diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2bf5254 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +buildroot/ +hafnium/ +linux/ +mbedtls/ +optee_benchmark/ +optee_examples/ +optee_rust/ +optee_test/ +out/ +out-br/ +qemu/ +toolchains/ +trusted-firmware-a/ +u-boot/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4849c47 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# TZmediator \ No newline at end of file diff --git a/build/.github/issue_template.md b/build/.github/issue_template.md new file mode 100644 index 0000000..0a6ae26 --- /dev/null +++ b/build/.github/issue_template.md @@ -0,0 +1,21 @@ + diff --git a/build/.github/pull_request_template.md b/build/.github/pull_request_template.md new file mode 100644 index 0000000..03cb067 --- /dev/null +++ b/build/.github/pull_request_template.md @@ -0,0 +1,19 @@ + diff --git a/build/.github/workflows/stale_issue.yml b/build/.github/workflows/stale_issue.yml new file mode 100644 index 0000000..34fd855 --- /dev/null +++ b/build/.github/workflows/stale_issue.yml @@ -0,0 +1,24 @@ +name: "Close stale issues" +on: + schedule: + - cron: "15 00 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v4.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue has been marked as a stale issue because it has been open (more than) 30 days with no activity. Remove the stale label or add a comment saying that you would like to have the label removed otherwise this issue will automatically be closed in 5 days. Note, that you can always re-open a closed issue at any time.' + stale-pr-message: 'This pull request has been marked as stale because it has been open (more than) 30 days with no activity. Remove the stale label or add a comment saying that you would like to have the label removed otherwise this pull request will automatically be closed in 5 days. Note, that you can always re-open a closed pull request at any time.' + exempt-issue-labels: bug,enhancement + exempt-pr-labels: bug,enhancement + days-before-stale: 30 + days-before-close: 5 + remove-issue-stale-when-updated: true + remove-pr-stale-when-updated: true diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..736ad05 --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,8 @@ +/Makefile +*.swp +/serial0.log +/serial1.log +.gdb_history +gdb +pi3.cfg +.hafnium_checkout diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..0e2fb9c --- /dev/null +++ b/build/README.md @@ -0,0 +1,13 @@ +# OP-TEE build.git + +This git contains makefiles etc to be able to build a full OP-TEE developer +setup for the OP-TEE project. + +All official OP-TEE documentation has moved to http://optee.readthedocs.io. The +pages that used to be here in this git can be found under [build] and [Device +specific information] at he new location for the OP-TEE documentation. + +// OP-TEE core maintainers + +[build]: https://optee.readthedocs.io/en/latest/building/index.html +[Device specific information]: https://optee.readthedocs.io/en/latest/building/devices/index.html diff --git a/build/am43xx.mk b/build/am43xx.mk new file mode 100644 index 0000000..db1b241 --- /dev/null +++ b/build/am43xx.mk @@ -0,0 +1,38 @@ +############################################################################### +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +############################################################################### +override COMPILE_NS_USER := 32 +override COMPILE_NS_KERNEL := 32 +override COMPILE_S_USER := 32 +override COMPILE_S_KERNEL := 32 + +# Need to set this before including common.mk +BR2_TARGET_GENERIC_GETTY_PORT ?= ttyS0 + +############################################################################### +# Includes +############################################################################### +include common.mk + +############################################################################### +# Paths to git projects and various binaries +############################################################################### +STAGING_AREA ?= $(ROOT)/out +U-BOOT_PATH ?= $(ROOT)/u-boot +UBOOT_SPL ?= $(U-BOOT_PATH)/u-boot-spl_HS_ISSW +UBOOT_IMG ?= $(U-BOOT_PATH)/u-boot_HS.img +UBOOT_ENV ?= $(BUILD_PATH)/ti/uEnv.txt +LINUX_IMAGE ?= $(LINUX_PATH)/arch/arm/boot/zImage +LINUX_DTBS ?= $(wildcard $(LINUX_PATH)/arch/arm/boot/dts/am43*.dtb) +FIT_SOURCE ?= $(BUILD_PATH)/ti/fitImage-am43xx.its +FIT_MAKEFILE ?= $(BUILD_PATH)/ti/Makefile +OPTEE_OS_PLATFORM ?= ti-am43xx +U-BOOT_CONFIG ?= am43xx_hs_evm_defconfig +CONFIG_TYPE ?= ti_sdk_am4x_debug + +############################################################################### +# Include common to TI builds +############################################################################### +include ti/ti-common.mk diff --git a/build/am57xx.mk b/build/am57xx.mk new file mode 100644 index 0000000..6971637 --- /dev/null +++ b/build/am57xx.mk @@ -0,0 +1,39 @@ +############################################################################### +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +############################################################################### +override COMPILE_NS_USER := 32 +override COMPILE_NS_KERNEL := 32 +override COMPILE_S_USER := 32 +override COMPILE_S_KERNEL := 32 + +# Need to set this before including common.mk +BR2_TARGET_GENERIC_GETTY_PORT ?= ttyS2 + +############################################################################### +# Includes +############################################################################### +include common.mk + +############################################################################### +# Paths to git projects and various binaries +############################################################################### +STAGING_AREA ?= $(ROOT)/out +U-BOOT_PATH ?= $(ROOT)/u-boot +UBOOT_SPL ?= $(U-BOOT_PATH)/u-boot-spl_HS_MLO +UBOOT_IMG ?= $(U-BOOT_PATH)/u-boot_HS.img +UBOOT_ENV ?= $(BUILD_PATH)/ti/uEnv.txt +LINUX_IMAGE ?= $(LINUX_PATH)/arch/arm/boot/zImage +LINUX_DTBS ?= $(wildcard $(LINUX_PATH)/arch/arm/boot/dts/am57*.dtb) +FIT_SOURCE ?= $(BUILD_PATH)/ti/fitImage-am57xx.its +FIT_MAKEFILE ?= $(BUILD_PATH)/ti/Makefile +OPTEE_OS_PLATFORM ?= ti-am57xx +U-BOOT_CONFIG ?= am57xx_hs_evm_defconfig +# using the same configs as for DRA7xx +CONFIG_TYPE ?= ti_sdk_dra7x_debug + +############################################################################### +# Include common to TI builds +############################################################################### +include ti/ti-common.mk diff --git a/build/br-ext/Config.in b/build/br-ext/Config.in new file mode 100644 index 0000000..279d6ff --- /dev/null +++ b/build/br-ext/Config.in @@ -0,0 +1,8 @@ +source "$BR2_EXTERNAL_OPTEE_PATH/package/optee_os_ext/Config.in" +source "$BR2_EXTERNAL_OPTEE_PATH/package/optee_client_ext/Config.in" +source "$BR2_EXTERNAL_OPTEE_PATH/package/optee_test_ext/Config.in" +source "$BR2_EXTERNAL_OPTEE_PATH/package/optee_examples_ext/Config.in" +source "$BR2_EXTERNAL_OPTEE_PATH/package/optee_rust_examples_ext/Config.in" +source "$BR2_EXTERNAL_OPTEE_PATH/package/optee_benchmark_ext/Config.in" +source "$BR2_EXTERNAL_OPTEE_PATH/package/ftpm_optee_ext/Config.in" +source "$BR2_EXTERNAL_OPTEE_PATH/package/linux_ftpm_mod_ext/Config.in" diff --git a/build/br-ext/board/fvp/overlay/etc/profile.d/ftpm_alias.sh b/build/br-ext/board/fvp/overlay/etc/profile.d/ftpm_alias.sh new file mode 100644 index 0000000..9201c4a --- /dev/null +++ b/build/br-ext/board/fvp/overlay/etc/profile.d/ftpm_alias.sh @@ -0,0 +1,6 @@ +alias ftpm_mod='insmod /lib/modules/extra/tpm_ftpm_tee.ko' +alias ftpm_getpcr='tpm2_pcrread' + +alias ftpm='ftpm_mod && ftpm_getpcr' + +alias ll='ls -al' diff --git a/build/br-ext/board/fvp/overlay/mnt/host/README b/build/br-ext/board/fvp/overlay/mnt/host/README new file mode 100644 index 0000000..1897799 --- /dev/null +++ b/build/br-ext/board/fvp/overlay/mnt/host/README @@ -0,0 +1,2 @@ +This directory is intended to be mounted onto a shared directory on the host. +See FVP_VIRTFS_AUTOMOUNT / FVP_VIRTFS_MOUNTPOINT in build/fvp.mk. diff --git a/build/br-ext/board/fvp/post-build.sh b/build/br-ext/board/fvp/post-build.sh new file mode 100755 index 0000000..b4c7d3a --- /dev/null +++ b/build/br-ext/board/fvp/post-build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2020, Roland Nagy +# Copyright (c) 2022, Arm Limited + +TARGET_DIR="$1" +VIRTFS_AUTOMOUNT="$2" +VIRTFS_MOUNTPOINT="$3" + +if [[ -z $TARGET_DIR ]]; then + echo "TARGET_DIR missing" + exit 1 +fi + +if [[ -z $VIRTFS_AUTOMOUNT ]]; then + echo "VIRTFS_AUTOMOUNT missing" + exit 1 +fi + +if [[ -z $VIRTFS_MOUNTPOINT ]]; then + echo "VIRTFS_MOUNTPOINT missing" + exit 1 +fi + +if [[ $VIRTFS_AUTOMOUNT == "y" ]]; then + grep FM "$TARGET_DIR"/etc/fstab > /dev/null || \ + echo "FM $VIRTFS_MOUNTPOINT 9p trans=virtio,version=9p2000.L 0 0" >> "$TARGET_DIR"/etc/fstab +fi diff --git a/build/br-ext/board/hikey960/overlay/etc/network/interfaces b/build/br-ext/board/hikey960/overlay/etc/network/interfaces new file mode 100644 index 0000000..8b6ba18 --- /dev/null +++ b/build/br-ext/board/hikey960/overlay/etc/network/interfaces @@ -0,0 +1,8 @@ +auto lo +iface lo inet loopback + +iface wlan0 inet dhcp + pre-up wpa_supplicant -i wlan0 -c /etc/wpa_supplicant.conf -B \ + && sleep 6 # wait before starting udhcpc + post-down killall -q wpa_supplicant + diff --git a/build/br-ext/board/hikey960/overlay/etc/wpa_supplicant.conf b/build/br-ext/board/hikey960/overlay/etc/wpa_supplicant.conf new file mode 100644 index 0000000..ee95e3b --- /dev/null +++ b/build/br-ext/board/hikey960/overlay/etc/wpa_supplicant.conf @@ -0,0 +1,6 @@ +ctrl_interface=/var/run/wpa_supplicant + +network={ + ssid="Network name" + psk="Password" +} diff --git a/build/br-ext/board/rockpi4/post-build.sh b/build/br-ext/board/rockpi4/post-build.sh new file mode 100755 index 0000000..6ca4388 --- /dev/null +++ b/build/br-ext/board/rockpi4/post-build.sh @@ -0,0 +1,5 @@ +#! /bin/bash +# SPDX-License-Identifier: BSD-2-Clause + +sed -i -e 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/' "$TARGET_DIR"/etc/ssh/sshd_config +sed -i -e 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' "$TARGET_DIR"/etc/ssh/sshd_config diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP135F-DK.cfg b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP135F-DK.cfg new file mode 100644 index 0000000..b458ff5 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP135F-DK.cfg @@ -0,0 +1,34 @@ +image sdcard.img { + hdimage { + gpt = "true" + } + + partition fsbl { + image = "tf-a-stm32mp135f-dk.stm32" + size = 256K + } + + partition fip { + image = "fip.bin" + partition-type-uuid = L + size = 4M + } + + partition u-boot-env { + partition-type-uuid = L + size = 4M + } + + partition bootfs { + image = "bootfs.ext2" + partition-type-uuid = L + size = 32M + bootable = "yes" + } + + partition rootfs { + image = "rootfs.ext2" + partition-type-uuid = L + size = 128M + } +} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157A-DHCOR-AVENGER96.cfg b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157A-DHCOR-AVENGER96.cfg new file mode 100644 index 0000000..8ba48d8 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157A-DHCOR-AVENGER96.cfg @@ -0,0 +1,34 @@ +image sdcard.img { + hdimage { + gpt = "true" + } + + partition fsbl { + image = "tf-a-stm32mp157a-dhcor-avenger96.stm32" + size = 256K + } + + partition fip { + image = "fip.bin" + partition-type-uuid = L + size = 4M + } + + partition u-boot-env { + partition-type-uuid = L + size = 4M + } + + partition bootfs { + image = "bootfs.ext2" + partition-type-uuid = L + size = 32M + bootable = "yes" + } + + partition rootfs { + image = "rootfs.ext2" + partition-type-uuid = L + size = 128M + } +} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157A-DK1.cfg b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157A-DK1.cfg new file mode 100644 index 0000000..b40fd46 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157A-DK1.cfg @@ -0,0 +1,34 @@ +image sdcard.img { + hdimage { + gpt = "true" + } + + partition fsbl { + image = "tf-a-stm32mp157a-dk1.stm32" + size = 256K + } + + partition fip { + image = "fip.bin" + partition-type-uuid = L + size = 4M + } + + partition u-boot-env { + partition-type-uuid = L + size = 4M + } + + partition bootfs { + image = "bootfs.ext2" + partition-type-uuid = L + size = 32M + bootable = "yes" + } + + partition rootfs { + image = "rootfs.ext2" + partition-type-uuid = L + size = 128M + } +} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-DHCOM-PDK2.cfg b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-DHCOM-PDK2.cfg new file mode 100644 index 0000000..137ff62 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-DHCOM-PDK2.cfg @@ -0,0 +1,34 @@ +image sdcard.img { + hdimage { + gpt = "true" + } + + partition fsbl { + image = "tf-a-stm32mp157c-dhcom-pdk2.stm32" + size = 256K + } + + partition fip { + image = "fip.bin" + partition-type-uuid = L + size = 4M + } + + partition u-boot-env { + partition-type-uuid = L + size = 4M + } + + partition bootfs { + image = "bootfs.ext2" + partition-type-uuid = L + size = 32M + bootable = "yes" + } + + partition rootfs { + image = "rootfs.ext2" + partition-type-uuid = L + size = 128M + } +} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-DK2.cfg b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-DK2.cfg new file mode 100644 index 0000000..c2aa647 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-DK2.cfg @@ -0,0 +1,34 @@ +image sdcard.img { + hdimage { + gpt = "true" + } + + partition fsbl { + image = "tf-a-stm32mp157c-dk2.stm32" + size = 256K + } + + partition fip { + image = "fip.bin" + partition-type-uuid = L + size = 4M + } + + partition u-boot-env { + partition-type-uuid = L + size = 4M + } + + partition bootfs { + image = "bootfs.ext2" + partition-type-uuid = L + size = 32M + bootable = "yes" + } + + partition rootfs { + image = "rootfs.ext2" + partition-type-uuid = L + size = 128M + } +} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-ED1.cfg b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-ED1.cfg new file mode 100644 index 0000000..6ac195d --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-ED1.cfg @@ -0,0 +1,34 @@ +image sdcard.img { + hdimage { + gpt = "true" + } + + partition fsbl { + image = "tf-a-stm32mp157c-ed1.stm32" + size = 256K + } + + partition fip { + image = "fip.bin" + partition-type-uuid = L + size = 4M + } + + partition u-boot-env { + partition-type-uuid = L + size = 4M + } + + partition bootfs { + image = "bootfs.ext2" + partition-type-uuid = L + size = 32M + bootable = "yes" + } + + partition rootfs { + image = "rootfs.ext2" + partition-type-uuid = L + size = 100M + } +} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-EV1.cfg b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-EV1.cfg new file mode 100644 index 0000000..946e461 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/genimage-STM32MP157C-EV1.cfg @@ -0,0 +1,34 @@ +image sdcard.img { + hdimage { + gpt = "true" + } + + partition fsbl { + image = "tf-a-stm32mp157c-ev1.stm32" + size = 256K + } + + partition fip { + image = "fip.bin" + partition-type-uuid = L + size = 4M + } + + partition u-boot-env { + partition-type-uuid = L + size = 4M + } + + partition bootfs { + image = "bootfs.ext2" + partition-type-uuid = L + size = 32M + bootable = "yes" + } + + partition rootfs { + image = "rootfs.ext2" + partition-type-uuid = L + size = 100M + } +} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP135F-DK/boot/extlinux/extlinux.conf b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP135F-DK/boot/extlinux/extlinux.conf new file mode 100644 index 0000000..990af59 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP135F-DK/boot/extlinux/extlinux.conf @@ -0,0 +1,6 @@ +TIMEOUT 20 +DEFAULT stm32mp13-buildroot +LABEL stm32mp13-buildroot + kernel /boot/uImage + devicetree /boot/stm32mp135f-dk.dtb + append root=/dev/mmcblk0p5 rootwait rw console=ttySTM0,115200 earlyprintk diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157A-DHCOR-AVENGER96/boot/extlinux/extlinux.conf b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157A-DHCOR-AVENGER96/boot/extlinux/extlinux.conf new file mode 100644 index 0000000..103d6cb --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157A-DHCOR-AVENGER96/boot/extlinux/extlinux.conf @@ -0,0 +1,6 @@ +TIMEOUT 20 +DEFAULT stm32mp15-buildroot +LABEL stm32mp15-buildroot + kernel /boot/uImage + devicetree /boot/stm32mp157a-dhcor-avenger96.dtb + append root=/dev/mmcblk0p5 rootwait rw console=ttySTM0,115200 diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157A-DK1/boot/extlinux/extlinux.conf b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157A-DK1/boot/extlinux/extlinux.conf new file mode 100644 index 0000000..41b5cce --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157A-DK1/boot/extlinux/extlinux.conf @@ -0,0 +1,6 @@ +TIMEOUT 20 +DEFAULT stm32mp15-buildroot +LABEL stm32mp15-buildroot + kernel /boot/uImage + devicetree /boot/stm32mp157a-dk1.dtb + append root=/dev/mmcblk0p5 rootwait rw console=ttySTM0,115200 diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-DHCOM-PDK2/boot/extlinux/extlinux.conf b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-DHCOM-PDK2/boot/extlinux/extlinux.conf new file mode 100644 index 0000000..0a862be --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-DHCOM-PDK2/boot/extlinux/extlinux.conf @@ -0,0 +1,6 @@ +TIMEOUT 20 +DEFAULT stm32mp15-buildroot +LABEL stm32mp15-buildroot + kernel /boot/uImage + devicetree /boot/stm32mp157c-dhcom-pdk2.dtb + append root=/dev/mmcblk1p5 rootwait rw console=ttySTM0,115200 diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-DK2/boot/extlinux/extlinux.conf b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-DK2/boot/extlinux/extlinux.conf new file mode 100644 index 0000000..2f8531f --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-DK2/boot/extlinux/extlinux.conf @@ -0,0 +1,6 @@ +TIMEOUT 20 +DEFAULT stm32mp15-buildroot +LABEL stm32mp15-buildroot + kernel /boot/uImage + devicetree /boot/stm32mp157c-dk2.dtb + append root=/dev/mmcblk0p5 rootwait rw console=ttySTM0,115200 diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-ED1/boot/extlinux/extlinux.conf b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-ED1/boot/extlinux/extlinux.conf new file mode 100644 index 0000000..f0ffe82 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-ED1/boot/extlinux/extlinux.conf @@ -0,0 +1,6 @@ +TIMEOUT 20 +DEFAULT stm32mp15-buildroot +LABEL stm32mp15-buildroot + kernel /boot/uImage + devicetree /boot/stm32mp157c-ed1.dtb + append root=/dev/mmcblk0p5 rootwait rw console=ttySTM0,115200 diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-EV1/boot/extlinux/extlinux.conf b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-EV1/boot/extlinux/extlinux.conf new file mode 100644 index 0000000..d916a85 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/overlay-STM32MP157C-EV1/boot/extlinux/extlinux.conf @@ -0,0 +1,6 @@ +TIMEOUT 20 +DEFAULT stm32mp15-buildroot +LABEL stm32mp15-buildroot + kernel /boot/uImage + devicetree /boot/stm32mp157c-ev1.dtb + append root=/dev/mmcblk0p5 rootwait rw console=ttySTM0,115200 diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/post-image.sh b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/post-image.sh new file mode 100755 index 0000000..499dcb6 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/post-image.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +die() { + cat <&2 +Error: $@ + +Usage: ${0} IMG_DIR GENIMAGE_CFG EXT_BIN_DIR [BOOTFS_OVERLAY] +EOF + exit 1 +} + +echo "Creating bootfs image" +echo "- image directory path: ${1}" # Generic argument set by Buildroot +echo "- genimage config file: ${2}" +echo "- external bins path: ${3}" +echo "- bootfs overlay path: ${4}" + +[ ! -z "${2}" -a -e ${2} ] || die "Error: missing argument genimage config file" +[ ! -z "${3}" -a -d ${3} ] || die "Error: missing argument external binaries dir" +[ -z "${4}" -o -d ${4} ] || die "Error: invalid bootfs overlay directory path" + +GENIMAGE_TMP="${BUILD_DIR}/genimage.tmp" + +# Create target bootfs filesystem +# - Copy uImage and DTBs from path provided in arg $3 into boot/ +# - Copy bootfs overlay filetree from path provided in optional arg $4 + +BOOTFS_DIR=${BASE_DIR}/target-bootfs +rm -f ${BINARIES_DIR}/bootfs.ext2 || exit 1 +rm -rf ${BOOTFS_DIR} && mkdir -p ${BOOTFS_DIR}/boot || exit 1 +cp --dereference ${3}/uImage ${BOOTFS_DIR}/boot || exit 1 +for f in ${3}/*.dtb; do + test -f $f && { cp --dereference $f ${BOOTFS_DIR}/boot || exit 1; } +done +[ -z "${4}" ] || { cp -ar ${4}/* ${BOOTFS_DIR} || exit 1; } + +mkfs.ext2 -L bootfs -d ${BOOTFS_DIR} ${BINARIES_DIR}/bootfs.ext2 32M || exit 1 + +# Generate image from generated partition images and genimage config file + +rm -rf "${GENIMAGE_TMP}" + +genimage --rootpath "${ROOTPATH_TMP}" \ + --tmppath "${GENIMAGE_TMP}" \ + --inputpath "${BINARIES_DIR}" \ + --outputpath "${BINARIES_DIR}" \ + --config ${2} diff --git a/build/br-ext/board/stmicroelectronics/stm32mp1-tz/readme.txt b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/readme.txt new file mode 100644 index 0000000..62cbb48 --- /dev/null +++ b/build/br-ext/board/stmicroelectronics/stm32mp1-tz/readme.txt @@ -0,0 +1,46 @@ +STM32MP1 Based platforms + +Intro +===== + +This directory provides support for few STM32MP1 based development boards. + + https://www.st.com/en/evaluation-tools/stm32mp157a-dk1.html + https://www.st.com/en/evaluation-tools/stm32mp157c-dk2.html + https://www.st.com/en/evaluation-tools/stm32mp157c-ev1.html + +How to build +============ + +From OP-TEE build repository: + + $ make PLATFORM=stm32mp1-157A_DK1 all +or + $ make PLATFORM=stm32mp1-157C_DK2 all +or + $ make PLATFORM=stm32mp1-157C_EV1 all + +How to write the microSD card +============================= + +Once the build process is finished you will have an image called +"sdcard.img" in directory ../out-br/images/ (relative to build +repository root path). + +Copy the bootable "sdcard.img" onto an microSD card with "dd": + + $ sudo dd if=../out-br/images/sdcard.img of=/dev/sdX \ + conv=fdatasync status=progress + +Boot the board +============== + + (1) Insert the microSD card in connector CN15 + + (2) Plug a micro-USB cable in connector CN11 and run your serial + communication program on /dev/ttyACM0. + + (3) Plug a USB-C cable in CN6 to power-up the board. + + (4) The system will start, with the console on UART. Both secure + and non-secure console consoles are on the same UART bus. diff --git a/build/br-ext/configs/ftpm_optee b/build/br-ext/configs/ftpm_optee new file mode 100644 index 0000000..b4a284e --- /dev/null +++ b/build/br-ext/configs/ftpm_optee @@ -0,0 +1 @@ +BR2_PACKAGE_FTPM_OPTEE_EXT=y diff --git a/build/br-ext/configs/gdbserver.conf b/build/br-ext/configs/gdbserver.conf new file mode 100644 index 0000000..726736a --- /dev/null +++ b/build/br-ext/configs/gdbserver.conf @@ -0,0 +1,6 @@ +BR2_ENABLE_DEBUG=y +BR2_PACKAGE_GDB=y +BR2_PACKAGE_HOST_GDB=y +BR2_TOOLCHAIN_BUILDROOT_CXX=y +BR2_TOOLCHAIN_BUILDROOT_GLIBC=y +BR2_PACKAGE_GLIBC=y diff --git a/build/br-ext/configs/linux_ftpm b/build/br-ext/configs/linux_ftpm new file mode 100644 index 0000000..82e47bd --- /dev/null +++ b/build/br-ext/configs/linux_ftpm @@ -0,0 +1 @@ +BR2_PACKAGE_LINUX_FTPM_MOD_EXT=y diff --git a/build/br-ext/configs/optee_aarch32 b/build/br-ext/configs/optee_aarch32 new file mode 100644 index 0000000..a7f8886 --- /dev/null +++ b/build/br-ext/configs/optee_aarch32 @@ -0,0 +1,5 @@ +# Architecture +BR2_arm=y +BR2_cortex_a15=y +BR2_ARM_FPU_VFPV3D16=y +BR2_ARM_INSTRUCTIONS_THUMB2=y diff --git a/build/br-ext/configs/optee_aarch64 b/build/br-ext/configs/optee_aarch64 new file mode 100644 index 0000000..d740c58 --- /dev/null +++ b/build/br-ext/configs/optee_aarch64 @@ -0,0 +1,2 @@ +# Architecture +BR2_aarch64=y diff --git a/build/br-ext/configs/optee_examples b/build/br-ext/configs/optee_examples new file mode 100644 index 0000000..94d84c1 --- /dev/null +++ b/build/br-ext/configs/optee_examples @@ -0,0 +1 @@ +BR2_PACKAGE_OPTEE_EXAMPLES_EXT=y diff --git a/build/br-ext/configs/optee_generic b/build/br-ext/configs/optee_generic new file mode 100644 index 0000000..8d231f5 --- /dev/null +++ b/build/br-ext/configs/optee_generic @@ -0,0 +1,7 @@ +BR2_CCACHE=y +BR2_TARGET_GENERIC_ISSUE="Welcome to Buildroot, type root or test to login" + +BR2_TARGET_ROOTFS_CPIO=y +BR2_TARGET_ROOTFS_CPIO_GZIP=y +BR2_PACKAGE_HOST_E2FSPROGS=y +BR2_PACKAGE_OPTEE_CLIENT_EXT=y diff --git a/build/br-ext/configs/optee_riscv64 b/build/br-ext/configs/optee_riscv64 new file mode 100755 index 0000000..9a55a03 --- /dev/null +++ b/build/br-ext/configs/optee_riscv64 @@ -0,0 +1,2 @@ +BR2_riscv=y +BR2_RISCV_64=y diff --git a/build/br-ext/configs/optee_test b/build/br-ext/configs/optee_test new file mode 100644 index 0000000..1f25dfa --- /dev/null +++ b/build/br-ext/configs/optee_test @@ -0,0 +1 @@ +BR2_PACKAGE_OPTEE_TEST_EXT=y diff --git a/build/br-ext/configs/sdk-aarch32 b/build/br-ext/configs/sdk-aarch32 new file mode 100644 index 0000000..6811f03 --- /dev/null +++ b/build/br-ext/configs/sdk-aarch32 @@ -0,0 +1,5 @@ +BR2_arm=y +BR2_ARM_ENABLE_VFP=y +BR2_ARM_EABIHF=y +BR2_ARM_FPU_VFPV2=y +BR2_GCC_TARGET_FLOAT_ABI="hard" diff --git a/build/br-ext/configs/sdk-aarch64 b/build/br-ext/configs/sdk-aarch64 new file mode 100644 index 0000000..1c77147 --- /dev/null +++ b/build/br-ext/configs/sdk-aarch64 @@ -0,0 +1 @@ +BR2_aarch64=y diff --git a/build/br-ext/configs/sdk-common b/build/br-ext/configs/sdk-common new file mode 100644 index 0000000..3af1a37 --- /dev/null +++ b/build/br-ext/configs/sdk-common @@ -0,0 +1,10 @@ +BR2_CCACHE=y +BR2_CCACHE_USE_BASEDIR=y +BR2_GCC_VERSION_10_X=y +BR2_INIT_NONE=y +BR2_KERNEL_HEADERS_5_10=y +BR2_PACKAGE_BUSYBOX=n +BR2_SYSTEM_BIN_SH_NONE=y +BR2_TARGET_ROOTFS_TAR=n +BR2_TOOLCHAIN_BUILDROOT_CXX=y +BR2_TOOLCHAIN_BUILDROOT_GLIBC=y diff --git a/build/br-ext/configs/toolchain-aarch32 b/build/br-ext/configs/toolchain-aarch32 new file mode 100644 index 0000000..378a50f --- /dev/null +++ b/build/br-ext/configs/toolchain-aarch32 @@ -0,0 +1,9 @@ +BR2_TOOLCHAIN_EXTERNAL=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y +BR2_TOOLCHAIN_EXTERNAL_PATH="%TOP_DIR%/toolchains/aarch32" +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX="arm-linux-gnueabihf" +BR2_TOOLCHAIN_EXTERNAL_GCC_11=y +BR2_TOOLCHAIN_EXTERNAL_HEADERS_4_20=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC=y +BR2_TOOLCHAIN_EXTERNAL_CXX=y +BR2_TOOLCHAIN_EXTERNAL_INET_RPC=n diff --git a/build/br-ext/configs/toolchain-aarch32-sdk b/build/br-ext/configs/toolchain-aarch32-sdk new file mode 100644 index 0000000..52ae1d2 --- /dev/null +++ b/build/br-ext/configs/toolchain-aarch32-sdk @@ -0,0 +1,9 @@ +BR2_TOOLCHAIN_EXTERNAL=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y +BR2_TOOLCHAIN_EXTERNAL_PATH="%TOP_DIR%/toolchains/aarch32" +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX="arm-linux" +BR2_TOOLCHAIN_EXTERNAL_GCC_10=y +BR2_TOOLCHAIN_EXTERNAL_HEADERS_5_10=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC=y +BR2_TOOLCHAIN_EXTERNAL_CXX=y +BR2_TOOLCHAIN_EXTERNAL_INET_RPC=n diff --git a/build/br-ext/configs/toolchain-aarch64 b/build/br-ext/configs/toolchain-aarch64 new file mode 100644 index 0000000..d9879b6 --- /dev/null +++ b/build/br-ext/configs/toolchain-aarch64 @@ -0,0 +1,9 @@ +BR2_TOOLCHAIN_EXTERNAL=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y +BR2_TOOLCHAIN_EXTERNAL_PATH="%TOP_DIR%/toolchains/aarch64" +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX="aarch64-linux-gnu" +BR2_TOOLCHAIN_EXTERNAL_GCC_11=y +BR2_TOOLCHAIN_EXTERNAL_HEADERS_4_20=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC=y +BR2_TOOLCHAIN_EXTERNAL_CXX=y +BR2_TOOLCHAIN_EXTERNAL_INET_RPC=n diff --git a/build/br-ext/configs/toolchain-aarch64-sdk b/build/br-ext/configs/toolchain-aarch64-sdk new file mode 100644 index 0000000..c820053 --- /dev/null +++ b/build/br-ext/configs/toolchain-aarch64-sdk @@ -0,0 +1,9 @@ +BR2_TOOLCHAIN_EXTERNAL=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y +BR2_TOOLCHAIN_EXTERNAL_PATH="%TOP_DIR%/toolchains/aarch64" +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX="aarch64-linux" +BR2_TOOLCHAIN_EXTERNAL_GCC_10=y +BR2_TOOLCHAIN_EXTERNAL_HEADERS_5_10=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC=y +BR2_TOOLCHAIN_EXTERNAL_CXX=y +BR2_TOOLCHAIN_EXTERNAL_INET_RPC=n diff --git a/build/br-ext/configs/toolchain-br b/build/br-ext/configs/toolchain-br new file mode 100644 index 0000000..d03e90e --- /dev/null +++ b/build/br-ext/configs/toolchain-br @@ -0,0 +1,3 @@ +BR2_TOOLCHAIN_BUILDROOT=y +BR2_GCC_VERSION_10_X=y +BR2_KERNEL_HEADERS_4_19=y diff --git a/build/br-ext/configs/toolchain-riscv64 b/build/br-ext/configs/toolchain-riscv64 new file mode 100755 index 0000000..d0674c5 --- /dev/null +++ b/build/br-ext/configs/toolchain-riscv64 @@ -0,0 +1,9 @@ +BR2_TOOLCHAIN_EXTERNAL=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y +BR2_TOOLCHAIN_EXTERNAL_PATH="%TOP_DIR%/toolchains/riscv64" +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX="riscv64-unknown-linux-gnu" +BR2_TOOLCHAIN_EXTERNAL_GCC_12=y +BR2_TOOLCHAIN_EXTERNAL_HEADERS_5_10=y +BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC=y +BR2_TOOLCHAIN_EXTERNAL_CXX=y +BR2_TOOLCHAIN_EXTERNAL_INET_RPC=n diff --git a/build/br-ext/configs/tss b/build/br-ext/configs/tss new file mode 100644 index 0000000..c10abeb --- /dev/null +++ b/build/br-ext/configs/tss @@ -0,0 +1 @@ +BR2_PACKAGE_TPM2_TOOLS=y diff --git a/build/br-ext/configs/xen.conf b/build/br-ext/configs/xen.conf new file mode 100644 index 0000000..4111fa2 --- /dev/null +++ b/build/br-ext/configs/xen.conf @@ -0,0 +1,6 @@ +BR2_PACKAGE_XEN=y +BR2_PACKAGE_XEN_TOOLS=y +BR2_PACKAGE_XEN_HYPERVISOR=y +BR2_PACKAGE_BUSYBOX_SHOW_OTHERS=y +BR2_PACKAGE_BASH=y +BR2_SYSTEM_BIN_SH_BASH=y diff --git a/build/br-ext/external.desc b/build/br-ext/external.desc new file mode 100644 index 0000000..02582f5 --- /dev/null +++ b/build/br-ext/external.desc @@ -0,0 +1,2 @@ +name: OPTEE +desc: OP-TEE br2-external tree diff --git a/build/br-ext/external.mk b/build/br-ext/external.mk new file mode 100644 index 0000000..fcf49b2 --- /dev/null +++ b/build/br-ext/external.mk @@ -0,0 +1 @@ +include $(sort $(wildcard $(BR2_EXTERNAL_OPTEE_PATH)/package/*/*.mk)) diff --git a/build/br-ext/package/ftpm_optee_ext/CMakeLists.txt b/build/br-ext/package/ftpm_optee_ext/CMakeLists.txt new file mode 100644 index 0000000..907b41d --- /dev/null +++ b/build/br-ext/package/ftpm_optee_ext/CMakeLists.txt @@ -0,0 +1,3 @@ +# This is a dummy Makefile. When this package is invoked, the fTPM service +# has been built already. +install(FILES /dev/null DESTINATION /dev/null) diff --git a/build/br-ext/package/ftpm_optee_ext/Config.in b/build/br-ext/package/ftpm_optee_ext/Config.in new file mode 100644 index 0000000..092b586 --- /dev/null +++ b/build/br-ext/package/ftpm_optee_ext/Config.in @@ -0,0 +1,28 @@ +config BR2_PACKAGE_FTPM_OPTEE_EXT + bool "Enable fTPM based on OPTEE" + select BR2_PACKAGE_OPTEE_OS_EXT + help + fTPM, http://github.com/microsoft/ms-tpm-20-ref. + NOTE: This package currently only takes care of installing files + into the root FS, that have been compiled already. + The build of the OPTEE fTPM service is assumed to have been done + previously. + +config BR2_PACKAGE_FTPM_OPTEE_EXT_SITE + string "FTPM_OPTEE installation package path" + default "" + help + The path to this installation package. + +config BR2_PACKAGE_FTPM_OPTEE_PACKAGE_SITE + string "Path to the TPM 2.0 Reference Implementation" + default "" + help + The path to this installation package. + +config BR2_PACKAGE_FTPM_OPTEE_EXT_TA_SRC + string "Path of the fTPM sources within the TPM Ref. Implementation" + default "Samples/ARM32-FirmwareTPM/optee_ta" + help + The path, relative to where the TPM 2.0 Reference Implementation + is installed, where the sources for the fTPM can be found. diff --git a/build/br-ext/package/ftpm_optee_ext/ftpm_optee_ext.mk b/build/br-ext/package/ftpm_optee_ext/ftpm_optee_ext.mk new file mode 100644 index 0000000..9473cca --- /dev/null +++ b/build/br-ext/package/ftpm_optee_ext/ftpm_optee_ext.mk @@ -0,0 +1,18 @@ +FTPM_OPTEE_EXT_VERSION = 1.0 +FTPM_OPTEE_EXT_SOURCE = local +FTPM_OPTEE_EXT_SITE = $(BR2_PACKAGE_FTPM_OPTEE_EXT_SITE) +FTPM_OPTEE_EXT_SRC = $(BR2_PACKAGE_FTPM_OPTEE_PACKAGE_SITE) +FTPM_OPTEE_EXT_SITE_METHOD = local +FTPM_OPTEE_EXT_TA_SRC = $(BR2_PACKAGE_FTPM_OPTEE_EXT_TA_SRC) + +define FTPM_OPTEE_EXT_INSTALL_TA + echo "Installing fTPM based on OPTEE" && \ + mkdir -p $(TARGET_DIR)/lib/optee_armtz && \ + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(TARGET_DIR)/lib/optee_armtz \ + $(FTPM_OPTEE_EXT_SRC)/$(FTPM_OPTEE_EXT_TA_SRC)/out/fTPM/bc50d971-d4c9-42c4-82cb-343fb7f37896.ta +endef + +FTPM_OPTEE_EXT_POST_INSTALL_TARGET_HOOKS += FTPM_OPTEE_EXT_INSTALL_TA + +$(eval $(cmake-package)) diff --git a/build/br-ext/package/linux_ftpm_mod_ext/CMakeLists.txt b/build/br-ext/package/linux_ftpm_mod_ext/CMakeLists.txt new file mode 100644 index 0000000..d2a0632 --- /dev/null +++ b/build/br-ext/package/linux_ftpm_mod_ext/CMakeLists.txt @@ -0,0 +1,3 @@ +# This is a dummy Makefile. When this package is invoked, the TPM Kernel +# module has been built already. +install(FILES /dev/null DESTINATION /dev/null) diff --git a/build/br-ext/package/linux_ftpm_mod_ext/Config.in b/build/br-ext/package/linux_ftpm_mod_ext/Config.in new file mode 100644 index 0000000..02d40d5 --- /dev/null +++ b/build/br-ext/package/linux_ftpm_mod_ext/Config.in @@ -0,0 +1,20 @@ +config BR2_PACKAGE_LINUX_FTPM_MOD_EXT + bool "Enable TPM Kernel module" + help + Enable TPM Kernel module. + NOTE: This package currently only takes care of installing files + into the root FS, that have been compiled already. + The build of the TPM Kernel module is assumed to have been done + previously. + +config BR2_PACKAGE_LINUX_FTPM_MOD_EXT_SITE + string "TPM Kernel Module installation package path" + default "" + help + The path to this installation package. + +config BR2_PACKAGE_LINUX_FTPM_MOD_EXT_PATH + string "Path to the TPM Kernel module" + default "" + help + The path of the TPM Kernel sources. diff --git a/build/br-ext/package/linux_ftpm_mod_ext/linux_ftpm_mod_ext.mk b/build/br-ext/package/linux_ftpm_mod_ext/linux_ftpm_mod_ext.mk new file mode 100644 index 0000000..6df8a73 --- /dev/null +++ b/build/br-ext/package/linux_ftpm_mod_ext/linux_ftpm_mod_ext.mk @@ -0,0 +1,18 @@ +LINUX_FTPM_MOD_EXT_VERSION = 1.0 +LINUX_FTPM_MOD_EXT_SOURCE = local +LINUX_FTPM_MOD_EXT_SITE = $(BR2_PACKAGE_LINUX_FTPM_MOD_EXT_SITE) +LINUX_FTPM_MOD_EXT_PATH = $(BR2_PACKAGE_LINUX_FTPM_MOD_EXT_PATH) +LINUX_FTPM_MOD_EXT_SITE_METHOD = local +LINUX_FTPM_MOD_EXT_INSTALL_DIR=$(TARGET_DIR)/lib/modules/extra + +define LINUX_FTPM_MOD_EXT_INSTALL + echo "Installing TPM kernel module" && \ + mkdir -p $(LINUX_FTPM_MOD_EXT_INSTALL_DIR) + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(LINUX_FTPM_MOD_EXT_INSTALL_DIR) \ + $(LINUX_FTPM_MOD_EXT_PATH)/drivers/char/tpm/tpm_ftpm_tee.ko +endef + +LINUX_FTPM_MOD_EXT_POST_INSTALL_TARGET_HOOKS += LINUX_FTPM_MOD_EXT_INSTALL + +$(eval $(cmake-package)) diff --git a/build/br-ext/package/optee_benchmark_ext/Config.in b/build/br-ext/package/optee_benchmark_ext/Config.in new file mode 100644 index 0000000..e6638ab --- /dev/null +++ b/build/br-ext/package/optee_benchmark_ext/Config.in @@ -0,0 +1,11 @@ +config BR2_PACKAGE_OPTEE_BENCHMARK_EXT + bool "optee_benchmark" + select BR2_PACKAGE_LIBYAML + help + http://github.org/OP-TEE/optee_benchmark + +config BR2_PACKAGE_OPTEE_BENCHMARK_EXT_SITE + string "OP-TEE benchmark path" + default "" + help + The path to the source directory of OP-TEE benchmark diff --git a/build/br-ext/package/optee_benchmark_ext/optee_benchmark_ext.mk b/build/br-ext/package/optee_benchmark_ext/optee_benchmark_ext.mk new file mode 100644 index 0000000..f55500e --- /dev/null +++ b/build/br-ext/package/optee_benchmark_ext/optee_benchmark_ext.mk @@ -0,0 +1,8 @@ +OPTEE_BENCHMARK_EXT_VERSION = 1.0 +OPTEE_BENCHMARK_EXT_SOURCE = local +OPTEE_BENCHMARK_EXT_SITE = $(BR2_PACKAGE_OPTEE_BENCHMARK_EXT_SITE) +OPTEE_BENCHMARK_EXT_SITE_METHOD = local +OPTEE_BENCHMARK_EXT_INSTALL_STAGING = YES +OPTEE_BENCHMARK_EXT_DEPENDENCIES = optee_client_ext libyaml + +$(eval $(cmake-package)) diff --git a/build/br-ext/package/optee_client_ext/Config.in b/build/br-ext/package/optee_client_ext/Config.in new file mode 100644 index 0000000..f9fc964 --- /dev/null +++ b/build/br-ext/package/optee_client_ext/Config.in @@ -0,0 +1,19 @@ +config BR2_PACKAGE_OPTEE_CLIENT_EXT + bool "optee_client" + select BR2_PACKAGE_UTIL_LINUX + select BR2_PACKAGE_UTIL_LINUX_LIBS + select BR2_PACKAGE_UTIL_LINUX_LIBUUID + help + http://github.org/OP-TEE/optee_client + +config BR2_PACKAGE_OPTEE_CLIENT_EXT_SITE + string "OP-TEE client path" + default "" + help + The path to the source directory of OP-TEE client + +config BR2_PACKAGE_OPTEE_CLIENT_EXT_RPMB_EMU + bool "Enable RPMB emulation" + default y + help + Enable RPMB device emulation in tee-supplicant. diff --git a/build/br-ext/package/optee_client_ext/S30optee b/build/br-ext/package/optee_client_ext/S30optee new file mode 100755 index 0000000..869bb5b --- /dev/null +++ b/build/br-ext/package/optee_client_ext/S30optee @@ -0,0 +1,71 @@ +#!/bin/sh + +DAEMON="tee-supplicant" +DAEMON_PATH="/usr/sbin" +DAEMON_ARGS="-d /dev/teepriv0" +PIDFILE="/var/run/$DAEMON.pid" + +start() { + # tee-supplicant and the client applications need not run as + # root provided that the TEE devices and the data store have + # proper permissions + printf 'Set permissions on %s: ' "/dev/tee*" + chown root:tee /dev/teepriv0 && chmod 0660 /dev/teepriv0 && \ + chown root:teeclnt /dev/tee0 && chmod 0660 /dev/tee0 + status=$? + if [ "$status" -eq 0 ]; then + echo "OK" + else + echo "FAIL" + return "$status" + fi + printf 'Create/set permissions on %s: ' "/data/tee" + mkdir -p /data/tee && chown -R tee:tee /data/tee && chmod 0770 /data/tee + status=$? + if [ "$status" -eq 0 ]; then + echo "OK" + else + echo "FAIL" + return "$status" + fi + printf 'Starting %s: ' "$DAEMON" + start-stop-daemon -S -q -p "$PIDFILE" -c tee -x "$DAEMON_PATH/$DAEMON" \ + -- $DAEMON_ARGS + status=$? + if [ "$status" -eq 0 ]; then + echo "OK" + else + echo "FAIL" + fi + return "$status" +} + +stop() { + printf 'Stopping %s: ' "$DAEMON" + start-stop-daemon -K -q -p "$PIDFILE" + status=$? + if [ "$status" -eq 0 ]; then + echo "OK" + else + echo "FAIL" + fi + return "$status" +} + +restart() { + stop + sleep 1 + start +} + +case "$1" in + start|stop|restart) + "$1";; + reload) + # Restart, since there is no true "reload" feature (does not + # reconfigure/restart on SIGHUP, just closes all open files). + restart;; + *) + echo "Usage: $0 {start|stop|restart|reload}" + exit 1 +esac diff --git a/build/br-ext/package/optee_client_ext/optee_client_ext.mk b/build/br-ext/package/optee_client_ext/optee_client_ext.mk new file mode 100644 index 0000000..2613272 --- /dev/null +++ b/build/br-ext/package/optee_client_ext/optee_client_ext.mk @@ -0,0 +1,40 @@ +OPTEE_CLIENT_EXT_VERSION = 1.0 +OPTEE_CLIENT_EXT_SOURCE = local +OPTEE_CLIENT_EXT_SITE = $(BR2_PACKAGE_OPTEE_CLIENT_EXT_SITE) +OPTEE_CLIENT_EXT_SITE_METHOD = local +OPTEE_CLIENT_EXT_DEPENDENCIES = host-pkgconf util-linux-libs +OPTEE_CLIENT_EXT_INSTALL_STAGING = YES + +ifeq ($(BR2_PACKAGE_OPTEE_BENCHMARK_EXT),y) +OPTEE_CLIENT_EXT_CONF_OPTS = -DCFG_TEE_BENCHMARK=ON +else +OPTEE_CLIENT_EXT_CONF_OPTS = -DCFG_TEE_BENCHMARK=OFF +endif + +ifeq ($(BR2_PACKAGE_OPTEE_CLIENT_EXT_RPMB_EMU),y) +OPTEE_CLIENT_EXT_CONF_OPTS += -DRPMB_EMU=ON +else +OPTEE_CLIENT_EXT_CONF_OPTS += -DRPMB_EMU=OFF +endif + +OPTEE_CLIENT_EXT_CONF_OPTS += -DCFG_TA_TEST_PATH=y + +define OPTEE_CLIENT_EXT_INSTALL_SUPPLICANT_SCRIPT + $(INSTALL) -m 0755 -D $(OPTEE_CLIENT_EXT_PKGDIR)/S30optee \ + $(TARGET_DIR)/etc/init.d/S30optee +endef + +define OPTEE_CLIENT_EXT_INSTALL_INIT_SYSV + $(OPTEE_CLIENT_EXT_INSTALL_SUPPLICANT_SCRIPT) +endef + +# User tee is used to run tee-supplicant because access to /dev/teepriv0 is +# restricted to group tee. +# Any user in group teeclnt (such as test) may run client applications. +define OPTEE_CLIENT_EXT_USERS + tee -1 tee -1 * - /bin/sh - TEE user + - -1 teeclnt -1 - - - - TEE users group + test -1 test -1 - - /bin/sh teeclnt Test user, may run TEE client applications +endef + +$(eval $(cmake-package)) diff --git a/build/br-ext/package/optee_examples_ext/Config.in b/build/br-ext/package/optee_examples_ext/Config.in new file mode 100644 index 0000000..e4e5a0c --- /dev/null +++ b/build/br-ext/package/optee_examples_ext/Config.in @@ -0,0 +1,26 @@ +config BR2_PACKAGE_OPTEE_EXAMPLES_EXT + bool "optee_examples" + help + http://github.org/OP-TEE/optee_examples + +config BR2_PACKAGE_OPTEE_EXAMPLES_EXT_SITE + string "OP-TEE examples path" + default "" + help + The path to the source directory of OP-TEE examples + +if BR2_PACKAGE_OPTEE_EXAMPLES_EXT + +config BR2_PACKAGE_OPTEE_EXAMPLES_EXT_SDK + string "OPTEE SDK path" + default "" + help + some help + +config BR2_PACKAGE_OPTEE_EXAMPLES_EXT_CROSS_COMPILE + string "OPTEE cross compiler" + default "" + help + some help + +endif diff --git a/build/br-ext/package/optee_examples_ext/optee_examples_ext.mk b/build/br-ext/package/optee_examples_ext/optee_examples_ext.mk new file mode 100644 index 0000000..113a9b3 --- /dev/null +++ b/build/br-ext/package/optee_examples_ext/optee_examples_ext.mk @@ -0,0 +1,30 @@ +OPTEE_EXAMPLES_EXT_VERSION = 1.0 +OPTEE_EXAMPLES_EXT_SOURCE = local +OPTEE_EXAMPLES_EXT_SITE = $(BR2_PACKAGE_OPTEE_EXAMPLES_EXT_SITE) +OPTEE_EXAMPLES_EXT_SITE_METHOD = local +OPTEE_EXAMPLES_EXT_INSTALL_STAGING = YES +OPTEE_EXAMPLES_EXT_DEPENDENCIES = optee_client_ext host-python-cryptography +OPTEE_EXAMPLES_EXT_SDK = $(BR2_PACKAGE_OPTEE_EXAMPLES_EXT_SDK) +OPTEE_EXAMPLES_EXT_CONF_OPTS = -DOPTEE_EXAMPLES_SDK=$(OPTEE_EXAMPLES_EXT_SDK) + +define OPTEE_EXAMPLES_EXT_BUILD_TAS + @$(foreach f,$(wildcard $(@D)/*/ta/Makefile), \ + echo Building $f && \ + $(MAKE) CROSS_COMPILE="$(shell echo $(BR2_PACKAGE_OPTEE_EXAMPLES_EXT_CROSS_COMPILE))" \ + O=out TA_DEV_KIT_DIR=$(OPTEE_EXAMPLES_EXT_SDK) \ + PYTHON3=$(HOST_DIR)/bin/python3 \ + $(TARGET_CONFIGURE_OPTS) -C $(dir $f) all &&) true +endef + +define OPTEE_EXAMPLES_EXT_INSTALL_TAS + @$(foreach f,$(wildcard $(@D)/*/ta/out/*.ta), \ + mkdir -p $(TARGET_DIR)/lib/optee_armtz && \ + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(TARGET_DIR)/lib/optee_armtz $f \ + &&) true +endef + +OPTEE_EXAMPLES_EXT_POST_BUILD_HOOKS += OPTEE_EXAMPLES_EXT_BUILD_TAS +OPTEE_EXAMPLES_EXT_POST_INSTALL_TARGET_HOOKS += OPTEE_EXAMPLES_EXT_INSTALL_TAS + +$(eval $(cmake-package)) diff --git a/build/br-ext/package/optee_os_ext/CMakeLists.txt b/build/br-ext/package/optee_os_ext/CMakeLists.txt new file mode 100644 index 0000000..c502988 --- /dev/null +++ b/build/br-ext/package/optee_os_ext/CMakeLists.txt @@ -0,0 +1,3 @@ +# This is a dummy Makefile. When this package is invoked, OP-TEE has been built +# already, and the only installation step occurs in optee_os.mk. +install(FILES /dev/null DESTINATION /dev/null) diff --git a/build/br-ext/package/optee_os_ext/Config.in b/build/br-ext/package/optee_os_ext/Config.in new file mode 100644 index 0000000..2326850 --- /dev/null +++ b/build/br-ext/package/optee_os_ext/Config.in @@ -0,0 +1,25 @@ +config BR2_PACKAGE_OPTEE_OS_EXT + bool "optee_os" + help + OP-TEE OS, http://github.org/OP-TEE/optee_os. + NOTE: this package currently only takes care of installing files into + the root FS, that have been compiled already. For example, shared + libraries used by TAs when CFG_ULIBS_SHARED=y. + The build of optee_os itself is assumed to have been done previously. + +config BR2_PACKAGE_OPTEE_OS_EXT_SITE + string "OP-TEE OS installation package path" + default "" + help + The path to this installation package. + +if BR2_PACKAGE_OPTEE_OS_EXT + +config BR2_PACKAGE_OPTEE_OS_EXT_SDK + string "OPTEE SDK path" + default "" + help + The path to export-ta_arm32 or export-ta_arm64 in the optee_os output + directory. + +endif diff --git a/build/br-ext/package/optee_os_ext/optee_os_ext.mk b/build/br-ext/package/optee_os_ext/optee_os_ext.mk new file mode 100644 index 0000000..6d414e1 --- /dev/null +++ b/build/br-ext/package/optee_os_ext/optee_os_ext.mk @@ -0,0 +1,31 @@ +OPTEE_OS_EXT_VERSION = 1.0 +OPTEE_OS_EXT_SOURCE = local +OPTEE_OS_EXT_SITE = $(BR2_PACKAGE_OPTEE_OS_EXT_SITE) +OPTEE_OS_EXT_SITE_METHOD = local +OPTEE_OS_EXT_SDK = $(BR2_PACKAGE_OPTEE_OS_EXT_SDK) + +define OPTEE_OS_EXT_INSTALL_DEVKIT_SHLIBS + @mkdir -p $(TARGET_DIR)/lib/optee_armtz && \ + for f in $(OPTEE_OS_EXT_SDK)/lib/*.ta; do \ + [ -f "$$f" ] || continue; \ + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(TARGET_DIR)/lib/optee_armtz $$f; \ + done +endef + +define OPTEE_OS_EXT_INSTALL_DEVKIT_TAS + for f in $(OPTEE_OS_EXT_SDK)/ta/*.ta; do \ + [ -f "$$f" ] || continue; \ + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(TARGET_DIR)/lib/optee_armtz $$f; \ + done +endef + +define OPTEE_OS_EXT_DO_INSTALL + $(OPTEE_OS_EXT_INSTALL_DEVKIT_SHLIBS) + $(OPTEE_OS_EXT_INSTALL_DEVKIT_TAS) +endef + +OPTEE_OS_EXT_POST_INSTALL_TARGET_HOOKS += OPTEE_OS_EXT_DO_INSTALL + +$(eval $(cmake-package)) diff --git a/build/br-ext/package/optee_rust_examples_ext/Config.in b/build/br-ext/package/optee_rust_examples_ext/Config.in new file mode 100644 index 0000000..8162a33 --- /dev/null +++ b/build/br-ext/package/optee_rust_examples_ext/Config.in @@ -0,0 +1,25 @@ +config BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT + bool "optee_rust_examples" + help + https://github.com/apache/incubator-teaclave-trustzone-sdk + +config BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT_SITE + string "OP-TEE Rust examples path" + default "" + help + The path to the source directory of OP-TEE Rust SDK + +if BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT + +config BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT_CROSS_COMPILE + string "OP-TEE Rust examples cross compiler" + default "" + help + some help + +config BR2_PACKAGE_OPTEE_RUST_EXAMPLES_TC_PATH_ENV + string "OP-TEE Rust examples toolchain path" + default "" + help + some help +endif diff --git a/build/br-ext/package/optee_rust_examples_ext/optee_rust_examples_ext.mk b/build/br-ext/package/optee_rust_examples_ext/optee_rust_examples_ext.mk new file mode 100644 index 0000000..e19e8b5 --- /dev/null +++ b/build/br-ext/package/optee_rust_examples_ext/optee_rust_examples_ext.mk @@ -0,0 +1,51 @@ +OPTEE_RUST_EXAMPLES_EXT_VERSION = 1.0 +OPTEE_RUST_EXAMPLES_EXT_SOURCE = local +OPTEE_RUST_EXAMPLES_EXT_SITE = $(BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT_SITE) +OPTEE_RUST_EXAMPLES_EXT_SITE_METHOD = local +OPTEE_RUST_EXAMPLES_EXT_INSTALL_STAGING = YES +OPTEE_RUST_EXAMPLES_EXT_DEPENDENCIES = optee_client_ext + +ifneq (,$(BR2_PACKAGE_OPTEE_RUST_EXAMPLES_TC_PATH_ENV)) +OPTEE_RUST_EXAMPLES_TC_PATH_ENV = PATH=$(BR2_PACKAGE_OPTEE_RUST_EXAMPLES_TC_PATH_ENV) +endif + +EXAMPLE = $(wildcard examples/*) + +HOST_TARGET := aarch64-unknown-linux-gnu +TA_TARGET := aarch64-unknown-optee-trustzone + +export RUST_TARGET_PATH = $(@D) +export RUST_COMPILER_RT_ROOT = $(RUST_TARGET_PATH)/rust/rust/src/llvm-project/compiler-rt +export OPTEE_DIR = $(@D)/../../.. +export OPTEE_OS_DIR = $(OPTEE_DIR)/optee_os +export OPTEE_CLIENT_DIR = $(OPTEE_DIR)/out-br/build/optee_client_ext-1.0 +export OPTEE_CLIENT_INCLUDE = $(OPTEE_CLIENT_DIR)/out/export/usr/include +export VENDOR = qemu_v8.mk +export OPTEE_OS_INCLUDE = $(OPTEE_DIR)/optee_os/out/arm/export-ta_arm64/include +export CC = $(OPTEE_DIR)/toolchains/aarch64/bin/aarch64-linux-gnu-gcc + +define OPTEE_RUST_EXAMPLES_EXT_BUILD_CMDS + @$(foreach f,$(wildcard $(@D)/examples/*/Makefile), \ + echo Building $f && \ + $(OPTEE_RUST_EXAMPLES_TC_PATH_ENV) $(MAKE) -C $(dir $f) &&) true +endef + +define OPTEE_RUST_EXAMPLES_EXT_INSTALL_TARGET_CMDS + @$(foreach f,$(wildcard $(@D)/examples/*/ta/target/$(TA_TARGET)/release/*.ta), \ + mkdir -p $(TARGET_DIR)/lib/optee_armtz && \ + echo Installing $f && \ + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(TARGET_DIR)/lib/optee_armtz $f \ + &&) true + @$(foreach f,$(wildcard $(@D)/examples/*/host/target/$(HOST_TARGET)/release/*-rs), \ + echo Installing $f && \ + $(INSTALL) -v -p --target-directory=$(TARGET_DIR)/usr/bin $f \ + &&) true + @$(foreach f,$(wildcard $(@D)/examples/*/plugin/target/$(HOST_TARGET)/release/*.plugin.so), \ + mkdir -p $(TARGET_DIR)/usr/lib/tee-supplicant/plugins && \ + echo Installing $f && \ + $(INSTALL) -v -p --target-directory=$(TARGET_DIR)/usr/lib/tee-supplicant/plugins $f \ + &&) true +endef + +$(eval $(generic-package)) diff --git a/build/br-ext/package/optee_test_ext/Config.in b/build/br-ext/package/optee_test_ext/Config.in new file mode 100644 index 0000000..dd4a4d7 --- /dev/null +++ b/build/br-ext/package/optee_test_ext/Config.in @@ -0,0 +1,52 @@ +config BR2_PACKAGE_OPTEE_TEST_EXT + bool "optee_test" + select BR2_PACKAGE_OPTEE_CLIENT_EXT + help + http://github.org/OP-TEE/optee_test + +config BR2_PACKAGE_OPTEE_TEST_EXT_SITE + string "OP-TEE test path" + default "" + help + The path to the source directory of OP-TEE test + +if BR2_PACKAGE_OPTEE_TEST_EXT + +config BR2_PACKAGE_OPTEE_TEST_EXT_SDK + string "OPTEE SDK path" + default "" + help + some help + +config BR2_PACKAGE_OPTEE_TEST_EXT_CROSS_COMPILE + string "OPTEE cross compiler" + default "" + help + some help + +config BR2_PACKAGE_OPTEE_TEST_EXT_GP_PACKAGE + string "GP package" + default "" + help + Path to the GlobalPlatform package containing description files + and TAs needed to build the complete GP test suite. The package + is free for GP members, but can be acquired directly from + GlobalPlatform for non-members. See + https://globalplatform.org/test-suites/tee-initial-configuration-test-suite-with-excluded-tests-list/ + The name of the package is currently "TEE Initial Configuration + Test Suite with Excluded Tests List v2.0.0.2" with the file name + TEE_Initial_Configuration-Test_Suite_v2_0_0_2-2017_06_09.7z + +config BR2_PACKAGE_OPTEE_TEST_EXT_WITH_TLS_TESTS + bool "with-tls-tests" + help + Select Thread Local Storage tests in Trusted Applications. Some + toolchains may not support this. + +config BR2_PACKAGE_OPTEE_TEST_EXT_WITH_CXX_TESTS + bool "with-cxx-tests" + help + Select C++ tests in Trusted Applications. Some toolchains may not + support this. + +endif diff --git a/build/br-ext/package/optee_test_ext/optee_test_ext.mk b/build/br-ext/package/optee_test_ext/optee_test_ext.mk new file mode 100644 index 0000000..8037789 --- /dev/null +++ b/build/br-ext/package/optee_test_ext/optee_test_ext.mk @@ -0,0 +1,71 @@ +OPTEE_TEST_EXT_VERSION = 1.0 +OPTEE_TEST_EXT_SOURCE = local +OPTEE_TEST_EXT_SITE = $(BR2_PACKAGE_OPTEE_TEST_EXT_SITE) +OPTEE_TEST_EXT_SITE_METHOD = local +OPTEE_TEST_EXT_INSTALL_STAGING = YES +OPTEE_TEST_EXT_DEPENDENCIES = optee_client_ext openssl host-python-cryptography +OPTEE_TEST_EXT_SDK = $(BR2_PACKAGE_OPTEE_TEST_EXT_SDK) +OPTEE_TEST_EXT_CONF_OPTS = -DOPTEE_TEST_SDK=$(OPTEE_TEST_EXT_SDK) +# os_test has dependencies, this enforces a valid build order +OPTEE_TEST_EXT_TAS = os_test_lib os_test_lib_dl os_test * +uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1))) + +ifneq ($(BR2_PACKAGE_OPTEE_TEST_EXT_GP_PACKAGE),"") +OPTEE_TEST_EXT_CONF_OPTS += -DWITH_GP_TESTS=1 +OPTEE_TEST_EXT_PRE_CONFIGURE_HOOKS += OPTEE_TEST_EXT_PREPARE_GP_SUITE +endif + +ifneq ($(BR2_PACKAGE_OPTEE_TEST_EXT_WITH_TLS_TESTS),) +TARGET_CONFIGURE_OPTS += WITH_TLS_TESTS=$(BR2_PACKAGE_OPTEE_TEST_EXT_WITH_TLS_TESTS) +endif + +ifneq ($(BR2_PACKAGE_OPTEE_TEST_EXT_WITH_CXX_TESTS),) +TARGET_CONFIGURE_OPTS += WITH_CXX_TESTS=$(BR2_PACKAGE_OPTEE_TEST_EXT_WITH_CXX_TESTS) +endif + +define OPTEE_TEST_EXT_PREPARE_GP_SUITE + sh $(@D)/host/xtest/gp/prepare_suite.sh $(@D) \ + $(BR2_PACKAGE_OPTEE_TEST_EXT_GP_PACKAGE) +endef + +define OPTEE_TEST_EXT_BUILD_TAS + @$(foreach f,$(call uniq,$(foreach t,$(OPTEE_TEST_EXT_TAS),$(wildcard $(@D)/ta/$(t)/Makefile))), \ + echo Building $f && \ + $(MAKE) CROSS_COMPILE="$(shell echo $(BR2_PACKAGE_OPTEE_TEST_EXT_CROSS_COMPILE))" \ + O=out TA_DEV_KIT_DIR=$(OPTEE_TEST_EXT_SDK) \ + PYTHON3=$(HOST_DIR)/bin/python3 \ + $(TARGET_CONFIGURE_OPTS) -C $(dir $f) all &&) true +endef + +define OPTEE_TEST_EXT_INSTALL_TAS + @$(foreach f,$(wildcard $(@D)/ta/*/out/*.ta), \ + mkdir -p $(TARGET_DIR)/lib/optee_armtz && \ + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(TARGET_DIR)/lib/optee_armtz $f \ + &&) true +endef + + +define OPTEE_TEST_EXT_BUILD_GP_TAS + @$(foreach f,$(wildcard $(shell echo $(@D)/host/xtest/gp-suite/TTAs_Internal_API_1_1_1/*/*/{*/,}code_files/Makefile)), \ + echo Building $f && \ + $(MAKE) CROSS_COMPILE="$(shell echo $(BR2_PACKAGE_OPTEE_TEST_EXT_CROSS_COMPILE))" \ + O=out TA_DEV_KIT_DIR=$(OPTEE_TEST_EXT_SDK) \ + $(TARGET_CONFIGURE_OPTS) -C $(dir $f) all &&) true + +endef + +define OPTEE_TEST_EXT_INSTALL_GP_TAS + @$(foreach f,$(wildcard $(shell echo $(@D)/host/xtest/gp-suite/TTAs_Internal_API_1_1_1/*/*/{*/,}code_files/out/*.ta)), \ + mkdir -p $(TARGET_DIR)/lib/optee_armtz && \ + $(INSTALL) -v -p --mode=444 \ + --target-directory=$(TARGET_DIR)/lib/optee_armtz $f \ + &&) true +endef + +OPTEE_TEST_EXT_POST_BUILD_HOOKS += OPTEE_TEST_EXT_BUILD_TAS +OPTEE_TEST_EXT_POST_BUILD_HOOKS += OPTEE_TEST_EXT_BUILD_GP_TAS +OPTEE_TEST_EXT_POST_INSTALL_TARGET_HOOKS += OPTEE_TEST_EXT_INSTALL_TAS +OPTEE_TEST_EXT_POST_INSTALL_TARGET_HOOKS += OPTEE_TEST_EXT_INSTALL_GP_TAS + +$(eval $(cmake-package)) diff --git a/build/br-ext/patches/xen/0001-Revert-xen-arm-Restrict-the-amount-of-memory-that-do.patch b/build/br-ext/patches/xen/0001-Revert-xen-arm-Restrict-the-amount-of-memory-that-do.patch new file mode 100644 index 0000000..75083a6 --- /dev/null +++ b/build/br-ext/patches/xen/0001-Revert-xen-arm-Restrict-the-amount-of-memory-that-do.patch @@ -0,0 +1,37 @@ +From 82a8e6a916f1a7bb3910c365197319b37085d58c Mon Sep 17 00:00:00 2001 +From: Jerome Forissier +Date: Fri, 6 May 2022 17:28:28 +0200 +Subject: [PATCH] Revert "xen/arm: Restrict the amount of memory that dom0less + domU and dom0 can allocate" + +This reverts commit c439f5e97b0229851ba76249ccb224695a1baa29. +--- + xen/arch/arm/domain_build.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c +index f49dbf1ca1..b07461f5d3 100644 +--- a/xen/arch/arm/domain_build.c ++++ b/xen/arch/arm/domain_build.c +@@ -2439,8 +2439,7 @@ static int __init construct_domU(struct domain *d, + + if ( vcpu_create(d, 0) == NULL ) + return -ENOMEM; +- +- d->max_pages = ((paddr_t)mem * SZ_1K) >> PAGE_SHIFT; ++ d->max_pages = ~0U; + + kinfo.d = d; + +@@ -2541,7 +2540,7 @@ int __init construct_dom0(struct domain *d) + + iommu_hwdom_init(d); + +- d->max_pages = dom0_mem >> PAGE_SHIFT; ++ d->max_pages = ~0U; + + kinfo.unassigned_mem = dom0_mem; + kinfo.d = d; +-- +2.34.1 + diff --git a/build/br-ext/patches/xen/0001-Set-config-for-OP-TEE.patch b/build/br-ext/patches/xen/0001-Set-config-for-OP-TEE.patch new file mode 100644 index 0000000..dddff4c --- /dev/null +++ b/build/br-ext/patches/xen/0001-Set-config-for-OP-TEE.patch @@ -0,0 +1,25 @@ +From 2c634e035c3a5cbd0b6d752cbfa1fabca3f6e530 Mon Sep 17 00:00:00 2001 +From: Jerome Forissier +Date: Fri, 6 May 2022 15:25:57 +0200 +Subject: [PATCH] Set config for OP-TEE + +Enables OP-TEE in the default arm64 configuration. + +Signed-off-by: Jerome Forissier +--- + xen/arch/arm/configs/arm64_defconfig | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/xen/arch/arm/configs/arm64_defconfig b/xen/arch/arm/configs/arm64_defconfig +index e69de29bb2..fa571067ee 100644 +--- a/xen/arch/arm/configs/arm64_defconfig ++++ b/xen/arch/arm/configs/arm64_defconfig +@@ -0,0 +1,5 @@ ++CONFIG_EXPERT=y ++CONFIG_TEE=y ++CONFIG_OPTEE=y ++CONFIG_SCHED_CREDIT2_DEFAULT=y ++CONFIG_DEBUG_INFO=n +-- +2.34.1 + diff --git a/build/br-ext/patches/xen/0001-optee-enable-OPTEE_SMC_SEC_CAP_MEMREF_NULL-capabilit.patch b/build/br-ext/patches/xen/0001-optee-enable-OPTEE_SMC_SEC_CAP_MEMREF_NULL-capabilit.patch new file mode 100644 index 0000000..8383b87 --- /dev/null +++ b/build/br-ext/patches/xen/0001-optee-enable-OPTEE_SMC_SEC_CAP_MEMREF_NULL-capabilit.patch @@ -0,0 +1,71 @@ +From d4fb5f166c2bfbaf9ba0de69da0d411288f437a9 Mon Sep 17 00:00:00 2001 +From: Volodymyr Babchuk +Date: Fri, 7 May 2021 01:39:47 +0000 +Subject: [PATCH] optee: enable OPTEE_SMC_SEC_CAP_MEMREF_NULL capability + +OP-TEE mediator already have support for NULL memory references. It +was added in patch 0dbed3ad336 ("optee: allow plain TMEM buffers with +NULL address"). But it does not propagate +OPTEE_SMC_SEC_CAP_MEMREF_NULL capability flag to a guest, so well +behaving guest can't use this feature. + +Note: linux optee driver honors this capability flag when handling +buffers from userspace clients, but ignores it when working with +internal calls. For instance, __optee_enumerate_devices() function +uses NULL argument to get buffer size hint from OP-TEE. This was the +reason, why "optee: allow plain TMEM buffers with NULL address" was +introduced in the first place. + +This patch adds the mentioned capability to list of known +capabilities. From Linux point of view it means that userspace clients +can use this feature, which is confirmed by OP-TEE test suite: + +* regression_1025 Test memref NULL and/or 0 bytes size +o regression_1025.1 Invalid NULL buffer memref registration + regression_1025.1 OK +o regression_1025.2 Input/Output MEMREF Buffer NULL - Size 0 bytes + regression_1025.2 OK +o regression_1025.3 Input MEMREF Buffer NULL - Size non 0 bytes + regression_1025.3 OK +o regression_1025.4 Input MEMREF Buffer NULL over PTA invocation + regression_1025.4 OK + regression_1025 OK + +Signed-off-by: Volodymyr Babchuk +Acked-by: Julien Grall +--- + xen/arch/arm/tee/optee.c | 3 ++- + xen/include/asm-arm/tee/optee_smc.h | 3 +++ + 2 files changed, 5 insertions(+), 1 deletion(-) + +diff --git a/xen/arch/arm/tee/optee.c b/xen/arch/arm/tee/optee.c +index 980eebe847..345361566e 100644 +--- a/xen/arch/arm/tee/optee.c ++++ b/xen/arch/arm/tee/optee.c +@@ -96,7 +96,8 @@ + #define OPTEE_KNOWN_NSEC_CAPS OPTEE_SMC_NSEC_CAP_UNIPROCESSOR + #define OPTEE_KNOWN_SEC_CAPS (OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM | \ + OPTEE_SMC_SEC_CAP_UNREGISTERED_SHM | \ +- OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) ++ OPTEE_SMC_SEC_CAP_DYNAMIC_SHM | \ ++ OPTEE_SMC_SEC_CAP_MEMREF_NULL) + + enum optee_call_state { + OPTEE_CALL_NORMAL, +diff --git a/xen/include/asm-arm/tee/optee_smc.h b/xen/include/asm-arm/tee/optee_smc.h +index d568bb2fe1..2f5c702326 100644 +--- a/xen/include/asm-arm/tee/optee_smc.h ++++ b/xen/include/asm-arm/tee/optee_smc.h +@@ -244,6 +244,9 @@ + */ + #define OPTEE_SMC_SEC_CAP_DYNAMIC_SHM (1 << 2) + ++/* Secure world supports Shared Memory with a NULL reference */ ++#define OPTEE_SMC_SEC_CAP_MEMREF_NULL (1 << 4) ++ + #define OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES 9 + #define OPTEE_SMC_EXCHANGE_CAPABILITIES \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES) +-- +2.34.1 + diff --git a/build/br-ext/patches/xen/0001-optee-immediately-free-RPC-buffers.patch b/build/br-ext/patches/xen/0001-optee-immediately-free-RPC-buffers.patch new file mode 100644 index 0000000..897c10d --- /dev/null +++ b/build/br-ext/patches/xen/0001-optee-immediately-free-RPC-buffers.patch @@ -0,0 +1,70 @@ +From e6059b14107cc061cc35fad44afdd9ad25ba1181 Mon Sep 17 00:00:00 2001 +From: Jens Wiklander +Date: Thu, 28 Apr 2022 09:07:41 +0200 +Subject: [PATCH] optee: immediately free RPC buffers that are released by + OP-TEE + +This commit fixes a case overlooked in [1]. + +There are two kinds of shared memory buffers used by OP-TEE: +1. Normal payload buffer +2. Internal command structure buffers + +The internal command structure buffers are represented with a shadow +copy internally in Xen since this buffer can contain physical addresses +that may need to be translated between real physical address and guest +physical address without leaking information to the guest. + +[1] fixes the problem when releasing the normal payload buffers. The +internal command structure buffers must be released in the same way. +Failure to follow this order opens a window where the guest has freed +the shared memory but Xen is still tracking the buffer. + +During this window the guest may happen to recycle this particular +shared memory in some other thread and try to use it. Xen will block +this which will lead to spurious failures to register a new shared +memory block. + +Fix this by freeing the internal command structure buffers first before +informing the guest that the buffer can be freed. + +[1] 5b13eb1d978e ("optee: immediately free buffers that are released by OP-TEE") + +Signed-off-by: Jens Wiklander +--- + xen/arch/arm/tee/optee.c | 13 ++++++------- + 1 file changed, 6 insertions(+), 7 deletions(-) + +diff --git a/xen/arch/arm/tee/optee.c b/xen/arch/arm/tee/optee.c +index 8a39fe33b0ef..f6d2d689ab60 100644 +--- a/xen/arch/arm/tee/optee.c ++++ b/xen/arch/arm/tee/optee.c +@@ -1149,8 +1149,14 @@ static int handle_rpc_return(struct optee_domain *ctx, + call->rpc_data_cookie = 0; + } + unmap_domain_page(shm_rpc->xen_arg); ++ } else if (call->rpc_op == OPTEE_SMC_RPC_FUNC_FREE) { ++ uint64_t cookie = regpair_to_uint64(get_user_reg(regs, 1), ++ get_user_reg(regs, 2)); ++ ++ free_shm_rpc(ctx, cookie); + } + ++ + return ret; + } + +@@ -1598,13 +1604,6 @@ static void handle_rpc(struct optee_domain *ctx, struct cpu_user_regs *regs) + case OPTEE_SMC_RPC_FUNC_ALLOC: + handle_rpc_func_alloc(ctx, regs, call); + return; +- case OPTEE_SMC_RPC_FUNC_FREE: +- { +- uint64_t cookie = regpair_to_uint64(call->rpc_params[0], +- call->rpc_params[1]); +- free_shm_rpc(ctx, cookie); +- break; +- } + case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR: + break; + case OPTEE_SMC_RPC_FUNC_CMD: diff --git a/build/br-ext/patches/xen/0001-xen-arm-optee-Fix-arm_smccc_smc-s-a0-for-OPTEE_SMC_D.patch b/build/br-ext/patches/xen/0001-xen-arm-optee-Fix-arm_smccc_smc-s-a0-for-OPTEE_SMC_D.patch new file mode 100644 index 0000000..09abdbd --- /dev/null +++ b/build/br-ext/patches/xen/0001-xen-arm-optee-Fix-arm_smccc_smc-s-a0-for-OPTEE_SMC_D.patch @@ -0,0 +1,40 @@ +From 1c3ed9c908732d19660fbe83580674d585464d4c Mon Sep 17 00:00:00 2001 +From: Oleksandr Tyshchenko +Date: Mon, 27 Sep 2021 16:54:10 +0300 +Subject: [PATCH] xen/arm: optee: Fix arm_smccc_smc's a0 for + OPTEE_SMC_DISABLE_SHM_CACHE + +Fix a possible copy-paste error in arm_smccc_smc's first argument (a0) +for OPTEE_SMC_DISABLE_SHM_CACHE case. + +This error causes Linux > v5.14-rc5 (b5c10dd04b7418793517e3286cde5c04759a86de +optee: Clear stale cache entries during initialization) to stuck +repeatedly issuing OPTEE_SMC_DISABLE_SHM_CACHE call and waiting for +the result to be OPTEE_SMC_RETURN_ENOTAVAIL which will never happen. + +Signed-off-by: Oleksandr Tyshchenko +Reviewed-by: Bertrand Marquis +Reviewed-by: Volodymyr Babchuk +Acked-by: Stefano Stabellini +Fixes: 2e35cdf9b2ca ("xen/arm: optee: add OP-TEE mediator skeleton") +Backport: 4.13+ +--- + xen/arch/arm/tee/optee.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/xen/arch/arm/tee/optee.c b/xen/arch/arm/tee/optee.c +index 345361566e..6df0d44eb9 100644 +--- a/xen/arch/arm/tee/optee.c ++++ b/xen/arch/arm/tee/optee.c +@@ -1692,7 +1692,7 @@ static bool optee_handle_call(struct cpu_user_regs *regs) + return true; + + case OPTEE_SMC_DISABLE_SHM_CACHE: +- arm_smccc_smc(OPTEE_SMC_ENABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0, ++ arm_smccc_smc(OPTEE_SMC_DISABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0, + OPTEE_CLIENT_ID(current->domain), &resp); + set_user_reg(regs, 0, resp.a0); + if ( resp.a0 == OPTEE_SMC_RETURN_OK ) { +-- +2.34.1 + diff --git a/build/br-ext/scripts/make_def_config.py b/build/br-ext/scripts/make_def_config.py new file mode 100644 index 0000000..4fc6666 --- /dev/null +++ b/build/br-ext/scripts/make_def_config.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: BSD-2-Clause +# Copyright (c) 2018, Linaro Limited + +import argparse +import shutil +import os +import re + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--br', required=True, + help='Path to buildroot tree') + parser.add_argument('--out', required=True, + help='Path to buildroot out directory') + parser.add_argument('--top-dir', required=True, + help='Replaces %TOP_DIR% in defconfig files') + parser.add_argument('--br-ext-optee', required=True, + help='Path the OP-TEE external buildroot tree') + parser.add_argument('--br-defconfig', required=True, action='append', + help='Buildroot defconfig file') + parser.add_argument('--make-cmd', required=True, + help='Make command') + return parser.parse_args() + +def concatenate_files(top_dir, dst, srcs): + with open(dst, 'w') as outfile: + for fname in srcs: + with open(fname) as infile: + for line in infile: + outfile.write(line.replace('%TOP_DIR%', top_dir)) + +def main(): + args = get_args() + + if not os.path.isdir(args.out): + os.makedirs(args.out) + + concatenate_files(args.top_dir, args.out + '/defconfig', args.br_defconfig) + + if os.path.isabs(args.out): + out = args.out + else: + out = '../' + args.out + + if os.path.isabs(args.br_ext_optee): + br_ext_optee = args.br_ext_optee + else: + br_ext_optee = '../' + args.br_ext_optee + + os.execlp(args.make_cmd, args.make_cmd, '-C', args.br, 'O=' + out, + 'BR2_EXTERNAL=' + br_ext_optee, + 'BR2_DEFCONFIG=' + out + '/defconfig', 'defconfig') + +if __name__ == "__main__": + main() diff --git a/build/common.mk b/build/common.mk new file mode 100644 index 0000000..a217fb0 --- /dev/null +++ b/build/common.mk @@ -0,0 +1,598 @@ +# +# Common definition to all platforms +# + +# Set a variable or error out if it was previously set to a different value +# The reason message (3rd parameter) is optional +# Example: +# $(call force,CFG_FOO,foo,required by CFG_BAR) +define force +$(eval $(call _force,$(1),$(2),$(3))) +endef + +define _force +ifdef $(1) +ifneq ($($(1)),$(2)) +ifneq (,$(3)) +_reason := $$(_empty) [$(3)] +endif +$$(error $(1) is set to '$($(1))' (from $(origin $(1))) but its value must be '$(2)'$$(_reason)) +endif +endif +$(1) := $(2) +endef + +SHELL := bash +BASH ?= bash +PYTHON3 ?= python3 +ROOT ?= $(shell pwd)/.. + +UNAME_M := $(shell uname -m) +ARCH ?= arm +BUILD_PATH ?= $(ROOT)/build +LINUX_PATH ?= $(ROOT)/linux +UBOOT_PATH ?= $(ROOT)/u-boot +BENCHMARK_APP_PATH ?= $(ROOT)/optee_benchmark +OPTEE_OS_PATH ?= $(ROOT)/optee_os +OPTEE_CLIENT_PATH ?= $(ROOT)/optee_client +OPTEE_TEST_PATH ?= $(ROOT)/optee_test +OPTEE_EXAMPLES_PATH ?= $(ROOT)/optee_examples +OPTEE_RUST_PATH ?= $(ROOT)/optee_rust +BUILDROOT_TARGET_ROOT ?= $(ROOT)/out-br/target + +# default high verbosity. slow uarts shall specify lower if prefered +CFG_TEE_CORE_LOG_LEVEL ?= 3 + +# default disable latency benchmarks (over all OP-TEE layers) +CFG_TEE_BENCHMARK ?= n + +# optee_test +WITH_TLS_TESTS ?= y +ifneq ($(COMPILER),clang) +# assuming GCC toolchain from toolchain.mk +WITH_CXX_TESTS ?= y +endif + +CCACHE ?= $(shell which ccache) # Don't remove this comment (space is needed) + +# QEMU shared folders settings +# +# TL;DR: +# 1) make QEMU_VIRTFS_AUTOMOUNT=y run +# will mount the project's root on the host as /mnt/host in QEMU. +# 2) mkdir -p /tmp/qemu-data-tee && make QEMU_PSS_AUTOMOUNT=y run +# will mount the host directory /tmp/qemu-data-tee as /data/tee +# in QEMU, thus creating persistent secure storage. + +ifeq ($(QEMU_VIRTFS_AUTOMOUNT),y) +$(call force,QEMU_VIRTFS_ENABLE,y,required by QEMU_VIRTFS_AUTOMOUNT) +endif + +ifeq ($(QEMU_PSS_AUTOMOUNT),y) +$(call force,QEMU_PSS_ENABLE,y,required by QEMU_PSS_AUTOMOUNT) +endif + +ifeq ($(QEMU_PSS_ENABLE),y) +$(call force,QEMU_VIRTFS_ENABLE,y,required by QEMU_PSS_ENABLE) +endif + +# Accessing a shared folder on the host from QEMU: +# # Set QEMU_VIRTFS_ENABLE to 'y' and adjust QEMU_VIRTFS_HOST_DIR +# # Then in QEMU, run: +# # $ mount -t 9p -o trans=virtio host +# # Or enable QEMU_VIRTFS_AUTOMOUNT +QEMU_VIRTFS_ENABLE ?= n +QEMU_VIRTFS_HOST_DIR ?= $(ROOT) + +# Persistent Secure Storage via shared folder +# # Set QEMU_PSS_ENABLE to 'y' and adjust QEMU_PSS_HOST_DIR +# # Then in QEMU, run: +# # $ mount -t 9p -o trans=virtio secure /data/tee +# # Or enable QEMU_PSS_AUTOMOUNT +QEMU_PSS_ENABLE ?= n +QEMU_PSS_HOST_DIR ?= /tmp/qemu-data-tee + +# Warning: when these variables are modified, you must remake the buildroot +# target directory. This can be done without rebuilding everything as follows: +# rm -rf ../out-br/target; find ../out-br/ -name .stamp_target_installed | xargs rm +# make run +QEMU_VIRTFS_AUTOMOUNT ?= n +QEMU_PSS_AUTOMOUNT ?= n +# Mount point for the shared directory inside QEMU +# Used by the post-build script, this is written to /etc/fstab as the mount +# point of the shared directory +QEMU_VIRTFS_MOUNTPOINT ?= /mnt/host + +# End of QEMU shared folder settings + +################################################################################ +# Mandatory for autotools (for specifying --host) +################################################################################ +ifeq ($(COMPILE_NS_USER),64) +ifeq ($(UNAME_M),x86_64) +MULTIARCH := aarch64-linux-gnu +else ifeq ($(UNAME_M),aarch64) +MULTIARCH := aarch64-linux +else +MULTIARCH := aarch64-linux +endif +else +ifeq ($(UNAME_M),x86_64) +MULTIARCH := arm-linux-gnueabihf +else ifeq ($(UNAME_M),aarch64) +MULTIARCH := arm-linux-gnueabihf +else +MULTIARCH := arm-linux +endif +endif + +################################################################################ +# Check coherency of compilation mode +################################################################################ + +ifneq ($(COMPILE_NS_USER),) +ifeq ($(COMPILE_NS_KERNEL),) +$(error COMPILE_NS_KERNEL must be defined as COMPILE_NS_USER=$(COMPILE_NS_USER) is defined) +endif +ifeq (,$(filter $(COMPILE_NS_USER),32 64)) +$(error COMPILE_NS_USER=$(COMPILE_NS_USER) - Should be 32 or 64) +endif +endif + +ifneq ($(COMPILE_NS_KERNEL),) +ifeq ($(COMPILE_NS_USER),) +$(error COMPILE_NS_USER must be defined as COMPILE_NS_KERNEL=$(COMPILE_NS_KERNEL) is defined) +endif +ifeq (,$(filter $(COMPILE_NS_KERNEL),32 64)) +$(error COMPILE_NS_KERNEL=$(COMPILE_NS_KERNEL) - Should be 32 or 64) +endif +endif + +ifeq ($(COMPILE_NS_KERNEL),32) +ifneq ($(COMPILE_NS_USER),32) +$(error COMPILE_NS_USER=$(COMPILE_NS_USER) - Should be 32 as COMPILE_NS_KERNEL=$(COMPILE_NS_KERNEL)) +endif +endif + +ifneq ($(COMPILE_S_USER),) +ifeq ($(COMPILE_S_KERNEL),) +$(error COMPILE_S_KERNEL must be defined as COMPILE_S_USER=$(COMPILE_S_USER) is defined) +endif +ifeq (,$(filter $(COMPILE_S_USER),32 64)) +$(error COMPILE_S_USER=$(COMPILE_S_USER) - Should be 32 or 64) +endif +endif + +ifneq ($(COMPILE_S_KERNEL),) +OPTEE_OS_COMMON_EXTRA_FLAGS ?= O=out/$(ARCH) +OPTEE_OS_BIN ?= $(OPTEE_OS_PATH)/out/$(ARCH)/core/tee.bin +OPTEE_OS_HEADER_V2_BIN ?= $(OPTEE_OS_PATH)/out/$(ARCH)/core/tee-header_v2.bin +OPTEE_OS_PAGER_V2_BIN ?= $(OPTEE_OS_PATH)/out/$(ARCH)/core/tee-pager_v2.bin +OPTEE_OS_PAGEABLE_V2_BIN ?= $(OPTEE_OS_PATH)/out/$(ARCH)/core/tee-pageable_v2.bin +ifeq ($(COMPILE_S_USER),) +$(error COMPILE_S_USER must be defined as COMPILE_S_KERNEL=$(COMPILE_S_KERNEL) is defined) +endif +ifeq (,$(filter $(COMPILE_S_KERNEL),32 64)) +$(error COMPILE_S_KERNEL=$(COMPILE_S_KERNEL) - Should be 32 or 64) +endif +endif + +ifeq ($(COMPILE_S_KERNEL),32) +ifneq ($(COMPILE_S_USER),32) +$(error COMPILE_S_USER=$(COMPILE_S_USER) - Should be 32 as COMPILE_S_KERNEL=$(COMPILE_S_KERNEL)) +endif +endif + + +################################################################################ +# set the compiler when COMPILE_xxx are defined +################################################################################ +ifeq ($(ARCH),arm) +CROSS_COMPILE_NS_USER ?= "$(CCACHE)$(AARCH$(COMPILE_NS_USER)_CROSS_COMPILE)" +CROSS_COMPILE_NS_KERNEL ?= "$(CCACHE)$(AARCH$(COMPILE_NS_KERNEL)_CROSS_COMPILE)" +CROSS_COMPILE_S_USER ?= "$(CCACHE)$(AARCH$(COMPILE_S_USER)_CROSS_COMPILE)" +CROSS_COMPILE_S_KERNEL ?= "$(CCACHE)$(AARCH$(COMPILE_S_KERNEL)_CROSS_COMPILE)" +else ifeq ($(ARCH),riscv) +CROSS_COMPILE_NS_USER ?= "$(CCACHE)$(RISCV$(COMPILE_NS_USER)_CROSS_COMPILE)" +CROSS_COMPILE_NS_KERNEL ?= "$(CCACHE)$(RISCV$(COMPILE_NS_KERNEL)_CROSS_COMPILE)" +CROSS_COMPILE_S_USER ?= "$(CCACHE)$(RISCV$(COMPILE_S_USER)_CROSS_COMPILE)" +CROSS_COMPILE_S_KERNEL ?= "$(CCACHE)$(RISCV$(COMPILE_S_KERNEL)_CROSS_COMPILE)" +endif + +################################################################################ +# defines, macros, configuration etc +################################################################################ +define KERNEL_VERSION +$(shell cd $(LINUX_PATH) && $(MAKE) --no-print-directory kernelversion) +endef + +# Read stdin, expand ${VAR} environment variables, output to stdout +# http://superuser.com/a/302847 +define expand-env-var +awk '{while(match($$0,"[$$]{[^}]*}")) {var=substr($$0,RSTART+2,RLENGTH -3);gsub("[$$]{"var"}",ENVIRON[var])}}1' +endef + +DEBUG ?= 0 + +# Macro to check if a compiler supports a given option +# For example: $(call cc-option,gcc,-Wno-error=stringop-truncation,) +# ...will return -Wno-error=stringop-truncation if gcc supports it, empty +# otherwise. +__cc-option = $(if $(shell $(1) $(2) -c -x c /dev/null -o /dev/null 2>&1 >/dev/null),$(3),$(2)) +_cc-opt-cached-var-name = cached-cc-option$(subst =,~,$(strip $(2)))$(subst $(empty) $(empty),,$(1)) +define _cc-option +$(eval _cached := $(call _cc-opt-cached-var-name,$1,$2)) +$(eval $(_cached) := $(if $(filter $(origin $(_cached)),undefined),$(call __cc-option,$(1),$(2),$(3)),$($(_cached)))) +$($(_cached)) +endef +cc-option = $(strip $(call _cc-option,$(1),$(2),$(3))) + +################################################################################ +# default target is all +################################################################################ +.PHONY: all +all: + +################################################################################ +# Build root +################################################################################ +ifeq ($(ARCH),arm) +BUILDROOT_ARCH=aarch$(COMPILE_NS_USER) +else ifeq ($(ARCH),riscv) +BUILDROOT_ARCH=riscv$(COMPILE_NS_USER) +endif +ifeq ($(GDBSERVER),y) +BUILDROOT_TOOLCHAIN=toolchain-br # Use toolchain supplied by buildroot +DEFCONFIG_GDBSERVER=--br-defconfig build/br-ext/configs/gdbserver.conf +else +# Local toolchains (downloaded by "make toolchains") +ifeq ($(UNAME_M),x86_64) +ifeq ($(ARCH),arm) +BUILDROOT_TOOLCHAIN=toolchain-aarch$(COMPILE_NS_USER) +else ifeq ($(ARCH),riscv) +BUILDROOT_TOOLCHAIN=toolchain-riscv$(COMPILE_NS_USER) +endif +else ifeq ($(UNAME_M),aarch64) +ifeq ($(COMPILE_NS_USER),64) +BUILDROOT_TOOLCHAIN=toolchain-aarch64-sdk +else +BUILDROOT_TOOLCHAIN=toolchain-aarch32 +endif +else +BUILDROOT_TOOLCHAIN=toolchain-aarch$(COMPILE_NS_USER)-sdk +endif +endif + +ifeq ($(XEN_BOOT),y) +DEFCONFIG_XEN=--br-defconfig build/br-ext/configs/xen.conf +# The version of Xen provided by Buildroot needs a few patches to work with +# OP-TEE +BR2_GLOBAL_PATCH_DIR=../build/br-ext/patches +endif + +BR2_PER_PACKAGE_DIRECTORIES ?= y +BR2_PACKAGE_LIBOPENSSL ?= y +BR2_PACKAGE_MMC_UTILS ?= y +BR2_PACKAGE_OPENSSL ?= y +BR2_PACKAGE_OPTEE_BENCHMARK_EXT ?= $(CFG_TEE_BENCHMARK) +BR2_PACKAGE_OPTEE_BENCHMARK_EXT_SITE ?= $(BENCHMARK_APP_PATH) +BR2_PACKAGE_OPTEE_CLIENT_EXT_SITE ?= $(OPTEE_CLIENT_PATH) +BR2_PACKAGE_OPTEE_EXAMPLES_EXT ?= y +BR2_PACKAGE_OPTEE_EXAMPLES_EXT_CROSS_COMPILE ?= $(CROSS_COMPILE_S_USER) +BR2_PACKAGE_OPTEE_EXAMPLES_EXT_SDK ?= $(OPTEE_OS_TA_DEV_KIT_DIR) +BR2_PACKAGE_OPTEE_EXAMPLES_EXT_SITE ?= $(OPTEE_EXAMPLES_PATH) +OPTEE_RUST_ENABLE ?= n +ifeq ($(OPTEE_RUST_ENABLE),y) +BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT ?= y +BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT_CROSS_COMPILE ?= $(CROSS_COMPILE_S_USER) +BR2_PACKAGE_OPTEE_RUST_EXAMPLES_EXT_SITE ?= $(OPTEE_RUST_PATH) +BR2_PACKAGE_OPTEE_RUST_EXAMPLES_TC_PATH_ENV = $(ROOT)/toolchains/aarch64/bin:$(HOME)/.cargo/bin:$(PATH) +endif +# The OPTEE_OS package builds nothing, it just installs files into the +# root FS when applicable (for example: shared libraries) +BR2_PACKAGE_OPTEE_OS_EXT ?= y +BR2_PACKAGE_OPTEE_OS_EXT_SDK ?= $(OPTEE_OS_TA_DEV_KIT_DIR) +BR2_PACKAGE_OPTEE_OS_EXT_SITE ?= $(CURDIR)/br-ext/package/optee_os_ext +BR2_PACKAGE_OPTEE_TEST_EXT ?= y +BR2_PACKAGE_OPTEE_TEST_EXT_CROSS_COMPILE ?= $(CROSS_COMPILE_S_USER) +BR2_PACKAGE_OPTEE_TEST_EXT_SDK ?= $(OPTEE_OS_TA_DEV_KIT_DIR) +BR2_PACKAGE_OPTEE_TEST_EXT_SITE ?= $(OPTEE_TEST_PATH) +BR2_PACKAGE_OPTEE_TEST_EXT_GP_PACKAGE := $(GP_PACKAGE) +BR2_PACKAGE_OPTEE_TEST_EXT_WITH_TLS_TESTS := $(WITH_TLS_TESTS) +BR2_PACKAGE_OPTEE_TEST_EXT_WITH_CXX_TESTS := $(WITH_CXX_TESTS) +BR2_PACKAGE_STRACE ?= y +ifeq ($(XEN_BOOT),y) +BR2_TARGET_GENERIC_GETTY_PORT ?= "console" +else +BR2_TARGET_GENERIC_GETTY_PORT ?= $(if $(CFG_NW_CONSOLE_UART),ttyAMA$(CFG_NW_CONSOLE_UART),ttyAMA0) +endif + +# Embed opensc for pkcs11-tool +BR2_PACKAGE_OPENSC ?= y + +# Embed keyutils for trusted-keys +BR2_PACKAGE_KEYUTILS ?= y + +# All BR2_* variables from the makefile or the environment are appended to +# ../out-br/extra.conf. All values are quoted "..." except y and n. +double-quote = "#" # This really sets the variable to " and avoids upsetting vim's syntax highlighting +streq = $(and $(findstring $(1),$(2)),$(findstring $(2),$(1))) +y-or-n = $(or $(call streq,y,$(1)),$(call streq,n,$(1))) +append-var_ = echo '$(1)=$(3)'$($(1))'$(3)' >>$(2); +append-var = $(call append-var_,$(1),$(2),$(if $(call y-or-n,$($(1))),,$(double-quote))) +append-br2-vars = $(foreach var,$(filter BR2_%,$(.VARIABLES)),$(call append-var,$(var),$(1))) + +ifneq (y,$(BR2_PER_PACKAGE_DIRECTORIES)) +br-make-flags := -j1 +endif + +.PHONY: buildroot +buildroot: optee-os optee-rust + @mkdir -p ../out-br + @rm -f ../out-br/build/optee_*/.stamp_* + @rm -f ../out-br/extra.conf + @$(call append-br2-vars,../out-br/extra.conf) + @(cd .. && $(PYTHON3) build/br-ext/scripts/make_def_config.py \ + --br buildroot --out out-br --br-ext build/br-ext \ + --top-dir "$(ROOT)" \ + --br-defconfig build/br-ext/configs/optee_$(BUILDROOT_ARCH) \ + --br-defconfig build/br-ext/configs/optee_generic \ + --br-defconfig build/br-ext/configs/$(BUILDROOT_TOOLCHAIN) \ + $(DEFCONFIG_GDBSERVER) \ + $(DEFCONFIG_XEN) \ + $(DEFCONFIG_TSS) \ + $(DEFCONFIG_TPM_MODULE) \ + $(DEFCONFIG_FTPM) \ + --br-defconfig out-br/extra.conf \ + --make-cmd $(MAKE)) + @$(MAKE) $(br-make-flags) -C ../out-br all + +.PHONY: buildroot-clean +buildroot-clean: + @test ! -d $(ROOT)/out-br || $(MAKE) -C $(ROOT)/out-br clean + +.PHONY: buildroot-cleaner +buildroot-cleaner: + @rm -rf $(ROOT)/out-br + +################################################################################ +# Linux +################################################################################ +ifeq ($(CFG_TEE_BENCHMARK),y) +LINUX_DEFCONFIG_BENCH ?= $(CURDIR)/kconfigs/tee_bench.conf +endif + +LINUX_COMMON_FLAGS ?= LOCALVERSION= CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) + +.PHONY: linux-common +linux-common: linux-defconfig + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) + +$(LINUX_PATH)/.config: $(LINUX_DEFCONFIG_COMMON_FILES) + cd $(LINUX_PATH) && \ + ARCH=$(LINUX_DEFCONFIG_COMMON_ARCH) \ + CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) \ + scripts/kconfig/merge_config.sh $(LINUX_DEFCONFIG_COMMON_FILES) \ + $(LINUX_DEFCONFIG_BENCH) + +.PHONY: linux-defconfig-clean-common +linux-defconfig-clean-common: + rm -f $(LINUX_PATH)/.config + +# LINUX_CLEAN_COMMON_FLAGS should be defined in specific makefiles (hikey.mk,...) +.PHONY: linux-clean-common +linux-clean-common: linux-defconfig-clean + $(MAKE) -C $(LINUX_PATH) $(LINUX_CLEAN_COMMON_FLAGS) clean + +# LINUX_CLEANER_COMMON_FLAGS should be defined in specific makefiles (hikey.mk,...) +.PHONY: linux-cleaner-common +linux-cleaner-common: linux-defconfig-clean + $(MAKE) -C $(LINUX_PATH) $(LINUX_CLEANER_COMMON_FLAGS) distclean + +################################################################################ +# EDK2 / Tianocore +################################################################################ +.PHONY: edk2-common +edk2-common: + $(call edk2-env) && \ + export PACKAGES_PATH=$(EDK2_PATH):$(EDK2_PLATFORMS_PATH) && \ + source $(EDK2_PATH)/edksetup.sh && \ + $(MAKE) -j1 -C $(EDK2_PATH)/BaseTools && \ + $(call edk2-call) all + +.PHONY: edk2-clean-common +edk2-clean-common: + $(call edk2-env) && \ + export PACKAGES_PATH=$(EDK2_PATH):$(EDK2_PLATFORMS_PATH) && \ + source $(EDK2_PATH)/edksetup.sh && \ + $(MAKE) -j1 -C $(EDK2_PATH)/BaseTools clean && \ + $(call edk2-call) cleanall + +################################################################################ +# QEMU / QEMUv8 +################################################################################ +QEMU_CONFIGURE_PARAMS_COMMON = --cc="$(CCACHE)gcc" --extra-cflags="-Wno-error" \ + --disable-docs +QEMU_EXTRA_ARGS +=\ + -object rng-random,filename=/dev/urandom,id=rng0 \ + -device virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000 + +ifeq ($(QEMU_VIRTFS_ENABLE),y) +QEMU_CONFIGURE_PARAMS_COMMON += --enable-virtfs +QEMU_EXTRA_ARGS +=\ + -fsdev local,id=fsdev0,path=$(QEMU_VIRTFS_HOST_DIR),security_model=none \ + -device virtio-9p-device,fsdev=fsdev0,mount_tag=host +ifeq ($(QEMU_PSS_ENABLE),y) +QEMU_EXTRA_ARGS +=\ + -fsdev local,id=fsdev1,path=$(QEMU_PSS_HOST_DIR),security_model=mapped-xattr \ + -device virtio-9p-device,fsdev=fsdev1,mount_tag=secure +endif +endif + +ifeq ($(GDBSERVER),y) +HOSTFWD := ,hostfwd=tcp::12345-:12345 +endif +# Enable QEMU SLiRP user networking +QEMU_EXTRA_ARGS +=\ + -netdev user,id=vmnic$(HOSTFWD) -device virtio-net-device,netdev=vmnic + +define run-help + @echo + @echo \* QEMU is now waiting to start the execution + @echo \* Start execution with either a \'c\' followed by \ in the QEMU console or + @echo \* attach a debugger and continue from there. + @echo \* + @echo \* To run OP-TEE tests, use the xtest command in the \'Normal World\' terminal + @echo \* Enter \'xtest -h\' for help. + @echo +endef + +ifneq (, $(LAUNCH_TERMINAL)) +define launch-terminal + @nc -z 127.0.0.1 $(1) || \ + $(LAUNCH_TERMINAL) "$(BUILD_PATH)/soc_term.py $(1)" & +endef +else +gnome-terminal := $(shell command -v gnome-terminal 2>/dev/null) +konsole := $(shell command -v konsole 2>/dev/null) +xterm := $(shell command -v xterm 2>/dev/null) +ifdef gnome-terminal +define launch-terminal + @nc -z 127.0.0.1 $(1) || \ + $(gnome-terminal) -t $(2) -x $(BUILD_PATH)/soc_term.py $(1) & +endef +else +ifdef konsole +define launch-terminal + @nc -z 127.0.0.1 $(1) || \ + $(konsole) --new-tab -p tabtitle=$(2) -e $(BUILD_PATH)/soc_term.py $(1) & +endef +else +ifdef xterm +define launch-terminal + @nc -z 127.0.0.1 $(1) || \ + $(xterm) -title $(2) -e $(BASH) -c "$(BUILD_PATH)/soc_term.py $(1)" & +endef +else +check-terminal := @echo "Error: could not find gnome-terminal, konsole nor xterm" ; false +endif +endif +endif +endif + +define wait-for-ports + @while ! nc -z 127.0.0.1 $(1) || ! nc -z 127.0.0.1 $(2); do sleep 1; done +endef + +################################################################################ +# OP-TEE +################################################################################ +ifeq ($(ARCH),arm) +ifeq ($(COMPILE_S_USER),32) +OPTEE_OS_TA_DEV_KIT_DIR ?= $(OPTEE_OS_PATH)/out/arm/export-ta_arm32 +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_USER_TA_TARGETS=ta_arm32 +endif +ifeq ($(COMPILE_S_USER),64) +OPTEE_OS_TA_DEV_KIT_DIR ?= $(OPTEE_OS_PATH)/out/arm/export-ta_arm64 +ifeq ($(MEASURED_BOOT_FTPM),y) +# The fTPM TA can only be built for 32-bit so enable the 32-bit libraries as well +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_USER_TA_TARGETS="ta_arm64 ta_arm32" +else +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_USER_TA_TARGETS=ta_arm64 +endif +endif + +ifeq ($(COMPILE_S_KERNEL),64) +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_ARM64_core=y +else +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_ARM64_core=n +endif + +OPTEE_OS_TA_CROSS_COMPILE_FLAGS += CROSS_COMPILE_ta_arm64="$(CCACHE)$(AARCH64_CROSS_COMPILE)" +OPTEE_OS_TA_CROSS_COMPILE_FLAGS += CROSS_COMPILE_ta_arm32="$(CCACHE)$(AARCH32_CROSS_COMPILE)" + +else ifeq ($(ARCH),riscv) + +ifeq ($(COMPILE_S_USER),32) +OPTEE_OS_TA_DEV_KIT_DIR ?= $(OPTEE_OS_PATH)/out/riscv/export-ta_rv32 +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_USER_TA_TARGETS=ta_rv32 +endif +ifeq ($(COMPILE_S_USER),64) +OPTEE_OS_TA_DEV_KIT_DIR ?= $(OPTEE_OS_PATH)/out/riscv/export-ta_rv64 +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_USER_TA_TARGETS=ta_rv64 +endif + +ifeq ($(COMPILE_S_KERNEL),64) +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_RV64_core=y +else +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_RV64_core=n +endif + +OPTEE_OS_TA_CROSS_COMPILE_FLAGS += CROSS_COMPILE_ta_rv64="$(CCACHE)$(RISCV64_CROSS_COMPILE)" +OPTEE_OS_TA_CROSS_COMPILE_FLAGS += CROSS_COMPILE_ta_rv32="$(CCACHE)$(RISCV32_CROSS_COMPILE)" +endif + +CFG_IN_TREE_EARLY_TAS ?= trusted_keys/f04a0fe7-1f5d-4b9b-abf7-619b85b4ce8c + +OPTEE_OS_COMMON_FLAGS ?= \ + $(OPTEE_OS_COMMON_EXTRA_FLAGS) \ + PLATFORM=$(OPTEE_OS_PLATFORM) \ + CROSS_COMPILE=$(CROSS_COMPILE_S_USER) \ + CROSS_COMPILE_core=$(CROSS_COMPILE_S_KERNEL) \ + $(OPTEE_OS_TA_CROSS_COMPILE_FLAGS) \ + CFG_TEE_CORE_LOG_LEVEL=$(CFG_TEE_CORE_LOG_LEVEL) \ + DEBUG=$(DEBUG) \ + CFG_TEE_BENCHMARK=$(CFG_TEE_BENCHMARK) \ + CFG_IN_TREE_EARLY_TAS="$(CFG_IN_TREE_EARLY_TAS)" + +.PHONY: optee-os-common +optee-os-common: + $(MAKE) -C $(OPTEE_OS_PATH) $(OPTEE_OS_COMMON_FLAGS) + +.PHONY: optee-os-clean-common +optee-os-clean-common: + $(MAKE) -C $(OPTEE_OS_PATH) $(OPTEE_OS_COMMON_FLAGS) clean + +################################################################################ +# OP-TEE Rust +################################################################################ +.PHONY: optee-rust +optee-rust: $(OPTEE_RUST_PATH)/.done + +$(OPTEE_RUST_PATH)/.done: +ifeq ($(OPTEE_RUST_ENABLE),y) + @(export OPTEE_DIR=$(ROOT) && \ + export CARGO_NET_GIT_FETCH_WITH_CLI=true && \ + cd $(OPTEE_RUST_PATH) && ./setup.sh && touch .done) +endif + +optee-rust-clean: + rm -f $(OPTEE_RUST_PATH)/.done + +################################################################################ +# fTPM Rules +################################################################################ + +# The fTPM implementation is based on ARM32 architecture whereas the rest of the +# system is built to run on 64-bit mode (COMPILE_S_USER = 64). Therefore set +# TA_DEV_KIT_DIR manually to the arm32 OPTEE toolkit rather than relying on +# OPTEE_OS_TA_DEV_KIT_DIR variable. +FTPM_FLAGS ?= \ + TA_CPU=cortex-a9 \ + TA_CROSS_COMPILE=$(AARCH32_CROSS_COMPILE) \ + TA_DEV_KIT_DIR=$(OPTEE_OS_PATH)/out/arm/export-ta_arm32 \ + CFG_TA_DEBUG=y CFG_TEE_TA_LOG_LEVEL=4 CFG_TA_MEASURED_BOOT=y + +.PHONY: ftpm +ftpm: +ifeq ($(MEASURED_BOOT_FTPM),y) +ftpm: optee-os + $(FTPM_FLAGS) $(MAKE) -C $(FTPM_PATH) +endif + +.PHONY: ftpm-clean +ftpm-clean: +ifeq ($(MEASURED_BOOT_FTPM),y) +ftpm-clean: + -$(FTPM_FLAGS) $(MAKE) -C $(FTPM_PATH) clean +endif diff --git a/build/dra7xx.mk b/build/dra7xx.mk new file mode 100644 index 0000000..f40f579 --- /dev/null +++ b/build/dra7xx.mk @@ -0,0 +1,38 @@ +############################################################################### +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +############################################################################### +override COMPILE_NS_USER := 32 +override COMPILE_NS_KERNEL := 32 +override COMPILE_S_USER := 32 +override COMPILE_S_KERNEL := 32 + +# Need to set this before including common.mk +BR2_TARGET_GENERIC_GETTY_PORT ?= ttyS0 + +############################################################################### +# Includes +############################################################################### +include common.mk + +############################################################################### +# Paths to git projects and various binaries +############################################################################### +STAGING_AREA ?= $(ROOT)/out +U-BOOT_PATH ?= $(ROOT)/u-boot +UBOOT_SPL ?= $(U-BOOT_PATH)/u-boot-spl_HS_MLO +UBOOT_IMG ?= $(U-BOOT_PATH)/u-boot_HS.img +UBOOT_ENV ?= $(BUILD_PATH)/ti/uEnv.txt +LINUX_IMAGE ?= $(LINUX_PATH)/arch/arm/boot/zImage +LINUX_DTBS ?= $(wildcard $(LINUX_PATH)/arch/arm/boot/dts/dra7*.dtb) +FIT_SOURCE ?= $(BUILD_PATH)/ti/fitImage-dra7xx.its +FIT_MAKEFILE ?= $(BUILD_PATH)/ti/Makefile +OPTEE_OS_PLATFORM ?= ti-dra7xx +U-BOOT_CONFIG ?= dra7xx_hs_evm_defconfig +CONFIG_TYPE ?= ti_sdk_dra7x_debug + +############################################################################### +# Include common to TI builds +############################################################################### +include ti/ti-common.mk diff --git a/build/fvp-psa-sp.mk b/build/fvp-psa-sp.mk new file mode 100644 index 0000000..848ad53 --- /dev/null +++ b/build/fvp-psa-sp.mk @@ -0,0 +1,131 @@ +FVP_USE_BASE_PLAT ?= y +FVP_VIRTFS_ENABLE ?= y +FVP_VIRTFS_AUTOMOUNT ?= y +MEASURED_BOOT ?= y +MEASURED_BOOT_FTPM ?= n +TS_SMM_GATEWAY ?= y +TS_UEFI_TESTS ?= n +# Supported values: embedded, fip +SP_PACKAGING_METHOD ?= embedded +SPMC_TESTS ?= n + +# Enable the "HArdware Volatile Entropy Gathering and Expansion" daemon to +# overcome low-entropy conditions in the FVP +BR2_PACKAGE_HAVEGED ?= y + +# Disable packages not used by this configuration +BR2_PACKAGE_HOST_E2FSPROGS ?= n +BR2_PACKAGE_KEYUTILS ?= n +BR2_PACKAGE_MMC_UTILS ?= n +BR2_PACKAGE_OPENSC ?= n +BR2_PACKAGE_OPTEE_EXAMPLES_EXT ?= n +BR2_PACKAGE_STRACE ?= n + +# Building xtest is not necessary if we don't want to run the SPMC tests +ifneq ($(SPMC_TESTS),y) +BR2_PACKAGE_OPTEE_TEST_EXT ?= n +BR2_PACKAGE_LIBOPENSSL ?= n +BR2_PACKAGE_OPENSSL ?= n +endif + +# TS SP configurations +DEFAULT_SP_CONFIG ?= default-opteesp +SP_BLOCK_STORAGE_CONFIG ?= $(DEFAULT_SP_CONFIG) +SP_PSA_ITS_CONFIG ?= $(DEFAULT_SP_CONFIG) +SP_PSA_PS_CONFIG ?= $(DEFAULT_SP_CONFIG) +SP_PSA_CRYPTO_CONFIG ?= $(DEFAULT_SP_CONFIG) +SP_PSA_ATTESTATION_CONFIG ?= $(DEFAULT_SP_CONFIG) +SP_SMM_GATEWAY_CONFIG ?= $(DEFAULT_SP_CONFIG) + +TF_A_FLAGS ?= \ + BL32=$(OPTEE_OS_PAGER_V2_BIN) \ + BL33=$(EDK2_BIN) \ + PLAT=fvp \ + SPD=spmd \ + SPMD_SPM_AT_SEL2=0 \ + ARM_SPMC_MANIFEST_DTS=$(ROOT)/build/fvp/spmc_manifest.dts \ + $(TF_A_FIP_SP_FLAGS) + +LINUX_DEFCONFIG_COMMON_FILES ?= $(CURDIR)/kconfigs/fvp_trusted-services.conf + +include fvp.mk +include trusted-services.mk + +# The macros used in bl2_sp_list.dts and spmc_manifest.dts has to be passed to +# TF-A because it handles the preprocessing of these files. +define add-dtc-define +DTC_CPPFLAGS+=-D$1=$(subst y,1,$(subst n,0,$($1))) +endef + +ifeq ($(SP_PACKAGING_METHOD),fip) +$(eval $(call add-dtc-define,SPMC_TESTS)) +$(eval $(call add-dtc-define,TS_SMM_GATEWAY)) + +TF_A_EXPORTS += DTC_CPPFLAGS="$(DTC_CPPFLAGS)" +endif + +OPTEE_OS_COMMON_EXTRA_FLAGS += \ + CFG_SECURE_PARTITION=y \ + CFG_CORE_SEL1_SPMC=y \ + CFG_CORE_HEAP_SIZE=131072 \ + CFG_DT=y \ + CFG_MAP_EXT_DT_SECURE=y + + +# The boot order of the SPs is determined by the order of calls here. This is +# due to the SPMC not (yet) supporting the boot order field of the SP manifest. +ifeq ($(SPMC_TESTS),n) +# PSA SPs +$(eval $(call build-sp,block-storage,config/$(SP_BLOCK_STORAGE_CONFIG),63646e80-eb52-462f-ac4f-8cdf3987519c,$(SP_BLOCK_STORAGE_EXTRA_FLAGS))) +$(eval $(call build-sp,internal-trusted-storage,config/$(SP_PSA_ITS_CONFIG),dc1eef48-b17a-4ccf-ac8b-dfcff7711b14,$(SP_PSA_ITS_EXTRA_FLAGS))) +$(eval $(call build-sp,protected-storage,config/$(SP_PSA_PS_CONFIG),751bf801-3dde-4768-a514-0f10aeed1790,$(SP_PSA_PS_EXTRA_FLAGS))) +$(eval $(call build-sp,crypto,config/$(SP_PSA_CRYPTO_CONFIG),d9df52d5-16a2-4bb2-9aa4-d26d3b84e8c0,$(SP_PSA_CRYPTO_EXTRA_FLAGS))) +ifeq ($(MEASURED_BOOT),y) +$(eval $(call build-sp,attestation,config/$(SP_PSA_ATTESTATION_CONFIG),a1baf155-8876-4695-8f7c-54955e8db974,$(SP_PSA_ATTESTATION_EXTRA_FLAGS))) +endif +ifeq ($(TS_SMM_GATEWAY),y) +$(eval $(call build-sp,smm-gateway,config/$(SP_SMM_GATEWAY_CONFIG),ed32d533-99e6-4209-9cc0-2d72cdd998a7,$(SP_SMM_GATEWAY_EXTRA_FLAGS))) +endif +else +# SPMC test SPs +OPTEE_OS_COMMON_EXTRA_FLAGS += CFG_SPMC_TESTS=y +$(eval $(call build-sp,spm-test1,opteesp,5c9edbc3-7b3a-4367-9f83-7c191ae86a37,$(SP_SPMC_TEST_EXTRA_FLAGS))) +$(eval $(call build-sp,spm-test2,opteesp,7817164c-c40c-4d1a-867a-9bb2278cf41a,$(SP_SPMC_TEST_EXTRA_FLAGS))) +$(eval $(call build-sp,spm-test3,opteesp,23eb0100-e32a-4497-9052-2f11e584afa6,$(SP_SPMC_TEST_EXTRA_FLAGS))) +$(eval $(call build-sp,spm-test4,opteesp,423762ed-7772-406f-99d8-0c27da0abbf8,$(SP_SPMC_TEST_EXTRA_FLAGS))) +endif + +# Linux user space applications +ifeq ($(SPMC_TESTS),n) +$(eval $(call build-ts-app,libts)) +$(eval $(call build-ts-app,ts-service-test)) +$(eval $(call build-ts-app,psa-api-test/internal_trusted_storage)) +$(eval $(call build-ts-app,psa-api-test/protected_storage)) +$(eval $(call build-ts-app,psa-api-test/crypto)) +ifeq ($(MEASURED_BOOT),y) +$(eval $(call build-ts-app,psa-api-test/initial_attestation)) +endif +ifeq ($(TS_UEFI_TESTS),y) +$(eval $(call build-ts-app,uefi-test)) + +# uefi-test uses MM Communicate via the arm-ffa-user driver and the message +# payload is forwarded in a carveout memory area. Adding reserved-memory node to +# the device tree to prevent Linux from using the carveout area for other +# purposes. + +ORIGINAL_DTB := $(FVP_LINUX_DTB) +CARVEOUT_ENTRY = $(ROOT)/build/fvp/mm_communicate_carveout.dtsi +FVP_LINUX_DTB = $(ROOT)/out/fvp_with_mm_carveout.dtb + +$(FVP_LINUX_DTB): $(CARVEOUT_ENTRY) | linux + { dtc -Idtb -Odts $(ORIGINAL_DTB); cat $(CARVEOUT_ENTRY); } | dtc -Idts -Odtb -o $(FVP_LINUX_DTB) + +boot-img: $(FVP_LINUX_DTB) + +.PHONY: carveout-dtb-clean +carveout-dtb-clean: + rm -f $(FVP_LINUX_DTB) + +boot-img-clean: carveout-dtb-clean +endif +endif diff --git a/build/fvp.mk b/build/fvp.mk new file mode 100644 index 0000000..2f1f658 --- /dev/null +++ b/build/fvp.mk @@ -0,0 +1,340 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +COMPILE_NS_USER ?= 64 +override COMPILE_NS_KERNEL := 64 +COMPILE_S_USER ?= 64 +COMPILE_S_KERNEL ?= 64 + +OPTEE_OS_PLATFORM = vexpress-fvp + +include common.mk + +################################################################################ +# Variables used for TPM configuration. +################################################################################ +BR2_ROOTFS_OVERLAY = $(ROOT)/build/br-ext/board/fvp/overlay +BR2_PACKAGE_FTPM_OPTEE_EXT_SITE ?= $(CURDIR)/br-ext/package/ftpm_optee_ext +BR2_PACKAGE_FTPM_OPTEE_PACKAGE_SITE ?= $(ROOT)/ms-tpm-20-ref + +# The fTPM implementation is based on ARM32 architecture whereas the rest of the +# system is built to run on 64-bit mode (COMPILE_S_USER = 64). Therefore set +# BR2_PACKAGE_FTPM_OPTEE_EXT_SDK manually to the arm32 OPTEE toolkit rather than +# relying on OPTEE_OS_TA_DEV_KIT_DIR variable. +BR2_PACKAGE_FTPM_OPTEE_EXT_SDK ?= $(OPTEE_OS_PATH)/out/arm/export-ta_arm32 + +BR2_PACKAGE_LINUX_FTPM_MOD_EXT_SITE ?= $(CURDIR)/br-ext/package/linux_ftpm_mod_ext +BR2_PACKAGE_LINUX_FTPM_MOD_EXT_PATH ?= $(LINUX_PATH) + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +MEASURED_BOOT ?= n +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +ifeq ($(MEASURED_BOOT),y) +# Prefer release mode for TF-A if using Measured Boot, debug may exhaust memory. +TF_A_BUILD ?= release +endif +ifeq ($(DEBUG),1) +TF_A_BUILD ?= debug +else +TF_A_BUILD ?= release +endif +EDK2_PATH ?= $(ROOT)/edk2 +EDK2_PLATFORMS_PATH ?= $(ROOT)/edk2-platforms +EDK2_TOOLCHAIN ?= GCC49 +EDK2_ARCH ?= AARCH64 +ifeq ($(DEBUG),1) +EDK2_BUILD ?= DEBUG +else +EDK2_BUILD ?= RELEASE +endif +EDK2_BIN ?= $(EDK2_PLATFORMS_PATH)/Build/ArmVExpress-FVP-AArch64/$(EDK2_BUILD)_$(EDK2_TOOLCHAIN)/FV/FVP_$(EDK2_ARCH)_EFI.fd +FVP_USE_BASE_PLAT ?= n +ifeq ($(FVP_USE_BASE_PLAT),y) +FVP_PATH ?= $(ROOT)/Base_RevC_AEMvA_pkg/models/Linux64_GCC-9.3 +FVP_BIN ?= FVP_Base_RevC-2xAEMvA +FVP_LINUX_DTB ?= $(LINUX_PATH)/arch/arm64/boot/dts/arm/fvp-base-revc.dtb +else +FVP_PATH ?= $(ROOT)/Foundation_Platformpkg/models/Linux64_GCC-9.3 +FVP_BIN ?= Foundation_Platform +FVP_LINUX_DTB ?= $(LINUX_PATH)/arch/arm64/boot/dts/arm/foundation-v8-gicv3-psci.dtb +endif +ifeq ($(wildcard $(FVP_PATH)),) +$(error $(FVP_PATH) does not exist) +endif +GRUB_PATH ?= $(ROOT)/grub +GRUB_CONFIG_PATH ?= $(BUILD_PATH)/fvp/grub +OUT_PATH ?= $(ROOT)/out +GRUB_BIN ?= $(OUT_PATH)/bootaa64.efi +BOOT_IMG ?= $(OUT_PATH)/boot-fat.uefi.img +FTPM_PATH ?= $(ROOT)/ms-tpm-20-ref/Samples/ARM32-FirmwareTPM/optee_ta + +ifeq ($(MEASURED_BOOT),y) +# By default enable FTPM for backwards compatibility. +MEASURED_BOOT_FTPM ?= y +else +$(call force,MEASURED_BOOT_FTPM,n,requires MEASURED_BOOT enabled) +endif + +# Build ancillary components to access fTPM if Measured Boot is enabled. +ifeq ($(MEASURED_BOOT_FTPM),y) +DEFCONFIG_FTPM ?= --br-defconfig build/br-ext/configs/ftpm_optee +DEFCONFIG_TPM_MODULE ?= --br-defconfig build/br-ext/configs/linux_ftpm +DEFCONFIG_TSS ?= --br-defconfig build/br-ext/configs/tss +endif + +################################################################################ +# Targets +################################################################################ +all: arm-tf optee-os ftpm boot-img linux edk2 +clean: arm-tf-clean boot-img-clean buildroot-clean edk2-clean grub-clean \ + ftpm-clean optee-os-clean + +include toolchain.mk + +################################################################################ +# Folders +################################################################################ +$(OUT_PATH): + mkdir -p $@ + +################################################################################ +# Shared folder +################################################################################ +# Enable accessing the host directory FVP_VIRTFS_HOST_DIR from the FVP. +# The shared folder can be mounted in the following ways: +# - Run 'mount -t 9p -o trans=virtio,version=9p2000.L FM ' or, +# - enable FVP_VIRTFS_AUTOMOUNT. +# The latter will use the Buildroot post-build script to add an entry to the +# target's /etc/fstab, mounting the shared directory to FVP_VIRTFS_MOUNTPOINT +# on the FVP. +# Note: the post-build script can only append to fstab. If FVP_VIRTFS_AUTOMOUNT +# is changed from "y" to "n", run 'rm -r ../out-br/build/skeleton-init-sysv' so +# the target's fstab will be replaced with the unmodified original again. +FVP_VIRTFS_ENABLE ?= n +FVP_VIRTFS_HOST_DIR ?= $(ROOT) +FVP_VIRTFS_AUTOMOUNT ?= n +FVP_VIRTFS_MOUNTPOINT ?= /mnt/host + +ifeq ($(FVP_VIRTFS_AUTOMOUNT),y) +$(call force,FVP_VIRTFS_ENABLE,y,required by FVP_VIRTFS_AUTOMOUNT) +endif + +ifneq ($(FVP_USE_BASE_PLAT),y) +$(call force,FVP_VIRTFS_ENABLE,n,only supported on FVP Base Platform) +endif + +BR2_ROOTFS_POST_BUILD_SCRIPT = $(ROOT)/build/br-ext/board/fvp/post-build.sh +BR2_ROOTFS_POST_SCRIPT_ARGS = "$(FVP_VIRTFS_AUTOMOUNT) $(FVP_VIRTFS_MOUNTPOINT)" + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +TF_A_FLAGS ?= \ + BL32=$(OPTEE_OS_HEADER_V2_BIN) \ + BL32_EXTRA1=$(OPTEE_OS_PAGER_V2_BIN) \ + BL32_EXTRA2=$(OPTEE_OS_PAGEABLE_V2_BIN) \ + BL33=$(EDK2_BIN) \ + ARM_TSP_RAM_LOCATION=tdram \ + FVP_USE_GIC_DRIVER=FVP_GICV3 \ + PLAT=fvp \ + SPD=opteed + +ifneq ($(MEASURED_BOOT),y) + TF_A_FLAGS += DEBUG=$(DEBUG) \ + MEASURED_BOOT=0 +else + TF_A_FLAGS += DEBUG=0 \ + MBEDTLS_DIR=$(ROOT)/mbedtls \ + ARM_ROTPK_LOCATION=devel_rsa \ + GENERATE_COT=1 \ + MEASURED_BOOT=1 \ + ROT_KEY=plat/arm/board/common/rotpk/arm_rotprivk_rsa.pem \ + TPM_HASH_ALG=sha256 \ + TRUSTED_BOARD_BOOT=1 \ + EVENT_LOG_LEVEL=20 +endif + +arm-tf: optee-os edk2 + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + +arm-tf-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# EDK2 / Tianocore +################################################################################ +define edk2-env + export WORKSPACE=$(EDK2_PLATFORMS_PATH) +endef + +define edk2-call + $(EDK2_TOOLCHAIN)_$(EDK2_ARCH)_PREFIX=$(AARCH64_CROSS_COMPILE) \ + build -n `getconf _NPROCESSORS_ONLN` -a $(EDK2_ARCH) \ + -t $(EDK2_TOOLCHAIN) -p Platform/ARM/VExpressPkg/ArmVExpress-FVP-AArch64.dsc -b $(EDK2_BUILD) +endef + +edk2: edk2-common + +edk2-clean: edk2-clean-common + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES ?= \ + $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/fvp.conf + +.PHONY: linux-ftpm-module +linux-ftpm-module: linux +ifeq ($(MEASURED_BOOT_FTPM),y) +linux-ftpm-module: + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) M=drivers/char/tpm \ + modules_install INSTALL_MOD_PATH=$(LINUX_PATH) +endif + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 + +linux: linux-common + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_COMMON_FLAGS += CFG_ARM_GICV3=y + +ifeq ($(MEASURED_BOOT),y) + OPTEE_OS_COMMON_FLAGS += CFG_DT=y CFG_CORE_TPM_EVENT_LOG=y +endif + +optee-os: optee-os-common + +optee-os-clean: ftpm-clean optee-os-clean-common + +################################################################################ +# Buildroot +################################################################################ + +buildroot: linux-ftpm-module + +################################################################################ +# grub +################################################################################ +grub-flags := CC="$(CCACHE)gcc" \ + TARGET_CC="$(AARCH64_CROSS_COMPILE)gcc" \ + TARGET_OBJCOPY="$(AARCH64_CROSS_COMPILE)objcopy" \ + TARGET_NM="$(AARCH64_CROSS_COMPILE)nm" \ + TARGET_RANLIB="$(AARCH64_CROSS_COMPILE)ranlib" \ + TARGET_STRIP="$(AARCH64_CROSS_COMPILE)strip" \ + --disable-werror + +GRUB_MODULES += boot chain configfile echo efinet eval ext2 fat font gettext \ + gfxterm gzio help linux loadenv lsefi normal part_gpt \ + part_msdos read regexp search search_fs_file search_fs_uuid \ + search_label terminal terminfo test tftp time + +$(GRUB_PATH)/configure: $(GRUB_PATH)/configure.ac + cd $(GRUB_PATH) && ./autogen.sh + +$(GRUB_PATH)/Makefile: $(GRUB_PATH)/configure + cd $(GRUB_PATH) && ./configure --target=aarch64 --enable-boot-time $(grub-flags) + +.PHONY: grub +grub: $(GRUB_PATH)/Makefile | $(OUT_PATH) + $(MAKE) -C $(GRUB_PATH) && \ + cd $(GRUB_PATH) && ./grub-mkimage \ + --output=$(GRUB_BIN) \ + --config=$(GRUB_CONFIG_PATH)/grub.cfg \ + --format=arm64-efi \ + --directory=grub-core \ + --prefix=/boot/grub \ + $(GRUB_MODULES) + +.PHONY: grub-clean +grub-clean: + @if [ -e $(GRUB_PATH)/Makefile ]; then $(MAKE) -C $(GRUB_PATH) clean; fi + @rm -f $(GRUB_BIN) + @rm -f $(GRUB_PATH)/configure + + +################################################################################ +# Boot Image +################################################################################ + +.PHONY: boot-img +boot-img: grub buildroot + rm -f $(BOOT_IMG) + mformat -i $(BOOT_IMG) -n 64 -h 255 -T 131072 -v "BOOT IMG" -C :: + mcopy -i $(BOOT_IMG) $(LINUX_PATH)/arch/arm64/boot/Image :: + mcopy -i $(BOOT_IMG) $(FVP_LINUX_DTB) ::/fvp.dtb + mmd -i $(BOOT_IMG) ::/EFI + mmd -i $(BOOT_IMG) ::/EFI/BOOT + mcopy -i $(BOOT_IMG) $(ROOT)/out-br/images/rootfs.cpio.gz ::/initrd.img + mcopy -i $(BOOT_IMG) $(GRUB_BIN) ::/EFI/BOOT/bootaa64.efi + mcopy -i $(BOOT_IMG) $(GRUB_CONFIG_PATH)/grub.cfg ::/EFI/BOOT/grub.cfg + +.PHONY: boot-img-clean +boot-img-clean: + rm -f $(BOOT_IMG) + +################################################################################ +# Run targets +################################################################################ +# This target enforces updating root fs etc +run: all + $(MAKE) run-only + +ifeq ($(FVP_USE_BASE_PLAT),y) +FVP_ARGS ?= \ + -C bp.ve_sysregs.exit_on_shutdown=1 \ + -C cache_state_modelled=0 \ + -C pctl.startup=0.0.0.0 \ + -C cluster0.NUM_CORES=4 \ + -C cluster1.NUM_CORES=4 \ + -C cluster0.cpu0.enable_crc32=1 \ + -C cluster0.cpu1.enable_crc32=1 \ + -C cluster0.cpu2.enable_crc32=1 \ + -C cluster0.cpu3.enable_crc32=1 \ + -C cluster1.cpu0.enable_crc32=1 \ + -C cluster1.cpu1.enable_crc32=1 \ + -C cluster1.cpu2.enable_crc32=1 \ + -C cluster1.cpu3.enable_crc32=1 \ + -C bp.secure_memory=1 \ + -C bp.secureflashloader.fname=$(TF_A_PATH)/build/fvp/$(TF_A_BUILD)/bl1.bin \ + -C bp.flashloader0.fname=$(TF_A_PATH)/build/fvp/$(TF_A_BUILD)/fip.bin \ + -C bp.virtioblockdevice.image_path=$(BOOT_IMG) +ifeq ($(FVP_VIRTFS_ENABLE),y) + FVP_ARGS += -C bp.virtiop9device.root_path=$(FVP_VIRTFS_HOST_DIR) +endif +else +FVP_ARGS ?= \ + --arm-v8.0 \ + --cores=4 \ + --secure-memory \ + --visualization \ + --gicv3 \ + --data="$(TF_A_PATH)/build/fvp/$(TF_A_BUILD)/bl1.bin"@0x0 \ + --data="$(TF_A_PATH)/build/fvp/$(TF_A_BUILD)/fip.bin"@0x8000000 \ + --block-device=$(BOOT_IMG) +endif + +run-only: + $(FVP_PATH)/$(FVP_BIN) $(FVP_ARGS) $(FVP_EXTRA_ARGS) diff --git a/build/fvp/bl2_sp_list.dtsi b/build/fvp/bl2_sp_list.dtsi new file mode 100644 index 0000000..597dcb8 --- /dev/null +++ b/build/fvp/bl2_sp_list.dtsi @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2022-2023, Arm Limited. All rights reserved. + */ + +#if !SPMC_TESTS +block_storage { + uuid = "63646e80-eb52-462f-ac4f-8cdf3987519c"; + load-address = <0x7a00000>; +}; + +internal_trusted_storage { + uuid = "dc1eef48-b17a-4ccf-ac8b-dfcff7711b14"; + load-address = <0x7a80000>; +}; + +protected_storage_sp { + uuid = "751bf801-3dde-4768-a514-0f10aeed1790"; + load-address = <0x7b00000>; +}; + +crypto_sp { + uuid = "d9df52d5-16a2-4bb2-9aa4-d26d3b84e8c0"; + load-address = <0x7b80000>; +}; + +#if MEASURED_BOOT +initial_attestation_sp { + uuid = "a1baf155-8876-4695-8f7c-54955e8db974"; + load-address = <0x7c80000>; +}; +#endif /* MEASURED_BOOT */ + +#if TS_SMM_GATEWAY +smm_gateway { + uuid = "ed32d533-99e6-4209-9cc0-2d72cdd998a7"; + load-address = <0x7d00000>; +}; +#endif /* TS_SMM_GATEWAY */ + +#else /* SPMC_TESTS */ + +test_sp1 { + uuid = "5c9edbc3-7b3a-4367-9f83-7c191ae86a37"; + load-address = <0x7a00000>; +}; + +test_sp2 { + uuid = "7817164c-c40c-4d1a-867a-9bb2278cf41a"; + load-address = <0x7a20000>; +}; + +test_sp3 { + uuid = "23eb0100-e32a-4497-9052-2f11e584afa6"; + load-address = <0x7a40000>; +}; + +test_sp4 { + uuid = "423762ed-7772-406f-99d8-0c27da0abbf8"; + load-address = <0x7a80000>; +}; + +#endif /* SPMC_TESTS */ diff --git a/build/fvp/grub/grub.cfg b/build/fvp/grub/grub.cfg new file mode 100644 index 0000000..4f399f7 --- /dev/null +++ b/build/fvp/grub/grub.cfg @@ -0,0 +1,10 @@ +set prefix='/EFI/BOOT' + +set default="0" +set timeout=10 + +menuentry 'GNU/Linux (OP-TEE)' { + linux /Image console=tty0 console=ttyAMA0,115200 earlycon=pl011,0x1c090000 root=/dev/disk/by-partlabel/system rootwait rw ignore_loglevel efi=noruntime + initrd /initrd.img + devicetree /fvp.dtb +} diff --git a/build/fvp/mm_communicate_carveout.dtsi b/build/fvp/mm_communicate_carveout.dtsi new file mode 100644 index 0000000..3472aa7 --- /dev/null +++ b/build/fvp/mm_communicate_carveout.dtsi @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + */ + +/ { + reserved-memory { + mm_communicate_carveout@881000000 { + reg = <0x00000008 0x81000000 0 0x00008000>; + }; + }; +}; diff --git a/build/fvp/spmc_manifest.dts b/build/fvp/spmc_manifest.dts new file mode 100644 index 0000000..d88b473 --- /dev/null +++ b/build/fvp/spmc_manifest.dts @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2022-2023, Arm Limited. All rights reserved. + */ + +/dts-v1/; + +/ { + compatible = "arm,ffa-core-manifest-1.0"; + #address-cells = <2>; + #size-cells = <1>; + + attribute { + spmc_id = <0x8000>; + maj_ver = <0x1>; + min_ver = <0x0>; + exec_state = <0x0>; + load_address = <0x0 0x6000000>; + entrypoint = <0x0 0x6000000>; + binary_size = <0x80000>; + }; + +/* + * This file will be preprocessed by TF-A's build system. If Measured Boot is + * enabled in TF-A's config, the build system will add the MEASURED_BOOT=1 macro + * to the preprocessor arguments. + */ +#if MEASURED_BOOT + tpm_event_log { + compatible = "arm,tpm_event_log"; + tpm_event_log_addr = <0x0 0x0>; + tpm_event_log_size = <0x0>; + tpm_event_log_max_size = <0x0>; + }; +#endif + +/* If the ARM_BL2_SP_LIST_DTS is defined, SPs should be loaded from FIP */ +#ifdef ARM_BL2_SP_LIST_DTS + sp_packages { + compatible = "arm,sp_pkg"; +#if !SPMC_TESTS + block_storage { + uuid = <0x806e6463 0x2f4652eb 0xdf8c4fac 0x9c518739>; + load-address = <0x0 0x7a00000>; + }; + internal_trusted_storage { + uuid = <0x48ef1edc 0xcf4c7ab1 0xcfdf8bac 0x141b71f7>; + load-address = <0x0 0x7a80000>; + }; + + protected_storage_sp { + uuid = <0x01f81b75 0x6847de3d 0x100f14a5 0x9017edae>; + load-address = <0x0 0x7b00000>; + }; + + crypto_sp { + uuid = <0xd552dfd9 0xb24ba216 0x6dd2a49a 0xc0e8843b>; + load-address = <0x0 0x7b80000>; + }; + +#if MEASURED_BOOT + initial_attestation_sp { + uuid = <0x55f1baa1 0x95467688 0x95547c8f 0x74b98d5e>; + load-address = <0x0 0x7c80000>; + }; +#endif + +#if TS_SMM_GATEWAY + smm_gateway { + uuid = <0x33d532ed 0x0942e699 0x722dc09c 0xa798d9cd>; + load-address = <0x0 0x7d00000>; + }; +#endif /* TS_SMM_GATEWAY */ + +#else /* SPMC_TESTS */ + test_sp1 { + uuid = <0xc3db9e5c 0x67433a7b 0x197c839f 0x376ae81a>; + load-address = <0x0 0x7a00000>; + }; + + test_sp2 { + uuid = <0x4c161778 0x1a4d0cc4 0xb29b7a86 0x1af48c27>; + load-address = <0x0 0x7a20000>; + }; + + test_sp3 { + uuid = <0x0001eb23 0x97442ae3 0x112f5290 0xa6af84e5>; + load-address = <0x0 0x7a40000>; + }; + + test_sp4 { + /* SP binary UUID */ + uuid = <0xed623742 0x6f407277 0x270cd899 0xf8bb0ada>; + load-address = <0x0 0x7a80000>; + }; +#endif /* SPMC_TESTS */ + + }; +#endif /* ARM_BL2_SP_LIST_DTS */ +}; diff --git a/build/get_clang.sh b/build/get_clang.sh new file mode 100755 index 0000000..06329da --- /dev/null +++ b/build/get_clang.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Download and extract Clang from the GitHub release page. +# We want a x86_64 cross-compiler capable of generating aarch64 and armv7a code +# *and* we want the compiler-rt libraries for these architectures +# (libclang_rt.*.a). +# Clang is configured to be able to cross-compile to all the supported +# architectures by default (see /bin/llc --version) which is great, +# but compiler-rt is included only for the host architecture. Therefore we need +# to combine several packages into one, which is the purpose of this script. + +[ "$1" ] || { echo "Usage: get_clang.sh version [path]"; exit 1; } + +VER=${1} +DEST=${2:-./clang-${VER}} +X86_64=clang+llvm-${VER}-x86_64-linux-gnu-ubuntu-16.04 +AARCH64=clang+llvm-${VER}-aarch64-linux-gnu +ARMV7A=clang+llvm-${VER}-armv7a-linux-gnueabihf + +set -x + +TMPDEST=${DEST}_tmp${RANDOM} + +if [ -e ${TMPDEST} ]; then + echo Error: ${TMPDEST} exists + exit 1 +fi + +function cleanup() { + rm -f ${X86_64}.tar.xz ${AARCH64}.tar.xz ${ARMV7A}.tar.xz + rm -rf ${AARCH64} ${ARMV7A} +} + +trap "{ exit 2; }" INT +trap cleanup EXIT + +(wget -nv https://github.com/llvm/llvm-project/releases/download/llvmorg-${VER}/${X86_64}.tar.xz && tar xf ${X86_64}.tar.xz) & +pids=$! +(wget -nv https://github.com/llvm/llvm-project/releases/download/llvmorg-${VER}/${AARCH64}.tar.xz && tar xf ${AARCH64}.tar.xz) & +pids="$pids $!" +(wget -nv https://github.com/llvm/llvm-project/releases/download/llvmorg-${VER}/${ARMV7A}.tar.xz && tar xf ${ARMV7A}.tar.xz) & +pids="$pids $!" + +wait $pids || exit 1 + +mv ${X86_64} ${TMPDEST} || exit 1 +cp ${AARCH64}/lib/clang/${VER}/lib/linux/* ${TMPDEST}/lib/clang/${VER}/lib/linux || exit 1 +cp ${ARMV7A}/lib/clang/${VER}/lib/linux/* ${TMPDEST}/lib/clang/${VER}/lib/linux || exit 1 +mv ${TMPDEST} ${DEST} diff --git a/build/hikey.mk b/build/hikey.mk new file mode 100644 index 0000000..4c74a82 --- /dev/null +++ b/build/hikey.mk @@ -0,0 +1,380 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +COMPILE_NS_USER ?= 64 +override COMPILE_NS_KERNEL := 64 +COMPILE_S_USER ?= 32 +COMPILE_S_KERNEL ?= 64 + +# Normal/secure world console UARTs: 3, 2, or 0 [default 3] +CFG_NW_CONSOLE_UART ?= 3 +CFG_SW_CONSOLE_UART ?= 3 + +# eMMC flash size: 8 or 4 GB [default 8] +CFG_FLASH_SIZE ?= 8 + +OPTEE_OS_PLATFORM = hikey + +################################################################################ +# Includes +################################################################################ +include common.mk + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +ATF_FB_PATH ?= $(ROOT)/atf-fastboot +ifeq ($(DEBUG),1) +TF_A_BUILD ?= debug +ATF_FB_BUILD ?= debug +else +TF_A_BUILD ?= release +ATF_FB_BUILD ?= release +endif + +EDK2_PATH ?= $(ROOT)/edk2 +ifeq ($(DEBUG),1) +EDK2_BUILD ?= DEBUG +else +EDK2_BUILD ?= RELEASE +endif +EDK2_BIN ?= $(EDK2_PATH)/Build/HiKey/$(EDK2_BUILD)_$(EDK2_TOOLCHAIN)/FV/BL33_AP_UEFI.fd +OPENPLATPKG_PATH ?= $(ROOT)/OpenPlatformPkg + +OUT_PATH ?=$(ROOT)/out +MCUIMAGE_BIN ?= $(OPENPLATPKG_PATH)/Platforms/Hisilicon/HiKey/Binary/mcuimage.bin +BOOT_IMG ?=$(ROOT)/out/boot-fat.uefi.img +NVME_IMG ?=$(ROOT)/out/nvme.img +GRUB_PATH ?=$(ROOT)/grub +LLOADER_PATH ?=$(ROOT)/l-loader +PATCHES_PATH ?=$(ROOT)/patches_hikey + +################################################################################ +# Targets +################################################################################ +.PHONY: all +all: prepare arm-tf boot-img lloader nvme + +.PHONY: clean +clean: arm-tf-clean atf-fb-clean buildroot-clean edk2-clean linux-clean \ + optee-os-clean boot-img-clean lloader-clean grub-clean + +.PHONY: cleaner +cleaner: clean prepare-cleaner buildroot-cleaner linux-cleaner \ + nvme-cleaner grub-cleaner + +include toolchain.mk + +.PHONY: prepare +prepare: + mkdir -p $(OUT_PATH) + +.PHONY: prepare-cleaner +prepare-cleaner: + rm -rf $(ROOT)/out + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +TF_A_FLAGS ?= \ + BL32=$(OPTEE_OS_HEADER_V2_BIN) \ + BL32_EXTRA1=$(OPTEE_OS_PAGER_V2_BIN) \ + BL32_EXTRA2=$(OPTEE_OS_PAGEABLE_V2_BIN) \ + BL33=$(EDK2_BIN) \ + SCP_BL2=$(MCUIMAGE_BIN) \ + DEBUG=$(DEBUG) \ + PLAT=hikey \ + SPD=opteed + +TF_A_CONSOLE_UART ?= $(CFG_SW_CONSOLE_UART) +ifeq ($(TF_A_CONSOLE_UART),0) + TF_A_FLAGS += CONSOLE_BASE=PL011_UART0_BASE \ + CRASH_CONSOLE_BASE=PL011_UART0_BASE +endif +ifeq ($(TF_A_CONSOLE_UART),2) + TF_A_FLAGS += CONSOLE_BASE=PL011_UART2_BASE \ + CRASH_CONSOLE_BASE=PL011_UART2_BASE +endif + +.PHONY: arm-tf +arm-tf: optee-os edk2 + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + +.PHONY: arm-tf-clean +arm-tf-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# EDK2 / Tianocore +################################################################################ +EDK2_ARCH ?= AARCH64 +EDK2_DSC ?= OpenPlatformPkg/Platforms/Hisilicon/HiKey/HiKey.dsc +EDK2_TOOLCHAIN ?= GCC49 + +EDK2_CONSOLE_UART ?= $(CFG_NW_CONSOLE_UART) +ifeq ($(EDK2_CONSOLE_UART),0) + EDK2_BUILDFLAGS += -DSERIAL_BASE=0xF8015000 +endif +ifeq ($(EDK2_CONSOLE_UART),2) + EDK2_BUILDFLAGS += -DSERIAL_BASE=0xF7112000 +endif + +define edk2-call + $(EDK2_TOOLCHAIN)_$(EDK2_ARCH)_PREFIX=$(AARCH64_CROSS_COMPILE) \ + build -n `getconf _NPROCESSORS_ONLN` -a $(EDK2_ARCH) \ + -t $(EDK2_TOOLCHAIN) -p $(EDK2_DSC) \ + -b $(EDK2_BUILD) $(EDK2_BUILDFLAGS) +endef + +.PHONY: edk2 +edk2: + sed -i 's/\(^DEFINE GCC_ALL_CC_FLAGS.*-Wno-array-bounds\) -include/\1 -Wno-stringop-overflow -include/' $(EDK2_PATH)/BaseTools/Conf/tools_def.template + cd $(EDK2_PATH) && rm -rf OpenPlatformPkg && \ + ln -s $(OPENPLATPKG_PATH) + set -e && cd $(EDK2_PATH) && source edksetup.sh && \ + $(MAKE) -j1 -C $(EDK2_PATH)/BaseTools \ + BUILD_CC="gcc $(call cc-option,gcc,-Wno-error=stringop-truncation,)" && \ + $(call edk2-call) + +.PHONY: edk2-clean +edk2-clean: + set -e && cd $(EDK2_PATH) && source edksetup.sh && \ + $(call edk2-call) cleanall && \ + $(MAKE) -j1 -C $(EDK2_PATH)/BaseTools clean + rm -rf $(EDK2_PATH)/Build + rm -rf $(EDK2_PATH)/Conf/.cache + rm -f $(EDK2_PATH)/Conf/build_rule.txt + rm -f $(EDK2_PATH)/Conf/target.txt + rm -f $(EDK2_PATH)/Conf/tools_def.txt + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH ?= arm64 +LINUX_DEFCONFIG_COMMON_FILES ?= $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/hikey.conf \ + $(PATCHES_PATH)/kernel_config/usb_net_dm9601.conf \ + $(PATCHES_PATH)/kernel_config/ftrace.conf + +.PHONY: linux-defconfig +linux-defconfig: $(LINUX_PATH)/.config + +.PHONY: linux-gen_init_cpio +linux-gen_init_cpio: linux-defconfig + $(MAKE) -C $(LINUX_PATH)/usr \ + CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) \ + ARCH=arm64 \ + LOCALVERSION= \ + gen_init_cpio + +LINUX_COMMON_FLAGS += ARCH=arm64 Image hisilicon/hi6220-hikey.dtb + +.PHONY: linux +linux: linux-common + +.PHONY: linux-defconfig-clean +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +.PHONY: linux-clean +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +.PHONY: linux-cleaner +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_COMMON_FLAGS += CFG_CONSOLE_UART=$(CFG_SW_CONSOLE_UART) + +.PHONY: optee-os +optee-os: optee-os-common + +.PHONY: optee-os-clean +optee-os-clean: optee-os-clean-common + +################################################################################ +# grub +################################################################################ +grub-flags := CC="$(CCACHE)gcc" \ + TARGET_CC="$(AARCH64_CROSS_COMPILE)gcc" \ + TARGET_OBJCOPY="$(AARCH64_CROSS_COMPILE)objcopy" \ + TARGET_NM="$(AARCH64_CROSS_COMPILE)nm" \ + TARGET_RANLIB="$(AARCH64_CROSS_COMPILE)ranlib" \ + TARGET_STRIP="$(AARCH64_CROSS_COMPILE)strip" \ + --disable-werror + +GRUB_MODULES += boot chain configfile efinet ext2 fat gettext \ + help linux loadenv lsefi normal part_gpt \ + part_msdos read search search_fs_file search_fs_uuid \ + search_label terminal terminfo tftp time + +$(GRUB_PATH)/configure: $(GRUB_PATH)/configure.ac + cd $(GRUB_PATH) && ./autogen.sh + +$(GRUB_PATH)/Makefile: $(GRUB_PATH)/configure + cd $(GRUB_PATH) && ./configure --target=aarch64 --enable-boot-time $(grub-flags) + +.PHONY: grub +grub: prepare $(GRUB_PATH)/Makefile + $(MAKE) -C $(GRUB_PATH); \ + cd $(GRUB_PATH) && ./grub-mkimage \ + --verbose \ + --output=$(OUT_PATH)/grubaa64.efi \ + --config=$(PATCHES_PATH)/grub/grub.configfile \ + --format=arm64-efi \ + --directory=grub-core \ + --prefix=/boot/grub \ + $(GRUB_MODULES) + +.PHONY: grub-clean +grub-clean: + if [ -e $(GRUB_PATH)/Makefile ]; then $(MAKE) -C $(GRUB_PATH) clean; fi + rm -f $(OUT_PATH)/grubaa64.efi + +.PHONY: grub-cleaner +grub-cleaner: grub-clean + if [ -e $(GRUB_PATH)/Makefile ]; then $(MAKE) -C $(GRUB_PATH) distclean; fi + rm -f $(GRUB_PATH)/configure + +################################################################################ +# Boot Image +################################################################################ +ifeq ($(CFG_NW_CONSOLE_UART),3) +GRUBCFG = $(PATCHES_PATH)/grub/grub_uart3.cfg +else ifeq ($(CFG_NW_CONSOLE_UART),2) +GRUBCFG = $(PATCHES_PATH)/grub/grub_uart2.cfg +else +GRUBCFG = $(PATCHES_PATH)/grub/grub_uart0.cfg +endif + +.PHONY: boot-img +boot-img: linux buildroot edk2 grub + rm -f $(BOOT_IMG) + mformat -i $(BOOT_IMG) -n 64 -h 255 -T 131072 -v "BOOT IMG" -C :: + mcopy -i $(BOOT_IMG) $(LINUX_PATH)/arch/arm64/boot/Image :: + mcopy -i $(BOOT_IMG) $(LINUX_PATH)/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dtb :: + mmd -i $(BOOT_IMG) ::/EFI + mmd -i $(BOOT_IMG) ::/EFI/BOOT + mcopy -i $(BOOT_IMG) $(OUT_PATH)/grubaa64.efi ::/EFI/BOOT/ + mcopy -i $(BOOT_IMG) $(GRUBCFG) ::/EFI/BOOT/grub.cfg + mcopy -i $(BOOT_IMG) $(ROOT)/out-br/images/rootfs.cpio.gz ::/initrd.img + mcopy -i $(BOOT_IMG) $(EDK2_PATH)/Build/HiKey/$(EDK2_BUILD)_$(EDK2_TOOLCHAIN)/$(EDK2_ARCH)/AndroidFastbootApp.efi ::/EFI/BOOT/fastboot.efi + +.PHONY: boot-img-clean +boot-img-clean: + rm -f $(BOOT_IMG) + +################################################################################ +# atf-fastboot +################################################################################ +ATF_FB_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +ATF_FB_FLAGS ?= \ + DEBUG=$(DEBUG) \ + PLAT=hikey + +.PHONY: atf-fb +atf-fb: + $(ATF_FB_EXPORTS) $(MAKE) -C $(ATF_FB_PATH) $(ATF_FB_FLAGS) + +.PHONY: atf-fb-clean +atf-fb-clean: + $(ATF_FB_EXPORTS) $(MAKE) -C $(ATF_FB_PATH) $(ATF_FB_FLAGS) clean + +################################################################################ +# l-loader +################################################################################ +.PHONY: lloader +lloader: arm-tf atf-fb + cd $(LLOADER_PATH) && \ + ln -sf $(TF_A_PATH)/build/hikey/$(TF_A_BUILD)/bl1.bin && \ + ln -sf $(TF_A_PATH)/build/hikey/$(TF_A_BUILD)/bl2.bin && \ + ln -sf $(ATF_FB_PATH)/build/hikey/$(ATF_FB_BUILD)/bl1.bin fastboot.bin && \ + $(MAKE) hikey PTABLE_LST="linux-$(CFG_FLASH_SIZE)g" CROSS_COMPILE="$(CCACHE)$(AARCH32_CROSS_COMPILE)" + +.PHONY: lloader-clean +lloader-clean: + $(MAKE) -C $(LLOADER_PATH) hikey-clean + +################################################################################ +# nvme image +# +# nvme partition is used to store UEFI non-volatile variables, +# but nvme image is currently an empty list +################################################################################ +.PHONY: nvme +nvme: prepare +ifeq ("$(wildcard $(NVME_IMG))","") + dd if=/dev/zero of=$(NVME_IMG) bs=1K count=128 +endif + +.PHONY: nvme-cleaner +nvme-cleaner: + rm -f $(NVME_IMG) + +################################################################################ +# Flash +################################################################################ +define flash_help + @read -r -p "1. Connect USB OTG cable, the micro USB cable (press enter)" dummy + @read -r -p "2. Connect HiKey to power up (press enter)" dummy +endef + +.PHONY: recovery +recovery: + @echo "Enter recovery mode to flash a new bootloader" + @echo + @echo "Make sure udev permissions are set appropriately:" + @echo " # /etc/udev/rules.d/hikey.rules" + @echo ' SUBSYSTEM=="usb", ATTRS{idVendor}=="18d1", ATTRS{idProduct}=="d00d", MODE="0666"' + @echo ' SUBSYSTEM=="usb", ATTRS{idVendor}=="12d1", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1"' + @echo + @echo "Set jumpers as follows:" + @echo "Jumper 1-2: Closed (Auto power up = Boot up when power is applied)" + @echo " 3-4: Closed (Boot Select = Recovery: program eMMC from USB OTG)" + @echo " 5-6: Open (GPIO3-1 = High: UEFI runs normally)" + @read -r -p "Press enter to continue" dummy + @echo + $(call flash_help) + @echo + $(ROOT)/burn-boot/hisi-idt.py --img1=$(LLOADER_PATH)/recovery.bin + fastboot getvar partition-size:ptable + fastboot flash loader $(LLOADER_PATH)/l-loader.bin + @echo + @echo "3. Wait until you see the (UART) message" + @echo " \"Enter fastboot mode...\"" + @$(MAKE) --no-print flash FROM_RECOVERY=1 + +.PHONY: flash +flash: +ifneq ($(FROM_RECOVERY),1) + @echo "Flash binaries using fastboot" + @echo + @echo "Set jumpers as follows:" + @echo "Jumper 1-2: Closed (Auto power up = Boot up when power is applied)" + @echo " 3-4: Open (Boot Select = Boot from eMMC)" + @echo " 5-6: Closed (GPIO3-1 = Low: UEFI runs Fastboot app)" + @read -r -p "Press enter to continue" dummy + @echo + $(call flash_help) + @echo "3. Wait until you see the (UART) message" + @echo " \"Android Fastboot mode - version x.x.\"" + @echo " Press RETURN or SPACE key to quit.\"" +endif + @read -r -p "Then press enter to continue flashing" dummy + @echo + fastboot flash ptable $(LLOADER_PATH)/ptable-linux-$(CFG_FLASH_SIZE)g.img + fastboot flash fastboot $(TF_A_PATH)/build/hikey/$(TF_A_BUILD)/fip.bin + fastboot flash nvme $(NVME_IMG) + fastboot flash boot $(BOOT_IMG) diff --git a/build/hikey960.mk b/build/hikey960.mk new file mode 100644 index 0000000..9fc8ba2 --- /dev/null +++ b/build/hikey960.mk @@ -0,0 +1,358 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +COMPILE_NS_USER ?= 64 +override COMPILE_NS_KERNEL := 64 +COMPILE_S_USER ?= 32 +COMPILE_S_KERNEL ?= 64 + +# Normal and secure world console UART: 6 (v2 or newer board) or 5 (v1 board) +CFG_CONSOLE_UART ?= 6 +# Needed by buildroot +CFG_NW_CONSOLE_UART ?= $(CFG_CONSOLE_UART) + +# To enable Wi-Fi on HiKey960: +# 1. Set the network name (SSID) and password in +# $(BR2_ROOTFS_OVERLAY)/etc/wpa_supplicant.conf before building +# 2. Boot the board, login as root and run: ifup wlan0 +BR2_PACKAGE_LINUX_FIRMWARE = y +BR2_PACKAGE_LINUX_FIRMWARE_TI_WL18XX = y +BR2_PACKAGE_WPA_SUPPLICANT = y +BR2_PACKAGE_WIRELESS_REGDB = y + +BR2_ROOTFS_OVERLAY = $(ROOT)/build/br-ext/board/hikey960/overlay + +OPTEE_OS_PLATFORM = hikey-hikey960 + +################################################################################ +# Includes +################################################################################ +include common.mk + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +ifeq ($(DEBUG),1) +TF_A_BUILD ?= debug +else +TF_A_BUILD ?= release +endif + +EDK2_PATH ?= $(ROOT)/edk2 +ifeq ($(DEBUG),1) +EDK2_BUILD ?= DEBUG +else +EDK2_BUILD ?= RELEASE +endif +EDK2_BIN ?= $(EDK2_PATH)/Build/HiKey960/$(EDK2_BUILD)_$(EDK2_TOOLCHAIN)/FV/BL33_AP_UEFI.fd +OPENPLATPKG_PATH ?= $(ROOT)/OpenPlatformPkg + +OUT_PATH ?=$(ROOT)/out +MCUIMAGE_BIN ?= $(OPENPLATPKG_PATH)/Platforms/Hisilicon/HiKey960/Binary/lpm3.img +BOOT_IMG ?=$(ROOT)/out/boot-fat.uefi.img +GRUB_PATH ?=$(ROOT)/grub +LLOADER_PATH ?=$(ROOT)/l-loader +IMAGE_TOOLS_PATH ?=$(ROOT)/tools-images-hikey960 +IMAGE_TOOLS_CONFIG ?=$(OUT_PATH)/config +PATCHES_PATH ?=$(ROOT)/patches_hikey + +################################################################################ +# Targets +################################################################################ +.PHONY: all +all: arm-tf boot-img lloader + +.PHONY: clean +clean: arm-tf-clean buildroot-clean edk2-clean linux-clean optee-os-clean \ + boot-img-clean lloader-clean grub-clean + +.PHONY: cleaner +cleaner: clean prepare-cleaner buildroot-cleaner linux-cleaner grub-cleaner + +include toolchain.mk + +.PHONY: prepare +prepare: + mkdir -p $(OUT_PATH) + +.PHONY: prepare-cleaner +prepare-cleaner: + rm -rf $(ROOT)/out + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +TF_A_FLAGS ?= \ + BL32=$(OPTEE_OS_HEADER_V2_BIN) \ + BL32_EXTRA1=$(OPTEE_OS_PAGER_V2_BIN) \ + BL32_EXTRA2=$(OPTEE_OS_PAGEABLE_V2_BIN) \ + BL33=$(EDK2_BIN) \ + SCP_BL2=$(MCUIMAGE_BIN) \ + DEBUG=$(DEBUG) \ + PLAT=hikey960 \ + SPD=opteed + +ifeq ($(CFG_CONSOLE_UART),5) + TF_A_FLAGS += CRASH_CONSOLE_BASE=PL011_UART5_BASE +endif + +.PHONY: arm-tf +arm-tf: optee-os edk2 + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + +.PHONY: arm-tf-clean +arm-tf-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# EDK2 / Tianocore +################################################################################ +EDK2_ARCH ?= AARCH64 +EDK2_DSC ?= OpenPlatformPkg/Platforms/Hisilicon/HiKey960/HiKey960.dsc +EDK2_TOOLCHAIN ?= GCC49 + +ifeq ($(CFG_CONSOLE_UART),5) + EDK2_BUILDFLAGS += -DSERIAL_BASE=0xFDF05000 +endif + +define edk2-call + $(EDK2_TOOLCHAIN)_$(EDK2_ARCH)_PREFIX=$(AARCH64_CROSS_COMPILE) \ + build -n `getconf _NPROCESSORS_ONLN` -a $(EDK2_ARCH) \ + -t $(EDK2_TOOLCHAIN) -p $(EDK2_DSC) \ + -b $(EDK2_BUILD) $(EDK2_BUILDFLAGS) +endef + +.PHONY: edk2 +edk2: + sed -i 's/\(^DEFINE GCC_ALL_CC_FLAGS.*-Wno-array-bounds\) -include/\1 -Wno-stringop-overflow -include/' $(EDK2_PATH)/BaseTools/Conf/tools_def.template + cd $(EDK2_PATH) && rm -rf OpenPlatformPkg && \ + ln -s $(OPENPLATPKG_PATH) + set -e && cd $(EDK2_PATH) && source edksetup.sh && \ + $(MAKE) -j1 -C $(EDK2_PATH)/BaseTools \ + BUILD_CC="gcc $(call cc-option,gcc,-Wno-error=stringop-truncation,)" && \ + $(call edk2-call) + +.PHONY: edk2-clean +edk2-clean: + set -e && cd $(EDK2_PATH) && source edksetup.sh && \ + $(call edk2-call) cleanall && \ + $(MAKE) -j1 -C $(EDK2_PATH)/BaseTools clean + rm -rf $(EDK2_PATH)/Build + rm -rf $(EDK2_PATH)/Conf/.cache + rm -f $(EDK2_PATH)/Conf/build_rule.txt + rm -f $(EDK2_PATH)/Conf/target.txt + rm -f $(EDK2_PATH)/Conf/tools_def.txt + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH ?= arm64 +LINUX_DEFCONFIG_COMMON_FILES ?= $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/hikey960.conf \ + $(PATCHES_PATH)/kernel_config/usb_net_dm9601.conf \ + $(PATCHES_PATH)/kernel_config/ftrace.conf + +.PHONY: linux-defconfig +linux-defconfig: $(LINUX_PATH)/.config + +.PHONY: linux-gen_init_cpio +linux-gen_init_cpio: linux-defconfig + $(MAKE) -C $(LINUX_PATH)/usr \ + CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) \ + ARCH=arm64 \ + LOCALVERSION= \ + gen_init_cpio + +LINUX_COMMON_FLAGS += ARCH=arm64 Image hisilicon/hi3660-hikey960.dtb + +.PHONY: linux +linux: linux-common + +.PHONY: linux-defconfig-clean +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +.PHONY: linux-clean +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +.PHONY: linux-cleaner +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_COMMON_FLAGS += CFG_CONSOLE_UART=$(CFG_CONSOLE_UART) + +.PHONY: optee-os +optee-os: optee-os-common + +.PHONY: optee-os-clean +optee-os-clean: optee-os-clean-common + +################################################################################ +# grub +################################################################################ +grub-flags := CC="$(CCACHE)gcc" \ + TARGET_CC="$(AARCH64_CROSS_COMPILE)gcc" \ + TARGET_OBJCOPY="$(AARCH64_CROSS_COMPILE)objcopy" \ + TARGET_NM="$(AARCH64_CROSS_COMPILE)nm" \ + TARGET_RANLIB="$(AARCH64_CROSS_COMPILE)ranlib" \ + TARGET_STRIP="$(AARCH64_CROSS_COMPILE)strip" \ + --disable-werror + +GRUB_MODULES += boot chain configfile echo efinet eval ext2 fat font gettext \ + gfxterm gzio help linux loadenv lsefi normal part_gpt \ + part_msdos read regexp search search_fs_file search_fs_uuid \ + search_label terminal terminfo test tftp time + +$(GRUB_PATH)/configure: $(GRUB_PATH)/configure.ac + cd $(GRUB_PATH) && ./autogen.sh + +$(GRUB_PATH)/Makefile: $(GRUB_PATH)/configure + cd $(GRUB_PATH) && ./configure --target=aarch64 --enable-boot-time $(grub-flags) + +.PHONY: grub +grub: prepare $(GRUB_PATH)/Makefile + $(MAKE) -C $(GRUB_PATH); \ + cd $(GRUB_PATH) && ./grub-mkimage \ + --verbose \ + --output=$(OUT_PATH)/grubaa64.efi \ + --config=$(PATCHES_PATH)/grub/grub.configfile \ + --format=arm64-efi \ + --directory=grub-core \ + --prefix=/boot/grub \ + $(GRUB_MODULES) + +.PHONY: grub-clean +grub-clean: + if [ -e $(GRUB_PATH)/Makefile ]; then $(MAKE) -C $(GRUB_PATH) clean; fi + rm -f $(OUT_PATH)/grubaa64.efi + +.PHONY: grub-cleaner +grub-cleaner: grub-clean + if [ -e $(GRUB_PATH)/Makefile ]; then $(MAKE) -C $(GRUB_PATH) distclean; fi + rm -f $(GRUB_PATH)/configure + +################################################################################ +# Boot Image +################################################################################ +ifeq ($(CFG_CONSOLE_UART),6) +GRUBCFG = $(PATCHES_PATH)/grub/grub_uart6.cfg +else +GRUBCFG = $(PATCHES_PATH)/grub/grub_uart5.cfg +endif + +.PHONY: boot-img +boot-img: linux buildroot edk2 grub + rm -f $(BOOT_IMG) + mformat -i $(BOOT_IMG) -n 64 -h 255 -T 131072 -v "BOOT IMG" -C :: + mcopy -i $(BOOT_IMG) $(LINUX_PATH)/arch/arm64/boot/Image :: + mcopy -i $(BOOT_IMG) $(LINUX_PATH)/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dtb :: + mmd -i $(BOOT_IMG) ::/EFI + mmd -i $(BOOT_IMG) ::/EFI/BOOT + mcopy -i $(BOOT_IMG) $(OUT_PATH)/grubaa64.efi ::/EFI/BOOT/ + mcopy -i $(BOOT_IMG) $(GRUBCFG) ::/EFI/BOOT/grub.cfg + mcopy -i $(BOOT_IMG) $(ROOT)/out-br/images/rootfs.cpio.gz ::/initrd.img + mcopy -i $(BOOT_IMG) $(EDK2_PATH)/Build/HiKey960/$(EDK2_BUILD)_$(EDK2_TOOLCHAIN)/$(EDK2_ARCH)/AndroidFastbootApp.efi ::/EFI/BOOT/fastboot.efi + +.PHONY: boot-img-clean +boot-img-clean: + rm -f $(BOOT_IMG) + +################################################################################ +# l-loader +################################################################################ +.PHONY: lloader +lloader: arm-tf edk2 + cd $(LLOADER_PATH) && \ + ln -sf $(TF_A_PATH)/build/hikey960/$(TF_A_BUILD)/bl1.bin && \ + ln -sf $(TF_A_PATH)/build/hikey960/$(TF_A_BUILD)/bl2.bin && \ + ln -sf $(EDK2_BIN) && \ + $(MAKE) hikey960 PTABLE_LST=linux-32g + +.PHONY: lloader-clean +lloader-clean: + $(MAKE) -C $(LLOADER_PATH) hikey960-clean + +################################################################################ +# Flash +################################################################################ +define flash_help + @read -r -p "Connect HiKey960 to power up (press enter)" dummy + @read -r -p "Connect USB OTG cable, the micro USB cable (press enter)" dummy +endef + +.PHONY: recov_cfg +recov_cfg: + @echo "./hisi-sec_usb_xloader.img 0x00020000" > $(IMAGE_TOOLS_CONFIG) + @echo "./hisi-sec_uce_boot.img 0x6A908000" >> $(IMAGE_TOOLS_CONFIG) + @echo "./recovery.bin 0x1AC00000" >> $(IMAGE_TOOLS_CONFIG) + +.PHONY: recovery +recovery: recov_cfg + @echo "Enter recovery mode to flash a new bootloader" + @echo + @echo "Make sure udev permissions are set appropriately:" + @echo " # /etc/udev/rules.d/hikey960.rules" + @echo ' SUBSYSTEM=="usb", ATTRS{idVendor}=="18d1", ATTRS{idProduct}=="d00d", MODE="0666"' + @echo ' SUBSYSTEM=="usb", ATTRS{idVendor}=="12d1", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1"' + @echo + @echo "Set jumpers or switches as follows:" + @echo "Jumper 1-2: Closed or Switch 1: On" + @echo " 3-4: Closed or 2: On" + @echo " 5-6: Open or 3: Off" + @read -r -p "Press enter to continue" dummy + @echo + $(call flash_help) + @echo + @echo "Check the device node (/dev/ttyUSBx) of the micro USB connection" + @echo "Note the value x of the device node. Default is 1" + @read -r -p "Enter the device node. Press enter for /dev/ttyUSB1: " DEV && \ + DEV=$${DEV:-/dev/ttyUSB1} && \ + cd $(IMAGE_TOOLS_PATH) && \ + ln -sf $(LLOADER_PATH)/recovery.bin && \ + sudo ./hikey_idt -c $(IMAGE_TOOLS_CONFIG) -p $$DEV && \ + rm -f $(IMAGE_TOOLS_CONFIG) + @echo + @echo "If you see dots starting to appear on the console," + @echo "press f ON THE CONSOLE (NOT HERE!) to run fastboot." + @echo "You have 10 seconds! Go!" + @echo "If not, fastboot should load automatically." + @read -r -p "Press enter (HERE) to continue flashing" dummy + @$(MAKE) --no-print flash FROM_RECOVERY=1 + +.PHONY: flash +flash: +ifneq ($(FROM_RECOVERY),1) + @echo "Flash binaries using fastboot" + @echo + @echo "Set jumpers or switches as follows:" + @echo "Jumper 1-2: Closed or Switch 1: On" + @echo " 3-4: Open or 2: Off" + @echo " 5-6: Closed or 3: On" + @read -r -p "Press enter to continue" dummy + @echo + $(call flash_help) +endif + @echo "Wait until you see the (UART) message" + @echo " \"Android Fastboot mode - version x.x.\"" + @echo " Press RETURN or SPACE key to quit.\"" + @echo "It can take a while for the fastboot device to come up," + @echo "so please wait ~10 seconds." + @read -r -p "Then press enter to continue flashing" dummy + @echo + fastboot flash ptable $(LLOADER_PATH)/prm_ptable.img + fastboot flash xloader $(IMAGE_TOOLS_PATH)/hisi-sec_xloader.img + fastboot flash fastboot $(LLOADER_PATH)/l-loader.bin + fastboot flash fip $(TF_A_PATH)/build/hikey960/$(TF_A_BUILD)/fip.bin + fastboot flash nvme $(IMAGE_TOOLS_PATH)/hisi-nvme.img + fastboot flash boot $(BOOT_IMG) diff --git a/build/imx.mk b/build/imx.mk new file mode 100644 index 0000000..a5e76fc --- /dev/null +++ b/build/imx.mk @@ -0,0 +1,241 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +override COMPILE_NS_USER := 64 +override COMPILE_NS_KERNEL := 64 +override COMPILE_S_USER := 64 +override COMPILE_S_KERNEL := 64 + +OPTEE_OS_PLATFORM = imx-mx8mqevk +BR2_TARGET_GENERIC_GETTY_PORT = ttymxc0 +BR2_TARGET_ROOTFS_EXT2=y +BR2_TARGET_ROOTFS_EXT2_4=y + +include common.mk + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a + +U-BOOT_PATH ?= $(ROOT)/u-boot +U-BOOT_BIN ?= $(U-BOOT_PATH)/u-boot.bin + +FIRMWARE_BIN_SHA256_SUM ?= 63ec62f5d229cbed00918c8449173933f1c9d594c59396b8dd217e94f47138b0 +FIRMWARE_BIN ?= firmware-imx-8.0.bin +FIRMWARE_BIN ?= firmware-imx-8.0.bin +FIRMWARE_BIN_URL ?= https://www.nxp.com/lgfiles/NMG/MAD/YOCTO/$(FIRMWARE_BIN) +FIRMWARE_VERSION ?= firmware-imx-8.0 +FIRMWARE_PATH ?= $(ROOT)/out-firmware/$(FIRMWARE_VERSION) +LPDDR_BIN_PATH ?= $(FIRMWARE_PATH)/firmware/ddr/synopsys + +MKIMAGE_PATH ?= $(ROOT)/imx-mkimage +BOOT_IMG ?= $(ROOT)/out/boot.img + +################################################################################ +# Targets +################################################################################ +all: tfa u-boot linux optee-os buildroot flash-image +clean: tfa-clean buildroot-clean u-boot-clean optee-os-clean + +include toolchain.mk + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS = CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +# BL32=$(OPTEE_OS_HEADER_V2_BIN) \ +# BL32_EXTRA1=$(OPTEE_OS_PAGER_V2_BIN) \ +# BL32_EXTRA2=$(OPTEE_OS_PAGEABLE_V2_BIN) \ + +TF_A_FLAGS = PLAT=imx8mq SPD=opteed DEBUG_CONSOLE=1 DEBUG=0 V=1 +TF_A_FLAGS += BL32=$(ROOT)/optee_os/out/arm/core/tee-raw.bin + +tfa: optee-os u-boot + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + +tfa-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# U-Boot +################################################################################ + +U-BOOT_EXPORTS = CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +U-BOOT_DEFCONFIG_FILES := $(U-BOOT_PATH)/configs/imx8mq_evk_defconfig \ + $(BUILD_PATH)/kconfigs/uboot_imx8.conf + +$(U-BOOT_PATH)/.config: $(U-BOOT_DEFCONFIG_FILES) + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) imx8mq_evk_defconfig + (cd $(U-BOOT_PATH) && \ + ARCH=arm64 \ + scripts/kconfig/merge_config.sh $(U-BOOT_DEFCONFIG_FILES)) + +.PHONY: u-boot-defconfig +u-boot-defconfig: $(U-BOOT_PATH)/.config + +.PHONY: u-boot +u-boot: u-boot-defconfig + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) + +.PHONY: u-boot-clean +u-boot-clean: + cd $(U-BOOT_PATH) && git clean -xdf + +.PHONY: u-boot-cscope +u-boot-cscope: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) cscope + + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/imx.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 + +linux: linux-common + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +optee-os: optee-os-common +optee-os-clean: optee-os-clean-common + +################################################################################ +# DDR firmware +################################################################################ +# This is prebuilt binaries by NXP, download them and use them. Update path if +# it changes in the future. + +$(ROOT)/out-firmware/$(FIRMWARE_BIN): + mkdir -p $(ROOT)/out-firmware + (cd $(ROOT)/out-firmware && wget $(FIRMWARE_BIN_URL)) + +$(FIRMWARE_PATH)/.unpacked: $(ROOT)/out-firmware/$(FIRMWARE_BIN) + (cd $(ROOT)/out-firmware && \ + echo $(FIRMWARE_BIN_SHA256_SUM) $(FIRMWARE_BIN) | sha256sum -c) + (cd $(ROOT)/out-firmware && \ + chmod 711 $(FIRMWARE_BIN) && ./$(FIRMWARE_BIN) --auto-accept) + touch $(FIRMWARE_PATH)/.unpacked + +.PHONY: ddr-firmware +ddr-firmware: $(FIRMWARE_PATH)/.unpacked + +ddr-firmware-clean: + rm -rf $(ROOT)/out-firmware + +################################################################################ +# imx-mkimage +################################################################################ +mkimage: u-boot tfa ddr-firmware + ln -sf $(ROOT)/optee_os/out/arm/core/tee-raw.bin \ + $(MKIMAGE_PATH)/iMX8M/tee.bin + ln -sf $(ROOT)/trusted-firmware-a/build/imx8mq/release/bl31.bin \ + $(MKIMAGE_PATH)/iMX8M/ + ln -sf $(LPDDR_BIN_PATH)/lpddr4_pmu_train_*.bin $(MKIMAGE_PATH)/iMX8M/ + ln -sf $(U-BOOT_PATH)/u-boot-nodtb.bin $(MKIMAGE_PATH)/iMX8M/ + ln -sf $(U-BOOT_PATH)/spl/u-boot-spl.bin $(MKIMAGE_PATH)/iMX8M/ + ln -sf $(U-BOOT_PATH)/arch/arm/dts/imx8mq-evk.dtb \ + $(MKIMAGE_PATH)/iMX8M/fsl-imx8mq-evk.dtb + ln -sf $(U-BOOT_PATH)/tools/mkimage $(MKIMAGE_PATH)/iMX8M/mkimage_uboot + $(MAKE) -C $(MKIMAGE_PATH) SOC=iMX8M flash_spl_uboot +#> +If you want to run with HDMI, copy signed_hdmi_imx8m.bin to imx-mkimage/iMX8M +#> +make SOC=iMX8M flash_spl_uboot or make SOC=iMX8M flash_hdmi_spl_uboot to +#> +generate flash.bin. +mkimage-clean: + cd $(MKIMAGE_PATH) && git clean -xdf + rm -f $(ROOT)/build/mkimage_imx8 + +$(ROOT)/out-br/images/ramdisk.img: $(ROOT)/out-br/images/rootfs.cpio.gz + $(U-BOOT_PATH)/tools/mkimage -A arm64 -O linux -T ramdisk -C gzip \ + -d $< $@ + +$(ROOT)/out: + mkdir -p $@ + +$(ROOT)/out/boot.scr: $(ROOT)/build/imx/u-boot_boot_script | $(ROOT)/out + $(U-BOOT_PATH)/tools/mkimage -T script -C none -n 'Boot script' \ + -d $< $@ + +################################################################################ +# Flash images +################################################################################ +USE_PERSISTENT_ROOTFS ?= 0 + +# Configuration of the BOOT partition +FLASH_PARTITIONS_BLOCK_SIZE = 512 +FLASH_PARTITION_BOOT_START_BLOCK = 16384 +FLASH_PARTITION_BOOT_SIZE_IN_BYTES = \ + $(shell echo $$(( 64 * 1024 * 1024 ))) +FLASH_PARTITION_BOOT_SIZE_IN_BLOCKS = \ + $(shell echo $$(( $(FLASH_PARTITION_BOOT_SIZE_IN_BYTES) / $(FLASH_PARTITIONS_BLOCK_SIZE) ))) +FLASH_PARTITIONS_TABLE = "\ + start=$(FLASH_PARTITION_BOOT_START_BLOCK) \ + size=$(FLASH_PARTITION_BOOT_SIZE_IN_BLOCKS) \ + type=7\n" +FLASH_IMAGE_SIZE = \ + $(shell echo $$(( $(FLASH_PARTITION_BOOT_START_BLOCK) * $(FLASH_PARTITIONS_BLOCK_SIZE) \ + + $(FLASH_PARTITION_BOOT_SIZE_IN_BYTES) ))) + +# Configuration of the ROOTFS partition if enabled +ifeq ($(USE_PERSISTENT_ROOTFS),1) +FLASH_PARTITION_ROOTFS_IMAGE_PATH = $(ROOT)/out-br/images/rootfs.ext4 +FLASH_PARTITION_ROOTFS_START_BLOCK = \ + $(shell echo $$(( $(FLASH_PARTITION_BOOT_START_BLOCK) + $(FLASH_PARTITION_BOOT_SIZE_IN_BLOCKS) ))) +FLASH_PARTITION_ROOTFS_SIZE_IN_BYTES = \ + $(shell stat -L --printf="%s" $(FLASH_PARTITION_ROOTFS_IMAGE_PATH)) +FLASH_PARTITION_ROOTFS_SIZE_IN_BLOCKS = \ + $(shell echo $$(( $(FLASH_PARTITION_ROOTFS_SIZE_IN_BYTES) / $(FLASH_PARTITIONS_BLOCK_SIZE) ))) +FLASH_PARTITIONS_TABLE += "\ + start=$(FLASH_PARTITION_ROOTFS_START_BLOCK) \ + size=$(FLASH_PARTITION_ROOTFS_SIZE_IN_BLOCKS) \ + type=83\n" +FLASH_IMAGE_SIZE := $(shell echo $$(( $(FLASH_IMAGE_SIZE) + $(FLASH_PARTITION_ROOTFS_SIZE_IN_BYTES) ))) +endif + +.PHONY: flash-image +flash-image: buildroot mkimage linux + $(MAKE) flash-image-only + +.PHONY: flash-image-only +flash-image-only: $(ROOT)/out-br/images/ramdisk.img $(ROOT)/out/boot.scr + rm -f $(BOOT_IMG) + truncate -s $(FLASH_IMAGE_SIZE) $(BOOT_IMG) + echo -ne $(FLASH_PARTITIONS_TABLE) | sfdisk $(BOOT_IMG) + mformat -i $(BOOT_IMG).fat -n 64 -h 255 -T 131072 -v "BOOT IMG" -C :: + mcopy -i $(BOOT_IMG).fat $(LINUX_PATH)/arch/arm64/boot/Image :: + mcopy -i $(BOOT_IMG).fat \ + $(LINUX_PATH)/arch/arm64/boot/dts/freescale/imx8mq-evk.dtb :: + mcopy -i $(BOOT_IMG).fat $(ROOT)/out/boot.scr :: + +ifeq ($(USE_PERSISTENT_ROOTFS),1) + dd if=$(FLASH_PARTITION_ROOTFS_IMAGE_PATH) of=$(BOOT_IMG) bs=$(FLASH_PARTITIONS_BLOCK_SIZE) \ + seek=$(FLASH_PARTITION_ROOTFS_START_BLOCK) conv=fsync,notrunc +else + mcopy -i $(BOOT_IMG).fat $(ROOT)/out-br/images/ramdisk.img :: +endif + + dd if=$(BOOT_IMG).fat of=$(BOOT_IMG) bs=$(FLASH_PARTITIONS_BLOCK_SIZE) \ + seek=$(FLASH_PARTITION_BOOT_START_BLOCK) conv=fsync,notrunc + dd if=$(ROOT)/imx-mkimage/iMX8M/flash.bin of=$(BOOT_IMG) bs=1k seek=33 \ + conv=fsync,notrunc diff --git a/build/imx/u-boot_boot_script b/build/imx/u-boot_boot_script new file mode 100644 index 0000000..a2ee4fe --- /dev/null +++ b/build/imx/u-boot_boot_script @@ -0,0 +1,22 @@ +run loadfdt; +fdt addr ${fdt_addr}; +fdt get value optee_compatible /firmware/optee compatible; +if test "${optee_compatible}" = "linaro,optee-tz"; then + echo Found optee node, using it; +else + echo optee node missing, trying to add it; + fdt resize 4096; + fdt mknode / firmware; + fdt mknode /firmware optee; + fdt set /firmware/optee compatible "linaro,optee-tz"; + fdt set /firmware/optee method "smc"; +fi; +fdt print /firmware/optee; +run loadimage; +if fatload mmc ${mmcdev}:${mmcpart} ${initrd_addr} ramdisk.img; then + booti ${loadaddr} ${initrd_addr} ${fdt_addr}; +else + echo "Booting on the persistent file system ..." + run loadimage;run mmcargs; + booti ${loadaddr} - ${fdt_addr}; +fi; \ No newline at end of file diff --git a/build/juno.mk b/build/juno.mk new file mode 100644 index 0000000..d508db5 --- /dev/null +++ b/build/juno.mk @@ -0,0 +1,151 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +override COMPILE_NS_USER := 64 +override COMPILE_NS_KERNEL := 64 +override COMPILE_S_USER := 64 +override COMPILE_S_KERNEL := 64 + +OPTEE_OS_PLATFORM = vexpress-juno + +include common.mk + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a + +U-BOOT_PATH ?= $(ROOT)/u-boot +U-BOOT_BIN ?= $(U-BOOT_PATH)/u-boot.bin + +SCP_BLX_URL ?= https://downloads.trustedfirmware.org/tf-a/css_scp_2.8.0/juno + +################################################################################ +# Targets +################################################################################ +all: scp-blx arm-tf u-boot linux optee-os buildroot +clean: scp-blx-clean arm-tf-clean buildroot-clean u-boot-clean optee-os-clean + +include toolchain.mk + +################################################################################ +# SCP BL1 and BL2 +################################################################################ +.PHONY: scp-blx +scp-blx: $(ROOT)/out-firmware/scp_bl1.bin $(ROOT)/out-firmware/scp_bl2.bin + +.PHONY: scp-blx +scp-blx-clean: + @rm -f $(ROOT)/out-firmware/scp_bl1.bin + @rm -f $(ROOT)/out-firmware/scp_bl2.bin + @rm -f $(ROOT)/out-firmware/tmp/scp_bl1.bin + @rm -f $(ROOT)/out-firmware/tmp/scp_bl2.bin + +define dlscp + @mkdir -p $(ROOT)/out-firmware/tmp + @rm -f $(ROOT)/out-firmware/tmp/$1 + @rm -f $(ROOT)/out-firmware/$1 + @(cd $(ROOT)/out-firmware/tmp/ && wget $(SCP_BLX_URL)/$1) + @(echo $2 $(ROOT)/out-firmware/tmp/$1 | sha256sum -c) + @(mv $(ROOT)/out-firmware/tmp/$1 $(ROOT)/out-firmware/$1) +endef + +$(ROOT)/out-firmware/scp_bl1.bin: + $(call dlscp,scp_bl1.bin,1c690a7d93c82d39d18b720920ced7a712fdcfc744224b4f067e56013522fece) + +$(ROOT)/out-firmware/scp_bl2.bin: + $(call dlscp,scp_bl2.bin,533eb4a3f9d91e759b98288edf4c1441abe6f7fbc5da87825a8f1bb1b7942aa5) + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +TF_A_FLAGS ?= \ + SCP_BL2=$(ROOT)/out-firmware/scp_bl2.bin \ + BL32=$(OPTEE_OS_HEADER_V2_BIN) \ + BL32_EXTRA1=$(OPTEE_OS_PAGER_V2_BIN) \ + BL32_EXTRA2=$(OPTEE_OS_PAGEABLE_V2_BIN) \ + BL33=$(U-BOOT_BIN) \ + DEBUG=0 \ + ARM_TSP_RAM_LOCATION=dram \ + PLAT=juno \ + SPD=opteed + +arm-tf: scp-blx optee-os u-boot + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + +arm-tf-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# Das U-Boot +################################################################################ + +U-BOOT_EXPORTS ?= CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +U-BOOT_DEFCONFIG_FILES := \ + $(U-BOOT_PATH)/configs/vexpress_aemv8a_juno_defconfig \ + $(ROOT)/build/kconfigs/u-boot_juno.conf + +.PHONY: u-boot +u-boot: + cd $(U-BOOT_PATH) && \ + scripts/kconfig/merge_config.sh $(U-BOOT_DEFCONFIG_FILES) + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) all + +u-boot-clean: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) clean + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/juno.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 + +linux: linux-common + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +optee-os: optee-os-common +optee-os-clean: optee-os-clean-common + + +$(ROOT)/out-br/images/ramdisk.img: $(ROOT)/out-br/images/rootfs.cpio.gz + $(U-BOOT_PATH)/tools/mkimage -A arm64 -O linux -T ramdisk -C gzip \ + -d $< $@ + +FTP-UPLOAD = ftp-upload -v --host $(JUNO_IP) --dir SOFTWARE + +.PHONY: flash +flash: $(ROOT)/out-br/images/ramdisk.img + @test -n "$(JUNO_IP)" || \ + (echo "JUNO_IP not set" ; exit 1) + $(FTP-UPLOAD) $(ROOT)/out-firmware/scp_bl1.bin + $(FTP-UPLOAD) $(TF_A_PATH)/build/juno/release/bl1.bin + $(FTP-UPLOAD) $(TF_A_PATH)/build/juno/release/fip.bin + $(FTP-UPLOAD) $(ROOT)/linux/arch/arm64/boot/Image + $(FTP-UPLOAD) $(ROOT)/linux/arch/arm64/boot/dts/arm/juno.dtb + $(FTP-UPLOAD) $(ROOT)/linux/arch/arm64/boot/dts/arm/juno-r1.dtb + $(FTP-UPLOAD) $(ROOT)/linux/arch/arm64/boot/dts/arm/juno-r2.dtb + $(FTP-UPLOAD) $(ROOT)/out-br/images/ramdisk.img diff --git a/build/kconfigs/fvp.conf b/build/kconfigs/fvp.conf new file mode 100644 index 0000000..62b0881 --- /dev/null +++ b/build/kconfigs/fvp.conf @@ -0,0 +1,8 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_TCG_TPM=y +CONFIG_TCG_FTPM_TEE=m +CONFIG_9P_FS=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_ARM_FFA_TRANSPORT=y diff --git a/build/kconfigs/fvp_trusted-services.conf b/build/kconfigs/fvp_trusted-services.conf new file mode 100644 index 0000000..3f7b12b --- /dev/null +++ b/build/kconfigs/fvp_trusted-services.conf @@ -0,0 +1,21 @@ +CONFIG_BLK_DEV_INITRD=y +CONFIG_ARCH_VEXPRESS=y +CONFIG_MODULES=y +CONFIG_NET=y +CONFIG_UNIX=y +CONFIG_INET=y +# CONFIG_IPV6 is not set +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_ARM_FFA_TRANSPORT=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_VIRTIO_MMIO=y +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_TMPFS=y +CONFIG_9P_FS=y +CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_FS=y diff --git a/build/kconfigs/fvp_trusted-services_uefi.conf b/build/kconfigs/fvp_trusted-services_uefi.conf new file mode 100644 index 0000000..d372aca --- /dev/null +++ b/build/kconfigs/fvp_trusted-services_uefi.conf @@ -0,0 +1 @@ +CONFIG_STRICT_DEVMEM=n diff --git a/build/kconfigs/hikey.conf b/build/kconfigs/hikey.conf new file mode 100644 index 0000000..4e6e1ce --- /dev/null +++ b/build/kconfigs/hikey.conf @@ -0,0 +1,7 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_PINCTRL_SINGLE=y +CONFIG_STAGING=y +CONFIG_ANDROID=y +CONFIG_ANDROID_TIMED_OUTPUT=n +CONFIG_ANDROID_LOW_MEMORY_KILLER=n diff --git a/build/kconfigs/hikey960.conf b/build/kconfigs/hikey960.conf new file mode 100644 index 0000000..b84a48f --- /dev/null +++ b/build/kconfigs/hikey960.conf @@ -0,0 +1,26 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_PINCTRL_SINGLE=y + +CONFIG_F2FS_FS=y +CONFIG_SQUASHFS_FILE_DIRECT=y +CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_4K_DEVBLK_SIZE=y + +CONFIG_SCSI_UFSHCD=y +CONFIG_SCSI_UFSHCD_PLATFORM=y +CONFIG_SCSI_UFS_HISI=y + +CONFIG_STAGING=y +CONFIG_ANDROID=y + +CONFIG_CFG80211=y +CONFIG_MAC80211=y +CONFIG_RFKILL=y +CONFIG_WLCORE=y +CONFIG_WL18XX=y +CONFIG_WLCORE_SDIO=y diff --git a/build/kconfigs/hikey_debian.conf b/build/kconfigs/hikey_debian.conf new file mode 100644 index 0000000..b85868e --- /dev/null +++ b/build/kconfigs/hikey_debian.conf @@ -0,0 +1,5 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_PINCTRL_SINGLE=y +CONFIG_DEBUG_INFO=n +CONFIG_EXT4_FS=y diff --git a/build/kconfigs/imx.conf b/build/kconfigs/imx.conf new file mode 100644 index 0000000..0c77338 --- /dev/null +++ b/build/kconfigs/imx.conf @@ -0,0 +1,3 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_IMX_SDMA=n diff --git a/build/kconfigs/juno.conf b/build/kconfigs/juno.conf new file mode 100644 index 0000000..07554cf --- /dev/null +++ b/build/kconfigs/juno.conf @@ -0,0 +1,2 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y diff --git a/build/kconfigs/mediatek.conf b/build/kconfigs/mediatek.conf new file mode 100644 index 0000000..6fb1c53 --- /dev/null +++ b/build/kconfigs/mediatek.conf @@ -0,0 +1,40 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_MODULES=n +CONFIG_VIRTUALIZATION=n +CONFIG_SYS_HYPERVISOR=n +CONFIG_CRASH_DUMP=n +CONFIG_IPV6=n +CONFIG_ACPI=n +CONFIG_NETWORK_FILESYSTEMS=n +CONFIG_DEBUG_KERNEL=n +CONFIG_MAGIC_SYSRQ=n +CONFIG_ARCH_SUNXI=n +CONFIG_ARCH_ALPINE=n +CONFIG_ARCH_BCM2835=n +CONFIG_ARCH_BCM_IPROC=n +CONFIG_ARCH_BERLIN=n +CONFIG_ARCH_EXYNOS=n +CONFIG_ARCH_LAYERSCAPE=n +CONFIG_ARCH_LG1K=n +CONFIG_ARCH_HISI=n +CONFIG_ARCH_MESON=n +CONFIG_ARCH_MVEBU=n +CONFIG_ARCH_QCOM=n +CONFIG_ARCH_ROCKCHIP=n +CONFIG_ARCH_SEATTLE=n +CONFIG_ARCH_SHMOBILE=n +CONFIG_ARCH_RENESAS=n +CONFIG_ARCH_R8A7795=n +CONFIG_ARCH_R8A7796=n +CONFIG_ARCH_STRATIX10=n +CONFIG_ARCH_TEGRA=n +CONFIG_ARCH_SPRD=n +CONFIG_ARCH_THUNDER=n +CONFIG_ARCH_THUNDER2=n +CONFIG_ARCH_UNIPHIER=n +CONFIG_ARCH_VEXPRESS=n +CONFIG_ARCH_VULCAN=n +CONFIG_ARCH_XGENE=n +CONFIG_ARCH_ZX=n +CONFIG_ARCH_ZYNQMP=n diff --git a/build/kconfigs/poplar.conf b/build/kconfigs/poplar.conf new file mode 100644 index 0000000..d13ba5d --- /dev/null +++ b/build/kconfigs/poplar.conf @@ -0,0 +1,6 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_STAGING=y +CONFIG_ANDROID=y +CONFIG_ANDROID_TIMED_OUTPUT=n +CONFIG_ANDROID_LOW_MEMORY_KILLER=n diff --git a/build/kconfigs/qemu.conf b/build/kconfigs/qemu.conf new file mode 100644 index 0000000..418888c --- /dev/null +++ b/build/kconfigs/qemu.conf @@ -0,0 +1,15 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +### Enable 9P VFS +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_9P_FS=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER=y +CONFIG_GDB_SCRIPTS=y +CONFIG_KEYS=y +CONFIG_TRUSTED_KEYS=y +CONFIG_ENCRYPTED_KEYS=y +CONFIG_ARM_FFA_TRANSPORT=y diff --git a/build/kconfigs/rockpi4.conf b/build/kconfigs/rockpi4.conf new file mode 100644 index 0000000..07554cf --- /dev/null +++ b/build/kconfigs/rockpi4.conf @@ -0,0 +1,2 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y diff --git a/build/kconfigs/rpi3.conf b/build/kconfigs/rpi3.conf new file mode 100644 index 0000000..07554cf --- /dev/null +++ b/build/kconfigs/rpi3.conf @@ -0,0 +1,2 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y diff --git a/build/kconfigs/stm32mp1.conf b/build/kconfigs/stm32mp1.conf new file mode 100644 index 0000000..121eddb --- /dev/null +++ b/build/kconfigs/stm32mp1.conf @@ -0,0 +1,8 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +# SCMI support +CONFIG_ARM_SCMI_PROTOCOL=y +CONFIG_COMMON_CLK_SCMI=y +CONFIG_REGULATOR_ARM_SCMI=y +# CONFIG_SENSORS_ARM_SCMI is not set +CONFIG_RESET_SCMI=y diff --git a/build/kconfigs/tee_bench.conf b/build/kconfigs/tee_bench.conf new file mode 100644 index 0000000..cf6cba9 --- /dev/null +++ b/build/kconfigs/tee_bench.conf @@ -0,0 +1 @@ +CONFIG_OPTEE_BENCHMARK=y diff --git a/build/kconfigs/u-boot_juno.conf b/build/kconfigs/u-boot_juno.conf new file mode 100644 index 0000000..e69de29 diff --git a/build/kconfigs/u-boot_qemu_v8.conf b/build/kconfigs/u-boot_qemu_v8.conf new file mode 100644 index 0000000..0e74125 --- /dev/null +++ b/build/kconfigs/u-boot_qemu_v8.conf @@ -0,0 +1,3 @@ +CONFIG_SYS_TEXT_BASE=0x60000000 +CONFIG_BOOTCOMMAND="setenv kernel_addr_r 0x42200000 && setenv ramdisk_addr_r 0x45000000 && load hostfs - ${kernel_addr_r} uImage && load hostfs - ${ramdisk_addr_r} rootfs.cpio.uboot && setenv bootargs console=ttyAMA0,115200 earlyprintk=serial,ttyAMA0,115200 root=/dev/ram && bootm ${kernel_addr_r} ${ramdisk_addr_r} ${fdt_addr}" +CONFIG_SEMIHOSTING=y diff --git a/build/kconfigs/u-boot_qemu_virt_v7.conf b/build/kconfigs/u-boot_qemu_virt_v7.conf new file mode 100644 index 0000000..36f92dd --- /dev/null +++ b/build/kconfigs/u-boot_qemu_virt_v7.conf @@ -0,0 +1,7 @@ +CONFIG_SYS_TEXT_BASE=0x60000000 +CONFIG_BOOTCOMMAND="fdt addr ${fdt_addr} && fdt resize 1000 && smhload zImage ${kernel_addr_r} && smhload rootfs.cpio.gz ${ramdisk_addr_r} ramdisk_addr_end && setenv bootargs console=ttyAMA0,115200 earlyprintk=serial,ttyAMA0,115200 && fdt chosen ${ramdisk_addr_r} ${ramdisk_addr_end} && bootz ${kernel_addr_r} - ${fdt_addr}" +CONFIG_SEMIHOSTING=y +CONFIG_ENV_IS_NOWHERE=y +# CONFIG_ENV_IS_IN_FLASH is not set +# CONFIG_MTD is not set +# CONFIG_MTD_NOR_FLASH is not set diff --git a/build/kconfigs/u-boot_rockpi4.conf b/build/kconfigs/u-boot_rockpi4.conf new file mode 100644 index 0000000..9ca219f --- /dev/null +++ b/build/kconfigs/u-boot_rockpi4.conf @@ -0,0 +1,32 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +# CONFIG_OPTEE_TA_AVB is not set +# CONFIG_OPTEE_TA_SCP03 is not set +# CONFIG_DISTRO_DEFAULTS is not set +# CONFIG_ENV_IS_IN_MMC is not set +CONFIG_ENV_IS_NOWHERE=y +# CONFIG_EFI_LOADER is not set +# CONFIG_CMD_OPTEE_RPMB is not set +# CONFIG_FIT_SIGNATURE is not set +# CONFIG_CHIMP_OPTEE is not set +# CONFIG_USB is not set +# CONFIG_CMD_NET is not set +# CONFIG_CMD_TPM is not set +# CONFIG_TPM is not set +# CONFIG_TPM_V1 is not set +# CONFIG_TPM_V2 is not set +# CONFIG_NET is not set +# CONFIG_PCI is not set +# CONFIG_CMD_ELF is not set +# CONFIG_CMD_GO is not set +CONFIG_HUSH_PARSER=y +CONFIG_CMD_FS_GENERIC=y +CONFIG_CMD_EXT2=y +CONFIG_USE_BOOTARGS=y +CONFIG_BOOTARGS="console=ttyS2,1500000 root=PARTUUID=17d61bff-8fdc-4089-b675-9be21b9f6ac7 loglevel=6 rootwait" +CONFIG_DEFAULT_FDT_FILE="/boot/rk3399-rock-pi-4b.dtb" +CONFIG_USE_BOOTCOMMAND=y +## Note: kernel_addr_gz is arbitrarily chosen to be 100M (0xa0000000) above kernel_addr_r +## That leaves quite a lot of room to decompress the kernel which by default is ~50MB +CONFIG_BOOTCOMMAND="setenv bootdev unknown; setenv kernel_addr_gz 0xa2080000; setenv kernel /boot/Image.gz; for d in 1 0; do test ${bootdev} = unknown && echo .. Looking for ${kernel} in mmc ${d}:5 && test -e mmc ${d}:5 ${kernel} && setenv bootdev ${d} && echo .. Found; done; if test ${bootdev} = unknown; then echo .. Kernel not found; else echo .. Loading kernel; ext2load mmc ${bootdev}:5 ${kernel_addr_gz} ${kernel}; unzip ${kernel_addr_gz} ${kernel_addr_r}; echo .. Loading DTB: mmc ${bootdev}:5 ${fdtfile}; ext2load mmc ${bootdev}:5 ${fdt_addr_r} ${fdtfile}; echo .. Booting kernel; booti ${kernel_addr_r} - ${fdt_addr_r}; fi" +CONFIG_BOOTDELAY=0 diff --git a/build/kconfigs/u-boot_rpi3.conf b/build/kconfigs/u-boot_rpi3.conf new file mode 100644 index 0000000..304f129 --- /dev/null +++ b/build/kconfigs/u-boot_rpi3.conf @@ -0,0 +1 @@ +CONFIG_SYS_TEXT_BASE=0x11000000 diff --git a/build/kconfigs/u-boot_xen_qemu_v8.conf b/build/kconfigs/u-boot_xen_qemu_v8.conf new file mode 100644 index 0000000..c7f81a4 --- /dev/null +++ b/build/kconfigs/u-boot_xen_qemu_v8.conf @@ -0,0 +1,3 @@ +CONFIG_SYS_TEXT_BASE=0x60000000 +CONFIG_BOOTCOMMAND="load virtio 0 $kernel_addr_r xen.efi && bootefi $kernel_addr_r" +CONFIG_SEMIHOSTING=y diff --git a/build/kconfigs/uboot_imx8.conf b/build/kconfigs/uboot_imx8.conf new file mode 100644 index 0000000..04f4422 --- /dev/null +++ b/build/kconfigs/uboot_imx8.conf @@ -0,0 +1,17 @@ +CONFIG_CMD_USB=y +CONFIG_USB_UHCI=y +CONFIG_USB_HOST_ETHER=y +CONFIG_USB_ETHER_ASIX=y +CONFIG_USB_ETHER_ASIX88179=y + +CONFIG_CMD_NET=y +CONFIG_CMD_PING=y +CONFIG_CMD_DHCP=y + +CONFIG_BOOTP_SUBNETMASK=y +CONFIG_BOOTP_GATEWAY=y +CONFIG_BOOTP_HOSTNAME=y +CONFIG_BOOTP_BOOTPATH=y + +CONFIG_USB=y +CONFIG_DM_USB=y diff --git a/build/kconfigs/versal.conf b/build/kconfigs/versal.conf new file mode 100644 index 0000000..0ffa982 --- /dev/null +++ b/build/kconfigs/versal.conf @@ -0,0 +1,4 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_DEVTMPFS=y + diff --git a/build/kconfigs/xen.conf b/build/kconfigs/xen.conf new file mode 100644 index 0000000..7450658 --- /dev/null +++ b/build/kconfigs/xen.conf @@ -0,0 +1,30 @@ +CONFIG_EXPERT=y +CONFIG_TEE=y +CONFIG_OPTEE=y + +# CONFIG_ACPI is not set +# CONFIG_HAS_ITS is not set +# CONFIG_EFI_SET_VIRTUAL_ADDRESS_MAP is not set +# CONFIG_ARGO is not set + +# CONFIG_SCHED_ARINC653 is not set + +# CONFIG_SCHED_CREDIT_DEFAULT is not set +CONFIG_SCHED_CREDIT2_DEFAULT=y +# CONFIG_SCHED_RTDS_DEFAULT is not set +# CONFIG_SCHED_NULL_DEFAULT is not set + +# CONFIG_IPMMU_VMSA is not set +# +# CONFIG_DEBUG is not set +CONFIG_DEBUG_INFO=n +# CONFIG_FRAME_POINTER is not set +# CONFIG_COVERAGE is not set +# CONFIG_DEBUG_LOCK_PROFILE is not set +# CONFIG_DEBUG_LOCKS is not set +# CONFIG_PERF_COUNTERS is not set +# CONFIG_VERBOSE_DEBUG is not set +# CONFIG_DEVICE_TREE_DEBUG is not set +# CONFIG_SCRUB_DEBUG is not set +# CONFIG_DEBUG_TRACE is not set +# CONFIG_XMEM_POOL_POISON is not set diff --git a/build/kconfigs/zynqmp.conf b/build/kconfigs/zynqmp.conf new file mode 100644 index 0000000..07554cf --- /dev/null +++ b/build/kconfigs/zynqmp.conf @@ -0,0 +1,2 @@ +CONFIG_TEE=y +CONFIG_OPTEE=y diff --git a/build/poplar.mk b/build/poplar.mk new file mode 100644 index 0000000..e4019d4 --- /dev/null +++ b/build/poplar.mk @@ -0,0 +1,222 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +COMPILE_NS_USER ?= 64 +override COMPILE_NS_KERNEL := 64 +COMPILE_S_USER ?= 32 +COMPILE_S_KERNEL ?= 64 + +OPTEE_OS_PLATFORM = poplar + +################################################################################ +# Includes +################################################################################ +include common.mk + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +ifeq ($(DEBUG),1) +TF_A_BUILD ?= debug +else +TF_A_BUILD ?= release +endif + +OUT_PATH ?= $(ROOT)/out +ROOTFS_BIN ?= $(ROOT)/out-br/images/rootfs.tar +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +LLOADER_PATH ?= $(ROOT)/l-loader +UBOOT_PATH ?= $(ROOT)/u-boot +OPTEE_PATH ?= $(ROOT)/optee_os +LINUX_PATH ?= $(ROOT)/linux +TOOLS_PATH ?= $(ROOT)/poplar-tools + +BL1_BIN ?= $(TF_A_PATH)/build/poplar/$(TF_A_BUILD)/bl1.bin +FIP_BIN ?= $(TF_A_PATH)/build/poplar/$(TF_A_BUILD)/fip.bin +LLOADER_BIN ?= $(LLOADER_PATH)/l-loader.bin + +LINUX_DTB ?= $(LINUX_PATH)/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dtb + +UBOOT_BIN ?= $(UBOOT_PATH)/u-boot.bin +OPTEE_BIN ?= $(OPTEE_PATH)/out/arm/core/tee-header_v2.bin +OPTEE_BIN_EXTRA1 ?= $(OPTEE_PATH)/out/arm/core/tee-pager_v2.bin +OPTEE_BIN_EXTRA2 ?= $(OPTEE_PATH)/out/arm/core/tee-pageable_v2.bin + +################################################################################ +# Targets +################################################################################ +.PHONY: all +all: u-boot arm-tf buildroot l-loader linux prepare-images | toolchains + +.PHONY: clean +clean: u-boot-clean arm-tf-clean l-loader-clean linux-clean optee-os-clean \ + buildroot-clean + +################################################################################ +# Toolchain +################################################################################ +include toolchain.mk + +################################################################################ +# U-Boot +################################################################################ +.PHONY: u-boot-config +u-boot-config: +ifeq ($(wildcard $(UBOOT_PATH)/.config),) + $(MAKE) -C $(UBOOT_PATH) \ + CROSS_COMPILE=$(AARCH64_CROSS_COMPILE) poplar_defconfig +endif + +.PHONY: u-boot-menuconfig +u-boot-menuconfig: u-boot-config + $(MAKE) -C $(UBOOT_PATH) \ + CROSS_COMPILE=$(AARCH64_CROSS_COMPILE) menuconfig + +.PHONY: u-boot +u-boot: u-boot-config + $(MAKE) -C $(UBOOT_PATH) \ + CROSS_COMPILE="$(AARCH64_CROSS_COMPILE)" + +.PHONY: u-boot-clean +u-boot-clean: + cd $(UBOOT_PATH) && git clean -xdf + +################################################################################ +# ARM Trusted Firmware +################################################################################ +.PHONY: arm-tf +arm-tf: u-boot optee-os + $(MAKE) -C $(TF_A_PATH) \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" \ + all fip \ + DEBUG=$(DEBUG) \ + PLAT=poplar \ + SPD=opteed \ + BL32=$(OPTEE_BIN) \ + BL33=$(UBOOT_BIN) \ + BL32_EXTRA1=$(OPTEE_BIN_EXTRA1) \ + BL32_EXTRA2=$(OPTEE_BIN_EXTRA2) + +.PHONY: arm-tf-clean +arm-tf-clean: + cd $(TF_A_PATH) && git clean -xdf + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_COMMON_FLAGS += CFG_ARM64_core=y CFG_DRAM_SIZE_GB=1 \ + CFG_ENABLE_EMBEDDED_TESTS=y + +.PHONY: optee-os +optee-os: optee-os-common + +.PHONY: optee-os-clean +optee-os-clean: optee-os-clean-common + +################################################################################ +# l-loader +################################################################################ +l-loader: arm-tf + cp $(BL1_BIN) $(LLOADER_PATH)/atf + cp $(FIP_BIN) $(LLOADER_PATH)/atf + $(MAKE) -C $(LLOADER_PATH) CROSS_COMPILE="$(AARCH32_CROSS_COMPILE)" \ + ARM_TRUSTED_FIRMWARE="$(TF_A_PATH)" + +.PHONY: l-loader-clean +l-loader-clean: + cd $(LLOADER_PATH) && git clean -xdf + +################################################################################ +# Linux +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/poplar.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 + +linux: linux-common + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +.PHONY: prepare-images +prepare-images: linux l-loader buildroot + @mkdir -p $(OUT_PATH) + @cp $(TOOLS_PATH)/poplar_recovery_builder.sh $(OUT_PATH) + @cp $(LLOADER_BIN) $(OUT_PATH) + @cp $(LINUX_PATH)/arch/arm64/boot/Image $(OUT_PATH) + @cp $(LINUX_DTB) $(OUT_PATH) + @cd $(OUT_PATH) && PATH=$(UBOOT_PATH)/tools:$$PATH \ + EMMC_DEV=/dev/mmcblk0 ./poplar_recovery_builder.sh \ + all "$(ROOTFS_BIN)" + +################################################################################ +# Buildroot/RootFS +################################################################################ +.PHONY: update_rootfs +update_rootfs: arm-tf u-boot + +.PHONY: buildroot +buildroot: update_rootfs + +################################################################################ +# Flash images +################################################################################ +.PHONY: flash-help +flash-help: + @echo "1. Install and configure TFTP server on your host PC:" + @echo "" + @echo " $$ sudo apt-get install atftpd # install atftpd server" + @echo " $$ sudo vim /etc/default/atftpd # edit atftpd server config" + @echo " $$ sudo service atftpd restart # restart atftpd server" + @echo "" + @echo "2. Proper configuration should look like:" + @echo "" + @echo " $$ cat /etc/default/atftpd" + @echo " USE_INETD=false" + @echo " OPTIONS=\"--tftpd-timeout 300 --retry-timeout 5 --mcast-port 1758 --mcast-addr 239.239.239.0-255 --mcast-ttl 1 --maxthread 100 --verbose=5 /path/to/rep/out/dir\"" + @echo "" + @echo "3. Flash proper U-boot build to USB stick." + @echo " It should be flashed to the first FAT32 partition. Then boot" + @echo " from it by pressing USB_BOOT switch on the board" + @echo "" + @echo "4. Connect to Poplar board over serial console (run on host PC):" + @echo "" + @echo " $$ screen /dev/ttyUSB0 115200" + @echo "" + @echo "5. Configure network interface in Poplar U-boot shell. If you can't" + @echo "get into U-boot console, press and hold Ctrl+C while booting:" + @echo "" + @echo " => setenv ipaddr 192.168.0.2" + @echo " => setenv netmask 255.255.255.0" + @echo " => setenv serverip 192.168.0.3" + @echo " ETH1: PHY(phyaddr=3, rgmii) link UP: DUPLEX=FULL : SPEED=1000M" + @echo " MAC: 00-16-8E-62-66-84" + @echo " host 192.168.0.3 is alive" + @echo "" + @echo "6. Verify connection is working (run in U-boot shell):" + @echo "" + @echo " => ping 192.168.0.3" + @echo "" + @echo "7. Run installer (run in U-boot shell):" + @echo "" + @echo " => tftp 0x08000000 recovery_files/install.scr" + @echo " => source 0x08000000" + @echo "" + @echo "8. After successful flashing reboot your board (U-boot shell):" + @echo "" + @echo " => reset" + @echo "" diff --git a/build/qemu-check.exp b/build/qemu-check.exp new file mode 100644 index 0000000..c15a4f9 --- /dev/null +++ b/build/qemu-check.exp @@ -0,0 +1,180 @@ +#!/usr/bin/expect -f +# +# This scripts starts QEMU, loads and boots Linux/OP-TEE, then runs +# tests in the guest. The return code is 0 for success, >0 for error. +# +# Options: +# --bios Path to the binary to be run [../out/bios-qemu/bios.bin] +# -q Suppress output to stdout (quiet) +# --tests Type of tests to run, values: all, xtest and trusted-keys +# --timeout Timeout for each test (sub)case, in seconds [480] +# --xtest-args Optional arguments to xtest + +set bios "../out/bios-qemu/bios.bin" +set cmd1 "cd /mnt/host/build/qemu_v8/xen" +set cmd2 "xl create guest.cfg" +set cmd3 "xl console domu" +set quiet 0 +set xtest_args "" + +# The time required to run some tests (e.g., key generation tests [4007.*]) +# can be significant and vary widely -- typically, from about one minute to +# several minutes depending on the host machine. +# The value here should be sufficient to run the whole optee_test suite +# ('xtest') with all testsuites enabled (regression+gp+pkcs11). +set timeout 900 +set tests "all" + +# Parse command line +set myargs $argv +while {[llength $myargs]} { + set myargs [lassign $myargs arg] + switch -exact -- $arg { + "--bios" {set myargs [lassign $myargs ::bios]} + "--tests" {set myargs [lassign $myargs ::tests]} + "--timeout" {set myargs [lassign $myargs ::timeout]} + "-q" {set ::quiet 1} + "--xtest-args" {set myargs [lassign $myargs ::xtest_args]} + } +} + +set cmd "xtest $xtest_args" + +proc info arg { + if {$::quiet==1} { return } + puts -nonewline $arg + flush stdout +} + +proc check_test_result arg { + set casenum "none" + expect { + # Exit with error status as soon as a test fails + -re { ([^ ]+) FAIL} { + info " $expect_out(1,string) FAIL\n" + exit 1 + } + -re {rcu_sched detected stalls} { + info " Kernel error: '$expect_out(0,string)'\n" + exit 1 + } + # Crude progress indicator: print one # when each test [sub]case starts + -re {([\*o]) ([^ ]+) } { + set casenum $expect_out(2,string) + if {$expect_out(1,string) == "o"} { + if {$star == 1} { + # Do not count first subcase ('o') since start + # of test ('*') was counted already + set star 0 + exp_continue + } + } else { + set star 1 + } + info "#" + incr ncases + if {$ncases % 50 == 0} { info "\n" } + exp_continue + } + # Exit when result separator is seen + "+-----------------------------------------------------\r\r" {} + # Handle errors in TEE core output + -i $arg -re {(..TC:[^\n]*assertion[^\n]*failed at[^\n]*)} { + info "!!! $expect_out(1,string)\n" + exit 1 + } + -i $arg -re {(..TC:[^\n]*Panic at[^\n]*)} { + info "!!! $expect_out(1,string)\n" + exit 1 + } + timeout { + info "!!! Timeout\n" + info "TIMEOUT - test case too long or hung? (last test started: $casenum)\n" + exit 2 + } + } + info "\nStatus: PASS ($ncases test cases)\n" +} + +# Disable echoing of guest output +log_user 0 +# Save guest console output to a file +log_file -a -noappend "serial0.log" +info "Starting QEMU..." +open "serial1.log" "w+" +spawn -open [open "|tail -f serial1.log"] +set teecore $spawn_id +if {[string first "aarch64" $::env(QEMU)] != -1} { + if {$::env(XEN_BOOT) == "y"} { + spawn $::env(QEMU) -nographic -serial mon:stdio -serial file:serial1.log -smp $::env(QEMU_SMP) -machine virt,acpi=off,secure=on,mte=$::env(QEMU_MTE),gic-version=$::env(QEMU_GIC),virtualization=true -cpu $::env(QEMU_CPU) -d unimp -semihosting-config enable=on,target=native -m $::env(QEMU_MEM) -bios bl1.bin -initrd rootfs.cpio.gz -kernel Image -append "console=ttyAMA0,38400 keep_bootcon root=/dev/vda2" -drive if=none,file=xen.ext4,format=raw,id=hd1 -device virtio-blk-device,drive=hd1 -fsdev local,id=fsdev0,path=../..,security_model=none -device virtio-9p-device,fsdev=fsdev0,mount_tag=host + } else { + spawn $::env(QEMU) -nographic -serial mon:stdio -serial file:serial1.log -smp $::env(QEMU_SMP) -machine virt,acpi=off,secure=on,mte=$::env(QEMU_MTE),gic-version=$::env(QEMU_GIC),virtualization=$::env(QEMU_VIRT) -cpu $::env(QEMU_CPU) -d unimp -semihosting-config enable=on,target=native -m $::env(QEMU_MEM) -bios bl1.bin -initrd rootfs.cpio.gz -kernel Image -append "console=ttyAMA0,38400 keep_bootcon root=/dev/vda2" + } +} else { + spawn $::env(QEMU) -nographic -monitor none -machine virt -machine secure=on -cpu cortex-a15 -smp $::env(QEMU_SMP) -d unimp -semihosting-config enable=on,target=native -m 1057 -serial stdio -serial file:serial1.log -bios $bios +} +expect { + "Kernel panic" { + info "!!! Kernel panic\n" + exit 1 + } + timeout { + info "!!! Timeout\n" + exit 1 + } + "ogin:" +} +send -- "root\r\r" +expect "# " +info " done, guest is booted" +if {$::env(XEN_BOOT) == "y"} { + info " (Xen Dom0)" +} +info ".\n" +# Toolchain libraries might be here or there +send -- "export LD_LIBRARY_PATH=/lib:/lib/arm-linux-gnueabihf\r" +if {$tests == "all" || $tests == "xtest"} { + info "Running: $cmd...\n" + expect "# " + send -- "$cmd\r" + check_test_result $teecore +} +if {$::env(XEN_BOOT) == "y"} { + info " Booting DomU.\n" + expect "# " + info "Running: $cmd1...\n" + send -- "$cmd1\r" + expect "# " + info "Running: $cmd2...\n" + send -- "$cmd2\r" + expect "# " + info "Running: $cmd3...\n" + send -- "$cmd3\r" + expect { + "Kernel panic" { + info "!!! Kernel panic\n" + exit 1 + } + timeout { + info "!!! Timeout\n" + exit 1 + } + "login:" + } + send -- "root\r\r" + expect "# " + info " done, DomU is booted.\n" + # Toolchain libraries might be here or there + send -- "export LD_LIBRARY_PATH=/lib:/lib/arm-linux-gnueabihf\r" + expect "# " + if {$tests == "all" || $tests == "xtest"} { + info "Running: $cmd...\n" + send -- "$cmd\r" + check_test_result $teecore + } +} +if {$tests == "all" || $tests == "trusted-keys"} { + # Invoke Trusted Keys tests + set basedir [file dirname $argv0] + source $basedir/trusted-keys.exp +} diff --git a/build/qemu.mk b/build/qemu.mk new file mode 100644 index 0000000..cf37705 --- /dev/null +++ b/build/qemu.mk @@ -0,0 +1,211 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +override COMPILE_NS_USER := 32 +override COMPILE_NS_KERNEL := 32 +override COMPILE_S_USER := 32 +override COMPILE_S_KERNEL := 32 + +BR2_ROOTFS_OVERLAY = $(ROOT)/build/br-ext/board/qemu/overlay +BR2_ROOTFS_POST_BUILD_SCRIPT = $(ROOT)/build/br-ext/board/qemu/post-build.sh +BR2_ROOTFS_POST_SCRIPT_ARGS = "$(QEMU_VIRTFS_AUTOMOUNT) $(QEMU_VIRTFS_MOUNTPOINT) $(QEMU_PSS_AUTOMOUNT)" + +OPTEE_OS_PLATFORM = vexpress-qemu_virt + +include common.mk + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +BINARIES_PATH ?= $(ROOT)/out/bin +U-BOOT_PATH ?= $(ROOT)/u-boot +QEMU_PATH ?= $(ROOT)/qemu +QEMU_BUILD ?= $(QEMU_PATH)/build + +DEBUG = 1 + +################################################################################ +# Targets +################################################################################ +all: arm-tf u-boot buildroot linux optee-os qemu +clean: arm-tf-clean u-boot-clean buildroot-clean linux-clean optee-os-clean \ + qemu-clean check-clean + +include toolchain.mk + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH32_CROSS_COMPILE)" + +TF_A_DEBUG ?= $(DEBUG) +ifeq ($(TF_A_DEBUG),0) +TF_A_LOGLVL ?= 30 +TF_A_OUT = $(TF_A_PATH)/build/qemu/release +else +TF_A_LOGLVL ?= 50 +TF_A_OUT = $(TF_A_PATH)/build/qemu/debug +endif + +TF_A_FLAGS ?= \ + BL32=$(OPTEE_OS_HEADER_V2_BIN) \ + BL32_EXTRA1=$(OPTEE_OS_PAGER_V2_BIN) \ + BL32_EXTRA2=$(OPTEE_OS_PAGEABLE_V2_BIN) \ + BL33=$(ROOT)/u-boot/u-boot.bin \ + ARM_ARCH_MAJOR=7 \ + ARCH=aarch32 \ + PLAT=qemu \ + ARM_TSP_RAM_LOCATION=tdram \ + BL32_RAM_LOCATION=tdram \ + AARCH32_SP=optee \ + DEBUG=$(TF_A_DEBUG) \ + LOG_LEVEL=$(TF_A_LOGLVL) \ + +arm-tf: optee-os u-boot + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + mkdir -p $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/bl1.bin $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/bl2.bin $(BINARIES_PATH) + ln -sf $(OPTEE_OS_HEADER_V2_BIN) $(BINARIES_PATH)/bl32.bin + ln -sf $(OPTEE_OS_PAGER_V2_BIN) $(BINARIES_PATH)/bl32_extra1.bin + ln -sf $(OPTEE_OS_PAGEABLE_V2_BIN) $(BINARIES_PATH)/bl32_extra2.bin + ln -sf $(ROOT)/u-boot/u-boot.bin $(BINARIES_PATH)/bl33.bin + +arm-tf-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# QEMU +################################################################################ +$(QEMU_BUILD)/config-host.mak: + cd $(QEMU_PATH); ./configure --target-list=arm-softmmu\ + $(QEMU_CONFIGURE_PARAMS_COMMON) + +qemu: $(QEMU_BUILD)/config-host.mak + $(MAKE) -C $(QEMU_PATH) + +qemu-clean: + $(MAKE) -C $(QEMU_PATH) distclean + +################################################################################ +# U-boot +################################################################################ +U-BOOT_EXPORTS ?= CROSS_COMPILE="$(CCACHE)$(AARCH32_CROSS_COMPILE)" + +U-BOOT_DEFCONFIG_FILES := \ + $(U-BOOT_PATH)/configs/qemu_arm_defconfig \ + $(ROOT)/build/kconfigs/u-boot_qemu_virt_v7.conf + +.PHONY: u-boot +u-boot: + cd $(U-BOOT_PATH) && \ + scripts/kconfig/merge_config.sh $(U-BOOT_DEFCONFIG_FILES) + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) all + +.PHONY: u-boot-clean +u-boot-clean: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) clean + +.PHONY: u-boot-cscope +u-boot-cscope: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) cscope + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm/configs/vexpress_defconfig \ + $(CURDIR)/kconfigs/qemu.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm zImage + +linux: linux-common + mkdir -p $(BINARIES_PATH) + ln -sf $(LINUX_PATH)/arch/arm/boot/zImage $(BINARIES_PATH) + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm + +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +optee-os: optee-os-common +optee-os-clean: optee-os-clean-common + +################################################################################ +# Run targets +################################################################################ +.PHONY: run +# This target enforces updating root fs etc +run: all + $(MAKE) run-only + +QEMU_SMP ?= 2 + +.PHONY: run-only +run-only: + ln -sf $(ROOT)/out-br/images/rootfs.cpio.gz $(BINARIES_PATH)/ + $(call check-terminal) + $(call run-help) + $(call launch-terminal,54320,"Normal World") + $(call launch-terminal,54321,"Secure World") + $(call wait-for-ports,54320,54321) + cd $(BINARIES_PATH) && $(QEMU_BUILD)/arm-softmmu/qemu-system-arm \ + -nographic \ + -serial tcp:127.0.0.1:54320 -serial tcp:127.0.0.1:54321 \ + -smp $(QEMU_SMP) \ + -s -S -machine virt,secure=on -cpu cortex-a15 \ + -d unimp -semihosting-config enable=on,target=native \ + -m 1057 \ + -bios bl1.bin \ + $(QEMU_EXTRA_ARGS) + +ifneq ($(filter check,$(MAKECMDGOALS)),) +CHECK_DEPS := all +endif + +check-args := --bios $(BINARIES_PATH)/bl1.bin +ifneq ($(TIMEOUT),) +check-args += --timeout $(TIMEOUT) +endif +ifneq ($(CHECK_TESTS),) +check-args += --tests $(CHECK_TESTS) +endif +ifneq ($(XTEST_ARGS),) +check-args += --xtest-args "$(XTEST_ARGS)" +endif + +check: $(CHECK_DEPS) + ln -sf $(ROOT)/out-br/images/rootfs.cpio.gz $(BINARIES_PATH)/ + cd $(BINARIES_PATH) && \ + export QEMU=$(QEMU_BUILD)/arm-softmmu/qemu-system-arm && \ + export QEMU_SMP=$(QEMU_SMP) && \ + export XEN_BOOT=n && \ + expect $(ROOT)/build/qemu-check.exp -- $(check-args) || \ + (if [ "$(DUMP_LOGS_ON_ERROR)" ]; then \ + echo "== $$PWD/serial0.log:"; \ + cat serial0.log; \ + echo "== end of $$PWD/serial0.log:"; \ + echo "== $$PWD/serial1.log:"; \ + cat serial1.log; \ + echo "== end of $$PWD/serial1.log:"; \ + fi; false) + +check-only: check + +check-clean: + rm -f serial0.log serial1.log diff --git a/build/qemu_v8.mk b/build/qemu_v8.mk new file mode 100644 index 0000000..d981b64 --- /dev/null +++ b/build/qemu_v8.mk @@ -0,0 +1,553 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +COMPILE_NS_USER ?= 64 +override COMPILE_NS_KERNEL := 64 +COMPILE_S_USER ?= 64 +COMPILE_S_KERNEL ?= 64 + +################################################################################ +# If you change this, you MUST run `make arm-tf-clean` first before rebuilding +################################################################################ +TF_A_TRUSTED_BOARD_BOOT ?= n + +BR2_ROOTFS_OVERLAY = $(ROOT)/build/br-ext/board/qemu/overlay +BR2_ROOTFS_POST_BUILD_SCRIPT = $(ROOT)/build/br-ext/board/qemu/post-build.sh +BR2_ROOTFS_POST_SCRIPT_ARGS = "$(QEMU_VIRTFS_AUTOMOUNT) $(QEMU_VIRTFS_MOUNTPOINT) $(QEMU_PSS_AUTOMOUNT)" + +OPTEE_OS_PLATFORM = vexpress-qemu_armv8a + +######################################################################################## +# If you change this, you MUST run `make arm-tf-clean optee-os-clean` before rebuilding +######################################################################################## +XEN_BOOT ?= n +ifeq ($(XEN_BOOT),y) +GICV3 = y +# For DomU, guest.cfg and other images can be picked up from mounted folder +QEMU_VIRTFS_AUTOMOUNT = y +endif + +include common.mk + +DEBUG ?= 1 + +# Option to build with GICV3 enabled +GICV3 ?= y + +# Option to configure FF-A and SPM: +# n: disabled +# 3: SPMC and SPMD at EL3 (in TF-A) +# 2: SPMC at S-EL2 (in Hafnium), SPMD at EL3 (in TF-A) +# 1: SPMC at S-EL1 (in OP-TEE), SPMD at EL3 (in TF-A) +SPMC_AT_EL ?= n +ifneq ($(filter-out n 1 2 3,$(SPMC_AT_EL)),) +$(error Unsupported SPMC_AT_EL value $(SPMC_AT_EL)) +endif + +# Option to configure Pointer Authentication for TA's +PAUTH ?= n + +# Option to configure Memory Tagging Extension +MEMTAG ?= n + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +BINARIES_PATH ?= $(ROOT)/out/bin +QEMU_PATH ?= $(ROOT)/qemu +QEMU_BUILD ?= $(QEMU_PATH)/build +MODULE_OUTPUT ?= $(ROOT)/out/kernel_modules +UBOOT_PATH ?= $(ROOT)/u-boot +UBOOT_BIN ?= $(UBOOT_PATH)/u-boot.bin +MKIMAGE_PATH ?= $(UBOOT_PATH)/tools +HAFNIUM_PATH ?= $(ROOT)/hafnium +HAFNIUM_BIN ?= $(HAFNIUM_PATH)/out/reference/secure_qemu_aarch64_clang/hafnium.bin + +ROOTFS_GZ ?= $(BINARIES_PATH)/rootfs.cpio.gz +ROOTFS_UGZ ?= $(BINARIES_PATH)/rootfs.cpio.uboot + +KERNEL_IMAGE ?= $(LINUX_PATH)/arch/arm64/boot/Image +KERNEL_IMAGEGZ ?= $(LINUX_PATH)/arch/arm64/boot/Image.gz +KERNEL_UIMAGE ?= $(BINARIES_PATH)/uImage + +# Load and entry addresses (u-boot only) +# If you change this please also change in kconfigs/u-boot_qemu_v8.conf +KERNEL_ENTRY ?= 0x42200000 +KERNEL_LOADADDR ?= 0x42200000 +ROOTFS_ENTRY ?= 0x45000000 +ROOTFS_LOADADDR ?= 0x45000000 + +ifeq ($(SPMC_AT_EL),2) +BL32_DEPS ?= hafnium optee-os +else +BL32_DEPS ?= optee-os +endif + +BL33_BIN ?= $(UBOOT_BIN) +BL33_DEPS ?= u-boot + +XEN_PATH ?= $(ROOT)/xen +XEN_IMAGE ?= $(ROOT)/out-br/build/xen-4.14.5/xen/xen.efi +XEN_EXT4 ?= $(BINARIES_PATH)/xen.ext4 +XEN_CFG ?= $(ROOT)/build/qemu_v8/xen/xen.cfg + +ifeq ($(GICV3),y) + TFA_GIC_DRIVER ?= QEMU_GICV3 + QEMU_GIC_VERSION = 3 +else + TFA_GIC_DRIVER ?= QEMU_GICV2 + QEMU_GIC_VERSION = 2 +endif + +################################################################################ +# Targets +################################################################################ +TARGET_DEPS := arm-tf buildroot linux optee-os qemu +TARGET_CLEAN := arm-tf-clean buildroot-clean linux-clean optee-os-clean \ + qemu-clean check-clean + +TARGET_DEPS += $(BL33_DEPS) + +TARGET_DEPS += $(KERNEL_UIMAGE) $(ROOTFS_UGZ) +TARGET_CLEAN += u-boot-clean + +ifeq ($(XEN_BOOT),y) +TARGET_DEPS += xen-create-image +endif + +all: $(TARGET_DEPS) + +clean: $(TARGET_CLEAN) + +$(BINARIES_PATH): + mkdir -p $@ + +include toolchain.mk + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +TF_A_DEBUG ?= $(DEBUG) +ifeq ($(TF_A_DEBUG),0) +TF_A_LOGLVL ?= 30 +TF_A_OUT = $(TF_A_PATH)/build/qemu/release +else +TF_A_LOGLVL ?= 40 +TF_A_OUT = $(TF_A_PATH)/build/qemu/debug +endif + +TF_A_FLAGS ?= \ + BL33=$(BL33_BIN) \ + PLAT=qemu \ + QEMU_USE_GIC_DRIVER=$(TFA_GIC_DRIVER) \ + ENABLE_SVE_FOR_NS=2 \ + ENABLE_SME_FOR_NS=2 \ + ENABLE_SVE_FOR_SWD=1 \ + ENABLE_SME_FOR_SWD=1 \ + ENABLE_FEAT_FGT=2 \ + BL32_RAM_LOCATION=tdram \ + DEBUG=$(TF_A_DEBUG) \ + LOG_LEVEL=$(TF_A_LOGLVL) + +TF_A_FLAGS_BL32_OPTEE = BL32=$(OPTEE_OS_HEADER_V2_BIN) +TF_A_FLAGS_BL32_OPTEE += BL32_EXTRA1=$(OPTEE_OS_PAGER_V2_BIN) +TF_A_FLAGS_BL32_OPTEE += BL32_EXTRA2=$(OPTEE_OS_PAGEABLE_V2_BIN) + +TF_A_FLAGS_SPMC_AT_EL_n = $(TF_A_FLAGS_BL32_OPTEE) SPD=opteed +TF_A_FLAGS_SPMC_AT_EL_1 = $(TF_A_FLAGS_BL32_OPTEE) SPD=spmd +TF_A_FLAGS_SPMC_AT_EL_1 += CTX_INCLUDE_EL2_REGS=0 SPMD_SPM_AT_SEL2=0 +TF_A_FLAGS_SPMC_AT_EL_1 += ENABLE_SME_FOR_NS=0 ENABLE_SME_FOR_SWD=0 +TF_A_FLAGS_SPMC_AT_EL_1 += QEMU_TOS_FW_CONFIG_DTS=../build/qemu_v8/spmc_el1_manifest.dts +TF_A_FLAGS_SPMC_AT_EL_1 += SPMC_OPTEE=1 +TF_A_FLAGS_SPMC_AT_EL_1 += QEMU_TOS_FW_CONFIG_DTS=../build/qemu_v8/spmc_el1_manifest.dts +TF_A_FLAGS_SPMC_AT_EL_2 = SPD=spmd +TF_A_FLAGS_SPMC_AT_EL_2 += ENABLE_SPE_FOR_LOWER_ELS=0 +TF_A_FLAGS_SPMC_AT_EL_2 += ENABLE_SME_FOR_NS=0 ENABLE_SME_FOR_SWD=0 +TF_A_FLAGS_SPMC_AT_EL_2 += ENABLE_FEAT_SEL2=1 +TF_A_FLAGS_SPMC_AT_EL_2 += SP_LAYOUT_FILE=../build/qemu_v8/sp_layout.json +TF_A_FLAGS_SPMC_AT_EL_2 += NEED_FDT=yes +TF_A_FLAGS_SPMC_AT_EL_2 += BL32=$(HAFNIUM_BIN) +TF_A_FLAGS_SPMC_AT_EL_2 += QEMU_TOS_FW_CONFIG_DTS=../build/qemu_v8/spmc_el2_manifest.dts +TF_A_FLAGS_SPMC_AT_EL_2 += QEMU_TB_FW_CONFIG_DTS=../build/qemu_v8/tb_fw_config.dts +ifneq ($(PAUTH),y) +TF_A_FLAGS_SPMC_AT_EL_2 += CTX_INCLUDE_PAUTH_REGS=1 +endif +ifneq ($(MEMTAG),y) +TF_A_FLAGS_SPMC_AT_EL_2 += CTX_INCLUDE_MTE_REGS=1 +endif +TF_A_FLAGS_SPMC_AT_EL_3 = SPD=spmd SPMC_AT_EL3=1 +TF_A_FLAGS_SPMC_AT_EL_3 += CTX_INCLUDE_EL2_REGS=0 SPMD_SPM_AT_SEL2=0 +TF_A_FLAGS_SPMC_AT_EL_3 += ENABLE_SME_FOR_NS=0 ENABLE_SME_FOR_SWD=0 +TF_A_FLAGS_SPMC_AT_EL_3 += BL32=$(OPTEE_OS_PAGER_V2_BIN) +TF_A_FLAGS_SPMC_AT_EL_3 += QEMU_TOS_FW_CONFIG_DTS=../build/qemu_v8/spmc_el3_manifest.dts + +TF_A_FLAGS += $(TF_A_FLAGS_SPMC_AT_EL_$(SPMC_AT_EL)) + +ifeq ($(TF_A_TRUSTED_BOARD_BOOT),y) +TF_A_FLAGS += \ + MBEDTLS_DIR=$(ROOT)/mbedtls \ + TRUSTED_BOARD_BOOT=1 \ + GENERATE_COT=1 +endif + +ifeq ($(PAUTH),y) +TF_A_FLAGS += CTX_INCLUDE_PAUTH_REGS=1 +endif +ifeq ($(MEMTAG),y) +TF_A_FLAGS += CTX_INCLUDE_MTE_REGS=1 +endif + +arm-tf: $(BL32_DEPS) $(BL33_DEPS) + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + mkdir -p $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/bl1.bin $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/bl2.bin $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/bl31.bin $(BINARIES_PATH) +ifeq ($(TF_A_TRUSTED_BOARD_BOOT),y) + ln -sf $(TF_A_OUT)/trusted_key.crt $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/tos_fw_key.crt $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/tos_fw_content.crt $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/tb_fw.crt $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/soc_fw_key.crt $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/soc_fw_content.crt $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/nt_fw_key.crt $(BINARIES_PATH) + ln -sf $(TF_A_OUT)/nt_fw_content.crt $(BINARIES_PATH) +endif + rm -f $(BINARIES_PATH)/bl32.bin + rm -f $(BINARIES_PATH)/bl32_extra1.bin + rm -f $(BINARIES_PATH)/bl32_extra2.bin + rm -f $(BINARIES_PATH)/tos_fw_config.dtb + rm -f $(BINARIES_PATH)/op-tee.pkg +ifeq ($(SPMC_AT_EL),1) + ln -sf $(TF_A_OUT)/fdts/spmc_el1_manifest.dtb \ + $(BINARIES_PATH)/tos_fw_config.dtb + ln -sf $(OPTEE_OS_HEADER_V2_BIN) $(BINARIES_PATH)/bl32.bin + ln -sf $(OPTEE_OS_PAGER_V2_BIN) $(BINARIES_PATH)/bl32_extra1.bin + ln -sf $(OPTEE_OS_PAGEABLE_V2_BIN) $(BINARIES_PATH)/bl32_extra2.bin +else ifeq ($(SPMC_AT_EL),2) + ln -sf $(TF_A_OUT)/fdts/spmc_el2_manifest.dtb \ + $(BINARIES_PATH)/tos_fw_config.dtb + ln -sf $(TF_A_OUT)/fdts/tb_fw_config.dtb \ + $(BINARIES_PATH)/tb_fw_config.dtb + ln -sf $(HAFNIUM_BIN) $(BINARIES_PATH)/bl32.bin + ln -sf $(TF_A_OUT)/op-tee.pkg $(BINARIES_PATH)/op-tee.pkg +else ifeq ($(SPMC_AT_EL),3) + ln -sf $(TF_A_OUT)/fdts/spmc_el3_manifest.dtb \ + $(BINARIES_PATH)/tos_fw_config.dtb + ln -sf $(OPTEE_OS_PAGER_V2_BIN) $(BINARIES_PATH)/bl32.bin +else + ln -sf $(OPTEE_OS_HEADER_V2_BIN) $(BINARIES_PATH)/bl32.bin + ln -sf $(OPTEE_OS_PAGER_V2_BIN) $(BINARIES_PATH)/bl32_extra1.bin + ln -sf $(OPTEE_OS_PAGEABLE_V2_BIN) $(BINARIES_PATH)/bl32_extra2.bin +endif + ln -sf $(BL33_BIN) $(BINARIES_PATH)/bl33.bin + +arm-tf-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# QEMU +################################################################################ +$(QEMU_BUILD)/config-host.mak: + cd $(QEMU_PATH); ./configure --target-list=aarch64-softmmu --enable-slirp\ + $(QEMU_CONFIGURE_PARAMS_COMMON) + +qemu: $(QEMU_BUILD)/config-host.mak + $(MAKE) -C $(QEMU_PATH) + +qemu-clean: + $(MAKE) -C $(QEMU_PATH) distclean + +################################################################################ +# U-Boot +################################################################################ +ifeq ($(XEN_BOOT),y) +UBOOT_DEFCONFIG_FILES := $(UBOOT_PATH)/configs/qemu_arm64_defconfig \ + $(ROOT)/build/kconfigs/u-boot_xen_qemu_v8.conf +else +UBOOT_DEFCONFIG_FILES := $(UBOOT_PATH)/configs/qemu_arm64_defconfig \ + $(ROOT)/build/kconfigs/u-boot_qemu_v8.conf +endif + +UBOOT_COMMON_FLAGS ?= CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) + +$(UBOOT_PATH)/.config: $(UBOOT_DEFCONFIG_FILES) + cd $(UBOOT_PATH) && \ + scripts/kconfig/merge_config.sh $(UBOOT_DEFCONFIG_FILES) + +.PHONY: u-boot-defconfig +u-boot-defconfig: $(UBOOT_PATH)/.config + +.PHONY: u-boot +u-boot: u-boot-defconfig + $(MAKE) -C $(UBOOT_PATH) $(UBOOT_COMMON_FLAGS) + +.PHONY: u-boot-clean +u-boot-clean: + $(MAKE) -C $(UBOOT_PATH) $(UBOOT_COMMON_FLAGS) distclean + +################################################################################ + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/qemu.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 Image scripts_gdb + +linux: linux-common + mkdir -p $(BINARIES_PATH) + ln -sf $(LINUX_PATH)/arch/arm64/boot/Image $(BINARIES_PATH) + +linux-modules: linux + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) modules + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$(MODULE_OUTPUT) modules_install + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_COMMON_FLAGS += DEBUG=$(DEBUG) CFG_ARM_GICV3=$(GICV3) +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_1 = CFG_CORE_SEL1_SPMC=y +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_2 = CFG_CORE_SEL2_SPMC=y +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_2 += CFG_ARM_GICV3=n CFG_CORE_HAFNIUM_INTC=y +# [0e00.0000 0e2f.ffff] is reserved to early boot and SPMC +# [0e30.0000 0e33.ffff] is reserved manifest etc (op-tee.pkg) +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_2 += CFG_TZDRAM_START=0x0e304000 +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_2 += CFG_TZDRAM_SIZE=0x00cfc000 +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_2 += CFG_CORE_WORKAROUND_NSITR_CACHE_PRIME=n +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_3 = CFG_CORE_EL3_SPMC=y +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_3 += CFG_DT_ADDR=0x40000000 +OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_3 += CFG_CORE_RESERVED_SHM=n + +ifeq ($(XEN_BOOT),y) +OPTEE_OS_COMMON_FLAGS += CFG_VIRTUALIZATION=y +endif + +ifeq ($(PAUTH),y) +OPTEE_OS_COMMON_FLAGS += CFG_TA_PAUTH=y +OPTEE_OS_COMMON_FLAGS += CFG_CORE_PAUTH=y +endif +ifeq ($(MEMTAG),y) +OPTEE_OS_COMMON_FLAGS += CFG_MEMTAG=y +endif + +ifneq ($(QEMU_SMP),) +CFG_TEE_CORE_NB_CORE ?= $(QEMU_SMP) +OPTEE_OS_COMMON_FLAGS += CFG_TEE_CORE_NB_CORE=$(CFG_TEE_CORE_NB_CORE) +endif + +OPTEE_OS_COMMON_FLAGS += $(OPTEE_OS_COMMON_FLAGS_SPMC_AT_EL_$(SPMC_AT_EL)) + +optee-os: optee-os-common + +optee-os-clean: optee-os-clean-common + +################################################################################ +# Hafnium +################################################################################ + +HAFNIUM_EXPORTS = PATH=$(HAFNIUM_PATH)/prebuilts/linux-x64/clang/bin:$(HAFNIUM_PATH)/prebuilts/linux-x64/dtc:$(PATH) + +.hafnium_checkout: + git -C $(HAFNIUM_PATH) submodule update --init + touch $@ + +hafnium: $(HAFNIUM_BIN) + +$(HAFNIUM_BIN): .hafnium_checkout | $(OUT_PATH) + $(HAFNIUM_EXPORTS) $(MAKE) -C $(HAFNIUM_PATH) $(HAFNIUM_FLAGS) all + + +################################################################################ +# mkimage - create images to be loaded by U-Boot +################################################################################ +# Without the objcopy, the uImage will be 10x bigger. +$(KERNEL_UIMAGE): u-boot linux | $(BINARIES_PATH) + ${AARCH64_CROSS_COMPILE}objcopy -O binary \ + -R .note \ + -R .comment \ + -S $(LINUX_PATH)/vmlinux \ + $(BINARIES_PATH)/linux.bin + $(MKIMAGE_PATH)/mkimage -A arm64 \ + -O linux \ + -T kernel \ + -C none \ + -a $(KERNEL_LOADADDR) \ + -e $(KERNEL_ENTRY) \ + -n "Linux kernel" \ + -d $(BINARIES_PATH)/linux.bin $(KERNEL_UIMAGE) + +.PHONY: uImage +uImage: $(KERNEL_UIMAGE) + +$(ROOTFS_UGZ): u-boot buildroot | $(BINARIES_PATH) + ln -sf $(ROOT)/out-br/images/rootfs.cpio.gz $(BINARIES_PATH) + $(MKIMAGE_PATH)/mkimage -A arm64 \ + -T ramdisk \ + -C gzip \ + -a $(ROOTFS_LOADADDR) \ + -e $(ROOTFS_ENTRY) \ + -n "Root file system" \ + -d $(ROOTFS_GZ) $(ROOTFS_UGZ) + +################################################################################ +# XEN +################################################################################ + +XEN_TMP ?= $(BINARIES_PATH)/xen_files + +$(XEN_TMP): + mkdir -p $@ + +xen-create-image: linux buildroot | $(XEN_TMP) + cp $(KERNEL_IMAGE) $(XEN_TMP) + cp $(XEN_IMAGE) $(XEN_TMP) + cp $(XEN_CFG) $(XEN_TMP) + cp $(ROOT)/out-br/images/rootfs.cpio.gz $(XEN_TMP) + rm -f $(XEN_EXT4) + mke2fs -t ext4 -d $(XEN_TMP) $(XEN_EXT4) 100M + + +################################################################################ +# Run targets +################################################################################ +.PHONY: run +# This target enforces updating root fs etc +run: all + $(MAKE) run-only + + +ifeq ($(XEN_BOOT),y) +QEMU_CPU ?= cortex-a57 +QEMU_MEM ?= 3072 +QEMU_SMP ?= 4 +QEMU_VIRT = true +QEMU_XEN ?= -drive if=none,file=$(XEN_EXT4),format=raw,id=hd1 \ + -device virtio-blk-device,drive=hd1 +else +ifeq ($(SPMC_AT_EL),2) +QEMU_VIRT = true +else +QEMU_VIRT = false +endif +ifeq ($(SPMC_AT_EL),n) +QEMU_SME = on +else +QEMU_SME = off +endif +QEMU_CPU ?= max,sme=$(QEMU_SME),pauth-impdef=on +QEMU_SMP ?= 2 +QEMU_MEM ?= 1057 +endif + +ifeq ($(MEMTAG),y) +QEMU_MTE = on +else ifeq ($(SPMC_AT_EL),2) +QEMU_MTE = on +else +QEMU_MTE = off +endif + +.PHONY: run-only +run-only: + ln -sf $(ROOT)/out-br/images/rootfs.cpio.gz $(BINARIES_PATH)/ + $(call check-terminal) + $(call run-help) + $(call launch-terminal,54320,"Normal World") + $(call launch-terminal,54321,"Secure World") + $(call wait-for-ports,54320,54321) + cd $(BINARIES_PATH) && $(QEMU_BUILD)/aarch64-softmmu/qemu-system-aarch64 \ + -nographic \ + -serial tcp:127.0.0.1:54320 -serial tcp:127.0.0.1:54321 \ + -smp $(QEMU_SMP) \ + -s -S -machine virt,acpi=off,secure=on,mte=$(QEMU_MTE),gic-version=$(QEMU_GIC_VERSION),virtualization=$(QEMU_VIRT) \ + -cpu $(QEMU_CPU) \ + -d unimp -semihosting-config enable=on,target=native \ + -m $(QEMU_MEM) \ + -bios bl1.bin \ + -initrd rootfs.cpio.gz \ + -kernel Image \ + -append 'console=ttyAMA0,38400 keep_bootcon root=/dev/vda2 $(QEMU_KERNEL_BOOTARGS)' \ + $(QEMU_XEN) \ + $(QEMU_EXTRA_ARGS) + +ifneq ($(filter check check-rust,$(MAKECMDGOALS)),) +CHECK_DEPS := all +endif + +ifneq ($(TIMEOUT),) +check-args := --timeout $(TIMEOUT) +endif +ifneq ($(CHECK_TESTS),) +check-args += --tests $(CHECK_TESTS) +endif +ifneq ($(XTEST_ARGS),) +check-args += --xtest-args "$(XTEST_ARGS)" +endif + +check: $(CHECK_DEPS) + ln -sf $(ROOT)/out-br/images/rootfs.cpio.gz $(BINARIES_PATH)/ + cd $(BINARIES_PATH) && \ + export QEMU=$(QEMU_BUILD)/aarch64-softmmu/qemu-system-aarch64 && \ + export QEMU_SMP=$(QEMU_SMP) && \ + export QEMU_MTE=$(QEMU_MTE) && \ + export QEMU_GIC=$(QEMU_GIC_VERSION) && \ + export QEMU_MEM=$(QEMU_MEM) && \ + export QEMU_CPU=$(QEMU_CPU) && \ + export QEMU_VIRT=$(QEMU_VIRT) && \ + export XEN_BOOT=$(XEN_BOOT) && \ + expect $(ROOT)/build/qemu-check.exp -- $(check-args) || \ + (if [ "$(DUMP_LOGS_ON_ERROR)" ]; then \ + echo "== $$PWD/serial0.log:"; \ + cat serial0.log; \ + echo "== end of $$PWD/serial0.log:"; \ + echo "== $$PWD/serial1.log:"; \ + cat serial1.log; \ + echo "== end of $$PWD/serial1.log:"; \ + fi; false) + +check-only: check + +check-rust: $(CHECK_DEPS) + ln -sf $(ROOT)/out-br/images/rootfs.cpio.gz $(BINARIES_PATH)/ + cd $(BINARIES_PATH) && \ + export QEMU=$(QEMU_BUILD)/aarch64-softmmu/qemu-system-aarch64 && \ + export QEMU_SMP=$(QEMU_SMP) && \ + export QEMU_MTE=$(QEMU_MTE) && \ + export QEMU_GIC=$(QEMU_GIC_VERSION) && \ + export QEMU_MEM=$(QEMU_MEM) && \ + expect $(ROOT)/optee_rust/ci/qemu-check.exp -- $(check-args) || \ + (if [ "$(DUMP_LOGS_ON_ERROR)" ]; then \ + echo "== $$PWD/serial0.log:"; \ + cat serial0.log; \ + echo "== end of $$PWD/serial0.log:"; \ + echo "== $$PWD/serial1.log:"; \ + cat serial1.log; \ + echo "== end of $$PWD/serial1.log:"; \ + fi; false) + +check-only-rust: check-rust + +check-clean: + rm -f serial0.log serial1.log diff --git a/build/qemu_v8/optee_sp_manifest.dts b/build/qemu_v8/optee_sp_manifest.dts new file mode 100644 index 0000000..024efe7 --- /dev/null +++ b/build/qemu_v8/optee_sp_manifest.dts @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020, Arm Limited. All rights reserved. + * Copyright (c) 2022, Linaro Limited + * + * SPDX-License-Identifier: BSD-3-Clause + * + * This file is a Partition Manifest (PM) for a minimal Secure Partition (SP) + * that has additional optional properties defined. + * + */ + +/dts-v1/; + +/ { + compatible = "arm,ffa-manifest-1.0"; + + /* Properties */ + description = "op-tee"; + ffa-version = <0x00010000>; /* 31:16 - Major, 15:0 - Minor */ + uuid = <0xe0786148 0xe311f8e7 0x02005ebc 0x1bc5d5a5>; + id = <1>; + execution-ctx-count = <8>; + exception-level = <2>; /* S-EL1 */ + execution-state = <0>; /* AARCH64 */ + load-address = <0xe300000>; + mem-size = <0xd00000>; /* OP-TEE specific extension */ + entrypoint-offset = <0x4000>; + xlat-granule = <0>; /* 4KiB */ + boot-order = <0>; + messaging-method = <0x3>; /* Direct messaging only */ + ns-interrupts-action = <1>; /* NS_ACTION_ME */ + + /* Boot protocol */ + gp-register-num = <0x0>; + + /* Boot Info */ + boot-info { + compatible = "arm,ffa-manifest-boot-info"; + ffa_manifest; + }; + + device-regions { + compatible = "arm,ffa-manifest-device-regions"; + + uart1 { + base-address = <0x00000000 0x09040000>; + pages-count = <1>; + attributes = <0x3>; /* read-write */ + /* SPI, level-triggered, secure, priority=1 */ + interrupts = <0x28 0xb01>; + }; + }; +}; diff --git a/build/qemu_v8/sp_layout.json b/build/qemu_v8/sp_layout.json new file mode 100644 index 0000000..c6a02a9 --- /dev/null +++ b/build/qemu_v8/sp_layout.json @@ -0,0 +1,6 @@ +{ + "op-tee" : { + "image": "../../optee_os/out/arm/core/tee-pager_v2.bin", + "pm": "optee_sp_manifest.dts" + } +} diff --git a/build/qemu_v8/spmc_el1_manifest.dts b/build/qemu_v8/spmc_el1_manifest.dts new file mode 100644 index 0000000..5c856d3 --- /dev/null +++ b/build/qemu_v8/spmc_el1_manifest.dts @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The Hafnium Authors. + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/BSD-3-Clause. + */ + +/dts-v1/; + +/ { + compatible = "arm,ffa-core-manifest-1.0"; + #address-cells = <2>; + #size-cells = <1>; + + attribute { + spmc_id = <0x8000>; + maj_ver = <0x1>; + min_ver = <0x0>; + exec_state = <0x0>; + load_address = <0x0 0xe100000>; + entrypoint = <0x0 0xe100000>; + binary_size = <0x80000>; + }; + + cpus { + #address-cells = <0x02>; + #size-cells = <0x00>; + + cpu@0 { + device_type = "cpu"; + reg = <0x0 0x0>; + }; + + cpu@3 { + device_type = "cpu"; + reg = <0x0 0x3>; + }; + + cpu@2 { + device_type = "cpu"; + reg = <0x0 0x2>; + }; + + cpu@1 { + device_type = "cpu"; + reg = <0x0 0x1>; + }; + }; + + /* VIRT_SECURE_MEM */ + memory@e000000 { + device_type = "memory"; + reg = <0x0 0xe000000 0x1000000>; + }; +}; diff --git a/build/qemu_v8/spmc_el2_manifest.dts b/build/qemu_v8/spmc_el2_manifest.dts new file mode 100644 index 0000000..7acaa38 --- /dev/null +++ b/build/qemu_v8/spmc_el2_manifest.dts @@ -0,0 +1,67 @@ +/* + * Copyright 2022 The Hafnium Authors. + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/BSD-3-Clause. + */ + +/dts-v1/; + +/ { + compatible = "arm,ffa-core-manifest-1.0"; + #address-cells = <2>; + #size-cells = <1>; + + attribute { + spmc_id = <0x8000>; + maj_ver = <0x1>; + min_ver = <0x1>; + exec_state = <0x0>; + load_address = <0x0 0xe100000>; + entrypoint = <0x0 0xe100000>; + binary_size = <0x80000>; + }; + + hypervisor { + compatible = "hafnium,hafnium"; + vm1 { + is_ffa_partition; + load_address = <0xe300000>; + debug_name = "op-tee"; + vcpu_count = <4>; + mem_size = <0xd00000>; + }; + }; + + cpus { + #address-cells = <0x02>; + #size-cells = <0x00>; + + cpu@0 { + device_type = "cpu"; + reg = <0x0 0x0>; + }; + + cpu@3 { + device_type = "cpu"; + reg = <0x0 0x3>; + }; + + cpu@2 { + device_type = "cpu"; + reg = <0x0 0x2>; + }; + + cpu@1 { + device_type = "cpu"; + reg = <0x0 0x1>; + }; + }; + + /* VIRT_SECURE_MEM */ + memory@e000000 { + device_type = "memory"; + reg = <0x0 0xe000000 0x1000000>; + }; +}; diff --git a/build/qemu_v8/spmc_el3_manifest.dts b/build/qemu_v8/spmc_el3_manifest.dts new file mode 100644 index 0000000..cb5d5c8 --- /dev/null +++ b/build/qemu_v8/spmc_el3_manifest.dts @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2020-2021, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +/dts-v1/; + +#define MODE_S_EL1 (0x2) + +/* For consumption by EL3 SPMC. */ +/ { + compatible = "arm,ffa-manifest-1.0"; + #address-cells = <2>; + #size-cells = <1>; + + ffa-version = <0x00010000>; /* 31:16 - Major, 15:0 - Minor */ + id = <0x8001>; + uuid = <0xe0786148 0xe311f8e7 0x02005ebc 0x1bc5d5a5>; + messaging-method = <1>; /* Direct Messaging Only */ + exception-level = ; + execution-state = <0>; + execution-ctx-count = <8>; + gp-register-num = <0>; + power-management-messages = <0x1>; /* Subscribe to CPU_OFF PM Msgs */ +}; diff --git a/build/qemu_v8/tb_fw_config.dts b/build/qemu_v8/tb_fw_config.dts new file mode 100644 index 0000000..68f7b3b --- /dev/null +++ b/build/qemu_v8/tb_fw_config.dts @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited. + */ + +#include + +/dts-v1/; + +/ { + secure-partitions { + compatible = "arm,sp"; + op-tee { + uuid = "486178e0-e7f8-11e3-bc5e-0002a5d5c51b"; + load-address = <0xe300000>; + }; + }; +}; diff --git a/build/qemu_v8/xen/guest.cfg b/build/qemu_v8/xen/guest.cfg new file mode 100644 index 0000000..903912b --- /dev/null +++ b/build/qemu_v8/xen/guest.cfg @@ -0,0 +1,7 @@ +kernel="/mnt/host/linux/arch/arm64/boot/Image" +memory=1024 +vcpus=1 +command="console=hvc0 earlycon=xenboot" +ramdisk = "/mnt/host/out-br/images/rootfs.cpio.gz" +name="domu" +tee="optee" diff --git a/build/qemu_v8/xen/xen.cfg b/build/qemu_v8/xen/xen.cfg new file mode 100644 index 0000000..4ce3d77 --- /dev/null +++ b/build/qemu_v8/xen/xen.cfg @@ -0,0 +1,7 @@ +[global] +default=buildroot + +[buildroot] +options=console=dtuart noreboot dom0_mem=1G +kernel=Image rw root=/dev/ram console=hvc0 earlycon=xenboot +ramdisk=rootfs.cpio.gz diff --git a/build/rockpi4.mk b/build/rockpi4.mk new file mode 100644 index 0000000..3ee7f34 --- /dev/null +++ b/build/rockpi4.mk @@ -0,0 +1,263 @@ +COMPILE_NS_USER ?= 64 +override COMPILE_NS_KERNEL := 64 +COMPILE_S_USER ?= 64 +COMPILE_S_KERNEL ?= 64 + +include common.mk + +DEBUG ?= 1 + +# Do not leave a partially downloaded binary in case wget fails midway +.DELETE_ON_ERROR: + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +BINARIES_PATH ?= $(ROOT)/out +UBOOT_PATH ?= $(ROOT)/u-boot +UBOOT_BIN ?= $(UBOOT_PATH)/u-boot.bin +ROOT_IMG ?= $(ROOT)/out-br/images/rootfs.ext2 +BOOT_IMG ?= $(ROOT)/out/rockpi4.img +RKDEVELOPTOOL_PATH ?= $(ROOT)/rkdeveloptool +RKDEVELOPTOOL_BIN ?= $(RKDEVELOPTOOL_PATH)/rkdeveloptool +LOADER_BIN ?= $(BINARIES_PATH)/rk3399_loader_v1.20.119.bin + +LINUX_MODULES ?= n + +BR2_TARGET_ROOTFS_CPIO = n +BR2_TARGET_ROOTFS_CPIO_GZIP = n +BR2_TARGET_ROOTFS_EXT2 = y +BR2_TARGET_GENERIC_GETTY_PORT = ttyS2 +ifeq ($(LINUX_MODULES),y) +# If modules are installed... +# ...enable automatic device detection and driver loading +BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV = y +# ...and configure eth0 automatically based on ifup helpers +BR2_PACKAGE_IFUPDOWN_SCRIPTS = y +BR2_SYSTEM_DHCP = eth0 +# An image with module takes more space +BR2_TARGET_ROOTFS_EXT2_SIZE = 384M +# Enable SSH daemon for remote login +BR2_PACKAGE_OPENSSH = y +BR2_PACKAGE_OPENSSH_SERVER = y +BR2_ROOTFS_POST_BUILD_SCRIPT = $(ROOT)/build/br-ext/board/rockpi4/post-build.sh +else +BR2_TARGET_ROOTFS_EXT2_SIZE = 112M +endif + +################################################################################ +# Targets +################################################################################ + +all: boot-img + +clean: buildroot-clean + +include toolchain.mk + +################################################################################ +# Arm Trusted Firmware-A +################################################################################ +TF_A_EXPORTS ?= CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" \ + M0_CROSS_COMPILE="$(CCACHE)$(AARCH32_CROSS_COMPILE)" + +TF_A_DEBUG ?= $(DEBUG) +ifeq ($(TF_A_DEBUG),0) +TF_A_LOGLVL ?= 30 +TF_A_OUT = $(TF_A_PATH)/build/rk3399/release +else +TF_A_LOGLVL ?= 40 +TF_A_OUT = $(TF_A_PATH)/build/rk3399/debug +endif + +TF_A_FLAGS ?= ARCH=aarch64 PLAT=rk3399 SPD=opteed DEBUG=$(TF_A_DEBUG) \ + LOG_LEVEL=$(TF_A_LOGLVL) + +.PHONY: tfa +tfa: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) bl31 + +.PHONY: tfa-clean +tfa-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +clean: tfa-clean + +################################################################################ +# U-Boot +################################################################################ +UBOOT_DEFCONFIG_FILES := $(UBOOT_PATH)/configs/rock-pi-4-rk3399_defconfig \ + $(ROOT)/build/kconfigs/u-boot_rockpi4.conf + +UBOOT_FLAGS ?= CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) \ + CC=$(CROSS_COMPILE_NS_KERNEL)gcc \ + HOSTCC="$(CCACHE) gcc" + +UBOOT_EXPORTS ?= BL31=$(TF_A_OUT)/bl31/bl31.elf TEE=$(OPTEE_OS_BIN) + +u-boot-defconfig: $(UBOOT_PATH)/.config + +$(UBOOT_PATH)/.config: $(UBOOT_DEFCONFIG_FILES) + cd $(UBOOT_PATH) && \ + scripts/kconfig/merge_config.sh $(UBOOT_DEFCONFIG_FILES) + +.PHONY: u-boot-defconfig + +.PHONY: u-boot +u-boot: $(UBOOT_PATH)/.config optee-os tfa + $(UBOOT_EXPORTS) $(MAKE) -C $(UBOOT_PATH) $(UBOOT_FLAGS) + +.PHONY: u-boot-clean +u-boot-clean: + $(UBOOT_EXPORTS) $(MAKE) -C $(UBOOT_PATH) $(UBOOT_FLAGS) distclean + +clean: u-boot-clean + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH ?= arm64 +LINUX_DEFCONFIG_COMMON_FILES ?= $(LINUX_PATH)/arch/arm64/configs/defconfig \ + $(CURDIR)/kconfigs/rockpi4.conf + +.PHONY: linux-defconfig +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 Image rockchip/rk3399-rock-pi-4b.dtb \ + $(if $(filter y,$(LINUX_MODULES)),modules) + +.PHONY: linux +linux: linux-common +ifeq ($(LINUX_MODULES),y) + $(MAKE) -C $(LINUX_PATH) ARCH=arm64 modules_install \ + INSTALL_MOD_PATH=$(BINARIES_PATH)/modules +endif + +$(LINUX_PATH)/arch/arm64/boot/Image.gz: linux + gzip -c $(LINUX_PATH)/arch/arm64/boot/Image >$@ + +.PHONY: linux-defconfig-clean +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +.PHONY: linux-clean +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +.PHONY: linux-cleaner +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_PLATFORM = rockchip-rk3399 +OPTEE_OS_COMMON_FLAGS += CFG_ENABLE_EMBEDDED_TESTS=y + +.PHONY: optee-os +optee-os: optee-os-common + +.PHONY: optee-os-clean +optee-os-clean: optee-os-clean-common + +clean: optee-os-clean + +################################################################################ +# Boot image, shall be copied to SD card +################################################################################ + +# U-Boot offset comes from CONFIG_SYS_MMCSD_RAW_MODE_U_BOOT_SECTOR=0x4000 +# Partition no. 5 ends at 12288 + BR2_TARGET_ROOTFS_EXT2_SIZE (in kiB) +# File size needs to be slightly bigger to accomodate for whatever meta-data +rootfs-size-kib := $(shell echo $(BR2_TARGET_ROOTFS_EXT2_SIZE) | sed 's/M/*1024/') +p5-end-kib := $(shell echo $$((12288 + $(rootfs-size-kib)))) +img-size-kib := $(shell echo $$(($(p5-end-kib) + 1024))) + +.PHONY: boot-img +boot-img: u-boot buildroot $(LINUX_PATH)/arch/arm64/boot/Image.gz + mkdir -p $(BINARIES_PATH) + rm -f $(BOOT_IMG) + truncate -s $(img-size-kib)KiB $(BOOT_IMG) + parted -s $(BOOT_IMG) \ + unit kiB \ + mklabel gpt \ + mkpart idbloader 32 4032 \ + mkpart primary fat32 4032 4096 \ + mkpart primary fat32 4096 8192 \ + mkpart uboot 8192 12288 \ + mkpart root fat32 12288 $(p5-end-kib) + sgdisk -u 5:17d61bff-8fdc-4089-b675-9be21b9f6ac7 $(BOOT_IMG) + dd if=$(UBOOT_PATH)/idbloader.img of=$(BOOT_IMG) bs=1kiB seek=32 conv=notrunc + dd if=$(UBOOT_PATH)/u-boot.itb of=$(BOOT_IMG) bs=1kiB seek=8192 conv=notrunc + e2mkdir $(ROOT_IMG):/boot + e2cp $(LINUX_PATH)/arch/arm64/boot/Image.gz $(ROOT_IMG):/boot + e2cp $(LINUX_PATH)/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4b.dtb $(ROOT_IMG):/boot +ifeq ($(LINUX_MODULES),y) + find $(BINARIES_PATH)/modules -type f | while read f; do e2cp -a $$f $(ROOT_IMG):$$(echo $$f | sed s@$(BINARIES_PATH)/modules@@); done +endif + dd if=$(ROOT_IMG) of=$(BOOT_IMG) bs=1kiB seek=12288 conv=notrunc + +.PHONY: boot-img-clean +boot-img-clean: + rm -f $(BOOT_IMG) + +clean: boot-img-clean + +################################################################################ +# rkdeveloptool +################################################################################ + +$(RKDEVELOPTOOL_PATH)/Makefile: + cd $(RKDEVELOPTOOL_PATH) && \ + autoreconf -i && \ + ./configure CXXFLAGS=-Wno-format-truncation + +$(RKDEVELOPTOOL_BIN): $(RKDEVELOPTOOL_PATH)/Makefile + $(MAKE) -C $(RKDEVELOPTOOL_PATH) + +rkdeveloptool: $(RKDEVELOPTOOL_BIN) + +rkdeveloptool-clean: + $(MAKE) -C $(RKDEVELOPTOOL_PATH) clean + +rkdeveloptool-distclean: + $(MAKE) -C $(RKDEVELOPTOOL_PATH) clean + +clean: rkdeveloptool-clean + +$(LOADER_BIN): + cd $(BINARIES_PATH) && \ + wget https://dl.radxa.com/rockpi/images/loader/$(notdir $(LOADER_BIN)) + +################################################################################ +# Flash the image via USB onto the onboard eMMC +################################################################################ + +define flash-help + @echo + @echo "Please connect the board to the computer via a USB cable." + @echo "The cable must be connected to the upper USB 3 (blue) port." + @echo "Then press and hold the mask ROM button (first one on the left" + @echo "under the HDMI connector), apply power and release the button." + @echo "(More details at https://wiki.radxa.com/Rockpi4/dev/usb-install)" + @echo + @read -r -p "Press enter to continue, Ctrl-C to cancel:" dummy +endef + +flash: $(BOOT_IMG) $(LOADER_BIN) $(RKDEVELOPTOOL_BIN) + $(call flash-help) + $(RKDEVELOPTOOL_BIN) db $(LOADER_BIN) + sleep 1 + $(RKDEVELOPTOOL_BIN) wl 0 $(BOOT_IMG) + +nuke-emmc: $(LOADER_BIN) $(RKDEVELOPTOOL_BIN) + @echo + @echo "** WARNING: this command will make the onboard eMMC unbootable!" + @echo "It can be used to boot from the SD card again." + $(call flash-help) + dd if=/dev/zero of=$(BINARIES_PATH)/zero.img bs=1M count=64 + $(RKDEVELOPTOOL_BIN) db $(LOADER_BIN) + sleep 1 + $(RKDEVELOPTOOL_BIN) wl 0 $(BINARIES_PATH)/zero.img diff --git a/build/rpi3.mk b/build/rpi3.mk new file mode 100644 index 0000000..85bb232 --- /dev/null +++ b/build/rpi3.mk @@ -0,0 +1,207 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +override COMPILE_NS_USER := 64 +override COMPILE_NS_KERNEL := 64 +override COMPILE_S_USER := 64 +override COMPILE_S_KERNEL := 64 + +# Need to set this before including common.mk +BR2_TARGET_GENERIC_GETTY_PORT ?= ttyS0 +BR2_ROOTFS_POST_BUILD_SCRIPT ?= "board/raspberrypi3-64/post-build.sh" + +# Network support related packages: +BR2_PACKAGE_DHCPCD ?= y +BR2_PACKAGE_ETHTOOL ?= y +BR2_PACKAGE_XINETD ?= y + +# SSH Packages : +BR2_PACKAGE_OPENSSH ?= y +BR2_PACKAGE_OPENSSH_SERVER ?= y +BR2_PACKAGE_OPENSSH_KEY_UTILS ?= y + +OPTEE_OS_PLATFORM = rpi3 + +include common.mk + +# Required tools to create the SD image +BR2_PACKAGE_HOST_GENIMAGE=y + +# We need the ext2/4 image to be generated, so we will be able to copy that +# directly into a parition on the image. +BR2_TARGET_ROOTFS_EXT2=y +BR2_TARGET_ROOTFS_EXT2_4=y +BR2_TARGET_ROOTFS_EXT2_SIZE="128M" + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +TF_A_OUT ?= $(TF_A_PATH)/build/rpi3/debug +TF_A_BOOT ?= $(TF_A_OUT)/armstub8.bin + +OPTEE_PATH ?= $(ROOT)/optee_os +U-BOOT_PATH ?= $(ROOT)/u-boot +U-BOOT_BIN ?= $(U-BOOT_PATH)/u-boot.bin + +RPI3_FIRMWARE_PATH ?= $(BUILD_PATH)/rpi3/firmware +RPI3_BOOT_CONFIG ?= $(RPI3_FIRMWARE_PATH)/config.txt +RPI3_UBOOT_ENV ?= $(ROOT)/out/uboot.env +RPI3_UBOOT_ENV_TXT ?= $(RPI3_FIRMWARE_PATH)/uboot.env.txt +RPI3_STOCK_FW_PATH ?= $(ROOT)/firmware +RPI3_STOCK_FW_PATH_BOOT ?= $(RPI3_STOCK_FW_PATH)/boot +OPTEE_BIN ?= $(OPTEE_PATH)/out/arm/core/tee-header_v2.bin +OPTEE_BIN_EXTRA1 ?= $(OPTEE_PATH)/out/arm/core/tee-pager_v2.bin +OPTEE_BIN_EXTRA2 ?= $(OPTEE_PATH)/out/arm/core/tee-pageable_v2.bin +BOOT_PARTITION_FILES ?= $(ROOT)/out/boot +CREATE_IMAGE ?= $(BUILD_PATH)/rpi3/scripts/create-image.sh + +LINUX_IMAGE ?= $(LINUX_PATH)/arch/arm64/boot/Image +LINUX_DTB_RPI3_B ?= $(LINUX_PATH)/arch/arm64/boot/dts/broadcom/bcm2710-rpi-3-b.dtb +LINUX_DTB_RPI3_BPLUS ?= $(LINUX_PATH)/arch/arm64/boot/dts/broadcom/bcm2710-rpi-3-b-plus.dtb +MODULE_OUTPUT ?= $(ROOT)/module_output + +################################################################################ +# Targets +################################################################################ +all: arm-tf buildroot optee-os u-boot linux update_bootfs update_rootfs \ + sdcard-image +clean: arm-tf-clean buildroot-clean u-boot-clean \ + optee-os-clean + +include toolchain.mk + +################################################################################ +# ARM Trusted Firmware +################################################################################ +TF_A_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +TF_A_FLAGS ?= \ + NEED_BL32=yes \ + BL32=$(OPTEE_BIN) \ + BL32_EXTRA1=$(OPTEE_BIN_EXTRA1) \ + BL32_EXTRA2=$(OPTEE_BIN_EXTRA2) \ + BL33=$(U-BOOT_BIN) \ + DEBUG=1 \ + V=0 \ + CRASH_REPORTING=1 \ + LOG_LEVEL=40 \ + PLAT=rpi3 \ + RPI3_PRELOADED_DTB_BASE=0x00010000 \ + SPD=opteed + +arm-tf: optee-os u-boot-env + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) all fip + +arm-tf-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# Das U-Boot +################################################################################ +U-BOOT_EXPORTS ?= CROSS_COMPILE=$(AARCH64_CROSS_COMPILE) ARCH=arm64 +U-BOOT_DEFCONFIG_COMMON_FILES := \ + $(U-BOOT_PATH)/configs/rpi_3_defconfig \ + $(CURDIR)/kconfigs/u-boot_rpi3.conf + +.PHONY: u-boot +u-boot: u-boot-defconfig + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) all + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) tools + +u-boot-clean: u-boot-defconfig-clean + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) clean + +u-boot-env: $(RPI3_UBOOT_ENV_TXT) u-boot + mkdir -p $(ROOT)/out + $(U-BOOT_PATH)/tools/mkenvimage -s 0x4000 -o $(RPI3_UBOOT_ENV) \ + $(RPI3_UBOOT_ENV_TXT) + +u-boot-env-clean: + rm -f $(RPI3_UBOOT_ENV) + +u-boot-defconfig: $(U-BOOT_DEFCONFIG_COMMON_FILES) + cd $(U-BOOT_PATH) && \ + ARCH=arm64 \ + scripts/kconfig/merge_config.sh $(U-BOOT_DEFCONFIG_COMMON_FILES) + +.PHONY: u-boot-defconfig-clean +u-boot-defconfig-clean: + rm -f $(U-BOOT_PATH)/.config + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/bcmrpi3_defconfig \ + $(CURDIR)/kconfigs/rpi3.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 + +linux: linux-common + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$(MODULE_OUTPUT) modules_install + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +################################################################################ +# OP-TEE +################################################################################ +optee-os: optee-os-common +optee-os-clean: optee-os-clean-common + +################################################################################ +# Root FS +################################################################################ +.PHONY: update_rootfs +# Make sure this is built before the buildroot target which will create the +# root file system based on what's in $(BUILDROOT_TARGET_ROOT) +buildroot: update_rootfs +update_rootfs: linux + @mkdir -p --mode=755 $(BUILDROOT_TARGET_ROOT)/usr/bin + @cd $(MODULE_OUTPUT) && find . | cpio -pudm $(BUILDROOT_TARGET_ROOT) + +update_bootfs: arm-tf u-boot + @mkdir -p --mode=755 $(BOOT_PARTITION_FILES) + @install -v -p --mode=755 $(LINUX_DTB_RPI3_B) $(BOOT_PARTITION_FILES)/bcm2710-rpi-3-b.dtb + @install -v -p --mode=755 $(LINUX_DTB_RPI3_BPLUS) $(BOOT_PARTITION_FILES)/bcm2710-rpi-3-b-plus.dtb + @install -v -p --mode=755 $(RPI3_BOOT_CONFIG) $(BOOT_PARTITION_FILES)/config.txt + @install -v -p --mode=755 $(LINUX_IMAGE) $(BOOT_PARTITION_FILES)/kernel8.img + @install -v -p --mode=755 $(TF_A_BOOT) $(BOOT_PARTITION_FILES)/armstub8.bin + @install -v -p --mode=755 $(RPI3_UBOOT_ENV) $(BOOT_PARTITION_FILES)/uboot.env + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/bootcode.bin $(BOOT_PARTITION_FILES)/bootcode.bin + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/COPYING.linux $(BOOT_PARTITION_FILES)/COPYING.linux + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/fixup_cd.dat $(BOOT_PARTITION_FILES)/fixup_cd.dat + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/fixup.dat $(BOOT_PARTITION_FILES)/fixup.dat + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/fixup_db.dat $(BOOT_PARTITION_FILES)/fixup_db.dat + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/fixup_x.dat $(BOOT_PARTITION_FILES)/fixup_x.dat + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/LICENCE.broadcom $(BOOT_PARTITION_FILES)/LICENCE.broadcom + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/start_cd.elf $(BOOT_PARTITION_FILES)/start_cd.elf + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/start_db.elf $(BOOT_PARTITION_FILES)/start_db.elf + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/start.elf $(BOOT_PARTITION_FILES)/start.elf + @install -v -p --mode=755 $(RPI3_STOCK_FW_PATH)/boot/start_x.elf $(BOOT_PARTITION_FILES)/start_x.elf + +.PHONY: sdcard-image +sdcard-image: update_bootfs update_rootfs buildroot + $(CREATE_IMAGE) -w $(ROOT) + +# Creating images etc, could wipe out a drive on the system, therefore we don't +# want to automate that in script or make target. Instead we just simply provide +# the steps here. +.PHONY: img-help +img-help: + @echo "Use 'dmesg' to find your device/SD-card name, then run the following as root:" + @echo " $$ sudo dd if=$(ROOT)/out/rpi3-sdcard.img of=/dev/ bs=1024k conv=fsync status=progress" diff --git a/build/rpi3/debugger/.gdbinit b/build/rpi3/debugger/.gdbinit new file mode 100644 index 0000000..62d82a0 --- /dev/null +++ b/build/rpi3/debugger/.gdbinit @@ -0,0 +1,28 @@ +target extended-remote localhost:3333 + +# to debug single core: +# in u-boot: setenv smp off ; boot + +# substitute appropriate value from u-boot startup message (relocation offset) +add-symbol-file ../../../u-boot/u-boot 0x3af3a000 +add-symbol-file ../../../trusted-firmware-a/build/rpi3/debug/bl31/bl31.elf 0x08400000 +add-symbol-file ../../../optee_os/out/arm/core/tee.elf 0x08420000 +add-symbol-file ../../../linux/vmlinux 0xffffffc000080800 + +mon gdb_breakpoint_override hard + +flushregs + +# bl31_entrypoint +#b *0x08400000 + +# TEE entrypoint +#b *0x08420000 + +# Linux; primary core entry. +# Symbols for initial setup _do not align with vmlinux (elf)_. Debug at instruction +# level only. +# Symbols post MMU enable are correct. +#b *0x080000 + +mon halt diff --git a/build/rpi3/firmware/config.txt b/build/rpi3/firmware/config.txt new file mode 100644 index 0000000..a9cf780 --- /dev/null +++ b/build/rpi3/firmware/config.txt @@ -0,0 +1,3 @@ +enable_uart=1 +kernel_address=0x02000000 +device_tree_address=0x01000000 diff --git a/build/rpi3/firmware/uboot.env.txt b/build/rpi3/firmware/uboot.env.txt new file mode 100644 index 0000000..658b7dd --- /dev/null +++ b/build/rpi3/firmware/uboot.env.txt @@ -0,0 +1,36 @@ +# generic params +bootdelay=3 +stderr=serial,lcd +stdin=serial,usbkbd +stdout=serial,lcd + +# CPU config +cpu=armv8 +smp=on + +# Console config +baudrate=115200 +sttyconsole=ttyS0 +ttyconsole=tty0 + +# Kernel/dtb filenames & load addresses +boot_it=booti ${kernel_addr_r} - ${fdt_addr_r} +fdt_addr_r=0x01000000 +kernel_addr_r=0x02000000 + +# NFS/TFTP boot configuraton +gatewayip=192.168.1.1 +netmask=255.255.255.0 +nfsserverip=192.168.1.100 +nfspath=/srv/nfs/rpi + +# bootcmd & bootargs configuration +preboot=usb start +bootcmd=run mmcboot +load_kernel=fatload mmc 0:1 ${kernel_addr_r} kernel8.img +mmcboot=run load_kernel; run set_bootargs_tty set_bootargs_mmc set_common_args; run boot_it +nfsboot=run load_kernel; run set_bootargs_tty set_bootargs_nfs set_common_args; run boot_it +set_bootargs_tty=setenv bootargs console=${ttyconsole} console=${sttyconsole},${baudrate} +set_bootargs_nfs=setenv bootargs ${bootargs} root=/dev/nfs rw rootfstype=nfs nfsroot=${nfsserverip}:${nfspath},udp,vers=3 ip=dhcp +set_bootargs_mmc=setenv bootargs ${bootargs} root=/dev/mmcblk0p2 rw rootfs=ext4 +set_common_args=setenv bootargs ${bootargs} smsc95xx.macaddr=${ethaddr} 'ignore_loglevel dma.dmachans=0x7f35 rootwait 8250.nr_uarts=1 elevator=deadline fsck.repair=yes bcm2708_fb.fbwidth=1920 bcm2708_fb.fbheight=1080 vc_mem.mem_base=0x3ec00000 vc_mem.mem_size=0x40000000 dwc_otg.fiq_enable=0 dwc_otg.fiq_fsm_enable=0 dwc_otg.nak_holdoff=0' diff --git a/build/rpi3/scripts/create-image.sh b/build/rpi3/scripts/create-image.sh new file mode 100755 index 0000000..e79ca82 --- /dev/null +++ b/build/rpi3/scripts/create-image.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Helper script to make a SD-card image for the official OP-TEE Raspberry Pi 3 +# build. + +set -e +WORKDIR="" + +display_usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " -w|--workdir : The OP-TEE RPI3 working directory" + echo " -h|--help : Display this help message" +} + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -w | --workdir) + WORKDIR="$2" + shift 2 + ;; + -h | --help) + display_usage + exit 0 + ;; + *) + echo "Invalid option: $1" + display_usage + exit 1 + ;; + esac +done + +# No point to continue if files/folders we depend on are not existing. +check_exist() { + local dir="$1" + if [ ! -e "${dir}" ]; then + echo "Folder ${dir} does not exist. Exiting..." + exit 1 + fi +} + +check_exist ${WORKDIR} + +IMAGE_FILE=${WORKDIR}/out/rpi3-sdcard.img +rm -f ${IMAGE_FILE} + +################################################################################ +# Configuration +################################################################################ +OFFSET_KIB=1024 +BOOT_PARTITION_SIZE_MB=64 +# ROOT FS size is read out from the rootfs file created by Buildroot, therefore +# we don't need to configure that here. + +################################################################################ +# Image and partition generation +################################################################################ +# In bytes, always 512, when creating msdos/MBR images. +SECTOR_SIZE=512 + +# 1. Compute the sector size for the offset to the first partition. +OFFSET_SECTOR_SIZE=$((OFFSET_KIB * 1024 / SECTOR_SIZE)) +echo "OFFSET_SECTOR_SIZE: $((OFFSET_KIB * 1024)) bytes ($((OFFSET_KIB / 1024))MB in ${OFFSET_SECTOR_SIZE} sectors)" + +# 2. Calculate the boot partition sizes. +BOOT_PARTITION_SIZE_KIB=$((BOOT_PARTITION_SIZE_MB * 1024)) +BOOT_PARTITION_SIZE_BYTES=$((BOOT_PARTITION_SIZE_KIB * 1024)) +BOOT_PARTITION_SECTOR_SIZE=$((BOOT_PARTITION_SIZE_BYTES / SECTOR_SIZE)) +echo "BOOT PARTITION SIZE: ${BOOT_PARTITION_SIZE_BYTES} bytes ($(($BOOT_PARTITION_SIZE_BYTES / 1024 / 1024))MB in ${BOOT_PARTITION_SECTOR_SIZE} sectors)" + +# 3. Find out the size of the rootfs produced by Buildroot and calculate the +# rootfs partition sizes. +ROOTFS_IMG=${WORKDIR}/out-br/images/rootfs.ext2 +check_exist ${ROOTFS_IMG} +ROOTFS_SIZE_BYTES=$(stat -c %s ${ROOTFS_IMG}) +ROOTFS_SECTOR_SIZE=$((ROOTFS_SIZE_BYTES / SECTOR_SIZE)) +echo "ROOTFS PARTITION SIZE: ${ROOTFS_SIZE_BYTES} bytes ($((ROOTFS_SIZE_BYTES / 1024 / 1024))MB in ${ROOTFS_SECTOR_SIZE} sectors)" + +# 4. Compute the images size, based on the rootfs size, the +# desired boot image size and the offset to the first partition. +IMG_SECTOR_SIZE=$((OFFSET_SECTOR_SIZE + BOOT_PARTITION_SECTOR_SIZE + ROOTFS_SECTOR_SIZE)) +IMG_SIZE_KIB=$((IMG_SECTOR_SIZE * SECTOR_SIZE / 1024)) +echo "TOTAL IMAGE SIZE: ${IMG_SIZE_KIB} bytes ($((IMG_SIZE_KIB / 1024))MB in ${IMG_SECTOR_SIZE} sectors)" + +# Create empty disk image +truncate -s ${IMG_SIZE_KIB}KiB ${IMAGE_FILE} + +parted -s ${IMAGE_FILE} unit kiB mklabel msdos +parted -a optimal ${IMAGE_FILE} unit kiB mkpart primary fat16 ${OFFSET_KIB} ${BOOT_PARTITION_SIZE_KIB} +parted -a optimal ${IMAGE_FILE} set 1 boot on +parted -a optimal ${IMAGE_FILE} unit kiB mkpart primary ext4 $((OFFSET_KIB + BOOT_PARTITION_SIZE_KIB)) 100% + +# Show some useful information about the image we just created. +echo "" +fdisk -l ${IMAGE_FILE} + +################################################################################ +# Copy the ext2 partition from Buildroot to the second partition. +################################################################################ +dd if=${ROOTFS_IMG} of=${IMAGE_FILE} bs=${SECTOR_SIZE} seek=$((OFFSET_SECTOR_SIZE + BOOT_PARTITION_SECTOR_SIZE)) conv=notrunc,fsync + +################################################################################ +# Create the boot image and copy it to the boot partition +################################################################################ +GENIMAGE=${WORKDIR}/out-br/host/bin/genimage +check_exist ${GENIMAGE} + +# Used for a temporary genimage config file. +RPI3_GENIMAGE_CFG=$(mktemp) + +# Used for temporarily storing files when creating the boot vfat image. +RPI3_GENIMAGE_TMPPATH=$(mktemp -d) +echo "RPI3_GENIMAGE_CFG: ${RPI3_GENIMAGE_CFG}" +echo "RPI3_GENIMAGE_TMPPATH: ${RPI3_GENIMAGE_TMPPATH}" + +trap "rm -rf ${RPI3_GENIMAGE_CFG} ${RPI3_GENIMAGE_TMPPATH}" EXIT + +BOOT_PARTITION_FILES=${WORKDIR}/out/boot +check_exist ${BOOT_PARTITION_FILES} + +# Create the config file for genimage on the fly. +cat <${RPI3_GENIMAGE_CFG} +image boot.vfat { + vfat { + } + + size = ${BOOT_PARTITION_SIZE_MB}M + mountpoint = "/" +} +EOF + +# Create the boot.vfat image. +${GENIMAGE} --rootpath ${BOOT_PARTITION_FILES} --config ${RPI3_GENIMAGE_CFG} --tmppath=${RPI3_GENIMAGE_TMPPATH} --outputpath=${WORKDIR}/out + +# Copy the contents of boot.vfat to the boot partition in the image we prepared. +dd if=${WORKDIR}/out/boot.vfat of=${IMAGE_FILE} bs=${SECTOR_SIZE} seek=${OFFSET_SECTOR_SIZE} conv=notrunc,fsync diff --git a/build/soc_term.py b/build/soc_term.py new file mode 100755 index 0000000..5178b60 --- /dev/null +++ b/build/soc_term.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2014, Linaro Limited +# Copyright (c) 2021, Huawei Technologies Co., Ltd +# + +import argparse +import os +import select +import socket +import sys +import termios + +handle_telnet = False +cmd_bytes = bytearray() + +TELNET_IAC = 0xff +TELNET_DO = 0xfd +TELNET_WILL = 0xfb +TELNET_SUPRESS_GO_AHEAD = 0x1 + + +def get_args(): + + parser = argparse.ArgumentParser(description='Starts a TCP server to be ' + 'used as a terminal (for QEMU or FVP). ' + 'When the server receives a connection ' + 'it puts the terminal in raw mode so ' + 'that control characters (Ctrl-C etc.) ' + 'are interpreted remotely. Only when the ' + 'peer has closed the connection the ' + 'terminal settings are restored.') + parser.add_argument('port', nargs=1, type=int, + help='local TCP port to listen on') + parser.add_argument('-t', '--telnet', action='store_true', + help='handle telnet commands (FVP)') + return parser.parse_args() + + +def set_stty_noncanonical(): + + t = termios.tcgetattr(sys.stdin.fileno()) + # iflag + t[0] = t[0] & ~termios.ICRNL + # lflag + t[3] = t[3] & ~(termios.ICANON | termios.ECHO | termios.ISIG) + t[6][termios.VMIN] = 1 # Character-at-a-time input + t[6][termios.VTIME] = 0 # with blocking + termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, t) + + +def handle_telnet_codes(fd, buf): + + global handle_telnet + global cmd_bytes + + if (not handle_telnet): + return + + if (fd == -1): + cmd_bytes.clear() + return + + # Iterate on a copy because buf is modified in the loop + for c in bytearray(buf): + if (len(cmd_bytes) or c == TELNET_IAC): + cmd_bytes.append(c) + del buf[0] + if (len(cmd_bytes) == 3): + if (cmd_bytes[1] == TELNET_DO): + cmd_bytes[1] = TELNET_WILL + elif (cmd_bytes[1] == TELNET_WILL): + if (cmd_bytes[2] == TELNET_SUPRESS_GO_AHEAD): + # We're done after responding to this + handle_telnet = False + cmd_bytes[1] = TELNET_DO + else: + # Unknown command, ignore it + cmd_bytes.clear() + if (len(cmd_bytes)): + os.write(fd, cmd_bytes) + cmd_bytes.clear() + + +def serve_conn(conn): + + fd = conn.fileno() + poll = select.poll() + poll.register(sys.stdin.fileno(), select.POLLIN) + poll.register(fd, select.POLLIN) + while (True): + for readyfd, _ in poll.poll(): + try: + data = os.read(readyfd, 512) + if (len(data) == 0): + print('soc_term: read fd EOF') + return + buf = bytearray(data) + handle_telnet_codes(readyfd, buf) + if (readyfd == fd): + to = sys.stdout.fileno() + else: + to = fd + except ConnectionResetError: + print('soc_term: connection reset') + return + try: + # Python >= 3.5 handles EINTR internally so no loop required + os.write(to, buf) + except WriteErrorException: + print('soc_term: write error') + return + + +def main(): + + global handle_telnet + args = get_args() + port = args.port[0] + sock = socket.socket() + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('127.0.0.1', port)) + sock.listen(5) + print(f'listening on port {port}') + if (args.telnet): + print('Handling telnet commands') + old_term = termios.tcgetattr(sys.stdin.fileno()) + while True: + try: + conn, _ = sock.accept() + print(f'soc_term: accepted fd {conn.fileno()}') + handle_telnet = args.telnet + handle_telnet_codes(-1, bytearray()) # Reset internal state + set_stty_noncanonical() + serve_conn(conn) + conn.close() + except KeyboardInterrupt: + return + finally: + termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, old_term) + + +if __name__ == "__main__": + main() diff --git a/build/stm32mp1.mk b/build/stm32mp1.mk new file mode 100644 index 0000000..dd8f32a --- /dev/null +++ b/build/stm32mp1.mk @@ -0,0 +1,235 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +override COMPILE_NS_USER := 32 +override COMPILE_NS_KERNEL := 32 +override COMPILE_S_USER := 32 +override COMPILE_S_KERNEL := 32 + +PLATFORM ?= stm32mp1-157C_DK2 +OPTEE_OS_PLATFORM := $(PLATFORM) + +include common.mk + +ifeq ($(PLATFORM),stm32mp1-157A_DK1) +BREXT_FLAVOR = STM32MP157A-DK1 +STM32MP1_DTS_BASENAME = stm32mp157a-dk1 +else ifeq ($(PLATFORM),stm32mp1-157A_DHCOR_AVENGER96) +BREXT_FLAVOR = STM32MP157A-DHCOR-AVENGER96 +STM32MP1_DTS_BASENAME = stm32mp157a-dhcor-avenger96 +STM32MP1_DTS_U_BOOT = stm32mp15xx-dhcor-avenger96 +else ifeq ($(PLATFORM),stm32mp1-157C_DHCOM_PDK2) +BREXT_FLAVOR = STM32MP157C-DHCOM-PDK2 +STM32MP1_DTS_BASENAME = stm32mp157c-dhcom-pdk2 +STM32MP1_DTS_U_BOOT = stm32mp15xx-dhcom-pdk2 +else ifeq ($(PLATFORM),stm32mp1-157C_DK2) +BREXT_FLAVOR = STM32MP157C-DK2 +STM32MP1_DTS_BASENAME = stm32mp157c-dk2 +else ifeq ($(PLATFORM),stm32mp1-157C_EV1) +BREXT_FLAVOR = STM32MP157C-EV1 +STM32MP1_DTS_BASENAME = stm32mp157c-ev1 +else ifeq ($(PLATFORM),stm32mp1-157C_ED1) +BREXT_FLAVOR = STM32MP157C-ED1 +STM32MP1_DTS_BASENAME = stm32mp157c-ed1 +else ifeq ($(PLATFORM),stm32mp1-135F_DK) +BREXT_FLAVOR = STM32MP135F-DK +STM32MP1_DTS_BASENAME = stm32mp135f-dk +STM32MP1_DEFCONFIG_U_BOOT = stm32mp13_defconfig +else +$(error Unknown PLATFORM $(PLATFORM)) +endif + +STM32MP1_DTS_U_BOOT ?= $(STM32MP1_DTS_BASENAME) +STM32MP1_DEFCONFIG_U_BOOT ?= stm32mp15_defconfig + +################################################################################ +# Binary images names +################################################################################ + +TFA_BIN := tf-a-$(STM32MP1_DTS_BASENAME).stm32 +TFA_FIP_BIN := fip.bin +OPTEE_HEADER_BIN := tee-header_v2.bin +OPTEE_PAGER_BIN := tee-pager_v2.bin +OPTEE_PAGEABLE_BIN := tee-pageable_v2.bin +U_BOOT_BIN := u-boot.bin +U_BOOT_DTB := u-boot.dtb +LINUX_KERNEL_BIN := uImage +LINUX_DTB_BIN := $(STM32MP1_DTS_BASENAME).dtb + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +BINARIES_PATH ?= $(ROOT)/out/bin +TFA_PATH ?= $(ROOT)/trusted-firmware-a +U_BOOT_PATH ?= $(ROOT)/u-boot +SCPFW_PATH ?= $(ROOT)/scp-firmware + +define install_in_binaries + echo " INSTALL $(shell basename $1) to $(BINARIES_PATH)" && \ + mkdir -p $(BINARIES_PATH) && \ + ln -sf $1 $(BINARIES_PATH) +endef + +################################################################################ +# Main targets +################################################################################ +all: tfa optee-os u-boot linux buildroot + @$(call install_in_binaries,$(ROOT)/out-br/images/sdcard.img) + @echo Build for platform $(PLATFORM) completed + +clean: tfa-clean optee-os-clean u-boot-clean linux-clean buildroot-clean + +include toolchain.mk + +################################################################################ +# OP-TEE OS +################################################################################ + +# Provide scp-firmware source tree path in case CFG_SCMI_SERVER is enabled +OPTEE_OS_COMMON_FLAGS += CFG_SCP_FIRMWARE=$(SCPFW_PATH) + +optee-os: optee-os-common + @$(call install_in_binaries,$(OPTEE_OS_PATH)/out/arm/core/$(OPTEE_HEADER_BIN)) + @$(call install_in_binaries,$(OPTEE_OS_PATH)/out/arm/core/$(OPTEE_PAGER_BIN)) + @$(call install_in_binaries,$(OPTEE_OS_PATH)/out/arm/core/$(OPTEE_PAGEABLE_BIN)) + +optee-os-clean: optee-os-clean-common + +################################################################################ +# TrustedFirmware-A +################################################################################ +TFA_EXPORTS ?= CROSS_COMPILE="$(CCACHE)$(AARCH32_CROSS_COMPILE)" + +TFA_DEBUG ?= $(DEBUG) +ifeq ($(TFA_DEBUG),0) +TFA_LOGLVL ?= 30 +TFA_OUT = $(TFA_PATH)/build/stm32mp1/release +else +TFA_LOGLVL ?= 50 +TFA_OUT = $(TFA_PATH)/build/stm32mp1/debug +endif + +TFA_FLAGS ?= \ + BL32=$(BINARIES_PATH)/$(OPTEE_HEADER_BIN) \ + BL32_EXTRA1=$(BINARIES_PATH)/$(OPTEE_PAGER_BIN) \ + BL32_EXTRA2=$(BINARIES_PATH)/$(OPTEE_PAGEABLE_BIN) \ + BL33=$(BINARIES_PATH)/$(U_BOOT_BIN) \ + BL33_CFG=$(BINARIES_PATH)/$(U_BOOT_DTB) \ + ARM_ARCH_MAJOR=7 \ + ARCH=aarch32 \ + PLAT=stm32mp1 \ + DTB_FILE_NAME=$(STM32MP1_DTS_BASENAME).dtb \ + AARCH32_SP=optee \ + DEBUG=$(TFA_DEBUG) \ + LOG_LEVEL=$(TFA_LOGLVL) \ + STM32MP_EMMC=1 STM32MP_SDMMC=1 \ + STM32MP_RAW_NAND=0 STM32MP_SPI_NAND=0 STM32MP_SPI_NOR=0 + +tfa: optee-os u-boot + $(TFA_EXPORTS) $(MAKE) -C $(TFA_PATH) $(TFA_FLAGS) all fip + @$(call install_in_binaries,$(TFA_OUT)/$(TFA_BIN)) + @$(call install_in_binaries,$(TFA_OUT)/$(TFA_FIP_BIN)) + +tfa-clean: + $(TFA_EXPORTS) $(MAKE) -C $(TFA_PATH) $(TFA_FLAGS) clean + +################################################################################ +# U-Boot +################################################################################ +U_BOOT_EXPORTS ?= CROSS_COMPILE="$(CCACHE)$(AARCH32_CROSS_COMPILE)" + +u-boot: + $(U_BOOT_EXPORTS) $(MAKE) -C $(U_BOOT_PATH) $(STM32MP1_DEFCONFIG_U_BOOT) + $(U_BOOT_EXPORTS) $(MAKE) -C $(U_BOOT_PATH) DEVICE_TREE=$(STM32MP1_DTS_U_BOOT) all + @$(call install_in_binaries,$(U_BOOT_PATH)/$(U_BOOT_BIN)) + @$(call install_in_binaries,$(U_BOOT_PATH)/$(U_BOOT_DTB)) + +u-boot-clean: + $(U_BOOT_EXPORTS) $(MAKE) -C $(U_BOOT_PATH) clean + +################################################################################ +# Linux kernel +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm/configs/multi_v7_defconfig \ + $(CURDIR)/kconfigs/stm32mp1.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm uImage LOADADDR=0xc2000000 \ + CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) \ + $(STM32MP1_DTS_BASENAME).dtb \ + PATH=$$PATH:$(U_BOOT_PATH)/tools + +linux: linux-common + @$(call install_in_binaries,$(LINUX_PATH)/arch/arm/boot/$(LINUX_KERNEL_BIN)) + @$(call install_in_binaries,$(LINUX_PATH)/arch/arm/boot/dts/$(LINUX_DTB_BIN)) + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm + +linux-cleaner: linux-cleaner-common + +################################################################################ +# Buildroot +################################################################################ + +BR2_TARGET_GENERIC_ISSUE="OP-TEE embedded distrib for $(PLATFORM)" +BR2_TARGET_GENERIC_GETTY_PORT=ttySTM0 +BR2_TARGET_ROOTFS_EXT2=y +BR2_PACKAGE_BUSYBOX_WATCHDOG=y + +BREXT_BOARD_PATH=$(ROOT)/build/br-ext/board/stmicroelectronics/stm32mp1-tz +BREXT_GENIMAGE_CONFIG=$(BREXT_BOARD_PATH)/genimage-$(BREXT_FLAVOR).cfg +BREXT_BOOTFS_OVERLAY=$(BREXT_BOARD_PATH)/overlay-$(BREXT_FLAVOR) + +BR2_PACKAGE_HOST_GENIMAGE=y +BR2_ROOTFS_POST_SCRIPT_ARGS="$(BREXT_GENIMAGE_CONFIG) $(BINARIES_PATH) $(BREXT_BOOTFS_OVERLAY)" +BR2_ROOTFS_POST_IMAGE_SCRIPT=$(BREXT_BOARD_PATH)/post-image.sh + +# TF-A, Linux kernel, U-Boot and OP-TEE OS/Client/... are not built from their +# related Buildroot native package. +BR2_TARGET_ARM_TRUSTED_FIRMWARE=n +BR2_LINUX_KERNEL=n +BR2_TARGET_OPTEE_OS=n +BR2_TARGET_UBOOT=n +BR2_PACKAGE_OPTEE_CLIENT=n +BR2_PACKAGE_OPTEE_TEST=n +BR2_PACKAGE_OPTEE_EXAMPLES=n +BR2_PACKAGE_OPTEE_BENCHMARK=n + +################################################################################ +# We build the SD card image from Builroot but TF-A, OP-TEE OS, U-Boot and +# Linux kernel are build outside Buildroot. The get a clear picture of built +# images, images built outside Buildroot are installed (ln -s) in BINARIES_PATH +# and copied (cp -f) to Buildroot output images/ directory before make taget +# buildroot is processed. +################################################################################ +define install_in_br_images + cp -f $(BINARIES_PATH)/$1 $(ROOT)/out-br/images +endef + +.PHONY: optee-os optee-os-clean +.PHONY: u-boot u-boot-clean +.PHONY: copy_images_to_br + +buildroot: copy_images_to_br +copy_images_to_br: tfa optee-os u-boot linux + @mkdir -p $(ROOT)/out-br/images + $(call install_in_br_images,$(TFA_BIN)) + $(call install_in_br_images,$(TFA_FIP_BIN)) + $(call install_in_br_images,$(U_BOOT_BIN)) + $(call install_in_br_images,$(U_BOOT_DTB)) + $(call install_in_br_images,$(LINUX_KERNEL_BIN)) + $(call install_in_br_images,$(LINUX_DTB_BIN)) + $(call install_in_br_images,$(OPTEE_HEADER_BIN)) + $(call install_in_br_images,$(OPTEE_PAGER_BIN)) + $(call install_in_br_images,$(OPTEE_PAGEABLE_BIN)) diff --git a/build/synquacer.mk b/build/synquacer.mk new file mode 100644 index 0000000..b9064a3 --- /dev/null +++ b/build/synquacer.mk @@ -0,0 +1,169 @@ +################################################################################ +# Following variables defines how the NS_USER (Non-Secure User - Client +# Application), NS_KERNEL (Non-Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +COMPILE_NS_USER ?= 64 +COMPILE_NS_KERNEL ?= 64 +COMPILE_S_USER ?= 64 +COMPILE_S_KERNEL ?= 64 + +################################################################################ +# Includes +################################################################################ +include common.mk +include toolchain.mk + +ifeq ($(DEBUG),1) +SCP_BUILD ?= debug +TFA_BUILD ?= debug +EDK2_BUILD ?= DEBUG +else +SCP_BUILD ?= release +TFA_BUILD ?= release +EDK2_BUILD ?= RELEASE +endif + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +BINARIES_PATH ?= $(ROOT)/out/bin +SCP_PATH ?= $(ROOT)/SCP-firmware +SCP_OUT ?= $(SCP_PATH)/build/product/synquacer +SCP_ROM ?= $(SCP_OUT)/scp_romfw/$(SCP_BUILD)/bin/scp_romfw.bin +SCP_RAM ?= $(SCP_OUT)/scp_ramfw/$(SCP_BUILD)/bin/scp_ramfw.bin +SCP_ROMRAM ?= $(SCP_OUT)/scp_romramfw.bin +TFA_PATH ?= $(ROOT)/trusted-firmware-a +TFA_OUT ?= $(TFA_PATH)/build/synquacer/$(TFA_BUILD) +TFA_BL31 ?= $(TFA_OUT)/bl31.bin +TFA_BL32 ?= $(TFA_OUT)/bl32.bin +TFA_FIP ?= $(TFA_OUT)/fip_all_arm_tf.bin +FIPTOOL ?= $(TFA_PATH)/tools/fiptool/fiptool +EDK2_PATH ?= $(ROOT)/edk2 +EDK2_PLATFORMS_PATH ?= $(ROOT)/edk2-platforms +EDK2_NON_OSI_PATH ?= $(ROOT)/edk2-non-osi +EDK2_PKGS_PATH := "$(EDK2_PATH):$(EDK2_PLATFORMS_PATH):$(EDK2_NON_OSI_PATH)" +EDK2_FIP ?= $(EDK2_NON_OSI_PATH)/Platform/Socionext/DeveloperBox/fip_all_arm_tf.bin +EDK2_TOOLCHAIN ?= GCC5 +EDK2_ARCH ?= AARCH64 + +################################################################################ +# Targets +################################################################################ +.PHONY: all +all: edk2 optee-os scp tfa + +.PHONY: clean +clean: edk2-clean optee-os-clean scp-clean tfa-clean + +################################################################################ +# Toolchains +################################################################################ +AARCH32_NONE_PATH ?= $(TOOLCHAIN_ROOT)/aarch32-none +AARCH32_NONE_CROSS_COMPILE ?= $(AARCH32_NONE_PATH)/bin/arm-none-eabi- +AARCH32_NONE_GCC_VERSION ?= gcc-arm-none-eabi-9-2019-q4-major-x86_64-linux +SRC_AARCH32_NONE_GCC ?= https://developer.arm.com/-/media/Files/downloads/gnu-rm/9-2019q4/RC2.1/$(AARCH32_NONE_GCC_VERSION).tar.bz2 + +toolchains: aarch32-none + +.PHONY: aarch32-none +aarch32-none: + $(call dltc,$(AARCH32_NONE_PATH),$(SRC_AARCH32_NONE_GCC),$(AARCH32_NONE_GCC_VERSION)) + +################################################################################ +# SCP +################################################################################ +SCP_FLAGS ?= \ + CC=$(AARCH32_NONE_CROSS_COMPILE)gcc \ + PRODUCT=synquacer \ + MODE=$(SCP_BUILD) + +.PHONY: scp +scp: aarch32-none + $(MAKE) -C $(SCP_PATH) $(SCP_FLAGS) all + tr "\000" "\377" < /dev/zero | dd of=$(SCP_ROMRAM) bs=1 count=196608 + dd of=$(SCP_ROMRAM) if=$(SCP_ROM) bs=1 conv=notrunc seek=0 + dd of=$(SCP_ROMRAM) if=$(SCP_RAM) bs=1 seek=65536 + ln -sf $(SCP_ROMRAM) $(BINARIES_PATH) + +.PHONY: scp-clean +scp-clean: + rm -f $(SCP_ROMRAM) + $(MAKE) -C $(SCP_PATH) $(SCP_FLAGS) clean + +################################################################################ +# Trusted Firmware A +################################################################################ +TFA_EXPORTS ?= \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" + +TFA_FLAGS ?= \ + BL32=$(TFA_BL32) \ + PRELOADED_BL33_BASE=0x08200000 \ + DEBUG=$(DEBUG) \ + PLAT=synquacer \ + SPD=opteed \ + SQ_USE_SCMI_DRIVER=1 + +.PHONY: tfa +tfa: $(TFA_FIP) + +.PHONY: tfa-clean +tfa-clean: + $(TFA_EXPORTS) $(MAKE) -C $(TFA_PATH) $(TFA_FLAGS) clean + +$(TFA_FIP): $(TFA_BL32) + $(TFA_EXPORTS) $(MAKE) -C $(TFA_PATH) $(TFA_FLAGS) all fiptool + $(FIPTOOL) create \ + --tb-fw $(TFA_BL31) \ + --soc-fw $(TFA_BL31) \ + --scp-fw $(TFA_BL31) \ + --tos-fw $< \ + $@ + +$(TFA_BL32): optee-os + mkdir -p $(dir $@) + $(AARCH64_CROSS_COMPILE)objcopy \ + -O binary $(OPTEE_OS_PATH)/out/arm/core/tee.elf $@ + +################################################################################ +# EDK2 / Tianocore +################################################################################ +define edk2-env + export WORKSPACE=$(EDK2_PLATFORMS_PATH) +endef + +define edk2-call + $(EDK2_TOOLCHAIN)_$(EDK2_ARCH)_PREFIX=$(AARCH64_CROSS_COMPILE) \ + build -n `getconf _NPROCESSORS_ONLN` \ + -a $(EDK2_ARCH) -t $(EDK2_TOOLCHAIN) -b $(EDK2_BUILD) \ + -p Platform/Socionext/DeveloperBox/DeveloperBox.dsc +endef + +.PHONY: edk2 +edk2: $(EDK2_FIP) + $(call edk2-env) && \ + export PACKAGES_PATH=$(EDK2_PKGS_PATH) && \ + source $(EDK2_PATH)/edksetup.sh && \ + $(MAKE) -j1 -C $(EDK2_PATH)/BaseTools && \ + $(call edk2-call) all + +.PHONY: edk2-clean +edk2-clean: edk2-clean-common + cd $(EDK2_NON_OSI_PATH) && \ + git checkout $(EDK2_FIP) + +$(EDK2_FIP): $(TFA_FIP) + cp $< $@ + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_COMMON_FLAGS += PLATFORM=synquacer +OPTEE_OS_CLEAN_COMMON_FLAGS += PLATFORM=synquacer + +.PHONY: optee-os +optee-os: optee-os-common + +.PHONY: optee-os-clean +optee-os-clean: optee-os-clean-common diff --git a/build/ti/Makefile b/build/ti/Makefile new file mode 100644 index 0000000..3b1e16a --- /dev/null +++ b/build/ti/Makefile @@ -0,0 +1,51 @@ +#!/usr/bin/make -f +# +# Sign/encrypt a Linux Kernel zImage, a DTB file, and a rootfs (to be used +# as an initramfs) into a FIT image tree blob for loading through U-Boot on a +# TI High Security (HS) SoC. +# +# Copyright (C) 2016-2017, Texas Instruments, Incorporated - http://www.ti.com/ +# Andreas Dannenberg +# Andrew F. Davis +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# + +ITB ?= fitImage +ITS ?= $(ITB:=.its) + +SEC_IMAGES := $(shell sed -n 's/.*\/incbin\/.*\"\(.*\.sec\)\".*/\1/p' $(ITS)) + +.PHONY: all +all: $(ITB) + +# Invoke signing tool from the TI Secure Dev package to sign and optionally +# encrypt a binary blob. This tool is accessed through the use of the +# TI_SECURE_DEV_PKG environmental variable in the same fashion as it is used +# when building secure U-Boot for TI devices. +%.sec: % +ifneq ($(TI_SECURE_DEV_PKG),) +ifneq ($(wildcard $(TI_SECURE_DEV_PKG)/scripts/secure-binary-image.sh),) + $(TI_SECURE_DEV_PKG)/scripts/secure-binary-image.sh $(patsubst %.sec,%,$@) $@ +else + @echo "ERROR: $(TI_SECURE_DEV_PKG)/scripts/secure-binary-image.sh not found." \ + "$@ was NOT created!"; exit 1 +endif +else + @echo "ERROR: TI_SECURE_DEV_PKG environment variable must be defined" \ + "for TI secure devices. $@ was NOT created!"; exit 1 +endif + +# Compile a FIT image tree source file describing the final image tree blob. +# Use the mkimage tool that comes with U-Boot to make sure we have the latest/ +# greatest as we are using advanced features such as FIT... +MKIMAGE ?= mkimage +$(ITB): $(ITS) $(SEC_IMAGES) + $(MKIMAGE) -f $< -r $@ + +.PHONY: clean +clean: + -$(RM) -v $(SEC_IMAGES) + -$(RM) -v $(ITB) diff --git a/build/ti/fitImage-am43xx.its b/build/ti/fitImage-am43xx.its new file mode 100644 index 0000000..ee88b76 --- /dev/null +++ b/build/ti/fitImage-am43xx.its @@ -0,0 +1,62 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage"; + #address-cells = <1>; + + images { + kernel@1 { + description = "Linux kernel"; + data = /incbin/("zImage.sec"); + type = "kernel"; + arch = "arm"; + os = "linux"; + compression = "none"; + load = <0x82000000>; + entry = <0x82000000>; + }; + + am437x-gp-evm.dtb { + description = "Flattened Device Tree blob"; + data = /incbin/("am437x-gp-evm.dtb.sec"); + type = "flat_dt"; + arch = "arm"; + compression = "none"; + }; + + am43x-epos-evm.dtb { + description = "Flattened Device Tree blob"; + data = /incbin/("am43x-epos-evm.dtb.sec"); + type = "flat_dt"; + arch = "arm"; + compression = "none"; + }; + + optee { + description = "OPTEE OS Image"; + data = /incbin/("tee.bin.sec"); + type = "tee"; + arch = "arm"; + compression = "none"; + load = <0xbdb000e4>; + }; + }; + + configurations { + default = "am437x-gp-evm.dtb"; + + am437x-gp-evm.dtb { + description = "Linux kernel, FDT blob, OPTEE OS"; + kernel = "kernel@1"; + fdt = "am437x-gp-evm.dtb"; + loadables = "optee"; + }; + + am43x-epos-evm.dtb { + description = "Linux kernel, FDT blob, OPTEE OS"; + kernel = "kernel@1"; + fdt = "am43x-epos-evm.dtb"; + loadables = "optee"; + }; + }; +}; diff --git a/build/ti/fitImage-am57xx.its b/build/ti/fitImage-am57xx.its new file mode 100644 index 0000000..0820026 --- /dev/null +++ b/build/ti/fitImage-am57xx.its @@ -0,0 +1,61 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage"; + #address-cells = <1>; + + images { + kernel@1 { + description = "Linux kernel"; + data = /incbin/("zImage.sec"); + type = "kernel"; + arch = "arm"; + os = "linux"; + compression = "none"; + load = <0x82000000>; + entry = <0x82000000>; + }; + + am57xx-evm.dtb { + description = "Flattened Device Tree blob"; + data = /incbin/("am57xx-evm.dtb.sec"); + type = "flat_dt"; + arch = "arm"; + compression = "none"; + }; + + am57xx-evm-reva3.dtb { + description = "Flattened Device Tree blob"; + data = /incbin/("am57xx-evm-reva3.dtb.sec"); + type = "flat_dt"; + arch = "arm"; + compression = "none"; + }; + + optee { + description = "OPTEE OS Image"; + data = /incbin/("tee.bin.sec"); + type = "tee"; + arch = "arm"; + compression = "none"; + }; + }; + + configurations { + default = "am57xx-evm-reva3.dtb"; + + am57xx-evm.dtb { + description = "Linux kernel, FDT blob, OPTEE OS"; + kernel = "kernel@1"; + fdt = "am57xx-evm.dtb"; + loadables = "optee"; + }; + + am57xx-evm-reva3.dtb { + description = "Linux kernel, FDT blob, OPTEE OS"; + kernel = "kernel@1"; + fdt = "am57xx-evm-reva3.dtb"; + loadables = "optee"; + }; + }; +}; diff --git a/build/ti/fitImage-dra7xx.its b/build/ti/fitImage-dra7xx.its new file mode 100644 index 0000000..191466e --- /dev/null +++ b/build/ti/fitImage-dra7xx.its @@ -0,0 +1,76 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage"; + #address-cells = <1>; + + images { + kernel@1 { + description = "Linux kernel"; + data = /incbin/("zImage.sec"); + type = "kernel"; + arch = "arm"; + os = "linux"; + compression = "none"; + load = <0x82000000>; + entry = <0x82000000>; + }; + + dra7-evm.dtb { + description = "Flattened Device Tree blob"; + data = /incbin/("dra7-evm.dtb.sec"); + type = "flat_dt"; + arch = "arm"; + compression = "none"; + }; + + dra72-evm-revc.dtb { + description = "Flattened Device Tree blob"; + data = /incbin/("dra72-evm-revc.dtb.sec"); + type = "flat_dt"; + arch = "arm"; + compression = "none"; + }; + + dra71-evm.dtb { + description = "Flattened Device Tree blob"; + data = /incbin/("dra71-evm.dtb.sec"); + type = "flat_dt"; + arch = "arm"; + compression = "none"; + }; + + optee { + description = "OPTEE OS Image"; + data = /incbin/("tee.bin.sec"); + type = "tee"; + arch = "arm"; + compression = "none"; + }; + }; + + configurations { + default = "dra7-evm.dtb"; + + dra7-evm.dtb { + description = "Linux kernel, FDT blob, OPTEE OS"; + kernel = "kernel@1"; + fdt = "dra7-evm.dtb"; + loadables = "optee"; + }; + + dra72-evm-revc.dtb { + description = "Linux kernel, FDT blob, OPTEE OS"; + kernel = "kernel@1"; + fdt = "dra72-evm-revc.dtb"; + loadables = "optee"; + }; + + dra71-evm.dtb { + description = "Linux kernel, FDT blob, OPTEE OS"; + kernel = "kernel@1"; + fdt = "dra71-evm.dtb"; + loadables = "optee"; + }; + }; +}; diff --git a/build/ti/ti-common.mk b/build/ti/ti-common.mk new file mode 100644 index 0000000..920685c --- /dev/null +++ b/build/ti/ti-common.mk @@ -0,0 +1,98 @@ +############################################################################### +# Targets +############################################################################### +.PHONY: all clean cleaner prepare + +all: u-boot linux optee-os build-fit buildroot +clean: linux-clean u-boot-clean optee-os-clean build-fit-clean buildroot-clean +cleaner: clean prepare-cleaner linux-cleaner + +include toolchain.mk + +prepare: + @if [ ! -d $(STAGING_AREA) ]; then mkdir -p $(STAGING_AREA); fi + +.PHONY: prepare-cleaner +prepare-cleaner: + rm -r $(STAGING_AREA) + +############################################################################### +# Das U-Boot +############################################################################### +.PHONY: u-boot u-boot-clean + +U-BOOT_EXPORTS ?= CROSS_COMPILE=$(CROSS_COMPILE_NS_KERNEL) ARCH=arm + +u-boot: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) $(U-BOOT_CONFIG) + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) all + +u-boot-clean: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) clean + +############################################################################### +# Linux kernel +############################################################################### +.PHONY: linux-defconfig linux linux-defconfig-clean linux-clean linux-cleaner + +linux-defconfig: + cd $(LINUX_PATH) && \ + ti_config_fragments/defconfig_builder.sh -t $(CONFIG_TYPE) + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) $(CONFIG_TYPE)_defconfig + +LINUX_COMMON_FLAGS += ARCH=arm +linux: linux-common +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm +linux-cleaner: linux-cleaner-common + +############################################################################### +# OP-TEE +############################################################################### +.PHONY: optee-os optee-os-clean optee-client optee-client-clean + +OPTEE_OS_COMMON_FLAGS += PLATFORM=$(OPTEE_OS_PLATFORM) +optee-os: optee-os-common + +OPTEE_OS_CLEAN_COMMON_FLAGS += PLATFORM=$(OPTEE_OS_PLATFORM) +optee-os-clean: optee-os-clean-common + +############################################################################### +# Build FIT +############################################################################### +.PHONY: build-fit build-fit-clean + +build-fit: prepare linux optee-os + cp $(LINUX_IMAGE) $(STAGING_AREA)/ + cp $(LINUX_DTBS) $(STAGING_AREA)/ + cp $(OPTEE_OS_BIN) $(STAGING_AREA)/ + cp $(FIT_SOURCE) $(STAGING_AREA)/fitImage.its + cp $(FIT_MAKEFILE) $(STAGING_AREA)/ + MKIMAGE=$(U-BOOT_PATH)/tools/mkimage $(MAKE) -C $(STAGING_AREA) + +build-fit-clean: + $(RM) $(STAGING_AREA)/Makefile + $(RM) $(STAGING_AREA)/fitImage.its + $(RM) $(STAGING_AREA)/tee.bin + $(RM) $(STAGING_AREA)/*.dtb + $(RM) $(STAGING_AREA)/zImage + +############################################################################### +# Root FS +############################################################################### +.PHONY: update_rootfs +# Make sure this is built before the buildroot target which will create the +# root file system based on what's in $(BUILDROOT_TARGET_ROOT) +buildroot: update_rootfs + +update_rootfs: u-boot build-fit + @mkdir -p --mode=755 $(BUILDROOT_TARGET_ROOT)/boot + @install -v -p --mode=644 $(UBOOT_SPL) $(BUILDROOT_TARGET_ROOT)/boot/MLO + @install -v -p --mode=644 $(UBOOT_IMG) \ + $(BUILDROOT_TARGET_ROOT)/boot/u-boot.img + @install -v -p --mode=644 $(STAGING_AREA)/fitImage \ + $(BUILDROOT_TARGET_ROOT)/boot/fitImage diff --git a/build/toolchain.mk b/build/toolchain.mk new file mode 100644 index 0000000..517b913 --- /dev/null +++ b/build/toolchain.mk @@ -0,0 +1,141 @@ +################################################################################ +# Toolchains +################################################################################ +SHELL = /bin/bash +ROOT ?= $(CURDIR)/.. +TOOLCHAIN_ROOT ?= $(ROOT)/toolchains +UNAME_M := $(shell uname -m) +ARCH ?= arm + +# Download toolchain macro for saving some repetition +# $(1) is $AARCH.._PATH : i.e., path to the destination +# $(2) is $SRC_AARCH.._GCC : is the downloaded tar.gz file +# $(3) is $.._GCC_VERSION : the name of the file to download +define dltc + @if [ ! -d "$(1)" ]; then \ + echo "Downloading $(3) ..."; \ + mkdir -p $(1); \ + curl --retry 5 -k -s -S -L $(2) -o $(TOOLCHAIN_ROOT)/$(3).tar.xz || \ + { rm -f $(TOOLCHAIN_ROOT)/$(3).tar.xz; cd $(TOOLCHAIN_ROOT) && rmdir $(1); echo Download failed; exit 1; }; \ + tar xf $(TOOLCHAIN_ROOT)/$(3).tar.xz -C $(1) --strip-components=1 || \ + { rm $(TOOLCHAIN_ROOT)/$(3).tar.xz; echo Downloaded file is damaged; \ + cd $(TOOLCHAIN_ROOT) && rm -rf $(1); exit 1; }; \ + (cd $(1)/bin && shopt -s nullglob && for f in *-none-linux*; do ln -s $$f $${f//-none} ; done;) \ + fi +endef + +# Build buildroot toolchain macro for saving some repetition +# $(1) is $ARCH : target architecture +# $(2) is $AARCH.._PATH : i.e., path to the destination +# $(3) & $(4) : parts of toolchain target triplet +define build_toolchain + @echo Building $1 toolchain + @mkdir -p ../out-$1-sdk $2 + @(cd .. && $(PYTHON3) build/br-ext/scripts/make_def_config.py \ + --br buildroot --out out-$1-sdk --br-ext build/br-ext \ + --top-dir "$(ROOT)" \ + --br-defconfig build/br-ext/configs/sdk-$1 \ + --br-defconfig build/br-ext/configs/sdk-common \ + --make-cmd $(MAKE)) + +@$(MAKE) -C ../out-$1-sdk clean + +@$(MAKE) -C ../out-$1-sdk sdk + @tar xf ../out-$1-sdk/images/$3-buildroot-linux-$4_sdk-buildroot.tar.gz \ + -C $2 --strip-components=1 + @touch $2/.done +endef + +ifeq ($(UNAME_M),x86_64) +ifeq ($(ARCH),arm) +AARCH32_PATH ?= $(TOOLCHAIN_ROOT)/aarch32 +AARCH32_CROSS_COMPILE ?= $(AARCH32_PATH)/bin/arm-linux-gnueabihf- +AARCH32_GCC_VERSION ?= arm-gnu-toolchain-11.3.rel1-x86_64-arm-none-linux-gnueabihf +SRC_AARCH32_GCC ?= https://developer.arm.com/-/media/Files/downloads/gnu/11.3.rel1/binrel/$(AARCH32_GCC_VERSION).tar.xz + +AARCH64_PATH ?= $(TOOLCHAIN_ROOT)/aarch64 +AARCH64_CROSS_COMPILE ?= $(AARCH64_PATH)/bin/aarch64-linux-gnu- +AARCH64_GCC_VERSION ?= arm-gnu-toolchain-11.3.rel1-x86_64-aarch64-none-linux-gnu +SRC_AARCH64_GCC ?= https://developer.arm.com/-/media/Files/downloads/gnu/11.3.rel1/binrel/$(AARCH64_GCC_VERSION).tar.xz + +.PHONY: toolchains +toolchains: aarch32 aarch64 + +.PHONY: aarch32 +aarch32: + $(call dltc,$(AARCH32_PATH),$(SRC_AARCH32_GCC),$(AARCH32_GCC_VERSION)) + +.PHONY: aarch64 +aarch64: + $(call dltc,$(AARCH64_PATH),$(SRC_AARCH64_GCC),$(AARCH64_GCC_VERSION)) + +CLANG_VER ?= 12.0.0 +CLANG_PATH ?= $(ROOT)/clang-$(CLANG_VER) + +# Download the Clang compiler with LLVM tools and compiler-rt libraries +define dl-clang + @if [ ! -d "$(2)" ]; then \ + ./get_clang.sh $(1) $(2); \ + else \ + echo "$(2) already exists"; \ + fi +endef + +.PHONY: clang-toolchains +clang-toolchains: + $(call dl-clang,$(CLANG_VER),$(CLANG_PATH)) + +else ifeq ($(ARCH),riscv) +RISCV64_PATH ?= $(TOOLCHAIN_ROOT)/riscv64 +RISCV64_CROSS_COMPILE ?= $(RISCV64_PATH)/bin/riscv64-unknown-linux-gnu- +RISCV64_GCC_RELEASE_DATE ?= 2023.07.07 +RISCV64_GCC_VERSION ?= riscv64-glibc-ubuntu-22.04-gcc-nightly-$(RISCV64_GCC_RELEASE_DATE)-nightly +SRC_RISCV64_GCC ?= https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/$(RISCV64_GCC_RELEASE_DATE)/$(RISCV64_GCC_VERSION).tar.gz + +.PHONY: toolchains +toolchains: riscv64 + +.PHONY: riscv64 +riscv64: + $(call dltc,$(RISCV64_PATH),$(SRC_RISCV64_GCC),$(RISCV64_GCC_VERSION)) + +endif + +else ifeq ($(UNAME_M),aarch64) + +AARCH32_PATH ?= $(TOOLCHAIN_ROOT)/aarch32 +AARCH32_CROSS_COMPILE ?= $(AARCH32_PATH)/bin/arm-linux-gnueabihf- +AARCH32_GCC_VERSION ?= gcc-arm-10.2-2020.11-aarch64-arm-none-linux-gnueabihf +SRC_AARCH32_GCC ?= https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/$(AARCH32_GCC_VERSION).tar.xz + +# There isn't any native aarch64 toolchain released from Arm and buildroot +# doesn't support distribution toolchain [1]. So we are left with no choice +# but to build buildroot toolchain from source and use it. +# +# [1] https://buildroot.org/downloads/manual/manual.html#_cross_compilation_toolchain +AARCH64_PATH ?= $(TOOLCHAIN_ROOT)/aarch64 +AARCH64_CROSS_COMPILE ?= $(AARCH64_PATH)/bin/aarch64-linux- + +.PHONY: toolchains +toolchains: aarch32 $(AARCH64_PATH)/.done + +.PHONY: aarch32 +aarch32: + $(call dltc,$(AARCH32_PATH),$(SRC_AARCH32_GCC),$(AARCH32_GCC_VERSION)) + +$(AARCH64_PATH)/.done: + $(call build_toolchain,aarch64,$(AARCH64_PATH),aarch64,gnu) + +else # $(UNAME_M) != x86_64 or $(UNAME_M) != aarch64 +AARCH32_PATH := $(TOOLCHAIN_ROOT)/aarch32 +AARCH32_CROSS_COMPILE := $(AARCH32_PATH)/bin/arm-linux- +AARCH64_PATH := $(TOOLCHAIN_ROOT)/aarch64 +AARCH64_CROSS_COMPILE := $(AARCH64_PATH)/bin/aarch64-linux- + +.PHONY: toolchains +toolchains: $(AARCH64_PATH)/.done $(AARCH32_PATH)/.done + +$(AARCH64_PATH)/.done: + $(call build_toolchain,aarch64,$(AARCH64_PATH),aarch64,gnu) + +$(AARCH32_PATH)/.done: + $(call build_toolchain,aarch32,$(AARCH32_PATH),arm,gnueabihf) +endif diff --git a/build/trusted-keys.exp b/build/trusted-keys.exp new file mode 100644 index 0000000..546b487 --- /dev/null +++ b/build/trusted-keys.exp @@ -0,0 +1,65 @@ +#!/usr/bin/expect -f +# +# This script test Linux trusted keys support using OP-TEE as a trust +# source. The return code is 0 for success, >0 for error. +# + +set timeout 5 +set tk_id 0 +set ek_id 0 +# Wait for next prompt, dealing with key ID, failure message and timeout +proc check_keyctl_result arg { + expect { + -re {(\d+)\r} { + set ::$arg $expect_out(1,string) + exp_continue + } + "add_key: No such device" { + info [join {"Skipping test due to 'No such device':" + "trusted keys are not supported" + "(missing driver? CFG_CORE_DYN_SHM=n?)\n"}] + exit 0 + } + "FAILED" { + info "!!! Error\n" + exit 1 + } + timeout { + info "!!! Timeout\n" + exit 1 + } + "# " + } +} +proc run_cmd arg { + send -- [append arg " || fail\r"] +} +info "Running: keyctl tests...\n" +expect "# " +send -- "function fail { echo FAILED ; }\r" +expect "# " +run_cmd "keyctl add trusted kmk \"new 32\" @u" +check_keyctl_result tk_id +run_cmd "keyctl add encrypted evm \"new trusted:kmk 32\" @u" +check_keyctl_result ek_id +run_cmd "keyctl pipe $tk_id > kmk.blob" +check_keyctl_result tk_id +run_cmd "keyctl pipe $ek_id > evm.blob" +check_keyctl_result ek_id +run_cmd "keyctl revoke $ek_id" +check_keyctl_result ek_id +run_cmd "keyctl revoke $tk_id" +check_keyctl_result tk_id +run_cmd "keyctl add trusted kmk \"load `cat kmk.blob`\" @u" +check_keyctl_result tk_id +run_cmd "keyctl add encrypted evm \"load `cat evm.blob`\" @u" +check_keyctl_result ek_id +run_cmd "keyctl pipe $tk_id > kmk.blob2" +check_keyctl_result tk_id +run_cmd "keyctl pipe $ek_id > evm.blob2" +check_keyctl_result ek_id +run_cmd "diff kmk.blob kmk.blob2" +check_keyctl_result tk_id +run_cmd "diff evm.blob evm.blob2" +check_keyctl_result ek_id +info "Status: keyctl tests successful\n" diff --git a/build/trusted-services.mk b/build/trusted-services.mk new file mode 100644 index 0000000..82af02f --- /dev/null +++ b/build/trusted-services.mk @@ -0,0 +1,191 @@ +################################################################################ +# Paths to Trusted Services source and output +################################################################################ +TS_PATH ?= $(ROOT)/trusted-services +TS_BUILD_PATH ?= $(OUT_PATH)/ts-build +TS_INSTALL_PREFIX ?= $(OUT_PATH)/ts-install + +################################################################################ +# Secure Partitions +################################################################################ +.PHONY: ffa-sp-all ffa-sp-all-clean ffa-sp-all-realclean + +optee-os-common: ffa-sp-all +optee-os-clean: ffa-sp-all-clean + +ffa-sp-all-realclean: + rm -rf $(TS_INSTALL_PREFIX)/opteesp $(TS_INSTALL_PREFIX)/sp + +ifneq ($(COMPILE_S_USER),64) +$(error Trusted Services SPs only support AArch64) +endif + +SP_EXT-opteesp := stripped.elf +SP_EXT-sp := bin + +# The macro sets a variable if the source variable is defined, otherwise it +# results in an error. +# Parameter list: +# 1 - Destination variable name +# 2 - Source variable name +# 3 - Error message +define set_if_source_defined +ifndef $(2) +$$(error $(3)) +else +$(1) := $($(2)) +endif +endef + +# Helper macro to build and install Trusted Services Secure Partitions (SPs). +# Invokes CMake to configure, and make to build and install the SP. (CMake's +# Makefile generator backend is used, we can run make in the build directory). +# Adds the SP output image to the optee_os_sp_paths list and complies the SP +# manifest dts to dtb. +# +# For information about the additional dependencies of the project, please see +# https://trusted-services.readthedocs.io/en/latest/developer/software-requirements.html +# +# Parameter list: +# 1 - SP deployment name (e.g. internal-trusted-storage, crypto, etc.) +# 2 - Build configuration name (e.g. config/shared-flash) +# 3 - SP canonical UUID (e.g. dc1eef48-b17a-4ccf-ac8b-dfcff7711b14) +# 4 - SP additional build flags (e.g. -DTS_PLATFORM=<...>) +define build-sp +$(eval SP_DIR := $(lastword $(subst -, ,$(2)))) +$(eval $(call set_if_source_defined,SP_EXT,SP_EXT-$(lastword $(subst -, ,$(2))),Invalid $(1) SP configuration: $(2))) + +.PHONY: ffa-$1-sp +ffa-$1-sp: + CROSS_COMPILE=$(subst $(CCACHE),,$(CROSS_COMPILE_S_USER)) cmake -G"Unix Makefiles" \ + -S $(TS_PATH)/deployments/$1/$2 -B $(TS_BUILD_PATH)/$1 \ + -DCMAKE_INSTALL_PREFIX=$(TS_INSTALL_PREFIX) \ + -DCMAKE_C_COMPILER_LAUNCHER=$(CCACHE) $(SP_COMMON_FLAGS) $4 + $$(MAKE) -C $(TS_BUILD_PATH)/$1 install + dtc -I dts -O dtb -o $(TS_INSTALL_PREFIX)/$(SP_DIR)/manifest/$3.dtb \ + $(TS_INSTALL_PREFIX)/$(SP_DIR)/manifest/$3.dts + +.PHONY: ffa-$1-sp-clean +ffa-$1-sp-clean: + - $$(MAKE) -C $(TS_BUILD_PATH)/$1 clean + +.PHONY: ffa-$1-sp-realclean +ffa-$1-sp-realclean: + rm -rf $(TS_BUILD_PATH)/$1 + +ffa-sp-all: ffa-$1-sp +ffa-sp-all-clean: ffa-$1-sp-clean +ffa-sp-all-realclean: ffa-$1-sp-realclean + +optee_os_sp_paths += $(TS_INSTALL_PREFIX)/$(SP_DIR)/bin/$3.$(SP_EXT) +fip_sp_json_paths += $(TS_INSTALL_PREFIX)/$(SP_DIR)/json/$1.json +endef + +ifeq ($(SP_PACKAGING_METHOD),embedded) +# Add the list of SP paths to the optee_os config +OPTEE_OS_COMMON_EXTRA_FLAGS += SP_PATHS="$(optee_os_sp_paths)" +else ifeq ($(SP_PACKAGING_METHOD),fip) +$(TS_INSTALL_PREFIX)/sp_layout.json: ffa-sp-all + $(PYTHON3) $(TS_PATH)/tools/python/merge_json.py $@ $(fip_sp_json_paths) + +optee-os-common: $(TS_INSTALL_PREFIX)/sp_layout.json + +# Configure TF-A to load the SPs from FIP by BL2 +TF_A_FIP_SP_FLAGS += ARM_BL2_SP_LIST_DTS=$(ROOT)/build/fvp/bl2_sp_list.dtsi \ + SP_LAYOUT_FILE=$(TS_INSTALL_PREFIX)/sp_layout.json +endif + +################################################################################ +# Linux FF-A user space drivers +################################################################################ +.PHONY: linux-arm-ffa-tee linux-arm-ffa-tee-clean +all: linux-arm-ffa-tee + +linux-arm-ffa-tee: linux + mkdir -p $(OUT_PATH)/linux-arm-ffa-tee + $(MAKE) -C $(ROOT)/linux-arm-ffa-tee $(LINUX_COMMON_FLAGS) install \ + TARGET_DIR=$(OUT_PATH)/linux-arm-ffa-tee + +linux-arm-ffa-tee-clean: + $(MAKE) -C $(ROOT)/linux-arm-ffa-tee clean + +# This driver is only used by the uefi-test app or the spmc tests +ifneq ($(filter y, $(TS_UEFI_TESTS) $(SPMC_TESTS)),) +.PHONY: linux-arm-ffa-user linux-arm-ffa-user-clean +all: linux-arm-ffa-user + +linux-arm-ffa-user: linux + mkdir -p $(OUT_PATH)/linux-arm-ffa-user + $(MAKE) -C $(ROOT)/linux-arm-ffa-user $(LINUX_COMMON_FLAGS) install \ + TARGET_DIR=$(OUT_PATH)/linux-arm-ffa-user + echo "ed32d533-99e6-4209-9cc0-2d72cdd998a7,\ + 5c9edbc3-7b3a-4367-9f83-7c191ae86a37,\ + 7817164c-c40c-4d1a-867a-9bb2278cf41a,\ + 23eb0100-e32a-4497-9052-2f11e584afa6,\ + bdcd76d7-825e-4751-963b-86d4f84943ac" > \ + $(OUT_PATH)/linux-arm-ffa-user/sp_uuid_list.txt + +linux-arm-ffa-user-clean: + $(MAKE) -C $(ROOT)/linux-arm-ffa-user clean + +# Disable CONFIG_STRICT_DEVMEM option in the Linux kernel config. This allows +# userspace access to the whole NS physical address space through /dev/mem. It's +# needed by the uefi-test app to communicate with the smm-gateway SP using a +# static carveout. If changed, run "make linux-defconfig-clean" to take effect. +LINUX_DEFCONFIG_COMMON_FILES += $(CURDIR)/kconfigs/fvp_trusted-services_uefi.conf +endif + +################################################################################ +# Trusted Services test applications +################################################################################ +.PHONY: ffa-test-all ffa-test-all-clean ffa-test-all-realclean +all: ffa-test-all + +ffa-test-all-realclean: + rm -rf $(TS_INSTALL_PREFIX)/arm-linux + +ifneq ($(COMPILE_NS_USER),64) +$(error Trusted Services test apps only support AArch64) +endif + +# Helper macro to build and install Trusted Services test applications. +# Invokes CMake to configure, and make to build and install the apps. +# +# Parameter list: +# 1 - SP deployment name (e.g. psa-api-test/internal-trusted-storage, +# ts-demo, etc.) +# +# Each target will pass TS_APP_COMMON_FLAGS and +# TS_APP__EXTRA_FLAGS to cmake. ucfdpn is the upper case +# deployment name with all / characters replaced by _ characters. These +# variables allow setting extra build flags trough the environment. + +define build-ts-app +.PHONY: ffa-$1 +FFA_$1_UC_NAME:=$(shell echo $1 | tr a-z/ A-Z_) +ffa-$1: + CROSS_COMPILE=$(subst $(CCACHE),,$(CROSS_COMPILE_NS_USER)) cmake -G"Unix Makefiles" \ + -S $(TS_PATH)/deployments/$1/arm-linux -B $(TS_BUILD_PATH)/$1 \ + -DCMAKE_INSTALL_PREFIX=$(TS_INSTALL_PREFIX) \ + -Dlibts_DIR=${TS_INSTALL_PREFIX}/arm-linux/lib/cmake/libts \ + -DCFG_FORCE_PREBUILT_LIBTS=On \ + -DCMAKE_C_COMPILER_LAUNCHER=$(CCACHE) $(TS_APP_COMMON_FLAGS) \ + $(TS_APP_${FFA_$1_UC_NAME}_EXTRA_FLAGS) + $$(MAKE) -C $(TS_BUILD_PATH)/$1 install + +ifneq ($1,libts) +ffa-$1: ffa-libts +endif + +.PHONY: ffa-$1-clean +ffa-$1-clean: + - $$(MAKE) -C $(TS_BUILD_PATH)/$1 clean + +.PHONY: ffa-$1-realclean +ffa-$1-realclean: + rm -rf $(TS_BUILD_PATH)/$1 + +ffa-test-all: ffa-$1 +ffa-test-all-clean: ffa-$1-clean +ffa-test-all-realclean: ffa-$1-realclean +endef diff --git a/build/verdin.mk b/build/verdin.mk new file mode 100644 index 0000000..8319f96 --- /dev/null +++ b/build/verdin.mk @@ -0,0 +1,179 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +COMPILE_NS_USER ?= 64 +override COMPILE_NS_KERNEL := 64 +COMPILE_S_USER ?= 64 +COMPILE_S_KERNEL ?= 64 + +BR2_TARGET_GENERIC_GETTY_PORT = ttymxc0 +################################################################################ +# Includes +################################################################################ +include common.mk + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +OUT_PATH ?= $(ROOT)/out +ROOTFS_BIN ?= $(ROOT)/out-br/images/rootfs.tar +TF_A_PATH ?= $(ROOT)/trusted-firmware-a +UBOOT_PATH ?= $(ROOT)/u-boot +OPTEE_PATH ?= $(ROOT)/optee_os +LINUX_PATH ?= $(ROOT)/linux + +LINUX_DTB ?= $(LINUX_PATH)/arch/arm64/boot/dts/freescale/fsl-imx8mm-verdin-dev.dtb +MODULE_OUTPUT ?= $(ROOT)/module_output + +UBOOT_BIN ?= $(UBOOT_PATH)/flash.bin +OPTEE_ELF ?= $(OPTEE_PATH)/out/arm/core/tee.elf + +DDR_URL ?= https://www.nxp.com/lgfiles/NMG/MAD/YOCTO/firmware-imx-8.1.1.bin +DDR_PATH ?= $(ROOT)/ddr-firmware + +ATF_LOAD_ADDR ?= 0x920000 +TEE_LOAD_ADDR ?= 0xbe000000 + + +################################################################################ +# Targets +################################################################################ +.PHONY: all +all: u-boot arm-tf buildroot linux prepare-images | toolchains + +.PHONY: clean +clean: u-boot-clean arm-tf-clean linux-clean optee-os-clean \ + buildroot-clean + +################################################################################ +# Toolchain +################################################################################ +include toolchain.mk + +################################################################################ +# U-Boot +################################################################################ +.PHONY: u-boot-config +u-boot-config: +ifeq ($(wildcard $(UBOOT_PATH)/.config),) + $(MAKE) -C $(UBOOT_PATH) \ + CROSS_COMPILE=$(AARCH64_CROSS_COMPILE) verdin-imx8mm_defconfig +endif + +.PHONY: u-boot-menuconfig +u-boot-menuconfig: u-boot-config + $(MAKE) -C $(UBOOT_PATH) \ + CROSS_COMPILE=$(AARCH64_CROSS_COMPILE) menuconfig + +.PHONY: u-boot +u-boot: u-boot-config arm-tf optee-os ddr-firmware + # Copy BL31 binary from TF-A + cp $(TF_A_PATH)/build/imx8mm/release/bl31.bin $(UBOOT_PATH) + # Prepare proper tee.bin + $(AARCH64_CROSS_COMPILE)objcopy -O binary \ + $(OPTEE_ELF) $(UBOOT_PATH)/tee.bin + # Copy DDR4 firmware + cp $(DDR_PATH)/firmware-imx-8.1.1/firmware/ddr/synopsys/lpddr4*.bin \ + $(UBOOT_PATH) + # Build U-Boot and final ready-to-flash flash.bin image + ATF_LOAD_ADDR=$(ATF_LOAD_ADDR) TEE_LOAD_ADDR=$(TEE_LOAD_ADDR) \ + $(MAKE) -C $(UBOOT_PATH) \ + CROSS_COMPILE="$(AARCH64_CROSS_COMPILE)" flash.bin + +.PHONY: u-boot-clean +u-boot-clean: + cd $(UBOOT_PATH) && git clean -xdf + +################################################################################ +# DDR4 Firmware +################################################################################ +.PHONY: ddr-firmware +ddr-firmware: + # DDR is exported to the $PWD only, so cd to $(DDR_PATH) + # before unpacking + if [ ! -d "$(DDR_PATH)" ]; then \ + mkdir -p $(DDR_PATH) && \ + wget $(DDR_URL) -O $(DDR_PATH)/firmware.bin && \ + chmod +x $(DDR_PATH)/firmware.bin && \ + cd $(DDR_PATH) && \ + $(DDR_PATH)/firmware.bin --auto-accept && \ + cd $(ROOT)/build; \ + fi; + +.PHONY: ddr-firmware-clean +ddr-firmware-clean: + rm -rf $(DDR_PATH) + +################################################################################ +# ARM Trusted Firmware +################################################################################ +.PHONY: arm-tf +arm-tf: + $(MAKE) -C $(TF_A_PATH) \ + PLAT=imx8mm \ + CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" \ + SPD=opteed \ + bl31 + +.PHONY: arm-tf-clean +arm-tf-clean: + cd $(TF_A_PATH) && git clean -xdf + +################################################################################ +# OP-TEE +################################################################################ +OPTEE_OS_COMMON_FLAGS += PLATFORM=imx PLATFORM_FLAVOR=mx8mmevk CFG_ARM64_core=y CFG_UART_BASE=0x30860000 +OPTEE_OS_CLEAN_COMMON_FLAGS += PLATFORM=imx-mx8mmevk + +.PHONY: optee-os +optee-os: optee-os-common + +.PHONY: optee-os-clean +optee-os-clean: optee-os-clean-common + +################################################################################ +# Linux +################################################################################ +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/defconfig \ + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 + +linux: linux-common + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) freescale/fsl-imx8mm-verdin-dev.dtb + $(MAKE) -C $(LINUX_PATH) $(LINUX_COMMON_FLAGS) INSTALL_MOD_STRIP=1 \ + INSTALL_MOD_PATH=$(MODULE_OUTPUT) modules_install + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +.PHONY: prepare-images +prepare-images: linux u-boot buildroot + @mkdir -p $(OUT_PATH) + @cp $(UBOOT_BIN) $(OUT_PATH) + @cp $(LINUX_PATH)/arch/arm64/boot/Image $(OUT_PATH) + @cp $(LINUX_DTB) $(OUT_PATH) + @cp $(ROOT)/out-br/images/rootfs.tar $(OUT_PATH) + +################################################################################ +# Buildroot/RootFS +################################################################################ +.PHONY: update_rootfs +update_rootfs: u-boot linux + @cd $(MODULE_OUTPUT) && find . | cpio -pudm $(BUILDROOT_TARGET_ROOT) + @cd $(ROOT)/build + +.PHONY: buildroot +buildroot: update_rootfs diff --git a/build/versal.mk b/build/versal.mk new file mode 100644 index 0000000..3d9945f --- /dev/null +++ b/build/versal.mk @@ -0,0 +1,175 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +override COMPILE_NS_USER := 64 +override COMPILE_NS_KERNEL := 64 +override COMPILE_S_USER := 64 +override COMPILE_S_KERNEL := 64 + +# Network support related packages: +BR2_PACKAGE_DHCPCD ?= y +BR2_PACKAGE_ETHTOOL ?= y +BR2_PACKAGE_XINETD ?= y + +# SSH Packages : +BR2_PACKAGE_OPENSSH ?= y +BR2_PACKAGE_OPENSSH_SERVER ?= y +BR2_PACKAGE_OPENSSH_KEY_UTILS ?= y + +# Openssl binary +BR2_PACKAGE_LIBOPENSSL_BIN ?= y + +PLATFORM = versal-vck190 +OPTEE_OS_PLATFORM = versal +OPTEE_OS_COMMON_EXTRA_FLAGS = CFG_PKCS11_TA=y CFG_USER_TA_TARGET_pkcs11=ta_arm64 O=out/arm + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/arm-trusted-firmware +U-BOOT_PATH ?= $(ROOT)/u-boot +BOOTGEN_PATH ?= $(ROOT)/bootgen +LINUX_PATH ?= $(ROOT)/linux + +include common.mk + +################################################################################ +# Targets +################################################################################ + +all: tfa optee-os u-boot linux dtbo buildroot +clean: tfa-clean optee-os-clean u-boot-clean linux-clean dtbo-clean buildroot-clean + +include toolchain.mk + +################################################################################ +# ARM Trusted Firmware +################################################################################ + +TF_A_EXPORTS = CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" +TF_A_FLAGS = PLAT=versal VERSAL_CONSOLE=pl011 RESET_TO_BL31=1 SPD=opteed DEBUG=1 + +tfa: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) bl31 + +tfa-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# OP-TEE +################################################################################# + +optee-os: optee-os-common + +optee-os-clean: optee-os-clean-common + rm -f ${OPTEE_OS_PATH}/out/arm/core/tee_raw.bin + +################################################################################ +# U-Boot +################################################################################ + +U-BOOT_EXPORTS = CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" +U-BOOT_CONFIG = xilinx_versal_virt_defconfig +U-BOOT_DTS = versal-vck190-revA + +u-boot: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) $(U-BOOT_CONFIG) + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) DEVICE_TREE=$(U-BOOT_DTS) DTC_FLAGS="-@" + +u-boot-clean: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) clean + +############################################################################### +# Device-Tree +############################################################################### +dtbo: linux + ${LINUX_PATH}/scripts/dtc/dtc -@ -I dts -O dtb -o versal/versal-optee.dtbo versal/versal-optee.dtso + +dtbo-clean: + rm -f versal/versal-optee.dtbo + +################################################################################ +# Linux kernel +################################################################################ + +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/xilinx_versal_defconfig \ + $(CURDIR)/kconfigs/versal.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 -j8 + +linux: linux-common + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +############################################################################### +# Bouildroot +############################################################################### + +BR2_TARGET_GENERIC_ISSUE="OP-TEE embedded distrib for $(PLATFORM)" +BR2_TARGET_ROOTFS_EXT2=y +BR2_PACKAGE_BUSYBOX_WATCHDOG=y + +# TF-A, Linux kernel, U-Boot and OP-TEE OS/Client/... are not built from their +# related Buildroot native package. +BR2_TARGET_ARM_TRUSTED_FIRMWARE=n +BR2_LINUX_KERNEL=n +BR2_TARGET_OPTEE_OS=n +BR2_TARGET_UBOOT=n +BR2_PACKAGE_OPTEE_CLIENT=n +BR2_PACKAGE_OPTEE_TEST=n +BR2_PACKAGE_OPTEE_EXAMPLES=n +BR2_PACKAGE_OPTEE_BENCHMARK=n + + +############################################################################### +# Images +############################################################################### +image: bootimage fitimage +image-clean: bootimage-clean fitimage-clean + +############################################################################### +# Boot Image +############################################################################### + +bootimage: bootgen tfa optee-os u-boot + $(BOOTGEN_PATH)/bootgen -arch versal -image versal/bootImage-${PLATFORM}.bif -w -o versal/BOOT.BIN + +bootimage-clean: bootgen-clean tfa-clean optee-os-clean u-boot-clean + rm -f versal/BOOT.BIN + + +############################################################################### +# Bootgen +############################################################################### + +bootgen: + make -C $(BOOTGEN_PATH) + +bootgen-clean: + make -C $(BOOTGEN_PATH) clean + + +############################################################################### +# FIT Image +############################################################################### + +fitimage: linux dtbo buildroot + ${U-BOOT_PATH}/tools/mkimage -f versal/fitImage-${PLATFORM}.its versal/${PLATFORM}.ub + +fitimage-clean: linux-clean dtbo-clean buildroot-clean + rm -f versal/${PLATFORM}.ub + diff --git a/build/versal/bootImage-versal-vck190.bif b/build/versal/bootImage-versal-vck190.bif new file mode 100644 index 0000000..f85c012 --- /dev/null +++ b/build/versal/bootImage-versal-vck190.bif @@ -0,0 +1,16 @@ +the_ROM_image: +{ + image { + { type=bootimage, file=../versal-vck190-bsp/project-spec/hw-description/vpl_gen_fixed.pdi } + { type=bootloader, file=../versal-vck190-bsp/pre-built/linux/images/plm.elf } + { core=psm, file=../versal-vck190-bsp/pre-built/linux/images/psmfw.elf } + } + + image { + id = 0x1c000000, name=apu_subsystem + { type=raw, load=0x00001000, file=../u-boot/arch/arm/dts/versal-vck190-revA-x-ebm-01-revA.dtb } + { core=a72-0, exception_level=el-3, trustzone, file=../arm-trusted-firmware/build/versal/debug/bl31/bl31.elf } + { core=a72-0, exception_level=el-2, file=../u-boot/u-boot.elf } + { type=raw, load=0x60000000, file=../optee_os/out/arm/core/tee-raw.bin } + } +} diff --git a/build/versal/boot_jtag.sh b/build/versal/boot_jtag.sh new file mode 100755 index 0000000..e4316b9 --- /dev/null +++ b/build/versal/boot_jtag.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +read -p "Enter full path to the Petalinux install [i.e: /opt/petalinux.2022.1]: " path + +. "${path}"/settings.sh + +xsct ./versal.tcl diff --git a/build/versal/fitImage-versal-vck190.its b/build/versal/fitImage-versal-vck190.its new file mode 100644 index 0000000..828a258 --- /dev/null +++ b/build/versal/fitImage-versal-vck190.its @@ -0,0 +1,67 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage for plnx_aarch64 kernel"; + #address-cells = <1>; + + images { + kernel { + description = "Linux Kernel"; + data = /incbin/("./../../linux/arch/arm64/boot/Image"); + type = "kernel"; + arch = "arm64"; + os = "linux"; + compression = "none"; + load = <0x80000>; + entry = <0x80000>; + hash1 { + algo = "sha1"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + data = /incbin/("../../u-boot/arch/arm/dts/versal-vck190-revA-x-ebm-01-revA.dtb"); + type = "flat_dt"; + arch = "arm64"; + load = <0x10000000>; + compression = "none"; + hash1 { + algo = "sha1"; + }; + }; + fdt-2 { + description = "Flattened Device Tree overlay"; + data = /incbin/("versal-optee.dtbo"); + type = "flat_dt"; + arch = "arm64"; + load = <0x11000000>; + compression = "none"; + hash1 { + algo = "sha1"; + }; + }; + ramdisk { + description = "ramdisk"; + data = /incbin/("../../out-br/images/rootfs.cpio.gz"); + type = "ramdisk"; + arch = "arm64"; + os = "linux"; + compression = "gzip"; + hash1 { + algo = "sha1"; + }; + }; + }; + configurations { + default = "conf1"; + conf1 { + description = "Boot Linux kernel with FDT blob + ramdisk"; + kernel = "kernel"; + fdt = "fdt-1", "fdt-2"; + ramdisk = "ramdisk"; + hash1 { + algo = "sha1"; + }; + }; + }; +}; diff --git a/build/versal/versal-optee.dtso b/build/versal/versal-optee.dtso new file mode 100644 index 0000000..0132ae0 --- /dev/null +++ b/build/versal/versal-optee.dtso @@ -0,0 +1,13 @@ +/dts-v1/; +/plugin/; +/ { + fragment@1 { + target-path = "/firmware"; + __overlay__ { + optee { + compatible = "linaro,optee-tz"; + method = "smc"; + }; + }; + }; +}; diff --git a/build/versal/versal.tcl b/build/versal/versal.tcl new file mode 100644 index 0000000..c7523dc --- /dev/null +++ b/build/versal/versal.tcl @@ -0,0 +1,6 @@ +connect +after 1000 +target 1 +rst +targets -set -nocase -filter {name =~ "*Versal*"} +device program "./BOOT.BIN" diff --git a/build/zynqmp.mk b/build/zynqmp.mk new file mode 100644 index 0000000..1ca7fce --- /dev/null +++ b/build/zynqmp.mk @@ -0,0 +1,184 @@ +################################################################################ +# Following variables defines how the NS_USER (Non Secure User - Client +# Application), NS_KERNEL (Non Secure Kernel), S_KERNEL (Secure Kernel) and +# S_USER (Secure User - TA) are compiled +################################################################################ +override COMPILE_NS_USER := 64 +override COMPILE_NS_KERNEL := 64 +override COMPILE_S_USER := 64 +override COMPILE_S_KERNEL := 64 + +PLATFORM := zynqmp-zcu102 +OPTEE_OS_PLATFORM = $(PLATFORM) + +DTS_zynqmp-zcu102 = zynqmp-zcu102-rev1.0 +DTS_zynqmp-zcu104 = zynqmp-zcu104-revC +DTS_zynqmp-zcu106 = zynqmp-zcu106-revA +DTS_zynqmp-ultra96 = avnet-ultra96-rev1 +U-BOOT_DTS = $(DTS_$(PLATFORM)) + +################################################################################ +# Paths to git projects and various binaries +################################################################################ +TF_A_PATH ?= $(ROOT)/arm-trusted-firmware +U-BOOT_PATH ?= $(ROOT)/u-boot-xlnx +BOOTGEN_PATH ?= $(ROOT)/bootgen +LINUX_PATH ?= $(ROOT)/linux-xlnx + +include common.mk + +################################################################################ +# Targets +################################################################################ + +all: tfa optee-os u-boot linux dtbo buildroot +clean: tfa-clean optee-os-clean u-boot-clean linux-clean dtbo-clean buildroot-clean + +include toolchain.mk + +################################################################################ +# ARM Trusted Firmware +################################################################################ + +TF_A_EXPORTS = CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" +TF_A_FLAGS = PLAT=zynqmp RESET_TO_BL31=1 NEED_BL32=yes SPD=opteed LOG_LEVEL=LOG_LEVEL_INFO + + +tfa: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) bl31 + +tfa-clean: + $(TF_A_EXPORTS) $(MAKE) -C $(TF_A_PATH) $(TF_A_FLAGS) clean + +################################################################################ +# OP-TEE +################################################################################# + +optee-os: optee-os-common + ${OPTEE_OS_PATH}/scripts/gen_tee_bin.py --input ${OPTEE_OS_PATH}/out/arm/core/tee.elf --out_tee_raw_bin ${OPTEE_OS_PATH}/out/arm/core/tee_raw.bin + +optee-os-clean: optee-os-clean-common + rm -f ${OPTEE_OS_PATH}/out/arm/core/tee_raw.bin + +################################################################################ +# U-Boot +################################################################################ + +U-BOOT_EXPORTS = CROSS_COMPILE="$(CCACHE)$(AARCH64_CROSS_COMPILE)" +U-BOOT_CONFIG = xilinx_zynqmp_virt_defconfig + +u-boot: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) $(U-BOOT_CONFIG) + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) DEVICE_TREE=$(U-BOOT_DTS) DTC_FLAGS="-@" + +u-boot-clean: + $(U-BOOT_EXPORTS) $(MAKE) -C $(U-BOOT_PATH) clean + +############################################################################### +# Device-Tree +############################################################################### +dtbo: linux + ${LINUX_PATH}/scripts/dtc/dtc -@ -I dts -O dtb -o zynqmp/zynqmp-optee.dtbo zynqmp/zynqmp-optee.dtso + +dtbo-clean: + rm -f zynqmp/zynqmp-optee.dtbo + +################################################################################ +# Linux kernel +################################################################################ + +LINUX_DEFCONFIG_COMMON_ARCH := arm64 +LINUX_DEFCONFIG_COMMON_FILES := \ + $(LINUX_PATH)/arch/arm64/configs/xilinx_zynqmp_defconfig \ + $(CURDIR)/kconfigs/zynqmp.conf + +linux-defconfig: $(LINUX_PATH)/.config + +LINUX_COMMON_FLAGS += ARCH=arm64 + +linux: linux-common + +linux-defconfig-clean: linux-defconfig-clean-common + +LINUX_CLEAN_COMMON_FLAGS += ARCH=arm64 + +linux-clean: linux-clean-common + +LINUX_CLEANER_COMMON_FLAGS += ARCH=arm64 + +linux-cleaner: linux-cleaner-common + +############################################################################### +# Bouildroot +############################################################################### + +BR2_TARGET_GENERIC_ISSUE="OP-TEE embedded distrib for $(PLATFORM)" +BR2_TARGET_ROOTFS_EXT2=y +BR2_PACKAGE_BUSYBOX_WATCHDOG=y +BR2_TARGET_GENERIC_GETTY_PORT=ttyPS0 + +# TF-A, Linux kernel, U-Boot and OP-TEE OS/Client/... are not built from their +# related Buildroot native package. +BR2_TARGET_ARM_TRUSTED_FIRMWARE=n +BR2_LINUX_KERNEL=n +BR2_TARGET_OPTEE_OS=n +BR2_TARGET_UBOOT=n +BR2_PACKAGE_OPTEE_CLIENT=n +BR2_PACKAGE_OPTEE_TEST=n +BR2_PACKAGE_OPTEE_EXAMPLES=n +BR2_PACKAGE_OPTEE_BENCHMARK=n + + +############################################################################### +# Images +############################################################################### +image: bootimage fitimage +image-clean: bootimage-clean fitimage-clean + +############################################################################### +# Boot Image +############################################################################### +FIRMWARE_TARBALL = $(subst zynqmp-,2021.1-,$(PLATFORM))-release.tar.xz + +bootimage: bootgen firmware tfa optee-os u-boot + $(BOOTGEN_PATH)/bootgen -arch zynqmp -image zynqmp/bootImage-${PLATFORM}.bif -w -o zynqmp/BOOT.bin + +bootimage-clean: bootgen-clean firmware-clean tfa-clean optee-os-clean u-boot-clean + rm -f zynqmp/BOOT.bin + + +############################################################################### +# Bootgen +############################################################################### + +bootgen: + make -C $(BOOTGEN_PATH) + +bootgen-clean: + make -C $(BOOTGEN_PATH) clean + + +################################################################################ +# ZynqMPSoC Firmware mandatory for the boot +################################################################################ + +firmware: +ifeq ("$(wildcard ../$(FIRMWARE_TARBALL))","") + $(error Release image tarball not present ../$(FIRMWARE_TARBALL)) +else + mkdir -p ../$(PLATFORM)-release && tar -xvf ../$(FIRMWARE_TARBALL) -C ../$(PLATFORM)-release --strip-components=1 +endif + +firmware-clean: + rm -rf ../$(PLATFORM)-release + +############################################################################### +# FIT Image +############################################################################### + +fitimage: linux dtbo buildroot + ${U-BOOT_PATH}/tools/mkimage -f zynqmp/fitImage-${PLATFORM}.its zynqmp/${PLATFORM}.ub + +fitimage-clean: linux-clean dtbo-clean buildroot-clean + rm -f zynqmp/${PLATFORM}.ub + diff --git a/build/zynqmp/bootImage-zynqmp-ultra96.bif b/build/zynqmp/bootImage-zynqmp-ultra96.bif new file mode 100644 index 0000000..687c8f5 --- /dev/null +++ b/build/zynqmp/bootImage-zynqmp-ultra96.bif @@ -0,0 +1,8 @@ +all: +{ + [pmufw_image] ../zynqmp-ultra96-release/pmufw.elf + [bootloader, destination_cpu = a53-0] ../zynqmp-ultra96-release/zynqmp_fsbl.elf + [destination_cpu = a53-0, exception_level = el-3, trustzone] ../arm-trusted-firmware/build/zynqmp/release/bl31/bl31.elf + [destination_cpu = a53-0, exception_level = el-2] ../u-boot-xlnx/u-boot.elf + [destination_cpu = a53-0, load=0x60000000, startup=0x60000000, exception_level = el-1, trustzone] ../optee_os/out/arm/core/tee_raw.bin +} diff --git a/build/zynqmp/bootImage-zynqmp-zcu102.bif b/build/zynqmp/bootImage-zynqmp-zcu102.bif new file mode 100644 index 0000000..42bf904 --- /dev/null +++ b/build/zynqmp/bootImage-zynqmp-zcu102.bif @@ -0,0 +1,8 @@ +all: +{ + [pmufw_image] ../zynqmp-zcu102-release/pmufw.elf + [bootloader, destination_cpu = a53-0] ../zynqmp-zcu102-release/zynqmp_fsbl.elf + [destination_cpu = a53-0, exception_level = el-3, trustzone] ../arm-trusted-firmware/build/zynqmp/release/bl31/bl31.elf + [destination_cpu = a53-0, exception_level = el-2] ../u-boot-xlnx/u-boot.elf + [destination_cpu = a53-0, load=0x60000000, startup=0x60000000, exception_level = el-1, trustzone] ../optee_os/out/arm/core/tee_raw.bin +} diff --git a/build/zynqmp/bootImage-zynqmp-zcu104.bif b/build/zynqmp/bootImage-zynqmp-zcu104.bif new file mode 100644 index 0000000..9d093b6 --- /dev/null +++ b/build/zynqmp/bootImage-zynqmp-zcu104.bif @@ -0,0 +1,8 @@ +all: +{ + [pmufw_image] ../zynqmp-zcu104-release/pmufw.elf + [bootloader, destination_cpu = a53-0] ../zynqmp-zcu104-release/zynqmp_fsbl.elf + [destination_cpu = a53-0, exception_level = el-3, trustzone] ../arm-trusted-firmware/build/zynqmp/release/bl31/bl31.elf + [destination_cpu = a53-0, exception_level = el-2] ../u-boot-xlnx/u-boot.elf + [destination_cpu = a53-0, load=0x60000000, startup=0x60000000, exception_level = el-1, trustzone] ../optee_os/out/arm/core/tee_raw.bin +} diff --git a/build/zynqmp/bootImage-zynqmp-zcu106.bif b/build/zynqmp/bootImage-zynqmp-zcu106.bif new file mode 100644 index 0000000..8965e04 --- /dev/null +++ b/build/zynqmp/bootImage-zynqmp-zcu106.bif @@ -0,0 +1,8 @@ +all: +{ + [pmufw_image] ../zynqmp-zcu106-release/pmufw.elf + [bootloader, destination_cpu = a53-0] ../zynqmp-zcu106-release/zynqmp_fsbl.elf + [destination_cpu = a53-0, exception_level = el-3, trustzone] ../arm-trusted-firmware/build/zynqmp/release/bl31/bl31.elf + [destination_cpu = a53-0, exception_level = el-2] ../u-boot-xlnx/u-boot.elf + [destination_cpu = a53-0, load=0x60000000, startup=0x60000000, exception_level = el-1, trustzone] ../optee_os/out/arm/core/tee_raw.bin +} diff --git a/build/zynqmp/fitImage-zynqmp-ultra96.its b/build/zynqmp/fitImage-zynqmp-ultra96.its new file mode 100644 index 0000000..c7b578b --- /dev/null +++ b/build/zynqmp/fitImage-zynqmp-ultra96.its @@ -0,0 +1,67 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage for plnx_aarch64 kernel"; + #address-cells = <1>; + + images { + kernel { + description = "Linux Kernel"; + data = /incbin/("./../../linux-xlnx/arch/arm64/boot/Image"); + type = "kernel"; + arch = "arm64"; + os = "linux"; + compression = "none"; + load = <0x80000>; + entry = <0x80000>; + hash { + algo = "sha1"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + data = /incbin/("../../u-boot-xlnx/arch/arm/dts/avnet-ultra96-rev1.dtb"); + type = "flat_dt"; + arch = "arm64"; + load = <0x10000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + fdt-2 { + description = "Flattened Device Tree overlay"; + data = /incbin/("zynqmp-optee.dtbo"); + type = "flat_dt"; + arch = "arm64"; + load = <0x11000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + ramdisk { + description = "ramdisk"; + data = /incbin/("../../out-br/images/rootfs.cpio.gz"); + type = "ramdisk"; + arch = "arm64"; + os = "linux"; + compression = "gzip"; + hash { + algo = "sha1"; + }; + }; + }; + configurations { + default = "conf"; + conf { + description = "Boot Linux kernel with FDT blob + ramdisk"; + kernel = "kernel"; + fdt = "fdt-1", "fdt-2"; + ramdisk = "ramdisk"; + hash { + algo = "sha1"; + }; + }; + }; +}; diff --git a/build/zynqmp/fitImage-zynqmp-zcu102.its b/build/zynqmp/fitImage-zynqmp-zcu102.its new file mode 100644 index 0000000..58a5f40 --- /dev/null +++ b/build/zynqmp/fitImage-zynqmp-zcu102.its @@ -0,0 +1,67 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage for plnx_aarch64 kernel"; + #address-cells = <1>; + + images { + kernel { + description = "Linux Kernel"; + data = /incbin/("./../../linux-xlnx/arch/arm64/boot/Image"); + type = "kernel"; + arch = "arm64"; + os = "linux"; + compression = "none"; + load = <0x80000>; + entry = <0x80000>; + hash { + algo = "sha1"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + data = /incbin/("../../u-boot-xlnx/arch/arm/dts/zynqmp-zcu102-rev1.0.dtb"); + type = "flat_dt"; + arch = "arm64"; + load = <0x10000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + fdt-2 { + description = "Flattened Device Tree overlay"; + data = /incbin/("zynqmp-optee.dtbo"); + type = "flat_dt"; + arch = "arm64"; + load = <0x11000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + ramdisk { + description = "ramdisk"; + data = /incbin/("../../out-br/images/rootfs.cpio.gz"); + type = "ramdisk"; + arch = "arm64"; + os = "linux"; + compression = "gzip"; + hash { + algo = "sha1"; + }; + }; + }; + configurations { + default = "conf"; + conf { + description = "Boot Linux kernel with FDT blob + ramdisk"; + kernel = "kernel"; + fdt = "fdt-1", "fdt-2"; + ramdisk = "ramdisk"; + hash { + algo = "sha1"; + }; + }; + }; +}; diff --git a/build/zynqmp/fitImage-zynqmp-zcu104.its b/build/zynqmp/fitImage-zynqmp-zcu104.its new file mode 100644 index 0000000..64cbc1d --- /dev/null +++ b/build/zynqmp/fitImage-zynqmp-zcu104.its @@ -0,0 +1,67 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage for plnx_aarch64 kernel"; + #address-cells = <1>; + + images { + kernel { + description = "Linux Kernel"; + data = /incbin/("./../../linux-xlnx/arch/arm64/boot/Image"); + type = "kernel"; + arch = "arm64"; + os = "linux"; + compression = "none"; + load = <0x80000>; + entry = <0x80000>; + hash { + algo = "sha1"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + data = /incbin/("../../u-boot-xlnx/arch/arm/dts/zynqmp-zcu104-revC.dtb"); + type = "flat_dt"; + arch = "arm64"; + load = <0x10000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + fdt-2 { + description = "Flattened Device Tree overlay"; + data = /incbin/("zynqmp-optee.dtbo"); + type = "flat_dt"; + arch = "arm64"; + load = <0x11000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + ramdisk { + description = "ramdisk"; + data = /incbin/("../../out-br/images/rootfs.cpio.gz"); + type = "ramdisk"; + arch = "arm64"; + os = "linux"; + compression = "gzip"; + hash { + algo = "sha1"; + }; + }; + }; + configurations { + default = "conf"; + conf { + description = "Boot Linux kernel with FDT blob + ramdisk"; + kernel = "kernel"; + fdt = "fdt-1", "fdt-2"; + ramdisk = "ramdisk"; + hash { + algo = "sha1"; + }; + }; + }; +}; diff --git a/build/zynqmp/fitImage-zynqmp-zcu106.its b/build/zynqmp/fitImage-zynqmp-zcu106.its new file mode 100644 index 0000000..bb32575 --- /dev/null +++ b/build/zynqmp/fitImage-zynqmp-zcu106.its @@ -0,0 +1,67 @@ +/dts-v1/; + +/ { + description = "U-Boot fitImage for plnx_aarch64 kernel"; + #address-cells = <1>; + + images { + kernel { + description = "Linux Kernel"; + data = /incbin/("./../../linux-xlnx/arch/arm64/boot/Image"); + type = "kernel"; + arch = "arm64"; + os = "linux"; + compression = "none"; + load = <0x80000>; + entry = <0x80000>; + hash { + algo = "sha1"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + data = /incbin/("../../u-boot-xlnx/arch/arm/dts/zynqmp-zcu106-revA.dtb"); + type = "flat_dt"; + arch = "arm64"; + load = <0x10000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + fdt-2 { + description = "Flattened Device Tree overlay"; + data = /incbin/("zynqmp-optee.dtbo"); + type = "flat_dt"; + arch = "arm64"; + load = <0x11000000>; + compression = "none"; + hash { + algo = "sha1"; + }; + }; + ramdisk { + description = "ramdisk"; + data = /incbin/("../../out-br/images/rootfs.cpio.gz"); + type = "ramdisk"; + arch = "arm64"; + os = "linux"; + compression = "gzip"; + hash { + algo = "sha1"; + }; + }; + }; + configurations { + default = "conf"; + conf { + description = "Boot Linux kernel with FDT blob + ramdisk"; + kernel = "kernel"; + fdt = "fdt-1", "fdt-2"; + ramdisk = "ramdisk"; + hash { + algo = "sha1"; + }; + }; + }; +}; diff --git a/build/zynqmp/zynqmp-optee.dtso b/build/zynqmp/zynqmp-optee.dtso new file mode 100644 index 0000000..1e214b1 --- /dev/null +++ b/build/zynqmp/zynqmp-optee.dtso @@ -0,0 +1,34 @@ +/dts-v1/; +/plugin/; +/ { + fragment@1 { + target-path = "/firmware"; + __overlay__ { + optee { + compatible = "linaro,optee-tz"; + method = "smc"; + }; + }; + }; + + fragment@2 { + target-path = "/"; + __overlay__ { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + atf@0 { + reg = <0x0 0x1000 0x0 0x1f000>; + no-map; + }; + + optee@60000000 { + reg = <0x0 0x60000000 0x0 0x10000000>; + no-map; + }; + }; + }; + }; +}; diff --git a/clone.sh b/clone.sh new file mode 100644 index 0000000..8aa859d --- /dev/null +++ b/clone.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# OP-TEE gits +git clone --branch 4.0.0 --depth 1 https://github.com/OP-TEE/optee_test.git optee_test + +# linaro-swg gits +git clone --branch optee-4.0.0 --depth 1 https://github.com/linaro-swg/linux.git linux +git clone --branch 4.0.0 --depth 1 https://github.com/linaro-swg/optee_benchmark.git optee_benchmark +git clone --branch 4.0.0 --depth 1 https://github.com/linaro-swg/optee_examples.git optee_examples + +# Misc gits +git clone --branch 2022.11.1 --depth 1 https://github.com/buildroot/buildroot.git buildroot +git clone --branch mbedtls-2.26.0 --depth 1 https://github.com/Mbed-TLS/mbedtls.git mbedtls +git clone https://github.com/apache/incubator-teaclave-trustzone-sdk.git optee_rust +cd optee_rust +git checkout 4031e7282a8f398f54faa19acb2b84fab05de877 +cd .. +git clone --branch v8.0.0 --depth 1 https://github.com/qemu/qemu.git qemu +git clone --branch v2.9 --depth 1 https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git trusted-firmware-a +git clone --branch v2.9 --depth 1 https://git.trustedfirmware.org/hafnium/hafnium.git hafnium +git clone --branch v2023.07.02 --depth 1 https://source.denx.de/u-boot/u-boot.git u-boot diff --git a/optee_client/.gitattributes b/optee_client/.gitattributes new file mode 100644 index 0000000..15e97ea --- /dev/null +++ b/optee_client/.gitattributes @@ -0,0 +1,4 @@ +# For git archive +.gitignore export-ignore +.gitattributes export-ignore +.travis.yml export-ignore diff --git a/optee_client/.github/workflows/ci.yml b/optee_client/.github/workflows/ci.yml new file mode 100644 index 0000000..1297791 --- /dev/null +++ b/optee_client/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI +on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + debian: + name: build (Debian) + runs-on: ubuntu-latest + container: jforissier/optee_client_ci_debian + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: arm64 with make + run: make -j O=out-make-aarch64 CROSS_COMPILE=aarch64-linux-gnu- + - name: armhf with make + run: make -j O=out-make-armhf CROSS_COMPILE=arm-linux-gnueabihf- + - name: arm64 with cmake + run: | + set -e -v + mkdir out-cmake-aarch64 && cd out-cmake-aarch64 + PKG_CONFIG=aarch64-linux-gnu-pkg-config cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc + make -j + - name: armhf with cmake + run: | + set -e -v + mkdir out-cmake-armhf && cd out-cmake-armhf + PKG_CONFIG=arm-linux-gnueabihf-pkg-config cmake .. -DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc + make -j + ubuntu: + name: build (Ubuntu) + runs-on: ubuntu-latest + container: jforissier/optee_client_ci_ubuntu + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: arm64 with make + run: make -j O=out-make-aarch64 CROSS_COMPILE=aarch64-linux-gnu- + - name: armhf with make + run: make -j O=out-make-armhf CROSS_COMPILE=arm-linux-gnueabihf- + - name: arm64 with cmake + run: | + set -e -v + mkdir out-cmake-aarch64 && cd out-cmake-aarch64 + PKG_CONFIG=aarch64-linux-gnu-pkg-config cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc + make -j + - name: armhf with cmake + run: | + set -e -v + mkdir out-cmake-armhf && cd out-cmake-armhf + PKG_CONFIG=arm-linux-gnueabihf-pkg-config cmake .. -DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc + make -j diff --git a/optee_client/.github/workflows/stales.yml b/optee_client/.github/workflows/stales.yml new file mode 100644 index 0000000..e0d47f3 --- /dev/null +++ b/optee_client/.github/workflows/stales.yml @@ -0,0 +1,27 @@ +name: 'Close stale issues and pull requests with no recent activity' +on: + schedule: + - cron: "15 00 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v4.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue has been marked as a stale issue because it has been open (more than) 30 days with no activity. Remove the stale label or add a comment, otherwise this issue will automatically be closed in 5 days. Note, that you can always re-open a closed issue at any time.' + stale-pr-message: 'This pull request has been marked as a stale pull request because it has been open (more than) 30 days with no activity. Remove the stale label or add a comment, otherwise this pull request will automatically be closed in 5 days. Note, that you can always re-open a closed issue at any time.' + stale-issue-label: Stale + stale-pr-label: Stale + exempt-issue-labels: bug,enhancement + exempt-pr-labels: bug,enhancement + days-before-stale: 30 + days-before-close: 5 + remove-stale-when-updated: true + remove-issue-stale-when-updated: true + remove-pr-stale-when-updated: true diff --git a/optee_client/.gitignore b/optee_client/.gitignore new file mode 100644 index 0000000..285b7ee --- /dev/null +++ b/optee_client/.gitignore @@ -0,0 +1,17 @@ +build +CMakeCache.txt +CMakeFiles +cmake_install.cmake +.cproject +cscope.* +Debug +GPATH +GRTAGS +GTAGS +tags +TAGS +/libs +/obj +out +.project +*.swp diff --git a/optee_client/Android.mk b/optee_client/Android.mk new file mode 100644 index 0000000..08e72e8 --- /dev/null +++ b/optee_client/Android.mk @@ -0,0 +1,106 @@ +################################################################################ +# Android optee-client, libckteec and optee-supplicant makefile # +################################################################################ +LOCAL_PATH := $(call my-dir) + +# 3 (debug) is too noisy +CFG_TEE_CLIENT_LOG_LEVEL ?= 2 +CFG_TEE_SUPP_LOG_LEVEL ?= 2 + +# Define Android-specific configuration before including config.mk +CFG_TEE_CLIENT_LOAD_PATH ?= /vendor/lib +TEEC_TEST_LOAD_PATH ?= /data/vendor/tee +CFG_TEE_FS_PARENT_PATH ?= /data/vendor/tee +CFG_TEE_SUPP_PLUGINS ?= y +ifneq ($(strip $($(combo_2nd_arch_prefix)TARGET_TOOLS_PREFIX)),) +CFG_TEE_PLUGIN_LOAD_PATH ?= /vendor/lib64/tee-supplicant/plugins/ +else +CFG_TEE_PLUGIN_LOAD_PATH ?= /vendor/lib/tee-supplicant/plugins/ +endif + +$(info CFG_TEE_PLUGIN_LOAD_PATH = ${CFG_TEE_PLUGIN_LOAD_PATH}) + + +################################################################################ +# Include optee-client common config and flags # +################################################################################ +include $(LOCAL_PATH)/config.mk +include $(LOCAL_PATH)/android_flags.mk + +optee_CFLAGS = $(CFLAGS) + +################################################################################ +# Build libteec.so - TEE (Trusted Execution Environment) shared library # +################################################################################ +include $(CLEAR_VARS) +LOCAL_CFLAGS += $(optee_CFLAGS) + +ifneq ($(CFG_TEE_CLIENT_LOG_FILE),) +LOCAL_CFLAGS += -DTEEC_LOG_FILE=\"$(CFG_TEE_CLIENT_LOG_FILE)\" +endif + +LOCAL_CFLAGS += -DDEBUGLEVEL_$(CFG_TEE_CLIENT_LOG_LEVEL) +LOCAL_CFLAGS += -DBINARY_PREFIX=\"TEEC\" + +LOCAL_SRC_FILES := libteec/src/tee_client_api.c \ + libteec/src/teec_trace.c +ifeq ($(CFG_TEE_BENCHMARK),y) +LOCAL_CFLAGS += -DCFG_TEE_BENCHMARK +LOCAL_SRC_FILES += teec_benchmark.c +endif + +LOCAL_C_INCLUDES := $(LOCAL_PATH)/libteec/include + +LOCAL_PRELINK_MODULE := false +LOCAL_MODULE := libteec + +LOCAL_MODULE_TAGS := optional +LOCAL_VENDOR_MODULE := true + +# Build the 32-bit and 64-bit versions. +LOCAL_MULTILIB := both +LOCAL_MODULE_TARGET_ARCH := arm arm64 + +LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/libteec/include + +include $(BUILD_SHARED_LIBRARY) + +################################################################################ +# Build libckteec.so # +################################################################################ +include $(CLEAR_VARS) + +LOCAL_CFLAGS += $(optee_CFLAGS) + +LOCAL_SRC_FILES := libckteec/src/pkcs11_api.c \ + libckteec/src/ck_debug.c \ + libckteec/src/ck_helpers.c \ + libckteec/src/invoke_ta.c \ + libckteec/src/pkcs11_processing.c \ + libckteec/src/pkcs11_token.c \ + libckteec/src/serializer.c \ + libckteec/src/serialize_ck.c + +LOCAL_C_INCLUDES := $(LOCAL_PATH)/libteec/include \ + $(LOCAL_PATH)/libckteec/include + +LOCAL_SHARED_LIBRARIES := libteec + +LOCAL_PRELINK_MODULE := false +LOCAL_MODULE := libckteec + +LOCAL_MODULE_TAGS := optional +LOCAL_VENDOR_MODULE := true + +# Build the 32-bit and 64-bit versions. +LOCAL_MULTILIB := both +LOCAL_MODULE_TARGET_ARCH := arm arm64 + +LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/libckteec/include + +include $(BUILD_SHARED_LIBRARY) + +################################################################################ +# Build TEE Supplicant # +################################################################################ +include $(LOCAL_PATH)/tee-supplicant/tee_supplicant_android.mk diff --git a/optee_client/CMakeLists.txt b/optee_client/CMakeLists.txt new file mode 100644 index 0000000..83107f4 --- /dev/null +++ b/optee_client/CMakeLists.txt @@ -0,0 +1,49 @@ +cmake_minimum_required(VERSION 3.4) +project(optee_client C) + +# https://cmake.org/Wiki/CMake_Useful_Variables +set(CMAKE_TOOLCHAIN_FILE CMakeToolchain.txt) + +option(CFG_WERROR "Build with -Werror" TRUE) +option(WITH_TEEACL "Build libteeacl" TRUE) + +include(GNUInstallDirs) + +################################################################################ +# Compiler flags: +# We want to use the same flags in the entire optee_client git +################################################################################ +add_compile_options( + -Wall -Wbad-function-cast -Wcast-align + -Werror-implicit-function-declaration -Wextra + -Wfloat-equal -Wformat-nonliteral -Wformat-security + -Wformat=2 -Winit-self -Wmissing-declarations + -Wmissing-format-attribute -Wmissing-include-dirs + -Wmissing-noreturn -Wmissing-prototypes -Wnested-externs + -Wpointer-arith -Wshadow -Wstrict-prototypes + -Wswitch-default -Wwrite-strings -fPIC +) +if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") + add_compile_options( + -Wunsafe-loop-optimizations + ) +endif() +if(CFG_WERROR) + add_compile_options(-Werror) +endif(CFG_WERROR) + +find_program(CCACHE_FOUND ccache) +if(CCACHE_FOUND) + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) +endif(CCACHE_FOUND) + +add_subdirectory(libteec) +add_subdirectory(tee-supplicant) +add_subdirectory(libckteec) +if(WITH_TEEACL) + find_package(PkgConfig REQUIRED) + pkg_check_modules(uuid REQUIRED IMPORTED_TARGET uuid) + add_subdirectory(libteeacl) +endif(WITH_TEEACL) +add_subdirectory(libseteec) diff --git a/optee_client/CMakeToolchain.txt b/optee_client/CMakeToolchain.txt new file mode 100644 index 0000000..71eb52f --- /dev/null +++ b/optee_client/CMakeToolchain.txt @@ -0,0 +1,3 @@ +set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_SYSTEM_PROCESSOR arm) diff --git a/optee_client/LICENSE b/optee_client/LICENSE new file mode 100644 index 0000000..76c5d00 --- /dev/null +++ b/optee_client/LICENSE @@ -0,0 +1,27 @@ +Unless it has its own copyright/license embedded in its body, each source file +is subject to the following license terms: + +Copyright (c) 2015, Linaro Limited +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/optee_client/Makefile b/optee_client/Makefile new file mode 100644 index 0000000..0d5a500 --- /dev/null +++ b/optee_client/Makefile @@ -0,0 +1,174 @@ +# Public variables are stored in config.mk +include ./config.mk + +######################################################################### +# Set Internal Variables # +# May be modified to match your setup # +######################################################################### +ifneq ($(V),1) +VPREFIX := @ +endif +export VPREFIX + +EXPORT_DIR ?= $(O)/export +DESTDIR ?= $(EXPORT_DIR) +SBINDIR ?= /usr/sbin +LIBDIR ?= /usr/lib +INCLUDEDIR ?= /usr/include +sbindir ?= $(SBINDIR) +libdir ?= $(LIBDIR) +includedir ?= $(INCLUDEDIR) + +WITH_TEEACL ?= 1 + +.PHONY: all build build-libteec build-libckteec build-libseteec \ + build-libteeacl install copy_export clean cscope \ + clean-cscope \ + checkpatch-pre-req checkpatch-modified-patch checkpatch-modified-file \ + checkpatch-last-commit-patch checkpatch-last-commit-file \ + checkpatch-base-commit-patch checkpatch-base-commit-file \ + checkpatch-all-files distclean + +all: build install + +build-libteec: + @echo "Building libteec.so" + @$(MAKE) --directory=libteec --no-print-directory --no-builtin-variables \ + CFG_TEE_BENCHMARK=$(CFG_TEE_BENCHMARK) CFG_TEE_CLIENT_LOG_LEVEL=$(CFG_TEE_CLIENT_LOG_LEVEL) + +build-tee-supplicant: build-libteec + @echo "Building tee-supplicant" + $(MAKE) --directory=tee-supplicant --no-print-directory --no-builtin-variables CFG_TEE_SUPP_LOG_LEVEL=$(CFG_TEE_SUPP_LOG_LEVEL) + +build: build-libteec build-tee-supplicant build-libckteec build-libseteec +ifeq ($(WITH_TEEACL),1) +build: build-libteeacl +endif + +build-libckteec: build-libteec + @echo "Building libckteec.so" + @$(MAKE) --directory=libckteec --no-print-directory --no-builtin-variables + +build-libseteec: build-libteec + @echo "Building libseteec.so" + @$(MAKE) --directory=libseteec --no-print-directory --no-builtin-variables + +build-libteeacl: + @echo "Building libteeacl.so" + @$(MAKE) --directory=libteeacl --no-print-directory --no-builtin-variables + +install: copy_export + +clean: clean-libteec clean-tee-supplicant clean-cscope clean-libckteec \ + clean-libseteec +ifeq ($(WITH_TEEACL),1) +clean: clean-libteeacl +endif + +clean-libteec: + @$(MAKE) --directory=libteec --no-print-directory clean + +clean-tee-supplicant: + @$(MAKE) --directory=tee-supplicant --no-print-directory clean + +clean-libckteec: + @$(MAKE) --directory=libckteec --no-print-directory clean + +clean-libseteec: + @$(MAKE) --directory=libseteec --no-print-directory clean + +clean-libteeacl: + @$(MAKE) --directory=libteeacl --no-print-directory clean + +cscope: + @echo " CSCOPE" + ${VPREFIX}find ${CURDIR} -name "*.[chsS]" > cscope.files + ${VPREFIX}cscope -b -q -k + +clean-cscope: + ${VPREFIX}rm -f cscope.* + +# Various checkpatch targets. The ones ending with "patch" only considers the +# patch, whilst the ones ending with "file" checks the complete file. +# +-------------------------------+------------+----------------------------+ +# | Target commit | File/Patch | Comment | +# +-------------------------------+------------+----------------------------+ +# | checkpatch-modified-patch | Patch | Check local modifications | +# +-------------------------------+------------+----------------------------+ +# | checkpatch-modified-file | File | Check Local modifications | +# +-------------------------------+------------+----------------------------+ +# | checkpatch-last-commit-patch | Patch | Check against HEAD^ | +# +-------------------------------+------------+----------------------------+ +# | checkpatch-last-commit-file | File | Check against HEAD^ | +# +-------------------------------+------------+----------------------------+ +# | checkpatch-base-commit-patch | Patch | Against specic commit | +# +-------------------------------+------------+----------------------------+ +# | checkpatch-base-commit-file | File | Against specic commit | +# +-------------------------------+------------+----------------------------+ +# | checkpatch-all-files | File | Check all tracked files | +# +-------------------------------+------------+----------------------------+ +CHECKPATCH_IGNORE ?= --ignore NEW_TYPEDEFS --no-signoff +CHECKPATCH_STRICT ?= --strict +CHECKPATCH_ARGS ?= $(CHECKPATCH_IGNORE) $(CHECKPATCH_STRICT) --no-tree --terse +CHECKPATCH_PATCH_ARGS := $(CHECKPATCH_ARGS) --patch +CHECKPATCH_FILE_ARGS := $(CHECKPATCH_ARGS) --file --no-patch + +checkpatch-pre-req: + @echo " CHECKPATCH" +ifndef CHECKPATCH + $(error "Environment variable CHECKPATCH must point to Linux kernels checkpatch script") +else +ifeq (,$(wildcard ${CHECKPATCH})) + $(error "CHECKPATCH points to the incorrect file") +endif +endif + +checkpatch-modified-patch: checkpatch-pre-req + ${VPREFIX}git diff | ${CHECKPATCH} $(CHECKPATCH_PATCH_ARGS) - || true + +checkpatch-modified-file: checkpatch-pre-req + ${VPREFIX}${CHECKPATCH} $(CHECKPATCH_FILE_ARGS) $(shell git diff --name-only) + + +checkpatch-last-commit-patch: checkpatch-pre-req + ${VPREFIX}git diff HEAD^ | ${CHECKPATCH} $(CHECKPATCH_PATCH_ARGS) - || true + +checkpatch-last-commit-file: checkpatch-pre-req + ${VPREFIX}${CHECKPATCH} $(CHECKPATCH_FILE_ARGS) $(shell git diff --name-only HEAD^) + + +checkpatch-base-commit-patch: checkpatch-pre-req +ifndef BASE_COMMIT + $(error "Environment variable BASE_COMMIT must contain a valid commit") +endif + ${VPREFIX}git diff $(BASE_COMMIT) | ${CHECKPATCH} $(CHECKPATCH_PATCH_ARGS) - || true + +checkpatch-base-commit-file: checkpatch-pre-req +ifndef BASE_COMMIT + $(error "Environment variable BASE_COMMIT must contain a valid commit") +endif + ${VPREFIX}${CHECKPATCH} $(CHECKPATCH_FILE_ARGS) $(shell git diff --name-only ${BASE_COMMIT}) + +checkpatch-all-files: checkpatch-pre-req + ${VPREFIX}${CHECKPATCH} $(CHECKPATCH_FILE_ARGS) $(shell git ls-files) + +distclean: clean + +copy_export: build + mkdir -p $(DESTDIR)$(sbindir) $(DESTDIR)$(libdir) $(DESTDIR)$(includedir) + cp config.mk $(DESTDIR)/$(includedir)/optee_client_config.mk + cp -d ${O}/libteec/libteec.so* $(DESTDIR)$(libdir) + cp -d ${O}/libteec/libteec.a $(DESTDIR)$(libdir) + cp ${O}/tee-supplicant/tee-supplicant $(DESTDIR)$(sbindir) + cp libteec/include/*.h $(DESTDIR)$(includedir) + cp libckteec/include/*.h $(DESTDIR)$(includedir) + cp -d ${O}/libckteec/libckteec.so* $(DESTDIR)$(libdir) + cp -d ${O}/libckteec/libckteec.a $(DESTDIR)$(libdir) +ifeq ($(WITH_TEEACL),1) + cp libteeacl/include/*.h $(DESTDIR)$(includedir) + cp -d ${O}/libteeacl/libteeacl.so* $(DESTDIR)$(libdir) + cp -d ${O}/libteeacl/libteeacl.a $(DESTDIR)$(libdir) +endif + cp libseteec/include/*.h $(DESTDIR)$(includedir) + cp -d ${O}/libseteec/libseteec.so* $(DESTDIR)$(libdir) + cp -d ${O}/libseteec/libseteec.a $(DESTDIR)$(libdir) diff --git a/optee_client/README.md b/optee_client/README.md new file mode 100644 index 0000000..045dbfd --- /dev/null +++ b/optee_client/README.md @@ -0,0 +1,10 @@ +# OP-TEE Client API +This git contains source code for the non-secure side implementation of the +OP-TEE project making up the client library and tee-supplicant. + +All official OP-TEE documentation has moved to http://optee.readthedocs.io. The +information that used to be here in this git can be found under [optee_client]. + +// OP-TEE core maintainers + +[optee_client]: https://optee.readthedocs.io/en/latest/building/gits/optee_client.html diff --git a/optee_client/android_flags.mk b/optee_client/android_flags.mk new file mode 100644 index 0000000..17c57fe --- /dev/null +++ b/optee_client/android_flags.mk @@ -0,0 +1,27 @@ +######################################################################### +# COMMON COMPILATION FLAGS # +######################################################################### +CFLAGS := -Wall -Wbad-function-cast -Wcast-align \ + -Werror-implicit-function-declaration -Wextra \ + -Wfloat-equal -Wformat-nonliteral -Wformat-security \ + -Wformat=2 -Winit-self -Wmissing-declarations \ + -Wmissing-format-attribute -Wmissing-include-dirs \ + -Wmissing-noreturn -Wmissing-prototypes -Wnested-externs \ + -Wpointer-arith -Wshadow -Wstrict-prototypes \ + -Wswitch-default \ + -Wwrite-strings +ifeq ($(CFG_WERROR),y) +CFLAGS += -Werror +endif +CFLAGS += -c -fPIC + +DEBUG ?= 0 +ifeq ($(DEBUG), 1) +CFLAGS += -DDEBUG -O0 -g +endif + +RM := rm -rf + +define rmdir +if [ -d "$(1)" ] ; then rmdir --ignore-fail-on-non-empty $(1) ; fi +endef diff --git a/optee_client/ci/Dockerfile.debian b/optee_client/ci/Dockerfile.debian new file mode 100644 index 0000000..875f113 --- /dev/null +++ b/optee_client/ci/Dockerfile.debian @@ -0,0 +1,21 @@ +# Dockerfile for CI image used in ../.github/workflows/ci.yml + +FROM debian:bullseye-slim +MAINTAINER Jerome Forissier + +ENV LANG=C.UTF-8 + +RUN dpkg --add-architecture armhf +RUN dpkg --add-architecture arm64 + +RUN apt update +RUN apt upgrade -y +RUN apt install -y \ + cmake \ + dpkg-dev \ + gcc-aarch64-linux-gnu \ + gcc-arm-linux-gnueabihf \ + make \ + pkg-config \ + uuid-dev:armhf \ + uuid-dev:arm64 diff --git a/optee_client/ci/Dockerfile.ubuntu b/optee_client/ci/Dockerfile.ubuntu new file mode 100644 index 0000000..9a4f8a5 --- /dev/null +++ b/optee_client/ci/Dockerfile.ubuntu @@ -0,0 +1,30 @@ +# Dockerfile for CI image used in ../.github/workflows/ci.yml + +FROM ubuntu:22.04 +MAINTAINER Jerome Forissier + +ENV LANG=C.UTF-8 + +RUN dpkg --add-architecture armhf +RUN dpkg --add-architecture arm64 + +RUN echo 'deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricted universe multiverse' > /etc/apt/sources.list +RUN echo 'deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe multiverse' >> /etc/apt/sources.list +RUN echo 'deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse' >> /etc/apt/sources.list +RUN echo 'deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse' >> /etc/apt/sources.list +RUN echo 'deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe multiverse' >> /etc/apt/sources.list +RUN echo 'deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe multiverse' >> /etc/apt/sources.list +RUN echo 'deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted universe multiverse' >> /etc/apt/sources.list +RUN echo 'deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse' >> /etc/apt/sources.list + +RUN apt update +RUN apt upgrade -y +RUN apt install -y \ + cmake \ + dpkg-dev \ + gcc-aarch64-linux-gnu \ + gcc-arm-linux-gnueabihf \ + make \ + pkg-config \ + uuid-dev:armhf \ + uuid-dev:arm64 diff --git a/optee_client/config.mk b/optee_client/config.mk new file mode 100644 index 0000000..24904af --- /dev/null +++ b/optee_client/config.mk @@ -0,0 +1,79 @@ +######################################################################### +# Public variables # +# Developers may override these values when calling the makefile, # +# as for example # +# CFG_TEE_CLIENT_LOG_LEVEL=1 make # +# Note: # +# Please do not use export to declare the variables, so that to avoid # +# compiling problem for android platform # +######################################################################### + +# CFG_TEE_CLIENT_LOG_LEVEL +# Client (User Non Secure) log level +# Supported values: 0 (no traces) to 4 (all traces) +CFG_TEE_CLIENT_LOG_LEVEL?=1 + +# CFG_TEE_SUPP_LOG_LEVEL +# Supplicant log level +# Supported values: 0 (no traces) to 4 (all traces) +CFG_TEE_SUPP_LOG_LEVEL?=1 + +# CFG_TEE_FS_PARENT_PATH +# Path to folder that will contain TEE filesystem. +# This folder can be created with the required permission in an init +# script during boot, else it will be created by the tee-supplicant on +# first REE FS access. +CFG_TEE_FS_PARENT_PATH ?= /data/tee + +# CFG_TEE_CLIENT_LOG_FILE +# The location of the client log file when logging to file is enabled. +CFG_TEE_CLIENT_LOG_FILE ?= $(CFG_TEE_FS_PARENT_PATH)/teec.log + +# CFG_TEE_CLIENT_LOAD_PATH +# Colon-separated list of paths where tee-supplicant loads TAs from. +# For example: CFG_TEE_CLIENT_LOAD_PATH ?= /lib:/vendor/lib +# Note that the TA files are typically in a sub-directory (see the +# --ta-dir option). +CFG_TEE_CLIENT_LOAD_PATH ?= /lib + +# CFG_TEE_SUPP_PLUGINS +# Enable (y) or disable (n) TEE supplicant plugin support +CFG_TEE_SUPP_PLUGINS ?= y + +# CFG_TEE_PLUGIN_LOAD_PATH +# The location of the user plugins +CFG_TEE_PLUGIN_LOAD_PATH ?= /usr/lib/tee-supplicant/plugins/ + +# CFG_TA_TEST_PATH +# Enable the tee test path. When enabled, the supplicant will try +# loading from a debug path before the regular path. This allows test +# such as 1008.5 that test loading of corrupt TAs. +CFG_TA_TEST_PATH ?= n + +# CFG_GP_SOCKETS +# Enable Global Platform Sockets support +CFG_GP_SOCKETS ?= y + +# CFG_TA_GPROF_SUPPORT +# Enable dumping gprof data, not used unless secure world decides +# to dump something +CFG_TA_GPROF_SUPPORT ?= y + +# CFG_FTRACE_SUPPORT +# Enable dumping ftrace data, not used unless secure world decides +# to dump something +CFG_FTRACE_SUPPORT ?= y + +# Default output directory. +# May be absolute, or relative to the optee_client source directory. +O ?= out + +# To be used instead of $(O) in sub-directories +OO := $(if $(filter /%,$(O)),$(O),$(CURDIR)/../$(O)) + +######################################################################### +# Private Values # +######################################################################### + +# Check that settings are coherent. + diff --git a/optee_client/flags.mk b/optee_client/flags.mk new file mode 100644 index 0000000..be4088c --- /dev/null +++ b/optee_client/flags.mk @@ -0,0 +1,37 @@ +######################################################################### +# COMMON COMPILATION FLAGS # +######################################################################### + +CROSS_COMPILE ?= arm-linux-gnueabihf- +CC ?= $(CROSS_COMPILE)gcc +AR ?= $(CROSS_COMPILE)ar +PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config + +C_COMPILER=$(shell readlink -f $$(which $(CC))) + +override CFLAGS += -Wall -Wbad-function-cast -Wcast-align \ + -Werror-implicit-function-declaration -Wextra \ + -Wfloat-equal -Wformat-nonliteral -Wformat-security \ + -Wformat=2 -Winit-self -Wmissing-declarations \ + -Wmissing-format-attribute -Wmissing-include-dirs \ + -Wmissing-noreturn -Wmissing-prototypes -Wnested-externs \ + -Wpointer-arith -Wshadow -Wstrict-prototypes \ + -Wswitch-default -Wwrite-strings -D_FILE_OFFSET_BITS=64 +ifneq (,$(findstring gcc,$(C_COMPILER))) +override CFLAGS += -Wunsafe-loop-optimizations +endif +ifeq ($(CFG_WERROR),y) +override CFLAGS += -Werror +endif +override CFLAGS += -c -fPIC + +DEBUG ?= 0 +ifeq ($(DEBUG), 1) +override CFLAGS += -DDEBUG -O0 -g +endif + +RM := rm -f + +define rmdir +if [ -d "$(1)" ] ; then rmdir --ignore-fail-on-non-empty $(1) ; fi +endef diff --git a/optee_client/libckteec/CMakeLists.txt b/optee_client/libckteec/CMakeLists.txt new file mode 100644 index 0000000..978e665 --- /dev/null +++ b/optee_client/libckteec/CMakeLists.txt @@ -0,0 +1,77 @@ +project(ckteec + VERSION 0.1.0 + LANGUAGES C) + +################################################################################ +# Packages +################################################################################ +find_package(Threads REQUIRED) +if(NOT THREADS_FOUND) + message(FATAL_ERROR "Threads not found") +endif() + +include(GNUInstallDirs) + +################################################################################ +# Source files +################################################################################ +set(SRC + src/pkcs11_api.c + src/ck_debug.c + src/ck_helpers.c + src/invoke_ta.c + src/pkcs11_processing.c + src/pkcs11_token.c + src/serializer.c + src/serialize_ck.c +) + +################################################################################ +# Built library +################################################################################ +add_library(ckteec ${SRC}) + +set_target_properties(ckteec PROPERTIES + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR} +) + +################################################################################ +# Flags always set +################################################################################ +target_compile_definitions(ckteec + PRIVATE -D_GNU_SOURCE + PRIVATE -DBINARY_PREFIX="LT" +) + +################################################################################ +# Optional flags +################################################################################ + +################################################################################ +# Public and private header and library dependencies +################################################################################ +target_include_directories(ckteec + PUBLIC $ + $ + PRIVATE src +) + +target_include_directories(teec + PUBLIC include +) + +target_link_libraries(ckteec + PRIVATE pthread + PRIVATE teec + PRIVATE m +) + +################################################################################ +# Install targets +################################################################################ +install(TARGETS ckteec + DESTINATION ${CMAKE_INSTALL_LIBDIR} +) + +add_subdirectory(include) diff --git a/optee_client/libckteec/Makefile b/optee_client/libckteec/Makefile new file mode 100644 index 0000000..de7e330 --- /dev/null +++ b/optee_client/libckteec/Makefile @@ -0,0 +1,78 @@ +include ../flags.mk +include ../config.mk + +OUT_DIR := $(OO)/libckteec + +.PHONY: all libckteec clean + +all: libckteec +install: libckteec + +LIB_NAME := libckteec +MAJOR_VERSION := 0 +MINOR_VERSION := 1 +PATCH_VERSION := 0 + +LIB_MAJOR := $(LIB_NAME).so.$(MAJOR_VERSION) +LIB_MAJ_MIN := $(LIB_NAME).so.$(MAJOR_VERSION).$(MINOR_VERSION) +LIB_MAJ_MIN_PAT := $(LIB_NAME).so.$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION) +LIBCKTEEC_SO_LIBRARY := $(LIB_MAJ_MIN_PAT) +LIBCKTEEC_AR_LIBRARY := $(LIB_NAME).a + +LIBCKTEEC_SRC_DIR := src + +LIBCKTEEC_SRCS = pkcs11_api.c +LIBCKTEEC_SRCS += ck_debug.c +LIBCKTEEC_SRCS += ck_helpers.c +LIBCKTEEC_SRCS += invoke_ta.c +LIBCKTEEC_SRCS += pkcs11_processing.c +LIBCKTEEC_SRCS += pkcs11_token.c +LIBCKTEEC_SRCS += serializer.c +LIBCKTEEC_SRCS += serialize_ck.c + +LIBCKTEEC_INCLUDES = ${CURDIR}/include +LIBCKTEEC_INCLUDES += ${CURDIR}/../libteec/include + +LIBCKTEEC_CFLAGS := $(addprefix -I, $(LIBCKTEEC_INCLUDES)) \ + $(CFLAGS) -D_GNU_SOURCE -fPIC + +LIBCKTEEC_LFLAGS := $(LDFLAGS) -L$(OUT_DIR)/../libteec -lteec + +LIBCKTEEC_OBJ_DIR := $(OUT_DIR) +LIBCKTEEC_OBJS := $(patsubst %.c,$(LIBCKTEEC_OBJ_DIR)/%.o, $(LIBCKTEEC_SRCS)) + +$(LIBCKTEEC_OBJ_DIR)/%.o: ${LIBCKTEEC_SRC_DIR}/%.c + $(VPREFIX)mkdir -p $(LIBCKTEEC_OBJ_DIR) + @echo " CC $<" + $(VPREFIX)$(CC) $(LIBCKTEEC_CFLAGS) -c $< -o $@ + +libckteec: $(OUT_DIR)/$(LIBCKTEEC_SO_LIBRARY) + +$(OUT_DIR)/$(LIBCKTEEC_SO_LIBRARY): $(LIBCKTEEC_OBJS) + @echo " LINK $@" + $(VPREFIX)$(CC) -shared -Wl,-soname,$(LIB_MAJOR) -o $@ $+ $(LIBCKTEEC_LFLAGS) + @echo "" + +libckteec: $(OUT_DIR)/$(LIBCKTEEC_AR_LIBRARY) + +$(OUT_DIR)/$(LIBCKTEEC_AR_LIBRARY): $(LIBCKTEEC_OBJS) + @echo " AR $@" + $(VPREFIX)$(AR) rcs $@ $+ + +libckteec: + $(VPREFIX)ln -sf $(LIB_MAJ_MIN_PAT) $(OUT_DIR)/$(LIB_MAJ_MIN) + $(VPREFIX)ln -sf $(LIB_MAJ_MIN) $(OUT_DIR)/$(LIB_MAJOR) + $(VPREFIX)ln -sf $(LIB_MAJOR) $(OUT_DIR)/$(LIB_NAME).so + +################################################################################ +# Cleaning up configuration +################################################################################ +clean: + $(RM) $(LIBCKTEEC_OBJS) + $(RM) $(OUT_DIR)/$(LIB_MAJ_MIN_PAT) + $(RM) $(OUT_DIR)/$(LIB_MAJ_MIN) + $(RM) $(OUT_DIR)/$(LIB_MAJOR) + $(RM) $(OUT_DIR)/$(LIBCKTEEC_SO_LIBRARY) + $(RM) $(OUT_DIR)/$(LIBCKTEEC_AR_LIBRARY) + $(call rmdir,$(OUT_DIR)) + diff --git a/optee_client/libckteec/include/CMakeLists.txt b/optee_client/libckteec/include/CMakeLists.txt new file mode 100644 index 0000000..c342012 --- /dev/null +++ b/optee_client/libckteec/include/CMakeLists.txt @@ -0,0 +1,12 @@ +project(libckteec-headers C) + +FILE(GLOB INSTALL_HEADERS "*.h") + +add_library(${PROJECT_NAME} INTERFACE) +target_include_directories( + ${PROJECT_NAME} + INTERFACE $ + $ +) + +install(FILES ${INSTALL_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/optee_client/libckteec/include/ck_debug.h b/optee_client/libckteec/include/ck_debug.h new file mode 100644 index 0000000..3e7e210 --- /dev/null +++ b/optee_client/libckteec/include/ck_debug.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#ifndef LIBCKTEEC_CK_DEBUG_H +#define LIBCKTEEC_CK_DEBUG_H + +#include + +/* Return a pointer to a string buffer of "CKR_xxx\0" return value ID */ +const char *ckr2str(CK_RV id); + +/* ckm2str - Return string buffer of "CKM_xxx\0" for a mechanism ID */ +const char *ckm2str(CK_MECHANISM_TYPE id); + +/* slot_ckf2str - Return string buffer of "CKF_xxx\0" for a slot flag */ +const char *slot_ckf2str(CK_ULONG flag); + +/* token_ckf2str - Return string buffer "CKF_xxx\0" for a token flag */ +const char *token_ckf2str(CK_ULONG flag); + +/* mecha_ckf2str - Return string buffer "CKF_xxx\0" for a mechanism flag */ +const char *mecha_ckf2str(CK_ULONG flag); + +/* session_ckf2str - Return string buffer "CKF_xxx\0" for a session flag */ +const char *session_ckf2str(CK_ULONG flag); + +/* session_cks2str - Return string buffer "CKS_xxx\0" for a session state */ +const char *session_cks2str(CK_ULONG flag); + +/* Return a pointer to a string buffer of "CKA_xxx\0" attribute ID */ +const char *cka2str(CK_ATTRIBUTE_TYPE id); + +/* Return a pointer to a string buffer of "CKO_xxx\0" object class ID */ +const char *cko2str(CK_OBJECT_CLASS id); + +/* Return a pointer to a string buffer of "CKK_xxx\0" key type ID */ +const char *ckk2str(CK_KEY_TYPE id); + +#endif /*LIBCKTEEC_CK_DEBUG_H*/ diff --git a/optee_client/libckteec/include/pkcs11.h b/optee_client/libckteec/include/pkcs11.h new file mode 100644 index 0000000..6d7a0f7 --- /dev/null +++ b/optee_client/libckteec/include/pkcs11.h @@ -0,0 +1,1248 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#ifndef PKCS11_H +#define PKCS11_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * PKCS#11 Cryptoki API v2.40-errata01, See specification from: + * http://docs.oasis-open.org/pkcs11/pkcs11-base/v2.40/errata01/os/pkcs11-base-v2.40-errata01-os-complete.html + */ +#define CK_PKCS11_VERSION_MAJOR 2 +#define CK_PKCS11_VERSION_MINOR 40 +#define CK_PKCS11_VERSION_PATCH 1 + +typedef unsigned char CK_BYTE; +typedef unsigned long CK_ULONG; +typedef long CK_LONG; + +typedef CK_BYTE CK_CHAR; +typedef CK_BYTE CK_UTF8CHAR; + +typedef CK_BYTE *CK_BYTE_PTR; + +typedef CK_ULONG *CK_ULONG_PTR; + +typedef CK_CHAR *CK_CHAR_PTR; +typedef CK_UTF8CHAR *CK_UTF8CHAR_PTR; + +typedef void *CK_VOID_PTR; +typedef CK_VOID_PTR *CK_VOID_PTR_PTR; + +typedef CK_BYTE CK_BBOOL; + +#define CK_TRUE 1 +#define CK_FALSE 0 + +typedef CK_ULONG CK_FLAGS; + +#define CK_UNAVAILABLE_INFORMATION (~0UL) +#define CK_EFFECTIVELY_INFINITE 0UL + +typedef CK_ULONG CK_SESSION_HANDLE; +typedef CK_SESSION_HANDLE *CK_SESSION_HANDLE_PTR; + +typedef CK_ULONG CK_OBJECT_HANDLE; +typedef CK_OBJECT_HANDLE *CK_OBJECT_HANDLE_PTR; + +#define CK_INVALID_HANDLE 0 + +typedef CK_ULONG CK_SLOT_ID; +typedef CK_SLOT_ID *CK_SLOT_ID_PTR; + +typedef struct CK_VERSION CK_VERSION; +typedef struct CK_VERSION *CK_VERSION_PTR; + +struct CK_VERSION { + CK_BYTE major; + CK_BYTE minor; +}; + +typedef struct CK_DATE CK_DATE; +typedef struct CK_DATE *CK_DATE_PTR; + +struct CK_DATE { + CK_CHAR year[4]; + CK_CHAR month[2]; + CK_CHAR day[2]; +}; + +/* + * PKCS#11 Objects attributes + */ + +typedef CK_ULONG CK_ATTRIBUTE_TYPE; + +typedef struct CK_ATTRIBUTE CK_ATTRIBUTE; +typedef struct CK_ATTRIBUTE *CK_ATTRIBUTE_PTR; + +struct CK_ATTRIBUTE { + CK_ATTRIBUTE_TYPE type; + CK_VOID_PTR pValue; + CK_ULONG ulValueLen; +}; + +/* + * Values for CK_ATTRIBUTE_TYPE + * + * This does not cover the full PKCS#11 IDs. + */ +#define CKF_ARRAY_ATTRIBUTE (1U << 30) +#define CKA_VENDOR_DEFINED (1U << 31) +#define CKA_CLASS 0x0000 +#define CKA_TOKEN 0x0001 +#define CKA_PRIVATE 0x0002 +#define CKA_LABEL 0x0003 +#define CKA_APPLICATION 0x0010 +#define CKA_VALUE 0x0011 +#define CKA_OBJECT_ID 0x0012 +#define CKA_CERTIFICATE_TYPE 0x0080 +#define CKA_ISSUER 0x0081 +#define CKA_SERIAL_NUMBER 0x0082 +#define CKA_AC_ISSUER 0x0083 +#define CKA_OWNER 0x0084 +#define CKA_ATTR_TYPES 0x0085 +#define CKA_TRUSTED 0x0086 +#define CKA_CERTIFICATE_CATEGORY 0x0087 +#define CKA_JAVA_MIDP_SECURITY_DOMAIN 0x0088 +#define CKA_URL 0x0089 +#define CKA_HASH_OF_SUBJECT_PUBLIC_KEY 0x008a +#define CKA_HASH_OF_ISSUER_PUBLIC_KEY 0x008b +#define CKA_NAME_HASH_ALGORITHM 0x008c +#define CKA_CHECK_VALUE 0x0090 +#define CKA_KEY_TYPE 0x0100 +#define CKA_SUBJECT 0x0101 +#define CKA_ID 0x0102 +#define CKA_SENSITIVE 0x0103 +#define CKA_ENCRYPT 0x0104 +#define CKA_DECRYPT 0x0105 +#define CKA_WRAP 0x0106 +#define CKA_UNWRAP 0x0107 +#define CKA_SIGN 0x0108 +#define CKA_SIGN_RECOVER 0x0109 +#define CKA_VERIFY 0x010a +#define CKA_VERIFY_RECOVER 0x010b +#define CKA_DERIVE 0x010c +#define CKA_START_DATE 0x0110 +#define CKA_END_DATE 0x0111 +#define CKA_MODULUS 0x0120 +#define CKA_MODULUS_BITS 0x0121 +#define CKA_PUBLIC_EXPONENT 0x0122 +#define CKA_PRIVATE_EXPONENT 0x0123 +#define CKA_PRIME_1 0x0124 +#define CKA_PRIME_2 0x0125 +#define CKA_EXPONENT_1 0x0126 +#define CKA_EXPONENT_2 0x0127 +#define CKA_COEFFICIENT 0x0128 +#define CKA_PUBLIC_KEY_INFO 0x0129 +#define CKA_PRIME 0x0130 +#define CKA_SUBPRIME 0x0131 +#define CKA_BASE 0x0132 +#define CKA_PRIME_BITS 0x0133 +#define CKA_SUBPRIME_BITS 0x0134 +#define CKA_VALUE_BITS 0x0160 +#define CKA_VALUE_LEN 0x0161 +#define CKA_EXTRACTABLE 0x0162 +#define CKA_LOCAL 0x0163 +#define CKA_NEVER_EXTRACTABLE 0x0164 +#define CKA_ALWAYS_SENSITIVE 0x0165 +#define CKA_KEY_GEN_MECHANISM 0x0166 +#define CKA_MODIFIABLE 0x0170 +#define CKA_COPYABLE 0x0171 +#define CKA_DESTROYABLE 0x0172 +#define CKA_EC_PARAMS 0x0180 +#define CKA_EC_POINT 0x0181 +#define CKA_ALWAYS_AUTHENTICATE 0x0202 +#define CKA_WRAP_WITH_TRUSTED 0x0210 +#define CKA_WRAP_TEMPLATE (0x0211 | CKF_ARRAY_ATTRIBUTE) +#define CKA_UNWRAP_TEMPLATE (0x0212 | CKF_ARRAY_ATTRIBUTE) +#define CKA_DERIVE_TEMPLATE (0x0213 | CKF_ARRAY_ATTRIBUTE) +#define CKA_OTP_FORMAT 0x0220 +#define CKA_OTP_LENGTH 0x0221 +#define CKA_OTP_TIME_INTERVAL 0x0222 +#define CKA_OTP_USER_FRIENDLY_MODE 0x0223 +#define CKA_OTP_CHALLENGE_REQUIREMENT 0x0224 +#define CKA_OTP_TIME_REQUIREMENT 0x0225 +#define CKA_OTP_COUNTER_REQUIREMENT 0x0226 +#define CKA_OTP_PIN_REQUIREMENT 0x0227 +#define CKA_OTP_COUNTER 0x022e +#define CKA_OTP_TIME 0x022f +#define CKA_OTP_USER_IDENTIFIER 0x022a +#define CKA_OTP_SERVICE_IDENTIFIER 0x022b +#define CKA_OTP_SERVICE_LOGO 0x022c +#define CKA_OTP_SERVICE_LOGO_TYPE 0x022d +#define CKA_GOSTR3410_PARAMS 0x0250 +#define CKA_GOSTR3411_PARAMS 0x0251 +#define CKA_GOST28147_PARAMS 0x0252 +#define CKA_HW_FEATURE_TYPE 0x0300 +#define CKA_RESET_ON_INIT 0x0301 +#define CKA_HAS_RESET 0x0302 +#define CKA_PIXEL_X 0x0400 +#define CKA_PIXEL_Y 0x0401 +#define CKA_RESOLUTION 0x0402 +#define CKA_CHAR_ROWS 0x0403 +#define CKA_CHAR_COLUMNS 0x0404 +#define CKA_COLOR 0x0405 +#define CKA_BITS_PER_PIXEL 0x0406 +#define CKA_CHAR_SETS 0x0480 +#define CKA_ENCODING_METHODS 0x0481 +#define CKA_MIME_TYPES 0x0482 +#define CKA_MECHANISM_TYPE 0x0500 +#define CKA_REQUIRED_CMS_ATTRIBUTES 0x0501 +#define CKA_DEFAULT_CMS_ATTRIBUTES 0x0502 +#define CKA_SUPPORTED_CMS_ATTRIBUTES 0x0503 +#define CKA_ALLOWED_MECHANISMS (0x0600 | CKF_ARRAY_ATTRIBUTE) + +/* Attribute CKA_CLASS refers to a CK_OBJECT_CLASS typed value */ +typedef CK_ULONG CK_OBJECT_CLASS; +typedef CK_OBJECT_CLASS *CK_OBJECT_CLASS_PTR; + +/* Values for type CK_OBJECT_CLASS */ +#define CKO_VENDOR_DEFINED (1U << 31) +#define CKO_DATA 0x0 +#define CKO_CERTIFICATE 0x1 +#define CKO_PUBLIC_KEY 0x2 +#define CKO_PRIVATE_KEY 0x3 +#define CKO_SECRET_KEY 0x4 +#define CKO_HW_FEATURE 0x5 +#define CKO_DOMAIN_PARAMETERS 0x6 +#define CKO_MECHANISM 0x7 +#define CKO_OTP_KEY 0x8 + +/* Attribute CKA_KEY_TYPE refers to a CK_KEY_TYPE typed value */ +typedef CK_ULONG CK_KEY_TYPE; +typedef CK_KEY_TYPE *CK_KEY_TYPE_PTR; + +/* + * Values for type CK_KEY_TYPE + * + * This does not cover the full PKCS#11 IDs. + */ +#define CKK_VENDOR_DEFINED (1U << 31) +#define CKK_RSA 0x000 +#define CKK_DSA 0x001 +#define CKK_DH 0x002 +#define CKK_ECDSA 0x003 +#define CKK_EC 0x003 +#define CKK_GENERIC_SECRET 0x010 +#define CKK_DES3 0x015 +#define CKK_AES 0x01f +#define CKK_HOTP 0x023 +#define CKK_MD5_HMAC 0x027 +#define CKK_SHA_1_HMAC 0x028 +#define CKK_SHA256_HMAC 0x02b +#define CKK_SHA384_HMAC 0x02c +#define CKK_SHA512_HMAC 0x02d +#define CKK_SHA224_HMAC 0x02e +#define CKK_EC_EDWARDS 0x040 /* PKCS#11 v3.1-cs01 */ + +/* + * Certificates + */ +typedef CK_ULONG CK_CERTIFICATE_TYPE; +typedef CK_ULONG CK_CERTIFICATE_CATEGORY; + +/* + * Valid values for attribute CKA_CERTIFICATE_TYPE + */ +#define CKC_X_509 0x00000000UL +#define CKC_X_509_ATTR_CERT 0x00000001UL +#define CKC_WTLS 0x00000002UL + +/* + * Valid values for attribute CKA_CERTIFICATE_CATEGORY + */ +#define CK_CERTIFICATE_CATEGORY_UNSPECIFIED 0UL +#define CK_CERTIFICATE_CATEGORY_TOKEN_USER 1UL +#define CK_CERTIFICATE_CATEGORY_AUTHORITY 2UL +#define CK_CERTIFICATE_CATEGORY_OTHER_ENTITY 3UL + +/* + * Mechanisms + * + * Note: a mechanism can be referenced as object reference in some PKCS#11 API + * functions. In such case, the object hold attribute CKA_MECHANISM_TYPE which + * refers to a CK_MECHANISM_TYPE typed value that defines the target mechanism. + */ + +typedef CK_ULONG CK_MECHANISM_TYPE; +typedef CK_MECHANISM_TYPE *CK_MECHANISM_TYPE_PTR; + +/* + * Values for type CK_MECHANISM_TYPE + * + * This does not cover the full PKCS#11 IDs. + */ +#define CKM_VENDOR_DEFINED (1U << 31) +#define CKM_RSA_PKCS_KEY_PAIR_GEN 0x00000 +#define CKM_RSA_PKCS 0x00001 +#define CKM_RSA_9796 0x00002 +#define CKM_RSA_X_509 0x00003 +#define CKM_MD5_RSA_PKCS 0x00005 +#define CKM_SHA1_RSA_PKCS 0x00006 +#define CKM_RSA_PKCS_OAEP 0x00009 +#define CKM_RSA_PKCS_PSS 0x0000d +#define CKM_SHA1_RSA_PKCS_PSS 0x0000e +#define CKM_SHA256_RSA_PKCS 0x00040 +#define CKM_SHA384_RSA_PKCS 0x00041 +#define CKM_SHA512_RSA_PKCS 0x00042 +#define CKM_SHA256_RSA_PKCS_PSS 0x00043 +#define CKM_SHA384_RSA_PKCS_PSS 0x00044 +#define CKM_SHA512_RSA_PKCS_PSS 0x00045 +#define CKM_SHA224_RSA_PKCS 0x00046 +#define CKM_SHA224_RSA_PKCS_PSS 0x00047 +#define CKM_SHA512_224 0x00048 +#define CKM_SHA512_224_HMAC 0x00049 +#define CKM_SHA512_224_HMAC_GENERAL 0x0004a +#define CKM_SHA512_224_KEY_DERIVATION 0x0004b +#define CKM_SHA512_256 0x0004c +#define CKM_SHA512_256_HMAC 0x0004d +#define CKM_SHA512_256_HMAC_GENERAL 0x0004e +#define CKM_SHA512_256_KEY_DERIVATION 0x0004f +#define CKM_DES3_ECB 0x00132 +#define CKM_DES3_CBC 0x00133 +#define CKM_DES3_MAC 0x00134 +#define CKM_DES3_MAC_GENERAL 0x00135 +#define CKM_DES3_CBC_PAD 0x00136 +#define CKM_DES3_CMAC_GENERAL 0x00137 +#define CKM_DES3_CMAC 0x00138 +#define CKM_MD5 0x00210 +#define CKM_MD5_HMAC 0x00211 +#define CKM_MD5_HMAC_GENERAL 0x00212 +#define CKM_SHA_1 0x00220 +#define CKM_SHA_1_HMAC 0x00221 +#define CKM_SHA_1_HMAC_GENERAL 0x00222 +#define CKM_SHA256 0x00250 +#define CKM_SHA256_HMAC 0x00251 +#define CKM_SHA256_HMAC_GENERAL 0x00252 +#define CKM_SHA224 0x00255 +#define CKM_SHA224_HMAC 0x00256 +#define CKM_SHA224_HMAC_GENERAL 0x00257 +#define CKM_SHA384 0x00260 +#define CKM_SHA384_HMAC 0x00261 +#define CKM_SHA384_HMAC_GENERAL 0x00262 +#define CKM_SHA512 0x00270 +#define CKM_SHA512_HMAC 0x00271 +#define CKM_SHA512_HMAC_GENERAL 0x00272 +#define CKM_HOTP_KEY_GEN 0x00290 +#define CKM_HOTP 0x00291 +#define CKM_GENERIC_SECRET_KEY_GEN 0x00350 +#define CKM_MD5_KEY_DERIVATION 0x00390 +#define CKM_MD2_KEY_DERIVATION 0x00391 +#define CKM_SHA1_KEY_DERIVATION 0x00392 +#define CKM_SHA256_KEY_DERIVATION 0x00393 +#define CKM_SHA384_KEY_DERIVATION 0x00394 +#define CKM_SHA512_KEY_DERIVATION 0x00395 +#define CKM_SHA224_KEY_DERIVATION 0x00396 +#define CKM_EC_KEY_PAIR_GEN 0x01040 +#define CKM_ECDSA 0x01041 +#define CKM_ECDSA_SHA1 0x01042 +#define CKM_ECDSA_SHA224 0x01043 +#define CKM_ECDSA_SHA256 0x01044 +#define CKM_ECDSA_SHA384 0x01045 +#define CKM_ECDSA_SHA512 0x01046 +#define CKM_ECDH1_DERIVE 0x01050 +#define CKM_ECDH1_COFACTOR_DERIVE 0x01051 +#define CKM_ECMQV_DERIVE 0x01052 +#define CKM_ECDH_AES_KEY_WRAP 0x01053 +#define CKM_RSA_AES_KEY_WRAP 0x01054 +#define CKM_EC_EDWARDS_KEY_PAIR_GEN 0x01055 +#define CKM_EDDSA 0x01057 +#define CKM_AES_KEY_GEN 0x01080 +#define CKM_AES_ECB 0x01081 +#define CKM_AES_CBC 0x01082 +#define CKM_AES_MAC 0x01083 +#define CKM_AES_MAC_GENERAL 0x01084 +#define CKM_AES_CBC_PAD 0x01085 +#define CKM_AES_CTR 0x01086 +#define CKM_AES_GCM 0x01087 +#define CKM_AES_CCM 0x01088 +#define CKM_AES_CTS 0x01089 +#define CKM_AES_CMAC 0x0108a +#define CKM_AES_CMAC_GENERAL 0x0108b +#define CKM_AES_XCBC_MAC 0x0108c +#define CKM_AES_XCBC_MAC_96 0x0108d +#define CKM_AES_GMAC 0x0108e +#define CKM_DES3_ECB_ENCRYPT_DATA 0x01102 +#define CKM_DES3_CBC_ENCRYPT_DATA 0x01103 +#define CKM_AES_ECB_ENCRYPT_DATA 0x01104 +#define CKM_AES_CBC_ENCRYPT_DATA 0x01105 +#define CKM_AES_KEY_WRAP 0x02109 +#define CKM_AES_KEY_WRAP_PAD 0x0210a + +typedef struct CK_MECHANISM_INFO CK_MECHANISM_INFO; +typedef struct CK_MECHANISM_INFO *CK_MECHANISM_INFO_PTR; + +struct CK_MECHANISM_INFO { + CK_ULONG ulMinKeySize; + CK_ULONG ulMaxKeySize; + CK_FLAGS flags; +}; + +/* Flags for field flags of struct ck_mechanism_info */ +#define CKF_HW (1U << 0) +#define CKF_ENCRYPT (1U << 8) +#define CKF_DECRYPT (1U << 9) +#define CKF_DIGEST (1U << 10) +#define CKF_SIGN (1U << 11) +#define CKF_SIGN_RECOVER (1U << 12) +#define CKF_VERIFY (1U << 13) +#define CKF_VERIFY_RECOVER (1U << 14) +#define CKF_GENERATE (1U << 15) +#define CKF_GENERATE_KEY_PAIR (1U << 16) +#define CKF_WRAP (1U << 17) +#define CKF_UNWRAP (1U << 18) +#define CKF_DERIVE (1U << 19) +#define CKF_EC_F_P (1U << 20) +#define CKF_EC_F_2M (1U << 21) +#define CKF_EC_ECPARAMETERS (1U << 22) +#define CKF_EC_NAMEDCURVE (1U << 23) +#define CKF_EC_UNCOMPRESS (1U << 24) +#define CKF_EC_COMPRESS (1U << 25) +#define CKF_EXTENSION (1U << 31) + +/* + * Mechanism parameter structures + * + * This does not cover the whole mechanism parameter structures defined by + * the PKCS#11. To be updated when needed. + */ + +typedef struct CK_MECHANISM CK_MECHANISM; +typedef struct CK_MECHANISM *CK_MECHANISM_PTR; + +struct CK_MECHANISM { + CK_MECHANISM_TYPE mechanism; + CK_VOID_PTR pParameter; + CK_ULONG ulParameterLen; +}; + +typedef CK_ULONG CK_RSA_PKCS_MGF_TYPE; + +/* Values for type CK_RSA_PKCS_MGF_TYPE */ +#define CKG_MGF1_SHA1 0x0001UL +#define CKG_MGF1_SHA224 0x0005UL +#define CKG_MGF1_SHA256 0x0002UL +#define CKG_MGF1_SHA384 0x0003UL +#define CKG_MGF1_SHA512 0x0004UL + +typedef CK_ULONG CK_RSA_PKCS_OAEP_SOURCE_TYPE; + +/* Values for type CK_RSA_PKCS_OAEP_SOURCE_TYPE */ +#define CKZ_DATA_SPECIFIED 0x0001UL + +/* MAC General parameters */ +typedef CK_ULONG CK_MAC_GENERAL_PARAMS; +typedef CK_MAC_GENERAL_PARAMS *CK_MAC_GENERAL_PARAMS_PTR; + +/* + * CK_EC_KDF_TYPE is used to indicate the Key Derivation Function (KDF) applied + * to derive keying data from a shared secret. + */ +typedef CK_ULONG CK_EC_KDF_TYPE; + +/* + * Elliptic curve Diffie-Hellman key derivation + * Elliptic curve Diffie-Hellman cofactor key derivation parameters + */ +typedef struct CK_ECDH1_DERIVE_PARAMS CK_ECDH1_DERIVE_PARAMS; +typedef struct CK_ECDH1_DERIVE_PARAMS *CK_ECDH1_DERIVE_PARAMS_PTR; + +struct CK_ECDH1_DERIVE_PARAMS { + CK_EC_KDF_TYPE kdf; + CK_ULONG ulSharedDataLen; + CK_BYTE_PTR pSharedData; + CK_ULONG ulPublicDataLen; + CK_BYTE_PTR pPublicData; +}; + +/* AES CBC encryption parameters */ +typedef struct CK_AES_CBC_ENCRYPT_DATA_PARAMS CK_AES_CBC_ENCRYPT_DATA_PARAMS; +typedef struct CK_AES_CBC_ENCRYPT_DATA_PARAMS + *CK_AES_CBC_ENCRYPT_DATA_PARAMS_PTR; + +struct CK_AES_CBC_ENCRYPT_DATA_PARAMS { + CK_BYTE iv[16]; + CK_BYTE_PTR pData; + CK_ULONG length; +}; + +/* AES CTR parameters */ +typedef struct CK_AES_CTR_PARAMS CK_AES_CTR_PARAMS; +typedef struct CK_AES_CTR_PARAMS *CK_AES_CTR_PARAMS_PTR; + +struct CK_AES_CTR_PARAMS { + CK_ULONG ulCounterBits; + CK_BYTE cb[16]; +}; + +/* AES GCM parameters */ +typedef struct CK_GCM_PARAMS CK_GCM_PARAMS; +typedef struct CK_GCM_PARAMS *CK_GCM_PARAMS_PTR; + +struct CK_GCM_PARAMS { + CK_BYTE_PTR pIv; + CK_ULONG ulIvLen; + CK_ULONG ulIvBits; + CK_BYTE_PTR pAAD; + CK_ULONG ulAADLen; + CK_ULONG ulTagBits; +}; + +/* EdDSA (RFC 8032) */ +typedef struct CK_EDDSA_PARAMS { + CK_BYTE phFlag; + CK_ULONG ulContextDataLen; + CK_BYTE_PTR pContextData; +} CK_EDDSA_PARAMS; + +typedef CK_EDDSA_PARAMS *CK_EDDSA_PARAMS_PTR; + +/* AES CCM parameters */ +typedef struct CK_CCM_PARAMS CK_CCM_PARAMS; +typedef struct CK_CCM_PARAMS *CK_CCM_PARAMS_PTR; + +struct CK_CCM_PARAMS { + CK_ULONG ulDataLen; + CK_BYTE_PTR pNonce; + CK_ULONG ulNonceLen; + CK_BYTE_PTR pAAD; + CK_ULONG ulAADLen; + CK_ULONG ulMACLen; +}; + +typedef struct CK_KEY_DERIVATION_STRING_DATA CK_KEY_DERIVATION_STRING_DATA; +typedef struct CK_KEY_DERIVATION_STRING_DATA + *CK_KEY_DERIVATION_STRING_DATA_PTR; + +struct CK_KEY_DERIVATION_STRING_DATA { + CK_BYTE_PTR pData; + CK_ULONG ulLen; +}; + +/* Parameters for CKM_RSA_PKCS_PSS */ +typedef struct CK_RSA_PKCS_PSS_PARAMS CK_RSA_PKCS_PSS_PARAMS; +typedef struct CK_RSA_PKCS_PSS_PARAMS *CK_RSA_PKCS_PSS_PARAMS_PTR; + +struct CK_RSA_PKCS_PSS_PARAMS { + CK_MECHANISM_TYPE hashAlg; + CK_RSA_PKCS_MGF_TYPE mgf; + CK_ULONG sLen; +}; + +/* Parameters for CKM_RSA_PKCS_OAEP */ +typedef struct CK_RSA_PKCS_OAEP_PARAMS CK_RSA_PKCS_OAEP_PARAMS; +typedef struct CK_RSA_PKCS_OAEP_PARAMS *CK_RSA_PKCS_OAEP_PARAMS_PTR; + +struct CK_RSA_PKCS_OAEP_PARAMS { + CK_MECHANISM_TYPE hashAlg; + CK_RSA_PKCS_MGF_TYPE mgf; + CK_RSA_PKCS_OAEP_SOURCE_TYPE source; + CK_VOID_PTR pSourceData; + CK_ULONG ulSourceDataLen; +}; + +typedef struct CK_RSA_AES_KEY_WRAP_PARAMS { + CK_ULONG ulAESKeyBits; + CK_RSA_PKCS_OAEP_PARAMS_PTR pOAEPParams; +} CK_RSA_AES_KEY_WRAP_PARAMS; + +typedef CK_RSA_AES_KEY_WRAP_PARAMS *CK_RSA_AES_KEY_WRAP_PARAMS_PTR; + +/* + * PKCS#11 return values + */ +typedef CK_ULONG CK_RV; + +/* Values for type CK_RV */ +#define CKR_VENDOR_DEFINED (1U << 31) +#define CKR_OK 0x0000 +#define CKR_CANCEL 0x0001 +#define CKR_HOST_MEMORY 0x0002 +#define CKR_SLOT_ID_INVALID 0x0003 +#define CKR_GENERAL_ERROR 0x0005 +#define CKR_FUNCTION_FAILED 0x0006 +#define CKR_ARGUMENTS_BAD 0x0007 +#define CKR_NO_EVENT 0x0008 +#define CKR_NEED_TO_CREATE_THREADS 0x0009 +#define CKR_CANT_LOCK 0x000a +#define CKR_ATTRIBUTE_READ_ONLY 0x0010 +#define CKR_ATTRIBUTE_SENSITIVE 0x0011 +#define CKR_ATTRIBUTE_TYPE_INVALID 0x0012 +#define CKR_ATTRIBUTE_VALUE_INVALID 0x0013 +#define CKR_ACTION_PROHIBITED 0x001b +#define CKR_DATA_INVALID 0x0020 +#define CKR_DATA_LEN_RANGE 0x0021 +#define CKR_DEVICE_ERROR 0x0030 +#define CKR_DEVICE_MEMORY 0x0031 +#define CKR_DEVICE_REMOVED 0x0032 +#define CKR_ENCRYPTED_DATA_INVALID 0x0040 +#define CKR_ENCRYPTED_DATA_LEN_RANGE 0x0041 +#define CKR_FUNCTION_CANCELED 0x0050 +#define CKR_FUNCTION_NOT_PARALLEL 0x0051 +#define CKR_FUNCTION_NOT_SUPPORTED 0x0054 +#define CKR_KEY_HANDLE_INVALID 0x0060 +#define CKR_KEY_SIZE_RANGE 0x0062 +#define CKR_KEY_TYPE_INCONSISTENT 0x0063 +#define CKR_KEY_NOT_NEEDED 0x0064 +#define CKR_KEY_CHANGED 0x0065 +#define CKR_KEY_NEEDED 0x0066 +#define CKR_KEY_INDIGESTIBLE 0x0067 +#define CKR_KEY_FUNCTION_NOT_PERMITTED 0x0068 +#define CKR_KEY_NOT_WRAPPABLE 0x0069 +#define CKR_KEY_UNEXTRACTABLE 0x006a +#define CKR_MECHANISM_INVALID 0x0070 +#define CKR_MECHANISM_PARAM_INVALID 0x0071 +#define CKR_OBJECT_HANDLE_INVALID 0x0082 +#define CKR_OPERATION_ACTIVE 0x0090 +#define CKR_OPERATION_NOT_INITIALIZED 0x0091 +#define CKR_PIN_INCORRECT 0x00a0 +#define CKR_PIN_INVALID 0x00a1 +#define CKR_PIN_LEN_RANGE 0x00a2 +#define CKR_PIN_EXPIRED 0x00a3 +#define CKR_PIN_LOCKED 0x00a4 +#define CKR_SESSION_CLOSED 0x00b0 +#define CKR_SESSION_COUNT 0x00b1 +#define CKR_SESSION_HANDLE_INVALID 0x00b3 +#define CKR_SESSION_PARALLEL_NOT_SUPPORTED 0x00b4 +#define CKR_SESSION_READ_ONLY 0x00b5 +#define CKR_SESSION_EXISTS 0x00b6 +#define CKR_SESSION_READ_ONLY_EXISTS 0x00b7 +#define CKR_SESSION_READ_WRITE_SO_EXISTS 0x00b8 +#define CKR_SIGNATURE_INVALID 0x00c0 +#define CKR_SIGNATURE_LEN_RANGE 0x00c1 +#define CKR_TEMPLATE_INCOMPLETE 0x00d0 +#define CKR_TEMPLATE_INCONSISTENT 0x00d1 +#define CKR_TOKEN_NOT_PRESENT 0x00e0 +#define CKR_TOKEN_NOT_RECOGNIZED 0x00e1 +#define CKR_TOKEN_WRITE_PROTECTED 0x00e2 +#define CKR_UNWRAPPING_KEY_HANDLE_INVALID 0x00f0 +#define CKR_UNWRAPPING_KEY_SIZE_RANGE 0x00f1 +#define CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT 0x00f2 +#define CKR_USER_ALREADY_LOGGED_IN 0x0100 +#define CKR_USER_NOT_LOGGED_IN 0x0101 +#define CKR_USER_PIN_NOT_INITIALIZED 0x0102 +#define CKR_USER_TYPE_INVALID 0x0103 +#define CKR_USER_ANOTHER_ALREADY_LOGGED_IN 0x0104 +#define CKR_USER_TOO_MANY_TYPES 0x0105 +#define CKR_WRAPPED_KEY_INVALID 0x0110 +#define CKR_WRAPPED_KEY_LEN_RANGE 0x0112 +#define CKR_WRAPPING_KEY_HANDLE_INVALID 0x0113 +#define CKR_WRAPPING_KEY_SIZE_RANGE 0x0114 +#define CKR_WRAPPING_KEY_TYPE_INCONSISTENT 0x0115 +#define CKR_RANDOM_SEED_NOT_SUPPORTED 0x0120 +#define CKR_RANDOM_NO_RNG 0x0121 +#define CKR_DOMAIN_PARAMS_INVALID 0x0130 +#define CKR_CURVE_NOT_SUPPORTED 0x0140 +#define CKR_BUFFER_TOO_SMALL 0x0150 +#define CKR_SAVED_STATE_INVALID 0x0160 +#define CKR_INFORMATION_SENSITIVE 0x0170 +#define CKR_STATE_UNSAVEABLE 0x0180 +#define CKR_CRYPTOKI_NOT_INITIALIZED 0x0190 +#define CKR_CRYPTOKI_ALREADY_INITIALIZED 0x0191 +#define CKR_MUTEX_BAD 0x01a0 +#define CKR_MUTEX_NOT_LOCKED 0x01a1 +#define CKR_NEW_PIN_MODE 0x01b0 +#define CKR_NEXT_OTP 0x01b1 +#define CKR_EXCEEDED_MAX_ITERATIONS 0x01b5 +#define CKR_FIPS_SELF_TEST_FAILED 0x01b6 +#define CKR_LIBRARY_LOAD_FAILED 0x01b7 +#define CKR_PIN_TOO_WEAK 0x01b8 +#define CKR_PUBLIC_KEY_INVALID 0x01b9 +#define CKR_FUNCTION_REJECTED 0x0200 + +/* + * PKCS#11 API functions + */ + +/* Argument for C_GetInfo */ +typedef struct CK_INFO CK_INFO; +typedef struct CK_INFO *CK_INFO_PTR; + +struct CK_INFO { + CK_VERSION cryptokiVersion; + CK_UTF8CHAR manufacturerID[32]; + CK_FLAGS flags; + CK_UTF8CHAR libraryDescription[32]; + CK_VERSION libraryVersion; +}; + +/* Argument for C_GetSlotInfo */ +typedef struct CK_SLOT_INFO CK_SLOT_INFO; +typedef struct CK_SLOT_INFO *CK_SLOT_INFO_PTR; + +struct CK_SLOT_INFO { + CK_UTF8CHAR slotDescription[64]; + CK_UTF8CHAR manufacturerID[32]; + CK_FLAGS flags; + CK_VERSION hardwareVersion; + CK_VERSION firmwareVersion; +}; + +/* Values for field flags of struct ck_slot_info */ +#define CKF_TOKEN_PRESENT (1U << 0) +#define CKF_REMOVABLE_DEVICE (1U << 1) +#define CKF_HW_SLOT (1U << 2) + +/* Argument for C_GetTokenInfo */ +typedef struct CK_TOKEN_INFO CK_TOKEN_INFO; +typedef struct CK_TOKEN_INFO *CK_TOKEN_INFO_PTR; + +struct CK_TOKEN_INFO { + CK_UTF8CHAR label[32]; + CK_UTF8CHAR manufacturerID[32]; + CK_UTF8CHAR model[16]; + CK_CHAR serialNumber[16]; + CK_FLAGS flags; + CK_ULONG ulMaxSessionCount; + CK_ULONG ulSessionCount; + CK_ULONG ulMaxRwSessionCount; + CK_ULONG ulRwSessionCount; + CK_ULONG ulMaxPinLen; + CK_ULONG ulMinPinLen; + CK_ULONG ulTotalPublicMemory; + CK_ULONG ulFreePublicMemory; + CK_ULONG ulTotalPrivateMemory; + CK_ULONG ulFreePrivateMemory; + CK_VERSION hardwareVersion; + CK_VERSION firmwareVersion; + CK_CHAR utcTime[16]; +}; + +/* Values for field flags of struct ck_token_info */ +#define CKF_RNG (1U << 0) +#define CKF_WRITE_PROTECTED (1U << 1) +#define CKF_LOGIN_REQUIRED (1U << 2) +#define CKF_USER_PIN_INITIALIZED (1U << 3) +#define CKF_RESTORE_KEY_NOT_NEEDED (1U << 5) +#define CKF_CLOCK_ON_TOKEN (1U << 6) +#define CKF_PROTECTED_AUTHENTICATION_PATH (1U << 8) +#define CKF_DUAL_CRYPTO_OPERATIONS (1U << 9) +#define CKF_TOKEN_INITIALIZED (1U << 10) +#define CKF_SECONDARY_AUTHENTICATION (1U << 11) +#define CKF_USER_PIN_COUNT_LOW (1U << 16) +#define CKF_USER_PIN_FINAL_TRY (1U << 17) +#define CKF_USER_PIN_LOCKED (1U << 18) +#define CKF_USER_PIN_TO_BE_CHANGED (1U << 19) +#define CKF_SO_PIN_COUNT_LOW (1U << 20) +#define CKF_SO_PIN_FINAL_TRY (1U << 21) +#define CKF_SO_PIN_LOCKED (1U << 22) +#define CKF_SO_PIN_TO_BE_CHANGED (1U << 23) +#define CKF_ERROR_STATE (1U << 24) + +/* Argument for C_GetSessionInfo */ +typedef struct CK_SESSION_INFO CK_SESSION_INFO; +typedef struct CK_SESSION_INFO *CK_SESSION_INFO_PTR; + +typedef CK_ULONG CK_STATE; + +/* Values for CK_STATE */ +#define CKS_RO_PUBLIC_SESSION 0 +#define CKS_RO_USER_FUNCTIONS 1 +#define CKS_RW_PUBLIC_SESSION 2 +#define CKS_RW_USER_FUNCTIONS 3 +#define CKS_RW_SO_FUNCTIONS 4 + +struct CK_SESSION_INFO { + CK_SLOT_ID slotID; + CK_STATE state; + CK_FLAGS flags; + CK_ULONG ulDeviceError; +}; + +/* Values for field flags of struct ck_session_info */ +#define CKF_RW_SESSION (1U << 1) +#define CKF_SERIAL_SESSION (1U << 2) + +/* Argument for C_Login */ +typedef CK_ULONG CK_USER_TYPE; + +/* Values for CK_USER_TYPE */ +#define CKU_SO 0 +#define CKU_USER 1 +#define CKU_CONTEXT_SPECIFIC 2 + +/* Values for argument flags of C_WaitForSlotEvent */ +#define CKF_DONT_BLOCK 1 + +/* Argument for CK_NOTIFY typed callback function */ +typedef CK_ULONG CK_NOTIFICATION; + +/* Values for CK_NOTIFICATION */ +#define CKN_SURRENDER 0 +#define CKN_OTP_CHANGED 1 + +/* Callback handler types */ +typedef CK_RV (*CK_NOTIFY) (CK_SESSION_HANDLE hSession, CK_NOTIFICATION event, + CK_VOID_PTR pApplication); +typedef CK_RV (*CK_CREATEMUTEX) (CK_VOID_PTR_PTR ppMutex); +typedef CK_RV (*CK_DESTROYMUTEX) (CK_VOID_PTR pMutex); +typedef CK_RV (*CK_LOCKMUTEX) (CK_VOID_PTR pMutex); +typedef CK_RV (*CK_UNLOCKMUTEX) (CK_VOID_PTR pMutex); + +/* Argument for C_GetFunctionList */ +typedef struct CK_FUNCTION_LIST CK_FUNCTION_LIST; +typedef struct CK_FUNCTION_LIST *CK_FUNCTION_LIST_PTR; +typedef struct CK_FUNCTION_LIST **CK_FUNCTION_LIST_PTR_PTR; + +struct CK_FUNCTION_LIST { + CK_VERSION version; + CK_RV (*C_Initialize)(CK_VOID_PTR pInitArgs); + CK_RV (*C_Finalize)(CK_VOID_PTR pReserved); + CK_RV (*C_GetInfo)(CK_INFO_PTR pInfo); + CK_RV (*C_GetFunctionList)(CK_FUNCTION_LIST_PTR_PTR ppFunctionList); + CK_RV (*C_GetSlotList)(CK_BBOOL tokenPresent, + CK_SLOT_ID_PTR pSlotList, CK_ULONG_PTR pulCount); + CK_RV (*C_GetSlotInfo)(CK_SLOT_ID slotID, CK_SLOT_INFO_PTR pInfo); + CK_RV (*C_GetTokenInfo)(CK_SLOT_ID slotID, CK_TOKEN_INFO_PTR pInfo); + CK_RV (*C_GetMechanismList)(CK_SLOT_ID slotID, + CK_MECHANISM_TYPE_PTR pMechanismList, + CK_ULONG_PTR pulCount); + CK_RV (*C_GetMechanismInfo)(CK_SLOT_ID slotID, CK_MECHANISM_TYPE type, + CK_MECHANISM_INFO_PTR pInfo); + CK_RV (*C_InitToken)(CK_SLOT_ID slotID, CK_UTF8CHAR_PTR pPin, + CK_ULONG ulPinLen, CK_UTF8CHAR_PTR pLabel); + CK_RV (*C_InitPIN)(CK_SESSION_HANDLE hSession, + CK_UTF8CHAR_PTR pPin, CK_ULONG ulPinLen); + CK_RV (*C_SetPIN)(CK_SESSION_HANDLE hSession, + CK_UTF8CHAR_PTR pOldPin, CK_ULONG ulOldLen, + CK_UTF8CHAR_PTR pNewPin, CK_ULONG ulNewLen); + CK_RV (*C_OpenSession)(CK_SLOT_ID slotID, CK_FLAGS flags, + CK_VOID_PTR pApplication, CK_NOTIFY Notify, + CK_SESSION_HANDLE_PTR phSession); + CK_RV (*C_CloseSession)(CK_SESSION_HANDLE hSession); + CK_RV (*C_CloseAllSessions)(CK_SLOT_ID slotID); + CK_RV (*C_GetSessionInfo)(CK_SESSION_HANDLE hSession, + CK_SESSION_INFO_PTR pInfo); + CK_RV (*C_GetOperationState)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pOperationState, + CK_ULONG_PTR pulOperationStateLen); + CK_RV (*C_SetOperationState)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pOperationState, + CK_ULONG ulOperationStateLen, + CK_OBJECT_HANDLE hEncryptionKey, + CK_OBJECT_HANDLE hAuthenticationKey); + CK_RV (*C_Login)(CK_SESSION_HANDLE hSession, CK_USER_TYPE userType, + CK_UTF8CHAR_PTR pPin, CK_ULONG ulPinLen); + CK_RV (*C_Logout)(CK_SESSION_HANDLE hSession); + CK_RV (*C_CreateObject)(CK_SESSION_HANDLE hSession, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phObject); + CK_RV (*C_CopyObject)(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phNewObject); + CK_RV (*C_DestroyObject)(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject); + CK_RV (*C_GetObjectSize)(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ULONG_PTR pulSize); + CK_RV (*C_GetAttributeValue)(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount); + CK_RV (*C_SetAttributeValue)(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount); + CK_RV (*C_FindObjectsInit)(CK_SESSION_HANDLE hSession, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount); + CK_RV (*C_FindObjects)(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE_PTR phObject, + CK_ULONG ulMaxObjectCount, + CK_ULONG_PTR pulObjectCount); + CK_RV (*C_FindObjectsFinal)(CK_SESSION_HANDLE hSession); + CK_RV (*C_EncryptInit)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + CK_RV (*C_Encrypt)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pEncryptedData, + CK_ULONG_PTR pulEncryptedDataLen); + CK_RV (*C_EncryptUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedData, + CK_ULONG_PTR pulEncryptedDataLen); + CK_RV (*C_EncryptFinal)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pLastEncryptedPart, + CK_ULONG_PTR pulLastEncryptedPartLen); + CK_RV (*C_DecryptInit)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + CK_RV (*C_Decrypt)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedData, + CK_ULONG ulEncryptedDataLen, + CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen); + CK_RV (*C_DecryptUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, CK_ULONG_PTR pulPartLen); + CK_RV (*C_DecryptFinal)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pLastPart, + CK_ULONG_PTR pulLastPartLen); + CK_RV (*C_DigestInit)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism); + CK_RV (*C_Digest)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen); + CK_RV (*C_DigestUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen); + CK_RV (*C_DigestKey)(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hKey); + CK_RV (*C_DigestFinal)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen); + CK_RV (*C_SignInit)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + CK_RV (*C_Sign)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, CK_ULONG_PTR pulSignatureLen); + CK_RV (*C_SignUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen); + CK_RV (*C_SignFinal)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, + CK_ULONG_PTR pulSignatureLen); + CK_RV (*C_SignRecoverInit)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + CK_RV (*C_SignRecover)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, + CK_ULONG_PTR pulSignatureLen); + CK_RV (*C_VerifyInit)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + CK_RV (*C_Verify)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, + CK_ULONG ulSignatureLen); + CK_RV (*C_VerifyUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen); + CK_RV (*C_VerifyFinal)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, + CK_ULONG ulSignatureLen); + CK_RV (*C_VerifyRecoverInit)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + CK_RV (*C_VerifyRecover)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, + CK_ULONG ulSignatureLen, + CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen); + CK_RV (*C_DigestEncryptUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG_PTR pulEncryptedPartLen); + CK_RV (*C_DecryptDigestUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, + CK_ULONG_PTR pulPartLen); + CK_RV (*C_SignEncryptUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG_PTR pulEncryptedPartLen); + CK_RV (*C_DecryptVerifyUpdate)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, + CK_ULONG_PTR pulPartLen); + CK_RV (*C_GenerateKey)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey); + CK_RV (*C_GenerateKeyPair)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_ATTRIBUTE_PTR pPublicKeyTemplate, + CK_ULONG ulPublicKeyAttributeCount, + CK_ATTRIBUTE_PTR pPrivateKeyTemplate, + CK_ULONG ulPrivateKeyAttributeCount, + CK_OBJECT_HANDLE_PTR phPublicKey, + CK_OBJECT_HANDLE_PTR phPrivateKey); + CK_RV (*C_WrapKey)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hWrappingKey, + CK_OBJECT_HANDLE hKey, + CK_BYTE_PTR pWrappedKey, + CK_ULONG_PTR pulWrappedKeyLen); + CK_RV (*C_UnwrapKey)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hUnwrappingKey, + CK_BYTE_PTR pWrappedKey, + CK_ULONG ulWrappedKeyLen, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey); + CK_RV (*C_DeriveKey)(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hBaseKey, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey); + CK_RV (*C_SeedRandom)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSeed, CK_ULONG ulSeedLen); + CK_RV (*C_GenerateRandom)(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pRandomData, + CK_ULONG ulRandomLen); + CK_RV (*C_GetFunctionStatus)(CK_SESSION_HANDLE hSession); + CK_RV (*C_CancelFunction)(CK_SESSION_HANDLE hSession); + CK_RV (*C_WaitForSlotEvent)(CK_FLAGS flags, CK_SLOT_ID_PTR slotID, + CK_VOID_PTR pReserved); +}; + +/* Optional init_args structure for C_Initialize */ +typedef struct CK_C_INITIALIZE_ARGS CK_C_INITIALIZE_ARGS; +typedef struct CK_C_INITIALIZE_ARGS *CK_C_INITIALIZE_ARGS_PTR; + +struct CK_C_INITIALIZE_ARGS { + CK_CREATEMUTEX CreateMutex; + CK_DESTROYMUTEX DestroyMutex; + CK_LOCKMUTEX LockMutex; + CK_UNLOCKMUTEX UnlockMutex; + CK_FLAGS flags; + CK_VOID_PTR reserved; +}; + +/* Flags for field flags of struct ck_c_initialize_args */ +#define CKF_LIBRARY_CANT_CREATE_OS_THREADS (1U << 0) +#define CKF_OS_LOCKING_OK (1U << 1) + +CK_RV C_Initialize(CK_VOID_PTR pInitArgs); + +CK_RV C_Finalize(CK_VOID_PTR pReserved); + +CK_RV C_GetInfo(CK_INFO_PTR pInfo); + +CK_RV C_GetFunctionList(CK_FUNCTION_LIST_PTR_PTR ppFunctionList); + +CK_RV C_GetSlotList(CK_BBOOL tokenPresent, + CK_SLOT_ID_PTR pSlotList, CK_ULONG_PTR pulCount); + +CK_RV C_GetSlotInfo(CK_SLOT_ID slotID, CK_SLOT_INFO_PTR pInfo); + +CK_RV C_GetTokenInfo(CK_SLOT_ID slotID, CK_TOKEN_INFO_PTR pInfo); + +CK_RV C_GetMechanismList(CK_SLOT_ID slotID, + CK_MECHANISM_TYPE_PTR pMechanismList, + CK_ULONG_PTR pulCount); + +CK_RV C_GetMechanismInfo(CK_SLOT_ID slotID, CK_MECHANISM_TYPE type, + CK_MECHANISM_INFO_PTR pInfo); + +CK_RV C_InitToken(CK_SLOT_ID slotID, CK_UTF8CHAR_PTR pPin, + CK_ULONG ulPinLen, CK_UTF8CHAR_PTR pLabel); + +CK_RV C_InitPIN(CK_SESSION_HANDLE hSession, + CK_UTF8CHAR_PTR pPin, CK_ULONG ulPinLen); + +CK_RV C_SetPIN(CK_SESSION_HANDLE hSession, + CK_UTF8CHAR_PTR pOldPin, CK_ULONG ulOldLen, + CK_UTF8CHAR_PTR pNewPin, CK_ULONG ulNewLen); + +CK_RV C_OpenSession(CK_SLOT_ID slotID, CK_FLAGS flags, + CK_VOID_PTR pApplication, CK_NOTIFY Notify, + CK_SESSION_HANDLE_PTR phSession); + +CK_RV C_CloseSession(CK_SESSION_HANDLE hSession); + +CK_RV C_CloseAllSessions(CK_SLOT_ID slotID); + +CK_RV C_GetSessionInfo(CK_SESSION_HANDLE hSession, CK_SESSION_INFO_PTR pInfo); + +CK_RV C_GetOperationState(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pOperationState, + CK_ULONG_PTR pulOperationStateLen); + +CK_RV C_SetOperationState(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pOperationState, + CK_ULONG ulOperationStateLen, + CK_OBJECT_HANDLE hEncryptionKey, + CK_OBJECT_HANDLE hAuthenticationKey); + +CK_RV C_Login(CK_SESSION_HANDLE hSession, CK_USER_TYPE userType, + CK_UTF8CHAR_PTR pPin, CK_ULONG ulPinLen); + +CK_RV C_Logout(CK_SESSION_HANDLE hSession); + +CK_RV C_CreateObject(CK_SESSION_HANDLE hSession, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phObject); + +CK_RV C_CopyObject(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phNewObject); + +CK_RV C_DestroyObject(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hObject); + +CK_RV C_GetObjectSize(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hObject, + CK_ULONG_PTR pulSize); + +CK_RV C_GetAttributeValue(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount); + +CK_RV C_SetAttributeValue(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount); + +CK_RV C_FindObjectsInit(CK_SESSION_HANDLE hSession, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount); + +CK_RV C_FindObjects(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE_PTR phObject, + CK_ULONG ulMaxObjectCount, CK_ULONG_PTR pulObjectCount); + +CK_RV C_FindObjectsFinal(CK_SESSION_HANDLE hSession); + +CK_RV C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + +CK_RV C_Encrypt(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pEncryptedData, CK_ULONG_PTR pulEncryptedDataLen); + +CK_RV C_EncryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedData, + CK_ULONG_PTR pulEncryptedDataLen); + +CK_RV C_EncryptFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pLastEncryptedPart, + CK_ULONG_PTR pulLastEncryptedPartLen); + +CK_RV C_DecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + +CK_RV C_Decrypt(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedData, CK_ULONG ulEncryptedDataLen, + CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen); + +CK_RV C_DecryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, CK_ULONG_PTR pulPartLen); + +CK_RV C_DecryptFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pLastPart, CK_ULONG_PTR pulLastPartLen); + +CK_RV C_DigestInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism); + +CK_RV C_Digest(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen); + +CK_RV C_DigestUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen); + +CK_RV C_DigestKey(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hKey); + +CK_RV C_DigestFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen); + +CK_RV C_SignInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + +CK_RV C_Sign(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, CK_ULONG_PTR pulSignatureLen); + +CK_RV C_SignUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen); + +CK_RV C_SignFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, CK_ULONG_PTR pulSignatureLen); + +CK_RV C_SignRecoverInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + +CK_RV C_SignRecover(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, CK_ULONG_PTR pulSignatureLen); + +CK_RV C_VerifyInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + +CK_RV C_Verify(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, CK_ULONG ulSignatureLen); + +CK_RV C_VerifyUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen); + +CK_RV C_VerifyFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, CK_ULONG ulSignatureLen); + +CK_RV C_VerifyRecoverInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey); + +CK_RV C_VerifyRecover(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, CK_ULONG ulSignatureLen, + CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen); + +CK_RV C_DigestEncryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG_PTR pulEncryptedPartLen); + +CK_RV C_DecryptDigestUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, CK_ULONG_PTR pulPartLen); + +CK_RV C_SignEncryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, + CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG_PTR pulEncryptedPartLen); + +CK_RV C_DecryptVerifyUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, CK_ULONG_PTR pulPartLen); + +CK_RV C_GenerateKey(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey); + +CK_RV C_GenerateKeyPair(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_ATTRIBUTE_PTR pPublicKeyTemplate, + CK_ULONG ulPublicKeyAttributeCount, + CK_ATTRIBUTE_PTR pPrivateKeyTemplate, + CK_ULONG ulPrivateKeyAttributeCount, + CK_OBJECT_HANDLE_PTR phPublicKey, + CK_OBJECT_HANDLE_PTR phPrivateKey); + +CK_RV C_WrapKey(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hWrappingKey, CK_OBJECT_HANDLE hKey, + CK_BYTE_PTR pWrappedKey, CK_ULONG_PTR pulWrappedKeyLen); + +CK_RV C_UnwrapKey(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hUnwrappingKey, + CK_BYTE_PTR pWrappedKey, CK_ULONG ulWrappedKeyLen, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey); + +CK_RV C_DeriveKey(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hBaseKey, + CK_ATTRIBUTE_PTR pTemplate, CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey); + +CK_RV C_SeedRandom(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSeed, CK_ULONG ulSeedLen); + +CK_RV C_GenerateRandom(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pRandomData, CK_ULONG ulRandomLen); + +CK_RV C_GetFunctionStatus(CK_SESSION_HANDLE hSession); + +CK_RV C_CancelFunction(CK_SESSION_HANDLE hSession); + +CK_RV C_WaitForSlotEvent(CK_FLAGS flags, CK_SLOT_ID_PTR slotID, + CK_VOID_PTR pReserved); + +#ifdef __cplusplus +} +#endif + +#endif /*PKCS11_H*/ diff --git a/optee_client/libckteec/include/pkcs11_ta.h b/optee_client/libckteec/include/pkcs11_ta.h new file mode 100644 index 0000000..063198b --- /dev/null +++ b/optee_client/libckteec/include/pkcs11_ta.h @@ -0,0 +1,1332 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_H +#define PKCS11_TA_H + +#include +#include + +#define PKCS11_TA_UUID { 0xfd02c9da, 0x306c, 0x48c7, \ + { 0xa4, 0x9c, 0xbb, 0xd8, 0x27, 0xae, 0x86, 0xee } } + +/* PKCS11 trusted application version information */ +#define PKCS11_TA_VERSION_MAJOR 0 +#define PKCS11_TA_VERSION_MINOR 1 +#define PKCS11_TA_VERSION_PATCH 0 + +/* Attribute specific values */ +#define PKCS11_CK_UNAVAILABLE_INFORMATION UINT32_C(0xFFFFFFFF) +#define PKCS11_UNDEFINED_ID UINT32_C(0xFFFFFFFF) +#define PKCS11_FALSE false +#define PKCS11_TRUE true + +/* + * Note on PKCS#11 TA commands ABI + * + * For evolution of the TA API and to not mess with the GPD TEE 4 parameters + * constraint, all the PKCS11 TA invocation commands use a subset of available + * the GPD TEE invocation parameter types. + * + * Param#0 is used for the so-called control arguments of the invoked command + * and for providing a PKCS#11 compliant status code for the request command. + * Param#0 is an in/out memory reference (aka memref[0]). The input buffer + * stores serialized arguments for the command. The output buffer store the + * 32bit TA return code for the command. As a consequence, param#0 shall + * always be an input/output memory reference of at least 32bit, more if + * the command expects more input arguments. + * + * When the TA returns with TEE_SUCCESS result, client shall always get the + * 32bit value stored in param#0 output buffer and use the value as TA + * return code for the invoked command. + * + * Param#1 can be used for input data arguments of the invoked command. + * It is unused or is an input memory reference, aka memref[1]. + * Evolution of the API may use memref[1] for output data as well. + * + * Param#2 is mostly used for output data arguments of the invoked command + * and for output handles generated from invoked commands. + * Few commands uses it for a secondary input data buffer argument. + * It is unused or is an input/output/in-out memory reference, aka memref[2]. + * + * Param#3 is currently unused and reserved for evolution of the API. + */ + +enum pkcs11_ta_cmd { + /* + * PKCS11_CMD_PING Ack TA presence and return version info + * + * [in] memref[0] = 32bit, unused, must be 0 + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = [ + * 32bit version major value, + * 32bit version minor value + * 32bit version patch value + * ] + */ + PKCS11_CMD_PING = 0, + + /* + * PKCS11_CMD_SLOT_LIST - Get the table of the valid slot IDs + * + * [in] memref[0] = 32bit, unused, must be 0 + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit array slot_ids[slot counts] + * + * The TA instance may represent several PKCS#11 slots and + * associated tokens. This commadn reports the IDs of embedded tokens. + * This command relates the PKCS#11 API function C_GetSlotList(). + */ + PKCS11_CMD_SLOT_LIST = 1, + + /* + * PKCS11_CMD_SLOT_INFO - Get cryptoki structured slot information + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_slot_info)info + * + * The TA instance may represent several PKCS#11 slots/tokens. + * This command relates the PKCS#11 API function C_GetSlotInfo(). + */ + PKCS11_CMD_SLOT_INFO = 2, + + /* + * PKCS11_CMD_TOKEN_INFO - Get cryptoki structured token information + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_token_info)info + * + * The TA instance may represent several PKCS#11 slots/tokens. + * This command relates the PKCS#11 API function C_GetTokenInfo(). + */ + PKCS11_CMD_TOKEN_INFO = 3, + + /* + * PKCS11_CMD_MECHANISM_IDS - Get list of the supported mechanisms + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit array mechanism IDs + * + * This command relates to the PKCS#11 API function + * C_GetMechanismList(). + */ + PKCS11_CMD_MECHANISM_IDS = 4, + + /* + * PKCS11_CMD_MECHANISM_INFO - Get information on a specific mechanism + * + * [in] memref[0] = [ + * 32bit slot ID, + * 32bit mechanism ID (PKCS11_CKM_*) + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_mechanism_info)info + * + * This command relates to the PKCS#11 API function + * C_GetMechanismInfo(). + */ + PKCS11_CMD_MECHANISM_INFO = 5, + + /* + * PKCS11_CMD_OPEN_SESSION - Open a session + * + * [in] memref[0] = [ + * 32bit slot ID, + * 32bit session flags, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit session handle + * + * This command relates to the PKCS#11 API function C_OpenSession(). + */ + PKCS11_CMD_OPEN_SESSION = 6, + + /* + * PKCS11_CMD_CLOSE_SESSION - Close an opened session + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_CloseSession(). + */ + PKCS11_CMD_CLOSE_SESSION = 7, + + /* + * PKCS11_CMD_CLOSE_ALL_SESSIONS - Close all client sessions on token + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function + * C_CloseAllSessions(). + */ + PKCS11_CMD_CLOSE_ALL_SESSIONS = 8, + + /* + * PKCS11_CMD_SESSION_INFO - Get Cryptoki information on a session + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_session_info)info + * + * This command relates to the PKCS#11 API function C_GetSessionInfo(). + */ + PKCS11_CMD_SESSION_INFO = 9, + + /* + * PKCS11_CMD_INIT_TOKEN - Initialize PKCS#11 token + * + * [in] memref[0] = [ + * 32bit slot ID, + * 32bit PIN length, + * byte array label[32] + * byte array PIN[PIN length], + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_InitToken(). + */ + PKCS11_CMD_INIT_TOKEN = 10, + + /* + * PKCS11_CMD_INIT_PIN - Initialize user PIN + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit PIN byte size, + * byte array: PIN data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_InitPIN(). + */ + PKCS11_CMD_INIT_PIN = 11, + + /* + * PKCS11_CMD_SET_PIN - Change user PIN + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit old PIN byte size, + * 32bit new PIN byte size, + * byte array: PIN data, + * byte array: new PIN data, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_SetPIN(). + */ + PKCS11_CMD_SET_PIN = 12, + + /* + * PKCS11_CMD_LOGIN - Initialize user PIN + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit user identifier, enum pkcs11_user_type + * 32bit PIN byte size, + * byte array: PIN data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_Login(). + */ + PKCS11_CMD_LOGIN = 13, + + /* + * PKCS11_CMD_LOGOUT - Log out from token + * + * [in] memref[0] = [ + * 32bit session handle, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_Logout(). + */ + PKCS11_CMD_LOGOUT = 14, + + /* + * PKCS11_CMD_CREATE_OBJECT - Create a raw client assembled object in + * the session or token + * + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_CreateObject(). + */ + PKCS11_CMD_CREATE_OBJECT = 15, + + /* + * PKCS11_CMD_DESTROY_OBJECT - Destroy an object + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_DestroyObject(). + */ + PKCS11_CMD_DESTROY_OBJECT = 16, + + /* + * PKCS11_CMD_ENCRYPT_INIT - Initialize encryption processing + * PKCS11_CMD_DECRYPT_INIT - Initialize decryption processing + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle of the key, + * (struct pkcs11_attribute_head)mechanism + mecha params + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * These commands relate to the PKCS#11 API functions + * C_EncryptInit() and C_DecryptInit(). + */ + PKCS11_CMD_ENCRYPT_INIT = 17, + PKCS11_CMD_DECRYPT_INIT = 18, + + /* + * PKCS11_CMD_ENCRYPT_UPDATE - Update encryption processing + * PKCS11_CMD_DECRYPT_UPDATE - Update decryption processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = output processed data + * + * These commands relate to the PKCS#11 API functions + * C_EncryptUpdate() and C_DecryptUpdate(). + */ + PKCS11_CMD_ENCRYPT_UPDATE = 19, + PKCS11_CMD_DECRYPT_UPDATE = 20, + + /* + * PKCS11_CMD_ENCRYPT_FINAL - Finalize encryption processing + * PKCS11_CMD_DECRYPT_FINAL - Finalize decryption processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = output processed data + * + * These commands relate to the PKCS#11 API functions + * C_EncryptFinal() and C_DecryptFinal(). + */ + PKCS11_CMD_ENCRYPT_FINAL = 21, + PKCS11_CMD_DECRYPT_FINAL = 22, + + /* + * PKCS11_CMD_ENCRYPT_ONESHOT - Update and finalize encryption + * processing + * PKCS11_CMD_DECRYPT_ONESHOT - Update and finalize decryption + * processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = output processed data + * + * These commands relate to the PKCS#11 API functions C_Encrypt and + * C_Decrypt. + */ + PKCS11_CMD_ENCRYPT_ONESHOT = 23, + PKCS11_CMD_DECRYPT_ONESHOT = 24, + + /* + * PKCS11_CMD_SIGN_INIT - Initialize a signature computation + * processing + * PKCS11_CMD_VERIFY_INIT - Initialize a signature verification + * processing + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit key handle, + * (struct pkcs11_attribute_head)mechanism + + * mechanism params, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * These commands relate to the PKCS#11 API functions C_SignInit() and + * C_VerifyInit(). + */ + PKCS11_CMD_SIGN_INIT = 25, + PKCS11_CMD_VERIFY_INIT = 26, + + /* + * PKCS11_CMD_SIGN_UPDATE - Update a signature computation processing + * PKCS11_CMD_VERIFY_UPDATE - Update a signature verification processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * + * These commands relate to the PKCS#11 API functions C_SignUpdate() and + * C_VerifyUpdate(). + */ + PKCS11_CMD_SIGN_UPDATE = 27, + PKCS11_CMD_VERIFY_UPDATE = 28, + + /* + * PKCS11_CMD_SIGN_FINAL - Finalize a signature computation processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = output signature + * + * This command relates to the PKCS#11 API functions C_SignFinal(). + */ + PKCS11_CMD_SIGN_FINAL = 29, + + /* + * PKCS11_CMD_VERIFY_FINAL - Finalize a signature verification + * processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[2] = input signature to be processed + * + * This command relates to the PKCS#11 API functions C_VerifyFinal(). + */ + PKCS11_CMD_VERIFY_FINAL = 30, + + /* + * PKCS11_CMD_SIGN_ONESHOT - Compute a signature + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = byte array: generated signature + * + * This command relates to the PKCS#11 API function C_Sign(). + */ + PKCS11_CMD_SIGN_ONESHOT = 31, + + /* + * PKCS11_CMD_VERIFY_ONESHOT - Compute and compare a signature + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [in] memref[2] = input signature to be processed + * + * This command relates to the PKCS#11 API function C_Verify(). + */ + PKCS11_CMD_VERIFY_ONESHOT = 32, + + /* + * PKCS11_CMD_GENERATE_KEY - Generate a symmetric key or domain + * parameters + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_GenerateKey(). + */ + PKCS11_CMD_GENERATE_KEY = 33, + + /* + * PKCS11_CMD_FIND_OBJECTS_INIT - Initialize an object search + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_FindOjectsInit(). + */ + PKCS11_CMD_FIND_OBJECTS_INIT = 34, + + /* + * PKCS11_CMD_FIND_OBJECTS - Get handles of matching objects + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit array object_handle_array[N] + * + * This command relates to the PKCS#11 API function C_FindOjects(). + * The size of object_handle_array depends on the size of the output + * buffer provided by the client. + */ + PKCS11_CMD_FIND_OBJECTS = 35, + + /* + * PKCS11_CMD_FIND_OBJECTS_FINAL - Finalize current objects search + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_FindOjectsFinal(). + */ + PKCS11_CMD_FIND_OBJECTS_FINAL = 36, + + /* + * PKCS11_CMD_GET_OBJECT_SIZE - Get byte size used by object in the TEE + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object_byte_size + * + * This command relates to the PKCS#11 API function C_GetObjectSize(). + */ + PKCS11_CMD_GET_OBJECT_SIZE = 37, + + /* + * PKCS11_CMD_GET_ATTRIBUTE_VALUE - Get the value of object attribute(s) + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_object_head)attribs + attributes + * data + * + * This command relates to the PKCS#11 API function C_GetAttributeValue. + * Caller provides an attribute template as 3rd argument in memref[0] + * (referred here as attribs + attributes data). Upon successful + * completion, the TA returns the provided template filled with expected + * data through output argument memref[2] (referred here again as + * attribs + attributes data). + */ + PKCS11_CMD_GET_ATTRIBUTE_VALUE = 38, + + /* + * PKCS11_CMD_SET_ATTRIBUTE_VALUE - Set the value of object attribute(s) + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_SetAttributeValue. + * Caller provides an attribute template as 3rd argument in memref[0] + * (referred here as attribs + attributes data). + */ + PKCS11_CMD_SET_ATTRIBUTE_VALUE = 39, + + /* + * PKCS11_CMD_COPY_OBJECT - Copies an object, creating a new object for + * the copy. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_CopyObject(). + * Caller provides an attribute template as 3rd argument in memref[0] + * (referred here as attribs + attributes data). + */ + PKCS11_CMD_COPY_OBJECT = 40, + + /* + * PKCS11_CMD_SEED_RANDOM - Seed random data generator + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = byte array: seed material to feed into the RNG + * + * This command relates to the PKCS#11 API function C_SeedRandom(). + */ + PKCS11_CMD_SEED_RANDOM = 41, + + /* + * PKCS11_CMD_GENERATE_RANDOM - Generate random data + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = byte array: generated random + * + * This command relates to the PKCS#11 API function C_GenerateRandom(). + */ + PKCS11_CMD_GENERATE_RANDOM = 42, + + /* + * PKCS11_CMD_DERIVE_KEY - Derive a key from a parent key. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit parent key handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_DeriveKey(). + */ + PKCS11_CMD_DERIVE_KEY = 43, + + /* + * PKCS11_CMD_RELEASE_ACTIVE_PROCESSING - Release active processing + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit enum pkcs11_ta_cmd + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command is used to release active processing in case of + * Cryptoki API invocation error is detected in user space processing. + * Function derived from pkcs11_ta_cmd is used to verify that active + * processing matches. + */ + PKCS11_CMD_RELEASE_ACTIVE_PROCESSING = 44, + + /* + * PKCS11_CMD_DIGEST_INIT - Initialize a digest computation processing + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_attribute_head)mechanism + mecha params + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_DigestInit(). + */ + PKCS11_CMD_DIGEST_INIT = 45, + + /* + * PKCS11_CMD_DIGEST_KEY - Update digest with a key + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit key handle + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_DigestKey(). + */ + PKCS11_CMD_DIGEST_KEY = 46, + + /* + * PKCS11_CMD_DIGEST_UPDATE - Update digest with data + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * + * This command relates to the PKCS#11 API function C_DigestUpdate(). + */ + PKCS11_CMD_DIGEST_UPDATE = 47, + + /* + * PKCS11_CMD_DIGEST_FINAL - Finalize a digest computation processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = output digest + * + * This command relates to the PKCS#11 API function C_DigestFinal(). + */ + PKCS11_CMD_DIGEST_FINAL = 48, + + /* + * PKCS11_CMD_DIGEST_ONESHOT - Compute a digest + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = byte array: generated digest + * + * This command relates to the PKCS#11 API function C_Digest(). + */ + PKCS11_CMD_DIGEST_ONESHOT = 49, + + /* + * PKCS11_CMD_GENERATE_KEY_PAIR - Generate an asymmetric key pair + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)public key attribs + + * attributes data, + * (struct pkcs11_object_head)private key attribs + + * attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = [ + * 32bit public key object handle, + * 32bit private key object handle + * ] + * + * This command relates to the PKCS#11 API function + * C_GenerateKeyPair(). + */ + PKCS11_CMD_GENERATE_KEY_PAIR = 50, + + /* + * PKCS11_CMD_WRAP_KEY - Wraps a private or secret key. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit wrapping key handle, + * 32bit key handle, + * (struct pkcs11_attribute_head)mechanism + mecha params + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = wrapped key + * + * This command relates to the PKCS#11 API function C_WrapKey(). + */ + PKCS11_CMD_WRAP_KEY = 51, + + /* + * PKCS11_CMD_UNWRAP_KEY - Unwraps a wrapped key, creating a new + * private or secret key object. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit unwrapping key handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = wrapped key + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_UnwrapKey(). + */ + PKCS11_CMD_UNWRAP_KEY = 52, +}; + +/* + * Command return codes + * PKCS11_ relates CryptoKi client API CKR_ + */ +enum pkcs11_rc { + PKCS11_CKR_OK = 0, + PKCS11_CKR_CANCEL = 0x0001, + PKCS11_CKR_SLOT_ID_INVALID = 0x0003, + PKCS11_CKR_GENERAL_ERROR = 0x0005, + PKCS11_CKR_FUNCTION_FAILED = 0x0006, + PKCS11_CKR_ARGUMENTS_BAD = 0x0007, + PKCS11_CKR_ATTRIBUTE_READ_ONLY = 0x0010, + PKCS11_CKR_ATTRIBUTE_SENSITIVE = 0x0011, + PKCS11_CKR_ATTRIBUTE_TYPE_INVALID = 0x0012, + PKCS11_CKR_ATTRIBUTE_VALUE_INVALID = 0x0013, + PKCS11_CKR_ACTION_PROHIBITED = 0x001b, + PKCS11_CKR_DATA_INVALID = 0x0020, + PKCS11_CKR_DATA_LEN_RANGE = 0x0021, + PKCS11_CKR_DEVICE_ERROR = 0x0030, + PKCS11_CKR_DEVICE_MEMORY = 0x0031, + PKCS11_CKR_DEVICE_REMOVED = 0x0032, + PKCS11_CKR_ENCRYPTED_DATA_INVALID = 0x0040, + PKCS11_CKR_ENCRYPTED_DATA_LEN_RANGE = 0x0041, + PKCS11_CKR_KEY_HANDLE_INVALID = 0x0060, + PKCS11_CKR_KEY_SIZE_RANGE = 0x0062, + PKCS11_CKR_KEY_TYPE_INCONSISTENT = 0x0063, + PKCS11_CKR_KEY_INDIGESTIBLE = 0x0067, + PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED = 0x0068, + PKCS11_CKR_KEY_NOT_WRAPPABLE = 0x0069, + PKCS11_CKR_KEY_UNEXTRACTABLE = 0x006a, + PKCS11_CKR_MECHANISM_INVALID = 0x0070, + PKCS11_CKR_MECHANISM_PARAM_INVALID = 0x0071, + PKCS11_CKR_OBJECT_HANDLE_INVALID = 0x0082, + PKCS11_CKR_OPERATION_ACTIVE = 0x0090, + PKCS11_CKR_OPERATION_NOT_INITIALIZED = 0x0091, + PKCS11_CKR_PIN_INCORRECT = 0x00a0, + PKCS11_CKR_PIN_INVALID = 0x00a1, + PKCS11_CKR_PIN_LEN_RANGE = 0x00a2, + PKCS11_CKR_PIN_EXPIRED = 0x00a3, + PKCS11_CKR_PIN_LOCKED = 0x00a4, + PKCS11_CKR_SESSION_CLOSED = 0x00b0, + PKCS11_CKR_SESSION_COUNT = 0x00b1, + PKCS11_CKR_SESSION_HANDLE_INVALID = 0x00b3, + PKCS11_CKR_SESSION_PARALLEL_NOT_SUPPORTED = 0x00b4, + PKCS11_CKR_SESSION_READ_ONLY = 0x00b5, + PKCS11_CKR_SESSION_EXISTS = 0x00b6, + PKCS11_CKR_SESSION_READ_ONLY_EXISTS = 0x00b7, + PKCS11_CKR_SESSION_READ_WRITE_SO_EXISTS = 0x00b8, + PKCS11_CKR_SIGNATURE_INVALID = 0x00c0, + PKCS11_CKR_SIGNATURE_LEN_RANGE = 0x00c1, + PKCS11_CKR_TEMPLATE_INCOMPLETE = 0x00d0, + PKCS11_CKR_TEMPLATE_INCONSISTENT = 0x00d1, + PKCS11_CKR_TOKEN_NOT_PRESENT = 0x00e0, + PKCS11_CKR_TOKEN_NOT_RECOGNIZED = 0x00e1, + PKCS11_CKR_TOKEN_WRITE_PROTECTED = 0x00e2, + PKCS11_CKR_UNWRAPPING_KEY_HANDLE_INVALID = 0x00f0, + PKCS11_CKR_UNWRAPPING_KEY_SIZE_RANGE = 0x00f1, + PKCS11_CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT = 0x00f2, + PKCS11_CKR_USER_ALREADY_LOGGED_IN = 0x0100, + PKCS11_CKR_USER_NOT_LOGGED_IN = 0x0101, + PKCS11_CKR_USER_PIN_NOT_INITIALIZED = 0x0102, + PKCS11_CKR_USER_TYPE_INVALID = 0x0103, + PKCS11_CKR_USER_ANOTHER_ALREADY_LOGGED_IN = 0x0104, + PKCS11_CKR_USER_TOO_MANY_TYPES = 0x0105, + PKCS11_CKR_WRAPPED_KEY_INVALID = 0x0110, + PKCS11_CKR_WRAPPED_KEY_LEN_RANGE = 0x0112, + PKCS11_CKR_WRAPPING_KEY_HANDLE_INVALID = 0x0113, + PKCS11_CKR_WRAPPING_KEY_SIZE_RANGE = 0x0114, + PKCS11_CKR_WRAPPING_KEY_TYPE_INCONSISTENT = 0x0115, + PKCS11_CKR_RANDOM_SEED_NOT_SUPPORTED = 0x0120, + PKCS11_CKR_RANDOM_NO_RNG = 0x0121, + PKCS11_CKR_DOMAIN_PARAMS_INVALID = 0x0130, + PKCS11_CKR_CURVE_NOT_SUPPORTED = 0x0140, + PKCS11_CKR_BUFFER_TOO_SMALL = 0x0150, + PKCS11_CKR_SAVED_STATE_INVALID = 0x0160, + PKCS11_CKR_INFORMATION_SENSITIVE = 0x0170, + PKCS11_CKR_STATE_UNSAVEABLE = 0x0180, + PKCS11_CKR_PIN_TOO_WEAK = 0x01b8, + PKCS11_CKR_PUBLIC_KEY_INVALID = 0x01b9, + PKCS11_CKR_FUNCTION_REJECTED = 0x0200, + /* Vendor specific IDs not returned to client */ + PKCS11_RV_NOT_FOUND = 0x80000000, + PKCS11_RV_NOT_IMPLEMENTED = 0x80000001, +}; + +/* + * Arguments for PKCS11_CMD_SLOT_INFO + */ +#define PKCS11_SLOT_DESC_SIZE 64 +#define PKCS11_SLOT_MANUFACTURER_SIZE 32 +#define PKCS11_SLOT_VERSION_SIZE 2 + +struct pkcs11_slot_info { + uint8_t slot_description[PKCS11_SLOT_DESC_SIZE]; + uint8_t manufacturer_id[PKCS11_SLOT_MANUFACTURER_SIZE]; + uint32_t flags; + uint8_t hardware_version[PKCS11_SLOT_VERSION_SIZE]; + uint8_t firmware_version[PKCS11_SLOT_VERSION_SIZE]; +}; + +/* + * Values for pkcs11_slot_info::flags. + * PKCS11_CKFS_ reflects CryptoKi client API slot flags CKF_. + */ +#define PKCS11_CKFS_TOKEN_PRESENT (1U << 0) +#define PKCS11_CKFS_REMOVABLE_DEVICE (1U << 1) +#define PKCS11_CKFS_HW_SLOT (1U << 2) + +/* + * Arguments for PKCS11_CMD_TOKEN_INFO + */ +#define PKCS11_TOKEN_LABEL_SIZE 32 +#define PKCS11_TOKEN_MANUFACTURER_SIZE 32 +#define PKCS11_TOKEN_MODEL_SIZE 16 +#define PKCS11_TOKEN_SERIALNUM_SIZE 16 + +struct pkcs11_token_info { + uint8_t label[PKCS11_TOKEN_LABEL_SIZE]; + uint8_t manufacturer_id[PKCS11_TOKEN_MANUFACTURER_SIZE]; + uint8_t model[PKCS11_TOKEN_MODEL_SIZE]; + uint8_t serial_number[PKCS11_TOKEN_SERIALNUM_SIZE]; + uint32_t flags; + uint32_t max_session_count; + uint32_t session_count; + uint32_t max_rw_session_count; + uint32_t rw_session_count; + uint32_t max_pin_len; + uint32_t min_pin_len; + uint32_t total_public_memory; + uint32_t free_public_memory; + uint32_t total_private_memory; + uint32_t free_private_memory; + uint8_t hardware_version[2]; + uint8_t firmware_version[2]; + uint8_t utc_time[16]; +}; + +/* + * Values for pkcs11_token_info::flags. + * PKCS11_CKFT_ reflects CryptoKi client API token flags CKF_. + */ +#define PKCS11_CKFT_RNG (1U << 0) +#define PKCS11_CKFT_WRITE_PROTECTED (1U << 1) +#define PKCS11_CKFT_LOGIN_REQUIRED (1U << 2) +#define PKCS11_CKFT_USER_PIN_INITIALIZED (1U << 3) +#define PKCS11_CKFT_RESTORE_KEY_NOT_NEEDED (1U << 5) +#define PKCS11_CKFT_CLOCK_ON_TOKEN (1U << 6) +#define PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH (1U << 8) +#define PKCS11_CKFT_DUAL_CRYPTO_OPERATIONS (1U << 9) +#define PKCS11_CKFT_TOKEN_INITIALIZED (1U << 10) +#define PKCS11_CKFT_SECONDARY_AUTHENTICATION (1U << 11) +#define PKCS11_CKFT_USER_PIN_COUNT_LOW (1U << 16) +#define PKCS11_CKFT_USER_PIN_FINAL_TRY (1U << 17) +#define PKCS11_CKFT_USER_PIN_LOCKED (1U << 18) +#define PKCS11_CKFT_USER_PIN_TO_BE_CHANGED (1U << 19) +#define PKCS11_CKFT_SO_PIN_COUNT_LOW (1U << 20) +#define PKCS11_CKFT_SO_PIN_FINAL_TRY (1U << 21) +#define PKCS11_CKFT_SO_PIN_LOCKED (1U << 22) +#define PKCS11_CKFT_SO_PIN_TO_BE_CHANGED (1U << 23) +#define PKCS11_CKFT_ERROR_STATE (1U << 24) + +/* Values for user identity */ +enum pkcs11_user_type { + PKCS11_CKU_SO = 0x000, + PKCS11_CKU_USER = 0x001, + PKCS11_CKU_CONTEXT_SPECIFIC = 0x002, +}; + +/* + * TEE Identity based authentication for tokens + * + * When configuration CFG_PKCS11_TA_AUTH_TEE_IDENTITY is enabled TEE Identity + * based authentication scheme is enabled. + * + * Feature enablement per token basis is controlled by token flag: + * pkcs11_token_info->flags & PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH + * + * When calling C_InitToken() mode is determined based on SO PIN value. + * - If the PIN is empty (or NULL_PTR) then active client TEE Identity will be + * used as SO TEE Identity + * - If the PIN is given then normal PIN behavior is used + * + * Once TEE Identity based authentication is activated following operational + * changes happen: + * - PIN failure counters are disabled to prevent token authentication lockups + * - Switching to different authentication mode needs C_InitToken() + * - When C_Login() or so is performed actual PIN value is ignored and active + * client TEE Identity will be used + * + * Different types of TEE Identity authentication methods can be configured: + * - Configured with C_InitToken(), C_InitPIN() or by C_SetPIN() + * - PIN value follows below PIN syntax + * + * TEE Identity based authenticate PIN syntax: + * - PIN value: NULL_PTR or empty + * - Use active client TEE Identity + * - PIN value: public + * - TEE public login + * - PIN value: user: + * - TEE user login with client UUID matching user credentials + * - PIN value: group: + * - TEE group login with client UUID matching group credentials + */ + +/* Keywords for protected authenticated path PIN parser */ +#define PKCS11_AUTH_TEE_IDENTITY_PUBLIC "public" +#define PKCS11_AUTH_TEE_IDENTITY_USER "user:" +#define PKCS11_AUTH_TEE_IDENTITY_GROUP "group:" + +/* + * Values for 32bit session flags argument to PKCS11_CMD_OPEN_SESSION + * and pkcs11_session_info::flags. + * PKCS11_CKFSS_ reflects CryptoKi client API session flags CKF_. + */ +#define PKCS11_CKFSS_RW_SESSION (1U << 1) +#define PKCS11_CKFSS_SERIAL_SESSION (1U << 2) + +/* + * Arguments for PKCS11_CMD_SESSION_INFO + */ + +struct pkcs11_session_info { + uint32_t slot_id; + uint32_t state; + uint32_t flags; + uint32_t device_error; +}; + +/* Valid values for pkcs11_session_info::state */ +enum pkcs11_session_state { + PKCS11_CKS_RO_PUBLIC_SESSION = 0, + PKCS11_CKS_RO_USER_FUNCTIONS = 1, + PKCS11_CKS_RW_PUBLIC_SESSION = 2, + PKCS11_CKS_RW_USER_FUNCTIONS = 3, + PKCS11_CKS_RW_SO_FUNCTIONS = 4, +}; + +/* + * Arguments for PKCS11_CMD_MECHANISM_INFO + */ + +struct pkcs11_mechanism_info { + uint32_t min_key_size; + uint32_t max_key_size; + uint32_t flags; +}; + +/* + * Values for pkcs11_mechanism_info::flags. + * PKCS11_CKFM_ reflects CryptoKi client API mechanism flags CKF_. + */ +#define PKCS11_CKFM_HW (1U << 0) +#define PKCS11_CKFM_ENCRYPT (1U << 8) +#define PKCS11_CKFM_DECRYPT (1U << 9) +#define PKCS11_CKFM_DIGEST (1U << 10) +#define PKCS11_CKFM_SIGN (1U << 11) +#define PKCS11_CKFM_SIGN_RECOVER (1U << 12) +#define PKCS11_CKFM_VERIFY (1U << 13) +#define PKCS11_CKFM_VERIFY_RECOVER (1U << 14) +#define PKCS11_CKFM_GENERATE (1U << 15) +#define PKCS11_CKFM_GENERATE_KEY_PAIR (1U << 16) +#define PKCS11_CKFM_WRAP (1U << 17) +#define PKCS11_CKFM_UNWRAP (1U << 18) +#define PKCS11_CKFM_DERIVE (1U << 19) +#define PKCS11_CKFM_EC_F_P (1U << 20) +#define PKCS11_CKFM_EC_F_2M (1U << 21) +#define PKCS11_CKFM_EC_ECPARAMETERS (1U << 22) +#define PKCS11_CKFM_EC_NAMEDCURVE (1U << 23) +#define PKCS11_CKFM_EC_UNCOMPRESS (1U << 24) +#define PKCS11_CKFM_EC_COMPRESS (1U << 25) + +/* + * pkcs11_object_head - Header of object whose data are serialized in memory + * + * An object is made of several attributes. Attributes are stored one next to + * the other with byte alignment as a serialized byte array. The byte array + * of serialized attributes is prepended with the size of the attrs[] array + * in bytes and the number of attributes in the array, yielding the struct + * pkcs11_object_head. + * + * @attrs_size - byte size of whole byte array attrs[] + * @attrs_count - number of attribute items stored in attrs[] + * @attrs - then starts the attributes data + */ +struct pkcs11_object_head { + uint32_t attrs_size; + uint32_t attrs_count; + uint8_t attrs[]; +}; + +/* + * Attribute reference in the TA ABI. Each attribute starts with a header + * structure followed by the attribute value. The attribute byte size is + * defined in the attribute header. + * + * @id - the 32bit identifier of the attribute, see PKCS11_CKA_ + * @size - the 32bit value attribute byte size + * @data - then starts the attribute value + */ +struct pkcs11_attribute_head { + uint32_t id; + uint32_t size; + uint8_t data[]; +}; + +/* + * Attribute identification IDs as of v2.40 excluding deprecated IDs. + * Valid values for struct pkcs11_attribute_head::id + * PKCS11_CKA_ reflects CryptoKi client API attribute IDs CKA_. + */ +enum pkcs11_attr_id { + PKCS11_CKA_CLASS = 0x0000, + PKCS11_CKA_TOKEN = 0x0001, + PKCS11_CKA_PRIVATE = 0x0002, + PKCS11_CKA_LABEL = 0x0003, + PKCS11_CKA_APPLICATION = 0x0010, + PKCS11_CKA_VALUE = 0x0011, + PKCS11_CKA_OBJECT_ID = 0x0012, + PKCS11_CKA_CERTIFICATE_TYPE = 0x0080, + PKCS11_CKA_ISSUER = 0x0081, + PKCS11_CKA_SERIAL_NUMBER = 0x0082, + PKCS11_CKA_AC_ISSUER = 0x0083, + PKCS11_CKA_OWNER = 0x0084, + PKCS11_CKA_ATTR_TYPES = 0x0085, + PKCS11_CKA_TRUSTED = 0x0086, + PKCS11_CKA_CERTIFICATE_CATEGORY = 0x0087, + PKCS11_CKA_JAVA_MIDP_SECURITY_DOMAIN = 0x0088, + PKCS11_CKA_URL = 0x0089, + PKCS11_CKA_HASH_OF_SUBJECT_PUBLIC_KEY = 0x008a, + PKCS11_CKA_HASH_OF_ISSUER_PUBLIC_KEY = 0x008b, + PKCS11_CKA_NAME_HASH_ALGORITHM = 0x008c, + PKCS11_CKA_CHECK_VALUE = 0x0090, + PKCS11_CKA_KEY_TYPE = 0x0100, + PKCS11_CKA_SUBJECT = 0x0101, + PKCS11_CKA_ID = 0x0102, + PKCS11_CKA_SENSITIVE = 0x0103, + PKCS11_CKA_ENCRYPT = 0x0104, + PKCS11_CKA_DECRYPT = 0x0105, + PKCS11_CKA_WRAP = 0x0106, + PKCS11_CKA_UNWRAP = 0x0107, + PKCS11_CKA_SIGN = 0x0108, + PKCS11_CKA_SIGN_RECOVER = 0x0109, + PKCS11_CKA_VERIFY = 0x010a, + PKCS11_CKA_VERIFY_RECOVER = 0x010b, + PKCS11_CKA_DERIVE = 0x010c, + PKCS11_CKA_START_DATE = 0x0110, + PKCS11_CKA_END_DATE = 0x0111, + PKCS11_CKA_MODULUS = 0x0120, + PKCS11_CKA_MODULUS_BITS = 0x0121, + PKCS11_CKA_PUBLIC_EXPONENT = 0x0122, + PKCS11_CKA_PRIVATE_EXPONENT = 0x0123, + PKCS11_CKA_PRIME_1 = 0x0124, + PKCS11_CKA_PRIME_2 = 0x0125, + PKCS11_CKA_EXPONENT_1 = 0x0126, + PKCS11_CKA_EXPONENT_2 = 0x0127, + PKCS11_CKA_COEFFICIENT = 0x0128, + PKCS11_CKA_PUBLIC_KEY_INFO = 0x0129, + PKCS11_CKA_PRIME = 0x0130, + PKCS11_CKA_SUBPRIME = 0x0131, + PKCS11_CKA_BASE = 0x0132, + PKCS11_CKA_PRIME_BITS = 0x0133, + PKCS11_CKA_SUBPRIME_BITS = 0x0134, + PKCS11_CKA_VALUE_BITS = 0x0160, + PKCS11_CKA_VALUE_LEN = 0x0161, + PKCS11_CKA_EXTRACTABLE = 0x0162, + PKCS11_CKA_LOCAL = 0x0163, + PKCS11_CKA_NEVER_EXTRACTABLE = 0x0164, + PKCS11_CKA_ALWAYS_SENSITIVE = 0x0165, + PKCS11_CKA_KEY_GEN_MECHANISM = 0x0166, + PKCS11_CKA_MODIFIABLE = 0x0170, + PKCS11_CKA_COPYABLE = 0x0171, + PKCS11_CKA_DESTROYABLE = 0x0172, + PKCS11_CKA_EC_PARAMS = 0x0180, + PKCS11_CKA_EC_POINT = 0x0181, + PKCS11_CKA_ALWAYS_AUTHENTICATE = 0x0202, + PKCS11_CKA_WRAP_WITH_TRUSTED = 0x0210, + /* + * The leading 4 comes from the PKCS#11 spec or:ing with + * CKF_ARRAY_ATTRIBUTE = 0x40000000. + */ + PKCS11_CKA_WRAP_TEMPLATE = 0x40000211, + PKCS11_CKA_UNWRAP_TEMPLATE = 0x40000212, + PKCS11_CKA_DERIVE_TEMPLATE = 0x40000213, + PKCS11_CKA_OTP_FORMAT = 0x0220, + PKCS11_CKA_OTP_LENGTH = 0x0221, + PKCS11_CKA_OTP_TIME_INTERVAL = 0x0222, + PKCS11_CKA_OTP_USER_FRIENDLY_MODE = 0x0223, + PKCS11_CKA_OTP_CHALLENGE_REQUIREMENT = 0x0224, + PKCS11_CKA_OTP_TIME_REQUIREMENT = 0x0225, + PKCS11_CKA_OTP_COUNTER_REQUIREMENT = 0x0226, + PKCS11_CKA_OTP_PIN_REQUIREMENT = 0x0227, + PKCS11_CKA_OTP_COUNTER = 0x022e, + PKCS11_CKA_OTP_TIME = 0x022f, + PKCS11_CKA_OTP_USER_IDENTIFIER = 0x022a, + PKCS11_CKA_OTP_SERVICE_IDENTIFIER = 0x022b, + PKCS11_CKA_OTP_SERVICE_LOGO = 0x022c, + PKCS11_CKA_OTP_SERVICE_LOGO_TYPE = 0x022d, + PKCS11_CKA_GOSTR3410_PARAMS = 0x0250, + PKCS11_CKA_GOSTR3411_PARAMS = 0x0251, + PKCS11_CKA_GOST28147_PARAMS = 0x0252, + PKCS11_CKA_HW_FEATURE_TYPE = 0x0300, + PKCS11_CKA_RESET_ON_INIT = 0x0301, + PKCS11_CKA_HAS_RESET = 0x0302, + PKCS11_CKA_PIXEL_X = 0x0400, + PKCS11_CKA_PIXEL_Y = 0x0401, + PKCS11_CKA_RESOLUTION = 0x0402, + PKCS11_CKA_CHAR_ROWS = 0x0403, + PKCS11_CKA_CHAR_COLUMNS = 0x0404, + PKCS11_CKA_COLOR = 0x0405, + PKCS11_CKA_BITS_PER_PIXEL = 0x0406, + PKCS11_CKA_CHAR_SETS = 0x0480, + PKCS11_CKA_ENCODING_METHODS = 0x0481, + PKCS11_CKA_MIME_TYPES = 0x0482, + PKCS11_CKA_MECHANISM_TYPE = 0x0500, + PKCS11_CKA_REQUIRED_CMS_ATTRIBUTES = 0x0501, + PKCS11_CKA_DEFAULT_CMS_ATTRIBUTES = 0x0502, + PKCS11_CKA_SUPPORTED_CMS_ATTRIBUTES = 0x0503, + /* + * The leading 4 comes from the PKCS#11 spec or:ing with + * CKF_ARRAY_ATTRIBUTE = 0x40000000. + */ + PKCS11_CKA_ALLOWED_MECHANISMS = 0x40000600, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKA_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_CLASS + * PKCS11_CKO_ reflects CryptoKi client API object class IDs CKO_. + */ +enum pkcs11_class_id { + PKCS11_CKO_DATA = 0x000, + PKCS11_CKO_CERTIFICATE = 0x001, + PKCS11_CKO_PUBLIC_KEY = 0x002, + PKCS11_CKO_PRIVATE_KEY = 0x003, + PKCS11_CKO_SECRET_KEY = 0x004, + PKCS11_CKO_HW_FEATURE = 0x005, + PKCS11_CKO_DOMAIN_PARAMETERS = 0x006, + PKCS11_CKO_MECHANISM = 0x007, + PKCS11_CKO_OTP_KEY = 0x008, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKO_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_KEY_TYPE + * PKCS11_CKK_ reflects CryptoKi client API key type IDs CKK_. + * Note that this is only a subset of the PKCS#11 specification. + */ +enum pkcs11_key_type { + PKCS11_CKK_RSA = 0x000, + PKCS11_CKK_DSA = 0x001, + PKCS11_CKK_DH = 0x002, + PKCS11_CKK_EC = 0x003, + PKCS11_CKK_GENERIC_SECRET = 0x010, + PKCS11_CKK_AES = 0x01f, + PKCS11_CKK_MD5_HMAC = 0x027, + PKCS11_CKK_SHA_1_HMAC = 0x028, + PKCS11_CKK_SHA256_HMAC = 0x02b, + PKCS11_CKK_SHA384_HMAC = 0x02c, + PKCS11_CKK_SHA512_HMAC = 0x02d, + PKCS11_CKK_SHA224_HMAC = 0x02e, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKK_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_CERTIFICATE_TYPE + */ +enum pkcs11_certificate_type { + PKCS11_CKC_X_509 = 0x00000000UL, + PKCS11_CKC_X_509_ATTR_CERT = 0x00000001UL, + PKCS11_CKC_WTLS = 0x00000002UL, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKC_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_CERTIFICATE_CATEGORY + */ +enum pkcs11_certificate_category { + PKCS11_CK_CERTIFICATE_CATEGORY_UNSPECIFIED = 0UL, + PKCS11_CK_CERTIFICATE_CATEGORY_TOKEN_USER = 1UL, + PKCS11_CK_CERTIFICATE_CATEGORY_AUTHORITY = 2UL, + PKCS11_CK_CERTIFICATE_CATEGORY_OTHER_ENTITY = 3UL, +}; + +/* + * Valid values for mechanism IDs + * PKCS11_CKM_ reflects CryptoKi client API mechanism IDs CKM_. + */ +enum pkcs11_mechanism_id { + PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN = 0x00000, + PKCS11_CKM_RSA_PKCS = 0x00001, + PKCS11_CKM_RSA_9796 = 0x00002, + PKCS11_CKM_RSA_X_509 = 0x00003, + PKCS11_CKM_MD5_RSA_PKCS = 0x00005, + PKCS11_CKM_SHA1_RSA_PKCS = 0x00006, + PKCS11_CKM_RSA_PKCS_OAEP = 0x00009, + PKCS11_CKM_RSA_PKCS_PSS = 0x0000d, + PKCS11_CKM_SHA1_RSA_PKCS_PSS = 0x0000e, + PKCS11_CKM_SHA256_RSA_PKCS = 0x00040, + PKCS11_CKM_SHA384_RSA_PKCS = 0x00041, + PKCS11_CKM_SHA512_RSA_PKCS = 0x00042, + PKCS11_CKM_SHA256_RSA_PKCS_PSS = 0x00043, + PKCS11_CKM_SHA384_RSA_PKCS_PSS = 0x00044, + PKCS11_CKM_SHA512_RSA_PKCS_PSS = 0x00045, + PKCS11_CKM_SHA224_RSA_PKCS = 0x00046, + PKCS11_CKM_SHA224_RSA_PKCS_PSS = 0x00047, + PKCS11_CKM_SHA512_224 = 0x00048, + PKCS11_CKM_SHA512_224_HMAC = 0x00049, + PKCS11_CKM_SHA512_224_HMAC_GENERAL = 0x0004a, + PKCS11_CKM_SHA512_224_KEY_DERIVATION = 0x0004b, + PKCS11_CKM_SHA512_256 = 0x0004c, + PKCS11_CKM_SHA512_256_HMAC = 0x0004d, + PKCS11_CKM_SHA512_256_HMAC_GENERAL = 0x0004e, + PKCS11_CKM_SHA512_256_KEY_DERIVATION = 0x0004f, + PKCS11_CKM_DES3_ECB = 0x00132, + PKCS11_CKM_DES3_CBC = 0x00133, + PKCS11_CKM_DES3_MAC = 0x00134, + PKCS11_CKM_DES3_MAC_GENERAL = 0x00135, + PKCS11_CKM_DES3_CBC_PAD = 0x00136, + PKCS11_CKM_DES3_CMAC_GENERAL = 0x00137, + PKCS11_CKM_DES3_CMAC = 0x00138, + PKCS11_CKM_MD5 = 0x00210, + PKCS11_CKM_MD5_HMAC = 0x00211, + PKCS11_CKM_MD5_HMAC_GENERAL = 0x00212, + PKCS11_CKM_SHA_1 = 0x00220, + PKCS11_CKM_SHA_1_HMAC = 0x00221, + PKCS11_CKM_SHA_1_HMAC_GENERAL = 0x00222, + PKCS11_CKM_SHA256 = 0x00250, + PKCS11_CKM_SHA256_HMAC = 0x00251, + PKCS11_CKM_SHA256_HMAC_GENERAL = 0x00252, + PKCS11_CKM_SHA224 = 0x00255, + PKCS11_CKM_SHA224_HMAC = 0x00256, + PKCS11_CKM_SHA224_HMAC_GENERAL = 0x00257, + PKCS11_CKM_SHA384 = 0x00260, + PKCS11_CKM_SHA384_HMAC = 0x00261, + PKCS11_CKM_SHA384_HMAC_GENERAL = 0x00262, + PKCS11_CKM_SHA512 = 0x00270, + PKCS11_CKM_SHA512_HMAC = 0x00271, + PKCS11_CKM_SHA512_HMAC_GENERAL = 0x00272, + PKCS11_CKM_HOTP_KEY_GEN = 0x00290, + PKCS11_CKM_HOTP = 0x00291, + PKCS11_CKM_GENERIC_SECRET_KEY_GEN = 0x00350, + PKCS11_CKM_MD5_KEY_DERIVATION = 0x00390, + PKCS11_CKM_MD2_KEY_DERIVATION = 0x00391, + PKCS11_CKM_SHA1_KEY_DERIVATION = 0x00392, + PKCS11_CKM_SHA256_KEY_DERIVATION = 0x00393, + PKCS11_CKM_SHA384_KEY_DERIVATION = 0x00394, + PKCS11_CKM_SHA512_KEY_DERIVATION = 0x00395, + PKCS11_CKM_SHA224_KEY_DERIVATION = 0x00396, + PKCS11_CKM_EC_KEY_PAIR_GEN = 0x01040, + PKCS11_CKM_ECDSA = 0x01041, + PKCS11_CKM_ECDSA_SHA1 = 0x01042, + PKCS11_CKM_ECDSA_SHA224 = 0x01043, + PKCS11_CKM_ECDSA_SHA256 = 0x01044, + PKCS11_CKM_ECDSA_SHA384 = 0x01045, + PKCS11_CKM_ECDSA_SHA512 = 0x01046, + PKCS11_CKM_ECDH1_DERIVE = 0x01050, + PKCS11_CKM_ECDH1_COFACTOR_DERIVE = 0x01051, + PKCS11_CKM_ECMQV_DERIVE = 0x01052, + PKCS11_CKM_ECDH_AES_KEY_WRAP = 0x01053, + PKCS11_CKM_RSA_AES_KEY_WRAP = 0x01054, + PKCS11_CKM_EDDSA = 0x01057, + PKCS11_CKM_AES_KEY_GEN = 0x01080, + PKCS11_CKM_AES_ECB = 0x01081, + PKCS11_CKM_AES_CBC = 0x01082, + PKCS11_CKM_AES_MAC = 0x01083, + PKCS11_CKM_AES_MAC_GENERAL = 0x01084, + PKCS11_CKM_AES_CBC_PAD = 0x01085, + PKCS11_CKM_AES_CTR = 0x01086, + PKCS11_CKM_AES_GCM = 0x01087, + PKCS11_CKM_AES_CCM = 0x01088, + PKCS11_CKM_AES_CTS = 0x01089, + PKCS11_CKM_AES_CMAC = 0x0108a, + PKCS11_CKM_AES_CMAC_GENERAL = 0x0108b, + PKCS11_CKM_AES_XCBC_MAC = 0x0108c, + PKCS11_CKM_AES_XCBC_MAC_96 = 0x0108d, + PKCS11_CKM_AES_GMAC = 0x0108e, + PKCS11_CKM_DES3_ECB_ENCRYPT_DATA = 0x01102, + PKCS11_CKM_DES3_CBC_ENCRYPT_DATA = 0x01103, + PKCS11_CKM_AES_ECB_ENCRYPT_DATA = 0x01104, + PKCS11_CKM_AES_CBC_ENCRYPT_DATA = 0x01105, + PKCS11_CKM_AES_KEY_WRAP = 0x02109, + PKCS11_CKM_AES_KEY_WRAP_PAD = 0x0210a, + /* + * Vendor extensions below. + * PKCS11 added IDs for operation not related to a CK mechanism ID + */ + PKCS11_PROCESSING_IMPORT = 0x80000000, + PKCS11_CKM_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values MG function identifiers + * PKCS11_CKG_ reflects CryptoKi client API MG function IDs CKG_. + */ +enum pkcs11_mgf_id { + PKCS11_CKG_MGF1_SHA1 = 0x0001, + PKCS11_CKG_MGF1_SHA224 = 0x0005, + PKCS11_CKG_MGF1_SHA256 = 0x0002, + PKCS11_CKG_MGF1_SHA384 = 0x0003, + PKCS11_CKG_MGF1_SHA512 = 0x0004, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKG_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for RSA PKCS/OAEP source type identifier + * PKCS11_CKZ_ reflects CryptoKi client API source type IDs CKZ_. + */ +#define PKCS11_CKZ_DATA_SPECIFIED 0x0001 + +#endif /*PKCS11_TA_H*/ diff --git a/optee_client/libckteec/src/ck_debug.c b/optee_client/libckteec/src/ck_debug.c new file mode 100644 index 0000000..49a5015 --- /dev/null +++ b/optee_client/libckteec/src/ck_debug.c @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#include "local_utils.h" + +#define CK2STR_ENTRY(id) case id: return #id + +const char *ckr2str(CK_RV id) +{ + switch (id) { + CK2STR_ENTRY(CKR_OK); + CK2STR_ENTRY(CKR_CANCEL); + CK2STR_ENTRY(CKR_HOST_MEMORY); + CK2STR_ENTRY(CKR_SLOT_ID_INVALID); + CK2STR_ENTRY(CKR_GENERAL_ERROR); + CK2STR_ENTRY(CKR_FUNCTION_FAILED); + CK2STR_ENTRY(CKR_ARGUMENTS_BAD); + CK2STR_ENTRY(CKR_NO_EVENT); + CK2STR_ENTRY(CKR_NEED_TO_CREATE_THREADS); + CK2STR_ENTRY(CKR_CANT_LOCK); + CK2STR_ENTRY(CKR_ATTRIBUTE_READ_ONLY); + CK2STR_ENTRY(CKR_ATTRIBUTE_SENSITIVE); + CK2STR_ENTRY(CKR_ATTRIBUTE_TYPE_INVALID); + CK2STR_ENTRY(CKR_ATTRIBUTE_VALUE_INVALID); + CK2STR_ENTRY(CKR_ACTION_PROHIBITED); + CK2STR_ENTRY(CKR_DATA_INVALID); + CK2STR_ENTRY(CKR_DATA_LEN_RANGE); + CK2STR_ENTRY(CKR_DEVICE_ERROR); + CK2STR_ENTRY(CKR_DEVICE_MEMORY); + CK2STR_ENTRY(CKR_DEVICE_REMOVED); + CK2STR_ENTRY(CKR_ENCRYPTED_DATA_INVALID); + CK2STR_ENTRY(CKR_ENCRYPTED_DATA_LEN_RANGE); + CK2STR_ENTRY(CKR_FUNCTION_CANCELED); + CK2STR_ENTRY(CKR_FUNCTION_NOT_PARALLEL); + CK2STR_ENTRY(CKR_FUNCTION_NOT_SUPPORTED); + CK2STR_ENTRY(CKR_KEY_HANDLE_INVALID); + CK2STR_ENTRY(CKR_KEY_SIZE_RANGE); + CK2STR_ENTRY(CKR_KEY_TYPE_INCONSISTENT); + CK2STR_ENTRY(CKR_KEY_NOT_NEEDED); + CK2STR_ENTRY(CKR_KEY_CHANGED); + CK2STR_ENTRY(CKR_KEY_NEEDED); + CK2STR_ENTRY(CKR_KEY_INDIGESTIBLE); + CK2STR_ENTRY(CKR_KEY_FUNCTION_NOT_PERMITTED); + CK2STR_ENTRY(CKR_KEY_NOT_WRAPPABLE); + CK2STR_ENTRY(CKR_KEY_UNEXTRACTABLE); + CK2STR_ENTRY(CKR_MECHANISM_INVALID); + CK2STR_ENTRY(CKR_MECHANISM_PARAM_INVALID); + CK2STR_ENTRY(CKR_OBJECT_HANDLE_INVALID); + CK2STR_ENTRY(CKR_OPERATION_ACTIVE); + CK2STR_ENTRY(CKR_OPERATION_NOT_INITIALIZED); + CK2STR_ENTRY(CKR_PIN_INCORRECT); + CK2STR_ENTRY(CKR_PIN_INVALID); + CK2STR_ENTRY(CKR_PIN_LEN_RANGE); + CK2STR_ENTRY(CKR_PIN_EXPIRED); + CK2STR_ENTRY(CKR_PIN_LOCKED); + CK2STR_ENTRY(CKR_SESSION_CLOSED); + CK2STR_ENTRY(CKR_SESSION_COUNT); + CK2STR_ENTRY(CKR_SESSION_HANDLE_INVALID); + CK2STR_ENTRY(CKR_SESSION_PARALLEL_NOT_SUPPORTED); + CK2STR_ENTRY(CKR_SESSION_READ_ONLY); + CK2STR_ENTRY(CKR_SESSION_EXISTS); + CK2STR_ENTRY(CKR_SESSION_READ_ONLY_EXISTS); + CK2STR_ENTRY(CKR_SESSION_READ_WRITE_SO_EXISTS); + CK2STR_ENTRY(CKR_SIGNATURE_INVALID); + CK2STR_ENTRY(CKR_SIGNATURE_LEN_RANGE); + CK2STR_ENTRY(CKR_TEMPLATE_INCOMPLETE); + CK2STR_ENTRY(CKR_TEMPLATE_INCONSISTENT); + CK2STR_ENTRY(CKR_TOKEN_NOT_PRESENT); + CK2STR_ENTRY(CKR_TOKEN_NOT_RECOGNIZED); + CK2STR_ENTRY(CKR_TOKEN_WRITE_PROTECTED); + CK2STR_ENTRY(CKR_UNWRAPPING_KEY_HANDLE_INVALID); + CK2STR_ENTRY(CKR_UNWRAPPING_KEY_SIZE_RANGE); + CK2STR_ENTRY(CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT); + CK2STR_ENTRY(CKR_USER_ALREADY_LOGGED_IN); + CK2STR_ENTRY(CKR_USER_NOT_LOGGED_IN); + CK2STR_ENTRY(CKR_USER_PIN_NOT_INITIALIZED); + CK2STR_ENTRY(CKR_USER_TYPE_INVALID); + CK2STR_ENTRY(CKR_USER_ANOTHER_ALREADY_LOGGED_IN); + CK2STR_ENTRY(CKR_USER_TOO_MANY_TYPES); + CK2STR_ENTRY(CKR_WRAPPED_KEY_INVALID); + CK2STR_ENTRY(CKR_WRAPPED_KEY_LEN_RANGE); + CK2STR_ENTRY(CKR_WRAPPING_KEY_HANDLE_INVALID); + CK2STR_ENTRY(CKR_WRAPPING_KEY_SIZE_RANGE); + CK2STR_ENTRY(CKR_WRAPPING_KEY_TYPE_INCONSISTENT); + CK2STR_ENTRY(CKR_RANDOM_SEED_NOT_SUPPORTED); + CK2STR_ENTRY(CKR_RANDOM_NO_RNG); + CK2STR_ENTRY(CKR_DOMAIN_PARAMS_INVALID); + CK2STR_ENTRY(CKR_CURVE_NOT_SUPPORTED); + CK2STR_ENTRY(CKR_BUFFER_TOO_SMALL); + CK2STR_ENTRY(CKR_SAVED_STATE_INVALID); + CK2STR_ENTRY(CKR_INFORMATION_SENSITIVE); + CK2STR_ENTRY(CKR_STATE_UNSAVEABLE); + CK2STR_ENTRY(CKR_CRYPTOKI_NOT_INITIALIZED); + CK2STR_ENTRY(CKR_CRYPTOKI_ALREADY_INITIALIZED); + CK2STR_ENTRY(CKR_MUTEX_BAD); + CK2STR_ENTRY(CKR_MUTEX_NOT_LOCKED); + CK2STR_ENTRY(CKR_NEW_PIN_MODE); + CK2STR_ENTRY(CKR_NEXT_OTP); + CK2STR_ENTRY(CKR_EXCEEDED_MAX_ITERATIONS); + CK2STR_ENTRY(CKR_FIPS_SELF_TEST_FAILED); + CK2STR_ENTRY(CKR_LIBRARY_LOAD_FAILED); + CK2STR_ENTRY(CKR_PIN_TOO_WEAK); + CK2STR_ENTRY(CKR_PUBLIC_KEY_INVALID); + CK2STR_ENTRY(CKR_FUNCTION_REJECTED); + default: + if (id & CKR_VENDOR_DEFINED) + return "Vendor defined"; + else + return "Unknown ID"; + } +} + +const char *ckm2str(CK_MECHANISM_TYPE id) +{ + switch (id) { + CK2STR_ENTRY(CKM_RSA_PKCS_KEY_PAIR_GEN); + CK2STR_ENTRY(CKM_RSA_PKCS); + CK2STR_ENTRY(CKM_RSA_9796); + CK2STR_ENTRY(CKM_RSA_X_509); + CK2STR_ENTRY(CKM_MD5_RSA_PKCS); + CK2STR_ENTRY(CKM_SHA1_RSA_PKCS); + CK2STR_ENTRY(CKM_RSA_PKCS_OAEP); + CK2STR_ENTRY(CKM_RSA_PKCS_PSS); + CK2STR_ENTRY(CKM_SHA1_RSA_PKCS_PSS); + CK2STR_ENTRY(CKM_SHA256_RSA_PKCS); + CK2STR_ENTRY(CKM_SHA384_RSA_PKCS); + CK2STR_ENTRY(CKM_SHA512_RSA_PKCS); + CK2STR_ENTRY(CKM_SHA256_RSA_PKCS_PSS); + CK2STR_ENTRY(CKM_SHA384_RSA_PKCS_PSS); + CK2STR_ENTRY(CKM_SHA512_RSA_PKCS_PSS); + CK2STR_ENTRY(CKM_SHA224_RSA_PKCS); + CK2STR_ENTRY(CKM_SHA224_RSA_PKCS_PSS); + CK2STR_ENTRY(CKM_SHA512_224); + CK2STR_ENTRY(CKM_SHA512_224_HMAC); + CK2STR_ENTRY(CKM_SHA512_224_HMAC_GENERAL); + CK2STR_ENTRY(CKM_SHA512_224_KEY_DERIVATION); + CK2STR_ENTRY(CKM_SHA512_256); + CK2STR_ENTRY(CKM_SHA512_256_HMAC); + CK2STR_ENTRY(CKM_SHA512_256_HMAC_GENERAL); + CK2STR_ENTRY(CKM_SHA512_256_KEY_DERIVATION); + CK2STR_ENTRY(CKM_MD5); + CK2STR_ENTRY(CKM_MD5_HMAC); + CK2STR_ENTRY(CKM_MD5_HMAC_GENERAL); + CK2STR_ENTRY(CKM_SHA_1); + CK2STR_ENTRY(CKM_SHA_1_HMAC); + CK2STR_ENTRY(CKM_SHA_1_HMAC_GENERAL); + CK2STR_ENTRY(CKM_SHA256); + CK2STR_ENTRY(CKM_SHA256_HMAC); + CK2STR_ENTRY(CKM_SHA256_HMAC_GENERAL); + CK2STR_ENTRY(CKM_SHA224); + CK2STR_ENTRY(CKM_SHA224_HMAC); + CK2STR_ENTRY(CKM_SHA224_HMAC_GENERAL); + CK2STR_ENTRY(CKM_SHA384); + CK2STR_ENTRY(CKM_SHA384_HMAC); + CK2STR_ENTRY(CKM_SHA384_HMAC_GENERAL); + CK2STR_ENTRY(CKM_SHA512); + CK2STR_ENTRY(CKM_SHA512_HMAC); + CK2STR_ENTRY(CKM_SHA512_HMAC_GENERAL); + CK2STR_ENTRY(CKM_HOTP_KEY_GEN); + CK2STR_ENTRY(CKM_HOTP); + CK2STR_ENTRY(CKM_GENERIC_SECRET_KEY_GEN); + CK2STR_ENTRY(CKM_SHA1_KEY_DERIVATION); + CK2STR_ENTRY(CKM_SHA256_KEY_DERIVATION); + CK2STR_ENTRY(CKM_SHA384_KEY_DERIVATION); + CK2STR_ENTRY(CKM_SHA512_KEY_DERIVATION); + CK2STR_ENTRY(CKM_SHA224_KEY_DERIVATION); + CK2STR_ENTRY(CKM_EC_KEY_PAIR_GEN); + CK2STR_ENTRY(CKM_ECDSA); + CK2STR_ENTRY(CKM_ECDSA_SHA1); + CK2STR_ENTRY(CKM_ECDSA_SHA224); + CK2STR_ENTRY(CKM_ECDSA_SHA256); + CK2STR_ENTRY(CKM_ECDSA_SHA384); + CK2STR_ENTRY(CKM_ECDSA_SHA512); + CK2STR_ENTRY(CKM_ECDH1_DERIVE); + CK2STR_ENTRY(CKM_ECDH1_COFACTOR_DERIVE); + CK2STR_ENTRY(CKM_ECMQV_DERIVE); + CK2STR_ENTRY(CKM_ECDH_AES_KEY_WRAP); + CK2STR_ENTRY(CKM_RSA_AES_KEY_WRAP); + CK2STR_ENTRY(CKM_AES_KEY_GEN); + CK2STR_ENTRY(CKM_AES_ECB); + CK2STR_ENTRY(CKM_AES_CBC); + CK2STR_ENTRY(CKM_AES_MAC); + CK2STR_ENTRY(CKM_AES_MAC_GENERAL); + CK2STR_ENTRY(CKM_AES_CBC_PAD); + CK2STR_ENTRY(CKM_AES_CTR); + CK2STR_ENTRY(CKM_AES_GCM); + CK2STR_ENTRY(CKM_AES_CCM); + CK2STR_ENTRY(CKM_AES_CTS); + CK2STR_ENTRY(CKM_AES_CMAC); + CK2STR_ENTRY(CKM_AES_CMAC_GENERAL); + CK2STR_ENTRY(CKM_AES_XCBC_MAC); + CK2STR_ENTRY(CKM_AES_XCBC_MAC_96); + CK2STR_ENTRY(CKM_AES_GMAC); + CK2STR_ENTRY(CKM_DES3_ECB_ENCRYPT_DATA); + CK2STR_ENTRY(CKM_DES3_CBC_ENCRYPT_DATA); + CK2STR_ENTRY(CKM_AES_ECB_ENCRYPT_DATA); + CK2STR_ENTRY(CKM_AES_CBC_ENCRYPT_DATA); + CK2STR_ENTRY(CKM_AES_KEY_WRAP); + CK2STR_ENTRY(CKM_AES_KEY_WRAP_PAD); + default: + if (id & CKM_VENDOR_DEFINED) + return "Vendor defined"; + else + return "Unknown ID"; + } +} + +const char *slot_ckf2str(CK_ULONG flag) +{ + switch (flag) { + CK2STR_ENTRY(CKF_TOKEN_PRESENT); + CK2STR_ENTRY(CKF_REMOVABLE_DEVICE); + CK2STR_ENTRY(CKF_HW_SLOT); + default: + return "Unknown flag"; + } +} + +const char *token_ckf2str(CK_ULONG flag) +{ + switch (flag) { + CK2STR_ENTRY(CKF_RNG); + CK2STR_ENTRY(CKF_WRITE_PROTECTED); + CK2STR_ENTRY(CKF_LOGIN_REQUIRED); + CK2STR_ENTRY(CKF_USER_PIN_INITIALIZED); + CK2STR_ENTRY(CKF_RESTORE_KEY_NOT_NEEDED); + CK2STR_ENTRY(CKF_CLOCK_ON_TOKEN); + CK2STR_ENTRY(CKF_PROTECTED_AUTHENTICATION_PATH); + CK2STR_ENTRY(CKF_DUAL_CRYPTO_OPERATIONS); + CK2STR_ENTRY(CKF_TOKEN_INITIALIZED); + CK2STR_ENTRY(CKF_SECONDARY_AUTHENTICATION); + CK2STR_ENTRY(CKF_USER_PIN_COUNT_LOW); + CK2STR_ENTRY(CKF_USER_PIN_FINAL_TRY); + CK2STR_ENTRY(CKF_USER_PIN_LOCKED); + CK2STR_ENTRY(CKF_USER_PIN_TO_BE_CHANGED); + CK2STR_ENTRY(CKF_SO_PIN_COUNT_LOW); + CK2STR_ENTRY(CKF_SO_PIN_FINAL_TRY); + CK2STR_ENTRY(CKF_SO_PIN_LOCKED); + CK2STR_ENTRY(CKF_SO_PIN_TO_BE_CHANGED); + CK2STR_ENTRY(CKF_ERROR_STATE); + default: + return "Unknown flag"; + } +} + +const char *mecha_ckf2str(CK_ULONG flag) +{ + switch (flag) { + CK2STR_ENTRY(CKF_HW); + CK2STR_ENTRY(CKF_ENCRYPT); + CK2STR_ENTRY(CKF_DECRYPT); + CK2STR_ENTRY(CKF_DIGEST); + CK2STR_ENTRY(CKF_SIGN); + CK2STR_ENTRY(CKF_SIGN_RECOVER); + CK2STR_ENTRY(CKF_VERIFY); + CK2STR_ENTRY(CKF_VERIFY_RECOVER); + CK2STR_ENTRY(CKF_GENERATE); + CK2STR_ENTRY(CKF_GENERATE_KEY_PAIR); + CK2STR_ENTRY(CKF_WRAP); + CK2STR_ENTRY(CKF_UNWRAP); + CK2STR_ENTRY(CKF_DERIVE); + CK2STR_ENTRY(CKF_EC_F_P); + CK2STR_ENTRY(CKF_EC_F_2M); + CK2STR_ENTRY(CKF_EC_ECPARAMETERS); + CK2STR_ENTRY(CKF_EC_NAMEDCURVE); + CK2STR_ENTRY(CKF_EC_UNCOMPRESS); + CK2STR_ENTRY(CKF_EC_COMPRESS); + CK2STR_ENTRY(CKF_EXTENSION); + default: + return "Unknown flag"; + } +} + +const char *session_ckf2str(CK_ULONG flag) +{ + switch (flag) { + CK2STR_ENTRY(CKF_RW_SESSION); + CK2STR_ENTRY(CKF_SERIAL_SESSION); + default: + return "Unknown flag"; + } +} + +const char *session_cks2str(CK_ULONG flag) +{ + switch (flag) { + CK2STR_ENTRY(CKS_RO_PUBLIC_SESSION); + CK2STR_ENTRY(CKS_RO_USER_FUNCTIONS); + CK2STR_ENTRY(CKS_RW_PUBLIC_SESSION); + CK2STR_ENTRY(CKS_RW_USER_FUNCTIONS); + CK2STR_ENTRY(CKS_RW_SO_FUNCTIONS); + default: + return "Unknown flag"; + } +} + +const char *cka2str(CK_ATTRIBUTE_TYPE id) +{ + switch (id) { + /* Standard CK attributes */ + CK2STR_ENTRY(CKA_CLASS); + CK2STR_ENTRY(CKA_TOKEN); + CK2STR_ENTRY(CKA_PRIVATE); + CK2STR_ENTRY(CKA_LABEL); + CK2STR_ENTRY(CKA_APPLICATION); + CK2STR_ENTRY(CKA_VALUE); + CK2STR_ENTRY(CKA_OBJECT_ID); + CK2STR_ENTRY(CKA_CERTIFICATE_TYPE); + CK2STR_ENTRY(CKA_ISSUER); + CK2STR_ENTRY(CKA_SERIAL_NUMBER); + CK2STR_ENTRY(CKA_AC_ISSUER); + CK2STR_ENTRY(CKA_OWNER); + CK2STR_ENTRY(CKA_ATTR_TYPES); + CK2STR_ENTRY(CKA_TRUSTED); + CK2STR_ENTRY(CKA_CERTIFICATE_CATEGORY); + CK2STR_ENTRY(CKA_JAVA_MIDP_SECURITY_DOMAIN); + CK2STR_ENTRY(CKA_URL); + CK2STR_ENTRY(CKA_HASH_OF_SUBJECT_PUBLIC_KEY); + CK2STR_ENTRY(CKA_HASH_OF_ISSUER_PUBLIC_KEY); + CK2STR_ENTRY(CKA_NAME_HASH_ALGORITHM); + CK2STR_ENTRY(CKA_CHECK_VALUE); + CK2STR_ENTRY(CKA_KEY_TYPE); + CK2STR_ENTRY(CKA_SUBJECT); + CK2STR_ENTRY(CKA_ID); + CK2STR_ENTRY(CKA_SENSITIVE); + CK2STR_ENTRY(CKA_ENCRYPT); + CK2STR_ENTRY(CKA_DECRYPT); + CK2STR_ENTRY(CKA_WRAP); + CK2STR_ENTRY(CKA_UNWRAP); + CK2STR_ENTRY(CKA_SIGN); + CK2STR_ENTRY(CKA_SIGN_RECOVER); + CK2STR_ENTRY(CKA_VERIFY); + CK2STR_ENTRY(CKA_VERIFY_RECOVER); + CK2STR_ENTRY(CKA_DERIVE); + CK2STR_ENTRY(CKA_START_DATE); + CK2STR_ENTRY(CKA_END_DATE); + CK2STR_ENTRY(CKA_MODULUS); + CK2STR_ENTRY(CKA_MODULUS_BITS); + CK2STR_ENTRY(CKA_PUBLIC_EXPONENT); + CK2STR_ENTRY(CKA_PRIVATE_EXPONENT); + CK2STR_ENTRY(CKA_PRIME_1); + CK2STR_ENTRY(CKA_PRIME_2); + CK2STR_ENTRY(CKA_EXPONENT_1); + CK2STR_ENTRY(CKA_EXPONENT_2); + CK2STR_ENTRY(CKA_COEFFICIENT); + CK2STR_ENTRY(CKA_PUBLIC_KEY_INFO); + CK2STR_ENTRY(CKA_PRIME); + CK2STR_ENTRY(CKA_SUBPRIME); + CK2STR_ENTRY(CKA_BASE); + CK2STR_ENTRY(CKA_PRIME_BITS); + CK2STR_ENTRY(CKA_SUBPRIME_BITS); + CK2STR_ENTRY(CKA_VALUE_BITS); + CK2STR_ENTRY(CKA_VALUE_LEN); + CK2STR_ENTRY(CKA_EXTRACTABLE); + CK2STR_ENTRY(CKA_LOCAL); + CK2STR_ENTRY(CKA_NEVER_EXTRACTABLE); + CK2STR_ENTRY(CKA_ALWAYS_SENSITIVE); + CK2STR_ENTRY(CKA_KEY_GEN_MECHANISM); + CK2STR_ENTRY(CKA_MODIFIABLE); + CK2STR_ENTRY(CKA_COPYABLE); + CK2STR_ENTRY(CKA_DESTROYABLE); + CK2STR_ENTRY(CKA_EC_PARAMS); + CK2STR_ENTRY(CKA_EC_POINT); + CK2STR_ENTRY(CKA_ALWAYS_AUTHENTICATE); + CK2STR_ENTRY(CKA_WRAP_WITH_TRUSTED); + CK2STR_ENTRY(CKA_WRAP_TEMPLATE); + CK2STR_ENTRY(CKA_UNWRAP_TEMPLATE); + CK2STR_ENTRY(CKA_DERIVE_TEMPLATE); + CK2STR_ENTRY(CKA_OTP_FORMAT); + CK2STR_ENTRY(CKA_OTP_LENGTH); + CK2STR_ENTRY(CKA_OTP_TIME_INTERVAL); + CK2STR_ENTRY(CKA_OTP_USER_FRIENDLY_MODE); + CK2STR_ENTRY(CKA_OTP_CHALLENGE_REQUIREMENT); + CK2STR_ENTRY(CKA_OTP_TIME_REQUIREMENT); + CK2STR_ENTRY(CKA_OTP_COUNTER_REQUIREMENT); + CK2STR_ENTRY(CKA_OTP_PIN_REQUIREMENT); + CK2STR_ENTRY(CKA_OTP_COUNTER); + CK2STR_ENTRY(CKA_OTP_TIME); + CK2STR_ENTRY(CKA_OTP_USER_IDENTIFIER); + CK2STR_ENTRY(CKA_OTP_SERVICE_IDENTIFIER); + CK2STR_ENTRY(CKA_OTP_SERVICE_LOGO); + CK2STR_ENTRY(CKA_OTP_SERVICE_LOGO_TYPE); + CK2STR_ENTRY(CKA_GOSTR3410_PARAMS); + CK2STR_ENTRY(CKA_GOSTR3411_PARAMS); + CK2STR_ENTRY(CKA_GOST28147_PARAMS); + CK2STR_ENTRY(CKA_HW_FEATURE_TYPE); + CK2STR_ENTRY(CKA_RESET_ON_INIT); + CK2STR_ENTRY(CKA_HAS_RESET); + CK2STR_ENTRY(CKA_PIXEL_X); + CK2STR_ENTRY(CKA_PIXEL_Y); + CK2STR_ENTRY(CKA_RESOLUTION); + CK2STR_ENTRY(CKA_CHAR_ROWS); + CK2STR_ENTRY(CKA_CHAR_COLUMNS); + CK2STR_ENTRY(CKA_COLOR); + CK2STR_ENTRY(CKA_BITS_PER_PIXEL); + CK2STR_ENTRY(CKA_CHAR_SETS); + CK2STR_ENTRY(CKA_ENCODING_METHODS); + CK2STR_ENTRY(CKA_MIME_TYPES); + CK2STR_ENTRY(CKA_MECHANISM_TYPE); + CK2STR_ENTRY(CKA_REQUIRED_CMS_ATTRIBUTES); + CK2STR_ENTRY(CKA_DEFAULT_CMS_ATTRIBUTES); + CK2STR_ENTRY(CKA_SUPPORTED_CMS_ATTRIBUTES); + CK2STR_ENTRY(CKA_ALLOWED_MECHANISMS); + CK2STR_ENTRY(CKA_VENDOR_DEFINED); + default: + if (id & CKA_VENDOR_DEFINED) + return "Vendor defined"; + else + return "Unknown ID"; + } +} + +const char *cko2str(CK_OBJECT_CLASS id) +{ + switch (id) { + CK2STR_ENTRY(CKO_DATA); + CK2STR_ENTRY(CKO_CERTIFICATE); + CK2STR_ENTRY(CKO_PUBLIC_KEY); + CK2STR_ENTRY(CKO_PRIVATE_KEY); + CK2STR_ENTRY(CKO_SECRET_KEY); + CK2STR_ENTRY(CKO_HW_FEATURE); + CK2STR_ENTRY(CKO_DOMAIN_PARAMETERS); + CK2STR_ENTRY(CKO_MECHANISM); + CK2STR_ENTRY(CKO_OTP_KEY); + CK2STR_ENTRY(CKO_VENDOR_DEFINED); + default: + return "Unknown"; + } +} + +const char *ckk2str(CK_KEY_TYPE id) +{ + switch (id) { + CK2STR_ENTRY(CKK_RSA); + CK2STR_ENTRY(CKK_DSA); + CK2STR_ENTRY(CKK_DH); + CK2STR_ENTRY(CKK_ECDSA); + CK2STR_ENTRY(CKK_GENERIC_SECRET); + CK2STR_ENTRY(CKK_DES3); + CK2STR_ENTRY(CKK_AES); + CK2STR_ENTRY(CKK_HOTP); + CK2STR_ENTRY(CKK_MD5_HMAC); + CK2STR_ENTRY(CKK_SHA_1_HMAC); + CK2STR_ENTRY(CKK_SHA256_HMAC); + CK2STR_ENTRY(CKK_SHA384_HMAC); + CK2STR_ENTRY(CKK_SHA512_HMAC); + CK2STR_ENTRY(CKK_SHA224_HMAC); + CK2STR_ENTRY(CKK_VENDOR_DEFINED); + default: + return "Unknown"; + } +} diff --git a/optee_client/libckteec/src/ck_helpers.c b/optee_client/libckteec/src/ck_helpers.c new file mode 100644 index 0000000..776c115 --- /dev/null +++ b/optee_client/libckteec/src/ck_helpers.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#include "ck_helpers.h" + +#ifdef DEBUG +void ckteec_assert_expected_rv(const char *function, CK_RV rv, + const CK_RV *expected_rv, size_t expected_count) +{ + size_t n = 0; + + for (n = 0; n < expected_count; n++) + if (rv == expected_rv[n]) + return; + + fprintf(stderr, "libckteec: %s: unexpected return value 0x%lx (%s)\n", + function, rv, ckr2str(rv)); + + assert(0); +} +#endif diff --git a/optee_client/libckteec/src/ck_helpers.h b/optee_client/libckteec/src/ck_helpers.h new file mode 100644 index 0000000..e00a798 --- /dev/null +++ b/optee_client/libckteec/src/ck_helpers.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Linaro Limited + */ + +#ifndef LIBCKTEEC_CK_HELPERS_H +#define LIBCKTEEC_CK_HELPERS_H + +#include +#include + +#include "local_utils.h" + +#ifdef DEBUG +#define ASSERT_CK_RV(_rv, ...) \ + do { \ + const CK_RV ref[] = { __VA_ARGS__ }; \ + size_t count = ARRAY_SIZE(ref); \ + \ + ckteec_assert_expected_rv(__func__, (_rv), ref, count); \ + } while (0) + +void ckteec_assert_expected_rv(const char *function, CK_RV rv, + const CK_RV *expected_rv, size_t expected_count); +#else +#define ASSERT_CK_RV(_rv, ...) (void)0 +#endif /*DEBUG*/ + +#endif /*LIBCKTEEC_CK_HELPERS_H*/ diff --git a/optee_client/libckteec/src/invoke_ta.c b/optee_client/libckteec/src/invoke_ta.c new file mode 100644 index 0000000..07b110e --- /dev/null +++ b/optee_client/libckteec/src/invoke_ta.c @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +/* BINARY_PREFIX is expected by teec_trace.h */ +#ifndef BINARY_PREFIX +#define BINARY_PREFIX "ckteec" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ck_helpers.h" +#include "invoke_ta.h" +#include "local_utils.h" + +struct ta_context { + pthread_mutex_t init_mutex; + bool initiated; + TEEC_Context context; + TEEC_Session session; +}; + +static struct ta_context ta_ctx = { + .init_mutex = PTHREAD_MUTEX_INITIALIZER, +}; + +bool ckteec_invoke_initiated(void) +{ + return ta_ctx.initiated; +} + +TEEC_SharedMemory *ckteec_alloc_shm(size_t size, enum ckteec_shm_dir dir) +{ + TEEC_SharedMemory *shm = NULL; + + switch (dir) { + case CKTEEC_SHM_IN: + case CKTEEC_SHM_OUT: + case CKTEEC_SHM_INOUT: + break; + default: + return NULL; + } + + shm = calloc(1, sizeof(TEEC_SharedMemory)); + if (!shm) + return NULL; + + shm->size = size; + + if (dir == CKTEEC_SHM_IN || dir == CKTEEC_SHM_INOUT) + shm->flags |= TEEC_MEM_INPUT; + if (dir == CKTEEC_SHM_OUT || dir == CKTEEC_SHM_INOUT) + shm->flags |= TEEC_MEM_OUTPUT; + + if (TEEC_AllocateSharedMemory(&ta_ctx.context, shm)) { + free(shm); + return NULL; + } + + return shm; +} + +TEEC_SharedMemory *ckteec_register_shm(void *buffer, size_t size, + enum ckteec_shm_dir dir) +{ + TEEC_SharedMemory *shm = NULL; + + switch (dir) { + case CKTEEC_SHM_IN: + case CKTEEC_SHM_OUT: + case CKTEEC_SHM_INOUT: + break; + default: + return NULL; + } + + shm = calloc(1, sizeof(TEEC_SharedMemory)); + if (!shm) + return NULL; + + shm->buffer = buffer; + shm->size = size; + + if (dir == CKTEEC_SHM_IN || dir == CKTEEC_SHM_INOUT) + shm->flags |= TEEC_MEM_INPUT; + if (dir == CKTEEC_SHM_OUT || dir == CKTEEC_SHM_INOUT) + shm->flags |= TEEC_MEM_OUTPUT; + + if (TEEC_RegisterSharedMemory(&ta_ctx.context, shm)) { + free(shm); + return NULL; + } + + return shm; +} + +void ckteec_free_shm(TEEC_SharedMemory *shm) +{ + TEEC_ReleaseSharedMemory(shm); + free(shm); +} + +static bool is_output_shm(TEEC_SharedMemory *shm) +{ + return shm && (shm->flags & TEEC_MEM_OUTPUT); +} + +CK_RV ckteec_invoke_ta(unsigned long cmd, TEEC_SharedMemory *ctrl, + TEEC_SharedMemory *io1, + TEEC_SharedMemory *io2, size_t *out2_size, + TEEC_SharedMemory *io3, size_t *out3_size) +{ + uint32_t command = (uint32_t)cmd; + TEEC_Operation op; + uint32_t origin = 0; + TEEC_Result res = TEEC_ERROR_GENERIC; + uint32_t ta_rc = PKCS11_CKR_GENERAL_ERROR; + + if ((is_output_shm(io2) && !out2_size) || + (is_output_shm(io3) && !out3_size)) + return CKR_ARGUMENTS_BAD; + + memset(&op, 0, sizeof(op)); + + if (ctrl && !(ctrl->flags & TEEC_MEM_INPUT && + ctrl->flags & TEEC_MEM_OUTPUT)) + return CKR_ARGUMENTS_BAD; + + if (ctrl) { + op.paramTypes |= TEEC_PARAM_TYPES(TEEC_MEMREF_WHOLE, 0, 0, 0); + op.params[0].memref.parent = ctrl; + } else { + /* TA mandates param#0 as in/out memref for output status */ + op.paramTypes |= TEEC_PARAM_TYPES(TEEC_MEMREF_TEMP_INOUT, + 0, 0, 0); + op.params[0].tmpref.buffer = &ta_rc; + op.params[0].tmpref.size = sizeof(ta_rc); + } + + if (io1) { + op.paramTypes |= TEEC_PARAM_TYPES(0, TEEC_MEMREF_WHOLE, 0, 0); + op.params[1].memref.parent = io1; + } + + if (io2) { + op.paramTypes |= TEEC_PARAM_TYPES(0, 0, TEEC_MEMREF_WHOLE, 0); + op.params[2].memref.parent = io2; + } + + if (io3) { + op.paramTypes |= TEEC_PARAM_TYPES(0, 0, 0, TEEC_MEMREF_WHOLE); + op.params[3].memref.parent = io3; + } + + res = TEEC_InvokeCommand(&ta_ctx.session, command, &op, &origin); + switch (res) { + case TEEC_SUCCESS: + /* Get PKCS11 TA return value from ctrl buffer */ + if (ctrl) { + if (op.params[0].memref.size == sizeof(ta_rc)) + memcpy(&ta_rc, ctrl->buffer, sizeof(ta_rc)); + } else { + if (op.params[0].tmpref.size != sizeof(ta_rc)) + ta_rc = PKCS11_CKR_GENERAL_ERROR; + } + break; + case TEEC_ERROR_SHORT_BUFFER: + ta_rc = CKR_BUFFER_TOO_SMALL; + break; + case TEEC_ERROR_OUT_OF_MEMORY: + return CKR_DEVICE_MEMORY; + default: + return CKR_GENERAL_ERROR; + } + + if (ta_rc == CKR_OK || ta_rc == CKR_BUFFER_TOO_SMALL) { + if (is_output_shm(io2)) + *out2_size = op.params[2].memref.size; + if (is_output_shm(io3)) + *out3_size = op.params[3].memref.size; + } + + return ta_rc; +} + +static CK_RV ping_ta(void) +{ + TEEC_Operation op = { 0 }; + uint32_t origin = 0; + TEEC_Result res = TEEC_SUCCESS; + uint32_t ta_version[3] = { 0 }; + uint32_t status = 0; + + memset(&op, 0, sizeof(op)); + op.params[0].tmpref.buffer = &status; + op.params[0].tmpref.size = sizeof(status); + op.params[2].tmpref.buffer = ta_version; + op.params[2].tmpref.size = sizeof(ta_version); + op.paramTypes = TEEC_PARAM_TYPES(TEEC_MEMREF_TEMP_INOUT, TEEC_NONE, + TEEC_MEMREF_TEMP_OUTPUT, TEEC_NONE); + + res = TEEC_InvokeCommand(&ta_ctx.session, PKCS11_CMD_PING, &op, + &origin); + + if (res != TEEC_SUCCESS || + origin != TEEC_ORIGIN_TRUSTED_APP || + op.params[0].tmpref.size != sizeof(status) || + status != PKCS11_CKR_OK) + return CKR_DEVICE_ERROR; + + if (ta_version[0] != PKCS11_TA_VERSION_MAJOR && + ta_version[1] > PKCS11_TA_VERSION_MINOR) { + EMSG("PKCS11 TA version mismatch: %"PRIu32".%"PRIu32".%"PRIu32, + ta_version[0], ta_version[1], ta_version[2]); + + return CKR_DEVICE_ERROR; + } + + DMSG("PKCS11 TA version %"PRIu32".%"PRIu32".%"PRIu32, + ta_version[0], ta_version[1], ta_version[2]); + + return CKR_OK; +} + +CK_RV ckteec_invoke_init(void) +{ + TEEC_UUID uuid = PKCS11_TA_UUID; + uint32_t origin = 0; + TEEC_Result res = TEEC_SUCCESS; + CK_RV rv = CKR_CRYPTOKI_ALREADY_INITIALIZED; + const char *login_type_env = NULL; + const char *login_gid_env = NULL; + uint32_t login_method = TEEC_LOGIN_PUBLIC; + void *login_data = NULL; + gid_t login_gid = 0; + unsigned long tmpconv = 0; + char *endp = NULL; + int e = 0; + + login_type_env = getenv("CKTEEC_LOGIN_TYPE"); + + if (login_type_env) { + if (strcmp(login_type_env, "public") == 0) { + login_method = TEEC_LOGIN_PUBLIC; + } else if (strcmp(login_type_env, "user") == 0) { + login_method = TEEC_LOGIN_USER; + } else if (strcmp(login_type_env, "group") == 0) { + login_gid_env = getenv("CKTEEC_LOGIN_GID"); + if (!login_gid_env || !strlen(login_gid_env)) { + EMSG("missing CKTEEC_LOGIN_GID"); + rv = CKR_ARGUMENTS_BAD; + goto out; + } + + login_method = TEEC_LOGIN_GROUP; + tmpconv = strtoul(login_gid_env, &endp, 10); + if (errno == ERANGE || tmpconv > (gid_t)-1 || + (login_gid_env + strlen(login_gid_env) != endp)) { + EMSG("failed to convert CKTEEC_LOGIN_GID"); + rv = CKR_ARGUMENTS_BAD; + goto out; + } + + login_gid = (gid_t)tmpconv; + login_data = &login_gid; + } else { + EMSG("invalid value for CKTEEC_LOGIN_TYPE"); + rv = CKR_ARGUMENTS_BAD; + goto out; + } + } + + e = pthread_mutex_lock(&ta_ctx.init_mutex); + if (e) + return CKR_CANT_LOCK; + + if (ta_ctx.initiated) { + rv = CKR_CRYPTOKI_ALREADY_INITIALIZED; + goto out; + } + + res = TEEC_InitializeContext(NULL, &ta_ctx.context); + if (res != TEEC_SUCCESS) { + EMSG("TEEC init context failed\n"); + rv = CKR_DEVICE_ERROR; + goto out; + } + + res = TEEC_OpenSession(&ta_ctx.context, &ta_ctx.session, &uuid, + login_method, login_data, NULL, &origin); + if (res != TEEC_SUCCESS) { + EMSG("TEEC open session failed %x from %d\n", res, origin); + TEEC_FinalizeContext(&ta_ctx.context); + rv = CKR_DEVICE_ERROR; + goto out; + } + + rv = ping_ta(); + + if (rv == CKR_OK) { + ta_ctx.initiated = true; + } else { + TEEC_CloseSession(&ta_ctx.session); + TEEC_FinalizeContext(&ta_ctx.context); + } + +out: + e = pthread_mutex_unlock(&ta_ctx.init_mutex); + if (e) { + EMSG("pthread_mutex_unlock: %s", strerror(e)); + EMSG("terminating..."); + exit(EXIT_FAILURE); + } + + return rv; +} + +CK_RV ckteec_invoke_terminate(void) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + int e = 0; + + e = pthread_mutex_lock(&ta_ctx.init_mutex); + if (e) { + EMSG("pthread_mutex_lock: %s", strerror(e)); + EMSG("terminating..."); + exit(EXIT_FAILURE); + } + + if (!ta_ctx.initiated) + goto out; + + ta_ctx.initiated = false; + TEEC_CloseSession(&ta_ctx.session); + TEEC_FinalizeContext(&ta_ctx.context); + + rv = CKR_OK; + +out: + e = pthread_mutex_unlock(&ta_ctx.init_mutex); + if (e) { + EMSG("pthread_mutex_unlock: %s", strerror(e)); + EMSG("terminating..."); + exit(EXIT_FAILURE); + } + + return rv; +} diff --git a/optee_client/libckteec/src/invoke_ta.h b/optee_client/libckteec/src/invoke_ta.h new file mode 100644 index 0000000..b54f319 --- /dev/null +++ b/optee_client/libckteec/src/invoke_ta.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 20187-2020, Linaro Limited + */ + +#ifndef LIBCKTEEC_INVOKE_TA_H +#define LIBCKTEEC_INVOKE_TA_H + +#include +#include + +enum ckteec_shm_dir { + CKTEEC_SHM_IN, + CKTEEC_SHM_OUT, + CKTEEC_SHM_INOUT, +}; + +/** + * ckteec_alloc_shm - Allocate memory in the TEE SHM (in, out or in/out) + * + * @size - Allocated size in byte + * @dir - Data direction used for the shared memory + * + * Return a shm reference or NULL on failure. + */ +TEEC_SharedMemory *ckteec_alloc_shm(size_t size, enum ckteec_shm_dir dir); + +/** + * ckteec_register_shm - Register memory as shared in the TEE SHM + * + * @buffer - Base address of buffer to register + * @size - Allocated size in byte + * @dir - Data direction used for the shared memory + * + * Return a shm reference or NULL on failure. + */ +TEEC_SharedMemory *ckteec_register_shm(void *buffer, size_t size, + enum ckteec_shm_dir dir); + +/** + * ckteec_free_shm - Release allocated or registered emory in the TEE SHM + * + * @shm - memory reference + */ +void ckteec_free_shm(TEEC_SharedMemory *shm); + +/** + * ckteec_invoke_ta - Invoke PKCS11 TA for a target request through the TEE + * + * @cmd - PKCS11 TA command ID + * @ctrl - shared memory with serialized request input arguments or NULL + * @io1 - In memory buffer argument #1 for the command or NULL + * @io2 - In and/or out memory buffer argument #2 for the command or NULL + * @out2_size - Reference to @io2 output buffer size or NULL if not applicable + * @io3 - In and/or out memory buffer argument #3 for the command or NULL + * @out3_size - Reference to @io3 output buffer size or NULL if not applicable + * + * Return a CR_RV compliant return value + */ +CK_RV ckteec_invoke_ta(unsigned long cmd, TEEC_SharedMemory *ctrl, + TEEC_SharedMemory *io1, + TEEC_SharedMemory *io2, size_t *out2_size, + TEEC_SharedMemory *io3, size_t *out3_size); + +static inline CK_RV ckteec_invoke_ctrl(unsigned long cmd, + TEEC_SharedMemory *ctrl) +{ + return ckteec_invoke_ta(cmd, ctrl, NULL, NULL, NULL, NULL, NULL); +} + +static inline CK_RV ckteec_invoke_ctrl_in(unsigned long cmd, + TEEC_SharedMemory *ctrl, + TEEC_SharedMemory *io1) +{ + return ckteec_invoke_ta(cmd, ctrl, io1, NULL, NULL, NULL, NULL); +} + +static inline CK_RV ckteec_invoke_ctrl_out(unsigned long cmd, + TEEC_SharedMemory *ctrl, + TEEC_SharedMemory *io2, + size_t *out_sz) +{ + return ckteec_invoke_ta(cmd, ctrl, NULL, io2, out_sz, NULL, NULL); +} + +/* + * ckteec_invoke_init - Initialize TEE session with the PKCS11 TA + * + * Return a CR_RV compliant return value + */ +CK_RV ckteec_invoke_init(void); + +/* + * ckteec_invoke_terminate - Release all allocated invocation resources + * + * Return a CR_RV compliant return value + */ +CK_RV ckteec_invoke_terminate(void); + +/* Return true if and only if the PKCS11 TA invocation context is initiated */ +bool ckteec_invoke_initiated(void); + +#endif /*LIBCKTEEC_INVOKE_TA_H*/ diff --git a/optee_client/libckteec/src/local_utils.h b/optee_client/libckteec/src/local_utils.h new file mode 100644 index 0000000..57e46fd --- /dev/null +++ b/optee_client/libckteec/src/local_utils.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Linaro Limited + */ + +#ifndef LIBCKTEEC_LOCAL_UTILS_H +#define LIBCKTEEC_LOCAL_UTILS_H + +#define ARRAY_SIZE(array) (sizeof(array) / sizeof(array[0])) + +#define COMPILE_TIME_ASSERT(x) \ + do { \ + switch (0) { case 0: case ((x) ? 1: 0) : default : break; } \ + } while (0) + +#endif /*LIBCKTEEC_LOCAL_UTILS_H*/ diff --git a/optee_client/libckteec/src/pkcs11_api.c b/optee_client/libckteec/src/pkcs11_api.c new file mode 100644 index 0000000..9b83454 --- /dev/null +++ b/optee_client/libckteec/src/pkcs11_api.c @@ -0,0 +1,1473 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include + +#include "ck_helpers.h" +#include "invoke_ta.h" +#include "pkcs11_processing.h" +#include "pkcs11_token.h" + +static const CK_FUNCTION_LIST libckteec_function_list = { + .version = { + .major = CK_PKCS11_VERSION_MAJOR, + .minor = CK_PKCS11_VERSION_MINOR, + }, + .C_Initialize = C_Initialize, + .C_Finalize = C_Finalize, + .C_GetInfo = C_GetInfo, + .C_GetFunctionList = C_GetFunctionList, + .C_GetSlotList = C_GetSlotList, + .C_GetSlotInfo = C_GetSlotInfo, + .C_GetTokenInfo = C_GetTokenInfo, + .C_GetMechanismList = C_GetMechanismList, + .C_GetMechanismInfo = C_GetMechanismInfo, + .C_InitToken = C_InitToken, + .C_InitPIN = C_InitPIN, + .C_SetPIN = C_SetPIN, + .C_OpenSession = C_OpenSession, + .C_CloseSession = C_CloseSession, + .C_CloseAllSessions = C_CloseAllSessions, + .C_GetSessionInfo = C_GetSessionInfo, + .C_GetOperationState = C_GetOperationState, + .C_SetOperationState = C_SetOperationState, + .C_Login = C_Login, + .C_Logout = C_Logout, + .C_CreateObject = C_CreateObject, + .C_CopyObject = C_CopyObject, + .C_DestroyObject = C_DestroyObject, + .C_GetObjectSize = C_GetObjectSize, + .C_GetAttributeValue = C_GetAttributeValue, + .C_SetAttributeValue = C_SetAttributeValue, + .C_FindObjectsInit = C_FindObjectsInit, + .C_FindObjects = C_FindObjects, + .C_FindObjectsFinal = C_FindObjectsFinal, + .C_EncryptInit = C_EncryptInit, + .C_Encrypt = C_Encrypt, + .C_EncryptUpdate = C_EncryptUpdate, + .C_EncryptFinal = C_EncryptFinal, + .C_DecryptInit = C_DecryptInit, + .C_Decrypt = C_Decrypt, + .C_DecryptUpdate = C_DecryptUpdate, + .C_DecryptFinal = C_DecryptFinal, + .C_DigestInit = C_DigestInit, + .C_Digest = C_Digest, + .C_DigestUpdate = C_DigestUpdate, + .C_DigestKey = C_DigestKey, + .C_DigestFinal = C_DigestFinal, + .C_SignInit = C_SignInit, + .C_Sign = C_Sign, + .C_SignUpdate = C_SignUpdate, + .C_SignFinal = C_SignFinal, + .C_SignRecoverInit = C_SignRecoverInit, + .C_SignRecover = C_SignRecover, + .C_VerifyInit = C_VerifyInit, + .C_Verify = C_Verify, + .C_VerifyUpdate = C_VerifyUpdate, + .C_VerifyFinal = C_VerifyFinal, + .C_VerifyRecoverInit = C_VerifyRecoverInit, + .C_VerifyRecover = C_VerifyRecover, + .C_DigestEncryptUpdate = C_DigestEncryptUpdate, + .C_DecryptDigestUpdate = C_DecryptDigestUpdate, + .C_SignEncryptUpdate = C_SignEncryptUpdate, + .C_DecryptVerifyUpdate = C_DecryptVerifyUpdate, + .C_GenerateKey = C_GenerateKey, + .C_GenerateKeyPair = C_GenerateKeyPair, + .C_WrapKey = C_WrapKey, + .C_UnwrapKey = C_UnwrapKey, + .C_DeriveKey = C_DeriveKey, + .C_SeedRandom = C_SeedRandom, + .C_GenerateRandom = C_GenerateRandom, + .C_GetFunctionStatus = C_GetFunctionStatus, + .C_CancelFunction = C_CancelFunction, + .C_WaitForSlotEvent = C_WaitForSlotEvent, +}; + +static bool lib_initiated(void) +{ + return ckteec_invoke_initiated(); +} + +CK_RV C_Initialize(CK_VOID_PTR pInitArgs) +{ + CK_C_INITIALIZE_ARGS_PTR args = NULL; + CK_RV rv = 0; + + if (pInitArgs) { + args = (CK_C_INITIALIZE_ARGS_PTR)pInitArgs; + + /* Reserved must be set to NULL in this version of PKCS#11 */ + if (args->reserved) + return CKR_ARGUMENTS_BAD; + } + + rv = ckteec_invoke_init(); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CANT_LOCK, + CKR_CRYPTOKI_ALREADY_INITIALIZED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_NEED_TO_CREATE_THREADS, CKR_OK, + CKR_MUTEX_BAD); + + return rv; +} + +CK_RV C_Finalize(CK_VOID_PTR pReserved) +{ + CK_RV rv = 0; + + /* Reserved must be set to NULL in this version of PKCS#11 */ + if (pReserved) + return CKR_ARGUMENTS_BAD; + + rv = ckteec_invoke_terminate(); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK); + + return rv; +} + +CK_RV C_GetInfo(CK_INFO_PTR pInfo) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_get_info(pInfo); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK); + + return rv; +} + +CK_RV C_GetFunctionList(CK_FUNCTION_LIST_PTR_PTR ppFunctionList) +{ + if (!ppFunctionList) + return CKR_ARGUMENTS_BAD; + + /* Discard the const attribute when exporting the list address */ + *ppFunctionList = (void *)&libckteec_function_list; + + return CKR_OK; +} + +CK_RV C_GetSlotList(CK_BBOOL tokenPresent, + CK_SLOT_ID_PTR pSlotList, + CK_ULONG_PTR pulCount) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_slot_get_list(tokenPresent, pSlotList, pulCount); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK); + + return rv; +} + +CK_RV C_GetSlotInfo(CK_SLOT_ID slotID, + CK_SLOT_INFO_PTR pInfo) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_slot_get_info(slotID, pInfo); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, + CKR_HOST_MEMORY, CKR_OK, CKR_SLOT_ID_INVALID); + + return rv; +} + +CK_RV C_InitToken(CK_SLOT_ID slotID, + CK_UTF8CHAR_PTR pPin, + CK_ULONG ulPinLen, + CK_UTF8CHAR_PTR pLabel) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_init_token(slotID, pPin, ulPinLen, pLabel); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_PIN_INCORRECT, CKR_PIN_LOCKED, CKR_SESSION_EXISTS, + CKR_SLOT_ID_INVALID, CKR_TOKEN_NOT_PRESENT, + CKR_TOKEN_NOT_RECOGNIZED, CKR_TOKEN_WRITE_PROTECTED); + + return rv; +} + +CK_RV C_GetTokenInfo(CK_SLOT_ID slotID, + CK_TOKEN_INFO_PTR pInfo) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_token_get_info(slotID, pInfo); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_SLOT_ID_INVALID, CKR_TOKEN_NOT_PRESENT, + CKR_TOKEN_NOT_RECOGNIZED, CKR_ARGUMENTS_BAD); + + return rv; +} + +CK_RV C_GetMechanismList(CK_SLOT_ID slotID, + CK_MECHANISM_TYPE_PTR pMechanismList, + CK_ULONG_PTR pulCount) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_token_mechanism_ids(slotID, pMechanismList, pulCount); + + ASSERT_CK_RV(rv, CKR_BUFFER_TOO_SMALL, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_SLOT_ID_INVALID, CKR_TOKEN_NOT_PRESENT, + CKR_TOKEN_NOT_RECOGNIZED, CKR_ARGUMENTS_BAD); + + return rv; +} + +CK_RV C_GetMechanismInfo(CK_SLOT_ID slotID, + CK_MECHANISM_TYPE type, + CK_MECHANISM_INFO_PTR pInfo) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_token_mechanism_info(slotID, type, pInfo); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_MECHANISM_INVALID, + CKR_OK, CKR_SLOT_ID_INVALID, CKR_TOKEN_NOT_PRESENT, + CKR_TOKEN_NOT_RECOGNIZED, CKR_ARGUMENTS_BAD); + + return rv; +} + +CK_RV C_OpenSession(CK_SLOT_ID slotID, + CK_FLAGS flags, + CK_VOID_PTR pApplication, + CK_NOTIFY Notify, + CK_SESSION_HANDLE_PTR phSession) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_open_session(slotID, flags, pApplication, Notify, + phSession); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_SESSION_COUNT, CKR_SESSION_PARALLEL_NOT_SUPPORTED, + CKR_SESSION_READ_WRITE_SO_EXISTS, CKR_SLOT_ID_INVALID, + CKR_TOKEN_NOT_PRESENT, CKR_TOKEN_NOT_RECOGNIZED, + CKR_TOKEN_WRITE_PROTECTED, CKR_ARGUMENTS_BAD); + + return rv; +} + +CK_RV C_CloseSession(CK_SESSION_HANDLE hSession) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_close_session(hSession); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_CloseAllSessions(CK_SLOT_ID slotID) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_close_all_sessions(slotID); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_SLOT_ID_INVALID, CKR_TOKEN_NOT_PRESENT); + + return rv; +} + +CK_RV C_GetSessionInfo(CK_SESSION_HANDLE hSession, + CK_SESSION_INFO_PTR pInfo) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_get_session_info(hSession, pInfo); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_ARGUMENTS_BAD); + + return rv; +} + +CK_RV C_InitPIN(CK_SESSION_HANDLE hSession, + CK_UTF8CHAR_PTR pPin, + CK_ULONG ulPinLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_init_pin(hSession, pPin, ulPinLen); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_PIN_INVALID, CKR_PIN_LEN_RANGE, CKR_SESSION_CLOSED, + CKR_SESSION_READ_ONLY, CKR_SESSION_HANDLE_INVALID, + CKR_TOKEN_WRITE_PROTECTED, CKR_USER_NOT_LOGGED_IN, + CKR_ARGUMENTS_BAD); + + return rv; +} + +CK_RV C_SetPIN(CK_SESSION_HANDLE hSession, + CK_UTF8CHAR_PTR pOldPin, + CK_ULONG ulOldLen, + CK_UTF8CHAR_PTR pNewPin, + CK_ULONG ulNewLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_set_pin(hSession, pOldPin, ulOldLen, pNewPin, ulNewLen); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_PIN_INCORRECT, CKR_PIN_INVALID, + CKR_PIN_LEN_RANGE, CKR_PIN_LOCKED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SESSION_READ_ONLY, + CKR_TOKEN_WRITE_PROTECTED, CKR_ARGUMENTS_BAD); + + return rv; +} + +CK_RV C_Login(CK_SESSION_HANDLE hSession, + CK_USER_TYPE userType, + CK_UTF8CHAR_PTR pPin, + CK_ULONG ulPinLen) + +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_login(hSession, userType, pPin, ulPinLen); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_PIN_INCORRECT, + CKR_PIN_LOCKED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SESSION_READ_ONLY_EXISTS, + CKR_USER_ALREADY_LOGGED_IN, + CKR_USER_ANOTHER_ALREADY_LOGGED_IN, + CKR_USER_PIN_NOT_INITIALIZED, CKR_USER_TOO_MANY_TYPES, + CKR_USER_TYPE_INVALID); + + return rv; +} + +CK_RV C_Logout(CK_SESSION_HANDLE hSession) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_logout(hSession); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_GetOperationState(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pOperationState, + CK_ULONG_PTR pulOperationStateLen) +{ + (void)hSession; + (void)pOperationState; + (void)pulOperationStateLen; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_SetOperationState(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pOperationState, + CK_ULONG ulOperationStateLen, + CK_OBJECT_HANDLE hEncryptionKey, + CK_OBJECT_HANDLE hAuthenticationKey) +{ + (void)hSession; + (void)pOperationState; + (void)ulOperationStateLen; + (void)hEncryptionKey; + (void)hAuthenticationKey; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_CreateObject(CK_SESSION_HANDLE hSession, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phObject) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_create_object(hSession, pTemplate, ulCount, phObject); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_ATTRIBUTE_READ_ONLY, + CKR_ATTRIBUTE_TYPE_INVALID, CKR_ATTRIBUTE_VALUE_INVALID, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_CURVE_NOT_SUPPORTED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_DOMAIN_PARAMS_INVALID, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_PIN_EXPIRED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SESSION_READ_ONLY, + CKR_TEMPLATE_INCOMPLETE, CKR_TEMPLATE_INCONSISTENT, + CKR_TOKEN_WRITE_PROTECTED, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_CopyObject(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phNewObject) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_copy_object(hSession, hObject, pTemplate, ulCount, + phNewObject); + + ASSERT_CK_RV(rv, CKR_ACTION_PROHIBITED, CKR_ARGUMENTS_BAD, + CKR_ATTRIBUTE_READ_ONLY, CKR_ATTRIBUTE_TYPE_INVALID, + CKR_ATTRIBUTE_VALUE_INVALID, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OBJECT_HANDLE_INVALID, CKR_OK, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_SESSION_READ_ONLY, CKR_TEMPLATE_INCONSISTENT, + CKR_TOKEN_WRITE_PROTECTED, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_DestroyObject(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_destroy_object(hSession, hObject); + + ASSERT_CK_RV(rv, CKR_ACTION_PROHIBITED, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OBJECT_HANDLE_INVALID, CKR_OK, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_SESSION_READ_ONLY, CKR_TOKEN_WRITE_PROTECTED); + + return rv; +} + +CK_RV C_GetObjectSize(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ULONG_PTR pulSize) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_get_object_size(hSession, hObject, pulSize); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_INFORMATION_SENSITIVE, CKR_OBJECT_HANDLE_INVALID, + CKR_OK, CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_GetAttributeValue(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_get_attribute_value(hSession, hObject, pTemplate, ulCount); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_ATTRIBUTE_SENSITIVE, + CKR_ATTRIBUTE_TYPE_INVALID, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OBJECT_HANDLE_INVALID, CKR_OK, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_SetAttributeValue(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hObject, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_set_attribute_value(hSession, hObject, pTemplate, ulCount); + + ASSERT_CK_RV(rv, CKR_ACTION_PROHIBITED, CKR_ARGUMENTS_BAD, + CKR_ATTRIBUTE_READ_ONLY, CKR_ATTRIBUTE_TYPE_INVALID, + CKR_ATTRIBUTE_VALUE_INVALID, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OBJECT_HANDLE_INVALID, CKR_OK, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SESSION_READ_ONLY, + CKR_TEMPLATE_INCONSISTENT, CKR_TOKEN_WRITE_PROTECTED, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_FindObjectsInit(CK_SESSION_HANDLE hSession, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_find_objects_init(hSession, pTemplate, ulCount); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_ATTRIBUTE_TYPE_INVALID, + CKR_ATTRIBUTE_VALUE_INVALID, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_OPERATION_ACTIVE, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_FindObjects(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE_PTR phObject, + CK_ULONG ulMaxObjectCount, + CK_ULONG_PTR pulObjectCount) + +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_find_objects(hSession, phObject, + ulMaxObjectCount, pulObjectCount); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_FindObjectsFinal(CK_SESSION_HANDLE hSession) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_find_objects_final(hSession); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_EncryptInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_init(hSession, pMechanism, hKey, CK_FALSE); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_KEY_FUNCTION_NOT_PERMITTED, CKR_KEY_HANDLE_INVALID, + CKR_KEY_SIZE_RANGE, CKR_KEY_TYPE_INCONSISTENT, + CKR_MECHANISM_INVALID, CKR_MECHANISM_PARAM_INVALID, + CKR_OK, CKR_OPERATION_ACTIVE, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_Encrypt(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, + CK_ULONG ulDataLen, + CK_BYTE_PTR pEncryptedData, + CK_ULONG_PTR pulEncryptedDataLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_oneshot(hSession, pData, ulDataLen, + pEncryptedData, pulEncryptedDataLen, + CK_FALSE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DATA_LEN_RANGE, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_EncryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, + CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedData, + CK_ULONG_PTR pulEncryptedDataLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_update(hSession, pPart, ulPartLen, + pEncryptedData, + pulEncryptedDataLen, CK_FALSE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DATA_LEN_RANGE, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_EncryptFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pLastEncryptedPart, + CK_ULONG_PTR pulLastEncryptedPartLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_final(hSession, pLastEncryptedPart, + pulLastEncryptedPartLen, CK_FALSE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DATA_LEN_RANGE, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_DecryptInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_init(hSession, pMechanism, hKey, CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_KEY_FUNCTION_NOT_PERMITTED, CKR_KEY_HANDLE_INVALID, + CKR_KEY_SIZE_RANGE, CKR_KEY_TYPE_INCONSISTENT, + CKR_MECHANISM_INVALID, CKR_MECHANISM_PARAM_INVALID, + CKR_OK, CKR_OPERATION_ACTIVE, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_Decrypt(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedData, + CK_ULONG ulEncryptedDataLen, + CK_BYTE_PTR pData, + CK_ULONG_PTR pulDataLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_oneshot(hSession, pEncryptedData, + ulEncryptedDataLen, + pData, pulDataLen, CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_ENCRYPTED_DATA_INVALID, CKR_ENCRYPTED_DATA_LEN_RANGE, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_DecryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, + CK_ULONG_PTR pulPartLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_update(hSession, pEncryptedPart, + ulEncryptedPartLen, + pPart, pulPartLen, CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_ENCRYPTED_DATA_INVALID, CKR_ENCRYPTED_DATA_LEN_RANGE, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_DecryptFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pLastPart, + CK_ULONG_PTR pulLastPartLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_encdecrypt_final(hSession, pLastPart, pulLastPartLen, + CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_ENCRYPTED_DATA_INVALID, CKR_ENCRYPTED_DATA_LEN_RANGE, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_DigestInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_digest_init(hSession, pMechanism); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_MECHANISM_INVALID, + CKR_MECHANISM_PARAM_INVALID, CKR_OK, CKR_OPERATION_ACTIVE, + CKR_PIN_EXPIRED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_Digest(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, + CK_ULONG ulDataLen, + CK_BYTE_PTR pDigest, + CK_ULONG_PTR pulDigestLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_digest_oneshot(hSession, pData, ulDataLen, pDigest, + pulDigestLen); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_DigestUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, + CK_ULONG ulPartLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_digest_update(hSession, pPart, ulPartLen); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_DigestKey(CK_SESSION_HANDLE hSession, + CK_OBJECT_HANDLE hKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_digest_key(hSession, hKey); + + ASSERT_CK_RV(rv, CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_KEY_HANDLE_INVALID, CKR_KEY_INDIGESTIBLE, + CKR_KEY_SIZE_RANGE, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_DigestFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pDigest, + CK_ULONG_PTR pulDigestLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_digest_final(hSession, pDigest, pulDigestLen); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_SignInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_signverify_init(hSession, pMechanism, hKey, CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_KEY_FUNCTION_NOT_PERMITTED, CKR_KEY_HANDLE_INVALID, + CKR_KEY_SIZE_RANGE, CKR_KEY_TYPE_INCONSISTENT, + CKR_MECHANISM_INVALID, CKR_MECHANISM_PARAM_INVALID, + CKR_OK, CKR_OPERATION_ACTIVE, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_Sign(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, + CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, + CK_ULONG_PTR pulSignatureLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_signverify_oneshot(hSession, pData, ulDataLen, + pSignature, pulSignatureLen, + CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DATA_INVALID, + CKR_DATA_LEN_RANGE, CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, + CKR_DEVICE_REMOVED, CKR_FUNCTION_CANCELED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN, + CKR_FUNCTION_REJECTED); + + return rv; +} + +CK_RV C_SignUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, + CK_ULONG ulPartLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_signverify_update(hSession, pPart, ulPartLen, CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DATA_LEN_RANGE, CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, + CKR_DEVICE_REMOVED, CKR_FUNCTION_CANCELED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_SignFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, + CK_ULONG_PTR pulSignatureLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_signverify_final(hSession, pSignature, pulSignatureLen, + CK_TRUE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DATA_LEN_RANGE, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN, + CKR_FUNCTION_REJECTED); + + return rv; +} + +CK_RV C_SignRecoverInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey) +{ + (void)hSession; + (void)pMechanism; + (void)hKey; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_SignRecover(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, + CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, + CK_ULONG_PTR pulSignatureLen) +{ + (void)hSession; + (void)pData; + (void)ulDataLen; + (void)pSignature; + (void)pulSignatureLen; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_VerifyInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_signverify_init(hSession, pMechanism, hKey, CK_FALSE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_KEY_FUNCTION_NOT_PERMITTED, CKR_KEY_HANDLE_INVALID, + CKR_KEY_SIZE_RANGE, CKR_KEY_TYPE_INCONSISTENT, + CKR_MECHANISM_INVALID, CKR_MECHANISM_PARAM_INVALID, + CKR_OK, CKR_OPERATION_ACTIVE, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_Verify(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pData, + CK_ULONG ulDataLen, + CK_BYTE_PTR pSignature, + CK_ULONG ulSignatureLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + CK_ULONG out_size = ulSignatureLen; + + if (lib_initiated()) + rv = ck_signverify_oneshot(hSession, pData, ulDataLen, + pSignature, &out_size, + CK_FALSE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DATA_INVALID, CKR_DATA_LEN_RANGE, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SIGNATURE_INVALID, + CKR_SIGNATURE_LEN_RANGE); + + return rv; +} + +CK_RV C_VerifyUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, + CK_ULONG ulPartLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_signverify_update(hSession, pPart, ulPartLen, CK_FALSE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DATA_LEN_RANGE, CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, + CKR_DEVICE_REMOVED, CKR_FUNCTION_CANCELED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID); + + return rv; +} + +CK_RV C_VerifyFinal(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, + CK_ULONG ulSignatureLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_signverify_final(hSession, pSignature, &ulSignatureLen, + CK_FALSE); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DATA_LEN_RANGE, CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, + CKR_DEVICE_REMOVED, CKR_FUNCTION_CANCELED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_OK, CKR_OPERATION_NOT_INITIALIZED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SIGNATURE_INVALID, + CKR_SIGNATURE_LEN_RANGE); + + return rv; +} + +CK_RV C_VerifyRecoverInit(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hKey) +{ + (void)hSession; + (void)pMechanism; + (void)hKey; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_VerifyRecover(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSignature, + CK_ULONG ulSignatureLen, + CK_BYTE_PTR pData, + CK_ULONG_PTR pulDataLen) +{ + (void)hSession; + (void)pSignature; + (void)ulSignatureLen; + (void)pData; + (void)pulDataLen; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_DigestEncryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, + CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG_PTR pulEncryptedPartLen) +{ + (void)hSession; + (void)pPart; + (void)ulPartLen; + (void)pEncryptedPart; + (void)pulEncryptedPartLen; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_DecryptDigestUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, + CK_ULONG_PTR pulPartLen) +{ + (void)hSession; + (void)pEncryptedPart; + (void)ulEncryptedPartLen; + (void)pPart; + (void)pulPartLen; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_SignEncryptUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pPart, + CK_ULONG ulPartLen, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG_PTR pulEncryptedPartLen) +{ + (void)hSession; + (void)pPart; + (void)ulPartLen; + (void)pEncryptedPart; + (void)pulEncryptedPartLen; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_DecryptVerifyUpdate(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pEncryptedPart, + CK_ULONG ulEncryptedPartLen, + CK_BYTE_PTR pPart, + CK_ULONG_PTR pulPartLen) +{ + (void)hSession; + (void)pEncryptedPart; + (void)ulEncryptedPartLen; + (void)pPart; + (void)pulPartLen; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} + +CK_RV C_GenerateKey(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_generate_key(hSession, pMechanism, pTemplate, ulCount, + phKey); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_ATTRIBUTE_READ_ONLY, + CKR_ATTRIBUTE_TYPE_INVALID, CKR_ATTRIBUTE_VALUE_INVALID, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_CURVE_NOT_SUPPORTED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_MECHANISM_INVALID, + CKR_MECHANISM_PARAM_INVALID, CKR_OK, CKR_OPERATION_ACTIVE, + CKR_PIN_EXPIRED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SESSION_READ_ONLY, + CKR_TEMPLATE_INCOMPLETE, CKR_TEMPLATE_INCONSISTENT, + CKR_TOKEN_WRITE_PROTECTED, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_GenerateKeyPair(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_ATTRIBUTE_PTR pPublicKeyTemplate, + CK_ULONG ulPublicKeyAttributeCount, + CK_ATTRIBUTE_PTR pPrivateKeyTemplate, + CK_ULONG ulPrivateKeyAttributeCount, + CK_OBJECT_HANDLE_PTR phPublicKey, + CK_OBJECT_HANDLE_PTR phPrivateKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_generate_key_pair(hSession, pMechanism, + pPublicKeyTemplate, + ulPublicKeyAttributeCount, + pPrivateKeyTemplate, + ulPrivateKeyAttributeCount, + phPublicKey, phPrivateKey); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_ATTRIBUTE_READ_ONLY, + CKR_ATTRIBUTE_TYPE_INVALID, CKR_ATTRIBUTE_VALUE_INVALID, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_CURVE_NOT_SUPPORTED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_DOMAIN_PARAMS_INVALID, CKR_FUNCTION_CANCELED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_MECHANISM_INVALID, CKR_MECHANISM_PARAM_INVALID, + CKR_OK, CKR_OPERATION_ACTIVE, CKR_PIN_EXPIRED, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_SESSION_READ_ONLY, CKR_TEMPLATE_INCOMPLETE, + CKR_TEMPLATE_INCONSISTENT, CKR_TOKEN_WRITE_PROTECTED, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_WrapKey(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hWrappingKey, + CK_OBJECT_HANDLE hKey, + CK_BYTE_PTR pWrappedKey, + CK_ULONG_PTR pulWrappedKeyLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_wrap_key(hSession, pMechanism, hWrappingKey, hKey, + pWrappedKey, pulWrappedKeyLen); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_BUFFER_TOO_SMALL, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_KEY_HANDLE_INVALID, + CKR_KEY_NOT_WRAPPABLE, CKR_KEY_SIZE_RANGE, + CKR_KEY_UNEXTRACTABLE, CKR_MECHANISM_INVALID, + CKR_MECHANISM_PARAM_INVALID, CKR_OK, CKR_OPERATION_ACTIVE, + CKR_PIN_EXPIRED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN, + CKR_WRAPPING_KEY_HANDLE_INVALID, + CKR_WRAPPING_KEY_SIZE_RANGE, + CKR_WRAPPING_KEY_TYPE_INCONSISTENT); + + return rv; +} + +CK_RV C_UnwrapKey(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hUnwrappingKey, + CK_BYTE_PTR pWrappedKey, + CK_ULONG ulWrappedKeyLen, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_unwrap_key(hSession, pMechanism, hUnwrappingKey, + pWrappedKey, ulWrappedKeyLen, pTemplate, + ulCount, phKey); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_ATTRIBUTE_READ_ONLY, + CKR_ATTRIBUTE_TYPE_INVALID, CKR_ATTRIBUTE_VALUE_INVALID, + CKR_BUFFER_TOO_SMALL, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_CURVE_NOT_SUPPORTED, CKR_DEVICE_ERROR, + CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_DOMAIN_PARAMS_INVALID, CKR_FUNCTION_CANCELED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_MECHANISM_INVALID, CKR_MECHANISM_PARAM_INVALID, CKR_OK, + CKR_OPERATION_ACTIVE, CKR_PIN_EXPIRED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SESSION_READ_ONLY, + CKR_TEMPLATE_INCOMPLETE, CKR_TEMPLATE_INCONSISTENT, + CKR_TOKEN_WRITE_PROTECTED, + CKR_UNWRAPPING_KEY_HANDLE_INVALID, + CKR_UNWRAPPING_KEY_SIZE_RANGE, + CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT, + CKR_USER_NOT_LOGGED_IN, CKR_WRAPPED_KEY_INVALID, + CKR_WRAPPED_KEY_LEN_RANGE); + + return rv; +} + +CK_RV C_DeriveKey(CK_SESSION_HANDLE hSession, + CK_MECHANISM_PTR pMechanism, + CK_OBJECT_HANDLE hBaseKey, + CK_ATTRIBUTE_PTR pTemplate, + CK_ULONG ulCount, + CK_OBJECT_HANDLE_PTR phKey) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_derive_key(hSession, pMechanism, hBaseKey, pTemplate, + ulCount, phKey); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_ATTRIBUTE_READ_ONLY, + CKR_ATTRIBUTE_TYPE_INVALID, CKR_ATTRIBUTE_VALUE_INVALID, + CKR_CRYPTOKI_NOT_INITIALIZED, CKR_CURVE_NOT_SUPPORTED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_DOMAIN_PARAMS_INVALID, CKR_FUNCTION_CANCELED, + CKR_FUNCTION_FAILED, CKR_GENERAL_ERROR, CKR_HOST_MEMORY, + CKR_KEY_HANDLE_INVALID, CKR_KEY_SIZE_RANGE, + CKR_KEY_TYPE_INCONSISTENT, CKR_MECHANISM_INVALID, + CKR_MECHANISM_PARAM_INVALID, CKR_OK, CKR_OPERATION_ACTIVE, + CKR_PIN_EXPIRED, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_SESSION_READ_ONLY, + CKR_TEMPLATE_INCOMPLETE, CKR_TEMPLATE_INCONSISTENT, + CKR_TOKEN_WRITE_PROTECTED, CKR_USER_NOT_LOGGED_IN, + CKR_DATA_LEN_RANGE); + + return rv; +} + +CK_RV C_SeedRandom(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pSeed, + CK_ULONG ulSeedLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_seed_random(hSession, pSeed, ulSeedLen); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_ACTIVE, CKR_RANDOM_SEED_NOT_SUPPORTED, + CKR_RANDOM_NO_RNG, CKR_SESSION_CLOSED, + CKR_SESSION_HANDLE_INVALID, CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_GenerateRandom(CK_SESSION_HANDLE hSession, + CK_BYTE_PTR pRandomData, + CK_ULONG ulRandomLen) +{ + CK_RV rv = CKR_CRYPTOKI_NOT_INITIALIZED; + + if (lib_initiated()) + rv = ck_generate_random(hSession, pRandomData, ulRandomLen); + + ASSERT_CK_RV(rv, CKR_ARGUMENTS_BAD, CKR_CRYPTOKI_NOT_INITIALIZED, + CKR_DEVICE_ERROR, CKR_DEVICE_MEMORY, CKR_DEVICE_REMOVED, + CKR_FUNCTION_CANCELED, CKR_FUNCTION_FAILED, + CKR_GENERAL_ERROR, CKR_HOST_MEMORY, CKR_OK, + CKR_OPERATION_ACTIVE, CKR_RANDOM_NO_RNG, + CKR_SESSION_CLOSED, CKR_SESSION_HANDLE_INVALID, + CKR_USER_NOT_LOGGED_IN); + + return rv; +} + +CK_RV C_GetFunctionStatus(CK_SESSION_HANDLE hSession) +{ + (void)hSession; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_PARALLEL; +} + +CK_RV C_CancelFunction(CK_SESSION_HANDLE hSession) +{ + (void)hSession; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_PARALLEL; +} + +CK_RV C_WaitForSlotEvent(CK_FLAGS flags, + CK_SLOT_ID_PTR slotID, + CK_VOID_PTR pReserved) +{ + (void)flags; + (void)slotID; + (void)pReserved; + + if (!lib_initiated()) + return CKR_CRYPTOKI_NOT_INITIALIZED; + + return CKR_FUNCTION_NOT_SUPPORTED; +} diff --git a/optee_client/libckteec/src/pkcs11_processing.c b/optee_client/libckteec/src/pkcs11_processing.c new file mode 100644 index 0000000..bb26c1f --- /dev/null +++ b/optee_client/libckteec/src/pkcs11_processing.c @@ -0,0 +1,1705 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2018, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "pkcs11_processing.h" +#include "invoke_ta.h" +#include "serializer.h" +#include "serialize_ck.h" + +CK_RV ck_create_object(CK_SESSION_HANDLE session, CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, CK_OBJECT_HANDLE_PTR handle) +{ + CK_RV rv = CKR_GENERAL_ERROR; + struct serializer obj = { 0 }; + size_t ctrl_size = 0; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + uint32_t session_handle = session; + uint32_t key_handle = 0; + char *buf = NULL; + size_t out_size = 0; + + if (!handle || !attribs || !count) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_attributes(&obj, attribs, count); + if (rv) + goto out; + + /* Shm io0: (i/o) [session-handle][serialized-attributes] / [status] */ + ctrl_size = sizeof(session_handle) + obj.size; + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, obj.buffer, obj.size); + + /* Shm io2: (out) [object handle] */ + out_shm = ckteec_alloc_shm(sizeof(key_handle), CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_CREATE_OBJECT, + ctrl, out_shm, &out_size); + + if (rv != CKR_OK || out_size != out_shm->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto out; + } + + memcpy(&key_handle, out_shm->buffer, sizeof(key_handle)); + *handle = key_handle; + +out: + release_serial_object(&obj); + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_destroy_object(CK_SESSION_HANDLE session, CK_OBJECT_HANDLE obj) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + size_t ctrl_size = 0; + char *buf = NULL; + uint32_t session_handle = session; + uint32_t obj_id = obj; + + /* Shm io0: (i/o) ctrl = [session-handle][object-handle] / [status] */ + ctrl_size = sizeof(session_handle) + sizeof(obj_id); + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &obj_id, sizeof(obj_id)); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_DESTROY_OBJECT, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_encdecrypt_init(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE key, + int decrypt) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + struct serializer obj = { 0 }; + uint32_t session_handle = session; + uint32_t key_handle = key; + size_t ctrl_size = 0; + char *buf = NULL; + + if (!mechanism) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_mecha_params(&obj, mechanism); + if (rv) + return rv; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][key-handle][serialized-mechanism-blob] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + sizeof(key_handle) + obj.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &key_handle, sizeof(key_handle)); + buf += sizeof(key_handle); + + memcpy(buf, obj.buffer, obj.size); + + rv = ckteec_invoke_ctrl(decrypt ? PKCS11_CMD_DECRYPT_INIT : + PKCS11_CMD_ENCRYPT_INIT, ctrl); + +bail: + ckteec_free_shm(ctrl); + release_serial_object(&obj); + + return rv; +} + +CK_RV ck_encdecrypt_update(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int decrypt) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *in_shm = NULL; + TEEC_SharedMemory *out_shm = NULL; + uint32_t session_handle = session; + size_t out_size = 0; + + if (!out_len || (in_len && !in)) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io1: input data buffer if any */ + if (in_len) { + in_shm = ckteec_register_shm(in, in_len, CKTEEC_SHM_IN); + if (!in_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + } + + /* Shm io2: output data buffer */ + if (out && out_len && *out_len) { + out_shm = ckteec_register_shm(out, *out_len, CKTEEC_SHM_OUT); + } else { + /* Query output data size */ + out_shm = ckteec_alloc_shm(0, CKTEEC_SHM_OUT); + } + + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + /* Invoke */ + rv = ckteec_invoke_ta(decrypt ? PKCS11_CMD_DECRYPT_UPDATE : + PKCS11_CMD_ENCRYPT_UPDATE, ctrl, + in_shm, out_shm, &out_size, NULL, NULL); + + if (out_len && (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)) + *out_len = out_size; + + if (rv == CKR_BUFFER_TOO_SMALL && out_size && !out) + rv = CKR_OK; + + goto bail_invoked; + +bail: + ck_release_active_processing(session, decrypt ? + PKCS11_CMD_DECRYPT_UPDATE : + PKCS11_CMD_ENCRYPT_UPDATE); +bail_invoked: + ckteec_free_shm(out_shm); + ckteec_free_shm(in_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_encdecrypt_oneshot(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int decrypt) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *in_shm = NULL; + TEEC_SharedMemory *out_shm = NULL; + uint32_t session_handle = session; + size_t out_size = 0; + + if (!out_len || (in_len && !in)) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io1: input data buffer */ + in_shm = ckteec_register_shm(in, in_len, CKTEEC_SHM_IN); + if (!in_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + /* Shm io2: output data buffer */ + if (out && out_len && *out_len) { + out_shm = ckteec_register_shm(out, *out_len, CKTEEC_SHM_OUT); + } else { + /* Query output data size */ + out_shm = ckteec_alloc_shm(0, CKTEEC_SHM_OUT); + } + + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ta(decrypt ? PKCS11_CMD_DECRYPT_ONESHOT : + PKCS11_CMD_ENCRYPT_ONESHOT, ctrl, + in_shm, out_shm, &out_size, NULL, NULL); + + if (out_len && (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)) + *out_len = out_size; + + if (rv == CKR_BUFFER_TOO_SMALL && out_size && !out) + rv = CKR_OK; + + goto bail_invoked; + +bail: + ck_release_active_processing(session, decrypt ? + PKCS11_CMD_DECRYPT_ONESHOT : + PKCS11_CMD_ENCRYPT_ONESHOT); +bail_invoked: + ckteec_free_shm(out_shm); + ckteec_free_shm(in_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_encdecrypt_final(CK_SESSION_HANDLE session, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int decrypt) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + uint32_t session_handle = session; + size_t out_size = 0; + + if (!out_len) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io2: output buffer reference */ + if (out && out_len && *out_len) { + out_shm = ckteec_register_shm(out, *out_len, CKTEEC_SHM_OUT); + } else { + /* Query output data size */ + out_shm = ckteec_alloc_shm(0, CKTEEC_SHM_OUT); + } + + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(decrypt ? PKCS11_CMD_DECRYPT_FINAL : + PKCS11_CMD_ENCRYPT_FINAL, + ctrl, out_shm, &out_size); + + if (out_len && (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)) + *out_len = out_size; + + if (rv == CKR_BUFFER_TOO_SMALL && out_size && !out) + rv = CKR_OK; + + goto bail_invoked; + +bail: + ck_release_active_processing(session, decrypt ? + PKCS11_CMD_DECRYPT_FINAL : + PKCS11_CMD_ENCRYPT_FINAL); +bail_invoked: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_digest_init(CK_SESSION_HANDLE session, CK_MECHANISM_PTR mechanism) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + struct serializer obj = { 0 }; + uint32_t session_handle = session; + size_t ctrl_size = 0; + uint8_t *buf = NULL; + + if (!mechanism) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_mecha_params(&obj, mechanism); + if (rv) + return rv; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][serialized-mechanism-blob] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + obj.size; + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, obj.buffer, obj.size); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_DIGEST_INIT, ctrl); + +bail: + ckteec_free_shm(ctrl); + release_serial_object(&obj); + + return rv; +} + +CK_RV ck_digest_key(CK_SESSION_HANDLE session, CK_OBJECT_HANDLE key) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t session_handle = session; + uint32_t key_handle = key; + size_t ctrl_size = 0; + uint8_t *buf = NULL; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][key-handle] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + sizeof(key_handle); + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &key_handle, sizeof(key_handle)); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_DIGEST_KEY, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_digest_update(CK_SESSION_HANDLE session, CK_BYTE_PTR in, + CK_ULONG in_len) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *io1 = NULL; + uint32_t session_handle = session; + uint8_t *buf = NULL; + + if (!in && in_len) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* + * Shm io0: (in/out) ctrl = [session-handle] / [status] + */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + + /* Shm io1: input payload */ + if (in_len && in) { + io1 = ckteec_register_shm(in, in_len, CKTEEC_SHM_IN); + if (!io1) { + rv = CKR_HOST_MEMORY; + goto bail; + } + } + + rv = ckteec_invoke_ctrl_in(PKCS11_CMD_DIGEST_UPDATE, ctrl, io1); + + goto bail_invoked; + +bail: + ck_release_active_processing(session, PKCS11_CMD_DIGEST_UPDATE); +bail_invoked: + ckteec_free_shm(io1); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_digest_oneshot(CK_SESSION_HANDLE session, CK_BYTE_PTR in, + CK_ULONG in_len, CK_BYTE_PTR digest_ref, + CK_ULONG_PTR digest_len) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *io1 = NULL; + TEEC_SharedMemory *io2 = NULL; + uint32_t session_handle = session; + size_t out_size = 0; + uint8_t *buf = NULL; + + if ((!in && in_len) || !digest_len) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* + * Shm io0: (in/out) ctrl = [session-handle] / [status] + */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + + /* Shm io1: input payload */ + if (in_len && in) { + io1 = ckteec_register_shm(in, in_len, CKTEEC_SHM_IN); + if (!io1) { + rv = CKR_HOST_MEMORY; + goto bail; + } + } + + /* Shm io2: output digest or null sized shm */ + if (digest_ref) { + io2 = ckteec_register_shm(digest_ref, *digest_len, + CKTEEC_SHM_OUT); + } else { + /* Query size if output signature */ + io2 = ckteec_alloc_shm(0, CKTEEC_SHM_OUT); + } + + if (!io2) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ta(PKCS11_CMD_DIGEST_ONESHOT, ctrl, + io1, io2, &out_size, NULL, NULL); + + if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) + *digest_len = out_size; + + if (rv == CKR_BUFFER_TOO_SMALL && out_size && !digest_ref) + rv = CKR_OK; + + goto bail_invoked; + +bail: + ck_release_active_processing(session, PKCS11_CMD_DIGEST_ONESHOT); +bail_invoked: + ckteec_free_shm(io1); + ckteec_free_shm(io2); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_digest_final(CK_SESSION_HANDLE session, CK_BYTE_PTR digest_ref, + CK_ULONG_PTR digest_len) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *io2 = NULL; + uint32_t session_handle = session; + size_t io2_size = 0; + uint8_t *buf = NULL; + + /* + * - If digest_ref == NULL AND digest_len == NULL -> need to call TA to + * terminate session. + * - If digest_len == NULL -> need to call to TA to terminate session. + * - If digest_ref == NULL BUT digest_len != NULL just operate normally + * to get size of the required buffer + */ + if (!digest_len) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* + * Shm io0: (in/out) ctrl = [session-handle] / [status] + */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + + /* Shm io2: output digest or null sized shm */ + if (digest_ref) + io2 = ckteec_register_shm(digest_ref, *digest_len, + CKTEEC_SHM_OUT); + else + io2 = ckteec_alloc_shm(0, CKTEEC_SHM_OUT); + + if (!io2) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ta(PKCS11_CMD_DIGEST_FINAL, ctrl, NULL, io2, + &io2_size, NULL, NULL); + + if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) + *digest_len = io2_size; + + if (rv == CKR_BUFFER_TOO_SMALL && io2_size && !digest_ref) + rv = CKR_OK; + + goto bail_invoked; + +bail: + ck_release_active_processing(session, PKCS11_CMD_DIGEST_FINAL); +bail_invoked: + ckteec_free_shm(io2); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_signverify_init(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE key, + int sign) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + struct serializer obj = { 0 }; + uint32_t session_handle = session; + uint32_t key_handle = key; + size_t ctrl_size = 0; + char *buf = NULL; + + if (!mechanism) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_mecha_params(&obj, mechanism); + if (rv) + return rv; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][key-handle][serialized-mechanism-blob] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + sizeof(key_handle) + obj.size; + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &key_handle, sizeof(key_handle)); + buf += sizeof(key_handle); + + memcpy(buf, obj.buffer, obj.size); + + rv = ckteec_invoke_ctrl(sign ? PKCS11_CMD_SIGN_INIT : + PKCS11_CMD_VERIFY_INIT, ctrl); + +bail: + ckteec_free_shm(ctrl); + release_serial_object(&obj); + + return rv; +} + +CK_RV ck_signverify_update(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + int sign) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *in_shm = NULL; + uint32_t session_handle = session; + + if (!in && in_len) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io1: input data */ + in_shm = ckteec_register_shm(in, in_len, CKTEEC_SHM_IN); + if (!in_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_in(sign ? PKCS11_CMD_SIGN_UPDATE : + PKCS11_CMD_VERIFY_UPDATE, ctrl, in_shm); + + goto bail_invoked; + +bail: + ck_release_active_processing(session, sign ? PKCS11_CMD_SIGN_UPDATE : + PKCS11_CMD_VERIFY_UPDATE); +bail_invoked: + ckteec_free_shm(in_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_signverify_oneshot(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + CK_BYTE_PTR sign_ref, + CK_ULONG_PTR sign_len, + int sign) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *io1 = NULL; + TEEC_SharedMemory *io2 = NULL; + uint32_t session_handle = session; + size_t out_size = 0; + + if ((in_len && !in) || (!sign && sign_len && *sign_len && !sign_ref) || + (sign && !sign_len)) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io1: input payload */ + if (in_len) { + io1 = ckteec_register_shm(in, in_len, CKTEEC_SHM_IN); + if (!io1) { + rv = CKR_HOST_MEMORY; + goto bail; + } + } + + /* + * Shm io2: input signature (if verifying) or null sized shm + * or + * Shm io2: output signature (if signing) or null sized shm + */ + if (sign_ref && sign_len && *sign_len) { + io2 = ckteec_register_shm(sign_ref, *sign_len, + sign ? CKTEEC_SHM_OUT : CKTEEC_SHM_IN); + } else { + /* Query size if output signature */ + io2 = ckteec_alloc_shm(0, sign ? CKTEEC_SHM_OUT : CKTEEC_SHM_IN); + } + + if (!io2) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ta(sign ? PKCS11_CMD_SIGN_ONESHOT : + PKCS11_CMD_VERIFY_ONESHOT, ctrl, + io1, io2, &out_size, NULL, NULL); + + if (sign && (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)) + *sign_len = out_size; + + if (rv == CKR_BUFFER_TOO_SMALL && out_size && !sign_ref) + rv = CKR_OK; + + goto bail_invoked; + +bail: + ck_release_active_processing(session, sign ? PKCS11_CMD_SIGN_ONESHOT : + PKCS11_CMD_VERIFY_ONESHOT); +bail_invoked: + ckteec_free_shm(io1); + ckteec_free_shm(io2); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_signverify_final(CK_SESSION_HANDLE session, + CK_BYTE_PTR sign_ref, + CK_ULONG_PTR sign_len, + int sign) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *io = NULL; + uint32_t session_handle = session; + size_t io_size = 0; + + if ((!sign && sign_len && *sign_len && !sign_ref) || + (sign && !sign_len)) { + rv = CKR_ARGUMENTS_BAD; + goto bail; + } + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* + * Shm io1: input signature (if verifying) or null sized shm + * or + * Shm io1: output signature (if signing) or null sized shm + */ + if (sign_ref && sign_len && *sign_len) + io = ckteec_register_shm(sign_ref, *sign_len, + sign ? CKTEEC_SHM_OUT : CKTEEC_SHM_IN); + else + io = ckteec_alloc_shm(0, sign ? CKTEEC_SHM_OUT : CKTEEC_SHM_IN); + + if (!io) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ta(sign ? PKCS11_CMD_SIGN_FINAL : + PKCS11_CMD_VERIFY_FINAL, ctrl, NULL, io, + &io_size, NULL, NULL); + + if (sign && sign_len && (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)) + *sign_len = io_size; + + if (rv == CKR_BUFFER_TOO_SMALL && io_size && !sign_ref) + rv = CKR_OK; + + goto bail_invoked; + +bail: + ck_release_active_processing(session, sign ? PKCS11_CMD_SIGN_FINAL : + PKCS11_CMD_VERIFY_FINAL); +bail_invoked: + ckteec_free_shm(io); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_generate_key(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, + CK_OBJECT_HANDLE_PTR handle) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + struct serializer smecha = { 0 }; + struct serializer sattr = { 0 }; + uint32_t session_handle = session; + size_t ctrl_size = 0; + uint32_t key_handle = 0; + char *buf = NULL; + size_t out_size = 0; + + if (!handle || !mechanism || (count && !attribs)) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_mecha_params(&smecha, mechanism); + if (rv) + return rv; + + rv = serialize_ck_attributes(&sattr, attribs, count); + if (rv) + goto bail; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][serialized-mecha][serialized-attributes] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + smecha.size + sattr.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, smecha.buffer, smecha.size); + buf += smecha.size; + + memcpy(buf, sattr.buffer, sattr.size); + + /* Shm io2: (out) [object handle] */ + out_shm = ckteec_alloc_shm(sizeof(key_handle), CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_GENERATE_KEY, + ctrl, out_shm, &out_size); + + if (rv != CKR_OK || out_size != out_shm->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto bail; + } + + memcpy(&key_handle, out_shm->buffer, sizeof(key_handle)); + *handle = key_handle; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + release_serial_object(&sattr); + release_serial_object(&smecha); + + return rv; +} + +CK_RV ck_find_objects_init(CK_SESSION_HANDLE session, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t session_handle = session; + struct serializer obj = { 0 }; + size_t ctrl_size = 0; + char *buf = NULL; + + if (count && !attribs) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_attributes(&obj, attribs, count); + if (rv) + return rv; + + /* Shm io0: (in/out) ctrl + * (in) [session-handle][headed-serialized-attributes] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + obj.size; + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, obj.buffer, obj.size); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_FIND_OBJECTS_INIT, ctrl); + +bail: + ckteec_free_shm(ctrl); + release_serial_object(&obj); + + return rv; +} + +CK_RV ck_find_objects(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE_PTR obj, + CK_ULONG max_count, + CK_ULONG_PTR count) + +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + uint32_t session_handle = session; + uint32_t *handles = NULL; + size_t handles_size = max_count * sizeof(uint32_t); + CK_ULONG n = 0; + CK_ULONG last = 0; + size_t out_size = 0; + + if (!count || (max_count && !obj)) + return CKR_ARGUMENTS_BAD; + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io2: (out) [object handle list] */ + out_shm = ckteec_alloc_shm(handles_size, CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_FIND_OBJECTS, + ctrl, out_shm, &out_size); + + if (rv || out_size > out_shm->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto bail; + } + + handles = out_shm->buffer; + last = out_size / sizeof(uint32_t); + *count = last; + + for (n = 0; n < last; n++) + obj[n] = handles[n]; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_find_objects_final(CK_SESSION_HANDLE session) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t session_handle = session; + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_FIND_OBJECTS_FINAL, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_get_object_size(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ULONG_PTR p_size) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + size_t ctrl_size = 0; + uint32_t session_handle = session; + uint32_t obj_handle = obj; + char *buf = NULL; + size_t out_size = 0; + uint32_t u32_sz = 0; + + if (!p_size) + return CKR_ARGUMENTS_BAD; + + /* Shm io0: (in/out) [session][obj-handle] / [status] */ + ctrl_size = sizeof(session_handle) + sizeof(obj_handle); + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &obj_handle, sizeof(obj_handle)); + + /* Shm io2: (out) [object size] */ + out_shm = ckteec_alloc_shm(sizeof(uint32_t), CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_GET_OBJECT_SIZE, + ctrl, out_shm, &out_size); + if (rv) + goto bail; + + if (out_shm->size != sizeof(uint32_t)) { + rv = CKR_DEVICE_ERROR; + goto bail; + } + + memcpy(&u32_sz, out_shm->buffer, out_shm->size); + *p_size = u32_sz; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_get_attribute_value(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count) +{ + CK_RV rv = CKR_GENERAL_ERROR; + CK_RV rv2 = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + struct serializer sattr = { 0 }; + size_t ctrl_size = 0; + uint32_t session_handle = session; + uint32_t obj_handle = obj; + char *buf = NULL; + size_t out_size = 0; + + if (count && !attribs) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_attributes(&sattr, attribs, count); + if (rv) + goto bail; + + /* Shm io0: (in/out) [session][obj-handle][attributes] / [status] */ + ctrl_size = sizeof(session_handle) + sizeof(obj_handle) + sattr.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &obj_handle, sizeof(obj_handle)); + buf += sizeof(obj_handle); + + memcpy(buf, sattr.buffer, sattr.size); + + /* Shm io2: (out) [attributes] */ + out_shm = ckteec_alloc_shm(sattr.size, CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_GET_ATTRIBUTE_VALUE, + ctrl, out_shm, &out_size); + + if (rv == CKR_OK || rv == CKR_ATTRIBUTE_SENSITIVE || + rv == CKR_ATTRIBUTE_TYPE_INVALID || rv == CKR_BUFFER_TOO_SMALL) { + rv2 = deserialize_ck_attributes(out_shm->buffer, attribs, + count); + if (rv2) + rv = CKR_GENERAL_ERROR; + } + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + release_serial_object(&sattr); + + return rv; +} + +CK_RV ck_set_attribute_value(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + struct serializer sattr = { 0 }; + size_t ctrl_size = 0; + uint32_t session_handle = session; + uint32_t obj_handle = obj; + char *buf = NULL; + + if (count && !attribs) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_attributes(&sattr, attribs, count); + if (rv) + goto bail; + + /* Shm io0: (in/out) [session][obj-handle][attributes] / [status] */ + ctrl_size = sizeof(session_handle) + sizeof(obj_handle) + sattr.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &obj_handle, sizeof(obj_handle)); + buf += sizeof(obj_handle); + + memcpy(buf, sattr.buffer, sattr.size); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_SET_ATTRIBUTE_VALUE, ctrl); + +bail: + ckteec_free_shm(ctrl); + release_serial_object(&sattr); + + return rv; +} + +CK_RV ck_copy_object(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, + CK_OBJECT_HANDLE_PTR handle) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + struct serializer sattr = { 0 }; + size_t ctrl_size = 0; + uint32_t session_handle = session; + uint32_t obj_handle = obj; + uint32_t key_handle = 0; + char *buf = NULL; + size_t out_size = 0; + + if (!handle || (count && !attribs)) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_attributes(&sattr, attribs, count); + if (rv) + goto bail; + + /* Shm io0: (in/out) [session][obj-handle][attributes] / [status] */ + ctrl_size = sizeof(session_handle) + sizeof(obj_handle) + sattr.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &obj_handle, sizeof(obj_handle)); + buf += sizeof(obj_handle); + + memcpy(buf, sattr.buffer, sattr.size); + + /* Shm io2: (out) [object handle] */ + out_shm = ckteec_alloc_shm(sizeof(key_handle), CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_COPY_OBJECT, + ctrl, out_shm, &out_size); + if (rv != CKR_OK || out_size != out_shm->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto bail; + } + + memcpy(&key_handle, out_shm->buffer, sizeof(key_handle)); + *handle = key_handle; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + release_serial_object(&sattr); + + return rv; +} + +CK_RV ck_derive_key(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE parent_key, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, + CK_OBJECT_HANDLE_PTR handle) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + struct serializer smecha = { 0 }; + struct serializer sattr = { 0 }; + uint32_t session_handle = session; + uint32_t obj_handle = parent_key; + size_t ctrl_size = 0; + uint32_t key_handle = 0; + char *buf = NULL; + size_t out_size = 0; + + if (!handle || !mechanism || (count && !attribs)) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_mecha_params(&smecha, mechanism); + if (rv) + return rv; + + rv = serialize_ck_attributes(&sattr, attribs, count); + if (rv) + goto bail; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][obj-handle][serialized-mecha][serialized-attributes] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + sizeof(obj_handle) + smecha.size + + sattr.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &obj_handle, sizeof(obj_handle)); + buf += sizeof(obj_handle); + + memcpy(buf, smecha.buffer, smecha.size); + buf += smecha.size; + + memcpy(buf, sattr.buffer, sattr.size); + + /* Shm io2: (out) [object handle] */ + out_shm = ckteec_alloc_shm(sizeof(key_handle), CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_DERIVE_KEY, + ctrl, out_shm, &out_size); + + if (rv != CKR_OK || out_size != out_shm->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto bail; + } + + memcpy(&key_handle, out_shm->buffer, sizeof(key_handle)); + *handle = key_handle; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + release_serial_object(&sattr); + release_serial_object(&smecha); + + return rv; +} + +CK_RV ck_release_active_processing(CK_SESSION_HANDLE session, + enum pkcs11_ta_cmd command) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t session_handle = session; + uint32_t cmd = command; + size_t ctrl_size = 0; + char *buf = NULL; + + /* Shm io0: (in/out) ctrl = [session-handle][command] / [status] */ + ctrl_size = sizeof(session_handle) + sizeof(cmd); + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &cmd, sizeof(cmd)); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_RELEASE_ACTIVE_PROCESSING, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_generate_key_pair(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_ATTRIBUTE_PTR pub_attribs, + CK_ULONG pub_count, + CK_ATTRIBUTE_PTR priv_attribs, + CK_ULONG priv_count, + CK_OBJECT_HANDLE_PTR pub_key, + CK_OBJECT_HANDLE_PTR priv_key) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + struct serializer smecha = { 0 }; + struct serializer pub_sattr = { 0 }; + struct serializer priv_sattr = { 0 }; + uint32_t session_handle = session; + size_t ctrl_size = 0; + uint32_t *key_handle = NULL; + size_t key_handle_size = 2 * sizeof(*key_handle); + char *buf = NULL; + size_t out_size = 0; + + if (!(mechanism && pub_attribs && priv_attribs && pub_key && priv_key)) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_mecha_params(&smecha, mechanism); + if (rv) + return rv; + + rv = serialize_ck_attributes(&pub_sattr, pub_attribs, pub_count); + if (rv) + goto bail; + + rv = serialize_ck_attributes(&priv_sattr, priv_attribs, priv_count); + if (rv) + goto bail; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][serialized-mecha][serialized-pub_attribs] + * [serialized-priv_attribs] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + smecha.size + pub_sattr.size + + priv_sattr.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, smecha.buffer, smecha.size); + buf += smecha.size; + + memcpy(buf, pub_sattr.buffer, pub_sattr.size); + buf += pub_sattr.size; + + memcpy(buf, priv_sattr.buffer, priv_sattr.size); + + /* + * Shm io2: (out) public key object handle][private key object handle] + */ + out_shm = ckteec_alloc_shm(key_handle_size, CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_GENERATE_KEY_PAIR, + ctrl, out_shm, &out_size); + + if (rv != CKR_OK || out_size != out_shm->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto bail; + } + + key_handle = out_shm->buffer; + *pub_key = key_handle[0]; + *priv_key = key_handle[1]; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + release_serial_object(&priv_sattr); + release_serial_object(&pub_sattr); + release_serial_object(&smecha); + + return rv; +} + +CK_RV ck_wrap_key(CK_SESSION_HANDLE session, CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE wrapping_key, CK_OBJECT_HANDLE key, + CK_BYTE_PTR wrapped_key, CK_ULONG_PTR wrapped_key_len) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + struct serializer smecha = { 0 }; + uint32_t session_handle = session; + uint32_t wrp_key_handle = wrapping_key; + uint32_t key_handle = key; + size_t ctrl_size = 0; + size_t out_size = 0; + char *buf = NULL; + + if (!mechanism || !wrapped_key_len) + return CKR_ARGUMENTS_BAD; + + rv = serialize_ck_mecha_params(&smecha, mechanism); + if (rv) + return rv; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][wrapping-key-handle][key-handle] + * [serialized-mecha] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + sizeof(wrp_key_handle) + + sizeof(key_handle) + smecha.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &wrp_key_handle, sizeof(wrp_key_handle)); + buf += sizeof(wrp_key_handle); + + memcpy(buf, &key_handle, sizeof(key_handle)); + buf += sizeof(key_handle); + + memcpy(buf, smecha.buffer, smecha.size); + + /* Shm io2: output buffer reference - wrapped key */ + if (wrapped_key && *wrapped_key_len) + out_shm = ckteec_register_shm(wrapped_key, *wrapped_key_len, + CKTEEC_SHM_OUT); + else + /* Query output data size */ + out_shm = ckteec_alloc_shm(0, CKTEEC_SHM_OUT); + + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_WRAP_KEY, ctrl, out_shm, + &out_size); + + if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) + *wrapped_key_len = out_size; + + if (rv == CKR_BUFFER_TOO_SMALL && out_size && !wrapped_key) + rv = CKR_OK; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + release_serial_object(&smecha); + + return rv; +} + +CK_RV ck_unwrap_key(CK_SESSION_HANDLE session, CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE unwrapping_key, CK_BYTE_PTR wrapped_key, + CK_ULONG wrapped_key_len, CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, CK_OBJECT_HANDLE_PTR handle) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *in_shm = NULL; + TEEC_SharedMemory *out_shm = NULL; + struct serializer smecha = { 0 }; + struct serializer sattr = { 0 }; + uint32_t session_handle = session; + uint32_t unwrapping_key_handle = unwrapping_key; + size_t ctrl_size = 0; + uint32_t key_handle = 0; + char *buf = NULL; + size_t out_size = 0; + + if (!handle || !mechanism || (count && !attribs) || + (wrapped_key_len && !wrapped_key)) + return CKR_ARGUMENTS_BAD; + + if (!unwrapping_key) + return CKR_UNWRAPPING_KEY_HANDLE_INVALID; + + rv = serialize_ck_mecha_params(&smecha, mechanism); + if (rv) + return rv; + + rv = serialize_ck_attributes(&sattr, attribs, count); + if (rv) + goto bail; + + /* + * Shm io0: (in/out) ctrl + * (in) [session-handle][unwrapping-key-handle][serialized-mecha] + * [serialized-attributes] + * (out) [status] + */ + ctrl_size = sizeof(session_handle) + sizeof(unwrapping_key_handle) + + smecha.size + sattr.size; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + buf = ctrl->buffer; + + memcpy(buf, &session_handle, sizeof(session_handle)); + buf += sizeof(session_handle); + + memcpy(buf, &unwrapping_key_handle, sizeof(unwrapping_key_handle)); + buf += sizeof(unwrapping_key_handle); + + memcpy(buf, smecha.buffer, smecha.size); + buf += smecha.size; + + memcpy(buf, sattr.buffer, sattr.size); + + /* Shm io1: input - wrapped key buffer */ + in_shm = ckteec_register_shm(wrapped_key, wrapped_key_len, + CKTEEC_SHM_IN); + if (!in_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + /* Shm io2: (out) [object handle] */ + out_shm = ckteec_alloc_shm(sizeof(key_handle), CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto bail; + } + + rv = ckteec_invoke_ta(PKCS11_CMD_UNWRAP_KEY, ctrl, in_shm, out_shm, + &out_size, NULL, NULL); + + if (rv != CKR_OK || out_size != out_shm->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto bail; + } + + memcpy(&key_handle, out_shm->buffer, sizeof(key_handle)); + *handle = key_handle; + +bail: + ckteec_free_shm(out_shm); + ckteec_free_shm(in_shm); + ckteec_free_shm(ctrl); + release_serial_object(&sattr); + release_serial_object(&smecha); + + return rv; +} diff --git a/optee_client/libckteec/src/pkcs11_processing.h b/optee_client/libckteec/src/pkcs11_processing.h new file mode 100644 index 0000000..aee8fb2 --- /dev/null +++ b/optee_client/libckteec/src/pkcs11_processing.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2018, Linaro Limited + */ + +#ifndef LIBCKTEEC_PKCS11_PROCESSING_H +#define LIBCKTEEC_PKCS11_PROCESSING_H + +#include +#include + +CK_RV ck_create_object(CK_SESSION_HANDLE session, CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, CK_OBJECT_HANDLE_PTR phObject); + +CK_RV ck_destroy_object(CK_SESSION_HANDLE session, CK_OBJECT_HANDLE obj); + +CK_RV ck_encdecrypt_init(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE key, + int decrypt); + +CK_RV ck_encdecrypt_update(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int decrypt); + +CK_RV ck_encdecrypt_oneshot(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int decrypt); + +CK_RV ck_encdecrypt_final(CK_SESSION_HANDLE session, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int decrypt); + +CK_RV ck_digest_init(CK_SESSION_HANDLE session, CK_MECHANISM_PTR mechanism); + +CK_RV ck_digest_key(CK_SESSION_HANDLE session, CK_OBJECT_HANDLE key); + +CK_RV ck_digest_update(CK_SESSION_HANDLE session, CK_BYTE_PTR in, + CK_ULONG in_len); + +CK_RV ck_digest_oneshot(CK_SESSION_HANDLE session, CK_BYTE_PTR in, + CK_ULONG in_len, CK_BYTE_PTR out, + CK_ULONG_PTR out_len); + +CK_RV ck_digest_final(CK_SESSION_HANDLE session, CK_BYTE_PTR out, + CK_ULONG_PTR out_len); + +CK_RV ck_signverify_init(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE key, + int sign); + +CK_RV ck_signverify_update(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + int sign); + +CK_RV ck_signverify_oneshot(CK_SESSION_HANDLE session, + CK_BYTE_PTR in, + CK_ULONG in_len, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int sign); + +CK_RV ck_signverify_final(CK_SESSION_HANDLE session, + CK_BYTE_PTR out, + CK_ULONG_PTR out_len, + int sign); + +CK_RV ck_generate_key(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, + CK_OBJECT_HANDLE_PTR handle); + +CK_RV ck_find_objects_init(CK_SESSION_HANDLE session, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count); + +CK_RV ck_find_objects(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE_PTR obj, + CK_ULONG max_count, + CK_ULONG_PTR count); + +CK_RV ck_find_objects_final(CK_SESSION_HANDLE session); + +CK_RV ck_get_object_size(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ULONG_PTR p_size); + +CK_RV ck_get_attribute_value(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count); + +CK_RV ck_set_attribute_value(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count); + +CK_RV ck_copy_object(CK_SESSION_HANDLE session, + CK_OBJECT_HANDLE obj, + CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, + CK_OBJECT_HANDLE_PTR handle); + +CK_RV ck_derive_key(CK_SESSION_HANDLE session, CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE base_key, CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, CK_OBJECT_HANDLE_PTR handle); + +CK_RV ck_release_active_processing(CK_SESSION_HANDLE session, + enum pkcs11_ta_cmd command); + +CK_RV ck_generate_key_pair(CK_SESSION_HANDLE session, + CK_MECHANISM_PTR mechanism, + CK_ATTRIBUTE_PTR pub_attribs, + CK_ULONG pub_count, + CK_ATTRIBUTE_PTR priv_attribs, + CK_ULONG priv_count, + CK_OBJECT_HANDLE_PTR pub_key, + CK_OBJECT_HANDLE_PTR priv_key); + +CK_RV ck_wrap_key(CK_SESSION_HANDLE session, CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE wrapping_key, CK_OBJECT_HANDLE key, + CK_BYTE_PTR wrapped_key, CK_ULONG_PTR wrapped_key_len); + +CK_RV ck_unwrap_key(CK_SESSION_HANDLE session, CK_MECHANISM_PTR mechanism, + CK_OBJECT_HANDLE unwrapping_key, CK_BYTE_PTR wrapped_key, + CK_ULONG wrapped_key_len, CK_ATTRIBUTE_PTR attribs, + CK_ULONG count, CK_OBJECT_HANDLE_PTR handle); + +#endif /*LIBCKTEEC_PKCS11_PROCESSING_H*/ diff --git a/optee_client/libckteec/src/pkcs11_token.c b/optee_client/libckteec/src/pkcs11_token.c new file mode 100644 index 0000000..670162e --- /dev/null +++ b/optee_client/libckteec/src/pkcs11_token.c @@ -0,0 +1,841 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#include "ck_helpers.h" +#include "invoke_ta.h" +#include "local_utils.h" +#include "pkcs11_token.h" + +#define PKCS11_LIB_MANUFACTURER "Linaro" +#define PKCS11_LIB_DESCRIPTION "OP-TEE PKCS11 Cryptoki library" + +/** + * ck_get_info - Get local information for C_GetInfo + */ +CK_RV ck_get_info(CK_INFO_PTR info) +{ + const CK_INFO lib_info = { + .cryptokiVersion = { + CK_PKCS11_VERSION_MAJOR, + CK_PKCS11_VERSION_MINOR, + }, + .manufacturerID = PKCS11_LIB_MANUFACTURER, + .flags = 0, /* must be zero per the PKCS#11 2.40 */ + .libraryDescription = PKCS11_LIB_DESCRIPTION, + .libraryVersion = { + PKCS11_TA_VERSION_MAJOR, + PKCS11_TA_VERSION_MINOR + }, + }; + int n = 0; + + if (!info) + return CKR_ARGUMENTS_BAD; + + *info = lib_info; + + /* Pad strings with blank characters */ + n = strnlen((char *)info->manufacturerID, + sizeof(info->manufacturerID)); + memset(&info->manufacturerID[n], ' ', + sizeof(info->manufacturerID) - n); + + n = strnlen((char *)info->libraryDescription, + sizeof(info->libraryDescription)); + memset(&info->libraryDescription[n], ' ', + sizeof(info->libraryDescription) - n); + + return CKR_OK; +} + +/** + * ck_slot_get_list - Wrap C_GetSlotList into PKCS11_CMD_SLOT_LIST + */ +CK_RV ck_slot_get_list(CK_BBOOL present, + CK_SLOT_ID_PTR slots, CK_ULONG_PTR count) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *shm = NULL; + uint32_t *slot_ids = NULL; + size_t client_count = 0; + size_t size = 0; + size_t n = 0; + + /* Discard @present: all slots reported by TA are present */ + (void)present; + + if (!count) + return CKR_ARGUMENTS_BAD; + + /* + * As per spec, if @slots is NULL, "The contents of *pulCount on + * entry to C_GetSlotList has no meaning in this case (...)" + */ + if (slots) + client_count = *count; + + size = client_count * sizeof(*slot_ids); + + shm = ckteec_alloc_shm(size, CKTEEC_SHM_OUT); + if (!shm) + return CKR_HOST_MEMORY; + + rv = ckteec_invoke_ta(PKCS11_CMD_SLOT_LIST, NULL, + NULL, shm, &size, NULL, NULL); + + if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) + *count = size / sizeof(*slot_ids); + + if (!slots && rv == CKR_BUFFER_TOO_SMALL) + rv = CKR_OK; + if (!slots || rv) + goto out; + + slot_ids = shm->buffer; + for (n = 0; n < *count; n++) + slots[n] = slot_ids[n]; + +out: + ckteec_free_shm(shm); + + return rv; +} + +/** + * ck_slot_get_info - Wrap C_GetSlotInfo into PKCS11_CMD_SLOT_INFO + */ +CK_RV ck_slot_get_info(CK_SLOT_ID slot, CK_SLOT_INFO_PTR info) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out = NULL; + uint32_t slot_id = slot; + struct pkcs11_slot_info *ta_info = NULL; + size_t out_size = 0; + + if (!info) + return CKR_ARGUMENTS_BAD; + + ctrl = ckteec_alloc_shm(sizeof(slot_id), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + memcpy(ctrl->buffer, &slot_id, sizeof(slot_id)); + + out = ckteec_alloc_shm(sizeof(*ta_info), CKTEEC_SHM_OUT); + if (!out) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_SLOT_INFO, ctrl, out, &out_size); + if (rv != CKR_OK || out_size != out->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto out; + } + + ta_info = out->buffer; + + COMPILE_TIME_ASSERT(sizeof(info->slotDescription) == + sizeof(ta_info->slot_description)); + memcpy(info->slotDescription, ta_info->slot_description, + sizeof(info->slotDescription)); + + COMPILE_TIME_ASSERT(sizeof(info->manufacturerID) == + sizeof(ta_info->manufacturer_id)); + memcpy(info->manufacturerID, ta_info->manufacturer_id, + sizeof(info->manufacturerID)); + + info->flags = ta_info->flags; + + COMPILE_TIME_ASSERT(sizeof(info->hardwareVersion) == + sizeof(ta_info->hardware_version)); + memcpy(&info->hardwareVersion, ta_info->hardware_version, + sizeof(info->hardwareVersion)); + + COMPILE_TIME_ASSERT(sizeof(info->firmwareVersion) == + sizeof(ta_info->firmware_version)); + memcpy(&info->firmwareVersion, ta_info->firmware_version, + sizeof(info->firmwareVersion)); + +out: + ckteec_free_shm(ctrl); + ckteec_free_shm(out); + + return rv; +} + +/** + * ck_token_get_info - Wrap C_GetTokenInfo into PKCS11_CMD_TOKEN_INFO + */ +CK_RV ck_token_get_info(CK_SLOT_ID slot, CK_TOKEN_INFO_PTR info) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + uint32_t slot_id = slot; + struct pkcs11_token_info *ta_info = NULL; + size_t out_size = 0; + + if (!info) + return CKR_ARGUMENTS_BAD; + + ctrl = ckteec_alloc_shm(sizeof(slot_id), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + memcpy(ctrl->buffer, &slot_id, sizeof(slot_id)); + + out_shm = ckteec_alloc_shm(sizeof(*ta_info), CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_TOKEN_INFO, ctrl, + out_shm, &out_size); + if (rv) + goto out; + + if (out_size != out_shm->size) { + rv = CKR_DEVICE_ERROR; + goto out; + } + + ta_info = out_shm->buffer; + + COMPILE_TIME_ASSERT(sizeof(info->label) == sizeof(ta_info->label)); + memcpy(info->label, ta_info->label, sizeof(info->label)); + + COMPILE_TIME_ASSERT(sizeof(info->manufacturerID) == + sizeof(ta_info->manufacturer_id)); + memcpy(info->manufacturerID, ta_info->manufacturer_id, + sizeof(info->manufacturerID)); + + COMPILE_TIME_ASSERT(sizeof(info->model) == sizeof(ta_info->model)); + memcpy(info->model, ta_info->model, sizeof(info->model)); + + COMPILE_TIME_ASSERT(sizeof(info->serialNumber) == + sizeof(ta_info->serial_number)); + memcpy(info->serialNumber, ta_info->serial_number, + sizeof(info->serialNumber)); + + info->flags = ta_info->flags; + info->ulMaxSessionCount = ta_info->max_session_count; + info->ulSessionCount = ta_info->session_count; + info->ulMaxRwSessionCount = ta_info->max_rw_session_count; + info->ulRwSessionCount = ta_info->rw_session_count; + info->ulMaxPinLen = ta_info->max_pin_len; + info->ulMinPinLen = ta_info->min_pin_len; + info->ulTotalPublicMemory = ta_info->total_public_memory; + info->ulFreePublicMemory = ta_info->free_public_memory; + info->ulTotalPrivateMemory = ta_info->total_private_memory; + info->ulFreePrivateMemory = ta_info->free_private_memory; + + COMPILE_TIME_ASSERT(sizeof(info->hardwareVersion) == + sizeof(ta_info->hardware_version)); + memcpy(&info->hardwareVersion, ta_info->hardware_version, + sizeof(info->hardwareVersion)); + + COMPILE_TIME_ASSERT(sizeof(info->firmwareVersion) == + sizeof(ta_info->firmware_version)); + memcpy(&info->firmwareVersion, ta_info->firmware_version, + sizeof(info->firmwareVersion)); + + COMPILE_TIME_ASSERT(sizeof(info->utcTime) == sizeof(ta_info->utc_time)); + memcpy(&info->utcTime, ta_info->utc_time, sizeof(info->utcTime)); + +out: + ckteec_free_shm(ctrl); + ckteec_free_shm(out_shm); + + return rv; +} + +/** + * ck_token_mechanism_ids - Wrap C_GetMechanismList + */ +CK_RV ck_token_mechanism_ids(CK_SLOT_ID slot, + CK_MECHANISM_TYPE_PTR mechanisms, + CK_ULONG_PTR count) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out = NULL; + uint32_t slot_id = slot; + uint32_t *mecha_ids = NULL; + size_t out_size = 0; + size_t n = 0; + + if (!count) + return CKR_ARGUMENTS_BAD; + + /* + * As per spec, if @mechanism is NULL, "The contents of *pulCount on + * entry to C_GetMechanismList has no meaning in this case (...)" + */ + if (mechanisms) + out_size = *count * sizeof(*mecha_ids); + + ctrl = ckteec_alloc_shm(sizeof(slot_id), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + memcpy(ctrl->buffer, &slot_id, sizeof(slot_id)); + + out = ckteec_alloc_shm(out_size, CKTEEC_SHM_OUT); + if (!out) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_MECHANISM_IDS, + ctrl, out, &out_size); + + if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) + *count = out_size / sizeof(*mecha_ids); + + if (!mechanisms && rv == CKR_BUFFER_TOO_SMALL) + rv = CKR_OK; + if (!mechanisms || rv) + goto out; + + mecha_ids = out->buffer; + for (n = 0; n < *count; n++) + mechanisms[n] = mecha_ids[n]; + +out: + ckteec_free_shm(ctrl); + ckteec_free_shm(out); + + return rv; +} + +/** + * ck_token_mechanism_info - Wrap C_GetMechanismInfo into command MECHANISM_INFO + */ +CK_RV ck_token_mechanism_info(CK_SLOT_ID slot, CK_MECHANISM_TYPE type, + CK_MECHANISM_INFO_PTR info) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out = NULL; + uint32_t slot_id = slot; + uint32_t mecha_type = type; + struct pkcs11_mechanism_info *ta_info = NULL; + char *buf = NULL; + size_t out_size = 0; + + if (!info) + return CKR_ARGUMENTS_BAD; + + ctrl = ckteec_alloc_shm(sizeof(slot_id) + sizeof(mecha_type), + CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + + buf = ctrl->buffer; + + memcpy(buf, &slot_id, sizeof(slot_id)); + buf += sizeof(slot_id); + + memcpy(buf, &mecha_type, sizeof(mecha_type)); + + out = ckteec_alloc_shm(sizeof(*ta_info), CKTEEC_SHM_OUT); + if (!out) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_MECHANISM_INFO, + ctrl, out, &out_size); + + if (rv != CKR_OK || out_size != out->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto out; + } + + ta_info = out->buffer; + + info->ulMinKeySize = ta_info->min_key_size; + info->ulMaxKeySize = ta_info->max_key_size; + info->flags = ta_info->flags; + +out: + ckteec_free_shm(ctrl); + ckteec_free_shm(out); + + return rv; +} + +/** + * ck_open_session - Wrap C_OpenSession into PKCS11_CMD_OPEN_{RW|RO}_SESSION + * + * Note: cookie and callback are not utilized by libckteec and are silently + * sinked in to have better out-of-box compatibility with 3rd party libraries + * and applications which provides the callback. + */ +CK_RV ck_open_session(CK_SLOT_ID slot, CK_FLAGS flags, CK_VOID_PTR cookie, + CK_NOTIFY callback, CK_SESSION_HANDLE_PTR session) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out = NULL; + uint32_t slot_id = slot; + uint32_t u32_flags = flags; + uint32_t handle = 0; + size_t out_size = 0; + uint8_t *buf; + + /* Ignore notify callback */ + (void)cookie; + (void)callback; + + if ((flags & ~(CKF_RW_SESSION | CKF_SERIAL_SESSION)) || !session) + return CKR_ARGUMENTS_BAD; + + /* Shm io0: (in/out) ctrl = [slot-id][flags] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(slot_id) + sizeof(u32_flags), + CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + buf = (uint8_t *)ctrl->buffer; + memcpy(buf, &slot_id, sizeof(slot_id)); + buf += sizeof(slot_id); + memcpy(buf, &u32_flags, sizeof(u32_flags)); + + /* Shm io2: (out) [session handle] */ + out = ckteec_alloc_shm(sizeof(handle), CKTEEC_SHM_OUT); + if (!out) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_OPEN_SESSION, + ctrl, out, &out_size); + if (rv != CKR_OK || out_size != out->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto out; + } + + memcpy(&handle, out->buffer, sizeof(handle)); + *session = handle; + +out: + ckteec_free_shm(ctrl); + ckteec_free_shm(out); + + return rv; +} + +/** + * ck_open_session - Wrap C_OpenSession into PKCS11_CMD_CLOSE_SESSION + */ +CK_RV ck_close_session(CK_SESSION_HANDLE session) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t session_handle = session; + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_CLOSE_SESSION, ctrl); + +out: + ckteec_free_shm(ctrl); + + return rv; +} + +/** + * ck_close_all_sessions - Wrap C_CloseAllSessions into TA command + */ +CK_RV ck_close_all_sessions(CK_SLOT_ID slot) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t slot_id = slot; + + /* Shm io0: (in/out) ctrl = [slot-id] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(slot_id), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + memcpy(ctrl->buffer, &slot_id, sizeof(slot_id)); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_CLOSE_ALL_SESSIONS, ctrl); + +out: + ckteec_free_shm(ctrl); + + return rv; +} + +/** + * ck_get_session_info - Wrap C_GetSessionInfo into PKCS11_CMD_SESSION_INFO + */ +CK_RV ck_get_session_info(CK_SESSION_HANDLE session, + CK_SESSION_INFO_PTR info) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out = NULL; + uint32_t session_handle = session; + struct pkcs11_session_info *ta_info = NULL; + size_t out_size = 0; + + if (!info) + return CKR_ARGUMENTS_BAD; + + /* Shm io0: (in/out) ctrl = [session-handle] / [status] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) { + rv = CKR_HOST_MEMORY; + goto out; + } + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io2: (out) [session info] */ + out = ckteec_alloc_shm(sizeof(struct pkcs11_session_info), + CKTEEC_SHM_OUT); + if (!out) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_SESSION_INFO, + ctrl, out, &out_size); + + if (rv != CKR_OK || out_size != out->size) { + if (rv == CKR_OK) + rv = CKR_DEVICE_ERROR; + goto out; + } + + ta_info = (struct pkcs11_session_info *)out->buffer; + info->slotID = ta_info->slot_id; + info->state = ta_info->state; + info->flags = ta_info->flags; + info->ulDeviceError = ta_info->device_error; + +out: + ckteec_free_shm(ctrl); + ckteec_free_shm(out); + + return rv; +} + +/** + * ck_init_token - Wrap C_InitToken into PKCS11_CMD_INIT_TOKEN + */ +CK_RV ck_init_token(CK_SLOT_ID slot, CK_UTF8CHAR_PTR pin, + CK_ULONG pin_len, CK_UTF8CHAR_PTR label) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t slot_id = slot; + uint32_t pkcs11_pin_len = pin_len; + size_t ctrl_size = 0; + char *buf = NULL; + + if (!pin && pin_len) + return CKR_ARGUMENTS_BAD; + + if (!label) + return CKR_ARGUMENTS_BAD; + + /* Shm io0: (in/out) ctrl = [slot-id][pin_len][label][pin] / [status] */ + ctrl_size = sizeof(slot_id) + sizeof(pkcs11_pin_len) + + 32 * sizeof(uint8_t) + pkcs11_pin_len; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + buf = ctrl->buffer; + + memcpy(buf, &slot_id, sizeof(slot_id)); + buf += sizeof(slot_id); + + memcpy(buf, &pkcs11_pin_len, sizeof(pkcs11_pin_len)); + buf += sizeof(pkcs11_pin_len); + + memcpy(buf, label, 32 * sizeof(uint8_t)); + buf += 32 * sizeof(uint8_t); + + memcpy(buf, pin, pkcs11_pin_len); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_INIT_TOKEN, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +/** + * ck_init_pin - Wrap C_InitPIN into PKCS11_CMD_INIT_PIN + */ +CK_RV ck_init_pin(CK_SESSION_HANDLE session, + CK_UTF8CHAR_PTR pin, CK_ULONG pin_len) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t pkcs11_session = session; + uint32_t pkcs11_pin_len = pin_len; + size_t ctrl_size = 0; + char *buf = NULL; + + if (!pin && pin_len) + return CKR_ARGUMENTS_BAD; + + /* Shm io0: (in/out) ctrl = [session][pin_len][pin] / [status] */ + ctrl_size = sizeof(pkcs11_session) + sizeof(pkcs11_pin_len) + + pkcs11_pin_len; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + buf = ctrl->buffer; + + memcpy(buf, &pkcs11_session, sizeof(pkcs11_session)); + buf += sizeof(pkcs11_session); + + memcpy(buf, &pkcs11_pin_len, sizeof(pkcs11_pin_len)); + buf += sizeof(pkcs11_pin_len); + + memcpy(buf, pin, pkcs11_pin_len); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_INIT_PIN, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +/** + * ck_set_pin - Wrap C_SetPIN into PKCS11_CMD_SET_PIN + */ +CK_RV ck_set_pin(CK_SESSION_HANDLE session, + CK_UTF8CHAR_PTR old, CK_ULONG old_len, + CK_UTF8CHAR_PTR new, CK_ULONG new_len) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t pkcs11_session = session; + uint32_t pkcs11_old_len = old_len; + uint32_t pkcs11_new_len = new_len; + size_t ctrl_size = 0; + char *buf; + + if ((!old && old_len) || (!new && new_len)) + return CKR_ARGUMENTS_BAD; + + /* + * Shm io0: (in/out) ctrl + * (in) [session][old_pin_len][new_pin_len][old pin][new pin] + * (out) [status] + */ + ctrl_size = sizeof(pkcs11_session) + sizeof(pkcs11_old_len) + + sizeof(pkcs11_new_len) + pkcs11_old_len + pkcs11_new_len; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + buf = ctrl->buffer; + + memcpy(buf, &pkcs11_session, sizeof(pkcs11_session)); + buf += sizeof(pkcs11_session); + + memcpy(buf, &pkcs11_old_len, sizeof(pkcs11_old_len)); + buf += sizeof(pkcs11_old_len); + + memcpy(buf, &pkcs11_new_len, sizeof(pkcs11_new_len)); + buf += sizeof(pkcs11_new_len); + + memcpy(buf, old, pkcs11_old_len); + buf += pkcs11_old_len; + + memcpy(buf, new, pkcs11_new_len); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_SET_PIN, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +/** + * ck_login - Wrap C_Login into PKCS11_CMD_LOGIN + */ +CK_RV ck_login(CK_SESSION_HANDLE session, CK_USER_TYPE user_type, + CK_UTF8CHAR_PTR pin, CK_ULONG pin_len) + +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t pkcs11_session = session; + uint32_t pkcs11_user = user_type; + uint32_t pkcs11_pin_len = pin_len; + size_t ctrl_size = 0; + char *buf = NULL; + + if (!pin && pin_len) + return CKR_ARGUMENTS_BAD; + + /* Shm io0: (i/o) ctrl = [session][user][pin length][pin] / [status] */ + ctrl_size = sizeof(pkcs11_session) + sizeof(pkcs11_user) + + sizeof(pkcs11_pin_len) + pkcs11_pin_len; + + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + buf = ctrl->buffer; + + memcpy(buf, &pkcs11_session, sizeof(pkcs11_session)); + buf += sizeof(pkcs11_session); + + memcpy(buf, &pkcs11_user, sizeof(pkcs11_user)); + buf += sizeof(pkcs11_user); + + memcpy(buf, &pkcs11_pin_len, sizeof(pkcs11_pin_len)); + buf += sizeof(pkcs11_pin_len); + + memcpy(buf, pin, pkcs11_pin_len); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_LOGIN, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +/** + * ck_logout - Wrap C_Logout into PKCS11_CMD_LOGOUT + */ +CK_RV ck_logout(CK_SESSION_HANDLE session) +{ + CK_RV rv = CKR_GENERAL_ERROR; + TEEC_SharedMemory *ctrl = NULL; + uint32_t session_handle = session; + + /* io0 = [session-handle] */ + ctrl = ckteec_alloc_shm(sizeof(session_handle), CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + rv = ckteec_invoke_ctrl(PKCS11_CMD_LOGOUT, ctrl); + + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_seed_random(CK_SESSION_HANDLE session, CK_BYTE_PTR seed, + CK_ULONG length) +{ + CK_RV rv = CKR_GENERAL_ERROR; + size_t ctrl_size = 0; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *in_shm = NULL; + uint32_t session_handle = session; + + if (!seed && length) + return CKR_ARGUMENTS_BAD; + + if (!seed) + return CKR_OK; + + /* Shm io0: (i/o) [session-handle] / [status] */ + ctrl_size = sizeof(session_handle); + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io1: (in) [seed data] */ + in_shm = ckteec_register_shm(seed, length, CKTEEC_SHM_IN); + if (!in_shm) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_in(PKCS11_CMD_SEED_RANDOM, ctrl, in_shm); + +out: + ckteec_free_shm(in_shm); + ckteec_free_shm(ctrl); + + return rv; +} + +CK_RV ck_generate_random(CK_SESSION_HANDLE session, CK_BYTE_PTR data, + CK_ULONG length) +{ + CK_RV rv = CKR_GENERAL_ERROR; + size_t ctrl_size = 0; + TEEC_SharedMemory *ctrl = NULL; + TEEC_SharedMemory *out_shm = NULL; + uint32_t session_handle = session; + size_t out_size = 0; + + if (!data && length) + return CKR_ARGUMENTS_BAD; + + if (!data) + return CKR_OK; + + /* Shm io0: (i/o) [session-handle] / [status] */ + ctrl_size = sizeof(session_handle); + ctrl = ckteec_alloc_shm(ctrl_size, CKTEEC_SHM_INOUT); + if (!ctrl) + return CKR_HOST_MEMORY; + + memcpy(ctrl->buffer, &session_handle, sizeof(session_handle)); + + /* Shm io2: (out) [generated random] */ + out_shm = ckteec_register_shm(data, length, CKTEEC_SHM_OUT); + if (!out_shm) { + rv = CKR_HOST_MEMORY; + goto out; + } + + rv = ckteec_invoke_ctrl_out(PKCS11_CMD_GENERATE_RANDOM, ctrl, out_shm, + &out_size); + +out: + ckteec_free_shm(out_shm); + ckteec_free_shm(ctrl); + + return rv; +} diff --git a/optee_client/libckteec/src/pkcs11_token.h b/optee_client/libckteec/src/pkcs11_token.h new file mode 100644 index 0000000..7c3e9bb --- /dev/null +++ b/optee_client/libckteec/src/pkcs11_token.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef LIBCKTEEC_PKCS11_TOKEN_H +#define LIBCKTEEC_PKCS11_TOKEN_H + +#include + +#include "invoke_ta.h" + +CK_RV ck_get_info(CK_INFO_PTR info); + +CK_RV ck_slot_get_list(CK_BBOOL present, + CK_SLOT_ID_PTR slots, CK_ULONG_PTR count); + +CK_RV ck_slot_get_info(CK_SLOT_ID slot, CK_SLOT_INFO_PTR info); + +CK_RV ck_token_get_info(CK_SLOT_ID slot, CK_TOKEN_INFO_PTR info); + +CK_RV ck_token_mechanism_ids(CK_SLOT_ID slot, + CK_MECHANISM_TYPE_PTR mechanisms, + CK_ULONG_PTR count); + +CK_RV ck_token_mechanism_info(CK_SLOT_ID slot, CK_MECHANISM_TYPE type, + CK_MECHANISM_INFO_PTR info); + +CK_RV ck_open_session(CK_SLOT_ID slot, CK_FLAGS flags, CK_VOID_PTR cookie, + CK_NOTIFY callback, CK_SESSION_HANDLE_PTR session); + +CK_RV ck_close_session(CK_SESSION_HANDLE session); + +CK_RV ck_close_all_sessions(CK_SLOT_ID slot); + +CK_RV ck_get_session_info(CK_SESSION_HANDLE session, CK_SESSION_INFO_PTR info); + +CK_RV ck_init_token(CK_SLOT_ID slot, CK_UTF8CHAR_PTR pin, + CK_ULONG pin_len, CK_UTF8CHAR_PTR label); + +CK_RV ck_init_pin(CK_SESSION_HANDLE session, + CK_UTF8CHAR_PTR pin, CK_ULONG pin_len); + +CK_RV ck_set_pin(CK_SESSION_HANDLE session, + CK_UTF8CHAR_PTR old, CK_ULONG old_len, + CK_UTF8CHAR_PTR new, CK_ULONG new_len); + +CK_RV ck_login(CK_SESSION_HANDLE session, CK_USER_TYPE user_type, + CK_UTF8CHAR_PTR pin, CK_ULONG pin_len); + +CK_RV ck_logout(CK_SESSION_HANDLE session); + +CK_RV ck_seed_random(CK_SESSION_HANDLE session, CK_BYTE_PTR seed, + CK_ULONG length); + +CK_RV ck_generate_random(CK_SESSION_HANDLE session, CK_BYTE_PTR data, + CK_ULONG length); + +#endif /*LIBCKTEEC_PKCS11_TOKEN_H*/ diff --git a/optee_client/libckteec/src/serialize_ck.c b/optee_client/libckteec/src/serialize_ck.c new file mode 100644 index 0000000..6fe9d10 --- /dev/null +++ b/optee_client/libckteec/src/serialize_ck.c @@ -0,0 +1,791 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "ck_helpers.h" +#include "local_utils.h" +#include "serializer.h" +#include "serialize_ck.h" + +/* + * Serialization and de-serialization logic + * + * Cryptoki API works in a way that user application uses memory references + * in object attributes description. TA can be invoked with only a small set + * of possible references to caller memory. Thus a Cryptoki object, made of + * data and pointers to data, is reassembled into a byte array where each + * attribute info (ID, value size, value) is appended with byte alignment. This + * so-called serialized object can be passed through the TA API. + * + * Initial entry to PKCS11 TA uses serialize_ck_attributes(). When TA + * returns with updated serialized data to be passed back to caller, we call + * deserialize_ck_attributes(). + * + * Special handling is performed for CK_ULONG passing which may be either 32 + * bits or 64 bits depending on target device architecture. In TA interface + * this is handled as unsigned 32 bit data type. + * + * When user application is querying attributes in example with + * C_GetAttributeValue() user may allocate larger value buffers. During entry + * to TA shared buffer is allocated in serialize_ck_attributes() based on + * caller's arguments. For each attribute TA verifies if value fits in + * the buffer and if it does, value is returned. Value size in buffer is + * updated to indicate real size of the value. When call is returned back to + * REE deserialize_ck_attributes() is invoked and then both input arguments and + * serialization buffer are used to return values to caller. Provided input + * arguments from caller are used to determine serialization buffer structure + * and then actual values and value sizes are then decoded from serialization + * buffer and returned to caller in caller's allocated memory. + */ + +/* + * Generic way of serializing CK keys, certificates, mechanism parameters, ... + * In cryptoki 2.40 parameters are almost all packaged as structure below: + */ +struct ck_ref { + CK_ULONG id; + CK_BYTE_PTR ptr; + CK_ULONG len; +}; + +/* + * This is for attributes that contains data memory indirections. + * In other words, an attributes that defines a list of attributes. + * They are identified from the attribute type CKA_... + * + * @obj - ref used to track the serial object being created + * @attribute - pointer to a structure aligned of the CK_ATTRIBUTE struct + */ +static CK_RV serialize_indirect_attribute(struct serializer *obj, + CK_ATTRIBUTE_PTR attribute) +{ + CK_ATTRIBUTE_PTR attr = NULL; + CK_ULONG count = 0; + CK_RV rv = CKR_GENERAL_ERROR; + struct serializer obj2 = { 0 }; + + switch (attribute->type) { + /* These are serialized each separately */ + case CKA_DERIVE_TEMPLATE: + case CKA_WRAP_TEMPLATE: + case CKA_UNWRAP_TEMPLATE: + count = attribute->ulValueLen / sizeof(CK_ATTRIBUTE); + attr = (CK_ATTRIBUTE_PTR)attribute->pValue; + break; + default: + return CKR_NO_EVENT; + } + + /* Create a serialized object for the content */ + rv = serialize_ck_attributes(&obj2, attr, count); + if (rv) + return rv; + + /* + * Append the created serialized object into target object: + * [attrib-id][byte-size][attributes-data] + */ + rv = serialize_32b(obj, attribute->type); + if (rv) + return rv; + + rv = serialize_32b(obj, obj2.size); + if (rv) + return rv; + + rv = serialize_buffer(obj, obj2.buffer, obj2.size); + if (rv) + return rv; + + obj->item_count++; + + return rv; +} + +static CK_RV deserialize_indirect_attribute(struct pkcs11_attribute_head *obj, + CK_ATTRIBUTE_PTR attribute) +{ + CK_ULONG count = 0; + CK_ATTRIBUTE_PTR attr = NULL; + + switch (attribute->type) { + /* These are serialized each separately */ + case CKA_DERIVE_TEMPLATE: + case CKA_WRAP_TEMPLATE: + case CKA_UNWRAP_TEMPLATE: + count = attribute->ulValueLen / sizeof(CK_ATTRIBUTE); + attr = (CK_ATTRIBUTE_PTR)attribute->pValue; + break; + default: + return CKR_GENERAL_ERROR; + } + + return deserialize_ck_attributes(obj->data, attr, count); +} + +static int ck_attr_is_ulong(CK_ATTRIBUTE_TYPE attribute_id) +{ + switch (attribute_id) { + case CKA_CLASS: + case CKA_CERTIFICATE_TYPE: + case CKA_CERTIFICATE_CATEGORY: + case CKA_NAME_HASH_ALGORITHM: + case CKA_KEY_TYPE: + case CKA_HW_FEATURE_TYPE: + case CKA_MECHANISM_TYPE: + case CKA_KEY_GEN_MECHANISM: + case CKA_VALUE_LEN: + case CKA_MODULUS_BITS: + return true; + default: + return false; + } +} + +static CK_RV serialize_ck_attribute(struct serializer *obj, CK_ATTRIBUTE *attr) +{ + CK_MECHANISM_TYPE *type = NULL; + uint32_t pkcs11_size = 0; + uint32_t pkcs11_data32 = 0; + void *pkcs11_pdata = NULL; + uint32_t *mech_buf = NULL; + CK_RV rv = CKR_GENERAL_ERROR; + unsigned int n = 0; + unsigned int m = 0; + + if (attr->type == PKCS11_UNDEFINED_ID) + return CKR_ATTRIBUTE_TYPE_INVALID; + + switch (attr->type) { + case CKA_DERIVE_TEMPLATE: + case CKA_WRAP_TEMPLATE: + case CKA_UNWRAP_TEMPLATE: + return serialize_indirect_attribute(obj, attr); + case CKA_ALLOWED_MECHANISMS: + n = attr->ulValueLen / sizeof(CK_ULONG); + pkcs11_size = n * sizeof(uint32_t); + mech_buf = malloc(pkcs11_size); + if (!mech_buf) + return CKR_HOST_MEMORY; + + type = attr->pValue; + for (m = 0; m < n; m++) { + mech_buf[m] = type[m]; + if (mech_buf[m] == PKCS11_UNDEFINED_ID) { + rv = CKR_MECHANISM_INVALID; + goto out; + } + } + pkcs11_pdata = mech_buf; + break; + /* Attributes which data value do not need conversion (aside ulong) */ + default: + pkcs11_pdata = attr->pValue; + if (!attr->pValue) { + pkcs11_size = 0; + } else if (ck_attr_is_ulong(attr->type)) { + CK_ULONG ck_ulong = 0; + + if (attr->ulValueLen < sizeof(CK_ULONG)) + return CKR_ATTRIBUTE_VALUE_INVALID; + + memcpy(&ck_ulong, attr->pValue, sizeof(ck_ulong)); + pkcs11_data32 = ck_ulong; + pkcs11_pdata = &pkcs11_data32; + pkcs11_size = sizeof(uint32_t); + } else { + pkcs11_size = attr->ulValueLen; + } + break; + } + + rv = serialize_32b(obj, attr->type); + if (rv) + goto out; + + rv = serialize_32b(obj, pkcs11_size); + if (rv) + goto out; + + rv = serialize_buffer(obj, pkcs11_pdata, pkcs11_size); + if (rv) + goto out; + + obj->item_count++; +out: + free(mech_buf); + + return rv; +} + +/* CK attribute reference arguments are list of attribute item */ +CK_RV serialize_ck_attributes(struct serializer *obj, + CK_ATTRIBUTE_PTR attributes, CK_ULONG count) +{ + CK_ULONG n = 0; + CK_RV rv = CKR_OK; + + rv = init_serial_object(obj); + if (rv) + return rv; + + for (n = 0; n < count; n++) { + rv = serialize_ck_attribute(obj, attributes + n); + if (rv) + break; + } + + if (rv) + release_serial_object(obj); + else + finalize_serial_object(obj); + + return rv; +} + +static CK_RV deserialize_mecha_list(CK_MECHANISM_TYPE *dst, void *src, + size_t count) +{ + char *ta_src = src; + size_t n = 0; + uint32_t mecha_id = 0; + + for (n = 0; n < count; n++) { + memcpy(&mecha_id, ta_src + n * sizeof(mecha_id), + sizeof(mecha_id)); + dst[n] = mecha_id; + } + + return CKR_OK; +} + +static CK_RV deserialize_ck_attribute(struct pkcs11_attribute_head *in, + uint8_t *data, CK_ATTRIBUTE_PTR out) +{ + CK_ULONG ck_ulong = 0; + uint32_t pkcs11_data32 = 0; + CK_RV rv = CKR_OK; + + out->type = in->id; + + if (in->size == PKCS11_CK_UNAVAILABLE_INFORMATION) { + out->ulValueLen = CK_UNAVAILABLE_INFORMATION; + return CKR_OK; + } + + if (!out->pValue && ck_attr_is_ulong(out->type)) { + out->ulValueLen = sizeof(CK_ULONG); + return CKR_OK; + } + + if (out->ulValueLen < in->size) { + out->ulValueLen = in->size; + return CKR_OK; + } + + if (!out->pValue) + return CKR_OK; + + /* Specific ulong encoded as 32bit in PKCS11 TA API */ + if (ck_attr_is_ulong(out->type)) { + if (out->ulValueLen < sizeof(CK_ULONG)) + return CKR_ATTRIBUTE_VALUE_INVALID; + + memcpy(&pkcs11_data32, data, sizeof(uint32_t)); + if (out->type == CKA_KEY_GEN_MECHANISM && + pkcs11_data32 == PKCS11_CK_UNAVAILABLE_INFORMATION) + ck_ulong = CK_UNAVAILABLE_INFORMATION; + else + ck_ulong = pkcs11_data32; + + memcpy(out->pValue, &ck_ulong, sizeof(CK_ULONG)); + out->ulValueLen = sizeof(CK_ULONG); + return CKR_OK; + } + + switch (out->type) { + case CKA_DERIVE_TEMPLATE: + case CKA_WRAP_TEMPLATE: + case CKA_UNWRAP_TEMPLATE: + rv = deserialize_indirect_attribute(in, out->pValue); + break; + case CKA_ALLOWED_MECHANISMS: + rv = deserialize_mecha_list(out->pValue, data, + in->size / sizeof(uint32_t)); + out->ulValueLen = in->size / sizeof(uint32_t) * + sizeof(CK_ULONG); + break; + /* Attributes which data value do not need conversion (aside ulong) */ + default: + memcpy(out->pValue, data, in->size); + out->ulValueLen = in->size; + break; + } + + return rv; +} + +CK_RV deserialize_ck_attributes(uint8_t *in, CK_ATTRIBUTE_PTR attributes, + CK_ULONG count) +{ + CK_ATTRIBUTE_PTR cur_attr = attributes; + CK_ULONG n = 0; + CK_RV rv = CKR_OK; + uint8_t *curr_head = in; + size_t len = 0; + + curr_head += sizeof(struct pkcs11_object_head); + + for (n = count; n > 0; n--, cur_attr++, curr_head += len) { + struct pkcs11_attribute_head *cli_ref = (void *)curr_head; + struct pkcs11_attribute_head cli_head = { 0 }; + void *data_ptr = NULL; + + /* Make copy if header so that is aligned properly. */ + memcpy(&cli_head, cli_ref, sizeof(cli_head)); + + /* Get real data pointer from template data */ + data_ptr = cli_ref->data; + + len = sizeof(cli_head); + + /* Advance by size provisioned in input serialized buffer */ + if (cur_attr->pValue) { + if (ck_attr_is_ulong(cur_attr->type)) + len += sizeof(uint32_t); + else + len += cur_attr->ulValueLen; + } + + rv = deserialize_ck_attribute(&cli_head, data_ptr, cur_attr); + if (rv) + return rv; + } + + return rv; +} + +/* + * Serialization of CK mechanism parameters + * + * Most mechanism have no parameters. + * Some mechanism have a single 32bit parameter. + * Some mechanism have a specific parameter structure which may contain + * indirected data (data referred by a buffer pointer). + * + * Below are each structure specific mechanisms parameters. + */ + +static CK_RV serialize_mecha_aes_ctr(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_AES_CTR_PARAMS_PTR param = mecha->pParameter; + CK_RV rv = CKR_GENERAL_ERROR; + uint32_t size = 0; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + size = sizeof(uint32_t) + sizeof(param->cb); + rv = serialize_32b(obj, size); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, param->ulCounterBits); + if (rv) + return rv; + + rv = serialize_buffer(obj, param->cb, sizeof(param->cb)); + if (rv) + return rv; + + return rv; +} + +static CK_RV serialize_mecha_aes_iv(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + uint32_t iv_size = mecha->ulParameterLen; + CK_RV rv = CKR_GENERAL_ERROR; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + rv = serialize_32b(obj, iv_size); + if (rv) + return rv; + + return serialize_buffer(obj, mecha->pParameter, mecha->ulParameterLen); +} + +static CK_RV serialize_mecha_key_deriv_str(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_KEY_DERIVATION_STRING_DATA_PTR param = mecha->pParameter; + CK_RV rv = CKR_GENERAL_ERROR; + uint32_t size = 0; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + size = sizeof(uint32_t) + param->ulLen; + rv = serialize_32b(obj, size); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, param->ulLen); + if (rv) + return rv; + + return serialize_buffer(obj, param->pData, param->ulLen); +} + +static CK_RV serialize_mecha_ecdh1_derive_param(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_ECDH1_DERIVE_PARAMS *params = mecha->pParameter; + CK_RV rv = CKR_GENERAL_ERROR; + size_t params_size = 3 * sizeof(uint32_t) + params->ulSharedDataLen + + params->ulPublicDataLen; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + rv = serialize_32b(obj, params_size); + if (rv) + return rv; + + rv = serialize_32b(obj, params->kdf); + if (rv) + return rv; + + rv = serialize_32b(obj, params->ulSharedDataLen); + if (rv) + return rv; + + rv = serialize_buffer(obj, params->pSharedData, + params->ulSharedDataLen); + if (rv) + return rv; + + rv = serialize_32b(obj, params->ulPublicDataLen); + if (rv) + return rv; + + return serialize_buffer(obj, params->pPublicData, + params->ulPublicDataLen); +} + +static CK_RV serialize_mecha_aes_cbc_encrypt_data(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_AES_CBC_ENCRYPT_DATA_PARAMS_PTR param = mecha->pParameter; + CK_RV rv = CKR_GENERAL_ERROR; + uint32_t size = 0; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + size = sizeof(param->iv) + sizeof(uint32_t) + param->length; + rv = serialize_32b(obj, size); + if (rv) + return rv; + + rv = serialize_buffer(obj, param->iv, sizeof(param->iv)); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, param->length); + if (rv) + return rv; + + return serialize_buffer(obj, param->pData, param->length); +} + +static CK_RV serialize_mecha_rsa_pss_param(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_RSA_PKCS_PSS_PARAMS *params = mecha->pParameter; + CK_RV rv = CKR_GENERAL_ERROR; + uint32_t params_size = 3 * sizeof(uint32_t); + + if (mecha->ulParameterLen != sizeof(*params)) + return CKR_ARGUMENTS_BAD; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + rv = serialize_32b(obj, params_size); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, params->hashAlg); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, params->mgf); + if (rv) + return rv; + + return serialize_ck_ulong(obj, params->sLen); +} + +static CK_RV serialize_mecha_rsa_oaep_param(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_RSA_PKCS_OAEP_PARAMS *params = mecha->pParameter; + CK_RV rv = CKR_GENERAL_ERROR; + size_t params_size = 4 * sizeof(uint32_t) + params->ulSourceDataLen; + + if (mecha->ulParameterLen != sizeof(*params)) + return CKR_ARGUMENTS_BAD; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + rv = serialize_32b(obj, params_size); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, params->hashAlg); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, params->mgf); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, params->source); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, params->ulSourceDataLen); + if (rv) + return rv; + + return serialize_buffer(obj, params->pSourceData, + params->ulSourceDataLen); +} + +static CK_RV serialize_mecha_rsa_aes_key_wrap(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_RSA_AES_KEY_WRAP_PARAMS *params = mecha->pParameter; + CK_RSA_PKCS_OAEP_PARAMS *aes_params = params->pOAEPParams; + CK_RV rv = CKR_GENERAL_ERROR; + size_t params_size = 5 * sizeof(uint32_t) + aes_params->ulSourceDataLen; + + if (mecha->ulParameterLen != sizeof(*params)) + return CKR_ARGUMENTS_BAD; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + rv = serialize_32b(obj, params_size); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, params->ulAESKeyBits); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, aes_params->hashAlg); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, aes_params->mgf); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, aes_params->source); + if (rv) + return rv; + + rv = serialize_ck_ulong(obj, aes_params->ulSourceDataLen); + if (rv) + return rv; + + return serialize_buffer(obj, aes_params->pSourceData, + aes_params->ulSourceDataLen); +} + +static CK_RV serialize_mecha_eddsa(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_RV rv = CKR_GENERAL_ERROR; + CK_EDDSA_PARAMS *params = mecha->pParameter; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + rv = serialize_32b(obj, 2 * sizeof(uint32_t) + params->ulContextDataLen); + if (rv) + return rv; + + rv = serialize_32b(obj, params->phFlag); + if (rv) + return rv; + + rv = serialize_32b(obj, params->ulContextDataLen); + if (rv) + return rv; + + return serialize_buffer(obj, params->pContextData, params->ulContextDataLen); +} + +static CK_RV serialize_mecha_mac_general_param(struct serializer *obj, + CK_MECHANISM_PTR mecha) +{ + CK_RV rv = CKR_GENERAL_ERROR; + CK_ULONG ck_data = 0; + + if (mecha->ulParameterLen != sizeof(ck_data)) + return CKR_ARGUMENTS_BAD; + + memcpy(&ck_data, mecha->pParameter, mecha->ulParameterLen); + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + rv = serialize_32b(obj, sizeof(uint32_t)); + if (rv) + return rv; + + return serialize_ck_ulong(obj, ck_data); +} + +/** + * serialize_ck_mecha_params - serialize a mechanism type & params + * + * @obj - serializer used to track the serialization + * @mechanism - pointer of the in structure aligned CK_MECHANISM. + * + * Serialized content: + * [mechanism-type][mechanism-param-blob] + * + * [mechanism-param-blob] depends on mechanism type ID, see + * serialize_mecha_XXX(). + */ +CK_RV serialize_ck_mecha_params(struct serializer *obj, + CK_MECHANISM_PTR mechanism) +{ + CK_MECHANISM mecha = { 0 }; + CK_RV rv = CKR_GENERAL_ERROR; + + memset(obj, 0, sizeof(*obj)); + + obj->object = PKCS11_CKO_MECHANISM; + + mecha = *mechanism; + obj->type = mecha.mechanism; + if (obj->type == PKCS11_UNDEFINED_ID) + return CKR_MECHANISM_INVALID; + + switch (mecha.mechanism) { + case CKM_GENERIC_SECRET_KEY_GEN: + case CKM_AES_KEY_GEN: + case CKM_AES_ECB: + case CKM_AES_CMAC: + case CKM_MD5: + case CKM_SHA_1: + case CKM_SHA224: + case CKM_SHA256: + case CKM_SHA384: + case CKM_SHA512: + case CKM_MD5_HMAC: + case CKM_SHA_1_HMAC: + case CKM_SHA224_HMAC: + case CKM_SHA256_HMAC: + case CKM_SHA384_HMAC: + case CKM_SHA512_HMAC: + case CKM_EC_KEY_PAIR_GEN: + case CKM_EC_EDWARDS_KEY_PAIR_GEN: + case CKM_ECDSA: + case CKM_ECDSA_SHA1: + case CKM_ECDSA_SHA224: + case CKM_ECDSA_SHA256: + case CKM_ECDSA_SHA384: + case CKM_ECDSA_SHA512: + case CKM_RSA_PKCS_KEY_PAIR_GEN: + case CKM_RSA_PKCS: + case CKM_MD5_RSA_PKCS: + case CKM_SHA1_RSA_PKCS: + case CKM_SHA224_RSA_PKCS: + case CKM_SHA256_RSA_PKCS: + case CKM_SHA384_RSA_PKCS: + case CKM_SHA512_RSA_PKCS: + /* No parameter expected, size shall be 0 */ + if (mechanism->ulParameterLen) + return CKR_MECHANISM_PARAM_INVALID; + + rv = serialize_32b(obj, obj->type); + if (rv) + return rv; + + return serialize_32b(obj, 0); + + case CKM_EDDSA: + return serialize_mecha_eddsa(obj, &mecha); + + case CKM_AES_CBC: + case CKM_AES_CBC_PAD: + case CKM_AES_CTS: + return serialize_mecha_aes_iv(obj, &mecha); + + case CKM_AES_CTR: + return serialize_mecha_aes_ctr(obj, &mecha); + + case CKM_AES_ECB_ENCRYPT_DATA: + return serialize_mecha_key_deriv_str(obj, &mecha); + + case CKM_AES_CBC_ENCRYPT_DATA: + return serialize_mecha_aes_cbc_encrypt_data(obj, &mecha); + + case CKM_ECDH1_DERIVE: + case CKM_ECDH1_COFACTOR_DERIVE: + return serialize_mecha_ecdh1_derive_param(obj, &mecha); + + case CKM_RSA_PKCS_PSS: + case CKM_SHA1_RSA_PKCS_PSS: + case CKM_SHA256_RSA_PKCS_PSS: + case CKM_SHA384_RSA_PKCS_PSS: + case CKM_SHA512_RSA_PKCS_PSS: + case CKM_SHA224_RSA_PKCS_PSS: + return serialize_mecha_rsa_pss_param(obj, &mecha); + + case CKM_RSA_PKCS_OAEP: + return serialize_mecha_rsa_oaep_param(obj, &mecha); + + case CKM_AES_CMAC_GENERAL: + case CKM_MD5_HMAC_GENERAL: + case CKM_SHA_1_HMAC_GENERAL: + case CKM_SHA224_HMAC_GENERAL: + case CKM_SHA256_HMAC_GENERAL: + case CKM_SHA384_HMAC_GENERAL: + case CKM_SHA512_HMAC_GENERAL: + return serialize_mecha_mac_general_param(obj, &mecha); + case CKM_RSA_AES_KEY_WRAP: + return serialize_mecha_rsa_aes_key_wrap(obj, &mecha); + + default: + return CKR_MECHANISM_INVALID; + } +} diff --git a/optee_client/libckteec/src/serialize_ck.h b/optee_client/libckteec/src/serialize_ck.h new file mode 100644 index 0000000..f7a205a --- /dev/null +++ b/optee_client/libckteec/src/serialize_ck.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ +#ifndef LIBCKTEEC_SERIALIZE_CK_H +#define LIBCKTEEC_SERIALIZE_CK_H + +#include + +#include "serializer.h" + +/* Create (and allocate) a serial object for CK_ATTRIBUTE array */ +CK_RV serialize_ck_attributes(struct serializer *obj, + CK_ATTRIBUTE_PTR attributes, CK_ULONG count); + +/* Convert PKCS11 TA attributes back to CK_ATTRIBUTE array */ +CK_RV deserialize_ck_attributes(uint8_t *in, + CK_ATTRIBUTE_PTR attributes, CK_ULONG count); + +/* Create (and allocate) a serial object for CK_MECHANISM array */ +CK_RV serialize_ck_mecha_params(struct serializer *obj, + CK_MECHANISM_PTR mechanisms); +#endif /*LIBCKTEEC_SERIALIZE_CK_H*/ diff --git a/optee_client/libckteec/src/serializer.c b/optee_client/libckteec/src/serializer.c new file mode 100644 index 0000000..d94932b --- /dev/null +++ b/optee_client/libckteec/src/serializer.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include + +#include "ck_helpers.h" +#include "local_utils.h" +#include "serializer.h" + +CK_RV init_serial_object(struct serializer *obj) +{ + struct pkcs11_object_head head = { 0 }; + + memset(obj, 0, sizeof(*obj)); + + return serialize_buffer(obj, &head, sizeof(head)); +} + +void finalize_serial_object(struct serializer *obj) +{ + struct pkcs11_object_head head = { 0 }; + + head.attrs_size = obj->size - sizeof(head); + head.attrs_count = obj->item_count; + memcpy(obj->buffer, &head, sizeof(head)); +} + +void release_serial_object(struct serializer *obj) +{ + free(obj->buffer); + obj->buffer = NULL; +} + +/** + * serialize - append data in a serialized buffer + * + * Serialize data in provided buffer. + * Ensure 64byte alignment of appended data in the buffer. + */ +static CK_RV serialize(char **bstart, size_t *blen, void *data, size_t len) +{ + size_t nlen = *blen + len; + char *buf = realloc(*bstart, nlen); + + if (!buf) + return CKR_HOST_MEMORY; + + memcpy(buf + *blen, data, len); + + *blen = nlen; + *bstart = buf; + + return CKR_OK; +} + +CK_RV serialize_buffer(struct serializer *obj, void *data, size_t size) +{ + return serialize(&obj->buffer, &obj->size, data, size); +} + +CK_RV serialize_32b(struct serializer *obj, uint32_t data) +{ + return serialize_buffer(obj, &data, sizeof(data)); +} + +CK_RV serialize_ck_ulong(struct serializer *obj, CK_ULONG data) +{ + uint32_t data32 = data; + + return serialize_buffer(obj, &data32, sizeof(data32)); +} diff --git a/optee_client/libckteec/src/serializer.h b/optee_client/libckteec/src/serializer.h new file mode 100644 index 0000000..0c9264a --- /dev/null +++ b/optee_client/libckteec/src/serializer.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ +#ifndef LIBCKTEEC_SERIALIZER_H +#define LIBCKTEEC_SERIALIZER_H + +#include +#include +#include +#include + +/* + * Struct used to create the buffer storing the serialized data. + * Contains some fields to help parsing content (type/boolprops). + */ +struct serializer { + char *buffer; /* serial buffer base address */ + size_t size; /* serial buffer current byte size */ + size_t item_count; /* number of items in entry table */ + uint32_t object; + uint32_t type; +}; + +/* Init/finalize/release a serializer object */ +CK_RV init_serial_object(struct serializer *obj); +void finalize_serial_object(struct serializer *obj); +void release_serial_object(struct serializer *obj); + +CK_RV serialize_buffer(struct serializer *obj, void *data, size_t size); +CK_RV serialize_32b(struct serializer *obj, uint32_t data); +CK_RV serialize_ck_ulong(struct serializer *obj, CK_ULONG data); + +#endif /*LIBCKTEEC_SERIALIZER_H*/ diff --git a/optee_client/libseteec/CMakeLists.txt b/optee_client/libseteec/CMakeLists.txt new file mode 100644 index 0000000..cabf92f --- /dev/null +++ b/optee_client/libseteec/CMakeLists.txt @@ -0,0 +1,65 @@ +project(seteec + VERSION 0.1.0 + LANGUAGES C) + +################################################################################ +# Packages +################################################################################ +find_package(Threads REQUIRED) +if(NOT THREADS_FOUND) + message(FATAL_ERROR "Threads not found") +endif() + +include(GNUInstallDirs) + +################################################################################ +# Source files +################################################################################ +set(SRC + src/se.c +) + +################################################################################ +# Built library +################################################################################ +add_library(seteec ${SRC}) + +set_target_properties(seteec PROPERTIES + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR} +) + +################################################################################ +# Flags always set +################################################################################ +target_compile_definitions(seteec + PRIVATE -D_GNU_SOURCE + PRIVATE -DBINARY_PREFIX="LT" +) + +################################################################################ +# Optional flags +################################################################################ + +################################################################################ +# Public and private header and library dependencies +################################################################################ +target_include_directories(seteec + PUBLIC $ + $ + PRIVATE src +) + +target_link_libraries(seteec + PRIVATE pthread + PRIVATE teec +) + +################################################################################ +# Install targets +################################################################################ +install(TARGETS seteec + DESTINATION ${CMAKE_INSTALL_LIBDIR} +) + +add_subdirectory(include) diff --git a/optee_client/libseteec/Makefile b/optee_client/libseteec/Makefile new file mode 100644 index 0000000..ce8b217 --- /dev/null +++ b/optee_client/libseteec/Makefile @@ -0,0 +1,71 @@ +include ../flags.mk +include ../config.mk + +OUT_DIR := $(OO)/libseteec + +.PHONY: all libseteec clean + +all: libseteec +install: libseteec + +LIB_NAME := libseteec +MAJOR_VERSION := 0 +MINOR_VERSION := 1 +PATCH_VERSION := 0 + +LIB_MAJOR := $(LIB_NAME).so.$(MAJOR_VERSION) +LIB_MAJ_MIN := $(LIB_NAME).so.$(MAJOR_VERSION).$(MINOR_VERSION) +LIB_MAJ_MIN_PAT := $(LIB_NAME).so.$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION) +LIBSETEEC_SO_LIBRARY := $(LIB_MAJ_MIN_PAT) +LIBSETEEC_AR_LIBRARY := $(LIB_NAME).a + +LIBSETEEC_SRC_DIR := src + +LIBSETEEC_SRCS = se.c + +LIBSETEEC_INCLUDES = ${CURDIR}/include +LIBSETEEC_INCLUDES += ${CURDIR}/../libteec/include + +LIBSETEEC_CFLAGS := $(addprefix -I, $(LIBSETEEC_INCLUDES)) \ + $(CFLAGS) -D_GNU_SOURCE -fPIC + +LIBSETEEC_LFLAGS := $(LDFLAGS) -L$(OUT_DIR)/../libteec -lteec -lpthread + +LIBSETEEC_OBJ_DIR := $(OUT_DIR) +LIBSETEEC_OBJS := $(patsubst %.c,$(LIBSETEEC_OBJ_DIR)/%.o, $(LIBSETEEC_SRCS)) + +$(LIBSETEEC_OBJ_DIR)/%.o: ${LIBSETEEC_SRC_DIR}/%.c + $(VPREFIX)mkdir -p $(LIBSETEEC_OBJ_DIR) + @echo " CC $<" + $(VPREFIX)$(CC) $(LIBSETEEC_CFLAGS) -c $< -o $@ + +libseteec: $(OUT_DIR)/$(LIBSETEEC_SO_LIBRARY) + +$(OUT_DIR)/$(LIBSETEEC_SO_LIBRARY): $(LIBSETEEC_OBJS) + @echo " LINK $@" + $(VPREFIX)$(CC) -shared -Wl,-soname,$(LIB_MAJOR) -o $@ $+ $(LIBSETEEC_LFLAGS) + @echo "" + +libseteec: $(OUT_DIR)/$(LIBSETEEC_AR_LIBRARY) + +$(OUT_DIR)/$(LIBSETEEC_AR_LIBRARY): $(LIBSETEEC_OBJS) + @echo " AR $@" + $(VPREFIX)$(AR) rcs $@ $+ + +libseteec: + $(VPREFIX)ln -sf $(LIB_MAJ_MIN_PAT) $(OUT_DIR)/$(LIB_MAJ_MIN) + $(VPREFIX)ln -sf $(LIB_MAJ_MIN) $(OUT_DIR)/$(LIB_MAJOR) + $(VPREFIX)ln -sf $(LIB_MAJOR) $(OUT_DIR)/$(LIB_NAME).so + +################################################################################ +# Cleaning up configuration +################################################################################ +clean: + $(RM) $(LIBSETEEC_OBJS) + $(RM) $(OUT_DIR)/$(LIB_MAJ_MIN_PAT) + $(RM) $(OUT_DIR)/$(LIB_MAJ_MIN) + $(RM) $(OUT_DIR)/$(LIB_MAJOR) + $(RM) $(OUT_DIR)/$(LIBSETEEC_SO_LIBRARY) + $(RM) $(OUT_DIR)/$(LIBSETEEC_AR_LIBRARY) + $(call rmdir,$(OUT_DIR)) + diff --git a/optee_client/libseteec/include/CMakeLists.txt b/optee_client/libseteec/include/CMakeLists.txt new file mode 100644 index 0000000..ee6fadd --- /dev/null +++ b/optee_client/libseteec/include/CMakeLists.txt @@ -0,0 +1,7 @@ +project(libseteec-headers C) + +FILE(GLOB INSTALL_HEADERS "*.h") + +add_library(${PROJECT_NAME} INTERFACE) + +install(FILES ${INSTALL_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/optee_client/libseteec/include/se_tee.h b/optee_client/libseteec/include/se_tee.h new file mode 100644 index 0000000..4da2342 --- /dev/null +++ b/optee_client/libseteec/include/se_tee.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Foundries.io + * Jorge Ramirez-Ortiz + */ + +#ifndef SE_TEE_H +#define SE_TEE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned long SE_ULONG; +typedef SE_ULONG SE_RV; + +/* Values for type SR_RV */ +#define SER_OK 0x0000 +#define SER_CANT_OPEN_SESSION 0x0001 +#define SER_ERROR_GENERIC 0x0002 + +/* + * Type identifier for the APDU message as described by Smart Card Standard ISO7816-4 + * about ADPU message bodies decoding convention: + * + * https://cardwerk.com/smart-card-standard-iso7816-4-section-5-basic-organizations/#chap5_3_2 + */ +enum se_apdu_type { + SE_APDU_NO_HINT, + SE_APDU_CASE_1, + SE_APDU_CASE_2, + SE_APDU_CASE_2E, + SE_APDU_CASE_3, + SE_APDU_CASE_3E, + SE_APDU_CASE_4, + SE_APDU_CASE_4E, +}; + +/** + * se_apdu_request() - Send an APDU message and get response. + * + * @param type Type of the APDU command. + * @param hdr Pointer to APDU message header. + * @param hdr_len Byte length of message header @hdr. + * @param src Pointer to APDU message payload. + * @param src_len Byte length of message payload @src. + * @param dst Pointer to APDU message reponse buffer. + * @param dst_len Byte length of reponse buffer @dst. + * + * @return SER_CANT_OPEN_SESSION Error opening the TEE session. + * @return SER_ERROR_GENERIC Error unspecified. + * @return SER_OK On success. + */ +SE_RV se_apdu_request(enum se_apdu_type type, + unsigned char *hdr, size_t hdr_len, + unsigned char *src, size_t src_len, + unsigned char *dst, size_t *dst_len); + +/** + * se_scp03_enable() - Enable the SCP03 protocol using the keys active in the + * Secure Element. + * + * Enables the SCP03 session with the Secure Element. + * + * @return SER_CANT_OPEN_SESSION Error opening the TEE session. + * @return SER_ERROR_GENERIC Error unspecified. + * @return SER_OK On success. + */ +SE_RV se_scp03_enable(void); + +/** + * se_scp03_rotate_keys_and_enable() - Attempt to replace the active SCP03 keys + * and enable the SCP03 session. + * + * Generates secure keys for the board and writes them in the Secure Element non + * volatile memory. Then re-enables the session. + * + * @return SER_CANT_OPEN_SESSION Error opening the TEE session. + * @return SER_ERROR_GENERIC Error unspecified. + * @return SER_OK On success. + */ +SE_RV se_scp03_rotate_keys_and_enable(void); + +#ifdef __cplusplus +} +#endif + +#endif /*SE_TEE_H*/ diff --git a/optee_client/libseteec/src/pta_apdu.h b/optee_client/libseteec/src/pta_apdu.h new file mode 100644 index 0000000..8fed78f --- /dev/null +++ b/optee_client/libseteec/src/pta_apdu.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (C) 2021, Foundries Limited + * Author: Jorge Ramirez + */ + +#ifndef __PTA_APDU_H +#define __PTA_APDU_H + +#define PTA_APDU_UUID { 0x3f3eb880, 0x3639, 0x11ec, \ + { 0x9b, 0x9d, 0x0f, 0x3f, 0xc9, 0x46, 0x8f, 0x50 } } + +/* + * ADPU based communication with the Secure Element + * + * [in] value[0].a Use APDU TXRX hints: PTA_APDU_TXRX_CASE_* + * [in] memref[1].buffer APDU header. + * [in] memref[1].size APDU header length. + * [in] memref[2].buffer request (APDU raw frame). + * [in] memref[2].size request length. + * [out] memref[3].buffer response (APDU raw frame). + * [out] memref[3].size response length. + * + * Result: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_NOT_IMPLEMENTED - Invoke command not implemented + * TEE_ERROR_GENERIC - Invoke command failure + */ +#define PTA_CMD_TXRX_APDU_RAW_FRAME 0 + +/* + * Type identifier for the APDU message as described by Smart Card Standard + * ISO7816-4 about ADPU message bodies decoding convention: + * + * https://cardwerk.com/smart-card-standard-iso7816-4-section-5-basic-organizations/#chap5_3_2 + */ +#define PTA_APDU_TXRX_CASE_NO_HINT 0 +#define PTA_APDU_TXRX_CASE_1 1 +#define PTA_APDU_TXRX_CASE_2 2 +#define PTA_APDU_TXRX_CASE_2E 3 +#define PTA_APDU_TXRX_CASE_3 4 +#define PTA_APDU_TXRX_CASE_3E 5 +#define PTA_APDU_TXRX_CASE_4 6 +#define PTA_APDU_TXRX_CASE_4E 7 + +#endif /* __PTA_APDU_H */ diff --git a/optee_client/libseteec/src/pta_scp03.h b/optee_client/libseteec/src/pta_scp03.h new file mode 100644 index 0000000..393ca56 --- /dev/null +++ b/optee_client/libseteec/src/pta_scp03.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (C) 2020, Foundries Limited + */ + +#ifndef __PTA_SCP03_H +#define __PTA_SCP03_H + +#define PTA_SCP03_UUID { 0xbe0e5821, 0xe718, 0x4f77, \ + { 0xab, 0x3e, 0x8e, 0x6c, 0x73, 0xa9, 0xc7, 0x35 } } + +/* + * Enable SCP03 support on the SE + * + * [in] value[0].a Use session keys PTA_SCP03_SESSION_* + * + * Result: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_NOT_IMPLEMENTED - Invoke command not implemented + * TEE_ERROR_GENERIC - Invoke command failure + */ +#define PTA_CMD_ENABLE_SCP03 0 + +/* Enable the session using the current keys in the Secure Element */ +#define PTA_SCP03_SESSION_CURRENT_KEYS 0 + +/* Enable the session after replacing the current keys in the Secure Element */ +#define PTA_SCP03_SESSION_ROTATE_KEYS 1 + +#endif /* __PTA_SCP03_H */ diff --git a/optee_client/libseteec/src/se.c b/optee_client/libseteec/src/se.c new file mode 100644 index 0000000..4f6be5a --- /dev/null +++ b/optee_client/libseteec/src/se.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Foundries.io Ltd + */ + +#ifndef BINARY_PREFIX +#define BINARY_PREFIX "seteec" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pta_apdu.h" +#include "pta_scp03.h" + +struct ta_context { + pthread_mutex_t lock; + TEEC_Context context; + TEEC_Session session; + TEEC_UUID uuid; + bool open; +}; + +static struct ta_context apdu_ta_ctx = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .uuid = PTA_APDU_UUID, +}; + +static struct ta_context scp03_ta_ctx = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .uuid = PTA_SCP03_UUID, +}; + +static bool open_session(struct ta_context *ctx) +{ + TEEC_Result res = TEEC_SUCCESS; + + if (pthread_mutex_lock(&ctx->lock)) + return false; + + if (!ctx->open) { + res = TEEC_InitializeContext(NULL, &ctx->context); + if (!res) { + res = TEEC_OpenSession(&ctx->context, &ctx->session, + &ctx->uuid, TEEC_LOGIN_PUBLIC, + NULL, NULL, NULL); + if (!res) + ctx->open = true; + } + } + + return !pthread_mutex_unlock(&ctx->lock) && !res; +} + +static SE_RV do_scp03(uint32_t cmd) +{ + TEEC_Operation op = { 0 }; + + if (!open_session(&scp03_ta_ctx)) + return SER_CANT_OPEN_SESSION; + + op.paramTypes = TEEC_PARAM_TYPES(TEEC_VALUE_INPUT, 0, 0, 0); + op.params[0].value.a = cmd; + + if (TEEC_InvokeCommand(&scp03_ta_ctx.session, + PTA_CMD_ENABLE_SCP03, &op, NULL)) + return SER_ERROR_GENERIC; + + return SER_OK; +} + +SE_RV se_scp03_enable(void) +{ + return do_scp03(PTA_SCP03_SESSION_CURRENT_KEYS); +} + +SE_RV se_scp03_rotate_keys_and_enable(void) +{ + return do_scp03(PTA_SCP03_SESSION_ROTATE_KEYS); +} + +SE_RV se_apdu_request(enum se_apdu_type apdu_type, + unsigned char *hdr, size_t hdr_len, + unsigned char *src, size_t src_len, + unsigned char *dst, size_t *dst_len) +{ + uint32_t type = PTA_APDU_TXRX_CASE_NO_HINT; + TEEC_Operation op = { 0 }; + + switch (apdu_type) { + case SE_APDU_NO_HINT: + type = PTA_APDU_TXRX_CASE_NO_HINT; + break; + case SE_APDU_CASE_1: + type = PTA_APDU_TXRX_CASE_1; + break; + case SE_APDU_CASE_2: + type = PTA_APDU_TXRX_CASE_2; + break; + case SE_APDU_CASE_2E: + type = PTA_APDU_TXRX_CASE_2E; + break; + case SE_APDU_CASE_3: + type = PTA_APDU_TXRX_CASE_3; + break; + case SE_APDU_CASE_3E: + type = PTA_APDU_TXRX_CASE_3E; + break; + case SE_APDU_CASE_4: + type = PTA_APDU_TXRX_CASE_4; + break; + case SE_APDU_CASE_4E: + type = PTA_APDU_TXRX_CASE_4E; + break; + default: + return SER_ERROR_GENERIC; + } + + if (!open_session(&apdu_ta_ctx)) + return SER_CANT_OPEN_SESSION; + + op.paramTypes = TEEC_PARAM_TYPES(TEEC_VALUE_INPUT, + TEEC_MEMREF_TEMP_INPUT, + TEEC_MEMREF_TEMP_INPUT, + TEEC_MEMREF_TEMP_OUTPUT); + op.params[0].value.a = type; + op.params[1].tmpref.buffer = hdr; + op.params[1].tmpref.size = hdr_len; + op.params[2].tmpref.buffer = src; + op.params[2].tmpref.size = src_len; + op.params[3].tmpref.buffer = dst; + op.params[3].tmpref.size = *dst_len; + + if (TEEC_InvokeCommand(&apdu_ta_ctx.session, + PTA_CMD_TXRX_APDU_RAW_FRAME, &op, NULL)) + return SER_ERROR_GENERIC; + + *dst_len = op.params[3].tmpref.size; + + return SER_OK; +} diff --git a/optee_client/libteeacl/CMakeLists.txt b/optee_client/libteeacl/CMakeLists.txt new file mode 100644 index 0000000..ce78058 --- /dev/null +++ b/optee_client/libteeacl/CMakeLists.txt @@ -0,0 +1,53 @@ +project(teeacl + VERSION 0.1.0 + LANGUAGES C +) + +include(GNUInstallDirs) + +################################################################################ +# Source files +################################################################################ +set(SRC + src/group.c + src/tee_uuid.c +) + +################################################################################ +# Built library +################################################################################ +add_library(teeacl ${SRC}) + +set_target_properties(teeacl PROPERTIES + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR} +) + +################################################################################ +# Flags always set +################################################################################ +target_compile_definitions(teeacl + PRIVATE -D_GNU_SOURCE + PRIVATE -DBINARY_PREFIX="LT" +) + +################################################################################ +# Public and private header and library dependencies +################################################################################ +target_include_directories(teeacl PUBLIC + $ + $ +) + +target_link_libraries(teeacl + PUBLIC PkgConfig::uuid +) + +################################################################################ +# Install targets +################################################################################ +install(TARGETS teeacl + DESTINATION ${CMAKE_INSTALL_LIBDIR} +) + +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") diff --git a/optee_client/libteeacl/Makefile b/optee_client/libteeacl/Makefile new file mode 100644 index 0000000..fe50c51 --- /dev/null +++ b/optee_client/libteeacl/Makefile @@ -0,0 +1,77 @@ +include ../flags.mk +include ../config.mk + +OUT_DIR := $(OO)/libteeacl + +.PHONY: all libteeacl check-libuuid clean + +all: libteeacl +install: libteeacl + +LIB_NAME := libteeacl +MAJOR_VERSION := 0 +MINOR_VERSION := 1 +PATCH_VERSION := 0 + +LIB_MAJOR := $(LIB_NAME).so.$(MAJOR_VERSION) +LIB_MAJ_MIN := $(LIB_NAME).so.$(MAJOR_VERSION).$(MINOR_VERSION) +LIB_MAJ_MIN_PAT := $(LIB_NAME).so.$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION) +LIBTEEACL_SO_LIBRARY := $(LIB_MAJ_MIN_PAT) +LIBTEEACL_AR_LIBRARY := $(LIB_NAME).a + +LIBTEEACL_SRC_DIR := src + +LIBTEEACL_SRCS = group.c +LIBTEEACL_SRCS += tee_uuid.c + +LIBTEEACL_INCLUDES = ${CURDIR}/include + +LIBTEEACL_CFLAGS := $(addprefix -I, $(LIBTEEACL_INCLUDES)) \ + $(shell $(PKG_CONFIG) --cflags uuid) \ + $(CFLAGS) -D_GNU_SOURCE -fPIC + +LIBTEEACL_LFLAGS := $(LDFLAGS) $(shell $(PKG_CONFIG) --libs uuid) + +LIBTEEACL_OBJ_DIR := $(OUT_DIR) +LIBTEEACL_OBJS := $(patsubst %.c,$(LIBTEEACL_OBJ_DIR)/%.o, $(LIBTEEACL_SRCS)) + +$(LIBTEEACL_OBJ_DIR)/%.o: ${LIBTEEACL_SRC_DIR}/%.c + $(VPREFIX)mkdir -p $(LIBTEEACL_OBJ_DIR) + @echo " CC $<" + $(VPREFIX)$(CC) $(LIBTEEACL_CFLAGS) -c $< -o $@ + +libteeacl: check-libuuid + +check-libuuid: + @echo " Finding uuid.pc" + $(VPREFIX)$(PKG_CONFIG) --atleast-version=2.34 uuid + +libteeacl: $(OUT_DIR)/$(LIBTEEACL_SO_LIBRARY) + +$(OUT_DIR)/$(LIBTEEACL_SO_LIBRARY): $(LIBTEEACL_OBJS) + @echo " LINK $@" + $(VPREFIX)$(CC) -shared -Wl,-soname,$(LIBTEEACL_SO_LIBRARY) -o $@ $+ $(LIBTEEACL_LFLAGS) + @echo "" + +libteeacl: $(OUT_DIR)/$(LIBTEEACL_AR_LIBRARY) + +$(OUT_DIR)/$(LIBTEEACL_AR_LIBRARY): $(LIBTEEACL_OBJS) + @echo " AR $@" + $(VPREFIX)$(AR) rcs $@ $+ + +libteeacl: + $(VPREFIX)ln -sf $(LIB_MAJ_MIN_PAT) $(OUT_DIR)/$(LIB_MAJ_MIN) + $(VPREFIX)ln -sf $(LIB_MAJ_MIN) $(OUT_DIR)/$(LIB_MAJOR) + $(VPREFIX)ln -sf $(LIB_MAJOR) $(OUT_DIR)/$(LIB_NAME).so + +################################################################################ +# Cleaning up configuration +################################################################################ +clean: + $(RM) $(LIBTEEACL_OBJS) + $(RM) $(OUT_DIR)/$(LIB_MAJ_MIN_PAT) + $(RM) $(OUT_DIR)/$(LIB_MAJ_MIN) + $(RM) $(OUT_DIR)/$(LIB_MAJOR) + $(RM) $(OUT_DIR)/$(LIBTEEACL_SO_LIBRARY) + $(RM) $(OUT_DIR)/$(LIBTEEACL_AR_LIBRARY) + $(call rmdir,$(OUT_DIR)) diff --git a/optee_client/libteeacl/include/teeacl.h b/optee_client/libteeacl/include/teeacl.h new file mode 100644 index 0000000..1d7a5ee --- /dev/null +++ b/optee_client/libteeacl/include/teeacl.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Vaisala Oyj. + */ + +/* + * Definitions for configuring and using Access Control List (ACL) + * based login methods. + */ + +#ifndef TEEACL_H +#define TEEACL_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// TEE Client UUID name space identifier (UUIDv4) +// same as `tee_client_uuid_ns` in linux kernel drivers/tee/tee_core.c +#define KERNEL_NAMESPACE "58ac9ca0-2086-4683-a1b8-ec4bc08e01b6" + +/* + * len UUID = 36 characters + * + * Prefixes: + * public + * user: + * group: + * + * + '\0' character totals 43, roundup. + */ + +/** + * Required length for UUID char buffers + */ +#define TEEACL_L_UUID 48 + +/** + * teeacl_gid_from_name - Try to resolve gid_t for a given `group_name`. + * + * If a matching group is found, zero is returned and `gid_out` will be set to + * the found value. + * If no group is found, -ENOENT is returned. + * If memory allocation fails, -ENOMEM is returned. + * For other failures, errno is returned. + * + * @param gid_out Ptr to gid result. Will be set to group id if a matching + * group is found. + * @param group_name Name of group to resolve. + * @return 0 if a matching group is found, see detailed description for other + * cases. + */ +int teeacl_gid_from_name(gid_t *gid_out, const char *group_name); + +/** + * teeacl_group_acl_uuid() - Encode a group login ACL string to the + * provided uuid_buf + * + * @param uuid_buf A buffer of length TEEACL_L_UUID. + * @param group Group id to encode for login. + * @return 0 on success, otherwise a negative number is returned in case of failure. + */ +int teeacl_group_acl_uuid(char uuid_buf[TEEACL_L_UUID], gid_t group); +/** + * The possible return values of the *_user_is_member_of functions. + */ +enum rv_groupmember { + NOT_MEMBER, + IS_MEMBER, + E_MEMORY, /**< Failed to allocate memory. */ + E_GROUPLIST /**< Failed to read group listing. */ +}; + +/** + * teeacl_current_user_is_member_of() - Check if the effective user ID of + * the process is a member in `group`. + * + * @param group Group id to check membership of. + * @return enum rv_groupmember form result. + */ +enum rv_groupmember teeacl_current_user_is_member_of(gid_t group); + +/** + * teeacl_user_is_member_of() - Check if `user` is a member in `group`. + * + * @param user Username string. + * @param group Group id to check membership of. + * @return enum rv_groupmember form result. + */ +enum rv_groupmember teeacl_user_is_member_of(const char *user, gid_t group); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* TEEACL_H */ diff --git a/optee_client/libteeacl/src/group.c b/optee_client/libteeacl/src/group.c new file mode 100644 index 0000000..bd165a1 --- /dev/null +++ b/optee_client/libteeacl/src/group.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Vaisala Oyj. + */ + +#include + +#include +#include +#include +#include +#include +#include + +static long teeacl_getgr_r_size_max(void) +{ + long s = sysconf(_SC_GETGR_R_SIZE_MAX); + + if (s == -1) + return 1024; + return s; +}; + +int teeacl_gid_from_name(gid_t *gid_out, const char *group_name) +{ + struct group grp = { 0 }; + char *buffer = NULL; + struct group *result = NULL; + size_t b_size = 0; + int rv = 0; + + b_size = teeacl_getgr_r_size_max(); + buffer = calloc(1, b_size); + if (!buffer) + return -ENOMEM; + + rv = getgrnam_r(group_name, &grp, buffer, b_size, &result); + + free(buffer); + if (!result) { + if (rv == 0) + return -ENOENT; + else + return rv; + } else { + *gid_out = grp.gr_gid; + return 0; + } +} + +enum rv_groupmember teeacl_current_user_is_member_of(gid_t group) +{ + char username[L_cuserid] = { 0 }; + + cuserid(username); + return teeacl_user_is_member_of(username, group); +} + +enum rv_groupmember teeacl_user_is_member_of(const char *user, gid_t group) +{ + enum rv_groupmember result = E_MEMORY; + int ret = 0; + int i = 0; + int grouplistsize = 8; /* initial guess */ + gid_t *p_groups = NULL; + gid_t *groups = calloc(grouplistsize, sizeof(gid_t)); + + if (!groups) + return E_MEMORY; + ret = getgrouplist(user, group, groups, &grouplistsize); + + if (ret == -1) { + p_groups = groups; + + /* we use realloc, since uClibc does not implement reallocarray */ + groups = realloc(groups, grouplistsize * sizeof(gid_t)); + if (!groups) { + free(p_groups); + return E_MEMORY; + } + ret = getgrouplist(user, group, groups, &grouplistsize); + if (ret == -1) { + result = E_GROUPLIST; + goto out; + } + } + + for (i = 0; i < grouplistsize; ++i) { + if (group == groups[i]) { + result = IS_MEMBER; + goto out; + } + } + result = NOT_MEMBER; +out: + free(groups); + return result; +} diff --git a/optee_client/libteeacl/src/tee_uuid.c b/optee_client/libteeacl/src/tee_uuid.c new file mode 100644 index 0000000..8c727e2 --- /dev/null +++ b/optee_client/libteeacl/src/tee_uuid.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Vaisala Oyj. + */ + +#include +#include +#include +#include + +int teeacl_group_acl_uuid(char uuid_buf[TEEACL_L_UUID], gid_t group) +{ + uuid_t g_uuid = { 0 }; + uuid_t k_uuid = { 0 }; + char gid_buf[TEEACL_L_UUID] = { 0 }; + size_t gid_buf_len = 0; + uint gstr_len = 6; + int rv = snprintf(gid_buf, TEEACL_L_UUID, "gid=%x", group); + + if (rv < 0) + return rv; + if (rv >= TEEACL_L_UUID) + return -1; + rv = uuid_parse(KERNEL_NAMESPACE, k_uuid); + if (rv < 0) + return rv; + + gid_buf_len = strnlen(gid_buf, TEEACL_L_UUID); + + uuid_generate_sha1(g_uuid, k_uuid, gid_buf, gid_buf_len); + + memcpy(uuid_buf, "group:", gstr_len); + uuid_unparse(g_uuid, uuid_buf + gstr_len); + return 0; +} diff --git a/optee_client/libteec/CMakeLists.txt b/optee_client/libteec/CMakeLists.txt new file mode 100644 index 0000000..0c73eba --- /dev/null +++ b/optee_client/libteec/CMakeLists.txt @@ -0,0 +1,89 @@ +project(libteec + VERSION 1.0.0 + LANGUAGES C) + +################################################################################ +# Packages +################################################################################ +find_package(Threads REQUIRED) +if(NOT THREADS_FOUND) + message(FATAL_ERROR "Threads not found") +endif() + +################################################################################ +# Configuration flags always included +################################################################################ +option(CFG_TEE_BENCHMARK "Build with benchmark support" OFF) + +set(CFG_TEE_CLIENT_LOG_LEVEL "1" CACHE STRING "libteec log level") +set(CFG_TEE_CLIENT_LOG_FILE "/data/tee/teec.log" CACHE STRING "Location of libteec log") + +################################################################################ +# Source files +################################################################################ +set(SRC + src/tee_client_api.c + src/teec_trace.c +) + +if(CFG_TEE_BENCHMARK) + set(SRC ${SRC} src/teec_benchmark.c) +endif() + +################################################################################ +# Built library +################################################################################ +add_library(teec ${SRC}) + +set(libteectgt teec) +configure_file(libteec.pc.in libteec.pc @ONLY) + +set_target_properties(teec PROPERTIES + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR} +) + +################################################################################ +# Flags always set +################################################################################ +target_compile_definitions(teec + PRIVATE -D_GNU_SOURCE + PRIVATE -DDEBUGLEVEL_${CFG_TEE_CLIENT_LOG_LEVEL} + PRIVATE -DTEEC_LOG_FILE="${CFG_TEE_CLIENT_LOG_FILE}" + PRIVATE -DBINARY_PREFIX="LT" +) + +################################################################################ +# Optional flags +################################################################################ +if(CFG_TEE_BENCHMARK) + target_compile_definitions(teec PRIVATE -DCFG_TEE_BENCHMARK) +endif() + +################################################################################ +# Public and private header and library dependencies +################################################################################ +target_include_directories(teec + PUBLIC include + PRIVATE src +) + +target_link_libraries(teec + PUBLIC ${CMAKE_THREAD_LIBS_INIT} # this is pthreads +) + +################################################################################ +# Install targets +################################################################################ +# FIXME: This should in someway harmonize with CFG_TEE_CLIENT_LOAD_PATH +# FIXME: Should we change this to /usr/local/lib? +install(TARGETS teec LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +install(DIRECTORY include + DESTINATION ${CMAKE_INSTALL_PREFIX} +) + +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libteec.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" +) diff --git a/optee_client/libteec/Makefile b/optee_client/libteec/Makefile new file mode 100644 index 0000000..41a21bb --- /dev/null +++ b/optee_client/libteec/Makefile @@ -0,0 +1,70 @@ +include ../flags.mk +include ../config.mk + +OUT_DIR := $(OO)/libteec + +.PHONY: all libteec clean + +all: libteec +################################################################################ +# Teec configuration +################################################################################ +MAJOR_VERSION := 1 +MINOR_VERSION := 0 +PATCH_VERSION := 0 +LIB_NAME := libteec.so +LIB_MAJOR := $(LIB_NAME).$(MAJOR_VERSION) +LIB_MAJ_MIN := $(LIB_NAME).$(MAJOR_VERSION).$(MINOR_VERSION) +LIB_MAJ_MIN_P := $(LIB_NAME).$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION) + +TEEC_SRCS := tee_client_api.c \ + teec_trace.c +ifeq ($(CFG_TEE_BENCHMARK),y) +TEEC_SRCS += teec_benchmark.c +endif + +TEEC_SRC_DIR := src +TEEC_OBJ_DIR := $(OUT_DIR) +TEEC_OBJS := $(patsubst %.c,$(TEEC_OBJ_DIR)/%.o, $(TEEC_SRCS)) +TEEC_INCLUDES := \ + ${CURDIR}/include \ + ${CURDIR}/src + +TEEC_CFLAGS := $(addprefix -I, $(TEEC_INCLUDES)) $(CFLAGS) -D_GNU_SOURCE \ + -DDEBUGLEVEL_$(CFG_TEE_CLIENT_LOG_LEVEL) \ + -DBINARY_PREFIX=\"TEEC\" + +ifeq ($(CFG_TEE_BENCHMARK),y) +TEEC_CFLAGS += -DCFG_TEE_BENCHMARK +endif + +TEEC_LFLAGS := $(LDFLAGS) -lpthread +TEEC_LIBRARY := $(OUT_DIR)/$(LIB_MAJ_MIN_P) + +libteec: $(TEEC_LIBRARY) $(OUT_DIR)/libteec.a + $(VPREFIX)ln -sf $(LIB_MAJ_MIN_P) $(OUT_DIR)/$(LIB_MAJOR) + $(VPREFIX)ln -sf $(LIB_MAJ_MIN_P) $(OUT_DIR)/$(LIB_MAJ_MIN) + $(VPREFIX)ln -sf $(LIB_MAJOR) $(OUT_DIR)/$(LIB_NAME) + +$(TEEC_LIBRARY): $(TEEC_OBJS) + @echo " LINK $@" + $(VPREFIX)$(CC) -shared -Wl,-soname,$(LIB_MAJOR) -o $@ $+ $(TEEC_LFLAGS) + @echo "" + +$(OUT_DIR)/libteec.a: $(TEEC_OBJS) + @echo " AR $@" + $(VPREFIX)$(AR) rcs $@ $+ + +$(TEEC_OBJ_DIR)/%.o: ${TEEC_SRC_DIR}/%.c + $(VPREFIX)mkdir -p $(TEEC_OBJ_DIR) + @echo " CC $<" + $(VPREFIX)$(CC) $(TEEC_CFLAGS) -c $< -o $@ + +################################################################################ +# Cleaning up configuration +################################################################################ +clean: + $(RM) $(TEEC_OBJS) $(TEEC_LIBRARY) $(OUT_DIR)/$(LIB_MAJOR) \ + $(OUT_DIR)/$(LIB_MAJ_MIN) $(OUT_DIR)/$(LIB_NAME) + $(RM) $(OUT_DIR)/libteec.a + $(call rmdir,$(OUT_DIR)) diff --git a/optee_client/libteec/include/tee_bench.h b/optee_client/libteec/include/tee_bench.h new file mode 100644 index 0000000..600407f --- /dev/null +++ b/optee_client/libteec/include/tee_bench.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEE_BENCH_H +#define TEE_BENCH_H + +#include + +#define PTA_BENCHMARK_UUID \ + { 0x0b9a63b0, 0xb4c6, 0x4c85, \ + { 0xa2, 0x84, 0xa2, 0x28, 0xef, 0x54, 0x7b, 0x4e } } + +#define BENCHMARK_CMD(id) (0xFA190000 | ((id) & 0xFFFF)) +#define BENCHMARK_CMD_REGISTER_MEMREF BENCHMARK_CMD(1) +#define BENCHMARK_CMD_GET_MEMREF BENCHMARK_CMD(2) +#define BENCHMARK_CMD_UNREGISTER BENCHMARK_CMD(3) + +/* + * Cycle count divider is enabled (in PMCR), + * CCNT value is incremented every 64th clock cycle + */ +#define TEE_BENCH_DIVIDER 64 +/* max amount of timestamps per buffer */ +#define TEE_BENCH_MAX_STAMPS 32 +#define TEE_BENCH_MAX_MASK (TEE_BENCH_MAX_STAMPS - 1) + +/* OP-TEE susbsystems ids */ +#define TEE_BENCH_CLIENT 0x10000000 +#define TEE_BENCH_KMOD 0x20000000 +#define TEE_BENCH_CORE 0x30000000 +#define TEE_BENCH_UTEE 0x40000000 +#define TEE_BENCH_DUMB_TA 0xF0000001 + +/* storing timestamp */ +struct tee_time_st { + uint64_t cnt; /* stores value from CNTPCT register */ + uint64_t addr; /* stores value from program counter register */ + uint64_t src; /* OP-TEE subsystem id */ +}; + +/* per-cpu circular buffer for timestamps */ +struct tee_ts_cpu_buf { + uint64_t head; + uint64_t tail; + struct tee_time_st stamps[TEE_BENCH_MAX_STAMPS]; +}; + +/* memory layout for shared memory, where timestamps will be stored */ +struct tee_ts_global { + uint64_t cores; + struct tee_ts_cpu_buf cpu_buf[]; +}; +#endif /* TEE_BENCH_H */ diff --git a/optee_client/libteec/include/tee_client_api.h b/optee_client/libteec/include/tee_client_api.h new file mode 100644 index 0000000..1693998 --- /dev/null +++ b/optee_client/libteec/include/tee_client_api.h @@ -0,0 +1,555 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef TEE_CLIENT_API_H +#define TEE_CLIENT_API_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +/* + * Defines the number of available memory references in an open session or + * invoke command operation payload. + */ +#define TEEC_CONFIG_PAYLOAD_REF_COUNT 4 + +/** + * Defines the maximum size of a single shared memory block, in bytes, of both + * API allocated and API registered memory. There is no good value to put here + * (limits depend on specific config used), so this define does not provide any + * restriction in this implementation. + */ +#define TEEC_CONFIG_SHAREDMEM_MAX_SIZE ULONG_MAX + +/** + * Flag constants indicating the type of parameters encoded inside the + * operation payload (TEEC_Operation), Type is uint32_t. + * + * TEEC_NONE The Parameter is not used + * + * TEEC_VALUE_INPUT The Parameter is a TEEC_Value tagged as input. + * + * TEEC_VALUE_OUTPUT The Parameter is a TEEC_Value tagged as output. + * + * TEEC_VALUE_INOUT The Parameter is a TEEC_Value tagged as both as + * input and output, i.e., for which both the + * behaviors of TEEC_VALUE_INPUT and + * TEEC_VALUE_OUTPUT apply. + * + * TEEC_MEMREF_TEMP_INPUT The Parameter is a TEEC_TempMemoryReference + * describing a region of memory which needs to be + * temporarily registered for the duration of the + * Operation and is tagged as input. + * + * TEEC_MEMREF_TEMP_OUTPUT Same as TEEC_MEMREF_TEMP_INPUT, but the Memory + * Reference is tagged as output. The + * Implementation may update the size field to + * reflect the required output size in some use + * cases. + * + * TEEC_MEMREF_TEMP_INOUT A Temporary Memory Reference tagged as both + * input and output, i.e., for which both the + * behaviors of TEEC_MEMREF_TEMP_INPUT and + * TEEC_MEMREF_TEMP_OUTPUT apply. + * + * TEEC_MEMREF_WHOLE The Parameter is a Registered Memory Reference + * that refers to the entirety of its parent Shared + * Memory block. The parameter structure is a + * TEEC_MemoryReference. In this structure, the + * Implementation MUST read only the parent field + * and MAY update the size field when the operation + * completes. + * + * TEEC_MEMREF_PARTIAL_INPUT A Registered Memory Reference structure that + * refers to a partial region of its parent Shared + * Memory block and is tagged as input. + * + * TEEC_MEMREF_PARTIAL_OUTPUT Registered Memory Reference structure that + * refers to a partial region of its parent Shared + * Memory block and is tagged as output. + * + * TEEC_MEMREF_PARTIAL_INOUT The Registered Memory Reference structure that + * refers to a partial region of its parent Shared + * Memory block and is tagged as both input and + * output, i.e., for which both the behaviors of + * TEEC_MEMREF_PARTIAL_INPUT and + * TEEC_MEMREF_PARTIAL_OUTPUT apply. + */ +#define TEEC_NONE 0x00000000 +#define TEEC_VALUE_INPUT 0x00000001 +#define TEEC_VALUE_OUTPUT 0x00000002 +#define TEEC_VALUE_INOUT 0x00000003 +#define TEEC_MEMREF_TEMP_INPUT 0x00000005 +#define TEEC_MEMREF_TEMP_OUTPUT 0x00000006 +#define TEEC_MEMREF_TEMP_INOUT 0x00000007 +#define TEEC_MEMREF_WHOLE 0x0000000C +#define TEEC_MEMREF_PARTIAL_INPUT 0x0000000D +#define TEEC_MEMREF_PARTIAL_OUTPUT 0x0000000E +#define TEEC_MEMREF_PARTIAL_INOUT 0x0000000F + +/** + * Flag constants indicating the data transfer direction of memory in + * TEEC_Parameter. TEEC_MEM_INPUT signifies data transfer direction from the + * client application to the TEE. TEEC_MEM_OUTPUT signifies data transfer + * direction from the TEE to the client application. Type is uint32_t. + * + * TEEC_MEM_INPUT The Shared Memory can carry data from the client + * application to the Trusted Application. + * TEEC_MEM_OUTPUT The Shared Memory can carry data from the Trusted + * Application to the client application. + */ +#define TEEC_MEM_INPUT 0x00000001 +#define TEEC_MEM_OUTPUT 0x00000002 + +/** + * Return values. Type is TEEC_Result + * + * TEEC_SUCCESS The operation was successful. + * TEEC_ERROR_GENERIC Non-specific cause. + * TEEC_ERROR_ACCESS_DENIED Access privileges are not sufficient. + * TEEC_ERROR_CANCEL The operation was canceled. + * TEEC_ERROR_ACCESS_CONFLICT Concurrent accesses caused conflict. + * TEEC_ERROR_EXCESS_DATA Too much data for the requested operation was + * passed. + * TEEC_ERROR_BAD_FORMAT Input data was of invalid format. + * TEEC_ERROR_BAD_PARAMETERS Input parameters were invalid. + * TEEC_ERROR_BAD_STATE Operation is not valid in the current state. + * TEEC_ERROR_ITEM_NOT_FOUND The requested data item is not found. + * TEEC_ERROR_NOT_IMPLEMENTED The requested operation should exist but is not + * yet implemented. + * TEEC_ERROR_NOT_SUPPORTED The requested operation is valid but is not + * supported in this implementation. + * TEEC_ERROR_NO_DATA Expected data was missing. + * TEEC_ERROR_OUT_OF_MEMORY System ran out of resources. + * TEEC_ERROR_BUSY The system is busy working on something else. + * TEEC_ERROR_COMMUNICATION Communication with a remote party failed. + * TEEC_ERROR_SECURITY A security fault was detected. + * TEEC_ERROR_SHORT_BUFFER The supplied buffer is too short for the + * generated output. + * TEEC_ERROR_TARGET_DEAD Trusted Application has panicked + * during the operation. + */ + +/** + * Standard defined error codes. + */ +#define TEEC_SUCCESS 0x00000000 +#define TEEC_ERROR_STORAGE_NOT_AVAILABLE 0xF0100003 +#define TEEC_ERROR_GENERIC 0xFFFF0000 +#define TEEC_ERROR_ACCESS_DENIED 0xFFFF0001 +#define TEEC_ERROR_CANCEL 0xFFFF0002 +#define TEEC_ERROR_ACCESS_CONFLICT 0xFFFF0003 +#define TEEC_ERROR_EXCESS_DATA 0xFFFF0004 +#define TEEC_ERROR_BAD_FORMAT 0xFFFF0005 +#define TEEC_ERROR_BAD_PARAMETERS 0xFFFF0006 +#define TEEC_ERROR_BAD_STATE 0xFFFF0007 +#define TEEC_ERROR_ITEM_NOT_FOUND 0xFFFF0008 +#define TEEC_ERROR_NOT_IMPLEMENTED 0xFFFF0009 +#define TEEC_ERROR_NOT_SUPPORTED 0xFFFF000A +#define TEEC_ERROR_NO_DATA 0xFFFF000B +#define TEEC_ERROR_OUT_OF_MEMORY 0xFFFF000C +#define TEEC_ERROR_BUSY 0xFFFF000D +#define TEEC_ERROR_COMMUNICATION 0xFFFF000E +#define TEEC_ERROR_SECURITY 0xFFFF000F +#define TEEC_ERROR_SHORT_BUFFER 0xFFFF0010 +#define TEEC_ERROR_EXTERNAL_CANCEL 0xFFFF0011 +#define TEEC_ERROR_TARGET_DEAD 0xFFFF3024 +#define TEEC_ERROR_STORAGE_NO_SPACE 0xFFFF3041 + +/** + * Function error origins, of type TEEC_ErrorOrigin. These indicate where in + * the software stack a particular return value originates from. + * + * TEEC_ORIGIN_API The error originated within the TEE Client API + * implementation. + * TEEC_ORIGIN_COMMS The error originated within the underlying + * communications stack linking the rich OS with + * the TEE. + * TEEC_ORIGIN_TEE The error originated within the common TEE code. + * TEEC_ORIGIN_TRUSTED_APP The error originated within the Trusted Application + * code. + */ +#define TEEC_ORIGIN_API 0x00000001 +#define TEEC_ORIGIN_COMMS 0x00000002 +#define TEEC_ORIGIN_TEE 0x00000003 +#define TEEC_ORIGIN_TRUSTED_APP 0x00000004 + +/** + * Session login methods, for use in TEEC_OpenSession() as parameter + * connectionMethod. Type is uint32_t. + * + * TEEC_LOGIN_PUBLIC No login data is provided. + * TEEC_LOGIN_USER Login data about the user running the Client + * Application process is provided. + * TEEC_LOGIN_GROUP Login data about the group running the Client + * Application process is provided. + * TEEC_LOGIN_APPLICATION Login data about the running Client Application + * itself is provided. + * TEEC_LOGIN_USER_APPLICATION Login data about the user and the running + * Client Application itself is provided. + * TEEC_LOGIN_GROUP_APPLICATION Login data about the group and the running + * Client Application itself is provided. + */ +#define TEEC_LOGIN_PUBLIC 0x00000000 +#define TEEC_LOGIN_USER 0x00000001 +#define TEEC_LOGIN_GROUP 0x00000002 +#define TEEC_LOGIN_APPLICATION 0x00000004 +#define TEEC_LOGIN_USER_APPLICATION 0x00000005 +#define TEEC_LOGIN_GROUP_APPLICATION 0x00000006 + +/** + * Encode the paramTypes according to the supplied types. + * + * @param p0 The first param type. + * @param p1 The second param type. + * @param p2 The third param type. + * @param p3 The fourth param type. + */ +#define TEEC_PARAM_TYPES(p0, p1, p2, p3) \ + ((p0) | ((p1) << 4) | ((p2) << 8) | ((p3) << 12)) + +/** + * Get the i_th param type from the paramType. + * + * @param p The paramType. + * @param i The i-th parameter to get the type for. + */ +#define TEEC_PARAM_TYPE_GET(p, i) (((p) >> (i * 4)) & 0xF) + +typedef uint32_t TEEC_Result; + +/** + * struct TEEC_Context - Represents a connection between a client application + * and a TEE. + */ +typedef struct { + /* Implementation defined */ + int fd; + bool reg_mem; + bool memref_null; +} TEEC_Context; + +/** + * This type contains a Universally Unique Resource Identifier (UUID) type as + * defined in RFC4122. These UUID values are used to identify Trusted + * Applications. + */ +typedef struct { + uint32_t timeLow; + uint16_t timeMid; + uint16_t timeHiAndVersion; + uint8_t clockSeqAndNode[8]; +} TEEC_UUID; + +/** + * struct TEEC_SharedMemory - Memory to transfer data between a client + * application and trusted code. + * + * @param buffer The memory buffer which is to be, or has been, shared + * with the TEE. + * @param size The size, in bytes, of the memory buffer. + * @param flags Bit-vector which holds properties of buffer. + * The bit-vector can contain either or both of the + * TEEC_MEM_INPUT and TEEC_MEM_OUTPUT flags. + * + * A shared memory block is a region of memory allocated in the context of the + * client application memory space that can be used to transfer data between + * that client application and a trusted application. The user of this struct + * is responsible to populate the buffer pointer. + */ +typedef struct { + void *buffer; + size_t size; + uint32_t flags; + /* + * Implementation-Defined + */ + int id; + size_t alloced_size; + void *shadow_buffer; + int registered_fd; + union { + bool dummy; + uint8_t flags; + } internal; +} TEEC_SharedMemory; + +/** + * struct TEEC_TempMemoryReference - Temporary memory to transfer data between + * a client application and trusted code, only used for the duration of the + * operation. + * + * @param buffer The memory buffer which is to be, or has been shared with + * the TEE. + * @param size The size, in bytes, of the memory buffer. + * + * A memory buffer that is registered temporarily for the duration of the + * operation to be called. + */ +typedef struct { + void *buffer; + size_t size; +} TEEC_TempMemoryReference; + +/** + * struct TEEC_RegisteredMemoryReference - use a pre-registered or + * pre-allocated shared memory block of memory to transfer data between + * a client application and trusted code. + * + * @param parent Points to a shared memory structure. The memory reference + * may utilize the whole shared memory or only a part of it. + * Must not be NULL + * + * @param size The size, in bytes, of the memory buffer. + * + * @param offset The offset, in bytes, of the referenced memory region from + * the start of the shared memory block. + * + */ +typedef struct { + TEEC_SharedMemory *parent; + size_t size; + size_t offset; +} TEEC_RegisteredMemoryReference; + +/** + * struct TEEC_Value - Small raw data container + * + * Instead of allocating a shared memory buffer this structure can be used + * to pass small raw data between a client application and trusted code. + * + * @param a The first integer value. + * + * @param b The second value. + */ +typedef struct { + uint32_t a; + uint32_t b; +} TEEC_Value; + +/** + * union TEEC_Parameter - Memory container to be used when passing data between + * client application and trusted code. + * + * Either the client uses a shared memory reference, parts of it or a small raw + * data container. + * + * @param tmpref A temporary memory reference only valid for the duration + * of the operation. + * + * @param memref The entire shared memory or parts of it. + * + * @param value The small raw data container to use + */ +typedef union { + TEEC_TempMemoryReference tmpref; + TEEC_RegisteredMemoryReference memref; + TEEC_Value value; +} TEEC_Parameter; + +/** + * struct TEEC_Session - Represents a connection between a client application + * and a trusted application. + */ +typedef struct { + /* Implementation defined */ + TEEC_Context *ctx; + uint32_t session_id; +} TEEC_Session; + +/** + * struct TEEC_Operation - Holds information and memory references used in + * TEEC_InvokeCommand(). + * + * @param started Client must initialize to zero if it needs to cancel + * an operation about to be performed. + * @param paramTypes Type of data passed. Use TEEC_PARAM_TYPES macro to + * create the correct flags. + * 0 means TEEC_NONE is passed for all params. + * @param params Array of parameters of type TEEC_Parameter. + * @param session Internal pointer to the last session used by + * TEEC_InvokeCommand with this operation. + * + */ +typedef struct { + uint32_t started; + uint32_t paramTypes; + TEEC_Parameter params[TEEC_CONFIG_PAYLOAD_REF_COUNT]; + /* Implementation-Defined */ + TEEC_Session *session; +} TEEC_Operation; + +/** + * TEEC_InitializeContext() - Initializes a context holding connection + * information on the specific TEE, designated by the name string. + + * @param name A zero-terminated string identifying the TEE to connect to. + * If name is set to NULL, the default TEE is connected to. NULL + * is the only supported value in this version of the API + * implementation. + * + * @param context The context structure which is to be initialized. + * + * @return TEEC_SUCCESS The initialization was successful. + * @return TEEC_Result Something failed. + */ +TEEC_Result TEEC_InitializeContext(const char *name, TEEC_Context *context); + +/** + * TEEC_FinalizeContext() - Destroys a context holding connection information + * on the specific TEE. + * + * This function destroys an initialized TEE context, closing the connection + * between the client application and the TEE. This function must only be + * called when all sessions related to this TEE context have been closed and + * all shared memory blocks have been released. + * + * @param context The context to be destroyed. + */ +void TEEC_FinalizeContext(TEEC_Context *context); + +/** + * TEEC_OpenSession() - Opens a new session with the specified trusted + * application. + * + * @param context The initialized TEE context structure in which + * scope to open the session. + * @param session The session to initialize. + * @param destination A structure identifying the trusted application + * with which to open a session. + * + * @param connectionMethod The connection method to use. + * @param connectionData Any data necessary to connect with the chosen + * connection method. Not supported, should be set to + * NULL. + * @param operation An operation structure to use in the session. May + * be set to NULL to signify no operation structure + * needed. + * + * @param returnOrigin A parameter which will hold the error origin if + * this function returns any value other than + * TEEC_SUCCESS. + * + * @return TEEC_SUCCESS OpenSession successfully opened a new session. + * @return TEEC_Result Something failed. + * + */ +TEEC_Result TEEC_OpenSession(TEEC_Context *context, + TEEC_Session *session, + const TEEC_UUID *destination, + uint32_t connectionMethod, + const void *connectionData, + TEEC_Operation *operation, + uint32_t *returnOrigin); + +/** + * TEEC_CloseSession() - Closes the session which has been opened with the + * specific trusted application. + * + * @param session The opened session to close. + */ +void TEEC_CloseSession(TEEC_Session *session); + +/** + * TEEC_InvokeCommand() - Executes a command in the specified trusted + * application. + * + * @param session A handle to an open connection to the trusted + * application. + * @param commandID Identifier of the command in the trusted application + * to invoke. + * @param operation An operation structure to use in the invoke command. + * May be set to NULL to signify no operation structure + * needed. + * @param returnOrigin A parameter which will hold the error origin if this + * function returns any value other than TEEC_SUCCESS. + * + * @return TEEC_SUCCESS OpenSession successfully opened a new session. + * @return TEEC_Result Something failed. + */ +TEEC_Result TEEC_InvokeCommand(TEEC_Session *session, + uint32_t commandID, + TEEC_Operation *operation, + uint32_t *returnOrigin); + +/** + * TEEC_RegisterSharedMemory() - Register a block of existing memory as a + * shared block within the scope of the specified context. + * + * @param context The initialized TEE context structure in which scope to + * open the session. + * @param sharedMem pointer to the shared memory structure to register. + * + * @return TEEC_SUCCESS The registration was successful. + * @return TEEC_ERROR_OUT_OF_MEMORY Memory exhaustion. + * @return TEEC_Result Something failed. + */ +TEEC_Result TEEC_RegisterSharedMemory(TEEC_Context *context, + TEEC_SharedMemory *sharedMem); + +/** + * TEEC_AllocateSharedMemory() - Allocate shared memory for TEE. + * + * @param context The initialized TEE context structure in which scope to + * open the session. + * @param sharedMem Pointer to the allocated shared memory. + * + * @return TEEC_SUCCESS The registration was successful. + * @return TEEC_ERROR_OUT_OF_MEMORY Memory exhaustion. + * @return TEEC_Result Something failed. + */ +TEEC_Result TEEC_AllocateSharedMemory(TEEC_Context *context, + TEEC_SharedMemory *sharedMem); + +/** + * TEEC_ReleaseSharedMemory() - Free or deregister the shared memory. + * + * @param sharedMem Pointer to the shared memory to be freed. + */ +void TEEC_ReleaseSharedMemory(TEEC_SharedMemory *sharedMemory); + +/** + * TEEC_RequestCancellation() - Request the cancellation of a pending open + * session or command invocation. + * + * @param operation Pointer to an operation previously passed to open session + * or invoke. + */ +void TEEC_RequestCancellation(TEEC_Operation *operation); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/optee_client/libteec/include/tee_client_api_extensions.h b/optee_client/libteec/include/tee_client_api_extensions.h new file mode 100644 index 0000000..85298aa --- /dev/null +++ b/optee_client/libteec/include/tee_client_api_extensions.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef TEE_CLIENT_API_EXTENSIONS_H +#define TEE_CLIENT_API_EXTENSIONS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * TEEC_RegisterMemoryFileDescriptor() - Register a block of existing memory as + * a shared block within the scope of the specified context. + * + * @param context The initialized TEE context structure in which scope to + * open the session. + * @param sharedMem pointer to the shared memory structure to register. + * @param fd file descriptor of the target memory. + * + * @return TEEC_SUCCESS The registration was successful. + * @return TEEC_ERROR_OUT_OF_MEMORY Memory exhaustion. + * @return TEEC_Result Something failed. + */ +TEEC_Result TEEC_RegisterSharedMemoryFileDescriptor(TEEC_Context *context, + TEEC_SharedMemory *sharedMem, + int fd); + +#ifdef __cplusplus +} +#endif + +#endif /* TEE_CLIENT_API_EXTENSIONS_H */ diff --git a/optee_client/libteec/include/tee_plugin_method.h b/optee_client/libteec/include/tee_plugin_method.h new file mode 100644 index 0000000..c08dc12 --- /dev/null +++ b/optee_client/libteec/include/tee_plugin_method.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#ifndef TEE_PLUGIN_METHOD_H +#define TEE_PLUGIN_METHOD_H + +#include +#include + +struct plugin_method { + const char *name; /* short friendly name of the plugin */ + TEEC_UUID uuid; + TEEC_Result (*init)(void); + TEEC_Result (*invoke)(unsigned int cmd, unsigned int sub_cmd, + void *data, size_t in_len, size_t *out_len); +}; + +#endif /* TEE_PLUGIN_METHOD_H */ diff --git a/optee_client/libteec/include/teec_trace.h b/optee_client/libteec/include/teec_trace.h new file mode 100644 index 0000000..f75358f --- /dev/null +++ b/optee_client/libteec/include/teec_trace.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef TEEC_TRACE_H +#define TEEC_TRACE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#ifndef BINARY_PREFIX +#error "BINARY_PREFIX not defined" +#endif + +/* + * Trace levels. + * + * ERROR is used when some kind of error has happened, this is most likely the + * print you will use most of the time when you report some kind of error. + * + * INFO is used when you want to print some 'normal' text to the user. + * This is the default level. + * + * DEBUG is used to print extra information to enter deeply in the module. + * + * FLOW is used to print the execution flox, typically the in/out of functions. + * + * */ + +#define TRACE_ERROR 1 +#define TRACE_INFO 2 +#define TRACE_DEBUG 3 +#define TRACE_FLOW 4 + +#if defined(DEBUGLEVEL_0) && !defined(DEBUGLEVEL) +#define DEBUGLEVEL TRACE_ERROR +#endif + +#if defined(DEBUGLEVEL_1) && !defined(DEBUGLEVEL) +#define DEBUGLEVEL TRACE_ERROR +#endif + +#if defined(DEBUGLEVEL_2) && !defined(DEBUGLEVEL) +#define DEBUGLEVEL TRACE_INFO +#endif + +#if defined(DEBUGLEVEL_3) && !defined(DEBUGLEVEL) +#define DEBUGLEVEL TRACE_DEBUG +#endif + +#if defined(DEBUGLEVEL_4) && !defined(DEBUGLEVEL) +#define DEBUGLEVEL TRACE_FLOW +#endif + +#ifndef DEBUGLEVEL +/* Default debug level. */ +#define DEBUGLEVEL TRACE_INFO +#endif + +/* + * This define make sure that parameters are checked in the same manner as it + * is done in the normal printf function. + */ +#define __PRINTFLIKE(__fmt, __varargs) __attribute__\ + ((__format__(__printf__, __fmt, __varargs))) + +void _dprintf(const char *function, int line, int level, const char *prefix, + const char *fmt, ...) __PRINTFLIKE(5, 6); + +#define dprintf(level, x...) do { \ + if ((level) <= DEBUGLEVEL) { \ + _dprintf(__func__, __LINE__, level, \ + BINARY_PREFIX, x); \ + } \ + } while (0) + +#define EMSG(fmt, ...) dprintf(TRACE_ERROR, fmt "\n", ##__VA_ARGS__) +#define IMSG(fmt, ...) dprintf(TRACE_INFO, fmt "\n", ##__VA_ARGS__) +#define DMSG(fmt, ...) dprintf(TRACE_DEBUG, fmt "\n", ##__VA_ARGS__) +#define FMSG(fmt, ...) dprintf(TRACE_FLOW, fmt "\n", ##__VA_ARGS__) + +#define INMSG(fmt, ...) FMSG("> " fmt, ##__VA_ARGS__) +#define OUTMSG(fmt, ...) FMSG("< " fmt, ##__VA_ARGS__) +#define OUTRMSG(r) \ + do { \ + if (r) \ + EMSG("Function returns with [%d]", r); \ + OUTMSG("r=[%d]", r); \ + return r; \ + } while (0) + +#define dprintf_raw(level, x...) do { \ + if ((level) <= DEBUGLEVEL) \ + _dprintf(0, 0, (level), BINARY_PREFIX, x); \ + } while (0) + +#define EMSG_RAW(fmt, ...) dprintf_raw(TRACE_ERROR, fmt, ##__VA_ARGS__) +#define IMSG_RAW(fmt, ...) dprintf_raw(TRACE_INFO, fmt, ##__VA_ARGS__) +#define DMSG_RAW(fmt, ...) dprintf_raw(TRACE_DEBUG, fmt, ##__VA_ARGS__) +#define FMSG_RAW(fmt, ...) dprintf_raw(TRACE_FLOW, fmt, ##__VA_ARGS__) + +/* + * This function will hex and ascii dump a buffer. + * + * Note that this function will only print if debug flag + * DEBUGLEVEL is INFO or FLOOD. + * + * @param bname Information string describing the buffer. + * @param buffer Pointer to the buffer. + * @param blen Length of the buffer. + * + * @return void + */ +void dump_buffer(const char *bname, const uint8_t *buffer, size_t blen); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/optee_client/libteec/libteec.pc.in b/optee_client/libteec/libteec.pc.in new file mode 100644 index 0000000..e91e690 --- /dev/null +++ b/optee_client/libteec/libteec.pc.in @@ -0,0 +1,14 @@ +prefix="@CMAKE_INSTALL_PREFIX@" +exec_prefix="${prefix}" +libdir="${prefix}/lib" +includedir="${prefix}/include" + +Name: @PROJECT_NAME@ +Description: @CMAKE_PROJECT_DESCRIPTION@ +URL: @CMAKE_PROJECT_HOMEPAGE_URL@ +Version: @PROJECT_VERSION@ +Requires: @pc_req_public@ +Requires.private: @pc_req_private@ +Cflags: -I"${includedir}" +Libs: -L"${libdir}" -l@libteectgt@ +Libs.private: -L"${libdir}" -l@libteectgt@ @pc_libs_private@ diff --git a/optee_client/libteec/src/tee_client_api.c b/optee_client/libteec/src/tee_client_api.c new file mode 100644 index 0000000..6d398ab --- /dev/null +++ b/optee_client/libteec/src/tee_client_api.c @@ -0,0 +1,960 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef __aligned +#define __aligned(x) __attribute__((__aligned__(x))) +#endif +#include + +#include "teec_benchmark.h" + +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) + +/* How many device sequence numbers will be tried before giving up */ +#define TEEC_MAX_DEV_SEQ 10 + +/* Helpers to access memref parts of a struct tee_ioctl_param */ +#define MEMREF_SHM_ID(p) ((p)->c) +#define MEMREF_SHM_OFFS(p) ((p)->a) +#define MEMREF_SIZE(p) ((p)->b) + +/* + * Internal flags of TEEC_SharedMemory::internal.flags + */ +#define SHM_FLAG_BUFFER_ALLOCED (1u << 0) +#define SHM_FLAG_SHADOW_BUFFER_ALLOCED (1u << 1) + +static pthread_mutex_t teec_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void teec_mutex_lock(pthread_mutex_t *mu) +{ + pthread_mutex_lock(mu); +} + +static void teec_mutex_unlock(pthread_mutex_t *mu) +{ + pthread_mutex_unlock(mu); +} + +static void *teec_paged_aligned_alloc(size_t sz) +{ + void *p = NULL; + size_t page_sz = sysconf(_SC_PAGESIZE); + size_t aligned_sz = ((sz + page_sz - 1) / page_sz) * page_sz; + + if (!posix_memalign(&p, page_sz, aligned_sz)) + return p; + + return NULL; +} + +static int teec_open_dev(const char *devname, const char *capabilities, + uint32_t *gen_caps) +{ + int fd = 0; + struct tee_ioctl_version_data vers; + + memset(&vers, 0, sizeof(vers)); + + fd = open(devname, O_RDWR); + if (fd < 0) + return -1; + + if (ioctl(fd, TEE_IOC_VERSION, &vers)) { + EMSG("TEE_IOC_VERSION failed"); + goto err; + } + + /* We can only handle GP TEEs */ + if (!(vers.gen_caps & TEE_GEN_CAP_GP)) + goto err; + + if (capabilities) { + if (strcmp(capabilities, "optee-tz") == 0) { + if (vers.impl_id != TEE_IMPL_ID_OPTEE) + goto err; + if (!(vers.impl_caps & TEE_OPTEE_CAP_TZ)) + goto err; + } else { + /* Unrecognized capability requested */ + goto err; + } + } + + *gen_caps = vers.gen_caps; + return fd; +err: + close(fd); + return -1; +} + +static int teec_shm_alloc(int fd, size_t size, int *id) +{ + int shm_fd = 0; + struct tee_ioctl_shm_alloc_data data; + + memset(&data, 0, sizeof(data)); + + data.size = size; + shm_fd = ioctl(fd, TEE_IOC_SHM_ALLOC, &data); + if (shm_fd < 0) + return -1; + *id = data.id; + return shm_fd; +} + +static int teec_shm_register(int fd, void *buf, size_t size, int *id) +{ + int shm_fd = 0; + struct tee_ioctl_shm_register_data data; + + memset(&data, 0, sizeof(data)); + + data.addr = (uintptr_t)buf; + data.length = size; + shm_fd = ioctl(fd, TEE_IOC_SHM_REGISTER, &data); + if (shm_fd < 0) + return -1; + *id = data.id; + return shm_fd; +} + +TEEC_Result TEEC_InitializeContext(const char *name, TEEC_Context *ctx) +{ + char devname[PATH_MAX] = { 0 }; + int fd = 0; + size_t n = 0; + + if (!ctx) + return TEEC_ERROR_BAD_PARAMETERS; + + for (n = 0; n < TEEC_MAX_DEV_SEQ; n++) { + uint32_t gen_caps = 0; + + snprintf(devname, sizeof(devname), "/dev/tee%zu", n); + fd = teec_open_dev(devname, name, &gen_caps); + if (fd >= 0) { + ctx->fd = fd; + ctx->reg_mem = gen_caps & TEE_GEN_CAP_REG_MEM; + ctx->memref_null = gen_caps & TEE_GEN_CAP_MEMREF_NULL; + return TEEC_SUCCESS; + } + } + + return TEEC_ERROR_ITEM_NOT_FOUND; +} + +void TEEC_FinalizeContext(TEEC_Context *ctx) +{ + if (ctx) + close(ctx->fd); +} + + +static TEEC_Result teec_pre_process_tmpref(TEEC_Context *ctx, + uint32_t param_type, TEEC_TempMemoryReference *tmpref, + struct tee_ioctl_param *param, + TEEC_SharedMemory *shm) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + + switch (param_type) { + case TEEC_MEMREF_TEMP_INPUT: + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; + shm->flags = TEEC_MEM_INPUT; + break; + case TEEC_MEMREF_TEMP_OUTPUT: + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT; + shm->flags = TEEC_MEM_OUTPUT; + break; + case TEEC_MEMREF_TEMP_INOUT: + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT; + shm->flags = TEEC_MEM_INPUT | TEEC_MEM_OUTPUT; + break; + default: + return TEEC_ERROR_BAD_PARAMETERS; + } + shm->size = tmpref->size; + + if (!tmpref->buffer) { + if (tmpref->size) + return TEEC_ERROR_BAD_PARAMETERS; + + if (ctx->memref_null) { + /* Null pointer, indicate no shared memory attached */ + MEMREF_SHM_ID(param) = TEE_MEMREF_NULL; + shm->id = -1; + } else { + res = TEEC_AllocateSharedMemory(ctx, shm); + if (res != TEEC_SUCCESS) + return res; + MEMREF_SHM_ID(param) = shm->id; + } + } else { + shm->buffer = tmpref->buffer; + res = TEEC_RegisterSharedMemory(ctx, shm); + if (res != TEEC_SUCCESS) + return res; + + if (shm->shadow_buffer) + memcpy(shm->shadow_buffer, tmpref->buffer, + tmpref->size); + + MEMREF_SHM_ID(param) = shm->id; + } + + MEMREF_SIZE(param) = tmpref->size; + + return TEEC_SUCCESS; +} + +static TEEC_Result teec_pre_process_whole( + TEEC_RegisteredMemoryReference *memref, + struct tee_ioctl_param *param) +{ + const uint32_t inout = TEEC_MEM_INPUT | TEEC_MEM_OUTPUT; + uint32_t flags = memref->parent->flags & inout; + TEEC_SharedMemory *shm = NULL; + + if (flags == inout) + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT; + else if (flags & TEEC_MEM_INPUT) + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; + else if (flags & TEEC_MEM_OUTPUT) + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT; + else + return TEEC_ERROR_BAD_PARAMETERS; + + shm = memref->parent; + /* + * We're using a shadow buffer in this reference, copy the real buffer + * into the shadow buffer if needed. We'll copy it back once we've + * returned from the call to secure world. + */ + if (shm->shadow_buffer && (flags & TEEC_MEM_INPUT)) + memcpy(shm->shadow_buffer, shm->buffer, shm->size); + + MEMREF_SHM_ID(param) = shm->id; + MEMREF_SIZE(param) = shm->size; + + return TEEC_SUCCESS; +} + +static TEEC_Result teec_pre_process_partial(uint32_t param_type, + TEEC_RegisteredMemoryReference *memref, + struct tee_ioctl_param *param) +{ + uint32_t req_shm_flags = 0; + TEEC_SharedMemory *shm = NULL; + + switch (param_type) { + case TEEC_MEMREF_PARTIAL_INPUT: + req_shm_flags = TEEC_MEM_INPUT; + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; + break; + case TEEC_MEMREF_PARTIAL_OUTPUT: + req_shm_flags = TEEC_MEM_OUTPUT; + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT; + break; + case TEEC_MEMREF_PARTIAL_INOUT: + req_shm_flags = TEEC_MEM_OUTPUT | TEEC_MEM_INPUT; + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT; + break; + default: + return TEEC_ERROR_BAD_PARAMETERS; + } + + shm = memref->parent; + + if ((shm->flags & req_shm_flags) != req_shm_flags) + return TEEC_ERROR_BAD_PARAMETERS; + + if ((memref->offset + memref->size < memref->offset) || + (memref->offset + memref->size > shm->size)) + return TEEC_ERROR_BAD_PARAMETERS; + + /* + * We're using a shadow buffer in this reference, copy the real buffer + * into the shadow buffer if needed. We'll copy it back once we've + * returned from the call to secure world. + */ + if (shm->shadow_buffer && param_type != TEEC_MEMREF_PARTIAL_OUTPUT) + memcpy((char *)shm->shadow_buffer + memref->offset, + (char *)shm->buffer + memref->offset, memref->size); + + MEMREF_SHM_ID(param) = shm->id; + MEMREF_SHM_OFFS(param) = memref->offset; + MEMREF_SIZE(param) = memref->size; + + return TEEC_SUCCESS; +} + +static TEEC_Result teec_pre_process_operation(TEEC_Context *ctx, + TEEC_Operation *operation, + struct tee_ioctl_param *params, + TEEC_SharedMemory *shms) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + size_t n = 0; + + memset(shms, 0, sizeof(TEEC_SharedMemory) * + TEEC_CONFIG_PAYLOAD_REF_COUNT); + + for (n = 0; n < TEEC_CONFIG_PAYLOAD_REF_COUNT; n++) + shms[n].id = -1; + + if (!operation) { + memset(params, 0, sizeof(struct tee_ioctl_param) * + TEEC_CONFIG_PAYLOAD_REF_COUNT); + return TEEC_SUCCESS; + } + + for (n = 0; n < TEEC_CONFIG_PAYLOAD_REF_COUNT; n++) { + uint32_t param_type = 0; + + param_type = TEEC_PARAM_TYPE_GET(operation->paramTypes, n); + switch (param_type) { + case TEEC_NONE: + params[n].attr = param_type; + break; + case TEEC_VALUE_INPUT: + case TEEC_VALUE_OUTPUT: + case TEEC_VALUE_INOUT: + params[n].attr = param_type; + params[n].a = operation->params[n].value.a; + params[n].b = operation->params[n].value.b; + break; + case TEEC_MEMREF_TEMP_INPUT: + case TEEC_MEMREF_TEMP_OUTPUT: + case TEEC_MEMREF_TEMP_INOUT: + res = teec_pre_process_tmpref(ctx, param_type, + &operation->params[n].tmpref, params + n, + shms + n); + if (res != TEEC_SUCCESS) + return res; + break; + case TEEC_MEMREF_WHOLE: + res = teec_pre_process_whole( + &operation->params[n].memref, + params + n); + if (res != TEEC_SUCCESS) + return res; + break; + case TEEC_MEMREF_PARTIAL_INPUT: + case TEEC_MEMREF_PARTIAL_OUTPUT: + case TEEC_MEMREF_PARTIAL_INOUT: + res = teec_pre_process_partial(param_type, + &operation->params[n].memref, params + n); + if (res != TEEC_SUCCESS) + return res; + break; + default: + return TEEC_ERROR_BAD_PARAMETERS; + } + } + + return TEEC_SUCCESS; +} + +static void teec_post_process_tmpref(uint32_t param_type, + TEEC_TempMemoryReference *tmpref, + struct tee_ioctl_param *param, + TEEC_SharedMemory *shm) +{ + if (param_type != TEEC_MEMREF_TEMP_INPUT) { + if (tmpref->buffer && shm->shadow_buffer) + memcpy(tmpref->buffer, shm->shadow_buffer, + MIN(MEMREF_SIZE(param), tmpref->size)); + + tmpref->size = MEMREF_SIZE(param); + } +} + +static void teec_post_process_whole(TEEC_RegisteredMemoryReference *memref, + struct tee_ioctl_param *param) +{ + TEEC_SharedMemory *shm = memref->parent; + + if (shm->flags & TEEC_MEM_OUTPUT) { + + /* + * We're using a shadow buffer in this reference, copy back + * the shadow buffer into the real buffer now that we've + * returned from secure world. + */ + if (shm->shadow_buffer && MEMREF_SIZE(param) <= shm->size) + memcpy(shm->buffer, shm->shadow_buffer, + MEMREF_SIZE(param)); + + memref->size = MEMREF_SIZE(param); + } +} + +static void teec_post_process_partial(uint32_t param_type, + TEEC_RegisteredMemoryReference *memref, + struct tee_ioctl_param *param) +{ + if (param_type != TEEC_MEMREF_PARTIAL_INPUT) { + TEEC_SharedMemory *shm = memref->parent; + + /* + * We're using a shadow buffer in this reference, copy back + * the shadow buffer into the real buffer now that we've + * returned from secure world. + */ + if (shm->shadow_buffer && MEMREF_SIZE(param) <= memref->size) + memcpy((char *)shm->buffer + memref->offset, + (char *)shm->shadow_buffer + memref->offset, + MEMREF_SIZE(param)); + + memref->size = MEMREF_SIZE(param); + } +} + +static void teec_post_process_operation(TEEC_Operation *operation, + struct tee_ioctl_param *params, + TEEC_SharedMemory *shms) +{ + size_t n = 0; + + if (!operation) + return; + + for (n = 0; n < TEEC_CONFIG_PAYLOAD_REF_COUNT; n++) { + uint32_t param_type = 0; + + param_type = TEEC_PARAM_TYPE_GET(operation->paramTypes, n); + switch (param_type) { + case TEEC_VALUE_INPUT: + break; + case TEEC_VALUE_OUTPUT: + case TEEC_VALUE_INOUT: + operation->params[n].value.a = params[n].a; + operation->params[n].value.b = params[n].b; + break; + case TEEC_MEMREF_TEMP_INPUT: + case TEEC_MEMREF_TEMP_OUTPUT: + case TEEC_MEMREF_TEMP_INOUT: + teec_post_process_tmpref(param_type, + &operation->params[n].tmpref, params + n, + shms + n); + break; + case TEEC_MEMREF_WHOLE: + teec_post_process_whole(&operation->params[n].memref, + params + n); + break; + case TEEC_MEMREF_PARTIAL_INPUT: + case TEEC_MEMREF_PARTIAL_OUTPUT: + case TEEC_MEMREF_PARTIAL_INOUT: + teec_post_process_partial(param_type, + &operation->params[n].memref, params + n); + default: + break; + } + } +} + +static void teec_free_temp_refs(TEEC_Operation *operation, + TEEC_SharedMemory *shms) +{ + size_t n = 0; + + if (!operation) + return; + + for (n = 0; n < TEEC_CONFIG_PAYLOAD_REF_COUNT; n++) { + switch (TEEC_PARAM_TYPE_GET(operation->paramTypes, n)) { + case TEEC_MEMREF_TEMP_INPUT: + case TEEC_MEMREF_TEMP_OUTPUT: + case TEEC_MEMREF_TEMP_INOUT: + TEEC_ReleaseSharedMemory(shms + n); + break; + default: + break; + } + } +} + +static TEEC_Result ioctl_errno_to_res(int err) +{ + switch (err) { + case ENOMEM: + return TEEC_ERROR_OUT_OF_MEMORY; + case EINVAL: + return TEEC_ERROR_BAD_PARAMETERS; + default: + return TEEC_ERROR_GENERIC; + } +} + +static void uuid_to_octets(uint8_t d[TEE_IOCTL_UUID_LEN], const TEEC_UUID *s) +{ + d[0] = s->timeLow >> 24; + d[1] = s->timeLow >> 16; + d[2] = s->timeLow >> 8; + d[3] = s->timeLow; + d[4] = s->timeMid >> 8; + d[5] = s->timeMid; + d[6] = s->timeHiAndVersion >> 8; + d[7] = s->timeHiAndVersion; + memcpy(d + 8, s->clockSeqAndNode, sizeof(s->clockSeqAndNode)); +} + +static void setup_client_data(struct tee_ioctl_open_session_arg *arg, + uint32_t connection_method, + const void *connection_data) +{ + arg->clnt_login = connection_method; + + switch (connection_method) { + case TEE_IOCTL_LOGIN_PUBLIC: + /* No connection data to pass */ + break; + case TEE_IOCTL_LOGIN_USER: + /* Kernel auto-fills UID and forms client UUID */ + break; + case TEE_IOCTL_LOGIN_GROUP: + /* + * Connection data for group login is uint32_t and rest of + * clnt_uuid is set as zero. + * + * Kernel verifies group membership and then forms client UUID. + */ + memcpy(arg->clnt_uuid, connection_data, sizeof(gid_t)); + break; + case TEE_IOCTL_LOGIN_APPLICATION: + /* + * Kernel auto-fills application identifier and forms client + * UUID. + */ + break; + case TEE_IOCTL_LOGIN_USER_APPLICATION: + /* + * Kernel auto-fills application identifier, UID and forms + * client UUID. + */ + break; + case TEE_IOCTL_LOGIN_GROUP_APPLICATION: + /* + * Connection data for group login is uint32_t rest of + * clnt_uuid is set as zero. + * + * Kernel verifies group membership, auto-fills application + * identifier and then forms client UUID. + */ + memcpy(arg->clnt_uuid, connection_data, sizeof(gid_t)); + break; + default: + /* + * Unknown login method, don't pass any connection data as we + * don't know size. + */ + break; + } +} + +TEEC_Result TEEC_OpenSession(TEEC_Context *ctx, TEEC_Session *session, + const TEEC_UUID *destination, + uint32_t connection_method, const void *connection_data, + TEEC_Operation *operation, uint32_t *ret_origin) +{ + struct tee_ioctl_open_session_arg *arg = NULL; + struct tee_ioctl_param *params = NULL; + TEEC_Result res = TEEC_ERROR_GENERIC; + uint32_t eorig = 0; + int rc = 0; + const size_t arg_size = sizeof(struct tee_ioctl_open_session_arg) + + TEEC_CONFIG_PAYLOAD_REF_COUNT * + sizeof(struct tee_ioctl_param); + union { + struct tee_ioctl_open_session_arg arg; + uint8_t data[arg_size]; + } buf; + struct tee_ioctl_buf_data buf_data; + TEEC_SharedMemory shm[TEEC_CONFIG_PAYLOAD_REF_COUNT]; + + memset(&buf, 0, sizeof(buf)); + memset(&shm, 0, sizeof(shm)); + memset(&buf_data, 0, sizeof(buf_data)); + + if (!ctx || !session) { + eorig = TEEC_ORIGIN_API; + res = TEEC_ERROR_BAD_PARAMETERS; + goto out; + } + + buf_data.buf_ptr = (uintptr_t)&buf; + buf_data.buf_len = sizeof(buf); + + arg = &buf.arg; + arg->num_params = TEEC_CONFIG_PAYLOAD_REF_COUNT; + params = (struct tee_ioctl_param *)(arg + 1); + + uuid_to_octets(arg->uuid, destination); + + setup_client_data(arg, connection_method, connection_data); + + res = teec_pre_process_operation(ctx, operation, params, shm); + if (res != TEEC_SUCCESS) { + eorig = TEEC_ORIGIN_API; + goto out_free_temp_refs; + } + + rc = ioctl(ctx->fd, TEE_IOC_OPEN_SESSION, &buf_data); + if (rc) { + EMSG("TEE_IOC_OPEN_SESSION failed"); + eorig = TEEC_ORIGIN_COMMS; + res = ioctl_errno_to_res(errno); + goto out_free_temp_refs; + } + res = arg->ret; + eorig = arg->ret_origin; + if (res == TEEC_SUCCESS) { + session->ctx = ctx; + session->session_id = arg->session; + } + teec_post_process_operation(operation, params, shm); + +out_free_temp_refs: + teec_free_temp_refs(operation, shm); +out: + if (ret_origin) + *ret_origin = eorig; + return res; +} + +void TEEC_CloseSession(TEEC_Session *session) +{ + struct tee_ioctl_close_session_arg arg; + + memset(&arg, 0, sizeof(arg)); + + if (!session) + return; + + arg.session = session->session_id; + if (ioctl(session->ctx->fd, TEE_IOC_CLOSE_SESSION, &arg)) + EMSG("Failed to close session 0x%x", session->session_id); +} + +TEEC_Result TEEC_InvokeCommand(TEEC_Session *session, uint32_t cmd_id, + TEEC_Operation *operation, uint32_t *error_origin) +{ + struct tee_ioctl_invoke_arg *arg = NULL; + struct tee_ioctl_param *params = NULL; + TEEC_Result res = TEEC_ERROR_GENERIC; + uint32_t eorig = 0; + int rc = 0; + const size_t arg_size = sizeof(struct tee_ioctl_invoke_arg) + + TEEC_CONFIG_PAYLOAD_REF_COUNT * + sizeof(struct tee_ioctl_param); + union { + struct tee_ioctl_invoke_arg arg; + uint8_t data[arg_size]; + } buf; + struct tee_ioctl_buf_data buf_data; + TEEC_SharedMemory shm[TEEC_CONFIG_PAYLOAD_REF_COUNT]; + + memset(&buf, 0, sizeof(buf)); + memset(&buf_data, 0, sizeof(buf_data)); + memset(&shm, 0, sizeof(shm)); + + if (!session) { + eorig = TEEC_ORIGIN_API; + res = TEEC_ERROR_BAD_PARAMETERS; + goto out; + } + + bm_timestamp(); + + buf_data.buf_ptr = (uintptr_t)&buf; + buf_data.buf_len = sizeof(buf); + + arg = &buf.arg; + arg->num_params = TEEC_CONFIG_PAYLOAD_REF_COUNT; + params = (struct tee_ioctl_param *)(arg + 1); + + arg->session = session->session_id; + arg->func = cmd_id; + + if (operation) { + teec_mutex_lock(&teec_mutex); + operation->session = session; + teec_mutex_unlock(&teec_mutex); + } + + res = teec_pre_process_operation(session->ctx, operation, params, shm); + if (res != TEEC_SUCCESS) { + eorig = TEEC_ORIGIN_API; + goto out_free_temp_refs; + } + + rc = ioctl(session->ctx->fd, TEE_IOC_INVOKE, &buf_data); + if (rc) { + EMSG("TEE_IOC_INVOKE failed"); + eorig = TEEC_ORIGIN_COMMS; + res = ioctl_errno_to_res(errno); + goto out_free_temp_refs; + } + + res = arg->ret; + eorig = arg->ret_origin; + teec_post_process_operation(operation, params, shm); + + bm_timestamp(); + +out_free_temp_refs: + teec_free_temp_refs(operation, shm); +out: + if (error_origin) + *error_origin = eorig; + return res; +} + +void TEEC_RequestCancellation(TEEC_Operation *operation) +{ + TEEC_Session *session = NULL; + struct tee_ioctl_cancel_arg arg; + + memset(&arg, 0, sizeof(arg)); + + if (!operation) + return; + + teec_mutex_lock(&teec_mutex); + session = operation->session; + teec_mutex_unlock(&teec_mutex); + + if (!session) + return; + + arg.session = session->session_id; + arg.cancel_id = 0; + + if (ioctl(session->ctx->fd, TEE_IOC_CANCEL, &arg)) + EMSG("TEE_IOC_CANCEL: %s", strerror(errno)); +} + +TEEC_Result TEEC_RegisterSharedMemory(TEEC_Context *ctx, TEEC_SharedMemory *shm) +{ + TEEC_Result res = TEEC_SUCCESS; + int fd = 0; + size_t s = 0; + + if (!ctx || !shm) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!shm->flags || (shm->flags & ~(TEEC_MEM_INPUT | TEEC_MEM_OUTPUT))) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!shm->buffer) + return TEEC_ERROR_BAD_PARAMETERS; + + s = shm->size; + if (!s) + s = 8; + if (ctx->reg_mem) { + fd = teec_shm_register(ctx->fd, shm->buffer, s, &shm->id); + if (fd >= 0) { + shm->registered_fd = fd; + shm->shadow_buffer = NULL; + shm->internal.flags = 0; + goto out; + } + + /* + * If we're here TEE_IOC_SHM_REGISTER failed, probably + * because some read-only memory was supplied and the Linux + * kernel doesn't like that at the moment. + * + * The error could also have some other origin. In any case + * we're not making matters worse by trying to allocate and + * register a shadow buffer before giving up. + */ + shm->shadow_buffer = teec_paged_aligned_alloc(s); + if (!shm->shadow_buffer) + return TEEC_ERROR_OUT_OF_MEMORY; + fd = teec_shm_register(ctx->fd, shm->shadow_buffer, s, + &shm->id); + if (fd >= 0) { + shm->registered_fd = fd; + shm->internal.flags = SHM_FLAG_SHADOW_BUFFER_ALLOCED; + goto out; + } + + if (errno == ENOMEM) + res = TEEC_ERROR_OUT_OF_MEMORY; + else + res = TEEC_ERROR_GENERIC; + free(shm->shadow_buffer); + shm->shadow_buffer = NULL; + return res; + } else { + fd = teec_shm_alloc(ctx->fd, s, &shm->id); + if (fd < 0) + return TEEC_ERROR_OUT_OF_MEMORY; + + shm->shadow_buffer = mmap(NULL, s, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + close(fd); + if (shm->shadow_buffer == (void *)MAP_FAILED) { + shm->id = -1; + return TEEC_ERROR_OUT_OF_MEMORY; + } + shm->registered_fd = -1; + shm->internal.flags = 0; + } + +out: + shm->alloced_size = s; + return TEEC_SUCCESS; +} + +TEEC_Result TEEC_RegisterSharedMemoryFileDescriptor(TEEC_Context *ctx, + TEEC_SharedMemory *shm, + int fd) +{ + int rfd = 0; + struct tee_ioctl_shm_register_fd_data data; + + memset(&data, 0, sizeof(data)); + + if (!ctx || !shm || fd < 0) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!shm->flags || (shm->flags & ~(TEEC_MEM_INPUT | TEEC_MEM_OUTPUT))) + return TEEC_ERROR_BAD_PARAMETERS; + + data.fd = fd; + rfd = ioctl(ctx->fd, TEE_IOC_SHM_REGISTER_FD, &data); + if (rfd < 0) + return TEEC_ERROR_BAD_PARAMETERS; + + shm->buffer = NULL; + shm->shadow_buffer = NULL; + shm->registered_fd = rfd; + shm->id = data.id; + shm->size = data.size; + return TEEC_SUCCESS; +} + +TEEC_Result TEEC_AllocateSharedMemory(TEEC_Context *ctx, TEEC_SharedMemory *shm) +{ + int fd = 0; + size_t s = 0; + + if (!ctx || !shm) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!shm->flags || (shm->flags & ~(TEEC_MEM_INPUT | TEEC_MEM_OUTPUT))) + return TEEC_ERROR_BAD_PARAMETERS; + + s = shm->size; + if (!s) + s = 8; + + if (ctx->reg_mem) { + shm->buffer = teec_paged_aligned_alloc(s); + if (!shm->buffer) + return TEEC_ERROR_OUT_OF_MEMORY; + + fd = teec_shm_register(ctx->fd, shm->buffer, s, &shm->id); + if (fd < 0) { + free(shm->buffer); + shm->buffer = NULL; + return TEEC_ERROR_OUT_OF_MEMORY; + } + shm->registered_fd = fd; + } else { + fd = teec_shm_alloc(ctx->fd, s, &shm->id); + if (fd < 0) + return TEEC_ERROR_OUT_OF_MEMORY; + + shm->buffer = mmap(NULL, s, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + close(fd); + if (shm->buffer == (void *)MAP_FAILED) { + shm->id = -1; + return TEEC_ERROR_OUT_OF_MEMORY; + } + shm->registered_fd = -1; + } + + shm->shadow_buffer = NULL; + shm->alloced_size = s; + shm->internal.flags = SHM_FLAG_BUFFER_ALLOCED; + return TEEC_SUCCESS; +} + +void TEEC_ReleaseSharedMemory(TEEC_SharedMemory *shm) +{ + if (!shm || shm->id == -1) + return; + + if (shm->shadow_buffer) { + if (shm->registered_fd >= 0) { + if (shm->internal.flags & + SHM_FLAG_SHADOW_BUFFER_ALLOCED) + free(shm->shadow_buffer); + close(shm->registered_fd); + } else { + munmap(shm->shadow_buffer, shm->alloced_size); + } + } else if (shm->buffer) { + if (shm->registered_fd >= 0) { + if (shm->internal.flags & SHM_FLAG_BUFFER_ALLOCED) + free(shm->buffer); + close(shm->registered_fd); + } else { + munmap(shm->buffer, shm->alloced_size); + } + } else if (shm->registered_fd >= 0) { + close(shm->registered_fd); + } + + shm->id = -1; + shm->shadow_buffer = NULL; + shm->buffer = NULL; + shm->registered_fd = -1; + shm->internal.flags = 0; +} diff --git a/optee_client/libteec/src/teec_benchmark.c b/optee_client/libteec/src/teec_benchmark.c new file mode 100644 index 0000000..0d04c19 --- /dev/null +++ b/optee_client/libteec/src/teec_benchmark.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "teec_benchmark.h" + +struct tee_ts_global *bench_ts_global; +static const TEEC_UUID pta_benchmark_uuid = PTA_BENCHMARK_UUID; + +static TEEC_Context bench_ctx; +static TEEC_Session bench_sess; + +static pthread_mutex_t teec_bench_mu = PTHREAD_MUTEX_INITIALIZER; + +/* Cycle counter */ +static inline uint64_t read_ccounter(void) +{ + uint64_t ccounter = 0; +#ifdef __aarch64__ + asm volatile("mrs %0, PMCCNTR_EL0" : "=r"(ccounter)); +#else + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(ccounter)); +#endif + return ccounter * TEE_BENCH_DIVIDER; +} + +static TEEC_Result benchmark_pta_open(void) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + uint32_t ret_orig = 0; + + res = TEEC_InitializeContext(NULL, &bench_ctx); + if (res != TEEC_SUCCESS) + return res; + + res = TEEC_OpenSession(&bench_ctx, &bench_sess, + &pta_benchmark_uuid, + TEEC_LOGIN_PUBLIC, NULL, NULL, &ret_orig); + if (res != TEEC_SUCCESS) { + TEEC_FinalizeContext(&bench_ctx); + return res; + } + + return res; +} + +static void benchmark_pta_close(void) +{ + TEEC_CloseSession(&bench_sess); + TEEC_FinalizeContext(&bench_ctx); +} + +static TEEC_Result benchmark_get_bench_buf_paddr(uint64_t *paddr_ts_buf, + uint64_t *size) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + uint32_t ret_orig = 0; + TEEC_Operation op; + + memset(&op, 0, sizeof(op)); + + res = benchmark_pta_open(); + if (res != TEEC_SUCCESS) + return res; + + op.paramTypes = TEEC_PARAM_TYPES(TEEC_VALUE_OUTPUT, TEEC_NONE, + TEEC_NONE, TEEC_NONE); + + res = TEEC_InvokeCommand(&bench_sess, BENCHMARK_CMD_GET_MEMREF, + &op, &ret_orig); + if (res != TEEC_SUCCESS) + return res; + + *paddr_ts_buf = op.params[0].value.a; + *size = op.params[0].value.b; + + benchmark_pta_close(); + + return res; +} + +static void *mmap_paddr(intptr_t paddr, uint64_t size) +{ + int devmem = 0; + off_t offset = 0; + off_t page_addr = 0; + intptr_t *hw_addr = NULL; + + devmem = open("/dev/mem", O_RDWR); + if (!devmem) + return NULL; + + offset = (off_t)paddr % getpagesize(); + page_addr = (off_t)(paddr - offset); + + hw_addr = (intptr_t *)mmap(0, size + offset, PROT_READ|PROT_WRITE, + MAP_SHARED, devmem, page_addr); + if (hw_addr == MAP_FAILED) { + close(devmem); + return NULL; + } + + close(devmem); + return (hw_addr + offset); +} + +/* check if we are in benchmark mode */ +static bool benchmark_check_mode(void) +{ + uint64_t ts_buf_raw = 0; + uint64_t ts_buf_size = 0; + bool res = true; + + if (!bench_ts_global) { + /* receive buffer from Benchmark PTA and register it */ + benchmark_get_bench_buf_paddr(&ts_buf_raw, &ts_buf_size); + if (ts_buf_raw && ts_buf_size) { + bench_ts_global = mmap_paddr(ts_buf_raw, ts_buf_size); + res = (bench_ts_global) ? true : false; + } else { + res = false; + } + } + + return res; +} + +/* Adding timestamp to buffer */ +void bm_timestamp(void) +{ + struct tee_ts_cpu_buf *cpu_buf = NULL; + uint64_t ts_i = 0; + void *ret_addr = NULL; + uint32_t cur_cpu = 0; + int ret = 0; + cpu_set_t cpu_set_old; + cpu_set_t cpu_set_tmp; + struct tee_time_st ts_data; + + memset(&cpu_set_old, 0, sizeof(cpu_set_old)); + memset(&cpu_set_tmp, 0, sizeof(cpu_set_tmp)); + memset(&ts_data, 0, sizeof(ts_data)); + + if (pthread_mutex_trylock(&teec_bench_mu)) + return; + + if (!benchmark_check_mode()) + goto error; + + CPU_ZERO(&cpu_set_old); + ret = sched_getaffinity(0, sizeof(cpu_set_old), &cpu_set_old); + if (ret) + goto error; + + /* stick to the same core while putting timestamp */ + cur_cpu = sched_getcpu(); + CPU_ZERO(&cpu_set_tmp); + CPU_SET(cur_cpu, &cpu_set_tmp); + ret = sched_setaffinity(0, sizeof(cpu_set_tmp), &cpu_set_tmp); + if (ret) + goto error; + + /* fill timestamp data */ + if (cur_cpu >= bench_ts_global->cores) { + ret = sched_setaffinity(0, sizeof(cpu_set_old), &cpu_set_old); + goto error; + } + + ret_addr = __builtin_return_address(0); + + cpu_buf = &bench_ts_global->cpu_buf[cur_cpu]; + ts_i = __sync_fetch_and_add(&cpu_buf->head, 1); + ts_data.cnt = read_ccounter(); + ts_data.addr = (uintptr_t)ret_addr; + ts_data.src = TEE_BENCH_CLIENT; + cpu_buf->stamps[ts_i & TEE_BENCH_MAX_MASK] = ts_data; + + /* set back affinity mask */ + sched_setaffinity(0, sizeof(cpu_set_old), &cpu_set_old); + +error: + pthread_mutex_unlock(&teec_bench_mu); +} + diff --git a/optee_client/libteec/src/teec_benchmark.h b/optee_client/libteec/src/teec_benchmark.h new file mode 100644 index 0000000..17d99ac --- /dev/null +++ b/optee_client/libteec/src/teec_benchmark.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __TEEC_BENCHMARK_H +#define __TEEC_BENCHMARK_H + +#ifdef CFG_TEE_BENCHMARK +void bm_timestamp(void); +#else +static inline void bm_timestamp(void) {} +#endif + +#endif /* __TEEC_BENCHMARK_H */ diff --git a/optee_client/libteec/src/teec_trace.c b/optee_client/libteec/src/teec_trace.c new file mode 100644 index 0000000..7194c8c --- /dev/null +++ b/optee_client/libteec/src/teec_trace.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "teec_trace.h" + +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) + +/* + * The length of the prefix is 37, for example it looks like this: + * P = Prefix + * M = Message + * F = Function name + * L = Line number + * PPPP: MMMMM [FFFFFFFFFFFFFFF : LLLLL] + */ +#define MAX_PRINT_SIZE 256 + +#ifdef TEEC_LOG_FILE +static void log_to_file(const char *buffer) +{ + FILE *log_file = fopen(TEEC_LOG_FILE, "a"); + + if (log_file != NULL) { + fprintf(log_file, "%s", buffer); + fclose(log_file); + log_file = NULL; + } +} +#else +#define log_to_file(buffer) +#endif + +static const char * const trace_level_strings[] = { + "", "ERR", "INF", "DBG", "FLW" +}; + +void _dprintf(const char *function, int line, int level, const char *prefix, + const char *fmt, ...) +{ + char msg[MAX_PRINT_SIZE]; + int n = 0; + va_list ap; + + if (function) { + int thread_id = syscall(SYS_gettid); + + n = snprintf(msg, sizeof(msg), "%s [%d] %s:%s:%d: ", + trace_level_strings[level], thread_id, prefix, + function, line); + if (n < 0) + return; + } + + if ((size_t)n < sizeof(msg)) { + va_start(ap, fmt); + n = vsnprintf(msg + n, sizeof(msg) - n, fmt, ap); + va_end(ap); + if (n < 0) + return; + } + + fprintf(stdout, "%s", msg); + log_to_file(msg); +} + +#if (defined(DEBUGLEVEL_3) || defined(DEBUGLEVEL_true) || defined(DEBUGLEVEL_4)) +void dump_buffer(const char *bname, const uint8_t *buffer, size_t blen) +{ + fprintf(stderr, "#### %s\n", bname); + + while (blen > 0) { + size_t n = 0; + + for (n = 0; n < 16; n++) { + if (n < blen) + fprintf(stderr, "%02x ", (int)buffer[n]); + else + fprintf(stderr, " "); + + if (n == 7) + fprintf(stderr, " "); + } + + fprintf(stderr, " |"); + + for (n = 0; n < 16; n++) { + if (n < blen) { + if (isprint(buffer[n])) + fprintf(stderr, "%c", (int)buffer[n]); + else + fprintf(stderr, "."); + } + } + + fprintf(stderr, "|\n"); + + blen -= MIN(blen, 16); + buffer += 16; + } +} +#else +void dump_buffer(const char *bname, const uint8_t *buffer, size_t blen) +{ + (void)bname; + (void)buffer; + (void)blen; +} +#endif diff --git a/optee_client/scripts/checkpatch_inc.sh b/optee_client/scripts/checkpatch_inc.sh new file mode 100644 index 0000000..23a503c --- /dev/null +++ b/optee_client/scripts/checkpatch_inc.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-2-Clause + +CHECKPATCH="${CHECKPATCH:-checkpatch.pl}" +# checkpatch.pl will ignore the following paths +CHECKPATCH_IGNORE=$(echo ) +_CP_EXCL=$(for p in $CHECKPATCH_IGNORE; do echo ":(exclude)$p" ; done) + +function _checkpatch() { + # Use --typedefsfile if supported by the checkpatch tool + typedefs_opt="--typedefsfile typedefs.checkpatch" + $CHECKPATCH --help 2>&1 | grep -q -- --typedefsfile || \ + typedefs_opt=""; + # Ignore NOT_UNIFIED_DIFF in case patch has no diff + # (e.g., all paths filtered out) + $CHECKPATCH --quiet --ignore FILE_PATH_CHANGES \ + --ignore GERRIT_CHANGE_ID \ + --ignore NOT_UNIFIED_DIFF \ + --ignore CAMELCASE \ + --ignore PREFER_KERNEL_TYPES \ + --ignore CONCATENATED_STRING \ + --no-tree \ + --strict \ + $typedefs_opt \ + - +} + +function checkpatch() { + git show --oneline --no-patch $1 + # The first git 'format-patch' shows the commit message + # The second one produces the diff (might be empty if _CP_EXCL + # filters out all diffs) + (git format-patch $1^..$1 --stdout | sed -n '/^diff --git/q;p'; \ + git format-patch $1^..$1 --stdout -- $_CP_EXCL . | \ + sed -n '/^diff --git/,$p') | _checkpatch +} + +function checkstaging() { + git diff --cached -- . $_CP_EXCL | _checkpatch +} + +function checkworking() { + git diff -- . $_CP_EXCL | _checkpatch +} + +function checkdiff() { + git diff $1...$2 -- . $_CP_EXCL | _checkpatch +} + diff --git a/optee_client/tee-supplicant/CMakeLists.txt b/optee_client/tee-supplicant/CMakeLists.txt new file mode 100644 index 0000000..b98fe2e --- /dev/null +++ b/optee_client/tee-supplicant/CMakeLists.txt @@ -0,0 +1,117 @@ +project(tee-supplicant C) + +################################################################################ +# Configuration flags always included +################################################################################ +option(CFG_TA_TEST_PATH "Enable tee-supplicant to load from test/debug path" OFF) +option(RPMB_EMU "Enable tee-supplicant to emulate RPMB" ON) +option(CFG_TA_GPROF_SUPPORT "Enable tee-supplicant support for TAs instrumented with gprof" ON) +option(CFG_FTRACE_SUPPORT "Enable tee-supplicant support for TAs instrumented with ftrace" ON) +option(CFG_TEE_SUPP_PLUGINS "Enable tee-supplicant plugin support" ON) + +set(CFG_TEE_SUPP_LOG_LEVEL "1" CACHE STRING "tee-supplicant log level") +# FIXME: Question is, is this really needed? Should just use defaults from # GNUInstallDirs? +set(CFG_TEE_CLIENT_LOAD_PATH "/lib" CACHE STRING "Colon-separated list of paths where to look for TAs (see also --ta-dir)") +set(CFG_TEE_FS_PARENT_PATH "/data/tee" CACHE STRING "Location of TEE filesystem (secure storage)") +# FIXME: Why do we have if defined(CFG_GP_SOCKETS) && CFG_GP_SOCKETS == 1 in the c-file? +set(CFG_GP_SOCKETS "1" CACHE STRING "Enable GlobalPlatform Socket API support") +set(CFG_TEE_PLUGIN_LOAD_PATH "/usr/lib/tee-supplicant/plugins/" CACHE STRING "tee-supplicant's plugins path") + +if(CFG_TEE_SUPP_PLUGINS) + set(CMAKE_INSTALL_RPATH "${CFG_TEE_PLUGIN_LOAD_PATH}") +endif() + +################################################################################ +# Source files +################################################################################ +set(SRC + src/handle.c + src/hmac_sha2.c + src/rpmb.c + src/sha2.c + src/tee_supp_fs.c + src/tee_supplicant.c + src/teec_ta_load.c +) + +if(CFG_GP_SOCKETS) + set(SRC ${SRC} src/tee_socket.c) +endif() + +if(CFG_TA_GPROF_SUPPORT OR CFG_FTRACE_SUPPORT) + set(SRC ${SRC} src/prof.c) +endif() + +if(CFG_TEE_SUPP_PLUGINS) + set(SRC ${SRC} src/plugin.c) +endif() + +################################################################################ +# Built binary +################################################################################ +add_executable(${PROJECT_NAME} ${SRC}) + +################################################################################ +# Flags always set +################################################################################ +target_compile_definitions(${PROJECT_NAME} + PRIVATE -DDEBUGLEVEL_${CFG_TEE_SUPP_LOG_LEVEL} + PRIVATE -DTEEC_LOAD_PATH="${CFG_TEE_CLIENT_LOAD_PATH}" + PRIVATE -DTEE_FS_PARENT_PATH="${CFG_TEE_FS_PARENT_PATH}" + PRIVATE -DBINARY_PREFIX="TSUP" +) + +################################################################################ +# Optional flags +################################################################################ +if(CFG_GP_SOCKETS) + target_compile_definitions(${PROJECT_NAME} + PRIVATE -DCFG_GP_SOCKETS=${CFG_GP_SOCKETS}) +endif() + +if(CFG_TA_TEST_PATH) + target_compile_definitions(${PROJECT_NAME} + PRIVATE -DCFG_TA_TEST_PATH=${CFG_TA_TEST_PATH}) +endif() + +if(RPMB_EMU) + target_compile_definitions(${PROJECT_NAME} + PRIVATE -DRPMB_EMU=1) +endif() + +if(CFG_TA_GPROF_SUPPORT) + target_compile_definitions(${PROJECT_NAME} + PRIVATE -DCFG_TA_GPROF_SUPPORT) +endif() + +if(CFG_FTRACE_SUPPORT) + target_compile_definitions(${PROJECT_NAME} + PRIVATE -DCFG_FTRACE_SUPPORT) +endif() + +if(CFG_TEE_SUPP_PLUGINS) + target_compile_definitions(${PROJECT_NAME} + PRIVATE -DTEE_SUPP_PLUGINS + PRIVATE -DTEE_PLUGIN_LOAD_PATH="${CFG_TEE_PLUGIN_LOAD_PATH}" + ) +endif() + +################################################################################ +# Public and private header and library dependencies +################################################################################ +target_include_directories(${PROJECT_NAME} PRIVATE src) + +target_link_libraries(${PROJECT_NAME} + PRIVATE teec +) + +if(CFG_TEE_SUPP_PLUGINS) + target_link_libraries(${PROJECT_NAME} + PRIVATE dl + ) +endif() + +################################################################################ +# Install targets +################################################################################ +install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}) diff --git a/optee_client/tee-supplicant/Makefile b/optee_client/tee-supplicant/Makefile new file mode 100644 index 0000000..6f59c8c --- /dev/null +++ b/optee_client/tee-supplicant/Makefile @@ -0,0 +1,104 @@ +include ../flags.mk +include ../config.mk + +OUT_DIR := $(OO)/tee-supplicant + +# Emulate RPMB ioctl's by default +RPMB_EMU ?= 1 + +.PHONY: all tee-supplicant clean + +all: tee-supplicant +################################################################################ +# Teec configuration +################################################################################ +PACKAGE_NAME := tee-supplicant + +TEES_SRCS := tee_supplicant.c \ + teec_ta_load.c \ + tee_supp_fs.c \ + rpmb.c \ + handle.c + +ifeq ($(CFG_GP_SOCKETS),y) +TEES_SRCS += tee_socket.c +endif + +ifeq ($(RPMB_EMU),1) +TEES_SRCS += sha2.c hmac_sha2.c +endif +ifneq (,$(filter y,$(CFG_TA_GPROF_SUPPORT) $(CFG_FTRACE_SUPPORT))) +TEES_SRCS += prof.c +endif + +ifeq ($(CFG_TEE_SUPP_PLUGINS),y) +TEES_SRCS += plugin.c +endif + +TEES_SRC_DIR := src +TEES_OBJ_DIR := $(OUT_DIR) +TEES_OBJS := $(patsubst %.c,$(TEES_OBJ_DIR)/%.o, $(TEES_SRCS)) +TEES_INCLUDES := ${CURDIR}/../libteec/include \ + ${CURDIR}/src \ + ${CURDIR}/../libteec/include \ + +TEES_CFLAGS := $(addprefix -I, $(TEES_INCLUDES)) $(CFLAGS) \ + -DDEBUGLEVEL_$(CFG_TEE_SUPP_LOG_LEVEL) \ + -DBINARY_PREFIX=\"TEES\" \ + -DTEE_FS_PARENT_PATH=\"$(CFG_TEE_FS_PARENT_PATH)\" \ + -DTEEC_LOAD_PATH=\"$(CFG_TEE_CLIENT_LOAD_PATH)\" \ + +ifeq ($(CFG_GP_SOCKETS),y) +TEES_CFLAGS += -DCFG_GP_SOCKETS=1 +endif +ifeq ($(RPMB_EMU),1) +TEES_CFLAGS += -DRPMB_EMU=1 +endif +ifeq ($(CFG_TA_TEST_PATH),y) +TEES_CFLAGS += -DCFG_TA_TEST_PATH=1 +endif +TEES_FILE := $(OUT_DIR)/$(PACKAGE_NAME) +TEES_LFLAGS := $(LDFLAGS) -L$(OUT_DIR)/../libteec -lteec + +ifeq ($(CFG_TA_GPROF_SUPPORT),y) +TEES_CFLAGS += -DCFG_TA_GPROF_SUPPORT +endif + +ifeq ($(CFG_FTRACE_SUPPORT),y) +TEES_CFLAGS += -DCFG_FTRACE_SUPPORT +endif + +ifeq ($(CFG_TEE_SUPP_PLUGINS),y) +TEES_CFLAGS += -DTEE_SUPP_PLUGINS \ + -DTEE_PLUGIN_LOAD_PATH=\"$(CFG_TEE_PLUGIN_LOAD_PATH)\" +endif + +TEES_LFLAGS += -lpthread +# Needed to get clock_gettime() for for glibc versions before 2.17 +TEES_LFLAGS += -lrt + +ifeq ($(CFG_TEE_SUPP_PLUGINS),y) +# Needed to dynamically load user plugins +TEES_LFLAGS += -ldl +# Needed for dlopen() +TEES_LFLAGS += -Wl,-rpath=$(CFG_TEE_PLUGIN_LOAD_PATH) +endif + +tee-supplicant: $(TEES_FILE) + +$(TEES_FILE): $(TEES_OBJS) + @echo " LINK $@" + $(VPREFIX)$(CC) -o $@ $+ $(TEES_LFLAGS) + @echo "" + +$(TEES_OBJ_DIR)/%.o: $(TEES_SRC_DIR)/%.c + $(VPREFIX)mkdir -p $(dir $@) + @echo " CC $<" + $(VPREFIX)$(CC) $(TEES_CFLAGS) $(TEES_CFLAGS_$(notdir $<)) -c $< -o $@ + +################################################################################ +# Cleaning up configuration +################################################################################ +clean: + $(RM) $(TEES_OBJS) $(TEES_FILE) + $(call rmdir,$(OUT_DIR)) diff --git a/optee_client/tee-supplicant/src/__tee_ipsocket.h b/optee_client/tee-supplicant/src/__tee_ipsocket.h new file mode 100644 index 0000000..7c33666 --- /dev/null +++ b/optee_client/tee-supplicant/src/__tee_ipsocket.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef ____TEE_IPSOCKET_H +#define ____TEE_IPSOCKET_H + +typedef enum TEE_ipSocket_ipVersion_e { + TEE_IP_VERSION_DC = 0, /* don’t care */ + TEE_IP_VERSION_4 = 1, + TEE_IP_VERSION_6 = 2 +} TEE_ipSocket_ipVersion; + +#endif /*____TEE_IPSOCKET_H*/ diff --git a/optee_client/tee-supplicant/src/__tee_isocket_defines.h b/optee_client/tee-supplicant/src/__tee_isocket_defines.h new file mode 100644 index 0000000..b38457a --- /dev/null +++ b/optee_client/tee-supplicant/src/__tee_isocket_defines.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef ____TEE_ISOCKET_DEFINES_H +#define ____TEE_ISOCKET_DEFINES_H + +#define TEE_ISOCKET_VERSION 0x01000000 + +#define TEE_ISOCKET_ERROR_PROTOCOL 0xF1007001 +#define TEE_ISOCKET_ERROR_REMOTE_CLOSED 0xF1007002 +#define TEE_ISOCKET_ERROR_TIMEOUT 0xF1007003 +#define TEE_ISOCKET_ERROR_OUT_OF_RESOURCES 0xF1007004 +#define TEE_ISOCKET_ERROR_LARGE_BUFFER 0xF1007005 +#define TEE_ISOCKET_WARNING_PROTOCOL 0xF1007006 +#define TEE_ISOCKET_ERROR_HOSTNAME 0xF1007007 + +#endif /*____TEE_ISOCKET_DEFINES_H*/ diff --git a/optee_client/tee-supplicant/src/__tee_tcpsocket_defines.h b/optee_client/tee-supplicant/src/__tee_tcpsocket_defines.h new file mode 100644 index 0000000..21853e0 --- /dev/null +++ b/optee_client/tee-supplicant/src/__tee_tcpsocket_defines.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef ____TEE_TCPSOCKET_DEFINES_H +#define ____TEE_TCPSOCKET_DEFINES_H + +/* Protocol identifier */ +#define TEE_ISOCKET_PROTOCOLID_TCP 0x65 + +/* Instance specific errors */ +#define TEE_ISOCKET_TCP_WARNING_UNKNOWN_OUT_OF_BAND 0xF1010002 + +#endif /*____TEE_TCPSOCKET_DEFINES_H*/ diff --git a/optee_client/tee-supplicant/src/__tee_tcpsocket_defines_extensions.h b/optee_client/tee-supplicant/src/__tee_tcpsocket_defines_extensions.h new file mode 100644 index 0000000..3a03a2c --- /dev/null +++ b/optee_client/tee-supplicant/src/__tee_tcpsocket_defines_extensions.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef ____TEE_TCPSOCKET_DEFINES_EXTENSIONS_H +#define ____TEE_TCPSOCKET_DEFINES_EXTENSIONS_H + +/* Instance and implementation specific ioctl functions */ +#define TEE_TCP_SET_RECVBUF 0x65f00000 +#define TEE_TCP_SET_SENDBUF 0x65f00001 + +#endif /*____TEE_TCPSOCKET_DEFINES_EXTENSIONS_H*/ diff --git a/optee_client/tee-supplicant/src/__tee_udpsocket_defines.h b/optee_client/tee-supplicant/src/__tee_udpsocket_defines.h new file mode 100644 index 0000000..64ec09c --- /dev/null +++ b/optee_client/tee-supplicant/src/__tee_udpsocket_defines.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef ____TEE_UDPSOCKET_DEFINES_H +#define ____TEE_UDPSOCKET_DEFINES_H + +/* Protocol identifier */ +#define TEE_ISOCKET_PROTOCOLID_UDP 0x66 + +/* Instance specific errors */ +#define TEE_ISOCKET_UDP_WARNING_UNKNOWN_OUT_OF_BAND 0xF1020002 + +/* Instance specific ioctl functions */ +#define TEE_UDP_CHANGEADDR 0x66000001 +#define TEE_UDP_CHANGEPORT 0x66000002 + +#endif /*____TEE_UDPSOCKET_DEFINES_H*/ diff --git a/optee_client/tee-supplicant/src/handle.c b/optee_client/tee-supplicant/src/handle.c new file mode 100644 index 0000000..58254bd --- /dev/null +++ b/optee_client/tee-supplicant/src/handle.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include "handle.h" + +/* + * Define the initial capacity of the database. It should be a low number + * multiple of 2 since some databases a likely to only use a few handles. + * Since the algorithm is to doubles up when growing it shouldn't cause a + * noticable overhead on large databases. + */ +#define HANDLE_DB_INITIAL_MAX_PTRS 4 + +static void mutex_lock(struct handle_db *db) +{ + if (db->mu) + pthread_mutex_lock(db->mu); +} + +static void mutex_unlock(struct handle_db *db) +{ + if (db->mu) + pthread_mutex_unlock(db->mu); +} + + +void handle_db_set_mutex(struct handle_db *db, pthread_mutex_t *mu) +{ + db->mu = mu; +} + +void handle_db_destroy(struct handle_db *db) +{ + if (db) { + mutex_lock(db); + free(db->ptrs); + db->ptrs = NULL; + db->max_ptrs = 0; + mutex_unlock(db); + } +} + +int handle_get(struct handle_db *db, void *ptr) +{ + size_t n = 0; + void *p = NULL; + size_t new_max_ptrs = 0; + int ret = 0; + + if (!db || !ptr) + return -1; + + mutex_lock(db); + + /* Try to find an empty location */ + for (n = 0; n < db->max_ptrs; n++) { + if (!db->ptrs[n]) { + db->ptrs[n] = ptr; + ret = n; + goto out; + } + } + + /* No location available, grow the ptrs array */ + if (db->max_ptrs) + new_max_ptrs = db->max_ptrs * 2; + else + new_max_ptrs = HANDLE_DB_INITIAL_MAX_PTRS; + p = realloc(db->ptrs, new_max_ptrs * sizeof(void *)); + if (!p) { + ret = -1; + goto out; + } + db->ptrs = p; + memset(db->ptrs + db->max_ptrs, 0, + (new_max_ptrs - db->max_ptrs) * sizeof(void *)); + db->max_ptrs = new_max_ptrs; + + /* Since n stopped at db->max_ptrs there is an empty location there */ + db->ptrs[n] = ptr; + ret = n; + +out: + mutex_unlock(db); + return ret; +} + +void *handle_put(struct handle_db *db, int handle) +{ + void *p = NULL; + + if (!db || handle < 0) + return NULL; + + mutex_lock(db); + + if ((size_t)handle >= db->max_ptrs) { + p = NULL; + goto out; + } + + p = db->ptrs[handle]; + db->ptrs[handle] = NULL; + +out: + mutex_unlock(db); + return p; +} + +void *handle_lookup(struct handle_db *db, int handle) +{ + void *p = NULL; + + if (!db || handle < 0) + return NULL; + + mutex_lock(db); + + if ((size_t)handle >= db->max_ptrs) { + p = NULL; + goto out; + } + + p = db->ptrs[handle]; + +out: + mutex_unlock(db); + return p; +} + +void handle_foreach_put(struct handle_db *db, + void (*cb)(int handle, void *ptr, void *arg), + void *arg) +{ + size_t n = 0; + + if (!db || !cb) + return; + + mutex_lock(db); + + for (n = 0; n < db->max_ptrs; n++) { + if (db->ptrs[n]) { + cb(n, db->ptrs[n], arg); + db->ptrs[n] = NULL; + } + } + + mutex_unlock(db); +} diff --git a/optee_client/tee-supplicant/src/handle.h b/optee_client/tee-supplicant/src/handle.h new file mode 100644 index 0000000..5be5461 --- /dev/null +++ b/optee_client/tee-supplicant/src/handle.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef HANDLE_H +#define HANDLE_H + +#include +#include + +struct handle_db { + void **ptrs; + size_t max_ptrs; + pthread_mutex_t *mu; +}; + +#define HANDLE_DB_INITIALIZER { NULL, 0, NULL } +#define HANDLE_DB_INITIALIZER_WITH_MUTEX(mu) { NULL, 0, (mu) } + +/* + * Assigns a mutex for the database. If mu != NULL the mutex will be + * acquired before each access to the database and released when + * the operation is done. + */ +void handle_db_set_mutex(struct handle_db *db, pthread_mutex_t *mu); + +/* + * Frees all internal data structures of the database, but does not free + * the db pointer. The database is safe to reuse after it's destroyed, it + * just be empty again. The assigned mutex is also preserved. + */ +void handle_db_destroy(struct handle_db *db); + +/* + * Allocates a new handle and assigns the supplied pointer to it, + * ptr must not be NULL. + * The function returns + * >= 0 on success and + * -1 on failure + */ +int handle_get(struct handle_db *db, void *ptr); + +/* + * Deallocates a handle. Returns the assiciated pointer of the handle + * the the handle was valid or NULL if it's invalid. + */ +void *handle_put(struct handle_db *db, int handle); + +/* + * Returns the assiciated pointer of the handle if the handle is a valid + * handle. + * Returns NULL on failure. + */ +void *handle_lookup(struct handle_db *db, int handle); + +void handle_foreach_put(struct handle_db *db, + void (*cb)(int handle, void *ptr, void *arg), + void *arg); + +#endif /*HANDLE_H*/ diff --git a/optee_client/tee-supplicant/src/hmac_sha2.c b/optee_client/tee-supplicant/src/hmac_sha2.c new file mode 100644 index 0000000..20e994a --- /dev/null +++ b/optee_client/tee-supplicant/src/hmac_sha2.c @@ -0,0 +1,126 @@ +/* + * HMAC-SHA-224/256/384/512 implementation + * Last update: 06/15/2005 + * Issue date: 06/15/2005 + * + * Copyright (C) 2005 Olivier Gay + * All rights reserved. + * + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include "hmac_sha2.h" + +/* HMAC-SHA-256 functions */ + +void hmac_sha256_init(hmac_sha256_ctx *ctx, const unsigned char *key, + unsigned int key_size) +{ + unsigned int fill = 0; + unsigned int num = 0; + const unsigned char *key_used = NULL; + unsigned char key_temp[SHA256_DIGEST_SIZE] = { 0 }; + int i = 0; + + if (key_size == SHA256_BLOCK_SIZE) { + key_used = key; + num = SHA256_BLOCK_SIZE; + } else { + if (key_size > SHA256_BLOCK_SIZE){ + num = SHA256_DIGEST_SIZE; + sha256(key, key_size, key_temp); + key_used = key_temp; + } else { /* key_size > SHA256_BLOCK_SIZE */ + key_used = key; + num = key_size; + } + fill = SHA256_BLOCK_SIZE - num; + + memset(ctx->block_ipad + num, 0x36, fill); + memset(ctx->block_opad + num, 0x5c, fill); + } + + for (i = 0; i < (int) num; i++) { + ctx->block_ipad[i] = key_used[i] ^ 0x36; + ctx->block_opad[i] = key_used[i] ^ 0x5c; + } + + sha256_init(&ctx->ctx_inside); + sha256_update(&ctx->ctx_inside, ctx->block_ipad, SHA256_BLOCK_SIZE); + + sha256_init(&ctx->ctx_outside); + sha256_update(&ctx->ctx_outside, ctx->block_opad, + SHA256_BLOCK_SIZE); + + /* for hmac_reinit */ + memcpy(&ctx->ctx_inside_reinit, &ctx->ctx_inside, + sizeof(sha256_ctx)); + memcpy(&ctx->ctx_outside_reinit, &ctx->ctx_outside, + sizeof(sha256_ctx)); +} + +void hmac_sha256_reinit(hmac_sha256_ctx *ctx) +{ + memcpy(&ctx->ctx_inside, &ctx->ctx_inside_reinit, + sizeof(sha256_ctx)); + memcpy(&ctx->ctx_outside, &ctx->ctx_outside_reinit, + sizeof(sha256_ctx)); +} + +void hmac_sha256_update(hmac_sha256_ctx *ctx, const unsigned char *message, + unsigned int message_len) +{ + sha256_update(&ctx->ctx_inside, message, message_len); +} + +void hmac_sha256_final(hmac_sha256_ctx *ctx, unsigned char *mac, + unsigned int mac_size) +{ + unsigned char digest_inside[SHA256_DIGEST_SIZE] = { 0 }; + unsigned char mac_temp[SHA256_DIGEST_SIZE] = { 0 }; + + sha256_final(&ctx->ctx_inside, digest_inside); + sha256_update(&ctx->ctx_outside, digest_inside, SHA256_DIGEST_SIZE); + sha256_final(&ctx->ctx_outside, mac_temp); + memcpy(mac, mac_temp, mac_size); +} + +void hmac_sha256(const unsigned char *key, unsigned int key_size, + const unsigned char *message, unsigned int message_len, + unsigned char *mac, unsigned mac_size) +{ + hmac_sha256_ctx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + hmac_sha256_init(&ctx, key, key_size); + hmac_sha256_update(&ctx, message, message_len); + hmac_sha256_final(&ctx, mac, mac_size); +} diff --git a/optee_client/tee-supplicant/src/hmac_sha2.h b/optee_client/tee-supplicant/src/hmac_sha2.h new file mode 100644 index 0000000..1044524 --- /dev/null +++ b/optee_client/tee-supplicant/src/hmac_sha2.h @@ -0,0 +1,74 @@ +/* + * HMAC-SHA-224/256/384/512 implementation + * Last update: 06/15/2005 + * Issue date: 06/15/2005 + * + * Copyright (C) 2005 Olivier Gay + * All rights reserved. + * + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef HMAC_SHA2_H +#define HMAC_SHA2_H + +#include "sha2.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + sha256_ctx ctx_inside; + sha256_ctx ctx_outside; + + /* for hmac_reinit */ + sha256_ctx ctx_inside_reinit; + sha256_ctx ctx_outside_reinit; + + unsigned char block_ipad[SHA256_BLOCK_SIZE]; + unsigned char block_opad[SHA256_BLOCK_SIZE]; +} hmac_sha256_ctx; + +void hmac_sha256_init(hmac_sha256_ctx *ctx, const unsigned char *key, + unsigned int key_size); +void hmac_sha256_reinit(hmac_sha256_ctx *ctx); +void hmac_sha256_update(hmac_sha256_ctx *ctx, const unsigned char *message, + unsigned int message_len); +void hmac_sha256_final(hmac_sha256_ctx *ctx, unsigned char *mac, + unsigned int mac_size); +void hmac_sha256(const unsigned char *key, unsigned int key_size, + const unsigned char *message, unsigned int message_len, + unsigned char *mac, unsigned mac_size); + +#ifdef __cplusplus +} +#endif + +#endif /* !HMAC_SHA2_H */ + diff --git a/optee_client/tee-supplicant/src/optee_msg_supplicant.h b/optee_client/tee-supplicant/src/optee_msg_supplicant.h new file mode 100644 index 0000000..40fa74d --- /dev/null +++ b/optee_client/tee-supplicant/src/optee_msg_supplicant.h @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OPTEE_MSG_SUPPLICANT_H +#define __OPTEE_MSG_SUPPLICANT_H + +/* + * Load a TA into memory + */ +#define OPTEE_MSG_RPC_CMD_LOAD_TA 0 + +/* + * Replay Protected Memory Block access + */ +#define OPTEE_MSG_RPC_CMD_RPMB 1 + +/* + * File system access + */ +#define OPTEE_MSG_RPC_CMD_FS 2 + +/* + * Define protocol for messages with .cmd == OPTEE_MSG_RPC_CMD_FS and first + * parameter has the attribute OPTEE_MSG_ATTR_TYPE_VALUE_INPUT. + */ + +/* + * Open a file + * + * [in] param[0].u.value.a OPTEE_MRF_OPEN + * [in] param[1].u.tmem a string holding the file name + * [out] param[2].u.value.a file descriptor of open file + */ +#define OPTEE_MRF_OPEN 0 + +/* + * Create a file + * + * [in] param[0].u.value.a OPTEE_MRF_CREATE + * [in] param[1].u.tmem a string holding the file name + * [out] param[2].u.value.a file descriptor of open file + */ +#define OPTEE_MRF_CREATE 1 + +/* + * Close a file + * + * [in] param[0].u.value.a OPTEE_MRF_CLOSE + * [in] param[0].u.value.b file descriptor of open file. + */ +#define OPTEE_MRF_CLOSE 2 + +/* + * Read from a file + * + * [in] param[0].u.value.a OPTEE_MRF_READ + * [in] param[0].u.value.b file descriptor of open file + * [in] param[0].u.value.c offset into file + * [out] param[1].u.tmem buffer to hold returned data + */ +#define OPTEE_MRF_READ 3 + +/* + * Write to a file + * + * [in] param[0].u.value.a OPTEE_MRF_WRITE + * [in] param[0].u.value.b file descriptor of open file + * [in] param[0].u.value.c offset into file + * [in] param[1].u.tmem buffer holding data to be written + */ +#define OPTEE_MRF_WRITE 4 + +/* + * Truncate a file + * + * [in] param[0].u.value.a OPTEE_MRF_TRUNCATE + * [in] param[0].u.value.b file descriptor of open file + * [in] param[0].u.value.c length of file. + */ +#define OPTEE_MRF_TRUNCATE 5 + +/* + * Remove a file + * + * [in] param[0].u.value.a OPTEE_MRF_REMOVE + * [in] param[1].u.tmem a string holding the file name + */ +#define OPTEE_MRF_REMOVE 6 + +/* + * Rename a file + * + * [in] param[0].u.value.a OPTEE_MRF_RENAME + * [in] param[0].u.value.b true if existing target should be removed + * [in] param[1].u.tmem a string holding the old file name + * [in] param[2].u.tmem a string holding the new file name + */ +#define OPTEE_MRF_RENAME 7 + +/* + * Opens a directory for file listing + * + * [in] param[0].u.value.a OPTEE_MRF_OPENDIR + * [in] param[1].u.tmem a string holding the name of the directory + * [out] param[2].u.value.a handle to open directory + */ +#define OPTEE_MRF_OPENDIR 8 + +/* + * Closes a directory handle + * + * [in] param[0].u.value.a OPTEE_MRF_CLOSEDIR + * [in] param[0].u.value.b handle to open directory + */ +#define OPTEE_MRF_CLOSEDIR 9 + +/* + * Read next file name of directory + * + * + * [in] param[0].u.value.a OPTEE_MRF_READDIR + * [in] param[0].u.value.b handle to open directory + * [out] param[1].u.tmem a string holding the file name + */ +#define OPTEE_MRF_READDIR 10 + +/* + * End of definitions for messages with .cmd == OPTEE_MSG_RPC_CMD_FS + */ + +/* + * Command Ids 3, 4 and 5 of OPTEE_MSG_RPC_CMD_xxx macros are reserved for use + * by the kernel driver. + */ + +/* + * Shared memory allocation + */ +#define OPTEE_MSG_RPC_CMD_SHM_ALLOC 6 +#define OPTEE_MSG_RPC_CMD_SHM_FREE 7 + +/* + * Was OPTEE_MSG_RPC_CMD_SQL_FS, which isn't supported any longer + */ +#define OPTEE_MSG_RPC_CMD_SQL_FS_RESERVED 8 + +/* + * GPROF support management commands + */ +#define OPTEE_MSG_RPC_CMD_GPROF 9 + +/* + * Socket commands + */ +#define OPTEE_MSG_RPC_CMD_SOCKET 10 + +/* + * Function tracing support management commands + */ +#define OPTEE_MSG_RPC_CMD_FTRACE 11 + +/* + * Plugin commands + */ +#define OPTEE_MSG_RPC_CMD_PLUGIN 12 + +/* + * Define protocol for messages with .cmd == OPTEE_MSG_RPC_CMD_SOCKET + */ + +#define OPTEE_MRC_SOCKET_TIMEOUT_NONBLOCKING 0 +#define OPTEE_MRC_SOCKET_TIMEOUT_BLOCKING 0xffffffff + +/* + * Open socket + * + * [in] param[0].u.value.a OPTEE_MRC_SOCKET_OPEN + * [in] param[0].u.value.b TA instance id + * [in] param[1].u.value.a server port number + * [in] param[1].u.value.b protocol, TEE_ISOCKET_PROTOCOLID_* + * [in] param[1].u.value.c ip version TEE_IP_VERSION_* from tee_ipsocket.h + * [in] param[2].u.tmem server address + * [out] param[3].u.value.a socket handle (32-bit) + */ +#define OPTEE_MRC_SOCKET_OPEN 0 + +/* + * Close socket + * + * [in] param[0].u.value.a OPTEE_MRC_SOCKET_CLOSE + * [in] param[0].u.value.b TA instance id + * [in] param[0].u.value.c socket handle + */ +#define OPTEE_MRC_SOCKET_CLOSE 1 + +/* + * Close all sockets + * + * [in] param[0].u.value.a OPTEE_MRC_SOCKET_CLOSE_ALL + * [in] param[0].u.value.b TA instance id + */ +#define OPTEE_MRC_SOCKET_CLOSE_ALL 2 + +/* + * Send data on socket + * + * [in] param[0].u.value.a OPTEE_MRC_SOCKET_SEND + * [in] param[0].u.value.b TA instance id + * [in] param[0].u.value.c socket handle + * [in] param[1].u.tmem buffer to transmit + * [in] param[2].u.value.a timeout ms or OPTEE_MRC_SOCKET_TIMEOUT_* + * [out] param[2].u.value.b number of transmitted bytes + */ +#define OPTEE_MRC_SOCKET_SEND 3 + +/* + * Receive data on socket + * + * [in] param[0].u.value.a OPTEE_MRC_SOCKET_RECV + * [in] param[0].u.value.b TA instance id + * [in] param[0].u.value.c socket handle + * [out] param[1].u.tmem buffer to receive + * [in] param[2].u.value.a timeout ms or OPTEE_MRC_SOCKET_TIMEOUT_* + */ +#define OPTEE_MRC_SOCKET_RECV 4 + +/* + * Perform IOCTL on socket + * + * [in] param[0].u.value.a OPTEE_MRC_SOCKET_IOCTL + * [in] param[0].u.value.b TA instance id + * [in] param[0].u.value.c socket handle + * [in/out] param[1].u.tmem buffer + * [in] param[2].u.value.a ioctl command + */ +#define OPTEE_MRC_SOCKET_IOCTL 5 + +/* + * End of definitions for messages with .cmd == OPTEE_MSG_RPC_CMD_SOCKET + */ + +/* + * Define protocol for messages with .cmd == OPTEE_MSG_RPC_CMD_PLUGIN + */ + +/* + * Invoke a tee-supplicant plugin. + * + * [in] param[0].u.value.a OPTEE_INVOKE_PLUGIN + * [in] param[0].u.value.b uuid.d1 + * [in] param[0].u.value.c uuid.d2 + * [in] param[1].u.value.a uuid.d3 + * [in] param[1].u.value.b uuid.d4 + * [in] param[1].u.value.c cmd for plugin + * [in] param[2].u.value.a sub_cmd for plugin + * [out] param[2].u.value.b length of the outbuf (param[3]), + * if out is needed. + * [in/out] param[3].u.tmem buffer holding data for plugin + * + * UUID serialized into octets: + * b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 b11 b12 b13 b14 b15 + * d1 | d2 | d3 | d4 + * + * The endianness of words d1, d2, d3 and d4 from SWd is little-endian. + * d1 word contains [b3 b2 b1 b0] + * d2 word contains [b7 b6 b5 b4] + * d3 word contains [b11 b10 b9 b8] + * d4 word contains [b15 b14 b13 b12] + */ +#define OPTEE_INVOKE_PLUGIN 0 + +/* + * End of definitions for messages with .cmd == OPTEE_MSG_RPC_CMD_PLUGIN + */ + +#endif /*__OPTEE_MSG_SUPPLICANT_H*/ diff --git a/optee_client/tee-supplicant/src/plugin.c b/optee_client/tee-supplicant/src/plugin.c new file mode 100644 index 0000000..cca22a1 --- /dev/null +++ b/optee_client/tee-supplicant/src/plugin.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "optee_msg_supplicant.h" + +#ifndef __aligned +#define __aligned(x) __attribute__((__aligned__(x))) +#endif +#include + +/* internal possible returned values */ +enum plugin_err { + PLUGIN_OK = 0, + PLUGIN_DL_OPEN_ERR = -1, + PLUGIN_DL_SYM_ERR = -2, +}; + +static struct plugin *plugin_list_head; + +/* returns 0, if u1 and u2 are equal */ +static int uuid_cmp(TEEC_UUID *u1, TEEC_UUID *u2) +{ + if (!memcmp(u1, u2, sizeof(TEEC_UUID))) + return 0; + + return 1; +} + +static void uuid_from_octets(TEEC_UUID *d, const uint8_t s[TEE_IOCTL_UUID_LEN]) +{ + d->timeLow = ((uint32_t)s[0] << 24) | ((uint32_t)s[1] << 16) | + ((uint32_t)s[2] << 8) | s[3]; + d->timeMid = ((uint32_t)s[4] << 8) | s[5]; + d->timeHiAndVersion = ((uint32_t)s[6] << 8) | s[7]; + memcpy(d->clockSeqAndNode, s + 8, sizeof(d->clockSeqAndNode)); +} + +static void push_plugin(struct plugin *p) +{ + p->next = plugin_list_head; + plugin_list_head = p; +} + +static struct plugin *find_plugin(TEEC_UUID *u) +{ + struct plugin *p = plugin_list_head; + + while (p) { + if (!uuid_cmp(&p->method->uuid, u)) + return p; + + p = p->next; + } + + return NULL; +} + +static enum plugin_err load_plugin(const char *name, struct plugin *p) +{ + void *handle = NULL; + struct plugin_method *m = NULL; + + handle = dlopen(name, RTLD_LAZY); + if (!handle) + return PLUGIN_DL_OPEN_ERR; + + m = (struct plugin_method *)dlsym(handle, "plugin_method"); + if (!m || !m->name || !m->invoke) { + dlclose(handle); + return PLUGIN_DL_SYM_ERR; + } + + p->handle = handle; + p->method = m; + + return PLUGIN_OK; +} + +static TEEC_Result plugin_invoke(TEEC_UUID *u, unsigned int cmd, + unsigned int sub_cmd, void *data, + size_t in_len, size_t *out_len) +{ + struct plugin *p = NULL; + + p = find_plugin(u); + if (!p) + return TEEC_ERROR_ITEM_NOT_FOUND; + + assert(p->method->invoke); + + return p->method->invoke(cmd, sub_cmd, data, in_len, out_len); +} + +TEEC_Result plugin_load_all(void) +{ + DIR *dir = NULL; + enum plugin_err res = PLUGIN_OK; + TEEC_Result teec_res = TEEC_SUCCESS; + struct dirent *entry = NULL; + char path[PATH_MAX] = { 0 }; + + dir = opendir(supplicant_params.plugin_load_path); + + if (!dir) { + IMSG("could not open directory %s", supplicant_params.plugin_load_path); + + /* don't generate error if there is no the dir */ + return TEEC_SUCCESS; + } + + while ((entry = readdir(dir))) { + struct plugin *p; + int r; + + if (!strcmp(entry->d_name, "..") || !strcmp(entry->d_name, ".")) + continue; + + p = calloc(1, sizeof(struct plugin)); + if (!p) { + EMSG("allocate mem for plugin <%s> failed", + entry->d_name); + closedir(dir); + return TEEC_ERROR_OUT_OF_MEMORY; + } + r = snprintf(path, sizeof(path), "%s/%s", + supplicant_params.plugin_load_path, entry->d_name); + if (r < 0 || r >= (int)sizeof(path)) { + EMSG("assemble of full path for plugin <%s> failed", + entry->d_name); + closedir(dir); + return TEEC_ERROR_GENERIC ; + } + res = load_plugin(path, p); + switch (res) { + case PLUGIN_DL_OPEN_ERR: + EMSG("open plugin <%s> failed: %s", + entry->d_name, dlerror()); + free(p); + continue; + case PLUGIN_DL_SYM_ERR: + EMSG("find 'plugin_method' sym in <%s> failed: %s", + entry->d_name, dlerror()); + free(p); + continue; + default: + DMSG("loading the <%s> plugin were successful", + p->method->name); + break; + } + + /* Init the plugin */ + if (p->method->init) { + teec_res = p->method->init(); + if (teec_res) { + EMSG("init the <%s> plugin failed with 0x%x", + p->method->name, teec_res); + free(p); + continue; + } + } + + push_plugin(p); + } + + closedir(dir); + return TEEC_SUCCESS; +} + +TEEC_Result plugin_process(size_t num_params, struct tee_ioctl_param *params) +{ + unsigned int cmd = 0; + unsigned int sub_cmd = 0; + void *data = NULL; + uint32_t data_len = 0; + TEEC_UUID uuid = { }; + uint32_t uuid_words[4] = { }; + size_t outlen = 0; + TEEC_Result res = TEEC_ERROR_NOT_SUPPORTED; + + if (num_params != 4 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[2].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT || + (params[3].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT) + return TEEC_ERROR_BAD_PARAMETERS; + + uuid_words[0] = params[0].b; + uuid_words[1] = params[0].c; + uuid_words[2] = params[1].a; + uuid_words[3] = params[1].b; + + uuid_from_octets(&uuid, (const uint8_t *)uuid_words); + + cmd = params[1].c; + sub_cmd = params[2].a; + + data = tee_supp_param_to_va(params + 3); + data_len = MEMREF_SIZE(params + 3); + + if (data_len && !data) + return TEEC_ERROR_BAD_PARAMETERS; + + switch (params[0].a) { + case OPTEE_INVOKE_PLUGIN: + res = plugin_invoke(&uuid, cmd, sub_cmd, data, data_len, + &outlen); + params[2].b = outlen; + default: + break; + } + + return res; +} diff --git a/optee_client/tee-supplicant/src/plugin.h b/optee_client/tee-supplicant/src/plugin.h new file mode 100644 index 0000000..f0ee829 --- /dev/null +++ b/optee_client/tee-supplicant/src/plugin.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#ifndef PLUGIN_H +#define PLUGIN_H + +#include +#include +#include + +struct tee_ioctl_param; + +/* This structure describes one plugin for the supplicant */ +struct plugin { + void *handle; + struct plugin_method *method; /* Implemented in the plugin */ + struct plugin *next; +}; + +#ifdef TEE_SUPP_PLUGINS +/* + * Loads all shared objects from 'CFG_TEE_PLUGIN_LOAD_PATH' + * and binds all functions. + * + * @return 'TEEC_SUCCESS' if all plugins were successfully loaded. + */ +TEEC_Result plugin_load_all(void); + +/* Plugin RPC handler */ +TEEC_Result plugin_process(size_t num_params, struct tee_ioctl_param *params); +#else +static inline TEEC_Result plugin_load_all(void) +{ + return TEEC_SUCCESS; +} + +static inline TEEC_Result plugin_process(size_t num_params, + struct tee_ioctl_param *params) +{ + (void)num_params; + (void)params; + + return TEEC_ERROR_NOT_SUPPORTED; +} +#endif /*TEE_SUPP_PLUGINS*/ + +#endif /* PLUGIN_H */ + diff --git a/optee_client/tee-supplicant/src/prof.c b/optee_client/tee-supplicant/src/prof.c new file mode 100644 index 0000000..3a3056e --- /dev/null +++ b/optee_client/tee-supplicant/src/prof.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "prof.h" + +#ifndef __aligned +#define __aligned(x) __attribute__((__aligned__(x))) +#endif +#include + +TEEC_Result prof_process(size_t num_params, struct tee_ioctl_param *params, + const char *prefix) +{ + char vers[5] = ""; + char path[255] = { 0 }; + size_t bufsize = 0; + TEEC_UUID *u = NULL; + int fd = -1; + void *buf = NULL; + int flags = 0; + int id = 0; + int st = 0; + int n = 0; + + if (num_params != 3 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT || + (params[2].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + id = params[0].a; + + if (MEMREF_SIZE(params + 1) != sizeof(TEEC_UUID)) + return TEEC_ERROR_BAD_PARAMETERS; + + u = tee_supp_param_to_va(params + 1); + if (!u) + return TEEC_ERROR_BAD_PARAMETERS; + + buf = tee_supp_param_to_va(params + 2); + if (!buf) + return TEEC_ERROR_BAD_PARAMETERS; + + bufsize = MEMREF_SIZE(params + 2); + + if (id < 0 || id > 100) + return TEEC_ERROR_BAD_PARAMETERS; + + flags = O_APPEND | O_WRONLY; + if (!id) { + /* id == 0 means create file */ + flags |= O_CREAT | O_EXCL; + id = 1; + } + + for (;;) { + if (id > 1) { + /* + * id == 1 is file 0 (no suffix), id == 2 is file .1 + * etc. + */ + if (id > 100) + id = 100; /* Avoid GCC truncation warning */ + snprintf(vers, sizeof(vers), ".%d", id - 1); + } + n = snprintf(path, sizeof(path), + "/tmp/%s" + "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" + "%s.out", + prefix, + u->timeLow, u->timeMid, u->timeHiAndVersion, + u->clockSeqAndNode[0], u->clockSeqAndNode[1], + u->clockSeqAndNode[2], u->clockSeqAndNode[3], + u->clockSeqAndNode[4], u->clockSeqAndNode[5], + u->clockSeqAndNode[6], u->clockSeqAndNode[7], + vers); + if ((n < 0) || (n >= (int)sizeof(path))) + break; + fd = open(path, flags, 0644); + if (fd >= 0) { + do { + st = write(fd, buf, bufsize); + } while (st < 0 && errno == EINTR); + close(fd); + if (st < 0 || st != (int)bufsize) + break; + params[0].a = id; + goto success; + } + if (errno != EEXIST) + break; + if (id++ == 100) + break; + } + + /* An error occurred */ + return TEEC_ERROR_GENERIC; + +success: + return TEEC_SUCCESS; +} diff --git a/optee_client/tee-supplicant/src/prof.h b/optee_client/tee-supplicant/src/prof.h new file mode 100644 index 0000000..48c8993 --- /dev/null +++ b/optee_client/tee-supplicant/src/prof.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PROF_H +#define PROF_H + +#include + +struct tee_ioctl_param; + +#if defined(CFG_TA_GPROF_SUPPORT) || defined(CFG_FTRACE_SUPPORT) + +TEEC_Result prof_process(size_t num_params, struct tee_ioctl_param *params, + const char *prefix); + +#else + +static inline TEEC_Result prof_process(size_t num_params, + struct tee_ioctl_param *params, + const char *prefix) +{ + (void)num_params; + (void)params; + (void)prefix; + + return TEEC_ERROR_NOT_SUPPORTED; +} + +#endif /* CFG_TA_GPROF_SUPPORT || CFG_FTRACE_SUPPORT */ +#endif /* PROF_H */ diff --git a/optee_client/tee-supplicant/src/rpmb.c b/optee_client/tee-supplicant/src/rpmb.c new file mode 100644 index 0000000..7643ed6 --- /dev/null +++ b/optee_client/tee-supplicant/src/rpmb.c @@ -0,0 +1,989 @@ +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef RPMB_EMU +#include +#include "hmac_sha2.h" +#else +#include +#endif + +/* + * Request and response definitions must be in sync with the secure side + */ + +/* Request */ +struct rpmb_req { + uint16_t cmd; +#define RPMB_CMD_DATA_REQ 0x00 +#define RPMB_CMD_GET_DEV_INFO 0x01 + uint16_t dev_id; + uint16_t block_count; + /* Optional data frames (rpmb_data_frame) follow */ +}; +#define RPMB_REQ_DATA(req) ((void *)((struct rpmb_req *)(req) + 1)) + +#define RPMB_CID_SZ 16 + +/* Response to device info request */ +struct rpmb_dev_info { + uint8_t cid[RPMB_CID_SZ]; + uint8_t rpmb_size_mult; /* EXT CSD-slice 168: RPMB Size */ + uint8_t rel_wr_sec_c; /* EXT CSD-slice 222: Reliable Write Sector */ + /* Count */ + uint8_t ret_code; +#define RPMB_CMD_GET_DEV_INFO_RET_OK 0x00 +#define RPMB_CMD_GET_DEV_INFO_RET_ERROR 0x01 +}; + +/* + * This structure is shared with OP-TEE and the MMC ioctl layer. + * It is the "data frame for RPMB access" defined by JEDEC, minus the + * start and stop bits. + */ +struct rpmb_data_frame { + uint8_t stuff_bytes[196]; + uint8_t key_mac[32]; + uint8_t data[256]; + uint8_t nonce[16]; + uint32_t write_counter; + uint16_t address; + uint16_t block_count; + uint16_t op_result; +#define RPMB_RESULT_OK 0x00 +#define RPMB_RESULT_GENERAL_FAILURE 0x01 +#define RPMB_RESULT_AUTH_FAILURE 0x02 +#define RPMB_RESULT_ADDRESS_FAILURE 0x04 +#define RPMB_RESULT_AUTH_KEY_NOT_PROGRAMMED 0x07 + uint16_t msg_type; +#define RPMB_MSG_TYPE_REQ_AUTH_KEY_PROGRAM 0x0001 +#define RPMB_MSG_TYPE_REQ_WRITE_COUNTER_VAL_READ 0x0002 +#define RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE 0x0003 +#define RPMB_MSG_TYPE_REQ_AUTH_DATA_READ 0x0004 +#define RPMB_MSG_TYPE_REQ_RESULT_READ 0x0005 +#define RPMB_MSG_TYPE_RESP_AUTH_KEY_PROGRAM 0x0100 +#define RPMB_MSG_TYPE_RESP_WRITE_COUNTER_VAL_READ 0x0200 +#define RPMB_MSG_TYPE_RESP_AUTH_DATA_WRITE 0x0300 +#define RPMB_MSG_TYPE_RESP_AUTH_DATA_READ 0x0400 +}; + + +static pthread_mutex_t rpmb_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * ioctl() interface + * Comes from: uapi/linux/major.h, linux/mmc/core.h + */ + +#define MMC_BLOCK_MAJOR 179 + +/* mmc_ioc_cmd.opcode */ +#define MMC_READ_MULTIPLE_BLOCK 18 +#define MMC_WRITE_MULTIPLE_BLOCK 25 + +/* mmc_ioc_cmd.flags */ +#define MMC_RSP_PRESENT (1 << 0) +#define MMC_RSP_136 (1 << 1) /* 136 bit response */ +#define MMC_RSP_CRC (1 << 2) /* Expect valid CRC */ +#define MMC_RSP_OPCODE (1 << 4) /* Response contains opcode */ + +#define MMC_RSP_R1 (MMC_RSP_PRESENT|MMC_RSP_CRC|MMC_RSP_OPCODE) + +#define MMC_CMD_ADTC (1 << 5) /* Addressed data transfer command */ + +/* mmc_ioc_cmd.write_flag */ +#define MMC_CMD23_ARG_REL_WR (1 << 31) /* CMD23 reliable write */ + +/* Maximum number of commands used in a multiple ioc command request */ +#define RPMB_MAX_IOC_MULTI_CMDS 3 + +#ifndef RPMB_EMU + +#define IOCTL(fd, request, ...) \ + ({ \ + int ret; \ + ret = ioctl((fd), (request), ##__VA_ARGS__); \ + if (ret < 0) \ + EMSG("ioctl ret=%d errno=%d", ret, errno); \ + ret; \ + }) + + +/* Open and/or return file descriptor to RPMB partition of device dev_id */ +static int mmc_rpmb_fd(uint16_t dev_id) +{ + static int id; + static int fd = -1; + char path[PATH_MAX] = { 0 }; + + DMSG("dev_id = %u", dev_id); + if (fd < 0) { +#ifdef __ANDROID__ + snprintf(path, sizeof(path), "/dev/mmcblk%urpmb", dev_id); +#else + snprintf(path, sizeof(path), "/dev/mmcblk%urpmb", dev_id); +#endif + fd = open(path, O_RDWR); + if (fd < 0) { + EMSG("Could not open %s (%s)", path, strerror(errno)); + return -1; + } + id = dev_id; + } + if (id != dev_id) { + EMSG("Only one MMC device is supported"); + return -1; + } + return fd; +} + +/* + * Read @n bytes from @fd, takes care of short reads and EINTR. + * Adapted from “Advanced Programming In the UNIX Environment” by W. Richard + * Stevens and Stephen A. Rago, 2013, 3rd Edition, Addison-Wesley + * (EINTR handling was added) + */ +static ssize_t readn(int fd, void *ptr, size_t n) +{ + size_t nleft = n; + ssize_t nread = 0; + uint8_t *p = ptr; + + while (nleft > 0) { + if ((nread = read(fd, p, nleft)) < 0) { + if (errno == EINTR) + continue; + if (nleft == n) + return -1; /* error, nothing read, return -1 */ + else + break; /* error, return amount read so far */ + } else if (nread == 0) { + break; /* EOF */ + } + nleft -= nread; + p += nread; + } + return n - nleft; /* return >= 0 */ +} + +/* Size of CID printed in hexadecimal */ +#define CID_STR_SZ (2 * RPMB_CID_SZ) + +static TEEC_Result read_cid_str(uint16_t dev_id, char cid[CID_STR_SZ + 1]) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + char path[48] = { 0 }; + int fd = 0; + int st = 0; + + snprintf(path, sizeof(path), + "/sys/class/mmc_host/mmc%u/mmc%u:0001/cid", dev_id, dev_id); + fd = open(path, O_RDONLY); + if (fd < 0) + return TEEC_ERROR_ITEM_NOT_FOUND; + st = readn(fd, cid, CID_STR_SZ); + if (st != CID_STR_SZ) { + EMSG("Read CID error"); + if (errno) + EMSG("%s", strerror(errno)); + res = TEEC_ERROR_NO_DATA; + goto out; + } + res = TEEC_SUCCESS; +out: + close(fd); + return res; +} + +static int hexchar2int(char c) +{ + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + return -1; +} + +static int hexbyte2int(char *hex) +{ + int v1 = hexchar2int(hex[0]); + int v2 = hexchar2int(hex[1]); + + if (v1 < 0 || v2 < 0) + return -1; + return 16 * v1 + v2; +} + +/* Device Identification (CID) register is 16 bytes. It is read from sysfs. */ +static TEEC_Result read_cid(uint16_t dev_id, uint8_t *cid) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + char cid_str[CID_STR_SZ + 1] = { }; + int i = 0; + int v = 0; + + res = read_cid_str(dev_id, cid_str); + if (res) + return res; + + for (i = 0; i < RPMB_CID_SZ; i++) { + v = hexbyte2int(cid_str + 2 * i); + if (v < 0) { + EMSG("Invalid CID string: %s", cid_str); + return TEEC_ERROR_NO_DATA; + } + cid[i] = v; + } + return TEEC_SUCCESS; +} + +static TEEC_Result read_mmc_sysfs_hex(uint16_t dev_id, const char *file, uint8_t *value) +{ + TEEC_Result status = TEEC_SUCCESS; + char path[255] = { 0 }; + char buf[255] = { 0 }; + char *endp = buf; + int fd = 0; + int ret = 0; + + snprintf(path, sizeof(path), "/sys/class/mmc_host/mmc%u/mmc%u:0001/%s", + dev_id, dev_id, file); + + fd = open(path, O_RDONLY); + if (fd < 0) { + EMSG("Could not open %s (%s)", path, strerror(errno)); + return TEEC_ERROR_ITEM_NOT_FOUND; + } + + ret = readn(fd, buf, sizeof(buf)); + if (ret < 0) { + EMSG("Read error (%s)", strerror(errno)); + status = TEEC_ERROR_NO_DATA; + goto out; + } + + errno = 0; + *value = strtol(buf, &endp, 16); + if (errno || endp == buf) + status = TEEC_ERROR_GENERIC; + +out: + close(fd); + return status; +} + +static TEEC_Result read_size_mult(uint16_t dev_id, uint8_t *value) +{ + return read_mmc_sysfs_hex(dev_id, "raw_rpmb_size_mult", value); +} + +static TEEC_Result read_rel_wr_sec_c(uint16_t dev_id, uint8_t *value) +{ + return read_mmc_sysfs_hex(dev_id, "rel_sectors", value); +} + +/* + * - If --rpmb-cid is given, find the eMMC RPMB device number with the specified + * CID, cache the number, copy it to @ndev_id and return true. If not found + * return false. + * - If --rpmb-cid is not given, copy @dev_id to @ndev_id and return true. + */ +static bool remap_rpmb_dev_id(uint16_t dev_id, uint16_t *ndev_id) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + static bool found = false; + static bool err = false; + static uint16_t id = 0; + char cid[CID_STR_SZ + 1] = { }; + struct dirent *dent = NULL; + DIR *dir = NULL; + int num = 0; + + if (err || found) + goto out; + + if (!supplicant_params.rpmb_cid) { + id = dev_id; + found = true; + goto out; + } + + dir = opendir("/sys/class/mmc_host"); + if (!dir) { + EMSG("Could not open /sys/class/mmc_host (%s)", + strerror(errno)); + err = true; + goto out; + } + + while ((dent = readdir(dir))) { + if (sscanf(dent->d_name, "%*[^0123456789]%d", &num) != 1) + continue; + if (num > UINT16_MAX) { + EMSG("Too many MMC devices"); + err = true; + break; + } + id = (uint16_t)num; + res = read_cid_str(id, cid); + if (res) + continue; + if (strcmp(cid, supplicant_params.rpmb_cid)) + continue; + IMSG("RPMB device %s is at /dev/mmcblk%urpmb\n", cid, id); + found = true; + break; + } + + closedir(dir); + + if (!found) + err = true; +out: + if (found) + *ndev_id = id; + return found; +} + +#else /* RPMB_EMU */ + +#define IOCTL(fd, request, ...) ioctl_emu((fd), (request), ##__VA_ARGS__) + +/* Emulated rel_wr_sec_c value (reliable write size, *256 bytes) */ +#define EMU_RPMB_REL_WR_SEC_C 1 +/* Emulated rpmb_size_mult value (RPMB size, *128 kB) */ +#define EMU_RPMB_SIZE_MULT 2 + +#define EMU_RPMB_SIZE_BYTES (EMU_RPMB_SIZE_MULT * 128 * 1024) + +/* Emulated eMMC device state */ +struct rpmb_emu { + uint8_t buf[EMU_RPMB_SIZE_BYTES]; + size_t size; + uint8_t key[32]; + bool key_set; + uint8_t nonce[16]; + uint32_t write_counter; + struct { + uint16_t msg_type; + uint16_t op_result; + uint16_t address; + } last_op; +}; +static struct rpmb_emu rpmb_emu = { + .size = EMU_RPMB_SIZE_BYTES +}; + +static struct rpmb_emu *mem_for_fd(int fd) +{ + static int sfd = -1; + + if (sfd == -1) + sfd = fd; + if (sfd != fd) { + EMSG("Emulating more than 1 RPMB partition is not supported"); + return NULL; + } + + return &rpmb_emu; +} + +#if (DEBUGLEVEL >= TRACE_FLOW) +static void dump_blocks(size_t startblk, size_t numblk, uint8_t *ptr, + bool to_mmc) +{ + char msg[100] = { 0 }; + size_t i = 0; + + for (i = 0; i < numblk; i++) { + snprintf(msg, sizeof(msg), "%s MMC block %zu", + to_mmc ? "Write" : "Read", startblk + i); + dump_buffer(msg, ptr, 256); + ptr += 256; + } +} +#else +static void dump_blocks(size_t startblk, size_t numblk, uint8_t *ptr, + bool to_mmc) +{ + (void)startblk; + (void)numblk; + (void)ptr; + (void)to_mmc; +} +#endif + +#define CUC(x) ((const unsigned char *)(x)) +static void hmac_update_frm(hmac_sha256_ctx *ctx, struct rpmb_data_frame *frm) +{ + hmac_sha256_update(ctx, CUC(frm->data), 256); + hmac_sha256_update(ctx, CUC(frm->nonce), 16); + hmac_sha256_update(ctx, CUC(&frm->write_counter), 4); + hmac_sha256_update(ctx, CUC(&frm->address), 2); + hmac_sha256_update(ctx, CUC(&frm->block_count), 2); + hmac_sha256_update(ctx, CUC(&frm->op_result), 2); + hmac_sha256_update(ctx, CUC(&frm->msg_type), 2); +} + +static bool is_hmac_valid(struct rpmb_emu *mem, struct rpmb_data_frame *frm, + size_t nfrm) +{ + uint8_t mac[32] = { 0 }; + size_t i = 0; + hmac_sha256_ctx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + if (!mem->key_set) { + EMSG("Cannot check MAC (key not set)"); + return false; + } + + hmac_sha256_init(&ctx, mem->key, sizeof(mem->key)); + for (i = 0; i < nfrm; i++, frm++) + hmac_update_frm(&ctx, frm); + frm--; + hmac_sha256_final(&ctx, mac, 32); + + if (memcmp(mac, frm->key_mac, 32)) { + EMSG("Invalid MAC"); + return false; + } + return true; +} + +static uint16_t gen_msb1st_result(uint8_t byte) +{ + return (uint16_t)byte << 8; +} + +static uint16_t compute_hmac(struct rpmb_emu *mem, struct rpmb_data_frame *frm, + size_t nfrm) +{ + size_t i = 0; + hmac_sha256_ctx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + if (!mem->key_set) { + EMSG("Cannot compute MAC (key not set)"); + return gen_msb1st_result(RPMB_RESULT_AUTH_KEY_NOT_PROGRAMMED); + } + + hmac_sha256_init(&ctx, mem->key, sizeof(mem->key)); + for (i = 0; i < nfrm; i++, frm++) + hmac_update_frm(&ctx, frm); + frm--; + hmac_sha256_final(&ctx, frm->key_mac, 32); + + return gen_msb1st_result(RPMB_RESULT_OK); +} + +static uint16_t ioctl_emu_mem_transfer(struct rpmb_emu *mem, + struct rpmb_data_frame *frm, + size_t nfrm, int to_mmc) +{ + size_t start = mem->last_op.address * 256; + size_t size = nfrm * 256; + size_t i = 0; + uint8_t *memptr = NULL; + + if (start > mem->size || start + size > mem->size) { + EMSG("Transfer bounds exceeed emulated memory"); + return gen_msb1st_result(RPMB_RESULT_ADDRESS_FAILURE); + } + if (to_mmc && !is_hmac_valid(mem, frm, nfrm)) + return gen_msb1st_result(RPMB_RESULT_AUTH_FAILURE); + + DMSG("Transferring %zu 256-byte data block%s %s MMC (block offset=%zu)", + nfrm, (nfrm > 1) ? "s" : "", to_mmc ? "to" : "from", start / 256); + for (i = 0; i < nfrm; i++) { + memptr = mem->buf + start + i * 256; + if (to_mmc) { + memcpy(memptr, frm[i].data, 256); + mem->write_counter++; + frm[i].write_counter = htonl(mem->write_counter); + frm[i].msg_type = + htons(RPMB_MSG_TYPE_RESP_AUTH_DATA_WRITE); + } else { + memcpy(frm[i].data, memptr, 256); + frm[i].msg_type = + htons(RPMB_MSG_TYPE_RESP_AUTH_DATA_READ); + frm[i].address = htons(mem->last_op.address); + frm[i].block_count = nfrm; + memcpy(frm[i].nonce, mem->nonce, 16); + } + frm[i].op_result = gen_msb1st_result(RPMB_RESULT_OK); + } + dump_blocks(mem->last_op.address, nfrm, mem->buf + start, to_mmc); + + if (!to_mmc) + compute_hmac(mem, frm, nfrm); + + return gen_msb1st_result(RPMB_RESULT_OK); +} + +static void ioctl_emu_get_write_result(struct rpmb_emu *mem, + struct rpmb_data_frame *frm) +{ + frm->msg_type = htons(RPMB_MSG_TYPE_RESP_AUTH_DATA_WRITE); + frm->op_result = mem->last_op.op_result; + frm->address = htons(mem->last_op.address); + frm->write_counter = htonl(mem->write_counter); + compute_hmac(mem, frm, 1); +} + +static uint16_t ioctl_emu_setkey(struct rpmb_emu *mem, + struct rpmb_data_frame *frm) +{ + if (mem->key_set) { + EMSG("Key already set"); + return gen_msb1st_result(RPMB_RESULT_GENERAL_FAILURE); + } + dump_buffer("Setting key", frm->key_mac, 32); + memcpy(mem->key, frm->key_mac, 32); + mem->key_set = true; + + return gen_msb1st_result(RPMB_RESULT_OK); +} + +static void ioctl_emu_get_keyprog_result(struct rpmb_emu *mem, + struct rpmb_data_frame *frm) +{ + frm->msg_type = + htons(RPMB_MSG_TYPE_RESP_AUTH_KEY_PROGRAM); + frm->op_result = mem->last_op.op_result; +} + +static void ioctl_emu_read_ctr(struct rpmb_emu *mem, + struct rpmb_data_frame *frm) +{ + DMSG("Reading counter"); + frm->msg_type = htons(RPMB_MSG_TYPE_RESP_WRITE_COUNTER_VAL_READ); + frm->write_counter = htonl(mem->write_counter); + memcpy(frm->nonce, mem->nonce, 16); + frm->op_result = compute_hmac(mem, frm, 1); +} + +static uint32_t read_cid(uint16_t dev_id, uint8_t *cid) +{ + /* Taken from an actual eMMC chip */ + static const uint8_t test_cid[] = { + /* MID (Manufacturer ID): Micron */ + 0xfe, + /* CBX (Device/BGA): BGA */ + 0x01, + /* OID (OEM/Application ID) */ + 0x4e, + /* PNM (Product name) "MMC04G" */ + 0x4d, 0x4d, 0x43, 0x30, 0x34, 0x47, + /* PRV (Product revision): 4.2 */ + 0x42, + /* PSN (Product serial number) */ + 0xc8, 0xf6, 0x55, 0x2a, + /* + * MDT (Manufacturing date): + * June, 2014 + */ + 0x61, + /* (CRC7 (0xA) << 1) | 0x1 */ + 0x15 + }; + + (void)dev_id; + memcpy(cid, test_cid, sizeof(test_cid)); + + return TEEC_SUCCESS; +} + +/* A crude emulation of the MMC ioc commands we need for RPMB */ +static int ioctl_emu_cmd(int fd, struct mmc_ioc_cmd *cmd) +{ + struct rpmb_data_frame *frm = NULL; + uint16_t msg_type = 0; + struct rpmb_emu *mem = mem_for_fd(fd); + + if (!mem) + return -1; + + switch (cmd->opcode) { + case MMC_WRITE_MULTIPLE_BLOCK: + frm = (struct rpmb_data_frame *)(uintptr_t)cmd->data_ptr; + msg_type = ntohs(frm->msg_type); + + switch (msg_type) { + case RPMB_MSG_TYPE_REQ_AUTH_KEY_PROGRAM: + mem->last_op.msg_type = msg_type; + mem->last_op.op_result = ioctl_emu_setkey(mem, frm); + break; + + case RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE: + mem->last_op.msg_type = msg_type; + mem->last_op.address = ntohs(frm->address); + mem->last_op.op_result = + ioctl_emu_mem_transfer(mem, frm, + cmd->blocks, 1); + break; + + case RPMB_MSG_TYPE_REQ_WRITE_COUNTER_VAL_READ: + case RPMB_MSG_TYPE_REQ_AUTH_DATA_READ: + memcpy(mem->nonce, frm->nonce, 16); + mem->last_op.msg_type = msg_type; + mem->last_op.address = ntohs(frm->address); + break; + default: + break; + } + break; + + case MMC_READ_MULTIPLE_BLOCK: + frm = (struct rpmb_data_frame *)(uintptr_t)cmd->data_ptr; + msg_type = ntohs(frm->msg_type); + + switch (mem->last_op.msg_type) { + case RPMB_MSG_TYPE_REQ_AUTH_KEY_PROGRAM: + ioctl_emu_get_keyprog_result(mem, frm); + break; + + case RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE: + ioctl_emu_get_write_result(mem, frm); + break; + + case RPMB_MSG_TYPE_REQ_WRITE_COUNTER_VAL_READ: + ioctl_emu_read_ctr(mem, frm); + break; + + case RPMB_MSG_TYPE_REQ_AUTH_DATA_READ: + ioctl_emu_mem_transfer(mem, frm, cmd->blocks, 0); + break; + + default: + EMSG("Unexpected"); + break; + } + break; + + default: + EMSG("Unsupported ioctl opcode 0x%08x", cmd->opcode); + return -1; + } + + return 0; +} + +static int ioctl_emu(int fd, unsigned long request, ...) +{ + struct mmc_ioc_multi_cmd *mcmd = NULL; + struct mmc_ioc_cmd *cmd = NULL; + size_t i = 0; + int res = 0; + va_list ap; + + if (request == MMC_IOC_CMD) { + va_start(ap, request); + cmd = va_arg(ap, struct mmc_ioc_cmd *); + va_end(ap); + + res = ioctl_emu_cmd(fd, cmd); + } else if (request == MMC_IOC_MULTI_CMD) { + va_start(ap, request); + mcmd = va_arg(ap, struct mmc_ioc_multi_cmd *); + va_end(ap); + + for (i = 0; i < mcmd->num_of_cmds; i++) { + res = ioctl_emu_cmd(fd, &mcmd->cmds[i]); + if (res) + return res; + } + } else { + EMSG("Unsupported ioctl: 0x%lx", request); + return -1; + } + + return res; +} + +static int mmc_rpmb_fd(uint16_t dev_id) +{ + (void)dev_id; + + /* Any value != -1 will do in test mode */ + return 0; +} + +static TEEC_Result read_size_mult(uint16_t dev_id, uint8_t *value) +{ + (void)dev_id; + + *value = EMU_RPMB_SIZE_MULT; + return TEEC_SUCCESS; +} + +static TEEC_Result read_rel_wr_sec_c(uint16_t dev_id, uint8_t *value) +{ + (void)dev_id; + + *value = EMU_RPMB_REL_WR_SEC_C; + return TEEC_SUCCESS; +} + +static bool remap_rpmb_dev_id(uint16_t dev_id, uint16_t *ndev_id) +{ + *ndev_id = dev_id; + return true; +} + +#endif /* RPMB_EMU */ + +static inline void set_mmc_io_cmd(struct mmc_ioc_cmd *cmd, unsigned int blocks, + __u32 opcode, int write_flag) +{ + cmd->blksz = 512; + cmd->blocks = blocks; + cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC; + cmd->opcode = opcode; + cmd->write_flag = write_flag; +} + +static uint32_t rpmb_data_req(int fd, struct rpmb_data_frame *req_frm, + size_t req_nfrm, struct rpmb_data_frame *rsp_frm, + size_t rsp_nfrm) +{ + TEEC_Result res = TEEC_SUCCESS; + int st = 0; + size_t i = 0; + uint16_t msg_type = ntohs(req_frm->msg_type); + struct mmc_ioc_multi_cmd *mcmd = NULL; + struct mmc_ioc_cmd *cmd = NULL; + + for (i = 1; i < req_nfrm; i++) { + if (req_frm[i].msg_type != msg_type) { + EMSG("All request frames shall be of the same type"); + return TEEC_ERROR_BAD_PARAMETERS; + } + } + + DMSG("Req: %zu frame(s) of type 0x%04x", req_nfrm, msg_type); + DMSG("Rsp: %zu frame(s)", rsp_nfrm); + + mcmd = (struct mmc_ioc_multi_cmd *) + calloc(1, sizeof(struct mmc_ioc_multi_cmd) + + RPMB_MAX_IOC_MULTI_CMDS * sizeof(struct mmc_ioc_cmd)); + if (!mcmd) + return TEEC_ERROR_OUT_OF_MEMORY; + + switch(msg_type) { + case RPMB_MSG_TYPE_REQ_AUTH_KEY_PROGRAM: + case RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE: + if (rsp_nfrm != 1) { + EMSG("Expected only one response frame"); + res = TEEC_ERROR_BAD_PARAMETERS; + goto out; + } + + mcmd->num_of_cmds = 3; + + /* Send write request frame(s) */ + cmd = &mcmd->cmds[0]; + set_mmc_io_cmd(cmd, req_nfrm, MMC_WRITE_MULTIPLE_BLOCK, + 1 | MMC_CMD23_ARG_REL_WR); + /* + * Black magic: tested on a HiKey board with a HardKernel eMMC + * module. When postsleep values are zero, the kernel logs + * random errors: "mmc_blk_ioctl_cmd: Card Status=0x00000E00" + * and ioctl() fails. + */ + cmd->postsleep_min_us = 20000; + cmd->postsleep_max_us = 50000; + mmc_ioc_cmd_set_data((*cmd), (uintptr_t)req_frm); + + /* Send result request frame */ + cmd = &mcmd->cmds[1]; + set_mmc_io_cmd(cmd, req_nfrm, MMC_WRITE_MULTIPLE_BLOCK, 1); + memset(rsp_frm, 0, 1); + rsp_frm->msg_type = htons(RPMB_MSG_TYPE_REQ_RESULT_READ); + mmc_ioc_cmd_set_data((*cmd), (uintptr_t)rsp_frm); + + /* Read response frame */ + cmd = &mcmd->cmds[2]; + set_mmc_io_cmd(cmd, rsp_nfrm, MMC_READ_MULTIPLE_BLOCK, 0); + mmc_ioc_cmd_set_data((*cmd), (uintptr_t)rsp_frm); + break; + + case RPMB_MSG_TYPE_REQ_WRITE_COUNTER_VAL_READ: + if (rsp_nfrm != 1) { + EMSG("Expected only one response frame"); + res = TEEC_ERROR_BAD_PARAMETERS; + goto out; + } +#if __GNUC__ > 6 + __attribute__((fallthrough)); +#endif + + case RPMB_MSG_TYPE_REQ_AUTH_DATA_READ: + if (req_nfrm != 1) { + EMSG("Expected only one request frame"); + res = TEEC_ERROR_BAD_PARAMETERS; + goto out; + } + + mcmd->num_of_cmds = 2; + + /* Send request frame */ + cmd = &mcmd->cmds[0]; + set_mmc_io_cmd(cmd, req_nfrm, MMC_WRITE_MULTIPLE_BLOCK, 1); + mmc_ioc_cmd_set_data((*cmd), (uintptr_t)req_frm); + + /* Read response frames */ + cmd = &mcmd->cmds[1]; + set_mmc_io_cmd(cmd, rsp_nfrm, MMC_READ_MULTIPLE_BLOCK, 0); + mmc_ioc_cmd_set_data((*cmd), (uintptr_t)rsp_frm); + break; + + default: + EMSG("Unsupported message type: %d", msg_type); + res = TEEC_ERROR_GENERIC; + goto out; + } + + st = IOCTL(fd, MMC_IOC_MULTI_CMD, mcmd); + if (st < 0) + res = TEEC_ERROR_GENERIC; + +out: + free(mcmd); + + return res; +} + +static uint32_t rpmb_get_dev_info(uint16_t dev_id, struct rpmb_dev_info *info) +{ + TEEC_Result res = TEEC_SUCCESS; + uint8_t rpmb_size_mult = 0; + uint8_t rel_wr_sec_c = 0; + + res = read_cid(dev_id, info->cid); + if (res != TEEC_SUCCESS) + return res; + + res = read_size_mult(dev_id, &rpmb_size_mult); + if (res != TEEC_SUCCESS) + return res; + info->rpmb_size_mult = rpmb_size_mult; + + res = read_rel_wr_sec_c(dev_id, &rel_wr_sec_c); + if (res != TEEC_SUCCESS) + return res; + info->rel_wr_sec_c = rel_wr_sec_c; + + info->ret_code = RPMB_CMD_GET_DEV_INFO_RET_OK; + + return res; +} + + +/* + * req is one struct rpmb_req followed by one or more struct rpmb_data_frame + * rsp is either one struct rpmb_dev_info or one or more struct rpmb_data_frame + */ +static uint32_t rpmb_process_request_unlocked(void *req, size_t req_size, + void *rsp, size_t rsp_size) +{ + struct rpmb_req *sreq = req; + size_t req_nfrm = 0; + size_t rsp_nfrm = 0; + uint16_t dev_id = 0; + uint32_t res = 0; + int fd = 0; + + if (req_size < sizeof(*sreq)) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!remap_rpmb_dev_id(sreq->dev_id, &dev_id)) + return TEEC_ERROR_ITEM_NOT_FOUND; + + switch (sreq->cmd) { + case RPMB_CMD_DATA_REQ: + req_nfrm = (req_size - sizeof(struct rpmb_req)) / 512; + rsp_nfrm = rsp_size / 512; + fd = mmc_rpmb_fd(dev_id); + if (fd < 0) + return TEEC_ERROR_BAD_PARAMETERS; + res = rpmb_data_req(fd, RPMB_REQ_DATA(req), req_nfrm, rsp, + rsp_nfrm); + break; + + case RPMB_CMD_GET_DEV_INFO: + if (req_size != sizeof(struct rpmb_req) || + rsp_size != sizeof(struct rpmb_dev_info)) { + EMSG("Invalid req/rsp size"); + return TEEC_ERROR_BAD_PARAMETERS; + } + res = rpmb_get_dev_info(dev_id, (struct rpmb_dev_info *)rsp); + break; + + default: + EMSG("Unsupported RPMB command: %d", sreq->cmd); + res = TEEC_ERROR_BAD_PARAMETERS; + break; + } + + return res; +} + + +uint32_t rpmb_process_request(void *req, size_t req_size, void *rsp, + size_t rsp_size) +{ + uint32_t res = 0; + + tee_supp_mutex_lock(&rpmb_mutex); + res = rpmb_process_request_unlocked(req, req_size, rsp, rsp_size); + tee_supp_mutex_unlock(&rpmb_mutex); + + return res; +} diff --git a/optee_client/tee-supplicant/src/rpmb.h b/optee_client/tee-supplicant/src/rpmb.h new file mode 100644 index 0000000..feeed5b --- /dev/null +++ b/optee_client/tee-supplicant/src/rpmb.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RPMB_H +#define RPMB_H + +#include +#include + +uint32_t rpmb_process_request(void *req, size_t req_size, void *rsp, + size_t rsp_size); + +#endif /* RPMB_H */ diff --git a/optee_client/tee-supplicant/src/sha2.c b/optee_client/tee-supplicant/src/sha2.c new file mode 100644 index 0000000..b884033 --- /dev/null +++ b/optee_client/tee-supplicant/src/sha2.c @@ -0,0 +1,249 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "sha2.h" + +#define SHFR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) +#define ROTL(x, n) ((x << n) | (x >> ((sizeof(x) << 3) - n))) +#define CH(x, y, z) ((x & y) ^ (~x & z)) +#define MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) + +#define SHA256_F1(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define SHA256_F2(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define SHA256_F3(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHFR(x, 3)) +#define SHA256_F4(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHFR(x, 10)) + +#define UNPACK32(x, str) \ +{ \ + *((str) + 3) = (uint8) ((x) ); \ + *((str) + 2) = (uint8) ((x) >> 8); \ + *((str) + 1) = (uint8) ((x) >> 16); \ + *((str) + 0) = (uint8) ((x) >> 24); \ +} + +#define PACK32(str, x) \ +{ \ + *(x) = ((uint32) *((str) + 3) ) \ + | ((uint32) *((str) + 2) << 8) \ + | ((uint32) *((str) + 1) << 16) \ + | ((uint32) *((str) + 0) << 24); \ +} + +#define UNPACK64(x, str) \ +{ \ + *((str) + 7) = (uint8) ((x) ); \ + *((str) + 6) = (uint8) ((x) >> 8); \ + *((str) + 5) = (uint8) ((x) >> 16); \ + *((str) + 4) = (uint8) ((x) >> 24); \ + *((str) + 3) = (uint8) ((x) >> 32); \ + *((str) + 2) = (uint8) ((x) >> 40); \ + *((str) + 1) = (uint8) ((x) >> 48); \ + *((str) + 0) = (uint8) ((x) >> 56); \ +} + +#define PACK64(str, x) \ +{ \ + *(x) = ((uint64) *((str) + 7) ) \ + | ((uint64) *((str) + 6) << 8) \ + | ((uint64) *((str) + 5) << 16) \ + | ((uint64) *((str) + 4) << 24) \ + | ((uint64) *((str) + 3) << 32) \ + | ((uint64) *((str) + 2) << 40) \ + | ((uint64) *((str) + 1) << 48) \ + | ((uint64) *((str) + 0) << 56); \ +} + +#define SHA256_SCR(i) \ +{ \ + w[i] = SHA256_F4(w[i - 2]) + w[i - 7] \ + + SHA256_F3(w[i - 15]) + w[i - 16]; \ +} + +uint32 sha256_h0[8] = + {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + +uint32 sha256_k[64] = + {0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +/* SHA-256 functions */ + +static void sha256_transf(sha256_ctx *ctx, const unsigned char *message, + unsigned int block_nb) +{ + uint32 w[64] = { 0 }; + uint32 wv[8] = { 0 }; + uint32 t1 = 0; + uint32 t2 = 0; + const unsigned char *sub_block = NULL; + int i = 0; + int j = 0; + + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 6); + + for (j = 0; j < 16; j++) { + PACK32(&sub_block[j << 2], &w[j]); + } + + for (j = 16; j < 64; j++) { + SHA256_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 64; j++) { + t1 = wv[7] + SHA256_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + + sha256_k[j] + w[j]; + t2 = SHA256_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } + } +} + +void sha256(const unsigned char *message, unsigned int len, + unsigned char *digest) +{ + sha256_ctx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + sha256_init(&ctx); + sha256_update(&ctx, message, len); + sha256_final(&ctx, digest); +} + +void sha256_init(sha256_ctx *ctx) +{ + int i = 0; + + for (i = 0; i < 8; i++) { + ctx->h[i] = sha256_h0[i]; + } + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb = 0; + unsigned int new_len = 0; + unsigned int rem_len = 0; + unsigned int tmp_len = 0; + const unsigned char *shifted_message = NULL; + + tmp_len = SHA256_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA256_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA256_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA256_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +void sha256_final(sha256_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb = 0; + unsigned int pm_len = 0; + unsigned int len_b = 0; + int i = 0; + + block_nb = (1 + ((SHA256_BLOCK_SIZE - 9) + < (ctx->len % SHA256_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + + for (i = 0 ; i < 8; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +} diff --git a/optee_client/tee-supplicant/src/sha2.h b/optee_client/tee-supplicant/src/sha2.h new file mode 100644 index 0000000..3a17fae --- /dev/null +++ b/optee_client/tee-supplicant/src/sha2.h @@ -0,0 +1,75 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef SHA2_H +#define SHA2_H + +#define SHA256_DIGEST_SIZE ( 256 / 8) +#define SHA256_BLOCK_SIZE ( 512 / 8) + +#ifndef SHA2_TYPES +#define SHA2_TYPES +typedef unsigned char uint8; +typedef unsigned int uint32; +typedef unsigned long long uint64; +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA256_BLOCK_SIZE]; + uint32 h[8]; +} sha256_ctx; + +typedef sha256_ctx sha224_ctx; + +void sha256_init(sha256_ctx * ctx); +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha256_final(sha256_ctx *ctx, unsigned char *digest); +void sha256(const unsigned char *message, unsigned int len, + unsigned char *digest); + +#ifdef __cplusplus +} +#endif + +#endif /* !SHA2_H */ + diff --git a/optee_client/tee-supplicant/src/tee_socket.c b/optee_client/tee-supplicant/src/tee_socket.c new file mode 100644 index 0000000..b9ae613 --- /dev/null +++ b/optee_client/tee-supplicant/src/tee_socket.c @@ -0,0 +1,848 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "handle.h" +#include "__tee_isocket_defines.h" +#include "__tee_ipsocket.h" +#include "__tee_tcpsocket_defines.h" +#include "__tee_tcpsocket_defines_extensions.h" +#include "__tee_udpsocket_defines.h" + +#ifndef __aligned +#define __aligned(x) __attribute__((__aligned__(x))) +#endif +#include + +/* + * Used when checking how much data we have queued. + * + * For SOCK_DGRAM we try to be accurate up to 4096 bytes as + * that's our arbitrary chosen sensible upper size (with + * some margin). Larger size doesn't make much sense since + * anything larger than the MTU is bound to cause trouble + * on a congested network. + * + * For SOCK_STREAM we chose the same upper limit for + * simplicity. It doesn't matter if there's more queued, + * no data will be lost. + */ +#define SUPP_MAX_PEEK_LEN 4096 + +struct sock_instance { + uint32_t id; + struct handle_db db; + TAILQ_ENTRY(sock_instance) link; +}; + +static pthread_mutex_t sock_mutex = PTHREAD_MUTEX_INITIALIZER; +TAILQ_HEAD(, sock_instance) sock_instances = + TAILQ_HEAD_INITIALIZER(sock_instances); + +static void sock_lock(void) +{ + pthread_mutex_lock(&sock_mutex); +} + +static void sock_unlock(void) +{ + pthread_mutex_unlock(&sock_mutex); +} + +static struct sock_instance *sock_instance_find(uint32_t instance_id) +{ + struct sock_instance *si = NULL; + + TAILQ_FOREACH(si, &sock_instances, link) { + if (si->id == instance_id) + return si; + } + return NULL; +} + +static void *fd_to_handle_ptr(int fd) +{ + uintptr_t ptr = 0; + + assert(fd >= 0); + ptr = fd + 1; + return (void *)ptr; +} + +static int handle_ptr_to_fd(void *ptr) +{ + assert(ptr); + return (uintptr_t)ptr - 1; +} + +static int sock_handle_get(uint32_t instance_id, int fd) +{ + int handle = -1; + struct sock_instance *si = NULL; + + sock_lock(); + + si = sock_instance_find(instance_id); + if (!si) { + si = calloc(1, sizeof(*si)); + if (!si) + goto out; + si->id = instance_id; + TAILQ_INSERT_TAIL(&sock_instances, si, link); + } + + handle = handle_get(&si->db, fd_to_handle_ptr(fd)); +out: + sock_unlock(); + return handle; +} + +static int sock_handle_to_fd(uint32_t instance_id, uint32_t handle) +{ + int fd = -1; + struct sock_instance *si = NULL; + + sock_lock(); + si = sock_instance_find(instance_id); + if (si) + fd = handle_ptr_to_fd(handle_lookup(&si->db, handle)); + sock_unlock(); + return fd; +} + +static void sock_handle_put(uint32_t instance_id, uint32_t handle) +{ + struct sock_instance *si = NULL; + + sock_lock(); + si = sock_instance_find(instance_id); + if (si) + handle_put(&si->db, handle); + sock_unlock(); +} + +static bool chk_pt(struct tee_ioctl_param *param, uint32_t type) +{ + return (param->attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) == type; +} + +static int fd_flags_add(int fd, int flags) +{ + int val = 0; + + val = fcntl(fd, F_GETFD, 0); + if (val == -1) + return -1; + + val |= flags; + + return fcntl(fd, F_SETFL, val); +} + +static TEEC_Result sock_connect(uint32_t ip_vers, unsigned int protocol, + const char *server, uint16_t port, int *ret_fd) +{ + TEEC_Result r = TEEC_ERROR_GENERIC; + struct addrinfo *res0 = NULL; + struct addrinfo *res = NULL; + int fd = -1; + char port_name[10] = { 0 }; + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); + + snprintf(port_name, sizeof(port_name), "%" PRIu16, port); + + switch (ip_vers) { + case TEE_IP_VERSION_DC: + hints.ai_family = AF_UNSPEC; + break; + case TEE_IP_VERSION_4: + hints.ai_family = AF_INET; + break; + case TEE_IP_VERSION_6: + hints.ai_family = AF_INET6; + break; + default: + return TEEC_ERROR_BAD_PARAMETERS; + } + + if (protocol == TEE_ISOCKET_PROTOCOLID_TCP) + hints.ai_socktype = SOCK_STREAM; + else if (protocol == TEE_ISOCKET_PROTOCOLID_UDP) + hints.ai_socktype = SOCK_DGRAM; + else + return TEEC_ERROR_BAD_PARAMETERS; + + if (getaddrinfo(server, port_name, &hints, &res0)) + return TEE_ISOCKET_ERROR_HOSTNAME; + + for (res = res0; res; res = res->ai_next) { + fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (fd == -1) { + if (errno == ENOMEM || errno == ENOBUFS) + r = TEE_ISOCKET_ERROR_OUT_OF_RESOURCES; + else + r = TEEC_ERROR_GENERIC; + continue; + } + + if (connect(fd, res->ai_addr, res->ai_addrlen)) { + if (errno == ETIMEDOUT) + r = TEE_ISOCKET_ERROR_TIMEOUT; + else + r = TEEC_ERROR_COMMUNICATION; + + close(fd); + fd = -1; + continue; + } + + if (fd_flags_add(fd, O_NONBLOCK)) { + close(fd); + fd = -1; + r = TEEC_ERROR_GENERIC; + break; + } + + r = TEEC_SUCCESS; + break; + } + + freeaddrinfo(res0); + *ret_fd = fd; + return r; +} + +static TEEC_Result tee_socket_open(size_t num_params, + struct tee_ioctl_param *params) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + int handle = 0; + int fd = 0; + uint32_t instance_id = 0; + char *server = NULL; + uint32_t ip_vers = 0; + uint16_t port = 0; + uint32_t protocol = 0; + + if (num_params != 4 || + !chk_pt(params + 0, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) || + !chk_pt(params + 1, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) || + !chk_pt(params + 2, TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT) || + !chk_pt(params + 3, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT)) + return TEEC_ERROR_BAD_PARAMETERS; + + instance_id = params[0].b; + port = params[1].a; + protocol = params[1].b; + ip_vers = params[1].c; + + server = tee_supp_param_to_va(params + 2); + if (!server || server[MEMREF_SIZE(params + 2) - 1] != '\0') + return TEE_ISOCKET_ERROR_HOSTNAME; + + res = sock_connect(ip_vers, protocol, server, port, &fd); + if (res != TEEC_SUCCESS) + return res; + + handle = sock_handle_get(instance_id, fd); + if (handle < 0) { + close(fd); + return TEEC_ERROR_OUT_OF_MEMORY; + } + + params[3].a = handle; + return TEEC_SUCCESS; +} + +static TEEC_Result tee_socket_close(size_t num_params, + struct tee_ioctl_param *params) +{ + int handle = 0; + uint32_t instance_id = 0; + int fd = 0; + + if (num_params != 1 || + !chk_pt(params + 0, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT)) + return TEEC_ERROR_BAD_PARAMETERS; + + instance_id = params[0].b; + handle = params[0].c; + fd = sock_handle_to_fd(instance_id, handle); + if (fd < 0) + return TEEC_ERROR_BAD_PARAMETERS; + sock_handle_put(instance_id, handle); + if (close(fd)) { + EMSG("tee_socket_close: close(%d): %s", fd, strerror(errno)); + return TEEC_ERROR_GENERIC; + } + return TEEC_SUCCESS; +} + +static void sock_close_cb(int handle, void *ptr, void *arg) +{ + struct sock_instance *si = arg; + int fd = handle_ptr_to_fd(ptr); + + if (close(fd)) + EMSG("sock_close_cb instance_id %d handle %d fd %d: %s", + si->id, handle, fd, strerror(errno)); +} + +static TEEC_Result tee_socket_close_all(size_t num_params, + struct tee_ioctl_param *params) +{ + uint32_t instance_id = 0; + struct sock_instance *si = NULL; + + if (num_params != 1 || + !chk_pt(params + 0, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT)) + return TEEC_ERROR_BAD_PARAMETERS; + + instance_id = params[0].b; + sock_lock(); + si = sock_instance_find(instance_id); + if (si) + handle_foreach_put(&si->db, sock_close_cb, si); + sock_unlock(); + + return TEEC_SUCCESS; +} + +#define TS_NSEC_PER_SEC 1000000000 + +static void ts_add(const struct timespec *a, const struct timespec *b, + struct timespec *res) +{ + res->tv_sec = a->tv_sec + b->tv_sec; + res->tv_nsec = a->tv_nsec + b->tv_nsec; + if (res->tv_nsec >= TS_NSEC_PER_SEC) { + res->tv_sec++; + res->tv_nsec -= TS_NSEC_PER_SEC; + } +} + +static int ts_diff_to_polltimeout(const struct timespec *a, + const struct timespec *b) +{ + struct timespec diff; + + memset(&diff, 0, sizeof(diff)); + + diff.tv_sec = a->tv_sec - b->tv_sec; + diff.tv_nsec = a->tv_nsec - b->tv_nsec; + if (a->tv_nsec < b->tv_nsec) { + diff.tv_nsec += TS_NSEC_PER_SEC; + diff.tv_sec--; + } + + if ((diff.tv_sec - 1) > (INT_MAX / 1000)) + return INT_MAX; + return diff.tv_sec * 1000 + diff.tv_nsec / (TS_NSEC_PER_SEC / 1000); +} + +static void ts_delay_from_millis(uint32_t millis, struct timespec *res) +{ + res->tv_sec = millis / 1000; + res->tv_nsec = (millis % 1000) * (TS_NSEC_PER_SEC / 1000); +} + +static TEEC_Result poll_with_timeout(struct pollfd *pfd, nfds_t nfds, + uint32_t timeout) +{ + int to = 0; + int r = 0; + struct timespec now; + struct timespec until; + + memset(&now, 0, sizeof(now)); + memset(&until, 0, sizeof(until)); + + if (timeout == OPTEE_MRC_SOCKET_TIMEOUT_BLOCKING) { + to = -1; + } else { + struct timespec delay; + + memset(&delay, 0, sizeof(delay)); + + ts_delay_from_millis(timeout, &delay); + + if (clock_gettime(CLOCK_REALTIME, &now)) + return TEEC_ERROR_GENERIC; + + ts_add(&now, &delay, &until); + } + + while (true) { + if (to != -1) + to = ts_diff_to_polltimeout(&until, &now); + + r = poll(pfd, nfds, to); + if (!r) + return TEE_ISOCKET_ERROR_TIMEOUT; + if (r == -1) { + /* + * If we're interrupted by a signal treat + * recalculate the timeout (if needed) and wait + * again. + */ + if (errno == EINTR) { + if (to != -1 && + clock_gettime(CLOCK_REALTIME, &now)) + return TEEC_ERROR_GENERIC; + continue; + } + return TEEC_ERROR_BAD_PARAMETERS; + } + return TEEC_SUCCESS; + } +} + +static TEEC_Result write_with_timeout(int fd, const void *buf, size_t *blen, + uint32_t timeout) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + struct pollfd pfd = { .fd = fd, .events = POLLOUT }; + ssize_t r = 0; + + res = poll_with_timeout(&pfd, 1, timeout); + if (res != TEEC_SUCCESS) + return res; + + r = write(fd, buf, *blen); + if (r == -1) { + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) + return TEE_ISOCKET_ERROR_TIMEOUT; + return TEEC_ERROR_BAD_PARAMETERS; + } + *blen = r; + return TEEC_SUCCESS; +} + +static TEEC_Result tee_socket_send(size_t num_params, + struct tee_ioctl_param *params) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + int handle = 0; + int fd = 0; + uint32_t instance_id = 0; + void *buf = NULL; + size_t bytes = 0; + + if (num_params != 3 || + !chk_pt(params + 0, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) || + !chk_pt(params + 1, TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT) || + !chk_pt(params + 2, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT)) + return TEEC_ERROR_BAD_PARAMETERS; + + instance_id = params[0].b; + handle = params[0].c; + fd = sock_handle_to_fd(instance_id, handle); + if (fd < 0) + return TEEC_ERROR_BAD_PARAMETERS; + + buf = tee_supp_param_to_va(params + 1); + bytes = MEMREF_SIZE(params + 1); + res = write_with_timeout(fd, buf, &bytes, params[2].a); + if (res == TEEC_SUCCESS) + params[2].b = bytes; + return res; +} + +static ssize_t recv_with_out_flags(int fd, void *buf, size_t len, int inflags, + int *out_flags) +{ + ssize_t r = 0; + + while (true) { + struct iovec iov = { .iov_base = buf, .iov_len = len, }; + struct msghdr msg = { .msg_iov = &iov, .msg_iovlen = 1, }; + + r = recvmsg(fd, &msg, inflags); + if (r < 0) { + /* + * If the syscall was just interrupted by a signal + * delivery, try again. + */ + if (errno == EINTR) + continue; + return r; + } + + *out_flags = msg.msg_flags; + return r; + } +} + +static TEEC_Result read_with_timeout(int fd, void *buf, size_t *blen, + uint32_t timeout) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + struct pollfd pfd = { .fd = fd, .events = POLLIN }; + int socktype = 0; + socklen_t l = sizeof(socktype); + size_t peek_len = 0; + int out_flags = 0; + ssize_t r = 0; + int e = 0; + + if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &socktype, &l)) + return TEEC_ERROR_BAD_PARAMETERS; + + if (*blen) { + /* If *blen == 0, the timeout parameter has no effect. */ + res = poll_with_timeout(&pfd, 1, timeout); + if (res != TEEC_SUCCESS) + return res; + } + + if ((socktype == SOCK_DGRAM && *blen < SUPP_MAX_PEEK_LEN) || !*blen) { + /* Check how much data we have queued. */ + void *b = malloc(SUPP_MAX_PEEK_LEN); + + if (!b) + return TEEC_ERROR_OUT_OF_MEMORY; + r = recv_with_out_flags(fd, b, SUPP_MAX_PEEK_LEN, + MSG_PEEK | MSG_DONTWAIT, &out_flags); + e = errno; + free(b); + if (r < 0) + goto err; + + /* + * If the message was truncated we know that it's at least + * one byte larger. + */ + if (out_flags & MSG_TRUNC) + r++; + + if (!*blen) { + *blen = r; + return TEEC_SUCCESS; + } + + peek_len = r; + } + + r = recv_with_out_flags(fd, buf, *blen, MSG_DONTWAIT, &out_flags); + if (r == -1) { + e = errno; + goto err; + } + if (socktype == SOCK_DGRAM && (out_flags & MSG_TRUNC)) { + /* + * The datagram has been truncated, return the best length + * we have to indicate that. + */ + if (peek_len > (size_t)r) + *blen = peek_len; + else + *blen = r + 1; + } else { + *blen = r; + } + return TEEC_SUCCESS; + +err: + if (e == EAGAIN || e == EWOULDBLOCK) { + /* + * If *blen is supplied as 0 then we're not supposed wait + * for data so the call to poll has been skipped. In case + * there is no data available recvmsg() will return an + * error with errno set to EAGAIN or EWOULDBLOCK. + */ + if (!*blen) + return TEEC_SUCCESS; + return TEE_ISOCKET_ERROR_TIMEOUT; + } + return TEEC_ERROR_BAD_PARAMETERS; +} + +static TEEC_Result tee_socket_recv(size_t num_params, + struct tee_ioctl_param *params) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + int handle = 0; + int fd = 0; + uint32_t instance_id = 0; + void *buf = NULL; + size_t bytes = 0; + + if (num_params != 3 || + !chk_pt(params + 0, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) || + !chk_pt(params + 1, TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT) || + !chk_pt(params + 2, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT)) + return TEEC_ERROR_BAD_PARAMETERS; + + instance_id = params[0].b; + handle = params[0].c; + fd = sock_handle_to_fd(instance_id, handle); + if (fd < 0) + return TEEC_ERROR_BAD_PARAMETERS; + + buf = tee_supp_param_to_va(params + 1); + + bytes = MEMREF_SIZE(params + 1); + res = read_with_timeout(fd, buf, &bytes, params[2].a); + if (res == TEEC_SUCCESS) + MEMREF_SIZE(params + 1) = bytes; + + return res; +} + +static TEEC_Result tee_socket_ioctl_tcp(int fd, uint32_t command, + void *buf, size_t *blen) +{ + switch (command) { + case TEE_TCP_SET_RECVBUF: + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, buf, *blen)) + return TEEC_ERROR_BAD_PARAMETERS; + return TEEC_SUCCESS; + case TEE_TCP_SET_SENDBUF: + if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, buf, *blen)) + return TEEC_ERROR_BAD_PARAMETERS; + return TEEC_SUCCESS; + default: + return TEEC_ERROR_NOT_SUPPORTED; + } +} + +static TEEC_Result sa_set_port(struct sockaddr *sa, socklen_t slen, + uint16_t port) +{ + if (sa->sa_family == AF_INET) { + struct sockaddr_in *sain = (void *)sa; + + if (slen < (socklen_t)sizeof(*sain)) + return TEEC_ERROR_BAD_PARAMETERS; + sain->sin_port = htons(port); + + return TEEC_SUCCESS; + } + + if (sa->sa_family == AF_INET6) { + struct sockaddr_in6 *sain6 = (void *)sa; + + if (slen < (socklen_t)sizeof(*sain6)) + return TEEC_ERROR_BAD_PARAMETERS; + sain6->sin6_port = htons(port); + + return TEEC_SUCCESS; + } + + return TEEC_ERROR_BAD_PARAMETERS; +} + +static TEEC_Result sa_get_port(struct sockaddr *sa, socklen_t slen, + uint16_t *port) +{ + if (sa->sa_family == AF_INET) { + struct sockaddr_in *sain = (void *)sa; + + if (slen < (socklen_t)sizeof(*sain)) + return TEEC_ERROR_BAD_PARAMETERS; + *port = ntohs(sain->sin_port); + + return TEEC_SUCCESS; + } + + if (sa->sa_family == AF_INET6) { + struct sockaddr_in6 *sain6 = (void *)sa; + + if (slen < (socklen_t)sizeof(*sain6)) + return TEEC_ERROR_BAD_PARAMETERS; + *port = ntohs(sain6->sin6_port); + + return TEEC_SUCCESS; + } + + return TEEC_ERROR_BAD_PARAMETERS; +} + +static TEEC_Result udp_changeaddr(int fd, int family, const char *server, + uint16_t port) +{ + TEEC_Result r = TEE_ISOCKET_ERROR_HOSTNAME; + struct addrinfo *res0 = NULL; + struct addrinfo *res = NULL; + char port_name[10] = { 0 }; + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); + + snprintf(port_name, sizeof(port_name), "%" PRIu16, port); + + hints.ai_family = family; + hints.ai_socktype = SOCK_DGRAM; + if (getaddrinfo(server, port_name, &hints, &res0)) + return TEE_ISOCKET_ERROR_HOSTNAME; + for (res = res0; res; res = res->ai_next) { + if (connect(fd, res->ai_addr, res->ai_addrlen)) { + if (errno == ETIMEDOUT) + r = TEE_ISOCKET_ERROR_TIMEOUT; + else + r = TEEC_ERROR_COMMUNICATION; + continue; + } + r = TEEC_SUCCESS; + break; + } + freeaddrinfo(res0); + + return r; +} + +static TEEC_Result tee_socket_ioctl_udp(int fd, uint32_t command, + void *buf, size_t *blen) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + uint16_t port = 0; + struct sockaddr_storage sass; + struct sockaddr *sa = (struct sockaddr *)&sass; + socklen_t len = sizeof(sass); + + memset(&sass, 0, sizeof(sass)); + + if (getpeername(fd, sa, &len)) + return TEEC_ERROR_BAD_PARAMETERS; + + switch (command) { + case TEE_UDP_CHANGEADDR: + res = sa_get_port(sa, len, &port); + if (res != TEEC_SUCCESS) + return res; + + if (!blen || *((char *)buf + *blen - 1) != '\0') + return TEE_ISOCKET_ERROR_HOSTNAME; + + return udp_changeaddr(fd, sa->sa_family, buf, port); + case TEE_UDP_CHANGEPORT: + if (*blen != sizeof(port)) + return TEEC_ERROR_BAD_PARAMETERS; + memcpy(&port, buf, sizeof(port)); + res = sa_set_port(sa, len, port); + if (res != TEEC_SUCCESS) + return res; + if (connect(fd, sa, len)) + return TEEC_ERROR_GENERIC; + return TEEC_SUCCESS; + default: + return TEEC_ERROR_NOT_SUPPORTED; + } +} + +static TEEC_Result tee_socket_ioctl(size_t num_params, + struct tee_ioctl_param *params) +{ + TEEC_Result res = TEEC_ERROR_GENERIC; + int handle = 0; + int fd = 0; + uint32_t instance_id = 0; + uint32_t command = 0; + void *buf = NULL; + int socktype = 0; + socklen_t l = 0; + size_t sz = 0; + + if (num_params != 3 || + !chk_pt(params + 0, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) || + !chk_pt(params + 1, TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT) || + !chk_pt(params + 2, TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT)) + return TEEC_ERROR_BAD_PARAMETERS; + + instance_id = params[0].b; + handle = params[0].c; + command = params[2].a; + fd = sock_handle_to_fd(instance_id, handle); + if (fd < 0) + return TEEC_ERROR_BAD_PARAMETERS; + + l = sizeof(socktype); + if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &socktype, &l)) + return TEEC_ERROR_BAD_PARAMETERS; + + buf = tee_supp_param_to_va(params + 1); + + switch (socktype) { + case SOCK_STREAM: + sz = MEMREF_SIZE(params + 1); + res = tee_socket_ioctl_tcp(fd, command, buf, &sz); + MEMREF_SIZE(params + 1) = sz; + return res; + case SOCK_DGRAM: + sz = MEMREF_SIZE(params + 1); + res = tee_socket_ioctl_udp(fd, command, buf, &sz); + MEMREF_SIZE(params + 1) = sz; + return res; + default: + return TEEC_ERROR_BAD_PARAMETERS; + } +} + +TEEC_Result tee_socket_process(size_t num_params, + struct tee_ioctl_param *params) +{ + if (!num_params || !tee_supp_param_is_value(params)) + return TEEC_ERROR_BAD_PARAMETERS; + + switch (params->a) { + case OPTEE_MRC_SOCKET_OPEN: + return tee_socket_open(num_params, params); + case OPTEE_MRC_SOCKET_CLOSE: + return tee_socket_close(num_params, params); + case OPTEE_MRC_SOCKET_CLOSE_ALL: + return tee_socket_close_all(num_params, params); + case OPTEE_MRC_SOCKET_SEND: + return tee_socket_send(num_params, params); + case OPTEE_MRC_SOCKET_RECV: + return tee_socket_recv(num_params, params); + case OPTEE_MRC_SOCKET_IOCTL: + return tee_socket_ioctl(num_params, params); + default: + return TEEC_ERROR_BAD_PARAMETERS; + } +} diff --git a/optee_client/tee-supplicant/src/tee_socket.h b/optee_client/tee-supplicant/src/tee_socket.h new file mode 100644 index 0000000..800c27c --- /dev/null +++ b/optee_client/tee-supplicant/src/tee_socket.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __TEE_SOCKET_H +#define __TEE_SOCKET_H + +#include + +struct tee_ioctl_param; + +#if defined(CFG_GP_SOCKETS) && CFG_GP_SOCKETS == 1 +TEEC_Result tee_socket_process(size_t num_params, + struct tee_ioctl_param *params); +#else +static inline TEEC_Result tee_socket_process(size_t num_params, + struct tee_ioctl_param *params) +{ + (void)num_params; + (void)params; + + return TEEC_ERROR_NOT_SUPPORTED; +} +#endif + +#endif /*__SOCKET_H*/ diff --git a/optee_client/tee-supplicant/src/tee_supp_fs.c b/optee_client/tee-supplicant/src/tee_supp_fs.c new file mode 100644 index 0000000..cc4120f --- /dev/null +++ b/optee_client/tee-supplicant/src/tee_supp_fs.c @@ -0,0 +1,681 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef __aligned +#define __aligned(x) __attribute__((__aligned__(x))) +#endif +#include + +#ifndef PATH_MAX +#define PATH_MAX 255 +#endif + +/* Path to all secure storage files. */ +static char tee_fs_root[PATH_MAX]; + +#define TEE_FS_FILENAME_MAX_LENGTH 150 + +static pthread_mutex_t dir_handle_db_mutex = PTHREAD_MUTEX_INITIALIZER; +static struct handle_db dir_handle_db = + HANDLE_DB_INITIALIZER_WITH_MUTEX(&dir_handle_db_mutex); + +static TEEC_Result errno_to_teec(int err) +{ + switch (err) { + case ENOSPC: + return TEEC_ERROR_STORAGE_NO_SPACE; + case ENOENT: + return TEEC_ERROR_ITEM_NOT_FOUND; + default: + break; + } + return TEEC_ERROR_GENERIC; +} + +static size_t tee_fs_get_absolute_filename(char *file, char *out, + size_t out_size) +{ + int s = 0; + + if (!file || !out || (out_size <= strlen(tee_fs_root) + 1)) + return 0; + + s = snprintf(out, out_size, "%s%s", tee_fs_root, file); + if (s < 0 || (size_t)s >= out_size) + return 0; + + /* Safe to cast since we have checked that sizes are OK */ + return (size_t)s; +} + +static void fs_fsync(void) +{ + int fd = 0; + + fd = open(tee_fs_root, O_RDONLY | O_DIRECTORY); + if (fd > 0) { + fsync(fd); + close(fd); + } +} + +static int do_mkdir(const char *path, mode_t mode) +{ + struct stat st; + + memset(&st, 0, sizeof(st)); + + if (mkdir(path, mode) != 0 && errno != EEXIST) + return -1; + + if (stat(path, &st) != 0 && !S_ISDIR(st.st_mode)) + return -1; + + fs_fsync(); + return 0; +} + +static int mkpath(const char *path, mode_t mode) +{ + int status = 0; + char *subpath = strdup(path); + char *prev = subpath; + char *curr = NULL; + + while (status == 0 && (curr = strchr(prev, '/')) != 0) { + /* + * Check for root or double slash + */ + if (curr != prev) { + *curr = '\0'; + status = do_mkdir(subpath, mode); + *curr = '/'; + } + prev = curr + 1; + } + if (status == 0) + status = do_mkdir(path, mode); + + free(subpath); + return status; +} + +static int tee_supp_fs_init(void) +{ + size_t n = 0; + mode_t mode = 0700; + + n = snprintf(tee_fs_root, sizeof(tee_fs_root), "%s/", supplicant_params.fs_parent_path); + + if (n >= sizeof(tee_fs_root)) + return -1; + + if (mkpath(tee_fs_root, mode) != 0) + return -1; + + return 0; +} + +static int open_wrapper(const char *fname, int flags) +{ + int fd = 0; + + while (true) { + fd = open(fname, flags | O_SYNC, 0600); + if (fd >= 0 || errno != EINTR) + return fd; + } +} + +static TEEC_Result ree_fs_new_open(size_t num_params, + struct tee_ioctl_param *params) +{ + char abs_filename[PATH_MAX] = { 0 }; + char *fname = NULL; + int fd = 0; + + if (num_params != 3 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT || + (params[2].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fname = tee_supp_param_to_va(params + 1); + if (!fname) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!tee_fs_get_absolute_filename(fname, abs_filename, + sizeof(abs_filename))) + return TEEC_ERROR_BAD_PARAMETERS; + + fd = open_wrapper(abs_filename, O_RDWR); + if (fd < 0) { + /* + * In case the problem is the filesystem is RO, retry with the + * open flags restricted to RO. + */ + fd = open_wrapper(abs_filename, O_RDONLY); + if (fd < 0) + return TEEC_ERROR_ITEM_NOT_FOUND; + } + + params[2].a = fd; + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_create(size_t num_params, + struct tee_ioctl_param *params) +{ + char abs_filename[PATH_MAX] = { 0 }; + char abs_dir[PATH_MAX] = { 0 }; + char *fname = NULL; + char *d = NULL; + int fd = 0; + const int flags = O_RDWR | O_CREAT | O_TRUNC; + + if (num_params != 3 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT || + (params[2].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fname = tee_supp_param_to_va(params + 1); + if (!fname) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!tee_fs_get_absolute_filename(fname, abs_filename, + sizeof(abs_filename))) + return TEEC_ERROR_BAD_PARAMETERS; + + fd = open_wrapper(abs_filename, flags); + if (fd >= 0) + goto out; + if (errno != ENOENT) + return errno_to_teec(errno); + + /* Directory for file missing, try make to it */ + strncpy(abs_dir, abs_filename, sizeof(abs_dir)); + abs_dir[sizeof(abs_dir) - 1] = '\0'; + d = dirname(abs_dir); + if (!mkdir(d, 0700)) { + int err = 0; + + fd = open_wrapper(abs_filename, flags); + if (fd >= 0) + goto out; + /* + * The directory was made but the file could still not be + * created. + */ + err = errno; + rmdir(d); + return errno_to_teec(err); + } + if (errno != ENOENT) + return errno_to_teec(errno); + + /* Parent directory for file missing, try to make it */ + d = dirname(d); + if (mkdir(d, 0700)) + return errno_to_teec(errno); + + /* Try to make directory for file again */ + strncpy(abs_dir, abs_filename, sizeof(abs_dir)); + abs_dir[sizeof(abs_dir) - 1] = '\0'; + d = dirname(abs_dir); + if (mkdir(d, 0700)) { + int err = errno; + + d = dirname(d); + rmdir(d); + return errno_to_teec(err); + } + + fd = open_wrapper(abs_filename, flags); + if (fd < 0) { + int err = errno; + + rmdir(d); + d = dirname(d); + rmdir(d); + return errno_to_teec(err); + } + +out: + fs_fsync(); + params[2].a = fd; + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_close(size_t num_params, + struct tee_ioctl_param *params) +{ + int fd = 0; + + if (num_params != 1 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fd = params[0].b; + while (close(fd)) { + if (errno != EINTR) + return errno_to_teec(errno); + } + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_read(size_t num_params, + struct tee_ioctl_param *params) +{ + uint8_t *buf = NULL; + size_t len = 0; + off_t offs = 0; + int fd = 0; + ssize_t r = 0; + size_t s = 0; + + if (num_params != 2 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fd = params[0].b; + offs = params[0].c; + + buf = tee_supp_param_to_va(params + 1); + if (!buf) + return TEEC_ERROR_BAD_PARAMETERS; + len = MEMREF_SIZE(params + 1); + + s = 0; + r = -1; + while (r && len) { + r = pread(fd, buf, len, offs); + if (r < 0) { + if (errno == EINTR) + continue; + return errno_to_teec(errno); + } + assert((size_t)r <= len); + buf += r; + len -= r; + offs += r; + s += r; + } + + MEMREF_SIZE(params + 1) = s; + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_write(size_t num_params, + struct tee_ioctl_param *params) +{ + uint8_t *buf = NULL; + size_t len = 0; + off_t offs = 0; + int fd = 0; + ssize_t r = 0; + + if (num_params != 2 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fd = params[0].b; + offs = params[0].c; + + buf = tee_supp_param_to_va(params + 1); + if (!buf) + return TEEC_ERROR_BAD_PARAMETERS; + len = MEMREF_SIZE(params + 1); + + while (len) { + r = pwrite(fd, buf, len, offs); + if (r < 0) { + if (errno == EINTR) + continue; + return errno_to_teec(errno); + } + assert((size_t)r <= len); + buf += r; + len -= r; + offs += r; + } + + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_truncate(size_t num_params, + struct tee_ioctl_param *params) +{ + size_t len = 0; + int fd = 0; + + if (num_params != 1 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fd = params[0].b; + len = params[0].c; + + while (ftruncate(fd, len)) { + if (errno != EINTR) + return errno_to_teec(errno); + } + + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_remove(size_t num_params, + struct tee_ioctl_param *params) +{ + char abs_filename[PATH_MAX] = { 0 }; + char *fname = NULL; + char *d = NULL; + + if (num_params != 2 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fname = tee_supp_param_to_va(params + 1); + if (!fname) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!tee_fs_get_absolute_filename(fname, abs_filename, + sizeof(abs_filename))) + return TEEC_ERROR_BAD_PARAMETERS; + + if (unlink(abs_filename)) + return errno_to_teec(errno); + + /* If a file is removed, maybe the directory can be removed to? */ + d = dirname(abs_filename); + if (!rmdir(d)) { + /* + * If the directory was removed, maybe the parent directory + * can be removed too? + */ + d = dirname(d); + rmdir(d); + } + + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_rename(size_t num_params, + struct tee_ioctl_param *params) +{ + char old_abs_filename[PATH_MAX] = { 0 }; + char new_abs_filename[PATH_MAX] = { 0 }; + char *old_fname = NULL; + char *new_fname = NULL; + bool overwrite = false; + + if (num_params != 3 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT || + (params[2].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + overwrite = !!params[0].b; + + old_fname = tee_supp_param_to_va(params + 1); + if (!old_fname) + return TEEC_ERROR_BAD_PARAMETERS; + + new_fname = tee_supp_param_to_va(params + 2); + if (!new_fname) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!tee_fs_get_absolute_filename(old_fname, old_abs_filename, + sizeof(old_abs_filename))) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!tee_fs_get_absolute_filename(new_fname, new_abs_filename, + sizeof(new_abs_filename))) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!overwrite) { + struct stat st; + + if (!stat(new_abs_filename, &st)) + return TEEC_ERROR_ACCESS_CONFLICT; + } + if (rename(old_abs_filename, new_abs_filename)) { + if (errno == ENOENT) + return TEEC_ERROR_ITEM_NOT_FOUND; + } + + fs_fsync(); + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_opendir(size_t num_params, + struct tee_ioctl_param *params) +{ + char abs_filename[PATH_MAX] = { 0 }; + char *fname = NULL; + DIR *dir = NULL; + int handle = 0; + struct dirent *dent = NULL; + bool empty = true; + + if (num_params != 3 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT || + (params[2].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + fname = tee_supp_param_to_va(params + 1); + if (!fname) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!tee_fs_get_absolute_filename(fname, abs_filename, + sizeof(abs_filename))) + return TEEC_ERROR_BAD_PARAMETERS; + + dir = opendir(abs_filename); + if (!dir) + return TEEC_ERROR_ITEM_NOT_FOUND; + + /* + * Ignore empty directories. Works around an issue when the + * data path is mounted over NFS. Due to the way OP-TEE implements + * TEE_CloseAndDeletePersistentObject1() currently, tee-supplicant + * still has a file descriptor open to the file when it's removed in + * ree_fs_new_remove(). In this case the NFS server may create a + * temporary reference called .nfs????, and the rmdir() call fails + * so that the TA directory is left over. Checking this special case + * here avoids that TEE_StartPersistentObjectEnumerator() returns + * TEE_SUCCESS when it should return TEEC_ERROR_ITEM_NOT_FOUND. + * Test case: "xtest 6009 6010". + */ + while ((dent = readdir(dir))) { + if (dent->d_name[0] == '.') + continue; + empty = false; + break; + } + if (empty) { + closedir(dir); + return TEEC_ERROR_ITEM_NOT_FOUND; + } + rewinddir(dir); + + handle = handle_get(&dir_handle_db, dir); + if (handle < 0) { + closedir(dir); + return TEEC_ERROR_OUT_OF_MEMORY; + } + + params[2].a = handle; + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_closedir(size_t num_params, + struct tee_ioctl_param *params) +{ + DIR *dir = NULL; + + if (num_params != 1 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + dir = handle_put(&dir_handle_db, params[0].b); + if (!dir) + return TEEC_ERROR_BAD_PARAMETERS; + + closedir(dir); + + return TEEC_SUCCESS; +} + +static TEEC_Result ree_fs_new_readdir(size_t num_params, + struct tee_ioctl_param *params) +{ + DIR *dir = NULL; + struct dirent *dirent = NULL; + char *buf = NULL; + size_t len = 0; + size_t fname_len = 0; + + if (num_params != 2 || + (params[0].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT || + (params[1].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) != + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT) + return TEEC_ERROR_BAD_PARAMETERS; + + + buf = tee_supp_param_to_va(params + 1); + if (!buf) + return TEEC_ERROR_BAD_PARAMETERS; + len = MEMREF_SIZE(params + 1); + + dir = handle_lookup(&dir_handle_db, params[0].b); + if (!dir) + return TEEC_ERROR_BAD_PARAMETERS; + + while (true) { + dirent = readdir(dir); + if (!dirent) + return TEEC_ERROR_ITEM_NOT_FOUND; + if (dirent->d_name[0] != '.') + break; + } + + fname_len = strlen(dirent->d_name) + 1; + MEMREF_SIZE(params + 1) = fname_len; + if (fname_len > len) + return TEEC_ERROR_SHORT_BUFFER; + + memcpy(buf, dirent->d_name, fname_len); + + return TEEC_SUCCESS; +} + +TEEC_Result tee_supp_fs_process(size_t num_params, + struct tee_ioctl_param *params) +{ + if (!num_params || !tee_supp_param_is_value(params)) + return TEEC_ERROR_BAD_PARAMETERS; + + if (!tee_fs_root[0]) { + if (tee_supp_fs_init() != 0) { + EMSG("error tee_supp_fs_init: failed to create %s/", + tee_fs_root); + memset(tee_fs_root, 0, sizeof(tee_fs_root)); + return TEEC_ERROR_STORAGE_NOT_AVAILABLE; + } + } + + switch (params->a) { + case OPTEE_MRF_OPEN: + return ree_fs_new_open(num_params, params); + case OPTEE_MRF_CREATE: + return ree_fs_new_create(num_params, params); + case OPTEE_MRF_CLOSE: + return ree_fs_new_close(num_params, params); + case OPTEE_MRF_READ: + return ree_fs_new_read(num_params, params); + case OPTEE_MRF_WRITE: + return ree_fs_new_write(num_params, params); + case OPTEE_MRF_TRUNCATE: + return ree_fs_new_truncate(num_params, params); + case OPTEE_MRF_REMOVE: + return ree_fs_new_remove(num_params, params); + case OPTEE_MRF_RENAME: + return ree_fs_new_rename(num_params, params); + case OPTEE_MRF_OPENDIR: + return ree_fs_new_opendir(num_params, params); + case OPTEE_MRF_CLOSEDIR: + return ree_fs_new_closedir(num_params, params); + case OPTEE_MRF_READDIR: + return ree_fs_new_readdir(num_params, params); + default: + return TEEC_ERROR_BAD_PARAMETERS; + } +} diff --git a/optee_client/tee-supplicant/src/tee_supp_fs.h b/optee_client/tee-supplicant/src/tee_supp_fs.h new file mode 100644 index 0000000..43e13fe --- /dev/null +++ b/optee_client/tee-supplicant/src/tee_supp_fs.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef TEE_SUPP_FS_H +#define TEE_SUPP_FS_H + +#include + +struct tee_ioctl_param; + +TEEC_Result tee_supp_fs_process(size_t num_params, + struct tee_ioctl_param *params); + +#endif diff --git a/optee_client/tee-supplicant/src/tee_supplicant.c b/optee_client/tee-supplicant/src/tee_supplicant.c new file mode 100644 index 0000000..850ab6f --- /dev/null +++ b/optee_client/tee-supplicant/src/tee_supplicant.c @@ -0,0 +1,974 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "optee_msg_supplicant.h" + +#ifndef __aligned +#define __aligned(x) __attribute__((__aligned__(x))) +#endif +#include + +#define RPC_NUM_PARAMS 5 + +#define RPC_BUF_SIZE (sizeof(struct tee_iocl_supp_send_arg) + \ + RPC_NUM_PARAMS * sizeof(struct tee_ioctl_param)) + +char **ta_path; +char *ta_path_str; + +union tee_rpc_invoke { + uint64_t buf[(RPC_BUF_SIZE - 1) / sizeof(uint64_t) + 1]; + struct tee_iocl_supp_recv_arg recv; + struct tee_iocl_supp_send_arg send; +}; + +struct tee_shm { + int id; + void *p; + size_t size; + bool registered; + int fd; + struct tee_shm *next; +}; + +struct thread_arg { + int fd; + uint32_t gen_caps; + bool abort; + size_t num_waiters; + pthread_mutex_t mutex; +}; + +struct param_value { + uint64_t a; + uint64_t b; + uint64_t c; +}; + +static pthread_mutex_t shm_mutex = PTHREAD_MUTEX_INITIALIZER; +static struct tee_shm *shm_head; + +struct tee_supplicant_params supplicant_params = { + .ta_dir = "optee_armtz", +#ifdef TEE_PLUGIN_LOAD_PATH + .plugin_load_path = TEE_PLUGIN_LOAD_PATH, +#endif + .fs_parent_path = TEE_FS_PARENT_PATH, +}; + +static void *thread_main(void *a); + +static size_t num_waiters_inc(struct thread_arg *arg) +{ + size_t ret = 0; + + tee_supp_mutex_lock(&arg->mutex); + arg->num_waiters++; + assert(arg->num_waiters); + ret = arg->num_waiters; + tee_supp_mutex_unlock(&arg->mutex); + + return ret; +} + +static size_t num_waiters_dec(struct thread_arg *arg) +{ + size_t ret = 0; + + tee_supp_mutex_lock(&arg->mutex); + assert(arg->num_waiters); + arg->num_waiters--; + ret = arg->num_waiters; + tee_supp_mutex_unlock(&arg->mutex); + + return ret; +} + +static void *paged_aligned_alloc(size_t sz) +{ + void *p = NULL; + + if (!posix_memalign(&p, sysconf(_SC_PAGESIZE), sz)) + return p; + + return NULL; +} + +static int get_value(size_t num_params, struct tee_ioctl_param *params, + const uint32_t idx, struct param_value **value) +{ + if (idx >= num_params) + return -1; + + switch (params[idx].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + *value = (void *)¶ms[idx].a; + return 0; + default: + return -1; + } +} + +static struct tee_shm *find_tshm(int id) +{ + struct tee_shm *tshm = NULL; + + tee_supp_mutex_lock(&shm_mutex); + + tshm = shm_head; + while (tshm && tshm->id != id) + tshm = tshm->next; + + tee_supp_mutex_unlock(&shm_mutex); + + return tshm; +} + +static struct tee_shm *pop_tshm(int id) +{ + struct tee_shm *tshm = NULL; + struct tee_shm *prev = NULL; + + tee_supp_mutex_lock(&shm_mutex); + + tshm = shm_head; + if (!tshm) + goto out; + + if (tshm->id == id) { + shm_head = tshm->next; + goto out; + } + + do { + prev = tshm; + tshm = tshm->next; + if (!tshm) + goto out; + } while (tshm->id != id); + prev->next = tshm->next; + +out: + tee_supp_mutex_unlock(&shm_mutex); + + return tshm; +} + +static void push_tshm(struct tee_shm *tshm) +{ + tee_supp_mutex_lock(&shm_mutex); + + tshm->next = shm_head; + shm_head = tshm; + + tee_supp_mutex_unlock(&shm_mutex); +} + +/* Get parameter allocated by secure world */ +static int get_param(size_t num_params, struct tee_ioctl_param *params, + const uint32_t idx, TEEC_SharedMemory *shm) +{ + struct tee_shm *tshm = NULL; + size_t offs = 0; + size_t sz = 0; + + if (idx >= num_params) + return -1; + + switch (params[idx].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + break; + default: + return -1; + } + + memset(shm, 0, sizeof(*shm)); + + tshm = find_tshm(MEMREF_SHM_ID(params + idx)); + if (!tshm) { + /* + * It doesn't make sense to query required size of an + * input buffer. + */ + if ((params[idx].attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) == + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT) + return -1; + + /* + * Buffer isn't found, the caller is querying required size + * of the buffer. + */ + return 0; + } + + sz = MEMREF_SIZE(params + idx); + offs = MEMREF_SHM_OFFS(params + idx); + if ((sz + offs) < sz) + return -1; + if ((sz + offs) > tshm->size) + return -1; + + shm->flags = TEEC_MEM_INPUT | TEEC_MEM_OUTPUT; + shm->size = sz; + shm->id = MEMREF_SHM_ID(params + idx); + shm->buffer = (uint8_t *)tshm->p + offs; + + return 0; +} + +static void uuid_from_octets(TEEC_UUID *d, const uint8_t s[TEE_IOCTL_UUID_LEN]) +{ + d->timeLow = (s[0] << 24) | (s[1] << 16) | (s[2] << 8) | s[3]; + d->timeMid = (s[4] << 8) | s[5]; + d->timeHiAndVersion = (s[6] << 8) | s[7]; + memcpy(d->clockSeqAndNode, s + 8, sizeof(d->clockSeqAndNode)); +} + +static uint32_t load_ta(size_t num_params, struct tee_ioctl_param *params) +{ + int ta_found = 0; + size_t size = 0; + struct param_value *val_cmd = NULL; + TEEC_UUID uuid; + TEEC_SharedMemory shm_ta; + + memset(&uuid, 0, sizeof(uuid)); + memset(&shm_ta, 0, sizeof(shm_ta)); + + if (num_params != 2 || get_value(num_params, params, 0, &val_cmd) || + get_param(num_params, params, 1, &shm_ta)) + return TEEC_ERROR_BAD_PARAMETERS; + + uuid_from_octets(&uuid, (void *)val_cmd); + + size = shm_ta.size; + ta_found = TEECI_LoadSecureModule(supplicant_params.ta_dir, &uuid, shm_ta.buffer, &size); + if (ta_found != TA_BINARY_FOUND) { + EMSG(" TA not found"); + return TEEC_ERROR_ITEM_NOT_FOUND; + } + + MEMREF_SIZE(params + 1) = size; + + /* + * If a buffer wasn't provided, just tell which size it should be. + * If it was provided but isn't big enough, report an error. + */ + if (shm_ta.buffer && size > shm_ta.size) + return TEEC_ERROR_SHORT_BUFFER; + + return TEEC_SUCCESS; +} + +static struct tee_shm *alloc_shm(int fd, size_t size) +{ + struct tee_shm *shm = NULL; + struct tee_ioctl_shm_alloc_data data; + + memset(&data, 0, sizeof(data)); + + shm = calloc(1, sizeof(*shm)); + if (!shm) + return NULL; + + data.size = size; + shm->fd = ioctl(fd, TEE_IOC_SHM_ALLOC, &data); + if (shm->fd < 0) { + free(shm); + return NULL; + } + + shm->p = mmap(NULL, data.size, PROT_READ | PROT_WRITE, MAP_SHARED, + shm->fd, 0); + if (shm->p == (void *)MAP_FAILED) { + close(shm->fd); + free(shm); + return NULL; + } + + shm->id = data.id; + shm->registered = false; + return shm; +} + +static struct tee_shm *register_local_shm(int fd, size_t size) +{ + struct tee_shm *shm = NULL; + void *buf = NULL; + struct tee_ioctl_shm_register_data data; + + memset(&data, 0, sizeof(data)); + + buf = paged_aligned_alloc(size); + if (!buf) + return NULL; + + shm = calloc(1, sizeof(*shm)); + if (!shm) { + free(buf); + return NULL; + } + + data.addr = (uintptr_t)buf; + data.length = size; + + shm->fd = ioctl(fd, TEE_IOC_SHM_REGISTER, &data); + if (shm->fd < 0) { + free(shm); + free(buf); + return NULL; + } + + shm->p = buf; + shm->registered = true; + shm->id = data.id; + + return shm; +} + +static uint32_t process_alloc(struct thread_arg *arg, size_t num_params, + struct tee_ioctl_param *params) +{ + struct param_value *val = NULL; + struct tee_shm *shm = NULL; + + if (num_params != 1 || get_value(num_params, params, 0, &val)) + return TEEC_ERROR_BAD_PARAMETERS; + + if (arg->gen_caps & TEE_GEN_CAP_REG_MEM) + shm = register_local_shm(arg->fd, val->b); + else + shm = alloc_shm(arg->fd, val->b); + + if (!shm) + return TEEC_ERROR_OUT_OF_MEMORY; + + shm->size = val->b; + val->c = shm->id; + push_tshm(shm); + + return TEEC_SUCCESS; +} + +static uint32_t process_free(size_t num_params, struct tee_ioctl_param *params) +{ + struct param_value *val = NULL; + struct tee_shm *shm = NULL; + int id = 0; + + if (num_params != 1 || get_value(num_params, params, 0, &val)) + return TEEC_ERROR_BAD_PARAMETERS; + + id = val->b; + + shm = pop_tshm(id); + if (!shm) + return TEEC_ERROR_BAD_PARAMETERS; + + close(shm->fd); + if (shm->registered) { + free(shm->p); + } else { + if (munmap(shm->p, shm->size) != 0) { + EMSG("munmap(%p, %zu) failed - Error = %s", + shm->p, shm->size, strerror(errno)); + free(shm); + return TEEC_ERROR_BAD_PARAMETERS; + } + } + + free(shm); + return TEEC_SUCCESS; +} + + + +/* How many device sequence numbers will be tried before giving up */ +#define MAX_DEV_SEQ 10 + +static int open_dev(const char *devname, uint32_t *gen_caps) +{ + int fd = 0; + struct tee_ioctl_version_data vers; + + memset(&vers, 0, sizeof(vers)); + + fd = open(devname, O_RDWR); + if (fd < 0) + return -1; + + if (ioctl(fd, TEE_IOC_VERSION, &vers)) + goto err; + + /* Only OP-TEE supported */ + if (vers.impl_id != TEE_IMPL_ID_OPTEE) + goto err; + + if (gen_caps) + *gen_caps = vers.gen_caps; + + DMSG("using device \"%s\"", devname); + return fd; +err: + close(fd); + return -1; +} + +static int get_dev_fd(uint32_t *gen_caps) +{ + int fd = 0; + char name[PATH_MAX] = { 0 }; + size_t n = 0; + + for (n = 0; n < MAX_DEV_SEQ; n++) { + snprintf(name, sizeof(name), "/dev/teepriv%zu", n); + fd = open_dev(name, gen_caps); + if (fd >= 0) + return fd; + } + return -1; +} + +static int usage(int status) +{ + fprintf(stderr, "Usage: tee-supplicant [options] []\n"); + fprintf(stderr, "\t-h, --help: this help\n"); + fprintf(stderr, "\t-d, --daemonize: run as a daemon (fork and return " + "after child has opened the TEE device or on error)\n"); + fprintf(stderr, "\t-f, --fs-parent-path: secure fs parent path [%s]\n", + supplicant_params.fs_parent_path); + fprintf(stderr, "\t-t, --ta-dir: TAs dirname under %s [%s]\n", TEEC_LOAD_PATH, + supplicant_params.ta_dir); + fprintf(stderr, "\t-p, --plugin-path: plugin load path [%s]\n", + supplicant_params.plugin_load_path); + fprintf(stderr, "\t-r, --rpmb-cid: RPMB device identification register " + "(CID) in hexadecimal\n"); + return status; +} + +static uint32_t process_rpmb(size_t num_params, struct tee_ioctl_param *params) +{ + TEEC_SharedMemory req; + TEEC_SharedMemory rsp; + + memset(&req, 0, sizeof(req)); + memset(&rsp, 0, sizeof(rsp)); + + if (get_param(num_params, params, 0, &req) || + get_param(num_params, params, 1, &rsp)) + return TEEC_ERROR_BAD_PARAMETERS; + + return rpmb_process_request(req.buffer, req.size, rsp.buffer, rsp.size); +} + +static bool read_request(int fd, union tee_rpc_invoke *request) +{ + struct tee_ioctl_buf_data data; + + memset(&data, 0, sizeof(data)); + + data.buf_ptr = (uintptr_t)request; + data.buf_len = sizeof(*request); + if (ioctl(fd, TEE_IOC_SUPPL_RECV, &data)) { + EMSG("TEE_IOC_SUPPL_RECV: %s", strerror(errno)); + return false; + } + return true; +} + +static bool write_response(int fd, union tee_rpc_invoke *request) +{ + struct tee_ioctl_buf_data data; + + memset(&data, 0, sizeof(data)); + + data.buf_ptr = (uintptr_t)&request->send; + data.buf_len = sizeof(struct tee_iocl_supp_send_arg) + + sizeof(struct tee_ioctl_param) * + (__u64)request->send.num_params; + if (ioctl(fd, TEE_IOC_SUPPL_SEND, &data)) { + EMSG("TEE_IOC_SUPPL_SEND: %s", strerror(errno)); + return false; + } + return true; +} + +static bool find_params(union tee_rpc_invoke *request, uint32_t *func, + size_t *num_params, struct tee_ioctl_param **params, + size_t *num_meta) +{ + struct tee_ioctl_param *p = NULL; + size_t n = 0; + + p = (struct tee_ioctl_param *)(&request->recv + 1); + + /* Skip meta parameters in the front */ + for (n = 0; n < request->recv.num_params; n++) + if (!(p[n].attr & TEE_IOCTL_PARAM_ATTR_META)) + break; + + *func = request->recv.func; + *num_params = request->recv.num_params - n; + *params = p + n; + *num_meta = n; + + /* Make sure that no meta parameters follows a non-meta parameter */ + for (; n < request->recv.num_params; n++) { + if (p[n].attr & TEE_IOCTL_PARAM_ATTR_META) { + EMSG("Unexpected meta parameter"); + return false; + } + } + + return true; +} + +static bool spawn_thread(struct thread_arg *arg) +{ + int e = 0; + pthread_t tid; + + memset(&tid, 0, sizeof(tid)); + + DMSG("Spawning a new thread"); + + /* + * Increase number of waiters now to avoid starting another thread + * before this thread has been scheduled. + */ + num_waiters_inc(arg); + + e = pthread_create(&tid, NULL, thread_main, arg); + if (e) { + EMSG("pthread_create: %s", strerror(e)); + num_waiters_dec(arg); + return false; + } + + e = pthread_detach(tid); + if (e) + EMSG("pthread_detach: %s", strerror(e)); + + return true; +} + +static bool process_one_request(struct thread_arg *arg) +{ + size_t num_params = 0; + size_t num_meta = 0; + struct tee_ioctl_param *params = NULL; + uint32_t func = 0; + uint32_t ret = 0; + union tee_rpc_invoke request; + + memset(&request, 0, sizeof(request)); + + DMSG("looping"); + request.recv.num_params = RPC_NUM_PARAMS; + + /* Let it be known that we can deal with meta parameters */ + params = (struct tee_ioctl_param *)(&request.send + 1); + params->attr = TEE_IOCTL_PARAM_ATTR_META; + + num_waiters_inc(arg); + + if (!read_request(arg->fd, &request)) + return false; + + if (!find_params(&request, &func, &num_params, ¶ms, &num_meta)) + return false; + + if (num_meta && !num_waiters_dec(arg) && !spawn_thread(arg)) + return false; + + switch (func) { + case OPTEE_MSG_RPC_CMD_LOAD_TA: + ret = load_ta(num_params, params); + break; + case OPTEE_MSG_RPC_CMD_FS: + ret = tee_supp_fs_process(num_params, params); + break; + case OPTEE_MSG_RPC_CMD_RPMB: + ret = process_rpmb(num_params, params); + break; + case OPTEE_MSG_RPC_CMD_SHM_ALLOC: + ret = process_alloc(arg, num_params, params); + break; + case OPTEE_MSG_RPC_CMD_SHM_FREE: + ret = process_free(num_params, params); + break; + case OPTEE_MSG_RPC_CMD_GPROF: + ret = prof_process(num_params, params, "gmon-"); + break; + case OPTEE_MSG_RPC_CMD_SOCKET: + ret = tee_socket_process(num_params, params); + break; + case OPTEE_MSG_RPC_CMD_FTRACE: + ret = prof_process(num_params, params, "ftrace-"); + break; + case OPTEE_MSG_RPC_CMD_PLUGIN: + ret = plugin_process(num_params, params); + break; + default: + EMSG("Cmd [0x%" PRIx32 "] not supported", func); + /* Not supported. */ + ret = TEEC_ERROR_NOT_SUPPORTED; + break; + } + + request.send.ret = ret; + return write_response(arg->fd, &request); +} + +static void *thread_main(void *a) +{ + struct thread_arg *arg = a; + + /* + * Now that this thread has been scheduled, compensate for the + * initial increase in spawn_thread() before. + */ + num_waiters_dec(arg); + + while (!arg->abort) { + if (!process_one_request(arg)) + arg->abort = true; + } + + return NULL; +} + +#define TEEC_TEST_LOAD_PATH "/foo:/bar::/baz" + +static void set_ta_path(void) +{ + char *p = NULL; + char *saveptr = NULL; + const char *path = (char *) +#ifdef TEEC_TEST_LOAD_PATH + TEEC_TEST_LOAD_PATH ":" +#endif + TEEC_LOAD_PATH; + size_t n = 0; + + ta_path_str = strdup(path); + if (!ta_path_str) + goto err; + + p = ta_path_str; + while (strtok_r(p, ":", &saveptr)) { + p = NULL; + n++; + } + n++; /* NULL terminator */ + + ta_path = malloc(n * sizeof(char *)); + if (!ta_path) + goto err; + + n = 0; + strcpy(ta_path_str, path); + p = ta_path_str; + while ((ta_path[n++] = strtok_r(p, ":", &saveptr))) + p = NULL; + + return; +err: + EMSG("out of memory"); + exit(EXIT_FAILURE); +} + +/* + * Similar to the standard libc function daemon(0, 0) but the parent process + * issues a blocking read on pipefd[0] before exiting. + * Returns 0 on success, <0 on error. + */ +static int make_daemon(int pipefd[2]) +{ + int fd = 0; + char c = 0; + int n = 0; + + switch (fork()) { + case -1: + return -1; + case 0: + /* In child */ + close(pipefd[0]); + break; + default: + /* In parent */ + close(pipefd[1]); + n = read(pipefd[0], &c, 1); + close(pipefd[0]); + if (!n) { + /* + * Nothing has been read: child has closed without + * writing (either exited on error or crashed) + */ + return -1; + } + /* Child is done with the opening of the TEE device */ + _exit(EXIT_SUCCESS); + } + + if (setsid() < 0) + return -2; + + if (chdir("/") < 0) + return -3; + + fd = open("/dev/null", O_RDWR); + if (fd < 0) + return -4; + dup2(fd, 0); + dup2(fd, 1); + dup2(fd, 2); + close(fd); + + return 0; +} + +int main(int argc, char *argv[]) +{ + struct thread_arg arg = { .fd = -1 }; + int pipefd[2] = { 0, }; + bool daemonize = false; + char *dev = NULL; + int e = 0; + int long_index = 0; + int opt = 0; + + e = pthread_mutex_init(&arg.mutex, NULL); + if (e) { + EMSG("pthread_mutex_init: %s", strerror(e)); + EMSG("terminating..."); + exit(EXIT_FAILURE); + } + + static struct option long_options[] = { + /* long name | has argument | flag | short value */ + { "help", no_argument, 0, 'h' }, + { "daemonize", no_argument, 0, 'd' }, + { "fs-parent-path", required_argument, 0, 'f' }, + { "ta-dir", required_argument, 0, 't' }, + { "plugin-path", required_argument, 0, 'p' }, + { "rpmb-cid", required_argument, 0, 'r' }, + { 0, 0, 0, 0 } + }; + + while ((opt = getopt_long(argc, argv, "hdf:t:p:r:", + long_options, &long_index )) != -1) { + switch (opt) { + case 'h' : + return usage(EXIT_SUCCESS); + break; + case 'd': + daemonize = true; + break; + case 'f': + supplicant_params.fs_parent_path = optarg; + break; + case 't': + supplicant_params.ta_dir = optarg; + break; + case 'p': + supplicant_params.plugin_load_path = optarg; + break; + case 'r': + supplicant_params.rpmb_cid = optarg; + break; + default: + return usage(EXIT_FAILURE); + } + } + /* check for non option argument, which is device name */ + if (argv[optind]) { + fprintf(stderr, "Using device %s.\n", argv[optind]); + dev = argv[optind]; + /* check that we do not have too many arguments */ + if (argv[optind + 1]) { + fprintf(stderr, "Too many arguments passed: extra argument: %s.\n", + argv[optind+1]); + return usage(EXIT_FAILURE); + } + } + + + set_ta_path(); + + if (plugin_load_all() != 0) { + EMSG("failed to load plugins"); + exit(EXIT_FAILURE); + } + + if (daemonize) { + if (pipe(pipefd) < 0) { + EMSG("pipe(): %s", strerror(errno)); + exit(EXIT_FAILURE); + } + e = make_daemon(pipefd); + if (e < 0) { + EMSG("make_daemon(): %d", e); + exit(EXIT_FAILURE); + } + } + + if (dev) { + arg.fd = open_dev(dev, &arg.gen_caps); + if (arg.fd < 0) { + EMSG("failed to open \"%s\"", argv[1]); + exit(EXIT_FAILURE); + } + } else { + arg.fd = get_dev_fd(&arg.gen_caps); + if (arg.fd < 0) { + EMSG("failed to find an OP-TEE supplicant device"); + exit(EXIT_FAILURE); + } + } + + if (daemonize) { + /* Release parent */ + if (write(pipefd[1], "", 1) != 1) { + EMSG("write(): %s", strerror(errno)); + exit(EXIT_FAILURE); + } + close(pipefd[1]); + } + + while (!arg.abort) { + if (!process_one_request(&arg)) + arg.abort = true; + } + + close(arg.fd); + + return EXIT_FAILURE; +} + +bool tee_supp_param_is_memref(struct tee_ioctl_param *param) +{ + switch (param->attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + return true; + default: + return false; + } +} + +bool tee_supp_param_is_value(struct tee_ioctl_param *param) +{ + switch (param->attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + return true; + default: + return false; + } +} + +void *tee_supp_param_to_va(struct tee_ioctl_param *param) +{ + struct tee_shm *tshm = NULL; + size_t end_offs = 0; + + if (!tee_supp_param_is_memref(param)) + return NULL; + + end_offs = MEMREF_SIZE(param) + MEMREF_SHM_OFFS(param); + if (end_offs < MEMREF_SIZE(param) || end_offs < MEMREF_SHM_OFFS(param)) + return NULL; + + tshm = find_tshm(MEMREF_SHM_ID(param)); + if (!tshm) + return NULL; + + if (end_offs > tshm->size) + return NULL; + + return (uint8_t *)tshm->p + MEMREF_SHM_OFFS(param); +} + +void tee_supp_mutex_lock(pthread_mutex_t *mu) +{ + int e = pthread_mutex_lock(mu); + + if (e) { + EMSG("pthread_mutex_lock: %s", strerror(e)); + EMSG("terminating..."); + exit(EXIT_FAILURE); + } +} + +void tee_supp_mutex_unlock(pthread_mutex_t *mu) +{ + int e = pthread_mutex_unlock(mu); + + if (e) { + EMSG("pthread_mutex_unlock: %s", strerror(e)); + EMSG("terminating..."); + exit(EXIT_FAILURE); + } +} diff --git a/optee_client/tee-supplicant/src/tee_supplicant.h b/optee_client/tee-supplicant/src/tee_supplicant.h new file mode 100644 index 0000000..cf98ac8 --- /dev/null +++ b/optee_client/tee-supplicant/src/tee_supplicant.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef TEE_SUPPLICANT_H +#define TEE_SUPPLICANT_H + +#include +#include + +/* Helpers to access memref parts of a struct tee_ioctl_param */ +#define MEMREF_SHM_ID(p) ((p)->c) +#define MEMREF_SHM_OFFS(p) ((p)->a) +#define MEMREF_SIZE(p) ((p)->b) + +struct tee_ioctl_param; + +/* Global tee-supplicant parameters */ +struct tee_supplicant_params { + const char *ta_dir; + const char *plugin_load_path; + const char *fs_parent_path; + const char *rpmb_cid; +}; + +extern struct tee_supplicant_params supplicant_params; + +bool tee_supp_param_is_memref(struct tee_ioctl_param *param); +bool tee_supp_param_is_value(struct tee_ioctl_param *param); +void *tee_supp_param_to_va(struct tee_ioctl_param *param); + +void tee_supp_mutex_lock(pthread_mutex_t *mu); +void tee_supp_mutex_unlock(pthread_mutex_t *mu); + +#endif /*TEE_SUPPLICANT_H*/ diff --git a/optee_client/tee-supplicant/src/teec_ta_load.c b/optee_client/tee-supplicant/src/teec_ta_load.c new file mode 100644 index 0000000..2cb5e98 --- /dev/null +++ b/optee_client/tee-supplicant/src/teec_ta_load.c @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* + * Attempt to first load TAs from a writable directory. This is + * intended for testing (xtest 1008, load_corrupt_ta specifically), + * and should not be enabled in a production system, as it would + * greatly facilitate loading rogue TA code. + */ +#ifdef CFG_TA_TEST_PATH +# ifndef TEEC_TEST_LOAD_PATH +# ifdef __ANDROID__ +# define TEEC_TEST_LOAD_PATH "/data/vendor/tee" +# else +# define TEEC_TEST_LOAD_PATH "/tmp" +# endif +# endif +#endif + +#ifndef PATH_MAX +#define PATH_MAX 255 +#endif + +struct tee_rpc_cmd { + void *buffer; + uint32_t size; + uint32_t type; + int fd; +}; + +/* + * Based on the uuid this function will try to find a TA-binary on the + * filesystem and return it back to the caller in the parameter ta. + * + * @param: prefix Prefix for TA load path + * @param: dev_path Where to load the TA from. The full path to the TA + * binary is @prefix/@dev_path/@destination.ta. + * @param: destination The uuid of the TA we are searching for. + * @param: ta A pointer which this function will allocate and copy + * the TA from the filesystem to the pointer itself. It is + * the callers responsibility to free the pointer. + * @param: ta_size The size of the TA found on file system. It will be 0 + * if no TA was not found. + * + * @return 0 if TA was found, otherwise -1. + */ +static int try_load_secure_module(const char* prefix, + const char* dev_path, + const TEEC_UUID *destination, void *ta, + size_t *ta_size) +{ + char fname[PATH_MAX] = { 0 }; + FILE *file = NULL; + bool first_try = true; + size_t s = 0; + long l = 0; + int n = 0; + + if (!ta_size || !destination) { + DMSG("wrong inparameter to TEECI_LoadSecureModule"); + return TA_BINARY_NOT_FOUND; + } + + /* + * We expect the TA binary to be named after the UUID as per RFC4122, + * that is: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.ta + * If the file cannot be open, try the deprecated format: + * xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx.ta + */ +again: + n = snprintf(fname, PATH_MAX, + "%s/%s/%08x-%04x-%04x-%02x%02x%s%02x%02x%02x%02x%02x%02x.ta", + prefix, dev_path, + destination->timeLow, + destination->timeMid, + destination->timeHiAndVersion, + destination->clockSeqAndNode[0], + destination->clockSeqAndNode[1], + first_try ? "-" : "", + destination->clockSeqAndNode[2], + destination->clockSeqAndNode[3], + destination->clockSeqAndNode[4], + destination->clockSeqAndNode[5], + destination->clockSeqAndNode[6], + destination->clockSeqAndNode[7]); + + DMSG("Attempt to load %s", fname); + + if ((n < 0) || (n >= PATH_MAX)) { + EMSG("wrong TA path [%s]", fname); + return TA_BINARY_NOT_FOUND; + } + + file = fopen(fname, "r"); + if (file == NULL) { + DMSG("failed to open the ta %s TA-file", fname); + if (first_try) { + first_try = false; + goto again; + } + return TA_BINARY_NOT_FOUND; + } + + if (fseek(file, 0, SEEK_END) != 0) { + fclose(file); + return TA_BINARY_NOT_FOUND; + } + + l = ftell(file); + if (l < 0) { + DMSG("failed to ftell the ta %s TA-file", fname); + fclose(file); + return TA_BINARY_NOT_FOUND; + } + + s = l; + if (s > *ta_size || !ta) { + /* + * Buffer isn't large enough, return the required size to + * let the caller increase the size of the buffer and try + * again. + */ + goto out; + } + + if (fseek(file, 0, SEEK_SET) != 0) { + fclose(file); + return TA_BINARY_NOT_FOUND; + } + + if (s != fread(ta, 1, s, file)) { + DMSG("failed to fread the ta %s TA-file", fname); + fclose(file); + return TA_BINARY_NOT_FOUND; + } + +out: + *ta_size = s; + fclose(file); + return TA_BINARY_FOUND; +} + +int TEECI_LoadSecureModule(const char* dev_path, + const TEEC_UUID *destination, void *ta, + size_t *ta_size) +{ + int res = TA_BINARY_NOT_FOUND; + char **path = NULL; + + for (path = ta_path; *path; path++) { + res = try_load_secure_module(*path, dev_path, destination, ta, + ta_size); + if (res == TA_BINARY_FOUND) + break; + } + return res; +} diff --git a/optee_client/tee-supplicant/src/teec_ta_load.h b/optee_client/tee-supplicant/src/teec_ta_load.h new file mode 100644 index 0000000..c89f4a0 --- /dev/null +++ b/optee_client/tee-supplicant/src/teec_ta_load.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _TEEC_TA_LOAD_H +#define _TEEC_TA_LOAD_H +#include + +#define TA_BINARY_FOUND 0 +#define TA_BINARY_NOT_FOUND -1 + +/* Heap copy of TA load paths, separated by '\0' (access via ta_path) */ +extern char *ta_path_str; +/* NULL-terminated list of paths (pointers into ta_path_str) */ +extern char **ta_path; + +/** + * Based on the uuid this function will try to find a TA-binary on the + * filesystem and return it back to the caller in the parameter ta. + * + * @param: destination The uuid of the TA we are searching for. + * @param: ta A pointer which this function will copy + * the TA from the filesystem to if *@ta_size i large + * enough. + * @param: ta_size The size of the TA found on file system. It will be 0 + * if no TA was not found. + * + * @return 0 if TA was found, otherwise -1. + */ +int TEECI_LoadSecureModule(const char *name, + const TEEC_UUID *destination, void *ta, + size_t *ta_size); +#endif diff --git a/optee_client/tee-supplicant/tee_supplicant_android.mk b/optee_client/tee-supplicant/tee_supplicant_android.mk new file mode 100644 index 0000000..04eee2a --- /dev/null +++ b/optee_client/tee-supplicant/tee_supplicant_android.mk @@ -0,0 +1,74 @@ +################################################################################ +# Build tee supplicant # +################################################################################ +LOCAL_PATH := $(call my-dir) + +include $(CLEAR_VARS) +LOCAL_CFLAGS += $(optee_CFLAGS) + +LOCAL_CFLAGS += -DDEBUGLEVEL_$(CFG_TEE_SUPP_LOG_LEVEL) \ + -DBINARY_PREFIX=\"TEES\" \ + -DTEE_FS_PARENT_PATH=\"$(CFG_TEE_FS_PARENT_PATH)\" \ + -DTEEC_LOAD_PATH=\"$(CFG_TEE_CLIENT_LOAD_PATH)\" + +ifneq ($(TEEC_TEST_LOAD_PATH),) +LOCAL_CFLAGS += -DTEEC_TEST_LOAD_PATH=\"$(TEEC_TEST_LOAD_PATH)\" +endif + +ifeq ($(CFG_TA_TEST_PATH),y) +LOCAL_CFLAGS += -DCFG_TA_TEST_PATH=1 +endif + +LOCAL_SRC_FILES += src/tee_supplicant.c \ + src/teec_ta_load.c \ + src/tee_supp_fs.c \ + src/rpmb.c \ + src/handle.c + +ifeq ($(CFG_GP_SOCKETS),y) +LOCAL_SRC_FILES += src/tee_socket.c +LOCAL_CFLAGS += -DCFG_GP_SOCKETS=1 +endif + +RPMB_EMU ?= 1 +ifeq ($(RPMB_EMU),1) +LOCAL_SRC_FILES += src/sha2.c src/hmac_sha2.c +LOCAL_CFLAGS += -DRPMB_EMU=1 +endif + +ifneq (,$(filter y,$(CFG_TA_GPROF_SUPPORT) $(CFG_FTRACE_SUPPORT))) +LOCAL_SRC_FILES += src/prof.c +endif + +ifeq ($(CFG_TA_GPROF_SUPPORT),y) +LOCAL_CFLAGS += -DCFG_TA_GPROF_SUPPORT +endif + +ifeq ($(CFG_TEE_SUPP_PLUGINS),y) +LOCAL_SRC_FILES += src/plugin.c + +LOCAL_CFLAGS += -DTEE_SUPP_PLUGINS \ + -DTEE_PLUGIN_LOAD_PATH=\"$(CFG_TEE_PLUGIN_LOAD_PATH)\" + +# Needed to dynamically load user plugins +LOCAL_LDFLAGS += -ldl +# Needed for dlopen() +LOCAL_LDFLAGS += -Wl,-rpath=$(CFG_TEE_PLUGIN_LOAD_PATH) +endif + +LOCAL_CFLAGS += -pthread + +ifeq ($(CFG_FTRACE_SUPPORT),y) +LOCAL_CFLAGS += -DCFG_FTRACE_SUPPORT +endif + +LOCAL_C_INCLUDES := $(LOCAL_PATH)/../libteec/include \ + $(LOCAL_PATH)/../libteec/include \ + $(LOCAL_PATH)/src + +LOCAL_SHARED_LIBRARIES := libteec + +LOCAL_MODULE := tee-supplicant +LOCAL_MODULE_TAGS := optional +LOCAL_VENDOR_MODULE := true +include $(BUILD_EXECUTABLE) diff --git a/optee_client/typedefs.checkpatch b/optee_client/typedefs.checkpatch new file mode 100644 index 0000000..84d846e --- /dev/null +++ b/optee_client/typedefs.checkpatch @@ -0,0 +1,80 @@ +# Note: please keep the entries in this file sorted in reverse alphabetical +# order (sort -r) +TEEC_Result +TEEC_Context +TEEC_UUID +TEEC_SharedMemory +TEEC_TempMemoryReference +TEEC_RegisteredMemoryReference +TEEC_Value +TEEC_Parameter +TEEC_Session +TEEC_Operation +CK_VOID_PTR_PTR +CK_VOID_PTR +CK_VERSION_PTR +CK_VERSION +CK_UTF8CHAR_PTR +CK_UTF8CHAR +CK_USER_TYPE +CK_UNLOCKMUTEX +CK_ULONG_PTR +CK_ULONG +CK_TOKEN_INFO_PTR +CK_TOKEN_INFO +CK_STATE +CK_SLOT_INFO_PTR +CK_SLOT_INFO +CK_SLOT_ID_PTR +CK_SLOT_ID +CK_SESSION_INFO_PTR +CK_SESSION_INFO +CK_SESSION_HANDLE_PTR +CK_SESSION_HANDLE +CK_RV +CK_OBJECT_HANDLE_PTR +CK_OBJECT_HANDLE +CK_OBJECT_CLASS_PTR +CK_OBJECT_CLASS +CK_NOTIFY +CK_NOTIFICATION +CK_MECHANISM_TYPE_PTR +CK_MECHANISM_TYPE +CK_MECHANISM_PTR +CK_MECHANISM_INFO_PTR +CK_MECHANISM_INFO +CK_MECHANISM +CK_MAC_GENERAL_PARAMS_PTR +CK_MAC_GENERAL_PARAMS +CK_LONG +CK_LOCKMUTEX +CK_KEY_TYPE_PTR +CK_KEY_TYPE +CK_INFO_PTR +CK_INFO +CK_GCM_PARAMS_PTR +CK_GCM_PARAMS +CK_FUNCTION_LIST_PTR_PTR +CK_FUNCTION_LIST_PTR +CK_FUNCTION_LIST +CK_FLAGS +CK_DESTROYMUTEX +CK_DATE_PTR +CK_DATE +CK_CREATEMUTEX +CK_C_INITIALIZE_ARGS_PTR +CK_C_INITIALIZE_ARGS +CK_CHAR_PTR +CK_CHAR +CK_CCM_PARAMS_PTR +CK_CCM_PARAMS +CK_BYTE_PTR +CK_BYTE +CK_BBOOL +CK_ATTRIBUTE_TYPE +CK_ATTRIBUTE_PTR +CK_ATTRIBUTE +CK_AES_CTR_PARAMS_PTR +CK_AES_CTR_PARAMS +CK_AES_CBC_ENCRYPT_DATA_PARAMS_PTR +CK_AES_CBC_ENCRYPT_DATA_PARAMS diff --git a/optee_os/.checkpatch.conf b/optee_os/.checkpatch.conf new file mode 100644 index 0000000..8d1e1d9 --- /dev/null +++ b/optee_os/.checkpatch.conf @@ -0,0 +1,13 @@ +--codespell +--no-tree +--quiet +--strict +--max-line-length=80 +--ignore=CAMELCASE +--ignore=CONCATENATED_STRING +--ignore=FILE_PATH_CHANGES +--ignore=GERRIT_CHANGE_ID +--ignore=NOT_UNIFIED_DIFF +--ignore=PREFER_KERNEL_TYPES +--ignore=USLEEP_RANGE +--kconfig-prefix=CFG_ diff --git a/optee_os/.clang-format b/optee_os/.clang-format new file mode 100644 index 0000000..4a78b29 --- /dev/null +++ b/optee_os/.clang-format @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# clang-format configuration file. Intended for clang-format >= 11. +# +# This is not perfect, but should help in most cases. +# Please note that the maintainers have the last word if there's +# disagrement with this tool. +# +# For more information, see: +# https://clang.llvm.org/docs/ClangFormat.html +# https://clang.llvm.org/docs/ClangFormatStyleOptions.html +# +--- +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: false +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeComma +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 8 +ContinuationIndentWidth: 8 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: false + +# Taken from: +# find lib core -name "*.h" -path '*/include/*' -print0 | xargs -0 grep -E -h \ +# '^#define[[:blank:]]*[^[:blank:]]*(FOREACH|for_each)[^[:blank:]]' | \ +# sed \ +# -e "s,^#define[ \t]*\([^[:space:]]*for_each[^[:space:]]*\)(.*$, - '\1'," \ +# -e "s,^#define[ \t]*\([^[:space:]]*FOREACH[^[:space:]]*\)(.*$, - '\1'," | \ +# LC_ALL=C sort -u +ForEachMacros: + - 'CIRCLEQ_FOREACH' + - 'CIRCLEQ_FOREACH_REVERSE' + - 'LIST_FOREACH' + - 'LIST_FOREACH_SAFE' + - 'SCATTERED_ARRAY_FOREACH' + - 'SIMPLEQ_FOREACH' + - 'SLIST_FOREACH' + - 'SLIST_FOREACH_SAFE' + - 'STAILQ_FOREACH' + - 'STAILQ_FOREACH_SAFE' + - 'TAILQ_FOREACH' + - 'TAILQ_FOREACH_REVERSE' + - 'TAILQ_FOREACH_REVERSE_SAFE' + - 'TAILQ_FOREACH_SAFE' + - 'fdt_for_each_property_offset' + - 'fdt_for_each_subnode' + - 'for_each_dt_driver' + - 'for_each_early_ta' + - 'for_each_fip_sp' + - 'for_each_secure_partition' + +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' +IndentCaseLabels: false +IndentGotoLabels: false +IndentPPDirectives: None +IndentWidth: 8 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 8 +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true + +# Taken from git's rules +PenaltyBreakAssignment: 10 +PenaltyBreakBeforeFirstCallParameter: 30 +PenaltyBreakComment: 10 +PenaltyBreakFirstLessLess: 0 +PenaltyBreakString: 10 +PenaltyExcessCharacter: 100 +PenaltyReturnTypeOnItsOwnLine: 60 + +PointerAlignment: Right +ReflowComments: false +SortIncludes: false +SortUsingDeclarations: false +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatementsExceptForEachMacros +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp03 +TabWidth: 8 +UseTab: Always +... diff --git a/optee_os/.gitattributes b/optee_os/.gitattributes new file mode 100644 index 0000000..62e73fc --- /dev/null +++ b/optee_os/.gitattributes @@ -0,0 +1,6 @@ +# For git archive +.gitignore export-ignore +.gitattributes export-ignore +.travis.yml export-ignore +.github export-ignore +.shippable.yml export-ignore diff --git a/optee_os/.github/issue_template.md b/optee_os/.github/issue_template.md new file mode 100644 index 0000000..0b6f432 --- /dev/null +++ b/optee_os/.github/issue_template.md @@ -0,0 +1,21 @@ + diff --git a/optee_os/.github/pull_request_template.md b/optee_os/.github/pull_request_template.md new file mode 100644 index 0000000..03cb067 --- /dev/null +++ b/optee_os/.github/pull_request_template.md @@ -0,0 +1,19 @@ + diff --git a/optee_os/.github/workflows/ci-cancel.yml b/optee_os/.github/workflows/ci-cancel.yml new file mode 100644 index 0000000..19119f5 --- /dev/null +++ b/optee_os/.github/workflows/ci-cancel.yml @@ -0,0 +1,22 @@ +# Cancel previous CI workflows that are still running when a new one is +# requested with the same ID. Happens when a branch is pushed to, +# including when a PR is updated. It would be wasteful to leave CI +# running on obsolete content. +# See https://github.com/marketplace/actions/cancel-workflow-action#advanced-pull-requests-from-forks +name: Cancel obsolete CI +on: + workflow_run: + workflows: [CI] + types: [requested] + branches-ignore: [master] +permissions: {} +jobs: + cancel: + permissions: + actions: write # to cancel/stop running workflows (styfle/cancel-workflow-action) + name: Cancel obsolete CI workflows + runs-on: ubuntu-latest + steps: + - uses: styfle/cancel-workflow-action@0.10.0 + with: + workflow_id: ${{ github.event.workflow.id }} diff --git a/optee_os/.github/workflows/ci.yml b/optee_os/.github/workflows/ci.yml new file mode 100644 index 0000000..ba0056e --- /dev/null +++ b/optee_os/.github/workflows/ci.yml @@ -0,0 +1,432 @@ +name: CI +on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + code_style: + name: Code style + runs-on: ubuntu-latest + container: jforissier/optee_os_ci + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 # full history so checkpatch can check commit IDs in commit messages + - name: Update Git config + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Run checkpatch + shell: bash + run: | + # checkpatch task + set -e + git config --global --add safe.directory /__w/optee_os/optee_os + pushd . >/dev/null + mkdir -p /tmp/linux/scripts + cd /tmp/linux/scripts + wget --quiet https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/plain/scripts/checkpatch.pl + chmod +x checkpatch.pl + wget --quiet https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/plain/scripts/spelling.txt + echo "invalid.struct.name" >const_structs.checkpatch + export PATH=/tmp/linux/scripts:$PATH + popd >/dev/null + source scripts/checkpatch_inc.sh + function _do() { echo '>>' $*; $*; } + # Run checkpatch.pl: + # - on the tip of the branch only if we're not in a pull request + # - otherwise: + # * on each commit in the development branch that is not in the target (merge to) branch + # * on the global diff if the PR contains more than one commit (useful to check if fixup + # commits do solve previous checkpatch errors) + if [ "${GITHUB_EVENT_NAME}" = "push" ]; then \ + _do checkpatch HEAD || failed=1; \ + else \ + for c in $(git rev-list HEAD^1..HEAD^2); do \ + _do checkpatch $c || failed=1; \ + done; \ + if [ "$(git rev-list --count HEAD^1..HEAD^2)" -gt 1 ]; then \ + _do checkdiff $(git rev-parse HEAD^1) $(git rev-parse HEAD^2) || failed=1; \ + fi; \ + fi + [ -z "$failed" ] + - name: Run pycodestyle + if: success() || failure() + run: | + # pycodestyle task + sudo -E bash -c "apt update -qq -y && apt install -qq -y pycodestyle" + pycodestyle scripts/*.py core/arch/arm/plat-stm32mp1/scripts/stm32image.py + builds: + name: make (multi-platform) + runs-on: ubuntu-latest + container: jforissier/optee_os_ci + steps: + - name: Restore build cache + uses: actions/cache@v3 + with: + path: /github/home/.cache/ccache + key: builds-cache-${{ github.sha }} + restore-keys: | + builds-cache- + - name: Checkout + uses: actions/checkout@v3 + - shell: bash + run: | + # build task + set -e -v + export LC_ALL=C + export PATH=/usr/local/bin:$PATH # clang + export CROSS_COMPILE32="ccache arm-linux-gnueabihf-" + export CROSS_COMPILE64="ccache aarch64-linux-gnu-" + export CFG_DEBUG_INFO=n + export CFG_WERROR=y + + function _make() { make -j$(nproc) -s O=out $*; } + function download_plug_and_trust() { mkdir -p $HOME/se050 && git clone --single-branch -b v0.4.2 https://github.com/foundriesio/plug-and-trust $HOME/se050/plug-and-trust || (rm -rf $HOME/se050 ; echo Nervermind); } + + function download_scp_firmware() { git clone --single-branch https://github.com/ARM-software/SCP-firmware.git $HOME/scp-firmware || echo Nervermind; } + + ccache -s -v + download_plug_and_trust + download_scp_firmware + + _make + _make COMPILER=clang + _make CFG_TEE_CORE_LOG_LEVEL=4 CFG_TEE_CORE_DEBUG=y CFG_TEE_TA_LOG_LEVEL=4 CFG_CC_OPT_LEVEL=0 CFG_DEBUG_INFO=y + _make CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_CORE_DEBUG=n CFG_TEE_TA_LOG_LEVEL=0 CFG_DEBUG_INFO=n CFG_ENABLE_EMBEDDED_TESTS=n + _make CFG_TEE_CORE_MALLOC_DEBUG=y CFG_CORE_DEBUG_CHECK_STACKS=y + _make CFG_CORE_SANITIZE_KADDRESS=y CFG_CORE_ASLR=n + _make CFG_LOCKDEP=y + _make CFG_CRYPTO=n + _make CFG_CRYPTO_{AES,DES}=n + _make CFG_CRYPTO_{DSA,RSA,DH}=n + _make CFG_CRYPTO_{DSA,RSA,DH,ECC}=n + _make CFG_CRYPTO_{H,C,CBC_}MAC=n + _make CFG_CRYPTO_{G,C}CM=n + _make CFG_CRYPTO_{MD5,SHA{1,224,256,384,512,512_256}}=n + _make CFG_WITH_PAGER=y out/core/tee{,-pager,-pageable}.bin + _make CFG_WITH_PAGER=y CFG_CRYPTOLIB_NAME=mbedtls CFG_CRYPTOLIB_DIR=lib/libmbedtls + _make CFG_WITH_PAGER=y CFG_WITH_LPAE=y + _make CFG_WITH_LPAE=y + _make CFG_CORE_PREALLOC_EL0_TBLS=y + _make CFG_RPMB_FS=y + _make CFG_RPMB_FS=y CFG_RPMB_TESTKEY=y + _make CFG_REE_FS=n CFG_RPMB_FS=y + _make CFG_WITH_PAGER=y CFG_WITH_LPAE=y CFG_RPMB_FS=y CFG_DT=y CFG_TEE_CORE_LOG_LEVEL=1 CFG_TEE_CORE_DEBUG=y CFG_CC_OPT_LEVEL=0 CFG_DEBUG_INFO=y + _make CFG_WITH_PAGER=y CFG_WITH_LPAE=y CFG_RPMB_FS=y CFG_DT=y CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_CORE_DEBUG=n DEBUG=0 + _make CFG_BUILT_IN_ARGS=y CFG_PAGEABLE_ADDR=0 CFG_NS_ENTRY_ADDR=0 CFG_DT_ADDR=0 CFG_DT=y + _make CFG_FTRACE_SUPPORT=y CFG_ULIBS_MCOUNT=y CFG_ULIBS_SHARED=y + _make CFG_TA_GPROF_SUPPORT=y CFG_FTRACE_SUPPORT=y CFG_SYSCALL_FTRACE=y CFG_ULIBS_MCOUNT=y + _make CFG_SECURE_DATA_PATH=y + _make CFG_REE_FS_TA_BUFFERED=y + _make CFG_WITH_USER_TA=n + _make CFG_{ATTESTATION,DEVICE_ENUM,RTC,SCMI,SECSTOR_TA_MGT}_PTA=y CFG_WITH_STATS=y CFG_TA_STATS=y + _make PLATFORM=vexpress-qemu_armv8a + _make PLATFORM=vexpress-qemu_armv8a COMPILER=clang + _make PLATFORM=vexpress-qemu_armv8a CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_CORE_DEBUG=n CFG_TEE_TA_LOG_LEVEL=0 CFG_DEBUG_INFO=n + _make PLATFORM=vexpress-qemu_armv8a CFG_TEE_CORE_LOG_LEVEL=4 CFG_TEE_CORE_DEBUG=y CFG_TEE_TA_LOG_LEVEL=4 CFG_CC_OPT_LEVEL=0 CFG_DEBUG_INFO=y + _make PLATFORM=vexpress-qemu_armv8a CFG_WITH_PAGER=y + _make PLATFORM=vexpress-qemu_armv8a CFG_FTRACE_SUPPORT=y CFG_ULIBS_MCOUNT=y CFG_ULIBS_SHARED=y + _make PLATFORM=vexpress-qemu_armv8a CFG_TA_GPROF_SUPPORT=y CFG_FTRACE_SUPPORT=y CFG_SYSCALL_FTRACE=y CFG_ULIBS_MCOUNT=y + _make PLATFORM=vexpress-qemu_armv8a CFG_NS_VIRTUALIZATION=y + _make PLATFORM=vexpress-qemu_armv8a CFG_CORE_PREALLOC_EL0_TBLS=y + _make PLATFORM=vexpress-qemu_armv8a CFG_CORE_SEL1_SPMC=y + _make PLATFORM=vexpress-qemu_armv8a CFG_CORE_SEL2_SPMC=y CFG_CORE_PHYS_RELOCATABLE=y CFG_TZDRAM_START=0x0d304000 CFG_TZDRAM_SIZE=0x00cfc000 + _make PLATFORM=vexpress-qemu_armv8a CFG_{ATTESTATION,DEVICE_ENUM,RTC,SCMI,SECSTOR_TA_MGT}_PTA=y CFG_WITH_STATS=y CFG_TA_STATS=y + _make PLATFORM=vexpress-qemu_armv8a CFG_CORE_SEL1_SPMC=y CFG_NS_VIRTUALIZATION=y + dd if=/dev/urandom of=BL32_AP_MM.fd bs=2621440 count=1 && _make PLATFORM=vexpress-qemu_armv8a CFG_STMM_PATH=BL32_AP_MM.fd CFG_RPMB_FS=y CFG_CORE_HEAP_SIZE=524288 CFG_TEE_RAM_VA_SIZE=0x00400000 + if [ -d $HOME/scp-firmware ]; then _make PLATFORM=vexpress-qemu_armv8a CFG_SCMI_SCPFW=y CFG_SCP_FIRMWARE=$HOME/scp-firmware; fi + _make PLATFORM=stm-b2260 + _make PLATFORM=stm-cannes + _make PLATFORM=stm32mp1 + _make PLATFORM=stm32mp1-135F_DK + if [ -d $HOME/scp-firmware ]; then _make PLATFORM=stm32mp1-157C_DK2 CFG_SCMI_SCPFW=y CFG_SCP_FIRMWARE=$HOME/scp-firmware; fi + _make PLATFORM=vexpress-fvp + _make PLATFORM=vexpress-fvp CFG_ARM64_core=y + _make PLATFORM=vexpress-fvp CFG_ARM64_core=y CFG_CORE_SEL1_SPMC=y CFG_SECURE_PARTITION=y + if [ -d $HOME/scp-firmware ]; then _make PLATFORM=vexpress-fvp CFG_SCMI_SCPFW=y CFG_SCP_FIRMWARE=$HOME/scp-firmware; fi + _make PLATFORM=vexpress-juno + _make PLATFORM=vexpress-juno CFG_ARM64_core=y + _make PLATFORM=hikey + _make PLATFORM=hikey CFG_ARM64_core=y + _make PLATFORM=mediatek-mt8173 + _make PLATFORM=mediatek-mt8175 + _make PLATFORM=mediatek-mt8183 + _make PLATFORM=mediatek-mt8516 + _make PLATFORM=imx-mx6ulevk + _make PLATFORM=imx-mx6ulevk CFG_NXP_CAAM=y CFG_CRYPTO_DRIVER=y + _make PLATFORM=imx-mx6ul9x9evk + _make PLATFORM=imx-mx6ullevk CFG_WITH_SOFTWARE_PRNG=n CFG_IMX_RNGB=y + if [ -d $HOME/se050/plug-and-trust ]; then _make PLATFORM=imx-mx6ullevk CFG_NXP_SE05X=y CFG_IMX_I2C=y CFG_STACK_{THREAD,TMP}_EXTRA=8192 CFG_CRYPTO_DRV_{CIPHER,ACIPHER}=y CFG_WITH_SOFTWARE_PRNG=n CFG_NXP_SE05X_{DIEID,RNG,RSA,ECC,CTR}_DRV=y CFG_NXP_SE05X_RSA_DRV_FALLBACK=y CFG_NXP_SE05X_ECC_DRV_FALLBACK=y CFG_NXP_SE05X_PLUG_AND_TRUST=$HOME/se050/plug-and-trust ; fi + _make PLATFORM=imx-mx6ulzevk + _make PLATFORM=imx-mx6slevk + _make PLATFORM=imx-mx6sllevk + _make PLATFORM=imx-mx6sxsabreauto + _make PLATFORM=imx-mx6sxsabresd + _make PLATFORM=imx-mx6sxsabresd CFG_NXP_CAAM=y CFG_CRYPTO_DRIVER=y + _make PLATFORM=imx-mx6solosabresd + _make PLATFORM=imx-mx6solosabreauto + _make PLATFORM=imx-mx6sxsabreauto + _make PLATFORM=imx-mx6qsabrelite + _make PLATFORM=imx-mx6qsabresd + _make PLATFORM=imx-mx6qsabresd CFG_RPMB_FS=y + _make PLATFORM=imx-mx6qsabreauto + _make PLATFORM=imx-mx6qsabreauto CFG_NXP_CAAM=y CFG_CRYPTO_DRIVER=y + _make PLATFORM=imx-mx6qpsabreauto + _make PLATFORM=imx-mx6qpsabresd + _make PLATFORM=imx-mx6dlsabresd + _make PLATFORM=imx-mx6dlsabreauto + _make PLATFORM=imx-mx6dapalis + _make PLATFORM=imx-mx6qapalis + _make PLATFORM=imx-mx7dsabresd + _make PLATFORM=imx-mx7dsabresd CFG_NXP_CAAM=y CFG_CRYPTO_DRIVER=y + _make PLATFORM=imx-mx7ulpevk + _make PLATFORM=imx-mx8mmevk + _make PLATFORM=imx-mx8mmevk CFG_NXP_CAAM=y CFG_CRYPTO_DRIVER=y + if [ -d $HOME/se050/plug-and-trust ]; then _make PLATFORM=imx-mx8mmevk CFG_NXP_CAAM=y CFG_NXP_CAAM_RNG_DRV=y CFG_NXP_SE05X=y CFG_IMX_I2C=y CFG_STACK_{THREAD,TMP}_EXTRA=8192 CFG_CRYPTO_DRV_{CIPHER,ACIPHER}=y CFG_NXP_SE05X_RNG_DRV=n CFG_WITH_SOFTWARE_PRNG=n CFG_NXP_SE05X_{DIEID,RSA,ECC,CTR}_DRV=y CFG_NXP_SE05X_RSA_DRV_FALLBACK=y CFG_NXP_SE05X_ECC_DRV_FALLBACK=y CFG_NXP_SE05X_PLUG_AND_TRUST=$HOME/se050/plug-and-trust ; fi + _make PLATFORM=imx-mx8mnevk + _make PLATFORM=imx-mx8mqevk + _make PLATFORM=imx-mx8mpevk + _make PLATFORM=imx-mx8qxpmek + _make PLATFORM=imx-mx8qmmek + _make PLATFORM=imx-mx8dxlevk + _make PLATFORM=imx-mx8ulpevk + _make PLATFORM=imx-mx8ulpevk CFG_NXP_CAAM=y CFG_CRYPTO_DRIVER=y + _make PLATFORM=imx-mx93evk + _make PLATFORM=k3-j721e + _make PLATFORM=k3-j721e CFG_ARM64_core=y + _make PLATFORM=k3-j784s4 + _make PLATFORM=k3-j784s4 CFG_ARM64_core=y + _make PLATFORM=k3-am65x + _make PLATFORM=k3-am65x CFG_ARM64_core=y + _make PLATFORM=k3-am64x + _make PLATFORM=k3-am64x CFG_ARM64_core=y + _make PLATFORM=k3-am62x + _make PLATFORM=k3-am62x CFG_ARM64_core=y + _make PLATFORM=ti-dra7xx out/core/tee{,-pager,-pageable}.bin + _make PLATFORM=ti-am57xx + _make PLATFORM=ti-am43xx + _make PLATFORM=sprd-sc9860 + _make PLATFORM=sprd-sc9860 CFG_ARM64_core=y + _make PLATFORM=ls-ls1043ardb + _make PLATFORM=ls-ls1046ardb + _make PLATFORM=ls-ls1012ardb + _make PLATFORM=ls-ls1028ardb + _make PLATFORM=ls-ls1088ardb + _make PLATFORM=ls-ls2088ardb + _make PLATFORM=ls-lx2160ardb + _make PLATFORM=ls-lx2160aqds + _make PLATFORM=zynq7k-zc702 + _make PLATFORM=zynqmp-zcu102 + _make PLATFORM=zynqmp-zcu102 CFG_ARM64_core=y + _make PLATFORM=zynqmp-zcu102 CFG_ARM64_core=y CFG_WITH_SOFTWARE_PRNG=n CFG_XIPHERA_TRNG=y CFG_ZYNQMP_HUK=y + _make PLATFORM=d02 + _make PLATFORM=d02 CFG_ARM64_core=y + _make PLATFORM=rcar + _make PLATFORM=rzg + _make PLATFORM=rzg CFG_ARM64_core=y + _make PLATFORM=rpi3 + _make PLATFORM=rpi3 CFG_ARM64_core=y + _make PLATFORM=hikey-hikey960 + _make PLATFORM=hikey-hikey960 COMPILER=clang + _make PLATFORM=hikey-hikey960 CFG_ARM64_core=y + _make PLATFORM=hikey-hikey960 CFG_ARM64_core=y COMPILER=clang + _make PLATFORM=hikey-hikey960 CFG_SECURE_DATA_PATH=n + _make PLATFORM=poplar + _make PLATFORM=poplar CFG_ARM64_core=y + _make PLATFORM=rockchip-rk322x + _make PLATFORM=rockchip-rk3399 + _make PLATFORM=sam + _make PLATFORM=sam-sama5d2_xplained + _make PLATFORM=sam-sama5d27_som1_ek + _make PLATFORM=sam-sama5d27_wlsom1_ek + _make PLATFORM=marvell-armada7k8k + _make PLATFORM=marvell-armada3700 + _make PLATFORM=marvell-otx2t96 + _make PLATFORM=marvell-otx2f95 + _make PLATFORM=marvell-otx2t98 + _make PLATFORM=marvell-cn10ka + _make PLATFORM=marvell-cn10kb + _make PLATFORM=marvell-cnf10ka + _make PLATFORM=marvell-cnf10kb + _make PLATFORM=synquacer + _make PLATFORM=sunxi-bpi_zero + _make PLATFORM=sunxi-sun50i_a64 + _make PLATFORM=bcm-ns3 CFG_ARM64_core=y + _make PLATFORM=hisilicon-hi3519av100_demo + _make PLATFORM=amlogic + _make PLATFORM=rzn1 + _make PLATFORM=versal CFG_VERSAL_FPGA_DDR_ADDR=0x40000000 + _make PLATFORM=corstone1000 + _make PLATFORM=nuvoton + + QEMUv8_check: + name: make check (QEMUv8) + runs-on: ubuntu-latest + container: jforissier/optee_os_ci:qemuv8_check2 + steps: + - name: Restore build cache + uses: actions/cache@v3 + with: + path: /github/home/.cache/ccache + key: qemuv8_check-cache-${{ github.sha }} + restore-keys: | + qemuv8_check-cache- + - name: Checkout + uses: actions/checkout@v3 + - shell: bash + run: | + # make check task + set -e -v + export LC_ALL=C + export BR2_CCACHE_DIR=/github/home/.cache/ccache + export CFG_TEE_CORE_LOG_LEVEL=0 + export CFG_ATTESTATION_PTA=y + export CFG_ATTESTATION_PTA_KEY_SIZE=1024 + WD=$(pwd) + cd .. + TOP=$(pwd)/optee_repo_qemu_v8 + /root/get_optee_qemuv8.sh ${TOP} + mv ${TOP}/optee_os ${TOP}/optee_os_old + ln -s ${WD} ${TOP}/optee_os + cd ${TOP}/build + + make -j$(nproc) check + make -j$(nproc) check CFG_CRYPTO_WITH_CE82=y + make -j$(nproc) check CFG_FTRACE_SUPPORT=y CFG_SYSCALL_FTRACE=y XTEST_ARGS=regression_1001 + make -j$(nproc) check CFG_PAN=y + + QEMUv8_Xen_check: + name: make check (QEMUv8, Xen) + runs-on: ubuntu-latest + container: jforissier/optee_os_ci:qemuv8_check2 + steps: + - name: Restore build cache + uses: actions/cache@v3 + with: + path: /github/home/.cache/ccache + key: qemuv8_xen_check-cache-${{ github.sha }} + restore-keys: | + qemuv8_xen_check-cache- + - name: Checkout + uses: actions/checkout@v3 + - shell: bash + run: | + # make check task + set -e -v + export LC_ALL=C + export CFG_TEE_CORE_LOG_LEVEL=0 + export BR2_CCACHE_DIR=/github/home/.cache/ccache + WD=$(pwd) + cd .. + TOP=$(pwd)/optee_repo_qemu_v8 + /root/get_optee_qemuv8.sh ${TOP} + mv ${TOP}/optee_os ${TOP}/optee_os_old + ln -s ${WD} ${TOP}/optee_os + cd ${TOP}/build + + make -j$(nproc) check XEN_BOOT=y + + QEMUv8_Hafnium_check: + name: make check (QEMUv8, Hafnium) + runs-on: ubuntu-latest + container: jforissier/optee_os_ci:qemuv8_check2 + steps: + - name: Remove /__t/* + run: rm -rf /__t/* + - name: Restore build cache + uses: actions/cache@v3 + with: + path: /github/home/.cache/ccache + key: qemuv8_hafnium_check-cache-${{ github.sha }} + restore-keys: | + qemuv8_hafnium_check-cache- + - name: Checkout + uses: actions/checkout@v3 + - shell: bash + run: | + # make check task + set -e -v + export LC_ALL=C + export CFG_TEE_CORE_LOG_LEVEL=0 + export BR2_CCACHE_DIR=/github/home/.cache/ccache + WD=$(pwd) + cd .. + TOP=$(pwd)/optee_repo_qemu_v8 + /root/get_optee_qemuv8.sh ${TOP} + mv ${TOP}/optee_os ${TOP}/optee_os_old + ln -s ${WD} ${TOP}/optee_os + cd ${TOP}/build + + make -j$(nproc) check SPMC_AT_EL=2 + + QEMUv8_check_BTI_MTE_PAC: + name: make check (QEMUv8, BTI+MTE+PAC) + runs-on: ubuntu-latest + container: jforissier/optee_os_ci:qemuv8_check2 + steps: + - name: Restore build cache + uses: actions/cache@v3 + with: + path: /github/home/.cache/ccache + key: qemuv8_check_bti_mte_pac-cache-${{ github.sha }} + restore-keys: | + qemuv8_check_bti_mte_pac-cache- + - name: Checkout + uses: actions/checkout@v3 + - shell: bash + run: | + # make check task + set -e -v + export LC_ALL=C + # The BTI-enabled toolchain is aarch64-unknown-linux-uclibc-gcc in /usr/local/bin + export PATH=/usr/local/bin:$PATH + export AARCH64_CROSS_COMPILE=aarch64-unknown-linux-uclibc- + export BR2_CCACHE_DIR=/github/home/.cache/ccache + export CFG_TEE_CORE_LOG_LEVEL=0 + export CFG_USER_TA_TARGETS=ta_arm64 + WD=$(pwd) + cd .. + TOP=$(pwd)/optee_repo_qemu_v8 + # TF-A v2.6 fails to build with the above toolchain so override it + export TF_A_EXPORTS="CROSS_COMPILE=${TOP}/toolchains/aarch64/bin/aarch64-linux-gnu-" + /root/get_optee_qemuv8.sh ${TOP} + # QEMU v7.2.0 has an issue with MTE + # https://github.com/OP-TEE/optee_os/issues/5759#issuecomment-1380590951 + cd ${TOP}/qemu && git fetch github && git checkout 13356edb87 + mv ${TOP}/optee_os ${TOP}/optee_os_old + ln -s ${WD} ${TOP}/optee_os + cd ${TOP}/build + + # xtest 1031 is excluded because 1031.4 (C++ exception from shared library) fails with this cross-compiler + make -j$(nproc) CFG_CORE_BTI=y CFG_TA_BTI=y MEMTAG=y PAUTH=y XTEST_ARGS="-x 1031" check + + QEMUv8_check_rust: + name: make check-rust (QEMUv8) + runs-on: ubuntu-latest + container: jforissier/optee_os_ci:qemuv8_check2 + steps: + - name: Remove /__t/* + run: rm -rf /__t/* + - name: Restore build cache + uses: actions/cache@v3 + with: + path: /github/home/.cache/ccache + key: qemuv8_check_rust-cache-${{ github.sha }} + restore-keys: | + qemuv8_check_rust-cache- + - name: Checkout + uses: actions/checkout@v3 + - shell: bash + run: | + # make check-rust task + set -e -v + export LC_ALL=C + export BR2_CCACHE_DIR=/github/home/.cache/ccache + WD=$(pwd) + cd .. + TOP=$(pwd)/optee_repo_qemu_v8 + /root/get_optee_qemuv8.sh ${TOP} + mv ${TOP}/optee_os ${TOP}/optee_os_old + ln -s ${WD} ${TOP}/optee_os + cd ${TOP}/build + + make -j$(nproc) OPTEE_RUST_ENABLE=y check-rust diff --git a/optee_os/.github/workflows/stales.yml b/optee_os/.github/workflows/stales.yml new file mode 100644 index 0000000..e0d47f3 --- /dev/null +++ b/optee_os/.github/workflows/stales.yml @@ -0,0 +1,27 @@ +name: 'Close stale issues and pull requests with no recent activity' +on: + schedule: + - cron: "15 00 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v4.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue has been marked as a stale issue because it has been open (more than) 30 days with no activity. Remove the stale label or add a comment, otherwise this issue will automatically be closed in 5 days. Note, that you can always re-open a closed issue at any time.' + stale-pr-message: 'This pull request has been marked as a stale pull request because it has been open (more than) 30 days with no activity. Remove the stale label or add a comment, otherwise this pull request will automatically be closed in 5 days. Note, that you can always re-open a closed issue at any time.' + stale-issue-label: Stale + stale-pr-label: Stale + exempt-issue-labels: bug,enhancement + exempt-pr-labels: bug,enhancement + days-before-stale: 30 + days-before-close: 5 + remove-stale-when-updated: true + remove-issue-stale-when-updated: true + remove-pr-stale-when-updated: true diff --git a/optee_os/.gitignore b/optee_os/.gitignore new file mode 100644 index 0000000..776f51c --- /dev/null +++ b/optee_os/.gitignore @@ -0,0 +1,7 @@ +.cproject +cscope.* +tags +TAGS +out +.project +*.swp diff --git a/optee_os/CHANGELOG.md b/optee_os/CHANGELOG.md new file mode 100644 index 0000000..44f5f99 --- /dev/null +++ b/optee_os/CHANGELOG.md @@ -0,0 +1,1735 @@ +# OP-TEE - version 4.0.0 (2023-10-20) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_4_0], [commits][OP_TEE_optee_os_commits_4_0] and [pull requests][OP_TEE_optee_os_pr_4_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_4_0], [commits][OP_TEE_optee_client_commits_4_0] and [pull requests][OP_TEE_optee_client_pr_4_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_4_0], [commits][OP_TEE_optee_test_commits_4_0] and [pull requests][OP_TEE_optee_test_pr_4_0] + - OP-TEE/build: [release page][OP_TEE_build_release_4_0], [commits][OP_TEE_build_commits_4_0] and [pull requests][OP_TEE_build_pr_4_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_4_0], [commits][linaro_swg_optee_examples_commits_4_0] and [pull requests][linaro_swg_optee_examples_pr_4_0] + + +[OP_TEE_optee_os_release_4_0]: https://github.com/OP-TEE/optee_os/releases/tag/4.0.0 +[OP_TEE_optee_os_commits_4_0]: https://github.com/OP-TEE/optee_os/compare/3.22.0...4.0.0 +[OP_TEE_optee_os_pr_4_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-10-20 + +[OP_TEE_optee_client_release_4_0]: https://github.com/OP-TEE/optee_client/releases/tag/4.0.0 +[OP_TEE_optee_client_commits_4_0]: https://github.com/OP-TEE/optee_client/compare/3.22.0...4.0.0 +[OP_TEE_optee_client_pr_4_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-10-20 + +[OP_TEE_optee_test_release_4_0]: https://github.com/OP-TEE/optee_test/releases/tag/4.0.0 +[OP_TEE_optee_test_commits_4_0]: https://github.com/OP-TEE/optee_test/compare/3.22.0...4.0.0 +[OP_TEE_optee_test_pr_4_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-10-20 + +[OP_TEE_build_release_4_0]: https://github.com/OP-TEE/build/releases/tag/4.0.0 +[OP_TEE_build_commits_4_0]: https://github.com/OP-TEE/build/compare/3.22.0...4.0.0 +[OP_TEE_build_pr_4_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-10-20 + +[linaro_swg_optee_examples_release_4_0]: https://github.com/linaro-swg/optee_examples/releases/tag/4.0.0 +[linaro_swg_optee_examples_commits_4_0]: https://github.com/linaro-swg/optee_examples/compare/3.22.0...4.0.0 +[linaro_swg_optee_examples_pr_4_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-10-20 + +# OP-TEE - version 3.22 (2023-07-07) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_22], [commits][OP_TEE_optee_os_commits_3_22] and [pull requests][OP_TEE_optee_os_pr_3_22] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_22], [commits][OP_TEE_optee_client_commits_3_22] and [pull requests][OP_TEE_optee_client_pr_3_22] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_22], [commits][OP_TEE_optee_test_commits_3_22] and [pull requests][OP_TEE_optee_test_pr_3_22] + - OP-TEE/build: [release page][OP_TEE_build_release_3_22], [commits][OP_TEE_build_commits_3_22] and [pull requests][OP_TEE_build_pr_3_22] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_22], [commits][linaro_swg_optee_examples_commits_3_22] and [pull requests][linaro_swg_optee_examples_pr_3_22] + + +[OP_TEE_optee_os_release_3_22]: https://github.com/OP-TEE/optee_os/releases/tag/3.22 +[OP_TEE_optee_os_commits_3_22]: https://github.com/OP-TEE/optee_os/compare/3.21...3.22 +[OP_TEE_optee_os_pr_3_22]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-07-07 + +[OP_TEE_optee_client_release_3_22]: https://github.com/OP-TEE/optee_client/releases/tag/3.22 +[OP_TEE_optee_client_commits_3_22]: https://github.com/OP-TEE/optee_client/compare/3.21...3.22 +[OP_TEE_optee_client_pr_3_22]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-07-07 + +[OP_TEE_optee_test_release_3_22]: https://github.com/OP-TEE/optee_test/releases/tag/3.22 +[OP_TEE_optee_test_commits_3_22]: https://github.com/OP-TEE/optee_test/compare/3.21...3.22 +[OP_TEE_optee_test_pr_3_22]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-07-07 + +[OP_TEE_build_release_3_22]: https://github.com/OP-TEE/build/releases/tag/3.22 +[OP_TEE_build_commits_3_22]: https://github.com/OP-TEE/build/compare/3.21...3.22 +[OP_TEE_build_pr_3_22]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-07-07 + +[linaro_swg_optee_examples_release_3_22]: https://github.com/linaro-swg/optee_examples/releases/tag/3.22 +[linaro_swg_optee_examples_commits_3_22]: https://github.com/linaro-swg/optee_examples/compare/3.21...3.22 +[linaro_swg_optee_examples_pr_3_22]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A..2023-07-07 + +# OP-TEE - version 3.21.0 (2023-04-14) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_21], [commits][OP_TEE_optee_os_commits_3_21] and [pull requests][OP_TEE_optee_os_pr_3_21] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_21], [commits][OP_TEE_optee_client_commits_3_21] and [pull requests][OP_TEE_optee_client_pr_3_21] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_21], [commits][OP_TEE_optee_test_commits_3_21] and [pull requests][OP_TEE_optee_test_pr_3_21] + - OP-TEE/build: [release page][OP_TEE_build_release_3_21], [commits][OP_TEE_build_commits_3_21] and [pull requests][OP_TEE_build_pr_3_21] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_21], [commits][linaro_swg_optee_examples_commits_3_21] and [pull requests][linaro_swg_optee_examples_pr_3_21] + + +[OP_TEE_optee_os_release_3_21]: https://github.com/OP-TEE/optee_os/releases/tag/3.21.0 +[OP_TEE_optee_os_commits_3_21]: https://github.com/OP-TEE/optee_os/compare/3.20.0...3.21.0 +[OP_TEE_optee_os_pr_3_21]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2023-01-20..2023-04-14 + +[OP_TEE_optee_client_release_3_21]: https://github.com/OP-TEE/optee_client/releases/tag/3.21.0 +[OP_TEE_optee_client_commits_3_21]: https://github.com/OP-TEE/optee_client/compare/3.20.0...3.21.0 +[OP_TEE_optee_client_pr_3_21]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2023-01-20..2023-04-14 + +[OP_TEE_optee_test_release_3_21]: https://github.com/OP-TEE/optee_test/releases/tag/3.21.0 +[OP_TEE_optee_test_commits_3_21]: https://github.com/OP-TEE/optee_test/compare/3.20.0...3.21.0 +[OP_TEE_optee_test_pr_3_21]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2023-01-20..2023-04-14 + +[OP_TEE_build_release_3_21]: https://github.com/OP-TEE/build/releases/tag/3.21.0 +[OP_TEE_build_commits_3_21]: https://github.com/OP-TEE/build/compare/3.20.0...3.21.0 +[OP_TEE_build_pr_3_21]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2023-01-20..2023-04-14 + +[linaro_swg_optee_examples_release_3_21]: https://github.com/linaro-swg/optee_examples/releases/tag/3.21.0 +[linaro_swg_optee_examples_commits_3_21]: https://github.com/linaro-swg/optee_examples/compare/3.20.0...3.21.0 +[linaro_swg_optee_examples_pr_3_21]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2023-01-20..2023-04-14 + +# OP-TEE - version 3.20.0 (2023-01-20) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_20], [commits][OP_TEE_optee_os_commits_3_20] and [pull requests][OP_TEE_optee_os_pr_3_20] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_20], [commits][OP_TEE_optee_client_commits_3_20] and [pull requests][OP_TEE_optee_client_pr_3_20] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_20], [commits][OP_TEE_optee_test_commits_3_20] and [pull requests][OP_TEE_optee_test_pr_3_20] + - OP-TEE/build: [release page][OP_TEE_build_release_3_20], [commits][OP_TEE_build_commits_3_20] and [pull requests][OP_TEE_build_pr_3_20] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_20], [commits][linaro_swg_optee_examples_commits_3_20] and [pull requests][linaro_swg_optee_examples_pr_3_20] + + +[OP_TEE_optee_os_release_3_20]: https://github.com/OP-TEE/optee_os/releases/tag/3.20.0 +[OP_TEE_optee_os_commits_3_20]: https://github.com/OP-TEE/optee_os/compare/3.19.0...3.20.0 +[OP_TEE_optee_os_pr_3_20]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-10-14..2023-01-20 + +[OP_TEE_optee_client_release_3_20]: https://github.com/OP-TEE/optee_client/releases/tag/3.20.0 +[OP_TEE_optee_client_commits_3_20]: https://github.com/OP-TEE/optee_client/compare/3.19.0...3.20.0 +[OP_TEE_optee_client_pr_3_20]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-10-14..2023-01-20 + +[OP_TEE_optee_test_release_3_20]: https://github.com/OP-TEE/optee_test/releases/tag/3.20.0 +[OP_TEE_optee_test_commits_3_20]: https://github.com/OP-TEE/optee_test/compare/3.19.0...3.20.0 +[OP_TEE_optee_test_pr_3_20]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-10-14..2023-01-20 + +[OP_TEE_build_release_3_20]: https://github.com/OP-TEE/build/releases/tag/3.20.0 +[OP_TEE_build_commits_3_20]: https://github.com/OP-TEE/build/compare/3.19.0...3.20.0 +[OP_TEE_build_pr_3_20]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-10-14..2023-01-20 + +[linaro_swg_optee_examples_release_3_20]: https://github.com/linaro-swg/optee_examples/releases/tag/3.20.0 +[linaro_swg_optee_examples_commits_3_20]: https://github.com/linaro-swg/optee_examples/compare/3.19.0...3.20.0 +[linaro_swg_optee_examples_pr_3_20]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-10-14..2023-01-20 + +# OP-TEE - version 3.19.0 (2022-10-14) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_19], [commits][OP_TEE_optee_os_commits_3_19] and [ +pull requests][OP_TEE_optee_os_pr_3_19] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_19], [commits][OP_TEE_optee_client_commits_3_19] and [pull requests][OP_TEE_optee_client_pr_3_19] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_19], [commits][OP_TEE_optee_test_commits_3_19] and [pull requests][OP_TEE_optee_test_pr_3_19] + - OP-TEE/build: [release page][OP_TEE_build_release_3_19], [commits][OP_TEE_build_commits_3_19] and [pull requests][OP_TEE_build_pr_3_19] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_19], [commits][linaro_swg_optee_examples_commits_3_19] and [pull requests][linaro_swg_optee_examples_pr_3_19] + + +[OP_TEE_optee_os_release_3_19]: https://github.com/OP-TEE/optee_os/releases/tag/3.19.0 +[OP_TEE_optee_os_commits_3_19]: https://github.com/OP-TEE/optee_os/compare/3.18.0...3.19.0 +[OP_TEE_optee_os_pr_3_19]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-07-15..2022-10-14 + +[OP_TEE_optee_client_release_3_19]: https://github.com/OP-TEE/optee_client/releases/tag/3.19.0 +[OP_TEE_optee_client_commits_3_19]: https://github.com/OP-TEE/optee_client/compare/3.18.0...3.19.0 +[OP_TEE_optee_client_pr_3_19]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-07-15..2022-10-14 + +[OP_TEE_optee_test_release_3_19]: https://github.com/OP-TEE/optee_test/releases/tag/3.19.0 +[OP_TEE_optee_test_commits_3_19]: https://github.com/OP-TEE/optee_test/compare/3.18.0...3.19.0 +[OP_TEE_optee_test_pr_3_19]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-07-15..2022-10-14 + +[OP_TEE_build_release_3_19]: https://github.com/OP-TEE/build/releases/tag/3.19.0 +[OP_TEE_build_commits_3_19]: https://github.com/OP-TEE/build/compare/3.18.0...3.19.0 +[OP_TEE_build_pr_3_19]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-07-15..2022-10-14 + +[linaro_swg_optee_examples_release_3_19]: https://github.com/linaro-swg/optee_examples/releases/tag/3.19.0 +[linaro_swg_optee_examples_commits_3_19]: https://github.com/linaro-swg/optee_examples/compare/3.18.0...3.19.0 +[linaro_swg_optee_examples_pr_3_19]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-07-15..2022-10-14 + +# OP-TEE - version 3.18.0 (2022-07-15) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_18_0], [commits][OP_TEE_optee_os_commits_3_18_0] and [pull requests][OP_TEE_optee_os_pr_3_18_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_18_0], [commits][OP_TEE_optee_client_commits_3_18_0] and [pull requests][OP_TEE_optee_client_pr_3_18_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_18_0], [commits][OP_TEE_optee_test_commits_3_18_0] and [pull requests][OP_TEE_optee_test_pr_3_18_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_18_0], [commits][OP_TEE_build_commits_3_18_0] and [pull requests][OP_TEE_build_pr_3_18_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_18_0], [commits][linaro_swg_optee_examples_commits_3_18_0] and [pull requests][linaro_swg_optee_examples_pr_3_18_0] + + +[OP_TEE_optee_os_release_3_18_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.18.0 +[OP_TEE_optee_os_commits_3_18_0]: https://github.com/OP-TEE/optee_os/compare/3.17.0...3.18.0 +[OP_TEE_optee_os_pr_3_18_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-04-15..2022-07-15 + +[OP_TEE_optee_client_release_3_18_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.18.0 +[OP_TEE_optee_client_commits_3_18_0]: https://github.com/OP-TEE/optee_client/compare/3.17.0...3.18.0 +[OP_TEE_optee_client_pr_3_18_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-04-15..2022-07-15 + +[OP_TEE_optee_test_release_3_18_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.18.0 +[OP_TEE_optee_test_commits_3_18_0]: https://github.com/OP-TEE/optee_test/compare/3.17.0...3.18.0 +[OP_TEE_optee_test_pr_3_18_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-04-15..2022-07-15 + +[OP_TEE_build_release_3_18_0]: https://github.com/OP-TEE/build/releases/tag/3.18.0 +[OP_TEE_build_commits_3_18_0]: https://github.com/OP-TEE/build/compare/3.17.0...3.18.0 +[OP_TEE_build_pr_3_18_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-04-15..2022-07-15 + +[linaro_swg_optee_examples_release_3_18_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.18.0 +[linaro_swg_optee_examples_commits_3_18_0]: https://github.com/linaro-swg/optee_examples/compare/3.17.0...3.18.0 +[linaro_swg_optee_examples_pr_3_18_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-04-15..2022-07-15 + +# OP-TEE - version 3.17.0 (2022-04-15) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_17_0], [commits][OP_TEE_optee_os_commits_3_17_0] and [pull requests][OP_TEE_optee_os_pr_3_17_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_17_0], [commits][OP_TEE_optee_client_commits_3_17_0] and [pull requests][OP_TEE_optee_client_pr_3_17_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_17_0], [commits][OP_TEE_optee_test_commits_3_17_0] and [pull requests][OP_TEE_optee_test_pr_3_17_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_17_0], [commits][OP_TEE_build_commits_3_17_0] and [pull requests][OP_TEE_build_pr_3_17_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_17_0], [commits][linaro_swg_optee_examples_commits_3_17_0] and [pull requests][linaro_swg_optee_examples_pr_3_17_0] + + +[OP_TEE_optee_os_release_3_17_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.17.0 +[OP_TEE_optee_os_commits_3_17_0]: https://github.com/OP-TEE/optee_os/compare/3.16.0...3.17.0 +[OP_TEE_optee_os_pr_3_17_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-01-28..2022-04-15 + +[OP_TEE_optee_client_release_3_17_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.17.0 +[OP_TEE_optee_client_commits_3_17_0]: https://github.com/OP-TEE/optee_client/compare/3.16.0...3.17.0 +[OP_TEE_optee_client_pr_3_17_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-01-28..2022-04-15 + +[OP_TEE_optee_test_release_3_17_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.17.0 +[OP_TEE_optee_test_commits_3_17_0]: https://github.com/OP-TEE/optee_test/compare/3.16.0...3.17.0 +[OP_TEE_optee_test_pr_3_17_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-01-28..2022-04-15 + +[OP_TEE_build_release_3_17_0]: https://github.com/OP-TEE/build/releases/tag/3.17.0 +[OP_TEE_build_commits_3_17_0]: https://github.com/OP-TEE/build/compare/3.16.0...3.17.0 +[OP_TEE_build_pr_3_17_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-01-28..2022-04-15 + +[linaro_swg_optee_examples_release_3_17_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.17.0 +[linaro_swg_optee_examples_commits_3_17_0]: https://github.com/linaro-swg/optee_examples/compare/3.16.0...3.17.0 +[linaro_swg_optee_examples_pr_3_17_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2022-01-28..2022-04-15 + +# OP-TEE - version 3.16.0 (2022-01-28) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_16_0], [commits][OP_TEE_optee_os_commits_3_16_0] and [pull requests][OP_TEE_optee_os_pr_3_16_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_16_0], [commits][OP_TEE_optee_client_commits_3_16_0] and [pull requests][OP_TEE_optee_client_pr_3_16_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_16_0], [commits][OP_TEE_optee_test_commits_3_16_0] and [pull requests][OP_TEE_optee_test_pr_3_16_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_16_0], [commits][OP_TEE_build_commits_3_16_0] and [pull requests][OP_TEE_build_pr_3_16_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_16_0], [commits][linaro_swg_optee_examples_commits_3_16_0] and [pull requests][linaro_swg_optee_examples_pr_3_16_0] + + +[OP_TEE_optee_os_release_3_16_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.16.0 +[OP_TEE_optee_os_commits_3_16_0]: https://github.com/OP-TEE/optee_os/compare/3.15.0...3.16.0 +[OP_TEE_optee_os_pr_3_16_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-10-18..2022-01-28 + +[OP_TEE_optee_client_release_3_16_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.16.0 +[OP_TEE_optee_client_commits_3_16_0]: https://github.com/OP-TEE/optee_client/compare/3.15.0...3.16.0 +[OP_TEE_optee_client_pr_3_16_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-10-18..2022-01-28 + +[OP_TEE_optee_test_release_3_16_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.16.0 +[OP_TEE_optee_test_commits_3_16_0]: https://github.com/OP-TEE/optee_test/compare/3.15.0...3.16.0 +[OP_TEE_optee_test_pr_3_16_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-10-18..2022-01-28 + +[OP_TEE_build_release_3_16_0]: https://github.com/OP-TEE/build/releases/tag/3.16.0 +[OP_TEE_build_commits_3_16_0]: https://github.com/OP-TEE/build/compare/3.15.0...3.16.0 +[OP_TEE_build_pr_3_16_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-10-18..2022-01-28 + +[linaro_swg_optee_examples_release_3_16_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.16.0 +[linaro_swg_optee_examples_commits_3_16_0]: https://github.com/linaro-swg/optee_examples/compare/3.15.0...3.16.0 +[linaro_swg_optee_examples_pr_3_16_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-10-18..2022-01-28 + +# OP-TEE - version 3.15.0 (2021-10-18) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_15_0], [commits][OP_TEE_optee_os_commits_3_15_0] and [pull requests][OP_TEE_optee_os_pr_3_15_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_15_0], [commits][OP_TEE_optee_client_commits_3_15_0] and [pull requests][OP_TEE_optee_client_pr_3_15_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_15_0], [commits][OP_TEE_optee_test_commits_3_15_0] and [pull requests][OP_TEE_optee_test_pr_3_15_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_15_0], [commits][OP_TEE_build_commits_3_15_0] and [pull requests][OP_TEE_build_pr_3_15_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_15_0], [commits][linaro_swg_optee_examples_commits_3_15_0] and [pull requests][linaro_swg_optee_examples_pr_3_15_0] + + +[OP_TEE_optee_os_release_3_15_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.15.0 +[OP_TEE_optee_os_commits_3_15_0]: https://github.com/OP-TEE/optee_os/compare/3.14.0...3.15.0 +[OP_TEE_optee_os_pr_3_15_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-07-16..2021-10-18 + +[OP_TEE_optee_client_release_3_15_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.15.0 +[OP_TEE_optee_client_commits_3_15_0]: https://github.com/OP-TEE/optee_client/compare/3.14.0...3.15.0 +[OP_TEE_optee_client_pr_3_15_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-07-16..2021-10-18 + +[OP_TEE_optee_test_release_3_15_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.15.0 +[OP_TEE_optee_test_commits_3_15_0]: https://github.com/OP-TEE/optee_test/compare/3.14.0...3.15.0 +[OP_TEE_optee_test_pr_3_15_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-07-16..2021-10-18 + +[OP_TEE_build_release_3_15_0]: https://github.com/OP-TEE/build/releases/tag/3.15.0 +[OP_TEE_build_commits_3_15_0]: https://github.com/OP-TEE/build/compare/3.14.0...3.15.0 +[OP_TEE_build_pr_3_15_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-07-16..2021-10-18 + +[linaro_swg_optee_examples_release_3_15_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.15.0 +[linaro_swg_optee_examples_commits_3_15_0]: https://github.com/linaro-swg/optee_examples/compare/3.14.0...3.15.0 +[linaro_swg_optee_examples_pr_3_15_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-07-16..2021-10-18 + +# OP-TEE - version 3.14.0 (target date: 2021-07-16) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_14_0], [commits][OP_TEE_optee_os_commits_3_14_0] and [pull requests][OP_TEE_optee_os_pr_3_14_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_14_0], [commits][OP_TEE_optee_client_commits_3_14_0] and [pull requests][OP_TEE_optee_client_pr_3_14_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_14_0], [commits][OP_TEE_optee_test_commits_3_14_0] and [pull requests][OP_TEE_optee_test_pr_3_14_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_14_0], [commits][OP_TEE_build_commits_3_14_0] and [pull requests][OP_TEE_build_pr_3_14_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_14_0], [commits][linaro_swg_optee_examples_commits_3_14_0] and [pull requests][linaro_swg_optee_examples_pr_3_14_0] + + +[OP_TEE_optee_os_release_3_14_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.14.0 +[OP_TEE_optee_os_commits_3_14_0]: https://github.com/OP-TEE/optee_os/compare/3.13.0...3.14.0 +[OP_TEE_optee_os_pr_3_14_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-05-01..2021-07-16 + +[OP_TEE_optee_client_release_3_14_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.14.0 +[OP_TEE_optee_client_commits_3_14_0]: https://github.com/OP-TEE/optee_client/compare/3.13.0...3.14.0 +[OP_TEE_optee_client_pr_3_14_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-05-01-..2021-07-16 + +[OP_TEE_optee_test_release_3_14_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.14.0 +[OP_TEE_optee_test_commits_3_14_0]: https://github.com/OP-TEE/optee_test/compare/3.13.0...3.14.0 +[OP_TEE_optee_test_pr_3_14_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-05-01..2021-07-16 + +[OP_TEE_build_release_3_14_0]: https://github.com/OP-TEE/build/releases/tag/3.14.0 +[OP_TEE_build_commits_3_14_0]: https://github.com/OP-TEE/build/compare/3.13.0...3.14.0 +[OP_TEE_build_pr_3_14_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-05-01..2021-07-16 + +[linaro_swg_optee_examples_release_3_14_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.14.0 +[linaro_swg_optee_examples_commits_3_14_0]: https://github.com/linaro-swg/optee_examples/compare/3.13.0...3.14.0 +[linaro_swg_optee_examples_pr_3_14_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-05-01..2021-07-16 + +# OP-TEE - version 3.13.0 (2021-04-30) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_13_0], [commits][OP_TEE_optee_os_commits_3_13_0] and [pull requests][OP_TEE_optee_os_pr_3_13_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_13_0], [commits][OP_TEE_optee_client_commits_3_13_0] and [pull requests][OP_TEE_optee_client_pr_3_13_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_13_0], [commits][OP_TEE_optee_test_commits_3_13_0] and [pull requests][OP_TEE_optee_test_pr_3_13_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_13_0], [commits][OP_TEE_build_commits_3_13_0] and [pull requests][OP_TEE_build_pr_3_13_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_13_0], [commits][linaro_swg_optee_examples_commits_3_13_0] and [pull requests][linaro_swg_optee_examples_pr_3_13_0] + + +[OP_TEE_optee_os_release_3_13_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.13.0 +[OP_TEE_optee_os_commits_3_13_0]: https://github.com/OP-TEE/optee_os/compare/3.12.0...3.13.0 +[OP_TEE_optee_os_pr_3_13_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-01-20..2021-04-30 + +[OP_TEE_optee_client_release_3_13_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.13.0 +[OP_TEE_optee_client_commits_3_13_0]: https://github.com/OP-TEE/optee_client/compare/3.12.0...3.13.0 +[OP_TEE_optee_client_pr_3_13_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-01-20..2021-04-30 + +[OP_TEE_optee_test_release_3_13_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.13.0 +[OP_TEE_optee_test_commits_3_13_0]: https://github.com/OP-TEE/optee_test/compare/3.12.0...3.13.0 +[OP_TEE_optee_test_pr_3_13_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-01-20..2021-04-30 + +[OP_TEE_build_release_3_13_0]: https://github.com/OP-TEE/build/releases/tag/3.13.0 +[OP_TEE_build_commits_3_13_0]: https://github.com/OP-TEE/build/compare/3.12.0...3.13.0 +[OP_TEE_build_pr_3_13_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-01-20..2021-04-30 + +[linaro_swg_optee_examples_release_3_13_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.13.0 +[linaro_swg_optee_examples_commits_3_13_0]: https://github.com/linaro-swg/optee_examples/compare/3.12.0...3.13.0 +[linaro_swg_optee_examples_pr_3_13_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2021-01-20..2021-04-30 + +# OP-TEE - version 3.12.0 (2021-01-20) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_12_0], [commits][OP_TEE_optee_os_commits_3_12_0] and [pull requests][OP_TEE_optee_os_pr_3_12_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_12_0], [commits][OP_TEE_optee_client_commits_3_12_0] and [pull requests][OP_TEE_optee_client_pr_3_12_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_12_0], [commits][OP_TEE_optee_test_commits_3_12_0] and [pull requests][OP_TEE_optee_test_pr_3_12_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_12_0], [commits][OP_TEE_build_commits_3_12_0] and [pull requests][OP_TEE_build_pr_3_12_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_12_0], [commits][linaro_swg_optee_examples_commits_3_12_0] and [pull requests][linaro_swg_optee_examples_pr_3_12_0] + + +[OP_TEE_optee_os_release_3_12_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.12.0 +[OP_TEE_optee_os_commits_3_12_0]: https://github.com/OP-TEE/optee_os/compare/3.11.0...3.12.0 +[OP_TEE_optee_os_pr_3_12_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-10-16..2021-01-20 + +[OP_TEE_optee_client_release_3_12_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.12.0 +[OP_TEE_optee_client_commits_3_12_0]: https://github.com/OP-TEE/optee_client/compare/3.11.0...3.12.0 +[OP_TEE_optee_client_pr_3_12_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-10-16..2021-01-20 + +[OP_TEE_optee_test_release_3_12_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.12.0 +[OP_TEE_optee_test_commits_3_12_0]: https://github.com/OP-TEE/optee_test/compare/3.11.0...3.12.0 +[OP_TEE_optee_test_pr_3_12_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-10-16..2021-01-20 + +[OP_TEE_build_release_3_12_0]: https://github.com/OP-TEE/build/releases/tag/3.12.0 +[OP_TEE_build_commits_3_12_0]: https://github.com/OP-TEE/build/compare/3.11.0...3.12.0 +[OP_TEE_build_pr_3_12_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-10-16..2021-01-20 + +[linaro_swg_optee_examples_release_3_12_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.12.0 +[linaro_swg_optee_examples_commits_3_12_0]: https://github.com/linaro-swg/optee_examples/compare/3.11.0...3.12.0 +[linaro_swg_optee_examples_pr_3_12_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-10-16..2021-01-20 + +# OP-TEE - version 3.11.0 (2020-10-16) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_11_0], [commits][OP_TEE_optee_os_commits_3_11_0] and [pull requests][OP_TEE_optee_os_pr_3_11_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_11_0], [commits][OP_TEE_optee_client_commits_3_11_0] and [pull requests][OP_TEE_optee_client_pr_3_11_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_11_0], [commits][OP_TEE_optee_test_commits_3_11_0] and [pull requests][OP_TEE_optee_test_pr_3_11_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_11_0], [commits][OP_TEE_build_commits_3_11_0] and [pull requests][OP_TEE_build_pr_3_11_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_11_0], [commits][linaro_swg_optee_examples_commits_3_11_0] and [pull requests][linaro_swg_optee_examples_pr_3_11_0] + + +[OP_TEE_optee_os_release_3_11_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.11.0 +[OP_TEE_optee_os_commits_3_11_0]: https://github.com/OP-TEE/optee_os/compare/3.10.0...3.11.0 +[OP_TEE_optee_os_pr_3_11_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-08-21..2020-10-16 + +[OP_TEE_optee_client_release_3_11_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.11.0 +[OP_TEE_optee_client_commits_3_11_0]: https://github.com/OP-TEE/optee_client/compare/3.10.0...3.11.0 +[OP_TEE_optee_client_pr_3_11_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-08-21..2020-10-16 + +[OP_TEE_optee_test_release_3_11_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.11.0 +[OP_TEE_optee_test_commits_3_11_0]: https://github.com/OP-TEE/optee_test/compare/3.10.0...3.11.0 +[OP_TEE_optee_test_pr_3_11_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-08-21..2020-10-16 + +[OP_TEE_build_release_3_11_0]: https://github.com/OP-TEE/build/releases/tag/3.11.0 +[OP_TEE_build_commits_3_11_0]: https://github.com/OP-TEE/build/compare/3.10.0...3.11.0 +[OP_TEE_build_pr_3_11_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-08-21..2020-10-16 + +[linaro_swg_optee_examples_release_3_11_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.11.0 +[linaro_swg_optee_examples_commits_3_11_0]: https://github.com/linaro-swg/optee_examples/compare/3.10.0...3.11.0 +[linaro_swg_optee_examples_pr_3_11_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-08-21..2020-10-16 + +# OP-TEE - version 3.10.0 (2020-08-21) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_10_0], [commits][OP_TEE_optee_os_commits_3_10_0] and [pull requests][OP_TEE_optee_os_pr_3_10_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_10_0], [commits][OP_TEE_optee_client_commits_3_10_0] and [pull requests][OP_TEE_optee_client_pr_3_10_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_10_0], [commits][OP_TEE_optee_test_commits_3_10_0] and [pull requests][OP_TEE_optee_test_pr_3_10_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_10_0], [commits][OP_TEE_build_commits_3_10_0] and [pull requests][OP_TEE_build_pr_3_10_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_10_0], [commits][linaro_swg_optee_examples_commits_3_10_0] and [pull requests][linaro_swg_optee_examples_pr_3_10_0] + + +[OP_TEE_optee_os_release_3_10_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.10.0 +[OP_TEE_optee_os_commits_3_10_0]: https://github.com/OP-TEE/optee_os/compare/3.9.0...3.10.0 +[OP_TEE_optee_os_pr_3_10_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-04-22..2020-08-21 + +[OP_TEE_optee_client_release_3_10_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.10.0 +[OP_TEE_optee_client_commits_3_10_0]: https://github.com/OP-TEE/optee_client/compare/3.9.0...3.10.0 +[OP_TEE_optee_client_pr_3_10_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-04-22..2020-08-21 + +[OP_TEE_optee_test_release_3_10_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.10.0 +[OP_TEE_optee_test_commits_3_10_0]: https://github.com/OP-TEE/optee_test/compare/3.9.0...3.10.0 +[OP_TEE_optee_test_pr_3_10_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-04-22..2020-08-21 + +[OP_TEE_build_release_3_10_0]: https://github.com/OP-TEE/build/releases/tag/3.10.0 +[OP_TEE_build_commits_3_10_0]: https://github.com/OP-TEE/build/compare/3.9.0...3.10.0 +[OP_TEE_build_pr_3_10_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-04-22..2020-08-21 + +[linaro_swg_optee_examples_release_3_10_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.10.0 +[linaro_swg_optee_examples_commits_3_10_0]: https://github.com/linaro-swg/optee_examples/compare/3.9.0...3.10.0 +[linaro_swg_optee_examples_pr_3_10_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-04-22..2020-08-21 + +# OP-TEE - version 3.9.0 (2020-04-22) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_9_0], [commits][OP_TEE_optee_os_commits_3_9_0] and [pull requests][OP_TEE_optee_os_pr_3_9_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_9_0], [commits][OP_TEE_optee_client_commits_3_9_0] and [pull requests][OP_TEE_optee_client_pr_3_9_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_9_0], [commits][OP_TEE_optee_test_commits_3_9_0] and [pull requests][OP_TEE_optee_test_pr_3_9_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_9_0], [commits][OP_TEE_build_commits_3_9_0] and [pull requests][OP_TEE_build_pr_3_9_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_9_0], [commits][linaro_swg_optee_examples_commits_3_9_0] and [pull requests][linaro_swg_optee_examples_pr_3_9_0] + + +[OP_TEE_optee_os_release_3_9_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.9.0 +[OP_TEE_optee_os_commits_3_9_0]: https://github.com/OP-TEE/optee_os/compare/3.8.0...3.9.0 +[OP_TEE_optee_os_pr_3_9_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-01-24..2020-05-22 + +[OP_TEE_optee_client_release_3_9_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.9.0 +[OP_TEE_optee_client_commits_3_9_0]: https://github.com/OP-TEE/optee_client/compare/3.8.0...3.9.0 +[OP_TEE_optee_client_pr_3_9_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-01-24..2020-05-22 + +[OP_TEE_optee_test_release_3_9_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.9.0 +[OP_TEE_optee_test_commits_3_9_0]: https://github.com/OP-TEE/optee_test/compare/3.8.0...3.9.0 +[OP_TEE_optee_test_pr_3_9_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-01-24..2020-05-22 + +[OP_TEE_build_release_3_9_0]: https://github.com/OP-TEE/build/releases/tag/3.9.0 +[OP_TEE_build_commits_3_9_0]: https://github.com/OP-TEE/build/compare/3.8.0...3.9.0 +[OP_TEE_build_pr_3_9_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-01-24..2020-05-22 + +[linaro_swg_optee_examples_release_3_9_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.9.0 +[linaro_swg_optee_examples_commits_3_9_0]: https://github.com/linaro-swg/optee_examples/compare/3.8.0...3.9.0 +[linaro_swg_optee_examples_pr_3_9_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-01-24..2020-05-22 + +# OP-TEE - version 3.8.0 (2020-01-24) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_8_0], [commits][OP_TEE_optee_os_commits_3_8_0] and [pull requests][OP_TEE_optee_os_pr_3_8_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_8_0], [commits][OP_TEE_optee_client_commits_3_8_0] and [pull requests][OP_TEE_optee_client_pr_3_8_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_8_0], [commits][OP_TEE_optee_test_commits_3_8_0] and [pull requests][OP_TEE_optee_test_pr_3_8_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_8_0], [commits][OP_TEE_build_commits_3_8_0] and [pull requests][OP_TEE_build_pr_3_8_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_8_0], [commits][linaro_swg_optee_examples_commits_3_8_0] and [pull requests][linaro_swg_optee_examples_pr_3_8_0] + + +[OP_TEE_optee_os_release_3_8_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.8.0 +[OP_TEE_optee_os_commits_3_8_0]: https://github.com/OP-TEE/optee_os/compare/3.7.0...3.8.0 +[OP_TEE_optee_os_pr_3_8_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2020-01-24..2020-01-24 + +[OP_TEE_optee_client_release_3_8_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.8.0 +[OP_TEE_optee_client_commits_3_8_0]: https://github.com/OP-TEE/optee_client/compare/3.7.0...3.8.0 +[OP_TEE_optee_client_pr_3_8_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2020-01-24 + +[OP_TEE_optee_test_release_3_8_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.8.0 +[OP_TEE_optee_test_commits_3_8_0]: https://github.com/OP-TEE/optee_test/compare/3.7.0...3.8.0 +[OP_TEE_optee_test_pr_3_8_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2020-01-24 + +[OP_TEE_build_release_3_8_0]: https://github.com/OP-TEE/build/releases/tag/3.8.0 +[OP_TEE_build_commits_3_8_0]: https://github.com/OP-TEE/build/compare/3.7.0...3.8.0 +[OP_TEE_build_pr_3_8_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2020-01-24 + +[linaro_swg_optee_examples_release_3_8_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.8.0 +[linaro_swg_optee_examples_commits_3_8_0]: https://github.com/linaro-swg/optee_examples/compare/3.7.0...3.8.0 +[linaro_swg_optee_examples_pr_3_8_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2020-01-24 + +# OP-TEE - version 3.7.0 (2019-10-18) + +- Links to the release pages, commits and pull requests merged into this release for: + - OP-TEE/optee_os: [release page][OP_TEE_optee_os_release_3_7_0], [commits][OP_TEE_optee_os_commits_3_7_0] and [pull requests][OP_TEE_optee_os_pr_3_7_0] + - OP-TEE/optee_client: [release page][OP_TEE_optee_client_release_3_7_0], [commits][OP_TEE_optee_client_commits_3_7_0] and [pull requests][OP_TEE_optee_client_pr_3_7_0] + - OP-TEE/optee_test: [release page][OP_TEE_optee_test_release_3_7_0], [commits][OP_TEE_optee_test_commits_3_7_0] and [pull requests][OP_TEE_optee_test_pr_3_7_0] + - OP-TEE/build: [release page][OP_TEE_build_release_3_7_0], [commits][OP_TEE_build_commits_3_7_0] and [pull requests][OP_TEE_build_pr_3_7_0] + - linaro-swg/optee_examples: [release page][linaro_swg_optee_examples_release_3_7_0], [commits][linaro_swg_optee_examples_commits_3_7_0] and [pull requests][linaro_swg_optee_examples_pr_3_7_0] + + +[OP_TEE_optee_os_release_3_7_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.7.0 +[OP_TEE_optee_os_commits_3_7_0]: https://github.com/OP-TEE/optee_os/compare/3.6.0...3.7.0 +[OP_TEE_optee_os_pr_3_7_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2019-10-18 + +[OP_TEE_optee_client_release_3_7_0]: https://github.com/OP-TEE/optee_client/releases/tag/3.7.0 +[OP_TEE_optee_client_commits_3_7_0]: https://github.com/OP-TEE/optee_client/compare/3.6.0...3.7.0 +[OP_TEE_optee_client_pr_3_7_0]: https://github.com/OP-TEE/optee_client/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2019-10-18 + +[OP_TEE_optee_test_release_3_7_0]: https://github.com/OP-TEE/optee_test/releases/tag/3.7.0 +[OP_TEE_optee_test_commits_3_7_0]: https://github.com/OP-TEE/optee_test/compare/3.6.0...3.7.0 +[OP_TEE_optee_test_pr_3_7_0]: https://github.com/OP-TEE/optee_test/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2019-10-18 + +[OP_TEE_build_release_3_7_0]: https://github.com/OP-TEE/build/releases/tag/3.7.0 +[OP_TEE_build_commits_3_7_0]: https://github.com/OP-TEE/build/compare/3.6.0...3.7.0 +[OP_TEE_build_pr_3_7_0]: https://github.com/OP-TEE/build/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2019-10-18 + +[linaro_swg_optee_examples_release_3_7_0]: https://github.com/linaro-swg/optee_examples/releases/tag/3.7.0 +[linaro_swg_optee_examples_commits_3_7_0]: https://github.com/linaro-swg/optee_examples/compare/3.6.0...3.7.0 +[linaro_swg_optee_examples_pr_3_7_0]: https://github.com/linaro-swg/optee_examples/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-07-05..2019-10-18 + +# OP-TEE - version 3.6.0 (2019-07-05) + +- Link to the GitHub [release page][github_release_3_6_0]. +- Links to the [commits][github_commits_3_6_0] and +[pull requests][github_pr_3_6_0] merged into this release. + +[github_release_3_6_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.6.0 +[github_commits_3_6_0]: https://github.com/OP-TEE/optee_os/compare/3.5.0...3.6.0 +[github_pr_3_6_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-04-26..2019-07-05 + +# OP-TEE - version 3.5.0 (2019-04-26) + +- Link to the GitHub [release page][github_release_3_5_0]. +- Links to the [commits][github_commits_3_5_0] and +[pull requests][github_pr_3_5_0] merged into this release. + +[github_release_3_5_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.5.0 +[github_commits_3_5_0]: https://github.com/OP-TEE/optee_os/compare/3.4.0...3.5.0 +[github_pr_3_5_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2019-01-26..2019-04-26 + +# OP-TEE - version 3.4.0 (2019-01-25) + +- Link to the GitHub [release page][github_release_3_4_0]. +- Links to the [commits][github_commits_3_4_0] and +[pull requests][github_pr_3_4_0] merged into this release. + +[github_release_3_4_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.4.0 +[github_commits_3_4_0]: https://github.com/OP-TEE/optee_os/compare/3.3.0...3.4.0 +[github_pr_3_4_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2018-10-12..2019-01-25 + +# OP-TEE - version 3.3.0 (2018-10-12) + +- Link to the GitHub [release page][github_release_3_3_0]. +- Links to the [commits][github_commits_3_3_0] and +[pull requests][github_pr_3_3_0] merged into this release. + +[github_release_3_3_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.3.0 +[github_commits_3_3_0]: https://github.com/OP-TEE/optee_os/compare/3.2.0...3.3.0 +[github_pr_3_3_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2018-07-04..2018-10-12 + +# OP-TEE - version 3.2.0 (2018-07-04) + +- Link to the GitHub [release page][github_release_3_2_0]. +- Links to the [commits][github_commits_3_2_0] and +[pull requests][github_pr_3_2_0] merged into this release. + +## Known issues + +* HiKey960: assertion in bget create_free_block() \[memalign() self-test\] ([#2414]) +* D02: assertion '!have_spinlock()' ([#2437]) + +[github_commits_3_2_0]: https://github.com/OP-TEE/optee_os/compare/3.1.0...3.2.0 +[github_pr_3_2_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2018-04-13..2018-07-04 +[github_release_3_2_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.2.0 +[#2414]: https://github.com/OP-TEE/optee_os/issues/2414 +[#2437]: https://github.com/OP-TEE/optee_os/issues/2437 + +# OP-TEE - version 3.1.0 (2018-04-13) + +- Link to the GitHub [release page][github_release_3_1_0]. +- Links to the [commits][github_commits_3_1_0] and +[pull requests][github_pr_3_1_0] merged into this release. + +[github_commits_3_1_0]: https://github.com/OP-TEE/optee_os/compare/3.0.0...3.1.0 +[github_pr_3_1_0]: https://github.com/OP-TEE/optee_os/pulls?q=is%3Apr+is%3Amerged+base%3Amaster+merged%3A2018-01-26..2018-04-13 +[github_release_3_1_0]: https://github.com/OP-TEE/optee_os/releases/tag/3.1.0 + +# OP-TEE - version 3.0.0 (2018-01-26) + +[Link][github_commits_3_0_0] to a list of all commits between this release and +the previous one (2.6.0). + +About backwards compatibility: Trusted Applications built with OP-TEE 2.5.0 or +earlier will not run properly with a *debug* build of this release due +to commit [0e1c6e8e][commit_0e1c6e8e] ("Dump call stack on TA panic"). +Non-debug builds are not affected. + +## New features + +* New supported platforms: Armada 3700 ([#1946]), Poplar ([#1999]), 64-bit + support for FSL ls1012ardb ([#1941]), i.MX6SX Sabreauto ([#1974]). +* arm32: sm: init CNTVOFF ([#2052]) +* Debug/info/error traces: make output more compact ([#2011]) +* tzc380: implement new functions ([#1994]) +* Secure Data Path: add pseudo-TA to convert VA to PA (#1993]) +* Pager: use NEON AES GCM implementation ([#1959]) +* Crypto: add optimized AES GCM implementation using NEON ([#1949]) +* Add support for using secure storage for TA anti-rollback ([#1928]) +* Crypto: replace struct crypto_ops with function interface ([#1923], + [#1931]) +* aosp_optee.mk: define OPTEE_BIN ([#1922]) +* Add build option to allow concurrent execution of single-instance TAs + ([#1915]) +* Pager: support for address sanitizer ([#1856]) +* Pager: make memory between CFG_TEE_RAM_START and TEE load address usable by + pager ([#1826]) + +## Bug fixes + +* Fix crash in tee_mmu_final() on TA loading error ([#2092]) +* LibTomCrypt: fix issue causing invalid output when using AES CTR with +hardware acceleration (CFG_CRYPTO_WITH_CE) ([#2086]) +* pl310: fix cache sync ([#2035]) +* tzc380: do not write reserved bits ([#1994]) +* Fix potential double free in ta_open() ([#1970]) +* libfdt: fix undefined behaviour in fdt_offset_ptr() ([#1969]) +* imx_wdog: fix register access ([#1966]) +* Secure storage: fix potential memory leak after early return ([#1961]) +* LibTomCrypt: fix double free in dsa_import() ([#1963]) +* RPMB: fix TA independance issue in secure storage ([#1921]) +* RPMB: return TEE_ERROR_ACCESS_CONFLICT instead of panicking when a TA + attempts to create an existing persistent object without the overwrite flag + ([#1919]) +* PSCI: pass non-secure context to psci_system_suspend() ([#1916]) +* Fix "Argument list too long" during "make clean" ([#1897]) + +## Security fixes + +* Mitigations and hardening against the Spectre and Meltdown vulnerabilities + (CVE-2017-5753, CVE-2017-5715, CVE-2017-5754). + +## Known issues + +* Secure storage (REE FS): storage size not updated after +TEE_TruncateObjectData() ([#2094]) +* Possible deadlock with CFG_WITH_PAGER=y when loading a TA and not enough +page tables are available in pgt_cache ([#2080]) + +## Tested on + +The release was tested successfully on the platforms listed below. + + +* d02 +* hikey +* hikey-hikey960 +* imx-mx6ulevk +* imx-mx7dsabresd +* marvell-armada7k8k +* marvell-armada3700 +* mediatek-mt8173 +* rcar-salvator_m3 +* rockchip-rk322x +* rpi3 +* sam +* ti +* vexpress-juno +* vexpress-qemu_armv8a +* vexpress-qemu_virt + +[commit_0e1c6e8e]: https://github.com/OP-TEE/optee_os/commit/0e1c6e8e +[github_commits_3_0_0]: https://github.com/OP-TEE/optee_os/compare/2.6.0...3.0.0 +[#2092]: https://github.com/OP-TEE/optee_os/pull/2092 +[#2086]: https://github.com/OP-TEE/optee_os/pull/2086 +[#2094]: https://github.com/OP-TEE/optee_os/issues/2094 +[#2080]: https://github.com/OP-TEE/optee_os/issues/2080 +[#2052]: https://github.com/OP-TEE/optee_os/pull/2052 +[#2035]: https://github.com/OP-TEE/optee_os/pull/2035 +[#2011]: https://github.com/OP-TEE/optee_os/pull/2011 +[#1999]: https://github.com/OP-TEE/optee_os/pull/1999 +[#1994]: https://github.com/OP-TEE/optee_os/pull/1994 +[#1993]: https://github.com/OP-TEE/optee_os/pull/1993 +[#1974]: https://github.com/OP-TEE/optee_os/pull/1974 +[#1970]: https://github.com/OP-TEE/optee_os/pull/1970 +[#1969]: https://github.com/OP-TEE/optee_os/pull/1969 +[#1966]: https://github.com/OP-TEE/optee_os/pull/1966 +[#1963]: https://github.com/OP-TEE/optee_os/pull/1963 +[#1961]: https://github.com/OP-TEE/optee_os/pull/1961 +[#1959]: https://github.com/OP-TEE/optee_os/pull/1959 +[#1949]: https://github.com/OP-TEE/optee_os/pull/1949 +[#1946]: https://github.com/OP-TEE/optee_os/pull/1946 +[#1941]: https://github.com/OP-TEE/optee_os/pull/1941 +[#1931]: https://github.com/OP-TEE/optee_os/pull/1931 +[#1928]: https://github.com/OP-TEE/optee_os/pull/1928 +[#1923]: https://github.com/OP-TEE/optee_os/pull/1923 +[#1922]: https://github.com/OP-TEE/optee_os/pull/1922 +[#1921]: https://github.com/OP-TEE/optee_os/pull/1921 +[#1919]: https://github.com/OP-TEE/optee_os/pull/1919 +[#1916]: https://github.com/OP-TEE/optee_os/pull/1916 +[#1915]: https://github.com/OP-TEE/optee_os/pull/1915 +[#1897]: https://github.com/OP-TEE/optee_os/pull/1897 +[#1856]: https://github.com/OP-TEE/optee_os/pull/1856 +[#1826]: https://github.com/OP-TEE/optee_os/pull/1826 + +# OP-TEE - version 2.6.0 + +[Link][github_commits_2_6_0] to a list of all commits between this release and +the previous one (2.5.0). + +## New features + +* New supported platforms: Atmel SAMA5 ([#1714]), HiSilicon HiKey960 ([#1684]), + Rockchip RK322X ([#1666]), NXP LS1043A-RDB/LS1046A-RDB ([#1787]), Marvell + Armada 70x0/80x0 ([#1807]). +* Dynamic shared memory (non-contiguous, non-secure memory can be mapped into + Trusted Applications VA space) ([#1631]) +* Dump TA call stack on panic ([#1858]) +* i.MX: PSCI reset ([#1849]) +* plat-ti: AM43xx: suspend/resume support ([#1822]) +* QEMU SMP support ([#1820]) +* plat-ti: AM43xx: disable TRNG ([#1816]) +* plat-ti: enable Secure Data Path by default ([#1815]) +* Improve symbolize.py ([#1778], [#1767], [#1766]) +* Early TAs (TAs linked in tee.bin) ([#1733]) +* Suspend/resume framework for arm32 and imx7d support ([#1729]) +* RK322X PSCI version, features and suspend support ([#1720]) +* arm32: handle aborts in system mode ([#1703]) +* i.MX: add SNVS SRTC support ([#1700]) +* GCC7 support ([#1693]) +* Improve detection of programming errors in locking code ([#1671], [#1670]) +* Support TEE RAM size larger than page directory size ([#1669]) + +## Removed features + +* Remove TUI code ([#1842]) + +## Bug fixes + +* Add missing synchronization barrier in core_mmu_map_pages() ([#1827]) +* Secure storage: REE FS: fix bug in error path ([#1801]) +* ASAN bug fixes ([#1799]) +* Fix race in core_mmu_user_mapping_is_active() ([#1785]) +* libutee: printf() and puts() fixes, add putchar() ([#1759], [#1754]) +* arm32: GICv3: fix FIQ masking in IRQ/ABT/SVC/UND handlers ([#1748]) +* arm32: preserve r12 in native_intr_handler() ([#1682]) +* arm64: fix print_kernel_stack() ([#1664]) +* benchmark: fix core data-abort ([#1658]) + +## Security fixes or enhancements + +* crypto: fix software PRNG weaknesses + ([OP-TEE-2017-0001][OP-TEE-2017-0001]) ([#1843]) + +## Tested on + +The release was tested successfully on the platforms listed below. +If a platform is not listed, it means the release was not tested on this +platform. + + +* d02 +* hikey +* hikey-hikey960 +* imx-mx6ulevk +* imx-mx7dsabresd +* ls-ls1021a??? (single core) +* ls-ls1043ardb +* ls-ls1046ardb +* mediatek-mt8173 +* rcar +* rockchip-rk322x +* rpi3 +* sam +* stm-b2260 +* stm-cannes +* ti-??? +* vexpress-fvp +* vexpress-juno +* vexpress-qemu_armv8a +* vexpress-qemu_virt + +[github_commits_2_6_0]: https://github.com/OP-TEE/optee_os/compare/2.5.0...2.6.0 +[#1858]: https://github.com/OP-TEE/optee_os/issues/1858 +[#1849]: https://github.com/OP-TEE/optee_os/issues/1849 +[#1843]: https://github.com/OP-TEE/optee_os/issues/1843 +[#1842]: https://github.com/OP-TEE/optee_os/issues/1842 +[#1827]: https://github.com/OP-TEE/optee_os/issues/1827 +[#1822]: https://github.com/OP-TEE/optee_os/issues/1822 +[#1820]: https://github.com/OP-TEE/optee_os/issues/1820 +[#1816]: https://github.com/OP-TEE/optee_os/issues/1816 +[#1815]: https://github.com/OP-TEE/optee_os/issues/1815 +[#1807]: https://github.com/OP-TEE/optee_os/issues/1807 +[#1801]: https://github.com/OP-TEE/optee_os/issues/1801 +[#1799]: https://github.com/OP-TEE/optee_os/issues/1799 +[#1787]: https://github.com/OP-TEE/optee_os/issues/1787 +[#1785]: https://github.com/OP-TEE/optee_os/issues/1785 +[#1778]: https://github.com/OP-TEE/optee_os/issues/1778 +[#1767]: https://github.com/OP-TEE/optee_os/issues/1767 +[#1766]: https://github.com/OP-TEE/optee_os/issues/1766 +[#1759]: https://github.com/OP-TEE/optee_os/issues/1759 +[#1754]: https://github.com/OP-TEE/optee_os/issues/1754 +[#1748]: https://github.com/OP-TEE/optee_os/issues/1748 +[#1733]: https://github.com/OP-TEE/optee_os/issues/1733 +[#1729]: https://github.com/OP-TEE/optee_os/issues/1729 +[#1720]: https://github.com/OP-TEE/optee_os/issues/1720 +[#1714]: https://github.com/OP-TEE/optee_os/issues/1714 +[#1703]: https://github.com/OP-TEE/optee_os/issues/1703 +[#1700]: https://github.com/OP-TEE/optee_os/issues/1700 +[#1693]: https://github.com/OP-TEE/optee_os/issues/1693 +[#1684]: https://github.com/OP-TEE/optee_os/issues/1684 +[#1682]: https://github.com/OP-TEE/optee_os/issues/1682 +[#1671]: https://github.com/OP-TEE/optee_os/issues/1671 +[#1670]: https://github.com/OP-TEE/optee_os/issues/1670 +[#1669]: https://github.com/OP-TEE/optee_os/issues/1669 +[#1666]: https://github.com/OP-TEE/optee_os/issues/1666 +[#1664]: https://github.com/OP-TEE/optee_os/issues/1664 +[#1658]: https://github.com/OP-TEE/optee_os/issues/1658 +[#1631]: https://github.com/OP-TEE/optee_os/issues/1631 +[OP-TEE-2017-0001]: https://www.op-tee.org/security-advisories/ + +# OP-TEE - version 2.5.0 + +[Link][github_commits_2_5_0] to a list of all commits between this release and +the previous one (2.4.0). + +## New features + +* New supported platform: i.MX7D ([#1639]) +* Secure storage: anti-rollback protection for REE FS using RPMB FS ([#1630]) +* Assign non-secure DDR configuration from DT if CFG_DT=y ([#1623]) +* Add new image format: split image into three separate binaries suitable for + upcoming ARM Trusted Firmware ([#1589]). +* Make alignment check configurable ([#1586]) +* drivers: add TZC380 driver ([#1578]) +* plat-imx: PSCI CPU off ([#1577]) +* 64-bit paging on QEMU v8 and HiKey ([#1575], [#1592]) +* Benchmark framework ([#1365]) +* Dump call stack of user TAs on abort ([#1552]) +* plat-hikey: enable Secure Data Path ([#1440]) +* Add interface to load and decrypt/authenticate user TAs ([#1513]) +* plat-ti: add secure paging support ([#1493]) +* plat-ti: add OTP hardware key support ([#1492]) +* Support ARM GICv3 ([#1465]) + +## Removed features + +* stm-orly2 is not supported anymore ([#1650]) +* Remove secure storage based on SQL FS (`CFG_SQL_FS=y`) ([#1490]) +* Remove support for mapping user TAs with 1 MiB or 2 MiB granularity + (`CFG_SMALL_PAGE_USER_TA=n`) ([#1559]). TAs are always mapped using small + pages. + +## Bug fixes + +* Reduce size of non-pageable code ([#1621]) +* Ignore `TA_FLAG_MULTI_SESSION` and `TA_FLAG_INSTANCE_KEEP_ALIVE` when + `TA_FLAG_SINGLE_INSTANCE` is not set ([#1574]) +* libutee: remove buffering for AES GCM (PR#1573) and AES CTR ([#1580]) +* Fix ROUNDUP()/ROUNDDOWN() macros ([#1519]) +* Do not touch other bits in GICD_CTLR ([#1508]) +* Fix build issue with `DEBUG=y` and `CFG_TEE_CORE_LOG_LEVEL=0` ([#1502]) +* crypto: do not restrict hash size when algorithm is ECDSA ([#1497]) + +## Security fixes or enhancements + +- crypto: fix RSA key leakage after fault injection attack + ([OP-TEE-2016-0003][OP-TEE-2016-0003]) ([#1610]) +* crypto: fix RSA key leakage after side channel attack + ([OP-TEE-2016-0002][OP-TEE-2016-0002]) ([#1610]) +* Make pager aliased pages not always writable ([#1551]) +* Support for no-exec RO and RW data ([#1459], [#1550]) + +## New issues + +* armv7: some platform-specific code (`plat_cpu_reset_early()`) overwrites +SCTLR bits configured by generic code. This affects alignment checks (`SCTLR.A`) +and write-implies-no-exec (`SCTLR.WXN`, `SCTLR.UWXN`), which can therefore not +be configured via the compile-time `CFG_` variables. +* armv7: plat-imx: Cortex-A9 cores should enable branch prediction (`SCLTR.Z`) +for improved performance. +* [#1656] qemu_armv8a: init hangs when secure data path and pager are both + enabled. + +## Tested on + +In the list below, _standard_ means that the `xtest` program passed with +its default configuration, while _extended_ means it was run successfully +with the additional GlobalPlatform™ TEE Initial Configuration Test Suite +v1.1.0.4. + +If a platform is not listed, it means the release was not tested on this +platform. + + +* d02: extended +* hikey: extended +* imx-mx6ulevk: standard +* imx-mx6ullevk: standard +* imx-mx7dsabresd: standard +* ls-ls1021atwr: standard +* mediatek-mt8173: standard +* rcar-h3: standard +* rpi3: standard +* stm-b2260: extended +* stm-cannes: extended +* ti-am43xx: standard +* ti-am57xx: standard +* ti-dra7xx: standard +* vexpress-fvp: standard +* vexpress-juno: standard +* vexpress-qemu_armv8a: standard +* vexpress-qemu_virt: standard + +[github_commits_2_5_0]: https://github.com/OP-TEE/optee_os/compare/2.4.0...2.5.0-rc1 +[#1656]: https://github.com/OP-TEE/optee_os/issues/1656 +[#1650]: https://github.com/OP-TEE/optee_os/pull/1650 +[#1639]: https://github.com/OP-TEE/optee_os/pull/1639 +[#1630]: https://github.com/OP-TEE/optee_os/pull/1630 +[#1623]: https://github.com/OP-TEE/optee_os/pull/1623 +[#1621]: https://github.com/OP-TEE/optee_os/pull/1621 +[#1610]: https://github.com/OP-TEE/optee_os/pull/1610 +[#1592]: https://github.com/OP-TEE/optee_os/pull/1592 +[#1589]: https://github.com/OP-TEE/optee_os/pull/1589 +[#1586]: https://github.com/OP-TEE/optee_os/pull/1586 +[#1580]: https://github.com/OP-TEE/optee_os/pull/1580 +[#1578]: https://github.com/OP-TEE/optee_os/pull/1578 +[#1577]: https://github.com/OP-TEE/optee_os/pull/1577 +[#1574]: https://github.com/OP-TEE/optee_os/pull/1574 +[#1559]: https://github.com/OP-TEE/optee_os/pull/1559 +[#1551]: https://github.com/OP-TEE/optee_os/pull/1551 +[#1550]: https://github.com/OP-TEE/optee_os/pull/1550 +[#1519]: https://github.com/OP-TEE/optee_os/pull/1519 +[#1502]: https://github.com/OP-TEE/optee_os/pull/1502 +[#1365]: https://github.com/OP-TEE/optee_os/pull/1365 +[#1552]: https://github.com/OP-TEE/optee_os/pull/1552 +[#1513]: https://github.com/OP-TEE/optee_os/pull/1513 +[#1508]: https://github.com/OP-TEE/optee_os/pull/1508 +[#1493]: https://github.com/OP-TEE/optee_os/pull/1493 +[#1497]: https://github.com/OP-TEE/optee_os/pull/1497 +[#1492]: https://github.com/OP-TEE/optee_os/pull/1492 +[#1490]: https://github.com/OP-TEE/optee_os/pull/1490 +[#1465]: https://github.com/OP-TEE/optee_os/pull/1465 +[#1459]: https://github.com/OP-TEE/optee_os/pull/1459 +[#1440]: https://github.com/OP-TEE/optee_os/pull/1440 +[OP-TEE-2016-0003]: https://www.op-tee.org/security-advisories/ +[OP-TEE-2016-0002]: https://www.op-tee.org/security-advisories/ + +# OP-TEE - version 2.4.0 + +[Link][github_commits_2_4_0] to a list of all commits between this release and +the previous one (2.3.0). + +Please note: this release is API-compatible with the previous one, but the +Secure Storage internal format for the REE and SQL FS is not compatible due to +commits [a238b74][commit_a238b74] ("core: REE FS: use the new hash tree +interface") and [44e900e][commit_44e900e] ("core: SQL FS: use the new hash tree +interface"). + +## New features + +* Add porting guidelines + +* Add support for Secure Data Path which allows Client and Trusted Applications + to share references to secure memory + +* New supported platform: Texas Instruments AM57xx (`PLATFORM=ti-am57xx`) + +* ARMv7-A: add support for platform services in secure monitor and add these + services for the DRA7xx platform + +* SPI framework and PL022 driver cleanup and improvements + +* Use CNTPCT (when available) to add entropy to the software PRNG + +* Add GlobalPlatform Socket API for UDP and TCP (IPv4 and IPv6) + +* DRA7: add TRNG driver, enable GICv2 driver + +* Support load address larger than 4G + +* libutee: preserve error code when calling TEE_Panic() for easier + troubleshooting + +* Support TA profiling with gprof (-pg compiler switch) + +* Optimize the ELF loader for TAs when pager is enabled + +* Update documentation + +* Add paged secure shared memory that can be transferred between TAs as + needed + +* Introduce MOBJ abstraction + +* i.MX6: add PSCI "on" function + +* arm32: introduce PSCI framework + +## Bug fixes + +* Secure storage: improve integrity checking of the REE and SQL filesystems by + adding a hash tree on the internal data structures. Any external modification + is detected, except full rollback. Fixes [#1188][issue1188]. + +* The linux driver will set the 'privileged' flag (TEE_GEN_CAP_PRIVILEGED) on + the device intended for use by tee-supplicant. Fixes [#1199][issue1199]. + +* RPMB: don't try to program the RPMB key by default + +* Fix "make clean" error cases + +* Fix issue when resetting persistent storage enumerator [#1332][issue1332] + +* Fix TA panic when doing AES CTS with specific buffer sizes + [#1203][issue1203]. + +## Known issues + +* On RPi3 xtest sometimes stall (rcu_sched self-detected stall on CPU) [#1353][issue1353] +* For multi-core PSCI support is to be added for ls1021atwr in OP-TEE. +* USB keyboard cannot be used to stop the u-boot timeout ([build issue131]). +* Travis service (build.git) seems unstable from time to time. + +## Tested on + +In the list below, _standard_ means that the `xtest` program passed with +its default configuration, while _extended_ means it was run successfully +with the additional GlobalPlatform™ TEE Initial Configuration Test Suite +v1.1.0.4. + +If a platform is not listed, it means the release was not tested on this +platform. + + +* d02: extended +* hikey: extended +* imx-mx6ulevk: standard +* ls-ls1021atwr: standard (single core) +* mediatek-mt8173: standard +* rcar-h3: standard +* rpi3: standard +* stm-b2260: extended +* ti-dra7xx: standard +* vexpress-fvp: standard +* vexpress-juno: standard +* vexpress-qemu_armv8a: standard +* vexpress-qemu_virt: standard +* zynqmp-zc1751_dc1: standard +* zynqmp-zc1751_dc2: standard +* zynqmp-zcu102: standard + +[github_commits_2_4_0]: https://github.com/OP-TEE/optee_os/compare/2.3.0...2.4.0 +[issue1332]: https://github.com/OP-TEE/optee_os/issues/1332 +[issue1353]: https://github.com/OP-TEE/optee_os/issues/1353 +[build issue131]: https://github.com/OP-TEE/build/issues/131 +[commit_a238b74]: https://github.com/OP-TEE/optee_os/commit/a238b744b1b3 +[commit_44e900e]: https://github.com/OP-TEE/optee_os/commit/44e900eabfc1 + +# OP-TEE - version 2.3.0 + +[Link][github_commits_2_3_0] to a list of all commits between this release and +the previous one (2.2.0). + +Please note: this release is API-compatible with the previous one, but the +Secure Storage internal format for the REE FS is not compatible due to commit +[361fb3e][commit_361fb3e] ("core: REE FS: use a single file per object"). + +[commit_361fb3e]: https://github.com/OP-TEE/optee_os/commit/361fb3e + +## New features + +* New supported platform: Xilinx Zynq 7000 ZC702 (`PLATFORM=zynq7k-zc702`) + +* Add debug assertions to spinlocks and mutexes + +* Add more CP15 register access macros for Cortex-A9 + +* ARMv7-A: redesign secure monitor to make it easier to register services + +* ARMv7-A: cleanup boot arguments + +* libutee: extend `TEE_CheckMemoryAccessRights()` with + `TEE_MEMORY_ACCESS_SECURE` and `TEE_MEMORY_ACCESS_NONSECURE` + +* plat-hikey: enable SPI by default and add sample test code + +* Consider `CFLAGS_ta_arm64` and `CFLAGS_ta_arm32` when building TAs + +* Secure storage refactoring + - Simplify interface with tee-supplicant. Minimize round trips with normal + world, especially by adding a cache for FS RPC payload data. + - REE FS: use a single file per object, remove block cache. + +* Print call stack in panic() + +## Bug fixes + +* Fix UUID encoding when communicating with normal world (use big endian + mode instead of native endianness). Related to this, the string format + for UUIDs has changed in tee-supplicant, so that TA file names now follow + the format defined in RFC4122 (a missing hyphen was added). The old format + is still supported, but deprecated, and will likely be removed with the + next major release. + +* Drop write permission to non-writable ELF segments after TA loading is + complete. + +* mm: fix confusing memory mapping debug traces + +* plat-ti: fix issues with MMU mapping + +* crypto: fix clearing of big numbers + +* build: allow spaces and double quotes in CFG_ variables + +* mm: use paddr_t to support both 32- and 64-bit architectures properly. + Resolves 32-bit truncation error when pool is at top of 32 bit address + space on 64-bit architecture. + +* plat-stm: support pager. Fix pager on ARMv7-A SMP boards. + +* Fix debug output of Trusted Applications (remove "ERROR: TEE-CORE:" prefix) + +* Do not consider TA memref parameters as TA private memory + +* crypto: fix `cipher_final()` which would not call `cbc_done()` for CBC_MAC + algorithms + +* fix for 16-way PL310 + +* arm32: fix call stack unwinding (`print_stack()`) + +* arm32: fix spinlock assembly code + +* plat-stm, plat-imx: fix SCR initalization + +* Fix user L1 MMU entries calculation (non-LPAE), allowing TTBCR.N values + other than 7. + +* mtk-mt8173: fix panic caused by incorrect size of SHMEM + +* plat-stm: fix RNG driver (non-flat mapping) + +## Known issues + +* New issues open on GitHub + * [#1203][issue1203] AES-CTS mode will fail when inlen=0x100, in_incr=0x80 + * [#1199][issue1199] Both tee and teepriv reported GlobalPlatform compliant + * [#1188][issue1188] Secure storage (SQL FS and REE FS): blocks not tied to + current meta header + * [#1172][issue1172] paddr_t should be larger than 32 bits when + CFG_WITH_LPAE is enabled + +## Tested on + +In the list below, _standard_ means that the `xtest` program passed with +its default configuration, while _extended_ means it was run successfully +with the additional GlobalPlatform™ TEE Initial Configuration Test Suite +v1.1.0.4. + +If a platform is not listed, it means the release was not tested on this +platform. + + +* d02: extended +* hikey: extended +* imx-mx6ulevk: standard +* ls-ls1021atwr: standard +* mediatek-mt8173: standard +* rcar-h3: standard +* rpi3: standard +* stm-b2260: extended +* stm-cannes: extended +* ti-dra7xx: standard +* vexpress-fvp: standard +* vexpress-juno: standard +* vexpress-qemu_armv8a: standard +* vexpress-qemu_virt: extended +* zynqmp-zcu102: standard + +[github_commits_2_3_0]: https://github.com/OP-TEE/optee_os/compare/2.2.0...2.3.0 +[issue1172]: https://github.com/OP-TEE/optee_os/issues/1172 +[issue1188]: https://github.com/OP-TEE/optee_os/issues/1188 +[issue1199]: https://github.com/OP-TEE/optee_os/issues/1199 +[issue1203]: https://github.com/OP-TEE/optee_os/issues/1203 + +# OP-TEE - version 2.2.0 + +[Link][github_commits_2_2_0] to a list of all commits between this release and +the previous one (2.1.0). + +Please note: this release is API-compatible with the previous one, but the +Secure Storage internal format is not compatible due to commit +[fde4a75][commit_fde4a75] ("storage: encrypt the FEK with a TA-specific key"). + +[commit_fde4a75]: https://github.com/OP-TEE/optee_os/commit/fde4a75 + +## New features + +* New supported platforms: + * Freescale i.MX6 Quad SABRE Lite & SD + * HiSilicon D02 + * Raspberry Pi3 + * Renesas RCAR H3 + * STMicroelectronics b2260 - h410 + +* Pager: Support paging of read/write pages by encrypting them with AES-GCM. + Support paging of user TAs. Add global setting for TZSRAM size + (CFG_CORE_TZSRAM_EMUL_SIZE), defaults to 300K. + +* Support for more than 8 CPU cores + +* Added SPI framework and PL022 driver + +* GPIO: framework supports multiple instances, PL061 driver now has get/set + interrupt and mode control functions + +* Secure storage: Encrypt the File Encryption Key with a TA-specific key for + better TA isolation. Add build-time and run-time support for multiple storage + backends. Add SQLite backend. + +* Trusted User Interface: some code is introduced to support the implementation + of TUI. This includes: a generic framebuffer driver, display and serial + abstractions, and drivers for PL111 (LCD) / PL050 (KMI) / TZC400 and PS2 + mouse. + +* AES acceleration using ARMv8-A Cryptographic Extensions instructions is + now supported in AArch32 mode + +* Add support for GCC flags: -fsanitize=undefined and -fsanitize=kernel-address + +* Use a global setting for core heap size (CFG_CORE_HEAP_SIZE), 64K by default. + +* Add macros to unwind and print the call stack of TEE core + +* Libtomcrypt: sync with the latest `develop` branch. + +* The Trusted Application SDK (ta_dev_kit.mk) can produce libraries (.a) + +* Rework assertions and TEE core panics and properly honor NDEBUG + +## Bug fixes + +* Fix incorrect algorithm passed to cipher.final() + +* scripts: support Python 2.x and 3.x + +* Secure storage: Add proper locking to support concurrent access. Fix sign + extension bug with offset parameter of syscall storage_obj_seek which could + cause errors in Aarch32 mode. Fix reading beyond end of file. + +* Aarch64: mask all maskable exceptions before doing a normal return from call. + +* Device Tree: add no-map property to OP-TEE node in reserved-memory. + +* LibTomcrypt: fix CVE-2016-6129 + +## Known issues + +* New issues open on GitHub + * [#1093][issue1093] rcar-h3: xtest 6010 hangs + * [#1092][issue1092] rcar-h3: xtest 4010 fails + * [#1081][issue1081] Bad mapping of TA secure memref parameters + * [#1071][issue1071] __data_end may not correctly represent text start position when using CFG_WITH_PAGER + * [#1069][issue1069] armv7/Aarch32: crash in stack unwind (DPRINT_STACK()) + +## Tested on + +In the list below, _standard_ means that the `xtest` program passed with +its default configuration, while _extended_ means it was run successfully +with the additional GlobalPlatform™ TEE Initial Configuration Test Suite +v1.1.0.4. + +If a platform is not listed, it means the release was not tested on this +platform. + + +* d02: extended +* hikey: extended +* imx-mx6qsabrelite: standard +* imx-mx6qsabresd: standard +* rcar-h3: standard, pass except issues [#1092][issue1092] and [#1093][issue1093] +* rpi3: standard +* stm-b2260: standard +* stm-cannes: standard +* ti-dra7xx: standard +* vexpress-fvp: standard +* vexpress-juno: standard +* vexpress-qemu_armv8a: standard +* vexpress-qemu_virt: extended +* zynqmp-zcu102: standard + +[github_commits_2_2_0]: https://github.com/OP-TEE/optee_os/compare/2.1.0...2.2.0 +[issue1081]: https://github.com/OP-TEE/optee_os/issues/1081 +[issue1071]: https://github.com/OP-TEE/optee_os/issues/1071 +[issue1069]: https://github.com/OP-TEE/optee_os/issues/1069 +[issue1092]: https://github.com/OP-TEE/optee_os/issues/1092 +[issue1093]: https://github.com/OP-TEE/optee_os/issues/1093 + +# OP-TEE - version 2.1.0 + +## New features + +* New supported platforms: + * Xilinx Zynq UltraScale+ MPSOC + * Spreadtrum SC9860 + +* GCC5 support + +* Non Linear Mapping support: In OP-TEE kernel mode, the physical to virtual + addresses was linear until this release, meaning the virtual addresses + were equal to the physical addresses. This is no more the case in this + release. + +* Font rendering routines have been introduced in order to ease an + implementation of Trusted UI. + +* File Storage: Possibility to use the normal world filesystem and the RPMB + implementations simultaneously. + +* AOSP: There is a [local manifest][aosp_local_manifest] to build OP-TEE into an AOSP build, running on HiKey. + Please refer to the README in that repo for instructions. + +* OpenEmbedded: In addition to the makefile-based build described in the optee_os README, there is an + [OpenEmbedded-based build][oe_build] that supports Qemu (32-bit), FVP (64-bit), and HiKey (64-bit). + Please refer to the README in that repo for instructions. + +* [Link][github_commits_2_1_0] to a list of all commits between this and + previous release. + + +## Tested on +Definitions: + +| Type | Meaning | +| ---- | ------- | +| Standard tests | The [optee_test][optee_test] project. | +| Extended tests | optee_test with tests from the GlobalPlatform™ TEE Initial Configuration Test Suite v1.1.0.4. | + +* ARM Juno Board (vexpress-juno), standard. +* Foundation Models (vexpress-fvp), standard tests + extended tests, + using FVP ARM V8 Foundation Platformr0p0 (platform build 10.0.37) +* FSL i.MX6 UltraLite EVK (imx), standard. +* FSL ls1021a (ls-ls1021atwr), standard tests. +* HiKey (hikey), standard + extended tests. +* QEMU (vexpress-qemu), standard + extended tests. +* Xilinx Zynq UltraScale+ MPSOC, standard tests + +Note that the following platform has not been tested: +* MTK8173-EVB (mediatek-mt8173) + + +## Known issues +* Issue(s) open on GitHub + * [#868][pr868]: python-wand font generation sometimes times out + * [#863][pr863]: "double free or corruption" error when building optee_os + * [#858][pr858]: UUIDs in binary format have wrong endinanness + * [#857][pr857]: Formatting of UUIDs is incorrect + * [#847][pr847]: optee_os panic(TEE-CORE: Assertion) + * [#838][pr838]: TUI font rendering is _very_ slow + * [#814][pr814]: Persistent objects : save informations after close + * [#665][pr665]: xtest 1013 stalled on HiKey when log levels are 4 and optee_os is on its own UART + * [#506][pr506]: tee-supplicant panic & ta panic + +[github_commits_2_1_0]: https://github.com/OP-TEE/optee_os/compare/2.0.0...2.1.0 +[pr868]: https://github.com/OP-TEE/optee_os/issues/868 +[pr863]: https://github.com/OP-TEE/optee_os/issues/863 +[pr858]: https://github.com/OP-TEE/optee_os/issues/858 +[pr857]: https://github.com/OP-TEE/optee_os/issues/857 +[pr847]: https://github.com/OP-TEE/optee_os/issues/847 +[pr838]: https://github.com/OP-TEE/optee_os/issues/838 +[pr814]: https://github.com/OP-TEE/optee_os/issues/814 +[pr665]: https://github.com/OP-TEE/optee_os/issues/665 +[aosp_local_manifest]: https://github.com/linaro-swg/optee_android_manifest +[oe_build]: https://github.com/linaro-swg/oe-optee + +# OP-TEE - version 2.0.0 + +## New features + +* Generic driver: A new generic TEE driver is in the process of being + [upstreamed][gendrv_v9]. + In this release, [OP-TEE/optee_linuxdriver][optee_linuxdriver] is no more used. + Instead, linux v4.5 is being patched using the proposed Generic TEE Driver, + as it can be found in [https://github.com/linaro-swg/linux/tree/optee][linux_optee] + +* RPMB support: Secure Storage can now use Replay Protected Memory Block (RPMB) partition + of an eMMC device. Check the [full documentation][rpmb_doc] + +* Hard-float ABI is now available. + +* [Link][github_commits_2_0_0] to a list of all commits between this and + previous release. + + +## Tested on +Definitions: + +| Type | Meaning | +| ---- | ------- | +| Standard tests | The [optee_test][optee_test] project. | +| Extended tests | optee_test with tests from the GlobalPlatform™ TEE Initial Configuration Test Suite v1.1.0.4. | + +* ARM Juno Board (vexpress-juno), standard. +* Foundation Models (vexpress-fvp), standard tests + extended tests, + using FVP ARM V8 Foundation Platformr0p0 (platform build 9.5.40) +* FSL ls1021a (ls-ls1021atwr), standard. +* HiKey (hikey), standard. +* MTK8173-EVB (mediatek-mt8173), standard. +* QEMU (vexpress-qemu), standard + extended tests. +* STM Cannes (stm-cannes), standard + extended tests. + +## Known issues +* Issue(s) open on GitHub + * [#40][prld40] BUG_ON() when re-using RPC buffer to tee-supplicant + * [#506][pr506]: tee-supplicant panic & ta panic + +[github_commits_2_0_0]: https://github.com/OP-TEE/optee_os/compare/1.1.0...2.0.0 +[rpmb_doc]: https://github.com/OP-TEE/optee_os/blob/master/documentation/secure_storage_rpmb.md +[optee_linuxdriver]: https://github.com/OP-TEE/optee_linuxdriver +[gendrv_v9]: https://lkml.org/lkml/2016/4/1/205 +[linux_optee]: https://github.com/linaro-swg/linux/tree/optee + + +# OP-TEE - version 1.1.0 + + +## New features + +* Softfloat library: floating point support is now available in 32bits TA. + +* Support running 64-bits TA: on ARMv8-A platform, TA can be compiled in + AArch32 and/or in AArch64 in case the core is compiled in AArch64. + An example can be found in HiKey configuration file. Using the following + excerpt code, the user TA libraries are compiled in both AArch32 and + AArch64, and can be found in `out/arm-plat-hikey/export-ta_arm32` and + `out/arm-plat-hikey/export-ta_arm64` + +``` + ta-targets = ta_arm32 + ta-targets += ta_arm64 +``` + +* Concurrent TA support: multiple TA can run in parallel on + several cores. + +* New tests added in xtest test suite: concurrent TA (xtest 1013), + floating point tests (xtest 1006 and os_test TA) and corruption + file storage (xtest 20000) + +* [Link][github_commits_1_1_0] to a list of all commits between this and + previous release. + + +## Tested on +Definitions: + +| Type | Meaning | +| ---- | ------- | +| Standard tests | The [optee_test][optee_test] project. | +| Extended tests | optee_test with tests from the GlobalPlatform™ TEE Initial Configuration Test Suite v1.1.0.4. | +| Hello world test | Plain hello world Trusted Application such as [this][hello_world]. | + +* Foundation Models (vexpress-fvp), standard tests + extended tests, + using FVP ARM V8 Foundation Platformr0p0 (platform build 9.5.40) +* HiKey (hikey), standard + extended tests. +* MT8173 (mediatek), standard tests. +* QEMU (vexpress-qemu), standard + extended tests. +* STM Cannes (stm-cannes), standard + extended tests. + +## Known issues +* Secure Storage is implemented, but note that anti-rollback protection + is not implemented yet. + +* Issue(s) open on GitHub + * [#40][prld40] BUG_ON() when re-using RPC buffer to tee-supplicant + * [#296][pr296]: Connecting RPMB to the storage APIs. + * [#493][pr493]: setup_juno_optee: unable to find pre-built binaries + * [#506][pr506]: tee-supplicant panic & ta panic + +[prld40]: https://github.com/OP-TEE/optee_linuxdriver/issues/40 +[pr506]: https://github.com/OP-TEE/optee_os/issues/506 +[github_commits_1_1_0]: https://github.com/OP-TEE/optee_os/compare/1.0.1...1.1.0 + + + +# OP-TEE - version 1.0.0 + +OP-TEE is now maintained by Linaro. Contributors do not need to +sign a CLA anymore, but must follow the rules of the [DCO][DCO] +(Developer Certificate of Origin) instead. + + +## New features + +* Add hardware support for Texas Instruments DRA7xx, ARMv7 (plat-ti) + +* GlobalPlatform™ TEE Internal Core API Specification v1.1, + including ECC algorithms. + +* Secure Storage: Files stored by the REE are now encrypted. Operations + are made atomic in order to prevent inconsistencies in case of errors + during the storage operations. [Slides][LCStorage] describing the + Secure Storage have been presented at the Linaro Connect SFO15. + +* Change of format of the Trusted Applications: they follow a + [signed ELF format][elf] + +* Rework thread [synchronization][synchro] in optee_os. + +* Use of ARMv8 native cryptographic support. + +* [OP-TEE/optee_test][optee_test] test suite is released. + +* Introduce [OP-TEE/manifest][manifest] and [OP-TEE/build][build] + to setup and build QEMU, FVP, HiKey and Mediatek platforms. Setup scripts + that used to be in optee_os have been removed, except for Juno board. + +* [Link][github_commits_1_0_0] to a list of all commits between this and + previous release. + + +## Tested on +Definitions: + +| Type | Meaning | +| ---- | ------- | +| Standard tests | The [optee_test][optee_test] project. | +| Extended tests | optee_test with tests from the GlobalPlatform™ TEE Initial Configuration Test Suite v1.1.0.4. | +| Hello world test | Plain hello world Trusted Application such as [this][hello_world]. | + +* ARM Juno Board (vexpress-juno), standard + extended tests. +* Foundation Models (vexpress-fvp), standard tests. +* HiKey (hikey), standard + extended tests. +* MT8173 (mediatek), standard tests. +* QEMU (vexpress-qemu), standard + extended tests. +* STM Cannes (stm-cannes), standard + extended tests. + +## Known issues +* Secure Storage is implemented, but note that anti-rollback protection + is not implemented yet. + +* Issue(s) open on GitHub + * [#210][pr210]: libteec.so 32-bit does not communicate well + with 64-bit kernel module + * [#296][pr296]: Connecting RPMB to the storage APIs. + * [#493][pr493]: setup_juno_optee: unable to find pre-built binaries + * [#494][pr494]: HiKey: xtest 7671 fails (1.0.0-rc2) + +[pr210]: https://github.com/OP-TEE/optee_os/issues/210 +[pr296]: https://github.com/OP-TEE/optee_os/issues/296 +[pr493]: https://github.com/OP-TEE/optee_os/issues/493 +[pr494]: https://github.com/OP-TEE/optee_os/issues/494 +[github_commits_1_0_0]: https://github.com/OP-TEE/optee_os/compare/0.3.0...1.0.0 +[DCO]: https://github.com/OP-TEE/optee_os/blob/master/Notice.md#contributions +[LCStorage]: http://www.slideshare.net/linaroorg/sfo15503-secure-storage-in-optee +[synchro]: https://github.com/OP-TEE/optee_os/blob/master/documentation/optee_design.md#4-thread-handling +[elf]: https://github.com/OP-TEE/optee_os/blob/master/documentation/optee_design.md#format +[optee_test]: https://github.com/OP-TEE/optee_test +[manifest]: https://github.com/OP-TEE/manifest +[build]: https://github.com/OP-TEE/build + + + +# OP-TEE - version 0.3.0 + +## New features + +* Add hardware support for + * Mediatek MT8173 Board, ARMv8-A (plat-mediatek) + * Hisilicon HiKey Board, ARMv8-A (plat-hikey) +* AArch64 build of optee_os is now possible through the configuration `CFG_ARM64_core=y` +* Secure Storage: Data can be encrypted prior to their storage in the non-secure. + Build is configured using `CFG_ENC_FS=y` +* A generic boot scheme can be used. Boot configuration is commonalized. This helps + new board support. It is applied on plat-hikey, plat-vexpress, plat-mediatek, plat-stm + and plat-vexpress. + +## Tested on +Definitions: + +| Type | Meaning | +| ---- | ------- | +| Standard tests | The optee_test project. | +| Extended tests | optee_test with tests from the GlobalPlatform™ TEE Initial Configuration Test Suite v1.1.0.4. | +| Hello world test | Plain hello world Trusted Application such as [this][hello_world]. | + +* ARM Juno Board (vexpress-juno), standard tests. +* Foundation Models (vexpress-fvp), standard tests. +* HiKey (hikey), standard tests. +* MT8173 (mediatek), standard tests. +* QEMU (vexpress-qemu), standard + extended tests. +* STM Cannes (stm-cannes), standard + extended tests. + +------------------------------------------- + +# OP-TEE - version 0.2.0 + +## New features + +### Linux Driver Refactoring + +Linux Driver has been refactored. It is now split in two parts: +* optee.ko, the generic Linux driver. It contains all functionality + common to all backends. +* optee_armtz.ko, a specific backend dedicated to the TrustZone optee. + It depends on optee.ko. + +Loading the TrustZone optee linux driver module is now performed using + + modprobe optee_armtz + +Thanks to the dependency between the generic and the backend modules, optee.ko is then automatically loaded. + +### Misc new features +* support PL310 lock down at TEE boot +* add 64bits support (division / print) + +## Tested on +Definitions: + +| Type | Meaning | +| ---- | ------- | +| Standard tests | The optee_test project. | +| Extended tests | optee_test with tests from the GlobalPlatform™ TEE Initial Configuration Test Suite v1.1.0.4. | +| Hello world test | Plain hello world Trusted Application such as [this][hello_world]. | + +* ARM Juno Board (vexpress-juno), standard tests + extended tests. + +* Foundation Models (vexpress-fvp), standard + extended tests. + +* QEMU (vexpress-qemu), standard + extended tests. + +* STM Cannes (stm-cannes), standard + extended tests. + + +## Issues resolved since last release +* Fix user TA trace issue, in order each TA is able to select its own trace level + + +------------------------------------------- +# OP-TEE - version 0.1.0 + +## New features +Below is a summary of the most important features added, but at the end you will +find a link that present you all commits between the current and previous +release tag. + +* GlobalPlatform Client API v1.0 support. + +* GlobalPlatform Internal API v1.0 support. + +* GlobalPlatform Secure Elements v1.0 support. + +* Add hardware support for + + * Allwinner A80, ARMv7-A. + + * ARM Juno Board, ARMv8-A. + + * Foundation Models, ARMv8-A. + + * Fast Models, ARMv8-A. + + * QEMU, ARMv7-A. + + * STM Cannes, ARMv7-A. + + * STM Orly2, ARMv7-A. + +* Add LibTomCrypt as the default software cryptographic library. + +* Add cryptographic abstraction layer in on secure side to ease the use of + other cryptographic software libraries or adding support for hardware + acceleration. + +* Extended cryptographic API with support for HKDF, Concat KDF and PBKDF2. + +* SHA-1 and SHA-256 ARMv8-A crypto extension implementation. + +* Enabled paging support in OP-TEE OS. + +* Add support for xtest (both standard and extended) in QEMU and FVP setup + scripts. + +* Add documentation for the OS design, cryptographic abstraction layer, secure + elements design, the build system, GitHub usage, key derivation extensions, + ARM-Trusted Firmware usage within OP-TEE and GlobalPlatform usage within + OP-TEE. + +* Integrate support for Travis CI. + +* [Link][github_commits_0_1_0] to a list of all commits between this and + previous release. + + +## Tested on +Definitions: + +| Type | Meaning | +| ---- | ------- | +| Standard tests | The optee_test project. | +| Extended tests | optee_test with tests from the GlobalPlatform™ TEE Initial Configuration Test Suite v1.0.0. | +| Hello world test | Plain hello world Trusted Application such as [this][hello_world]. | + +* Allwinner A80 (plat-sunxi), hello world test. + +* ARM Juno Board (vexpress-juno), standard tests. + +* Foundation Models (plat-vexpress-fvp), standard + extended tests + +* QEMU (plat-vexpress-qemu), standard + extended tests (and Secure Elements + tested separately). + +* STM Cannes (plat-stm-cannes), standard + extended tests. + + +## Issues resolved since last release +N/A since this is the first release tag on OP-TEE. + + +## Known issues +* Storage is implemented, but not "Secure storage", meaning that a client + needs to do encrypt files on their own before storing the files. + +* Issue(s) open on GitHub + * [#95][pr95]: An error about building the test code of libtomcrypt. + + * [#149][pr149]: when testing optee os with arm trusted firmware (I + utilized optee os tee.bin as bl32 image) on juno platform, I got an + error. + + * [#161][pr161]: tee_svc_cryp.c lacks accessibility checks on + user-supplied TEE_Attributes. + +[hello_world]: https://github.com/jenswi-linaro/lcu14_optee_hello_world +[github_commits_0_1_0]: https://github.com/OP-TEE/optee_os/compare/b01047730e77127c23a36591643eeb8bb0487d68...999e4a6c0f64d3177fd3d0db234107b6fb860884 +[pr95]: https://github.com/OP-TEE/optee_os/issues/95 +[pr149]: https://github.com/OP-TEE/optee_os/issues/149 +[pr161]: https://github.com/OP-TEE/optee_os/issues/161 + +* Global Platform Device Internal Core API v1.1 + * [#230][pr230]: Persistent object corruption support (TEE_ERROR_CORRUPT_OBJECT/_2) + * [#230][pr230]: Persistent object access support (TEE_ERROR_STORAGE_NOT_AVAILABLE/_2) diff --git a/optee_os/LICENSE b/optee_os/LICENSE new file mode 100644 index 0000000..657de32 --- /dev/null +++ b/optee_os/LICENSE @@ -0,0 +1,27 @@ +Unless it has its own copyright/license embedded in its body, each file +is subject to the following license terms: + +Copyright (c) 2015, Linaro Limited +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/optee_os/MAINTAINERS b/optee_os/MAINTAINERS new file mode 100644 index 0000000..b480529 --- /dev/null +++ b/optee_os/MAINTAINERS @@ -0,0 +1,328 @@ + Maintainers in the OP-TEE project + +Linaro as such maintains OP-TEE, but for individual devices which might not be +available to Linaro and/or in public in general we have assigned +sub-maintainers for these platforms. + +Please keep this list in alphabetical order, and refer to the Linux kernel's +MAINTAINER file for the description of section entries [1]. + +Note that OP-TEE development mainly occurs on GitHub [2] so this file should be +used a bit differently compared to the Linux MAINTAINERS file: + +1. GitHub accounts may be given in square brackets, starting with an @ sign. +For example, [@jbech-linaro]. + +2. Patches should generally be submitted as GitHub pull requests (more details +in documentation/github.md). Therefore, please do NOT send patches to the +e-mail addresses below. Maintainers normally are subscribed to GitHub +notifications, so they should see your pull requests. If you need to 'ping' +some maintainer, just mention his/her GitHub account in a comment in the pull +request. + +2. Bug reports and questions should be posted to the GitHub project as well +(use the "Issues" tab). + +3. The last entry ("THE REST") lists the overall maintainers (M:) and the +members of the Linaro Security Working Group who provide reviews on a regular +basis (R:). + +[1] https://www.kernel.org/doc/linux/MAINTAINERS +[2] https://github.com/OP-TEE/optee_os + +---------- + +ARM Foundation FVP +R: Jens Wiklander [@jenswi-linaro] +S: Maintained +F: core/arch/arm/plat-vexpress/ + +ARM Juno +R: Jens Wiklander [@jenswi-linaro] +S: Maintained +F: core/arch/arm/plat-vexpress/ + +ARM Corstone1000 +R: Emekcan.Aras@arm.com [@ememarar] +S: Maintained +F: core/arch/arm/plat-corstone1000/ + +AllWinner sun8i H2+ +R: Ying-Chun Liu (PaulLiu) [@grandpaul] +S: Maintained +F: core/arch/arm/plat-sunxi/ + +AllWinner sun50i A64 +R: Amit Singh Tomar [@Amit-Radur] +S: Maintained +F: core/arch/arm/plat-sunxi/ + +AmLogic AXG (A113D) +R: Carlo Caione [@carlocaione] +S: Maintained +F: core/arch/arm/plat-amlogic/ + +Atmel ATSAMA5D2-XULT +R: Akshay Bhat [@nodeax] +S: Maintained +F: core/arch/arm/plat-sam/ + +Broadcom ns3 +L: Broadcom +S: Orphan +F: core/arch/arm/plat-bcm/ + +Core Drivers I2C +R: Jorge Ramirez [@ldts] +S: Maintained +F: core/drivers/imx_i2c.c + +Core Drivers RNGB +R: Jorge Ramirez [@ldts] +S: Maintained +F: core/drivers/imx_rngb.c + +Core Drivers SE050 +R: Jorge Ramirez [@ldts] +S: Maintained +F: core/drivers/crypto/se050 + +Core Drivers Versal ACAP +R: Jorge Ramirez [@ldts] +S: Maintained +F: core/drivers/crypto/versal/authenc.c +F: core/drivers/crypto/versal/ecc.c +F: core/drivers/crypto/versal/ipi.c +F: core/drivers/crypto/versal/rsa.c +F: core/drivers/versal_gpio.c +F: core/drivers/versal_huk.c +F: core/drivers/versal_mbox.c +F: core/drivers/versal_nvm.c +F: core/drivers/versal_pm.c +F: core/drivers/versal_puf.c +F: core/drivers/versal_sha3_384.c +F: core/drivers/versal_trng.c + +Core Drivers ZYNQMP +R: Jorge Ramirez [@ldts] +S: Maintained +F: core/drivers/zynqmp_csu_aes.c +F: core/drivers/zynqmp_csu_puf.c +F: core/drivers/zynqmp_csudma.c +F: core/drivers/zynqmp_huk.c +F: core/drivers/zynqmp_pm.c + +Function Graph Tracer (ftrace) support +R: Sumit Garg [@b49020] +S: Maintained +F: ldelf/ftrace.c +F: lib/libutils/ext/ftrace/ + +HiSilicon D02 +S: Orphan +F: core/arch/arm/plat-d02/ + +HiSilicon D06 +R: Xiaoxu Zeng [@xiaoxuZeng] +S: Maintained +F: core/arch/arm/plat-d06/ + +HiSilicon HiKey (Kirin 620), HiKey960 (Kirin 960) +R: Jerome Forissier [@jforissier] +S: Maintained +F: core/arch/arm/plat-hikey/ + +HiSilicon Poplar (Hi3798C V200) +R: Igor Opaniuk [@igoropaniuk] +S: Maintained +F: core/arch/arm/plat-poplar/ + +Hisilicon Hi3519AV100 family +R: Jerome Forissier [@jforissier] +S: Maintained +F: core/arch/arm/plat-hisilicon + +Marvell Armada 70x0, Armada 80x0, Armada 3700, OcteonTX2 CN96XX, OcteonTX2 CFN95XX, OcteonTX2 CN98XX, CN10K +R: Tao Lu [@taovcu] +S: Maintained +F: core/arch/arm/plat-marvell/ + +MediaTek MT8173 EVB +S: Orphan +F: core/arch/arm/plat-mediatek/ + +NXP LS1043A-RDB, LS1046A-RDB, LS1012A-RDB, LS1028A-RDB, LS1088A-RDB, LS2088A-RDB, LX2160A-RDB, LX2160A-QDS +R: Pankaj Gupta [@pangupta] +R: Sahil Malhotra [@sahilnxp] +S: Maintained +F: core/arch/arm/plat-ls/ + +Core Drivers I2C +R: Sahil Malhotra [@sahilnxp] +S: Maintained +F: core/drivers/ls_i2c.c + +LS Core Drivers GPIO +R: Sahil Malhotra [@sahilnxp] +S: Maintained +F: core/drivers/ls_gpio.c + +LS Core Drivers DSPI +R: Sahil Malhotra [@sahilnxp] +S: Maintained +F: core/drivers/ls_dspi.c + +NXP (Freescale) i.MX family +R: Peng Fan [@MrVan] +R: Silvano Di Ninno [@sdininno] +R: Clement Faure [@clementfaure] +S: Maintained +F: core/arch/arm/plat-imx/ +F: core/arch/arm/plat-imx/registers +F: core/drivers/crypto/caam/ +F: core/drivers/imx/ +F: core/drivers/imx_csu.c +F: core/drivers/imx_ele.c +F: core/drivers/imx_lpuart.c +F: core/drivers/imx_ocotp.c +F: core/drivers/imx_sc_api.c +F: core/drivers/imx_scu.c +F: core/drivers/imx_snvs.c +F: core/drivers/imx_uart.c +F: core/drivers/imx_wdog.c + +Clock driver framework +R: Clement Leger [@clementleger] +S: Maintained +F: core/drivers/clk/ + +Crypto driver interface +R: Cedric Neveux [@cneveux] +R: Silvano Di Ninno [@sdininno] +R: Clement Faure [@clementfaure] +S: Maintained +F: core/drivers/crypto/ + +NXP (Freescale) i.MX7 WaRP7 +R: Peng Fan [@MrVan] +R: Bryan O'Donoghue [@bryanodonoghue] +S: Maintained +F: core/arch/arm/plat-imx/conf.mk + +PKCS#11 TA +R: Etienne Carriere [@etienne-lms] +R: Ruchika Gupta [@ruchi393] +R: Vesa Jääskeläinen [@vesajaaskelainen] +S: Maintained +F: ta/pkcs11 + +QEMU (32 and 64 bits) +R: Jens Wiklander [@jenswi-linaro] +S: Maintained +F: core/arch/arm/plat-vexpress/ + +Raspberry Pi3 +R: Joakim Bech [@jbech-linaro] +S: Maintained +F: core/arch/arm/plat-rpi3/ + +Renesas RCAR +R: Volodymyr Babchuk [@lorc] +S: Maintained +F: core/arch/arm/plat-rcar/ + +Renesas RZ/G2 +R: Lad Prabhakar [@prabhakarlad] +R: Biju Das [@bijucdas] +S: Maintained +F: core/arch/arm/plat-rzg/ + +Renesas RZ/N1 +R: Sumit Garg [@b49020] +S: Maintained +F: core/arch/arm/plat-rzn1/ + +Rockchip RK322X +R: Rockchip +S: Maintained +F: core/arch/arm/plat-rockchip/ + +Socionext DeveloperBox (Synquacer SC2A11) +R: Sumit Garg [@b49020] +S: Maintained +F: core/arch/arm/plat-synquacer/ + +Socionext UniPhier +R: Kunihiko Hayashi +R: [@96boards-akebi96/optee] +S: Maintained +F: core/arch/arm/plat-uniphier/ + +Spreadtrum SC9860 +S: Orphan +F: core/arch/arm/plat-sprd/ + +STMicroelectronics b2260-h410, b2120-h310/h410 +R: Etienne Carriere [@etienne-lms] +S: Maintained +F: core/arch/arm/plat-stm/ + +STMicroelectronics stm32mp1 +R: Etienne Carriere [@etienne-lms] +S: Maintained +F: core/arch/arm/plat-stm32mp1/ +F: core/drivers/stm32_* +F: core/drivers/stm32mp15_huk.c +F: core/drivers/stpmic1.c + +Texas Instruments AM43xx, AM57xx, DRA7xx, AM65x, J721E, J784S4, AM64x, AM62x +R: Andrew Davis [@glneo] +S: Maintained +F: core/arch/arm/plat-ti/ +F: core/arch/arm/plat-k3/ + +Xilinx Zynq 7000 ZC702 Board +R: Yan Yan +R: Feng Yu +S: Maintained +F: core/arch/arm/plat-zynq7k/ + +Xilinx Zynq UltraScale+ MPSOC +R: Ricardo Salveti [@ricardosalveti] +S: Maintained +F: core/arch/arm/plat-zynqmp/ + +Xilinx Versal ACAP +R: Jorge Ramirez-Ortiz [@ldts] +S: Maintained +F: core/arch/arm/plat-versal/ + +Virtualization support +R: Volodymyr Babchuk [@lorc] +S: Maintained +F: core/arch/arm/kernel/virtualization.c + +Aspeed AST2600, AST2700 +R: Chia-Wei Wang [@ChiaweiW] +R: Neal Liu [@Neal-liu] +S: Maintained +F: core/arch/arm/plat-aspeed/ + +RISC-V +R: Marouene Boubakri [@maroueneboubakri] +S: Maintained +F: core/arch/riscv/ +F: lib/libutee/arch/riscv/ +F: lib/libutils/ext/arch/riscv/ +F: lib/libutils/isoc/arch/riscv/ +F: ta/arch/riscv/ + +THE REST +M: Joakim Bech [@jbech-linaro] +M: Jens Wiklander [@jenswi-linaro] +M: Jerome Forissier [@jforissier] +R: Etienne Carriere [@etienne-lms] +L: op-tee@lists.trustedfirmware.org +S: Maintained +F: * +F: */ diff --git a/optee_os/Makefile b/optee_os/Makefile new file mode 100644 index 0000000..8db6678 --- /dev/null +++ b/optee_os/Makefile @@ -0,0 +1,128 @@ +SHELL = bash + +# It can happen that a makefile calls us, which contains an 'export' directive +# or the '.EXPORT_ALL_VARIABLES:' special target. In this case, all the make +# variables are added to the environment for each line of the recipes, so that +# any sub-makefile can use them. +# We have observed this can cause issues such as 'Argument list too long' +# errors as the shell runs out of memory. +# Since this Makefile won't call any sub-makefiles, and since the commands do +# not expect to implicitely obtain any make variable from the environment, we +# can safely cancel this export mechanism. Unfortunately, it can't be done +# globally, only by name. Let's unexport MAKEFILE_LIST which is by far the +# biggest one due to our way of tracking dependencies and compile flags +# (we include many *.cmd and *.d files). +unexport MAKEFILE_LIST + +# Automatically delete corrupt targets (file updated but recipe exits with a +# nonzero status). Useful since a few recipes use shell redirection. +.DELETE_ON_ERROR: + +include mk/macros.mk +include mk/checkconf.mk + +.PHONY: all +all: + +.PHONY: mem_usage +mem_usage: + +# log and load eventual tee config file +# path is absolute or relative to current source root directory. +ifdef CFG_OPTEE_CONFIG +$(info Loading OPTEE configuration file $(CFG_OPTEE_CONFIG)) +include $(CFG_OPTEE_CONFIG) +endif + +# If $(PLATFORM) is defined and contains a hyphen, parse it as +# $(PLATFORM)-$(PLATFORM_FLAVOR) for convenience +ifneq (,$(findstring -,$(PLATFORM))) +ops := $(join PLATFORM PLATFORM_FLAVOR,$(addprefix =,$(subst -, ,$(PLATFORM)))) +$(foreach op,$(ops),$(eval override $(op))) +endif + +# Make these default for now +ARCH ?= arm +PLATFORM ?= vexpress +# Default value for PLATFORM_FLAVOR is set in plat-$(PLATFORM)/conf.mk +ifeq ($O,) +O := out +out-dir := $(O)/$(ARCH)-plat-$(PLATFORM) +else +out-dir := $(O) +endif + +arch_$(ARCH) := y + +ifneq ($V,1) +q := @ +cmd-echo := true +cmd-echo-silent := echo +else +q := +cmd-echo := echo +cmd-echo-silent := true +endif + +ifneq ($(filter 4.%,$(MAKE_VERSION)),) # make-4 +ifneq ($(filter %s ,$(firstword x$(MAKEFLAGS))),) +cmd-echo-silent := true +endif +else # make-3.8x +ifneq ($(findstring s, $(MAKEFLAGS)),) +cmd-echo-silent := true +endif +endif + +SCRIPTS_DIR := scripts + +include core/core.mk + +# Platform/arch config is supposed to assign the targets +ta-targets ?= invalid +$(call force,default-user-ta-target,$(firstword $(ta-targets))) + +ifeq ($(CFG_WITH_USER_TA),y) +include ldelf/ldelf.mk +define build-ta-target +ta-target := $(1) +include ta/ta.mk +endef +$(foreach t, $(ta-targets), $(eval $(call build-ta-target, $(t)))) + +# Build user TAs included in this git +ifeq ($(CFG_BUILD_IN_TREE_TA),y) +define build-user-ta +ta-mk-file := $(1) +include ta/mk/build-user-ta.mk +endef +$(foreach t, $(sort $(wildcard ta/*/user_ta.mk)), $(eval $(call build-user-ta,$(t)))) +endif +endif + +include mk/cleandirs.mk + +.PHONY: clean +clean: + @$(cmd-echo-silent) ' CLEAN $(out-dir)' + $(call do-rm-f, $(cleanfiles)) + ${q}dirs="$(call cleandirs-for-rmdir)"; if [ "$$dirs" ]; then $(RMDIR) $$dirs; fi + @if [ "$(out-dir)" != "$(O)" ]; then $(cmd-echo-silent) ' CLEAN $(O)'; fi + ${q}if [ -d "$(O)" ]; then $(RMDIR) $(O); fi + +.PHONY: cscope +cscope: + @echo ' CSCOPE .' + ${q}rm -f cscope.* + ${q}find $(PWD) -name "*.[chSs]" | grep -v export-ta_ | \ + grep -v -F _init.ld.S | grep -v -F _unpaged.ld.S > cscope.files + ${q}cscope -b -q -k + +.PHONY: checkpatch checkpatch-staging checkpatch-working +checkpatch: checkpatch-staging checkpatch-working + +checkpatch-working: + ${q}./scripts/checkpatch.sh + +checkpatch-staging: + ${q}./scripts/checkpatch.sh --cached diff --git a/optee_os/README.md b/optee_os/README.md new file mode 100644 index 0000000..328078e --- /dev/null +++ b/optee_os/README.md @@ -0,0 +1,7 @@ +# OP-TEE Trusted OS +This git contains source code for the secure side implementation of OP-TEE +project. + +All official OP-TEE documentation has moved to http://optee.readthedocs.io. + +// OP-TEE core maintainers diff --git a/optee_os/core/arch/arm/arm.mk b/optee_os/core/arch/arm/arm.mk new file mode 100644 index 0000000..706556b --- /dev/null +++ b/optee_os/core/arch/arm/arm.mk @@ -0,0 +1,456 @@ +# Setup compiler for the core module +ifeq ($(CFG_ARM64_core),y) +arch-bits-core := 64 +else +arch-bits-core := 32 +endif +CROSS_COMPILE_core := $(CROSS_COMPILE$(arch-bits-core)) +COMPILER_core := $(COMPILER) +include mk/$(COMPILER_core).mk + +# Defines the cc-option macro using the compiler set for the core module +include mk/cc-option.mk + +# Size of emulated TrustZone protected SRAM, 448 kB. +# Only applicable when paging is enabled. +CFG_CORE_TZSRAM_EMUL_SIZE ?= 458752 + +ifneq ($(CFG_LPAE_ADDR_SPACE_SIZE),) +$(warning Error: CFG_LPAE_ADDR_SPACE_SIZE is not supported any longer) +$(error Error: Please use CFG_LPAE_ADDR_SPACE_BITS instead) +endif + +CFG_LPAE_ADDR_SPACE_BITS ?= 32 + +CFG_MMAP_REGIONS ?= 13 +CFG_RESERVED_VASPACE_SIZE ?= (1024 * 1024 * 10) + +ifeq ($(CFG_ARM64_core),y) +ifeq ($(CFG_ARM32_core),y) +$(error CFG_ARM64_core and CFG_ARM32_core cannot be both 'y') +endif +CFG_KERN_LINKER_FORMAT ?= elf64-littleaarch64 +CFG_KERN_LINKER_ARCH ?= aarch64 +# TCR_EL1.IPS needs to be initialized according to the largest physical +# address that we need to map. +# Physical address size +# 32 bits, 4GB. +# 36 bits, 64GB. +# (etc.) +CFG_CORE_ARM64_PA_BITS ?= 32 +$(call force,CFG_WITH_LPAE,y) +else +$(call force,CFG_ARM32_core,y) +CFG_KERN_LINKER_FORMAT ?= elf32-littlearm +CFG_KERN_LINKER_ARCH ?= arm +endif + +ifeq ($(CFG_TA_FLOAT_SUPPORT),y) +# Use hard-float for floating point support in user TAs instead of +# soft-float +CFG_WITH_VFP ?= y +ifeq ($(CFG_ARM64_core),y) +# AArch64 has no fallback to soft-float +$(call force,CFG_WITH_VFP,y) +endif +ifeq ($(CFG_WITH_VFP),y) +arm64-platform-hard-float-enabled := y +ifneq ($(CFG_TA_ARM32_NO_HARD_FLOAT_SUPPORT),y) +arm32-platform-hard-float-enabled := y +endif +endif +endif + +# Adds protection against CVE-2017-5715 also know as Spectre +# (https://spectreattack.com) +# See also https://developer.arm.com/-/media/Files/pdf/Cache_Speculation_Side-channels.pdf +# Variant 2 +CFG_CORE_WORKAROUND_SPECTRE_BP ?= y +# Same as CFG_CORE_WORKAROUND_SPECTRE_BP but targeting exceptions from +# secure EL0 instead of non-secure world, including mitigation for +# CVE-2022-23960. +CFG_CORE_WORKAROUND_SPECTRE_BP_SEC ?= $(CFG_CORE_WORKAROUND_SPECTRE_BP) + +# Adds protection against a tool like Cachegrab +# (https://github.com/nccgroup/cachegrab), which uses non-secure interrupts +# to prime and later analyze the L1D, L1I and BTB caches to gain +# information from secure world execution. +CFG_CORE_WORKAROUND_NSITR_CACHE_PRIME ?= y +ifeq ($(CFG_CORE_WORKAROUND_NSITR_CACHE_PRIME),y) +$(call force,CFG_CORE_WORKAROUND_SPECTRE_BP,y,Required by CFG_CORE_WORKAROUND_NSITR_CACHE_PRIME) +endif + +# Adds workarounds against if ARM core is configured with Non-maskable FIQ +# (NMFI) support. This is indicated by SCTLR.NMFI being true. NMFI cannot be +# disabled by software and as it affects atomic context end result will be +# prohibiting FIQ signal usage in OP-TEE and applying some tweaks to make sure +# FIQ is enabled in critical places. +CFG_CORE_WORKAROUND_ARM_NMFI ?= n + +CFG_CORE_RWDATA_NOEXEC ?= y +CFG_CORE_RODATA_NOEXEC ?= n +ifeq ($(CFG_CORE_RODATA_NOEXEC),y) +$(call force,CFG_CORE_RWDATA_NOEXEC,y) +endif +# 'y' to set the Alignment Check Enable bit in SCTLR/SCTLR_EL1, 'n' to clear it +CFG_SCTLR_ALIGNMENT_CHECK ?= n + +ifeq ($(CFG_CORE_LARGE_PHYS_ADDR),y) +$(call force,CFG_WITH_LPAE,y) +endif + +# SPMC configuration "S-EL1 SPMC" where SPM Core is implemented at S-EL1, +# that is, OP-TEE. +ifeq ($(CFG_CORE_SEL1_SPMC),y) +$(call force,CFG_CORE_FFA,y) +$(call force,CFG_CORE_SEL2_SPMC,n) +$(call force,CFG_CORE_EL3_SPMC,n) +endif +# SPMC configuration "S-EL2 SPMC" where SPM Core is implemented at S-EL2, +# that is, the hypervisor sandboxing OP-TEE +ifeq ($(CFG_CORE_SEL2_SPMC),y) +$(call force,CFG_CORE_FFA,y) +$(call force,CFG_CORE_SEL1_SPMC,n) +$(call force,CFG_CORE_EL3_SPMC,n) +CFG_CORE_HAFNIUM_INTC ?= y +# Enable support in OP-TEE to relocate itself to allow it to run from a +# physical address that differs from the link address +CFG_CORE_PHYS_RELOCATABLE ?= y +endif +# SPMC configuration "EL3 SPMC" where SPM Core is implemented at EL3, that +# is, in TF-A +ifeq ($(CFG_CORE_EL3_SPMC),y) +$(call force,CFG_CORE_FFA,y) +$(call force,CFG_CORE_SEL2_SPMC,n) +$(call force,CFG_CORE_SEL1_SPMC,n) +endif + +ifeq ($(CFG_CORE_FFA),y) +ifneq ($(CFG_DT),y) +$(error CFG_CORE_FFA depends on CFG_DT) +endif +endif + +ifeq ($(CFG_CORE_PHYS_RELOCATABLE)-$(CFG_WITH_PAGER),y-y) +$(error CFG_CORE_PHYS_RELOCATABLE and CFG_WITH_PAGER are not compatible) +endif +ifeq ($(CFG_CORE_PHYS_RELOCATABLE),y) +ifneq ($(CFG_CORE_SEL2_SPMC),y) +$(error CFG_CORE_PHYS_RELOCATABLE depends on CFG_CORE_SEL2_SPMC) +endif +endif + +ifeq ($(CFG_CORE_FFA)-$(CFG_WITH_PAGER),y-y) +$(error CFG_CORE_FFA and CFG_WITH_PAGER are not compatible) +endif +ifeq ($(CFG_GIC),y) +ifeq ($(CFG_ARM_GICV3),y) +$(call force,CFG_CORE_IRQ_IS_NATIVE_INTR,y) +else +$(call force,CFG_CORE_IRQ_IS_NATIVE_INTR,n) +endif +endif + +CFG_CORE_HAFNIUM_INTC ?= n +ifeq ($(CFG_CORE_HAFNIUM_INTC),y) +$(call force,CFG_CORE_IRQ_IS_NATIVE_INTR,y) +endif + +# Selects if IRQ is used to signal native interrupt +# if CFG_CORE_IRQ_IS_NATIVE_INTR == y: +# IRQ signals a native interrupt pending +# FIQ signals a foreign non-secure interrupt or a managed exit pending +# else: (vice versa) +# IRQ signals a foreign non-secure interrupt or a managed exit pending +# FIQ signals a native interrupt pending +CFG_CORE_IRQ_IS_NATIVE_INTR ?= n + +# Unmaps all kernel mode code except the code needed to take exceptions +# from user space and restore kernel mode mapping again. This gives more +# strict control over what is accessible while in user mode. +# Addresses CVE-2017-5715 (aka Meltdown) known to affect Arm Cortex-A75 +CFG_CORE_UNMAP_CORE_AT_EL0 ?= y + +# Initialize PMCR.DP to 1 to prohibit cycle counting in secure state, and +# save/restore PMCR during world switch. +CFG_SM_NO_CYCLE_COUNTING ?= y + + +# CFG_CORE_ASYNC_NOTIF_GIC_INTID is defined by the platform to some free +# interrupt. Setting it to a non-zero number enables support for using an +# Arm-GIC to notify normal world. This config variable should use a value +# larger the 32 to make it of the type SPI. +# Note that asynchronous notifactions must be enabled with +# CFG_CORE_ASYNC_NOTIF=y for this variable to be used. +CFG_CORE_ASYNC_NOTIF_GIC_INTID ?= 0 + +ifeq ($(CFG_ARM32_core),y) +# Configration directive related to ARMv7 optee boot arguments. +# CFG_PAGEABLE_ADDR: if defined, forces pageable data physical address. +# CFG_NS_ENTRY_ADDR: if defined, forces NS World physical entry address. +# CFG_DT_ADDR: if defined, forces Device Tree data physical address. +endif + +# CFG_MAX_CACHE_LINE_SHIFT is used to define platform specific maximum cache +# line size in address lines. This must cover all inner and outer cache levels. +# When data is aligned with this and cache operations are performed then those +# only affect correct data. +# +# Default value (6 lines or 64 bytes) should cover most architectures, override +# this in platform config if different. +CFG_MAX_CACHE_LINE_SHIFT ?= 6 + +core-platform-cppflags += -I$(arch-dir)/include +core-platform-subdirs += \ + $(addprefix $(arch-dir)/, kernel crypto mm tee) $(platform-dir) + +ifneq ($(CFG_WITH_ARM_TRUSTED_FW),y) +core-platform-subdirs += $(arch-dir)/sm +endif + +arm64-platform-cppflags += -DARM64=1 -D__LP64__=1 +arm32-platform-cppflags += -DARM32=1 -D__ILP32__=1 + +platform-cflags-generic ?= -ffunction-sections -fdata-sections -pipe +platform-aflags-generic ?= -pipe + +arm32-platform-aflags += -marm + +arm32-platform-cflags-no-hard-float ?= -mfloat-abi=soft +arm32-platform-cflags-hard-float ?= -mfloat-abi=hard -funsafe-math-optimizations +arm32-platform-cflags-generic-thumb ?= -mthumb \ + -fno-short-enums -fno-common -mno-unaligned-access +arm32-platform-cflags-generic-arm ?= -marm -fno-omit-frame-pointer -mapcs \ + -fno-short-enums -fno-common -mno-unaligned-access +arm32-platform-aflags-no-hard-float ?= + +arm64-platform-cflags-no-hard-float ?= -mgeneral-regs-only +arm64-platform-cflags-hard-float ?= +arm64-platform-cflags-generic := -mstrict-align $(call cc-option,-mno-outline-atomics,) + +ifeq ($(CFG_MEMTAG),y) +arm64-platform-cflags += -march=armv8.5-a+memtag +arm64-platform-aflags += -march=armv8.5-a+memtag +endif + +platform-cflags-optimization ?= -O$(CFG_CC_OPT_LEVEL) + +ifeq ($(CFG_DEBUG_INFO),y) +platform-cflags-debug-info ?= -g3 +platform-aflags-debug-info ?= -g +endif + +core-platform-cflags += $(platform-cflags-optimization) +core-platform-cflags += $(platform-cflags-generic) +core-platform-cflags += $(platform-cflags-debug-info) + +core-platform-aflags += $(platform-aflags-generic) +core-platform-aflags += $(platform-aflags-debug-info) + +ifeq ($(call cfg-one-enabled, CFG_CORE_ASLR CFG_CORE_PHYS_RELOCATABLE),y) +core-platform-cflags += -fpie +endif + +ifeq ($(CFG_CORE_PAUTH),y) +bp-core-opt := $(call cc-option,-mbranch-protection=pac-ret+leaf) +endif + +ifeq ($(CFG_CORE_BTI),y) +bp-core-opt := $(call cc-option,-mbranch-protection=bti) +endif + +ifeq (y-y,$(CFG_CORE_PAUTH)-$(CFG_CORE_BTI)) +bp-core-opt := $(call cc-option,-mbranch-protection=pac-ret+leaf+bti) +endif + +ifeq (y,$(filter $(CFG_CORE_BTI) $(CFG_CORE_PAUTH),y)) +ifeq (,$(bp-core-opt)) +$(error -mbranch-protection not supported) +endif +core-platform-cflags += $(bp-core-opt) +endif + +ifeq ($(CFG_ARM64_core),y) +core-platform-cppflags += $(arm64-platform-cppflags) +core-platform-cflags += $(arm64-platform-cflags) +core-platform-cflags += $(arm64-platform-cflags-generic) +core-platform-cflags += $(arm64-platform-cflags-no-hard-float) +core-platform-aflags += $(arm64-platform-aflags) +else +core-platform-cppflags += $(arm32-platform-cppflags) +core-platform-cflags += $(arm32-platform-cflags) +core-platform-cflags += $(arm32-platform-cflags-no-hard-float) +ifeq ($(CFG_UNWIND),y) +core-platform-cflags += -funwind-tables +endif +ifeq ($(CFG_SYSCALL_FTRACE),y) +core-platform-cflags += $(arm32-platform-cflags-generic-arm) +else +core-platform-cflags += $(arm32-platform-cflags-generic-thumb) +endif +core-platform-aflags += $(core_arm32-platform-aflags) +core-platform-aflags += $(arm32-platform-aflags) +endif + +# Provide default supported-ta-targets if not set by the platform config +ifeq (,$(supported-ta-targets)) +supported-ta-targets = ta_arm32 +ifeq ($(CFG_ARM64_core),y) +supported-ta-targets += ta_arm64 +endif +endif + +ta-targets := $(if $(CFG_USER_TA_TARGETS),$(filter $(supported-ta-targets),$(CFG_USER_TA_TARGETS)),$(supported-ta-targets)) +unsup-targets := $(filter-out $(ta-targets),$(CFG_USER_TA_TARGETS)) +ifneq (,$(unsup-targets)) +$(error CFG_USER_TA_TARGETS contains unsupported value(s): $(unsup-targets). Valid values: $(supported-ta-targets)) +endif + +ifneq ($(filter ta_arm32,$(ta-targets)),) +# Variables for ta-target/sm "ta_arm32" +CFG_ARM32_ta_arm32 := y +arch-bits-ta_arm32 := 32 +ta_arm32-platform-cppflags += $(arm32-platform-cppflags) +ta_arm32-platform-cflags += $(arm32-platform-cflags) +ta_arm32-platform-cflags += $(platform-cflags-optimization) +ta_arm32-platform-cflags += $(platform-cflags-debug-info) +ta_arm32-platform-cflags += -fpic + +# Thumb mode doesn't support function graph tracing due to missing +# frame pointer support required to trace function call chain. So +# rather compile in ARM mode if function tracing is enabled. +ifeq ($(CFG_FTRACE_SUPPORT),y) +ta_arm32-platform-cflags += $(arm32-platform-cflags-generic-arm) +else +ta_arm32-platform-cflags += $(arm32-platform-cflags-generic-thumb) +endif + +ifeq ($(arm32-platform-hard-float-enabled),y) +ta_arm32-platform-cflags += $(arm32-platform-cflags-hard-float) +else +ta_arm32-platform-cflags += $(arm32-platform-cflags-no-hard-float) +endif +ifeq ($(CFG_UNWIND),y) +ta_arm32-platform-cflags += -funwind-tables +endif +ta_arm32-platform-aflags += $(platform-aflags-generic) +ta_arm32-platform-aflags += $(platform-aflags-debug-info) +ta_arm32-platform-aflags += $(arm32-platform-aflags) + +ta_arm32-platform-cxxflags += -fpic +ta_arm32-platform-cxxflags += $(arm32-platform-cxxflags) +ta_arm32-platform-cxxflags += $(platform-cflags-optimization) +ta_arm32-platform-cxxflags += $(platform-cflags-debug-info) + +ifeq ($(arm32-platform-hard-float-enabled),y) +ta_arm32-platform-cxxflags += $(arm32-platform-cflags-hard-float) +else +ta_arm32-platform-cxxflags += $(arm32-platform-cflags-no-hard-float) +endif + +ta-mk-file-export-vars-ta_arm32 += CFG_ARM32_ta_arm32 +ta-mk-file-export-vars-ta_arm32 += ta_arm32-platform-cppflags +ta-mk-file-export-vars-ta_arm32 += ta_arm32-platform-cflags +ta-mk-file-export-vars-ta_arm32 += ta_arm32-platform-aflags +ta-mk-file-export-vars-ta_arm32 += ta_arm32-platform-cxxflags + +ta-mk-file-export-add-ta_arm32 += CROSS_COMPILE ?= arm-linux-gnueabihf-_nl_ +ta-mk-file-export-add-ta_arm32 += CROSS_COMPILE32 ?= $$(CROSS_COMPILE)_nl_ +ta-mk-file-export-add-ta_arm32 += CROSS_COMPILE_ta_arm32 ?= $$(CROSS_COMPILE32)_nl_ +ta-mk-file-export-add-ta_arm32 += COMPILER ?= gcc_nl_ +ta-mk-file-export-add-ta_arm32 += COMPILER_ta_arm32 ?= $$(COMPILER)_nl_ +ta-mk-file-export-add-ta_arm32 += PYTHON3 ?= python3_nl_ +endif + +ifneq ($(filter ta_arm64,$(ta-targets)),) +# Variables for ta-target/sm "ta_arm64" +CFG_ARM64_ta_arm64 := y +arch-bits-ta_arm64 := 64 +ta_arm64-platform-cppflags += $(arm64-platform-cppflags) +ta_arm64-platform-cflags += $(arm64-platform-cflags) +ta_arm64-platform-cflags += $(platform-cflags-optimization) +ta_arm64-platform-cflags += $(platform-cflags-debug-info) +ta_arm64-platform-cflags += -fpic +ta_arm64-platform-cflags += $(arm64-platform-cflags-generic) +ifeq ($(arm64-platform-hard-float-enabled),y) +ta_arm64-platform-cflags += $(arm64-platform-cflags-hard-float) +else +ta_arm64-platform-cflags += $(arm64-platform-cflags-no-hard-float) +endif +ta_arm64-platform-aflags += $(platform-aflags-generic) +ta_arm64-platform-aflags += $(platform-aflags-debug-info) +ta_arm64-platform-aflags += $(arm64-platform-aflags) + +ta_arm64-platform-cxxflags += -fpic +ta_arm64-platform-cxxflags += $(platform-cflags-optimization) +ta_arm64-platform-cxxflags += $(platform-cflags-debug-info) + +ifeq ($(CFG_TA_PAUTH),y) +bp-ta-opt := $(call cc-option,-mbranch-protection=pac-ret+leaf) +endif + +ifeq ($(CFG_TA_BTI),y) +bp-ta-opt := $(call cc-option,-mbranch-protection=bti) +endif + +ifeq (y-y,$(CFG_TA_PAUTH)-$(CFG_TA_BTI)) +bp-ta-opt := $(call cc-option,-mbranch-protection=pac-ret+leaf+bti) +endif + +ifeq (y,$(filter $(CFG_TA_BTI) $(CFG_TA_PAUTH),y)) +ifeq (,$(bp-ta-opt)) +$(error -mbranch-protection not supported) +endif +ta_arm64-platform-cflags += $(bp-ta-opt) +endif + +ta-mk-file-export-vars-ta_arm64 += CFG_ARM64_ta_arm64 +ta-mk-file-export-vars-ta_arm64 += ta_arm64-platform-cppflags +ta-mk-file-export-vars-ta_arm64 += ta_arm64-platform-cflags +ta-mk-file-export-vars-ta_arm64 += ta_arm64-platform-aflags +ta-mk-file-export-vars-ta_arm64 += ta_arm64-platform-cxxflags + +ta-mk-file-export-add-ta_arm64 += CROSS_COMPILE64 ?= $$(CROSS_COMPILE)_nl_ +ta-mk-file-export-add-ta_arm64 += CROSS_COMPILE_ta_arm64 ?= $$(CROSS_COMPILE64)_nl_ +ta-mk-file-export-add-ta_arm64 += COMPILER ?= gcc_nl_ +ta-mk-file-export-add-ta_arm64 += COMPILER_ta_arm64 ?= $$(COMPILER)_nl_ +ta-mk-file-export-add-ta_arm64 += PYTHON3 ?= python3_nl_ +endif + +# Set cross compiler prefix for each TA target +$(foreach sm, $(ta-targets), $(eval CROSS_COMPILE_$(sm) ?= $(CROSS_COMPILE$(arch-bits-$(sm))))) + +arm32-sysreg-txt = core/arch/arm/kernel/arm32_sysreg.txt +arm32-sysregs-$(arm32-sysreg-txt)-h := arm32_sysreg.h +arm32-sysregs-$(arm32-sysreg-txt)-s := arm32_sysreg.S +arm32-sysregs += $(arm32-sysreg-txt) + +ifeq ($(CFG_ARM_GICV3),y) +arm32-gicv3-sysreg-txt = core/arch/arm/kernel/arm32_gicv3_sysreg.txt +arm32-sysregs-$(arm32-gicv3-sysreg-txt)-h := arm32_gicv3_sysreg.h +arm32-sysregs-$(arm32-gicv3-sysreg-txt)-s := arm32_gicv3_sysreg.S +arm32-sysregs += $(arm32-gicv3-sysreg-txt) +endif + +arm32-sysregs-out := $(out-dir)/$(sm)/include/generated + +define process-arm32-sysreg +FORCE-GENSRC$(sm): $$(arm32-sysregs-out)/$$(arm32-sysregs-$(1)-h) +cleanfiles := $$(cleanfiles) $$(arm32-sysregs-out)/$$(arm32-sysregs-$(1)-h) + +$$(arm32-sysregs-out)/$$(arm32-sysregs-$(1)-h): $(1) scripts/arm32_sysreg.py + @$(cmd-echo-silent) ' GEN $$@' + $(q)mkdir -p $$(dir $$@) + $(q)scripts/arm32_sysreg.py --guard __$$(arm32-sysregs-$(1)-h) \ + < $$< > $$@ + +FORCE-GENSRC$(sm): $$(arm32-sysregs-out)/$$(arm32-sysregs-$(1)-s) +cleanfiles := $$(cleanfiles) $$(arm32-sysregs-out)/$$(arm32-sysregs-$(1)-s) + +$$(arm32-sysregs-out)/$$(arm32-sysregs-$(1)-s): $(1) scripts/arm32_sysreg.py + @$(cmd-echo-silent) ' GEN $$@' + $(q)mkdir -p $$(dir $$@) + $(q)scripts/arm32_sysreg.py --s_file < $$< > $$@ +endef #process-arm32-sysreg + +$(foreach sr, $(arm32-sysregs), $(eval $(call process-arm32-sysreg,$(sr)))) diff --git a/optee_os/core/arch/arm/cpu/cortex-a15.mk b/optee_os/core/arch/arm/cpu/cortex-a15.mk new file mode 100644 index 0000000..b01014e --- /dev/null +++ b/optee_os/core/arch/arm/cpu/cortex-a15.mk @@ -0,0 +1,10 @@ +$(call force,CFG_ARM32_core,y) +$(call force,CFG_ARM64_core,n) +$(call force,CFG_HWSUPP_MEM_PERM_WXN,y) +$(call force,CFG_HWSUPP_MEM_PERM_PXN,y) +arm32-platform-cpuarch := cortex-a15 +arm32-platform-cflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-aflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-cxxflags += -mcpu=$(arm32-platform-cpuarch) +# Program flow prediction may need manual enablement +CFG_ENABLE_SCTLR_Z ?= y diff --git a/optee_os/core/arch/arm/cpu/cortex-a5.mk b/optee_os/core/arch/arm/cpu/cortex-a5.mk new file mode 100644 index 0000000..11714dd --- /dev/null +++ b/optee_os/core/arch/arm/cpu/cortex-a5.mk @@ -0,0 +1,10 @@ +$(call force,CFG_ARM32_core,y) +$(call force,CFG_ARM64_core,n) +$(call force,CFG_WITH_LPAE,n) +$(call force,CFG_HWSUPP_MEM_PERM_WXN,n) +$(call force,CFG_HWSUPP_MEM_PERM_PXN,n) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,n) +arm32-platform-cpuarch := cortex-a5 +arm32-platform-cflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-aflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-cxxflags += -mcpu=$(arm32-platform-cpuarch) diff --git a/optee_os/core/arch/arm/cpu/cortex-a7.mk b/optee_os/core/arch/arm/cpu/cortex-a7.mk new file mode 100644 index 0000000..46d4d1a --- /dev/null +++ b/optee_os/core/arch/arm/cpu/cortex-a7.mk @@ -0,0 +1,9 @@ +$(call force,CFG_ARM32_core,y) +$(call force,CFG_ARM64_core,n) +$(call force,CFG_HWSUPP_MEM_PERM_WXN,y) +$(call force,CFG_HWSUPP_MEM_PERM_PXN,y) +$(call force,CFG_ENABLE_SCTLR_Z,n) +arm32-platform-cpuarch := cortex-a7 +arm32-platform-cflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-aflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-cxxflags += -mcpu=$(arm32-platform-cpuarch) diff --git a/optee_os/core/arch/arm/cpu/cortex-a9.mk b/optee_os/core/arch/arm/cpu/cortex-a9.mk new file mode 100644 index 0000000..6da50be --- /dev/null +++ b/optee_os/core/arch/arm/cpu/cortex-a9.mk @@ -0,0 +1,12 @@ +$(call force,CFG_ARM32_core,y) +$(call force,CFG_ARM64_core,n) +$(call force,CFG_WITH_LPAE,n) +$(call force,CFG_HWSUPP_MEM_PERM_WXN,n) +$(call force,CFG_HWSUPP_MEM_PERM_PXN,n) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,n) +arm32-platform-cpuarch := cortex-a9 +arm32-platform-cflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-aflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-cxxflags += -mcpu=$(arm32-platform-cpuarch) +# Program flow prediction may need manual enablement +CFG_ENABLE_SCTLR_Z ?= y diff --git a/optee_os/core/arch/arm/cpu/cortex-armv8-0.mk b/optee_os/core/arch/arm/cpu/cortex-armv8-0.mk new file mode 100644 index 0000000..624b7cf --- /dev/null +++ b/optee_os/core/arch/arm/cpu/cortex-armv8-0.mk @@ -0,0 +1,10 @@ +$(call force,CFG_HWSUPP_MEM_PERM_WXN,y) +$(call force,CFG_HWSUPP_MEM_PERM_PXN,y) +$(call force,CFG_ENABLE_SCTLR_RR,n) +$(call force,CFG_ENABLE_SCTLR_Z,n) +# cortex-a53 and cortex-a57 complies on arm32 architectures +arm32-platform-cpuarch := cortex-a53 +arm32-platform-cflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-aflags += -mcpu=$(arm32-platform-cpuarch) +arm32-platform-cxxflags += -mcpu=$(arm32-platform-cpuarch) +platform-flavor-armv8 := 1 diff --git a/optee_os/core/arch/arm/crypto/aes-gcm-ce.c b/optee_os/core/arch/arm/crypto/aes-gcm-ce.c new file mode 100644 index 0000000..054dda3 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/aes-gcm-ce.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aes_armv8a_ce.h" + +static void get_be_block(void *dst, const void *src) +{ + uint64_t *d = dst; + + d[1] = get_be64(src); + d[0] = get_be64((const uint8_t *)src + 8); +} + +static void put_be_block(void *dst, const void *src) +{ + const uint64_t *s = src; + + put_be64(dst, s[1]); + put_be64((uint8_t *)dst + 8, s[0]); +} + +static void ghash_reflect(uint64_t h[2], const uint64_t k[2]) +{ + uint64_t b = get_be64(k); + uint64_t a = get_be64(k + 1); + + h[0] = (a << 1) | (b >> 63); + h[1] = (b << 1) | (a >> 63); + if (b >> 63) + h[1] ^= 0xc200000000000000UL; +} + +void internal_aes_gcm_set_key(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *enc_key) +{ + uint64_t k[2] = { 0 }; + uint64_t h[2] = { 0 }; + + crypto_aes_enc_block(enc_key->data, sizeof(enc_key->data), + enc_key->rounds, state->ctr, k); + + ghash_reflect(state->ghash_key.h, k); + + internal_aes_gcm_gfmul(k, k, h); + ghash_reflect(state->ghash_key.h2, h); + + internal_aes_gcm_gfmul(k, h, h); + ghash_reflect(state->ghash_key.h3, h); + + internal_aes_gcm_gfmul(k, h, h); + ghash_reflect(state->ghash_key.h4, h); +} + +static void pmull_ghash_update(int num_blocks, uint64_t dg[2], + const uint8_t *src, + const struct internal_ghash_key *ghash_key, + const uint8_t *head) +{ +#ifdef CFG_HWSUPP_PMULT_64 + pmull_ghash_update_p64(num_blocks, dg, src, ghash_key, head); +#else + pmull_ghash_update_p8(num_blocks, dg, src, ghash_key, head); +#endif +} + +void internal_aes_gcm_ghash_update(struct internal_aes_gcm_state *state, + const void *head, const void *data, + size_t num_blocks) +{ + uint32_t vfp_state; + uint64_t dg[2]; + + get_be_block(dg, state->hash_state); + + vfp_state = thread_kernel_enable_vfp(); + + pmull_ghash_update(num_blocks, dg, data, &state->ghash_key, head); + + thread_kernel_disable_vfp(vfp_state); + + put_be_block(state->hash_state, dg); +} + +static void encrypt_pl(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, uint64_t dg[2], + const uint8_t *src, size_t num_blocks, uint8_t *dst) +{ + void *buf_cryp = state->buf_cryp; + + while (num_blocks) { + ce_aes_xor_block(buf_cryp, buf_cryp, src); + + pmull_ghash_update(1, dg, buf_cryp, &state->ghash_key, NULL); + memcpy(dst, buf_cryp, TEE_AES_BLOCK_SIZE); + + ce_aes_ecb_encrypt(buf_cryp, (const uint8_t *)state->ctr, + (const uint8_t *)ek->data, ek->rounds, + 1, 1); + internal_aes_gcm_inc_ctr(state); + + src += TEE_AES_BLOCK_SIZE; + dst += TEE_AES_BLOCK_SIZE; + num_blocks--; + } +} + +static void decrypt_pl(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, uint64_t dg[2], + const uint8_t *src, size_t num_blocks, uint8_t *dst) +{ + while (num_blocks) { + ce_aes_ctr_encrypt(dst, src, (const uint8_t *)ek->data, + ek->rounds, 1, (uint8_t *)state->ctr, 1); + pmull_ghash_update(1, dg, src, &state->ghash_key, NULL); + + src += TEE_AES_BLOCK_SIZE; + dst += TEE_AES_BLOCK_SIZE; + num_blocks--; + } +} + +#ifdef ARM64 +static void update_payload_2block(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + uint64_t dg[2], TEE_OperationMode mode, + const void *src, size_t num_blocks, void *dst) +{ + assert(num_blocks && !(num_blocks % 2)); + + if (mode == TEE_MODE_ENCRYPT) { + uint8_t ks[sizeof(state->buf_cryp) * 2] = { 0 }; + + /* + * ks holds the encrypted counters of the next two blocks. + * pmull_gcm_encrypt() uses this to encrypt the first two + * blocks. When pmull_gcm_encrypt() returns is ks updated + * with the encrypted counters of the next two blocks. As + * we're only keeping one of these blocks we throw away + * block number two consequently decreases the counter by + * one. + */ + memcpy(ks, state->buf_cryp, sizeof(state->buf_cryp)); + + pmull_gcm_load_round_keys(ek->data, ek->rounds); + pmull_gcm_encrypt_block(ks + sizeof(state->buf_cryp), + (uint8_t *)state->ctr, ek->rounds); + internal_aes_gcm_inc_ctr(state); + pmull_gcm_encrypt(num_blocks, dg, dst, src, &state->ghash_key, + state->ctr, NULL, ek->rounds, ks); + memcpy(state->buf_cryp, ks, TEE_AES_BLOCK_SIZE); + internal_aes_gcm_dec_ctr(state); + } else { + pmull_gcm_decrypt(num_blocks, dg, dst, src, &state->ghash_key, + state->ctr, ek->data, ek->rounds); + } +} + +/* Overriding the __weak function */ +void +internal_aes_gcm_update_payload_blocks(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + TEE_OperationMode mode, const void *src, + size_t num_blocks, void *dst) +{ + size_t nb = ROUNDDOWN(num_blocks, 2); + uint32_t vfp_state = 0; + uint64_t dg[2] = { 0 }; + + get_be_block(dg, state->hash_state); + vfp_state = thread_kernel_enable_vfp(); + + /* + * pmull_gcm_encrypt() and pmull_gcm_decrypt() can only handle + * blocks in multiples of two. + */ + if (nb) + update_payload_2block(state, ek, dg, mode, src, nb, dst); + + if (nb != num_blocks) { + /* There's a final block */ + const void *s = (const uint8_t *)src + nb * TEE_AES_BLOCK_SIZE; + void *d = (uint8_t *)dst + nb * TEE_AES_BLOCK_SIZE; + + if (mode == TEE_MODE_ENCRYPT) + encrypt_pl(state, ek, dg, s, 1, d); + else + decrypt_pl(state, ek, dg, s, 1, d); + } + + thread_kernel_disable_vfp(vfp_state); + put_be_block(state->hash_state, dg); +} +#endif /*ARM64*/ + +#ifdef ARM32 +/* Overriding the __weak function */ +void +internal_aes_gcm_update_payload_blocks(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + TEE_OperationMode mode, const void *src, + size_t num_blocks, void *dst) +{ + uint64_t dg[2] = { 0 }; + uint32_t vfp_state = 0; + + assert(!state->buf_pos && num_blocks); + get_be_block(dg, state->hash_state); + vfp_state = thread_kernel_enable_vfp(); + + if (mode == TEE_MODE_ENCRYPT) + encrypt_pl(state, ek, dg, src, num_blocks, dst); + else + decrypt_pl(state, ek, dg, src, num_blocks, dst); + + thread_kernel_disable_vfp(vfp_state); + put_be_block(state->hash_state, dg); +} +#endif diff --git a/optee_os/core/arch/arm/crypto/aes_armv8a_ce.c b/optee_os/core/arch/arm/crypto/aes_armv8a_ce.c new file mode 100644 index 0000000..725eacb --- /dev/null +++ b/optee_os/core/arch/arm/crypto/aes_armv8a_ce.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, 2020 Linaro Limited + * Copyright (C) 2013 Linaro Ltd + * Copyright (c) 2001-2007, Tom St Denis + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtom.org + */ + +/* + * AES cipher for ARMv8 with Crypto Extensions + */ + +#include +#include +#include +#include +#include +#include + +#include "aes_armv8a_ce.h" + +struct aes_block { + uint8_t b[TEE_AES_BLOCK_SIZE]; +}; + +static uint32_t ror32(uint32_t val, unsigned int shift) +{ + return (val >> shift) | (val << (32 - shift)); +} + +static void expand_enc_key(uint32_t *enc_key, size_t key_len) +{ + /* The AES key schedule round constants */ + static uint8_t const rcon[] = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + }; + unsigned int kwords = key_len / sizeof(uint32_t); + unsigned int i = 0; + + for (i = 0; i < sizeof(rcon); i++) { + uint32_t *rki = enc_key + i * kwords; + uint32_t *rko = rki + kwords; + + rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8) ^ + rcon[i] ^ rki[0]; + rko[1] = rko[0] ^ rki[1]; + rko[2] = rko[1] ^ rki[2]; + rko[3] = rko[2] ^ rki[3]; + + if (key_len == 24) { + if (i >= 7) + break; + rko[4] = rko[3] ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + } else if (key_len == 32) { + if (i >= 6) + break; + rko[4] = ce_aes_sub(rko[3]) ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + rko[6] = rko[5] ^ rki[6]; + rko[7] = rko[6] ^ rki[7]; + } + } +} + +static void make_dec_key(unsigned int round_count, + const struct aes_block *key_enc, + struct aes_block *key_dec) +{ + unsigned int i = 0; + unsigned int j = round_count; + + /* + * Generate the decryption keys for the Equivalent Inverse Cipher. + * This involves reversing the order of the round keys, and applying + * the Inverse Mix Columns transformation on all but the first and + * the last ones. + */ + j = round_count; + + key_dec[0] = key_enc[j]; + for (i = 1, j--; j > 0; i++, j--) + ce_aes_invert(key_dec + i, key_enc + j); + key_dec[i] = key_enc[0]; +} + +TEE_Result crypto_accel_aes_expand_keys(const void *key, size_t key_len, + void *enc_key, void *dec_key, + size_t expanded_key_len, + unsigned int *round_count) +{ + unsigned int num_rounds = 0; + uint32_t vfp_state = 0; + + if (!key || !enc_key) + return TEE_ERROR_BAD_PARAMETERS; + if (key_len != 16 && key_len != 24 && key_len != 32) + return TEE_ERROR_BAD_PARAMETERS; + if (!IS_ALIGNED_WITH_TYPE(enc_key, struct aes_block) || + !IS_ALIGNED_WITH_TYPE(dec_key, struct aes_block)) + return TEE_ERROR_BAD_PARAMETERS; + + num_rounds = 10 + ((key_len / 8) - 2) * 2; + + if (expanded_key_len < (num_rounds + 1) * sizeof(struct aes_block)) + return TEE_ERROR_BAD_PARAMETERS; + + *round_count = num_rounds; + memset(enc_key, 0, expanded_key_len); + memcpy(enc_key, key, key_len); + + vfp_state = thread_kernel_enable_vfp(); + + expand_enc_key(enc_key, key_len); + if (dec_key) + make_dec_key(num_rounds, enc_key, dec_key); + + thread_kernel_disable_vfp(vfp_state); + + return TEE_SUCCESS; +} + +void crypto_accel_aes_ecb_enc(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count) +{ + uint32_t vfp_state = 0; + + assert(out && in && key); + + vfp_state = thread_kernel_enable_vfp(); + ce_aes_ecb_encrypt(out, in, key, round_count, block_count, 1); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_aes_ecb_dec(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count) +{ + uint32_t vfp_state = 0; + + assert(out && in && key); + + vfp_state = thread_kernel_enable_vfp(); + ce_aes_ecb_decrypt(out, in, key, round_count, block_count, 1); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_aes_cbc_enc(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && iv); + + vfp_state = thread_kernel_enable_vfp(); + ce_aes_cbc_encrypt(out, in, key, round_count, block_count, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_aes_cbc_dec(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && iv); + + vfp_state = thread_kernel_enable_vfp(); + ce_aes_cbc_decrypt(out, in, key, round_count, block_count, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_aes_ctr_be_enc(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && iv); + + vfp_state = thread_kernel_enable_vfp(); + ce_aes_ctr_encrypt(out, in, key, round_count, block_count, iv, 1); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_aes_xts_enc(void *out, const void *in, const void *key1, + unsigned int round_count, + unsigned int block_count, const void *key2, + void *tweak) +{ + uint32_t vfp_state = 0; + + assert(out && in && key1 && key2 && tweak); + + vfp_state = thread_kernel_enable_vfp(); + ce_aes_xts_encrypt(out, in, key1, round_count, block_count, key2, + tweak); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_aes_xts_dec(void *out, const void *in, const void *key1, + unsigned int round_count, + unsigned int block_count, const void *key2, + void *tweak) +{ + uint32_t vfp_state = 0; + + assert(out && in && key1 && key2 && tweak); + + vfp_state = thread_kernel_enable_vfp(); + ce_aes_xts_decrypt(out, in, key1, round_count, block_count, key2, + tweak); + thread_kernel_disable_vfp(vfp_state); +} diff --git a/optee_os/core/arch/arm/crypto/aes_armv8a_ce.h b/optee_os/core/arch/arm/crypto/aes_armv8a_ce.h new file mode 100644 index 0000000..fccc27c --- /dev/null +++ b/optee_os/core/arch/arm/crypto/aes_armv8a_ce.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020 Linaro Limited + */ + +#ifndef __AES_ARMV8_CE_H +#define __AES_ARMV8_CE_H + +#include + +/* Prototypes for assembly functions */ +uint32_t ce_aes_sub(uint32_t in); +void ce_aes_invert(void *dst, const void *src); +void ce_aes_ecb_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + int rounds, int blocks, int first); +void ce_aes_ecb_decrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + int rounds, int blocks, int first); +void ce_aes_cbc_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + int rounds, int blocks, uint8_t iv[]); +void ce_aes_cbc_decrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + int rounds, int blocks, uint8_t iv[]); +void ce_aes_ctr_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + int rounds, int blocks, uint8_t ctr[], int first); +void ce_aes_xts_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk1[], + int rounds, int blocks, uint8_t const rk2[], + uint8_t iv[]); +void ce_aes_xts_decrypt(uint8_t out[], uint8_t const in[], uint8_t const rk1[], + int rounds, int blocks, uint8_t const rk2[], + uint8_t iv[]); +void ce_aes_xor_block(uint8_t out[], uint8_t const op1[], uint8_t const op2[]); + +#endif /*__AES_ARMV8_CE_H*/ diff --git a/optee_os/core/arch/arm/crypto/aes_modes_armv8a_ce_a32.S b/optee_os/core/arch/arm/crypto/aes_modes_armv8a_ce_a32.S new file mode 100644 index 0000000..30a431a --- /dev/null +++ b/optee_os/core/arch/arm/crypto/aes_modes_armv8a_ce_a32.S @@ -0,0 +1,531 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, 2020 Linaro Limited + * Copyright (C) 2015 Linaro Ltd + * + * aes-ce-core.S - AES in CBC/CTR/XTS mode using ARMv8 Crypto Extensions + */ + +#include + + .fpu crypto-neon-fp-armv8 + + .macro enc_round, state, key + aese.8 \state, \key + aesmc.8 \state, \state + .endm + + .macro dec_round, state, key + aesd.8 \state, \key + aesimc.8 \state, \state + .endm + + .macro enc_dround, key1, key2 + enc_round q0, \key1 + enc_round q0, \key2 + .endm + + .macro dec_dround, key1, key2 + dec_round q0, \key1 + dec_round q0, \key2 + .endm + + .macro enc_fround, key1, key2, key3 + enc_round q0, \key1 + aese.8 q0, \key2 + veor q0, q0, \key3 + .endm + + .macro dec_fround, key1, key2, key3 + dec_round q0, \key1 + aesd.8 q0, \key2 + veor q0, q0, \key3 + .endm + + .macro enc_dround_3x, key1, key2 + enc_round q0, \key1 + enc_round q1, \key1 + enc_round q2, \key1 + enc_round q0, \key2 + enc_round q1, \key2 + enc_round q2, \key2 + .endm + + .macro dec_dround_3x, key1, key2 + dec_round q0, \key1 + dec_round q1, \key1 + dec_round q2, \key1 + dec_round q0, \key2 + dec_round q1, \key2 + dec_round q2, \key2 + .endm + + .macro enc_fround_3x, key1, key2, key3 + enc_round q0, \key1 + enc_round q1, \key1 + enc_round q2, \key1 + aese.8 q0, \key2 + aese.8 q1, \key2 + aese.8 q2, \key2 + veor q0, q0, \key3 + veor q1, q1, \key3 + veor q2, q2, \key3 + .endm + + .macro dec_fround_3x, key1, key2, key3 + dec_round q0, \key1 + dec_round q1, \key1 + dec_round q2, \key1 + aesd.8 q0, \key2 + aesd.8 q1, \key2 + aesd.8 q2, \key2 + veor q0, q0, \key3 + veor q1, q1, \key3 + veor q2, q2, \key3 + .endm + + .macro do_block, dround, fround + cmp r3, #12 @ which key size? + vld1.8 {q10-q11}, [ip]! + \dround q8, q9 + vld1.8 {q12-q13}, [ip]! + \dround q10, q11 + vld1.8 {q10-q11}, [ip]! + \dround q12, q13 + vld1.8 {q12-q13}, [ip]! + \dround q10, q11 + blo 0f @ AES-128: 10 rounds + vld1.8 {q10-q11}, [ip]! + \dround q12, q13 + beq 1f @ AES-192: 12 rounds + vld1.8 {q12-q13}, [ip] + \dround q10, q11 +0: \fround q12, q13, q14 + bx lr + +1: \fround q10, q11, q14 + bx lr + .endm + + /* + * Internal, non-AAPCS compliant functions that implement the core + * AES transforms. These should preserve all registers except q0 - + * q2 and ip. + * Arguments: + * q0 : first in/output block + * q1 : second in/output block (_3x version only) + * q2 : third in/output block (_3x version only) + * q8 : first round key + * q9 : secound round key + * q14 : final round key + * r2 : address of round key array + * r3 : number of rounds + */ + .section .text.ce_aes_helpers + .align 6 +aes_encrypt: + add ip, r2, #32 @ 3rd round key +.Laes_encrypt_tweak: + do_block enc_dround, enc_fround + + .align 6 +aes_decrypt: + add ip, r2, #32 @ 3rd round key + do_block dec_dround, dec_fround + + .align 6 +aes_encrypt_3x: + add ip, r2, #32 @ 3rd round key + do_block enc_dround_3x, enc_fround_3x + + .align 6 +aes_decrypt_3x: + add ip, r2, #32 @ 3rd round key + do_block dec_dround_3x, dec_fround_3x + + .macro prepare_key, rk, rounds + add ip, \rk, \rounds, lsl #4 + vld1.8 {q8-q9}, [\rk] @ load first 2 round keys + vld1.8 {q14}, [ip] @ load last round key + .endm + + /* + * void ce_aes_ecb_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * int first) + */ +FUNC ce_aes_ecb_encrypt , : + push {r4, lr} + ldr r4, [sp, #8] + prepare_key r2, r3 +.Lecbencloop3x: + subs r4, r4, #3 + bmi .Lecbenc1x + vld1.8 {q0-q1}, [r1]! + vld1.8 {q2}, [r1]! + bl aes_encrypt_3x + vst1.8 {q0-q1}, [r0]! + vst1.8 {q2}, [r0]! + b .Lecbencloop3x +.Lecbenc1x: + adds r4, r4, #3 + beq .Lecbencout +.Lecbencloop: + vld1.8 {q0}, [r1]! + bl aes_encrypt + vst1.8 {q0}, [r0]! + subs r4, r4, #1 + bne .Lecbencloop +.Lecbencout: + pop {r4, pc} +END_FUNC ce_aes_ecb_encrypt + + /* + * void ce_aes_ecb_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * int first) + */ +FUNC ce_aes_ecb_decrypt , : + push {r4, lr} + ldr r4, [sp, #8] + prepare_key r2, r3 +.Lecbdecloop3x: + subs r4, r4, #3 + bmi .Lecbdec1x + vld1.8 {q0-q1}, [r1]! + vld1.8 {q2}, [r1]! + bl aes_decrypt_3x + vst1.8 {q0-q1}, [r0]! + vst1.8 {q2}, [r0]! + b .Lecbdecloop3x +.Lecbdec1x: + adds r4, r4, #3 + beq .Lecbdecout +.Lecbdecloop: + vld1.8 {q0}, [r1]! + bl aes_decrypt + vst1.8 {q0}, [r0]! + subs r4, r4, #1 + bne .Lecbdecloop +.Lecbdecout: + pop {r4, pc} +END_FUNC ce_aes_ecb_decrypt + + /* + * void ce_aes_cbc_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * uint8_t iv[]) + */ +FUNC ce_aes_cbc_encrypt , : + push {r4-r6, lr} + ldrd r4, r5, [sp, #16] + vld1.8 {q0}, [r5] + prepare_key r2, r3 +.Lcbcencloop: + vld1.8 {q1}, [r1]! @ get next pt block + veor q0, q0, q1 @ ..and xor with iv + bl aes_encrypt + vst1.8 {q0}, [r0]! + subs r4, r4, #1 + bne .Lcbcencloop + vst1.8 {q0}, [r5] + pop {r4-r6, pc} +END_FUNC ce_aes_cbc_encrypt + + /* + * void ce_aes_cbc_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * uint8_t iv[]) + */ +FUNC ce_aes_cbc_decrypt , : + push {r4-r6, lr} + ldrd r4, r5, [sp, #16] + vld1.8 {q6}, [r5] @ keep iv in q6 + prepare_key r2, r3 +.Lcbcdecloop3x: + subs r4, r4, #3 + bmi .Lcbcdec1x + vld1.8 {q0-q1}, [r1]! + vld1.8 {q2}, [r1]! + vmov q3, q0 + vmov q4, q1 + vmov q5, q2 + bl aes_decrypt_3x + veor q0, q0, q6 + veor q1, q1, q3 + veor q2, q2, q4 + vmov q6, q5 + vst1.8 {q0-q1}, [r0]! + vst1.8 {q2}, [r0]! + b .Lcbcdecloop3x +.Lcbcdec1x: + adds r4, r4, #3 + beq .Lcbcdecout + vmov q15, q14 @ preserve last round key +.Lcbcdecloop: + vld1.8 {q0}, [r1]! @ get next ct block + veor q14, q15, q6 @ combine prev ct with last key + vmov q6, q0 + bl aes_decrypt + vst1.8 {q0}, [r0]! + subs r4, r4, #1 + bne .Lcbcdecloop +.Lcbcdecout: + vst1.8 {q6}, [r5] @ keep iv in q6 + pop {r4-r6, pc} +END_FUNC ce_aes_cbc_decrypt + + /* + * void ce_aes_ctr_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * uint8_t ctr[], int first) + */ +FUNC ce_aes_ctr_encrypt , : + push {r4-r6, lr} + ldrd r4, r5, [sp, #16] + vld1.8 {q6}, [r5] @ load ctr + prepare_key r2, r3 + vmov r6, s27 @ keep swabbed ctr in r6 + rev r6, r6 + cmn r6, r4 @ 32 bit overflow? + bcs .Lctrloop +.Lctrloop3x: + subs r4, r4, #3 + bmi .Lctr1x + add r6, r6, #1 + vmov q0, q6 + vmov q1, q6 + rev ip, r6 + add r6, r6, #1 + vmov q2, q6 + vmov s7, ip + rev ip, r6 + add r6, r6, #1 + vmov s11, ip + vld1.8 {q3-q4}, [r1]! + vld1.8 {q5}, [r1]! + bl aes_encrypt_3x + veor q0, q0, q3 + veor q1, q1, q4 + veor q2, q2, q5 + rev ip, r6 + vst1.8 {q0-q1}, [r0]! + vst1.8 {q2}, [r0]! + vmov s27, ip + b .Lctrloop3x +.Lctr1x: + adds r4, r4, #3 + beq .Lctrout +.Lctrloop: + vmov q0, q6 + bl aes_encrypt + subs r4, r4, #1 + bmi .Lctrtailblock @ blocks < 0 means tail block + vld1.8 {q3}, [r1]! + veor q3, q0, q3 + vst1.8 {q3}, [r0]! + + adds r6, r6, #1 @ increment BE ctr + rev ip, r6 + vmov s27, ip + bcs .Lctrcarry + teq r4, #0 + bne .Lctrloop +.Lctrout: + vst1.8 {q6}, [r5] + pop {r4-r6, pc} + +.Lctrtailblock: + vst1.8 {q0}, [r0, :64] @ return just the key stream + pop {r4-r6, pc} + +.Lctrcarry: + .irp sreg, s26, s25, s24 + vmov ip, \sreg @ load next word of ctr + rev ip, ip @ ... to handle the carry + adds ip, ip, #1 + rev ip, ip + vmov \sreg, ip + bcc 0f + .endr +0: teq r4, #0 + beq .Lctrout + b .Lctrloop +END_FUNC ce_aes_ctr_encrypt + + /* + * void ce_aes_xts_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], int rounds, int blocks, + * uint8_t const rk2[], uint8_t iv[]) + * void ce_aes_xts_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], int rounds, int blocks, + * uint8_t const rk2[], uint8_t iv[]); + */ + + .macro next_tweak, out, in, const, tmp + vshr.s64 \tmp, \in, #63 + vand \tmp, \tmp, \const + vadd.u64 \out, \in, \in + vext.8 \tmp, \tmp, \tmp, #8 + veor \out, \out, \tmp + .endm + +LOCAL_FUNC ce_aes_xts_init , : + vldr d14, .Lxts_mul_x + vldr d15, .Lxts_mul_x + 8 + + ldr r4, [sp, #16] @ load args + ldr r5, [sp, #24] + vld1.8 {q0}, [r5] @ load iv + + @ Encrypt the IV in q0 with the second AES key. This should only + @ be done at the start of a block. + ldr r6, [sp, #20] @ load AES key 2 + prepare_key r6, r3 + add ip, r6, #32 @ 3rd round key of key 2 + b .Laes_encrypt_tweak @ tail call + + .align 3 +.Lxts_mul_x: + .quad 1, 0x87 +END_FUNC ce_aes_xts_init + +FUNC ce_aes_xts_encrypt , : + push {r4-r6, lr} + + bl ce_aes_xts_init @ run shared prologue + prepare_key r2, r3 + vmov q3, q0 + + teq r6, #0 @ start of a block? + bne .Lxtsenc3x + +.Lxtsencloop3x: + next_tweak q3, q3, q7, q6 +.Lxtsenc3x: + subs r4, r4, #3 + bmi .Lxtsenc1x + vld1.8 {q0-q1}, [r1]! @ get 3 pt blocks + vld1.8 {q2}, [r1]! + next_tweak q4, q3, q7, q6 + veor q0, q0, q3 + next_tweak q5, q4, q7, q6 + veor q1, q1, q4 + veor q2, q2, q5 + bl aes_encrypt_3x + veor q0, q0, q3 + veor q1, q1, q4 + veor q2, q2, q5 + vst1.8 {q0-q1}, [r0]! @ write 3 ct blocks + vst1.8 {q2}, [r0]! + vmov q3, q5 + teq r4, #0 + beq .Lxtsencout + b .Lxtsencloop3x +.Lxtsenc1x: + adds r4, r4, #3 + beq .Lxtsencout +.Lxtsencloop: + vld1.8 {q0}, [r1]! + veor q0, q0, q3 + bl aes_encrypt + veor q0, q0, q3 + vst1.8 {q0}, [r0]! + subs r4, r4, #1 + beq .Lxtsencout + next_tweak q3, q3, q7, q6 + b .Lxtsencloop +.Lxtsencout: + next_tweak q3, q3, q7, q6 + vst1.8 {q3}, [r5] + pop {r4-r6, pc} +END_FUNC ce_aes_xts_encrypt + +FUNC ce_aes_xts_decrypt , : + push {r4-r6, lr} + + bl ce_aes_xts_init @ run shared prologue + prepare_key r2, r3 + vmov q3, q0 + + teq r6, #0 @ start of a block? + bne .Lxtsdec3x + +.Lxtsdecloop3x: + next_tweak q3, q3, q7, q6 +.Lxtsdec3x: + subs r4, r4, #3 + bmi .Lxtsdec1x + vld1.8 {q0-q1}, [r1]! @ get 3 ct blocks + vld1.8 {q2}, [r1]! + next_tweak q4, q3, q7, q6 + veor q0, q0, q3 + next_tweak q5, q4, q7, q6 + veor q1, q1, q4 + veor q2, q2, q5 + bl aes_decrypt_3x + veor q0, q0, q3 + veor q1, q1, q4 + veor q2, q2, q5 + vst1.8 {q0-q1}, [r0]! @ write 3 pt blocks + vst1.8 {q2}, [r0]! + vmov q3, q5 + teq r4, #0 + beq .Lxtsdecout + b .Lxtsdecloop3x +.Lxtsdec1x: + adds r4, r4, #3 + beq .Lxtsdecout +.Lxtsdecloop: + vld1.8 {q0}, [r1]! + veor q0, q0, q3 + add ip, r2, #32 @ 3rd round key + bl aes_decrypt + veor q0, q0, q3 + vst1.8 {q0}, [r0]! + subs r4, r4, #1 + beq .Lxtsdecout + next_tweak q3, q3, q7, q6 + b .Lxtsdecloop +.Lxtsdecout: + next_tweak q3, q3, q7, q6 + vst1.8 {q3}, [r5] + pop {r4-r6, pc} +END_FUNC ce_aes_xts_decrypt + + /* + * u32 ce_aes_sub(u32 input) - use the aese instruction to perform the + * AES sbox substitution on each byte in + * 'input' + */ +FUNC ce_aes_sub , : + vdup.32 q1, r0 + veor q0, q0, q0 + aese.8 q0, q1 + vmov r0, s0 + bx lr +END_FUNC ce_aes_sub + + /* + * void ce_aes_invert(void *dst, const void *src) + * + * perform the Inverse MixColumns operation on round key in + */ +FUNC ce_aes_invert , : + vld1.8 {q0}, [r1] + aesimc.8 q0, q0 + vst1.8 {q0}, [r0] + bx lr +END_FUNC ce_aes_invert + + /* + * void ce_aes_xor_block(uint8_t out[], uint8_t const op1[], + * uint8_t const op2[]); + */ +FUNC ce_aes_xor_block , : + vld1.8 {q0}, [r1] + vld1.8 {q1}, [r2] + veor q0, q0, q1 + vst1.8 {q0}, [r0] + bx lr +END_FUNC ce_aes_xor_block diff --git a/optee_os/core/arch/arm/crypto/aes_modes_armv8a_ce_a64.S b/optee_os/core/arch/arm/crypto/aes_modes_armv8a_ce_a64.S new file mode 100644 index 0000000..72dfea7 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/aes_modes_armv8a_ce_a64.S @@ -0,0 +1,664 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, 2020 Linaro Limited + * Copyright (C) 2013 Linaro Ltd + * + * - AES cipher for ARMv8 with Crypto Extensions + * - Chaining mode wrappers for AES + */ + +#include + + .arch armv8-a+crypto + + /* Preload all round keys */ + .macro load_round_keys, rounds, rk + cmp \rounds, #12 + blo 2222f /* 128 bits */ + beq 1111f /* 192 bits */ + ld1 {v17.16b-v18.16b}, [\rk], #32 +1111: ld1 {v19.16b-v20.16b}, [\rk], #32 +2222: ld1 {v21.16b-v24.16b}, [\rk], #64 + ld1 {v25.16b-v28.16b}, [\rk], #64 + ld1 {v29.16b-v31.16b}, [\rk] + .endm + + /* Prepare for encryption with key in rk[] */ + .macro enc_prepare, rounds, rk, ignore + load_round_keys \rounds, \rk + .endm + + /* Prepare for encryption (again) but with new key in rk[] */ + .macro enc_switch_key, rounds, rk, ignore + load_round_keys \rounds, \rk + .endm + + /* Prepare for decryption with key in rk[] */ + .macro dec_prepare, rounds, rk, ignore + load_round_keys \rounds, \rk + .endm + + .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3 + aes\de \i0\().16b, \k\().16b + aes\mc \i0\().16b, \i0\().16b + .ifnb \i1 + aes\de \i1\().16b, \k\().16b + aes\mc \i1\().16b, \i1\().16b + .ifnb \i3 + aes\de \i2\().16b, \k\().16b + aes\mc \i2\().16b, \i2\().16b + aes\de \i3\().16b, \k\().16b + aes\mc \i3\().16b, \i3\().16b + .endif + .endif + .endm + + /* Up to 4 interleaved encryption rounds with the same round key */ + .macro round_Nx, enc, k, i0, i1, i2, i3 + .ifc \enc, e + do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3 + .else + do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3 + .endif + .endm + + /* Up to 4 interleaved final rounds */ + .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3 + aes\de \i0\().16b, \k\().16b + .ifnb \i1 + aes\de \i1\().16b, \k\().16b + .ifnb \i3 + aes\de \i2\().16b, \k\().16b + aes\de \i3\().16b, \k\().16b + .endif + .endif + eor \i0\().16b, \i0\().16b, \k2\().16b + .ifnb \i1 + eor \i1\().16b, \i1\().16b, \k2\().16b + .ifnb \i3 + eor \i2\().16b, \i2\().16b, \k2\().16b + eor \i3\().16b, \i3\().16b, \k2\().16b + .endif + .endif + .endm + + /* Up to 4 interleaved blocks */ + .macro do_block_Nx, enc, rounds, i0, i1, i2, i3 + cmp \rounds, #12 + blo 2222f /* 128 bits */ + beq 1111f /* 192 bits */ + round_Nx \enc, v17, \i0, \i1, \i2, \i3 + round_Nx \enc, v18, \i0, \i1, \i2, \i3 +1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3 + round_Nx \enc, v20, \i0, \i1, \i2, \i3 +2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 + round_Nx \enc, \key, \i0, \i1, \i2, \i3 + .endr + fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3 + .endm + + .macro encrypt_block, in, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \in + .endm + + .macro encrypt_block2x, i0, i1, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1 + .endm + + .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 + .endm + + .macro decrypt_block, in, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \in + .endm + + .macro decrypt_block2x, i0, i1, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1 + .endm + + .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 + .endm + + +/* + * There are several ways to instantiate this code: + * - no interleave, all inline + * - 2-way interleave, 2x calls out of line (-DINTERLEAVE=2) + * - 2-way interleave, all inline (-DINTERLEAVE=2 -DINTERLEAVE_INLINE) + * - 4-way interleave, 4x calls out of line (-DINTERLEAVE=4) + * - 4-way interleave, all inline (-DINTERLEAVE=4 -DINTERLEAVE_INLINE) + * + * Macros imported by this code: + * - enc_prepare - setup NEON registers for encryption + * - dec_prepare - setup NEON registers for decryption + * - enc_switch_key - change to new key after having prepared for encryption + * - encrypt_block - encrypt a single block + * - decrypt block - decrypt a single block + * - encrypt_block2x - encrypt 2 blocks in parallel (if INTERLEAVE == 2) + * - decrypt_block2x - decrypt 2 blocks in parallel (if INTERLEAVE == 2) + * - encrypt_block4x - encrypt 4 blocks in parallel (if INTERLEAVE == 4) + * - decrypt_block4x - decrypt 4 blocks in parallel (if INTERLEAVE == 4) + */ + +#if defined(INTERLEAVE) && !defined(INTERLEAVE_INLINE) +#define FRAME_PUSH stp x29, x30, [sp,#-16]! ; mov x29, sp +#define FRAME_POP ldp x29, x30, [sp],#16 + +#if INTERLEAVE == 2 + +LOCAL_FUNC aes_encrypt_block2x , : + encrypt_block2x v0, v1, w3, x2, x6, w7 + ret +END_FUNC aes_encrypt_block2x + +LOCAL_FUNC aes_decrypt_block2x , : + decrypt_block2x v0, v1, w3, x2, x6, w7 + ret +END_FUNC aes_decrypt_block2x + +#elif INTERLEAVE == 4 + +LOCAL_FUNC aes_encrypt_block4x , : + encrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7 + ret +END_FUNC aes_encrypt_block4x + +LOCAL_FUNC aes_decrypt_block4x , : + decrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7 + ret +END_FUNC aes_decrypt_block4x + +#else +#error INTERLEAVE should equal 2 or 4 +#endif + + .macro do_encrypt_block2x + bl aes_encrypt_block2x + .endm + + .macro do_decrypt_block2x + bl aes_decrypt_block2x + .endm + + .macro do_encrypt_block4x + bl aes_encrypt_block4x + .endm + + .macro do_decrypt_block4x + bl aes_decrypt_block4x + .endm + +#else +#define FRAME_PUSH +#define FRAME_POP + + .macro do_encrypt_block2x + encrypt_block2x v0, v1, w3, x2, x6, w7 + .endm + + .macro do_decrypt_block2x + decrypt_block2x v0, v1, w3, x2, x6, w7 + .endm + + .macro do_encrypt_block4x + encrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7 + .endm + + .macro do_decrypt_block4x + decrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7 + .endm + +#endif + + /* + * uint32_t ce_aes_sub(uint32_t in) - use the aese instruction to + * perform the AES sbox substitution on each byte in 'input' + */ +FUNC ce_aes_sub , : + dup v1.4s, w0 + movi v0.16b, #0 + aese v0.16b, v1.16b + umov w0, v0.s[0] + ret +END_FUNC ce_aes_sub + + /* + * void ce_aes_invert(void *dst, const void *src); + */ +FUNC ce_aes_invert , : + ld1 {v0.16b}, [x1] + aesimc v1.16b, v0.16b + st1 {v1.16b}, [x0] + ret +END_FUNC ce_aes_invert + + /* + * ce_aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], + * int rounds, int blocks, int first) + */ +FUNC ce_aes_ecb_encrypt , : + FRAME_PUSH + cbz w5, .LecbencloopNx + + enc_prepare w3, x2, x5 + +.LecbencloopNx: +#if INTERLEAVE >= 2 + subs w4, w4, #INTERLEAVE + bmi .Lecbenc1x +#if INTERLEAVE == 2 + ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 pt blocks */ + do_encrypt_block2x + st1 {v0.16b-v1.16b}, [x0], #32 +#else + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ + do_encrypt_block4x + st1 {v0.16b-v3.16b}, [x0], #64 +#endif + b .LecbencloopNx +.Lecbenc1x: + adds w4, w4, #INTERLEAVE + beq .Lecbencout +#endif +.Lecbencloop: + ld1 {v0.16b}, [x1], #16 /* get next pt block */ + encrypt_block v0, w3, x2, x5, w6 + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lecbencloop +.Lecbencout: + FRAME_POP + ret +END_FUNC ce_aes_ecb_encrypt + + /* + * ce_aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], + * int rounds, int blocks, int first) + */ +FUNC ce_aes_ecb_decrypt , : + FRAME_PUSH + cbz w5, .LecbdecloopNx + + dec_prepare w3, x2, x5 + +.LecbdecloopNx: +#if INTERLEAVE >= 2 + subs w4, w4, #INTERLEAVE + bmi .Lecbdec1x +#if INTERLEAVE == 2 + ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 ct blocks */ + do_decrypt_block2x + st1 {v0.16b-v1.16b}, [x0], #32 +#else + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ + do_decrypt_block4x + st1 {v0.16b-v3.16b}, [x0], #64 +#endif + b .LecbdecloopNx +.Lecbdec1x: + adds w4, w4, #INTERLEAVE + beq .Lecbdecout +#endif +.Lecbdecloop: + ld1 {v0.16b}, [x1], #16 /* get next ct block */ + decrypt_block v0, w3, x2, x5, w6 + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lecbdecloop +.Lecbdecout: + FRAME_POP + ret +END_FUNC ce_aes_ecb_decrypt + + /* + * void ce_aes_cbc_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * uint8_t iv[]) + */ +FUNC ce_aes_cbc_encrypt , : + ld1 {v4.16b}, [x5] /* get iv */ + enc_prepare w3, x2, x6 + +.Lcbcencloop4x: + subs w4, w4, #4 + bmi .Lcbcenc1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ + eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */ + encrypt_block v0, w3, x2, x6, w7 + eor v1.16b, v1.16b, v0.16b + encrypt_block v1, w3, x2, x6, w7 + eor v2.16b, v2.16b, v1.16b + encrypt_block v2, w3, x2, x6, w7 + eor v3.16b, v3.16b, v2.16b + encrypt_block v3, w3, x2, x6, w7 + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v3.16b + b .Lcbcencloop4x +.Lcbcenc1x: + adds w4, w4, #4 + beq .Lcbcencout +.Lcbcencloop: + ld1 {v0.16b}, [x1], #16 /* get next pt block */ + eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */ + encrypt_block v4, w3, x2, x6, w7 + st1 {v4.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lcbcencloop +.Lcbcencout: + st1 {v4.16b}, [x5] /* return iv */ + ret +END_FUNC ce_aes_cbc_encrypt + + /* + * void ce_aes_cbc_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * uint8_t iv[]) + */ +FUNC ce_aes_cbc_decrypt , : + stp x29, x30, [sp, #-16]! + mov x29, sp + + ld1 {v7.16b}, [x5] /* get iv */ + dec_prepare w3, x2, x6 + +.LcbcdecloopNx: + subs w4, w4, #4 + bmi .Lcbcdec1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ + mov v4.16b, v0.16b + mov v5.16b, v1.16b + mov v6.16b, v2.16b + bl aes_decrypt_block4x + sub x1, x1, #16 + eor v0.16b, v0.16b, v7.16b + eor v1.16b, v1.16b, v4.16b + ld1 {v7.16b}, [x1], #16 /* reload 1 ct block */ + eor v2.16b, v2.16b, v5.16b + eor v3.16b, v3.16b, v6.16b + st1 {v0.16b-v3.16b}, [x0], #64 + b .LcbcdecloopNx +.Lcbcdec1x: + adds w4, w4, #4 + beq .Lcbcdecout +.Lcbcdecloop: + ld1 {v1.16b}, [x1], #16 /* get next ct block */ + mov v0.16b, v1.16b /* ...and copy to v0 */ + decrypt_block v0, w3, x2, x6, w7 + eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */ + mov v7.16b, v1.16b /* ct is next iv */ + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lcbcdecloop +.Lcbcdecout: + st1 {v7.16b}, [x5] /* return iv */ + ldp x29, x30, [sp], #16 + ret +END_FUNC ce_aes_cbc_decrypt + + + /* + * void ce_aes_ctr_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], int rounds, int blocks, + * uint8_t ctr[], int first) + */ +FUNC ce_aes_ctr_encrypt , : + stp x29, x30, [sp, #-16]! + mov x29, sp + + enc_prepare w3, x2, x6 + ld1 {v4.16b}, [x5] + + umov x6, v4.d[1] /* keep swabbed ctr in reg */ + rev x6, x6 + cmn w6, w4 /* 32 bit overflow? */ + bcs .Lctrloop +.LctrloopNx: + subs w4, w4, #4 + bmi .Lctr1x + add w7, w6, #1 + mov v0.16b, v4.16b + add w8, w6, #2 + mov v1.16b, v4.16b + add w9, w6, #3 + mov v2.16b, v4.16b + rev w7, w7 + mov v3.16b, v4.16b + rev w8, w8 + mov v1.s[3], w7 + rev w9, w9 + mov v2.s[3], w8 + mov v3.s[3], w9 + ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */ + bl aes_encrypt_block4x + eor v0.16b, v5.16b, v0.16b + ld1 {v5.16b}, [x1], #16 /* get 1 input block */ + eor v1.16b, v6.16b, v1.16b + eor v2.16b, v7.16b, v2.16b + eor v3.16b, v5.16b, v3.16b + st1 {v0.16b-v3.16b}, [x0], #64 + add x6, x6, #4 + rev x7, x6 + ins v4.d[1], x7 + cbz w4, .Lctrout + b .LctrloopNx +.Lctr1x: + adds w4, w4, #4 + beq .Lctrout +.Lctrloop: + mov v0.16b, v4.16b + encrypt_block v0, w3, x2, x8, w7 + + adds x6, x6, #1 /* increment BE ctr */ + rev x7, x6 + ins v4.d[1], x7 + bcs .Lctrcarry /* overflow? */ + +.Lctrcarrydone: + subs w4, w4, #1 + bmi .Lctrtailblock /* blocks <0 means tail block */ + ld1 {v3.16b}, [x1], #16 + eor v3.16b, v0.16b, v3.16b + st1 {v3.16b}, [x0], #16 + bne .Lctrloop + +.Lctrout: + st1 {v4.16b}, [x5] /* return next CTR value */ + ldp x29, x30, [sp], #16 + ret + +.Lctrtailblock: + st1 {v0.16b}, [x0] + ldp x29, x30, [sp], #16 + ret + +.Lctrcarry: + umov x7, v4.d[0] /* load upper word of ctr */ + rev x7, x7 /* ... to handle the carry */ + add x7, x7, #1 + rev x7, x7 + ins v4.d[0], x7 + b .Lctrcarrydone +END_FUNC ce_aes_ctr_encrypt + + + .macro next_tweak, out, in, const, tmp + sshr \tmp\().2d, \in\().2d, #63 + and \tmp\().16b, \tmp\().16b, \const\().16b + add \out\().2d, \in\().2d, \in\().2d + ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8 + eor \out\().16b, \out\().16b, \tmp\().16b + .endm + + /* + * void ce_aes_xts_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], int rounds, int blocks, + * uint8_t const rk2[], uint8_t iv[]) + */ +FUNC ce_aes_xts_encrypt , : + FRAME_PUSH + + ld1 {v4.16b}, [x6] + enc_prepare w3, x5, x6 + encrypt_block v4, w3, x5, x6, w7 /* first tweak */ + enc_switch_key w3, x2, x6 + ldr q7, .Lxts_mul_x + b .LxtsencNx + +.LxtsencloopNx: + ldr q7, .Lxts_mul_x + next_tweak v4, v4, v7, v8 +.LxtsencNx: +#if INTERLEAVE >= 2 + subs w4, w4, #INTERLEAVE + bmi .Lxtsenc1x +#if INTERLEAVE == 2 + ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 pt blocks */ + next_tweak v5, v4, v7, v8 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + do_encrypt_block2x + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + st1 {v0.16b-v1.16b}, [x0], #32 + cbz w4, .LxtsencoutNx + next_tweak v4, v5, v7, v8 + b .LxtsencNx +.LxtsencoutNx: + mov v4.16b, v5.16b + b .Lxtsencout +#else + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ + next_tweak v5, v4, v7, v8 + eor v0.16b, v0.16b, v4.16b + next_tweak v6, v5, v7, v8 + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + next_tweak v7, v6, v7, v8 + eor v3.16b, v3.16b, v7.16b + do_encrypt_block4x + eor v3.16b, v3.16b, v7.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v7.16b + cbz w4, .Lxtsencout + b .LxtsencloopNx +#endif +.Lxtsenc1x: + adds w4, w4, #INTERLEAVE + beq .Lxtsencout +#endif +.Lxtsencloop: + ld1 {v1.16b}, [x1], #16 + eor v0.16b, v1.16b, v4.16b + encrypt_block v0, w3, x2, x6, w7 + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + beq .Lxtsencout + next_tweak v4, v4, v7, v8 + b .Lxtsencloop +.Lxtsencout: + next_tweak v4, v4, v7, v8 + st1 {v4.16b}, [x6], #16 + FRAME_POP + ret + + .align 4 +.Lxts_mul_x: + .word 1, 0, 0x87, 0 +END_FUNC ce_aes_xts_encrypt + + /* + * void ce_aes_xts_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], int rounds, int blocks, + * uint8_t const rk2[], uint8_t iv[]) + */ +FUNC ce_aes_xts_decrypt , : + FRAME_PUSH + + ld1 {v4.16b}, [x6] + enc_prepare w3, x5, x6 + encrypt_block v4, w3, x5, x6, w7 /* first tweak */ + dec_prepare w3, x2, x6 + ldr q7, .Lxts_mul_x + b .LxtsdecNx + +.LxtsdecloopNx: + ldr q7, .Lxts_mul_x + next_tweak v4, v4, v7, v8 +.LxtsdecNx: +#if INTERLEAVE >= 2 + subs w4, w4, #INTERLEAVE + bmi .Lxtsdec1x +#if INTERLEAVE == 2 + ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 ct blocks */ + next_tweak v5, v4, v7, v8 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + do_decrypt_block2x + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + st1 {v0.16b-v1.16b}, [x0], #32 + cbz w4, .LxtsdecoutNx + next_tweak v4, v5, v7, v8 + b .LxtsdecNx +.LxtsdecoutNx: + mov v4.16b, v5.16b + b .Lxtsdecout +#else + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ + next_tweak v5, v4, v7, v8 + eor v0.16b, v0.16b, v4.16b + next_tweak v6, v5, v7, v8 + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + next_tweak v7, v6, v7, v8 + eor v3.16b, v3.16b, v7.16b + do_decrypt_block4x + eor v3.16b, v3.16b, v7.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v7.16b + cbz w4, .Lxtsdecout + b .LxtsdecloopNx +#endif +.Lxtsdec1x: + adds w4, w4, #INTERLEAVE + beq .Lxtsdecout +#endif +.Lxtsdecloop: + ld1 {v1.16b}, [x1], #16 + eor v0.16b, v1.16b, v4.16b + decrypt_block v0, w3, x2, x6, w7 + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + beq .Lxtsdecout + next_tweak v4, v4, v7, v8 + b .Lxtsdecloop +.Lxtsdecout: + FRAME_POP + next_tweak v4, v4, v7, v8 + st1 {v4.16b}, [x6], #16 + ret +END_FUNC ce_aes_xts_decrypt + + /* + * void ce_aes_xor_block(uint8_t out[], uint8_t const op1[], + * uint8_t const op2[]); + */ +FUNC ce_aes_xor_block , : + ld1 {v0.16b}, [x1] + ld1 {v1.16b}, [x2] + eor v0.16b, v0.16b, v1.16b + st1 {v0.16b}, [x0] + ret +END_FUNC ce_aes_xor_block + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/crypto/ghash-ce-core_a32.S b/optee_os/core/arch/arm/crypto/ghash-ce-core_a32.S new file mode 100644 index 0000000..08d4665 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/ghash-ce-core_a32.S @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Accelerated GHASH implementation with ARMv8 PMULL instructions. + * + * Copyright (C) 2014 - 2017 Linaro Ltd. + */ + +#include +#include + +#define CPU_LE(x...) x + + SHASH .req q0 + T1 .req q1 + XL .req q2 + XM .req q3 + XH .req q4 + IN1 .req q4 + + SHASH_L .req d0 + SHASH_H .req d1 + T1_L .req d2 + T1_H .req d3 + XL_L .req d4 + XL_H .req d5 + XM_L .req d6 + XM_H .req d7 + XH_L .req d8 + + t0l .req d10 + t0h .req d11 + t1l .req d12 + t1h .req d13 + t2l .req d14 + t2h .req d15 + t3l .req d16 + t3h .req d17 + t4l .req d18 + t4h .req d19 + + t0q .req q5 + t1q .req q6 + t2q .req q7 + t3q .req q8 + t4q .req q9 + T2 .req q9 + + s1l .req d20 + s1h .req d21 + s2l .req d22 + s2h .req d23 + s3l .req d24 + s3h .req d25 + s4l .req d26 + s4h .req d27 + + MASK .req d28 + SHASH2_p8 .req d28 + + k16 .req d29 + k32 .req d30 + k48 .req d31 + SHASH2_p64 .req d31 + + .text + .fpu crypto-neon-fp-armv8 + + .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4 + vmull.p64 \rd, \rn, \rm + .endm + + /* + * This implementation of 64x64 -> 128 bit polynomial multiplication + * using vmull.p8 instructions (8x8 -> 16) is taken from the paper + * "Fast Software Polynomial Multiplication on ARM Processors Using + * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and + * Ricardo Dahab (https://hal.inria.fr/hal-01506572) + * + * It has been slightly tweaked for in-order performance, and to allow + * 'rq' to overlap with 'ad' or 'bd'. + */ + .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l + vext.8 t0l, \ad, \ad, #1 @ A1 + .ifc \b1, t4l + vext.8 t4l, \bd, \bd, #1 @ B1 + .endif + vmull.p8 t0q, t0l, \bd @ F = A1*B + vext.8 t1l, \ad, \ad, #2 @ A2 + vmull.p8 t4q, \ad, \b1 @ E = A*B1 + .ifc \b2, t3l + vext.8 t3l, \bd, \bd, #2 @ B2 + .endif + vmull.p8 t1q, t1l, \bd @ H = A2*B + vext.8 t2l, \ad, \ad, #3 @ A3 + vmull.p8 t3q, \ad, \b2 @ G = A*B2 + veor t0q, t0q, t4q @ L = E + F + .ifc \b3, t4l + vext.8 t4l, \bd, \bd, #3 @ B3 + .endif + vmull.p8 t2q, t2l, \bd @ J = A3*B + veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 + veor t1q, t1q, t3q @ M = G + H + .ifc \b4, t3l + vext.8 t3l, \bd, \bd, #4 @ B4 + .endif + vmull.p8 t4q, \ad, \b3 @ I = A*B3 + veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 + vmull.p8 t3q, \ad, \b4 @ K = A*B4 + vand t0h, t0h, k48 + vand t1h, t1h, k32 + veor t2q, t2q, t4q @ N = I + J + veor t0l, t0l, t0h + veor t1l, t1l, t1h + veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 + vand t2h, t2h, k16 + veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 + vmov.i64 t3h, #0 + vext.8 t0q, t0q, t0q, #15 + veor t2l, t2l, t2h + vext.8 t1q, t1q, t1q, #14 + vmull.p8 \rq, \ad, \bd @ D = A*B + vext.8 t2q, t2q, t2q, #13 + vext.8 t3q, t3q, t3q, #12 + veor t0q, t0q, t1q + veor t2q, t2q, t3q + veor \rq, \rq, t0q + veor \rq, \rq, t2q + .endm + + // + // PMULL (64x64->128) based reduction for CPUs that can do + // it in a single instruction. + // + .macro __pmull_reduce_p64 + vmull.p64 T1, XL_L, MASK + + veor XH_L, XH_L, XM_H + vext.8 T1, T1, T1, #8 + veor XL_H, XL_H, XM_L + veor T1, T1, XL + + vmull.p64 XL, T1_H, MASK + .endm + + // + // Alternative reduction for CPUs that lack support for the + // 64x64->128 PMULL instruction + // + .macro __pmull_reduce_p8 + veor XL_H, XL_H, XM_L + veor XH_L, XH_L, XM_H + + vshl.i64 T1, XL, #57 + vshl.i64 T2, XL, #62 + veor T1, T1, T2 + vshl.i64 T2, XL, #63 + veor T1, T1, T2 + veor XL_H, XL_H, T1_L + veor XH_L, XH_L, T1_H + + vshr.u64 T1, XL, #1 + veor XH, XH, XL + veor XL, XL, T1 + vshr.u64 T1, T1, #6 + vshr.u64 XL, XL, #1 + .endm + + .macro ghash_update, pn + vld1.64 {XL}, [r1] + + /* do the head block first, if supplied */ + ldr ip, [sp] + teq ip, #0 + beq 0f + vld1.8 {T1}, [ip] + teq r0, #0 + b 1f + +0: vld1.8 {T1}, [r2]! + subs r0, r0, #1 + +1: /* multiply XL by SHASH in GF(2^128) */ +#ifndef CONFIG_CPU_BIG_ENDIAN + vrev64.8 T1, T1 +#endif + vext.8 IN1, T1, T1, #8 + veor T1_L, T1_L, XL_H + veor XL, XL, IN1 + + __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + veor T1, T1, XL + __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 + __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0) + + veor T1, XL, XH + veor XM, XM, T1 + + __pmull_reduce_\pn + + veor T1, T1, XH + veor XL, XL, T1 + + bne 0b + + vst1.64 {XL}, [r1] + bx lr + .endm + + /* + * void pmull_ghash_update(int blocks, u64 dg[], const char *src, + * struct ghash_key const *k, const char *head) + */ +FUNC pmull_ghash_update_p64 , : + vld1.64 {SHASH}, [r3] + veor SHASH2_p64, SHASH_L, SHASH_H + + vmov.i8 MASK, #0xe1 + vshl.u64 MASK, MASK, #57 + + ghash_update p64 +END_FUNC pmull_ghash_update_p64 + +FUNC pmull_ghash_update_p8 , : + vld1.64 {SHASH}, [r3] + veor SHASH2_p8, SHASH_L, SHASH_H + + vext.8 s1l, SHASH_L, SHASH_L, #1 + vext.8 s2l, SHASH_L, SHASH_L, #2 + vext.8 s3l, SHASH_L, SHASH_L, #3 + vext.8 s4l, SHASH_L, SHASH_L, #4 + vext.8 s1h, SHASH_H, SHASH_H, #1 + vext.8 s2h, SHASH_H, SHASH_H, #2 + vext.8 s3h, SHASH_H, SHASH_H, #3 + vext.8 s4h, SHASH_H, SHASH_H, #4 + + vmov.i64 k16, #0xffff + vmov.i64 k32, #0xffffffff + vmov.i64 k48, #0xffffffffffff + + ghash_update p8 +END_FUNC pmull_ghash_update_p8 diff --git a/optee_os/core/arch/arm/crypto/ghash-ce-core_a64.S b/optee_os/core/arch/arm/crypto/ghash-ce-core_a64.S new file mode 100644 index 0000000..dc86132 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/ghash-ce-core_a64.S @@ -0,0 +1,650 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020 Linaro Limited + * Copyright (C) 2014 - 2017 Linaro Ltd. + * + * Accelerated GHASH implementation with ARMv8 PMULL instructions. + */ + +#include +#include +#define CPU_LE(x...) x + +/* + * If the lower half of CTR is initialized with zeroes or a low value we + * can expect that the upper half will remain unchanged. As an optimization + * make the code to increase the upper half optional. + */ +#define INC_HALF_CTR 0 + + SHASH .req v0 + SHASH2 .req v1 + T1 .req v2 + T2 .req v3 + MASK .req v4 + XL .req v5 + XM .req v6 + XH .req v7 + IN1 .req v7 + + k00_16 .req v8 + k32_48 .req v9 + + t3 .req v10 + t4 .req v11 + t5 .req v12 + t6 .req v13 + t7 .req v14 + t8 .req v15 + t9 .req v16 + + perm1 .req v17 + perm2 .req v18 + perm3 .req v19 + + sh1 .req v20 + sh2 .req v21 + sh3 .req v22 + sh4 .req v23 + + ss1 .req v24 + ss2 .req v25 + ss3 .req v26 + ss4 .req v27 + + XL2 .req v8 + XM2 .req v9 + XH2 .req v10 + XL3 .req v11 + XM3 .req v12 + XH3 .req v13 + TT3 .req v14 + TT4 .req v15 + HH .req v16 + HH3 .req v17 + HH4 .req v18 + HH34 .req v19 + + .arch armv8-a+crypto + + .macro __pmull_p64, rd, rn, rm + pmull \rd\().1q, \rn\().1d, \rm\().1d + .endm + + .macro __pmull2_p64, rd, rn, rm + pmull2 \rd\().1q, \rn\().2d, \rm\().2d + .endm + + .macro __pmull_p8, rq, ad, bd + ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 + ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 + ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 + + __pmull_p8_\bd \rq, \ad + .endm + + .macro __pmull2_p8, rq, ad, bd + tbl t3.16b, {\ad\().16b}, perm1.16b // A1 + tbl t5.16b, {\ad\().16b}, perm2.16b // A2 + tbl t7.16b, {\ad\().16b}, perm3.16b // A3 + + __pmull2_p8_\bd \rq, \ad + .endm + + .macro __pmull_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_SHASH2, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 + .endm + + .macro __pmull2_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 + pmull\t t3.8h, t3.\nb, \bd // F = A1*B + pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 + pmull\t t5.8h, t5.\nb, \bd // H = A2*B + pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 + pmull\t t7.8h, t7.\nb, \bd // J = A3*B + pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 + pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 + pmull\t \rq\().8h, \ad, \bd // D = A*B + + eor t3.16b, t3.16b, t4.16b // L = E + F + eor t5.16b, t5.16b, t6.16b // M = G + H + eor t7.16b, t7.16b, t8.16b // N = I + J + + uzp1 t4.2d, t3.2d, t5.2d + uzp2 t3.2d, t3.2d, t5.2d + uzp1 t6.2d, t7.2d, t9.2d + uzp2 t7.2d, t7.2d, t9.2d + + // t3 = (L) (P0 + P1) << 8 + // t5 = (M) (P2 + P3) << 16 + eor t4.16b, t4.16b, t3.16b + and t3.16b, t3.16b, k32_48.16b + + // t7 = (N) (P4 + P5) << 24 + // t9 = (K) (P6 + P7) << 32 + eor t6.16b, t6.16b, t7.16b + and t7.16b, t7.16b, k00_16.16b + + eor t4.16b, t4.16b, t3.16b + eor t6.16b, t6.16b, t7.16b + + zip2 t5.2d, t4.2d, t3.2d + zip1 t3.2d, t4.2d, t3.2d + zip2 t9.2d, t6.2d, t7.2d + zip1 t7.2d, t6.2d, t7.2d + + ext t3.16b, t3.16b, t3.16b, #15 + ext t5.16b, t5.16b, t5.16b, #14 + ext t7.16b, t7.16b, t7.16b, #13 + ext t9.16b, t9.16b, t9.16b, #12 + + eor t3.16b, t3.16b, t5.16b + eor t7.16b, t7.16b, t9.16b + eor \rq\().16b, \rq\().16b, t3.16b + eor \rq\().16b, \rq\().16b, t7.16b + .endm + + .macro __pmull_pre_p64 + add x8, x3, #16 + ld1 {HH.2d-HH4.2d}, [x8] + + trn1 SHASH2.2d, SHASH.2d, HH.2d + trn2 T1.2d, SHASH.2d, HH.2d + eor SHASH2.16b, SHASH2.16b, T1.16b + + trn1 HH34.2d, HH3.2d, HH4.2d + trn2 T1.2d, HH3.2d, HH4.2d + eor HH34.16b, HH34.16b, T1.16b + + movi MASK.16b, #0xe1 + shl MASK.2d, MASK.2d, #57 + .endm + + .macro __pmull_pre_p8 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + + // k00_16 := 0x0000000000000000_000000000000ffff + // k32_48 := 0x00000000ffffffff_0000ffffffffffff + movi k32_48.2d, #0xffffffff + mov k32_48.h[2], k32_48.h[0] + ushr k00_16.2d, k32_48.2d, #32 + + // prepare the permutation vectors + mov_imm x5, 0x080f0e0d0c0b0a09 + movi T1.8b, #8 + dup perm1.2d, x5 + eor perm1.16b, perm1.16b, T1.16b + ushr perm2.2d, perm1.2d, #8 + ushr perm3.2d, perm1.2d, #16 + ushr T1.2d, perm1.2d, #24 + sli perm2.2d, perm1.2d, #56 + sli perm3.2d, perm1.2d, #48 + sli T1.2d, perm1.2d, #40 + + // precompute loop invariants + tbl sh1.16b, {SHASH.16b}, perm1.16b + tbl sh2.16b, {SHASH.16b}, perm2.16b + tbl sh3.16b, {SHASH.16b}, perm3.16b + tbl sh4.16b, {SHASH.16b}, T1.16b + ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 + ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 + ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 + ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 + .endm + + // + // PMULL (64x64->128) based reduction for CPUs that can do + // it in a single instruction. + // + .macro __pmull_reduce_p64 + pmull T2.1q, XL.1d, MASK.1d + eor XM.16b, XM.16b, T1.16b + + mov XH.d[0], XM.d[1] + mov XM.d[1], XL.d[0] + + eor XL.16b, XM.16b, T2.16b + ext T2.16b, XL.16b, XL.16b, #8 + pmull XL.1q, XL.1d, MASK.1d + .endm + + // + // Alternative reduction for CPUs that lack support for the + // 64x64->128 PMULL instruction + // + .macro __pmull_reduce_p8 + eor XM.16b, XM.16b, T1.16b + + mov XL.d[1], XM.d[0] + mov XH.d[0], XM.d[1] + + shl T1.2d, XL.2d, #57 + shl T2.2d, XL.2d, #62 + eor T2.16b, T2.16b, T1.16b + shl T1.2d, XL.2d, #63 + eor T2.16b, T2.16b, T1.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor T2.16b, T2.16b, T1.16b + + mov XL.d[1], T2.d[0] + mov XH.d[0], T2.d[1] + + ushr T2.2d, XL.2d, #1 + eor XH.16b, XH.16b, XL.16b + eor XL.16b, XL.16b, T2.16b + ushr T2.2d, T2.2d, #6 + ushr XL.2d, XL.2d, #1 + .endm + + .macro __pmull_ghash, pn + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] + + __pmull_pre_\pn + + /* do the head block first, if supplied */ + cbz x4, 0f + ld1 {T1.16b}, [x4] + mov x4, xzr + b 3f + +0: .ifc \pn, p64 + tbnz w0, #0, 2f // skip until #blocks is a + tbnz w0, #1, 2f // round multiple of 4 + +1: ld1 {XM3.16b-TT4.16b}, [x2], #64 + + sub w0, w0, #4 + + rev64 T1.16b, XM3.16b + rev64 T2.16b, XH3.16b + rev64 TT4.16b, TT4.16b + rev64 TT3.16b, TT3.16b + + ext IN1.16b, TT4.16b, TT4.16b, #8 + ext XL3.16b, TT3.16b, TT3.16b, #8 + + eor TT4.16b, TT4.16b, IN1.16b + pmull2 XH2.1q, SHASH.2d, IN1.2d // a1 * b1 + pmull XL2.1q, SHASH.1d, IN1.1d // a0 * b0 + pmull XM2.1q, SHASH2.1d, TT4.1d // (a1 + a0)(b1 + b0) + + eor TT3.16b, TT3.16b, XL3.16b + pmull2 XH3.1q, HH.2d, XL3.2d // a1 * b1 + pmull XL3.1q, HH.1d, XL3.1d // a0 * b0 + pmull2 XM3.1q, SHASH2.2d, TT3.2d // (a1 + a0)(b1 + b0) + + ext IN1.16b, T2.16b, T2.16b, #8 + eor XL2.16b, XL2.16b, XL3.16b + eor XH2.16b, XH2.16b, XH3.16b + eor XM2.16b, XM2.16b, XM3.16b + + eor T2.16b, T2.16b, IN1.16b + pmull2 XH3.1q, HH3.2d, IN1.2d // a1 * b1 + pmull XL3.1q, HH3.1d, IN1.1d // a0 * b0 + pmull XM3.1q, HH34.1d, T2.1d // (a1 + a0)(b1 + b0) + + eor XL2.16b, XL2.16b, XL3.16b + eor XH2.16b, XH2.16b, XH3.16b + eor XM2.16b, XM2.16b, XM3.16b + + ext IN1.16b, T1.16b, T1.16b, #8 + ext TT3.16b, XL.16b, XL.16b, #8 + eor XL.16b, XL.16b, IN1.16b + eor T1.16b, T1.16b, TT3.16b + + pmull2 XH.1q, HH4.2d, XL.2d // a1 * b1 + eor T1.16b, T1.16b, XL.16b + pmull XL.1q, HH4.1d, XL.1d // a0 * b0 + pmull2 XM.1q, HH34.2d, T1.2d // (a1 + a0)(b1 + b0) + + eor XL.16b, XL.16b, XL2.16b + eor XH.16b, XH.16b, XH2.16b + eor XM.16b, XM.16b, XM2.16b + + eor T2.16b, XL.16b, XH.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor XM.16b, XM.16b, T2.16b + + __pmull_reduce_p64 + + eor T2.16b, T2.16b, XH.16b + eor XL.16b, XL.16b, T2.16b + + cbz w0, 5f + b 1b + .endif + +2: ld1 {T1.16b}, [x2], #16 + sub w0, w0, #1 + +3: /* multiply XL by SHASH in GF(2^128) */ +CPU_LE( rev64 T1.16b, T1.16b ) + + ext T2.16b, XL.16b, XL.16b, #8 + ext IN1.16b, T1.16b, T1.16b, #8 + eor T1.16b, T1.16b, T2.16b + eor XL.16b, XL.16b, IN1.16b + + __pmull2_\pn XH, XL, SHASH // a1 * b1 + eor T1.16b, T1.16b, XL.16b + __pmull_\pn XL, XL, SHASH // a0 * b0 + __pmull_\pn XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + +4: eor T2.16b, XL.16b, XH.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor XM.16b, XM.16b, T2.16b + + __pmull_reduce_\pn + + eor T2.16b, T2.16b, XH.16b + eor XL.16b, XL.16b, T2.16b + + cbnz w0, 0b + +5: st1 {XL.2d}, [x1] + ret + .endm + +/* + * void pmull_ghash_update_p64(int blocks, uint64_t dg[2], const uint8_t *src, + * const struct internal_ghash_key *ghash_key, + * const uint8_t *head); + */ +FUNC pmull_ghash_update_p64 , : + __pmull_ghash p64 +END_FUNC pmull_ghash_update_p64 + +/* + * void pmull_ghash_update_p8(int blocks, uint64_t dg[2], const uint8_t *src, + * const struct internal_ghash_key *ghash_key, + * const uint8_t *head); + */ +FUNC pmull_ghash_update_p8 , : + __pmull_ghash p8 +END_FUNC pmull_ghash_update_p8 + + KS0 .req v12 + KS1 .req v13 + INP0 .req v14 + INP1 .req v15 + + .macro load_round_keys, rounds, rk + cmp \rounds, #12 + blo 2222f /* 128 bits */ + beq 1111f /* 192 bits */ + ld1 {v17.4s-v18.4s}, [\rk], #32 +1111: ld1 {v19.4s-v20.4s}, [\rk], #32 +2222: ld1 {v21.4s-v24.4s}, [\rk], #64 + ld1 {v25.4s-v28.4s}, [\rk], #64 + ld1 {v29.4s-v31.4s}, [\rk] + .endm + + .macro enc_round, state, key + aese \state\().16b, \key\().16b + aesmc \state\().16b, \state\().16b + .endm + + .macro enc_block, state, rounds + cmp \rounds, #12 + b.lo 2222f /* 128 bits */ + b.eq 1111f /* 192 bits */ + enc_round \state, v17 + enc_round \state, v18 +1111: enc_round \state, v19 + enc_round \state, v20 +2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 + enc_round \state, \key + .endr + aese \state\().16b, v30.16b + eor \state\().16b, \state\().16b, v31.16b + .endm + + .macro pmull_gcm_do_crypt, enc + ld1 {SHASH.2d}, [x4], #16 + ld1 {HH.2d}, [x4] + ld1 {XL.2d}, [x1] +#if INC_HALF_CTR + ldr x8, [x5, #8] // load lower counter +#else + ldp x9, x8, [x5] // load counter +#endif + + movi MASK.16b, #0xe1 + trn1 SHASH2.2d, SHASH.2d, HH.2d + trn2 T1.2d, SHASH.2d, HH.2d +CPU_LE( rev x8, x8 ) +#if !INC_HALF_CTR +CPU_LE( rev x9, x9 ) +#endif + shl MASK.2d, MASK.2d, #57 + eor SHASH2.16b, SHASH2.16b, T1.16b + + .if \enc == 1 + ldr x10, [sp] + ld1 {KS0.16b-KS1.16b}, [x10] + .endif + + cbnz x6, 4f + +0: ld1 {INP0.16b-INP1.16b}, [x3], #32 + +#if INC_HALF_CTR + rev x9, x8 + add x11, x8, #1 + add x8, x8, #2 +#endif + + .if \enc == 1 + eor INP0.16b, INP0.16b, KS0.16b // encrypt input + eor INP1.16b, INP1.16b, KS1.16b + .endif + + sub w0, w0, #2 + +#if INC_HALF_CTR + ld1 {KS0.8b}, [x5] // load upper counter + rev x11, x11 + mov KS1.8b, KS0.8b + ins KS0.d[1], x9 // set lower counter + ins KS1.d[1], x11 +#else + ins KS0.d[1], x8 + ins KS0.d[0], x9 + rev64 KS0.16b, KS0.16b + + add x8, x8, #1 + cbnz x8, 10f + add x9, x9, #1 +10: + ins KS1.d[1], x8 + ins KS1.d[0], x9 + rev64 KS1.16b, KS1.16b + + add x8, x8, #1 + cbnz x8, 11f + add x9, x9, #1 +11: +#endif + + rev64 T1.16b, INP1.16b + + cmp w7, #12 + b.ge 2f // AES-192/256? + +1: enc_round KS0, v21 + ext IN1.16b, T1.16b, T1.16b, #8 + + enc_round KS1, v21 + pmull2 XH2.1q, SHASH.2d, IN1.2d // a1 * b1 + + enc_round KS0, v22 + eor T1.16b, T1.16b, IN1.16b + + enc_round KS1, v22 + pmull XL2.1q, SHASH.1d, IN1.1d // a0 * b0 + + enc_round KS0, v23 + pmull XM2.1q, SHASH2.1d, T1.1d // (a1 + a0)(b1 + b0) + + enc_round KS1, v23 + rev64 T1.16b, INP0.16b + ext T2.16b, XL.16b, XL.16b, #8 + + enc_round KS0, v24 + ext IN1.16b, T1.16b, T1.16b, #8 + eor T1.16b, T1.16b, T2.16b + + enc_round KS1, v24 + eor XL.16b, XL.16b, IN1.16b + + enc_round KS0, v25 + eor T1.16b, T1.16b, XL.16b + + enc_round KS1, v25 + pmull2 XH.1q, HH.2d, XL.2d // a1 * b1 + + enc_round KS0, v26 + pmull XL.1q, HH.1d, XL.1d // a0 * b0 + + enc_round KS1, v26 + pmull2 XM.1q, SHASH2.2d, T1.2d // (a1 + a0)(b1 + b0) + + enc_round KS0, v27 + eor XL.16b, XL.16b, XL2.16b + eor XH.16b, XH.16b, XH2.16b + + enc_round KS1, v27 + eor XM.16b, XM.16b, XM2.16b + ext T1.16b, XL.16b, XH.16b, #8 + + enc_round KS0, v28 + eor T2.16b, XL.16b, XH.16b + eor XM.16b, XM.16b, T1.16b + + enc_round KS1, v28 + eor XM.16b, XM.16b, T2.16b + + enc_round KS0, v29 + pmull T2.1q, XL.1d, MASK.1d + + enc_round KS1, v29 + mov XH.d[0], XM.d[1] + mov XM.d[1], XL.d[0] + + aese KS0.16b, v30.16b + eor XL.16b, XM.16b, T2.16b + + aese KS1.16b, v30.16b + ext T2.16b, XL.16b, XL.16b, #8 + + eor KS0.16b, KS0.16b, v31.16b + pmull XL.1q, XL.1d, MASK.1d + eor T2.16b, T2.16b, XH.16b + + eor KS1.16b, KS1.16b, v31.16b + eor XL.16b, XL.16b, T2.16b + + .if \enc == 0 + eor INP0.16b, INP0.16b, KS0.16b + eor INP1.16b, INP1.16b, KS1.16b + .endif + + st1 {INP0.16b-INP1.16b}, [x2], #32 + + cbnz w0, 0b + +CPU_LE( rev x8, x8 ) +#if !INC_HALF_CTR +CPU_LE( rev x9, x9 ) +#endif + st1 {XL.2d}, [x1] +#if INC_HALF_CTR + str x8, [x5, #8] // store lower counter +#else + stp x9, x8, [x5] // store counter +#endif + + .if \enc == 1 + st1 {KS0.16b-KS1.16b}, [x10] + .endif + + ret + +2: b.eq 3f // AES-192? + enc_round KS0, v17 + enc_round KS1, v17 + enc_round KS0, v18 + enc_round KS1, v18 +3: enc_round KS0, v19 + enc_round KS1, v19 + enc_round KS0, v20 + enc_round KS1, v20 + b 1b + +4: load_round_keys w7, x6 + b 0b + .endm + +/* + * void pmull_gcm_encrypt(int blocks, uint64_t dg[2], uint8_t dst[], + * const uint8_t src[], + * const struct internal_ghash_key *ghash_key, + * uint64_t ctr[], const uint64_t rk[], int rounds, + * uint8_t ks[]); + */ +FUNC pmull_gcm_encrypt , : + pmull_gcm_do_crypt 1 +END_FUNC pmull_gcm_encrypt + +/* + * void pmull_gcm_decrypt(int blocks, uint64_t dg[2], uint8_t dst[], + * const uint8_t src[], + * const struct internal_ghash_key *ghash_key, + * uint64_t ctr[], const uint64_t rk[], int rounds); + */ +FUNC pmull_gcm_decrypt , : + pmull_gcm_do_crypt 0 +END_FUNC pmull_gcm_decrypt + +/* + * void pmull_gcm_encrypt_block(uint8_t dst[], const uint8_t src[], int rounds) + */ +FUNC pmull_gcm_encrypt_block , : + ld1 {v0.16b}, [x1] + enc_block v0, w2 + st1 {v0.16b}, [x0] + ret +END_FUNC pmull_gcm_encrypt_block + +/* + * void pmull_gcm_load_round_keys(const uint64_t rk[30], int rounds) + */ +FUNC pmull_gcm_load_round_keys , : + load_round_keys w1, x0 + ret +END_FUNC pmull_gcm_load_round_keys + +/* + * uint32_t pmull_gcm_aes_sub(uint32_t input) + * + * use the aese instruction to perform the AES sbox substitution + * on each byte in 'input' + */ +FUNC pmull_gcm_aes_sub , : + dup v1.4s, w0 + movi v0.16b, #0 + aese v0.16b, v1.16b + umov w0, v0.s[0] + ret +END_FUNC pmull_gcm_aes_sub + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/crypto/sha1_armv8a_ce.c b/optee_os/core/arch/arm/crypto/sha1_armv8a_ce.c new file mode 100644 index 0000000..4689ce0 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha1_armv8a_ce.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include + +/* Prototype for assembly function */ +void sha1_ce_transform(uint32_t state[5], const void *src, + unsigned int block_count); + +void crypto_accel_sha1_compress(uint32_t state[5], const void *src, + unsigned int block_count) +{ + uint32_t vfp_state = 0; + + vfp_state = thread_kernel_enable_vfp(); + sha1_ce_transform(state, src, block_count); + thread_kernel_disable_vfp(vfp_state); +} diff --git a/optee_os/core/arch/arm/crypto/sha1_armv8a_ce_a32.S b/optee_os/core/arch/arm/crypto/sha1_armv8a_ce_a32.S new file mode 100644 index 0000000..0c75821 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha1_armv8a_ce_a32.S @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2020, Linaro Limited + */ + +/* SHA-1 secure hash using ARMv8 Crypto Extensions */ + +#include + + .fpu crypto-neon-fp-armv8 + + k0 .req q0 + k1 .req q1 + k2 .req q2 + k3 .req q3 + + ta0 .req q4 + ta1 .req q5 + tb0 .req q5 + tb1 .req q4 + + dga .req q6 + dgb .req q7 + dgbs .req s28 + + dg0 .req q12 + + dg1a0 .req q13 + dg1a1 .req q14 + dg1b0 .req q14 + dg1b1 .req q13 + + .macro add_only, op, ev, rc, s0, dg1 + .ifnb \s0 + vadd.u32 tb\ev, q\s0, \rc + .endif + sha1h.32 dg1b\ev, dg0 + .ifb \dg1 + sha1\op\().32 dg0, dg1a\ev, ta\ev + .else + sha1\op\().32 dg0, \dg1, ta\ev + .endif + .endm + + .macro add_update, op, ev, rc, s0, s1, s2, s3, dg1 + sha1su0.32 q\s0, q\s1, q\s2 + add_only \op, \ev, \rc, \s1, \dg1 + sha1su1.32 q\s0, q\s3 + .endm + + +FUNC sha1_ce_transform , : + /* load round constants */ + adr r3, .Lsha1_rcon + vld1.32 {k0-k1}, [r3]! + vld1.32 {k2-k3}, [r3] + + /* load state */ + vld1.32 {dga}, [r0] + vldr dgbs, [r0, #16] + +0: /* load input */ + vld1.8 {q8-q9}, [r1]! + vrev32.8 q8, q8 + vrev32.8 q9, q9 + vld1.8 {q10-q11}, [r1]! + vrev32.8 q10, q10 + vrev32.8 q11, q11 + subs r2, r2, #1 + + vadd.u32 ta0, q8, k0 + vmov dg0, dga + + add_update c, 0, k0, 8, 9, 10, 11, dgb + add_update c, 1, k0, 9, 10, 11, 8 + add_update c, 0, k0, 10, 11, 8, 9 + add_update c, 1, k0, 11, 8, 9, 10 + add_update c, 0, k1, 8, 9, 10, 11 + + add_update p, 1, k1, 9, 10, 11, 8 + add_update p, 0, k1, 10, 11, 8, 9 + add_update p, 1, k1, 11, 8, 9, 10 + add_update p, 0, k1, 8, 9, 10, 11 + add_update p, 1, k2, 9, 10, 11, 8 + + add_update m, 0, k2, 10, 11, 8, 9 + add_update m, 1, k2, 11, 8, 9, 10 + add_update m, 0, k2, 8, 9, 10, 11 + add_update m, 1, k2, 9, 10, 11, 8 + add_update m, 0, k3, 10, 11, 8, 9 + + add_update p, 1, k3, 11, 8, 9, 10 + add_only p, 0, k3, 9 + add_only p, 1, k3, 10 + add_only p, 0, k3, 11 + add_only p, 1 + + /* update state */ + vadd.u32 dga, dga, dg0 + vadd.u32 dgb, dgb, dg1a0 + bne 0b + + /* store new state */ + vst1.32 {dga}, [r0] + vstr dgbs, [r0, #16] + bx lr + + .align 4 +.Lsha1_rcon: + .word 0x5a827999, 0x5a827999, 0x5a827999, 0x5a827999 + .word 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1 + .word 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc + .word 0xca62c1d6, 0xca62c1d6, 0xca62c1d6, 0xca62c1d6 +END_FUNC sha1_ce_transform diff --git a/optee_os/core/arch/arm/crypto/sha1_armv8a_ce_a64.S b/optee_os/core/arch/arm/crypto/sha1_armv8a_ce_a64.S new file mode 100644 index 0000000..b8ed17f --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha1_armv8a_ce_a64.S @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2020, Linaro Limited + * Copyright (C) 2014 Linaro Ltd + */ + + /* SHA-1 secure hash using ARMv8 Crypto Extensions */ + +#include + + .arch armv8-a+crypto + + k0 .req v0 + k1 .req v1 + k2 .req v2 + k3 .req v3 + + t0 .req v4 + t1 .req v5 + + dga .req q6 + dgav .req v6 + dgb .req s7 + dgbv .req v7 + + dg0q .req q12 + dg0s .req s12 + dg0v .req v12 + dg1s .req s13 + dg1v .req v13 + dg2s .req s14 + + .macro add_only, op, ev, rc, s0, dg1 + .ifc \ev, ev + add t1.4s, v\s0\().4s, \rc\().4s + sha1h dg2s, dg0s + .ifnb \dg1 + sha1\op dg0q, \dg1, t0.4s + .else + sha1\op dg0q, dg1s, t0.4s + .endif + .else + .ifnb \s0 + add t0.4s, v\s0\().4s, \rc\().4s + .endif + sha1h dg1s, dg0s + sha1\op dg0q, dg2s, t1.4s + .endif + .endm + + .macro add_update, op, ev, rc, s0, s1, s2, s3, dg1 + sha1su0 v\s0\().4s, v\s1\().4s, v\s2\().4s + add_only \op, \ev, \rc, \s1, \dg1 + sha1su1 v\s0\().4s, v\s3\().4s + .endm + + /* + * void sha1_ce_transform(u32 state[5], u8 const *src, int blocks) + */ +FUNC sha1_ce_transform , : + /* load round constants */ + adr x6, .Lsha1_rcon + ld1r {k0.4s}, [x6], #4 + ld1r {k1.4s}, [x6], #4 + ld1r {k2.4s}, [x6], #4 + ld1r {k3.4s}, [x6] + + /* load state */ + ld1 {dgav.4s}, [x0] + ldr dgb, [x0, #16] + + /* load input */ +0: ld1 {v8.16b-v11.16b}, [x1], #64 + sub w2, w2, #1 + + rev32 v8.16b, v8.16b + rev32 v9.16b, v9.16b + rev32 v10.16b, v10.16b + rev32 v11.16b, v11.16b + +1: add t0.4s, v8.4s, k0.4s + mov dg0v.16b, dgav.16b + + add_update c, ev, k0, 8, 9, 10, 11, dgb + add_update c, od, k0, 9, 10, 11, 8 + add_update c, ev, k0, 10, 11, 8, 9 + add_update c, od, k0, 11, 8, 9, 10 + add_update c, ev, k1, 8, 9, 10, 11 + + add_update p, od, k1, 9, 10, 11, 8 + add_update p, ev, k1, 10, 11, 8, 9 + add_update p, od, k1, 11, 8, 9, 10 + add_update p, ev, k1, 8, 9, 10, 11 + add_update p, od, k2, 9, 10, 11, 8 + + add_update m, ev, k2, 10, 11, 8, 9 + add_update m, od, k2, 11, 8, 9, 10 + add_update m, ev, k2, 8, 9, 10, 11 + add_update m, od, k2, 9, 10, 11, 8 + add_update m, ev, k3, 10, 11, 8, 9 + + add_update p, od, k3, 11, 8, 9, 10 + add_only p, ev, k3, 9 + add_only p, od, k3, 10 + add_only p, ev, k3, 11 + add_only p, od + + /* update state */ + add dgbv.2s, dgbv.2s, dg1v.2s + add dgav.4s, dgav.4s, dg0v.4s + + cbnz w2, 0b + + /* store new state */ +3: st1 {dgav.4s}, [x0] + str dgb, [x0, #16] + ret + + /* The SHA1 round constants */ + .align 4 +.Lsha1_rcon: + .word 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6 +END_FUNC sha1_ce_transform + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/crypto/sha256_armv8a_ce.c b/optee_os/core/arch/arm/crypto/sha256_armv8a_ce.c new file mode 100644 index 0000000..6114d85 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha256_armv8a_ce.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include + +/* Prototype for assembly function */ +void sha256_ce_transform(uint32_t state[8], const void *src, + unsigned int block_count); + +void crypto_accel_sha256_compress(uint32_t state[8], const void *src, + unsigned int block_count) +{ + uint32_t vfp_state = 0; + + vfp_state = thread_kernel_enable_vfp(); + sha256_ce_transform(state, src, block_count); + thread_kernel_disable_vfp(vfp_state); +} diff --git a/optee_os/core/arch/arm/crypto/sha256_armv8a_ce_a32.S b/optee_os/core/arch/arm/crypto/sha256_armv8a_ce_a32.S new file mode 100644 index 0000000..ccad7a9 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha256_armv8a_ce_a32.S @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2020, Linaro Limited + */ + +/* SHA-256 secure hash using ARMv8 Crypto Extensions */ + +#include + + .fpu crypto-neon-fp-armv8 + + k0 .req q7 + k1 .req q8 + + ta0 .req q9 + ta1 .req q10 + tb0 .req q10 + tb1 .req q9 + + dga .req q11 + dgb .req q12 + + dg0 .req q13 + dg1 .req q14 + dg2 .req q15 + + .macro add_only, ev, s0 + vmov dg2, dg0 + .ifnb \s0 + vld1.32 {k\ev}, [r3]! + .endif + sha256h.32 dg0, dg1, tb\ev + sha256h2.32 dg1, dg2, tb\ev + .ifnb \s0 + vadd.u32 ta\ev, q\s0, k\ev + .endif + .endm + + .macro add_update, ev, s0, s1, s2, s3 + sha256su0.32 q\s0, q\s1 + add_only \ev, \s1 + sha256su1.32 q\s0, q\s2, q\s3 + .endm + + +FUNC sha256_ce_transform , : + /* load state */ + vld1.8 {dga-dgb}, [r0] + + /* load input */ +0: vld1.8 {q0-q1}, [r1]! + vrev32.8 q0, q0 + vrev32.8 q1, q1 + vld1.8 {q2-q3}, [r1]! + vrev32.8 q2, q2 + vrev32.8 q3, q3 + subs r2, r2, #1 + + /* load round constants */ + adr r3, .Lsha256_rcon + vld1.32 {k0}, [r3]! + + vadd.u32 ta0, q0, k0 + vmov dg0, dga + vmov dg1, dgb + + add_update 1, 0, 1, 2, 3 + add_update 0, 1, 2, 3, 0 + add_update 1, 2, 3, 0, 1 + add_update 0, 3, 0, 1, 2 + add_update 1, 0, 1, 2, 3 + add_update 0, 1, 2, 3, 0 + add_update 1, 2, 3, 0, 1 + add_update 0, 3, 0, 1, 2 + add_update 1, 0, 1, 2, 3 + add_update 0, 1, 2, 3, 0 + add_update 1, 2, 3, 0, 1 + add_update 0, 3, 0, 1, 2 + + add_only 1, 1 + add_only 0, 2 + add_only 1, 3 + add_only 0 + + /* update state */ + vadd.u32 dga, dga, dg0 + vadd.u32 dgb, dgb, dg1 + bne 0b + + /* store new state */ + vst1.8 {dga-dgb}, [r0] + bx lr + + .align 6 +.Lsha256_rcon: + .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 + .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 + .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 + .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 + .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc + .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da + .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 + .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 + .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 + .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 + .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 + .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 + .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 + .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 + .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 + .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +END_FUNC sha256_ce_transform diff --git a/optee_os/core/arch/arm/crypto/sha256_armv8a_ce_a64.S b/optee_os/core/arch/arm/crypto/sha256_armv8a_ce_a64.S new file mode 100644 index 0000000..5fcfc65 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha256_armv8a_ce_a64.S @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2020, Linaro Limited + * Copyright (C) 2014 Linaro Ltd + */ + +/* Core SHA-224/SHA-256 transform using v8 Crypto Extensions */ + +#include + + .arch armv8-a+crypto + + dga .req q20 + dgav .req v20 + dgb .req q21 + dgbv .req v21 + + t0 .req v22 + t1 .req v23 + + dg0q .req q24 + dg0v .req v24 + dg1q .req q25 + dg1v .req v25 + dg2q .req q26 + dg2v .req v26 + + .macro add_only, ev, rc, s0 + mov dg2v.16b, dg0v.16b + .ifeq \ev + add t1.4s, v\s0\().4s, \rc\().4s + sha256h dg0q, dg1q, t0.4s + sha256h2 dg1q, dg2q, t0.4s + .else + .ifnb \s0 + add t0.4s, v\s0\().4s, \rc\().4s + .endif + sha256h dg0q, dg1q, t1.4s + sha256h2 dg1q, dg2q, t1.4s + .endif + .endm + + .macro add_update, ev, rc, s0, s1, s2, s3 + sha256su0 v\s0\().4s, v\s1\().4s + add_only \ev, \rc, \s1 + sha256su1 v\s0\().4s, v\s2\().4s, v\s3\().4s + .endm + + + /* + * void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, + * int blocks) + */ +FUNC sha256_ce_transform , : + /* load round constants */ + adr x8, .Lsha2_rcon + ld1 { v0.4s- v3.4s}, [x8], #64 + ld1 { v4.4s- v7.4s}, [x8], #64 + ld1 { v8.4s-v11.4s}, [x8], #64 + ld1 {v12.4s-v15.4s}, [x8] + + /* load state */ + mov x9, x0 + ld1 {dgav.4s}, [x9], #16 + ld1 {dgbv.4s}, [x9] + + /* load input */ +0: ld1 {v16.16b-v19.16b}, [x1], #64 + sub w2, w2, #1 + + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b + +1: add t0.4s, v16.4s, v0.4s + mov dg0v.16b, dgav.16b + mov dg1v.16b, dgbv.16b + + add_update 0, v1, 16, 17, 18, 19 + add_update 1, v2, 17, 18, 19, 16 + add_update 0, v3, 18, 19, 16, 17 + add_update 1, v4, 19, 16, 17, 18 + + add_update 0, v5, 16, 17, 18, 19 + add_update 1, v6, 17, 18, 19, 16 + add_update 0, v7, 18, 19, 16, 17 + add_update 1, v8, 19, 16, 17, 18 + + add_update 0, v9, 16, 17, 18, 19 + add_update 1, v10, 17, 18, 19, 16 + add_update 0, v11, 18, 19, 16, 17 + add_update 1, v12, 19, 16, 17, 18 + + add_only 0, v13, 17 + add_only 1, v14, 18 + add_only 0, v15, 19 + add_only 1 + + /* update state */ + add dgav.4s, dgav.4s, dg0v.4s + add dgbv.4s, dgbv.4s, dg1v.4s + + /* handled all input blocks? */ + cbnz w2, 0b + + /* store new state */ +3: mov x9, x0 + st1 {dgav.16b}, [x9], #16 + st1 {dgbv.16b}, [x9] + ret + + /* + * The SHA-256 round constants + */ + .align 4 +.Lsha2_rcon: + .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 + .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 + .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 + .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 + .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc + .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da + .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 + .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 + .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 + .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 + .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 + .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 + .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 + .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 + .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 + .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +END_FUNC sha256_ce_transform + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/crypto/sha3_armv8a_ce.c b/optee_os/core/arch/arm/crypto/sha3_armv8a_ce.c new file mode 100644 index 0000000..ad9152d --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha3_armv8a_ce.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023, Linaro Limited + */ + +#include +#include + +/* Prototype for assembly function */ +int sha3_ce_transform(uint64_t state[25], const void *src, + unsigned int block_count, unsigned int digest_size); + +void crypto_accel_sha3_compress(uint64_t state[25], const void *src, + unsigned int block_count, + unsigned int digest_size) +{ + uint32_t vfp_state = 0; + int res = 0; + + vfp_state = thread_kernel_enable_vfp(); + res = sha3_ce_transform(state, src, block_count, digest_size); + thread_kernel_disable_vfp(vfp_state); + assert(!res); +} + diff --git a/optee_os/core/arch/arm/crypto/sha3_armv8a_ce_a64.S b/optee_os/core/arch/arm/crypto/sha3_armv8a_ce_a64.S new file mode 100644 index 0000000..a468998 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha3_armv8a_ce_a64.S @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2018 Linaro Ltd + * Copyright (c) 2023 Linaro Limited + */ + +/* Core SHA-3 transform using v8 Crypto Extensions */ + +#include +#include + + .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,\ + 21,22,23,24,25,26,27,28,29,30,31 + .set .Lv\b\().2d, \b + .set .Lv\b\().16b, \b + .endr + + /* + * ARMv8.2 Crypto Extensions instructions + */ + .macro eor3, rd, rn, rm, ra + .inst 0xce000000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | \ + (.L\rm << 16) + .endm + + .macro rax1, rd, rn, rm + .inst 0xce608c00 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro bcax, rd, rn, rm, ra + .inst 0xce200000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | \ + (.L\rm << 16) + .endm + + .macro xar, rd, rn, rm, imm6 + .inst 0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | \ + (.L\rm << 16) + .endm + + /* + * int sha3_ce_transform(u64 *st, const u8 *data, int blocks, + * int dg_size) + */ + .text +FUNC sha3_ce_transform , : + /* load state */ + add x8, x0, #32 + ld1 { v0.1d- v3.1d}, [x0] + ld1 { v4.1d- v7.1d}, [x8], #32 + ld1 { v8.1d-v11.1d}, [x8], #32 + ld1 {v12.1d-v15.1d}, [x8], #32 + ld1 {v16.1d-v19.1d}, [x8], #32 + ld1 {v20.1d-v23.1d}, [x8], #32 + ld1 {v24.1d}, [x8] + +0: sub w2, w2, #1 + mov w8, #24 + adr_l x9, .Lsha3_rcon + + /* load input */ + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v31.8b}, [x1], #24 + eor v0.8b, v0.8b, v25.8b + eor v1.8b, v1.8b, v26.8b + eor v2.8b, v2.8b, v27.8b + eor v3.8b, v3.8b, v28.8b + eor v4.8b, v4.8b, v29.8b + eor v5.8b, v5.8b, v30.8b + eor v6.8b, v6.8b, v31.8b + + /* Bit 6 set? -> SHA3-512 */ + tbnz x3, #6, 3f + + /* SHA3-384, SHA3-256, SHA3-224 or SHA3-128 */ + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v30.8b}, [x1], #16 + eor v7.8b, v7.8b, v25.8b + eor v8.8b, v8.8b, v26.8b + eor v9.8b, v9.8b, v27.8b + eor v10.8b, v10.8b, v28.8b + eor v11.8b, v11.8b, v29.8b + eor v12.8b, v12.8b, v30.8b + + /* bit 4 set? -> SHA3-384, SHA3-224 or SHA3-128 */ + tbnz x3, #4, 1f + + /* SHA3-256: digest size 32 bytes, block size 136 bytes */ + ld1 {v25.8b-v28.8b}, [x1], #32 + eor v13.8b, v13.8b, v25.8b + eor v14.8b, v14.8b, v26.8b + eor v15.8b, v15.8b, v27.8b + eor v16.8b, v16.8b, v28.8b + b 4f + + /* bit 5 set? -> SHA-384 */ +1: tbnz x3, #5, 4f + + /* SHA3-224 or SHA3-128 */ + ld1 {v25.8b-v28.8b}, [x1], #32 + eor v13.8b, v13.8b, v25.8b + eor v14.8b, v14.8b, v26.8b + eor v15.8b, v15.8b, v27.8b + eor v16.8b, v16.8b, v28.8b + + /* bit 2 set? -> SHA-224 */ + tbnz x3, #2, 2f + + /* SHA3-128: digest size 16 bytes, block size 168 bytes */ + ld1 {v25.8b-v28.8b}, [x1], #32 + eor v17.8b, v17.8b, v25.8b + eor v18.8b, v18.8b, v26.8b + eor v19.8b, v19.8b, v27.8b + eor v20.8b, v20.8b, v28.8b + b 4f + + /* SHA3-224: digest size 28 bytes, block size 144 bytes */ +2: ld1 {v25.8b-v26.8b}, [x1], #16 + eor v17.8b, v17.8b, v25.8b + eor v18.8b, v18.8b, v26.8b + b 4f + + /* SHA3-512: digest size 64 bytes , block size 72 bytes */ +3: ld1 {v25.8b-v26.8b}, [x1], #16 + eor v7.8b, v7.8b, v25.8b + eor v8.8b, v8.8b, v26.8b + +4: sub w8, w8, #1 + + eor3 v29.16b, v4.16b, v9.16b, v14.16b + eor3 v26.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v3.16b, v8.16b, v13.16b + eor3 v25.16b, v0.16b, v5.16b, v10.16b + eor3 v27.16b, v2.16b, v7.16b, v12.16b + eor3 v29.16b, v29.16b, v19.16b, v24.16b + eor3 v26.16b, v26.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v18.16b, v23.16b + eor3 v25.16b, v25.16b, v15.16b, v20.16b + eor3 v27.16b, v27.16b, v17.16b, v22.16b + + rax1 v30.2d, v29.2d, v26.2d // bc[0] + rax1 v26.2d, v26.2d, v28.2d // bc[2] + rax1 v28.2d, v28.2d, v25.2d // bc[4] + rax1 v25.2d, v25.2d, v27.2d // bc[1] + rax1 v27.2d, v27.2d, v29.2d // bc[3] + + eor v0.16b, v0.16b, v30.16b + xar v29.2d, v1.2d, v25.2d, (64 - 1) + xar v1.2d, v6.2d, v25.2d, (64 - 44) + xar v6.2d, v9.2d, v28.2d, (64 - 20) + xar v9.2d, v22.2d, v26.2d, (64 - 61) + xar v22.2d, v14.2d, v28.2d, (64 - 39) + xar v14.2d, v20.2d, v30.2d, (64 - 18) + xar v31.2d, v2.2d, v26.2d, (64 - 62) + xar v2.2d, v12.2d, v26.2d, (64 - 43) + xar v12.2d, v13.2d, v27.2d, (64 - 25) + xar v13.2d, v19.2d, v28.2d, (64 - 8) + xar v19.2d, v23.2d, v27.2d, (64 - 56) + xar v23.2d, v15.2d, v30.2d, (64 - 41) + xar v15.2d, v4.2d, v28.2d, (64 - 27) + xar v28.2d, v24.2d, v28.2d, (64 - 14) + xar v24.2d, v21.2d, v25.2d, (64 - 2) + xar v8.2d, v8.2d, v27.2d, (64 - 55) + xar v4.2d, v16.2d, v25.2d, (64 - 45) + xar v16.2d, v5.2d, v30.2d, (64 - 36) + xar v5.2d, v3.2d, v27.2d, (64 - 28) + xar v27.2d, v18.2d, v27.2d, (64 - 21) + xar v3.2d, v17.2d, v26.2d, (64 - 15) + xar v25.2d, v11.2d, v25.2d, (64 - 10) + xar v26.2d, v7.2d, v26.2d, (64 - 6) + xar v30.2d, v10.2d, v30.2d, (64 - 3) + + bcax v20.16b, v31.16b, v22.16b, v8.16b + bcax v21.16b, v8.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v31.16b, v24.16b + bcax v24.16b, v24.16b, v8.16b, v31.16b + + ld1r {v31.2d}, [x9], #8 + + bcax v17.16b, v25.16b, v19.16b, v3.16b + bcax v18.16b, v3.16b, v15.16b, v19.16b + bcax v19.16b, v19.16b, v16.16b, v15.16b + bcax v15.16b, v15.16b, v25.16b, v16.16b + bcax v16.16b, v16.16b, v3.16b, v25.16b + + bcax v10.16b, v29.16b, v12.16b, v26.16b + bcax v11.16b, v26.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v29.16b, v14.16b + bcax v14.16b, v14.16b, v26.16b, v29.16b + + bcax v7.16b, v30.16b, v9.16b, v4.16b + bcax v8.16b, v4.16b, v5.16b, v9.16b + bcax v9.16b, v9.16b, v6.16b, v5.16b + bcax v5.16b, v5.16b, v30.16b, v6.16b + bcax v6.16b, v6.16b, v4.16b, v30.16b + + bcax v3.16b, v27.16b, v0.16b, v28.16b + bcax v4.16b, v28.16b, v1.16b, v0.16b + bcax v0.16b, v0.16b, v2.16b, v1.16b + bcax v1.16b, v1.16b, v27.16b, v2.16b + bcax v2.16b, v2.16b, v28.16b, v27.16b + + eor v0.16b, v0.16b, v31.16b + + cbnz w8, 4b + cbnz w2, 0b + + /* save state */ + st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v4.1d- v7.1d}, [x0], #32 + st1 { v8.1d-v11.1d}, [x0], #32 + st1 {v12.1d-v15.1d}, [x0], #32 + st1 {v16.1d-v19.1d}, [x0], #32 + st1 {v20.1d-v23.1d}, [x0], #32 + st1 {v24.1d}, [x0] + mov w0, w2 + ret +END_FUNC sha3_ce_transform + + .section ".rodata", "a" + .align 8 +LOCAL_DATA .Lsha3_rcon , : + .quad 0x0000000000000001, 0x0000000000008082, 0x800000000000808a + .quad 0x8000000080008000, 0x000000000000808b, 0x0000000080000001 + .quad 0x8000000080008081, 0x8000000000008009, 0x000000000000008a + .quad 0x0000000000000088, 0x0000000080008009, 0x000000008000000a + .quad 0x000000008000808b, 0x800000000000008b, 0x8000000000008089 + .quad 0x8000000000008003, 0x8000000000008002, 0x8000000000000080 + .quad 0x000000000000800a, 0x800000008000000a, 0x8000000080008081 + .quad 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/crypto/sha512_armv8a_ce.c b/optee_os/core/arch/arm/crypto/sha512_armv8a_ce.c new file mode 100644 index 0000000..165e718 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha512_armv8a_ce.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +#include +#include + +/* Prototype for assembly function */ +void sha512_ce_transform(uint64_t state[8], const void *src, + unsigned int block_count); + +void crypto_accel_sha512_compress(uint64_t state[8], const void *src, + unsigned int block_count) +{ + uint32_t vfp_state = 0; + + vfp_state = thread_kernel_enable_vfp(); + sha512_ce_transform(state, src, block_count); + thread_kernel_disable_vfp(vfp_state); +} diff --git a/optee_os/core/arch/arm/crypto/sha512_armv8a_ce_a64.S b/optee_os/core/arch/arm/crypto/sha512_armv8a_ce_a64.S new file mode 100644 index 0000000..d6a83fd --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sha512_armv8a_ce_a64.S @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Linaro Limited + * Copyright (C) 2018 Linaro Ltd + */ + +/* Core SHA-384/SHA-512 transform using v8 Crypto Extensions */ + +#include +#include + + .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19 + .set .Lq\b, \b + .set .Lv\b\().2d, \b + .endr + + .macro sha512h, rd, rn, rm + .inst 0xce608000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sha512h2, rd, rn, rm + .inst 0xce608400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sha512su0, rd, rn + .inst 0xcec08000 | .L\rd | (.L\rn << 5) + .endm + + .macro sha512su1, rd, rn, rm + .inst 0xce608800 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + /* + * The SHA-512 round constants + */ + .section ".rodata", "a" + .align 4 +LOCAL_DATA .Lsha512_rcon , : + .quad 0x428a2f98d728ae22, 0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538, 0x59f111f1b605d019 + .quad 0x923f82a4af194f9b, 0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242, 0x12835b0145706fbe + .quad 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f, 0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235, 0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2, 0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275, 0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5 + .quad 0x983e5152ee66dfab, 0xa831c66d2db43210 + .quad 0xb00327c898fb213f, 0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2, 0xd5a79147930aa725 + .quad 0x06ca6351e003826f, 0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc, 0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df + .quad 0x650a73548baf63de, 0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6, 0x92722c851482353b + .quad 0xa2bfe8a14cf10364, 0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791, 0xc76c51a30654be30 + .quad 0xd192e819d6ef5218, 0xd69906245565a910 + .quad 0xf40e35855771202a, 0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8, 0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc, 0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72, 0x8cc702081a6439ec + .quad 0x90befffa23631e28, 0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915, 0xc67178f2e372532b + .quad 0xca273eceea26619c, 0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba, 0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae, 0x1b710b35131c471b + .quad 0x28db77f523047d84, 0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 +END_DATA .Lsha512_rcon + + .macro dround, i0, i1, i2, i3, i4, rc0, rc1, in0, in1, in2, in3, in4 + .ifnb \rc1 + ld1 {v\rc1\().2d}, [x4], #16 + .endif + add v5.2d, v\rc0\().2d, v\in0\().2d + ext v6.16b, v\i2\().16b, v\i3\().16b, #8 + ext v5.16b, v5.16b, v5.16b, #8 + ext v7.16b, v\i1\().16b, v\i2\().16b, #8 + add v\i3\().2d, v\i3\().2d, v5.2d + .ifnb \in1 + ext v5.16b, v\in3\().16b, v\in4\().16b, #8 + sha512su0 v\in0\().2d, v\in1\().2d + .endif + sha512h q\i3, q6, v7.2d + .ifnb \in1 + sha512su1 v\in0\().2d, v\in2\().2d, v5.2d + .endif + add v\i4\().2d, v\i1\().2d, v\i3\().2d + sha512h2 q\i3, q\i1, v\i0\().2d + .endm + + /* + * void sha512_ce_transform(struct sha512_state *sst, u8 const *src, + * int blocks) + */ +FUNC sha512_ce_transform , : + /* load state */ + ld1 {v8.2d-v11.2d}, [x0] + + /* load first 4 round constants */ + adr_l x3, .Lsha512_rcon + ld1 {v20.2d-v23.2d}, [x3], #64 + + /* load input */ +0: ld1 {v12.2d-v15.2d}, [x1], #64 + ld1 {v16.2d-v19.2d}, [x1], #64 + sub w2, w2, #1 + + rev64 v12.16b, v12.16b + rev64 v13.16b, v13.16b + rev64 v14.16b, v14.16b + rev64 v15.16b, v15.16b + rev64 v16.16b, v16.16b + rev64 v17.16b, v17.16b + rev64 v18.16b, v18.16b + rev64 v19.16b, v19.16b + + mov x4, x3 // rc pointer + + mov v0.16b, v8.16b + mov v1.16b, v9.16b + mov v2.16b, v10.16b + mov v3.16b, v11.16b + + // v0 ab cd -- ef gh ab + // v1 cd -- ef gh ab cd + // v2 ef gh ab cd -- ef + // v3 gh ab cd -- ef gh + // v4 -- ef gh ab cd -- + + dround 0, 1, 2, 3, 4, 20, 24, 12, 13, 19, 16, 17 + dround 3, 0, 4, 2, 1, 21, 25, 13, 14, 12, 17, 18 + dround 2, 3, 1, 4, 0, 22, 26, 14, 15, 13, 18, 19 + dround 4, 2, 0, 1, 3, 23, 27, 15, 16, 14, 19, 12 + dround 1, 4, 3, 0, 2, 24, 28, 16, 17, 15, 12, 13 + + dround 0, 1, 2, 3, 4, 25, 29, 17, 18, 16, 13, 14 + dround 3, 0, 4, 2, 1, 26, 30, 18, 19, 17, 14, 15 + dround 2, 3, 1, 4, 0, 27, 31, 19, 12, 18, 15, 16 + dround 4, 2, 0, 1, 3, 28, 24, 12, 13, 19, 16, 17 + dround 1, 4, 3, 0, 2, 29, 25, 13, 14, 12, 17, 18 + + dround 0, 1, 2, 3, 4, 30, 26, 14, 15, 13, 18, 19 + dround 3, 0, 4, 2, 1, 31, 27, 15, 16, 14, 19, 12 + dround 2, 3, 1, 4, 0, 24, 28, 16, 17, 15, 12, 13 + dround 4, 2, 0, 1, 3, 25, 29, 17, 18, 16, 13, 14 + dround 1, 4, 3, 0, 2, 26, 30, 18, 19, 17, 14, 15 + + dround 0, 1, 2, 3, 4, 27, 31, 19, 12, 18, 15, 16 + dround 3, 0, 4, 2, 1, 28, 24, 12, 13, 19, 16, 17 + dround 2, 3, 1, 4, 0, 29, 25, 13, 14, 12, 17, 18 + dround 4, 2, 0, 1, 3, 30, 26, 14, 15, 13, 18, 19 + dround 1, 4, 3, 0, 2, 31, 27, 15, 16, 14, 19, 12 + + dround 0, 1, 2, 3, 4, 24, 28, 16, 17, 15, 12, 13 + dround 3, 0, 4, 2, 1, 25, 29, 17, 18, 16, 13, 14 + dround 2, 3, 1, 4, 0, 26, 30, 18, 19, 17, 14, 15 + dround 4, 2, 0, 1, 3, 27, 31, 19, 12, 18, 15, 16 + dround 1, 4, 3, 0, 2, 28, 24, 12, 13, 19, 16, 17 + + dround 0, 1, 2, 3, 4, 29, 25, 13, 14, 12, 17, 18 + dround 3, 0, 4, 2, 1, 30, 26, 14, 15, 13, 18, 19 + dround 2, 3, 1, 4, 0, 31, 27, 15, 16, 14, 19, 12 + dround 4, 2, 0, 1, 3, 24, 28, 16, 17, 15, 12, 13 + dround 1, 4, 3, 0, 2, 25, 29, 17, 18, 16, 13, 14 + + dround 0, 1, 2, 3, 4, 26, 30, 18, 19, 17, 14, 15 + dround 3, 0, 4, 2, 1, 27, 31, 19, 12, 18, 15, 16 + dround 2, 3, 1, 4, 0, 28, 24, 12 + dround 4, 2, 0, 1, 3, 29, 25, 13 + dround 1, 4, 3, 0, 2, 30, 26, 14 + + dround 0, 1, 2, 3, 4, 31, 27, 15 + dround 3, 0, 4, 2, 1, 24, , 16 + dround 2, 3, 1, 4, 0, 25, , 17 + dround 4, 2, 0, 1, 3, 26, , 18 + dround 1, 4, 3, 0, 2, 27, , 19 + + /* update state */ + add v8.2d, v8.2d, v0.2d + add v9.2d, v9.2d, v1.2d + add v10.2d, v10.2d, v2.2d + add v11.2d, v11.2d, v3.2d + + /* handled all input blocks? */ + cbnz w2, 0b + + /* store new state */ +3: st1 {v8.2d-v11.2d}, [x0] + mov w0, w2 + ret +END_FUNC sha512_ce_transform + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/crypto/sm3_armv8a_ce.c b/optee_os/core/arch/arm/crypto/sm3_armv8a_ce.c new file mode 100644 index 0000000..2646849 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm3_armv8a_ce.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022 Linaro Limited + */ + +#include +#include + +/* Prototype for assembly function */ +void sm3_ce_transform(uint32_t state[8], const void *src, + unsigned int block_count); + +void crypto_accel_sm3_compress(uint32_t state[8], const void *src, + unsigned int block_count) +{ + uint32_t vfp_state = 0; + + vfp_state = thread_kernel_enable_vfp(); + sm3_ce_transform(state, src, block_count); + thread_kernel_disable_vfp(vfp_state); +} + diff --git a/optee_os/core/arch/arm/crypto/sm3_armv8a_ce_a64.S b/optee_os/core/arch/arm/crypto/sm3_armv8a_ce_a64.S new file mode 100644 index 0000000..13ed507 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm3_armv8a_ce_a64.S @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022 Linaro Limited + * Copyright (C) 2018 Linaro Ltd + */ + +/* Core SM3 secure hash using ARMv8.2 Crypto Extensions */ + +#include +#include + + .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 + .set .Lv\b\().4s, \b + .endr + + .macro sm3partw1, rd, rn, rm + .inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3partw2, rd, rn, rm + .inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3ss1, rd, rn, rm, ra + .inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16) + .endm + + .macro sm3tt1a, rd, rn, rm, imm2 + .inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt1b, rd, rn, rm, imm2 + .inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2a, rd, rn, rm, imm2 + .inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2b, rd, rn, rm, imm2 + .inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro round, ab, s0, t0, t1, i + sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s + shl \t1\().4s, \t0\().4s, #1 + sri \t1\().4s, \t0\().4s, #31 + sm3tt1\ab v8.4s, v5.4s, v10.4s, \i + sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i + .endm + + .macro qround, ab, s0, s1, s2, s3, s4 + .ifnb \s4 + ext \s4\().16b, \s1\().16b, \s2\().16b, #12 + ext v6.16b, \s0\().16b, \s1\().16b, #12 + ext v7.16b, \s2\().16b, \s3\().16b, #8 + sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s + .endif + + eor v10.16b, \s0\().16b, \s1\().16b + + round \ab, \s0, v11, v12, 0 + round \ab, \s0, v12, v11, 1 + round \ab, \s0, v11, v12, 2 + round \ab, \s0, v12, v11, 3 + + .ifnb \s4 + sm3partw2 \s4\().4s, v7.4s, v6.4s + .endif + .endm + + /* + * void sm3_ce_transform(struct sm3_state *sst, u8 const *src, + * int blocks) + */ +FUNC sm3_ce_transform , : + /* load state */ + ld1 {v8.4s-v9.4s}, [x0] + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + + adr_l x8, .Lt + ldp s13, s14, [x8] + + /* load input */ +0: ld1 {v0.16b-v3.16b}, [x1], #64 + sub w2, w2, #1 + + mov v15.16b, v8.16b + mov v16.16b, v9.16b + + rev32 v0.16b, v0.16b + rev32 v1.16b, v1.16b + rev32 v2.16b, v2.16b + rev32 v3.16b, v3.16b + + ext v11.16b, v13.16b, v13.16b, #4 + + qround a, v0, v1, v2, v3, v4 + qround a, v1, v2, v3, v4, v0 + qround a, v2, v3, v4, v0, v1 + qround a, v3, v4, v0, v1, v2 + + ext v11.16b, v14.16b, v14.16b, #4 + + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4, v0, v1, v2 + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4 + qround b, v4, v0 + qround b, v0, v1 + + eor v8.16b, v8.16b, v15.16b + eor v9.16b, v9.16b, v16.16b + + /* handled all input blocks? */ + cbnz w2, 0b + + /* save state */ + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + st1 {v8.4s-v9.4s}, [x0] + ret +END_FUNC sm3_ce_transform + + .section ".rodata", "a" + .align 3 +LOCAL_DATA .Lt , : + .word 0x79cc4519, 0x9d8a7a87 +END_DATA .Lt + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/crypto/sm4_armv8a_aese_a64.S b/optee_os/core/arch/arm/crypto/sm4_armv8a_aese_a64.S new file mode 100644 index 0000000..ca51956 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm4_armv8a_aese_a64.S @@ -0,0 +1,1122 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) Hisilicon Technologies Co., Ltd. 2023. All rights reserved. + * Copyright (C) 2022, Alibaba Group. + * Copyright (C) 2022 Tianjia Zhang + * + * SM4 optimization for ARMv8 by NEON and AES HW instruction, which is an + * optional Cryptographic Extension for ARMv8-A. + * + * The NEON implementation refers to Linux kernel (sm4-neon-core.S contributed + * by Tianjia Zhang ). + * + * The AES trick refers to sm4ni (https://github.com/mjosaarinen/sm4ni). The + * constants used in load_sbox_matrix are from this blog (https://www.cnblogs. + * com/kentle/p/15826075.html). We've done some further optimizations so the + * constants don't look the same. + */ + +#include + +.arch armv8-a+crypto + +#define m0 w9 +#define m1 w10 +#define m2 w11 +#define m3 w12 +#define tw0l x7 +#define tw0h x8 +#define tw1l x9 +#define tw1h x10 +#define tw2l x11 +#define tw2h x12 +#define tw3l x13 +#define tw3h x14 +#define tw4l x15 +#define tw4h x16 +#define tw5l x17 +#define tw5h x18 +#define tw6l x19 +#define tw6h x20 +#define tw7l x21 +#define tw7h x22 +#define tmpw0 w23 +#define tmpx0 x23 +#define tmpw1 w24 +#define tmpx1 x24 +#define tmpw2 w25 + +/* round keys: v0-v7 */ +#define RK0 v0 +#define RK1 v1 +#define RK2 v2 +#define RK3 v3 +#define RK4 v4 +#define RK5 v5 +#define RK6 v6 +#define RK7 v7 + +/* plain blocks: v8-v15 */ +#define BLK0 v8 +#define BLK1 v9 +#define BLK2 v10 +#define BLK3 v11 +#define BLK4 v12 +#define BLK5 v13 +#define BLK6 v14 +#define BLK7 v15 + +#define TMP0 v16 +#define TMP1 v17 +#define TMP2 v18 +#define TMP3 v19 +#define TMP4 v20 +#define TMP5 v21 +#define TMP6 v22 +#define TMP7 v23 +#define TMP8 v24 +#define IV v25 +#define ANDMASKV v26 +#define ANDMASKQ q26 +#define ATALMaskV v27 +#define ATALMaskQ q27 +#define ATAHMaskV v28 +#define ATAHMaskQ q28 +#define TALMaskV v29 +#define TALMaskQ q29 +#define TAHMaskV v30 +#define TAHMaskQ q30 +#define MASKV v31 +#define MASKQ q31 + +.macro frame_push + stp x15, x16, [sp, #-0x10]! + stp x17, x18, [sp, #-0x10]! + stp x19, x20, [sp, #-0x10]! + stp x21, x22, [sp, #-0x10]! + stp x23, x24, [sp, #-0x10]! + stp x25, x26, [sp, #-0x10]! + stp x27, x28, [sp, #-0x10]! + stp x29, x30, [sp, #-0x10]! + stp d8, d9, [sp, #-0x10]! + stp d10, d11, [sp, #-0x10]! + stp d12, d13, [sp, #-0x10]! + stp d14, d15, [sp, #-0x10]! +.endm + +.macro frame_pop + ldp d14, d15, [sp], #0x10 + ldp d12, d13, [sp], #0x10 + ldp d10, d11, [sp], #0x10 + ldp d8, d9, [sp], #0x10 + ldp x29, x30, [sp], #0x10 + ldp x27, x28, [sp], #0x10 + ldp x25, x26, [sp], #0x10 + ldp x23, x24, [sp], #0x10 + ldp x21, x22, [sp], #0x10 + ldp x19, x20, [sp], #0x10 + ldp x17, x18, [sp], #0x10 + ldp x15, x16, [sp], #0x10 +.endm + +.macro load_sbox_matrix + ldr MASKQ, .Lsbox_magic + ldr TAHMaskQ, .Lsbox_magic+16 + ldr TALMaskQ, .Lsbox_magic+32 + ldr ATAHMaskQ, .Lsbox_magic+48 + ldr ATALMaskQ, .Lsbox_magic+64 + ldr ANDMASKQ, .Lsbox_magic+80 +.endm + +.macro multi_matrix, x, high, low, tmp + ushr \tmp\().16b, \x\().16b, 4 + and \x\().16b, \x\().16b, ANDMASKV.16b + tbl \x\().16b, {\low\().16b}, \x\().16b + tbl \tmp\().16b, {\high\().16b}, \tmp\().16b + eor \x\().16b, \x\().16b, \tmp\().16b +.endm + +.macro sbox, des, src, tmp1, tmp2 + tbl \des\().16b, {\src\().16b}, MASKV.16b + multi_matrix \des, TAHMaskV, TALMaskV, \tmp2 + eor \tmp1\().16b, \tmp1\().16b, \tmp1\().16b + aese \des\().16b, \tmp1\().16b + multi_matrix \des, ATAHMaskV, ATALMaskV, \tmp2 +.endm + +.macro sbox_double, des0, src0, des1, src1, tmp1, tmp2 + tbl \des0\().16b, {\src0\().16b}, MASKV.16b + tbl \des1\().16b, {\src1\().16b}, MASKV.16b + multi_matrix \des0, TAHMaskV, TALMaskV, \tmp2 + multi_matrix \des1, TAHMaskV, TALMaskV, \tmp2 + eor \tmp1\().16b, \tmp1\().16b, \tmp1\().16b + aese \des0\().16b, \tmp1\().16b + multi_matrix \des0, ATAHMaskV, ATALMaskV, \tmp2 + aese \des1\().16b, \tmp1\().16b + multi_matrix \des1, ATAHMaskV, ATALMaskV, \tmp2 +.endm + +.macro round, c0, c1, c2, c3, k + mov tmpw0, \k + eor tmpw1, \c1, \c2 + eor tmpw0, \c3, tmpw0 + eor tmpw2, tmpw1, tmpw0 + mov TMP0.s[0], tmpw2 + /* nonlinear transformation */ + sbox TMP1, TMP0, TMP2, TMP3 + /* linear transformation */ + mov tmpw2, TMP1.s[0] + ror tmpw0, tmpw2, #(32-10) + eor tmpw0, tmpw0, tmpw2, ror #(32-2) + ror tmpw1, tmpw2, #(32-24) + eor tmpw1, tmpw1, tmpw2, ror #(32-18) + eor tmpw0, tmpw0, tmpw1 + eor tmpw2, tmpw0, tmpw2 + eor \c0, \c0, tmpw2 +.endm + +.macro round4_enc, k + round m0, m1, m2, m3, \k\().s[0] + round m1, m2, m3, m0, \k\().s[1] + round m2, m3, m0, m1, \k\().s[2] + round m3, m0, m1, m2, \k\().s[3] +.endm + +.macro round4_dec, k + round m0, m1, m2, m3, \k\().s[3] + round m1, m2, m3, m0, \k\().s[2] + round m2, m3, m0, m1, \k\().s[1] + round m3, m0, m1, m2, \k\().s[0] +.endm + +.macro encrypt_block_no_rev, in + mov m0, \in\().s[0] + mov m1, \in\().s[1] + mov m2, \in\().s[2] + mov m3, \in\().s[3] + round4_enc RK0 + round4_enc RK1 + round4_enc RK2 + round4_enc RK3 + round4_enc RK4 + round4_enc RK5 + round4_enc RK6 + round4_enc RK7 + mov \in\().s[0], m3 + mov \in\().s[1], m2 + mov \in\().s[2], m1 + mov \in\().s[3], m0 +.endm + +.macro encrypt_block, in + rev32 \in\().16b, \in\().16b + encrypt_block_no_rev \in + rev32 \in\().16b, \in\().16b +.endm + +.macro decrypt_block_no_rev, in + mov m0, \in\().s[0] + mov m1, \in\().s[1] + mov m2, \in\().s[2] + mov m3, \in\().s[3] + round4_dec RK7 + round4_dec RK6 + round4_dec RK5 + round4_dec RK4 + round4_dec RK3 + round4_dec RK2 + round4_dec RK1 + round4_dec RK0 + mov \in\().s[0], m3 + mov \in\().s[1], m2 + mov \in\().s[2], m1 + mov \in\().s[3], m0 +.endm + +.macro decrypt_block, in + rev32 \in\().16b, \in\().16b + decrypt_block_no_rev \in + rev32 \in\().16b, \in\().16b +.endm + +LOCAL_FUNC sm4_encrypt_block1x , : + encrypt_block BLK0 + ret +END_FUNC sm4_encrypt_block1x + +LOCAL_FUNC sm4_decrypt_block1x , : + decrypt_block BLK0 + ret +END_FUNC sm4_decrypt_block1x + +.macro transpose_4x4, s0, s1, s2, s3 + zip1 TMP0.4s, \s0\().4s, \s1\().4s + zip1 TMP1.4s, \s2\().4s, \s3\().4s + zip2 TMP2.4s, \s0\().4s, \s1\().4s + zip2 TMP3.4s, \s2\().4s, \s3\().4s + zip1 \s0\().2d, TMP0.2d, TMP1.2d + zip2 \s1\().2d, TMP0.2d, TMP1.2d + zip1 \s2\().2d, TMP2.2d, TMP3.2d + zip2 \s3\().2d, TMP2.2d, TMP3.2d +.endm + +.macro rotate_clockwise_90, s0, s1, s2, s3 + zip1 TMP0.4s, \s1\().4s, \s0\().4s + zip2 TMP1.4s, \s1\().4s, \s0\().4s + zip1 TMP2.4s, \s3\().4s, \s2\().4s + zip2 TMP3.4s, \s3\().4s, \s2\().4s + zip1 \s0\().2d, TMP2.2d, TMP0.2d + zip2 \s1\().2d, TMP2.2d, TMP0.2d + zip1 \s2\().2d, TMP3.2d, TMP1.2d + zip2 \s3\().2d, TMP3.2d, TMP1.2d +.endm + + +.macro round_4x, s0, s1, s2, s3, k + dup TMP8.4s, \k + eor TMP1.16b, \s2\().16b, \s3\().16b + eor TMP8.16b, TMP8.16b, \s1\().16b + eor TMP8.16b, TMP8.16b, TMP1.16b + + /* nonlinear transformation */ + sbox TMP0, TMP8, TMP2, TMP3 + + /* linear transformation */ + shl TMP1.4s, TMP0.4s, #2 + shl TMP2.4s, TMP0.4s, #10 + shl TMP3.4s, TMP0.4s, #18 + shl TMP4.4s, TMP0.4s, #24 + sri TMP1.4s, TMP0.4s, #(32-2) + sri TMP2.4s, TMP0.4s, #(32-10) + sri TMP3.4s, TMP0.4s, #(32-18) + sri TMP4.4s, TMP0.4s, #(32-24) + eor TMP0.16b, TMP0.16b, TMP1.16b + eor TMP2.16b, TMP2.16b, TMP3.16b + eor TMP4.16b, TMP4.16b, \s0\().16b + eor TMP0.16b, TMP0.16b, TMP2.16b + eor \s0\().16b, TMP0.16b, TMP4.16b +.endm + +.macro round4_4x, k + round_4x BLK0, BLK1, BLK2, BLK3, \k\().s[0] + round_4x BLK1, BLK2, BLK3, BLK0, \k\().s[1] + round_4x BLK2, BLK3, BLK0, BLK1, \k\().s[2] + round_4x BLK3, BLK0, BLK1, BLK2, \k\().s[3] +.endm + +LOCAL_FUNC sm4_encrypt_block4x , : + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + + transpose_4x4 BLK0, BLK1, BLK2, BLK3 + + round4_4x RK0 + round4_4x RK1 + round4_4x RK2 + round4_4x RK3 + round4_4x RK4 + round4_4x RK5 + round4_4x RK6 + round4_4x RK7 + + rotate_clockwise_90 BLK0, BLK1, BLK2, BLK3 + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + ret +END_FUNC sm4_encrypt_block4x + +.macro round_8x, s0, s1, s2, s3, t0, t1, t2, t3, k + dup TMP8.4s, \k + eor TMP0.16b, \s2\().16b, \s3\().16b + mov TMP7.16b, TMP8.16b + eor TMP1.16b, \t2\().16b, \t3\().16b + eor TMP8.16b, TMP8.16b, \s1\().16b + eor TMP7.16b, TMP7.16b, \t1\().16b + eor TMP8.16b, TMP8.16b, TMP0.16b + eor TMP7.16b, TMP7.16b, TMP1.16b + + /* nonlinear transformation */ + sbox_double TMP0, TMP8, TMP1, TMP7, TMP2, TMP3 + + /* linear transformation */ + shl TMP6.4s, TMP0.4s, #2 + shl TMP8.4s, TMP1.4s, #2 + shl TMP2.4s, TMP0.4s, #10 + shl TMP5.4s, TMP1.4s, #10 + shl TMP3.4s, TMP0.4s, #18 + shl TMP4.4s, TMP0.4s, #24 + sri TMP6.4s, TMP0.4s, #(32-2) + sri TMP2.4s, TMP0.4s, #(32-10) + sri TMP3.4s, TMP0.4s, #(32-18) + sri TMP4.4s, TMP0.4s, #(32-24) + eor TMP0.16b, TMP0.16b, TMP6.16b + eor TMP2.16b, TMP2.16b, TMP3.16b + shl TMP6.4s, TMP1.4s, #18 + shl TMP7.4s, TMP1.4s, #24 + sri TMP8.4s, TMP1.4s, #(32-2) + sri TMP5.4s, TMP1.4s, #(32-10) + sri TMP6.4s, TMP1.4s, #(32-18) + sri TMP7.4s, TMP1.4s, #(32-24) + eor TMP4.16b, TMP4.16b, \s0\().16b + eor TMP1.16b, TMP1.16b, TMP8.16b + eor \s0\().16b, TMP0.16b, TMP2.16b + eor \s0\().16b, \s0\().16b, TMP4.16b + eor TMP5.16b, TMP5.16b, TMP6.16b + eor TMP7.16b, TMP7.16b, \t0\().16b + eor TMP1.16b, TMP1.16b, TMP5.16b + eor \t0\().16b, TMP1.16b, TMP7.16b +.endm + +.macro round4_8x, k + round_8x BLK0, BLK1, BLK2, BLK3, BLK4, BLK5, BLK6, BLK7, \k\().s[0] + round_8x BLK1, BLK2, BLK3, BLK0, BLK5, BLK6, BLK7, BLK4, \k\().s[1] + round_8x BLK2, BLK3, BLK0, BLK1, BLK6, BLK7, BLK4, BLK5, \k\().s[2] + round_8x BLK3, BLK0, BLK1, BLK2, BLK7, BLK4, BLK5, BLK6, \k\().s[3] +.endm + +LOCAL_FUNC sm4_encrypt_block8x , : + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + rev32 BLK4.16b, BLK4.16b + rev32 BLK5.16b, BLK5.16b + rev32 BLK6.16b, BLK6.16b + rev32 BLK7.16b, BLK7.16b + + transpose_4x4 BLK0, BLK1, BLK2, BLK3 + transpose_4x4 BLK4, BLK5, BLK6, BLK7 + + round4_8x RK0 + round4_8x RK1 + round4_8x RK2 + round4_8x RK3 + round4_8x RK4 + round4_8x RK5 + round4_8x RK6 + round4_8x RK7 + + rotate_clockwise_90 BLK0, BLK1, BLK2, BLK3 + rotate_clockwise_90 BLK4, BLK5, BLK6, BLK7 + + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + rev32 BLK4.16b, BLK4.16b + rev32 BLK5.16b, BLK5.16b + rev32 BLK6.16b, BLK6.16b + rev32 BLK7.16b, BLK7.16b + ret +END_FUNC sm4_encrypt_block8x + +.macro inc_le128, vctr, low, high + mov \vctr\().d[1], \high + mov \vctr\().d[0], \low + adds \high, \high, #1 + adc \low, \low, xzr + rev64 \vctr\().16b, \vctr\().16b +.endm + +.macro mov_reg_to_vec, desv, src0, src1 + mov \desv\().d[0], \src0 + mov \desv\().d[1], \src1 +.endm + +.macro next_tweak, des0, des1, src0, src1 + mov tmpw2, 0x87 + extr tmpx0, \src1, \src1, #32 + extr \des1, \src1, \src0, #63 + and tmpw1, tmpw2, tmpw0, asr#31 + eor \des0, tmpx1, \src0, lsl#1 +.endm + +.macro next_tweak_vec, desv, srcv + mov tw0l, \srcv\().d[0] + mov tw0h, \srcv\().d[1] + next_tweak tw1l, tw1h, tw0l, tw0h + mov \desv\().d[0], tw1l + mov \desv\().d[1], tw1h +.endm + +LOCAL_DATA .Lck , : + .long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 + .long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 + .long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 + .long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 + .long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 + .long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 + .long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 + .long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 +END_DATA .Lck + +LOCAL_DATA .Lfk , : + .long 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc +END_DATA .Lfk + +LOCAL_DATA .Lshuffles , : + .long 0x07060504, 0x0B0A0908, 0x0F0E0D0C, 0x03020100 +END_DATA .Lshuffles + +LOCAL_DATA .Lsbox_magic , : + .dword 0x0b0e0104070a0d00, 0x0306090c0f020508 + .dword 0x62185a2042387a00, 0x22581a6002783a40 + .dword 0x15df62a89e54e923, 0xc10bb67c4a803df7 + .dword 0xb9aa6b78c1d21300, 0x1407c6d56c7fbead + .dword 0x6404462679195b3b, 0xe383c1a1fe9edcbc + .dword 0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f +END_DATA .Lsbox_magic + +.macro sm4_setkey + ld1 {v5.4s}, [x1] + load_sbox_matrix + rev32 v5.16b, v5.16b + adr x5, .Lfk + ld1 {v6.4s}, [x5] + eor v5.16b, v5.16b, v6.16b + mov x6, #32 + adr x5, .Lck +1: + mov w7, v5.s[1] + ldr w8, [x5], #4 + eor w8, w8, w7 + mov w7, v5.s[2] + eor w8, w8, w7 + mov w7, v5.s[3] + eor w8, w8, w7 + + /* optimize sbox using AESE instruction */ + mov TMP0.s[0], w8 + sbox TMP1, TMP0, TMP2, TMP3 + mov w7, TMP1.s[0] + + /* linear transformation */ + eor w8, w7, w7, ror #19 + eor w8, w8, w7, ror #9 + mov w7, v5.s[0] + eor w8, w8, w7 + mov v5.s[0], w8 + ext v5.16b, v5.16b, v5.16b, 4 + subs x6, x6, #1 +.endm + +/* + * void neon_sm4_setkey_enc(uint32_t sk[32], uint8_t const key[16]); + * x0: round key + * x1: user key + */ +FUNC neon_sm4_setkey_enc , : + sm4_setkey + str w8, [x0], #4 + b.ne 1b + ret +END_FUNC neon_sm4_setkey_enc + +/* + * void neon_sm4_setkey_dec(uint32_t sk[32], uint8_t const key[16]); + * x0: round key + * x1: user key + */ +FUNC neon_sm4_setkey_dec , : + add x0, x0, 124 + sm4_setkey + str w8, [x0], #-4 + b.ne 1b + ret +END_FUNC neon_sm4_setkey_dec + +/* + * void neon_sm4_ecb_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len); + * x0: output + * x1: input + * x2: round key + * w3: length + */ +FUNC neon_sm4_ecb_encrypt , : + frame_push + load_sbox_matrix + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + +.Lecbloop8x: + cmp w3, 8 + b.lt .Lecb4x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + ld1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x1], #64 + bl sm4_encrypt_block8x + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w3, w3, #8 + b.gt .Lecbloop8x + +.Lecb4x: + cmp w3, 1 + b.lt .Lecbout + cmp w3, 2 + b.lt .Lecb1x + cmp w3, 3 + b.lt .Lecb2x + cmp w3, 4 + b.lt .Lecb3x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + bl sm4_encrypt_block4x + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w3, w3, #4 + b .Lecb4x + +.Lecb3x: + ld1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x1], #48 + bl sm4_encrypt_block4x + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w3, w3, #3 + b.le .Lecbout + +.Lecb2x: + ld1 {BLK0.16b, BLK1.16b}, [x1], #32 + bl sm4_encrypt_block4x + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w3, w3, #2 + b.le .Lecbout + +.Lecb1x: + ld1 {BLK0.16b}, [x1], #16 + bl sm4_encrypt_block1x + st1 {BLK0.16b}, [x0], #16 + +.Lecbout: + frame_pop + ret + +END_FUNC neon_sm4_ecb_encrypt + +/* + * void neon_sm4_cbc_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len, + * uint8_t iv[]); + * x0: output + * x1: input + * x2: round key + * w3: length + * x4: iv + */ +FUNC neon_sm4_cbc_encrypt , : + frame_push + load_sbox_matrix + + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + ld1 {IV.16b}, [x4] + +.Lcbcencloop4x: + cmp w3, 4 + b.lt .Lcbcenc1x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + eor BLK0.16b, BLK0.16b, IV.16b + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + encrypt_block_no_rev BLK0 + eor BLK1.16b, BLK1.16b, BLK0.16b + encrypt_block_no_rev BLK1 + rev32 BLK0.16b, BLK0.16b + eor BLK2.16b, BLK2.16b, BLK1.16b + encrypt_block_no_rev BLK2 + rev32 BLK1.16b, BLK1.16b + eor BLK3.16b, BLK3.16b, BLK2.16b + encrypt_block_no_rev BLK3 + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + mov IV.16b, BLK3.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + subs w3, w3, #4 + b .Lcbcencloop4x +.Lcbcenc1x: + cmp w3, 1 + b.lt .Lcbcencout +.Lcbcencloop: + ld1 {BLK0.16b}, [x1], #16 + eor BLK0.16b, BLK0.16b, IV.16b + bl sm4_encrypt_block1x + mov IV.16b, BLK0.16b + st1 {BLK0.16b}, [x0], #16 + subs w3, w3, #1 + bne .Lcbcencloop +.Lcbcencout: + st1 {IV.16b}, [x4] + frame_pop + ret +END_FUNC neon_sm4_cbc_encrypt + +/* + * void neon_sm4_cbc_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len, + * uint8_t iv[]); + * x0: output + * x1: input + * x2: round key + * w3: length + * x4: iv + */ +FUNC neon_sm4_cbc_decrypt , : + frame_push + load_sbox_matrix + + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + ld1 {IV.16b}, [x4] + +.Lcbcdecloop8x: + cmp w3, 8 + b.lt .Lcbcdec4x + + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + ld1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x1], #64 + bl sm4_encrypt_block8x + sub x5, x1, #128 + eor BLK0.16b, BLK0.16b, IV.16b + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x5], #64 + eor BLK1.16b, BLK1.16b, TMP0.16b + eor BLK2.16b, BLK2.16b, TMP1.16b + eor BLK3.16b, BLK3.16b, TMP2.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + ld1 {TMP4.16b, TMP5.16b, TMP6.16b, TMP7.16b}, [x5], #64 + eor BLK4.16b, BLK4.16b, TMP3.16b + eor BLK5.16b, BLK5.16b, TMP4.16b + mov IV.16b, TMP7.16b + eor BLK6.16b, BLK6.16b, TMP5.16b + eor BLK7.16b, BLK7.16b, TMP6.16b + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w3, w3, #8 + b.gt .Lcbcdecloop8x + +.Lcbcdec4x: + cmp w3, 1 + b.lt .Lcbcdecout + cmp w3, 2 + b.lt .Lcbcdec1x + cmp w3, 3 + b.lt .Lcbcdec2x + cmp w3, 4 + b.lt .Lcbcdec3x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + bl sm4_encrypt_block4x + sub x5, x1, 64 + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x5], #64 + eor BLK0.16b, BLK0.16b, IV.16b + eor BLK1.16b, BLK1.16b, TMP0.16b + eor BLK2.16b, BLK2.16b, TMP1.16b + eor BLK3.16b, BLK3.16b, TMP2.16b + mov IV.16b, TMP3.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w3, w3, #4 + b .Lcbcdec4x + +.Lcbcdec3x: + ld1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x1], #48 + bl sm4_encrypt_block4x + sub x5, x1, 48 + ld1 {TMP0.16b, TMP1.16b, TMP2.16b}, [x5], #48 + eor BLK0.16b, BLK0.16b, IV.16b + eor BLK1.16b, BLK1.16b, TMP0.16b + eor BLK2.16b, BLK2.16b, TMP1.16b + mov IV.16b, TMP2.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w3, w3, #3 + b.le .Lcbcdecout + +.Lcbcdec2x: + ld1 {BLK0.16b, BLK1.16b}, [x1], #32 + bl sm4_encrypt_block4x + sub x5, x1, 32 + ld1 {TMP0.16b, TMP1.16b}, [x5], #32 + eor BLK0.16b, BLK0.16b, IV.16b + eor BLK1.16b, BLK1.16b, TMP0.16b + mov IV.16b, TMP1.16b + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w3, w3, #2 + b.le .Lcbcdecout + +.Lcbcdec1x: + ld1 {BLK0.16b}, [x1], #16 + bl sm4_encrypt_block1x + sub x5, x1, 16 + ld1 {TMP0.16b}, [x5], #16 + eor BLK0.16b, BLK0.16b, IV.16b + mov IV.16b, TMP0.16b + st1 {BLK0.16b}, [x0], #16 + +.Lcbcdecout: + st1 {IV.16b}, [x4] + frame_pop + ret +END_FUNC neon_sm4_cbc_decrypt + +/* + * void neon_sm4_ctr_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len, + * uint8_t iv[]); + * x0: output + * x1: input + * x2: round key + * w3: length + * x4: iv + */ +FUNC neon_sm4_ctr_encrypt , : + frame_push + load_sbox_matrix + + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + ldp x7, x8, [x4] + rev x7, x7 + rev x8, x8 + +.Lctrloop8x: + cmp w3, 8 + b.lt .Lctr4x + + /* construct CTRs */ + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + inc_le128 BLK2, x7, x8 + inc_le128 BLK3, x7, x8 + inc_le128 BLK4, x7, x8 + inc_le128 BLK5, x7, x8 + inc_le128 BLK6, x7, x8 + inc_le128 BLK7, x7, x8 + bl sm4_encrypt_block8x + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x1], #64 + ld1 {TMP4.16b, TMP5.16b, TMP6.16b, TMP7.16b}, [x1], #64 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + eor BLK4.16b, BLK4.16b, TMP4.16b + eor BLK5.16b, BLK5.16b, TMP5.16b + eor BLK6.16b, BLK6.16b, TMP6.16b + eor BLK7.16b, BLK7.16b, TMP7.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w3, w3, #8 + b.gt .Lctrloop8x + +.Lctr4x: + cmp w3, 1 + b.lt .Lctrout + cmp w3, 2 + b.lt .Lctr1x + cmp w3, 3 + b.lt .Lctr2x + cmp w3, 4 + b.lt .Lctr3x + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + inc_le128 BLK2, x7, x8 + inc_le128 BLK3, x7, x8 + bl sm4_encrypt_block4x + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x1], #64 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w3, w3, #4 + b .Lctr4x + +.Lctr3x: + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + inc_le128 BLK2, x7, x8 + bl sm4_encrypt_block4x + ld1 {TMP0.16b, TMP1.16b, TMP2.16b}, [x1], #48 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w3, w3, #3 + b.le .Lctrout + +.Lctr2x: + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + bl sm4_encrypt_block4x + ld1 {TMP0.16b, TMP1.16b}, [x1], #32 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w3, w3, #2 + b.le .Lctrout + +.Lctr1x: + inc_le128 BLK0, x7, x8 + bl sm4_encrypt_block1x + ld1 {TMP0.16b}, [x1], #16 + eor BLK0.16b, BLK0.16b, TMP0.16b + st1 {BLK0.16b}, [x0], #16 + +.Lctrout: + rev x7, x7 + rev x8, x8 + stp x7, x8, [x4] + frame_pop + ret +END_FUNC neon_sm4_ctr_encrypt + +/* + * x0: output + * x1: input + * x2: round key1 + * x3: round key2 + * w4: blocks + * x26: enc/dec + */ +LOCAL_FUNC xts_do_cipher , : + stp x29, x30, [sp, #-16]! + mov x29, sp + load_sbox_matrix + ld1 {IV.16b}, [x5] + /* load round key2 for first tweak */ + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x3], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x3], #64 + encrypt_block IV + /* load round key1 for block cipher */ + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + /* w6: remain */ + and w6, w4, #0x0F + /* w4: blocks */ + lsr w4, w4, 4 + /* blocks == 0: ret */ + cmp w4, #1 + b.lt .Lxtsout + cmp w6, 0 + b.eq .Lxtsblks + subs w4, w4, #1 + b.eq .Lxtstail +.Lxtsblks: + mov tw0l, IV.d[0] + mov tw0h, IV.d[1] + next_tweak tw1l, tw1h, tw0l, tw0h + next_tweak tw2l, tw2h, tw1l, tw1h + next_tweak tw3l, tw3h, tw2l, tw2h + next_tweak tw4l, tw4h, tw3l, tw3h + next_tweak tw5l, tw5h, tw4l, tw4h + next_tweak tw6l, tw6h, tw5l, tw5h + next_tweak tw7l, tw7h, tw6l, tw6h +.Lxtsloop8x: + cmp w4, 8 + b.lt .Lxts4x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + mov_reg_to_vec TMP0, tw0l, tw0h + mov_reg_to_vec TMP1, tw1l, tw1h + mov_reg_to_vec TMP2, tw2l, tw2h + mov_reg_to_vec TMP3, tw3l, tw3h + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + ld1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x1], #64 + mov_reg_to_vec TMP4, tw4l, tw4h + mov_reg_to_vec TMP5, tw5l, tw5h + mov_reg_to_vec TMP6, tw6l, tw6h + mov_reg_to_vec IV, tw7l, tw7h + eor BLK4.16b, BLK4.16b, TMP4.16b + eor BLK5.16b, BLK5.16b, TMP5.16b + eor BLK6.16b, BLK6.16b, TMP6.16b + eor BLK7.16b, BLK7.16b, IV.16b + + bl sm4_encrypt_block8x + + mov_reg_to_vec TMP0, tw0l, tw0h + next_tweak tw0l, tw0h, tw7l, tw7h + mov_reg_to_vec TMP1, tw1l, tw1h + next_tweak tw1l, tw1h, tw0l, tw0h + mov_reg_to_vec TMP2, tw2l, tw2h + next_tweak tw2l, tw2h, tw1l, tw1h + mov_reg_to_vec TMP3, tw3l, tw3h + next_tweak tw3l, tw3h, tw2l, tw2h + mov_reg_to_vec TMP4, tw4l, tw4h + next_tweak tw4l, tw4h, tw3l, tw3h + mov_reg_to_vec TMP5, tw5l, tw5h + next_tweak tw5l, tw5h, tw4l, tw4h + mov_reg_to_vec TMP6, tw6l, tw6h + next_tweak tw6l, tw6h, tw5l, tw5h + mov_reg_to_vec IV, tw7l, tw7h + next_tweak tw7l, tw7h, tw6l, tw6h + + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + eor BLK4.16b, BLK4.16b, TMP4.16b + eor BLK5.16b, BLK5.16b, TMP5.16b + eor BLK6.16b, BLK6.16b, TMP6.16b + eor BLK7.16b, BLK7.16b, IV.16b + + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w4, w4, #8 + b.gt .Lxtsloop8x + +.Lxts4x: + cmp w4, 1 + b.lt .Lxtsblksout + cmp w4, 2 + b.lt .Lxts1x + cmp w4, 3 + b.lt .Lxts2x + cmp w4, 4 + b.lt .Lxts3x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + mov_reg_to_vec BLK4, tw0l, tw0h + mov_reg_to_vec BLK5, tw1l, tw1h + mov_reg_to_vec BLK6, tw2l, tw2h + mov_reg_to_vec IV, tw3l, tw3h + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, BLK6.16b + eor BLK3.16b, BLK3.16b, IV.16b + bl sm4_encrypt_block4x + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, BLK6.16b + eor BLK3.16b, BLK3.16b, IV.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w4, w4, #4 + + mov tw0l, tw4l + mov tw0h, tw4h + mov tw1l, tw5l + mov tw1h, tw5h + mov tw2l, tw6l + mov tw2h, tw6h + b .Lxts4x + +.Lxts3x: + ld1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x1], #48 + mov_reg_to_vec BLK4, tw0l, tw0h + mov_reg_to_vec BLK5, tw1l, tw1h + mov_reg_to_vec IV, tw2l, tw2h + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, IV.16b + bl sm4_encrypt_block4x + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, IV.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w4, w4, #3 + b.le .Lxtsblksout + +.Lxts2x: + ld1 {BLK0.16b, BLK1.16b}, [x1], #32 + mov_reg_to_vec BLK4, tw0l, tw0h + mov_reg_to_vec IV, tw1l, tw1h + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, IV.16b + bl sm4_encrypt_block4x + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, IV.16b + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w4, w4, #2 + b.le .Lxtsblksout + +.Lxts1x: + ld1 {BLK0.16b}, [x1], #16 + mov_reg_to_vec IV, tw0l, tw0h + eor BLK0.16b, BLK0.16b, IV.16b + bl sm4_encrypt_block1x + eor BLK0.16b, BLK0.16b, IV.16b + st1 {BLK0.16b}, [x0], #16 +.Lxtsblksout: + cmp w6, 0 + /* if encrypt some blocks with a partial block */ + next_tweak_vec IV, IV + b.eq .Lxtsout +.Lxtstail: + next_tweak_vec TMP7, IV + cmp x26, 1 + b.eq 1f + /* The last two tweaks IV, TMP7 need to be swapped for decryption */ + mov TMP8.16b, IV.16b + mov IV.16b, TMP7.16b + mov TMP7.16b, TMP8.16b + 1: + ld1 {BLK0.16b}, [x1], #16 + eor BLK0.16b, BLK0.16b, IV.16b + bl sm4_encrypt_block1x + eor BLK0.16b, BLK0.16b, IV.16b + st1 {BLK0.16b}, [x0], #16 + sub x7, x0, 16 + 10: + subs x6, x6, 1 + ldrb tmpw0, [x7, x6] + strb tmpw0, [x0, x6] + ldrb tmpw0, [x1, x6] + strb tmpw0, [x7, x6] + b.gt 10b + ld1 {BLK0.16b}, [x7] + eor BLK0.16b, BLK0.16b, TMP7.16b + bl sm4_encrypt_block1x + eor BLK0.16b, BLK0.16b, TMP7.16b + st1 {BLK0.16b}, [x7] + +.Lxtsout: + /* load round key2 for last tweak */ + sub x3, x3, #128 + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x3], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x3], #64 + /* decrypt last tweak for next update */ + decrypt_block IV + st1 {IV.16b}, [x5] + ldp x29, x30, [sp], #16 + ret +END_FUNC xts_do_cipher + +/* + * void neon_sm4_xts_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], uint8_t const rk2[], + * size_t len, uint8_t iv[]) + * x0: output + * x1: input + * x2: round key1 + * x3: round key2 + * w4: len + * x5: iv + */ +FUNC neon_sm4_xts_encrypt , : + frame_push + mov x26, 1 + bl xts_do_cipher + frame_pop + ret +END_FUNC neon_sm4_xts_encrypt + +/* + * void neon_sm4_xts_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], uint8_t const rk2[], + * size_t len, uint8_t iv[]) + * x0: output + * x1: input + * x2: round key1 + * x3: round key2 + * w4: len + * x5: iv + */ +FUNC neon_sm4_xts_decrypt , : + frame_push + mov x26, 0 + bl xts_do_cipher + frame_pop + ret +END_FUNC neon_sm4_xts_decrypt diff --git a/optee_os/core/arch/arm/crypto/sm4_armv8a_ce.c b/optee_os/core/arch/arm/crypto/sm4_armv8a_ce.c new file mode 100644 index 0000000..cd8f35b --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm4_armv8a_ce.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) Hisilicon Technologies Co., Ltd. 2023. All rights reserved. + * + * SM4 optimization for ARMv8 by SM4 HW instruction, which is an optional + * Cryptographic Extension for ARMv8.2-A. + */ +#include +#include + +#include "sm4_armv8a_ce.h" + +void crypto_accel_sm4_setkey_enc(uint32_t sk[32], const uint8_t key[16]) +{ + uint32_t vfp_state = 0; + + assert(sk && key); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_setkey_enc(sk, key); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_setkey_dec(uint32_t sk[32], const uint8_t key[16]) +{ + uint32_t vfp_state = 0; + + assert(sk && key); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_setkey_dec(sk, key); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_ecb_enc(void *out, const void *in, const void *key, + unsigned int len) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_ecb_encrypt(out, in, key, len); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_cbc_enc(void *out, const void *in, const void *key, + unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_cbc_encrypt(out, in, key, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_cbc_dec(void *out, const void *in, const void *key, + unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_cbc_decrypt(out, in, key, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_ctr_enc(void *out, const void *in, const void *key, + unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_ctr_encrypt(out, in, key, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_xts_enc(void *out, const void *in, const void *key1, + const void *key2, unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key1 && key2 && (len >= 16)); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_xts_encrypt(out, in, key1, key2, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_xts_dec(void *out, const void *in, const void *key1, + const void *key2, unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key1 && key2 && (len >= 16)); + + vfp_state = thread_kernel_enable_vfp(); + ce_sm4_xts_decrypt(out, in, key1, key2, len, iv); + thread_kernel_disable_vfp(vfp_state); +} diff --git a/optee_os/core/arch/arm/crypto/sm4_armv8a_ce.h b/optee_os/core/arch/arm/crypto/sm4_armv8a_ce.h new file mode 100644 index 0000000..5e1c2ed --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm4_armv8a_ce.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) Hisilicon Technologies Co., Ltd. 2023. All rights reserved. + */ + +#ifndef __SM4_ARMV8_CE_H +#define __SM4_ARMV8_CE_H + +#include + +/* Prototypes for assembly functions */ +void ce_sm4_setkey_enc(uint32_t sk[32], uint8_t const key[16]); +void ce_sm4_setkey_dec(uint32_t sk[32], uint8_t const key[16]); +void ce_sm4_ecb_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len); +void ce_sm4_cbc_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len, uint8_t iv[]); +void ce_sm4_cbc_decrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len, uint8_t iv[]); +void ce_sm4_ctr_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len, uint8_t iv[]); +void ce_sm4_xts_encrypt(uint8_t out[], uint8_t const in[], + uint8_t const rk1[], uint8_t const rk2[], size_t len, + uint8_t iv[]); +void ce_sm4_xts_decrypt(uint8_t out[], uint8_t const in[], uint8_t const rk1[], + uint8_t const rk2[], size_t len, uint8_t iv[]); + +#endif /*__SM4_ARMV8_CE_H*/ diff --git a/optee_os/core/arch/arm/crypto/sm4_armv8a_ce_a64.S b/optee_os/core/arch/arm/crypto/sm4_armv8a_ce_a64.S new file mode 100644 index 0000000..1f96bbf --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm4_armv8a_ce_a64.S @@ -0,0 +1,1030 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) Hisilicon Technologies Co., Ltd. 2023. All rights reserved. + * Copyright (C) 2022, Alibaba Group. + * Copyright (C) 2022 Tianjia Zhang + * + * SM4 optimization for ARMv8 by SM4 HW instruction, which is an optional + * Cryptographic Extension for ARMv8.2-A. + * + * The CE implementation refers to Linux kernel (sm4-ce-core.S contributed + * by Tianjia Zhang ). + */ + +#include + +.arch armv8.2-a+crypto+sm4 + +#define tw0l x7 +#define tw0h x8 +#define tw1l x9 +#define tw1h x10 +#define tw2l x11 +#define tw2h x12 +#define tw3l x13 +#define tw3h x14 +#define tw4l x15 +#define tw4h x16 +#define tw5l x17 +#define tw5h x18 +#define tw6l x19 +#define tw6h x20 +#define tw7l x21 +#define tw7h x22 +#define tmpw0 w23 +#define tmpx0 x23 +#define tmpw1 w24 +#define tmpx1 x24 +#define tmpw2 w25 + +/* round keys: v0-v7 */ +#define RK0 v0 +#define RK1 v1 +#define RK2 v2 +#define RK3 v3 +#define RK4 v4 +#define RK5 v5 +#define RK6 v6 +#define RK7 v7 + +/* plain blocks: v8-v15 */ +#define BLK0 v8 +#define BLK1 v9 +#define BLK2 v10 +#define BLK3 v11 +#define BLK4 v12 +#define BLK5 v13 +#define BLK6 v14 +#define BLK7 v15 + +#define TMP0 v16 +#define TMP1 v17 +#define TMP2 v18 +#define TMP3 v19 +#define TMP4 v20 +#define TMP5 v21 +#define TMP6 v22 +#define TMP7 v23 +#define TMP8 v24 +#define IV v25 + +.macro frame_push + stp x15, x16, [sp, #-0x10]! + stp x17, x18, [sp, #-0x10]! + stp x19, x20, [sp, #-0x10]! + stp x21, x22, [sp, #-0x10]! + stp x23, x24, [sp, #-0x10]! + stp x25, x26, [sp, #-0x10]! + stp x27, x28, [sp, #-0x10]! + stp x29, x30, [sp, #-0x10]! + stp d8, d9, [sp, #-0x10]! + stp d10, d11, [sp, #-0x10]! + stp d12, d13, [sp, #-0x10]! + stp d14, d15, [sp, #-0x10]! +.endm + +.macro frame_pop + ldp d14, d15, [sp], #0x10 + ldp d12, d13, [sp], #0x10 + ldp d10, d11, [sp], #0x10 + ldp d8, d9, [sp], #0x10 + ldp x29, x30, [sp], #0x10 + ldp x27, x28, [sp], #0x10 + ldp x25, x26, [sp], #0x10 + ldp x23, x24, [sp], #0x10 + ldp x21, x22, [sp], #0x10 + ldp x19, x20, [sp], #0x10 + ldp x17, x18, [sp], #0x10 + ldp x15, x16, [sp], #0x10 +.endm + +.macro encrypt_block_no_rev, in + sm4e \in\().4s, RK0.4s + sm4e \in\().4s, RK1.4s + sm4e \in\().4s, RK2.4s + sm4e \in\().4s, RK3.4s + sm4e \in\().4s, RK4.4s + sm4e \in\().4s, RK5.4s + sm4e \in\().4s, RK6.4s + sm4e \in\().4s, RK7.4s + rev64 \in\().4s, \in\().4s + ext \in\().16b, \in\().16b, \in\().16b, #8 +.endm + +.macro encrypt_block, in + rev32 \in\().16b, \in\().16b + sm4e \in\().4s, RK0.4s + sm4e \in\().4s, RK1.4s + sm4e \in\().4s, RK2.4s + sm4e \in\().4s, RK3.4s + sm4e \in\().4s, RK4.4s + sm4e \in\().4s, RK5.4s + sm4e \in\().4s, RK6.4s + sm4e \in\().4s, RK7.4s + rev64 \in\().16b, \in\().16b + ext \in\().16b, \in\().16b, \in\().16b, #8 +.endm + +.macro decrypt_block, in + rev32 \in\().16b, \in\().16b + rev64 RK7.4s, RK7.4s; + rev64 RK6.4s, RK6.4s; + rev64 RK5.4s, RK5.4s; + rev64 RK4.4s, RK4.4s; + ext RK7.16b, RK7.16b, RK7.16b, #8; + ext RK6.16b, RK6.16b, RK6.16b, #8; + ext RK5.16b, RK5.16b, RK5.16b, #8; + ext RK4.16b, RK4.16b, RK4.16b, #8; + sm4e \in\().4s, RK7.4s + sm4e \in\().4s, RK6.4s + sm4e \in\().4s, RK5.4s + sm4e \in\().4s, RK4.4s + rev64 RK3.4s, RK3.4s; + rev64 RK2.4s, RK2.4s; + rev64 RK1.4s, RK1.4s; + rev64 RK0.4s, RK0.4s; + ext RK3.16b, RK3.16b, RK3.16b, #8; + ext RK2.16b, RK2.16b, RK2.16b, #8; + ext RK1.16b, RK1.16b, RK1.16b, #8; + ext RK0.16b, RK0.16b, RK0.16b, #8; + sm4e \in\().4s, RK3.4s + sm4e \in\().4s, RK2.4s + sm4e \in\().4s, RK1.4s + sm4e \in\().4s, RK0.4s + rev64 \in\().16b, \in\().16b + ext \in\().16b, \in\().16b, \in\().16b, #8 +.endm + +LOCAL_FUNC sm4_encrypt_block1x , : + encrypt_block BLK0 + ret +END_FUNC sm4_encrypt_block1x + +LOCAL_FUNC sm4_decrypt_block1x , : + decrypt_block BLK0 + ret +END_FUNC sm4_decrypt_block1x + +LOCAL_FUNC sm4_encrypt_block4x , : + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + + sm4e BLK0.4s, RK0.4s + sm4e BLK1.4s, RK0.4s + sm4e BLK2.4s, RK0.4s + sm4e BLK3.4s, RK0.4s + + sm4e BLK0.4s, RK1.4s + sm4e BLK1.4s, RK1.4s + sm4e BLK2.4s, RK1.4s + sm4e BLK3.4s, RK1.4s + + sm4e BLK0.4s, RK2.4s + sm4e BLK1.4s, RK2.4s + sm4e BLK2.4s, RK2.4s + sm4e BLK3.4s, RK2.4s + + sm4e BLK0.4s, RK3.4s + sm4e BLK1.4s, RK3.4s + sm4e BLK2.4s, RK3.4s + sm4e BLK3.4s, RK3.4s + + sm4e BLK0.4s, RK4.4s + sm4e BLK1.4s, RK4.4s + sm4e BLK2.4s, RK4.4s + sm4e BLK3.4s, RK4.4s + + sm4e BLK0.4s, RK5.4s + sm4e BLK1.4s, RK5.4s + sm4e BLK2.4s, RK5.4s + sm4e BLK3.4s, RK5.4s + + sm4e BLK0.4s, RK6.4s + sm4e BLK1.4s, RK6.4s + sm4e BLK2.4s, RK6.4s + sm4e BLK3.4s, RK6.4s + + sm4e BLK0.4s, RK7.4s + sm4e BLK1.4s, RK7.4s + sm4e BLK2.4s, RK7.4s + sm4e BLK3.4s, RK7.4s + + rev64 BLK0.16b, BLK0.16b + rev64 BLK1.16b, BLK1.16b + rev64 BLK2.16b, BLK2.16b + rev64 BLK3.16b, BLK3.16b + + ext BLK0.16b, BLK0.16b, BLK0.16b, #8 + ext BLK1.16b, BLK1.16b, BLK1.16b, #8 + ext BLK2.16b, BLK2.16b, BLK2.16b, #8 + ext BLK3.16b, BLK3.16b, BLK3.16b, #8 + ret +END_FUNC sm4_encrypt_block4x + +LOCAL_FUNC sm4_encrypt_block8x , : + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + rev32 BLK4.16b, BLK4.16b + rev32 BLK5.16b, BLK5.16b + rev32 BLK6.16b, BLK6.16b + rev32 BLK7.16b, BLK7.16b + + sm4e BLK0.4s, RK0.4s + sm4e BLK1.4s, RK0.4s + sm4e BLK2.4s, RK0.4s + sm4e BLK3.4s, RK0.4s + sm4e BLK4.4s, RK0.4s + sm4e BLK5.4s, RK0.4s + sm4e BLK6.4s, RK0.4s + sm4e BLK7.4s, RK0.4s + + sm4e BLK0.4s, RK1.4s + sm4e BLK1.4s, RK1.4s + sm4e BLK2.4s, RK1.4s + sm4e BLK3.4s, RK1.4s + sm4e BLK4.4s, RK1.4s + sm4e BLK5.4s, RK1.4s + sm4e BLK6.4s, RK1.4s + sm4e BLK7.4s, RK1.4s + + sm4e BLK0.4s, RK2.4s + sm4e BLK1.4s, RK2.4s + sm4e BLK2.4s, RK2.4s + sm4e BLK3.4s, RK2.4s + sm4e BLK4.4s, RK2.4s + sm4e BLK5.4s, RK2.4s + sm4e BLK6.4s, RK2.4s + sm4e BLK7.4s, RK2.4s + + sm4e BLK0.4s, RK3.4s + sm4e BLK1.4s, RK3.4s + sm4e BLK2.4s, RK3.4s + sm4e BLK3.4s, RK3.4s + sm4e BLK4.4s, RK3.4s + sm4e BLK5.4s, RK3.4s + sm4e BLK6.4s, RK3.4s + sm4e BLK7.4s, RK3.4s + + sm4e BLK0.4s, RK4.4s + sm4e BLK1.4s, RK4.4s + sm4e BLK2.4s, RK4.4s + sm4e BLK3.4s, RK4.4s + sm4e BLK4.4s, RK4.4s + sm4e BLK5.4s, RK4.4s + sm4e BLK6.4s, RK4.4s + sm4e BLK7.4s, RK4.4s + + sm4e BLK0.4s, RK5.4s + sm4e BLK1.4s, RK5.4s + sm4e BLK2.4s, RK5.4s + sm4e BLK3.4s, RK5.4s + sm4e BLK4.4s, RK5.4s + sm4e BLK5.4s, RK5.4s + sm4e BLK6.4s, RK5.4s + sm4e BLK7.4s, RK5.4s + + sm4e BLK0.4s, RK6.4s + sm4e BLK1.4s, RK6.4s + sm4e BLK2.4s, RK6.4s + sm4e BLK3.4s, RK6.4s + sm4e BLK4.4s, RK6.4s + sm4e BLK5.4s, RK6.4s + sm4e BLK6.4s, RK6.4s + sm4e BLK7.4s, RK6.4s + + sm4e BLK0.4s, RK7.4s + sm4e BLK1.4s, RK7.4s + sm4e BLK2.4s, RK7.4s + sm4e BLK3.4s, RK7.4s + sm4e BLK4.4s, RK7.4s + sm4e BLK5.4s, RK7.4s + sm4e BLK6.4s, RK7.4s + sm4e BLK7.4s, RK7.4s + + rev64 BLK0.16b, BLK0.16b + rev64 BLK1.16b, BLK1.16b + rev64 BLK2.16b, BLK2.16b + rev64 BLK3.16b, BLK3.16b + rev64 BLK4.16b, BLK4.16b + rev64 BLK5.16b, BLK5.16b + rev64 BLK6.16b, BLK6.16b + rev64 BLK7.16b, BLK7.16b + + ext BLK0.16b, BLK0.16b, BLK0.16b, #8 + ext BLK1.16b, BLK1.16b, BLK1.16b, #8 + ext BLK2.16b, BLK2.16b, BLK2.16b, #8 + ext BLK3.16b, BLK3.16b, BLK3.16b, #8 + ext BLK4.16b, BLK4.16b, BLK4.16b, #8 + ext BLK5.16b, BLK5.16b, BLK5.16b, #8 + ext BLK6.16b, BLK6.16b, BLK6.16b, #8 + ext BLK7.16b, BLK7.16b, BLK7.16b, #8 + ret +END_FUNC sm4_encrypt_block8x + +.macro inc_le128, vctr, low, high + mov \vctr\().d[1], \high + mov \vctr\().d[0], \low + adds \high, \high, #1 + adc \low, \low, xzr + rev64 \vctr\().16b, \vctr\().16b +.endm + +.macro mov_reg_to_vec, desv, src0, src1 + mov \desv\().d[0], \src0 + mov \desv\().d[1], \src1 +.endm + +.macro next_tweak, des0, des1, src0, src1 + mov tmpw2, 0x87 + extr tmpx0, \src1, \src1, #32 + extr \des1, \src1, \src0, #63 + and tmpw1, tmpw2, tmpw0, asr#31 + eor \des0, tmpx1, \src0, lsl#1 +.endm + +.macro next_tweak_vec, desv, srcv + mov tw0l, \srcv\().d[0] + mov tw0h, \srcv\().d[1] + next_tweak tw1l, tw1h, tw0l, tw0h + mov \desv\().d[0], tw1l + mov \desv\().d[1], tw1h +.endm + +LOCAL_DATA .Lck , : + .long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 + .long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 + .long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 + .long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 + .long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 + .long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 + .long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 + .long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 +END_DATA .Lck + +LOCAL_DATA .Lfk , : + .long 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc +END_DATA .Lfk + +/* + * void ce_sm4_setkey_enc(uint32_t sk[32], uint8_t const key[16]); + * x0: round key + * x1: user key + */ +FUNC ce_sm4_setkey_enc , : + ld1 {RK0.4s}, [x1] + adr x2, .Lfk + ld1 {TMP8.4s}, [x2] + adr x2, .Lck + ld1 {TMP0.4s, TMP1.4s, TMP2.4s, TMP3.4s}, [x2], 64 + rev32 RK0.16b, RK0.16b + ld1 {TMP4.4s, TMP5.4s, TMP6.4s, TMP7.4s}, [x2] + eor RK0.16b, RK0.16b, TMP8.16b + sm4ekey RK0.4s, RK0.4s, TMP0.4s + sm4ekey RK1.4s, RK0.4s, TMP1.4s + sm4ekey RK2.4s, RK1.4s, TMP2.4s + sm4ekey RK3.4s, RK2.4s, TMP3.4s + sm4ekey RK4.4s, RK3.4s, TMP4.4s + st1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x0], 64 + sm4ekey RK5.4s, RK4.4s, TMP5.4s + sm4ekey RK6.4s, RK5.4s, TMP6.4s + sm4ekey RK7.4s, RK6.4s, TMP7.4s + st1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x0] + ret +END_FUNC ce_sm4_setkey_enc + +/* + * void ce_sm4_setkey_dec(uint32_t sk[32], uint8_t const key[16]); + * x0: round key + * x1: user key + */ +FUNC ce_sm4_setkey_dec , : + ld1 {RK7.4s}, [x1] + adr x2, .Lfk + ld1 {TMP8.4s}, [x2] + adr x2, .Lck + ld1 {TMP0.4s, TMP1.4s, TMP2.4s, TMP3.4s}, [x2], 64 + rev32 RK7.16b, RK7.16b + ld1 {TMP4.4s, TMP5.4s, TMP6.4s, TMP7.4s}, [x2] + eor RK7.16b, RK7.16b, TMP8.16b; + sm4ekey RK7.4s, RK7.4s, TMP0.4s + sm4ekey RK6.4s, RK7.4s, TMP1.4s + sm4ekey RK5.4s, RK6.4s, TMP2.4s + rev64 RK7.4s, RK7.4s + rev64 RK6.4s, RK6.4s + ext RK7.16b, RK7.16b, RK7.16b, #8 + ext RK6.16b, RK6.16b, RK6.16b, #8 + sm4ekey RK4.4s, RK5.4s, TMP3.4s + sm4ekey RK3.4s, RK4.4s, TMP4.4s + rev64 RK5.4s, RK5.4s + rev64 RK4.4s, RK4.4s + ext RK5.16b, RK5.16b, RK5.16b, #8 + ext RK4.16b, RK4.16b, RK4.16b, #8 + sm4ekey RK2.4s, RK3.4s, TMP5.4s + sm4ekey RK1.4s, RK2.4s, TMP6.4s + rev64 RK3.4s, RK3.4s + rev64 RK2.4s, RK2.4s + ext RK3.16b, RK3.16b, RK3.16b, #8 + ext RK2.16b, RK2.16b, RK2.16b, #8 + sm4ekey RK0.4s, RK1.4s, TMP7.4s + rev64 RK1.4s, RK1.4s + rev64 RK0.4s, RK0.4s + ext RK1.16b, RK1.16b, RK1.16b, #8 + ext RK0.16b, RK0.16b, RK0.16b, #8 + st1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x0], 64 + st1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x0] + ret +END_FUNC ce_sm4_setkey_dec + +/* + * void ce_sm4_ecb_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len); + * x0: output + * x1: input + * x2: round key + * w3: length + */ +FUNC ce_sm4_ecb_encrypt , : + frame_push + + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + +.Lecbloop8x: + cmp w3, 8 + b.lt .Lecb4x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + ld1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x1], #64 + bl sm4_encrypt_block8x + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w3, w3, #8 + b.gt .Lecbloop8x + +.Lecb4x: + cmp w3, 1 + b.lt .Lecbout + cmp w3, 2 + b.lt .Lecb1x + cmp w3, 3 + b.lt .Lecb2x + cmp w3, 4 + b.lt .Lecb3x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + bl sm4_encrypt_block4x + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w3, w3, #4 + b .Lecb4x + +.Lecb3x: + ld1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x1], #48 + bl sm4_encrypt_block4x + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w3, w3, #3 + b.le .Lecbout + +.Lecb2x: + ld1 {BLK0.16b, BLK1.16b}, [x1], #32 + bl sm4_encrypt_block4x + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w3, w3, #2 + b.le .Lecbout + +.Lecb1x: + ld1 {BLK0.16b}, [x1], #16 + bl sm4_encrypt_block1x + st1 {BLK0.16b}, [x0], #16 + +.Lecbout: + frame_pop + ret + +END_FUNC ce_sm4_ecb_encrypt + +/* + * void ce_sm4_cbc_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len, + * uint8_t iv[]); + * x0: output + * x1: input + * x2: round key + * w3: length + * x4: iv + */ +FUNC ce_sm4_cbc_encrypt , : + frame_push + + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + ld1 {IV.16b}, [x4] + +.Lcbcencloop4x: + cmp w3, 4 + b.lt .Lcbcenc1x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + eor BLK0.16b, BLK0.16b, IV.16b + rev32 BLK0.16b, BLK0.16b + rev32 BLK1.16b, BLK1.16b + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + encrypt_block_no_rev BLK0 + eor BLK1.16b, BLK1.16b, BLK0.16b + encrypt_block_no_rev BLK1 + rev32 BLK0.16b, BLK0.16b + eor BLK2.16b, BLK2.16b, BLK1.16b + encrypt_block_no_rev BLK2 + rev32 BLK1.16b, BLK1.16b + eor BLK3.16b, BLK3.16b, BLK2.16b + encrypt_block_no_rev BLK3 + rev32 BLK2.16b, BLK2.16b + rev32 BLK3.16b, BLK3.16b + mov IV.16b, BLK3.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + subs w3, w3, #4 + b .Lcbcencloop4x +.Lcbcenc1x: + cmp w3, 1 + b.lt .Lcbcencout +.Lcbcencloop: + ld1 {BLK0.16b}, [x1], #16 + eor BLK0.16b, BLK0.16b, IV.16b + bl sm4_encrypt_block1x + mov IV.16b, BLK0.16b + st1 {BLK0.16b}, [x0], #16 + subs w3, w3, #1 + bne .Lcbcencloop +.Lcbcencout: + st1 {IV.16b}, [x4] + frame_pop + ret +END_FUNC ce_sm4_cbc_encrypt + +/* + * void ce_sm4_cbc_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len, + * uint8_t iv[]); + * x0: output + * x1: input + * x2: round key + * w3: length + * x4: iv + */ +FUNC ce_sm4_cbc_decrypt , : + frame_push + + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + ld1 {IV.16b}, [x4] + +.Lcbcdecloop8x: + cmp w3, 8 + b.lt .Lcbcdec4x + + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + ld1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x1], #64 + bl sm4_encrypt_block8x + sub x5, x1, #128 + eor BLK0.16b, BLK0.16b, IV.16b + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x5], #64 + eor BLK1.16b, BLK1.16b, TMP0.16b + eor BLK2.16b, BLK2.16b, TMP1.16b + eor BLK3.16b, BLK3.16b, TMP2.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + ld1 {TMP4.16b, TMP5.16b, TMP6.16b, TMP7.16b}, [x5], #64 + eor BLK4.16b, BLK4.16b, TMP3.16b + eor BLK5.16b, BLK5.16b, TMP4.16b + mov IV.16b, TMP7.16b + eor BLK6.16b, BLK6.16b, TMP5.16b + eor BLK7.16b, BLK7.16b, TMP6.16b + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w3, w3, #8 + b.gt .Lcbcdecloop8x + +.Lcbcdec4x: + cmp w3, 1 + b.lt .Lcbcdecout + cmp w3, 2 + b.lt .Lcbcdec1x + cmp w3, 3 + b.lt .Lcbcdec2x + cmp w3, 4 + b.lt .Lcbcdec3x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + bl sm4_encrypt_block4x + sub x5, x1, 64 + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x5], #64 + eor BLK0.16b, BLK0.16b, IV.16b + eor BLK1.16b, BLK1.16b, TMP0.16b + eor BLK2.16b, BLK2.16b, TMP1.16b + eor BLK3.16b, BLK3.16b, TMP2.16b + mov IV.16b, TMP3.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w3, w3, #4 + b .Lcbcdec4x + +.Lcbcdec3x: + ld1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x1], #48 + bl sm4_encrypt_block4x + sub x5, x1, 48 + ld1 {TMP0.16b, TMP1.16b, TMP2.16b}, [x5], #48 + eor BLK0.16b, BLK0.16b, IV.16b + eor BLK1.16b, BLK1.16b, TMP0.16b + eor BLK2.16b, BLK2.16b, TMP1.16b + mov IV.16b, TMP2.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w3, w3, #3 + b.le .Lcbcdecout + +.Lcbcdec2x: + ld1 {BLK0.16b, BLK1.16b}, [x1], #32 + bl sm4_encrypt_block4x + sub x5, x1, 32 + ld1 {TMP0.16b, TMP1.16b}, [x5], #32 + eor BLK0.16b, BLK0.16b, IV.16b + eor BLK1.16b, BLK1.16b, TMP0.16b + mov IV.16b, TMP1.16b + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w3, w3, #2 + b.le .Lcbcdecout + +.Lcbcdec1x: + ld1 {BLK0.16b}, [x1], #16 + bl sm4_encrypt_block1x + sub x5, x1, 16 + ld1 {TMP0.16b}, [x5], #16 + eor BLK0.16b, BLK0.16b, IV.16b + mov IV.16b, TMP0.16b + st1 {BLK0.16b}, [x0], #16 + +.Lcbcdecout: + st1 {IV.16b}, [x4] + frame_pop + ret +END_FUNC ce_sm4_cbc_decrypt + +/* + * void ce_sm4_ctr_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk[], size_t len, + * uint8_t iv[]); + * x0: output + * x1: input + * x2: round key + * w3: length + * x4: iv + */ +FUNC ce_sm4_ctr_encrypt , : + frame_push + + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + lsr w3, w3, 4 + ldp x7, x8, [x4] + rev x7, x7 + rev x8, x8 + +.Lctrloop8x: + cmp w3, 8 + b.lt .Lctr4x + + /* construct CTRs */ + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + inc_le128 BLK2, x7, x8 + inc_le128 BLK3, x7, x8 + inc_le128 BLK4, x7, x8 + inc_le128 BLK5, x7, x8 + inc_le128 BLK6, x7, x8 + inc_le128 BLK7, x7, x8 + bl sm4_encrypt_block8x + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x1], #64 + ld1 {TMP4.16b, TMP5.16b, TMP6.16b, TMP7.16b}, [x1], #64 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + eor BLK4.16b, BLK4.16b, TMP4.16b + eor BLK5.16b, BLK5.16b, TMP5.16b + eor BLK6.16b, BLK6.16b, TMP6.16b + eor BLK7.16b, BLK7.16b, TMP7.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w3, w3, #8 + b.gt .Lctrloop8x + +.Lctr4x: + cmp w3, 1 + b.lt .Lctrout + cmp w3, 2 + b.lt .Lctr1x + cmp w3, 3 + b.lt .Lctr2x + cmp w3, 4 + b.lt .Lctr3x + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + inc_le128 BLK2, x7, x8 + inc_le128 BLK3, x7, x8 + bl sm4_encrypt_block4x + ld1 {TMP0.16b, TMP1.16b, TMP2.16b, TMP3.16b}, [x1], #64 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w3, w3, #4 + b .Lctr4x + +.Lctr3x: + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + inc_le128 BLK2, x7, x8 + bl sm4_encrypt_block4x + ld1 {TMP0.16b, TMP1.16b, TMP2.16b}, [x1], #48 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w3, w3, #3 + b.le .Lctrout + +.Lctr2x: + inc_le128 BLK0, x7, x8 + inc_le128 BLK1, x7, x8 + bl sm4_encrypt_block4x + ld1 {TMP0.16b, TMP1.16b}, [x1], #32 + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w3, w3, #2 + b.le .Lctrout + +.Lctr1x: + inc_le128 BLK0, x7, x8 + bl sm4_encrypt_block1x + ld1 {TMP0.16b}, [x1], #16 + eor BLK0.16b, BLK0.16b, TMP0.16b + st1 {BLK0.16b}, [x0], #16 + +.Lctrout: + rev x7, x7 + rev x8, x8 + stp x7, x8, [x4] + frame_pop + ret +END_FUNC ce_sm4_ctr_encrypt + +/* + * x0: output + * x1: input + * x2: round key1 + * x3: round key2 + * w4: blocks + * x26: enc/dec + */ +LOCAL_FUNC xts_do_cipher , : + stp x29, x30, [sp, #-16]! + mov x29, sp + + ld1 {IV.16b}, [x5] + /* load round key2 for first tweak */ + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x3], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x3], #64 + encrypt_block IV + /* load round key1 for block cipher */ + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x2], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x2], #64 + /* w6: remain */ + and w6, w4, #0x0F + /* w4: blocks */ + lsr w4, w4, 4 + /* blocks == 0: ret */ + cmp w4, #1 + b.lt .Lxtsout + cmp w6, 0 + b.eq .Lxtsblks + subs w4, w4, #1 + b.eq .Lxtstail +.Lxtsblks: + mov tw0l, IV.d[0] + mov tw0h, IV.d[1] + next_tweak tw1l, tw1h, tw0l, tw0h + next_tweak tw2l, tw2h, tw1l, tw1h + next_tweak tw3l, tw3h, tw2l, tw2h + next_tweak tw4l, tw4h, tw3l, tw3h + next_tweak tw5l, tw5h, tw4l, tw4h + next_tweak tw6l, tw6h, tw5l, tw5h + next_tweak tw7l, tw7h, tw6l, tw6h +.Lxtsloop8x: + cmp w4, 8 + b.lt .Lxts4x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + mov_reg_to_vec TMP0, tw0l, tw0h + mov_reg_to_vec TMP1, tw1l, tw1h + mov_reg_to_vec TMP2, tw2l, tw2h + mov_reg_to_vec TMP3, tw3l, tw3h + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + ld1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x1], #64 + mov_reg_to_vec TMP4, tw4l, tw4h + mov_reg_to_vec TMP5, tw5l, tw5h + mov_reg_to_vec TMP6, tw6l, tw6h + mov_reg_to_vec IV, tw7l, tw7h + eor BLK4.16b, BLK4.16b, TMP4.16b + eor BLK5.16b, BLK5.16b, TMP5.16b + eor BLK6.16b, BLK6.16b, TMP6.16b + eor BLK7.16b, BLK7.16b, IV.16b + + bl sm4_encrypt_block8x + + mov_reg_to_vec TMP0, tw0l, tw0h + next_tweak tw0l, tw0h, tw7l, tw7h + mov_reg_to_vec TMP1, tw1l, tw1h + next_tweak tw1l, tw1h, tw0l, tw0h + mov_reg_to_vec TMP2, tw2l, tw2h + next_tweak tw2l, tw2h, tw1l, tw1h + mov_reg_to_vec TMP3, tw3l, tw3h + next_tweak tw3l, tw3h, tw2l, tw2h + mov_reg_to_vec TMP4, tw4l, tw4h + next_tweak tw4l, tw4h, tw3l, tw3h + mov_reg_to_vec TMP5, tw5l, tw5h + next_tweak tw5l, tw5h, tw4l, tw4h + mov_reg_to_vec TMP6, tw6l, tw6h + next_tweak tw6l, tw6h, tw5l, tw5h + mov_reg_to_vec IV, tw7l, tw7h + next_tweak tw7l, tw7h, tw6l, tw6h + + eor BLK0.16b, BLK0.16b, TMP0.16b + eor BLK1.16b, BLK1.16b, TMP1.16b + eor BLK2.16b, BLK2.16b, TMP2.16b + eor BLK3.16b, BLK3.16b, TMP3.16b + eor BLK4.16b, BLK4.16b, TMP4.16b + eor BLK5.16b, BLK5.16b, TMP5.16b + eor BLK6.16b, BLK6.16b, TMP6.16b + eor BLK7.16b, BLK7.16b, IV.16b + + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + st1 {BLK4.16b, BLK5.16b, BLK6.16b, BLK7.16b}, [x0], #64 + subs w4, w4, #8 + b.gt .Lxtsloop8x + +.Lxts4x: + cmp w4, 1 + b.lt .Lxtsblksout + cmp w4, 2 + b.lt .Lxts1x + cmp w4, 3 + b.lt .Lxts2x + cmp w4, 4 + b.lt .Lxts3x + ld1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x1], #64 + mov_reg_to_vec BLK4, tw0l, tw0h + mov_reg_to_vec BLK5, tw1l, tw1h + mov_reg_to_vec BLK6, tw2l, tw2h + mov_reg_to_vec IV, tw3l, tw3h + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, BLK6.16b + eor BLK3.16b, BLK3.16b, IV.16b + bl sm4_encrypt_block4x + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, BLK6.16b + eor BLK3.16b, BLK3.16b, IV.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b, BLK3.16b}, [x0], #64 + sub w4, w4, #4 + + mov tw0l, tw4l + mov tw0h, tw4h + mov tw1l, tw5l + mov tw1h, tw5h + mov tw2l, tw6l + mov tw2h, tw6h + b .Lxts4x + +.Lxts3x: + ld1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x1], #48 + mov_reg_to_vec BLK4, tw0l, tw0h + mov_reg_to_vec BLK5, tw1l, tw1h + mov_reg_to_vec IV, tw2l, tw2h + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, IV.16b + bl sm4_encrypt_block4x + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, BLK5.16b + eor BLK2.16b, BLK2.16b, IV.16b + st1 {BLK0.16b, BLK1.16b, BLK2.16b}, [x0], #48 + subs w4, w4, #3 + b.le .Lxtsblksout + +.Lxts2x: + ld1 {BLK0.16b, BLK1.16b}, [x1], #32 + mov_reg_to_vec BLK4, tw0l, tw0h + mov_reg_to_vec IV, tw1l, tw1h + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, IV.16b + bl sm4_encrypt_block4x + eor BLK0.16b, BLK0.16b, BLK4.16b + eor BLK1.16b, BLK1.16b, IV.16b + st1 {BLK0.16b, BLK1.16b}, [x0], #32 + subs w4, w4, #2 + b.le .Lxtsblksout + +.Lxts1x: + ld1 {BLK0.16b}, [x1], #16 + mov_reg_to_vec IV, tw0l, tw0h + eor BLK0.16b, BLK0.16b, IV.16b + bl sm4_encrypt_block1x + eor BLK0.16b, BLK0.16b, IV.16b + st1 {BLK0.16b}, [x0], #16 +.Lxtsblksout: + cmp w6, 0 + /* if encrypt some blocks with a partial block */ + next_tweak_vec IV, IV + b.eq .Lxtsout +.Lxtstail: + next_tweak_vec TMP7, IV + cmp x26, 1 + b.eq 1f + /* The last two tweaks IV, TMP7 need to be swapped for decryption */ + mov TMP8.16b, IV.16b + mov IV.16b, TMP7.16b + mov TMP7.16b, TMP8.16b + 1: + ld1 {BLK0.16b}, [x1], #16 + eor BLK0.16b, BLK0.16b, IV.16b + bl sm4_encrypt_block1x + eor BLK0.16b, BLK0.16b, IV.16b + st1 {BLK0.16b}, [x0], #16 + sub x7, x0, 16 + 10: + subs x6, x6, 1 + ldrb tmpw0, [x7, x6] + strb tmpw0, [x0, x6] + ldrb tmpw0, [x1, x6] + strb tmpw0, [x7, x6] + b.gt 10b + ld1 {BLK0.16b}, [x7] + eor BLK0.16b, BLK0.16b, TMP7.16b + bl sm4_encrypt_block1x + eor BLK0.16b, BLK0.16b, TMP7.16b + st1 {BLK0.16b}, [x7] + +.Lxtsout: + /* load round key2 for last tweak */ + sub x3, x3, #128 + ld1 {RK0.4s, RK1.4s, RK2.4s, RK3.4s}, [x3], #64 + ld1 {RK4.4s, RK5.4s, RK6.4s, RK7.4s}, [x3], #64 + /* decrypt last tweak for next update */ + decrypt_block IV + st1 {IV.16b}, [x5] + ldp x29, x30, [sp], #16 + ret +END_FUNC xts_do_cipher + +/* + * void ce_sm4_xts_encrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], uint8_t const rk2[], + * size_t len, uint8_t iv[]) + * x0: output + * x1: input + * x2: round key1 + * x3: round key2 + * w4: len + * x5: iv + */ +FUNC ce_sm4_xts_encrypt , : + frame_push + mov x26, 1 + bl xts_do_cipher + frame_pop + ret + +END_FUNC ce_sm4_xts_encrypt + +/* + * void ce_sm4_xts_decrypt(uint8_t out[], uint8_t const in[], + * uint8_t const rk1[], uint8_t const rk2[], + * size_t len, uint8_t iv[]) + * x0: output + * x1: input + * x2: round key1 + * x3: round key2 + * w4: len + * x5: iv + */ +FUNC ce_sm4_xts_decrypt , : + frame_push + mov x26, 0 + bl xts_do_cipher + frame_pop + ret +END_FUNC ce_sm4_xts_decrypt diff --git a/optee_os/core/arch/arm/crypto/sm4_armv8a_neon.c b/optee_os/core/arch/arm/crypto/sm4_armv8a_neon.c new file mode 100644 index 0000000..6d1f759 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm4_armv8a_neon.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) Hisilicon Technologies Co., Ltd. 2023. All rights reserved. + * + * SM4 optimization for ARMv8 by NEON and AES HW instruction, which is an + * optional Cryptographic Extension for ARMv8-A. + */ +#include +#include + +#include "sm4_armv8a_neon.h" + +void crypto_accel_sm4_setkey_enc(uint32_t sk[32], const uint8_t key[16]) +{ + uint32_t vfp_state = 0; + + assert(sk && key); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_setkey_enc(sk, key); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_setkey_dec(uint32_t sk[32], const uint8_t key[16]) +{ + uint32_t vfp_state = 0; + + assert(sk && key); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_setkey_dec(sk, key); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_ecb_enc(void *out, const void *in, const void *key, + unsigned int len) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_ecb_encrypt(out, in, key, len); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_cbc_enc(void *out, const void *in, const void *key, + unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_cbc_encrypt(out, in, key, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_cbc_dec(void *out, const void *in, const void *key, + unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_cbc_decrypt(out, in, key, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_ctr_enc(void *out, const void *in, const void *key, + unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key && !(len % 16)); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_ctr_encrypt(out, in, key, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_xts_enc(void *out, const void *in, const void *key1, + const void *key2, unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key1 && key2 && (len >= 16)); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_xts_encrypt(out, in, key1, key2, len, iv); + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_accel_sm4_xts_dec(void *out, const void *in, const void *key1, + const void *key2, unsigned int len, void *iv) +{ + uint32_t vfp_state = 0; + + assert(out && in && key1 && key2 && (len >= 16)); + + vfp_state = thread_kernel_enable_vfp(); + neon_sm4_xts_decrypt(out, in, key1, key2, len, iv); + thread_kernel_disable_vfp(vfp_state); +} diff --git a/optee_os/core/arch/arm/crypto/sm4_armv8a_neon.h b/optee_os/core/arch/arm/crypto/sm4_armv8a_neon.h new file mode 100644 index 0000000..addf9d3 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sm4_armv8a_neon.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) Hisilicon Technologies Co., Ltd. 2023. All rights reserved. + */ + +#ifndef __SM4_ARMV8_NEON_H +#define __SM4_ARMV8_NEON_H + +#include + +/* Prototypes for assembly functions */ +void neon_sm4_setkey_enc(uint32_t sk[32], uint8_t const key[16]); +void neon_sm4_setkey_dec(uint32_t sk[32], uint8_t const key[16]); +void neon_sm4_ecb_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len); +void neon_sm4_cbc_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len, uint8_t iv[]); +void neon_sm4_cbc_decrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len, uint8_t iv[]); +void neon_sm4_ctr_encrypt(uint8_t out[], uint8_t const in[], uint8_t const rk[], + size_t len, uint8_t iv[]); +void neon_sm4_xts_encrypt(uint8_t out[], uint8_t const in[], + uint8_t const rk1[], uint8_t const rk2[], size_t len, + uint8_t iv[]); +void neon_sm4_xts_decrypt(uint8_t out[], uint8_t const in[], + uint8_t const rk1[], uint8_t const rk2[], size_t len, + uint8_t iv[]); + +#endif /*__SM4_ARMV8_NEON_H*/ diff --git a/optee_os/core/arch/arm/crypto/sub.mk b/optee_os/core/arch/arm/crypto/sub.mk new file mode 100644 index 0000000..9179916 --- /dev/null +++ b/optee_os/core/arch/arm/crypto/sub.mk @@ -0,0 +1,47 @@ +ifeq ($(CFG_CRYPTO_WITH_CE),y) +srcs-$(CFG_ARM64_core) += ghash-ce-core_a64.S +srcs-$(CFG_ARM32_core) += ghash-ce-core_a32.S +srcs-y += aes-gcm-ce.c +endif + +ifeq ($(CFG_CRYPTO_AES_ARM_CE),y) +srcs-y += aes_armv8a_ce.c +srcs-$(CFG_ARM64_core) += aes_modes_armv8a_ce_a64.S +aflags-aes_modes_armv8a_ce_a64.S-y += -DINTERLEAVE=4 +srcs-$(CFG_ARM32_core) += aes_modes_armv8a_ce_a32.S +endif + +ifeq ($(CFG_CRYPTO_SHA1_ARM_CE),y) +srcs-y += sha1_armv8a_ce.c +srcs-$(CFG_ARM64_core) += sha1_armv8a_ce_a64.S +srcs-$(CFG_ARM32_core) += sha1_armv8a_ce_a32.S +endif + +ifeq ($(CFG_CRYPTO_SHA256_ARM_CE),y) +srcs-y += sha256_armv8a_ce.c +srcs-$(CFG_ARM64_core) += sha256_armv8a_ce_a64.S +srcs-$(CFG_ARM32_core) += sha256_armv8a_ce_a32.S +endif + +ifeq ($(CFG_CRYPTO_SHA512_ARM_CE),y) +srcs-y += sha512_armv8a_ce.c +srcs-$(CFG_ARM64_core) += sha512_armv8a_ce_a64.S +endif + +ifeq ($(CFG_CRYPTO_SHA3_ARM_CE),y) +srcs-y += sha3_armv8a_ce.c +srcs-$(CFG_ARM64_core) += sha3_armv8a_ce_a64.S +endif + +ifeq ($(CFG_CRYPTO_SM3_ARM_CE),y) +srcs-y += sm3_armv8a_ce.c +srcs-$(CFG_ARM64_core) += sm3_armv8a_ce_a64.S +endif + +ifeq ($(CFG_CRYPTO_SM4_ARM_CE),y) +srcs-$(CFG_ARM64_core) += sm4_armv8a_ce.c +srcs-$(CFG_ARM64_core) += sm4_armv8a_ce_a64.S +else ifeq ($(CFG_CRYPTO_SM4_ARM_AESE),y) +srcs-$(CFG_ARM64_core) += sm4_armv8a_neon.c +srcs-$(CFG_ARM64_core) += sm4_armv8a_aese_a64.S +endif diff --git a/optee_os/core/arch/arm/dts/at91-sama5d27_som1.dtsi b/optee_os/core/arch/arm/dts/at91-sama5d27_som1.dtsi new file mode 100644 index 0000000..efd725a --- /dev/null +++ b/optee_os/core/arch/arm/dts/at91-sama5d27_som1.dtsi @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * at91-sama5d27_som1.dtsi - Device Tree file for SAMA5D27 SoM1 board + * + * Copyright (c) 2017, Microchip Technology Inc. + * 2017 Cristian Birsan + * 2017 Claudiu Beznea + */ +#include "sama5d2.dtsi" +#include "sama5d2-pinfunc.h" + +/ { + model = "Atmel SAMA5D27 SoM1"; + compatible = "atmel,sama5d27-som1", "atmel,sama5d27", "atmel,sama5d2", "atmel,sama5"; + + aliases { + i2c0 = &i2c0; + }; + + clocks { + slow_xtal { + clock-frequency = <32768>; + }; + + main_xtal { + clock-frequency = <24000000>; + }; + }; + + ahb { + sdmmc0: sdio-host@a0000000 { + microchip,sdcal-inverted; + }; + + apb { + qspi1: spi@f0024000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_qspi1_default>; + + flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <80000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; + m25p,fast-read; + + at91bootstrap@0 { + label = "at91bootstrap"; + reg = <0x00000000 0x00040000>; + }; + + bootloader@40000 { + label = "bootloader"; + reg = <0x00040000 0x000c0000>; + }; + + bootloaderenvred@100000 { + label = "bootloader env redundant"; + reg = <0x00100000 0x00040000>; + }; + + bootloaderenv@140000 { + label = "bootloader env"; + reg = <0x00140000 0x00040000>; + }; + + dtb@180000 { + label = "device tree"; + reg = <0x00180000 0x00080000>; + }; + + kernel@200000 { + label = "kernel"; + reg = <0x00200000 0x00600000>; + }; + }; + }; + + macb0: ethernet@f8008000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_macb0_default>; + phy-mode = "rmii"; + + ethernet-phy@7 { + reg = <0x7>; + interrupt-parent = <&pioA>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_macb0_phy_irq>; + }; + }; + + i2c0: i2c@f8028000 { + dmas = <0>, <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c0_default>; + status = "okay"; + + at24@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + pagesize = <8>; + }; + }; + + pinctrl@fc039000 { + pinctrl_i2c0_default: i2c0_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_qspi1_default: qspi1_default { + sck_cs { + pinmux = , + ; + bias-disable; + }; + + data { + pinmux = , + , + , + ; + bias-pull-up; + }; + }; + + pinctrl_macb0_default: macb0_default { + pinmux = , + , + , + , + , + , + , + , + , + ; + bias-disable; + }; + + pinctrl_macb0_phy_irq: macb0_phy_irq { + pinmux = ; + bias-disable; + }; + }; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/at91-sama5d27_som1_ek.dts b/optee_os/core/arch/arm/dts/at91-sama5d27_som1_ek.dts new file mode 100644 index 0000000..461934b --- /dev/null +++ b/optee_os/core/arch/arm/dts/at91-sama5d27_som1_ek.dts @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * at91-sama5d27_som1_ek.dts - Device Tree file for SAMA5D27-SOM1-EK board + * + * Copyright (c) 2017, Microchip Technology Inc. + * 2016 Nicolas Ferre + * 2017 Cristian Birsan + * 2017 Claudiu Beznea + */ +/dts-v1/; +#include "at91-sama5d27_som1.dtsi" +#include +#include + +/ { + model = "Atmel SAMA5D27 SOM1 EK"; + compatible = "atmel,sama5d27-som1-ek", "atmel,sama5d27-som1", "atmel,sama5d27", "atmel,sama5d2", "atmel,sama5"; + + aliases { + serial0 = &uart1; /* DBGU */ + serial1 = &uart4; /* mikro BUS 1 */ + serial2 = &uart2; /* mikro BUS 2 */ + i2c1 = &i2c1; + i2c2 = &i2c3; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + ahb { + usb0: gadget@300000 { + atmel,vbus-gpio = <&pioA PIN_PD20 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usba_vbus>; + status = "okay"; + }; + + usb1: ohci@400000 { + num-ports = <3>; + atmel,vbus-gpio = <0 /* &pioA PIN_PD20 GPIO_ACTIVE_HIGH */ + &pioA PIN_PA27 GPIO_ACTIVE_HIGH + 0 + >; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb_default>; + status = "okay"; + }; + + usb2: ehci@500000 { + status = "okay"; + }; + + sdmmc0: sdio-host@a0000000 { + bus-width = <8>; + mmc-ddr-3_3v; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdmmc0_default>; + status = "okay"; + }; + + sdmmc1: sdio-host@b0000000 { + bus-width = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdmmc1_default>; + status = "okay"; + }; + + apb { + isc: isc@f0008000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_isc_base &pinctrl_isc_data_8bit &pinctrl_isc_data_9_10 &pinctrl_isc_data_11_12>; + status = "okay"; + }; + + qspi1: spi@f0024000 { + status = "okay"; + }; + + spi0: spi@f8000000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spi0_default>; + status = "okay"; + }; + + macb0: ethernet@f8008000 { + status = "okay"; + }; + + tcb0: timer@f800c000 { + timer0: timer@0 { + compatible = "atmel,tcb-timer"; + reg = <0>; + }; + + timer1: timer@1 { + compatible = "atmel,tcb-timer"; + reg = <1>; + }; + }; + + uart1: serial@f8020000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; + atmel,use-dma-rx; + atmel,use-dma-tx; + status = "okay"; + }; + + uart2: serial@f8024000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_mikrobus2_uart>; + atmel,use-dma-rx; + atmel,use-dma-tx; + status = "okay"; + }; + + pwm0: pwm@f802c000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_mikrobus1_pwm &pinctrl_mikrobus2_pwm>; + status = "disabled"; /* Conflict with leds. */ + }; + + flx1: flexcom@f8038000 { + atmel,flexcom-mode = ; + status = "okay"; + + i2c3: i2c@600 { + dmas = <0>, <0>; + i2c-analog-filter; + i2c-digital-filter; + i2c-digital-filter-width-ns = <35>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_mikrobus_i2c>; + status = "okay"; + }; + }; + + shdwc@f8048010 { + debounce-delay-us = <976>; + atmel,wakeup-rtc-timer; + + input@0 { + reg = <0>; + }; + }; + + uart3: serial@fc008000 { + atmel,use-dma-rx; + atmel,use-dma-tx; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart3_default>; + status = "disabled"; /* Conflict with isc. */ + }; + + uart4: serial@fc00c000 { + atmel,use-dma-rx; + atmel,use-dma-tx; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_mikrobus1_uart>; + status = "okay"; + }; + + flx3: flexcom@fc014000 { + atmel,flexcom-mode = ; + status = "disabled"; + + uart8: serial@200 { + dmas = <0>, <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flx3_default>; + status = "disabled"; /* Conflict with isc. */ + }; + + spi5: spi@400 { + dmas = <0>, <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flx3_default>; + status = "disabled"; /* Conflict with isc. */ + }; + }; + + flx4: flexcom@fc018000 { + atmel,flexcom-mode = ; + status = "okay"; + + uart9: serial@200 { + dmas = <0>, <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flx4_default>; + status = "disabled"; /* Conflict with spi6 and i2c6. */ + }; + + spi6: spi@400 { + dmas = <0>, <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_mikrobus_spi &pinctrl_mikrobus1_spi_cs &pinctrl_mikrobus2_spi_cs>; + status = "okay"; /* Conflict with uart5 and i2c6. */ + }; + + i2c6: i2c@600 { + dmas = <0>, <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flx4_default>; + status = "disabled"; /* Conflict with uart5 and spi6. */ + }; + }; + + i2c1: i2c@fc028000 { + dmas = <0>, <0>; + i2c-analog-filter; + i2c-digital-filter; + i2c-digital-filter-width-ns = <35>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c1_default>; + status = "okay"; + }; + + adc: adc@fc030000 { + vddana-supply = <&vddana>; + vref-supply = <&advref>; + + status = "disabled"; + }; + + pinctrl@fc039000 { + + pinctrl_can1_default: can1_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_flx3_default: flx3_default { + pinmux = , + , + , + , + ; + bias-disable; + }; + + pinctrl_i2c1_default: i2c1_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_isc_base: isc_base { + pinmux = , + , + , + ; + bias-disable; + }; + + pinctrl_isc_data_8bit: isc_data_8bit { + pinmux = , + , + , + , + , + , + , + ; + bias-disable; + }; + + pinctrl_isc_data_9_10: isc_data_9_10 { + pinmux = , + ; + bias-disable; + }; + + pinctrl_isc_data_11_12: isc_data_11_12 { + pinmux = , + ; + bias-disable; + }; + + pinctrl_key_gpio_default: key_gpio_default { + pinmux = ; + bias-pull-up; + }; + + pinctrl_led_gpio_default: led_gpio_default { + pinmux = , + , + ; + bias-pull-up; + }; + + pinctrl_sdmmc0_default: sdmmc0_default { + cmd_data { + pinmux = , + , + , + , + , + , + , + , + ; + bias-disable; + }; + + ck_cd_vddsel { + pinmux = , + , + ; + bias-disable; + }; + }; + + pinctrl_sdmmc1_default: sdmmc1_default { + cmd_data { + pinmux = , + , + , + , + ; + bias-disable; + }; + + conf-ck_cd { + pinmux = , + ; + bias-disable; + }; + }; + + pinctrl_spi0_default: spi0_default { + pinmux = , + , + , + ; + bias-disable; + }; + + pinctrl_uart1_default: uart1_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_uart3_default: uart3_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_usb_default: usb_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_usba_vbus: usba_vbus { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus1_an: mikrobus1_an { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus2_an: mikrobus2_an { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus1_rst: mikrobus1_rst { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus2_rst: mikrobus2_rst { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus1_spi_cs: mikrobus1_spi_cs { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus2_spi_cs: mikrobus2_spi_cs { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus_spi: mikrobus_spi { + pinmux = , + , + ; + bias-disable; + }; + + pinctrl_mikrobus1_pwm: mikrobus1_pwm { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus2_pwm: mikrobus2_pwm { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus1_int: mikrobus1_int { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus2_int: mikrobus2_int { + pinmux = ; + bias-disable; + }; + + pinctrl_mikrobus1_uart: mikrobus1_uart { + pinmux = , + ; + bias-disable; + }; + + pinctrl_mikrobus2_uart: mikrobus2_uart { + pinmux = , + ; + bias-disable; + }; + + pinctrl_mikrobus_i2c: mikrobus1_i2c { + pinmux = , + ; + bias-disable; + }; + + pinctrl_flx4_default: flx4_uart_default { + pinmux = , + , + , + , + ; + bias-disable; + }; + }; + + can1: can@fc050000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can1_default>; + status = "okay"; + }; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_key_gpio_default>; + + pb4 { + label = "USER"; + gpios = <&pioA PIN_PA29 GPIO_ACTIVE_LOW>; + /* linux,code = ; BSD license issue */ + wakeup-source; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_led_gpio_default>; + status = "okay"; /* Conflict with pwm0. */ + + red { + label = "red"; + gpios = <&pioA PIN_PA10 GPIO_ACTIVE_HIGH>; + }; + + green { + label = "green"; + gpios = <&pioA PIN_PB1 GPIO_ACTIVE_HIGH>; + }; + + blue { + label = "blue"; + gpios = <&pioA PIN_PA31 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + }; + + vddin_3v3: fixed-regulator-vddin_3v3 { + compatible = "regulator-fixed"; + + regulator-name = "VDDIN_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + status = "okay"; + }; + + vddana: fixed-regulator-vddana { + compatible = "regulator-fixed"; + + regulator-name = "VDDANA"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vddin_3v3>; + status = "okay"; + }; + + advref: fixed-regulator-advref { + compatible = "regulator-fixed"; + + regulator-name = "advref"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vddana>; + status = "okay"; + }; +}; diff --git a/optee_os/core/arch/arm/dts/at91-sama5d27_wlsom1.dtsi b/optee_os/core/arch/arm/dts/at91-sama5d27_wlsom1.dtsi new file mode 100644 index 0000000..f177c46 --- /dev/null +++ b/optee_os/core/arch/arm/dts/at91-sama5d27_wlsom1.dtsi @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * at91-sama5d27_wlsom1.dtsi - Device Tree file for SAMA5D27 WLSOM1 + * + * Copyright (C) 2019 Microchip Technology Inc. and its subsidiaries + * + * Author: Nicolas Ferre + * Author: Eugen Hristev + */ +#include "sama5d2.dtsi" +#include "sama5d2-pinfunc.h" +#include +#include + +/ { + model = "Microchip SAMA5D27 WLSOM1"; + compatible = "microchip,sama5d27-wlsom1", "atmel,sama5d27", "atmel,sama5d2", "atmel,sama5"; + + aliases { + i2c0 = &i2c0; + }; + + clocks { + slow_xtal { + clock-frequency = <32768>; + }; + + main_xtal { + clock-frequency = <24000000>; + }; + }; +}; + +&flx1 { + atmel,flexcom-mode = ; + + uart6: serial@200 { + pinctrl-0 = <&pinctrl_flx1_default>; + pinctrl-names = "default"; + }; +}; + +&i2c0 { + pinctrl-0 = <&pinctrl_i2c0_default>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + pinctrl-names = "default", "gpio"; + sda-gpios = <&pioA PIN_PD21 GPIO_ACTIVE_HIGH>; + scl-gpios = <&pioA PIN_PD22 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + status = "okay"; +}; + +&i2c1 { + dmas = <0>, <0>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + sda-gpios = <&pioA PIN_PD19 GPIO_ACTIVE_HIGH>; + scl-gpios = <&pioA PIN_PD20 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + status = "disable"; + secure-status = "okay"; + + mcp16502@5b { + compatible = "microchip,mcp16502"; + reg = <0x5b>; + status = "okay"; + lpm-gpios = <&pioBU 0 GPIO_ACTIVE_LOW>; + + regulators { + vdd_3v3: VDD_IO { + regulator-name = "VDD_IO"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3700000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + vddio_ddr: VDD_DDR { + regulator-name = "VDD_DDR"; + regulator-min-microvolt = <600000>; + regulator-max-microvolt = <1850000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1200000>; + regulator-changeable-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1200000>; + regulator-changeable-in-suspend; + regulator-mode = <4>; + }; + }; + + vdd_core: VDD_CORE { + regulator-name = "VDD_CORE"; + regulator-min-microvolt = <600000>; + regulator-max-microvolt = <1850000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + vdd_ddr: VDD_OTHER { + regulator-name = "VDD_OTHER"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + regulator-changeable-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + regulator-changeable-in-suspend; + regulator-mode = <4>; + }; + }; + + LDO1 { + regulator-name = "LDO1"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3700000>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + }; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + LDO2 { + regulator-name = "LDO2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3700000>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + }; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&macb0 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_macb0_default>; + phy-mode = "rmii"; + + ethernet-phy@0 { + reg = <0x0>; + interrupt-parent = <&pioA>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_macb0_phy_irq>; + }; +}; + +&pmc { + atmel,osc-bypass; +}; + +&qspi1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_qspi1_default>; + status = "disabled"; + + qspi1_flash: spi_flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <80000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + m25p,fast-read; + status = "disabled"; + + at91bootstrap@0 { + label = "at91bootstrap"; + reg = <0x0 0x40000>; + }; + + bootloader@40000 { + label = "bootloader"; + reg = <0x40000 0xc0000>; + }; + + bootloaderenvred@100000 { + label = "bootloader env redundant"; + reg = <0x100000 0x40000>; + }; + + bootloaderenv@140000 { + label = "bootloader env"; + reg = <0x140000 0x40000>; + }; + + dtb@180000 { + label = "device tree"; + reg = <0x180000 0x80000>; + }; + + kernel@200000 { + label = "kernel"; + reg = <0x200000 0x600000>; + }; + }; +}; + +&pioA { + pinctrl_flx1_default: flx1_usart_default { + pinmux = , + , + , + ; + bias-disable; + }; + + pinctrl_i2c0_default: i2c0_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c0_gpio: i2c0_gpio { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c1_default: i2c1_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c1_gpio: i2c1_gpio { + pinmux = , + ; + bias-disable; + }; + + pinctrl_macb0_default: macb0_default { + pinmux = , + , + , + , + , + , + , + , + , + ; + bias-disable; + }; + + pinctrl_macb0_phy_irq: macb0_phy_irq { + pinmux = ; + bias-disable; + }; + + pinctrl_qspi1_default: qspi1_default { + pinmux = , + , + , + , + , + ; + bias-pull-up; + }; +}; + diff --git a/optee_os/core/arch/arm/dts/at91-sama5d27_wlsom1_ek.dts b/optee_os/core/arch/arm/dts/at91-sama5d27_wlsom1_ek.dts new file mode 100644 index 0000000..c6c3fc8 --- /dev/null +++ b/optee_os/core/arch/arm/dts/at91-sama5d27_wlsom1_ek.dts @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * at91-sama5d27_wlsom1_ek.dts - Device Tree file for SAMA5D27 WLSOM1 EK + * + * Copyright (C) 2019 Microchip Technology Inc. and its subsidiaries + * + * Author: Nicolas Ferre + */ +/dts-v1/; +#include "at91-sama5d27_wlsom1.dtsi" + +/ { + model = "Microchip SAMA5D27 WLSOM1 EK"; + compatible = "microchip,sama5d27-wlsom1-ek", "microchip,sama5d27-wlsom1", "atmel,sama5d27", "atmel,sama5d2", "atmel,sama5"; + + aliases { + serial0 = &uart0; /* DBGU */ + serial1 = &uart6; /* BT */ + serial2 = &uart5; /* mikro BUS 2 */ + serial3 = &uart3; /* mikro BUS 1 */ + i2c1 = &i2c1; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_led_gpio_default>; + status = "okay"; + + red { + label = "red"; + gpios = <&pioA PIN_PA6 GPIO_ACTIVE_HIGH>; + }; + + green { + label = "green"; + gpios = <&pioA PIN_PA7 GPIO_ACTIVE_HIGH>; + }; + + blue { + label = "blue"; + gpios = <&pioA PIN_PA8 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + }; +}; + +&adc { + vddana-supply = <&vdd_3v3>; + vref-supply = <&vdd_3v3>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_adc_default>; + status = "okay"; +}; + +&flx0 { + atmel,flexcom-mode = ; + status = "okay"; + + uart5: serial@200 { + pinctrl-0 = <&pinctrl_flx0_default>; + pinctrl-names = "default"; + atmel,use-dma-rx; + atmel,use-dma-tx; + status = "okay"; + }; +}; + +&flx1 { + status = "okay"; + + uart6: serial@200 { + atmel,use-dma-rx; + atmel,use-dma-tx; + status = "okay"; + }; +}; + +&macb0 { + status = "okay"; +}; + +&pioA { + /* + * There is no real pinmux for ADC, if the pin + * is not requested by another peripheral then + * the muxing is done when channel is enabled. + * Requesting pins for ADC is GPIO is + * encouraged to prevent conflicts and to + * disable bias in order to be in the same + * state when the pin is not muxed to the adc. + */ + pinctrl_adc_default: adc_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_flx0_default: flx0_usart_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_key_gpio_default: key_gpio_default { + pinmux = ; + bias-pull-up; + }; + + pinctrl_led_gpio_default: led_gpio_default { + pinmux = , + , + ; + bias-pull-down; + }; + + pinctrl_sdmmc0_default: sdmmc0_default { + cmd_data { + pinmux = , + , + , + , + ; + bias-disable; + }; + + ck_cd_vddsel { + pinmux = , + , + , + ; + bias-disable; + }; + }; + + pinctrl_uart0_default: uart0_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_uart3_default: uart3_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_pwm0_default: pwm0_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_usb_default: usb_default { + pinmux = ; + bias-disable; + }; + + pinctrl_usba_vbus: usba_vbus { + pinmux = ; + bias-disable; + }; +}; + +&pwm0 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm0_default>; + status = "okay"; +}; + +&qspi1 { + status = "okay"; + + qspi1_flash: spi_flash@0 { + status = "okay"; + }; +}; + +&sdmmc0 { + bus-width = <4>; + mmc-ddr-3_3v; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdmmc0_default>; + status = "okay"; +}; + +&shutdown_controller { + atmel,shdwc-debouncer = <976>; + atmel,wakeup-rtc-timer; + + input@0 { + reg = <0>; + }; +}; + +&tcb0 { + timer0: timer@0 { + compatible = "atmel,tcb-timer"; + reg = <0>; + }; + + timer1: timer@1 { + compatible = "atmel,tcb-timer"; + reg = <1>; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; + atmel,use-dma-rx; + atmel,use-dma-tx; + status = "okay"; +}; + +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart3_default>; + atmel,use-dma-rx; + atmel,use-dma-tx; + status = "okay"; +}; + +&usb0 { + atmel,vbus-gpio = <&pioA PIN_PA16 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usba_vbus>; + status = "okay"; +}; + +&usb1 { + num-ports = <3>; + atmel,vbus-gpio = <0 + &pioA PIN_PA10 GPIO_ACTIVE_HIGH + 0 + >; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb_default>; + status = "okay"; +}; + +&usb2 { + phy_type = "hsic"; + status = "okay"; +}; + diff --git a/optee_os/core/arch/arm/dts/at91-sama5d2_xplained.dts b/optee_os/core/arch/arm/dts/at91-sama5d2_xplained.dts new file mode 100644 index 0000000..1909b7f --- /dev/null +++ b/optee_os/core/arch/arm/dts/at91-sama5d2_xplained.dts @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * at91-sama5d2_xplained.dts - Device Tree file for SAMA5D2 Xplained board + * + * Copyright (C) 2015 Atmel, + * 2015 Nicolas Ferre + */ +/dts-v1/; +#include "sama5d2.dtsi" +#include "sama5d2-pinfunc.h" +#include +#include +#include + +/ { + model = "Atmel SAMA5D2 Xplained"; + compatible = "atmel,sama5d2-xplained", "atmel,sama5d2", "atmel,sama5"; + + aliases { + serial0 = &uart1; /* DBGU */ + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; /* XPRO EXT2 */ + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + clocks { + slow_xtal { + clock-frequency = <32768>; + }; + + main_xtal { + clock-frequency = <12000000>; + }; + }; + + ahb { + usb0: gadget@300000 { + atmel,vbus-gpio = <&pioA PIN_PA31 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usba_vbus>; + status = "okay"; + }; + + usb1: ohci@400000 { + num-ports = <3>; + atmel,vbus-gpio = <0 /* &pioA PIN_PB9 GPIO_ACTIVE_HIGH */ + &pioA PIN_PB10 GPIO_ACTIVE_HIGH + 0 + >; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb_default>; + status = "okay"; + }; + + usb2: ehci@500000 { + status = "okay"; + }; + + sdmmc0: sdio-host@a0000000 { + bus-width = <8>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdmmc0_default>; + non-removable; + mmc-ddr-1_8v; + status = "okay"; + }; + + sdmmc1: sdio-host@b0000000 { + bus-width = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdmmc1_default>; + status = "okay"; /* conflict with qspi0 */ + vqmmc-supply = <&vdd_3v3_reg>; + vmmc-supply = <&vdd_3v3_reg>; + }; + + apb { + qspi0: spi@f0020000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_qspi0_default>; + status = "disabled"; /* conflict with sdmmc1 */ + + flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <80000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; + m25p,fast-read; + + at91bootstrap@0 { + label = "at91bootstrap"; + reg = <0x00000000 0x00040000>; + }; + + bootloader@40000 { + label = "bootloader"; + reg = <0x00040000 0x000c0000>; + }; + + bootloaderenvred@100000 { + label = "bootloader env redundant"; + reg = <0x00100000 0x00040000>; + }; + + bootloaderenv@140000 { + label = "bootloader env"; + reg = <0x00140000 0x00040000>; + }; + + dtb@180000 { + label = "device tree"; + reg = <0x00180000 0x00080000>; + }; + + kernel@200000 { + label = "kernel"; + reg = <0x00200000 0x00600000>; + }; + + misc@800000 { + label = "misc"; + reg = <0x00800000 0x00000000>; + }; + }; + }; + + spi0: spi@f8000000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spi0_default>; + status = "okay"; + + m25p80@0 { + compatible = "atmel,at25df321a"; + reg = <0>; + spi-max-frequency = <50000000>; + }; + }; + + macb0: ethernet@f8008000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_macb0_default &pinctrl_macb0_phy_irq>; + phy-mode = "rmii"; + status = "okay"; + + ethernet-phy@1 { + reg = <0x1>; + interrupt-parent = <&pioA>; + interrupts = ; + }; + }; + + tcb0: timer@f800c000 { + timer0: timer@0 { + compatible = "atmel,tcb-timer"; + reg = <0>; + }; + + timer1: timer@1 { + compatible = "atmel,tcb-timer"; + reg = <1>; + }; + }; + + uart1: serial@f8020000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; + atmel,use-dma-rx; + atmel,use-dma-tx; + status = "okay"; + }; + + i2c0: i2c@f8028000 { + dmas = <0>, <0>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c0_default>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + sda-gpios = <&pioA PIN_PD21 GPIO_ACTIVE_HIGH>; + scl-gpios = <&pioA PIN_PD22 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + i2c-sda-hold-time-ns = <350>; + status = "okay"; + + pmic@5b { + compatible = "active-semi,act8945a"; + reg = <0x5b>; + active-semi,vsel-high; + status = "okay"; + + regulators { + vdd_1v35_reg: REG_DCDC1 { + regulator-name = "VDD_1V35"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-allowed-modes = , + ; + regulator-initial-mode = ; + regulator-always-on; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-min-microvolt=<1400000>; + regulator-suspend-max-microvolt=<1400000>; + regulator-changeable-in-suspend; + regulator-mode=; + }; + }; + + vdd_1v2_reg: REG_DCDC2 { + regulator-name = "VDD_1V2"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1300000>; + regulator-allowed-modes = , + ; + regulator-initial-mode = ; + regulator-always-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_3v3_reg: REG_DCDC3 { + regulator-name = "VDD_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-allowed-modes = , + ; + regulator-initial-mode = ; + regulator-always-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_fuse_reg: REG_LDO1 { + regulator-name = "VDD_FUSE"; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + regulator-allowed-modes = , + ; + regulator-initial-mode = ; + regulator-always-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_3v3_lp_reg: REG_LDO2 { + regulator-name = "VDD_3V3_LP"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-allowed-modes = , + ; + regulator-initial-mode = ; + regulator-always-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_led_reg: REG_LDO3 { + regulator-name = "VDD_LED"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-allowed-modes = , + ; + regulator-initial-mode = ; + regulator-always-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_sdhc_1v8_reg: REG_LDO4 { + regulator-name = "VDD_SDHC_1V8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-allowed-modes = , + ; + regulator-initial-mode = ; + regulator-always-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + charger { + compatible = "active-semi,act8945a-charger"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_charger_chglev &pinctrl_charger_lbo &pinctrl_charger_irq>; + interrupt-parent = <&pioA>; + interrupts = ; + + active-semi,chglev-gpios = <&pioA PIN_PA12 GPIO_ACTIVE_HIGH>; + active-semi,lbo-gpios = <&pioA PIN_PC8 GPIO_ACTIVE_LOW>; + active-semi,input-voltage-threshold-microvolt = <6600>; + active-semi,precondition-timeout = <40>; + active-semi,total-timeout = <3>; + status = "okay"; + }; + }; + }; + + pwm0: pwm@f802c000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm0_pwm2_default>; + status = "disabled"; /* conflict with leds */ + }; + + flx0: flexcom@f8034000 { + atmel,flexcom-mode = ; + status = "disabled"; /* conflict with ISC_D2 & ISC_D3 data pins */ + + uart5: serial@200 { + dmas = <0>, <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flx0_default>; + status = "okay"; + }; + + i2c2: i2c@600 { + dmas = <0>, <0>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_flx0_default>; + pinctrl-1 = <&pinctrl_i2c2_gpio>; + sda-gpios = <&pioA PIN_PB28 GPIO_ACTIVE_HIGH>; + scl-gpios = <&pioA PIN_PB29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + i2c-sda-hold-time-ns = <350>; + i2c-analog-filter; + i2c-digital-filter; + i2c-digital-filter-width-ns = <35>; + status = "disabled"; /* conflict with ISC_D2 & ISC_D3 data pins */ + }; + }; + + shdwc@f8048010 { + debounce-delay-us = <976>; + atmel,wakeup-rtc-timer; + + input@0 { + reg = <0>; + }; + }; + + i2s0: i2s@f8050000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2s0_default>; + status = "disabled"; /* conflict with can0 */ + }; + + can0: can@f8054000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can0_default>; + status = "okay"; + }; + + uart3: serial@fc008000 { + atmel,use-dma-rx; + atmel,use-dma-tx; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart3_default>; + status = "okay"; + }; + + flx4: flexcom@fc018000 { + atmel,flexcom-mode = ; + status = "okay"; + + i2c6: i2c@600 { + dmas = <0>, <0>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_flx4_default>; + pinctrl-1 = <&pinctrl_flx4_gpio>; + sda-gpios = <&pioA PIN_PD12 GPIO_ACTIVE_HIGH>; + scl-gpios = <&pioA PIN_PD13 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + i2c-analog-filter; + i2c-digital-filter; + i2c-digital-filter-width-ns = <35>; + status = "okay"; + }; + }; + + i2c1: i2c@fc028000 { + dmas = <0>, <0>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + i2c-analog-filter; + i2c-digital-filter; + i2c-digital-filter-width-ns = <35>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + sda-gpios = <&pioA PIN_PD4 GPIO_ACTIVE_HIGH>; + scl-gpios = <&pioA PIN_PD5 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + status = "okay"; + + at24@54 { + compatible = "atmel,24c02"; + reg = <0x54>; + pagesize = <16>; + }; + }; + + adc: adc@fc030000 { + vddana-supply = <&vdd_3v3_lp_reg>; + vref-supply = <&vdd_3v3_lp_reg>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_adc_default &pinctrl_adtrg_default>; + status = "okay"; + }; + + pinctrl@fc039000 { + /* + * There is no real pinmux for ADC, if the pin + * is not requested by another peripheral then + * the muxing is done when channel is enabled. + * Requesting pins for ADC is GPIO is + * encouraged to prevent conflicts and to + * disable bias in order to be in the same + * state when the pin is not muxed to the adc. + */ + pinctrl_adc_default: adc_default { + pinmux = ; + bias-disable; + }; + + pinctrl_can0_default: can0_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_can1_default: can1_default { + pinmux = , + ; + bias-disable; + }; + + /* + * The ADTRG pin can work on any edge type. + * In here it's being pulled up, so need to + * connect it to ground to get an edge e.g. + * Trigger can be configured on falling, rise + * or any edge, and the pull-up can be changed + * to pull-down or left floating according to + * needs. + */ + pinctrl_adtrg_default: adtrg_default { + pinmux = ; + bias-pull-up; + }; + + pinctrl_charger_chglev: charger_chglev { + pinmux = ; + bias-disable; + }; + + pinctrl_charger_irq: charger_irq { + pinmux = ; + bias-disable; + }; + + pinctrl_charger_lbo: charger_lbo { + pinmux = ; + bias-pull-up; + }; + + pinctrl_classd_default_pfets: classd_default_pfets { + pinmux = , + ; + bias-pull-up; + }; + + pinctrl_classd_default_nfets: classd_default_nfets { + pinmux = , + ; + bias-pull-down; + }; + + pinctrl_flx0_default: flx0_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_flx4_default: flx4_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_flx4_gpio: flx4_gpio { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c0_default: i2c0_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c0_gpio: i2c0_gpio { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c1_default: i2c1_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c1_gpio: i2c1_gpio { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2c2_gpio: i2c2_gpio { + pinmux = , + ; + bias-disable; + }; + + pinctrl_i2s0_default: i2s0_default { + pinmux = , + , + , + , + ; + bias-disable; + }; + + pinctrl_i2s1_default: i2s1_default { + pinmux = , + , + , + , + ; + bias-disable; + }; + + pinctrl_key_gpio_default: key_gpio_default { + pinmux = ; + bias-pull-up; + }; + + pinctrl_led_gpio_default: led_gpio_default { + pinmux = , + , + ; + bias-pull-up; + }; + + pinctrl_macb0_default: macb0_default { + pinmux = , + , + , + , + , + , + , + , + , + ; + bias-disable; + }; + + pinctrl_macb0_phy_irq: macb0_phy_irq { + pinmux = ; + bias-disable; + }; + + pinctrl_qspi0_default: qspi0_default { + sck_cs { + pinmux = , + ; + bias-disable; + }; + + data { + pinmux = , + , + , + ; + bias-pull-up; + }; + }; + + pinctrl_sdmmc0_default: sdmmc0_default { + cmd_data { + pinmux = , + , + , + , + , + , + , + , + ; + bias-disable; + }; + + ck_cd_rstn_vddsel { + pinmux = , + , + , + ; + bias-disable; + }; + }; + + pinctrl_sdmmc1_default: sdmmc1_default { + cmd_data { + pinmux = , + , + , + , + ; + bias-disable; + }; + + conf-ck_cd { + pinmux = , + ; + bias-disable; + }; + }; + + pinctrl_spi0_default: spi0_default { + pinmux = , + , + , + ; + bias-disable; + }; + + pinctrl_uart1_default: uart1_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_uart3_default: uart3_default { + pinmux = , + ; + bias-disable; + }; + + pinctrl_usb_default: usb_default { + pinmux = ; + bias-disable; + }; + + pinctrl_usba_vbus: usba_vbus { + pinmux = ; + bias-disable; + }; + + pinctrl_pwm0_pwm2_default: pwm0_pwm2_default { + pinmux = , + ; + bias-pull-up; + }; + }; + + classd: classd@fc048000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_classd_default_pfets &pinctrl_classd_default_nfets>; + atmel,pwm-type = "diff"; + atmel,non-overlap-time = <10>; + status = "okay"; + }; + + i2s1: i2s@fc04c000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2s1_default>; + status = "disabled"; /* conflict with spi0, sdmmc1 */ + }; + + can1: can@fc050000 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can1_default>; + status = "okay"; + }; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_key_gpio_default>; + + bp1 { + label = "PB_USER"; + gpios = <&pioA PIN_PB9 GPIO_ACTIVE_LOW>; + /* linux,code = ; BSD license issue */ + wakeup-source; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_led_gpio_default>; + status = "okay"; /* conflict with pwm0 */ + + red { + label = "red"; + gpios = <&pioA PIN_PB6 GPIO_ACTIVE_LOW>; + }; + + + green { + label = "green"; + gpios = <&pioA PIN_PB5 GPIO_ACTIVE_LOW>; + }; + + blue { + label = "blue"; + gpios = <&pioA PIN_PB0 GPIO_ACTIVE_LOW>; + linux,default-trigger = "heartbeat"; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/dt_driver_test.dtsi b/optee_os/core/arch/arm/dts/dt_driver_test.dtsi new file mode 100644 index 0000000..69bae56 --- /dev/null +++ b/optee_os/core/arch/arm/dts/dt_driver_test.dtsi @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + #include + +/ { + /* + * Device dt-test-consumer consumes resource, it is expected + * to defer its probe until resources are ready. + */ + dt-test-consumer { + compatible = "linaro,dt-test-consumer"; + clocks = <&dt_test_provider 3>, <&dt_test_provider 7>; + clock-names = "clk0", "clk1"; + resets = <&dt_test_provider 35>, <&dt_test_provider 5>; + reset-names = "rst0", "rst1"; + test-gpios = <&dt_test_provider 1 GPIO_ACTIVE_LOW>, + <&dt_test_provider 2 GPIO_PULL_UP>; + }; + + /* + * Resource device are discovered from subnode added to probe + * list by related drivers (here all simple-bus). + */ + dt-test-bus-b0 { + compatible = "simple-bus"; + + dt-test-bus-b1 { + compatible = "simple-bus"; + + dt-test-bus-b2 { + compatible = "simple-bus"; + + dt-test-bus-b3 { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <0>; + + dt_test_provider: dt-test-provider@0 { + compatible = "linaro,dt-test-provider"; + reg = <0>; + #clock-cells = <1>; + #reset-cells = <1>; + #gpio-cells = <2>; + }; + }; + }; + }; + }; + + dt-test-crypt-consumer { + compatible = "linaro,dt-test-crypt-consumer"; + }; +}; diff --git a/optee_os/core/arch/arm/dts/embedded_dtb_test.dts b/optee_os/core/arch/arm/dts/embedded_dtb_test.dts new file mode 100644 index 0000000..48e2d0d --- /dev/null +++ b/optee_os/core/arch/arm/dts/embedded_dtb_test.dts @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ +/dts-v1/; + +#include "dt_driver_test.dtsi" + +/ { + model = "Embedded device tree for OP-TEE/Qemu platform test purpose"; +}; diff --git a/optee_os/core/arch/arm/dts/fsl-lx2160a-qds.dts b/optee_os/core/arch/arm/dts/fsl-lx2160a-qds.dts new file mode 100644 index 0000000..0cf72cb --- /dev/null +++ b/optee_os/core/arch/arm/dts/fsl-lx2160a-qds.dts @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +// +// Device Tree file for LX2160AQDS +// +// Copyright 2018-2020 NXP + +/dts-v1/; + +#include "fsl-lx2160a.dtsi" + +/ { + model = "NXP Layerscape LX2160AQDS"; + compatible = "fsl,lx2160a-qds", "fsl,lx2160a"; + + aliases { + crypto = &crypto; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + sb_3v3: regulator-sb3v3 { + compatible = "regulator-fixed"; + regulator-name = "MC34717-3.3VSB"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + mdio-mux-1 { + compatible = "mdio-mux-multiplexer"; + mux-controls = <&mux 0>; + mdio-parent-bus = <&emdio1>; + #address-cells=<1>; + #size-cells = <0>; + + mdio@0 { /* On-board PHY #1 RGMI1*/ + reg = <0x00>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@8 { /* On-board PHY #2 RGMI2*/ + reg = <0x8>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@18 { /* Slot #1 */ + reg = <0x18>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@19 { /* Slot #2 */ + reg = <0x19>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1a { /* Slot #3 */ + reg = <0x1a>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1b { /* Slot #4 */ + reg = <0x1b>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1c { /* Slot #5 */ + reg = <0x1c>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1d { /* Slot #6 */ + reg = <0x1d>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1e { /* Slot #7 */ + reg = <0x1e>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1f { /* Slot #8 */ + reg = <0x1f>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + mdio-mux-2 { + compatible = "mdio-mux-multiplexer"; + mux-controls = <&mux 1>; + mdio-parent-bus = <&emdio2>; + #address-cells=<1>; + #size-cells = <0>; + + mdio@0 { /* Slot #1 (secondary EMI) */ + reg = <0x00>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1 { /* Slot #2 (secondary EMI) */ + reg = <0x01>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@2 { /* Slot #3 (secondary EMI) */ + reg = <0x02>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@3 { /* Slot #4 (secondary EMI) */ + reg = <0x03>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@4 { /* Slot #5 (secondary EMI) */ + reg = <0x04>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@5 { /* Slot #6 (secondary EMI) */ + reg = <0x05>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@6 { /* Slot #7 (secondary EMI) */ + reg = <0x06>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@7 { /* Slot #8 (secondary EMI) */ + reg = <0x07>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; +}; + +&can0 { + status = "okay"; +}; + +&can1 { + status = "okay"; +}; + +&crypto { + status = "okay"; +}; + +&dspi0 { + status = "okay"; + + dflash0: flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <1000000>; + }; +}; + +&dspi1 { + status = "okay"; + + dflash1: flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <1000000>; + }; +}; + +&dspi2 { + status = "okay"; + + dflash2: flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <1000000>; + }; +}; + +&emdio1 { + status = "okay"; +}; + +&emdio2 { + status = "okay"; +}; + +&esdhc0 { + status = "okay"; +}; + +&esdhc1 { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + fpga@66 { + compatible = "fsl,lx2160aqds-fpga", "fsl,fpga-qixis-i2c", + "simple-mfd"; + reg = <0x66>; + + mux: mux-controller { + compatible = "reg-mux"; + #mux-control-cells = <1>; + mux-reg-masks = <0x54 0xf8>, /* 0: reg 0x54, bits 7:3 */ + <0x54 0x07>; /* 1: reg 0x54, bit 2:0 */ + }; + }; + + i2c-mux@77 { + compatible = "nxp,pca9547"; + reg = <0x77>; + #address-cells = <1>; + #size-cells = <0>; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x2>; + + power-monitor@40 { + compatible = "ti,ina220"; + reg = <0x40>; + shunt-resistor = <500>; + }; + + power-monitor@41 { + compatible = "ti,ina220"; + reg = <0x41>; + shunt-resistor = <1000>; + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x3>; + + temperature-sensor@4c { + compatible = "nxp,sa56004"; + reg = <0x4c>; + vcc-supply = <&sb_3v3>; + }; + + temperature-sensor@4d { + compatible = "nxp,sa56004"; + reg = <0x4d>; + vcc-supply = <&sb_3v3>; + }; + + rtc@51 { + compatible = "nxp,pcf2129"; + reg = <0x51>; + }; + }; + }; +}; + +&sata0 { + status = "okay"; +}; + +&sata1 { + status = "okay"; +}; + +&sata2 { + status = "okay"; +}; + +&sata3 { + status = "okay"; +}; + +&uart0 { + status = "okay"; +}; + +&uart1 { + status = "okay"; +}; + +&usb0 { + status = "okay"; +}; + +&usb1 { + status = "okay"; +}; + +&sata0 { + status = "okay"; +}; + +&sata1 { + status = "okay"; +}; + +&sata2 { + status = "okay"; +}; + +&sata3 { + status = "okay"; +}; diff --git a/optee_os/core/arch/arm/dts/fsl-lx2160a-rdb.dts b/optee_os/core/arch/arm/dts/fsl-lx2160a-rdb.dts new file mode 100644 index 0000000..c2817b7 --- /dev/null +++ b/optee_os/core/arch/arm/dts/fsl-lx2160a-rdb.dts @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +// +// Device Tree file for LX2160ARDB +// +// Copyright 2018 NXP + +/dts-v1/; + +#include "fsl-lx2160a.dtsi" + +/ { + model = "NXP Layerscape LX2160ARDB"; + compatible = "fsl,lx2160a-rdb", "fsl,lx2160a"; + + aliases { + crypto = &crypto; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + sb_3v3: regulator-sb3v3 { + compatible = "regulator-fixed"; + regulator-name = "MC34717-3.3VSB"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; +}; + +&crypto { + status = "okay"; +}; + +&esdhc0 { + sd-uhs-sdr104; + sd-uhs-sdr50; + sd-uhs-sdr25; + sd-uhs-sdr12; + status = "okay"; +}; + +&esdhc1 { + mmc-hs200-1_8v; + mmc-hs400-1_8v; + bus-width = <8>; + status = "okay"; +}; + +&fspi { + status = "okay"; + + mt35xu512aba0: flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "spansion,m25p80"; + m25p,fast-read; + spi-max-frequency = <50000000>; + reg = <0>; + spi-rx-bus-width = <8>; + spi-tx-bus-width = <8>; + }; + + mt35xu512aba1: flash@1 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "spansion,m25p80"; + m25p,fast-read; + spi-max-frequency = <50000000>; + reg = <1>; + spi-rx-bus-width = <8>; + spi-tx-bus-width = <8>; + }; +}; + +&i2c0 { + status = "okay"; + + i2c-mux@77 { + compatible = "nxp,pca9547"; + reg = <0x77>; + #address-cells = <1>; + #size-cells = <0>; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x2>; + + power-monitor@40 { + compatible = "ti,ina220"; + reg = <0x40>; + shunt-resistor = <1000>; + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x3>; + + temperature-sensor@4c { + compatible = "nxp,sa56004"; + reg = <0x4c>; + vcc-supply = <&sb_3v3>; + }; + + temperature-sensor@4d { + compatible = "nxp,sa56004"; + reg = <0x4d>; + vcc-supply = <&sb_3v3>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; + + rtc@51 { + compatible = "nxp,pcf2129"; + reg = <0x51>; + // IRQ10_B + interrupts = <0 150 0x4>; + }; +}; + +&sata0 { + status = "okay"; +}; + +&sata1 { + status = "okay"; +}; + +&sata2 { + status = "okay"; +}; + +&sata3 { + status = "okay"; +}; + +&uart0 { + status = "okay"; +}; + +&uart1 { + status = "okay"; +}; + +&usb0 { + status = "okay"; +}; + +&usb1 { + status = "okay"; +}; diff --git a/optee_os/core/arch/arm/dts/fsl-lx2160a.dtsi b/optee_os/core/arch/arm/dts/fsl-lx2160a.dtsi new file mode 100644 index 0000000..c5bdfc8 --- /dev/null +++ b/optee_os/core/arch/arm/dts/fsl-lx2160a.dtsi @@ -0,0 +1,1385 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +// +// Device Tree Include file for Layerscape-LX2160A family SoC. +// +// Copyright 2018-2020 NXP + +#include +#include + +/memreserve/ 0x80000000 0x00010000; + +/ { + compatible = "fsl,lx2160a"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + rtc1 = &ftm_alarm0; + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + // 8 clusters having 2 Cortex-A72 cores each + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x0>; + clocks = <&clockgen 1 0>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster0_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x1>; + clocks = <&clockgen 1 0>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster0_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu100: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x100>; + clocks = <&clockgen 1 1>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster1_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu101: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x101>; + clocks = <&clockgen 1 1>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster1_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu200: cpu@200 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x200>; + clocks = <&clockgen 1 2>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster2_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu201: cpu@201 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x201>; + clocks = <&clockgen 1 2>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster2_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu300: cpu@300 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x300>; + clocks = <&clockgen 1 3>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster3_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu301: cpu@301 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x301>; + clocks = <&clockgen 1 3>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster3_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu400: cpu@400 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x400>; + clocks = <&clockgen 1 4>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster4_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu401: cpu@401 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x401>; + clocks = <&clockgen 1 4>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster4_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu500: cpu@500 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x500>; + clocks = <&clockgen 1 5>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster5_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu501: cpu@501 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x501>; + clocks = <&clockgen 1 5>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster5_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu600: cpu@600 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x600>; + clocks = <&clockgen 1 6>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster6_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu601: cpu@601 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x601>; + clocks = <&clockgen 1 6>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster6_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu700: cpu@700 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x700>; + clocks = <&clockgen 1 7>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster7_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cpu701: cpu@701 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + enable-method = "psci"; + reg = <0x701>; + clocks = <&clockgen 1 7>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <192>; + next-level-cache = <&cluster7_l2>; + cpu-idle-states = <&cpu_pw15>; + #cooling-cells = <2>; + }; + + cluster0_l2: l2-cache0 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cluster1_l2: l2-cache1 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cluster2_l2: l2-cache2 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cluster3_l2: l2-cache3 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cluster4_l2: l2-cache4 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cluster5_l2: l2-cache5 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cluster6_l2: l2-cache6 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cluster7_l2: l2-cache7 { + compatible = "cache"; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-level = <2>; + }; + + cpu_pw15: cpu-pw15 { + compatible = "arm,idle-state"; + idle-state-name = "PW15"; + arm,psci-suspend-param = <0x0>; + entry-latency-us = <2000>; + exit-latency-us = <2000>; + min-residency-us = <6000>; + }; + }; + + gic: interrupt-controller@6000000 { + compatible = "arm,gic-v3"; + reg = <0x0 0x06000000 0 0x10000>, // GIC Dist + <0x0 0x06200000 0 0x200000>, // GICR (RD_base + + // SGI_base) + <0x0 0x0c0c0000 0 0x2000>, // GICC + <0x0 0x0c0d0000 0 0x1000>, // GICH + <0x0 0x0c0e0000 0 0x20000>; // GICV + #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + interrupt-controller; + interrupts = ; + + its: gic-its@6020000 { + compatible = "arm,gic-v3-its"; + msi-controller; + reg = <0x0 0x6020000 0 0x20000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + }; + + pmu { + compatible = "arm,cortex-a72-pmu"; + interrupts = ; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + memory@80000000 { + // DRAM space - 1, size : 2 GB DRAM + device_type = "memory"; + reg = <0x00000000 0x80000000 0 0x80000000>; + }; + + ddr1: memory-controller@1080000 { + compatible = "fsl,qoriq-memory-controller"; + reg = <0x0 0x1080000 0x0 0x1000>; + interrupts = ; + little-endian; + }; + + ddr2: memory-controller@1090000 { + compatible = "fsl,qoriq-memory-controller"; + reg = <0x0 0x1090000 0x0 0x1000>; + interrupts = ; + little-endian; + }; + + // One clock unit-sysclk node which bootloader require during DT fix-up + sysclk: sysclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; // fixed up by bootloader + clock-output-names = "sysclk"; + }; + + thermal-zones { + core_thermal1: core-thermal1 { + polling-delay-passive = <1000>; + polling-delay = <5000>; + thermal-sensors = <&tmu 0>; + + trips { + core_cluster_alert: core-cluster-alert { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + + core_cluster_crit: core-cluster-crit { + temperature = <95000>; + hysteresis = <2000>; + type = "critical"; + }; + }; + + }; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + dma-ranges = <0x0 0x0 0x0 0x0 0x10000 0x00000000>; + + crypto: crypto@8000000 { + compatible = "fsl,sec-v5.0", "fsl,sec-v4.0"; + fsl,sec-era = <10>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x00 0x8000000 0x100000>; + reg = <0x00 0x8000000 0x0 0x100000>; + interrupts = ; + dma-coherent; + status = "disabled"; + + sec_jr0: jr@10000 { + compatible = "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x10000 0x10000>; + interrupts = ; + status = "okay"; secure-status = "disabled"; /* NS-only */ + }; + + sec_jr1: jr@20000 { + compatible = "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x20000 0x10000>; + interrupts = ; + status = "okay"; secure-status = "disabled"; /* NS-only */ + }; + + sec_jr2: jr@30000 { + compatible = "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x30000 0x10000>; + interrupts = ; + status = "disabled"; secure-status = "okay"; /* S-only */ + }; + + sec_jr3: jr@40000 { + compatible = "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x40000 0x10000>; + interrupts = ; + status = "okay"; secure-status = "disabled"; /* workaround for ATF */ + }; + }; + + clockgen: clock-controller@1300000 { + compatible = "fsl,lx2160a-clockgen"; + reg = <0 0x1300000 0 0xa0000>; + #clock-cells = <2>; + clocks = <&sysclk>; + }; + + dcfg: syscon@1e00000 { + compatible = "fsl,lx2160a-dcfg", "syscon"; + reg = <0x0 0x1e00000 0x0 0x10000>; + little-endian; + }; + + sfp: sfp@1e80000 { + compatible = "fsl,lx2160a-sfp"; + reg = <0x0 0x1e80000 0x0 0x1000>; + }; + + sec_mon: sec-mon@1e90000 { + compatible = "fsl,lx2160a-sec-mon"; + reg = <0x0 0x1e90000 0x0 0x1000>; + status = "disabled"; + secure-status = "okay"; + }; + + /* WRIOP0: 0x8b8_0000, E-MDIO1: 0x1_6000 */ + emdio1: mdio@8b96000 { + compatible = "fsl,fman-memac-mdio"; + reg = <0x0 0x8b96000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + little-endian; /* force the driver in LE mode */ + status = "disabled"; + }; + + /* WRIOP0: 0x8b8_0000, E-MDIO2: 0x1_7000 */ + emdio2: mdio@8b97000 { + compatible = "fsl,fman-memac-mdio"; + reg = <0x0 0x8b97000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + little-endian; /* force the driver in LE mode */ + status = "disabled"; + }; + + i2c0: i2c@2000000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2000000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + scl-gpios = <&gpio2 15 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; + + i2c1: i2c@2010000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2010000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + status = "disabled"; + }; + + i2c2: i2c@2020000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2020000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + status = "disabled"; + }; + + i2c3: i2c@2030000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2030000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + status = "disabled"; + }; + + i2c4: i2c@2040000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2040000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + scl-gpios = <&gpio2 16 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; + + i2c5: i2c@2050000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2050000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + status = "disabled"; + }; + + i2c6: i2c@2060000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2060000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + status = "disabled"; + }; + + i2c7: i2c@2070000 { + compatible = "fsl,vf610-i2c"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2070000 0x0 0x10000>; + interrupts = ; + clock-names = "i2c"; + clocks = <&clockgen 4 15>; + status = "disabled"; + }; + + fspi: spi@20c0000 { + compatible = "nxp,lx2160a-fspi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x20c0000 0x0 0x10000>, + <0x0 0x20000000 0x0 0x10000000>; + reg-names = "fspi_base", "fspi_mmap"; + interrupts = ; + clocks = <&clockgen 4 3>, <&clockgen 4 3>; + clock-names = "fspi_en", "fspi"; + status = "disabled"; + }; + + dspi0: spi@2100000 { + compatible = "fsl,lx2160a-dspi", "fsl,ls2085a-dspi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2100000 0x0 0x10000>; + interrupts = ; + clocks = <&clockgen 4 7>; + clock-names = "dspi"; + spi-num-chipselects = <5>; + bus-num = <0>; + status = "disabled"; + }; + + dspi1: spi@2110000 { + compatible = "fsl,lx2160a-dspi", "fsl,ls2085a-dspi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2110000 0x0 0x10000>; + interrupts = ; + clocks = <&clockgen 4 7>; + clock-names = "dspi"; + spi-num-chipselects = <5>; + bus-num = <1>; + status = "disabled"; + }; + + dspi2: spi@2120000 { + compatible = "fsl,lx2160a-dspi", "fsl,ls2085a-dspi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x2120000 0x0 0x10000>; + interrupts = ; + clocks = <&clockgen 4 7>; + clock-names = "dspi"; + spi-num-chipselects = <5>; + bus-num = <2>; + status = "disabled"; + }; + + esdhc0: esdhc@2140000 { + compatible = "fsl,esdhc"; + reg = <0x0 0x2140000 0x0 0x10000>; + interrupts = <0 28 0x4>; /* Level high type */ + clocks = <&clockgen 4 1>; + voltage-ranges = <1800 1800 3300 3300>; + sdhci,auto-cmd12; + little-endian; + bus-width = <4>; + status = "disabled"; + }; + + esdhc1: esdhc@2150000 { + compatible = "fsl,esdhc"; + reg = <0x0 0x2150000 0x0 0x10000>; + interrupts = <0 63 0x4>; /* Level high type */ + clocks = <&clockgen 4 1>; + voltage-ranges = <1800 1800 3300 3300>; + sdhci,auto-cmd12; + broken-cd; + little-endian; + bus-width = <4>; + status = "disabled"; + }; + + can0: can@2180000 { + compatible = "fsl,lx2160ar1-flexcan"; + reg = <0x0 0x2180000 0x0 0x10000>; + interrupts = ; + clocks = <&sysclk>, <&clockgen 4 7>; + clock-names = "ipg", "per"; + status = "disabled"; + }; + + can1: can@2190000 { + compatible = "fsl,lx2160ar1-flexcan"; + reg = <0x0 0x2190000 0x0 0x10000>; + interrupts = ; + clocks = <&sysclk>, <&clockgen 4 7>; + clock-names = "ipg", "per"; + status = "disabled"; + }; + + tmu: tmu@1f80000 { + compatible = "fsl,qoriq-tmu"; + reg = <0x0 0x1f80000 0x0 0x10000>; + interrupts = <0 23 0x4>; + fsl,tmu-range = <0x800000E6 0x8001017D>; + fsl,tmu-calibration = + /* Calibration data group 1 */ + <0x00000000 0x00000035 + /* Calibration data group 2 */ + 0x00010001 0x00000154>; + little-endian; + #thermal-sensor-cells = <1>; + }; + + uart0: serial@21c0000 { + compatible = "arm,sbsa-uart","arm,pl011"; + reg = <0x0 0x21c0000 0x0 0x1000>; + interrupts = ; + current-speed = <115200>; + status = "disabled"; + }; + + uart1: serial@21d0000 { + compatible = "arm,sbsa-uart","arm,pl011"; + reg = <0x0 0x21d0000 0x0 0x1000>; + interrupts = ; + current-speed = <115200>; + status = "disabled"; + }; + + uart2: serial@21e0000 { + compatible = "arm,sbsa-uart","arm,pl011"; + reg = <0x0 0x21e0000 0x0 0x1000>; + interrupts = ; + current-speed = <115200>; + status = "disabled"; + }; + + uart3: serial@21f0000 { + compatible = "arm,sbsa-uart","arm,pl011"; + reg = <0x0 0x21f0000 0x0 0x1000>; + interrupts = ; + current-speed = <115200>; + status = "disabled"; + }; + + gpio0: gpio@2300000 { + compatible = "fsl,qoriq-gpio"; + reg = <0x0 0x2300000 0x0 0x10000>; + interrupts = ; + gpio-controller; + little-endian; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio1: gpio@2310000 { + compatible = "fsl,qoriq-gpio"; + reg = <0x0 0x2310000 0x0 0x10000>; + interrupts = ; + gpio-controller; + little-endian; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio2: gpio@2320000 { + compatible = "fsl,qoriq-gpio"; + reg = <0x0 0x2320000 0x0 0x10000>; + interrupts = ; + gpio-controller; + little-endian; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio3: gpio@2330000 { + compatible = "fsl,qoriq-gpio"; + reg = <0x0 0x2330000 0x0 0x10000>; + interrupts = ; + gpio-controller; + little-endian; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + watchdog@23a0000 { + compatible = "arm,sbsa-gwdt"; + reg = <0x0 0x23a0000 0 0x1000>, + <0x0 0x2390000 0 0x1000>; + interrupts = ; + timeout-sec = <30>; + }; + + rcpm: rcpm@1e34040 { + compatible = "fsl,lx2160a-rcpm", "fsl,qoriq-rcpm-2.1+"; + reg = <0x0 0x1e34040 0x0 0x1c>; + #fsl,rcpm-wakeup-cells = <7>; + little-endian; + }; + + ftm_alarm0: timer@2800000 { + compatible = "fsl,lx2160a-ftm-alarm"; + reg = <0x0 0x2800000 0x0 0x10000>; + fsl,rcpm-wakeup = <&rcpm 0x0 0x0 0x0 0x0 0x4000 0x0 0x0>; + interrupts = <0 44 4>; + }; + + usb0: usb@3100000 { + compatible = "fsl,lx2160a-dwc3", "snps,dwc3"; + reg = <0x0 0x3100000 0x0 0x10000>; + interrupts = ; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + usb3-lpm-capable; + snps,dis-u1u2-when-u3-quirk; + snps,dis_rxdet_inp3_quirk; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + snps,host-vbus-glitches; + dma-coherent; + status = "disabled"; + }; + + usb1: usb@3110000 { + compatible = "fsl,lx2160a-dwc3", "snps,dwc3"; + reg = <0x0 0x3110000 0x0 0x10000>; + interrupts = ; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + usb3-lpm-capable; + snps,dis-u1u2-when-u3-quirk; + snps,dis_rxdet_inp3_quirk; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + snps,host-vbus-glitches; + status = "disabled"; + }; + + sata0: sata@3200000 { + compatible = "fsl,lx2160a-ahci"; + reg = <0x0 0x3200000 0x0 0x10000>, + <0x7 0x100520 0x0 0x4>; + reg-names = "ahci", "sata-ecc"; + interrupts = ; + clocks = <&clockgen 4 3>; + dma-coherent; + status = "disabled"; + }; + + sata1: sata@3210000 { + compatible = "fsl,lx2160a-ahci"; + reg = <0x0 0x3210000 0x0 0x10000>, + <0x7 0x100520 0x0 0x4>; + reg-names = "ahci", "sata-ecc"; + interrupts = ; + clocks = <&clockgen 4 3>; + dma-coherent; + status = "disabled"; + }; + + sata2: sata@3220000 { + compatible = "fsl,lx2160a-ahci"; + reg = <0x0 0x3220000 0x0 0x10000>, + <0x7 0x100520 0x0 0x4>; + reg-names = "ahci", "sata-ecc"; + interrupts = ; + clocks = <&clockgen 4 3>; + dma-coherent; + status = "disabled"; + }; + + sata3: sata@3230000 { + compatible = "fsl,lx2160a-ahci"; + reg = <0x0 0x3230000 0x0 0x10000>, + <0x7 0x100520 0x0 0x4>; + reg-names = "ahci", "sata-ecc"; + interrupts = ; + clocks = <&clockgen 4 3>; + dma-coherent; + status = "disabled"; + }; + + pcie@3400000 { + compatible = "fsl,lx2160a-pcie"; + reg = <0x00 0x03400000 0x0 0x00100000 /* controller registers */ + 0x80 0x00000000 0x0 0x00001000>; /* configuration space */ + reg-names = "csr_axi_slave", "config_axi_slave"; + interrupts = , /* AER interrupt */ + , /* PME interrupt */ + ; /* controller interrupt */ + interrupt-names = "aer", "pme", "intr"; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + dma-coherent; + apio-wins = <8>; + ppio-wins = <8>; + bus-range = <0x0 0xff>; + ranges = <0x82000000 0x0 0x40000000 0x80 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */ + msi-parent = <&its>; + iommu-map = <0 &smmu 0 1>; /* This is fixed-up by u-boot */ + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0000 0 0 1 &gic 0 0 GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 2 &gic 0 0 GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 3 &gic 0 0 GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 4 &gic 0 0 GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + pcie_ep@3400000 { + compatible = "fsl,lx2160a-pcie-ep"; + reg = <0x00 0x03400000 0x0 0x00100000 + 0x80 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ob-windows = <256>; + status = "disabled"; + }; + + pcie@3500000 { + compatible = "fsl,lx2160a-pcie"; + reg = <0x00 0x03500000 0x0 0x00100000 /* controller registers */ + 0x88 0x00000000 0x0 0x00001000>; /* configuration space */ + reg-names = "csr_axi_slave", "config_axi_slave"; + interrupts = , /* AER interrupt */ + , /* PME interrupt */ + ; /* controller interrupt */ + interrupt-names = "aer", "pme", "intr"; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + dma-coherent; + apio-wins = <8>; + ppio-wins = <8>; + bus-range = <0x0 0xff>; + ranges = <0x82000000 0x0 0x40000000 0x88 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */ + msi-parent = <&its>; + iommu-map = <0 &smmu 0 1>; /* This is fixed-up by u-boot */ + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0000 0 0 1 &gic 0 0 GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 2 &gic 0 0 GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 3 &gic 0 0 GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 4 &gic 0 0 GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + pcie_ep@3500000 { + compatible = "fsl,lx2160a-pcie-ep"; + reg = <0x00 0x03500000 0x0 0x00100000 + 0x88 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ob-windows = <256>; + status = "disabled"; + }; + + pcie@3600000 { + compatible = "fsl,lx2160a-pcie"; + reg = <0x00 0x03600000 0x0 0x00100000 /* controller registers */ + 0x90 0x00000000 0x0 0x00001000>; /* configuration space */ + reg-names = "csr_axi_slave", "config_axi_slave"; + interrupts = , /* AER interrupt */ + , /* PME interrupt */ + ; /* controller interrupt */ + interrupt-names = "aer", "pme", "intr"; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + dma-coherent; + apio-wins = <8>; + ppio-wins = <8>; + bus-range = <0x0 0xff>; + ranges = <0x82000000 0x0 0x40000000 0x90 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */ + msi-parent = <&its>; + iommu-map = <0 &smmu 0 1>; /* This is fixed-up by u-boot */ + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0000 0 0 1 &gic 0 0 GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 2 &gic 0 0 GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 3 &gic 0 0 GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 4 &gic 0 0 GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + pcie_ep@3600000 { + compatible = "fsl,lx2160a-pcie-ep"; + reg = <0x00 0x03600000 0x0 0x00100000 + 0x90 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ob-windows = <256>; + max-functions = <2>; + status = "disabled"; + }; + + pcie@3700000 { + compatible = "fsl,lx2160a-pcie"; + reg = <0x00 0x03700000 0x0 0x00100000 /* controller registers */ + 0x98 0x00000000 0x0 0x00001000>; /* configuration space */ + reg-names = "csr_axi_slave", "config_axi_slave"; + interrupts = , /* AER interrupt */ + , /* PME interrupt */ + ; /* controller interrupt */ + interrupt-names = "aer", "pme", "intr"; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + dma-coherent; + apio-wins = <8>; + ppio-wins = <8>; + bus-range = <0x0 0xff>; + ranges = <0x82000000 0x0 0x40000000 0x98 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */ + msi-parent = <&its>; + iommu-map = <0 &smmu 0 1>; /* This is fixed-up by u-boot */ + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0000 0 0 1 &gic 0 0 GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 2 &gic 0 0 GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 3 &gic 0 0 GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 4 &gic 0 0 GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + pcie_ep@3700000 { + compatible = "fsl,lx2160a-pcie-ep"; + reg = <0x00 0x03700000 0x0 0x00100000 + 0x98 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ob-windows = <256>; + status = "disabled"; + }; + + pcie@3800000 { + compatible = "fsl,lx2160a-pcie"; + reg = <0x00 0x03800000 0x0 0x00100000 /* controller registers */ + 0xa0 0x00000000 0x0 0x00001000>; /* configuration space */ + reg-names = "csr_axi_slave", "config_axi_slave"; + interrupts = , /* AER interrupt */ + , /* PME interrupt */ + ; /* controller interrupt */ + interrupt-names = "aer", "pme", "intr"; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + dma-coherent; + apio-wins = <8>; + ppio-wins = <8>; + bus-range = <0x0 0xff>; + ranges = <0x82000000 0x0 0x40000000 0xa0 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */ + msi-parent = <&its>; + iommu-map = <0 &smmu 0 1>; /* This is fixed-up by u-boot */ + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0000 0 0 1 &gic 0 0 GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 2 &gic 0 0 GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 3 &gic 0 0 GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 4 &gic 0 0 GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + pcie_ep@3800000 { + compatible = "fsl,lx2160a-pcie-ep"; + reg = <0x00 0x03800000 0x0 0x00100000 + 0xa0 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ob-windows = <256>; + max-functions = <2>; + status = "disabled"; + }; + + pcie@3900000 { + compatible = "fsl,lx2160a-pcie"; + reg = <0x00 0x03900000 0x0 0x00100000 /* controller registers */ + 0xa8 0x00000000 0x0 0x00001000>; /* configuration space */ + reg-names = "csr_axi_slave", "config_axi_slave"; + interrupts = , /* AER interrupt */ + , /* PME interrupt */ + ; /* controller interrupt */ + interrupt-names = "aer", "pme", "intr"; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + dma-coherent; + apio-wins = <8>; + ppio-wins = <8>; + bus-range = <0x0 0xff>; + ranges = <0x82000000 0x0 0x40000000 0xa8 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */ + msi-parent = <&its>; + iommu-map = <0 &smmu 0 1>; /* This is fixed-up by u-boot */ + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0000 0 0 1 &gic 0 0 GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 2 &gic 0 0 GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 3 &gic 0 0 GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>, + <0000 0 0 4 &gic 0 0 GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + pcie_ep@3900000 { + compatible = "fsl,lx2160a-pcie-ep"; + reg = <0x00 0x03900000 0x0 0x00100000 + 0xa8 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ob-windows = <256>; + status = "disabled"; + }; + + smmu: iommu@5000000 { + compatible = "arm,mmu-500"; + reg = <0 0x5000000 0 0x800000>; + #iommu-cells = <1>; + #global-interrupts = <14>; + // global secure fault + interrupts = , + // combined secure + , + // global non-secure fault + , + // combined non-secure + , + // performance counter interrupts 0-9 + , + , + , + , + , + , + , + , + , + , + // per context interrupt, 64 interrupts + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + dma-coherent; + }; + + console@8340020 { + compatible = "fsl,dpaa2-console"; + reg = <0x00000000 0x08340020 0 0x2>; + }; + + ptp-timer@8b95000 { + compatible = "fsl,dpaa2-ptp"; + reg = <0x0 0x8b95000 0x0 0x100>; + clocks = <&clockgen 4 1>; + little-endian; + fsl,extts-fifo; + }; + + fsl_mc: fsl-mc@80c000000 { + compatible = "fsl,qoriq-mc"; + reg = <0x00000008 0x0c000000 0 0x40>, + <0x00000000 0x08340000 0 0x40000>; + msi-parent = <&its>; + /* iommu-map property is fixed up by u-boot */ + iommu-map = <0 &smmu 0 0>; + dma-coherent; + #address-cells = <3>; + #size-cells = <1>; + + /* + * Region type 0x0 - MC portals + * Region type 0x1 - QBMAN portals + */ + ranges = <0x0 0x0 0x0 0x8 0x0c000000 0x4000000 + 0x1 0x0 0x0 0x8 0x18000000 0x8000000>; + + /* + * Define the maximum number of MACs present on the SoC. + */ + dpmacs { + #address-cells = <1>; + #size-cells = <0>; + + dpmac1: dpmac@1 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x1>; + }; + + dpmac2: dpmac@2 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x2>; + }; + + dpmac3: dpmac@3 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x3>; + }; + + dpmac4: dpmac@4 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x4>; + }; + + dpmac5: dpmac@5 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x5>; + }; + + dpmac6: dpmac@6 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x6>; + }; + + dpmac7: dpmac@7 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x7>; + }; + + dpmac8: dpmac@8 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x8>; + }; + + dpmac9: dpmac@9 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x9>; + }; + + dpmac10: dpmac@a { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0xa>; + }; + + dpmac11: dpmac@b { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0xb>; + }; + + dpmac12: dpmac@c { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0xc>; + }; + + dpmac13: dpmac@d { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0xd>; + }; + + dpmac14: dpmac@e { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0xe>; + }; + + dpmac15: dpmac@f { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0xf>; + }; + + dpmac16: dpmac@10 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x10>; + }; + + dpmac17: dpmac@11 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x11>; + }; + + dpmac18: dpmac@12 { + compatible = "fsl,qoriq-mc-dpmac"; + reg = <0x12>; + }; + }; + }; + }; + + firmware { + optee { + compatible = "linaro,optee-tz"; + method = "smc"; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/hikey.dts b/optee_os/core/arch/arm/dts/hikey.dts new file mode 100644 index 0000000..0ac794b --- /dev/null +++ b/optee_os/core/arch/arm/dts/hikey.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <0>; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + + sdp@3e800000 { + compatible = "linaro,secure-heap"; + no-map; + reg = <0x3E800000 0x00400000>; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/sama5d2-pinfunc.h b/optee_os/core/arch/arm/dts/sama5d2-pinfunc.h new file mode 100644 index 0000000..25d64d7 --- /dev/null +++ b/optee_os/core/arch/arm/dts/sama5d2-pinfunc.h @@ -0,0 +1,885 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (C) 2021 Microchip + */ + +#define PINMUX_PIN(no, func, ioset) \ +(((no) & 0xffff) | (((func) & 0xf) << 16) | (((ioset) & 0xff) << 20)) + +#define PIN_PA0 0 +#define PIN_PA0__GPIO PINMUX_PIN(PIN_PA0, 0, 0) +#define PIN_PA0__SDMMC0_CK PINMUX_PIN(PIN_PA0, 1, 1) +#define PIN_PA0__QSPI0_SCK PINMUX_PIN(PIN_PA0, 2, 1) +#define PIN_PA0__D0 PINMUX_PIN(PIN_PA0, 6, 2) +#define PIN_PA1 1 +#define PIN_PA1__GPIO PINMUX_PIN(PIN_PA1, 0, 0) +#define PIN_PA1__SDMMC0_CMD PINMUX_PIN(PIN_PA1, 1, 1) +#define PIN_PA1__QSPI0_CS PINMUX_PIN(PIN_PA1, 2, 1) +#define PIN_PA1__D1 PINMUX_PIN(PIN_PA1, 6, 2) +#define PIN_PA2 2 +#define PIN_PA2__GPIO PINMUX_PIN(PIN_PA2, 0, 0) +#define PIN_PA2__SDMMC0_DAT0 PINMUX_PIN(PIN_PA2, 1, 1) +#define PIN_PA2__QSPI0_IO0 PINMUX_PIN(PIN_PA2, 2, 1) +#define PIN_PA2__D2 PINMUX_PIN(PIN_PA2, 6, 2) +#define PIN_PA3 3 +#define PIN_PA3__GPIO PINMUX_PIN(PIN_PA3, 0, 0) +#define PIN_PA3__SDMMC0_DAT1 PINMUX_PIN(PIN_PA3, 1, 1) +#define PIN_PA3__QSPI0_IO1 PINMUX_PIN(PIN_PA3, 2, 1) +#define PIN_PA3__D3 PINMUX_PIN(PIN_PA3, 6, 2) +#define PIN_PA4 4 +#define PIN_PA4__GPIO PINMUX_PIN(PIN_PA4, 0, 0) +#define PIN_PA4__SDMMC0_DAT2 PINMUX_PIN(PIN_PA4, 1, 1) +#define PIN_PA4__QSPI0_IO2 PINMUX_PIN(PIN_PA4, 2, 1) +#define PIN_PA4__D4 PINMUX_PIN(PIN_PA4, 6, 2) +#define PIN_PA5 5 +#define PIN_PA5__GPIO PINMUX_PIN(PIN_PA5, 0, 0) +#define PIN_PA5__SDMMC0_DAT3 PINMUX_PIN(PIN_PA5, 1, 1) +#define PIN_PA5__QSPI0_IO3 PINMUX_PIN(PIN_PA5, 2, 1) +#define PIN_PA5__D5 PINMUX_PIN(PIN_PA5, 6, 2) +#define PIN_PA6 6 +#define PIN_PA6__GPIO PINMUX_PIN(PIN_PA6, 0, 0) +#define PIN_PA6__SDMMC0_DAT4 PINMUX_PIN(PIN_PA6, 1, 1) +#define PIN_PA6__QSPI1_SCK PINMUX_PIN(PIN_PA6, 2, 1) +#define PIN_PA6__TIOA5 PINMUX_PIN(PIN_PA6, 4, 1) +#define PIN_PA6__FLEXCOM2_IO0 PINMUX_PIN(PIN_PA6, 5, 1) +#define PIN_PA6__D6 PINMUX_PIN(PIN_PA6, 6, 2) +#define PIN_PA7 7 +#define PIN_PA7__GPIO PINMUX_PIN(PIN_PA7, 0, 0) +#define PIN_PA7__SDMMC0_DAT5 PINMUX_PIN(PIN_PA7, 1, 1) +#define PIN_PA7__QSPI1_IO0 PINMUX_PIN(PIN_PA7, 2, 1) +#define PIN_PA7__TIOB5 PINMUX_PIN(PIN_PA7, 4, 1) +#define PIN_PA7__FLEXCOM2_IO1 PINMUX_PIN(PIN_PA7, 5, 1) +#define PIN_PA7__D7 PINMUX_PIN(PIN_PA7, 6, 2) +#define PIN_PA8 8 +#define PIN_PA8__GPIO PINMUX_PIN(PIN_PA8, 0, 0) +#define PIN_PA8__SDMMC0_DAT6 PINMUX_PIN(PIN_PA8, 1, 1) +#define PIN_PA8__QSPI1_IO1 PINMUX_PIN(PIN_PA8, 2, 1) +#define PIN_PA8__TCLK5 PINMUX_PIN(PIN_PA8, 4, 1) +#define PIN_PA8__FLEXCOM2_IO2 PINMUX_PIN(PIN_PA8, 5, 1) +#define PIN_PA8__NWE_NANDWE PINMUX_PIN(PIN_PA8, 6, 2) +#define PIN_PA9 9 +#define PIN_PA9__GPIO PINMUX_PIN(PIN_PA9, 0, 0) +#define PIN_PA9__SDMMC0_DAT7 PINMUX_PIN(PIN_PA9, 1, 1) +#define PIN_PA9__QSPI1_IO2 PINMUX_PIN(PIN_PA9, 2, 1) +#define PIN_PA9__TIOA4 PINMUX_PIN(PIN_PA9, 4, 1) +#define PIN_PA9__FLEXCOM2_IO3 PINMUX_PIN(PIN_PA9, 5, 1) +#define PIN_PA9__NCS3 PINMUX_PIN(PIN_PA9, 6, 2) +#define PIN_PA10 10 +#define PIN_PA10__GPIO PINMUX_PIN(PIN_PA10, 0, 0) +#define PIN_PA10__SDMMC0_RSTN PINMUX_PIN(PIN_PA10, 1, 1) +#define PIN_PA10__QSPI1_IO3 PINMUX_PIN(PIN_PA10, 2, 1) +#define PIN_PA10__TIOB4 PINMUX_PIN(PIN_PA10, 4, 1) +#define PIN_PA10__FLEXCOM2_IO4 PINMUX_PIN(PIN_PA10, 5, 1) +#define PIN_PA10__A21_NANDALE PINMUX_PIN(PIN_PA10, 6, 2) +#define PIN_PA11 11 +#define PIN_PA11__GPIO PINMUX_PIN(PIN_PA11, 0, 0) +#define PIN_PA11__SDMMC0_VDDSEL PINMUX_PIN(PIN_PA11, 1, 1) +#define PIN_PA11__QSPI1_CS PINMUX_PIN(PIN_PA11, 2, 1) +#define PIN_PA11__TCLK4 PINMUX_PIN(PIN_PA11, 4, 1) +#define PIN_PA11__A22_NANDCLE PINMUX_PIN(PIN_PA11, 6, 2) +#define PIN_PA12 12 +#define PIN_PA12__GPIO PINMUX_PIN(PIN_PA12, 0, 0) +#define PIN_PA12__SDMMC0_WP PINMUX_PIN(PIN_PA12, 1, 1) +#define PIN_PA12__IRQ PINMUX_PIN(PIN_PA12, 2, 1) +#define PIN_PA12__NRD_NANDOE PINMUX_PIN(PIN_PA12, 6, 2) +#define PIN_PA13 13 +#define PIN_PA13__GPIO PINMUX_PIN(PIN_PA13, 0, 0) +#define PIN_PA13__SDMMC0_CD PINMUX_PIN(PIN_PA13, 1, 1) +#define PIN_PA13__FLEXCOM3_IO1 PINMUX_PIN(PIN_PA13, 5, 1) +#define PIN_PA13__D8 PINMUX_PIN(PIN_PA13, 6, 2) +#define PIN_PA14 14 +#define PIN_PA14__GPIO PINMUX_PIN(PIN_PA14, 0, 0) +#define PIN_PA14__SPI0_SPCK PINMUX_PIN(PIN_PA14, 1, 1) +#define PIN_PA14__TK1 PINMUX_PIN(PIN_PA14, 2, 1) +#define PIN_PA14__QSPI0_SCK PINMUX_PIN(PIN_PA14, 3, 2) +#define PIN_PA14__I2SC1_MCK PINMUX_PIN(PIN_PA14, 4, 2) +#define PIN_PA14__FLEXCOM3_IO2 PINMUX_PIN(PIN_PA14, 5, 1) +#define PIN_PA14__D9 PINMUX_PIN(PIN_PA14, 6, 2) +#define PIN_PA15 15 +#define PIN_PA15__GPIO PINMUX_PIN(PIN_PA15, 0, 0) +#define PIN_PA15__SPI0_MOSI PINMUX_PIN(PIN_PA15, 1, 1) +#define PIN_PA15__TF1 PINMUX_PIN(PIN_PA15, 2, 1) +#define PIN_PA15__QSPI0_CS PINMUX_PIN(PIN_PA15, 3, 2) +#define PIN_PA15__I2SC1_CK PINMUX_PIN(PIN_PA15, 4, 2) +#define PIN_PA15__FLEXCOM3_IO0 PINMUX_PIN(PIN_PA15, 5, 1) +#define PIN_PA15__D10 PINMUX_PIN(PIN_PA15, 6, 2) +#define PIN_PA16 16 +#define PIN_PA16__GPIO PINMUX_PIN(PIN_PA16, 0, 0) +#define PIN_PA16__SPI0_MISO PINMUX_PIN(PIN_PA16, 1, 1) +#define PIN_PA16__TD1 PINMUX_PIN(PIN_PA16, 2, 1) +#define PIN_PA16__QSPI0_IO0 PINMUX_PIN(PIN_PA16, 3, 2) +#define PIN_PA16__I2SC1_WS PINMUX_PIN(PIN_PA16, 4, 2) +#define PIN_PA16__FLEXCOM3_IO3 PINMUX_PIN(PIN_PA16, 5, 1) +#define PIN_PA16__D11 PINMUX_PIN(PIN_PA16, 6, 2) +#define PIN_PA17 17 +#define PIN_PA17__GPIO PINMUX_PIN(PIN_PA17, 0, 0) +#define PIN_PA17__SPI0_NPCS0 PINMUX_PIN(PIN_PA17, 1, 1) +#define PIN_PA17__RD1 PINMUX_PIN(PIN_PA17, 2, 1) +#define PIN_PA17__QSPI0_IO1 PINMUX_PIN(PIN_PA17, 3, 2) +#define PIN_PA17__I2SC1_DI0 PINMUX_PIN(PIN_PA17, 4, 2) +#define PIN_PA17__FLEXCOM3_IO4 PINMUX_PIN(PIN_PA17, 5, 1) +#define PIN_PA17__D12 PINMUX_PIN(PIN_PA17, 6, 2) +#define PIN_PA18 18 +#define PIN_PA18__GPIO PINMUX_PIN(PIN_PA18, 0, 0) +#define PIN_PA18__SPI0_NPCS1 PINMUX_PIN(PIN_PA18, 1, 1) +#define PIN_PA18__RK1 PINMUX_PIN(PIN_PA18, 2, 1) +#define PIN_PA18__QSPI0_IO2 PINMUX_PIN(PIN_PA18, 3, 2) +#define PIN_PA18__I2SC1_DO0 PINMUX_PIN(PIN_PA18, 4, 2) +#define PIN_PA18__SDMMC1_DAT0 PINMUX_PIN(PIN_PA18, 5, 1) +#define PIN_PA18__D13 PINMUX_PIN(PIN_PA18, 6, 2) +#define PIN_PA19 19 +#define PIN_PA19__GPIO PINMUX_PIN(PIN_PA19, 0, 0) +#define PIN_PA19__SPI0_NPCS2 PINMUX_PIN(PIN_PA19, 1, 1) +#define PIN_PA19__RF1 PINMUX_PIN(PIN_PA19, 2, 1) +#define PIN_PA19__QSPI0_IO3 PINMUX_PIN(PIN_PA19, 3, 2) +#define PIN_PA19__TIOA0 PINMUX_PIN(PIN_PA19, 4, 1) +#define PIN_PA19__SDMMC1_DAT1 PINMUX_PIN(PIN_PA19, 5, 1) +#define PIN_PA19__D14 PINMUX_PIN(PIN_PA19, 6, 2) +#define PIN_PA20 20 +#define PIN_PA20__GPIO PINMUX_PIN(PIN_PA20, 0, 0) +#define PIN_PA20__SPI0_NPCS3 PINMUX_PIN(PIN_PA20, 1, 1) +#define PIN_PA20__TIOB0 PINMUX_PIN(PIN_PA20, 4, 1) +#define PIN_PA20__SDMMC1_DAT2 PINMUX_PIN(PIN_PA20, 5, 1) +#define PIN_PA20__D15 PINMUX_PIN(PIN_PA20, 6, 2) +#define PIN_PA21 21 +#define PIN_PA21__GPIO PINMUX_PIN(PIN_PA21, 0, 0) +#define PIN_PA21__IRQ PINMUX_PIN(PIN_PA21, 1, 2) +#define PIN_PA21__PCK2 PINMUX_PIN(PIN_PA21, 2, 3) +#define PIN_PA21__TCLK0 PINMUX_PIN(PIN_PA21, 4, 1) +#define PIN_PA21__SDMMC1_DAT3 PINMUX_PIN(PIN_PA21, 5, 1) +#define PIN_PA21__NANDRDY PINMUX_PIN(PIN_PA21, 6, 2) +#define PIN_PA22 22 +#define PIN_PA22__GPIO PINMUX_PIN(PIN_PA22, 0, 0) +#define PIN_PA22__FLEXCOM1_IO2 PINMUX_PIN(PIN_PA22, 1, 1) +#define PIN_PA22__D0 PINMUX_PIN(PIN_PA22, 2, 1) +#define PIN_PA22__TCK PINMUX_PIN(PIN_PA22, 3, 4) +#define PIN_PA22__SPI1_SPCK PINMUX_PIN(PIN_PA22, 4, 2) +#define PIN_PA22__SDMMC1_CK PINMUX_PIN(PIN_PA22, 5, 1) +#define PIN_PA22__QSPI0_SCK PINMUX_PIN(PIN_PA22, 6, 3) +#define PIN_PA23 23 +#define PIN_PA23__GPIO PINMUX_PIN(PIN_PA23, 0, 0) +#define PIN_PA23__FLEXCOM1_IO1 PINMUX_PIN(PIN_PA23, 1, 1) +#define PIN_PA23__D1 PINMUX_PIN(PIN_PA23, 2, 1) +#define PIN_PA23__TDI PINMUX_PIN(PIN_PA23, 3, 4) +#define PIN_PA23__SPI1_MOSI PINMUX_PIN(PIN_PA23, 4, 2) +#define PIN_PA23__QSPI0_CS PINMUX_PIN(PIN_PA23, 6, 3) +#define PIN_PA24 24 +#define PIN_PA24__GPIO PINMUX_PIN(PIN_PA24, 0, 0) +#define PIN_PA24__FLEXCOM1_IO0 PINMUX_PIN(PIN_PA24, 1, 1) +#define PIN_PA24__D2 PINMUX_PIN(PIN_PA24, 2, 1) +#define PIN_PA24__TDO PINMUX_PIN(PIN_PA24, 3, 4) +#define PIN_PA24__SPI1_MISO PINMUX_PIN(PIN_PA24, 4, 2) +#define PIN_PA24__QSPI0_IO0 PINMUX_PIN(PIN_PA24, 6, 3) +#define PIN_PA25 25 +#define PIN_PA25__GPIO PINMUX_PIN(PIN_PA25, 0, 0) +#define PIN_PA25__FLEXCOM1_IO3 PINMUX_PIN(PIN_PA25, 1, 1) +#define PIN_PA25__D3 PINMUX_PIN(PIN_PA25, 2, 1) +#define PIN_PA25__TMS PINMUX_PIN(PIN_PA25, 3, 4) +#define PIN_PA25__SPI1_NPCS0 PINMUX_PIN(PIN_PA25, 4, 2) +#define PIN_PA25__QSPI0_IO1 PINMUX_PIN(PIN_PA25, 6, 3) +#define PIN_PA26 26 +#define PIN_PA26__GPIO PINMUX_PIN(PIN_PA26, 0, 0) +#define PIN_PA26__FLEXCOM1_IO4 PINMUX_PIN(PIN_PA26, 1, 1) +#define PIN_PA26__D4 PINMUX_PIN(PIN_PA26, 2, 1) +#define PIN_PA26__NTRST PINMUX_PIN(PIN_PA26, 3, 4) +#define PIN_PA26__SPI1_NPCS1 PINMUX_PIN(PIN_PA26, 4, 2) +#define PIN_PA26__QSPI0_IO2 PINMUX_PIN(PIN_PA26, 6, 3) +#define PIN_PA27 27 +#define PIN_PA27__GPIO PINMUX_PIN(PIN_PA27, 0, 0) +#define PIN_PA27__TIOA1 PINMUX_PIN(PIN_PA27, 1, 2) +#define PIN_PA27__D5 PINMUX_PIN(PIN_PA27, 2, 1) +#define PIN_PA27__SPI0_NPCS2 PINMUX_PIN(PIN_PA27, 3, 2) +#define PIN_PA27__SPI1_NPCS2 PINMUX_PIN(PIN_PA27, 4, 2) +#define PIN_PA27__SDMMC1_RSTN PINMUX_PIN(PIN_PA27, 5, 1) +#define PIN_PA27__QSPI0_IO3 PINMUX_PIN(PIN_PA27, 6, 3) +#define PIN_PA28 28 +#define PIN_PA28__GPIO PINMUX_PIN(PIN_PA28, 0, 0) +#define PIN_PA28__TIOB1 PINMUX_PIN(PIN_PA28, 1, 2) +#define PIN_PA28__D6 PINMUX_PIN(PIN_PA28, 2, 1) +#define PIN_PA28__SPI0_NPCS3 PINMUX_PIN(PIN_PA28, 3, 2) +#define PIN_PA28__SPI1_NPCS3 PINMUX_PIN(PIN_PA28, 4, 2) +#define PIN_PA28__SDMMC1_CMD PINMUX_PIN(PIN_PA28, 5, 1) +#define PIN_PA28__CLASSD_L0 PINMUX_PIN(PIN_PA28, 6, 1) +#define PIN_PA29 29 +#define PIN_PA29__GPIO PINMUX_PIN(PIN_PA29, 0, 0) +#define PIN_PA29__TCLK1 PINMUX_PIN(PIN_PA29, 1, 2) +#define PIN_PA29__D7 PINMUX_PIN(PIN_PA29, 2, 1) +#define PIN_PA29__SPI0_NPCS1 PINMUX_PIN(PIN_PA29, 3, 2) +#define PIN_PA29__SDMMC1_WP PINMUX_PIN(PIN_PA29, 5, 1) +#define PIN_PA29__CLASSD_L1 PINMUX_PIN(PIN_PA29, 6, 1) +#define PIN_PA30 30 +#define PIN_PA30__GPIO PINMUX_PIN(PIN_PA30, 0, 0) +#define PIN_PA30__NWE_NANDWE PINMUX_PIN(PIN_PA30, 2, 1) +#define PIN_PA30__SPI0_NPCS0 PINMUX_PIN(PIN_PA30, 3, 2) +#define PIN_PA30__PWMH0 PINMUX_PIN(PIN_PA30, 4, 1) +#define PIN_PA30__SDMMC1_CD PINMUX_PIN(PIN_PA30, 5, 1) +#define PIN_PA30__CLASSD_L2 PINMUX_PIN(PIN_PA30, 6, 1) +#define PIN_PA31 31 +#define PIN_PA31__GPIO PINMUX_PIN(PIN_PA31, 0, 0) +#define PIN_PA31__NCS3 PINMUX_PIN(PIN_PA31, 2, 1) +#define PIN_PA31__SPI0_MISO PINMUX_PIN(PIN_PA31, 3, 2) +#define PIN_PA31__PWML0 PINMUX_PIN(PIN_PA31, 4, 1) +#define PIN_PA31__CLASSD_L3 PINMUX_PIN(PIN_PA31, 6, 1) +#define PIN_PB0 32 +#define PIN_PB0__GPIO PINMUX_PIN(PIN_PB0, 0, 0) +#define PIN_PB0__A21_NANDALE PINMUX_PIN(PIN_PB0, 2, 1) +#define PIN_PB0__SPI0_MOSI PINMUX_PIN(PIN_PB0, 3, 2) +#define PIN_PB0__PWMH1 PINMUX_PIN(PIN_PB0, 4, 1) +#define PIN_PB1 33 +#define PIN_PB1__GPIO PINMUX_PIN(PIN_PB1, 0, 0) +#define PIN_PB1__A22_NANDCLE PINMUX_PIN(PIN_PB1, 2, 1) +#define PIN_PB1__SPI0_SPCK PINMUX_PIN(PIN_PB1, 3, 2) +#define PIN_PB1__PWML1 PINMUX_PIN(PIN_PB1, 4, 1) +#define PIN_PB1__CLASSD_R0 PINMUX_PIN(PIN_PB1, 6, 1) +#define PIN_PB2 34 +#define PIN_PB2__GPIO PINMUX_PIN(PIN_PB2, 0, 0) +#define PIN_PB2__NRD_NANDOE PINMUX_PIN(PIN_PB2, 2, 1) +#define PIN_PB2__PWMFI0 PINMUX_PIN(PIN_PB2, 4, 1) +#define PIN_PB2__CLASSD_R1 PINMUX_PIN(PIN_PB2, 6, 1) +#define PIN_PB3 35 +#define PIN_PB3__GPIO PINMUX_PIN(PIN_PB3, 0, 0) +#define PIN_PB3__URXD4 PINMUX_PIN(PIN_PB3, 1, 1) +#define PIN_PB3__D8 PINMUX_PIN(PIN_PB3, 2, 1) +#define PIN_PB3__IRQ PINMUX_PIN(PIN_PB3, 3, 3) +#define PIN_PB3__PWMEXTRG0 PINMUX_PIN(PIN_PB3, 4, 1) +#define PIN_PB3__CLASSD_R2 PINMUX_PIN(PIN_PB3, 6, 1) +#define PIN_PB4 36 +#define PIN_PB4__GPIO PINMUX_PIN(PIN_PB4, 0, 0) +#define PIN_PB4__UTXD4 PINMUX_PIN(PIN_PB4, 1, 1) +#define PIN_PB4__D9 PINMUX_PIN(PIN_PB4, 2, 1) +#define PIN_PB4__FIQ PINMUX_PIN(PIN_PB4, 3, 4) +#define PIN_PB4__CLASSD_R3 PINMUX_PIN(PIN_PB4, 6, 1) +#define PIN_PB5 37 +#define PIN_PB5__GPIO PINMUX_PIN(PIN_PB5, 0, 0) +#define PIN_PB5__TCLK2 PINMUX_PIN(PIN_PB5, 1, 1) +#define PIN_PB5__D10 PINMUX_PIN(PIN_PB5, 2, 1) +#define PIN_PB5__PWMH2 PINMUX_PIN(PIN_PB5, 3, 1) +#define PIN_PB5__QSPI1_SCK PINMUX_PIN(PIN_PB5, 4, 2) +#define PIN_PB5__GTSUCOMP PINMUX_PIN(PIN_PB5, 6, 3) +#define PIN_PB6 38 +#define PIN_PB6__GPIO PINMUX_PIN(PIN_PB6, 0, 0) +#define PIN_PB6__TIOA2 PINMUX_PIN(PIN_PB6, 1, 1) +#define PIN_PB6__D11 PINMUX_PIN(PIN_PB6, 2, 1) +#define PIN_PB6__PWML2 PINMUX_PIN(PIN_PB6, 3, 1) +#define PIN_PB6__QSPI1_CS PINMUX_PIN(PIN_PB6, 4, 2) +#define PIN_PB6__GTXER PINMUX_PIN(PIN_PB6, 6, 3) +#define PIN_PB7 39 +#define PIN_PB7__GPIO PINMUX_PIN(PIN_PB7, 0, 0) +#define PIN_PB7__TIOB2 PINMUX_PIN(PIN_PB7, 1, 1) +#define PIN_PB7__D12 PINMUX_PIN(PIN_PB7, 2, 1) +#define PIN_PB7__PWMH3 PINMUX_PIN(PIN_PB7, 3, 1) +#define PIN_PB7__QSPI1_IO0 PINMUX_PIN(PIN_PB7, 4, 2) +#define PIN_PB7__GRXCK PINMUX_PIN(PIN_PB7, 6, 3) +#define PIN_PB8 40 +#define PIN_PB8__GPIO PINMUX_PIN(PIN_PB8, 0, 0) +#define PIN_PB8__TCLK3 PINMUX_PIN(PIN_PB8, 1, 1) +#define PIN_PB8__D13 PINMUX_PIN(PIN_PB8, 2, 1) +#define PIN_PB8__PWML3 PINMUX_PIN(PIN_PB8, 3, 1) +#define PIN_PB8__QSPI1_IO1 PINMUX_PIN(PIN_PB8, 4, 2) +#define PIN_PB8__GCRS PINMUX_PIN(PIN_PB8, 6, 3) +#define PIN_PB9 41 +#define PIN_PB9__GPIO PINMUX_PIN(PIN_PB9, 0, 0) +#define PIN_PB9__TIOA3 PINMUX_PIN(PIN_PB9, 1, 1) +#define PIN_PB9__D14 PINMUX_PIN(PIN_PB9, 2, 1) +#define PIN_PB9__PWMFI1 PINMUX_PIN(PIN_PB9, 3, 1) +#define PIN_PB9__QSPI1_IO2 PINMUX_PIN(PIN_PB9, 4, 2) +#define PIN_PB9__GCOL PINMUX_PIN(PIN_PB9, 6, 3) +#define PIN_PB10 42 +#define PIN_PB10__GPIO PINMUX_PIN(PIN_PB10, 0, 0) +#define PIN_PB10__TIOB3 PINMUX_PIN(PIN_PB10, 1, 1) +#define PIN_PB10__D15 PINMUX_PIN(PIN_PB10, 2, 1) +#define PIN_PB10__PWMEXTRG1 PINMUX_PIN(PIN_PB10, 3, 1) +#define PIN_PB10__QSPI1_IO3 PINMUX_PIN(PIN_PB10, 4, 2) +#define PIN_PB10__GRX2 PINMUX_PIN(PIN_PB10, 6, 3) +#define PIN_PB11 43 +#define PIN_PB11__GPIO PINMUX_PIN(PIN_PB11, 0, 0) +#define PIN_PB11__LCDDAT0 PINMUX_PIN(PIN_PB11, 1, 1) +#define PIN_PB11__A0_NBS0 PINMUX_PIN(PIN_PB11, 2, 1) +#define PIN_PB11__URXD3 PINMUX_PIN(PIN_PB11, 3, 3) +#define PIN_PB11__PDMIC_DAT PINMUX_PIN(PIN_PB11, 4, 2) +#define PIN_PB11__GRX3 PINMUX_PIN(PIN_PB11, 6, 3) +#define PIN_PB12 44 +#define PIN_PB12__GPIO PINMUX_PIN(PIN_PB12, 0, 0) +#define PIN_PB12__LCDDAT1 PINMUX_PIN(PIN_PB12, 1, 1) +#define PIN_PB12__A1 PINMUX_PIN(PIN_PB12, 2, 1) +#define PIN_PB12__UTXD3 PINMUX_PIN(PIN_PB12, 3, 3) +#define PIN_PB12__PDMIC_CLK PINMUX_PIN(PIN_PB12, 4, 2) +#define PIN_PB12__GTX2 PINMUX_PIN(PIN_PB12, 6, 3) +#define PIN_PB13 45 +#define PIN_PB13__GPIO PINMUX_PIN(PIN_PB13, 0, 0) +#define PIN_PB13__LCDDAT2 PINMUX_PIN(PIN_PB13, 1, 1) +#define PIN_PB13__A2 PINMUX_PIN(PIN_PB13, 2, 1) +#define PIN_PB13__PCK1 PINMUX_PIN(PIN_PB13, 3, 3) +#define PIN_PB13__GTX3 PINMUX_PIN(PIN_PB13, 6, 3) +#define PIN_PB14 46 +#define PIN_PB14__GPIO PINMUX_PIN(PIN_PB14, 0, 0) +#define PIN_PB14__LCDDAT3 PINMUX_PIN(PIN_PB14, 1, 1) +#define PIN_PB14__A3 PINMUX_PIN(PIN_PB14, 2, 1) +#define PIN_PB14__TK1 PINMUX_PIN(PIN_PB14, 3, 2) +#define PIN_PB14__I2SC1_MCK PINMUX_PIN(PIN_PB14, 4, 1) +#define PIN_PB14__QSPI1_SCK PINMUX_PIN(PIN_PB14, 5, 3) +#define PIN_PB14__GTXCK PINMUX_PIN(PIN_PB14, 6, 3) +#define PIN_PB15 47 +#define PIN_PB15__GPIO PINMUX_PIN(PIN_PB15, 0, 0) +#define PIN_PB15__LCDDAT4 PINMUX_PIN(PIN_PB15, 1, 1) +#define PIN_PB15__A4 PINMUX_PIN(PIN_PB15, 2, 1) +#define PIN_PB15__TF1 PINMUX_PIN(PIN_PB15, 3, 2) +#define PIN_PB15__I2SC1_CK PINMUX_PIN(PIN_PB15, 4, 1) +#define PIN_PB15__QSPI1_CS PINMUX_PIN(PIN_PB15, 5, 3) +#define PIN_PB15__GTXEN PINMUX_PIN(PIN_PB15, 6, 3) +#define PIN_PB16 48 +#define PIN_PB16__GPIO PINMUX_PIN(PIN_PB16, 0, 0) +#define PIN_PB16__LCDDAT5 PINMUX_PIN(PIN_PB16, 1, 1) +#define PIN_PB16__A5 PINMUX_PIN(PIN_PB16, 2, 1) +#define PIN_PB16__TD1 PINMUX_PIN(PIN_PB16, 3, 2) +#define PIN_PB16__I2SC1_WS PINMUX_PIN(PIN_PB16, 4, 1) +#define PIN_PB16__QSPI1_IO0 PINMUX_PIN(PIN_PB16, 5, 3) +#define PIN_PB16__GRXDV PINMUX_PIN(PIN_PB16, 6, 3) +#define PIN_PB17 49 +#define PIN_PB17__GPIO PINMUX_PIN(PIN_PB17, 0, 0) +#define PIN_PB17__LCDDAT6 PINMUX_PIN(PIN_PB17, 1, 1) +#define PIN_PB17__A6 PINMUX_PIN(PIN_PB17, 2, 1) +#define PIN_PB17__RD1 PINMUX_PIN(PIN_PB17, 3, 2) +#define PIN_PB17__I2SC1_DI0 PINMUX_PIN(PIN_PB17, 4, 1) +#define PIN_PB17__QSPI1_IO1 PINMUX_PIN(PIN_PB17, 5, 3) +#define PIN_PB17__GRXER PINMUX_PIN(PIN_PB17, 6, 3) +#define PIN_PB18 50 +#define PIN_PB18__GPIO PINMUX_PIN(PIN_PB18, 0, 0) +#define PIN_PB18__LCDDAT7 PINMUX_PIN(PIN_PB18, 1, 1) +#define PIN_PB18__A7 PINMUX_PIN(PIN_PB18, 2, 1) +#define PIN_PB18__RK1 PINMUX_PIN(PIN_PB18, 3, 2) +#define PIN_PB18__I2SC1_DO0 PINMUX_PIN(PIN_PB18, 4, 1) +#define PIN_PB18__QSPI1_IO2 PINMUX_PIN(PIN_PB18, 5, 3) +#define PIN_PB18__GRX0 PINMUX_PIN(PIN_PB18, 6, 3) +#define PIN_PB19 51 +#define PIN_PB19__GPIO PINMUX_PIN(PIN_PB19, 0, 0) +#define PIN_PB19__LCDDAT8 PINMUX_PIN(PIN_PB19, 1, 1) +#define PIN_PB19__A8 PINMUX_PIN(PIN_PB19, 2, 1) +#define PIN_PB19__RF1 PINMUX_PIN(PIN_PB19, 3, 2) +#define PIN_PB19__TIOA3 PINMUX_PIN(PIN_PB19, 4, 2) +#define PIN_PB19__QSPI1_IO3 PINMUX_PIN(PIN_PB19, 5, 3) +#define PIN_PB19__GRX1 PINMUX_PIN(PIN_PB19, 6, 3) +#define PIN_PB20 52 +#define PIN_PB20__GPIO PINMUX_PIN(PIN_PB20, 0, 0) +#define PIN_PB20__LCDDAT9 PINMUX_PIN(PIN_PB20, 1, 1) +#define PIN_PB20__A9 PINMUX_PIN(PIN_PB20, 2, 1) +#define PIN_PB20__TK0 PINMUX_PIN(PIN_PB20, 3, 1) +#define PIN_PB20__TIOB3 PINMUX_PIN(PIN_PB20, 4, 2) +#define PIN_PB20__PCK1 PINMUX_PIN(PIN_PB20, 5, 4) +#define PIN_PB20__GTX0 PINMUX_PIN(PIN_PB20, 6, 3) +#define PIN_PB21 53 +#define PIN_PB21__GPIO PINMUX_PIN(PIN_PB21, 0, 0) +#define PIN_PB21__LCDDAT10 PINMUX_PIN(PIN_PB21, 1, 1) +#define PIN_PB21__A10 PINMUX_PIN(PIN_PB21, 2, 1) +#define PIN_PB21__TF0 PINMUX_PIN(PIN_PB21, 3, 1) +#define PIN_PB21__TCLK3 PINMUX_PIN(PIN_PB21, 4, 2) +#define PIN_PB21__FLEXCOM3_IO2 PINMUX_PIN(PIN_PB21, 5, 3) +#define PIN_PB21__GTX1 PINMUX_PIN(PIN_PB21, 6, 3) +#define PIN_PB22 54 +#define PIN_PB22__GPIO PINMUX_PIN(PIN_PB22, 0, 0) +#define PIN_PB22__LCDDAT11 PINMUX_PIN(PIN_PB22, 1, 1) +#define PIN_PB22__A11 PINMUX_PIN(PIN_PB22, 2, 1) +#define PIN_PB22__TD0 PINMUX_PIN(PIN_PB22, 3, 1) +#define PIN_PB22__TIOA2 PINMUX_PIN(PIN_PB22, 4, 2) +#define PIN_PB22__FLEXCOM3_IO1 PINMUX_PIN(PIN_PB22, 5, 3) +#define PIN_PB22__GMDC PINMUX_PIN(PIN_PB22, 6, 3) +#define PIN_PB23 55 +#define PIN_PB23__GPIO PINMUX_PIN(PIN_PB23, 0, 0) +#define PIN_PB23__LCDDAT12 PINMUX_PIN(PIN_PB23, 1, 1) +#define PIN_PB23__A12 PINMUX_PIN(PIN_PB23, 2, 1) +#define PIN_PB23__RD0 PINMUX_PIN(PIN_PB23, 3, 1) +#define PIN_PB23__TIOB2 PINMUX_PIN(PIN_PB23, 4, 2) +#define PIN_PB23__FLEXCOM3_IO0 PINMUX_PIN(PIN_PB23, 5, 3) +#define PIN_PB23__GMDIO PINMUX_PIN(PIN_PB23, 6, 3) +#define PIN_PB24 56 +#define PIN_PB24__GPIO PINMUX_PIN(PIN_PB24, 0, 0) +#define PIN_PB24__LCDDAT13 PINMUX_PIN(PIN_PB24, 1, 1) +#define PIN_PB24__A13 PINMUX_PIN(PIN_PB24, 2, 1) +#define PIN_PB24__RK0 PINMUX_PIN(PIN_PB24, 3, 1) +#define PIN_PB24__TCLK2 PINMUX_PIN(PIN_PB24, 4, 2) +#define PIN_PB24__FLEXCOM3_IO3 PINMUX_PIN(PIN_PB24, 5, 3) +#define PIN_PB24__ISC_D10 PINMUX_PIN(PIN_PB24, 6, 3) +#define PIN_PB25 57 +#define PIN_PB25__GPIO PINMUX_PIN(PIN_PB25, 0, 0) +#define PIN_PB25__LCDDAT14 PINMUX_PIN(PIN_PB25, 1, 1) +#define PIN_PB25__A14 PINMUX_PIN(PIN_PB25, 2, 1) +#define PIN_PB25__RF0 PINMUX_PIN(PIN_PB25, 3, 1) +#define PIN_PB25__FLEXCOM3_IO4 PINMUX_PIN(PIN_PB25, 5, 3) +#define PIN_PB25__ISC_D11 PINMUX_PIN(PIN_PB25, 6, 3) +#define PIN_PB26 58 +#define PIN_PB26__GPIO PINMUX_PIN(PIN_PB26, 0, 0) +#define PIN_PB26__LCDDAT15 PINMUX_PIN(PIN_PB26, 1, 1) +#define PIN_PB26__A15 PINMUX_PIN(PIN_PB26, 2, 1) +#define PIN_PB26__URXD0 PINMUX_PIN(PIN_PB26, 3, 1) +#define PIN_PB26__PDMIC_DAT PINMUX_PIN(PIN_PB26, 4, 1) +#define PIN_PB26__ISC_D0 PINMUX_PIN(PIN_PB26, 6, 3) +#define PIN_PB27 59 +#define PIN_PB27__GPIO PINMUX_PIN(PIN_PB27, 0, 0) +#define PIN_PB27__LCDDAT16 PINMUX_PIN(PIN_PB27, 1, 1) +#define PIN_PB27__A16 PINMUX_PIN(PIN_PB27, 2, 1) +#define PIN_PB27__UTXD0 PINMUX_PIN(PIN_PB27, 3, 1) +#define PIN_PB27__PDMIC_CLK PINMUX_PIN(PIN_PB27, 4, 1) +#define PIN_PB27__ISC_D1 PINMUX_PIN(PIN_PB27, 6, 3) +#define PIN_PB28 60 +#define PIN_PB28__GPIO PINMUX_PIN(PIN_PB28, 0, 0) +#define PIN_PB28__LCDDAT17 PINMUX_PIN(PIN_PB28, 1, 1) +#define PIN_PB28__A17 PINMUX_PIN(PIN_PB28, 2, 1) +#define PIN_PB28__FLEXCOM0_IO0 PINMUX_PIN(PIN_PB28, 3, 1) +#define PIN_PB28__TIOA5 PINMUX_PIN(PIN_PB28, 4, 2) +#define PIN_PB28__ISC_D2 PINMUX_PIN(PIN_PB28, 6, 3) +#define PIN_PB29 61 +#define PIN_PB29__GPIO PINMUX_PIN(PIN_PB29, 0, 0) +#define PIN_PB29__LCDDAT18 PINMUX_PIN(PIN_PB29, 1, 1) +#define PIN_PB29__A18 PINMUX_PIN(PIN_PB29, 2, 1) +#define PIN_PB29__FLEXCOM0_IO1 PINMUX_PIN(PIN_PB29, 3, 1) +#define PIN_PB29__TIOB5 PINMUX_PIN(PIN_PB29, 4, 2) +#define PIN_PB29__ISC_D3 PINMUX_PIN(PIN_PB29, 7, 3) +#define PIN_PB30 62 +#define PIN_PB30__GPIO PINMUX_PIN(PIN_PB30, 0, 0) +#define PIN_PB30__LCDDAT19 PINMUX_PIN(PIN_PB30, 1, 1) +#define PIN_PB30__A19 PINMUX_PIN(PIN_PB30, 2, 1) +#define PIN_PB30__FLEXCOM0_IO2 PINMUX_PIN(PIN_PB30, 3, 1) +#define PIN_PB30__TCLK5 PINMUX_PIN(PIN_PB30, 4, 2) +#define PIN_PB30__ISC_D4 PINMUX_PIN(PIN_PB30, 6, 3) +#define PIN_PB31 63 +#define PIN_PB31__GPIO PINMUX_PIN(PIN_PB31, 0, 0) +#define PIN_PB31__LCDDAT20 PINMUX_PIN(PIN_PB31, 1, 1) +#define PIN_PB31__A20 PINMUX_PIN(PIN_PB31, 2, 1) +#define PIN_PB31__FLEXCOM0_IO3 PINMUX_PIN(PIN_PB31, 3, 1) +#define PIN_PB31__TWD0 PINMUX_PIN(PIN_PB31, 4, 1) +#define PIN_PB31__ISC_D5 PINMUX_PIN(PIN_PB31, 6, 3) +#define PIN_PC0 64 +#define PIN_PC0__GPIO PINMUX_PIN(PIN_PC0, 0, 0) +#define PIN_PC0__LCDDAT21 PINMUX_PIN(PIN_PC0, 1, 1) +#define PIN_PC0__A23 PINMUX_PIN(PIN_PC0, 2, 1) +#define PIN_PC0__FLEXCOM0_IO4 PINMUX_PIN(PIN_PC0, 3, 1) +#define PIN_PC0__TWCK0 PINMUX_PIN(PIN_PC0, 4, 1) +#define PIN_PC0__ISC_D6 PINMUX_PIN(PIN_PC0, 6, 3) +#define PIN_PC1 65 +#define PIN_PC1__GPIO PINMUX_PIN(PIN_PC1, 0, 0) +#define PIN_PC1__LCDDAT22 PINMUX_PIN(PIN_PC1, 1, 1) +#define PIN_PC1__A24 PINMUX_PIN(PIN_PC1, 2, 1) +#define PIN_PC1__CANTX0 PINMUX_PIN(PIN_PC1, 3, 1) +#define PIN_PC1__SPI1_SPCK PINMUX_PIN(PIN_PC1, 4, 1) +#define PIN_PC1__I2SC0_CK PINMUX_PIN(PIN_PC1, 5, 1) +#define PIN_PC1__ISC_D7 PINMUX_PIN(PIN_PC1, 6, 3) +#define PIN_PC2 66 +#define PIN_PC2__GPIO PINMUX_PIN(PIN_PC2, 0, 0) +#define PIN_PC2__LCDDAT23 PINMUX_PIN(PIN_PC2, 1, 1) +#define PIN_PC2__A25 PINMUX_PIN(PIN_PC2, 2, 1) +#define PIN_PC2__CANRX0 PINMUX_PIN(PIN_PC2, 3, 1) +#define PIN_PC2__SPI1_MOSI PINMUX_PIN(PIN_PC2, 4, 1) +#define PIN_PC2__I2SC0_MCK PINMUX_PIN(PIN_PC2, 5, 1) +#define PIN_PC2__ISC_D8 PINMUX_PIN(PIN_PC2, 6, 3) +#define PIN_PC3 67 +#define PIN_PC3__GPIO PINMUX_PIN(PIN_PC3, 0, 0) +#define PIN_PC3__LCDPWM PINMUX_PIN(PIN_PC3, 1, 1) +#define PIN_PC3__NWAIT PINMUX_PIN(PIN_PC3, 2, 1) +#define PIN_PC3__TIOA1 PINMUX_PIN(PIN_PC3, 3, 1) +#define PIN_PC3__SPI1_MISO PINMUX_PIN(PIN_PC3, 4, 1) +#define PIN_PC3__I2SC0_WS PINMUX_PIN(PIN_PC3, 5, 1) +#define PIN_PC3__ISC_D9 PINMUX_PIN(PIN_PC3, 6, 3) +#define PIN_PC4 68 +#define PIN_PC4__GPIO PINMUX_PIN(PIN_PC4, 0, 0) +#define PIN_PC4__LCDDISP PINMUX_PIN(PIN_PC4, 1, 1) +#define PIN_PC4__NWR1_NBS1 PINMUX_PIN(PIN_PC4, 2, 1) +#define PIN_PC4__TIOB1 PINMUX_PIN(PIN_PC4, 3, 1) +#define PIN_PC4__SPI1_NPCS0 PINMUX_PIN(PIN_PC4, 4, 1) +#define PIN_PC4__I2SC0_DI0 PINMUX_PIN(PIN_PC4, 5, 1) +#define PIN_PC4__ISC_PCK PINMUX_PIN(PIN_PC4, 6, 3) +#define PIN_PC5 69 +#define PIN_PC5__GPIO PINMUX_PIN(PIN_PC5, 0, 0) +#define PIN_PC5__LCDVSYNC PINMUX_PIN(PIN_PC5, 1, 1) +#define PIN_PC5__NCS0 PINMUX_PIN(PIN_PC5, 2, 1) +#define PIN_PC5__TCLK1 PINMUX_PIN(PIN_PC5, 3, 1) +#define PIN_PC5__SPI1_NPCS1 PINMUX_PIN(PIN_PC5, 4, 1) +#define PIN_PC5__I2SC0_DO0 PINMUX_PIN(PIN_PC5, 5, 1) +#define PIN_PC5__ISC_VSYNC PINMUX_PIN(PIN_PC5, 6, 3) +#define PIN_PC6 70 +#define PIN_PC6__GPIO PINMUX_PIN(PIN_PC6, 0, 0) +#define PIN_PC6__LCDHSYNC PINMUX_PIN(PIN_PC6, 1, 1) +#define PIN_PC6__NCS1 PINMUX_PIN(PIN_PC6, 2, 1) +#define PIN_PC6__TWD1 PINMUX_PIN(PIN_PC6, 3, 1) +#define PIN_PC6__SPI1_NPCS2 PINMUX_PIN(PIN_PC6, 4, 1) +#define PIN_PC6__ISC_HSYNC PINMUX_PIN(PIN_PC6, 6, 3) +#define PIN_PC7 71 +#define PIN_PC7__GPIO PINMUX_PIN(PIN_PC7, 0, 0) +#define PIN_PC7__LCDPCK PINMUX_PIN(PIN_PC7, 1, 1) +#define PIN_PC7__NCS2 PINMUX_PIN(PIN_PC7, 2, 1) +#define PIN_PC7__TWCK1 PINMUX_PIN(PIN_PC7, 3, 1) +#define PIN_PC7__SPI1_NPCS3 PINMUX_PIN(PIN_PC7, 4, 1) +#define PIN_PC7__URXD1 PINMUX_PIN(PIN_PC7, 5, 2) +#define PIN_PC7__ISC_MCK PINMUX_PIN(PIN_PC7, 6, 3) +#define PIN_PC8 72 +#define PIN_PC8__GPIO PINMUX_PIN(PIN_PC8, 0, 0) +#define PIN_PC8__LCDDEN PINMUX_PIN(PIN_PC8, 1, 1) +#define PIN_PC8__NANDRDY PINMUX_PIN(PIN_PC8, 2, 1) +#define PIN_PC8__FIQ PINMUX_PIN(PIN_PC8, 3, 1) +#define PIN_PC8__PCK0 PINMUX_PIN(PIN_PC8, 4, 3) +#define PIN_PC8__UTXD1 PINMUX_PIN(PIN_PC8, 5, 2) +#define PIN_PC8__ISC_FIELD PINMUX_PIN(PIN_PC8, 6, 3) +#define PIN_PC9 73 +#define PIN_PC9__GPIO PINMUX_PIN(PIN_PC9, 0, 0) +#define PIN_PC9__FIQ PINMUX_PIN(PIN_PC9, 1, 3) +#define PIN_PC9__GTSUCOMP PINMUX_PIN(PIN_PC9, 2, 1) +#define PIN_PC9__ISC_D0 PINMUX_PIN(PIN_PC9, 3, 1) +#define PIN_PC9__TIOA4 PINMUX_PIN(PIN_PC9, 4, 2) +#define PIN_PC10 74 +#define PIN_PC10__GPIO PINMUX_PIN(PIN_PC10, 0, 0) +#define PIN_PC10__LCDDAT2 PINMUX_PIN(PIN_PC10, 1, 2) +#define PIN_PC10__GTXCK PINMUX_PIN(PIN_PC10, 2, 1) +#define PIN_PC10__ISC_D1 PINMUX_PIN(PIN_PC10, 3, 1) +#define PIN_PC10__TIOB4 PINMUX_PIN(PIN_PC10, 4, 2) +#define PIN_PC10__CANTX0 PINMUX_PIN(PIN_PC10, 5, 2) +#define PIN_PC11 75 +#define PIN_PC11__GPIO PINMUX_PIN(PIN_PC11, 0, 0) +#define PIN_PC11__LCDDAT3 PINMUX_PIN(PIN_PC11, 1, 2) +#define PIN_PC11__GTXEN PINMUX_PIN(PIN_PC11, 2, 1) +#define PIN_PC11__ISC_D2 PINMUX_PIN(PIN_PC11, 3, 1) +#define PIN_PC11__TCLK4 PINMUX_PIN(PIN_PC11, 4, 2) +#define PIN_PC11__CANRX0 PINMUX_PIN(PIN_PC11, 5, 2) +#define PIN_PC11__A0_NBS0 PINMUX_PIN(PIN_PC11, 6, 2) +#define PIN_PC12 76 +#define PIN_PC12__GPIO PINMUX_PIN(PIN_PC12, 0, 0) +#define PIN_PC12__LCDDAT4 PINMUX_PIN(PIN_PC12, 1, 2) +#define PIN_PC12__GRXDV PINMUX_PIN(PIN_PC12, 2, 1) +#define PIN_PC12__ISC_D3 PINMUX_PIN(PIN_PC12, 3, 1) +#define PIN_PC12__URXD3 PINMUX_PIN(PIN_PC12, 4, 1) +#define PIN_PC12__TK0 PINMUX_PIN(PIN_PC12, 5, 2) +#define PIN_PC12__A1 PINMUX_PIN(PIN_PC12, 6, 2) +#define PIN_PC13 77 +#define PIN_PC13__GPIO PINMUX_PIN(PIN_PC13, 0, 0) +#define PIN_PC13__LCDDAT5 PINMUX_PIN(PIN_PC13, 1, 2) +#define PIN_PC13__GRXER PINMUX_PIN(PIN_PC13, 2, 1) +#define PIN_PC13__ISC_D4 PINMUX_PIN(PIN_PC13, 3, 1) +#define PIN_PC13__UTXD3 PINMUX_PIN(PIN_PC13, 4, 1) +#define PIN_PC13__TF0 PINMUX_PIN(PIN_PC13, 5, 2) +#define PIN_PC13__A2 PINMUX_PIN(PIN_PC13, 6, 2) +#define PIN_PC14 78 +#define PIN_PC14__GPIO PINMUX_PIN(PIN_PC14, 0, 0) +#define PIN_PC14__LCDDAT6 PINMUX_PIN(PIN_PC14, 1, 2) +#define PIN_PC14__GRX0 PINMUX_PIN(PIN_PC14, 2, 1) +#define PIN_PC14__ISC_D5 PINMUX_PIN(PIN_PC14, 3, 1) +#define PIN_PC14__TD0 PINMUX_PIN(PIN_PC14, 5, 2) +#define PIN_PC14__A3 PINMUX_PIN(PIN_PC14, 6, 2) +#define PIN_PC15 79 +#define PIN_PC15__GPIO PINMUX_PIN(PIN_PC15, 0, 0) +#define PIN_PC15__LCDDAT7 PINMUX_PIN(PIN_PC15, 1, 2) +#define PIN_PC15__GRX1 PINMUX_PIN(PIN_PC15, 2, 1) +#define PIN_PC15__ISC_D6 PINMUX_PIN(PIN_PC15, 3, 1) +#define PIN_PC15__RD0 PINMUX_PIN(PIN_PC15, 5, 2) +#define PIN_PC15__A4 PINMUX_PIN(PIN_PC15, 6, 2) +#define PIN_PC16 80 +#define PIN_PC16__GPIO PINMUX_PIN(PIN_PC16, 0, 0) +#define PIN_PC16__LCDDAT10 PINMUX_PIN(PIN_PC16, 1, 2) +#define PIN_PC16__GTX0 PINMUX_PIN(PIN_PC16, 2, 1) +#define PIN_PC16__ISC_D7 PINMUX_PIN(PIN_PC16, 3, 1) +#define PIN_PC16__RK0 PINMUX_PIN(PIN_PC16, 5, 2) +#define PIN_PC16__A5 PINMUX_PIN(PIN_PC16, 6, 2) +#define PIN_PC17 81 +#define PIN_PC17__GPIO PINMUX_PIN(PIN_PC17, 0, 0) +#define PIN_PC17__LCDDAT11 PINMUX_PIN(PIN_PC17, 1, 2) +#define PIN_PC17__GTX1 PINMUX_PIN(PIN_PC17, 2, 1) +#define PIN_PC17__ISC_D8 PINMUX_PIN(PIN_PC17, 3, 1) +#define PIN_PC17__RF0 PINMUX_PIN(PIN_PC17, 5, 2) +#define PIN_PC17__A6 PINMUX_PIN(PIN_PC17, 6, 2) +#define PIN_PC18 82 +#define PIN_PC18__GPIO PINMUX_PIN(PIN_PC18, 0, 0) +#define PIN_PC18__LCDDAT12 PINMUX_PIN(PIN_PC18, 1, 2) +#define PIN_PC18__GMDC PINMUX_PIN(PIN_PC18, 2, 1) +#define PIN_PC18__ISC_D9 PINMUX_PIN(PIN_PC18, 3, 1) +#define PIN_PC18__FLEXCOM3_IO2 PINMUX_PIN(PIN_PC18, 5, 2) +#define PIN_PC18__A7 PINMUX_PIN(PIN_PC18, 6, 2) +#define PIN_PC19 83 +#define PIN_PC19__GPIO PINMUX_PIN(PIN_PC19, 0, 0) +#define PIN_PC19__LCDDAT13 PINMUX_PIN(PIN_PC19, 1, 2) +#define PIN_PC19__GMDIO PINMUX_PIN(PIN_PC19, 2, 1) +#define PIN_PC19__ISC_D10 PINMUX_PIN(PIN_PC19, 3, 1) +#define PIN_PC19__FLEXCOM3_IO1 PINMUX_PIN(PIN_PC19, 5, 2) +#define PIN_PC19__A8 PINMUX_PIN(PIN_PC19, 6, 2) +#define PIN_PC20 84 +#define PIN_PC20__GPIO PINMUX_PIN(PIN_PC20, 0, 0) +#define PIN_PC20__LCDDAT14 PINMUX_PIN(PIN_PC20, 1, 2) +#define PIN_PC20__GRXCK PINMUX_PIN(PIN_PC20, 2, 1) +#define PIN_PC20__ISC_D11 PINMUX_PIN(PIN_PC20, 3, 1) +#define PIN_PC20__FLEXCOM3_IO0 PINMUX_PIN(PIN_PC20, 5, 2) +#define PIN_PC20__A9 PINMUX_PIN(PIN_PC20, 6, 2) +#define PIN_PC21 85 +#define PIN_PC21__GPIO PINMUX_PIN(PIN_PC21, 0, 0) +#define PIN_PC21__LCDDAT15 PINMUX_PIN(PIN_PC21, 1, 2) +#define PIN_PC21__GTXER PINMUX_PIN(PIN_PC21, 2, 1) +#define PIN_PC21__ISC_PCK PINMUX_PIN(PIN_PC21, 3, 1) +#define PIN_PC21__FLEXCOM3_IO3 PINMUX_PIN(PIN_PC21, 5, 2) +#define PIN_PC21__A10 PINMUX_PIN(PIN_PC21, 6, 2) +#define PIN_PC22 86 +#define PIN_PC22__GPIO PINMUX_PIN(PIN_PC22, 0, 0) +#define PIN_PC22__LCDDAT18 PINMUX_PIN(PIN_PC22, 1, 2) +#define PIN_PC22__GCRS PINMUX_PIN(PIN_PC22, 2, 1) +#define PIN_PC22__ISC_VSYNC PINMUX_PIN(PIN_PC22, 3, 1) +#define PIN_PC22__FLEXCOM3_IO4 PINMUX_PIN(PIN_PC22, 5, 2) +#define PIN_PC22__A11 PINMUX_PIN(PIN_PC22, 6, 2) +#define PIN_PC23 87 +#define PIN_PC23__GPIO PINMUX_PIN(PIN_PC23, 0, 0) +#define PIN_PC23__LCDDAT19 PINMUX_PIN(PIN_PC23, 1, 2) +#define PIN_PC23__GCOL PINMUX_PIN(PIN_PC23, 2, 1) +#define PIN_PC23__ISC_HSYNC PINMUX_PIN(PIN_PC23, 3, 1) +#define PIN_PC23__A12 PINMUX_PIN(PIN_PC23, 6, 2) +#define PIN_PC24 88 +#define PIN_PC24__GPIO PINMUX_PIN(PIN_PC24, 0, 0) +#define PIN_PC24__LCDDAT20 PINMUX_PIN(PIN_PC24, 1, 2) +#define PIN_PC24__GRX2 PINMUX_PIN(PIN_PC24, 2, 1) +#define PIN_PC24__ISC_MCK PINMUX_PIN(PIN_PC24, 3, 1) +#define PIN_PC24__A13 PINMUX_PIN(PIN_PC24, 6, 2) +#define PIN_PC25 89 +#define PIN_PC25__GPIO PINMUX_PIN(PIN_PC25, 0, 0) +#define PIN_PC25__LCDDAT21 PINMUX_PIN(PIN_PC25, 1, 2) +#define PIN_PC25__GRX3 PINMUX_PIN(PIN_PC25, 2, 1) +#define PIN_PC25__ISC_FIELD PINMUX_PIN(PIN_PC25, 3, 1) +#define PIN_PC25__A14 PINMUX_PIN(PIN_PC25, 6, 2) +#define PIN_PC26 90 +#define PIN_PC26__GPIO PINMUX_PIN(PIN_PC26, 0, 0) +#define PIN_PC26__LCDDAT22 PINMUX_PIN(PIN_PC26, 1, 2) +#define PIN_PC26__GTX2 PINMUX_PIN(PIN_PC26, 2, 1) +#define PIN_PC26__CANTX1 PINMUX_PIN(PIN_PC26, 4, 1) +#define PIN_PC26__A15 PINMUX_PIN(PIN_PC26, 6, 2) +#define PIN_PC27 91 +#define PIN_PC27__GPIO PINMUX_PIN(PIN_PC27, 0, 0) +#define PIN_PC27__LCDDAT23 PINMUX_PIN(PIN_PC27, 1, 2) +#define PIN_PC27__GTX3 PINMUX_PIN(PIN_PC27, 2, 1) +#define PIN_PC27__PCK1 PINMUX_PIN(PIN_PC27, 3, 2) +#define PIN_PC27__CANRX1 PINMUX_PIN(PIN_PC27, 4, 1) +#define PIN_PC27__TWD0 PINMUX_PIN(PIN_PC27, 5, 2) +#define PIN_PC27__A16 PINMUX_PIN(PIN_PC27, 6, 2) +#define PIN_PC28 92 +#define PIN_PC28__GPIO PINMUX_PIN(PIN_PC28, 0, 0) +#define PIN_PC28__LCDPWM PINMUX_PIN(PIN_PC28, 1, 2) +#define PIN_PC28__FLEXCOM4_IO0 PINMUX_PIN(PIN_PC28, 2, 1) +#define PIN_PC28__PCK2 PINMUX_PIN(PIN_PC28, 3, 2) +#define PIN_PC28__TWCK0 PINMUX_PIN(PIN_PC28, 5, 2) +#define PIN_PC28__A17 PINMUX_PIN(PIN_PC28, 6, 2) +#define PIN_PC29 93 +#define PIN_PC29__GPIO PINMUX_PIN(PIN_PC29, 0, 0) +#define PIN_PC29__LCDDISP PINMUX_PIN(PIN_PC29, 1, 2) +#define PIN_PC29__FLEXCOM4_IO1 PINMUX_PIN(PIN_PC29, 2, 1) +#define PIN_PC29__A18 PINMUX_PIN(PIN_PC29, 6, 2) +#define PIN_PC30 94 +#define PIN_PC30__GPIO PINMUX_PIN(PIN_PC30, 0, 0) +#define PIN_PC30__LCDVSYNC PINMUX_PIN(PIN_PC30, 1, 2) +#define PIN_PC30__FLEXCOM4_IO2 PINMUX_PIN(PIN_PC30, 2, 1) +#define PIN_PC30__A19 PINMUX_PIN(PIN_PC30, 6, 2) +#define PIN_PC31 95 +#define PIN_PC31__GPIO PINMUX_PIN(PIN_PC31, 0, 0) +#define PIN_PC31__LCDHSYNC PINMUX_PIN(PIN_PC31, 1, 2) +#define PIN_PC31__FLEXCOM4_IO3 PINMUX_PIN(PIN_PC31, 2, 1) +#define PIN_PC31__URXD3 PINMUX_PIN(PIN_PC31, 3, 2) +#define PIN_PC31__A20 PINMUX_PIN(PIN_PC31, 6, 2) +#define PIN_PD0 96 +#define PIN_PD0__GPIO PINMUX_PIN(PIN_PD0, 0, 0) +#define PIN_PD0__LCDPCK PINMUX_PIN(PIN_PD0, 1, 2) +#define PIN_PD0__FLEXCOM4_IO4 PINMUX_PIN(PIN_PD0, 2, 1) +#define PIN_PD0__UTXD3 PINMUX_PIN(PIN_PD0, 3, 2) +#define PIN_PD0__GTSUCOMP PINMUX_PIN(PIN_PD0, 4, 2) +#define PIN_PD0__A23 PINMUX_PIN(PIN_PD0, 6, 2) +#define PIN_PD1 97 +#define PIN_PD1__GPIO PINMUX_PIN(PIN_PD1, 0, 0) +#define PIN_PD1__LCDDEN PINMUX_PIN(PIN_PD1, 1, 2) +#define PIN_PD1__GRXCK PINMUX_PIN(PIN_PD1, 4, 2) +#define PIN_PD1__A24 PINMUX_PIN(PIN_PD1, 6, 2) +#define PIN_PD2 98 +#define PIN_PD2__GPIO PINMUX_PIN(PIN_PD2, 0, 0) +#define PIN_PD2__URXD1 PINMUX_PIN(PIN_PD2, 1, 1) +#define PIN_PD2__GTXER PINMUX_PIN(PIN_PD2, 4, 2) +#define PIN_PD2__ISC_MCK PINMUX_PIN(PIN_PD2, 5, 2) +#define PIN_PD2__A25 PINMUX_PIN(PIN_PD2, 6, 2) +#define PIN_PD3 99 +#define PIN_PD3__GPIO PINMUX_PIN(PIN_PD3, 0, 0) +#define PIN_PD3__UTXD1 PINMUX_PIN(PIN_PD3, 1, 1) +#define PIN_PD3__FIQ PINMUX_PIN(PIN_PD3, 2, 2) +#define PIN_PD3__GCRS PINMUX_PIN(PIN_PD3, 4, 2) +#define PIN_PD3__ISC_D11 PINMUX_PIN(PIN_PD3, 5, 2) +#define PIN_PD3__NWAIT PINMUX_PIN(PIN_PD3, 6, 2) +#define PIN_PD4 100 +#define PIN_PD4__GPIO PINMUX_PIN(PIN_PD4, 0, 0) +#define PIN_PD4__TWD1 PINMUX_PIN(PIN_PD4, 1, 2) +#define PIN_PD4__URXD2 PINMUX_PIN(PIN_PD4, 2, 1) +#define PIN_PD4__GCOL PINMUX_PIN(PIN_PD4, 4, 2) +#define PIN_PD4__ISC_D10 PINMUX_PIN(PIN_PD4, 5, 2) +#define PIN_PD4__NCS0 PINMUX_PIN(PIN_PD4, 6, 2) +#define PIN_PD5 101 +#define PIN_PD5__GPIO PINMUX_PIN(PIN_PD5, 0, 0) +#define PIN_PD5__TWCK1 PINMUX_PIN(PIN_PD5, 1, 2) +#define PIN_PD5__UTXD2 PINMUX_PIN(PIN_PD5, 2, 1) +#define PIN_PD5__GRX2 PINMUX_PIN(PIN_PD5, 4, 2) +#define PIN_PD5__ISC_D9 PINMUX_PIN(PIN_PD5, 5, 2) +#define PIN_PD5__NCS1 PINMUX_PIN(PIN_PD5, 6, 2) +#define PIN_PD6 102 +#define PIN_PD6__GPIO PINMUX_PIN(PIN_PD6, 0, 0) +#define PIN_PD6__TCK PINMUX_PIN(PIN_PD6, 1, 2) +#define PIN_PD6__PCK1 PINMUX_PIN(PIN_PD6, 2, 1) +#define PIN_PD6__GRX3 PINMUX_PIN(PIN_PD6, 4, 2) +#define PIN_PD6__ISC_D8 PINMUX_PIN(PIN_PD6, 5, 2) +#define PIN_PD6__NCS2 PINMUX_PIN(PIN_PD6, 6, 2) +#define PIN_PD7 103 +#define PIN_PD7__GPIO PINMUX_PIN(PIN_PD7, 0, 0) +#define PIN_PD7__TDI PINMUX_PIN(PIN_PD7, 1, 2) +#define PIN_PD7__UTMI_RXVAL PINMUX_PIN(PIN_PD7, 3, 1) +#define PIN_PD7__GTX2 PINMUX_PIN(PIN_PD7, 4, 2) +#define PIN_PD7__ISC_D0 PINMUX_PIN(PIN_PD7, 5, 2) +#define PIN_PD7__NWR1_NBS1 PINMUX_PIN(PIN_PD7, 6, 2) +#define PIN_PD8 104 +#define PIN_PD8__GPIO PINMUX_PIN(PIN_PD8, 0, 0) +#define PIN_PD8__TDO PINMUX_PIN(PIN_PD8, 1, 2) +#define PIN_PD8__UTMI_RXERR PINMUX_PIN(PIN_PD8, 3, 1) +#define PIN_PD8__GTX3 PINMUX_PIN(PIN_PD8, 4, 2) +#define PIN_PD8__ISC_D1 PINMUX_PIN(PIN_PD8, 5, 2) +#define PIN_PD8__NANDRDY PINMUX_PIN(PIN_PD8, 6, 2) +#define PIN_PD9 105 +#define PIN_PD9__GPIO PINMUX_PIN(PIN_PD9, 0, 0) +#define PIN_PD9__TMS PINMUX_PIN(PIN_PD9, 1, 2) +#define PIN_PD9__UTMI_RXACT PINMUX_PIN(PIN_PD9, 3, 1) +#define PIN_PD9__GTXCK PINMUX_PIN(PIN_PD9, 4, 2) +#define PIN_PD9__ISC_D2 PINMUX_PIN(PIN_PD9, 5, 2) +#define PIN_PD10 106 +#define PIN_PD10__GPIO PINMUX_PIN(PIN_PD10, 0, 0) +#define PIN_PD10__NTRST PINMUX_PIN(PIN_PD10, 1, 2) +#define PIN_PD10__UTMI_HDIS PINMUX_PIN(PIN_PD10, 3, 1) +#define PIN_PD10__GTXEN PINMUX_PIN(PIN_PD10, 4, 2) +#define PIN_PD10__ISC_D3 PINMUX_PIN(PIN_PD10, 5, 2) +#define PIN_PD11 107 +#define PIN_PD11__GPIO PINMUX_PIN(PIN_PD11, 0, 0) +#define PIN_PD11__TIOA1 PINMUX_PIN(PIN_PD11, 1, 3) +#define PIN_PD11__PCK2 PINMUX_PIN(PIN_PD11, 2, 2) +#define PIN_PD11__UTMI_LS0 PINMUX_PIN(PIN_PD11, 3, 1) +#define PIN_PD11__GRXDV PINMUX_PIN(PIN_PD11, 4, 2) +#define PIN_PD11__ISC_D4 PINMUX_PIN(PIN_PD11, 5, 2) +#define PIN_PD11__ISC_MCK PINMUX_PIN(PIN_PD11, 7, 4) +#define PIN_PD12 108 +#define PIN_PD12__GPIO PINMUX_PIN(PIN_PD12, 0, 0) +#define PIN_PD12__TIOB1 PINMUX_PIN(PIN_PD12, 1, 3) +#define PIN_PD12__FLEXCOM4_IO0 PINMUX_PIN(PIN_PD12, 2, 2) +#define PIN_PD12__UTMI_LS1 PINMUX_PIN(PIN_PD12, 3, 1) +#define PIN_PD12__GRXER PINMUX_PIN(PIN_PD12, 4, 2) +#define PIN_PD12__ISC_D5 PINMUX_PIN(PIN_PD12, 5, 2) +#define PIN_PD12__ISC_D4 PINMUX_PIN(PIN_PD12, 6, 4) +#define PIN_PD13 109 +#define PIN_PD13__GPIO PINMUX_PIN(PIN_PD13, 0, 0) +#define PIN_PD13__TCLK1 PINMUX_PIN(PIN_PD13, 1, 3) +#define PIN_PD13__FLEXCOM4_IO1 PINMUX_PIN(PIN_PD13, 2, 2) +#define PIN_PD13__UTMI_CDRPCSEL0 PINMUX_PIN(PIN_PD13, 3, 1) +#define PIN_PD13__GRX0 PINMUX_PIN(PIN_PD13, 4, 2) +#define PIN_PD13__ISC_D6 PINMUX_PIN(PIN_PD13, 5, 2) +#define PIN_PD13__ISC_D5 PINMUX_PIN(PIN_PD13, 6, 4) +#define PIN_PD14 110 +#define PIN_PD14__GPIO PINMUX_PIN(PIN_PD14, 0, 0) +#define PIN_PD14__TCK PINMUX_PIN(PIN_PD14, 1, 1) +#define PIN_PD14__FLEXCOM4_IO2 PINMUX_PIN(PIN_PD14, 2, 2) +#define PIN_PD14__UTMI_CDRPCSEL1 PINMUX_PIN(PIN_PD14, 3, 1) +#define PIN_PD14__GRX1 PINMUX_PIN(PIN_PD14, 4, 2) +#define PIN_PD14__ISC_D7 PINMUX_PIN(PIN_PD14, 5, 2) +#define PIN_PD14__ISC_D6 PINMUX_PIN(PIN_PD14, 6, 4) +#define PIN_PD15 111 +#define PIN_PD15__GPIO PINMUX_PIN(PIN_PD15, 0, 0) +#define PIN_PD15__TDI PINMUX_PIN(PIN_PD15, 1, 1) +#define PIN_PD15__FLEXCOM4_IO3 PINMUX_PIN(PIN_PD15, 2, 2) +#define PIN_PD15__UTMI_CDRCPDIVEN PINMUX_PIN(PIN_PD15, 3, 1) +#define PIN_PD15__GTX0 PINMUX_PIN(PIN_PD15, 4, 2) +#define PIN_PD15__ISC_PCK PINMUX_PIN(PIN_PD15, 5, 2) +#define PIN_PD15__ISC_D7 PINMUX_PIN(PIN_PD15, 6, 4) +#define PIN_PD16 112 +#define PIN_PD16__GPIO PINMUX_PIN(PIN_PD16, 0, 0) +#define PIN_PD16__TDO PINMUX_PIN(PIN_PD16, 1, 1) +#define PIN_PD16__FLEXCOM4_IO4 PINMUX_PIN(PIN_PD16, 2, 2) +#define PIN_PD16__UTMI_CDRBISTEN PINMUX_PIN(PIN_PD16, 3, 1) +#define PIN_PD16__GTX1 PINMUX_PIN(PIN_PD16, 4, 2) +#define PIN_PD16__ISC_VSYNC PINMUX_PIN(PIN_PD16, 5, 2) +#define PIN_PD16__ISC_D8 PINMUX_PIN(PIN_PD16, 6, 4) +#define PIN_PD17 113 +#define PIN_PD17__GPIO PINMUX_PIN(PIN_PD17, 0, 0) +#define PIN_PD17__TMS PINMUX_PIN(PIN_PD17, 1, 1) +#define PIN_PD17__UTMI_CDRCPSELDIV PINMUX_PIN(PIN_PD17, 3, 1) +#define PIN_PD17__GMDC PINMUX_PIN(PIN_PD17, 4, 2) +#define PIN_PD17__ISC_HSYNC PINMUX_PIN(PIN_PD17, 5, 2) +#define PIN_PD17__ISC_D9 PINMUX_PIN(PIN_PD17, 6, 4) +#define PIN_PD18 114 +#define PIN_PD18__GPIO PINMUX_PIN(PIN_PD18, 0, 0) +#define PIN_PD18__NTRST PINMUX_PIN(PIN_PD18, 1, 1) +#define PIN_PD18__GMDIO PINMUX_PIN(PIN_PD18, 4, 2) +#define PIN_PD18__ISC_FIELD PINMUX_PIN(PIN_PD18, 5, 2) +#define PIN_PD18__ISC_D10 PINMUX_PIN(PIN_PD18, 6, 4) +#define PIN_PD19 115 +#define PIN_PD19__GPIO PINMUX_PIN(PIN_PD19, 0, 0) +#define PIN_PD19__PCK0 PINMUX_PIN(PIN_PD19, 1, 1) +#define PIN_PD19__TWD1 PINMUX_PIN(PIN_PD19, 2, 3) +#define PIN_PD19__URXD2 PINMUX_PIN(PIN_PD19, 3, 3) +#define PIN_PD19__I2SC0_CK PINMUX_PIN(PIN_PD19, 5, 2) +#define PIN_PD19__ISC_D11 PINMUX_PIN(PIN_PD19, 6, 4) +#define PIN_PD20 116 +#define PIN_PD20__GPIO PINMUX_PIN(PIN_PD20, 0, 0) +#define PIN_PD20__TIOA2 PINMUX_PIN(PIN_PD20, 1, 3) +#define PIN_PD20__TWCK1 PINMUX_PIN(PIN_PD20, 2, 3) +#define PIN_PD20__UTXD2 PINMUX_PIN(PIN_PD20, 3, 3) +#define PIN_PD20__I2SC0_MCK PINMUX_PIN(PIN_PD20, 5, 2) +#define PIN_PD20__ISC_PCK PINMUX_PIN(PIN_PD20, 6, 4) +#define PIN_PD21 117 +#define PIN_PD21__GPIO PINMUX_PIN(PIN_PD21, 0, 0) +#define PIN_PD21__TIOB2 PINMUX_PIN(PIN_PD21, 1, 3) +#define PIN_PD21__TWD0 PINMUX_PIN(PIN_PD21, 2, 4) +#define PIN_PD21__FLEXCOM4_IO0 PINMUX_PIN(PIN_PD21, 3, 3) +#define PIN_PD21__I2SC0_WS PINMUX_PIN(PIN_PD21, 5, 2) +#define PIN_PD21__ISC_VSYNC PINMUX_PIN(PIN_PD21, 6, 4) +#define PIN_PD22 118 +#define PIN_PD22__GPIO PINMUX_PIN(PIN_PD22, 0, 0) +#define PIN_PD22__TCLK2 PINMUX_PIN(PIN_PD22, 1, 3) +#define PIN_PD22__TWCK0 PINMUX_PIN(PIN_PD22, 2, 4) +#define PIN_PD22__FLEXCOM4_IO1 PINMUX_PIN(PIN_PD22, 3, 3) +#define PIN_PD22__I2SC0_DI0 PINMUX_PIN(PIN_PD22, 5, 2) +#define PIN_PD22__ISC_HSYNC PINMUX_PIN(PIN_PD22, 6, 4) +#define PIN_PD23 119 +#define PIN_PD23__GPIO PINMUX_PIN(PIN_PD23, 0, 0) +#define PIN_PD23__URXD2 PINMUX_PIN(PIN_PD23, 1, 2) +#define PIN_PD23__FLEXCOM4_IO2 PINMUX_PIN(PIN_PD23, 3, 3) +#define PIN_PD23__I2SC0_DO0 PINMUX_PIN(PIN_PD23, 5, 2) +#define PIN_PD23__ISC_FIELD PINMUX_PIN(PIN_PD23, 6, 4) +#define PIN_PD24 120 +#define PIN_PD24__GPIO PINMUX_PIN(PIN_PD24, 0, 0) +#define PIN_PD24__UTXD2 PINMUX_PIN(PIN_PD24, 1, 2) +#define PIN_PD24__FLEXCOM4_IO3 PINMUX_PIN(PIN_PD24, 3, 3) +#define PIN_PD25 121 +#define PIN_PD25__GPIO PINMUX_PIN(PIN_PD25, 0, 0) +#define PIN_PD25__SPI1_SPCK PINMUX_PIN(PIN_PD25, 1, 3) +#define PIN_PD25__FLEXCOM4_IO4 PINMUX_PIN(PIN_PD25, 3, 3) +#define PIN_PD26 122 +#define PIN_PD26__GPIO PINMUX_PIN(PIN_PD26, 0, 0) +#define PIN_PD26__SPI1_MOSI PINMUX_PIN(PIN_PD26, 1, 3) +#define PIN_PD26__FLEXCOM2_IO0 PINMUX_PIN(PIN_PD26, 3, 2) +#define PIN_PD27 123 +#define PIN_PD27__GPIO PINMUX_PIN(PIN_PD27, 0, 0) +#define PIN_PD27__SPI1_MISO PINMUX_PIN(PIN_PD27, 1, 3) +#define PIN_PD27__TCK PINMUX_PIN(PIN_PD27, 2, 3) +#define PIN_PD27__FLEXCOM2_IO1 PINMUX_PIN(PIN_PD27, 3, 2) +#define PIN_PD28 124 +#define PIN_PD28__GPIO PINMUX_PIN(PIN_PD28, 0, 0) +#define PIN_PD28__SPI1_NPCS0 PINMUX_PIN(PIN_PD28, 1, 3) +#define PIN_PD28__TCI PINMUX_PIN(PIN_PD28, 2, 3) +#define PIN_PD28__FLEXCOM2_IO2 PINMUX_PIN(PIN_PD28, 3, 2) +#define PIN_PD29 125 +#define PIN_PD29__GPIO PINMUX_PIN(PIN_PD29, 0, 0) +#define PIN_PD29__SPI1_NPCS1 PINMUX_PIN(PIN_PD29, 1, 3) +#define PIN_PD29__TDO PINMUX_PIN(PIN_PD29, 2, 3) +#define PIN_PD29__FLEXCOM2_IO3 PINMUX_PIN(PIN_PD29, 3, 2) +#define PIN_PD29__TIOA3 PINMUX_PIN(PIN_PD29, 4, 3) +#define PIN_PD29__TWD0 PINMUX_PIN(PIN_PD29, 5, 3) +#define PIN_PD30 126 +#define PIN_PD30__GPIO PINMUX_PIN(PIN_PD30, 0, 0) +#define PIN_PD30__SPI1_NPCS2 PINMUX_PIN(PIN_PD30, 1, 3) +#define PIN_PD30__TMS PINMUX_PIN(PIN_PD30, 2, 3) +#define PIN_PD30__FLEXCOM2_IO4 PINMUX_PIN(PIN_PD30, 3, 2) +#define PIN_PD30__TIOB3 PINMUX_PIN(PIN_PD30, 4, 3) +#define PIN_PD30__TWCK0 PINMUX_PIN(PIN_PD30, 5, 3) +#define PIN_PD31 127 +#define PIN_PD31__GPIO PINMUX_PIN(PIN_PD31, 0, 0) +#define PIN_PD31__ADTRG PINMUX_PIN(PIN_PD31, 1, 1) +#define PIN_PD31__NTRST PINMUX_PIN(PIN_PD31, 2, 3) +#define PIN_PD31__IRQ PINMUX_PIN(PIN_PD31, 3, 4) +#define PIN_PD31__TCLK3 PINMUX_PIN(PIN_PD31, 4, 3) +#define PIN_PD31__PCK0 PINMUX_PIN(PIN_PD31, 5, 2) diff --git a/optee_os/core/arch/arm/dts/sama5d2.dtsi b/optee_os/core/arch/arm/dts/sama5d2.dtsi new file mode 100644 index 0000000..1a04292 --- /dev/null +++ b/optee_os/core/arch/arm/dts/sama5d2.dtsi @@ -0,0 +1,1207 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * sama5d2.dtsi - Device Tree Include file for SAMA5D2 family SoC + * + * Copyright (C) 2015 Atmel, + * 2015 Ludovic Desroches + */ + +#include +#include +#include +#include + +/ { + #address-cells = <1>; + #size-cells = <1>; + model = "Atmel SAMA5D2 family SoC"; + compatible = "atmel,sama5d2"; + interrupt-parent = <&aic>; + + aliases { + serial0 = &uart1; + serial1 = &uart3; + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a5"; + reg = <0>; + next-level-cache = <&L2>; + clocks = <&pmc PMC_TYPE_CORE PMC_MCK_PRES>; + clock-names = "cpu"; + }; + }; + + pmu { + compatible = "arm,cortex-a5-pmu"; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH 0>; + }; + + etb@740000 { + compatible = "arm,coresight-etb10", "arm,primecell"; + reg = <0x740000 0x1000>; + + clocks = <&pmc PMC_TYPE_CORE PMC_MCK>; + clock-names = "apb_pclk"; + + in-ports { + port { + etb_in: endpoint { + remote-endpoint = <&etm_out>; + }; + }; + }; + }; + + etm@73c000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0x73c000 0x1000>; + + clocks = <&pmc PMC_TYPE_CORE PMC_MCK>; + clock-names = "apb_pclk"; + + out-ports { + port { + etm_out: endpoint { + remote-endpoint = <&etb_in>; + }; + }; + }; + }; + + memory@20000000 { + device_type = "memory"; + reg = <0x20000000 0x20000000>; + }; + + clocks { + slow_xtal: slow_xtal { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + }; + + main_xtal: main_xtal { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + }; + }; + + ns_sram: sram@200000 { + compatible = "atmel,sama5d2-sram", "mmio-sram"; + reg = <0x00200000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x00200000 0x20000>; + status = "disabled"; + secure-status = "okay"; + }; + + ahb { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + nfc_sram: sram@100000 { + compatible = "mmio-sram"; + no-memory-wc; + reg = <0x00100000 0x2400>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x00100000 0x2400>; + + }; + + usb0: gadget@300000 { + compatible = "atmel,sama5d3-udc"; + reg = <0x00300000 0x100000 + 0xfc02c000 0x400>; + interrupts = <42 IRQ_TYPE_LEVEL_HIGH 2>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 42>, <&pmc PMC_TYPE_CORE PMC_UTMI>; + clock-names = "pclk", "hclk"; + assigned-clocks = <&pmc PMC_TYPE_CORE PMC_USBCK>; + assigned-clock-parents = <&pmc PMC_TYPE_CORE PMC_UTMI>; + assigned-clock-rates = <48000000>; + status = "disabled"; + }; + + usb1: ohci@400000 { + compatible = "atmel,at91rm9200-ohci", "usb-ohci"; + reg = <0x00400000 0x100000>; + interrupts = <41 IRQ_TYPE_LEVEL_HIGH 2>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 41>, <&pmc PMC_TYPE_PERIPHERAL 41>, <&pmc PMC_TYPE_SYSTEM 6>; + clock-names = "ohci_clk", "hclk", "uhpck"; + assigned-clocks = <&pmc PMC_TYPE_CORE PMC_USBCK>; + assigned-clock-parents = <&pmc PMC_TYPE_CORE PMC_UTMI>; + assigned-clock-rates = <48000000>; + status = "disabled"; + }; + + usb2: ehci@500000 { + compatible = "atmel,at91sam9g45-ehci", "usb-ehci"; + reg = <0x00500000 0x100000>; + interrupts = <41 IRQ_TYPE_LEVEL_HIGH 2>; + clocks = <&pmc PMC_TYPE_CORE PMC_UTMI>, <&pmc PMC_TYPE_PERIPHERAL 41>; + clock-names = "usb_clk", "ehci_clk"; + assigned-clocks = <&pmc PMC_TYPE_CORE PMC_USBCK>; + assigned-clock-parents = <&pmc PMC_TYPE_CORE PMC_UTMI>; + assigned-clock-rates = <48000000>; + status = "disabled"; + }; + + L2: cache-controller@a00000 { + compatible = "arm,pl310-cache"; + reg = <0x00a00000 0x1000>; + interrupts = <63 IRQ_TYPE_LEVEL_HIGH 4>; + cache-unified; + cache-level = <2>; + }; + + ebi: ebi@10000000 { + compatible = "atmel,sama5d3-ebi"; + #address-cells = <2>; + #size-cells = <1>; + atmel,smc = <&hsmc>; + reg = <0x10000000 0x10000000 + 0x60000000 0x30000000>; + ranges = <0x0 0x0 0x10000000 0x10000000 + 0x1 0x0 0x60000000 0x10000000 + 0x2 0x0 0x70000000 0x10000000 + 0x3 0x0 0x80000000 0x10000000>; + clocks = <&pmc PMC_TYPE_CORE PMC_MCK2>; + status = "disabled"; + + nand_controller: nand-controller { + compatible = "atmel,sama5d3-nand-controller"; + atmel,nfc-sram = <&nfc_sram>; + atmel,nfc-io = <&nfc_io>; + ecc-engine = <&pmecc>; + #address-cells = <2>; + #size-cells = <1>; + ranges; + status = "disabled"; + }; + }; + + sdmmc0: sdio-host@a0000000 { + compatible = "atmel,sama5d2-sdhci"; + reg = <0xa0000000 0x300>; + interrupts = <31 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 31>, <&pmc PMC_TYPE_GCK 31>, <&pmc PMC_TYPE_CORE PMC_MAIN>; + clock-names = "hclock", "multclk", "baseclk"; + assigned-clocks = <&pmc PMC_TYPE_GCK 31>; + assigned-clock-parents = <&pmc PMC_TYPE_CORE PMC_UTMI>; + assigned-clock-rates = <480000000>; + status = "disabled"; + }; + + sdmmc1: sdio-host@b0000000 { + compatible = "atmel,sama5d2-sdhci"; + reg = <0xb0000000 0x300>; + interrupts = <32 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 32>, <&pmc PMC_TYPE_GCK 32>, <&pmc PMC_TYPE_CORE PMC_MAIN>; + clock-names = "hclock", "multclk", "baseclk"; + assigned-clocks = <&pmc PMC_TYPE_GCK 32>; + assigned-clock-parents = <&pmc PMC_TYPE_CORE PMC_UTMI>; + assigned-clock-rates = <480000000>; + status = "disabled"; + }; + + nfc_io: nfc-io@c0000000 { + compatible = "atmel,sama5d3-nfc-io", "syscon"; + reg = <0xc0000000 0x8000000>; + }; + + apb { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + hlcdc: hlcdc@f0000000 { + compatible = "atmel,sama5d2-hlcdc"; + reg = <0xf0000000 0x2000>; + interrupts = <45 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 45>, <&pmc PMC_TYPE_SYSTEM 3>, <&clk32k>; + clock-names = "periph_clk","sys_clk", "slow_clk"; + status = "disabled"; + + hlcdc-display-controller { + compatible = "atmel,hlcdc-display-controller"; + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + }; + }; + + hlcdc_pwm: hlcdc-pwm { + compatible = "atmel,hlcdc-pwm"; + #pwm-cells = <3>; + }; + }; + + isc: isc@f0008000 { + compatible = "atmel,sama5d2-isc"; + reg = <0xf0008000 0x4000>; + interrupts = <46 IRQ_TYPE_LEVEL_HIGH 5>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 46>, <&pmc PMC_TYPE_SYSTEM 18>, <&pmc PMC_TYPE_GCK 46>; + clock-names = "hclock", "iscck", "gck"; + #clock-cells = <0>; + clock-output-names = "isc-mck"; + status = "disabled"; + }; + + ramc0: ramc@f000c000 { + compatible = "atmel,sama5d3-ddramc"; + reg = <0xf000c000 0x200>; + clocks = <&pmc PMC_TYPE_SYSTEM 2>, <&pmc PMC_TYPE_PERIPHERAL 13>; + clock-names = "ddrck", "mpddr"; + }; + + dma0: dma-controller@f0010000 { + compatible = "atmel,sama5d4-dma"; + reg = <0xf0010000 0x1000>; + interrupts = <6 IRQ_TYPE_LEVEL_HIGH 0>; + #dma-cells = <1>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 6>; + clock-names = "dma_clk"; + }; + + /* Place dma1 here despite its address */ + dma1: dma-controller@f0004000 { + compatible = "atmel,sama5d4-dma"; + reg = <0xf0004000 0x1000>; + interrupts = <7 IRQ_TYPE_LEVEL_HIGH 0>; + #dma-cells = <1>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 7>; + clock-names = "dma_clk"; + }; + + pmc: pmc@f0014000 { + compatible = "atmel,sama5d2-pmc", "syscon"; + reg = <0xf0014000 0x160>; + interrupts = <74 IRQ_TYPE_LEVEL_HIGH 7>; + #clock-cells = <2>; + clocks = <&clk32k>, <&main_xtal>; + clock-names = "slow_clk", "main_xtal"; + status = "disabled"; + secure-status = "okay"; + }; + + qspi0: spi@f0020000 { + compatible = "atmel,sama5d2-qspi"; + reg = <0xf0020000 0x100>, <0xd0000000 0x08000000>; + reg-names = "qspi_base", "qspi_mmap"; + interrupts = <52 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 52>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + qspi1: spi@f0024000 { + compatible = "atmel,sama5d2-qspi"; + reg = <0xf0024000 0x100>, <0xd8000000 0x08000000>; + reg-names = "qspi_base", "qspi_mmap"; + interrupts = <53 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 53>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + sha@f0028000 { + compatible = "atmel,at91sam9g46-sha"; + reg = <0xf0028000 0x100>; + interrupts = <12 IRQ_TYPE_LEVEL_HIGH 0>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(30))>; + dma-names = "tx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 12>; + clock-names = "sha_clk"; + status = "okay"; + }; + + aes@f002c000 { + compatible = "atmel,at91sam9g46-aes"; + reg = <0xf002c000 0x100>; + interrupts = <9 IRQ_TYPE_LEVEL_HIGH 0>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(26))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(27))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 9>; + clock-names = "aes_clk"; + status = "okay"; + }; + + spi0: spi@f8000000 { + compatible = "atmel,at91rm9200-spi"; + reg = <0xf8000000 0x100>; + interrupts = <33 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(6))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(7))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 33>; + clock-names = "spi_clk"; + atmel,fifo-size = <16>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + ssc0: ssc@f8004000 { + compatible = "atmel,at91sam9g45-ssc"; + reg = <0xf8004000 0x4000>; + interrupts = <43 IRQ_TYPE_LEVEL_HIGH 4>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(21))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(22))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 43>; + clock-names = "pclk"; + status = "disabled"; + }; + + macb0: ethernet@f8008000 { + compatible = "atmel,sama5d2-gem"; + reg = <0xf8008000 0x1000>; + interrupts = <5 IRQ_TYPE_LEVEL_HIGH 3 /* Queue 0 */ + 66 IRQ_TYPE_LEVEL_HIGH 3 /* Queue 1 */ + 67 IRQ_TYPE_LEVEL_HIGH 3>; /* Queue 2 */ + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 5>, <&pmc PMC_TYPE_PERIPHERAL 5>; + clock-names = "hclk", "pclk"; + status = "disabled"; + }; + + tcb0: timer@f800c000 { + compatible = "atmel,sama5d2-tcb", "simple-mfd", "syscon"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0xf800c000 0x100>; + interrupts = <35 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 35>, <&pmc PMC_TYPE_GCK 35>, <&clk32k>; + clock-names = "t0_clk", "gclk", "slow_clk"; + }; + + tcb1: timer@f8010000 { + compatible = "atmel,sama5d2-tcb", "simple-mfd", "syscon"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0xf8010000 0x100>; + interrupts = <36 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 36>, <&pmc PMC_TYPE_GCK 36>, <&clk32k>; + clock-names = "t0_clk", "gclk", "slow_clk"; + status = "disabled"; + secure-status = "okay"; + }; + + hsmc: hsmc@f8014000 { + compatible = "atmel,sama5d2-smc", "syscon", "simple-mfd"; + reg = <0xf8014000 0x1000>; + interrupts = <17 IRQ_TYPE_LEVEL_HIGH 6>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 17>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + pmecc: ecc-engine@f8014070 { + compatible = "atmel,sama5d2-pmecc"; + reg = <0xf8014070 0x490>, + <0xf8014500 0x100>; + }; + }; + + pdmic: pdmic@f8018000 { + compatible = "atmel,sama5d2-pdmic"; + reg = <0xf8018000 0x124>; + interrupts = <48 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) + | AT91_XDMAC_DT_PERID(50))>; + dma-names = "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 48>, <&pmc PMC_TYPE_GCK 48>; + clock-names = "pclk", "gclk"; + status = "disabled"; + }; + + uart0: serial@f801c000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xf801c000 0x100>; + interrupts = <24 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(35))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(36))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 24>; + clock-names = "usart"; + status = "disabled"; + }; + + uart1: serial@f8020000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xf8020000 0x100>; + interrupts = <25 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(37))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(38))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 25>; + clock-names = "usart"; + status = "disabled"; + }; + + uart2: serial@f8024000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xf8024000 0x100>; + interrupts = <26 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(39))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(40))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 26>; + clock-names = "usart"; + status = "disabled"; + }; + + i2c0: i2c@f8028000 { + compatible = "atmel,sama5d2-i2c"; + reg = <0xf8028000 0x100>; + interrupts = <29 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(0))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(1))>; + dma-names = "tx", "rx"; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 29>; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + pwm0: pwm@f802c000 { + compatible = "atmel,sama5d2-pwm"; + reg = <0xf802c000 0x4000>; + interrupts = <38 IRQ_TYPE_LEVEL_HIGH 7>; + #pwm-cells = <3>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 38>; + status = "disabled"; + }; + + sfr: sfr@f8030000 { + compatible = "atmel,sama5d2-sfr", "syscon"; + reg = <0xf8030000 0x98>; + status = "disabled"; + secure-status = "okay"; + }; + + flx0: flexcom@f8034000 { + compatible = "atmel,sama5d2-flexcom"; + reg = <0xf8034000 0x200>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 19>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0xf8034000 0x800>; + status = "disabled"; + + uart5: serial@200 { + compatible = "atmel,at91sam9260-usart"; + reg = <0x200 0x200>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 19>; + clock-names = "usart"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(11))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(12))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <32>; + status = "disabled"; + }; + + spi2: spi@400 { + compatible = "atmel,at91rm9200-spi"; + reg = <0x400 0x200>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 19>; + clock-names = "spi_clk"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(11))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(12))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + i2c2: i2c@600 { + compatible = "atmel,sama5d2-i2c"; + reg = <0x600 0x200>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 19>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(11))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(12))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + }; + + flx1: flexcom@f8038000 { + compatible = "atmel,sama5d2-flexcom"; + reg = <0xf8038000 0x200>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 20>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0xf8038000 0x800>; + status = "disabled"; + + uart6: serial@200 { + compatible = "atmel,at91sam9260-usart"; + reg = <0x200 0x200>; + interrupts = <20 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 20>; + clock-names = "usart"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(13))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(14))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <32>; + status = "disabled"; + }; + + spi3: spi@400 { + compatible = "atmel,at91rm9200-spi"; + reg = <0x400 0x200>; + interrupts = <20 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 20>; + clock-names = "spi_clk"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(13))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(14))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + i2c3: i2c@600 { + compatible = "atmel,sama5d2-i2c"; + reg = <0x600 0x200>; + interrupts = <20 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 20>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(13))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(14))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + }; + + securam: sram@f8044000 { + compatible = "atmel,sama5d2-securam", "mmio-sram"; + reg = <0xf8044000 0x1420>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 51>; + #address-cells = <1>; + #size-cells = <1>; + no-memory-wc; + ranges = <0 0xf8044000 0x1420>; + status = "disabled"; + secure-status = "okay"; + }; + + reset_controller: rstc@f8048000 { + compatible = "atmel,sama5d3-rstc"; + reg = <0xf8048000 0x10>; + clocks = <&clk32k>; + status = "disabled"; + secure-status = "okay"; + }; + + shutdown_controller: shdwc@f8048010 { + compatible = "atmel,sama5d2-shdwc"; + reg = <0xf8048010 0x10>; + clocks = <&clk32k>; + #address-cells = <1>; + #size-cells = <0>; + atmel,wakeup-rtc-timer; + status = "disabled"; + secure-status = "okay"; + }; + + pit: timer@f8048030 { + compatible = "atmel,at91sam9260-pit"; + reg = <0xf8048030 0x10>; + interrupts = <3 IRQ_TYPE_LEVEL_HIGH 5>; + clocks = <&pmc PMC_TYPE_CORE PMC_MCK2>; + }; + + watchdog: watchdog@f8048040 { + compatible = "atmel,sama5d4-wdt"; + reg = <0xf8048040 0x10>; + interrupts = <4 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&clk32k>; + status = "disabled"; + secure-status = "okay"; + }; + + clk32k: sckc@f8048050 { + compatible = "atmel,sama5d4-sckc"; + reg = <0xf8048050 0x4>; + + clocks = <&slow_xtal>; + #clock-cells = <0>; + status = "disabled"; + secure-status = "okay"; + }; + + rtc: rtc@f80480b0 { + compatible = "atmel,sama5d2-rtc"; + reg = <0xf80480b0 0x30>; + interrupts = <74 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&clk32k>; + status = "disabled"; + secure-status = "okay"; + }; + + i2s0: i2s@f8050000 { + compatible = "atmel,sama5d2-i2s"; + reg = <0xf8050000 0x100>; + interrupts = <54 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(31))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(32))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 54>, <&pmc PMC_TYPE_GCK 54>; + clock-names = "pclk", "gclk"; + assigned-clocks = <&pmc PMC_TYPE_CORE PMC_I2S0_MUX>; + assigned-clock-parents = <&pmc PMC_TYPE_GCK 54>; + status = "disabled"; + }; + + can0: can@f8054000 { + compatible = "bosch,m_can"; + reg = <0xf8054000 0x4000>, <0x210000 0x1c00>; + reg-names = "m_can", "message_ram"; + interrupts = <56 IRQ_TYPE_LEVEL_HIGH 7>, + <64 IRQ_TYPE_LEVEL_HIGH 7>; + interrupt-names = "int0", "int1"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 56>, <&pmc PMC_TYPE_GCK 56>; + clock-names = "hclk", "cclk"; + assigned-clocks = <&pmc PMC_TYPE_GCK 56>; + assigned-clock-parents = <&pmc PMC_TYPE_CORE PMC_UTMI>; + assigned-clock-rates = <40000000>; + bosch,mram-cfg = <0x0 0 0 64 0 0 32 32>; + status = "disabled"; + }; + + spi1: spi@fc000000 { + compatible = "atmel,at91rm9200-spi"; + reg = <0xfc000000 0x100>; + interrupts = <34 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(8))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(9))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 34>; + clock-names = "spi_clk"; + atmel,fifo-size = <16>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart3: serial@fc008000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xfc008000 0x100>; + interrupts = <27 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma1 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(41))>, + <&dma1 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(42))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 27>; + clock-names = "usart"; + status = "disabled"; + }; + + uart4: serial@fc00c000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xfc00c000 0x100>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(43))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(44))>; + dma-names = "tx", "rx"; + interrupts = <28 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 28>; + clock-names = "usart"; + status = "disabled"; + }; + + flx2: flexcom@fc010000 { + compatible = "atmel,sama5d2-flexcom"; + reg = <0xfc010000 0x200>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 21>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0xfc010000 0x800>; + status = "disabled"; + + uart7: serial@200 { + compatible = "atmel,at91sam9260-usart"; + reg = <0x200 0x200>; + interrupts = <21 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 21>; + clock-names = "usart"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(15))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(16))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <32>; + status = "disabled"; + }; + + spi4: spi@400 { + compatible = "atmel,at91rm9200-spi"; + reg = <0x400 0x200>; + interrupts = <21 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 21>; + clock-names = "spi_clk"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(15))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(16))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + i2c4: i2c@600 { + compatible = "atmel,sama5d2-i2c"; + reg = <0x600 0x200>; + interrupts = <21 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 21>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(15))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(16))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + }; + + flx3: flexcom@fc014000 { + compatible = "atmel,sama5d2-flexcom"; + reg = <0xfc014000 0x200>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 22>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0xfc014000 0x800>; + status = "disabled"; + + uart8: serial@200 { + compatible = "atmel,at91sam9260-usart"; + reg = <0x200 0x200>; + interrupts = <22 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 22>; + clock-names = "usart"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(17))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(18))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <32>; + status = "disabled"; + }; + + spi5: spi@400 { + compatible = "atmel,at91rm9200-spi"; + reg = <0x400 0x200>; + interrupts = <22 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 22>; + clock-names = "spi_clk"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(17))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(18))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + i2c5: i2c@600 { + compatible = "atmel,sama5d2-i2c"; + reg = <0x600 0x200>; + interrupts = <22 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 22>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(17))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(18))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + }; + + flx4: flexcom@fc018000 { + compatible = "atmel,sama5d2-flexcom"; + reg = <0xfc018000 0x200>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 23>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0xfc018000 0x800>; + status = "disabled"; + + uart9: serial@200 { + compatible = "atmel,at91sam9260-usart"; + reg = <0x200 0x200>; + interrupts = <23 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 23>; + clock-names = "usart"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(19))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(20))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <32>; + status = "disabled"; + }; + + spi6: spi@400 { + compatible = "atmel,at91rm9200-spi"; + reg = <0x400 0x200>; + interrupts = <23 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 23>; + clock-names = "spi_clk"; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(19))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(20))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + i2c6: i2c@600 { + compatible = "atmel,sama5d2-i2c"; + reg = <0x600 0x200>; + interrupts = <23 IRQ_TYPE_LEVEL_HIGH 7>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 23>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(19))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | + AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(20))>; + dma-names = "tx", "rx"; + atmel,fifo-size = <16>; + status = "disabled"; + }; + }; + + trng@fc01c000 { + compatible = "atmel,at91sam9g45-trng"; + reg = <0xfc01c000 0x100>; + interrupts = <47 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 47>; + status = "disabled"; + secure-status = "okay"; + }; + + aic: interrupt-controller@fc020000 { + #interrupt-cells = <3>; + compatible = "atmel,sama5d2-aic"; + interrupt-controller; + reg = <0xfc020000 0x200>; + atmel,external-irqs = <49>; + }; + + saic: interrupt-controller@f803c000 { + #interrupt-cells = <3>; + compatible = "atmel,sama5d2-saic"; + interrupt-controller; + reg = <0xf803c000 0x200>; + atmel,external-irqs = <49>; + status = "disabled"; + secure-status = "okay"; + }; + + i2c1: i2c@fc028000 { + compatible = "atmel,sama5d2-i2c"; + reg = <0xfc028000 0x100>; + interrupts = <30 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(2))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(3))>; + dma-names = "tx", "rx"; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 30>; + atmel,fifo-size = <16>; + status = "disabled"; + }; + + adc: adc@fc030000 { + compatible = "atmel,sama5d2-adc"; + reg = <0xfc030000 0x100>; + interrupts = <40 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 40>; + clock-names = "adc_clk"; + dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(25))>; + dma-names = "rx"; + atmel,min-sample-rate-hz = <200000>; + atmel,max-sample-rate-hz = <20000000>; + atmel,startup-time-ms = <4>; + atmel,trigger-edge-type = ; + #io-channel-cells = <1>; + status = "disabled"; + }; + + resistive_touch: resistive-touch { + compatible = "resistive-adc-touch"; + io-channels = <&adc AT91_SAMA5D2_ADC_X_CHANNEL>, + <&adc AT91_SAMA5D2_ADC_Y_CHANNEL>, + <&adc AT91_SAMA5D2_ADC_P_CHANNEL>; + io-channel-names = "x", "y", "pressure"; + touchscreen-min-pressure = <50000>; + status = "disabled"; + }; + + pioA: pinctrl@fc039000 { + compatible = "atmel,sama5d2-pinctrl"; + reg = <0xfc039000 0x600>; + interrupts = <18 IRQ_TYPE_LEVEL_HIGH 7>, + <68 IRQ_TYPE_LEVEL_HIGH 7>, + <69 IRQ_TYPE_LEVEL_HIGH 7>, + <70 IRQ_TYPE_LEVEL_HIGH 7>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 18>; + status = "disabled"; + secure-status = "okay"; + }; + + pioBU: secumod@fc040000 { + compatible = "atmel,sama5d2-secumod", "syscon"; + reg = <0xfc040000 0x100>; + + gpio-controller; + #gpio-cells = <2>; + status = "disabled"; + secure-status = "okay"; + }; + + tdes@fc044000 { + compatible = "atmel,at91sam9g46-tdes"; + reg = <0xfc044000 0x100>; + interrupts = <11 IRQ_TYPE_LEVEL_HIGH 0>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(28))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(29))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 11>; + clock-names = "tdes_clk"; + status = "okay"; + }; + + classd: classd@fc048000 { + compatible = "atmel,sama5d2-classd"; + reg = <0xfc048000 0x100>; + interrupts = <59 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(47))>; + dma-names = "tx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 59>, <&pmc PMC_TYPE_GCK 59>; + clock-names = "pclk", "gclk"; + status = "disabled"; + }; + + i2s1: i2s@fc04c000 { + compatible = "atmel,sama5d2-i2s"; + reg = <0xfc04c000 0x100>; + interrupts = <55 IRQ_TYPE_LEVEL_HIGH 7>; + dmas = <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(33))>, + <&dma0 + (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | + AT91_XDMAC_DT_PERID(34))>; + dma-names = "tx", "rx"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 55>, <&pmc PMC_TYPE_GCK 55>; + clock-names = "pclk", "gclk"; + assigned-clocks = <&pmc PMC_TYPE_CORE PMC_I2S1_MUX>; + assigned-parrents = <&pmc PMC_TYPE_GCK 55>; + status = "disabled"; + }; + + can1: can@fc050000 { + compatible = "bosch,m_can"; + reg = <0xfc050000 0x4000>, <0x210000 0x3800>; + reg-names = "m_can", "message_ram"; + interrupts = <57 IRQ_TYPE_LEVEL_HIGH 7>, + <65 IRQ_TYPE_LEVEL_HIGH 7>; + interrupt-names = "int0", "int1"; + clocks = <&pmc PMC_TYPE_PERIPHERAL 57>, <&pmc PMC_TYPE_GCK 57>; + clock-names = "hclk", "cclk"; + assigned-clocks = <&pmc PMC_TYPE_GCK 57>; + assigned-clock-parents = <&pmc PMC_TYPE_CORE PMC_UTMI>; + assigned-clock-rates = <40000000>; + bosch,mram-cfg = <0x1c00 0 0 64 0 0 32 32>; + status = "disabled"; + }; + + sfrbu: sfr@fc05c000 { + compatible = "atmel,sama5d2-sfrbu", "syscon"; + reg = <0xfc05c000 0x20>; + }; + + chipid@fc069000 { + compatible = "atmel,sama5d2-chipid"; + reg = <0xfc069000 0x8>; + }; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp13-pinctrl.dtsi b/optee_os/core/arch/arm/dts/stm32mp13-pinctrl.dtsi new file mode 100644 index 0000000..6bf8419 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp13-pinctrl.dtsi @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019-2022 - All Rights Reserved + * Author: Alexandre Torgue + */ +#include + +&pinctrl { + i2c4_pins_a: i2c4-0 { + pins { + pinmux = , /* I2C4_SCL */ + ; /* I2C4_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + rcc_mco_pins_a: rcc-pins-0 { + pins { + pinmux = ; /* RCC_MCO_1 */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + }; + + uart4_pins_a: uart4-0 { + pins1 { + pinmux = ; /* UART4_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* UART4_RX */ + bias-disable; + }; + }; + + usart1_pins_a: usart1-0 { + pins1 { + pinmux = , /* USART1_TX */ + ; /* USART1_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* USART1_RX */ + ; /* USART1_CTS_NSS */ + bias-pull-up; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp131.dtsi b/optee_os/core/arch/arm/dts/stm32mp131.dtsi new file mode 100644 index 0000000..fb13a53 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp131.dtsi @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2021-2022 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +#include +#include +#include +#include +#include + +/ { + #address-cells = <1>; + #size-cells = <1>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu0: cpu@0 { + compatible = "arm,cortex-a7"; + device_type = "cpu"; + reg = <0>; + }; + }; + + hse_monitor: hse-monitor { + compatible = "st,freq-monitor"; + counter = <&lptimer3 1 1 0 0>; + status = "disabled"; + }; + + intc: interrupt-controller@a0021000 { + compatible = "arm,cortex-a7-gic"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0xa0021000 0x1000>, + <0xa0022000 0x2000>; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + + clocks { + clk_hse: clk-hse { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24000000>; + }; + + clk_hsi: clk-hsi { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <64000000>; + }; + + clk_lse: clk-lse { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <32768>; + }; + + clk_lsi: clk-lsi { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <32000>; + }; + + clk_csi: clk-csi { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <4000000>; + }; + + clk_i2sin: clk-i2sin { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <19000000>; + }; + + }; + + soc { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + interrupt-parent = <&intc>; + ranges; + + usart3: serial@4000f000 { + compatible = "st,stm32h7-uart"; + reg = <0x4000f000 0x400>; + interrupts = ; + clocks = <&rcc USART3_K>; + resets = <&rcc USART3_R>; + status = "disabled"; + }; + + uart4: serial@40010000 { + compatible = "st,stm32h7-uart"; + reg = <0x40010000 0x400>; + interrupts = ; + clocks = <&rcc UART4_K>; + resets = <&rcc UART4_R>; + status = "disabled"; + }; + + uart5: serial@40011000 { + compatible = "st,stm32h7-uart"; + reg = <0x40011000 0x400>; + interrupts = ; + clocks = <&rcc UART5_K>; + resets = <&rcc UART5_R>; + status = "disabled"; + }; + + uart7: serial@40018000 { + compatible = "st,stm32h7-uart"; + reg = <0x40018000 0x400>; + interrupts = ; + clocks = <&rcc UART7_K>; + resets = <&rcc UART7_R>; + status = "disabled"; + }; + + uart8: serial@40019000 { + compatible = "st,stm32h7-uart"; + reg = <0x40019000 0x400>; + interrupts = ; + clocks = <&rcc UART8_K>; + resets = <&rcc UART8_R>; + status = "disabled"; + }; + + usart6: serial@44003000 { + compatible = "st,stm32h7-uart"; + reg = <0x44003000 0x400>; + interrupts = ; + clocks = <&rcc USART6_K>; + resets = <&rcc USART6_R>; + status = "disabled"; + }; + + rcc: rcc@50000000 { + compatible = "st,stm32mp13-rcc", "syscon"; + reg = <0x50000000 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + #clock-cells = <1>; + #reset-cells = <1>; + clocks = <&clk_hse>, <&clk_hsi>, <&clk_lse>, <&clk_lsi>, <&clk_csi>, <&clk_i2sin>; + clock-names = "clk-hse", "clk-hsi", "clk-lse", "clk-lsi", "clk-csi", "clk-i2sin"; + interrupts = ; + secure-interrupts = ; + secure-interrupt-names = "wakeup"; + }; + + pwr_regulators: pwr@50001000 { + compatible = "st,stm32mp1,pwr-reg"; + reg = <0x50001000 0x10>; + + reg11: reg11 { + regulator-name = "reg11"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + }; + + reg18: reg18 { + regulator-name = "reg18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + usb33: usb33 { + regulator-name = "usb33"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + }; + + pwr_irq: pwr@50001010 { + compatible = "st,stm32mp1,pwr-irq"; + status = "disabled"; + }; + + syscfg: syscon@50020000 { + compatible = "st,stm32mp157-syscfg", "syscon"; + reg = <0x50020000 0x400>; + }; + + iwdg2: watchdog@5a002000 { + compatible = "st,stm32mp1-iwdg"; + reg = <0x5a002000 0x400>; + interrupts = ; + clocks = <&rcc IWDG2>, <&rcc CK_LSI>; + clock-names = "pclk", "lsi"; + status = "disabled"; + }; + + rtc: rtc@5c004000 { + compatible = "st,stm32mp13-rtc"; + reg = <0x5c004000 0x400>; + clocks = <&rcc RTCAPB>, <&rcc RTC>; + clock-names = "pclk", "rtc_ck"; + status = "disabled"; + }; + + bsec: efuse@5c005000 { + compatible = "st,stm32mp13-bsec"; + reg = <0x5c005000 0x400>; + #address-cells = <1>; + #size-cells = <1>; + + cfg0_otp: cfg0_otp@0 { + reg = <0x0 0x2>; + }; + part_number_otp: part_number_otp@4 { + reg = <0x4 0x2>; + bits = <0 12>; + }; + monotonic_otp: monotonic_otp@10 { + reg = <0x10 0x4>; + }; + nand_otp: cfg9_otp@24 { + reg = <0x24 0x4>; + }; + uid_otp: uid_otp@34 { + reg = <0x34 0xc>; + }; + hw2_otp: hw2_otp@48 { + reg = <0x48 0x4>; + }; + ts_cal1: calib@5c { + reg = <0x5c 0x2>; + }; + ts_cal2: calib@5e { + reg = <0x5e 0x2>; + }; + pkh_otp: pkh_otp@60 { + reg = <0x60 0x20>; + }; + ethernet_mac1_address: mac1@e4 { + reg = <0xe4 0xc>; + st,non-secure-otp; + }; + oem_enc_key: oem_enc_key@170 { + reg = <0x170 0x10>; + }; + }; + + tzc400: tzc@5c006000 { + compatible = "st,stm32mp1-tzc"; + reg = <0x5c006000 0x1000>; + interrupts = ; + st,mem-map = <0xc0000000 0x40000000>; + clocks = <&rcc TZC>; + }; + + tamp: tamp@5c00a000 { + compatible = "st,stm32mp13-tamp"; + reg = <0x5c00a000 0x400>; + interrupts = ; + clocks = <&rcc RTCAPB>; + }; + + pinctrl: pin-controller@50002000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "st,stm32mp135-pinctrl"; + ranges = <0 0x50002000 0x8400>; + pins-are-numbered; + + gpioa: gpio@50002000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOA>; + reg = <0x0 0x400>; + st,bank-name = "GPIOA"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 0 16>; + }; + + gpiob: gpio@50003000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOB>; + reg = <0x1000 0x400>; + st,bank-name = "GPIOB"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 16 16>; + }; + + gpioc: gpio@50004000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOC>; + reg = <0x2000 0x400>; + st,bank-name = "GPIOC"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 32 16>; + }; + + gpiod: gpio@50005000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOD>; + reg = <0x3000 0x400>; + st,bank-name = "GPIOD"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 48 16>; + }; + + gpioe: gpio@50006000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOE>; + reg = <0x4000 0x400>; + st,bank-name = "GPIOE"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 64 16>; + }; + + gpiof: gpio@50007000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOF>; + reg = <0x5000 0x400>; + st,bank-name = "GPIOF"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 80 16>; + }; + + gpiog: gpio@50008000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOG>; + reg = <0x6000 0x400>; + st,bank-name = "GPIOG"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 96 16>; + }; + + gpioh: gpio@50009000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOH>; + reg = <0x7000 0x400>; + st,bank-name = "GPIOH"; + ngpios = <15>; + gpio-ranges = <&pinctrl 0 112 15>; + }; + + gpioi: gpio@5000a000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&rcc GPIOI>; + reg = <0x8000 0x400>; + st,bank-name = "GPIOI"; + ngpios = <8>; + gpio-ranges = <&pinctrl 0 128 8>; + }; + }; + + etzpc: etzpc@5c007000 { + compatible = "st,stm32-etzpc", "firewall-bus"; + reg = <0x5C007000 0x400>; + clocks = <&rcc TZPC>; + #address-cells = <1>; + #size-cells = <1>; + + adc_2: adc@48004000 { + reg = <0x48004000 0x400>; + compatible = "st,stm32mp13-adc-core"; + interrupts = ; + clocks = <&rcc ADC2>, <&rcc ADC2_K>; + clock-names = "bus", "adc"; + interrupt-controller; + #interrupt-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + adc2: adc@0 { + compatible = "st,stm32mp13-adc"; + reg = <0x0>; + #io-channel-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&adc_2>; + interrupts = <0>; + status = "disabled"; + + channel@13 { + reg = <13>; + label = "vrefint"; + }; + + channel@14 { + reg = <14>; + label = "vddcore"; + }; + + channel@16 { + reg = <16>; + label = "vddcpu"; + }; + + channel@17 { + reg = <17>; + label = "vddq_ddr"; + }; + }; + }; + + usart1: serial@4c000000 { + compatible = "st,stm32h7-uart"; + reg = <0x4c000000 0x400>; + interrupts = ; + clocks = <&rcc USART1_K>; + resets = <&rcc USART1_R>; + status = "disabled"; + }; + + usart2: serial@4c001000 { + compatible = "st,stm32h7-uart"; + reg = <0x4c001000 0x400>; + interrupts = ; + clocks = <&rcc USART2_K>; + resets = <&rcc USART2_R>; + status = "disabled"; + }; + + i2c3: i2c@4c004000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x4c004000 0x400>; + clocks = <&rcc I2C3_K>; + resets = <&rcc I2C3_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x4>; + i2c-analog-filter; + status = "disabled"; + }; + + i2c4: i2c@4c005000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x4c005000 0x400>; + clocks = <&rcc I2C4_K>; + resets = <&rcc I2C4_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x8>; + i2c-analog-filter; + status = "disabled"; + }; + + i2c5: i2c@4c006000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x4c006000 0x400>; + clocks = <&rcc I2C5_K>; + resets = <&rcc I2C5_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x10>; + i2c-analog-filter; + status = "disabled"; + }; + + timers12: timer@4c007000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x4c007000 0x400>; + interrupts = ; + clocks = <&rcc TIM12_K>; + clock-names = "int"; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + timers13: timer@4c008000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x4c008000 0x400>; + clocks = <&rcc TIM13_K>; + clock-names = "int"; + status = "disabled"; + }; + + timers14: timer@4c009000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x4c009000 0x400>; + clocks = <&rcc TIM14_K>; + clock-names = "int"; + status = "disabled"; + }; + + timers15: timer@4c00a000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x4c00a000 0x400>; + interrupts = ; + clocks = <&rcc TIM15_K>; + clock-names = "int"; + status = "disabled"; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + timers16: timer@4c00b000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x4c00b000 0x400>; + clocks = <&rcc TIM16_K>; + clock-names = "int"; + status = "disabled"; + + }; + + timers17: timer@4c00c000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x4c00c000 0x400>; + clocks = <&rcc TIM17_K>; + clock-names = "int"; + status = "disabled"; + }; + + lptimer2: timer@50021000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-lptimer"; + reg = <0x50021000 0x400>; + interrupts = ; + clocks = <&rcc LPTIM2_K>; + clock-names = "mux"; + status = "disabled"; + }; + + lptimer3: timer@50022000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-lptimer"; + reg = <0x50022000 0x400>; + interrupts = ; + clocks = <&rcc LPTIM3_K>; + clock-names = "mux"; + status = "disabled"; + + counter { + compatible = "st,stm32-lptimer-counter"; + status = "disabled"; + }; + }; + + vrefbuf: vrefbuf@50025000 { + compatible = "st,stm32mp13-vrefbuf"; + reg = <0x50025000 0x8>; + regulator-name = "vrefbuf"; + regulator-min-microvolt = <1650000>; + regulator-max-microvolt = <2500000>; + clocks = <&rcc VREF>; + status = "disabled"; + }; + + hash: hash@54003000 { + compatible = "st,stm32mp13-hash"; + reg = <0x54003000 0x400>; + interrupts = ; + clocks = <&rcc HASH1>; + resets = <&rcc HASH1_R>; + status = "disabled"; + }; + + rng: rng@54004000 { + compatible = "st,stm32mp13-rng"; + reg = <0x54004000 0x400>; + clocks = <&rcc RNG1_K>; + resets = <&rcc RNG1_R>; + status = "disabled"; + }; + + iwdg1: watchdog@5c003000 { + compatible = "st,stm32mp1-iwdg"; + reg = <0x5C003000 0x400>; + interrupts = ; + clocks = <&rcc IWDG1>, <&rcc CK_LSI>; + clock-names = "pclk", "lsi"; + status = "disabled"; + }; + + stgen: stgen@5c008000 { + compatible = "st,stm32-stgen"; + reg = <0x5C008000 0x1000>; + }; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp133.dtsi b/optee_os/core/arch/arm/dts/stm32mp133.dtsi new file mode 100644 index 0000000..d69614e --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp133.dtsi @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2021-2022 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +#include "stm32mp131.dtsi" diff --git a/optee_os/core/arch/arm/dts/stm32mp135.dtsi b/optee_os/core/arch/arm/dts/stm32mp135.dtsi new file mode 100644 index 0000000..1e790f9 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp135.dtsi @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2021-2022 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +#include "stm32mp133.dtsi" + +/ { + soc { + etzpc: etzpc@5c007000 { + ltdc: display-controller@5a001000 { + compatible = "st,stm32-ltdc"; + reg = <0x5a001000 0x400>; + interrupts = , + ; + clocks = <&rcc LTDC_PX>; + clock-names = "lcd"; + resets = <&rcc LTDC_R>; + status = "disabled"; + }; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp135f-dk.dts b/optee_os/core/arch/arm/dts/stm32mp135f-dk.dts new file mode 100644 index 0000000..af22a61 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp135f-dk.dts @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2021-2022 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +/dts-v1/; + +#include +#include +#include +#include "stm32mp135.dtsi" +#include "stm32mp13xf.dtsi" +#include "stm32mp13-pinctrl.dtsi" + +/ { + model = "STMicroelectronics STM32MP135F-DK Discovery Board"; + compatible = "st,stm32mp135f-dk", "st,stm32mp135"; + + aliases { + serial0 = &uart4; + serial1 = &usart1; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory@c0000000 { + device_type = "memory"; + reg = <0xc0000000 0x20000000>; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + optee_framebuffer: optee-framebuffer@dd000000 { + /* Secure framebuffer memory */ + reg = <0xdd000000 0x1000000>; + no-map; + }; + }; + + vin: vin { + compatible = "regulator-fixed"; + regulator-name = "vin"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; + + v3v3_ao: v3v3_ao { + compatible = "regulator-fixed"; + regulator-name = "v3v3_ao"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; +}; + +&adc_2 { + vdda-supply = <&vdd_adc>; + vref-supply = <&vdd_adc>; + status = "okay"; + + adc2: adc@0 { + status = "okay"; + + channel@15 { + reg = <15>; + label = "vbat"; + }; + }; +}; + +&bsec { + board_id: board_id@f0 { + reg = <0xf0 0x4>; + st,non-secure-otp; + }; +}; + +&i2c4 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + clock-frequency = <400000>; + status = "okay"; + + pmic: stpmic@33 { + compatible = "st,stpmic1"; + reg = <0x33>; + status = "okay"; + st,wakeup-pin-number = <1>; + st,notif-it-id = <0>; + + regulators { + compatible = "st,stpmic1-regulators"; + buck1-supply = <&vin>; + buck2-supply = <&vin>; + buck3-supply = <&vin>; + buck4-supply = <&vin>; + ldo1-supply = <&vin>; + ldo4-supply = <&vin>; + ldo5-supply = <&vin>; + ldo6-supply = <&vin>; + vref_ddr-supply = <&vin>; + pwr_sw1-supply = <&bst_out>; + pwr_sw2-supply = <&v3v3_ao>; + + vddcpu: buck1 { + regulator-name = "vddcpu"; + regulator-min-microvolt = <1250000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-over-current-protection; + + lp-stop { + regulator-suspend-microvolt = <1250000>; + }; + lplv-stop { + regulator-suspend-microvolt = <900000>; + }; + lplv-stop2 { + regulator-off-in-suspend; + }; + standby-ddr-sr { + regulator-off-in-suspend; + }; + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + vdd_ddr: buck2 { + regulator-name = "vdd_ddr"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-over-current-protection; + + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + vdd: buck3 { + regulator-name = "vdd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + st,mask-reset; + regulator-over-current-protection; + }; + + vddcore: buck4 { + regulator-name = "vddcore"; + regulator-min-microvolt = <1250000>; + regulator-max-microvolt = <1250000>; + regulator-always-on; + regulator-over-current-protection; + + lplv-stop { + regulator-suspend-microvolt = <900000>; + }; + lplv-stop2 { + regulator-suspend-microvolt = <900000>; + }; + standby-ddr-sr { + regulator-off-in-suspend; + }; + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + vdd_adc: ldo1 { + regulator-name = "vdd_adc"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + standby-ddr-sr { + regulator-off-in-suspend; + }; + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + unused1: ldo2 { + regulator-name = "ldo2"; + }; + + unused2: ldo3 { + regulator-name = "ldo3"; + }; + + vdd_usb: ldo4 { + regulator-name = "vdd_usb"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + standby-ddr-sr { + regulator-off-in-suspend; + }; + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + vdd_sd: ldo5 { + regulator-name = "vdd_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + + standby-ddr-sr { + regulator-off-in-suspend; + }; + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + v1v8_periph: ldo6 { + regulator-name = "v1v8_periph"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + standby-ddr-sr { + regulator-off-in-suspend; + }; + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + vref_ddr: vref_ddr { + regulator-name = "vref_ddr"; + regulator-always-on; + + standby-ddr-sr { + regulator-off-in-suspend; + }; + standby-ddr-off { + regulator-off-in-suspend; + }; + }; + + bst_out: boost { + regulator-name = "bst_out"; + }; + + v3v3_sw: pwr_sw2 { + regulator-name = "v3v3_sw"; + regulator-active-discharge = <1>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + }; + }; +}; + +&iwdg1 { + timeout-sec = <32>; + status = "okay"; +}; + +&oem_enc_key { + st,non-secure-otp-provisioning; +}; + +&pwr_regulators { + vdd-supply = <&vdd>; + vdd_3v3_usbfs-supply = <&vdd_usb>; +}; + +&rcc { + compatible = "st,stm32mp13-rcc", "syscon"; + + st,clksrc = < + CLK_MPU_PLL1P + CLK_AXI_PLL2P + CLK_MLAHBS_PLL3 + CLK_RTC_LSE + CLK_MCO1_HSE + CLK_MCO2_DISABLED + CLK_CKPER_HSE + CLK_ETH1_PLL4P + CLK_ETH2_PLL4P + CLK_SDMMC1_PLL4P + CLK_SDMMC2_PLL4P + CLK_STGEN_HSE + CLK_USBPHY_HSE + CLK_I2C4_HSI + CLK_I2C5_HSI + CLK_USBO_USBPHY + CLK_ADC2_CKPER + CLK_I2C12_HSI + CLK_UART1_HSI + CLK_UART2_HSI + CLK_UART35_HSI + CLK_UART4_HSI + CLK_UART6_HSI + CLK_UART78_HSI + CLK_SAES_AXI + CLK_DCMIPP_PLL2Q + CLK_LPTIM3_PCLK3 + CLK_RNG1_PLL4R + >; + + st,clkdiv = < + DIV(DIV_MPU, 1) + DIV(DIV_AXI, 0) + DIV(DIV_MLAHB, 0) + DIV(DIV_APB1, 1) + DIV(DIV_APB2, 1) + DIV(DIV_APB3, 1) + DIV(DIV_APB4, 1) + DIV(DIV_APB5, 2) + DIV(DIV_APB6, 1) + DIV(DIV_RTC, 0) + DIV(DIV_MCO1, 0) + DIV(DIV_MCO2, 0) + >; + + st,pll_vco { + pll1_vco_2000Mhz: pll1-vco-2000Mhz { + src = ; + divmn = <1 82>; + frac = <0xAAA>; + }; + + pll1_vco_1300Mhz: pll1-vco-1300Mhz { + src = ; + divmn = <2 80>; + frac = <0x800>; + }; + + pll2_vco_1066Mhz: pll2-vco-1066Mhz { + src = ; + divmn = <2 65>; + frac = <0x1400>; + }; + + pll3_vco_417Mhz: pll3-vco-417Mhz { + src = ; + divmn = <1 33>; + frac = <0x1a04>; + }; + + pll4_vco_600Mhz: pll4-vco-600Mhz { + src = ; + divmn = <1 49>; + }; + }; + + /* VCO = 1300.0 MHz => P = 650 (CPU) */ + pll1: st,pll@0 { + compatible = "st,stm32mp1-pll"; + reg = <0>; + + st,pll = <&pll1_cfg1>; + + pll1_cfg1: pll1_cfg1 { + st,pll_vco = <&pll1_vco_1300Mhz>; + st,pll_div_pqr = <0 1 1>; + }; + + pll1_cfg2: pll1_cfg2 { + st,pll_vco = <&pll1_vco_2000Mhz>; + st,pll_div_pqr = <0 1 1>; + }; + }; + + /* VCO = 1066.0 MHz => P = 266 (AXI), Q = 266, R = 533 (DDR) */ + pll2: st,pll@1 { + compatible = "st,stm32mp1-pll"; + reg = <1>; + + st,pll = <&pll2_cfg1>; + + pll2_cfg1: pll2_cfg1 { + st,pll_vco = <&pll2_vco_1066Mhz>; + st,pll_div_pqr = <1 1 0>; + }; + }; + + /* VCO = 417.8 MHz => P = 209, Q = 24, R = 11 */ + pll3: st,pll@2 { + compatible = "st,stm32mp1-pll"; + reg = <2>; + + st,pll = <&pll3_cfg1>; + + pll3_cfg1: pll3_cfg1 { + st,pll_vco = <&pll3_vco_417Mhz>; + st,pll_div_pqr = <1 16 36>; + }; + }; + + /* VCO = 600.0 MHz => P = 50, Q = 10, R = 50 */ + pll4: st,pll@3 { + compatible = "st,stm32mp1-pll"; + reg = <3>; + st,pll = <&pll4_cfg1>; + + pll4_cfg1: pll4_cfg1 { + st,pll_vco = <&pll4_vco_600Mhz>; + st,pll_div_pqr = <11 59 11>; + }; + }; + + st,clk_opp { + /* CK_MPU clock config for MP13 */ + st,ck_mpu { + + cfg_1 { + hz = <650000000>; + st,clksrc = ; + st,pll = <&pll1_cfg1>; + }; + + cfg_2 { + hz = <1000000000>; + st,clksrc = ; + st,pll = <&pll1_cfg2>; + }; + }; + }; +}; + +&rng { + status = "okay"; + clock-error-detect; +}; + +&rtc { + status = "okay"; +}; + +&saes { + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_pins_a>; + status = "okay"; +}; + +&usart1 { + pinctrl-names = "default"; + pinctrl-0 = <&usart1_pins_a>; + uart-has-rtscts; + status = "disabled"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp13xc.dtsi b/optee_os/core/arch/arm/dts/stm32mp13xc.dtsi new file mode 100644 index 0000000..4527f11 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp13xc.dtsi @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2021-2022 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +&etzpc { + cryp: crypto@54002000 { + compatible = "st,stm32mp1-cryp"; + reg = <0x54002000 0x400>; + interrupts = ; + clocks = <&rcc CRYP1>; + resets = <&rcc CRYP1_R>; + status = "disabled"; + }; + + saes: saes@54005000 { + compatible = "st,stm32mp13-saes"; + reg = <0x54005000 0x400>; + interrupts = ; + clocks = <&rcc SAES_K>; + resets = <&rcc SAES_R>; + status = "disabled"; + }; + + pka: pka@54006000 { + compatible = "st,stm32mp13-pka64"; + reg = <0x54006000 0x2000>; + interrupts = ; + clocks = <&rcc PKA>; + resets = <&rcc PKA_R>; + status = "disabled"; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp13xf.dtsi b/optee_os/core/arch/arm/dts/stm32mp13xf.dtsi new file mode 100644 index 0000000..4527f11 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp13xf.dtsi @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2021-2022 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +&etzpc { + cryp: crypto@54002000 { + compatible = "st,stm32mp1-cryp"; + reg = <0x54002000 0x400>; + interrupts = ; + clocks = <&rcc CRYP1>; + resets = <&rcc CRYP1_R>; + status = "disabled"; + }; + + saes: saes@54005000 { + compatible = "st,stm32mp13-saes"; + reg = <0x54005000 0x400>; + interrupts = ; + clocks = <&rcc SAES_K>; + resets = <&rcc SAES_R>; + status = "disabled"; + }; + + pka: pka@54006000 { + compatible = "st,stm32mp13-pka64"; + reg = <0x54006000 0x2000>; + interrupts = ; + clocks = <&rcc PKA>; + resets = <&rcc PKA_R>; + status = "disabled"; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15-pinctrl.dtsi b/optee_os/core/arch/arm/dts/stm32mp15-pinctrl.dtsi new file mode 100644 index 0000000..7edff9c --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15-pinctrl.dtsi @@ -0,0 +1,2279 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2017 - All Rights Reserved + * Author: Ludovic Barre for STMicroelectronics. + */ +#include + +&pinctrl { + adc1_in6_pins_a: adc1-in6-0 { + pins { + pinmux = ; + }; + }; + + adc12_ain_pins_a: adc12-ain-0 { + pins { + pinmux = , /* ADC1 in13 */ + , /* ADC1 in6 */ + , /* ADC2 in2 */ + ; /* ADC2 in6 */ + }; + }; + + adc12_ain_pins_b: adc12-ain-1 { + pins { + pinmux = , /* ADC1 in6 */ + ; /* ADC2 in2 */ + }; + }; + + adc12_usb_cc_pins_a: adc12-usb-cc-pins-0 { + pins { + pinmux = , /* ADC12 in18 */ + ; /* ADC12 in19 */ + }; + }; + + cec_pins_a: cec-0 { + pins { + pinmux = ; + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + cec_sleep_pins_a: cec-sleep-0 { + pins { + pinmux = ; /* HDMI_CEC */ + }; + }; + + cec_pins_b: cec-1 { + pins { + pinmux = ; + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + cec_sleep_pins_b: cec-sleep-1 { + pins { + pinmux = ; /* HDMI_CEC */ + }; + }; + + dac_ch1_pins_a: dac-ch1-0 { + pins { + pinmux = ; + }; + }; + + dac_ch2_pins_a: dac-ch2-0 { + pins { + pinmux = ; + }; + }; + + dcmi_pins_a: dcmi-0 { + pins { + pinmux = ,/* DCMI_HSYNC */ + ,/* DCMI_VSYNC */ + ,/* DCMI_PIXCLK */ + ,/* DCMI_D0 */ + ,/* DCMI_D1 */ + ,/* DCMI_D2 */ + ,/* DCMI_D3 */ + ,/* DCMI_D4 */ + ,/* DCMI_D5 */ + ,/* DCMI_D6 */ + ,/* DCMI_D7 */ + ,/* DCMI_D8 */ + ,/* DCMI_D9 */ + ,/* DCMI_D10 */ + ;/* DCMI_D11 */ + bias-disable; + }; + }; + + dcmi_sleep_pins_a: dcmi-sleep-0 { + pins { + pinmux = ,/* DCMI_HSYNC */ + ,/* DCMI_VSYNC */ + ,/* DCMI_PIXCLK */ + ,/* DCMI_D0 */ + ,/* DCMI_D1 */ + ,/* DCMI_D2 */ + ,/* DCMI_D3 */ + ,/* DCMI_D4 */ + ,/* DCMI_D5 */ + ,/* DCMI_D6 */ + ,/* DCMI_D7 */ + ,/* DCMI_D8 */ + ,/* DCMI_D9 */ + ,/* DCMI_D10 */ + ;/* DCMI_D11 */ + }; + }; + + dcmi_pins_b: dcmi-1 { + pins { + pinmux = ,/* DCMI_HSYNC */ + ,/* DCMI_VSYNC */ + ,/* DCMI_PIXCLK */ + ,/* DCMI_D0 */ + ,/* DCMI_D1 */ + ,/* DCMI_D2 */ + ,/* DCMI_D3 */ + ,/* DCMI_D4 */ + ,/* DCMI_D5 */ + ,/* DCMI_D6 */ + ;/* DCMI_D7 */ + bias-disable; + }; + }; + + dcmi_sleep_pins_b: dcmi-sleep-1 { + pins { + pinmux = ,/* DCMI_HSYNC */ + ,/* DCMI_VSYNC */ + ,/* DCMI_PIXCLK */ + ,/* DCMI_D0 */ + ,/* DCMI_D1 */ + ,/* DCMI_D2 */ + ,/* DCMI_D3 */ + ,/* DCMI_D4 */ + ,/* DCMI_D5 */ + ,/* DCMI_D6 */ + ;/* DCMI_D7 */ + }; + }; + + ethernet0_rgmii_pins_a: rgmii-0 { + pins1 { + pinmux = , /* ETH_RGMII_CLK125 */ + , /* ETH_RGMII_GTX_CLK */ + , /* ETH_RGMII_TXD0 */ + , /* ETH_RGMII_TXD1 */ + , /* ETH_RGMII_TXD2 */ + , /* ETH_RGMII_TXD3 */ + , /* ETH_RGMII_TX_CTL */ + ; /* ETH_MDC */ + bias-disable; + drive-push-pull; + slew-rate = <2>; + }; + pins2 { + pinmux = ; /* ETH_MDIO */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { + pinmux = , /* ETH_RGMII_RXD0 */ + , /* ETH_RGMII_RXD1 */ + , /* ETH_RGMII_RXD2 */ + , /* ETH_RGMII_RXD3 */ + , /* ETH_RGMII_RX_CLK */ + ; /* ETH_RGMII_RX_CTL */ + bias-disable; + }; + }; + + ethernet0_rgmii_sleep_pins_a: rgmii-sleep-0 { + pins1 { + pinmux = , /* ETH_RGMII_CLK125 */ + , /* ETH_RGMII_GTX_CLK */ + , /* ETH_RGMII_TXD0 */ + , /* ETH_RGMII_TXD1 */ + , /* ETH_RGMII_TXD2 */ + , /* ETH_RGMII_TXD3 */ + , /* ETH_RGMII_TX_CTL */ + , /* ETH_MDIO */ + , /* ETH_MDC */ + , /* ETH_RGMII_RXD0 */ + , /* ETH_RGMII_RXD1 */ + , /* ETH_RGMII_RXD2 */ + , /* ETH_RGMII_RXD3 */ + , /* ETH_RGMII_RX_CLK */ + ; /* ETH_RGMII_RX_CTL */ + }; + }; + + ethernet0_rgmii_pins_b: rgmii-1 { + pins1 { + pinmux = , /* ETH_RGMII_CLK125 */ + , /* ETH_RGMII_GTX_CLK */ + , /* ETH_RGMII_TXD0 */ + , /* ETH_RGMII_TXD1 */ + , /* ETH_RGMII_TXD2 */ + , /* ETH_RGMII_TXD3 */ + , /* ETH_RGMII_TX_CTL */ + ; /* ETH_MDC */ + bias-disable; + drive-push-pull; + slew-rate = <2>; + }; + pins2 { + pinmux = ; /* ETH_MDIO */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { + pinmux = , /* ETH_RGMII_RXD0 */ + , /* ETH_RGMII_RXD1 */ + , /* ETH_RGMII_RXD2 */ + , /* ETH_RGMII_RXD3 */ + , /* ETH_RGMII_RX_CLK */ + ; /* ETH_RGMII_RX_CTL */ + bias-disable; + }; + }; + + ethernet0_rgmii_sleep_pins_b: rgmii-sleep-1 { + pins1 { + pinmux = , /* ETH_RGMII_CLK125 */ + , /* ETH_RGMII_GTX_CLK */ + , /* ETH_RGMII_TXD0 */ + , /* ETH_RGMII_TXD1 */ + , /* ETH_RGMII_TXD2 */ + , /* ETH_RGMII_TXD3 */ + , /* ETH_RGMII_TX_CTL */ + , /* ETH_MDC */ + , /* ETH_MDIO */ + , /* ETH_RGMII_RXD0 */ + , /* ETH_RGMII_RXD1 */ + , /* ETH_RGMII_RXD2 */ + , /* ETH_RGMII_RXD3 */ + , /* ETH_RGMII_RX_CLK */ + ; /* ETH_RGMII_RX_CTL */ + }; + }; + + ethernet0_rgmii_pins_c: rgmii-2 { + pins1 { + pinmux = , /* ETH_RGMII_CLK125 */ + , /* ETH_RGMII_GTX_CLK */ + , /* ETH_RGMII_TXD0 */ + , /* ETH_RGMII_TXD1 */ + , /* ETH_RGMII_TXD2 */ + , /* ETH_RGMII_TXD3 */ + , /* ETH_RGMII_TX_CTL */ + ; /* ETH_MDC */ + bias-disable; + drive-push-pull; + slew-rate = <2>; + }; + pins2 { + pinmux = ; /* ETH_MDIO */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { + pinmux = , /* ETH_RGMII_RXD0 */ + , /* ETH_RGMII_RXD1 */ + , /* ETH_RGMII_RXD2 */ + , /* ETH_RGMII_RXD3 */ + , /* ETH_RGMII_RX_CLK */ + ; /* ETH_RGMII_RX_CTL */ + bias-disable; + }; + }; + + ethernet0_rgmii_sleep_pins_c: rgmii-sleep-2 { + pins1 { + pinmux = , /* ETH_RGMII_CLK125 */ + , /* ETH_RGMII_GTX_CLK */ + , /* ETH_RGMII_TXD0 */ + , /* ETH_RGMII_TXD1 */ + , /* ETH_RGMII_TXD2 */ + , /* ETH_RGMII_TXD3 */ + , /* ETH_RGMII_TX_CTL */ + , /* ETH_MDIO */ + , /* ETH_MDC */ + , /* ETH_RGMII_RXD0 */ + , /* ETH_RGMII_RXD1 */ + , /* ETH_RGMII_RXD2 */ + , /* ETH_RGMII_RXD3 */ + , /* ETH_RGMII_RX_CLK */ + ; /* ETH_RGMII_RX_CTL */ + }; + }; + + ethernet0_rmii_pins_a: rmii-0 { + pins1 { + pinmux = , /* ETH1_RMII_TXD0 */ + , /* ETH1_RMII_TXD1 */ + , /* ETH1_RMII_TX_EN */ + , /* ETH1_RMII_REF_CLK */ + , /* ETH1_MDIO */ + ; /* ETH1_MDC */ + bias-disable; + drive-push-pull; + slew-rate = <2>; + }; + pins2 { + pinmux = , /* ETH1_RMII_RXD0 */ + , /* ETH1_RMII_RXD1 */ + ; /* ETH1_RMII_CRS_DV */ + bias-disable; + }; + }; + + ethernet0_rmii_sleep_pins_a: rmii-sleep-0 { + pins1 { + pinmux = , /* ETH1_RMII_TXD0 */ + , /* ETH1_RMII_TXD1 */ + , /* ETH1_RMII_TX_EN */ + , /* ETH1_MDIO */ + , /* ETH1_MDC */ + , /* ETH1_RMII_RXD0 */ + , /* ETH1_RMII_RXD1 */ + , /* ETH1_RMII_REF_CLK */ + ; /* ETH1_RMII_CRS_DV */ + }; + }; + + ethernet0_rmii_pins_b: rmii-1 { + pins1 { + pinmux = , /* ETH1_CLK */ + , /* ETH1_MDC */ + , /* ETH1_TXD0 */ + ; /* ETH1_TXD1 */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + pins2 { + pinmux = ; /* ETH1_MDIO */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { + pinmux = , /* ETH1_CRS_DV */ + , /* ETH1_RXD0 */ + ; /* ETH1_RXD1 */ + bias-disable; + }; + pins4 { + pinmux = ; /* ETH1_TX_EN */ + }; + }; + + ethernet0_rmii_sleep_pins_b: rmii-sleep-1 { + pins1 { + pinmux = , /* ETH1_MDIO */ + , /* ETH1_CRS_DV */ + , /* ETH1_CLK */ + , /* ETH1_TX_EN */ + , /* ETH1_MDC */ + , /* ETH1_RXD0 */ + , /* ETH1_RXD1 */ + , /* ETH1_TXD0 */ + ; /* ETH1_TXD1 */ + }; + }; + + ethernet0_rmii_pins_c: rmii-2 { + pins1 { + pinmux = , /* ETH1_RMII_TXD0 */ + , /* ETH1_RMII_TXD1 */ + , /* ETH1_RMII_TX_EN */ + , /* ETH1_RMII_REF_CLK */ + , /* ETH1_MDIO */ + ; /* ETH1_MDC */ + bias-disable; + drive-push-pull; + slew-rate = <2>; + }; + pins2 { + pinmux = , /* ETH1_RMII_RXD0 */ + , /* ETH1_RMII_RXD1 */ + ; /* ETH1_RMII_CRS_DV */ + bias-disable; + }; + }; + + ethernet0_rmii_sleep_pins_c: rmii-sleep-2 { + pins1 { + pinmux = , /* ETH1_RMII_TXD0 */ + , /* ETH1_RMII_TXD1 */ + , /* ETH1_RMII_TX_EN */ + , /* ETH1_MDIO */ + , /* ETH1_MDC */ + , /* ETH1_RMII_RXD0 */ + , /* ETH1_RMII_RXD1 */ + , /* ETH1_RMII_REF_CLK */ + ; /* ETH1_RMII_CRS_DV */ + }; + }; + + fmc_pins_a: fmc-0 { + pins1 { + pinmux = , /* FMC_NOE */ + , /* FMC_NWE */ + , /* FMC_A16_FMC_CLE */ + , /* FMC_A17_FMC_ALE */ + , /* FMC_D0 */ + , /* FMC_D1 */ + , /* FMC_D2 */ + , /* FMC_D3 */ + , /* FMC_D4 */ + , /* FMC_D5 */ + , /* FMC_D6 */ + , /* FMC_D7 */ + ; /* FMC_NE2_FMC_NCE */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + pins2 { + pinmux = ; /* FMC_NWAIT */ + bias-pull-up; + }; + }; + + fmc_sleep_pins_a: fmc-sleep-0 { + pins { + pinmux = , /* FMC_NOE */ + , /* FMC_NWE */ + , /* FMC_A16_FMC_CLE */ + , /* FMC_A17_FMC_ALE */ + , /* FMC_D0 */ + , /* FMC_D1 */ + , /* FMC_D2 */ + , /* FMC_D3 */ + , /* FMC_D4 */ + , /* FMC_D5 */ + , /* FMC_D6 */ + , /* FMC_D7 */ + , /* FMC_NWAIT */ + ; /* FMC_NE2_FMC_NCE */ + }; + }; + + fmc_pins_b: fmc-1 { + pins { + pinmux = , /* FMC_NOE */ + , /* FMC_NWE */ + , /* FMC_NL */ + , /* FMC_D0 */ + , /* FMC_D1 */ + , /* FMC_D2 */ + , /* FMC_D3 */ + , /* FMC_D4 */ + , /* FMC_D5 */ + , /* FMC_D6 */ + , /* FMC_D7 */ + , /* FMC_D8 */ + , /* FMC_D9 */ + , /* FMC_D10 */ + , /* FMC_D11 */ + , /* FMC_D12 */ + , /* FMC_D13 */ + , /* FMC_D14 */ + , /* FMC_D15 */ + , /* FMC_NE2_FMC_NCE */ + ; /* FMC_NE4 */ + bias-disable; + drive-push-pull; + slew-rate = <3>; + }; + }; + + fmc_sleep_pins_b: fmc-sleep-1 { + pins { + pinmux = , /* FMC_NOE */ + , /* FMC_NWE */ + , /* FMC_NL */ + , /* FMC_D0 */ + , /* FMC_D1 */ + , /* FMC_D2 */ + , /* FMC_D3 */ + , /* FMC_D4 */ + , /* FMC_D5 */ + , /* FMC_D6 */ + , /* FMC_D7 */ + , /* FMC_D8 */ + , /* FMC_D9 */ + , /* FMC_D10 */ + , /* FMC_D11 */ + , /* FMC_D12 */ + , /* FMC_D13 */ + , /* FMC_D14 */ + , /* FMC_D15 */ + , /* FMC_NE2_FMC_NCE */ + ; /* FMC_NE4 */ + }; + }; + + i2c1_pins_a: i2c1-0 { + pins { + pinmux = , /* I2C1_SCL */ + ; /* I2C1_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c1_sleep_pins_a: i2c1-sleep-0 { + pins { + pinmux = , /* I2C1_SCL */ + ; /* I2C1_SDA */ + }; + }; + + i2c1_pins_b: i2c1-1 { + pins { + pinmux = , /* I2C1_SCL */ + ; /* I2C1_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c1_sleep_pins_b: i2c1-sleep-1 { + pins { + pinmux = , /* I2C1_SCL */ + ; /* I2C1_SDA */ + }; + }; + + i2c2_pins_a: i2c2-0 { + pins { + pinmux = , /* I2C2_SCL */ + ; /* I2C2_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c2_sleep_pins_a: i2c2-sleep-0 { + pins { + pinmux = , /* I2C2_SCL */ + ; /* I2C2_SDA */ + }; + }; + + i2c2_pins_b1: i2c2-1 { + pins { + pinmux = ; /* I2C2_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c2_sleep_pins_b1: i2c2-sleep-1 { + pins { + pinmux = ; /* I2C2_SDA */ + }; + }; + + i2c2_pins_c: i2c2-2 { + pins { + pinmux = , /* I2C2_SCL */ + ; /* I2C2_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c2_pins_sleep_c: i2c2-sleep-2 { + pins { + pinmux = , /* I2C2_SCL */ + ; /* I2C2_SDA */ + }; + }; + + i2c5_pins_a: i2c5-0 { + pins { + pinmux = , /* I2C5_SCL */ + ; /* I2C5_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c5_sleep_pins_a: i2c5-sleep-0 { + pins { + pinmux = , /* I2C5_SCL */ + ; /* I2C5_SDA */ + + }; + }; + + i2c5_pins_b: i2c5-1 { + pins { + pinmux = , /* I2C5_SCL */ + ; /* I2C5_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c5_sleep_pins_b: i2c5-sleep-1 { + pins { + pinmux = , /* I2C5_SCL */ + ; /* I2C5_SDA */ + }; + }; + + i2s2_pins_a: i2s2-0 { + pins { + pinmux = , /* I2S2_SDO */ + , /* I2S2_WS */ + ; /* I2S2_CK */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + }; + + i2s2_sleep_pins_a: i2s2-sleep-0 { + pins { + pinmux = , /* I2S2_SDO */ + , /* I2S2_WS */ + ; /* I2S2_CK */ + }; + }; + + ltdc_pins_a: ltdc-0 { + pins { + pinmux = , /* LCD_CLK */ + , /* LCD_HSYNC */ + , /* LCD_VSYNC */ + , /* LCD_DE */ + , /* LCD_R0 */ + , /* LCD_R1 */ + , /* LCD_R2 */ + , /* LCD_R3 */ + , /* LCD_R4 */ + , /* LCD_R5 */ + , /* LCD_R6 */ + , /* LCD_R7 */ + , /* LCD_G0 */ + , /* LCD_G1 */ + , /* LCD_G2 */ + , /* LCD_G3 */ + , /* LCD_G4 */ + , /* LCD_G5 */ + , /* LCD_G6 */ + , /* LCD_G7 */ + , /* LCD_B0 */ + , /* LCD_B1 */ + , /* LCD_B2 */ + , /* LCD_B3 */ + , /* LCD_B4 */ + , /* LCD_B5 */ + , /* LCD_B6 */ + ; /* LCD_B7 */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + }; + + ltdc_sleep_pins_a: ltdc-sleep-0 { + pins { + pinmux = , /* LCD_CLK */ + , /* LCD_HSYNC */ + , /* LCD_VSYNC */ + , /* LCD_DE */ + , /* LCD_R0 */ + , /* LCD_R1 */ + , /* LCD_R2 */ + , /* LCD_R3 */ + , /* LCD_R4 */ + , /* LCD_R5 */ + , /* LCD_R6 */ + , /* LCD_R7 */ + , /* LCD_G0 */ + , /* LCD_G1 */ + , /* LCD_G2 */ + , /* LCD_G3 */ + , /* LCD_G4 */ + , /* LCD_G5 */ + , /* LCD_G6 */ + , /* LCD_G7 */ + , /* LCD_B0 */ + , /* LCD_B1 */ + , /* LCD_B2 */ + , /* LCD_B3 */ + , /* LCD_B4 */ + , /* LCD_B5 */ + , /* LCD_B6 */ + ; /* LCD_B7 */ + }; + }; + + ltdc_pins_b: ltdc-1 { + pins { + pinmux = , /* LCD_CLK */ + , /* LCD_HSYNC */ + , /* LCD_VSYNC */ + , /* LCD_DE */ + , /* LCD_R0 */ + , /* LCD_R1 */ + , /* LCD_R2 */ + , /* LCD_R3 */ + , /* LCD_R4 */ + , /* LCD_R5 */ + , /* LCD_R6 */ + , /* LCD_R7 */ + , /* LCD_G0 */ + , /* LCD_G1 */ + , /* LCD_G2 */ + , /* LCD_G3 */ + , /* LCD_G4 */ + , /* LCD_G5 */ + , /* LCD_G6 */ + , /* LCD_G7 */ + , /* LCD_B0 */ + , /* LCD_B1 */ + , /* LCD_B2 */ + , /* LCD_B3 */ + , /* LCD_B4 */ + , /* LCD_B5 */ + , /* LCD_B6 */ + ; /* LCD_B7 */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + }; + + ltdc_sleep_pins_b: ltdc-sleep-1 { + pins { + pinmux = , /* LCD_CLK */ + , /* LCD_HSYNC */ + , /* LCD_VSYNC */ + , /* LCD_DE */ + , /* LCD_R0 */ + , /* LCD_R1 */ + , /* LCD_R2 */ + , /* LCD_R3 */ + , /* LCD_R4 */ + , /* LCD_R5 */ + , /* LCD_R6 */ + , /* LCD_R7 */ + , /* LCD_G0 */ + , /* LCD_G1 */ + , /* LCD_G2 */ + , /* LCD_G3 */ + , /* LCD_G4 */ + , /* LCD_G5 */ + , /* LCD_G6 */ + , /* LCD_G7 */ + , /* LCD_B0 */ + , /* LCD_B1 */ + , /* LCD_B2 */ + , /* LCD_B3 */ + , /* LCD_B4 */ + , /* LCD_B5 */ + , /* LCD_B6 */ + ; /* LCD_B7 */ + }; + }; + + ltdc_pins_c: ltdc-2 { + pins1 { + pinmux = , /* LTDC_R6 */ + , /* LTDC_B7 */ + , /* LTDC_R5 */ + , /* LTDC_G7 */ + , /* LTDC_B2 */ + , /* LTDC_B3 */ + , /* LTDC_G3 */ + , /* LTDC_B4 */ + , /* LTDC_DE */ + , /* LTDC_R7 */ + , /* LTDC_G5 */ + , /* LTDC_R2 */ + , /* LTDC_R3 */ + , /* LTDC_R4 */ + , /* LTDC_G2 */ + , /* LTDC_G4 */ + , /* LTDC_G6 */ + , /* LTDC_B5 */ + , /* LTDC_B6 */ + , /* LTDC_VSYNC */ + ; /* LTDC_HSYNC */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* LTDC_CLK */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + }; + + ltdc_sleep_pins_c: ltdc-sleep-2 { + pins1 { + pinmux = , /* LTDC_R6 */ + , /* LTDC_B7 */ + , /* LTDC_R5 */ + , /* LTDC_G7 */ + , /* LTDC_B2 */ + , /* LTDC_B3 */ + , /* LTDC_G3 */ + , /* LTDC_B4 */ + , /* LTDC_DE */ + , /* LTDC_R7 */ + , /* LTDC_G5 */ + , /* LTDC_R2 */ + , /* LTDC_R3 */ + , /* LTDC_R4 */ + , /* LTDC_G2 */ + , /* LTDC_G4 */ + , /* LTDC_G6 */ + , /* LTDC_B5 */ + , /* LTDC_B6 */ + , /* LTDC_VSYNC */ + , /* LTDC_HSYNC */ + ; /* LTDC_CLK */ + }; + }; + + ltdc_pins_d: ltdc-3 { + pins1 { + pinmux = ; /* LCD_CLK */ + bias-disable; + drive-push-pull; + slew-rate = <3>; + }; + pins2 { + pinmux = , /* LCD_HSYNC */ + , /* LCD_VSYNC */ + , /* LCD_DE */ + , /* LCD_R0 */ + , /* LCD_R1 */ + , /* LCD_R2 */ + , /* LCD_R3 */ + , /* LCD_R4 */ + , /* LCD_R5 */ + , /* LCD_R6 */ + , /* LCD_R7 */ + , /* LCD_G0 */ + , /* LCD_G1 */ + , /* LCD_G2 */ + , /* LCD_G3 */ + , /* LCD_G4 */ + , /* LCD_G5 */ + , /* LCD_G6 */ + , /* LCD_G7 */ + , /* LCD_B0 */ + , /* LCD_B1 */ + , /* LCD_B2 */ + , /* LCD_B3 */ + , /* LCD_B4 */ + , /* LCD_B5 */ + , /* LCD_B6 */ + ; /* LCD_B7 */ + bias-disable; + drive-push-pull; + slew-rate = <2>; + }; + }; + + ltdc_sleep_pins_d: ltdc-sleep-3 { + pins { + pinmux = , /* LCD_CLK */ + , /* LCD_HSYNC */ + , /* LCD_VSYNC */ + , /* LCD_DE */ + , /* LCD_R0 */ + , /* LCD_R1 */ + , /* LCD_R2 */ + , /* LCD_R3 */ + , /* LCD_R4 */ + , /* LCD_R5 */ + , /* LCD_R6 */ + , /* LCD_R7 */ + , /* LCD_G0 */ + , /* LCD_G1 */ + , /* LCD_G2 */ + , /* LCD_G3 */ + , /* LCD_G4 */ + , /* LCD_G5 */ + , /* LCD_G6 */ + , /* LCD_G7 */ + , /* LCD_B0 */ + , /* LCD_B1 */ + , /* LCD_B2 */ + , /* LCD_B3 */ + , /* LCD_B4 */ + , /* LCD_B5 */ + , /* LCD_B6 */ + ; /* LCD_B7 */ + }; + }; + + mco2_pins_a: mco2-0 { + pins { + pinmux = ; /* MCO2 */ + bias-disable; + drive-push-pull; + slew-rate = <2>; + }; + }; + + mco2_sleep_pins_a: mco2-sleep-0 { + pins { + pinmux = ; /* MCO2 */ + }; + }; + + m_can1_pins_a: m-can1-0 { + pins1 { + pinmux = ; /* CAN1_TX */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* CAN1_RX */ + bias-disable; + }; + }; + + m_can1_sleep_pins_a: m_can1-sleep-0 { + pins { + pinmux = , /* CAN1_TX */ + ; /* CAN1_RX */ + }; + }; + + m_can1_pins_b: m-can1-1 { + pins1 { + pinmux = ; /* CAN1_TX */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* CAN1_RX */ + bias-disable; + }; + }; + + m_can1_sleep_pins_b: m_can1-sleep-1 { + pins { + pinmux = , /* CAN1_TX */ + ; /* CAN1_RX */ + }; + }; + + m_can2_pins_a: m-can2-0 { + pins1 { + pinmux = ; /* CAN2_TX */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* CAN2_RX */ + bias-disable; + }; + }; + + m_can2_sleep_pins_a: m_can2-sleep-0 { + pins { + pinmux = , /* CAN2_TX */ + ; /* CAN2_RX */ + }; + }; + + pwm1_pins_a: pwm1-0 { + pins { + pinmux = , /* TIM1_CH1 */ + , /* TIM1_CH2 */ + ; /* TIM1_CH4 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm1_sleep_pins_a: pwm1-sleep-0 { + pins { + pinmux = , /* TIM1_CH1 */ + , /* TIM1_CH2 */ + ; /* TIM1_CH4 */ + }; + }; + + pwm1_pins_b: pwm1-1 { + pins { + pinmux = ; /* TIM1_CH1 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm1_sleep_pins_b: pwm1-sleep-1 { + pins { + pinmux = ; /* TIM1_CH1 */ + }; + }; + + pwm2_pins_a: pwm2-0 { + pins { + pinmux = ; /* TIM2_CH4 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm2_sleep_pins_a: pwm2-sleep-0 { + pins { + pinmux = ; /* TIM2_CH4 */ + }; + }; + + pwm3_pins_a: pwm3-0 { + pins { + pinmux = ; /* TIM3_CH2 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm3_sleep_pins_a: pwm3-sleep-0 { + pins { + pinmux = ; /* TIM3_CH2 */ + }; + }; + + pwm3_pins_b: pwm3-1 { + pins { + pinmux = ; /* TIM3_CH2 */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm3_sleep_pins_b: pwm3-sleep-1 { + pins { + pinmux = ; /* TIM3_CH2 */ + }; + }; + + pwm4_pins_a: pwm4-0 { + pins { + pinmux = , /* TIM4_CH3 */ + ; /* TIM4_CH4 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm4_sleep_pins_a: pwm4-sleep-0 { + pins { + pinmux = , /* TIM4_CH3 */ + ; /* TIM4_CH4 */ + }; + }; + + pwm4_pins_b: pwm4-1 { + pins { + pinmux = ; /* TIM4_CH2 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm4_sleep_pins_b: pwm4-sleep-1 { + pins { + pinmux = ; /* TIM4_CH2 */ + }; + }; + + pwm5_pins_a: pwm5-0 { + pins { + pinmux = ; /* TIM5_CH2 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm5_sleep_pins_a: pwm5-sleep-0 { + pins { + pinmux = ; /* TIM5_CH2 */ + }; + }; + + pwm5_pins_b: pwm5-1 { + pins { + pinmux = , /* TIM5_CH2 */ + , /* TIM5_CH3 */ + ; /* TIM5_CH4 */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm5_sleep_pins_b: pwm5-sleep-1 { + pins { + pinmux = , /* TIM5_CH2 */ + , /* TIM5_CH3 */ + ; /* TIM5_CH4 */ + }; + }; + + pwm8_pins_a: pwm8-0 { + pins { + pinmux = ; /* TIM8_CH4 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm8_sleep_pins_a: pwm8-sleep-0 { + pins { + pinmux = ; /* TIM8_CH4 */ + }; + }; + + pwm12_pins_a: pwm12-0 { + pins { + pinmux = ; /* TIM12_CH1 */ + bias-pull-down; + drive-push-pull; + slew-rate = <0>; + }; + }; + + pwm12_sleep_pins_a: pwm12-sleep-0 { + pins { + pinmux = ; /* TIM12_CH1 */ + }; + }; + + qspi_clk_pins_a: qspi-clk-0 { + pins { + pinmux = ; /* QSPI_CLK */ + bias-disable; + drive-push-pull; + slew-rate = <3>; + }; + }; + + qspi_clk_sleep_pins_a: qspi-clk-sleep-0 { + pins { + pinmux = ; /* QSPI_CLK */ + }; + }; + + qspi_bk1_pins_a: qspi-bk1-0 { + pins1 { + pinmux = , /* QSPI_BK1_IO0 */ + , /* QSPI_BK1_IO1 */ + , /* QSPI_BK1_IO2 */ + ; /* QSPI_BK1_IO3 */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + pins2 { + pinmux = ; /* QSPI_BK1_NCS */ + bias-pull-up; + drive-push-pull; + slew-rate = <1>; + }; + }; + + qspi_bk1_sleep_pins_a: qspi-bk1-sleep-0 { + pins { + pinmux = , /* QSPI_BK1_IO0 */ + , /* QSPI_BK1_IO1 */ + , /* QSPI_BK1_IO2 */ + , /* QSPI_BK1_IO3 */ + ; /* QSPI_BK1_NCS */ + }; + }; + + qspi_bk2_pins_a: qspi-bk2-0 { + pins1 { + pinmux = , /* QSPI_BK2_IO0 */ + , /* QSPI_BK2_IO1 */ + , /* QSPI_BK2_IO2 */ + ; /* QSPI_BK2_IO3 */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + pins2 { + pinmux = ; /* QSPI_BK2_NCS */ + bias-pull-up; + drive-push-pull; + slew-rate = <1>; + }; + }; + + qspi_bk2_sleep_pins_a: qspi-bk2-sleep-0 { + pins { + pinmux = , /* QSPI_BK2_IO0 */ + , /* QSPI_BK2_IO1 */ + , /* QSPI_BK2_IO2 */ + , /* QSPI_BK2_IO3 */ + ; /* QSPI_BK2_NCS */ + }; + }; + + sai2a_pins_a: sai2a-0 { + pins { + pinmux = , /* SAI2_SCK_A */ + , /* SAI2_SD_A */ + , /* SAI2_FS_A */ + ; /* SAI2_MCLK_A */ + slew-rate = <0>; + drive-push-pull; + bias-disable; + }; + }; + + sai2a_sleep_pins_a: sai2a-sleep-0 { + pins { + pinmux = , /* SAI2_SCK_A */ + , /* SAI2_SD_A */ + , /* SAI2_FS_A */ + ; /* SAI2_MCLK_A */ + }; + }; + + sai2a_pins_b: sai2a-1 { + pins1 { + pinmux = , /* SAI2_SD_A */ + , /* SAI2_FS_A */ + ; /* SAI2_SCK_A */ + slew-rate = <0>; + drive-push-pull; + bias-disable; + }; + }; + + sai2a_sleep_pins_b: sai2a-sleep-1 { + pins { + pinmux = , /* SAI2_SD_A */ + , /* SAI2_FS_A */ + ; /* SAI2_SCK_A */ + }; + }; + + sai2a_pins_c: sai2a-2 { + pins { + pinmux = , /* SAI2_SCK_A */ + , /* SAI2_SD_A */ + ; /* SAI2_FS_A */ + slew-rate = <0>; + drive-push-pull; + bias-disable; + }; + }; + + sai2a_sleep_pins_c: sai2a-sleep-2 { + pins { + pinmux = , /* SAI2_SCK_A */ + , /* SAI2_SD_A */ + ; /* SAI2_FS_A */ + }; + }; + + sai2b_pins_a: sai2b-0 { + pins1 { + pinmux = , /* SAI2_SCK_B */ + , /* SAI2_FS_B */ + ; /* SAI2_MCLK_B */ + slew-rate = <0>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* SAI2_SD_B */ + bias-disable; + }; + }; + + sai2b_sleep_pins_a: sai2b-sleep-0 { + pins { + pinmux = , /* SAI2_SD_B */ + , /* SAI2_SCK_B */ + , /* SAI2_FS_B */ + ; /* SAI2_MCLK_B */ + }; + }; + + sai2b_pins_b: sai2b-1 { + pins { + pinmux = ; /* SAI2_SD_B */ + bias-disable; + }; + }; + + sai2b_sleep_pins_b: sai2b-sleep-1 { + pins { + pinmux = ; /* SAI2_SD_B */ + }; + }; + + sai2b_pins_c: sai2b-2 { + pins1 { + pinmux = ; /* SAI2_SD_B */ + bias-disable; + }; + }; + + sai2b_sleep_pins_c: sai2b-sleep-2 { + pins { + pinmux = ; /* SAI2_SD_B */ + }; + }; + + sai4a_pins_a: sai4a-0 { + pins { + pinmux = ; /* SAI4_SD_A */ + slew-rate = <0>; + drive-push-pull; + bias-disable; + }; + }; + + sai4a_sleep_pins_a: sai4a-sleep-0 { + pins { + pinmux = ; /* SAI4_SD_A */ + }; + }; + + sdmmc1_b4_pins_a: sdmmc1-b4-0 { + pins1 { + pinmux = , /* SDMMC1_D0 */ + , /* SDMMC1_D1 */ + , /* SDMMC1_D2 */ + , /* SDMMC1_D3 */ + ; /* SDMMC1_CMD */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* SDMMC1_CK */ + slew-rate = <2>; + drive-push-pull; + bias-disable; + }; + }; + + sdmmc1_b4_od_pins_a: sdmmc1-b4-od-0 { + pins1 { + pinmux = , /* SDMMC1_D0 */ + , /* SDMMC1_D1 */ + , /* SDMMC1_D2 */ + ; /* SDMMC1_D3 */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* SDMMC1_CK */ + slew-rate = <2>; + drive-push-pull; + bias-disable; + }; + pins3 { + pinmux = ; /* SDMMC1_CMD */ + slew-rate = <1>; + drive-open-drain; + bias-disable; + }; + }; + + sdmmc1_b4_init_pins_a: sdmmc1-b4-init-0 { + pins1 { + pinmux = , /* SDMMC1_D0 */ + , /* SDMMC1_D1 */ + , /* SDMMC1_D2 */ + ; /* SDMMC1_D3 */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + }; + + sdmmc1_b4_sleep_pins_a: sdmmc1-b4-sleep-0 { + pins { + pinmux = , /* SDMMC1_D0 */ + , /* SDMMC1_D1 */ + , /* SDMMC1_D2 */ + , /* SDMMC1_D3 */ + , /* SDMMC1_CK */ + ; /* SDMMC1_CMD */ + }; + }; + + sdmmc1_dir_pins_a: sdmmc1-dir-0 { + pins1 { + pinmux = , /* SDMMC1_D0DIR */ + , /* SDMMC1_D123DIR */ + ; /* SDMMC1_CDIR */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2{ + pinmux = ; /* SDMMC1_CKIN */ + bias-pull-up; + }; + }; + + sdmmc1_dir_init_pins_a: sdmmc1-dir-init-0 { + pins1 { + pinmux = , /* SDMMC1_D0DIR */ + , /* SDMMC1_D123DIR */ + ; /* SDMMC1_CDIR */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + }; + + sdmmc1_dir_sleep_pins_a: sdmmc1-dir-sleep-0 { + pins { + pinmux = , /* SDMMC1_D0DIR */ + , /* SDMMC1_D123DIR */ + , /* SDMMC1_CDIR */ + ; /* SDMMC1_CKIN */ + }; + }; + + sdmmc1_dir_pins_b: sdmmc1-dir-1 { + pins1 { + pinmux = , /* SDMMC1_D0DIR */ + , /* SDMMC1_D123DIR */ + ; /* SDMMC1_CDIR */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2{ + pinmux = ; /* SDMMC1_CKIN */ + bias-pull-up; + }; + }; + + sdmmc1_dir_sleep_pins_b: sdmmc1-dir-sleep-1 { + pins { + pinmux = , /* SDMMC1_D0DIR */ + , /* SDMMC1_D123DIR */ + , /* SDMMC1_CDIR */ + ; /* SDMMC1_CKIN */ + }; + }; + + sdmmc2_b4_pins_a: sdmmc2-b4-0 { + pins1 { + pinmux = , /* SDMMC2_D0 */ + , /* SDMMC2_D1 */ + , /* SDMMC2_D2 */ + , /* SDMMC2_D3 */ + ; /* SDMMC2_CMD */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2 { + pinmux = ; /* SDMMC2_CK */ + slew-rate = <2>; + drive-push-pull; + bias-pull-up; + }; + }; + + sdmmc2_b4_od_pins_a: sdmmc2-b4-od-0 { + pins1 { + pinmux = , /* SDMMC2_D0 */ + , /* SDMMC2_D1 */ + , /* SDMMC2_D2 */ + ; /* SDMMC2_D3 */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2 { + pinmux = ; /* SDMMC2_CK */ + slew-rate = <2>; + drive-push-pull; + bias-pull-up; + }; + pins3 { + pinmux = ; /* SDMMC2_CMD */ + slew-rate = <1>; + drive-open-drain; + bias-pull-up; + }; + }; + + sdmmc2_b4_sleep_pins_a: sdmmc2-b4-sleep-0 { + pins { + pinmux = , /* SDMMC2_D0 */ + , /* SDMMC2_D1 */ + , /* SDMMC2_D2 */ + , /* SDMMC2_D3 */ + , /* SDMMC2_CK */ + ; /* SDMMC2_CMD */ + }; + }; + + sdmmc2_b4_pins_b: sdmmc2-b4-1 { + pins1 { + pinmux = , /* SDMMC2_D0 */ + , /* SDMMC2_D1 */ + , /* SDMMC2_D2 */ + , /* SDMMC2_D3 */ + ; /* SDMMC2_CMD */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* SDMMC2_CK */ + slew-rate = <2>; + drive-push-pull; + bias-disable; + }; + }; + + sdmmc2_b4_od_pins_b: sdmmc2-b4-od-1 { + pins1 { + pinmux = , /* SDMMC2_D0 */ + , /* SDMMC2_D1 */ + , /* SDMMC2_D2 */ + ; /* SDMMC2_D3 */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + pins2 { + pinmux = ; /* SDMMC2_CK */ + slew-rate = <2>; + drive-push-pull; + bias-disable; + }; + pins3 { + pinmux = ; /* SDMMC2_CMD */ + slew-rate = <1>; + drive-open-drain; + bias-disable; + }; + }; + + sdmmc2_d47_pins_a: sdmmc2-d47-0 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + }; + + sdmmc2_d47_sleep_pins_a: sdmmc2-d47-sleep-0 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + }; + }; + + sdmmc2_d47_pins_b: sdmmc2-d47-1 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + slew-rate = <1>; + drive-push-pull; + bias-disable; + }; + }; + + sdmmc2_d47_sleep_pins_b: sdmmc2-d47-sleep-1 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + }; + }; + + sdmmc2_d47_pins_c: sdmmc2-d47-2 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + }; + + sdmmc2_d47_sleep_pins_c: sdmmc2-d47-sleep-2 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + }; + }; + + sdmmc2_d47_pins_d: sdmmc2-d47-3 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + }; + }; + + sdmmc2_d47_sleep_pins_d: sdmmc2-d47-sleep-3 { + pins { + pinmux = , /* SDMMC2_D4 */ + , /* SDMMC2_D5 */ + , /* SDMMC2_D6 */ + ; /* SDMMC2_D7 */ + }; + }; + + sdmmc3_b4_pins_a: sdmmc3-b4-0 { + pins1 { + pinmux = , /* SDMMC3_D0 */ + , /* SDMMC3_D1 */ + , /* SDMMC3_D2 */ + , /* SDMMC3_D3 */ + ; /* SDMMC3_CMD */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2 { + pinmux = ; /* SDMMC3_CK */ + slew-rate = <2>; + drive-push-pull; + bias-pull-up; + }; + }; + + sdmmc3_b4_od_pins_a: sdmmc3-b4-od-0 { + pins1 { + pinmux = , /* SDMMC3_D0 */ + , /* SDMMC3_D1 */ + , /* SDMMC3_D2 */ + ; /* SDMMC3_D3 */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2 { + pinmux = ; /* SDMMC3_CK */ + slew-rate = <2>; + drive-push-pull; + bias-pull-up; + }; + pins3 { + pinmux = ; /* SDMMC2_CMD */ + slew-rate = <1>; + drive-open-drain; + bias-pull-up; + }; + }; + + sdmmc3_b4_sleep_pins_a: sdmmc3-b4-sleep-0 { + pins { + pinmux = , /* SDMMC3_D0 */ + , /* SDMMC3_D1 */ + , /* SDMMC3_D2 */ + , /* SDMMC3_D3 */ + , /* SDMMC3_CK */ + ; /* SDMMC3_CMD */ + }; + }; + + sdmmc3_b4_pins_b: sdmmc3-b4-1 { + pins1 { + pinmux = , /* SDMMC3_D0 */ + , /* SDMMC3_D1 */ + , /* SDMMC3_D2 */ + , /* SDMMC3_D3 */ + ; /* SDMMC3_CMD */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2 { + pinmux = ; /* SDMMC3_CK */ + slew-rate = <2>; + drive-push-pull; + bias-pull-up; + }; + }; + + sdmmc3_b4_od_pins_b: sdmmc3-b4-od-1 { + pins1 { + pinmux = , /* SDMMC3_D0 */ + , /* SDMMC3_D1 */ + , /* SDMMC3_D2 */ + ; /* SDMMC3_D3 */ + slew-rate = <1>; + drive-push-pull; + bias-pull-up; + }; + pins2 { + pinmux = ; /* SDMMC3_CK */ + slew-rate = <2>; + drive-push-pull; + bias-pull-up; + }; + pins3 { + pinmux = ; /* SDMMC2_CMD */ + slew-rate = <1>; + drive-open-drain; + bias-pull-up; + }; + }; + + sdmmc3_b4_sleep_pins_b: sdmmc3-b4-sleep-1 { + pins { + pinmux = , /* SDMMC3_D0 */ + , /* SDMMC3_D1 */ + , /* SDMMC3_D2 */ + , /* SDMMC3_D3 */ + , /* SDMMC3_CK */ + ; /* SDMMC3_CMD */ + }; + }; + + spdifrx_pins_a: spdifrx-0 { + pins { + pinmux = ; /* SPDIF_IN1 */ + bias-disable; + }; + }; + + spdifrx_sleep_pins_a: spdifrx-sleep-0 { + pins { + pinmux = ; /* SPDIF_IN1 */ + }; + }; + + spi2_pins_a: spi2-0 { + pins1 { + pinmux = , /* SPI1_SCK */ + ; /* SPI1_MOSI */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + + pins2 { + pinmux = ; /* SPI1_MISO */ + bias-disable; + }; + }; + + spi4_pins_a: spi4-0 { + pins { + pinmux = , /* SPI4_SCK */ + ; /* SPI4_MOSI */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + pins2 { + pinmux = ; /* SPI4_MISO */ + bias-disable; + }; + }; + + stusb1600_pins_a: stusb1600-0 { + pins { + pinmux = ; + bias-pull-up; + }; + }; + + uart4_pins_a: uart4-0 { + pins1 { + pinmux = ; /* UART4_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* UART4_RX */ + bias-disable; + }; + }; + + uart4_idle_pins_a: uart4-idle-0 { + pins1 { + pinmux = ; /* UART4_TX */ + }; + pins2 { + pinmux = ; /* UART4_RX */ + bias-disable; + }; + }; + + uart4_sleep_pins_a: uart4-sleep-0 { + pins { + pinmux = , /* UART4_TX */ + ; /* UART4_RX */ + }; + }; + + uart4_pins_b: uart4-1 { + pins1 { + pinmux = ; /* UART4_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* UART4_RX */ + bias-disable; + }; + }; + + uart4_pins_c: uart4-2 { + pins1 { + pinmux = ; /* UART4_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* UART4_RX */ + bias-disable; + }; + }; + + uart7_pins_a: uart7-0 { + pins1 { + pinmux = ; /* UART7_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* UART7_RX */ + , /* UART7_CTS */ + ; /* UART7_RTS */ + bias-disable; + }; + }; + + uart7_pins_b: uart7-1 { + pins1 { + pinmux = ; /* UART7_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* UART7_RX */ + bias-disable; + }; + }; + + uart7_pins_c: uart7-2 { + pins1 { + pinmux = ; /* UART7_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* UART7_RX */ + bias-pull-up; + }; + }; + + uart7_idle_pins_c: uart7-idle-2 { + pins1 { + pinmux = ; /* UART7_TX */ + }; + pins2 { + pinmux = ; /* UART7_RX */ + bias-pull-up; + }; + }; + + uart7_sleep_pins_c: uart7-sleep-2 { + pins { + pinmux = , /* UART7_TX */ + ; /* UART7_RX */ + }; + }; + + uart8_pins_a: uart8-0 { + pins1 { + pinmux = ; /* UART8_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* UART8_RX */ + bias-disable; + }; + }; + + uart8_rtscts_pins_a: uart8rtscts-0 { + pins { + pinmux = , /* UART8_RTS */ + ; /* UART8_CTS */ + bias-disable; + }; + }; + + usart2_pins_a: usart2-0 { + pins1 { + pinmux = , /* USART2_TX */ + ; /* USART2_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* USART2_RX */ + ; /* USART2_CTS_NSS */ + bias-disable; + }; + }; + + usart2_sleep_pins_a: usart2-sleep-0 { + pins { + pinmux = , /* USART2_TX */ + , /* USART2_RTS */ + , /* USART2_RX */ + ; /* USART2_CTS_NSS */ + }; + }; + + usart2_pins_b: usart2-1 { + pins1 { + pinmux = , /* USART2_TX */ + ; /* USART2_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* USART2_RX */ + ; /* USART2_CTS_NSS */ + bias-disable; + }; + }; + + usart2_sleep_pins_b: usart2-sleep-1 { + pins { + pinmux = , /* USART2_TX */ + , /* USART2_RTS */ + , /* USART2_RX */ + ; /* USART2_CTS_NSS */ + }; + }; + + usart2_pins_c: usart2-2 { + pins1 { + pinmux = , /* USART2_TX */ + ; /* USART2_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* USART2_RX */ + ; /* USART2_CTS_NSS */ + bias-disable; + }; + }; + + usart2_idle_pins_c: usart2-idle-2 { + pins1 { + pinmux = , /* USART2_TX */ + ; /* USART2_CTS_NSS */ + }; + pins2 { + pinmux = ; /* USART2_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { + pinmux = ; /* USART2_RX */ + bias-disable; + }; + }; + + usart2_sleep_pins_c: usart2-sleep-2 { + pins { + pinmux = , /* USART2_TX */ + , /* USART2_RTS */ + , /* USART2_RX */ + ; /* USART2_CTS_NSS */ + }; + }; + + usart3_pins_a: usart3-0 { + pins1 { + pinmux = ; /* USART3_TX */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = ; /* USART3_RX */ + bias-disable; + }; + }; + + usart3_pins_b: usart3-1 { + pins1 { + pinmux = , /* USART3_TX */ + ; /* USART3_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* USART3_RX */ + ; /* USART3_CTS_NSS */ + bias-pull-up; + }; + }; + + usart3_idle_pins_b: usart3-idle-1 { + pins1 { + pinmux = , /* USART3_TX */ + ; /* USART3_CTS_NSS */ + }; + pins2 { + pinmux = ; /* USART3_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { + pinmux = ; /* USART3_RX */ + bias-pull-up; + }; + }; + + usart3_sleep_pins_b: usart3-sleep-1 { + pins { + pinmux = , /* USART3_TX */ + , /* USART3_RTS */ + , /* USART3_CTS_NSS */ + ; /* USART3_RX */ + }; + }; + + usart3_pins_c: usart3-2 { + pins1 { + pinmux = , /* USART3_TX */ + ; /* USART3_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* USART3_RX */ + ; /* USART3_CTS_NSS */ + bias-pull-up; + }; + }; + + usart3_idle_pins_c: usart3-idle-2 { + pins1 { + pinmux = , /* USART3_TX */ + ; /* USART3_CTS_NSS */ + }; + pins2 { + pinmux = ; /* USART3_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { + pinmux = ; /* USART3_RX */ + bias-pull-up; + }; + }; + + usart3_sleep_pins_c: usart3-sleep-2 { + pins { + pinmux = , /* USART3_TX */ + , /* USART3_RTS */ + , /* USART3_CTS_NSS */ + ; /* USART3_RX */ + }; + }; + + usart3_pins_d: usart3-3 { + pins1 { + pinmux = , /* USART3_TX */ + ; /* USART3_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins2 { + pinmux = , /* USART3_RX */ + ; /* USART3_CTS_NSS */ + bias-disable; + }; + }; + + usart3_idle_pins_d: usart3-idle-3 { + pins1 { + pinmux = , /* USART3_TX */ + , /* USART3_RTS */ + ; /* USART3_CTS_NSS */ + }; + pins2 { + pinmux = ; /* USART3_RX */ + bias-disable; + }; + }; + + usart3_sleep_pins_d: usart3-sleep-3 { + pins { + pinmux = , /* USART3_TX */ + , /* USART3_RTS */ + , /* USART3_CTS_NSS */ + ; /* USART3_RX */ + }; + }; + + usbotg_hs_pins_a: usbotg-hs-0 { + pins { + pinmux = ; /* OTG_ID */ + }; + }; + + usbotg_fs_dp_dm_pins_a: usbotg-fs-dp-dm-0 { + pins { + pinmux = , /* OTG_FS_DM */ + ; /* OTG_FS_DP */ + }; + }; +}; + +&pinctrl_z { + i2c2_pins_b2: i2c2-0 { + pins { + pinmux = ; /* I2C2_SCL */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c2_sleep_pins_b2: i2c2-sleep-0 { + pins { + pinmux = ; /* I2C2_SCL */ + }; + }; + + i2c4_pins_a: i2c4-0 { + pins { + pinmux = , /* I2C4_SCL */ + ; /* I2C4_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c4_sleep_pins_a: i2c4-sleep-0 { + pins { + pinmux = , /* I2C4_SCL */ + ; /* I2C4_SDA */ + }; + }; + + i2c6_pins_a: i2c6-0 { + pins { + pinmux = , /* I2C6_SCL */ + ; /* I2C6_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c6_sleep_pins_a: i2c6-sleep-0 { + pins { + pinmux = , /* I2C6_SCL */ + ; /* I2C6_SDA */ + }; + }; + + spi1_pins_a: spi1-0 { + pins1 { + pinmux = , /* SPI1_SCK */ + ; /* SPI1_MOSI */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + + pins2 { + pinmux = ; /* SPI1_MISO */ + bias-disable; + }; + }; + + spi1_pins_b: spi1-1 { + pins1 { + pinmux = , /* SPI1_SCK */ + ; /* SPI1_MOSI */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + + pins2 { + pinmux = ; /* SPI1_MISO */ + bias-disable; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp151.dtsi b/optee_os/core/arch/arm/dts/stm32mp151.dtsi new file mode 100644 index 0000000..dce4095 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp151.dtsi @@ -0,0 +1,1851 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2017 - All Rights Reserved + * Author: Ludovic Barre for STMicroelectronics. + */ +#include +#include +#include + +/ { + #address-cells = <1>; + #size-cells = <1>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu0: cpu@0 { + compatible = "arm,cortex-a7"; + clock-frequency = <650000000>; + device_type = "cpu"; + reg = <0>; + }; + }; + + arm-pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + interrupt-affinity = <&cpu0>; + interrupt-parent = <&intc>; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + + intc: interrupt-controller@a0021000 { + compatible = "arm,cortex-a7-gic"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0xa0021000 0x1000>, + <0xa0022000 0x2000>; + }; + + timer { + compatible = "arm,armv7-timer"; + interrupts = , + , + , + ; + interrupt-parent = <&intc>; + }; + + clocks { + clk_hse: clk-hse { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24000000>; + }; + + clk_hsi: clk-hsi { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <64000000>; + }; + + clk_lse: clk-lse { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <32768>; + }; + + clk_lsi: clk-lsi { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <32000>; + }; + + clk_csi: clk-csi { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <4000000>; + }; + }; + + thermal-zones { + cpu_thermal: cpu-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + thermal-sensors = <&dts>; + + trips { + cpu_alert1: cpu-alert1 { + temperature = <85000>; + hysteresis = <0>; + type = "passive"; + }; + + cpu-crit { + temperature = <120000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + }; + }; + }; + + booster: regulator-booster { + compatible = "st,stm32mp1-booster"; + st,syscfg = <&syscfg>; + status = "disabled"; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + interrupt-parent = <&intc>; + ranges; + + timers2: timer@40000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40000000 0x400>; + clocks = <&rcc TIM2_K>; + clock-names = "int"; + dmas = <&dmamux1 18 0x400 0x1>, + <&dmamux1 19 0x400 0x1>, + <&dmamux1 20 0x400 0x1>, + <&dmamux1 21 0x400 0x1>, + <&dmamux1 22 0x400 0x1>; + dma-names = "ch1", "ch2", "ch3", "ch4", "up"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@1 { + compatible = "st,stm32h7-timer-trigger"; + reg = <1>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + timers3: timer@40001000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40001000 0x400>; + clocks = <&rcc TIM3_K>; + clock-names = "int"; + dmas = <&dmamux1 23 0x400 0x1>, + <&dmamux1 24 0x400 0x1>, + <&dmamux1 25 0x400 0x1>, + <&dmamux1 26 0x400 0x1>, + <&dmamux1 27 0x400 0x1>, + <&dmamux1 28 0x400 0x1>; + dma-names = "ch1", "ch2", "ch3", "ch4", "up", "trig"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@2 { + compatible = "st,stm32h7-timer-trigger"; + reg = <2>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + timers4: timer@40002000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40002000 0x400>; + clocks = <&rcc TIM4_K>; + clock-names = "int"; + dmas = <&dmamux1 29 0x400 0x1>, + <&dmamux1 30 0x400 0x1>, + <&dmamux1 31 0x400 0x1>, + <&dmamux1 32 0x400 0x1>; + dma-names = "ch1", "ch2", "ch3", "ch4"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@3 { + compatible = "st,stm32h7-timer-trigger"; + reg = <3>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + timers5: timer@40003000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40003000 0x400>; + clocks = <&rcc TIM5_K>; + clock-names = "int"; + dmas = <&dmamux1 55 0x400 0x1>, + <&dmamux1 56 0x400 0x1>, + <&dmamux1 57 0x400 0x1>, + <&dmamux1 58 0x400 0x1>, + <&dmamux1 59 0x400 0x1>, + <&dmamux1 60 0x400 0x1>; + dma-names = "ch1", "ch2", "ch3", "ch4", "up", "trig"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@4 { + compatible = "st,stm32h7-timer-trigger"; + reg = <4>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + timers6: timer@40004000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40004000 0x400>; + clocks = <&rcc TIM6_K>; + clock-names = "int"; + dmas = <&dmamux1 69 0x400 0x1>; + dma-names = "up"; + status = "disabled"; + + timer@5 { + compatible = "st,stm32h7-timer-trigger"; + reg = <5>; + status = "disabled"; + }; + }; + + timers7: timer@40005000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40005000 0x400>; + clocks = <&rcc TIM7_K>; + clock-names = "int"; + dmas = <&dmamux1 70 0x400 0x1>; + dma-names = "up"; + status = "disabled"; + + timer@6 { + compatible = "st,stm32h7-timer-trigger"; + reg = <6>; + status = "disabled"; + }; + }; + + timers12: timer@40006000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40006000 0x400>; + clocks = <&rcc TIM12_K>; + clock-names = "int"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@11 { + compatible = "st,stm32h7-timer-trigger"; + reg = <11>; + status = "disabled"; + }; + }; + + timers13: timer@40007000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40007000 0x400>; + clocks = <&rcc TIM13_K>; + clock-names = "int"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@12 { + compatible = "st,stm32h7-timer-trigger"; + reg = <12>; + status = "disabled"; + }; + }; + + timers14: timer@40008000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x40008000 0x400>; + clocks = <&rcc TIM14_K>; + clock-names = "int"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@13 { + compatible = "st,stm32h7-timer-trigger"; + reg = <13>; + status = "disabled"; + }; + }; + + lptimer1: timer@40009000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-lptimer"; + reg = <0x40009000 0x400>; + interrupts-extended = <&exti 47 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc LPTIM1_K>; + clock-names = "mux"; + wakeup-source; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm-lp"; + #pwm-cells = <3>; + status = "disabled"; + }; + + trigger@0 { + compatible = "st,stm32-lptimer-trigger"; + reg = <0>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-lptimer-counter"; + status = "disabled"; + }; + }; + + spi2: spi@4000b000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32h7-spi"; + reg = <0x4000b000 0x400>; + interrupts = ; + clocks = <&rcc SPI2_K>; + resets = <&rcc SPI2_R>; + dmas = <&dmamux1 39 0x400 0x05>, + <&dmamux1 40 0x400 0x05>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + i2s2: audio-controller@4000b000 { + compatible = "st,stm32h7-i2s"; + #sound-dai-cells = <0>; + reg = <0x4000b000 0x400>; + interrupts = ; + dmas = <&dmamux1 39 0x400 0x01>, + <&dmamux1 40 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + spi3: spi@4000c000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32h7-spi"; + reg = <0x4000c000 0x400>; + interrupts = ; + clocks = <&rcc SPI3_K>; + resets = <&rcc SPI3_R>; + dmas = <&dmamux1 61 0x400 0x05>, + <&dmamux1 62 0x400 0x05>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + i2s3: audio-controller@4000c000 { + compatible = "st,stm32h7-i2s"; + #sound-dai-cells = <0>; + reg = <0x4000c000 0x400>; + interrupts = ; + dmas = <&dmamux1 61 0x400 0x01>, + <&dmamux1 62 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + spdifrx: audio-controller@4000d000 { + compatible = "st,stm32h7-spdifrx"; + #sound-dai-cells = <0>; + reg = <0x4000d000 0x400>; + clocks = <&rcc SPDIF_K>; + clock-names = "kclk"; + interrupts = ; + dmas = <&dmamux1 93 0x400 0x01>, + <&dmamux1 94 0x400 0x01>; + dma-names = "rx", "rx-ctrl"; + status = "disabled"; + }; + + usart2: serial@4000e000 { + compatible = "st,stm32h7-uart"; + reg = <0x4000e000 0x400>; + interrupts-extended = <&exti 27 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc USART2_K>; + wakeup-source; + dmas = <&dmamux1 43 0x400 0x15>, + <&dmamux1 44 0x400 0x11>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + usart3: serial@4000f000 { + compatible = "st,stm32h7-uart"; + reg = <0x4000f000 0x400>; + interrupts-extended = <&exti 28 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc USART3_K>; + wakeup-source; + dmas = <&dmamux1 45 0x400 0x15>, + <&dmamux1 46 0x400 0x11>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + uart4: serial@40010000 { + compatible = "st,stm32h7-uart"; + reg = <0x40010000 0x400>; + interrupts-extended = <&exti 30 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc UART4_K>; + wakeup-source; + dmas = <&dmamux1 63 0x400 0x15>, + <&dmamux1 64 0x400 0x11>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + uart5: serial@40011000 { + compatible = "st,stm32h7-uart"; + reg = <0x40011000 0x400>; + interrupts-extended = <&exti 31 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc UART5_K>; + wakeup-source; + dmas = <&dmamux1 65 0x400 0x15>, + <&dmamux1 66 0x400 0x11>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + i2c1: i2c@40012000 { + compatible = "st,stm32mp15-i2c"; + reg = <0x40012000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C1_K>; + resets = <&rcc I2C1_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x1>; + wakeup-source; + i2c-analog-filter; + status = "disabled"; + }; + + i2c2: i2c@40013000 { + compatible = "st,stm32mp15-i2c"; + reg = <0x40013000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C2_K>; + resets = <&rcc I2C2_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x2>; + wakeup-source; + i2c-analog-filter; + status = "disabled"; + }; + + i2c3: i2c@40014000 { + compatible = "st,stm32mp15-i2c"; + reg = <0x40014000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C3_K>; + resets = <&rcc I2C3_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x4>; + wakeup-source; + i2c-analog-filter; + status = "disabled"; + }; + + i2c5: i2c@40015000 { + compatible = "st,stm32mp15-i2c"; + reg = <0x40015000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C5_K>; + resets = <&rcc I2C5_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x10>; + wakeup-source; + i2c-analog-filter; + status = "disabled"; + }; + + cec: cec@40016000 { + compatible = "st,stm32-cec"; + reg = <0x40016000 0x400>; + interrupts = ; + clocks = <&rcc CEC_K>, <&rcc CEC>; + clock-names = "cec", "hdmi-cec"; + status = "disabled"; + }; + + dac: dac@40017000 { + compatible = "st,stm32h7-dac-core"; + reg = <0x40017000 0x400>; + clocks = <&rcc DAC12>; + clock-names = "pclk"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + dac1: dac@1 { + compatible = "st,stm32-dac"; + #io-channel-cells = <1>; + reg = <1>; + status = "disabled"; + }; + + dac2: dac@2 { + compatible = "st,stm32-dac"; + #io-channel-cells = <1>; + reg = <2>; + status = "disabled"; + }; + }; + + uart7: serial@40018000 { + compatible = "st,stm32h7-uart"; + reg = <0x40018000 0x400>; + interrupts-extended = <&exti 32 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc UART7_K>; + wakeup-source; + dmas = <&dmamux1 79 0x400 0x15>, + <&dmamux1 80 0x400 0x11>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + uart8: serial@40019000 { + compatible = "st,stm32h7-uart"; + reg = <0x40019000 0x400>; + interrupts-extended = <&exti 33 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc UART8_K>; + wakeup-source; + dmas = <&dmamux1 81 0x400 0x15>, + <&dmamux1 82 0x400 0x11>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + timers1: timer@44000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x44000000 0x400>; + clocks = <&rcc TIM1_K>; + clock-names = "int"; + dmas = <&dmamux1 11 0x400 0x1>, + <&dmamux1 12 0x400 0x1>, + <&dmamux1 13 0x400 0x1>, + <&dmamux1 14 0x400 0x1>, + <&dmamux1 15 0x400 0x1>, + <&dmamux1 16 0x400 0x1>, + <&dmamux1 17 0x400 0x1>; + dma-names = "ch1", "ch2", "ch3", "ch4", + "up", "trig", "com"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@0 { + compatible = "st,stm32h7-timer-trigger"; + reg = <0>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + timers8: timer@44001000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x44001000 0x400>; + clocks = <&rcc TIM8_K>; + clock-names = "int"; + dmas = <&dmamux1 47 0x400 0x1>, + <&dmamux1 48 0x400 0x1>, + <&dmamux1 49 0x400 0x1>, + <&dmamux1 50 0x400 0x1>, + <&dmamux1 51 0x400 0x1>, + <&dmamux1 52 0x400 0x1>, + <&dmamux1 53 0x400 0x1>; + dma-names = "ch1", "ch2", "ch3", "ch4", + "up", "trig", "com"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@7 { + compatible = "st,stm32h7-timer-trigger"; + reg = <7>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-timer-counter"; + status = "disabled"; + }; + }; + + usart6: serial@44003000 { + compatible = "st,stm32h7-uart"; + reg = <0x44003000 0x400>; + interrupts-extended = <&exti 29 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc USART6_K>; + wakeup-source; + dmas = <&dmamux1 71 0x400 0x15>, + <&dmamux1 72 0x400 0x11>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + spi1: spi@44004000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32h7-spi"; + reg = <0x44004000 0x400>; + interrupts = ; + clocks = <&rcc SPI1_K>; + resets = <&rcc SPI1_R>; + dmas = <&dmamux1 37 0x400 0x05>, + <&dmamux1 38 0x400 0x05>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + i2s1: audio-controller@44004000 { + compatible = "st,stm32h7-i2s"; + #sound-dai-cells = <0>; + reg = <0x44004000 0x400>; + interrupts = ; + dmas = <&dmamux1 37 0x400 0x01>, + <&dmamux1 38 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + spi4: spi@44005000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32h7-spi"; + reg = <0x44005000 0x400>; + interrupts = ; + clocks = <&rcc SPI4_K>; + resets = <&rcc SPI4_R>; + dmas = <&dmamux1 83 0x400 0x05>, + <&dmamux1 84 0x400 0x05>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + timers15: timer@44006000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x44006000 0x400>; + clocks = <&rcc TIM15_K>; + clock-names = "int"; + dmas = <&dmamux1 105 0x400 0x1>, + <&dmamux1 106 0x400 0x1>, + <&dmamux1 107 0x400 0x1>, + <&dmamux1 108 0x400 0x1>; + dma-names = "ch1", "up", "trig", "com"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@14 { + compatible = "st,stm32h7-timer-trigger"; + reg = <14>; + status = "disabled"; + }; + }; + + timers16: timer@44007000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x44007000 0x400>; + clocks = <&rcc TIM16_K>; + clock-names = "int"; + dmas = <&dmamux1 109 0x400 0x1>, + <&dmamux1 110 0x400 0x1>; + dma-names = "ch1", "up"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + timer@15 { + compatible = "st,stm32h7-timer-trigger"; + reg = <15>; + status = "disabled"; + }; + }; + + timers17: timer@44008000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-timers"; + reg = <0x44008000 0x400>; + clocks = <&rcc TIM17_K>; + clock-names = "int"; + dmas = <&dmamux1 111 0x400 0x1>, + <&dmamux1 112 0x400 0x1>; + dma-names = "ch1", "up"; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm"; + #pwm-cells = <3>; + status = "disabled"; + }; + + timer@16 { + compatible = "st,stm32h7-timer-trigger"; + reg = <16>; + status = "disabled"; + }; + }; + + spi5: spi@44009000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32h7-spi"; + reg = <0x44009000 0x400>; + interrupts = ; + clocks = <&rcc SPI5_K>; + resets = <&rcc SPI5_R>; + dmas = <&dmamux1 85 0x400 0x05>, + <&dmamux1 86 0x400 0x05>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + sai1: sai@4400a000 { + compatible = "st,stm32h7-sai"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x4400a000 0x400>; + reg = <0x4400a000 0x4>, <0x4400a3f0 0x10>; + interrupts = ; + resets = <&rcc SAI1_R>; + status = "disabled"; + + sai1a: audio-controller@4400a004 { + #sound-dai-cells = <0>; + + compatible = "st,stm32-sai-sub-a"; + reg = <0x4 0x20>; + clocks = <&rcc SAI1_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 87 0x400 0x01>; + status = "disabled"; + }; + + sai1b: audio-controller@4400a024 { + #sound-dai-cells = <0>; + compatible = "st,stm32-sai-sub-b"; + reg = <0x24 0x20>; + clocks = <&rcc SAI1_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 88 0x400 0x01>; + status = "disabled"; + }; + }; + + sai2: sai@4400b000 { + compatible = "st,stm32h7-sai"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x4400b000 0x400>; + reg = <0x4400b000 0x4>, <0x4400b3f0 0x10>; + interrupts = ; + resets = <&rcc SAI2_R>; + status = "disabled"; + + sai2a: audio-controller@4400b004 { + #sound-dai-cells = <0>; + compatible = "st,stm32-sai-sub-a"; + reg = <0x4 0x20>; + clocks = <&rcc SAI2_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 89 0x400 0x01>; + status = "disabled"; + }; + + sai2b: audio-controller@4400b024 { + #sound-dai-cells = <0>; + compatible = "st,stm32-sai-sub-b"; + reg = <0x24 0x20>; + clocks = <&rcc SAI2_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 90 0x400 0x01>; + status = "disabled"; + }; + }; + + sai3: sai@4400c000 { + compatible = "st,stm32h7-sai"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x4400c000 0x400>; + reg = <0x4400c000 0x4>, <0x4400c3f0 0x10>; + interrupts = ; + resets = <&rcc SAI3_R>; + status = "disabled"; + + sai3a: audio-controller@4400c004 { + #sound-dai-cells = <0>; + compatible = "st,stm32-sai-sub-a"; + reg = <0x04 0x20>; + clocks = <&rcc SAI3_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 113 0x400 0x01>; + status = "disabled"; + }; + + sai3b: audio-controller@4400c024 { + #sound-dai-cells = <0>; + compatible = "st,stm32-sai-sub-b"; + reg = <0x24 0x20>; + clocks = <&rcc SAI3_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 114 0x400 0x01>; + status = "disabled"; + }; + }; + + dfsdm: dfsdm@4400d000 { + compatible = "st,stm32mp1-dfsdm"; + reg = <0x4400d000 0x800>; + clocks = <&rcc DFSDM_K>; + clock-names = "dfsdm"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + dfsdm0: filter@0 { + compatible = "st,stm32-dfsdm-adc"; + #io-channel-cells = <1>; + reg = <0>; + interrupts = ; + dmas = <&dmamux1 101 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + + dfsdm1: filter@1 { + compatible = "st,stm32-dfsdm-adc"; + #io-channel-cells = <1>; + reg = <1>; + interrupts = ; + dmas = <&dmamux1 102 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + + dfsdm2: filter@2 { + compatible = "st,stm32-dfsdm-adc"; + #io-channel-cells = <1>; + reg = <2>; + interrupts = ; + dmas = <&dmamux1 103 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + + dfsdm3: filter@3 { + compatible = "st,stm32-dfsdm-adc"; + #io-channel-cells = <1>; + reg = <3>; + interrupts = ; + dmas = <&dmamux1 104 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + + dfsdm4: filter@4 { + compatible = "st,stm32-dfsdm-adc"; + #io-channel-cells = <1>; + reg = <4>; + interrupts = ; + dmas = <&dmamux1 91 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + + dfsdm5: filter@5 { + compatible = "st,stm32-dfsdm-adc"; + #io-channel-cells = <1>; + reg = <5>; + interrupts = ; + dmas = <&dmamux1 92 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + }; + + dma1: dma-controller@48000000 { + compatible = "st,stm32-dma"; + reg = <0x48000000 0x400>; + interrupts = , + , + , + , + , + , + , + ; + clocks = <&rcc DMA1>; + resets = <&rcc DMA1_R>; + #dma-cells = <4>; + st,mem2mem; + dma-requests = <8>; + }; + + dma2: dma-controller@48001000 { + compatible = "st,stm32-dma"; + reg = <0x48001000 0x400>; + interrupts = , + , + , + , + , + , + , + ; + clocks = <&rcc DMA2>; + resets = <&rcc DMA2_R>; + #dma-cells = <4>; + st,mem2mem; + dma-requests = <8>; + }; + + dmamux1: dma-router@48002000 { + compatible = "st,stm32h7-dmamux"; + reg = <0x48002000 0x40>; + #dma-cells = <3>; + dma-requests = <128>; + dma-masters = <&dma1 &dma2>; + dma-channels = <16>; + clocks = <&rcc DMAMUX>; + resets = <&rcc DMAMUX_R>; + }; + + adc: adc@48003000 { + compatible = "st,stm32mp1-adc-core"; + reg = <0x48003000 0x400>; + interrupts = , + ; + clocks = <&rcc ADC12>, <&rcc ADC12_K>; + clock-names = "bus", "adc"; + interrupt-controller; + st,syscfg = <&syscfg>; + #interrupt-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + adc1: adc@0 { + compatible = "st,stm32mp1-adc"; + #io-channel-cells = <1>; + reg = <0x0>; + interrupt-parent = <&adc>; + interrupts = <0>; + dmas = <&dmamux1 9 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + + adc2: adc@100 { + compatible = "st,stm32mp1-adc"; + #io-channel-cells = <1>; + reg = <0x100>; + interrupt-parent = <&adc>; + interrupts = <1>; + dmas = <&dmamux1 10 0x400 0x01>; + dma-names = "rx"; + status = "disabled"; + }; + }; + + sdmmc3: mmc@48004000 { + compatible = "st,stm32-sdmmc2", "arm,pl18x", "arm,primecell"; + arm,primecell-periphid = <0x00253180>; + reg = <0x48004000 0x400>; + interrupts = ; + interrupt-names = "cmd_irq"; + clocks = <&rcc SDMMC3_K>; + clock-names = "apb_pclk"; + resets = <&rcc SDMMC3_R>; + cap-sd-highspeed; + cap-mmc-highspeed; + max-frequency = <120000000>; + status = "disabled"; + }; + + usbotg_hs: usb-otg@49000000 { + compatible = "st,stm32mp15-hsotg", "snps,dwc2"; + reg = <0x49000000 0x10000>; + clocks = <&rcc USBO_K>; + clock-names = "otg"; + resets = <&rcc USBO_R>; + reset-names = "dwc2"; + interrupts = ; + g-rx-fifo-size = <512>; + g-np-tx-fifo-size = <32>; + g-tx-fifo-size = <256 16 16 16 16 16 16 16>; + dr_mode = "otg"; + otg-rev = <0x200>; + usb33d-supply = <&usb33>; + status = "disabled"; + }; + + ipcc: mailbox@4c001000 { + compatible = "st,stm32mp1-ipcc"; + #mbox-cells = <1>; + reg = <0x4c001000 0x400>; + st,proc-id = <0>; + interrupts-extended = + <&intc GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>, + <&intc GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>, + <&exti 61 1>; + interrupt-names = "rx", "tx", "wakeup"; + clocks = <&rcc IPCC>; + wakeup-source; + status = "disabled"; + }; + + dcmi: dcmi@4c006000 { + compatible = "st,stm32-dcmi"; + reg = <0x4c006000 0x400>; + interrupts = ; + resets = <&rcc CAMITF_R>; + clocks = <&rcc DCMI>; + clock-names = "mclk"; + dmas = <&dmamux1 75 0x400 0x01>; + dma-names = "tx"; + status = "disabled"; + }; + + rcc: rcc@50000000 { + compatible = "st,stm32mp1-rcc", "syscon"; + reg = <0x50000000 0x1000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + pwr_regulators: pwr@50001000 { + compatible = "st,stm32mp1,pwr-reg"; + reg = <0x50001000 0x10>; + + reg11: reg11 { + regulator-name = "reg11"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + }; + + reg18: reg18 { + regulator-name = "reg18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + usb33: usb33 { + regulator-name = "usb33"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + }; + + pwr_mcu: pwr_mcu@50001014 { + compatible = "st,stm32mp151-pwr-mcu", "syscon"; + reg = <0x50001014 0x4>; + }; + + exti: interrupt-controller@5000d000 { + compatible = "st,stm32mp1-exti", "syscon"; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x5000d000 0x400>; + }; + + syscfg: syscon@50020000 { + compatible = "st,stm32mp157-syscfg", "syscon"; + reg = <0x50020000 0x400>; + clocks = <&rcc SYSCFG>; + }; + + lptimer2: timer@50021000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-lptimer"; + reg = <0x50021000 0x400>; + interrupts-extended = <&exti 48 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc LPTIM2_K>; + clock-names = "mux"; + wakeup-source; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm-lp"; + #pwm-cells = <3>; + status = "disabled"; + }; + + trigger@1 { + compatible = "st,stm32-lptimer-trigger"; + reg = <1>; + status = "disabled"; + }; + + counter { + compatible = "st,stm32-lptimer-counter"; + status = "disabled"; + }; + }; + + lptimer3: timer@50022000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32-lptimer"; + reg = <0x50022000 0x400>; + interrupts-extended = <&exti 50 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc LPTIM3_K>; + clock-names = "mux"; + wakeup-source; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm-lp"; + #pwm-cells = <3>; + status = "disabled"; + }; + + trigger@2 { + compatible = "st,stm32-lptimer-trigger"; + reg = <2>; + status = "disabled"; + }; + }; + + lptimer4: timer@50023000 { + compatible = "st,stm32-lptimer"; + reg = <0x50023000 0x400>; + interrupts-extended = <&exti 52 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc LPTIM4_K>; + clock-names = "mux"; + wakeup-source; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm-lp"; + #pwm-cells = <3>; + status = "disabled"; + }; + }; + + lptimer5: timer@50024000 { + compatible = "st,stm32-lptimer"; + reg = <0x50024000 0x400>; + interrupts-extended = <&exti 53 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc LPTIM5_K>; + clock-names = "mux"; + wakeup-source; + status = "disabled"; + + pwm { + compatible = "st,stm32-pwm-lp"; + #pwm-cells = <3>; + status = "disabled"; + }; + }; + + vrefbuf: vrefbuf@50025000 { + compatible = "st,stm32-vrefbuf"; + reg = <0x50025000 0x8>; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <2500000>; + clocks = <&rcc VREF>; + status = "disabled"; + }; + + sai4: sai@50027000 { + compatible = "st,stm32h7-sai"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x50027000 0x400>; + reg = <0x50027000 0x4>, <0x500273f0 0x10>; + interrupts = ; + resets = <&rcc SAI4_R>; + status = "disabled"; + + sai4a: audio-controller@50027004 { + #sound-dai-cells = <0>; + compatible = "st,stm32-sai-sub-a"; + reg = <0x04 0x20>; + clocks = <&rcc SAI4_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 99 0x400 0x01>; + status = "disabled"; + }; + + sai4b: audio-controller@50027024 { + #sound-dai-cells = <0>; + compatible = "st,stm32-sai-sub-b"; + reg = <0x24 0x20>; + clocks = <&rcc SAI4_K>; + clock-names = "sai_ck"; + dmas = <&dmamux1 100 0x400 0x01>; + status = "disabled"; + }; + }; + + dts: thermal@50028000 { + compatible = "st,stm32-thermal"; + reg = <0x50028000 0x100>; + interrupts = ; + clocks = <&rcc TMPSENS>; + clock-names = "pclk"; + #thermal-sensor-cells = <0>; + status = "disabled"; + }; + + hash1: hash@54002000 { + compatible = "st,stm32f756-hash"; + reg = <0x54002000 0x400>; + interrupts = ; + clocks = <&rcc HASH1>; + resets = <&rcc HASH1_R>; + dmas = <&mdma1 31 0x2 0x1000A02 0x0 0x0>; + dma-names = "in"; + dma-maxburst = <2>; + status = "disabled"; + }; + + rng1: rng@54003000 { + compatible = "st,stm32-rng"; + reg = <0x54003000 0x400>; + clocks = <&rcc RNG1_K>; + resets = <&rcc RNG1_R>; + status = "disabled"; + }; + + mdma1: dma-controller@58000000 { + compatible = "st,stm32h7-mdma"; + reg = <0x58000000 0x1000>; + interrupts = ; + clocks = <&rcc MDMA>; + resets = <&rcc MDMA_R>; + #dma-cells = <5>; + dma-channels = <32>; + dma-requests = <48>; + }; + + fmc: memory-controller@58002000 { + #address-cells = <2>; + #size-cells = <1>; + compatible = "st,stm32mp1-fmc2-ebi"; + reg = <0x58002000 0x1000>; + clocks = <&rcc FMC_K>; + resets = <&rcc FMC_R>; + status = "disabled"; + + ranges = <0 0 0x60000000 0x04000000>, /* EBI CS 1 */ + <1 0 0x64000000 0x04000000>, /* EBI CS 2 */ + <2 0 0x68000000 0x04000000>, /* EBI CS 3 */ + <3 0 0x6c000000 0x04000000>, /* EBI CS 4 */ + <4 0 0x80000000 0x10000000>; /* NAND */ + + nand-controller@4,0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32mp1-fmc2-nfc"; + reg = <4 0x00000000 0x1000>, + <4 0x08010000 0x1000>, + <4 0x08020000 0x1000>, + <4 0x01000000 0x1000>, + <4 0x09010000 0x1000>, + <4 0x09020000 0x1000>; + interrupts = ; + dmas = <&mdma1 20 0x2 0x12000a02 0x0 0x0>, + <&mdma1 20 0x2 0x12000a08 0x0 0x0>, + <&mdma1 21 0x2 0x12000a0a 0x0 0x0>; + dma-names = "tx", "rx", "ecc"; + status = "disabled"; + }; + }; + + qspi: spi@58003000 { + compatible = "st,stm32f469-qspi"; + reg = <0x58003000 0x1000>, <0x70000000 0x10000000>; + reg-names = "qspi", "qspi_mm"; + interrupts = ; + dmas = <&mdma1 22 0x2 0x10100002 0x0 0x0>, + <&mdma1 22 0x2 0x10100008 0x0 0x0>; + dma-names = "tx", "rx"; + clocks = <&rcc QSPI_K>; + resets = <&rcc QSPI_R>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + sdmmc1: mmc@58005000 { + compatible = "st,stm32-sdmmc2", "arm,pl18x", "arm,primecell"; + arm,primecell-periphid = <0x00253180>; + reg = <0x58005000 0x1000>; + interrupts = ; + interrupt-names = "cmd_irq"; + clocks = <&rcc SDMMC1_K>; + clock-names = "apb_pclk"; + resets = <&rcc SDMMC1_R>; + cap-sd-highspeed; + cap-mmc-highspeed; + max-frequency = <120000000>; + status = "disabled"; + }; + + sdmmc2: mmc@58007000 { + compatible = "st,stm32-sdmmc2", "arm,pl18x", "arm,primecell"; + arm,primecell-periphid = <0x00253180>; + reg = <0x58007000 0x1000>; + interrupts = ; + interrupt-names = "cmd_irq"; + clocks = <&rcc SDMMC2_K>; + clock-names = "apb_pclk"; + resets = <&rcc SDMMC2_R>; + cap-sd-highspeed; + cap-mmc-highspeed; + max-frequency = <120000000>; + status = "disabled"; + }; + + crc1: crc@58009000 { + compatible = "st,stm32f7-crc"; + reg = <0x58009000 0x400>; + clocks = <&rcc CRC1>; + status = "disabled"; + }; + + ethernet0: ethernet@5800a000 { + compatible = "st,stm32mp1-dwmac", "snps,dwmac-4.20a"; + reg = <0x5800a000 0x2000>; + reg-names = "stmmaceth"; + interrupts-extended = <&intc GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq"; + clock-names = "stmmaceth", + "mac-clk-tx", + "mac-clk-rx", + "eth-ck", + "ptp_ref", + "ethstp"; + clocks = <&rcc ETHMAC>, + <&rcc ETHTX>, + <&rcc ETHRX>, + <&rcc ETHCK_K>, + <&rcc ETHPTP_K>, + <&rcc ETHSTP>; + st,syscon = <&syscfg 0x4>; + snps,mixed-burst; + snps,pbl = <2>; + snps,en-tx-lpi-clockgating; + snps,axi-config = <&stmmac_axi_config_0>; + snps,tso; + status = "disabled"; + + stmmac_axi_config_0: stmmac-axi-config { + snps,wr_osr_lmt = <0x7>; + snps,rd_osr_lmt = <0x7>; + snps,blen = <0 0 0 0 16 8 4>; + }; + }; + + usbh_ohci: usb@5800c000 { + compatible = "generic-ohci"; + reg = <0x5800c000 0x1000>; + clocks = <&usbphyc>, <&rcc USBH>; + resets = <&rcc USBH_R>; + interrupts = ; + status = "disabled"; + }; + + usbh_ehci: usb@5800d000 { + compatible = "generic-ehci"; + reg = <0x5800d000 0x1000>; + clocks = <&usbphyc>, <&rcc USBH>; + resets = <&rcc USBH_R>; + interrupts = ; + companion = <&usbh_ohci>; + status = "disabled"; + }; + + ltdc: display-controller@5a001000 { + compatible = "st,stm32-ltdc"; + reg = <0x5a001000 0x400>; + interrupts = , + ; + clocks = <&rcc LTDC_PX>; + clock-names = "lcd"; + resets = <&rcc LTDC_R>; + status = "disabled"; + + port { + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + iwdg2: watchdog@5a002000 { + compatible = "st,stm32mp1-iwdg"; + reg = <0x5a002000 0x400>; + interrupts = ; + clocks = <&rcc IWDG2>, <&rcc CK_LSI>; + clock-names = "pclk", "lsi"; + status = "disabled"; + }; + + usbphyc: usbphyc@5a006000 { + #address-cells = <1>; + #size-cells = <0>; + #clock-cells = <0>; + compatible = "st,stm32mp1-usbphyc"; + reg = <0x5a006000 0x1000>; + clocks = <&rcc USBPHY_K>; + resets = <&rcc USBPHY_R>; + vdda1v1-supply = <®11>; + vdda1v8-supply = <®18>; + status = "disabled"; + + usbphyc_port0: usb-phy@0 { + #phy-cells = <0>; + reg = <0>; + }; + + usbphyc_port1: usb-phy@1 { + #phy-cells = <1>; + reg = <1>; + }; + }; + + usart1: serial@5c000000 { + compatible = "st,stm32h7-uart"; + reg = <0x5c000000 0x400>; + interrupts-extended = <&exti 26 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc USART1_K>; + wakeup-source; + status = "disabled"; + }; + + spi6: spi@5c001000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "st,stm32h7-spi"; + reg = <0x5c001000 0x400>; + interrupts = ; + clocks = <&rcc SPI6_K>; + resets = <&rcc SPI6_R>; + dmas = <&mdma1 34 0x0 0x40008 0x0 0x0>, + <&mdma1 35 0x0 0x40002 0x0 0x0>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + i2c4: i2c@5c002000 { + compatible = "st,stm32mp15-i2c"; + reg = <0x5c002000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C4_K>; + resets = <&rcc I2C4_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x8>; + wakeup-source; + i2c-analog-filter; + status = "disabled"; + }; + + iwdg1: watchdog@5c003000 { + compatible = "st,stm32mp1-iwdg"; + reg = <0x5C003000 0x400>; + interrupts = ; + clocks = <&rcc IWDG1>, <&rcc CK_LSI>; + clock-names = "pclk", "lsi"; + status = "disabled"; + }; + + rtc: rtc@5c004000 { + compatible = "st,stm32mp1-rtc"; + reg = <0x5c004000 0x400>; + clocks = <&rcc RTCAPB>, <&rcc RTC>; + clock-names = "pclk", "rtc_ck"; + interrupts-extended = <&exti 19 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + bsec: efuse@5c005000 { + compatible = "st,stm32mp15-bsec"; + reg = <0x5c005000 0x400>; + #address-cells = <1>; + #size-cells = <1>; + + cfg0_otp: cfg0_otp@0 { + reg = <0x0 0x1>; + }; + part_number_otp: part_number_otp@4 { + reg = <0x4 0x1>; + }; + monotonic_otp: monotonic_otp@10 { + reg = <0x10 0x4>; + }; + nand_otp: nand_otp@24 { + reg = <0x24 0x4>; + }; + uid_otp: uid_otp@34 { + reg = <0x34 0xc>; + }; + package_otp: package_otp@40 { + reg = <0x40 0x4>; + }; + hw2_otp: hw2_otp@48 { + reg = <0x48 0x4>; + }; + ts_cal1: calib@5c { + reg = <0x5c 0x2>; + }; + ts_cal2: calib@5e { + reg = <0x5e 0x2>; + }; + pkh_otp: pkh_otp@60 { + reg = <0x60 0x20>; + }; + ethernet_mac_address: mac@e4 { + reg = <0xe4 0x8>; + st,non-secure-otp; + }; + }; + + etzpc: etzpc@5c007000 { + compatible = "st,stm32-etzpc"; + reg = <0x5C007000 0x400>; + clocks = <&rcc TZPC>; + status = "disabled"; + secure-status = "okay"; + }; + + i2c6: i2c@5c009000 { + compatible = "st,stm32mp15-i2c"; + reg = <0x5c009000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C6_K>; + resets = <&rcc I2C6_R>; + #address-cells = <1>; + #size-cells = <0>; + st,syscfg-fmp = <&syscfg 0x4 0x20>; + wakeup-source; + i2c-analog-filter; + status = "disabled"; + }; + + tamp: tamp@5c00a000 { + compatible = "st,stm32-tamp", "syscon", "simple-mfd"; + reg = <0x5c00a000 0x400>; + clocks = <&rcc RTCAPB>; + }; + + /* + * Break node order to solve dependency probe issue between + * pinctrl and exti. + */ + pinctrl: pinctrl@50002000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "st,stm32mp157-pinctrl"; + ranges = <0 0x50002000 0xa400>; + interrupt-parent = <&exti>; + st,syscfg = <&exti 0x60 0xff>; + pins-are-numbered; + + gpioa: gpio@50002000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x0 0x400>; + clocks = <&rcc GPIOA>; + st,bank-name = "GPIOA"; + status = "disabled"; + }; + + gpiob: gpio@50003000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x1000 0x400>; + clocks = <&rcc GPIOB>; + st,bank-name = "GPIOB"; + status = "disabled"; + }; + + gpioc: gpio@50004000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x2000 0x400>; + clocks = <&rcc GPIOC>; + st,bank-name = "GPIOC"; + status = "disabled"; + }; + + gpiod: gpio@50005000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x3000 0x400>; + clocks = <&rcc GPIOD>; + st,bank-name = "GPIOD"; + status = "disabled"; + }; + + gpioe: gpio@50006000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x4000 0x400>; + clocks = <&rcc GPIOE>; + st,bank-name = "GPIOE"; + status = "disabled"; + }; + + gpiof: gpio@50007000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x5000 0x400>; + clocks = <&rcc GPIOF>; + st,bank-name = "GPIOF"; + status = "disabled"; + }; + + gpiog: gpio@50008000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x6000 0x400>; + clocks = <&rcc GPIOG>; + st,bank-name = "GPIOG"; + status = "disabled"; + }; + + gpioh: gpio@50009000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x7000 0x400>; + clocks = <&rcc GPIOH>; + st,bank-name = "GPIOH"; + status = "disabled"; + }; + + gpioi: gpio@5000a000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x8000 0x400>; + clocks = <&rcc GPIOI>; + st,bank-name = "GPIOI"; + status = "disabled"; + }; + + gpioj: gpio@5000b000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0x9000 0x400>; + clocks = <&rcc GPIOJ>; + st,bank-name = "GPIOJ"; + status = "disabled"; + }; + + gpiok: gpio@5000c000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0xa000 0x400>; + clocks = <&rcc GPIOK>; + st,bank-name = "GPIOK"; + status = "disabled"; + }; + }; + + pinctrl_z: pinctrl@54004000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "st,stm32mp157-z-pinctrl"; + ranges = <0 0x54004000 0x400>; + pins-are-numbered; + interrupt-parent = <&exti>; + st,syscfg = <&exti 0x60 0xff>; + + gpioz: gpio@54004000 { + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reg = <0 0x400>; + clocks = <&rcc GPIOZ>; + st,bank-name = "GPIOZ"; + st,bank-ioport = <11>; + status = "disabled"; + }; + }; + }; + + mlahb: ahb { + compatible = "st,mlahb", "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + dma-ranges = <0x00000000 0x38000000 0x10000>, + <0x10000000 0x10000000 0x60000>, + <0x30000000 0x30000000 0x60000>; + + m4_rproc: m4@10000000 { + compatible = "st,stm32mp1-m4"; + reg = <0x10000000 0x40000>, + <0x30000000 0x40000>, + <0x38000000 0x10000>; + resets = <&rcc MCU_R>; + st,syscfg-holdboot = <&rcc 0x10C 0x1>; + st,syscfg-tz = <&rcc 0x000 0x1>; + st,syscfg-pdds = <&pwr_mcu 0x0 0x1>; + st,syscfg-rsc-tbl = <&tamp 0x144 0xFFFFFFFF>; + st,syscfg-m4-state = <&tamp 0x148 0xFFFFFFFF>; + status = "disabled"; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp153.dtsi b/optee_os/core/arch/arm/dts/stm32mp153.dtsi new file mode 100644 index 0000000..486084e --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp153.dtsi @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +#include "stm32mp151.dtsi" + +/ { + cpus { + cpu1: cpu@1 { + compatible = "arm,cortex-a7"; + clock-frequency = <650000000>; + device_type = "cpu"; + reg = <1>; + }; + }; + + arm-pmu { + interrupts = , + ; + interrupt-affinity = <&cpu0>, <&cpu1>; + }; + + timer { + interrupts = , + , + , + ; + }; + + soc { + m_can1: can@4400e000 { + compatible = "bosch,m_can"; + reg = <0x4400e000 0x400>, <0x44011000 0x1400>; + reg-names = "m_can", "message_ram"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + clocks = <&rcc CK_HSE>, <&rcc FDCAN_K>; + clock-names = "hclk", "cclk"; + bosch,mram-cfg = <0x0 0 0 32 0 0 2 2>; + status = "disabled"; + }; + + m_can2: can@4400f000 { + compatible = "bosch,m_can"; + reg = <0x4400f000 0x400>, <0x44011000 0x2800>; + reg-names = "m_can", "message_ram"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + clocks = <&rcc CK_HSE>, <&rcc FDCAN_K>; + clock-names = "hclk", "cclk"; + bosch,mram-cfg = <0x1400 0 0 32 0 0 2 2>; + status = "disabled"; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp157.dtsi b/optee_os/core/arch/arm/dts/stm32mp157.dtsi new file mode 100644 index 0000000..54e73cc --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp157.dtsi @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +#include "stm32mp153.dtsi" + +/ { + soc { + gpu: gpu@59000000 { + compatible = "vivante,gc"; + reg = <0x59000000 0x800>; + interrupts = ; + clocks = <&rcc GPU>, <&rcc GPU_K>; + clock-names = "bus" ,"core"; + resets = <&rcc GPU_R>; + }; + + dsi: dsi@5a000000 { + compatible = "st,stm32-dsi"; + reg = <0x5a000000 0x800>; + clocks = <&rcc DSI_K>, <&clk_hse>, <&rcc DSI_PX>; + clock-names = "pclk", "ref", "px_clk"; + resets = <&rcc DSI_R>; + reset-names = "apb"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + }; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp157a-dhcor-avenger96.dts b/optee_os/core/arch/arm/dts/stm32mp157a-dhcor-avenger96.dts new file mode 100644 index 0000000..2e3c9fb --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp157a-dhcor-avenger96.dts @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright (C) Linaro Ltd 2019 - All Rights Reserved + * Author: Manivannan Sadhasivam + * Copyright (C) 2020 Marek Vasut + * + * DHCOR STM32MP1 variant: + * DHCR-STM32MP157A-C065-R102-V18-SPI-C-01LG + * DHCOR PCB number: 586-100 or newer + * Avenger96 PCB number: 588-200 or newer + */ + +/dts-v1/; + +#include "stm32mp157.dtsi" +#include "stm32mp15xc.dtsi" +#include "stm32mp15xx-dhcor-som.dtsi" +#include "stm32mp15xx-dhcor-avenger96.dtsi" + +/ { + model = "Arrow Electronics STM32MP157A Avenger96 board"; + compatible = "arrow,stm32mp157a-avenger96", "dh,stm32mp157a-dhcor-som", + "st,stm32mp157"; +}; + +&m_can1 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&m_can1_pins_b>; + pinctrl-1 = <&m_can1_sleep_pins_b>; + status = "disabled"; +}; + +&m_can2 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&m_can2_pins_a>; + pinctrl-1 = <&m_can2_sleep_pins_a>; + status = "disabled"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp157a-dk1.dts b/optee_os/core/arch/arm/dts/stm32mp157a-dk1.dts new file mode 100644 index 0000000..7ba971e --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp157a-dk1.dts @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +/dts-v1/; + +#include "stm32mp157.dtsi" +#include "stm32mp15-pinctrl.dtsi" +#include "stm32mp15xxac-pinctrl.dtsi" +#include "stm32mp15xx-dkx.dtsi" + +/ { + model = "STMicroelectronics STM32MP157A-DK1 Discovery Board"; + compatible = "st,stm32mp157a-dk1", "st,stm32mp157"; + + aliases { + ethernet0 = ðernet0; + serial0 = &uart4; + serial1 = &usart3; + serial2 = &uart7; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; +}; + +&bsec { + board_id: board_id@ec { + reg = <0xec 0x4>; + st,non-secure-otp; + }; + huk_otp: huk-otp@f0 { + reg = <0xf0 0x10>; + }; +}; + +&rcc { + status = "okay"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp157c-dhcom-pdk2.dts b/optee_os/core/arch/arm/dts/stm32mp157c-dhcom-pdk2.dts new file mode 100644 index 0000000..a2e8e56 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp157c-dhcom-pdk2.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause +/* + * Copyright (C) 2019-2020 Marek Vasut + * Copyright (C) 2022 DH electronics GmbH + * + * DHCOM STM32MP1 variant: + * DHCM-STM32MP157C-C065-R102-F0819-SPI-E2-CAN2-SDR104-RTC-WBT-T-DSI-I-01D2 + * DHCOM PCB number: 587-200 or newer + * PDK2 PCB number: 516-400 or newer + */ +/dts-v1/; + +#include "stm32mp157.dtsi" +#include "stm32mp15xc.dtsi" +#include "stm32mp15xx-dhcom-som.dtsi" +#include "stm32mp15xx-dhcom-pdk2.dtsi" + +/ { + model = "DH electronics STM32MP157C DHCOM Premium Developer Kit (2)"; + compatible = "dh,stm32mp157c-dhcom-pdk2", "dh,stm32mp157c-dhcom-som", + "st,stm32mp157"; +}; + +&m_can1 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&m_can1_pins_a>; + pinctrl-1 = <&m_can1_sleep_pins_a>; + status = "okay"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp157c-dk2.dts b/optee_os/core/arch/arm/dts/stm32mp157c-dk2.dts new file mode 100644 index 0000000..b82730c --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp157c-dk2.dts @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +/dts-v1/; + +#include "stm32mp157.dtsi" +#include "stm32mp15xc.dtsi" +#include "stm32mp15-pinctrl.dtsi" +#include "stm32mp15xxac-pinctrl.dtsi" +#include "stm32mp15xx-dkx.dtsi" + +/ { + model = "STMicroelectronics STM32MP157C-DK2 Discovery Board"; + compatible = "st,stm32mp157c-dk2", "st,stm32mp157"; + + aliases { + ethernet0 = ðernet0; + serial0 = &uart4; + serial1 = &usart3; + serial2 = &uart7; + serial3 = &usart2; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; +}; + +&bsec { + board_id: board_id@ec { + reg = <0xec 0x4>; + st,non-secure-otp; + }; + huk_otp: huk-otp@f0 { + reg = <0xf0 0x10>; + }; +}; + +&dsi { + status = "disabled"; + phy-dsi-supply = <®18>; + + ports { + port@0 { + reg = <0>; + dsi_in: endpoint { + remote-endpoint = <<dc_ep1_out>; + }; + }; + + port@1 { + reg = <1>; + dsi_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; + + panel@0 { + compatible = "orisetech,otm8009a"; + reg = <0>; + reset-gpios = <&gpioe 4 GPIO_ACTIVE_LOW>; + power-supply = <&v3v3>; + status = "okay"; + + port { + panel_in: endpoint { + remote-endpoint = <&dsi_out>; + }; + }; + }; +}; + +&i2c1 { + touchscreen@38 { + compatible = "focaltech,ft6236"; + reg = <0x38>; + interrupts = <2 2>; + interrupt-parent = <&gpiof>; + interrupt-controller; + touchscreen-size-x = <480>; + touchscreen-size-y = <800>; + status = "okay"; + }; +}; + +<dc { + status = "disabled"; + + port { + ltdc_ep1_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&dsi_in>; + }; + }; +}; + +&rcc { + status = "okay"; +}; + +&usart2 { + pinctrl-names = "default", "sleep", "idle"; + pinctrl-0 = <&usart2_pins_c>; + pinctrl-1 = <&usart2_sleep_pins_c>; + pinctrl-2 = <&usart2_idle_pins_c>; + status = "disabled"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp157c-ed1.dts b/optee_os/core/arch/arm/dts/stm32mp157c-ed1.dts new file mode 100644 index 0000000..1c5ab2d --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp157c-ed1.dts @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2017 - All Rights Reserved + * Author: Ludovic Barre for STMicroelectronics. + */ +/dts-v1/; + +#include "stm32mp157.dtsi" +#include "stm32mp15xc.dtsi" +#include "stm32mp15-pinctrl.dtsi" +#include "stm32mp15xxaa-pinctrl.dtsi" +#include +#include + +/ { + model = "STMicroelectronics STM32MP157C eval daughter"; + compatible = "st,stm32mp157c-ed1", "st,stm32mp157"; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory@c0000000 { + device_type = "memory"; + reg = <0xC0000000 0x40000000>; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + mcuram2: mcuram2@10000000 { + compatible = "shared-dma-pool"; + reg = <0x10000000 0x40000>; + no-map; + }; + + vdev0vring0: vdev0vring0@10040000 { + compatible = "shared-dma-pool"; + reg = <0x10040000 0x1000>; + no-map; + }; + + vdev0vring1: vdev0vring1@10041000 { + compatible = "shared-dma-pool"; + reg = <0x10041000 0x1000>; + no-map; + }; + + vdev0buffer: vdev0buffer@10042000 { + compatible = "shared-dma-pool"; + reg = <0x10042000 0x4000>; + no-map; + }; + + mcuram: mcuram@30000000 { + compatible = "shared-dma-pool"; + reg = <0x30000000 0x40000>; + no-map; + }; + + retram: retram@38000000 { + compatible = "shared-dma-pool"; + reg = <0x38000000 0x10000>; + no-map; + }; + + gpu_reserved: gpu@e8000000 { + reg = <0xe8000000 0x8000000>; + no-map; + }; + }; + + aliases { + serial0 = &uart4; + }; + + sd_switch: regulator-sd_switch { + compatible = "regulator-gpio"; + regulator-name = "sd_switch"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2900000>; + regulator-type = "voltage"; + regulator-always-on; + + gpios = <&gpiof 14 GPIO_ACTIVE_HIGH>; + gpios-states = <0>; + states = <1800000 0x1>, + <2900000 0x0>; + }; + + vin: vin { + compatible = "regulator-fixed"; + regulator-name = "vin"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; +}; + +&adc { + /* ANA0, ANA1 are dedicated pins and don't need pinctrl: only in6. */ + pinctrl-0 = <&adc1_in6_pins_a>; + pinctrl-names = "default"; + vdd-supply = <&vdd>; + vdda-supply = <&vdda>; + vref-supply = <&vdda>; + status = "disabled"; + adc1: adc@0 { + st,adc-channels = <0 1 6>; + /* 16.5 ck_cycles sampling time */ + st,min-sample-time-nsecs = <400>; + status = "okay"; + }; +}; + +&bsec { + board_id: board_id@ec { + reg = <0xec 0x4>; + st,non-secure-otp; + }; + huk_otp: huk-otp@f0 { + reg = <0xf0 0x10>; + }; +}; + +&crc1 { + status = "disabled"; +}; + +&dac { + pinctrl-names = "default"; + pinctrl-0 = <&dac_ch1_pins_a &dac_ch2_pins_a>; + vref-supply = <&vdda>; + status = "disabled"; + dac1: dac@1 { + status = "okay"; + }; + dac2: dac@2 { + status = "okay"; + }; +}; + +&dts { + status = "disabled"; +}; + +&gpu { + contiguous-area = <&gpu_reserved>; +}; + +&hash1 { + status = "disabled"; +}; + +&i2c4 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c4_pins_a>; + pinctrl-1 = <&i2c4_sleep_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + clock-frequency = <400000>; + status = "okay"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; + + pmic: stpmic@33 { + compatible = "st,stpmic1"; + reg = <0x33>; + interrupts-extended = <&gpioa 0 IRQ_TYPE_EDGE_FALLING>; + interrupt-controller; + #interrupt-cells = <2>; + status = "okay"; + + regulators { + compatible = "st,stpmic1-regulators"; + buck1-supply = <&vin>; + buck2-supply = <&vin>; + buck3-supply = <&vin>; + buck4-supply = <&vin>; + ldo1-supply = <&v3v3>; + ldo2-supply = <&v3v3>; + ldo3-supply = <&vdd_ddr>; + ldo4-supply = <&vin>; + ldo5-supply = <&v3v3>; + ldo6-supply = <&v3v3>; + vref_ddr-supply = <&vin>; + boost-supply = <&vin>; + pwr_sw1-supply = <&bst_out>; + pwr_sw2-supply = <&bst_out>; + + vddcore: buck1 { + regulator-name = "vddcore"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd_ddr: buck2 { + regulator-name = "vdd_ddr"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd: buck3 { + regulator-name = "vdd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + st,mask-reset; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + v3v3: buck4 { + regulator-name = "v3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-over-current-protection; + regulator-initial-mode = <0>; + }; + + vdda: ldo1 { + regulator-name = "vdda"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + interrupts = ; + }; + + v2v8: ldo2 { + regulator-name = "v2v8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + interrupts = ; + }; + + vtt_ddr: ldo3 { + regulator-name = "vtt_ddr"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <750000>; + regulator-always-on; + regulator-over-current-protection; + }; + + vdd_usb: ldo4 { + regulator-name = "vdd_usb"; + interrupts = ; + }; + + vdd_sd: ldo5 { + regulator-name = "vdd_sd"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + interrupts = ; + regulator-boot-on; + }; + + v1v8: ldo6 { + regulator-name = "v1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + interrupts = ; + }; + + vref_ddr: vref_ddr { + regulator-name = "vref_ddr"; + regulator-always-on; + }; + + bst_out: boost { + regulator-name = "bst_out"; + interrupts = ; + }; + + vbus_otg: pwr_sw1 { + regulator-name = "vbus_otg"; + interrupts = ; + }; + + vbus_sw: pwr_sw2 { + regulator-name = "vbus_sw"; + interrupts = ; + regulator-active-discharge = <1>; + }; + }; + + onkey { + compatible = "st,stpmic1-onkey"; + interrupts = , ; + interrupt-names = "onkey-falling", "onkey-rising"; + power-off-time-sec = <10>; + status = "disabled"; + }; + + watchdog { + compatible = "st,stpmic1-wdt"; + status = "disabled"; + }; + }; +}; + +&ipcc { + status = "disabled"; +}; + +&iwdg1 { + timeout-sec = <32>; +}; + +&iwdg2 { + timeout-sec = <32>; + status = "okay"; + secure-status = "disabled"; +}; + +&m4_rproc { + memory-region = <&retram>, <&mcuram>, <&mcuram2>, <&vdev0vring0>, + <&vdev0vring1>, <&vdev0buffer>; + mboxes = <&ipcc 0>, <&ipcc 1>, <&ipcc 2>, <&ipcc 3>; + mbox-names = "vq0", "vq1", "shutdown", "detach"; + interrupt-parent = <&exti>; + interrupts = <68 1>; + status = "disabled"; +}; + +&pwr_regulators { + vdd-supply = <&vdd>; + vdd_3v3_usbfs-supply = <&vdd_usb>; +}; + +&rcc { + status = "okay"; +}; + +&rng1 { + status = "okay"; +}; + +&rtc { + status = "disabled"; +}; + +&sdmmc1 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc1_b4_pins_a &sdmmc1_dir_pins_a>; + pinctrl-1 = <&sdmmc1_b4_od_pins_a &sdmmc1_dir_pins_a>; + pinctrl-2 = <&sdmmc1_b4_sleep_pins_a &sdmmc1_dir_sleep_pins_a>; + cd-gpios = <&gpiog 1 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; + disable-wp; + st,sig-dir; + st,neg-edge; + st,use-ckin; + bus-width = <4>; + vmmc-supply = <&vdd_sd>; + vqmmc-supply = <&sd_switch>; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-ddr50; + status = "disabled"; +}; + +&sdmmc2 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc2_b4_pins_a &sdmmc2_d47_pins_a>; + pinctrl-1 = <&sdmmc2_b4_od_pins_a &sdmmc2_d47_pins_a>; + pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_a>; + non-removable; + no-sd; + no-sdio; + st,neg-edge; + bus-width = <8>; + vmmc-supply = <&v3v3>; + vqmmc-supply = <&vdd>; + mmc-ddr-3_3v; + status = "disabled"; +}; + +&timers6 { + status = "disabled"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; + timer@5 { + status = "okay"; + }; +}; + +&uart4 { + pinctrl-names = "default", "sleep", "idle"; + pinctrl-0 = <&uart4_pins_a>; + pinctrl-1 = <&uart4_sleep_pins_a>; + pinctrl-2 = <&uart4_idle_pins_a>; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; +}; + +&usbotg_hs { + vbus-supply = <&vbus_otg>; +}; + +&usbphyc_port0 { + phy-supply = <&vdd_usb>; +}; + +&usbphyc_port1 { + phy-supply = <&vdd_usb>; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp157c-ev1.dts b/optee_os/core/arch/arm/dts/stm32mp157c-ev1.dts new file mode 100644 index 0000000..a973a2d --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp157c-ev1.dts @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2017 - All Rights Reserved + * Author: Ludovic Barre for STMicroelectronics. + */ +/dts-v1/; + +#include "stm32mp157c-ed1.dts" +#include + +/ { + model = "STMicroelectronics STM32MP157C eval daughter on eval mother"; + compatible = "st,stm32mp157c-ev1", "st,stm32mp157c-ed1", "st,stm32mp157"; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + aliases { + serial0 = &uart4; + serial1 = &usart3; + ethernet0 = ðernet0; + }; + + clocks { + clk_ext_camera: clk-ext-camera { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24000000>; + }; + }; + + joystick { + compatible = "gpio-keys"; + pinctrl-0 = <&joystick_pins>; + pinctrl-names = "default"; + button-0 { + label = "JoySel"; + interrupt-parent = <&stmfx_pinctrl>; + interrupts = <0 IRQ_TYPE_EDGE_RISING>; + }; + button-1 { + label = "JoyDown"; + interrupt-parent = <&stmfx_pinctrl>; + interrupts = <1 IRQ_TYPE_EDGE_RISING>; + }; + button-2 { + label = "JoyLeft"; + interrupt-parent = <&stmfx_pinctrl>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + }; + button-3 { + label = "JoyRight"; + interrupt-parent = <&stmfx_pinctrl>; + interrupts = <3 IRQ_TYPE_EDGE_RISING>; + }; + button-4 { + label = "JoyUp"; + interrupt-parent = <&stmfx_pinctrl>; + interrupts = <4 IRQ_TYPE_EDGE_RISING>; + }; + }; + + panel_backlight: panel-backlight { + compatible = "gpio-backlight"; + gpios = <&gpiod 13 GPIO_ACTIVE_LOW>; + default-on; + status = "okay"; + }; +}; + +&cec { + pinctrl-names = "default"; + pinctrl-0 = <&cec_pins_a>; + status = "disabled"; +}; + +&dcmi { + status = "disabled"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&dcmi_pins_a>; + pinctrl-1 = <&dcmi_sleep_pins_a>; + + port { + dcmi_0: endpoint { + remote-endpoint = <&ov5640_0>; + bus-type = <5>; + bus-width = <8>; + hsync-active = <0>; + vsync-active = <0>; + pclk-sample = <1>; + }; + }; +}; + +&dsi { + phy-dsi-supply = <®18>; + status = "disabled"; + + ports { + port@0 { + reg = <0>; + dsi_in: endpoint { + remote-endpoint = <<dc_ep0_out>; + }; + }; + + port@1 { + reg = <1>; + dsi_out: endpoint { + remote-endpoint = <&dsi_panel_in>; + }; + }; + }; + + panel-dsi@0 { + compatible = "raydium,rm68200"; + reg = <0>; + reset-gpios = <&gpiof 15 GPIO_ACTIVE_LOW>; + backlight = <&panel_backlight>; + power-supply = <&v3v3>; + status = "okay"; + + port { + dsi_panel_in: endpoint { + remote-endpoint = <&dsi_out>; + }; + }; + }; +}; + +ðernet0 { + status = "disabled"; + pinctrl-0 = <ðernet0_rgmii_pins_a>; + pinctrl-1 = <ðernet0_rgmii_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + phy-mode = "rgmii-id"; + max-speed = <1000>; + phy-handle = <&phy0>; + + mdio0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwmac-mdio"; + phy0: ethernet-phy@0 { + reg = <0>; + }; + }; +}; + +&fmc { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&fmc_pins_a>; + pinctrl-1 = <&fmc_sleep_pins_a>; + status = "disabled"; + + nand-controller@4,0 { + status = "okay"; + + nand@0 { + reg = <0>; + nand-on-flash-bbt; + #address-cells = <1>; + #size-cells = <1>; + }; + }; +}; + +&i2c2 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c2_pins_a>; + pinctrl-1 = <&i2c2_sleep_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "disabled"; + + ov5640: camera@3c { + compatible = "ovti,ov5640"; + reg = <0x3c>; + clocks = <&clk_ext_camera>; + clock-names = "xclk"; + DOVDD-supply = <&v2v8>; + powerdown-gpios = <&stmfx_pinctrl 18 (GPIO_ACTIVE_HIGH | GPIO_PUSH_PULL)>; + reset-gpios = <&stmfx_pinctrl 19 (GPIO_ACTIVE_LOW | GPIO_PUSH_PULL)>; + rotation = <180>; + status = "okay"; + + port { + ov5640_0: endpoint { + remote-endpoint = <&dcmi_0>; + bus-width = <8>; + data-shift = <2>; /* lines 9:2 are used */ + hsync-active = <0>; + vsync-active = <0>; + pclk-sample = <1>; + }; + }; + }; + + stmfx: stmfx@42 { + compatible = "st,stmfx-0300"; + reg = <0x42>; + interrupts = <8 IRQ_TYPE_EDGE_RISING>; + interrupt-parent = <&gpioi>; + vdd-supply = <&v3v3>; + + stmfx_pinctrl: pinctrl { + compatible = "st,stmfx-0300-pinctrl"; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&stmfx_pinctrl 0 0 24>; + + joystick_pins: joystick-pins { + pins = "gpio0", "gpio1", "gpio2", "gpio3", "gpio4"; + bias-pull-down; + }; + }; + }; +}; + +&i2c5 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c5_pins_a>; + pinctrl-1 = <&i2c5_sleep_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "disabled"; +}; + +<dc { + status = "disabled"; + + port { + ltdc_ep0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&dsi_in>; + }; + }; +}; + +&m_can1 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&m_can1_pins_a>; + pinctrl-1 = <&m_can1_sleep_pins_a>; + status = "disabled"; +}; + +&qspi { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&qspi_clk_pins_a &qspi_bk1_pins_a &qspi_bk2_pins_a>; + pinctrl-1 = <&qspi_clk_sleep_pins_a &qspi_bk1_sleep_pins_a &qspi_bk2_sleep_pins_a>; + reg = <0x58003000 0x1000>, <0x70000000 0x4000000>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + flash0: flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + flash1: flash@1 { + compatible = "jedec,spi-nor"; + reg = <1>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; + #address-cells = <1>; + #size-cells = <1>; + }; +}; + +&sdmmc3 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc3_b4_pins_a>; + pinctrl-1 = <&sdmmc3_b4_od_pins_a>; + pinctrl-2 = <&sdmmc3_b4_sleep_pins_a>; + broken-cd; + st,neg-edge; + bus-width = <4>; + vmmc-supply = <&v3v3>; + status = "disabled"; +}; + +&spi1 { + pinctrl-names = "default"; + pinctrl-0 = <&spi1_pins_a>; + status = "disabled"; +}; + +&timers2 { + /* spare dmas for other usage (un-delete to enable pwm capture) */ + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm2_pins_a>; + pinctrl-1 = <&pwm2_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@1 { + status = "okay"; + }; +}; + +&timers8 { + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm8_pins_a>; + pinctrl-1 = <&pwm8_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@7 { + status = "okay"; + }; +}; + +&timers12 { + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm12_pins_a>; + pinctrl-1 = <&pwm12_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@11 { + status = "okay"; + }; +}; + +&usart3 { + pinctrl-names = "default", "sleep", "idle"; + pinctrl-0 = <&usart3_pins_b>; + pinctrl-1 = <&usart3_sleep_pins_b>; + pinctrl-2 = <&usart3_idle_pins_b>; + /* + * HW flow control USART3_RTS is optional, and isn't default wired to + * the connector. SB23 needs to be soldered in order to use it, and R77 + * (ETH_CLK) should be removed. + */ + uart-has-rtscts; + status = "disabled"; +}; + +&usbh_ehci { + phys = <&usbphyc_port0>; + status = "disabled"; +}; + +&usbotg_hs { + pinctrl-0 = <&usbotg_hs_pins_a>; + pinctrl-names = "default"; + phys = <&usbphyc_port1 0>; + phy-names = "usb2-phy"; + status = "disabled"; +}; + +&usbphyc { + status = "disabled"; +}; + +&usbphyc_port0 { + st,tune-hs-dc-level = <2>; + st,enable-fs-rftime-tuning; + st,enable-hs-rftime-reduction; + st,trim-hs-current = <15>; + st,trim-hs-impedance = <1>; + st,tune-squelch-level = <3>; + st,tune-hs-rx-offset = <2>; + st,no-lsfs-sc; +}; + +&usbphyc_port1 { + st,tune-hs-dc-level = <2>; + st,enable-fs-rftime-tuning; + st,enable-hs-rftime-reduction; + st,trim-hs-current = <15>; + st,trim-hs-impedance = <1>; + st,tune-squelch-level = <3>; + st,tune-hs-rx-offset = <2>; + st,no-lsfs-sc; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xc.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xc.dtsi new file mode 100644 index 0000000..b06a55a --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xc.dtsi @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +/ { + soc { + cryp1: cryp@54001000 { + compatible = "st,stm32mp1-cryp"; + reg = <0x54001000 0x400>; + interrupts = ; + clocks = <&rcc CRYP1>; + resets = <&rcc CRYP1_R>; + status = "disabled"; + }; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xx-dhcom-pdk2.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcom-pdk2.dtsi new file mode 100644 index 0000000..f1ff04f --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcom-pdk2.dtsi @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause +/* + * Copyright (C) 2019-2020 Marek Vasut + * Copyright (C) 2022 DH electronics GmbH + */ + +/ { + aliases { + serial0 = &uart4; + serial1 = &usart3; + serial2 = &uart8; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + clk_ext_audio_codec: clock-codec { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + }; + + display_bl: display-bl { + compatible = "pwm-backlight"; + pwms = <&pwm2 3 500000 1>; + brightness-levels = <0 16 22 30 40 55 75 102 138 188 255>; + default-brightness-level = <8>; + enable-gpios = <&gpioi 0 GPIO_ACTIVE_HIGH>; + power-supply = <®_panel_bl>; + status = "okay"; + }; + + gpio-keys-polled { + compatible = "gpio-keys-polled"; + poll-interval = <20>; + + /* + * The EXTi IRQ line 3 is shared with ethernet, + * so mark this as polled GPIO key. + */ + button-0 { + label = "TA1-GPIO-A"; + gpios = <&gpiof 3 GPIO_ACTIVE_LOW>; + }; + + /* + * The EXTi IRQ line 6 is shared with touchscreen, + * so mark this as polled GPIO key. + */ + button-1 { + label = "TA2-GPIO-B"; + gpios = <&gpiod 6 GPIO_ACTIVE_LOW>; + }; + + /* + * The EXTi IRQ line 0 is shared with PMIC, + * so mark this as polled GPIO key. + */ + button-2 { + label = "TA3-GPIO-C"; + gpios = <&gpiog 0 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + button-3 { + label = "TA4-GPIO-D"; + gpios = <&gpiod 12 GPIO_ACTIVE_LOW>; + wakeup-source; + }; + }; + + led { + compatible = "gpio-leds"; + + led-0 { + label = "green:led5"; + gpios = <&gpioc 6 GPIO_ACTIVE_HIGH>; + default-state = "off"; + status = "disabled"; + }; + + led-1 { + label = "green:led6"; + gpios = <&gpiod 11 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-2 { + label = "green:led7"; + gpios = <&gpioi 2 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-3 { + label = "green:led8"; + gpios = <&gpioi 3 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; + + panel { + compatible = "edt,etm0700g0edh6"; + backlight = <&display_bl>; + power-supply = <®_panel_bl>; + + port { + lcd_panel_in: endpoint { + remote-endpoint = <&lcd_display_out>; + }; + }; + }; + + reg_panel_bl: regulator-panel-bl { + compatible = "regulator-fixed"; + regulator-name = "panel_backlight"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <®_panel_supply>; + }; + + reg_panel_supply: regulator-panel-supply { + compatible = "regulator-fixed"; + regulator-name = "panel_supply"; + regulator-min-microvolt = <24000000>; + regulator-max-microvolt = <24000000>; + }; + + sound { + compatible = "audio-graph-card"; + routing = + "MIC_IN", "Capture", + "Capture", "Mic Bias", + "Playback", "HP_OUT"; + dais = <&sai2a_port &sai2b_port>; + status = "okay"; + }; +}; + +&cec { + pinctrl-names = "default"; + pinctrl-0 = <&cec_pins_a>; + status = "okay"; +}; + +&i2c2 { /* Header X22 */ + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "okay"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; +}; + +&i2c5 { /* Header X21 */ + pinctrl-names = "default"; + pinctrl-0 = <&i2c5_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "okay"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; + + sgtl5000: codec@a { + compatible = "fsl,sgtl5000"; + reg = <0x0a>; + #sound-dai-cells = <0>; + clocks = <&clk_ext_audio_codec>; + VDDA-supply = <&v3v3>; + VDDIO-supply = <&vdd>; + + sgtl5000_port: port { + #address-cells = <1>; + #size-cells = <0>; + + sgtl5000_tx_endpoint: endpoint@0 { + reg = <0>; + remote-endpoint = <&sai2a_endpoint>; + frame-master = <&sgtl5000_tx_endpoint>; + bitclock-master = <&sgtl5000_tx_endpoint>; + }; + + sgtl5000_rx_endpoint: endpoint@1 { + reg = <1>; + remote-endpoint = <&sai2b_endpoint>; + frame-master = <&sgtl5000_rx_endpoint>; + bitclock-master = <&sgtl5000_rx_endpoint>; + }; + }; + + }; + + touchscreen@38 { + compatible = "edt,edt-ft5406"; + reg = <0x38>; + interrupt-parent = <&gpioc>; + interrupts = <6 IRQ_TYPE_EDGE_FALLING>; /* GPIO E */ + }; +}; + +<dc { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <<dc_pins_b>; + pinctrl-1 = <<dc_sleep_pins_b>; + status = "okay"; + + port { + lcd_display_out: endpoint { + remote-endpoint = <&lcd_panel_in>; + }; + }; +}; + +&sai2 { + clocks = <&rcc SAI2>, <&rcc PLL3_Q>, <&rcc PLL3_R>; + clock-names = "pclk", "x8k", "x11k"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sai2a_pins_b &sai2b_pins_b>; + pinctrl-1 = <&sai2a_sleep_pins_b &sai2b_sleep_pins_b>; + status = "okay"; + + sai2a: audio-controller@4400b004 { + #clock-cells = <0>; + dma-names = "tx"; + clocks = <&rcc SAI2_K>; + clock-names = "sai_ck"; + status = "okay"; + + sai2a_port: port { + sai2a_endpoint: endpoint { + remote-endpoint = <&sgtl5000_tx_endpoint>; + format = "i2s"; + mclk-fs = <512>; + dai-tdm-slot-num = <2>; + dai-tdm-slot-width = <16>; + }; + }; + }; + + sai2b: audio-controller@4400b024 { + dma-names = "rx"; + st,sync = <&sai2a 2>; + clocks = <&rcc SAI2_K>, <&sai2a>; + clock-names = "sai_ck", "MCLK"; + status = "okay"; + + sai2b_port: port { + sai2b_endpoint: endpoint { + remote-endpoint = <&sgtl5000_rx_endpoint>; + format = "i2s"; + mclk-fs = <512>; + dai-tdm-slot-num = <2>; + dai-tdm-slot-width = <16>; + }; + }; + }; +}; + +&timers2 { + /* spare dmas for other usage (un-delete to enable pwm capture) */ + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; + pwm2: pwm { + pinctrl-0 = <&pwm2_pins_a>; + pinctrl-names = "default"; + status = "okay"; + }; + timer@1 { + status = "okay"; + }; +}; + +&usart3 { + pinctrl-names = "default"; + pinctrl-0 = <&usart3_pins_a>; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; +}; + +&uart8 { + pinctrl-names = "default"; + pinctrl-0 = <&uart8_pins_a &uart8_rtscts_pins_a>; + uart-has-rtscts; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; +}; + +&usbh_ehci { + phys = <&usbphyc_port0>; + status = "okay"; +}; + +&usbotg_hs { + dr_mode = "otg"; + pinctrl-0 = <&usbotg_hs_pins_a>; + pinctrl-names = "default"; + phy-names = "usb2-phy"; + phys = <&usbphyc_port1 0>; + vbus-supply = <&vbus_otg>; + status = "okay"; +}; + +&usbphyc { + status = "okay"; +}; + +&usbphyc_port0 { + phy-supply = <&vdd_usb>; +}; + +&usbphyc_port1 { + phy-supply = <&vdd_usb>; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xx-dhcom-som.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcom-som.dtsi new file mode 100644 index 0000000..b4fc6da --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcom-som.dtsi @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) 2019-2020 Marek Vasut + * Copyright (C) 2022 DH electronics GmbH + */ + +#include "stm32mp15-pinctrl.dtsi" +#include "stm32mp15xxaa-pinctrl.dtsi" +#include +#include + +/ { + aliases { + ethernet0 = ðernet0; + ethernet1 = &ksz8851; + rtc0 = &hwrtc; + rtc1 = &rtc; + }; + + memory@c0000000 { + device_type = "memory"; + reg = <0xC0000000 0x40000000>; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + mcuram2: mcuram2@10000000 { + compatible = "shared-dma-pool"; + reg = <0x10000000 0x40000>; + no-map; + }; + + vdev0vring0: vdev0vring0@10040000 { + compatible = "shared-dma-pool"; + reg = <0x10040000 0x1000>; + no-map; + }; + + vdev0vring1: vdev0vring1@10041000 { + compatible = "shared-dma-pool"; + reg = <0x10041000 0x1000>; + no-map; + }; + + vdev0buffer: vdev0buffer@10042000 { + compatible = "shared-dma-pool"; + reg = <0x10042000 0x4000>; + no-map; + }; + + mcuram: mcuram@30000000 { + compatible = "shared-dma-pool"; + reg = <0x30000000 0x40000>; + no-map; + }; + + retram: retram@38000000 { + compatible = "shared-dma-pool"; + reg = <0x38000000 0x10000>; + no-map; + }; + }; + + ethernet_vio: vioregulator { + compatible = "regulator-fixed"; + regulator-name = "vio"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&gpiog 3 GPIO_ACTIVE_LOW>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vdd>; + }; +}; + +&adc { + vdd-supply = <&vdd>; + vdda-supply = <&vdda>; + vref-supply = <&vdda>; + status = "okay"; + + adc1: adc@0 { + st,min-sample-time-nsecs = <5000>; + st,adc-channels = <0>; + status = "okay"; + }; + + adc2: adc@100 { + st,adc-channels = <1>; + st,min-sample-time-nsecs = <5000>; + status = "okay"; + }; +}; + +&crc1 { + status = "okay"; +}; + +&dac { + pinctrl-names = "default"; + pinctrl-0 = <&dac_ch1_pins_a &dac_ch2_pins_a>; + vref-supply = <&vdda>; + status = "okay"; + + dac1: dac@1 { + status = "okay"; + }; + dac2: dac@2 { + status = "okay"; + }; +}; + +&dts { + status = "okay"; +}; + +ðernet0 { + status = "okay"; + pinctrl-0 = <ðernet0_rmii_pins_c &mco2_pins_a>; + pinctrl-1 = <ðernet0_rmii_sleep_pins_c &mco2_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + phy-mode = "rmii"; + max-speed = <100>; + phy-handle = <&phy0>; + + mdio0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwmac-mdio"; + + phy0: ethernet-phy@1 { + reg = <1>; + /* LAN8710Ai */ + compatible = "ethernet-phy-id0007.c0f0", + "ethernet-phy-ieee802.3-c22"; + clocks = <&rcc CK_MCO2>; + reset-gpios = <&gpioh 3 GPIO_ACTIVE_LOW>; + reset-assert-us = <500>; + reset-deassert-us = <500>; + smsc,disable-energy-detect; + interrupt-parent = <&gpioi>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + }; + }; +}; + +&fmc { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&fmc_pins_b>; + pinctrl-1 = <&fmc_sleep_pins_b>; + status = "okay"; + + ksz8851: ethernet@1,0 { + compatible = "micrel,ks8851-mll"; + reg = <1 0x0 0x2>, <1 0x2 0x20000>; + interrupt-parent = <&gpioc>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + bank-width = <2>; + + /* Timing values are in nS */ + st,fmc2-ebi-cs-mux-enable; + st,fmc2-ebi-cs-transaction-type = <4>; + st,fmc2-ebi-cs-buswidth = <16>; + st,fmc2-ebi-cs-address-setup-ns = <5>; + st,fmc2-ebi-cs-address-hold-ns = <5>; + st,fmc2-ebi-cs-bus-turnaround-ns = <5>; + st,fmc2-ebi-cs-data-setup-ns = <45>; + st,fmc2-ebi-cs-data-hold-ns = <1>; + st,fmc2-ebi-cs-write-address-setup-ns = <5>; + st,fmc2-ebi-cs-write-address-hold-ns = <5>; + st,fmc2-ebi-cs-write-bus-turnaround-ns = <5>; + st,fmc2-ebi-cs-write-data-setup-ns = <45>; + st,fmc2-ebi-cs-write-data-hold-ns = <1>; + }; +}; + +&gpioa { + gpio-line-names = "", "", "", "", + "", "", "DHCOM-K", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpiob { + gpio-line-names = "", "", "", "", + "", "", "", "", + "DHCOM-Q", "", "", "", + "", "", "", ""; +}; + +&gpioc { + gpio-line-names = "", "", "", "", + "", "", "DHCOM-E", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpiod { + gpio-line-names = "", "", "", "", + "", "", "DHCOM-B", "", + "", "", "", "DHCOM-F", + "DHCOM-D", "", "", ""; +}; + +&gpioe { + gpio-line-names = "", "", "", "", + "", "", "DHCOM-P", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpiof { + gpio-line-names = "", "", "", "DHCOM-A", + "", "", "", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpiog { + gpio-line-names = "DHCOM-C", "", "", "", + "", "", "", "", + "DHCOM-L", "", "", "", + "", "", "", ""; +}; + +&gpioh { + gpio-line-names = "", "", "", "", + "", "", "", "DHCOM-N", + "DHCOM-J", "DHCOM-W", "DHCOM-V", "DHCOM-U", + "DHCOM-T", "", "DHCOM-S", ""; +}; + +&gpioi { + gpio-line-names = "DHCOM-G", "DHCOM-O", "DHCOM-H", "DHCOM-I", + "DHCOM-R", "DHCOM-M", "", "", + "", "", "", "", + "", "", "", ""; +}; + +&i2c4 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "okay"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; + + hwrtc: rtc@32 { + compatible = "microcrystal,rv8803"; + reg = <0x32>; + }; + + pmic: stpmic@33 { + compatible = "st,stpmic1"; + reg = <0x33>; + interrupts-extended = <&gpioa 0 IRQ_TYPE_EDGE_FALLING>; + interrupt-controller; + #interrupt-cells = <2>; + status = "okay"; + + regulators { + compatible = "st,stpmic1-regulators"; + ldo1-supply = <&v3v3>; + ldo2-supply = <&v3v3>; + ldo3-supply = <&vdd_ddr>; + ldo5-supply = <&v3v3>; + ldo6-supply = <&v3v3>; + pwr_sw1-supply = <&bst_out>; + pwr_sw2-supply = <&bst_out>; + + vddcore: buck1 { + regulator-name = "vddcore"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd_ddr: buck2 { + regulator-name = "vdd_ddr"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd: buck3 { + regulator-name = "vdd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + st,mask-reset; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + v3v3: buck4 { + regulator-name = "v3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-over-current-protection; + regulator-initial-mode = <0>; + }; + + vdda: ldo1 { + regulator-name = "vdda"; + regulator-always-on; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + interrupts = ; + }; + + v2v8: ldo2 { + regulator-name = "v2v8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + interrupts = ; + }; + + vtt_ddr: ldo3 { + regulator-name = "vtt_ddr"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <750000>; + regulator-always-on; + regulator-over-current-protection; + }; + + vdd_usb: ldo4 { + regulator-name = "vdd_usb"; + interrupts = ; + }; + + vdd_sd: ldo5 { + regulator-name = "vdd_sd"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + interrupts = ; + regulator-boot-on; + }; + + v1v8: ldo6 { + regulator-name = "v1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + interrupts = ; + }; + + vref_ddr: vref_ddr { + regulator-name = "vref_ddr"; + regulator-always-on; + }; + + bst_out: boost { + regulator-name = "bst_out"; + interrupts = ; + }; + + vbus_otg: pwr_sw1 { + regulator-name = "vbus_otg"; + interrupts = ; + }; + + vbus_sw: pwr_sw2 { + regulator-name = "vbus_sw"; + interrupts = ; + regulator-active-discharge = <1>; + }; + }; + + onkey { + compatible = "st,stpmic1-onkey"; + interrupts = , ; + interrupt-names = "onkey-falling", "onkey-rising"; + power-off-time-sec = <10>; + status = "okay"; + }; + + watchdog { + compatible = "st,stpmic1-wdt"; + status = "disabled"; + }; + }; + + touchscreen@49 { + compatible = "ti,tsc2004"; + reg = <0x49>; + vio-supply = <&v3v3>; + interrupts-extended = <&gpioh 15 IRQ_TYPE_EDGE_FALLING>; + }; + + eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + pagesize = <16>; + }; +}; + +&ipcc { + status = "okay"; +}; + +&iwdg2 { + timeout-sec = <32>; + status = "okay"; + secure-status = "disabled"; +}; + +&m4_rproc { + memory-region = <&retram>, <&mcuram>, <&mcuram2>, <&vdev0vring0>, + <&vdev0vring1>, <&vdev0buffer>; + mboxes = <&ipcc 0>, <&ipcc 1>, <&ipcc 2>; + mbox-names = "vq0", "vq1", "shutdown"; + interrupt-parent = <&exti>; + interrupts = <68 1>; + status = "okay"; +}; + +&pwr_regulators { + vdd-supply = <&vdd>; + vdd_3v3_usbfs-supply = <&vdd_usb>; +}; + +&qspi { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&qspi_clk_pins_a &qspi_bk1_pins_a>; + pinctrl-1 = <&qspi_clk_sleep_pins_a &qspi_bk1_sleep_pins_a>; + reg = <0x58003000 0x1000>, <0x70000000 0x4000000>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + flash0: flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; + #address-cells = <1>; + #size-cells = <1>; + }; +}; + +&rcc { + /* Connect MCO2 output to ETH_RX_CLK input via pad-pad connection */ + /* clocks = <&rcc CK_MCO2>; Not supported in OP-TEE OS */ + /* clock-names = "ETH_RX_CLK/ETH_REF_CLK"; Not supported */ + + /* + * Set PLL4P output to 100 MHz to supply SDMMC with faster clock, + * set MCO2 output to 50 MHz to supply ETHRX clock with PLL4P/2, + * so that MCO2 behaves as a divider for the ETHRX clock here. + */ + /* assigned-clocks = <&rcc CK_MCO2>, <&rcc PLL4_P>; Not supported */ + /* assigned-clock-parents = <&rcc PLL4_P>; Not supported */ + /* assigned-clock-rates = <50000000>, <100000000>; Not supported */ + + status = "okay"; +}; + +&rng1 { + status = "okay"; +}; + +&rtc { + status = "okay"; +}; + +&sdmmc1 { + pinctrl-names = "default", "opendrain", "sleep", "init"; + pinctrl-0 = <&sdmmc1_b4_pins_a &sdmmc1_dir_pins_a>; + pinctrl-1 = <&sdmmc1_b4_od_pins_a &sdmmc1_dir_pins_a>; + pinctrl-2 = <&sdmmc1_b4_sleep_pins_a &sdmmc1_dir_sleep_pins_a>; + pinctrl-3 = <&sdmmc1_b4_init_pins_a &sdmmc1_dir_init_pins_a>; + cd-gpios = <&gpiog 1 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; + disable-wp; + st,sig-dir; + st,neg-edge; + st,use-ckin; + st,cmd-gpios = <&gpiod 2 0>; + st,ck-gpios = <&gpioc 12 0>; + st,ckin-gpios = <&gpioe 4 0>; + bus-width = <4>; + vmmc-supply = <&vdd_sd>; + status = "okay"; +}; + +&sdmmc1_b4_pins_a { + /* + * SD bus pull-up resistors: + * - optional on SoMs with SD voltage translator + * - mandatory on SoMs without SD voltage translator + */ + pins1 { + bias-pull-up; + }; + pins2 { + bias-pull-up; + }; +}; + +&sdmmc2 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc2_b4_pins_a &sdmmc2_d47_pins_a>; + pinctrl-1 = <&sdmmc2_b4_od_pins_a &sdmmc2_d47_pins_a>; + pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_a>; + non-removable; + no-sd; + no-sdio; + st,neg-edge; + bus-width = <8>; + vmmc-supply = <&v3v3>; + vqmmc-supply = <&v3v3>; + mmc-ddr-3_3v; + status = "okay"; +}; + +&sdmmc3 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc3_b4_pins_a>; + pinctrl-1 = <&sdmmc3_b4_od_pins_a>; + pinctrl-2 = <&sdmmc3_b4_sleep_pins_a>; + broken-cd; + st,neg-edge; + bus-width = <4>; + vmmc-supply = <&v3v3>; + vqmmc-supply = <&v3v3>; + mmc-ddr-3_3v; + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_pins_a>; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-avenger96.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-avenger96.dtsi new file mode 100644 index 0000000..76c54b0 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-avenger96.dtsi @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright (C) Linaro Ltd 2019 - All Rights Reserved + * Author: Manivannan Sadhasivam + * Copyright (C) 2020 Marek Vasut + */ + +/* Avenger96 uses DHCOR SoM configured for 1V8 IO operation */ +#include "stm32mp15xx-dhcor-io1v8.dtsi" + +/ { + aliases { + ethernet0 = ðernet0; + mmc0 = &sdmmc1; + serial0 = &uart4; + serial1 = &uart7; + serial2 = &usart2; + spi0 = &qspi; + }; + + /* XTal Q1 */ + cec_clock: clk-cec-fixed { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24000000>; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + hdmi-out { + compatible = "hdmi-connector"; + type = "a"; + + port { + hdmi_con: endpoint { + remote-endpoint = <&adv7513_out>; + }; + }; + }; + + led { + compatible = "gpio-leds"; + led1 { + label = "green:user0"; + gpios = <&gpioz 7 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led2 { + label = "green:user1"; + gpios = <&gpiof 3 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led3 { + label = "green:user2"; + gpios = <&gpiog 0 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + + led4 { + label = "green:user3"; + gpios = <&gpiog 1 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "none"; + default-state = "off"; + panic-indicator; + }; + }; + + sd_switch: regulator-sd_switch { + compatible = "regulator-gpio"; + regulator-name = "sd_switch"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2900000>; + regulator-type = "voltage"; + regulator-always-on; + + gpios = <&gpioi 5 GPIO_ACTIVE_HIGH>; + gpios-states = <0>; + states = <1800000 0x1>, + <2900000 0x0>; + }; + + sound { + compatible = "audio-graph-card"; + label = "STM32MP1-AV96-HDMI"; + dais = <&sai2a_port>; + status = "okay"; + }; + + wlan_pwr: regulator-wlan { + compatible = "regulator-fixed"; + + regulator-name = "wl-reg"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpios = <&gpioz 3 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&adc { + pinctrl-names = "default"; + pinctrl-0 = <&adc12_ain_pins_b>; + vdd-supply = <&vdd>; + vdda-supply = <&vdda>; + vref-supply = <&vdda>; + status = "okay"; + + adc1: adc@0 { + st,adc-channels = <0 1 6>; + st,min-sample-time-nsecs = <5000>; + status = "okay"; + }; + + adc2: adc@100 { + st,adc-channels = <0 1 2>; + st,min-sample-time-nsecs = <5000>; + status = "okay"; + }; +}; + +ðernet0 { + status = "okay"; + pinctrl-0 = <ðernet0_rgmii_pins_c>; + pinctrl-1 = <ðernet0_rgmii_sleep_pins_c>; + pinctrl-names = "default", "sleep"; + phy-mode = "rgmii"; + max-speed = <1000>; + phy-handle = <&phy0>; + + mdio0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwmac-mdio"; + reset-gpios = <&gpioz 2 GPIO_ACTIVE_LOW>; + reset-delay-us = <1000>; + reset-post-delay-us = <1000>; + + phy0: ethernet-phy@7 { + reg = <7>; + + rxc-skew-ps = <1500>; + rxdv-skew-ps = <540>; + rxd0-skew-ps = <420>; + rxd1-skew-ps = <420>; + rxd2-skew-ps = <420>; + rxd3-skew-ps = <420>; + + txc-skew-ps = <1440>; + txen-skew-ps = <540>; + txd0-skew-ps = <420>; + txd1-skew-ps = <420>; + txd2-skew-ps = <420>; + txd3-skew-ps = <420>; + }; + }; +}; + +&gpioa { + gpio-line-names = "", "", "", "", + "", "", "", "", + "", "", "", "AV96-K", + "AV96-I", "", "AV96-A", ""; +}; + +&gpiob { + gpio-line-names = "", "", "", "", + "", "AV96-J", "", "", + "", "", "", "AV96-B", + "", "AV96-L", "", ""; +}; + +&gpioc { + gpio-line-names = "", "", "", "AV96-C", + "", "", "", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpiod { + gpio-line-names = "", "", "", "", + "", "", "", "", + "AV96-D", "", "", "", + "", "", "AV96-E", "AV96-F"; +}; + +&gpiof { + gpio-line-names = "", "", "", "", + "", "", "", "", + "", "", "", "", + "AV96-G", "AV96-H", "", ""; +}; + +&i2c1 { /* X6 I2C1 */ + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins_b>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "okay"; + /delete-property/dmas; + /delete-property/dma-names; +}; + +&i2c2 { /* X6 I2C2 */ + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins_c>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "okay"; + /delete-property/dmas; + /delete-property/dma-names; +}; + +&i2c4 { + hdmi-transmitter@3d { + compatible = "adi,adv7513"; + reg = <0x3d>, <0x4d>, <0x2d>, <0x5d>; + reg-names = "main", "edid", "cec", "packet"; + clocks = <&cec_clock>; + clock-names = "cec"; + + avdd-supply = <&v3v3>; + dvdd-supply = <&v3v3>; + pvdd-supply = <&v3v3>; + dvdd-3v-supply = <&v3v3>; + bgvdd-supply = <&v3v3>; + + interrupts = <9 IRQ_TYPE_EDGE_FALLING>; + interrupt-parent = <&gpiog>; + + status = "okay"; + + adi,input-depth = <8>; + adi,input-colorspace = "rgb"; + adi,input-clock = "1x"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + adv7513_in: endpoint { + remote-endpoint = <<dc_ep0_out>; + }; + }; + + port@1 { + reg = <1>; + adv7513_out: endpoint { + remote-endpoint = <&hdmi_con>; + }; + }; + + port@2 { + reg = <2>; + adv7513_i2s0: endpoint { + remote-endpoint = <&sai2a_endpoint>; + }; + }; + }; + }; +}; + +<dc { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <<dc_pins_d>; + pinctrl-1 = <<dc_sleep_pins_d>; + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + ltdc_ep0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&adv7513_in>; + }; + }; +}; + +&sai2 { + clocks = <&rcc SAI2>, <&rcc PLL3_Q>, <&rcc PLL3_R>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sai2a_pins_c>; + pinctrl-1 = <&sai2a_sleep_pins_c>; + clock-names = "pclk", "x8k", "x11k"; + status = "okay"; + + sai2a: audio-controller@4400b004 { + #clock-cells = <0>; + dma-names = "tx"; + clocks = <&rcc SAI2_K>; + clock-names = "sai_ck"; + status = "okay"; + + sai2a_port: port { + sai2a_endpoint: endpoint { + remote-endpoint = <&adv7513_i2s0>; + format = "i2s"; + mclk-fs = <256>; + }; + }; + }; +}; + +&sdmmc1 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc1_b4_pins_a &sdmmc1_dir_pins_b>; + pinctrl-1 = <&sdmmc1_b4_od_pins_a &sdmmc1_dir_pins_b>; + pinctrl-2 = <&sdmmc1_b4_sleep_pins_a &sdmmc1_dir_sleep_pins_b>; + cd-gpios = <&gpioi 8 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; + disable-wp; + st,sig-dir; + st,neg-edge; + st,use-ckin; + bus-width = <4>; + vmmc-supply = <&vdd_sd>; + vqmmc-supply = <&sd_switch>; + status = "okay"; +}; + +&sdmmc2 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc2_b4_pins_a &sdmmc2_d47_pins_c>; + pinctrl-1 = <&sdmmc2_b4_od_pins_a &sdmmc2_d47_pins_c>; + pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_c>; + bus-width = <8>; + mmc-ddr-1_8v; + no-sd; + no-sdio; + non-removable; + st,neg-edge; + vmmc-supply = <&v3v3>; + vqmmc-supply = <&vdd_io>; + status = "okay"; +}; + +&sdmmc3 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc3_b4_pins_b>; + pinctrl-1 = <&sdmmc3_b4_od_pins_b>; + pinctrl-2 = <&sdmmc3_b4_sleep_pins_b>; + broken-cd; + non-removable; + st,neg-edge; + bus-width = <4>; + vmmc-supply = <&wlan_pwr>; + status = "okay"; + + #address-cells = <1>; + #size-cells = <0>; + brcmf: bcrmf@1 { + reg = <1>; + compatible = "brcm,bcm4329-fmac"; + }; +}; + +&spi2 { + pinctrl-names = "default"; + pinctrl-0 = <&spi2_pins_a>; + cs-gpios = <&gpioi 0 0>; + status = "disabled"; + /delete-property/dmas; + /delete-property/dma-names; +}; + +&uart4 { + /* On Low speed expansion header */ + label = "LS-UART1"; + pinctrl-names = "default"; + pinctrl-0 = <&uart4_pins_b>; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; +}; + +&uart7 { + /* On Low speed expansion header */ + label = "LS-UART0"; + pinctrl-names = "default"; + pinctrl-0 = <&uart7_pins_a>; + uart-has-rtscts; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; +}; + +/* Bluetooth */ +&usart2 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&usart2_pins_a>; + pinctrl-1 = <&usart2_sleep_pins_a>; + st,hw-flow-ctrl; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; + + bluetooth { + compatible = "brcm,bcm43438-bt"; + max-speed = <3000000>; + shutdown-gpios = <&gpioz 6 GPIO_ACTIVE_HIGH>; + }; +}; + +&usbh_ehci { + phys = <&usbphyc_port0>; + phy-names = "usb"; + status = "okay"; +}; + +&usbotg_hs { + pinctrl-0 = <&usbotg_hs_pins_a>; + pinctrl-names = "default"; + phy-names = "usb2-phy"; + phys = <&usbphyc_port1 0>; + status = "okay"; + vbus-supply = <&vbus_otg>; +}; + +&usbphyc { + status = "okay"; +}; + +&usbphyc_port0 { + phy-supply = <&vdd_usb>; +}; + +&usbphyc_port1 { + phy-supply = <&vdd_usb>; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-io1v8.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-io1v8.dtsi new file mode 100644 index 0000000..7517231 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-io1v8.dtsi @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright (C) Linaro Ltd 2019 - All Rights Reserved + * Author: Manivannan Sadhasivam + * Copyright (C) 2020 Marek Vasut + */ + +/ { + /* Enpirion EP3A8LQI U2 on the DHCOR */ + vdd_io: regulator-buck-io { + compatible = "regulator-fixed"; + regulator-name = "buck-io"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vdd>; + }; +}; + +&pwr_regulators { + vdd-supply = <&vdd_io>; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-som.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-som.dtsi new file mode 100644 index 0000000..312f387 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xx-dhcor-som.dtsi @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright (C) Linaro Ltd 2019 - All Rights Reserved + * Author: Manivannan Sadhasivam + * Copyright (C) 2020 Marek Vasut + * Copyright (C) 2022 DH electronics GmbH + */ + +#include "stm32mp15-pinctrl.dtsi" +#include "stm32mp15xxac-pinctrl.dtsi" +#include +#include + +/ { + aliases { + spi0 = &qspi; + }; + + memory@c0000000 { + device_type = "memory"; + reg = <0xc0000000 0x40000000>; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + mcuram2: mcuram2@10000000 { + compatible = "shared-dma-pool"; + reg = <0x10000000 0x40000>; + no-map; + }; + + vdev0vring0: vdev0vring0@10040000 { + compatible = "shared-dma-pool"; + reg = <0x10040000 0x1000>; + no-map; + }; + + vdev0vring1: vdev0vring1@10041000 { + compatible = "shared-dma-pool"; + reg = <0x10041000 0x1000>; + no-map; + }; + + vdev0buffer: vdev0buffer@10042000 { + compatible = "shared-dma-pool"; + reg = <0x10042000 0x4000>; + no-map; + }; + + mcuram: mcuram@30000000 { + compatible = "shared-dma-pool"; + reg = <0x30000000 0x40000>; + no-map; + }; + + retram: retram@38000000 { + compatible = "shared-dma-pool"; + reg = <0x38000000 0x10000>; + no-map; + }; + }; +}; + +&crc1 { + status = "okay"; +}; + +&dts { + status = "okay"; +}; + +&i2c4 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + status = "okay"; + /delete-property/dmas; + /delete-property/dma-names; + + pmic: stpmic@33 { + compatible = "st,stpmic1"; + reg = <0x33>; + interrupts-extended = <&gpioa 0 IRQ_TYPE_EDGE_FALLING>; + interrupt-controller; + #interrupt-cells = <2>; + status = "okay"; + + regulators { + compatible = "st,stpmic1-regulators"; + + ldo1-supply = <&v3v3>; + ldo2-supply = <&v3v3>; + ldo3-supply = <&vdd_ddr>; + ldo5-supply = <&v3v3>; + ldo6-supply = <&v3v3>; + pwr_sw1-supply = <&bst_out>; + pwr_sw2-supply = <&bst_out>; + + vddcore: buck1 { + regulator-name = "vddcore"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd_ddr: buck2 { + regulator-name = "vdd_ddr"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd: buck3 { + regulator-name = "vdd"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + v3v3: buck4 { + regulator-name = "v3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-over-current-protection; + regulator-initial-mode = <0>; + }; + + vdda: ldo1 { + regulator-name = "vdda"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + interrupts = ; + }; + + v2v8: ldo2 { + regulator-name = "v2v8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + interrupts = ; + }; + + vtt_ddr: ldo3 { + regulator-name = "vtt_ddr"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <750000>; + regulator-always-on; + regulator-over-current-protection; + }; + + vdd_usb: ldo4 { + regulator-name = "vdd_usb"; + interrupts = ; + }; + + vdd_sd: ldo5 { + regulator-name = "vdd_sd"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + interrupts = ; + regulator-boot-on; + }; + + v1v8: ldo6 { + regulator-name = "v1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + interrupts = ; + regulator-enable-ramp-delay = <300000>; + }; + + vref_ddr: vref_ddr { + regulator-name = "vref_ddr"; + regulator-always-on; + }; + + bst_out: boost { + regulator-name = "bst_out"; + interrupts = ; + }; + + vbus_otg: pwr_sw1 { + regulator-name = "vbus_otg"; + interrupts = ; + regulator-active-discharge = <1>; + }; + + vbus_sw: pwr_sw2 { + regulator-name = "vbus_sw"; + interrupts = ; + regulator-active-discharge = <1>; + }; + }; + + onkey { + compatible = "st,stpmic1-onkey"; + interrupts = , ; + interrupt-names = "onkey-falling", "onkey-rising"; + status = "okay"; + }; + + watchdog { + compatible = "st,stpmic1-wdt"; + status = "disabled"; + }; + }; + + eeprom@53 { + compatible = "atmel,24c02"; + reg = <0x53>; + pagesize = <16>; + }; +}; + +&ipcc { + status = "okay"; +}; + +&iwdg2 { + timeout-sec = <32>; + status = "okay"; + secure-status = "disabled"; +}; + +&m4_rproc { + memory-region = <&retram>, <&mcuram>, <&mcuram2>, <&vdev0vring0>, + <&vdev0vring1>, <&vdev0buffer>; + mboxes = <&ipcc 0>, <&ipcc 1>, <&ipcc 2>; + mbox-names = "vq0", "vq1", "shutdown"; + interrupt-parent = <&exti>; + interrupts = <68 1>; + status = "okay"; +}; + +&pwr_regulators { + vdd-supply = <&vdd>; + vdd_3v3_usbfs-supply = <&vdd_usb>; +}; + +&qspi { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&qspi_clk_pins_a &qspi_bk1_pins_a>; + pinctrl-1 = <&qspi_clk_sleep_pins_a &qspi_bk1_sleep_pins_a>; + reg = <0x58003000 0x1000>, <0x70000000 0x200000>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + flash0: flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-rx-bus-width = <4>; + spi-max-frequency = <50000000>; + #address-cells = <1>; + #size-cells = <1>; + }; +}; + +&rcc { + status = "okay"; +}; + +&rng1 { + status = "okay"; +}; + +&rtc { + status = "okay"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xx-dkx.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xx-dkx.dtsi new file mode 100644 index 0000000..cb4b6ac --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xx-dkx.dtsi @@ -0,0 +1,734 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +#include +#include + +/ { + memory@c0000000 { + device_type = "memory"; + reg = <0xc0000000 0x20000000>; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + mcuram2: mcuram2@10000000 { + compatible = "shared-dma-pool"; + reg = <0x10000000 0x40000>; + no-map; + }; + + vdev0vring0: vdev0vring0@10040000 { + compatible = "shared-dma-pool"; + reg = <0x10040000 0x1000>; + no-map; + }; + + vdev0vring1: vdev0vring1@10041000 { + compatible = "shared-dma-pool"; + reg = <0x10041000 0x1000>; + no-map; + }; + + vdev0buffer: vdev0buffer@10042000 { + compatible = "shared-dma-pool"; + reg = <0x10042000 0x4000>; + no-map; + }; + + mcuram: mcuram@30000000 { + compatible = "shared-dma-pool"; + reg = <0x30000000 0x40000>; + no-map; + }; + + retram: retram@38000000 { + compatible = "shared-dma-pool"; + reg = <0x38000000 0x10000>; + no-map; + }; + + gpu_reserved: gpu@d4000000 { + reg = <0xd4000000 0x4000000>; + no-map; + }; + }; + + led { + compatible = "gpio-leds"; + led-blue { + label = "heartbeat"; + gpios = <&gpiod 11 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + status = "disabled"; + }; + }; + + sound { + compatible = "audio-graph-card"; + label = "STM32MP1-DK"; + routing = + "Playback" , "MCLK", + "Capture" , "MCLK", + "MICL" , "Mic Bias"; + dais = <&sai2a_port &sai2b_port &i2s2_port>; + status = "disabled"; + }; + + vin: vin { + compatible = "regulator-fixed"; + regulator-name = "vin"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; +}; + +&adc { + pinctrl-names = "default"; + pinctrl-0 = <&adc12_ain_pins_a>, <&adc12_usb_cc_pins_a>; + vdd-supply = <&vdd>; + vdda-supply = <&vdd>; + vref-supply = <&vrefbuf>; + status = "disabled"; + adc1: adc@0 { + /* + * Type-C USB_PWR_CC1 & USB_PWR_CC2 on in18 & in19. + * Use at least 5 * RC time, e.g. 5 * (Rp + Rd) * C: + * 5 * (56 + 47kOhms) * 5pF => 2.5us. + * Use arbitrary margin here (e.g. 5us). + */ + st,min-sample-time-nsecs = <5000>; + /* AIN connector, USB Type-C CC1 & CC2 */ + st,adc-channels = <0 1 6 13 18 19>; + status = "okay"; + }; + adc2: adc@100 { + /* AIN connector, USB Type-C CC1 & CC2 */ + st,adc-channels = <0 1 2 6 18 19>; + st,min-sample-time-nsecs = <5000>; + status = "okay"; + }; +}; + +&cec { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cec_pins_b>; + pinctrl-1 = <&cec_sleep_pins_b>; + status = "disabled"; +}; + +&crc1 { + status = "disabled"; +}; + +&dts { + status = "disabled"; +}; + +ðernet0 { + status = "disabled"; + pinctrl-0 = <ðernet0_rgmii_pins_a>; + pinctrl-1 = <ðernet0_rgmii_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + phy-mode = "rgmii-id"; + max-speed = <1000>; + phy-handle = <&phy0>; + + mdio0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwmac-mdio"; + phy0: ethernet-phy@0 { + reg = <0>; + }; + }; +}; + +&gpu { + contiguous-area = <&gpu_reserved>; +}; + +&hash1 { + status = "disabled"; +}; + +&i2c1 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c1_pins_a>; + pinctrl-1 = <&i2c1_sleep_pins_a>; + i2c-scl-rising-time-ns = <100>; + i2c-scl-falling-time-ns = <7>; + status = "disabled"; + /delete-property/dmas; + /delete-property/dma-names; + + hdmi-transmitter@39 { + compatible = "sil,sii9022"; + reg = <0x39>; + iovcc-supply = <&v3v3_hdmi>; + cvcc12-supply = <&v1v2_hdmi>; + reset-gpios = <&gpioa 10 GPIO_ACTIVE_LOW>; + interrupts = <1 IRQ_TYPE_EDGE_FALLING>; + interrupt-parent = <&gpiog>; + #sound-dai-cells = <0>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + sii9022_in: endpoint { + remote-endpoint = <<dc_ep0_out>; + }; + }; + + port@3 { + reg = <3>; + sii9022_tx_endpoint: endpoint { + remote-endpoint = <&i2s2_endpoint>; + }; + }; + }; + }; + + cs42l51: cs42l51@4a { + compatible = "cirrus,cs42l51"; + reg = <0x4a>; + #sound-dai-cells = <0>; + VL-supply = <&v3v3>; + VD-supply = <&v1v8_audio>; + VA-supply = <&v1v8_audio>; + VAHP-supply = <&v1v8_audio>; + reset-gpios = <&gpiog 9 GPIO_ACTIVE_LOW>; + clocks = <&sai2a>; + clock-names = "MCLK"; + status = "okay"; + + cs42l51_port: port { + #address-cells = <1>; + #size-cells = <0>; + + cs42l51_tx_endpoint: endpoint@0 { + reg = <0>; + remote-endpoint = <&sai2a_endpoint>; + frame-master = <&cs42l51_tx_endpoint>; + bitclock-master = <&cs42l51_tx_endpoint>; + }; + + cs42l51_rx_endpoint: endpoint@1 { + reg = <1>; + remote-endpoint = <&sai2b_endpoint>; + frame-master = <&cs42l51_rx_endpoint>; + bitclock-master = <&cs42l51_rx_endpoint>; + }; + }; + }; +}; + +&i2c4 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c4_pins_a>; + pinctrl-1 = <&i2c4_sleep_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + clock-frequency = <400000>; + status = "okay"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; + + stusb1600@28 { + compatible = "st,stusb1600"; + reg = <0x28>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + interrupt-parent = <&gpioi>; + pinctrl-names = "default"; + pinctrl-0 = <&stusb1600_pins_a>; + status = "okay"; + vdd-supply = <&vin>; + + connector { + compatible = "usb-c-connector"; + label = "USB-C"; + power-role = "dual"; + typec-power-opmode = "default"; + + port { + con_usbotg_hs_ep: endpoint { + remote-endpoint = <&usbotg_hs_ep>; + }; + }; + }; + }; + + pmic: stpmic@33 { + compatible = "st,stpmic1"; + reg = <0x33>; + interrupts-extended = <&gpioa 0 IRQ_TYPE_EDGE_FALLING>; + interrupt-controller; + #interrupt-cells = <2>; + status = "okay"; + + regulators { + compatible = "st,stpmic1-regulators"; + buck1-supply = <&vin>; + buck2-supply = <&vin>; + buck3-supply = <&vin>; + buck4-supply = <&vin>; + ldo1-supply = <&v3v3>; + ldo2-supply = <&vin>; + ldo3-supply = <&vdd_ddr>; + ldo4-supply = <&vin>; + ldo5-supply = <&vin>; + ldo6-supply = <&v3v3>; + vref_ddr-supply = <&vin>; + boost-supply = <&vin>; + pwr_sw1-supply = <&bst_out>; + pwr_sw2-supply = <&bst_out>; + + vddcore: buck1 { + regulator-name = "vddcore"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd_ddr: buck2 { + regulator-name = "vdd_ddr"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + vdd: buck3 { + regulator-name = "vdd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + st,mask-reset; + regulator-initial-mode = <0>; + regulator-over-current-protection; + }; + + v3v3: buck4 { + regulator-name = "v3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-over-current-protection; + regulator-initial-mode = <0>; + }; + + v1v8_audio: ldo1 { + regulator-name = "v1v8_audio"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + interrupts = ; + }; + + v3v3_hdmi: ldo2 { + regulator-name = "v3v3_hdmi"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + interrupts = ; + }; + + vtt_ddr: ldo3 { + regulator-name = "vtt_ddr"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <750000>; + regulator-always-on; + regulator-over-current-protection; + }; + + vdd_usb: ldo4 { + regulator-name = "vdd_usb"; + interrupts = ; + }; + + vdda: ldo5 { + regulator-name = "vdda"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <2900000>; + interrupts = ; + regulator-boot-on; + }; + + v1v2_hdmi: ldo6 { + regulator-name = "v1v2_hdmi"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-always-on; + interrupts = ; + }; + + vref_ddr: vref_ddr { + regulator-name = "vref_ddr"; + regulator-always-on; + }; + + bst_out: boost { + regulator-name = "bst_out"; + interrupts = ; + }; + + vbus_otg: pwr_sw1 { + regulator-name = "vbus_otg"; + interrupts = ; + }; + + vbus_sw: pwr_sw2 { + regulator-name = "vbus_sw"; + interrupts = ; + regulator-active-discharge = <1>; + }; + }; + + onkey { + compatible = "st,stpmic1-onkey"; + interrupts = , ; + interrupt-names = "onkey-falling", "onkey-rising"; + power-off-time-sec = <10>; + status = "okay"; + }; + + watchdog { + compatible = "st,stpmic1-wdt"; + status = "disabled"; + }; + }; +}; + +&i2c5 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c5_pins_a>; + pinctrl-1 = <&i2c5_sleep_pins_a>; + i2c-scl-rising-time-ns = <185>; + i2c-scl-falling-time-ns = <20>; + clock-frequency = <400000>; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; +}; + +&i2s2 { + clocks = <&rcc SPI2>, <&rcc SPI2_K>, <&rcc PLL3_Q>, <&rcc PLL3_R>; + clock-names = "pclk", "i2sclk", "x8k", "x11k"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2s2_pins_a>; + pinctrl-1 = <&i2s2_sleep_pins_a>; + status = "disabled"; + + i2s2_port: port { + i2s2_endpoint: endpoint { + remote-endpoint = <&sii9022_tx_endpoint>; + format = "i2s"; + mclk-fs = <256>; + }; + }; +}; + +&ipcc { + status = "disabled"; +}; + +&iwdg1 { + timeout-sec = <32>; +}; + +&iwdg2 { + timeout-sec = <32>; + status = "okay"; + secure-status = "disabled"; +}; + +<dc { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <<dc_pins_a>; + pinctrl-1 = <<dc_sleep_pins_a>; + status = "disabled"; + + port { + ltdc_ep0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&sii9022_in>; + }; + }; +}; + +&m4_rproc { + memory-region = <&retram>, <&mcuram>, <&mcuram2>, <&vdev0vring0>, + <&vdev0vring1>, <&vdev0buffer>; + mboxes = <&ipcc 0>, <&ipcc 1>, <&ipcc 2>, <&ipcc 3>; + mbox-names = "vq0", "vq1", "shutdown", "detach"; + interrupt-parent = <&exti>; + interrupts = <68 1>; + status = "okay"; +}; + +&pwr_regulators { + vdd-supply = <&vdd>; + vdd_3v3_usbfs-supply = <&vdd_usb>; +}; + +&rng1 { + status = "okay"; +}; + +&rtc { + status = "okay"; +}; + +&sai2 { + clocks = <&rcc SAI2>, <&rcc PLL3_Q>, <&rcc PLL3_R>; + clock-names = "pclk", "x8k", "x11k"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sai2a_pins_a>, <&sai2b_pins_b>; + pinctrl-1 = <&sai2a_sleep_pins_a>, <&sai2b_sleep_pins_b>; + status = "disabled"; + + sai2a: audio-controller@4400b004 { + #clock-cells = <0>; + dma-names = "tx"; + clocks = <&rcc SAI2_K>; + clock-names = "sai_ck"; + status = "okay"; + + sai2a_port: port { + sai2a_endpoint: endpoint { + remote-endpoint = <&cs42l51_tx_endpoint>; + format = "i2s"; + mclk-fs = <256>; + dai-tdm-slot-num = <2>; + dai-tdm-slot-width = <32>; + }; + }; + }; + + sai2b: audio-controller@4400b024 { + dma-names = "rx"; + st,sync = <&sai2a 2>; + clocks = <&rcc SAI2_K>, <&sai2a>; + clock-names = "sai_ck", "MCLK"; + status = "okay"; + + sai2b_port: port { + sai2b_endpoint: endpoint { + remote-endpoint = <&cs42l51_rx_endpoint>; + format = "i2s"; + mclk-fs = <256>; + dai-tdm-slot-num = <2>; + dai-tdm-slot-width = <32>; + }; + }; + }; +}; + +&sdmmc1 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc1_b4_pins_a>; + pinctrl-1 = <&sdmmc1_b4_od_pins_a>; + pinctrl-2 = <&sdmmc1_b4_sleep_pins_a>; + cd-gpios = <&gpiob 7 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; + disable-wp; + st,neg-edge; + bus-width = <4>; + vmmc-supply = <&v3v3>; + status = "disabled"; +}; + +&sdmmc3 { + pinctrl-names = "default", "opendrain", "sleep"; + pinctrl-0 = <&sdmmc3_b4_pins_a>; + pinctrl-1 = <&sdmmc3_b4_od_pins_a>; + pinctrl-2 = <&sdmmc3_b4_sleep_pins_a>; + broken-cd; + st,neg-edge; + bus-width = <4>; + vmmc-supply = <&v3v3>; + status = "disabled"; +}; + +&timers1 { + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm1_pins_a>; + pinctrl-1 = <&pwm1_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@0 { + status = "okay"; + }; +}; + +&timers3 { + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm3_pins_a>; + pinctrl-1 = <&pwm3_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@2 { + status = "okay"; + }; +}; + +&timers4 { + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm4_pins_a &pwm4_pins_b>; + pinctrl-1 = <&pwm4_sleep_pins_a &pwm4_sleep_pins_b>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@3 { + status = "okay"; + }; +}; + +&timers5 { + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm5_pins_a>; + pinctrl-1 = <&pwm5_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@4 { + status = "okay"; + }; +}; + +&timers6 { + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + timer@5 { + status = "okay"; + }; +}; + +&timers12 { + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; + pwm { + pinctrl-0 = <&pwm12_pins_a>; + pinctrl-1 = <&pwm12_sleep_pins_a>; + pinctrl-names = "default", "sleep"; + status = "okay"; + }; + timer@11 { + status = "okay"; + }; +}; + +&uart4 { + pinctrl-names = "default", "sleep", "idle"; + pinctrl-0 = <&uart4_pins_a>; + pinctrl-1 = <&uart4_sleep_pins_a>; + pinctrl-2 = <&uart4_idle_pins_a>; + /delete-property/dmas; + /delete-property/dma-names; + status = "okay"; +}; + +&uart7 { + pinctrl-names = "default", "sleep", "idle"; + pinctrl-0 = <&uart7_pins_c>; + pinctrl-1 = <&uart7_sleep_pins_c>; + pinctrl-2 = <&uart7_idle_pins_c>; + /delete-property/dmas; + /delete-property/dma-names; + status = "disabled"; +}; + +&usart3 { + pinctrl-names = "default", "sleep", "idle"; + pinctrl-0 = <&usart3_pins_c>; + pinctrl-1 = <&usart3_sleep_pins_c>; + pinctrl-2 = <&usart3_idle_pins_c>; + uart-has-rtscts; + status = "disabled"; +}; + +&usbh_ehci { + phys = <&usbphyc_port0>; + status = "disabled"; +}; + +&usbotg_hs { + phys = <&usbphyc_port1 0>; + phy-names = "usb2-phy"; + usb-role-switch; + status = "disabled"; + + port { + usbotg_hs_ep: endpoint { + remote-endpoint = <&con_usbotg_hs_ep>; + }; + }; +}; + +&usbphyc { + status = "disabled"; +}; + +&usbphyc_port0 { + phy-supply = <&vdd_usb>; + st,tune-hs-dc-level = <2>; + st,enable-fs-rftime-tuning; + st,enable-hs-rftime-reduction; + st,trim-hs-current = <15>; + st,trim-hs-impedance = <1>; + st,tune-squelch-level = <3>; + st,tune-hs-rx-offset = <2>; + st,no-lsfs-sc; +}; + +&usbphyc_port1 { + phy-supply = <&vdd_usb>; + st,tune-hs-dc-level = <2>; + st,enable-fs-rftime-tuning; + st,enable-hs-rftime-reduction; + st,trim-hs-current = <15>; + st,trim-hs-impedance = <1>; + st,tune-squelch-level = <3>; + st,tune-hs-rx-offset = <2>; + st,no-lsfs-sc; +}; + +&vrefbuf { + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vdda-supply = <&vdd>; + status = "okay"; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xxaa-pinctrl.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xxaa-pinctrl.dtsi new file mode 100644 index 0000000..04f7a43 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xxaa-pinctrl.dtsi @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +&pinctrl { + st,package = ; + + gpioa: gpio@50002000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 0 16>; + }; + + gpiob: gpio@50003000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 16 16>; + }; + + gpioc: gpio@50004000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 32 16>; + }; + + gpiod: gpio@50005000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 48 16>; + }; + + gpioe: gpio@50006000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 64 16>; + }; + + gpiof: gpio@50007000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 80 16>; + }; + + gpiog: gpio@50008000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 96 16>; + }; + + gpioh: gpio@50009000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 112 16>; + }; + + gpioi: gpio@5000a000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 128 16>; + }; + + gpioj: gpio@5000b000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 144 16>; + }; + + gpiok: gpio@5000c000 { + status = "okay"; + ngpios = <8>; + gpio-ranges = <&pinctrl 0 160 8>; + }; +}; + +&pinctrl_z { + st,package = ; + + gpioz: gpio@54004000 { + status = "okay"; + ngpios = <8>; + gpio-ranges = <&pinctrl_z 0 400 8>; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xxab-pinctrl.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xxab-pinctrl.dtsi new file mode 100644 index 0000000..328dad1 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xxab-pinctrl.dtsi @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +&pinctrl { + st,package = ; + + gpioa: gpio@50002000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 0 16>; + }; + + gpiob: gpio@50003000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 16 16>; + }; + + gpioc: gpio@50004000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 32 16>; + }; + + gpiod: gpio@50005000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 48 16>; + }; + + gpioe: gpio@50006000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 64 16>; + }; + + gpiof: gpio@50007000 { + status = "okay"; + ngpios = <6>; + gpio-ranges = <&pinctrl 6 86 6>; + }; + + gpiog: gpio@50008000 { + status = "okay"; + ngpios = <10>; + gpio-ranges = <&pinctrl 6 102 10>; + }; + + gpioh: gpio@50009000 { + status = "okay"; + ngpios = <2>; + gpio-ranges = <&pinctrl 0 112 2>; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xxac-pinctrl.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xxac-pinctrl.dtsi new file mode 100644 index 0000000..7eaa245 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xxac-pinctrl.dtsi @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +&pinctrl { + st,package = ; + + gpioa: gpio@50002000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 0 16>; + }; + + gpiob: gpio@50003000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 16 16>; + }; + + gpioc: gpio@50004000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 32 16>; + }; + + gpiod: gpio@50005000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 48 16>; + }; + + gpioe: gpio@50006000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 64 16>; + }; + + gpiof: gpio@50007000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 80 16>; + }; + + gpiog: gpio@50008000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 96 16>; + }; + + gpioh: gpio@50009000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 112 16>; + }; + + gpioi: gpio@5000a000 { + status = "okay"; + ngpios = <12>; + gpio-ranges = <&pinctrl 0 128 12>; + }; +}; + +&pinctrl_z { + st,package = ; + + gpioz: gpio@54004000 { + status = "okay"; + ngpios = <8>; + gpio-ranges = <&pinctrl_z 0 400 8>; + }; +}; diff --git a/optee_os/core/arch/arm/dts/stm32mp15xxad-pinctrl.dtsi b/optee_os/core/arch/arm/dts/stm32mp15xxad-pinctrl.dtsi new file mode 100644 index 0000000..b63e207 --- /dev/null +++ b/optee_os/core/arch/arm/dts/stm32mp15xxad-pinctrl.dtsi @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2019 - All Rights Reserved + * Author: Alexandre Torgue for STMicroelectronics. + */ + +&pinctrl { + st,package = ; + + gpioa: gpio@50002000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 0 16>; + }; + + gpiob: gpio@50003000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 16 16>; + }; + + gpioc: gpio@50004000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 32 16>; + }; + + gpiod: gpio@50005000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 48 16>; + }; + + gpioe: gpio@50006000 { + status = "okay"; + ngpios = <16>; + gpio-ranges = <&pinctrl 0 64 16>; + }; + + gpiof: gpio@50007000 { + status = "okay"; + ngpios = <6>; + gpio-ranges = <&pinctrl 6 86 6>; + }; + + gpiog: gpio@50008000 { + status = "okay"; + ngpios = <10>; + gpio-ranges = <&pinctrl 6 102 10>; + }; + + gpioh: gpio@50009000 { + status = "okay"; + ngpios = <2>; + gpio-ranges = <&pinctrl 0 112 2>; + }; +}; diff --git a/optee_os/core/arch/arm/include/arm.h b/optee_os/core/arch/arm/include/arm.h new file mode 100644 index 0000000..0eea438 --- /dev/null +++ b/optee_os/core/arch/arm/include/arm.h @@ -0,0 +1,203 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2019-2023, Arm Limited. All rights reserved. + */ +#ifndef ARM_H +#define ARM_H + +#include +#include +#include + +/* MIDR definitions */ +#define MIDR_PRIMARY_PART_NUM_SHIFT U(4) +#define MIDR_PRIMARY_PART_NUM_WIDTH U(12) +#define MIDR_PRIMARY_PART_NUM_MASK (BIT(MIDR_PRIMARY_PART_NUM_WIDTH) - 1) + +#define MIDR_IMPLEMENTER_SHIFT U(24) +#define MIDR_IMPLEMENTER_WIDTH U(8) +#define MIDR_IMPLEMENTER_MASK (BIT(MIDR_IMPLEMENTER_WIDTH) - 1) +#define MIDR_IMPLEMENTER_ARM U(0x41) + +#define MIDR_VARIANT_SHIFT U(20) +#define MIDR_VARIANT_WIDTH U(4) +#define MIDR_VARIANT_MASK (BIT(MIDR_VARIANT_WIDTH) - 1) + +#define MIDR_REVISION_SHIFT U(0) +#define MIDR_REVISION_WIDTH U(4) +#define MIDR_REVISION_MASK (BIT(MIDR_REVISION_WIDTH) - 1) + +#define CORTEX_A5_PART_NUM U(0xC05) +#define CORTEX_A7_PART_NUM U(0xC07) +#define CORTEX_A8_PART_NUM U(0xC08) +#define CORTEX_A9_PART_NUM U(0xC09) +#define CORTEX_A15_PART_NUM U(0xC0F) +#define CORTEX_A17_PART_NUM U(0xC0E) +#define CORTEX_A57_PART_NUM U(0xD07) +#define CORTEX_A72_PART_NUM U(0xD08) +#define CORTEX_A73_PART_NUM U(0xD09) +#define CORTEX_A75_PART_NUM U(0xD0A) +#define CORTEX_A65_PART_NUM U(0xD06) +#define CORTEX_A65AE_PART_NUM U(0xD43) +#define CORTEX_A76_PART_NUM U(0xD0B) +#define CORTEX_A76AE_PART_NUM U(0xD0E) +#define CORTEX_A77_PART_NUM U(0xD0D) +#define CORTEX_A78_PART_NUM U(0xD41) +#define CORTEX_A78AE_PART_NUM U(0xD42) +#define CORTEX_A78C_PART_NUM U(0xD4B) +#define CORTEX_A710_PART_NUM U(0xD47) +#define CORTEX_X1_PART_NUM U(0xD44) +#define CORTEX_X2_PART_NUM U(0xD48) +#define NEOVERSE_E1_PART_NUM U(0xD4A) +#define NEOVERSE_N1_PART_NUM U(0xD0C) +#define NEOVERSE_N2_PART_NUM U(0xD49) +#define NEOVERSE_V1_PART_NUM U(0xD40) + +/* MPIDR definitions */ +#define MPIDR_AFFINITY_BITS U(8) +#define MPIDR_AFFLVL_MASK ULL(0xff) +#define MPIDR_AFF0_SHIFT U(0) +#define MPIDR_AFF0_MASK (MPIDR_AFFLVL_MASK << MPIDR_AFF0_SHIFT) +#define MPIDR_AFF1_SHIFT U(8) +#define MPIDR_AFF1_MASK (MPIDR_AFFLVL_MASK << MPIDR_AFF1_SHIFT) +#define MPIDR_AFF2_SHIFT U(16) +#define MPIDR_AFF2_MASK (MPIDR_AFFLVL_MASK << MPIDR_AFF2_SHIFT) +#define MPIDR_AFF3_SHIFT U(32) +#define MPIDR_AFF3_MASK (MPIDR_AFFLVL_MASK << MPIDR_AFF3_SHIFT) + +#define MPIDR_MT_SHIFT U(24) +#define MPIDR_MT_MASK BIT(MPIDR_MT_SHIFT) + +#define MPIDR_CPU_MASK MPIDR_AFF0_MASK +#define MPIDR_CLUSTER_SHIFT MPIDR_AFF1_SHIFT +#define MPIDR_CLUSTER_MASK MPIDR_AFF1_MASK + +#define MPIDR_AARCH32_AFF_MASK (MPIDR_AFF0_MASK | MPIDR_AFF1_MASK | \ + MPIDR_AFF2_MASK) + +/* CLIDR definitions */ +#define CLIDR_LOUIS_SHIFT U(21) +#define CLIDR_LOC_SHIFT U(24) +#define CLIDR_FIELD_WIDTH U(3) + +/* CSSELR definitions */ +#define CSSELR_LEVEL_SHIFT U(1) + +/* CTR definitions */ +#define CTR_CWG_SHIFT U(24) +#define CTR_CWG_MASK U(0xf) +#define CTR_ERG_SHIFT U(20) +#define CTR_ERG_MASK U(0xf) +#define CTR_DMINLINE_SHIFT U(16) +#define CTR_DMINLINE_WIDTH U(4) +#define CTR_DMINLINE_MASK (BIT(4) - 1) +#define CTR_L1IP_SHIFT U(14) +#define CTR_L1IP_MASK U(0x3) +#define CTR_IMINLINE_SHIFT U(0) +#define CTR_IMINLINE_MASK U(0xf) +#define CTR_WORD_SIZE U(4) + +#define ARM32_CPSR_MODE_MASK U(0x1f) +#define ARM32_CPSR_MODE_USR U(0x10) +#define ARM32_CPSR_MODE_FIQ U(0x11) +#define ARM32_CPSR_MODE_IRQ U(0x12) +#define ARM32_CPSR_MODE_SVC U(0x13) +#define ARM32_CPSR_MODE_MON U(0x16) +#define ARM32_CPSR_MODE_ABT U(0x17) +#define ARM32_CPSR_MODE_UND U(0x1b) +#define ARM32_CPSR_MODE_SYS U(0x1f) + +#define ARM32_CPSR_T BIT(5) +#define ARM32_CPSR_F_SHIFT U(6) +#define ARM32_CPSR_F BIT(6) +#define ARM32_CPSR_I BIT(7) +#define ARM32_CPSR_A BIT(8) +#define ARM32_CPSR_E BIT(9) +#define ARM32_CPSR_FIA (ARM32_CPSR_F | ARM32_CPSR_I | ARM32_CPSR_A) +#define ARM32_CPSR_IT_MASK (ARM32_CPSR_IT_MASK1 | ARM32_CPSR_IT_MASK2) +#define ARM32_CPSR_IT_MASK1 U(0x06000000) +#define ARM32_CPSR_IT_MASK2 U(0x0000fc00) + +/* ARM Generic timer definitions */ +#define CNTKCTL_PL0PCTEN BIT(0) /* physical counter el0 access enable */ +#define CNTKCTL_PL0VCTEN BIT(1) /* virtual counter el0 access enable */ + +#ifdef ARM32 +#include +#endif + +#ifdef ARM64 +#include +#endif + +#ifndef __ASSEMBLER__ +static inline __noprof uint64_t barrier_read_counter_timer(void) +{ + isb(); +#ifdef CFG_CORE_SEL2_SPMC + return read_cntvct(); +#else + return read_cntpct(); +#endif +} + +static inline bool feat_bti_is_implemented(void) +{ +#ifdef ARM32 + return false; +#else + return ((read_id_aa64pfr1_el1() & ID_AA64PFR1_EL1_BT_MASK) == + FEAT_BTI_IMPLEMENTED); +#endif +} + +static inline unsigned int feat_mte_implemented(void) +{ +#ifdef ARM32 + return 0; +#else + return (read_id_aa64pfr1_el1() >> ID_AA64PFR1_EL1_MTE_SHIFT) & + ID_AA64PFR1_EL1_MTE_MASK; +#endif +} + +static inline unsigned int feat_pan_implemented(void) +{ +#ifdef ARM32 + return 0; +#else + return (read_id_aa64mmfr1_el1() >> ID_AA64MMFR1_EL1_PAN_SHIFT) & + ID_AA64MMFR1_EL1_PAN_MASK; +#endif +} + +static inline bool feat_crc32_implemented(void) +{ +#ifdef ARM32 + return false; +#else + return ((read_id_aa64isar0_el1() >> ID_AA64ISAR0_EL1_CRC32_SHIFT) & + ID_AA64ISAR0_EL1_CRC32_MASK) == FEAT_CRC32_IMPLEMENTED; +#endif +} + +static inline bool feat_pauth_is_implemented(void) +{ +#ifdef ARM32 + return false; +#else + uint64_t mask = + SHIFT_U64(ID_AA64ISAR1_GPI_MASK, ID_AA64ISAR1_GPI_SHIFT) | + SHIFT_U64(ID_AA64ISAR1_GPA_MASK, ID_AA64ISAR1_GPA_SHIFT) | + SHIFT_U64(ID_AA64ISAR1_API_MASK, ID_AA64ISAR1_API_SHIFT) | + SHIFT_U64(ID_AA64ISAR1_APA_MASK, ID_AA64ISAR1_APA_SHIFT); + + /* If any of the fields is not zero, PAuth is implemented by arch */ + return (read_id_aa64isar1_el1() & mask) != 0U; +#endif +} + +#endif + +#endif /*ARM_H*/ diff --git a/optee_os/core/arch/arm/include/arm32.h b/optee_os/core/arch/arm/include/arm32.h new file mode 100644 index 0000000..253b197 --- /dev/null +++ b/optee_os/core/arch/arm/include/arm32.h @@ -0,0 +1,281 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef ARM32_H +#define ARM32_H + +#include +#include +#include +#include + +#define CPSR_MODE_MASK ARM32_CPSR_MODE_MASK +#define CPSR_MODE_USR ARM32_CPSR_MODE_USR +#define CPSR_MODE_FIQ ARM32_CPSR_MODE_FIQ +#define CPSR_MODE_IRQ ARM32_CPSR_MODE_IRQ +#define CPSR_MODE_SVC ARM32_CPSR_MODE_SVC +#define CPSR_MODE_MON ARM32_CPSR_MODE_MON +#define CPSR_MODE_ABT ARM32_CPSR_MODE_ABT +#define CPSR_MODE_UND ARM32_CPSR_MODE_UND +#define CPSR_MODE_SYS ARM32_CPSR_MODE_SYS + +#define CPSR_T ARM32_CPSR_T +#define CPSR_F_SHIFT ARM32_CPSR_F_SHIFT +#define CPSR_F ARM32_CPSR_F +#define CPSR_I ARM32_CPSR_I +#define CPSR_A ARM32_CPSR_A +#define CPSR_FIA ARM32_CPSR_FIA +#define CPSR_IT_MASK ARM32_CPSR_IT_MASK +#define CPSR_IT_MASK1 ARM32_CPSR_IT_MASK1 +#define CPSR_IT_MASK2 ARM32_CPSR_IT_MASK2 + +#define PMCR_DP BIT32(5) + +#define SCR_NS BIT32(0) +#define SCR_IRQ BIT32(1) +#define SCR_FIQ BIT32(2) +#define SCR_EA BIT32(3) +#define SCR_FW BIT32(4) +#define SCR_AW BIT32(5) +#define SCR_NET BIT32(6) +#define SCR_SCD BIT32(7) +#define SCR_HCE BIT32(8) +#define SCR_SIF BIT32(9) + +#define SCTLR_M BIT32(0) +#define SCTLR_A BIT32(1) +#define SCTLR_C BIT32(2) +#define SCTLR_CP15BEN BIT32(5) +#define SCTLR_SW BIT32(10) +#define SCTLR_Z BIT32(11) +#define SCTLR_I BIT32(12) +#define SCTLR_V BIT32(13) +#define SCTLR_RR BIT32(14) +#define SCTLR_HA BIT32(17) +#define SCTLR_WXN BIT32(19) +#define SCTLR_UWXN BIT32(20) +#define SCTLR_FI BIT32(21) +#define SCTLR_SPAN BIT32(23) +#define SCTLR_VE BIT32(24) +#define SCTLR_EE BIT32(25) +#define SCTLR_NMFI BIT32(27) +#define SCTLR_TRE BIT32(28) +#define SCTLR_AFE BIT32(29) +#define SCTLR_TE BIT32(30) + +/* Only valid for Cortex-A15 */ +#define ACTLR_CA15_ENABLE_INVALIDATE_BTB BIT(0) +/* Only valid for Cortex-A8 */ +#define ACTLR_CA8_ENABLE_INVALIDATE_BTB BIT(6) +/* Only valid for Cortex-A9 */ +#define ACTLR_CA9_WFLZ BIT(3) + +#define ACTLR_SMP BIT32(6) + +#define NSACR_CP10 BIT32(10) +#define NSACR_CP11 BIT32(11) +#define NSACR_NSD32DIS BIT32(14) +#define NSACR_NSASEDIS BIT32(15) +#define NSACR_NS_L2ERR BIT32(17) +#define NSACR_NS_SMP BIT32(18) + +#define CPACR_ASEDIS BIT32(31) +#define CPACR_D32DIS BIT32(30) +#define CPACR_CP(co_proc, access) SHIFT_U32((access), ((co_proc) * 2)) +#define CPACR_CP_ACCESS_DENIED U(0x0) +#define CPACR_CP_ACCESS_PL1_ONLY U(0x1) +#define CPACR_CP_ACCESS_FULL U(0x3) + + +#define DACR_DOMAIN(num, perm) SHIFT_U32((perm), ((num) * 2)) +#define DACR_DOMAIN_PERM_NO_ACCESS U(0x0) +#define DACR_DOMAIN_PERM_CLIENT U(0x1) +#define DACR_DOMAIN_PERM_MANAGER U(0x3) + +#define PAR_F BIT32(0) +#define PAR_SS BIT32(1) +#define PAR_LPAE BIT32(11) +#define PAR_PA_SHIFT U(12) +#define PAR32_PA_MASK (BIT32(20) - 1) +#define PAR64_PA_MASK (BIT64(28) - 1) + +/* + * TTBCR has different register layout if LPAE is enabled or not. + * TTBCR.EAE == 0 => LPAE is not enabled + * TTBCR.EAE == 1 => LPAE is enabled + */ +#define TTBCR_EAE BIT32(31) + +/* When TTBCR.EAE == 0 */ +#define TTBCR_PD0 BIT32(4) +#define TTBCR_PD1 BIT32(5) + +/* When TTBCR.EAE == 1 */ +#define TTBCR_T0SZ_SHIFT U(0) +#define TTBCR_EPD0 BIT32(7) +#define TTBCR_IRGN0_SHIFT U(8) +#define TTBCR_ORGN0_SHIFT U(10) +#define TTBCR_SH0_SHIFT U(12) +#define TTBCR_T1SZ_SHIFT U(16) +#define TTBCR_A1 BIT32(22) +#define TTBCR_EPD1 BIT32(23) +#define TTBCR_IRGN1_SHIFT U(24) +#define TTBCR_ORGN1_SHIFT U(26) +#define TTBCR_SH1_SHIFT U(28) + +/* Normal memory, Inner/Outer Non-cacheable */ +#define TTBCR_XRGNX_NC U(0x0) +/* Normal memory, Inner/Outer Write-Back Write-Allocate Cacheable */ +#define TTBCR_XRGNX_WB U(0x1) +/* Normal memory, Inner/Outer Write-Through Cacheable */ +#define TTBCR_XRGNX_WT U(0x2) +/* Normal memory, Inner/Outer Write-Back no Write-Allocate Cacheable */ +#define TTBCR_XRGNX_WBWA U(0x3) + +/* Non-shareable */ +#define TTBCR_SHX_NSH U(0x0) +/* Outer Shareable */ +#define TTBCR_SHX_OSH U(0x2) +/* Inner Shareable */ +#define TTBCR_SHX_ISH U(0x3) + +#define TTBR_ASID_MASK U(0xff) +#define TTBR_ASID_SHIFT U(48) + +#define TLBI_MVA_SHIFT U(12) +#define TLBI_ASID_MASK U(0xff) + +#define FSR_LPAE BIT32(9) +#define FSR_WNR BIT32(11) + +/* Valid if FSR.LPAE is 1 */ +#define FSR_STATUS_MASK (BIT32(6) - 1) + +/* Valid if FSR.LPAE is 0 */ +#define FSR_FS_MASK (BIT32(10) | (BIT32(4) - 1)) + +/* ID_PFR1 bit fields */ +#define IDPFR1_VIRT_SHIFT U(12) +#define IDPFR1_VIRT_MASK SHIFT_U32(0xF, IDPFR1_VIRT_SHIFT) +#define IDPFR1_GENTIMER_SHIFT U(16) +#define IDPFR1_GENTIMER_MASK SHIFT_U32(0xF, IDPFR1_GENTIMER_SHIFT) + +#ifndef __ASSEMBLER__ +#include +#ifdef CFG_ARM_GICV3 +#include +#endif + +static inline __noprof void isb(void) +{ + asm volatile ("isb" : : : "memory"); +} + +static inline __noprof void dsb(void) +{ + asm volatile ("dsb" : : : "memory"); +} + +static inline __noprof void dsb_ish(void) +{ + asm volatile ("dsb ish" : : : "memory"); +} + +static inline __noprof void dsb_ishst(void) +{ + asm volatile ("dsb ishst" : : : "memory"); +} + +static inline __noprof void dmb(void) +{ + asm volatile ("dmb" : : : "memory"); +} + +static inline __noprof void sev(void) +{ + asm volatile ("sev" : : : "memory"); +} + +static inline __noprof void wfe(void) +{ + asm volatile ("wfe" : : : "memory"); +} + +static inline __noprof uint32_t read_cpsr(void) +{ + uint32_t cpsr; + + asm volatile ("mrs %[cpsr], cpsr" + : [cpsr] "=r" (cpsr) + ); + return cpsr; +} + +static inline __noprof void write_cpsr(uint32_t cpsr) +{ + asm volatile ("msr cpsr_fsxc, %[cpsr]" + : : [cpsr] "r" (cpsr) + ); +} + +static inline __noprof uint32_t read_spsr(void) +{ + uint32_t spsr; + + asm volatile ("mrs %[spsr], spsr" + : [spsr] "=r" (spsr) + ); + return spsr; +} + +static inline __noprof void wfi(void) +{ + asm volatile("wfi" : : : "memory"); +} + +static __always_inline __noprof uint32_t read_pc(void) +{ + uint32_t val; + + asm volatile ("adr %0, ." : "=r" (val)); + return val; +} + +static __always_inline __noprof uint32_t read_sp(void) +{ + uint32_t val; + + asm volatile ("mov %0, sp" : "=r" (val)); + return val; +} + +static __always_inline __noprof uint32_t read_lr(void) +{ + uint32_t val; + + asm volatile ("mov %0, lr" : "=r" (val)); + return val; +} + +static __always_inline __noprof uint32_t read_fp(void) +{ + uint32_t val; + + asm volatile ("mov %0, fp" : "=r" (val)); + return val; +} + +static __always_inline __noprof uint32_t read_r7(void) +{ + uint32_t val; + + asm volatile ("mov %0, r7" : "=r" (val)); + return val; +} + +#endif /*__ASSEMBLER__*/ + +#endif /*ARM32_H*/ diff --git a/optee_os/core/arch/arm/include/arm32_macros.S b/optee_os/core/arch/arm/include/arm32_macros.S new file mode 100644 index 0000000..95077a0 --- /dev/null +++ b/optee_os/core/arch/arm/include/arm32_macros.S @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#ifdef CFG_ARM_GICV3 +#include +#endif + + .macro mov_imm reg, val + .if ((\val) & 0xffff0000) == 0 + movw \reg, #(\val) + .else + movw \reg, #((\val) & 0xffff) + movt \reg, #((\val) >> 16) + .endif + .endm + + .macro panic_at_smc_return +#if defined(CFG_TEE_CORE_DEBUG) + bl __panic_at_smc_return +#else + b . +#endif + .endm diff --git a/optee_os/core/arch/arm/include/arm32_macros_cortex_a9.S b/optee_os/core/arch/arm/include/arm32_macros_cortex_a9.S new file mode 100644 index 0000000..4960e9f --- /dev/null +++ b/optee_os/core/arch/arm/include/arm32_macros_cortex_a9.S @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Freescale Semiconductor, Inc. + * All rights reserved. + * + * Peng Fan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + .macro write_pcr reg + mcr p15, 0, \reg, c15, c0, 0 + .endm + + .macro read_pcr reg + mrc p15, 0, \reg, c15, c0, 0 + .endm + + .macro write_diag reg + mcr p15, 0, \reg, c15, c0, 1 + .endm + + .macro read_diag reg + mrc p15, 0, \reg, c15, c0, 1 + .endm diff --git a/optee_os/core/arch/arm/include/arm64.h b/optee_os/core/arch/arm/include/arm64.h new file mode 100644 index 0000000..8c210b3 --- /dev/null +++ b/optee_os/core/arch/arm/include/arm64.h @@ -0,0 +1,508 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2023, Arm Limited + */ +#ifndef ARM64_H +#define ARM64_H + +#include +#include +#include +#include + +#define SCTLR_M BIT64(0) +#define SCTLR_A BIT64(1) +#define SCTLR_C BIT64(2) +#define SCTLR_SA BIT64(3) +#define SCTLR_I BIT64(12) +#define SCTLR_ENDB BIT64(13) +#define SCTLR_WXN BIT64(19) +#define SCTLR_SPAN BIT64(23) +#define SCTLR_ENDA BIT64(27) +#define SCTLR_ENIB BIT64(30) +#define SCTLR_ENIA BIT64(31) +#define SCTLR_BT0 BIT64(35) +#define SCTLR_BT1 BIT64(36) +#define SCTLR_ITFSB BIT64(37) + +#define SCTLR_TCF_MASK SHIFT_U64(0x3, 40) +#define SCTLR_TCF_NONE SHIFT_U64(0x0, 40) +#define SCTLR_TCF_SYNC SHIFT_U64(0x1, 40) +#define SCTLR_TCF_ASYNC SHIFT_U64(0x2, 40) +#define SCTLR_TCF_ASYMM SHIFT_U64(0x3, 40) + +#define SCTLR_TCF0_MASK SHIFT_U64(0x3, 38) +#define SCTLR_TCF0_NONE SHIFT_U64(0x0, 38) +#define SCTLR_TCF0_SYNC SHIFT_U64(0x1, 38) +#define SCTLR_TCF0_ASYNC SHIFT_U64(0x2, 38) +#define SCTLR_TCF0_ASYMM SHIFT_U64(0x3, 38) + +#define SCTLR_ATA0 BIT64(42) +#define SCTLR_ATA BIT64(43) + +#define TTBR_ASID_MASK U(0xff) +#define TTBR_ASID_SHIFT U(48) + +#define CLIDR_LOUIS_SHIFT U(21) +#define CLIDR_LOC_SHIFT U(24) +#define CLIDR_FIELD_WIDTH U(3) + +#define CSSELR_LEVEL_SHIFT U(1) + +#define DAIFBIT_FIQ BIT32(0) +#define DAIFBIT_IRQ BIT32(1) +#define DAIFBIT_ABT BIT32(2) +#define DAIFBIT_DBG BIT32(3) +#define DAIFBIT_ALL (DAIFBIT_FIQ | DAIFBIT_IRQ | \ + DAIFBIT_ABT | DAIFBIT_DBG) + +#define DAIF_F_SHIFT U(6) +#define DAIF_F BIT32(6) +#define DAIF_I BIT32(7) +#define DAIF_A BIT32(8) +#define DAIF_D BIT32(9) +#define DAIF_AIF (DAIF_A | DAIF_I | DAIF_F) + +#define SPSR_MODE_RW_SHIFT U(4) +#define SPSR_MODE_RW_MASK U(0x1) +#define SPSR_MODE_RW_64 U(0x0) +#define SPSR_MODE_RW_32 U(0x1) + +#define SPSR_64_MODE_SP_SHIFT U(0) +#define SPSR_64_MODE_SP_MASK U(0x1) +#define SPSR_64_MODE_SP_EL0 U(0x0) +#define SPSR_64_MODE_SP_ELX U(0x1) + +#define SPSR_64_MODE_EL_SHIFT U(2) +#define SPSR_64_MODE_EL_MASK U(0x3) +#define SPSR_64_MODE_EL1 U(0x1) +#define SPSR_64_MODE_EL0 U(0x0) + +#define SPSR_64_DAIF_SHIFT U(6) +#define SPSR_64_DAIF_MASK U(0xf) + +#define SPSR_64_PAN BIT64(22) + +#define SPSR_32_AIF_SHIFT U(6) +#define SPSR_32_AIF_MASK U(0x7) + +#define SPSR_32_E_SHIFT U(9) +#define SPSR_32_E_MASK U(0x1) +#define SPSR_32_E_LITTLE U(0x0) +#define SPSR_32_E_BIG U(0x1) + +#define SPSR_32_T_SHIFT U(5) +#define SPSR_32_T_MASK U(0x1) +#define SPSR_32_T_ARM U(0x0) +#define SPSR_32_T_THUMB U(0x1) + +#define SPSR_32_MODE_SHIFT U(0) +#define SPSR_32_MODE_MASK U(0xf) +#define SPSR_32_MODE_USR U(0x0) + + +#define SPSR_64(el, sp, daif) \ + (SPSR_MODE_RW_64 << SPSR_MODE_RW_SHIFT | \ + ((el) & SPSR_64_MODE_EL_MASK) << SPSR_64_MODE_EL_SHIFT | \ + ((sp) & SPSR_64_MODE_SP_MASK) << SPSR_64_MODE_SP_SHIFT | \ + ((daif) & SPSR_64_DAIF_MASK) << SPSR_64_DAIF_SHIFT) + +#define SPSR_32(mode, isa, aif) \ + (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT | \ + SPSR_32_E_LITTLE << SPSR_32_E_SHIFT | \ + ((mode) & SPSR_32_MODE_MASK) << SPSR_32_MODE_SHIFT | \ + ((isa) & SPSR_32_T_MASK) << SPSR_32_T_SHIFT | \ + ((aif) & SPSR_32_AIF_MASK) << SPSR_32_AIF_SHIFT) + + +#define TCR_T0SZ_SHIFT U(0) +#define TCR_EPD0 BIT64(7) +#define TCR_IRGN0_SHIFT U(8) +#define TCR_ORGN0_SHIFT U(10) +#define TCR_SH0_SHIFT U(12) +#define TCR_T1SZ_SHIFT U(16) +#define TCR_A1 BIT64(22) +#define TCR_EPD1 BIT64(23) +#define TCR_IRGN1_SHIFT U(24) +#define TCR_ORGN1_SHIFT U(26) +#define TCR_SH1_SHIFT U(28) +#define TCR_EL1_IPS_SHIFT U(32) +#define TCR_EL1_IPS_MASK UINT64_C(0x7) +#define TCR_TG1_4KB SHIFT_U64(2, 30) +#define TCR_RES1 BIT64(31) +#define TCR_TBI0 BIT64(37) +#define TCR_TBI1 BIT64(38) +#define TCR_TCMA0 BIT64(57) +#define TCR_TCMA1 BIT64(58) + + +/* Normal memory, Inner/Outer Non-cacheable */ +#define TCR_XRGNX_NC U(0x0) +/* Normal memory, Inner/Outer Write-Back Write-Allocate Cacheable */ +#define TCR_XRGNX_WB U(0x1) +/* Normal memory, Inner/Outer Write-Through Cacheable */ +#define TCR_XRGNX_WT U(0x2) +/* Normal memory, Inner/Outer Write-Back no Write-Allocate Cacheable */ +#define TCR_XRGNX_WBWA U(0x3) + +/* Non-shareable */ +#define TCR_SHX_NSH U(0x0) +/* Outer Shareable */ +#define TCR_SHX_OSH U(0x2) +/* Inner Shareable */ +#define TCR_SHX_ISH U(0x3) + +#define ESR_EC_SHIFT U(26) +#define ESR_EC_MASK U(0x3f) + +#define ESR_EC_UNKNOWN U(0x00) +#define ESR_EC_WFI U(0x01) +#define ESR_EC_AARCH32_CP15_32 U(0x03) +#define ESR_EC_AARCH32_CP15_64 U(0x04) +#define ESR_EC_AARCH32_CP14_MR U(0x05) +#define ESR_EC_AARCH32_CP14_LS U(0x06) +#define ESR_EC_FP_ASIMD U(0x07) +#define ESR_EC_AARCH32_CP10_ID U(0x08) +#define ESR_EC_PAUTH U(0x09) +#define ESR_EC_AARCH32_CP14_64 U(0x0c) +#define ESR_EC_BTI U(0x0d) +#define ESR_EC_ILLEGAL U(0x0e) +#define ESR_EC_AARCH32_SVC U(0x11) +#define ESR_EC_AARCH64_SVC U(0x15) +#define ESR_EC_AARCH64_SYS U(0x18) +#define ESR_EC_ERET U(0x1a) +#define ESR_EC_FPAC U(0x1c) +#define ESR_EC_IABT_EL0 U(0x20) +#define ESR_EC_IABT_EL1 U(0x21) +#define ESR_EC_PC_ALIGN U(0x22) +#define ESR_EC_DABT_EL0 U(0x24) +#define ESR_EC_DABT_EL1 U(0x25) +#define ESR_EC_SP_ALIGN U(0x26) +#define ESR_EC_AARCH32_FP U(0x28) +#define ESR_EC_AARCH64_FP U(0x2c) +#define ESR_EC_SERROR U(0x2f) +#define ESR_EC_BREAKPT_EL0 U(0x30) +#define ESR_EC_BREAKPT_EL1 U(0x31) +#define ESR_EC_SOFTSTP_EL0 U(0x32) +#define ESR_EC_SOFTSTP_EL1 U(0x33) +#define ESR_EC_WATCHPT_EL0 U(0x34) +#define ESR_EC_WATCHPT_EL1 U(0x35) +#define ESR_EC_AARCH32_BKPT U(0x38) +#define ESR_EC_AARCH64_BRK U(0x3c) + +/* Combined defines for DFSC and IFSC */ +#define ESR_FSC_MASK U(0x3f) +#define ESR_FSC_SIZE_L0 U(0x00) +#define ESR_FSC_SIZE_L1 U(0x01) +#define ESR_FSC_SIZE_L2 U(0x02) +#define ESR_FSC_SIZE_L3 U(0x03) +#define ESR_FSC_TRANS_L0 U(0x04) +#define ESR_FSC_TRANS_L1 U(0x05) +#define ESR_FSC_TRANS_L2 U(0x06) +#define ESR_FSC_TRANS_L3 U(0x07) +#define ESR_FSC_ACCF_L1 U(0x09) +#define ESR_FSC_ACCF_L2 U(0x0a) +#define ESR_FSC_ACCF_L3 U(0x0b) +#define ESR_FSC_PERMF_L1 U(0x0d) +#define ESR_FSC_PERMF_L2 U(0x0e) +#define ESR_FSC_PERMF_L3 U(0x0f) +#define ESR_FSC_TAG_CHECK U(0x11) +#define ESR_FSC_ALIGN U(0x21) + +/* WnR for DABT and RES0 for IABT */ +#define ESR_ABT_WNR BIT32(6) + +#define CPACR_EL1_FPEN_SHIFT U(20) +#define CPACR_EL1_FPEN_MASK U(0x3) +#define CPACR_EL1_FPEN_NONE U(0x0) +#define CPACR_EL1_FPEN_EL1 U(0x1) +#define CPACR_EL1_FPEN_EL0EL1 U(0x3) +#define CPACR_EL1_FPEN(x) ((x) >> CPACR_EL1_FPEN_SHIFT \ + & CPACR_EL1_FPEN_MASK) + + +#define PAR_F BIT32(0) +#define PAR_PA_SHIFT U(12) +#define PAR_PA_MASK (BIT64(36) - 1) + +#define TLBI_VA_SHIFT U(12) +#define TLBI_ASID_SHIFT U(48) +#define TLBI_ASID_MASK U(0xff) + +#define ID_AA64PFR1_EL1_BT_MASK ULL(0xf) +#define FEAT_BTI_IMPLEMENTED ULL(0x1) + +#define ID_AA64PFR1_EL1_MTE_MASK UL(0xf) +#define ID_AA64PFR1_EL1_MTE_SHIFT U(8) +#define FEAT_MTE_NOT_IMPLEMENTED U(0x0) +#define FEAT_MTE_IMPLEMENTED U(0x1) +#define FEAT_MTE2_IMPLEMENTED U(0x2) +#define FEAT_MTE3_IMPLEMENTED U(0x3) + +#define ID_AA64MMFR1_EL1_PAN_MASK UL(0xf) +#define ID_AA64MMFR1_EL1_PAN_SHIFT U(20) +#define FEAT_PAN_NOT_IMPLEMENTED U(0x0) +#define FEAT_PAN_IMPLEMENTED U(0x1) +#define FEAT_PAN2_IMPLEMENTED U(0x2) +#define FEAT_PAN3_IMPLEMENTED U(0x3) + +#define ID_AA64ISAR0_EL1_CRC32_MASK UL(0xf) +#define ID_AA64ISAR0_EL1_CRC32_SHIFT U(16) +#define FEAT_CRC32_NOT_IMPLEMENTED U(0x0) +#define FEAT_CRC32_IMPLEMENTED U(0x1) + +#define ID_AA64ISAR1_GPI_SHIFT U(28) +#define ID_AA64ISAR1_GPI_MASK U(0xf) +#define ID_AA64ISAR1_GPI_NI U(0x0) +#define ID_AA64ISAR1_GPI_IMP_DEF U(0x1) + +#define ID_AA64ISAR1_GPA_SHIFT U(24) +#define ID_AA64ISAR1_GPA_MASK U(0xf) +#define ID_AA64ISAR1_GPA_NI U(0x0) +#define ID_AA64ISAR1_GPA_ARCHITECTED U(0x1) + +#define ID_AA64ISAR1_API_SHIFT U(8) +#define ID_AA64ISAR1_API_MASK U(0xf) +#define ID_AA64ISAR1_API_NI U(0x0) +#define ID_AA64ISAR1_API_IMP_DEF U(0x1) +#define ID_AA64ISAR1_API_IMP_DEF_EPAC U(0x2) +#define ID_AA64ISAR1_API_IMP_DEF_EPAC2 U(0x3) +#define ID_AA64ISAR1_API_IMP_DEF_EPAC2_FPAC U(0x4) +#define ID_AA64ISAR1_API_IMP_DEF_EPAC2_FPAC_CMB U(0x5) + +#define ID_AA64ISAR1_APA_SHIFT U(4) +#define ID_AA64ISAR1_APA_MASK U(0xf) +#define ID_AA64ISAR1_APA_NI U(0x0) +#define ID_AA64ISAR1_APA_ARCHITECTED U(0x1) +#define ID_AA64ISAR1_APA_ARCH_EPAC U(0x2) +#define ID_AA64ISAR1_APA_ARCH_EPAC2 U(0x3) +#define ID_AA64ISAR1_APA_ARCH_EPAC2_FPAC U(0x4) +#define ID_AA64ISAR1_APA_ARCH_EPAC2_FPAC_CMB U(0x5) + +#define ID_MMFR3_EL1_PAN_SHIFT U(16) + +#define GCR_EL1_RRND BIT64(16) + +#ifndef __ASSEMBLER__ +static inline __noprof void isb(void) +{ + asm volatile ("isb" : : : "memory"); +} + +static inline __noprof void dsb(void) +{ + asm volatile ("dsb sy" : : : "memory"); +} + +static inline __noprof void dsb_ish(void) +{ + asm volatile ("dsb ish" : : : "memory"); +} + +static inline __noprof void dsb_ishst(void) +{ + asm volatile ("dsb ishst" : : : "memory"); +} + +static inline __noprof void dsb_osh(void) +{ + asm volatile ("dsb osh" : : : "memory"); +} + +static inline __noprof void sev(void) +{ + asm volatile ("sev" : : : "memory"); +} + +static inline __noprof void wfe(void) +{ + asm volatile ("wfe" : : : "memory"); +} + +static inline __noprof void wfi(void) +{ + asm volatile ("wfi" : : : "memory"); +} + +static inline __noprof void write_at_s1e1r(uint64_t va) +{ + asm volatile ("at S1E1R, %0" : : "r" (va)); +} + +static __always_inline __noprof uint64_t read_pc(void) +{ + uint64_t val; + + asm volatile ("adr %0, ." : "=r" (val)); + return val; +} + +static __always_inline __noprof uint64_t read_fp(void) +{ + uint64_t val; + + asm volatile ("mov %0, x29" : "=r" (val)); + return val; +} + +static inline __noprof uint64_t read_pmu_ccnt(void) +{ + uint64_t val; + + asm volatile("mrs %0, PMCCNTR_EL0" : "=r"(val)); + return val; +} + +static inline __noprof void tlbi_vaae1is(uint64_t va) +{ + asm volatile ("tlbi vaae1is, %0" : : "r" (va)); +} + +static inline __noprof void tlbi_vale1is(uint64_t va) +{ + asm volatile ("tlbi vale1is, %0" : : "r" (va)); +} + +static inline void write_64bit_pair(uint64_t dst, uint64_t hi, uint64_t lo) +{ + /* 128bits should be written to hardware at one time */ + asm volatile ("stp %1, %0, [%2]" : : + "r" (hi), "r" (lo), "r" (dst) : "memory"); +} + +/* + * Templates for register read/write functions based on mrs/msr + */ + +#define DEFINE_REG_READ_FUNC_(reg, type, asmreg) \ +static inline __noprof type read_##reg(void) \ +{ \ + uint64_t val64 = 0; \ + \ + asm volatile("mrs %0, " #asmreg : "=r" (val64)); \ + return val64; \ +} + +#define DEFINE_REG_WRITE_FUNC_(reg, type, asmreg) \ +static inline __noprof void write_##reg(type val) \ +{ \ + uint64_t val64 = val; \ + \ + asm volatile("msr " #asmreg ", %0" : : "r" (val64)); \ +} + +#define DEFINE_U32_REG_READ_FUNC(reg) \ + DEFINE_REG_READ_FUNC_(reg, uint32_t, reg) + +#define DEFINE_U32_REG_WRITE_FUNC(reg) \ + DEFINE_REG_WRITE_FUNC_(reg, uint32_t, reg) + +#define DEFINE_U32_REG_READWRITE_FUNCS(reg) \ + DEFINE_U32_REG_READ_FUNC(reg) \ + DEFINE_U32_REG_WRITE_FUNC(reg) + +#define DEFINE_U64_REG_READ_FUNC(reg) \ + DEFINE_REG_READ_FUNC_(reg, uint64_t, reg) + +#define DEFINE_U64_REG_WRITE_FUNC(reg) \ + DEFINE_REG_WRITE_FUNC_(reg, uint64_t, reg) + +#define DEFINE_U64_REG_READWRITE_FUNCS(reg) \ + DEFINE_U64_REG_READ_FUNC(reg) \ + DEFINE_U64_REG_WRITE_FUNC(reg) + +/* + * Define register access functions + */ + +DEFINE_U32_REG_READWRITE_FUNCS(cpacr_el1) +DEFINE_U32_REG_READWRITE_FUNCS(daif) +DEFINE_U32_REG_READWRITE_FUNCS(fpcr) +DEFINE_U32_REG_READWRITE_FUNCS(fpsr) + +DEFINE_U32_REG_READ_FUNC(ctr_el0) +#define read_ctr() read_ctr_el0() +DEFINE_U32_REG_READ_FUNC(contextidr_el1) +DEFINE_U64_REG_READ_FUNC(sctlr_el1) + +/* ARM Generic timer functions */ +DEFINE_REG_READ_FUNC_(cntfrq, uint32_t, cntfrq_el0) +DEFINE_REG_READ_FUNC_(cntvct, uint64_t, cntvct_el0) +DEFINE_REG_READ_FUNC_(cntpct, uint64_t, cntpct_el0) +DEFINE_REG_READ_FUNC_(cntkctl, uint32_t, cntkctl_el1) +DEFINE_REG_WRITE_FUNC_(cntkctl, uint32_t, cntkctl_el1) +DEFINE_REG_READ_FUNC_(cntps_ctl, uint32_t, cntps_ctl_el1) +DEFINE_REG_WRITE_FUNC_(cntps_ctl, uint32_t, cntps_ctl_el1) +DEFINE_REG_READ_FUNC_(cntps_tval, uint32_t, cntps_tval_el1) +DEFINE_REG_WRITE_FUNC_(cntps_tval, uint32_t, cntps_tval_el1) + +DEFINE_REG_READ_FUNC_(pmccntr, uint64_t, pmccntr_el0) + +DEFINE_U64_REG_READWRITE_FUNCS(ttbr0_el1) +DEFINE_U64_REG_READWRITE_FUNCS(ttbr1_el1) +DEFINE_U64_REG_READWRITE_FUNCS(tcr_el1) + +DEFINE_U64_REG_READ_FUNC(esr_el1) +DEFINE_U64_REG_READ_FUNC(far_el1) +DEFINE_U64_REG_READ_FUNC(mpidr_el1) +/* Alias for reading this register to avoid ifdefs in code */ +#define read_mpidr() read_mpidr_el1() +DEFINE_U64_REG_READ_FUNC(midr_el1) +/* Alias for reading this register to avoid ifdefs in code */ +#define read_midr() read_midr_el1() +DEFINE_U64_REG_READ_FUNC(par_el1) + +DEFINE_U64_REG_WRITE_FUNC(mair_el1) + +DEFINE_U64_REG_READ_FUNC(id_aa64mmfr1_el1) +DEFINE_U64_REG_READ_FUNC(id_aa64pfr1_el1) +DEFINE_U64_REG_READ_FUNC(id_aa64isar0_el1) +DEFINE_U64_REG_READ_FUNC(id_aa64isar1_el1) +DEFINE_REG_READ_FUNC_(apiakeylo, uint64_t, S3_0_c2_c1_0) +DEFINE_REG_READ_FUNC_(apiakeyhi, uint64_t, S3_0_c2_c1_1) + +DEFINE_REG_WRITE_FUNC_(apibkeylo, uint64_t, S3_0_c2_c1_2) +DEFINE_REG_WRITE_FUNC_(apibkeyhi, uint64_t, S3_0_c2_c1_3) + +DEFINE_REG_READ_FUNC_(apdakeylo, uint64_t, S3_0_c2_c2_0) +DEFINE_REG_READ_FUNC_(apdakeyhi, uint64_t, S3_0_c2_c2_1) + +DEFINE_REG_WRITE_FUNC_(apdbkeylo, uint64_t, S3_0_c2_c2_2) +DEFINE_REG_WRITE_FUNC_(apdbkeyhi, uint64_t, S3_0_c2_c2_3) + +DEFINE_REG_WRITE_FUNC_(apgakeylo, uint64_t, S3_0_c2_c3_0) +DEFINE_REG_WRITE_FUNC_(apgakeyhi, uint64_t, S3_0_c2_c3_1) + +/* Register read/write functions for GICC registers by using system interface */ +DEFINE_REG_READ_FUNC_(icc_ctlr, uint32_t, S3_0_C12_C12_4) +DEFINE_REG_WRITE_FUNC_(icc_ctlr, uint32_t, S3_0_C12_C12_4) +DEFINE_REG_WRITE_FUNC_(icc_pmr, uint32_t, S3_0_C4_C6_0) +DEFINE_REG_READ_FUNC_(icc_iar0, uint32_t, S3_0_c12_c8_0) +DEFINE_REG_READ_FUNC_(icc_iar1, uint32_t, S3_0_c12_c12_0) +DEFINE_REG_WRITE_FUNC_(icc_eoir0, uint32_t, S3_0_c12_c8_1) +DEFINE_REG_WRITE_FUNC_(icc_eoir1, uint32_t, S3_0_c12_c12_1) +DEFINE_REG_WRITE_FUNC_(icc_igrpen0, uint32_t, S3_0_C12_C12_6) +DEFINE_REG_WRITE_FUNC_(icc_igrpen1, uint32_t, S3_0_C12_C12_7) +DEFINE_REG_WRITE_FUNC_(icc_sgi1r, uint64_t, S3_0_C12_C11_5) +DEFINE_REG_WRITE_FUNC_(icc_asgi1r, uint64_t, S3_0_C12_C11_6) + +DEFINE_REG_WRITE_FUNC_(pan, uint64_t, S3_0_c4_c2_3) +DEFINE_REG_READ_FUNC_(pan, uint64_t, S3_0_c4_c2_3) + +static inline void write_pan_enable(void) +{ + /* msr pan, #1 */ + asm volatile("msr S0_0_c4_c1_4, xzr" ::: "memory" ); +} + +static inline void write_pan_disable(void) +{ + /* msr pan, #0 */ + asm volatile("msr S0_0_c4_c0_4, xzr" ::: "memory" ); +} + +#endif /*__ASSEMBLER__*/ + +#endif /*ARM64_H*/ + diff --git a/optee_os/core/arch/arm/include/arm64_macros.S b/optee_os/core/arch/arm/include/arm64_macros.S new file mode 100644 index 0000000..34295d6 --- /dev/null +++ b/optee_os/core/arch/arm/include/arm64_macros.S @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + + .altmacro + + /* + * This helper macro concatenates instr_prefix, instr_suffix, to + * create a ldp/stp instruction. It also selects register name x/w + * based on reg_bytes. + */ + .macro __do_dregs instr_prefix, instr_suffix, reg_bytes, base_reg, \ + base_offs, reg0, reg1 + .if \reg_bytes == 8 + \instr_prefix\instr_suffix \ + x\reg0, x\reg1, [\base_reg, #\base_offs] + .else + \instr_prefix\instr_suffix \ + w\reg0, w\reg1, [\base_reg, #\base_offs] + .endif + .endm + + /* + * This helper macro concatenates instr_prefix, instr_suffix, to + * create a ldr/str instruction. It also selects register name x/w + * based on reg_bytes. + */ + .macro __do_reg instr_prefix, instr_suffix, reg_bytes, base_reg, \ + base_offs, reg + .if \reg_bytes == 8 + \instr_prefix\instr_suffix \ + x\reg, [\base_reg, #\base_offs] + .else + \instr_prefix\instr_suffix \ + w\reg, [\base_reg, #\base_offs] + .endif + .endm + + /* + * This helper macro uses recursion to create a loop which will + * start with generating instructions for register pairs and if + * it's an odd number of registers end with a single load/store. + */ + .macro _do_regs instr_prefix, reg_bytes, base_reg, base_offs, \ + from_regnum, to_regnum + .if (\to_regnum - \from_regnum + 1) >= 2 + __do_dregs \instr_prefix, p, \reg_bytes, \base_reg, \ + \base_offs, \from_regnum, %(\from_regnum + 1) + .else + __do_reg \instr_prefix, r, \reg_bytes, \base_reg, \ + \base_offs, \from_regnum + .endif + .if (\to_regnum - \from_regnum + 1) > 2 + _do_regs \instr_prefix, \reg_bytes, \base_reg, \ + %(\base_offs + 2 * \reg_bytes), \ + %(\from_regnum + 2), \to_regnum + .endif + .endm + + /* + * Stores registers x[from_regnum]..x[to_regnum] at + * [base_reg, #base_offs] + */ + .macro store_xregs base_reg, base_offs, from_regnum, to_regnum + _do_regs st 8 \base_reg, \base_offs, \from_regnum, \to_regnum + .endm + + /* + * Stores registers w[from_regnum]..w[to_regnum] at + * [base_reg, #base_offs] + */ + .macro store_wregs base_reg, base_offs, from_regnum, to_regnum + _do_regs st 4 \base_reg, \base_offs, \from_regnum, \to_regnum + .endm + + /* + * Loads registers x[from_regnum]..x[to_regnum] at + * [base_reg, #base_offs] + */ + .macro load_xregs base_reg, base_offs, from_regnum, to_regnum + _do_regs ld 8 \base_reg, \base_offs, \from_regnum, \to_regnum + .endm + + /* + * Loads registers w[from_regnum]..w[to_regnum] at + * [base_reg, #base_offs] + */ + .macro load_wregs base_reg, base_offs, from_regnum, to_regnum + _do_regs ld 4 \base_reg, \base_offs, \from_regnum, \to_regnum + .endm + + + /* Push register pair on stack */ + .macro push, r1, r2 + stp \r1, \r2, [sp, #-16]! + .endm + + /* Pop register pair from stack */ + .macro pop, r1, r2 + ldp \r1, \r2, [sp], #16 + .endm + + .macro mov_imm _reg, _val + .if (((\_val) >> 31) == 0 || ((\_val) >> 31) == 0x1ffffffff) + movz \_reg, :abs_g1_s:\_val + .else + .if (((\_val) >> 47) == 0 || ((\_val) >> 47) == 0x1ffff) + movz \_reg, :abs_g2_s:\_val + .else + movz \_reg, :abs_g3:\_val + movk \_reg, :abs_g2_nc:\_val + .endif + movk \_reg, :abs_g1_nc:\_val + .endif + movk \_reg, :abs_g0_nc:\_val + .endm + + .macro add_imm _reg, _val + .if ((\_val) > 0xfff) + add \_reg, \_reg, ((\_val) >> 12), LSL #12 + .endif + .if (((\_val) & 0xfff) > 0) + add \_reg, \_reg, ((\_val) & 0xfff) + .endif + .endm + + .macro sub_imm _reg, _val + .if ((\_val) > 0xfff) + sub \_reg, \_reg, ((\_val) >> 12), LSL #12 + .endif + .if (((\_val) & 0xfff) > 0) + sub \_reg, \_reg, ((\_val) & 0xfff) + .endif + .endm + + /* + * Load address of into , being in the range + * +/- 4GB of the PC (note that 'adr reg, sym' is limited to +/- 1MB). + */ + .macro adr_l reg, sym + adrp \reg, \sym + add \reg, \reg, :lo12:\sym + .endm + + .macro panic_at_smc_return +#if defined(CFG_TEE_CORE_DEBUG) + bl __panic_at_smc_return +#else + b . +#endif + .endm + + .macro read_apiakeylo reg + mrs \reg, S3_0_c2_c1_0 + .endm + + .macro read_apiakeyhi reg + mrs \reg, S3_0_c2_c1_1 + .endm + + .macro write_apiakeylo reg + msr S3_0_c2_c1_0, \reg + .endm + + .macro write_apiakeyhi reg + msr S3_0_c2_c1_1, \reg + .endm + + .macro write_pan reg + /* msr pan, \reg */ + msr S3_0_c4_c2_3, \reg + .endm + + .macro write_pan_enable + /* msr pan, #1 */ + msr S0_0_c4_c1_4, xzr + .endm + + .macro write_pan_disable + /* msr pan, #0 */ + msr S0_0_c4_c0_4, xzr + .endm diff --git a/optee_os/core/arch/arm/include/crypto/ghash-ce-core.h b/optee_os/core/arch/arm/include/crypto/ghash-ce-core.h new file mode 100644 index 0000000..b0f9f54 --- /dev/null +++ b/optee_os/core/arch/arm/include/crypto/ghash-ce-core.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef __GHASH_CE_CORE_H +#define __GHASH_CE_CORE_H + +#include + +struct internal_ghash_key { + uint64_t h[2]; + uint64_t h2[2]; + uint64_t h3[2]; + uint64_t h4[2]; +}; + +void pmull_ghash_update_p64(int blocks, uint64_t dg[2], const uint8_t *src, + const struct internal_ghash_key *ghash_key, + const uint8_t *head); +void pmull_ghash_update_p8(int blocks, uint64_t dg[2], const uint8_t *src, + const struct internal_ghash_key *ghash_key, + const uint8_t *head); + +void pmull_gcm_load_round_keys(const uint64_t rk[30], int rounds); + +void pmull_gcm_encrypt(int blocks, uint64_t dg[2], uint8_t dst[], + const uint8_t src[], + const struct internal_ghash_key *ghash_key, + uint64_t ctr[], const uint64_t rk[], int rounds, + uint8_t ks[]); + + +void pmull_gcm_decrypt(int blocks, uint64_t dg[2], uint8_t dst[], + const uint8_t src[], + const struct internal_ghash_key *ghash_key, + uint64_t ctr[], const uint64_t rk[], int rounds); + +uint32_t pmull_gcm_aes_sub(uint32_t input); + +void pmull_gcm_encrypt_block(uint8_t dst[], const uint8_t src[], int rounds); + +#endif /*__GHASH_CE_CORE_H*/ diff --git a/optee_os/core/arch/arm/include/ffa.h b/optee_os/core/arch/arm/include/ffa.h new file mode 100644 index 0000000..c54479b --- /dev/null +++ b/optee_os/core/arch/arm/include/ffa.h @@ -0,0 +1,353 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, Linaro Limited + * Copyright (c) 2018-2023, Arm Limited. All rights reserved. + */ + +#ifndef __FFA_H +#define __FFA_H + +/* This is based on the FF-A 1.0 EAC specification */ + +#include +#include +#include + +/* Error codes */ +#define FFA_OK 0 +#define FFA_NOT_SUPPORTED -1 +#define FFA_INVALID_PARAMETERS -2 +#define FFA_NO_MEMORY -3 +#define FFA_BUSY -4 +#define FFA_INTERRUPTED -5 +#define FFA_DENIED -6 +#define FFA_RETRY -7 +#define FFA_ABORTED -8 + +/* FFA_VERSION helpers */ +#define FFA_VERSION_MAJOR U(1) +#define FFA_VERSION_MAJOR_SHIFT U(16) +#define FFA_VERSION_MAJOR_MASK U(0x7FFF) +#define FFA_VERSION_MINOR U(1) +#define FFA_VERSION_MINOR_SHIFT U(0) +#define FFA_VERSION_MINOR_MASK U(0xFFFF) +#define MAKE_FFA_VERSION(major, minor) \ + ((((major) & FFA_VERSION_MAJOR_MASK) << FFA_VERSION_MAJOR_SHIFT) | \ + ((minor) & FFA_VERSION_MINOR_MASK)) + +#define FFA_VERSION_1_0 MAKE_FFA_VERSION(1, 0) +#define FFA_VERSION_1_1 MAKE_FFA_VERSION(1, 1) + +/* Function IDs */ +#define FFA_ERROR U(0x84000060) +#define FFA_SUCCESS_32 U(0x84000061) +#define FFA_SUCCESS_64 U(0xC4000061) +#define FFA_INTERRUPT U(0x84000062) +#define FFA_VERSION U(0x84000063) +#define FFA_FEATURES U(0x84000064) +#define FFA_RX_RELEASE U(0x84000065) +#define FFA_RXTX_MAP_32 U(0x84000066) +#define FFA_RXTX_MAP_64 U(0xC4000066) +#define FFA_RXTX_UNMAP U(0x84000067) +#define FFA_PARTITION_INFO_GET U(0x84000068) +#define FFA_ID_GET U(0x84000069) +#define FFA_SPM_ID_GET U(0x84000085) +#define FFA_MSG_WAIT U(0x8400006B) +#define FFA_MSG_YIELD U(0x8400006C) +#define FFA_RUN U(0x8400006D) +#define FFA_MSG_SEND2 U(0x84000086) +#define FFA_MSG_SEND U(0x8400006E) +#define FFA_MSG_SEND_DIRECT_REQ_32 U(0x8400006F) +#define FFA_MSG_SEND_DIRECT_REQ_64 U(0xC400006F) +#define FFA_MSG_SEND_DIRECT_RESP_32 U(0x84000070) +#define FFA_MSG_SEND_DIRECT_RESP_64 U(0xC4000070) +#define FFA_MSG_POLL U(0x8400006A) +#define FFA_MEM_DONATE_32 U(0x84000071) +#define FFA_MEM_DONATE_64 U(0xC4000071) +#define FFA_MEM_LEND_32 U(0x84000072) +#define FFA_MEM_LEND_64 U(0xC4000072) +#define FFA_MEM_SHARE_32 U(0x84000073) +#define FFA_MEM_SHARE_64 U(0xC4000073) +#define FFA_MEM_RETRIEVE_REQ_32 U(0x84000074) +#define FFA_MEM_RETRIEVE_REQ_64 U(0xC4000074) +#define FFA_MEM_RETRIEVE_RESP U(0x84000075) +#define FFA_MEM_RELINQUISH U(0x84000076) +#define FFA_MEM_RECLAIM U(0x84000077) +#define FFA_MEM_FRAG_RX U(0x8400007A) +#define FFA_MEM_FRAG_TX U(0x8400007B) +#define FFA_NORMAL_WORLD_RESUME U(0x8400007C) +#define FFA_NOTIFICATION_BITMAP_CREATE U(0x8400007D) +#define FFA_NOTIFICATION_BITMAP_DESTROY U(0x8400007E) +#define FFA_NOTIFICATION_BIND U(0x8400007F) +#define FFA_NOTIFICATION_UNBIND U(0x84000080) +#define FFA_NOTIFICATION_SET U(0x84000081) +#define FFA_NOTIFICATION_GET U(0x84000082) +#define FFA_NOTIFICATION_INFO_GET_32 U(0x84000083) +#define FFA_NOTIFICATION_INFO_GET_64 U(0xC4000083) +#define FFA_SECONDARY_EP_REGISTER_64 U(0xC4000087) +#define FFA_MEM_PERM_GET_32 U(0x84000088) +#define FFA_MEM_PERM_GET_64 U(0xC4000088) +#define FFA_MEM_PERM_SET_32 U(0x84000089) +#define FFA_MEM_PERM_SET_64 U(0xC4000089) +#define FFA_CONSOLE_LOG_32 U(0x8400008A) +#define FFA_CONSOLE_LOG_64 U(0xC400008A) + +#define FFA_FEATURE_NOTIF_PEND_INTR U(0x1) +#define FFA_FEATURE_SCHEDULE_RECV_INTR U(0x2) +#define FFA_FEATURE_MANAGED_EXIT_INTR U(0x3) + +/* Special value for traffic targeted to the Hypervisor or SPM */ +#define FFA_TARGET_INFO_MBZ U(0x0) + +#define FFA_MSG_FLAG_FRAMEWORK BIT(31) +#define FFA_MSG_TYPE_MASK GENMASK_32(7, 0) +#define FFA_MSG_PSCI U(0x0) +#define FFA_MSG_SEND_VM_CREATED U(0x4) +#define FFA_MSG_RESP_VM_CREATED U(0x5) +#define FFA_MSG_SEND_VM_DESTROYED U(0x6) +#define FFA_MSG_RESP_VM_DESTROYED U(0x7) +#define FFA_MSG_VERSION_REQ U(0x8) +#define FFA_MSG_VERSION_RESP U(0x9) + +/* + * Flag used as parameter to FFA_PARTITION_INFO_GET to return partition + * count only. + */ +#define FFA_PARTITION_INFO_GET_COUNT_FLAG BIT(0) + +/* Memory attributes: Normal memory, Write-Back cacheable, Inner shareable */ +#define FFA_NORMAL_MEM_REG_ATTR U(0x2f) + +/* Memory access permissions: Read-write */ +#define FFA_MEM_ACC_RW BIT(1) + +/* Memory access permissions: executable */ +#define FFA_MEM_ACC_EXE BIT(3) + +/* Memory access permissions mask */ +#define FFA_MEM_ACC_MASK 0xf + +/* Clear memory before mapping in receiver */ +#define FFA_MEMORY_REGION_FLAG_CLEAR BIT(0) +/* Relayer may time slice this operation */ +#define FFA_MEMORY_REGION_FLAG_TIME_SLICE BIT(1) +/* Clear memory after receiver relinquishes it */ +#define FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH BIT(2) + +/* Share memory transaction */ +#define FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE SHIFT_U32(1, 3) +/* Relayer must choose the alignment boundary */ +#define FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT 0 + +#define FFA_MEM_PERM_DATA_PERM GENMASK_32(1, 0) +#define FFA_MEM_PERM_RW U(0x1) +#define FFA_MEM_PERM_RO U(0x3) + +#define FFA_MEM_PERM_INSTRUCTION_PERM BIT(2) +#define FFA_MEM_PERM_NX BIT(2) +#define FFA_MEM_PERM_X U(0) + +#define FFA_MEM_PERM_RESERVED GENMASK_32(31, 3) + +/* Special value for MBZ parameters */ +#define FFA_PARAM_MBZ U(0x0) + +/* + * The W1 register in FFA_INTERRUPT and FFA_RUN interfaces contains the target + * information. This value has two parts, the SP ID and vCPU ID. The SP ID + * identifies the SP to resume and the vCPU ID identifies the vCPU or execution + * context to resume (FF-A v1.1 section 4.8). + */ +#define FFA_TARGET_INFO_SET(sp_id, vcpu_id) (((sp_id) << 16) | (vcpu_id)) +#define FFA_TARGET_INFO_GET_SP_ID(info) (((info) >> 16) & 0xffff) +#define FFA_TARGET_INFO_GET_VCPU_ID(info) ((info) & 0xffff) + +/* + * Flags used for the FFA_PARTITION_INFO_GET return message: + * BIT(0): Supports receipt of direct requests + * BIT(1): Can send direct requests + * BIT(2): Can send and receive indirect messages + * BIT(3): Supports receipt of notifications + * BIT(4-5): Partition ID is a PE endpoint ID + */ +#define FFA_PART_PROP_DIRECT_REQ_RECV BIT(0) +#define FFA_PART_PROP_DIRECT_REQ_SEND BIT(1) +#define FFA_PART_PROP_INDIRECT_MSGS BIT(2) +#define FFA_PART_PROP_RECV_NOTIF BIT(3) +#define FFA_PART_PROP_IS_PE_ID SHIFT_U32(0, 4) +#define FFA_PART_PROP_IS_SEPID_INDEP SHIFT_U32(1, 4) +#define FFA_PART_PROP_IS_SEPID_DEP SHIFT_U32(2, 4) +#define FFA_PART_PROP_IS_AUX_ID SHIFT_U32(3, 4) +#define FFA_PART_PROP_NOTIF_CREATED BIT(6) +#define FFA_PART_PROP_NOTIF_DESTROYED BIT(7) +#define FFA_PART_PROP_AARCH64_STATE BIT(8) + +#define FFA_MEMORY_HANDLE_HYPERVISOR_BIT BIT64(63) +#define FFA_MEMORY_HANDLE_SECURE_BIT BIT64(45) +#define FFA_MEMORY_HANDLE_NON_SECURE_BIT BIT64(44) +/* + * Codes the OP-TEE partition/guest ID into a cookie in order to know which + * partition to activate when reclaiming the shared memory. This field is 0 + * unless CFG_NS_VIRTUALIZATION is enabled. + */ +#define FFA_MEMORY_HANDLE_PRTN_SHIFT 16 +#define FFA_MEMORY_HANDLE_PRTN_MASK GENMASK_32(16, 0) + + +#define FFA_BOOT_INFO_NAME_LEN U(16) + +/* Boot Info descriptors type */ +#define FFA_BOOT_INFO_TYPE_IMDEF BIT(7) +#define FFA_BOOT_INFO_TYPE_ID_MASK GENMASK_32(6, 0) +#define FFA_BOOT_INFO_TYPE_ID_FDT U(0) +#define FFA_BOOT_INFO_TYPE_ID_HOB U(1) + +/* Boot Info descriptors flags */ +#define FFA_BOOT_INFO_FLAG_NAME_FORMAT_MASK GENMASK_32(1, 0) +#define FFA_BOOT_INFO_FLAG_NAME_FORMAT_STRING U(0) +#define FFA_BOOT_INFO_FLAG_NAME_FORMAT_UUID U(1) + +/** Bits [3:2] encode the format of the content field in ffa_boot_info_desc. */ +#define FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_SHIFT U(2) +#define FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_MASK GENMASK_32(3, 2) +#define FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_VALUE U(1) +#define FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_ADDR U(0) + +#define FFA_BOOT_INFO_SIGNATURE U(0xFFA) +#define FFA_BOOT_INFO_VERSION U(0x10001) + +#define FFA_CONSOLE_LOG_CHAR_COUNT_MASK GENMASK_32(7, 0) +#define FFA_CONSOLE_LOG_32_MAX_MSG_LEN U(24) +#define FFA_CONSOLE_LOG_64_MAX_MSG_LEN U(48) + +#ifndef __ASSEMBLER__ +/* Constituent memory region descriptor */ +struct ffa_address_range { + uint64_t address; + uint32_t page_count; + uint32_t reserved; +}; + +/* Composite memory region descriptor */ +struct ffa_mem_region { + uint32_t total_page_count; + uint32_t address_range_count; + uint64_t reserved; + struct ffa_address_range address_range_array[]; +}; + +/* Memory access permissions descriptor */ +struct ffa_mem_access_perm { + uint16_t endpoint_id; + uint8_t perm; + uint8_t flags; +}; + +/* Endpoint memory access descriptor */ +struct ffa_mem_access { + struct ffa_mem_access_perm access_perm; + uint32_t region_offs; + uint64_t reserved; +}; + +/* Lend, donate or share memory transaction descriptor */ +struct ffa_mem_transaction_1_0 { + uint16_t sender_id; + uint8_t mem_reg_attr; + uint8_t reserved0; + uint32_t flags; + uint64_t global_handle; + uint64_t tag; + uint32_t reserved1; + uint32_t mem_access_count; + struct ffa_mem_access mem_access_array[]; +}; + +struct ffa_mem_transaction_1_1 { + uint16_t sender_id; + uint16_t mem_reg_attr; + uint32_t flags; + uint64_t global_handle; + uint64_t tag; + uint32_t mem_access_size; + uint32_t mem_access_count; + uint32_t mem_access_offs; + uint8_t reserved[12]; +}; + +/* + * The parts needed from struct ffa_mem_transaction_1_0 or struct + * ffa_mem_transaction_1_1, used to provide an abstraction of difference in + * data structures between version 1.0 and 1.1. This is just an internal + * interface and can be changed without changing any ABI. + */ +struct ffa_mem_transaction_x { + uint16_t sender_id; + uint8_t mem_reg_attr; + uint8_t flags; + uint8_t mem_access_size; + uint8_t mem_access_count; + uint16_t mem_access_offs; + uint64_t global_handle; + uint64_t tag; +}; + +#define FFA_UUID_SIZE 16 + +/* Partition information descriptor */ +struct ffa_partition_info_x { + uint16_t id; + uint16_t execution_context; + uint32_t partition_properties; + /* + * The uuid field is absent in FF-A 1.0, and an array of 16 + * (FFA_UUID_SIZE) from FF-A 1.1 + */ + uint8_t uuid[]; +}; + +/* Descriptor to relinquish a memory region (FFA_MEM_RELINQUISH) */ +struct ffa_mem_relinquish { + uint64_t handle; + uint32_t flags; + uint32_t endpoint_count; + uint16_t endpoint_id_array[]; +}; + +/* FF-A v1.0 boot information name-value pairs */ +struct ffa_boot_info_nvp_1_0 { + uint32_t name[4]; + uint64_t value; + uint64_t size; +}; + +/* FF-A v1.0 boot information descriptor */ +struct ffa_boot_info_1_0 { + uint32_t magic; + uint32_t count; + struct ffa_boot_info_nvp_1_0 nvp[]; +}; + +/* FF-A v1.1 boot information descriptor */ +struct ffa_boot_info_1_1 { + char name[FFA_BOOT_INFO_NAME_LEN]; + uint8_t type; + uint8_t reserved; + uint16_t flags; + uint32_t size; + uint64_t contents; +}; + +/* FF-A v1.1 boot information header */ +struct ffa_boot_info_header_1_1 { + uint32_t signature; + uint32_t version; + uint32_t blob_size; + uint32_t desc_size; + uint32_t desc_count; + uint32_t desc_offset; + uint64_t reserved; +}; + +#endif /*__ASSEMBLER__*/ +#endif /* __FFA_H */ diff --git a/optee_os/core/arch/arm/include/hafnium.h b/optee_os/core/arch/arm/include/hafnium.h new file mode 100644 index 0000000..542ae8f --- /dev/null +++ b/optee_os/core/arch/arm/include/hafnium.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright 2018 The Hafnium Authors. + * Copyright (c) 2022, Linaro Limited + */ + +#ifndef __HAFNIUM_H +#define __HAFNIUM_H + +/* + * This is based on inc/vmapi/hf/abi.h and inc/vmapi/hf/types.h from the + * Hafnium source tree. + */ + +/* + * Enables a given interrupt ID, returns 0 on success or -1 if the + * interrupt ID is invalid. + */ +#define HF_INTERRUPT_ENABLE 0xff03 + +/* + * Returns the ID of the next pending interrupt, and acknowledges it (i.e. + * marks it as no longer pending). Returns HF_INVALID_INTID if there are no + * pending interrupts. + */ +#define HF_INTERRUPT_GET 0xff04 + +/* + * Drops the current interrupt priority and deactivate the given interrupt + * ID. + */ +#define HF_INTERRUPT_DEACTIVATE 0xff08 + +/* Interrupt ID returned when there is no interrupt pending. */ +#define HF_INVALID_INTID 0xffffffff + +/* The virtual interrupt ID used for managed exit. */ +#define HF_MANAGED_EXIT_INTID 4 + +#define HF_INTERRUPT_TYPE_IRQ 0 +#define HF_INTERRUPT_TYPE_FIQ 1 +#define HF_ENABLE 1 +#define HF_DISABLE 0 + +#endif /*__HAFNIUM_H*/ diff --git a/optee_os/core/arch/arm/include/kernel/arch_scall.h b/optee_os/core/arch/arm/include/kernel/arch_scall.h new file mode 100644 index 0000000..cff6cc2 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/arch_scall.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2022, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ +#ifndef __KERNEL_ARCH_SCALL_H +#define __KERNEL_ARCH_SCALL_H + +#include +#include +#include + +static inline void scall_get_max_args(struct thread_scall_regs *regs, + size_t *scn, size_t *max_args) +{ +#ifdef ARM32 + *scn = regs->r7; + *max_args = regs->r6; +#endif +#ifdef ARM64 + if (((regs->spsr >> SPSR_MODE_RW_SHIFT) & SPSR_MODE_RW_MASK) == + SPSR_MODE_RW_32) { + *scn = regs->x7; + *max_args = regs->x6; + } else { + *scn = regs->x8; + *max_args = 0; + } +#endif +} + +static inline void scall_set_retval(struct thread_scall_regs *regs, + uint32_t ret_val) +{ +#ifdef ARM32 + regs->r0 = ret_val; +#endif +#ifdef ARM64 + regs->x0 = ret_val; +#endif +} + +static inline void scall_set_sys_return_regs(struct thread_scall_regs *regs, + bool panic, uint32_t panic_code) +{ +#ifdef ARM32 + regs->r1 = panic; + regs->r2 = panic_code; +#endif +#ifdef ARM64 + regs->x1 = panic; + regs->x2 = panic_code; +#endif +} +#endif /*__KERNEL_ARCH_SCALL_H*/ + diff --git a/optee_os/core/arch/arm/include/kernel/cache_helpers_arch.h b/optee_os/core/arch/arm/include/kernel/cache_helpers_arch.h new file mode 100644 index 0000000..aa136fe --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/cache_helpers_arch.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __KERNEL_CACHE_HELPERS_ARCH_H +#define __KERNEL_CACHE_HELPERS_ARCH_H + +#ifndef __ASSEMBLER__ +#include +#include +#endif + +#ifndef __ASSEMBLER__ + +static inline unsigned int dcache_get_line_size(void) +{ + uint32_t value = read_ctr(); + + return CTR_WORD_SIZE << + ((value >> CTR_DMINLINE_SHIFT) & CTR_DMINLINE_MASK); +} + +#endif /*!__ASSEMBLER__*/ + +#endif /*__KERNEL_CACHE_HELPERS_ARCH_H*/ diff --git a/optee_os/core/arch/arm/include/kernel/delay_arch.h b/optee_os/core/arch/arm/include/kernel/delay_arch.h new file mode 100644 index 0000000..12a2080 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/delay_arch.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __KERNEL_DELAY_ARCH_H +#define __KERNEL_DELAY_ARCH_H + +#include +#include +#include + +static inline uint64_t arm_cnt_us2cnt(uint32_t us) +{ + return ((uint64_t)us * (uint64_t)read_cntfrq()) / 1000000ULL; +} + +static inline uint64_t timeout_init_us(uint32_t us) +{ + return barrier_read_counter_timer() + arm_cnt_us2cnt(us); +} + +static inline bool timeout_elapsed(uint64_t expire) +{ + return barrier_read_counter_timer() > expire; +} + +#endif diff --git a/optee_os/core/arch/arm/include/kernel/misc_arch.h b/optee_os/core/arch/arm/include/kernel/misc_arch.h new file mode 100644 index 0000000..0653119 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/misc_arch.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef KERNEL_MISC_ARCH_H +#define KERNEL_MISC_ARCH_H + +#include +#include +#include + +size_t get_core_pos_mpidr(uint32_t mpidr); + +uint32_t read_mode_sp(int cpu_mode); +uint32_t read_mode_lr(int cpu_mode); + +void wait_cycles(unsigned long cycles); + +#endif /*KERNEL_MISC_ARCH_H*/ diff --git a/optee_os/core/arch/arm/include/kernel/secure_partition.h b/optee_os/core/arch/arm/include/kernel/secure_partition.h new file mode 100644 index 0000000..9924148 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/secure_partition.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020-2023, Arm Limited. + */ +#ifndef __KERNEL_SECURE_PARTITION_H +#define __KERNEL_SECURE_PARTITION_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TAILQ_HEAD(sp_sessions_head, sp_session); + +enum sp_status { sp_idle, sp_busy, sp_preempted, sp_dead }; + +struct sp_session { + struct ffa_rxtx rxtx; + enum sp_status state; + uint16_t endpoint_id; + uint16_t caller_id; + uint32_t boot_order; + struct ts_session ts_sess; + unsigned int spinlock; + const void *fdt; + bool is_initialized; + TEE_UUID ffa_uuid; + uint32_t ns_int_mode; + uint32_t ns_int_mode_inherited; + TAILQ_ENTRY(sp_session) link; +}; + +struct sp_ctx { + struct thread_ctx_regs sp_regs; + struct sp_session *open_session; + struct user_mode_ctx uctx; + struct ts_ctx ts_ctx; +}; + +struct sp_image { + struct embedded_ts image; + const void *fdt; +}; + +#ifdef CFG_SECURE_PARTITION +bool is_sp_ctx(struct ts_ctx *ctx); +#else +static inline bool is_sp_ctx(struct ts_ctx *ctx __unused) +{ + return false; +} +#endif + +static inline struct sp_session *__noprof +to_sp_session(struct ts_session *sess) +{ + assert(is_sp_ctx(sess->ctx)); + return container_of(sess, struct sp_session, ts_sess); +} + +static inline struct sp_ctx *to_sp_ctx(struct ts_ctx *ctx) +{ + assert(is_sp_ctx(ctx)); + return container_of(ctx, struct sp_ctx, ts_ctx); +} + +struct sp_session *sp_get_session(uint32_t session_id); +TEE_Result sp_enter(struct thread_smc_args *args, struct sp_session *sp); +TEE_Result sp_partition_info_get(uint32_t ffa_vers, void *buf, size_t buf_size, + const TEE_UUID *ffa_uuid, size_t *elem_count, + bool count_only); +bool sp_has_exclusive_access(struct sp_mem_map_region *mem, + struct user_mode_ctx *uctx); +TEE_Result sp_map_shared(struct sp_session *s, + struct sp_mem_receiver *receiver, + struct sp_mem *mem, + uint64_t *va); +TEE_Result sp_unmap_ffa_regions(struct sp_session *s, struct sp_mem *smem); + +#define for_each_secure_partition(_sp) \ + SCATTERED_ARRAY_FOREACH(_sp, sp_images, struct sp_image) + +struct fip_sp { + struct sp_image sp_img; + STAILQ_ENTRY(fip_sp) link; +}; + +STAILQ_HEAD(fip_sp_head, fip_sp); +extern struct fip_sp_head fip_sp_list; + +#define for_each_fip_sp(_sp) \ + STAILQ_FOREACH(_sp, &fip_sp_list, link) + +#endif /* __KERNEL_SECURE_PARTITION_H */ diff --git a/optee_os/core/arch/arm/include/kernel/spmc_sp_handler.h b/optee_os/core/arch/arm/include/kernel/spmc_sp_handler.h new file mode 100644 index 0000000..d43ee40 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/spmc_sp_handler.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Arm Limited. + */ +#ifndef __KERNEL_SPMC_SP_HANDLER_H +#define __KERNEL_SPMC_SP_HANDLER_H + +#include +#include +#include +#include +#include + +#define FFA_DST(x) ((x) & UINT16_MAX) +#define FFA_SRC(x) (((x) >> 16) & UINT16_MAX) + +void spmc_sp_thread_entry(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3); +void spmc_sp_msg_handler(struct thread_smc_args *args, + struct sp_session *caller_sp); +bool ffa_mem_reclaim(struct thread_smc_args *args, + struct sp_session *caller_sp); + +#ifdef CFG_SECURE_PARTITION +void spmc_sp_start_thread(struct thread_smc_args *args); +int spmc_sp_add_share(struct ffa_mem_transaction_x *mem_trans, + struct ffa_rxtx *rxtx, size_t blen, + uint64_t *global_handle, struct sp_session *owner_sp); +void spmc_sp_set_to_preempted(struct ts_session *ts_sess); +int spmc_sp_resume_from_preempted(uint16_t endpoint_id); +#else +static inline void spmc_sp_start_thread(struct thread_smc_args *args __unused) +{ +} + +static inline int +spmc_sp_add_share(struct ffa_mem_transaction_x *mem_trans __unused, + struct ffa_rxtx *rxtx __unused, size_t blen __unused, + uint64_t *global_handle __unused, + struct sp_session *owner_sp __unused) +{ + return FFA_NOT_SUPPORTED; +} + +static inline void spmc_sp_set_to_preempted(struct ts_session *ts_sess __unused) +{ +} + +static inline int spmc_sp_resume_from_preempted(uint16_t endpoint_id __unused) +{ + return FFA_NOT_SUPPORTED; +} +#endif + +#endif /* __KERNEL_SPMC_SP_HANDLER_H */ diff --git a/optee_os/core/arch/arm/include/kernel/stmm_sp.h b/optee_os/core/arch/arm/include/kernel/stmm_sp.h new file mode 100644 index 0000000..f9da6a0 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/stmm_sp.h @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019-2020, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ + +#ifndef __KERNEL_STMM_SP_H +#define __KERNEL_STMM_SP_H + +#include +#include +#include +#include +#include +#include +#include + +#define STMM_RET_SUCCESS 0 +#define STMM_RET_NOT_SUPPORTED -1 +#define STMM_RET_INVALID_PARAM -2 +#define STMM_RET_DENIED -3 +#define STMM_RET_NO_MEM -5 + +#define STMM_MEM_ATTR_ACCESS_MASK U(0x3) +#define STMM_MEM_ATTR_ACCESS_NONE U(0) +#define STMM_MEM_ATTR_ACCESS_RW U(1) +#define STMM_MEM_ATTR_ACCESS_RO U(3) +#define STMM_MEM_ATTR_EXEC_NEVER BIT(2) +#define STMM_MEM_ATTR_EXEC U(0) +#define STMM_MEM_ATTR_ALL (STMM_MEM_ATTR_ACCESS_RW | \ + STMM_MEM_ATTR_ACCESS_RO | \ + STMM_MEM_ATTR_EXEC_NEVER) + +/* + * Used for EDK2 StMM communication. Since StMM can be launched on an arbitrary + * address it uses these 2 syscalls to define the memory attributes for the + * data and code segments after dispatching the binaries. + * + * FFA_SVC_MEMORY_ATTRIBUTES_SET_64/FFA_SVC_MEMORY_ATTRIBUTES_SET_32: + * - x4: base address + * - x5: number of pages + * - x6: attributes of the remapping (described above) + * + * FFA_SVC_MEMORY_ATTRIBUTES_GET_32/FFA_SVC_MEMORY_ATTRIBUTES_GET_64: + * - x4: base address, currently only a single page is requested. + */ +#define FFA_SVC_MEMORY_ATTRIBUTES_GET_64 UINT32_C(0xC4000064) +#define FFA_SVC_MEMORY_ATTRIBUTES_SET_64 UINT32_C(0xC4000065) + +#define FFA_SVC_MEMORY_ATTRIBUTES_GET_32 UINT32_C(0x84000064) +#define FFA_SVC_MEMORY_ATTRIBUTES_SET_32 UINT32_C(0x84000065) + +/* + * We need to define the RPMB IDs formally, since the plan is + * for them to be included in the FFA spec (for SP-to-SP future communication). + * This is fine for now as it represents the internal contract between the + * EDK2 RPMB driver and Secure Partition + * + * FFA_SVC_RPMB_WRITE/FFA_SVC_RPMB_WRITE_32: + * - x4: virtual address of the buffer to write in the device + * - x5: buffer byte length + * - x6: byte offset in the device + * FFA_SVC_RPMB_READ/FFA_SVC_RPMB_READ_32: + * - x4: virtual address of the buffer were RPMB contents are copied + * - x5: buffer byte length to read + * - x6: byte offset in the device + */ +#define FFA_SVC_RPMB_READ UINT32_C(0xC4000066) +#define FFA_SVC_RPMB_WRITE UINT32_C(0xC4000067) + +#define FFA_SVC_RPMB_READ_32 UINT32_C(0x84000066) +#define FFA_SVC_RPMB_WRITE_32 UINT32_C(0x84000067) + +/* Param header types */ +#define STMM_PARAM_EP UINT8_C(0x01) +#define STMM_PARAM_IMAGE_BINARY UINT8_C(0x02) +#define STMM_PARAM_BL31 UINT8_C(0x03) +#define STMM_PARAM_BL_LOAD_INFO UINT8_C(0x04) +#define STMM_PARAM_BL_PARAMS UINT8_C(0x05) +#define STMM_PARAM_PSCI_LIB_ARGS UINT8_C(0x06) +#define STMM_PARAM_SP_IMAGE_BOOT_INFO UINT8_C(0x07) + +/* Param header version */ +#define STMM_PARAM_VERSION_1 UINT8_C(0x01) +#define STMM_PARAM_VERSION_2 UINT8_C(0x02) + +/* + * This structure provides information on format used to describe + * secure partition invocation parameters. + */ +struct stmm_param_header { + uint8_t type; /* type of the structure */ + uint8_t version; /* version of this structure */ + uint16_t size; /* size of this structure in bytes */ + uint32_t attr; /* attributes: unused bits SBZ */ +}; + +/* + * Flags used by the stmm_mp_info structure to describe the + * characteristics of a cpu. Only a single flag is defined at the moment to + * indicate the primary cpu. + */ +#define MP_INFO_FLAG_PRIMARY_CPU UINT32_C(0x00000001) + +/* + * This structure is used to provide information required to initialise a S-EL0 + * partition. + */ +struct stmm_mp_info { + uint64_t mpidr; + uint32_t linear_id; + uint32_t flags; +}; + +struct stmm_boot_info { + struct stmm_param_header h; + uint64_t sp_mem_base; + uint64_t sp_mem_limit; + uint64_t sp_image_base; + uint64_t sp_stack_base; + uint64_t sp_heap_base; + uint64_t sp_ns_comm_buf_base; + uint64_t sp_shared_buf_base; + uint64_t sp_image_size; + uint64_t sp_pcpu_stack_size; + uint64_t sp_heap_size; + uint64_t sp_ns_comm_buf_size; + uint64_t sp_shared_buf_size; + uint32_t num_sp_mem_regions; + uint32_t num_cpus; + struct stmm_mp_info *mp_info; +}; + +struct stmm_ctx { + struct user_mode_ctx uctx; + struct tee_ta_ctx ta_ctx; + struct thread_ctx_regs regs; + vaddr_t ns_comm_buf_addr; + unsigned int ns_comm_buf_size; + bool is_initializing; +}; + +extern const struct ts_ops stmm_sp_ops; + +static inline bool is_stmm_ctx(struct ts_ctx *ctx __maybe_unused) +{ + return IS_ENABLED(CFG_WITH_STMM_SP) && ctx && ctx->ops == &stmm_sp_ops; +} + +static inline struct stmm_ctx *to_stmm_ctx(struct ts_ctx *ctx) +{ + assert(is_stmm_ctx(ctx)); + return container_of(ctx, struct stmm_ctx, ta_ctx.ts_ctx); +} + +#ifdef CFG_WITH_STMM_SP +TEE_Result stmm_init_session(const TEE_UUID *uuid, + struct tee_ta_session *s); +#else +static inline TEE_Result +stmm_init_session(const TEE_UUID *uuid __unused, + struct tee_ta_session *s __unused) +{ + return TEE_ERROR_ITEM_NOT_FOUND; +} +#endif + +#ifdef CFG_WITH_STMM_SP +const TEE_UUID *stmm_get_uuid(void); +#else +static inline const TEE_UUID *stmm_get_uuid(void) { return NULL; } +#endif + +#endif /*__KERNEL_STMM_SP_H*/ diff --git a/optee_os/core/arch/arm/include/kernel/tee_l2cc_mutex.h b/optee_os/core/arch/arm/include/kernel/tee_l2cc_mutex.h new file mode 100644 index 0000000..1011fce --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/tee_l2cc_mutex.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TEE_L2CC_MUTEX_H +#define TEE_L2CC_MUTEX_H +#include +#include +#include +#include + +#if defined(CFG_PL310) +TEE_Result tee_enable_l2cc_mutex(void); +TEE_Result tee_disable_l2cc_mutex(void); +TEE_Result tee_get_l2cc_mutex(paddr_t *mutex); +TEE_Result tee_set_l2cc_mutex(paddr_t *mutex); +void tee_l2cc_mutex_lock(void); +void tee_l2cc_mutex_unlock(void); + +/* + * Store the pa of a mutex used for l2cc + * It is allocated from the boot + */ +void tee_l2cc_store_mutex_boot_pa(uint32_t pa); + +#else +static TEE_Result tee_enable_l2cc_mutex(void); +static TEE_Result tee_disable_l2cc_mutex(void); +static TEE_Result tee_get_l2cc_mutex(paddr_t *mutex); +static TEE_Result tee_set_l2cc_mutex(paddr_t *mutex); + +static inline TEE_Result tee_enable_l2cc_mutex(void) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +static inline TEE_Result tee_disable_l2cc_mutex(void) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +static inline TEE_Result tee_get_l2cc_mutex(paddr_t *mutex __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +static inline TEE_Result tee_set_l2cc_mutex(paddr_t *mutex __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +#endif /* TEE_L2CC_MUTEX_H */ diff --git a/optee_os/core/arch/arm/include/kernel/thread_arch.h b/optee_os/core/arch/arm/include/kernel/thread_arch.h new file mode 100644 index 0000000..37fffbc --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/thread_arch.h @@ -0,0 +1,465 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2016-2022, Linaro Limited + * Copyright (c) 2020-2021, Arm Limited + */ + +#ifndef __KERNEL_THREAD_ARCH_H +#define __KERNEL_THREAD_ARCH_H + +#ifndef __ASSEMBLER__ +#include +#include +#include +#include +#endif + +#ifndef __ASSEMBLER__ + +#ifdef ARM64 +/* + * struct thread_core_local needs to have alignment suitable for a stack + * pointer since SP_EL1 points to this + */ +#define THREAD_CORE_LOCAL_ALIGNED __aligned(16) +#else +#define THREAD_CORE_LOCAL_ALIGNED __aligned(8) +#endif + +struct mobj; + +/* + * Storage of keys used for pointer authentication. FEAT_PAuth supports a + * number of keys of which only the APIA key is currently used, depending on + * configuration. + */ +struct thread_pauth_keys { + uint64_t apia_hi; + uint64_t apia_lo; +}; + +struct thread_core_local { +#ifdef ARM32 + uint32_t r[2]; + paddr_t sm_pm_ctx_phys; +#endif +#ifdef ARM64 + uint64_t x[4]; +#endif +#ifdef CFG_CORE_PAUTH + struct thread_pauth_keys keys; +#endif + vaddr_t tmp_stack_va_end; + long kcode_offset; + short int curr_thread; + uint32_t flags; + vaddr_t abt_stack_va_end; +#ifdef CFG_TEE_CORE_DEBUG + unsigned int locked_count; /* Number of spinlocks held */ +#endif +#if defined(ARM64) && defined(CFG_CORE_FFA) + /* Function ID to use for a direct response, 32-bit vs 64-bit */ + uint32_t direct_resp_fid; +#endif +#if defined(ARM64) && defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) + uint8_t bhb_loop_count; +#endif +#ifdef CFG_CORE_DEBUG_CHECK_STACKS + bool stackcheck_recursion; +#endif +#ifdef CFG_FAULT_MITIGATION + struct ftmn_func_arg *ftmn_arg; +#endif +} THREAD_CORE_LOCAL_ALIGNED; + +struct thread_vector_table { + uint32_t std_smc_entry; + uint32_t fast_smc_entry; + uint32_t cpu_on_entry; + uint32_t cpu_off_entry; + uint32_t cpu_resume_entry; + uint32_t cpu_suspend_entry; + uint32_t fiq_entry; + uint32_t system_off_entry; + uint32_t system_reset_entry; +}; + +extern struct thread_vector_table thread_vector_table; + +struct thread_user_vfp_state { + struct vfp_state vfp; + bool lazy_saved; + bool saved; +}; + +#ifdef ARM32 +struct thread_smc_args { + uint32_t a0; /* SMC function ID */ + uint32_t a1; /* Parameter */ + uint32_t a2; /* Parameter */ + uint32_t a3; /* Thread ID when returning from RPC */ + uint32_t a4; /* Not used */ + uint32_t a5; /* Not used */ + uint32_t a6; /* Not used */ + uint32_t a7; /* Hypervisor Client ID */ +}; +#endif /*ARM32*/ +#ifdef ARM64 +struct thread_smc_args { + uint64_t a0; /* SMC function ID */ + uint64_t a1; /* Parameter */ + uint64_t a2; /* Parameter */ + uint64_t a3; /* Thread ID when returning from RPC */ + uint64_t a4; /* Not used */ + uint64_t a5; /* Not used */ + uint64_t a6; /* Not used */ + uint64_t a7; /* Hypervisor Client ID */ +}; +#endif /*ARM64*/ + +#ifdef ARM32 +struct thread_abort_regs { + uint32_t usr_sp; + uint32_t usr_lr; + uint32_t pad; + uint32_t spsr; + uint32_t elr; + uint32_t r0; + uint32_t r1; + uint32_t r2; + uint32_t r3; + uint32_t r4; + uint32_t r5; + uint32_t r6; + uint32_t r7; + uint32_t r8; + uint32_t r9; + uint32_t r10; + uint32_t r11; + uint32_t ip; +}; +#endif /*ARM32*/ +#ifdef ARM64 +struct thread_abort_regs { + uint64_t x0; /* r0_usr */ + uint64_t x1; /* r1_usr */ + uint64_t x2; /* r2_usr */ + uint64_t x3; /* r3_usr */ + uint64_t x4; /* r4_usr */ + uint64_t x5; /* r5_usr */ + uint64_t x6; /* r6_usr */ + uint64_t x7; /* r7_usr */ + uint64_t x8; /* r8_usr */ + uint64_t x9; /* r9_usr */ + uint64_t x10; /* r10_usr */ + uint64_t x11; /* r11_usr */ + uint64_t x12; /* r12_usr */ + uint64_t x13; /* r13/sp_usr */ + uint64_t x14; /* r14/lr_usr */ + uint64_t x15; + uint64_t x16; + uint64_t x17; + uint64_t x18; + uint64_t x19; + uint64_t x20; + uint64_t x21; + uint64_t x22; + uint64_t x23; + uint64_t x24; + uint64_t x25; + uint64_t x26; + uint64_t x27; + uint64_t x28; + uint64_t x29; + uint64_t x30; + uint64_t elr; + uint64_t spsr; + uint64_t sp_el0; +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + uint64_t apiakey_hi; + uint64_t apiakey_lo; +#endif +}; +#endif /*ARM64*/ + +#ifdef ARM32 +struct thread_scall_regs { + uint32_t spsr; + uint32_t r0; + uint32_t r1; + uint32_t r2; + uint32_t r3; + uint32_t r4; + uint32_t r5; + uint32_t r6; + uint32_t r7; + uint32_t lr; +}; +#endif /*ARM32*/ +#ifdef ARM64 +struct thread_scall_regs { + uint64_t elr; + uint64_t spsr; + uint64_t x0; /* r0_usr */ + uint64_t x1; /* r1_usr */ + uint64_t x2; /* r2_usr */ + uint64_t x3; /* r3_usr */ + uint64_t x4; /* r4_usr */ + uint64_t x5; /* r5_usr */ + uint64_t x6; /* r6_usr */ + uint64_t x7; /* r7_usr */ + uint64_t x8; /* r8_usr */ + uint64_t x9; /* r9_usr */ + uint64_t x10; /* r10_usr */ + uint64_t x11; /* r11_usr */ + uint64_t x12; /* r12_usr */ + uint64_t x13; /* r13/sp_usr */ + uint64_t x14; /* r14/lr_usr */ + uint64_t x30; + uint64_t sp_el0; +#ifdef CFG_SECURE_PARTITION + uint64_t x15; + uint64_t x16; + uint64_t x17; + uint64_t x18; + uint64_t x19; + uint64_t x20; + uint64_t x21; + uint64_t x22; + uint64_t x23; + uint64_t x24; + uint64_t x25; + uint64_t x26; + uint64_t x27; + uint64_t x28; + uint64_t x29; +#endif +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + uint64_t apiakey_hi; + uint64_t apiakey_lo; +#endif + uint64_t pad; +} __aligned(16); +#endif /*ARM64*/ + +#ifdef ARM32 +struct thread_ctx_regs { + uint32_t r0; + uint32_t r1; + uint32_t r2; + uint32_t r3; + uint32_t r4; + uint32_t r5; + uint32_t r6; + uint32_t r7; + uint32_t r8; + uint32_t r9; + uint32_t r10; + uint32_t r11; + uint32_t r12; + uint32_t usr_sp; + uint32_t usr_lr; + uint32_t svc_spsr; + uint32_t svc_sp; + uint32_t svc_lr; + uint32_t pc; + uint32_t cpsr; +}; +#endif /*ARM32*/ + +#ifdef ARM64 +struct thread_ctx_regs { + uint64_t sp; + uint64_t pc; + uint64_t cpsr; + uint64_t x[31]; + uint64_t tpidr_el0; +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + uint64_t apiakey_hi; + uint64_t apiakey_lo; +#endif +}; +#endif /*ARM64*/ + +struct user_mode_ctx; + +#ifdef CFG_WITH_ARM_TRUSTED_FW +/* + * These five functions have a __weak default implementation which does + * nothing. Platforms are expected to override them if needed. + */ +unsigned long thread_cpu_off_handler(unsigned long a0, unsigned long a1); +unsigned long thread_cpu_suspend_handler(unsigned long a0, unsigned long a1); +unsigned long thread_cpu_resume_handler(unsigned long a0, unsigned long a1); +unsigned long thread_system_off_handler(unsigned long a0, unsigned long a1); +unsigned long thread_system_reset_handler(unsigned long a0, unsigned long a1); +#endif /*CFG_WITH_ARM_TRUSTED_FW*/ + +/* + * Defines the bits for the exception mask used by the + * thread_*_exceptions() functions below. + * These definitions are compatible with both ARM32 and ARM64. + */ +#if defined(CFG_CORE_IRQ_IS_NATIVE_INTR) +#define THREAD_EXCP_FOREIGN_INTR (ARM32_CPSR_F >> ARM32_CPSR_F_SHIFT) +#define THREAD_EXCP_NATIVE_INTR (ARM32_CPSR_I >> ARM32_CPSR_F_SHIFT) +#else +#define THREAD_EXCP_FOREIGN_INTR (ARM32_CPSR_I >> ARM32_CPSR_F_SHIFT) +#define THREAD_EXCP_NATIVE_INTR (ARM32_CPSR_F >> ARM32_CPSR_F_SHIFT) +#endif +#define THREAD_EXCP_ALL (THREAD_EXCP_FOREIGN_INTR \ + | THREAD_EXCP_NATIVE_INTR \ + | (ARM32_CPSR_A >> ARM32_CPSR_F_SHIFT)) + +#ifdef CFG_WITH_VFP +/* + * thread_kernel_enable_vfp() - Temporarily enables usage of VFP + * + * Foreign interrupts are masked while VFP is enabled. User space must not be + * entered before thread_kernel_disable_vfp() has been called to disable VFP + * and restore the foreign interrupt status. + * + * This function may only be called from an active thread context and may + * not be called again before thread_kernel_disable_vfp() has been called. + * + * VFP state is saved as needed. + * + * Returns a state variable that should be passed to + * thread_kernel_disable_vfp(). + */ +uint32_t thread_kernel_enable_vfp(void); + +/* + * thread_kernel_disable_vfp() - Disables usage of VFP + * @state: state variable returned by thread_kernel_enable_vfp() + * + * Disables usage of VFP and restores foreign interrupt status after a call to + * thread_kernel_enable_vfp(). + * + * This function may only be called after a call to + * thread_kernel_enable_vfp(). + */ +void thread_kernel_disable_vfp(uint32_t state); + +/* + * thread_kernel_save_vfp() - Saves kernel vfp state if enabled + */ +void thread_kernel_save_vfp(void); + +/* + * thread_kernel_save_vfp() - Restores kernel vfp state + */ +void thread_kernel_restore_vfp(void); + +/* + * thread_user_enable_vfp() - Enables vfp for user mode usage + * @uvfp: pointer to where to save the vfp state if needed + */ +void thread_user_enable_vfp(struct thread_user_vfp_state *uvfp); +#else /*CFG_WITH_VFP*/ +static inline void thread_kernel_save_vfp(void) +{ +} + +static inline void thread_kernel_restore_vfp(void) +{ +} +#endif /*CFG_WITH_VFP*/ + +/* + * thread_user_save_vfp() - Saves the user vfp state if enabled + */ +#ifdef CFG_WITH_VFP +void thread_user_save_vfp(void); +#else +static inline void thread_user_save_vfp(void) +{ +} +#endif + +/* + * thread_user_clear_vfp() - Clears the vfp state + * @uctx: pointer to user mode context containing the saved state to clear + */ +#ifdef CFG_WITH_VFP +void thread_user_clear_vfp(struct user_mode_ctx *uctx); +#else +static inline void thread_user_clear_vfp(struct user_mode_ctx *uctx __unused) +{ +} +#endif + +#ifdef ARM64 +/* + * thread_get_saved_thread_sp() - Returns the saved sp of current thread + * + * When switching from the thread stack pointer the value is stored + * separately in the current thread context. This function returns this + * saved value. + * + * @returns stack pointer + */ +vaddr_t thread_get_saved_thread_sp(void); +#endif /*ARM64*/ + +/* + * Provides addresses and size of kernel code that must be mapped while in + * user mode. + */ +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 +void thread_get_user_kcode(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz); +#else +static inline void thread_get_user_kcode(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz) +{ + *mobj = NULL; + *offset = 0; + *va = 0; + *sz = 0; +} +#endif + +/* + * Provides addresses and size of kernel (rw) data that must be mapped + * while in user mode. + */ +#if defined(CFG_CORE_UNMAP_CORE_AT_EL0) && \ + defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) +void thread_get_user_kdata(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz); +#else +static inline void thread_get_user_kdata(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz) +{ + *mobj = NULL; + *offset = 0; + *va = 0; + *sz = 0; +} +#endif + +/* + * Disables and empties the prealloc RPC cache one reference at a time. If + * all threads are idle this function returns true and a cookie of one shm + * object which was removed from the cache. When the cache is empty *cookie + * is set to 0 and the cache is disabled else a valid cookie value. If one + * thread isn't idle this function returns false. + */ +bool thread_disable_prealloc_rpc_cache(uint64_t *cookie); + +/* + * Enabled the prealloc RPC cache. If all threads are idle the cache is + * enabled and this function returns true. If one thread isn't idle this + * function return false. + */ +bool thread_enable_prealloc_rpc_cache(void); + +unsigned long thread_hvc(unsigned long func_id, unsigned long a1, + unsigned long a2, unsigned long a3); +unsigned long thread_smc(unsigned long func_id, unsigned long a1, + unsigned long a2, unsigned long a3); +void thread_smccc(struct thread_smc_args *arg_res); +#endif /*__ASSEMBLER__*/ +#endif /*__KERNEL_THREAD_ARCH_H*/ diff --git a/optee_os/core/arch/arm/include/kernel/thread_private_arch.h b/optee_os/core/arch/arm/include/kernel/thread_private_arch.h new file mode 100644 index 0000000..d5b1884 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/thread_private_arch.h @@ -0,0 +1,263 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef __KERNEL_THREAD_PRIVATE_ARCH_H +#define __KERNEL_THREAD_PRIVATE_ARCH_H + +#ifndef __ASSEMBLER__ + +#include +#include +#include + +#ifdef CFG_WITH_ARM_TRUSTED_FW +#define STACK_TMP_OFFS 0 +#else +#define STACK_TMP_OFFS SM_STACK_TMP_RESERVE_SIZE +#endif + +#ifdef ARM32 +#ifdef CFG_CORE_SANITIZE_KADDRESS +#define STACK_TMP_SIZE (3072 + STACK_TMP_OFFS + CFG_STACK_TMP_EXTRA) +#else +#define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS + CFG_STACK_TMP_EXTRA) +#endif +#define STACK_THREAD_SIZE (8192 + CFG_STACK_THREAD_EXTRA) + +#if defined(CFG_CORE_SANITIZE_KADDRESS) || defined(__clang__) || \ + !defined(CFG_CRYPTO_WITH_CE) +#define STACK_ABT_SIZE 3072 +#else +#define STACK_ABT_SIZE 2048 +#endif + +#endif /*ARM32*/ + +#ifdef ARM64 +#if (defined(__clang__) && !defined(__OPTIMIZE_SIZE__)) || \ + defined(CFG_CORE_SANITIZE_KADDRESS) +#define STACK_TMP_SIZE (4096 + STACK_TMP_OFFS + CFG_STACK_TMP_EXTRA) +#else +#define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS + CFG_STACK_TMP_EXTRA) +#endif +#if defined(CFG_CORE_SANITIZE_KADDRESS) +#define STACK_THREAD_SIZE (10240 + CFG_STACK_THREAD_EXTRA) +#else +#define STACK_THREAD_SIZE (8192 + CFG_STACK_THREAD_EXTRA) +#endif + +#if TRACE_LEVEL > 0 +#define STACK_ABT_SIZE 3072 +#else +#define STACK_ABT_SIZE 1024 +#endif +#endif /*ARM64*/ + +#ifdef CFG_CORE_DEBUG_CHECK_STACKS +/* + * Extra space added to each stack in order to reliably detect and dump stack + * overflows. Should cover the maximum expected overflow size caused by any C + * function (say, 512 bytes; no function should have that much local variables), + * plus the maximum stack space needed by __cyg_profile_func_exit(): about 1 KB, + * a large part of which is used to print the call stack. Total: 1.5 KB. + */ +#define STACK_CHECK_EXTRA 1536 +#else +#define STACK_CHECK_EXTRA 0 +#endif + +#ifdef ARM64 +struct thread_user_mode_rec { + uint64_t ctx_regs_ptr; + uint64_t exit_status0_ptr; + uint64_t exit_status1_ptr; + uint64_t pad; + uint64_t x[31 - 19]; /* x19..x30 */ +}; +#endif /*ARM64*/ + +#ifdef CFG_WITH_VFP +struct thread_vfp_state { + bool ns_saved; + bool sec_saved; + bool sec_lazy_saved; + struct vfp_state ns; + struct vfp_state sec; + struct thread_user_vfp_state *uvfp; +}; + +#endif /*CFG_WITH_VFP*/ +#endif /*__ASSEMBLER__*/ + +#ifdef ARM64 +#ifdef CFG_WITH_VFP +#define THREAD_VFP_STATE_SIZE \ + (16 + (16 * 32 + 16) * 2 + 16) +#else +#define THREAD_VFP_STATE_SIZE 0 +#endif +#endif /*ARM64*/ + +#ifndef __ASSEMBLER__ + +/* + * During boot note the part of code and data that needs to be mapped while + * in user mode. The provided address and size have to be page aligned. + * Note that the code and data will be mapped at the lowest possible + * addresses available for user space (see core_mmu_get_user_va_range()). + */ +extern long thread_user_kcode_offset; + +/* + * Initializes VBAR for current CPU (called by thread_init_per_cpu() + */ +void thread_init_vbar(vaddr_t addr); + +void thread_excp_vect(void); +void thread_excp_vect_wa_spectre_v2(void); +void thread_excp_vect_wa_a15_spectre_v2(void); +void thread_excp_vect_wa_spectre_bhb(void); +void thread_excp_vect_end(void); + +/* + * Assembly function as the first function in a thread. Handles a stdcall, + * a0-a3 holds the parameters. Hands over to __thread_std_smc_entry() when + * everything is set up and does some post processing once + * __thread_std_smc_entry() returns. + */ +void thread_std_smc_entry(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5); +uint32_t __thread_std_smc_entry(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5); + +void thread_sp_alloc_and_run(struct thread_smc_args *args); + +/* + * Resumes execution of currently active thread by restoring context and + * jumping to the instruction where to continue execution. + * + * Arguments supplied by non-secure world will be copied into the saved + * context of the current thread if THREAD_FLAGS_COPY_ARGS_ON_RETURN is set + * in the flags field in the thread context. + */ +void thread_resume(struct thread_ctx_regs *regs); + +uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, + uint32_t *exit_status0, + uint32_t *exit_status1); + +/* + * Private functions made available for thread_asm.S + */ + +/* Returns the temp stack for current CPU */ +void *thread_get_tmp_sp(void); + +/* + * Marks the current thread as suspended. And updated the flags + * for the thread context (see thread resume for use of flags). + * Returns thread index of the thread that was suspended. + */ +int thread_state_suspend(uint32_t flags, uint32_t cpsr, vaddr_t pc); + +/* + * Marks the current thread as free. + */ +void thread_state_free(void); + +/* Returns a pointer to the saved registers in current thread context. */ +struct thread_ctx_regs *thread_get_ctx_regs(void); + +#ifdef ARM32 +/* Sets sp for abort mode */ +void thread_set_abt_sp(vaddr_t sp); + +/* Sets sp for undefined mode */ +void thread_set_und_sp(vaddr_t sp); + +/* Sets sp for irq mode */ +void thread_set_irq_sp(vaddr_t sp); + +/* Sets sp for fiq mode */ +void thread_set_fiq_sp(vaddr_t sp); + +/* Read usr_sp banked CPU register */ +uint32_t thread_get_usr_sp(void); +uint32_t thread_get_usr_lr(void); +void thread_set_usr_lr(uint32_t usr_lr); +#endif /*ARM32*/ + +void thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5); +void thread_resume_from_rpc(uint32_t thread_id, uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3); + +/* + * The thread_rpc() function suspends current thread and temporarily exits + * to non-secure world. This function returns later when non-secure world + * returns. + * + * The purpose of this function is to request services from non-secure + * world. + */ +#define THREAD_RPC_NUM_ARGS 4 +#ifdef ARM64 +void thread_rpc_spsr(uint32_t rv[THREAD_RPC_NUM_ARGS], uint64_t spsr); +void __thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]); + +#ifdef CFG_CORE_FFA +struct thread_rpc_arg { + union { + struct { + uint32_t w1; + uint32_t w4; + uint32_t w5; + uint32_t w6; + } call; + struct { + uint32_t w4; + uint32_t w5; + uint32_t w6; + } ret; + uint32_t pad[THREAD_RPC_NUM_ARGS]; + }; +}; + +static inline void thread_rpc(struct thread_rpc_arg *rpc_arg) +{ + __thread_rpc(rpc_arg->pad); +} +#else +static inline void thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]) +{ + __thread_rpc(rv); +} +#endif +#endif +#ifdef ARM32 +void thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]); +#endif + +/* + * Called from assembly only, vector_fast_smc_entry(). Handles a fast SMC + * by dispatching it to the registered fast SMC handler. + */ +void thread_handle_fast_smc(struct thread_smc_args *args); + +/* + * Called from assembly only, vector_std_smc_entry(). Handles a std SMC by + * dispatching it to the registered std SMC handler. + */ +uint32_t thread_handle_std_smc(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6, uint32_t a7); + +/* Called from assembly only. Handles a SVC from user mode. */ +void thread_scall_handler(struct thread_scall_regs *regs); + +void thread_spmc_register_secondary_ep(vaddr_t ep); +#endif /*__ASSEMBLER__*/ +#endif /*__KERNEL_THREAD_PRIVATE_ARCH_H*/ diff --git a/optee_os/core/arch/arm/include/kernel/thread_spmc.h b/optee_os/core/arch/arm/include/kernel/thread_spmc.h new file mode 100644 index 0000000..ffef090 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/thread_spmc.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Arm Limited. + */ +#ifndef __KERNEL_THREAD_SPMC_H +#define __KERNEL_THREAD_SPMC_H + +#include +#include + +/* FF-A endpoint base ID when OP-TEE is used as a S-EL1 endpoint */ +#define SPMC_ENDPOINT_ID 0x8001 + +struct ffa_rxtx { + void *rx; + void *tx; + unsigned int size; + unsigned int spinlock; + uint32_t ffa_vers; + bool tx_is_mine; +}; + +void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *buf); +void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *buf); +void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *buf); +uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx); + +void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, + uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5); +void spmc_handle_partition_info_get(struct thread_smc_args *args, + struct ffa_rxtx *rxtx); +TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, + size_t idx, uint16_t endpoint_id, + uint16_t execution_context, + uint32_t part_props, + const uint32_t uuid_words[4]); +int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, + struct ffa_mem_transaction_x *trans); + +#if !defined(CFG_CORE_SEL1_SPMC) +struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie); +void thread_spmc_relinquish(uint64_t memory_region_handle); +#endif + +#endif /* __KERNEL_THREAD_SPMC_H */ diff --git a/optee_os/core/arch/arm/include/kernel/tlb_helpers.h b/optee_os/core/arch/arm/include/kernel/tlb_helpers.h new file mode 100644 index 0000000..36a9af3 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/tlb_helpers.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef TLB_HELPERS_H +#define TLB_HELPERS_H + +#include + +#ifndef __ASSEMBLER__ +#include + +void tlbi_all(void); +void tlbi_asid(unsigned long asid); +void tlbi_va_allasid(unsigned long addr); + +static inline void tlbi_va_allasid_nosync(vaddr_t va) +{ +#ifdef ARM64 + tlbi_vaae1is(va >> TLBI_VA_SHIFT); +#else + write_tlbimvaais(va); +#endif +} + +static inline void tlbi_va_asid_nosync(vaddr_t va, uint32_t asid) +{ + uint32_t a = asid & TLBI_ASID_MASK; + +#ifdef ARM64 + tlbi_vale1is((va >> TLBI_VA_SHIFT) | SHIFT_U64(a, TLBI_ASID_SHIFT)); + tlbi_vale1is((va >> TLBI_VA_SHIFT) | + SHIFT_U64(a | 1, TLBI_ASID_SHIFT)); +#else + write_tlbimvais((va & ~(BIT32(TLBI_MVA_SHIFT) - 1)) | a); + write_tlbimvais((va & ~(BIT32(TLBI_MVA_SHIFT) - 1)) | a | 1); +#endif +} + +static inline void tlbi_va_asid(vaddr_t va, uint32_t asid) +{ + dsb_ishst(); + tlbi_va_asid_nosync(va, asid); + dsb_ish(); + isb(); +} +#endif /*!__ASSEMBLER__*/ + +#endif /* TLB_HELPERS_H */ diff --git a/optee_os/core/arch/arm/include/kernel/tz_proc_def.h b/optee_os/core/arch/arm/include/kernel/tz_proc_def.h new file mode 100644 index 0000000..6e9097d --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/tz_proc_def.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include + +/* + * General constants + */ + +/* + * CP15 Multiprocessor Affinity register (MPIDR) + */ +#define CP15_CONFIG_CPU_ID_MASK U(0x00000003) +#define CPU_ID0 U(0x00000000) +#define CPU_ID1 U(0x00000001) + +/* + * CP15 Secure configuration register + */ +#define CP15_CONFIG_NS_MASK U(0x00000001) +#define CP15_CONFIG_IRQ_MASK U(0x00000002) +#define CP15_CONFIG_FIQ_MASK U(0x00000004) +#define CP15_CONFIG_EA_MASK U(0x00000008) +#define CP15_CONFIG_FW_MASK U(0x00000010) +#define CP15_CONFIG_AW_MASK U(0x00000020) +#define CP15_CONFIG_nET_MASK U(0x00000040) + +/* + * CP15 Control register + */ +#define CP15_CONTROL_M_MASK U(0x00000001) +#define CP15_CONTROL_C_MASK U(0x00000004) +#define CP15_CONTROL_Z_MASK U(0x00000800) +#define CP15_CONTROL_I_MASK U(0x00001000) +#define CP15_CONTROL_V_MASK U(0x00002000) +#define CP15_CONTROL_HA_MASK U(0x00020000) +#define CP15_CONTROL_EE_MASK U(0x02000000) +#define CP15_CONTROL_NMFI_MASK U(0x08000000) +#define CP15_CONTROL_TRE_MASK U(0x10000000) +#define CP15_CONTROL_AFE_MASK U(0x20000000) +#define CP15_CONTROL_TE_MASK U(0x40000000) + +/* + * CP15 Auxiliary Control register + */ +#define CP15_CONTROL_SMP_MASK U(0x00000040) +#define CP15_CONTROL_EXCL_MASK U(0x00000080) + +/* + * CP15 Non secure access control register + */ +#define CP15_NSAC_TL_MASK U(0x10000) +#define CP15_NSAC_CL_MASK U(0x20000) +#define CP15_NSAC_CPN_MASK U(0x3FFF) + +/* + * CP15 Cache register + */ +#define CP15_CACHE_ADDR_R_BIT U(12) +#define CP15_CACHE_ADDR_L_BIT (U(32) - CP15_CACHE_ADDR_R_BIT) +#define CP15_CACHE_RESULT_MASK U(0x00000001) + +/* + * CP15 TCM register + * + * ITCM configuration (4kbytes, @0x20100000, enabled) + * DTCM configuration (4kbytes, @0x20101000, enabled) + */ +#define CP15_TCM_ENABLE_MASK U(0x00000001) +#define CP15_TCM_INSTR_TCM U(0x2010000C) +#define CP15_TCM_DATA_TCM U(0x2010100C) + +/* + * CP15 cache lockdown register + * + * ITCM configuration (4kbytes, @0x20100000, enabled) + * DTCM configuration (4kbytes, @0x20101000, enabled) + */ +#define CP15_CACHE_LOCK_ALLWAYS_MASK U(0x0000000F) + +/* + * CP15 cache cleaning constant definition + */ +/* start of line number field offset in way/index format */ +#define LINE_FIELD_OFFSET U(5) +/* Warning: this assumes a 256 lines/way cache (32kB cache) */ +#define LINE_FIELD_OVERFLOW U(13) +/* start of way number field offset in way/index format */ +#define WAY_FIELD_OFFSET U(30) diff --git a/optee_os/core/arch/arm/include/kernel/tz_ssvce_def.h b/optee_os/core/arch/arm/include/kernel/tz_ssvce_def.h new file mode 100644 index 0000000..25215b5 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/tz_ssvce_def.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TZ_SSVCE_DEF_H +#define TZ_SSVCE_DEF_H + +#include +#include + +/* + * ARMv7 Secure Services library + */ + +#define CPSR_OFFSET U(0x00) +#define STACK_INT_USAGE U(0x04) + +/* + * tee service IDs (TODO: align with the service ID list). + * Set by NSec in R4 before SMC to request a TEE service. + */ +#define SSAPI_RET_FROM_INT_SERV U(4) +#define SSAPI_RET_FROM_RPC_SERV U(5) + +/* + * TEE monitor: status returned by the routine that checks the entry + * reason (valid Service ID / secure context). + */ +#define SEC_INVALID_ENTRY U(0) +#define SEC_PRE_INIT_ENTRY U(1) +#define SEC_RET_FROM_INT_ENTRY U(2) +#define SEC_RET_FROM_RPC_ENTRY U(3) +#define SEC_NORMAL_ENTRY U(4) + +/* + * teecore exit reason. + * Set by Secure in R4 before SMC to request a switch to NSec. + */ +#define SEC_EXIT_NORMAL U(1) +#define SEC_EXIT_START_EXT_CODE U(2) +#define SEC_EXIT_INT U(3) +#define SEC_EXIT_RPC_CALL U(4) +#define SEC_EXIT_FIRST U(5) +#define SEC_EXIT_DEEP_SLEEP U(6) + +/* misc */ + +#define SEC_UNDEF_STACK_OFFSET U(4) +#define SEC_ABORT_STACK_OFFSET U(12) + +#define SEC_ENTRY_STATUS_NOK U(0) +#define SEC_ENTRY_STATUS_OK U(1) + +/* + * Outer cache iomem + */ +#define PL310_LINE_SIZE U(32) +#define PL310_8_WAYS U(8) + +#define PL310_CACHE_ID 0x0 +/* reg1 */ +#define PL310_CTRL U(0x100) +#define PL310_AUX_CTRL U(0x104) +#define PL310_TAG_RAM_CTRL U(0x108) +#define PL310_DATA_RAM_CTRL U(0x10C) +/* reg7 */ +#define PL310_SYNC U(0x730) +#define PL310_INV_BY_WAY U(0x77C) +#define PL310_CLEAN_BY_WAY U(0x7BC) +#define PL310_FLUSH_BY_WAY U(0x7FC) +#define PL310_INV_BY_PA U(0x770) +#define PL310_CLEAN_BY_PA U(0x7B0) +#define PL310_FLUSH_BY_PA U(0x7F0) +#define PL310_FLUSH_BY_INDEXWAY U(0x7F8) +/* reg9 */ +#define PL310_DCACHE_LOCKDOWN_BASE U(0x900) +#define PL310_ICACHE_LOCKDOWN_BASE U(0x904) +/* reg12 */ +#define PL310_ADDR_FILT_START U(0xC00) +#define PL310_ADDR_FILT_END U(0xC04) +/* reg15 */ +#define PL310_DEBUG_CTRL U(0xF40) +#define PL310_PREFETCH_CTRL U(0xF60) +#define PL310_POWER_CTRL U(0xF80) + +#define PL310_CTRL_ENABLE_BIT BIT32(0) +#define PL310_AUX_16WAY_BIT BIT32(16) + +#define PL310_CACHE_ID_PART_MASK GENMASK_32(9, 6) +#define PL310_CACHE_ID_PART_L310 0xC0 +#define PL310_CACHE_ID_RTL_MASK GENMASK_32(5, 0) +#define PL310_CACHE_ID_RTL_R3P2 0x8 + +/* + * SCU iomem + */ +#define SCU_CTRL U(0x00) +#define SCU_CONFIG U(0x04) +#define SCU_POWER U(0x08) +#define SCU_INV_SEC U(0x0C) +#define SCU_FILT_SA U(0x40) +#define SCU_FILT_EA U(0x44) +#define SCU_SAC U(0x50) +#define SCU_NSAC U(0x54) +#define SCU_SIZE U(0x58) +#define SCU_ERRATA744369 U(0x30) + +#define SCU_ACCESS_CONTROL_CPU0 BIT32(0) +#define SCU_ACCESS_CONTROL_CPU1 BIT32(1) +#define SCU_ACCESS_CONTROL_CPU2 BIT32(2) +#define SCU_ACCESS_CONTROL_CPU3 BIT32(3) +#define SCU_NSAC_SCU_SHIFT U(0) +#define SCU_NSAC_PTIMER_SHIFT U(4) +#define SCU_NSAC_GTIMER_SHIFT U(8) + +/* + * GIC iomem + */ +#define GIC_DIST_ISR0 U(0x080) +#define GIC_DIST_ISR1 U(0x084) +#define GIC_DIST_IPRIO U(0x400) + +/* + * CPU iomem + */ +#define CORE_ICC_ICCPMR U(0x0004) + +#endif /* TZ_SSVCE_DEF_H */ diff --git a/optee_os/core/arch/arm/include/kernel/tz_ssvce_pl310.h b/optee_os/core/arch/arm/include/kernel/tz_ssvce_pl310.h new file mode 100644 index 0000000..2bcd26e --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/tz_ssvce_pl310.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TZ_SSVCE_PL310_H +#define TZ_SSVCE_PL310_H + +#include +#include +#include + +vaddr_t pl310_base(void); +/* + * End address is included in the range (last address in range) + */ +void arm_cl2_cleaninvbyway(vaddr_t pl310_base); +void arm_cl2_invbyway(vaddr_t pl310_base); +void arm_cl2_cleanbyway(vaddr_t pl310_base); +void arm_cl2_cleanbypa(vaddr_t pl310_base, paddr_t start, paddr_t end); +void arm_cl2_invbypa(vaddr_t pl310_base, paddr_t start, paddr_t end); +void arm_cl2_cleaninvbypa(vaddr_t pl310_base, paddr_t start, paddr_t end); + +#endif /* TZ_SSVCE_PL310_H */ diff --git a/optee_os/core/arch/arm/include/kernel/user_access_arch.h b/optee_os/core/arch/arm/include/kernel/user_access_arch.h new file mode 100644 index 0000000..5dea613 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/user_access_arch.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023, Amazon.com Inc. or its affiliates. All rights Reserved. + */ + +#ifndef __KERNEL_USER_ACCESS_ARCH_H +#define __KERNEL_USER_ACCESS_ARCH_H + +#include + +#ifdef CFG_PAN +/* Enter a section where user mode access is temporarily enabled. */ +static inline void enter_user_access(void) +{ + write_pan_disable(); +} + +/* Exit from the section where user mode access was temporarily enabled. */ +static inline void exit_user_access(void) +{ + write_pan_enable(); +} +#else +static inline void enter_user_access(void) {} +static inline void exit_user_access(void) {} +#endif /* CFG_PAN */ + +#endif /* __KERNEL_USER_ACCESS_ARCH_H */ diff --git a/optee_os/core/arch/arm/include/kernel/vfp.h b/optee_os/core/arch/arm/include/kernel/vfp.h new file mode 100644 index 0000000..49c0d73 --- /dev/null +++ b/optee_os/core/arch/arm/include/kernel/vfp.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef KERNEL_VFP_H +#define KERNEL_VFP_H + +#include +#include + +#ifdef ARM32 +/* + * Advanced SIMD/floating point state on ARMv7-A or ARMv8-A AArch32 has: + * - 32 64-bit data registers + * - FPSCR (32 bits) + * - FPEXC (32 bits) + */ + +#define VFP_NUM_REGS U(32) + +struct vfp_reg { + uint64_t v; +}; + +struct vfp_state { + uint32_t fpexc; + uint32_t fpscr; + struct vfp_reg reg[VFP_NUM_REGS]; +}; +#endif + +#ifdef ARM64 +/* + * Advanced SIMD/floating point state on ARMv8-A AArch64 has: + * - 32 128-bit data registers + * - FPSR (32 bits) + * - FPCR (32 bits) + * - CPACR_EL1.FPEN (2 bits) + */ + +#define VFP_NUM_REGS U(32) + +struct vfp_reg { + uint8_t v[16]; +} __aligned(16); + +struct vfp_state { + struct vfp_reg reg[VFP_NUM_REGS]; + uint32_t fpsr; + uint32_t fpcr; + uint32_t cpacr_el1; +}; +#endif + +#ifdef CFG_WITH_VFP +/* vfp_is_enabled() - Returns true if VFP is enabled */ +bool vfp_is_enabled(void); + +/* vfp_enable() - Enables vfp */ +void vfp_enable(void); + +/* vfp_disable() - Disables vfp */ +void vfp_disable(void); +#else +static inline bool vfp_is_enabled(void) +{ + return false; +} + +static inline void vfp_enable(void) +{ +} + +static inline void vfp_disable(void) +{ +} +#endif + +/* + * vfp_lazy_save_state_init() - Saves VFP enable status and disables VFP + * @state: VFP state structure to initialize + */ +void vfp_lazy_save_state_init(struct vfp_state *state); + +/* + * vfp_lazy_save_state_final() - Saves rest of VFP state + * @state: VFP state to save to + * @force_save: Forces saving of state regardless of previous state if true. + * + * If VFP was enabled when vfp_lazy_save_state_init() was called or + * @force_save is true: save rest of state and disable VFP. Otherwise, do + * nothing. + */ +void vfp_lazy_save_state_final(struct vfp_state *state, bool force_save); + +/* + * vfp_lazy_restore_state() - Lazy restore VFP state + * @state: VFP state to restore + * + * Restores VFP enable status and also restores rest of VFP state if + * vfp_lazy_save_state_final() was called on this state. + */ +void vfp_lazy_restore_state(struct vfp_state *state, bool full_state); + +#endif /*KERNEL_VFP_H*/ diff --git a/optee_os/core/arch/arm/include/mm/core_mmu_arch.h b/optee_os/core/arch/arm/include/mm/core_mmu_arch.h new file mode 100644 index 0000000..42f382f --- /dev/null +++ b/optee_os/core/arch/arm/include/mm/core_mmu_arch.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __CORE_MMU_ARCH_H +#define __CORE_MMU_ARCH_H + +#ifndef __ASSEMBLER__ +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include + +/* + * Platforms can define TRUSTED_{S,D}RAM_* or TZ{S,D}RAM_*. We're helping + * here with the transition to TRUSTED_{S,D}RAM_* by defining these if + * missing based on the legacy defines. + */ +#ifdef TZSRAM_BASE +#ifdef TRUSTED_SRAM_BASE +#error TRUSTED_SRAM_BASE is already defined +#endif +#define TRUSTED_SRAM_BASE TZSRAM_BASE +#define TRUSTED_SRAM_SIZE TZSRAM_SIZE +#endif + +#ifdef TZDRAM_BASE +#ifdef TRUSTED_DRAM_BASE +#error TRUSTED_DRAM_BASE is already defined +#endif +#define TRUSTED_DRAM_BASE TZDRAM_BASE +#define TRUSTED_DRAM_SIZE TZDRAM_SIZE +#endif + +#define SMALL_PAGE_SHIFT U(12) + +#ifdef CFG_WITH_LPAE +#define CORE_MMU_PGDIR_SHIFT U(21) +#define CORE_MMU_PGDIR_LEVEL U(3) +#else +#define CORE_MMU_PGDIR_SHIFT U(20) +#define CORE_MMU_PGDIR_LEVEL U(2) +#endif + +#define CORE_MMU_USER_CODE_SHIFT SMALL_PAGE_SHIFT + +#define CORE_MMU_USER_PARAM_SHIFT SMALL_PAGE_SHIFT + +/* + * Level of base table (i.e. first level of page table), + * depending on address space + */ +#if !defined(CFG_WITH_LPAE) || (CFG_LPAE_ADDR_SPACE_BITS < 40) +#define CORE_MMU_BASE_TABLE_SHIFT U(30) +#define CORE_MMU_BASE_TABLE_LEVEL U(1) +#elif (CFG_LPAE_ADDR_SPACE_BITS <= 48) +#define CORE_MMU_BASE_TABLE_SHIFT U(39) +#define CORE_MMU_BASE_TABLE_LEVEL U(0) +#else /* (CFG_LPAE_ADDR_SPACE_BITS > 48) */ +#error "CFG_WITH_LPAE with CFG_LPAE_ADDR_SPACE_BITS > 48 isn't supported!" +#endif + +#ifdef CFG_WITH_LPAE +/* + * CORE_MMU_BASE_TABLE_OFFSET is used when switching to/from reduced kernel + * mapping. The actual value depends on internals in core_mmu_lpae.c which + * we rather not expose here. There's a compile time assertion to check + * that these magic numbers are correct. + */ +#define CORE_MMU_BASE_TABLE_OFFSET \ + (CFG_TEE_CORE_NB_CORE * \ + BIT(CFG_LPAE_ADDR_SPACE_BITS - CORE_MMU_BASE_TABLE_SHIFT) * \ + U(8)) +#endif + +#ifndef __ASSEMBLER__ + +/* + * Assembly code in enable_mmu() depends on the layout of this struct. + */ +struct core_mmu_config { +#if defined(ARM64) + uint64_t tcr_el1; + uint64_t mair_el1; + uint64_t ttbr0_el1_base; + uint64_t ttbr0_core_offset; + uint64_t map_offset; +#elif defined(CFG_WITH_LPAE) + uint32_t ttbcr; + uint32_t mair0; + uint32_t ttbr0_base; + uint32_t ttbr0_core_offset; + uint32_t map_offset; +#else + uint32_t prrr; + uint32_t nmrr; + uint32_t dacr; + uint32_t ttbcr; + uint32_t ttbr; + uint32_t map_offset; +#endif +}; + +#ifdef CFG_WITH_LPAE +/* + * struct core_mmu_user_map - current user mapping register state + * @user_map: physical address of user map translation table + * @asid: ASID for the user map + * + * Note that this struct should be treated as an opaque struct since + * the content depends on descriptor table format. + */ +struct core_mmu_user_map { + uint64_t user_map; + uint32_t asid; +}; +#else +/* + * struct core_mmu_user_map - current user mapping register state + * @ttbr0: content of ttbr0 + * @ctxid: content of contextidr + * + * Note that this struct should be treated as an opaque struct since + * the content depends on descriptor table format. + */ +struct core_mmu_user_map { + uint32_t ttbr0; + uint32_t ctxid; +}; +#endif + +#ifdef CFG_WITH_LPAE +bool core_mmu_user_va_range_is_defined(void); +#else +static inline bool __noprof core_mmu_user_va_range_is_defined(void) +{ + return true; +} +#endif + +/* Cache maintenance operation type */ +enum cache_op { + DCACHE_CLEAN, + DCACHE_AREA_CLEAN, + DCACHE_INVALIDATE, + DCACHE_AREA_INVALIDATE, + ICACHE_INVALIDATE, + ICACHE_AREA_INVALIDATE, + DCACHE_CLEAN_INV, + DCACHE_AREA_CLEAN_INV, +}; + +/* L1/L2 cache maintenance */ +TEE_Result cache_op_inner(enum cache_op op, void *va, size_t len); +#ifdef CFG_PL310 +TEE_Result cache_op_outer(enum cache_op op, paddr_t pa, size_t len); +#else +static inline TEE_Result cache_op_outer(enum cache_op op __unused, + paddr_t pa __unused, + size_t len __unused) +{ + /* Nothing to do about L2 Cache Maintenance when no PL310 */ + return TEE_SUCCESS; +} +#endif + +/* Do section mapping, not support on LPAE */ +void map_memarea_sections(const struct tee_mmap_region *mm, uint32_t *ttb); + +static inline bool core_mmu_check_max_pa(paddr_t pa __maybe_unused) +{ +#if defined(ARM64) + return pa <= (BIT64(CFG_CORE_ARM64_PA_BITS) - 1); +#elif defined(CFG_CORE_LARGE_PHYS_ADDR) + return pa <= (BIT64(40) - 1); +#else + COMPILE_TIME_ASSERT(sizeof(paddr_t) == sizeof(uint32_t)); + return true; +#endif +} + +/* + * Special barrier to make sure all the changes to translation tables are + * visible before returning. + */ +static inline void core_mmu_table_write_barrier(void) +{ + dsb_ishst(); +} + +static inline bool core_mmu_entry_have_security_bit(uint32_t attr) +{ + return !(attr & TEE_MATTR_TABLE) || !IS_ENABLED(CFG_WITH_LPAE); +} + +static inline unsigned int core_mmu_get_va_width(void) +{ + if (IS_ENABLED(ARM64)) { + COMPILE_TIME_ASSERT(CFG_LPAE_ADDR_SPACE_BITS >= 32); + COMPILE_TIME_ASSERT(CFG_LPAE_ADDR_SPACE_BITS <= 48); + return CFG_LPAE_ADDR_SPACE_BITS; + } + return 32; +} + +static inline bool core_mmu_level_in_range(unsigned int level) +{ +#if CORE_MMU_BASE_TABLE_LEVEL == 0 + return level <= CORE_MMU_PGDIR_LEVEL; +#else + return level >= CORE_MMU_BASE_TABLE_LEVEL && + level <= CORE_MMU_PGDIR_LEVEL; +#endif +} + +#endif /*__ASSEMBLER__*/ + +#endif /* CORE_MMU_H */ diff --git a/optee_os/core/arch/arm/include/mm/generic_ram_layout.h b/optee_os/core/arch/arm/include/mm/generic_ram_layout.h new file mode 100644 index 0000000..b3e0ac3 --- /dev/null +++ b/optee_os/core/arch/arm/include/mm/generic_ram_layout.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ + +#ifndef __MM_GENERIC_RAM_LAYOUT_H +#define __MM_GENERIC_RAM_LAYOUT_H + +#include + +/* + * Generic RAM layout configuration directives + * + * Mandatory directives: + * CFG_TZDRAM_START + * CFG_TZDRAM_SIZE + * CFG_SHMEM_START + * CFG_SHMEM_SIZE + * + * Optional directives: + * CFG_TEE_LOAD_ADDR If defined sets TEE_LOAD_ADDR. If not, TEE_LOAD_ADDR + * is set by the platform or defaults to TEE_RAM_START. + * CFG_TEE_RAM_VA_SIZE Some platforms may have specific needs + * + * Optional directives when pager is enabled: + * CFG_TZSRAM_START If no set, emulated at CFG_TZDRAM_START + * CFG_TZSRAM_SIZE Default to CFG_CORE_TZSRAM_EMUL_SIZE + * + * Optional directive when CFG_SECURE_DATA_PATH is enabled: + * CFG_TEE_SDP_MEM_SIZE If CFG_TEE_SDP_MEM_BASE is not defined, SDP test + * memory byte size can be set by CFG_TEE_SDP_MEM_SIZE. + * + * This header file produces the following generic macros upon the mandatory + * and optional configuration directives listed above: + * + * TEE_RAM_START TEE core RAM physical base address + * TEE_RAM_VA_SIZE TEE core virtual memory address range size + * TEE_RAM_PH_SIZE TEE core physical RAM byte size + * TA_RAM_START TA contexts/pagestore RAM physical base address + * TA_RAM_SIZE TA contexts/pagestore RAM byte size + * TEE_SHMEM_START Non-secure static shared memory physical base address + * TEE_SHMEM_SIZE Non-secure static shared memory byte size + * + * TZDRAM_BASE Main/external secure RAM base address + * TZDRAM_SIZE Main/external secure RAM byte size + * TZSRAM_BASE On-chip secure RAM base address, required by pager. + * TZSRAM_SIZE On-chip secure RAM byte size, required by pager. + * + * TEE_LOAD_ADDR Only defined here if CFG_TEE_LOAD_ADDR is defined. + * Otherwise we expect the platform_config.h to define it + * unless which LEE_LOAD_ADDR defaults to TEE_RAM_START. + * + * TEE_RAM_VA_SIZE Set to CFG_TEE_RAM_VA_SIZE or defaults to + * CORE_MMU_PGDIR_SIZE. + * + * TEE_SDP_TEST_MEM_BASE Define if a SDP memory pool is required and none set. + * Always defined in the inner top (high addresses) + * of CFG_TZDRAM_START/_SIZE. + * TEE_SDP_TEST_MEM_SIZE Set to CFG_TEE_SDP_MEM_SIZE or a default size. + * + * ---------------------------------------------------------------------------- + * TEE RAM layout without CFG_WITH_PAGER + *_ + * +----------------------------------+ <-- CFG_TZDRAM_START + * | TEE core secure RAM (TEE_RAM) | + * +----------------------------------+ + * | Trusted Application RAM (TA_RAM) | + * +----------------------------------+ + * | SDP test memory (optional) | + * +----------------------------------+ <-- CFG_TZDRAM_START + CFG_TZDRAM_SIZE + * + * +----------------------------------+ <-- CFG_SHMEM_START + * | Non-secure static SHM | + * +----------------------------------+ <-- CFG_SHMEM_START + CFG_SHMEM_SIZE + * + * ---------------------------------------------------------------------------- + * TEE RAM layout with CFG_WITH_PAGER=y and undefined CFG_TZSRAM_START/_SIZE + * + * +----------------------------------+ <-- CFG_TZDRAM_START + * | TEE core secure RAM (TEE_RAM) | | | CFG_CORE_TZSRAM_EMUL_SIZE + * +----------------------------------+ --|-' + * | reserved (for kasan) | | TEE_RAM_VA_SIZE + * +----------------------------------+ --' + * | TA RAM / Pagestore (TA_RAM) | + * +----------------------------------+ <---- align with CORE_MMU_PGDIR_SIZE + * +----------------------------------+ <-- + * | SDP test memory (optional) | | CFG_TEE_SDP_MEM_SIZE + * +----------------------------------+ <-+ CFG_TZDRAM_START + CFG_TZDRAM_SIZE + * + * +----------------------------------+ <-- CFG_SHMEM_START + * | Non-secure static SHM | | + * +----------------------------------+ v CFG_SHMEM_SIZE + * + * ---------------------------------------------------------------------------- + * TEE RAM layout with CFG_WITH_PAGER=y and define CFG_TZSRAM_START/_SIZE + * + * +----------------------------------+ <-- CFG_TZSRAM_START + * | TEE core secure RAM (TEE_RAM) | | CFG_TZSRAM_SIZE + * +----------------------------------+ --' + * + * +----------------------------------+ <- CFG_TZDRAM_START + * | TA RAM / Pagestore (TA_RAM) | + * |----------------------------------+ <---- align with CORE_MMU_PGDIR_SIZE + * |----------------------------------+ <-- + * | SDP test memory (optional) | | CFG_TEE_SDP_MEM_SIZE + * +----------------------------------+ <-+ CFG_TZDRAM_START + CFG_TZDRAM_SIZE + * + * +----------------------------------+ <-- CFG_SHMEM_START + * | Non-secure static SHM | | + * +----------------------------------+ v CFG_SHMEM_SIZE + */ + +#ifdef CFG_TEE_LOAD_ADDR +#define TEE_LOAD_ADDR CFG_TEE_LOAD_ADDR +#else +/* Platform specific platform_config.h may set TEE_LOAD_ADDR */ +#endif + +#ifdef CFG_TEE_RAM_VA_SIZE +#define TEE_RAM_VA_SIZE CFG_TEE_RAM_VA_SIZE +#else +#define TEE_RAM_VA_SIZE CORE_MMU_PGDIR_SIZE +#endif + +#ifdef CFG_SHMEM_SIZE +#define TEE_SHMEM_SIZE CFG_SHMEM_SIZE +#endif + +#ifdef CFG_SHMEM_START +#define TEE_SHMEM_START CFG_SHMEM_START +#ifndef CFG_SHMEM_SIZE +#error CFG_SHMEM_START mandates CFG_SHMEM_SIZE +#endif +#endif + +#if defined(CFG_TZSRAM_START) +#define TZSRAM_BASE CFG_TZSRAM_START +#define TZSRAM_SIZE CFG_TZSRAM_SIZE +#endif + +#ifdef CFG_TZDRAM_START +#if !defined(CFG_WITH_PAGER) || defined(CFG_TZSRAM_START) +#define TZDRAM_BASE CFG_TZDRAM_START +#define TZDRAM_SIZE CFG_TZDRAM_SIZE +#else +#define TZSRAM_BASE CFG_TZDRAM_START +#define TZSRAM_SIZE CFG_CORE_TZSRAM_EMUL_SIZE +#define TZDRAM_BASE ROUNDUP(TZSRAM_BASE + TZSRAM_SIZE, \ + TEE_RAM_VA_SIZE) +#define TZDRAM_SIZE (CFG_TZDRAM_START + (CFG_TZDRAM_SIZE - \ + TZDRAM_BASE)) +#endif + +#ifdef CFG_WITH_PAGER +#define TEE_RAM_START TZSRAM_BASE +#define TEE_RAM_PH_SIZE TZSRAM_SIZE +#else +#define TEE_RAM_START TZDRAM_BASE +#define TEE_RAM_PH_SIZE TEE_RAM_VA_SIZE +#endif /*CFG_WITH_PAGER*/ +#endif /*CFG_TZDRAM_START*/ + +/* + * Secure data path test memory pool + * - If SDP is disabled, no SDP test memory needed. + * - If SDP is enabled, if CFG_TEE_SDP_MEM_BASE, SDP test pool is not needed. + * - If SDP is enabled and CFG_TEE_SDP_MEM_BASE not defined, a SDP test pool + * is defined at the end of the secure RAM. CFG_TEE_SDP_MEM_SIZE can set + * its size otherwise it defaults to 4MB. + */ +#if !defined(CFG_SECURE_DATA_PATH) || defined(CFG_TEE_SDP_MEM_BASE) +#define TEE_SDP_TEST_MEM_SIZE 0 +#else +#ifdef CFG_TEE_SDP_MEM_SIZE +#define TEE_SDP_TEST_MEM_SIZE CFG_TEE_SDP_MEM_SIZE +#else +#define TEE_SDP_TEST_MEM_SIZE SIZE_4M +#endif +#define TEE_SDP_TEST_MEM_BASE (CFG_TZDRAM_START + (CFG_TZDRAM_SIZE - \ + TEE_SDP_TEST_MEM_SIZE)) +#endif + +#endif /*__MM_GENERIC_RAM_LAYOUT_H*/ + diff --git a/optee_os/core/arch/arm/include/optee_ffa.h b/optee_os/core/arch/arm/include/optee_ffa.h new file mode 100644 index 0000000..8f16039 --- /dev/null +++ b/optee_os/core/arch/arm/include/optee_ffa.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019-2021, Linaro Limited + */ + +/* + * This file is exported by OP-TEE and is kept in sync between secure world + * and normal world drivers. We're using ARM FF-A 1.0 specification. + */ + +#ifndef __OPTEE_FFA_H +#define __OPTEE_FFA_H + +#include + +/* + * Normal world sends requests with FFA_MSG_SEND_DIRECT_REQ and + * responses are returned with FFA_MSG_SEND_DIRECT_RESP for normal + * messages. + * + * All requests with FFA_MSG_SEND_DIRECT_REQ and FFA_MSG_SEND_DIRECT_RESP + * are using the AArch32 SMC calling convention with register usage as + * defined in FF-A specification: + * w0: Function ID (0x8400006F or 0x84000070) + * w1: Source/Destination IDs + * w2: Reserved (MBZ) + * w3-w7: Implementation defined, free to be used below + */ + +#define OPTEE_FFA_VERSION_MAJOR UINT32_C(1) +#define OPTEE_FFA_VERSION_MINOR UINT32_C(0) + +#define OPTEE_FFA_BLOCKING_CALL(id) UINT32_C(id) +#define OPTEE_FFA_YIELDING_CALL_BIT U(31) +#define OPTEE_FFA_YIELDING_CALL(id) (UINT32_C(id) | \ + BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) + +/* + * Returns the API version implemented, currently follows the FF-A version. + * Call register usage: + * w3: Service ID, OPTEE_FFA_GET_API_VERSION + * w4-w7: Not used (MBZ) + * + * Return register usage: + * w3: OPTEE_FFA_VERSION_MAJOR + * w4: OPTEE_FFA_VERSION_MINOR + * w5-w7: Not used (MBZ) + */ +#define OPTEE_FFA_GET_API_VERSION OPTEE_FFA_BLOCKING_CALL(0) + +/* + * Returns the revision of OP-TEE. + * + * Used by non-secure world to figure out which version of the Trusted OS + * is installed. Note that the returned revision is the revision of the + * Trusted OS, not of the API. + * + * Call register usage: + * w3: Service ID, OPTEE_FFA_GET_OS_VERSION + * w4-w7: Unused (MBZ) + * + * Return register usage: + * w3: CFG_OPTEE_REVISION_MAJOR + * w4: CFG_OPTEE_REVISION_MINOR + * w5: TEE_IMPL_GIT_SHA1 (or zero if not supported) + */ +#define OPTEE_FFA_GET_OS_VERSION OPTEE_FFA_BLOCKING_CALL(1) + +/* + * Exchange capabilities between normal world and secure world. + * + * Currently there are no defined capabilities. When features are added new + * capabilities may be added. + * + * Call register usage: + * w3: Service ID, OPTEE_FFA_EXCHANGE_CAPABILITIES + * w4-w7: Note used (MBZ) + * + * Return register usage: + * w3: Error code, 0 on success + * w4: Bit[7:0]: Number of parameters needed for RPC to be supplied + * as the second MSG arg struct for + * OPTEE_FFA_YIELDING_CALL_WITH_ARG. + * Bit[31:8]: Reserved (MBZ) + * w5: Bitfield of secure world capabilities OPTEE_FFA_SEC_CAP_* below, + * unused bits MBZ. + * w6-w7: Not used (MBZ) + */ +/* + * Secure world supports using an offset into the argument shared memory + * object, see also OPTEE_FFA_YIELDING_CALL_WITH_ARG + */ +#define OPTEE_FFA_SEC_CAP_ARG_OFFSET BIT(0) + +#define OPTEE_FFA_EXCHANGE_CAPABILITIES OPTEE_FFA_BLOCKING_CALL(2) + +/* + * Unregister shared memory + * + * Call register usage: + * w3: Service ID, OPTEE_FFA_YIELDING_CALL_UNREGISTER_SHM + * w4: Shared memory handle, lower bits + * w5: Shared memory handle, higher bits + * w6-w7: Not used (MBZ) + * + * Return register usage: + * w3: Error code, 0 on success + * w4-w7: Not used (MBZ) + */ +#define OPTEE_FFA_UNREGISTER_SHM OPTEE_FFA_BLOCKING_CALL(3) + +/* + * Call with struct optee_msg_arg as argument in the supplied shared memory + * with a zero internal offset and normal cached memory attributes. + * Register usage: + * w3: Service ID, OPTEE_FFA_YIELDING_CALL_WITH_ARG + * w4: Lower 32 bits of a 64-bit Shared memory handle + * w5: Upper 32 bits of a 64-bit Shared memory handle + * w6: Offset into shared memory pointing to a struct optee_msg_arg + * right after the parameters of this struct (at offset + * OPTEE_MSG_GET_ARG_SIZE(num_params) follows a struct optee_msg_arg + * for RPC, this struct has reserved space for the number of RPC + * parameters as returned by OPTEE_FFA_EXCHANGE_CAPABILITIES. + * MBZ unless the bit OPTEE_FFA_SEC_CAP_ARG_OFFSET is received with + * OPTEE_FFA_EXCHANGE_CAPABILITIES. + * w7: Not used (MBZ) + * Resume from RPC. Register usage: + * w3: Service ID, OPTEE_FFA_YIELDING_CALL_RESUME + * w4-w6: Not used (MBZ) + * w7: Resume info + * + * Normal return (yielding call is completed). Register usage: + * w3: Error code, 0 on success + * w4: OPTEE_FFA_YIELDING_CALL_RETURN_DONE + * w5-w7: Not used (MBZ) + * + * RPC interrupt return (RPC from secure world). Register usage: + * w3: Error code == 0 + * w4: Any defined RPC code but OPTEE_FFA_YIELDING_CALL_RETURN_DONE + * w5-w6: Not used (MBZ) + * w7: Resume info + * + * Possible error codes in register w3: + * 0: Success + * FFA_DENIED: w4 isn't one of OPTEE_FFA_YIELDING_CALL_START + * OPTEE_FFA_YIELDING_CALL_RESUME + * + * Possible error codes for OPTEE_FFA_YIELDING_CALL_START, + * FFA_BUSY: Number of OP-TEE OS threads exceeded, + * try again later + * FFA_DENIED: RPC shared memory object not found + * FFA_INVALID_PARAMETER: Bad shared memory handle or offset into the memory + * + * Possible error codes for OPTEE_FFA_YIELDING_CALL_RESUME + * FFA_INVALID_PARAMETER: Bad resume info + */ +#define OPTEE_FFA_YIELDING_CALL_WITH_ARG OPTEE_FFA_YIELDING_CALL(0) +#define OPTEE_FFA_YIELDING_CALL_RESUME OPTEE_FFA_YIELDING_CALL(1) + +#define OPTEE_FFA_YIELDING_CALL_RETURN_DONE U(0) +#define OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD U(1) +#define OPTEE_FFA_YIELDING_CALL_RETURN_INTERRUPT U(2) + +#endif /*__OPTEE_FFA_H*/ diff --git a/optee_os/core/arch/arm/include/pta_stmm.h b/optee_os/core/arch/arm/include/pta_stmm.h new file mode 100644 index 0000000..891647a --- /dev/null +++ b/optee_os/core/arch/arm/include/pta_stmm.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2020, Linaro Limited + */ +#ifndef __PTA_STMM_H +#define __PTA_STMM_H + +/* + * Interface to the pseudo TA, which provides a communication channel with + * the Standalone MM SP (StMM) running at S-EL0. + */ + +#define PTA_STMM_UUID { 0xed32d533, 0x99e6, 0x4209, {\ + 0x9c, 0xc0, 0x2d, 0x72, 0xcd, 0xd9, 0x98, 0xa7 } } + +/* + * Pass a buffer to Standalone MM SP + * + * [in/out] memref[0]: EFI Communication buffer + * [out] value[1].a: EFI return code + */ +#define PTA_STMM_CMD_COMMUNICATE 0 + +#endif /* __PTA_STMM_H */ + diff --git a/optee_os/core/arch/arm/include/scmi/scmi_server.h b/optee_os/core/arch/arm/include/scmi/scmi_server.h new file mode 100644 index 0000000..a28fd66 --- /dev/null +++ b/optee_os/core/arch/arm/include/scmi/scmi_server.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019-2022, Linaro Limited + */ +#ifndef SCMI_SERVER_H +#define SCMI_SERVER_H + +#include +#include + +#ifdef CFG_SCMI_SCPFW +/* + * Request processing of an incoming event in the SCMI server for a target + * MHU/SMT mailbox. + * + * @channel_id: SCMI channel handler + */ +TEE_Result scmi_server_smt_process_thread(unsigned int channel_id); + +/* + * Request processing of an incoming event in the SCMI server for a target + * MHU/MSG mailbox. + * + * @id: SCMI channel handler + * @in_buf: Input message MSG buffer + * @in_size: Input message MSG buffer size + * @out_buf: Output message MSG buffer + * @out_size: Reference to output message MSG buffer size + */ +TEE_Result scmi_server_msg_process_thread(unsigned int channel_id, void *in_buf, + size_t in_size, void *out_buf, + size_t *out_size); + +/* + * Get SCP-firmware channel device ID from the client channel ID. + * + * @channel_id: SCMI channel handler + * @handle: Output SCP-firmware device ID for the target SCMI mailbox + */ +TEE_Result scmi_server_get_channel(unsigned int channel_id, int *handle); + +/* Get number of channels supported by the SCMI platform/server */ +int scmi_server_get_channels_count(void); + +#else /* CFG_SCMI_SCPFW */ +static inline +TEE_Result scmi_server_smt_process_thread(unsigned int channel_id __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline +TEE_Result scmi_server_msg_process_thread(unsigned int channel_id __unused, + void *in_buf __unused, + size_t in_size __unused, + void *out_buf __unused, + size_t *out_size __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result scmi_server_get_channel(unsigned int id __unused, + int *handle __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline int scmi_server_get_channels_count(void) +{ + return 0; +} +#endif /* CFG_SCMI_SCPFW */ +#endif /* SCMI_SERVER_H */ diff --git a/optee_os/core/arch/arm/include/sm/optee_smc.h b/optee_os/core/arch/arm/include/sm/optee_smc.h new file mode 100644 index 0000000..04a3839 --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/optee_smc.h @@ -0,0 +1,708 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2021, Linaro Limited + */ +#ifndef OPTEE_SMC_H +#define OPTEE_SMC_H + +#include + +/* + * This file is exported by OP-TEE and is in kept in sync between secure + * world and normal world kernel driver. We're following ARM SMC Calling + * Convention as specified in + * http://infocenter.arm.com/help/topic/com.arm.doc.den0028a/index.html + * + * This file depends on optee_msg.h being included to expand the SMC id + * macros below. + */ + +#define OPTEE_SMC_32 U(0) +#define OPTEE_SMC_64 U(0x40000000) +#define OPTEE_SMC_FAST_CALL U(0x80000000) +#define OPTEE_SMC_STD_CALL U(0) + +#define OPTEE_SMC_OWNER_MASK U(0x3F) +#define OPTEE_SMC_OWNER_SHIFT U(24) + +#define OPTEE_SMC_FUNC_MASK U(0xFFFF) + +#define OPTEE_SMC_IS_FAST_CALL(smc_val) ((smc_val) & OPTEE_SMC_FAST_CALL) +#define OPTEE_SMC_IS_64(smc_val) ((smc_val) & OPTEE_SMC_64) +#define OPTEE_SMC_FUNC_NUM(smc_val) ((smc_val) & OPTEE_SMC_FUNC_MASK) +#define OPTEE_SMC_OWNER_NUM(smc_val) \ + (((smc_val) >> OPTEE_SMC_OWNER_SHIFT) & OPTEE_SMC_OWNER_MASK) + +#define OPTEE_SMC_CALL_VAL(type, calling_convention, owner, func_num) \ + ((type) | (calling_convention) | \ + (((owner) & OPTEE_SMC_OWNER_MASK) << \ + OPTEE_SMC_OWNER_SHIFT) |\ + ((func_num) & OPTEE_SMC_FUNC_MASK)) + +#define OPTEE_SMC_STD_CALL_VAL(func_num) \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_STD_CALL, \ + OPTEE_SMC_OWNER_TRUSTED_OS, (func_num)) +#define OPTEE_SMC_FAST_CALL_VAL(func_num) \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_FAST_CALL, \ + OPTEE_SMC_OWNER_TRUSTED_OS, (func_num)) + +#define OPTEE_SMC_OWNER_ARCH U(0) +#define OPTEE_SMC_OWNER_CPU U(1) +#define OPTEE_SMC_OWNER_SIP U(2) +#define OPTEE_SMC_OWNER_OEM U(3) +#define OPTEE_SMC_OWNER_STANDARD U(4) +#define OPTEE_SMC_OWNER_TRUSTED_APP U(48) +#define OPTEE_SMC_OWNER_TRUSTED_OS U(50) + +#define OPTEE_SMC_OWNER_TRUSTED_OS_OPTEED U(62) +#define OPTEE_SMC_OWNER_TRUSTED_OS_API U(63) + +/* + * Function specified by SMC Calling convention. + */ +#define OPTEE_SMC_FUNCID_CALLS_COUNT U(0xFF00) +#define OPTEE_SMC_CALLS_COUNT \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_FAST_CALL, \ + OPTEE_SMC_OWNER_TRUSTED_OS_API, \ + OPTEE_SMC_FUNCID_CALLS_COUNT) + +/* + * Normal cached memory (write-back), shareable for SMP systems and not + * shareable for UP systems. + */ +#define OPTEE_SMC_SHM_CACHED U(1) + +/* + * a0..a7 is used as register names in the descriptions below, on arm32 + * that translates to r0..r7 and on arm64 to w0..w7. In both cases it's + * 32-bit registers. + */ + +/* + * Function specified by SMC Calling convention + * + * Return the following UID if using API specified in this file + * without further extensions: + * 384fb3e0-e7f8-11e3-af63-0002a5d5c51b. + * see also OPTEE_MSG_UID_* in optee_msg.h + */ +#define OPTEE_SMC_FUNCID_CALLS_UID OPTEE_MSG_FUNCID_CALLS_UID +#define OPTEE_SMC_CALLS_UID \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_FAST_CALL, \ + OPTEE_SMC_OWNER_TRUSTED_OS_API, \ + OPTEE_SMC_FUNCID_CALLS_UID) + +/* + * Function specified by SMC Calling convention + * + * Returns 2.0 if using API specified in this file without further extensions. + * see also OPTEE_MSG_REVISION_* in optee_msg.h + */ +#define OPTEE_SMC_FUNCID_CALLS_REVISION OPTEE_MSG_FUNCID_CALLS_REVISION +#define OPTEE_SMC_CALLS_REVISION \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_FAST_CALL, \ + OPTEE_SMC_OWNER_TRUSTED_OS_API, \ + OPTEE_SMC_FUNCID_CALLS_REVISION) + +/* + * Get UUID of Trusted OS. + * + * Used by non-secure world to figure out which Trusted OS is installed. + * Note that returned UUID is the UUID of the Trusted OS, not of the API. + * + * Returns UUID in a0-4 in the same way as OPTEE_SMC_CALLS_UID + * described above. + */ +#define OPTEE_SMC_FUNCID_GET_OS_UUID OPTEE_MSG_FUNCID_GET_OS_UUID +#define OPTEE_SMC_CALL_GET_OS_UUID \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_OS_UUID) + +/* + * Get revision of Trusted OS. + * + * Used by non-secure world to figure out which version of the Trusted OS + * is installed. Note that the returned revision is the revision of the + * Trusted OS, not of the API. + * + * Returns revision in a0-1 in the same way as OPTEE_SMC_CALLS_REVISION + * described above. May optionally return a 32-bit build identifier in a2, + * with zero meaning unspecified. + */ +#define OPTEE_SMC_FUNCID_GET_OS_REVISION OPTEE_MSG_FUNCID_GET_OS_REVISION +#define OPTEE_SMC_CALL_GET_OS_REVISION \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_OS_REVISION) + +/* + * Call with struct optee_msg_arg as argument + * + * When called with OPTEE_SMC_CALL_WITH_RPC_ARG or + * OPTEE_SMC_CALL_WITH_REGD_ARG in a0 there is one RPC struct optee_msg_arg + * following after the first struct optee_msg_arg. The RPC struct + * optee_msg_arg has reserved space for the number of RPC parameters as + * returned by OPTEE_SMC_EXCHANGE_CAPABILITIES. + * + * When calling these functions normal world has a few responsibilities: + * 1. It must be able to handle eventual RPCs + * 2. Non-secure interrupts should not be masked + * 3. If asynchronous notifications has been negotiated successfully, then + * the interrupt for asynchronous notifications should be unmasked + * during this call. + * + * Call register usage, OPTEE_SMC_CALL_WITH_ARG and + * OPTEE_SMC_CALL_WITH_RPC_ARG: + * a0 SMC Function ID, OPTEE_SMC_CALL_WITH_ARG or OPTEE_SMC_CALL_WITH_RPC_ARG + * a1 Upper 32 bits of a 64-bit physical pointer to a struct optee_msg_arg + * a2 Lower 32 bits of a 64-bit physical pointer to a struct optee_msg_arg + * a3 Cache settings, not used if physical pointer is in a predefined shared + * memory area else per OPTEE_SMC_SHM_* + * a4-6 Not used + * a7 Hypervisor Client ID register + * + * Call register usage, OPTEE_SMC_CALL_WITH_REGD_ARG: + * a0 SMC Function ID, OPTEE_SMC_CALL_WITH_REGD_ARG + * a1 Upper 32 bits of a 64-bit shared memory cookie + * a2 Lower 32 bits of a 64-bit shared memory cookie + * a3 Offset of the struct optee_msg_arg in the shared memory with the + * supplied cookie + * a4-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 Return value, OPTEE_SMC_RETURN_* + * a1-3 Not used + * a4-7 Preserved + * + * OPTEE_SMC_RETURN_ETHREAD_LIMIT return register usage: + * a0 Return value, OPTEE_SMC_RETURN_ETHREAD_LIMIT + * a1-3 Preserved + * a4-7 Preserved + * + * RPC return register usage: + * a0 Return value, OPTEE_SMC_RETURN_IS_RPC(val) + * a1-2 RPC parameters + * a3-7 Resume information, must be preserved + * + * Possible return values: + * OPTEE_SMC_RETURN_UNKNOWN_FUNCTION Trusted OS does not recognize this + * function. + * OPTEE_SMC_RETURN_OK Call completed, result updated in + * the previously supplied struct + * optee_msg_arg. + * OPTEE_SMC_RETURN_ETHREAD_LIMIT Number of Trusted OS threads exceeded, + * try again later. + * OPTEE_SMC_RETURN_EBADADDR Bad physical pointer to struct + * optee_msg_arg. + * OPTEE_SMC_RETURN_EBADCMD Bad/unknown cmd in struct optee_msg_arg + * OPTEE_SMC_RETURN_IS_RPC() Call suspended by RPC call to normal + * world. + */ +#define OPTEE_SMC_FUNCID_CALL_WITH_ARG OPTEE_MSG_FUNCID_CALL_WITH_ARG +#define OPTEE_SMC_CALL_WITH_ARG \ + OPTEE_SMC_STD_CALL_VAL(OPTEE_SMC_FUNCID_CALL_WITH_ARG) +#define OPTEE_SMC_CALL_WITH_RPC_ARG \ + OPTEE_SMC_STD_CALL_VAL(OPTEE_SMC_FUNCID_CALL_WITH_RPC_ARG) +#define OPTEE_SMC_CALL_WITH_REGD_ARG \ + OPTEE_SMC_STD_CALL_VAL(OPTEE_SMC_FUNCID_CALL_WITH_REGD_ARG) + +/* + * Get Shared Memory Config + * + * Returns the Secure/Non-secure shared memory config. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_GET_SHM_CONFIG + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Have config return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 Physical address of start of SHM + * a2 Size of of SHM + * a3 Cache settings of memory, as defined by the + * OPTEE_SMC_SHM_* values above + * a4-7 Preserved + * + * Not available register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL + * a1-3 Not used + * a4-7 Preserved + */ +#define OPTEE_SMC_FUNCID_GET_SHM_CONFIG 7 +#define OPTEE_SMC_GET_SHM_CONFIG \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_SHM_CONFIG) + +/* + * Configures L2CC mutex + * + * Disables, enables usage of L2CC mutex. Returns or sets physical address + * of L2CC mutex. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_L2CC_MUTEX + * a1 OPTEE_SMC_L2CC_MUTEX_GET_ADDR Get physical address of mutex + * OPTEE_SMC_L2CC_MUTEX_SET_ADDR Set physical address of mutex + * OPTEE_SMC_L2CC_MUTEX_ENABLE Enable usage of mutex + * OPTEE_SMC_L2CC_MUTEX_DISABLE Disable usage of mutex + * a2 if a1 == OPTEE_SMC_L2CC_MUTEX_SET_ADDR, upper 32bit of a 64bit + * physical address of mutex + * a3 if a1 == OPTEE_SMC_L2CC_MUTEX_SET_ADDR, lower 32bit of a 64bit + * physical address of mutex + * a3-6 Not used + * a7 Hypervisor Client ID register + * + * Have config return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 Preserved + * a2 if a1 == OPTEE_SMC_L2CC_MUTEX_GET_ADDR, upper 32bit of a 64bit + * physical address of mutex + * a3 if a1 == OPTEE_SMC_L2CC_MUTEX_GET_ADDR, lower 32bit of a 64bit + * physical address of mutex + * a3-7 Preserved + * + * Error return register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL Physical address not available + * OPTEE_SMC_RETURN_EBADADDR Bad supplied physical address + * OPTEE_SMC_RETURN_EBADCMD Unsupported value in a1 + * a1-7 Preserved + */ +#define OPTEE_SMC_L2CC_MUTEX_GET_ADDR U(0) +#define OPTEE_SMC_L2CC_MUTEX_SET_ADDR U(1) +#define OPTEE_SMC_L2CC_MUTEX_ENABLE U(2) +#define OPTEE_SMC_L2CC_MUTEX_DISABLE U(3) +#define OPTEE_SMC_FUNCID_L2CC_MUTEX U(8) +#define OPTEE_SMC_L2CC_MUTEX \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_L2CC_MUTEX) + +/* + * Exchanges capabilities between normal world and secure world + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_EXCHANGE_CAPABILITIES + * a1 bitfield of normal world capabilities OPTEE_SMC_NSEC_CAP_* + * a2-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 bitfield of secure world capabilities OPTEE_SMC_SEC_CAP_* + * a2 The maximum secure world notification number + * a3 Bit[7:0]: Number of parameters needed for RPC to be supplied + * as the second MSG arg struct for + * OPTEE_SMC_CALL_WITH_ARG + * Bit[31:8]: Reserved (MBZ) + * a3-7 Preserved + * + * Error return register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL, can't use the capabilities from normal world + * a1 bitfield of secure world capabilities OPTEE_SMC_SEC_CAP_* + * a2-7 Preserved + */ +/* Normal world works as a uniprocessor system */ +#define OPTEE_SMC_NSEC_CAP_UNIPROCESSOR BIT(0) +/* Secure world has reserved shared memory for normal world to use */ +#define OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM BIT(0) +/* Secure world can communicate via previously unregistered shared memory */ +#define OPTEE_SMC_SEC_CAP_UNREGISTERED_SHM BIT(1) +/* + * Secure world supports commands "register/unregister shared memory", + * secure world accepts command buffers located in any parts of non-secure RAM + */ +#define OPTEE_SMC_SEC_CAP_DYNAMIC_SHM BIT(2) +/* Secure world is built with virtualization support */ +#define OPTEE_SMC_SEC_CAP_VIRTUALIZATION BIT(3) +/* Secure world supports Shared Memory with a NULL reference */ +#define OPTEE_SMC_SEC_CAP_MEMREF_NULL BIT(4) +/* Secure world supports asynchronous notification of normal world */ +#define OPTEE_SMC_SEC_CAP_ASYNC_NOTIF BIT(5) +/* Secure world supports pre-allocating RPC arg struct */ +#define OPTEE_SMC_SEC_CAP_RPC_ARG BIT(6) + +#define OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES U(9) +#define OPTEE_SMC_EXCHANGE_CAPABILITIES \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES) + +/* + * Disable and empties cache of shared memory objects + * + * Secure world can cache frequently used shared memory objects, for + * example objects used as RPC arguments. When secure world is idle this + * function returns one shared memory reference to free. To disable the + * cache and free all cached objects this function has to be called until + * it returns OPTEE_SMC_RETURN_ENOTAVAIL. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_DISABLE_SHM_CACHE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 Upper 32 bits of a 64-bit Shared memory cookie + * a2 Lower 32 bits of a 64-bit Shared memory cookie + * a3-7 Preserved + * + * Cache empty return register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_SMC_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_SMC_FUNCID_DISABLE_SHM_CACHE U(10) +#define OPTEE_SMC_DISABLE_SHM_CACHE \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_DISABLE_SHM_CACHE) + +/* + * Enable cache of shared memory objects + * + * Secure world can cache frequently used shared memory objects, for + * example objects used as RPC arguments. When secure world is idle this + * function returns OPTEE_SMC_RETURN_OK and the cache is enabled. If + * secure world isn't idle OPTEE_SMC_RETURN_EBUSY is returned. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_ENABLE_SHM_CACHE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_SMC_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_SMC_FUNCID_ENABLE_SHM_CACHE U(11) +#define OPTEE_SMC_ENABLE_SHM_CACHE \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_ENABLE_SHM_CACHE) + +/* + * Release of secondary cores + * + * OP-TEE in secure world is in charge of the release process of secondary + * cores. The Rich OS issue the this request to ask OP-TEE to boot up the + * secondary cores, go through the OP-TEE per-core initialization, and then + * switch to the Non-seCure world with the Rich OS provided entry address. + * The secondary cores enter Non-Secure world in SVC mode, with Thumb, FIQ, + * IRQ and Abort bits disabled. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_BOOT_SECONDARY + * a1 Index of secondary core to boot + * a2 Upper 32 bits of a 64-bit Non-Secure world entry physical address + * a3 Lower 32 bits of a 64-bit Non-Secure world entry physical address + * a4-7 Not used + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1-7 Preserved + * + * Error return: + * a0 OPTEE_SMC_RETURN_EBADCMD Core index out of range + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_SMC_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_SMC_FUNCID_BOOT_SECONDARY U(12) +#define OPTEE_SMC_BOOT_SECONDARY \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_BOOT_SECONDARY) + +/* + * Inform OP-TEE about a new virtual machine + * + * Hypervisor issues this call during virtual machine (guest) creation. + * OP-TEE records client id of new virtual machine and prepares + * to receive requests from it. This call is available only if OP-TEE + * was built with virtualization support. + * + * Call requests usage: + * a0 SMC Function ID, OPTEE_SMC_VM_CREATED + * a1 Hypervisor Client ID of newly created virtual machine + * a2-6 Not used + * a7 Hypervisor Client ID register. Must be 0, because only hypervisor + * can issue this call + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1-7 Preserved + * + * Error return: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL OP-TEE have no resources for + * another VM + * a1-7 Preserved + * + */ +#define OPTEE_SMC_FUNCID_VM_CREATED U(13) +#define OPTEE_SMC_VM_CREATED \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_VM_CREATED) + +/* + * Inform OP-TEE about shutdown of a virtual machine + * + * Hypervisor issues this call during virtual machine (guest) destruction. + * OP-TEE will clean up all resources associated with this VM. This call is + * available only if OP-TEE was built with virtualization support. + * + * Call requests usage: + * a0 SMC Function ID, OPTEE_SMC_VM_DESTROYED + * a1 Hypervisor Client ID of virtual machine being shut down + * a2-6 Not used + * a7 Hypervisor Client ID register. Must be 0, because only hypervisor + * can issue this call + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1-7 Preserved + * + */ +#define OPTEE_SMC_FUNCID_VM_DESTROYED U(14) +#define OPTEE_SMC_VM_DESTROYED \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_VM_DESTROYED) + +/* + * Query OP-TEE about number of supported threads + * + * Normal World OS or Hypervisor issues this call to find out how many + * threads OP-TEE supports. That is how many standard calls can be issued + * in parallel before OP-TEE will return OPTEE_SMC_RETURN_ETHREAD_LIMIT. + * + * Call requests usage: + * a0 SMC Function ID, OPTEE_SMC_GET_THREAD_COUNT + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 Number of threads + * a2-7 Preserved + * + * Error return: + * a0 OPTEE_SMC_RETURN_UNKNOWN_FUNCTION Requested call is not implemented + * a1-7 Preserved + */ +#define OPTEE_SMC_FUNCID_GET_THREAD_COUNT U(15) +#define OPTEE_SMC_GET_THREAD_COUNT \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_THREAD_COUNT) + +/* + * Inform OP-TEE that normal world is able to receive asynchronous + * notifications. + * + * Call requests usage: + * a0 SMC Function ID, OPTEE_SMC_ENABLE_ASYNC_NOTIF + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1-7 Preserved + * + * Not supported return register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL + * a1-7 Preserved + */ +#define OPTEE_SMC_FUNCID_ENABLE_ASYNC_NOTIF 16 +#define OPTEE_SMC_ENABLE_ASYNC_NOTIF \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_ENABLE_ASYNC_NOTIF) + +/* + * Retrieve a value of notifications pending since the last call of this + * function. + * + * OP-TEE keeps a record of all posted values. When an interrupt is + * received which indicates that there are posted values this function + * should be called until all pended values have been retrieved. When a + * value is retrieved, it's cleared from the record in secure world. + * + * It is expected that this function is called from an interrupt handler + * in normal world. + * + * Call requests usage: + * a0 SMC Function ID, OPTEE_SMC_GET_ASYNC_NOTIF_VALUE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 value + * a2 Bit[0]: OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID if the value in a1 is + * valid, else 0 if no values were pending + * a2 Bit[1]: OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING if another value is + * pending, else 0. + * Bit[31:2]: MBZ + * a3-7 Preserved + * + * Not supported return register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL + * a1-7 Preserved + */ +#define OPTEE_SMC_ASYNC_NOTIF_VALID BIT(0) +#define OPTEE_SMC_ASYNC_NOTIF_PENDING BIT(1) + +/* + * Notification that OP-TEE expects a yielding call to do some bottom half + * work in a driver. + */ +#define OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF 0 + +#define OPTEE_SMC_FUNCID_GET_ASYNC_NOTIF_VALUE 17 +#define OPTEE_SMC_GET_ASYNC_NOTIF_VALUE \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_ASYNC_NOTIF_VALUE) + +/* See OPTEE_SMC_CALL_WITH_RPC_ARG above */ +#define OPTEE_SMC_FUNCID_CALL_WITH_RPC_ARG U(18) + +/* See OPTEE_SMC_CALL_WITH_REGD_ARG above */ +#define OPTEE_SMC_FUNCID_CALL_WITH_REGD_ARG U(19) + +/* + * Resume from RPC (for example after processing a foreign interrupt) + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC + * a1-3 Value of a1-3 when OPTEE_SMC_CALL_WITH_ARG returned + * OPTEE_SMC_RETURN_RPC in a0 + * + * Return register usage is the same as for OPTEE_SMC_*CALL_WITH_ARG above. + * + * Possible return values + * OPTEE_SMC_RETURN_UNKNOWN_FUNCTION Trusted OS does not recognize this + * function. + * OPTEE_SMC_RETURN_OK Original call completed, result + * updated in the previously supplied. + * struct optee_msg_arg + * OPTEE_SMC_RETURN_RPC Call suspended by RPC call to normal + * world. + * OPTEE_SMC_RETURN_ERESUME Resume failed, the opaque resume + * information was corrupt. + */ +#define OPTEE_SMC_FUNCID_RETURN_FROM_RPC U(3) +#define OPTEE_SMC_CALL_RETURN_FROM_RPC \ + OPTEE_SMC_STD_CALL_VAL(OPTEE_SMC_FUNCID_RETURN_FROM_RPC) + +#define OPTEE_SMC_RETURN_RPC_PREFIX_MASK U(0xFFFF0000) +#define OPTEE_SMC_RETURN_RPC_PREFIX U(0xFFFF0000) +#define OPTEE_SMC_RETURN_RPC_FUNC_MASK U(0x0000FFFF) + +#define OPTEE_SMC_RETURN_GET_RPC_FUNC(ret) \ + ((ret) & OPTEE_SMC_RETURN_RPC_FUNC_MASK) + +#define OPTEE_SMC_RPC_VAL(func) ((func) | OPTEE_SMC_RETURN_RPC_PREFIX) + +/* + * Allocate memory for RPC parameter passing. The memory is used to hold a + * struct optee_msg_arg. + * + * "Call" register usage: + * a0 This value, OPTEE_SMC_RETURN_RPC_ALLOC + * a1 Size in bytes of required argument memory + * a2 Not used + * a3 Resume information, must be preserved + * a4-5 Not used + * a6-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1 Upper 32 bits of 64-bit physical pointer to allocated + * memory, (a1 == 0 && a2 == 0) if size was 0 or if memory can't + * be allocated. + * a2 Lower 32 bits of 64-bit physical pointer to allocated + * memory, (a1 == 0 && a2 == 0) if size was 0 or if memory can't + * be allocated + * a3 Preserved + * a4 Upper 32 bits of 64-bit Shared memory cookie used when freeing + * the memory or doing an RPC + * a5 Lower 32 bits of 64-bit Shared memory cookie used when freeing + * the memory or doing an RPC + * a6-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_ALLOC U(0) +#define OPTEE_SMC_RETURN_RPC_ALLOC \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_ALLOC) + +/* + * Free memory previously allocated by OPTEE_SMC_RETURN_RPC_ALLOC + * + * "Call" register usage: + * a0 This value, OPTEE_SMC_RETURN_RPC_FREE + * a1 Upper 32 bits of 64-bit shared memory cookie belonging to this + * argument memory + * a2 Lower 32 bits of 64-bit shared memory cookie belonging to this + * argument memory + * a3-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1-2 Not used + * a3-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_FREE U(2) +#define OPTEE_SMC_RETURN_RPC_FREE \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_FREE) + +/* + * Deliver a foreign interrupt in normal world. + * + * "Call" register usage: + * a0 OPTEE_SMC_RETURN_RPC_FOREIGN_INTR + * a1-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_FOREIGN_INTR U(4) +#define OPTEE_SMC_RETURN_RPC_FOREIGN_INTR \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_FOREIGN_INTR) + +/* + * Do an RPC request. The supplied struct optee_msg_arg tells which + * request to do and the parameters for the request. The following fields + * are used (the rest are unused): + * - cmd the Request ID + * - ret return value of the request, filled in by normal world + * - num_params number of parameters for the request + * - params the parameters + * - param_attrs attributes of the parameters + * + * "Call" register usage: + * a0 OPTEE_SMC_RETURN_RPC_CMD + * a1 Upper 32 bits of a 64-bit Shared memory cookie holding a + * struct optee_msg_arg, must be preserved, only the data should + * be updated + * a2 Lower 32 bits of a 64-bit Shared memory cookie holding a + * struct optee_msg_arg, must be preserved, only the data should + * be updated + * a3-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1-2 Not used + * a3-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_CMD U(5) +#define OPTEE_SMC_RETURN_RPC_CMD \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_CMD) + +/* Returned in a0 */ +#define OPTEE_SMC_RETURN_UNKNOWN_FUNCTION U(0xFFFFFFFF) + +/* Returned in a0 only from Trusted OS functions */ +#define OPTEE_SMC_RETURN_OK U(0x0) +#define OPTEE_SMC_RETURN_ETHREAD_LIMIT U(0x1) +#define OPTEE_SMC_RETURN_EBUSY U(0x2) +#define OPTEE_SMC_RETURN_ERESUME U(0x3) +#define OPTEE_SMC_RETURN_EBADADDR U(0x4) +#define OPTEE_SMC_RETURN_EBADCMD U(0x5) +#define OPTEE_SMC_RETURN_ENOMEM U(0x6) +#define OPTEE_SMC_RETURN_ENOTAVAIL U(0x7) +#define OPTEE_SMC_RETURN_IS_RPC(ret) \ + (((ret) != OPTEE_SMC_RETURN_UNKNOWN_FUNCTION) && \ + ((((ret) & OPTEE_SMC_RETURN_RPC_PREFIX_MASK) == \ + OPTEE_SMC_RETURN_RPC_PREFIX))) + +#endif /* OPTEE_SMC_H */ diff --git a/optee_os/core/arch/arm/include/sm/pm.h b/optee_os/core/arch/arm/include/sm/pm.h new file mode 100644 index 0000000..939f966 --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/pm.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017 NXP + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SM_PM_H +#define SM_PM_H +#include +#include + +struct sm_pm_ctx { + uint32_t sp; + paddr_t cpu_resume_addr; + uint32_t suspend_regs[16]; +}; + +/* suspend/resume core functions */ +void sm_pm_cpu_suspend_save(struct sm_pm_ctx *ptr, uint32_t sp); +void sm_pm_cpu_do_suspend(uint32_t *ptr); +void sm_pm_cpu_do_resume(void); + +/* + * Exported to platform suspend, arg will be passed to fn as r0 + * Return value: 0 - cpu resumed from suspended state. + * -1 - cpu not suspended. + */ +int sm_pm_cpu_suspend(uint32_t arg, int (*fn)(uint32_t)); +#endif diff --git a/optee_os/core/arch/arm/include/sm/psci.h b/optee_os/core/arch/arm/include/sm/psci.h new file mode 100644 index 0000000..b992e60 --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/psci.h @@ -0,0 +1,81 @@ +#include +#include +#include + +#define PSCI_VERSION_0_2 U(0x00000002) +#define PSCI_VERSION_1_0 U(0x00010000) +#define PSCI_VERSION_1_1 U(0x00010001) +#define PSCI_VERSION U(0x84000000) +#define PSCI_CPU_SUSPEND U(0x84000001) +#define PSCI_CPU_OFF U(0x84000002) +#define PSCI_CPU_ON U(0x84000003) +#define PSCI_CPU_ON_SMC64 (PSCI_CPU_ON | U(0x40000000)) +#define PSCI_AFFINITY_INFO U(0x84000004) +#define PSCI_MIGRATE U(0x84000005) +#define PSCI_MIGRATE_INFO_TYPE U(0x84000006) +#define PSCI_MIGRATE_INFO_UP_CPU U(0x84000007) +#define PSCI_SYSTEM_OFF U(0x84000008) +#define PSCI_SYSTEM_RESET U(0x84000009) +#define PSCI_PSCI_FEATURES U(0x8400000a) +#define PSCI_CPU_FREEZE U(0x8400000b) +#define PSCI_CPU_DEFAULT_SUSPEND U(0x8400000c) +#define PSCI_NODE_HW_STATE U(0x8400000d) +#define PSCI_SYSTEM_SUSPEND U(0x8400000e) +#define PSCI_PSCI_SET_SUSPEND_MODE U(0x8400000f) +#define PSCI_FN_STAT_RESIDENCY U(0x84000010) +#define PSCI_FN_STAT_COUNT U(0x84000011) +#define PSCI_SYSTEM_RESET2 U(0x84000012) +#define PSCI_MEM_PROTECT U(0x84000013) +#define PSCI_MEM_PROTECT_CHECK_RANGE U(0x84000014) + +#define PSCI_NUM_CALLS U(21) + +#define PSCI_AFFINITY_LEVEL_ON U(0) +#define PSCI_AFFINITY_LEVEL_OFF U(1) +#define PSCI_AFFINITY_LEVEL_ON_PENDING U(2) + +#define PSCI_POWER_STATE_ID_MASK U(0xffff) +#define PSCI_POWER_STATE_ID_SHIFT U(0) +#define PSCI_POWER_STATE_TYPE_SHIFT U(16) +#define PSCI_POWER_STATE_TYPE_MASK BIT32(PSCI_POWER_STATE_TYPE_SHIFT) +#define PSCI_POWER_STATE_AFFL_SHIFT U(24) +#define PSCI_POWER_STATE_AFFL_MASK SHIFT_U32(0x3, \ + PSCI_POWER_STATE_AFFL_SHIFT) + +#define PSCI_POWER_STATE_TYPE_STANDBY U(0) +#define PSCI_POWER_STATE_TYPE_POWER_DOWN U(1) + +#define PSCI_RET_SUCCESS (0) +#define PSCI_RET_NOT_SUPPORTED (-1) +#define PSCI_RET_INVALID_PARAMETERS (-2) +#define PSCI_RET_DENIED (-3) +#define PSCI_RET_ALREADY_ON (-4) +#define PSCI_RET_ON_PENDING (-5) +#define PSCI_RET_INTERNAL_FAILURE (-6) +#define PSCI_RET_NOT_PRESENT (-7) +#define PSCI_RET_DISABLED (-8) +#define PSCI_RET_INVALID_ADDRESS (-9) + +uint32_t psci_version(void); +int psci_cpu_suspend(uint32_t power_state, uintptr_t entry, + uint32_t context_id, struct sm_nsec_ctx *nsec); +int psci_cpu_off(void); +int psci_cpu_on(uint32_t cpu_id, uint32_t entry, uint32_t context_id); +int psci_affinity_info(uint32_t affinity, uint32_t lowest_affnity_level); +int psci_migrate(uint32_t cpu_id); +int psci_migrate_info_type(void); +int psci_migrate_info_up_cpu(void); +void psci_system_off(void); +void psci_system_reset(void); +int psci_features(uint32_t psci_fid); +int psci_system_reset2(uint32_t reset_type, uint32_t cookie); +int psci_mem_protect(uint32_t enable); +int psci_mem_chk_range(paddr_t base, size_t length); +int psci_node_hw_state(uint32_t cpu_id, uint32_t power_level); +int psci_system_suspend(uintptr_t entry, uint32_t context_id, + struct sm_nsec_ctx *nsec); +int psci_stat_residency(uint32_t cpu_id, uint32_t power_state); +int psci_stat_count(uint32_t cpu_id, uint32_t power_state); +void tee_psci_handler(struct thread_smc_args *args, struct sm_nsec_ctx *nsec); + +void psci_armv7_cpu_off(void); diff --git a/optee_os/core/arch/arm/include/sm/sm.h b/optee_os/core/arch/arm/include/sm/sm.h new file mode 100644 index 0000000..d33c5fc --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/sm.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef SM_SM_H +#define SM_SM_H + +#ifndef __ASSEMBLER__ + +#include +#include + +struct sm_unbanked_regs { + uint32_t usr_sp; + uint32_t usr_lr; + uint32_t irq_spsr; + uint32_t irq_sp; + uint32_t irq_lr; + uint32_t fiq_spsr; + uint32_t fiq_sp; + uint32_t fiq_lr; + /* + * Note that fiq_r{8-12} are not saved here. Instead thread_fiq_handler + * preserves r{8-12}. + */ + uint32_t svc_spsr; + uint32_t svc_sp; + uint32_t svc_lr; + uint32_t abt_spsr; + uint32_t abt_sp; + uint32_t abt_lr; + uint32_t und_spsr; + uint32_t und_sp; + uint32_t und_lr; +#ifdef CFG_SM_NO_CYCLE_COUNTING + uint32_t pmcr; +#endif +#ifdef CFG_FTRACE_SUPPORT + uint32_t cntkctl; + uint32_t pad; +#endif +}; + +struct sm_nsec_ctx { + struct sm_unbanked_regs ub_regs; + + uint32_t r8; + uint32_t r9; + uint32_t r10; + uint32_t r11; + uint32_t r12; + + uint32_t r0; + uint32_t r1; + uint32_t r2; + uint32_t r3; + uint32_t r4; + uint32_t r5; + uint32_t r6; + uint32_t r7; + + /* return state */ + uint32_t mon_lr; + uint32_t mon_spsr; +}; + +struct sm_sec_ctx { + struct sm_unbanked_regs ub_regs; + + uint32_t r0; + uint32_t r1; + uint32_t r2; + uint32_t r3; + uint32_t r4; + uint32_t r5; + uint32_t r6; + uint32_t r7; + + /* return state */ + uint32_t mon_lr; + uint32_t mon_spsr; +}; + +struct sm_ctx { +#ifndef CFG_SM_NO_CYCLE_COUNTING + uint32_t pad; +#endif + struct sm_sec_ctx sec; +#ifdef CFG_SM_NO_CYCLE_COUNTING + uint32_t pad; +#endif + struct sm_nsec_ctx nsec; +}; + +/* + * The secure monitor reserves space at top of stack_tmp to hold struct + * sm_ctx. + */ +#define SM_STACK_TMP_RESERVE_SIZE sizeof(struct sm_ctx) + +/* Returns storage location of non-secure context for current CPU */ +struct sm_nsec_ctx *sm_get_nsec_ctx(void); + +/* Returns stack pointer to use in monitor mode for current CPU */ +void *sm_get_sp(void); + +/* + * Initializes secure monitor, must be called by each CPU + */ +void sm_init(vaddr_t stack_pointer); + +enum sm_handler_ret { + SM_HANDLER_SMC_HANDLED = 0, + SM_HANDLER_PENDING_SMC, +}; + +/* + * Returns whether SMC was handled from platform handler in secure monitor + * or if it shall reach OP-TEE core . + */ +enum sm_handler_ret sm_platform_handler(struct sm_ctx *ctx); + +void sm_save_unbanked_regs(struct sm_unbanked_regs *regs); +void sm_restore_unbanked_regs(struct sm_unbanked_regs *regs); + +/* + * These function return to secure monitor by SMC instead of a normal + * function return. + */ +void vector_std_smc_entry(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5, uint32_t a6, uint32_t a7); +void vector_fast_smc_entry(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5, uint32_t a6, uint32_t a7); +void vector_fiq_entry(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5, uint32_t a6, uint32_t a7); + +#endif /*!__ASSEMBLER__*/ + +/* 32 bit return value for sm_from_nsec() */ +#define SM_EXIT_TO_NON_SECURE 0 +#define SM_EXIT_TO_SECURE 1 + +#endif /*SM_SM_H*/ diff --git a/optee_os/core/arch/arm/include/sm/std_smc.h b/optee_os/core/arch/arm/include/sm/std_smc.h new file mode 100644 index 0000000..ce8606d --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/std_smc.h @@ -0,0 +1,39 @@ +#ifndef __STD_SMC_H__ +#define __STD_SMC_H__ + +#include + +/* SMC function IDs for Standard Service queries */ + +#define ARM_STD_SVC_CALL_COUNT 0x8400ff00 +#define ARM_STD_SVC_UID 0x8400ff01 +/* 0x8400ff02 is reserved */ +#define ARM_STD_SVC_VERSION 0x8400ff03 + +#define ARM_SMCCC_VERSION 0x80000000 +#define ARM_SMCCC_ARCH_FEATURES 0x80000001 +#define ARM_SMCCC_ARCH_SOC_ID 0x80000002 +#define ARM_SMCCC_ARCH_WORKAROUND_1 0x80008000 +#define ARM_SMCCC_ARCH_WORKAROUND_2 0x80007fff + +#define ARM_SMCCC_RET_SUCCESS 0 +#define ARM_SMCCC_RET_NOT_SUPPORTED 0xffffffff +#define ARM_SMCCC_RET_NOT_REQUIRED 0xfffffffe +#define ARM_SMCCC_RET_INVALID_PARAMETER 0xfffffffd + +#define SMCCC_V_1_0 0x10000 +#define SMCCC_V_1_1 0x10001 +#define SMCCC_V_1_2 0x10002 + +/* ARM Standard Service Calls version numbers */ +#define STD_SVC_VERSION_MAJOR 0x0 +#define STD_SVC_VERSION_MINOR 0x1 + +/* The macros below are used to identify PSCI calls from the SMC function ID */ +#define PSCI_FID_MASK 0xffe0u +#define PSCI_FID_VALUE 0u +#define is_psci_fid(_fid) \ + (((_fid) & PSCI_FID_MASK) == PSCI_FID_VALUE) + +void smc_std_handler(struct thread_smc_args *args, struct sm_nsec_ctx *nsec); +#endif diff --git a/optee_os/core/arch/arm/include/sm/teesmc_opteed.h b/optee_os/core/arch/arm/include/sm/teesmc_opteed.h new file mode 100644 index 0000000..41e7204 --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/teesmc_opteed.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEESMC_OPTEED_H +#define TEESMC_OPTEED_H + +/* + * This file specify SMC function IDs used when returning from TEE to the + * secure monitor. + * + * All SMC Function IDs indicates SMC32 Calling Convention but will carry + * full 64 bit values in the argument registers if invoked from Aarch64 + * mode. This violates the SMC Calling Convention, but since this + * convention only coveres API towards Normwal World it's something that + * only concerns the OP-TEE Dispatcher in ARM Trusted Firmware and OP-TEE + * OS at Secure EL1. + */ + +/* + * Issued when returning from initial entry. + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_ENTRY_DONE + * r1/x1 Pointer to entry vector + */ +#define TEESMC_OPTEED_FUNCID_RETURN_ENTRY_DONE 0 +#define TEESMC_OPTEED_RETURN_ENTRY_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_ENTRY_DONE) + + + +/* + * Issued when returning from "cpu_on" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_ON_DONE + * r1/x1 0 on success and anything else to indicate error condition + */ +#define TEESMC_OPTEED_FUNCID_RETURN_ON_DONE 1 +#define TEESMC_OPTEED_RETURN_ON_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_ON_DONE) + +/* + * Issued when returning from "cpu_off" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_OFF_DONE + * r1/x1 0 on success and anything else to indicate error condition + */ +#define TEESMC_OPTEED_FUNCID_RETURN_OFF_DONE 2 +#define TEESMC_OPTEED_RETURN_OFF_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_OFF_DONE) + +/* + * Issued when returning from "cpu_suspend" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_SUSPEND_DONE + * r1/x1 0 on success and anything else to indicate error condition + */ +#define TEESMC_OPTEED_FUNCID_RETURN_SUSPEND_DONE 3 +#define TEESMC_OPTEED_RETURN_SUSPEND_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_SUSPEND_DONE) + +/* + * Issued when returning from "cpu_resume" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_RESUME_DONE + * r1/x1 0 on success and anything else to indicate error condition + */ +#define TEESMC_OPTEED_FUNCID_RETURN_RESUME_DONE 4 +#define TEESMC_OPTEED_RETURN_RESUME_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_RESUME_DONE) + +/* + * Issued when returning from "std_smc" or "fast_smc" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_CALL_DONE + * r1-4/x1-4 Return value 0-3 which will passed to normal world in + * r0-3/x0-3 + */ +#define TEESMC_OPTEED_FUNCID_RETURN_CALL_DONE 5 +#define TEESMC_OPTEED_RETURN_CALL_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_CALL_DONE) + +/* + * Issued when returning from "fiq" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_FIQ_DONE + */ +#define TEESMC_OPTEED_FUNCID_RETURN_FIQ_DONE 6 +#define TEESMC_OPTEED_RETURN_FIQ_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_FIQ_DONE) + +/* + * Issued when returning from "system_off" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_SYSTEM_OFF_DONE + */ +#define TEESMC_OPTEED_FUNCID_RETURN_SYSTEM_OFF_DONE 7 +#define TEESMC_OPTEED_RETURN_SYSTEM_OFF_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_SYSTEM_OFF_DONE) + +/* + * Issued when returning from "system_reset" vector + * + * Register usage: + * r0/x0 SMC Function ID, TEESMC_OPTEED_RETURN_SYSTEM_RESET_DONE + */ +#define TEESMC_OPTEED_FUNCID_RETURN_SYSTEM_RESET_DONE 8 +#define TEESMC_OPTEED_RETURN_SYSTEM_RESET_DONE \ + TEESMC_OPTEED_RV(TEESMC_OPTEED_FUNCID_RETURN_SYSTEM_RESET_DONE) + +#endif /*TEESMC_OPTEED_H*/ diff --git a/optee_os/core/arch/arm/include/sm/teesmc_opteed_macros.h b/optee_os/core/arch/arm/include/sm/teesmc_opteed_macros.h new file mode 100644 index 0000000..fcc84b7 --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/teesmc_opteed_macros.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEESMC_OPTEED_MACROS_H +#define TEESMC_OPTEED_MACROS_H + +#define TEESMC_OPTEED_RV(func_num) \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_FAST_CALL, \ + OPTEE_SMC_OWNER_TRUSTED_OS_OPTEED, (func_num)) + +#endif /*TEESMC_OPTEED_MACROS_H*/ diff --git a/optee_os/core/arch/arm/include/sm/watchdog_smc.h b/optee_os/core/arch/arm/include/sm/watchdog_smc.h new file mode 100644 index 0000000..b8b55a4 --- /dev/null +++ b/optee_os/core/arch/arm/include/sm/watchdog_smc.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) STMicroelectronics 2023 + */ +#ifndef SM_WATCHDOG_SMC_H +#define SM_WATCHDOG_SMC_H + +/* + * This file describes the secure watchdog management service. + * It exposes the SMC supported commands. + * We're following ARM SMC Calling Convention as specified in + * https://developer.arm.com/documentation/den0028. + */ + +/* + * Overall global call for watchdog interface + * Call requests usage: + * a0 [in] SMC Function ID, CFG_WDT_SM_HANDLER_ID + * [out] PSCI error code return + * a1 [in] Watchdog command (one of SMCWD_*) + * [out] Depends on watchdog command (input a1) + * a2 [in/out] Depends on watchdog command (input a1) + * a3-6 [in/out] Not used + * a7 [in/out] Hypervisor Client ID register + */ + +/* Watchdog supported commands */ + +#define SMCWD_INIT 0 +#define SMCWD_SET_TIMEOUT 1 +#define SMCWD_ENABLE 2 +#define SMCWD_PET 3 +#define SMCWD_GET_TIMELEFT 4 + +/* + * Command SMCWD_INIT : Watchdog initialization + * [in] a1 Set to SMCWD_INIT + * [out] a1 The minimal timeout value in seconds supported + * a2 The maximum timeout value in seconds supported + * Return codes: + * PSCI_RET_SUCCESS - Command success + * PSCI_RET_INTERNAL_FAILURE - Initialization failure + * + * Command SMCWD_SET_TIMEOUT : Watchdog set timeout + * [in] a1 Set to SMCWD_SET_TIMEOUT + * a2 The timeout value in seconds to set + * Return codes: + * PSCI_RET_SUCCESS - Command success + * PSCI_RET_INVALID_PARAMETERS - Incorrect input param + * + * Command SMCWD_ENABLE : Watchdog enable + * [in] a1 Set to SMCWD_ENABLE + * a2 Set to 0 to stop the watchdog, 1 to enable it + * Return codes: + * PSCI_RET_SUCCESS - Command success + * PSCI_RET_INVALID_PARAMETERS - Incorrect input param + * + * Command SMCWD_PET : Ping the watchdog for refresh + * [in] a1 Set to SMCWD_PET + * Return codes: + * PSCI_RET_SUCCESS - Command success + * PSCI_RET_DISABLED - The watchdog is not enabled + * + * Command SMCWD_GET_TIMELEFT : Get time left + * [in] a1 Set to SMCWD_GET_TIMELEFT + * [out] a1 The timeout value in seconds before watchdog expires + * Return codes: + * PSCI_RET_SUCCESS - Command success + * PSCI_RET_DISABLED - The watchdog is not enabled + * PSCI_RET_NOT_SUPPORTED - Function not supported + * + * Other commands + * [in] a1 Other values + * Return codes: + * PSCI_RET_NOT_SUPPORTED - Function not supported + * + * a3-6 Not used + * a7 Hypervisor Client ID register + */ + +#endif /* SM_WATCHDOG_SMC_H */ diff --git a/optee_os/core/arch/arm/include/smccc.h b/optee_os/core/arch/arm/include/smccc.h new file mode 100644 index 0000000..8146cbe --- /dev/null +++ b/optee_os/core/arch/arm/include/smccc.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ + +#ifndef __SMCCC_H +#define __SMCCC_H + +/* + * Describes features of SMC Calling Convention from v1.1 + * See also https://developer.arm.com/-/media/developer/pdf/ARM_DEN_0070A_Firmware_interfaces_for_mitigating_CVE-2017-5715.pdf + */ + +/* + * Retrieve the implemented version of the SMC Calling Convention + * Mandatory from SMCCC v1.1 + * Optional in SMCCC v1.0 + */ +#define SMCCC_VERSION 0x80000000 + +/* + * Determine the availability and capability of Arm Architecture Service + * functions. + * Mandatory from SMCCC v1.1 + * Optional for SMCCC v1.0 + */ +#define SMCCC_ARCH_FEATURES 0x80000001 + +/* + * Execute the mitigation for CVE-2017-5715 on the calling PE + * Optional from SMCCC v1.1 + * Not supported in SMCCC v1.0 + */ +#define SMCCC_ARCH_WORKAROUND_1 0x80008000 + +#endif /*__SMCCC_H*/ diff --git a/optee_os/core/arch/arm/include/tee/entry_fast.h b/optee_os/core/arch/arm/include/tee/entry_fast.h new file mode 100644 index 0000000..0f6fef8 --- /dev/null +++ b/optee_os/core/arch/arm/include/tee/entry_fast.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_ENTRY_FAST_H +#define TEE_ENTRY_FAST_H + +#include + +/* These functions are overridable by the specific target */ +void tee_entry_get_api_call_count(struct thread_smc_args *args); +void tee_entry_get_api_uuid(struct thread_smc_args *args); +void tee_entry_get_api_revision(struct thread_smc_args *args); +void tee_entry_get_os_uuid(struct thread_smc_args *args); +void tee_entry_get_os_revision(struct thread_smc_args *args); + +/* + * Returns the number of calls recognized by tee_entry(). Used by the + * specific target to calculate the total number of supported calls when + * overriding tee_entry_get_api_call_count(). + */ +size_t tee_entry_generic_get_api_call_count(void); + +/* + * Fast call entry, __weak, overridable. If overridden should call + * __tee_entry_fast() at the end in order to handle the standard functions. + */ +void tee_entry_fast(struct thread_smc_args *args); +void __tee_entry_fast(struct thread_smc_args *args); + +#endif /* TEE_ENTRY_FAST_H */ diff --git a/optee_os/core/arch/arm/kernel/abort.c b/optee_os/core/arch/arm/kernel/abort.c new file mode 100644 index 0000000..72fbfc8 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/abort.c @@ -0,0 +1,593 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum fault_type { + FAULT_TYPE_USER_MODE_PANIC, + FAULT_TYPE_USER_MODE_VFP, + FAULT_TYPE_PAGEABLE, + FAULT_TYPE_IGNORE, +}; + +#ifdef CFG_UNWIND + +#ifdef ARM32 +/* + * Kernel or user mode unwind (32-bit execution state). + */ +static void __print_stack_unwind(struct abort_info *ai) +{ + struct unwind_state_arm32 state = { }; + uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; + uint32_t sp = 0; + uint32_t lr = 0; + + assert(!abort_is_user_exception(ai)); + + if (mode == CPSR_MODE_SYS) { + sp = ai->regs->usr_sp; + lr = ai->regs->usr_lr; + } else { + sp = read_mode_sp(mode); + lr = read_mode_lr(mode); + } + + memset(&state, 0, sizeof(state)); + state.registers[0] = ai->regs->r0; + state.registers[1] = ai->regs->r1; + state.registers[2] = ai->regs->r2; + state.registers[3] = ai->regs->r3; + state.registers[4] = ai->regs->r4; + state.registers[5] = ai->regs->r5; + state.registers[6] = ai->regs->r6; + state.registers[7] = ai->regs->r7; + state.registers[8] = ai->regs->r8; + state.registers[9] = ai->regs->r9; + state.registers[10] = ai->regs->r10; + state.registers[11] = ai->regs->r11; + state.registers[13] = sp; + state.registers[14] = lr; + state.registers[15] = ai->pc; + + print_stack_arm32(&state, thread_stack_start(), thread_stack_size()); +} +#endif /* ARM32 */ + +#ifdef ARM64 +/* Kernel mode unwind (64-bit execution state) */ +static void __print_stack_unwind(struct abort_info *ai) +{ + struct unwind_state_arm64 state = { + .pc = ai->regs->elr, + .fp = ai->regs->x29, + }; + + print_stack_arm64(&state, thread_stack_start(), thread_stack_size()); +} +#endif /*ARM64*/ + +#else /* CFG_UNWIND */ +static void __print_stack_unwind(struct abort_info *ai __unused) +{ +} +#endif /* CFG_UNWIND */ + +static __maybe_unused const char *abort_type_to_str(uint32_t abort_type) +{ + if (abort_type == ABORT_TYPE_DATA) + return "data"; + if (abort_type == ABORT_TYPE_PREFETCH) + return "prefetch"; + return "undef"; +} + +static __maybe_unused const char *fault_to_str(uint32_t abort_type, + uint32_t fault_descr) +{ + /* fault_descr is only valid for data or prefetch abort */ + if (abort_type != ABORT_TYPE_DATA && abort_type != ABORT_TYPE_PREFETCH) + return ""; + + switch (core_mmu_get_fault_type(fault_descr)) { + case CORE_MMU_FAULT_ALIGNMENT: + return " (alignment fault)"; + case CORE_MMU_FAULT_TRANSLATION: + return " (translation fault)"; + case CORE_MMU_FAULT_READ_PERMISSION: + return " (read permission fault)"; + case CORE_MMU_FAULT_WRITE_PERMISSION: + return " (write permission fault)"; + case CORE_MMU_FAULT_TAG_CHECK: + return " (tag check fault)"; + default: + return ""; + } +} + +static __maybe_unused void +__print_abort_info(struct abort_info *ai __maybe_unused, + const char *ctx __maybe_unused) +{ + __maybe_unused size_t core_pos = 0; +#ifdef ARM32 + uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; + __maybe_unused uint32_t sp = 0; + __maybe_unused uint32_t lr = 0; + + if (mode == CPSR_MODE_USR || mode == CPSR_MODE_SYS) { + sp = ai->regs->usr_sp; + lr = ai->regs->usr_lr; + core_pos = thread_get_tsd()->abort_core; + } else { + sp = read_mode_sp(mode); + lr = read_mode_lr(mode); + core_pos = get_core_pos(); + } +#endif /*ARM32*/ +#ifdef ARM64 + if (abort_is_user_exception(ai)) + core_pos = thread_get_tsd()->abort_core; + else + core_pos = get_core_pos(); +#endif /*ARM64*/ + + EMSG_RAW(""); + if (IS_ENABLED(CFG_MEMTAG)) + EMSG_RAW("%s %s-abort at address 0x%" PRIxVA + " [tagged 0x%" PRIxVA "]%s", ctx, + abort_type_to_str(ai->abort_type), + memtag_strip_tag_vaddr((void *)ai->va), ai->va, + fault_to_str(ai->abort_type, ai->fault_descr)); + else + EMSG_RAW("%s %s-abort at address 0x%" PRIxVA "%s", ctx, + abort_type_to_str(ai->abort_type), ai->va, + fault_to_str(ai->abort_type, ai->fault_descr)); +#ifdef ARM32 + EMSG_RAW(" fsr 0x%08x ttbr0 0x%08x ttbr1 0x%08x cidr 0x%X", + ai->fault_descr, read_ttbr0(), read_ttbr1(), + read_contextidr()); + EMSG_RAW(" cpu #%zu cpsr 0x%08x", + core_pos, ai->regs->spsr); + EMSG_RAW(" r0 0x%08x r4 0x%08x r8 0x%08x r12 0x%08x", + ai->regs->r0, ai->regs->r4, ai->regs->r8, ai->regs->ip); + EMSG_RAW(" r1 0x%08x r5 0x%08x r9 0x%08x sp 0x%08x", + ai->regs->r1, ai->regs->r5, ai->regs->r9, sp); + EMSG_RAW(" r2 0x%08x r6 0x%08x r10 0x%08x lr 0x%08x", + ai->regs->r2, ai->regs->r6, ai->regs->r10, lr); + EMSG_RAW(" r3 0x%08x r7 0x%08x r11 0x%08x pc 0x%08x", + ai->regs->r3, ai->regs->r7, ai->regs->r11, ai->pc); +#endif /*ARM32*/ +#ifdef ARM64 + EMSG_RAW(" esr 0x%08x ttbr0 0x%08" PRIx64 " ttbr1 0x%08" PRIx64 + " cidr 0x%X", ai->fault_descr, read_ttbr0_el1(), + read_ttbr1_el1(), read_contextidr_el1()); + EMSG_RAW(" cpu #%zu cpsr 0x%08x", + core_pos, (uint32_t)ai->regs->spsr); + EMSG_RAW(" x0 %016" PRIx64 " x1 %016" PRIx64, + ai->regs->x0, ai->regs->x1); + EMSG_RAW(" x2 %016" PRIx64 " x3 %016" PRIx64, + ai->regs->x2, ai->regs->x3); + EMSG_RAW(" x4 %016" PRIx64 " x5 %016" PRIx64, + ai->regs->x4, ai->regs->x5); + EMSG_RAW(" x6 %016" PRIx64 " x7 %016" PRIx64, + ai->regs->x6, ai->regs->x7); + EMSG_RAW(" x8 %016" PRIx64 " x9 %016" PRIx64, + ai->regs->x8, ai->regs->x9); + EMSG_RAW(" x10 %016" PRIx64 " x11 %016" PRIx64, + ai->regs->x10, ai->regs->x11); + EMSG_RAW(" x12 %016" PRIx64 " x13 %016" PRIx64, + ai->regs->x12, ai->regs->x13); + EMSG_RAW(" x14 %016" PRIx64 " x15 %016" PRIx64, + ai->regs->x14, ai->regs->x15); + EMSG_RAW(" x16 %016" PRIx64 " x17 %016" PRIx64, + ai->regs->x16, ai->regs->x17); + EMSG_RAW(" x18 %016" PRIx64 " x19 %016" PRIx64, + ai->regs->x18, ai->regs->x19); + EMSG_RAW(" x20 %016" PRIx64 " x21 %016" PRIx64, + ai->regs->x20, ai->regs->x21); + EMSG_RAW(" x22 %016" PRIx64 " x23 %016" PRIx64, + ai->regs->x22, ai->regs->x23); + EMSG_RAW(" x24 %016" PRIx64 " x25 %016" PRIx64, + ai->regs->x24, ai->regs->x25); + EMSG_RAW(" x26 %016" PRIx64 " x27 %016" PRIx64, + ai->regs->x26, ai->regs->x27); + EMSG_RAW(" x28 %016" PRIx64 " x29 %016" PRIx64, + ai->regs->x28, ai->regs->x29); + EMSG_RAW(" x30 %016" PRIx64 " elr %016" PRIx64, + ai->regs->x30, ai->regs->elr); + EMSG_RAW(" sp_el0 %016" PRIx64, ai->regs->sp_el0); +#endif /*ARM64*/ +} + +/* + * Print abort info and (optionally) stack dump to the console + * @ai kernel-mode abort info. + * @stack_dump true to show a stack trace + */ +static void __abort_print(struct abort_info *ai, bool stack_dump) +{ + assert(!abort_is_user_exception(ai)); + + __print_abort_info(ai, "Core"); + + if (stack_dump) { + trace_printf_helper_raw(TRACE_ERROR, true, + "TEE load address @ %#"PRIxVA, + VCORE_START_VA); + __print_stack_unwind(ai); + } +} + +void abort_print(struct abort_info *ai) +{ + __abort_print(ai, false); +} + +void abort_print_error(struct abort_info *ai) +{ + __abort_print(ai, true); +} + +/* This function must be called from a normal thread */ +void abort_print_current_ts(void) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + struct abort_info ai = { }; + struct ts_session *s = ts_get_current_session(); + + ai.abort_type = tsd->abort_type; + ai.fault_descr = tsd->abort_descr; + ai.va = tsd->abort_va; + ai.pc = tsd->abort_regs.elr; + ai.regs = &tsd->abort_regs; + + if (ai.abort_type != ABORT_TYPE_USER_MODE_PANIC) + __print_abort_info(&ai, "User mode"); + + s->ctx->ops->dump_state(s->ctx); + +#if defined(CFG_FTRACE_SUPPORT) + if (s->ctx->ops->dump_ftrace) { + s->fbuf = NULL; + s->ctx->ops->dump_ftrace(s->ctx); + } +#endif +} + +static void save_abort_info_in_tsd(struct abort_info *ai) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + + tsd->abort_type = ai->abort_type; + tsd->abort_descr = ai->fault_descr; + tsd->abort_va = ai->va; + tsd->abort_regs = *ai->regs; + tsd->abort_core = get_core_pos(); +} + +#ifdef ARM32 +static void set_abort_info(uint32_t abort_type, struct thread_abort_regs *regs, + struct abort_info *ai) +{ + switch (abort_type) { + case ABORT_TYPE_DATA: + ai->fault_descr = read_dfsr(); + ai->va = read_dfar(); + break; + case ABORT_TYPE_PREFETCH: + ai->fault_descr = read_ifsr(); + ai->va = read_ifar(); + break; + default: + ai->fault_descr = 0; + ai->va = regs->elr; + break; + } + ai->abort_type = abort_type; + ai->pc = regs->elr; + ai->regs = regs; +} +#endif /*ARM32*/ + +#ifdef ARM64 +static void set_abort_info(uint32_t abort_type __unused, + struct thread_abort_regs *regs, struct abort_info *ai) +{ + ai->fault_descr = read_esr_el1(); + switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { + case ESR_EC_IABT_EL0: + case ESR_EC_IABT_EL1: + ai->abort_type = ABORT_TYPE_PREFETCH; + ai->va = read_far_el1(); + break; + case ESR_EC_DABT_EL0: + case ESR_EC_DABT_EL1: + case ESR_EC_SP_ALIGN: + ai->abort_type = ABORT_TYPE_DATA; + ai->va = read_far_el1(); + break; + default: + ai->abort_type = ABORT_TYPE_UNDEF; + ai->va = regs->elr; + } + ai->pc = regs->elr; + ai->regs = regs; +} +#endif /*ARM64*/ + +#ifdef ARM32 +static void handle_user_mode_panic(struct abort_info *ai) +{ + /* + * It was a user exception, stop user execution and return + * to TEE Core. + */ + ai->regs->r0 = TEE_ERROR_TARGET_DEAD; + ai->regs->r1 = true; + ai->regs->r2 = 0xdeadbeef; + ai->regs->elr = (uint32_t)thread_unwind_user_mode; + ai->regs->spsr &= CPSR_FIA; + ai->regs->spsr &= ~CPSR_MODE_MASK; + ai->regs->spsr |= CPSR_MODE_SVC; + /* Select Thumb or ARM mode */ + if (ai->regs->elr & 1) + ai->regs->spsr |= CPSR_T; + else + ai->regs->spsr &= ~CPSR_T; +} +#endif /*ARM32*/ + +#ifdef ARM64 +static void handle_user_mode_panic(struct abort_info *ai) +{ + struct thread_ctx *tc __maybe_unused = NULL; + uint32_t daif = 0; + uint32_t pan_bit = 0; + + /* + * It was a user exception, stop user execution and return + * to TEE Core. + */ + ai->regs->x0 = TEE_ERROR_TARGET_DEAD; + ai->regs->x1 = true; + ai->regs->x2 = 0xdeadbeef; + ai->regs->elr = (vaddr_t)thread_unwind_user_mode; + ai->regs->sp_el0 = thread_get_saved_thread_sp(); + +#if defined(CFG_CORE_PAUTH) + /* + * We're going to return to the privileged core thread, update the + * APIA key to match the key used by the thread. + */ + tc = threads + thread_get_id(); + ai->regs->apiakey_hi = tc->keys.apia_hi; + ai->regs->apiakey_lo = tc->keys.apia_lo; +#endif + + if (IS_ENABLED(CFG_PAN) && feat_pan_implemented() && read_pan()) + pan_bit = SPSR_64_PAN; + daif = (ai->regs->spsr >> SPSR_32_AIF_SHIFT) & SPSR_32_AIF_MASK; + /* XXX what about DAIF_D? */ + ai->regs->spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, daif) | + pan_bit; +} +#endif /*ARM64*/ + +#ifdef CFG_WITH_VFP +static void handle_user_mode_vfp(void) +{ + struct ts_session *s = ts_get_current_session(); + + thread_user_enable_vfp(&to_user_mode_ctx(s->ctx)->vfp); +} +#endif /*CFG_WITH_VFP*/ + +#ifdef CFG_WITH_USER_TA +#ifdef ARM32 +/* Returns true if the exception originated from user mode */ +bool abort_is_user_exception(struct abort_info *ai) +{ + return (ai->regs->spsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; +} +#endif /*ARM32*/ + +#ifdef ARM64 +/* Returns true if the exception originated from user mode */ +bool abort_is_user_exception(struct abort_info *ai) +{ + uint32_t spsr = ai->regs->spsr; + + if (spsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) + return true; + if (((spsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == + SPSR_64_MODE_EL0) + return true; + return false; +} +#endif /*ARM64*/ +#else /*CFG_WITH_USER_TA*/ +bool abort_is_user_exception(struct abort_info *ai __unused) +{ + return false; +} +#endif /*CFG_WITH_USER_TA*/ + +#if defined(CFG_WITH_VFP) && defined(CFG_WITH_USER_TA) +#ifdef ARM32 +static bool is_vfp_fault(struct abort_info *ai) +{ + if ((ai->abort_type != ABORT_TYPE_UNDEF) || vfp_is_enabled()) + return false; + + /* + * Not entirely accurate, but if it's a truly undefined instruction + * we'll end up in this function again, except this time + * vfp_is_enabled() so we'll return false. + */ + return true; +} +#endif /*ARM32*/ + +#ifdef ARM64 +static bool is_vfp_fault(struct abort_info *ai) +{ + switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { + case ESR_EC_FP_ASIMD: + case ESR_EC_AARCH32_FP: + case ESR_EC_AARCH64_FP: + return true; + default: + return false; + } +} +#endif /*ARM64*/ +#else /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ +static bool is_vfp_fault(struct abort_info *ai __unused) +{ + return false; +} +#endif /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ + +bool abort_is_write_fault(struct abort_info *ai) +{ +#ifdef ARM32 + unsigned int write_not_read = 11; +#endif +#ifdef ARM64 + unsigned int write_not_read = 6; +#endif + + return ai->abort_type == ABORT_TYPE_DATA && + (ai->fault_descr & BIT(write_not_read)); +} + +static enum fault_type get_fault_type(struct abort_info *ai) +{ + if (abort_is_user_exception(ai)) { + if (is_vfp_fault(ai)) + return FAULT_TYPE_USER_MODE_VFP; +#ifndef CFG_WITH_PAGER + return FAULT_TYPE_USER_MODE_PANIC; +#endif + } + + if (thread_is_from_abort_mode()) { + abort_print_error(ai); + panic("[abort] abort in abort handler (trap CPU)"); + } + + if (ai->abort_type == ABORT_TYPE_UNDEF) { + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] undefined abort (trap CPU)"); + } + + switch (core_mmu_get_fault_type(ai->fault_descr)) { + case CORE_MMU_FAULT_ALIGNMENT: + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] alignement fault! (trap CPU)"); + break; + + case CORE_MMU_FAULT_ACCESS_BIT: + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] access bit fault! (trap CPU)"); + break; + + case CORE_MMU_FAULT_DEBUG_EVENT: + if (!abort_is_user_exception(ai)) + abort_print(ai); + DMSG("[abort] Ignoring debug event!"); + return FAULT_TYPE_IGNORE; + + case CORE_MMU_FAULT_TRANSLATION: + case CORE_MMU_FAULT_WRITE_PERMISSION: + case CORE_MMU_FAULT_READ_PERMISSION: + return FAULT_TYPE_PAGEABLE; + + case CORE_MMU_FAULT_ASYNC_EXTERNAL: + if (!abort_is_user_exception(ai)) + abort_print(ai); + DMSG("[abort] Ignoring async external abort!"); + return FAULT_TYPE_IGNORE; + + case CORE_MMU_FAULT_TAG_CHECK: + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] Tag check fault! (trap CPU)"); + break; + + case CORE_MMU_FAULT_OTHER: + default: + if (!abort_is_user_exception(ai)) + abort_print(ai); + DMSG("[abort] Unhandled fault!"); + return FAULT_TYPE_IGNORE; + } +} + +void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs) +{ + struct abort_info ai; + bool handled; + + set_abort_info(abort_type, regs, &ai); + + switch (get_fault_type(&ai)) { + case FAULT_TYPE_IGNORE: + break; + case FAULT_TYPE_USER_MODE_PANIC: + DMSG("[abort] abort in User mode (TA will panic)"); + save_abort_info_in_tsd(&ai); + vfp_disable(); + handle_user_mode_panic(&ai); + break; +#ifdef CFG_WITH_VFP + case FAULT_TYPE_USER_MODE_VFP: + handle_user_mode_vfp(); + break; +#endif + case FAULT_TYPE_PAGEABLE: + default: + if (thread_get_id_may_fail() < 0) { + abort_print_error(&ai); + panic("abort outside thread context"); + } + thread_kernel_save_vfp(); + handled = tee_pager_handle_fault(&ai); + thread_kernel_restore_vfp(); + if (!handled) { + if (!abort_is_user_exception(&ai)) { + abort_print_error(&ai); + panic("unhandled pageable abort"); + } + DMSG("[abort] abort in User mode (TA will panic)"); + save_abort_info_in_tsd(&ai); + vfp_disable(); + handle_user_mode_panic(&ai); + } + break; + } +} diff --git a/optee_os/core/arch/arm/kernel/arch_scall.c b/optee_os/core/arch/arm/kernel/arch_scall.c new file mode 100644 index 0000000..1a865dc --- /dev/null +++ b/optee_os/core/arch/arm/kernel/arch_scall.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2022, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define TA32_CONTEXT_MAX_SIZE (14 * sizeof(uint32_t)) +#define TA64_CONTEXT_MAX_SIZE (2 * sizeof(uint64_t)) + +#ifdef CFG_UNWIND +#ifdef ARM32 +/* Get register values pushed onto the stack by _utee_panic() */ +static void save_panic_regs_a32_ta(struct thread_specific_data *tsd, + uint32_t *pushed) +{ + tsd->abort_regs = (struct thread_abort_regs){ + .elr = pushed[0], + .r0 = pushed[1], + .r1 = pushed[2], + .r2 = pushed[3], + .r3 = pushed[4], + .r4 = pushed[5], + .r5 = pushed[6], + .r6 = pushed[7], + .r7 = pushed[8], + .r8 = pushed[9], + .r9 = pushed[10], + .r10 = pushed[11], + .r11 = pushed[12], + .usr_sp = (uint32_t)pushed, + .usr_lr = pushed[13], + .spsr = read_spsr(), + }; +} + +void scall_save_panic_stack(struct thread_scall_regs *regs) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + struct ts_session *s = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(s->ctx); + + tsd->abort_type = ABORT_TYPE_USER_MODE_PANIC; + tsd->abort_descr = 0; + tsd->abort_va = 0; + + if (vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE, + (uaddr_t)regs->r1, TA32_CONTEXT_MAX_SIZE)) { + TAMSG_RAW(""); + TAMSG_RAW("Can't unwind invalid user stack 0x%"PRIxUA, + (uaddr_t)regs->r1); + return; + } + + save_panic_regs_a32_ta(tsd, (uint32_t *)regs->r1); +} +#endif /*ARM32*/ + +#ifdef ARM64 +/* Get register values pushed onto the stack by _utee_panic() (32-bit TA) */ +static void save_panic_regs_a32_ta(struct thread_specific_data *tsd, + uint32_t *pushed) +{ + tsd->abort_regs = (struct thread_abort_regs){ + .elr = pushed[0], + .x0 = pushed[1], + .x1 = pushed[2], + .x2 = pushed[3], + .x3 = pushed[4], + .x4 = pushed[5], + .x5 = pushed[6], + .x6 = pushed[7], + .x7 = pushed[8], + .x8 = pushed[9], + .x9 = pushed[10], + .x10 = pushed[11], + .x11 = pushed[12], + .x13 = (uint64_t)pushed, + .x14 = pushed[13], + .spsr = (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT), + }; +} + +/* Get register values pushed onto the stack by _utee_panic() (64-bit TA) */ +static void save_panic_regs_a64_ta(struct thread_specific_data *tsd, + uint64_t *pushed) +{ + TEE_Result res = TEE_SUCCESS; + uint64_t x29 = 0; + uint64_t elr = 0; + + res = GET_USER_SCALAR(x29, &pushed[0]); + if (res) + x29 = 0; + + res = GET_USER_SCALAR(elr, &pushed[1]); + if (res) + elr = 0; + + tsd->abort_regs = (struct thread_abort_regs){ + .x29 = x29, + .elr = elr, + .spsr = (SPSR_64_MODE_EL0 << SPSR_64_MODE_EL_SHIFT), + }; +} + +void scall_save_panic_stack(struct thread_scall_regs *regs) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + struct ts_session *s = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(s->ctx); + + if (vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE, + (uaddr_t)regs->x1, + utc->uctx.is_32bit ? + TA32_CONTEXT_MAX_SIZE : + TA64_CONTEXT_MAX_SIZE)) { + TAMSG_RAW(""); + TAMSG_RAW("Can't unwind invalid user stack 0x%"PRIxUA, + (uaddr_t)regs->x1); + return; + } + + tsd->abort_type = ABORT_TYPE_USER_MODE_PANIC; + tsd->abort_descr = 0; + tsd->abort_va = 0; + + if (utc->uctx.is_32bit) + save_panic_regs_a32_ta(tsd, (uint32_t *)regs->x1); + else + save_panic_regs_a64_ta(tsd, (uint64_t *)regs->x1); +} +#endif /*ARM64*/ + +#else /* CFG_UNWIND */ +void scall_save_panic_stack(struct thread_scall_regs *regs __unused) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + + tsd->abort_type = ABORT_TYPE_USER_MODE_PANIC; +} +#endif /* CFG_UNWIND */ diff --git a/optee_os/core/arch/arm/kernel/arch_scall_a32.S b/optee_os/core/arch/arm/kernel/arch_scall_a32.S new file mode 100644 index 0000000..2b572e2 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/arch_scall_a32.S @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include "tee_syscall_numbers.h" +#include "trace_levels.h" +#include +#include +#include +#include +#include + +/* + * uint32_t scall_do_call(struct thread_scall_regs *regs, syscall_t func); + * + * Called from scall_handle_user_ta() + */ +FUNC scall_do_call , : +UNWIND( .cantunwind) + push {r5-r9, lr} + mov r7, sp + mov r8, r0 + mov r9, r1 + ldr r5, [r8, #THREAD_SCALL_REG_R5] + ldr r6, [r8, #THREAD_SCALL_REG_R6] + + /* + * Copy eventual arguments passed on the user stack. + * + * r5 holds the address of the first word + * r6 holds the number of words + * + * scall_handle_user_ta() who calls this function has already checked + * that we don't copy too much data. + */ + cmp r6, #0 + beq .Lno_args + sub sp, sp, r6, lsl #2 + bic sp, sp, #7 /* make sure it's a multiple of 8 */ + mov r0, sp + mov r1, r5 + mov r2, r6, lsl #2 + ldr lr, =copy_from_user + blx lr + + /* If copy failed return the error */ + cmp r0, #0 + bne .Lret + +.Lno_args: + /* Load arguments to function */ + add lr, r8, #THREAD_SCALL_REG_R0 + ldm lr, {r0-r3} + blx r9 +.Lret: + mov sp, r7 + pop {r5-r9, pc} +END_FUNC scall_do_call + +/* + * syscall_sys_return() and syscall_panic() are two special cases for syscalls + * in the way that they do not return to the TA, instead execution is resumed + * as if __thread_enter_user_mode() had returned to thread_enter_user_mode(). + * + * In order to do this the functions need a way to get hold of a pointer to + * the struct thread_scall_regs provided by storing relevant registers on the + * stack in thread_scall_handler() and later load them into registers again + * when thread_scall_handler() is returning. + * + * scall_do_call() is supplied the pointer to struct thread_scall_regs in + * r0. This pointer can later be retrieved from r8. + */ + +/* + * User space sees this function as: + * void syscall_sys_return(uint32_t ret) __noreturn; + * + * But internally the function depends on being called from + * scall_do_call() with pointer to the struct thread_scall_regs saved by + * thread_scall_handler() in r8. + * + * The argument ret is already in r0 so we don't touch that and let it + * propagate as return value of the called scall_sys_return_helper(). + */ +FUNC syscall_sys_return , : + mov r1, #0 /* panic = false */ + mov r2, #0 /* panic_code = 0 */ + mov r3, r8 /* pointer to struct thread_scall_regs */ + b scall_sys_return_helper +END_FUNC syscall_sys_return + +/* + * User space sees this function as: + * void syscall_panic(uint32_t code) __noreturn; + * + * But internally the function depends on being called from + * scall_do_call() with pointer to the struct thread_scall_regs saved by + * thread_scall_handler() in r8. + */ +FUNC syscall_panic , : + mov r1, #1 /* panic = true */ + mov r2, r0 /* panic_code = 0 */ + mov r3, r8 /* pointer to struct thread_scall_regs */ + ldr r0, =TEE_ERROR_TARGET_DEAD + b scall_sys_return_helper +END_FUNC syscall_panic diff --git a/optee_os/core/arch/arm/kernel/arch_scall_a64.S b/optee_os/core/arch/arm/kernel/arch_scall_a64.S new file mode 100644 index 0000000..03a96f6 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/arch_scall_a64.S @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#include "tee_syscall_numbers.h" +#include "trace_levels.h" +#include +#include +#include +#include +#include +#include + +#if 0 +struct sc_rec { + uint64_t x0; + uint64_t x1; + uint64_t x19; + uint64_t x30; +} +#endif +#define SC_REC_X0 (8 * 0) +#define SC_REC_X1 (8 * 1) +#define SC_REC_X19 (8 * 2) +#define SC_REC_X30 (8 * 3) +#define SC_REC_SIZE (SC_REC_X30 + 8) + +/* + * uint32_t scall_do_call(struct thread_scall_regs *regs, syscall_t func); + * + * Called from scall_handle_user_ta() + */ +FUNC scall_do_call , : + sub sp, sp, #SC_REC_SIZE + stp x0, x1, [sp, #SC_REC_X0] + stp x19, x30, [sp, #SC_REC_X19] + mov x19, sp + + ldr x2, [x0, #THREAD_SCALL_REG_SPSR] + tst x2, #(SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT) + b.eq .Lcall_a64 + + ldp x5, x6, [x0, #THREAD_SCALL_REG_X5] + cmp x6, #0 + b.eq .Lno_args_a32 + + /* + * Calculate required space on stack to copy Aarch32 arguments + * and to transform them into Aarch64 arguments. + * x6 = nargs_on_stack + * n64 = (nargs_on_stack - 4) * 8 + * n32 = nargs_on_stack * 4 + * sp -= ROUNDUP(MAX(n32, n64), 16) + * + */ + /* n64 = (nargs_on_stack - 4) * 8 */ + sub x1, x6, #0x4 + lsl x1, x1, #3 + /* n32 = nargs_on_stack * 4 */ + lsl x0, x6, #2 + /* sp -= ROUNDUP(MAX(n32, n64), 16) */ + cmp x1, x0 + csel x0, x1, x0, ge + add x0, x0, #0xf + and x0, x0, #0xfffffffffffffff0 + sub sp, sp, x0 + + /* + * Find location on stack where to copy the Aarch32 arguments + * and do the copy. + * copy_from_user(sp, x5, nargs_on_stack * 4) + */ + mov x0, sp + mov x1, x5 + add x2, xzr, x6, lsl #2 + bl copy_from_user + /* If copy failed return the error */ + cmp x0, #0 + bne .Lret + + /* + * Load arguments into w4..w7, we're loading junk into unused + * registers, but it's quicker than trying to figure out how + * many registers to load into. + */ + /* x0 = nargs_on_stack */ + ldr x0, [x19, #SC_REC_X0] + ldr x0, [x0, #THREAD_SCALL_REG_X6] + load_wregs sp, 0, 4, 7 + + /* + * Convert remaining Aarch32 parameters passed on stack as Aarch64 + * parameters on stack. + * + * nargs_on_stack is initialized in x0 above + * n64 = (nargs_on_stack - 4) * 8 + * if n64 < 0 goro .Lno_args + * x0 = x2 = x19 - n64 + * x1 points to next argument + * while (x2 != x19) { + * w3 = *x1 + * x1 += 4 + * *x2 = x3 + * x2 += 8 + * } + * sp = x0 + */ + /* n64 = (nargs_on_stack - 4) * 8 */ + subs x2, x0, #0x4 + b.le .Lno_args_a32 + lsl x2, x2, #3 + mov x0, x2 + +.Lcpy_to_stack: + ldr w3, [x1], #4 + str x3, [x2], #8 + cmp x2, x19 + b.ne .Lcpy_to_stack + mov sp, x0 + + +.Lno_args_a32: /* Load the first 4 arguments to function */ + ldr x9, [x19, #SC_REC_X0] + load_xregs x9, THREAD_SCALL_REG_X0, 0, 3 + mov w0, w0 + mov w1, w1 + mov w2, w2 + mov w3, w3 + + /* Call the syscall function */ + ldr x16, [x19, #SC_REC_X1] + blr x16 + b .Lret + +.Lcall_a64: /* Load the first 8 arguments to function */ + ldr x9, [x19, #SC_REC_X0] + load_xregs x9, THREAD_SCALL_REG_X0, 0, 8 + + /* Call the syscall function */ + ldr x16, [x19, #SC_REC_X1] + blr x16 + +.Lret: + mov sp, x19 + ldp x19, x30, [sp, #SC_REC_X19] + add sp, sp, #SC_REC_SIZE + ret +END_FUNC scall_do_call + +/* + * syscall_sys_return() and syscall_panic() are two special cases for syscalls + * in the way that they do not return to the TA, instead execution is resumed + * as if __thread_enter_user_mode() had returned to thread_enter_user_mode(). + * + * In order to do this the functions need a way to get hold of a pointer to + * the struct thread_scall_regs provided by storing relevant registers on the + * stack in el0_svc() and later load them into registers again when el0_svc() + * is returning. + * + * scall_do_call() is supplied the pointer to struct thread_scall_regs in + * x0. This pointer can later be retrieved by chasing x19. + */ + +/* + * User space sees this function as: + * void syscall_sys_return(uint32_t ret) __noreturn; + * + * But internally the function depends on being called from + * scall_do_call() to be able to chase x19 in order to get hold of a + * pointer to struct thread_scall_regs. + * + * The argument ret is already in x0 so we don't touch that and let it + * propagate as return value of the called scall_sys_return_helper(). + */ +FUNC syscall_sys_return , : + mov x1, #0 /* panic = false */ + mov x2, #0 /* panic_code = 0 */ + ldr x3, [x19, #SC_REC_X0] /* pointer to struct thread_scall_regs */ + b scall_sys_return_helper +END_FUNC syscall_sys_return + +/* + * User space sees this function as: + * void syscall_panic(uint32_t code) __noreturn; + * + * But internally the function depends on being called from + * scall_do_call() to be able to chase x19 in order to get hold of a + * pointer to struct thread_scall_regs. + */ +FUNC syscall_panic , : + mov x1, #1 /* panic = true */ + mov x2, x0 /* code */ + ldr w0, =TEE_ERROR_TARGET_DEAD + ldr x3, [x19, #SC_REC_X0] /* pointer to struct thread_scall_regs */ + b scall_sys_return_helper +END_FUNC syscall_panic + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/arm32_gicv3_sysreg.txt b/optee_os/core/arch/arm/kernel/arm32_gicv3_sysreg.txt new file mode 100644 index 0000000..dd379e6 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/arm32_gicv3_sysreg.txt @@ -0,0 +1,40 @@ +# Format of file +# +# lines beginning with '@' will be printed as additional comments + +@ Based on register description in +@ ARM Generic Interrupt Controller +@ Architecture Specification +@ GIC architecture version 3.0 and version 4.0 + +@ Table 8-7 Mapping of MCR and MRC to physical and virtual CPU interface registers, AArch32 state +ICC_AP0R0 c12 0 c8 4 RW +ICC_AP0R1 c12 0 c8 5 RW +ICC_AP0R2 c12 0 c8 6 RW +ICC_AP0R3 c12 0 c8 7 RW +ICC_AP1R0 c12 0 c9 0 RW +ICC_AP1R1 c12 0 c9 1 RW +ICC_AP1R2 c12 0 c9 2 RW +ICC_AP1R3 c12 0 c9 3 RW +ICC_ASGI1R - 1 c12 - WO +ICC_BPR0 c12 0 c8 3 RW +ICC_BPR1 c12 0 c12 3 RW +ICC_CTLR c12 0 c12 4 RW +ICC_DIR c12 0 c11 1 WO +ICC_EOIR0 c12 0 c8 1 WO +ICC_EOIR1 c12 0 c12 1 WO +ICC_HPPIR0 c12 0 c8 2 RO +ICC_HPPIR1 c12 0 c12 2 RO +ICC_HSRE c12 4 c9 5 RW +ICC_IAR0 c12 0 c8 0 RO +ICC_IAR1 c12 0 c12 0 RO +ICC_IGRPEN0 c12 0 c12 6 RW +ICC_IGRPEN1 c12 0 c12 7 RW +ICC_MCTLR c12 6 c12 4 RW +ICC_MGRPEN1 c12 6 c12 7 RW +ICC_MSRE c12 6 c12 5 RW +ICC_PMR c4 0 c6 0 RW +ICC_RPR c12 0 c11 3 RO +ICC_SGI0R - 2 c12 - WO +ICC_SGI1R - 0 c12 - WO +ICC_SRE c12 0 c12 5 RW diff --git a/optee_os/core/arch/arm/kernel/arm32_sysreg.txt b/optee_os/core/arch/arm/kernel/arm32_sysreg.txt new file mode 100644 index 0000000..b69ebcb --- /dev/null +++ b/optee_os/core/arch/arm/kernel/arm32_sysreg.txt @@ -0,0 +1,146 @@ +# Format of file +# +# lines beginning with '@' will be printed as additional comments + +@ Based on register description in +@ ARM Architecture Reference Manual +@ ARMv7-A and ARMv7-R edition +@ Issue C.c +@ + +@ B3.18.1 Identification registers, functional group +AIDR c0 1 c0 7 RO IMPLEMENTATION DEFINED Auxiliary ID Register +CCSIDR c0 1 c0 0 RO Cache Size ID Registers +CLIDR c0 1 c0 1 RO Cache Level ID Register +CSSELR c0 2 c0 0 RW Cache Size Selection Register +CTR c0 0 c0 1 RO Cache Type Register +ID_AFR0 c0 0 c1 3 RO Auxiliary Feature Register 0 +ID_DFR0 c0 0 c1 2 RO Debug Feature Register 0 +ID_ISAR0 c0 0 c2 0 RO Instruction Set Attribute Register 0 +ID_ISAR1 c0 0 c2 1 RO Instruction Set Attribute Register 1 +ID_ISAR2 c0 0 c2 2 RO Instruction Set Attribute Register 2 +ID_ISAR3 c0 0 c2 3 RO Instruction Set Attribute Register 3 +ID_ISAR4 c0 0 c2 4 RO Instruction Set Attribute Register 4 +ID_ISAR5 c0 0 c2 5 RO Instruction Set Attribute Register 5 +ID_MMFR0 c0 0 c1 4 RO Memory Model Feature Register 0 +ID_MMFR1 c0 0 c1 5 RO Memory Model Feature Register 1 +ID_MMFR2 c0 0 c1 6 RO Memory Model Feature Register 2 +ID_MMFR3 c0 0 c1 7 RO Memory Model Feature Register 3 +ID_PFR0 c0 0 c1 0 RO Processor Feature Register 0 +ID_PFR1 c0 0 c1 1 RO Processor Feature Register 1 +MIDR c0 0 c0 0 RO Main ID Register +MPIDR c0 0 c0 5 RO Multiprocessor Affinity Register +REVIDR c0 0 c0 6 RO Revision ID Register +TCMTR c0 0 c0 2 RO TCM Type Register +TLBTR c0 0 c0 3 RO TLB Type Register + +@ B3.18.2 Virtual memory control registers, functional group +AMAIR0 c10 0 c3 0 RW Auxiliary Memory Attribute Indirection Register 0 +AMAIR1 c10 0 c3 1 RW Auxiliary Memory Attribute Indirection Register 1 +CONTEXTIDR c13 0 c0 1 RW Context ID Register +DACR c3 0 c0 0 RW Domain Access Control Register +MAIR0 c10 0 c2 0 RW Memory Attribute Indirection Register 0 +MAIR1 c10 0 c2 1 RW Memory Attribute Indirection Register 1 +NMRR c10 0 c2 1 RW Normal Memory Remap Register +PRRR c10 0 c2 0 RW Primary Region Remap Register +SCTLR c1 0 c0 0 RW System Control Register +TTBCR c2 0 c0 2 RW Translation Table Base Control Register +TTBR0 c2 0 c0 0 RW Translation Table Base Register 0 +TTBR0_64bit - 0 c2 - RW Translation Table Base Register 0 +TTBR1 c2 0 c0 1 RW Translation Table Base Register 1 +TTBR1_64bit - 1 c2 - RW Translation Table Base Register 1 + +@ B3.18.3 PL1 Fault handling registers, functional group +ADFSR c5 0 c1 0 RW Auxiliary Data Fault Status Register +AIFSR c5 0 c1 1 RW Auxiliary Instruction Fault Status Register +DFAR c6 0 c0 0 RW Data Fault Address Register +DFSR c5 0 c0 0 RW Data Fault Status Register +IFAR c6 0 c0 2 RW Instruction Fault Address Register +IFSR c5 0 c0 1 RW Instruction Fault Status Register + +@ B3.18.4 Other system control registers, functional group +ACTLR c1 0 c0 1 RW IMPLEMENTATION DEFINED Auxiliary Control Register +CPACR c1 0 c0 2 RW Coprocessor Access Control Register +FCSEIDR c13 0 c0 0 RW FCSE Process ID Register + +@ B3.18.6 Cache maintenance operations, functional group, VMSA +BPIALL c7 0 c5 6 WOD Branch predictor invalidate all +BPIALLIS c7 0 c1 6 WOD Branch predictor invalidate all IS +BPIMVA c7 0 c5 7 WO Branch predictor invalidate by MVA +DCCIMVAC c7 0 c14 1 WO Data cache clean and invalidate by MVA PoC +DCCISW c7 0 c14 2 WO Data cache clean and invalidate by set/way +DCCMVAC c7 0 c10 1 WO Data cache clean by MVA PoC +DCCMVAU c7 0 c11 1 WO Data cache clean by MVA PoU +DCCSW c7 0 c10 2 WO Data cache clean by set/way +DCIMVAC c7 0 c6 1 WO Data cache invalidate by MVA PoC +DCISW c7 0 c6 2 WO Data cache invalidate by set/way +ICIALLU c7 0 c5 0 WOD Instruction cache invalidate all PoU +ICIALLUIS c7 0 c1 0 WOD Instruction cache invalidate all PoU, IS +ICIMVAU c7 0 c5 1 WO Instruction cache invalidate by MVA PoU + +@ B3.18.7 TLB maintenance operations, functional group +TLBIALL c8 0 c7 0 WOD Invalidate entire unified TLB +TLBIALLIS c8 0 c3 0 WOD Invalidate entire unified TLB IS +TLBIASID c8 0 c7 2 WO Invalidate unified TLB by ASID +TLBIASIDIS c8 0 c3 2 WO Invalidate unified TLB by ASID IS +TLBIMVAA c8 0 c7 3 WO Invalidate unified TLB by MVA, all ASID +TLBIMVAAIS c8 0 c3 3 WO Invalidate unified TLB by MVA, all ASID IS +TLBIMVA c8 0 c7 1 WO Invalidate unified TLB by MVA +TLBIMVAIS c8 0 c3 1 WO Invalidate unified TLB by MVA IS + +@ B3.18.8 Address translation operations, functional group +ATS12NSOPR c7 0 c8 4 WO Stages 1 and 2 Non-secure only PL1 read +ATS12NSOPW c7 0 c8 5 WO Stages 1 and 2 Non-secure only PL1 write +ATS12NSOUR c7 0 c8 6 WO Stages 1 and 2 Non-secure only unprivileged read +ATS12NSOUW c7 0 c8 7 WO Stages 1 and 2 Non-secure only unprivileged write +ATS1CPR c7 0 c8 0 WO Stage 1 Current state PL1 read +ATS1CPW c7 0 c8 1 WO Stage 1 Current state PL1 write +ATS1CUR c7 0 c8 2 WO Stage 1 Current state unprivileged read +ATS1CUW c7 0 c8 3 WO Stage 1 Current state unprivileged write +ATS1HR c7 4 c8 0 WO Stage 1 Hyp mode read +ATS1HW c7 4 c8 1 WO Stage 1 Hyp mode write +PAR32 c7 0 c4 0 RW Physical Address Register +PAR64 - 0 c7 - RW Physical Address Register + +@ B3.18.9 Miscellaneous operations, functional group +TPIDRPRW c13 0 c0 4 RW PL1 only Thread ID Register +TPIDRURO c13 0 c0 3 RW PL0 User Read-Only Thread ID Register +TPIDRURW c13 0 c0 2 RW PL0 User Read/Write Thread ID Register + +@ B3.18.11 Security Extensions registers, functional group +ISR c12 0 c1 0 RO Interrupt Status Register +MVBAR c12 0 c0 1 RW Monitor Vector Base Address Register +NSACR c1 0 c1 2 RW Non-Secure Access Control Register +SCR c1 0 c1 0 RW Secure Configuration Register +SDER c1 0 c1 1 RW Secure Debug Enable Register +VBAR c12 0 c0 0 RW Vector Base Address Register + +@ B8.2 Generic Timer registers summary +CNTFRQ c14 0 c0 0 RW Counter Frequency register +CNTPCT - 0 c14 - RO Physical Count register +CNTKCTL c14 0 c1 0 RW Timer PL1 Control register +CNTP_TVAL c14 0 c2 0 RW PL1 Physical TimerValue register +CNTP_CTL c14 0 c2 1 RW PL1 Physical Timer Control register +CNTV_TVAL c14 0 c3 0 RW Virtual TimerValue register +CNTV_CTL c14 0 c3 1 RW Virtual Timer Control register +CNTVCT - 1 c14 - RO Virtual Count register +CNTP_CVAL - 2 c14 - RW PL1 Physical Timer CompareValue register +CNTV_CVAL - 3 c14 - RW Virtual Timer CompareValue register +CNTVOFF - 4 c14 - RW Virtual Offset register + +@ Table C12-7 Performance Monitors register summary +PMCR c9 0 c12 0 RW Performance Monitors Control Register +PMCNTENSET c9 0 c12 1 RW Performance Monitors Count Enable Set register +PMCNTENCLR c9 0 c12 2 RW Performance Monitors Count Enable Clear register +PMOVSR c9 0 c12 3 RW Performance Monitors Overflow Flag Status Register +PMSWINC c9 0 c12 4 WO Performance Monitors Software Increment register +PMSELR c9 0 c12 5 RW Performance Monitors Event Counter Selection Register +PMCEID0 c9 0 c12 6 RO Performance Monitors Common Event Identification reg 0 +PMCEID1 c9 0 c12 7 RO Performance Monitors Common Event Identification reg 1 +PMCCNTR c9 0 c13 0 RW Performance Monitors Cycle Count Register +PMXEVTYPER c9 0 c13 1 RW Performance Monitors Event Type Select Register +PMXEVCNTR c9 0 c13 2 RW Performance Monitors Event Count Register +PMUSERENR c9 0 c14 0 RW Performance Monitors User Enable Register +PMINTENSET c9 0 c14 1 RW Performance Monitors Interrupt Enable Set register +PMINTENCLR c9 0 c14 2 RW Performance Monitors Interrupt Enable Clear register +PMOVSSET c9 0 c14 3 RW Performance Monitors Overflow Flag Status Set register diff --git a/optee_os/core/arch/arm/kernel/asm-defines.c b/optee_os/core/arch/arm/kernel/asm-defines.c new file mode 100644 index 0000000..1bafd6d --- /dev/null +++ b/optee_os/core/arch/arm/kernel/asm-defines.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINES +{ +#ifdef ARM32 + DEFINE(SM_NSEC_CTX_R0, offsetof(struct sm_nsec_ctx, r0)); + DEFINE(SM_NSEC_CTX_R8, offsetof(struct sm_nsec_ctx, r8)); + DEFINE(SM_SEC_CTX_R0, offsetof(struct sm_sec_ctx, r0)); + DEFINE(SM_SEC_CTX_MON_LR, offsetof(struct sm_sec_ctx, mon_lr)); + DEFINE(SM_CTX_SEC_SIZE, sizeof(struct sm_sec_ctx)); + DEFINE(SM_CTX_SIZE, sizeof(struct sm_ctx)); + DEFINE(SM_CTX_NSEC, offsetof(struct sm_ctx, nsec)); + DEFINE(SM_CTX_SEC, offsetof(struct sm_ctx, sec)); + + DEFINE(THREAD_SCALL_REG_R0, offsetof(struct thread_scall_regs, r0)); + DEFINE(THREAD_SCALL_REG_R5, offsetof(struct thread_scall_regs, r5)); + DEFINE(THREAD_SCALL_REG_R6, offsetof(struct thread_scall_regs, r6)); + + /* struct thread_ctx */ + DEFINE(THREAD_CTX_STACK_VA_END, offsetof(struct thread_ctx, + stack_va_end)); + + /* struct thread_ctx_regs */ + DEFINE(THREAD_CTX_REGS_USR_SP, + offsetof(struct thread_ctx_regs, usr_sp)); + DEFINE(THREAD_CTX_REGS_PC, offsetof(struct thread_ctx_regs, pc)); + DEFINE(THREAD_CTX_REGS_CPSR, offsetof(struct thread_ctx_regs, cpsr)); + + /* struct thread_core_local */ + DEFINE(THREAD_CORE_LOCAL_R0, offsetof(struct thread_core_local, r[0])); + DEFINE(THREAD_CORE_LOCAL_SM_PM_CTX_PHYS, + offsetof(struct thread_core_local, sm_pm_ctx_phys)); + DEFINE(THREAD_CORE_LOCAL_SIZE, sizeof(struct thread_core_local)); + + DEFINE(SM_PM_CTX_SIZE, sizeof(struct sm_pm_ctx)); +#endif /*ARM32*/ + +#ifdef ARM64 + DEFINE(THREAD_SMC_ARGS_X0, offsetof(struct thread_smc_args, a0)); + DEFINE(THREAD_SMC_ARGS_SIZE, sizeof(struct thread_smc_args)); + + DEFINE(THREAD_SCALL_REG_X0, offsetof(struct thread_scall_regs, x0)); + DEFINE(THREAD_SCALL_REG_X2, offsetof(struct thread_scall_regs, x2)); + DEFINE(THREAD_SCALL_REG_X5, offsetof(struct thread_scall_regs, x5)); + DEFINE(THREAD_SCALL_REG_X6, offsetof(struct thread_scall_regs, x6)); + DEFINE(THREAD_SCALL_REG_X30, offsetof(struct thread_scall_regs, x30)); + DEFINE(THREAD_SCALL_REG_ELR, offsetof(struct thread_scall_regs, elr)); + DEFINE(THREAD_SCALL_REG_SPSR, offsetof(struct thread_scall_regs, spsr)); + DEFINE(THREAD_SCALL_REG_SP_EL0, offsetof(struct thread_scall_regs, + sp_el0)); +#ifdef CFG_TA_PAUTH + DEFINE(THREAD_SCALL_REG_APIAKEY_HI, offsetof(struct thread_scall_regs, + apiakey_hi)); +#endif + DEFINE(THREAD_SCALL_REG_SIZE, sizeof(struct thread_scall_regs)); + + /* struct thread_abort_regs */ + DEFINE(THREAD_ABT_REG_X0, offsetof(struct thread_abort_regs, x0)); + DEFINE(THREAD_ABT_REG_X2, offsetof(struct thread_abort_regs, x2)); + DEFINE(THREAD_ABT_REG_X30, offsetof(struct thread_abort_regs, x30)); + DEFINE(THREAD_ABT_REG_SPSR, offsetof(struct thread_abort_regs, spsr)); + DEFINE(THREAD_ABT_REGS_SIZE, sizeof(struct thread_abort_regs)); +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + DEFINE(THREAD_ABT_REGS_APIAKEY_HI, offsetof(struct thread_abort_regs, + apiakey_hi)); +#endif + + /* struct thread_ctx */ + DEFINE(THREAD_CTX_KERN_SP, offsetof(struct thread_ctx, kern_sp)); + DEFINE(THREAD_CTX_STACK_VA_END, offsetof(struct thread_ctx, + stack_va_end)); +#if defined(CFG_CORE_PAUTH) + DEFINE(THREAD_CTX_KEYS, offsetof(struct thread_ctx, keys)); +#endif + + /* struct thread_ctx_regs */ + DEFINE(THREAD_CTX_REGS_SP, offsetof(struct thread_ctx_regs, sp)); + DEFINE(THREAD_CTX_REGS_X0, offsetof(struct thread_ctx_regs, x[0])); + DEFINE(THREAD_CTX_REGS_X1, offsetof(struct thread_ctx_regs, x[1])); + DEFINE(THREAD_CTX_REGS_X2, offsetof(struct thread_ctx_regs, x[2])); + DEFINE(THREAD_CTX_REGS_X4, offsetof(struct thread_ctx_regs, x[4])); + DEFINE(THREAD_CTX_REGS_X19, offsetof(struct thread_ctx_regs, x[19])); + DEFINE(THREAD_CTX_REGS_TPIDR_EL0, offsetof(struct thread_ctx_regs, + tpidr_el0)); +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + DEFINE(THREAD_CTX_REGS_APIAKEY_HI, offsetof(struct thread_ctx_regs, + apiakey_hi)); +#endif + + /* struct thread_user_mode_rec */ + DEFINE(THREAD_USER_MODE_REC_CTX_REGS_PTR, + offsetof(struct thread_user_mode_rec, ctx_regs_ptr)); + DEFINE(THREAD_USER_MODE_REC_EXIT_STATUS0_PTR, + offsetof(struct thread_user_mode_rec, exit_status0_ptr)); + DEFINE(THREAD_USER_MODE_REC_X19, + offsetof(struct thread_user_mode_rec, x[0])); + DEFINE(THREAD_USER_MODE_REC_SIZE, sizeof(struct thread_user_mode_rec)); + + /* struct thread_core_local */ + DEFINE(THREAD_CORE_LOCAL_X0, offsetof(struct thread_core_local, x[0])); + DEFINE(THREAD_CORE_LOCAL_X2, offsetof(struct thread_core_local, x[2])); + DEFINE(THREAD_CORE_LOCAL_KCODE_OFFSET, + offsetof(struct thread_core_local, kcode_offset)); +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC + DEFINE(THREAD_CORE_LOCAL_BHB_LOOP_COUNT, + offsetof(struct thread_core_local, bhb_loop_count)); +#endif +#if defined(CFG_CORE_PAUTH) + DEFINE(THREAD_CORE_LOCAL_KEYS, + offsetof(struct thread_core_local, keys)); +#endif +#endif /*ARM64*/ + + /* struct thread_ctx */ + DEFINE(THREAD_CTX_SIZE, sizeof(struct thread_ctx)); +#ifdef CFG_CORE_FFA + DEFINE(THREAD_CTX_TSD_RPC_TARGET_INFO, + offsetof(struct thread_ctx, tsd.rpc_target_info)) + DEFINE(THREAD_CTX_FLAGS, + offsetof(struct thread_ctx, flags)) +#endif + + /* struct thread_core_local */ + DEFINE(THREAD_CORE_LOCAL_TMP_STACK_VA_END, + offsetof(struct thread_core_local, tmp_stack_va_end)); + DEFINE(THREAD_CORE_LOCAL_CURR_THREAD, + offsetof(struct thread_core_local, curr_thread)); + DEFINE(THREAD_CORE_LOCAL_FLAGS, + offsetof(struct thread_core_local, flags)); + DEFINE(THREAD_CORE_LOCAL_ABT_STACK_VA_END, + offsetof(struct thread_core_local, abt_stack_va_end)); +#if defined(ARM64) && defined(CFG_CORE_FFA) + DEFINE(THREAD_CORE_LOCAL_DIRECT_RESP_FID, + offsetof(struct thread_core_local, direct_resp_fid)); +#endif + + DEFINE(STACK_TMP_GUARD, STACK_CANARY_SIZE / 2 + STACK_TMP_OFFS); + + /* struct core_mmu_config */ + DEFINE(CORE_MMU_CONFIG_SIZE, sizeof(struct core_mmu_config)); + DEFINE(CORE_MMU_CONFIG_MAP_OFFSET, + offsetof(struct core_mmu_config, map_offset)); + + /* struct boot_embdata */ + DEFINE(BOOT_EMBDATA_HASHES_OFFSET, + offsetof(struct boot_embdata, hashes_offset)); + DEFINE(BOOT_EMBDATA_HASHES_LEN, + offsetof(struct boot_embdata, hashes_len)); + DEFINE(BOOT_EMBDATA_RELOC_OFFSET, + offsetof(struct boot_embdata, reloc_offset)); + DEFINE(BOOT_EMBDATA_RELOC_LEN, + offsetof(struct boot_embdata, reloc_len)); + +#ifdef CORE_MMU_BASE_TABLE_OFFSET + /* + * This define is too complex to be used as an argument for the + * macros add_imm and sub_imm so evaluate it here. + */ + DEFINE(__CORE_MMU_BASE_TABLE_OFFSET, CORE_MMU_BASE_TABLE_OFFSET); +#endif + +} diff --git a/optee_os/core/arch/arm/kernel/boot.c b/optee_os/core/arch/arm/kernel/boot.c new file mode 100644 index 0000000..9a13224 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/boot.c @@ -0,0 +1,1456 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2023, Linaro Limited + * Copyright (c) 2023, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if !defined(CFG_WITH_ARM_TRUSTED_FW) +#include +#endif + +#if defined(CFG_WITH_VFP) +#include +#endif + +/* + * In this file we're using unsigned long to represent physical pointers as + * they are received in a single register when OP-TEE is initially entered. + * This limits 32-bit systems to only use make use of the lower 32 bits + * of a physical address for initial parameters. + * + * 64-bit systems on the other hand can use full 64-bit physical pointers. + */ +#define PADDR_INVALID ULONG_MAX + +#if defined(CFG_BOOT_SECONDARY_REQUEST) +struct ns_entry_context { + uintptr_t entry_point; + uintptr_t context_id; +}; +struct ns_entry_context ns_entry_contexts[CFG_TEE_CORE_NB_CORE]; +static uint32_t spin_table[CFG_TEE_CORE_NB_CORE]; +#endif + +#ifdef CFG_BOOT_SYNC_CPU +/* + * Array used when booting, to synchronize cpu. + * When 0, the cpu has not started. + * When 1, it has started + */ +uint32_t sem_cpu_sync[CFG_TEE_CORE_NB_CORE]; +DECLARE_KEEP_PAGER(sem_cpu_sync); +#endif + +#ifdef CFG_CORE_FFA +static void *manifest_dt __nex_bss; +#endif + +#ifdef CFG_SECONDARY_INIT_CNTFRQ +static uint32_t cntfrq; +#endif + +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak void plat_primary_init_early(void) +{ +} +DECLARE_KEEP_PAGER(plat_primary_init_early); + +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak void boot_primary_init_intc(void) +{ +} + +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak void boot_secondary_init_intc(void) +{ +} + +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak unsigned long plat_get_aslr_seed(void) +{ + DMSG("Warning: no ASLR seed"); + + return 0; +} + +#if defined(_CFG_CORE_STACK_PROTECTOR) || defined(CFG_WITH_STACK_CANARIES) +/* Generate random stack canary value on boot up */ +__weak void plat_get_random_stack_canaries(void *buf, size_t ncan, size_t size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t i = 0; + + assert(buf && ncan && size); + + /* + * With virtualization the RNG is not initialized in Nexus core. + * Need to override with platform specific implementation. + */ + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + IMSG("WARNING: Using fixed value for stack canary"); + memset(buf, 0xab, ncan * size); + goto out; + } + + ret = crypto_rng_read(buf, ncan * size); + if (ret != TEE_SUCCESS) + panic("Failed to generate random stack canary"); + +out: + /* Leave null byte in canary to prevent string base exploit */ + for (i = 0; i < ncan; i++) + *((uint8_t *)buf + size * i) = 0; +} +#endif /* _CFG_CORE_STACK_PROTECTOR || CFG_WITH_STACK_CANARIES */ + +/* + * This function is called as a guard after each smc call which is not + * supposed to return. + */ +void __panic_at_smc_return(void) +{ + panic(); +} + +#if defined(CFG_WITH_ARM_TRUSTED_FW) +void init_sec_mon(unsigned long nsec_entry __maybe_unused) +{ + assert(nsec_entry == PADDR_INVALID); + /* Do nothing as we don't have a secure monitor */ +} +#else +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak void init_sec_mon(unsigned long nsec_entry) +{ + struct sm_nsec_ctx *nsec_ctx; + + assert(nsec_entry != PADDR_INVALID); + + /* Initialize secure monitor */ + nsec_ctx = sm_get_nsec_ctx(); + nsec_ctx->mon_lr = nsec_entry; + nsec_ctx->mon_spsr = CPSR_MODE_SVC | CPSR_I; + if (nsec_entry & 1) + nsec_ctx->mon_spsr |= CPSR_T; +} +#endif + +#if defined(CFG_WITH_ARM_TRUSTED_FW) +static void init_vfp_nsec(void) +{ +} +#else +static void init_vfp_nsec(void) +{ + /* Normal world can use CP10 and CP11 (SIMD/VFP) */ + write_nsacr(read_nsacr() | NSACR_CP10 | NSACR_CP11); +} +#endif + +#if defined(CFG_WITH_VFP) + +#ifdef ARM32 +static void init_vfp_sec(void) +{ + uint32_t cpacr = read_cpacr(); + + /* + * Enable Advanced SIMD functionality. + * Enable use of D16-D31 of the Floating-point Extension register + * file. + */ + cpacr &= ~(CPACR_ASEDIS | CPACR_D32DIS); + /* + * Enable usage of CP10 and CP11 (SIMD/VFP) (both kernel and user + * mode. + */ + cpacr |= CPACR_CP(10, CPACR_CP_ACCESS_FULL); + cpacr |= CPACR_CP(11, CPACR_CP_ACCESS_FULL); + write_cpacr(cpacr); +} +#endif /* ARM32 */ + +#ifdef ARM64 +static void init_vfp_sec(void) +{ + /* Not using VFP until thread_kernel_enable_vfp() */ + vfp_disable(); +} +#endif /* ARM64 */ + +#else /* CFG_WITH_VFP */ + +static void init_vfp_sec(void) +{ + /* Not using VFP */ +} +#endif + +#ifdef CFG_SECONDARY_INIT_CNTFRQ +static void primary_save_cntfrq(void) +{ + assert(cntfrq == 0); + + /* + * CNTFRQ should be initialized on the primary CPU by a + * previous boot stage + */ + cntfrq = read_cntfrq(); +} + +static void secondary_init_cntfrq(void) +{ + assert(cntfrq != 0); + write_cntfrq(cntfrq); +} +#else /* CFG_SECONDARY_INIT_CNTFRQ */ +static void primary_save_cntfrq(void) +{ +} + +static void secondary_init_cntfrq(void) +{ +} +#endif + +#ifdef CFG_CORE_SANITIZE_KADDRESS +static void init_run_constructors(void) +{ + const vaddr_t *ctor; + + for (ctor = &__ctor_list; ctor < &__ctor_end; ctor++) + ((void (*)(void))(*ctor))(); +} + +static void init_asan(void) +{ + + /* + * CFG_ASAN_SHADOW_OFFSET is also supplied as + * -fasan-shadow-offset=$(CFG_ASAN_SHADOW_OFFSET) to the compiler. + * Since all the needed values to calculate the value of + * CFG_ASAN_SHADOW_OFFSET isn't available in to make we need to + * calculate it in advance and hard code it into the platform + * conf.mk. Here where we have all the needed values we double + * check that the compiler is supplied the correct value. + */ + +#define __ASAN_SHADOW_START \ + ROUNDUP(TEE_RAM_START + (TEE_RAM_VA_SIZE * 8) / 9 - 8, 8) + assert(__ASAN_SHADOW_START == (vaddr_t)&__asan_shadow_start); +#define __CFG_ASAN_SHADOW_OFFSET \ + (__ASAN_SHADOW_START - (TEE_RAM_START / 8)) + COMPILE_TIME_ASSERT(CFG_ASAN_SHADOW_OFFSET == __CFG_ASAN_SHADOW_OFFSET); +#undef __ASAN_SHADOW_START +#undef __CFG_ASAN_SHADOW_OFFSET + + /* + * Assign area covered by the shadow area, everything from start up + * to the beginning of the shadow area. + */ + asan_set_shadowed((void *)TEE_LOAD_ADDR, &__asan_shadow_start); + + /* + * Add access to areas that aren't opened automatically by a + * constructor. + */ + asan_tag_access(&__ctor_list, &__ctor_end); + asan_tag_access(__rodata_start, __rodata_end); +#ifdef CFG_WITH_PAGER + asan_tag_access(__pageable_start, __pageable_end); +#endif /*CFG_WITH_PAGER*/ + asan_tag_access(__nozi_start, __nozi_end); +#ifdef ARM32 + asan_tag_access(__exidx_start, __exidx_end); + asan_tag_access(__extab_start, __extab_end); +#endif + + init_run_constructors(); + + /* Everything is tagged correctly, let's start address sanitizing. */ + asan_start(); +} +#else /*CFG_CORE_SANITIZE_KADDRESS*/ +static void init_asan(void) +{ +} +#endif /*CFG_CORE_SANITIZE_KADDRESS*/ + +#if defined(CFG_MEMTAG) +/* Called from entry_a64.S only when MEMTAG is configured */ +void boot_init_memtag(void) +{ + paddr_t base = 0; + paddr_size_t size = 0; + + memtag_init_ops(feat_mte_implemented()); + core_mmu_get_secure_memory(&base, &size); + memtag_set_tags((void *)(vaddr_t)base, size, 0); +} +#endif + +#ifdef CFG_WITH_PAGER + +#ifdef CFG_CORE_SANITIZE_KADDRESS +static void carve_out_asan_mem(tee_mm_pool_t *pool) +{ + const size_t s = pool->hi - pool->lo; + tee_mm_entry_t *mm; + paddr_t apa = ASAN_MAP_PA; + size_t asz = ASAN_MAP_SZ; + + if (core_is_buffer_outside(apa, asz, pool->lo, s)) + return; + + /* Reserve the shadow area */ + if (!core_is_buffer_inside(apa, asz, pool->lo, s)) { + if (apa < pool->lo) { + /* + * ASAN buffer is overlapping with the beginning of + * the pool. + */ + asz -= pool->lo - apa; + apa = pool->lo; + } else { + /* + * ASAN buffer is overlapping with the end of the + * pool. + */ + asz = pool->hi - apa; + } + } + mm = tee_mm_alloc2(pool, apa, asz); + assert(mm); +} +#else +static void carve_out_asan_mem(tee_mm_pool_t *pool __unused) +{ +} +#endif + +static void print_pager_pool_size(void) +{ + struct tee_pager_stats __maybe_unused stats; + + tee_pager_get_stats(&stats); + IMSG("Pager pool size: %zukB", + stats.npages_all * SMALL_PAGE_SIZE / 1024); +} + +static void init_vcore(tee_mm_pool_t *mm_vcore) +{ + const vaddr_t begin = VCORE_START_VA; + size_t size = TEE_RAM_VA_SIZE; + +#ifdef CFG_CORE_SANITIZE_KADDRESS + /* Carve out asan memory, flat maped after core memory */ + if (begin + size > ASAN_SHADOW_PA) + size = ASAN_MAP_PA - begin; +#endif + + if (!tee_mm_init(mm_vcore, begin, size, SMALL_PAGE_SHIFT, + TEE_MM_POOL_NO_FLAGS)) + panic("tee_mm_vcore init failed"); +} + +/* + * With CFG_CORE_ASLR=y the init part is relocated very early during boot. + * The init part is also paged just as the rest of the normal paged code, with + * the difference that it's preloaded during boot. When the backing store + * is configured the entire paged binary is copied in place and then also + * the init part. Since the init part has been relocated (references to + * addresses updated to compensate for the new load address) this has to be + * undone for the hashes of those pages to match with the original binary. + * + * If CFG_CORE_ASLR=n, nothing needs to be done as the code/ro pages are + * unchanged. + */ +static void undo_init_relocation(uint8_t *paged_store __maybe_unused) +{ +#ifdef CFG_CORE_ASLR + unsigned long *ptr = NULL; + const uint32_t *reloc = NULL; + const uint32_t *reloc_end = NULL; + unsigned long offs = boot_mmu_config.map_offset; + const struct boot_embdata *embdata = (const void *)__init_end; + vaddr_t addr_end = (vaddr_t)__init_end - offs - TEE_LOAD_ADDR; + vaddr_t addr_start = (vaddr_t)__init_start - offs - TEE_LOAD_ADDR; + + reloc = (const void *)((vaddr_t)embdata + embdata->reloc_offset); + reloc_end = reloc + embdata->reloc_len / sizeof(*reloc); + + for (; reloc < reloc_end; reloc++) { + if (*reloc < addr_start) + continue; + if (*reloc >= addr_end) + break; + ptr = (void *)(paged_store + *reloc - addr_start); + *ptr -= offs; + } +#endif +} + +static struct fobj *ro_paged_alloc(tee_mm_entry_t *mm, void *hashes, + void *store) +{ + const unsigned int num_pages = tee_mm_get_bytes(mm) / SMALL_PAGE_SIZE; +#ifdef CFG_CORE_ASLR + unsigned int reloc_offs = (vaddr_t)__pageable_start - VCORE_START_VA; + const struct boot_embdata *embdata = (const void *)__init_end; + const void *reloc = __init_end + embdata->reloc_offset; + + return fobj_ro_reloc_paged_alloc(num_pages, hashes, reloc_offs, + reloc, embdata->reloc_len, store); +#else + return fobj_ro_paged_alloc(num_pages, hashes, store); +#endif +} + +static void init_runtime(unsigned long pageable_part) +{ + size_t n; + size_t init_size = (size_t)(__init_end - __init_start); + size_t pageable_start = (size_t)__pageable_start; + size_t pageable_end = (size_t)__pageable_end; + size_t pageable_size = pageable_end - pageable_start; + vaddr_t tzsram_end = TZSRAM_BASE + TZSRAM_SIZE - TEE_LOAD_ADDR + + VCORE_START_VA; + size_t hash_size = (pageable_size / SMALL_PAGE_SIZE) * + TEE_SHA256_HASH_SIZE; + const struct boot_embdata *embdata = (const void *)__init_end; + const void *tmp_hashes = NULL; + tee_mm_entry_t *mm = NULL; + struct fobj *fobj = NULL; + uint8_t *paged_store = NULL; + uint8_t *hashes = NULL; + + assert(pageable_size % SMALL_PAGE_SIZE == 0); + assert(embdata->total_len >= embdata->hashes_offset + + embdata->hashes_len); + assert(hash_size == embdata->hashes_len); + + tmp_hashes = __init_end + embdata->hashes_offset; + + init_asan(); + + /* Add heap2 first as heap1 may be too small as initial bget pool */ + malloc_add_pool(__heap2_start, __heap2_end - __heap2_start); + malloc_add_pool(__heap1_start, __heap1_end - __heap1_start); + + /* + * This needs to be initialized early to support address lookup + * in MEM_AREA_TEE_RAM + */ + tee_pager_early_init(); + + hashes = malloc(hash_size); + IMSG_RAW("\n"); + IMSG("Pager is enabled. Hashes: %zu bytes", hash_size); + assert(hashes); + asan_memcpy_unchecked(hashes, tmp_hashes, hash_size); + + /* + * Need tee_mm_sec_ddr initialized to be able to allocate secure + * DDR below. + */ + core_mmu_init_ta_ram(); + + carve_out_asan_mem(&tee_mm_sec_ddr); + + mm = tee_mm_alloc(&tee_mm_sec_ddr, pageable_size); + assert(mm); + paged_store = phys_to_virt(tee_mm_get_smem(mm), MEM_AREA_TA_RAM, + pageable_size); + /* + * Load pageable part in the dedicated allocated area: + * - Move pageable non-init part into pageable area. Note bootloader + * may have loaded it anywhere in TA RAM hence use memmove(). + * - Copy pageable init part from current location into pageable area. + */ + memmove(paged_store + init_size, + phys_to_virt(pageable_part, + core_mmu_get_type_by_pa(pageable_part), + __pageable_part_end - __pageable_part_start), + __pageable_part_end - __pageable_part_start); + asan_memcpy_unchecked(paged_store, __init_start, init_size); + /* + * Undo eventual relocation for the init part so the hash checks + * can pass. + */ + undo_init_relocation(paged_store); + + /* Check that hashes of what's in pageable area is OK */ + DMSG("Checking hashes of pageable area"); + for (n = 0; (n * SMALL_PAGE_SIZE) < pageable_size; n++) { + const uint8_t *hash = hashes + n * TEE_SHA256_HASH_SIZE; + const uint8_t *page = paged_store + n * SMALL_PAGE_SIZE; + TEE_Result res; + + DMSG("hash pg_idx %zu hash %p page %p", n, hash, page); + res = hash_sha256_check(hash, page, SMALL_PAGE_SIZE); + if (res != TEE_SUCCESS) { + EMSG("Hash failed for page %zu at %p: res 0x%x", + n, (void *)page, res); + panic(); + } + } + + /* + * Assert prepaged init sections are page aligned so that nothing + * trails uninited at the end of the premapped init area. + */ + assert(!(init_size & SMALL_PAGE_MASK)); + + /* + * Initialize the virtual memory pool used for main_mmu_l2_ttb which + * is supplied to tee_pager_init() below. + */ + init_vcore(&tee_mm_vcore); + + /* + * Assign alias area for pager end of the small page block the rest + * of the binary is loaded into. We're taking more than needed, but + * we're guaranteed to not need more than the physical amount of + * TZSRAM. + */ + mm = tee_mm_alloc2(&tee_mm_vcore, + (vaddr_t)tee_mm_vcore.lo + + tee_mm_vcore.size - TZSRAM_SIZE, + TZSRAM_SIZE); + assert(mm); + tee_pager_set_alias_area(mm); + + /* + * Claim virtual memory which isn't paged. + * Linear memory (flat map core memory) ends there. + */ + mm = tee_mm_alloc2(&tee_mm_vcore, VCORE_UNPG_RX_PA, + (vaddr_t)(__pageable_start - VCORE_UNPG_RX_PA)); + assert(mm); + + /* + * Allocate virtual memory for the pageable area and let the pager + * take charge of all the pages already assigned to that memory. + */ + mm = tee_mm_alloc2(&tee_mm_vcore, (vaddr_t)__pageable_start, + pageable_size); + assert(mm); + fobj = ro_paged_alloc(mm, hashes, paged_store); + assert(fobj); + tee_pager_add_core_region(tee_mm_get_smem(mm), PAGED_REGION_TYPE_RO, + fobj); + fobj_put(fobj); + + tee_pager_add_pages(pageable_start, init_size / SMALL_PAGE_SIZE, false); + tee_pager_add_pages(pageable_start + init_size, + (pageable_size - init_size) / SMALL_PAGE_SIZE, + true); + if (pageable_end < tzsram_end) + tee_pager_add_pages(pageable_end, (tzsram_end - pageable_end) / + SMALL_PAGE_SIZE, true); + + /* + * There may be physical pages in TZSRAM before the core load address. + * These pages can be added to the physical pages pool of the pager. + * This setup may happen when a the secure bootloader runs in TZRAM + * and its memory can be reused by OP-TEE once boot stages complete. + */ + tee_pager_add_pages(tee_mm_vcore.lo, + (VCORE_UNPG_RX_PA - tee_mm_vcore.lo) / SMALL_PAGE_SIZE, + true); + + print_pager_pool_size(); +} +#else + +static void init_runtime(unsigned long pageable_part __unused) +{ + init_asan(); + + /* + * By default whole OP-TEE uses malloc, so we need to initialize + * it early. But, when virtualization is enabled, malloc is used + * only by TEE runtime, so malloc should be initialized later, for + * every virtual partition separately. Core code uses nex_malloc + * instead. + */ +#ifdef CFG_NS_VIRTUALIZATION + nex_malloc_add_pool(__nex_heap_start, __nex_heap_end - + __nex_heap_start); +#else + malloc_add_pool(__heap1_start, __heap1_end - __heap1_start); +#endif + + IMSG_RAW("\n"); +} +#endif + +#if defined(CFG_DT) +static int add_optee_dt_node(struct dt_descriptor *dt) +{ + int offs; + int ret; + + if (fdt_path_offset(dt->blob, "/firmware/optee") >= 0) { + DMSG("OP-TEE Device Tree node already exists!"); + return 0; + } + + offs = fdt_path_offset(dt->blob, "/firmware"); + if (offs < 0) { + offs = add_dt_path_subnode(dt, "/", "firmware"); + if (offs < 0) + return -1; + } + + offs = fdt_add_subnode(dt->blob, offs, "optee"); + if (offs < 0) + return -1; + + ret = fdt_setprop_string(dt->blob, offs, "compatible", + "linaro,optee-tz"); + if (ret < 0) + return -1; + ret = fdt_setprop_string(dt->blob, offs, "method", "smc"); + if (ret < 0) + return -1; + + if (CFG_CORE_ASYNC_NOTIF_GIC_INTID) { + /* + * The format of the interrupt property is defined by the + * binding of the interrupt domain root. In this case it's + * one Arm GIC v1, v2 or v3 so we must be compatible with + * these. + * + * An SPI type of interrupt is indicated with a 0 in the + * first cell. A PPI type is indicated with value 1. + * + * The interrupt number goes in the second cell where + * SPIs ranges from 0 to 987 and PPI ranges from 0 to 15. + * + * Flags are passed in the third cells. + */ + uint32_t itr_trigger = 0; + uint32_t itr_type = 0; + uint32_t itr_id = 0; + uint32_t val[3] = { }; + + /* PPI are visible only in current CPU cluster */ + static_assert(!CFG_CORE_ASYNC_NOTIF_GIC_INTID || + (CFG_CORE_ASYNC_NOTIF_GIC_INTID >= + GIC_SPI_BASE) || + ((CFG_TEE_CORE_NB_CORE <= 8) && + (CFG_CORE_ASYNC_NOTIF_GIC_INTID >= + GIC_PPI_BASE))); + + if (CFG_CORE_ASYNC_NOTIF_GIC_INTID >= GIC_SPI_BASE) { + itr_type = GIC_SPI; + itr_id = CFG_CORE_ASYNC_NOTIF_GIC_INTID - GIC_SPI_BASE; + itr_trigger = IRQ_TYPE_EDGE_RISING; + } else { + itr_type = GIC_PPI; + itr_id = CFG_CORE_ASYNC_NOTIF_GIC_INTID - GIC_PPI_BASE; + itr_trigger = IRQ_TYPE_EDGE_RISING | + GIC_CPU_MASK_SIMPLE(CFG_TEE_CORE_NB_CORE); + } + + val[0] = TEE_U32_TO_BIG_ENDIAN(itr_type); + val[1] = TEE_U32_TO_BIG_ENDIAN(itr_id); + val[2] = TEE_U32_TO_BIG_ENDIAN(itr_trigger); + + ret = fdt_setprop(dt->blob, offs, "interrupts", val, + sizeof(val)); + if (ret < 0) + return -1; + } + return 0; +} + +#ifdef CFG_PSCI_ARM32 +static int append_psci_compatible(void *fdt, int offs, const char *str) +{ + return fdt_appendprop(fdt, offs, "compatible", str, strlen(str) + 1); +} + +static int dt_add_psci_node(struct dt_descriptor *dt) +{ + int offs; + + if (fdt_path_offset(dt->blob, "/psci") >= 0) { + DMSG("PSCI Device Tree node already exists!"); + return 0; + } + + offs = add_dt_path_subnode(dt, "/", "psci"); + if (offs < 0) + return -1; + if (append_psci_compatible(dt->blob, offs, "arm,psci-1.0")) + return -1; + if (append_psci_compatible(dt->blob, offs, "arm,psci-0.2")) + return -1; + if (append_psci_compatible(dt->blob, offs, "arm,psci")) + return -1; + if (fdt_setprop_string(dt->blob, offs, "method", "smc")) + return -1; + if (fdt_setprop_u32(dt->blob, offs, "cpu_suspend", PSCI_CPU_SUSPEND)) + return -1; + if (fdt_setprop_u32(dt->blob, offs, "cpu_off", PSCI_CPU_OFF)) + return -1; + if (fdt_setprop_u32(dt->blob, offs, "cpu_on", PSCI_CPU_ON)) + return -1; + if (fdt_setprop_u32(dt->blob, offs, "sys_poweroff", PSCI_SYSTEM_OFF)) + return -1; + if (fdt_setprop_u32(dt->blob, offs, "sys_reset", PSCI_SYSTEM_RESET)) + return -1; + return 0; +} + +static int check_node_compat_prefix(struct dt_descriptor *dt, int offs, + const char *prefix) +{ + const size_t prefix_len = strlen(prefix); + size_t l; + int plen; + const char *prop; + + prop = fdt_getprop(dt->blob, offs, "compatible", &plen); + if (!prop) + return -1; + + while (plen > 0) { + if (memcmp(prop, prefix, prefix_len) == 0) + return 0; /* match */ + + l = strlen(prop) + 1; + prop += l; + plen -= l; + } + + return -1; +} + +static int dt_add_psci_cpu_enable_methods(struct dt_descriptor *dt) +{ + int offs = 0; + + while (1) { + offs = fdt_next_node(dt->blob, offs, NULL); + if (offs < 0) + break; + if (fdt_getprop(dt->blob, offs, "enable-method", NULL)) + continue; /* already set */ + if (check_node_compat_prefix(dt, offs, "arm,cortex-a")) + continue; /* no compatible */ + if (fdt_setprop_string(dt->blob, offs, "enable-method", "psci")) + return -1; + /* Need to restart scanning as offsets may have changed */ + offs = 0; + } + return 0; +} + +static int config_psci(struct dt_descriptor *dt) +{ + if (dt_add_psci_node(dt)) + return -1; + return dt_add_psci_cpu_enable_methods(dt); +} +#else +static int config_psci(struct dt_descriptor *dt __unused) +{ + return 0; +} +#endif /*CFG_PSCI_ARM32*/ + +#ifdef CFG_CORE_DYN_SHM +static uint64_t get_dt_val_and_advance(const void *data, size_t *offs, + uint32_t cell_size) +{ + uint64_t rv = 0; + + if (cell_size == 1) { + uint32_t v; + + memcpy(&v, (const uint8_t *)data + *offs, sizeof(v)); + *offs += sizeof(v); + rv = fdt32_to_cpu(v); + } else { + uint64_t v; + + memcpy(&v, (const uint8_t *)data + *offs, sizeof(v)); + *offs += sizeof(v); + rv = fdt64_to_cpu(v); + } + + return rv; +} + +/* + * Find all non-secure memory from DT. Memory marked inaccessible by Secure + * World is ignored since it could not be mapped to be used as dynamic shared + * memory. + */ +static int get_nsec_memory_helper(void *fdt, struct core_mmu_phys_mem *mem) +{ + const uint8_t *prop = NULL; + uint64_t a = 0; + uint64_t l = 0; + size_t prop_offs = 0; + size_t prop_len = 0; + int elems_total = 0; + int addr_size = 0; + int len_size = 0; + int offs = 0; + size_t n = 0; + int len = 0; + + addr_size = fdt_address_cells(fdt, 0); + if (addr_size < 0) + return 0; + + len_size = fdt_size_cells(fdt, 0); + if (len_size < 0) + return 0; + + while (true) { + offs = fdt_node_offset_by_prop_value(fdt, offs, "device_type", + "memory", + sizeof("memory")); + if (offs < 0) + break; + + if (fdt_get_status(fdt, offs) != (DT_STATUS_OK_NSEC | + DT_STATUS_OK_SEC)) + continue; + + prop = fdt_getprop(fdt, offs, "reg", &len); + if (!prop) + continue; + + prop_len = len; + for (n = 0, prop_offs = 0; prop_offs < prop_len; n++) { + a = get_dt_val_and_advance(prop, &prop_offs, addr_size); + if (prop_offs >= prop_len) { + n--; + break; + } + + l = get_dt_val_and_advance(prop, &prop_offs, len_size); + if (mem) { + mem->type = MEM_AREA_DDR_OVERALL; + mem->addr = a; + mem->size = l; + mem++; + } + } + + elems_total += n; + } + + return elems_total; +} + +static struct core_mmu_phys_mem *get_nsec_memory(void *fdt, size_t *nelems) +{ + struct core_mmu_phys_mem *mem = NULL; + int elems_total = 0; + + elems_total = get_nsec_memory_helper(fdt, NULL); + if (elems_total <= 0) + return NULL; + + mem = nex_calloc(elems_total, sizeof(*mem)); + if (!mem) + panic(); + + elems_total = get_nsec_memory_helper(fdt, mem); + assert(elems_total > 0); + + *nelems = elems_total; + + return mem; +} +#endif /*CFG_CORE_DYN_SHM*/ + +#ifdef CFG_CORE_RESERVED_SHM +static int mark_static_shm_as_reserved(struct dt_descriptor *dt) +{ + vaddr_t shm_start; + vaddr_t shm_end; + + core_mmu_get_mem_by_type(MEM_AREA_NSEC_SHM, &shm_start, &shm_end); + if (shm_start != shm_end) + return add_res_mem_dt_node(dt, "optee_shm", + virt_to_phys((void *)shm_start), + shm_end - shm_start); + + DMSG("No SHM configured"); + return -1; +} +#endif /*CFG_CORE_RESERVED_SHM*/ + +static int mark_tzdram_as_reserved(struct dt_descriptor *dt) +{ + return add_res_mem_dt_node(dt, "optee_core", CFG_TZDRAM_START, + CFG_TZDRAM_SIZE); +} + +static void update_external_dt(void) +{ + struct dt_descriptor *dt = get_external_dt_desc(); + + if (!dt || !dt->blob) + return; + + if (!IS_ENABLED(CFG_CORE_FFA) && add_optee_dt_node(dt)) + panic("Failed to add OP-TEE Device Tree node"); + + if (config_psci(dt)) + panic("Failed to config PSCI"); + +#ifdef CFG_CORE_RESERVED_SHM + if (mark_static_shm_as_reserved(dt)) + panic("Failed to config non-secure memory"); +#endif + + if (mark_tzdram_as_reserved(dt)) + panic("Failed to config secure memory"); +} +#else /*CFG_DT*/ +static void update_external_dt(void) +{ +} + +#ifdef CFG_CORE_DYN_SHM +static struct core_mmu_phys_mem *get_nsec_memory(void *fdt __unused, + size_t *nelems __unused) +{ + return NULL; +} +#endif /*CFG_CORE_DYN_SHM*/ +#endif /*!CFG_DT*/ + +#if defined(CFG_CORE_FFA) +void *get_manifest_dt(void) +{ + return manifest_dt; +} + +static void init_manifest_dt(unsigned long pa) +{ + void *fdt = NULL; + int ret = 0; + + if (!pa) { + EMSG("No manifest DT found"); + return; + } + + fdt = core_mmu_add_mapping(MEM_AREA_MANIFEST_DT, pa, CFG_DTB_MAX_SIZE); + if (!fdt) + panic("Failed to map manifest DT"); + + manifest_dt = fdt; + + ret = fdt_check_full(fdt, CFG_DTB_MAX_SIZE); + if (ret < 0) { + EMSG("Invalid manifest Device Tree at %#lx: error %d", pa, ret); + panic(); + } + + IMSG("manifest DT found"); +} + +static TEE_Result release_manifest_dt(void) +{ + if (!manifest_dt) + return TEE_SUCCESS; + + if (core_mmu_remove_mapping(MEM_AREA_MANIFEST_DT, manifest_dt, + CFG_DTB_MAX_SIZE)) + panic("Failed to remove temporary manifest DT mapping"); + manifest_dt = NULL; + + return TEE_SUCCESS; +} + +boot_final(release_manifest_dt); +#else +void *get_manifest_dt(void) +{ + return NULL; +} + +static void init_manifest_dt(unsigned long pa __unused) +{ +} +#endif /*CFG_CORE_FFA*/ + +#ifdef CFG_CORE_DYN_SHM +static void discover_nsec_memory(void) +{ + struct core_mmu_phys_mem *mem; + const struct core_mmu_phys_mem *mem_begin = NULL; + const struct core_mmu_phys_mem *mem_end = NULL; + size_t nelems; + void *fdt = get_external_dt(); + + if (fdt) { + mem = get_nsec_memory(fdt, &nelems); + if (mem) { + core_mmu_set_discovered_nsec_ddr(mem, nelems); + return; + } + + DMSG("No non-secure memory found in FDT"); + } + + mem_begin = phys_ddr_overall_begin; + mem_end = phys_ddr_overall_end; + nelems = mem_end - mem_begin; + if (nelems) { + /* + * Platform cannot use both register_ddr() and the now + * deprecated register_dynamic_shm(). + */ + assert(phys_ddr_overall_compat_begin == + phys_ddr_overall_compat_end); + } else { + mem_begin = phys_ddr_overall_compat_begin; + mem_end = phys_ddr_overall_compat_end; + nelems = mem_end - mem_begin; + if (!nelems) + return; + DMSG("Warning register_dynamic_shm() is deprecated, please use register_ddr() instead"); + } + + mem = nex_calloc(nelems, sizeof(*mem)); + if (!mem) + panic(); + + memcpy(mem, phys_ddr_overall_begin, sizeof(*mem) * nelems); + core_mmu_set_discovered_nsec_ddr(mem, nelems); +} +#else /*CFG_CORE_DYN_SHM*/ +static void discover_nsec_memory(void) +{ +} +#endif /*!CFG_CORE_DYN_SHM*/ + +#ifdef CFG_NS_VIRTUALIZATION +static TEE_Result virt_init_heap(void) +{ + /* We need to initialize pool for every virtual guest partition */ + malloc_add_pool(__heap1_start, __heap1_end - __heap1_start); + + return TEE_SUCCESS; +} +preinit_early(virt_init_heap); +#endif + +void init_tee_runtime(void) +{ +#ifndef CFG_WITH_PAGER + /* Pager initializes TA RAM early */ + core_mmu_init_ta_ram(); +#endif + /* + * With virtualization we call this function when creating the + * OP-TEE partition instead. + */ + if (!IS_ENABLED(CFG_NS_VIRTUALIZATION)) + call_preinitcalls(); + call_initcalls(); + + /* + * These two functions uses crypto_rng_read() to initialize the + * pauth keys. Once call_initcalls() returns we're guaranteed that + * crypto_rng_read() is ready to be used. + */ + thread_init_core_local_pauth_keys(); + thread_init_thread_pauth_keys(); + + /* + * Reinitialize canaries around the stacks with crypto_rng_read(). + * + * TODO: Updating canaries when CFG_NS_VIRTUALIZATION is enabled will + * require synchronization between thread_check_canaries() and + * thread_update_canaries(). + */ + if (!IS_ENABLED(CFG_NS_VIRTUALIZATION)) + thread_update_canaries(); +} + +static void init_primary(unsigned long pageable_part, unsigned long nsec_entry) +{ + thread_init_core_local_stacks(); + /* + * Mask asynchronous exceptions before switch to the thread vector + * as the thread handler requires those to be masked while + * executing with the temporary stack. The thread subsystem also + * asserts that the foreign interrupts are blocked when using most of + * its functions. + */ + thread_set_exceptions(THREAD_EXCP_ALL); + primary_save_cntfrq(); + init_vfp_sec(); + /* + * Pager: init_runtime() calls thread_kernel_enable_vfp() so we must + * set a current thread right now to avoid a chicken-and-egg problem + * (thread_init_boot_thread() sets the current thread but needs + * things set by init_runtime()). + */ + thread_get_core_local()->curr_thread = 0; + init_runtime(pageable_part); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + /* + * Virtualization: We can't initialize threads right now because + * threads belong to "tee" part and will be initialized + * separately per each new virtual guest. So, we'll clear + * "curr_thread" and call it done. + */ + thread_get_core_local()->curr_thread = -1; + } else { + thread_init_boot_thread(); + } + thread_init_primary(); + thread_init_per_cpu(); + init_sec_mon(nsec_entry); +} + +static bool cpu_nmfi_enabled(void) +{ +#if defined(ARM32) + return read_sctlr() & SCTLR_NMFI; +#else + /* Note: ARM64 does not feature non-maskable FIQ support. */ + return false; +#endif +} + +/* + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +void __weak boot_init_primary_late(unsigned long fdt, + unsigned long manifest) +{ + init_external_dt(fdt); + /* + * With an SPMC at S-EL2 we have saved the physical fdt address + * from the passed boot info. + */ + if (IS_ENABLED(CFG_CORE_SEL2_SPMC)) + manifest = (unsigned long)get_manifest_dt(); + init_manifest_dt(manifest); +#ifdef CFG_CORE_SEL1_SPMC + tpm_map_log_area(get_manifest_dt()); +#else + tpm_map_log_area(get_external_dt()); +#endif + discover_nsec_memory(); + update_external_dt(); + configure_console_from_dt(); + + IMSG("OP-TEE version: %s", core_v_str); + if (IS_ENABLED(CFG_WARN_INSECURE)) { + IMSG("WARNING: This OP-TEE configuration might be insecure!"); + IMSG("WARNING: Please check https://optee.readthedocs.io/en/latest/architecture/porting_guidelines.html"); + } + IMSG("Primary CPU initializing"); +#ifdef CFG_CORE_ASLR + DMSG("Executing at offset %#lx with virtual load address %#"PRIxVA, + (unsigned long)boot_mmu_config.map_offset, VCORE_START_VA); +#endif + if (IS_ENABLED(CFG_MEMTAG)) + DMSG("Memory tagging %s", + memtag_is_enabled() ? "enabled" : "disabled"); + + /* Check if platform needs NMFI workaround */ + if (cpu_nmfi_enabled()) { + if (!IS_ENABLED(CFG_CORE_WORKAROUND_ARM_NMFI)) + IMSG("WARNING: This ARM core has NMFI enabled, please apply workaround!"); + } else { + if (IS_ENABLED(CFG_CORE_WORKAROUND_ARM_NMFI)) + IMSG("WARNING: This ARM core does not have NMFI enabled, no need for workaround"); + } + + boot_primary_init_intc(); + init_vfp_nsec(); + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + IMSG("Initializing virtualization support"); + core_mmu_init_virtualization(); + } else { + init_tee_runtime(); + } + call_finalcalls(); + IMSG("Primary CPU switching to normal world boot"); +} + +static void init_secondary_helper(unsigned long nsec_entry) +{ + IMSG("Secondary CPU %zu initializing", get_core_pos()); + + /* + * Mask asynchronous exceptions before switch to the thread vector + * as the thread handler requires those to be masked while + * executing with the temporary stack. The thread subsystem also + * asserts that the foreign interrupts are blocked when using most of + * its functions. + */ + thread_set_exceptions(THREAD_EXCP_ALL); + + secondary_init_cntfrq(); + thread_init_per_cpu(); + init_sec_mon(nsec_entry); + boot_secondary_init_intc(); + init_vfp_sec(); + init_vfp_nsec(); + + IMSG("Secondary CPU %zu switching to normal world boot", get_core_pos()); +} + +/* + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area so that it lies in the init area. + */ +void __weak boot_init_primary_early(unsigned long pageable_part, + unsigned long nsec_entry __maybe_unused) +{ + unsigned long e = PADDR_INVALID; + +#if !defined(CFG_WITH_ARM_TRUSTED_FW) + e = nsec_entry; +#endif + + init_primary(pageable_part, e); +} + +#if defined(CFG_WITH_ARM_TRUSTED_FW) +unsigned long boot_cpu_on_handler(unsigned long a0 __maybe_unused, + unsigned long a1 __unused) +{ + init_secondary_helper(PADDR_INVALID); + return 0; +} +#else +void boot_init_secondary(unsigned long nsec_entry) +{ + init_secondary_helper(nsec_entry); +} +#endif + +#if defined(CFG_BOOT_SECONDARY_REQUEST) +void boot_set_core_ns_entry(size_t core_idx, uintptr_t entry, + uintptr_t context_id) +{ + ns_entry_contexts[core_idx].entry_point = entry; + ns_entry_contexts[core_idx].context_id = context_id; + dsb_ishst(); +} + +int boot_core_release(size_t core_idx, paddr_t entry) +{ + if (!core_idx || core_idx >= CFG_TEE_CORE_NB_CORE) + return -1; + + ns_entry_contexts[core_idx].entry_point = entry; + dmb(); + spin_table[core_idx] = 1; + dsb(); + sev(); + + return 0; +} + +/* + * spin until secondary boot request, then returns with + * the secondary core entry address. + */ +struct ns_entry_context *boot_core_hpen(void) +{ +#ifdef CFG_PSCI_ARM32 + return &ns_entry_contexts[get_core_pos()]; +#else + do { + wfe(); + } while (!spin_table[get_core_pos()]); + dmb(); + return &ns_entry_contexts[get_core_pos()]; +#endif +} +#endif + +#if defined(CFG_CORE_ASLR) +#if defined(CFG_DT) +unsigned long __weak get_aslr_seed(void *fdt) +{ + int rc = 0; + const uint64_t *seed = NULL; + int offs = 0; + int len = 0; + + if (!fdt) { + DMSG("No fdt"); + goto err; + } + + rc = fdt_check_header(fdt); + if (rc) { + DMSG("Bad fdt: %d", rc); + goto err; + } + + offs = fdt_path_offset(fdt, "/secure-chosen"); + if (offs < 0) { + DMSG("Cannot find /secure-chosen"); + goto err; + } + seed = fdt_getprop(fdt, offs, "kaslr-seed", &len); + if (!seed || len != sizeof(*seed)) { + DMSG("Cannot find valid kaslr-seed"); + goto err; + } + + return fdt64_to_cpu(*seed); + +err: + /* Try platform implementation */ + return plat_get_aslr_seed(); +} +#else /*!CFG_DT*/ +unsigned long __weak get_aslr_seed(void *fdt __unused) +{ + /* Try platform implementation */ + return plat_get_aslr_seed(); +} +#endif /*!CFG_DT*/ +#endif /*CFG_CORE_ASLR*/ + +#if defined(CFG_CORE_FFA) +static void *get_fdt_from_boot_info(struct ffa_boot_info_header_1_1 *hdr) +{ + struct ffa_boot_info_1_1 *desc = NULL; + uint8_t content_fmt = 0; + uint8_t name_fmt = 0; + void *fdt = NULL; + int ret = 0; + + if (hdr->signature != FFA_BOOT_INFO_SIGNATURE) { + EMSG("Bad boot info signature %#"PRIx32, hdr->signature); + panic(); + } + if (hdr->version != FFA_BOOT_INFO_VERSION) { + EMSG("Bad boot info version %#"PRIx32, hdr->version); + panic(); + } + if (hdr->desc_count != 1) { + EMSG("Bad boot info descriptor count %#"PRIx32, + hdr->desc_count); + panic(); + } + desc = (void *)((vaddr_t)hdr + hdr->desc_offset); + name_fmt = desc->flags & FFA_BOOT_INFO_FLAG_NAME_FORMAT_MASK; + if (name_fmt == FFA_BOOT_INFO_FLAG_NAME_FORMAT_STRING) + DMSG("Boot info descriptor name \"%16s\"", desc->name); + else if (name_fmt == FFA_BOOT_INFO_FLAG_NAME_FORMAT_UUID) + DMSG("Boot info descriptor UUID %pUl", (void *)desc->name); + else + DMSG("Boot info descriptor: unknown name format %"PRIu8, + name_fmt); + + content_fmt = (desc->flags & FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_MASK) >> + FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_SHIFT; + if (content_fmt != FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_ADDR) { + EMSG("Bad boot info content format %"PRIu8", expected %u (address)", + content_fmt, FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_ADDR); + panic(); + } + + fdt = (void *)(vaddr_t)desc->contents; + ret = fdt_check_full(fdt, desc->size); + if (ret < 0) { + EMSG("Invalid Device Tree at %p: error %d", fdt, ret); + panic(); + } + return fdt; +} + +static void get_sec_mem_from_manifest(void *fdt, paddr_t *base, size_t *size) +{ + int ret = 0; + uint64_t num = 0; + + ret = fdt_node_check_compatible(fdt, 0, "arm,ffa-manifest-1.0"); + if (ret < 0) { + EMSG("Invalid FF-A manifest at %p: error %d", fdt, ret); + panic(); + } + ret = dt_getprop_as_number(fdt, 0, "load-address", &num); + if (ret < 0) { + EMSG("Can't read \"load-address\" from FF-A manifest at %p: error %d", + fdt, ret); + panic(); + } + *base = num; + /* "mem-size" is currently an undocumented extension to the spec. */ + ret = dt_getprop_as_number(fdt, 0, "mem-size", &num); + if (ret < 0) { + EMSG("Can't read \"mem-size\" from FF-A manifest at %p: error %d", + fdt, ret); + panic(); + } + *size = num; +} + +void __weak boot_save_boot_info(void *boot_info) +{ + paddr_t base = 0; + size_t size = 0; + + manifest_dt = get_fdt_from_boot_info(boot_info); + if (IS_ENABLED(CFG_CORE_SEL2_SPMC) && + IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE)) { + get_sec_mem_from_manifest(manifest_dt, &base, &size); + core_mmu_set_secure_memory(base, size); + } +} +#endif /*CFG_CORE_FFA*/ diff --git a/optee_os/core/arch/arm/kernel/cache_helpers_a32.S b/optee_os/core/arch/arm/kernel/cache_helpers_a32.S new file mode 100644 index 0000000..50180dc --- /dev/null +++ b/optee_os/core/arch/arm/kernel/cache_helpers_a32.S @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2017, Linaro Limited. All rights reserved. + * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +/* + * Cache line size helpers + */ +.macro dcache_line_size reg, tmp + read_ctr \tmp + ubfx \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH + mov \reg, #CTR_WORD_SIZE + lsl \reg, \reg, \tmp +.endm + +.macro icache_line_size reg, tmp + read_ctr \tmp + and \tmp, \tmp, #CTR_IMINLINE_MASK + mov \reg, #CTR_WORD_SIZE + lsl \reg, \reg, \tmp +.endm + +/* + * This macro can be used for implementing various data cache operations `op` + */ +.macro do_dcache_maintenance_by_mva reg + dcache_line_size r2, r3 + add r1, r0, r1 + sub r3, r2, #1 + bic r0, r0, r3 +loop_\reg: + write_\reg r0 + add r0, r0, r2 + cmp r0, r1 + blo loop_\reg + dsb sy + bx lr +.endm + + /* ------------------------------------------ + * Clean+Invalidate from base address till + * size. 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +FUNC dcache_cleaninv_range , : + do_dcache_maintenance_by_mva dccimvac +END_FUNC dcache_cleaninv_range + + /* ------------------------------------------ + * Clean from base address till size. + * 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +FUNC dcache_clean_range , : + do_dcache_maintenance_by_mva dccmvac +END_FUNC dcache_clean_range + + /* ------------------------------------------ + * Invalidate from base address till + * size. 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +FUNC dcache_inv_range , : + do_dcache_maintenance_by_mva dcimvac +END_FUNC dcache_inv_range + + + /* ------------------------------------------ + * Clean from base address till size to point of unification + * 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +FUNC dcache_clean_range_pou , : + do_dcache_maintenance_by_mva dccmvau +END_FUNC dcache_clean_range_pou + + /* ---------------------------------------------------------------- + * Data cache operations by set/way to the level specified + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV, + * DCACHE_OP_CLEAN), as defined in cache_helpers.h + * r1: The cache level to begin operation from + * r2: clidr_el1 + * r3: The last cache level to operate on + * and will carry out the operation on each data cache from level 0 + * to the level in r3 in sequence + * + * The dcsw_op macro sets up the r2 and r3 parameters based on + * clidr_el1 cache information before invoking the main function + * ---------------------------------------------------------------- + */ + + .macro dcsw_op shift, fw, ls + read_clidr r2 + ubfx r3, r2, \shift, \fw + lsl r3, r3, \ls + mov r1, #0 + b do_dcsw_op + .endm + +LOCAL_FUNC do_dcsw_op , : + push {r4-r12,lr} + adr r11, dcsw_loop_table // compute cache op based on the operation type + add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions +loop1: + add r10, r1, r1, LSR #1 // Work out 3x current cache level + mov r12, r2, LSR r10 // extract cache type bits from clidr + and r12, r12, #7 // mask the bits for current cache only + cmp r12, #2 // see what cache we have at this level + blo level_done // no cache or only instruction cache at this level + + write_csselr r1 // select current cache level in csselr + isb // isb to sych the new cssr&csidr + read_ccsidr r12 // read the new ccsidr + and r10, r12, #7 // extract the length of the cache lines + add r10, r10, #4 // add 4 (r10 = line length offset) + ubfx r4, r12, #3, #10 // r4 = maximum way number (right aligned) + clz r5, r4 // r5 = the bit position of the way size increment + mov r9, r4 // r9 working copy of the aligned max way number + +loop2: + ubfx r7, r12, #13, #15 // r7 = max set number (right aligned) + +loop3: + orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0 + orr r0, r0, r7, LSL r10 // factor in the set number + + blx r6 + subs r7, r7, #1 // decrement the set number + bhs loop3 + subs r9, r9, #1 // decrement the way number + bhs loop2 +level_done: + add r1, r1, #2 // increment the cache number + cmp r3, r1 + dsb sy // ensure completion of previous cache maintenance instruction + bhi loop1 + + mov r6, #0 + write_csselr r6 //select cache level 0 in csselr + dsb sy + isb + pop {r4-r12,pc} + +dcsw_loop_table: + write_dcisw r0 + bx lr + write_dccisw r0 + bx lr + write_dccsw r0 + bx lr +END_FUNC do_dcsw_op + + /* --------------------------------------------------------------- + * Data cache operations by set/way till PoU. + * + * The function requires : + * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV, + * DCACHE_OP_CLEAN), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_louis , : + dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT +END_FUNC dcache_op_louis + + /* --------------------------------------------------------------- + * Data cache operations by set/way till PoC. + * + * The function requires : + * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV, + * DCACHE_OP_CLEAN), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_all , : + dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT +END_FUNC dcache_op_all + + + /* --------------------------------------------------------------- + * Helper macro for data cache operations by set/way for the + * level specified + * --------------------------------------------------------------- + */ + .macro dcsw_op_level level + read_clidr r2 + mov r3, \level + sub r1, r3, #2 + b do_dcsw_op + .endm + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 1 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV, + * DCACHE_OP_CLEAN), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_level1 , : + dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT) +END_FUNC dcache_op_level1 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 2 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV, + * DCACHE_OP_CLEAN), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_level2 , : + dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT) +END_FUNC dcache_op_level2 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 3 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV, + * DCACHE_OP_CLEAN), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_level3 , : + dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT) +END_FUNC dcache_op_level3 + +FUNC icache_inv_all , : + /* Invalidate Entire Instruction Cache (and branch predictors) */ + write_icialluis + + dsb ishst /* ensure that maintenance operations are seen */ + isb /* by the instructions rigth after the isb */ + + bx lr +END_FUNC icache_inv_all + + /* ------------------------------------------ + * Invalidate from base address till + * size. 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +FUNC icache_inv_range , : + icache_line_size r2, r3 + add r1, r0, r1 + sub r3, r2, #1 + bic r0, r0, r3 +loop_ic_inv: + write_icimvau r0 + add r0, r0, r2 + cmp r0, r1 + blo loop_ic_inv + + /* Invalidate entire branch predictor array inner shareable */ + write_bpiallis + + dsb ishst + isb + + bx lr +END_FUNC icache_inv_range diff --git a/optee_os/core/arch/arm/kernel/cache_helpers_a64.S b/optee_os/core/arch/arm/kernel/cache_helpers_a64.S new file mode 100644 index 0000000..2ed02a9 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/cache_helpers_a64.S @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2017, Linaro Limited. All rights reserved. + * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +.macro dcache_line_size reg, tmp + mrs \tmp, ctr_el0 + ubfx \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH + mov \reg, #CTR_WORD_SIZE + lsl \reg, \reg, \tmp +.endm + +.macro icache_line_size reg, tmp + mrs \tmp, ctr_el0 + and \tmp, \tmp, #CTR_IMINLINE_MASK + mov \reg, #CTR_WORD_SIZE + lsl \reg, \reg, \tmp +.endm + +/* + * This macro can be used for implementing various data cache operations `op` + */ +.macro do_dcache_maintenance_by_va op + dcache_line_size x2, x3 + add x1, x0, x1 + sub x3, x2, #1 + bic x0, x0, x3 +loop_\op: + dc \op, x0 + add x0, x0, x2 + cmp x0, x1 + b.lo loop_\op + dsb sy + ret +.endm + /* ------------------------------------------ + * Clean+Invalidate from base address till + * size. 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +FUNC dcache_cleaninv_range , : + do_dcache_maintenance_by_va civac +END_FUNC dcache_cleaninv_range + + /* ------------------------------------------ + * Clean from base address till size. + * 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +FUNC dcache_clean_range , : + do_dcache_maintenance_by_va cvac +END_FUNC dcache_clean_range + + /* ------------------------------------------ + * Invalidate from base address till + * size. 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +FUNC dcache_inv_range , : + do_dcache_maintenance_by_va ivac +END_FUNC dcache_inv_range + + /* ------------------------------------------ + * Clean from base address till size to point of unification + * 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +FUNC dcache_clean_range_pou , : + do_dcache_maintenance_by_va cvau +END_FUNC dcache_clean_range_pou + + /* --------------------------------------------------------------- + * Data cache operations by set/way to the level specified + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in cache_helpers.h + * x3: The last cache level to operate on + * x9: clidr_el1 + * x10: The cache level to begin operation from + * and will carry out the operation on each data cache from level 0 + * to the level in x3 in sequence + * + * The dcsw_op macro sets up the x3 and x9 parameters based on + * clidr_el1 cache information before invoking the main function + * --------------------------------------------------------------- + */ + + .macro dcsw_op shift, fw, ls + mrs x9, clidr_el1 + ubfx x3, x9, \shift, \fw + lsl x3, x3, \ls + mov x10, xzr + b do_dcsw_op + .endm + +LOCAL_FUNC do_dcsw_op , : + cbz x3, exit + adr x14, dcsw_loop_table // compute inner loop address + add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions +BTI( add x14, x14, x0, lsl #2) // inner loop is + "bti j" instruction + mov x0, x9 + mov w8, #1 +loop1: + add x2, x10, x10, lsr #1 // work out 3x current cache level + lsr x1, x0, x2 // extract cache type bits from clidr + and x1, x1, #7 // mask the bits for current cache only + cmp x1, #2 // see what cache we have at this level + b.lo level_done // nothing to do if no cache or icache + + msr csselr_el1, x10 // select current cache level in csselr + isb // isb to sych the new cssr&csidr + mrs x1, ccsidr_el1 // read the new ccsidr + and x2, x1, #7 // extract the length of the cache lines + add x2, x2, #4 // add 4 (line length offset) + ubfx x4, x1, #3, #10 // maximum way number + clz w5, w4 // bit position of way size increment + lsl w9, w4, w5 // w9 = aligned max way number + lsl w16, w8, w5 // w16 = way number loop decrement + orr w9, w10, w9 // w9 = combine way and cache number + ubfx w6, w1, #13, #15 // w6 = max set number + lsl w17, w8, w2 // w17 = set number loop decrement + dsb sy // barrier before we start this level + br x14 // jump to DC operation specific loop + + .macro dcsw_loop _op +BTI( bti j) +loop2_\_op: + lsl w7, w6, w2 // w7 = aligned max set number + +loop3_\_op: + orr w11, w9, w7 // combine cache, way and set number + dc \_op, x11 + subs w7, w7, w17 // decrement set number + b.hs loop3_\_op + + subs x9, x9, x16 // decrement way number + b.hs loop2_\_op + + b level_done + .endm + +level_done: + add x10, x10, #2 // increment cache number + cmp x3, x10 + b.hi loop1 + msr csselr_el1, xzr // select cache level 0 in csselr + dsb sy // barrier to complete final cache operation + isb +exit: + ret + +dcsw_loop_table: + dcsw_loop isw + dcsw_loop cisw + dcsw_loop csw +END_FUNC do_dcsw_op + +FUNC dcache_op_louis , : + dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT +END_FUNC dcache_op_louis + +FUNC dcache_op_all , : + dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT +END_FUNC dcache_op_all + + /* --------------------------------------------------------------- + * Helper macro for data cache operations by set/way for the + * level specified + * --------------------------------------------------------------- + */ + .macro dcsw_op_level level + mrs x9, clidr_el1 + mov x3, \level + sub x10, x3, #2 + b do_dcsw_op + .endm + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 1 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_level1 , : + dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT) +END_FUNC dcache_op_level1 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 2 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_level2 , : + dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT) +END_FUNC dcache_op_level2 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 3 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in cache_helpers.h + * --------------------------------------------------------------- + */ +FUNC dcache_op_level3 , : + dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT) +END_FUNC dcache_op_level3 + +FUNC icache_inv_all , : + /* Invalidate Entire Instruction Cache */ + ic ialluis + + dsb ish /* ensure that maintenance operations are seen */ + isb /* by the instructions rigth after the isb */ + + ret +END_FUNC icache_inv_all + +FUNC icache_inv_range , : + icache_line_size x2, x3 + add x1, x0, x1 + sub x3, x2, #1 + bic x0, x0, x3 +loop_ic_inv: + ic ivau, x0 + add x0, x0, x2 + cmp x0, x1 + b.lo loop_ic_inv + dsb ish + ret + +END_FUNC icache_inv_range + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/entry_a32.S b/optee_os/core/arch/arm/kernel/entry_a32.S new file mode 100644 index 0000000..2ef44c3 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/entry_a32.S @@ -0,0 +1,931 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + * Copyright (c) 2021-2023, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +.arch_extension sec + +.section .data +.balign 4 + +#ifdef CFG_BOOT_SYNC_CPU +.equ SEM_CPU_READY, 1 +#endif + +#ifdef CFG_PL310 +.section .rodata.init +panic_boot_file: + .asciz __FILE__ + +/* + * void assert_flat_mapped_range(uint32_t vaddr, uint32_t line) + */ +LOCAL_FUNC __assert_flat_mapped_range , : +UNWIND( .cantunwind) + push { r4-r6, lr } + mov r4, r0 + mov r5, r1 + bl cpu_mmu_enabled + cmp r0, #0 + beq 1f + mov r0, r4 + bl virt_to_phys + cmp r0, r4 + beq 1f + /* + * this must be compliant with the panic generic routine: + * __do_panic(__FILE__, __LINE__, __func__, str) + */ + ldr r0, =panic_boot_file + mov r1, r5 + mov r2, #0 + mov r3, #0 + bl __do_panic + b . /* should NOT return */ +1: pop { r4-r6, pc } +END_FUNC __assert_flat_mapped_range + + /* panic if mmu is enable and vaddr != paddr (scratch lr) */ + .macro assert_flat_mapped_range va, line + ldr r0, \va + ldr r1, =\line + bl __assert_flat_mapped_range + .endm +#endif /* CFG_PL310 */ + +WEAK_FUNC plat_cpu_reset_early , : + bx lr +END_FUNC plat_cpu_reset_early +DECLARE_KEEP_PAGER plat_cpu_reset_early + + .section .identity_map, "ax" + .align 5 +LOCAL_FUNC reset_vect_table , : , .identity_map + b . + b . /* Undef */ + b . /* Syscall */ + b . /* Prefetch abort */ + b . /* Data abort */ + b . /* Reserved */ + b . /* IRQ */ + b . /* FIQ */ +END_FUNC reset_vect_table + + .macro cpu_is_ready +#ifdef CFG_BOOT_SYNC_CPU + bl __get_core_pos + lsl r0, r0, #2 + ldr r1,=sem_cpu_sync + ldr r2, =SEM_CPU_READY + str r2, [r1, r0] + dsb + sev +#endif + .endm + + .macro wait_primary +#ifdef CFG_BOOT_SYNC_CPU + ldr r0, =sem_cpu_sync + mov r2, #SEM_CPU_READY + sev +1: + ldr r1, [r0] + cmp r1, r2 + wfene + bne 1b +#endif + .endm + + .macro wait_secondary +#ifdef CFG_BOOT_SYNC_CPU + ldr r0, =sem_cpu_sync + mov r3, #CFG_TEE_CORE_NB_CORE + mov r2, #SEM_CPU_READY + sev +1: + subs r3, r3, #1 + beq 3f + add r0, r0, #4 +2: + ldr r1, [r0] + cmp r1, r2 + wfene + bne 2b + b 1b +3: +#endif + .endm + + /* + * set_sctlr : Setup some core configuration in CP15 SCTLR + * + * Setup required by current implementation of the OP-TEE core: + * - Disable data and instruction cache. + * - MMU is expected off and exceptions trapped in ARM mode. + * - Enable or disable alignment checks upon platform configuration. + * - Optionally enable write-implies-execute-never. + * - Optionally enable round robin strategy for cache replacement. + * + * Clobbers r0. + */ + .macro set_sctlr + read_sctlr r0 + bic r0, r0, #(SCTLR_M | SCTLR_C) + bic r0, r0, #SCTLR_I + bic r0, r0, #SCTLR_TE + orr r0, r0, #SCTLR_SPAN +#if defined(CFG_SCTLR_ALIGNMENT_CHECK) + orr r0, r0, #SCTLR_A +#else + bic r0, r0, #SCTLR_A +#endif +#if defined(CFG_HWSUPP_MEM_PERM_WXN) && defined(CFG_CORE_RWDATA_NOEXEC) + orr r0, r0, #(SCTLR_WXN | SCTLR_UWXN) +#endif +#if defined(CFG_ENABLE_SCTLR_RR) + orr r0, r0, #SCTLR_RR +#endif + write_sctlr r0 + .endm + +#if defined(CFG_CORE_SEL1_SPMC) && defined(CFG_WITH_ARM_TRUSTED_FW) + /* + * With OP-TEE as SPMC at S-EL1 the SPMD (SPD_spmd) in TF-A passes + * the DTB in r0, pagable part in r1, and the rest of the registers + * are unused + * + * Save boot arguments passed + * entry r0, saved r6: device tree address + * entry r1, saved r4: pagestore + * saved r5, r7: Zero + */ + .macro bootargs_entry + mov r6, r0 + mov r4, r1 + mov r5, #0 + mov r7, #0 + .endm +#else + /* + * Save boot arguments + * entry r0, saved r4: pagestore + * entry r1, saved r7: (ARMv7 standard bootarg #1) + * entry r2, saved r6: device tree address, (ARMv7 standard bootarg #2) + * entry lr, saved r5: non-secure entry address (ARMv7 bootarg #0) + */ + .macro bootargs_entry +#if defined(CFG_NS_ENTRY_ADDR) + ldr r5, =CFG_NS_ENTRY_ADDR +#else + mov r5, lr +#endif +#if defined(CFG_PAGEABLE_ADDR) + ldr r4, =CFG_PAGEABLE_ADDR +#else + mov r4, r0 +#endif +#if defined(CFG_DT_ADDR) + ldr r6, =CFG_DT_ADDR +#else + mov r6, r2 +#endif + mov r7, r1 + .endm +#endif + + .macro maybe_init_spectre_workaround +#if !defined(CFG_WITH_ARM_TRUSTED_FW) && \ + (defined(CFG_CORE_WORKAROUND_SPECTRE_BP) || \ + defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC)) + read_midr r0 + ubfx r1, r0, #MIDR_IMPLEMENTER_SHIFT, #MIDR_IMPLEMENTER_WIDTH + cmp r1, #MIDR_IMPLEMENTER_ARM + bne 1f + ubfx r1, r0, #MIDR_PRIMARY_PART_NUM_SHIFT, \ + #MIDR_PRIMARY_PART_NUM_WIDTH + + movw r2, #CORTEX_A8_PART_NUM + cmp r1, r2 + moveq r2, #ACTLR_CA8_ENABLE_INVALIDATE_BTB + beq 2f + + movw r2, #CORTEX_A15_PART_NUM + cmp r1, r2 + moveq r2, #ACTLR_CA15_ENABLE_INVALIDATE_BTB + bne 1f /* Skip it for all other CPUs */ +2: + read_actlr r0 + orr r0, r0, r2 + write_actlr r0 + isb +1: +#endif + .endm + +FUNC _start , : +UNWIND( .cantunwind) + + bootargs_entry + + /* + * 32bit entry is expected to execute Supervisor mode, + * some bootloader may enter in Supervisor or Monitor + */ + cps #CPSR_MODE_SVC + + /* Early ARM secure MP specific configuration */ + bl plat_cpu_reset_early + maybe_init_spectre_workaround + + set_sctlr + isb + + ldr r0, =reset_vect_table + write_vbar r0 + +#if defined(CFG_WITH_ARM_TRUSTED_FW) + b reset_primary +#else + bl __get_core_pos + cmp r0, #0 + beq reset_primary + b reset_secondary +#endif +END_FUNC _start +DECLARE_KEEP_INIT _start + + /* + * Setup sp to point to the top of the tmp stack for the current CPU: + * sp is assigned: + * stack_tmp + (cpu_id + 1) * stack_tmp_stride - STACK_TMP_GUARD + */ + .macro set_sp + bl __get_core_pos + cmp r0, #CFG_TEE_CORE_NB_CORE + /* Unsupported CPU, park it before it breaks something */ + bge unhandled_cpu + add r0, r0, #1 + + /* r2 = stack_tmp - STACK_TMP_GUARD */ + adr r3, stack_tmp_rel + ldr r2, [r3] + add r2, r2, r3 + + /* + * stack_tmp_stride and stack_tmp_stride_rel are the + * equivalent of: + * extern const u32 stack_tmp_stride; + * u32 stack_tmp_stride_rel = (u32)&stack_tmp_stride - + * (u32)&stack_tmp_stride_rel + * + * To load the value of stack_tmp_stride we do the equivalent + * of: + * *(u32 *)(stack_tmp_stride + (u32)&stack_tmp_stride_rel) + */ + adr r3, stack_tmp_stride_rel + ldr r1, [r3] + ldr r1, [r1, r3] + + /* + * r0 is core pos + 1 + * r1 is value of stack_tmp_stride + * r2 is value of stack_tmp + guard + */ + mul r1, r0, r1 + add sp, r1, r2 + .endm + + /* + * Cache maintenance during entry: handle outer cache. + * End address is exclusive: first byte not to be changed. + * Note however arm_clX_inv/cleanbyva operate on full cache lines. + * + * Use ANSI #define to trap source file line number for PL310 assertion + */ + .macro __inval_cache_vrange vbase, vend, line +#if defined(CFG_PL310) && !defined(CFG_PL310_SIP_PROTOCOL) + assert_flat_mapped_range (\vbase), (\line) + bl pl310_base + ldr r1, \vbase + ldr r2, \vend + bl arm_cl2_invbypa +#endif + ldr r0, \vbase + ldr r1, \vend + sub r1, r1, r0 + bl dcache_inv_range + .endm + + .macro __flush_cache_vrange vbase, vend, line +#if defined(CFG_PL310) && !defined(CFG_PL310_SIP_PROTOCOL) + assert_flat_mapped_range (\vbase), (\line) + ldr r0, \vbase + ldr r1, \vend + sub r1, r1, r0 + bl dcache_clean_range + bl pl310_base + ldr r1, \vbase + ldr r2, \vend + bl arm_cl2_cleaninvbypa +#endif + ldr r0, \vbase + ldr r1, \vend + sub r1, r1, r0 + bl dcache_cleaninv_range + .endm + +#define inval_cache_vrange(vbase, vend) \ + __inval_cache_vrange vbase, vend, __LINE__ + +#define flush_cache_vrange(vbase, vend) \ + __flush_cache_vrange vbase, vend, __LINE__ + +#ifdef CFG_BOOT_SYNC_CPU +#define flush_cpu_semaphores \ + flush_cache_vrange(sem_cpu_sync_start, sem_cpu_sync_end) +#else +#define flush_cpu_semaphores +#endif + +LOCAL_FUNC reset_primary , : , .identity_map +UNWIND( .cantunwind) + + /* preserve r4-r7: bootargs */ + +#ifdef CFG_WITH_PAGER + /* + * Move init code into correct location and move hashes to a + * temporary safe location until the heap is initialized. + * + * The binary is built as: + * [Pager code, rodata and data] : In correct location + * [Init code and rodata] : Should be copied to __init_start + * [struct boot_embdata + data] : Should be saved before + * initializing pager, first uint32_t tells the length of the data + */ + ldr r0, =__init_start /* dst */ + ldr r1, =__data_end /* src */ + ldr r2, =__init_end + sub r2, r2, r0 /* init len */ + ldr r12, [r1, r2] /* length of hashes etc */ + add r2, r2, r12 /* length of init and hashes etc */ + /* Copy backwards (as memmove) in case we're overlapping */ + add r0, r0, r2 /* __init_start + len */ + add r1, r1, r2 /* __data_end + len */ + str r0, cached_mem_end + ldr r2, =__init_start +copy_init: + ldmdb r1!, {r3, r8-r12} + stmdb r0!, {r3, r8-r12} + cmp r0, r2 + bgt copy_init +#else + /* + * The binary is built as: + * [Core, rodata and data] : In correct location + * [struct boot_embdata + data] : Should be moved to __end, first + * uint32_t tells the length of the struct + data + */ + ldr r0, =__end /* dst */ + ldr r1, =__data_end /* src */ + ldr r2, [r1] /* struct boot_embdata::total_len */ + /* Copy backwards (as memmove) in case we're overlapping */ + add r0, r0, r2 + add r1, r1, r2 + str r0, cached_mem_end + ldr r2, =__end + +copy_init: + ldmdb r1!, {r3, r8-r12} + stmdb r0!, {r3, r8-r12} + cmp r0, r2 + bgt copy_init +#endif + + /* + * Clear .bss, this code obviously depends on the linker keeping + * start/end of .bss at least 8 byte aligned. + */ + ldr r0, =__bss_start + ldr r1, =__bss_end + mov r2, #0 + mov r3, #0 +clear_bss: + stmia r0!, {r2, r3} + cmp r0, r1 + bls clear_bss + +#ifdef CFG_NS_VIRTUALIZATION + /* + * Clear .nex_bss, this code obviously depends on the linker keeping + * start/end of .bss at least 8 byte aligned. + */ + ldr r0, =__nex_bss_start + ldr r1, =__nex_bss_end + mov r2, #0 + mov r3, #0 +clear_nex_bss: + stmia r0!, {r2, r3} + cmp r0, r1 + bls clear_nex_bss +#endif + +#ifdef CFG_CORE_SANITIZE_KADDRESS + /* First initialize the entire shadow area with no access */ + ldr r0, =__asan_shadow_start /* start */ + ldr r1, =__asan_shadow_end /* limit */ + mov r2, #ASAN_DATA_RED_ZONE +shadow_no_access: + str r2, [r0], #4 + cmp r0, r1 + bls shadow_no_access + + /* Mark the entire stack area as OK */ + ldr r2, =CFG_ASAN_SHADOW_OFFSET + ldr r0, =__nozi_stack_start /* start */ + lsr r0, r0, #ASAN_BLOCK_SHIFT + add r0, r0, r2 + ldr r1, =__nozi_stack_end /* limit */ + lsr r1, r1, #ASAN_BLOCK_SHIFT + add r1, r1, r2 + mov r2, #0 +shadow_stack_access_ok: + strb r2, [r0], #1 + cmp r0, r1 + bls shadow_stack_access_ok +#endif + + set_sp + + bl thread_init_thread_core_local + + /* complete ARM secure MP common configuration */ + bl plat_primary_init_early + + /* Enable Console */ + bl console_init + +#ifdef CFG_PL310 + bl pl310_base + bl arm_cl2_config +#endif + + /* + * Invalidate dcache for all memory used during initialization to + * avoid nasty surprices when the cache is turned on. We must not + * invalidate memory not used by OP-TEE since we may invalidate + * entries used by for instance ARM Trusted Firmware. + */ + inval_cache_vrange(cached_mem_start, cached_mem_end) + +#if defined(CFG_PL310) && !defined(CFG_PL310_SIP_PROTOCOL) + /* Enable PL310 if not yet enabled */ + bl pl310_base + bl arm_cl2_enable +#endif + +#ifdef CFG_CORE_ASLR + mov r0, r6 + bl get_aslr_seed +#else + mov r0, #0 +#endif + + ldr r1, =boot_mmu_config + bl core_init_mmu_map + +#ifdef CFG_CORE_ASLR + /* + * Process relocation information for updating with the virtual map + * offset. We're doing this now before MMU is enabled as some of + * the memory will become write protected. + */ + ldr r0, =boot_mmu_config + ldr r0, [r0, #CORE_MMU_CONFIG_MAP_OFFSET] + /* + * Update cached_mem_end address with load offset since it was + * calculated before relocation. + */ + ldr r2, cached_mem_end + add r2, r2, r0 + str r2, cached_mem_end + + bl relocate +#endif + + bl __get_core_pos + bl enable_mmu +#ifdef CFG_CORE_ASLR + /* + * Reinitialize console, since register_serial_console() has + * previously registered a PA and with ASLR the VA is different + * from the PA. + */ + bl console_init +#endif + +#ifdef CFG_NS_VIRTUALIZATION + /* + * Initialize partition tables for each partition to + * default_partition which has been relocated now to a different VA + */ + bl core_mmu_set_default_prtn_tbl +#endif + + mov r0, r4 /* pageable part address */ + mov r1, r5 /* ns-entry address */ + bl boot_init_primary_early +#ifndef CFG_NS_VIRTUALIZATION + mov r9, sp + ldr r0, =threads + ldr r0, [r0, #THREAD_CTX_STACK_VA_END] + mov sp, r0 + bl thread_get_core_local + mov r8, r0 + mov r0, #0 + str r0, [r8, #THREAD_CORE_LOCAL_FLAGS] +#endif + mov r0, r6 /* DT address */ + mov r1, #0 /* unused */ + bl boot_init_primary_late +#ifndef CFG_NS_VIRTUALIZATION + mov r0, #THREAD_CLF_TMP + str r0, [r8, #THREAD_CORE_LOCAL_FLAGS] + mov sp, r9 +#endif + +#ifdef _CFG_CORE_STACK_PROTECTOR + /* Update stack canary value */ + sub sp, sp, #0x8 + mov r0, sp + mov r1, #1 + mov r2, #0x4 + bl plat_get_random_stack_canaries + ldr r0, [sp] + ldr r1, =__stack_chk_guard + str r0, [r1] + add sp, sp, #0x8 +#endif + + /* + * In case we've touched memory that secondary CPUs will use before + * they have turned on their D-cache, clean and invalidate the + * D-cache before exiting to normal world. + */ + flush_cache_vrange(cached_mem_start, cached_mem_end) + + /* release secondary boot cores and sync with them */ + cpu_is_ready + flush_cpu_semaphores + wait_secondary + +#ifdef CFG_PL310_LOCKED +#ifdef CFG_PL310_SIP_PROTOCOL +#error "CFG_PL310_LOCKED must not be defined when CFG_PL310_SIP_PROTOCOL=y" +#endif + /* lock/invalidate all lines: pl310 behaves as if disable */ + bl pl310_base + bl arm_cl2_lockallways + bl pl310_base + bl arm_cl2_cleaninvbyway +#endif + + /* + * Clear current thread id now to allow the thread to be reused on + * next entry. Matches the thread_init_boot_thread() in + * boot.c. + */ +#ifndef CFG_NS_VIRTUALIZATION + bl thread_clr_boot_thread +#endif + +#ifdef CFG_CORE_FFA + ldr r0, =cpu_on_handler + /* + * Compensate for the virtual map offset since cpu_on_handler() is + * called with MMU off. + */ + ldr r1, boot_mmu_config + CORE_MMU_CONFIG_MAP_OFFSET + sub r0, r0, r1 + bl thread_spmc_register_secondary_ep + b thread_ffa_msg_wait +#else /* CFG_CORE_FFA */ + +#if defined(CFG_WITH_ARM_TRUSTED_FW) + ldr r0, =boot_mmu_config + ldr r0, [r0, #CORE_MMU_CONFIG_MAP_OFFSET] + ldr r1, =thread_vector_table + /* Pass the vector address returned from main_init */ + sub r1, r1, r0 +#else + /* realy standard bootarg #1 and #2 to non secure entry */ + mov r4, #0 + mov r3, r6 /* std bootarg #2 for register R2 */ + mov r2, r7 /* std bootarg #1 for register R1 */ + mov r1, #0 +#endif /* CFG_WITH_ARM_TRUSTED_FW */ + + mov r0, #TEESMC_OPTEED_RETURN_ENTRY_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +#endif /* CFG_CORE_FFA */ +END_FUNC reset_primary + +#ifdef CFG_BOOT_SYNC_CPU +LOCAL_DATA sem_cpu_sync_start , : + .word sem_cpu_sync +END_DATA sem_cpu_sync_start + +LOCAL_DATA sem_cpu_sync_end , : + .word sem_cpu_sync + (CFG_TEE_CORE_NB_CORE << 2) +END_DATA sem_cpu_sync_end +#endif + +LOCAL_DATA cached_mem_start , : + .word __text_start +END_DATA cached_mem_start + +LOCAL_DATA cached_mem_end , : + .skip 4 +END_DATA cached_mem_end + +LOCAL_FUNC unhandled_cpu , : + wfi + b unhandled_cpu +END_FUNC unhandled_cpu + +#ifdef CFG_CORE_ASLR +LOCAL_FUNC relocate , : + push {r4-r5} + /* r0 holds load offset */ +#ifdef CFG_WITH_PAGER + ldr r12, =__init_end +#else + ldr r12, =__end +#endif + ldr r2, [r12, #BOOT_EMBDATA_RELOC_OFFSET] + ldr r3, [r12, #BOOT_EMBDATA_RELOC_LEN] + + mov_imm r1, TEE_LOAD_ADDR + add r2, r2, r12 /* start of relocations */ + add r3, r3, r2 /* end of relocations */ + + /* + * Relocations are not formatted as Rel32, instead they are in a + * compressed format created by get_reloc_bin() in + * scripts/gen_tee_bin.py + * + * All the R_ARM_RELATIVE relocations are translated into a list of + * 32-bit offsets from TEE_LOAD_ADDR. At each address a 32-bit + * value pointed out which increased with the load offset. + */ + +#ifdef CFG_WITH_PAGER + /* + * With pager enabled we can only relocate the pager and init + * parts, the rest has to be done when a page is populated. + */ + sub r12, r12, r1 +#endif + + b 2f + /* Loop over the relocation addresses and process all entries */ +1: ldr r4, [r2], #4 +#ifdef CFG_WITH_PAGER + /* Skip too large addresses */ + cmp r4, r12 + bge 2f +#endif + ldr r5, [r4, r1] + add r5, r5, r0 + str r5, [r4, r1] + +2: cmp r2, r3 + bne 1b + + pop {r4-r5} + bx lr +END_FUNC relocate +#endif + +/* + * void enable_mmu(unsigned long core_pos); + * + * This function depends on being mapped with in the identity map where + * physical address and virtual address is the same. After MMU has been + * enabled the instruction pointer will be updated to execute as the new + * offset instead. Stack pointers and the return address are updated. + */ +LOCAL_FUNC enable_mmu , : , .identity_map + /* r0 = core pos */ + adr r1, boot_mmu_config + +#ifdef CFG_WITH_LPAE + ldm r1!, {r2, r3} + /* + * r2 = ttbcr + * r3 = mair0 + */ + write_ttbcr r2 + write_mair0 r3 + + ldm r1!, {r2, r3} + /* + * r2 = ttbr0_base + * r3 = ttbr0_core_offset + */ + + /* + * ttbr0_el1 = ttbr0_base + ttbr0_core_offset * core_pos + */ + mla r12, r0, r3, r2 + mov r0, #0 + write_ttbr0_64bit r12, r0 + write_ttbr1_64bit r0, r0 +#else + ldm r1!, {r2, r3} + /* + * r2 = prrr + * r3 = nmrr + */ + write_prrr r2 + write_nmrr r3 + + ldm r1!, {r2, r3} + /* + * r2 = dacr + * r3 = ttbcr + */ + write_dacr r2 + write_ttbcr r3 + + ldm r1!, {r2} + /* r2 = ttbr */ + write_ttbr0 r2 + write_ttbr1 r2 + + mov r2, #0 + write_contextidr r2 +#endif + ldm r1!, {r2} + /* r2 = load_offset (always 0 if CFG_CORE_ASLR=n) */ + isb + + /* Invalidate TLB */ + write_tlbiall + + /* + * Make sure translation table writes have drained into memory and + * the TLB invalidation is complete. + */ + dsb sy + isb + + read_sctlr r0 + orr r0, r0, #SCTLR_M +#ifndef CFG_WITH_LPAE + /* Enable Access flag (simplified access permissions) and TEX remap */ + orr r0, r0, #(SCTLR_AFE | SCTLR_TRE) +#endif + write_sctlr r0 + isb + + /* Update vbar */ + read_vbar r1 + add r1, r1, r2 + write_vbar r1 + isb + + /* Invalidate instruction cache and branch predictor */ + write_iciallu + write_bpiall + isb + + read_sctlr r0 + /* Enable I and D cache */ + orr r0, r0, #SCTLR_I + orr r0, r0, #SCTLR_C +#if defined(CFG_ENABLE_SCTLR_Z) + /* + * This is only needed on ARMv7 architecture and hence conditionned + * by configuration directive CFG_ENABLE_SCTLR_Z. For recent + * architectures, the program flow prediction is automatically + * enabled upon MMU enablement. + */ + orr r0, r0, #SCTLR_Z +#endif + write_sctlr r0 + isb + + /* Adjust stack pointer and return address */ + add sp, sp, r2 + add lr, lr, r2 + + bx lr +END_FUNC enable_mmu + +LOCAL_DATA stack_tmp_rel , : + .word stack_tmp - stack_tmp_rel - STACK_TMP_GUARD +END_DATA stack_tmp_rel + +LOCAL_DATA stack_tmp_stride_rel , : + .word stack_tmp_stride - stack_tmp_stride_rel +END_DATA stack_tmp_stride_rel + +DATA boot_mmu_config , : /* struct core_mmu_config */ + .skip CORE_MMU_CONFIG_SIZE +END_DATA boot_mmu_config + +#if defined(CFG_WITH_ARM_TRUSTED_FW) +FUNC cpu_on_handler , : , .identity_map +UNWIND( .cantunwind) + mov r4, r0 + mov r5, r1 + mov r6, lr + + set_sctlr + isb + + adr r0, reset_vect_table + write_vbar r0 + + mov r4, lr + + bl __get_core_pos + bl enable_mmu + + set_sp + + mov r0, r4 + mov r1, r5 + bl boot_cpu_on_handler +#ifdef CFG_CORE_FFA + b thread_ffa_msg_wait +#else + bx r6 +#endif +END_FUNC cpu_on_handler +DECLARE_KEEP_PAGER cpu_on_handler + +#else /* defined(CFG_WITH_ARM_TRUSTED_FW) */ + +LOCAL_FUNC reset_secondary , : , .identity_map +UNWIND( .cantunwind) + adr r0, reset_vect_table + write_vbar r0 + + wait_primary + + set_sp + +#if defined (CFG_BOOT_SECONDARY_REQUEST) + /* if L1 is not invalidated before, do it here */ + mov r0, #DCACHE_OP_INV + bl dcache_op_level1 +#endif + + bl __get_core_pos + bl enable_mmu + + cpu_is_ready + +#if defined (CFG_BOOT_SECONDARY_REQUEST) + /* + * boot_core_hpen() return value (r0) is address of + * ns entry context structure + */ + bl boot_core_hpen + ldm r0, {r0, r6} +#else + mov r0, r5 /* ns-entry address */ + mov r6, #0 +#endif + bl boot_init_secondary + + mov r0, #TEESMC_OPTEED_RETURN_ENTRY_DONE + mov r1, r6 + mov r2, #0 + mov r3, #0 + mov r4, #0 + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC reset_secondary +DECLARE_KEEP_PAGER reset_secondary +#endif /* defined(CFG_WITH_ARM_TRUSTED_FW) */ diff --git a/optee_os/core/arch/arm/kernel/entry_a64.S b/optee_os/core/arch/arm/kernel/entry_a64.S new file mode 100644 index 0000000..dd82e71 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/entry_a64.S @@ -0,0 +1,781 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2022, Linaro Limited + * Copyright (c) 2021-2023, Arm Limited + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + /* + * Setup SP_EL0 and SPEL1, SP will be set to SP_EL0. + * SP_EL0 is assigned: + * stack_tmp + (cpu_id + 1) * stack_tmp_stride - STACK_TMP_GUARD + * SP_EL1 is assigned thread_core_local[cpu_id] + */ + .macro set_sp + bl __get_core_pos + cmp x0, #CFG_TEE_CORE_NB_CORE + /* Unsupported CPU, park it before it breaks something */ + bge unhandled_cpu + add x0, x0, #1 + adr_l x1, stack_tmp_stride + ldr w1, [x1] + mul x1, x0, x1 + + /* x0 = stack_tmp - STACK_TMP_GUARD */ + adr_l x2, stack_tmp_rel + ldr w0, [x2] + add x0, x0, x2 + + msr spsel, #0 + add sp, x1, x0 + bl thread_get_core_local + msr spsel, #1 + mov sp, x0 + msr spsel, #0 + .endm + + .macro read_feat_mte reg + mrs \reg, id_aa64pfr1_el1 + ubfx \reg, \reg, #ID_AA64PFR1_EL1_MTE_SHIFT, #4 + .endm + + .macro read_feat_pan reg + mrs \reg, id_mmfr3_el1 + ubfx \reg, \reg, #ID_MMFR3_EL1_PAN_SHIFT, #4 + .endm + + .macro set_sctlr_el1 + mrs x0, sctlr_el1 + orr x0, x0, #SCTLR_I + orr x0, x0, #SCTLR_SA + orr x0, x0, #SCTLR_SPAN +#if defined(CFG_CORE_RWDATA_NOEXEC) + orr x0, x0, #SCTLR_WXN +#endif +#if defined(CFG_SCTLR_ALIGNMENT_CHECK) + orr x0, x0, #SCTLR_A +#else + bic x0, x0, #SCTLR_A +#endif +#ifdef CFG_MEMTAG + read_feat_mte x1 + cmp w1, #1 + b.ls 111f + orr x0, x0, #(SCTLR_ATA | SCTLR_ATA0) + bic x0, x0, #SCTLR_TCF_MASK + bic x0, x0, #SCTLR_TCF0_MASK +111: +#endif +#if defined(CFG_TA_PAUTH) && defined(CFG_TA_BTI) + orr x0, x0, #SCTLR_BT0 +#endif +#if defined(CFG_CORE_PAUTH) && defined(CFG_CORE_BTI) + orr x0, x0, #SCTLR_BT1 +#endif + msr sctlr_el1, x0 + .endm + + .macro init_memtag_per_cpu + read_feat_mte x0 + cmp w0, #1 + b.ls 11f + +#ifdef CFG_TEE_CORE_DEBUG + /* + * This together with GCR_EL1.RRND = 0 will make the tags + * acquired with the irg instruction deterministic. + */ + mov_imm x0, 0xcafe00 + msr rgsr_el1, x0 + /* Avoid tag = 0x0 and 0xf */ + mov x0, #0 +#else + /* + * Still avoid tag = 0x0 and 0xf as we use that tag for + * everything which isn't explicitly tagged. Setting + * GCR_EL1.RRND = 1 to allow an implementation specific + * method of generating the tags. + */ + mov x0, #GCR_EL1_RRND +#endif + orr x0, x0, #1 + orr x0, x0, #(1 << 15) + msr gcr_el1, x0 + + /* + * Enable the tag checks on the current CPU. + * + * Depends on boot_init_memtag() having cleared tags for + * TEE core memory. Well, not really, addresses with the + * tag value 0b0000 will use unchecked access due to + * TCR_TCMA0. + */ + mrs x0, tcr_el1 + orr x0, x0, #TCR_TBI0 + orr x0, x0, #TCR_TCMA0 + msr tcr_el1, x0 + + mrs x0, sctlr_el1 + orr x0, x0, #SCTLR_TCF_SYNC + orr x0, x0, #SCTLR_TCF0_SYNC + msr sctlr_el1, x0 + + isb +11: + .endm + + .macro init_pauth_per_cpu + msr spsel, #1 + ldp x0, x1, [sp, #THREAD_CORE_LOCAL_KEYS] + msr spsel, #0 + write_apiakeyhi x0 + write_apiakeylo x1 + mrs x0, sctlr_el1 + orr x0, x0, #SCTLR_ENIA + msr sctlr_el1, x0 + isb + .endm + + .macro init_pan + read_feat_pan x0 + cmp x0, #0 + b.eq 1f + mrs x0, sctlr_el1 + bic x0, x0, #SCTLR_SPAN + msr sctlr_el1, x0 + write_pan_enable + 1: + .endm + +FUNC _start , : + /* + * Register use: + * x0 - CFG_CORE_FFA=y && CFG_CORE_SEL2_SPMC=n: + * if non-NULL holds the TOS FW config [1] address + * - CFG_CORE_FFA=y && CFG_CORE_SEL2_SPMC=y: + * address of FF-A Boot Information Blob + * - CFG_CORE_FFA=n: + * if non-NULL holds the pagable part address + * x2 - CFG_CORE_SEL2_SPMC=n: + * if non-NULL holds the system DTB address + * + * x19 - saved x0 + * x20 - saved x2 + * + * [1] A TF-A concept: TOS_FW_CONFIG - Trusted OS Firmware + * configuration file. Used by Trusted OS (BL32), that is, OP-TEE + * here. + */ + mov x19, x0 +#if defined(CFG_DT_ADDR) + ldr x20, =CFG_DT_ADDR +#else + mov x20, x2 /* Save DT address */ +#endif + + adr x0, reset_vect_table + msr vbar_el1, x0 + isb + +#ifdef CFG_PAN + init_pan +#endif + + set_sctlr_el1 + isb + +#ifdef CFG_WITH_PAGER + /* + * Move init code into correct location and move hashes to a + * temporary safe location until the heap is initialized. + * + * The binary is built as: + * [Pager code, rodata and data] : In correct location + * [Init code and rodata] : Should be copied to __init_start + * [struct boot_embdata + data] : Should be saved before + * initializing pager, first uint32_t tells the length of the data + */ + adr x0, __init_start /* dst */ + adr x1, __data_end /* src */ + adr x2, __init_end + sub x2, x2, x0 /* init len */ + ldr w4, [x1, x2] /* length of hashes etc */ + add x2, x2, x4 /* length of init and hashes etc */ + /* Copy backwards (as memmove) in case we're overlapping */ + add x0, x0, x2 /* __init_start + len */ + add x1, x1, x2 /* __data_end + len */ + adr x3, cached_mem_end + str x0, [x3] + adr x2, __init_start +copy_init: + ldp x3, x4, [x1, #-16]! + stp x3, x4, [x0, #-16]! + cmp x0, x2 + b.gt copy_init +#else + /* + * The binary is built as: + * [Core, rodata and data] : In correct location + * [struct boot_embdata + data] : Should be moved to __end, first + * uint32_t tells the length of the struct + data + */ + adr_l x0, __end /* dst */ + adr_l x1, __data_end /* src */ + ldr w2, [x1] /* struct boot_embdata::total_len */ + /* Copy backwards (as memmove) in case we're overlapping */ + add x0, x0, x2 + add x1, x1, x2 + adr x3, cached_mem_end + str x0, [x3] + adr_l x2, __end + +copy_init: + ldp x3, x4, [x1, #-16]! + stp x3, x4, [x0, #-16]! + cmp x0, x2 + b.gt copy_init +#endif + + /* + * Clear .bss, this code obviously depends on the linker keeping + * start/end of .bss at least 8 byte aligned. + */ + adr_l x0, __bss_start + adr_l x1, __bss_end +clear_bss: + str xzr, [x0], #8 + cmp x0, x1 + b.lt clear_bss + +#ifdef CFG_NS_VIRTUALIZATION + /* + * Clear .nex_bss, this code obviously depends on the linker keeping + * start/end of .bss at least 8 byte aligned. + */ + adr_l x0, __nex_bss_start + adr_l x1, __nex_bss_end +clear_nex_bss: + str xzr, [x0], #8 + cmp x0, x1 + b.lt clear_nex_bss +#endif + + +#if defined(CFG_CORE_PHYS_RELOCATABLE) + /* + * Save the base physical address, it will not change after this + * point. + */ + adr_l x2, core_mmu_tee_load_pa + adr x1, _start /* Load address */ + str x1, [x2] + + mov_imm x0, TEE_LOAD_ADDR /* Compiled load address */ + sub x0, x1, x0 /* Relocatation offset */ + + cbz x0, 1f + bl relocate +1: +#endif + + /* Setup SP_EL0 and SP_EL1, SP will be set to SP_EL0 */ + set_sp + + bl thread_init_thread_core_local + + /* Enable aborts now that we can receive exceptions */ + msr daifclr, #DAIFBIT_ABT + + /* + * Invalidate dcache for all memory used during initialization to + * avoid nasty surprices when the cache is turned on. We must not + * invalidate memory not used by OP-TEE since we may invalidate + * entries used by for instance ARM Trusted Firmware. + */ + adr_l x0, __text_start + ldr x1, cached_mem_end + sub x1, x1, x0 + bl dcache_cleaninv_range + + /* Enable Console */ + bl console_init + +#if defined(CFG_CORE_SEL2_SPMC) + mov x0, x19 /* boot info */ + bl boot_save_boot_info +#endif + +#ifdef CFG_MEMTAG + /* + * If FEAT_MTE2 is available, initializes the memtag callbacks. + * Tags for OP-TEE core memory are then cleared to make it safe to + * enable MEMTAG below. + */ + bl boot_init_memtag +#endif + +#ifdef CFG_CORE_ASLR + mov x0, x20 /* DT address */ + bl get_aslr_seed +#else + mov x0, #0 +#endif + + adr x1, boot_mmu_config + bl core_init_mmu_map + +#ifdef CFG_CORE_ASLR + /* + * Process relocation information again updating for the virtual + * map offset. We're doing this now before MMU is enabled as some + * of the memory will become write protected. + */ + ldr x0, boot_mmu_config + CORE_MMU_CONFIG_MAP_OFFSET + cbz x0, 1f + /* + * Update cached_mem_end address with load offset since it was + * calculated before relocation. + */ + adr x5, cached_mem_end + ldr x6, [x5] + add x6, x6, x0 + str x6, [x5] + adr x1, _start /* Load address */ + bl relocate +1: +#endif + + bl __get_core_pos + bl enable_mmu +#ifdef CFG_CORE_ASLR + /* + * Reinitialize console, since register_serial_console() has + * previously registered a PA and with ASLR the VA is different + * from the PA. + */ + bl console_init +#endif + +#ifdef CFG_NS_VIRTUALIZATION + /* + * Initialize partition tables for each partition to + * default_partition which has been relocated now to a different VA + */ + bl core_mmu_set_default_prtn_tbl +#endif + +#ifdef CFG_CORE_SEL1_SPMC + mov x0, xzr /* pager not used */ +#else + mov x0, x19 /* pagable part address */ +#endif + mov x1, #-1 + bl boot_init_primary_early + +#ifdef CFG_MEMTAG + init_memtag_per_cpu +#endif + +#ifndef CFG_NS_VIRTUALIZATION + mov x21, sp + adr_l x0, threads + ldr x0, [x0, #THREAD_CTX_STACK_VA_END] + mov sp, x0 + bl thread_get_core_local + mov x22, x0 + str wzr, [x22, #THREAD_CORE_LOCAL_FLAGS] +#endif + mov x0, x20 /* DT address also known as HW_CONFIG */ +#ifdef CFG_CORE_SEL1_SPMC + mov x1, x19 /* TOS_FW_CONFIG DT address */ +#else + mov x1, xzr /* unused */ +#endif + bl boot_init_primary_late +#ifdef CFG_CORE_PAUTH + init_pauth_per_cpu +#endif + +#ifndef CFG_NS_VIRTUALIZATION + mov x0, #THREAD_CLF_TMP + str w0, [x22, #THREAD_CORE_LOCAL_FLAGS] + mov sp, x21 +#endif + +#ifdef _CFG_CORE_STACK_PROTECTOR + /* Update stack canary value */ + sub sp, sp, #0x10 + mov x0, sp + mov x1, #1 + mov x2, #0x8 + bl plat_get_random_stack_canaries + ldr x0, [sp] + adr_l x5, __stack_chk_guard + str x0, [x5] + add sp, sp, #0x10 +#endif + + /* + * In case we've touched memory that secondary CPUs will use before + * they have turned on their D-cache, clean and invalidate the + * D-cache before exiting to normal world. + */ + adr_l x0, __text_start + ldr x1, cached_mem_end + sub x1, x1, x0 + bl dcache_cleaninv_range + + + /* + * Clear current thread id now to allow the thread to be reused on + * next entry. Matches the thread_init_boot_thread in + * boot.c. + */ +#ifndef CFG_NS_VIRTUALIZATION + bl thread_clr_boot_thread +#endif + +#ifdef CFG_CORE_FFA + adr x0, cpu_on_handler + /* + * Compensate for the virtual map offset since cpu_on_handler() is + * called with MMU off. + */ + ldr x1, boot_mmu_config + CORE_MMU_CONFIG_MAP_OFFSET + sub x0, x0, x1 + bl thread_spmc_register_secondary_ep + b thread_ffa_msg_wait +#else + /* + * Pass the vector address returned from main_init Compensate for + * the virtual map offset since cpu_on_handler() is called with MMU + * off. + */ + ldr x0, boot_mmu_config + CORE_MMU_CONFIG_MAP_OFFSET + adr x1, thread_vector_table + sub x1, x1, x0 + mov x0, #TEESMC_OPTEED_RETURN_ENTRY_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +#endif +END_FUNC _start +DECLARE_KEEP_INIT _start + + .section .identity_map.data + .balign 8 +LOCAL_DATA cached_mem_end , : + .skip 8 +END_DATA cached_mem_end + +#if defined(CFG_CORE_ASLR) || defined(CFG_CORE_PHYS_RELOCATABLE) +LOCAL_FUNC relocate , : + /* + * x0 holds relocate offset + * x1 holds load address + */ +#ifdef CFG_WITH_PAGER + adr_l x6, __init_end +#else + adr_l x6, __end +#endif + ldp w2, w3, [x6, #BOOT_EMBDATA_RELOC_OFFSET] + + add x2, x2, x6 /* start of relocations */ + add x3, x3, x2 /* end of relocations */ + + /* + * Relocations are not formatted as Rela64, instead they are in a + * compressed format created by get_reloc_bin() in + * scripts/gen_tee_bin.py + * + * All the R_AARCH64_RELATIVE relocations are translated into a + * list of 32-bit offsets from TEE_LOAD_ADDR. At each address a + * 64-bit value pointed out which increased with the load offset. + */ + +#ifdef CFG_WITH_PAGER + /* + * With pager enabled we can only relocate the pager and init + * parts, the rest has to be done when a page is populated. + */ + sub x6, x6, x1 +#endif + + b 2f + /* Loop over the relocation addresses and process all entries */ +1: ldr w4, [x2], #4 +#ifdef CFG_WITH_PAGER + /* Skip too large addresses */ + cmp x4, x6 + b.ge 2f +#endif + add x4, x4, x1 + ldr x5, [x4] + add x5, x5, x0 + str x5, [x4] + +2: cmp x2, x3 + b.ne 1b + + ret +END_FUNC relocate +#endif + +/* + * void enable_mmu(unsigned long core_pos); + * + * This function depends on being mapped with in the identity map where + * physical address and virtual address is the same. After MMU has been + * enabled the instruction pointer will be updated to execute as the new + * offset instead. Stack pointers and the return address are updated. + */ +LOCAL_FUNC enable_mmu , : , .identity_map + adr x1, boot_mmu_config + load_xregs x1, 0, 2, 6 + /* + * x0 = core_pos + * x2 = tcr_el1 + * x3 = mair_el1 + * x4 = ttbr0_el1_base + * x5 = ttbr0_core_offset + * x6 = load_offset + */ + msr tcr_el1, x2 + msr mair_el1, x3 + + /* + * ttbr0_el1 = ttbr0_el1_base + ttbr0_core_offset * core_pos + */ + madd x1, x5, x0, x4 + msr ttbr0_el1, x1 + msr ttbr1_el1, xzr + isb + + /* Invalidate TLB */ + tlbi vmalle1 + + /* + * Make sure translation table writes have drained into memory and + * the TLB invalidation is complete. + */ + dsb sy + isb + + /* Enable the MMU */ + mrs x1, sctlr_el1 + orr x1, x1, #SCTLR_M + msr sctlr_el1, x1 + isb + + /* Update vbar */ + mrs x1, vbar_el1 + add x1, x1, x6 + msr vbar_el1, x1 + isb + + /* Invalidate instruction cache and branch predictor */ + ic iallu + isb + + /* Enable I and D cache */ + mrs x1, sctlr_el1 + orr x1, x1, #SCTLR_I + orr x1, x1, #SCTLR_C + msr sctlr_el1, x1 + isb + + /* Adjust stack pointers and return address */ + msr spsel, #1 + add sp, sp, x6 + msr spsel, #0 + add sp, sp, x6 + add x30, x30, x6 + + ret +END_FUNC enable_mmu + + .section .identity_map.data + .balign 8 +DATA boot_mmu_config , : /* struct core_mmu_config */ + .skip CORE_MMU_CONFIG_SIZE +END_DATA boot_mmu_config + +FUNC cpu_on_handler , : + mov x19, x0 + mov x20, x1 + mov x21, x30 + + adr x0, reset_vect_table + msr vbar_el1, x0 + isb + + set_sctlr_el1 + isb + +#ifdef CFG_PAN + init_pan +#endif + + /* Enable aborts now that we can receive exceptions */ + msr daifclr, #DAIFBIT_ABT + + bl __get_core_pos + bl enable_mmu + + /* Setup SP_EL0 and SP_EL1, SP will be set to SP_EL0 */ + set_sp + +#ifdef CFG_MEMTAG + init_memtag_per_cpu +#endif +#ifdef CFG_CORE_PAUTH + init_pauth_per_cpu +#endif + + mov x0, x19 + mov x1, x20 +#ifdef CFG_CORE_FFA + bl boot_cpu_on_handler + b thread_ffa_msg_wait +#else + mov x30, x21 + b boot_cpu_on_handler +#endif +END_FUNC cpu_on_handler +DECLARE_KEEP_PAGER cpu_on_handler + +LOCAL_FUNC unhandled_cpu , : + wfi + b unhandled_cpu +END_FUNC unhandled_cpu + +LOCAL_DATA stack_tmp_rel , : + .word stack_tmp - stack_tmp_rel - STACK_TMP_GUARD +END_DATA stack_tmp_rel + + /* + * This macro verifies that the a given vector doesn't exceed the + * architectural limit of 32 instructions. This is meant to be placed + * immedately after the last instruction in the vector. It takes the + * vector entry as the parameter + */ + .macro check_vector_size since + .if (. - \since) > (32 * 4) + .error "Vector exceeds 32 instructions" + .endif + .endm + + .section .identity_map, "ax", %progbits + .align 11 +LOCAL_FUNC reset_vect_table , :, .identity_map, , nobti + /* ----------------------------------------------------- + * Current EL with SP0 : 0x0 - 0x180 + * ----------------------------------------------------- + */ +SynchronousExceptionSP0: + b SynchronousExceptionSP0 + check_vector_size SynchronousExceptionSP0 + + .align 7 +IrqSP0: + b IrqSP0 + check_vector_size IrqSP0 + + .align 7 +FiqSP0: + b FiqSP0 + check_vector_size FiqSP0 + + .align 7 +SErrorSP0: + b SErrorSP0 + check_vector_size SErrorSP0 + + /* ----------------------------------------------------- + * Current EL with SPx: 0x200 - 0x380 + * ----------------------------------------------------- + */ + .align 7 +SynchronousExceptionSPx: + b SynchronousExceptionSPx + check_vector_size SynchronousExceptionSPx + + .align 7 +IrqSPx: + b IrqSPx + check_vector_size IrqSPx + + .align 7 +FiqSPx: + b FiqSPx + check_vector_size FiqSPx + + .align 7 +SErrorSPx: + b SErrorSPx + check_vector_size SErrorSPx + + /* ----------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x580 + * ----------------------------------------------------- + */ + .align 7 +SynchronousExceptionA64: + b SynchronousExceptionA64 + check_vector_size SynchronousExceptionA64 + + .align 7 +IrqA64: + b IrqA64 + check_vector_size IrqA64 + + .align 7 +FiqA64: + b FiqA64 + check_vector_size FiqA64 + + .align 7 +SErrorA64: + b SErrorA64 + check_vector_size SErrorA64 + + /* ----------------------------------------------------- + * Lower EL using AArch32 : 0x0 - 0x180 + * ----------------------------------------------------- + */ + .align 7 +SynchronousExceptionA32: + b SynchronousExceptionA32 + check_vector_size SynchronousExceptionA32 + + .align 7 +IrqA32: + b IrqA32 + check_vector_size IrqA32 + + .align 7 +FiqA32: + b FiqA32 + check_vector_size FiqA32 + + .align 7 +SErrorA32: + b SErrorA32 + check_vector_size SErrorA32 + +END_FUNC reset_vect_table + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/idle.c b/optee_os/core/arch/arm/kernel/idle.c new file mode 100644 index 0000000..c2824fc --- /dev/null +++ b/optee_os/core/arch/arm/kernel/idle.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Huawei Technology Co., Ltd + */ + +#include +#include + +void cpu_idle(void) +{ + dsb(); + wfi(); +} diff --git a/optee_os/core/arch/arm/kernel/kern.ld.S b/optee_os/core/arch/arm/kernel/kern.ld.S new file mode 100644 index 0000000..500c2d2 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/kern.ld.S @@ -0,0 +1,499 @@ +/* SPDX-License-Identifier: (BSD-2-Clause AND MIT) */ +/* + * Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright (c) 2008-2010 Travis Geiselbrecht + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +/* + * Note: + * Clang 11 (ld.lld) generates non-relocatable reference when using ROUNDDOWN() + * from , which does not work with ASLR. + */ +#define LD_ROUNDDOWN(x, y) ((x) - ((x) % (y))) + +OUTPUT_FORMAT(CFG_KERN_LINKER_FORMAT) +OUTPUT_ARCH(CFG_KERN_LINKER_ARCH) + +ENTRY(_start) +SECTIONS +{ + . = TEE_LOAD_ADDR; + ASSERT(!(TEE_LOAD_ADDR & (SMALL_PAGE_SIZE - 1)), + "text start should be page aligned") + __text_start = .; + + /* + * Memory between TEE_LOAD_ADDR and page aligned rounded down + * value will be mapped with unpaged "text" section attributes: + * likely to be read-only/executable. + */ + __flatmap_unpg_rx_start = LD_ROUNDDOWN(__text_start, SMALL_PAGE_SIZE); + + .text : { + KEEP(*(.text._start)) + __identity_map_init_start = .; + __text_data_start = .; + *(.identity_map.data) + __text_data_end = .; + *(.identity_map .identity_map.* \ + /* + * The one below is needed because it's a weak + * symbol that may be overridden by platform + * specific code. + */ + .text.get_core_pos_mpidr) + __identity_map_init_end = .; + KEEP(*(.text.init .text.plat_cpu_reset_early \ + .text.reset .text.reset_primary .text.unhandled_cpu \ + .text.__assert_flat_mapped_range)) + +#ifdef CFG_WITH_PAGER + *(.text) +/* Include list of sections needed for paging */ +#include +#else + *(.text .text.*) +#endif + *(.sram.text.glue_7* .gnu.linkonce.t.*) + . = ALIGN(8); + } + __text_end = .; + +#ifdef CFG_CORE_RODATA_NOEXEC + . = ALIGN(SMALL_PAGE_SIZE); +#endif + __flatmap_unpg_rx_size = . - __flatmap_unpg_rx_start; + __flatmap_unpg_ro_start = .; + + .rodata : ALIGN(8) { + __rodata_start = .; + *(.gnu.linkonce.r.*) +#ifdef CFG_WITH_PAGER + *(.rodata .rodata.__unpaged .rodata.__unpaged.*) +#include +#else + *(.rodata .rodata.*) +#ifndef CFG_CORE_ASLR + . = ALIGN(8); + KEEP(*(SORT(.scattered_array*))); +#endif +#endif + . = ALIGN(8); + __rodata_end = .; + } + +#if defined(CFG_CORE_ASLR) + .data.rel.ro : { +#if !defined(CFG_WITH_PAGER) + . = ALIGN(8); + KEEP(*(SORT(.scattered_array*))); +#endif + *(.data.rel.ro.__unpaged .data.rel.ro.__unpaged.*) + } +#endif + + .got : { *(.got.plt) *(.got) } + .note.gnu.property : { *(.note.gnu.property) } + .plt : { *(.plt) } + + .ctors : ALIGN(8) { + __ctor_list = .; + KEEP(*(.ctors .ctors.* .init_array .init_array.*)) + __ctor_end = .; + } + .dtors : ALIGN(8) { + __dtor_list = .; + KEEP(*(.dtors .dtors.* .fini_array .fini_array.*)) + __dtor_end = .; + } + + /* .ARM.exidx is sorted, so has to go in its own output section. */ + .ARM.exidx : { + __exidx_start = .; + *(.ARM.exidx* .gnu.linkonce.armexidx.*) + __exidx_end = .; + } + + .ARM.extab : { + __extab_start = .; + *(.ARM.extab*) + __extab_end = .; + } + + /* Start page aligned read-write memory */ +#ifdef CFG_CORE_RWDATA_NOEXEC + . = ALIGN(SMALL_PAGE_SIZE); +#endif + __flatmap_unpg_ro_size = . - __flatmap_unpg_ro_start; + +#ifdef CFG_NS_VIRTUALIZATION + __flatmap_nex_rw_start = . ; + .nex_data : ALIGN(8) { + *(.nex_data .nex_data.*) + } + + .nex_bss : ALIGN(8) { + __nex_bss_start = .; + *(.nex_bss .nex_bss.*) + __nex_bss_end = .; + } + + /* + * We want to keep all nexus memory in one place, because + * it should be always mapped and it is easier to map one + * memory region than two. + * Next section are NOLOAD ones, but they are followed + * by sections with data. Thus, this NOLOAD section will + * be included in the resulting binary, filled with zeroes + */ + .nex_stack (NOLOAD) : { + __nozi_stack_start = .; + KEEP(*(.nozi_stack.stack_tmp .nozi_stack.stack_abt)) + . = ALIGN(8); + __nozi_stack_end = .; + } + + .nex_heap (NOLOAD) : { + __nex_heap_start = .; + . += CFG_CORE_NEX_HEAP_SIZE; + . = ALIGN(16 * 1024); + __nex_heap_end = .; + } + .nex_nozi (NOLOAD) : { + ASSERT(!(ABSOLUTE(.) & (16 * 1024 - 1)), "align nozi to 16kB"); + KEEP(*(.nozi.mmu.base_table .nozi.mmu.l2)) + } + + . = ALIGN(SMALL_PAGE_SIZE); + + __flatmap_nex_rw_size = . - __flatmap_nex_rw_start; + __flatmap_nex_rw_end = .; +#endif + + __flatmap_unpg_rw_start = .; + + .data : ALIGN(8) { + /* writable data */ + __data_start_rom = .; + /* in one segment binaries, the rom data address is on top + of the ram data address */ + __data_start = .; + *(.data .data.* .gnu.linkonce.d.*) + . = ALIGN(8); + } + + /* unintialized data */ + .bss : { + __data_end = .; + __bss_start = .; + *(.bss .bss.*) + *(.gnu.linkonce.b.*) + *(COMMON) + . = ALIGN(8); + __bss_end = .; + } + + .heap1 (NOLOAD) : { + /* + * We're keeping track of the padding added before the + * .nozi section so we can do something useful with + * this otherwise wasted memory. + */ + __heap1_start = .; +#ifndef CFG_WITH_PAGER + . += CFG_CORE_HEAP_SIZE; +#endif +#ifdef CFG_WITH_LPAE + . = ALIGN(4 * 1024); +#else + . = ALIGN(16 * 1024); +#endif + __heap1_end = .; + } + /* + * Uninitialized data that shouldn't be zero initialized at + * runtime. + * + * L1 mmu table requires 16 KiB alignment + */ + .nozi (NOLOAD) : { + __nozi_start = .; + KEEP(*(.nozi .nozi.*)) + . = ALIGN(16); + __nozi_end = .; + /* + * If virtualization is enabled, abt and tmp stacks will placed + * at above .nex_stack section and thread stacks will go there + */ + __nozi_stack_start = .; + KEEP(*(.nozi_stack .nozi_stack.*)) + . = ALIGN(8); + __nozi_stack_end = .; + } + +#ifdef CFG_WITH_PAGER + .heap2 (NOLOAD) : { + __heap2_start = .; + /* + * Reserve additional memory for heap, the total should be + * at least CFG_CORE_HEAP_SIZE, but count what has already + * been reserved in .heap1 + */ + . += CFG_CORE_HEAP_SIZE - (__heap1_end - __heap1_start); + . = ALIGN(SMALL_PAGE_SIZE); + __heap2_end = .; + } + + /* Start page aligned read-only memory */ + __flatmap_unpg_rw_size = . - __flatmap_unpg_rw_start; + + __init_start = .; + __flatmap_init_rx_start = .; + + ASSERT(!(__flatmap_init_rx_start & (SMALL_PAGE_SIZE - 1)), + "read-write memory is not paged aligned") + + .text_init : { + __text_init_start = .; +/* + * Include list of sections needed for boot initialization, this list + * overlaps with unpaged.ld.S but since unpaged.ld.S is first all those + * sections will go into the unpaged area. + */ +#include + KEEP(*(.text.startup.*)); + /* Make sure constructor functions are available during init */ + KEEP(*(.text._GLOBAL__sub_*)); + . = ALIGN(8); + __text_init_end = .; + } + +#ifdef CFG_CORE_RODATA_NOEXEC + . = ALIGN(SMALL_PAGE_SIZE); +#endif + __flatmap_init_rx_size = . - __flatmap_init_rx_start; + __flatmap_init_ro_start = .; + + .rodata_init : { + __rodata_init_start = .; +#include +#ifndef CFG_CORE_ASLR + . = ALIGN(8); + KEEP(*(SORT(.scattered_array*))); +#endif + __rodata_init_end = .; + } +#ifdef CFG_CORE_ASLR + .data.rel.ro_init : ALIGN(8) { + KEEP(*(SORT(.scattered_array*))); + } +#endif + . = ALIGN(8); + __ro_and_relro_data_init_end = .; + + __init_end = ALIGN(__ro_and_relro_data_init_end, SMALL_PAGE_SIZE); + __get_tee_init_end = __init_end; + __init_size = __init_end - __init_start; + + /* vcore flat map stops here. No need to page align, rodata follows. */ + __flatmap_init_ro_size = __init_end - __flatmap_init_ro_start; + + .rodata_pageable : ALIGN(8) { + __rodata_pageable_start = .; + *(.rodata*) + __rodata_pageable_end = .; + } + +#ifdef CFG_CORE_RODATA_NOEXEC + . = ALIGN(SMALL_PAGE_SIZE); +#endif + + .text_pageable : ALIGN(8) { + __text_pageable_start = .; + *(.text*) + . = ALIGN(SMALL_PAGE_SIZE); + __text_pageable_end = .; + } + + __pageable_part_end = .; + __pageable_part_start = __init_end; + __pageable_start = __init_start; + __pageable_end = __pageable_part_end; + + ASSERT(TEE_LOAD_ADDR >= TEE_RAM_START, + "Load address before start of physical memory") + ASSERT(TEE_LOAD_ADDR < (TEE_RAM_START + TEE_RAM_PH_SIZE), + "Load address after end of physical memory") + ASSERT((TEE_RAM_START + TEE_RAM_PH_SIZE - __init_end) > + SMALL_PAGE_SIZE, "Too few free pages to initialize paging") + + +#endif /*CFG_WITH_PAGER*/ + +#ifdef CFG_CORE_SANITIZE_KADDRESS + . = TEE_RAM_START + (TEE_RAM_VA_SIZE * 8) / 9 - 8; + . = ALIGN(8); + .asan_shadow : { + __asan_shadow_start = .; + . += TEE_RAM_VA_SIZE / 9; + __asan_shadow_end = .; + __asan_shadow_size = __asan_shadow_end - __asan_shadow_start; + } +#endif /*CFG_CORE_SANITIZE_KADDRESS*/ + + __end = .; + +#ifndef CFG_WITH_PAGER + __init_size = __data_end - TEE_LOAD_ADDR; +#endif + /* + * Guard against moving the location counter backwards in the assignment + * below. + */ + ASSERT(. <= (TEE_RAM_START + TEE_RAM_VA_SIZE), + "TEE_RAM_VA_SIZE is too small") + . = TEE_RAM_START + TEE_RAM_VA_SIZE; + + _end_of_ram = .; + +#ifndef CFG_WITH_PAGER + __flatmap_unpg_rw_size = _end_of_ram - __flatmap_unpg_rw_start; + __get_tee_init_end = .; +#endif + + /* + * These regions will not become a normal part of the dumped + * binary, instead some are interpreted by the dump script and + * converted into suitable format for OP-TEE itself to use. + */ + .dynamic : { *(.dynamic) } + .hash : { *(.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + + .rel : { + *(.rel.*) + } + .rela : { + *(.rela.*) + } +#if !defined(CFG_CORE_ASLR) && !defined(CFG_CORE_PHYS_RELOCATABLE) + ASSERT(SIZEOF(.rel) == 0, "Relocation entries not expected") + ASSERT(SIZEOF(.rela) == 0, "Relocation entries not expected") +#endif + + /DISCARD/ : { + /* Strip unnecessary stuff */ + *(.comment .note .eh_frame .interp) + /* Strip meta variables */ + *(__keep_meta_vars*) + } + +} + +/* Unpaged read-only memories */ +__vcore_unpg_rx_start = __flatmap_unpg_rx_start; +__vcore_unpg_ro_start = __flatmap_unpg_ro_start; +#ifdef CFG_CORE_RODATA_NOEXEC +__vcore_unpg_rx_size = __flatmap_unpg_rx_size; +__vcore_unpg_ro_size = __flatmap_unpg_ro_size; +#else +__vcore_unpg_rx_size = __flatmap_unpg_rx_size + __flatmap_unpg_ro_size; +__vcore_unpg_ro_size = 0; +#endif +__vcore_unpg_rx_end = __vcore_unpg_rx_start + __vcore_unpg_rx_size; +__vcore_unpg_ro_end = __vcore_unpg_ro_start + __vcore_unpg_ro_size; + +/* Unpaged read-write memory */ +__vcore_unpg_rw_start = __flatmap_unpg_rw_start; +__vcore_unpg_rw_size = __flatmap_unpg_rw_size; +__vcore_unpg_rw_end = __vcore_unpg_rw_start + __vcore_unpg_rw_size; + +#ifdef CFG_NS_VIRTUALIZATION +/* Nexus read-write memory */ +__vcore_nex_rw_start = __flatmap_nex_rw_start; +__vcore_nex_rw_size = __flatmap_nex_rw_size; +__vcore_nex_rw_end = __vcore_nex_rw_start + __vcore_nex_rw_size; +#endif + +#ifdef CFG_WITH_PAGER +/* + * Core init mapping shall cover up to end of the physical RAM. + * This is required since the hash table is appended to the + * binary data after the firmware build sequence. + */ +#define __FLATMAP_PAGER_TRAILING_SPACE \ + (TEE_RAM_START + TEE_RAM_PH_SIZE - \ + (__flatmap_init_ro_start + __flatmap_init_ro_size)) + +/* Paged/init read-only memories */ +__vcore_init_rx_start = __flatmap_init_rx_start; +__vcore_init_ro_start = __flatmap_init_ro_start; +#ifdef CFG_CORE_RODATA_NOEXEC +__vcore_init_rx_size = __flatmap_init_rx_size; +__vcore_init_ro_size = __flatmap_init_ro_size + __FLATMAP_PAGER_TRAILING_SPACE; +#else +__vcore_init_rx_size = __flatmap_init_rx_size + __flatmap_init_ro_size + + __FLATMAP_PAGER_TRAILING_SPACE; +__vcore_init_ro_size = 0; +#endif /* CFG_CORE_RODATA_NOEXEC */ +__vcore_init_rx_end = __vcore_init_rx_start + __vcore_init_rx_size; +__vcore_init_ro_end = __vcore_init_ro_start + __vcore_init_ro_size; +#endif /* CFG_WITH_PAGER */ + +#ifdef CFG_CORE_SANITIZE_KADDRESS +__asan_map_start = (__asan_shadow_start / SMALL_PAGE_SIZE) * + SMALL_PAGE_SIZE; +__asan_map_end = ((__asan_shadow_end - 1) / SMALL_PAGE_SIZE) * + SMALL_PAGE_SIZE + SMALL_PAGE_SIZE; +__asan_map_size = __asan_map_end - __asan_map_start; +#endif /*CFG_CORE_SANITIZE_KADDRESS*/ diff --git a/optee_os/core/arch/arm/kernel/link.mk b/optee_os/core/arch/arm/kernel/link.mk new file mode 100644 index 0000000..49e9f4f --- /dev/null +++ b/optee_os/core/arch/arm/kernel/link.mk @@ -0,0 +1,263 @@ +link-out-dir = $(out-dir)/core + +link-script-dummy = $(arch-dir)/kernel/link_dummy.ld +link-script = $(if $(wildcard $(platform-dir)/kern.ld.S), \ + $(platform-dir)/kern.ld.S, \ + $(arch-dir)/kernel/kern.ld.S) +link-script-pp = $(link-out-dir)/kern.ld +link-script-dep = $(link-out-dir)/.kern.ld.d + +AWK = awk + +link-ldflags-common += $(call ld-option,--no-warn-rwx-segments) +ifeq ($(CFG_ARM32_core),y) +link-ldflags-common += $(call ld-option,--no-warn-execstack) +endif + +link-ldflags = $(LDFLAGS) +ifeq ($(call cfg-one-enabled, CFG_CORE_ASLR CFG_CORE_PHYS_RELOCATABLE),y) +link-ldflags += -pie -Bsymbolic -z norelro $(ldflag-apply-dynamic-relocs) +ifeq ($(CFG_ARM64_core),y) +link-ldflags += -z text +else +# Suppression of relocations in read-only segments has not been done yet +link-ldflags += -z notext +endif +endif +ifeq ($(CFG_CORE_BTI),y) +# force-bti tells the linker to warn if some object files lack the .note.gnu.property +# section with the BTI flag, and to turn on the BTI flag in the output anyway. The +# resulting executable would likely fail at runtime so we use this flag along +# with the --fatal-warnings below to check and prevent this situation (with useful +# diagnostics). +link-ldflags += $(call ld-option,-z force-bti) --fatal-warnings +endif +link-ldflags += -T $(link-script-pp) -Map=$(link-out-dir)/tee.map +link-ldflags += --sort-section=alignment +link-ldflags += --fatal-warnings +link-ldflags += --gc-sections +link-ldflags += $(link-ldflags-common) + +link-ldadd = $(LDADD) +link-ldadd += $(ldflags-external) +link-ldadd += $(libdeps) +link-objs := $(filter-out \ + $(out-dir)/$(platform-dir)/link_dummies_paged.o \ + $(out-dir)/$(platform-dir)/link_dummies_init.o \ + $(out-dir)/$(arch-dir)/kernel/link_dummies_paged.o \ + $(out-dir)/$(arch-dir)/kernel/link_dummies_init.o, \ + $(objs)) +link-objs-init := $(filter-out \ + $(out-dir)/$(platform-dir)/link_dummies_init.o \ + $(out-dir)/$(arch-dir)/kernel/link_dummies_init.o, \ + $(objs)) +ldargs-tee.elf := $(link-ldflags) $(link-objs) $(link-out-dir)/version.o \ + $(link-ldadd) $(libgcccore) + +link-script-cppflags := \ + $(filter-out $(CPPFLAGS_REMOVE) $(cppflags-remove), \ + $(nostdinccore) $(CPPFLAGS) \ + $(addprefix -I,$(incdirscore) $(link-out-dir)) \ + $(cppflagscore)) + +ldargs-all_objs := -T $(link-script-dummy) --no-check-sections \ + $(link-ldflags-common) \ + $(link-objs) $(link-ldadd) $(libgcccore) +cleanfiles += $(link-out-dir)/all_objs.o +$(link-out-dir)/all_objs.o: $(objs) $(libdeps) $(MAKEFILE_LIST) + @$(cmd-echo-silent) ' LD $@' + $(q)$(LDcore) $(ldargs-all_objs) -o $@ + +cleanfiles += $(link-out-dir)/unpaged_entries.txt +$(link-out-dir)/unpaged_entries.txt: $(link-out-dir)/all_objs.o + @$(cmd-echo-silent) ' GEN $@' + $(q)$(NMcore) $< | \ + $(AWK) '/ ____keep_pager/ { printf "-u%s ", $$3 }' > $@ + +unpaged-ldargs := -T $(link-script-dummy) --no-check-sections --gc-sections \ + $(link-ldflags-common) +unpaged-ldadd := $(objs) $(link-ldadd) $(libgcccore) +cleanfiles += $(link-out-dir)/unpaged.o +$(link-out-dir)/unpaged.o: $(link-out-dir)/unpaged_entries.txt + @$(cmd-echo-silent) ' LD $@' + $(q)$(LDcore) $(unpaged-ldargs) \ + `cat $(link-out-dir)/unpaged_entries.txt` \ + $(unpaged-ldadd) -o $@ + +cleanfiles += $(link-out-dir)/text_unpaged.ld.S +$(link-out-dir)/text_unpaged.ld.S: $(link-out-dir)/unpaged.o + @$(cmd-echo-silent) ' GEN $@' + $(q)$(READELFcore) -S -W $< | \ + $(PYTHON3) ./scripts/gen_ld_sects.py .text. > $@ + +cleanfiles += $(link-out-dir)/rodata_unpaged.ld.S +$(link-out-dir)/rodata_unpaged.ld.S: $(link-out-dir)/unpaged.o + @$(cmd-echo-silent) ' GEN $@' + $(q)$(READELFcore) -S -W $< | \ + $(PYTHON3) ./scripts/gen_ld_sects.py .rodata. > $@ + + +cleanfiles += $(link-out-dir)/init_entries.txt +$(link-out-dir)/init_entries.txt: $(link-out-dir)/all_objs.o + @$(cmd-echo-silent) ' GEN $@' + $(q)$(NMcore) $< | \ + $(AWK) '/ ____keep_init/ { printf "-u%s ", $$3 }' > $@ + +init-ldargs := -T $(link-script-dummy) --no-check-sections --gc-sections \ + $(link-ldflags-common) +init-ldadd := $(link-objs-init) $(link-out-dir)/version.o $(link-ldadd) \ + $(libgcccore) +cleanfiles += $(link-out-dir)/init.o +$(link-out-dir)/init.o: $(link-out-dir)/init_entries.txt + $(call gen-version-o) + @$(cmd-echo-silent) ' LD $@' + $(q)$(LDcore) $(init-ldargs) \ + `cat $(link-out-dir)/init_entries.txt` \ + $(init-ldadd) -o $@ + +cleanfiles += $(link-out-dir)/text_init.ld.S +$(link-out-dir)/text_init.ld.S: $(link-out-dir)/init.o + @$(cmd-echo-silent) ' GEN $@' + $(q)$(READELFcore) -S -W $< | \ + $(PYTHON3) ./scripts/gen_ld_sects.py .text. > $@ + +cleanfiles += $(link-out-dir)/rodata_init.ld.S +$(link-out-dir)/rodata_init.ld.S: $(link-out-dir)/init.o + @$(cmd-echo-silent) ' GEN $@' + $(q)$(READELFcore) -S -W $< | $(PYTHON3) ./scripts/gen_ld_sects.py .rodata. > $@ + +-include $(link-script-dep) + +link-script-extra-deps += $(link-out-dir)/text_unpaged.ld.S +link-script-extra-deps += $(link-out-dir)/rodata_unpaged.ld.S +link-script-extra-deps += $(link-out-dir)/text_init.ld.S +link-script-extra-deps += $(link-out-dir)/rodata_init.ld.S +link-script-extra-deps += $(conf-file) +cleanfiles += $(link-script-pp) $(link-script-dep) +$(link-script-pp): $(link-script) $(link-script-extra-deps) + @$(cmd-echo-silent) ' CPP $@' + @mkdir -p $(dir $@) + $(q)$(CPPcore) -P -MT $@ -MD -MF $(link-script-dep) \ + $(link-script-cppflags) $< -o $@ + +define update-buildcount + @$(cmd-echo-silent) ' UPD $(1)' + $(q)if [ ! -f $(1) ]; then \ + mkdir -p $(dir $(1)); \ + echo 1 >$(1); \ + else \ + expr 0`cat $(1)` + 1 >$(1); \ + fi +endef + +# filter-out to workaround objdump warning +version-o-cflags = $(filter-out -g3,$(core-platform-cflags) \ + $(platform-cflags) $(cflagscore)) +# SOURCE_DATE_EPOCH defined for reproducible builds +ifneq ($(SOURCE_DATE_EPOCH),) +date-opts = -d @$(SOURCE_DATE_EPOCH) +endif +DATE_STR = `LC_ALL=C date -u $(date-opts)` +BUILD_COUNT_STR = `cat $(link-out-dir)/.buildcount` +CORE_CC_VERSION = `$(CCcore) -v 2>&1 | grep "version " | sed 's/ *$$//'` +define gen-version-o + $(call update-buildcount,$(link-out-dir)/.buildcount) + @$(cmd-echo-silent) ' GEN $(link-out-dir)/version.o' + $(q)echo -e "const char core_v_str[] =" \ + "\"$(TEE_IMPL_VERSION) \"" \ + "\"($(CORE_CC_VERSION)) \"" \ + "\"#$(BUILD_COUNT_STR) \"" \ + "\"$(DATE_STR) \"" \ + "\"$(CFG_KERN_LINKER_ARCH)\";\n" \ + | $(CCcore) $(version-o-cflags) \ + -xc - -c -o $(link-out-dir)/version.o +endef +$(link-out-dir)/version.o: + $(call gen-version-o) + +-include $(link-out-dir)/.tee.elf.cmd +define check-link-objs +$(if $(strip $(filter-out $(link-objs), $(old-link-objs)) + $(filter-out $(old-link-objs), $(link-objs))), FORCE_LINK := FORCE) +endef +$(eval $(call check-link-objs)) + +all: $(link-out-dir)/tee.elf +cleanfiles += $(link-out-dir)/tee.elf $(link-out-dir)/tee.map +cleanfiles += $(link-out-dir)/version.o +cleanfiles += $(link-out-dir)/.buildcount +cleanfiles += $(link-out-dir)/.tee.elf.cmd +$(link-out-dir)/tee.elf: $(link-objs) $(libdeps) $(link-script-pp) $(FORCE_LINK) + @echo "old-link-objs := $(link-objs)" >$(link-out-dir)/.tee.elf.cmd + @$(cmd-echo-silent) ' LD $@' + $(q)$(LDcore) $(ldargs-tee.elf) -o $@ + +all: $(link-out-dir)/tee.dmp +cleanfiles += $(link-out-dir)/tee.dmp +$(link-out-dir)/tee.dmp: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' OBJDUMP $@' + $(q)$(OBJDUMPcore) -l -x -d $< > $@ + +cleanfiles += $(link-out-dir)/tee-pager.bin +$(link-out-dir)/tee-pager.bin: $(link-out-dir)/tee.elf scripts/gen_tee_bin.py + @echo Warning: $@ is deprecated + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) scripts/gen_tee_bin.py --input $< --out_tee_pager_bin $@ + +cleanfiles += $(link-out-dir)/tee-pageable.bin +$(link-out-dir)/tee-pageable.bin: $(link-out-dir)/tee.elf scripts/gen_tee_bin.py + @echo Warning: $@ is deprecated + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) scripts/gen_tee_bin.py --input $< --out_tee_pageable_bin $@ + +all: $(link-out-dir)/tee.bin +cleanfiles += $(link-out-dir)/tee.bin +$(link-out-dir)/tee.bin: $(link-out-dir)/tee.elf scripts/gen_tee_bin.py + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) scripts/gen_tee_bin.py --input $< --out_tee_bin $@ + +all: $(link-out-dir)/tee-header_v2.bin +cleanfiles += $(link-out-dir)/tee-header_v2.bin +$(link-out-dir)/tee-header_v2.bin: $(link-out-dir)/tee.elf \ + scripts/gen_tee_bin.py + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) scripts/gen_tee_bin.py --input $< --out_header_v2 $@ + +all: $(link-out-dir)/tee-pager_v2.bin +cleanfiles += $(link-out-dir)/tee-pager_v2.bin +$(link-out-dir)/tee-pager_v2.bin: $(link-out-dir)/tee.elf scripts/gen_tee_bin.py + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) scripts/gen_tee_bin.py --input $< --out_pager_v2 $@ + +all: $(link-out-dir)/tee-pageable_v2.bin +cleanfiles += $(link-out-dir)/tee-pageable_v2.bin +$(link-out-dir)/tee-pageable_v2.bin: $(link-out-dir)/tee.elf \ + scripts/gen_tee_bin.py + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) scripts/gen_tee_bin.py --input $< --out_pageable_v2 $@ + +all: $(link-out-dir)/tee.symb_sizes +cleanfiles += $(link-out-dir)/tee.symb_sizes +$(link-out-dir)/tee.symb_sizes: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' GEN $@' + $(q)$(NMcore) --print-size --reverse-sort --size-sort $< > $@ + +cleanfiles += $(link-out-dir)/tee.mem_usage +ifneq ($(filter mem_usage,$(MAKECMDGOALS)),) +mem_usage: $(link-out-dir)/tee.mem_usage + +$(link-out-dir)/tee.mem_usage: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) ./scripts/mem_usage.py $< > $@ +endif + +all: $(link-out-dir)/tee-raw.bin +cleanfiles += $(link-out-dir)/tee-raw.bin +$(link-out-dir)/tee-raw.bin: $(link-out-dir)/tee.elf scripts/gen_tee_bin.py + @$(cmd-echo-silent) ' GEN $@' + $(q)scripts/gen_tee_bin.py --input $< --out_tee_raw_bin $@ + +cleanfiles += $(link-out-dir)/tee.srec +$(link-out-dir)/tee.srec: $(link-out-dir)/tee-raw.bin + @$(cmd-echo-silent) ' SREC $@' + $(q)$(OBJCOPYcore) -I binary -O srec $(SRECFLAGS) $< $@ diff --git a/optee_os/core/arch/arm/kernel/link_dummies_init.c b/optee_os/core/arch/arm/kernel/link_dummies_init.c new file mode 100644 index 0000000..5c90271 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/link_dummies_init.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + */ +#include +#include +#include + +void __section(".text.dummy.boot_save_boot_info") +boot_save_boot_info(void *boot_info __unused) +{ +} + +unsigned long __section(".text.dummy.get_aslr_seed") +get_aslr_seed(void *fdt __unused) +{ + return 0; +} + +void __section(".text.dummy.core_init_mmu_map") +core_init_mmu_map(unsigned long seed __unused, + struct core_mmu_config *cfg __unused) +{ +} + +void __section(".text.dummy.boot_init_primary_early") +boot_init_primary_early(unsigned long pageable_part __unused, + unsigned long nsec_entry __unused) +{ +} diff --git a/optee_os/core/arch/arm/kernel/link_dummies_paged.c b/optee_os/core/arch/arm/kernel/link_dummies_paged.c new file mode 100644 index 0000000..d6d65c6 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/link_dummies_paged.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2021, Linaro Limited + * Copyright (c) 2023, Arm Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void __section(".text.dummy.call_preinitcalls") call_preinitcalls(void) +{ +} + +void __section(".text.dummy.call_initcalls") call_initcalls(void) +{ +} + +void __section(".text.dummy.call_finalcalls") call_finalcalls(void) +{ +} + +void __section(".text.dummy.boot_init_primary_late") +boot_init_primary_late(unsigned long fdt __unused, + unsigned long manifest_dt __unused) +{ +} + +uint32_t __section(".text.dummy.__thread_std_smc_entry") +__thread_std_smc_entry(uint32_t a0 __unused, uint32_t a1 __unused, + uint32_t a2 __unused, uint32_t a3 __unused, + uint32_t a4 __unused, uint32_t a5 __unused) +{ + return 0; +} + +const struct mobj_ops mobj_reg_shm_ops __rodata_dummy; +const struct mobj_ops mobj_phys_ops __rodata_dummy; +const struct mobj_ops mobj_virt_ops __rodata_dummy; +const struct mobj_ops mobj_shm_ops __rodata_dummy; +const struct mobj_ops mobj_with_fobj_ops __rodata_dummy; +const struct fobj_ops ops_rwp_paged_iv __rodata_dummy; +const struct fobj_ops ops_rwp_unpaged_iv __rodata_dummy; +const struct fobj_ops ops_ro_paged __rodata_dummy; +const struct fobj_ops ops_ro_reloc_paged __rodata_dummy; +const struct fobj_ops ops_locked_paged __rodata_dummy; +const struct fobj_ops ops_sec_mem __rodata_dummy; +const struct ts_ops user_ta_ops __rodata_dummy; +const struct ts_ops stmm_sp_ops __rodata_dummy; diff --git a/optee_os/core/arch/arm/kernel/link_dummy.ld b/optee_os/core/arch/arm/kernel/link_dummy.ld new file mode 100644 index 0000000..1393088 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/link_dummy.ld @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +SECTIONS +{ + /* + * This seems to make the ARMv7 linker happy with regards to glue_7 + * sections etc. + */ + ..dummy : { } +} + +__asan_map_end = .; +__asan_map_size = .; +__asan_map_start = .; +__asan_shadow_end = .; +__asan_shadow_size = .; +__asan_shadow_start = .; +__bss_end = .; +__bss_start = .; +__ctor_end = .; +__ctor_list = .; +__data_end = .; +__data_start = .; +__end = .; +__end_phys_ddr_overall_section = .; +__end_phys_mem_map_section = .; +__end_phys_nsec_ddr_section = .; +__end_phys_sdp_mem_section = .; +__exidx_end = .; +__exidx_start = .; +__extab_end = .; +__extab_start = .; +__heap1_end = .; +__heap1_start = .; +__heap2_end = .; +__heap2_start = .; +__identity_map_init_end = .; +__identity_map_init_start = .; +__initcall_end = .; +__initcall_start = .; +__init_end = .; +__init_size = .; +__init_start = .; +__nex_bss_end = .; +__nex_bss_start = .; +__nex_heap_end = .; +__nex_heap_start = .; +__nozi_end = .; +__nozi_stack_end = .; +__nozi_stack_start = .; +__nozi_start = .; +__pageable_end = .; +__pageable_part_end = .; +__pageable_part_start = .; +__pageable_start = .; +__rela_end = .; +__rela_start = .; +__rel_end = .; +__rel_start = .; +__ro_and_relro_data_init_end = .; +__rodata_early_ta_end = .; +__rodata_early_ta_start = .; +__rodata_end = .; +__rodata_init_end = .; +__rodata_init_start = .; +__rodata_pageable_end = .; +__rodata_pageable_start = .; +__rodata_start = .; +__start_phys_ddr_overall_section = .; +__start_phys_nsec_ddr_section = .; +__text_init_end = .; +__text_init_start = .; +__text_pageable_end = .; +__text_pageable_start = .; +__text_start = .; +__text_data_start = .; +__text_data_end = .; +__text_end = .; +__tmp_hashes_end = .; +__tmp_hashes_size = .; +__tmp_hashes_start = .; +__vcore_init_ro_end = .; +__vcore_init_ro_size = .; +__vcore_init_ro_start = .; +__vcore_init_rx_end = .; +__vcore_init_rx_size = .; +__vcore_init_rx_start = .; +__vcore_nex_rw_end = .; +__vcore_nex_rw_size = .; +__vcore_nex_rw_start = .; +__vcore_unpg_ro_end = .; +__vcore_unpg_ro_size = .; +__vcore_unpg_ro_start = .; +__vcore_unpg_rw_end = .; +__vcore_unpg_rw_size = .; +__vcore_unpg_rw_start = .; +__vcore_unpg_rx_end = .; +__vcore_unpg_rx_size = .; +__vcore_unpg_rx_start = .; +PROVIDE(core_v_str = 0); +PROVIDE(tee_entry_std = 0); +PROVIDE(init_teecore = 0); diff --git a/optee_os/core/arch/arm/kernel/misc_a32.S b/optee_os/core/arch/arm/kernel/misc_a32.S new file mode 100644 index 0000000..13983da --- /dev/null +++ b/optee_os/core/arch/arm/kernel/misc_a32.S @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2019, Arm Limited. All rights reserved. + */ + +#include +#include +#include +#include + + +/* size_t __get_core_pos(void); */ +FUNC __get_core_pos , : , .identity_map + read_mpidr r0 + b get_core_pos_mpidr +END_FUNC __get_core_pos + +/* size_t get_core_pos_mpidr(uint32_t mpidr); */ +/* Let platforms override this if needed */ +WEAK_FUNC get_core_pos_mpidr , : + mov r3, r0 + + /* + * Shift MPIDR value if it's not already shifted. + * Using logical shift ensures AFF0 to be filled with zeroes. + * This part is necessary even if CFG_CORE_THREAD_SHIFT is 0 because + * MT bit can be set on single threaded systems where all the AFF0 + * values are zeroes. + */ + tst r0, #MPIDR_MT_MASK + lsleq r3, r0, #MPIDR_AFFINITY_BITS + + /* + * At this point the MPIDR layout is always shifted so it looks + * as follows AFF2 -> cluster, AFF1 -> core, AFF0 -> thread + */ +#if CFG_CORE_THREAD_SHIFT == 0 + /* Calculate CorePos = (ClusterId * (cores/cluster)) + CoreId */ + ubfx r0, r3, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS + ubfx r1, r3, #MPIDR_AFF2_SHIFT, #MPIDR_AFFINITY_BITS + add r0, r0, r1, LSL #(CFG_CORE_CLUSTER_SHIFT) +#else + /* + * Calculate CorePos = + * ((ClusterId * (cores/cluster)) + CoreId) * (threads/core) + ThreadId + */ + ubfx r0, r3, #MPIDR_AFF0_SHIFT, #MPIDR_AFFINITY_BITS + ubfx r1, r3, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS + ubfx r2, r3, #MPIDR_AFF2_SHIFT, #MPIDR_AFFINITY_BITS + add r1, r1, r2, LSL #(CFG_CORE_CLUSTER_SHIFT) + add r0, r0, r1, LSL #(CFG_CORE_THREAD_SHIFT) +#endif + + bx lr +END_FUNC get_core_pos_mpidr + +/* + * uint32_t temp_set_mode(int cpu_mode) + * returns cpsr to be set + */ +LOCAL_FUNC temp_set_mode , : + mov r1, r0 + cmp r1, #CPSR_MODE_USR /* update mode: usr -> sys */ + moveq r1, #CPSR_MODE_SYS + cpsid aif /* disable interrupts */ + mrs r0, cpsr /* get cpsr with disabled its*/ + bic r0, #CPSR_MODE_MASK /* clear mode */ + orr r0, r1 /* set expected mode */ + bx lr +END_FUNC temp_set_mode + +/* uint32_t read_mode_sp(int cpu_mode) */ +FUNC read_mode_sp , : + push {r4, lr} +UNWIND( .save {r4, lr}) + mrs r4, cpsr /* save cpsr */ + bl temp_set_mode + msr cpsr, r0 /* set the new mode */ + mov r0, sp /* get the function result */ + msr cpsr, r4 /* back to the old mode */ + pop {r4, pc} +END_FUNC read_mode_sp + +/* uint32_t read_mode_lr(int cpu_mode) */ +FUNC read_mode_lr , : + push {r4, lr} +UNWIND( .save {r4, lr}) + mrs r4, cpsr /* save cpsr */ + bl temp_set_mode + msr cpsr, r0 /* set the new mode */ + mov r0, lr /* get the function result */ + msr cpsr, r4 /* back to the old mode */ + pop {r4, pc} +END_FUNC read_mode_lr + +/* void wait_cycles(unsigned long cycles) */ +FUNC wait_cycles , : + /* Divide by 4 since each loop will take 4 cycles to complete */ + lsrs r0, r0, #2 + bxeq lr +loop: + subs r0, r0, #1 + nop + bne loop + + bx lr +END_FUNC wait_cycles diff --git a/optee_os/core/arch/arm/kernel/misc_a64.S b/optee_os/core/arch/arm/kernel/misc_a64.S new file mode 100644 index 0000000..3e70a0d --- /dev/null +++ b/optee_os/core/arch/arm/kernel/misc_a64.S @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2019, Arm Limited. All rights reserved. + */ + +#include +#include +#include + +/* size_t __get_core_pos(void); */ +FUNC __get_core_pos , : , .identity_map + mrs x0, mpidr_el1 + b get_core_pos_mpidr +END_FUNC __get_core_pos + +/* size_t get_core_pos_mpidr(uint32_t mpidr); */ +/* Let platforms override this if needed */ +WEAK_FUNC get_core_pos_mpidr , : + /* + * Shift MPIDR value if it's not already shifted. + * Using logical shift ensures AFF0 to be filled with zeroes. + * This part is necessary even if CFG_CORE_THREAD_SHIFT is 0 because + * MT bit can be set on single threaded systems where all the AFF0 + * values are zeroes. + */ + tst x0, #MPIDR_MT_MASK + lsl x3, x0, #MPIDR_AFFINITY_BITS + csel x3, x3, x0, eq + + /* + * At this point the MPIDR layout is always shifted so it looks + * as follows AFF2 -> cluster, AFF1 -> core, AFF0 -> thread + */ +#if CFG_CORE_THREAD_SHIFT == 0 + /* Calculate CorePos = (ClusterId * (cores/cluster)) + CoreId */ + ubfx x0, x3, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS + ubfx x1, x3, #MPIDR_AFF2_SHIFT, #MPIDR_AFFINITY_BITS + add x0, x0, x1, LSL #(CFG_CORE_CLUSTER_SHIFT) +#else + /* + * Calculate CorePos = + * ((ClusterId * (cores/cluster)) + CoreId) * (threads/core) + ThreadId + */ + ubfx x0, x3, #MPIDR_AFF0_SHIFT, #MPIDR_AFFINITY_BITS + ubfx x1, x3, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS + ubfx x2, x3, #MPIDR_AFF2_SHIFT, #MPIDR_AFFINITY_BITS + add x1, x1, x2, LSL #(CFG_CORE_CLUSTER_SHIFT) + add x0, x0, x1, LSL #(CFG_CORE_THREAD_SHIFT) +#endif + + ret +END_FUNC get_core_pos_mpidr + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/rpc_io_i2c.c b/optee_os/core/arch/arm/kernel/rpc_io_i2c.c new file mode 100644 index 0000000..439ab9a --- /dev/null +++ b/optee_os/core/arch/arm/kernel/rpc_io_i2c.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020 Foundries Ltd + */ +#include +#include +#include +#include + +/* + * @brief: I2C master transfer request to an I2C slave device. + * It is the responsibility of the caller to validate the number of bytes + * processed by the REE. + * + * @param req: the secure world I2C master request + * @param len: the number of bytes processed by REE + * @returns: TEE_SUCCESS on success, TEE_ERROR_XXX on error. + */ +TEE_Result rpc_io_i2c_transfer(struct rpc_i2c_request *req, size_t *len) +{ + struct thread_param p[4] = { }; + TEE_Result res = TEE_SUCCESS; + struct mobj *mobj = NULL; + uint8_t *va = NULL; + + assert(req); + + if (!len) + return TEE_ERROR_BAD_PARAMETERS; + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_I2C, + THREAD_SHM_TYPE_KERNEL_PRIVATE, + req->buffer_len, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + if (req->mode == RPC_I2C_MODE_WRITE) + memcpy(va, req->buffer, req->buffer_len); + + p[0] = THREAD_PARAM_VALUE(IN, req->mode, req->bus, req->chip); + p[1] = THREAD_PARAM_VALUE(IN, req->flags, 0, 0); + p[2] = THREAD_PARAM_MEMREF(INOUT, mobj, 0, req->buffer_len); + p[3] = THREAD_PARAM_VALUE(OUT, 0, 0, 0); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_I2C_TRANSFER, ARRAY_SIZE(p), p); + if (res != TEE_SUCCESS) + return res; + + /* + * Reporting more bytes than supplied or requested from the I2C chip is + * an REE error + */ + if (p[3].u.value.a > req->buffer_len) + return TEE_ERROR_EXCESS_DATA; + + *len = p[3].u.value.a; + + if (req->mode == RPC_I2C_MODE_READ) + memcpy(req->buffer, va, *len); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/arch/arm/kernel/secure_partition.c b/optee_os/core/arch/arm/kernel/secure_partition.c new file mode 100644 index 0000000..8cdddbe --- /dev/null +++ b/optee_os/core/arch/arm/kernel/secure_partition.c @@ -0,0 +1,1936 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020-2023, Arm Limited. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BOUNCE_BUFFER_SIZE 4096 + +#define SP_MANIFEST_ATTR_READ BIT(0) +#define SP_MANIFEST_ATTR_WRITE BIT(1) +#define SP_MANIFEST_ATTR_EXEC BIT(2) +#define SP_MANIFEST_ATTR_NSEC BIT(3) + +#define SP_MANIFEST_ATTR_RO (SP_MANIFEST_ATTR_READ) +#define SP_MANIFEST_ATTR_RW (SP_MANIFEST_ATTR_READ | \ + SP_MANIFEST_ATTR_WRITE) +#define SP_MANIFEST_ATTR_RX (SP_MANIFEST_ATTR_READ | \ + SP_MANIFEST_ATTR_EXEC) +#define SP_MANIFEST_ATTR_RWX (SP_MANIFEST_ATTR_READ | \ + SP_MANIFEST_ATTR_WRITE | \ + SP_MANIFEST_ATTR_EXEC) + +#define SP_MANIFEST_FLAG_NOBITS BIT(0) + +#define SP_MANIFEST_NS_INT_QUEUED (0x0) +#define SP_MANIFEST_NS_INT_MANAGED_EXIT (0x1) +#define SP_MANIFEST_NS_INT_SIGNALED (0x2) + +#define SP_PKG_HEADER_MAGIC (0x474b5053) +#define SP_PKG_HEADER_VERSION_V1 (0x1) +#define SP_PKG_HEADER_VERSION_V2 (0x2) + +struct sp_pkg_header { + uint32_t magic; + uint32_t version; + uint32_t pm_offset; + uint32_t pm_size; + uint32_t img_offset; + uint32_t img_size; +}; + +struct fip_sp_head fip_sp_list = STAILQ_HEAD_INITIALIZER(fip_sp_list); + +static const struct ts_ops sp_ops; + +/* List that holds all of the loaded SP's */ +static struct sp_sessions_head open_sp_sessions = + TAILQ_HEAD_INITIALIZER(open_sp_sessions); + +static const struct embedded_ts *find_secure_partition(const TEE_UUID *uuid) +{ + const struct sp_image *sp = NULL; + const struct fip_sp *fip_sp = NULL; + + for_each_secure_partition(sp) { + if (!memcmp(&sp->image.uuid, uuid, sizeof(*uuid))) + return &sp->image; + } + + for_each_fip_sp(fip_sp) { + if (!memcmp(&fip_sp->sp_img.image.uuid, uuid, sizeof(*uuid))) + return &fip_sp->sp_img.image; + } + + return NULL; +} + +bool is_sp_ctx(struct ts_ctx *ctx) +{ + return ctx && (ctx->ops == &sp_ops); +} + +static void set_sp_ctx_ops(struct ts_ctx *ctx) +{ + ctx->ops = &sp_ops; +} + +struct sp_session *sp_get_session(uint32_t session_id) +{ + struct sp_session *s = NULL; + + TAILQ_FOREACH(s, &open_sp_sessions, link) { + if (s->endpoint_id == session_id) + return s; + } + + return NULL; +} + +TEE_Result sp_partition_info_get(uint32_t ffa_vers, void *buf, size_t buf_size, + const TEE_UUID *ffa_uuid, size_t *elem_count, + bool count_only) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t part_props = FFA_PART_PROP_DIRECT_REQ_RECV | + FFA_PART_PROP_DIRECT_REQ_SEND; + struct sp_session *s = NULL; + + TAILQ_FOREACH(s, &open_sp_sessions, link) { + if (ffa_uuid && + memcmp(&s->ffa_uuid, ffa_uuid, sizeof(*ffa_uuid))) + continue; + + if (s->state == sp_dead) + continue; + if (!count_only && !res) { + uint32_t uuid_words[4] = { 0 }; + + tee_uuid_to_octets((uint8_t *)uuid_words, &s->ffa_uuid); + res = spmc_fill_partition_entry(ffa_vers, buf, buf_size, + *elem_count, + s->endpoint_id, 1, + part_props, uuid_words); + } + *elem_count += 1; + } + + return res; +} + +bool sp_has_exclusive_access(struct sp_mem_map_region *mem, + struct user_mode_ctx *uctx) +{ + /* + * Check that we have access to the region if it is supposed to be + * mapped to the current context. + */ + if (uctx) { + struct vm_region *region = NULL; + + /* Make sure that each mobj belongs to the SP */ + TAILQ_FOREACH(region, &uctx->vm_info.regions, link) { + if (region->mobj == mem->mobj) + break; + } + + if (!region) + return false; + } + + /* Check that it is not shared with another SP */ + return !sp_mem_is_shared(mem); +} + +static TEE_Result new_session_id(uint16_t *endpoint_id) +{ + struct sp_session *session = NULL; + + *endpoint_id = SPMC_ENDPOINT_ID; + + /* Find the first available endpoint id */ + do { + if (*endpoint_id == UINT16_MAX) + return TEE_ERROR_BAD_FORMAT; + + (*endpoint_id)++; + + session = sp_get_session(*endpoint_id); + } while (session); + + return TEE_SUCCESS; +} + +static TEE_Result sp_create_ctx(const TEE_UUID *bin_uuid, struct sp_session *s) +{ + TEE_Result res = TEE_SUCCESS; + struct sp_ctx *spc = NULL; + + /* Register context */ + spc = calloc(1, sizeof(struct sp_ctx)); + if (!spc) + return TEE_ERROR_OUT_OF_MEMORY; + + spc->open_session = s; + s->ts_sess.ctx = &spc->ts_ctx; + spc->ts_ctx.uuid = *bin_uuid; + + res = vm_info_init(&spc->uctx, &spc->ts_ctx); + if (res) + goto err; + + set_sp_ctx_ops(&spc->ts_ctx); + + return TEE_SUCCESS; + +err: + free(spc); + return res; +} + +/* + * Insert a new sp_session to the sessions list, so that it is ordered + * by boot_order. + */ +static void insert_session_ordered(struct sp_sessions_head *open_sessions, + struct sp_session *session) +{ + struct sp_session *s = NULL; + + if (!open_sessions || !session) + return; + + TAILQ_FOREACH(s, &open_sp_sessions, link) { + if (s->boot_order > session->boot_order) + break; + } + + if (!s) + TAILQ_INSERT_TAIL(open_sessions, session, link); + else + TAILQ_INSERT_BEFORE(s, session, link); +} + +static TEE_Result sp_create_session(struct sp_sessions_head *open_sessions, + const TEE_UUID *bin_uuid, + const uint32_t boot_order, + struct sp_session **sess) +{ + TEE_Result res = TEE_SUCCESS; + struct sp_session *s = calloc(1, sizeof(struct sp_session)); + + if (!s) + return TEE_ERROR_OUT_OF_MEMORY; + + s->boot_order = boot_order; + + res = new_session_id(&s->endpoint_id); + if (res) + goto err; + + DMSG("Loading Secure Partition %pUl", (void *)bin_uuid); + res = sp_create_ctx(bin_uuid, s); + if (res) + goto err; + + insert_session_ordered(open_sessions, s); + *sess = s; + return TEE_SUCCESS; + +err: + free(s); + return res; +} + +static TEE_Result sp_init_set_registers(struct sp_ctx *ctx) +{ + struct thread_ctx_regs *sp_regs = &ctx->sp_regs; + + memset(sp_regs, 0, sizeof(*sp_regs)); + sp_regs->sp = ctx->uctx.stack_ptr; + sp_regs->pc = ctx->uctx.entry_func; + + return TEE_SUCCESS; +} + +TEE_Result sp_map_shared(struct sp_session *s, + struct sp_mem_receiver *receiver, + struct sp_mem *smem, + uint64_t *va) +{ + TEE_Result res = TEE_SUCCESS; + struct sp_ctx *ctx = NULL; + uint32_t perm = TEE_MATTR_UR; + struct sp_mem_map_region *reg = NULL; + + ctx = to_sp_ctx(s->ts_sess.ctx); + + /* Get the permission */ + if (receiver->perm.perm & FFA_MEM_ACC_EXE) + perm |= TEE_MATTR_UX; + + if (receiver->perm.perm & FFA_MEM_ACC_RW) { + if (receiver->perm.perm & FFA_MEM_ACC_EXE) + return TEE_ERROR_ACCESS_CONFLICT; + + perm |= TEE_MATTR_UW; + } + /* + * Currently we don't support passing a va. We can't guarantee that the + * full region will be mapped in a contiguous region. A smem->region can + * have multiple mobj for one share. Currently there doesn't seem to be + * an option to guarantee that these will be mapped in a contiguous va + * space. + */ + if (*va) + return TEE_ERROR_NOT_SUPPORTED; + + SLIST_FOREACH(reg, &smem->regions, link) { + res = vm_map(&ctx->uctx, va, reg->page_count * SMALL_PAGE_SIZE, + perm, 0, reg->mobj, reg->page_offset); + + if (res != TEE_SUCCESS) { + EMSG("Failed to map memory region %#"PRIx32, res); + return res; + } + } + return TEE_SUCCESS; +} + +TEE_Result sp_unmap_ffa_regions(struct sp_session *s, struct sp_mem *smem) +{ + TEE_Result res = TEE_SUCCESS; + vaddr_t vaddr = 0; + size_t len = 0; + struct sp_ctx *ctx = to_sp_ctx(s->ts_sess.ctx); + struct sp_mem_map_region *reg = NULL; + + SLIST_FOREACH(reg, &smem->regions, link) { + vaddr = (vaddr_t)sp_mem_get_va(&ctx->uctx, reg->page_offset, + reg->mobj); + len = reg->page_count * SMALL_PAGE_SIZE; + + res = vm_unmap(&ctx->uctx, vaddr, len); + if (res != TEE_SUCCESS) + return res; + } + + return TEE_SUCCESS; +} + +static TEE_Result sp_dt_get_u64(const void *fdt, int node, const char *property, + uint64_t *value) +{ + const fdt64_t *p = NULL; + int len = 0; + + p = fdt_getprop(fdt, node, property, &len); + if (!p) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (len != sizeof(*p)) + return TEE_ERROR_BAD_FORMAT; + + *value = fdt64_ld(p); + + return TEE_SUCCESS; +} + +static TEE_Result sp_dt_get_u32(const void *fdt, int node, const char *property, + uint32_t *value) +{ + const fdt32_t *p = NULL; + int len = 0; + + p = fdt_getprop(fdt, node, property, &len); + if (!p) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (len != sizeof(*p)) + return TEE_ERROR_BAD_FORMAT; + + *value = fdt32_to_cpu(*p); + + return TEE_SUCCESS; +} + +static TEE_Result sp_dt_get_u16(const void *fdt, int node, const char *property, + uint16_t *value) +{ + const fdt16_t *p = NULL; + int len = 0; + + p = fdt_getprop(fdt, node, property, &len); + if (!p) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (len != sizeof(*p)) + return TEE_ERROR_BAD_FORMAT; + + *value = fdt16_to_cpu(*p); + + return TEE_SUCCESS; +} + +static TEE_Result sp_dt_get_uuid(const void *fdt, int node, + const char *property, TEE_UUID *uuid) +{ + uint32_t uuid_array[4] = { 0 }; + const fdt32_t *p = NULL; + int len = 0; + int i = 0; + + p = fdt_getprop(fdt, node, property, &len); + if (!p) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (len != sizeof(TEE_UUID)) + return TEE_ERROR_BAD_FORMAT; + + for (i = 0; i < 4; i++) + uuid_array[i] = fdt32_to_cpu(p[i]); + + tee_uuid_from_octets(uuid, (uint8_t *)uuid_array); + + return TEE_SUCCESS; +} + +static TEE_Result sp_is_elf_format(const void *fdt, int sp_node, + bool *is_elf_format) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t elf_format = 0; + + res = sp_dt_get_u32(fdt, sp_node, "elf-format", &elf_format); + if (res != TEE_SUCCESS && res != TEE_ERROR_ITEM_NOT_FOUND) + return res; + + *is_elf_format = (elf_format != 0); + + return TEE_SUCCESS; +} + +static TEE_Result sp_binary_open(const TEE_UUID *uuid, + const struct ts_store_ops **ops, + struct ts_store_handle **handle) +{ + TEE_Result res = TEE_ERROR_ITEM_NOT_FOUND; + + SCATTERED_ARRAY_FOREACH(*ops, sp_stores, struct ts_store_ops) { + res = (*ops)->open(uuid, handle); + if (res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + break; + } + + return res; +} + +static TEE_Result load_binary_sp(struct ts_session *s, + struct user_mode_ctx *uctx) +{ + size_t bin_size = 0, bin_size_rounded = 0, bin_page_count = 0; + size_t bb_size = ROUNDUP(BOUNCE_BUFFER_SIZE, SMALL_PAGE_SIZE); + size_t bb_num_pages = bb_size / SMALL_PAGE_SIZE; + const struct ts_store_ops *store_ops = NULL; + struct ts_store_handle *handle = NULL; + TEE_Result res = TEE_SUCCESS; + tee_mm_entry_t *mm = NULL; + struct fobj *fobj = NULL; + struct mobj *mobj = NULL; + uaddr_t base_addr = 0; + uint32_t vm_flags = 0; + unsigned int idx = 0; + vaddr_t va = 0; + + if (!s || !uctx) + return TEE_ERROR_BAD_PARAMETERS; + + DMSG("Loading raw binary format SP %pUl", &uctx->ts_ctx->uuid); + + /* Initialize the bounce buffer */ + fobj = fobj_sec_mem_alloc(bb_num_pages); + mobj = mobj_with_fobj_alloc(fobj, NULL, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(fobj); + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + + res = vm_map(uctx, &va, bb_size, TEE_MATTR_PRW, 0, mobj, 0); + mobj_put(mobj); + if (res) + return res; + + uctx->bbuf = (uint8_t *)va; + uctx->bbuf_size = BOUNCE_BUFFER_SIZE; + + vm_set_ctx(uctx->ts_ctx); + + /* Find TS store and open SP binary */ + res = sp_binary_open(&uctx->ts_ctx->uuid, &store_ops, &handle); + if (res != TEE_SUCCESS) { + EMSG("Failed to open SP binary"); + return res; + } + + /* Query binary size and calculate page count */ + res = store_ops->get_size(handle, &bin_size); + if (res != TEE_SUCCESS) + goto err; + + if (ROUNDUP_OVERFLOW(bin_size, SMALL_PAGE_SIZE, &bin_size_rounded)) { + res = TEE_ERROR_OVERFLOW; + goto err; + } + + bin_page_count = bin_size_rounded / SMALL_PAGE_SIZE; + + /* Allocate memory */ + mm = tee_mm_alloc(&tee_mm_sec_ddr, bin_size_rounded); + if (!mm) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + base_addr = tee_mm_get_smem(mm); + + /* Create mobj */ + mobj = sp_mem_new_mobj(bin_page_count, TEE_MATTR_MEM_TYPE_CACHED, true); + if (!mobj) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err_free_tee_mm; + } + + res = sp_mem_add_pages(mobj, &idx, base_addr, bin_page_count); + if (res) + goto err_free_mobj; + + /* Map memory area for the SP binary */ + va = 0; + res = vm_map(uctx, &va, bin_size_rounded, TEE_MATTR_URWX, + vm_flags, mobj, 0); + if (res) + goto err_free_mobj; + + /* Read SP binary into the previously mapped memory area */ + res = store_ops->read(handle, NULL, (void *)va, bin_size); + if (res) + goto err_unmap; + + /* Set memory protection to allow execution */ + res = vm_set_prot(uctx, va, bin_size_rounded, TEE_MATTR_UX); + if (res) + goto err_unmap; + + mobj_put(mobj); + store_ops->close(handle); + + /* The entry point must be at the beginning of the SP binary. */ + uctx->entry_func = va; + uctx->load_addr = va; + uctx->is_32bit = false; + + s->handle_scall = s->ctx->ops->handle_scall; + + return TEE_SUCCESS; + +err_unmap: + vm_unmap(uctx, va, bin_size_rounded); + +err_free_mobj: + mobj_put(mobj); + +err_free_tee_mm: + tee_mm_free(mm); + +err: + store_ops->close(handle); + + return res; +} + +static TEE_Result sp_open_session(struct sp_session **sess, + struct sp_sessions_head *open_sessions, + const TEE_UUID *ffa_uuid, + const TEE_UUID *bin_uuid, + const uint32_t boot_order, + const void *fdt) +{ + TEE_Result res = TEE_SUCCESS; + struct sp_session *s = NULL; + struct sp_ctx *ctx = NULL; + bool is_elf_format = false; + + if (!find_secure_partition(bin_uuid)) + return TEE_ERROR_ITEM_NOT_FOUND; + + res = sp_create_session(open_sessions, bin_uuid, boot_order, &s); + if (res != TEE_SUCCESS) { + DMSG("sp_create_session failed %#"PRIx32, res); + return res; + } + + ctx = to_sp_ctx(s->ts_sess.ctx); + assert(ctx); + if (!ctx) + return TEE_ERROR_TARGET_DEAD; + *sess = s; + + ts_push_current_session(&s->ts_sess); + + res = sp_is_elf_format(fdt, 0, &is_elf_format); + if (res == TEE_SUCCESS) { + if (is_elf_format) { + /* Load the SP using ldelf. */ + ldelf_load_ldelf(&ctx->uctx); + res = ldelf_init_with_ldelf(&s->ts_sess, &ctx->uctx); + } else { + /* Raw binary format SP */ + res = load_binary_sp(&s->ts_sess, &ctx->uctx); + } + } else { + EMSG("Failed to detect SP format"); + } + + if (res != TEE_SUCCESS) { + EMSG("Failed loading SP %#"PRIx32, res); + ts_pop_current_session(); + return TEE_ERROR_TARGET_DEAD; + } + + /* + * Make the SP ready for its first run. + * Set state to busy to prevent other endpoints from sending messages to + * the SP before its boot phase is done. + */ + s->state = sp_busy; + s->caller_id = 0; + sp_init_set_registers(ctx); + memcpy(&s->ffa_uuid, ffa_uuid, sizeof(*ffa_uuid)); + ts_pop_current_session(); + + return TEE_SUCCESS; +} + +static TEE_Result fdt_get_uuid(const void * const fdt, TEE_UUID *uuid) +{ + const struct fdt_property *description = NULL; + int description_name_len = 0; + + if (fdt_node_check_compatible(fdt, 0, "arm,ffa-manifest-1.0")) { + EMSG("Failed loading SP, manifest not found"); + return TEE_ERROR_BAD_PARAMETERS; + } + + description = fdt_get_property(fdt, 0, "description", + &description_name_len); + if (description) + DMSG("Loading SP: %s", description->data); + + if (sp_dt_get_uuid(fdt, 0, "uuid", uuid)) { + EMSG("Missing or invalid UUID in SP manifest"); + return TEE_ERROR_BAD_FORMAT; + } + + return TEE_SUCCESS; +} + +static TEE_Result copy_and_map_fdt(struct sp_ctx *ctx, const void * const fdt, + void **fdt_copy, size_t *mapped_size) +{ + size_t total_size = ROUNDUP(fdt_totalsize(fdt), SMALL_PAGE_SIZE); + size_t num_pages = total_size / SMALL_PAGE_SIZE; + uint32_t perm = TEE_MATTR_UR | TEE_MATTR_PRW; + TEE_Result res = TEE_SUCCESS; + struct mobj *m = NULL; + struct fobj *f = NULL; + vaddr_t va = 0; + + f = fobj_sec_mem_alloc(num_pages); + m = mobj_with_fobj_alloc(f, NULL, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(f); + if (!m) + return TEE_ERROR_OUT_OF_MEMORY; + + res = vm_map(&ctx->uctx, &va, total_size, perm, 0, m, 0); + mobj_put(m); + if (res) + return res; + + if (fdt_open_into(fdt, (void *)va, total_size)) + return TEE_ERROR_GENERIC; + + *fdt_copy = (void *)va; + *mapped_size = total_size; + + return res; +} + +static void fill_boot_info_1_0(vaddr_t buf, const void *fdt) +{ + struct ffa_boot_info_1_0 *info = (struct ffa_boot_info_1_0 *)buf; + static const char fdt_name[16] = "TYPE_DT\0\0\0\0\0\0\0\0"; + + memcpy(&info->magic, "FF-A", 4); + info->count = 1; + + COMPILE_TIME_ASSERT(sizeof(info->nvp[0].name) == sizeof(fdt_name)); + memcpy(info->nvp[0].name, fdt_name, sizeof(fdt_name)); + info->nvp[0].value = (uintptr_t)fdt; + info->nvp[0].size = fdt_totalsize(fdt); +} + +static void fill_boot_info_1_1(vaddr_t buf, const void *fdt) +{ + size_t desc_offs = ROUNDUP(sizeof(struct ffa_boot_info_header_1_1), 8); + struct ffa_boot_info_header_1_1 *header = + (struct ffa_boot_info_header_1_1 *)buf; + struct ffa_boot_info_1_1 *desc = + (struct ffa_boot_info_1_1 *)(buf + desc_offs); + + header->signature = FFA_BOOT_INFO_SIGNATURE; + header->version = FFA_BOOT_INFO_VERSION; + header->blob_size = desc_offs + sizeof(struct ffa_boot_info_1_1); + header->desc_size = sizeof(struct ffa_boot_info_1_1); + header->desc_count = 1; + header->desc_offset = desc_offs; + + memset(&desc[0].name, 0, sizeof(desc[0].name)); + /* Type: Standard boot info (bit[7] == 0), FDT type */ + desc[0].type = FFA_BOOT_INFO_TYPE_ID_FDT; + /* Flags: Contents field contains an address */ + desc[0].flags = FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_ADDR << + FFA_BOOT_INFO_FLAG_CONTENT_FORMAT_SHIFT; + desc[0].size = fdt_totalsize(fdt); + desc[0].contents = (uintptr_t)fdt; +} + +static TEE_Result create_and_map_boot_info(struct sp_ctx *ctx, const void *fdt, + struct thread_smc_args *args, + vaddr_t *va, size_t *mapped_size) +{ + size_t total_size = ROUNDUP(CFG_SP_INIT_INFO_MAX_SIZE, SMALL_PAGE_SIZE); + size_t num_pages = total_size / SMALL_PAGE_SIZE; + uint32_t perm = TEE_MATTR_UR | TEE_MATTR_PRW; + TEE_Result res = TEE_SUCCESS; + uint32_t sp_ffa_version = 0; + struct fobj *f = NULL; + struct mobj *m = NULL; + uint32_t info_reg = 0; + + res = sp_dt_get_u32(fdt, 0, "ffa-version", &sp_ffa_version); + if (res) + return res; + + f = fobj_sec_mem_alloc(num_pages); + m = mobj_with_fobj_alloc(f, NULL, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(f); + if (!m) + return TEE_ERROR_OUT_OF_MEMORY; + + res = vm_map(&ctx->uctx, va, total_size, perm, 0, m, 0); + mobj_put(m); + if (res) + return res; + + *mapped_size = total_size; + + switch (sp_ffa_version) { + case MAKE_FFA_VERSION(1, 0): + fill_boot_info_1_0(*va, fdt); + break; + case MAKE_FFA_VERSION(1, 1): + fill_boot_info_1_1(*va, fdt); + break; + default: + EMSG("Unknown FF-A version: %#"PRIx32, sp_ffa_version); + return TEE_ERROR_NOT_SUPPORTED; + } + + res = sp_dt_get_u32(fdt, 0, "gp-register-num", &info_reg); + if (res) { + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + /* If the property is not present, set default to x0 */ + info_reg = 0; + } else { + return TEE_ERROR_BAD_FORMAT; + } + } + + switch (info_reg) { + case 0: + args->a0 = *va; + break; + case 1: + args->a1 = *va; + break; + case 2: + args->a2 = *va; + break; + case 3: + args->a3 = *va; + break; + default: + EMSG("Invalid register selected for passing boot info"); + return TEE_ERROR_BAD_FORMAT; + } + + return TEE_SUCCESS; +} + +static TEE_Result handle_fdt_load_relative_mem_regions(struct sp_ctx *ctx, + const void *fdt) +{ + int node = 0; + int subnode = 0; + tee_mm_entry_t *mm = NULL; + TEE_Result res = TEE_SUCCESS; + + /* + * Memory regions are optional in the SP manifest, it's not an error if + * we don't find any. + */ + node = fdt_node_offset_by_compatible(fdt, 0, + "arm,ffa-manifest-memory-regions"); + if (node < 0) + return TEE_SUCCESS; + + fdt_for_each_subnode(subnode, fdt, node) { + uint64_t load_rel_offset = 0; + uint32_t attributes = 0; + uint64_t base_addr = 0; + uint32_t pages_cnt = 0; + uint32_t flags = 0; + uint32_t perm = 0; + size_t size = 0; + vaddr_t va = 0; + + mm = NULL; + + /* Load address relative offset of a memory region */ + if (!sp_dt_get_u64(fdt, subnode, "load-address-relative-offset", + &load_rel_offset)) { + va = ctx->uctx.load_addr + load_rel_offset; + } else { + /* Skip non load address relative memory regions */ + continue; + } + + if (!sp_dt_get_u64(fdt, subnode, "base-address", &base_addr)) { + EMSG("Both base-address and load-address-relative-offset fields are set"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Size of memory region as count of 4K pages */ + if (sp_dt_get_u32(fdt, subnode, "pages-count", &pages_cnt)) { + EMSG("Mandatory field is missing: pages-count"); + return TEE_ERROR_BAD_FORMAT; + } + + if (MUL_OVERFLOW(pages_cnt, SMALL_PAGE_SIZE, &size)) + return TEE_ERROR_OVERFLOW; + + /* Memory region attributes */ + if (sp_dt_get_u32(fdt, subnode, "attributes", &attributes)) { + EMSG("Mandatory field is missing: attributes"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Check instruction and data access permissions */ + switch (attributes & SP_MANIFEST_ATTR_RWX) { + case SP_MANIFEST_ATTR_RO: + perm = TEE_MATTR_UR; + break; + case SP_MANIFEST_ATTR_RW: + perm = TEE_MATTR_URW; + break; + case SP_MANIFEST_ATTR_RX: + perm = TEE_MATTR_URX; + break; + default: + EMSG("Invalid memory access permissions"); + return TEE_ERROR_BAD_FORMAT; + } + + res = sp_dt_get_u32(fdt, subnode, "load-flags", &flags); + if (res != TEE_SUCCESS && res != TEE_ERROR_ITEM_NOT_FOUND) { + EMSG("Optional field with invalid value: flags"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Load relative regions must be secure */ + if (attributes & SP_MANIFEST_ATTR_NSEC) { + EMSG("Invalid memory security attribute"); + return TEE_ERROR_BAD_FORMAT; + } + + if (flags & SP_MANIFEST_FLAG_NOBITS) { + /* + * NOBITS flag is set, which means that loaded binary + * doesn't contain this area, so it's need to be + * allocated. + */ + struct mobj *m = NULL; + unsigned int idx = 0; + + mm = tee_mm_alloc(&tee_mm_sec_ddr, size); + if (!mm) + return TEE_ERROR_OUT_OF_MEMORY; + + base_addr = tee_mm_get_smem(mm); + + m = sp_mem_new_mobj(pages_cnt, + TEE_MATTR_MEM_TYPE_CACHED, true); + if (!m) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err_mm_free; + } + + res = sp_mem_add_pages(m, &idx, base_addr, pages_cnt); + if (res) { + mobj_put(m); + goto err_mm_free; + } + + res = vm_map(&ctx->uctx, &va, size, perm, 0, m, 0); + mobj_put(m); + if (res) + goto err_mm_free; + } else { + /* + * If NOBITS is not present the memory area is already + * mapped and only need to set the correct permissions. + */ + res = vm_set_prot(&ctx->uctx, va, size, perm); + if (res) + return res; + } + } + + return TEE_SUCCESS; + +err_mm_free: + tee_mm_free(mm); + return res; +} + +static TEE_Result handle_fdt_dev_regions(struct sp_ctx *ctx, void *fdt) +{ + int node = 0; + int subnode = 0; + TEE_Result res = TEE_SUCCESS; + const char *dt_device_match_table = { + "arm,ffa-manifest-device-regions", + }; + + /* + * Device regions are optional in the SP manifest, it's not an error if + * we don't find any + */ + node = fdt_node_offset_by_compatible(fdt, 0, dt_device_match_table); + if (node < 0) + return TEE_SUCCESS; + + fdt_for_each_subnode(subnode, fdt, node) { + uint64_t base_addr = 0; + uint32_t pages_cnt = 0; + uint32_t attributes = 0; + struct mobj *m = NULL; + bool is_secure = true; + uint32_t perm = 0; + vaddr_t va = 0; + unsigned int idx = 0; + + /* + * Physical base address of a device MMIO region. + * Currently only physically contiguous region is supported. + */ + if (sp_dt_get_u64(fdt, subnode, "base-address", &base_addr)) { + EMSG("Mandatory field is missing: base-address"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Total size of MMIO region as count of 4K pages */ + if (sp_dt_get_u32(fdt, subnode, "pages-count", &pages_cnt)) { + EMSG("Mandatory field is missing: pages-count"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Data access, instruction access and security attributes */ + if (sp_dt_get_u32(fdt, subnode, "attributes", &attributes)) { + EMSG("Mandatory field is missing: attributes"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Check instruction and data access permissions */ + switch (attributes & SP_MANIFEST_ATTR_RWX) { + case SP_MANIFEST_ATTR_RO: + perm = TEE_MATTR_UR; + break; + case SP_MANIFEST_ATTR_RW: + perm = TEE_MATTR_URW; + break; + default: + EMSG("Invalid memory access permissions"); + return TEE_ERROR_BAD_FORMAT; + } + + /* + * The SP is a secure endpoint, security attribute can be + * secure or non-secure + */ + if (attributes & SP_MANIFEST_ATTR_NSEC) + is_secure = false; + + /* Memory attributes must be Device-nGnRnE */ + m = sp_mem_new_mobj(pages_cnt, TEE_MATTR_MEM_TYPE_STRONGLY_O, + is_secure); + if (!m) + return TEE_ERROR_OUT_OF_MEMORY; + + res = sp_mem_add_pages(m, &idx, (paddr_t)base_addr, pages_cnt); + if (res) { + mobj_put(m); + return res; + } + + res = vm_map(&ctx->uctx, &va, pages_cnt * SMALL_PAGE_SIZE, + perm, 0, m, 0); + mobj_put(m); + if (res) + return res; + + /* + * Overwrite the device region's PA in the fdt with the VA. This + * fdt will be passed to the SP. + */ + res = fdt_setprop_u64(fdt, subnode, "base-address", va); + + /* + * Unmap the region if the overwrite failed since the SP won't + * be able to access it without knowing the VA. + */ + if (res) { + vm_unmap(&ctx->uctx, va, pages_cnt * SMALL_PAGE_SIZE); + return res; + } + } + + return TEE_SUCCESS; +} + +static TEE_Result swap_sp_endpoints(uint32_t endpoint_id, + uint32_t new_endpoint_id) +{ + struct sp_session *session = sp_get_session(endpoint_id); + uint32_t manifest_endpoint_id = 0; + + /* + * We don't know in which order the SPs are loaded. The endpoint ID + * defined in the manifest could already be generated by + * new_session_id() and used by another SP. If this is the case, we swap + * the ID's of the two SPs. We also have to make sure that the ID's are + * not defined twice in the manifest. + */ + + /* The endpoint ID was not assigned yet */ + if (!session) + return TEE_SUCCESS; + + /* + * Read the manifest file from the SP who originally had the endpoint. + * We can safely swap the endpoint ID's if the manifest file doesn't + * have an endpoint ID defined. + */ + if (!sp_dt_get_u32(session->fdt, 0, "id", &manifest_endpoint_id)) { + assert(manifest_endpoint_id == endpoint_id); + EMSG("SP: Found duplicated endpoint ID %#"PRIx32, endpoint_id); + return TEE_ERROR_ACCESS_CONFLICT; + } + + session->endpoint_id = new_endpoint_id; + + return TEE_SUCCESS; +} + +static TEE_Result read_manifest_endpoint_id(struct sp_session *s) +{ + uint32_t endpoint_id = 0; + + /* + * The endpoint ID can be optionally defined in the manifest file. We + * have to map the ID inside the manifest to the SP if it's defined. + * If not, the endpoint ID generated inside new_session_id() will be + * used. + */ + if (!sp_dt_get_u32(s->fdt, 0, "id", &endpoint_id)) { + TEE_Result res = TEE_ERROR_GENERIC; + + if (endpoint_id <= SPMC_ENDPOINT_ID) + return TEE_ERROR_BAD_FORMAT; + + res = swap_sp_endpoints(endpoint_id, s->endpoint_id); + if (res) + return res; + + DMSG("SP: endpoint ID (0x%"PRIx32") found in manifest", + endpoint_id); + /* Assign the endpoint ID to the current SP */ + s->endpoint_id = endpoint_id; + } + return TEE_SUCCESS; +} + +static TEE_Result handle_fdt_mem_regions(struct sp_ctx *ctx, void *fdt) +{ + int node = 0; + int subnode = 0; + tee_mm_entry_t *mm = NULL; + TEE_Result res = TEE_SUCCESS; + + /* + * Memory regions are optional in the SP manifest, it's not an error if + * we don't find any. + */ + node = fdt_node_offset_by_compatible(fdt, 0, + "arm,ffa-manifest-memory-regions"); + if (node < 0) + return TEE_SUCCESS; + + fdt_for_each_subnode(subnode, fdt, node) { + uint64_t load_rel_offset = 0; + bool alloc_needed = false; + uint32_t attributes = 0; + uint64_t base_addr = 0; + uint32_t pages_cnt = 0; + bool is_secure = true; + struct mobj *m = NULL; + unsigned int idx = 0; + uint32_t perm = 0; + size_t size = 0; + vaddr_t va = 0; + + mm = NULL; + + /* Load address relative offset of a memory region */ + if (!sp_dt_get_u64(fdt, subnode, "load-address-relative-offset", + &load_rel_offset)) { + /* + * At this point the memory region is already mapped by + * handle_fdt_load_relative_mem_regions. + * Only need to set the base-address in the manifest and + * then skip the rest of the mapping process. + */ + va = ctx->uctx.load_addr + load_rel_offset; + res = fdt_setprop_u64(fdt, subnode, "base-address", va); + if (res) + return res; + + continue; + } + + /* + * Base address of a memory region. + * If not present, we have to allocate the specified memory. + * If present, this field could specify a PA or VA. Currently + * only a PA is supported. + */ + if (sp_dt_get_u64(fdt, subnode, "base-address", &base_addr)) + alloc_needed = true; + + /* Size of memory region as count of 4K pages */ + if (sp_dt_get_u32(fdt, subnode, "pages-count", &pages_cnt)) { + EMSG("Mandatory field is missing: pages-count"); + return TEE_ERROR_BAD_FORMAT; + } + + if (MUL_OVERFLOW(pages_cnt, SMALL_PAGE_SIZE, &size)) + return TEE_ERROR_OVERFLOW; + + /* + * Memory region attributes: + * - Instruction/data access permissions + * - Cacheability/shareability attributes + * - Security attributes + * + * Cacheability/shareability attributes can be ignored for now. + * OP-TEE only supports a single type for normal cached memory + * and currently there is no use case that would require to + * change this. + */ + if (sp_dt_get_u32(fdt, subnode, "attributes", &attributes)) { + EMSG("Mandatory field is missing: attributes"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Check instruction and data access permissions */ + switch (attributes & SP_MANIFEST_ATTR_RWX) { + case SP_MANIFEST_ATTR_RO: + perm = TEE_MATTR_UR; + break; + case SP_MANIFEST_ATTR_RW: + perm = TEE_MATTR_URW; + break; + case SP_MANIFEST_ATTR_RX: + perm = TEE_MATTR_URX; + break; + default: + EMSG("Invalid memory access permissions"); + return TEE_ERROR_BAD_FORMAT; + } + + /* + * The SP is a secure endpoint, security attribute can be + * secure or non-secure. + * The SPMC cannot allocate non-secure memory, i.e. if the base + * address is missing this attribute must be secure. + */ + if (attributes & SP_MANIFEST_ATTR_NSEC) { + if (alloc_needed) { + EMSG("Invalid memory security attribute"); + return TEE_ERROR_BAD_FORMAT; + } + is_secure = false; + } + + if (alloc_needed) { + /* Base address is missing, we have to allocate */ + mm = tee_mm_alloc(&tee_mm_sec_ddr, size); + if (!mm) + return TEE_ERROR_OUT_OF_MEMORY; + + base_addr = tee_mm_get_smem(mm); + } + + m = sp_mem_new_mobj(pages_cnt, TEE_MATTR_MEM_TYPE_CACHED, + is_secure); + if (!m) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err_mm_free; + } + + res = sp_mem_add_pages(m, &idx, base_addr, pages_cnt); + if (res) { + mobj_put(m); + goto err_mm_free; + } + + res = vm_map(&ctx->uctx, &va, size, perm, 0, m, 0); + mobj_put(m); + if (res) + goto err_mm_free; + + /* + * Overwrite the memory region's base address in the fdt with + * the VA. This fdt will be passed to the SP. + * If the base-address field was not present in the original + * fdt, this function will create it. This doesn't cause issues + * since the necessary extra space has been allocated when + * opening the fdt. + */ + res = fdt_setprop_u64(fdt, subnode, "base-address", va); + + /* + * Unmap the region if the overwrite failed since the SP won't + * be able to access it without knowing the VA. + */ + if (res) { + vm_unmap(&ctx->uctx, va, size); + goto err_mm_free; + } + } + + return TEE_SUCCESS; + +err_mm_free: + tee_mm_free(mm); + return res; +} + +static TEE_Result handle_tpm_event_log(struct sp_ctx *ctx, void *fdt) +{ + uint32_t perm = TEE_MATTR_URW | TEE_MATTR_PRW; + uint32_t dummy_size __maybe_unused = 0; + TEE_Result res = TEE_SUCCESS; + size_t page_count = 0; + struct fobj *f = NULL; + struct mobj *m = NULL; + vaddr_t log_addr = 0; + size_t log_size = 0; + int node = 0; + + node = fdt_node_offset_by_compatible(fdt, 0, "arm,tpm_event_log"); + if (node < 0) + return TEE_SUCCESS; + + /* Checking the existence and size of the event log properties */ + if (sp_dt_get_u64(fdt, node, "tpm_event_log_addr", &log_addr)) { + EMSG("tpm_event_log_addr not found or has invalid size"); + return TEE_ERROR_BAD_FORMAT; + } + + if (sp_dt_get_u32(fdt, node, "tpm_event_log_size", &dummy_size)) { + EMSG("tpm_event_log_size not found or has invalid size"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Validating event log */ + res = tpm_get_event_log_size(&log_size); + if (res) + return res; + + if (!log_size) { + EMSG("Empty TPM event log was provided"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + /* Allocating memory area for the event log to share with the SP */ + page_count = ROUNDUP_DIV(log_size, SMALL_PAGE_SIZE); + + f = fobj_sec_mem_alloc(page_count); + m = mobj_with_fobj_alloc(f, NULL, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(f); + if (!m) + return TEE_ERROR_OUT_OF_MEMORY; + + res = vm_map(&ctx->uctx, &log_addr, log_size, perm, 0, m, 0); + mobj_put(m); + if (res) + return res; + + /* Copy event log */ + res = tpm_get_event_log((void *)log_addr, &log_size); + if (res) + goto err_unmap; + + /* Setting event log details in the manifest */ + res = fdt_setprop_u64(fdt, node, "tpm_event_log_addr", log_addr); + if (res) + goto err_unmap; + + res = fdt_setprop_u32(fdt, node, "tpm_event_log_size", log_size); + if (res) + goto err_unmap; + + return TEE_SUCCESS; + +err_unmap: + vm_unmap(&ctx->uctx, log_addr, log_size); + + return res; +} + +/* + * Note: this function is called only on the primary CPU. It assumes that the + * features present on the primary CPU are available on all of the secondary + * CPUs as well. + */ +static TEE_Result handle_hw_features(void *fdt) +{ + uint32_t val __maybe_unused = 0; + TEE_Result res = TEE_SUCCESS; + int node = 0; + + /* + * HW feature descriptions are optional in the SP manifest, it's not an + * error if we don't find any. + */ + node = fdt_node_offset_by_compatible(fdt, 0, "arm,hw-features"); + if (node < 0) + return TEE_SUCCESS; + + /* Modify the crc32 property only if it's already present */ + if (!sp_dt_get_u32(fdt, node, "crc32", &val)) { + res = fdt_setprop_u32(fdt, node, "crc32", + feat_crc32_implemented()); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +static TEE_Result read_ns_interrupts_action(const void *fdt, + struct sp_session *s) +{ + TEE_Result res = TEE_ERROR_BAD_PARAMETERS; + + res = sp_dt_get_u32(fdt, 0, "ns-interrupts-action", &s->ns_int_mode); + + if (res) { + EMSG("Mandatory property is missing: ns-interrupts-action"); + return res; + } + + switch (s->ns_int_mode) { + case SP_MANIFEST_NS_INT_QUEUED: + case SP_MANIFEST_NS_INT_SIGNALED: + /* OK */ + break; + + case SP_MANIFEST_NS_INT_MANAGED_EXIT: + EMSG("Managed exit is not implemented"); + return TEE_ERROR_NOT_IMPLEMENTED; + + default: + EMSG("Invalid ns-interrupts-action value: %"PRIu32, + s->ns_int_mode); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sp_init_uuid(const TEE_UUID *bin_uuid, const void * const fdt) +{ + TEE_Result res = TEE_SUCCESS; + struct sp_session *sess = NULL; + TEE_UUID ffa_uuid = {}; + uint16_t boot_order = 0; + uint32_t boot_order_arg = 0; + + res = fdt_get_uuid(fdt, &ffa_uuid); + if (res) + return res; + + res = sp_dt_get_u16(fdt, 0, "boot-order", &boot_order); + if (res == TEE_SUCCESS) { + boot_order_arg = boot_order; + } else if (res == TEE_ERROR_ITEM_NOT_FOUND) { + boot_order_arg = UINT32_MAX; + } else { + EMSG("Failed reading boot-order property err:%#"PRIx32, res); + return res; + } + + res = sp_open_session(&sess, + &open_sp_sessions, + &ffa_uuid, bin_uuid, boot_order_arg, fdt); + if (res) + return res; + + sess->fdt = fdt; + + res = read_manifest_endpoint_id(sess); + if (res) + return res; + DMSG("endpoint is 0x%"PRIx16, sess->endpoint_id); + + res = read_ns_interrupts_action(fdt, sess); + if (res) + return res; + + return TEE_SUCCESS; +} + +static TEE_Result sp_first_run(struct sp_session *sess) +{ + TEE_Result res = TEE_SUCCESS; + struct thread_smc_args args = { }; + struct sp_ctx *ctx = NULL; + vaddr_t boot_info_va = 0; + size_t boot_info_size = 0; + void *fdt_copy = NULL; + size_t fdt_size = 0; + + ctx = to_sp_ctx(sess->ts_sess.ctx); + ts_push_current_session(&sess->ts_sess); + sess->is_initialized = false; + + /* + * Load relative memory regions must be handled before doing any other + * mapping to prevent conflicts in the VA space. + */ + res = handle_fdt_load_relative_mem_regions(ctx, sess->fdt); + if (res) { + ts_pop_current_session(); + return res; + } + + res = copy_and_map_fdt(ctx, sess->fdt, &fdt_copy, &fdt_size); + if (res) + goto out; + + res = handle_fdt_dev_regions(ctx, fdt_copy); + if (res) + goto out; + + res = handle_fdt_mem_regions(ctx, fdt_copy); + if (res) + goto out; + + if (IS_ENABLED(CFG_CORE_TPM_EVENT_LOG)) { + res = handle_tpm_event_log(ctx, fdt_copy); + if (res) + goto out; + } + + res = handle_hw_features(fdt_copy); + if (res) + goto out; + + res = create_and_map_boot_info(ctx, fdt_copy, &args, &boot_info_va, + &boot_info_size); + if (res) + goto out; + + ts_pop_current_session(); + + res = sp_enter(&args, sess); + if (res) { + ts_push_current_session(&sess->ts_sess); + goto out; + } + + spmc_sp_msg_handler(&args, sess); + + ts_push_current_session(&sess->ts_sess); + sess->is_initialized = true; + +out: + /* Free the boot info page from the SP memory */ + vm_unmap(&ctx->uctx, boot_info_va, boot_info_size); + vm_unmap(&ctx->uctx, (vaddr_t)fdt_copy, fdt_size); + ts_pop_current_session(); + + return res; +} + +TEE_Result sp_enter(struct thread_smc_args *args, struct sp_session *sp) +{ + TEE_Result res = TEE_SUCCESS; + struct sp_ctx *ctx = to_sp_ctx(sp->ts_sess.ctx); + + ctx->sp_regs.x[0] = args->a0; + ctx->sp_regs.x[1] = args->a1; + ctx->sp_regs.x[2] = args->a2; + ctx->sp_regs.x[3] = args->a3; + ctx->sp_regs.x[4] = args->a4; + ctx->sp_regs.x[5] = args->a5; + ctx->sp_regs.x[6] = args->a6; + ctx->sp_regs.x[7] = args->a7; + + res = sp->ts_sess.ctx->ops->enter_invoke_cmd(&sp->ts_sess, 0); + + args->a0 = ctx->sp_regs.x[0]; + args->a1 = ctx->sp_regs.x[1]; + args->a2 = ctx->sp_regs.x[2]; + args->a3 = ctx->sp_regs.x[3]; + args->a4 = ctx->sp_regs.x[4]; + args->a5 = ctx->sp_regs.x[5]; + args->a6 = ctx->sp_regs.x[6]; + args->a7 = ctx->sp_regs.x[7]; + + return res; +} + +/* + * According to FF-A v1.1 section 8.3.1.4 if a caller requires less permissive + * active on NS interrupt than the callee, the callee must inherit the caller's + * configuration. + * Each SP's own NS action setting is stored in ns_int_mode. The effective + * action will be MIN([self action], [caller's action]) which is stored in the + * ns_int_mode_inherited field. + */ +static void sp_cpsr_configure_foreign_interrupts(struct sp_session *s, + struct ts_session *caller, + uint64_t *cpsr) +{ + if (caller) { + struct sp_session *caller_sp = to_sp_session(caller); + + s->ns_int_mode_inherited = MIN(caller_sp->ns_int_mode_inherited, + s->ns_int_mode); + } else { + s->ns_int_mode_inherited = s->ns_int_mode; + } + + if (s->ns_int_mode_inherited == SP_MANIFEST_NS_INT_QUEUED) + *cpsr |= SHIFT_U32(THREAD_EXCP_FOREIGN_INTR, + ARM32_CPSR_F_SHIFT); + else + *cpsr &= ~SHIFT_U32(THREAD_EXCP_FOREIGN_INTR, + ARM32_CPSR_F_SHIFT); +} + +static TEE_Result sp_enter_invoke_cmd(struct ts_session *s, + uint32_t cmd __unused) +{ + struct sp_ctx *ctx = to_sp_ctx(s->ctx); + TEE_Result res = TEE_SUCCESS; + uint32_t exceptions = 0; + struct sp_session *sp_s = to_sp_session(s); + struct ts_session *sess = NULL; + struct thread_ctx_regs *sp_regs = NULL; + uint32_t thread_id = THREAD_ID_INVALID; + struct ts_session *caller = NULL; + uint32_t rpc_target_info = 0; + uint32_t panicked = false; + uint32_t panic_code = 0; + + bm_timestamp(); + + sp_regs = &ctx->sp_regs; + ts_push_current_session(s); + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + /* Enable/disable foreign interrupts in CPSR/SPSR */ + caller = ts_get_calling_session(); + sp_cpsr_configure_foreign_interrupts(sp_s, caller, &sp_regs->cpsr); + + /* + * Store endpoint ID and thread ID in rpc_target_info. This will be used + * as w1 in FFA_INTERRUPT in case of a foreign interrupt. + */ + rpc_target_info = thread_get_tsd()->rpc_target_info; + thread_id = thread_get_id(); + assert(thread_id <= UINT16_MAX); + thread_get_tsd()->rpc_target_info = + FFA_TARGET_INFO_SET(sp_s->endpoint_id, thread_id); + + __thread_enter_user_mode(sp_regs, &panicked, &panic_code); + + /* Restore rpc_target_info */ + thread_get_tsd()->rpc_target_info = rpc_target_info; + + thread_unmask_exceptions(exceptions); + + thread_user_clear_vfp(&ctx->uctx); + + if (panicked) { + DMSG("SP panicked with code %#"PRIx32, panic_code); + abort_print_current_ts(); + + sess = ts_pop_current_session(); + cpu_spin_lock(&sp_s->spinlock); + sp_s->state = sp_dead; + cpu_spin_unlock(&sp_s->spinlock); + + return TEE_ERROR_TARGET_DEAD; + } + + sess = ts_pop_current_session(); + assert(sess == s); + + bm_timestamp(); + + return res; +} + +/* We currently don't support 32 bits */ +#ifdef ARM64 +static void sp_svc_store_registers(struct thread_scall_regs *regs, + struct thread_ctx_regs *sp_regs) +{ + COMPILE_TIME_ASSERT(sizeof(sp_regs->x[0]) == sizeof(regs->x0)); + memcpy(sp_regs->x, ®s->x0, 31 * sizeof(regs->x0)); + sp_regs->pc = regs->elr; + sp_regs->sp = regs->sp_el0; +} +#endif + +static bool sp_handle_scall(struct thread_scall_regs *regs) +{ + struct ts_session *ts = ts_get_current_session(); + struct sp_ctx *uctx = to_sp_ctx(ts->ctx); + struct sp_session *s = uctx->open_session; + + assert(s); + + sp_svc_store_registers(regs, &uctx->sp_regs); + + regs->x0 = 0; + regs->x1 = 0; /* panic */ + regs->x2 = 0; /* panic code */ + + /* + * All the registers of the SP are saved in the SP session by the SVC + * handler. + * We always return to S-El1 after handling the SVC. We will continue + * in sp_enter_invoke_cmd() (return from __thread_enter_user_mode). + * The sp_enter() function copies the FF-A parameters (a0-a7) from the + * saved registers to the thread_smc_args. The thread_smc_args object is + * afterward used by the spmc_sp_msg_handler() to handle the + * FF-A message send by the SP. + */ + return false; +} + +static void sp_dump_state(struct ts_ctx *ctx) +{ + struct sp_ctx *utc = to_sp_ctx(ctx); + + if (utc->uctx.dump_entry_func) { + TEE_Result res = ldelf_dump_state(&utc->uctx); + + if (!res || res == TEE_ERROR_TARGET_DEAD) + return; + } + + user_mode_ctx_print_mappings(&utc->uctx); +} + +static const struct ts_ops sp_ops = { + .enter_invoke_cmd = sp_enter_invoke_cmd, + .handle_scall = sp_handle_scall, + .dump_state = sp_dump_state, +}; + +static TEE_Result process_sp_pkg(uint64_t sp_pkg_pa, TEE_UUID *sp_uuid) +{ + enum teecore_memtypes mtype = MEM_AREA_TA_RAM; + struct sp_pkg_header *sp_pkg_hdr = NULL; + struct fip_sp *sp = NULL; + uint64_t sp_fdt_end = 0; + size_t sp_pkg_size = 0; + vaddr_t sp_pkg_va = 0; + + /* Process the first page which contains the SP package header */ + sp_pkg_va = (vaddr_t)phys_to_virt(sp_pkg_pa, mtype, SMALL_PAGE_SIZE); + if (!sp_pkg_va) { + EMSG("Cannot find mapping for PA %#" PRIxPA, sp_pkg_pa); + return TEE_ERROR_GENERIC; + } + + sp_pkg_hdr = (struct sp_pkg_header *)sp_pkg_va; + + if (sp_pkg_hdr->magic != SP_PKG_HEADER_MAGIC) { + EMSG("Invalid SP package magic"); + return TEE_ERROR_BAD_FORMAT; + } + + if (sp_pkg_hdr->version != SP_PKG_HEADER_VERSION_V1 && + sp_pkg_hdr->version != SP_PKG_HEADER_VERSION_V2) { + EMSG("Invalid SP header version"); + return TEE_ERROR_BAD_FORMAT; + } + + if (ADD_OVERFLOW(sp_pkg_hdr->img_offset, sp_pkg_hdr->img_size, + &sp_pkg_size)) { + EMSG("Invalid SP package size"); + return TEE_ERROR_BAD_FORMAT; + } + + if (ADD_OVERFLOW(sp_pkg_hdr->pm_offset, sp_pkg_hdr->pm_size, + &sp_fdt_end) || sp_fdt_end > sp_pkg_hdr->img_offset) { + EMSG("Invalid SP manifest size"); + return TEE_ERROR_BAD_FORMAT; + } + + /* Process the whole SP package now that the size is known */ + sp_pkg_va = (vaddr_t)phys_to_virt(sp_pkg_pa, mtype, sp_pkg_size); + if (!sp_pkg_va) { + EMSG("Cannot find mapping for PA %#" PRIxPA, sp_pkg_pa); + return TEE_ERROR_GENERIC; + } + + sp_pkg_hdr = (struct sp_pkg_header *)sp_pkg_va; + + sp = calloc(1, sizeof(struct fip_sp)); + if (!sp) + return TEE_ERROR_OUT_OF_MEMORY; + + memcpy(&sp->sp_img.image.uuid, sp_uuid, sizeof(*sp_uuid)); + sp->sp_img.image.ts = (uint8_t *)(sp_pkg_va + sp_pkg_hdr->img_offset); + sp->sp_img.image.size = sp_pkg_hdr->img_size; + sp->sp_img.image.flags = 0; + sp->sp_img.fdt = (uint8_t *)(sp_pkg_va + sp_pkg_hdr->pm_offset); + + STAILQ_INSERT_TAIL(&fip_sp_list, sp, link); + + return TEE_SUCCESS; +} + +static TEE_Result fip_sp_init_all(void) +{ + TEE_Result res = TEE_SUCCESS; + uint64_t sp_pkg_addr = 0; + const void *fdt = NULL; + TEE_UUID sp_uuid = { }; + int sp_pkgs_node = 0; + int subnode = 0; + int root = 0; + + fdt = get_manifest_dt(); + if (!fdt) { + EMSG("No SPMC manifest found"); + return TEE_ERROR_GENERIC; + } + + root = fdt_path_offset(fdt, "/"); + if (root < 0) + return TEE_ERROR_BAD_FORMAT; + + if (fdt_node_check_compatible(fdt, root, "arm,ffa-core-manifest-1.0")) + return TEE_ERROR_BAD_FORMAT; + + /* SP packages are optional, it's not an error if we don't find any */ + sp_pkgs_node = fdt_node_offset_by_compatible(fdt, root, "arm,sp_pkg"); + if (sp_pkgs_node < 0) + return TEE_SUCCESS; + + fdt_for_each_subnode(subnode, fdt, sp_pkgs_node) { + res = sp_dt_get_u64(fdt, subnode, "load-address", &sp_pkg_addr); + if (res) { + EMSG("Invalid FIP SP load address"); + return res; + } + + res = sp_dt_get_uuid(fdt, subnode, "uuid", &sp_uuid); + if (res) { + EMSG("Invalid FIP SP uuid"); + return res; + } + + res = process_sp_pkg(sp_pkg_addr, &sp_uuid); + if (res) { + EMSG("Invalid FIP SP package"); + return res; + } + } + + return TEE_SUCCESS; +} + +static void fip_sp_deinit_all(void) +{ + while (!STAILQ_EMPTY(&fip_sp_list)) { + struct fip_sp *sp = STAILQ_FIRST(&fip_sp_list); + + STAILQ_REMOVE_HEAD(&fip_sp_list, link); + free(sp); + } +} + +static TEE_Result sp_init_all(void) +{ + TEE_Result res = TEE_SUCCESS; + const struct sp_image *sp = NULL; + const struct fip_sp *fip_sp = NULL; + char __maybe_unused msg[60] = { '\0', }; + struct sp_session *s = NULL; + struct sp_session *prev_sp = NULL; + + for_each_secure_partition(sp) { + if (sp->image.uncompressed_size) + snprintf(msg, sizeof(msg), + " (compressed, uncompressed %u)", + sp->image.uncompressed_size); + else + msg[0] = '\0'; + DMSG("SP %pUl size %u%s", (void *)&sp->image.uuid, + sp->image.size, msg); + + res = sp_init_uuid(&sp->image.uuid, sp->fdt); + + if (res != TEE_SUCCESS) { + EMSG("Failed initializing SP(%pUl) err:%#"PRIx32, + &sp->image.uuid, res); + if (!IS_ENABLED(CFG_SP_SKIP_FAILED)) + panic(); + } + } + + res = fip_sp_init_all(); + if (res) + panic("Failed initializing FIP SPs"); + + for_each_fip_sp(fip_sp) { + sp = &fip_sp->sp_img; + + DMSG("SP %pUl size %u", (void *)&sp->image.uuid, + sp->image.size); + + res = sp_init_uuid(&sp->image.uuid, sp->fdt); + + if (res != TEE_SUCCESS) { + EMSG("Failed initializing SP(%pUl) err:%#"PRIx32, + &sp->image.uuid, res); + if (!IS_ENABLED(CFG_SP_SKIP_FAILED)) + panic(); + } + } + + /* + * At this point all FIP SPs are loaded by ldelf or by the raw binary SP + * loader, so the original images (loaded by BL2) are not needed anymore + */ + fip_sp_deinit_all(); + + /* + * Now that all SPs are loaded, check through the boot order values, + * and warn in case there is a non-unique value. + */ + TAILQ_FOREACH(s, &open_sp_sessions, link) { + /* User specified boot-order values are uint16 */ + if (s->boot_order > UINT16_MAX) + break; + + if (prev_sp && prev_sp->boot_order == s->boot_order) + IMSG("WARNING: duplicated boot-order (%pUl vs %pUl)", + &prev_sp->ts_sess.ctx->uuid, + &s->ts_sess.ctx->uuid); + + prev_sp = s; + } + + /* Continue the initialization and run the SP */ + TAILQ_FOREACH(s, &open_sp_sessions, link) { + DMSG("Starting SP: 0x%"PRIx16, s->endpoint_id); + + res = sp_first_run(s); + if (res != TEE_SUCCESS) { + EMSG("Failed starting SP(0x%"PRIx16") err:%#"PRIx32, + s->endpoint_id, res); + if (!IS_ENABLED(CFG_SP_SKIP_FAILED)) + panic(); + } + } + + return TEE_SUCCESS; +} + +boot_final(sp_init_all); + +static TEE_Result secure_partition_open(const TEE_UUID *uuid, + struct ts_store_handle **h) +{ + return emb_ts_open(uuid, h, find_secure_partition); +} + +REGISTER_SP_STORE(2) = { + .description = "SP store", + .open = secure_partition_open, + .get_size = emb_ts_get_size, + .get_tag = emb_ts_get_tag, + .read = emb_ts_read, + .close = emb_ts_close, +}; diff --git a/optee_os/core/arch/arm/kernel/spin_lock_a32.S b/optee_os/core/arch/arm/kernel/spin_lock_a32.S new file mode 100644 index 0000000..ab9c41b --- /dev/null +++ b/optee_os/core/arch/arm/kernel/spin_lock_a32.S @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of ARM nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +/* void __cpu_spin_lock(unsigned int *lock) */ +FUNC __cpu_spin_lock , : + mov r2, #SPINLOCK_LOCK +1: + ldrex r1, [r0] + cmp r1, #SPINLOCK_UNLOCK + wfene + strexeq r1, r2, [r0] + cmpeq r1, #0 + bne 1b + dmb + bx lr +END_FUNC __cpu_spin_lock + +/* int __cpu_spin_trylock(unsigned int *lock) - return 0 on success */ +FUNC __cpu_spin_trylock , : + mov r2, #SPINLOCK_LOCK + mov r1, r0 +1: + ldrex r0, [r1] + cmp r0, #0 + bne 1f + strex r0, r2, [r1] + cmp r0, #0 + bne 1b + dmb + bx lr +1: + clrex + dmb + bx lr +END_FUNC __cpu_spin_trylock + +/* void __cpu_spin_unlock(unsigned int *lock) */ +FUNC __cpu_spin_unlock , : + dmb + mov r1, #SPINLOCK_UNLOCK + str r1, [r0] + dsb + sev + bx lr +END_FUNC __cpu_spin_unlock diff --git a/optee_os/core/arch/arm/kernel/spin_lock_a64.S b/optee_os/core/arch/arm/kernel/spin_lock_a64.S new file mode 100644 index 0000000..10561fe --- /dev/null +++ b/optee_os/core/arch/arm/kernel/spin_lock_a64.S @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: (BSD-2-Clause AND BSD-3-Clause) */ +/* + * Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of ARM nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +/* void __cpu_spin_lock(unsigned int *lock); */ +FUNC __cpu_spin_lock , : + mov w2, #SPINLOCK_LOCK + sevl +l1: wfe +l2: ldaxr w1, [x0] + cbnz w1, l1 + stxr w1, w2, [x0] + cbnz w1, l2 + ret +END_FUNC __cpu_spin_lock + +/* unsigned int __cpu_spin_trylock(unsigned int *lock); */ +FUNC __cpu_spin_trylock , : + mov x1, x0 + mov w2, #SPINLOCK_LOCK +.loop: ldaxr w0, [x1] + cbnz w0, .cpu_spin_trylock_out + stxr w0, w2, [x1] + cbnz w0, .loop +.cpu_spin_trylock_out: + ret +END_FUNC __cpu_spin_trylock + +/* void __cpu_spin_unlock(unsigned int *lock); */ +FUNC __cpu_spin_unlock , : + stlr wzr, [x0] + ret +END_FUNC __cpu_spin_unlock + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/spmc_sp_handler.c b/optee_os/core/arch/arm/kernel/spmc_sp_handler.c new file mode 100644 index 0000000..87f9772 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/spmc_sp_handler.c @@ -0,0 +1,1287 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2023, Arm Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned int mem_ref_lock = SPINLOCK_UNLOCK; + +void spmc_sp_start_thread(struct thread_smc_args *args) +{ + thread_sp_alloc_and_run(args); +} + +static void ffa_set_error(struct thread_smc_args *args, uint32_t error) +{ + args->a0 = FFA_ERROR; + args->a1 = FFA_PARAM_MBZ; + args->a2 = error; + args->a3 = FFA_PARAM_MBZ; + args->a4 = FFA_PARAM_MBZ; + args->a5 = FFA_PARAM_MBZ; + args->a6 = FFA_PARAM_MBZ; + args->a7 = FFA_PARAM_MBZ; +} + +static void ffa_success(struct thread_smc_args *args) +{ + args->a0 = FFA_SUCCESS_32; +} + +static TEE_Result ffa_get_dst(struct thread_smc_args *args, + struct sp_session *caller, + struct sp_session **dst) +{ + struct sp_session *s = NULL; + + if (args->a2 != FFA_PARAM_MBZ) + return FFA_INVALID_PARAMETERS; + + s = sp_get_session(FFA_DST(args->a1)); + + /* Message came from the NW */ + if (!caller) { + if (!s) { + EMSG("Neither destination nor source is a SP"); + return FFA_INVALID_PARAMETERS; + } + } else { + /* Check if the source matches the endpoint we came from */ + if (FFA_SRC(args->a1) != caller->endpoint_id) { + EMSG("Source address doesn't match the endpoint id"); + return FFA_INVALID_PARAMETERS; + } + } + + *dst = s; + + return FFA_OK; +} + +static struct sp_mem_receiver *find_sp_mem_receiver(struct sp_session *s, + struct sp_mem *smem) +{ + struct sp_mem_receiver *receiver = NULL; + + /* + * FF-A Spec 8.10.2: + * Each Handle identifies a single unique composite memory region + * description that is, there is a 1:1 mapping between the two. + * + * Each memory share has an unique handle. We can only have each SP + * once as a receiver in the memory share. For each receiver of a + * memory share, we have one sp_mem_access_descr object. + * This means that there can only be one SP linked to a specific + * struct sp_mem_access_descr. + */ + SLIST_FOREACH(receiver, &smem->receivers, link) { + if (receiver->perm.endpoint_id == s->endpoint_id) + break; + } + return receiver; +} + +static int add_mem_region_to_sp(struct ffa_mem_access *mem_acc, + struct sp_mem *smem) +{ + struct ffa_mem_access_perm *access_perm = &mem_acc->access_perm; + struct sp_session *s = NULL; + struct sp_mem_receiver *receiver = NULL; + uint8_t perm = READ_ONCE(access_perm->perm); + uint16_t endpoint_id = READ_ONCE(access_perm->endpoint_id); + + s = sp_get_session(endpoint_id); + + /* Only add memory shares of loaded SPs */ + if (!s) + return FFA_DENIED; + + /* Only allow each endpoint once */ + if (find_sp_mem_receiver(s, smem)) + return FFA_DENIED; + + if (perm & ~FFA_MEM_ACC_MASK) + return FFA_DENIED; + + receiver = calloc(1, sizeof(struct sp_mem_receiver)); + if (!receiver) + return FFA_NO_MEMORY; + + receiver->smem = smem; + + receiver->perm.endpoint_id = endpoint_id; + receiver->perm.perm = perm; + receiver->perm.flags = READ_ONCE(access_perm->flags); + + SLIST_INSERT_HEAD(&smem->receivers, receiver, link); + + return FFA_OK; +} + +static void spmc_sp_handle_mem_share(struct thread_smc_args *args, + struct ffa_rxtx *rxtx, + struct sp_session *owner_sp) +{ + struct ffa_mem_transaction_x mem_trans = { }; + uint32_t tot_len = args->a1; + uint32_t frag_len = args->a2; + uint64_t global_handle = 0; + int res = FFA_OK; + + cpu_spin_lock(&rxtx->spinlock); + + /* Descriptor fragments aren't supported yet. */ + if (frag_len != tot_len) + res = FFA_NOT_SUPPORTED; + else if (frag_len > rxtx->size) + res = FFA_INVALID_PARAMETERS; + else + res = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, + frag_len, &mem_trans); + if (!res) + res = spmc_sp_add_share(&mem_trans, rxtx, tot_len, + &global_handle, owner_sp); + if (!res) { + args->a3 = high32_from_64(global_handle); + args->a2 = low32_from_64(global_handle); + args->a1 = FFA_PARAM_MBZ; + args->a0 = FFA_SUCCESS_32; + } else { + ffa_set_error(args, res); + } + + cpu_spin_unlock(&rxtx->spinlock); +} + +static int spmc_sp_add_sp_region(struct sp_mem *smem, + struct ffa_address_range *mem_reg, + struct sp_session *owner_sp, + uint8_t highest_permission) +{ + struct sp_ctx *sp_ctx = NULL; + uint64_t va = READ_ONCE(mem_reg->address); + int res = FFA_OK; + uint64_t region_len = READ_ONCE(mem_reg->page_count) * SMALL_PAGE_SIZE; + struct mobj *mobj = NULL; + + sp_ctx = to_sp_ctx(owner_sp->ts_sess.ctx); + + /* + * The memory region we try to share might not be linked to just one + * mobj. Create a new region for each mobj. + */ + while (region_len) { + size_t len = region_len; + struct sp_mem_map_region *region = NULL; + uint16_t prot = 0; + size_t offs = 0; + + /* + * There is already a mobj for each address that is in the SPs + * address range. + */ + mobj = vm_get_mobj(&sp_ctx->uctx, va, &len, &prot, &offs); + if (!mobj) + return FFA_DENIED; + + /* + * If we share memory from a SP, check if we are not sharing + * with a higher permission than the memory was originally + * mapped. + */ + if ((highest_permission & FFA_MEM_ACC_RW) && + !(prot & TEE_MATTR_UW)) { + res = FFA_DENIED; + goto err; + } + + if ((highest_permission & FFA_MEM_ACC_EXE) && + !(prot & TEE_MATTR_UX)) { + res = FFA_DENIED; + goto err; + } + + region = calloc(1, sizeof(*region)); + region->mobj = mobj; + region->page_offset = offs; + region->page_count = len / SMALL_PAGE_SIZE; + + if (!sp_has_exclusive_access(region, &sp_ctx->uctx)) { + free(region); + res = FFA_DENIED; + goto err; + } + + va += len; + region_len -= len; + SLIST_INSERT_HEAD(&smem->regions, region, link); + } + + return FFA_OK; +err: + mobj_put(mobj); + + return res; +} + +static int spmc_sp_add_nw_region(struct sp_mem *smem, + struct ffa_mem_region *mem_reg) +{ + uint64_t page_count = READ_ONCE(mem_reg->total_page_count); + struct sp_mem_map_region *region = NULL; + struct mobj *m = sp_mem_new_mobj(page_count, TEE_MATTR_MEM_TYPE_CACHED, + false); + unsigned int i = 0; + unsigned int idx = 0; + int res = FFA_OK; + uint64_t address_count = READ_ONCE(mem_reg->address_range_count); + + if (!m) + return FFA_NO_MEMORY; + + for (i = 0; i < address_count; i++) { + struct ffa_address_range *addr_range = NULL; + + addr_range = &mem_reg->address_range_array[i]; + if (sp_mem_add_pages(m, &idx, + READ_ONCE(addr_range->address), + READ_ONCE(addr_range->page_count))) { + res = FFA_DENIED; + goto clean_up; + } + } + + region = calloc(1, sizeof(*region)); + if (!region) { + res = FFA_NO_MEMORY; + goto clean_up; + } + + region->mobj = m; + region->page_count = page_count; + + if (!sp_has_exclusive_access(region, NULL)) { + free(region); + res = FFA_DENIED; + goto clean_up; + } + + SLIST_INSERT_HEAD(&smem->regions, region, link); + return FFA_OK; +clean_up: + mobj_put(m); + return res; +} + +int spmc_sp_add_share(struct ffa_mem_transaction_x *mem_trans, + struct ffa_rxtx *rxtx, size_t blen, + uint64_t *global_handle, struct sp_session *owner_sp) +{ + int res = FFA_INVALID_PARAMETERS; + unsigned int num_mem_accs = 0; + unsigned int i = 0; + struct ffa_mem_access *mem_acc = NULL; + size_t needed_size = 0; + size_t addr_range_offs = 0; + struct ffa_mem_region *mem_reg = NULL; + uint8_t highest_permission = 0; + struct sp_mem *smem = sp_mem_new(); + uint16_t sender_id = mem_trans->sender_id; + + if (!smem) + return FFA_NO_MEMORY; + + if ((owner_sp && owner_sp->endpoint_id != sender_id) || + (!owner_sp && sp_get_session(sender_id))) { + res = FFA_DENIED; + goto cleanup; + } + + num_mem_accs = mem_trans->mem_access_count; + mem_acc = (void *)((vaddr_t)rxtx->rx + mem_trans->mem_access_offs); + + if (!num_mem_accs) { + res = FFA_DENIED; + goto cleanup; + } + + /* Store the ffa_mem_transaction */ + smem->sender_id = sender_id; + smem->mem_reg_attr = mem_trans->mem_reg_attr; + smem->flags = mem_trans->flags; + smem->tag = mem_trans->tag; + + if (MUL_OVERFLOW(num_mem_accs, sizeof(*mem_acc), &needed_size) || + ADD_OVERFLOW(needed_size, mem_trans->mem_access_offs, + &needed_size) || needed_size > blen) { + res = FFA_NO_MEMORY; + goto cleanup; + } + + for (i = 0; i < num_mem_accs; i++) + highest_permission |= READ_ONCE(mem_acc[i].access_perm.perm); + + addr_range_offs = READ_ONCE(mem_acc[0].region_offs); + mem_reg = (void *)((char *)rxtx->rx + addr_range_offs); + + /* Iterate over all the addresses */ + if (owner_sp) { + size_t address_range = READ_ONCE(mem_reg->address_range_count); + + for (i = 0; i < address_range; i++) { + struct ffa_address_range *addr_range = NULL; + + addr_range = &mem_reg->address_range_array[i]; + + if (!core_is_buffer_inside((vaddr_t)addr_range, + sizeof(*addr_range), + (vaddr_t)rxtx->rx, + rxtx->size)) { + res = FFA_NO_MEMORY; + goto cleanup; + } + res = spmc_sp_add_sp_region(smem, addr_range, + owner_sp, + highest_permission); + if (res) + goto cleanup; + } + } else { + res = spmc_sp_add_nw_region(smem, mem_reg); + if (res) + goto cleanup; + } + + /* Add the memory address to the SP */ + for (i = 0; i < num_mem_accs; i++) { + res = add_mem_region_to_sp(&mem_acc[i], smem); + if (res) + goto cleanup; + } + *global_handle = smem->global_handle; + sp_mem_add(smem); + + return FFA_OK; + +cleanup: + sp_mem_remove(smem); + return res; +} + +void spmc_sp_set_to_preempted(struct ts_session *ts_sess) +{ + if (ts_sess && is_sp_ctx(ts_sess->ctx)) { + struct sp_session *sp_sess = to_sp_session(ts_sess); + + assert(sp_sess->state == sp_busy); + sp_sess->state = sp_preempted; + } +} + +int spmc_sp_resume_from_preempted(uint16_t endpoint_id) +{ + struct sp_session *sp_sess = sp_get_session(endpoint_id); + + if (!sp_sess) + return FFA_INVALID_PARAMETERS; + + if (sp_sess->state != sp_preempted) + return FFA_DENIED; + + sp_sess->state = sp_busy; + + return FFA_OK; +} + +static bool check_rxtx(struct ffa_rxtx *rxtx) +{ + return rxtx && rxtx->rx && rxtx->tx && rxtx->size > 0; +} + +static TEE_Result +check_retrieve_request(struct sp_mem_receiver *receiver, uint32_t ffa_vers, + struct ffa_mem_transaction_x *mem_trans, + void *rx, struct sp_mem *smem, int64_t tx_len) +{ + struct ffa_mem_access *retr_access = NULL; + uint8_t share_perm = receiver->perm.perm; + uint32_t retr_perm = 0; + uint32_t retr_flags = mem_trans->flags; + uint64_t retr_tag = mem_trans->tag; + struct sp_mem_map_region *reg = NULL; + + /* + * The request came from the endpoint. It should only have one + * ffa_mem_access element + */ + if (mem_trans->mem_access_count != 1) + return TEE_ERROR_BAD_PARAMETERS; + + retr_access = (void *)((vaddr_t)rx + mem_trans->mem_access_offs); + retr_perm = READ_ONCE(retr_access->access_perm.perm); + + /* Check if tag is correct */ + if (receiver->smem->tag != retr_tag) { + EMSG("Incorrect tag %#"PRIx64" %#"PRIx64, receiver->smem->tag, + retr_tag); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Check permissions and flags */ + if ((retr_perm & FFA_MEM_ACC_RW) && + !(share_perm & FFA_MEM_ACC_RW)) { + DMSG("Incorrect memshare permission set"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if ((retr_perm & FFA_MEM_ACC_EXE) && + !(share_perm & FFA_MEM_ACC_EXE)) { + DMSG("Incorrect memshare permission set"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (retr_flags & FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH) { + DMSG("CLEAR_RELINQUISH is not allowed for FFA_SHARE"); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* + * Check if there is enough space in the tx buffer to send the respons. + */ + if (ffa_vers <= FFA_VERSION_1_0) + tx_len -= sizeof(struct ffa_mem_transaction_1_0); + else + tx_len -= sizeof(struct ffa_mem_transaction_1_1); + tx_len -= sizeof(struct ffa_mem_access) + + sizeof(struct ffa_mem_region); + + if (tx_len < 0) + return FFA_NO_MEMORY; + + SLIST_FOREACH(reg, &smem->regions, link) { + tx_len -= sizeof(struct ffa_address_range); + if (tx_len < 0) + return FFA_NO_MEMORY; + } + + return TEE_SUCCESS; +} + +static void create_retrieve_response(uint32_t ffa_vers, void *dst_buffer, + struct sp_mem_receiver *receiver, + struct sp_mem *smem, struct sp_session *s) +{ + size_t off = 0; + struct ffa_mem_region *dst_region = NULL; + struct ffa_address_range *addr_dst = NULL; + struct sp_mem_map_region *reg = NULL; + struct ffa_mem_access *mem_acc = NULL; + + /* + * we respond with a ffa_mem_retrieve_resp which defines the + * following data in the rx buffer of the sp. + * struct mem_transaction_descr + * struct mem_access_descr (always 1 element) + * struct mem_region_descr + */ + if (ffa_vers <= FFA_VERSION_1_0) { + struct ffa_mem_transaction_1_0 *d_ds = dst_buffer; + + off = sizeof(*d_ds); + mem_acc = d_ds->mem_access_array; + + /* copy the mem_transaction_descr */ + d_ds->sender_id = receiver->smem->sender_id; + d_ds->mem_reg_attr = receiver->smem->mem_reg_attr; + d_ds->flags = receiver->smem->flags; + d_ds->tag = receiver->smem->tag; + d_ds->mem_access_count = 1; + } else { + struct ffa_mem_transaction_1_1 *d_ds = dst_buffer; + + off = sizeof(*d_ds); + mem_acc = (void *)(d_ds + 1); + + d_ds->sender_id = receiver->smem->sender_id; + d_ds->mem_reg_attr = receiver->smem->mem_reg_attr; + d_ds->flags = receiver->smem->flags; + d_ds->tag = receiver->smem->tag; + d_ds->mem_access_size = sizeof(*mem_acc); + d_ds->mem_access_count = 1; + d_ds->mem_access_offs = off; + } + + off += sizeof(struct ffa_mem_access); + dst_region = (struct ffa_mem_region *)(mem_acc + 1); + + /* Copy the mem_accsess_descr */ + mem_acc[0].region_offs = off; + memcpy(&mem_acc[0].access_perm, &receiver->perm, + sizeof(struct ffa_mem_access_perm)); + + /* Copy the mem_region_descr */ + dst_region->address_range_count = 0; + dst_region->total_page_count = 0; + + addr_dst = dst_region->address_range_array; + + SLIST_FOREACH(reg, &smem->regions, link) { + uint32_t offset = reg->page_offset; + struct sp_ctx *ctx = to_sp_ctx(s->ts_sess.ctx); + + addr_dst->address = (uint64_t)sp_mem_get_va(&ctx->uctx, + offset, + reg->mobj); + addr_dst->page_count = reg->page_count; + dst_region->address_range_count++; + + dst_region->total_page_count += addr_dst->page_count; + } +} + +static void ffa_mem_retrieve(struct thread_smc_args *args, + struct sp_session *caller_sp, + struct ffa_rxtx *rxtx) +{ + struct ffa_mem_transaction_x mem_trans = { }; + uint32_t tot_len = args->a1; + uint32_t frag_len = args->a2; + int ret = FFA_OK; + size_t tx_len = 0; + struct ffa_mem_access *mem_acc = NULL; + struct ffa_mem_region *mem_region = NULL; + uint64_t va = 0; + struct sp_mem *smem = NULL; + struct sp_mem_receiver *receiver = NULL; + uint32_t exceptions = 0; + uint32_t address_offset = 0; + size_t needed_size = 0; + + if (!check_rxtx(rxtx) || !rxtx->tx_is_mine) { + ret = FFA_DENIED; + goto err; + } + /* Descriptor fragments aren't supported yet. */ + if (frag_len != tot_len) { + ret = FFA_NOT_SUPPORTED; + goto err; + } + if (frag_len > rxtx->size) { + ret = FFA_INVALID_PARAMETERS; + goto err; + } + + tx_len = rxtx->size; + + ret = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, frag_len, + &mem_trans); + if (ret) + goto err; + + smem = sp_mem_get(mem_trans.global_handle); + if (!smem) { + DMSG("Incorrect handle"); + ret = FFA_DENIED; + goto err; + } + + receiver = sp_mem_get_receiver(caller_sp->endpoint_id, smem); + + mem_acc = (void *)((vaddr_t)rxtx->rx + mem_trans.mem_access_offs); + + address_offset = READ_ONCE(mem_acc[0].region_offs); + + if (ADD_OVERFLOW(address_offset, sizeof(struct ffa_mem_region), + &needed_size) || needed_size > tx_len) { + ret = FFA_INVALID_PARAMETERS; + goto err; + } + + if (check_retrieve_request(receiver, rxtx->ffa_vers, &mem_trans, + rxtx->rx, smem, tx_len) != TEE_SUCCESS) { + ret = FFA_INVALID_PARAMETERS; + goto err; + } + + exceptions = cpu_spin_lock_xsave(&mem_ref_lock); + + if (receiver->ref_count == UINT8_MAX) { + ret = FFA_DENIED; + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + goto err; + } + + receiver->ref_count++; + + /* We only need to map the region the first time we request it. */ + if (receiver->ref_count == 1) { + TEE_Result ret_map = TEE_SUCCESS; + + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + + /* + * Try to map the memory linked to the handle in + * sp_mem_access_descr. + */ + mem_region = (struct ffa_mem_region *)((vaddr_t)rxtx->rx + + address_offset); + + va = READ_ONCE(mem_region->address_range_array[0].address); + ret_map = sp_map_shared(caller_sp, receiver, smem, &va); + + if (ret_map) { + EMSG("Could not map memory region: %#"PRIx32, ret_map); + exceptions = cpu_spin_lock_xsave(&mem_ref_lock); + receiver->ref_count--; + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + ret = FFA_DENIED; + goto err; + } + } else { + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + } + + create_retrieve_response(rxtx->ffa_vers, rxtx->tx, receiver, smem, + caller_sp); + + args->a0 = FFA_MEM_RETRIEVE_RESP; + args->a1 = tx_len; + args->a2 = tx_len; + + rxtx->tx_is_mine = false; + + return; +err: + ffa_set_error(args, ret); +} + +static void ffa_mem_relinquish(struct thread_smc_args *args, + struct sp_session *caller_sp, + struct ffa_rxtx *rxtx) +{ + struct sp_mem *smem = NULL; + struct ffa_mem_relinquish *mem = rxtx->rx; + struct sp_mem_receiver *receiver = NULL; + int err = FFA_NOT_SUPPORTED; + uint32_t exceptions = 0; + + if (!check_rxtx(rxtx)) { + ffa_set_error(args, FFA_DENIED); + return; + } + + exceptions = cpu_spin_lock_xsave(&rxtx->spinlock); + smem = sp_mem_get(READ_ONCE(mem->handle)); + + if (!smem) { + DMSG("Incorrect handle"); + err = FFA_DENIED; + goto err_unlock_rxtwx; + } + + if (READ_ONCE(mem->endpoint_count) != 1) { + DMSG("Incorrect endpoint count"); + err = FFA_INVALID_PARAMETERS; + goto err_unlock_rxtwx; + } + + if (READ_ONCE(mem->endpoint_id_array[0]) != caller_sp->endpoint_id) { + DMSG("Incorrect endpoint id"); + err = FFA_DENIED; + goto err_unlock_rxtwx; + } + + cpu_spin_unlock_xrestore(&rxtx->spinlock, exceptions); + + receiver = sp_mem_get_receiver(caller_sp->endpoint_id, smem); + + exceptions = cpu_spin_lock_xsave(&mem_ref_lock); + if (!receiver->ref_count) { + DMSG("To many relinquish requests"); + err = FFA_DENIED; + goto err_unlock_memref; + } + + receiver->ref_count--; + if (!receiver->ref_count) { + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + if (sp_unmap_ffa_regions(caller_sp, smem) != TEE_SUCCESS) { + DMSG("Failed to unmap region"); + ffa_set_error(args, FFA_DENIED); + return; + } + } else { + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + } + + ffa_success(args); + return; + +err_unlock_rxtwx: + cpu_spin_unlock_xrestore(&rxtx->spinlock, exceptions); + ffa_set_error(args, err); + return; +err_unlock_memref: + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + ffa_set_error(args, err); +} + +static void zero_mem_region(struct sp_mem *smem, struct sp_session *s) +{ + void *addr = NULL; + struct sp_ctx *ctx = to_sp_ctx(s->ts_sess.ctx); + struct sp_mem_map_region *reg = NULL; + + ts_push_current_session(&s->ts_sess); + SLIST_FOREACH(reg, &smem->regions, link) { + size_t sz = reg->page_count * SMALL_PAGE_SIZE; + + addr = sp_mem_get_va(&ctx->uctx, reg->page_offset, reg->mobj); + + assert(addr); + memset(addr, 0, sz); + } + ts_pop_current_session(); +} + +/* + * ffa_mem_reclaim returns false if it couldn't process the reclaim message. + * This happens when the memory regions was shared with the OP-TEE endpoint. + * After this thread_spmc calls handle_mem_reclaim() to make sure that the + * region is reclaimed from the OP-TEE endpoint. + */ +bool ffa_mem_reclaim(struct thread_smc_args *args, + struct sp_session *caller_sp) +{ + uint64_t handle = reg_pair_to_64(args->a2, args->a1); + uint32_t flags = args->a3; + struct sp_mem *smem = NULL; + struct sp_mem_receiver *receiver = NULL; + uint32_t exceptions = 0; + + smem = sp_mem_get(handle); + if (!smem) + return false; + + /* + * If the caller is an SP, make sure that it is the owner of the share. + * If the call comes from NWd this is ensured by the hypervisor. + */ + if (caller_sp && caller_sp->endpoint_id != smem->sender_id) { + ffa_set_error(args, FFA_INVALID_PARAMETERS); + return true; + } + + exceptions = cpu_spin_lock_xsave(&mem_ref_lock); + + /* Make sure that all shares where relinquished */ + SLIST_FOREACH(receiver, &smem->receivers, link) { + if (receiver->ref_count != 0) { + ffa_set_error(args, FFA_DENIED); + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + return true; + } + } + + if (flags & FFA_MEMORY_REGION_FLAG_CLEAR) { + if (caller_sp) { + zero_mem_region(smem, caller_sp); + } else { + /* + * Currently we don't support zeroing Normal World + * memory. To do this we would have to map the memory + * again, zero it and unmap it. + */ + ffa_set_error(args, FFA_DENIED); + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + return true; + } + } + + sp_mem_remove(smem); + cpu_spin_unlock_xrestore(&mem_ref_lock, exceptions); + + ffa_success(args); + return true; +} + +static struct sp_session * +ffa_handle_sp_direct_req(struct thread_smc_args *args, + struct sp_session *caller_sp) +{ + struct sp_session *dst = NULL; + TEE_Result res = FFA_OK; + + if (args->a2 != FFA_PARAM_MBZ) { + ffa_set_error(args, FFA_INVALID_PARAMETERS); + return caller_sp; + } + + res = ffa_get_dst(args, caller_sp, &dst); + if (res) { + /* Tried to send message to an incorrect endpoint */ + ffa_set_error(args, res); + return caller_sp; + } + if (!dst) { + EMSG("Request to normal world not supported"); + ffa_set_error(args, FFA_NOT_SUPPORTED); + return caller_sp; + } + + if (dst == caller_sp) { + EMSG("Cannot send message to own ID"); + ffa_set_error(args, FFA_INVALID_PARAMETERS); + return caller_sp; + } + + cpu_spin_lock(&dst->spinlock); + if (dst->state != sp_idle) { + DMSG("SP is busy"); + ffa_set_error(args, FFA_BUSY); + cpu_spin_unlock(&dst->spinlock); + return caller_sp; + } + + dst->state = sp_busy; + cpu_spin_unlock(&dst->spinlock); + + /* + * Store the calling endpoint id. This will make it possible to check + * if the response is sent back to the correct endpoint. + */ + dst->caller_id = FFA_SRC(args->a1); + + /* Forward the message to the destination SP */ + res = sp_enter(args, dst); + if (res) { + /* The SP Panicked */ + ffa_set_error(args, FFA_ABORTED); + /* Return error to calling SP */ + return caller_sp; + } + + return dst; +} + +static struct sp_session * +ffa_handle_sp_direct_resp(struct thread_smc_args *args, + struct sp_session *caller_sp) +{ + struct sp_session *dst = NULL; + TEE_Result res = FFA_OK; + + if (!caller_sp) { + EMSG("Response from normal world not supported"); + ffa_set_error(args, FFA_NOT_SUPPORTED); + return NULL; + } + + res = ffa_get_dst(args, caller_sp, &dst); + if (res) { + /* Tried to send response to an incorrect endpoint */ + ffa_set_error(args, res); + return caller_sp; + } + + if (dst && dst->state != sp_busy) { + EMSG("SP is not waiting for a request"); + ffa_set_error(args, FFA_INVALID_PARAMETERS); + return caller_sp; + } + + if (caller_sp->caller_id != FFA_DST(args->a1)) { + EMSG("FFA_MSG_SEND_DIRECT_RESP to incorrect SP"); + ffa_set_error(args, FFA_INVALID_PARAMETERS); + return caller_sp; + } + + caller_sp->caller_id = 0; + + cpu_spin_lock(&caller_sp->spinlock); + caller_sp->state = sp_idle; + cpu_spin_unlock(&caller_sp->spinlock); + + if (!dst) { + /* Send message back to the NW */ + return NULL; + } + + /* Forward the message to the destination SP */ + res = sp_enter(args, dst); + if (res) { + /* The SP Panicked */ + ffa_set_error(args, FFA_ABORTED); + /* Return error to calling SP */ + return caller_sp; + } + return dst; +} + +static struct sp_session * +ffa_handle_sp_error(struct thread_smc_args *args, + struct sp_session *caller_sp) +{ + /* If caller_sp == NULL send message to Normal World */ + if (caller_sp && sp_enter(args, caller_sp)) { + /* + * We can not return the error. Unwind the call chain with one + * link. Set the state of the SP to dead. + */ + caller_sp->state = sp_dead; + /* Create error. */ + ffa_set_error(args, FFA_ABORTED); + return sp_get_session(caller_sp->caller_id); + } + + return caller_sp; +} + +static void handle_features(struct thread_smc_args *args) +{ + uint32_t ret_fid = 0; + uint32_t ret_w2 = FFA_PARAM_MBZ; + + switch (args->a1) { +#ifdef ARM64 + case FFA_RXTX_MAP_64: +#endif + case FFA_RXTX_MAP_32: + ret_fid = FFA_SUCCESS_32; + ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ + break; + case FFA_ERROR: + case FFA_VERSION: + case FFA_SUCCESS_32: +#ifdef ARM64 + case FFA_SUCCESS_64: +#endif + default: + ret_fid = FFA_ERROR; + ret_w2 = FFA_NOT_SUPPORTED; + break; + } + + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static void handle_spm_id_get(struct thread_smc_args *args) +{ + spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, SPMC_ENDPOINT_ID, + FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static void handle_mem_perm_get(struct thread_smc_args *args, + struct sp_session *sp_s) +{ + struct sp_ctx *sp_ctx = NULL; + TEE_Result res = TEE_ERROR_BAD_PARAMETERS; + uint16_t attrs = 0; + uint32_t ret_fid = FFA_ERROR; + uint32_t ret_val = FFA_INVALID_PARAMETERS; + + /* + * The FFA_MEM_PERM_GET interface is only allowed during initialization + */ + if (sp_s->is_initialized) { + ret_val = FFA_DENIED; + goto out; + } + + sp_ctx = to_sp_ctx(sp_s->ts_sess.ctx); + if (!sp_ctx) + goto out; + + /* Query memory attributes */ + ts_push_current_session(&sp_s->ts_sess); + res = vm_get_prot(&sp_ctx->uctx, args->a1, SMALL_PAGE_SIZE, &attrs); + ts_pop_current_session(); + if (res) + goto out; + + /* Build response value */ + ret_fid = FFA_SUCCESS_32; + ret_val = 0; + if ((attrs & TEE_MATTR_URW) == TEE_MATTR_URW) + ret_val |= FFA_MEM_PERM_RW; + else if (attrs & TEE_MATTR_UR) + ret_val |= FFA_MEM_PERM_RO; + + if ((attrs & TEE_MATTR_UX) == 0) + ret_val |= FFA_MEM_PERM_NX; + +out: + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_val, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static void handle_mem_perm_set(struct thread_smc_args *args, + struct sp_session *sp_s) +{ + struct sp_ctx *sp_ctx = NULL; + TEE_Result res = TEE_ERROR_BAD_PARAMETERS; + size_t region_size = 0; + uint32_t data_perm = 0; + uint32_t instruction_perm = 0; + uint16_t attrs = 0; + uint32_t ret_fid = FFA_ERROR; + uint32_t ret_val = FFA_INVALID_PARAMETERS; + + /* + * The FFA_MEM_PERM_GET interface is only allowed during initialization + */ + if (sp_s->is_initialized) { + ret_val = FFA_DENIED; + goto out; + } + + sp_ctx = to_sp_ctx(sp_s->ts_sess.ctx); + if (!sp_ctx) + goto out; + + if (MUL_OVERFLOW(args->a2, SMALL_PAGE_SIZE, ®ion_size)) + goto out; + + if (args->a3 & FFA_MEM_PERM_RESERVED) { + /* Non-zero reserved bits */ + goto out; + } + + data_perm = args->a3 & FFA_MEM_PERM_DATA_PERM; + instruction_perm = args->a3 & FFA_MEM_PERM_INSTRUCTION_PERM; + + /* RWX access right configuration is not permitted */ + if (data_perm == FFA_MEM_PERM_RW && instruction_perm == FFA_MEM_PERM_X) + goto out; + + switch (data_perm) { + case FFA_MEM_PERM_RO: + attrs = TEE_MATTR_UR; + break; + case FFA_MEM_PERM_RW: + attrs = TEE_MATTR_URW; + break; + default: + /* Invalid permission value */ + goto out; + } + + if (instruction_perm == FFA_MEM_PERM_X) + attrs |= TEE_MATTR_UX; + + /* Set access rights */ + ts_push_current_session(&sp_s->ts_sess); + res = vm_set_prot(&sp_ctx->uctx, args->a1, region_size, attrs); + ts_pop_current_session(); + if (res != TEE_SUCCESS) + goto out; + + ret_fid = FFA_SUCCESS_32; + ret_val = FFA_PARAM_MBZ; + +out: + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_val, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static void spmc_handle_version(struct thread_smc_args *args, + struct ffa_rxtx *rxtx) +{ + spmc_set_args(args, spmc_exchange_version(args->a1, rxtx), + FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static void handle_console_log(struct thread_smc_args *args) +{ + uint32_t ret_fid = FFA_ERROR; + uint32_t ret_val = FFA_INVALID_PARAMETERS; + size_t char_count = args->a1 & FFA_CONSOLE_LOG_CHAR_COUNT_MASK; + const void *reg_list[] = { + &args->a2, &args->a3, &args->a4, + &args->a5, &args->a6, &args->a7 + }; + char buffer[FFA_CONSOLE_LOG_64_MAX_MSG_LEN + 1] = { 0 }; + size_t max_length = 0; + size_t reg_size = 0; + size_t n = 0; + + if (args->a0 == FFA_CONSOLE_LOG_64) { + max_length = FFA_CONSOLE_LOG_64_MAX_MSG_LEN; + reg_size = sizeof(uint64_t); + } else { + max_length = FFA_CONSOLE_LOG_32_MAX_MSG_LEN; + reg_size = sizeof(uint32_t); + } + + if (char_count < 1 || char_count > max_length) + goto out; + + for (n = 0; n < char_count; n += reg_size) + memcpy(buffer + n, reg_list[n / reg_size], + MIN(char_count - n, reg_size)); + + buffer[char_count] = '\0'; + + trace_ext_puts(buffer); + + ret_fid = FFA_SUCCESS_32; + ret_val = FFA_PARAM_MBZ; + +out: + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_val, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +/* + * FF-A messages handler for SP. Every messages for or from a SP is handled + * here. This is the entry of the sp_spmc kernel thread. The caller_sp is set + * to NULL when it is the Normal World. + */ +void spmc_sp_msg_handler(struct thread_smc_args *args, + struct sp_session *caller_sp) +{ + thread_check_canaries(); + do { + switch (args->a0) { +#ifdef ARM64 + case FFA_MSG_SEND_DIRECT_REQ_64: +#endif + case FFA_MSG_SEND_DIRECT_REQ_32: + caller_sp = ffa_handle_sp_direct_req(args, caller_sp); + break; +#ifdef ARM64 + case FFA_MSG_SEND_DIRECT_RESP_64: +#endif + case FFA_MSG_SEND_DIRECT_RESP_32: + caller_sp = ffa_handle_sp_direct_resp(args, caller_sp); + break; + case FFA_ERROR: + caller_sp = ffa_handle_sp_error(args, caller_sp); + break; + case FFA_MSG_WAIT: + /* FFA_WAIT gives control back to NW */ + cpu_spin_lock(&caller_sp->spinlock); + caller_sp->state = sp_idle; + cpu_spin_unlock(&caller_sp->spinlock); + caller_sp = NULL; + break; +#ifdef ARM64 + case FFA_RXTX_MAP_64: +#endif + case FFA_RXTX_MAP_32: + ts_push_current_session(&caller_sp->ts_sess); + spmc_handle_rxtx_map(args, &caller_sp->rxtx); + ts_pop_current_session(); + sp_enter(args, caller_sp); + break; + case FFA_RXTX_UNMAP: + ts_push_current_session(&caller_sp->ts_sess); + spmc_handle_rxtx_unmap(args, &caller_sp->rxtx); + ts_pop_current_session(); + sp_enter(args, caller_sp); + break; + case FFA_RX_RELEASE: + ts_push_current_session(&caller_sp->ts_sess); + spmc_handle_rx_release(args, &caller_sp->rxtx); + ts_pop_current_session(); + sp_enter(args, caller_sp); + break; + case FFA_ID_GET: + args->a0 = FFA_SUCCESS_32; + args->a2 = caller_sp->endpoint_id; + sp_enter(args, caller_sp); + break; + case FFA_VERSION: + spmc_handle_version(args, &caller_sp->rxtx); + sp_enter(args, caller_sp); + break; + case FFA_FEATURES: + handle_features(args); + sp_enter(args, caller_sp); + break; + case FFA_SPM_ID_GET: + handle_spm_id_get(args); + sp_enter(args, caller_sp); + break; + case FFA_PARTITION_INFO_GET: + ts_push_current_session(&caller_sp->ts_sess); + spmc_handle_partition_info_get(args, &caller_sp->rxtx); + ts_pop_current_session(); + sp_enter(args, caller_sp); + break; +#ifdef ARM64 + case FFA_MEM_SHARE_64: +#endif + case FFA_MEM_SHARE_32: + ts_push_current_session(&caller_sp->ts_sess); + spmc_sp_handle_mem_share(args, &caller_sp->rxtx, + caller_sp); + ts_pop_current_session(); + sp_enter(args, caller_sp); + break; +#ifdef ARM64 + case FFA_MEM_RETRIEVE_REQ_64: +#endif + case FFA_MEM_RETRIEVE_REQ_32: + ts_push_current_session(&caller_sp->ts_sess); + ffa_mem_retrieve(args, caller_sp, &caller_sp->rxtx); + ts_pop_current_session(); + sp_enter(args, caller_sp); + break; + case FFA_MEM_RELINQUISH: + ts_push_current_session(&caller_sp->ts_sess); + ffa_mem_relinquish(args, caller_sp, &caller_sp->rxtx); + ts_pop_current_session(); + sp_enter(args, caller_sp); + break; + case FFA_MEM_RECLAIM: + ffa_mem_reclaim(args, caller_sp); + sp_enter(args, caller_sp); + break; +#ifdef ARM64 + case FFA_MEM_PERM_GET_64: +#endif + case FFA_MEM_PERM_GET_32: + handle_mem_perm_get(args, caller_sp); + sp_enter(args, caller_sp); + break; + +#ifdef ARM64 + case FFA_MEM_PERM_SET_64: +#endif + case FFA_MEM_PERM_SET_32: + handle_mem_perm_set(args, caller_sp); + sp_enter(args, caller_sp); + break; + +#ifdef ARM64 + case FFA_CONSOLE_LOG_64: +#endif + case FFA_CONSOLE_LOG_32: + handle_console_log(args); + sp_enter(args, caller_sp); + break; + + default: + EMSG("Unhandled FFA function ID %#"PRIx32, + (uint32_t)args->a0); + ffa_set_error(args, FFA_INVALID_PARAMETERS); + sp_enter(args, caller_sp); + } + } while (caller_sp); +} diff --git a/optee_os/core/arch/arm/kernel/stmm_sp.c b/optee_os/core/arch/arm/kernel/stmm_sp.c new file mode 100644 index 0000000..691bae8 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/stmm_sp.c @@ -0,0 +1,880 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef ARM64 +#define SVC_REGS_A0(_regs) ((_regs)->x0) +#define SVC_REGS_A1(_regs) ((_regs)->x1) +#define SVC_REGS_A2(_regs) ((_regs)->x2) +#define SVC_REGS_A3(_regs) ((_regs)->x3) +#define SVC_REGS_A4(_regs) ((_regs)->x4) +#define SVC_REGS_A5(_regs) ((_regs)->x5) +#define SVC_REGS_A6(_regs) ((_regs)->x6) +#define SVC_REGS_A7(_regs) ((_regs)->x7) +#define __FFA_SVC_RPMB_READ FFA_SVC_RPMB_READ +#define __FFA_SVC_RPMB_WRITE FFA_SVC_RPMB_WRITE +#define __FFA_SVC_MEMORY_ATTRIBUTES_GET FFA_SVC_MEMORY_ATTRIBUTES_GET_64 +#define __FFA_SVC_MEMORY_ATTRIBUTES_SET FFA_SVC_MEMORY_ATTRIBUTES_SET_64 +#define __FFA_MSG_SEND_DIRECT_RESP FFA_MSG_SEND_DIRECT_RESP_64 +#define __FFA_MSG_SEND_DIRECT_REQ FFA_MSG_SEND_DIRECT_REQ_64 +#endif +#ifdef ARM32 +#define SVC_REGS_A0(_regs) ((_regs)->r0) +#define SVC_REGS_A1(_regs) ((_regs)->r1) +#define SVC_REGS_A2(_regs) ((_regs)->r2) +#define SVC_REGS_A3(_regs) ((_regs)->r3) +#define SVC_REGS_A4(_regs) ((_regs)->r4) +#define SVC_REGS_A5(_regs) ((_regs)->r5) +#define SVC_REGS_A6(_regs) ((_regs)->r6) +#define SVC_REGS_A7(_regs) ((_regs)->r7) +#define __FFA_SVC_RPMB_READ FFA_SVC_RPMB_READ_32 +#define __FFA_SVC_RPMB_WRITE FFA_SVC_RPMB_WRITE_32 +#define __FFA_SVC_MEMORY_ATTRIBUTES_GET FFA_SVC_MEMORY_ATTRIBUTES_GET_32 +#define __FFA_SVC_MEMORY_ATTRIBUTES_SET FFA_SVC_MEMORY_ATTRIBUTES_SET_32 +#define __FFA_MSG_SEND_DIRECT_RESP FFA_MSG_SEND_DIRECT_RESP_32 +#define __FFA_MSG_SEND_DIRECT_REQ FFA_MSG_SEND_DIRECT_REQ_32 +#endif + +static const TEE_UUID stmm_uuid = PTA_STMM_UUID; + +/* + * Once a complete FFA spec is added, these will become discoverable. + * Until then these are considered part of the internal ABI between + * OP-TEE and StMM. + */ +static const uint16_t stmm_id = 1U; +static const uint16_t stmm_pta_id = 2U; +static const uint16_t mem_mgr_id = 3U; +static const uint16_t ffa_storage_id = 4U; + +static const unsigned int stmm_stack_size = 4 * SMALL_PAGE_SIZE; +static const unsigned int stmm_heap_size = 398 * SMALL_PAGE_SIZE; +static const unsigned int stmm_sec_buf_size = 4 * SMALL_PAGE_SIZE; +static const unsigned int stmm_ns_comm_buf_size = 4 * SMALL_PAGE_SIZE; + +extern unsigned char stmm_image[]; +extern const unsigned int stmm_image_size; +extern const unsigned int stmm_image_uncompressed_size; + +const TEE_UUID *stmm_get_uuid(void) +{ + return &stmm_uuid; +} + +static struct stmm_ctx *stmm_alloc_ctx(const TEE_UUID *uuid) +{ + TEE_Result res = TEE_SUCCESS; + struct stmm_ctx *spc = NULL; + + spc = calloc(1, sizeof(*spc)); + if (!spc) + return NULL; + + spc->ta_ctx.ts_ctx.ops = &stmm_sp_ops; + spc->ta_ctx.ts_ctx.uuid = *uuid; + spc->ta_ctx.flags = TA_FLAG_SINGLE_INSTANCE | + TA_FLAG_INSTANCE_KEEP_ALIVE; + + res = vm_info_init(&spc->uctx, &spc->ta_ctx.ts_ctx); + if (res) { + free(spc); + return NULL; + } + + spc->ta_ctx.ref_count = 1; + condvar_init(&spc->ta_ctx.busy_cv); + + return spc; +} + +static TEE_Result stmm_enter_user_mode(struct stmm_ctx *spc) +{ + uint32_t exceptions = 0; + uint32_t panic_code = 0; + uint32_t panicked = 0; + uint64_t cntkctl = 0; + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + cntkctl = read_cntkctl(); + write_cntkctl(cntkctl | CNTKCTL_PL0PCTEN); + +#ifdef ARM32 + /* Handle usr_lr in place of __thread_enter_user_mode() */ + thread_set_usr_lr(spc->regs.usr_lr); +#endif + + __thread_enter_user_mode(&spc->regs, &panicked, &panic_code); + +#ifdef ARM32 + spc->regs.usr_lr = thread_get_usr_lr(); +#endif + + write_cntkctl(cntkctl); + thread_unmask_exceptions(exceptions); + + thread_user_clear_vfp(&spc->uctx); + + if (panicked) { + abort_print_current_ts(); + DMSG("stmm panicked with code %#"PRIx32, panic_code); + return TEE_ERROR_TARGET_DEAD; + } + + return TEE_SUCCESS; +} + +#ifdef ARM64 +static void init_stmm_regs(struct stmm_ctx *spc, unsigned long a0, + unsigned long a1, unsigned long sp, unsigned long pc) +{ + spc->regs.x[0] = a0; + spc->regs.x[1] = a1; + spc->regs.sp = sp; + spc->regs.pc = pc; +} +#endif + +#ifdef ARM32 +static uint32_t __maybe_unused get_spsr(void) +{ + uint32_t s = 0; + + s = read_cpsr(); + s &= ~(CPSR_MODE_MASK | CPSR_T | ARM32_CPSR_IT_MASK); + s |= CPSR_MODE_USR; + + return s; +} + +static void init_stmm_regs(struct stmm_ctx *spc, unsigned long a0, + unsigned long a1, unsigned long sp, unsigned long pc) +{ + spc->regs.r0 = a0; + spc->regs.r1 = a1; + spc->regs.usr_sp = sp; + spc->regs.cpsr = get_spsr(); + spc->regs.pc = pc; +} +#endif + +static TEE_Result alloc_and_map_sp_fobj(struct stmm_ctx *spc, size_t sz, + uint32_t prot, vaddr_t *va) +{ + size_t num_pgs = ROUNDUP(sz, SMALL_PAGE_SIZE) / SMALL_PAGE_SIZE; + struct fobj *fobj = fobj_ta_mem_alloc(num_pgs); + TEE_Result res = TEE_SUCCESS; + struct mobj *mobj = NULL; + + mobj = mobj_with_fobj_alloc(fobj, NULL, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(fobj); + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + + res = vm_map(&spc->uctx, va, num_pgs * SMALL_PAGE_SIZE, + prot, 0, mobj, 0); + if (res) + mobj_put(mobj); + + return TEE_SUCCESS; +} + +static void *zalloc(void *opaque __unused, unsigned int items, + unsigned int size) +{ + return mempool_alloc(mempool_default, items * size); +} + +static void zfree(void *opaque __unused, void *address) +{ + mempool_free(mempool_default, address); +} + +static void uncompress_image(void *dst, size_t dst_size, void *src, + size_t src_size) +{ + z_stream strm = { + .next_in = src, + .avail_in = src_size, + .next_out = dst, + .avail_out = dst_size, + .zalloc = zalloc, + .zfree = zfree, + }; + + if (inflateInit(&strm) != Z_OK) + panic("inflateInit"); + + if (inflate(&strm, Z_SYNC_FLUSH) != Z_STREAM_END) + panic("inflate"); + + if (inflateEnd(&strm) != Z_OK) + panic("inflateEnd"); +} + +static TEE_Result load_stmm(struct stmm_ctx *spc) +{ + struct stmm_boot_info *boot_info = NULL; + struct stmm_mp_info *mp_info = NULL; + TEE_Result res = TEE_SUCCESS; + vaddr_t sp_addr = 0; + vaddr_t image_addr = 0; + vaddr_t heap_addr = 0; + vaddr_t stack_addr = 0; + vaddr_t sec_buf_addr = 0; + vaddr_t comm_buf_addr = 0; + unsigned int sp_size = 0; + unsigned int uncompressed_size_roundup = 0; + + uncompressed_size_roundup = ROUNDUP(stmm_image_uncompressed_size, + SMALL_PAGE_SIZE); + sp_size = uncompressed_size_roundup + stmm_stack_size + + stmm_heap_size + stmm_sec_buf_size; + res = alloc_and_map_sp_fobj(spc, sp_size, + TEE_MATTR_PRW, &sp_addr); + if (res) + return res; + + res = alloc_and_map_sp_fobj(spc, stmm_ns_comm_buf_size, + TEE_MATTR_URW | TEE_MATTR_PRW, + &comm_buf_addr); + /* + * We don't need to free the previous instance here, they'll all be + * handled during the destruction call (stmm_ctx_destroy()) + */ + if (res) + return res; + + image_addr = sp_addr; + heap_addr = image_addr + uncompressed_size_roundup; + stack_addr = heap_addr + stmm_heap_size; + sec_buf_addr = stack_addr + stmm_stack_size; + + vm_set_ctx(&spc->ta_ctx.ts_ctx); + uncompress_image((void *)image_addr, stmm_image_uncompressed_size, + stmm_image, stmm_image_size); + + res = vm_set_prot(&spc->uctx, image_addr, uncompressed_size_roundup, + TEE_MATTR_URX | TEE_MATTR_PR); + if (res) + return res; + + res = vm_set_prot(&spc->uctx, heap_addr, stmm_heap_size, + TEE_MATTR_URW | TEE_MATTR_PRW); + if (res) + return res; + + res = vm_set_prot(&spc->uctx, stack_addr, stmm_stack_size, + TEE_MATTR_URW | TEE_MATTR_PRW); + if (res) + return res; + + res = vm_set_prot(&spc->uctx, sec_buf_addr, stmm_sec_buf_size, + TEE_MATTR_URW | TEE_MATTR_PRW); + if (res) + return res; + + DMSG("stmm load address %#"PRIxVA, image_addr); + + boot_info = (struct stmm_boot_info *)sec_buf_addr; + mp_info = (struct stmm_mp_info *)(boot_info + 1); + *boot_info = (struct stmm_boot_info){ + .h.type = STMM_PARAM_SP_IMAGE_BOOT_INFO, + .h.version = STMM_PARAM_VERSION_1, + .h.size = sizeof(struct stmm_boot_info), + .h.attr = 0, + .sp_mem_base = sp_addr, + .sp_mem_limit = sp_addr + sp_size, + .sp_image_base = image_addr, + .sp_stack_base = stack_addr, + .sp_heap_base = heap_addr, + .sp_ns_comm_buf_base = comm_buf_addr, + .sp_shared_buf_base = sec_buf_addr, + .sp_image_size = stmm_image_size, + .sp_pcpu_stack_size = stmm_stack_size, + .sp_heap_size = stmm_heap_size, + .sp_ns_comm_buf_size = stmm_ns_comm_buf_size, + .sp_shared_buf_size = stmm_sec_buf_size, + .num_sp_mem_regions = 6, + .num_cpus = 1, + .mp_info = mp_info, + }; + mp_info->mpidr = read_mpidr(); + mp_info->linear_id = 0; + mp_info->flags = MP_INFO_FLAG_PRIMARY_CPU; + spc->ns_comm_buf_addr = comm_buf_addr; + spc->ns_comm_buf_size = stmm_ns_comm_buf_size; + + init_stmm_regs(spc, sec_buf_addr, + (vaddr_t)(mp_info + 1) - sec_buf_addr, + stack_addr + stmm_stack_size, image_addr); + + return stmm_enter_user_mode(spc); +} + +TEE_Result stmm_init_session(const TEE_UUID *uuid, struct tee_ta_session *sess) +{ + struct stmm_ctx *spc = NULL; + TEE_Result res = TEE_SUCCESS; + + if (memcmp(uuid, &stmm_uuid, sizeof(*uuid))) + return TEE_ERROR_ITEM_NOT_FOUND; + + spc = stmm_alloc_ctx(uuid); + if (!spc) + return TEE_ERROR_OUT_OF_MEMORY; + + spc->is_initializing = true; + + mutex_lock(&tee_ta_mutex); + sess->ts_sess.ctx = &spc->ta_ctx.ts_ctx; + sess->ts_sess.handle_scall = sess->ts_sess.ctx->ops->handle_scall; + mutex_unlock(&tee_ta_mutex); + + ts_push_current_session(&sess->ts_sess); + res = load_stmm(spc); + ts_pop_current_session(); + vm_set_ctx(NULL); + if (res) { + sess->ts_sess.ctx = NULL; + spc->ta_ctx.ts_ctx.ops->destroy(&spc->ta_ctx.ts_ctx); + + return res; + } + + mutex_lock(&tee_ta_mutex); + spc->is_initializing = false; + TAILQ_INSERT_TAIL(&tee_ctxes, &spc->ta_ctx, link); + mutex_unlock(&tee_ta_mutex); + + return TEE_SUCCESS; +} + +static TEE_Result stmm_enter_open_session(struct ts_session *s) +{ + struct stmm_ctx *spc = to_stmm_ctx(s->ctx); + struct tee_ta_session *ta_sess = to_ta_session(s); + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (ta_sess->param->types != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + if (spc->is_initializing) { + /* StMM is initialized in stmm_init_session() */ + ta_sess->err_origin = TEE_ORIGIN_TEE; + return TEE_ERROR_BAD_STATE; + } + + return TEE_SUCCESS; +} + +static TEE_Result stmm_enter_invoke_cmd(struct ts_session *s, uint32_t cmd) +{ + struct stmm_ctx *spc = to_stmm_ctx(s->ctx); + struct tee_ta_session *ta_sess = to_ta_session(s); + TEE_Result res = TEE_SUCCESS; + TEE_Result __maybe_unused tmp_res = TEE_SUCCESS; + unsigned int ns_buf_size = 0; + struct param_mem *mem = NULL; + void *va = NULL; + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (cmd != PTA_STMM_CMD_COMMUNICATE) + return TEE_ERROR_BAD_PARAMETERS; + + if (ta_sess->param->types != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + mem = &ta_sess->param->u[0].mem; + ns_buf_size = mem->size; + if (ns_buf_size > spc->ns_comm_buf_size) { + mem->size = spc->ns_comm_buf_size; + return TEE_ERROR_EXCESS_DATA; + } + + res = mobj_inc_map(mem->mobj); + if (res) + return res; + + va = mobj_get_va(mem->mobj, mem->offs, mem->size); + if (!va) { + EMSG("Can't get a valid VA for NS buffer"); + res = TEE_ERROR_BAD_PARAMETERS; + goto out_va; + } + +#ifdef ARM64 + spc->regs.x[0] = __FFA_MSG_SEND_DIRECT_REQ; + spc->regs.x[1] = (stmm_pta_id << 16) | stmm_id; + spc->regs.x[2] = FFA_PARAM_MBZ; + spc->regs.x[3] = spc->ns_comm_buf_addr; + spc->regs.x[4] = ns_buf_size; + spc->regs.x[5] = 0; + spc->regs.x[6] = 0; + spc->regs.x[7] = 0; +#endif +#ifdef ARM32 + spc->regs.r0 = __FFA_MSG_SEND_DIRECT_REQ; + spc->regs.r1 = (stmm_pta_id << 16) | stmm_id; + spc->regs.r2 = FFA_PARAM_MBZ; + spc->regs.r3 = spc->ns_comm_buf_addr; + spc->regs.r4 = ns_buf_size; + spc->regs.r5 = 0; + spc->regs.r6 = 0; + spc->regs.r7 = 0; +#endif + + ts_push_current_session(s); + + memcpy((void *)spc->ns_comm_buf_addr, va, ns_buf_size); + + res = stmm_enter_user_mode(spc); + if (res) + goto out_session; + /* + * Copy the SPM response from secure partition back to the non-secure + * buffer of the client that called us. + */ +#ifdef ARM64 + ta_sess->param->u[1].val.a = spc->regs.x[4]; +#endif +#ifdef ARM32 + ta_sess->param->u[1].val.a = spc->regs.r4; +#endif + + memcpy(va, (void *)spc->ns_comm_buf_addr, ns_buf_size); + +out_session: + ts_pop_current_session(); +out_va: + tmp_res = mobj_dec_map(mem->mobj); + assert(!tmp_res); + + return res; +} + +static void stmm_enter_close_session(struct ts_session *s __unused) +{ +} + +static void stmm_dump_state(struct ts_ctx *ctx) +{ + user_mode_ctx_print_mappings(to_user_mode_ctx(ctx)); +} +DECLARE_KEEP_PAGER(stmm_dump_state); + +static uint32_t stmm_get_instance_id(struct ts_ctx *ctx) +{ + return to_stmm_ctx(ctx)->uctx.vm_info.asid; +} + +static void stmm_ctx_destroy(struct ts_ctx *ctx) +{ + struct stmm_ctx *spc = to_stmm_ctx(ctx); + + vm_info_final(&spc->uctx); + free(spc); +} + +static uint32_t sp_svc_get_mem_attr(vaddr_t va) +{ + TEE_Result res = TEE_ERROR_BAD_PARAMETERS; + struct ts_session *sess = NULL; + struct stmm_ctx *spc = NULL; + uint16_t attrs = 0; + uint16_t perm = 0; + + if (!va) + goto err; + + sess = ts_get_current_session(); + spc = to_stmm_ctx(sess->ctx); + + res = vm_get_prot(&spc->uctx, va, SMALL_PAGE_SIZE, &attrs); + if (res) + goto err; + + if (attrs & TEE_MATTR_UR) + perm |= STMM_MEM_ATTR_ACCESS_RO; + else if (attrs & TEE_MATTR_UW) + perm |= STMM_MEM_ATTR_ACCESS_RW; + + if (attrs & TEE_MATTR_UX) + perm |= STMM_MEM_ATTR_EXEC; + + return perm; +err: + return STMM_RET_DENIED; +} + +static int sp_svc_set_mem_attr(vaddr_t va, unsigned int nr_pages, uint32_t perm) +{ + TEE_Result res = TEE_ERROR_BAD_PARAMETERS; + struct ts_session *sess = NULL; + struct stmm_ctx *spc = NULL; + size_t sz = 0; + uint32_t prot = 0; + + if (!va || !nr_pages || MUL_OVERFLOW(nr_pages, SMALL_PAGE_SIZE, &sz)) + return STMM_RET_INVALID_PARAM; + + if (perm & ~STMM_MEM_ATTR_ALL) + return STMM_RET_INVALID_PARAM; + + sess = ts_get_current_session(); + spc = to_stmm_ctx(sess->ctx); + + if ((perm & STMM_MEM_ATTR_ACCESS_MASK) == STMM_MEM_ATTR_ACCESS_RO) + prot |= TEE_MATTR_UR; + else if ((perm & STMM_MEM_ATTR_ACCESS_MASK) == STMM_MEM_ATTR_ACCESS_RW) + prot |= TEE_MATTR_URW; + + if ((perm & STMM_MEM_ATTR_EXEC_NEVER) == STMM_MEM_ATTR_EXEC) + prot |= TEE_MATTR_UX; + + res = vm_set_prot(&spc->uctx, va, sz, prot); + if (res) + return STMM_RET_DENIED; + + return STMM_RET_SUCCESS; +} + +#ifdef ARM64 +static void save_sp_ctx(struct stmm_ctx *spc, + struct thread_scall_regs *regs) +{ + size_t n = 0; + + /* Save the return values from StMM */ + for (n = 0; n <= 7; n++) + spc->regs.x[n] = *(®s->x0 + n); + + spc->regs.sp = regs->sp_el0; + spc->regs.pc = regs->elr; + spc->regs.cpsr = regs->spsr; +} +#endif + +#ifdef ARM32 +static void save_sp_ctx(struct stmm_ctx *spc, + struct thread_scall_regs *regs) +{ + spc->regs.r0 = regs->r0; + spc->regs.r1 = regs->r1; + spc->regs.r2 = regs->r2; + spc->regs.r3 = regs->r3; + spc->regs.r4 = regs->r4; + spc->regs.r5 = regs->r5; + spc->regs.r6 = regs->r6; + spc->regs.r7 = regs->r7; + spc->regs.pc = regs->lr; + spc->regs.cpsr = regs->spsr; + spc->regs.usr_sp = thread_get_usr_sp(); +} +#endif + +static void return_from_sp_helper(bool panic, uint32_t panic_code, + struct thread_scall_regs *regs) +{ + struct ts_session *sess = ts_get_current_session(); + struct stmm_ctx *spc = to_stmm_ctx(sess->ctx); + + if (panic) + spc->ta_ctx.panicked = true; + else + save_sp_ctx(spc, regs); + + SVC_REGS_A0(regs) = 0; + SVC_REGS_A1(regs) = panic; + SVC_REGS_A2(regs) = panic_code; +} + +static void service_compose_direct_resp(struct thread_scall_regs *regs, + uint32_t ret_val) +{ + uint16_t src_id = 0; + uint16_t dst_id = 0; + + /* extract from request */ + src_id = (SVC_REGS_A1(regs) >> 16) & UINT16_MAX; + dst_id = SVC_REGS_A1(regs) & UINT16_MAX; + + /* compose message */ + SVC_REGS_A0(regs) = __FFA_MSG_SEND_DIRECT_RESP; + /* swap endpoint ids */ + SVC_REGS_A1(regs) = SHIFT_U32(dst_id, 16) | src_id; + SVC_REGS_A2(regs) = FFA_PARAM_MBZ; + SVC_REGS_A3(regs) = ret_val; + SVC_REGS_A4(regs) = 0; + SVC_REGS_A5(regs) = 0; + SVC_REGS_A6(regs) = 0; + SVC_REGS_A7(regs) = 0; +} + +/* + * Combined read from secure partition, this will open, read and + * close the file object. + */ +static TEE_Result sec_storage_obj_read(unsigned long storage_id, char *obj_id, + unsigned long obj_id_len, void *data, + unsigned long len, unsigned long offset, + unsigned long flags) +{ + const struct tee_file_operations *fops = NULL; + TEE_Result res = TEE_ERROR_BAD_STATE; + struct ts_session *sess = NULL; + struct tee_file_handle *fh = NULL; + struct tee_pobj *po = NULL; + size_t file_size = 0; + size_t read_len = 0; + + fops = tee_svc_storage_file_ops(storage_id); + if (!fops) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (obj_id_len > TEE_OBJECT_ID_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + sess = ts_get_current_session(); + + res = tee_pobj_get(&sess->ctx->uuid, obj_id, obj_id_len, flags, + false, fops, &po); + if (res != TEE_SUCCESS) + return res; + + res = po->fops->open(po, &file_size, &fh); + if (res != TEE_SUCCESS) + goto out; + + read_len = len; + res = po->fops->read(fh, offset, NULL, data, &read_len); + if (res == TEE_ERROR_CORRUPT_OBJECT) { + EMSG("Object corrupt"); + po->fops->remove(po); + } else if (res == TEE_SUCCESS && len != read_len) { + res = TEE_ERROR_CORRUPT_OBJECT; + } + + po->fops->close(&fh); + +out: + tee_pobj_release(po); + + return res; +} + +/* + * Combined write from secure partition, this will create/open, write and + * close the file object. + */ +static TEE_Result sec_storage_obj_write(unsigned long storage_id, char *obj_id, + unsigned long obj_id_len, void *data, + unsigned long len, unsigned long offset, + unsigned long flags) + +{ + const struct tee_file_operations *fops = NULL; + struct ts_session *sess = NULL; + struct tee_file_handle *fh = NULL; + TEE_Result res = TEE_SUCCESS; + struct tee_pobj *po = NULL; + + fops = tee_svc_storage_file_ops(storage_id); + if (!fops) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (obj_id_len > TEE_OBJECT_ID_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + sess = ts_get_current_session(); + + res = tee_pobj_get(&sess->ctx->uuid, obj_id, obj_id_len, flags, + false, fops, &po); + if (res != TEE_SUCCESS) + return res; + + res = po->fops->open(po, NULL, &fh); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = po->fops->create(po, false, NULL, 0, NULL, 0, + NULL, NULL, 0, &fh); + if (res == TEE_SUCCESS) { + res = po->fops->write(fh, offset, NULL, data, len); + po->fops->close(&fh); + } + + tee_pobj_release(po); + + return res; +} + +static void stmm_handle_mem_mgr_service(struct thread_scall_regs *regs) +{ + uint32_t action = SVC_REGS_A3(regs); + uintptr_t va = SVC_REGS_A4(regs); + uint32_t nr_pages = SVC_REGS_A5(regs); + uint32_t perm = SVC_REGS_A6(regs); + + switch (action) { + case __FFA_SVC_MEMORY_ATTRIBUTES_GET: + service_compose_direct_resp(regs, sp_svc_get_mem_attr(va)); + break; + case __FFA_SVC_MEMORY_ATTRIBUTES_SET: + service_compose_direct_resp(regs, + sp_svc_set_mem_attr(va, nr_pages, + perm)); + break; + default: + EMSG("Undefined service id %#"PRIx32, action); + service_compose_direct_resp(regs, STMM_RET_INVALID_PARAM); + break; + } +} + +static uint32_t tee2stmm_ret_val(TEE_Result res) +{ + switch (res) { + case TEE_SUCCESS: + return STMM_RET_SUCCESS; + case TEE_ERROR_NOT_SUPPORTED: + return STMM_RET_NOT_SUPPORTED; + case TEE_ERROR_ACCESS_DENIED: + return STMM_RET_DENIED; + case TEE_ERROR_OUT_OF_MEMORY: + return STMM_RET_NO_MEM; + case TEE_ERROR_BAD_PARAMETERS: + default: + return STMM_RET_INVALID_PARAM; + } +} + +#define FILENAME "EFI_VARS" +static void stmm_handle_storage_service(struct thread_scall_regs *regs) +{ + uint32_t flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE | + TEE_DATA_FLAG_SHARE_READ | + TEE_DATA_FLAG_SHARE_WRITE; + uint32_t action = SVC_REGS_A3(regs); + void *va = (void *)SVC_REGS_A4(regs); + unsigned long len = SVC_REGS_A5(regs); + unsigned long offset = SVC_REGS_A6(regs); + char obj_id[] = FILENAME; + size_t obj_id_len = strlen(obj_id); + TEE_Result res = TEE_SUCCESS; + uint32_t stmm_rc = STMM_RET_INVALID_PARAM; + + switch (action) { + case __FFA_SVC_RPMB_READ: + DMSG("RPMB read"); + res = sec_storage_obj_read(TEE_STORAGE_PRIVATE_RPMB, obj_id, + obj_id_len, va, len, offset, flags); + stmm_rc = tee2stmm_ret_val(res); + break; + case __FFA_SVC_RPMB_WRITE: + DMSG("RPMB write"); + res = sec_storage_obj_write(TEE_STORAGE_PRIVATE_RPMB, obj_id, + obj_id_len, va, len, offset, flags); + stmm_rc = tee2stmm_ret_val(res); + break; + default: + EMSG("Undefined service id %#"PRIx32, action); + break; + } + + service_compose_direct_resp(regs, stmm_rc); +} + +static void spm_eret_error(int32_t error_code, struct thread_scall_regs *regs) +{ + SVC_REGS_A0(regs) = FFA_ERROR; + SVC_REGS_A1(regs) = FFA_PARAM_MBZ; + SVC_REGS_A2(regs) = error_code; + SVC_REGS_A3(regs) = FFA_PARAM_MBZ; + SVC_REGS_A4(regs) = FFA_PARAM_MBZ; + SVC_REGS_A5(regs) = FFA_PARAM_MBZ; + SVC_REGS_A6(regs) = FFA_PARAM_MBZ; + SVC_REGS_A7(regs) = FFA_PARAM_MBZ; +} + +static void spm_handle_direct_req(struct thread_scall_regs *regs) +{ + uint16_t dst_id = SVC_REGS_A1(regs) & UINT16_MAX; + + if (dst_id == mem_mgr_id) { + stmm_handle_mem_mgr_service(regs); + } else if (dst_id == ffa_storage_id) { + stmm_handle_storage_service(regs); + } else { + EMSG("Undefined endpoint id %#"PRIx16, dst_id); + spm_eret_error(STMM_RET_INVALID_PARAM, regs); + } +} + +/* Return true if returning to SP, false if returning to caller */ +static bool spm_handle_scall(struct thread_scall_regs *regs) +{ +#ifdef ARM64 + uint64_t *a0 = ®s->x0; +#endif +#ifdef ARM32 + uint32_t *a0 = ®s->r0; +#endif + + switch (*a0) { + case FFA_VERSION: + DMSG("Received FFA version"); + *a0 = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); + return true; + case __FFA_MSG_SEND_DIRECT_RESP: + DMSG("Received FFA direct response"); + return_from_sp_helper(false, 0, regs); + return false; + case __FFA_MSG_SEND_DIRECT_REQ: + DMSG("Received FFA direct request"); + spm_handle_direct_req(regs); + return true; + default: + EMSG("Undefined syscall %#"PRIx32, (uint32_t)*a0); + return_from_sp_helper(true /*panic*/, 0xabcd, regs); + return false; + } +} + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct ts_ops stmm_sp_ops __weak __relrodata_unpaged("stmm_sp_ops") = { + .enter_open_session = stmm_enter_open_session, + .enter_invoke_cmd = stmm_enter_invoke_cmd, + .enter_close_session = stmm_enter_close_session, + .dump_state = stmm_dump_state, + .destroy = stmm_ctx_destroy, + .get_instance_id = stmm_get_instance_id, + .handle_scall = spm_handle_scall, +}; diff --git a/optee_os/core/arch/arm/kernel/sub.mk b/optee_os/core/arch/arm/kernel/sub.mk new file mode 100644 index 0000000..49c5927 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/sub.mk @@ -0,0 +1,83 @@ +srcs-y += rpc_io_i2c.c +srcs-y += idle.c + +srcs-$(CFG_SECURE_TIME_SOURCE_CNTPCT) += tee_time_arm_cntpct.c +srcs-$(CFG_ARM64_core) += timer_a64.c + +srcs-$(CFG_ARM32_core) += spin_lock_a32.S +srcs-$(CFG_ARM64_core) += spin_lock_a64.S +srcs-$(CFG_ARM32_core) += tlb_helpers_a32.S +srcs-$(CFG_ARM64_core) += tlb_helpers_a64.S +srcs-$(CFG_ARM64_core) += cache_helpers_a64.S +srcs-$(CFG_ARM32_core) += cache_helpers_a32.S +srcs-$(CFG_PL310) += tz_ssvce_pl310_a32.S +srcs-$(CFG_PL310) += tee_l2cc_mutex.c + +srcs-$(CFG_ARM32_core) += thread_a32.S +srcs-$(CFG_ARM64_core) += thread_a64.S +srcs-y += thread.c +ifeq ($(CFG_WITH_USER_TA),y) +srcs-y += arch_scall.c +srcs-$(CFG_ARM32_core) += arch_scall_a32.S +srcs-$(CFG_ARM64_core) += arch_scall_a64.S +endif +ifeq ($(CFG_CORE_FFA),y) +srcs-y += thread_spmc.c +cppflags-thread_spmc.c-y += -DTEE_IMPL_GIT_SHA1=$(TEE_IMPL_GIT_SHA1) +srcs-$(CFG_ARM32_core) += thread_spmc_a32.S +srcs-$(CFG_ARM64_core) += thread_spmc_a64.S +else +srcs-y += thread_optee_smc.c +srcs-$(CFG_ARM32_core) += thread_optee_smc_a32.S +srcs-$(CFG_ARM64_core) += thread_optee_smc_a64.S +endif +srcs-y += abort.c +srcs-$(CFG_WITH_VFP) += vfp.c +ifeq ($(CFG_WITH_VFP),y) +srcs-$(CFG_ARM32_core) += vfp_a32.S +srcs-$(CFG_ARM64_core) += vfp_a64.S +endif +srcs-$(CFG_ARM32_core) += misc_a32.S +srcs-$(CFG_ARM64_core) += misc_a64.S +srcs-$(CFG_WITH_STMM_SP) += stmm_sp.c +srcs-$(CFG_SECURE_PARTITION) += secure_partition.c +srcs-$(CFG_SECURE_PARTITION) += spmc_sp_handler.c + +srcs-y += boot.c +srcs-$(CFG_ARM32_core) += entry_a32.S +srcs-$(CFG_ARM64_core) += entry_a64.S + +ifeq ($(CFG_UNWIND),y) +srcs-$(CFG_ARM32_core) += unwind_arm32.c +srcs-$(CFG_ARM64_core) += unwind_arm64.c +endif + +srcs-$(CFG_NS_VIRTUALIZATION) += virtualization.c + +srcs-y += link_dummies_paged.c +srcs-y += link_dummies_init.c + +asm-defines-y += asm-defines.c +# Reflect the following dependencies: +# asm-defines.c includes +# includes +# includes +# and (optional) +asm-defines-asm-defines.c-deps += $(out-dir)/core/include/generated/arm32_sysreg.h +ifeq ($(CFG_ARM_GICV3),y) +asm-defines-asm-defines.c-deps += $(out-dir)/core/include/generated/arm32_gicv3_sysreg.h +endif + +ifeq ($(CFG_SYSCALL_FTRACE),y) +# We would not like to profile thread.c file as it provide common APIs +# that are needed for ftrace framework to trace syscalls. So profiling +# this file could create an incorrect cyclic behaviour. +cflags-remove-thread.c-y += -pg +# Tracing abort dump files corrupts the stack trace. So exclude them +# from profiling. +cflags-remove-abort.c-y += -pg +ifeq ($(CFG_UNWIND),y) +cflags-remove-unwind_arm32.c-y += -pg +cflags-remove-unwind_arm64.c-$(CFG_ARM64_core) += -pg +endif +endif diff --git a/optee_os/core/arch/arm/kernel/tee_l2cc_mutex.c b/optee_os/core/arch/arm/kernel/tee_l2cc_mutex.c new file mode 100644 index 0000000..38e8f81 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/tee_l2cc_mutex.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * l2cc_mutex_va holds teecore virtual address of TZ L2CC mutex or NULL. + * + * l2cc_mutex_pa holds TZ L2CC mutex physical address. It is relevant only + * if 'l2cc_mutex_va' hold a non-NULL address. + */ +#define MUTEX_SZ sizeof(uint32_t) + +static uint32_t *l2cc_mutex_va; +static uint32_t l2cc_mutex_pa; +static uint32_t l2cc_mutex_boot_pa; +static unsigned int *l2cc_mutex; + +void tee_l2cc_store_mutex_boot_pa(uint32_t pa) +{ + l2cc_mutex_boot_pa = pa; +} + +/* + * Allocate public RAM to get a L2CC mutex to shared with NSec. + * Return 0 on success. + */ +static int l2cc_mutex_alloc(void) +{ + void *va; + + if (l2cc_mutex_va != NULL) + return -1; + + l2cc_mutex_pa = l2cc_mutex_boot_pa; + + va = phys_to_virt(l2cc_mutex_pa, MEM_AREA_NSEC_SHM, MUTEX_SZ); + if (!va) + return -1; + + *(uint32_t *)va = 0; + l2cc_mutex_va = va; + return 0; +} + +static void l2cc_mutex_set(void *mutex) +{ + l2cc_mutex = (unsigned int *)mutex; +} + +/* + * tee_xxx_l2cc_mutex(): Handle L2 mutex configuration requests from NSec + * + * Policy: + * - if NSec did not register a L2 mutex, default allocate it in public RAM. + * - if NSec disables L2 mutex, disable the current mutex and unregister it. + * + * Enable L2CC: NSec allows teecore to run safe outer maintance + * with shared mutex. + * Disable L2CC: NSec will run outer maintenance with locking + * shared mutex. teecore cannot run outer maintenance. + * Set L2CC: NSec proposes a Shared Memory locaiotn for the outer + * maintenance shared mutex. + * Get L2CC: NSec requests the outer maintenance shared mutex + * location. If NSec has successufully registered one, + * return its location, otherwise, allocated one in NSec + * and provided NSec the physical location. + */ +TEE_Result tee_enable_l2cc_mutex(void) +{ + int ret; + + if (!l2cc_mutex_va) { + ret = l2cc_mutex_alloc(); + if (ret) + return TEE_ERROR_GENERIC; + } + l2cc_mutex_set(l2cc_mutex_va); + return TEE_SUCCESS; +} + +TEE_Result tee_disable_l2cc_mutex(void) +{ + l2cc_mutex_va = NULL; + l2cc_mutex_set(NULL); + return TEE_SUCCESS; +} + +TEE_Result tee_get_l2cc_mutex(paddr_t *mutex) +{ + int ret; + + if (!l2cc_mutex_va) { + ret = l2cc_mutex_alloc(); + if (ret) + return TEE_ERROR_GENERIC; + } + *mutex = l2cc_mutex_pa; + return TEE_SUCCESS; +} + +TEE_Result tee_set_l2cc_mutex(paddr_t *mutex) +{ + uint32_t addr; + void *va; + + if (l2cc_mutex_va != NULL) + return TEE_ERROR_BAD_PARAMETERS; + addr = *mutex; + va = phys_to_virt(addr, MEM_AREA_NSEC_SHM, MUTEX_SZ); + if (!va) + return TEE_ERROR_BAD_PARAMETERS; + l2cc_mutex_pa = addr; + l2cc_mutex_va = va; + return TEE_SUCCESS; +} + +void tee_l2cc_mutex_lock(void) +{ + if (l2cc_mutex) + cpu_spin_lock(l2cc_mutex); +} + +void tee_l2cc_mutex_unlock(void) +{ + if (l2cc_mutex) + cpu_spin_unlock(l2cc_mutex); +} diff --git a/optee_os/core/arch/arm/kernel/tee_time_arm_cntpct.c b/optee_os/core/arch/arm/kernel/tee_time_arm_cntpct.c new file mode 100644 index 0000000..112dfc3 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/tee_time_arm_cntpct.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, 2015 Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static TEE_Result arm_cntpct_get_sys_time(TEE_Time *time) +{ + uint64_t cntpct = barrier_read_counter_timer(); + uint32_t cntfrq = read_cntfrq(); + + time->seconds = cntpct / cntfrq; + time->millis = (cntpct % cntfrq) / (cntfrq / TEE_TIME_MILLIS_BASE); + + return TEE_SUCCESS; +} + +static const struct time_source arm_cntpct_time_source = { + .name = "arm cntpct", + .protection_level = 1000, + .get_sys_time = arm_cntpct_get_sys_time, +}; + +REGISTER_TIME_SOURCE(arm_cntpct_time_source) + +/* + * We collect jitter using cntpct in 32- or 64-bit mode that is typically + * clocked at around 1MHz. + * + * The first time we are called, we add low 16 bits of the counter as entropy. + * + * Subsequently, accumulate 2 low bits each time by: + * + * - rotating the accumumlator by 2 bits + * - XORing it in 2-bit chunks with the whole CNTPCT contents + * + * and adding one byte of entropy when we reach 8 rotated bits. + */ + +void plat_prng_add_jitter_entropy(enum crypto_rng_src sid, unsigned int *pnum) +{ + uint64_t tsc = barrier_read_counter_timer(); + int bytes = 0, n; + static uint8_t first, bits; + static uint16_t acc; + + if (!first) { + acc = tsc; + bytes = 2; + first = 1; + } else { + acc = (acc << 2) | ((acc >> 6) & 3); + for (n = 0; n < 64; n += 2) + acc ^= (tsc >> n) & 3; + bits += 2; + if (bits >= 8) { + bits = 0; + bytes = 1; + } + } + if (bytes) { + FMSG("0x%02X", (int)acc & ((1 << (bytes * 8)) - 1)); + crypto_rng_add_event(sid, pnum, (uint8_t *)&acc, bytes); + } +} diff --git a/optee_os/core/arch/arm/kernel/thread.c b/optee_os/core/arch/arm/kernel/thread.c new file mode 100644 index 0000000..66833b3 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread.c @@ -0,0 +1,1198 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2022, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020-2021, Arm Limited + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 +static vaddr_t thread_user_kcode_va __nex_bss; +long thread_user_kcode_offset __nex_bss; +static size_t thread_user_kcode_size __nex_bss; +#endif + +#if defined(CFG_CORE_UNMAP_CORE_AT_EL0) && \ + defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) +long thread_user_kdata_sp_offset __nex_bss; +static uint8_t thread_user_kdata_page[ + ROUNDUP(sizeof(struct thread_core_local) * CFG_TEE_CORE_NB_CORE, + SMALL_PAGE_SIZE)] + __aligned(SMALL_PAGE_SIZE) +#ifndef CFG_NS_VIRTUALIZATION + __section(".nozi.kdata_page"); +#else + __section(".nex_nozi.kdata_page"); +#endif +#endif + +#ifdef ARM32 +uint32_t __nostackcheck thread_get_exceptions(void) +{ + uint32_t cpsr = read_cpsr(); + + return (cpsr >> CPSR_F_SHIFT) & THREAD_EXCP_ALL; +} + +void __nostackcheck thread_set_exceptions(uint32_t exceptions) +{ + uint32_t cpsr = read_cpsr(); + + /* Foreign interrupts must not be unmasked while holding a spinlock */ + if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) + assert_have_no_spinlock(); + + cpsr &= ~(THREAD_EXCP_ALL << CPSR_F_SHIFT); + cpsr |= ((exceptions & THREAD_EXCP_ALL) << CPSR_F_SHIFT); + + barrier(); + write_cpsr(cpsr); + barrier(); +} +#endif /*ARM32*/ + +#ifdef ARM64 +uint32_t __nostackcheck thread_get_exceptions(void) +{ + uint32_t daif = read_daif(); + + return (daif >> DAIF_F_SHIFT) & THREAD_EXCP_ALL; +} + +void __nostackcheck thread_set_exceptions(uint32_t exceptions) +{ + uint32_t daif = read_daif(); + + /* Foreign interrupts must not be unmasked while holding a spinlock */ + if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) + assert_have_no_spinlock(); + + daif &= ~(THREAD_EXCP_ALL << DAIF_F_SHIFT); + daif |= ((exceptions & THREAD_EXCP_ALL) << DAIF_F_SHIFT); + + barrier(); + write_daif(daif); + barrier(); +} +#endif /*ARM64*/ + +uint32_t __nostackcheck thread_mask_exceptions(uint32_t exceptions) +{ + uint32_t state = thread_get_exceptions(); + + thread_set_exceptions(state | (exceptions & THREAD_EXCP_ALL)); + return state; +} + +void __nostackcheck thread_unmask_exceptions(uint32_t state) +{ + thread_set_exceptions(state & THREAD_EXCP_ALL); +} + +static void thread_lazy_save_ns_vfp(void) +{ +#ifdef CFG_WITH_VFP + struct thread_ctx *thr = threads + thread_get_id(); + + thr->vfp_state.ns_saved = false; + vfp_lazy_save_state_init(&thr->vfp_state.ns); +#endif /*CFG_WITH_VFP*/ +} + +static void thread_lazy_restore_ns_vfp(void) +{ +#ifdef CFG_WITH_VFP + struct thread_ctx *thr = threads + thread_get_id(); + struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; + + assert(!thr->vfp_state.sec_lazy_saved && !thr->vfp_state.sec_saved); + + if (tuv && tuv->lazy_saved && !tuv->saved) { + vfp_lazy_save_state_final(&tuv->vfp, false /*!force_save*/); + tuv->saved = true; + } + + vfp_lazy_restore_state(&thr->vfp_state.ns, thr->vfp_state.ns_saved); + thr->vfp_state.ns_saved = false; +#endif /*CFG_WITH_VFP*/ +} + +#ifdef ARM32 +static void init_regs(struct thread_ctx *thread, uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6, uint32_t a7, void *pc) +{ + thread->regs.pc = (uint32_t)pc; + + /* + * Stdcalls starts in SVC mode with masked foreign interrupts, masked + * Asynchronous abort and unmasked native interrupts. + */ + thread->regs.cpsr = read_cpsr() & ARM32_CPSR_E; + thread->regs.cpsr |= CPSR_MODE_SVC | CPSR_A | + (THREAD_EXCP_FOREIGN_INTR << ARM32_CPSR_F_SHIFT); + /* Enable thumb mode if it's a thumb instruction */ + if (thread->regs.pc & 1) + thread->regs.cpsr |= CPSR_T; + /* Reinitialize stack pointer */ + thread->regs.svc_sp = thread->stack_va_end; + + /* + * Copy arguments into context. This will make the + * arguments appear in r0-r7 when thread is started. + */ + thread->regs.r0 = a0; + thread->regs.r1 = a1; + thread->regs.r2 = a2; + thread->regs.r3 = a3; + thread->regs.r4 = a4; + thread->regs.r5 = a5; + thread->regs.r6 = a6; + thread->regs.r7 = a7; +} +#endif /*ARM32*/ + +#ifdef ARM64 +static void init_regs(struct thread_ctx *thread, uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6, uint32_t a7, void *pc) +{ + thread->regs.pc = (uint64_t)pc; + + /* + * Stdcalls starts in SVC mode with masked foreign interrupts, masked + * Asynchronous abort and unmasked native interrupts. + */ + thread->regs.cpsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, + THREAD_EXCP_FOREIGN_INTR | DAIFBIT_ABT); + /* Reinitialize stack pointer */ + thread->regs.sp = thread->stack_va_end; + + /* + * Copy arguments into context. This will make the + * arguments appear in x0-x7 when thread is started. + */ + thread->regs.x[0] = a0; + thread->regs.x[1] = a1; + thread->regs.x[2] = a2; + thread->regs.x[3] = a3; + thread->regs.x[4] = a4; + thread->regs.x[5] = a5; + thread->regs.x[6] = a6; + thread->regs.x[7] = a7; + + /* Set up frame pointer as per the Aarch64 AAPCS */ + thread->regs.x[29] = 0; +} +#endif /*ARM64*/ + +static void __thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6, uint32_t a7, + void *pc, uint32_t flags) +{ + struct thread_core_local *l = thread_get_core_local(); + bool found_thread = false; + size_t n = 0; + + assert(l->curr_thread == THREAD_ID_INVALID); + + thread_lock_global(); + + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].state == THREAD_STATE_FREE) { + threads[n].state = THREAD_STATE_ACTIVE; + found_thread = true; + break; + } + } + + thread_unlock_global(); + + if (!found_thread) + return; + + l->curr_thread = n; + + threads[n].flags = flags; + init_regs(threads + n, a0, a1, a2, a3, a4, a5, a6, a7, pc); +#ifdef CFG_CORE_PAUTH + /* + * Copy the APIA key into the registers to be restored with + * thread_resume(). + */ + threads[n].regs.apiakey_hi = threads[n].keys.apia_hi; + threads[n].regs.apiakey_lo = threads[n].keys.apia_lo; +#endif + + thread_lazy_save_ns_vfp(); + + l->flags &= ~THREAD_CLF_TMP; + thread_resume(&threads[n].regs); + /*NOTREACHED*/ + panic(); +} + +void thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5) +{ + __thread_alloc_and_run(a0, a1, a2, a3, a4, a5, 0, 0, + thread_std_smc_entry, 0); +} + +#ifdef CFG_SECURE_PARTITION +void thread_sp_alloc_and_run(struct thread_smc_args *args __maybe_unused) +{ + __thread_alloc_and_run(args->a0, args->a1, args->a2, args->a3, args->a4, + args->a5, args->a6, args->a7, + spmc_sp_thread_entry, THREAD_FLAGS_FFA_ONLY); +} +#endif + +#ifdef ARM32 +static void copy_a0_to_a3(struct thread_ctx_regs *regs, uint32_t a0, + uint32_t a1, uint32_t a2, uint32_t a3) +{ + /* + * Update returned values from RPC, values will appear in + * r0-r3 when thread is resumed. + */ + regs->r0 = a0; + regs->r1 = a1; + regs->r2 = a2; + regs->r3 = a3; +} +#endif /*ARM32*/ + +#ifdef ARM64 +static void copy_a0_to_a3(struct thread_ctx_regs *regs, uint32_t a0, + uint32_t a1, uint32_t a2, uint32_t a3) +{ + /* + * Update returned values from RPC, values will appear in + * x0-x3 when thread is resumed. + */ + regs->x[0] = a0; + regs->x[1] = a1; + regs->x[2] = a2; + regs->x[3] = a3; +} +#endif /*ARM64*/ + +#ifdef ARM32 +static bool is_from_user(uint32_t cpsr) +{ + return (cpsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; +} +#endif + +#ifdef ARM64 +static bool is_from_user(uint32_t cpsr) +{ + if (cpsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) + return true; + if (((cpsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == + SPSR_64_MODE_EL0) + return true; + return false; +} +#endif + +#ifdef CFG_SYSCALL_FTRACE +static void __noprof ftrace_suspend(void) +{ + struct ts_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); + + if (s && s->fbuf) + s->fbuf->syscall_trace_suspended = true; +} + +static void __noprof ftrace_resume(void) +{ + struct ts_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); + + if (s && s->fbuf) + s->fbuf->syscall_trace_suspended = false; +} +#else +static void __noprof ftrace_suspend(void) +{ +} + +static void __noprof ftrace_resume(void) +{ +} +#endif + +static bool is_user_mode(struct thread_ctx_regs *regs) +{ + return is_from_user((uint32_t)regs->cpsr); +} + +void thread_resume_from_rpc(uint32_t thread_id, uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3) +{ + size_t n = thread_id; + struct thread_core_local *l = thread_get_core_local(); + bool found_thread = false; + + assert(l->curr_thread == THREAD_ID_INVALID); + + thread_lock_global(); + + if (n < CFG_NUM_THREADS && threads[n].state == THREAD_STATE_SUSPENDED) { + threads[n].state = THREAD_STATE_ACTIVE; + found_thread = true; + } + + thread_unlock_global(); + + if (!found_thread) + return; + + l->curr_thread = n; + + if (threads[n].have_user_map) { + core_mmu_set_user_map(&threads[n].user_map); + if (threads[n].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) + tee_ta_ftrace_update_times_resume(); + } + + if (is_user_mode(&threads[n].regs)) + tee_ta_update_session_utime_resume(); + + /* + * Return from RPC to request service of a foreign interrupt must not + * get parameters from non-secure world. + */ + if (threads[n].flags & THREAD_FLAGS_COPY_ARGS_ON_RETURN) { + copy_a0_to_a3(&threads[n].regs, a0, a1, a2, a3); + threads[n].flags &= ~THREAD_FLAGS_COPY_ARGS_ON_RETURN; + } + + thread_lazy_save_ns_vfp(); + + if (threads[n].have_user_map) + ftrace_resume(); + + l->flags &= ~THREAD_CLF_TMP; + thread_resume(&threads[n].regs); + /*NOTREACHED*/ + panic(); +} + +#ifdef ARM64 +static uint64_t spsr_from_pstate(void) +{ + uint64_t spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, 0); + + spsr |= read_daif(); + if (IS_ENABLED(CFG_PAN) && feat_pan_implemented() && read_pan()) + spsr |= SPSR_64_PAN; + + return spsr; +} + +void __thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]) +{ + thread_rpc_spsr(rv, spsr_from_pstate()); +} + +vaddr_t thread_get_saved_thread_sp(void) +{ + struct thread_core_local *l = thread_get_core_local(); + int ct = l->curr_thread; + + assert(ct != THREAD_ID_INVALID); + return threads[ct].kern_sp; +} +#endif /*ARM64*/ + +#ifdef ARM32 +bool thread_is_in_normal_mode(void) +{ + return (read_cpsr() & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_SVC; +} +#endif + +void thread_state_free(void) +{ + struct thread_core_local *l = thread_get_core_local(); + int ct = l->curr_thread; + + assert(ct != THREAD_ID_INVALID); + + thread_lazy_restore_ns_vfp(); + tee_pager_release_phys( + (void *)(threads[ct].stack_va_end - STACK_THREAD_SIZE), + STACK_THREAD_SIZE); + + thread_lock_global(); + + assert(threads[ct].state == THREAD_STATE_ACTIVE); + threads[ct].state = THREAD_STATE_FREE; + threads[ct].flags = 0; + l->curr_thread = THREAD_ID_INVALID; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + thread_unlock_global(); +} + +#ifdef CFG_WITH_PAGER +static void release_unused_kernel_stack(struct thread_ctx *thr, + uint32_t cpsr __maybe_unused) +{ +#ifdef ARM64 + /* + * If we're from user mode then thr->regs.sp is the saved user + * stack pointer and thr->kern_sp holds the last kernel stack + * pointer. But if we're from kernel mode then thr->kern_sp isn't + * up to date so we need to read from thr->regs.sp instead. + */ + vaddr_t sp = is_from_user(cpsr) ? thr->kern_sp : thr->regs.sp; +#else + vaddr_t sp = thr->regs.svc_sp; +#endif + vaddr_t base = thr->stack_va_end - STACK_THREAD_SIZE; + size_t len = sp - base; + + tee_pager_release_phys((void *)base, len); +} +#else +static void release_unused_kernel_stack(struct thread_ctx *thr __unused, + uint32_t cpsr __unused) +{ +} +#endif + +int thread_state_suspend(uint32_t flags, uint32_t cpsr, vaddr_t pc) +{ + struct thread_core_local *l = thread_get_core_local(); + int ct = l->curr_thread; + + assert(ct != THREAD_ID_INVALID); + + if (core_mmu_user_mapping_is_active()) + ftrace_suspend(); + + thread_check_canaries(); + + release_unused_kernel_stack(threads + ct, cpsr); + + if (is_from_user(cpsr)) { + thread_user_save_vfp(); + tee_ta_update_session_utime_suspend(); + tee_ta_gprof_sample_pc(pc); + } + thread_lazy_restore_ns_vfp(); + + thread_lock_global(); + + assert(threads[ct].state == THREAD_STATE_ACTIVE); + threads[ct].flags |= flags; + threads[ct].regs.cpsr = cpsr; + threads[ct].regs.pc = pc; + threads[ct].state = THREAD_STATE_SUSPENDED; + + threads[ct].have_user_map = core_mmu_user_mapping_is_active(); + if (threads[ct].have_user_map) { + if (threads[ct].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) + tee_ta_ftrace_update_times_suspend(); + core_mmu_get_user_map(&threads[ct].user_map); + core_mmu_set_user_map(NULL); + } + + if (IS_ENABLED(CFG_SECURE_PARTITION)) { + struct ts_session *ts_sess = + TAILQ_FIRST(&threads[ct].tsd.sess_stack); + + spmc_sp_set_to_preempted(ts_sess); + } + + l->curr_thread = THREAD_ID_INVALID; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + + thread_unlock_global(); + + return ct; +} + +bool thread_init_stack(uint32_t thread_id, vaddr_t sp) +{ + if (thread_id >= CFG_NUM_THREADS) + return false; + threads[thread_id].stack_va_end = sp; + return true; +} + +static void __maybe_unused +set_core_local_kcode_offset(struct thread_core_local *cls, long offset) +{ + size_t n = 0; + + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) + cls[n].kcode_offset = offset; +} + +static void init_user_kcode(void) +{ +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + vaddr_t v = (vaddr_t)thread_excp_vect; + vaddr_t ve = (vaddr_t)thread_excp_vect_end; + + thread_user_kcode_va = ROUNDDOWN(v, CORE_MMU_USER_CODE_SIZE); + ve = ROUNDUP(ve, CORE_MMU_USER_CODE_SIZE); + thread_user_kcode_size = ve - thread_user_kcode_va; + + core_mmu_get_user_va_range(&v, NULL); + thread_user_kcode_offset = thread_user_kcode_va - v; + + set_core_local_kcode_offset(thread_core_local, + thread_user_kcode_offset); +#if defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) + set_core_local_kcode_offset((void *)thread_user_kdata_page, + thread_user_kcode_offset); + /* + * When transitioning to EL0 subtract SP with this much to point to + * this special kdata page instead. SP is restored by add this much + * while transitioning back to EL1. + */ + v += thread_user_kcode_size; + thread_user_kdata_sp_offset = (vaddr_t)thread_core_local - v; +#endif +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ +} + +void thread_init_primary(void) +{ + /* Initialize canaries around the stacks */ + thread_init_canaries(); + + init_user_kcode(); +} + +static uint32_t __maybe_unused get_midr_implementer(uint32_t midr) +{ + return (midr >> MIDR_IMPLEMENTER_SHIFT) & MIDR_IMPLEMENTER_MASK; +} + +static uint32_t __maybe_unused get_midr_primary_part(uint32_t midr) +{ + return (midr >> MIDR_PRIMARY_PART_NUM_SHIFT) & + MIDR_PRIMARY_PART_NUM_MASK; +} + +static uint32_t __maybe_unused get_midr_variant(uint32_t midr) +{ + return (midr >> MIDR_VARIANT_SHIFT) & MIDR_VARIANT_MASK; +} + +static uint32_t __maybe_unused get_midr_revision(uint32_t midr) +{ + return (midr >> MIDR_REVISION_SHIFT) & MIDR_REVISION_MASK; +} + +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC +#ifdef ARM64 +static bool probe_workaround_available(uint32_t wa_id) +{ + int32_t r; + + r = thread_smc(SMCCC_VERSION, 0, 0, 0); + if (r < 0) + return false; + if (r < 0x10001) /* compare with version 1.1 */ + return false; + + /* Version >= 1.1, so SMCCC_ARCH_FEATURES is available */ + r = thread_smc(SMCCC_ARCH_FEATURES, wa_id, 0, 0); + return r >= 0; +} + +static vaddr_t __maybe_unused select_vector_wa_spectre_v2(void) +{ + if (probe_workaround_available(SMCCC_ARCH_WORKAROUND_1)) { + DMSG("SMCCC_ARCH_WORKAROUND_1 (%#08" PRIx32 ") available", + SMCCC_ARCH_WORKAROUND_1); + DMSG("SMC Workaround for CVE-2017-5715 used"); + return (vaddr_t)thread_excp_vect_wa_spectre_v2; + } + + DMSG("SMCCC_ARCH_WORKAROUND_1 (%#08" PRIx32 ") unavailable", + SMCCC_ARCH_WORKAROUND_1); + DMSG("SMC Workaround for CVE-2017-5715 not needed (if ARM-TF is up to date)"); + return (vaddr_t)thread_excp_vect; +} +#else +static vaddr_t __maybe_unused select_vector_wa_spectre_v2(void) +{ + return (vaddr_t)thread_excp_vect_wa_spectre_v2; +} +#endif +#endif + +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC +static vaddr_t select_vector_wa_spectre_bhb(uint8_t loop_count __maybe_unused) +{ + /* + * Spectre-BHB has only been analyzed for AArch64 so far. For + * AArch32 fall back to the Spectre-V2 workaround which is likely + * to work even if perhaps a bit more expensive than a more + * optimized workaround. + */ +#ifdef ARM64 +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + struct thread_core_local *cl = (void *)thread_user_kdata_page; + + cl[get_core_pos()].bhb_loop_count = loop_count; +#endif + thread_get_core_local()->bhb_loop_count = loop_count; + + DMSG("Spectre-BHB CVE-2022-23960 workaround enabled with \"K\" = %u", + loop_count); + + return (vaddr_t)thread_excp_vect_wa_spectre_bhb; +#else + return select_vector_wa_spectre_v2(); +#endif +} +#endif + +static vaddr_t get_excp_vect(void) +{ +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC + uint32_t midr = read_midr(); + uint8_t vers = 0; + + if (get_midr_implementer(midr) != MIDR_IMPLEMENTER_ARM) + return (vaddr_t)thread_excp_vect; + /* + * Variant rx, Revision py, for instance + * Variant 2 Revision 0 = r2p0 = 0x20 + */ + vers = (get_midr_variant(midr) << 4) | get_midr_revision(midr); + + /* + * Spectre-V2 (CVE-2017-5715) software workarounds covers what's + * needed for Spectre-BHB (CVE-2022-23960) too. The workaround for + * Spectre-V2 is more expensive than the one for Spectre-BHB so if + * possible select the workaround for Spectre-BHB. + */ + switch (get_midr_primary_part(midr)) { +#ifdef ARM32 + /* Spectre-V2 */ + case CORTEX_A8_PART_NUM: + case CORTEX_A9_PART_NUM: + case CORTEX_A17_PART_NUM: +#endif + /* Spectre-V2 */ + case CORTEX_A57_PART_NUM: + case CORTEX_A73_PART_NUM: + case CORTEX_A75_PART_NUM: + return select_vector_wa_spectre_v2(); +#ifdef ARM32 + /* Spectre-V2 */ + case CORTEX_A15_PART_NUM: + return (vaddr_t)thread_excp_vect_wa_a15_spectre_v2; +#endif + /* + * Spectre-V2 for vers < r1p0 + * Spectre-BHB for vers >= r1p0 + */ + case CORTEX_A72_PART_NUM: + if (vers < 0x10) + return select_vector_wa_spectre_v2(); + return select_vector_wa_spectre_bhb(8); + + /* + * Doing the more safe but expensive Spectre-V2 workaround for CPUs + * still being researched on the best mitigation sequence. + */ + case CORTEX_A65_PART_NUM: + case CORTEX_A65AE_PART_NUM: + case NEOVERSE_E1_PART_NUM: + return select_vector_wa_spectre_v2(); + + /* Spectre-BHB */ + case CORTEX_A76_PART_NUM: + case CORTEX_A76AE_PART_NUM: + case CORTEX_A77_PART_NUM: + return select_vector_wa_spectre_bhb(24); + case CORTEX_A78_PART_NUM: + case CORTEX_A78AE_PART_NUM: + case CORTEX_A78C_PART_NUM: + case CORTEX_A710_PART_NUM: + case CORTEX_X1_PART_NUM: + case CORTEX_X2_PART_NUM: + return select_vector_wa_spectre_bhb(32); + case NEOVERSE_N1_PART_NUM: + return select_vector_wa_spectre_bhb(24); + case NEOVERSE_N2_PART_NUM: + case NEOVERSE_V1_PART_NUM: + return select_vector_wa_spectre_bhb(32); + + default: + return (vaddr_t)thread_excp_vect; + } +#endif /*CFG_CORE_WORKAROUND_SPECTRE_BP_SEC*/ + + return (vaddr_t)thread_excp_vect; +} + +void thread_init_per_cpu(void) +{ +#ifdef ARM32 + struct thread_core_local *l = thread_get_core_local(); + +#if !defined(CFG_WITH_ARM_TRUSTED_FW) + /* Initialize secure monitor */ + sm_init(l->tmp_stack_va_end + STACK_TMP_OFFS); +#endif + thread_set_irq_sp(l->tmp_stack_va_end); + thread_set_fiq_sp(l->tmp_stack_va_end); + thread_set_abt_sp((vaddr_t)l); + thread_set_und_sp((vaddr_t)l); +#endif + + thread_init_vbar(get_excp_vect()); + +#ifdef CFG_FTRACE_SUPPORT + /* + * Enable accesses to frequency register and physical counter + * register in EL0/PL0 required for timestamping during + * function tracing. + */ + write_cntkctl(read_cntkctl() | CNTKCTL_PL0PCTEN); +#endif +} + +#ifdef CFG_WITH_VFP +uint32_t thread_kernel_enable_vfp(void) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + struct thread_ctx *thr = threads + thread_get_id(); + struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; + + assert(!vfp_is_enabled()); + + if (!thr->vfp_state.ns_saved) { + vfp_lazy_save_state_final(&thr->vfp_state.ns, + true /*force_save*/); + thr->vfp_state.ns_saved = true; + } else if (thr->vfp_state.sec_lazy_saved && + !thr->vfp_state.sec_saved) { + /* + * This happens when we're handling an abort while the + * thread was using the VFP state. + */ + vfp_lazy_save_state_final(&thr->vfp_state.sec, + false /*!force_save*/); + thr->vfp_state.sec_saved = true; + } else if (tuv && tuv->lazy_saved && !tuv->saved) { + /* + * This can happen either during syscall or abort + * processing (while processing a syscall). + */ + vfp_lazy_save_state_final(&tuv->vfp, false /*!force_save*/); + tuv->saved = true; + } + + vfp_enable(); + return exceptions; +} + +void thread_kernel_disable_vfp(uint32_t state) +{ + uint32_t exceptions; + + assert(vfp_is_enabled()); + + vfp_disable(); + exceptions = thread_get_exceptions(); + assert(exceptions & THREAD_EXCP_FOREIGN_INTR); + exceptions &= ~THREAD_EXCP_FOREIGN_INTR; + exceptions |= state & THREAD_EXCP_FOREIGN_INTR; + thread_set_exceptions(exceptions); +} + +void thread_kernel_save_vfp(void) +{ + struct thread_ctx *thr = threads + thread_get_id(); + + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + if (vfp_is_enabled()) { + vfp_lazy_save_state_init(&thr->vfp_state.sec); + thr->vfp_state.sec_lazy_saved = true; + } +} + +void thread_kernel_restore_vfp(void) +{ + struct thread_ctx *thr = threads + thread_get_id(); + + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + assert(!vfp_is_enabled()); + if (thr->vfp_state.sec_lazy_saved) { + vfp_lazy_restore_state(&thr->vfp_state.sec, + thr->vfp_state.sec_saved); + thr->vfp_state.sec_saved = false; + thr->vfp_state.sec_lazy_saved = false; + } +} + +void thread_user_enable_vfp(struct thread_user_vfp_state *uvfp) +{ + struct thread_ctx *thr = threads + thread_get_id(); + struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; + + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + assert(!vfp_is_enabled()); + + if (!thr->vfp_state.ns_saved) { + vfp_lazy_save_state_final(&thr->vfp_state.ns, + true /*force_save*/); + thr->vfp_state.ns_saved = true; + } else if (tuv && uvfp != tuv) { + if (tuv->lazy_saved && !tuv->saved) { + vfp_lazy_save_state_final(&tuv->vfp, + false /*!force_save*/); + tuv->saved = true; + } + } + + if (uvfp->lazy_saved) + vfp_lazy_restore_state(&uvfp->vfp, uvfp->saved); + uvfp->lazy_saved = false; + uvfp->saved = false; + + thr->vfp_state.uvfp = uvfp; + vfp_enable(); +} + +void thread_user_save_vfp(void) +{ + struct thread_ctx *thr = threads + thread_get_id(); + struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; + + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + if (!vfp_is_enabled()) + return; + + assert(tuv && !tuv->lazy_saved && !tuv->saved); + vfp_lazy_save_state_init(&tuv->vfp); + tuv->lazy_saved = true; +} + +void thread_user_clear_vfp(struct user_mode_ctx *uctx) +{ + struct thread_user_vfp_state *uvfp = &uctx->vfp; + struct thread_ctx *thr = threads + thread_get_id(); + + if (uvfp == thr->vfp_state.uvfp) + thr->vfp_state.uvfp = NULL; + uvfp->lazy_saved = false; + uvfp->saved = false; +} +#endif /*CFG_WITH_VFP*/ + +#ifdef ARM32 +static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) +{ + uint32_t s; + + if (!is_32bit) + return false; + + s = read_cpsr(); + s &= ~(CPSR_MODE_MASK | CPSR_T | CPSR_IT_MASK1 | CPSR_IT_MASK2); + s |= CPSR_MODE_USR; + if (entry_func & 1) + s |= CPSR_T; + *spsr = s; + return true; +} +#endif + +#ifdef ARM64 +static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) +{ + uint32_t s; + + if (is_32bit) { + s = read_daif() & (SPSR_32_AIF_MASK << SPSR_32_AIF_SHIFT); + s |= SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT; + s |= (entry_func & SPSR_32_T_MASK) << SPSR_32_T_SHIFT; + } else { + s = read_daif() & (SPSR_64_DAIF_MASK << SPSR_64_DAIF_SHIFT); + } + + *spsr = s; + return true; +} +#endif + +static void set_ctx_regs(struct thread_ctx_regs *regs, unsigned long a0, + unsigned long a1, unsigned long a2, unsigned long a3, + unsigned long user_sp, unsigned long entry_func, + uint32_t spsr, + struct thread_pauth_keys *keys __maybe_unused) +{ + /* + * First clear all registers to avoid leaking information from + * other TAs or even the Core itself. + */ + *regs = (struct thread_ctx_regs){ }; +#ifdef ARM32 + regs->r0 = a0; + regs->r1 = a1; + regs->r2 = a2; + regs->r3 = a3; + regs->usr_sp = user_sp; + regs->pc = entry_func; + regs->cpsr = spsr; +#endif +#ifdef ARM64 + regs->x[0] = a0; + regs->x[1] = a1; + regs->x[2] = a2; + regs->x[3] = a3; + regs->sp = user_sp; + regs->pc = entry_func; + regs->cpsr = spsr; + regs->x[13] = user_sp; /* Used when running TA in Aarch32 */ + regs->sp = user_sp; /* Used when running TA in Aarch64 */ +#ifdef CFG_TA_PAUTH + assert(keys); + regs->apiakey_hi = keys->apia_hi; + regs->apiakey_lo = keys->apia_lo; +#endif + /* Set frame pointer (user stack can't be unwound past this point) */ + regs->x[29] = 0; +#endif +} + +static struct thread_pauth_keys *thread_get_pauth_keys(void) +{ +#if defined(CFG_TA_PAUTH) + struct ts_session *s = ts_get_current_session(); + /* Only user TA's support the PAUTH keys */ + struct user_ta_ctx *utc = to_user_ta_ctx(s->ctx); + + return &utc->uctx.keys; +#else + return NULL; +#endif +} + +uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, + unsigned long a2, unsigned long a3, unsigned long user_sp, + unsigned long entry_func, bool is_32bit, + uint32_t *exit_status0, uint32_t *exit_status1) +{ + uint32_t spsr = 0; + uint32_t exceptions = 0; + uint32_t rc = 0; + struct thread_ctx_regs *regs = NULL; + struct thread_pauth_keys *keys = NULL; + + tee_ta_update_session_utime_resume(); + + keys = thread_get_pauth_keys(); + + /* Derive SPSR from current CPSR/PSTATE readout. */ + if (!get_spsr(is_32bit, entry_func, &spsr)) { + *exit_status0 = 1; /* panic */ + *exit_status1 = 0xbadbadba; + return 0; + } + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + /* + * We're using the per thread location of saved context registers + * for temporary storage. Now that exceptions are masked they will + * not be used for any thing else until they are eventually + * unmasked when user mode has been entered. + */ + regs = thread_get_ctx_regs(); + set_ctx_regs(regs, a0, a1, a2, a3, user_sp, entry_func, spsr, keys); + rc = __thread_enter_user_mode(regs, exit_status0, exit_status1); + thread_unmask_exceptions(exceptions); + return rc; +} + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 +void thread_get_user_kcode(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz) +{ + core_mmu_get_user_va_range(va, NULL); + *mobj = mobj_tee_ram_rx; + *sz = thread_user_kcode_size; + *offset = thread_user_kcode_va - (vaddr_t)mobj_get_va(*mobj, 0, *sz); +} +#endif + +#if defined(CFG_CORE_UNMAP_CORE_AT_EL0) && \ + defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) +void thread_get_user_kdata(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz) +{ + vaddr_t v; + + core_mmu_get_user_va_range(&v, NULL); + *va = v + thread_user_kcode_size; + *mobj = mobj_tee_ram_rw; + *sz = sizeof(thread_user_kdata_page); + *offset = (vaddr_t)thread_user_kdata_page - + (vaddr_t)mobj_get_va(*mobj, 0, *sz); +} +#endif + +static void setup_unwind_user_mode(struct thread_scall_regs *regs) +{ +#ifdef ARM32 + regs->lr = (uintptr_t)thread_unwind_user_mode; + regs->spsr = read_cpsr(); +#endif +#ifdef ARM64 + regs->elr = (uintptr_t)thread_unwind_user_mode; + regs->spsr = spsr_from_pstate(); + /* + * Regs is the value of stack pointer before calling the SVC + * handler. By the addition matches for the reserved space at the + * beginning of el0_sync_svc(). This prepares the stack when + * returning to thread_unwind_user_mode instead of a normal + * exception return. + */ + regs->sp_el0 = (uint64_t)(regs + 1); +#endif +} + +static void gprof_set_status(struct ts_session *s __maybe_unused, + enum ts_gprof_status status __maybe_unused) +{ +#ifdef CFG_TA_GPROF_SUPPORT + if (s->ctx->ops->gprof_set_status) + s->ctx->ops->gprof_set_status(status); +#endif +} + +/* + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +void __weak thread_scall_handler(struct thread_scall_regs *regs) +{ + struct ts_session *sess = NULL; + uint32_t state = 0; + + /* Enable native interrupts */ + state = thread_get_exceptions(); + thread_unmask_exceptions(state & ~THREAD_EXCP_NATIVE_INTR); + + thread_user_save_vfp(); + + sess = ts_get_current_session(); + /* + * User mode service has just entered kernel mode, suspend gprof + * collection until we're about to switch back again. + */ + gprof_set_status(sess, TS_GPROF_SUSPEND); + + /* Restore foreign interrupts which are disabled on exception entry */ + thread_restore_foreign_intr(); + + assert(sess && sess->handle_scall); + if (sess->handle_scall(regs)) { + /* We're about to switch back to user mode */ + gprof_set_status(sess, TS_GPROF_RESUME); + } else { + /* We're returning from __thread_enter_user_mode() */ + setup_unwind_user_mode(regs); + } +} + +#ifdef CFG_WITH_ARM_TRUSTED_FW +/* + * These five functions are __weak to allow platforms to override them if + * needed. + */ +unsigned long __weak thread_cpu_off_handler(unsigned long a0 __unused, + unsigned long a1 __unused) +{ + return 0; +} +DECLARE_KEEP_PAGER(thread_cpu_off_handler); + +unsigned long __weak thread_cpu_suspend_handler(unsigned long a0 __unused, + unsigned long a1 __unused) +{ + return 0; +} +DECLARE_KEEP_PAGER(thread_cpu_suspend_handler); + +unsigned long __weak thread_cpu_resume_handler(unsigned long a0 __unused, + unsigned long a1 __unused) +{ + return 0; +} +DECLARE_KEEP_PAGER(thread_cpu_resume_handler); + +unsigned long __weak thread_system_off_handler(unsigned long a0 __unused, + unsigned long a1 __unused) +{ + return 0; +} +DECLARE_KEEP_PAGER(thread_system_off_handler); + +unsigned long __weak thread_system_reset_handler(unsigned long a0 __unused, + unsigned long a1 __unused) +{ + return 0; +} +DECLARE_KEEP_PAGER(thread_system_reset_handler); +#endif /*CFG_WITH_ARM_TRUSTED_FW*/ + +#ifdef CFG_CORE_WORKAROUND_ARM_NMFI +void __noreturn interrupt_main_handler(void) +{ + /* + * Note: overrides the default implementation of this function so that + * if there would be another handler defined there would be duplicate + * symbol error during linking. + */ + panic("Secure interrupt received but it is not supported"); +} +#endif diff --git a/optee_os/core/arch/arm/kernel/thread_a32.S b/optee_os/core/arch/arm/kernel/thread_a32.S new file mode 100644 index 0000000..5a20421 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_a32.S @@ -0,0 +1,990 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2020, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + .syntax unified + .arch_extension sec + + .macro cmp_spsr_user_mode reg:req + /* + * We're only testing the lower 4 bits as bit 5 (0x10) + * always is set. + */ + tst \reg, #0x0f + .endm + +FUNC thread_set_abt_sp , : +UNWIND( .cantunwind) + mrs r1, cpsr + cps #CPSR_MODE_ABT + mov sp, r0 + msr cpsr, r1 + bx lr +END_FUNC thread_set_abt_sp + +FUNC thread_set_und_sp , : +UNWIND( .cantunwind) + mrs r1, cpsr + cps #CPSR_MODE_UND + mov sp, r0 + msr cpsr, r1 + bx lr +END_FUNC thread_set_und_sp + +FUNC thread_set_irq_sp , : +UNWIND( .cantunwind) + mrs r1, cpsr + cps #CPSR_MODE_IRQ + mov sp, r0 + msr cpsr, r1 + bx lr +END_FUNC thread_set_irq_sp + +FUNC thread_set_fiq_sp , : +UNWIND( .cantunwind) + mrs r1, cpsr + cps #CPSR_MODE_FIQ + mov sp, r0 + msr cpsr, r1 + bx lr +END_FUNC thread_set_fiq_sp + +FUNC thread_get_usr_sp , : + mrs r1, cpsr + cpsid aif + cps #CPSR_MODE_SYS + mov r0, sp + msr cpsr, r1 + bx lr +END_FUNC thread_get_usr_sp + +FUNC thread_get_usr_lr , : + mrs r1, cpsr + cpsid aif + cps #CPSR_MODE_SYS + mov r0, lr + msr cpsr, r1 + bx lr +END_FUNC thread_get_usr_lr + +FUNC thread_set_usr_lr , : + mrs r1, cpsr + cpsid aif + cps #CPSR_MODE_SYS + mov lr, r0 + msr cpsr, r1 + bx lr +END_FUNC thread_set_usr_lr + +/* void thread_resume(struct thread_ctx_regs *regs) */ +FUNC thread_resume , : +UNWIND( .cantunwind) + add r12, r0, #(13 * 4) /* Restore registers r0-r12 later */ + + cps #CPSR_MODE_SYS + ldr sp, [r12], #4 + ldr lr, [r12], #4 + + cps #CPSR_MODE_SVC + ldr r1, [r12], #4 + ldr sp, [r12], #4 + ldr lr, [r12], #4 + msr spsr_fsxc, r1 + + ldm r12, {r1, r2} + + /* + * Switching to some other mode than SVC as we need to set spsr in + * order to return into the old state properly and it may be SVC + * mode we're returning to. + */ + cps #CPSR_MODE_ABT + cmp_spsr_user_mode r2 + mov lr, r1 + msr spsr_fsxc, r2 + ldm r0, {r0-r12} + movsne pc, lr + b eret_to_user_mode +END_FUNC thread_resume + +/* + * Disables IRQ and FIQ and saves state of thread in fiq mode which has + * the banked r8-r12 registers, returns original CPSR. + */ +LOCAL_FUNC thread_save_state_fiq , : +UNWIND( .cantunwind) + mov r9, lr + + /* + * Uses stack for temporary storage, while storing needed + * context in the thread context struct. + */ + + mrs r8, cpsr + + cpsid aif /* Disable Async abort, IRQ and FIQ */ + + push {r4-r7} + push {r0-r3} + + mrs r6, cpsr /* Save current CPSR */ + + bl thread_get_ctx_regs + + pop {r1-r4} /* r0-r3 pushed above */ + stm r0!, {r1-r4} + pop {r1-r4} /* r4-r7 pushed above */ + stm r0!, {r1-r4} + + cps #CPSR_MODE_SYS + stm r0!, {r8-r12} + str sp, [r0], #4 + str lr, [r0], #4 + + cps #CPSR_MODE_SVC + mrs r1, spsr + str r1, [r0], #4 + str sp, [r0], #4 + str lr, [r0], #4 + + /* back to fiq mode */ + orr r6, r6, #ARM32_CPSR_FIA /* Disable Async abort, IRQ and FIQ */ + msr cpsr, r6 /* Restore mode */ + + mov r0, r8 /* Return original CPSR */ + bx r9 +END_FUNC thread_save_state_fiq + +/* + * Disables IRQ and FIQ and saves state of thread, returns original + * CPSR. + */ +FUNC thread_save_state , : +UNWIND( .cantunwind) + push {r12, lr} + /* + * Uses stack for temporary storage, while storing needed + * context in the thread context struct. + */ + + mrs r12, cpsr + + cpsid aif /* Disable Async abort, IRQ and FIQ */ + + push {r4-r7} + push {r0-r3} + + mov r5, r12 /* Save CPSR in a preserved register */ + mrs r6, cpsr /* Save current CPSR */ + + bl thread_get_ctx_regs + + pop {r1-r4} /* r0-r3 pushed above */ + stm r0!, {r1-r4} + pop {r1-r4} /* r4-r7 pushed above */ + stm r0!, {r1-r4} + stm r0!, {r8-r11} + + pop {r12, lr} + stm r0!, {r12} + + cps #CPSR_MODE_SYS + str sp, [r0], #4 + str lr, [r0], #4 + + cps #CPSR_MODE_SVC + mrs r1, spsr + str r1, [r0], #4 + str sp, [r0], #4 + str lr, [r0], #4 + + orr r6, r6, #ARM32_CPSR_FIA /* Disable Async abort, IRQ and FIQ */ + msr cpsr, r6 /* Restore mode */ + + mov r0, r5 /* Return original CPSR */ + bx lr +END_FUNC thread_save_state + +#ifdef CFG_CORE_SEL2_SPMC +/* + * unsigned long thread_hvc(unsigned long func_id, unsigned long a1, + * unsigned long a2, unsigned long a3) + */ +FUNC thread_hvc , : + push {r4-r7} +UNWIND( .save {r4-r7}) + hvc #0 + pop {r4-r7} + bx lr +END_FUNC thread_hvc +#endif /*CFG_CORE_SEL2_SPMC*/ + +/* + * unsigned long thread_smc(unsigned long func_id, unsigned long a1, + * unsigned long a2, unsigned long a3) + */ +FUNC thread_smc , : + push {r4-r7} +UNWIND( .save {r4-r7}) + smc #0 + pop {r4-r7} + bx lr +END_FUNC thread_smc + +/* void thread_smccc(struct thread_smc_args *arg_res) */ +FUNC thread_smccc , : + push {r4-r7} + push {r0, lr} + ldm r0, {r0-r7} +#ifdef CFG_CORE_SEL2_SPMC + hvc #0 +#else + smc #0 +#endif + pop {r12, lr} + stm r12, {r0-r7} + pop {r4-r7} + bx lr +END_FUNC thread_smccc + +FUNC thread_init_vbar , : + /* Set vector (VBAR) */ + write_vbar r0 + bx lr +END_FUNC thread_init_vbar +DECLARE_KEEP_PAGER thread_init_vbar + +/* + * Below are low level routines handling entry and return from user mode. + * + * thread_enter_user_mode() saves all that registers user mode can change + * so kernel mode can restore needed registers when resuming execution + * after the call to thread_enter_user_mode() has returned. + * thread_enter_user_mode() doesn't return directly since it enters user + * mode instead, it's thread_unwind_user_mode() that does the + * returning by restoring the registers saved by thread_enter_user_mode(). + * + * There's three ways for thread_enter_user_mode() to return to caller, + * user TA calls _utee_return, user TA calls _utee_panic or through an abort. + * + * Calls to _utee_return or _utee_panic are handled as: + * __thread_svc_handler() -> thread_scall_handler() -> scall_do_call() which + * calls syscall_return() or syscall_panic(). + * + * These function calls returns normally except thread_scall_handler() which + * which is an exception handling routine so it reads return address and + * SPSR to restore from the stack. syscall_return() and syscall_panic() + * changes return address and SPSR used by thread_scall_handler() to instead of + * returning into user mode as with other syscalls it returns into + * thread_unwind_user_mode() in kernel mode instead. When + * thread_scall_handler() returns the stack pointer at the point where + * thread_enter_user_mode() left it so this is where + * thread_unwind_user_mode() can operate. + * + * Aborts are handled in a similar way but by thread_abort_handler() + * instead, when the pager sees that it's an abort from user mode that + * can't be handled it updates SPSR and return address used by + * thread_abort_handler() to return into thread_unwind_user_mode() + * instead. + */ + +/* + * uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, + * uint32_t *exit_status0, + * uint32_t *exit_status1); + * + * This function depends on being called with exceptions masked. + */ +FUNC __thread_enter_user_mode , : +UNWIND( .cantunwind) + /* + * Save all registers to allow syscall_return() to resume execution + * as if this function would have returned. This is also used in + * syscall_panic(). + * + * If stack usage of this function is changed + * thread_unwind_user_mode() has to be updated. + */ + push {r4-r12,lr} + + /* + * Save old user sp and set new user sp. + */ + cps #CPSR_MODE_SYS + mov r4, sp + ldr sp, [r0, #THREAD_CTX_REGS_USR_SP] + cps #CPSR_MODE_SVC + + push {r1, r2, r4, r5} + + /* Prepare user mode entry via eret_to_user_mode */ + ldr lr, [r0, #THREAD_CTX_REGS_PC] + ldr r4, [r0, #THREAD_CTX_REGS_CPSR] + msr spsr_fsxc, r4 + + ldm r0, {r0-r12} + + b eret_to_user_mode +END_FUNC __thread_enter_user_mode + +/* + * void thread_unwind_user_mode(uint32_t ret, uint32_t exit_status0, + * uint32_t exit_status1); + * See description in thread.h + */ +FUNC thread_unwind_user_mode , : +UNWIND( .cantunwind) + /* Match push {r1, r2, r4, r5} in thread_enter_user_mode() */ + pop {r4-r7} + str r1, [r4] + str r2, [r5] + + /* Restore old user sp */ + cps #CPSR_MODE_SYS + mov sp, r6 + cps #CPSR_MODE_SVC + + /* Match push {r4-r12,lr} in thread_enter_user_mode() */ + pop {r4-r12,pc} +END_FUNC thread_unwind_user_mode + + .macro maybe_restore_mapping + /* + * This macro is a bit hard to read due to all the ifdefs, + * we're testing for two different configs which makes four + * different combinations. + * + * - With LPAE, and then some extra code if with + * CFG_CORE_UNMAP_CORE_AT_EL0 + * - Without LPAE, and then some extra code if with + * CFG_CORE_UNMAP_CORE_AT_EL0 + */ + + /* + * At this point we can't rely on any memory being writable + * yet, so we're using TPIDRPRW to store r0, and if with + * LPAE TPIDRURO to store r1 too. + */ + write_tpidrprw r0 +#if defined(CFG_CORE_UNMAP_CORE_AT_EL0) || defined(CFG_WITH_LPAE) + write_tpidruro r1 +#endif + +#ifdef CFG_WITH_LPAE + read_ttbr0_64bit r0, r1 + tst r1, #BIT(TTBR_ASID_SHIFT - 32) + beq 11f + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + /* + * Update the mapping to use the full kernel mode mapping. + * Since the translation table could reside above 4GB we'll + * have to use 64-bit arithmetics. + */ + subs r0, r0, #CORE_MMU_BASE_TABLE_OFFSET + sbc r1, r1, #0 +#endif + bic r1, r1, #BIT(TTBR_ASID_SHIFT - 32) + write_ttbr0_64bit r0, r1 + isb + +#else /*!CFG_WITH_LPAE*/ + read_contextidr r0 + tst r0, #1 + beq 11f + + /* Update the mapping to use the full kernel mode mapping. */ + bic r0, r0, #1 + write_contextidr r0 + isb +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + read_ttbcr r0 + bic r0, r0, #TTBCR_PD1 + write_ttbcr r0 + isb +#endif + +#endif /*!CFG_WITH_LPAE*/ + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + ldr r0, =thread_user_kcode_offset + ldr r0, [r0] + read_vbar r1 + add r1, r1, r0 + write_vbar r1 + isb + + 11: /* + * The PC is adjusted unconditionally to guard against the + * case there was an FIQ just before we did the "cpsid aif". + */ + ldr r0, =22f + bx r0 + 22: +#else + 11: +#endif + read_tpidrprw r0 +#if defined(CFG_CORE_UNMAP_CORE_AT_EL0) || defined(CFG_WITH_LPAE) + read_tpidruro r1 +#endif + .endm + +/* The handler of native interrupt. */ +.macro native_intr_handler mode:req + cpsid aif + maybe_restore_mapping + + /* + * FIQ and IRQ have a +4 offset for lr compared to preferred return + * address + */ + sub lr, lr, #4 + + /* + * We're always saving {r0-r3}. In IRQ mode we're saving r12 also. + * In FIQ mode we're saving the banked fiq registers {r8-r12} FIQ + * because the secure monitor doesn't save those. The treatment of + * the banked fiq registers is somewhat analogous to the lazy save + * of VFP registers. + */ + .ifc \mode\(),fiq + push {r0-r3, r8-r12, lr} + .else + push {r0-r3, r12, lr} + .endif + + bl thread_check_canaries + bl interrupt_main_handler + + mrs r0, spsr + cmp_spsr_user_mode r0 + + .ifc \mode\(),fiq + pop {r0-r3, r8-r12, lr} + .else + pop {r0-r3, r12, lr} + .endif + + movsne pc, lr + b eret_to_user_mode +.endm + +/* The handler of foreign interrupt. */ +.macro foreign_intr_handler mode:req + cpsid aif + maybe_restore_mapping + + sub lr, lr, #4 + push {r12} + + .ifc \mode\(),fiq + /* + * If a foreign (non-secure) interrupt is received as a FIQ we need + * to check that we're in a saveable state or if we need to mask + * the interrupt to be handled later. + * + * The window when this is needed is quite narrow, it's between + * entering the exception vector and until the "cpsid" instruction + * of the handler has been executed. + * + * Currently we can save the state properly if the FIQ is received + * while in user or svc (kernel) mode. + * + * If we're returning to abort, undef or irq mode we're returning + * with the mapping restored. This is OK since before the handler + * we're returning to eventually returns to user mode the reduced + * mapping will be restored. + */ + mrs r12, spsr + and r12, r12, #ARM32_CPSR_MODE_MASK + cmp r12, #ARM32_CPSR_MODE_USR + cmpne r12, #ARM32_CPSR_MODE_SVC + beq 1f + mrs r12, spsr + orr r12, r12, #ARM32_CPSR_F + msr spsr_fsxc, r12 + pop {r12} + movs pc, lr +1: + .endif + + push {lr} + + .ifc \mode\(),fiq + bl thread_save_state_fiq + .else + bl thread_save_state + .endif + +#ifdef CFG_CORE_WORKAROUND_NSITR_CACHE_PRIME + /* + * Prevent leaking information about which entries has been used in + * cache. We're relying on the secure monitor/dispatcher to take + * care of the BTB. + */ + mov r0, #DCACHE_OP_CLEAN_INV + bl dcache_op_louis + write_iciallu +#endif + + /* + * Use SP_abt to update core local flags. + * flags = (flags << THREAD_CLF_SAVED_SHIFT) | THREAD_CLF_TMP; + */ + cps #CPSR_MODE_ABT + ldr r1, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsl r1, r1, #THREAD_CLF_SAVED_SHIFT + orr r1, r1, #THREAD_CLF_TMP + str r1, [sp, #THREAD_CORE_LOCAL_FLAGS] + .ifc \mode\(),fiq + cps #CPSR_MODE_FIQ + .else + cps #CPSR_MODE_IRQ + .endif + + mov r0, #THREAD_FLAGS_EXIT_ON_FOREIGN_INTR + mrs r1, spsr + pop {r2} + pop {r12} + blx thread_state_suspend + + /* + * Switch to SVC mode and copy current stack pointer as it already + * is the tmp stack. + */ + mov r1, sp + cps #CPSR_MODE_SVC + mov sp, r1 + + /* Passing thread index in r0 */ + b thread_foreign_intr_exit +.endm + +FUNC thread_excp_vect , :, align=32 +UNWIND( .cantunwind) + b . /* Reset */ + b __thread_und_handler /* Undefined instruction */ + b __thread_svc_handler /* System call */ + b __thread_pabort_handler /* Prefetch abort */ + b __thread_dabort_handler /* Data abort */ + b . /* Reserved */ + b __thread_irq_handler /* IRQ */ + b __thread_fiq_handler /* FIQ */ +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC + .macro vector_prologue_spectre + /* + * This depends on SP being 8 byte aligned, that is, the + * lowest three bits in SP are zero. + * + * To avoid unexpected speculation we need to invalidate + * the branch predictor before we do the first branch. It + * doesn't matter if it's a conditional or an unconditional + * branch speculation can still occur. + * + * The idea is to form a specific bit pattern in the lowest + * three bits of SP depending on which entry in the vector + * we enter via. This is done by adding 1 to SP in each + * entry but the last. + */ + add sp, sp, #1 /* 7:111 Reset */ + add sp, sp, #1 /* 6:110 Undefined instruction */ + add sp, sp, #1 /* 5:101 Secure monitor call */ + add sp, sp, #1 /* 4:100 Prefetch abort */ + add sp, sp, #1 /* 3:011 Data abort */ + add sp, sp, #1 /* 2:010 Reserved */ + add sp, sp, #1 /* 1:001 IRQ */ + cpsid aif /* 0:000 FIQ */ + .endm + + .balign 32 + .global thread_excp_vect_wa_a15_spectre_v2 +thread_excp_vect_wa_a15_spectre_v2: + vector_prologue_spectre + write_tpidrprw r0 + mrs r0, spsr + cmp_spsr_user_mode r0 + bne 1f + /* + * Invalidate the branch predictor for the current processor. + * For Cortex-A8 ACTLR[6] has to be set to 1 for BPIALL to be + * effective. + * Note that the BPIALL instruction is not effective in + * invalidating the branch predictor on Cortex-A15. For that CPU, + * set ACTLR[0] to 1 during early processor initialisation, and + * invalidate the branch predictor by performing an ICIALLU + * instruction. See also: + * https://github.com/ARM-software/arm-trusted-firmware/wiki/Arm-Trusted-Firmware-Security-Advisory-TFV-6#variant-2-cve-2017-5715 + */ + write_iciallu + isb + b 1f + + .balign 32 + .global thread_excp_vect_wa_spectre_v2 +thread_excp_vect_wa_spectre_v2: + vector_prologue_spectre + write_tpidrprw r0 + mrs r0, spsr + cmp_spsr_user_mode r0 + bne 1f + /* Invalidate the branch predictor for the current processor. */ + write_bpiall + isb + +1: and r0, sp, #(BIT(0) | BIT(1) | BIT(2)) + bic sp, sp, #(BIT(0) | BIT(1) | BIT(2)) + add pc, pc, r0, LSL #3 + nop + + read_tpidrprw r0 + b __thread_fiq_handler /* FIQ */ + read_tpidrprw r0 + b __thread_irq_handler /* IRQ */ + read_tpidrprw r0 + b . /* Reserved */ + read_tpidrprw r0 + b __thread_dabort_handler /* Data abort */ + read_tpidrprw r0 + b __thread_pabort_handler /* Prefetch abort */ + read_tpidrprw r0 + b __thread_svc_handler /* System call */ + read_tpidrprw r0 + b __thread_und_handler /* Undefined instruction */ + read_tpidrprw r0 + b . /* Reset */ +#endif /*CFG_CORE_WORKAROUND_SPECTRE_BP_SEC*/ + +__thread_und_handler: + cpsid aif + maybe_restore_mapping + strd r0, r1, [sp, #THREAD_CORE_LOCAL_R0] + mrs r1, spsr + tst r1, #CPSR_T + subne lr, lr, #2 + subeq lr, lr, #4 + mov r0, #ABORT_TYPE_UNDEF + b __thread_abort_common + +__thread_dabort_handler: + cpsid aif + maybe_restore_mapping + strd r0, r1, [sp, #THREAD_CORE_LOCAL_R0] + sub lr, lr, #8 + mov r0, #ABORT_TYPE_DATA + b __thread_abort_common + +__thread_pabort_handler: + cpsid aif + maybe_restore_mapping + strd r0, r1, [sp, #THREAD_CORE_LOCAL_R0] + sub lr, lr, #4 + mov r0, #ABORT_TYPE_PREFETCH + +__thread_abort_common: + /* + * At this label: + * cpsr is in mode undef or abort + * sp is still pointing to struct thread_core_local belonging to + * this core. + * {r0, r1} are saved in struct thread_core_local pointed to by sp + * {r2-r11, ip} are untouched. + * r0 holds the first argument for abort_handler() + */ + + /* + * Update core local flags. + * flags = (flags << THREAD_CLF_SAVED_SHIFT) | THREAD_CLF_ABORT; + */ + ldr r1, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsl r1, r1, #THREAD_CLF_SAVED_SHIFT + orr r1, r1, #THREAD_CLF_ABORT + + /* + * Select stack and update flags accordingly + * + * Normal case: + * If the abort stack is unused select that. + * + * Fatal error handling: + * If we're already using the abort stack as noted by bit + * (THREAD_CLF_SAVED_SHIFT + THREAD_CLF_ABORT_SHIFT) in the flags + * field we're selecting the temporary stack instead to be able to + * make a stack trace of the abort in abort mode. + * + * r1 is initialized as a temporary stack pointer until we've + * switched to system mode. + */ + tst r1, #(THREAD_CLF_ABORT << THREAD_CLF_SAVED_SHIFT) + orrne r1, r1, #THREAD_CLF_TMP /* flags |= THREAD_CLF_TMP; */ + str r1, [sp, #THREAD_CORE_LOCAL_FLAGS] + ldrne r1, [sp, #THREAD_CORE_LOCAL_TMP_STACK_VA_END] + ldreq r1, [sp, #THREAD_CORE_LOCAL_ABT_STACK_VA_END] + + /* + * Store registers on stack fitting struct thread_abort_regs + * start from the end of the struct + * {r2-r11, ip} + * Load content of previously saved {r0-r1} and stores + * it up to the pad field. + * After this is only {usr_sp, usr_lr} missing in the struct + */ + stmdb r1!, {r2-r11, ip} /* Push on the selected stack */ + ldrd r2, r3, [sp, #THREAD_CORE_LOCAL_R0] + /* Push the original {r0-r1} on the selected stack */ + stmdb r1!, {r2-r3} + mrs r3, spsr + /* Push {pad, spsr, elr} on the selected stack */ + stmdb r1!, {r2, r3, lr} + + cps #CPSR_MODE_SYS + str lr, [r1, #-4]! + str sp, [r1, #-4]! + mov sp, r1 + + bl abort_handler + + mov ip, sp + ldr sp, [ip], #4 + ldr lr, [ip], #4 + + /* + * Even if we entered via CPSR_MODE_UND, we are returning via + * CPSR_MODE_ABT. It doesn't matter as lr and spsr are assigned + * here. + */ + cps #CPSR_MODE_ABT + ldm ip!, {r0, r1, lr} /* r0 is pad */ + msr spsr_fsxc, r1 + + /* Update core local flags */ + ldr r0, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsr r0, r0, #THREAD_CLF_SAVED_SHIFT + str r0, [sp, #THREAD_CORE_LOCAL_FLAGS] + + cmp_spsr_user_mode r1 + ldm ip, {r0-r11, ip} + movsne pc, lr + b eret_to_user_mode + /* end thread_abort_common */ + +__thread_svc_handler: + cpsid aif + + maybe_restore_mapping + + push {r0-r7, lr} + mrs r0, spsr + push {r0} + mov r0, sp + bl thread_scall_handler + cpsid aif /* In case something was unmasked */ + pop {r0} + msr spsr_fsxc, r0 + cmp_spsr_user_mode r0 + pop {r0-r7, lr} + movsne pc, lr + b eret_to_user_mode + /* end thread_svc_handler */ + +__thread_fiq_handler: +#if defined(CFG_CORE_IRQ_IS_NATIVE_INTR) + foreign_intr_handler fiq +#else + native_intr_handler fiq +#endif + /* end thread_fiq_handler */ + +__thread_irq_handler: +#if defined(CFG_CORE_IRQ_IS_NATIVE_INTR) + native_intr_handler irq +#else + foreign_intr_handler irq +#endif + /* end thread_irq_handler */ + + /* + * Returns to user mode. + * Expects to be jumped to with lr pointing to the user space + * address to jump to and spsr holding the desired cpsr. Async + * abort, irq and fiq should be masked. + */ +eret_to_user_mode: + write_tpidrprw r0 +#if defined(CFG_CORE_UNMAP_CORE_AT_EL0) || defined(CFG_WITH_LPAE) + write_tpidruro r1 +#endif + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + ldr r0, =thread_user_kcode_offset + ldr r0, [r0] + read_vbar r1 + sub r1, r1, r0 + write_vbar r1 + isb + + /* Jump into the reduced mapping before the full mapping is removed */ + ldr r1, =1f + sub r1, r1, r0 + bx r1 +1: +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + +#ifdef CFG_WITH_LPAE + read_ttbr0_64bit r0, r1 +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + add r0, r0, #CORE_MMU_BASE_TABLE_OFFSET +#endif + /* switch to user ASID */ + orr r1, r1, #BIT(TTBR_ASID_SHIFT - 32) + write_ttbr0_64bit r0, r1 + isb +#else /*!CFG_WITH_LPAE*/ +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + read_ttbcr r0 + orr r0, r0, #TTBCR_PD1 + write_ttbcr r0 + isb +#endif + read_contextidr r0 + orr r0, r0, #BIT(0) + write_contextidr r0 + isb +#endif /*!CFG_WITH_LPAE*/ + + read_tpidrprw r0 +#if defined(CFG_CORE_UNMAP_CORE_AT_EL0) || defined(CFG_WITH_LPAE) + read_tpidruro r1 +#endif + + movs pc, lr + + /* + * void icache_inv_user_range(void *addr, size_t size); + * + * This function has to execute with the user space ASID active, + * this means executing with reduced mapping and the code needs + * to be located here together with the vector. + */ + .global icache_inv_user_range + .type icache_inv_user_range , %function +icache_inv_user_range: + push {r4-r7} + + /* Mask all exceptions */ + mrs r4, cpsr /* This register must be preserved */ + cpsid aif + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + ldr r2, =thread_user_kcode_offset + ldr r2, [r2] + read_vbar r5 /* This register must be preserved */ + sub r3, r5, r2 + write_vbar r3 + isb + + /* Jump into the reduced mapping before the full mapping is removed */ + ldr r3, =1f + sub r3, r3, r2 + bx r3 +1: +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + +#ifdef CFG_WITH_LPAE + read_ttbr0_64bit r6, r7 /* These registers must be preseved */ + /* switch to user ASID */ + orr r3, r7, #BIT(TTBR_ASID_SHIFT - 32) +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + add r2, r6, #CORE_MMU_BASE_TABLE_OFFSET + write_ttbr0_64bit r2, r3 +#else + write_ttbr0_64bit r6, r3 +#endif + isb +#else /*!CFG_WITH_LPAE*/ +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + read_ttbcr r6 /* This register must be preserved */ + orr r2, r6, #TTBCR_PD1 + write_ttbcr r2 + isb +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + read_contextidr r7 /* This register must be preserved */ + orr r2, r7, #BIT(0) + write_contextidr r2 + isb +#endif /*!CFG_WITH_LPAE*/ + + /* + * Do the actual icache invalidation + */ + + /* Calculate minimum icache line size, result in r2 */ + read_ctr r3 + and r3, r3, #CTR_IMINLINE_MASK + mov r2, #CTR_WORD_SIZE + lsl r2, r2, r3 + + add r1, r0, r1 + sub r3, r2, #1 + bic r0, r0, r3 +1: + write_icimvau r0 + add r0, r0, r2 + cmp r0, r1 + blo 1b + + /* Invalidate entire branch predictor array inner shareable */ + write_bpiallis + + dsb ishst + isb + +#ifdef CFG_WITH_LPAE + write_ttbr0_64bit r6, r7 + isb +#else /*!CFG_WITH_LPAE*/ + write_contextidr r7 + isb +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + write_ttbcr r6 + isb +#endif +#endif /*!CFG_WITH_LPAE*/ + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + write_vbar r5 + isb + /* + * The PC is adjusted unconditionally to guard against the + * case there was an FIQ just before we did the "cpsid aif". + */ + ldr r0, =1f + bx r0 +1: +#endif + + msr cpsr_fsxc, r4 /* Restore exceptions */ + pop {r4-r7} + bx lr /* End of icache_inv_user_range() */ + + /* + * Make sure that literals are placed before the + * thread_excp_vect_end label. + */ + .pool + .global thread_excp_vect_end +thread_excp_vect_end: +END_FUNC thread_excp_vect diff --git a/optee_os/core/arch/arm/kernel/thread_a64.S b/optee_os/core/arch/arm/kernel/thread_a64.S new file mode 100644 index 0000000..256195e --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_a64.S @@ -0,0 +1,1336 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + .macro get_thread_ctx core_local, res, tmp0, tmp1 + ldrh w\tmp0, [\core_local, \ + #THREAD_CORE_LOCAL_CURR_THREAD] + adr_l x\res, threads + mov x\tmp1, #THREAD_CTX_SIZE + madd x\res, x\tmp0, x\tmp1, x\res + .endm + + .macro return_from_exception + eret + /* Guard against speculation past ERET */ + dsb nsh + isb + .endm + + .macro b_if_spsr_is_el0 reg, label + tbnz \reg, #(SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT), \label + tst \reg, #(SPSR_64_MODE_EL_MASK << SPSR_64_MODE_EL_SHIFT) + b.eq \label + .endm + + .macro pauth_el0_to_el1 reg + /* + * If pauth is only enabled in one of core or TA (xor) we + * need to update sctlr. + */ +#if (defined(CFG_TA_PAUTH) && !defined(CFG_CORE_PAUTH)) || \ + (!defined(CFG_TA_PAUTH) && defined(CFG_CORE_PAUTH)) + mrs \reg, sctlr_el1 + /* Flip the SCTLR_ENIA bit */ + eor \reg, \reg, #SCTLR_ENIA + msr sctlr_el1, \reg +#endif + .endm + + .macro pauth_el1_to_el0 reg + /* + * If pauth is only enabled in one of core or TA (xor) we + * need to update sctlr. + */ +#if (defined(CFG_TA_PAUTH) && !defined(CFG_CORE_PAUTH)) || \ + (!defined(CFG_TA_PAUTH) && defined(CFG_CORE_PAUTH)) + mrs \reg, sctlr_el1 + /* Flip the SCTLR_ENIA bit */ + eor \reg, \reg, #SCTLR_ENIA + msr sctlr_el1, \reg +#endif + .endm + +/* void thread_resume(struct thread_ctx_regs *regs) */ +FUNC thread_resume , : + load_xregs x0, THREAD_CTX_REGS_SP, 1, 3 + load_xregs x0, THREAD_CTX_REGS_X4, 4, 30 + mov sp, x1 + msr elr_el1, x2 + msr spsr_el1, x3 + ldr x1, [x0, THREAD_CTX_REGS_TPIDR_EL0] + msr tpidr_el0, x1 + +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + load_xregs x0, THREAD_CTX_REGS_APIAKEY_HI, 1, 2 + write_apiakeyhi x1 + write_apiakeylo x2 +#endif + b_if_spsr_is_el0 w3, 1f + +#if defined(CFG_CORE_PAUTH) || defined(CFG_TA_PAUTH) + /* SCTLR or the APIA key has changed */ + isb +#endif + load_xregs x0, THREAD_CTX_REGS_X1, 1, 3 + ldr x0, [x0, THREAD_CTX_REGS_X0] + return_from_exception + +1: + load_xregs x0, THREAD_CTX_REGS_X1, 1, 3 + ldr x0, [x0, THREAD_CTX_REGS_X0] + + msr spsel, #1 + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 1 + b eret_to_el0 +END_FUNC thread_resume + +#ifdef CFG_CORE_SEL2_SPMC +FUNC thread_hvc , : + hvc #0 + ret +END_FUNC thread_hvc +#endif + +FUNC thread_smc , : + smc #0 + ret +END_FUNC thread_smc + +/* void thread_smccc(struct thread_smc_args *arg_res) */ +FUNC thread_smccc , : + push x0, xzr + mov x8, x0 + load_xregs x8, 0, 0, 7 +#ifdef CFG_CORE_SEL2_SPMC + hvc #0 +#else + smc #0 +#endif + pop x8, xzr + store_xregs x8, 0, 0, 7 + ret +END_FUNC thread_smccc + +FUNC thread_init_vbar , : + msr vbar_el1, x0 + ret +END_FUNC thread_init_vbar +DECLARE_KEEP_PAGER thread_init_vbar + +/* + * uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, + * uint32_t *exit_status0, + * uint32_t *exit_status1); + * + * This function depends on being called with exceptions masked. + */ +FUNC __thread_enter_user_mode , : + /* + * Create the and fill in the struct thread_user_mode_rec + */ + sub sp, sp, #THREAD_USER_MODE_REC_SIZE + store_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, 0, 2 + store_xregs sp, THREAD_USER_MODE_REC_X19, 19, 30 + + /* + * Save kern sp in x19 + * Switch to SP_EL1 + */ + mov x19, sp + msr spsel, #1 + + /* + * Save the kernel stack pointer in the thread context + */ + /* get pointer to current thread context */ + get_thread_ctx sp, 21, 20, 22 + /* + * Save kernel stack pointer to ensure that el0_svc() uses + * correct stack pointer + */ + str x19, [x21, #THREAD_CTX_KERN_SP] + + /* + * Initialize SPSR, ELR_EL1, and SP_EL0 to enter user mode + */ + load_xregs x0, THREAD_CTX_REGS_SP, 1, 3 + msr sp_el0, x1 + msr elr_el1, x2 + msr spsr_el1, x3 + +#ifdef CFG_TA_PAUTH + /* Load APIAKEY */ + load_xregs x0, THREAD_CTX_REGS_APIAKEY_HI, 1, 2 + write_apiakeyhi x1 + write_apiakeylo x2 +#endif + + /* + * Save the values for x0 and x1 in struct thread_core_local to be + * restored later just before the eret. + */ + load_xregs x0, THREAD_CTX_REGS_X0, 1, 2 + store_xregs sp, THREAD_CORE_LOCAL_X0, 1, 2 + + /* Load the rest of the general purpose registers */ + load_xregs x0, THREAD_CTX_REGS_X2, 2, 30 + + /* Jump into user mode */ + b eret_to_el0 +END_FUNC __thread_enter_user_mode +DECLARE_KEEP_PAGER __thread_enter_user_mode + +/* + * void thread_unwind_user_mode(uint32_t ret, uint32_t exit_status0, + * uint32_t exit_status1); + * See description in thread.h + */ +FUNC thread_unwind_user_mode , : + /* Store the exit status */ + load_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, 3, 5 + str w1, [x4] + str w2, [x5] + /* Save x19..x30 */ + store_xregs x3, THREAD_CTX_REGS_X19, 19, 30 + /* Restore x19..x30 */ + load_xregs sp, THREAD_USER_MODE_REC_X19, 19, 30 + add sp, sp, #THREAD_USER_MODE_REC_SIZE + /* Return from the call of thread_enter_user_mode() */ + ret +END_FUNC thread_unwind_user_mode + + /* + * This macro verifies that the a given vector doesn't exceed the + * architectural limit of 32 instructions. This is meant to be placed + * immedately after the last instruction in the vector. It takes the + * vector entry as the parameter + */ + .macro check_vector_size since + .if (. - \since) > (32 * 4) + .error "Vector exceeds 32 instructions" + .endif + .endm + + .macro restore_mapping +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + /* Temporarily save x0, x1 */ + msr tpidr_el1, x0 + msr tpidrro_el0, x1 + + /* Update the mapping to use the full kernel mapping */ + mrs x0, ttbr0_el1 + sub_imm x0, __CORE_MMU_BASE_TABLE_OFFSET + /* switch to kernel mode ASID */ + bic x0, x0, #BIT(TTBR_ASID_SHIFT) + msr ttbr0_el1, x0 + isb + + /* Jump into the full mapping and continue execution */ + adr x0, 1f + ldr x1, [sp, #THREAD_CORE_LOCAL_KCODE_OFFSET] + add x0, x0, x1 + br x0 + 1: +BTI( bti j) + /* Point to the vector into the full mapping */ + adr_l x0, thread_user_kcode_offset + ldr x0, [x0] + mrs x1, vbar_el1 + add x1, x1, x0 + msr vbar_el1, x1 + isb + +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC + /* + * Update the SP with thread_user_kdata_sp_offset as + * described in init_user_kcode(). + */ + adr_l x0, thread_user_kdata_sp_offset + ldr x0, [x0] + add sp, sp, x0 +#endif + + /* Restore x0, x1 */ + mrs x0, tpidr_el1 + mrs x1, tpidrro_el0 + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 +#else + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + mrs x0, ttbr0_el1 + /* switch to kernel mode ASID */ + bic x0, x0, #BIT(TTBR_ASID_SHIFT) + msr ttbr0_el1, x0 + isb +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + .endm + +#define INV_INSN 0 +FUNC thread_excp_vect , : , default, 2048, nobti + /* ----------------------------------------------------- + * EL1 with SP0 : 0x0 - 0x180 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +el1_sync_sp0: + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + b el1_sync_abort + check_vector_size el1_sync_sp0 + + .balign 128, INV_INSN +el1_irq_sp0: + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + b elx_irq + check_vector_size el1_irq_sp0 + + .balign 128, INV_INSN +el1_fiq_sp0: + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + b elx_fiq + check_vector_size el1_fiq_sp0 + + .balign 128, INV_INSN +el1_serror_sp0: + b el1_serror_sp0 + check_vector_size el1_serror_sp0 + + /* ----------------------------------------------------- + * Current EL with SP1: 0x200 - 0x380 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +el1_sync_sp1: + b el1_sync_sp1 + check_vector_size el1_sync_sp1 + + .balign 128, INV_INSN +el1_irq_sp1: + b el1_irq_sp1 + check_vector_size el1_irq_sp1 + + .balign 128, INV_INSN +el1_fiq_sp1: + b el1_fiq_sp1 + check_vector_size el1_fiq_sp1 + + .balign 128, INV_INSN +el1_serror_sp1: + b el1_serror_sp1 + check_vector_size el1_serror_sp1 + + /* ----------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x580 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +el0_sync_a64: + restore_mapping + /* PAuth will be disabled later else check_vector_size will fail */ + + b el0_sync_a64_finish + check_vector_size el0_sync_a64 + + .balign 128, INV_INSN +el0_irq_a64: + restore_mapping + pauth_el0_to_el1 x1 + + b elx_irq + check_vector_size el0_irq_a64 + + .balign 128, INV_INSN +el0_fiq_a64: + restore_mapping + pauth_el0_to_el1 x1 + + b elx_fiq + check_vector_size el0_fiq_a64 + + .balign 128, INV_INSN +el0_serror_a64: + b el0_serror_a64 + check_vector_size el0_serror_a64 + + /* ----------------------------------------------------- + * Lower EL using AArch32 : 0x0 - 0x180 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +el0_sync_a32: + restore_mapping + + b el0_sync_a32_finish + check_vector_size el0_sync_a32 + + .balign 128, INV_INSN +el0_irq_a32: + restore_mapping + + b elx_irq + check_vector_size el0_irq_a32 + + .balign 128, INV_INSN +el0_fiq_a32: + restore_mapping + + b elx_fiq + check_vector_size el0_fiq_a32 + + .balign 128, INV_INSN +el0_serror_a32: + b el0_serror_a32 + check_vector_size el0_serror_a32 + +#if defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) + .macro invalidate_branch_predictor + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + mov_imm x0, SMCCC_ARCH_WORKAROUND_1 + smc #0 + load_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + .endm + + .balign 2048, INV_INSN + .global thread_excp_vect_wa_spectre_v2 +thread_excp_vect_wa_spectre_v2: + /* ----------------------------------------------------- + * EL1 with SP0 : 0x0 - 0x180 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_v2_el1_sync_sp0: + b el1_sync_sp0 + check_vector_size wa_spectre_v2_el1_sync_sp0 + + .balign 128, INV_INSN +wa_spectre_v2_el1_irq_sp0: + b el1_irq_sp0 + check_vector_size wa_spectre_v2_el1_irq_sp0 + + .balign 128, INV_INSN +wa_spectre_v2_el1_fiq_sp0: + b el1_fiq_sp0 + check_vector_size wa_spectre_v2_el1_fiq_sp0 + + .balign 128, INV_INSN +wa_spectre_v2_el1_serror_sp0: + b el1_serror_sp0 + check_vector_size wa_spectre_v2_el1_serror_sp0 + + /* ----------------------------------------------------- + * Current EL with SP1: 0x200 - 0x380 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_v2_el1_sync_sp1: + b wa_spectre_v2_el1_sync_sp1 + check_vector_size wa_spectre_v2_el1_sync_sp1 + + .balign 128, INV_INSN +wa_spectre_v2_el1_irq_sp1: + b wa_spectre_v2_el1_irq_sp1 + check_vector_size wa_spectre_v2_el1_irq_sp1 + + .balign 128, INV_INSN +wa_spectre_v2_el1_fiq_sp1: + b wa_spectre_v2_el1_fiq_sp1 + check_vector_size wa_spectre_v2_el1_fiq_sp1 + + .balign 128, INV_INSN +wa_spectre_v2_el1_serror_sp1: + b wa_spectre_v2_el1_serror_sp1 + check_vector_size wa_spectre_v2_el1_serror_sp1 + + /* ----------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x580 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_v2_el0_sync_a64: + invalidate_branch_predictor + b el0_sync_a64 + check_vector_size wa_spectre_v2_el0_sync_a64 + + .balign 128, INV_INSN +wa_spectre_v2_el0_irq_a64: + invalidate_branch_predictor + b el0_irq_a64 + check_vector_size wa_spectre_v2_el0_irq_a64 + + .balign 128, INV_INSN +wa_spectre_v2_el0_fiq_a64: + invalidate_branch_predictor + b el0_fiq_a64 + check_vector_size wa_spectre_v2_el0_fiq_a64 + + .balign 128, INV_INSN +wa_spectre_v2_el0_serror_a64: + b wa_spectre_v2_el0_serror_a64 + check_vector_size wa_spectre_v2_el0_serror_a64 + + /* ----------------------------------------------------- + * Lower EL using AArch32 : 0x0 - 0x180 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_v2_el0_sync_a32: + invalidate_branch_predictor + b el0_sync_a32 + check_vector_size wa_spectre_v2_el0_sync_a32 + + .balign 128, INV_INSN +wa_spectre_v2_el0_irq_a32: + invalidate_branch_predictor + b el0_irq_a32 + check_vector_size wa_spectre_v2_el0_irq_a32 + + .balign 128, INV_INSN +wa_spectre_v2_el0_fiq_a32: + invalidate_branch_predictor + b el0_fiq_a32 + check_vector_size wa_spectre_v2_el0_fiq_a32 + + .balign 128, INV_INSN +wa_spectre_v2_el0_serror_a32: + b wa_spectre_v2_el0_serror_a32 + check_vector_size wa_spectre_v2_el0_serror_a32 + + .macro discard_branch_history + str x0, [sp, #THREAD_CORE_LOCAL_X0] + ldrb w0, [sp, #THREAD_CORE_LOCAL_BHB_LOOP_COUNT] + 1: b 2f + 2: subs w0, w0, #1 + bne 1b + dsb sy + isb + ldr x0, [sp, #THREAD_CORE_LOCAL_X0] + .endm + + .balign 2048, INV_INSN + .global thread_excp_vect_wa_spectre_bhb +thread_excp_vect_wa_spectre_bhb: + /* ----------------------------------------------------- + * EL1 with SP0 : 0x0 - 0x180 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_bhb_el1_sync_sp0: + b el1_sync_sp0 + check_vector_size wa_spectre_bhb_el1_sync_sp0 + + .balign 128, INV_INSN +wa_spectre_bhb_el1_irq_sp0: + b el1_irq_sp0 + check_vector_size wa_spectre_bhb_el1_irq_sp0 + + .balign 128, INV_INSN +wa_spectre_bhb_el1_fiq_sp0: + b el1_fiq_sp0 + check_vector_size wa_spectre_bhb_el1_fiq_sp0 + + .balign 128, INV_INSN +wa_spectre_bhb_el1_serror_sp0: + b el1_serror_sp0 + check_vector_size wa_spectre_bhb_el1_serror_sp0 + + /* ----------------------------------------------------- + * Current EL with SP1: 0x200 - 0x380 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_bhb_el1_sync_sp1: + b wa_spectre_bhb_el1_sync_sp1 + check_vector_size wa_spectre_bhb_el1_sync_sp1 + + .balign 128, INV_INSN +wa_spectre_bhb_el1_irq_sp1: + b wa_spectre_bhb_el1_irq_sp1 + check_vector_size wa_spectre_bhb_el1_irq_sp1 + + .balign 128, INV_INSN +wa_spectre_bhb_el1_fiq_sp1: + b wa_spectre_bhb_el1_fiq_sp1 + check_vector_size wa_spectre_bhb_el1_fiq_sp1 + + .balign 128, INV_INSN +wa_spectre_bhb_el1_serror_sp1: + b wa_spectre_bhb_el1_serror_sp1 + check_vector_size wa_spectre_bhb_el1_serror_sp1 + + /* ----------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x580 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_bhb_el0_sync_a64: + discard_branch_history + b el0_sync_a64 + check_vector_size wa_spectre_bhb_el0_sync_a64 + + .balign 128, INV_INSN +wa_spectre_bhb_el0_irq_a64: + discard_branch_history + b el0_irq_a64 + check_vector_size wa_spectre_bhb_el0_irq_a64 + + .balign 128, INV_INSN +wa_spectre_bhb_el0_fiq_a64: + discard_branch_history + b el0_fiq_a64 + check_vector_size wa_spectre_bhb_el0_fiq_a64 + + .balign 128, INV_INSN +wa_spectre_bhb_el0_serror_a64: + b wa_spectre_bhb_el0_serror_a64 + check_vector_size wa_spectre_bhb_el0_serror_a64 + + /* ----------------------------------------------------- + * Lower EL using AArch32 : 0x0 - 0x180 + * ----------------------------------------------------- + */ + .balign 128, INV_INSN +wa_spectre_bhb_el0_sync_a32: + discard_branch_history + b el0_sync_a32 + check_vector_size wa_spectre_bhb_el0_sync_a32 + + .balign 128, INV_INSN +wa_spectre_bhb_el0_irq_a32: + discard_branch_history + b el0_irq_a32 + check_vector_size wa_spectre_bhb_el0_irq_a32 + + .balign 128, INV_INSN +wa_spectre_bhb_el0_fiq_a32: + discard_branch_history + b el0_fiq_a32 + check_vector_size wa_spectre_bhb_el0_fiq_a32 + + .balign 128, INV_INSN +wa_spectre_bhb_el0_serror_a32: + b wa_spectre_bhb_el0_serror_a32 + check_vector_size wa_spectre_bhb_el0_serror_a32 +#endif /*CFG_CORE_WORKAROUND_SPECTRE_BP_SEC*/ + +/* + * We're keeping this code in the same section as the vector to make sure + * that it's always available. + */ +eret_to_el0: + pauth_el1_to_el0 x1 + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + /* Point to the vector into the reduced mapping */ + adr_l x0, thread_user_kcode_offset + ldr x0, [x0] + mrs x1, vbar_el1 + sub x1, x1, x0 + msr vbar_el1, x1 + isb + +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC + /* Store the SP offset in tpidr_el1 to be used below to update SP */ + adr_l x1, thread_user_kdata_sp_offset + ldr x1, [x1] + msr tpidr_el1, x1 +#endif + + /* Jump into the reduced mapping and continue execution */ + adr_l x1, 1f + sub x1, x1, x0 + br x1 +1: +BTI( bti j) + load_xregs sp, THREAD_CORE_LOCAL_X0, 0, 1 + msr tpidrro_el0, x0 + + /* Update the mapping to exclude the full kernel mapping */ + mrs x0, ttbr0_el1 + add_imm x0, __CORE_MMU_BASE_TABLE_OFFSET + orr x0, x0, #BIT(TTBR_ASID_SHIFT) /* switch to user mode ASID */ + msr ttbr0_el1, x0 + isb + +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC + /* + * Update the SP with thread_user_kdata_sp_offset as described in + * init_user_kcode(). + */ + mrs x0, tpidr_el1 + sub sp, sp, x0 +#endif + + mrs x0, tpidrro_el0 +#else + mrs x0, ttbr0_el1 + orr x0, x0, #BIT(TTBR_ASID_SHIFT) /* switch to user mode ASID */ + msr ttbr0_el1, x0 + isb + load_xregs sp, THREAD_CORE_LOCAL_X0, 0, 1 +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + + return_from_exception + +el0_sync_a64_finish: + mrs x2, esr_el1 + mrs x3, sp_el0 + lsr x2, x2, #ESR_EC_SHIFT + cmp x2, #ESR_EC_AARCH64_SVC + b.eq el0_svc + b el0_sync_abort + +el0_sync_a32_finish: + mrs x2, esr_el1 + mrs x3, sp_el0 + lsr x2, x2, #ESR_EC_SHIFT + cmp x2, #ESR_EC_AARCH32_SVC + b.eq el0_svc + b el0_sync_abort + + /* + * void icache_inv_user_range(void *addr, size_t size); + * + * This function has to execute with the user space ASID active, + * this means executing with reduced mapping and the code needs + * to be located here together with the vector. + */ + .global icache_inv_user_range + .type icache_inv_user_range , %function +icache_inv_user_range: + /* Mask all exceptions */ + mrs x6, daif /* this register must be preserved */ + msr daifset, #DAIFBIT_ALL + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + /* Point to the vector into the reduced mapping */ + adr_l x7, thread_user_kcode_offset + ldr x7, [x7] /* this register must be preserved */ + mrs x4, vbar_el1 /* this register must be preserved */ + sub x3, x4, x7 + msr vbar_el1, x3 + isb + + /* Jump into the reduced mapping and continue execution */ + adr x3, 1f + sub x3, x3, x7 + br x3 +1: +BTI( bti j) + /* Update the mapping to exclude the full kernel mapping */ + mrs x5, ttbr0_el1 /* this register must be preserved */ + orr x2, x5, #BIT(TTBR_ASID_SHIFT) /* switch to user mode ASID */ + add_imm x2, __CORE_MMU_BASE_TABLE_OFFSET + msr ttbr0_el1, x2 + isb + +#else + mrs x5, ttbr0_el1 /* this register must be preserved */ + orr x2, x5, #BIT(TTBR_ASID_SHIFT) /* switch to user mode ASID */ + msr ttbr0_el1, x2 + isb +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + + /* + * Do the actual icache invalidation + */ + + /* Calculate minimum icache line size, result in x2 */ + mrs x3, ctr_el0 + and x3, x3, #CTR_IMINLINE_MASK + mov x2, #CTR_WORD_SIZE + lsl x2, x2, x3 + + add x1, x0, x1 + sub x3, x2, #1 + bic x0, x0, x3 +1: + ic ivau, x0 + add x0, x0, x2 + cmp x0, x1 + b.lo 1b + dsb ish + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + /* Update the mapping to use the full kernel mapping and ASID */ + msr ttbr0_el1, x5 + isb + + /* Jump into the full mapping and continue execution */ + adr x0, 1f + add x0, x0, x7 + br x0 +1: +BTI( bti j) + /* Point to the vector into the full mapping */ + msr vbar_el1, x4 + isb +#else + /* switch to kernel mode ASID */ + msr ttbr0_el1, x5 + isb +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + + msr daif, x6 /* restore exceptions */ + ret /* End of icache_inv_user_range() */ + + /* + * Make sure that literals are placed before the + * thread_excp_vect_end label. + */ + .pool + .global thread_excp_vect_end +thread_excp_vect_end: +END_FUNC thread_excp_vect + +LOCAL_FUNC el0_svc , : + pauth_el0_to_el1 x1 + /* get pointer to current thread context in x0 */ + get_thread_ctx sp, 0, 1, 2 + mrs x1, tpidr_el0 + str x1, [x0, #THREAD_CTX_REGS_TPIDR_EL0] + /* load saved kernel sp */ + ldr x3, [x0, #THREAD_CTX_KERN_SP] + /* Keep pointer to initial recod in x1 */ + mov x1, sp + /* Switch to SP_EL0 and restore kernel sp */ + msr spsel, #0 + mov x2, sp /* Save SP_EL0 */ + mov sp, x3 + + /* Make room for struct thread_scall_regs */ + sub sp, sp, #THREAD_SCALL_REG_SIZE + stp x30, x2, [sp, #THREAD_SCALL_REG_X30] + +#ifdef CFG_TA_PAUTH + /* Save APIAKEY */ + read_apiakeyhi x2 + read_apiakeylo x3 + stp x2, x3, [sp, #THREAD_SCALL_REG_APIAKEY_HI] +#endif + +#ifdef CFG_CORE_PAUTH + ldp x2, x3, [x0, #THREAD_CTX_KEYS] + write_apiakeyhi x2 + write_apiakeylo x3 +#endif +#if defined(CFG_CORE_PAUTH) || defined(CFG_TA_PAUTH) + /* SCTLR or the APIA key has changed */ + isb +#endif + + /* Restore x0-x3 */ + ldp x2, x3, [x1, #THREAD_CORE_LOCAL_X2] + ldp x0, x1, [x1, #THREAD_CORE_LOCAL_X0] + + /* Prepare the argument for the handler */ + store_xregs sp, THREAD_SCALL_REG_X0, 0, 14 + mrs x0, elr_el1 + mrs x1, spsr_el1 + store_xregs sp, THREAD_SCALL_REG_ELR, 0, 1 + + mov x0, sp + + /* + * Unmask native interrupts, Serror, and debug exceptions since we have + * nothing left in sp_el1. Note that the SVC handler is excepted to + * re-enable foreign interrupts by itself. + */ +#if defined(CFG_CORE_IRQ_IS_NATIVE_INTR) + msr daifclr, #(DAIFBIT_IRQ | DAIFBIT_ABT | DAIFBIT_DBG) +#else + msr daifclr, #(DAIFBIT_FIQ | DAIFBIT_ABT | DAIFBIT_DBG) +#endif + + /* Call the handler */ + bl thread_scall_handler + + /* Mask all maskable exceptions since we're switching back to sp_el1 */ + msr daifset, #DAIFBIT_ALL + + /* + * Save kernel sp we'll had at the beginning of this function. + * This is when this TA has called another TA because + * __thread_enter_user_mode() also saves the stack pointer in this + * field. + */ + msr spsel, #1 + get_thread_ctx sp, 0, 1, 2 + msr spsel, #0 + add x1, sp, #THREAD_SCALL_REG_SIZE + str x1, [x0, #THREAD_CTX_KERN_SP] + + /* Restore registers to the required state and return*/ + ldr x1, [x0, #THREAD_CTX_REGS_TPIDR_EL0] + msr tpidr_el0, x1 + load_xregs sp, THREAD_SCALL_REG_ELR, 0, 1 + msr elr_el1, x0 + msr spsr_el1, x1 + load_xregs sp, THREAD_SCALL_REG_X2, 2, 14 + mov x30, sp + ldr x0, [x30, #THREAD_SCALL_REG_SP_EL0] + mov sp, x0 + b_if_spsr_is_el0 w1, 1f + ldp x0, x1, [x30, THREAD_SCALL_REG_X0] + ldr x30, [x30, #THREAD_SCALL_REG_X30] + + return_from_exception + +1: +#ifdef CFG_TA_PAUTH + /* Restore APIAKEY */ + load_xregs x30, THREAD_SCALL_REG_APIAKEY_HI, 0, 1 + write_apiakeyhi x0 + write_apiakeylo x1 +#endif + + ldp x0, x1, [x30, THREAD_SCALL_REG_X0] + ldr x30, [x30, #THREAD_SCALL_REG_X30] + + msr spsel, #1 + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 1 + b eret_to_el0 +END_FUNC el0_svc + +LOCAL_FUNC el1_sync_abort , : + mov x0, sp + msr spsel, #0 + mov x3, sp /* Save original sp */ + + /* + * Update core local flags. + * flags = (flags << THREAD_CLF_SAVED_SHIFT) | THREAD_CLF_ABORT; + */ + ldr w1, [x0, #THREAD_CORE_LOCAL_FLAGS] + lsl w1, w1, #THREAD_CLF_SAVED_SHIFT + orr w1, w1, #THREAD_CLF_ABORT + tbnz w1, #(THREAD_CLF_SAVED_SHIFT + THREAD_CLF_ABORT_SHIFT), \ + .Lsel_tmp_sp + + /* Select abort stack */ + ldr x2, [x0, #THREAD_CORE_LOCAL_ABT_STACK_VA_END] + b .Lset_sp + +.Lsel_tmp_sp: + /* We have an abort while using the abort stack, select tmp stack */ + ldr x2, [x0, #THREAD_CORE_LOCAL_TMP_STACK_VA_END] + orr w1, w1, #THREAD_CLF_TMP /* flags |= THREAD_CLF_TMP; */ + +.Lset_sp: + mov sp, x2 + str w1, [x0, #THREAD_CORE_LOCAL_FLAGS] + + /* + * Save state on stack + */ + sub sp, sp, #THREAD_ABT_REGS_SIZE + mrs x2, spsr_el1 + /* Store spsr, sp_el0 */ + stp x2, x3, [sp, #THREAD_ABT_REG_SPSR] + /* Store original x0, x1 */ + ldp x2, x3, [x0, #THREAD_CORE_LOCAL_X0] + stp x2, x3, [sp, #THREAD_ABT_REG_X0] + /* Store original x2, x3 and x4 to x29 */ + ldp x2, x3, [x0, #THREAD_CORE_LOCAL_X2] + store_xregs sp, THREAD_ABT_REG_X2, 2, 29 + /* Store x30, elr_el1 */ + mrs x1, elr_el1 + stp x30, x1, [sp, #THREAD_ABT_REG_X30] + +#if defined(CFG_CORE_PAUTH) + read_apiakeyhi x2 + read_apiakeylo x3 + stp x2, x3, [sp, #THREAD_ABT_REGS_APIAKEY_HI] + ldp x2, x3, [x0, #THREAD_CORE_LOCAL_KEYS] + write_apiakeyhi x2 + write_apiakeylo x3 + isb +#endif + + /* + * Call handler + */ + mov x0, #0 + mov x1, sp + bl abort_handler + + /* + * Restore state from stack + */ + /* Load x30, elr_el1 */ + ldp x30, x0, [sp, #THREAD_ABT_REG_X30] + msr elr_el1, x0 + /* Load x0 to x29 */ + load_xregs sp, THREAD_ABT_REG_X0, 0, 29 + /* Switch to SP_EL1 */ + msr spsel, #1 + /* Save x0 to x3 in CORE_LOCAL */ + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + /* Restore spsr_el1 and sp_el0 */ + mrs x3, sp_el0 + ldp x0, x1, [x3, #THREAD_ABT_REG_SPSR] + msr spsr_el1, x0 + msr sp_el0, x1 + + /* Update core local flags */ + ldr w0, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsr w0, w0, #THREAD_CLF_SAVED_SHIFT + str w0, [sp, #THREAD_CORE_LOCAL_FLAGS] + +#if defined(CFG_CORE_PAUTH) + ldp x0, x1, [x3, #THREAD_ABT_REGS_APIAKEY_HI] + write_apiakeyhi x0 + write_apiakeylo x1 + isb +#endif + + /* Restore x0 to x3 */ + load_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + + /* Return from exception */ + return_from_exception +END_FUNC el1_sync_abort + + /* sp_el0 in x3 */ +LOCAL_FUNC el0_sync_abort , : + pauth_el0_to_el1 x1 + /* + * Update core local flags + */ + ldr w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsl w1, w1, #THREAD_CLF_SAVED_SHIFT + orr w1, w1, #THREAD_CLF_ABORT + str w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + + /* + * Save state on stack + */ + + /* load abt_stack_va_end */ + ldr x1, [sp, #THREAD_CORE_LOCAL_ABT_STACK_VA_END] + /* Keep pointer to initial record in x0 */ + mov x0, sp + /* Switch to SP_EL0 */ + msr spsel, #0 + mov sp, x1 + sub sp, sp, #THREAD_ABT_REGS_SIZE + mrs x2, spsr_el1 + /* Store spsr, sp_el0 */ + stp x2, x3, [sp, #THREAD_ABT_REG_SPSR] + /* Store original x0, x1 */ + ldp x2, x3, [x0, #THREAD_CORE_LOCAL_X0] + stp x2, x3, [sp, #THREAD_ABT_REG_X0] + /* Store original x2, x3 and x4 to x29 */ + ldp x2, x3, [x0, #THREAD_CORE_LOCAL_X2] + store_xregs sp, THREAD_ABT_REG_X2, 2, 29 + /* Store x30, elr_el1 */ + mrs x1, elr_el1 + stp x30, x1, [sp, #THREAD_ABT_REG_X30] + +#if defined(CFG_TA_PAUTH) + read_apiakeyhi x2 + read_apiakeylo x3 + stp x2, x3, [sp, #THREAD_ABT_REGS_APIAKEY_HI] +#endif + +#if defined(CFG_CORE_PAUTH) + ldp x2, x3, [x0, #THREAD_CORE_LOCAL_KEYS] + write_apiakeyhi x2 + write_apiakeylo x3 +#endif + +#if defined(CFG_CORE_PAUTH) || defined(CFG_TA_PAUTH) + /* SCTLR or the APIA key has changed */ + isb +#endif + + /* + * Call handler + */ + mov x0, #0 + mov x1, sp + bl abort_handler + + /* + * Restore state from stack + */ + + /* Load x30, elr_el1 */ + ldp x30, x0, [sp, #THREAD_ABT_REG_X30] + msr elr_el1, x0 + /* Load x0 to x29 */ + load_xregs sp, THREAD_ABT_REG_X0, 0, 29 + /* Switch to SP_EL1 */ + msr spsel, #1 + /* Save x0 to x3 in EL1_REC */ + store_xregs sp, THREAD_CORE_LOCAL_X0, 0, 3 + /* Restore spsr_el1 and sp_el0 */ + mrs x3, sp_el0 + ldp x0, x1, [x3, #THREAD_ABT_REG_SPSR] + msr spsr_el1, x0 + msr sp_el0, x1 + + /* Update core local flags */ + ldr w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsr w1, w1, #THREAD_CLF_SAVED_SHIFT + str w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + ldp x1, x2, [x3, #THREAD_ABT_REGS_APIAKEY_HI] + write_apiakeyhi x1 + write_apiakeylo x2 +#endif + + /* Restore x2 to x3 */ + load_xregs sp, THREAD_CORE_LOCAL_X2, 2, 3 + + b_if_spsr_is_el0 w0, 1f + +#if defined(CFG_CORE_PAUTH) + /* the APIA key has changed */ + isb +#endif + + /* Restore x0 to x1 */ + load_xregs sp, THREAD_CORE_LOCAL_X0, 0, 1 + + /* Return from exception */ + return_from_exception +1: b eret_to_el0 +END_FUNC el0_sync_abort + +/* The handler of foreign interrupt. */ +.macro foreign_intr_handler mode:req + /* + * Update core local flags + */ + ldr w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsl w1, w1, #THREAD_CLF_SAVED_SHIFT + orr w1, w1, #THREAD_CLF_TMP + .ifc \mode\(),fiq + orr w1, w1, #THREAD_CLF_FIQ + .else + orr w1, w1, #THREAD_CLF_IRQ + .endif + str w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + + /* get pointer to current thread context in x0 */ + get_thread_ctx sp, 0, 1, 2 + /* Keep original SP_EL0 */ + mrs x2, sp_el0 + + /* Store original sp_el0 */ + str x2, [x0, #THREAD_CTX_REGS_SP] + /* Store tpidr_el0 */ + mrs x2, tpidr_el0 + str x2, [x0, #THREAD_CTX_REGS_TPIDR_EL0] + /* Store x4..x30 */ + store_xregs x0, THREAD_CTX_REGS_X4, 4, 30 + /* Load original x0..x3 into x10..x13 */ + load_xregs sp, THREAD_CORE_LOCAL_X0, 10, 13 + /* Save original x0..x3 */ + store_xregs x0, THREAD_CTX_REGS_X0, 10, 13 + +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + /* Save APIAKEY */ + read_apiakeyhi x1 + read_apiakeylo x2 + store_xregs x0, THREAD_CTX_REGS_APIAKEY_HI, 1, 2 +#endif +#if defined(CFG_CORE_PAUTH) + ldp x1, x2, [sp, #THREAD_CORE_LOCAL_KEYS] + write_apiakeyhi x1 + write_apiakeylo x2 + isb +#endif + /* load tmp_stack_va_end */ + ldr x1, [sp, #THREAD_CORE_LOCAL_TMP_STACK_VA_END] + /* Switch to SP_EL0 */ + msr spsel, #0 + mov sp, x1 + +#ifdef CFG_CORE_WORKAROUND_NSITR_CACHE_PRIME + /* + * Prevent leaking information about which entries has been used in + * cache. We're relying on the dispatcher in TF-A to take care of + * the BTB. + */ + mov x0, #DCACHE_OP_CLEAN_INV + bl dcache_op_louis + ic iallu +#endif + /* + * Mark current thread as suspended + */ + mov w0, #THREAD_FLAGS_EXIT_ON_FOREIGN_INTR + mrs x1, spsr_el1 + mrs x2, elr_el1 + bl thread_state_suspend + + /* Update core local flags */ + /* Switch to SP_EL1 */ + msr spsel, #1 + ldr w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsr w1, w1, #THREAD_CLF_SAVED_SHIFT + orr w1, w1, #THREAD_CLF_TMP + str w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + msr spsel, #0 + + /* + * Note that we're exiting with SP_EL0 selected since the entry + * functions expects to have SP_EL0 selected with the tmp stack + * set. + */ + + /* Passing thread index in w0 */ + b thread_foreign_intr_exit +.endm + +/* + * This struct is never used from C it's only here to visualize the + * layout. + * + * struct elx_nintr_rec { + * uint64_t x[19 - 4]; x4..x18 + * uint64_t lr; + * uint64_t sp_el0; + * #if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + * uint64_t apiakey_hi; + * uint64_t apiakey_lo; + * #endif + * }; + */ +#define ELX_NINTR_REC_X(x) (8 * ((x) - 4)) +#define ELX_NINTR_REC_LR (8 + ELX_NINTR_REC_X(19)) +#define ELX_NINTR_REC_SP_EL0 (8 + ELX_NINTR_REC_LR) +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) +#define ELX_NINTR_REG_APIAKEY_HI (8 + ELX_NINTR_REC_SP_EL0) +#define ELX_NINTR_REG_APIAKEY_LO (8 + ELX_NINTR_REG_APIAKEY_HI) +#define ELX_NINTR_REC_SIZE (8 + ELX_NINTR_REG_APIAKEY_LO) +#else +#define ELX_NINTR_REC_SIZE (8 + ELX_NINTR_REC_SP_EL0) +#endif + + +/* The handler of native interrupt. */ +.macro native_intr_handler mode:req + /* + * Update core local flags + */ + ldr w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsl w1, w1, #THREAD_CLF_SAVED_SHIFT + .ifc \mode\(),fiq + orr w1, w1, #THREAD_CLF_FIQ + .else + orr w1, w1, #THREAD_CLF_IRQ + .endif + orr w1, w1, #THREAD_CLF_TMP + str w1, [sp, #THREAD_CORE_LOCAL_FLAGS] + + /* + * Save registers on the temp stack that can be corrupted by a call + * to a C function. + * + * Note that we're temporarily using x1 to access the temp stack + * until we're ready to switch to sp_el0 and update sp. + */ + /* load tmp_stack_va_end */ + ldr x1, [sp, #THREAD_CORE_LOCAL_TMP_STACK_VA_END] + /* Make room for struct elx_nintr_rec */ + sub x1, x1, #ELX_NINTR_REC_SIZE + /* Store lr and original sp_el0 */ + mrs x2, sp_el0 + stp x30, x2, [x1, #ELX_NINTR_REC_LR] + /* Store x4..x18 */ + store_xregs x1, ELX_NINTR_REC_X(4), 4, 18 + +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + read_apiakeyhi x2 + read_apiakeylo x3 + stp x2, x3, [x1, #ELX_NINTR_REG_APIAKEY_HI] +#if defined(CFG_CORE_PAUTH) + ldp x2, x3, [sp, #THREAD_CORE_LOCAL_KEYS] + write_apiakeyhi x2 + write_apiakeylo x3 +#endif + /* SCTLR or the APIA key has changed */ + isb +#endif + + /* Switch to SP_EL0 */ + msr spsel, #0 + mov sp, x1 + + bl thread_check_canaries + bl interrupt_main_handler + + /* + * Restore registers + */ + +#if defined(CFG_TA_PAUTH) || defined(CFG_CORE_PAUTH) + ldp x0, x1, [sp, #ELX_NINTR_REG_APIAKEY_HI] + write_apiakeyhi x0 + write_apiakeylo x1 +#endif + + /* Restore x4..x18 */ + load_xregs sp, ELX_NINTR_REC_X(4), 4, 18 + /* Load lr and original sp_el0 */ + ldp x30, x2, [sp, #ELX_NINTR_REC_LR] + /* Restore SP_El0 */ + mov sp, x2 + /* Switch back to SP_EL1 */ + msr spsel, #1 + + /* Update core local flags */ + ldr w0, [sp, #THREAD_CORE_LOCAL_FLAGS] + lsr w0, w0, #THREAD_CLF_SAVED_SHIFT + str w0, [sp, #THREAD_CORE_LOCAL_FLAGS] + + mrs x0, spsr_el1 + + /* Restore x2..x3 */ + load_xregs sp, THREAD_CORE_LOCAL_X2, 2, 3 + b_if_spsr_is_el0 w0, 1f + +#if defined(CFG_CORE_PAUTH) + /* APIA key has changed */ + isb +#endif + + /* Restore x0..x1 */ + load_xregs sp, THREAD_CORE_LOCAL_X0, 0, 1 + + /* Return from exception */ + return_from_exception +1: b eret_to_el0 +.endm + +LOCAL_FUNC elx_irq , : +#if defined(CFG_CORE_IRQ_IS_NATIVE_INTR) + native_intr_handler irq +#else + foreign_intr_handler irq +#endif +END_FUNC elx_irq + +LOCAL_FUNC elx_fiq , : +#if defined(CFG_CORE_IRQ_IS_NATIVE_INTR) + foreign_intr_handler fiq +#else + native_intr_handler fiq +#endif +END_FUNC elx_fiq + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/thread_optee_smc.c b/optee_os/core/arch/arm/kernel/thread_optee_smc.c new file mode 100644 index 0000000..bc69d8b --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_optee_smc.c @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2021, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool thread_prealloc_rpc_cache; +static unsigned int thread_rpc_pnum; + +static_assert(NOTIF_VALUE_DO_BOTTOM_HALF == + OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF); + +void thread_handle_fast_smc(struct thread_smc_args *args) +{ + thread_check_canaries(); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && + virt_set_guest(args->a7)) { + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; + goto out; + } + + tee_entry_fast(args); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + +out: + /* Fast handlers must not unmask any exceptions */ + assert(thread_get_exceptions() == THREAD_EXCP_ALL); +} + +uint32_t thread_handle_std_smc(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6 __unused, uint32_t a7 __maybe_unused) +{ + uint32_t rv = OPTEE_SMC_RETURN_OK; + + thread_check_canaries(); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && virt_set_guest(a7)) + return OPTEE_SMC_RETURN_ENOTAVAIL; + + /* + * thread_resume_from_rpc() and thread_alloc_and_run() only return + * on error. Successful return is done via thread_exit() or + * thread_rpc(). + */ + if (a0 == OPTEE_SMC_CALL_RETURN_FROM_RPC) { + thread_resume_from_rpc(a3, a1, a2, a4, a5); + rv = OPTEE_SMC_RETURN_ERESUME; + } else { + thread_alloc_and_run(a0, a1, a2, a3, 0, 0); + rv = OPTEE_SMC_RETURN_ETHREAD_LIMIT; + } + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + + return rv; +} + +/** + * Free physical memory previously allocated with thread_rpc_alloc_arg() + * + * @cookie: cookie received when allocating the buffer + */ +static void thread_rpc_free_arg(uint64_t cookie) +{ + if (cookie) { + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { + OPTEE_SMC_RETURN_RPC_FREE + }; + + reg_pair_from_64(cookie, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + } +} + +static uint32_t get_msg_arg(struct mobj *mobj, size_t offset, + size_t *num_params, struct optee_msg_arg **arg, + struct optee_msg_arg **rpc_arg) +{ + void *p = NULL; + size_t sz = 0; + + if (!mobj) + return OPTEE_SMC_RETURN_EBADADDR; + + p = mobj_get_va(mobj, offset, sizeof(struct optee_msg_arg)); + if (!p || !IS_ALIGNED_WITH_TYPE(p, struct optee_msg_arg)) + return OPTEE_SMC_RETURN_EBADADDR; + + *arg = p; + *num_params = READ_ONCE((*arg)->num_params); + if (*num_params > OPTEE_MSG_MAX_NUM_PARAMS) + return OPTEE_SMC_RETURN_EBADADDR; + + sz = OPTEE_MSG_GET_ARG_SIZE(*num_params); + if (!mobj_get_va(mobj, offset, sz)) + return OPTEE_SMC_RETURN_EBADADDR; + + if (rpc_arg) { + size_t rpc_sz = 0; + + rpc_sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + p = mobj_get_va(mobj, offset + sz, rpc_sz); + if (!p) + return OPTEE_SMC_RETURN_EBADADDR; + *rpc_arg = p; + } + + return OPTEE_SMC_RETURN_OK; +} + +static void clear_prealloc_rpc_cache(struct thread_ctx *thr) +{ + thread_rpc_free_arg(mobj_get_cookie(thr->rpc_mobj)); + mobj_put(thr->rpc_mobj); + thr->rpc_arg = NULL; + thr->rpc_mobj = NULL; +} + +static uint32_t call_entry_std(struct optee_msg_arg *arg, size_t num_params, + struct optee_msg_arg *rpc_arg) +{ + struct thread_ctx *thr = threads + thread_get_id(); + uint32_t rv = 0; + + if (rpc_arg) { + /* + * In case the prealloc RPC arg cache is enabled, clear the + * cached object for this thread. + * + * Normally it doesn't make sense to have the prealloc RPC + * arg cache enabled together with a supplied RPC arg + * struct. But if it is we must use the supplied struct and + * at the same time make sure to not break anything. + */ + if (IS_ENABLED(CFG_PREALLOC_RPC_CACHE) && + thread_prealloc_rpc_cache) + clear_prealloc_rpc_cache(thr); + thr->rpc_arg = rpc_arg; + } + + if (tee_entry_std(arg, num_params)) + rv = OPTEE_SMC_RETURN_EBADCMD; + else + rv = OPTEE_SMC_RETURN_OK; + + thread_rpc_shm_cache_clear(&thr->shm_cache); + if (rpc_arg) + thr->rpc_arg = NULL; + + if (rv == OPTEE_SMC_RETURN_OK && + !(IS_ENABLED(CFG_PREALLOC_RPC_CACHE) && thread_prealloc_rpc_cache)) + clear_prealloc_rpc_cache(thr); + + return rv; +} + +static uint32_t std_entry_with_parg(paddr_t parg, bool with_rpc_arg) +{ + size_t sz = sizeof(struct optee_msg_arg); + struct optee_msg_arg *rpc_arg = NULL; + struct optee_msg_arg *arg = NULL; + struct mobj *mobj = NULL; + size_t num_params = 0; + uint32_t rv = 0; + + /* Check if this region is in static shared space */ + if (core_pbuf_is(CORE_MEM_NSEC_SHM, parg, sz)) { + if (!IS_ALIGNED_WITH_TYPE(parg, struct optee_msg_arg)) + goto bad_addr; + + arg = phys_to_virt(parg, MEM_AREA_NSEC_SHM, + sizeof(struct optee_msg_arg)); + if (!arg) + goto bad_addr; + + num_params = READ_ONCE(arg->num_params); + if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) + return OPTEE_SMC_RETURN_EBADADDR; + + sz = OPTEE_MSG_GET_ARG_SIZE(num_params); + if (with_rpc_arg) { + rpc_arg = (void *)((uint8_t *)arg + sz); + sz += OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + } + if (!core_pbuf_is(CORE_MEM_NSEC_SHM, parg, sz)) + goto bad_addr; + + return call_entry_std(arg, num_params, rpc_arg); + } else { + if (parg & SMALL_PAGE_MASK) + goto bad_addr; + /* + * mobj_mapped_shm_alloc checks if parg resides in nonsec + * ddr. + */ + mobj = mobj_mapped_shm_alloc(&parg, 1, 0, 0); + if (!mobj) + goto bad_addr; + if (with_rpc_arg) + rv = get_msg_arg(mobj, 0, &num_params, &arg, &rpc_arg); + else + rv = get_msg_arg(mobj, 0, &num_params, &arg, NULL); + if (!rv) + rv = call_entry_std(arg, num_params, rpc_arg); + mobj_put(mobj); + return rv; + } + +bad_addr: + EMSG("Bad arg address 0x%"PRIxPA, parg); + return OPTEE_SMC_RETURN_EBADADDR; +} + +static uint32_t std_entry_with_regd_arg(uint64_t cookie, size_t offset) +{ + struct optee_msg_arg *rpc_arg = NULL; + struct optee_msg_arg *arg = NULL; + size_t num_params = 0; + struct mobj *mobj = NULL; + uint32_t rv = 0; + + mobj = mobj_reg_shm_get_by_cookie(cookie); + if (!mobj) { + EMSG("Bad arg cookie 0x%"PRIx64, cookie); + return OPTEE_SMC_RETURN_EBADADDR; + } + + if (mobj_inc_map(mobj)) { + rv = OPTEE_SMC_RETURN_ENOMEM; + goto out; + } + + rv = get_msg_arg(mobj, offset, &num_params, &arg, &rpc_arg); + if (!rv) + rv = call_entry_std(arg, num_params, rpc_arg); + + mobj_dec_map(mobj); +out: + mobj_put(mobj); + + return rv; +} + +static uint32_t std_smc_entry(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3 __unused) +{ + const bool with_rpc_arg = true; + + switch (a0) { + case OPTEE_SMC_CALL_WITH_ARG: + return std_entry_with_parg(reg_pair_to_64(a1, a2), + !with_rpc_arg); + case OPTEE_SMC_CALL_WITH_RPC_ARG: + return std_entry_with_parg(reg_pair_to_64(a1, a2), + with_rpc_arg); + case OPTEE_SMC_CALL_WITH_REGD_ARG: + return std_entry_with_regd_arg(reg_pair_to_64(a1, a2), a3); + default: + EMSG("Unknown SMC 0x%"PRIx32, a0); + return OPTEE_SMC_RETURN_EBADCMD; + } +} + +/* + * Helper routine for the assembly function thread_std_smc_entry() + * + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4 __unused, + uint32_t a5 __unused) +{ + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_on_stdcall(); + + return std_smc_entry(a0, a1, a2, a3); +} + +bool thread_disable_prealloc_rpc_cache(uint64_t *cookie) +{ + bool rv = false; + size_t n = 0; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + thread_lock_global(); + + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].state != THREAD_STATE_FREE) { + rv = false; + goto out; + } + } + + rv = true; + + if (IS_ENABLED(CFG_PREALLOC_RPC_CACHE)) { + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].rpc_arg) { + *cookie = mobj_get_cookie(threads[n].rpc_mobj); + mobj_put(threads[n].rpc_mobj); + threads[n].rpc_arg = NULL; + threads[n].rpc_mobj = NULL; + goto out; + } + } + } + + *cookie = 0; + thread_prealloc_rpc_cache = false; +out: + thread_unlock_global(); + thread_unmask_exceptions(exceptions); + return rv; +} + +bool thread_enable_prealloc_rpc_cache(void) +{ + bool rv = false; + size_t n = 0; + uint32_t exceptions = 0; + + if (!IS_ENABLED(CFG_PREALLOC_RPC_CACHE)) + return true; + + exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + thread_lock_global(); + + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].state != THREAD_STATE_FREE) { + rv = false; + goto out; + } + } + + rv = true; + thread_prealloc_rpc_cache = true; +out: + thread_unlock_global(); + thread_unmask_exceptions(exceptions); + return rv; +} + +static struct mobj *rpc_shm_mobj_alloc(paddr_t pa, size_t sz, uint64_t cookie) +{ + /* Check if this region is in static shared space */ + if (core_pbuf_is(CORE_MEM_NSEC_SHM, pa, sz)) + return mobj_shm_alloc(pa, sz, cookie); + + if (IS_ENABLED(CFG_CORE_DYN_SHM) && + !(pa & SMALL_PAGE_MASK) && sz <= SMALL_PAGE_SIZE) + return mobj_mapped_shm_alloc(&pa, 1, 0, cookie); + + return NULL; +} + +/** + * Allocates data for struct optee_msg_arg. + * + * @size: size in bytes of struct optee_msg_arg + * + * @returns mobj that describes allocated buffer or NULL on error + */ +static struct mobj *thread_rpc_alloc_arg(size_t size) +{ + paddr_t pa; + uint64_t co; + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { + OPTEE_SMC_RETURN_RPC_ALLOC, size + }; + struct mobj *mobj = NULL; + + thread_rpc(rpc_args); + + /* Registers 1 and 2 passed from normal world */ + pa = reg_pair_to_64(rpc_args[0], rpc_args[1]); + /* Registers 4 and 5 passed from normal world */ + co = reg_pair_to_64(rpc_args[2], rpc_args[3]); + + if (!IS_ALIGNED_WITH_TYPE(pa, struct optee_msg_arg)) + goto err; + + mobj = rpc_shm_mobj_alloc(pa, size, co); + if (!mobj) + goto err; + + return mobj; +err: + thread_rpc_free_arg(co); + mobj_put(mobj); + return NULL; +} + +static bool set_rmem(struct optee_msg_param *param, + struct thread_param *tpm) +{ + param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + + OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; + param->u.rmem.offs = tpm->u.memref.offs; + param->u.rmem.size = tpm->u.memref.size; + if (tpm->u.memref.mobj) { + param->u.rmem.shm_ref = mobj_get_cookie(tpm->u.memref.mobj); + if (!param->u.rmem.shm_ref) + return false; + } else { + param->u.rmem.shm_ref = 0; + } + + return true; +} + +static bool set_tmem(struct optee_msg_param *param, + struct thread_param *tpm) +{ + paddr_t pa = 0; + uint64_t shm_ref = 0; + struct mobj *mobj = tpm->u.memref.mobj; + + param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + + OPTEE_MSG_ATTR_TYPE_TMEM_INPUT; + if (mobj) { + shm_ref = mobj_get_cookie(mobj); + if (!shm_ref) + return false; + if (mobj_get_pa(mobj, tpm->u.memref.offs, 0, &pa)) + return false; + } + + param->u.tmem.size = tpm->u.memref.size; + param->u.tmem.buf_ptr = pa; + param->u.tmem.shm_ref = shm_ref; + + return true; +} + +static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, + struct thread_param *params, void **arg_ret, + uint64_t *carg_ret) +{ + struct thread_ctx *thr = threads + thread_get_id(); + struct optee_msg_arg *arg = thr->rpc_arg; + size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + + if (num_params > THREAD_RPC_MAX_NUM_PARAMS) + return TEE_ERROR_BAD_PARAMETERS; + + if (!arg) { + struct mobj *mobj = thread_rpc_alloc_arg(sz); + + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + + arg = mobj_get_va(mobj, 0, sz); + if (!arg) { + thread_rpc_free_arg(mobj_get_cookie(mobj)); + return TEE_ERROR_OUT_OF_MEMORY; + } + + thr->rpc_arg = arg; + thr->rpc_mobj = mobj; + } + + memset(arg, 0, OPTEE_MSG_GET_ARG_SIZE(num_params)); + arg->cmd = cmd; + arg->num_params = num_params; + arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ + + for (size_t n = 0; n < num_params; n++) { + switch (params[n].attr) { + case THREAD_PARAM_ATTR_NONE: + arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; + break; + case THREAD_PARAM_ATTR_VALUE_IN: + case THREAD_PARAM_ATTR_VALUE_OUT: + case THREAD_PARAM_ATTR_VALUE_INOUT: + arg->params[n].attr = params[n].attr - + THREAD_PARAM_ATTR_VALUE_IN + + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; + arg->params[n].u.value.a = params[n].u.value.a; + arg->params[n].u.value.b = params[n].u.value.b; + arg->params[n].u.value.c = params[n].u.value.c; + break; + case THREAD_PARAM_ATTR_MEMREF_IN: + case THREAD_PARAM_ATTR_MEMREF_OUT: + case THREAD_PARAM_ATTR_MEMREF_INOUT: + if (!params[n].u.memref.mobj || + mobj_matches(params[n].u.memref.mobj, + CORE_MEM_NSEC_SHM)) { + if (!set_tmem(arg->params + n, params + n)) + return TEE_ERROR_BAD_PARAMETERS; + } else if (mobj_matches(params[n].u.memref.mobj, + CORE_MEM_REG_SHM)) { + if (!set_rmem(arg->params + n, params + n)) + return TEE_ERROR_BAD_PARAMETERS; + } else { + return TEE_ERROR_BAD_PARAMETERS; + } + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + } + + *arg_ret = arg; + *carg_ret = mobj_get_cookie(thr->rpc_mobj); + + return TEE_SUCCESS; +} + +static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, + struct thread_param *params) +{ + for (size_t n = 0; n < num_params; n++) { + switch (params[n].attr) { + case THREAD_PARAM_ATTR_VALUE_OUT: + case THREAD_PARAM_ATTR_VALUE_INOUT: + params[n].u.value.a = arg->params[n].u.value.a; + params[n].u.value.b = arg->params[n].u.value.b; + params[n].u.value.c = arg->params[n].u.value.c; + break; + case THREAD_PARAM_ATTR_MEMREF_OUT: + case THREAD_PARAM_ATTR_MEMREF_INOUT: + /* + * rmem.size and tmem.size is the same type and + * location. + */ + params[n].u.memref.size = arg->params[n].u.rmem.size; + break; + default: + break; + } + } + + return arg->ret; +} + +uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, + struct thread_param *params) +{ + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; + void *arg = NULL; + uint64_t carg = 0; + uint32_t ret = 0; + + /* The source CRYPTO_RNG_SRC_JITTER_RPC is safe to use here */ + plat_prng_add_jitter_entropy(CRYPTO_RNG_SRC_JITTER_RPC, + &thread_rpc_pnum); + + ret = get_rpc_arg(cmd, num_params, params, &arg, &carg); + if (ret) + return ret; + + reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + + return get_rpc_arg_res(arg, num_params, params); +} + +/** + * Free physical memory previously allocated with thread_rpc_alloc() + * + * @cookie: cookie received when allocating the buffer + * @bt: must be the same as supplied when allocating + * @mobj: mobj that describes allocated buffer + * + * This function also frees corresponding mobj. + */ +static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) +{ + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; + void *arg = NULL; + uint64_t carg = 0; + struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); + uint32_t ret = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, + &arg, &carg); + + mobj_put(mobj); + + if (!ret) { + reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + } +} + +static struct mobj *get_rpc_alloc_res(struct optee_msg_arg *arg, + unsigned int bt, size_t size) +{ + struct mobj *mobj = NULL; + uint64_t cookie = 0; + size_t sz = 0; + paddr_t p = 0; + + if (arg->ret || arg->num_params != 1) + goto err; + + if (arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT && + arg->params[0].attr != (OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | + OPTEE_MSG_ATTR_NONCONTIG)) + goto err; + + p = arg->params[0].u.tmem.buf_ptr; + sz = READ_ONCE(arg->params[0].u.tmem.size); + cookie = arg->params[0].u.tmem.shm_ref; + if (sz < size) + goto err; + + if (arg->params[0].attr == OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT) + mobj = rpc_shm_mobj_alloc(p, sz, cookie); + else + mobj = msg_param_mobj_from_noncontig(p, sz, cookie, true); + + if (!mobj) { + thread_rpc_free(bt, cookie, mobj); + goto err; + } + + assert(mobj_is_nonsec(mobj)); + return mobj; +err: + EMSG("RPC allocation failed. Non-secure world result: ret=%#" + PRIx32" ret_origin=%#"PRIx32, arg->ret, arg->ret_origin); + return NULL; +} + +/** + * Allocates shared memory buffer via RPC + * + * @size: size in bytes of shared memory buffer + * @align: required alignment of buffer + * @bt: buffer type OPTEE_RPC_SHM_TYPE_* + * + * Returns a pointer to MOBJ for the memory on success, or NULL on failure. + */ +static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) +{ + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; + void *arg = NULL; + uint64_t carg = 0; + struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); + uint32_t ret = get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, + &arg, &carg); + + if (ret) + return NULL; + + reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + + return get_rpc_alloc_res(arg, bt, size); +} + +struct mobj *thread_rpc_alloc_payload(size_t size) +{ + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); +} + +struct mobj *thread_rpc_alloc_kernel_payload(size_t size) +{ + /* + * Error out early since kernel private dynamic shared memory + * allocations don't currently use the `OPTEE_MSG_ATTR_NONCONTIG` bit + * and therefore cannot be larger than a page. + */ + if (IS_ENABLED(CFG_CORE_DYN_SHM) && size > SMALL_PAGE_SIZE) + return NULL; + + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); +} + +void thread_rpc_free_kernel_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, mobj_get_cookie(mobj), mobj); +} + +void thread_rpc_free_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), + mobj); +} + +struct mobj *thread_rpc_alloc_global_payload(size_t size) +{ + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); +} + +void thread_rpc_free_global_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, mobj_get_cookie(mobj), + mobj); +} diff --git a/optee_os/core/arch/arm/kernel/thread_optee_smc_a32.S b/optee_os/core/arch/arm/kernel/thread_optee_smc_a32.S new file mode 100644 index 0000000..2a8c14c --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_optee_smc_a32.S @@ -0,0 +1,255 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +.arch_extension sec + +/* + * If ASLR is configured the identity mapped code may be mapped at two + * locations, the identity location where virtual and physical address is + * the same and at the runtime selected location to which OP-TEE has been + * relocated. Code executing at a location different compared to the + * runtime selected location works OK as long as it doesn't do relative + * addressing outside the identity mapped range. To allow relative + * addressing this macro jumps to the runtime selected location. + * + * Note that the identity mapped range and the runtime selected range can + * only differ if ASLR is configured. + */ + .macro readjust_pc +#ifdef CFG_CORE_ASLR + ldr r12, =1111f + bx r12 +1111: +#endif + .endm + +FUNC vector_std_smc_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + push {r4-r7} + bl thread_handle_std_smc + add sp, sp, #(4 * 4) + /* + * Normally thread_handle_std_smc() should return via + * thread_exit(), thread_rpc(), but if thread_handle_std_smc() + * hasn't switched stack (error detected) it will do a normal "C" + * return. + */ + mov r1, r0 + ldr r0, =TEESMC_OPTEED_RETURN_CALL_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_std_smc_entry + +FUNC vector_fast_smc_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + push {r0-r7} + mov r0, sp + bl thread_handle_fast_smc + pop {r1-r8} + ldr r0, =TEESMC_OPTEED_RETURN_CALL_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_fast_smc_entry + +FUNC vector_fiq_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + /* Secure Monitor received a FIQ and passed control to us. */ + bl thread_check_canaries + bl interrupt_main_handler + ldr r0, =TEESMC_OPTEED_RETURN_FIQ_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_fiq_entry + +#if defined(CFG_WITH_ARM_TRUSTED_FW) +LOCAL_FUNC vector_cpu_on_entry , : , .identity_map +UNWIND( .cantunwind) + bl cpu_on_handler + /* When cpu_on_handler() returns mmu is enabled */ + mov r1, r0 + ldr r0, =TEESMC_OPTEED_RETURN_ON_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_on_entry + +LOCAL_FUNC vector_cpu_off_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + bl thread_cpu_off_handler + mov r1, r0 + ldr r0, =TEESMC_OPTEED_RETURN_OFF_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_off_entry + +LOCAL_FUNC vector_cpu_suspend_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + bl thread_cpu_suspend_handler + mov r1, r0 + ldr r0, =TEESMC_OPTEED_RETURN_SUSPEND_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_suspend_entry + +LOCAL_FUNC vector_cpu_resume_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + bl thread_cpu_resume_handler + mov r1, r0 + ldr r0, =TEESMC_OPTEED_RETURN_RESUME_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_resume_entry + +LOCAL_FUNC vector_system_off_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + bl thread_system_off_handler + mov r1, r0 + ldr r0, =TEESMC_OPTEED_RETURN_SYSTEM_OFF_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_system_off_entry + +LOCAL_FUNC vector_system_reset_entry , : , .identity_map +UNWIND( .cantunwind) + readjust_pc + bl thread_system_reset_handler + mov r1, r0 + ldr r0, =TEESMC_OPTEED_RETURN_SYSTEM_RESET_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_system_reset_entry + +/* + * Vector table supplied to ARM Trusted Firmware (ARM-TF) at + * initialization. Also used when compiled with the internal monitor, but + * the cpu_*_entry and system_*_entry are not used then. + * + * Note that ARM-TF depends on the layout of this vector table, any change + * in layout has to be synced with ARM-TF. + */ +FUNC thread_vector_table , : , .identity_map +UNWIND( .cantunwind) + b vector_std_smc_entry + b vector_fast_smc_entry + b vector_cpu_on_entry + b vector_cpu_off_entry + b vector_cpu_resume_entry + b vector_cpu_suspend_entry + b vector_fiq_entry + b vector_system_off_entry + b vector_system_reset_entry +END_FUNC thread_vector_table +DECLARE_KEEP_PAGER thread_vector_table +#endif /*if defined(CFG_WITH_ARM_TRUSTED_FW)*/ + +FUNC thread_std_smc_entry , : +UNWIND( .cantunwind) + push {r4, r5} /* Pass these following the arm32 calling convention */ + bl __thread_std_smc_entry + add sp, sp, #8 /* There's nothing return, just restore the sp */ + mov r4, r0 /* Save return value for later */ + + /* Disable interrupts before switching to temporary stack */ + cpsid aif + bl thread_get_tmp_sp + mov sp, r0 + + bl thread_state_free + + ldr r0, =TEESMC_OPTEED_RETURN_CALL_DONE + mov r1, r4 + mov r2, #0 + mov r3, #0 + mov r4, #0 + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC thread_std_smc_entry + +/* void thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]) */ +FUNC thread_rpc , : + push {r0, lr} +UNWIND( .save {r0, lr}) + + bl thread_save_state + mov r4, r0 /* Save original CPSR */ + + /* + * Switch to temporary stack and SVC mode. Save CPSR to resume into. + */ + bl thread_get_tmp_sp + ldr r5, [sp] /* Get pointer to rv[] */ + cps #CPSR_MODE_SVC /* Change to SVC mode */ + mov sp, r0 /* Switch to tmp stack */ + + mov r0, #THREAD_FLAGS_COPY_ARGS_ON_RETURN + mov r1, r4 /* CPSR to restore */ + ldr r2, =.thread_rpc_return + bl thread_state_suspend + mov r4, r0 /* Supply thread index */ + ldr r0, =TEESMC_OPTEED_RETURN_CALL_DONE + ldm r5, {r1-r3} /* Load rv[] into r0-r2 */ + smc #0 + /* SMC should not return */ + panic_at_smc_return + +.thread_rpc_return: + /* + * At this point has the stack pointer been restored to the value + * it had when thread_save_state() was called above. + * + * Jumps here from thread_resume above when RPC has returned. The + * IRQ and FIQ bits are restored to what they where when this + * function was originally entered. + */ + pop {r12, lr} /* Get pointer to rv[] */ + stm r12, {r0-r3} /* Store r0-r3 into rv[] */ + bx lr +END_FUNC thread_rpc +DECLARE_KEEP_PAGER thread_rpc + +/* + * void thread_foreign_intr_exit(uint32_t thread_index) + * + * This function is jumped to at the end of macro foreign_intr_handler(). + * The current thread as indicated by @thread_index has just been + * suspended. The job here is just to inform normal world the thread id to + * resume when returning. + */ +FUNC thread_foreign_intr_exit , : + mov r4, r0 + ldr r0, =TEESMC_OPTEED_RETURN_CALL_DONE + ldr r1, =OPTEE_SMC_RETURN_RPC_FOREIGN_INTR + mov r2, #0 + mov r3, #0 + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC thread_foreign_intr_exit diff --git a/optee_os/core/arch/arm/kernel/thread_optee_smc_a64.S b/optee_os/core/arch/arm/kernel/thread_optee_smc_a64.S new file mode 100644 index 0000000..1b08414 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_optee_smc_a64.S @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * If ASLR is configured the identity mapped code may be mapped at two + * locations, the identity location where virtual and physical address is + * the same and at the runtime selected location to which OP-TEE has been + * relocated. Code executing at a location different compared to the + * runtime selected location works OK as long as it doesn't do relative + * addressing outside the identity mapped range. To allow relative + * addressing this macro jumps to the runtime selected location. + * + * Note that the identity mapped range and the runtime selected range can + * only differ if ASLR is configured. + */ + .macro readjust_pc +#ifdef CFG_CORE_ASLR + adr x16, 1111f + ldr x17, boot_mmu_config + CORE_MMU_CONFIG_MAP_OFFSET + add x16, x16, x17 + br x16 +1111: +BTI( bti j) +#endif + .endm + +LOCAL_FUNC vector_std_smc_entry , : , .identity_map + readjust_pc + bl thread_handle_std_smc + /* + * Normally thread_handle_std_smc() should return via + * thread_exit(), thread_rpc(), but if thread_handle_std_smc() + * hasn't switched stack (error detected) it will do a normal "C" + * return. + */ + mov w1, w0 + ldr x0, =TEESMC_OPTEED_RETURN_CALL_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_std_smc_entry + +LOCAL_FUNC vector_fast_smc_entry , : , .identity_map + readjust_pc + sub sp, sp, #THREAD_SMC_ARGS_SIZE + store_xregs sp, THREAD_SMC_ARGS_X0, 0, 7 + mov x0, sp + bl thread_handle_fast_smc + load_xregs sp, THREAD_SMC_ARGS_X0, 1, 8 + add sp, sp, #THREAD_SMC_ARGS_SIZE + ldr x0, =TEESMC_OPTEED_RETURN_CALL_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_fast_smc_entry + +LOCAL_FUNC vector_fiq_entry , : , .identity_map + readjust_pc + /* Secure Monitor received a FIQ and passed control to us. */ + bl thread_check_canaries + bl interrupt_main_handler + ldr x0, =TEESMC_OPTEED_RETURN_FIQ_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_fiq_entry + +LOCAL_FUNC vector_cpu_on_entry , : , .identity_map + bl cpu_on_handler + mov x1, x0 + ldr x0, =TEESMC_OPTEED_RETURN_ON_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_on_entry + +LOCAL_FUNC vector_cpu_off_entry , : , .identity_map + readjust_pc + bl thread_cpu_off_handler + mov x1, x0 + ldr x0, =TEESMC_OPTEED_RETURN_OFF_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_off_entry + +LOCAL_FUNC vector_cpu_suspend_entry , : , .identity_map + readjust_pc + bl thread_cpu_suspend_handler + mov x1, x0 + ldr x0, =TEESMC_OPTEED_RETURN_SUSPEND_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_suspend_entry + +LOCAL_FUNC vector_cpu_resume_entry , : , .identity_map + readjust_pc + bl thread_cpu_resume_handler + mov x1, x0 + ldr x0, =TEESMC_OPTEED_RETURN_RESUME_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_cpu_resume_entry + +LOCAL_FUNC vector_system_off_entry , : , .identity_map + readjust_pc + bl thread_system_off_handler + mov x1, x0 + ldr x0, =TEESMC_OPTEED_RETURN_SYSTEM_OFF_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_system_off_entry + +LOCAL_FUNC vector_system_reset_entry , : , .identity_map + readjust_pc + bl thread_system_reset_handler + mov x1, x0 + ldr x0, =TEESMC_OPTEED_RETURN_SYSTEM_RESET_DONE + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC vector_system_reset_entry + +/* + * Vector table supplied to ARM Trusted Firmware (ARM-TF) at + * initialization. + * + * Note that ARM-TF depends on the layout of this vector table, any change + * in layout has to be synced with ARM-TF. + */ +FUNC thread_vector_table , : , .identity_map, , nobti + b vector_std_smc_entry + b vector_fast_smc_entry + b vector_cpu_on_entry + b vector_cpu_off_entry + b vector_cpu_resume_entry + b vector_cpu_suspend_entry + b vector_fiq_entry + b vector_system_off_entry + b vector_system_reset_entry +END_FUNC thread_vector_table +DECLARE_KEEP_PAGER thread_vector_table + +FUNC thread_std_smc_entry , : + bl __thread_std_smc_entry + mov w20, w0 /* Save return value for later */ + + /* Mask all maskable exceptions before switching to temporary stack */ + msr daifset, #DAIFBIT_ALL + bl thread_get_tmp_sp + mov sp, x0 + + bl thread_state_free + + ldr x0, =TEESMC_OPTEED_RETURN_CALL_DONE + mov w1, w20 + mov x2, #0 + mov x3, #0 + mov x4, #0 + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC thread_std_smc_entry + +/* void thread_rpc_spsr(uint32_t rv[THREAD_RPC_NUM_ARGS], uint64_t spsr) */ +FUNC thread_rpc_spsr , : + /* Mask all maskable exceptions before switching to temporary stack */ + msr daifset, #DAIFBIT_ALL + push x0, xzr + push x1, x30 + bl thread_get_ctx_regs + ldr x30, [sp, #8] + store_xregs x0, THREAD_CTX_REGS_X19, 19, 30 + mov x19, x0 + +#if defined(CFG_CORE_PAUTH) + /* Save APIAKEY */ + read_apiakeyhi x1 + read_apiakeylo x2 + store_xregs x0, THREAD_CTX_REGS_APIAKEY_HI, 1, 2 +#endif + + bl thread_get_tmp_sp + pop x1, xzr /* Match "push x1, x30" above */ + mov x2, sp + str x2, [x19, #THREAD_CTX_REGS_SP] + ldr x20, [sp] /* Get pointer to rv[] */ + mov sp, x0 /* Switch to tmp stack */ + /* + * We need to read rv[] early, because thread_state_suspend + * can invoke virt_unset_guest() which will unmap pages, + * where rv[] resides + */ + load_wregs x20, 0, 21, 23 /* Load rv[] into w20-w22 */ + + adr x2, .thread_rpc_return + mov w0, #THREAD_FLAGS_COPY_ARGS_ON_RETURN + bl thread_state_suspend + mov x4, x0 /* Supply thread index */ + ldr w0, =TEESMC_OPTEED_RETURN_CALL_DONE + mov x1, x21 + mov x2, x22 + mov x3, x23 + smc #0 + /* SMC should not return */ + panic_at_smc_return + +.thread_rpc_return: + /* + * At this point has the stack pointer been restored to the value + * stored in THREAD_CTX above. + * + * Jumps here from thread_resume above when RPC has returned. The + * IRQ and FIQ bits are restored to what they where when this + * function was originally entered. + */ + pop x16, xzr /* Get pointer to rv[] */ + store_wregs x16, 0, 0, 3 /* Store w0-w3 into rv[] */ + ret +END_FUNC thread_rpc_spsr +DECLARE_KEEP_PAGER thread_rpc_spsr + +/* + * void thread_foreign_intr_exit(uint32_t thread_index) + * + * This function is jumped to at the end of macro foreign_intr_handler(). + * The current thread as indicated by @thread_index has just been + * suspended. The job here is just to inform normal world the thread id to + * resume when returning. + */ +FUNC thread_foreign_intr_exit , : + mov w4, w0 + ldr w0, =TEESMC_OPTEED_RETURN_CALL_DONE + ldr w1, =OPTEE_SMC_RETURN_RPC_FOREIGN_INTR + mov w2, #0 + mov w3, #0 + smc #0 + /* SMC should not return */ + panic_at_smc_return +END_FUNC thread_foreign_intr_exit + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/thread_spmc.c b/optee_os/core/arch/arm/kernel/thread_spmc.c new file mode 100644 index 0000000..59cdb51 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_spmc.c @@ -0,0 +1,1976 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020-2021, Linaro Limited. + * Copyright (c) 2019-2021, Arm Limited. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CFG_CORE_SEL1_SPMC) +struct mem_share_state { + struct mobj_ffa *mf; + unsigned int page_count; + unsigned int region_count; + unsigned int current_page_idx; +}; + +struct mem_frag_state { + struct mem_share_state share; + tee_mm_entry_t *mm; + unsigned int frag_offset; + SLIST_ENTRY(mem_frag_state) link; +}; +#endif + +/* Initialized in spmc_init() below */ +static uint16_t my_endpoint_id __nex_bss; +#ifdef CFG_CORE_SEL1_SPMC +static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | + FFA_PART_PROP_DIRECT_REQ_SEND | +#ifdef CFG_NS_VIRTUALIZATION + FFA_PART_PROP_NOTIF_CREATED | + FFA_PART_PROP_NOTIF_DESTROYED | +#endif +#ifdef ARM64 + FFA_PART_PROP_AARCH64_STATE | +#endif + FFA_PART_PROP_IS_PE_ID; + +static uint32_t my_uuid_words[] = { + /* + * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 + * SP, or + * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a + * logical partition, residing in the same exception level as the + * SPMC + * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b + */ + 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, +}; + +/* + * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. + * + * struct ffa_rxtx::spin_lock protects the variables below from concurrent + * access this includes the use of content of struct ffa_rxtx::rx and + * @frag_state_head. + * + * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct + * ffa_rxtx::tx and false when it is owned by normal world. + * + * Note that we can't prevent normal world from updating the content of + * these buffers so we must always be careful when reading. while we hold + * the lock. + */ + +static struct ffa_rxtx my_rxtx __nex_bss; + +static bool is_nw_buf(struct ffa_rxtx *rxtx) +{ + return rxtx == &my_rxtx; +} + +static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = + SLIST_HEAD_INITIALIZER(&frag_state_head); +#else +static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); +static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); +static struct ffa_rxtx my_rxtx = { + .rx = __rx_buf, + .tx = __tx_buf, + .size = sizeof(__rx_buf), +}; +#endif + +static uint32_t swap_src_dst(uint32_t src_dst) +{ + return (src_dst >> 16) | (src_dst << 16); +} + +static uint16_t get_sender_id(uint32_t src_dst) +{ + return src_dst >> 16; +} + +void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, + uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5) +{ + *args = (struct thread_smc_args){ .a0 = fid, + .a1 = src_dst, + .a2 = w2, + .a3 = w3, + .a4 = w4, + .a5 = w5, }; +} + +uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) +{ + /* + * No locking, if the caller does concurrent calls to this it's + * only making a mess for itself. We must be able to renegotiate + * the FF-A version in order to support differing versions between + * the loader and the driver. + */ + if (vers < FFA_VERSION_1_1) + rxtx->ffa_vers = FFA_VERSION_1_0; + else + rxtx->ffa_vers = FFA_VERSION_1_1; + + return rxtx->ffa_vers; +} + +#if defined(CFG_CORE_SEL1_SPMC) +static void handle_features(struct thread_smc_args *args) +{ + uint32_t ret_fid = 0; + uint32_t ret_w2 = FFA_PARAM_MBZ; + + switch (args->a1) { +#ifdef ARM64 + case FFA_RXTX_MAP_64: +#endif + case FFA_RXTX_MAP_32: + ret_fid = FFA_SUCCESS_32; + ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ + break; +#ifdef ARM64 + case FFA_MEM_SHARE_64: +#endif + case FFA_MEM_SHARE_32: + ret_fid = FFA_SUCCESS_32; + /* + * Partition manager supports transmission of a memory + * transaction descriptor in a buffer dynamically allocated + * by the endpoint. + */ + ret_w2 = BIT(0); + break; + + case FFA_ERROR: + case FFA_VERSION: + case FFA_SUCCESS_32: +#ifdef ARM64 + case FFA_SUCCESS_64: +#endif + case FFA_FEATURES: + case FFA_SPM_ID_GET: + case FFA_MEM_FRAG_TX: + case FFA_MEM_RECLAIM: + case FFA_MSG_SEND_DIRECT_REQ_64: + case FFA_MSG_SEND_DIRECT_REQ_32: + case FFA_INTERRUPT: + case FFA_PARTITION_INFO_GET: + case FFA_RXTX_UNMAP: + case FFA_RX_RELEASE: + case FFA_FEATURE_MANAGED_EXIT_INTR: + ret_fid = FFA_SUCCESS_32; + break; + default: + ret_fid = FFA_ERROR; + ret_w2 = FFA_NOT_SUPPORTED; + break; + } + + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) +{ + tee_mm_entry_t *mm = NULL; + + if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) + return FFA_INVALID_PARAMETERS; + + mm = tee_mm_alloc(&tee_mm_shm, sz); + if (!mm) + return FFA_NO_MEMORY; + + if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, + sz / SMALL_PAGE_SIZE, + MEM_AREA_NSEC_SHM)) { + tee_mm_free(mm); + return FFA_INVALID_PARAMETERS; + } + + *va_ret = (void *)tee_mm_get_smem(mm); + return 0; +} + +static void handle_spm_id_get(struct thread_smc_args *args) +{ + spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, my_endpoint_id, + FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static void unmap_buf(void *va, size_t sz) +{ + tee_mm_entry_t *mm = tee_mm_find(&tee_mm_shm, (vaddr_t)va); + + assert(mm); + core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); + tee_mm_free(mm); +} + +void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *rxtx) +{ + int rc = 0; + uint32_t ret_fid = FFA_ERROR; + unsigned int sz = 0; + paddr_t rx_pa = 0; + paddr_t tx_pa = 0; + void *rx = NULL; + void *tx = NULL; + + cpu_spin_lock(&rxtx->spinlock); + + if (args->a3 & GENMASK_64(63, 6)) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + + sz = args->a3 * SMALL_PAGE_SIZE; + if (!sz) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + /* TX/RX are swapped compared to the caller */ + tx_pa = args->a2; + rx_pa = args->a1; + + if (rxtx->size) { + rc = FFA_DENIED; + goto out; + } + + /* + * If the buffer comes from a SP the address is virtual and already + * mapped. + */ + if (is_nw_buf(rxtx)) { + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; + bool tx_alloced = false; + + /* + * With virtualization we establish this mapping in + * the nexus mapping which then is replicated to + * each partition. + * + * This means that this mapping must be done before + * any partition is created and then must not be + * changed. + */ + + /* + * core_mmu_add_mapping() may reuse previous + * mappings. First check if there's any mappings to + * reuse so we know how to clean up in case of + * failure. + */ + tx = phys_to_virt(tx_pa, mt, sz); + rx = phys_to_virt(rx_pa, mt, sz); + if (!tx) { + tx = core_mmu_add_mapping(mt, tx_pa, sz); + if (!tx) { + rc = FFA_NO_MEMORY; + goto out; + } + tx_alloced = true; + } + if (!rx) + rx = core_mmu_add_mapping(mt, rx_pa, sz); + + if (!rx) { + if (tx_alloced && tx) + core_mmu_remove_mapping(mt, tx, sz); + rc = FFA_NO_MEMORY; + goto out; + } + } else { + rc = map_buf(tx_pa, sz, &tx); + if (rc) + goto out; + rc = map_buf(rx_pa, sz, &rx); + if (rc) { + unmap_buf(tx, sz); + goto out; + } + } + rxtx->tx = tx; + rxtx->rx = rx; + } else { + if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + + if (!virt_to_phys((void *)tx_pa) || + !virt_to_phys((void *)rx_pa)) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + + rxtx->tx = (void *)tx_pa; + rxtx->rx = (void *)rx_pa; + } + + rxtx->size = sz; + rxtx->tx_is_mine = true; + ret_fid = FFA_SUCCESS_32; + DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); + DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); +out: + cpu_spin_unlock(&rxtx->spinlock); + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *rxtx) +{ + uint32_t ret_fid = FFA_ERROR; + int rc = FFA_INVALID_PARAMETERS; + + cpu_spin_lock(&rxtx->spinlock); + + if (!rxtx->size) + goto out; + + /* We don't unmap the SP memory as the SP might still use it */ + if (is_nw_buf(rxtx)) { + unmap_buf(rxtx->rx, rxtx->size); + unmap_buf(rxtx->tx, rxtx->size); + } + rxtx->size = 0; + rxtx->rx = NULL; + rxtx->tx = NULL; + ret_fid = FFA_SUCCESS_32; + rc = 0; +out: + cpu_spin_unlock(&rxtx->spinlock); + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *rxtx) +{ + uint32_t ret_fid = 0; + int rc = 0; + + cpu_spin_lock(&rxtx->spinlock); + /* The senders RX is our TX */ + if (!rxtx->size || rxtx->tx_is_mine) { + ret_fid = FFA_ERROR; + rc = FFA_DENIED; + } else { + ret_fid = FFA_SUCCESS_32; + rc = 0; + rxtx->tx_is_mine = true; + } + cpu_spin_unlock(&rxtx->spinlock); + + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) +{ + return !w0 && !w1 && !w2 && !w3; +} + +static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) +{ + /* + * This depends on which UUID we have been assigned. + * TODO add a generic mechanism to obtain our UUID. + * + * The test below is for the hard coded UUID + * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b + */ + return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && + w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; +} + +TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, + size_t idx, uint16_t endpoint_id, + uint16_t execution_context, + uint32_t part_props, + const uint32_t uuid_words[4]) +{ + struct ffa_partition_info_x *fpi = NULL; + size_t fpi_size = sizeof(*fpi); + + if (ffa_vers >= FFA_VERSION_1_1) + fpi_size += FFA_UUID_SIZE; + + if ((idx + 1) * fpi_size > blen) + return TEE_ERROR_OUT_OF_MEMORY; + + fpi = (void *)((vaddr_t)buf + idx * fpi_size); + fpi->id = endpoint_id; + /* Number of execution contexts implemented by this partition */ + fpi->execution_context = execution_context; + + fpi->partition_properties = part_props; + + if (ffa_vers >= FFA_VERSION_1_1) { + if (uuid_words) + memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); + else + memset(fpi->uuid, 0, FFA_UUID_SIZE); + } + + return TEE_SUCCESS; +} + +static int handle_partition_info_get_all(size_t *elem_count, + struct ffa_rxtx *rxtx, bool count_only) +{ + if (!count_only) { + /* Add OP-TEE SP */ + if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, + rxtx->size, 0, my_endpoint_id, + CFG_TEE_CORE_NB_CORE, + my_part_props, my_uuid_words)) + return FFA_NO_MEMORY; + } + *elem_count = 1; + + if (IS_ENABLED(CFG_SECURE_PARTITION)) { + if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, + NULL, elem_count, count_only)) + return FFA_NO_MEMORY; + } + + return FFA_OK; +} + +void spmc_handle_partition_info_get(struct thread_smc_args *args, + struct ffa_rxtx *rxtx) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t ret_fid = FFA_ERROR; + uint32_t rc = 0; + bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; + + if (!count_only) { + cpu_spin_lock(&rxtx->spinlock); + + if (!rxtx->size || !rxtx->tx_is_mine) { + rc = FFA_BUSY; + goto out; + } + } + + if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { + size_t elem_count = 0; + + ret_fid = handle_partition_info_get_all(&elem_count, rxtx, + count_only); + + if (ret_fid) { + rc = ret_fid; + ret_fid = FFA_ERROR; + } else { + ret_fid = FFA_SUCCESS_32; + rc = elem_count; + } + + goto out; + } + + if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { + if (!count_only) { + res = spmc_fill_partition_entry(rxtx->ffa_vers, + rxtx->tx, rxtx->size, 0, + my_endpoint_id, + CFG_TEE_CORE_NB_CORE, + my_part_props, + my_uuid_words); + if (res) { + ret_fid = FFA_ERROR; + rc = FFA_INVALID_PARAMETERS; + goto out; + } + } + rc = 1; + } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { + uint32_t uuid_array[4] = { 0 }; + TEE_UUID uuid = { }; + size_t count = 0; + + uuid_array[0] = args->a1; + uuid_array[1] = args->a2; + uuid_array[2] = args->a3; + uuid_array[3] = args->a4; + tee_uuid_from_octets(&uuid, (uint8_t *)uuid_array); + + res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, + rxtx->size, &uuid, &count, + count_only); + if (res != TEE_SUCCESS) { + ret_fid = FFA_ERROR; + rc = FFA_INVALID_PARAMETERS; + goto out; + } + rc = count; + } else { + ret_fid = FFA_ERROR; + rc = FFA_INVALID_PARAMETERS; + goto out; + } + + ret_fid = FFA_SUCCESS_32; + +out: + spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); + if (!count_only) { + rxtx->tx_is_mine = false; + cpu_spin_unlock(&rxtx->spinlock); + } +} + +static void spmc_handle_run(struct thread_smc_args *args) +{ + uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); + uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); + uint32_t rc = FFA_OK; + + if (endpoint != my_endpoint_id) { + /* + * The endpoint should be an SP, try to resume the SP from + * preempted into busy state. + */ + rc = spmc_sp_resume_from_preempted(endpoint); + if (rc) + goto out; + } + + thread_resume_from_rpc(thread_id, 0, 0, 0, 0); + + /* thread_resume_from_rpc return only of the thread_id is invalid */ + rc = FFA_INVALID_PARAMETERS; + +out: + spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, + FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} +#endif /*CFG_CORE_SEL1_SPMC*/ + +static void handle_yielding_call(struct thread_smc_args *args, + uint32_t direct_resp_fid) +{ + TEE_Result res = 0; + + thread_check_canaries(); + +#ifdef ARM64 + /* Saving this for an eventual RPC */ + thread_get_core_local()->direct_resp_fid = direct_resp_fid; +#endif + + if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { + /* Note connection to struct thread_rpc_arg::ret */ + thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, + 0); + res = TEE_ERROR_BAD_PARAMETERS; + } else { + thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, + args->a6, args->a7); + res = TEE_ERROR_BUSY; + } + spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), + 0, res, 0, 0); +} + +static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) +{ + uint64_t cookie = reg_pair_to_64(a5, a4); + uint32_t res = 0; + + res = mobj_ffa_unregister_by_cookie(cookie); + switch (res) { + case TEE_SUCCESS: + case TEE_ERROR_ITEM_NOT_FOUND: + return 0; + case TEE_ERROR_BUSY: + EMSG("res %#"PRIx32, res); + return FFA_BUSY; + default: + EMSG("res %#"PRIx32, res); + return FFA_INVALID_PARAMETERS; + } +} + +static void handle_blocking_call(struct thread_smc_args *args, + uint32_t direct_resp_fid) +{ + switch (args->a3) { + case OPTEE_FFA_GET_API_VERSION: + spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, + OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, + 0); + break; + case OPTEE_FFA_GET_OS_VERSION: + spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, + CFG_OPTEE_REVISION_MAJOR, + CFG_OPTEE_REVISION_MINOR, TEE_IMPL_GIT_SHA1); + break; + case OPTEE_FFA_EXCHANGE_CAPABILITIES: + spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, + 0, THREAD_RPC_MAX_NUM_PARAMS, + OPTEE_FFA_SEC_CAP_ARG_OFFSET); + break; + case OPTEE_FFA_UNREGISTER_SHM: + spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, + handle_unregister_shm(args->a4, args->a5), 0, 0); + break; + default: + EMSG("Unhandled blocking service ID %#"PRIx32, + (uint32_t)args->a3); + spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, + TEE_ERROR_BAD_PARAMETERS, 0, 0); + } +} + +static void handle_framework_direct_request(struct thread_smc_args *args, + struct ffa_rxtx *rxtx, + uint32_t direct_resp_fid) +{ + uint32_t w0 = FFA_ERROR; + uint32_t w1 = FFA_PARAM_MBZ; + uint32_t w2 = FFA_NOT_SUPPORTED; + uint32_t w3 = FFA_PARAM_MBZ; + + switch (args->a2 & FFA_MSG_TYPE_MASK) { + case FFA_MSG_SEND_VM_CREATED: + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + uint16_t guest_id = args->a5; + TEE_Result res = virt_guest_created(guest_id); + + w0 = direct_resp_fid; + w1 = swap_src_dst(args->a1); + w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; + if (res == TEE_SUCCESS) + w3 = FFA_OK; + else if (res == TEE_ERROR_OUT_OF_MEMORY) + w3 = FFA_DENIED; + else + w3 = FFA_INVALID_PARAMETERS; + } + break; + case FFA_MSG_SEND_VM_DESTROYED: + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + uint16_t guest_id = args->a5; + TEE_Result res = virt_guest_destroyed(guest_id); + + w0 = direct_resp_fid; + w1 = swap_src_dst(args->a1); + w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; + if (res == TEE_SUCCESS) + w3 = FFA_OK; + else + w3 = FFA_INVALID_PARAMETERS; + } + break; + case FFA_MSG_VERSION_REQ: + w0 = direct_resp_fid; + w1 = swap_src_dst(args->a1); + w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; + w3 = spmc_exchange_version(args->a3, rxtx); + break; + default: + break; + } + spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); +} + +static void handle_direct_request(struct thread_smc_args *args, + struct ffa_rxtx *rxtx) +{ + uint32_t direct_resp_fid = 0; + + if (IS_ENABLED(CFG_SECURE_PARTITION) && + FFA_DST(args->a1) != my_endpoint_id) { + spmc_sp_start_thread(args); + return; + } + + if (OPTEE_SMC_IS_64(args->a0)) + direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; + else + direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; + + if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { + handle_framework_direct_request(args, rxtx, direct_resp_fid); + return; + } + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && + virt_set_guest(get_sender_id(args->a1))) { + spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, + TEE_ERROR_ITEM_NOT_FOUND, 0, 0); + return; + } + + if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) + handle_yielding_call(args, direct_resp_fid); + else + handle_blocking_call(args, direct_resp_fid); + + /* + * Note that handle_yielding_call() typically only returns if a + * thread cannot be allocated or found. virt_unset_guest() is also + * called from thread_state_suspend() and thread_state_free(). + */ + virt_unset_guest(); +} + +int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, + struct ffa_mem_transaction_x *trans) +{ + uint16_t mem_reg_attr = 0; + uint32_t flags = 0; + uint32_t count = 0; + uint32_t offs = 0; + uint32_t size = 0; + size_t n = 0; + + if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) + return FFA_INVALID_PARAMETERS; + + if (ffa_vers >= FFA_VERSION_1_1) { + struct ffa_mem_transaction_1_1 *descr = NULL; + + if (blen < sizeof(*descr)) + return FFA_INVALID_PARAMETERS; + + descr = buf; + trans->sender_id = READ_ONCE(descr->sender_id); + mem_reg_attr = READ_ONCE(descr->mem_reg_attr); + flags = READ_ONCE(descr->flags); + trans->global_handle = READ_ONCE(descr->global_handle); + trans->tag = READ_ONCE(descr->tag); + + count = READ_ONCE(descr->mem_access_count); + size = READ_ONCE(descr->mem_access_size); + offs = READ_ONCE(descr->mem_access_offs); + } else { + struct ffa_mem_transaction_1_0 *descr = NULL; + + if (blen < sizeof(*descr)) + return FFA_INVALID_PARAMETERS; + + descr = buf; + trans->sender_id = READ_ONCE(descr->sender_id); + mem_reg_attr = READ_ONCE(descr->mem_reg_attr); + flags = READ_ONCE(descr->flags); + trans->global_handle = READ_ONCE(descr->global_handle); + trans->tag = READ_ONCE(descr->tag); + + count = READ_ONCE(descr->mem_access_count); + size = sizeof(struct ffa_mem_access); + offs = offsetof(struct ffa_mem_transaction_1_0, + mem_access_array); + } + + if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || + size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) + return FFA_INVALID_PARAMETERS; + + /* Check that the endpoint memory access descriptor array fits */ + if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || + n > blen) + return FFA_INVALID_PARAMETERS; + + trans->mem_reg_attr = mem_reg_attr; + trans->flags = flags; + trans->mem_access_size = size; + trans->mem_access_count = count; + trans->mem_access_offs = offs; + return 0; +} + +#if defined(CFG_CORE_SEL1_SPMC) +static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, + unsigned int mem_access_count, uint8_t *acc_perms, + unsigned int *region_offs) +{ + struct ffa_mem_access_perm *descr = NULL; + struct ffa_mem_access *mem_acc = NULL; + unsigned int n = 0; + + for (n = 0; n < mem_access_count; n++) { + mem_acc = (void *)(mem_acc_base + mem_access_size * n); + descr = &mem_acc->access_perm; + if (READ_ONCE(descr->endpoint_id) == my_endpoint_id) { + *acc_perms = READ_ONCE(descr->perm); + *region_offs = READ_ONCE(mem_acc[n].region_offs); + return 0; + } + } + + return FFA_INVALID_PARAMETERS; +} + +static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, + size_t blen, unsigned int *page_count, + unsigned int *region_count, size_t *addr_range_offs) +{ + const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; + const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; + struct ffa_mem_region *region_descr = NULL; + unsigned int region_descr_offs = 0; + uint8_t mem_acc_perm = 0; + size_t n = 0; + + if (mem_trans->mem_reg_attr != exp_mem_reg_attr) + return FFA_INVALID_PARAMETERS; + + /* Check that the access permissions matches what's expected */ + if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, + mem_trans->mem_access_size, + mem_trans->mem_access_count, + &mem_acc_perm, ®ion_descr_offs) || + mem_acc_perm != exp_mem_acc_perm) + return FFA_INVALID_PARAMETERS; + + /* Check that the Composite memory region descriptor fits */ + if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || + n > blen) + return FFA_INVALID_PARAMETERS; + + if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, + struct ffa_mem_region)) + return FFA_INVALID_PARAMETERS; + + region_descr = (struct ffa_mem_region *)((vaddr_t)buf + + region_descr_offs); + *page_count = READ_ONCE(region_descr->total_page_count); + *region_count = READ_ONCE(region_descr->address_range_count); + *addr_range_offs = n; + return 0; +} + +static int add_mem_share_helper(struct mem_share_state *s, void *buf, + size_t flen) +{ + unsigned int region_count = flen / sizeof(struct ffa_address_range); + struct ffa_address_range *arange = NULL; + unsigned int n = 0; + + if (region_count > s->region_count) + region_count = s->region_count; + + if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) + return FFA_INVALID_PARAMETERS; + arange = buf; + + for (n = 0; n < region_count; n++) { + unsigned int page_count = READ_ONCE(arange[n].page_count); + uint64_t addr = READ_ONCE(arange[n].address); + + if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, + addr, page_count)) + return FFA_INVALID_PARAMETERS; + } + + s->region_count -= region_count; + if (s->region_count) + return region_count * sizeof(*arange); + + if (s->current_page_idx != s->page_count) + return FFA_INVALID_PARAMETERS; + + return 0; +} + +static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) +{ + int rc = 0; + + rc = add_mem_share_helper(&s->share, buf, flen); + if (rc >= 0) { + if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { + /* We're not at the end of the descriptor yet */ + if (s->share.region_count) + return s->frag_offset; + + /* We're done */ + rc = 0; + } else { + rc = FFA_INVALID_PARAMETERS; + } + } + + SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); + if (rc < 0) + mobj_ffa_sel1_spmc_delete(s->share.mf); + else + mobj_ffa_push_to_inactive(s->share.mf); + free(s); + + return rc; +} + +static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, + void *buf) +{ + struct ffa_mem_access_perm *perm = NULL; + struct ffa_mem_access *mem_acc = NULL; + + if (!IS_ENABLED(CFG_SECURE_PARTITION)) + return false; + + if (mem_trans->mem_access_count < 1) + return false; + + mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); + perm = &mem_acc->access_perm; + + /* + * perm->endpoint_id is read here only to check if the endpoint is + * OP-TEE. We do read it later on again, but there are some additional + * checks there to make sure that the data is correct. + */ + return READ_ONCE(perm->endpoint_id) != my_endpoint_id; +} + +static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, + tee_mm_entry_t *mm, void *buf, size_t blen, + size_t flen, uint64_t *global_handle) +{ + int rc = 0; + struct mem_share_state share = { }; + size_t addr_range_offs = 0; + uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; + size_t n = 0; + + rc = mem_share_init(mem_trans, buf, flen, &share.page_count, + &share.region_count, &addr_range_offs); + if (rc) + return rc; + + if (MUL_OVERFLOW(share.region_count, + sizeof(struct ffa_address_range), &n) || + ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) + return FFA_INVALID_PARAMETERS; + + if (mem_trans->global_handle) + cookie = mem_trans->global_handle; + share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); + if (!share.mf) + return FFA_NO_MEMORY; + + if (flen != blen) { + struct mem_frag_state *s = calloc(sizeof(*s), 1); + + if (!s) { + rc = FFA_NO_MEMORY; + goto err; + } + s->share = share; + s->mm = mm; + s->frag_offset = addr_range_offs; + + SLIST_INSERT_HEAD(&frag_state_head, s, link); + rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, + flen - addr_range_offs); + + if (rc >= 0) + *global_handle = mobj_ffa_get_cookie(share.mf); + + return rc; + } + + rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, + flen - addr_range_offs); + if (rc) { + /* + * Number of consumed bytes may be returned instead of 0 for + * done. + */ + rc = FFA_INVALID_PARAMETERS; + goto err; + } + + *global_handle = mobj_ffa_push_to_inactive(share.mf); + + return 0; +err: + mobj_ffa_sel1_spmc_delete(share.mf); + return rc; +} + +static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, + unsigned int page_count, + uint64_t *global_handle, struct ffa_rxtx *rxtx) +{ + struct ffa_mem_transaction_x mem_trans = { }; + int rc = 0; + size_t len = 0; + void *buf = NULL; + tee_mm_entry_t *mm = NULL; + vaddr_t offs = pbuf & SMALL_PAGE_MASK; + + if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) + return FFA_INVALID_PARAMETERS; + if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) + return FFA_INVALID_PARAMETERS; + + /* + * Check that the length reported in flen is covered by len even + * if the offset is taken into account. + */ + if (len < flen || len - offs < flen) + return FFA_INVALID_PARAMETERS; + + mm = tee_mm_alloc(&tee_mm_shm, len); + if (!mm) + return FFA_NO_MEMORY; + + if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, + page_count, MEM_AREA_NSEC_SHM)) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + buf = (void *)(tee_mm_get_smem(mm) + offs); + + cpu_spin_lock(&rxtx->spinlock); + rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); + if (!rc && IS_ENABLED(CFG_NS_VIRTUALIZATION) && + virt_set_guest(mem_trans.sender_id)) + rc = FFA_DENIED; + if (!rc) + rc = add_mem_share(&mem_trans, mm, buf, blen, flen, + global_handle); + virt_unset_guest(); + cpu_spin_unlock(&rxtx->spinlock); + if (rc > 0) + return rc; + + core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); +out: + tee_mm_free(mm); + return rc; +} + +static int handle_mem_share_rxbuf(size_t blen, size_t flen, + uint64_t *global_handle, + struct ffa_rxtx *rxtx) +{ + struct ffa_mem_transaction_x mem_trans = { }; + int rc = FFA_DENIED; + + cpu_spin_lock(&rxtx->spinlock); + + if (!rxtx->rx || flen > rxtx->size) + goto out; + + rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, + &mem_trans); + if (rc) + goto out; + if (is_sp_share(&mem_trans, rxtx->rx)) { + rc = spmc_sp_add_share(&mem_trans, rxtx, blen, + global_handle, NULL); + goto out; + } + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && + virt_set_guest(mem_trans.sender_id)) + goto out; + + rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, + global_handle); + + virt_unset_guest(); + +out: + cpu_spin_unlock(&rxtx->spinlock); + + return rc; +} + +static void handle_mem_share(struct thread_smc_args *args, + struct ffa_rxtx *rxtx) +{ + uint32_t tot_len = args->a1; + uint32_t frag_len = args->a2; + uint64_t addr = args->a3; + uint32_t page_count = args->a4; + uint32_t ret_w1 = 0; + uint32_t ret_w2 = FFA_INVALID_PARAMETERS; + uint32_t ret_w3 = 0; + uint32_t ret_fid = FFA_ERROR; + uint64_t global_handle = 0; + int rc = 0; + + /* Check that the MBZs are indeed 0 */ + if (args->a5 || args->a6 || args->a7) + goto out; + + /* Check that fragment length doesn't exceed total length */ + if (frag_len > tot_len) + goto out; + + /* Check for 32-bit calling convention */ + if (args->a0 == FFA_MEM_SHARE_32) + addr &= UINT32_MAX; + + if (!addr) { + /* + * The memory transaction descriptor is passed via our rx + * buffer. + */ + if (page_count) + goto out; + rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, + rxtx); + } else { + rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, + &global_handle, rxtx); + } + if (rc < 0) { + ret_w2 = rc; + } else if (rc > 0) { + ret_fid = FFA_MEM_FRAG_RX; + ret_w3 = rc; + reg_pair_from_64(global_handle, &ret_w2, &ret_w1); + } else { + ret_fid = FFA_SUCCESS_32; + reg_pair_from_64(global_handle, &ret_w3, &ret_w2); + } +out: + spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); +} + +static struct mem_frag_state *get_frag_state(uint64_t global_handle) +{ + struct mem_frag_state *s = NULL; + + SLIST_FOREACH(s, &frag_state_head, link) + if (mobj_ffa_get_cookie(s->share.mf) == global_handle) + return s; + + return NULL; +} + +static void handle_mem_frag_tx(struct thread_smc_args *args, + struct ffa_rxtx *rxtx) +{ + uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); + size_t flen = args->a3; + uint32_t endpoint_id = args->a4; + struct mem_frag_state *s = NULL; + tee_mm_entry_t *mm = NULL; + unsigned int page_count = 0; + void *buf = NULL; + uint32_t ret_w1 = 0; + uint32_t ret_w2 = 0; + uint32_t ret_w3 = 0; + uint32_t ret_fid = 0; + int rc = 0; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + uint16_t guest_id = endpoint_id >> 16; + + if (!guest_id || virt_set_guest(guest_id)) { + rc = FFA_INVALID_PARAMETERS; + goto out_set_rc; + } + } + + /* + * Currently we're only doing this for fragmented FFA_MEM_SHARE_* + * requests. + */ + + cpu_spin_lock(&rxtx->spinlock); + + s = get_frag_state(global_handle); + if (!s) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + + mm = s->mm; + if (mm) { + if (flen > tee_mm_get_bytes(mm)) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + page_count = s->share.page_count; + buf = (void *)tee_mm_get_smem(mm); + } else { + if (flen > rxtx->size) { + rc = FFA_INVALID_PARAMETERS; + goto out; + } + buf = rxtx->rx; + } + + rc = add_mem_share_frag(s, buf, flen); +out: + virt_unset_guest(); + cpu_spin_unlock(&rxtx->spinlock); + + if (rc <= 0 && mm) { + core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); + tee_mm_free(mm); + } + +out_set_rc: + if (rc < 0) { + ret_fid = FFA_ERROR; + ret_w2 = rc; + } else if (rc > 0) { + ret_fid = FFA_MEM_FRAG_RX; + ret_w3 = rc; + reg_pair_from_64(global_handle, &ret_w2, &ret_w1); + } else { + ret_fid = FFA_SUCCESS_32; + reg_pair_from_64(global_handle, &ret_w3, &ret_w2); + } + + spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); +} + +static void handle_mem_reclaim(struct thread_smc_args *args) +{ + uint32_t ret_val = FFA_INVALID_PARAMETERS; + uint32_t ret_fid = FFA_ERROR; + uint64_t cookie = 0; + + if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) + goto out; + + cookie = reg_pair_to_64(args->a2, args->a1); + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + uint16_t guest_id = 0; + + if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { + guest_id = virt_find_guest_by_cookie(cookie); + } else { + guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & + FFA_MEMORY_HANDLE_PRTN_MASK; + } + if (!guest_id || virt_set_guest(guest_id)) + goto out; + } + + switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { + case TEE_SUCCESS: + ret_fid = FFA_SUCCESS_32; + ret_val = 0; + break; + case TEE_ERROR_ITEM_NOT_FOUND: + DMSG("cookie %#"PRIx64" not found", cookie); + ret_val = FFA_INVALID_PARAMETERS; + break; + default: + DMSG("cookie %#"PRIx64" busy", cookie); + ret_val = FFA_DENIED; + break; + } + + virt_unset_guest(); + +out: + spmc_set_args(args, ret_fid, ret_val, 0, 0, 0, 0); +} +#endif + +/* Only called from assembly */ +void thread_spmc_msg_recv(struct thread_smc_args *args); +void thread_spmc_msg_recv(struct thread_smc_args *args) +{ + assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); + switch (args->a0) { +#if defined(CFG_CORE_SEL1_SPMC) + case FFA_FEATURES: + handle_features(args); + break; + case FFA_SPM_ID_GET: + handle_spm_id_get(args); + break; +#ifdef ARM64 + case FFA_RXTX_MAP_64: +#endif + case FFA_RXTX_MAP_32: + spmc_handle_rxtx_map(args, &my_rxtx); + break; + case FFA_RXTX_UNMAP: + spmc_handle_rxtx_unmap(args, &my_rxtx); + break; + case FFA_RX_RELEASE: + spmc_handle_rx_release(args, &my_rxtx); + break; + case FFA_PARTITION_INFO_GET: + spmc_handle_partition_info_get(args, &my_rxtx); + break; + case FFA_RUN: + spmc_handle_run(args); + break; +#endif /*CFG_CORE_SEL1_SPMC*/ + case FFA_INTERRUPT: + interrupt_main_handler(); + if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) + spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, + 0, 0); + else + spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); + break; +#ifdef ARM64 + case FFA_MSG_SEND_DIRECT_REQ_64: +#endif + case FFA_MSG_SEND_DIRECT_REQ_32: + handle_direct_request(args, &my_rxtx); + break; +#if defined(CFG_CORE_SEL1_SPMC) +#ifdef ARM64 + case FFA_MEM_SHARE_64: +#endif + case FFA_MEM_SHARE_32: + handle_mem_share(args, &my_rxtx); + break; + case FFA_MEM_RECLAIM: + if (!IS_ENABLED(CFG_SECURE_PARTITION) || + !ffa_mem_reclaim(args, NULL)) + handle_mem_reclaim(args); + break; + case FFA_MEM_FRAG_TX: + handle_mem_frag_tx(args, &my_rxtx); + break; +#endif /*CFG_CORE_SEL1_SPMC*/ + default: + EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); + spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, + FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); + } +} + +static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) +{ + size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + struct thread_ctx *thr = threads + thread_get_id(); + TEE_Result res = TEE_ERROR_BAD_PARAMETERS; + struct optee_msg_arg *arg = NULL; + struct mobj *mobj = NULL; + uint32_t num_params = 0; + size_t sz = 0; + + mobj = mobj_ffa_get_by_cookie(cookie, 0); + if (!mobj) { + EMSG("Can't find cookie %#"PRIx64, cookie); + return TEE_ERROR_BAD_PARAMETERS; + } + + res = mobj_inc_map(mobj); + if (res) + goto out_put_mobj; + + res = TEE_ERROR_BAD_PARAMETERS; + arg = mobj_get_va(mobj, offset, sizeof(*arg)); + if (!arg) + goto out_dec_map; + + num_params = READ_ONCE(arg->num_params); + if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) + goto out_dec_map; + + sz = OPTEE_MSG_GET_ARG_SIZE(num_params); + + thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); + if (!thr->rpc_arg) + goto out_dec_map; + + virt_on_stdcall(); + res = tee_entry_std(arg, num_params); + + thread_rpc_shm_cache_clear(&thr->shm_cache); + thr->rpc_arg = NULL; + +out_dec_map: + mobj_dec_map(mobj); +out_put_mobj: + mobj_put(mobj); + return res; +} + +/* + * Helper routine for the assembly function thread_std_smc_entry() + * + * Note: this function is weak just to make link_dummies_paged.c happy. + */ +uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5 __unused) +{ + /* + * Arguments are supplied from handle_yielding_call() as: + * a0 <- w1 + * a1 <- w3 + * a2 <- w4 + * a3 <- w5 + * a4 <- w6 + * a5 <- w7 + */ + thread_get_tsd()->rpc_target_info = swap_src_dst(a0); + if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) + return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); + return FFA_DENIED; +} + +static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) +{ + uint64_t offs = tpm->u.memref.offs; + + param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + + OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; + + param->u.fmem.offs_low = offs; + param->u.fmem.offs_high = offs >> 32; + if (param->u.fmem.offs_high != offs >> 32) + return false; + + param->u.fmem.size = tpm->u.memref.size; + if (tpm->u.memref.mobj) { + uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); + + /* If a mobj is passed it better be one with a valid cookie. */ + if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) + return false; + param->u.fmem.global_id = cookie; + } else { + param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; + } + + return true; +} + +static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, + struct thread_param *params, + struct optee_msg_arg **arg_ret) +{ + size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + struct thread_ctx *thr = threads + thread_get_id(); + struct optee_msg_arg *arg = thr->rpc_arg; + + if (num_params > THREAD_RPC_MAX_NUM_PARAMS) + return TEE_ERROR_BAD_PARAMETERS; + + if (!arg) { + EMSG("rpc_arg not set"); + return TEE_ERROR_GENERIC; + } + + memset(arg, 0, sz); + arg->cmd = cmd; + arg->num_params = num_params; + arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ + + for (size_t n = 0; n < num_params; n++) { + switch (params[n].attr) { + case THREAD_PARAM_ATTR_NONE: + arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; + break; + case THREAD_PARAM_ATTR_VALUE_IN: + case THREAD_PARAM_ATTR_VALUE_OUT: + case THREAD_PARAM_ATTR_VALUE_INOUT: + arg->params[n].attr = params[n].attr - + THREAD_PARAM_ATTR_VALUE_IN + + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; + arg->params[n].u.value.a = params[n].u.value.a; + arg->params[n].u.value.b = params[n].u.value.b; + arg->params[n].u.value.c = params[n].u.value.c; + break; + case THREAD_PARAM_ATTR_MEMREF_IN: + case THREAD_PARAM_ATTR_MEMREF_OUT: + case THREAD_PARAM_ATTR_MEMREF_INOUT: + if (!set_fmem(arg->params + n, params + n)) + return TEE_ERROR_BAD_PARAMETERS; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + } + + if (arg_ret) + *arg_ret = arg; + + return TEE_SUCCESS; +} + +static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, + struct thread_param *params) +{ + for (size_t n = 0; n < num_params; n++) { + switch (params[n].attr) { + case THREAD_PARAM_ATTR_VALUE_OUT: + case THREAD_PARAM_ATTR_VALUE_INOUT: + params[n].u.value.a = arg->params[n].u.value.a; + params[n].u.value.b = arg->params[n].u.value.b; + params[n].u.value.c = arg->params[n].u.value.c; + break; + case THREAD_PARAM_ATTR_MEMREF_OUT: + case THREAD_PARAM_ATTR_MEMREF_INOUT: + params[n].u.memref.size = arg->params[n].u.fmem.size; + break; + default: + break; + } + } + + return arg->ret; +} + +uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, + struct thread_param *params) +{ + struct thread_rpc_arg rpc_arg = { .call = { + .w1 = thread_get_tsd()->rpc_target_info, + .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, + }, + }; + struct optee_msg_arg *arg = NULL; + uint32_t ret = 0; + + ret = get_rpc_arg(cmd, num_params, params, &arg); + if (ret) + return ret; + + thread_rpc(&rpc_arg); + + return get_rpc_arg_res(arg, num_params, params); +} + +static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) +{ + struct thread_rpc_arg rpc_arg = { .call = { + .w1 = thread_get_tsd()->rpc_target_info, + .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, + }, + }; + struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); + uint32_t res2 = 0; + uint32_t res = 0; + + DMSG("freeing cookie %#"PRIx64, cookie); + + res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); + + mobj_put(mobj); + res2 = mobj_ffa_unregister_by_cookie(cookie); + if (res2) + DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, + cookie, res2); + if (!res) + thread_rpc(&rpc_arg); +} + +static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) +{ + struct thread_rpc_arg rpc_arg = { .call = { + .w1 = thread_get_tsd()->rpc_target_info, + .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, + }, + }; + struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); + struct optee_msg_arg *arg = NULL; + unsigned int internal_offset = 0; + struct mobj *mobj = NULL; + uint64_t cookie = 0; + + if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) + return NULL; + + thread_rpc(&rpc_arg); + + if (arg->num_params != 1 || + arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) + return NULL; + + internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); + cookie = READ_ONCE(arg->params->u.fmem.global_id); + mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); + if (!mobj) { + DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", + cookie, internal_offset); + return NULL; + } + + assert(mobj_is_nonsec(mobj)); + + if (mobj->size < size) { + DMSG("Mobj %#"PRIx64": wrong size", cookie); + mobj_put(mobj); + return NULL; + } + + if (mobj_inc_map(mobj)) { + DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); + mobj_put(mobj); + return NULL; + } + + return mobj; +} + +struct mobj *thread_rpc_alloc_payload(size_t size) +{ + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); +} + +struct mobj *thread_rpc_alloc_kernel_payload(size_t size) +{ + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); +} + +void thread_rpc_free_kernel_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, mobj_get_cookie(mobj), mobj); +} + +void thread_rpc_free_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), + mobj); +} + +struct mobj *thread_rpc_alloc_global_payload(size_t size) +{ + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); +} + +void thread_rpc_free_global_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, mobj_get_cookie(mobj), + mobj); +} + +void thread_spmc_register_secondary_ep(vaddr_t ep) +{ + unsigned long ret = 0; + + /* Let the SPM know the entry point for secondary CPUs */ + ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); + + if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) + EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); +} + +#if defined(CFG_CORE_SEL1_SPMC) +static TEE_Result spmc_init(void) +{ + my_endpoint_id = SPMC_ENDPOINT_ID; + DMSG("My endpoint ID %#x", my_endpoint_id); + + /* + * If SPMD think we are version 1.0 it will report version 1.0 to + * normal world regardless of what version we query the SPM with. + * However, if SPMD think we are version 1.1 it will forward + * queries from normal world to let us negotiate version. So by + * setting version 1.0 here we should be compatible. + * + * Note that disagreement on negotiated version means that we'll + * have communication problems with normal world. + */ + my_rxtx.ffa_vers = FFA_VERSION_1_0; + + return TEE_SUCCESS; +} +#else /* !defined(CFG_CORE_SEL1_SPMC) */ +static bool is_ffa_success(uint32_t fid) +{ +#ifdef ARM64 + if (fid == FFA_SUCCESS_64) + return true; +#endif + return fid == FFA_SUCCESS_32; +} + +static void spmc_rxtx_map(struct ffa_rxtx *rxtx) +{ + struct thread_smc_args args = { +#ifdef ARM64 + .a0 = FFA_RXTX_MAP_64, +#else + .a0 = FFA_RXTX_MAP_32, +#endif + .a1 = virt_to_phys(rxtx->tx), + .a2 = virt_to_phys(rxtx->rx), + .a3 = 1, + }; + + thread_smccc(&args); + if (!is_ffa_success(args.a0)) { + if (args.a0 == FFA_ERROR) + EMSG("rxtx map failed with error %ld", args.a2); + else + EMSG("rxtx map failed"); + panic(); + } +} + +static uint16_t get_my_id(void) +{ + struct thread_smc_args args = { + .a0 = FFA_ID_GET, + }; + + thread_smccc(&args); + if (!is_ffa_success(args.a0)) { + if (args.a0 == FFA_ERROR) + EMSG("Get id failed with error %ld", args.a2); + else + EMSG("Get id failed"); + panic(); + } + + return args.a2; +} + +static uint32_t get_ffa_version(uint32_t my_version) +{ + struct thread_smc_args args = { + .a0 = FFA_VERSION, + .a1 = my_version, + }; + + thread_smccc(&args); + if (args.a0 & BIT(31)) { + EMSG("FF-A version failed with error %ld", args.a0); + panic(); + } + + return args.a0; +} + +static void *spmc_retrieve_req(uint64_t cookie, + struct ffa_mem_transaction_x *trans) +{ + struct ffa_mem_access *acc_descr_array = NULL; + struct ffa_mem_access_perm *perm_descr = NULL; + struct thread_smc_args args = { + .a0 = FFA_MEM_RETRIEVE_REQ_32, + .a3 = 0, /* Address, Using TX -> MBZ */ + .a4 = 0, /* Using TX -> MBZ */ + }; + size_t size = 0; + int rc = 0; + + if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { + struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; + + size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); + memset(trans_descr, 0, size); + trans_descr->sender_id = thread_get_tsd()->rpc_target_info; + trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; + trans_descr->global_handle = cookie; + trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | + FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; + trans_descr->mem_access_count = 1; + acc_descr_array = trans_descr->mem_access_array; + } else { + struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; + + size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); + memset(trans_descr, 0, size); + trans_descr->sender_id = thread_get_tsd()->rpc_target_info; + trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; + trans_descr->global_handle = cookie; + trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | + FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; + trans_descr->mem_access_count = 1; + trans_descr->mem_access_offs = sizeof(*trans_descr); + trans_descr->mem_access_size = sizeof(struct ffa_mem_access); + acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + + sizeof(*trans_descr)); + } + acc_descr_array->region_offs = 0; + acc_descr_array->reserved = 0; + perm_descr = &acc_descr_array->access_perm; + perm_descr->endpoint_id = my_endpoint_id; + perm_descr->perm = FFA_MEM_ACC_RW; + perm_descr->flags = 0; + + args.a1 = size; /* Total Length */ + args.a2 = size; /* Frag Length == Total length */ + thread_smccc(&args); + if (args.a0 != FFA_MEM_RETRIEVE_RESP) { + if (args.a0 == FFA_ERROR) + EMSG("Failed to fetch cookie %#"PRIx64" error code %d", + cookie, (int)args.a2); + else + EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, + cookie, args.a0); + return NULL; + } + rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, + my_rxtx.size, trans); + if (rc) { + EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", + cookie, rc); + return NULL; + } + + return my_rxtx.rx; +} + +void thread_spmc_relinquish(uint64_t cookie) +{ + struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; + struct thread_smc_args args = { + .a0 = FFA_MEM_RELINQUISH, + }; + + memset(relinquish_desc, 0, sizeof(*relinquish_desc)); + relinquish_desc->handle = cookie; + relinquish_desc->flags = 0; + relinquish_desc->endpoint_count = 1; + relinquish_desc->endpoint_id_array[0] = my_endpoint_id; + thread_smccc(&args); + if (!is_ffa_success(args.a0)) + EMSG("Failed to relinquish cookie %#"PRIx64, cookie); +} + +static int set_pages(struct ffa_address_range *regions, + unsigned int num_regions, unsigned int num_pages, + struct mobj_ffa *mf) +{ + unsigned int n = 0; + unsigned int idx = 0; + + for (n = 0; n < num_regions; n++) { + unsigned int page_count = READ_ONCE(regions[n].page_count); + uint64_t addr = READ_ONCE(regions[n].address); + + if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) + return FFA_INVALID_PARAMETERS; + } + + if (idx != num_pages) + return FFA_INVALID_PARAMETERS; + + return 0; +} + +struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) +{ + struct mobj_ffa *ret = NULL; + struct ffa_mem_transaction_x retrieve_desc = { }; + struct ffa_mem_access *descr_array = NULL; + struct ffa_mem_region *descr = NULL; + struct mobj_ffa *mf = NULL; + unsigned int num_pages = 0; + unsigned int offs = 0; + void *buf = NULL; + struct thread_smc_args ffa_rx_release_args = { + .a0 = FFA_RX_RELEASE + }; + + /* + * OP-TEE is only supporting a single mem_region while the + * specification allows for more than one. + */ + buf = spmc_retrieve_req(cookie, &retrieve_desc); + if (!buf) { + EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, + cookie); + return NULL; + } + + descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); + offs = READ_ONCE(descr_array->region_offs); + descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); + + num_pages = READ_ONCE(descr->total_page_count); + mf = mobj_ffa_spmc_new(cookie, num_pages); + if (!mf) + goto out; + + if (set_pages(descr->address_range_array, + READ_ONCE(descr->address_range_count), num_pages, mf)) { + mobj_ffa_spmc_delete(mf); + goto out; + } + + ret = mf; + +out: + /* Release RX buffer after the mem retrieve request. */ + thread_smccc(&ffa_rx_release_args); + + return ret; +} + +static TEE_Result spmc_init(void) +{ + unsigned int major = 0; + unsigned int minor __maybe_unused = 0; + uint32_t my_vers = 0; + uint32_t vers = 0; + + my_vers = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); + vers = get_ffa_version(my_vers); + major = (vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK; + minor = (vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK; + DMSG("SPMC reported version %u.%u", major, minor); + if (major != FFA_VERSION_MAJOR) { + EMSG("Incompatible major version %u, expected %u", + major, FFA_VERSION_MAJOR); + panic(); + } + if (vers < my_vers) + my_vers = vers; + DMSG("Using version %u.%u", + (my_vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK, + (my_vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK); + my_rxtx.ffa_vers = my_vers; + + spmc_rxtx_map(&my_rxtx); + my_endpoint_id = get_my_id(); + DMSG("My endpoint ID %#x", my_endpoint_id); + + return TEE_SUCCESS; +} +#endif /* !defined(CFG_CORE_SEL1_SPMC) */ + +/* + * boot_final() is always done before exiting at end of boot + * initialization. In case of virtualization the init-calls are done only + * once a OP-TEE partition has been created. So with virtualization we have + * to initialize via boot_final() to make sure we have a value assigned + * before it's used the first time. + */ +#ifdef CFG_NS_VIRTUALIZATION +boot_final(spmc_init); +#else +service_init(spmc_init); +#endif diff --git a/optee_os/core/arch/arm/kernel/thread_spmc_a32.S b/optee_os/core/arch/arm/kernel/thread_spmc_a32.S new file mode 100644 index 0000000..7a8d9da --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_spmc_a32.S @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Linaro Limited + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +FUNC thread_ffa_msg_wait , : + mov_imm r0, FFA_MSG_WAIT /* FID */ + mov r1, #FFA_TARGET_INFO_MBZ /* Target info MBZ */ + mov r2, #FFA_PARAM_MBZ /* Param MBZ */ + mov r3, #FFA_PARAM_MBZ /* Param MBZ */ + mov r4, #FFA_PARAM_MBZ /* Param MBZ */ + mov r5, #FFA_PARAM_MBZ /* Param MBZ */ + mov r6, #FFA_PARAM_MBZ /* Param MBZ */ + mov r7, #FFA_PARAM_MBZ /* Param MBZ */ + b .ffa_msg_loop +END_FUNC thread_ffa_msg_wait + + /* Caller provides r1, r3-r7 params */ +LOCAL_FUNC ffa_msg_send_direct_resp , : + ldr r0, =FFA_MSG_SEND_DIRECT_RESP_32 /* FID */ + mov r2, #FFA_PARAM_MBZ /* RES MBZ */ + +.ffa_msg_loop: + /* Invoke SMC with caller provided parameters */ + smc #0 + + /* Store the parameters as struct thread_smc_args on stack */ + push {r0-r7} + mov r0, sp + + /* parse and handle message */ + bl thread_spmc_msg_recv + + /* Load struct thread_smc_args into registers */ + pop {r0-r7} + b .ffa_msg_loop +END_FUNC ffa_msg_send_direct_resp + +FUNC thread_std_smc_entry , : +UNWIND( .cantunwind) + + push {r4, r5} /* Pass these following the arm32 calling convention */ + ror r4, r0, #16 /* Save target info with src and dst swapped */ + bl __thread_std_smc_entry + add sp, sp, #8 /* There's nothing return, just restore the sp */ + mov r5, r0 /* Save return value */ + + /* Mask all maskable exceptions before switching to temporary stack */ + cpsid aif + bl thread_get_tmp_sp + mov sp, r0 + + bl thread_state_free + + mov r1, r4 /* Target info */ + mov r3, r5 /* Return value */ + mov r4, #FFA_PARAM_MBZ /* Unused parameter */ + mov r5, #FFA_PARAM_MBZ /* Unused parameter */ + mov r6, #FFA_PARAM_MBZ /* Unused parameter */ + mov r7, #FFA_PARAM_MBZ /* Unused parameter */ + b ffa_msg_send_direct_resp +END_FUNC thread_std_smc_entry + +/* void thread_rpc(struct thread_rpc_arg *rpc_arg) */ +FUNC thread_rpc , : + push {r0, lr} +UNWIND( .save {r0, lr}) + + bl thread_save_state + mov r4, r0 /* Save original CPSR */ + + /* + * Switch to temporary stack and SVC mode. Save CPSR to resume into. + */ + bl thread_get_tmp_sp + ldr r8, [sp] /* Get pointer to rv[] */ + cps #CPSR_MODE_SVC /* Change to SVC mode */ + mov sp, r0 /* Switch to tmp stack */ + + mov r0, #THREAD_FLAGS_COPY_ARGS_ON_RETURN + mov r1, r4 /* CPSR to restore */ + ldr r2, =.thread_rpc_return + bl thread_state_suspend + mov r7, r0 /* Supply thread index */ + ldr r0, =FFA_MSG_SEND_DIRECT_RESP_32 + mov r2, #FFA_PARAM_MBZ + mov r3, #0 /* Error code = 0 */ + ldm r8, {r1, r4-r6} /* Load rv[] into r1,r4-r6 */ + b ffa_msg_send_direct_resp + +.thread_rpc_return: + /* + * At this point has the stack pointer been restored to the value + * it had when thread_save_state() was called above. + * + * Jumps here from thread_resume above when RPC has returned. The + * IRQ and FIQ bits are restored to what they where when this + * function was originally entered. + */ + pop {r12, lr} /* Get pointer to rv[] */ + stm r12, {r0-r3} /* Store r0-r3 into rv[] */ + bx lr +END_FUNC thread_rpc + +/* + * void thread_foreign_intr_exit(uint32_t thread_index) + * + * This function is jumped to at the end of macro foreign_intr_handler(). + * The current thread as indicated by @thread_index has just been + * suspended. The job here is just to inform normal world the thread id to + * resume when returning. + */ +FUNC thread_foreign_intr_exit , : + /* load threads[r0].tsd.rpc_target_info into r1 */ + mov r1, #THREAD_CTX_SIZE + ldr r2, =threads + mla r1, r1, r0, r2 + ldr r1, [r1, #THREAD_CTX_TSD_RPC_TARGET_INFO] + mov r2, #FFA_PARAM_MBZ + mov r3, #FFA_PARAM_MBZ + mov r4, #OPTEE_FFA_YIELDING_CALL_RETURN_INTERRUPT + mov r5, #FFA_PARAM_MBZ + mov r6, #FFA_PARAM_MBZ + mov r7, r0 + b ffa_msg_send_direct_resp +END_FUNC thread_foreign_intr_exit diff --git a/optee_os/core/arch/arm/kernel/thread_spmc_a64.S b/optee_os/core/arch/arm/kernel/thread_spmc_a64.S new file mode 100644 index 0000000..8ea2bce --- /dev/null +++ b/optee_os/core/arch/arm/kernel/thread_spmc_a64.S @@ -0,0 +1,214 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Linaro Limited + * Copyright (c) 2019-2021, Arm Limited + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#if CFG_SECURE_PARTITION +LOCAL_FUNC thread_ffa_interrupt , : + mov_imm x0, FFA_INTERRUPT /* FID */ + /* X1: Endpoint/vCPU IDs is set by caller */ + mov x2, #FFA_PARAM_MBZ /* Param MBZ */ + mov x3, #FFA_PARAM_MBZ /* Param MBZ */ + mov x4, #FFA_PARAM_MBZ /* Param MBZ */ + mov x5, #FFA_PARAM_MBZ /* Param MBZ */ + mov x6, #FFA_PARAM_MBZ /* Param MBZ */ + mov x7, #FFA_PARAM_MBZ /* Param MBZ */ + b .ffa_msg_loop +END_FUNC thread_ffa_interrupt +#endif /* CFG_SECURE_PARTITION */ + +FUNC thread_ffa_msg_wait , : + mov_imm x0, FFA_MSG_WAIT /* FID */ + mov x1, #FFA_TARGET_INFO_MBZ /* Target info MBZ */ + mov x2, #FFA_PARAM_MBZ /* Param MBZ */ + mov x3, #FFA_PARAM_MBZ /* Param MBZ */ + mov x4, #FFA_PARAM_MBZ /* Param MBZ */ + mov x5, #FFA_PARAM_MBZ /* Param MBZ */ + mov x6, #FFA_PARAM_MBZ /* Param MBZ */ + mov x7, #FFA_PARAM_MBZ /* Param MBZ */ + b .ffa_msg_loop +END_FUNC thread_ffa_msg_wait + + /* Caller provides x1, x3-x7 params */ +LOCAL_FUNC ffa_msg_send_direct_resp , : + msr spsel, #1 + ldr w0, [sp, #THREAD_CORE_LOCAL_DIRECT_RESP_FID] + msr spsel, #0 + mov x2, #FFA_PARAM_MBZ /* RES MBZ */ + +.ffa_msg_loop: + /* Invoke SMC with caller provided parameters */ + smc #0 + + /* Store the parameters as struct thread_smc_args on stack */ + sub sp, sp, #THREAD_SMC_ARGS_SIZE + store_xregs sp, THREAD_SMC_ARGS_X0, 0, 7 + mov x0, sp + + /* parse and handle message */ + bl thread_spmc_msg_recv + + /* Load struct thread_smc_args into registers */ + load_xregs sp, THREAD_SMC_ARGS_X0, 0, 7 + add sp, sp, #THREAD_SMC_ARGS_SIZE + b .ffa_msg_loop +END_FUNC ffa_msg_send_direct_resp + +FUNC thread_std_smc_entry , : + ror w19, w0, #16 /* Save target info with src and dst swapped */ + bl __thread_std_smc_entry + mov w20, w0 /* Save return value */ + + /* Mask all maskable exceptions before switching to temporary stack */ + msr daifset, #DAIFBIT_ALL + bl thread_get_tmp_sp + mov sp, x0 + + bl thread_state_free + + mov w1, w19 /* Target info */ + mov w3, w20 /* Return value */ + mov x4, #FFA_PARAM_MBZ /* Unused parameter */ + mov x5, #FFA_PARAM_MBZ /* Unused parameter */ + mov x6, #FFA_PARAM_MBZ /* Unused parameter */ + mov x7, #FFA_PARAM_MBZ /* Unused parameter */ + b ffa_msg_send_direct_resp +END_FUNC thread_std_smc_entry + +#ifdef CFG_SECURE_PARTITION +/* void spmc_sp_thread_entry(args) */ +FUNC spmc_sp_thread_entry , : + /* Store the parameters as struct thread_smc_args on stack */ + sub sp, sp, #THREAD_SMC_ARGS_SIZE + store_xregs sp, THREAD_SMC_ARGS_X0, 0, 7 + mov x0, sp + mov x1, #0 /* Pass NULL pointer for caller_sp, coming from NW */ + bl spmc_sp_msg_handler + load_xregs sp, THREAD_SMC_ARGS_X0, 20, 27 + + /* Mask all maskable exceptions before switching to temporary stack */ + msr daifset, #DAIFBIT_ALL + bl thread_get_tmp_sp + mov sp, x0 + + bl thread_state_free + + /* Restore the FF-A arguments before the SMC instruction. */ + mov w0, w20 + mov w1, w21 + mov w2, w22 + mov w3, w23 + mov w4, w24 + mov w5, w25 + mov w6, w26 + mov w7, w27 + b .ffa_msg_loop +END_FUNC spmc_sp_thread_entry +#endif + +/* void thread_rpc_spsr(uint32_t rv[THREAD_RPC_NUM_ARGS], uint64_t spsr) */ +FUNC thread_rpc_spsr , : + /* Mask all maskable exceptions before switching to temporary stack */ + msr daifset, #DAIFBIT_ALL + push x0, xzr + push x1, x30 + bl thread_get_ctx_regs + ldr x30, [sp, #8] + store_xregs x0, THREAD_CTX_REGS_X19, 19, 30 + mov x19, x0 + +#if defined(CFG_CORE_PAUTH) + /* Save APIAKEY */ + read_apiakeyhi x1 + read_apiakeylo x2 + store_xregs x0, THREAD_CTX_REGS_APIAKEY_HI, 1, 2 +#endif + + bl thread_get_tmp_sp + pop x1, xzr /* Match "push x1, x30" above */ + mov x2, sp + str x2, [x19, #THREAD_CTX_REGS_SP] + ldr x20, [sp] /* Get pointer to rpc_arg[] */ + mov sp, x0 /* Switch to tmp stack */ + /* + * We need to read rpc_arg[] early, because thread_state_suspend + * can invoke virt_unset_guest() which will unmap pages, + * where rpc_arg[] resides + */ + load_wregs x20, 0, 21, 24 /* Load rpc_arg[] into w21-w24 */ + + adr x2, .thread_rpc_return + mov w0, #THREAD_FLAGS_COPY_ARGS_ON_RETURN + bl thread_state_suspend + mov w1, w21 + mov w3, #0 /* Error code = 0 */ + mov w4, w22 + mov w5, w23 + mov w6, w24 + mov w7, w0 /* Supply thread index */ + b ffa_msg_send_direct_resp + +.thread_rpc_return: + /* + * At this point has the stack pointer been restored to the value + * stored in THREAD_CTX above. + * + * Jumps here from thread_resume above when RPC has returned. The + * IRQ and FIQ bits are restored to what they where when this + * function was originally entered. w0-w3 holds the values supplied + * to thread_resume_from_rpc() in a0-a3. + */ + pop x16, xzr /* Get pointer to rv[] */ + store_wregs x16, 0, 0, 3 /* Store w0-w3 into rv[] */ + ret +END_FUNC thread_rpc_spsr + +/* + * void thread_foreign_intr_exit(uint32_t thread_index) + * + * This function is jumped to at the end of macro foreign_intr_handler(). + * The current thread as indicated by @thread_index has just been + * suspended. The job here is just to inform normal world the thread id to + * resume when returning. + * If the active FF-A endpoint is OP-TEE (or a TA) then an this function send an + * OPTEE_FFA_YIELDING_CALL_RETURN_INTERRUPT message to the normal world via the + * FFA_MSG_SEND_DIRECT_RESP interface. This is handled by the OP-TEE + * driver in Linux so it can schedule task to the thread. + * If the active endpoint is an SP the function sends an FFA_INTERRUPT. This is + * handled by the FF-A driver and after taking care of the NWd interrupts it + * returns via an FFA_RUN call. + * The active endpoint is determined by checking the THREAD_FLAGS_FFA_ONLY flag + * in threads[w0].flags. This is only set for the thread which handles SPs. + */ +FUNC thread_foreign_intr_exit , : + /* load threads[w0].tsd.rpc_target_info into w1 */ + mov x1, #THREAD_CTX_SIZE + adr_l x2, threads + madd x2, x1, x0, x2 + ldr w1, [x2, #THREAD_CTX_TSD_RPC_TARGET_INFO] +#if CFG_SECURE_PARTITION + /* load threads[w0].flags into w2 */ + ldr w2, [x2, #THREAD_CTX_FLAGS] + and w2, w2, #THREAD_FLAGS_FFA_ONLY + cbnz w2, thread_ffa_interrupt +#endif /* CFG_SECURE_PARTITION */ + mov w3, #FFA_PARAM_MBZ + mov w4, #OPTEE_FFA_YIELDING_CALL_RETURN_INTERRUPT + mov x5, #FFA_PARAM_MBZ + mov w6, #FFA_PARAM_MBZ + mov w7, w0 + b ffa_msg_send_direct_resp +END_FUNC thread_foreign_intr_exit + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/timer_a64.c b/optee_os/core/arch/arm/kernel/timer_a64.c new file mode 100644 index 0000000..d64a997 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/timer_a64.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +#include +#include +#include + +static unsigned int timer_lock = SPINLOCK_UNLOCK; +static bool timer_running; + +void generic_timer_start(uint32_t time_ms) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + uint32_t timer_ticks; + + cpu_spin_lock(&timer_lock); + + if (timer_running == true) + goto exit; + + /* The timer will fire time_ms from now */ + timer_ticks = (read_cntfrq() * time_ms) / 1000; + write_cntps_tval(timer_ticks); + + /* Enable the secure physical timer */ + write_cntps_ctl(1); + + timer_running = true; + +exit: + cpu_spin_unlock(&timer_lock); + thread_set_exceptions(exceptions); +} + +void generic_timer_stop(void) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + cpu_spin_lock(&timer_lock); + + /* Disable the timer */ + write_cntps_ctl(0); + + timer_running = false; + + cpu_spin_unlock(&timer_lock); + thread_set_exceptions(exceptions); +} + +void generic_timer_handler(uint32_t time_ms) +{ + uint32_t timer_ticks; + + /* Ensure that the timer did assert the interrupt */ + assert((read_cntps_ctl() >> 2)); + + /* Disable the timer */ + write_cntps_ctl(0); + + /* Reconfigure timer to fire time_ms from now */ + timer_ticks = (read_cntfrq() * time_ms) / 1000; + write_cntps_tval(timer_ticks); + + /* Enable the secure physical timer */ + write_cntps_ctl(1); +} diff --git a/optee_os/core/arch/arm/kernel/tlb_helpers_a32.S b/optee_os/core/arch/arm/kernel/tlb_helpers_a32.S new file mode 100644 index 0000000..79bffdd --- /dev/null +++ b/optee_os/core/arch/arm/kernel/tlb_helpers_a32.S @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include + +/* void tlbi_all(void); */ +FUNC tlbi_all , : + dsb ishst /* Sync with table update */ + write_tlbiallis /* Invalidate TLBs */ + dsb ish /* Sync with tlb invalidation completion */ + isb /* Sync execution on tlb update */ + bx lr +END_FUNC tlbi_all + +/* void tlbi_va_allasid(vaddr_t va); */ +FUNC tlbi_va_allasid , : + dsb ishst /* Sync with table update */ + write_tlbimvaais r0 /* Inval TLB by MVA all ASID Inner Sharable */ + dsb ish /* Sync with tlb invalidation completion */ + isb /* Sync execution on tlb update */ + bx lr +END_FUNC tlbi_va_allasid + +/* void tlbi_asid(unsigned long asid); */ +FUNC tlbi_asid , : + dsb ishst /* Sync with table update */ + write_tlbiasidis r0 /* Inval unified TLB by ASID Inner Sharable */ + orr r0, r0, #1 /* Select the kernel ASID */ + write_tlbiasidis r0 /* Inval unified TLB by ASID Inner Sharable */ + dsb ish /* Sync with tlb invalidation completion */ + isb /* Sync execution on tlb update */ + bx lr +END_FUNC tlbi_asid diff --git a/optee_os/core/arch/arm/kernel/tlb_helpers_a64.S b/optee_os/core/arch/arm/kernel/tlb_helpers_a64.S new file mode 100644 index 0000000..fddc883 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/tlb_helpers_a64.S @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2017, Linaro Limited + */ + +#include +#include +#include + +/* void tlbi_all(void); */ +FUNC tlbi_all , : + dsb ishst /* Sync with table update */ + tlbi vmalle1is /* All tlb in inner shareable */ + dsb ish /* Sync with tlb invalidation completion */ + isb /* Sync execution on tlb update */ + ret +END_FUNC tlbi_all + +/* void tlbi_va_allasid(vaddr_t va); */ +FUNC tlbi_va_allasid , : + lsr x0, x0, #TLBI_VA_SHIFT + dsb ishst /* Sync with table update */ + tlbi vaae1is, x0 /* Invalidate tlb by va in inner shareable */ + dsb ish /* Sync with tlb invalidation completion */ + isb /* Sync execution on tlb update */ + ret +END_FUNC tlbi_va_allasid + +/* void tlbi_asid(unsigned int asid); */ +FUNC tlbi_asid , : + lsl x0, x0, #TLBI_ASID_SHIFT + dsb ishst /* Sync with table update */ + tlbi aside1is, x0 /* Invalidate tlb by asid in inner shareable */ + orr x0, x0, #BIT(TLBI_ASID_SHIFT) /* Select the kernel ASID */ + tlbi aside1is, x0 /* Invalidate tlb by asid in inner shareable */ + dsb ish /* Sync with tlb invalidation completion */ + isb /* Sync execution on tlb update */ + ret +END_FUNC tlbi_asid + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/tz_ssvce_pl310_a32.S b/optee_os/core/arch/arm/kernel/tz_ssvce_pl310_a32.S new file mode 100644 index 0000000..b3f85cc --- /dev/null +++ b/optee_os/core/arch/arm/kernel/tz_ssvce_pl310_a32.S @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include + +#define PL310_LOCKDOWN_NBREGS 8 +#define PL310_LOCKDOWN_SZREG 4 + +#define PL310_8WAYS_MASK 0x00FF +#define PL310_16WAYS_UPPERMASK 0xFF00 + +/* + * void arm_cl2_lockallways(vaddr_t base) + * + * lock all L2 caches ways for data and instruction + */ +FUNC arm_cl2_lockallways , : + add r1, r0, #PL310_DCACHE_LOCKDOWN_BASE + ldr r2, [r0, #PL310_AUX_CTRL] + tst r2, #PL310_AUX_16WAY_BIT + mov r2, #PL310_8WAYS_MASK + orrne r2, #PL310_16WAYS_UPPERMASK + mov r0, #PL310_LOCKDOWN_NBREGS +1: /* lock Dcache and Icache */ + str r2, [r1], #PL310_LOCKDOWN_SZREG + str r2, [r1], #PL310_LOCKDOWN_SZREG + subs r0, r0, #1 + bne 1b + + mov pc, lr +END_FUNC arm_cl2_lockallways + +/* + * Set sync operation mask according to ways associativity. + * Preserve r0 = pl310 iomem base address + */ +.macro syncbyway_set_mask reg + ldr \reg, [r0, #PL310_AUX_CTRL] + tst \reg, #PL310_AUX_16WAY_BIT + mov \reg, #PL310_8WAYS_MASK + orrne \reg, \reg, #PL310_16WAYS_UPPERMASK +.endm + +/* + * void arm_cl2_cleaninvbyway(vaddr_t base) + * clean & invalidate the whole L2 cache. + */ +FUNC arm_cl2_cleaninvbyway , : + + syncbyway_set_mask r1 + str r1, [r0, #PL310_FLUSH_BY_WAY] + + /* Wait for all cache ways to be cleaned and invalidated */ +loop_cli_way_done: + ldr r2, [r0, #PL310_FLUSH_BY_WAY] + and r2, r2, r1 + cmp r2, #0 + bne loop_cli_way_done + + /* Cache Sync */ + + /* + * Wait for writing cache sync + * To PL310, Cache sync is atomic opertion, no need to check + * the status. For PL220, this check is needed. Keeping the loop + * for PL310 is no harm for PL310. + */ +loop_cli_sync: + ldr r1, [r0, #PL310_SYNC] + cmp r1, #0 + bne loop_cli_sync + + mov r1, #0 + str r1, [r0, #PL310_SYNC] + +loop_cli_sync_done: + ldr r1, [r0, #PL310_SYNC] + cmp r1, #0 + bne loop_cli_sync_done + + mov pc, lr +END_FUNC arm_cl2_cleaninvbyway + +/* void arm_cl2_invbyway(vaddr_t base) */ +FUNC arm_cl2_invbyway , : + + syncbyway_set_mask r1 + str r1, [r0, #PL310_INV_BY_WAY] + +loop_inv_way_done: + ldr r2, [r0, #PL310_INV_BY_WAY] + and r2, r2, r1 + cmp r2, #0 + bne loop_inv_way_done + +loop_inv_way_sync: + ldr r1, [r0, #PL310_SYNC] + cmp r1, #0 + bne loop_inv_way_sync + + mov r1, #0 + str r1, [r0, #PL310_SYNC] + +loop_inv_way_sync_done: + ldr r1, [r0, #PL310_SYNC] + cmp r1, #0 + bne loop_inv_way_sync_done + + mov pc, lr +END_FUNC arm_cl2_invbyway + +/* void arm_cl2_cleanbyway(vaddr_t base) */ +FUNC arm_cl2_cleanbyway , : + + syncbyway_set_mask r1 + str r1, [r0, #PL310_CLEAN_BY_WAY] + +loop_cl_way_done: + ldr r2, [r0, #PL310_CLEAN_BY_WAY] + and r2, r2, r1 + cmp r2, #0 + bne loop_cl_way_done + +loop_cl_way_sync: + ldr r1, [r0, #PL310_SYNC] + cmp r1, #0 + bne loop_cl_way_sync + + mov r1, #0 + str r1, [r0, #PL310_SYNC] + +loop_cl_way_sync_done: + ldr r1, [r0, #PL310_SYNC] + cmp r1, #0 + bne loop_cl_way_sync_done + + mov pc, lr +END_FUNC arm_cl2_cleanbyway + +/* + * void _arm_cl2_xxxbypa(vaddr_t pl310_base, paddr_t start, paddr_t end, + * int pl310value); + * pl310value is one of PL310_CLEAN_BY_PA, PL310_INV_BY_PA or PL310_FLUSH_BY_PA + */ +LOCAL_FUNC _arm_cl2_xxxbypa , : + /* Align start address on PL310 line size */ + and r1, #(~(PL310_LINE_SIZE - 1)) +#ifdef SCU_BASE + /* + * ARM ERRATA #764369 + * Undocummented SCU Diagnostic Control Register + */ + /* + * NOTE: + * We're assuming that if mmu is enabled PL310_BASE and SCU_BASE + * still have the same relative offsets from each other. + */ + sub r0, r0, #(PL310_BASE - SCU_BASE) + mov r12, #1 + str r12, [r0, #SCU_ERRATA744369] + dsb + add r0, r0, #(PL310_BASE - SCU_BASE) +#endif +loop_cl2_xxxbypa: + str r1, [r0, r3] + +loop_xxx_pa_done: + ldr r12, [r0, r3] + and r12, r12, r1 + cmp r12, #0 + bne loop_xxx_pa_done + + add r1, r1, #PL310_LINE_SIZE + cmp r2, r1 + bpl loop_cl2_xxxbypa + +loop_xxx_pa_sync: + ldr r12, [r0, #PL310_SYNC] + cmp r12, #0 + bne loop_xxx_pa_sync + + mov r12, #0 + str r12, [r0, #PL310_SYNC] + +loop_xxx_pa_sync_done: + ldr r12, [r0, #PL310_SYNC] + cmp r12, #0 + bne loop_xxx_pa_sync_done + + mov pc, lr +END_FUNC _arm_cl2_xxxbypa + +/* + * void _arm_cl2_cleanbypa(vaddr_t pl310_base, paddr_t start, paddr_t end); + * clean L2 cache by physical address range. + */ +FUNC arm_cl2_cleanbypa , : + mov r3, #PL310_CLEAN_BY_PA + b _arm_cl2_xxxbypa +END_FUNC arm_cl2_cleanbypa + +/* + * void arm_cl2_invbypa(vaddr_t pl310_base, paddr_t start, paddr_t end); + * invalidate L2 cache by physical address range. + */ +FUNC arm_cl2_invbypa , : + mov r3, #PL310_INV_BY_PA + b _arm_cl2_xxxbypa +END_FUNC arm_cl2_invbypa + +/* + * void arm_cl2_cleaninvbypa(vaddr_t pl310_base, paddr_t start, paddr_t end); + * clean and invalidate L2 cache by physical address range. + */ +FUNC arm_cl2_cleaninvbypa , : + mov r3, #PL310_FLUSH_BY_PA + b _arm_cl2_xxxbypa +END_FUNC arm_cl2_cleaninvbypa + diff --git a/optee_os/core/arch/arm/kernel/unwind_arm32.c b/optee_os/core/arch/arm/kernel/unwind_arm32.c new file mode 100644 index 0000000..6dcdb6e --- /dev/null +++ b/optee_os/core/arch/arm/kernel/unwind_arm32.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2015 Linaro Limited + * Copyright 2013-2014 Andrew Turner. + * Copyright 2013-2014 Ian Lepore. + * Copyright 2013-2014 Rui Paulo. + * Copyright 2013 Eitan Adler. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include "unwind_private.h" + +/* The register names */ +#define FP 11 +#define SP 13 +#define LR 14 +#define PC 15 + +bool find_exidx(vaddr_t addr __unused, vaddr_t *idx_start, vaddr_t *idx_end) +{ + *idx_start = (vaddr_t)__exidx_start; + *idx_end = (vaddr_t)__exidx_end; + return true; +} + +vaddr_t *unw_get_kernel_stack(void) +{ + size_t n = 0; + size_t size = 0; + size_t exidx_sz = 0; + vaddr_t *tmp = NULL; + vaddr_t *addr = NULL; + struct unwind_state_arm32 state = { }; + vaddr_t stack = thread_stack_start(); + size_t stack_size = thread_stack_size(); + + if (SUB_OVERFLOW((vaddr_t)__exidx_end, (vaddr_t)__exidx_start, + &exidx_sz)) + return NULL; + + /* r7: Thumb-style frame pointer */ + state.registers[7] = read_r7(); + /* r11: ARM-style frame pointer */ + state.registers[FP] = read_fp(); + state.registers[SP] = read_sp(); + state.registers[LR] = read_lr(); + + /* + * Add 4 to make sure that we have an address well inside this function. + * This is needed because we're subtracting 2 from PC when calling + * find_index() above. See a comment there for more details. + */ + state.registers[PC] = (uint32_t)unw_get_kernel_stack + 4; + + while (unwind_stack_arm32(&state, stack, stack_size)) { + tmp = unw_grow(addr, &size, (n + 1) * sizeof(vaddr_t)); + if (!tmp) + goto err; + addr = tmp; + addr[n] = state.registers[PC]; + n++; + } + + if (addr) { + tmp = unw_grow(addr, &size, (n + 1) * sizeof(vaddr_t)); + if (!tmp) + goto err; + addr = tmp; + addr[n] = 0; + } + + return addr; +err: + EMSG("Out of memory"); + return NULL; +} + +#if (TRACE_LEVEL > 0) +void print_kernel_stack(void) +{ + struct unwind_state_arm32 state = { }; + vaddr_t stack_start = 0; + vaddr_t stack_end = 0; + + /* r7: Thumb-style frame pointer */ + state.registers[7] = read_r7(); + /* r11: ARM-style frame pointer */ + state.registers[FP] = read_fp(); + state.registers[SP] = read_sp(); + state.registers[LR] = read_lr(); + + /* + * Add 4 to make sure that we have an address well inside this function. + * This is needed because we're subtracting 2 from PC when calling + * find_index() above. See a comment there for more details. + */ + state.registers[PC] = (uint32_t)print_kernel_stack + 4; + + trace_printf_helper_raw(TRACE_ERROR, true, + "TEE load address @ %#"PRIxVA, VCORE_START_VA); + get_stack_hard_limits(&stack_start, &stack_end); + print_stack_arm32(&state, stack_start, stack_end - stack_start); +} +#endif diff --git a/optee_os/core/arch/arm/kernel/unwind_arm64.c b/optee_os/core/arch/arm/kernel/unwind_arm64.c new file mode 100644 index 0000000..7d994d7 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/unwind_arm64.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-2-Clause +/*- + * Copyright (c) 2015 Linaro Limited + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "unwind_private.h" + +#if defined(CFG_CORE_PAUTH) +void pauth_strip_pac(uint64_t *lr) +{ + const uint64_t va_mask = GENMASK_64(CFG_LPAE_ADDR_SPACE_BITS - 1, 0); + + *lr = *lr & va_mask; +} +#endif + +vaddr_t *unw_get_kernel_stack(void) +{ + size_t n = 0; + size_t size = 0; + vaddr_t *tmp = NULL; + vaddr_t *addr = NULL; + uaddr_t stack = thread_stack_start(); + size_t stack_size = thread_stack_size(); + struct unwind_state_arm64 state = { + .pc = read_pc(), + .fp = read_fp() + }; + + while (unwind_stack_arm64(&state, stack, stack_size)) { + tmp = unw_grow(addr, &size, (n + 1) * sizeof(vaddr_t)); + if (!tmp) + goto err; + addr = tmp; + addr[n] = state.pc; + n++; + } + + if (addr) { + tmp = unw_grow(addr, &size, (n + 1) * sizeof(vaddr_t)); + if (!tmp) + goto err; + addr = tmp; + addr[n] = 0; + } + + return addr; +err: + EMSG("Out of memory"); + free(addr); + return NULL; +} + +#if defined(CFG_UNWIND) && (TRACE_LEVEL > 0) +void print_kernel_stack(void) +{ + struct unwind_state_arm64 state = { }; + vaddr_t stack_start = 0; + vaddr_t stack_end = 0; + + state.pc = read_pc(); + state.fp = read_fp(); + + trace_printf_helper_raw(TRACE_ERROR, true, + "TEE load address @ %#"PRIxVA, VCORE_START_VA); + get_stack_hard_limits(&stack_start, &stack_end); + print_stack_arm64(&state, stack_start, stack_end - stack_start); +} +#endif diff --git a/optee_os/core/arch/arm/kernel/unwind_private.h b/optee_os/core/arch/arm/kernel/unwind_private.h new file mode 100644 index 0000000..df6f2ff --- /dev/null +++ b/optee_os/core/arch/arm/kernel/unwind_private.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ +#include +#include +#include + +static inline void *unw_grow(void *p, size_t *cur_size, size_t new_size) +{ + size_t rounded_size = 0; + void *tmp = NULL; + + if (*cur_size >= new_size) + return p; + + rounded_size = ROUNDUP(new_size, 16 * sizeof(vaddr_t)); + tmp = realloc(p, rounded_size); + + if (tmp) + *cur_size = rounded_size; + return tmp; +} diff --git a/optee_os/core/arch/arm/kernel/vfp.c b/optee_os/core/arch/arm/kernel/vfp.c new file mode 100644 index 0000000..8e945bf --- /dev/null +++ b/optee_os/core/arch/arm/kernel/vfp.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include +#include "vfp_private.h" + +#ifdef ARM32 +bool vfp_is_enabled(void) +{ + return !!(vfp_read_fpexc() & FPEXC_EN); +} + +void vfp_enable(void) +{ + vfp_write_fpexc(vfp_read_fpexc() | FPEXC_EN); +} + +void vfp_disable(void) +{ + vfp_write_fpexc(vfp_read_fpexc() & ~FPEXC_EN); +} + +void vfp_lazy_save_state_init(struct vfp_state *state) +{ + uint32_t fpexc = vfp_read_fpexc(); + + state->fpexc = fpexc; + vfp_write_fpexc(fpexc & ~FPEXC_EN); +} + +void vfp_lazy_save_state_final(struct vfp_state *state, bool force_save) +{ + if ((state->fpexc & FPEXC_EN) || force_save) { + uint32_t fpexc = vfp_read_fpexc(); + + assert(!(fpexc & FPEXC_EN)); + vfp_write_fpexc(fpexc | FPEXC_EN); + state->fpscr = vfp_read_fpscr(); + vfp_save_extension_regs(state->reg); + vfp_write_fpexc(fpexc); + } +} + +void vfp_lazy_restore_state(struct vfp_state *state, bool full_state) +{ + + if (full_state) { + /* + * Only restore VFP registers if they have been touched as they + * otherwise are intact. + */ + + /* FPEXC is restored to what's in state->fpexc below */ + vfp_write_fpexc(vfp_read_fpexc() | FPEXC_EN); + + vfp_write_fpscr(state->fpscr); + vfp_restore_extension_regs(state->reg); + } + vfp_write_fpexc(state->fpexc); +} +#endif /* ARM32 */ + +#ifdef ARM64 +bool vfp_is_enabled(void) +{ + return (CPACR_EL1_FPEN(read_cpacr_el1()) & CPACR_EL1_FPEN_EL0EL1); +} + +void vfp_enable(void) +{ + uint32_t val = read_cpacr_el1(); + + val |= (CPACR_EL1_FPEN_EL0EL1 << CPACR_EL1_FPEN_SHIFT); + write_cpacr_el1(val); + isb(); +} + +void vfp_disable(void) +{ + uint32_t val = read_cpacr_el1(); + + val &= ~(CPACR_EL1_FPEN_MASK << CPACR_EL1_FPEN_SHIFT); + write_cpacr_el1(val); + isb(); +} + +void vfp_lazy_save_state_init(struct vfp_state *state) +{ + state->cpacr_el1 = read_cpacr_el1(); + vfp_disable(); +} + +void vfp_lazy_save_state_final(struct vfp_state *state, bool force_save) +{ + if ((CPACR_EL1_FPEN(state->cpacr_el1) & CPACR_EL1_FPEN_EL0EL1) || + force_save) { + assert(!vfp_is_enabled()); + vfp_enable(); + state->fpcr = read_fpcr(); + state->fpsr = read_fpsr(); + vfp_save_extension_regs(state->reg); + vfp_disable(); + } +} + +void vfp_lazy_restore_state(struct vfp_state *state, bool full_state) +{ + if (full_state) { + /* + * Only restore VFP registers if they have been touched as they + * otherwise are intact. + */ + + /* CPACR_EL1 is restored to what's in state->cpacr_el1 below */ + vfp_enable(); + write_fpcr(state->fpcr); + write_fpsr(state->fpsr); + vfp_restore_extension_regs(state->reg); + } + write_cpacr_el1(state->cpacr_el1); + isb(); +} +#endif /* ARM64 */ diff --git a/optee_os/core/arch/arm/kernel/vfp_a32.S b/optee_os/core/arch/arm/kernel/vfp_a32.S new file mode 100644 index 0000000..c435de1 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/vfp_a32.S @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2016, Linaro Limited + */ + +#include +#include +#include + + .fpu neon + +/* void vfp_save_extension_regs(uint64_t regs[VFP_NUM_REGS]); */ +FUNC vfp_save_extension_regs , : + vstm r0!, {d0-d15} + read_cpacr r1 + tst r1, #CPACR_D32DIS + bxne lr + vstm r0, {d16-d31} + bx lr +END_FUNC vfp_save_extension_regs + +/* void vfp_restore_extension_regs(uint64_t regs[VFP_NUM_REGS]); */ +FUNC vfp_restore_extension_regs , : + vldm r0!, {d0-d15} + read_cpacr r1 + tst r1, #CPACR_D32DIS + bxne lr + vldm r0, {d16-d31} + bx lr +END_FUNC vfp_restore_extension_regs + +/* void vfp_write_fpexc(uint32_t fpexc) */ +FUNC vfp_write_fpexc , : + vmsr fpexc, r0 + bx lr +END_FUNC vfp_write_fpexc + +/* uint32_t vfp_read_fpexc(void) */ +FUNC vfp_read_fpexc , : + vmrs r0, fpexc + bx lr +END_FUNC vfp_read_fpexc + +/* void vfp_write_fpscr(uint32_t fpscr) */ +FUNC vfp_write_fpscr , : + vmsr fpscr, r0 + bx lr +END_FUNC vfp_write_fpscr + +/* uint32_t vfp_read_fpscr(void) */ +FUNC vfp_read_fpscr , : + vmrs r0, fpscr + bx lr +END_FUNC vfp_read_fpscr diff --git a/optee_os/core/arch/arm/kernel/vfp_a64.S b/optee_os/core/arch/arm/kernel/vfp_a64.S new file mode 100644 index 0000000..a7322cc --- /dev/null +++ b/optee_os/core/arch/arm/kernel/vfp_a64.S @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include + +/* void vfp_save_extension_regs(struct vfp_reg regs[VFP_NUM_REGS]); */ +FUNC vfp_save_extension_regs , : + stp q0, q1, [x0, #16 * 0] + stp q2, q3, [x0, #16 * 2] + stp q4, q5, [x0, #16 * 4] + stp q6, q7, [x0, #16 * 6] + stp q8, q9, [x0, #16 * 8] + stp q10, q11, [x0, #16 * 10] + stp q12, q13, [x0, #16 * 12] + stp q14, q15, [x0, #16 * 14] + stp q16, q17, [x0, #16 * 16] + stp q18, q19, [x0, #16 * 18] + stp q20, q21, [x0, #16 * 20] + stp q22, q23, [x0, #16 * 22] + stp q24, q25, [x0, #16 * 24] + stp q26, q27, [x0, #16 * 26] + stp q28, q29, [x0, #16 * 28] + stp q30, q31, [x0, #16 * 30] + ret +END_FUNC vfp_save_extension_regs + +/* void vfp_restore_extension_regs(struct vfp_reg regs[VFP_NUM_REGS]); */ +FUNC vfp_restore_extension_regs , : + ldp q0, q1, [x0, #16 * 0] + ldp q2, q3, [x0, #16 * 2] + ldp q4, q5, [x0, #16 * 4] + ldp q6, q7, [x0, #16 * 6] + ldp q8, q9, [x0, #16 * 8] + ldp q10, q11, [x0, #16 * 10] + ldp q12, q13, [x0, #16 * 12] + ldp q14, q15, [x0, #16 * 14] + ldp q16, q17, [x0, #16 * 16] + ldp q18, q19, [x0, #16 * 18] + ldp q20, q21, [x0, #16 * 20] + ldp q22, q23, [x0, #16 * 22] + ldp q24, q25, [x0, #16 * 24] + ldp q26, q27, [x0, #16 * 26] + ldp q28, q29, [x0, #16 * 28] + ldp q30, q31, [x0, #16 * 30] + ret +END_FUNC vfp_restore_extension_regs + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/core/arch/arm/kernel/vfp_private.h b/optee_os/core/arch/arm/kernel/vfp_private.h new file mode 100644 index 0000000..6fac336 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/vfp_private.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef VFP_PRIVATE +#define VFP_PRIVATE + +#include + +void vfp_save_extension_regs(struct vfp_reg regs[VFP_NUM_REGS]); +void vfp_restore_extension_regs(struct vfp_reg regs[VFP_NUM_REGS]); +void vfp_clear_extension_regs(void); + +#ifdef ARM32 + +#define FPEXC_EN (1 << 30) + +/* + * These functions can't be implemented in inline assembly when compiling + * for thumb mode, to make it easy always implement then in ARM assembly as + * ordinary functions. + */ +void vfp_write_fpexc(uint32_t fpexc); +uint32_t vfp_read_fpexc(void); +void vfp_write_fpscr(uint32_t fpscr); +uint32_t vfp_read_fpscr(void); + +#endif /* ARM32 */ + +#endif /*VFP_PRIVATE*/ diff --git a/optee_os/core/arch/arm/kernel/virtualization.c b/optee_os/core/arch/arm/kernel/virtualization.c new file mode 100644 index 0000000..54b73c7 --- /dev/null +++ b/optee_os/core/arch/arm/kernel/virtualization.c @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* Copyright (c) 2018, EPAM Systems. All rights reserved. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned int prtn_list_lock __nex_data = SPINLOCK_UNLOCK; + +static LIST_HEAD(prtn_list_head, guest_partition) prtn_list __nex_data = + LIST_HEAD_INITIALIZER(prtn_list_head); + +/* Free pages used for guest partitions */ +tee_mm_pool_t virt_mapper_pool __nex_bss; + +/* Memory used by OP-TEE core */ +struct tee_mmap_region *kmemory_map __nex_bss; + +struct guest_partition { + LIST_ENTRY(guest_partition) link; + struct mmu_partition *mmu_prtn; + struct tee_mmap_region *memory_map; + struct mutex mutex; + void *tables_va; + tee_mm_entry_t *tee_ram; + tee_mm_entry_t *ta_ram; + tee_mm_entry_t *tables; + bool runtime_initialized; + uint16_t id; + struct refcount refc; +#ifdef CFG_CORE_SEL1_SPMC + uint64_t cookies[64]; + uint8_t cookie_count; +#endif +}; + +struct guest_partition *current_partition[CFG_TEE_CORE_NB_CORE] __nex_bss; + +static struct guest_partition *get_current_prtn(void) +{ + struct guest_partition *ret; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + ret = current_partition[get_core_pos()]; + + thread_unmask_exceptions(exceptions); + + return ret; +} + +uint16_t virt_get_current_guest_id(void) +{ + struct guest_partition *prtn = get_current_prtn(); + + if (!prtn) + return 0; + return prtn->id; +} + +static void set_current_prtn(struct guest_partition *prtn) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + current_partition[get_core_pos()] = prtn; + + thread_unmask_exceptions(exceptions); +} + +static size_t get_ta_ram_size(void) +{ + size_t ta_size = 0; + + core_mmu_get_ta_range(NULL, &ta_size); + return ROUNDDOWN(ta_size / CFG_VIRT_GUEST_COUNT - VCORE_UNPG_RW_SZ - + core_mmu_get_total_pages_size(), SMALL_PAGE_SIZE); +} + +static struct tee_mmap_region *prepare_memory_map(paddr_t tee_data, + paddr_t ta_ram) +{ + int i, entries; + vaddr_t max_va = 0; + struct tee_mmap_region *map; + /* + * This function assumes that at time of operation, + * kmemory_map (aka static_memory_map from core_mmu.c) + * will not be altered. This is true, because all + * changes to static_memory_map are done during + * OP-TEE initialization, while this function will + * called when hypervisor creates a guest. + */ + + /* Count number of entries in nexus memory map */ + for (map = kmemory_map, entries = 1; map->type != MEM_AREA_END; + map++, entries++) + ; + + /* Allocate entries for virtual guest map */ + map = nex_calloc(entries + 1, sizeof(struct tee_mmap_region)); + if (!map) + return NULL; + + memcpy(map, kmemory_map, sizeof(*map) * entries); + + /* Map TEE .data and .bss sections */ + for (i = 0; i < entries; i++) { + if (map[i].va == (vaddr_t)(VCORE_UNPG_RW_PA)) { + map[i].type = MEM_AREA_TEE_RAM_RW; + map[i].attr = core_mmu_type_to_attr(map[i].type); + map[i].pa = tee_data; + } + if (map[i].va + map[i].size > max_va) + max_va = map[i].va + map[i].size; + } + + /* Map TA_RAM */ + assert(map[entries - 1].type == MEM_AREA_END); + map[entries] = map[entries - 1]; + map[entries - 1].region_size = SMALL_PAGE_SIZE; + map[entries - 1].va = ROUNDUP(max_va, map[entries - 1].region_size); + map[entries - 1].va += + (ta_ram - map[entries - 1].va) & CORE_MMU_PGDIR_MASK; + map[entries - 1].pa = ta_ram; + map[entries - 1].size = get_ta_ram_size(); + map[entries - 1].type = MEM_AREA_TA_RAM; + map[entries - 1].attr = core_mmu_type_to_attr(map[entries - 1].type); + + DMSG("New map (%08lx):", (vaddr_t)(VCORE_UNPG_RW_PA)); + + for (i = 0; i < entries; i++) + DMSG("T: %-16s rsz: %08x, pa: %08lx, va: %08lx, sz: %08lx attr: %x", + teecore_memtype_name(map[i].type), + map[i].region_size, map[i].pa, map[i].va, + map[i].size, map[i].attr); + return map; +} + +void virt_init_memory(struct tee_mmap_region *memory_map, paddr_t secmem0_base, + paddr_size_t secmem0_size, paddr_t secmem1_base, + paddr_size_t secmem1_size) +{ + struct tee_mmap_region *map = NULL; + paddr_size_t size = secmem0_size; + paddr_t base = secmem0_base; + + if (secmem1_size) { + assert(secmem0_base + secmem0_size <= secmem1_base); + size = secmem1_base + secmem1_size - base; + } + + /* Init page pool that covers all secure RAM */ + if (!tee_mm_init(&virt_mapper_pool, base, size, + SMALL_PAGE_SHIFT, TEE_MM_POOL_NEX_MALLOC)) + panic("Can't create pool with free pages"); + DMSG("Created virtual mapper pool from %"PRIxPA" to %"PRIxPA, + base, base + size); + + if (secmem1_size) { + /* Carve out an eventual gap between secmem0 and secmem1 */ + base = secmem0_base + secmem0_size; + size = secmem1_base - base; + if (size) { + DMSG("Carving out gap between secmem0 and secmem1 (0x%"PRIxPA":0x%"PRIxPASZ")", + base, size); + if (!tee_mm_alloc2(&virt_mapper_pool, base, size)) + panic("Can't carve out secmem gap"); + } + } + + + /* Carve out areas that are used by OP-TEE core */ + for (map = memory_map; map->type != MEM_AREA_END; map++) { + switch (map->type) { + case MEM_AREA_TEE_RAM_RX: + case MEM_AREA_TEE_RAM_RO: + case MEM_AREA_NEX_RAM_RO: + case MEM_AREA_NEX_RAM_RW: + DMSG("Carving out area of type %d (0x%08lx-0x%08lx)", + map->type, map->pa, map->pa + map->size); + if (!tee_mm_alloc2(&virt_mapper_pool, map->pa, + map->size)) + panic("Can't carve out used area"); + break; + default: + continue; + } + } + + kmemory_map = memory_map; +} + + +static TEE_Result configure_guest_prtn_mem(struct guest_partition *prtn) +{ + TEE_Result res = TEE_SUCCESS; + paddr_t original_data_pa = 0; + + prtn->tee_ram = tee_mm_alloc(&virt_mapper_pool, VCORE_UNPG_RW_SZ); + if (!prtn->tee_ram) { + EMSG("Can't allocate memory for TEE runtime context"); + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + DMSG("TEE RAM: %08" PRIxPA, tee_mm_get_smem(prtn->tee_ram)); + + prtn->ta_ram = tee_mm_alloc(&virt_mapper_pool, get_ta_ram_size()); + if (!prtn->ta_ram) { + EMSG("Can't allocate memory for TA data"); + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + DMSG("TA RAM: %08" PRIxPA, tee_mm_get_smem(prtn->ta_ram)); + + prtn->tables = tee_mm_alloc(&virt_mapper_pool, + core_mmu_get_total_pages_size()); + if (!prtn->tables) { + EMSG("Can't allocate memory for page tables"); + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + prtn->tables_va = phys_to_virt(tee_mm_get_smem(prtn->tables), + MEM_AREA_SEC_RAM_OVERALL, + core_mmu_get_total_pages_size()); + assert(prtn->tables_va); + + prtn->mmu_prtn = core_alloc_mmu_prtn(prtn->tables_va); + if (!prtn->mmu_prtn) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + prtn->memory_map = prepare_memory_map(tee_mm_get_smem(prtn->tee_ram), + tee_mm_get_smem(prtn->ta_ram)); + if (!prtn->memory_map) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + core_init_mmu_prtn(prtn->mmu_prtn, prtn->memory_map); + + original_data_pa = virt_to_phys(__data_start); + /* Switch to guest's mappings */ + core_mmu_set_prtn(prtn->mmu_prtn); + + /* clear .bss */ + memset((void *)(VCORE_UNPG_RW_PA), 0, VCORE_UNPG_RW_SZ); + + /* copy .data section from R/O original */ + memcpy(__data_start, + phys_to_virt(original_data_pa, MEM_AREA_SEC_RAM_OVERALL, + __data_end - __data_start), + __data_end - __data_start); + + return TEE_SUCCESS; + +err: + if (prtn->tee_ram) + tee_mm_free(prtn->tee_ram); + if (prtn->ta_ram) + tee_mm_free(prtn->ta_ram); + if (prtn->tables) + tee_mm_free(prtn->tables); + nex_free(prtn->mmu_prtn); + nex_free(prtn->memory_map); + + return res; +} + +TEE_Result virt_guest_created(uint16_t guest_id) +{ + struct guest_partition *prtn = NULL; + TEE_Result res = TEE_SUCCESS; + uint32_t exceptions = 0; + + prtn = nex_calloc(1, sizeof(*prtn)); + if (!prtn) + return TEE_ERROR_OUT_OF_MEMORY; + + prtn->id = guest_id; + mutex_init(&prtn->mutex); + refcount_set(&prtn->refc, 1); + res = configure_guest_prtn_mem(prtn); + if (res) { + nex_free(prtn); + return res; + } + + set_current_prtn(prtn); + + /* Initialize threads */ + thread_init_threads(); + /* Do the preinitcalls */ + call_preinitcalls(); + + exceptions = cpu_spin_lock_xsave(&prtn_list_lock); + LIST_INSERT_HEAD(&prtn_list, prtn, link); + cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); + + IMSG("Added guest %d", guest_id); + + set_current_prtn(NULL); + core_mmu_set_default_prtn(); + + return TEE_SUCCESS; +} + +TEE_Result virt_guest_destroyed(uint16_t guest_id) +{ + struct guest_partition *prtn; + uint32_t exceptions; + + IMSG("Removing guest %d", guest_id); + + exceptions = cpu_spin_lock_xsave(&prtn_list_lock); + + LIST_FOREACH(prtn, &prtn_list, link) { + if (prtn->id == guest_id) { + LIST_REMOVE(prtn, link); + break; + } + } + cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); + + if (prtn) { + if (!refcount_dec(&prtn->refc)) { + EMSG("Guest thread(s) is still running. refc = %d", + refcount_val(&prtn->refc)); + panic(); + } + + tee_mm_free(prtn->tee_ram); + tee_mm_free(prtn->ta_ram); + tee_mm_free(prtn->tables); + core_free_mmu_prtn(prtn->mmu_prtn); + nex_free(prtn->memory_map); + nex_free(prtn); + } else + EMSG("Client with id %d is not found", guest_id); + + return TEE_SUCCESS; +} + +TEE_Result virt_set_guest(uint16_t guest_id) +{ + struct guest_partition *prtn; + uint32_t exceptions; + + prtn = get_current_prtn(); + + /* This can be true only if we return from IRQ RPC */ + if (prtn && prtn->id == guest_id) + return TEE_SUCCESS; + + if (prtn) + panic("Virtual guest partition is already set"); + + exceptions = cpu_spin_lock_xsave(&prtn_list_lock); + LIST_FOREACH(prtn, &prtn_list, link) { + if (prtn->id == guest_id) { + set_current_prtn(prtn); + core_mmu_set_prtn(prtn->mmu_prtn); + refcount_inc(&prtn->refc); + cpu_spin_unlock_xrestore(&prtn_list_lock, + exceptions); + return TEE_SUCCESS; + } + } + cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); + + if (guest_id == HYP_CLNT_ID) + return TEE_SUCCESS; + return TEE_ERROR_ITEM_NOT_FOUND; +} + +void virt_unset_guest(void) +{ + struct guest_partition *prtn = get_current_prtn(); + + if (!prtn) + return; + + set_current_prtn(NULL); + core_mmu_set_default_prtn(); + if (refcount_dec(&prtn->refc)) + panic(); +} + +void virt_on_stdcall(void) +{ + struct guest_partition *prtn = get_current_prtn(); + + /* Initialize runtime on first std call */ + if (!prtn->runtime_initialized) { + mutex_lock(&prtn->mutex); + if (!prtn->runtime_initialized) { + init_tee_runtime(); + prtn->runtime_initialized = true; + } + mutex_unlock(&prtn->mutex); + } +} + +struct tee_mmap_region *virt_get_memory_map(void) +{ + struct guest_partition *prtn; + + prtn = get_current_prtn(); + + if (!prtn) + return NULL; + + return prtn->memory_map; +} + +void virt_get_ta_ram(vaddr_t *start, vaddr_t *end) +{ + struct guest_partition *prtn = get_current_prtn(); + + *start = (vaddr_t)phys_to_virt(tee_mm_get_smem(prtn->ta_ram), + MEM_AREA_TA_RAM, + tee_mm_get_bytes(prtn->ta_ram)); + *end = *start + tee_mm_get_bytes(prtn->ta_ram); +} + +#ifdef CFG_CORE_SEL1_SPMC +static int find_cookie(struct guest_partition *prtn, uint64_t cookie) +{ + int i = 0; + + for (i = 0; i < prtn->cookie_count; i++) + if (prtn->cookies[i] == cookie) + return i; + return -1; +} + +static struct guest_partition *find_prtn_cookie(uint64_t cookie, int *idx) +{ + struct guest_partition *prtn = NULL; + int i = 0; + + LIST_FOREACH(prtn, &prtn_list, link) { + i = find_cookie(prtn, cookie); + if (i >= 0) { + if (idx) + *idx = i; + return prtn; + } + } + + return NULL; +} + +TEE_Result virt_add_cookie_to_current_guest(uint64_t cookie) +{ + TEE_Result res = TEE_ERROR_ACCESS_DENIED; + struct guest_partition *prtn = NULL; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + if (find_prtn_cookie(cookie, NULL)) + goto out; + + prtn = current_partition[get_core_pos()]; + if (prtn->cookie_count < ARRAY_SIZE(prtn->cookies)) { + prtn->cookies[prtn->cookie_count] = cookie; + prtn->cookie_count++; + res = TEE_SUCCESS; + } +out: + thread_unmask_exceptions(exceptions); + + return res; +} + +void virt_remove_cookie(uint64_t cookie) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + struct guest_partition *prtn = NULL; + int i = 0; + + prtn = find_prtn_cookie(cookie, &i); + if (prtn) { + memmove(prtn->cookies + i, prtn->cookies + i + 1, + sizeof(uint64_t) * (prtn->cookie_count - i - 1)); + prtn->cookie_count--; + } + thread_unmask_exceptions(exceptions); +} + +uint16_t virt_find_guest_by_cookie(uint64_t cookie) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + struct guest_partition *prtn = NULL; + uint16_t ret = 0; + + prtn = find_prtn_cookie(cookie, NULL); + if (prtn) + ret = prtn->id; + + thread_unmask_exceptions(exceptions); + + return ret; +} +#endif /*CFG_CORE_SEL1_SPMC*/ diff --git a/optee_os/core/arch/arm/mm/core_mmu.c b/optee_os/core/arch/arm/mm/core_mmu.c new file mode 100644 index 0000000..e25f5cc --- /dev/null +++ b/optee_os/core/arch/arm/mm/core_mmu.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Two ASIDs per context, one for kernel mode and one for user mode. ASID 0 + * and 1 are reserved and not used. This means a maximum of 126 loaded user + * mode contexts. This value can be increased but not beyond the maximum + * ASID, which is architecture dependent (max 255 for ARMv7-A and ARMv8-A + * Aarch32). This constant defines number of ASID pairs. + */ +#define MMU_NUM_ASID_PAIRS 64 + +static bitstr_t bit_decl(g_asid, MMU_NUM_ASID_PAIRS) __nex_bss; +static unsigned int g_asid_spinlock __nex_bss = SPINLOCK_UNLOCK; + +void tlbi_va_range(vaddr_t va, size_t len, size_t granule) +{ + assert(granule == CORE_MMU_PGDIR_SIZE || granule == SMALL_PAGE_SIZE); + assert(!(va & (granule - 1)) && !(len & (granule - 1))); + + dsb_ishst(); + while (len) { + tlbi_va_allasid_nosync(va); + len -= granule; + va += granule; + } + dsb_ish(); + isb(); +} + +void tlbi_va_range_asid(vaddr_t va, size_t len, size_t granule, uint32_t asid) +{ + assert(granule == CORE_MMU_PGDIR_SIZE || granule == SMALL_PAGE_SIZE); + assert(!(va & (granule - 1)) && !(len & (granule - 1))); + + dsb_ishst(); + while (len) { + tlbi_va_asid_nosync(va, asid); + len -= granule; + va += granule; + } + dsb_ish(); + isb(); +} + +TEE_Result cache_op_inner(enum cache_op op, void *va, size_t len) +{ + switch (op) { + case DCACHE_CLEAN: + dcache_op_all(DCACHE_OP_CLEAN); + break; + case DCACHE_AREA_CLEAN: + dcache_clean_range(va, len); + break; + case DCACHE_INVALIDATE: + dcache_op_all(DCACHE_OP_INV); + break; + case DCACHE_AREA_INVALIDATE: + dcache_inv_range(va, len); + break; + case ICACHE_INVALIDATE: + icache_inv_all(); + break; + case ICACHE_AREA_INVALIDATE: + icache_inv_range(va, len); + break; + case DCACHE_CLEAN_INV: + dcache_op_all(DCACHE_OP_CLEAN_INV); + break; + case DCACHE_AREA_CLEAN_INV: + dcache_cleaninv_range(va, len); + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + return TEE_SUCCESS; +} + +#ifdef CFG_PL310 +TEE_Result cache_op_outer(enum cache_op op, paddr_t pa, size_t len) +{ + TEE_Result ret = TEE_SUCCESS; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + tee_l2cc_mutex_lock(); + switch (op) { + case DCACHE_INVALIDATE: + arm_cl2_invbyway(pl310_base()); + break; + case DCACHE_AREA_INVALIDATE: + if (len) + arm_cl2_invbypa(pl310_base(), pa, pa + len - 1); + break; + case DCACHE_CLEAN: + arm_cl2_cleanbyway(pl310_base()); + break; + case DCACHE_AREA_CLEAN: + if (len) + arm_cl2_cleanbypa(pl310_base(), pa, pa + len - 1); + break; + case DCACHE_CLEAN_INV: + arm_cl2_cleaninvbyway(pl310_base()); + break; + case DCACHE_AREA_CLEAN_INV: + if (len) + arm_cl2_cleaninvbypa(pl310_base(), pa, pa + len - 1); + break; + default: + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + tee_l2cc_mutex_unlock(); + thread_set_exceptions(exceptions); + return ret; +} +#endif /*CFG_PL310*/ + +unsigned int asid_alloc(void) +{ + uint32_t exceptions = cpu_spin_lock_xsave(&g_asid_spinlock); + unsigned int r; + int i; + + bit_ffc(g_asid, MMU_NUM_ASID_PAIRS, &i); + if (i == -1) { + r = 0; + } else { + bit_set(g_asid, i); + r = (i + 1) * 2; + } + + cpu_spin_unlock_xrestore(&g_asid_spinlock, exceptions); + return r; +} + +void asid_free(unsigned int asid) +{ + uint32_t exceptions = cpu_spin_lock_xsave(&g_asid_spinlock); + + /* Only even ASIDs are supposed to be allocated */ + assert(!(asid & 1)); + + if (asid) { + int i = (asid - 1) / 2; + + assert(i < MMU_NUM_ASID_PAIRS && bit_test(g_asid, i)); + bit_clear(g_asid, i); + } + + cpu_spin_unlock_xrestore(&g_asid_spinlock, exceptions); +} + +bool arch_va2pa_helper(void *va, paddr_t *pa) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + paddr_t par = 0; + paddr_t par_pa_mask = 0; + bool ret = false; + +#ifdef ARM32 + write_ats1cpr((vaddr_t)va); + isb(); +#ifdef CFG_WITH_LPAE + par = read_par64(); + par_pa_mask = PAR64_PA_MASK; +#else + par = read_par32(); + par_pa_mask = PAR32_PA_MASK; +#endif +#endif /*ARM32*/ + +#ifdef ARM64 + write_at_s1e1r((vaddr_t)va); + isb(); + par = read_par_el1(); + par_pa_mask = PAR_PA_MASK; +#endif + if (par & PAR_F) + goto out; + *pa = (par & (par_pa_mask << PAR_PA_SHIFT)) | + ((vaddr_t)va & (BIT64(PAR_PA_SHIFT) - 1)); + + ret = true; +out: + thread_unmask_exceptions(exceptions); + return ret; +} + +bool cpu_mmu_enabled(void) +{ + uint32_t sctlr; + +#ifdef ARM32 + sctlr = read_sctlr(); +#else + sctlr = read_sctlr_el1(); +#endif + + return sctlr & SCTLR_M ? true : false; +} diff --git a/optee_os/core/arch/arm/mm/core_mmu_lpae.c b/optee_os/core/arch/arm/mm/core_mmu_lpae.c new file mode 100644 index 0000000..7e79f78 --- /dev/null +++ b/optee_os/core/arch/arm/mm/core_mmu_lpae.c @@ -0,0 +1,1356 @@ +// SPDX-License-Identifier: (BSD-2-Clause AND BSD-3-Clause) +/* + * Copyright (c) 2015-2016, 2022 Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright (c) 2014, 2022, ARM Limited and Contributors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of ARM nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef DEBUG_XLAT_TABLE +#define DEBUG_XLAT_TABLE 0 +#endif + +#if DEBUG_XLAT_TABLE +#define debug_print(...) DMSG_RAW(__VA_ARGS__) +#else +#define debug_print(...) ((void)0) +#endif + + +/* + * Miscellaneous MMU related constants + */ + +#define INVALID_DESC 0x0 +#define BLOCK_DESC 0x1 +#define L3_BLOCK_DESC 0x3 +#define TABLE_DESC 0x3 +#define DESC_ENTRY_TYPE_MASK 0x3 + +#define XN (1ull << 2) +#define PXN (1ull << 1) +#define CONT_HINT (1ull << 0) + +#define UPPER_ATTRS(x) (((x) & 0x7) << 52) +#define GP BIT64(50) /* Guarded Page, Aarch64 FEAT_BTI */ +#define NON_GLOBAL (1ull << 9) +#define ACCESS_FLAG (1ull << 8) +#define NSH (0x0 << 6) +#define OSH (0x2 << 6) +#define ISH (0x3 << 6) + +#define AP_RO (0x1 << 5) +#define AP_RW (0x0 << 5) +#define AP_UNPRIV (0x1 << 4) + +#define NS (0x1 << 3) +#define LOWER_ATTRS_SHIFT 2 +#define LOWER_ATTRS(x) (((x) & 0xfff) << LOWER_ATTRS_SHIFT) + +#define ATTR_DEVICE_nGnRE_INDEX 0x0 +#define ATTR_IWBWA_OWBWA_NTR_INDEX 0x1 +#define ATTR_DEVICE_nGnRnE_INDEX 0x2 +#define ATTR_TAGGED_NORMAL_MEM_INDEX 0x3 +#define ATTR_INDEX_MASK 0x7 + +#define ATTR_DEVICE_nGnRnE (0x0) +#define ATTR_DEVICE_nGnRE (0x4) +#define ATTR_IWBWA_OWBWA_NTR (0xff) +/* Same as ATTR_IWBWA_OWBWA_NTR but with memory tagging. */ +#define ATTR_TAGGED_NORMAL_MEM (0xf0) + +#define MAIR_ATTR_SET(attr, index) (((uint64_t)attr) << ((index) << 3)) + +#define OUTPUT_ADDRESS_MASK (0x0000FFFFFFFFF000ULL) + +/* (internal) physical address size bits in EL3/EL1 */ +#define TCR_PS_BITS_4GB (0x0) +#define TCR_PS_BITS_64GB (0x1) +#define TCR_PS_BITS_1TB (0x2) +#define TCR_PS_BITS_4TB (0x3) +#define TCR_PS_BITS_16TB (0x4) +#define TCR_PS_BITS_256TB (0x5) + +#define UNSET_DESC ((uint64_t)-1) + +#define FOUR_KB_SHIFT 12 +#define PAGE_SIZE_SHIFT FOUR_KB_SHIFT +#define PAGE_SIZE (1 << PAGE_SIZE_SHIFT) +#define PAGE_SIZE_MASK (PAGE_SIZE - 1) +#define IS_PAGE_ALIGNED(addr) (((addr) & PAGE_SIZE_MASK) == 0) + +#define XLAT_ENTRY_SIZE_SHIFT 3 /* Each MMU table entry is 8 bytes (1 << 3) */ +#define XLAT_ENTRY_SIZE (1 << XLAT_ENTRY_SIZE_SHIFT) + +#define XLAT_TABLE_SIZE_SHIFT PAGE_SIZE_SHIFT +#define XLAT_TABLE_SIZE (1 << XLAT_TABLE_SIZE_SHIFT) + +#define XLAT_TABLE_LEVEL_MAX U(3) + +/* Values for number of entries in each MMU translation table */ +#define XLAT_TABLE_ENTRIES_SHIFT (XLAT_TABLE_SIZE_SHIFT - XLAT_ENTRY_SIZE_SHIFT) +#define XLAT_TABLE_ENTRIES (1 << XLAT_TABLE_ENTRIES_SHIFT) +#define XLAT_TABLE_ENTRIES_MASK (XLAT_TABLE_ENTRIES - 1) + +/* Values to convert a memory address to an index into a translation table */ +#define L3_XLAT_ADDRESS_SHIFT PAGE_SIZE_SHIFT +#define L2_XLAT_ADDRESS_SHIFT (L3_XLAT_ADDRESS_SHIFT + \ + XLAT_TABLE_ENTRIES_SHIFT) +#define L1_XLAT_ADDRESS_SHIFT (L2_XLAT_ADDRESS_SHIFT + \ + XLAT_TABLE_ENTRIES_SHIFT) +#define L0_XLAT_ADDRESS_SHIFT (L1_XLAT_ADDRESS_SHIFT + \ + XLAT_TABLE_ENTRIES_SHIFT) +#define XLAT_ADDR_SHIFT(level) (PAGE_SIZE_SHIFT + \ + ((XLAT_TABLE_LEVEL_MAX - (level)) * \ + XLAT_TABLE_ENTRIES_SHIFT)) + +#define XLAT_BLOCK_SIZE(level) (UL(1) << XLAT_ADDR_SHIFT(level)) + +/* Base table */ +#define BASE_XLAT_ADDRESS_SHIFT XLAT_ADDR_SHIFT(CORE_MMU_BASE_TABLE_LEVEL) +#define BASE_XLAT_BLOCK_SIZE XLAT_BLOCK_SIZE(CORE_MMU_BASE_TABLE_LEVEL) + +#define NUM_BASE_LEVEL_ENTRIES \ + BIT(CFG_LPAE_ADDR_SPACE_BITS - BASE_XLAT_ADDRESS_SHIFT) + +/* + * MMU L1 table, one for each core + * + * With CFG_CORE_UNMAP_CORE_AT_EL0, each core has one table to be used + * while in kernel mode and one to be used while in user mode. + */ +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 +#define NUM_BASE_TABLES 2 +#else +#define NUM_BASE_TABLES 1 +#endif + +#ifndef MAX_XLAT_TABLES +#ifdef CFG_NS_VIRTUALIZATION +# define XLAT_TABLE_VIRTUALIZATION_EXTRA 3 +#else +# define XLAT_TABLE_VIRTUALIZATION_EXTRA 0 +#endif +#ifdef CFG_CORE_ASLR +# define XLAT_TABLE_ASLR_EXTRA 3 +#else +# define XLAT_TABLE_ASLR_EXTRA 0 +#endif +#if (CORE_MMU_BASE_TABLE_LEVEL == 0) +# define XLAT_TABLE_TEE_EXTRA 8 +# define XLAT_TABLE_USER_EXTRA (NUM_BASE_TABLES * CFG_TEE_CORE_NB_CORE) +#else +# define XLAT_TABLE_TEE_EXTRA 5 +# define XLAT_TABLE_USER_EXTRA 0 +#endif +#define MAX_XLAT_TABLES (XLAT_TABLE_TEE_EXTRA + \ + XLAT_TABLE_VIRTUALIZATION_EXTRA + \ + XLAT_TABLE_ASLR_EXTRA + \ + XLAT_TABLE_USER_EXTRA) +#endif /*!MAX_XLAT_TABLES*/ + +#if (CORE_MMU_BASE_TABLE_LEVEL == 0) +#if (MAX_XLAT_TABLES <= UINT8_MAX) +typedef uint8_t l1_idx_t; +#elif (MAX_XLAT_TABLES <= UINT16_MAX) +typedef uint16_t l1_idx_t; +#else +#error MAX_XLAT_TABLES is suspiciously large, please check +#endif +#endif + +typedef uint64_t base_xlat_tbls_t[CFG_TEE_CORE_NB_CORE][NUM_BASE_LEVEL_ENTRIES]; +typedef uint64_t xlat_tbl_t[XLAT_TABLE_ENTRIES]; + +static base_xlat_tbls_t base_xlation_table[NUM_BASE_TABLES] + __aligned(NUM_BASE_LEVEL_ENTRIES * XLAT_ENTRY_SIZE) + __section(".nozi.mmu.base_table"); + +static xlat_tbl_t xlat_tables[MAX_XLAT_TABLES] + __aligned(XLAT_TABLE_SIZE) __section(".nozi.mmu.l2"); + +#define XLAT_TABLES_SIZE (sizeof(xlat_tbl_t) * MAX_XLAT_TABLES) + +/* MMU L2 table for TAs, one for each thread */ +static xlat_tbl_t xlat_tables_ul1[CFG_NUM_THREADS] + __aligned(XLAT_TABLE_SIZE) __section(".nozi.mmu.l2"); + +/* + * TAs page table entry inside a level 1 page table. + * + * TAs mapping is expected to start from level 2. + * + * If base level is 1 then this is the index of a level 1 entry, + * that will point directly into TA mapping table. + * + * If base level is 0 then entry 0 in base table is always used, and then + * we fallback to "base level == 1" like scenario. + */ +static int user_va_idx __nex_data = -1; + +struct mmu_partition { + base_xlat_tbls_t *base_tables; + xlat_tbl_t *xlat_tables; + xlat_tbl_t *l2_ta_tables; + unsigned int xlat_tables_used; + unsigned int asid; + +#if (CORE_MMU_BASE_TABLE_LEVEL == 0) + /* + * Indexes of the L1 table from 'xlat_tables' + * that points to the user mappings. + */ + l1_idx_t user_l1_table_idx[NUM_BASE_TABLES][CFG_TEE_CORE_NB_CORE]; +#endif +}; + +static struct mmu_partition default_partition __nex_data = { + .base_tables = base_xlation_table, + .xlat_tables = xlat_tables, + .l2_ta_tables = xlat_tables_ul1, + .xlat_tables_used = 0, + .asid = 0 +}; + +#ifdef CFG_NS_VIRTUALIZATION +static struct mmu_partition *current_prtn[CFG_TEE_CORE_NB_CORE] __nex_bss; +#endif + +static struct mmu_partition *get_prtn(void) +{ +#ifdef CFG_NS_VIRTUALIZATION + struct mmu_partition *ret; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + ret = current_prtn[get_core_pos()]; + + thread_unmask_exceptions(exceptions); + return ret; +#else + return &default_partition; +#endif +} + +static uint32_t desc_to_mattr(unsigned level, uint64_t desc) +{ + uint32_t a; + + if (!(desc & 1)) + return 0; + + if (level == XLAT_TABLE_LEVEL_MAX) { + if ((desc & DESC_ENTRY_TYPE_MASK) != L3_BLOCK_DESC) + return 0; + } else { + if ((desc & DESC_ENTRY_TYPE_MASK) == TABLE_DESC) + return TEE_MATTR_TABLE; + } + + a = TEE_MATTR_VALID_BLOCK; + + if (desc & LOWER_ATTRS(ACCESS_FLAG)) + a |= TEE_MATTR_PRX | TEE_MATTR_URX; + + if (!(desc & LOWER_ATTRS(AP_RO))) + a |= TEE_MATTR_PW | TEE_MATTR_UW; + + if (!(desc & LOWER_ATTRS(AP_UNPRIV))) + a &= ~TEE_MATTR_URWX; + + if (desc & UPPER_ATTRS(XN)) + a &= ~(TEE_MATTR_PX | TEE_MATTR_UX); + + if (desc & UPPER_ATTRS(PXN)) + a &= ~TEE_MATTR_PX; + + COMPILE_TIME_ASSERT(ATTR_DEVICE_nGnRnE_INDEX == + TEE_MATTR_MEM_TYPE_STRONGLY_O); + COMPILE_TIME_ASSERT(ATTR_DEVICE_nGnRE_INDEX == TEE_MATTR_MEM_TYPE_DEV); + COMPILE_TIME_ASSERT(ATTR_IWBWA_OWBWA_NTR_INDEX == + TEE_MATTR_MEM_TYPE_CACHED); + COMPILE_TIME_ASSERT(ATTR_TAGGED_NORMAL_MEM_INDEX == + TEE_MATTR_MEM_TYPE_TAGGED); + + a |= ((desc & LOWER_ATTRS(ATTR_INDEX_MASK)) >> LOWER_ATTRS_SHIFT) << + TEE_MATTR_MEM_TYPE_SHIFT; + + if (!(desc & LOWER_ATTRS(NON_GLOBAL))) + a |= TEE_MATTR_GLOBAL; + + if (!(desc & LOWER_ATTRS(NS))) + a |= TEE_MATTR_SECURE; + + if (desc & GP) + a |= TEE_MATTR_GUARDED; + + return a; +} + +static uint64_t mattr_to_desc(unsigned level, uint32_t attr) +{ + uint64_t desc; + uint32_t a = attr; + + if (a & TEE_MATTR_TABLE) + return TABLE_DESC; + + if (!(a & TEE_MATTR_VALID_BLOCK)) + return 0; + + if (a & (TEE_MATTR_PX | TEE_MATTR_PW)) + a |= TEE_MATTR_PR; + if (a & (TEE_MATTR_UX | TEE_MATTR_UW)) + a |= TEE_MATTR_UR; + if (a & TEE_MATTR_UR) + a |= TEE_MATTR_PR; + if (a & TEE_MATTR_UW) + a |= TEE_MATTR_PW; + + if (IS_ENABLED(CFG_CORE_BTI) && (a & TEE_MATTR_PX)) + a |= TEE_MATTR_GUARDED; + + if (level == XLAT_TABLE_LEVEL_MAX) + desc = L3_BLOCK_DESC; + else + desc = BLOCK_DESC; + + if (!(a & (TEE_MATTR_PX | TEE_MATTR_UX))) + desc |= UPPER_ATTRS(XN); + if (!(a & TEE_MATTR_PX)) + desc |= UPPER_ATTRS(PXN); + + if (a & TEE_MATTR_UR) + desc |= LOWER_ATTRS(AP_UNPRIV); + + if (!(a & TEE_MATTR_PW)) + desc |= LOWER_ATTRS(AP_RO); + + if (feat_bti_is_implemented() && (a & TEE_MATTR_GUARDED)) + desc |= GP; + + /* Keep in sync with core_mmu.c:core_mmu_mattr_is_ok */ + switch ((a >> TEE_MATTR_MEM_TYPE_SHIFT) & TEE_MATTR_MEM_TYPE_MASK) { + case TEE_MATTR_MEM_TYPE_STRONGLY_O: + desc |= LOWER_ATTRS(ATTR_DEVICE_nGnRnE_INDEX | OSH); + break; + case TEE_MATTR_MEM_TYPE_DEV: + desc |= LOWER_ATTRS(ATTR_DEVICE_nGnRE_INDEX | OSH); + break; + case TEE_MATTR_MEM_TYPE_CACHED: + desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); + break; + case TEE_MATTR_MEM_TYPE_TAGGED: + desc |= LOWER_ATTRS(ATTR_TAGGED_NORMAL_MEM_INDEX | ISH); + break; + default: + /* + * "Can't happen" the attribute is supposed to be checked + * with core_mmu_mattr_is_ok() before. + */ + panic(); + } + + if (a & (TEE_MATTR_UR | TEE_MATTR_PR)) + desc |= LOWER_ATTRS(ACCESS_FLAG); + + if (!(a & TEE_MATTR_GLOBAL)) + desc |= LOWER_ATTRS(NON_GLOBAL); + + desc |= a & TEE_MATTR_SECURE ? 0 : LOWER_ATTRS(NS); + + return desc; +} + +#ifdef CFG_NS_VIRTUALIZATION +size_t core_mmu_get_total_pages_size(void) +{ + return ROUNDUP(sizeof(base_xlation_table), SMALL_PAGE_SIZE) + + sizeof(xlat_tables) + sizeof(xlat_tables_ul1); +} + +struct mmu_partition *core_alloc_mmu_prtn(void *tables) +{ + struct mmu_partition *prtn; + uint8_t *tbl = tables; + unsigned int asid = asid_alloc(); + + assert(((vaddr_t)tbl) % SMALL_PAGE_SIZE == 0); + + if (!asid) + return NULL; + + prtn = nex_malloc(sizeof(*prtn)); + if (!prtn) { + asid_free(asid); + return NULL; + } + + prtn->base_tables = (void *)tbl; + COMPILE_TIME_ASSERT(sizeof(base_xlation_table) <= SMALL_PAGE_SIZE); + memset(prtn->base_tables, 0, SMALL_PAGE_SIZE); + tbl += ROUNDUP(sizeof(base_xlation_table), SMALL_PAGE_SIZE); + + prtn->xlat_tables = (void *)tbl; + memset(prtn->xlat_tables, 0, XLAT_TABLES_SIZE); + tbl += XLAT_TABLES_SIZE; + assert(((vaddr_t)tbl) % SMALL_PAGE_SIZE == 0); + + prtn->l2_ta_tables = (void *)tbl; + prtn->xlat_tables_used = 0; + prtn->asid = asid; + + return prtn; +} + +void core_free_mmu_prtn(struct mmu_partition *prtn) +{ + asid_free(prtn->asid); + nex_free(prtn); +} + +void core_mmu_set_prtn(struct mmu_partition *prtn) +{ + uint64_t ttbr; + /* + * We are changing mappings for current CPU, + * so make sure that we will not be rescheduled + */ + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + + current_prtn[get_core_pos()] = prtn; + + ttbr = virt_to_phys(prtn->base_tables[0][get_core_pos()]); + + write_ttbr0_el1(ttbr | ((paddr_t)prtn->asid << TTBR_ASID_SHIFT)); + isb(); + tlbi_all(); +} + +void core_mmu_set_default_prtn(void) +{ + core_mmu_set_prtn(&default_partition); +} + +void core_mmu_set_default_prtn_tbl(void) +{ + size_t n = 0; + + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) + current_prtn[n] = &default_partition; +} +#endif + +static uint64_t *core_mmu_xlat_table_alloc(struct mmu_partition *prtn) +{ + uint64_t *new_table = NULL; + + if (prtn->xlat_tables_used >= MAX_XLAT_TABLES) { + EMSG("%u xlat tables exhausted", MAX_XLAT_TABLES); + + return NULL; + } + + new_table = prtn->xlat_tables[prtn->xlat_tables_used++]; + + DMSG("xlat tables used %u / %u", + prtn->xlat_tables_used, MAX_XLAT_TABLES); + + return new_table; +} + +/* + * Given an entry that points to a table returns the virtual address + * of the pointed table. NULL otherwise. + */ +static void *core_mmu_xlat_table_entry_pa2va(struct mmu_partition *prtn, + unsigned int level, + uint64_t entry) +{ + paddr_t pa = 0; + void *va = NULL; + + if ((entry & DESC_ENTRY_TYPE_MASK) != TABLE_DESC || + level >= XLAT_TABLE_LEVEL_MAX) + return NULL; + + pa = entry & OUTPUT_ADDRESS_MASK; + + if (!IS_ENABLED(CFG_NS_VIRTUALIZATION) || prtn == &default_partition) + va = phys_to_virt(pa, MEM_AREA_TEE_RAM_RW_DATA, + XLAT_TABLE_SIZE); + else + va = phys_to_virt(pa, MEM_AREA_SEC_RAM_OVERALL, + XLAT_TABLE_SIZE); + + return va; +} + +/* + * For a table entry that points to a table - allocate and copy to + * a new pointed table. This is done for the requested entry, + * without going deeper into the pointed table entries. + * + * A success is returned for non-table entries, as nothing to do there. + */ +__maybe_unused +static bool core_mmu_entry_copy(struct core_mmu_table_info *tbl_info, + unsigned int idx) +{ + uint64_t *orig_table = NULL; + uint64_t *new_table = NULL; + uint64_t *entry = NULL; + struct mmu_partition *prtn = NULL; + +#ifdef CFG_NS_VIRTUALIZATION + prtn = tbl_info->prtn; +#else + prtn = &default_partition; +#endif + assert(prtn); + + if (idx >= tbl_info->num_entries) + return false; + + entry = (uint64_t *)tbl_info->table + idx; + + /* Nothing to do for non-table entries */ + if ((*entry & DESC_ENTRY_TYPE_MASK) != TABLE_DESC || + tbl_info->level >= XLAT_TABLE_LEVEL_MAX) + return true; + + new_table = core_mmu_xlat_table_alloc(prtn); + if (!new_table) + return false; + + orig_table = core_mmu_xlat_table_entry_pa2va(prtn, tbl_info->level, + *entry); + if (!orig_table) + return false; + + /* Copy original table content to new table */ + memcpy(new_table, orig_table, XLAT_TABLE_ENTRIES * XLAT_ENTRY_SIZE); + + /* Point to the new table */ + *entry = virt_to_phys(new_table) | (*entry & ~OUTPUT_ADDRESS_MASK); + + return true; +} + +static void core_init_mmu_prtn_tee(struct mmu_partition *prtn, + struct tee_mmap_region *mm) +{ + size_t n; + + assert(prtn && mm); + + for (n = 0; !core_mmap_is_end_of_table(mm + n); n++) { + debug_print(" %010" PRIxVA " %010" PRIxPA " %10zx %x", + mm[n].va, mm[n].pa, mm[n].size, mm[n].attr); + + if (!IS_PAGE_ALIGNED(mm[n].pa) || !IS_PAGE_ALIGNED(mm[n].size)) + panic("unaligned region"); + } + + /* Clear table before use */ + memset(prtn->base_tables, 0, sizeof(base_xlation_table)); + + for (n = 0; !core_mmap_is_end_of_table(mm + n); n++) + if (!core_mmu_is_dynamic_vaspace(mm + n)) + core_mmu_map_region(prtn, mm + n); + + /* + * Primary mapping table is ready at index `get_core_pos()` + * whose value may not be ZERO. Take this index as copy source. + */ + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { + if (n == get_core_pos()) + continue; + + memcpy(prtn->base_tables[0][n], + prtn->base_tables[0][get_core_pos()], + XLAT_ENTRY_SIZE * NUM_BASE_LEVEL_ENTRIES); + } +} + +/* + * In order to support 32-bit TAs we will have to find + * a user VA base in the region [1GB, 4GB[. + * Due to OP-TEE design limitation, TAs page table should be an entry + * inside a level 1 page table. + * + * Available options are only these: + * - base level 0 entry 0 - [0GB, 512GB[ + * - level 1 entry 0 - [0GB, 1GB[ + * - level 1 entry 1 - [1GB, 2GB[ <---- + * - level 1 entry 2 - [2GB, 3GB[ <---- + * - level 1 entry 3 - [3GB, 4GB[ <---- + * - level 1 entry 4 - [4GB, 5GB[ + * - ... + * - ... + * + * - base level 1 entry 0 - [0GB, 1GB[ + * - base level 1 entry 1 - [1GB, 2GB[ <---- + * - base level 1 entry 2 - [2GB, 3GB[ <---- + * - base level 1 entry 3 - [3GB, 4GB[ <---- + * - base level 1 entry 4 - [4GB, 5GB[ + * - ... + */ +static void set_user_va_idx(struct mmu_partition *prtn) +{ + uint64_t *tbl = NULL; + unsigned int n = 0; + + assert(prtn); + + tbl = prtn->base_tables[0][get_core_pos()]; + + /* + * If base level is 0, then we must use its entry 0. + */ + if (CORE_MMU_BASE_TABLE_LEVEL == 0) { + /* + * If base level 0 entry 0 is not used then + * it's clear that we can use level 1 entry 1 inside it. + * (will be allocated later). + */ + if ((tbl[0] & DESC_ENTRY_TYPE_MASK) == INVALID_DESC) { + user_va_idx = 1; + + return; + } + + assert((tbl[0] & DESC_ENTRY_TYPE_MASK) == TABLE_DESC); + + tbl = core_mmu_xlat_table_entry_pa2va(prtn, 0, tbl[0]); + assert(tbl); + } + + /* + * Search level 1 table (i.e. 1GB mapping per entry) for + * an empty entry in the range [1GB, 4GB[. + */ + for (n = 1; n < 4; n++) { + if ((tbl[n] & DESC_ENTRY_TYPE_MASK) == INVALID_DESC) { + user_va_idx = n; + break; + } + } + + assert(user_va_idx != -1); +} + +/* + * Setup an entry inside a core level 1 page table for TAs memory mapping + * + * If base table level is 1 - user_va_idx is already the index, + * so nothing to do. + * If base table level is 0 - we might need to allocate entry 0 of base table, + * as TAs page table is an entry inside a level 1 + * page table. + */ +static void core_init_mmu_prtn_ta_core(struct mmu_partition *prtn + __maybe_unused, + unsigned int base_idx __maybe_unused, + unsigned int core __maybe_unused) +{ +#if (CORE_MMU_BASE_TABLE_LEVEL == 0) + struct core_mmu_table_info tbl_info = { }; + uint64_t *tbl = NULL; + uintptr_t idx = 0; + + assert(user_va_idx != -1); + COMPILE_TIME_ASSERT(MAX_XLAT_TABLES < + (1 << (8 * sizeof(prtn->user_l1_table_idx[0][0])))); + + tbl = prtn->base_tables[base_idx][core]; + + /* + * If base level is 0, then user_va_idx refers to + * level 1 page table that's in base level 0 entry 0. + */ + core_mmu_set_info_table(&tbl_info, 0, 0, tbl); +#ifdef CFG_NS_VIRTUALIZATION + tbl_info.prtn = prtn; +#endif + + /* + * If this isn't the core that created the initial tables + * mappings, then the level 1 table must be copied, + * as it will hold pointer to the user mapping table + * that changes per core. + */ + if (core != get_core_pos()) { + if (!core_mmu_entry_copy(&tbl_info, 0)) + panic(); + } + + if (!core_mmu_entry_to_finer_grained(&tbl_info, 0, true)) + panic(); + + /* + * Now base level table should be ready with a table descriptor + */ + assert((tbl[0] & DESC_ENTRY_TYPE_MASK) == TABLE_DESC); + + tbl = core_mmu_xlat_table_entry_pa2va(prtn, 0, tbl[0]); + assert(tbl); + + idx = ((uintptr_t)&tbl[user_va_idx] - (uintptr_t)prtn->xlat_tables) / + sizeof(xlat_tbl_t); + assert(idx < prtn->xlat_tables_used); + + prtn->user_l1_table_idx[base_idx][core] = idx; +#endif +} + +static void core_init_mmu_prtn_ta(struct mmu_partition *prtn) +{ + unsigned int base_idx = 0; + unsigned int core = 0; + + assert(user_va_idx != -1); + + for (base_idx = 0; base_idx < NUM_BASE_TABLES; base_idx++) + for (core = 0; core < CFG_TEE_CORE_NB_CORE; core++) + core_init_mmu_prtn_ta_core(prtn, base_idx, core); +} + +void core_init_mmu_prtn(struct mmu_partition *prtn, struct tee_mmap_region *mm) +{ + core_init_mmu_prtn_tee(prtn, mm); + core_init_mmu_prtn_ta(prtn); +} + +void core_init_mmu(struct tee_mmap_region *mm) +{ + uint64_t max_va = 0; + size_t n; + + COMPILE_TIME_ASSERT(CORE_MMU_BASE_TABLE_SHIFT == + XLAT_ADDR_SHIFT(CORE_MMU_BASE_TABLE_LEVEL)); +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + COMPILE_TIME_ASSERT(CORE_MMU_BASE_TABLE_OFFSET == + sizeof(base_xlation_table) / 2); +#endif + COMPILE_TIME_ASSERT(XLAT_TABLES_SIZE == sizeof(xlat_tables)); + + /* Initialize default pagetables */ + core_init_mmu_prtn_tee(&default_partition, mm); + + for (n = 0; !core_mmap_is_end_of_table(mm + n); n++) { + vaddr_t va_end = mm[n].va + mm[n].size - 1; + + if (va_end > max_va) + max_va = va_end; + } + + set_user_va_idx(&default_partition); + + core_init_mmu_prtn_ta(&default_partition); + + COMPILE_TIME_ASSERT(CFG_LPAE_ADDR_SPACE_BITS > L1_XLAT_ADDRESS_SHIFT); + assert(max_va < BIT64(CFG_LPAE_ADDR_SPACE_BITS)); +} + +#ifdef CFG_WITH_PAGER +/* Prefer to consume only 1 base xlat table for the whole mapping */ +bool core_mmu_prefer_tee_ram_at_top(paddr_t paddr) +{ + size_t base_level_size = BASE_XLAT_BLOCK_SIZE; + paddr_t base_level_mask = base_level_size - 1; + + return (paddr & base_level_mask) > (base_level_size / 2); +} +#endif + +#ifdef ARM32 +void core_init_mmu_regs(struct core_mmu_config *cfg) +{ + uint32_t ttbcr = 0; + uint32_t mair = 0; + + cfg->ttbr0_base = virt_to_phys(base_xlation_table[0][0]); + cfg->ttbr0_core_offset = sizeof(base_xlation_table[0][0]); + + mair = MAIR_ATTR_SET(ATTR_DEVICE_nGnRE, ATTR_DEVICE_nGnRE_INDEX); + mair |= MAIR_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, ATTR_IWBWA_OWBWA_NTR_INDEX); + mair |= MAIR_ATTR_SET(ATTR_DEVICE_nGnRnE, ATTR_DEVICE_nGnRnE_INDEX); + /* + * Tagged memory isn't supported in 32-bit mode, map tagged memory + * as normal memory instead. + */ + mair |= MAIR_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, + ATTR_TAGGED_NORMAL_MEM_INDEX); + cfg->mair0 = mair; + + ttbcr = TTBCR_EAE; + ttbcr |= TTBCR_XRGNX_WBWA << TTBCR_IRGN0_SHIFT; + ttbcr |= TTBCR_XRGNX_WBWA << TTBCR_ORGN0_SHIFT; + ttbcr |= TTBCR_SHX_ISH << TTBCR_SH0_SHIFT; + ttbcr |= TTBCR_EPD1; /* Disable the use of TTBR1 */ + + /* TTBCR.A1 = 0 => ASID is stored in TTBR0 */ + cfg->ttbcr = ttbcr; +} +#endif /*ARM32*/ + +#ifdef ARM64 +static unsigned int get_physical_addr_size_bits(void) +{ + /* + * Intermediate Physical Address Size. + * 0b000 32 bits, 4GB. + * 0b001 36 bits, 64GB. + * 0b010 40 bits, 1TB. + * 0b011 42 bits, 4TB. + * 0b100 44 bits, 16TB. + * 0b101 48 bits, 256TB. + * 0b110 52 bits, 4PB (not supported) + */ + + COMPILE_TIME_ASSERT(CFG_CORE_ARM64_PA_BITS >= 32); + + if (CFG_CORE_ARM64_PA_BITS <= 32) + return TCR_PS_BITS_4GB; + + if (CFG_CORE_ARM64_PA_BITS <= 36) + return TCR_PS_BITS_64GB; + + if (CFG_CORE_ARM64_PA_BITS <= 40) + return TCR_PS_BITS_1TB; + + if (CFG_CORE_ARM64_PA_BITS <= 42) + return TCR_PS_BITS_4TB; + + if (CFG_CORE_ARM64_PA_BITS <= 44) + return TCR_PS_BITS_16TB; + + /* Physical address can't exceed 48 bits */ + COMPILE_TIME_ASSERT(CFG_CORE_ARM64_PA_BITS <= 48); + /* CFG_CORE_ARM64_PA_BITS <= 48 */ + return TCR_PS_BITS_256TB; +} + +void core_init_mmu_regs(struct core_mmu_config *cfg) +{ + uint64_t ips = get_physical_addr_size_bits(); + uint64_t mair = 0; + uint64_t tcr = 0; + + cfg->ttbr0_el1_base = virt_to_phys(base_xlation_table[0][0]); + cfg->ttbr0_core_offset = sizeof(base_xlation_table[0][0]); + + mair = MAIR_ATTR_SET(ATTR_DEVICE_nGnRE, ATTR_DEVICE_nGnRE_INDEX); + mair |= MAIR_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, ATTR_IWBWA_OWBWA_NTR_INDEX); + mair |= MAIR_ATTR_SET(ATTR_DEVICE_nGnRnE, ATTR_DEVICE_nGnRnE_INDEX); + /* + * If MEMTAG isn't enabled, map tagged memory as normal memory + * instead. + */ + if (memtag_is_enabled()) + mair |= MAIR_ATTR_SET(ATTR_TAGGED_NORMAL_MEM, + ATTR_TAGGED_NORMAL_MEM_INDEX); + else + mair |= MAIR_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, + ATTR_TAGGED_NORMAL_MEM_INDEX); + cfg->mair_el1 = mair; + + tcr = TCR_RES1; + tcr |= TCR_XRGNX_WBWA << TCR_IRGN0_SHIFT; + tcr |= TCR_XRGNX_WBWA << TCR_ORGN0_SHIFT; + tcr |= TCR_SHX_ISH << TCR_SH0_SHIFT; + tcr |= ips << TCR_EL1_IPS_SHIFT; + tcr |= 64 - CFG_LPAE_ADDR_SPACE_BITS; + + /* Disable the use of TTBR1 */ + tcr |= TCR_EPD1; + + /* + * TCR.A1 = 0 => ASID is stored in TTBR0 + * TCR.AS = 0 => Same ASID size as in Aarch32/ARMv7 + */ + cfg->tcr_el1 = tcr; +} +#endif /*ARM64*/ + +void core_mmu_set_info_table(struct core_mmu_table_info *tbl_info, + unsigned level, vaddr_t va_base, void *table) +{ + tbl_info->level = level; + tbl_info->next_level = level + 1; + tbl_info->table = table; + tbl_info->va_base = va_base; + tbl_info->shift = XLAT_ADDR_SHIFT(level); + +#if (CORE_MMU_BASE_TABLE_LEVEL > 0) + assert(level >= CORE_MMU_BASE_TABLE_LEVEL); +#endif + assert(level <= XLAT_TABLE_LEVEL_MAX); + + if (level == CORE_MMU_BASE_TABLE_LEVEL) + tbl_info->num_entries = NUM_BASE_LEVEL_ENTRIES; + else + tbl_info->num_entries = XLAT_TABLE_ENTRIES; +} + +void core_mmu_get_user_pgdir(struct core_mmu_table_info *pgd_info) +{ + vaddr_t va_range_base; + void *tbl = get_prtn()->l2_ta_tables[thread_get_id()]; + + core_mmu_get_user_va_range(&va_range_base, NULL); + core_mmu_set_info_table(pgd_info, 2, va_range_base, tbl); +} + +void core_mmu_create_user_map(struct user_mode_ctx *uctx, + struct core_mmu_user_map *map) +{ + struct core_mmu_table_info dir_info; + + COMPILE_TIME_ASSERT(sizeof(uint64_t) * XLAT_TABLE_ENTRIES == PGT_SIZE); + + core_mmu_get_user_pgdir(&dir_info); + memset(dir_info.table, 0, PGT_SIZE); + core_mmu_populate_user_map(&dir_info, uctx); + map->user_map = virt_to_phys(dir_info.table) | TABLE_DESC; + map->asid = uctx->vm_info.asid; +} + +bool core_mmu_find_table(struct mmu_partition *prtn, vaddr_t va, + unsigned max_level, + struct core_mmu_table_info *tbl_info) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + unsigned int num_entries = NUM_BASE_LEVEL_ENTRIES; + unsigned int level = CORE_MMU_BASE_TABLE_LEVEL; + vaddr_t va_base = 0; + bool ret = false; + uint64_t *tbl; + + if (!prtn) + prtn = get_prtn(); + tbl = prtn->base_tables[0][get_core_pos()]; + + while (true) { + unsigned int level_size_shift = XLAT_ADDR_SHIFT(level); + unsigned int n = (va - va_base) >> level_size_shift; + + if (n >= num_entries) + goto out; + + if (level == max_level || level == XLAT_TABLE_LEVEL_MAX || + (tbl[n] & TABLE_DESC) != TABLE_DESC) { + /* + * We've either reached max_level, a block + * mapping entry or an "invalid" mapping entry. + */ + + /* + * Base level is the CPU specific translation table. + * It doesn't make sense to return anything based + * on that unless foreign interrupts already are + * masked. + */ + if (level == CORE_MMU_BASE_TABLE_LEVEL && + !(exceptions & THREAD_EXCP_FOREIGN_INTR)) + goto out; + + tbl_info->table = tbl; + tbl_info->va_base = va_base; + tbl_info->level = level; + tbl_info->next_level = level + 1; + tbl_info->shift = level_size_shift; + tbl_info->num_entries = num_entries; +#ifdef CFG_NS_VIRTUALIZATION + tbl_info->prtn = prtn; +#endif + ret = true; + goto out; + } + + tbl = core_mmu_xlat_table_entry_pa2va(prtn, level, tbl[n]); + + if (!tbl) + goto out; + + va_base += (vaddr_t)n << level_size_shift; + level++; + num_entries = XLAT_TABLE_ENTRIES; + } +out: + thread_unmask_exceptions(exceptions); + return ret; +} + +bool core_mmu_entry_to_finer_grained(struct core_mmu_table_info *tbl_info, + unsigned int idx, bool secure __unused) +{ + uint64_t *new_table; + uint64_t *entry; + int i; + paddr_t pa; + uint64_t attr; + paddr_t block_size_on_next_lvl = XLAT_BLOCK_SIZE(tbl_info->level + 1); + struct mmu_partition *prtn; + +#ifdef CFG_NS_VIRTUALIZATION + prtn = tbl_info->prtn; +#else + prtn = &default_partition; +#endif + assert(prtn); + + if (tbl_info->level >= XLAT_TABLE_LEVEL_MAX || + idx >= tbl_info->num_entries) + return false; + + entry = (uint64_t *)tbl_info->table + idx; + + if ((*entry & DESC_ENTRY_TYPE_MASK) == TABLE_DESC) + return true; + + new_table = core_mmu_xlat_table_alloc(prtn); + if (!new_table) + return false; + + if (*entry) { + pa = *entry & OUTPUT_ADDRESS_MASK; + attr = *entry & ~(OUTPUT_ADDRESS_MASK | DESC_ENTRY_TYPE_MASK); + for (i = 0; i < XLAT_TABLE_ENTRIES; i++) { + new_table[i] = pa | attr | BLOCK_DESC; + pa += block_size_on_next_lvl; + } + } else { + memset(new_table, 0, XLAT_TABLE_ENTRIES * XLAT_ENTRY_SIZE); + } + + *entry = virt_to_phys(new_table) | TABLE_DESC; + + return true; +} + +void core_mmu_set_entry_primitive(void *table, size_t level, size_t idx, + paddr_t pa, uint32_t attr) +{ + uint64_t *tbl = table; + uint64_t desc = mattr_to_desc(level, attr); + + tbl[idx] = desc | pa; +} + +void core_mmu_get_entry_primitive(const void *table, size_t level, + size_t idx, paddr_t *pa, uint32_t *attr) +{ + const uint64_t *tbl = table; + + if (pa) + *pa = tbl[idx] & GENMASK_64(47, 12); + + if (attr) + *attr = desc_to_mattr(level, tbl[idx]); +} + +bool core_mmu_user_va_range_is_defined(void) +{ + return user_va_idx != -1; +} + +void core_mmu_get_user_va_range(vaddr_t *base, size_t *size) +{ + assert(user_va_idx != -1); + + if (base) + *base = (vaddr_t)user_va_idx << L1_XLAT_ADDRESS_SHIFT; + if (size) + *size = BIT64(L1_XLAT_ADDRESS_SHIFT); +} + +static uint64_t *core_mmu_get_user_mapping_entry(struct mmu_partition *prtn, + unsigned int base_idx) +{ +#if (CORE_MMU_BASE_TABLE_LEVEL == 0) + l1_idx_t idx = 0; + uint64_t *tbl = NULL; +#endif + + assert(user_va_idx != -1); + +#if (CORE_MMU_BASE_TABLE_LEVEL == 0) + idx = prtn->user_l1_table_idx[base_idx][get_core_pos()]; + tbl = prtn->xlat_tables[idx]; + + return &tbl[user_va_idx]; +#else + return &prtn->base_tables[base_idx][get_core_pos()][user_va_idx]; +#endif +} + +bool core_mmu_user_mapping_is_active(void) +{ + bool ret = false; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + uint64_t *entry = NULL; + + entry = core_mmu_get_user_mapping_entry(get_prtn(), 0); + ret = (*entry != 0); + + thread_unmask_exceptions(exceptions); + + return ret; +} + +#ifdef ARM32 +void core_mmu_get_user_map(struct core_mmu_user_map *map) +{ + struct mmu_partition *prtn = get_prtn(); + uint64_t *entry = NULL; + + entry = core_mmu_get_user_mapping_entry(prtn, 0); + + map->user_map = *entry; + if (map->user_map) { + map->asid = (read_ttbr0_64bit() >> TTBR_ASID_SHIFT) & + TTBR_ASID_MASK; + } else { + map->asid = 0; + } +} + +void core_mmu_set_user_map(struct core_mmu_user_map *map) +{ + uint64_t ttbr = 0; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + struct mmu_partition *prtn = get_prtn(); + uint64_t *entries[NUM_BASE_TABLES] = { }; + unsigned int i = 0; + + ttbr = read_ttbr0_64bit(); + /* Clear ASID */ + ttbr &= ~((uint64_t)TTBR_ASID_MASK << TTBR_ASID_SHIFT); + write_ttbr0_64bit(ttbr); + isb(); + + for (i = 0; i < NUM_BASE_TABLES; i++) + entries[i] = core_mmu_get_user_mapping_entry(prtn, i); + + /* Set the new map */ + if (map && map->user_map) { + for (i = 0; i < NUM_BASE_TABLES; i++) + *entries[i] = map->user_map; + + dsb(); /* Make sure the write above is visible */ + ttbr |= ((uint64_t)map->asid << TTBR_ASID_SHIFT); + write_ttbr0_64bit(ttbr); + isb(); + } else { + for (i = 0; i < NUM_BASE_TABLES; i++) + *entries[i] = INVALID_DESC; + + dsb(); /* Make sure the write above is visible */ + } + + tlbi_all(); + icache_inv_all(); + + thread_unmask_exceptions(exceptions); +} + +enum core_mmu_fault core_mmu_get_fault_type(uint32_t fault_descr) +{ + assert(fault_descr & FSR_LPAE); + + switch (fault_descr & FSR_STATUS_MASK) { + case 0x21: /* b100001 Alignment fault */ + return CORE_MMU_FAULT_ALIGNMENT; + case 0x11: /* b010001 Asynchronous extern abort (DFSR only) */ + return CORE_MMU_FAULT_ASYNC_EXTERNAL; + case 0x12: /* b100010 Debug event */ + return CORE_MMU_FAULT_DEBUG_EVENT; + default: + break; + } + + switch ((fault_descr & FSR_STATUS_MASK) >> 2) { + case 0x1: /* b0001LL Translation fault */ + return CORE_MMU_FAULT_TRANSLATION; + case 0x2: /* b0010LL Access flag fault */ + case 0x3: /* b0011LL Permission fault */ + if (fault_descr & FSR_WNR) + return CORE_MMU_FAULT_WRITE_PERMISSION; + else + return CORE_MMU_FAULT_READ_PERMISSION; + default: + return CORE_MMU_FAULT_OTHER; + } +} +#endif /*ARM32*/ + +#ifdef ARM64 +void core_mmu_get_user_map(struct core_mmu_user_map *map) +{ + struct mmu_partition *prtn = get_prtn(); + uint64_t *entry = NULL; + + entry = core_mmu_get_user_mapping_entry(prtn, 0); + + map->user_map = *entry; + if (map->user_map) { + map->asid = (read_ttbr0_el1() >> TTBR_ASID_SHIFT) & + TTBR_ASID_MASK; + } else { + map->asid = 0; + } +} + +void core_mmu_set_user_map(struct core_mmu_user_map *map) +{ + uint64_t ttbr = 0; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + struct mmu_partition *prtn = get_prtn(); + uint64_t *entries[NUM_BASE_TABLES] = { }; + unsigned int i = 0; + + ttbr = read_ttbr0_el1(); + /* Clear ASID */ + ttbr &= ~((uint64_t)TTBR_ASID_MASK << TTBR_ASID_SHIFT); + write_ttbr0_el1(ttbr); + isb(); + + for (i = 0; i < NUM_BASE_TABLES; i++) + entries[i] = core_mmu_get_user_mapping_entry(prtn, i); + + /* Set the new map */ + if (map && map->user_map) { + for (i = 0; i < NUM_BASE_TABLES; i++) + *entries[i] = map->user_map; + + dsb(); /* Make sure the write above is visible */ + ttbr |= ((uint64_t)map->asid << TTBR_ASID_SHIFT); + write_ttbr0_el1(ttbr); + isb(); + } else { + for (i = 0; i < NUM_BASE_TABLES; i++) + *entries[i] = INVALID_DESC; + + dsb(); /* Make sure the write above is visible */ + } + + tlbi_all(); + icache_inv_all(); + + thread_unmask_exceptions(exceptions); +} + +enum core_mmu_fault core_mmu_get_fault_type(uint32_t fault_descr) +{ + switch ((fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { + case ESR_EC_SP_ALIGN: + case ESR_EC_PC_ALIGN: + return CORE_MMU_FAULT_ALIGNMENT; + case ESR_EC_IABT_EL0: + case ESR_EC_DABT_EL0: + case ESR_EC_IABT_EL1: + case ESR_EC_DABT_EL1: + switch (fault_descr & ESR_FSC_MASK) { + case ESR_FSC_SIZE_L0: + case ESR_FSC_SIZE_L1: + case ESR_FSC_SIZE_L2: + case ESR_FSC_SIZE_L3: + case ESR_FSC_TRANS_L0: + case ESR_FSC_TRANS_L1: + case ESR_FSC_TRANS_L2: + case ESR_FSC_TRANS_L3: + return CORE_MMU_FAULT_TRANSLATION; + case ESR_FSC_ACCF_L1: + case ESR_FSC_ACCF_L2: + case ESR_FSC_ACCF_L3: + case ESR_FSC_PERMF_L1: + case ESR_FSC_PERMF_L2: + case ESR_FSC_PERMF_L3: + if (fault_descr & ESR_ABT_WNR) + return CORE_MMU_FAULT_WRITE_PERMISSION; + else + return CORE_MMU_FAULT_READ_PERMISSION; + case ESR_FSC_ALIGN: + return CORE_MMU_FAULT_ALIGNMENT; + case ESR_FSC_TAG_CHECK: + return CORE_MMU_FAULT_TAG_CHECK; + default: + return CORE_MMU_FAULT_OTHER; + } + default: + return CORE_MMU_FAULT_OTHER; + } +} +#endif /*ARM64*/ diff --git a/optee_os/core/arch/arm/mm/core_mmu_v7.c b/optee_os/core/arch/arm/mm/core_mmu_v7.c new file mode 100644 index 0000000..61e703d --- /dev/null +++ b/optee_os/core/arch/arm/mm/core_mmu_v7.c @@ -0,0 +1,828 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_WITH_LPAE +#error This file is not to be used with LPAE +#endif + +#ifdef CFG_NS_VIRTUALIZATION +#error Currently V7 MMU code does not support virtualization +#endif + +#ifndef DEBUG_XLAT_TABLE +#define DEBUG_XLAT_TABLE 0 +#endif + +#if DEBUG_XLAT_TABLE +#define debug_print(...) DMSG_RAW(__VA_ARGS__) +#else +#define debug_print(...) ((void)0) +#endif + +/* + * MMU related values + */ + +/* Sharable */ +#define TEE_MMU_TTB_S (1 << 1) + +/* Not Outer Sharable */ +#define TEE_MMU_TTB_NOS (1 << 5) + +/* Normal memory, Inner Non-cacheable */ +#define TEE_MMU_TTB_IRGN_NC 0 + +/* Normal memory, Inner Write-Back Write-Allocate Cacheable */ +#define TEE_MMU_TTB_IRGN_WBWA (1 << 6) + +/* Normal memory, Inner Write-Through Cacheable */ +#define TEE_MMU_TTB_IRGN_WT 1 + +/* Normal memory, Inner Write-Back no Write-Allocate Cacheable */ +#define TEE_MMU_TTB_IRGN_WB (1 | (1 << 6)) + +/* Normal memory, Outer Write-Back Write-Allocate Cacheable */ +#define TEE_MMU_TTB_RNG_WBWA (1 << 3) + +/* Normal memory, Outer Write-Back no Write-Allocate Cacheable */ +#define TEE_MMU_TTB_RNG_WB (3 << 3) + +#ifndef CFG_NO_SMP +#define TEE_MMU_DEFAULT_ATTRS \ + (TEE_MMU_TTB_S | TEE_MMU_TTB_NOS | \ + TEE_MMU_TTB_IRGN_WBWA | TEE_MMU_TTB_RNG_WBWA) +#else +#define TEE_MMU_DEFAULT_ATTRS (TEE_MMU_TTB_IRGN_WB | TEE_MMU_TTB_RNG_WB) +#endif + + +#define INVALID_DESC 0x0 + +#define SECTION_SHIFT 20 +#define SECTION_MASK 0x000fffff +#define SECTION_SIZE 0x00100000 + +/* armv7 memory mapping attributes: section mapping */ +#define SECTION_SECURE (0 << 19) +#define SECTION_NOTSECURE (1 << 19) +#define SECTION_SHARED (1 << 16) +#define SECTION_NOTGLOBAL (1 << 17) +#define SECTION_ACCESS_FLAG (1 << 10) +#define SECTION_UNPRIV (1 << 11) +#define SECTION_RO (1 << 15) +#define SECTION_TEXCB(texcb) ((((texcb) >> 2) << 12) | \ + ((((texcb) >> 1) & 0x1) << 3) | \ + (((texcb) & 0x1) << 2)) +#define SECTION_DEVICE SECTION_TEXCB(ATTR_DEVICE_INDEX) +#define SECTION_NORMAL SECTION_TEXCB(ATTR_DEVICE_INDEX) +#define SECTION_NORMAL_CACHED SECTION_TEXCB(ATTR_NORMAL_CACHED_INDEX) +#define SECTION_STRONG_O SECTION_TEXCB(ATTR_STRONG_O_INDEX) +#define SECTION_TAGGED_CACHED SECTION_TEXCB(ATTR_TAGGED_CACHED_INDEX) + +#define SECTION_XN (1 << 4) +#define SECTION_PXN (1 << 0) +#define SECTION_SECTION (2 << 0) + +#define SECTION_PT_NOTSECURE (1 << 3) +#define SECTION_PT_PT (1 << 0) + +#define SECTION_PT_ATTR_MASK ~((1 << 10) - 1) + +#define SMALL_PAGE_SMALL_PAGE (1 << 1) +#define SMALL_PAGE_SHARED (1 << 10) +#define SMALL_PAGE_NOTGLOBAL (1 << 11) +#define SMALL_PAGE_TEXCB(texcb) ((((texcb) >> 2) << 6) | \ + ((((texcb) >> 1) & 0x1) << 3) | \ + (((texcb) & 0x1) << 2)) +#define SMALL_PAGE_DEVICE SMALL_PAGE_TEXCB(ATTR_DEVICE_INDEX) +#define SMALL_PAGE_NORMAL SMALL_PAGE_TEXCB(ATTR_DEVICE_INDEX) +#define SMALL_PAGE_NORMAL_CACHED \ + SMALL_PAGE_TEXCB(ATTR_NORMAL_CACHED_INDEX) +#define SMALL_PAGE_STRONG_O SMALL_PAGE_TEXCB(ATTR_STRONG_O_INDEX) +#define SMALL_PAGE_TAGGED_CACHED \ + SMALL_PAGE_TEXCB(ATTR_TAGGED_CACHED_INDEX) +#define SMALL_PAGE_ACCESS_FLAG (1 << 4) +#define SMALL_PAGE_UNPRIV (1 << 5) +#define SMALL_PAGE_RO (1 << 9) +#define SMALL_PAGE_XN (1 << 0) + + +/* The TEX, C and B bits concatenated */ +#define ATTR_DEVICE_INDEX 0x0 +#define ATTR_NORMAL_CACHED_INDEX 0x1 +#define ATTR_STRONG_O_INDEX 0x2 +/* Compat with TEE_MATTR_MEM_TYPE_TAGGED */ +#define ATTR_TAGGED_CACHED_INDEX 0x3 + +#define PRRR_IDX(idx, tr, nos) (((tr) << (2 * (idx))) | \ + ((uint32_t)(nos) << ((idx) + 24))) +#define NMRR_IDX(idx, ir, or) (((ir) << (2 * (idx))) | \ + ((uint32_t)(or) << (2 * (idx) + 16))) +#define PRRR_DS0 (1 << 16) +#define PRRR_DS1 (1 << 17) +#define PRRR_NS0 (1 << 18) +#define PRRR_NS1 (1 << 19) + +#define ATTR_DEVICE_PRRR PRRR_IDX(ATTR_DEVICE_INDEX, 1, 0) +#define ATTR_DEVICE_NMRR NMRR_IDX(ATTR_DEVICE_INDEX, 0, 0) + +#define ATTR_STRONGLY_O_PRRR PRRR_IDX(ATTR_STRONG_O_INDEX, 0, 0) +#define ATTR_STRONGLY_O_NMRR NMRR_IDX(ATTR_STRONG_O_INDEX, 0, 0) + +#ifndef CFG_NO_SMP +#define ATTR_NORMAL_CACHED_PRRR PRRR_IDX(ATTR_NORMAL_CACHED_INDEX, 2, 1) +#define ATTR_NORMAL_CACHED_NMRR NMRR_IDX(ATTR_NORMAL_CACHED_INDEX, 1, 1) +#define ATTR_TAGGED_CACHED_PRRR PRRR_IDX(ATTR_TAGGED_CACHED_INDEX, 2, 1) +#define ATTR_TAGGED_CACHED_NMRR NMRR_IDX(ATTR_TAGGED_CACHED_INDEX, 1, 1) +#else +#define ATTR_NORMAL_CACHED_PRRR PRRR_IDX(ATTR_NORMAL_CACHED_INDEX, 2, 0) +#define ATTR_NORMAL_CACHED_NMRR NMRR_IDX(ATTR_NORMAL_CACHED_INDEX, 3, 3) +#define ATTR_TAGGED_CACHED_PRRR PRRR_IDX(ATTR_TAGGED_CACHED_INDEX, 2, 0) +#define ATTR_TAGGED_CACHED_NMRR NMRR_IDX(ATTR_TAGGED_CACHED_INDEX, 3, 3) +#endif + +#define NUM_L1_ENTRIES 4096 +#define NUM_L2_ENTRIES 256 + +#define L1_TBL_SIZE (NUM_L1_ENTRIES * 4) +#define L2_TBL_SIZE (NUM_L2_ENTRIES * 4) +#define L1_ALIGNMENT L1_TBL_SIZE +#define L2_ALIGNMENT L2_TBL_SIZE + +/* Defined to the smallest possible secondary L1 MMU table */ +#define TTBCR_N_VALUE 7 + +/* Number of sections in ttbr0 when user mapping activated */ +#define NUM_UL1_ENTRIES (1 << (12 - TTBCR_N_VALUE)) +#define UL1_ALIGNMENT (NUM_UL1_ENTRIES * 4) +/* TTB attributes */ + +/* TTB0 of TTBR0 (depends on TTBCR_N_VALUE) */ +#define TTB_UL1_MASK (~(UL1_ALIGNMENT - 1)) +/* TTB1 of TTBR1 */ +#define TTB_L1_MASK (~(L1_ALIGNMENT - 1)) + +#ifndef MAX_XLAT_TABLES +#ifdef CFG_CORE_ASLR +# define XLAT_TABLE_ASLR_EXTRA 2 +#else +# define XLAT_TABLE_ASLR_EXTRA 0 +#endif +#define MAX_XLAT_TABLES (4 + XLAT_TABLE_ASLR_EXTRA) +#endif /*!MAX_XLAT_TABLES*/ + +enum desc_type { + DESC_TYPE_PAGE_TABLE, + DESC_TYPE_SECTION, + DESC_TYPE_SUPER_SECTION, + DESC_TYPE_LARGE_PAGE, + DESC_TYPE_SMALL_PAGE, + DESC_TYPE_INVALID, +}; + +typedef uint32_t l1_xlat_tbl_t[NUM_L1_ENTRIES]; +typedef uint32_t l2_xlat_tbl_t[NUM_L2_ENTRIES]; +typedef uint32_t ul1_xlat_tbl_t[NUM_UL1_ENTRIES]; + +static l1_xlat_tbl_t main_mmu_l1_ttb + __aligned(L1_ALIGNMENT) __section(".nozi.mmu.l1"); + +/* L2 MMU tables */ +static l2_xlat_tbl_t main_mmu_l2_ttb[MAX_XLAT_TABLES] + __aligned(L2_ALIGNMENT) __section(".nozi.mmu.l2"); + +/* MMU L1 table for TAs, one for each thread */ +static ul1_xlat_tbl_t main_mmu_ul1_ttb[CFG_NUM_THREADS] + __aligned(UL1_ALIGNMENT) __section(".nozi.mmu.ul1"); + +struct mmu_partition { + l1_xlat_tbl_t *l1_table; + l2_xlat_tbl_t *l2_tables; + ul1_xlat_tbl_t *ul1_tables; + uint32_t tables_used; +}; + +static struct mmu_partition default_partition = { + .l1_table = &main_mmu_l1_ttb, + .l2_tables = main_mmu_l2_ttb, + .ul1_tables = main_mmu_ul1_ttb, + .tables_used = 0, +}; + +#ifdef CFG_NS_VIRTUALIZATION +static struct mmu_partition *current_prtn[CFG_TEE_CORE_NB_CORE]; + +void core_mmu_set_default_prtn_tbl(void) +{ + size_t n = 0; + + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) + current_prtn[n] = &default_partition; +} +#endif + +static struct mmu_partition *get_prtn(void) +{ +#ifdef CFG_NS_VIRTUALIZATION + return current_prtn[get_core_pos()]; +#else + return &default_partition; +#endif +} + +static vaddr_t core_mmu_get_main_ttb_va(struct mmu_partition *prtn) +{ + return (vaddr_t)prtn->l1_table; +} + +static paddr_t core_mmu_get_main_ttb_pa(struct mmu_partition *prtn) +{ + paddr_t pa = virt_to_phys((void *)core_mmu_get_main_ttb_va(prtn)); + + if (pa & ~TTB_L1_MASK) + panic("invalid core l1 table"); + return pa; +} + +static vaddr_t core_mmu_get_ul1_ttb_va(struct mmu_partition *prtn) +{ + return (vaddr_t)prtn->ul1_tables[thread_get_id()]; +} + +static paddr_t core_mmu_get_ul1_ttb_pa(struct mmu_partition *prtn) +{ + paddr_t pa = virt_to_phys((void *)core_mmu_get_ul1_ttb_va(prtn)); + + if (pa & ~TTB_UL1_MASK) + panic("invalid user l1 table"); + return pa; +} + +static void *core_mmu_alloc_l2(struct mmu_partition *prtn, size_t size) +{ + uint32_t to_alloc = ROUNDUP(size, NUM_L2_ENTRIES * SMALL_PAGE_SIZE) / + (NUM_L2_ENTRIES * SMALL_PAGE_SIZE); + + DMSG("L2 table used: %d/%d", prtn->tables_used + to_alloc, + MAX_XLAT_TABLES); + if (prtn->tables_used + to_alloc > MAX_XLAT_TABLES) + return NULL; + + memset(prtn->l2_tables[prtn->tables_used], 0, + sizeof(l2_xlat_tbl_t) * to_alloc); + prtn->tables_used += to_alloc; + return prtn->l2_tables[prtn->tables_used - to_alloc]; +} + +static enum desc_type get_desc_type(unsigned level, uint32_t desc) +{ + assert(level >= 1 && level <= 2); + + if (level == 1) { + if ((desc & 0x3) == 0x1) + return DESC_TYPE_PAGE_TABLE; + + if ((desc & 0x2) == 0x2) { + if (desc & (1 << 18)) + return DESC_TYPE_SUPER_SECTION; + return DESC_TYPE_SECTION; + } + } else { + if ((desc & 0x3) == 0x1) + return DESC_TYPE_LARGE_PAGE; + + if ((desc & 0x2) == 0x2) + return DESC_TYPE_SMALL_PAGE; + } + + return DESC_TYPE_INVALID; +} + +static uint32_t texcb_to_mattr(uint32_t texcb) +{ + COMPILE_TIME_ASSERT(ATTR_DEVICE_INDEX == TEE_MATTR_MEM_TYPE_DEV); + COMPILE_TIME_ASSERT(ATTR_NORMAL_CACHED_INDEX == + TEE_MATTR_MEM_TYPE_CACHED); + COMPILE_TIME_ASSERT(ATTR_STRONG_O_INDEX == + TEE_MATTR_MEM_TYPE_STRONGLY_O); + COMPILE_TIME_ASSERT(ATTR_TAGGED_CACHED_INDEX == + TEE_MATTR_MEM_TYPE_TAGGED); + + return texcb << TEE_MATTR_MEM_TYPE_SHIFT; +} + +static uint32_t mattr_to_texcb(uint32_t attr) +{ + /* Keep in sync with core_mmu.c:core_mmu_mattr_is_ok */ + return (attr >> TEE_MATTR_MEM_TYPE_SHIFT) & TEE_MATTR_MEM_TYPE_MASK; +} + + +static uint32_t desc_to_mattr(unsigned level, uint32_t desc) +{ + uint32_t a; + + switch (get_desc_type(level, desc)) { + case DESC_TYPE_PAGE_TABLE: + a = TEE_MATTR_TABLE; + if (!(desc & SECTION_PT_NOTSECURE)) + a |= TEE_MATTR_SECURE; + break; + case DESC_TYPE_SECTION: + a = TEE_MATTR_VALID_BLOCK; + if (desc & SECTION_ACCESS_FLAG) + a |= TEE_MATTR_PRX | TEE_MATTR_URX; + + if (!(desc & SECTION_RO)) + a |= TEE_MATTR_PW | TEE_MATTR_UW; + + if (desc & SECTION_XN) + a &= ~(TEE_MATTR_PX | TEE_MATTR_UX); + + if (desc & SECTION_PXN) + a &= ~TEE_MATTR_PX; + + a |= texcb_to_mattr(((desc >> 12) & 0x7) | ((desc >> 2) & 0x3)); + + if (!(desc & SECTION_NOTGLOBAL)) + a |= TEE_MATTR_GLOBAL; + + if (!(desc & SECTION_NOTSECURE)) + a |= TEE_MATTR_SECURE; + + break; + case DESC_TYPE_SMALL_PAGE: + a = TEE_MATTR_VALID_BLOCK; + if (desc & SMALL_PAGE_ACCESS_FLAG) + a |= TEE_MATTR_PRX | TEE_MATTR_URX; + + if (!(desc & SMALL_PAGE_RO)) + a |= TEE_MATTR_PW | TEE_MATTR_UW; + + if (desc & SMALL_PAGE_XN) + a &= ~(TEE_MATTR_PX | TEE_MATTR_UX); + + a |= texcb_to_mattr(((desc >> 6) & 0x7) | ((desc >> 2) & 0x3)); + + if (!(desc & SMALL_PAGE_NOTGLOBAL)) + a |= TEE_MATTR_GLOBAL; + break; + default: + return 0; + } + + return a; +} + +static uint32_t mattr_to_desc(unsigned level, uint32_t attr) +{ + uint32_t desc; + uint32_t a = attr; + unsigned texcb; + + if (level == 1 && (a & TEE_MATTR_TABLE)) { + desc = SECTION_PT_PT; + if (!(a & TEE_MATTR_SECURE)) + desc |= SECTION_PT_NOTSECURE; + return desc; + } + + if (!(a & TEE_MATTR_VALID_BLOCK)) + return INVALID_DESC; + + if (a & (TEE_MATTR_PX | TEE_MATTR_PW)) + a |= TEE_MATTR_PR; + if (a & (TEE_MATTR_UX | TEE_MATTR_UW)) + a |= TEE_MATTR_UR; + if (a & TEE_MATTR_UR) + a |= TEE_MATTR_PR; + if (a & TEE_MATTR_UW) + a |= TEE_MATTR_PW; + + + texcb = mattr_to_texcb(a); + + if (level == 1) { /* Section */ +#ifndef CFG_NO_SMP + desc = SECTION_SECTION | SECTION_SHARED; +#else + desc = SECTION_SECTION; +#endif + + if (!(a & (TEE_MATTR_PX | TEE_MATTR_UX))) + desc |= SECTION_XN; + +#ifdef CFG_HWSUPP_MEM_PERM_PXN + if (!(a & TEE_MATTR_PX)) + desc |= SECTION_PXN; +#endif + + if (a & TEE_MATTR_UR) + desc |= SECTION_UNPRIV; + + if (!(a & TEE_MATTR_PW)) + desc |= SECTION_RO; + + if (a & (TEE_MATTR_UR | TEE_MATTR_PR)) + desc |= SECTION_ACCESS_FLAG; + + if (!(a & TEE_MATTR_GLOBAL)) + desc |= SECTION_NOTGLOBAL; + + if (!(a & TEE_MATTR_SECURE)) + desc |= SECTION_NOTSECURE; + + desc |= SECTION_TEXCB(texcb); + } else { +#ifndef CFG_NO_SMP + desc = SMALL_PAGE_SMALL_PAGE | SMALL_PAGE_SHARED; +#else + desc = SMALL_PAGE_SMALL_PAGE; +#endif + + if (!(a & (TEE_MATTR_PX | TEE_MATTR_UX))) + desc |= SMALL_PAGE_XN; + + if (a & TEE_MATTR_UR) + desc |= SMALL_PAGE_UNPRIV; + + if (!(a & TEE_MATTR_PW)) + desc |= SMALL_PAGE_RO; + + if (a & (TEE_MATTR_UR | TEE_MATTR_PR)) + desc |= SMALL_PAGE_ACCESS_FLAG; + + if (!(a & TEE_MATTR_GLOBAL)) + desc |= SMALL_PAGE_NOTGLOBAL; + + desc |= SMALL_PAGE_TEXCB(texcb); + } + + return desc; +} + +void core_mmu_set_info_table(struct core_mmu_table_info *tbl_info, + unsigned level, vaddr_t va_base, void *table) +{ + tbl_info->level = level; + tbl_info->next_level = level + 1; + tbl_info->table = table; + tbl_info->va_base = va_base; + assert(level <= 2); + if (level == 1) { + tbl_info->shift = SECTION_SHIFT; + tbl_info->num_entries = NUM_L1_ENTRIES; + } else { + tbl_info->shift = SMALL_PAGE_SHIFT; + tbl_info->num_entries = NUM_L2_ENTRIES; + } +} + +void core_mmu_get_user_pgdir(struct core_mmu_table_info *pgd_info) +{ + void *tbl = (void *)core_mmu_get_ul1_ttb_va(get_prtn()); + + core_mmu_set_info_table(pgd_info, 1, 0, tbl); + pgd_info->num_entries = NUM_UL1_ENTRIES; +} + +void core_mmu_create_user_map(struct user_mode_ctx *uctx, + struct core_mmu_user_map *map) +{ + struct core_mmu_table_info dir_info = { }; + + COMPILE_TIME_ASSERT(L2_TBL_SIZE == PGT_SIZE); + + core_mmu_get_user_pgdir(&dir_info); + memset(dir_info.table, 0, dir_info.num_entries * sizeof(uint32_t)); + core_mmu_populate_user_map(&dir_info, uctx); + map->ttbr0 = core_mmu_get_ul1_ttb_pa(get_prtn()) | + TEE_MMU_DEFAULT_ATTRS; + map->ctxid = uctx->vm_info.asid; +} + +bool core_mmu_find_table(struct mmu_partition *prtn, vaddr_t va, + unsigned max_level, + struct core_mmu_table_info *tbl_info) +{ + uint32_t *tbl; + unsigned n = va >> SECTION_SHIFT; + + if (!prtn) + prtn = get_prtn(); + tbl = (uint32_t *)core_mmu_get_main_ttb_va(prtn); + + if (max_level == 1 || (tbl[n] & 0x3) != 0x1) { + core_mmu_set_info_table(tbl_info, 1, 0, tbl); + } else { + paddr_t ntbl = tbl[n] & ~((1 << 10) - 1); + void *l2tbl = phys_to_virt(ntbl, MEM_AREA_TEE_RAM_RW_DATA, + L2_TBL_SIZE); + + if (!l2tbl) + return false; + + core_mmu_set_info_table(tbl_info, 2, n << SECTION_SHIFT, l2tbl); + } + return true; +} + +void core_mmu_set_entry_primitive(void *table, size_t level, size_t idx, + paddr_t pa, uint32_t attr) +{ + uint32_t *tbl = table; + uint32_t desc = mattr_to_desc(level, attr); + + tbl[idx] = desc | pa; +} + +static paddr_t desc_to_pa(unsigned level, uint32_t desc) +{ + unsigned shift_mask; + + switch (get_desc_type(level, desc)) { + case DESC_TYPE_PAGE_TABLE: + shift_mask = 10; + break; + case DESC_TYPE_SECTION: + shift_mask = 20; + break; + case DESC_TYPE_SUPER_SECTION: + shift_mask = 24; /* We're ignoring bits 32 and above. */ + break; + case DESC_TYPE_LARGE_PAGE: + shift_mask = 16; + break; + case DESC_TYPE_SMALL_PAGE: + shift_mask = 12; + break; + default: + /* Invalid section */ + shift_mask = 4; + } + + return desc & ~((1 << shift_mask) - 1); +} + +bool core_mmu_entry_to_finer_grained(struct core_mmu_table_info *tbl_info, + unsigned int idx, bool secure) +{ + uint32_t *new_table; + uint32_t *entry; + uint32_t new_table_desc; + uint32_t attr; + uint32_t desc; + paddr_t pa; + int i; + + if (tbl_info->level != 1) + return false; + + if (idx >= NUM_L1_ENTRIES) + return false; + + entry = (uint32_t *)tbl_info->table + idx; + attr = desc_to_mattr(1, *entry); + + if (*entry && get_desc_type(1, *entry) == DESC_TYPE_PAGE_TABLE) { + /* + * If there is page table already, + * check the secure attribute fits + */ + return secure == (bool)(attr & TEE_MATTR_SECURE); + } + + /* If there is something mapped, check the secure access flag */ + if (attr && secure != (bool)(attr & TEE_MATTR_SECURE)) + return false; + + new_table = core_mmu_alloc_l2(get_prtn(), + NUM_L2_ENTRIES * SMALL_PAGE_SIZE); + + if (!new_table) + return false; + + new_table_desc = SECTION_PT_PT | virt_to_phys(new_table); + + if (!secure) + new_table_desc |= SECTION_PT_NOTSECURE; + + if (*entry) { + pa = desc_to_pa(1, *entry); + desc = mattr_to_desc(2, attr); + for (i = 0; i < NUM_L2_ENTRIES; i++, pa += SMALL_PAGE_SIZE) + new_table[i] = desc | pa; + } + + /* Update descriptor at current level */ + *entry = new_table_desc; + + return true; +} + +void core_mmu_get_entry_primitive(const void *table, size_t level, + size_t idx, paddr_t *pa, uint32_t *attr) +{ + const uint32_t *tbl = table; + + if (pa) + *pa = desc_to_pa(level, tbl[idx]); + + if (attr) + *attr = desc_to_mattr(level, tbl[idx]); +} + +void core_mmu_get_user_va_range(vaddr_t *base, size_t *size) +{ + if (base) { + /* Leaving the first entry unmapped to make NULL unmapped */ + *base = 1 << SECTION_SHIFT; + } + + if (size) + *size = (NUM_UL1_ENTRIES - 1) << SECTION_SHIFT; +} + +void core_mmu_get_user_map(struct core_mmu_user_map *map) +{ + map->ttbr0 = read_ttbr0(); + map->ctxid = read_contextidr(); +} + +void core_mmu_set_user_map(struct core_mmu_user_map *map) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + /* + * Update the reserved Context ID and TTBR0 + */ + + dsb(); /* ARM erratum 754322 */ + write_contextidr(0); + isb(); + + if (map) { + write_ttbr0(map->ttbr0); + isb(); + write_contextidr(map->ctxid); + isb(); + } else { + write_ttbr0(read_ttbr1()); + isb(); + } + + tlbi_all(); + icache_inv_all(); + + /* Restore interrupts */ + thread_unmask_exceptions(exceptions); +} + +bool core_mmu_user_mapping_is_active(void) +{ + bool ret; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + ret = read_ttbr0() != read_ttbr1(); + thread_unmask_exceptions(exceptions); + + return ret; +} + +static void print_mmap_area(const struct tee_mmap_region *mm __maybe_unused, + const char *str __maybe_unused) +{ + if (!(mm->attr & TEE_MATTR_VALID_BLOCK)) + debug_print("%s [%08" PRIxVA " %08" PRIxVA "] not mapped", + str, mm->va, mm->va + mm->size); + else + debug_print("%s [%08" PRIxVA " %08" PRIxVA "] %s-%s-%s-%s", + str, mm->va, mm->va + mm->size, + mattr_is_cached(mm->attr) ? "MEM" : "DEV", + mm->attr & TEE_MATTR_PW ? "RW" : "RO", + mm->attr & TEE_MATTR_PX ? "X" : "XN", + mm->attr & TEE_MATTR_SECURE ? "S" : "NS"); +} + +void map_memarea_sections(const struct tee_mmap_region *mm, uint32_t *ttb) +{ + uint32_t attr = mattr_to_desc(1, mm->attr); + size_t idx = mm->va >> SECTION_SHIFT; + paddr_t pa = 0; + size_t n; + + if (core_mmap_is_end_of_table(mm)) + return; + + print_mmap_area(mm, "section map"); + + attr = mattr_to_desc(1, mm->attr); + if (attr != INVALID_DESC) + pa = mm->pa; + + n = ROUNDUP(mm->size, SECTION_SIZE) >> SECTION_SHIFT; + while (n--) { + assert(!attr || !ttb[idx] || ttb[idx] == (pa | attr)); + + ttb[idx] = pa | attr; + idx++; + pa += SECTION_SIZE; + } +} + +void core_init_mmu_prtn(struct mmu_partition *prtn, struct tee_mmap_region *mm) +{ + void *ttb1 = (void *)core_mmu_get_main_ttb_va(prtn); + size_t n; + + /* reset L1 table */ + memset(ttb1, 0, L1_TBL_SIZE); + + for (n = 0; !core_mmap_is_end_of_table(mm + n); n++) + if (!core_mmu_is_dynamic_vaspace(mm + n)) + core_mmu_map_region(prtn, mm + n); +} + +void core_init_mmu(struct tee_mmap_region *mm) +{ + /* Initialize default pagetables */ + core_init_mmu_prtn(&default_partition, mm); +} + +void core_init_mmu_regs(struct core_mmu_config *cfg) +{ + cfg->ttbr = core_mmu_get_main_ttb_pa(&default_partition) | + TEE_MMU_DEFAULT_ATTRS; + + cfg->prrr = ATTR_DEVICE_PRRR | ATTR_NORMAL_CACHED_PRRR | + ATTR_STRONGLY_O_PRRR | ATTR_TAGGED_CACHED_PRRR; + cfg->nmrr = ATTR_DEVICE_NMRR | ATTR_NORMAL_CACHED_NMRR | + ATTR_STRONGLY_O_NMRR | ATTR_TAGGED_CACHED_NMRR; + + cfg->prrr |= PRRR_NS1 | PRRR_DS1; + + /* + * Program Domain access control register with two domains: + * domain 0: teecore + * domain 1: TA + */ + cfg->dacr = DACR_DOMAIN(0, DACR_DOMAIN_PERM_CLIENT) | + DACR_DOMAIN(1, DACR_DOMAIN_PERM_CLIENT); + + /* + * Enable lookups using TTBR0 and TTBR1 with the split of addresses + * defined by TEE_MMU_TTBCR_N_VALUE. + */ + cfg->ttbcr = TTBCR_N_VALUE; +} + +enum core_mmu_fault core_mmu_get_fault_type(uint32_t fsr) +{ + assert(!(fsr & FSR_LPAE)); + + switch (fsr & FSR_FS_MASK) { + case 0x1: /* DFSR[10,3:0] 0b00001 Alignment fault (DFSR only) */ + return CORE_MMU_FAULT_ALIGNMENT; + case 0x2: /* DFSR[10,3:0] 0b00010 Debug event */ + return CORE_MMU_FAULT_DEBUG_EVENT; + case 0x4: /* DFSR[10,3:0] b00100 Fault on instr cache maintenance */ + case 0x5: /* DFSR[10,3:0] b00101 Translation fault first level */ + case 0x7: /* DFSR[10,3:0] b00111 Translation fault second level */ + return CORE_MMU_FAULT_TRANSLATION; + case 0xd: /* DFSR[10,3:0] b01101 Permission fault first level */ + case 0xf: /* DFSR[10,3:0] b01111 Permission fault second level */ + if (fsr & FSR_WNR) + return CORE_MMU_FAULT_WRITE_PERMISSION; + else + return CORE_MMU_FAULT_READ_PERMISSION; + case 0x3: /* DFSR[10,3:0] b00011 access bit fault on section */ + case 0x6: /* DFSR[10,3:0] b00110 access bit fault on page */ + return CORE_MMU_FAULT_ACCESS_BIT; + case (1 << 10) | 0x6: + /* DFSR[10,3:0] 0b10110 Async external abort (DFSR only) */ + return CORE_MMU_FAULT_ASYNC_EXTERNAL; + + default: + return CORE_MMU_FAULT_OTHER; + } +} diff --git a/optee_os/core/arch/arm/mm/mobj_dyn_shm.c b/optee_os/core/arch/arm/mm/mobj_dyn_shm.c new file mode 100644 index 0000000..c5bdc30 --- /dev/null +++ b/optee_os/core/arch/arm/mm/mobj_dyn_shm.c @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct mutex shm_mu = MUTEX_INITIALIZER; +static struct condvar shm_cv = CONDVAR_INITIALIZER; +static size_t shm_release_waiters; + +/* + * mobj_reg_shm implementation. Describes shared memory provided by normal world + */ + +struct mobj_reg_shm { + struct mobj mobj; + SLIST_ENTRY(mobj_reg_shm) next; + uint64_t cookie; + tee_mm_entry_t *mm; + paddr_t page_offset; + struct refcount mapcount; + bool guarded; + bool releasing; + bool release_frees; + paddr_t pages[]; +}; + +static size_t mobj_reg_shm_size(size_t nr_pages) +{ + size_t s = 0; + + if (MUL_OVERFLOW(sizeof(paddr_t), nr_pages, &s)) + return 0; + if (ADD_OVERFLOW(sizeof(struct mobj_reg_shm), s, &s)) + return 0; + return s; +} + +static SLIST_HEAD(reg_shm_head, mobj_reg_shm) reg_shm_list = + SLIST_HEAD_INITIALIZER(reg_shm_head); + +static unsigned int reg_shm_slist_lock = SPINLOCK_UNLOCK; +static unsigned int reg_shm_map_lock = SPINLOCK_UNLOCK; + +static struct mobj_reg_shm *to_mobj_reg_shm(struct mobj *mobj); + +static TEE_Result mobj_reg_shm_get_pa(struct mobj *mobj, size_t offst, + size_t granule, paddr_t *pa) +{ + struct mobj_reg_shm *mobj_reg_shm = to_mobj_reg_shm(mobj); + size_t full_offset = 0; + paddr_t p = 0; + + if (!pa) + return TEE_ERROR_GENERIC; + + if (offst >= mobj->size) + return TEE_ERROR_GENERIC; + + full_offset = offst + mobj_reg_shm->page_offset; + switch (granule) { + case 0: + p = mobj_reg_shm->pages[full_offset / SMALL_PAGE_SIZE] + + (full_offset & SMALL_PAGE_MASK); + break; + case SMALL_PAGE_SIZE: + p = mobj_reg_shm->pages[full_offset / SMALL_PAGE_SIZE]; + break; + default: + return TEE_ERROR_GENERIC; + } + *pa = p; + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(mobj_reg_shm_get_pa); + +static size_t mobj_reg_shm_get_phys_offs(struct mobj *mobj, + size_t granule __maybe_unused) +{ + assert(granule >= mobj->phys_granule); + return to_mobj_reg_shm(mobj)->page_offset; +} + +static void *mobj_reg_shm_get_va(struct mobj *mobj, size_t offst, size_t len) +{ + struct mobj_reg_shm *mrs = to_mobj_reg_shm(mobj); + + if (!mrs->mm || !mobj_check_offset_and_len(mobj, offst, len)) + return NULL; + + return (void *)(vaddr_t)(tee_mm_get_smem(mrs->mm) + offst + + mrs->page_offset); +} + +static void reg_shm_unmap_helper(struct mobj_reg_shm *r) +{ + assert(r->mm); + assert(r->mm->pool->shift == SMALL_PAGE_SHIFT); + core_mmu_unmap_pages(tee_mm_get_smem(r->mm), r->mm->size); + tee_mm_free(r->mm); + r->mm = NULL; +} + +static void reg_shm_free_helper(struct mobj_reg_shm *mobj_reg_shm) +{ + uint32_t exceptions = cpu_spin_lock_xsave(®_shm_map_lock); + + if (mobj_reg_shm->mm) + reg_shm_unmap_helper(mobj_reg_shm); + + cpu_spin_unlock_xrestore(®_shm_map_lock, exceptions); + + SLIST_REMOVE(®_shm_list, mobj_reg_shm, mobj_reg_shm, next); + free(mobj_reg_shm); +} + +static void mobj_reg_shm_free(struct mobj *mobj) +{ + struct mobj_reg_shm *r = to_mobj_reg_shm(mobj); + uint32_t exceptions = 0; + + if (r->guarded && !r->releasing) { + /* + * Guarded registersted shared memory can't be released + * by cookie, only by mobj_put(). However, unguarded + * registered shared memory can also be freed by mobj_put() + * unless mobj_reg_shm_release_by_cookie() is waiting for + * the mobj to be released. + */ + exceptions = cpu_spin_lock_xsave(®_shm_slist_lock); + reg_shm_free_helper(r); + cpu_spin_unlock_xrestore(®_shm_slist_lock, exceptions); + } else { + /* + * We've reached the point where an unguarded reg shm can + * be released by cookie. Notify eventual waiters. + */ + exceptions = cpu_spin_lock_xsave(®_shm_slist_lock); + r->release_frees = true; + cpu_spin_unlock_xrestore(®_shm_slist_lock, exceptions); + + mutex_lock(&shm_mu); + if (shm_release_waiters) + condvar_broadcast(&shm_cv); + mutex_unlock(&shm_mu); + } +} + +static TEE_Result mobj_reg_shm_get_mem_type(struct mobj *mobj __unused, + uint32_t *mt) +{ + if (!mt) + return TEE_ERROR_GENERIC; + + *mt = TEE_MATTR_MEM_TYPE_CACHED; + + return TEE_SUCCESS; +} + +static TEE_Result mobj_reg_shm_inc_map(struct mobj *mobj) +{ + TEE_Result res = TEE_SUCCESS; + struct mobj_reg_shm *r = to_mobj_reg_shm(mobj); + uint32_t exceptions = 0; + size_t sz = 0; + + while (true) { + if (refcount_inc(&r->mapcount)) + return TEE_SUCCESS; + + exceptions = cpu_spin_lock_xsave(®_shm_map_lock); + + if (!refcount_val(&r->mapcount)) + break; /* continue to reinitialize */ + /* + * If another thread beat us to initialize mapcount, + * restart to make sure we still increase it. + */ + cpu_spin_unlock_xrestore(®_shm_map_lock, exceptions); + } + + /* + * If we have beaten another thread calling mobj_reg_shm_dec_map() + * to get the lock we need only to reinitialize mapcount to 1. + */ + if (!r->mm) { + sz = ROUNDUP(mobj->size + r->page_offset, SMALL_PAGE_SIZE); + r->mm = tee_mm_alloc(&tee_mm_shm, sz); + if (!r->mm) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = core_mmu_map_pages(tee_mm_get_smem(r->mm), r->pages, + sz / SMALL_PAGE_SIZE, + MEM_AREA_NSEC_SHM); + if (res) { + tee_mm_free(r->mm); + r->mm = NULL; + goto out; + } + } + + refcount_set(&r->mapcount, 1); +out: + cpu_spin_unlock_xrestore(®_shm_map_lock, exceptions); + + return res; +} + +static TEE_Result mobj_reg_shm_dec_map(struct mobj *mobj) +{ + struct mobj_reg_shm *r = to_mobj_reg_shm(mobj); + uint32_t exceptions = 0; + + if (!refcount_dec(&r->mapcount)) + return TEE_SUCCESS; + + exceptions = cpu_spin_lock_xsave(®_shm_map_lock); + + /* + * Check that another thread hasn't been able to: + * - increase the mapcount + * - or, increase the mapcount, decrease it again, and set r->mm to + * NULL + * before we acquired the spinlock + */ + if (!refcount_val(&r->mapcount) && r->mm) + reg_shm_unmap_helper(r); + + cpu_spin_unlock_xrestore(®_shm_map_lock, exceptions); + + return TEE_SUCCESS; +} + +static bool mobj_reg_shm_matches(struct mobj *mobj, enum buf_is_attr attr); + +static uint64_t mobj_reg_shm_get_cookie(struct mobj *mobj) +{ + return to_mobj_reg_shm(mobj)->cookie; +} + +/* + * When CFG_PREALLOC_RPC_CACHE is disabled, this variable is weak just + * to ease breaking its dependency chain when added to the unpaged area. + * When CFG_PREALLOC_RPC_CACHE is enabled, releasing RPC preallocated + * shm mandates these resources to be unpaged. + */ +const struct mobj_ops mobj_reg_shm_ops +__weak __relrodata_unpaged("mobj_reg_shm_ops") = { + .get_pa = mobj_reg_shm_get_pa, + .get_phys_offs = mobj_reg_shm_get_phys_offs, + .get_va = mobj_reg_shm_get_va, + .get_mem_type = mobj_reg_shm_get_mem_type, + .matches = mobj_reg_shm_matches, + .free = mobj_reg_shm_free, + .get_cookie = mobj_reg_shm_get_cookie, + .inc_map = mobj_reg_shm_inc_map, + .dec_map = mobj_reg_shm_dec_map, +}; + +#ifdef CFG_PREALLOC_RPC_CACHE +/* Releasing RPC preallocated shm mandates few resources to be unpaged */ +DECLARE_KEEP_PAGER(mobj_reg_shm_get_cookie); +DECLARE_KEEP_PAGER(mobj_reg_shm_matches); +DECLARE_KEEP_PAGER(mobj_reg_shm_free); +#endif + +static bool mobj_reg_shm_matches(struct mobj *mobj __maybe_unused, + enum buf_is_attr attr) +{ + assert(mobj->ops == &mobj_reg_shm_ops); + + return attr == CORE_MEM_NON_SEC || attr == CORE_MEM_REG_SHM; +} + +static struct mobj_reg_shm *to_mobj_reg_shm(struct mobj *mobj) +{ + assert(mobj->ops == &mobj_reg_shm_ops); + return container_of(mobj, struct mobj_reg_shm, mobj); +} + +struct mobj *mobj_reg_shm_alloc(paddr_t *pages, size_t num_pages, + paddr_t page_offset, uint64_t cookie) +{ + struct mobj_reg_shm *mobj_reg_shm = NULL; + size_t i = 0; + uint32_t exceptions = 0; + size_t s = 0; + + if (!num_pages || page_offset >= SMALL_PAGE_SIZE) + return NULL; + + s = mobj_reg_shm_size(num_pages); + if (!s) + return NULL; + mobj_reg_shm = calloc(1, s); + if (!mobj_reg_shm) + return NULL; + + mobj_reg_shm->mobj.ops = &mobj_reg_shm_ops; + mobj_reg_shm->mobj.size = num_pages * SMALL_PAGE_SIZE - page_offset; + mobj_reg_shm->mobj.phys_granule = SMALL_PAGE_SIZE; + refcount_set(&mobj_reg_shm->mobj.refc, 1); + mobj_reg_shm->cookie = cookie; + mobj_reg_shm->guarded = true; + mobj_reg_shm->page_offset = page_offset; + memcpy(mobj_reg_shm->pages, pages, sizeof(*pages) * num_pages); + + /* Ensure loaded references match format and security constraints */ + for (i = 0; i < num_pages; i++) { + if (mobj_reg_shm->pages[i] & SMALL_PAGE_MASK) + goto err; + + /* Only Non-secure memory can be mapped there */ + if (!core_pbuf_is(CORE_MEM_NON_SEC, mobj_reg_shm->pages[i], + SMALL_PAGE_SIZE)) + goto err; + } + + exceptions = cpu_spin_lock_xsave(®_shm_slist_lock); + SLIST_INSERT_HEAD(®_shm_list, mobj_reg_shm, next); + cpu_spin_unlock_xrestore(®_shm_slist_lock, exceptions); + + return &mobj_reg_shm->mobj; +err: + free(mobj_reg_shm); + return NULL; +} + +void mobj_reg_shm_unguard(struct mobj *mobj) +{ + uint32_t exceptions = cpu_spin_lock_xsave(®_shm_slist_lock); + + to_mobj_reg_shm(mobj)->guarded = false; + cpu_spin_unlock_xrestore(®_shm_slist_lock, exceptions); +} + +static struct mobj_reg_shm *reg_shm_find_unlocked(uint64_t cookie) +{ + struct mobj_reg_shm *mobj_reg_shm = NULL; + + SLIST_FOREACH(mobj_reg_shm, ®_shm_list, next) + if (mobj_reg_shm->cookie == cookie) + return mobj_reg_shm; + + return NULL; +} + +struct mobj *mobj_reg_shm_get_by_cookie(uint64_t cookie) +{ + uint32_t exceptions = cpu_spin_lock_xsave(®_shm_slist_lock); + struct mobj_reg_shm *r = reg_shm_find_unlocked(cookie); + + cpu_spin_unlock_xrestore(®_shm_slist_lock, exceptions); + if (!r) + return NULL; + + return mobj_get(&r->mobj); +} + +TEE_Result mobj_reg_shm_release_by_cookie(uint64_t cookie) +{ + uint32_t exceptions = 0; + struct mobj_reg_shm *r = NULL; + + /* + * Try to find r and see can be released by this function, if so + * call mobj_put(). Otherwise this function is called either by + * wrong cookie and perhaps a second time, regardless return + * TEE_ERROR_BAD_PARAMETERS. + */ + exceptions = cpu_spin_lock_xsave(®_shm_slist_lock); + r = reg_shm_find_unlocked(cookie); + if (!r || r->guarded || r->releasing) + r = NULL; + else + r->releasing = true; + + cpu_spin_unlock_xrestore(®_shm_slist_lock, exceptions); + + if (!r) + return TEE_ERROR_BAD_PARAMETERS; + + mobj_put(&r->mobj); + + /* + * We've established that this function can release the cookie. + * Now we wait until mobj_reg_shm_free() is called by the last + * mobj_put() needed to free this mobj. Note that the call to + * mobj_put() above could very well be that call. + * + * Once mobj_reg_shm_free() is called it will set r->release_frees + * to true and we can free the mobj here. + */ + mutex_lock(&shm_mu); + shm_release_waiters++; + assert(shm_release_waiters); + + while (true) { + exceptions = cpu_spin_lock_xsave(®_shm_slist_lock); + if (r->release_frees) { + reg_shm_free_helper(r); + r = NULL; + } + cpu_spin_unlock_xrestore(®_shm_slist_lock, exceptions); + + if (!r) + break; + condvar_wait(&shm_cv, &shm_mu); + } + + assert(shm_release_waiters); + shm_release_waiters--; + mutex_unlock(&shm_mu); + + return TEE_SUCCESS; +} + +struct mobj *mobj_mapped_shm_alloc(paddr_t *pages, size_t num_pages, + paddr_t page_offset, uint64_t cookie) +{ + struct mobj *mobj = mobj_reg_shm_alloc(pages, num_pages, + page_offset, cookie); + + if (!mobj) + return NULL; + + if (mobj_inc_map(mobj)) { + mobj_put(mobj); + return NULL; + } + + return mobj; +} + +static TEE_Result mobj_mapped_shm_init(void) +{ + vaddr_t pool_start = 0; + vaddr_t pool_end = 0; + + core_mmu_get_mem_by_type(MEM_AREA_SHM_VASPACE, &pool_start, &pool_end); + if (!pool_start || !pool_end) + panic("Can't find region for shmem pool"); + + if (!tee_mm_init(&tee_mm_shm, pool_start, pool_end - pool_start, + SMALL_PAGE_SHIFT, TEE_MM_POOL_NO_FLAGS)) + panic("Could not create shmem pool"); + + DMSG("Shared memory address range: %" PRIxVA ", %" PRIxVA, + pool_start, pool_end); + return TEE_SUCCESS; +} + +preinit(mobj_mapped_shm_init); diff --git a/optee_os/core/arch/arm/mm/mobj_ffa.c b/optee_os/core/arch/arm/mm/mobj_ffa.c new file mode 100644 index 0000000..e57c95a --- /dev/null +++ b/optee_os/core/arch/arm/mm/mobj_ffa.c @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Life cycle of struct mobj_ffa + * + * SPMC at S-EL1 (CFG_CORE_SEL1_SPMC=y) + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * During FFA_MEM_SHARE allocated in mobj_ffa_sel1_spmc_new() and finally + * added to the inactive list at the end of add_mem_share() once + * successfully filled in. + * registered_by_cookie = false + * mobj.refs.val = 0 + * inactive_refs = 0 + * + * During FFA_MEM_RECLAIM reclaimed/freed using + * mobj_ffa_sel1_spmc_reclaim(). This will always succeed if the normal + * world is only calling this when all other threads are done with the + * shared memory object. However, there are some conditions that must be + * met to make sure that this is the case: + * mobj not in the active list, else -> return TEE_ERROR_BUSY + * mobj not in inactive list, else -> return TEE_ERROR_ITEM_NOT_FOUND + * mobj inactive_refs is 0, else -> return TEE_ERROR_BUSY + * + * mobj is activated using mobj_ffa_get_by_cookie() which unless the mobj + * is active already: + * - move the mobj into the active list + * - if not registered_by_cookie -> + * set registered_by_cookie and increase inactive_refs + * - set mobj.refc.val to 1 + * - increase inactive_refs + * + * A previously activated mobj is made ready for reclaim using + * mobj_ffa_unregister_by_cookie() which only succeeds if the mobj is in + * the inactive list and registered_by_cookie is set and then: + * - clears registered_by_cookie + * - decreases inactive_refs + * + * Each successful call to mobj_ffa_get_by_cookie() must be matched by a + * call to mobj_put(). If the mobj.refc.val reaches 0 it's + * - moved to the inactive list + * - inactive_refs is decreased + * + * SPMC at S-EL2/EL3 (CFG_CORE_SEL1_SPMC=n) + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * mobj is activated/allocated using mobj_ffa_get_by_cookie() which if + * already active only is + * - increasing mobj.refc.val and inactive_refs + * if found in inactive list is + * - setting mobj.refc.val to 1 + * - increasing inactive_refs + * - moved into active list + * if not found is created using thread_spmc_populate_mobj_from_rx() and + * then: + * - setting mobj.refc.val to 1 + * - increasing inactive_refs + * - moved into active list + * + * A previously activated mobj is relinquished using + * mobj_ffa_unregister_by_cookie() which only succeeds if the mobj is in + * the inactive list and inactive_refs is 1 + */ +struct mobj_ffa { + struct mobj mobj; + SLIST_ENTRY(mobj_ffa) link; + uint64_t cookie; + tee_mm_entry_t *mm; + struct refcount mapcount; + unsigned int inactive_refs; + uint16_t page_offset; +#ifdef CFG_CORE_SEL1_SPMC + bool registered_by_cookie; +#endif + paddr_t pages[]; +}; + +SLIST_HEAD(mobj_ffa_head, mobj_ffa); + +#ifdef CFG_CORE_SEL1_SPMC +#define NUM_SHMS 64 +static bitstr_t bit_decl(shm_bits, NUM_SHMS); +#endif + +static struct mobj_ffa_head shm_head = SLIST_HEAD_INITIALIZER(shm_head); +static struct mobj_ffa_head shm_inactive_head = + SLIST_HEAD_INITIALIZER(shm_inactive_head); + +static unsigned int shm_lock = SPINLOCK_UNLOCK; + +static const struct mobj_ops mobj_ffa_ops; + +static struct mobj_ffa *to_mobj_ffa(struct mobj *mobj) +{ + assert(mobj->ops == &mobj_ffa_ops); + return container_of(mobj, struct mobj_ffa, mobj); +} + +static size_t shm_size(size_t num_pages) +{ + size_t s = 0; + + if (MUL_OVERFLOW(sizeof(paddr_t), num_pages, &s)) + return 0; + if (ADD_OVERFLOW(sizeof(struct mobj_ffa), s, &s)) + return 0; + return s; +} + +static struct mobj_ffa *ffa_new(unsigned int num_pages) +{ + struct mobj_ffa *mf = NULL; + size_t s = 0; + + if (!num_pages) + return NULL; + + s = shm_size(num_pages); + if (!s) + return NULL; + mf = calloc(1, s); + if (!mf) + return NULL; + + mf->mobj.ops = &mobj_ffa_ops; + mf->mobj.size = num_pages * SMALL_PAGE_SIZE; + mf->mobj.phys_granule = SMALL_PAGE_SIZE; + refcount_set(&mf->mobj.refc, 0); + mf->inactive_refs = 0; + + return mf; +} + +#ifdef CFG_CORE_SEL1_SPMC +struct mobj_ffa *mobj_ffa_sel1_spmc_new(uint64_t cookie, + unsigned int num_pages) +{ + struct mobj_ffa *mf = NULL; + uint32_t exceptions = 0; + int i = 0; + + if (cookie != OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) { + if (!(cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT)) + return NULL; + if (virt_add_cookie_to_current_guest(cookie)) + return NULL; + } + + mf = ffa_new(num_pages); + if (!mf) { + if (cookie != OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) + virt_remove_cookie(cookie); + return NULL; + } + + if (cookie != OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) { + mf->cookie = cookie; + return mf; + } + + exceptions = cpu_spin_lock_xsave(&shm_lock); + bit_ffc(shm_bits, NUM_SHMS, &i); + if (i != -1) { + bit_set(shm_bits, i); + mf->cookie = i; + mf->cookie |= FFA_MEMORY_HANDLE_NON_SECURE_BIT; + /* + * Encode the partition ID into the handle so we know which + * partition to switch to when reclaiming a handle. + */ + mf->cookie |= SHIFT_U64(virt_get_current_guest_id(), + FFA_MEMORY_HANDLE_PRTN_SHIFT); + } + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + + if (i == -1) { + free(mf); + return NULL; + } + + return mf; +} +#endif /*CFG_CORE_SEL1_SPMC*/ + +static size_t get_page_count(struct mobj_ffa *mf) +{ + return ROUNDUP(mf->mobj.size, SMALL_PAGE_SIZE) / SMALL_PAGE_SIZE; +} + +static bool cmp_cookie(struct mobj_ffa *mf, uint64_t cookie) +{ + return mf->cookie == cookie; +} + +static bool cmp_ptr(struct mobj_ffa *mf, uint64_t ptr) +{ + return mf == (void *)(vaddr_t)ptr; +} + +static struct mobj_ffa *pop_from_list(struct mobj_ffa_head *head, + bool (*cmp_func)(struct mobj_ffa *mf, + uint64_t val), + uint64_t val) +{ + struct mobj_ffa *mf = SLIST_FIRST(head); + struct mobj_ffa *p = NULL; + + if (!mf) + return NULL; + + if (cmp_func(mf, val)) { + SLIST_REMOVE_HEAD(head, link); + return mf; + } + + while (true) { + p = SLIST_NEXT(mf, link); + if (!p) + return NULL; + if (cmp_func(p, val)) { + SLIST_REMOVE_AFTER(mf, link); + return p; + } + mf = p; + } +} + +static struct mobj_ffa *find_in_list(struct mobj_ffa_head *head, + bool (*cmp_func)(struct mobj_ffa *mf, + uint64_t val), + uint64_t val) +{ + struct mobj_ffa *mf = NULL; + + SLIST_FOREACH(mf, head, link) + if (cmp_func(mf, val)) + return mf; + + return NULL; +} + +#if defined(CFG_CORE_SEL1_SPMC) +void mobj_ffa_sel1_spmc_delete(struct mobj_ffa *mf) +{ + + if (!IS_ENABLED(CFG_NS_VIRTUALIZATION) || + !(mf->cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT)) { + uint64_t mask = FFA_MEMORY_HANDLE_NON_SECURE_BIT; + uint32_t exceptions = 0; + int64_t i = 0; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + mask |= SHIFT_U64(FFA_MEMORY_HANDLE_PRTN_MASK, + FFA_MEMORY_HANDLE_PRTN_SHIFT); + i = mf->cookie & ~mask; + assert(i >= 0 && i < NUM_SHMS); + + exceptions = cpu_spin_lock_xsave(&shm_lock); + assert(bit_test(shm_bits, i)); + bit_clear(shm_bits, i); + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + } + + assert(!mf->mm); + free(mf); +} +#else /* !defined(CFG_CORE_SEL1_SPMC) */ +struct mobj_ffa *mobj_ffa_spmc_new(uint64_t cookie, unsigned int num_pages) +{ + struct mobj_ffa *mf = NULL; + + assert(cookie != OPTEE_MSG_FMEM_INVALID_GLOBAL_ID); + mf = ffa_new(num_pages); + if (mf) + mf->cookie = cookie; + return mf; +} + +void mobj_ffa_spmc_delete(struct mobj_ffa *mf) +{ + free(mf); +} +#endif /* !defined(CFG_CORE_SEL1_SPMC) */ + +TEE_Result mobj_ffa_add_pages_at(struct mobj_ffa *mf, unsigned int *idx, + paddr_t pa, unsigned int num_pages) +{ + unsigned int n = 0; + size_t tot_page_count = get_page_count(mf); + + if (ADD_OVERFLOW(*idx, num_pages, &n) || n > tot_page_count) + return TEE_ERROR_BAD_PARAMETERS; + + if (!IS_ENABLED(CFG_CORE_SEL2_SPMC) && + !core_pbuf_is(CORE_MEM_NON_SEC, pa, num_pages * SMALL_PAGE_SIZE)) + return TEE_ERROR_BAD_PARAMETERS; + + for (n = 0; n < num_pages; n++) + mf->pages[n + *idx] = pa + n * SMALL_PAGE_SIZE; + + (*idx) += n; + return TEE_SUCCESS; +} + +uint64_t mobj_ffa_get_cookie(struct mobj_ffa *mf) +{ + return mf->cookie; +} + +uint64_t mobj_ffa_push_to_inactive(struct mobj_ffa *mf) +{ + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&shm_lock); + assert(!find_in_list(&shm_inactive_head, cmp_ptr, (vaddr_t)mf)); + assert(!find_in_list(&shm_inactive_head, cmp_cookie, mf->cookie)); + assert(!find_in_list(&shm_head, cmp_cookie, mf->cookie)); + SLIST_INSERT_HEAD(&shm_inactive_head, mf, link); + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + + return mf->cookie; +} + +static void unmap_helper(struct mobj_ffa *mf) +{ + if (mf->mm) { + core_mmu_unmap_pages(tee_mm_get_smem(mf->mm), + get_page_count(mf)); + tee_mm_free(mf->mm); + mf->mm = NULL; + } +} + +#ifdef CFG_CORE_SEL1_SPMC +TEE_Result mobj_ffa_sel1_spmc_reclaim(uint64_t cookie) +{ + TEE_Result res = TEE_SUCCESS; + struct mobj_ffa *mf = NULL; + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&shm_lock); + mf = find_in_list(&shm_head, cmp_cookie, cookie); + /* + * If the mobj is found here it's still active and cannot be + * reclaimed. + */ + if (mf) { + DMSG("cookie %#"PRIx64" busy refc %u", + cookie, refcount_val(&mf->mobj.refc)); + res = TEE_ERROR_BUSY; + goto out; + } + + mf = find_in_list(&shm_inactive_head, cmp_cookie, cookie); + if (!mf) { + res = TEE_ERROR_ITEM_NOT_FOUND; + goto out; + } + /* + * If the mobj has been registered via mobj_ffa_get_by_cookie() + * but not unregistered yet with mobj_ffa_unregister_by_cookie(). + */ + if (mf->inactive_refs) { + DMSG("cookie %#"PRIx64" busy inactive_refs %u", + cookie, mf->inactive_refs); + res = TEE_ERROR_BUSY; + goto out; + } + + if (!pop_from_list(&shm_inactive_head, cmp_ptr, (vaddr_t)mf)) + panic(); + res = TEE_SUCCESS; +out: + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + if (!res) { + mobj_ffa_sel1_spmc_delete(mf); + virt_remove_cookie(cookie); + } + return res; +} +#endif /*CFG_CORE_SEL1_SPMC*/ + +TEE_Result mobj_ffa_unregister_by_cookie(uint64_t cookie) +{ + TEE_Result res = TEE_SUCCESS; + struct mobj_ffa *mf = NULL; + uint32_t exceptions = 0; + + assert(cookie != OPTEE_MSG_FMEM_INVALID_GLOBAL_ID); + exceptions = cpu_spin_lock_xsave(&shm_lock); + mf = find_in_list(&shm_head, cmp_cookie, cookie); + /* + * If the mobj is found here it's still active and cannot be + * unregistered. + */ + if (mf) { + EMSG("cookie %#"PRIx64" busy refc %u:%u", + cookie, refcount_val(&mf->mobj.refc), mf->inactive_refs); + res = TEE_ERROR_BUSY; + goto out; + } + mf = find_in_list(&shm_inactive_head, cmp_cookie, cookie); + /* + * If the mobj isn't found or if it already has been unregistered. + */ + if (!mf) { + EMSG("cookie %#"PRIx64" not found", cookie); + res = TEE_ERROR_ITEM_NOT_FOUND; + goto out; + } +#if defined(CFG_CORE_SEL1_SPMC) + if (!mf->registered_by_cookie) { + EMSG("cookie %#"PRIx64" not registered", cookie); + res = TEE_ERROR_ITEM_NOT_FOUND; + goto out; + } + assert(mf->inactive_refs); + mf->inactive_refs--; + mf->registered_by_cookie = false; +#else + if (mf->inactive_refs) { + EMSG("cookie %#"PRIx64" busy refc %u:%u", + cookie, refcount_val(&mf->mobj.refc), mf->inactive_refs); + res = TEE_ERROR_BUSY; + goto out; + } + mf = pop_from_list(&shm_inactive_head, cmp_cookie, cookie); + mobj_ffa_spmc_delete(mf); + thread_spmc_relinquish(cookie); +#endif + res = TEE_SUCCESS; + +out: + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + return res; +} + +struct mobj *mobj_ffa_get_by_cookie(uint64_t cookie, + unsigned int internal_offs) +{ + struct mobj_ffa *mf = NULL; + uint32_t exceptions = 0; + + if (internal_offs >= SMALL_PAGE_SIZE) + return NULL; + exceptions = cpu_spin_lock_xsave(&shm_lock); + mf = find_in_list(&shm_head, cmp_cookie, cookie); + if (mf) { + if (mf->page_offset == internal_offs) { + if (!refcount_inc(&mf->mobj.refc)) { + /* + * If refcount is 0 some other thread has + * called mobj_put() on this reached 0 and + * before ffa_inactivate() got the lock we + * found it. Let's reinitialize it. + */ + refcount_set(&mf->mobj.refc, 1); + mf->inactive_refs++; + } + DMSG("cookie %#"PRIx64" active: refc %u:%u", + cookie, refcount_val(&mf->mobj.refc), + mf->inactive_refs); + } else { + EMSG("cookie %#"PRIx64" mismatching internal_offs got %#"PRIx16" expected %#x", + cookie, mf->page_offset, internal_offs); + mf = NULL; + } + } else { + mf = pop_from_list(&shm_inactive_head, cmp_cookie, cookie); +#if !defined(CFG_CORE_SEL1_SPMC) + /* Try to retrieve it from the SPM at S-EL2 */ + if (mf) { + DMSG("cookie %#"PRIx64" resurrecting", cookie); + } else { + DMSG("Populating mobj from rx buffer, cookie %#"PRIx64, + cookie); + mf = thread_spmc_populate_mobj_from_rx(cookie); + } +#endif + if (mf) { +#if defined(CFG_CORE_SEL1_SPMC) + if (!mf->registered_by_cookie) { + mf->inactive_refs++; + mf->registered_by_cookie = true; + } +#endif + assert(refcount_val(&mf->mobj.refc) == 0); + refcount_set(&mf->mobj.refc, 1); + refcount_set(&mf->mapcount, 0); + mf->inactive_refs++; + + /* + * mf->page_offset is offset into the first page. + * This offset is assigned from the internal_offs + * parameter to this function. + * + * While a mobj_ffa is active (ref_count > 0) this + * will not change, but when being pushed to the + * inactive list it can be changed again. + * + * So below we're backing out the old + * mf->page_offset and then assigning a new from + * internal_offset. + */ + mf->mobj.size += mf->page_offset; + assert(!(mf->mobj.size & SMALL_PAGE_MASK)); + mf->mobj.size -= internal_offs; + mf->page_offset = internal_offs; + + SLIST_INSERT_HEAD(&shm_head, mf, link); + } + } + + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + + if (!mf) { + EMSG("Failed to get cookie %#"PRIx64" internal_offs %#x", + cookie, internal_offs); + return NULL; + } + return &mf->mobj; +} + +static TEE_Result ffa_get_pa(struct mobj *mobj, size_t offset, + size_t granule, paddr_t *pa) +{ + struct mobj_ffa *mf = to_mobj_ffa(mobj); + size_t full_offset = 0; + paddr_t p = 0; + + if (!pa) + return TEE_ERROR_GENERIC; + + if (offset >= mobj->size) + return TEE_ERROR_GENERIC; + + full_offset = offset + mf->page_offset; + switch (granule) { + case 0: + p = mf->pages[full_offset / SMALL_PAGE_SIZE] + + (full_offset & SMALL_PAGE_MASK); + break; + case SMALL_PAGE_SIZE: + p = mf->pages[full_offset / SMALL_PAGE_SIZE]; + break; + default: + return TEE_ERROR_GENERIC; + } + *pa = p; + + return TEE_SUCCESS; +} + +static size_t ffa_get_phys_offs(struct mobj *mobj, + size_t granule __maybe_unused) +{ + assert(granule >= mobj->phys_granule); + + return to_mobj_ffa(mobj)->page_offset; +} + +static void *ffa_get_va(struct mobj *mobj, size_t offset, size_t len) +{ + struct mobj_ffa *mf = to_mobj_ffa(mobj); + + if (!mf->mm || !mobj_check_offset_and_len(mobj, offset, len)) + return NULL; + + return (void *)(tee_mm_get_smem(mf->mm) + offset + mf->page_offset); +} + +static void ffa_inactivate(struct mobj *mobj) +{ + struct mobj_ffa *mf = to_mobj_ffa(mobj); + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&shm_lock); + /* + * If refcount isn't 0 some other thread has found this mobj in + * shm_head after the mobj_put() that put us here and before we got + * the lock. + */ + if (refcount_val(&mobj->refc)) { + DMSG("cookie %#"PRIx64" was resurrected", mf->cookie); + goto out; + } + + /* + * pop_from_list() can fail to find the mobj if we had just + * decreased the refcount to 0 in mobj_put() and was going to + * acquire the shm_lock but another thread found this mobj and + * reinitialized the refcount to 1. Then before we got cpu time the + * other thread called mobj_put() and deactivated the mobj again. + * + * However, we still have the inactive count that guarantees + * that the mobj can't be freed until it reaches 0. + * At this point the mobj is in the inactive list. + */ + if (pop_from_list(&shm_head, cmp_ptr, (vaddr_t)mf)) { + unmap_helper(mf); + SLIST_INSERT_HEAD(&shm_inactive_head, mf, link); + } +out: + if (!mf->inactive_refs) + panic(); + mf->inactive_refs--; + cpu_spin_unlock_xrestore(&shm_lock, exceptions); +} + +static TEE_Result ffa_get_mem_type(struct mobj *mobj __unused, uint32_t *mt) +{ + if (!mt) + return TEE_ERROR_GENERIC; + + *mt = TEE_MATTR_MEM_TYPE_CACHED; + + return TEE_SUCCESS; +} + +static bool ffa_matches(struct mobj *mobj __maybe_unused, enum buf_is_attr attr) +{ + assert(mobj->ops == &mobj_ffa_ops); + + return attr == CORE_MEM_NON_SEC || attr == CORE_MEM_REG_SHM; +} + +static uint64_t ffa_get_cookie(struct mobj *mobj) +{ + return to_mobj_ffa(mobj)->cookie; +} + +static TEE_Result ffa_inc_map(struct mobj *mobj) +{ + TEE_Result res = TEE_SUCCESS; + struct mobj_ffa *mf = to_mobj_ffa(mobj); + uint32_t exceptions = 0; + size_t sz = 0; + + while (true) { + if (refcount_inc(&mf->mapcount)) + return TEE_SUCCESS; + + exceptions = cpu_spin_lock_xsave(&shm_lock); + + if (!refcount_val(&mf->mapcount)) + break; /* continue to reinitialize */ + /* + * If another thread beat us to initialize mapcount, + * restart to make sure we still increase it. + */ + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + } + + /* + * If we have beated another thread calling ffa_dec_map() + * to get the lock we need only to reinitialize mapcount to 1. + */ + if (!mf->mm) { + sz = ROUNDUP(mobj->size + mf->page_offset, SMALL_PAGE_SIZE); + mf->mm = tee_mm_alloc(&tee_mm_shm, sz); + if (!mf->mm) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = core_mmu_map_pages(tee_mm_get_smem(mf->mm), mf->pages, + sz / SMALL_PAGE_SIZE, + MEM_AREA_NSEC_SHM); + if (res) { + tee_mm_free(mf->mm); + mf->mm = NULL; + goto out; + } + } + + refcount_set(&mf->mapcount, 1); +out: + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + + return res; +} + +static TEE_Result ffa_dec_map(struct mobj *mobj) +{ + struct mobj_ffa *mf = to_mobj_ffa(mobj); + uint32_t exceptions = 0; + + if (!refcount_dec(&mf->mapcount)) + return TEE_SUCCESS; + + exceptions = cpu_spin_lock_xsave(&shm_lock); + if (!refcount_val(&mf->mapcount)) + unmap_helper(mf); + cpu_spin_unlock_xrestore(&shm_lock, exceptions); + + return TEE_SUCCESS; +} + +static TEE_Result mapped_shm_init(void) +{ + vaddr_t pool_start = 0; + vaddr_t pool_end = 0; + + core_mmu_get_mem_by_type(MEM_AREA_SHM_VASPACE, &pool_start, &pool_end); + if (!pool_start || !pool_end) + panic("Can't find region for shmem pool"); + + if (!tee_mm_init(&tee_mm_shm, pool_start, pool_end - pool_start, + SMALL_PAGE_SHIFT, + TEE_MM_POOL_NO_FLAGS)) + panic("Could not create shmem pool"); + + DMSG("Shared memory address range: %#"PRIxVA", %#"PRIxVA, + pool_start, pool_end); + return TEE_SUCCESS; +} + +static const struct mobj_ops mobj_ffa_ops = { + .get_pa = ffa_get_pa, + .get_phys_offs = ffa_get_phys_offs, + .get_va = ffa_get_va, + .get_mem_type = ffa_get_mem_type, + .matches = ffa_matches, + .free = ffa_inactivate, + .get_cookie = ffa_get_cookie, + .inc_map = ffa_inc_map, + .dec_map = ffa_dec_map, +}; + +preinit(mapped_shm_init); diff --git a/optee_os/core/arch/arm/mm/sp_mem.c b/optee_os/core/arch/arm/mm/sp_mem.c new file mode 100644 index 0000000..bdba730 --- /dev/null +++ b/optee_os/core/arch/arm/mm/sp_mem.c @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2022, Arm Limited. All rights reserved. + */ +#include +#include +#include +#include +#include +#include + +#define NUM_SHARES 64 + +static bitstr_t bit_decl(share_bits, NUM_SHARES); +static unsigned int sp_mem_lock = SPINLOCK_UNLOCK; + +/* mem_shares stores all active FF-A shares. */ +SLIST_HEAD(sp_mem_head, sp_mem); +static struct sp_mem_head mem_shares = SLIST_HEAD_INITIALIZER(sp_mem_head); +static const struct mobj_ops mobj_sp_ops; + +struct mobj_sp { + struct mobj mobj; + uint32_t mem_type; + bool is_secure; + paddr_t pages[]; +}; + +static struct mobj_sp *to_mobj_sp(struct mobj *mobj) +{ + assert(mobj->ops == &mobj_sp_ops); + return container_of(mobj, struct mobj_sp, mobj); +} + +static size_t mobj_sp_size(size_t num_pages) +{ + size_t s = 0; + + if (MUL_OVERFLOW(sizeof(paddr_t), num_pages, &s)) + return 0; + if (ADD_OVERFLOW(sizeof(struct mobj_sp), s, &s)) + return 0; + return s; +} + +struct mobj *sp_mem_new_mobj(uint64_t pages, uint32_t mem_type, bool is_secure) +{ + struct mobj_sp *m = NULL; + size_t s = 0; + + s = mobj_sp_size(pages); + if (!s) + return NULL; + + m = calloc(1, s); + if (!m) + return NULL; + + m->mobj.ops = &mobj_sp_ops; + m->mobj.size = pages * SMALL_PAGE_SIZE; + m->mobj.phys_granule = SMALL_PAGE_SIZE; + + m->mem_type = mem_type; + m->is_secure = is_secure; + + refcount_set(&m->mobj.refc, 1); + return &m->mobj; +} + +static size_t get_page_count(struct mobj_sp *ms) +{ + return ROUNDUP(ms->mobj.size, SMALL_PAGE_SIZE) / SMALL_PAGE_SIZE; +} + +/* Add some physical pages to the mobj object. */ +int sp_mem_add_pages(struct mobj *mobj, unsigned int *idx, + paddr_t pa, unsigned int num_pages) +{ + struct mobj_sp *ms = to_mobj_sp(mobj); + unsigned int n = 0; + size_t tot_page_count = get_page_count(ms); + + if (ADD_OVERFLOW(*idx, num_pages, &n) || n > tot_page_count) + return TEE_ERROR_BAD_PARAMETERS; + + /* Don't check for device memory */ + if (ms->mem_type == TEE_MATTR_MEM_TYPE_CACHED) { + if (ms->is_secure) { + if (!tee_pbuf_is_sec(pa, num_pages * SMALL_PAGE_SIZE)) + return TEE_ERROR_BAD_PARAMETERS; + } else { + if (!tee_pbuf_is_non_sec(pa, + num_pages * SMALL_PAGE_SIZE)) + return TEE_ERROR_BAD_PARAMETERS; + } + } + + for (n = 0; n < num_pages; n++) + ms->pages[n + *idx] = pa + n * SMALL_PAGE_SIZE; + + *idx += n; + return TEE_SUCCESS; +} + +static TEE_Result get_mem_type(struct mobj *mobj, uint32_t *mt) +{ + struct mobj_sp *m = to_mobj_sp(mobj); + + *mt = m->mem_type; + + return TEE_SUCCESS; +} + +static bool mobj_sp_matches(struct mobj *mobj, enum buf_is_attr attr) +{ + struct mobj_sp *m = to_mobj_sp(mobj); + + if (m->is_secure) + return attr == CORE_MEM_SEC; + else + return attr == CORE_MEM_NON_SEC || attr == CORE_MEM_REG_SHM; +} + +static TEE_Result get_pa(struct mobj *mobj, size_t offset, + size_t granule, paddr_t *pa) +{ + struct mobj_sp *ms = to_mobj_sp(mobj); + paddr_t p = 0; + + if (!pa) + return TEE_ERROR_GENERIC; + + if (offset >= mobj->size) + return TEE_ERROR_GENERIC; + + switch (granule) { + case 0: + p = ms->pages[offset / SMALL_PAGE_SIZE] + + (offset & SMALL_PAGE_MASK); + break; + case SMALL_PAGE_SIZE: + p = ms->pages[offset / SMALL_PAGE_SIZE]; + break; + default: + return TEE_ERROR_GENERIC; + } + *pa = p; + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(get_pa); + +static size_t get_phys_offs(struct mobj *mobj __maybe_unused, + size_t granule __maybe_unused) +{ + return 0; +} + +static void inactivate(struct mobj *mobj) +{ + struct mobj_sp *ms = to_mobj_sp(mobj); + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&sp_mem_lock); + /* + * If refcount isn't 0 some other thread has found this mobj in + * shm_head after the mobj_put() that put us here and before we got + * the lock. + */ + if (!refcount_val(&mobj->refc)) + free(ms); + + cpu_spin_unlock_xrestore(&sp_mem_lock, exceptions); +} + +static const struct mobj_ops mobj_sp_ops = { + .get_pa = get_pa, + .get_phys_offs = get_phys_offs, + .get_mem_type = get_mem_type, + .matches = mobj_sp_matches, + .free = inactivate, +}; + +struct sp_mem_receiver *sp_mem_get_receiver(uint32_t s_id, struct sp_mem *smem) +{ + struct sp_mem_receiver *r = NULL; + + SLIST_FOREACH(r, &smem->receivers, link) { + if (r->perm.endpoint_id == s_id) + return r; + } + return NULL; +} + +struct sp_mem *sp_mem_get(uint64_t handle) +{ + struct sp_mem *smem = NULL; + uint32_t exceptions = cpu_spin_lock_xsave(&sp_mem_lock); + + SLIST_FOREACH(smem, &mem_shares, link) { + if (smem->global_handle == handle) + break; + } + + cpu_spin_unlock_xrestore(&sp_mem_lock, exceptions); + return smem; +} + +void *sp_mem_get_va(const struct user_mode_ctx *uctx, size_t offset, + struct mobj *mobj) +{ + struct vm_region *region = NULL; + + TAILQ_FOREACH(region, &uctx->vm_info.regions, link) { + if (region->mobj == mobj && region->offset == offset) + return (void *)region->va; + } + return NULL; +} + +struct sp_mem *sp_mem_new(void) +{ + struct sp_mem *smem = NULL; + uint32_t exceptions = 0; + int i = 0; + + smem = calloc(sizeof(*smem), 1); + if (!smem) + return NULL; + + exceptions = cpu_spin_lock_xsave(&sp_mem_lock); + + bit_ffc(share_bits, NUM_SHARES, &i); + if (i == -1) { + cpu_spin_unlock_xrestore(&sp_mem_lock, exceptions); + free(smem); + return NULL; + } + + bit_set(share_bits, i); + /* + * OP-TEE SHAREs use bit 44 use bit 45 instead. + */ + smem->global_handle = i | FFA_MEMORY_HANDLE_SECURE_BIT; + SLIST_INIT(&smem->regions); + SLIST_INIT(&smem->receivers); + + cpu_spin_unlock_xrestore(&sp_mem_lock, exceptions); + + return smem; +} + +void sp_mem_add(struct sp_mem *smem) +{ + uint32_t exceptions = cpu_spin_lock_xsave(&sp_mem_lock); + + SLIST_INSERT_HEAD(&mem_shares, smem, link); + + cpu_spin_unlock_xrestore(&sp_mem_lock, exceptions); +} + +bool sp_mem_is_shared(struct sp_mem_map_region *new_reg) +{ + struct sp_mem *smem = NULL; + uint32_t exceptions = cpu_spin_lock_xsave(&sp_mem_lock); + uint64_t new_reg_end = new_reg->page_offset + + (new_reg->page_count * SMALL_PAGE_SIZE); + + SLIST_FOREACH(smem, &mem_shares, link) { + struct sp_mem_map_region *reg = NULL; + + SLIST_FOREACH(reg, &smem->regions, link) { + if (new_reg->mobj == reg->mobj) { + uint64_t reg_end = 0; + + reg_end = reg->page_offset + + (reg->page_count * SMALL_PAGE_SIZE); + + if (new_reg->page_offset < reg_end && + new_reg_end > reg->page_offset) { + cpu_spin_unlock_xrestore(&sp_mem_lock, + exceptions); + return true; + } + } + } + } + + cpu_spin_unlock_xrestore(&sp_mem_lock, exceptions); + return false; +} + +void sp_mem_remove(struct sp_mem *smem) +{ + uint32_t exceptions = 0; + int i = 0; + struct sp_mem *tsmem = NULL; + + if (!smem) + return; + + /* Remove all receivers */ + while (!SLIST_EMPTY(&smem->receivers)) { + struct sp_mem_receiver *receiver = NULL; + + receiver = SLIST_FIRST(&smem->receivers); + SLIST_REMOVE_HEAD(&smem->receivers, link); + free(receiver); + } + /* Remove all regions */ + while (!SLIST_EMPTY(&smem->regions)) { + struct sp_mem_map_region *region = SLIST_FIRST(&smem->regions); + + mobj_put(region->mobj); + + SLIST_REMOVE_HEAD(&smem->regions, link); + free(region); + } + + exceptions = cpu_spin_lock_xsave(&sp_mem_lock); + + i = smem->global_handle & ~FFA_MEMORY_HANDLE_SECURE_BIT; + assert(i < NUM_SHARES); + + bit_clear(share_bits, i); + + SLIST_FOREACH(tsmem, &mem_shares, link) { + if (tsmem == smem) { + SLIST_REMOVE(&mem_shares, smem, sp_mem, link); + break; + } + } + + cpu_spin_unlock_xrestore(&sp_mem_lock, exceptions); + + free(smem); +} diff --git a/optee_os/core/arch/arm/mm/sub.mk b/optee_os/core/arch/arm/mm/sub.mk new file mode 100644 index 0000000..1fff635 --- /dev/null +++ b/optee_os/core/arch/arm/mm/sub.mk @@ -0,0 +1,24 @@ +srcs-y += core_mmu.c +srcs-$(CFG_WITH_PAGER) += tee_pager.c +ifeq ($(CFG_WITH_LPAE),y) +srcs-y += core_mmu_lpae.c +else +srcs-y += core_mmu_v7.c +endif +srcs-$(CFG_CORE_FFA) += mobj_ffa.c +srcs-$(CFG_SECURE_PARTITION) += sp_mem.c +ifneq ($(CFG_CORE_FFA),y) +srcs-$(CFG_CORE_DYN_SHM) += mobj_dyn_shm.c +endif + +ifeq ($(CFG_SYSCALL_FTRACE),y) +# We would not like to profile MMU APIs as these are used to switch TA +# context which may cause undesired behaviour as ftrace requires TA context +# to be active. Moreover profiling code uses some of MMU APIs to check +# if TA context is active or not. +ifeq ($(CFG_WITH_LPAE),y) +cflags-remove-core_mmu_lpae.c-y += -pg +else +cflags-remove-core_mmu_v7.c-y += -pg +endif +endif diff --git a/optee_os/core/arch/arm/mm/tee_pager.c b/optee_os/core/arch/arm/mm/tee_pager.c new file mode 100644 index 0000000..a9d3f39 --- /dev/null +++ b/optee_os/core/arch/arm/mm/tee_pager.c @@ -0,0 +1,2048 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2021, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static struct vm_paged_region_head core_vm_regions = + TAILQ_HEAD_INITIALIZER(core_vm_regions); + +#define INVALID_PGIDX UINT_MAX +#define PMEM_FLAG_DIRTY BIT(0) +#define PMEM_FLAG_HIDDEN BIT(1) + +/* + * struct tee_pager_pmem - Represents a physical page used for paging. + * + * @flags flags defined by PMEM_FLAG_* above + * @fobj_pgidx index of the page in the @fobj + * @fobj File object of which a page is made visible. + * @va_alias Virtual address where the physical page always is aliased. + * Used during remapping of the page when the content need to + * be updated before it's available at the new location. + */ +struct tee_pager_pmem { + unsigned int flags; + unsigned int fobj_pgidx; + struct fobj *fobj; + void *va_alias; + TAILQ_ENTRY(tee_pager_pmem) link; +}; + +struct tblidx { + struct pgt *pgt; + unsigned int idx; +}; + +/* The list of physical pages. The first page in the list is the oldest */ +TAILQ_HEAD(tee_pager_pmem_head, tee_pager_pmem); + +static struct tee_pager_pmem_head tee_pager_pmem_head = + TAILQ_HEAD_INITIALIZER(tee_pager_pmem_head); + +static struct tee_pager_pmem_head tee_pager_lock_pmem_head = + TAILQ_HEAD_INITIALIZER(tee_pager_lock_pmem_head); + +/* number of pages hidden */ +#define TEE_PAGER_NHIDE (tee_pager_npages / 3) + +/* Number of registered physical pages, used hiding pages. */ +static size_t tee_pager_npages; + +/* This area covers the IVs for all fobjs with paged IVs */ +static struct vm_paged_region *pager_iv_region; +/* Used by make_iv_available(), see make_iv_available() for details. */ +static struct tee_pager_pmem *pager_spare_pmem; + +#ifdef CFG_WITH_STATS +static struct tee_pager_stats pager_stats; + +static inline void incr_ro_hits(void) +{ + pager_stats.ro_hits++; +} + +static inline void incr_rw_hits(void) +{ + pager_stats.rw_hits++; +} + +static inline void incr_hidden_hits(void) +{ + pager_stats.hidden_hits++; +} + +static inline void incr_zi_released(void) +{ + pager_stats.zi_released++; +} + +static inline void incr_npages_all(void) +{ + pager_stats.npages_all++; +} + +static inline void set_npages(void) +{ + pager_stats.npages = tee_pager_npages; +} + +void tee_pager_get_stats(struct tee_pager_stats *stats) +{ + *stats = pager_stats; + + pager_stats.hidden_hits = 0; + pager_stats.ro_hits = 0; + pager_stats.rw_hits = 0; + pager_stats.zi_released = 0; +} + +#else /* CFG_WITH_STATS */ +static inline void incr_ro_hits(void) { } +static inline void incr_rw_hits(void) { } +static inline void incr_hidden_hits(void) { } +static inline void incr_zi_released(void) { } +static inline void incr_npages_all(void) { } +static inline void set_npages(void) { } + +void tee_pager_get_stats(struct tee_pager_stats *stats) +{ + memset(stats, 0, sizeof(struct tee_pager_stats)); +} +#endif /* CFG_WITH_STATS */ + +#define TBL_NUM_ENTRIES (CORE_MMU_PGDIR_SIZE / SMALL_PAGE_SIZE) +#define TBL_LEVEL CORE_MMU_PGDIR_LEVEL +#define TBL_SHIFT SMALL_PAGE_SHIFT + +#define EFFECTIVE_VA_SIZE \ + (ROUNDUP(VCORE_START_VA + TEE_RAM_VA_SIZE, CORE_MMU_PGDIR_SIZE) - \ + ROUNDDOWN(VCORE_START_VA, CORE_MMU_PGDIR_SIZE)) + +static struct pager_table { + struct pgt pgt; + struct core_mmu_table_info tbl_info; +} *pager_tables; +static unsigned int num_pager_tables; + +static unsigned pager_spinlock = SPINLOCK_UNLOCK; + +/* Defines the range of the alias area */ +static tee_mm_entry_t *pager_alias_area; +/* + * Physical pages are added in a stack like fashion to the alias area, + * @pager_alias_next_free gives the address of next free entry if + * @pager_alias_next_free is != 0 + */ +static uintptr_t pager_alias_next_free; + +#ifdef CFG_TEE_CORE_DEBUG +#define pager_lock(ai) pager_lock_dldetect(__func__, __LINE__, ai) + +static uint32_t pager_lock_dldetect(const char *func, const int line, + struct abort_info *ai) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + unsigned int retries = 0; + unsigned int reminder = 0; + + while (!cpu_spin_trylock(&pager_spinlock)) { + retries++; + if (!retries) { + /* wrapped, time to report */ + trace_printf(func, line, TRACE_ERROR, true, + "possible spinlock deadlock reminder %u", + reminder); + if (reminder < UINT_MAX) + reminder++; + if (ai) + abort_print(ai); + } + } + + return exceptions; +} +#else +static uint32_t pager_lock(struct abort_info __unused *ai) +{ + return cpu_spin_lock_xsave(&pager_spinlock); +} +#endif + +static uint32_t pager_lock_check_stack(size_t stack_size) +{ + if (stack_size) { + int8_t buf[stack_size]; + size_t n; + + /* + * Make sure to touch all pages of the stack that we expect + * to use with this lock held. We need to take eventual + * page faults before the lock is taken or we'll deadlock + * the pager. The pages that are populated in this way will + * eventually be released at certain save transitions of + * the thread. + */ + for (n = 0; n < stack_size; n += SMALL_PAGE_SIZE) + io_write8((vaddr_t)buf + n, 1); + io_write8((vaddr_t)buf + stack_size - 1, 1); + } + + return pager_lock(NULL); +} + +static void pager_unlock(uint32_t exceptions) +{ + cpu_spin_unlock_xrestore(&pager_spinlock, exceptions); +} + +void *tee_pager_phys_to_virt(paddr_t pa, size_t len) +{ + struct core_mmu_table_info ti; + unsigned idx; + uint32_t a; + paddr_t p; + vaddr_t v; + size_t n; + + if (pa & SMALL_PAGE_MASK || len > SMALL_PAGE_SIZE) + return NULL; + + /* + * Most addresses are mapped lineary, try that first if possible. + */ + if (!tee_pager_get_table_info(pa, &ti)) + return NULL; /* impossible pa */ + idx = core_mmu_va2idx(&ti, pa); + core_mmu_get_entry(&ti, idx, &p, &a); + if ((a & TEE_MATTR_VALID_BLOCK) && p == pa) + return (void *)core_mmu_idx2va(&ti, idx); + + n = 0; + idx = core_mmu_va2idx(&pager_tables[n].tbl_info, TEE_RAM_START); + while (true) { + while (idx < TBL_NUM_ENTRIES) { + v = core_mmu_idx2va(&pager_tables[n].tbl_info, idx); + if (v >= (TEE_RAM_START + TEE_RAM_VA_SIZE)) + return NULL; + + core_mmu_get_entry(&pager_tables[n].tbl_info, + idx, &p, &a); + if ((a & TEE_MATTR_VALID_BLOCK) && p == pa) + return (void *)v; + idx++; + } + + n++; + if (n >= num_pager_tables) + return NULL; + idx = 0; + } + + return NULL; +} + +static bool pmem_is_hidden(struct tee_pager_pmem *pmem) +{ + return pmem->flags & PMEM_FLAG_HIDDEN; +} + +static bool pmem_is_dirty(struct tee_pager_pmem *pmem) +{ + return pmem->flags & PMEM_FLAG_DIRTY; +} + +static bool pmem_is_covered_by_region(struct tee_pager_pmem *pmem, + struct vm_paged_region *reg) +{ + if (pmem->fobj != reg->fobj) + return false; + if (pmem->fobj_pgidx < reg->fobj_pgoffs) + return false; + if ((pmem->fobj_pgidx - reg->fobj_pgoffs) >= + (reg->size >> SMALL_PAGE_SHIFT)) + return false; + + return true; +} + +static size_t get_pgt_count(vaddr_t base, size_t size) +{ + assert(size); + + return (base + size - 1) / CORE_MMU_PGDIR_SIZE + 1 - + base / CORE_MMU_PGDIR_SIZE; +} + +static bool region_have_pgt(struct vm_paged_region *reg, struct pgt *pgt) +{ + size_t n = 0; + + for (n = 0; n < get_pgt_count(reg->base, reg->size); n++) + if (reg->pgt_array[n] == pgt) + return true; + + return false; +} + +static struct tblidx pmem_get_region_tblidx(struct tee_pager_pmem *pmem, + struct vm_paged_region *reg) +{ + size_t tbloffs = (reg->base & CORE_MMU_PGDIR_MASK) >> SMALL_PAGE_SHIFT; + size_t idx = pmem->fobj_pgidx - reg->fobj_pgoffs + tbloffs; + + assert(pmem->fobj && pmem->fobj_pgidx != INVALID_PGIDX); + assert(idx / TBL_NUM_ENTRIES < get_pgt_count(reg->base, reg->size)); + + return (struct tblidx){ + .idx = idx % TBL_NUM_ENTRIES, + .pgt = reg->pgt_array[idx / TBL_NUM_ENTRIES], + }; +} + +static struct pager_table *find_pager_table_may_fail(vaddr_t va) +{ + size_t n; + const vaddr_t mask = CORE_MMU_PGDIR_MASK; + + if (!pager_tables) + return NULL; + + n = ((va & ~mask) - pager_tables[0].tbl_info.va_base) >> + CORE_MMU_PGDIR_SHIFT; + if (n >= num_pager_tables) + return NULL; + + assert(va >= pager_tables[n].tbl_info.va_base && + va <= (pager_tables[n].tbl_info.va_base | mask)); + + return pager_tables + n; +} + +static struct pager_table *find_pager_table(vaddr_t va) +{ + struct pager_table *pt = find_pager_table_may_fail(va); + + assert(pt); + return pt; +} + +bool tee_pager_get_table_info(vaddr_t va, struct core_mmu_table_info *ti) +{ + struct pager_table *pt = find_pager_table_may_fail(va); + + if (!pt) + return false; + + *ti = pt->tbl_info; + return true; +} + +static struct core_mmu_table_info *find_table_info(vaddr_t va) +{ + return &find_pager_table(va)->tbl_info; +} + +static struct pgt *find_core_pgt(vaddr_t va) +{ + return &find_pager_table(va)->pgt; +} + +void tee_pager_set_alias_area(tee_mm_entry_t *mm) +{ + struct pager_table *pt; + unsigned idx; + vaddr_t smem = tee_mm_get_smem(mm); + size_t nbytes = tee_mm_get_bytes(mm); + vaddr_t v; + uint32_t a = 0; + + DMSG("0x%" PRIxVA " - 0x%" PRIxVA, smem, smem + nbytes); + + assert(!pager_alias_area); + pager_alias_area = mm; + pager_alias_next_free = smem; + + /* Clear all mapping in the alias area */ + pt = find_pager_table(smem); + idx = core_mmu_va2idx(&pt->tbl_info, smem); + while (pt <= (pager_tables + num_pager_tables - 1)) { + while (idx < TBL_NUM_ENTRIES) { + v = core_mmu_idx2va(&pt->tbl_info, idx); + if (v >= (smem + nbytes)) + goto out; + + core_mmu_get_entry(&pt->tbl_info, idx, NULL, &a); + core_mmu_set_entry(&pt->tbl_info, idx, 0, 0); + if (a & TEE_MATTR_VALID_BLOCK) + pgt_dec_used_entries(&pt->pgt); + idx++; + } + + pt++; + idx = 0; + } + +out: + tlbi_va_range(smem, nbytes, SMALL_PAGE_SIZE); +} + +static size_t tbl_usage_count(struct core_mmu_table_info *ti) +{ + size_t n; + uint32_t a = 0; + size_t usage = 0; + + for (n = 0; n < ti->num_entries; n++) { + core_mmu_get_entry(ti, n, NULL, &a); + if (a & TEE_MATTR_VALID_BLOCK) + usage++; + } + return usage; +} + +static void tblidx_get_entry(struct tblidx tblidx, paddr_t *pa, uint32_t *attr) +{ + assert(tblidx.pgt && tblidx.idx < TBL_NUM_ENTRIES); + core_mmu_get_entry_primitive(tblidx.pgt->tbl, TBL_LEVEL, tblidx.idx, + pa, attr); +} + +static void tblidx_set_entry(struct tblidx tblidx, paddr_t pa, uint32_t attr) +{ + assert(tblidx.pgt && tblidx.idx < TBL_NUM_ENTRIES); + core_mmu_set_entry_primitive(tblidx.pgt->tbl, TBL_LEVEL, tblidx.idx, + pa, attr); +} + +static struct tblidx region_va2tblidx(struct vm_paged_region *reg, vaddr_t va) +{ + paddr_t mask = CORE_MMU_PGDIR_MASK; + size_t n = 0; + + assert(va >= reg->base && va < (reg->base + reg->size)); + n = (va - (reg->base & ~mask)) / CORE_MMU_PGDIR_SIZE; + + return (struct tblidx){ + .idx = (va & mask) / SMALL_PAGE_SIZE, + .pgt = reg->pgt_array[n], + }; +} + +static vaddr_t tblidx2va(struct tblidx tblidx) +{ + return tblidx.pgt->vabase + (tblidx.idx << SMALL_PAGE_SHIFT); +} + +static void tblidx_tlbi_entry(struct tblidx tblidx) +{ + vaddr_t va = tblidx2va(tblidx); + +#if defined(CFG_PAGED_USER_TA) + if (tblidx.pgt->ctx) { + uint32_t asid = to_user_mode_ctx(tblidx.pgt->ctx)->vm_info.asid; + + tlbi_va_asid(va, asid); + return; + } +#endif + tlbi_va_allasid(va); +} + +static void pmem_assign_fobj_page(struct tee_pager_pmem *pmem, + struct vm_paged_region *reg, vaddr_t va) +{ + struct tee_pager_pmem *p = NULL; + unsigned int fobj_pgidx = 0; + + assert(!pmem->fobj && pmem->fobj_pgidx == INVALID_PGIDX); + + assert(va >= reg->base && va < (reg->base + reg->size)); + fobj_pgidx = (va - reg->base) / SMALL_PAGE_SIZE + reg->fobj_pgoffs; + + TAILQ_FOREACH(p, &tee_pager_pmem_head, link) + assert(p->fobj != reg->fobj || p->fobj_pgidx != fobj_pgidx); + + pmem->fobj = reg->fobj; + pmem->fobj_pgidx = fobj_pgidx; +} + +static void pmem_clear(struct tee_pager_pmem *pmem) +{ + pmem->fobj = NULL; + pmem->fobj_pgidx = INVALID_PGIDX; + pmem->flags = 0; +} + +static void pmem_unmap(struct tee_pager_pmem *pmem, struct pgt *only_this_pgt) +{ + struct vm_paged_region *reg = NULL; + struct tblidx tblidx = { }; + uint32_t a = 0; + + TAILQ_FOREACH(reg, &pmem->fobj->regions, fobj_link) { + /* + * If only_this_pgt points to a pgt then the pgt of this + * region has to match or we'll skip over it. + */ + if (only_this_pgt && !region_have_pgt(reg, only_this_pgt)) + continue; + if (!pmem_is_covered_by_region(pmem, reg)) + continue; + tblidx = pmem_get_region_tblidx(pmem, reg); + if (!tblidx.pgt) + continue; + tblidx_get_entry(tblidx, NULL, &a); + if (a & TEE_MATTR_VALID_BLOCK) { + tblidx_set_entry(tblidx, 0, 0); + pgt_dec_used_entries(tblidx.pgt); + tblidx_tlbi_entry(tblidx); + } + } +} + +void tee_pager_early_init(void) +{ + size_t n = 0; + + num_pager_tables = EFFECTIVE_VA_SIZE / CORE_MMU_PGDIR_SIZE; + pager_tables = calloc(num_pager_tables, sizeof(*pager_tables)); + if (!pager_tables) + panic("Cannot allocate pager_tables"); + + /* + * Note that this depends on add_pager_vaspace() adding vaspace + * after end of memory. + */ + for (n = 0; n < num_pager_tables; n++) { + if (!core_mmu_find_table(NULL, VCORE_START_VA + + n * CORE_MMU_PGDIR_SIZE, UINT_MAX, + &pager_tables[n].tbl_info)) + panic("can't find mmu tables"); + + if (pager_tables[n].tbl_info.shift != TBL_SHIFT) + panic("Unsupported page size in translation table"); + assert(pager_tables[n].tbl_info.num_entries == TBL_NUM_ENTRIES); + assert(pager_tables[n].tbl_info.level == TBL_LEVEL); + + pager_tables[n].pgt.tbl = pager_tables[n].tbl_info.table; + pager_tables[n].pgt.vabase = pager_tables[n].tbl_info.va_base; + pgt_set_used_entries(&pager_tables[n].pgt, + tbl_usage_count(&pager_tables[n].tbl_info)); + } +} + +static void *pager_add_alias_page(paddr_t pa) +{ + unsigned idx; + struct core_mmu_table_info *ti; + /* Alias pages mapped without write permission: runtime will care */ + uint32_t attr = TEE_MATTR_VALID_BLOCK | TEE_MATTR_SECURE | + TEE_MATTR_PR | (TEE_MATTR_MEM_TYPE_CACHED << + TEE_MATTR_MEM_TYPE_SHIFT); + + DMSG("0x%" PRIxPA, pa); + + ti = find_table_info(pager_alias_next_free); + idx = core_mmu_va2idx(ti, pager_alias_next_free); + core_mmu_set_entry(ti, idx, pa, attr); + pgt_inc_used_entries(find_core_pgt(pager_alias_next_free)); + pager_alias_next_free += SMALL_PAGE_SIZE; + if (pager_alias_next_free >= (tee_mm_get_smem(pager_alias_area) + + tee_mm_get_bytes(pager_alias_area))) + pager_alias_next_free = 0; + return (void *)core_mmu_idx2va(ti, idx); +} + +static void region_insert(struct vm_paged_region_head *regions, + struct vm_paged_region *reg, + struct vm_paged_region *r_prev) +{ + uint32_t exceptions = pager_lock_check_stack(8); + + if (r_prev) + TAILQ_INSERT_AFTER(regions, r_prev, reg, link); + else + TAILQ_INSERT_HEAD(regions, reg, link); + TAILQ_INSERT_TAIL(®->fobj->regions, reg, fobj_link); + + pager_unlock(exceptions); +} +DECLARE_KEEP_PAGER(region_insert); + +static struct vm_paged_region *alloc_region(vaddr_t base, size_t size) +{ + struct vm_paged_region *reg = NULL; + + if ((base & SMALL_PAGE_MASK) || !size) { + EMSG("invalid pager region [%" PRIxVA " +0x%zx]", base, size); + panic(); + } + + reg = calloc(1, sizeof(*reg)); + if (!reg) + return NULL; + reg->pgt_array = calloc(get_pgt_count(base, size), + sizeof(struct pgt *)); + if (!reg->pgt_array) { + free(reg); + return NULL; + } + + reg->base = base; + reg->size = size; + return reg; +} + +void tee_pager_add_core_region(vaddr_t base, enum vm_paged_region_type type, + struct fobj *fobj) +{ + struct vm_paged_region *reg = NULL; + size_t n = 0; + + assert(fobj); + + DMSG("0x%" PRIxPTR " - 0x%" PRIxPTR " : type %d", + base, base + fobj->num_pages * SMALL_PAGE_SIZE, type); + + reg = alloc_region(base, fobj->num_pages * SMALL_PAGE_SIZE); + if (!reg) + panic("alloc_region"); + + reg->fobj = fobj_get(fobj); + reg->fobj_pgoffs = 0; + reg->type = type; + + switch (type) { + case PAGED_REGION_TYPE_RO: + reg->flags = TEE_MATTR_PRX; + break; + case PAGED_REGION_TYPE_RW: + case PAGED_REGION_TYPE_LOCK: + reg->flags = TEE_MATTR_PRW; + break; + default: + panic(); + } + + for (n = 0; n < get_pgt_count(reg->base, reg->size); n++) + reg->pgt_array[n] = find_core_pgt(base + + n * CORE_MMU_PGDIR_SIZE); + region_insert(&core_vm_regions, reg, NULL); +} + +static struct vm_paged_region *find_region(struct vm_paged_region_head *regions, + vaddr_t va) +{ + struct vm_paged_region *reg; + + if (!regions) + return NULL; + + TAILQ_FOREACH(reg, regions, link) { + if (core_is_buffer_inside(va, 1, reg->base, reg->size)) + return reg; + } + return NULL; +} + +#ifdef CFG_PAGED_USER_TA +static struct vm_paged_region *find_uta_region(vaddr_t va) +{ + struct ts_ctx *ctx = thread_get_tsd()->ctx; + + if (!is_user_mode_ctx(ctx)) + return NULL; + return find_region(to_user_mode_ctx(ctx)->regions, va); +} +#else +static struct vm_paged_region *find_uta_region(vaddr_t va __unused) +{ + return NULL; +} +#endif /*CFG_PAGED_USER_TA*/ + + +static uint32_t get_region_mattr(uint32_t reg_flags) +{ + uint32_t attr = TEE_MATTR_VALID_BLOCK | TEE_MATTR_SECURE | + TEE_MATTR_MEM_TYPE_CACHED << TEE_MATTR_MEM_TYPE_SHIFT | + (reg_flags & (TEE_MATTR_PRWX | TEE_MATTR_URWX)); + + return attr; +} + +static paddr_t get_pmem_pa(struct tee_pager_pmem *pmem) +{ + struct core_mmu_table_info *ti; + paddr_t pa; + unsigned idx; + + ti = find_table_info((vaddr_t)pmem->va_alias); + idx = core_mmu_va2idx(ti, (vaddr_t)pmem->va_alias); + core_mmu_get_entry(ti, idx, &pa, NULL); + return pa; +} + +#ifdef CFG_PAGED_USER_TA +static void unlink_region(struct vm_paged_region_head *regions, + struct vm_paged_region *reg) +{ + uint32_t exceptions = pager_lock_check_stack(64); + + TAILQ_REMOVE(regions, reg, link); + TAILQ_REMOVE(®->fobj->regions, reg, fobj_link); + + pager_unlock(exceptions); +} +DECLARE_KEEP_PAGER(unlink_region); + +static void free_region(struct vm_paged_region *reg) +{ + fobj_put(reg->fobj); + free(reg->pgt_array); + free(reg); +} + +static TEE_Result pager_add_um_region(struct user_mode_ctx *uctx, vaddr_t base, + struct fobj *fobj, uint32_t prot) +{ + struct vm_paged_region *r_prev = NULL; + struct vm_paged_region *reg = NULL; + vaddr_t b = base; + size_t fobj_pgoffs = 0; + size_t s = fobj->num_pages * SMALL_PAGE_SIZE; + + if (!uctx->regions) { + uctx->regions = malloc(sizeof(*uctx->regions)); + if (!uctx->regions) + return TEE_ERROR_OUT_OF_MEMORY; + TAILQ_INIT(uctx->regions); + } + + reg = TAILQ_FIRST(uctx->regions); + while (reg) { + if (core_is_buffer_intersect(b, s, reg->base, reg->size)) + return TEE_ERROR_BAD_PARAMETERS; + if (b < reg->base) + break; + r_prev = reg; + reg = TAILQ_NEXT(reg, link); + } + + reg = alloc_region(b, s); + if (!reg) + return TEE_ERROR_OUT_OF_MEMORY; + + /* Table info will be set when the context is activated. */ + reg->fobj = fobj_get(fobj); + reg->fobj_pgoffs = fobj_pgoffs; + reg->type = PAGED_REGION_TYPE_RW; + reg->flags = prot; + + region_insert(uctx->regions, reg, r_prev); + + return TEE_SUCCESS; +} + +static void map_pgts(struct vm_paged_region *reg) +{ + struct core_mmu_table_info dir_info = { NULL }; + size_t n = 0; + + core_mmu_get_user_pgdir(&dir_info); + + for (n = 0; n < get_pgt_count(reg->base, reg->size); n++) { + struct pgt *pgt = reg->pgt_array[n]; + uint32_t attr = 0; + paddr_t pa = 0; + size_t idx = 0; + + idx = core_mmu_va2idx(&dir_info, pgt->vabase); + core_mmu_get_entry(&dir_info, idx, &pa, &attr); + + /* + * Check if the page table already is used, if it is, it's + * already registered. + */ + if (pgt->num_used_entries) { + assert(attr & TEE_MATTR_TABLE); + assert(pa == virt_to_phys(pgt->tbl)); + continue; + } + + attr = TEE_MATTR_SECURE | TEE_MATTR_TABLE; + pa = virt_to_phys(pgt->tbl); + assert(pa); + /* + * Note that the update of the table entry is guaranteed to + * be atomic. + */ + core_mmu_set_entry(&dir_info, idx, pa, attr); + } +} + +TEE_Result tee_pager_add_um_region(struct user_mode_ctx *uctx, vaddr_t base, + struct fobj *fobj, uint32_t prot) +{ + TEE_Result res = TEE_SUCCESS; + struct thread_specific_data *tsd = thread_get_tsd(); + struct vm_paged_region *reg = NULL; + + res = pager_add_um_region(uctx, base, fobj, prot); + if (res) + return res; + + if (uctx->ts_ctx == tsd->ctx) { + /* + * We're chaning the currently active utc. Assign page + * tables to the new regions and make sure that the page + * tables are registered in the upper table. + */ + tee_pager_assign_um_tables(uctx); + TAILQ_FOREACH(reg, uctx->regions, link) + map_pgts(reg); + } + + return TEE_SUCCESS; +} + +static void split_region(struct vm_paged_region *reg, + struct vm_paged_region *r2, vaddr_t va) +{ + uint32_t exceptions = pager_lock_check_stack(64); + size_t diff = va - reg->base; + size_t r2_pgt_count = 0; + size_t reg_pgt_count = 0; + size_t n0 = 0; + size_t n = 0; + + assert(r2->base == va); + assert(r2->size == reg->size - diff); + + r2->fobj = fobj_get(reg->fobj); + r2->fobj_pgoffs = reg->fobj_pgoffs + diff / SMALL_PAGE_SIZE; + r2->type = reg->type; + r2->flags = reg->flags; + + r2_pgt_count = get_pgt_count(r2->base, r2->size); + reg_pgt_count = get_pgt_count(reg->base, reg->size); + n0 = reg_pgt_count - r2_pgt_count; + for (n = n0; n < reg_pgt_count; n++) + r2->pgt_array[n - n0] = reg->pgt_array[n]; + reg->size = diff; + + TAILQ_INSERT_BEFORE(reg, r2, link); + TAILQ_INSERT_AFTER(®->fobj->regions, reg, r2, fobj_link); + + pager_unlock(exceptions); +} +DECLARE_KEEP_PAGER(split_region); + +TEE_Result tee_pager_split_um_region(struct user_mode_ctx *uctx, vaddr_t va) +{ + struct vm_paged_region *reg = NULL; + struct vm_paged_region *r2 = NULL; + + if (va & SMALL_PAGE_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + TAILQ_FOREACH(reg, uctx->regions, link) { + if (va == reg->base || va == reg->base + reg->size) + return TEE_SUCCESS; + if (va > reg->base && va < reg->base + reg->size) { + size_t diff = va - reg->base; + + r2 = alloc_region(va, reg->size - diff); + if (!r2) + return TEE_ERROR_OUT_OF_MEMORY; + split_region(reg, r2, va); + return TEE_SUCCESS; + } + } + + return TEE_SUCCESS; +} + +static struct pgt ** +merge_region_with_next(struct vm_paged_region_head *regions, + struct vm_paged_region *reg, + struct vm_paged_region *r_next, struct pgt **pgt_array) +{ + uint32_t exceptions = pager_lock_check_stack(64); + struct pgt **old_pgt_array = reg->pgt_array; + + reg->pgt_array = pgt_array; + TAILQ_REMOVE(regions, r_next, link); + TAILQ_REMOVE(&r_next->fobj->regions, r_next, fobj_link); + + pager_unlock(exceptions); + return old_pgt_array; +} +DECLARE_KEEP_PAGER(merge_region_with_next); + +static struct pgt **alloc_merged_pgt_array(struct vm_paged_region *a, + struct vm_paged_region *a_next) +{ + size_t a_next_pgt_count = get_pgt_count(a_next->base, a_next->size); + size_t a_pgt_count = get_pgt_count(a->base, a->size); + size_t pgt_count = get_pgt_count(a->base, a->size + a_next->size); + struct pgt **pgt_array = NULL; + bool have_shared_pgt = false; + + have_shared_pgt = ((a->base + a->size) & ~CORE_MMU_PGDIR_MASK) == + (a_next->base & ~CORE_MMU_PGDIR_MASK); + + if (have_shared_pgt) + assert(pgt_count == a_pgt_count + a_next_pgt_count - 1); + else + assert(pgt_count == a_pgt_count + a_next_pgt_count); + + /* In case there's a shared pgt they must match */ + if (have_shared_pgt && + a->pgt_array[a_pgt_count - 1] != a_next->pgt_array[0]) + return NULL; + + pgt_array = calloc(sizeof(struct pgt *), pgt_count); + if (!pgt_array) + return NULL; + + /* + * Copy and merge the two pgt_arrays, note the special case + * where a pgt is shared. + */ + memcpy(pgt_array, a->pgt_array, a_pgt_count * sizeof(struct pgt *)); + if (have_shared_pgt) + memcpy(pgt_array + a_pgt_count, a_next->pgt_array + 1, + (a_next_pgt_count - 1) * sizeof(struct pgt *)); + else + memcpy(pgt_array + a_pgt_count, a_next->pgt_array, + a_next_pgt_count * sizeof(struct pgt *)); + + return pgt_array; +} + +void tee_pager_merge_um_region(struct user_mode_ctx *uctx, vaddr_t va, + size_t len) +{ + struct vm_paged_region *r_next = NULL; + struct vm_paged_region *reg = NULL; + struct pgt **pgt_array = NULL; + vaddr_t end_va = 0; + + if ((va | len) & SMALL_PAGE_MASK) + return; + if (ADD_OVERFLOW(va, len, &end_va)) + return; + + for (reg = TAILQ_FIRST(uctx->regions);; reg = r_next) { + r_next = TAILQ_NEXT(reg, link); + if (!r_next) + return; + + /* Try merging with the area just before va */ + if (reg->base + reg->size < va) + continue; + + /* + * If reg->base is well past our range we're done. + * Note that if it's just the page after our range we'll + * try to merge. + */ + if (reg->base > end_va) + return; + + if (reg->base + reg->size != r_next->base) + continue; + if (reg->fobj != r_next->fobj || reg->type != r_next->type || + reg->flags != r_next->flags) + continue; + if (reg->fobj_pgoffs + reg->size / SMALL_PAGE_SIZE != + r_next->fobj_pgoffs) + continue; + + pgt_array = alloc_merged_pgt_array(reg, r_next); + if (!pgt_array) + continue; + + /* + * merge_region_with_next() returns the old pgt array which + * was replaced in reg. We don't want to call free() + * directly from merge_region_with_next() that would pull + * free() and its dependencies into the unpaged area. + */ + free(merge_region_with_next(uctx->regions, reg, r_next, + pgt_array)); + free_region(r_next); + r_next = reg; + } +} + +static void rem_region(struct vm_paged_region_head *regions, + struct vm_paged_region *reg) +{ + struct tee_pager_pmem *pmem; + size_t last_pgoffs = reg->fobj_pgoffs + + (reg->size >> SMALL_PAGE_SHIFT) - 1; + uint32_t exceptions; + struct tblidx tblidx = { }; + uint32_t a = 0; + + exceptions = pager_lock_check_stack(64); + + TAILQ_REMOVE(regions, reg, link); + TAILQ_REMOVE(®->fobj->regions, reg, fobj_link); + + TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { + if (pmem->fobj != reg->fobj || + pmem->fobj_pgidx < reg->fobj_pgoffs || + pmem->fobj_pgidx > last_pgoffs) + continue; + + tblidx = pmem_get_region_tblidx(pmem, reg); + tblidx_get_entry(tblidx, NULL, &a); + if (!(a & TEE_MATTR_VALID_BLOCK)) + continue; + + tblidx_set_entry(tblidx, 0, 0); + tblidx_tlbi_entry(tblidx); + pgt_dec_used_entries(tblidx.pgt); + } + + pager_unlock(exceptions); +} +DECLARE_KEEP_PAGER(rem_region); + +void tee_pager_rem_um_region(struct user_mode_ctx *uctx, vaddr_t base, + size_t size) +{ + struct vm_paged_region *reg; + struct vm_paged_region *r_next; + size_t s = ROUNDUP(size, SMALL_PAGE_SIZE); + + TAILQ_FOREACH_SAFE(reg, uctx->regions, link, r_next) { + if (core_is_buffer_inside(reg->base, reg->size, base, s)) { + rem_region(uctx->regions, reg); + free_region(reg); + } + } + tlbi_asid(uctx->vm_info.asid); +} + +void tee_pager_rem_um_regions(struct user_mode_ctx *uctx) +{ + struct vm_paged_region *reg = NULL; + + if (!uctx->regions) + return; + + while (true) { + reg = TAILQ_FIRST(uctx->regions); + if (!reg) + break; + unlink_region(uctx->regions, reg); + free_region(reg); + } + + free(uctx->regions); +} + +static bool __maybe_unused same_context(struct tee_pager_pmem *pmem) +{ + struct vm_paged_region *reg = TAILQ_FIRST(&pmem->fobj->regions); + void *ctx = reg->pgt_array[0]->ctx; + + do { + reg = TAILQ_NEXT(reg, fobj_link); + if (!reg) + return true; + } while (reg->pgt_array[0]->ctx == ctx); + + return false; +} + +bool tee_pager_set_um_region_attr(struct user_mode_ctx *uctx, vaddr_t base, + size_t size, uint32_t flags) +{ + bool ret = false; + vaddr_t b = base; + size_t s = size; + size_t s2 = 0; + struct vm_paged_region *reg = find_region(uctx->regions, b); + uint32_t exceptions = 0; + struct tee_pager_pmem *pmem = NULL; + uint32_t a = 0; + uint32_t f = 0; + uint32_t mattr = 0; + uint32_t f2 = 0; + struct tblidx tblidx = { }; + + f = (flags & TEE_MATTR_URWX) | TEE_MATTR_UR | TEE_MATTR_PR; + if (f & TEE_MATTR_UW) + f |= TEE_MATTR_PW; + mattr = get_region_mattr(f); + + exceptions = pager_lock_check_stack(SMALL_PAGE_SIZE); + + while (s) { + if (!reg) { + ret = false; + goto out; + } + s2 = MIN(reg->size, s); + b += s2; + s -= s2; + + if (reg->flags == f) + goto next_region; + + TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { + if (!pmem_is_covered_by_region(pmem, reg)) + continue; + + tblidx = pmem_get_region_tblidx(pmem, reg); + tblidx_get_entry(tblidx, NULL, &a); + if (a == f) + continue; + tblidx_set_entry(tblidx, 0, 0); + tblidx_tlbi_entry(tblidx); + + pmem->flags &= ~PMEM_FLAG_HIDDEN; + if (pmem_is_dirty(pmem)) + f2 = mattr; + else + f2 = mattr & ~(TEE_MATTR_UW | TEE_MATTR_PW); + tblidx_set_entry(tblidx, get_pmem_pa(pmem), f2); + if (!(a & TEE_MATTR_VALID_BLOCK)) + pgt_inc_used_entries(tblidx.pgt); + /* + * Make sure the table update is visible before + * continuing. + */ + dsb_ishst(); + + /* + * Here's a problem if this page already is shared. + * We need do icache invalidate for each context + * in which it is shared. In practice this will + * never happen. + */ + if (flags & TEE_MATTR_UX) { + void *va = (void *)tblidx2va(tblidx); + + /* Assert that the pmem isn't shared. */ + assert(same_context(pmem)); + + dcache_clean_range_pou(va, SMALL_PAGE_SIZE); + icache_inv_user_range(va, SMALL_PAGE_SIZE); + } + } + + reg->flags = f; +next_region: + reg = TAILQ_NEXT(reg, link); + } + + ret = true; +out: + pager_unlock(exceptions); + return ret; +} + +DECLARE_KEEP_PAGER(tee_pager_set_um_region_attr); +#endif /*CFG_PAGED_USER_TA*/ + +void tee_pager_invalidate_fobj(struct fobj *fobj) +{ + struct tee_pager_pmem *pmem; + uint32_t exceptions; + + exceptions = pager_lock_check_stack(64); + + TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) + if (pmem->fobj == fobj) + pmem_clear(pmem); + + pager_unlock(exceptions); +} +DECLARE_KEEP_PAGER(tee_pager_invalidate_fobj); + +static struct tee_pager_pmem *pmem_find(struct vm_paged_region *reg, vaddr_t va) +{ + struct tee_pager_pmem *pmem = NULL; + size_t fobj_pgidx = 0; + + assert(va >= reg->base && va < (reg->base + reg->size)); + fobj_pgidx = (va - reg->base) / SMALL_PAGE_SIZE + reg->fobj_pgoffs; + + TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) + if (pmem->fobj == reg->fobj && pmem->fobj_pgidx == fobj_pgidx) + return pmem; + + return NULL; +} + +static bool tee_pager_unhide_page(struct vm_paged_region *reg, vaddr_t page_va) +{ + struct tblidx tblidx = region_va2tblidx(reg, page_va); + struct tee_pager_pmem *pmem = pmem_find(reg, page_va); + uint32_t a = get_region_mattr(reg->flags); + uint32_t attr = 0; + paddr_t pa = 0; + + if (!pmem) + return false; + + tblidx_get_entry(tblidx, NULL, &attr); + if (attr & TEE_MATTR_VALID_BLOCK) + return false; + + /* + * The page is hidden, or not not mapped yet. Unhide the page and + * move it to the tail. + * + * Since the page isn't mapped there doesn't exist a valid TLB entry + * for this address, so no TLB invalidation is required after setting + * the new entry. A DSB is needed though, to make the write visible. + * + * For user executable pages it's more complicated. Those pages can + * be shared between multiple TA mappings and thus populated by + * another TA. The reference manual states that: + * + * "instruction cache maintenance is required only after writing + * new data to a physical address that holds an instruction." + * + * So for hidden pages we would not need to invalidate i-cache, but + * for newly populated pages we do. Since we don't know which we + * have to assume the worst and always invalidate the i-cache. We + * don't need to clean the d-cache though, since that has already + * been done earlier. + * + * Additional bookkeeping to tell if the i-cache invalidation is + * needed or not is left as a future optimization. + */ + + /* If it's not a dirty block, then it should be read only. */ + if (!pmem_is_dirty(pmem)) + a &= ~(TEE_MATTR_PW | TEE_MATTR_UW); + + pa = get_pmem_pa(pmem); + pmem->flags &= ~PMEM_FLAG_HIDDEN; + if (reg->flags & TEE_MATTR_UX) { + void *va = (void *)tblidx2va(tblidx); + + /* Set a temporary read-only mapping */ + assert(!(a & (TEE_MATTR_UW | TEE_MATTR_PW))); + tblidx_set_entry(tblidx, pa, a & ~TEE_MATTR_UX); + dsb_ishst(); + + icache_inv_user_range(va, SMALL_PAGE_SIZE); + + /* Set the final mapping */ + tblidx_set_entry(tblidx, pa, a); + tblidx_tlbi_entry(tblidx); + } else { + tblidx_set_entry(tblidx, pa, a); + dsb_ishst(); + } + pgt_inc_used_entries(tblidx.pgt); + + TAILQ_REMOVE(&tee_pager_pmem_head, pmem, link); + TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); + incr_hidden_hits(); + return true; +} + +static void tee_pager_hide_pages(void) +{ + struct tee_pager_pmem *pmem = NULL; + size_t n = 0; + + TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { + if (n >= TEE_PAGER_NHIDE) + break; + n++; + + /* we cannot hide pages when pmem->fobj is not defined. */ + if (!pmem->fobj) + continue; + + if (pmem_is_hidden(pmem)) + continue; + + pmem->flags |= PMEM_FLAG_HIDDEN; + pmem_unmap(pmem, NULL); + } +} + +static unsigned int __maybe_unused +num_regions_with_pmem(struct tee_pager_pmem *pmem) +{ + struct vm_paged_region *reg = NULL; + unsigned int num_matches = 0; + + TAILQ_FOREACH(reg, &pmem->fobj->regions, fobj_link) + if (pmem_is_covered_by_region(pmem, reg)) + num_matches++; + + return num_matches; +} + +/* + * Find mapped pmem, hide and move to pageble pmem. + * Return false if page was not mapped, and true if page was mapped. + */ +static bool tee_pager_release_one_phys(struct vm_paged_region *reg, + vaddr_t page_va) +{ + struct tee_pager_pmem *pmem = NULL; + struct tblidx tblidx = { }; + size_t fobj_pgidx = 0; + + assert(page_va >= reg->base && page_va < (reg->base + reg->size)); + fobj_pgidx = (page_va - reg->base) / SMALL_PAGE_SIZE + + reg->fobj_pgoffs; + + TAILQ_FOREACH(pmem, &tee_pager_lock_pmem_head, link) { + if (pmem->fobj != reg->fobj || pmem->fobj_pgidx != fobj_pgidx) + continue; + + /* + * Locked pages may not be shared. We're asserting that the + * number of regions using this pmem is one and only one as + * we're about to unmap it. + */ + assert(num_regions_with_pmem(pmem) == 1); + + tblidx = pmem_get_region_tblidx(pmem, reg); + tblidx_set_entry(tblidx, 0, 0); + pgt_dec_used_entries(tblidx.pgt); + TAILQ_REMOVE(&tee_pager_lock_pmem_head, pmem, link); + pmem_clear(pmem); + tee_pager_npages++; + set_npages(); + TAILQ_INSERT_HEAD(&tee_pager_pmem_head, pmem, link); + incr_zi_released(); + return true; + } + + return false; +} + +static void pager_deploy_page(struct tee_pager_pmem *pmem, + struct vm_paged_region *reg, vaddr_t page_va, + bool clean_user_cache, bool writable) +{ + struct tblidx tblidx = region_va2tblidx(reg, page_va); + uint32_t attr = get_region_mattr(reg->flags); + struct core_mmu_table_info *ti = NULL; + uint8_t *va_alias = pmem->va_alias; + paddr_t pa = get_pmem_pa(pmem); + unsigned int idx_alias = 0; + uint32_t attr_alias = 0; + paddr_t pa_alias = 0; + + /* Ensure we are allowed to write to aliased virtual page */ + ti = find_table_info((vaddr_t)va_alias); + idx_alias = core_mmu_va2idx(ti, (vaddr_t)va_alias); + core_mmu_get_entry(ti, idx_alias, &pa_alias, &attr_alias); + if (!(attr_alias & TEE_MATTR_PW)) { + attr_alias |= TEE_MATTR_PW; + core_mmu_set_entry(ti, idx_alias, pa_alias, attr_alias); + tlbi_va_allasid((vaddr_t)va_alias); + } + + asan_tag_access(va_alias, va_alias + SMALL_PAGE_SIZE); + if (fobj_load_page(pmem->fobj, pmem->fobj_pgidx, va_alias)) { + EMSG("PH 0x%" PRIxVA " failed", page_va); + panic(); + } + switch (reg->type) { + case PAGED_REGION_TYPE_RO: + TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); + incr_ro_hits(); + /* Forbid write to aliases for read-only (maybe exec) pages */ + attr_alias &= ~TEE_MATTR_PW; + core_mmu_set_entry(ti, idx_alias, pa_alias, attr_alias); + tlbi_va_allasid((vaddr_t)va_alias); + break; + case PAGED_REGION_TYPE_RW: + TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); + if (writable && (attr & (TEE_MATTR_PW | TEE_MATTR_UW))) + pmem->flags |= PMEM_FLAG_DIRTY; + incr_rw_hits(); + break; + case PAGED_REGION_TYPE_LOCK: + /* Move page to lock list */ + if (tee_pager_npages <= 0) + panic("Running out of pages"); + tee_pager_npages--; + set_npages(); + TAILQ_INSERT_TAIL(&tee_pager_lock_pmem_head, pmem, link); + break; + default: + panic(); + } + asan_tag_no_access(va_alias, va_alias + SMALL_PAGE_SIZE); + + if (!writable) + attr &= ~(TEE_MATTR_PW | TEE_MATTR_UW); + + /* + * We've updated the page using the aliased mapping and + * some cache maintenance is now needed if it's an + * executable page. + * + * Since the d-cache is a Physically-indexed, + * physically-tagged (PIPT) cache we can clean either the + * aliased address or the real virtual address. In this + * case we choose the real virtual address. + * + * The i-cache can also be PIPT, but may be something else + * too like VIPT. The current code requires the caches to + * implement the IVIPT extension, that is: + * "instruction cache maintenance is required only after + * writing new data to a physical address that holds an + * instruction." + * + * To portably invalidate the icache the page has to + * be mapped at the final virtual address but not + * executable. + */ + if (reg->flags & (TEE_MATTR_PX | TEE_MATTR_UX)) { + uint32_t mask = TEE_MATTR_PX | TEE_MATTR_UX | + TEE_MATTR_PW | TEE_MATTR_UW; + void *va = (void *)page_va; + + /* Set a temporary read-only mapping */ + tblidx_set_entry(tblidx, pa, attr & ~mask); + tblidx_tlbi_entry(tblidx); + + dcache_clean_range_pou(va, SMALL_PAGE_SIZE); + if (clean_user_cache) + icache_inv_user_range(va, SMALL_PAGE_SIZE); + else + icache_inv_range(va, SMALL_PAGE_SIZE); + + /* Set the final mapping */ + tblidx_set_entry(tblidx, pa, attr); + tblidx_tlbi_entry(tblidx); + } else { + tblidx_set_entry(tblidx, pa, attr); + /* + * No need to flush TLB for this entry, it was + * invalid. We should use a barrier though, to make + * sure that the change is visible. + */ + dsb_ishst(); + } + pgt_inc_used_entries(tblidx.pgt); + + FMSG("Mapped 0x%" PRIxVA " -> 0x%" PRIxPA, page_va, pa); +} + +static void make_dirty_page(struct tee_pager_pmem *pmem, + struct vm_paged_region *reg, struct tblidx tblidx, + paddr_t pa) +{ + assert(reg->flags & (TEE_MATTR_UW | TEE_MATTR_PW)); + assert(!(pmem->flags & PMEM_FLAG_DIRTY)); + + FMSG("Dirty %#"PRIxVA, tblidx2va(tblidx)); + pmem->flags |= PMEM_FLAG_DIRTY; + tblidx_set_entry(tblidx, pa, get_region_mattr(reg->flags)); + tblidx_tlbi_entry(tblidx); +} + +/* + * This function takes a reference to a page (@fobj + fobj_pgidx) and makes + * the corresponding IV available. + * + * In case the page needs to be saved the IV must be writable, consequently + * is the page holding the IV made dirty. If the page instead only is to + * be verified it's enough that the page holding the IV is readonly and + * thus doesn't have to be made dirty too. + * + * This function depends on pager_spare_pmem pointing to a free pmem when + * entered. In case the page holding the needed IV isn't mapped this spare + * pmem is used to map the page. If this function has used pager_spare_pmem + * and assigned it to NULL it must be reassigned with a new free pmem + * before this function can be called again. + */ +static void make_iv_available(struct fobj *fobj, unsigned int fobj_pgidx, + bool writable) +{ + struct vm_paged_region *reg = pager_iv_region; + struct tee_pager_pmem *pmem = NULL; + struct tblidx tblidx = { }; + vaddr_t page_va = 0; + uint32_t attr = 0; + paddr_t pa = 0; + + page_va = fobj_get_iv_vaddr(fobj, fobj_pgidx) & ~SMALL_PAGE_MASK; + if (!IS_ENABLED(CFG_CORE_PAGE_TAG_AND_IV) || !page_va) { + assert(!page_va); + return; + } + + assert(reg && reg->type == PAGED_REGION_TYPE_RW); + assert(pager_spare_pmem); + assert(core_is_buffer_inside(page_va, 1, reg->base, reg->size)); + + tblidx = region_va2tblidx(reg, page_va); + /* + * We don't care if tee_pager_unhide_page() succeeds or not, we're + * still checking the attributes afterwards. + */ + tee_pager_unhide_page(reg, page_va); + tblidx_get_entry(tblidx, &pa, &attr); + if (!(attr & TEE_MATTR_VALID_BLOCK)) { + /* + * We're using the spare pmem to map the IV corresponding + * to another page. + */ + pmem = pager_spare_pmem; + pager_spare_pmem = NULL; + pmem_assign_fobj_page(pmem, reg, page_va); + + if (writable) + pmem->flags |= PMEM_FLAG_DIRTY; + + pager_deploy_page(pmem, reg, page_va, + false /*!clean_user_cache*/, writable); + } else if (writable && !(attr & TEE_MATTR_PW)) { + pmem = pmem_find(reg, page_va); + /* Note that pa is valid since TEE_MATTR_VALID_BLOCK is set */ + make_dirty_page(pmem, reg, tblidx, pa); + } +} + +static void pager_get_page(struct vm_paged_region *reg, struct abort_info *ai, + bool clean_user_cache) +{ + vaddr_t page_va = ai->va & ~SMALL_PAGE_MASK; + struct tblidx tblidx = region_va2tblidx(reg, page_va); + struct tee_pager_pmem *pmem = NULL; + bool writable = false; + uint32_t attr = 0; + + /* + * Get a pmem to load code and data into, also make sure + * the corresponding IV page is available. + */ + while (true) { + pmem = TAILQ_FIRST(&tee_pager_pmem_head); + if (!pmem) { + EMSG("No pmem entries"); + abort_print(ai); + panic(); + } + + if (pmem->fobj) { + pmem_unmap(pmem, NULL); + if (pmem_is_dirty(pmem)) { + uint8_t *va = pmem->va_alias; + + make_iv_available(pmem->fobj, pmem->fobj_pgidx, + true /*writable*/); + asan_tag_access(va, va + SMALL_PAGE_SIZE); + if (fobj_save_page(pmem->fobj, pmem->fobj_pgidx, + pmem->va_alias)) + panic("fobj_save_page"); + asan_tag_no_access(va, va + SMALL_PAGE_SIZE); + + pmem_clear(pmem); + + /* + * If the spare pmem was used by + * make_iv_available() we need to replace + * it with the just freed pmem. + * + * See make_iv_available() for details. + */ + if (IS_ENABLED(CFG_CORE_PAGE_TAG_AND_IV) && + !pager_spare_pmem) { + TAILQ_REMOVE(&tee_pager_pmem_head, + pmem, link); + pager_spare_pmem = pmem; + pmem = NULL; + } + + /* + * Check if the needed virtual page was + * made available as a side effect of the + * call to make_iv_available() above. If so + * we're done. + */ + tblidx_get_entry(tblidx, NULL, &attr); + if (attr & TEE_MATTR_VALID_BLOCK) + return; + + /* + * The freed pmem was used to replace the + * consumed pager_spare_pmem above. Restart + * to find another pmem. + */ + if (!pmem) + continue; + } + } + + TAILQ_REMOVE(&tee_pager_pmem_head, pmem, link); + pmem_clear(pmem); + + pmem_assign_fobj_page(pmem, reg, page_va); + make_iv_available(pmem->fobj, pmem->fobj_pgidx, + false /*!writable*/); + if (!IS_ENABLED(CFG_CORE_PAGE_TAG_AND_IV) || pager_spare_pmem) + break; + + /* + * The spare pmem was used by make_iv_available(). We need + * to replace it with the just freed pmem. And get another + * pmem. + * + * See make_iv_available() for details. + */ + pmem_clear(pmem); + pager_spare_pmem = pmem; + } + + /* + * PAGED_REGION_TYPE_LOCK are always writable while PAGED_REGION_TYPE_RO + * are never writable. + * + * Pages from PAGED_REGION_TYPE_RW starts read-only to be + * able to tell when they are updated and should be tagged + * as dirty. + */ + if (reg->type == PAGED_REGION_TYPE_LOCK || + (reg->type == PAGED_REGION_TYPE_RW && abort_is_write_fault(ai))) + writable = true; + else + writable = false; + + pager_deploy_page(pmem, reg, page_va, clean_user_cache, writable); +} + +static bool pager_update_permissions(struct vm_paged_region *reg, + struct abort_info *ai, bool *handled) +{ + struct tblidx tblidx = region_va2tblidx(reg, ai->va); + struct tee_pager_pmem *pmem = NULL; + uint32_t attr = 0; + paddr_t pa = 0; + + *handled = false; + + tblidx_get_entry(tblidx, &pa, &attr); + + /* Not mapped */ + if (!(attr & TEE_MATTR_VALID_BLOCK)) + return false; + + /* Not readable, should not happen */ + if (abort_is_user_exception(ai)) { + if (!(attr & TEE_MATTR_UR)) + return true; + } else { + if (!(attr & TEE_MATTR_PR)) { + abort_print_error(ai); + panic(); + } + } + + switch (core_mmu_get_fault_type(ai->fault_descr)) { + case CORE_MMU_FAULT_TRANSLATION: + case CORE_MMU_FAULT_READ_PERMISSION: + if (ai->abort_type == ABORT_TYPE_PREFETCH) { + /* Check attempting to execute from an NOX page */ + if (abort_is_user_exception(ai)) { + if (!(attr & TEE_MATTR_UX)) + return true; + } else { + if (!(attr & TEE_MATTR_PX)) { + abort_print_error(ai); + panic(); + } + } + } + /* Since the page is mapped now it's OK */ + break; + case CORE_MMU_FAULT_WRITE_PERMISSION: + /* Check attempting to write to an RO page */ + pmem = pmem_find(reg, ai->va); + if (!pmem) + panic(); + if (abort_is_user_exception(ai)) { + if (!(reg->flags & TEE_MATTR_UW)) + return true; + if (!(attr & TEE_MATTR_UW)) + make_dirty_page(pmem, reg, tblidx, pa); + } else { + if (!(reg->flags & TEE_MATTR_PW)) { + abort_print_error(ai); + panic(); + } + if (!(attr & TEE_MATTR_PW)) + make_dirty_page(pmem, reg, tblidx, pa); + } + /* Since permissions has been updated now it's OK */ + break; + default: + /* Some fault we can't deal with */ + if (abort_is_user_exception(ai)) + return true; + abort_print_error(ai); + panic(); + } + *handled = true; + return true; +} + +#ifdef CFG_TEE_CORE_DEBUG +static void stat_handle_fault(void) +{ + static size_t num_faults; + static size_t min_npages = SIZE_MAX; + static size_t total_min_npages = SIZE_MAX; + + num_faults++; + if ((num_faults % 1024) == 0 || tee_pager_npages < total_min_npages) { + DMSG("nfaults %zu npages %zu (min %zu)", + num_faults, tee_pager_npages, min_npages); + min_npages = tee_pager_npages; /* reset */ + } + if (tee_pager_npages < min_npages) + min_npages = tee_pager_npages; + if (tee_pager_npages < total_min_npages) + total_min_npages = tee_pager_npages; +} +#else +static void stat_handle_fault(void) +{ +} +#endif + +bool tee_pager_handle_fault(struct abort_info *ai) +{ + struct vm_paged_region *reg; + vaddr_t page_va = ai->va & ~SMALL_PAGE_MASK; + uint32_t exceptions; + bool ret; + bool clean_user_cache = false; + +#ifdef TEE_PAGER_DEBUG_PRINT + if (!abort_is_user_exception(ai)) + abort_print(ai); +#endif + + /* + * We're updating pages that can affect several active CPUs at a + * time below. We end up here because a thread tries to access some + * memory that isn't available. We have to be careful when making + * that memory available as other threads may succeed in accessing + * that address the moment after we've made it available. + * + * That means that we can't just map the memory and populate the + * page, instead we use the aliased mapping to populate the page + * and once everything is ready we map it. + */ + exceptions = pager_lock(ai); + + stat_handle_fault(); + + /* check if the access is valid */ + if (abort_is_user_exception(ai)) { + reg = find_uta_region(ai->va); + clean_user_cache = true; + } else { + reg = find_region(&core_vm_regions, ai->va); + if (!reg) { + reg = find_uta_region(ai->va); + clean_user_cache = true; + } + } + if (!reg || !reg->pgt_array[0]) { + ret = false; + goto out; + } + + if (tee_pager_unhide_page(reg, page_va)) + goto out_success; + + /* + * The page wasn't hidden, but some other core may have + * updated the table entry before we got here or we need + * to make a read-only page read-write (dirty). + */ + if (pager_update_permissions(reg, ai, &ret)) { + /* + * Nothing more to do with the abort. The problem + * could already have been dealt with from another + * core or if ret is false the TA will be paniced. + */ + goto out; + } + + pager_get_page(reg, ai, clean_user_cache); + +out_success: + tee_pager_hide_pages(); + ret = true; +out: + pager_unlock(exceptions); + return ret; +} + +void tee_pager_add_pages(vaddr_t vaddr, size_t npages, bool unmap) +{ + size_t n = 0; + + DMSG("0x%" PRIxVA " - 0x%" PRIxVA " : %d", + vaddr, vaddr + npages * SMALL_PAGE_SIZE, (int)unmap); + + /* setup memory */ + for (n = 0; n < npages; n++) { + struct core_mmu_table_info *ti = NULL; + struct tee_pager_pmem *pmem = NULL; + vaddr_t va = vaddr + n * SMALL_PAGE_SIZE; + struct tblidx tblidx = { }; + unsigned int pgidx = 0; + paddr_t pa = 0; + uint32_t attr = 0; + + ti = find_table_info(va); + pgidx = core_mmu_va2idx(ti, va); + /* + * Note that we can only support adding pages in the + * valid range of this table info, currently not a problem. + */ + core_mmu_get_entry(ti, pgidx, &pa, &attr); + + /* Ignore unmapped pages/blocks */ + if (!(attr & TEE_MATTR_VALID_BLOCK)) + continue; + + pmem = calloc(1, sizeof(struct tee_pager_pmem)); + if (!pmem) + panic("out of mem"); + pmem_clear(pmem); + + pmem->va_alias = pager_add_alias_page(pa); + + if (unmap) { + core_mmu_set_entry(ti, pgidx, 0, 0); + pgt_dec_used_entries(find_core_pgt(va)); + } else { + struct vm_paged_region *reg = NULL; + + /* + * The page is still mapped, let's assign the region + * and update the protection bits accordingly. + */ + reg = find_region(&core_vm_regions, va); + assert(reg); + pmem_assign_fobj_page(pmem, reg, va); + tblidx = pmem_get_region_tblidx(pmem, reg); + assert(tblidx.pgt == find_core_pgt(va)); + assert(pa == get_pmem_pa(pmem)); + tblidx_set_entry(tblidx, pa, + get_region_mattr(reg->flags)); + } + + if (unmap && IS_ENABLED(CFG_CORE_PAGE_TAG_AND_IV) && + !pager_spare_pmem) { + pager_spare_pmem = pmem; + } else { + tee_pager_npages++; + incr_npages_all(); + set_npages(); + TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); + } + } + + /* + * As this is done at inits, invalidate all TLBs once instead of + * targeting only the modified entries. + */ + tlbi_all(); +} + +#ifdef CFG_PAGED_USER_TA +static struct pgt *find_pgt(struct pgt *pgt, vaddr_t va) +{ + struct pgt *p = pgt; + + while (p && (va & ~CORE_MMU_PGDIR_MASK) != p->vabase) + p = SLIST_NEXT(p, link); + return p; +} + +void tee_pager_assign_um_tables(struct user_mode_ctx *uctx) +{ + struct vm_paged_region *reg = NULL; + struct pgt *pgt = NULL; + size_t n = 0; + + if (!uctx->regions) + return; + + pgt = SLIST_FIRST(&uctx->pgt_cache); + TAILQ_FOREACH(reg, uctx->regions, link) { + for (n = 0; n < get_pgt_count(reg->base, reg->size); n++) { + vaddr_t va = reg->base + CORE_MMU_PGDIR_SIZE * n; + struct pgt *p __maybe_unused = find_pgt(pgt, va); + + if (!reg->pgt_array[n]) + reg->pgt_array[n] = p; + else + assert(reg->pgt_array[n] == p); + } + } +} + +void tee_pager_pgt_save_and_release_entries(struct pgt *pgt) +{ + struct tee_pager_pmem *pmem = NULL; + struct vm_paged_region *reg = NULL; + struct vm_paged_region_head *regions = NULL; + uint32_t exceptions = pager_lock_check_stack(SMALL_PAGE_SIZE); + size_t n = 0; + + if (!pgt->num_used_entries) + goto out; + + TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { + if (pmem->fobj) + pmem_unmap(pmem, pgt); + } + assert(!pgt->num_used_entries); + +out: + regions = to_user_mode_ctx(pgt->ctx)->regions; + if (regions) { + TAILQ_FOREACH(reg, regions, link) { + for (n = 0; n < get_pgt_count(reg->base, reg->size); + n++) { + if (reg->pgt_array[n] == pgt) { + reg->pgt_array[n] = NULL; + break; + } + } + } + } + + pager_unlock(exceptions); +} +DECLARE_KEEP_PAGER(tee_pager_pgt_save_and_release_entries); +#endif /*CFG_PAGED_USER_TA*/ + +void tee_pager_release_phys(void *addr, size_t size) +{ + bool unmaped = false; + vaddr_t va = (vaddr_t)addr; + vaddr_t begin = ROUNDUP(va, SMALL_PAGE_SIZE); + vaddr_t end = ROUNDDOWN(va + size, SMALL_PAGE_SIZE); + struct vm_paged_region *reg; + uint32_t exceptions; + + if (end <= begin) + return; + + exceptions = pager_lock_check_stack(128); + + for (va = begin; va < end; va += SMALL_PAGE_SIZE) { + reg = find_region(&core_vm_regions, va); + if (!reg) + panic(); + unmaped |= tee_pager_release_one_phys(reg, va); + } + + if (unmaped) + tlbi_va_range(begin, end - begin, SMALL_PAGE_SIZE); + + pager_unlock(exceptions); +} +DECLARE_KEEP_PAGER(tee_pager_release_phys); + +void *tee_pager_alloc(size_t size) +{ + tee_mm_entry_t *mm = NULL; + uint8_t *smem = NULL; + size_t num_pages = 0; + struct fobj *fobj = NULL; + + if (!size) + return NULL; + + mm = tee_mm_alloc(&tee_mm_vcore, ROUNDUP(size, SMALL_PAGE_SIZE)); + if (!mm) + return NULL; + + smem = (uint8_t *)tee_mm_get_smem(mm); + num_pages = tee_mm_get_bytes(mm) / SMALL_PAGE_SIZE; + fobj = fobj_locked_paged_alloc(num_pages); + if (!fobj) { + tee_mm_free(mm); + return NULL; + } + + tee_pager_add_core_region((vaddr_t)smem, PAGED_REGION_TYPE_LOCK, fobj); + fobj_put(fobj); + + asan_tag_access(smem, smem + num_pages * SMALL_PAGE_SIZE); + + return smem; +} + +vaddr_t tee_pager_init_iv_region(struct fobj *fobj) +{ + tee_mm_entry_t *mm = NULL; + uint8_t *smem = NULL; + + assert(!pager_iv_region); + + mm = tee_mm_alloc(&tee_mm_vcore, fobj->num_pages * SMALL_PAGE_SIZE); + if (!mm) + panic(); + + smem = (uint8_t *)tee_mm_get_smem(mm); + tee_pager_add_core_region((vaddr_t)smem, PAGED_REGION_TYPE_RW, fobj); + fobj_put(fobj); + + asan_tag_access(smem, smem + fobj->num_pages * SMALL_PAGE_SIZE); + + pager_iv_region = find_region(&core_vm_regions, (vaddr_t)smem); + assert(pager_iv_region && pager_iv_region->fobj == fobj); + + return (vaddr_t)smem; +} diff --git a/optee_os/core/arch/arm/plat-amlogic/conf.mk b/optee_os/core/arch/arm/plat-amlogic/conf.mk new file mode 100644 index 0000000..a28779c --- /dev/null +++ b/optee_os/core/arch/arm/plat-amlogic/conf.mk @@ -0,0 +1,17 @@ +PLATFORM_FLAVOR ?= axg + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_TEE_CORE_NB_CORE,4) + +CFG_TZDRAM_START ?= 0x05300000 +CFG_TZDRAM_SIZE ?= 0x00c00000 +CFG_SHMEM_START ?= 0x05000000 +CFG_SHMEM_SIZE ?= 0x00100000 + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_AMLOGIC_UART,y) + +$(call force,CFG_WITH_PAGER,n) +$(call force,CFG_ARM64_core,y) diff --git a/optee_os/core/arch/arm/plat-amlogic/link.mk b/optee_os/core/arch/arm/plat-amlogic/link.mk new file mode 100644 index 0000000..704996b --- /dev/null +++ b/optee_os/core/arch/arm/plat-amlogic/link.mk @@ -0,0 +1,17 @@ +include core/arch/arm/kernel/link.mk + +# Create BL32 image from the native binary images + +define aml_bin2img_cmd + @$(cmd-echo-silent) ' GEN $@' + $(q)./core/arch/arm/plat-amlogic/scripts/aml_bin2img.py +endef + +ifneq (,$(filter $(PLATFORM_FLAVOR),axg)) +all: $(link-out-dir)/bl32.img +cleanfiles += $(link-out-dir)/bl32.img +$(link-out-dir)/bl32.img: $(link-out-dir)/tee-pager_v2.bin + $(aml_bin2img_cmd) --source $< --dest $@ --entry 0x5300000 \ + --res_mem_start 0x5300000 --res_mem_size 0x1000000 \ + --sec_mem_start 0x5300000 --sec_mem_size 0xc00000 +endif diff --git a/optee_os/core/arch/arm/plat-amlogic/main.c b/optee_os/core/arch/arm/plat-amlogic/main.c new file mode 100644 index 0000000..fd5ede4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-amlogic/main.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020 Carlo Caione + */ + +#include +#include +#include +#include +#include + +static struct amlogic_uart_data console_data; +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, + CORE_MMU_PGDIR_SIZE); + +void console_init(void) +{ + amlogic_uart_init(&console_data, CONSOLE_UART_BASE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-amlogic/platform_config.h b/optee_os/core/arch/arm/plat-amlogic/platform_config.h new file mode 100644 index 0000000..a791e46 --- /dev/null +++ b/optee_os/core/arch/arm/plat-amlogic/platform_config.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020 Carlo Caione + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#define GIC_BASE 0xFFC01000 +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 + +#define CONSOLE_UART_BASE 0xFF803000 + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-amlogic/scripts/aml_bin2img.py b/optee_os/core/arch/arm/plat-amlogic/scripts/aml_bin2img.py new file mode 100755 index 0000000..28a037d --- /dev/null +++ b/optee_os/core/arch/arm/plat-amlogic/scripts/aml_bin2img.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2020 Carlo Caione +# +# Derived from plat-stm32mp1/scripts/stm32image.py +# + +import argparse +import struct +import mmap + +header_size = 0x200 +ext_magic_number = 0x12348765 +version = 0x00002710 + + +def get_size(file): + file.seek(0, 2) # End of the file + size = file.tell() + return size + + +def aml_set_header(dest_fd, entry, res_mem_start, res_mem_size, sec_mem_start, + sec_mem_size): + dest_fd.seek(0, 0) + + dest_fd.write(struct.pack(' 0: + mmsrc = mmap.mmap(src_fd.fileno(), 0, access=mmap.ACCESS_READ) + dest_fd.write(mmsrc[:sizesrc]) + mmsrc.close() + + src_fd.close() + + aml_set_header(dest_fd, entry, res_mem_start, res_mem_size, sec_mem_start, + sec_mem_size) + + dest_fd.close() + + +def auto_int(x): + return int(x, 0) + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument('--source', + required=True, + help='Source file') + + parser.add_argument('--dest', + required=True, + help='Destination file') + + parser.add_argument('--entry', + required=True, + type=auto_int, + help='Entry point') + + parser.add_argument('--res_mem_start', + required=True, + type=auto_int, + help='Reserved memory start') + + parser.add_argument('--res_mem_size', + required=True, + type=auto_int, + help='Reserved memory size') + + parser.add_argument('--sec_mem_start', + required=True, + type=auto_int, + help='Secure memory start') + + parser.add_argument('--sec_mem_size', + required=True, + type=auto_int, + help='Secure memory size') + + return parser.parse_args() + + +def main(): + args = get_args() + + source_file = args.source + destination_file = args.dest + entry_point = args.entry + res_mem_start = args.res_mem_start + res_mem_size = args.res_mem_size + sec_mem_start = args.sec_mem_start + sec_mem_size = args.sec_mem_size + + aml_create_header_file(source_file, + destination_file, + entry_point, + res_mem_start, + res_mem_size, + sec_mem_start, + sec_mem_size) + + +if __name__ == "__main__": + main() diff --git a/optee_os/core/arch/arm/plat-amlogic/sub.mk b/optee_os/core/arch/arm/plat-amlogic/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-amlogic/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-aspeed/conf.mk b/optee_os/core/arch/arm/plat-aspeed/conf.mk new file mode 100644 index 0000000..a45ccca --- /dev/null +++ b/optee_os/core/arch/arm/plat-aspeed/conf.mk @@ -0,0 +1,47 @@ +PLATFORM_FLAVOR ?= ast2600 + +ifeq ($(PLATFORM_FLAVOR),ast2600) +include core/arch/arm/cpu/cortex-a7.mk + +$(call force,CFG_8250_UART,y) +$(call force,CFG_ARM32_core,y) +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_GIC,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) + +CFG_NUM_THREADS ?= $(CFG_TEE_CORE_NB_CORE) + +CFG_DRAM_BASE ?= 0x80000000 +CFG_DRAM_SIZE ?= 0x40000000 + +CFG_TZDRAM_START ?= 0xb0000000 +CFG_TZDRAM_SIZE ?= 0x1000000 + +CFG_CORE_RESERVED_SHM ?= n +else ifeq ($(PLATFORM_FLAVOR),ast2700) +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_8250_UART,y) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_GIC,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) + +CFG_NUM_THREADS ?= $(CFG_TEE_CORE_NB_CORE) + +CFG_DRAM_BASE ?= 0x400000000 +CFG_DRAM_SIZE ?= 0x40000000 + +CFG_TZDRAM_START ?= 0x430080000 +CFG_TZDRAM_SIZE ?= 0x1000000 + +CFG_CORE_RESERVED_SHM ?= n + +supported-ta-targets = ta_arm64 +else +$(error Unsupported PLATFORM_FLAVOR "$(PLATFORM_FLAVOR)") +endif diff --git a/optee_os/core/arch/arm/plat-aspeed/core_pos_a32.S b/optee_os/core/arch/arm/plat-aspeed/core_pos_a32.S new file mode 100644 index 0000000..35932d6 --- /dev/null +++ b/optee_os/core/arch/arm/plat-aspeed/core_pos_a32.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Aspeed Technology Inc. + */ + +#include +#include +#include + +FUNC get_core_pos_mpidr , : + /* + * need this to correct core0 - 0xf00, core1 - 0xf01, ... + */ + and r0, r0, #MPIDR_CPU_MASK + bx lr +END_FUNC get_core_pos_mpidr diff --git a/optee_os/core/arch/arm/plat-aspeed/platform_ast2600.c b/optee_os/core/arch/arm/plat-aspeed/platform_ast2600.c new file mode 100644 index 0000000..f8b359c --- /dev/null +++ b/optee_os/core/arch/arm/plat-aspeed/platform_ast2600.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Aspeed Technology Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum TZM_PERM { + TZM_PERM_VGA_CURSOR_RD, + TZM_PERM_VGA_CRT_RD, + TZM_PERM_SOC_DISPLAY_RD, + TZM_PERM_PCIE_BUS1_RW, + TZM_PERM_VIDEO_HIGH_WR, + TZM_PERM_CPU_RW, + TZM_PERM_SLI_RW, + TZM_PERM_PCIE_BUS2_RW, + TZM_PERM_USB20_HUB_EHCI1_DMA_RW, + TZM_PERM_USB20_DEV_EHCI2_DMA_RW, + TZM_PERM_USB11_UCHI_HOST_RW, + TZM_PERM_AHB_RW, + TZM_PERM_CM3_DATA_RW, + TZM_PERM_CM3_INSN_RW, + TZM_PERM_MAC0_DMA_RW, + TZM_PERM_MAC1_DMA_RW, + TZM_PERM_SDIO_DMA_RW, + TZM_PERM_PILOT_RW, + TZM_PERM_XDMA1_RW, + TZM_PERM_MCTP1_RW, + TZM_PERM_VIDEO_FLAG_RW, + TZM_PERM_VIDEO_LOW_WR, + TZM_PERM_2D_DATA_RW, + TZM_PERM_ENCRYPT_RW, + TZM_PERM_MCTP2_RW, + TZM_PERM_XDMA2_RW, + TZM_PERM_ECC_RSA_RW, +}; + +register_phys_mem(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, SMALL_PAGE_SIZE); +register_phys_mem(MEM_AREA_IO_SEC, GIC_BASE + GICD_OFFSET, GIC_DIST_REG_SIZE); +register_phys_mem(MEM_AREA_IO_SEC, GIC_BASE + GICC_OFFSET, GIC_CPU_REG_SIZE); +register_phys_mem(MEM_AREA_IO_SEC, AHBC_BASE, SMALL_PAGE_SIZE); +register_phys_mem(MEM_AREA_IO_NSEC, SCU_BASE, SMALL_PAGE_SIZE); + +#define AHBC_REG_WR_PROT 0x204 +#define AHBC_TZP_ACCESS1 0x280 +#define AHBC_TZP_HACE BIT(20) +#define AHBC_TZM_ST(i) (0x300 + ((i) * 0x10)) +#define AHBC_TZM_ED(i) (0x304 + ((i) * 0x10)) +#define AHBC_TZM_PERM(i) (0x308 + ((i) * 0x10)) + +register_ddr(CFG_DRAM_BASE, CFG_DRAM_SIZE); + +static struct serial8250_uart_data console_data; + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +void console_init(void) +{ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +void plat_primary_init_early(void) +{ + vaddr_t ahbc_virt = 0; + uint32_t tzm_perm = 0; + + ahbc_virt = core_mmu_get_va(AHBC_BASE, + MEM_AREA_IO_SEC, SMALL_PAGE_SIZE); + if (!ahbc_virt) + panic(); + + tzm_perm = BIT(TZM_PERM_CPU_RW); + if (IS_ENABLED(CFG_ASPEED_CRYPTO_DRIVER)) { + tzm_perm |= BIT(TZM_PERM_ENCRYPT_RW); + io_write32(ahbc_virt + AHBC_TZP_ACCESS1, AHBC_TZP_HACE); + } + + io_write32(ahbc_virt + AHBC_TZM_PERM(0), tzm_perm); + io_write32(ahbc_virt + AHBC_TZM_ED(0), + CFG_TZDRAM_START + CFG_TZDRAM_SIZE - 1); + io_write32(ahbc_virt + AHBC_TZM_ST(0), + CFG_TZDRAM_START | BIT(0)); + io_write32(ahbc_virt + AHBC_REG_WR_PROT, BIT(16)); +} diff --git a/optee_os/core/arch/arm/plat-aspeed/platform_ast2700.c b/optee_os/core/arch/arm/plat-aspeed/platform_ast2700.c new file mode 100644 index 0000000..a1b3ac4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-aspeed/platform_ast2700.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023, Aspeed Technology Inc. + */ + +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem(MEM_AREA_IO_SEC, UART_BASE, SMALL_PAGE_SIZE); +register_phys_mem(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); + +register_ddr(CFG_DRAM_BASE, CFG_DRAM_SIZE); + +static struct serial8250_uart_data console_data; + +void boot_primary_init_intc(void) +{ + gic_init(0, GICD_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +void console_init(void) +{ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-aspeed/platform_config.h b/optee_os/core/arch/arm/plat-aspeed/platform_config.h new file mode 100644 index 0000000..69f2679 --- /dev/null +++ b/optee_os/core/arch/arm/plat-aspeed/platform_config.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Aspeed Technology Inc. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#if defined(PLATFORM_FLAVOR_ast2600) +#define GIC_BASE 0x40460000 +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 + +#define AHBC_BASE 0x1e600000 +#define SCU_BASE 0x1e6e2000 +#define UART5_BASE 0x1e784000 + +#define CONSOLE_UART_BASE UART5_BASE +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 1846153 +#elif defined(PLATFORM_FLAVOR_ast2700) +#define GICD_BASE 0x12200000 +#define GICR_BASE 0x12280000 + +#define UART_BASE 0x14c33000 +#define UART12_BASE (UART_BASE + 0xb00) + +#define CONSOLE_UART_BASE UART12_BASE +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 1846153 +#else +#error "Unknown platform flavor" +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-aspeed/sub.mk b/optee_os/core/arch/arm/plat-aspeed/sub.mk new file mode 100644 index 0000000..a9ca2b8 --- /dev/null +++ b/optee_os/core/arch/arm/plat-aspeed/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +srcs-$(PLATFORM_FLAVOR_ast2600) += platform_ast2600.c core_pos_a32.S +srcs-$(PLATFORM_FLAVOR_ast2700) += platform_ast2700.c diff --git a/optee_os/core/arch/arm/plat-bcm/bcm_elog.c b/optee_os/core/arch/arm/plat-bcm/bcm_elog.c new file mode 100644 index 0000000..2e5c9ad --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/bcm_elog.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include + +static struct bcm_elog global_elog; + +void bcm_elog_putchar(char ch) +{ + struct bcm_elog *elog = &global_elog; + uint32_t offset = 0, len = 0; + vaddr_t base = 0; + + base = io_pa_or_va(&elog->base, elog->max_size); + + offset = io_read32(base + BCM_ELOG_OFF_OFFSET); + len = io_read32(base + BCM_ELOG_LEN_OFFSET); + io_write8(base + offset, ch); + offset++; + + /* Log buffer is now full and need to wrap around */ + if (offset >= elog->max_size) + offset = BCM_ELOG_HEADER_LEN; + + /* Only increment length when log buffer is not full */ + if (len < elog->max_size - BCM_ELOG_HEADER_LEN) + len++; + + io_write32(base + BCM_ELOG_OFF_OFFSET, offset); + io_write32(base + BCM_ELOG_LEN_OFFSET, len); +} + +void bcm_elog_init(uintptr_t pa_base, uint32_t size) +{ + struct bcm_elog *elog = &global_elog; + uint32_t val = 0; + vaddr_t base = 0; + + elog->base.pa = pa_base; + elog->max_size = size; + + base = io_pa_or_va(&elog->base, BCM_ELOG_HEADER_LEN); + + /* + * If a valid signature is found, it means logging is already + * initialized. In this case, we should not re-initialize the entry + * header in the designated memory + */ + val = io_read32(base + BCM_ELOG_SIG_OFFSET); + if (val != BCM_ELOG_SIG_VAL) { + io_write32(base + BCM_ELOG_SIG_OFFSET, BCM_ELOG_SIG_VAL); + io_write32(base + BCM_ELOG_OFF_OFFSET, BCM_ELOG_HEADER_LEN); + io_write32(base + BCM_ELOG_LEN_OFFSET, 0); + } +} diff --git a/optee_os/core/arch/arm/plat-bcm/bcm_elog.h b/optee_os/core/arch/arm/plat-bcm/bcm_elog.h new file mode 100644 index 0000000..218da37 --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/bcm_elog.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef BCM_ELOG_H +#define BCM_ELOG_H + +#include + +/* Error logging signature offset and value */ +#define BCM_ELOG_SIG_OFFSET 0x0000 +#define BCM_ELOG_SIG_VAL 0x75767971 + +/* Current logging offset that points to where new logs should be added */ +#define BCM_ELOG_OFF_OFFSET 0x0004 + +/* Current logging length (excluding header) */ +#define BCM_ELOG_LEN_OFFSET 0x0008 + +#define BCM_ELOG_HEADER_LEN 12 + +/* + * @base: base address of memory where log is saved + * @max_size: max size of memory reserved for logging + */ +struct bcm_elog { + struct io_pa_va base; + uint32_t max_size; +}; + +void bcm_elog_init(uintptr_t pa_base, uint32_t size); +void bcm_elog_putchar(char ch); + +#endif /* BCM_ELOG_H */ diff --git a/optee_os/core/arch/arm/plat-bcm/conf.mk b/optee_os/core/arch/arm/plat-bcm/conf.mk new file mode 100644 index 0000000..5f4b816 --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/conf.mk @@ -0,0 +1,44 @@ +PLATFORM_FLAVOR ?= ns3 + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_8250_UART,y) +$(call force,CFG_TEE_CORE_DEBUG,n) +$(call force,CFG_GIC,y) + +$(call force,CFG_WITH_LPAE,y) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +CFG_TZDRAM_START ?= 0x8e000000 +CFG_TZDRAM_SIZE ?= 0x01000000 # 16MB +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) - $(CFG_SHMEM_SIZE)) +CFG_SHMEM_SIZE ?= 0x01000000 # 16MB +CFG_TEE_RAM_VA_SIZE := 0x400000 # 4MB + +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) + +ifeq ($(PLATFORM_FLAVOR),ns3) +$(call force,CFG_PL022,y) +$(call force,CFG_SP805_WDT,y) +$(call force,CFG_BCM_HWRNG,y) +$(call force,CFG_BCM_SOTP,y) +$(call force,CFG_BCM_GPIO,y) +CFG_BNXT_FW ?= y +CFG_BCM_ELOG_DUMP ?= y +endif + +CFG_BCM_ELOG_AP_UART_LOG_BASE ?= 0x8f110000 +CFG_BCM_ELOG_AP_UART_LOG_SIZE ?= 0x10000 + +CFG_BCM_ELOG_BASE ?= 0x8f120000 +CFG_BCM_ELOG_SIZE ?= 0x100000 + +ifeq ($(DEBUG),1) +platform-cflags += -gdwarf-2 +platform-aflags += -gdwarf-2 +endif + +CFG_WITH_STACK_CANARIES ?= n diff --git a/optee_os/core/arch/arm/plat-bcm/crc32.c b/optee_os/core/arch/arm/plat-bcm/crc32.c new file mode 100644 index 0000000..c716f37 --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/crc32.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include + +static const uint32_t crc32tbl[] = { /* CRC polynomial 0xedb88320 */ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +static uint32_t ucrc32(const char ch, uint32_t crc) +{ + return crc32tbl[(crc ^ ch) & 0xff] ^ (crc >> 8); +} + +uint32_t crc32i(uint32_t crc, const char *buf, size_t len) +{ + size_t l = 0; + + for (l = 0; l < len; l++) + crc = ucrc32(buf[l], crc); + + return ~crc; +} diff --git a/optee_os/core/arch/arm/plat-bcm/crc32.h b/optee_os/core/arch/arm/plat-bcm/crc32.h new file mode 100644 index 0000000..acfb15c --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/crc32.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef CRC32_H +#define CRC32_H + +#include +#include + +#define CRC32_INIT_VAL (~0) +#define CRC32 crc32i + +uint32_t crc32i(uint32_t crc, const char *buf, size_t len); + +#endif /* CRC32_H */ diff --git a/optee_os/core/arch/arm/plat-bcm/main.c b/optee_os/core/arch/arm/plat-bcm/main.c new file mode 100644 index 0000000..d6f37fa --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/main.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct serial8250_uart_data console_data; + +#ifdef BCM_DEVICE0_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, BCM_DEVICE0_BASE, BCM_DEVICE0_SIZE); +#endif +#ifdef BCM_DEVICE1_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, BCM_DEVICE1_BASE, BCM_DEVICE1_SIZE); +#endif +#ifdef BCM_DEVICE2_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, BCM_DEVICE2_BASE, BCM_DEVICE2_SIZE); +#endif +#ifdef BCM_DEVICE3_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, BCM_DEVICE3_BASE, BCM_DEVICE3_SIZE); +#endif +#ifdef BCM_DEVICE4_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, BCM_DEVICE4_BASE, BCM_DEVICE4_SIZE); +#endif +#ifdef BCM_DEVICE5_BASE +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, BCM_DEVICE5_BASE, BCM_DEVICE5_SIZE); +#endif +#ifdef BCM_DRAM0_NS_BASE +register_dynamic_shm(BCM_DRAM0_NS_BASE, BCM_DRAM0_NS_SIZE); +#endif +#ifdef BCM_DRAM1_NS_BASE +register_dynamic_shm(BCM_DRAM1_NS_BASE, BCM_DRAM1_NS_SIZE); +#endif +#ifdef BCM_DRAM2_NS_BASE +register_dynamic_shm(BCM_DRAM2_NS_BASE, BCM_DRAM2_NS_SIZE); +#endif +#ifdef BCM_DRAM0_SEC_BASE +register_phys_mem(MEM_AREA_RAM_SEC, BCM_DRAM0_SEC_BASE, BCM_DRAM0_SEC_SIZE); +#endif +#ifdef CFG_BCM_ELOG_AP_UART_LOG_BASE +register_phys_mem(MEM_AREA_IO_NSEC, CFG_BCM_ELOG_AP_UART_LOG_BASE, + CFG_BCM_ELOG_AP_UART_LOG_SIZE); +#endif +#ifdef CFG_BCM_ELOG_BASE +register_phys_mem(MEM_AREA_RAM_NSEC, CFG_BCM_ELOG_BASE, CFG_BCM_ELOG_SIZE); +#endif + +void plat_trace_ext_puts(const char *str) +{ + const char *p; + + for (p = str; *p; p++) + bcm_elog_putchar(*p); +} + +void console_init(void) +{ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); + + bcm_elog_init(CFG_BCM_ELOG_AP_UART_LOG_BASE, + CFG_BCM_ELOG_AP_UART_LOG_SIZE); +} + +void boot_primary_init_intc(void) +{ + gic_init(0, GICD_BASE); +} diff --git a/optee_os/core/arch/arm/plat-bcm/platform_config.h b/optee_os/core/arch/arm/plat-bcm/platform_config.h new file mode 100644 index 0000000..3d59c82 --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/platform_config.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +#define STACK_ALIGNMENT 64 + +#define CONSOLE_UART_CLK_IN_HZ 25000000 +#define CONSOLE_BAUDRATE 115200 + +#define CONSOLE_UART_BASE 0x68a10000 + +#define GICD_BASE 0x63c00000 + +#define SMBUS0_BASE 0x689b0000 +#define SMBUS0_END (SMBUS0_BASE + 0xB0) + +#define SECURE_GPIO_BASE0 0x68b00000 +#define ASIU_GPIO_INTR 190 +#define GPIO_NUM_START0 0 +#define NUM_GPIOS0 151 + +#define CHIP_SECURE_GPIO_CONTROL0_BASE 0x68b60000 + +#define SPI_0_BASE 0x68a80000 +#define SPI_0_END (SPI_0_BASE + 0x1000) +#define SPI_0_CLK_HZ 175000000 +#define SPI_0_CS_MUX_PAD 0x68a40490 + +#define HWRNG_BASE 0x68b20000 +#define HWRNG_END (HWRNG_BASE + 0x28) + +#define SOTP_BASE 0x68b50000 + +/* NO ECC bits are present from ROW_0 to ROW_20, i.e Section 0 to Section 3 */ +#define SOTP_NO_ECC_ROWS 20 + +/* Secure Watch Dog */ +#define SEC_WDT_BASE 0x68B30000 +#define SEC_WDT_END (SEC_WDT_BASE + 0x1000) +#define SEC_WDT_CLK_HZ 12500000 +#define SEC_WDT_INTR 192 + +#define BNXT_BASE 0x60800000 + +#define QSPI_MEM_BASE 0x70000000 + +/* device memory ranges */ +#define BCM_DEVICE0_BASE GICD_BASE +#define BCM_DEVICE0_SIZE CORE_MMU_PGDIR_SIZE +#define BCM_DEVICE1_BASE SMBUS0_BASE +#define BCM_DEVICE1_SIZE CORE_MMU_PGDIR_SIZE +#define BCM_DEVICE4_BASE BNXT_BASE +#define BCM_DEVICE4_SIZE 0x800000 +#define BCM_DEVICE5_BASE QSPI_MEM_BASE +#define BCM_DEVICE5_SIZE 0x800000 + +/* NS DDR ranges */ +#define BCM_DRAM0_NS_BASE 0x80000000 +#define BCM_DRAM0_NS_SIZE 0xae00000 +#define BCM_DRAM1_NS_BASE 0x90000000 +#define BCM_DRAM1_NS_SIZE 0x70000000 +#define BCM_DRAM2_NS_BASE 0x880100000 +#define BCM_DRAM2_NS_SIZE 0x17ff00000 + +/* Secure DDR ranges */ +#define BCM_DRAM0_SEC_BASE 0x8ae00000 +#define BCM_DRAM0_SEC_SIZE 0x2200000 + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-bcm/sub.mk b/optee_os/core/arch/arm/plat-bcm/sub.mk new file mode 100644 index 0000000..d3616b4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-bcm/sub.mk @@ -0,0 +1,4 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-y += crc32.c +srcs-y += bcm_elog.c diff --git a/optee_os/core/arch/arm/plat-corstone1000/conf.mk b/optee_os/core/arch/arm/plat-corstone1000/conf.mk new file mode 100644 index 0000000..98347b1 --- /dev/null +++ b/optee_os/core/arch/arm/plat-corstone1000/conf.mk @@ -0,0 +1,44 @@ +PLATFORM_FLAVOR ?= mps3 + +# ARM debugger needs this +platform-cflags-debug-info = -gdwarf-2 +platform-aflags-debug-info = -gdwarf-2 + +$(call force,CFG_HWSUPP_MEM_PERM_WXN,y) +$(call force,CFG_HWSUPP_MEM_PERM_PXN,y) +$(call force,CFG_ENABLE_SCTLR_RR,n) +$(call force,CFG_ENABLE_SCTLR_Z,n) + +$(call force,CFG_WITH_LPAE,y) +$(call force,CFG_PSCI_ARM64,y) +$(call force,CFG_DT,y) +$(call force,CFG_EXTERNAL_DTB_OVERLAY,y) + +$(call force,CFG_CORE_SEL1_SPMC,y) +$(call force,CFG_CORE_FFA,y) +$(call force,CFG_SECURE_PARTITION,y) + +$(call force,CFG_GIC,y) +$(call force,CFG_PL011,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_ARM64_core,y) + +arm64-platform-cpuarch := cortex-a35 +arm64-platform-cflags += -mcpu=$(arm64-platform-cpuarch) +arm64-platform-aflags += -mcpu=$(arm64-platform-cpuarch) + +CFG_WITH_STATS ?= y +CFG_WITH_ARM_TRUSTED_FW ?= y + +CFG_TEE_CORE_NB_CORE ?= 1 +CFG_TZDRAM_START ?= 0x02002000 + +# TEE_RAM (OPTEE kernel + DATA) + TA_RAM = 3MB +CFG_TZDRAM_SIZE ?= 0x300000 +CFG_SHMEM_START ?= 0x86000000 +CFG_SHMEM_SIZE ?= 0x00200000 + +CFG_DDR_SIZE ?= 0x7f000000 +CFG_DT_ADDR ?= 0x82100000 +CFG_DTB_MAX_SIZE ?= 0x100000 +CFG_CORE_HEAP_SIZE ?= 131072 diff --git a/optee_os/core/arch/arm/plat-corstone1000/main.c b/optee_os/core/arch/arm/plat-corstone1000/main.c new file mode 100644 index 0000000..846b601 --- /dev/null +++ b/optee_os/core/arch/arm/plat-corstone1000/main.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static struct pl011_data console_data __nex_bss; + +register_ddr(DRAM0_BASE, DRAM0_SIZE); +register_ddr(MM_COMM_BUF_BASE, MM_COMM_BUF_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, PL011_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GIC_CPU_REG_SIZE); + +void boot_primary_init_intc(void) +{ + gic_init(GICC_BASE, GICD_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, CONSOLE_UART_CLK_IN_HZ, + CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-corstone1000/platform_config.h b/optee_os/core/arch/arm/plat-corstone1000/platform_config.h new file mode 100644 index 0000000..6653d38 --- /dev/null +++ b/optee_os/core/arch/arm/plat-corstone1000/platform_config.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Arm Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#define GIC_BASE 0x1c000000 + +#define UART0_BASE 0x1a510000 +#define UART1_BASE 0x1a520000 +#define CONSOLE_UART_BASE UART1_BASE + +#define DRAM0_BASE 0x80000000 +#define DRAM0_SIZE CFG_DDR_SIZE + +#define GICD_OFFSET 0x10000 +#define GICC_OFFSET 0x2F000 + +#define GICD_BASE (GIC_BASE + GICD_OFFSET) +#define GICC_BASE (GIC_BASE + GICC_OFFSET) + +#define MM_COMM_BUF_BASE 0x02000000 +#define MM_COMM_BUF_SIZE 0x1000 + +#define UART_BAUDRATE 115200 +#define CONSOLE_BAUDRATE UART_BAUDRATE + +#define SYS_COUNTER_FREQ_IN_TICKS UL(50000000) /* 50MHz */ + +#define CONSOLE_UART_CLK_IN_HZ UL(50000000) /* 50MHz*/ + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-corstone1000/sub.mk b/optee_os/core/arch/arm/plat-corstone1000/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-corstone1000/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-d02/conf.mk b/optee_os/core/arch/arm/plat-d02/conf.mk new file mode 100644 index 0000000..f96895d --- /dev/null +++ b/optee_os/core/arch/arm/plat-d02/conf.mk @@ -0,0 +1,34 @@ +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_TEE_CORE_NB_CORE,16) +CFG_NUM_THREADS ?= 16 +CFG_CRYPTO_WITH_CE ?= y +CFG_WITH_SOFTWARE_PRNG ?= n +# Overrides default in mk/config.mk with 96 kB +CFG_CORE_HEAP_SIZE ?= 98304 + +$(call force,CFG_HI16XX_UART,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +ifneq ($(CFG_WITH_SOFTWARE_PRNG),y) +$(call force,CFG_HI16XX_RNG,y) +endif +$(call force,CFG_WITH_LPAE,y) + +ifeq ($(CFG_ARM64_core),y) +CFG_CORE_TZSRAM_EMUL_SIZE ?= 655360 +else +CFG_CORE_TZSRAM_EMUL_SIZE ?= 524288 +endif + +# 20MB-384kB of secure RAM +ifeq ($(CFG_WITH_PAGER),y) +CFG_TEE_RAM_VA_SIZE ?= 0x00400000 +else +CFG_TEE_RAM_VA_SIZE ?= 0x00200000 +endif +CFG_TZDRAM_START ?= 0x50400000 +CFG_TZDRAM_SIZE ?= 0x013a00000 +CFG_SHMEM_START ?= 0x50000000 +CFG_SHMEM_SIZE ?= 0x00400000 + diff --git a/optee_os/core/arch/arm/plat-d02/main.c b/optee_os/core/arch/arm/plat-d02/main.c new file mode 100644 index 0000000..759ca3b --- /dev/null +++ b/optee_os/core/arch/arm/plat-d02/main.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +static struct hi16xx_uart_data console_data; + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, + HI16XX_UART_REG_SIZE); + +void console_init(void) +{ + hi16xx_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-d02/platform_config.h b/optee_os/core/arch/arm/plat-d02/platform_config.h new file mode 100644 index 0000000..eb91def --- /dev/null +++ b/optee_os/core/arch/arm/plat-d02/platform_config.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +/* UART */ +#define PERI_SUB_CTRL_ADDR 0x80000000 +#define CONSOLE_UART_BASE (PERI_SUB_CTRL_ADDR + 0x00300000) +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 200000000 + +/* ALG sub-controller */ +#define ALG_SC_BASE 0xD0000000 +#define ALG_SC_REG_SIZE 0xF010 + +/* RNG */ +#define RNG_BASE 0xD1010000 +#define RNG_REG_SIZE 0x18 + +/* + * HiSilicon D02 memory map + * + * Note: the physical address ranges below correspond to DRAM which is + * non-secure by default. Therefore, the terms TZDRAM and TZSRAM may not + * reflect the reality and only indicate areas that "would normally be" + * secure DRAM and secure SRAM in a more complete implementation. + * The memory map was defined like this for lack of better documentation. + * It is good enough for development/testing purposes. + * + * CFG_WITH_PAGER=n + * + * 0x7FC0_0000 - + * Linux/other | DRAM1 + * 0x5180_0000 - + * TA RAM: 16 MiB | + * 0x5080_0000 | TZDRAM + * TEE RAM: 4 MiB (TEE_RAM_VA_SIZE) | + * 0x5040_0000 [TZDRAM_BASE, TEE_LOAD_ADDR] - + * Shared memory: 4 MiB | SHMEM + * 0x5000_0000 - + * Linux/other | DRAM0 + * 0x0000_0000 [DRAM0_BASE] - + * + * CFG_WITH_PAGER=y + * + * 0x7FC0_0000 - + * Linux/other | DRAM1 + * 0x5180_0000 - + * TA RAM: 20096 KiB (TZDRAM_SIZE) | TZDRAM + * 0x5046_0000 - + * TEE RAM: 384 KiB (TZSRAM_SIZE) | TZSRAM + * 0x5040_0000 [TZSRAM_BASE, TEE_LOAD_ADDR] - + * Shared memory: 4 MiB | SHMEM + * 0x5000_0000 - + * Linux/other | DRAM0 + * 0x0000_0000 [DRAM0_BASE] - + */ + +#define DRAM0_BASE 0x00000000 +#define DRAM0_SIZE 0x50000000 + +#define DRAM1_BASE 0x51800000 +#define DRAM1_SIZE 0x2E400000 + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-d02/sub.mk b/optee_os/core/arch/arm/plat-d02/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-d02/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-d06/conf.mk b/optee_os/core/arch/arm/plat-d06/conf.mk new file mode 100644 index 0000000..98a8fa0 --- /dev/null +++ b/optee_os/core/arch/arm/plat-d06/conf.mk @@ -0,0 +1,28 @@ +include core/arch/arm/cpu/cortex-armv8-0.mk + +CFG_TEE_CORE_NB_CORE ?= 128 +CFG_NUM_THREADS ?= 96 +CFG_CRYPTO_WITH_CE ?= y + +CFG_WITH_PAGER ?= n +CFG_WITH_SOFTWARE_PRNG ?= y +CFG_WITH_STATS ?= y +CFG_TEE_CORE_EMBED_INTERNAL_TESTS ?= y + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_WITH_LPAE,y) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,48) +$(call force,CFG_LPC_UART,y) + +CFG_TEE_CORE_LOG_LEVEL ?= 4 + +CFG_CORE_HEAP_SIZE ?= 0x008000000 +CFG_CORE_ARM64_PA_BITS ?= 40 +CFG_TEE_RAM_VA_SIZE ?= 0x009000000 +CFG_TZDRAM_START ?= 0x20C0000000 +CFG_TZDRAM_SIZE ?= 0x32000000 +CFG_SHMEM_START ?= 0x50000000 +CFG_SHMEM_SIZE ?= 0x04000000 diff --git a/optee_os/core/arch/arm/plat-d06/core_pos_a64.S b/optee_os/core/arch/arm/plat-d06/core_pos_a64.S new file mode 100644 index 0000000..d744cf7 --- /dev/null +++ b/optee_os/core/arch/arm/plat-d06/core_pos_a64.S @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2019, Arm Limited. All rights reserved. + * Copyright (c) 2020, Marek Vasut + * Copyright (c) 2022, Huawei Technologies Co., Ltd + */ + +#include +#include +#include + +/* + * bit8~bit10: core index + * bit16~bit18: ccl index + * bit20~bit22: sccl index + * 96 cores: index = sccl * 24 + ccl * 4 + core + * 128 cores: index = sccl * 32 + ccl * 4 + core (now used) + */ + +FUNC get_core_pos_mpidr , : + lsr x1, x0, 8 + and x2, x1, 0x7 + + lsr x1, x0, 16 + and x3, x1, 0x7 + + lsr x1, x0, 20 + and x4, x1, 0x7 + + mov x5, x4 +#if (CFG_TEE_CORE_NB_CORE == 96) + lsl x5, x5, 1 + add x5, x5, x4 + lsl x5, x5, 1 +#elif (CFG_TEE_CORE_NB_CORE == 128) + lsl x5, x5, 3 +#else + static_assert(0); +#endif + add x5, x5, x3 + lsl x5, x5, 2 + add x5, x5, x2 + + mov x0, x5 + + ret +END_FUNC get_core_pos_mpidr diff --git a/optee_os/core/arch/arm/plat-d06/main.c b/optee_os/core/arch/arm/plat-d06/main.c new file mode 100644 index 0000000..7736944 --- /dev/null +++ b/optee_os/core/arch/arm/plat-d06/main.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2022, Huawei Technologies Co., Ltd + */ +#include +#include +#include + +static struct lpc_uart_data console_data __nex_bss; + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, LPC_BASE, LPC_SIZE); + +void console_init(void) +{ + lpc_uart_init(&console_data, LPC_BASE, CONSOLE_UART_CLK_IN_HZ, + CONSOLE_BAUDRATE); + + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-d06/platform_config.h b/optee_os/core/arch/arm/plat-d06/platform_config.h new file mode 100644 index 0000000..81979bd --- /dev/null +++ b/optee_os/core/arch/arm/plat-d06/platform_config.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2022, Huawei Technologies Co., Ltd + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +/* UART */ +#define UART_BASE 0x2f8 +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 200 + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-d06/sub.mk b/optee_os/core/arch/arm/plat-d06/sub.mk new file mode 100644 index 0000000..4324853 --- /dev/null +++ b/optee_os/core/arch/arm/plat-d06/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-y += core_pos_a64.S diff --git a/optee_os/core/arch/arm/plat-hikey/conf.mk b/optee_os/core/arch/arm/plat-hikey/conf.mk new file mode 100644 index 0000000..1fe28b9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hikey/conf.mk @@ -0,0 +1,64 @@ +PLATFORM_FLAVOR ?= hikey + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_PL011,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) + +CFG_NUM_THREADS ?= 8 +CFG_CRYPTO_WITH_CE ?= y + +ifeq ($(PLATFORM_FLAVOR),hikey) +CFG_CORE_HEAP_SIZE ?= 73728 +CFG_PL061 ?= y +CFG_PL022 ?= y +CFG_SPI ?= y + +ifeq ($(CFG_SPI_TEST),y) +$(call force,CFG_SPI,y) +endif + +ifeq ($(CFG_SPI),y) +$(call force,CFG_PL061,y) +$(call force,CFG_PL022,y) +endif + +ifeq ($(CFG_PL061),y) +core-platform-cppflags += -DPLAT_PL061_MAX_GPIOS=160 +endif +endif + +CFG_CACHE_API ?= y +CFG_SECURE_DATA_PATH ?= n +CFG_TEE_SDP_MEM_BASE ?= 0x3E800000 +CFG_TEE_SDP_MEM_SIZE ?= 0x00400000 + +ifeq ($(PLATFORM_FLAVOR),hikey) +CFG_CONSOLE_UART ?= 3 +CFG_DRAM_SIZE_GB ?= 2 +endif + +ifeq ($(PLATFORM_FLAVOR),hikey960) +CFG_CONSOLE_UART ?= 6 +CFG_DRAM_SIZE_GB ?= 3 +CFG_CORE_BGET_BESTFIT ?= y +ifeq ($(CFG_ARM32_core),y) +CFG_ASAN_SHADOW_OFFSET ?= 0x372E38E0 +endif +# Hikey960 4G/6G versions have physical addresses above 4G range +ifneq (,$(filter 4 6,$(CFG_DRAM_SIZE_GB))) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +endif +endif + +CFG_TZDRAM_START ?= 0x3F000000 +CFG_TZDRAM_SIZE ?= 0x01000000 +CFG_SHMEM_START ?= 0x3EE00000 +CFG_SHMEM_SIZE ?= 0x00200000 +CFG_TEE_RAM_VA_SIZE ?= 0x00200000 + +CFG_IN_TREE_EARLY_TAS += avb/023f8f1a-292a-432b-8fc4-de8471358067 + +CFG_EMBED_DTB_SOURCE_FILE ?= hikey.dts diff --git a/optee_os/core/arch/arm/plat-hikey/hikey_peripherals.h b/optee_os/core/arch/arm/plat-hikey/hikey_peripherals.h new file mode 100644 index 0000000..ee89715 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hikey/hikey_peripherals.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2016, Linaro Ltd and Contributors. All rights reserved. + * Copyright (c) 2016, Hisilicon Ltd and Contributors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of ARM nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIKEY_PERIPHERALS_H__ +#define __HIKEY_PERIPHERALS_H__ + +#include + +#define PMUSSI_BASE 0xF8000000 +#define PERI_BASE 0xF7030000 +#define PMX0_BASE 0xF7010000 +#define PMX1_BASE 0xF7010800 +#define GPIO6_BASE 0xF7022000 +#define SPI_BASE 0xF7106000 + +#define PMUSSI_REG_SIZE 0x1000 +#define PERI_BASE_REG_SIZE 0x2000 +#define PMX0_REG_SIZE 0x27c +#define PMX1_REG_SIZE 0x28c + +/* register offsets */ +#define PMUSSI_LDO21_REG_ADJ SHIFT_U32(0x86, 2) +#define PMUSSI_ENA_LDO17_22 SHIFT_U32(0x2F, 2) + +#define PERI_SC_PERIPH_RSTDIS3 0x334 +#define PERI_SC_PERIPH_RSTSTAT3 0x338 +#define PERI_SC_PERIPH_CLKEN3 0x230 +#define PERI_SC_PERIPH_CLKSTAT3 0x238 + +#define PMX0_IOMG104 0x1a0 +#define PMX0_IOMG105 0x1a4 +#define PMX0_IOMG106 0x1a8 +#define PMX0_IOMG107 0x1ac + +#define PMX1_IOCG104 0x1b0 +#define PMX1_IOCG105 0x1b4 +#define PMX1_IOCG106 0x1b8 +#define PMX1_IOCG107 0x1bc +/* end register offsets */ + +#define PMUSSI_LDO21_REG_VL_MASK 0x7 +#define PMUSSI_LDO21_REG_VL_1V8 0x3 +#define PMUSSI_ENA_LDO21 BIT(4) + +#define PERI_RST3_SSP BIT(9) +#define PERI_CLK3_SSP BIT(9) + +#define PINMUX_GPIO 0 +#define PINMUX_SPI 1 + +#define PINCFG_NOPULL 0 +#define PINCFG_PULLUP 1 +#define PINCFG_PULLDN 2 + +#define GPIO6_2 50 +#define SPI_CLK_HZ 150000000 /* 150mhz */ +#define SPI_500_KHZ 500000 +#define SPI_10_KHZ 10000 + +#ifdef CFG_SPI +void spi_init(void); +#ifdef CFG_SPI_TEST +void spi_test(void); +#endif /* CFG_SPI_TEST */ +#endif /* CFG_SPI */ + +#endif /* __HIKEY_PERIPHERALS_H__ */ diff --git a/optee_os/core/arch/arm/plat-hikey/main.c b/optee_os/core/arch/arm/plat-hikey/main.c new file mode 100644 index 0000000..9e0543a --- /dev/null +++ b/optee_os/core/arch/arm/plat-hikey/main.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#ifdef CFG_SPI +#include +#include +#endif +#if defined(PLATFORM_FLAVOR_hikey) +#include +#endif +#include +#include +#include +#include +#include +#include +#include + +static struct pl011_data console_data __nex_bss; + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, PL011_REG_SIZE); +#if defined(PLATFORM_FLAVOR_hikey) +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, PMUSSI_BASE, PMUSSI_REG_SIZE); +#endif +#if defined(CFG_SPI) && defined(PLATFORM_FLAVOR_hikey) +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, PERI_BASE, PERI_BASE_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, PMX0_BASE, PMX0_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, PMX1_BASE, PMX1_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, GPIO6_BASE, PL061_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, SPI_BASE, PL022_REG_SIZE); +#endif +register_dynamic_shm(DRAM0_BASE, DRAM0_SIZE_NSEC); +#ifdef DRAM1_SIZE_NSEC +register_dynamic_shm(DRAM1_BASE, DRAM1_SIZE_NSEC); +#endif +#ifdef DRAM2_SIZE_NSEC +register_dynamic_shm(DRAM2_BASE, DRAM2_SIZE_NSEC); +#endif + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, CONSOLE_UART_CLK_IN_HZ, + CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +#if defined(PLATFORM_FLAVOR_hikey) +#ifdef CFG_SPI +void spi_init(void) +{ + uint32_t shifted_val, read_val; + vaddr_t peri_base = core_mmu_get_va(PERI_BASE, MEM_AREA_IO_NSEC, + PERI_BASE_REG_SIZE); + vaddr_t pmx0_base = core_mmu_get_va(PMX0_BASE, MEM_AREA_IO_NSEC, + PMX0_REG_SIZE); + vaddr_t pmx1_base = core_mmu_get_va(PMX1_BASE, MEM_AREA_IO_NSEC, + PMX1_REG_SIZE); + + DMSG("take SPI0 out of reset\n"); + shifted_val = PERI_RST3_SSP; + /* + * no need to read PERI_SC_PERIPH_RSTDIS3 first + * as all the bits are processed and cleared after writing + */ + io_write32(peri_base + PERI_SC_PERIPH_RSTDIS3, shifted_val); + DMSG("PERI_SC_PERIPH_RSTDIS3: 0x%x\n", + io_read32(peri_base + PERI_SC_PERIPH_RSTDIS3)); + + /* + * wait until the requested device is out of reset + * and ready to be used + */ + do { + read_val = io_read32(peri_base + PERI_SC_PERIPH_RSTSTAT3); + } while (read_val & shifted_val); + DMSG("PERI_SC_PERIPH_RSTSTAT3: 0x%x\n", read_val); + + DMSG("enable SPI clock\n"); + /* + * no need to read PERI_SC_PERIPH_CLKEN3 first + * as all the bits are processed and cleared after writing + */ + shifted_val = PERI_CLK3_SSP; + io_write32(peri_base + PERI_SC_PERIPH_CLKEN3, shifted_val); + DMSG("PERI_SC_PERIPH_CLKEN3: 0x%x\n", + io_read32(peri_base + PERI_SC_PERIPH_CLKEN3)); + + DMSG("PERI_SC_PERIPH_CLKSTAT3: 0x%x\n", + io_read32(peri_base + PERI_SC_PERIPH_CLKSTAT3)); + + /* + * GPIO6_2 can be configured as PINMUX_GPIO, but as PINMUX_SPI, HW IP + * will control the chip select pin so we don't have to manually do it. + * The only concern is that the IP will pulse it between each packet, + * which might not work with certain clients. There seems to be no + * option to configure it to stay enabled for the total duration of the + * transfer. + * ref: http://infocenter.arm.com/help/topic/com.arm.doc.ddi0194h/CJACFAFG.html + */ + DMSG("configure gpio6 pins 0-3 as SPI\n"); + io_write32(pmx0_base + PMX0_IOMG104, PINMUX_SPI); + io_write32(pmx0_base + PMX0_IOMG105, PINMUX_SPI); + io_write32(pmx0_base + PMX0_IOMG106, PINMUX_SPI); + io_write32(pmx0_base + PMX0_IOMG107, PINMUX_SPI); + + DMSG("configure gpio6 pins 0-3 as nopull\n"); + io_write32(pmx1_base + PMX1_IOCG104, PINCFG_NOPULL); + io_write32(pmx1_base + PMX1_IOCG105, PINCFG_NOPULL); + io_write32(pmx1_base + PMX1_IOCG106, PINCFG_NOPULL); + io_write32(pmx1_base + PMX1_IOCG107, PINCFG_NOPULL); + +#ifdef CFG_SPI_TEST + spi_test(); +#endif +} +#endif + +static TEE_Result peripherals_init(void) +{ + vaddr_t pmussi_base = core_mmu_get_va(PMUSSI_BASE, MEM_AREA_IO_NSEC, + PMUSSI_REG_SIZE); + + DMSG("enable LD021_1V8 source (pin 35) on LS connector\n"); + /* + * Mezzanine cards usually use this to source level shifters for + * UART, GPIO, SPI, I2C, etc so if not enabled, connected + * peripherals will not work either (during bootloader stage) + * until linux is booted. + */ + io_mask8(pmussi_base + PMUSSI_LDO21_REG_ADJ, PMUSSI_LDO21_REG_VL_1V8, + PMUSSI_LDO21_REG_VL_MASK); + io_write8(pmussi_base + PMUSSI_ENA_LDO17_22, PMUSSI_ENA_LDO21); + +#ifdef CFG_SPI + spi_init(); +#endif + return TEE_SUCCESS; +} + +driver_init(peripherals_init); +#endif /* PLATFORM_FLAVOR_hikey */ diff --git a/optee_os/core/arch/arm/plat-hikey/platform_config.h b/optee_os/core/arch/arm/plat-hikey/platform_config.h new file mode 100644 index 0000000..35e6f3d --- /dev/null +++ b/optee_os/core/arch/arm/plat-hikey/platform_config.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +/* PL011 UART */ +#if defined(PLATFORM_FLAVOR_hikey) + +#define PL011_UART0_BASE 0xF8015000 +#define PL011_UART2_BASE 0xF7112000 +#define PL011_UART3_BASE 0xF7113000 +#if (CFG_CONSOLE_UART == 3) +#define CONSOLE_UART_BASE PL011_UART3_BASE +#elif (CFG_CONSOLE_UART == 2) +#define CONSOLE_UART_BASE PL011_UART2_BASE +#elif (CFG_CONSOLE_UART == 0) +#define CONSOLE_UART_BASE PL011_UART0_BASE +#else +#error Unknown console UART +#endif + +#elif defined(PLATFORM_FLAVOR_hikey960) + +#define PL011_UART5_BASE 0xFDF05000 +#define PL011_UART6_BASE 0xFFF32000 +#if (CFG_CONSOLE_UART == 6) +#define CONSOLE_UART_BASE PL011_UART6_BASE +#elif (CFG_CONSOLE_UART == 5) +#define CONSOLE_UART_BASE PL011_UART5_BASE +#else +#error Unknown console UART +#endif + +#else /* PLATFORM_FLAVOR_hikey */ +#error Unknown HiKey PLATFORM_FLAVOR +#endif /* PLATFORM_FLAVOR_hikey */ + +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 19200000 + +/* + * HiKey and HiKey960 memory map + * + * Refer to the default configuration from conf.mk for description below. + * + * TZDRAM is secured (firewalled) by the DDR controller, see ARM-TF, but note + * that security of this type of memory is weak for two reasons: + * 1. It is prone to physical tampering since DRAM is external to the SoC + * 2. It is still somewhat prone to software attacks because the memory + * protection may be reverted by the non-secure kernel with a piece of + * code similar to the one that sets the protection in ARM-TF (we're + * missing a "lockdown" step which would prevent any change to the DDRC + * configuration until the next SoC reset). + * TZSRAM is emulated in the TZDRAM area, because the on-chip SRAM of the + * HiKey SoC is too small to run OP-TEE (72K total with 64K available, see + * "SRAM Memory Region Layout" in ARM-TF plat/hikey/include/hisi_sram_map.h), + * while the SRAM of the HiKey960 SoC is not available to the public at the + * moment. + * + * CFG_WITH_PAGER=n + * + * 0x4000_0000 - + * TA RAM: 14 MiB | + * 0x3F20_0000 | TZDRAM + * TEE RAM: 2 MiB (TEE_RAM_VA_SIZE) | + * 0x3F00_0000 [TZDRAM_BASE, BL32_LOAD_ADDR] - + * Shared memory: 2 MiB | + * 0x3EE0_0000 | DRAM0 + * Reserved by UEFI for OP-TEE, unused | + * 0x3EC0_0000 - + * Secure Data Path buffers: 4 MiB | DRAM0 (secure) + * 0x3E80_0000 [CFG_TEE_SDP_MEM_BASE] - + * Reserved by UEFI for OP-TEE, unused | + * 0x3E00_0000 | DRAM0 + * Available to Linux | + * 0x0000_0000 [DRAM0_BASE] - + * + * CFG_WITH_PAGER=y + * + * 0x4000_0000 - + * TA RAM: 14 MiB | TZDRAM + * 0x3F20_0000 - + * Unused + * 0x3F03_2000 - + * TEE RAM: 200 KiB | TZSRAM + * 0x3F00_0000 [TZSRAM_BASE, BL32_LOAD_ADDR] - + * Shared memory: 2 MiB | + * 0x3EE0_0000 | DRAM0 + * Reserved by UEFI for OP-TEE, unused | + * 0x3EC0_0000 - + * Secure Data Path buffers: 4 MiB | DRAM0 (secure) + * 0x3E80_0000 [CFG_TEE_SDP_MEM_BASE] - + * Reserved by UEFI for OP-TEE, unused | + * 0x3E00_0000 | DRAM0 + * Available to Linux | + * 0x0000_0000 [DRAM0_BASE] - + */ + +#define DRAM0_BASE 0x00000000 +#define DRAM0_SIZE 0x3F000000 +#define DRAM0_SIZE_NSEC 0x3E000000 +#define DRAM1_BASE 0x40000000 + +#if defined(PLATFORM_FLAVOR_hikey) + +#if (CFG_DRAM_SIZE_GB == 2) +#define DRAM1_SIZE_NSEC 0x40000000 +#elif (CFG_DRAM_SIZE_GB == 1) +/* do nothing */ +#else +#error Unknown DRAM size +#endif + +#elif defined(PLATFORM_FLAVOR_hikey960) + +/* + * Physical address ranges for HiKey960 RAM: + * 3G board: 0~3G + * 4G board: 0~3G 3~3.5G 8~8.5G + * 6G board: 0~3G 4~7G + */ +#if (CFG_DRAM_SIZE_GB == 3) +#define DRAM1_SIZE_NSEC 0x80000000 +#elif (CFG_DRAM_SIZE_GB == 4) +#define DRAM1_SIZE_NSEC 0xA0000000 +#define DRAM2_BASE 0x200000000 +#define DRAM2_SIZE_NSEC 0x20000000 +#elif (CFG_DRAM_SIZE_GB == 6) +#define DRAM1_SIZE_NSEC 0x80000000 +#define DRAM2_BASE 0x100000000 +#define DRAM2_SIZE_NSEC 0xC0000000 +#else +#error Unknown DRAM size +#endif + +#if (CFG_DRAM_SIZE_GB >= 4 && defined(CFG_ARM32_core) && \ + defined(CFG_CORE_DYN_SHM) && !defined(CFG_LARGE_PHYS_ADDR)) +#error 32-bit TEE with CFG_CORE_DYN_SHM and without CFG_LARGE_PHYS_ADDR \ + cannot support boards with 4G RAM or more +#endif + +#else /* PLATFORM_FLAVOR_hikey */ +#error Unknown HiKey PLATFORM_FLAVOR +#endif /* PLATFORM_FLAVOR_hikey */ + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-hikey/spi_test.c b/optee_os/core/arch/arm/plat-hikey/spi_test.c new file mode 100644 index 0000000..ea7ec25 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hikey/spi_test.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PL022_STAT 0x00C +#define PL022_STAT_BSY SHIFT_U32(1, 4) + +static void spi_cs_callback(enum gpio_level value) +{ + static bool inited; + static struct pl061_data pd; + vaddr_t gpio6_base = core_mmu_get_va(GPIO6_BASE, MEM_AREA_IO_NSEC, + PL061_REG_SIZE); + vaddr_t spi_base = core_mmu_get_va(SPI_BASE, MEM_AREA_IO_NSEC, + PL022_REG_SIZE); + + if (!inited) { + pl061_init(&pd); + pl061_register(gpio6_base, 6); + pl061_set_mode_control(GPIO6_2, PL061_MC_SW); + pd.chip.ops->set_interrupt(NULL, GPIO6_2, + GPIO_INTERRUPT_DISABLE); + pd.chip.ops->set_direction(NULL, GPIO6_2, GPIO_DIR_OUT); + inited = true; + } + + if (io_read8(spi_base + PL022_STAT) & PL022_STAT_BSY) + DMSG("pl022 busy - do NOT set CS!"); + while (io_read8(spi_base + PL022_STAT) & PL022_STAT_BSY) + ; + DMSG("pl022 done - set CS!"); + + pd.chip.ops->set_value(NULL, GPIO6_2, value); +} + +static void spi_set_cs_mux(uint32_t val) +{ + uint32_t data; + vaddr_t pmx0_base = core_mmu_get_va(PMX0_BASE, MEM_AREA_IO_NSEC, + PMX0_REG_SIZE); + + if (val == PINMUX_SPI) { + DMSG("Configure gpio6 pin2 as SPI"); + io_write32(pmx0_base + PMX0_IOMG106, PINMUX_SPI); + } else { + DMSG("Configure gpio6 pin2 as GPIO"); + io_write32(pmx0_base + PMX0_IOMG106, PINMUX_GPIO); + } + + data = io_read32(pmx0_base + PMX0_IOMG106); + if (data) + DMSG("gpio6 pin2 is SPI"); + else + DMSG("gpio6 pin2 is GPIO"); +} + +static void spi_test_with_manual_cs_control(void) +{ + struct pl022_data pd; + vaddr_t spi_base = core_mmu_get_va(SPI_BASE, MEM_AREA_IO_NSEC, + PL022_REG_SIZE); + uint8_t tx[3] = {0x01, 0x80, 0x00}; + uint8_t rx[3] = {0}; + size_t i, j, len = 3; + enum spi_result res; + + spi_set_cs_mux(PINMUX_GPIO); + + DMSG("Set CS callback"); + pd.cs_control = PL022_CS_CTRL_MANUAL; + + DMSG("spi_base: 0x%" PRIxVA "\n", spi_base); + DMSG("Configure SPI"); + pd.base = spi_base; + pd.clk_hz = SPI_CLK_HZ; + pd.speed_hz = SPI_10_KHZ; + pd.mode = SPI_MODE0; + pd.data_size_bits = 8; + pd.loopback = true; + + pl022_init(&pd); + pd.chip.ops->configure(&pd.chip); + pd.chip.ops->start(&pd.chip); + + /* + * Pulse CS only once for the whole transmission. + * This is the scheme used by the pl022 driver. + */ + spi_cs_callback(GPIO_LEVEL_HIGH); + tee_time_busy_wait(2); + spi_cs_callback(GPIO_LEVEL_LOW); + for (j = 0; j < 10; j++) { + DMSG("SPI test loop: %zu", j); + res = pd.chip.ops->txrx8(&pd.chip, tx, rx, len); + if (res) { + EMSG("SPI transceive error %d", res); + break; + } + + for (i = 0; i < len; i++) + DMSG("rx[%zu] = 0x%x", i, rx[i]); + + tee_time_busy_wait(20); + } + spi_cs_callback(GPIO_LEVEL_HIGH); + + /* Pulse CS once per transfer */ + spi_cs_callback(GPIO_LEVEL_HIGH); + tee_time_busy_wait(2); + for (j = 10; j < 20; j++) { + DMSG("SPI test loop: %zu", j); + spi_cs_callback(GPIO_LEVEL_LOW); + res = pd.chip.ops->txrx8(&pd.chip, tx, rx, len); + if (res) { + EMSG("SPI transceive error %d", res); + break; + } + + for (i = 0; i < len; i++) + DMSG("rx[%zu] = 0x%x", i, rx[i]); + + tee_time_busy_wait(20); + spi_cs_callback(GPIO_LEVEL_HIGH); + } + + /* Pulse CS once per word/byte */ + spi_set_cs_mux(PINMUX_SPI); + tee_time_busy_wait(2); + for (j = 20; j < 30; j++) { + DMSG("SPI test loop: %zu", j); + res = pd.chip.ops->txrx8(&pd.chip, tx, rx, len); + if (res) { + EMSG("SPI transceive error %d", res); + break; + } + + for (i = 0; i < len; i++) + DMSG("rx[%zu] = 0x%x", i, rx[i]); + + tee_time_busy_wait(20); + } + + pd.chip.ops->end(&pd.chip); +} + +static void spi_test_with_registered_cs_cb(void) +{ + struct pl022_data pd; + vaddr_t spi_base = core_mmu_get_va(SPI_BASE, MEM_AREA_IO_NSEC, + PL022_REG_SIZE); + uint8_t tx[3] = {0x01, 0x80, 0x00}; + uint8_t rx[3] = {0}; + size_t i, j, len = 3; + enum spi_result res; + + spi_set_cs_mux(PINMUX_GPIO); + + DMSG("Set CS callback"); + pd.cs_data.cs_cb = spi_cs_callback; + pd.cs_control = PL022_CS_CTRL_CB; + + DMSG("spi_base: 0x%" PRIxVA "\n", spi_base); + DMSG("Configure SPI"); + pd.base = spi_base; + pd.clk_hz = SPI_CLK_HZ; + pd.speed_hz = SPI_10_KHZ; + pd.mode = SPI_MODE0; + pd.data_size_bits = 8; + pd.loopback = true; + + pl022_init(&pd); + pd.chip.ops->configure(&pd.chip); + pd.chip.ops->start(&pd.chip); + + for (j = 0; j < 20; j++) { + DMSG("SPI test loop: %zu", j); + res = pd.chip.ops->txrx8(&pd.chip, tx, rx, len); + if (res) { + EMSG("SPI transceive error %d", res); + break; + } + + for (i = 0; i < len; i++) + DMSG("rx[%zu] = 0x%x", i, rx[i]); + + tee_time_busy_wait(20); + } + + pd.chip.ops->end(&pd.chip); +} + +static void spi_test_with_builtin_cs_control(void) +{ + struct pl061_data pd061; + struct pl022_data pd022; + vaddr_t gpio6_base = core_mmu_get_va(GPIO6_BASE, MEM_AREA_IO_NSEC, + PL061_REG_SIZE); + vaddr_t spi_base = core_mmu_get_va(SPI_BASE, MEM_AREA_IO_NSEC, + PL022_REG_SIZE); + uint8_t tx[3] = {0x01, 0x80, 0x00}; + uint8_t rx[3] = {0}; + size_t i, j, len = 3; + enum spi_result res; + + spi_set_cs_mux(PINMUX_GPIO); + + DMSG("gpio6_base: 0x%" PRIxVA "\n", gpio6_base); + DMSG("Configure GPIO"); + pl061_init(&pd061); + pl061_register(gpio6_base, 6); + DMSG("Enable software mode control for chip select"); + pl061_set_mode_control(GPIO6_2, PL061_MC_SW); + + pd022.cs_data.gpio_data.chip = &pd061.chip; + pd022.cs_data.gpio_data.pin_num = GPIO6_2; + pd022.cs_control = PL022_CS_CTRL_AUTO_GPIO; + + DMSG("spi_base: 0x%" PRIxVA "\n", spi_base); + DMSG("Configure SPI"); + pd022.base = spi_base; + pd022.clk_hz = SPI_CLK_HZ; + pd022.speed_hz = SPI_10_KHZ; + pd022.mode = SPI_MODE0; + pd022.data_size_bits = 8; + pd022.loopback = true; + + pl022_init(&pd022); + pd022.chip.ops->configure(&pd022.chip); + pd022.chip.ops->start(&pd022.chip); + + for (j = 0; j < 20; j++) { + DMSG("SPI test loop: %zu", j); + res = pd022.chip.ops->txrx8(&pd022.chip, tx, rx, len); + if (res) { + EMSG("SPI transceive error %d", res); + break; + } + + for (i = 0; i < len; i++) + DMSG("rx[%zu] = 0x%x", i, rx[i]); + + tee_time_busy_wait(20); + } + + pd022.chip.ops->end(&pd022.chip); +} + +/* + * spi_init() MUST be run before calling this function! + * + * spi_test runs some loopback tests, so the SPI module will just receive + * what is transmitted, i.e. 0x01, 0x80, 0x00. + * + * In non-loopback mode, the transmitted value will elicit a readback of + * the measured value from the ADC chip on the Linksprite 96Boards + * Mezzanine card [1], which can be connected to either a sliding + * rheostat [2] or photoresistor [3]. + * + * [1] http://linksprite.com/wiki/index.php5?title=Linker_Mezzanine_card_for_96board + * [2] http://learn.linksprite.com/96-board/sliding-rheostat + * [3] http://learn.linksprite.com/96-board/photoresistor + */ +void spi_test(void) +{ + spi_test_with_builtin_cs_control(); + spi_test_with_registered_cs_cb(); + spi_test_with_manual_cs_control(); +} diff --git a/optee_os/core/arch/arm/plat-hikey/sub.mk b/optee_os/core/arch/arm/plat-hikey/sub.mk new file mode 100644 index 0000000..1acd560 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hikey/sub.mk @@ -0,0 +1,5 @@ +global-incdirs-y += . +srcs-y += main.c +ifeq ($(PLATFORM_FLAVOR),hikey) +srcs-$(CFG_SPI_TEST) += spi_test.c +endif diff --git a/optee_os/core/arch/arm/plat-hisilicon/conf.mk b/optee_os/core/arch/arm/plat-hisilicon/conf.mk new file mode 100644 index 0000000..dc118d8 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hisilicon/conf.mk @@ -0,0 +1,61 @@ +PLATFORM_FLAVOR ?= hi3519av100_demo + +hi3519av100-flavorlist = hi3519av100_demo hi3519av100_tst + +ifneq (,$(filter $(PLATFORM_FLAVOR),$(hi3519av100-flavorlist))) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_HI3519AV100,y) +$(call force,CFG_TEE_CORE_NB_CORE,2) +# Hi3519av100 has got two clusters, one core per cluster +$(call force,CFG_CORE_CLUSTER_SHIFT,0) + +$(call force,CFG_PL011,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_ARM32_core,y) +$(call force,CFG_PSCI_ARM32,y) + +CFG_BOOT_SECONDARY_REQUEST ?= y +CFG_NUM_THREADS ?= 4 +CFG_CRYPTO_WITH_CE ?= y +CFG_NS_ENTRY_ADDR ?= 0x22008000 +CFG_CORE_HEAP_SIZE ?= 131072 + +# +# Hi3519av100 memory map +# +# This is a general memory map for demo board, and for your own board, +# you have to define your own memory map. +# +# 0x4000_0000 [DRAM_LIMIT] +# other (media memory zone/uboot and other) +# +# 0x3360_0000 - +# TA RAM: 12 MiB | TZDRAM +# 0x32a0_0000 - +# +# CFG_WITH_PAGER=n - +# TEE RAM: 4 MiB (TEE_RAM_VA_SIZE) | TZDRAM +# 0x3260_0000 [TZDRAM_BASE, TEE_LOAD_ADDR] - +# +# CFG_WITH_PAGER=y +# Unused +# 0x32607_0000 - +# TEE RAM: 448 KiB (TZSRAM_SIZE) | TZSRAM +# 0x3260_0000 [TZDRAM_BASE, TZSRAM_BASE, TEE_LOAD_ADDR] +# OP-TEE Future Use: 2 MiB +# 0x3240_0000 +# Shared memory: 4 MB +# 0x3200_0000 +# Linux memory: 256MB +# 0x2200_0000 +# DSP reserved memory: 32MB +# 0x2000_0000 [DRAM_BASE] +# +CFG_TZDRAM_START ?= 0x32600000 +CFG_TZDRAM_SIZE ?= 0x01000000 +CFG_TEE_RAM_VA_SIZE ?= 0x00400000 +CFG_SHMEM_START ?= 0x32000000 +CFG_SHMEM_SIZE ?= 0x00400000 +else +$(error Error: Not supported PLATFORM_FLAVOR or NULL PLATFORM_FLAVOR) +endif diff --git a/optee_os/core/arch/arm/plat-hisilicon/hi3519av100.h b/optee_os/core/arch/arm/plat-hisilicon/hi3519av100.h new file mode 100644 index 0000000..1d4b7ea --- /dev/null +++ b/optee_os/core/arch/arm/plat-hisilicon/hi3519av100.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, HiSilicon Technologies Co., Ltd. + */ + +#ifndef __HI3519AV100_H__ +#define __HI3519AV100_H__ + +#include + +/* PL011 */ +#define PL011_UART0_BASE 0x04540000 +#define PL011_BAUDRATE 115200 +#define PL011_UART0_CLK_IN_HZ 24000000 + +/* BootSRAM */ +#define BOOTSRAM_BASE 0x04200000 +#define BOOTSRAM_SIZE 0x1000 + +/* CPU Reset Control */ +#define CPU_CRG_BASE 0x04510000 +#define CPU_CRG_SIZE 0x1000 + +/* Sysctrl Register */ +#define SYS_CTRL_BASE 0x04520000 +#define SYS_CTRL_SIZE 0x1000 + +#endif /* __HI3519AV100_H__ */ diff --git a/optee_os/core/arch/arm/plat-hisilicon/hi3519av100_plat_init.S b/optee_os/core/arch/arm/plat-hisilicon/hi3519av100_plat_init.S new file mode 100644 index 0000000..bf11645 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hisilicon/hi3519av100_plat_init.S @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, HiSilicon Technologies Co., Ltd. + */ + +/* + * Entry points for the Hi3519AV100 a53 aarch32 mode init. + * It is assumed no stack is available when these routines are called. + * It is assumed each routine is called with return address in LR + * and with ARM registers R0, R1, R2, R3 being scratched. + */ + +#include +#include +#include +#include + +#define CCI_BASE 0x04528000 +#define CPUECTLR_A53_SMPEN BIT(6) +#define ACTRL_CPUECTLR BIT(1) +#define HACTRL_CPUECTLR BIT(1) + +.section .text +.balign 4 +.code 32 + +/* + * Hi3519AV100 a53 aarch32 configuration early configuration + * + * Use scratch registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + */ +FUNC plat_cpu_reset_early , : + /* + * Write the CPU Extended Control Register + * Set the SMPEN bit, this Cortex-A53 core's register + */ + mrrc p15, 1, r0, r1, c15 + orr r0, r0, #CPUECTLR_A53_SMPEN + mcrr p15, 1, r0, r1, c15 + + /* + * Enable Non-Secure EL1 write access to CPUECTLR + */ + mrs r1, cpsr + cps #CPSR_MODE_MON + + read_scr r0 + orr r0, r0, #SCR_NS /* Set NS bit in SCR */ + write_scr r0 + isb + + /* Write HACTLR register */ + mrc p15, 4, r2, c1, c0, 1 + orr r2, r2, #HACTRL_CPUECTLR + mcr p15, 4, r2, c1, c0, 1 + + bic r0, r0, #SCR_NS /* Clr NS bit in SCR */ + write_scr r0 + isb + + /* Write ACTLR register */ + mrc p15, 0, r2, c1, c0, 1 + orr r2, r2, #ACTRL_CPUECTLR + mcr p15, 0, r2, c1, c0, 1 + + msr cpsr, r1 + /* + * Enable cci for secondary core + */ + mov r3, lr + bl __get_core_pos + mov lr, r3 + cmp r0, #0 + beq out + ldr r0, =CCI_BASE + ldr r1, [r0] + orr r1, r1, #BIT(9) /* bit 9 set to 1 */ + str r1, [r0] +out: + bx lr +END_FUNC plat_cpu_reset_early diff --git a/optee_os/core/arch/arm/plat-hisilicon/main.c b/optee_os/core/arch/arm/plat-hisilicon/main.c new file mode 100644 index 0000000..4840b98 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hisilicon/main.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static struct pl011_data console_data; +register_phys_mem(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, PL011_REG_SIZE); +#ifdef BOOTSRAM_BASE +register_phys_mem(MEM_AREA_IO_SEC, BOOTSRAM_BASE, BOOTSRAM_SIZE); +#endif +#ifdef CPU_CRG_BASE +register_phys_mem(MEM_AREA_IO_SEC, CPU_CRG_BASE, CPU_CRG_SIZE); +#endif +#ifdef SYS_CTRL_BASE +register_phys_mem(MEM_AREA_IO_SEC, SYS_CTRL_BASE, SYS_CTRL_SIZE); +#endif + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-hisilicon/platform_config.h b/optee_os/core/arch/arm/plat-hisilicon/platform_config.h new file mode 100644 index 0000000..c5dc82b --- /dev/null +++ b/optee_os/core/arch/arm/plat-hisilicon/platform_config.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, HiSilicon Technologies Co., Ltd. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +/* PL011 UART */ +#define CONSOLE_UART_BASE PL011_UART0_BASE +#define CONSOLE_BAUDRATE PL011_BAUDRATE +#define CONSOLE_UART_CLK_IN_HZ PL011_UART0_CLK_IN_HZ + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-hisilicon/psci.c b/optee_os/core/arch/arm/plat-hisilicon/psci.c new file mode 100644 index 0000000..9009e22 --- /dev/null +++ b/optee_os/core/arch/arm/plat-hisilicon/psci.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REG_CPU_SUSSYS_RESET 0xcc +#define REG_CPU_START_COMMAND 0x0 +#define REG_CPU_START_ADDR 0x4 +#define REG_SYSCTRL_RESET 0x4 +#define RELEASE_CORE_MASK (BIT32(25) | BIT32(1)) + +int psci_features(uint32_t psci_fid) +{ + switch (psci_fid) { + case ARM_SMCCC_VERSION: + case PSCI_PSCI_FEATURES: + case PSCI_VERSION: + case PSCI_SYSTEM_RESET: +#ifdef CFG_BOOT_SECONDARY_REQUEST + case PSCI_CPU_ON: +#endif + return PSCI_RET_SUCCESS; + default: + return PSCI_RET_NOT_SUPPORTED; + } +} + +uint32_t psci_version(void) +{ + return PSCI_VERSION_1_0; +} + +void psci_system_reset(void) +{ + vaddr_t sysctrl = core_mmu_get_va(SYS_CTRL_BASE, MEM_AREA_IO_SEC, + SYS_CTRL_SIZE); + + if (!sysctrl) { + EMSG("no sysctrl mapping, hang here"); + panic(); + } + + io_write32(sysctrl + REG_SYSCTRL_RESET, 0xdeadbeef); +} + +#ifdef CFG_BOOT_SECONDARY_REQUEST +int psci_cpu_on(uint32_t core_idx, uint32_t entry, + uint32_t context_id) +{ + uint32_t val = 0; + size_t pos = get_core_pos_mpidr(core_idx); + vaddr_t bootsram = core_mmu_get_va(BOOTSRAM_BASE, MEM_AREA_IO_SEC, + BOOTSRAM_SIZE); + vaddr_t crg = core_mmu_get_va(CPU_CRG_BASE, MEM_AREA_IO_SEC, + CPU_CRG_SIZE); + + if (!bootsram || !crg) { + EMSG("No bootsram or crg mapping"); + return PSCI_RET_INVALID_PARAMETERS; + } + + if ((pos == 0) || (pos >= CFG_TEE_CORE_NB_CORE)) + return PSCI_RET_INVALID_PARAMETERS; + + /* set secondary core's NS entry addresses */ + boot_set_core_ns_entry(pos, entry, context_id); + + val = virt_to_phys((void *)TEE_LOAD_ADDR); + io_write32(bootsram + REG_CPU_START_ADDR, val); + io_write32(bootsram + REG_CPU_START_COMMAND, 0xe51ff004); + + /* release secondary core */ + io_clrbits32(crg + REG_CPU_SUSSYS_RESET, RELEASE_CORE_MASK); + + return PSCI_RET_SUCCESS; +} +#endif diff --git a/optee_os/core/arch/arm/plat-hisilicon/sub.mk b/optee_os/core/arch/arm/plat-hisilicon/sub.mk new file mode 100644 index 0000000..ed69d4c --- /dev/null +++ b/optee_os/core/arch/arm/plat-hisilicon/sub.mk @@ -0,0 +1,4 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-$(CFG_HI3519AV100) += hi3519av100_plat_init.S +srcs-$(CFG_PSCI_ARM32) += psci.c diff --git a/optee_os/core/arch/arm/plat-imx/a7_plat_init.S b/optee_os/core/arch/arm/plat-imx/a7_plat_init.S new file mode 100644 index 0000000..f96a18a --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/a7_plat_init.S @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Peng Fan + */ + +/* + * Entry points for the A7 init. + * It is assume no stack is available when these routines are called. + * It is assume each routine is called with return address in LR + * and with ARM registers R0, R1, R2, R3 being scratchable. + */ + +#include +#include +#include +#include +#include + +.section .text +.balign 4 +.code 32 + +/* + * Cortex A7 configuration early configuration + * + * Use scratables registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + */ +FUNC plat_cpu_reset_early , : + /* + * DDI: Disable dual issue [bit28=0] + * DDVM: Disable Distributed Virtual Memory transactions [bit15=0] + * L1PCTL: L1 Data prefetch control [bit14:13=2b11] + * L1RADIS: L1 Data Cache read-allocate mode disable [bit12=0] + * L2RADIS: L2 Data Cache read-allocate mode disable [bit11=0] + * DODMBS: Disable optimized data memory barrier behavior [bit10=0] + * SMP: Enables coherent requests to the processor [bit6=0] + */ + mov_imm r0, 0x00006040 + write_actlr r0 + + mov_imm r0, 0x00040C00 + write_nsacr r0 + + bx lr +END_FUNC plat_cpu_reset_early + +FUNC get_core_pos_mpidr , : + /* Drop ClusterId. There is no SoCs with more than 4 A7 Cores. */ + and r0, r0, #MPIDR_CPU_MASK + bx lr +END_FUNC get_core_pos_mpidr diff --git a/optee_os/core/arch/arm/plat-imx/a9_plat_init.S b/optee_os/core/arch/arm/plat-imx/a9_plat_init.S new file mode 100644 index 0000000..4050684 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/a9_plat_init.S @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * Copyright 2019 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Entry points for the A9 inits, A9 revision specific or not. + * It is assume no stack is available when these routines are called. + * It is assume each routine is called with return address in LR + * and with ARM registers R0, R1, R2, R3 being scratchable. + */ + +#include +#include +#include +#include +#include +#include + +.section .text +.balign 4 +.code 32 + +/* + * Cortex A9 early configuration + * + * Use registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + */ +FUNC plat_cpu_reset_early , : + /* + * Under very rare timing circumstances, transition into streaming + * mode might create a data corruption + * Configurations affected + * This erratum affects configurations with either: + * - One processor if the ACP is present + * - Two or more processors + * This erratum can be worked round by setting bit[22] of the + * undocumented Diagnostic Control Register to 1. This + * register is encoded as CP15 c15 0 c0 1. + * The bit can be written in Secure state only, with the following + * Read/Modify/Write code sequence: + * MRC p15,0,rt,c15,c0,1 + * ORR rt,rt,#0x00400000 + * MCR p15,0,rt,c15,c0,1 + * When this bit is set, the processor is unable to switch into + * Read-Allocate (streaming) mode, which means this erratum cannot + * occur. Setting this bit could possibly result in a visible drop + * in performance for routines that perform intensive memory + * accesses, such as memset() or memcpy(). However, the workaround + * is not expected to create any significant performance degradation + * in most standard applications. + */ +#if defined(CFG_MX6QP) || defined(CFG_MX6Q) || defined(CFG_MX6D) || \ + defined(CFG_MX6DL) + read_diag r0 + orr r0, r0, #1 << 22 + write_diag r0 +#endif + /* + * Disallow NSec to mask FIQ [bit4: FW=0] + * Allow NSec to manage Imprecise Abort [bit5: AW=1] + * Imprecise Abort trapped to Abort Mode [bit3: EA=0] + * In Sec world, FIQ trapped to FIQ Mode [bit2: FIQ=0] + * IRQ always trapped to IRQ Mode [bit1: IRQ=0] + * Secure World [bit0: NS=0] + */ + mov r0, #SCR_AW + write_scr r0 + + /* + * Mandated HW config loaded + * + * SCTLR = 0x00004000 + * - Round-Robin replac. for icache, btac, i/duTLB (bit14: RoundRobin) + * + * ACTRL = 0x000000[46(i.MX6SLL),47] + * - core always in full SMP (FW bit0=[0(i.MX6SLL),1], SMP bit6=1) + * - L2 write full line of zero disabled (bit3=0) + * (keep WFLZ low. Will be set once outer L2 is ready) + * - L1 Prefetch enable (bit2=1) + * - L2 Prefetch hint enable (bit1=1) + * + * NSACR = 0x00020C00 + * - NSec cannot change ACTRL.SMP (NS_SMP bit18=0) + * - Nsec can lockdown TLB (TL bit17=1) + * - NSec cannot access PLE (PLE bit16=0) + * - NSec can use SIMD/VFP (CP10/CP11) (bit15:14=2b00, bit11:10=2b11) + * + * PCR + * - no change latency, enable clk gating + */ + mov_imm r0, 0x00004000 + write_sctlr r0 + +#ifdef CFG_MX6SLL + mov_imm r0, 0x00000046 +#else + mov_imm r0, 0x00000047 +#endif + write_actlr r0 + + mov_imm r0, 0x00020C00 + write_nsacr r0 + + read_pcr r0 + orr r0, r0, #0x1 + write_pcr r0 + + mov pc, lr +END_FUNC plat_cpu_reset_early diff --git a/optee_os/core/arch/arm/plat-imx/conf.mk b/optee_os/core/arch/arm/plat-imx/conf.mk new file mode 100644 index 0000000..4e85e52 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/conf.mk @@ -0,0 +1,530 @@ +PLATFORM_FLAVOR ?= mx6ulevk + +# Get SoC associated with the PLATFORM_FLAVOR +mx6ul-flavorlist = \ + mx6ulevk \ + mx6ul9x9evk \ + mx6ulccimx6ulsbcpro \ + mx6ulccbv2 \ + +mx6ull-flavorlist = \ + mx6ullevk \ + mx6ulzevk \ + +mx6q-flavorlist = \ + mx6qsabrelite \ + mx6qsabreauto \ + mx6qsabresd \ + mx6qhmbedge \ + mx6qapalis \ + +mx6qp-flavorlist = \ + mx6qpsabreauto \ + mx6qpsabresd \ + +mx6sl-flavorlist = \ + mx6slevk + +mx6sll-flavorlist = \ + mx6sllevk + +mx6sx-flavorlist = \ + mx6sxsabreauto \ + mx6sxsabresd \ + mx6sxudooneofull \ + +mx6d-flavorlist = \ + mx6dhmbedge \ + mx6dapalis \ + +mx6dl-flavorlist = \ + mx6dlsabreauto \ + mx6dlsabresd \ + mx6dlhmbedge \ + +mx6s-flavorlist = \ + mx6shmbedge \ + mx6solosabresd \ + mx6solosabreauto \ + +mx7d-flavorlist = \ + mx7dsabresd \ + mx7dpico_mbl \ + mx7dclsom \ + +mx7s-flavorlist = \ + mx7swarp7 \ + mx7swarp7_mbl \ + +mx7ulp-flavorlist = \ + mx7ulpevk + +mx8mq-flavorlist = \ + mx8mqevk + +mx8mm-flavorlist = \ + mx8mmevk \ + mx8mm_cl_iot_gate + +mx8mn-flavorlist = \ + mx8mnevk + +mx8mp-flavorlist = \ + mx8mpevk \ + mx8mp_rsb3720_6g + +mx8qm-flavorlist = \ + mx8qmmek \ + +mx8qx-flavorlist = \ + mx8qxpmek \ + +mx8dxl-flavorlist = \ + mx8dxlevk \ + +mx8ulp-flavorlist = \ + mx8ulpevk \ + +mx93-flavorlist = \ + mx93evk \ + +ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6ul-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6UL,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_TZC380,y) +include core/arch/arm/cpu/cortex-a7.mk +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6ull-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6ULL,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_TZC380,y) +$(call force,CFG_IMX_CAAM,n) +$(call force,CFG_NXP_CAAM,n) +$(call force,CFG_IMX_DCP,y) +include core/arch/arm/cpu/cortex-a7.mk +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6q-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6Q,y) +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_TZC380,y) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6qp-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6QP,y) +$(call force,CFG_TEE_CORE_NB_CORE,4) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6d-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6D,y) +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_TZC380,y) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6dl-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6DL,y) +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_TZC380,y) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6s-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6S,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6sl-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6SL,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_IMX_CAAM,n) +$(call force,CFG_NXP_CAAM,n) +$(call force,CFG_IMX_DCP,y) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6sll-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6SLL,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_IMX_CAAM,n) +$(call force,CFG_NXP_CAAM,n) +$(call force,CFG_IMX_DCP,y) +$(call force,CFG_NO_SMP,y) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6sx-flavorlist))) +$(call force,CFG_MX6,y) +$(call force,CFG_MX6SX,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx7s-flavorlist))) +$(call force,CFG_MX7,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +include core/arch/arm/cpu/cortex-a7.mk +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx7d-flavorlist))) +$(call force,CFG_MX7,y) +$(call force,CFG_TEE_CORE_NB_CORE,2) +include core/arch/arm/cpu/cortex-a7.mk +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx7ulp-flavorlist))) +$(call force,CFG_MX7ULP,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_TZC380,n) +$(call force,CFG_IMX_CSU,n) +include core/arch/arm/cpu/cortex-a7.mk +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8mq-flavorlist))) +$(call force,CFG_MX8MQ,y) +$(call force,CFG_MX8M,y) +$(call force,CFG_ARM64_core,y) +CFG_DRAM_BASE ?= 0x40000000 +CFG_TEE_CORE_NB_CORE ?= 4 +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8mm-flavorlist))) +$(call force,CFG_MX8MM,y) +$(call force,CFG_MX8M,y) +$(call force,CFG_ARM64_core,y) +CFG_DRAM_BASE ?= 0x40000000 +CFG_TEE_CORE_NB_CORE ?= 4 +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8mn-flavorlist))) +$(call force,CFG_MX8MN,y) +$(call force,CFG_MX8M,y) +$(call force,CFG_ARM64_core,y) +CFG_DRAM_BASE ?= 0x40000000 +CFG_TEE_CORE_NB_CORE ?= 4 +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8mp-flavorlist))) +$(call force,CFG_MX8MP,y) +$(call force,CFG_MX8M,y) +$(call force,CFG_ARM64_core,y) +CFG_DRAM_BASE ?= 0x40000000 +CFG_TEE_CORE_NB_CORE ?= 4 +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8qm-flavorlist))) +$(call force,CFG_MX8QM,y) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_IMX_SNVS,n) +CFG_IMX_LPUART ?= y +CFG_DRAM_BASE ?= 0x80000000 +CFG_TEE_CORE_NB_CORE ?= 6 +$(call force,CFG_IMX_OCOTP,n) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8qx-flavorlist))) +$(call force,CFG_MX8QX,y) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_IMX_SNVS,n) +CFG_IMX_LPUART ?= y +CFG_DRAM_BASE ?= 0x80000000 +CFG_TEE_CORE_NB_CORE ?= 4 +$(call force,CFG_IMX_OCOTP,n) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8dxl-flavorlist))) +$(call force,CFG_MX8DXL,y) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_IMX_SNVS,n) +CFG_IMX_LPUART ?= y +CFG_DRAM_BASE ?= 0x80000000 +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_IMX_OCOTP,n) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8ulp-flavorlist))) +$(call force,CFG_MX8ULP,y) +$(call force,CFG_ARM64_core,y) +CFG_IMX_LPUART ?= y +CFG_DRAM_BASE ?= 0x80000000 +CFG_TEE_CORE_NB_CORE ?= 2 +$(call force,CFG_NXP_SNVS,n) +$(call force,CFG_IMX_OCOTP,n) +CFG_IMX_MU ?= y +CFG_IMX_ELE ?= y +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx93-flavorlist))) +$(call force,CFG_MX93,y) +$(call force,CFG_ARM64_core,y) +CFG_IMX_LPUART ?= y +CFG_DRAM_BASE ?= 0x80000000 +CFG_TEE_CORE_NB_CORE ?= 2 +$(call force,CFG_NXP_SNVS,n) +$(call force,CFG_IMX_OCOTP,n) +$(call force,CFG_TZC380,n) +$(call force,CFG_CRYPTO_DRIVER,n) +$(call force,CFG_NXP_CAAM,n) +CFG_IMX_MU ?= y +CFG_IMX_ELE ?= y +else +$(error Unsupported PLATFORM_FLAVOR "$(PLATFORM_FLAVOR)") +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx7dsabresd)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_NS_ENTRY_ADDR ?= 0x80800000 +CFG_IMX_WDOG_EXT_RESET ?= y +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx7dclsom)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_UART_BASE ?= UART1_BASE +CFG_IMX_WDOG_EXT_RESET ?= y +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx7dpico_mbl)) +CFG_DDR_SIZE ?= 0x20000000 +CFG_NS_ENTRY_ADDR ?= 0x87800000 +CFG_DT_ADDR ?= 0x83100000 +CFG_UART_BASE ?= UART5_BASE +CFG_BOOT_SECONDARY_REQUEST ?= n +CFG_EXTERNAL_DTB_OVERLAY ?= y +CFG_IMX_WDOG_EXT_RESET ?= y +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx7swarp7)) +CFG_DDR_SIZE ?= 0x20000000 +CFG_NS_ENTRY_ADDR ?= 0x80800000 +CFG_BOOT_SECONDARY_REQUEST ?= n +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx7swarp7_mbl)) +CFG_DDR_SIZE ?= 0x20000000 +CFG_NS_ENTRY_ADDR ?= 0x87800000 +CFG_DT_ADDR ?= 0x83100000 +CFG_BOOT_SECONDARY_REQUEST ?= n +CFG_EXTERNAL_DTB_OVERLAY = y +CFG_IMX_WDOG_EXT_RESET = y +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx7ulpevk)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_NS_ENTRY_ADDR ?= 0x60800000 +CFG_UART_BASE ?= UART4_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6qpsabresd mx6qsabresd mx6dlsabresd \ + mx6dlsabrelite mx6dhmbedge mx6dlhmbedge mx6solosabresd \ + mx6dapalis mx6qapalis)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_NS_ENTRY_ADDR ?= 0x12000000 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6qpsabreauto mx6qsabreauto \ + mx6dlsabreauto mx6solosabreauto)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_NS_ENTRY_ADDR ?= 0x12000000 +CFG_UART_BASE ?= UART4_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6qhmbedge)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_UART_BASE ?= UART1_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6shmbedge)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_NS_ENTRY_ADDR ?= 0x12000000 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6qsabrelite mx6dlsabrelite)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_NS_ENTRY_ADDR ?= 0x12000000 +CFG_UART_BASE ?= UART2_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6slevk)) +CFG_NS_ENTRY_ADDR ?= 0x80800000 +CFG_DDR_SIZE ?= 0x40000000 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6sllevk)) +CFG_NS_ENTRY_ADDR ?= 0x80800000 +CFG_DDR_SIZE ?= 0x80000000 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6sxsabreauto)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_NS_ENTRY_ADDR ?= 0x80800000 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6sxsabresd)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_NS_ENTRY_ADDR ?= 0x80800000 +endif + +ifeq ($(PLATFORM_FLAVOR), mx6sxudooneofull) +CFG_DDR_SIZE ?= 0x40000000 +CFG_UART_BASE ?= UART1_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6ulevk mx6ullevk mx6ulzevk)) +CFG_DDR_SIZE ?= 0x20000000 +CFG_NS_ENTRY_ADDR ?= 0x80800000 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6ulccimx6ulsbcpro)) +CFG_DDR_SIZE ?= 0x10000000 +CFG_NS_ENTRY_ADDR ?= 0x80800000 +CFG_UART_BASE ?= UART5_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6ul9x9evk)) +CFG_DDR_SIZE ?= 0x10000000 +CFG_NS_ENTRY_ADDR ?= 0x80800000 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx6ulccbv2)) +CFG_DDR_SIZE ?= 0x10000000 +CFG_UART_BASE ?= UART7_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8mqevk)) +CFG_DDR_SIZE ?= 0xc0000000 +CFG_UART_BASE ?= UART1_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8mmevk)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_UART_BASE ?= UART2_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8mm_cl_iot_gate)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_UART_BASE ?= UART3_BASE +CFG_NSEC_DDR_1_BASE ?= 0x80000000UL +CFG_NSEC_DDR_1_SIZE ?= 0x40000000UL +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8mnevk)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_UART_BASE ?= UART2_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8mpevk)) +CFG_DDR_SIZE ?= UL(0x180000000) +CFG_UART_BASE ?= UART2_BASE +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8mp_rsb3720_6g)) +CFG_DDR_SIZE ?= UL(0x180000000) +CFG_UART_BASE ?= UART3_BASE +CFG_TZDRAM_START ?= 0x56000000 +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8qxpmek mx8qmmek)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_UART_BASE ?= UART0_BASE +CFG_NSEC_DDR_1_BASE ?= 0x880000000UL +CFG_NSEC_DDR_1_SIZE ?= 0x380000000UL +CFG_CORE_ARM64_PA_BITS ?= 40 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8dxlevk)) +CFG_DDR_SIZE ?= 0x40000000 +CFG_UART_BASE ?= UART0_BASE +CFG_NSEC_DDR_1_BASE ?= 0x800000000UL +CFG_NSEC_DDR_1_SIZE ?= 0x400000000UL +CFG_CORE_ARM64_PA_BITS ?= 40 +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx8ulpevk)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_UART_BASE ?= UART5_BASE +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),mx93evk)) +CFG_DDR_SIZE ?= 0x80000000 +CFG_UART_BASE ?= UART1_BASE +endif + +# i.MX6 Solo/SL/SoloX/DualLite/Dual/Quad specific config +ifeq ($(filter y, $(CFG_MX6QP) $(CFG_MX6Q) $(CFG_MX6D) $(CFG_MX6DL) $(CFG_MX6S) \ + $(CFG_MX6SL) $(CFG_MX6SLL) $(CFG_MX6SX)), y) +include core/arch/arm/cpu/cortex-a9.mk + +$(call force,CFG_PL310,y) + +CFG_PL310_LOCKED ?= y +CFG_ENABLE_SCTLR_RR ?= y +CFG_IMX_SCU ?= y +endif + +ifeq ($(filter y, $(CFG_MX6QP) $(CFG_MX6Q) $(CFG_MX6D) $(CFG_MX6DL) $(CFG_MX6S)), y) +CFG_DRAM_BASE ?= 0x10000000 +endif + +ifneq (,$(filter y, $(CFG_MX6UL) $(CFG_MX6ULL) $(CFG_MX6SL) $(CFG_MX6SLL) \ + $(CFG_MX6SX))) +CFG_DRAM_BASE ?= 0x80000000 +endif + +ifeq ($(filter y, $(CFG_MX7)), y) +CFG_INIT_CNTVOFF ?= y +CFG_DRAM_BASE ?= 0x80000000 +endif + +ifeq ($(filter y, $(CFG_MX7ULP)), y) +CFG_INIT_CNTVOFF ?= y +CFG_DRAM_BASE ?= UL(0x60000000) +$(call force,CFG_IMX_LPUART,y) +$(call force,CFG_BOOT_SECONDARY_REQUEST,n) +endif + +ifneq (,$(filter y, $(CFG_MX6) $(CFG_MX7) $(CFG_MX7ULP))) +$(call force,CFG_GIC,y) + +CFG_BOOT_SECONDARY_REQUEST ?= y +CFG_DT ?= y +CFG_DTB_MAX_SIZE ?= 0x20000 +CFG_PAGEABLE_ADDR ?= 0 +CFG_PSCI_ARM32 ?= y +CFG_SECURE_TIME_SOURCE_REE ?= y +CFG_UART_BASE ?= UART1_BASE +endif + +ifneq (,$(filter y, $(CFG_MX6) $(CFG_MX7) $(CFG_MX8M))) +$(call force,CFG_IMX_UART,y) +CFG_IMX_SNVS ?= y +endif + +ifneq (,$(filter y, $(CFG_MX6) $(CFG_MX7))) +CFG_IMX_CSU ?= y +endif + +ifeq ($(filter y, $(CFG_PSCI_ARM32)), y) +CFG_HWSUPP_MEM_PERM_WXN = n +CFG_IMX_WDOG ?= y +endif + +ifeq ($(CFG_ARM64_core),y) +# arm-v8 platforms +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_GIC,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) + +CFG_CRYPTO_WITH_CE ?= y + +supported-ta-targets = ta_arm64 +endif + +CFG_TZDRAM_SIZE ?= 0x01e00000 +CFG_SHMEM_SIZE ?= 0x00200000 +CFG_TZDRAM_START ?= ($(CFG_DRAM_BASE) - $(CFG_TZDRAM_SIZE) - $(CFG_SHMEM_SIZE) + $(CFG_DDR_SIZE)) +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) + +# Enable embedded tests by default +CFG_ENABLE_EMBEDDED_TESTS ?= y + +# Set default heap size for imx platforms to 128k +CFG_CORE_HEAP_SIZE ?= 131072 + +CFG_CRYPTO_SIZE_OPTIMIZATION ?= n +CFG_MMAP_REGIONS ?= 24 + +# SE05X and OCOTP both implement tee_otp_get_die_id() +ifeq ($(CFG_NXP_SE05X),y) +$(call force,CFG_IMX_OCOTP,n) +endif +CFG_IMX_OCOTP ?= y +CFG_IMX_DIGPROG ?= y +CFG_PKCS11_TA ?= y + +# Almost all platforms include CAAM HW Modules, except the +# ones forced to be disabled +CFG_NXP_CAAM ?= n + +ifeq ($(CFG_NXP_CAAM),y) +ifeq ($(filter y, $(CFG_MX8QM) $(CFG_MX8QX) $(CFG_MX8DXL)), y) +CFG_IMX_SC ?= y +CFG_IMX_MU ?= y +endif + +else + +ifneq (,$(filter y, $(CFG_MX6) $(CFG_MX7) $(CFG_MX7ULP))) +CFG_IMX_CAAM ?= y +endif + +endif + diff --git a/optee_os/core/arch/arm/plat-imx/config/imx6qdlsolo.h b/optee_os/core/arch/arm/plat-imx/config/imx6qdlsolo.h new file mode 100644 index 0000000..45bbff8 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/config/imx6qdlsolo.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * Copyright 2017-2019 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef CONFIG_IMX6QDLSOLO_H +#define CONFIG_IMX6QDLSOLO_H + +/* + * PL310 TAG RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_TAG_RAM_CTRL_INIT +#define PL310_TAG_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 DATA RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_DATA_RAM_CTRL_INIT +#define PL310_DATA_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 Auxiliary Control Register + */ +#ifndef PL310_AUX_CTRL_INIT +#if defined(CFG_MX6QP) || defined(CFG_MX6Q) || defined(CFG_MX6D) +/* + * Early BRESP enabled (bit30=1) + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockown cache lines (bit26=1) + * Pseudo-random replacement policy (bit25=1) + * Force write allocated (default) (bit24:23=00) + * Shared attribute internally ignored (bit22=1, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * 64kb way size (bit19:17=3b011) + * 16-way associativity (bit16=1) + * Store buffer device limitation disabled (bit11=0) + * Cacheable accesses have high prio (bit10=0) + * Full Line Zero (FLZ) enabled (bit0=1) + */ +#define PL310_AUX_CTRL_INIT 0x7E470001 +#else +/* + * Early BRESP enabled (bit30=0) + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockown cache lines (bit26=1) + * Pseudo-random replacement policy (bit25=0) + * Force write allocated (default) (bit24:23=00) + * Shared attribute internally ignored (bit22=1, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * 32kb way size (bit19:17=3b010) + * 8-way associativity (bit16=0) + * Store buffer device limitation enabled (bit11=1) + * Cacheable accesses have high prio (bit10=0) + * Full Line Zero (FLZ) disabled (bit0=0) + */ +#define PL310_AUX_CTRL_INIT 0x3C440800 +#endif +#endif + +/* + * PL310 Prefetch Control Register + * + * Double linefill enabled (bit30=1) + * I/D prefetch enabled (bit29:28=2b11) + * Prefetch drop disabled (bit24=0) + * Incr double linefill disable (bit23=0) + * Prefetch offset = 0xF (bit4:0) + */ +#define PL310_PREFETCH_CTRL_INIT 0x7000000F + +/* + * PL310 Power Register + * + * Dynamic clock gating enabled + * Standby mode enabled + */ +#define PL310_POWER_CTRL_INIT 0x00000003 +#endif diff --git a/optee_os/core/arch/arm/plat-imx/config/imx6sl.h b/optee_os/core/arch/arm/plat-imx/config/imx6sl.h new file mode 100644 index 0000000..f781892 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/config/imx6sl.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ + +#ifndef _CONFIG_IMX6SL_H +#define _CONFIG_IMX6SL_H + +/* + * PL310 TAG RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_TAG_RAM_CTRL_INIT +#define PL310_TAG_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 DATA RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_DATA_RAM_CTRL_INIT +#define PL310_DATA_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 Auxiliary Control Register + * + * Early BRESP enabled (bit30=1) + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockown cache lines (bit26=1) + * Pseudo-random replacement policy (bit25=1) + * Force write allocated (default) (bit24:23=00) + * Shared attribute internally ignored (bit22=1, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * 16kb way size (bit19:17=3b001) + * 16-way associativity (bit16=1) + * Store buffer device limitation enabled (bit11=0) + * Cacheable accesses have high prio (bit10=0) + * Full Line Zero (FLZ) enabled (bit0=1) + */ +#ifndef PL310_AUX_CTRL_INIT +#define PL310_AUX_CTRL_INIT 0x7E430001 +#endif + +/* + * PL310 Prefetch Control Register + * + * Double linefill enabled (bit30=1) + * I/D prefetch enabled (bit29:28=2b11) + * Prefetch drop disabled (bit24=0) + * Incr double linefill disable (bit23=0) + * Prefetch offset = 0xF (bit4:0) + */ +#define PL310_PREFETCH_CTRL_INIT 0x7000000F + +/* + * PL310 Power Register + * + * Dynamic clock gating enabled + * Standby mode enabled + */ +#define PL310_POWER_CTRL_INIT 0x00000003 +#endif diff --git a/optee_os/core/arch/arm/plat-imx/config/imx6sll.h b/optee_os/core/arch/arm/plat-imx/config/imx6sll.h new file mode 100644 index 0000000..048f748 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/config/imx6sll.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ + +#ifndef _CONFIG_IMX6SLL_H +#define _CONFIG_IMX6SLL_H + +/* + * PL310 TAG RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_TAG_RAM_CTRL_INIT +#define PL310_TAG_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 DATA RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_DATA_RAM_CTRL_INIT +#define PL310_DATA_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 Auxiliary Control Register + * + * Early BRESP enabled (bit30=1) + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockown cache lines (bit26=1) + * Pseudo-random replacement policy (bit25=1) + * Force write allocated (default) (bit24:23=00) + * Shared attribute internally ignored (bit22=1, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * 16kb way size (bit19:17=3b001) + * 16-way associativity (bit16=1) + * Store buffer device limitation enabled (bit11=0) + * Cacheable accesses have high prio (bit10=0) + * Full Line Zero (FLZ) enabled (bit0=1) + */ +#ifndef PL310_AUX_CTRL_INIT +#define PL310_AUX_CTRL_INIT 0x7E430001 +#endif + +/* + * PL310 Prefetch Control Register + * + * Double linefill enabled (bit30=1) + * I/D prefetch enabled (bit29:28=2b11) + * Prefetch drop disabled (bit24=0) + * Incr double linefill disable (bit23=0) + * Prefetch offset = 0xF (bit4:0) + */ +#define PL310_PREFETCH_CTRL_INIT 0x7000000F + +/* + * PL310 Power Register + * + * Dynamic clock gating enabled + * Standby mode enabled + */ +#define PL310_POWER_CTRL_INIT 0x00000003 +#endif diff --git a/optee_os/core/arch/arm/plat-imx/config/imx6sx.h b/optee_os/core/arch/arm/plat-imx/config/imx6sx.h new file mode 100644 index 0000000..2636855 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/config/imx6sx.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Peng Fan + */ + +#ifndef __CONFIG_IMX6SX_H +#define __CONFIG_IMX6SX_H + +/* + * PL310 TAG RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_TAG_RAM_CTRL_INIT +#define PL310_TAG_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 DATA RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:3 - 4 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_DATA_RAM_CTRL_INIT +#define PL310_DATA_RAM_CTRL_INIT 0x00000132 +#endif + +/* + * PL310 Auxiliary Control Register + * + * Early BRESP enabled (bit30=1) + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockown cache lines (bit26=1) + * Pseudo-random replacement policy (bit25=1) + * Force write allocated (default) (bit24:23=00) + * Shared attribute internally ignored (bit22=1, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * 16kb way size (bit19:17=3b001) + * 16-way associativity (bit16=1) + * Store buffer device limitation enabled (bit11=0) + * Cacheable accesses have high prio (bit10=0) + * Full Line Zero (FLZ) enabled (bit0=1) + */ +#ifndef PL310_AUX_CTRL_INIT +#define PL310_AUX_CTRL_INIT 0x7E430001 +#endif + +/* + * PL310 Prefetch Control Register + * + * Double linefill enabled (bit30=1) + * I/D prefetch enabled (bit29:28=2b11) + * Prefetch drop disabled (bit24=0) + * Incr double linefill disable (bit23=0) + * Prefetch offset = 0xF (bit4:0) + */ +#define PL310_PREFETCH_CTRL_INIT 0x7000000F + +/* + * PL310 Power Register + * + * Dynamic clock gating enabled + * Standby mode enabled + */ +#define PL310_POWER_CTRL_INIT 0x00000003 +#endif diff --git a/optee_os/core/arch/arm/plat-imx/imx-common.c b/optee_os/core/arch/arm/plat-imx/imx-common.c new file mode 100644 index 0000000..7339137 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/imx-common.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2016 Freescale Semiconductor, Inc. + * Copyright 2017-2019, 2021 NXP + * + * Peng Fan + */ + +#include +#include +#include +#include +#include +#include +#include + +#define SOC_TYPE(reg) (((reg) & (0x00FF0000)) >> 16) +#define SOC_REV_MAJOR(reg) (((reg) & (0x0000FF00)) >> 8) +#define SOC_REV_MINOR(reg) ((reg) & (0x0000000F)) +#define SOC_REV_MINOR_MX7(reg) ((reg) & (0x000000FF)) + +static uint32_t imx_digprog; + +#ifdef ANATOP_BASE +uint32_t imx_get_digprog(void) +{ + vaddr_t addr = 0; + + if (imx_digprog) + return imx_digprog; + + addr = core_mmu_get_va(ANATOP_BASE, MEM_AREA_IO_SEC, 0x1000); + if (!addr) + return 0; + + imx_digprog = io_read32(addr + DIGPROG_OFFSET); + +#ifdef CFG_MX8MQ + /* + * On the i.MX8MQ, the minor revision number must be updated to make + * the difference between B0 chip and the newer chips. + */ + addr = core_mmu_get_va(OCOTP_BASE, MEM_AREA_IO_SEC, OCOTP_SIZE); + if (!addr) + return 0; + + if (io_read32(addr + OCOTP_SW_INFO_B1) == OCOTP_SW_MAGIC_B1) + imx_digprog |= BIT32(0); +#endif /* CFG_MX8MQ */ + + return imx_digprog; +} +#else /* ANATOP_BASE */ +uint32_t imx_get_digprog(void) +{ + if (imx_digprog) + return imx_digprog; + + if (IS_ENABLED(CFG_MX7ULP)) + imx_digprog = SOC_MX7ULP << 16; + else if (IS_ENABLED(CFG_MX8QX)) + imx_digprog = SOC_MX8QX << 16; + else if (IS_ENABLED(CFG_MX8QM)) + imx_digprog = SOC_MX8QM << 16; + else if (IS_ENABLED(CFG_MX8DXL)) + imx_digprog = SOC_MX8DXL << 16; + else if (IS_ENABLED(CFG_MX8ULP)) + imx_digprog = SOC_MX8ULP << 16; + else if (IS_ENABLED(CFG_MX93)) + imx_digprog = SOC_MX93 << 16; + + return imx_digprog; +} +#endif /* ANATOP_BASE */ + +uint32_t imx_soc_rev_major(void) +{ + if (imx_digprog == 0) + imx_get_digprog(); + + return SOC_REV_MAJOR(imx_digprog); +} + +uint32_t imx_soc_rev_minor(void) +{ + if (imx_digprog == 0) + imx_get_digprog(); + + if (IS_ENABLED(CFG_MX7)) + return SOC_REV_MINOR_MX7(imx_digprog); + else + return SOC_REV_MINOR(imx_digprog); +} + +uint32_t imx_soc_type(void) +{ + if (imx_digprog == 0) + imx_get_digprog(); + + return SOC_TYPE(imx_digprog); +} + +bool soc_is_imx6sl(void) +{ + return imx_soc_type() == SOC_MX6SL; +} + +bool soc_is_imx6sll(void) +{ + return imx_soc_type() == SOC_MX6SLL; +} + +bool soc_is_imx6sx(void) +{ + return imx_soc_type() == SOC_MX6SX; +} + +bool soc_is_imx6ul(void) +{ + return imx_soc_type() == SOC_MX6UL; +} + +bool soc_is_imx6ull(void) +{ + return imx_soc_type() == SOC_MX6ULL; +} + +bool soc_is_imx6sdl(void) +{ + return imx_soc_type() == SOC_MX6DL; +} + +bool soc_is_imx6dq(void) +{ + return (imx_soc_type() == SOC_MX6Q) && (imx_soc_rev_major() == 0); +} + +bool soc_is_imx6dqp(void) +{ + return (imx_soc_type() == SOC_MX6Q) && (imx_soc_rev_major() == 1); +} + +bool soc_is_imx6(void) +{ + uint32_t soc = imx_soc_type(); + + return (soc == SOC_MX6SLL) || (soc == SOC_MX6SL) || + (soc == SOC_MX6D) || (soc == SOC_MX6SX) || + (soc == SOC_MX6UL) || (soc == SOC_MX6ULL) || + (soc == SOC_MX6DL) || (soc == SOC_MX6Q); +} + +bool soc_is_imx7ds(void) +{ + return imx_soc_type() == SOC_MX7D; +} + +bool soc_is_imx7ulp(void) +{ + return imx_soc_type() == SOC_MX7ULP; +} + +bool soc_is_imx8mq(void) +{ + return imx_soc_type() == SOC_MX8M && imx_soc_rev_major() == 0x40; +} + +bool soc_is_imx8mm(void) +{ + return imx_soc_type() == SOC_MX8M && imx_soc_rev_major() == 0x41; +} + +bool soc_is_imx8mn(void) +{ + return imx_soc_type() == SOC_MX8M && imx_soc_rev_major() == 0x42; +} + +bool soc_is_imx8mp(void) +{ + return imx_soc_type() == SOC_MX8M && imx_soc_rev_major() == 0x43; +} + +bool soc_is_imx8m(void) +{ + return soc_is_imx8mq() || soc_is_imx8mm() || soc_is_imx8mn() || + soc_is_imx8mp(); +} + +bool soc_is_imx8mq_b0_layer(void) +{ + if (soc_is_imx8mq() && imx_soc_rev_minor() == 0x0) + return true; + else + return false; +} diff --git a/optee_os/core/arch/arm/plat-imx/imx-regs.h b/optee_os/core/arch/arm/plat-imx/imx-regs.h new file mode 100644 index 0000000..f846286 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/imx-regs.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * Copyright 2019, 2023 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef PLAT_IMX_IMX_REGS_H +#define PLAT_IMX_IMX_REGS_H + +#ifdef CFG_MX6 +#include +#elif defined(CFG_MX7) +#include +#elif defined(CFG_MX7ULP) +#include +#elif defined(CFG_MX8MQ) || defined(CFG_MX8MM) || defined(CFG_MX8MN) || \ + defined(CFG_MX8MP) +#include +#elif defined(CFG_MX8QX) || defined(CFG_MX8QM) || defined(CFG_MX8DXL) +#include +#elif defined(CFG_MX8ULP) +#include +#elif defined(CFG_MX93) +#include +#else +#error "CFG_MX* not defined" +#endif +#endif diff --git a/optee_os/core/arch/arm/plat-imx/imx.h b/optee_os/core/arch/arm/plat-imx/imx.h new file mode 100644 index 0000000..dfdee0b --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/imx.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Peng Fan + */ +#ifndef PLAT_IMX_IMX_H +#define PLAT_IMX_IMX_H + +#include +#include + +#define SOC_MX6SL 0x60 +#define SOC_MX6DL 0x61 +#define SOC_MX6SX 0x62 +#define SOC_MX6Q 0x63 +#define SOC_MX6UL 0x64 +#define SOC_MX6ULL 0x65 +#define SOC_MX6SLL 0x67 +#define SOC_MX6D 0x6A +#define SOC_MX7D 0x72 +#define SOC_MX7ULP 0xE1 +#define SOC_MX8QX 0xE2 +#define SOC_MX8QM 0xE3 +#define SOC_MX8DXL 0xE4 +#define SOC_MX8M 0x82 +#define SOC_MX8ULP 0x83 +#define SOC_MX93 0xC1 + +#ifndef __ASSEMBLER__ +bool soc_is_imx6(void); +bool soc_is_imx6sx(void); +bool soc_is_imx6sl(void); +bool soc_is_imx6sll(void); +bool soc_is_imx6ul(void); +bool soc_is_imx6ull(void); +bool soc_is_imx6sdl(void); +bool soc_is_imx6dq(void); +bool soc_is_imx6dqp(void); +bool soc_is_imx7ds(void); +bool soc_is_imx7ulp(void); +bool soc_is_imx8m(void); +bool soc_is_imx8mq(void); +bool soc_is_imx8mm(void); +bool soc_is_imx8mn(void); +bool soc_is_imx8mp(void); +bool soc_is_imx8mq_b0_layer(void); +uint32_t imx_soc_type(void); +uint32_t imx_soc_rev_major(void); +uint32_t imx_soc_rev_minor(void); +uint32_t imx_get_digprog(void); +#endif /* __ASSEMBLER__ */ +#endif diff --git a/optee_os/core/arch/arm/plat-imx/imx_pl310.c b/optee_os/core/arch/arm/plat-imx/imx_pl310.c new file mode 100644 index 0000000..202d64b --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/imx_pl310.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2016 Freescale Semiconductor, Inc. + * + * Peng Fan + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "imx_pl310.h" + +#define PL310_AUX_CTRL_FLZW BIT(0) +#define PL310_DEBUG_CTRL_DISABLE_WRITEBACK BIT(1) +#define PL310_DEBUG_CTRL_DISABLE_LINEFILL BIT(0) +#define PL310_PREFETCH_DOUBLE_LINEFILL BIT(30) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, PL310_BASE, CORE_MMU_PGDIR_SIZE); + +void arm_cl2_config(vaddr_t pl310_base) +{ + uint32_t val = 0; + uint32_t id = 0; + + /* Disable PL310 */ + io_write32(pl310_base + PL310_CTRL, 0); + + io_write32(pl310_base + PL310_TAG_RAM_CTRL, PL310_TAG_RAM_CTRL_INIT); + io_write32(pl310_base + PL310_DATA_RAM_CTRL, PL310_DATA_RAM_CTRL_INIT); + io_write32(pl310_base + PL310_AUX_CTRL, PL310_AUX_CTRL_INIT); + /* + * The L2 cache controller(PL310) version on the i.MX6D/Q + * is r3p1-50rel0 + * The L2 cache controller(PL310) version on the + * i.MX6DL/SOLO/SL/SX/DQP is r3p2. + * + * According to ARM PL310 errata: 752271 + * ID: 752271: Double linefill feature can cause data corruption + * Fault Status: Present in: r3p0, r3p1, r3p1-50rel0. Fixed in r3p2 + * Workaround: The only workaround to this erratum is to disable the + * double linefill feature. This is the default behavior. + */ + val = PL310_PREFETCH_CTRL_INIT; + + id = io_read32(pl310_base + PL310_CACHE_ID); + + if (((id & PL310_CACHE_ID_PART_MASK) == PL310_CACHE_ID_PART_L310) && + ((id & PL310_CACHE_ID_RTL_MASK) < PL310_CACHE_ID_RTL_R3P2)) + val &= ~PL310_PREFETCH_DOUBLE_LINEFILL; + + io_write32(pl310_base + PL310_PREFETCH_CTRL, val); + + io_write32(pl310_base + PL310_POWER_CTRL, PL310_POWER_CTRL_INIT); + + /* invalidate all cache ways */ + arm_cl2_invbyway(pl310_base); +} + +void arm_cl2_enable(vaddr_t pl310_base) +{ + uint32_t val __maybe_unused; + + /* Enable PL310 ctrl -> only set lsb bit */ + io_write32(pl310_base + PL310_CTRL, 1); + +#ifndef CFG_PL310_SIP_PROTOCOL + /* if L2 FLZW enable, enable in L1 */ + val = io_read32(pl310_base + PL310_AUX_CTRL); + if (val & PL310_AUX_CTRL_FLZW) + write_actlr(read_actlr() | ACTLR_CA9_WFLZ); +#endif +} + +vaddr_t pl310_base(void) +{ + return core_mmu_get_va(PL310_BASE, MEM_AREA_IO_SEC, 1); +} + +#ifdef CFG_PL310_SIP_PROTOCOL +uint32_t pl310_enable(void) +{ + vaddr_t base = pl310_base(); + + arm_cl2_config(base); + arm_cl2_enable(base); + return OPTEE_SMC_RETURN_OK; +} + +uint32_t pl310_disable(void) +{ + EMSG("not implemented"); + return OPTEE_SMC_RETURN_ENOTAVAIL; +} + +uint32_t pl310_enable_writeback(void) +{ + vaddr_t base = pl310_base(); + + io_write32(base + PL310_DEBUG_CTRL, 0); + return OPTEE_SMC_RETURN_OK; +} + +uint32_t pl310_disable_writeback(void) +{ + vaddr_t base = pl310_base(); + uint32_t val = PL310_DEBUG_CTRL_DISABLE_WRITEBACK | + PL310_DEBUG_CTRL_DISABLE_LINEFILL; + + io_write32(base + PL310_DEBUG_CTRL, val); + return OPTEE_SMC_RETURN_OK; +} + +uint32_t pl310_enable_wflz(void) +{ + write_actlr(read_actlr() | ACTLR_CA9_WFLZ); + return OPTEE_SMC_RETURN_OK; +} +#endif diff --git a/optee_os/core/arch/arm/plat-imx/imx_pl310.h b/optee_os/core/arch/arm/plat-imx/imx_pl310.h new file mode 100644 index 0000000..4fef57b --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/imx_pl310.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + */ +#ifndef __IMX_PL310_H__ +#define __IMX_PL310_H__ + +uint32_t pl310_enable(void); +uint32_t pl310_disable(void); +uint32_t pl310_enable_writeback(void); +uint32_t pl310_disable_writeback(void); +uint32_t pl310_enable_wflz(void); + +#endif /* __IMX_PL310_H__ */ + diff --git a/optee_os/core/arch/arm/plat-imx/link.mk b/optee_os/core/arch/arm/plat-imx/link.mk new file mode 100644 index 0000000..c5a3673 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/link.mk @@ -0,0 +1,9 @@ +include core/arch/arm/kernel/link.mk + +.PHONY: uTee +uTee: $(link-out-dir)/uTee +cleanfiles += $(link-out-dir)/uTee +$(link-out-dir)/uTee: $(link-out-dir)/tee-raw.bin + @$(cmd-echo-silent) ' MKIMAGE $@' + $(q)ADDR=`printf 0x%x $$(($(subst UL,,$(CFG_TZDRAM_START))))`; \ + mkimage -A arm -O linux -C none -a $$ADDR -e $$ADDR -d $< $@ diff --git a/optee_os/core/arch/arm/plat-imx/main.c b/optee_os/core/arch/arm/plat-imx/main.c new file mode 100644 index 0000000..1a7087d --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/main.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * Copyright 2019, 2023 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct imx_uart_data console_data __nex_bss; + +#ifdef CONSOLE_UART_BASE +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, + CORE_MMU_PGDIR_SIZE); +#endif +#ifdef GIC_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE, CORE_MMU_PGDIR_SIZE); +#endif +#ifdef ANATOP_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, ANATOP_BASE, CORE_MMU_PGDIR_SIZE); +#endif +#ifdef GICD_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, 0x10000); +#endif +#ifdef AIPS0_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AIPS0_BASE, + ROUNDUP(AIPS0_SIZE, CORE_MMU_PGDIR_SIZE)); +#endif +#ifdef AIPS1_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AIPS1_BASE, + ROUNDUP(AIPS1_SIZE, CORE_MMU_PGDIR_SIZE)); +#endif +#ifdef AIPS2_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AIPS2_BASE, + ROUNDUP(AIPS2_SIZE, CORE_MMU_PGDIR_SIZE)); +#endif +#ifdef AIPS3_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AIPS3_BASE, + ROUNDUP(AIPS3_SIZE, CORE_MMU_PGDIR_SIZE)); +#endif +#ifdef IRAM_BASE +register_phys_mem(MEM_AREA_TEE_COHERENT, + ROUNDDOWN(IRAM_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); +#endif +#ifdef M4_AIPS_BASE +register_phys_mem(MEM_AREA_IO_SEC, M4_AIPS_BASE, M4_AIPS_SIZE); +#endif +#ifdef IRAM_S_BASE +register_phys_mem(MEM_AREA_TEE_COHERENT, + ROUNDDOWN(IRAM_S_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); +#endif + +#if defined(CFG_PL310) +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(PL310_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); +#endif + +#ifdef CFG_DRAM_BASE +register_ddr(CFG_DRAM_BASE, CFG_DDR_SIZE); +#endif +#ifdef CFG_NSEC_DDR_1_BASE +register_ddr(CFG_NSEC_DDR_1_BASE, CFG_NSEC_DDR_1_SIZE); +#endif + +void console_init(void) +{ +#ifdef CONSOLE_UART_BASE + imx_uart_init(&console_data, CONSOLE_UART_BASE); + register_serial_console(&console_data.chip); +#endif +} + +void boot_primary_init_intc(void) +{ +#ifdef GICD_BASE + gic_init(0, GICD_BASE); +#else + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +#endif +} + +#if CFG_TEE_CORE_NB_CORE > 1 +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} +#endif diff --git a/optee_os/core/arch/arm/plat-imx/platform_config.h b/optee_os/core/arch/arm/plat-imx/platform_config.h new file mode 100644 index 0000000..d7ebe0d --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/platform_config.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * Copyright 2019 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include +#include +#include + +#ifndef CFG_DDR_SIZE +#error "CFG_DDR_SIZE not defined" +#endif + +#define STACK_ALIGNMENT 64 +#define CONSOLE_UART_BASE (CFG_UART_BASE) + +/* For i.MX6 Quad SABRE Lite and Smart Device board */ +#if defined(CFG_MX6QP) || defined(CFG_MX6Q) || defined(CFG_MX6D) || \ + defined(CFG_MX6DL) || defined(CFG_MX6S) +#include +#elif defined(CFG_MX6SX) +#include +/* For i.MX 6SL */ +#elif defined(CFG_MX6SL) +#include +/* For i.MX 6SLL */ +#elif defined(CFG_MX6SLL) +#include +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx6-crm.h b/optee_os/core/arch/arm/plat-imx/registers/imx6-crm.h new file mode 100644 index 0000000..032058d --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx6-crm.h @@ -0,0 +1,860 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ +#ifndef __IMX6_CRM_H__ +#define __IMX6_CRM_H__ + +#define CCM_CCR 0x0000 +#define CCM_CCDR 0x0004 +#define CCM_CSR 0x0008 +#define CCM_CCSR 0x000C +#define CCM_CACRR 0x0010 +#define CCM_CBCDR 0x0014 +#define CCM_CBCMR 0x0018 +#define CCM_CSCMR1 0x001C +#define CCM_CSCMR2 0x0020 +#define CCM_CSCDR1 0x0024 +#define CCM_CS1CDR 0x0028 +#define CCM_CS2CDR 0x002C +#define CCM_CDCDR 0x0030 +#define CCM_CHSCCDR 0x0034 +#define CCM_CSCDR2 0x0038 +#define CCM_CSCDR3 0x003C +#define CCM_CSCDR4 0x0040 +#define CCM_CWDR 0x0044 +#define CCM_CDHIPR 0x0048 +#define CCM_CDCR 0x004C +#define CCM_CTOR 0x0050 +#define CCM_CLPCR 0x0054 +#define CCM_CISR 0x0058 +#define CCM_CIMR 0x005C +#define CCM_CCOSR 0x0060 +#define CCM_CGPR 0x0064 +#define CCM_CCGR0 0x0068 +#define CCM_CCGR1 0x006C +#define CCM_CCGR2 0x0070 +#define CCM_CCGR3 0x0074 +#define CCM_CCGR4 0x0078 +#define CCM_CCGR5 0x007C +#define CCM_CCGR6 0x0080 +#define CCM_CCGR7 0x0084 +#define CCM_CMEOR 0x0088 + +#define CCM_ANALOG_PLL_SYS 0x4000 +#define CCM_ANALOG_PLL_SYS_SET 0x4004 +#define CCM_ANALOG_PLL_SYS_CLR 0x4008 +#define CCM_ANALOG_PLL_SYS_TOG 0x400C +#define CCM_ANALOG_USB1_PLL_480_CTRL 0x4010 +#define CCM_ANALOG_USB1_PLL_480_CTRL_SET 0x4014 +#define CCM_ANALOG_USB1_PLL_480_CTRL_CLR 0x4018 +#define CCM_ANALOG_USB1_PLL_480_CTRL_TOG 0x401C + +#define CCM_ANALOG_PLL_528 0x4030 +#define CCM_ANALOG_PLL_528_SET 0x4034 +#define CCM_ANALOG_PLL_528_CLR 0x4038 +#define CCM_ANALOG_PLL_528_TOG 0x403C +#define CCM_ANALOG_PLL_528_SS 0x4040 +#define CCM_ANALOG_PLL_528_NUM 0x4050 +#define CCM_ANALOG_PLL_528_DENOM 0x4060 +#define CCM_ANALOG_PLL_AUDIO 0x4070 +#define CCM_ANALOG_PLL_AUDIO_SET 0x4074 +#define CCM_ANALOG_PLL_AUDIO_CLR 0x4078 +#define CCM_ANALOG_PLL_AUDIO_TOG 0x407C +#define CCM_ANALOG_PLL_AUDIO_NUM 0x4080 +#define CCM_ANALOG_PLL_AUDIO_DENOM 0x4090 +#define CCM_ANALOG_PLL_VIDEO 0x40A0 +#define CCM_ANALOG_PLL_VIDEO_SET 0x40A4 +#define CCM_ANALOG_PLL_VIDEO_CLR 0x40A8 +#define CCM_ANALOG_PLL_VIDEO_TOG 0x40AC +#define CCM_ANALOG_PLL_VIDEO_NUM 0x40B0 +#define CCM_ANALOG_PLL_VEDIO_DENON 0x40C0 +#define CCM_ANALOG_PLL_ENET 0x40E0 +#define CCM_ANALOG_PLL_ENET_SET 0x40E4 +#define CCM_ANALOG_PLL_ENET_CLR 0x40E8 +#define CCM_ANALOG_PLL_ENET_TOG 0x40EC +#define CCM_ANALOG_PFD_480 0x40F0 +#define CCM_ANALOG_PFD_480_SET 0x40F4 +#define CCM_ANALOG_PFD_480_CLR 0x40F8 +#define CCM_ANALOG_PFD_480_TOG 0x40FC +#define CCM_ANALOG_PFD_528 0x4100 +#define CCM_ANALOG_PFD_528_SET 0x4104 +#define CCM_ANALOG_PFD_528_CLR 0x4108 +#define CCM_ANALOG_PFD_528_TOG 0x410C + +/* Define the bits in register CCR */ +#define BS_CCM_CCR_RBC_EN 27 +#define BM_CCM_CCR_RBC_EN BIT32(BS_CCM_CCR_RBC_EN) +#define BS_CCM_CCR_REG_BYPASS_COUNT 21 +#define BM_CCM_CCR_REG_BYPASS_COUNT \ + SHIFT_U32(0x3F, BS_CCM_CCR_REG_BYPASS_COUNT) +#define BS_CCM_CCR_WB_COUNT 16 +#define BM_CCM_CCR_WB_COUNT SHIFT_U32(0x7, BS_CCM_CCR_WB_COUNT) +#define BS_CCM_CCR_OSCNT 0 +#define BM_CCM_CCR_OSCNT SHIFT_U32(0xFF, BS_CCM_CCR_OSCNT) +#define CCM_CCR_COSC_EN SHIFT_U32((1 << 12), BS_CCM_CCR_OSCNT) + +/* Define the bits in register CCDR */ +#define BS_CCM_CCDR_MMDC_CH1_HS_MASK 16 +#define BM_CCM_CCDR_MMDC_CH1_HS_MASK BIT32(BS_CCM_CCDR_MMDC_CH1_HS_MASK) +#define BS_CCM_CCDR_MMDC_CH0_HS_MASK 17 +#define BM_CCM_CCDR_MMDC_CH0_HS_MASK BIT32(BS_CCM_CCDR_MMDC_CH0_HS_MASK) + +/* Define the bits in register CSR */ +#define BS_CCM_CSR_COSC_READY 5 +#define BM_CCM_CSR_COSC_READY BIT32(BS_CCM_CSR_COSC_READY) +#define BS_CCM_CSR_REF_EN_B 0 +#define BM_CCM_CSR_REF_EN_B BIT32(BS_CCM_CSR_REF_EN_B) + +/* Define the bits in register CCSR */ +#define BS_CCM_CCSR_PDF_540M_AUTO_DIS 15 +#define BM_CCM_CCSR_PDF_540M_AUTO_DIS BIT32(BS_CCM_CCSR_PDF_540M_AUTO_DIS) +#define BS_CCM_CCSR_PDF_720M_AUTO_DIS 14 +#define BM_CCM_CCSR_PDF_720M_AUTO_DIS BIT32(BS_CCM_CCSR_PDF_720M_AUTO_DIS) +#define BS_CCM_CCSR_PDF_454M_AUTO_DIS 13 +#define BM_CCM_CCSR_PDF_454M_AUTO_DIS BIT32(BS_CCM_CCSR_PDF_454M_AUTO_DIS) +#define BS_CCM_CCSR_PDF_508M_AUTO_DIS 12 +#define BM_CCM_CCSR_PDF_508M_AUTO_DIS BIT32(BS_CCM_CCSR_PDF_508M_AUTO_DIS) +#define BS_CCM_CCSR_PDF_594M_AUTO_DIS 11 +#define BM_CCM_CCSR_PDF_594M_AUTO_DIS BIT32(BS_CCM_CCSR_PDF_594M_AUTO_DIS) +#define BS_CCM_CCSR_PDF_352M_AUTO_DIS 10 +#define BM_CCM_CCSR_PDF_352M_AUTO_DIS BIT32(BS_CCM_CCSR_PDF_352M_AUTO_DIS) +#define BS_CCM_CCSR_PDF_400M_AUTO_DIS 9 +#define BM_CCM_CCSR_PDF_400M_AUTO_DIS BIT32(BS_CCM_CCSR_PDF_400M_AUTO_DIS) +#define BS_CCM_CCSR_STEP_SEL 8 +#define BM_CCM_CCSR_STEP_SEL BIT32(BS_CCM_CCSR_STEP_SEL) +#define BS_CCM_CCSR_PLL1_SW_CLK_SEL 2 +#define BM_CCM_CCSR_PLL1_SW_CLK_SEL BIT32(BS_CCM_CCSR_PLL1_SW_CLK_SEL) +#define BS_CCM_CCSR_PLL2_SW_CLK_SEL 1 +#define BM_CCM_CCSR_PLL2_SW_CLK_SEL BIT32(BS_CCM_CCSR_PLL2_SW_CLK_SEL) +#define BS_CCM_CCSR_PLL3_SW_CLK_SEL 0 +#define BM_CCM_CCSR_PLL3_SW_CLK_SEL BIT32(BS_CCM_CCSR_PLL3_SW_CLK_SEL) + +/* Define the bits in register CACRR */ +#define BS_CCM_CACRR_ARM_PODF 0 +#define BM_CCM_CACRR_ARM_PODF SHIFT_U32(0x7, BS_CCM_CACRR_ARM_PODF) + +/* Define the bits in register CBCDR */ +#define BS_CCM_CBCDR_PERIPH_CLK2_PODF 27 +#define BM_CCM_CBCDR_PERIPH_CLK2_PODF \ + SHIFT_U32(0x7, BS_CCM_CBCDR_PERIPH_CLK2_PODF) +#define BS_CCM_CBCDR_PERIPH2_CLK2_SEL 26 +#define BM_CCM_CBCDR_PERIPH2_CLK2_SEL BIT32(BS_CCM_CBCDR_PERIPH2_CLK2_SEL) +#define BS_CCM_CBCDR_PERIPH_CLK_SEL 25 +#define BM_CCM_CBCDR_PERIPH_CLK_SEL BIT32(BS_CCM_CBCDR_PERIPH_CLK_SEL) +#define BS_CCM_CBCDR_MMDC_CH0_PODF 19 +#define BM_CCM_CBCDR_MMDC_CH0_PODF \ + SHIFT_U32(0x7, BS_CCM_CBCDR_MMDC_CH0_PODF) +#define BS_CCM_CBCDR_AXI_PODF 16 +#define BM_CCM_CBCDR_AXI_PODF SHIFT_U32(0x7, BS_CCM_CBCDR_AXI_PODF) +#define BS_CCM_CBCDR_AHB_PODF 10 +#define BM_CCM_CBCDR_AHB_PODF SHIFT_U32(0x7, BS_CCM_CBCDR_AHB_PODF) +#define BS_CCM_CBCDR_IPG_PODF 8 +#define BM_CCM_CBCDR_IPG_PODF SHIFT_U32(0x3, BS_CCM_CBCDR_IPG_PODF) +#define BS_CCM_CBCDR_AXI_ALT_SEL 7 +#define BM_CCM_CBCDR_AXI_ALT_SEL BIT32(BS_CCM_CBCDR_AXI_ALT_SEL) +#define BS_CCM_CBCDR_AXI_SEL 6 +#define BM_CCM_CBCDR_AXI_SEL BIT32(BS_CCM_CBCDR_AXI_SEL) +#define BS_CCM_CBCDR_MMDC_CH1_PODF 3 +#define BM_CCM_CBCDR_MMDC_CH1_PODF \ + SHIFT_U32(0x7, BS_CCM_CBCDR_MMDC_CH1_PODF) +#define BS_CCM_CBCDR_PERIPH2_CLK2_PODF 0 +#define BM_CCM_CBCDR_PERIPH2_CLK2_PODF \ + SHIFT_U32(0x7, BS_CCM_CBCDR_PERIPH2_CLK2_PODF) + +/* Define the bits in register CBCMR */ +#define BS_CCM_CBCMR_GPU3D_SHADER_PODF 29 +#define BM_CCM_CBCMR_GPU3D_SHADER_PODF \ + SHIFT_U32(0x7, BS_CCM_CBCMR_GPU3D_SHADER_PODF) +#define BS_CCM_CBCMR_GPU3D_CORE_PODF 26 +#define BM_CCM_CBCMR_GPU3D_CORE_PODF \ + SHIFT_U32(0x7, BS_CCM_CBCMR_GPU3D_CORE_PODF) +#define BS_CCM_CBCMR_GPU2D_CORE_PODF 23 +#define BM_CCM_CBCMR_GPU2D_CORE_PODF \ + SHIFT_U32(0x7, BS_CCM_CBCMR_GPU2D_CORE_PODF) +#define BS_CCM_CBCMR_PRE_PERIPH2_CLK_SEL 21 +#define BM_CCM_CBCMR_PRE_PERIPH2_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CBCMR_PRE_PERIPH2_CLK_SEL) +#define BS_CCM_CBCMR_PRE_PERIPH2_CLK2_SEL 20 +#define BM_CCM_CBCMR_PRE_PERIPH2_CLK2_SEL \ + BIT32(BS_CCM_CBCMR_PRE_PERIPH2_CLK2_SEL) +#define BS_CCM_CBCMR_PRE_PERIPH_CLK_SEL 18 +#define BM_CCM_CBCMR_PRE_PERIPH_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CBCMR_PRE_PERIPH_CLK_SEL) +#define BS_CCM_CBCMR_GPU2D_CLK_SEL 16 +#define BM_CCM_CBCMR_GPU2D_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CBCMR_GPU2D_CLK_SEL) +#define BS_CCM_CBCMR_VPU_AXI_CLK_SEL 14 +#define BM_CCM_CBCMR_VPU_AXI_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CBCMR_VPU_AXI_CLK_SEL) +#define BS_CCM_CBCMR_PERIPH_CLK2_SEL 12 +#define BM_CCM_CBCMR_PERIPH_CLK2_SEL \ + SHIFT_U32(0x3, BS_CCM_CBCMR_PERIPH_CLK2_SEL) +#define BS_CCM_CBCMR_VDOAXI_CLK_SEL 11 +#define BM_CCM_CBCMR_VDOAXI_CLK_SEL BIT32(BS_CCM_CBCMR_VDOAXI_CLK_SEL) +#define BS_CCM_CBCMR_PCIE_AXI_CLK_SEL 10 +#define BM_CCM_CBCMR_PCIE_AXI_CLK_SEL BIT32(BS_CCM_CBCMR_PCIE_AXI_CLK_SE) +#define BS_CCM_CBCMR_GPU3D_SHADER_CLK_SEL 8 +#define BM_CCM_CBCMR_GPU3D_SHADER_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CBCMR_GPU3D_SHADER_CLK_SEL) +#define BS_CCM_CBCMR_GPU3D_CORE_CLK_SEL 4 +#define BM_CCM_CBCMR_GPU3D_CORE_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CBCMR_GPU3D_CORE_CLK_SEL) +#define BS_CCM_CBCMR_GPU3D_AXI_CLK_SEL 1 +#define BM_CCM_CBCMR_GPU3D_AXI_CLK_SEL \ + BIT32(BS_CCM_CBCMR_GPU3D_AXI_CLK_SEL) +#define BS_CCM_CBCMR_GPU2D_AXI_CLK_SEL 0 +#define BM_CCM_CBCMR_GPU2D_AXI_CLK_SEL \ + BIT32(BS_CCM_CBCMR_GPU2D_AXI_CLK_SELx0) + +/* Define the bits in register CSCMR1 */ +#define BS_CCM_CSCMR1_ACLK_EMI_SLOW 29 +#define BM_CCM_CSCMR1_ACLK_EMI_SLOW \ + SHIFT_U32(0x3, BS_CCM_CSCMR1_ACLK_EMI_SLOW) +#define BS_CCM_CSCMR1_ACLK_EMI 27 +#define BM_CCM_CSCMR1_ACLK_EMI SHIFT_U32(0x3, BS_CCM_CSCMR1_ACLK_EMI) +#define BS_CCM_CSCMR1_ACLK_EMI_SLOW_PODF 23 +#define BM_CCM_CSCMR1_ACLK_EMI_SLOW_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCMR1_ACLK_EMI_SLOW_PODF) +#define BS_CCM_CSCMR1_ACLK_EMI_PODF 20 +#define BM_CCM_CSCMR1_ACLK_EMI_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCMR1_ACLK_EMI_PODF) +#define BS_CCM_CSCMR1_USDHC4_CLK_SEL 19 +#define BM_CCM_CSCMR1_USDHC4_CLK_SEL \ + BIT32(BS_CCM_CSCMR1_USDHC4_CLK_SEL) +#define BS_CCM_CSCMR1_USDHC3_CLK_SEL 18 +#define BM_CCM_CSCMR1_USDHC3_CLK_SEL \ + BIT32(BS_CCM_CSCMR1_USDHC3_CLK_SEL) +#define BS_CCM_CSCMR1_USDHC2_CLK_SEL 17 +#define BM_CCM_CSCMR1_USDHC2_CLK_SEL \ + BIT32(BS_CCM_CSCMR1_USDHC2_CLK_SEL) +#define BS_CCM_CSCMR1_USDHC1_CLK_SEL 16 +#define BM_CCM_CSCMR1_USDHC1_CLK_SEL \ + BIT32(BS_CCM_CSCMR1_USDHC1_CLK_SEL) +#define BS_CCM_CSCMR1_SSI3_CLK_SEL 14 +#define BM_CCM_CSCMR1_SSI3_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CSCMR1_SSI3_CLK_SEL) +#define BS_CCM_CSCMR1_SSI2_CLK_SEL 12 +#define BM_CCM_CSCMR1_SSI2_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CSCMR1_SSI2_CLK_SEL) +#define BS_CCM_CSCMR1_SSI1_CLK_SEL 10 +#define BM_CCM_CSCMR1_SSI1_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CSCMR1_SSI1_CLK_SEL) +#define BS_CCM_CSCMR1_PERCLK_PODF 0 +#define BM_CCM_CSCMR1_PERCLK_PODF \ + SHIFT_U32(0x3F, BS_CCM_CSCMR1_PERCLK_PODF) + +/* Define the bits in register CSCMR2 */ +#define BS_CCM_CSCMR2_ESAI_PRE_SEL 19 +#define BM_CCM_CSCMR2_ESAI_PRE_SEL \ + SHIFT_U32(0x3, BS_CCM_CSCMR2_ESAI_PRE_SEL) +#define BS_CCM_CSCMR2_LDB_DI1_IPU_DIV 11 +#define BM_CCM_CSCMR2_LDB_DI1_IPU_DIV BIT32(BS_CCM_CSCMR2_LDB_DI1_IPU_DIV) +#define BS_CCM_CSCMR2_LDB_DI0_IPU_DIV 10 +#define BM_CCM_CSCMR2_LDB_DI0_IPU_DIV BIT32(BS_CCM_CSCMR2_LDB_DI1_IPU_DIV) +#define BS_CCM_CSCMR2_CAN_CLK_SEL 2 +#define BM_CCM_CSCMR2_CAN_CLK_SEL \ + SHIFT_U32(0x3F, BS_CCM_CSCMR2_CAN_CLK_SEL) + +/* Define the bits in register CSCDR1 */ +#define BS_CCM_CSCDR1_VPU_AXI_PODF 25 +#define BM_CCM_CSCDR1_VPU_AXI_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCDR1_VPU_AXI_PODF) +#define BS_CCM_CSCDR1_USDHC4_PODF 22 +#define BM_CCM_CSCDR1_USDHC4_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCDR1_USDHC4_PODF) +#define BS_CCM_CSCDR1_USDHC3_PODF 19 +#define BM_CCM_CSCDR1_USDHC3_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCDR1_USDHC3_PODF) +#define BS_CCM_CSCDR1_USDHC2_PODF 16 +#define BM_CCM_CSCDR1_USDHC2_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCDR1_USDHC2_PODF) +#define BS_CCM_CSCDR1_USDHC1_PODF 11 +#define BM_CCM_CSCDR1_USDHC1_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCDR1_USDHC1_PODF) +#define BS_CCM_CSCDR1_USBOH3_CLK_PRED 8 +#define BM_CCM_CSCDR1_USBOH3_CLK_PRED \ + SHIFT_U32(0x7, BS_CCM_CSCDR1_USBOH3_CLK_PRED) +#define BS_CCM_CSCDR1_USBOH3_CLK_PODF 6 +#define BM_CCM_CSCDR1_USBOH3_CLK_PODF \ + SHIFT_U32(0x3, BS_CCM_CSCDR1_USBOH3_CLK_PODF) +#ifdef CONFIG_MX6SL +#define BS_CCM_CSCDR1_UART_CLK_SEL 6 +#define BM_CCM_CSCDR1_UART_CLK_SEL BIT32(BS_CCM_CSCDR1_UART_CLK_SEL) +#define BS_CCM_CSCDR1_UART_CLK_PODF 0 +#define BM_CCM_CSCDR1_UART_CLK_PODF SHIFT_U32(0x1F, BS_CCM_CSCDR1_UA) +#else +#define BS_CCM_CSCDR1_UART_CLK_PODF 0 +#define BM_CCM_CSCDR1_UART_CLK_PODF \ + SHIFT_U32(0x3F, BS_CCM_CSCDR1_UART_CLK_PODF) +#endif + +/* Define the bits in register CS1CDR */ +#define BS_CCM_CS1CDR_ESAI_CLK_PODF 25 +#define BM_CCM_CS1CDR_ESAI_CLK_PODF \ + SHIFT_U32(0x3F, BS_CCM_CS1CDR_ESAI_CLK_PODF) +#define BS_CCM_CS1CDR_SSI3_CLK_PODF 16 +#define BM_CCM_CS1CDR_SSI3_CLK_PODF SHIFT_U32(0x3F, BS_CCM_CS1CDR_SSI3) +#define BS_CCM_CS1CDR_ESAI_CLK_PRED 9 +#define BM_CCM_CS1CDR_ESAI_CLK_PRED \ + SHIFT_U32(0x3, BS_CCM_CS1CDR_ESAI_CLK_PRED) +#define BS_CCM_CS1CDR_SSI1_CLK_PRED 6 +#define BM_CCM_CS1CDR_SSI1_CLK_PRED \ + SHIFT_U32(0x7, BS_CCM_CS1CDR_SSI1_CLK_PRED) +#define BS_CCM_CS1CDR_SSI1_CLK_PODF 0 +#define BM_CCM_CS1CDR_SSI1_CLK_PODF \ + SHIFT_U32(0x3F, BS_CCM_CS1CDR_SSI1_CLK_PODF) + +/* Define the bits in register CS2CDR */ +#define BS_CCM_CS2CDR_ENFC_CLK_PODF 21 +#define BM_CCM_CS2CDR_ENFC_CLK_PODF \ + SHIFT_U32(0x3F, BS_CCM_CS2CDR_ENFC_CLK_PODF) +#define CCM_CS2CDR_ENFC_CLK_PODF(v) \ + (SHIFT_U32(v, BS_CCM_CS2CDR_ENFC_CLK_PODF) & \ + BM_CCM_CS2CDR_ENFC_CLK_PODF) +#define BS_CCM_CS2CDR_ENFC_CLK_PRED 18 +#define BM_CCM_CS2CDR_ENFC_CLK_PRED \ + SHIFT_U32(0x7, BS_CCM_CS2CDR_ENFC_CLK_PRED) +#define CCM_CS2CDR_ENFC_CLK_PRED(v) \ + (SHIFT_U32(v, BS_CCM_CS2CDR_ENFC_CLK_PRED) & \ + BM_CCM_CS2CDR_ENFC_CLK_PRED) +#define BS_CCM_CS2CDR_ENFC_CLK_SEL 16 +#define BM_CCM_CS2CDR_ENFC_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CS2CDR_ENFC_CLK_SEL_OFFSET) +#define CCM_CS2CDR_ENFC_CLK_SEL(v) \ + (SHIFT_U32(v, BS_CCM_CS2CDR_ENFC_CLK_SEL) & \ + BM_CCM_CS2CDR_ENFC_CLK_SEL) +#define BS_CCM_CS2CDR_LDB_DI1_CLK_SEL 12 +#define BM_CCM_CS2CDR_LDB_DI1_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CS2CDR_LDB_DI1_CLK_SEL) +#define BS_CCM_CS2CDR_LDB_DI0_CLK_SEL 9 +#define BM_CCM_CS2CDR_LDB_DI0_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CS2CDR_LDB_DI0_CLK_SEL) +#define BS_CCM_CS2CDR_SSI2_CLK_PRED 6 +#define BM_CCM_CS2CDR_SSI2_CLK_PRED \ + SHIFT_U32(0x7, BS_CCM_CS2CDR_SSI2_CLK_PRED) +#define BS_CCM_CS2CDR_SSI2_CLK_PODF 0 +#define BM_CCM_CS2CDR_SSI2_CLK_PODF \ + SHIFT_U32(0x3F, BS_CCM_CS2CDR_SSI2_CLK_PODF) + +/* Define the bits in register CDCDR */ +#define BS_CCM_CDCDR_HSI_TX_PODF 29 +#define BM_CCM_CDCDR_HSI_TX_PODF \ + SHIFT_U32(0x7, BS_CCM_CDCDR_HSI_TX_PODF) +#define BS_CCM_CDCDR_SPDIF0_CLK_PRED 25 +#define BM_CCM_CDCDR_SPDIF0_CLK_PRED \ + SHIFT_U32(0x7, BS_CCM_CDCDR_SPDIF0_CLK_PRED) +#define BS_CCM_CDCDR_HSI_TX_CLK_SEL 28 +#define BM_CCM_CDCDR_HSI_TX_CLK_SEL \ + BIT32(BS_CCM_CDCDR_HSI_TX_CLK_SEL) +#define BS_CCM_CDCDR_SPDIF0_CLK_PODF 19 +#define BM_CCM_CDCDR_SPDIF0_CLK_PODF \ + SHIFT_U32(0x7, BS_CCM_CDCDR_SPDIF0_CLK_PODF) +#define BS_CCM_CDCDR_SPDIF0_CLK_SEL 20 +#define BM_CCM_CDCDR_SPDIF0_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CDCDR_SPDIF0_CLK_SEL) +#define BS_CCM_CDCDR_SPDIF1_CLK_PRED 12 +#define BM_CCM_CDCDR_SPDIF1_CLK_PRED \ + SHIFT_U32(0x7, BS_CCM_CDCDR_SPDIF1_CLK_PRED) +#define BS_CCM_CDCDR_SPDIF1_CLK_PODF 9 +#define BM_CCM_CDCDR_SPDIF1_CLK_PODF \ + SHIFT_U32(0x7, BS_CCM_CDCDR_SPDIF1_CLK_PODF) +#define BS_CCM_CDCDR_SPDIF1_CLK_SEL 7 +#define BM_CCM_CDCDR_SPDIF1_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CDCDR_SPDIF1_CLK_SEL) + +/* Define the bits in register CHSCCDR */ +#define BS_CCM_CHSCCDR_IPU1_DI1_PRE_CLK_SEL 15 +#define BM_CCM_CHSCCDR_IPU1_DI1_PRE_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU1_DI1_PRE_CLK_SEL) +#define BS_CCM_CHSCCDR_IPU1_DI1_PODF 12 +#define BM_CCM_CHSCCDR_IPU1_DI1_PODF \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU1_DI1_PODF) +#define BS_CCM_CHSCCDR_IPU1_DI1_CLK_SEL 9 +#define BM_CCM_CHSCCDR_IPU1_DI1_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU1_DI1_CLK_SEL) +#define BS_CCM_CHSCCDR_IPU1_DI0_PRE_CLK_SEL 6 +#define BM_CCM_CHSCCDR_IPU1_DI0_PRE_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU1_DI0_PRE_CLK_SEL) +#define BS_CCM_CHSCCDR_IPU1_DI0_PODF 3 +#define BM_CCM_CHSCCDR_IPU1_DI0_PODF \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU1_DI0_PODF) +#define BS_CCM_CHSCCDR_IPU1_DI0_CLK_SEL 0 +#define BM_CCM_CHSCCDR_IPU1_DI0_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU1_DI0_CLK_SEL) + +#define CHSCCDR_CLK_SEL_LDB_DI0 3 +#define CHSCCDR_PODF_DIVIDE_BY_3 2 +#define CHSCCDR_IPU_PRE_CLK_540M_PFD 5 + +/* Define the bits in register CSCDR2 */ +#define BS_CCM_CSCDR2_ECSPI_CLK_PODF 19 +#define BM_CCM_CSCDR2_ECSPI_CLK_PODF \ + SHIFT_U32(0x3F, BS_CCM_CSCDR2_ECSPI_CLK_PODF) +#define BS_CCM_CHSCCDR_IPU2_DI1_PRE_CLK_SEL 15 +#define BM_CCM_CHSCCDR_IPU2_DI1_PRE_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU2_DI1_PRE_CLK_SEL) +#define BS_CCM_CHSCCDR_IPU2_DI1_PODF 12 +#define BM_CCM_CHSCCDR_IPU2_DI1_PODF \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU2_DI1_PODF) +#define BS_CCM_CHSCCDR_IPU2_DI1_CLK_SEL 9 +#define BM_CCM_CHSCCDR_IPU2_DI1_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU2_DI1_CLK_SEL) +#define BS_CCM_CHSCCDR_IPU2_DI0_PRE_CLK_SEL 6 +#define BM_CCM_CHSCCDR_IPU2_DI0_PRE_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU2_DI0_PRE_CLK_SEL) +#define BS_CCM_CHSCCDR_IPU2_DI0_PODF 3 +#define BM_CCM_CHSCCDR_IPU2_DI0_PODF \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU2_DI0_PODF) +#define BS_CCM_CHSCCDR_IPU2_DI0_CLK_SEL 0 +#define BM_CCM_CHSCCDR_IPU2_DI0_CLK_SEL \ + SHIFT_U32(0x7, BS_CCM_CHSCCDR_IPU2_DI0_CLK_SEL) + +/* Define the bits in register CSCDR3 */ +#define BS_CCM_CSCDR3_IPU2_HSP_PODF 16 +#define BM_CCM_CSCDR3_IPU2_HSP_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCDR3_IPU2_HSP_PODF) +#define BS_CCM_CSCDR3_IPU2_HSP_CLK_SEL 14 +#define BM_CCM_CSCDR3_IPU2_HSP_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CSCDR3_IPU2_HSP_CLK_SEL) +#define BS_CCM_CSCDR3_IPU1_HSP_PODF 11 +#define BM_CCM_CSCDR3_IPU1_HSP_PODF \ + SHIFT_U32(0x7, BS_CCM_CSCDR3_IPU1_HSP_PODF) +#define BS_CCM_CSCDR3_IPU1_HSP_CLK_SEL 9 +#define BM_CCM_CSCDR3_IPU1_HSP_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CSCDR3_IPU1_HSP_CLK_SEL) + +/* Define the bits in register CDHIPR */ +#define BS_CCM_CDHIPR_ARM_PODF_BUSY 16 +#define BM_CCM_CDHIPR_ARM_PODF_BUSY \ + BIT32(BS_CCM_CDHIPR_ARM_PODF_BUSY) +#define BS_CCM_CDHIPR_PERIPH_CLK_SEL_BUSY 5 +#define BM_CCM_CDHIPR_PERIPH_CLK_SEL_BUSY \ + BIT32(BS_CCM_CDHIPR_PERIPH_CLK_SEL_BUSY) +#define BS_CCM_CDHIPR_MMDC_CH0_PODF_BUSY 4 +#define BM_CCM_CDHIPR_MMDC_CH0_PODF_BUSY \ + BIT32(BS_CCM_CDHIPR_MMDC_CH0_PODF_BUSY) +#define BS_CCM_CDHIPR_PERIPH2_CLK_SEL_BUSY 3 +#define BM_CCM_CDHIPR_PERIPH2_CLK_SEL_BUSY \ + BIT32(BS_CCM_CDHIPR_PERIPH2_CLK_SEL_BUSY) +#define BS_CCM_CDHIPR_MMDC_CH1_PODF_BUSY 2 +#define BM_CCM_CDHIPR_MMDC_CH1_PODF_BUSY \ + BIT32(BS_CCM_CDHIPR_MMDC_CH1_PODF_BUSY) +#define BS_CCM_CDHIPR_AHB_PODF_BUSY 1 +#define BM_CCM_CDHIPR_AHB_PODF_BUSY \ + BIT32(BS_CCM_CDHIPR_AHB_PODF_BUSY) +#define BS_CCM_CDHIPR_AXI_PODF_BUSY 0 +#define BM_CCM_CDHIPR_AXI_PODF_BUSY \ + BIT32(BS_CCM_CDHIPR_AXI_PODF_BUSY) + +/* Define the bits in register CLPCR */ +#define BS_CCM_CLPCR_MASK_L2CC_IDLE 27 +#define BM_CCM_CLPCR_MASK_L2CC_IDLE \ + BIT32(BS_CCM_CLPCR_MASK_L2CC_IDLE) +#define BS_CCM_CLPCR_MASK_SCU_IDLE 26 +#define BM_CCM_CLPCR_MASK_SCU_IDLE \ + BIT32(BS_CCM_CLPCR_MASK_SCU_IDLE) +#define BS_CCM_CLPCR_MASK_CORE3_WFI 25 +#define BM_CCM_CLPCR_MASK_CORE3_WFI \ + BIT32(BS_CCM_CLPCR_MASK_CORE3_WFI) +#define BS_CCM_CLPCR_MASK_CORE2_WFI 24 +#define BM_CCM_CLPCR_MASK_CORE2_WFI \ + BIT32(BS_CCM_CLPCR_MASK_CORE2_WFI) +#define BS_CCM_CLPCR_MASK_CORE1_WFI 23 +#define BM_CCM_CLPCR_MASK_CORE1_WFI \ + BIT32(BS_CCM_CLPCR_MASK_CORE1_WFI) +#define BS_CCM_CLPCR_MASK_CORE0_WFI 22 +#define BM_CCM_CLPCR_MASK_CORE0_WFI \ + BIT32(BS_CCM_CLPCR_MASK_CORE0_WFI) +#define BS_CCM_CLPCR_BYP_MMDC_CH1_LPM_HS 21 +#define BM_CCM_CLPCR_BYP_MMDC_CH1_LPM_HS \ + BIT32(BS_CCM_CLPCR_BYP_MMDC_CH1_LPM_HS) +#define BS_CCM_CLPCR_BYP_MMDC_CH0_LPM_HS 19 +#define BM_CCM_CLPCR_BYP_MMDC_CH0_LPM_HS \ + BIT32(BS_CCM_CLPCR_BYP_MMDC_CH0_LPM_HS) +#define BS_CCM_CLPCR_WB_CORE_AT_LPM 17 +#define BM_CCM_CLPCR_WB_CORE_AT_LPM \ + BIT32(BS_CCM_CLPCR_WB_CORE_AT_LPM) +#define BS_CCM_CLPCR_WB_PER_AT_LPM 16 +#define BM_CCM_CLPCR_WB_PER_AT_LPM \ + BIT32(BS_CCM_CLPCR_WB_PER_AT_LPM) +#define BS_CCM_CLPCR_COSC_PWRDOWN 11 +#define BM_CCM_CLPCR_COSC_PWRDOWN \ + BIT32(BS_CCM_CLPCR_COSC_PWRDOWN) +#define BS_CCM_CLPCR_STBY_COUNT 9 +#define BM_CCM_CLPCR_STBY_COUNT \ + SHIFT_U32(0x3, BS_CCM_CLPCR_STBY_COUNT) +#define BS_CCM_CLPCR_VSTBY 8 +#define BM_CCM_CLPCR_VSTBY \ + BIT32(BS_CCM_CLPCR_VSTBY) +#define BS_CCM_CLPCR_DIS_REF_OSC 7 +#define BM_CCM_CLPCR_DIS_REF_OSC \ + BIT32(BS_CCM_CLPCR_DIS_REF_OSC) +#define BS_CCM_CLPCR_SBYOS 6 +#define BM_CCM_CLPCR_SBYOS \ + BIT32(BS_CCM_CLPCR_SBYOS) +#define BS_CCM_CLPCR_ARM_CLK_DIS_ON_LPM 5 +#define BM_CCM_CLPCR_ARM_CLK_DIS_ON_LPM \ + BIT32(BS_CCM_CLPCR_ARM_CLK_DIS_ON_LPM) +#define BS_CCM_CLPCR_LPSR_CLK_SEL 3 +#define BM_CCM_CLPCR_LPSR_CLK_SEL \ + SHIFT_U32(0x3, BS_CCM_CLPCR_LPSR_CLK_SEL) +#define BS_CCM_CLPCR_BYPASS_PMIC_VFUNC_RDY 2 +#define BM_CCM_CLPCR_BYPASS_PMIC_VFUNC_RDY \ + BIT32(BS_CCM_CLPCR_BYPASS_PMIC_VFUNC_RDY) +#define BS_CCM_CLPCR_LPM 0 +#define BM_CCM_CLPCR_LPM \ + SHIFT_U32(0x3, BS_CCM_CLPCR_LPM) + +/* Define the bits in register CISR */ +#define BS_CCM_CISR_ARM_PODF_LOADED 26 +#define BM_CCM_CISR_ARM_PODF_LOADED \ + BIT32(BS_CCM_CISR_ARM_PODF_LOADED) +#define BS_CCM_CISR_MMDC_CH0_PODF_LOADED 23 +#define BM_CCM_CISR_MMDC_CH0_PODF_LOADED \ + BIT32(BS_CCM_CISR_MMDC_CH0_PODF_LOADED) +#define BS_CCM_CISR_PERIPH_CLK_SEL_LOADED 22 +#define BM_CCM_CISR_PERIPH_CLK_SEL_LOADED \ + BIT32(BS_CCM_CISR_PERIPH_CLK_SEL_LOADED) +#define BS_CCM_CISR_MMDC_CH1_PODF_LOADED 21 +#define BM_CCM_CISR_MMDC_CH1_PODF_LOADED \ + BIT32(BS_CCM_CISR_MMDC_CH1_PODF_LOADED) +#define BS_CCM_CISR_AHB_PODF_LOADED 20 +#define BM_CCM_CISR_AHB_PODF_LOADED \ + BIT32(BS_CCM_CISR_AHB_PODF_LOADED) +#define BS_CCM_CISR_PERIPH2_CLK_SEL_LOADED 19 +#define BM_CCM_CISR_PERIPH2_CLK_SEL_LOADED \ + BIT32(BS_CCM_CISR_PERIPH2_CLK_SEL_LOADED) +#define BS_CCM_CISR_AXI_PODF_LOADED 17 +#define BM_CCM_CISR_AXI_PODF_LOADED \ + BIT32(BS_CCM_CISR_AXI_PODF_LOADED) +#define BS_CCM_CISR_COSC_READY 6 +#define BM_CCM_CISR_COSC_READY \ + BIT32(BS_CCM_CISR_COSC_READY) +#define BS_CCM_CISR_LRF_PLL 0 +#define BM_CCM_CISR_LRF_PLL \ + BIT32(BS_CCM_CISR_LRF_PLL) + +/* Define the bits in register CIMR */ +#define BS_CCM_CIMR_MASK_ARM_PODF_LOADED 26 +#define BM_CCM_CIMR_MASK_ARM_PODF_LOADED \ + BIT32(BS_CCM_CIMR_MASK_ARM_PODF_LOADED) +#define BS_CCM_CIMR_MASK_MMDC_CH0_PODF_LOADED 23 +#define BM_CCM_CIMR_MASK_MMDC_CH0_PODF_LOADED \ + BIT32(BS_CCM_CIMR_MASK_MMDC_CH0_PODF_LOADED) +#define BS_CCM_CIMR_MASK_PERIPH_CLK_SEL_LOADED 22 +#define BM_CCM_CIMR_MASK_PERIPH_CLK_SEL_LOADED \ + BIT32(BS_CCM_CIMR_MASK_PERIPH_CLK_SEL_LOADED) +#define BS_CCM_CIMR_MASK_MMDC_CH1_PODF_LOADED 21 +#define BM_CCM_CIMR_MASK_MMDC_CH1_PODF_LOADED \ + BIT32(BS_CCM_CIMR_MASK_MMDC_CH1_PODF_LOADED) +#define BS_CCM_CIMR_MASK_AHB_PODF_LOADED 20 +#define BM_CCM_CIMR_MASK_AHB_PODF_LOADED \ + BIT32(BS_CCM_CIMR_MASK_AHB_PODF_LOADED) +#define BS_CCM_CIMR_MASK_PERIPH2_CLK_SEL_LOADED 19 +#define BM_CCM_CIMR_MASK_PERIPH2_CLK_SEL_LOADED \ + BIT32(BS_CCM_CIMR_MASK_PERIPH2_CLK_SEL_LOADED) +#define BS_CCM_CIMR_MASK_AXI_PODF_LOADED 17 +#define BM_CCM_CIMR_MASK_AXI_PODF_LOADED \ + BIT32(BS_CCM_CIMR_MASK_AXI_PODF_LOADED) +#define BS_CCM_CIMR_MASK_COSC_READY 6 +#define BM_CCM_CIMR_MASK_COSC_READY \ + BIT32(BS_CCM_CIMR_MASK_COSC_READY) +#define BS_CCM_CIMR_MASK_LRF_PLL 0 +#define BM_CCM_CIMR_MASK_LRF_PLL \ + BIT32(BS_CCM_CIMR_MASK_LRF_PLL) + +/* Define the bits in register CCOSR */ +#define BS_CCM_CCOSR_CKO2_EN 24 +#define BM_CCM_CCOSR_CKO2_EN \ + BIT32(BS_CCM_CCOSR_CKO2_EN) +#define BS_CCM_CCOSR_CKO2_DIV 21 +#define BM_CCM_CCOSR_CKO2_DIV \ + SHIFT_U32(0x7, BS_CCM_CCOSR_CKO2_DIV) +#define BS_CCM_CCOSR_CKO2_SEL 16 +#define BM_CCM_CCOSR_CKO2_SEL \ + SHIFT_U32(0x1F, BS_CCM_CCOSR_CKO2_SEL) +#define BS_CCM_CCOSR_CLK_OUT_SEL 8 +#define BM_CCM_CCOSR_CLK_OUT_SEL_CKO2 \ + BIT32(BS_CCM_CCOSR_CLK_OUT_SEL) +#define BS_CCM_CCOSR_CKOL_EN 7 +#define BM_CCM_CCOSR_CKOL_EN \ + BIT32(BS_CCM_CCOSR_CKOL_EN) +#define BS_CCM_CCOSR_CKOL_DIV 4 +#define BM_CCM_CCOSR_CKOL_DIV \ + SHIFT_U32(0x7, BS_CCM_CCOSR_CKOL_DIV) +#define BS_CCM_CCOSR_CKOL_SEL 0 +#define BM_CCM_CCOSR_CKOL_SEL \ + SHIFT_U32(0xF, BS_CCM_CCOSR_CKOL_SEL) + +/* Define the bits in registers CGPR */ +#define BS_CCM_CGPR_INT_MEM_CLK_LPM 17 +#define BM_CCM_CGPR_INT_MEM_CLK_LPM \ + BIT32(BS_CCM_CGPR_INT_MEM_CLK_LPM) +#define BS_CCM_CGPR_EFUSE_PROG_SUPPLY_GATE 4 +#define BM_CCM_CGPR_EFUSE_PROG_SUPPLY_GATE \ + BIT32(BS_CCM_CGPR_EFUSE_PROG_SUPPLY_GATE) +#define BS_CCM_CGPR_MMDC_EXT_CLK_DIS 2 +#define BM_CCM_CGPR_MMDC_EXT_CLK_DIS \ + BIT32(BS_CCM_CGPR_MMDC_EXT_CLK_DIS) +#define BS_CCM_CGPR_PMIC_DELAY_SCALER 0 +#define BM_CCM_CGPR_PMIC_DELAY_SCALER \ + BIT32(BS_CCM_CGPR_PMIC_DELAY_SCALER) + +/* Define the bits in registers CCGRx */ +#define BS_CCM_CCGR0_AIPS_TZ1 0 +#define BM_CCM_CCGR0_AIPS_TZ1 \ + SHIFT_U32(3, BS_CCM_CCGR0_AIPS_TZ1) +#define BS_CCM_CCGR0_AIPS_TZ2 2 +#define BM_CCM_CCGR0_AIPS_TZ2 \ + SHIFT_U32(3, BS_CCM_CCGR0_AIPS_TZ2) +#define BS_CCM_CCGR0_APBHDMA 4 +#define BM_CCM_CCGR0_APBHDMA \ + SHIFT_U32(3, BS_CCM_CCGR0_APBHDMA) +#define BS_CCM_CCGR0_ASRC 6 +#define BM_CCM_CCGR0_ASRC \ + SHIFT_U32(3, BS_CCM_CCGR0_ASRC) +#define BS_CCM_CCGR0_CAAM_SECURE_MEM 8 +#define BM_CCM_CCGR0_CAAM_SECURE_MEM \ + SHIFT_U32(3, BS_CCM_CCGR0_CAAM_SECURE_MEM) +#define BS_CCM_CCGR0_CAAM_WRAPPER_ACLK 10 +#define BM_CCM_CCGR0_CAAM_WRAPPER_ACLK \ + SHIFT_U32(3, BS_CCM_CCGR0_CAAM_WRAPPER_ACLK) +#define BS_CCM_CCGR0_CAAM_WRAPPER_IPG 12 +#define BM_CCM_CCGR0_CAAM_WRAPPER_IPG \ + SHIFT_U32(3, BS_CCM_CCGR0_CAAM_WRAPPER_IPG) +#define BS_CCM_CCGR0_CAN1 14 +#define BM_CCM_CCGR0_CAN1 SHIFT_U32(3, BS_CCM_CCGR0_CAN1) +#define BS_CCM_CCGR0_CAN1_SERIAL 16 +#define BM_CCM_CCGR0_CAN1_SERIAL \ + SHIFT_U32(3, BS_CCM_CCGR0_CAN1_SERIAL) +#define BS_CCM_CCGR0_CAN2 18 +#define BM_CCM_CCGR0_CAN2 SHIFT_U32(3, BS_CCM_CCGR0_CAN2) +#define BS_CCM_CCGR0_CAN2_SERIAL 20 +#define BM_CCM_CCGR0_CAN2_SERIAL \ + SHIFT_U32(3, BS_CCM_CCGR0_CAN2_SERIAL) +#define BS_CCM_CCGR0_CHEETAH_DBG_CLK 22 +#define BM_CCM_CCGR0_CHEETAH_DBG_CLK \ + SHIFT_U32(3, BS_CCM_CCGR0_CHEETAH_DBG_CLK) +#define BS_CCM_CCGR0_DCIC1 24 +#define BM_CCM_CCGR0_DCIC1 SHIFT_U32(3, BS_CCM_CCGR0_DCIC1) +#define BS_CCM_CCGR0_DCIC2 26 +#define BM_CCM_CCGR0_DCIC2 SHIFT_U32(3, BS_CCM_CCGR0_DCIC2) +#define BS_CCM_CCGR0_DTCP 28 +#define BM_CCM_CCGR0_DTCP SHIFT_U32(3, BS_CCM_CCGR0_DTCP) + +#define BS_CCM_CCGR1_ECSPI1S 0 +#define BM_CCM_CCGR1_ECSPI1S SHIFT_U32(3, BS_CCM_CCGR1_ECSPI1S) +#define BS_CCM_CCGR1_ECSPI2S 2 +#define BM_CCM_CCGR1_ECSPI2S SHIFT_U32(3, BS_CCM_CCGR1_ECSPI2S) +#define BS_CCM_CCGR1_ECSPI3S 4 +#define BM_CCM_CCGR1_ECSPI3S SHIFT_U32(3, BS_CCM_CCGR1_ECSPI3S) +#define BS_CCM_CCGR1_ECSPI4S 6 +#define BM_CCM_CCGR1_ECSPI4S SHIFT_U32(3, BS_CCM_CCGR1_ECSPI4S) +#define BS_CCM_CCGR1_ECSPI5S 8 +#define BM_CCM_CCGR1_ECSPI5S SHIFT_U32(3, BS_CCM_CCGR1_ECSPI5S) +#define BS_CCM_CCGR1_ENET_CLK_ENABLE 10 +#define BM_CCM_CCGR1_ENET_CLK_ENABLE \ + (3 << BS_CCM_CCGR1_ENET_CLK_ENABLE) +#define BS_CCM_CCGR1_EPIT1S 12 +#define BM_CCM_CCGR1_EPIT1S SHIFT_U32(3, BS_CCM_CCGR1_EPIT1S) +#define BS_CCM_CCGR1_EPIT2S 14 +#define BM_CCM_CCGR1_EPIT2S SHIFT_U32(3, BS_CCM_CCGR1_EPIT2S) +#define BS_CCM_CCGR1_ESAIS 16 +#define BM_CCM_CCGR1_ESAIS SHIFT_U32(3, BS_CCM_CCGR1_ESAIS) +#define BS_CCM_CCGR1_GPT_BUS 20 +#define BM_CCM_CCGR1_GPT_BUS SHIFT_U32(3, BS_CCM_CCGR1_GPT_BUS) +#define BS_CCM_CCGR1_GPT_SERIAL 22 +#define BM_CCM_CCGR1_GPT_SERIAL SHIFT_U32(3, BS_CCM_CCGR1_GPT_SERIAL) +#define BS_CCM_CCGR1_GPU2D 24 +#define BM_CCM_CCGR1_GPU2D SHIFT_U32(3, BS_CCM_CCGR1_GPU2D) +#define BS_CCM_CCGR1_GPU3D 26 +#define BM_CCM_CCGR1_GPU3D SHIFT_U32(3, BS_CCM_CCGR1_GPU3D) + +#define BS_CCM_CCGR2_HDMI_TX_IAHBCLK 0 +#define BM_CCM_CCGR2_HDMI_TX_IAHBCLK \ + SHIFT_U32(3, BS_CCM_CCGR2_HDMI_TX_IAHBCLK) +#define BS_CCM_CCGR2_HDMI_TX_ISFRCLK 4 +#define BM_CCM_CCGR2_HDMI_TX_ISFRCLK \ + SHIFT_U32(3, BS_CCM_CCGR2_HDMI_TX_ISFRCLK) +#define BS_CCM_CCGR2_I2C1_SERIAL 6 +#define BM_CCM_CCGR2_I2C1_SERIAL SHIFT_U32(3, BS_CCM_CCGR2_I2C1_SERIAL) +#define BS_CCM_CCGR2_I2C2_SERIAL 8 +#define BM_CCM_CCGR2_I2C2_SERIAL SHIFT_U32(3, BS_CCM_CCGR2_I2C2_SERIAL) +#define BS_CCM_CCGR2_I2C3_SERIAL 10 +#define BM_CCM_CCGR2_I2C3_SERIAL SHIFT_U32(3, BS_CCM_CCGR2_I2C3_SERIAL) +#define BS_CCM_CCGR2_OCOTP_CTRL 12 +#define BM_CCM_CCGR2_OCOTP_CTRL SHIFT_U32(3, BS_CCM_CCGR2_OCOTP_CTRL) +#define BS_CCM_CCGR2_IOMUX_IPT_CLK_IO 14 +#define BM_CCM_CCGR2_IOMUX_IPT_CLK_IO \ + SHIFT_U32(3, BS_CCM_CCGR2_IOMUX_IPT_CLK_IO) +#define BS_CCM_CCGR2_IPMUX1 16 +#define BM_CCM_CCGR2_IPMUX1 SHIFT_U32(3, BS_CCM_CCGR2_IPMUX1) +#define BS_CCM_CCGR2_IPMUX2 18 +#define BM_CCM_CCGR2_IPMUX2 SHIFT_U32(3, BS_CCM_CCGR2_IPMUX2) +#define BS_CCM_CCGR2_IPMUX3 20 +#define BM_CCM_CCGR2_IPMUX3 SHIFT_U32(3, BS_CCM_CCGR2_IPMUX3) +#define BS_CCM_CCGR2_IPSYNC_IP2APB_TZASC1_IPGS 22 +#define BM_CCM_CCGR2_IPSYNC_IP2APB_TZASC1_IPGS \ + SHIFT_U32(3, BS_CCM_CCGR2_IPSYNC_IP2APB_TZASC1_IPGS) +#define BS_CCM_CCGR2_IPSYNC_IP2APB_TZASC2_IPG 24 +#define BM_CCM_CCGR2_IPSYNC_IP2APB_TZASC2_IPG \ + SHIFT_U32(3, BS_CCM_CCGR2_IPSYNC_IP2APB_TZASC2_IPG) +#define BS_CCM_CCGR2_IPSYNC_VDOA_IPG_MASTER_CLK 26 +#define BM_CCM_CCGR2_IPSYNC_VDOA_IPG_MASTER_CLK \ + SHIFT_U32(3, BS_CCM_CCGR2_IPSYNC_VDOA_IPG_MASTER_CLK) + +#define BS_CCM_CCGR3_IPU1_IPU 0 +#define BM_CCM_CCGR3_IPU1_IPU SHIFT_U32(3, BS_CCM_CCGR3_IPU1_IPU) +#define BS_CCM_CCGR3_IPU1_IPU_DI0 2 +#define BM_CCM_CCGR3_IPU1_IPU_DI0 SHIFT_U32(3, BS_CCM_CCGR3_IPU1_IPU_DI0) +#define BS_CCM_CCGR3_IPU1_IPU_DI1 4 +#define BM_CCM_CCGR3_IPU1_IPU_DI1 SHIFT_U32(3, BS_CCM_CCGR3_IPU1_IPU_DI1) +#define BS_CCM_CCGR3_IPU2_IPU 6 +#define BM_CCM_CCGR3_IPU2_IPU SHIFT_U32(3, BS_CCM_CCGR3_IPU2_IPU) +#define BS_CCM_CCGR3_IPU2_IPU_DI0 8 +#define BM_CCM_CCGR3_IPU2_IPU_DI0 SHIFT_U32(3, BS_CCM_CCGR3_IPU2_IPU_DI0) +#define BS_CCM_CCGR3_IPU2_IPU_DI1 10 +#define BM_CCM_CCGR3_IPU2_IPU_DI1 SHIFT_U32(3, BS_CCM_CCGR3_IPU2_IPU_DI1) +#define BS_CCM_CCGR3_LDB_DI0 12 +#define BM_CCM_CCGR3_LDB_DI0 SHIFT_U32(3, BS_CCM_CCGR3_LDB_DI0) +#define BS_CCM_CCGR3_LDB_DI1 14 +#define BM_CCM_CCGR3_LDB_DI1 SHIFT_U32(3, BS_CCM_CCGR3_LDB_DI1) +#define BS_CCM_CCGR3_MIPI_CORE_CFG 16 +#define BM_CCM_CCGR3_MIPI_CORE_CFG \ + SHIFT_U32(3, BS_CCM_CCGR3_MIPI_CORE_CFG) +#define BS_CCM_CCGR3_MLB 18 +#define BM_CCM_CCGR3_MLB SHIFT_U32(3, BS_CCM_CCGR3_MLB) +#define BS_CCM_CCGR3_MMDC_CORE_ACLK_FAST_CORE_P0 20 +#define BM_CCM_CCGR3_MMDC_CORE_ACLK_FAST_CORE_P0 \ + SHIFT_U32(3, BS_CCM_CCGR3_MMDC_CORE_ACLK_FAST_CORE_P0) +#define BS_CCM_CCGR3_MMDC_CORE_ACLK_FAST_CORE_P1 22 +#define BM_CCM_CCGR3_MMDC_CORE_ACLK_FAST_CORE_P1 \ + SHIFT_U32(3, BS_CCM_CCGR3_MMDC_CORE_ACLK_FAST_CORE_P1) +#define BS_CCM_CCGR3_MMDC_CORE_IPG_CLK_P0 24 +#define BM_CCM_CCGR3_MMDC_CORE_IPG_CLK_P0 \ + SHIFT_U32(3, BS_CCM_CCGR3_MMDC_CORE_IPG_CLK_P0) +#define BS_CCM_CCGR3_MMDC_CORE_IPG_CLK_P1 26 +#define BM_CCM_CCGR3_MMDC_CORE_IPG_CLK_P1 \ + SHIFT_U32(3, BS_CCM_CCGR3_MMDC_CORE_IPG_CLK_P1) +#define BS_CCM_CCGR3_OCRAM 28 +#define BM_CCM_CCGR3_OCRAM SHIFT_U32(3, BS_CCM_CCGR3_OCRAM) +#define BS_CCM_CCGR3_OPENVGAXICLK 30 +#define BM_CCM_CCGR3_OPENVGAXICLK \ + SHIFT_U32(3, BS_CCM_CCGR3_OPENVGAXICLK) + +#define BS_CCM_CCGR4_PCIE 0 +#define BM_CCM_CCGR4_PCIE SHIFT_U32(3, BS_CCM_CCGR4_PCIE) +#define BS_CCM_CCGR4_PL301_MX6QFAST1_S133 8 +#define BM_CCM_CCGR4_PL301_MX6QFAST1_S133 \ + SHIFT_U32(3, BS_CCM_CCGR4_PL301_MX6QFAST1_S133) +#define BS_CCM_CCGR4_PL301_MX6QPER1_BCH 12 +#define BM_CCM_CCGR4_PL301_MX6QPER1_BCH \ + SHIFT_U32(3, BS_CCM_CCGR4_PL301_MX6QPER1_BCH) +#define BS_CCM_CCGR4_PL301_MX6QPER2_MAINCLK_ENABLE 14 +#define BM_CCM_CCGR4_PL301_MX6QPER2_MAINCLK_ENABLE \ + SHIFT_U32(3, BS_CCM_CCGR4_PL301_MX6QPER2_MAINCLK_ENABLE) +#define BS_CCM_CCGR4_PWM1 16 +#define BM_CCM_CCGR4_PWM1 \ + SHIFT_U32(3, BS_CCM_CCGR4_PWM1) +#define BS_CCM_CCGR4_PWM2 18 +#define BM_CCM_CCGR4_PWM2 \ + SHIFT_U32(3, BS_CCM_CCGR4_PWM2) +#define BS_CCM_CCGR4_PWM3 20 +#define BM_CCM_CCGR4_PWM3 \ + SHIFT_U32(3, BS_CCM_CCGR4_PWM3) +#define BS_CCM_CCGR4_PWM4 22 +#define BM_CCM_CCGR4_PWM4 \ + SHIFT_U32(3, BS_CCM_CCGR4_PWM4) +#define BS_CCM_CCGR4_RAWNAND_U_BCH_INPUT_APB 24 +#define BM_CCM_CCGR4_RAWNAND_U_BCH_INPUT_APB \ + SHIFT_U32(3, BS_CCM_CCGR4_RAWNAND_U_BCH_INPUT_APB) +#define BS_CCM_CCGR4_RAWNAND_U_GPMI_BCH_INPUT_BCH 26 +#define BM_CCM_CCGR4_RAWNAND_U_GPMI_BCH_INPUT_BCH \ + SHIFT_U32(3, BS_CCM_CCGR4_RAWNAND_U_GPMI_BCH_INPUT_BCH) +#define BS_CCM_CCGR4_RAWNAND_U_GPMI_BCH_INPUT_GPMI_IO 28 +#define BM_CCM_CCGR4_RAWNAND_U_GPMI_BCH_INPUT_GPMI_IO \ + SHIFT_U32(3, BS_CCM_CCGR4_RAWNAND_U_GPMI_BCH_INPUT_GPMI_IO) +#define BS_CCM_CCGR4_RAWNAND_U_GPMI_INPUT_APB 30 +#define BM_CCM_CCGR4_RAWNAND_U_GPMI_INPUT_APB \ + SHIFT_U32(3, BS_CCM_CCGR4_RAWNAND_U_GPMI_INPUT_APB) + +#define BS_CCM_CCGR5_ROM 0 +#define BM_CCM_CCGR5_ROM SHIFT_U32(3, BS_CCM_CCGR5_ROM) +#define BS_CCM_CCGR5_SATA 4 +#define BM_CCM_CCGR5_SATA SHIFT_U32(3, BS_CCM_CCGR5_SATA) +#define BS_CCM_CCGR5_SDMA 6 +#define BM_CCM_CCGR5_SDMA SHIFT_U32(3, BS_CCM_CCGR5_SDMA) +#define BS_CCM_CCGR5_SPBA 12 +#define BM_CCM_CCGR5_SPBA SHIFT_U32(3, BS_CCM_CCGR5_SPBA) +#define BS_CCM_CCGR5_SPDIF 14 +#define BM_CCM_CCGR5_SPDIF SHIFT_U32(3, BS_CCM_CCGR5_SPDIF) +#define BS_CCM_CCGR5_SSI1 18 +#define BM_CCM_CCGR5_SSI1 SHIFT_U32(3, BS_CCM_CCGR5_SSI1) +#define BS_CCM_CCGR5_SSI2 20 +#define BM_CCM_CCGR5_SSI2 SHIFT_U32(3, BS_CCM_CCGR5_SSI2) +#define BS_CCM_CCGR5_SSI3 22 +#define BM_CCM_CCGR5_SSI3 SHIFT_U32(3, BS_CCM_CCGR5_SSI3) +#define BS_CCM_CCGR5_UART 24 +#define BM_CCM_CCGR5_UART SHIFT_U32(3, BS_CCM_CCGR5_UART) +#define BS_CCM_CCGR5_UART_SERIAL 26 +#define BM_CCM_CCGR5_UART_SERIAL SHIFT_U32(3, BS_CCM_CCGR5_UART_SERIAL) + +#define BS_CCM_CCGR6_USBOH3 0 +#define BM_CCM_CCGR6_USBOH3 SHIFT_U32(3, BS_CCM_CCGR6_USBOH3) +#define BS_CCM_CCGR6_USDHC1 2 +#define BM_CCM_CCGR6_USDHC1 SHIFT_U32(3, BS_CCM_CCGR6_USDHC1) +#define BS_CCM_CCGR6_USDHC2 4 +#define BM_CCM_CCGR6_USDHC2 SHIFT_U32(3, BS_CCM_CCGR6_USDHC2) +#define BS_CCM_CCGR6_USDHC3 6 +#define BM_CCM_CCGR6_USDHC3 SHIFT_U32(3, BS_CCM_CCGR6_USDHC3) +#define BS_CCM_CCGR6_USDHC4 8 +#define BM_CCM_CCGR6_USDHC4 SHIFT_U32(3, BS_CCM_CCGR6_USDHC4) +#define BS_CCM_CCGR6_EMI_SLOW 10 +#define BM_CCM_CCGR6_EMI_SLOW SHIFT_U32(3, BS_CCM_CCGR6_EMI_SLOW) +#define BS_CCM_CCGR6_VDOAXICLK 12 +#define BM_CCM_CCGR6_VDOAXICLK SHIFT_U32(3, BS_CCM_CCGR6_VDOAXICLK) +#define BS_CCM_CCGR6_I2C4_SERIAL 24 +#define BM_CCM_CCGR6_I2C4_SERIAL SHIFT_U32(3, BS_CCM_CCGR6_I2C4_SERIAL) + +/* + * Define Analog Macros and common bits + */ +#define BS_CCM_ANALOG_PLL_LOCK 31 +#define BM_CCM_ANALOG_PLL_LOCK \ + BIT32(BS_CCM_ANALOG_PLL_LOCK) +#define BS_CCM_ANALOG_PLL_BYPASS 16 +#define BM_CCM_ANALOG_PLL_BYPASS \ + BIT32(BS_CCM_ANALOG_PLL_BYPASS) +#define BS_CCM_ANALOG_PLL_BYPASS_CLK_SRC 14 +#define BM_CCM_ANALOG_PLL_BYPASS_CLK_SRC \ + SHIFT_U32(0x3, BS_CCM_ANALOG_PLL_BYPASS_CLK_SRC) +#define CCM_ANALOG_PLL_BYPASS_CLK_SRC(clk) \ + (SHIFT_U32(clk, BS_CCM_ANALOG_PLL_BYPASS_CLK_SRC) & \ + BM_CCM_ANALOG_PLL_BYPASS_CLK_SRC) +#define CCM_ANALOG_PLL_BYPASS_CLK_SRC_CLK24M 0x0 +#define CCM_ANALOG_PLL_BYPASS_CLK_SRC_CLK1 0x1 +#define CCM_ANALOG_PLL_BYPASS_CLK_SRC_CLK2 0x2 +#define CCM_ANALOG_PLL_BYPASS_CLK_SRC_XOR 0x3 + +#define BS_CCM_ANALOG_PLL_ENABLE 13 +#define BM_CCM_ANALOG_PLL_ENABLE BIT32(BS_CCM_ANALOG_PLL_ENABLE) +#define BS_CCM_ANALOG_PLL_POWERDOWN 12 +#define BM_CCM_ANALOG_PLL_POWERDOWN BIT32(BS_CCM_ANALOG_PLL_POWERDOWN) +#define BS_CCM_ANALOG_PLL_DIV_SELECT 0 + +/* + * Specific Analog bits definition + */ +#define BS_CCM_ANALOG_PLL_ARM_PLL_SEL 19 +#define BM_CCM_ANALOG_PLL_ARM_PLL_SEL BIT32(BS_CCM_ANALOG_PLL_ARM_PLL_SEL) +#define BS_CCM_ANALOG_PLL_ARM_LVDS_24MHZ_SEL 18 +#define BM_CCM_ANALOG_PLL_ARM_LVDS_24MHZ_SEL \ + BIT32(BS_CCM_ANALOG_PLL_ARM_LVDS_24MHZ_SEL) +#define BS_CCM_ANALOG_PLL_ARM_LVDS_SEL 17 +#define BM_CCM_ANALOG_PLL_ARM_LVDS_SEL \ + BIT32(BS_CCM_ANALOG_PLL_ARM_LVDS_SEL) +#define BM_CCM_ANALOG_PLL_ARM_DIV_SELECT \ + SHIFT_U32(0x7F, BS_CCM_ANALOG_PLL_DIV_SELECT) + +#endif /* __IMX6_CRM_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx6-dcp.h b/optee_os/core/arch/arm/plat-imx/registers/imx6-dcp.h new file mode 100644 index 0000000..0067575 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx6-dcp.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020 NXP + */ + +#ifndef __IMX6_DCP_H__ +#define __IMX6_DCP_H__ + +/* DCP base address */ +#ifdef CFG_MX6ULL +#define DCP_BASE 0x2280000 +#endif +#if defined(CFG_MX6SL) || defined(CFG_MX6SLL) +#define DCP_BASE 0x20FC000 +#endif + +/* DCP registers address offset */ +#define DCP_CTRL 0x00 +#define DCP_CTRL_SET 0x04 +#define DCP_CTRL_CLR 0x08 +#define DCP_STAT 0x10 +#define DCP_STAT_CLR 0x18 +#define DCP_CHANNELCTRL 0x20 +#define DCP_CAPABILITY0 0x30 +#define DCP_CAPABILITY1 0x40 +#define DCP_CONTEXT 0x50 +#define DCP_KEY 0x60 +#define DCP_KEYDATA 0x70 +#define DCP_PACKET0 0x80 +#define DCP_PACKET1 0x90 +#define DCP_PACKET2 0xA0 +#define DCP_PACKET3 0xB0 +#define DCP_PACKET4 0xC0 +#define DCP_PACKET5 0xD0 +#define DCP_PACKET6 0xE0 +#define DCP_CH_N_CMDPTR(n) (0x100 + (n) * 0x40) +#define DCP_CH_N_SEMA(n) (0x110 + (n) * 0x40) +#define DCP_CH_N_STAT(n) (0x120 + (n) * 0x40) +#define DCP_CHOCMDPTR 0x100 +#define DCP_CH0SEMA 0x110 +#define DCP_CH0STAT 0x120 +#define DCP_CH1CMDPTR 0x140 +#define DCP_CH2CMDPTR 0x180 +#define DCP_CH3CMDPTR 0x1C0 + +/* DCP CHANNELCTRL register configuration */ +#define DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK GENMASK_32(7, 0) +#define DCP_STAT_CLEAR GENMASK_32(31, 0) +#define DCP_CH_STAT_ERROR_MASK GENMASK_32(23, 0) + +/* DCP CTRL register configuration */ +#define DCP_CTRL_SFTRST BIT32(31) +#define DCP_CTRL_CLKGATE BIT32(30) +#define DCP_CTRL_GATHER_RESIDUAL_WRITES BIT32(23) +#define DCP_CTRL_ENABLE_CONTEXT_CACHING BIT32(22) +#define DCP_CTRL_ENABLE_CONTEXT_SWITCHING BIT32(21) +#define DCP_CTRL_CH3_INTERRUPT_ENABLE BIT32(3) +#define DCP_CTRL_CH2_INTERRUPT_ENABLE BIT32(2) +#define DCP_CTRL_CH1_INTERRUPT_ENABLE BIT32(1) +#define DCP_CTRL_CH0_INTERRUPT_ENABLE BIT32(0) + +/* DCP CAPABILITY0 register configuration */ +#define DCP_CAPABILITY0_DISABLE_UNIQUE_KEY BIT32(29) + +/* Work Packet control0 configuration */ +#define DCP_CONTROL0_OUTPUT_WORDSWAP BIT32(23) +#define DCP_CONTROL0_OUTPUT_BYTESWAP BIT32(22) +#define DCP_CONTROL0_INPUT_WORDSWAP BIT32(21) +#define DCP_CONTROL0_INPUT_BYTESWAP BIT32(20) +#define DCP_CONTROL0_KEY_WORDSWAP BIT32(19) +#define DCP_CONTROL0_KEY_BYTESWA BIT32(18) +#define DCP_CONTROL0_TEST_SEMA_IRQ BIT32(17) +#define DCP_CONTROL0_CONSTANT_FILL BIT32(16) +#define DCP_CONTROL0_HASH_OUTPUT BIT32(15) +#define DCP_CONTROL0_HASH_CHECK BIT32(14) +#define DCP_CONTROL0_HASH_TERM BIT32(13) +#define DCP_CONTROL0_HASH_INIT BIT32(12) +#define DCP_CONTROL0_PAYLOAD_KEY BIT32(11) +#define DCP_CONTROL0_OTP_KEY BIT32(10) +#define DCP_CONTROL0_CIPHER_INIT BIT32(9) +#define DCP_CONTROL0_CIPHER_ENCRYPT BIT32(8) +#define DCP_CONTROL0_ENABLE_BLIT BIT32(7) +#define DCP_CONTROL0_ENABLE_HASH BIT32(6) +#define DCP_CONTROL0_ENABLE_CIPHER BIT32(5) +#define DCP_CONTROL0_ENABLE_MEMCOPY BIT32(4) +#define DCP_CONTROL0_CHAIN_CONTINUOUS BIT32(3) +#define DCP_CONTROL0_CHAIN BIT32(2) +#define DCP_CONTROL0_DECR_SEMAPHORE BIT32(1) +#define DCP_CONTROL0_INTERRUPT_ENABLE BIT32(0) + +/* Work Packet control1 configuration */ +#define DCP_CONTROL1_HASH_SELECT_SHA256 SHIFT_U32(2, 16) +#define DCP_CONTROL1_HASH_SELECT_CRC32 BIT32(16) +#define DCP_CONTROL1_HASH_SELECT_SHA1 SHIFT_U32(0, 16) +#define DCP_CONTROL1_CIPHER_MODE_CBC BIT32(4) +#define DCP_CONTROL1_CIPHER_MODE_ECB SHIFT_U32(0, 4) +#define DCP_CONTROL1_CIPHER_SELECT_AES128 0 +#define DCP_CONTROL1_KEY_SELECT_OTP_CRYPTO SHIFT_U32(0xfe, 8) + +#endif /* __IMX6_DCP_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx6.h b/optee_os/core/arch/arm/plat-imx/registers/imx6.h new file mode 100644 index 0000000..3889989 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx6.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * Copyright 2017-2020 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __IMX6_H__ +#define __IMX6_H__ + +#include +#include + +#define UART1_BASE 0x2020000 +#define IOMUXC_BASE 0x020E0000 +#define IOMUXC_SIZE 0x4000 +#define IOMUXC_GPR_BASE 0x020E4000 +#define SRC_BASE 0x020D8000 +#define SRC_SIZE 0x4000 +#define CCM_BASE 0x020C4000 +#define CCM_SIZE 0x4000 +#define ANATOP_BASE 0x020C8000 +#define ANATOP_SIZE 0x1000 +#define SNVS_BASE 0x020CC000 +#define SNVS_SIZE 0x4000 +#define GPC_BASE 0x020DC000 +#define GPC_SIZE 0x4000 +#define WDOG_BASE 0x020BC000 +#define CSU_BASE 0x021C0000 +#define SEMA4_BASE 0x02290000 +#define SEMA4_SIZE 0x4000 +#define MMDC_P0_BASE 0x021B0000 +#define MMDC_P0_SIZE 0x4000 +#define MMDC_P1_BASE 0x021B4000 +#define MMDC_P1_SIZE 0x4000 +#define TZASC_BASE 0x21D0000 +#define TZASC_SIZE 0x4000 +#define TZASC2_BASE 0x21D4000 +#define UART2_BASE 0x021E8000 +#define UART3_BASE 0x021EC000 +#define UART4_BASE 0x021F0000 +#define UART5_BASE 0x021F4000 +#define AIPS1_BASE 0x02000000 +#define AIPS1_SIZE 0x100000 +#define AIPS2_BASE 0x02100000 +#define AIPS2_SIZE 0x100000 +#define AIPS3_BASE 0x02200000 +#define AIPS3_SIZE 0x100000 + +#if defined(CFG_MX6ULL) +#define RNGB_BASE 0x02284000 +#elif defined(CFG_MX6SL) || defined(CFG_MX6SLL) +#define RNGB_BASE 0x021b4000 +#endif + +#define SCU_BASE 0x00A00000 +#define PL310_BASE 0x00A02000 +#define IRAM_BASE 0x00900000 + +#define OCOTP_BASE 0x021BC000 +#define OCOTP_SIZE 0x4000 + +#define GIC_BASE 0x00A00000 +#define GICD_OFFSET 0x1000 + +#if defined(CFG_MX6UL) || defined(CFG_MX6ULL) +#define GICC_OFFSET 0x2000 +#define UART6_BASE 0x021FC000 +#define UART7_BASE 0x02018000 +/* No CAAM on i.MX6ULL */ +#define CAAM_BASE 0x02140000 +#else +#define GICC_OFFSET 0x100 +#define CAAM_BASE 0x02100000 +#endif +#define CAAM_SIZE 0x40000 + +#define GIC_CPU_BASE (GIC_BASE + GICC_OFFSET) +#define GIC_DIST_BASE (GIC_BASE + GICD_OFFSET) + +/* Central Security Unit register values */ +#define CSU_CSL_START 0x0 +#define CSU_CSL_END 0xA0 +#define CSU_ACCESS_ALL 0x00FF00FF +#define CSU_SETTING_LOCK 0x01000100 +#define CSU_SA 0x218 + +/* Used in suspend/resume and low power idle */ +#define MX6Q_SRC_GPR1 0x20 +#define MX6Q_SRC_GPR2 0x24 +#define MX6Q_MMDC_MISC 0x18 +#define MX6Q_MMDC_MAPSR 0x404 +#define MX6Q_MMDC_MPDGCTRL0 0x83c +#define MX6Q_GPC_IMR1 0x08 +#define MX6Q_GPC_IMR2 0x0c +#define MX6Q_GPC_IMR3 0x10 +#define MX6Q_GPC_IMR4 0x14 +#define MX6Q_CCM_CCR 0x0 +#define MX6Q_ANATOP_CORE 0x140 + +#define IOMUXC_GPR9_OFFSET 0x24 +#define IOMUXC_GPR10_OFFSET 0x28 + +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_OFFSET 5 +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_MASK GENMASK_32(10, 5) + +#define IOMUXC_GPR10_OCRAM_TZ_EN_OFFSET 4 +#define IOMUXC_GPR10_OCRAM_TZ_EN_MASK GENMASK_32(4, 4) + +#define IOMUXC_GPR10_OCRAM_TZ_EN_LOCK_OFFSET 20 +#define IOMUXC_GPR10_OCRAM_TZ_EN_LOCK_MASK GENMASK_32(20, 20) +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_LOCK_OFFSET 21 +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_LOCK_MASK GENMASK_32(26, 21) + +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_OFFSET_6UL 11 +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_MASK_6UL GENMASK_32(15, 11) +#define IOMUXC_GPR10_OCRAM_TZ_EN_OFFSET_6UL 10 +#define IOMUXC_GPR10_OCRAM_TZ_EN_MASK_6UL GENMASK_32(10, 10) + +#define IOMUXC_GPR10_OCRAM_TZ_EN_LOCK_OFFSET_6UL 26 +#define IOMUXC_GPR10_OCRAM_TZ_EN_LOCK_MASK_6UL GENMASK_32(26, 26) +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_LOCK_OFFSET_6UL (27) +#define IOMUXC_GPR10_OCRAM_TZ_ADDR_LOCK_MASK_6UL GENMASK_32(31, 27) + +#ifdef CFG_MX6SL +#define DIGPROG_OFFSET 0x280 +#else +#define DIGPROG_OFFSET 0x260 +#endif + +#if defined(CFG_MX6ULL) +#define I2C1_BASE 0x021a0000 +#define I2C2_BASE 0x021a4000 +#define I2C3_BASE 0x021a8000 +#define I2C4_BASE 0x021f8000 + +#define IOMUXC_I2C1_SCL_CFG_OFF 0x340 +#define IOMUXC_I2C1_SDA_CFG_OFF 0x344 +#define IOMUXC_I2C1_SCL_MUX_OFF 0xb4 +#define IOMUXC_I2C1_SDA_MUX_OFF 0xb8 +#define IOMUXC_I2C1_SCL_INP_OFF 0x5a4 +#define IOMUXC_I2C1_SDA_INP_OFF 0x5a8 +#endif + +#endif /* __IMX6_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx7-crm.h b/optee_os/core/arch/arm/plat-imx/registers/imx7-crm.h new file mode 100644 index 0000000..74a60e4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx7-crm.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ + +#ifndef __IMX7_CRM_H__ +#define __IMX7_CRM_H__ + +/* + * GPRx Registers + */ +#define CCM_GPR0 0x0 +#define CCM_GPRx_OFFSET 0x10 +#define CCM_GPRx(idx) (((idx) * CCM_GRPx_OFFSET) + CCM_GPR0) +#define CCM_GPRx_SET(idx) (CCM_GPRx(idx) + 0x4) +#define CCM_GPRx_CLR(idx) (CCM_GPRx(idx) + 0x8) +#define CCM_GPRx_TOG(idx) (CCM_GPRx(idx) + 0xC) + +/* + * PLL_CTRLx Registers (PLL Control) + */ +#define CCM_PLL_CTRL0 0x800 +#define CCM_PLL_CTRLx_OFFSET 0x10 +#define CCM_PLL_CTRLx(idx) (((idx) * CCM_PLL_CTRLx_OFFSET) + CCM_PLL_CTRL0) +#define CCM_PLL_CTRLx_SET(idx) (CCM_PLL_CTRLx(idx) + 0x4) +#define CCM_PLL_CTRLx_CLR(idx) (CCM_PLL_CTRLx(idx) + 0x8) +#define CCM_PLL_CTRLx_TOG(idx) (CCM_PLL_CTRLx(idx) + 0xC) + +/* + * CCGRx Registers (Clock Gating) + */ +#define CCM_CCGR0 0x4000 +#define CCM_CCGRx_OFFSET 0x10 +#define CCM_CCGRx(idx) (((idx) * CCM_CCGRx_OFFSET) + CCM_CCGR0) +#define CCM_CCGRx_SET(idx) (CCM_CCGRx(idx) + 0x4) +#define CCM_CCGRx_CLR(idx) (CCM_CCGRx(idx) + 0x8) +#define CCM_CCGRx_TOG(idx) (CCM_CCGRx(idx) + 0xC) + +#define BS_CCM_CCGRx_SETTING(idx) ((idx) * 4) +#define BM_CCM_CCGRx_SETTING(idx) \ + SHIFT_U32(0x3, BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_DISABLE(idx) \ + SHIFT_U32(0, BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_RUN(idx) \ + BIT32(BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_RUN_WAIT(idx) \ + SHIFT_U32(0x2, BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_ALWAYS_ON(idx) \ + SHIFT_U32(0x3, BS_CCM_CCGRx_SETTING(idx)) + +/* + * TARGET_ROOTx Registers (Target) + */ +#define CCM_TARGET_ROOT0 0x8000 +#define CCM_TARGET_ROOTx_OFFSET 0x80 +#define CCM_TARGET_ROOTx(idx) \ + (((idx) * CCM_TARGET_ROOTx_OFFSET) + CCM_TARGET_ROOT0) +#define CCM_TARGET_ROOTx_SET(idx) (CCM_TARGET_ROOTx(idx) + 0x4) +#define CCM_TARGET_ROOTx_CLR(idx) (CCM_TARGET_ROOTx(idx) + 0x8) +#define CCM_TARGET_ROOTx_TOG(idx) (CCM_TARGET_ROOTx(idx) + 0xC) + +/* + * MISC_ROOTx Registers (Miscellaneous) + */ +#define CCM_MISC_ROOT0 0x8010 +#define CCM_MISC_ROOTx_OFFSET 0x80 +#define CCM_MISC_ROOTx(idx) \ + (((idx) * CCM_MISC_ROOTx_OFFSET) + CCM_MISC_ROOT0) +#define CCM_MISC_ROOTx_SET(idx) (CCM_MISC_ROOTx(idx) + 0x4) +#define CCM_MISC_ROOTx_CLR(idx) (CCM_MISC_ROOTx(idx) + 0x8) +#define CCM_MISC_ROOTx_TOG(idx) (CCM_MISC_ROOTx(idx) + 0xC) + +/* + * POST_ROOTx Registers (Post Divider) + */ +#define CCM_POST_ROOT0 0x8020 +#define CCM_POST_ROOTx_OFFSET 0x80 +#define CCM_POST_ROOTx(idx) \ + (((idx) * CCM_POST_ROOTx_OFFSET) + CCM_POST_ROOT0) +#define CCM_POST_ROOTx_SET(idx) (CCM_POST_ROOTx(idx) + 0x4) +#define CCM_POST_ROOTx_CLR(idx) (CCM_POST_ROOTx(idx) + 0x8) +#define CCM_POST_ROOTx_TOG(idx) (CCM_POST_ROOTx(idx) + 0xC) + +/* + * PRE_ROOTx Registers (Pre Divider) + */ +#define CCM_PRE_ROOT0 0x8030 +#define CCM_PRE_ROOTx_OFFSET 0x80 +#define CCM_PRE_ROOTx(idx) \ + (((idx) * CCM_PRE_ROOTx_OFFSET) + CCM_PRE_ROOT0) +#define CCM_PRE_ROOTx_SET(idx) (CCM_PRE_ROOTx(idx) + 0x4) +#define CCM_PRE_ROOTx_CLR(idx) (CCM_PRE_ROOTx(idx) + 0x8) +#define CCM_PRE_ROOTx_TOG(idx) (CCM_PRE_ROOTx(idx) + 0xC) + +/* + * ACCESS_CTRL_ROOTx Registers (Access Control) + */ +#define CCM_ACCESS_CTRL_ROOT0 0x8030 +#define CCM_ACCESS_CTRL_ROOTx_OFFSET 0x80 +#define CCM_ACCESS_CTRL_ROOTx(idx) \ + (((idx) * CCM_ACCESS_CTRL_ROOTx_OFFSET) + CCM_ACCESS_CTRL_ROOT0) +#define CCM_ACCESS_CTRL_ROOTx_SET(idx) (CCM_ACCESS_CTRL_ROOTx(idx) + 0x4) +#define CCM_ACCESS_CTRL_ROOTx_CLR(idx) (CCM_ACCESS_CTRL_ROOTx(idx) + 0x8) +#define CCM_ACCESS_CTRL_ROOTx_TOG(idx) (CCM_ACCESS_CTRL_ROOTx(idx) + 0xC) + +/* + * Clock Domain ID + */ +#define CCM_CLOCK_DOMAIN_OCOTP 35 +#define CCM_CLOCK_DOMAIN_CAAM 36 + +#endif /* __IMX7_CRM_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx7.h b/optee_os/core/arch/arm/plat-imx/registers/imx7.h new file mode 100644 index 0000000..6c3baf4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx7.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ +#ifndef __IMX7_H__ +#define __IMX7_H__ + +#include + +#define GIC_BASE 0x31000000 +#define GIC_SIZE 0x8000 +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 + +#define CAAM_BASE 0x30900000 +#define CAAM_SIZE 0x40000 +#define UART1_BASE 0x30860000 +#define UART2_BASE 0x30890000 +#define UART3_BASE 0x30880000 +#define UART4_BASE 0x30A60000 +#define UART5_BASE 0x30A70000 + +#define AIPS1_BASE 0x30000000 +#define AIPS1_SIZE 0x400000 +#define AIPS2_BASE 0x30400000 +#define AIPS2_SIZE 0x400000 +#define AIPS3_BASE 0x30800000 +#define AIPS3_SIZE 0x400000 + +#define WDOG_BASE 0x30280000 +#define LPSR_BASE 0x30270000 +#define IOMUXC_BASE 0x30330000 +#define IOMUXC_SIZE 0x4000 +#define IOMUXC_GPR_BASE 0x30340000 +#define OCOTP_BASE 0x30350000 +#define OCOTP_SIZE 0x10000 +#define ANATOP_BASE 0x30360000 +#define SNVS_BASE 0x30370000 +#define SNVS_SIZE 0x10000 +#define CCM_BASE 0x30380000 +#define CCM_SIZE 0x10000 +#define SRC_BASE 0x30390000 +#define SRC_SIZE 0x4000 +#define GPC_BASE 0x303A0000 +#define GPC_SIZE 0x4000 +#define CSU_BASE 0x303E0000 +#define TZASC_BASE 0x30780000 +#define TZASC_SIZE 0x10000 +#define DDRC_PHY_BASE 0x30790000 +#define MMDC_P0_BASE 0x307A0000 +#define DDRC_BASE 0x307A0000 +#define IRAM_BASE 0x00900000 +#define IRAM_S_BASE 0x00180000 + +#define CSU_CSL_START 0x0 +#define CSU_CSL_END 0x100 +#define CSU_ACCESS_ALL 0x00FF00FF +#define CSU_SETTING_LOCK 0x01000100 +#define CSU_SA 0x218 + +#define TRUSTZONE_OCRAM_START 0x180000 + +#define IOMUXC_GPR9_OFFSET 0x24 +#define IOMUXC_GPR9_TZASC1_MUX_CONTROL_OFFSET 0 + +#define IOMUXC_GPR11_OFFSET 0x2C +#define IOMUXC_GPR11_OCRAM_S_TZ_ADDR_OFFSET 11 +#define IOMUXC_GPR11_OCRAM_S_TZ_ADDR_MASK GENMASK_32(13, 11) + +#define IOMUXC_GPR11_OCRAM_S_TZ_EN_OFFSET 10 +#define IOMUXC_GPR11_OCRAM_S_TZ_EN_MASK GENMASK_32(10, 10) + +#define IOMUXC_GPR11_OCRAM_S_TZ_EN_LOCK_OFFSET 26 +#define IOMUXC_GPR11_OCRAM_S_TZ_EN_LOCK_MASK GENMASK_32(26, 26) +#define IOMUXC_GPR11_OCRAM_S_TZ_ADDR_LOCK_OFFSET GENMASK_32(29, 27) + +#define DIGPROG_OFFSET 0x800 + +#endif /* __IMX7_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx7ulp-crm.h b/optee_os/core/arch/arm/plat-imx/registers/imx7ulp-crm.h new file mode 100644 index 0000000..a14aed3 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx7ulp-crm.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ +#ifndef __IMX7ULP_CRM_H__ +#define __IMX7ULP_CRM_H__ + +#include + +#define PCC_CGC_BIT_SHIFT 30 + +#define PCC_ENABLE_CLOCK BIT32(PCC_CGC_BIT_SHIFT) +#define PCC_DISABLE_CLOCK BIT32(0) + +#define PCC_CAAM 0x90 + +#endif /* __IMX7ULP_CRM_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx7ulp.h b/optee_os/core/arch/arm/plat-imx/registers/imx7ulp.h new file mode 100644 index 0000000..a5abc50 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx7ulp.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ + +#ifndef __IMX7ULP_H__ +#define __IMX7ULP_H__ + +#include + +#define GIC_BASE 0x40020000 +#define GIC_SIZE 0x8000 +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 + +#define AIPS0_BASE 0x40000000 +#define AIPS0_SIZE 0x800000 +#define AIPS1_BASE 0x40800000 +#define AIPS1_SIZE 0x800000 +#define M4_AIPS_BASE 0x41000000 +#define M4_AIPS_SIZE 0x100000 +#define M4_AIPS0_BASE 0x41000000 +#define M4_AIPS0_SIZE 0x80000 +#define M4_AIPS1_BASE 0x41080000 +#define M4_AIPS1_SIZE 0x80000 + +#define GPIOC_BASE 0x400f0000 +#define GPIOD_BASE 0x400f0040 +#define GPIOE_BASE 0x400f0080 +#define GPIOF_BASE 0x400f00c0 +#define TPM5_BASE 0x40260000 +#define WDOG_BASE 0x403d0000 +#define WDOG_SIZE 0x10 +#define SCG1_BASE 0x403e0000 +#define PCC2_BASE 0x403f0000 +#define PMC1_BASE 0x40400000 +#define SMC1_BASE 0x40410000 +#define MMDC_BASE 0x40ab0000 +#define IOMUXC1_BASE 0x40ac0000 +#define MMDC_IO_BASE 0x40ad0000 +#define PCC3_BASE 0x40b30000 +#define OCOTP_BASE 0x410A6000 +#define OCOTP_SIZE 0x4000 +#define PMC0_BASE 0x410a1000 +#define SIM_BASE 0x410a3000 +#define OCOTP_BASE 0x410A6000 +#define OCOTP_SIZE 0x4000 + +#define CAAM_BASE 0x40240000 +#define CAAM_SIZE 0x10000 +#define UART4_BASE 0x402d0000 +#define UART5_BASE 0x402e0000 +#define UART6_BASE 0x40a60000 +#define UART7_BASE 0x40a70000 + +#define IRAM_BASE 0x1FFFC000 +#define IRAM_SIZE 0x4000 + +#define LP_OCRAM_START IRAM_BASE + +#endif /* __IMX7ULP_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx8m-crm.h b/optee_os/core/arch/arm/plat-imx/registers/imx8m-crm.h new file mode 100644 index 0000000..58b92b8 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx8m-crm.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020 Foundries Ltd + */ + +#ifndef __IMX8M_CRM_H +#define __IMX8M_CRM_H + +/* CCGRx Registers (Clock Gating) */ +#define CCM_CCGR0 0x4000 +#define CCM_CCGRx_OFFSET 0x10 +#define CCM_CCGRx(idx) (((idx) * CCM_CCGRx_OFFSET) + CCM_CCGR0) +#define CCM_CCGRx_SET(idx) (CCM_CCGRx(idx) + 0x4) +#define CCM_CCGRx_CLR(idx) (CCM_CCGRx(idx) + 0x8) +#define CCM_CCGRx_TOG(idx) (CCM_CCGRx(idx) + 0xC) + +#define BS_CCM_CCGRx_SETTING(idx) ((idx) * 4) +#define BM_CCM_CCGRx_SETTING(idx) \ + SHIFT_U32(0x3, BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_DISABLE(idx) \ + SHIFT_U32(0, BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_RUN(idx) \ + BIT32(BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_RUN_WAIT(idx) \ + SHIFT_U32(0x2, BS_CCM_CCGRx_SETTING(idx)) +#define CCM_CCGRx_ALWAYS_ON(idx) \ + SHIFT_U32(0x3, BS_CCM_CCGRx_SETTING(idx)) + +#define CCM_CCRG_I2C1 23 +#define CCM_CCRG_I2C2 24 +#define CCM_CCRG_I2C3 25 +#define CCM_CCRG_I2C4 26 +#define CCM_CCRG_OCOTP 34 + +#if defined(CFG_MX8MP) +#define CCM_CCRG_I2C5 51 +#define CCM_CCRG_I2C6 52 +#endif + +#endif /* __IMX8M_CRM_H */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx8m.h b/optee_os/core/arch/arm/plat-imx/registers/imx8m.h new file mode 100644 index 0000000..9bda805 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx8m.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + */ + +#ifndef __IMX8M_H__ +#define __IMX8M_H__ + +#include + +#define GICD_BASE 0x38800000 +#define GICR_BASE 0x38880000 +#define UART1_BASE 0x30860000 +#define UART2_BASE 0x30890000 +#define UART3_BASE 0x30880000 +#define UART4_BASE 0x30A60000 +#define TZASC_BASE 0x32F80000 +#define TZASC_SIZE 0x10000 +#define CAAM_BASE 0x30900000 +#define CAAM_SIZE 0x40000 +#define CCM_BASE 0x30380000 +#define CCM_SIZE 0x10000 +#define ANATOP_BASE 0x30360000 +#define IOMUXC_BASE 0x30330000 +#define OCOTP_BASE 0x30350000 +#define OCOTP_SIZE 0x10000 +#define SNVS_BASE 0x30370000 +#define SNVS_SIZE 0x10000 +#define SECMEM_BASE 0x00100000 +#define SECMEM_SIZE 0x8000 + +#ifdef CFG_MX8MQ +#define DIGPROG_OFFSET 0x06c +#define OCOTP_SW_INFO_B1 0x40 +#define OCOTP_SW_MAGIC_B1 0xFF0055AA +#endif +#if defined(CFG_MX8MM) || defined(CFG_MX8MN) || defined(CFG_MX8MP) +#define DIGPROG_OFFSET 0x800 +#endif + +#if defined(CFG_MX8MM) || defined(CFG_MX8MQ) || defined(CFG_MX8MN) +#define I2C1_BASE 0x30a20000 +#define I2C2_BASE 0x30a30000 +#define I2C3_BASE 0x30a40000 +#define I2C4_BASE 0x30a50000 + +#define IOMUXC_I2C1_SCL_CFG_OFF 0x47C +#define IOMUXC_I2C1_SDA_CFG_OFF 0x480 +#define IOMUXC_I2C1_SCL_MUX_OFF 0x214 +#define IOMUXC_I2C1_SDA_MUX_OFF 0x218 +#endif + +#if defined(CFG_MX8MP) +#define I2C1_BASE 0x30a20000 +#define I2C2_BASE 0x30a30000 +#define I2C3_BASE 0x30a40000 +#define I2C4_BASE 0x30a50000 +#define I2C5_BASE 0x30ad0000 +#define I2C6_BASE 0x30ae0000 + +#define IOMUXC_I2C1_SCL_CFG_OFF 0x460 +#define IOMUXC_I2C1_SDA_CFG_OFF 0x464 +#define IOMUXC_I2C1_SCL_MUX_OFF 0x200 +#define IOMUXC_I2C1_SDA_MUX_OFF 0x204 +#endif + +#endif /* __IMX8M_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx8q.h b/optee_os/core/arch/arm/plat-imx/registers/imx8q.h new file mode 100644 index 0000000..b085e62 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx8q.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + */ + +#ifndef __IMX8Q_H__ +#define __IMX8Q_H__ + +#define GICD_BASE 0x51a00000 +#define GICR_BASE 0x51b00000 +#define UART0_BASE 0x5a060000 +#define UART1_BASE 0x5a070000 +#define UART2_BASE 0x5a080000 +#define UART3_BASE 0x5a090000 +#define UART4_BASE 0x5a0a0000 +#define CAAM_BASE 0x31400000 +#define CAAM_SIZE 0x40000 +#define SC_IPC_BASE_SECURE 0x5d1b0000 +#define SC_IPC_SIZE 0x10000 + +#endif /* __IMX8Q_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx8ulp-crm.h b/optee_os/core/arch/arm/plat-imx/registers/imx8ulp-crm.h new file mode 100644 index 0000000..1882b54 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx8ulp-crm.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + */ +#ifndef __IMX8ULP_CRM_H__ +#define __IMX8ULP_CRM_H__ + +#include + +#define PCC_CGC_BIT_SHIFT 30 +#define PCC_ENABLE_CLOCK BIT32(PCC_CGC_BIT_SHIFT) +#define PCC_CAAM 0xB8 + +#endif /* __IMX8ULP_CRM_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx8ulp.h b/optee_os/core/arch/arm/plat-imx/registers/imx8ulp.h new file mode 100644 index 0000000..7fe824b --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx8ulp.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + */ +#ifndef __IMX8ULP_H__ +#define __IMX8ULP_H__ + +#include + +#define GICD_BASE 0x2d400000 +#define GICR_BASE 0x2d440000 +#define UART4_BASE 0x29390000 +#define UART5_BASE 0x293a0000 +#define CAAM_BASE 0x292e0000 +#define CAAM_SIZE 0x10000 +#define PCC3_BASE 0x292d0000 +#define PCC3_SIZE 0x1000 +#define AIPS3_BASE 0x29000000 +#define AIPS3_SIZE 0x400000 +#define SECMEM_BASE 0x00100000 +#define SECMEM_SIZE 0x80000 +#define MU_BASE 0x27020000 +#define MU_SIZE 0x10000 + +#endif /* __IMX8ULP_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/registers/imx93.h b/optee_os/core/arch/arm/plat-imx/registers/imx93.h new file mode 100644 index 0000000..e916075 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/registers/imx93.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ +#ifndef __IMX93_H__ +#define __IMX93_H__ + +#define GICD_BASE 0x48000000 +#define GICR_BASE 0x48040000 + +#define UART1_BASE 0x44380000 +#define MU_BASE 0x47520000 +#define MU_SIZE 0x10000 + +#endif /* __IMX93_H__ */ diff --git a/optee_os/core/arch/arm/plat-imx/sm_platform_handler.c b/optee_os/core/arch/arm/plat-imx/sm_platform_handler.c new file mode 100644 index 0000000..0828891 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/sm_platform_handler.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + */ + +#include +#include +#include +#include +#include "imx_pl310.h" + +#define IMX_SIP_PL310_ENABLE 1 +#define IMX_SIP_PL310_DISABLE 2 +#define IMX_SIP_PL310_ENABLE_WRITEBACK 3 +#define IMX_SIP_PL310_DISABLE_WRITEBACK 4 +#define IMX_SIP_PL310_ENABLE_WFLZ 5 + +static enum sm_handler_ret imx_sip_handler(struct thread_smc_args *smc_args) +{ + uint16_t sip_func = OPTEE_SMC_FUNC_NUM(smc_args->a0); + + switch (sip_func) { +#ifdef CFG_PL310_SIP_PROTOCOL + case IMX_SIP_PL310_ENABLE: + smc_args->a0 = pl310_enable(); + break; + case IMX_SIP_PL310_DISABLE: + smc_args->a0 = pl310_disable(); + break; + case IMX_SIP_PL310_ENABLE_WRITEBACK: + smc_args->a0 = pl310_enable_writeback(); + break; + case IMX_SIP_PL310_DISABLE_WRITEBACK: + smc_args->a0 = pl310_disable_writeback(); + break; + case IMX_SIP_PL310_ENABLE_WFLZ: + smc_args->a0 = pl310_enable_wflz(); + break; +#endif + default: + EMSG("Invalid SIP function code: 0x%x", sip_func); + smc_args->a0 = OPTEE_SMC_RETURN_EBADCMD; + break; + } + + return SM_HANDLER_SMC_HANDLED; +} + +enum sm_handler_ret sm_platform_handler(struct sm_ctx *ctx) +{ + uint32_t *nsec_r0 = (uint32_t *)(&ctx->nsec.r0); + uint16_t smc_owner = OPTEE_SMC_OWNER_NUM(*nsec_r0); + + switch (smc_owner) { + case OPTEE_SMC_OWNER_SIP: + return imx_sip_handler((struct thread_smc_args *)nsec_r0); + default: + return SM_HANDLER_PENDING_SMC; + } +} diff --git a/optee_os/core/arch/arm/plat-imx/sub.mk b/optee_os/core/arch/arm/plat-imx/sub.mk new file mode 100644 index 0000000..0849d03 --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/sub.mk @@ -0,0 +1,20 @@ +global-incdirs-y += . +srcs-y += main.c imx-common.c + +srcs-$(CFG_PL310) += imx_pl310.c +ifeq ($(CFG_PSCI_ARM32),y) +$(call force,CFG_PM_ARM32,y) +CFG_IMX_PM ?= y +endif + +ifneq (,$(filter y, $(CFG_MX6Q) $(CFG_MX6QP) $(CFG_MX6D) $(CFG_MX6DL) \ + $(CFG_MX6S) $(CFG_MX6SL) $(CFG_MX6SLL) $(CFG_MX6SX))) +srcs-y += a9_plat_init.S +srcs-$(CFG_SM_PLATFORM_HANDLER) += sm_platform_handler.c +endif + +ifneq (,$(filter y, $(CFG_MX7) $(CFG_MX7ULP) $(CFG_MX6UL) $(CFG_MX6ULL))) +srcs-y += a7_plat_init.S +endif + +srcs-$(CFG_TZC380) += tzc380.c diff --git a/optee_os/core/arch/arm/plat-imx/tzc380.c b/optee_os/core/arch/arm/plat-imx/tzc380.c new file mode 100644 index 0000000..ecc545b --- /dev/null +++ b/optee_os/core/arch/arm/plat-imx/tzc380.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Pengutronix + * All rights reserved. + * Copyright 2023 NXP + * + * Rouven Czerwinski + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * TZASC2_BASE is asserted non null when used. + * This is needed to compile the code for i.MX6UL/L + * and i.MX8MQ. + */ +#ifndef TZASC2_BASE +#define TZASC2_BASE 0 +#else +register_phys_mem(MEM_AREA_IO_SEC, TZASC2_BASE, TZASC_SIZE); +#endif + +register_phys_mem(MEM_AREA_IO_SEC, TZASC_BASE, TZASC_SIZE); + +static int imx_tzc_auto_configure(vaddr_t addr, vaddr_t rsize, uint32_t attr, + uint8_t region) +{ + vaddr_t addr_imx = 0; + + /* + * On 8mscale platforms, the TZASC controller for the DRAM protection, + * has the memory regions starting at address 0x0 instead of the DRAM + * base address (0x40000000) + */ + if (IS_ENABLED(CFG_MX8M)) + addr_imx = addr - CFG_DRAM_BASE; + else + addr_imx = addr; + + return tzc_auto_configure(addr_imx, rsize, attr, region); +} + +static TEE_Result imx_configure_tzasc(void) +{ + vaddr_t addr[2] = {0}; + int end = 1; + int i = 0; + + addr[0] = core_mmu_get_va(TZASC_BASE, MEM_AREA_IO_SEC, 1); + + if (IS_ENABLED(CFG_MX6Q) || IS_ENABLED(CFG_MX6D) || + IS_ENABLED(CFG_MX6DL)) { + assert(TZASC2_BASE != 0); + addr[1] = core_mmu_get_va(TZASC2_BASE, MEM_AREA_IO_SEC, 1); + end = 2; + } + + for (i = 0; i < end; i++) { + uint8_t region = 1; + + tzc_init(addr[i]); + + region = imx_tzc_auto_configure(CFG_DRAM_BASE, CFG_DDR_SIZE, + TZC_ATTR_SP_NS_RW, region); + region = imx_tzc_auto_configure(CFG_TZDRAM_START, + CFG_TZDRAM_SIZE, + TZC_ATTR_SP_S_RW, region); + region = imx_tzc_auto_configure(CFG_SHMEM_START, CFG_SHMEM_SIZE, + TZC_ATTR_SP_ALL, region); + + if (tzc_regions_lockdown() != TEE_SUCCESS) + panic("Region lockdown failed!"); + + tzc_dump_state(); + } + return TEE_SUCCESS; +} + +static TEE_Result +pm_enter_resume(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *pm_handle __unused) +{ + if (op == PM_OP_RESUME) + return imx_configure_tzasc(); + + return TEE_SUCCESS; +} + +static TEE_Result tzasc_init(void) +{ + register_pm_driver_cb(pm_enter_resume, NULL, "imx-tzasc"); + + return imx_configure_tzasc(); +} +driver_init(tzasc_init); diff --git a/optee_os/core/arch/arm/plat-k3/conf.mk b/optee_os/core/arch/arm/plat-k3/conf.mk new file mode 100644 index 0000000..2fd235e --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/conf.mk @@ -0,0 +1,37 @@ +CFG_WITH_STATS ?= y +CFG_CRYPTO_WITH_CE ?= y +CFG_CONSOLE_UART ?= 0 + +CFG_TZDRAM_START ?= 0x9e800000 +CFG_TZDRAM_SIZE ?= 0x01400000 # 20MB +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) +CFG_SHMEM_SIZE ?= 0x00400000 # 4MB + +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_8250_UART,y) +$(call force,CFG_HWSUPP_MEM_PERM_PXN,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_GIC,y) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_K3_OTP_KEYWRITING,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) + +ifneq (,$(filter ${PLATFORM_FLAVOR},am65x)) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +endif + +ifneq (,$(filter ${PLATFORM_FLAVOR},am65x j721e j784s4 am64x am62x)) +CFG_WITH_SOFTWARE_PRNG ?= n +else +$(call force,CFG_WITH_SOFTWARE_PRNG,y) +endif + +ifneq ($(CFG_WITH_SOFTWARE_PRNG),y) +$(call force,CFG_SA2UL,y) +CFG_HWRNG_QUALITY ?= 1024 +CFG_HWRNG_PTA ?= y +endif + +include core/arch/arm/cpu/cortex-armv8-0.mk diff --git a/optee_os/core/arch/arm/plat-k3/drivers/sa2ul.c b/optee_os/core/arch/arm/plat-k3/drivers/sa2ul.c new file mode 100644 index 0000000..e11423e --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/sa2ul.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Texas Instruments K3 SA2UL Driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * Andrew Davis + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sa2ul.h" + +#define SA2UL_ES 0x0008 +#define SA2UL_ES_TRNG BIT(3) +#define SA2UL_EEC 0x1000 +#define SA2UL_EEC_TRNG BIT(3) + +#define FW_ENABLE_REGION 0x0a +#define FW_BACKGROUND_REGION BIT(8) +#define FW_BIG_ARM_PRIVID 0x01 +#define FW_TIFS_PRIVID 0xca +#define FW_WILDCARD_PRIVID 0xc3 +#define FW_SECURE_ONLY GENMASK_32(7, 0) +#define FW_NON_SECURE GENMASK_32(15, 0) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SA2UL_BASE, SA2UL_REG_SIZE); + +static TEE_Result sa2ul_init(void) +{ + vaddr_t sa2ul = (vaddr_t)phys_to_virt(SA2UL_BASE, MEM_AREA_IO_SEC, + RNG_REG_SIZE); + uint16_t fwl_id = SA2UL_TI_SCI_FW_ID; + uint16_t sa2ul_region = SA2UL_TI_SCI_FW_RGN_ID; + uint16_t rng_region = RNG_TI_SCI_FW_RGN_ID; + uint8_t owner_index = OPTEE_HOST_ID; + uint8_t owner_privid = 0; + uint16_t owner_permission_bits = 0; + uint32_t control = 0; + uint32_t permissions[FWL_MAX_PRIVID_SLOTS] = { }; + uint32_t num_perm = 0; + uint64_t start_address = 0; + uint64_t end_address = 0; + uint32_t val = 0; + TEE_Result result = TEE_SUCCESS; + int ret = 0; + + if (SA2UL_TI_SCI_DEV_ID != -1) { + /* Power on the SA2UL device */ + ret = ti_sci_device_get(SA2UL_TI_SCI_DEV_ID); + if (ret) { + EMSG("Failed to get SA2UL device"); + return TEE_ERROR_GENERIC; + } + } + + IMSG("Activated SA2UL device"); + + /* Try to claim the SA2UL firewall for ourselves */ + ret = ti_sci_change_fwl_owner(fwl_id, sa2ul_region, owner_index, + &owner_privid, &owner_permission_bits); + if (ret) { + /* + * This is not fatal, it just means we are on an HS device + * where the DMSC already owns the SA2UL. On GP we need + * to do additional setup for access permissions below. + */ + DMSG("Could not change SA2UL firewall owner"); + } else { + IMSG("Fixing SA2UL firewall owner for GP device"); + + /* Get current SA2UL firewall configuration */ + ret = ti_sci_get_fwl_region(fwl_id, sa2ul_region, 1, + &control, permissions, + &start_address, &end_address); + if (ret) { + EMSG("Could not get firewall region information"); + return TEE_ERROR_GENERIC; + } + + /* Modify SA2UL firewall to allow all others access*/ + control = FW_BACKGROUND_REGION | FW_ENABLE_REGION; + permissions[0] = (FW_WILDCARD_PRIVID << 16) | FW_NON_SECURE; + ret = ti_sci_set_fwl_region(fwl_id, sa2ul_region, 1, + control, permissions, + 0x0, UINT32_MAX); + if (ret) { + EMSG("Could not set firewall region information"); + return TEE_ERROR_GENERIC; + } + } + + /* Claim the TRNG firewall for ourselves */ + ret = ti_sci_change_fwl_owner(fwl_id, rng_region, owner_index, + &owner_privid, &owner_permission_bits); + if (ret) { + EMSG("Could not change TRNG firewall owner"); + return TEE_ERROR_GENERIC; + } + + /* Get current TRNG firewall configuration */ + ret = ti_sci_get_fwl_region(fwl_id, rng_region, 1, + &control, permissions, + &start_address, &end_address); + if (ret) { + EMSG("Could not get firewall region information"); + return TEE_ERROR_GENERIC; + } + + /* Modify TRNG firewall to block all others access */ + control = FW_ENABLE_REGION; + start_address = RNG_BASE; + end_address = RNG_BASE + RNG_REG_SIZE - 1; + permissions[num_perm++] = (FW_BIG_ARM_PRIVID << 16) | FW_SECURE_ONLY; +#if defined(PLATFORM_FLAVOR_am62x) + permissions[num_perm++] = (FW_TIFS_PRIVID << 16) | FW_NON_SECURE; +#endif + ret = ti_sci_set_fwl_region(fwl_id, rng_region, num_perm, + control, permissions, + start_address, end_address); + if (ret) { + EMSG("Could not set firewall region information"); + return TEE_ERROR_GENERIC; + } + + IMSG("Enabled firewalls for SA2UL TRNG device"); + + /* Enable RNG engine in SA2UL if not already enabled */ + val = io_read32(sa2ul + SA2UL_ES); + if (!(val & SA2UL_ES_TRNG)) { + IMSG("Enabling SA2UL TRNG engine"); + io_setbits32(sa2ul + SA2UL_EEC, SA2UL_EEC_TRNG); + } + + /* Initialize the RNG Module */ + result = sa2ul_rng_init(); + if (result != TEE_SUCCESS) + return result; + + IMSG("SA2UL Drivers initialized"); + + return TEE_SUCCESS; +} +driver_init(sa2ul_init); diff --git a/optee_os/core/arch/arm/plat-k3/drivers/sa2ul.h b/optee_os/core/arch/arm/plat-k3/drivers/sa2ul.h new file mode 100644 index 0000000..58a3eec --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/sa2ul.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Texas Instruments K3 SA2UL Driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * Andrew Davis + */ + +#ifndef __DRIVERS_SA2UL_H +#define __DRIVERS_SA2UL_H + +#include + +TEE_Result sa2ul_rng_init(void); + +#endif /* __DRIVERS_SA2UL_H */ diff --git a/optee_os/core/arch/arm/plat-k3/drivers/sa2ul_rng.c b/optee_os/core/arch/arm/plat-k3/drivers/sa2ul_rng.c new file mode 100644 index 0000000..1684bff --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/sa2ul_rng.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Texas Instruments K3 SA2UL RNG Driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * Andrew Davis + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sa2ul.h" + +#define RNG_OUTPUT_0 0x00 +#define RNG_OUTPUT_1 0x04 +#define RNG_OUTPUT_2 0x08 +#define RNG_OUTPUT_3 0x0C +#define RNG_STATUS 0x10 +#define RNG_READY BIT(0) +#define SHUTDOWN_OFLO BIT(1) +#define RNG_INTACK 0x10 +#define RNG_CONTROL 0x14 +#define ENABLE_TRNG BIT(10) +#define RNG_CONFIG 0x18 +#define RNG_ALARMCNT 0x1C +#define RNG_FROENABLE 0x20 +#define RNG_FRODETUNE 0x24 +#define RNG_ALARMMASK 0x28 +#define RNG_ALARMSTOP 0x2C +#define RNG_OPTIONS 0x78 +#define RNG_EIP_REV 0x7C + +#define RNG_CONTROL_STARTUP_CYCLES_SHIFT 16 +#define RNG_CONTROL_STARTUP_CYCLES_MASK GENMASK_32(31, 16) + +#define RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT 16 +#define RNG_CONFIG_MAX_REFIL_CYCLES_MASK GENMASK_32(31, 16) +#define RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT 0 +#define RNG_CONFIG_MIN_REFIL_CYCLES_MASK GENMASK_32(7, 0) + +#define RNG_ALARMCNT_ALARM_TH_SHIFT 0 +#define RNG_ALARMCNT_ALARM_TH_MASK GENMASK_32(7, 0) +#define RNG_ALARMCNT_SHUTDOWN_TH_SHIFT 16 +#define RNG_ALARMCNT_SHUTDOWN_TH_MASK GENMASK_32(20, 16) + +#define RNG_CONTROL_STARTUP_CYCLES 0xff +#define RNG_CONFIG_MIN_REFIL_CYCLES 0x5 +#define RNG_CONFIG_MAX_REFIL_CYCLES 0x22 +#define RNG_ALARM_THRESHOLD 0xff +#define RNG_SHUTDOWN_THRESHOLD 0x4 + +#define RNG_FRO_MASK GENMASK_32(23, 0) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, RNG_BASE, RNG_REG_SIZE); + +static unsigned int rng_lock = SPINLOCK_UNLOCK; +static vaddr_t rng; + +static void sa2ul_rng_read128(uint32_t *word0, uint32_t *word1, + uint32_t *word2, uint32_t *word3) +{ + /* Is the result ready (available)? */ + while (!(io_read32(rng + RNG_STATUS) & RNG_READY)) { + /* Is the shutdown threshold reached? */ + if (io_read32(rng + RNG_STATUS) & SHUTDOWN_OFLO) { + uint32_t alarm = io_read32(rng + RNG_ALARMSTOP); + uint32_t tune = io_read32(rng + RNG_FRODETUNE); + + /* Clear the alarm events */ + io_write32(rng + RNG_ALARMMASK, 0x0); + io_write32(rng + RNG_ALARMSTOP, 0x0); + /* De-tune offending FROs */ + io_write32(rng + RNG_FRODETUNE, tune ^ alarm); + /* Re-enable the shut down FROs */ + io_write32(rng + RNG_FROENABLE, RNG_FRO_MASK); + /* Clear the shutdown overflow event */ + io_write32(rng + RNG_INTACK, SHUTDOWN_OFLO); + + DMSG("Fixed FRO shutdown"); + } + } + /* Read random value */ + *word0 = io_read32(rng + RNG_OUTPUT_0); + *word1 = io_read32(rng + RNG_OUTPUT_1); + *word2 = io_read32(rng + RNG_OUTPUT_2); + *word3 = io_read32(rng + RNG_OUTPUT_3); + /* Acknowledge read complete */ + io_write32(rng + RNG_INTACK, RNG_READY); +} + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + static union { + uint32_t val[4]; + uint8_t byte[16]; + } fifo; + static size_t fifo_pos; + uint8_t *buffer = buf; + size_t buffer_pos = 0; + + while (buffer_pos < len) { + uint32_t exceptions = cpu_spin_lock_xsave(&rng_lock); + + /* Refill our FIFO */ + if (fifo_pos == 0) + sa2ul_rng_read128(&fifo.val[0], &fifo.val[1], + &fifo.val[2], &fifo.val[3]); + + buffer[buffer_pos++] = fifo.byte[fifo_pos++]; + fifo_pos %= 16; + + cpu_spin_unlock_xrestore(&rng_lock, exceptions); + } + + return TEE_SUCCESS; +} + +TEE_Result sa2ul_rng_init(void) +{ + uint32_t val = 0; + + rng = (vaddr_t)phys_to_virt(RNG_BASE, MEM_AREA_IO_SEC, RNG_REG_SIZE); + + /* Ensure initial latency */ + val |= RNG_CONFIG_MIN_REFIL_CYCLES << RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT; + val |= RNG_CONFIG_MAX_REFIL_CYCLES << RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT; + io_write32(rng + RNG_CONFIG, val); + + /* Configure the desired FROs */ + io_write32(rng + RNG_FRODETUNE, 0x0); + + /* Enable all FROs */ + io_write32(rng + RNG_FROENABLE, 0xffffff); + + io_write32(rng + RNG_CONTROL, ENABLE_TRNG); + + IMSG("SA2UL TRNG initialized"); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/arch/arm/plat-k3/drivers/sec_proxy.c b/optee_os/core/arch/arm/plat-k3/drivers/sec_proxy.c new file mode 100644 index 0000000..3961d72 --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/sec_proxy.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Texas Instruments K3 Secure Proxy Driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * Manorit Chawdhry + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sec_proxy.h" + +/* SEC PROXY RT THREAD STATUS */ +#define RT_THREAD_STATUS_REG 0x0 +#define RT_THREAD_STATUS_ERROR_MASK BIT(31) +#define RT_THREAD_STATUS_CUR_CNT_MASK GENMASK_32(7, 0) + +/* SEC PROXY SCFG THREAD CTRL */ +#define SCFG_THREAD_CTRL_REG 0x1000 +#define SCFG_THREAD_CTRL_DIR_SHIFT 31 +#define SCFG_THREAD_CTRL_DIR_MASK BIT(31) + +/* SECURE PROXY GENERIC HELPERS */ +enum threads { + SEC_PROXY_TX_THREAD, + SEC_PROXY_RX_THREAD, + SEC_PROXY_MAX_THREADS +}; + +#define SEC_PROXY_THREAD(base, x) ((base) + (0x1000 * (x))) +#define SEC_PROXY_DATA_START_OFFS 0x4 +#define SEC_PROXY_DATA_END_OFFS 0x3c + +#define THREAD_DIR_TX (0) +#define THREAD_DIR_RX (1) + +/** + * struct k3_sec_proxy_thread - Description of a Secure Proxy Thread + * @id: Thread ID + * @data: Thread Data path region for target + * @scfg: Secure Config Region for Thread + * @rt: RealTime Region for Thread + */ +struct k3_sec_proxy_thread { + const char *name; + vaddr_t data; + vaddr_t scfg; + vaddr_t rt; +} spts[SEC_PROXY_MAX_THREADS]; + +/** + * k3_sec_proxy_verify_thread() - Verify thread status before + * sending/receiving data + * @dir: Direction of the thread + */ +static TEE_Result k3_sec_proxy_verify_thread(uint32_t dir) +{ + struct k3_sec_proxy_thread *spt = &spts[dir]; + uint64_t timeout = 0; + uint32_t val = 0; + unsigned int retry = 2; + + FMSG("Check for thread corruption"); + val = io_read32(spt->rt + RT_THREAD_STATUS_REG); + + /* Check for any errors already available */ + while ((val & RT_THREAD_STATUS_ERROR_MASK) && retry--) { + if (!retry) { + EMSG("Thread %s is corrupted, cannot send data.", + spt->name); + return TEE_ERROR_BAD_STATE; + } + + /* Write Bit 0 to this location */ + IMSG("Resetting proxy thread %s", spt->name); + val ^= RT_THREAD_STATUS_ERROR_MASK; + io_write32(spt->rt + RT_THREAD_STATUS_REG, val); + } + + FMSG("Check for thread direction"); + /* Make sure thread is configured for right direction */ + if ((io_read32(spt->scfg + SCFG_THREAD_CTRL_REG) & + SCFG_THREAD_CTRL_DIR_MASK) >> SCFG_THREAD_CTRL_DIR_SHIFT != dir) { + if (dir == SEC_PROXY_TX_THREAD) + EMSG("Trying to receive data on tx Thread %s", + spt->name); + else + EMSG("Trying to send data on rx Thread %s", spt->name); + return TEE_ERROR_COMMUNICATION; + } + + FMSG("Check for thread queue"); + /* Check the message queue before sending/receiving data */ + timeout = timeout_init_us(SEC_PROXY_TIMEOUT_US); + while (!(io_read32(spt->rt + RT_THREAD_STATUS_REG) & + RT_THREAD_STATUS_CUR_CNT_MASK)) { + DMSG("Waiting for thread %s to %s", spt->name, + (dir == THREAD_DIR_TX) ? "empty" : "fill"); + if (timeout_elapsed(timeout)) { + EMSG("Queue is busy"); + return TEE_ERROR_BUSY; + } + } + + FMSG("Success"); + return TEE_SUCCESS; +} + +/** + * k3_sec_proxy_send() - Send data over a Secure Proxy thread + * @msg: Pointer to k3_sec_proxy_msg + */ +TEE_Result k3_sec_proxy_send(const struct k3_sec_proxy_msg *msg) +{ + struct k3_sec_proxy_thread *spt = &spts[SEC_PROXY_TX_THREAD]; + int num_words = 0; + int trail_bytes = 0; + int i = 0; + uintptr_t data_reg = 0; + uint32_t data_word = 0; + TEE_Result ret = TEE_SUCCESS; + + FMSG("Verifying the thread"); + ret = k3_sec_proxy_verify_thread(THREAD_DIR_TX); + if (ret) { + EMSG("Thread %s verification failed. ret = %d", spt->name, ret); + return ret; + } + + /* Check the message size. */ + if (msg->len > SEC_PROXY_MAX_MSG_SIZE) { + EMSG("Thread %s message length %zu > max msg size %d", + spt->name, msg->len, SEC_PROXY_MAX_MSG_SIZE); + return TEE_ERROR_BAD_STATE; + } + + /* Send the message */ + data_reg = spt->data + SEC_PROXY_DATA_START_OFFS; + num_words = msg->len / sizeof(uint32_t); + for (i = 0; i < num_words; i++) { + memcpy(&data_word, &msg->buf[i * 4], sizeof(uint32_t)); + io_write32(data_reg, data_word); + data_reg += sizeof(uint32_t); + } + + trail_bytes = msg->len % sizeof(uint32_t); + if (trail_bytes) { + uint32_t data_trail = 0; + + i = msg->len - trail_bytes; + while (trail_bytes--) { + data_trail <<= 8; + data_trail |= msg->buf[i++]; + } + + io_write32(data_reg, data_trail); + data_reg += sizeof(uint32_t); + } + + /* + * 'data_reg' indicates next register to write. If we did not already + * write on tx complete reg(last reg), we must do so for transmit + */ + if (data_reg <= (spt->data + SEC_PROXY_DATA_END_OFFS)) + io_write32(spt->data + SEC_PROXY_DATA_END_OFFS, 0); + + return TEE_SUCCESS; +} + +/** + * k3_sec_proxy_recv() - Receive data from a Secure Proxy thread + * @msg: Pointer to k3_sec_proxy_msg + */ +TEE_Result k3_sec_proxy_recv(struct k3_sec_proxy_msg *msg) +{ + struct k3_sec_proxy_thread *spt = &spts[SEC_PROXY_RX_THREAD]; + int num_words = 0; + int i = 0; + int trail_bytes = 0; + uint32_t data_trail = 0; + uint32_t data_word = 0; + uintptr_t data_reg = 0; + TEE_Result ret = TEE_SUCCESS; + + FMSG("Verifying thread"); + ret = k3_sec_proxy_verify_thread(THREAD_DIR_RX); + if (ret) { + EMSG("Thread %s verification failed. ret = %d", spt->name, ret); + return ret; + } + + /* Receive the message */ + data_reg = spt->data + SEC_PROXY_DATA_START_OFFS; + num_words = msg->len / sizeof(uint32_t); + for (i = 0; i < num_words; i++) { + data_word = io_read32(data_reg); + memcpy(&msg->buf[i * 4], &data_word, sizeof(uint32_t)); + data_reg += sizeof(uint32_t); + } + + trail_bytes = msg->len % sizeof(uint32_t); + if (trail_bytes) { + data_trail = io_read32(data_reg); + data_reg += sizeof(uint32_t); + + i = msg->len - trail_bytes; + while (trail_bytes--) { + msg->buf[i] = data_trail & 0xff; + data_trail >>= 8; + } + } + + /* + * 'data_reg' indicates next register to read. If we did not already + * read on rx complete reg(last reg), we must do so for receive + */ + if (data_reg <= (spt->data + SEC_PROXY_DATA_END_OFFS)) + io_read32(spt->data + SEC_PROXY_DATA_END_OFFS); + + return TEE_SUCCESS; +} + +/** + * k3_sec_proxy_init() - Initialize the secure proxy threads + */ +TEE_Result k3_sec_proxy_init(void) +{ + struct k3_sec_proxy_thread *thread; + int rx_thread = SEC_PROXY_RESPONSE_THREAD; + int tx_thread = SEC_PROXY_REQUEST_THREAD; + uint32_t target_data = 0; + uint32_t cfg_scfg = 0; + uint32_t cfg_rt = 0; + + DMSG("tx_thread: %d, rx_thread: %d", tx_thread, rx_thread); + + /* TX_THREAD */ + target_data = SEC_PROXY_THREAD(SEC_PROXY_DATA_BASE, tx_thread); + cfg_scfg = SEC_PROXY_THREAD(SEC_PROXY_SCFG_BASE, tx_thread); + cfg_rt = SEC_PROXY_THREAD(SEC_PROXY_RT_BASE, tx_thread); + + thread = &spts[SEC_PROXY_TX_THREAD]; + thread->name = "SEC_PROXY_LOW_PRIORITY_THREAD"; + + thread->data = core_mmu_get_va(target_data, MEM_AREA_IO_SEC, + SEC_PROXY_DATA_SIZE); + if (!thread->data) + return TEE_ERROR_OUT_OF_MEMORY; + + thread->scfg = core_mmu_get_va(cfg_scfg, MEM_AREA_IO_SEC, + SEC_PROXY_SCFG_SIZE); + if (!thread->scfg) + return TEE_ERROR_OUT_OF_MEMORY; + + thread->rt = core_mmu_get_va(cfg_rt, MEM_AREA_IO_SEC, + SEC_PROXY_RT_SIZE); + if (!thread->rt) + return TEE_ERROR_OUT_OF_MEMORY; + + /* RX_THREAD */ + target_data = SEC_PROXY_THREAD(SEC_PROXY_DATA_BASE, rx_thread); + cfg_scfg = SEC_PROXY_THREAD(SEC_PROXY_SCFG_BASE, rx_thread); + cfg_rt = SEC_PROXY_THREAD(SEC_PROXY_RT_BASE, rx_thread); + + thread = &spts[SEC_PROXY_RX_THREAD]; + thread->name = "SEC_PROXY_RESPONSE_THREAD"; + + thread->data = core_mmu_get_va(target_data, MEM_AREA_IO_SEC, + SEC_PROXY_DATA_SIZE); + if (!thread->data) + return TEE_ERROR_OUT_OF_MEMORY; + + thread->scfg = core_mmu_get_va(cfg_scfg, MEM_AREA_IO_SEC, + SEC_PROXY_SCFG_SIZE); + if (!thread->scfg) + return TEE_ERROR_OUT_OF_MEMORY; + + thread->rt = core_mmu_get_va(cfg_rt, MEM_AREA_IO_SEC, + SEC_PROXY_RT_SIZE); + if (!thread->rt) + return TEE_ERROR_OUT_OF_MEMORY; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/arch/arm/plat-k3/drivers/sec_proxy.h b/optee_os/core/arch/arm/plat-k3/drivers/sec_proxy.h new file mode 100644 index 0000000..0fdfb65 --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/sec_proxy.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Texas Instruments K3 Secure Proxy Driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * Manorit Chawdhry + */ + +#ifndef SEC_PROXY_H +#define SEC_PROXY_H + +#include +#include +#include + +#define SEC_PROXY_MAX_MSG_SIZE 60 + +/** + * struct k3_sec_proxy_msg - Secure proxy message structure + * @len: Length of data in the Buffer + * @buf: Buffer pointer + * + * This is the structure for data used in k3_sec_proxy_{send,recv}() + */ +struct k3_sec_proxy_msg { + size_t len; + uint8_t *buf; +}; + +/** + * k3_sec_proxy_send() - Send data over a Secure Proxy thread + * @msg: Pointer to k3_sec_proxy_msg + */ +TEE_Result k3_sec_proxy_send(const struct k3_sec_proxy_msg *msg); + +/** + * k3_sec_proxy_recv() - Receive data from a Secure Proxy thread + * @msg: Pointer to k3_sec_proxy_msg + */ +TEE_Result k3_sec_proxy_recv(struct k3_sec_proxy_msg *msg); + +/** + * k3_sec_proxy_init() - Initialize the secure proxy threads + */ +TEE_Result k3_sec_proxy_init(void); + +#endif /* __SEC_PROXY_H */ diff --git a/optee_os/core/arch/arm/plat-k3/drivers/sub.mk b/optee_os/core/arch/arm/plat-k3/drivers/sub.mk new file mode 100644 index 0000000..0bcd2ba --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/sub.mk @@ -0,0 +1,4 @@ +srcs-y += sec_proxy.c +srcs-y += ti_sci.c +srcs-$(CFG_SA2UL) += sa2ul.c +srcs-$(CFG_SA2UL) += sa2ul_rng.c diff --git a/optee_os/core/arch/arm/plat-k3/drivers/ti_sci.c b/optee_os/core/arch/arm/plat-k3/drivers/ti_sci.c new file mode 100644 index 0000000..9d7c7dc --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/ti_sci.c @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Texas Instruments System Control Interface Driver + * Based on TF-A implementation + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * Manorit Chawdhry + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "sec_proxy.h" +#include "ti_sci.h" +#include "ti_sci_protocol.h" + +static uint8_t message_sequence; + +/** + * struct ti_sci_xfer - Structure representing a message flow + * @tx_message: Transmit message + * @rx_message: Receive message + */ +struct ti_sci_xfer { + struct k3_sec_proxy_msg tx_message; + struct k3_sec_proxy_msg rx_message; +}; + +/** + * ti_sci_setup_xfer() - Setup message transfer + * + * @msg_type: Message type + * @msg_flags: Flag to set for the message + * @tx_buf: Buffer to be sent to mailbox channel + * @tx_message_size: transmit message size + * @rx_buf: Buffer to be received from mailbox channel + * @rx_message_size: receive message size + * @xfer: Transfer message + * + * Helper function which is used by various command functions that are + * exposed to clients of this driver for allocating a message traffic event. + * + * Return: 0 if all goes well, else appropriate error message + */ +static int ti_sci_setup_xfer(uint16_t msg_type, uint32_t msg_flags, + void *tx_buf, + size_t tx_message_size, + void *rx_buf, + size_t rx_message_size, + struct ti_sci_xfer *xfer) +{ + struct ti_sci_msg_hdr *hdr = NULL; + + /* Ensure we have sane transfer sizes */ + if (rx_message_size > SEC_PROXY_MAX_MSG_SIZE || + tx_message_size > SEC_PROXY_MAX_MSG_SIZE || + rx_message_size < sizeof(*hdr) || + tx_message_size < sizeof(*hdr)) { + EMSG("Message transfer size not sane"); + return TEE_ERROR_SHORT_BUFFER; + } + + hdr = (struct ti_sci_msg_hdr *)tx_buf; + hdr->seq = ++message_sequence; + hdr->type = msg_type; + hdr->host = OPTEE_HOST_ID; + hdr->flags = msg_flags | TI_SCI_FLAG_REQ_ACK_ON_PROCESSED; + + xfer->tx_message.buf = tx_buf; + xfer->tx_message.len = tx_message_size; + + xfer->rx_message.buf = rx_buf; + xfer->rx_message.len = rx_message_size; + + return 0; +} + +/** + * ti_sci_get_response() - Receive response from mailbox channel + * + * @xfer: Transfer to initiate and wait for response + * + * Return: 0 if all goes well, else appropriate error message + */ +static inline int ti_sci_get_response(struct ti_sci_xfer *xfer) +{ + struct k3_sec_proxy_msg *msg = &xfer->rx_message; + struct ti_sci_msg_hdr *hdr = NULL; + unsigned int retry = 5; + int ret = 0; + + for (; retry > 0; retry--) { + /* Receive the response */ + ret = k3_sec_proxy_recv(msg); + if (ret) { + EMSG("Message receive failed (%d)", ret); + return ret; + } + + /* msg is updated by Secure Proxy driver */ + hdr = (struct ti_sci_msg_hdr *)msg->buf; + + /* Sanity check for message response */ + if (hdr->seq == message_sequence) + break; + + IMSG("Message with sequence ID %u is not expected", hdr->seq); + } + if (!retry) { + EMSG("Timed out waiting for message"); + return TEE_ERROR_BUSY; + } + + if (!(hdr->flags & TI_SCI_FLAG_RESP_GENERIC_ACK)) { + DMSG("Message not acknowledged"); + return TEE_ERROR_ACCESS_DENIED; + } + + return 0; +} + +/** + * ti_sci_do_xfer() - Do one transfer + * + * @xfer: Transfer to initiate and wait for response + * + * Return: 0 if all goes well, else appropriate error message + */ +static inline int ti_sci_do_xfer(struct ti_sci_xfer *xfer) +{ + struct k3_sec_proxy_msg *msg = &xfer->tx_message; + int ret = 0; + + /* Send the message */ + ret = k3_sec_proxy_send(msg); + if (ret) { + EMSG("Message sending failed (%d)", ret); + return ret; + } + + /* Get the response */ + ret = ti_sci_get_response(xfer); + if (ret) { + if ((TEE_Result)ret != TEE_ERROR_ACCESS_DENIED) + EMSG("Failed to get response (%d)", ret); + return ret; + } + + return 0; +} + +int ti_sci_get_revision(struct ti_sci_msg_resp_version *rev_info) +{ + struct ti_sci_msg_req_version req = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_VERSION, 0x0, + &req, sizeof(req), + rev_info, sizeof(*rev_info), + &xfer); + if (ret) + return ret; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + return 0; +} + +static int ti_sci_device_set_state(uint32_t id, uint32_t flags, uint8_t state) +{ + struct ti_sci_msg_req_set_device_state req = { }; + struct ti_sci_msg_resp_set_device_state resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_SET_DEVICE_STATE, flags, + &req, sizeof(req), + &resp, sizeof(resp), + &xfer); + if (ret) + return ret; + + req.id = id; + req.state = state; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + return 0; +} + +int ti_sci_device_get(uint32_t id) +{ + return ti_sci_device_set_state(id, 0, MSG_DEVICE_SW_STATE_ON); +} + +int ti_sci_device_put(uint32_t id) +{ + return ti_sci_device_set_state(id, 0, MSG_DEVICE_SW_STATE_AUTO_OFF); +} + +int ti_sci_set_fwl_region(uint16_t fwl_id, uint16_t region, + uint32_t n_permission_regs, uint32_t control, + const uint32_t permissions[FWL_MAX_PRIVID_SLOTS], + uint64_t start_address, uint64_t end_address) +{ + struct ti_sci_msg_req_fwl_set_firewall_region req = { }; + struct ti_sci_msg_resp_fwl_set_firewall_region resp = { }; + struct ti_sci_xfer xfer = { }; + unsigned int i = 0; + int ret = 0; + + assert(n_permission_regs <= FWL_MAX_PRIVID_SLOTS); + + ret = ti_sci_setup_xfer(TI_SCI_MSG_FWL_SET, 0, + &req, sizeof(req), + &resp, sizeof(resp), + &xfer); + if (ret) + return ret; + + req.fwl_id = fwl_id; + req.region = region; + req.n_permission_regs = n_permission_regs; + req.control = control; + for (i = 0; i < n_permission_regs; i++) + req.permissions[i] = permissions[i]; + req.start_address = start_address; + req.end_address = end_address; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + return 0; +} + +int ti_sci_get_fwl_region(uint16_t fwl_id, uint16_t region, + uint32_t n_permission_regs, uint32_t *control, + uint32_t permissions[FWL_MAX_PRIVID_SLOTS], + uint64_t *start_address, uint64_t *end_address) +{ + struct ti_sci_msg_req_fwl_get_firewall_region req = { }; + struct ti_sci_msg_resp_fwl_get_firewall_region resp = { }; + struct ti_sci_xfer xfer = { }; + unsigned int i = 0; + int ret = 0; + + assert(n_permission_regs <= FWL_MAX_PRIVID_SLOTS); + + ret = ti_sci_setup_xfer(TI_SCI_MSG_FWL_GET, 0, + &req, sizeof(req), + &resp, sizeof(resp), + &xfer); + if (ret) + return ret; + + req.fwl_id = fwl_id; + req.region = region; + req.n_permission_regs = n_permission_regs; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + *control = resp.control; + for (i = 0; i < n_permission_regs; i++) + permissions[i] = resp.permissions[i]; + *start_address = resp.start_address; + *end_address = resp.end_address; + + return 0; +} + +int ti_sci_change_fwl_owner(uint16_t fwl_id, uint16_t region, + uint8_t owner_index, uint8_t *owner_privid, + uint16_t *owner_permission_bits) +{ + struct ti_sci_msg_req_fwl_change_owner_info req = { }; + struct ti_sci_msg_resp_fwl_change_owner_info resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_FWL_CHANGE_OWNER, 0, + &req, sizeof(req), + &resp, sizeof(resp), + &xfer); + if (ret) + return ret; + + req.fwl_id = fwl_id; + req.region = region; + req.owner_index = owner_index; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + *owner_privid = resp.owner_privid; + *owner_permission_bits = resp.owner_permission_bits; + + return 0; +} + +int ti_sci_get_dkek(uint8_t sa2ul_instance, + const char *context, const char *label, + uint8_t dkek[SA2UL_DKEK_KEY_LEN]) +{ + struct ti_sci_msg_req_sa2ul_get_dkek req = { }; + struct ti_sci_msg_resp_sa2ul_get_dkek resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_SA2UL_GET_DKEK, 0, + &req, sizeof(req), &resp, sizeof(resp), &xfer); + if (ret) + return ret; + + req.sa2ul_instance = sa2ul_instance; + req.kdf_label_len = strlen(label); + req.kdf_context_len = strlen(context); + if (req.kdf_label_len + req.kdf_context_len > + KDF_LABEL_AND_CONTEXT_LEN_MAX) { + EMSG("Context and Label too long"); + return TEE_ERROR_BAD_PARAMETERS; + } + memcpy(req.kdf_label_and_context, label, strlen(label)); + memcpy(req.kdf_label_and_context + strlen(label), context, + strlen(context)); + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + memcpy(dkek, resp.dkek, sizeof(resp.dkek)); + memzero_explicit(&resp, sizeof(resp)); + return 0; +} + +int ti_sci_read_otp_mmr(uint8_t mmr_idx, uint32_t *val) +{ + struct ti_sci_msg_req_read_otp_mmr req = { }; + struct ti_sci_msg_resp_read_otp_mmr resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_READ_OTP_MMR, 0, + &req, sizeof(req), &resp, sizeof(resp), &xfer); + if (ret) + goto exit; + + req.mmr_idx = mmr_idx; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + goto exit; + + *val = resp.mmr_val; + +exit: + memzero_explicit(&resp, sizeof(resp)); + return ret; +} + +int ti_sci_write_otp_row(uint8_t row_idx, uint32_t row_val, uint32_t row_mask) +{ + struct ti_sci_msg_req_write_otp_row req = { }; + struct ti_sci_msg_resp_write_otp_row resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_WRITE_OTP_ROW, 0, + &req, sizeof(req), &resp, sizeof(resp), &xfer); + if (ret) + goto exit; + + req.row_idx = row_idx; + req.row_val = row_val; + req.row_mask = row_mask; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + goto exit; + + DMSG("resp.row_val: 0x%08x", resp.row_val); + + if (resp.row_val != (req.row_val & req.row_mask)) { + EMSG("Value not written correctly"); + DMSG("req.row_val : 0x%08"PRIx32, req.row_val); + DMSG("req.row_mask: 0x%08"PRIx32, req.row_mask); + ret = TEE_ERROR_BAD_STATE; + } + +exit: + memzero_explicit(&resp, sizeof(resp)); + memzero_explicit(&req, sizeof(req)); + return ret; +} + +int ti_sci_lock_otp_row(uint8_t row_idx, uint8_t hw_write_lock, + uint8_t hw_read_lock, uint8_t row_soft_lock) +{ + struct ti_sci_msg_req_lock_otp_row req = { }; + struct ti_sci_msg_resp_lock_otp_row resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_LOCK_OTP_ROW, 0, + &req, sizeof(req), &resp, sizeof(resp), &xfer); + if (ret) + return ret; + + req.row_idx = row_idx; + req.hw_write_lock = hw_write_lock; + req.hw_read_lock = hw_read_lock; + req.row_soft_lock = row_soft_lock; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + return 0; +} + +int ti_sci_get_swrev(uint32_t *swrev) +{ + struct ti_sci_msq_req_get_swrev req = { }; + struct ti_sci_msq_resp_get_swrev resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_READ_SWREV, 0, + &req, sizeof(req), &resp, sizeof(resp), &xfer); + if (ret) + return ret; + + req.identifier = OTP_REV_ID_SEC_BRDCFG; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + *swrev = resp.swrev; + memzero_explicit(&resp, sizeof(resp)); + return 0; +} + +int ti_sci_get_keycnt_keyrev(uint32_t *key_cnt, uint32_t *key_rev) +{ + struct ti_sci_msq_req_get_keycnt_keyrev req = { }; + struct ti_sci_msq_resp_get_keycnt_keyrev resp = { }; + struct ti_sci_xfer xfer = { }; + int ret = 0; + + ret = ti_sci_setup_xfer(TI_SCI_MSG_READ_KEYCNT_KEYREV, 0, + &req, sizeof(req), &resp, sizeof(resp), &xfer); + if (ret) + return ret; + + ret = ti_sci_do_xfer(&xfer); + if (ret) + return ret; + + *key_cnt = resp.keycnt; + *key_rev = resp.keyrev; + memzero_explicit(&resp, sizeof(resp)); + return 0; +} + +int ti_sci_init(void) +{ + struct ti_sci_msg_resp_version rev_info = { }; + int ret = 0; + + ret = ti_sci_get_revision(&rev_info); + if (ret) { + EMSG("Unable to communicate with control firmware (%d)", ret); + return ret; + } + + IMSG("SYSFW ABI: %d.%d (firmware rev 0x%04x '%s')", + rev_info.abi_major, rev_info.abi_minor, + rev_info.firmware_revision, + rev_info.firmware_description); + + return 0; +} diff --git a/optee_os/core/arch/arm/plat-k3/drivers/ti_sci.h b/optee_os/core/arch/arm/plat-k3/drivers/ti_sci.h new file mode 100644 index 0000000..497c4d9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/ti_sci.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * Manorit Chawdhry + */ + +#ifndef TI_SCI_H +#define TI_SCI_H + +#include +#include +#include + +#include "ti_sci_protocol.h" + +/** + * ti_sci_get_revision() - Get the revision of the SCI entity + * + * Updates the SCI information in the internal data structure. + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_get_revision(struct ti_sci_msg_resp_version *rev_info); + +/** + * Device control operations + * + * - ti_sci_device_get - Get access to device managed by TISCI + * - ti_sci_device_put - Release access to device managed by TISCI + * + * NOTE: for all these functions, the following are generic in nature: + * @id: Device Identifier + * + * Returns 0 for successful request, else returns corresponding error message. + * + * Request for the device - NOTE: the client MUST maintain integrity of + * usage count by balancing get_device with put_device. No refcounting is + * managed by driver for that purpose. + */ +int ti_sci_device_get(uint32_t id); +int ti_sci_device_put(uint32_t id); + +/** + * ti_sci_set_fwl_region() - Request for configuring a firewall region + * + * @fwl_id: Firewall ID in question. fwl_id is defined in the TRM. + * @region: Region or channel number to set config info. This field + * is unused in case of a simple firewall and must be + * initialized to zero. In case of a region based + * firewall, this field indicates the region in question + * (index starting from 0). In case of a channel based + * firewall, this field indicates the channel in question + * (index starting from 0). + * @n_permission_regs: Number of permission registers to set + * @control: Contents of the firewall CONTROL register to set + * @permissions: Contents of the firewall PERMISSION register to set + * @start_address: Contents of the firewall START_ADDRESS register to set + * @end_address: Contents of the firewall END_ADDRESS register to set + * + * Return: 0 if all went well, else returns appropriate error value. + */ +int ti_sci_set_fwl_region(uint16_t fwl_id, uint16_t region, + uint32_t n_permission_regs, uint32_t control, + const uint32_t permissions[FWL_MAX_PRIVID_SLOTS], + uint64_t start_address, uint64_t end_address); +/** + * ti_sci_cmd_get_fwl_region() - Request for getting a firewall region + * + * @fwl_id: Firewall ID in question. fwl_id is defined in the TRM. + * @region: Region or channel number to set config info. This field + * is unused in case of a simple firewall and must be + * initialized to zero. In case of a region based + * firewall, this field indicates the region in question + * (index starting from 0). In case of a channel based + * firewall, this field indicates the channel in question + * (index starting from 0). + * @n_permission_regs: Region or channel number to set config info + * @control: Contents of the firewall CONTROL register + * @permissions: Contents of the firewall PERMISSION register + * @start_address: Contents of the firewall START_ADDRESS register + * @end_address: Contents of the firewall END_ADDRESS register + * + * Return: 0 if all went well, else returns appropriate error value. + */ +int ti_sci_get_fwl_region(uint16_t fwl_id, uint16_t region, + uint32_t n_permission_regs, uint32_t *control, + uint32_t permissions[FWL_MAX_PRIVID_SLOTS], + uint64_t *start_address, uint64_t *end_address); +/** + * ti_sci_change_fwl_owner() - Request for changing a firewall owner + * + * @fwl_id: Firewall ID in question. fwl_id is defined in the TRM. + * @region: Region or channel number to set config info. This field + * is unused in case of a simple firewall and must be + * initialized to zero. In case of a region based + * firewall, this field indicates the region in question + * (index starting from 0). In case of a channel based + * firewall, this field indicates the channel in question + * (index starting from 0). + * @owner_index: New owner index to transfer ownership to + * @owner_privid: New owner priv-ID returned by DMSC. This field is + * currently initialized to zero by DMSC. + * @owner_permission_bits: New owner permission bits returned by DMSC. This + * field is currently initialized to zero by DMSC. + * + * Return: 0 if all went well, else returns appropriate error value. + */ +int ti_sci_change_fwl_owner(uint16_t fwl_id, uint16_t region, + uint8_t owner_index, uint8_t *owner_privid, + uint16_t *owner_permission_bits); + +/** + * ti_sci_get_dkek() - Get the DKEK + * @sa2ul_instance: SA2UL instance to get key + * @context: Context string input to KDF + * @label: Label string input to KDF + * @dkek: Returns with DKEK populated + * + * Updates the DKEK the internal data structure. + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_get_dkek(uint8_t sa2ul_instance, + const char *context, const char *label, + uint8_t dkek[SA2UL_DKEK_KEY_LEN]); + +/** + * ti_sci_read_otp_mmr() - Get the Extended OTP + * @mmr_idx: 32-bit MMR index + * @val: Value of the 32-bit MMR + * + * Reads the extended OTP bits from efuse + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_read_otp_mmr(uint8_t mmr_idx, uint32_t *val); + +/** + * ti_sci_write_otp_row() - Write the extended OTP row + * @row_idx: Index of the OTP row. Zero indexing + * @row_val: Value to be written + * @row_mask: Mask bits for row_val to be written + * + * Writes a Row in the extended OTP field + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_write_otp_row(uint8_t row_idx, uint32_t row_val, uint32_t row_mask); + +/** + * ti_sci_lock_otp_row - Locking the Extended OTP row + * @row_idx: Index of the OTP row. Zero indexing + * @hw_write_lock: Hardware write lock + * @hw_read_lock: Hardware read lock + * @row_soft_lock: Software write lock + * + * Lockes a Row in the extended OTP field to prevent read/writes + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_lock_otp_row(uint8_t row_idx, uint8_t hw_write_lock, + uint8_t hw_read_lock, uint8_t row_soft_lock); + +/** + * ti_sci_get_swrev - Read Software Revision + * @swrev: Software Revision + * + * Reads the software revision. The System Firmware currently supports reading + * only the software revision from the Secure Board Configuration. + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_get_swrev(uint32_t *swrev); + +/** + * ti_sci_get_keycnt_keyrev - Read Key Count and Key Revision values + * @key_cnt: Key Count + * @key_rev: Key Revision + * + * Reads the Key Count and Key Revision in OTP + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_get_keycnt_keyrev(uint32_t *key_cnt, uint32_t *key_rev); + +/** + * ti_sci_init() - Basic initialization + * + * Return: 0 if all goes well, else appropriate error message + */ +int ti_sci_init(void); + +#endif diff --git a/optee_os/core/arch/arm/plat-k3/drivers/ti_sci_protocol.h b/optee_os/core/arch/arm/plat-k3/drivers/ti_sci_protocol.h new file mode 100644 index 0000000..12e3098 --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/drivers/ti_sci_protocol.h @@ -0,0 +1,452 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2016-2022 Texas Instruments Incorporated - https://www.ti.com/ + * Lokesh Vutla + * Manorit Chawdhry + */ + +#ifndef TI_SCI_PROTOCOL_H +#define TI_SCI_PROTOCOL_H + +#include +#include +#include + +/* Generic Messages */ +#define TI_SCI_MSG_VERSION 0x0002 + +/* Device requests */ +#define TI_SCI_MSG_SET_DEVICE_STATE 0x0200 + +/* Security Management Messages */ +#define TI_SCI_MSG_FWL_SET 0x9000 +#define TI_SCI_MSG_FWL_GET 0x9001 +#define TI_SCI_MSG_FWL_CHANGE_OWNER 0x9002 +#define TI_SCI_MSG_SA2UL_GET_DKEK 0x9029 +#define TI_SCI_MSG_READ_OTP_MMR 0x9022 +#define TI_SCI_MSG_WRITE_OTP_ROW 0x9023 +#define TI_SCI_MSG_LOCK_OTP_ROW 0x9024 + +/* OTP Revision Read/Write Message Description */ +#define TI_SCI_MSG_WRITE_SWREV 0x9032 +#define TI_SCI_MSG_READ_SWREV 0x9033 +#define TI_SCI_MSG_READ_KEYCNT_KEYREV 0x9034 +#define TI_SCI_MSG_WRITE_KEYREV 0x9035 + +/** + * struct ti_sci_secure_msg_hdr - Secure Message Header for All messages + * and responses + * + * @checksum: Integrity check for HS devices + * @reserved: Reserved for future uses + */ +struct ti_sci_secure_msg_hdr { + uint16_t checksum; + uint16_t reserved; +} __packed; + +/** + * struct ti_sci_msg_hdr - Generic Message Header for All messages and responses + * @type: Type of messages: One of TI_SCI_MSG* values + * @host: Host of the message + * @seq: Message identifier indicating a transfer sequence + * @flags: Flag for the message + */ +struct ti_sci_msg_hdr { + struct ti_sci_secure_msg_hdr sec_hdr; + uint16_t type; + uint8_t host; + uint8_t seq; +#define TI_SCI_MSG_FLAG(val) BIT(val) +#define TI_SCI_FLAG_REQ_GENERIC_NORESPONSE 0x0 +#define TI_SCI_FLAG_REQ_ACK_ON_RECEIVED TI_SCI_MSG_FLAG(0) +#define TI_SCI_FLAG_REQ_ACK_ON_PROCESSED TI_SCI_MSG_FLAG(1) +#define TI_SCI_FLAG_RESP_GENERIC_NACK 0x0 +#define TI_SCI_FLAG_RESP_GENERIC_ACK TI_SCI_MSG_FLAG(1) + /* Additional Flags */ + uint32_t flags; +} __packed; + +/** + * struct ti_sci_msg_version_req - Request for firmware version information + * @hdr: Generic header + * + * Request for TI_SCI_MSG_VERSION + */ +struct ti_sci_msg_req_version { + struct ti_sci_msg_hdr hdr; +} __packed; + +/** + * struct ti_sci_msg_resp_version - Response for firmware version information + * @hdr: Generic header + * @firmware_description: String describing the firmware + * @firmware_revision: Firmware revision + * @abi_major: Major version of the ABI that firmware supports + * @abi_minor: Minor version of the ABI that firmware supports + * @sub_version: Sub-version number of the firmware + * @patch_version: Patch-version number of the firmware. + * + * In general, ABI version changes follow the rule that minor version increments + * are backward compatible. Major revision changes in ABI may not be + * backward compatible. + * + * Response to request TI_SCI_MSG_VERSION + */ +struct ti_sci_msg_resp_version { + struct ti_sci_msg_hdr hdr; +#define FIRMWARE_DESCRIPTION_LENGTH 32 + char firmware_description[FIRMWARE_DESCRIPTION_LENGTH]; + uint16_t firmware_revision; + uint8_t abi_major; + uint8_t abi_minor; + uint8_t sub_version; + uint8_t patch_version; +} __packed; + +/** + * struct ti_sci_msg_req_set_device_state - Set the desired state of the device + * @hdr: Generic header + * @id: Indicates which device to modify + * @reserved: Reserved space in message, must be 0 for backward compatibility + * @state: The desired state of the device. + * + * Certain flags can also be set to alter the device state: + * + MSG_FLAG_DEVICE_WAKE_ENABLED - Configure the device to be a wake source. + * The meaning of this flag will vary slightly from device to device and from + * SoC to SoC but it generally allows the device to wake the SoC out of deep + * suspend states. + * + MSG_FLAG_DEVICE_RESET_ISO - Enable reset isolation for this device. + * + MSG_FLAG_DEVICE_EXCLUSIVE - Claim this device exclusively. When passed + * with STATE_RETENTION or STATE_ON, it will claim the device exclusively. + * If another host already has this device set to STATE_RETENTION or STATE_ON, + * the message will fail. Once successful, other hosts attempting to set + * STATE_RETENTION or STATE_ON will fail. + * + * Request type is TI_SCI_MSG_SET_DEVICE_STATE, responded with a generic + * ACK/NACK message. + */ +struct ti_sci_msg_req_set_device_state { + /* Additional hdr->flags options */ +#define MSG_FLAG_DEVICE_WAKE_ENABLED TI_SCI_MSG_FLAG(8) +#define MSG_FLAG_DEVICE_RESET_ISO TI_SCI_MSG_FLAG(9) +#define MSG_FLAG_DEVICE_EXCLUSIVE TI_SCI_MSG_FLAG(10) + struct ti_sci_msg_hdr hdr; + uint32_t id; + uint32_t reserved; + +#define MSG_DEVICE_SW_STATE_AUTO_OFF 0 +#define MSG_DEVICE_SW_STATE_RETENTION 1 +#define MSG_DEVICE_SW_STATE_ON 2 + uint8_t state; +} __packed; + +/** + * struct ti_sci_msg_resp_set_device_state - Response for set device state + * @hdr: Generic header + * + * Response to request TI_SCI_MSG_SET_DEVICE_STATE + */ +struct ti_sci_msg_resp_set_device_state { + struct ti_sci_msg_hdr hdr; +} __packed; + +#define FWL_MAX_PRIVID_SLOTS 3U + +/** + * struct ti_sci_msg_req_fwl_set_firewall_region - Set firewall permissions + * @hdr: Generic Header + * @fwl_id: Firewall ID + * @region: Region or channel number to set config info. + * This field is unused in case of a simple firewall and + * must be initialized to zero. In case of a region based + * firewall, this field indicates the region (index + * starting from 0). In case of a channel based firewall, + * this field indicates the channel (index starting + * from 0). + * @n_permission_regs: Number of permission registers to set + * @control: Contents of the firewall CONTROL register to set + * @permissions: Contents of the firewall PERMISSION register to set + * @start_address: Contents of the firewall START_ADDRESS register to set + * @end_address: Contents of the firewall END_ADDRESS register to set + */ +struct ti_sci_msg_req_fwl_set_firewall_region { + struct ti_sci_msg_hdr hdr; + uint16_t fwl_id; + uint16_t region; + uint32_t n_permission_regs; + uint32_t control; + uint32_t permissions[FWL_MAX_PRIVID_SLOTS]; + uint64_t start_address; + uint64_t end_address; +} __packed; + +struct ti_sci_msg_resp_fwl_set_firewall_region { + struct ti_sci_msg_hdr hdr; +} __packed; + +/** + * struct ti_sci_msg_req_fwl_get_firewall_region - Retrieve firewall permissions + * @hdr: Generic Header + * @fwl_id: Firewall ID in question + * @region: Region or channel number to set config info. + * This field is unused in case of a simple firewall and + * must be initialized to zero. In case of a region based + * firewall, this field indicates the region (index + * starting from 0). In case of a channel based firewall, + * this field indicates the channel (index starting + * from 0). + * @n_permission_regs: Number of permission registers to retrieve + */ +struct ti_sci_msg_req_fwl_get_firewall_region { + struct ti_sci_msg_hdr hdr; + uint16_t fwl_id; + uint16_t region; + uint32_t n_permission_regs; +} __packed; + +/** + * struct ti_sci_msg_resp_fwl_get_firewall_region - Response for retrieving the + * firewall permissions + * + * @hdr: Generic Header + * + * @fwl_id: Firewall ID in question + * @region: Region or channel number to set config info. + * This field is unused in case of a simple firewall and + * must be initialized to zero. In case of a region based + * firewall, this field indicates the region (index + * starting from 0). In case of a channel based firewall, + * this field indicates the channel (index starting + * from 0). + * @n_permission_regs: Number of permission registers retrieved + * @control: Contents of the firewall CONTROL register + * @permissions: Contents of the firewall PERMISSION registers + * @start_address: Contents of the firewall START_ADDRESS register + * @end_address: Contents of the firewall END_ADDRESS register + */ +struct ti_sci_msg_resp_fwl_get_firewall_region { + struct ti_sci_msg_hdr hdr; + uint16_t fwl_id; + uint16_t region; + uint32_t n_permission_regs; + uint32_t control; + uint32_t permissions[FWL_MAX_PRIVID_SLOTS]; + uint64_t start_address; + uint64_t end_address; +} __packed; + +/** + * struct ti_sci_msg_req_fwl_change_owner_info - Request change firewall owner + * + * @hdr: Generic Header + * + * @fwl_id: Firewall ID in question + * @region: Region or channel number if applicable + * @owner_index: New owner index to transfer ownership to + */ +struct ti_sci_msg_req_fwl_change_owner_info { + struct ti_sci_msg_hdr hdr; + uint16_t fwl_id; + uint16_t region; + uint8_t owner_index; +} __packed; + +/** + * struct ti_sci_msg_resp_fwl_change_owner_info - Response for change + * firewall owner + * + * @hdr: Generic Header + * + * @fwl_id: Firewall ID specified in request + * @region: Region or channel number specified in request + * @owner_index: Owner index specified in request + * @owner_privid: New owner priv-ID returned by DMSC. + * @owner_permission_bits: New owner permission bits returned by DMSC. + */ +struct ti_sci_msg_resp_fwl_change_owner_info { + struct ti_sci_msg_hdr hdr; + uint16_t fwl_id; + uint16_t region; + uint8_t owner_index; + uint8_t owner_privid; + uint16_t owner_permission_bits; +} __packed; + +/** + * struct ti_sci_msg_sa2ul_get_dkek_req - Request for DKEK value + * @hdr: Generic header + * @sa2ul_instance: SA2UL instance number - set to 0 + * @kdf_label_len: Length of "Label" input to KDF + * @kdf_context_len: Length of "Context" input to KDF + * @kdf_label_and_context: "Label" and "Context" bytes + * + * Request for TI_SCI_MSG_SA2UL_GET_DKEK + */ +struct ti_sci_msg_req_sa2ul_get_dkek { + struct ti_sci_msg_hdr hdr; + uint8_t sa2ul_instance; + uint8_t kdf_label_len; + uint8_t kdf_context_len; +#define KDF_LABEL_AND_CONTEXT_LEN_MAX 41 + uint8_t kdf_label_and_context[KDF_LABEL_AND_CONTEXT_LEN_MAX]; +} __packed; + +/** + * struct ti_sci_msg_sa2ul_get_dkek_req - Response for DKEK value + * @hdr: Generic header + * @dkek: Array containing Derived KEK + * + * Response to request TI_SCI_MSG_SA2UL_GET_DKEK + */ +struct ti_sci_msg_resp_sa2ul_get_dkek { + struct ti_sci_msg_hdr hdr; +#define SA2UL_DKEK_KEY_LEN 32 + uint8_t dkek[SA2UL_DKEK_KEY_LEN]; +} __packed; + +/** + * struct ti_sci_msg_resp_read_otp_mmr - Request for reading extended OTP + * @hdr: Generic header + * @mmr_idx: Index of 32 bit MMR + * + * Request for TI_SCI_MSG_READ_OTP_MMR + */ +struct ti_sci_msg_req_read_otp_mmr { + struct ti_sci_msg_hdr hdr; + uint8_t mmr_idx; +} __packed; + +/** + * struct ti_sci_msg_resp_read_otp_mmr - Response for reading extended OTP + * @hdr: Generic header + * @mmr_val: Value written in the OTP + * + * Response to request TI_SCI_MSG_READ_OTP_MMR + */ +struct ti_sci_msg_resp_read_otp_mmr { + struct ti_sci_msg_hdr hdr; + uint32_t mmr_val; +} __packed; + +/** + * struct ti_sci_msg_req_write_otp_row - Request for writing Extended OTP + * @hdr: Generic header + * @row_idx: Index of the OTP row. Zero indexing + * @row_val: Value to be written + * @row_mask: Mask bits for row_val to be written + * + * Request for TI_SCI_MSG_WRITE_OTP_ROW + */ +struct ti_sci_msg_req_write_otp_row { + struct ti_sci_msg_hdr hdr; + uint8_t row_idx; + uint32_t row_val; + uint32_t row_mask; +} __packed; + +/** + * struct ti_sci_msg_resp_write_otp_row - Response for writing Extended OTP + * @hdr: Generic header + * @row_val: Value that is written + * + * Response to request TI_SCI_MSG_WRITE_OTP_ROW + */ +struct ti_sci_msg_resp_write_otp_row { + struct ti_sci_msg_hdr hdr; + uint32_t row_val; +} __packed; + +/** + * struct ti_sci_msg_req_lock_otp_row - Request for Lock OTP row + * @hdr: Generic header + * @row_idx: Index of the OTP row. Zero indexing + * @hw_write_lock: 0x5A indicates row will be write protected + * @hw_read_lock: 0x5A indicates row will be read protected + * @row_soft_lock: Software write lock + * + * Request for TI_SCI_MSG_LOCK_OTP_ROW + */ +struct ti_sci_msg_req_lock_otp_row { + struct ti_sci_msg_hdr hdr; + uint8_t row_idx; + uint8_t hw_write_lock; + uint8_t hw_read_lock; + uint8_t row_soft_lock; +} __packed; + +/** + * struct ti_sci_msg_resp_lock_otp_row - Response for Lock OTP row + * @hdr: Generic header + * + * Response to request TI_SCI_MSG_LOCK_OTP_ROW + */ +struct ti_sci_msg_resp_lock_otp_row { + struct ti_sci_msg_hdr hdr; +} __packed; + +/** + * \brief OTP Revision Identifiers + */ +enum tisci_otp_revision_identifier { + /** Software Revision SBL */ + OTP_REV_ID_SBL = 0, + /** Software Revision SYSFW */ + OTP_REV_ID_SYSFW = 1, + /** Software Revision Secure Board Configuration */ + OTP_REV_ID_SEC_BRDCFG = 2, +}; + +/** + * struct ti_sci_msq_req_get_swrev - Request for reading the Software Revision + * in OTP + * @hdr: Generic header + * @identifier: One of the entries from enum tisci_otp_revision_identifier + * (Current support only for OTP_REV_ID_SEC_BRDCFG) + * + * Request for TI_SCI_MSG_READ_SWREV + */ +struct ti_sci_msq_req_get_swrev { + struct ti_sci_msg_hdr hdr; + uint8_t identifier; +} __packed; + +/** + * struct ti_sci_msq_req_get_swrev - Response for reading the Software Revision + * in OTP + * @hdr: Generic header + * @swrev: Decoded Sofrware Revision value from efuses + * + * Response for TI_SCI_MSG_READ_SWREV + */ +struct ti_sci_msq_resp_get_swrev { + struct ti_sci_msg_hdr hdr; + uint32_t swrev; +} __packed; + +/** + * struct ti_sci_msq_req_get_keycnt_keyrev - Request for reading the Key Count + * and Key Revision in OTP + * @hdr: Generic header + * + * Request for TI_SCI_MSG_READ_KEYCNT_KEYREV + */ +struct ti_sci_msq_req_get_keycnt_keyrev { + struct ti_sci_msg_hdr hdr; +} __packed; + +/** + * struct ti_sci_msq_req_get_swrev - Response for reading the Key Count and Key + * Revision in OTP + * @hdr: Generic header + * @keycnt: Key Count integer value + * @keyrev: Key Revision integer value + * + * Response for TI_SCI_MSG_READ_SWREV + */ +struct ti_sci_msq_resp_get_keycnt_keyrev { + struct ti_sci_msg_hdr hdr; + uint32_t keycnt; + uint32_t keyrev; +} __packed; +#endif diff --git a/optee_os/core/arch/arm/plat-k3/main.c b/optee_os/core/arch/arm/plat-k3/main.c new file mode 100644 index 0000000..531da53 --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/main.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct serial8250_uart_data console_data; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GICC_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GICD_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, + SERIAL8250_UART_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SEC_PROXY_DATA_BASE, + SEC_PROXY_DATA_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SEC_PROXY_SCFG_BASE, + SEC_PROXY_SCFG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SEC_PROXY_RT_BASE, SEC_PROXY_RT_SIZE); +register_ddr(DRAM0_BASE, DRAM0_SIZE); +register_ddr(DRAM1_BASE, DRAM1_SIZE); + +void boot_primary_init_intc(void) +{ + gic_init(GICC_BASE, GICD_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +void console_init(void) +{ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +static TEE_Result init_ti_sci(void) +{ + TEE_Result ret = TEE_SUCCESS; + + ret = k3_sec_proxy_init(); + if (ret != TEE_SUCCESS) + return ret; + + ret = ti_sci_init(); + if (ret) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +service_init(init_ti_sci); + +static TEE_Result secure_boot_information(void) +{ + uint32_t keycnt = 0; + uint32_t keyrev = 0; + uint32_t swrev = 0; + + if (!ti_sci_get_swrev(&swrev)) + IMSG("Secure Board Configuration Software: Rev %"PRIu32, + swrev); + + if (!ti_sci_get_keycnt_keyrev(&keycnt, &keyrev)) + IMSG("Secure Boot Keys: Count %"PRIu32 ", Rev %"PRIu32, + keycnt, keyrev); + + return TEE_SUCCESS; +} + +service_init_late(secure_boot_information); + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + uint8_t dkek[SA2UL_DKEK_KEY_LEN] = { }; + int ret = 0; + + assert(SA2UL_DKEK_KEY_LEN >= HW_UNIQUE_KEY_LENGTH); + + ret = ti_sci_get_dkek(0, "OP-TEE", "DKEK", dkek); + if (ret) { + EMSG("Could not get HUK"); + return TEE_ERROR_SECURITY; + } + + memcpy(&hwkey->data[0], dkek, sizeof(hwkey->data)); + memzero_explicit(&dkek, sizeof(dkek)); + + IMSG("HUK Initialized"); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/arch/arm/plat-k3/platform_config.h b/optee_os/core/arch/arm/plat-k3/platform_config.h new file mode 100644 index 0000000..cffca0a --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/platform_config.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +#define UART0_BASE 0x02800000 + +#define CONSOLE_UART_BASE (UART0_BASE + CFG_CONSOLE_UART * 0x10000) +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 48000000 + +#define DRAM0_BASE 0x80000000 +#define DRAM0_SIZE 0x80000000 + +#define DRAM1_BASE 0x880000000 +#define DRAM1_SIZE 0x780000000 + +#define SCU_BASE 0x01800000 +#if defined(PLATFORM_FLAVOR_j721e) || defined(PLATFORM_FLAVOR_j784s4) +#define GICC_OFFSET 0x100000 +#define GICC_SIZE 0x100000 +#define GICD_OFFSET 0x0 +#define GICD_SIZE 0x10000 +#else +#define GICC_OFFSET 0x80000 +#define GICC_SIZE 0x90000 +#define GICD_OFFSET 0x0 +#define GICD_SIZE 0x10000 +#endif +#if defined(PLATFORM_FLAVOR_am65x) || defined(PLATFORM_FLAVOR_j721e) || \ + defined(PLATFORM_FLAVOR_j784s4) +#define SEC_PROXY_DATA_BASE 0x32c00000 +#define SEC_PROXY_DATA_SIZE 0x100000 +#define SEC_PROXY_SCFG_BASE 0x32800000 +#define SEC_PROXY_SCFG_SIZE 0x100000 +#define SEC_PROXY_RT_BASE 0x32400000 +#define SEC_PROXY_RT_SIZE 0x100000 +#define SEC_PROXY_RESPONSE_THREAD 6 +#define SEC_PROXY_REQUEST_THREAD 7 +#else +#define SEC_PROXY_DATA_BASE 0x4d000000 +#define SEC_PROXY_DATA_SIZE 0x80000 +#define SEC_PROXY_SCFG_BASE 0x4a400000 +#define SEC_PROXY_SCFG_SIZE 0x80000 +#define SEC_PROXY_RT_BASE 0x4a600000 +#define SEC_PROXY_RT_SIZE 0x80000 +#define SEC_PROXY_RESPONSE_THREAD 10 +#define SEC_PROXY_REQUEST_THREAD 11 +#endif +#define OPTEE_HOST_ID 11 +#define SEC_PROXY_TIMEOUT_US 1000000 +#define GICC_BASE (SCU_BASE + GICC_OFFSET) +#define GICD_BASE (SCU_BASE + GICD_OFFSET) + +/* SA2UL */ +#if defined(PLATFORM_FLAVOR_am65x) +#define SA2UL_BASE 0x04e00000 +#define SA2UL_TI_SCI_DEV_ID 136 +#define SA2UL_TI_SCI_FW_ID 2112 +#define SA2UL_TI_SCI_FW_RGN_ID 0 +#elif defined(PLATFORM_FLAVOR_j721e) +#define SA2UL_BASE 0x40900000 +#define SA2UL_TI_SCI_DEV_ID 265 +#define SA2UL_TI_SCI_FW_ID 1196 +#define SA2UL_TI_SCI_FW_RGN_ID 0 +#elif defined(PLATFORM_FLAVOR_j784s4) +#define SA2UL_BASE 0x40900000 +#define SA2UL_TI_SCI_DEV_ID -1 +#define SA2UL_TI_SCI_FW_ID 1196 +#define SA2UL_TI_SCI_FW_RGN_ID 0 +#elif defined(PLATFORM_FLAVOR_am64x) +#define SA2UL_BASE 0x40900000 +#define SA2UL_TI_SCI_DEV_ID 133 +#define SA2UL_TI_SCI_FW_ID 35 +#define SA2UL_TI_SCI_FW_RGN_ID 0 +#elif defined(PLATFORM_FLAVOR_am62x) +#define SA2UL_BASE 0x40900000 +#define SA2UL_TI_SCI_DEV_ID -1 +#define SA2UL_TI_SCI_FW_ID 66 +#define SA2UL_TI_SCI_FW_RGN_ID 1 +#endif +#define SA2UL_REG_SIZE 0x1000 + +/* RNG */ +#define RNG_BASE (SA2UL_BASE + 0x10000) +#define RNG_REG_SIZE 0x1000 +#if defined(PLATFORM_FLAVOR_am62x) +#define RNG_TI_SCI_FW_RGN_ID 2 +#else +#define RNG_TI_SCI_FW_RGN_ID 3 +#endif + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-k3/sub.mk b/optee_os/core/arch/arm/plat-k3/sub.mk new file mode 100644 index 0000000..9dc7571 --- /dev/null +++ b/optee_os/core/arch/arm/plat-k3/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +srcs-y += main.c +subdirs-y += drivers diff --git a/optee_os/core/arch/arm/plat-ls/conf.mk b/optee_os/core/arch/arm/plat-ls/conf.mk new file mode 100644 index 0000000..8402e62 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ls/conf.mk @@ -0,0 +1,131 @@ +PLATFORM_FLAVOR ?= ls1012ardb + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_GIC,y) +$(call force,CFG_16550_UART,y) +$(call force,CFG_LS,y) + +$(call force,CFG_DRAM0_BASE,0x80000000) +$(call force,CFG_TEE_OS_DRAM0_SIZE,0x4000000) + +CFG_ENABLE_EMBEDDED_TESTS ?= y +CFG_PKCS11_TA ?= y + +CFG_CORE_HEAP_SIZE ?= 131072 + +ifeq ($(PLATFORM_FLAVOR),ls1012ardb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_DRAM0_SIZE,0x40000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +CFG_NUM_THREADS ?= 2 +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),ls1043ardb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_DRAM0_SIZE,0x80000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),ls1046ardb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_DRAM0_SIZE,0x80000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),ls1088ardb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_DRAM0_SIZE,0x80000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +$(call force,CFG_ARM_GICV3,y) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),ls2088ardb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_DRAM0_SIZE,0x80000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +$(call force,CFG_ARM_GICV3,y) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),lx2160aqds) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,16) +$(call force,CFG_DRAM0_SIZE,0x80000000) +$(call force,CFG_DRAM1_BASE,0x2080000000) +$(call force,CFG_DRAM1_SIZE,0x1F80000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_PL011,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_EMBED_DTB,y) +$(call force,CFG_EMBED_DTB_SOURCE_FILE,fsl-lx2160a-qds.dts) +CFG_LS_I2C ?= y +CFG_LS_GPIO ?= y +CFG_LS_DSPI ?= y +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),lx2160ardb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,16) +$(call force,CFG_DRAM0_SIZE,0x80000000) +$(call force,CFG_DRAM1_BASE,0x2080000000) +$(call force,CFG_DRAM1_SIZE,0x1F80000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_PL011,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_EMBED_DTB,y) +$(call force,CFG_EMBED_DTB_SOURCE_FILE,fsl-lx2160a-rdb.dts) +CFG_LS_I2C ?= y +CFG_LS_GPIO ?= y +CFG_LS_DSPI ?= y +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),ls1028ardb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_DRAM0_SIZE,0x80000000) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +$(call force,CFG_ARM_GICV3,y) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(platform-flavor-armv8),1) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +CFG_TZDRAM_START ?= ((CFG_DRAM0_BASE + CFG_DRAM0_SIZE) - CFG_TEE_OS_DRAM0_SIZE) +CFG_TZDRAM_SIZE ?= ( CFG_TEE_OS_DRAM0_SIZE - CFG_SHMEM_SIZE) +#CFG_SHMEM_START (Non-Secure shared memory) needs to be 2MB aligned boundary for TZASC 380 configuration. +CFG_SHMEM_START ?= ((CFG_DRAM0_BASE + CFG_DRAM0_SIZE) - CFG_SHMEM_SIZE) +$(call force,CFG_ARM64_core,y) +CFG_USER_TA_TARGETS ?= ta_arm64 +else +#In ARMv7 platform CFG_SHMEM_SIZE is different to that of ARMv8 platforms. +CFG_TZDRAM_START ?= ((CFG_DRAM0_BASE + CFG_DRAM0_SIZE) - CFG_TEE_OS_DRAM0_SIZE) +CFG_TZDRAM_SIZE ?= ( CFG_TEE_OS_DRAM0_SIZE - (2*CFG_SHMEM_SIZE)) +#CFG_SHMEM_START (Non-Secure shared memory) needs to be 2MB aligned boundary for TZASC 380 configuration. +CFG_SHMEM_START ?= ((CFG_DRAM0_BASE + CFG_DRAM0_SIZE) - (2*CFG_SHMEM_SIZE)) +endif + +#Keeping Number of TEE thread equal to number of cores on the SoC +CFG_NUM_THREADS ?= $(CFG_TEE_CORE_NB_CORE) + +ifneq ($(CFG_ARM64_core),y) +$(call force,CFG_SECONDARY_INIT_CNTFRQ,y) +endif + +CFG_CRYPTO_SIZE_OPTIMIZATION ?= n + +# NXP CAAM support is not enabled by default and can be enabled +# on the command line +CFG_NXP_CAAM ?= n diff --git a/optee_os/core/arch/arm/plat-ls/main.c b/optee_os/core/arch/arm/plat-ls/main.c new file mode 100644 index 0000000..3aa97ba --- /dev/null +++ b/optee_os/core/arch/arm/plat-ls/main.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018 NXP + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#ifdef CFG_PL011 +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_PL011 +static struct pl011_data console_data; +#else +static struct ns16550_data console_data; +#endif + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, + CORE_MMU_PGDIR_SIZE); +#if !defined(PLATFORM_FLAVOR_lx2160aqds) && !defined(PLATFORM_FLAVOR_lx2160ardb) +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE, CORE_MMU_PGDIR_SIZE); +#endif + +#if defined(PLATFORM_FLAVOR_lx2160ardb) || defined(PLATFORM_FLAVOR_lx2160aqds) +register_ddr(CFG_DRAM0_BASE, (CFG_TZDRAM_START - CFG_DRAM0_BASE)); +#ifdef CFG_DRAM1_BASE +register_ddr(CFG_DRAM1_BASE, CFG_DRAM1_SIZE); +#endif +#endif +#ifdef DCFG_BASE +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, DCFG_BASE, CORE_MMU_PGDIR_SIZE); +#endif + +#ifdef CFG_ARM32_core +void plat_primary_init_early(void) +{ + vaddr_t addr; + +#if defined(CFG_BOOT_SECONDARY_REQUEST) + /* set secondary entry address */ + io_write32(DCFG_BASE + DCFG_SCRATCHRW1, + __compiler_bswap32(TEE_LOAD_ADDR)); + + /* release secondary cores */ + io_write32(DCFG_BASE + DCFG_CCSR_BRR /* cpu1 */, + __compiler_bswap32(0x1 << 1)); + dsb(); + sev(); +#endif + + /* configure CSU */ + + /* first grant all peripherals */ + for (addr = CSU_BASE + CSU_CSL_START; + addr != CSU_BASE + CSU_CSL_END; + addr += 4) + io_write32(addr, __compiler_bswap32(CSU_ACCESS_ALL)); + + /* restrict key preipherals from NS */ + io_write32(CSU_BASE + CSU_CSL30, + __compiler_bswap32(CSU_ACCESS_SEC_ONLY)); + io_write32(CSU_BASE + CSU_CSL37, + __compiler_bswap32(CSU_ACCESS_SEC_ONLY)); + + /* lock the settings */ + for (addr = CSU_BASE + CSU_CSL_START; + addr != CSU_BASE + CSU_CSL_END; + addr += 4) + io_setbits32(addr, + __compiler_bswap32(CSU_SETTING_LOCK)); +} +#endif + +void console_init(void) +{ +#ifdef CFG_PL011 + /* + * Everything for uart driver initialization is done in bootloader. + * So not reinitializing console. + */ + pl011_init(&console_data, CONSOLE_UART_BASE, 0, 0); +#else + ns16550_init(&console_data, CONSOLE_UART_BASE, IO_WIDTH_U8, 0); +#endif + register_serial_console(&console_data.chip); +} + +#if defined(PLATFORM_FLAVOR_lx2160aqds) || defined(PLATFORM_FLAVOR_lx2160ardb) +static TEE_Result get_gic_base_addr_from_dt(paddr_t *gic_addr) +{ + paddr_t paddr = 0; + size_t size = 0; + + void *fdt = get_embedded_dt(); + int gic_offset = 0; + + gic_offset = fdt_path_offset(fdt, "/soc/interrupt-controller@6000000"); + + if (gic_offset < 0) + gic_offset = fdt_path_offset(fdt, + "/interrupt-controller@6000000"); + + if (gic_offset > 0) { + paddr = fdt_reg_base_address(fdt, gic_offset); + if (paddr == DT_INFO_INVALID_REG) { + EMSG("GIC: Unable to get base addr from DT"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + size = fdt_reg_size(fdt, gic_offset); + if (size == DT_INFO_INVALID_REG_SIZE) { + EMSG("GIC: Unable to get size of base addr from DT"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + } else { + EMSG("Unable to get gic offset node"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + /* make entry in page table */ + if (!core_mmu_add_mapping(MEM_AREA_IO_SEC, paddr, size)) { + EMSG("GIC controller base MMU PA mapping failure"); + return TEE_ERROR_GENERIC; + } + + *gic_addr = paddr; + return TEE_SUCCESS; +} +#endif + +#define SVR_MINOR_MASK 0xF + +static void get_gic_offset(uint32_t *offsetc, uint32_t *offsetd) +{ +#ifdef PLATFORM_FLAVOR_ls1043ardb + vaddr_t addr = 0; + uint32_t rev = 0; + + addr = (vaddr_t)phys_to_virt(DCFG_BASE + DCFG_SVR_OFFSET, + MEM_AREA_IO_NSEC, 1); + if (!addr) { + EMSG("Failed to get virtual address for SVR register"); + panic(); + } + + rev = get_be32((void *)addr); + + if ((rev & SVR_MINOR_MASK) == 1) { + *offsetc = GICC_OFFSET_REV1_1; + *offsetd = GICD_OFFSET_REV1_1; + } else { + *offsetc = GICC_OFFSET_REV1; + *offsetd = GICD_OFFSET_REV1; + } +#else + *offsetc = GICC_OFFSET; + *offsetd = GICD_OFFSET; +#endif +} + +void boot_primary_init_intc(void) +{ + paddr_t gic_base = 0; + uint32_t gicc_offset = 0; + uint32_t gicd_offset = 0; + +#if defined(PLATFORM_FLAVOR_lx2160aqds) || defined(PLATFORM_FLAVOR_lx2160ardb) + if (get_gic_base_addr_from_dt(&gic_base)) + EMSG("Failed to get GIC base addr from DT"); +#else + gic_base = GIC_BASE; +#endif + get_gic_offset(&gicc_offset, &gicd_offset); + gic_init(gic_base + gicc_offset, gic_base + gicd_offset); +} + +#if !defined(CFG_WITH_ARM_TRUSTED_FW) +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} +#endif diff --git a/optee_os/core/arch/arm/plat-ls/plat_init.S b/optee_os/core/arch/arm/plat-ls/plat_init.S new file mode 100644 index 0000000..a6350b9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ls/plat_init.S @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Entry points for the A9 inits, A9 revision specific or not. + * It is assume no stack is available when these routines are called. + * It is assume each routine is called with return address in LR + * and with ARM registers R0, R1, R2, R3 being scratchable. + */ + +#include +#include +#include +#include + +.section .text +.balign 4 +.code 32 + +/* + * platform early configuration + * + * Use scratables registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + */ +FUNC plat_cpu_reset_early , : + /* + * Disallow NSec to mask FIQ [bit4: FW=0] + * Allow NSec to manage Imprecise Abort [bit5: AW=1] + * Imprecise Abort trapped to Abort Mode [bit3: EA=0] + * In Sec world, FIQ trapped to FIQ Mode [bit2: FIQ=0] + * IRQ always trapped to IRQ Mode [bit1: IRQ=0] + * Secure World [bit0: NS=0] + */ + mov r0, #SCR_AW + write_scr r0 /* write Secure Configuration Register */ + + /* + * Mandated HW config loaded + * + * SCTLR = 0x00000000 + * + * ACTRL = 0x00000040 + * - core NOT booted in full SMP (FW bit0=0) + * + * NSACR = 0x00000C00 + * - NSec cannot change ACTRL.SMP (NS_SMP bit18=0) + * - NSec can use SIMD/VFP (CP10/CP11) (bit15:14=2b00, bit11:10=2b11) + */ + mov_imm r0, 0x00000000 + write_sctlr r0 + + mov_imm r0, 0x00000040 + write_actlr r0 + + mov_imm r0, 0x00000C00 + write_nsacr r0 + + mov pc, lr +END_FUNC plat_cpu_reset_early diff --git a/optee_os/core/arch/arm/plat-ls/platform_config.h b/optee_os/core/arch/arm/plat-ls/platform_config.h new file mode 100644 index 0000000..3924b5b --- /dev/null +++ b/optee_os/core/arch/arm/plat-ls/platform_config.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * All rights reserved. + * Copyright 2021 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +#define STACK_ALIGNMENT 64 + +/* console uart define */ +#define CONSOLE_UART_BASE UART0_BASE + +/* Platform specific defines */ +#if defined(PLATFORM_FLAVOR_ls1012ardb) +/* DUART 1 */ +#define UART0_BASE 0x021C0500 +#define GIC_BASE 0x01400000 +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 +#define CAAM_BASE 0x01700000 +#define CAAM_SIZE 0x100000 +#endif + +#if defined(PLATFORM_FLAVOR_ls1043ardb) +#define UART0_BASE 0x021C0500 +#define GIC_BASE 0x01400000 +#define GICC_OFFSET_REV1 0x2000 +#define GICD_OFFSET_REV1 0x1000 +#define GICC_OFFSET_REV1_1 0x20000 +#define GICD_OFFSET_REV1_1 0x10000 +#define CAAM_BASE 0x01700000 +#define CAAM_SIZE 0x100000 +#define DCFG_BASE 0x01EE0000 +#define DCFG_SVR_OFFSET 0xA4 +#endif + +#if defined(PLATFORM_FLAVOR_ls1046ardb) +/* DUART 1 */ +#define UART0_BASE 0x021C0500 +#define GIC_BASE 0x01400000 +#define GICC_OFFSET 0x20000 +#define GICD_OFFSET 0x10000 +#define CAAM_BASE 0x01700000 +#define CAAM_SIZE 0x100000 +#endif + +#if defined(PLATFORM_FLAVOR_ls1088ardb) +/* DUART 1 */ +#define UART0_BASE 0x021C0500 +#define GIC_BASE 0x06000000 +#define GICC_OFFSET 0x0 +#define GICD_OFFSET 0x0 +#define CAAM_BASE 0x08000000 +#define CAAM_SIZE 0x100000 +#endif + +#if defined(PLATFORM_FLAVOR_ls2088ardb) +/* DUART 1 */ +#define UART0_BASE 0x021C0600 +#define GIC_BASE 0x06000000 +#define GICC_OFFSET 0x0 +#define GICD_OFFSET 0x0 +#define CAAM_BASE 0x08000000 +#define CAAM_SIZE 0x100000 +#endif + +#if defined(PLATFORM_FLAVOR_ls1028ardb) +/* DUART 1 */ +#define UART0_BASE 0x021C0500 +#define GIC_BASE 0x06000000 +#define GICC_OFFSET 0x0 +#define GICD_OFFSET 0x0 +#define CAAM_BASE 0x08000000 +#define CAAM_SIZE 0x100000 +#endif + +#if defined(PLATFORM_FLAVOR_lx2160ardb) +/* DUART 1 */ +#define UART0_BASE 0x021C0000 +#define GIC_BASE 0x06000000 +#define GICC_OFFSET 0x0 +#define GICD_OFFSET 0x0 +#define CAAM_BASE 0x08000000 +#define CAAM_SIZE 0x100000 +#endif + +#if defined(PLATFORM_FLAVOR_lx2160aqds) +/* DUART 1 */ +#define UART0_BASE 0x021C0000 +#define GIC_BASE 0x06000000 +#define GICC_OFFSET 0x0 +#define GICD_OFFSET 0x0 +#define CAAM_BASE 0x08000000 +#define CAAM_SIZE 0x100000 +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-ls/sub.mk b/optee_os/core/arch/arm/plat-ls/sub.mk new file mode 100644 index 0000000..17fcf7a --- /dev/null +++ b/optee_os/core/arch/arm/plat-ls/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-$(CFG_ARM32_core) += plat_init.S diff --git a/optee_os/core/arch/arm/plat-marvell/armada3700/hal_sec_perf.c b/optee_os/core/arch/arm/plat-marvell/armada3700/hal_sec_perf.c new file mode 100644 index 0000000..daecbe1 --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/armada3700/hal_sec_perf.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Marvell International Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#define PHY_2_VIR(addr) ((vaddr_t)phys_to_virt((addr), MEM_AREA_IO_SEC, 1)) + +#define MCU_BASE 0xD0000000 +#define MCU_MC_CONTROL_0_REG PHY_2_VIR(MCU_BASE + 0x044) +#define TRUSTZONE_LOCK BIT(31) + +#define MCU_TZ_RANGE_HIGH_REG(x) PHY_2_VIR(MCU_BASE + 0x84 + ((x) << 3)) +#define MCU_TZ_RANGE_LOW_REG(x) PHY_2_VIR(MCU_BASE + 0x80 + ((x) << 3)) + +#define RW_PERM 0x0 +#define RO_PERM 0x1 +#define WO_PERM 0x2 +#define ABORT_PERM 0x3 + +#define MAX_RANGE_NUM 16 +#define INVALID_SIZE_CODE 0xff + +#ifdef TEE_RES_CFG_16M +#define RSVD_SEC_MEM (SIZE_8M + SIZE_8M) +#elif defined(TEE_RES_CFG_24M) +#define RSVD_SEC_MEM (SIZE_8M + SIZE_8M + SIZE_8M) +#elif defined(TEE_RES_CFG_8M) +#define RSVD_SEC_MEM SIZE_8M +#else +#error "no reserved secure memory defined." +#endif + +#define RA_ADDR TZDRAM_BASE +#define RA_SIZE TZDRAM_SIZE +#define RA_PERM ABORT_PERM + +#define TZ_IS_VALID(data) ((data) & (0x1)) +#define TZ_SET_VALID(data) ((data) |= (0x1)) + +#define TZ_GET_PERM(data, ret) ((ret) = (((data) & (0x3 << 1)) >> 1)) +#define TZ_SET_PERM(data, val) \ + do { \ + (data) &= (~(0x3 << 1)); \ + (data) |= (((val) & 0x3) << 1); \ + } while (0) + +#define TZ_GET_RZ_EN(data, ret) ((ret) = (((data) & (0x1 << 3)) >> 3)) +#define TZ_SET_RZ_EN(data, val) \ + do { \ + (data) &= (~(0x1 << 3)); \ + (data) |= (((val) & 0x1) << 3); \ + } while (0) + +#define TZ_GET_AREA_LEN_CODE(data, ret) ((ret) = (((data) & (0x1F << 8)) >> 8)) + +#define TZ_SET_AREA_LEN_CODE(data, val) \ + do { \ + (data) &= (~(0x1F << 8)); \ + (data) |= (((val) & 0x1F) << 8); \ + } while (0) + +#define TZ_GET_START_ADDR_L(data, ret) \ + ((ret) = (((data) & 0xFFF00000))) + +#define TZ_SET_START_ADDR_L(data, val) \ + do { \ + (data) &= (~0xFFF00000); \ + (data) |= (((val) & 0xFFF00000)); \ + } while (0) + +#define TZ_GET_UR_PERM(data, val) ((ret) = (((data) & (0x3 << 4)) >> 4)) +#define TZ_SET_UR_PERM(data, val) \ + do { \ + (data) &= (~(0x3 << 4)); \ + (data) |= (((val) & 0x3) << 4); \ + } while (0) + +#define TZ_GET_UR_RZ_EN(data, val) \ + ((ret) = (((data) & (0x1 << 6)) >> 6)) + +#define TZ_SET_UR_RZ_EN(data, val) \ + do { \ + (data) &= (~(0x1 << 6)); \ + (data) |= (((val) & 0x1) << 6); \ + } while (0) + + /* armada3700 mini region size is 1M */ +#define RANGE_SIZE_TO_CODE(size, code, i) \ + do { \ + (code) = INVALID_SIZE_CODE; \ + for ((i) = 0; (i) <= 0x1d; (i)++) { \ + if (((uint32_t)0x1 << (i)) == ((size) >> 20)) { \ + (code) = (i); \ + break; \ + } \ + } \ + } while (0) + +#define TZ_LOCK_MC(x) \ + do { \ + (x) = io_read32(MCU_MC_CONTROL_0_REG); \ + (x) |= (TRUSTZONE_LOCK); \ + io_write32(MCU_MC_CONTROL_0_REG, (x)); \ + } while (0) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, MCU_BASE, CORE_MMU_PGDIR_SIZE); + +static int32_t _find_valid_range(void) +{ + uint32_t i; + uint32_t tmp; + + for (i = 0; i < MAX_RANGE_NUM; i++) { + tmp = io_read32(MCU_TZ_RANGE_LOW_REG(i)); + if (!TZ_IS_VALID(tmp)) + return i; + } + return -1; +} + +static int32_t set_range(uint32_t addr, uint32_t size, uint32_t perm) +{ + uint32_t data; + uint32_t sizecode; + int32_t valid_range; + uint32_t i; + + if (!IS_ALIGNED(addr, SIZE_1M)) { + EMSG("region addr(0x%" PRIx32 ") is not aligned with 1M!", + addr); + return -1; + } + + if (!IS_ALIGNED(size, SIZE_1M)) { + EMSG("region size(0x%" PRIx32 ") is not aligned with 1M!", + size); + return -1; + } + + RANGE_SIZE_TO_CODE(size, sizecode, i); + if (sizecode == INVALID_SIZE_CODE) { + EMSG("not valid region size(2^n)! size:0x%" PRIx32, size); + return -1; + } + + valid_range = _find_valid_range(); + if (valid_range == -1) { + EMSG("ERR: can't find valid range!"); + return -1; + } + + data = io_read32(MCU_TZ_RANGE_LOW_REG(valid_range)); + + TZ_SET_VALID(data); + TZ_SET_PERM(data, perm); + TZ_SET_AREA_LEN_CODE(data, sizecode); + TZ_SET_START_ADDR_L(data, addr); + + if (!valid_range) { + /* Set Undefine Range RW */ + TZ_SET_UR_PERM(data, RW_PERM); + TZ_SET_UR_RZ_EN(data, 0); + } + + io_write32(MCU_TZ_RANGE_LOW_REG(valid_range), data); + + return 0; +} + +static void _dump_range(void) +{ + uint32_t i; + uint32_t tmp; + uint32_t __maybe_unused sizecode_read; + uint32_t __maybe_unused sizem; + uint32_t __maybe_unused addr_read; + uint32_t __maybe_unused perm_read; + + for (i = 0; i < MAX_RANGE_NUM; i++) { + tmp = io_read32(MCU_TZ_RANGE_LOW_REG(i)); + + if (TZ_IS_VALID(tmp)) { + TZ_GET_PERM(tmp, perm_read); + TZ_GET_AREA_LEN_CODE(tmp, sizecode_read); + TZ_GET_START_ADDR_L(tmp, addr_read); + + DMSG("Range Num%" PRIu32 + ": Reg 0x%" PRIx64 " = 0x%" PRIx32, + i, MCU_TZ_RANGE_LOW_REG(i), tmp); + DMSG("AddrL: 0x%08" PRIx32, addr_read); + DMSG("Size: %" PRIu32 "M", (0x1 << sizecode_read)); + DMSG("Perm: %" PRIu32, perm_read); + } + } +} + +static void _set_range(uint32_t addr, uint32_t size, uint32_t perm) +{ + uint32_t rgn_addr = addr; + uint32_t rgn_size = size; + /* minimum region size is 1M and must be times of 1M */ + uint32_t p = 0x100000; + + while (1) { + if ((p * 2) > rgn_size) { + set_range(rgn_addr, p, perm); + rgn_addr += p; + rgn_size -= p; + if (rgn_size == 0) + break; + p = 0x100000; + } else + p <<= 1; + } +} + +static TEE_Result init_sec_perf(void) +{ + uint32_t tmp; + + /* Set Secure Memory Region */ + DMSG("sec-rgn size: ra = 0x%" PRIx32 ", size = 0x%" PRIx64, + RA_ADDR, RA_SIZE); + _set_range(RA_ADDR, RA_SIZE, RA_PERM); + + /* Close TZ register modification */ + TZ_LOCK_MC(tmp); + + _dump_range(); + + return TEE_SUCCESS; +} + +service_init(init_sec_perf); diff --git a/optee_os/core/arch/arm/plat-marvell/armada7k8k/hal_sec_perf.c b/optee_os/core/arch/arm/plat-marvell/armada7k8k/hal_sec_perf.c new file mode 100644 index 0000000..cbfda55 --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/armada7k8k/hal_sec_perf.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Marvell International Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#define PHY_2_VIR(addr) ((vaddr_t)phys_to_virt((addr), MEM_AREA_IO_SEC, 1)) + +#define MCU_MC_CONTROL_0_REG PHY_2_VIR(MCU_BASE + 0x044) +#define TRUSTZONE_LOCK BIT(31) + +#define MCU_TZ_RANGE_HIGH_REG(x) PHY_2_VIR(MCU_BASE + 0x84 + ((x) << 3)) +#define MCU_TZ_RANGE_LOW_REG(x) PHY_2_VIR(MCU_BASE + 0x80 + ((x) << 3)) + +#define RW_PERM 0x0 +#define RO_PERM 0x1 +#define WO_PERM 0x2 +#define ABORT_PERM 0x3 + +#define MAX_RANGE_NUM 16 +#define INVALID_SIZE_CODE 0xff + +#ifdef TEE_RES_CFG_16M +#define RSVD_SEC_MEM (SIZE_8M + SIZE_8M) +#elif defined(TEE_RES_CFG_24M) +#define RSVD_SEC_MEM (SIZE_8M + SIZE_8M + SIZE_8M) +#elif defined(TEE_RES_CFG_8M) +#define RSVD_SEC_MEM SIZE_8M +#else +#error "no reserved secure memory defined." +#endif + +#define RA_ADDR TZDRAM_BASE +#define RA_SIZE TZDRAM_SIZE +#define RA_PERM ABORT_PERM + +#define TZ_IS_VALID(data) ((data) & (0x1)) +#define TZ_SET_VALID(data) ((data) |= (0x1)) + +#define TZ_GET_PERM(data, ret) ((ret) = (((data) & (0x3 << 1)) >> 1)) +#define TZ_SET_PERM(data, val) \ + do { \ + (data) &= (~(0x3 << 1)); \ + (data) |= (((val) & 0x3) << 1); \ + } while (0) + +#define TZ_GET_RZ_EN(data, ret) ((ret) = (((data) & (0x1 << 3)) >> 3)) +#define TZ_SET_RZ_EN(data, val) \ + do { \ + (data) &= (~(0x1 << 3)); \ + (data) |= (((val) & 0x1) << 3); \ + } while (0) + +#define TZ_GET_AREA_LEN_CODE(data, ret) ((ret) = (((data) & (0x1F << 7)) >> 7)) + +#define TZ_SET_AREA_LEN_CODE(data, val) \ + do { \ + (data) &= (~(0x1F << 7)); \ + (data) |= (((val) & 0x1F) << 7); \ + } while (0) + +#define TZ_GET_START_ADDR_L(data, ret) \ + ((ret) = (((data) & 0xFFFFF000))) + +#define TZ_SET_START_ADDR_L(data, val) \ + do { \ + (data) &= (~0xFFFFF000); \ + (data) |= (((val) & 0xFFFFF000)); \ + } while (0) + +#define TZ_GET_UR_PERM(data, val) ((ret) = (((data) & (0x3 << 4)) >> 4)) +#define TZ_SET_UR_PERM(data, val) \ + do { \ + (data) &= (~(0x3 << 4)); \ + (data) |= (((val) & 0x3) << 4); \ + } while (0) + +#define TZ_GET_UR_RZ_EN(data, val) \ + ((ret) = (((data) & (0x1 << 6)) >> 6)) + +#define TZ_SET_UR_RZ_EN(data, val) \ + do { \ + (data) &= (~(0x1 << 6)); \ + (data) |= (((val) & 0x1) << 6); \ + } while (0) + + /* armada mini region size is 1M */ +#define RANGE_SIZE_TO_CODE(size, code, i) \ + do { \ + (code) = INVALID_SIZE_CODE; \ + for ((i) = 8; (i) <= 0x1f; (i)++) { \ + if (((uint32_t)0x1 << (i)) == ((size) >> 12)) { \ + (code) = (i); \ + break; \ + } \ + } \ + } while (0) + +#define RANGE_CODE_TO_SIZE_K(code, sizek) ((sizek) = ((4) << (code))) + +#define TZ_LOCK_MC(x) \ + do { \ + (x) = io_read32(MCU_MC_CONTROL_0_REG); \ + (x) |= (TRUSTZONE_LOCK); \ + io_write32(MCU_MC_CONTROL_0_REG, (x)); \ + } while (0) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, MCU_BASE, CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, MC_SCR_REGISTER, CORE_MMU_PGDIR_SIZE); + +static int32_t _find_valid_range(void) +{ + uint32_t i; + uint32_t tmp; + + for (i = 0; i < MAX_RANGE_NUM; i++) { + tmp = io_read32(MCU_TZ_RANGE_LOW_REG(i)); + if (!TZ_IS_VALID(tmp)) + return i; + } + return -1; +} + +static int32_t set_range(uint32_t addr, uint32_t size, uint32_t perm) +{ + uint32_t data; + uint32_t sizecode; + int32_t valid_range; + uint32_t i; + + if (!IS_ALIGNED(addr, SIZE_1M)) { + EMSG("region addr(0x%" PRIx32 ") is not aligned with 1M!", + addr); + return -1; + } + + if (!IS_ALIGNED(size, SIZE_1M)) { + EMSG("region size(0x%" PRIx32 ") is not aligned with 1M!", + size); + return -1; + } + + if (!IS_ALIGNED(addr, size)) { + EMSG("region size(0x%" PRIx32 + ") not align with addr(0x%" PRIx32 ")", + size, addr); + return -1; + } + + RANGE_SIZE_TO_CODE(size, sizecode, i); + if (sizecode == INVALID_SIZE_CODE) { + EMSG("not valid region size(2^n)! size:0x%" PRIx32, size); + return -1; + } + + valid_range = _find_valid_range(); + if (valid_range == -1) { + EMSG("ERR: can't find valid range!"); + return -1; + } + + data = io_read32(MCU_TZ_RANGE_LOW_REG(valid_range)); + + TZ_SET_VALID(data); + TZ_SET_PERM(data, perm); + TZ_SET_AREA_LEN_CODE(data, sizecode); + TZ_SET_START_ADDR_L(data, addr); + + if (!valid_range) { + /* Set Undefine Range RW */ + TZ_SET_UR_PERM(data, RW_PERM); + TZ_SET_UR_RZ_EN(data, 0); + } + + io_write32(MCU_TZ_RANGE_LOW_REG(valid_range), data); + + return 0; +} + +static void _dump_range(void) +{ + uint32_t i; + uint32_t tmp; + uint32_t sizek; + uint32_t sizecode_read; + uint32_t __maybe_unused sizem; + uint32_t __maybe_unused addr_read; + uint32_t __maybe_unused perm_read; + + for (i = 0; i < MAX_RANGE_NUM; i++) { + tmp = io_read32(MCU_TZ_RANGE_LOW_REG(i)); + + if (TZ_IS_VALID(tmp)) { + TZ_GET_PERM(tmp, perm_read); + TZ_GET_AREA_LEN_CODE(tmp, sizecode_read); + TZ_GET_START_ADDR_L(tmp, addr_read); + + DMSG("Range Num%" PRIu32 + ": Reg 0x%" PRIx64 " = 0x%" PRIx32, + i, MCU_TZ_RANGE_LOW_REG(i), tmp); + DMSG("AddrL: 0x%08" PRIx32, addr_read); + RANGE_CODE_TO_SIZE_K(sizecode_read, sizek); + sizem = sizek >> 10; + DMSG("Size: %" PRIu32 "K, %" PRIu32 "M", sizek, sizem); + DMSG("Perm: %" PRIu32, perm_read); + } + } +} + +static uint32_t _find_granule(uint32_t addr, uint32_t size) +{ + /* max supported granule for armada is 8TB + * but 2GB is far enough here + */ + uint32_t max_granule = SIZE_2G; + + while (max_granule >= SIZE_4K) { /* min granule is 4kB */ + if (max_granule <= size && IS_ALIGNED(addr, max_granule)) + return max_granule; + + max_granule >>= 1; + } + + return 0; /* cannot find a valid granule */ +} + +static void _set_range(uint32_t addr, uint32_t size, uint32_t perm) +{ + uint32_t rgn_addr = addr; + uint32_t rgn_size = size; + uint32_t p; + + while (rgn_size) { + p = _find_granule(rgn_addr, rgn_size); + if (!p) + panic("cannot find a suitable granule!"); + if (set_range(rgn_addr, p, perm)) + panic("set_range failed!"); + + rgn_addr += p; + rgn_size -= p; + } +} + +static TEE_Result init_sec_perf(void) +{ + uint32_t tmp; + + /* MC_SCR config: deny NS access to MC registers */ + tmp = io_read32(PHY_2_VIR(MC_SCR_REGISTER)); + tmp |= 0x1; + io_write32(PHY_2_VIR(MC_SCR_REGISTER), tmp); + + /* Set Secure Memory Region */ + DMSG("sec-rgn size: ra = 0x%08" PRIx32 ", size = 0x%" PRIx32, + RA_ADDR, RA_SIZE); + _set_range(RA_ADDR, RA_SIZE, RA_PERM); + + /* Close TZ register modification */ + TZ_LOCK_MC(tmp); + + _dump_range(); + + return TEE_SUCCESS; +} + +service_init(init_sec_perf); diff --git a/optee_os/core/arch/arm/plat-marvell/cn10k/core_pos.S b/optee_os/core/arch/arm/plat-marvell/cn10k/core_pos.S new file mode 100644 index 0000000..dd7cced --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/cn10k/core_pos.S @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2023 Marvell. + */ + +#include +#include +#include + +FUNC get_core_pos_mpidr , : +/* No clusters, core position is affinity2 of MPIDR_EL1 */ + mov_imm x1, MPIDR_AFFLVL_MASK + and x0, x1, x0, LSR #MPIDR_AFF2_SHIFT + ret +END_FUNC get_core_pos_mpidr diff --git a/optee_os/core/arch/arm/plat-marvell/conf.mk b/optee_os/core/arch/arm/plat-marvell/conf.mk new file mode 100644 index 0000000..473fd91 --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/conf.mk @@ -0,0 +1,169 @@ +PLATFORM_FLAVOR ?= armada7k8k + +ifeq ($(PLATFORM_FLAVOR),armada7k8k) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_TZDRAM_START,0x04400000) +$(call force,CFG_TZDRAM_SIZE,0x00C00000) +$(call force,CFG_SHMEM_START,0x05000000) +$(call force,CFG_SHMEM_SIZE,0x00400000) +$(call force,CFG_TEE_RAM_VA_SIZE,0x00400000) +# If Secure Data Path is enabled, uses the TZDRAM last 4MByte +$(call force,CFG_TEE_SDP_MEM_SIZE,0x00400000) +platform-debugger-arm := 1 +$(call force,CFG_8250_UART,y) +endif + +ifeq ($(PLATFORM_FLAVOR),armada3700) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_TZDRAM_START,0x04400000) +$(call force,CFG_TZDRAM_SIZE,0x00C00000) +$(call force,CFG_SHMEM_START,0x05000000) +$(call force,CFG_SHMEM_SIZE,0x00400000) +$(call force,CFG_TEE_RAM_VA_SIZE,0x00400000) +# If Secure Data Path is enabled, uses the TZDRAM last 4MByte +$(call force,CFG_TEE_SDP_MEM_SIZE,0x00400000) +platform-debugger-arm := 1 +$(call force,CFG_MVEBU_UART,y) +$(call force,CFG_ARM_GICV3,y) +endif + +ifeq ($(PLATFORM_FLAVOR),otx2t96) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,24) +$(call force,CFG_CLUSTERS_PER_NODE,4) +$(call force,CFG_TZDRAM_START,0x00001000) +$(call force,CFG_TZDRAM_SIZE,0x000a00000) +$(call force,CFG_SHMEM_START,0x01000000) +$(call force,CFG_SHMEM_SIZE,0x00800000) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) +$(call force,CFG_PL011,y) +$(call force,CFG_ARM_GICV3,y) +CFG_HW_UNQ_KEY_SUPPORT ?= y +CFG_USER_TA_TARGETS ?= ta_arm64 +CFG_NUM_THREADS ?= CFG_TEE_CORE_NB_CORE +CFG_CORE_HEAP_SIZE ?= 131072 +endif + +ifeq ($(PLATFORM_FLAVOR),otx2f95) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,6) +$(call force,CFG_CLUSTERS_PER_NODE,1) +$(call force,CFG_TZDRAM_START,0x00001000) +$(call force,CFG_TZDRAM_SIZE,0x000a00000) +$(call force,CFG_SHMEM_START,0x01000000) +$(call force,CFG_SHMEM_SIZE,0x00800000) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) +$(call force,CFG_PL011,y) +$(call force,CFG_ARM_GICV3,y) +CFG_HW_UNQ_KEY_SUPPORT ?= y +CFG_USER_TA_TARGETS ?= ta_arm64 +CFG_NUM_THREADS ?= CFG_TEE_CORE_NB_CORE +CFG_CORE_HEAP_SIZE ?= 131072 +endif + +ifeq ($(PLATFORM_FLAVOR),otx2t98) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,36) +$(call force,CFG_CLUSTERS_PER_NODE,6) +$(call force,CFG_TZDRAM_START,0x00001000) +$(call force,CFG_TZDRAM_SIZE,0x000a00000) +$(call force,CFG_SHMEM_START,0x01000000) +$(call force,CFG_SHMEM_SIZE,0x00800000) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +#$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) +$(call force,CFG_PL011,y) +$(call force,CFG_ARM_GICV3,y) +CFG_HW_UNQ_KEY_SUPPORT ?= y +CFG_USER_TA_TARGETS ?= ta_arm64 +CFG_NUM_THREADS ?= CFG_TEE_CORE_NB_CORE +CFG_CORE_HEAP_SIZE ?= 131072 +endif + +ifeq ($(PLATFORM_FLAVOR),cn10ka) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,24) +$(call force,CFG_TZDRAM_START,0x00001000) +$(call force,CFG_TZDRAM_SIZE,0x000a00000) +$(call force,CFG_SHMEM_START,0x03400000) +$(call force,CFG_SHMEM_SIZE,0x00800000) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) +$(call force,CFG_PL011,y) +$(call force,CFG_ARM_GICV3,y) +CFG_USER_TA_TARGETS ?= ta_arm64 +CFG_NUM_THREADS ?= CFG_TEE_CORE_NB_CORE +CFG_CORE_HEAP_SIZE ?= 131072 +endif + +ifeq ($(PLATFORM_FLAVOR),cn10kb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_TZDRAM_START,0x00001000) +$(call force,CFG_TZDRAM_SIZE,0x000a00000) +$(call force,CFG_SHMEM_START,0x03400000) +$(call force,CFG_SHMEM_SIZE,0x00800000) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) +$(call force,CFG_PL011,y) +$(call force,CFG_ARM_GICV3,y) +CFG_USER_TA_TARGETS ?= ta_arm64 +CFG_NUM_THREADS ?= CFG_TEE_CORE_NB_CORE +CFG_CORE_HEAP_SIZE ?= 131072 +endif + +ifeq ($(PLATFORM_FLAVOR),cnf10ka) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,18) +$(call force,CFG_TZDRAM_START,0x00001000) +$(call force,CFG_TZDRAM_SIZE,0x000a00000) +$(call force,CFG_SHMEM_START,0x03400000) +$(call force,CFG_SHMEM_SIZE,0x00800000) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) +$(call force,CFG_PL011,y) +$(call force,CFG_ARM_GICV3,y) +CFG_USER_TA_TARGETS ?= ta_arm64 +CFG_NUM_THREADS ?= CFG_TEE_CORE_NB_CORE +CFG_CORE_HEAP_SIZE ?= 131072 +endif + +ifeq ($(PLATFORM_FLAVOR),cnf10kb) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,12) +$(call force,CFG_TZDRAM_START,0x00001000) +$(call force,CFG_TZDRAM_SIZE,0x000a00000) +$(call force,CFG_SHMEM_START,0x03400000) +$(call force,CFG_SHMEM_SIZE,0x00800000) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,48) +$(call force,CFG_LPAE_ADDR_SPACE_BITS,36) +$(call force,CFG_PL011,y) +$(call force,CFG_ARM_GICV3,y) +CFG_USER_TA_TARGETS ?= ta_arm64 +CFG_NUM_THREADS ?= CFG_TEE_CORE_NB_CORE +CFG_CORE_HEAP_SIZE ?= 131072 +endif + +ifeq ($(platform-debugger-arm),1) +# ARM debugger needs this +platform-cflags-debug-info = -gdwarf-2 +platform-aflags-debug-info = -gdwarf-2 +endif + +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_GIC,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) + +CFG_WITH_STATS ?= y diff --git a/optee_os/core/arch/arm/plat-marvell/main.c b/optee_os/core/arch/arm/plat-marvell/main.c new file mode 100644 index 0000000..a299b21 --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/main.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Marvell International Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#if defined(PLATFORM_FLAVOR_armada7k8k) +#include +#elif defined(PLATFORM_FLAVOR_armada3700) +#include +#endif +#ifdef CFG_PL011 +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(PLATFORM_FLAVOR_armada7k8k) +static struct serial8250_uart_data console_data; +#elif defined(PLATFORM_FLAVOR_armada3700) +static struct mvebu_uart_data console_data; +#elif CFG_PL011 +static struct pl011_data console_data; +#endif + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, + CORE_MMU_PGDIR_SIZE); +#ifdef CFG_HW_UNQ_KEY_SUPPORT +register_phys_mem(MEM_AREA_IO_SEC, PLAT_MARVELL_FUSF_FUSE_BASE, + SMALL_PAGE_SIZE); +#endif + +#ifdef GIC_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, CORE_MMU_PGDIR_SIZE); +#ifdef GICC_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, CORE_MMU_PGDIR_SIZE); +#endif + +void boot_primary_init_intc(void) +{ + paddr_t gicd_base = 0; + paddr_t gicc_base = 0; + +#ifdef GICC_BASE + gicc_base = GIC_BASE + GICC_OFFSET; +#endif + gicd_base = GIC_BASE + GICD_OFFSET; + + gic_init(gicc_base, gicd_base); +} +#endif + +void console_init(void) +{ +#if defined(PLATFORM_FLAVOR_armada7k8k) + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); +#elif defined(PLATFORM_FLAVOR_armada3700) + mvebu_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); +#elif CFG_PL011 + pl011_init(&console_data, CONSOLE_UART_BASE, CONSOLE_UART_CLK_IN_HZ, + CONSOLE_BAUDRATE); +#endif + register_serial_console(&console_data.chip); +} + +#ifdef CFG_HW_UNQ_KEY_SUPPORT +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + void *huk = phys_to_virt(PLAT_MARVELL_FUSF_FUSE_BASE + + PLAT_MARVELL_FUSF_HUK_OFFSET, + MEM_AREA_IO_SEC, sizeof(hwkey->data)); + if (!huk) { + EMSG("\nH/W Unique key is not fetched from the platform."); + return TEE_ERROR_SECURITY; + } + + memcpy(&hwkey->data[0], huk, sizeof(hwkey->data)); + return TEE_SUCCESS; +} +#endif diff --git a/optee_os/core/arch/arm/plat-marvell/otx2/core_pos.S b/optee_os/core/arch/arm/plat-marvell/otx2/core_pos.S new file mode 100644 index 0000000..a5a612f --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/otx2/core_pos.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Marvell International Ltd. + */ + +#include +#include + +FUNC get_core_pos_mpidr , : + ubfx x1, x0, #MPIDR_AFF0_SHIFT, #MPIDR_AFFINITY_BITS + mov x2, #CFG_CLUSTERS_PER_NODE + mul x1, x1, x2 + ubfx x2, x0, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS + add x0, x1, x2 + ret +END_FUNC get_core_pos_mpidr diff --git a/optee_os/core/arch/arm/plat-marvell/platform_config.h b/optee_os/core/arch/arm/plat-marvell/platform_config.h new file mode 100644 index 0000000..365a6e6 --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/platform_config.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017 Marvell International Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#ifdef ARM64 +#ifdef CFG_WITH_PAGER +#error "Pager not supported yet" +#endif +#else +#error "32 bit mode not supported yet" +#endif /*ARM64*/ + +#if defined(PLATFORM_FLAVOR_armada7k8k) +/* + * armada7k8k specifics. + */ +#define TEE_RES_CFG_8M + +#define MVEBU_REGS_BASE 0xF0000000 + +/* GICv2 */ +#define MVEBU_GICD_BASE 0x210000 +#define MVEBU_GICC_BASE 0x220000 +#define GIC_DIST_BASE (MVEBU_REGS_BASE + MVEBU_GICD_BASE) +#define GIC_CPU_BASE (MVEBU_REGS_BASE + MVEBU_GICC_BASE) + +#define GIC_BASE GIC_DIST_BASE + +/* UART */ +#define PLAT_MARVELL_BOOT_UART_BASE (MVEBU_REGS_BASE + 0x512000) +#define PLAT_MARVELL_BOOT_UART_CLK_IN_HZ 200000000 +#define MARVELL_CONSOLE_BAUDRATE 115200 + +#define CONSOLE_UART_BASE PLAT_MARVELL_BOOT_UART_BASE + +#define GICC_OFFSET 0x10000 +#define GICD_OFFSET 0x0 + +#define GICD_BASE (GIC_BASE + GICD_OFFSET) +#define GICC_BASE (GIC_BASE + GICC_OFFSET) + +/* MCU */ +#define MCU_BASE 0xF0020000 +#define MCU_REG_SIZE SIZE_4K +#define MC_SCR_REGISTER 0xF06F0204 +#define MC_SCR_REG_SIZE SIZE_4K + +#elif defined(PLATFORM_FLAVOR_armada3700) +/* + * armada3700 specifics. + */ +#define TEE_RES_CFG_8M + +#define MVEBU_REGS_BASE 0xD0000000 + +/* GICv3 */ +#define MVEBU_GICD_BASE 0x1D00000 +#define MVEBU_GICR_BASE 0x1D40000 +#define MVEBU_GICC_BASE 0x1D80000 + +#define GIC_DIST_BASE (MVEBU_REGS_BASE + MVEBU_GICD_BASE) +#define GIC_RDIS_BASE (MVEBU_REGS_BASE + MVEBU_GICR_BASE) +#define GIC_CPU_BASE (MVEBU_REGS_BASE + MVEBU_GICC_BASE) + +#define GIC_BASE GIC_DIST_BASE +#define GICC_OFFSET (0x80000) +#define GICR_OFFSET (0x40000) +#define GICD_OFFSET (0x0) + +#define GICD_BASE (GIC_BASE + GICD_OFFSET) +#define GICC_BASE (GIC_BASE + GICC_OFFSET) + +/* UART */ +#define PLAT_MARVELL_BOOT_UART_BASE (MVEBU_REGS_BASE + 0x12000) +#define PLAT_MARVELL_BOOT_UART_CLK_IN_HZ 25804800 +#define MARVELL_CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_BASE PLAT_MARVELL_BOOT_UART_BASE + +#elif defined(PLATFORM_FLAVOR_otx2t96) || defined(PLATFORM_FLAVOR_otx2f95) || \ + defined(PLATFORM_FLAVOR_otx2t98) +/* + * OcteonTX2(otx2) specifics. + */ + +/* GICv3 */ +#define GIC_BASE 0x801000000000ll +#define GICD_OFFSET (0x0) + +#define GICD_BASE (GIC_BASE + GICD_OFFSET) + +/* UART */ +#define PLAT_MARVELL_BOOT_UART_BASE 0x87E028000000ll +#define PLAT_MARVELL_BOOT_UART_CLK_IN_HZ 16666656 +#define MARVELL_CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_BASE PLAT_MARVELL_BOOT_UART_BASE + +/* eFUSE */ +#define PLAT_MARVELL_FUSF_FUSE_BASE 0x87E004000000ll +#define PLAT_MARVELL_FUSF_HUK_OFFSET (0x90) + +#elif defined(PLATFORM_FLAVOR_cn10ka) || defined(PLATFORM_FLAVOR_cn10kb) || \ + defined(PLATFORM_FLAVOR_cnf10ka) || defined(PLATFORM_FLAVOR_cnf10kb) +/* + * cn10k specifics. + */ + +/* GICv3 */ +#define GIC_BASE 0x801000000000ll +#define GICD_OFFSET 0x0 + +#define GICD_BASE (GIC_BASE + GICD_OFFSET) + +/* UART */ +#define PLAT_MARVELL_BOOT_UART_BASE 0x87E028000000ll +#define PLAT_MARVELL_BOOT_UART_CLK_IN_HZ 16666656 +#define MARVELL_CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_BASE PLAT_MARVELL_BOOT_UART_BASE + +#else +#error "Unknown platform flavor" +#endif + +#define UART_BAUDRATE MARVELL_CONSOLE_BAUDRATE +#define CONSOLE_BAUDRATE UART_BAUDRATE +#define CONSOLE_UART_CLK_IN_HZ PLAT_MARVELL_BOOT_UART_CLK_IN_HZ + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-marvell/sub.mk b/optee_os/core/arch/arm/plat-marvell/sub.mk new file mode 100644 index 0000000..e72a8ec --- /dev/null +++ b/optee_os/core/arch/arm/plat-marvell/sub.mk @@ -0,0 +1,10 @@ +global-incdirs-y += . +srcs-y += main.c +ifneq (,$(filter $(PLATFORM_FLAVOR),otx2t96 otx2f95 otx2t98)) +srcs-$(CFG_ARM64_core) += otx2/core_pos.S +endif +ifneq (,$(filter $(PLATFORM_FLAVOR),cn10ka cn10kb cnf10ka cnf10kb)) +srcs-$(CFG_ARM64_core) += cn10k/core_pos.S +endif +srcs-$(PLATFORM_FLAVOR_armada7k8k) += armada7k8k/hal_sec_perf.c +srcs-$(PLATFORM_FLAVOR_armada3700) += armada3700/hal_sec_perf.c diff --git a/optee_os/core/arch/arm/plat-mediatek/conf.mk b/optee_os/core/arch/arm/plat-mediatek/conf.mk new file mode 100644 index 0000000..ed15acb --- /dev/null +++ b/optee_os/core/arch/arm/plat-mediatek/conf.mk @@ -0,0 +1,80 @@ +PLATFORM_FLAVOR ?= mt8173 + +CFG_ARM64_core ?= y + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_8250_UART,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) + +# default DRAM base address +CFG_DRAM_BASE ?= 0x40000000 + +# default DRAM size 1 GiB +CFG_DRAM_SIZE ?= 0x40000000 + +ifeq ($(PLATFORM_FLAVOR),mt8173) +# 2**1 = 2 cores per cluster +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +CFG_TZDRAM_START ?= 0xbe000000 +CFG_TZDRAM_SIZE ?= 0x01e00000 +CFG_SHMEM_START ?= 0xbfe00000 +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),mt8175) +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_GIC,y) +CFG_TZDRAM_START ?= 0x43200000 +CFG_TZDRAM_SIZE ?= 0x00a00000 +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),mt8516) +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +CFG_TZDRAM_START ?= 0x4fd00000 +CFG_TZDRAM_SIZE ?= 0x00300000 +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),mt8183) +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_GIC,y) +CFG_TZDRAM_START ?= 0x4fd00000 +CFG_TZDRAM_SIZE ?= 0x00300000 +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),mt8195) +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_GIC,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +CFG_TZDRAM_START ?= 0x43200000 +CFG_TZDRAM_SIZE ?= 0x00a00000 +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),mt8188) +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_CORE_CLUSTER_SHIFT,2) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_GIC,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +CFG_TZDRAM_START ?= 0x43200000 +CFG_TZDRAM_SIZE ?= 0x00a00000 +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) +CFG_SHMEM_SIZE ?= 0x00200000 +endif diff --git a/optee_os/core/arch/arm/plat-mediatek/main.c b/optee_os/core/arch/arm/plat-mediatek/main.c new file mode 100644 index 0000000..2470def --- /dev/null +++ b/optee_os/core/arch/arm/plat-mediatek/main.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, + CONSOLE_UART_BASE, SERIAL8250_UART_REG_SIZE); + +static struct serial8250_uart_data console_data; + +register_ddr(CFG_DRAM_BASE, CFG_DRAM_SIZE); + +#ifdef CFG_GIC +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE + GICD_OFFSET, + CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE + GICC_OFFSET, + CORE_MMU_PGDIR_SIZE); + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} +#endif + +void console_init(void) +{ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-mediatek/platform_config.h b/optee_os/core/arch/arm/plat-mediatek/platform_config.h new file mode 100644 index 0000000..17e0705 --- /dev/null +++ b/optee_os/core/arch/arm/plat-mediatek/platform_config.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#ifdef ARM64 +#ifdef CFG_WITH_PAGER +#error "Pager not supported for ARM64" +#endif +#endif /*ARM64*/ + +#if defined(PLATFORM_FLAVOR_mt8173) + +#define GIC_BASE 0x10220000 +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 + +#define UART0_BASE 0x11002000 +#define UART1_BASE 0x11003000 +#define UART2_BASE 0x11004000 +#define UART3_BASE 0x11005000 + +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_BAUDRATE 921600 +#define CONSOLE_UART_CLK_IN_HZ 26000000 + +#define DRAM0_BASE 0x40000000 +#define DRAM0_SIZE 0x80000000 + +#elif defined(PLATFORM_FLAVOR_mt8175) + +#define GIC_BASE 0x0C000000 +#define GICC_OFFSET 0x400000 +#define GICD_OFFSET 0x0 + +#define UART0_BASE 0x11002000 +#define UART1_BASE 0x11103000 +#define UART2_BASE 0x11104000 + +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_BAUDRATE 921600 +#define CONSOLE_UART_CLK_IN_HZ 26000000 + +#elif defined(PLATFORM_FLAVOR_mt8516) + +#define GIC_BASE 0x10310000 +#define GICC_OFFSET 0x10000 +#define GICD_OFFSET 0x00000 + +#define UART0_BASE 0x11005000 +#define UART1_BASE 0x11106000 +#define UART2_BASE 0x11107000 + +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_BAUDRATE 921600 +#define CONSOLE_UART_CLK_IN_HZ 26000000 + +#elif defined(PLATFORM_FLAVOR_mt8183) + +#define GIC_BASE 0x0C000000 +#define GICC_OFFSET 0x400000 +#define GICD_OFFSET 0x0 + +#define UART0_BASE 0x11002000 +#define UART1_BASE 0x11103000 +#define UART2_BASE 0x11104000 + +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_BAUDRATE 921600 +#define CONSOLE_UART_CLK_IN_HZ 26000000 + +#elif defined(PLATFORM_FLAVOR_mt8195) + +#define GIC_BASE 0x0C000000 +#define GICC_OFFSET 0x400000 +#define GICD_OFFSET 0x0 + +#define UART0_BASE 0x11001100 +#define UART1_BASE 0x11101200 +#define UART2_BASE 0x11101300 + +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_BAUDRATE 921600 +#define CONSOLE_UART_CLK_IN_HZ 26000000 + +#elif defined(PLATFORM_FLAVOR_mt8188) + +#define GIC_BASE 0x0C000000 +#define GICC_OFFSET 0x400000 +#define GICD_OFFSET 0x0 + +#define UART0_BASE 0x11001100 +#define UART1_BASE 0x11101200 +#define UART2_BASE 0x11101300 + +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 26000000 + +#else +#error "Unknown platform flavor" +#endif + +#ifdef CFG_WITH_LPAE +#define MAX_XLAT_TABLES 5 +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-mediatek/sub.mk b/optee_os/core/arch/arm/plat-mediatek/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-mediatek/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-nuvoton/conf.mk b/optee_os/core/arch/arm/plat-nuvoton/conf.mk new file mode 100644 index 0000000..c3fc96b --- /dev/null +++ b/optee_os/core/arch/arm/plat-nuvoton/conf.mk @@ -0,0 +1,41 @@ +PLATFORM_FLAVOR ?= npcm845x + +ifeq ($(PLATFORM_FLAVOR),npcm845x) +include core/arch/arm/cpu/cortex-armv8-0.mk +CFG_ARM64_core ?= y +endif #npcm845x + +CFG_USER_TA_TARGETS ?= ta_arm64 + +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_GIC,y) +$(call force,CFG_ARM_GICV2,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_16550_UART,y) + +# Not used now for current platform +$(call force,CFG_EXTERNAL_DT,n) + +CFG_WITH_STATS ?= y + +# To enable version printing with color in main +CFG_NPCM_DEBUG ?= n + +ifeq ($(PLATFORM_FLAVOR),npcm845x) +CFG_TEE_CORE_NB_CORE ?= 4 +# [3000.0000 031f.ffff] is reserved to early boot +CFG_TZDRAM_START ?= 0x02100000 +CFG_TZDRAM_SIZE ?= 0x03f00000 +# SHM chosen arbitrary, in a way that it does not interfere +# with initial location of linux kernel, dtb and initrd. +CFG_SHMEM_START ?= 0x06000000 +CFG_SHMEM_SIZE ?= 0x00200000 +# When Secure Data Path is enable, last MByte of TZDRAM is SDP test memory. +CFG_TEE_SDP_MEM_BASE ?= 0x05F00000 +CFG_TEE_SDP_MEM_SIZE ?= 0x00100000 +$(call force,CFG_DT,y) +CFG_DTB_MAX_SIZE ?= 0x100000 +$(call force,CFG_WITH_PAGER,n,Pager is not supported for NPCM845x) +else +$(error Unsupported platform flavor "$(PLATFORM_FLAVOR)") +endif #npcm845x diff --git a/optee_os/core/arch/arm/plat-nuvoton/main.c b/optee_os/core/arch/arm/plat-nuvoton/main.c new file mode 100644 index 0000000..933fb2d --- /dev/null +++ b/optee_os/core/arch/arm/plat-nuvoton/main.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2023, Linaro Limited + * Copyright (c) 2014-2023, STMicroelectronics International N.V. + * Copyright (C) 2022-2023 Nuvoton Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define COLOR_NORMAL "\x1B[0m" +#define COLOR_RED "\x1B[31m" +#define COLOR_GREEN "\x1B[32m" +#define COLOR_YELLOW "\x1B[33m" +#define COLOR_BLUE "\x1B[34m" +#define COLOR_MAGENTA "\x1B[35m" +#define COLOR_CYAN "\x1B[36m" +#define COLOR_WHITE "\x1B[37m" + +#define NPCM_MEASURE_BASE 0xF0848000 +#define NPCM_MEASURE_DME 0x0E0 +#define NPCM_MEASURE_SIZE 64 + +static struct ns16550_data console_data __nex_bss; + +static struct { + uint8_t data[HW_UNIQUE_KEY_LENGTH]; + bool ready; +} npcm_hwkey; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, UART_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GIC_DIST_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_RAM_NSEC, NPCM_MEASURE_BASE, SMALL_PAGE_SIZE); + +register_ddr(DRAM0_BASE, DRAM0_SIZE); + +static void print_version(void) +{ + IMSG(COLOR_MAGENTA); + IMSG(">================================================"); + IMSG("OP-TEE OS Version %s", core_v_str); + IMSG(">================================================"); + IMSG(COLOR_NORMAL); +} + +void boot_primary_init_intc(void) +{ + if (IS_ENABLED(CFG_NPCM_DEBUG)) + print_version(); + + gic_init(GICC_BASE, GICD_BASE); +} + +void console_init(void) +{ + ns16550_init(&console_data, CONSOLE_UART_BASE, IO_WIDTH_U32, 2); + register_serial_console(&console_data.chip); +} + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + void *vaddr = NULL; + TEE_Result res = TEE_SUCCESS; + uint32_t bin[HW_UNIQUE_KEY_LENGTH / sizeof(uint32_t)] = {}; + uint8_t *bin_val = (uint8_t *)(&bin[0]); + + if (npcm_hwkey.ready) + goto out; + + vaddr = phys_to_virt(NPCM_MEASURE_BASE + NPCM_MEASURE_DME, + MEM_AREA_RAM_NSEC, NPCM_MEASURE_SIZE); + if (!vaddr) { + EMSG("Not enough memory mapped"); + return TEE_ERROR_SECURITY; + } + + res = tee_hash_createdigest(TEE_ALG_SHA256, (uint8_t *)vaddr, + NPCM_MEASURE_SIZE, bin_val, + HW_UNIQUE_KEY_LENGTH); + if (res != TEE_SUCCESS) { + EMSG("Can't create a digest for HUK"); + return TEE_ERROR_SECURITY; + } + + memcpy(&npcm_hwkey.data[0], bin, HW_UNIQUE_KEY_LENGTH); + npcm_hwkey.ready = true; + + IMSG("HUK Initialized"); + +out: + memcpy(hwkey->data, npcm_hwkey.data, HW_UNIQUE_KEY_LENGTH); + + return res; +} diff --git a/optee_os/core/arch/arm/plat-nuvoton/platform_config.h b/optee_os/core/arch/arm/plat-nuvoton/platform_config.h new file mode 100644 index 0000000..2d7dc77 --- /dev/null +++ b/optee_os/core/arch/arm/plat-nuvoton/platform_config.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2023, Linaro Limited + * Copyright (C) 2022-2023 Nuvoton Ltd. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 +#define DRAM0_BASE 0x00000000 +#define DRAM0_SIZE 0x40000000 /* 1G DDR */ +#define GIC_BASE 0xDFFF8000 +#define UART0_BASE 0xf0000000 +#define UART_REG_SIZE 0x100 +#define CONSOLE_UART_BASE UART0_BASE +#define GICD_OFFSET 0x1000 +#define GICC_OFFSET 0x2000 +#define GICD_BASE (GIC_BASE + GICD_OFFSET) +#define GICC_BASE (GIC_BASE + GICC_OFFSET) + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-nuvoton/sub.mk b/optee_os/core/arch/arm/plat-nuvoton/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-nuvoton/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-poplar/conf.mk b/optee_os/core/arch/arm/plat-poplar/conf.mk new file mode 100644 index 0000000..167cb40 --- /dev/null +++ b/optee_os/core/arch/arm/plat-poplar/conf.mk @@ -0,0 +1,29 @@ +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_TEE_CORE_NB_CORE,4) + +$(call force,CFG_PL011,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) + +ifeq ($(CFG_ARM64_core),y) +CFG_CORE_TZSRAM_EMUL_SIZE ?= 655360 +else +CFG_CORE_TZSRAM_EMUL_SIZE ?= 524288 +endif + +CFG_NUM_THREADS ?= 4 +CFG_CRYPTO_WITH_CE ?= y +# Overrides default in mk/config.mk with 96 kB +CFG_CORE_HEAP_SIZE ?= 98304 + +CFG_PL061 ?= y + +ifeq ($(CFG_PL061),y) +core-platform-cppflags += -DPLAT_PL061_MAX_GPIOS=104 +endif + +CFG_TEE_SDP_MEM_BASE ?= 0x02800000 +CFG_TEE_SDP_MEM_SIZE ?= 0x00400000 + +CFG_DRAM_SIZE_GB ?= 2 diff --git a/optee_os/core/arch/arm/plat-poplar/hi3798cv200.h b/optee_os/core/arch/arm/plat-poplar/hi3798cv200.h new file mode 100644 index 0000000..1b6b61e --- /dev/null +++ b/optee_os/core/arch/arm/plat-poplar/hi3798cv200.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2017, Linaro Limited + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef __HI3798cv200_H__ +#define __HI3798cv200_H__ + +/* PL011 */ +#define PL011_UART0_BASE (0xF8B00000) +#define PL011_BAUDRATE (115200) +#define PL011_UART0_CLK_IN_HZ (75000000) + +#endif /* __HI3798cv200_H__ */ diff --git a/optee_os/core/arch/arm/plat-poplar/main.c b/optee_os/core/arch/arm/plat-poplar/main.c new file mode 100644 index 0000000..ec244ce --- /dev/null +++ b/optee_os/core/arch/arm/plat-poplar/main.c @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2017, Linaro Limited + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#ifdef CFG_PL061 +#include +#endif +#include +#include +#include +#include +#include + +static struct pl011_data console_data; + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, PL011_REG_SIZE); +/* for dynamic shared memory */ +register_dynamic_shm(DRAM0_BASE_NSEC, DRAM0_SIZE_NSEC); + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-poplar/platform_config.h b/optee_os/core/arch/arm/plat-poplar/platform_config.h new file mode 100644 index 0000000..ff60d52 --- /dev/null +++ b/optee_os/core/arch/arm/plat-poplar/platform_config.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017, Linaro Limited + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +/* PL011 UART */ +#define CONSOLE_UART_BASE PL011_UART0_BASE +#define CONSOLE_BAUDRATE PL011_BAUDRATE +#define CONSOLE_UART_CLK_IN_HZ PL011_UART0_CLK_IN_HZ + +/* + * Poplar memory map + * + * Note: the physical address ranges below correspond to DRAM which is + * non-secure by default. Therefore, the terms TZDRAM and TZSRAM may not + * reflect the reality and only indicate areas that "would normally be" + * secure DRAM and secure SRAM in a more complete implementation. + * The memory map was defined like this for lack of better documentation. + * It is good enough for development/testing purposes. + * + * 0xFF00_0000 [DRAM2_LIMIT] + * other (devmem) + * 0xF000_0000 [DRAM2_BASE] + * + * 0x8000_0000 (0x4000_0000 for 1GB board) [DRAM0_LIMIT] + * u-boot + ree memory: 1144 MiB (144 MiB for 1GB board) + * 0x3700_0000 CONFIG_SYS_TEXT_BASE (u-boot) + * PLAT_POPLAR_NS_IMAGE_OFFSET (arm-tf) + * ramdisk: 76 MiB + * 0x3240_0000 + * fdt: 2 MiB + * 0x3220_0000 + * pxe file or script addr: 2 MiB + * 0x3200_0000 + * kernel/android: 32 MiB + * 0x3000_0000 + * ree memory: 696 MiB + * 0x0480_0000 CONFIG_SYS_LOAD_ADDR (defined in u-boot) + * other: 6 MiB + * 0x0420_0000 CONFIG_SYS_INIT_SP_ADDR (defined in u-boot) + * 0x0408_0000 KERNEL_TEXT_OFFSET (defined in u-boot) + * unused: 512 KiB + * 0x0400_0000 + * + * 0x0400_0000 - + * TA RAM: 14 MiB | TZDRAM + * 0x0320_0000 - + * + * CFG_WITH_PAGER=n - + * TEE RAM: 2 MiB (TEE_RAM_VA_SIZE) | TZDRAM + * 0x0300_0000 [TZDRAM_BASE, TEE_LOAD_ADDR] - + * + * CFG_WITH_PAGER=y + * Unused + * 0x030A_0000 - + * TEE RAM: 640 KiB (TZSRAM_SIZE) | TZSRAM + * 0x0300_0000 [TZSRAM_BASE, TEE_LOAD_ADDR] - + * + * 0x0300_0000 [TZDRAM_BASE, TZSRAM_BASE, TEE_LOAD_ADDR] + * OP-TEE Future Use: 4 MiB + * 0x02C0_0000 + * + * 0x02C0_0000 + * Secure Data Path buffers: 4 MiB + * 0x0280_0000 [CFG_TEE_SDP_MEM_BASE] + * Shared memory: 4 MiB + * 0x0240_0000 + * OP-TEE Future Use: 2 MiB + * 0x0220_0000 + * + * 0x0220_0000 + * unused: 64 KiB + * 0x021F_0000 l-loader limit (len/size set by poplar-l-loader.git) + * unused (cannot be used) + * 0x0210_0000 l-loader limit (max bootrom can accept) + * fip.bin load zone: 768 KiB + * 0x0204_0000 + * bl31: 80 KiB + * 0x0202_A000 + * bl2: 48 KiB + * 0x0201_E000 + * bl1: 64 KiB + * 0x0200_E000 + * l-loader text: 52 KiB + * 0x0200_1000 + * unused + * 0x0200_0000 + * TA virtual memory space + * 0x0000_0000 [DRAM0_BASE] + */ +#define DRAM0_BASE 0x00000000 +#if (CFG_DRAM_SIZE_GB == 2) +#define DRAM0_SIZE 0x80000000 +#elif (CFG_DRAM_SIZE_GB == 1) +#define DRAM0_SIZE 0x40000000 +#else +#error Unsupported DRAM size +#endif + +#define DRAM0_BASE_NSEC 0x04080000 +#define DRAM0_SIZE_NSEC (DRAM0_SIZE - DRAM0_BASE_NSEC) + +#define DRAM2_BASE 0xF0000000 +#define DRAM2_SIZE 0x0F000000 + +#ifdef CFG_WITH_PAGER + +#define TZSRAM_BASE 0x03000000 +#define TZSRAM_SIZE CFG_CORE_TZSRAM_EMUL_SIZE + +#define TZDRAM_BASE 0x03200000 +#define TZDRAM_SIZE (14 * 1024 * 1024) + +#define TEE_RAM_START TZSRAM_BASE +#define TEE_RAM_PH_SIZE TZSRAM_SIZE +#define TA_RAM_START ROUNDUP(TZDRAM_BASE, CORE_MMU_PGDIR_SIZE) +#define TA_RAM_SIZE ROUNDDOWN(TZDRAM_SIZE, CORE_MMU_PGDIR_SIZE) + +#else /* CFG_WITH_PAGER */ + +#define TZDRAM_BASE 0x03000000 +#define TZDRAM_SIZE (16 * 1024 * 1024) + +#define TEE_RAM_START TZDRAM_BASE +#define TEE_RAM_PH_SIZE TEE_RAM_VA_SIZE +#define TA_RAM_START ROUNDUP((TZDRAM_BASE + TEE_RAM_VA_SIZE), \ + CORE_MMU_PGDIR_SIZE) + +#define TA_RAM_SIZE ROUNDDOWN((TZDRAM_SIZE - TEE_RAM_VA_SIZE),\ + CORE_MMU_PGDIR_SIZE) + +#endif /* CFG_WITH_PAGER */ + +#define TEE_SHMEM_START 0x02400000 +#define TEE_SHMEM_SIZE (4 * 1024 * 1024) + +#define TEE_RAM_VA_SIZE (2 * 1024 * 1024) + +#define TEE_LOAD_ADDR 0x03000000 /* BL32_BASE */ + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-poplar/sub.mk b/optee_os/core/arch/arm/plat-poplar/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-poplar/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-rcar/conf.mk b/optee_os/core/arch/arm/plat-rcar/conf.mk new file mode 100644 index 0000000..0de4143 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/conf.mk @@ -0,0 +1,42 @@ +PLATFORM_FLAVOR ?= generic_dt + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_SCIF,y) +$(call force,CFG_GIC,y) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_WITH_LPAE,y) + +ifeq ($(PLATFORM_FLAVOR), spider_s4) +$(call force,CFG_RCAR_GEN4, y) +else +$(call force,CFG_RCAR_GEN3, y) +endif + +CFG_TZDRAM_START ?= 0x44100000 +CFG_TZDRAM_SIZE ?= 0x03D00000 +CFG_TEE_RAM_VA_SIZE ?= 0x100000 +supported-ta-targets = ta_arm64 + +ifeq ($(CFG_RCAR_GEN3), y) +CFG_WITH_SOFTWARE_PRNG ?= n +CFG_HWRNG_QUALITY ?= 1024 +CFG_HWRNG_PTA ?= y +CFG_DT ?= y +$(call force,CFG_RCAR_ROMAPI, y) +endif + +ifeq ($(CFG_RCAR_GEN4), y) +# 1xx - for SCIFxx +# 2xx - for HSCIFxx +CFG_RCAR_UART ?= 200 +$(call force,CFG_RCAR_ROMAPI, n) +$(call force,CFG_CORE_CLUSTER_SHIFT, 1) +$(call force,CFG_ARM_GICV3, y) +endif + diff --git a/optee_os/core/arch/arm/plat-rcar/core_pos_a64.S b/optee_os/core/arch/arm/plat-rcar/core_pos_a64.S new file mode 100644 index 0000000..ff25c85 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/core_pos_a64.S @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2019, Arm Limited. All rights reserved. + * Copyright (c) 2020, Marek Vasut + */ + +#include +#include +#include +#include +#include "rcar.h" + +FUNC get_core_pos_mpidr , : + /* + * Shift MPIDR value if it's not already shifted. + * Using logical shift ensures AFF0 to be filled with zeroes. + * This part is necessary even if CFG_CORE_THREAD_SHIFT is 0 because + * MT bit can be set on single threaded systems where all the AFF0 + * values are zeroes. + */ + tst x0, #MPIDR_MT_MASK + lsl x3, x0, #MPIDR_AFFINITY_BITS + csel x3, x3, x0, eq + + /* + * At this point the MPIDR layout is always shifted so it looks + * as follows AFF2 -> cluster, AFF1 -> core, AFF0 -> thread + */ + /* Calculate CorePos = (ClusterId * (cores/cluster)) + CoreId */ + ubfx x0, x3, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS + ubfx x1, x3, #MPIDR_AFF2_SHIFT, #MPIDR_AFFINITY_BITS + + /* + * R-Car M3W/M3W+ have 6 cores, but internally cluster 0 has two + * cores (0, 1) and cluster 1 has four cores (4, 5, 6, 7). Other + * R-Car SoCs either have two full clusters (4xCA57 + 4xCA53) or + * they have one cluster. + * + * The code below normalizes the M3W/M3W+ core enumeration such + * that cluster 0 returns core IDs {0, 1} and cluster 1 returns + * core IDs {2, 3, 4, 5}. This is achieved by calculating the + * core ID as CorePos = CoreId + (ClusterId << (IsM3W ? 1 : 0)) + */ + + adr_l x2, rcar_prr_value + ldr w3, [x2] + cmp w3, wzr + bne 1f + + /* Load PRR PRODUCT into x3 */ + mov x2, #PRR_BASE + ldr w3, [x2, #PRR_OFFSET] + /* + * Cache the PRR register value. PRR value does not change at runtime. + * This function is first called with MMU disabled, so it is possible + * to read the PRR register via its physical address, but once MMU is + * enabled, this is no longer possible as the virtual address is not + * available here, so in that case, use the cached value of the PRR. + */ + adr_l x2, rcar_prr_value + str w3, [x2] + +1: mov w2, #PRR_PRODUCT_M3W + and w3, w3, #PRR_PRODUCT_MASK + cmp w2, w3 + beq 2f /* if (!IsM3W) { x1 <<= 2; } else { x1 <<= 1} */ + lsl x1, x1, #1 +2: lsl x1, x1, #1 + add x0, x0, x1 + + ret +END_FUNC get_core_pos_mpidr diff --git a/optee_os/core/arch/arm/plat-rcar/hw_rng.c b/optee_os/core/arch/arm/plat-rcar/hw_rng.c new file mode 100644 index 0000000..b8496e8 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/hw_rng.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* Copyright (c) 2021, EPAM Systems. All rights reserved. */ + +#include +#include +#include +#include +#include +#include + +#include "romapi.h" + +#define SCRATCH_BUF_SZ 4096 + +static uint8_t scratch_buf[SCRATCH_BUF_SZ] __nex_bss + __aligned(RCAR_CACHE_LINE_SZ); +static unsigned int spin_lock __nex_data = SPINLOCK_UNLOCK; + +/* + * It is inefficient to call ROM_GetRndVector() every time we want 8 bits of + * random data, so we will cache the unused values for latter use. + */ +static uint8_t rng_cache[PLAT_RND_VECTOR_SZ] __nex_bss + __aligned(RCAR_CACHE_LINE_SZ); +static uint8_t rng_cache_pos __nex_data; + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + uint32_t exceptions; + uint8_t *buffer = buf; + size_t buffer_pos = 0; + uint8_t ret_val = 0; + + assert(rng_cache_pos < PLAT_RND_VECTOR_SZ); + + while (buffer_pos < len) { + exceptions = cpu_spin_lock_xsave(&spin_lock); + /* Refill our FIFO */ + if (rng_cache_pos == 0) { + uint32_t ret = plat_rom_getrndvector(rng_cache, + scratch_buf, + sizeof(scratch_buf)); + if (ret != 0) + panic("ROM_GetRndVector() returned error!"); + } + + buffer[buffer_pos++] = rng_cache[rng_cache_pos++]; + if (rng_cache_pos == PLAT_RND_VECTOR_SZ) + rng_cache_pos = 0; + + cpu_spin_unlock_xrestore(&spin_lock, exceptions); + } + + return ret_val; +} diff --git a/optee_os/core/arch/arm/plat-rcar/link.mk b/optee_os/core/arch/arm/plat-rcar/link.mk new file mode 100644 index 0000000..51fc371 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/link.mk @@ -0,0 +1,5 @@ +include core/arch/arm/kernel/link.mk + +SRECFLAGS ?= --srec-forceS3 --adjust-vma=$(CFG_TZDRAM_START) + +all: $(link-out-dir)/tee.srec diff --git a/optee_os/core/arch/arm/plat-rcar/main.c b/optee_os/core/arch/arm/plat-rcar/main.c new file mode 100644 index 0000000..9b443f9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/main.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, GlobalLogic + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, SCIF_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GIC_CPU_REG_SIZE); +#ifdef PRR_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, PRR_BASE, SMALL_PAGE_SIZE); +#endif + +/* Legacy platforms */ +#if defined(PLATFORM_FLAVOR_salvator_h3) || \ + defined(PLATFORM_FLAVOR_salvator_h3_4x2g) || \ + defined(PLATFORM_FLAVOR_salvator_m3) || \ + defined(PLATFORM_FLAVOR_salvator_m3_2x4g) || \ + defined(PLATFORM_FLAVOR_spider_s4) +register_ddr(NSEC_DDR_0_BASE, NSEC_DDR_0_SIZE); +register_ddr(NSEC_DDR_1_BASE, NSEC_DDR_1_SIZE); +#ifdef NSEC_DDR_2_BASE +register_ddr(NSEC_DDR_2_BASE, NSEC_DDR_2_SIZE); +#endif +#ifdef NSEC_DDR_3_BASE +register_ddr(NSEC_DDR_3_BASE, NSEC_DDR_3_SIZE); +#endif +#endif + +static struct scif_uart_data console_data __nex_bss; + +#ifdef PRR_BASE +uint32_t rcar_prr_value __nex_bss; +#endif + +void console_init(void) +{ + scif_uart_init(&console_data, CONSOLE_UART_BASE); + register_serial_console(&console_data.chip); +} + +#ifdef CFG_RCAR_ROMAPI +/* Should only seed from a hardware random number generator */ +static_assert(!IS_ENABLED(CFG_WITH_SOFTWARE_PRNG)); + +unsigned long plat_get_aslr_seed(void) +{ + unsigned long seed = 0; + + /* On RCAR we can get hw random bytes on early boot stages */ + if (crypto_rng_read(&seed, sizeof(seed))) + panic(); + + return seed; +} +#endif + +void boot_primary_init_intc(void) +{ + gic_init(GICC_BASE, GICD_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} diff --git a/optee_os/core/arch/arm/plat-rcar/platform_config.h b/optee_os/core/arch/arm/plat-rcar/platform_config.h new file mode 100644 index 0000000..f9d5d71 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/platform_config.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, GlobalLogic + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +#define RCAR_CACHE_LINE_SZ 64 + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT RCAR_CACHE_LINE_SZ + +#if defined(CFG_RCAR_GEN3) + +#define GIC_BASE 0xF1000000 +#define GICC_BASE 0xF1020000 +#define GICD_BASE 0xF1010000 + +#define CONSOLE_UART_BASE 0xE6E88000 + +#define PRR_BASE 0xFFF00000 + +#elif defined(CFG_RCAR_GEN4) + +#define GICC_BASE 0xF1060000 +#define GICD_BASE 0xF1000000 + +#if CFG_RCAR_UART == 103 /* SCIF3 */ +#define CONSOLE_UART_BASE 0xE6C50000 +#elif CFG_RCAR_UART == 200 /* HSCIF0 */ +#define CONSOLE_UART_BASE 0xE6540000 +#endif + +#endif /* CFG_RCAR_GENx */ + +#if defined(PLATFORM_FLAVOR_salvator_h3) +#define NSEC_DDR_0_BASE 0x47E00000 +#define NSEC_DDR_0_SIZE 0x38200000 +#define NSEC_DDR_1_BASE 0x500000000U +#define NSEC_DDR_1_SIZE 0x40000000 +#define NSEC_DDR_2_BASE 0x600000000U +#define NSEC_DDR_2_SIZE 0x40000000 +#define NSEC_DDR_3_BASE 0x700000000U +#define NSEC_DDR_3_SIZE 0x40000000 + +#elif defined(PLATFORM_FLAVOR_salvator_h3_4x2g) +#define NSEC_DDR_0_BASE 0x47E00000 +#define NSEC_DDR_0_SIZE 0x78200000 +#define NSEC_DDR_1_BASE 0x500000000U +#define NSEC_DDR_1_SIZE 0x80000000 +#define NSEC_DDR_2_BASE 0x600000000U +#define NSEC_DDR_2_SIZE 0x80000000 +#define NSEC_DDR_3_BASE 0x700000000U +#define NSEC_DDR_3_SIZE 0x80000000 + +#elif defined(PLATFORM_FLAVOR_salvator_m3) +#define NSEC_DDR_0_BASE 0x47E00000 +#define NSEC_DDR_0_SIZE 0x78200000 +#define NSEC_DDR_1_BASE 0x600000000U +#define NSEC_DDR_1_SIZE 0x80000000 + +#elif defined(PLATFORM_FLAVOR_salvator_m3_2x4g) +#define NSEC_DDR_0_BASE 0x47E00000 +#define NSEC_DDR_0_SIZE 0x78200000 +#define NSEC_DDR_1_BASE 0x480000000U +#define NSEC_DDR_1_SIZE 0x80000000 +#define NSEC_DDR_2_BASE 0x600000000U +#define NSEC_DDR_2_SIZE 0x100000000U + +#elif defined(PLATFORM_FLAVOR_spider_s4) +#define NSEC_DDR_0_BASE 0x48000000 +#define NSEC_DDR_0_SIZE 0x78000000 +#define NSEC_DDR_1_BASE 0x480000000U +#define NSEC_DDR_1_SIZE 0x80000000U + +#else + +/* Generic DT-based platform */ + +#endif + +/* Full GlobalPlatform test suite requires TEE_SHMEM_SIZE to be at least 2MB */ +#define TEE_SHMEM_START (TZDRAM_BASE + TZDRAM_SIZE) +#define TEE_SHMEM_SIZE 0x100000 + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-rcar/rcar.h b/optee_os/core/arch/arm/plat-rcar/rcar.h new file mode 100644 index 0000000..a14cbf9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/rcar.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2021, EPAM Systems. All rights reserved. */ + +#ifndef PLAT_RCAR_RCAR_H +#define PLAT_RCAR_RCAR_H + +#define PRR_OFFSET 0x44 +#define PRR_PRODUCT_H3 0x4F00 +#define PRR_PRODUCT_M3W 0x5200 +#define PRR_PRODUCT_MASK 0xFF00 +#define PRR_CUT_MASK 0xFF +#define PRR_CUT_10 0x00 /* Ver 1.0 */ +#define PRR_CUT_11 0x01 /* Ver 1.1 */ +#define PRR_CUT_20 0x10 /* Ver 2.0 */ +#define PRR_CUT_30 0x20 /* Ver.3.0 */ + +#ifndef __ASSEMBLER__ +extern uint32_t rcar_prr_value; +#endif + +#endif /* PLAT_RCAR_RCAR_H */ diff --git a/optee_os/core/arch/arm/plat-rcar/romapi.c b/optee_os/core/arch/arm/plat-rcar/romapi.c new file mode 100644 index 0000000..ad0a0d9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/romapi.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, EPAM Systems + */ +#include +#include +#include +#include + +#include "rcar.h" +#include "romapi.h" + +static int get_api_table_index(void) +{ + /* + * Depending on SoC type and version, there are 4 possible addresses + * for each ROMAPI function + */ + static int index __nex_data = -1; + + if (index != -1) + return index; + + switch (rcar_prr_value & PRR_PRODUCT_MASK) { + case PRR_PRODUCT_H3: + switch (rcar_prr_value & PRR_CUT_MASK) { + case PRR_CUT_10: /* H3 ES1.0 */ + case PRR_CUT_11: /* H3 ES1.1 */ + index = 0; + break; + case PRR_CUT_20: /* H3 ES2.0 */ + index = 1; + break; + default: /* Newer H3 versions use unified table */ + index = 3; + break; + } + break; + case PRR_PRODUCT_M3W: + switch (rcar_prr_value & PRR_CUT_MASK) { + case PRR_CUT_10: /* M3 ES1.0 */ + index = 2; + break; + default: /* Newer M3 versions use unified table */ + index = 3; + break; + } + break; + default: /* All other SoCs use unified table */ + index = 3; + break; + } + + return index; +} + +/* implemented in romapi_call.S */ +extern uint32_t __plat_romapi_wrapper(paddr_t func, uint64_t arg1, + uint64_t arg2, uint64_t arg3); + +static uint32_t __plat_romapi_direct(paddr_t func, uint64_t arg1, + uint64_t arg2, uint64_t arg3) +{ + uint32_t (*fptr)(uint64_t arg1, uint64_t arg2, uint64_t arg3) = NULL; + + assert(!cpu_mmu_enabled()); + + fptr = (typeof(fptr))func; + + return fptr(arg1, arg2, arg3); +} + +static uint32_t plat_call_romapi(paddr_t func, uint64_t arg1, + uint64_t arg2, uint64_t arg3) +{ + uint32_t (*fptr)(paddr_t func, uint64_t arg1, uint64_t arg2, + uint64_t arg3) = NULL; + + /* + * If MMU is enabled, we need to use trampoline function that will + * disable MMU and switch stack pointer to physical address. On other + * hand, if MMU is disabled, we can call the ROM function directly. + */ + if (cpu_mmu_enabled()) + /* + * With ASLR enabled __plat_romapi_wrapper() function will be + * mapped at two addresses: at random address (with the rest of + * OP-TEE) and at identity address. We need to map it at + * identity address and call it at identity address because this + * function turns off MMU to perform ROM API call. But + * __plat_romapi_wrapper *symbol* will be relocated by ASLR + * code. To get identity address of the function we need to use + * virt_to_phys(). + */ + fptr = (void *)virt_to_phys(__plat_romapi_wrapper); + else + /* + * With MMU disabled we can call ROM code directly. + */ + fptr = __plat_romapi_direct; + + return fptr(func, arg1, arg2, arg3); +} + +static paddr_t va2pa(void *ptr) +{ + if (cpu_mmu_enabled()) + return virt_to_phys(ptr); + else + return (paddr_t)ptr; +} + +static const paddr_t romapi_getrndvector[] = { + 0xEB10DFC4, /* H3 1.0/1.1, needs confirmation */ + 0xEB117134, /* H3 2.0 */ + 0xEB11055C, /* M3 1.0/1.05, needs confirmation */ + 0xEB100188, /* H3 3.0, M3 1.1+, M3N, E3, D3, V3M 2.0 */ +}; + +uint32_t plat_rom_getrndvector(uint8_t rndbuff[PLAT_RND_VECTOR_SZ], + uint8_t *scratch, uint32_t scratch_sz) +{ + uint32_t ret = -1; + paddr_t func_addr = romapi_getrndvector[get_api_table_index()]; + paddr_t rndbuff_pa = va2pa(rndbuff); + paddr_t scratch_pa = va2pa(scratch); + + assert(scratch_sz >= 4096); + assert(rndbuff_pa % RCAR_CACHE_LINE_SZ == 0); + assert(scratch_pa % RCAR_CACHE_LINE_SZ == 0); + + ret = plat_call_romapi(func_addr, rndbuff_pa, scratch_pa, scratch_sz); + + /* + * ROM code is called with MMU turned off, so any accesses to rndbuff + * are not affected by data cache. This can lead to two problems: + * + * 1. Any prior writes can be cached but may not reach memory. So staled + * values can be flushed to memory later and overwrite new data written + * by ROM code. This includes stack as well. + * + * 2. ROM code will write new data to the buffer, but we may see old, + * cached values. + * + * ROM code wrapper will issue dcache_op_all(DCACHE_OP_CLEAN). This will + * ensure that all writes reached memory. After the call we need to + * invalidate the cache to see new data. + * + * We are not accessing scratch area, so no need to do cache maintenance + * for that buffer. + */ + cache_op_inner(DCACHE_AREA_INVALIDATE, rndbuff, PLAT_RND_VECTOR_SZ); + + return ret; +} diff --git a/optee_os/core/arch/arm/plat-rcar/romapi.h b/optee_os/core/arch/arm/plat-rcar/romapi.h new file mode 100644 index 0000000..8efa0eb --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/romapi.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, EPAM Systems + */ + +#ifndef __ROMAPI_H +#define __ROMAPI_H + +#include + +/* + * Mask ROM provides number of facilities, including function that returns 32 + * byte random vector. + */ +#define PLAT_RND_VECTOR_SZ 32 + +/* + * Call to this function must be protected by a spinlock, because ROM code + * accesses hardware. This function requires at least 4kb scratch buffer to + * work. All parameters should be aligned to 8 bytes. + */ +uint32_t plat_rom_getrndvector(uint8_t rndbuff[PLAT_RND_VECTOR_SZ], + uint8_t *scratch, + uint32_t scratch_sz); + +#endif diff --git a/optee_os/core/arch/arm/plat-rcar/romapi_call.S b/optee_os/core/arch/arm/plat-rcar/romapi_call.S new file mode 100644 index 0000000..975eddb --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/romapi_call.S @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, EPAM Systems + */ + +#include +#include +#include +#include +#include + +/* uint32_t __plat_romapi_wrapper(paddr_t func, uint64_t arg1, uint64_t arg2, + * uint64_t arg3) + * Call MaskROM function func(arg1, arg2, arg3). + * We need to disable MMU before calling any MaskROM API functions + */ +FUNC __plat_romapi_wrapper , : , .identity_map + + push fp, lr + push x19, x20 + push x21, x22 + push x23, x24 + + mov x19, x0 + mov x20, x1 + mov x21, x2 + mov x22, x3 + + /* Get PA of stack pointer */ + mov x0, sp +#ifdef CFG_CORE_ASLR + /* + * We are running at identity location, so we can't use bl there, + * because assembler will generate relative address to virt_to_phys(), + * which is not identity mapped. + */ + adr_l x9, virt_to_phys + ldr x10, boot_mmu_config + CORE_MMU_CONFIG_MAP_OFFSET + add x9, x9, x10 + blr x9 +#else + bl virt_to_phys +#endif + mov x23, x0 + + /* We about to disable MMU. Make sure that all writes reached memory */ + mov x0, #DCACHE_OP_CLEAN +#ifdef CFG_CORE_ASLR + /* See the comment above */ + adr_l x9, dcache_op_all + ldr x10, boot_mmu_config + CORE_MMU_CONFIG_MAP_OFFSET + add x9, x9, x10 + blr x9 +#else + bl dcache_op_all +#endif + + /* Disable MMU */ + mrs x9, sctlr_el1 + bic x9, x9, #SCTLR_M + bic x9, x9, #SCTLR_C + msr sctlr_el1, x9 + isb + /* Invalidate instruction cache and branch predictor */ + ic ialluis + dsb ish /* ensure that maintenance operations are seen */ + isb + + /* Save old SP to x24 and switch to a new stack */ + mov x24, sp + mov sp, x23 + + /* call the function */ + mov x0, x20 /* x20: uint64_t arg1 */ + mov x1, x21 /* x21: uint64_t arg2 */ + mov x2, x22 /* x22: uint64_t arg3 */ + blr x19 /* x19: paddr_t func */ + + /* restore sp */ + mov sp, x24 + + /* Enable MMU */ + mrs x9, sctlr_el1 + orr x9, x9, #SCTLR_M + orr x9, x9, #SCTLR_C + msr sctlr_el1, x9 + isb + + /* Invalidate instruction cache and branch predictor */ + ic iallu + isb + + pop x23, x24 + pop x21, x22 + pop x19, x20 + pop fp, lr + ret +END_FUNC __plat_romapi_wrapper diff --git a/optee_os/core/arch/arm/plat-rcar/sub.mk b/optee_os/core/arch/arm/plat-rcar/sub.mk new file mode 100644 index 0000000..2cbed17 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rcar/sub.mk @@ -0,0 +1,6 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-${CFG_RCAR_GEN3} += core_pos_a64.S +srcs-${CFG_RCAR_ROMAPI} += romapi.c +srcs-${CFG_RCAR_ROMAPI} += romapi_call.S +srcs-${CFG_RCAR_ROMAPI} += hw_rng.c diff --git a/optee_os/core/arch/arm/plat-rockchip/common.h b/optee_os/core/arch/arm/plat-rockchip/common.h new file mode 100644 index 0000000..7111891 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/common.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + */ + +#ifndef PLAT_ROCKCHIP_COMMON_H +#define PLAT_ROCKCHIP_COMMON_H + +/* For SMP cpu bootup, they are common for rockchip platforms */ +#define LOCK_TAG 0xDEADBEAF +#define LOCK_ADDR_OFFSET 4 +#define BOOT_ADDR_OFFSET 8 + +/* + * Some register has write-mask bits, it means if you want to set the bits, + * you need set the write-mask bits at the same time, the write-mask bits is + * in high 16-bits. The following macro definition helps you access register + * efficiently. + */ +#define REG_MSK_SHIFT 16 +#define WMSK_BIT(nr) BIT((nr) + REG_MSK_SHIFT) +#define BIT_WITH_WMSK(nr) (BIT(nr) | WMSK_BIT(nr)) +#define BITS_WMSK(msk, shift) SHIFT_U32(msk, (shift) + REG_MSK_SHIFT) +#define BITS_WITH_WMASK(bits, msk, shift) \ + (SHIFT_U32(bits, shift) | BITS_WMSK(msk, shift)) + +#endif diff --git a/optee_os/core/arch/arm/plat-rockchip/conf.mk b/optee_os/core/arch/arm/plat-rockchip/conf.mk new file mode 100644 index 0000000..e8e9f4c --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/conf.mk @@ -0,0 +1,63 @@ +PLATFORM_FLAVOR ?= rk322x + +$(call force,CFG_GIC,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_8250_UART,y) + +CFG_DT ?= y +CFG_WITH_STATS ?= y +CFG_NUM_THREADS ?= 4 + +ifeq ($(PLATFORM_FLAVOR),rk322x) +include ./core/arch/arm/cpu/cortex-a7.mk +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_PSCI_ARM32,y) +$(call force,CFG_BOOT_SECONDARY_REQUEST,y) + +CFG_TZDRAM_START ?= 0x68400000 +CFG_TZDRAM_SIZE ?= 0x00200000 +CFG_SHMEM_START ?= 0x68600000 +CFG_SHMEM_SIZE ?= 0x00100000 + +CFG_EARLY_CONSOLE ?= y +CFG_EARLY_CONSOLE_BASE ?= UART2_BASE +CFG_EARLY_CONSOLE_SIZE ?= UART2_SIZE +CFG_EARLY_CONSOLE_BAUDRATE ?= 1500000 +CFG_EARLY_CONSOLE_CLK_IN_HZ ?= 24000000 +endif + +ifeq ($(PLATFORM_FLAVOR),rk3399) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,6) +$(call force,CFG_ARM_GICV3,y) +CFG_CRYPTO_WITH_CE ?= y + +CFG_TZDRAM_START ?= 0x30000000 +CFG_TZDRAM_SIZE ?= 0x02000000 +CFG_SHMEM_START ?= 0x32000000 +CFG_SHMEM_SIZE ?= 0x00400000 + +CFG_EARLY_CONSOLE ?= y +CFG_EARLY_CONSOLE_BASE ?= UART2_BASE +CFG_EARLY_CONSOLE_SIZE ?= UART2_SIZE +CFG_EARLY_CONSOLE_BAUDRATE ?= 1500000 +CFG_EARLY_CONSOLE_CLK_IN_HZ ?= 24000000 +endif + +ifeq ($(PLATFORM_FLAVOR),px30) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,4) + +CFG_TZDRAM_START ?= 0x30000000 +CFG_TZDRAM_SIZE ?= 0x02000000 +CFG_SHMEM_START ?= 0x32000000 +CFG_SHMEM_SIZE ?= 0x00400000 + +CFG_EARLY_CONSOLE ?= n +endif + +ifeq ($(platform-flavor-armv8),1) +$(call force,CFG_ARM64_core,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +ta-targets = ta_arm64 +endif diff --git a/optee_os/core/arch/arm/plat-rockchip/core_pos_a32.S b/optee_os/core/arch/arm/plat-rockchip/core_pos_a32.S new file mode 100644 index 0000000..5994c46 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/core_pos_a32.S @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include + +FUNC get_core_pos_mpidr , : + /* + * Because mpidr is designed mistake in hardware, ie. core0 is 0xf00, + * core1 is 0xf01..., so we need implement the function to correct this. + */ + and r0, r0, #MPIDR_CPU_MASK + bx lr +END_FUNC get_core_pos_mpidr + diff --git a/optee_os/core/arch/arm/plat-rockchip/cru.h b/optee_os/core/arch/arm/plat-rockchip/cru.h new file mode 100644 index 0000000..afbf387 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/cru.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + */ + +#ifndef PLAT_ROCKCHIP_CRU_H +#define PLAT_ROCKCHIP_CRU_H + +#include +#include + +#if defined(PLATFORM_FLAVOR_rk322x) + +enum plls_id { + APLL_ID, + DPLL_ID, + CPLL_ID, + GPLL_ID, + PLL_END, +}; + +#define CRU_SOFTRST_CON(i) (0x110 + ((i) * 4)) +#define CRU_MODE_CON 0x040 +#define CRU_GLBRST_CFG_BASE 0x140 +#define CRU_FSTRST_VAL_BASE 0x1f0 +#define CRU_SNDRST_VAL_BASE 0x1f4 +#define CRU_FSTRST_VAL 0xfdb9 +#define CRU_SNDRST_VAL 0xeca8 +#define PLLS_SLOW_MODE 0x11030000 + +#define CORE_SOFT_RESET(core) SHIFT_U32(0x100010, (core)) +#define CORE_SOFT_RELEASE(core) SHIFT_U32(0x100000, (core)) +#define CORE_HELD_IN_RESET(core) SHIFT_U32(0x000010, (core)) +#define NONBOOT_CORES_SOFT_RESET 0x00e000e0 + +#define CRU_CLKGATE_CON_CNT 16 +#define CRU_CLKSEL_CON(i) (0x044 + ((i) * 4)) +#define CRU_CLKGATE_CON(i) (0x0d0 + ((i) * 4)) +#define CRU_PLL_CON0(pll) ((pll) * 0x0c + 0x0) +#define CRU_PLL_CON1(pll) ((pll) * 0x0c + 0x4) +#define CRU_PLL_CON2(pll) ((pll) * 0x0c + 0x8) + +#define PLL_LOCK BIT(10) +#define PLL_POWER_UP BITS_WITH_WMASK(0, 1, 13) +#define PLL_POWER_DOWN BITS_WITH_WMASK(1, 1, 13) + +#define PLL_MODE_BIT(pll) ((pll) * 4) +#define PLL_MODE_MSK(pll) BIT(PLL_MODE_BIT(pll)) +#define PLL_SLOW_MODE(pll) BITS_WITH_WMASK(0, 1, PLL_MODE_BIT(pll)) +#define PLL_NORM_MODE(pll) BITS_WITH_WMASK(1, 1, PLL_MODE_BIT(pll)) +#endif + +#endif diff --git a/optee_os/core/arch/arm/plat-rockchip/grf.h b/optee_os/core/arch/arm/plat-rockchip/grf.h new file mode 100644 index 0000000..40caae6 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/grf.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + */ + +#ifndef PLAT_ROCKCHIP_GRF_H +#define PLAT_ROCKCHIP_GRF_H + +#if defined(PLATFORM_FLAVOR_rk322x) +#define GRF_CPU_STATUS1 0x524 + +#define CORE_WFE_MASK(core) SHIFT_U32(0x02, (core)) +#define CORE_WFI_MASK(core) SHIFT_U32(0x20, (core)) +#define CORE_WFE_I_MASK(core) (CORE_WFI_MASK(core) | CORE_WFE_MASK(core)) +#endif + +#endif diff --git a/optee_os/core/arch/arm/plat-rockchip/main.c b/optee_os/core/arch/arm/plat-rockchip/main.c new file mode 100644 index 0000000..b7501dd --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/main.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + * Copyright (C) 2019, Theobroma Systems Design und Consulting GmbH + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CFG_EARLY_CONSOLE) +static struct serial8250_uart_data early_console_data; +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, + CFG_EARLY_CONSOLE_BASE, CFG_EARLY_CONSOLE_SIZE); +#endif + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE, GIC_SIZE); + +void boot_primary_init_intc(void) +{ + gic_init(GICC_BASE, GICD_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +void console_init(void) +{ +#if defined(CFG_EARLY_CONSOLE) + /* + * Console devices can vary a lot between devices and + * OP-TEE will switch to the DT-based real console later, + * based on DT-devices and the systems chosen node. + * So early console is only needed for early debugging. + */ + serial8250_uart_init(&early_console_data, + CFG_EARLY_CONSOLE_BASE, + CFG_EARLY_CONSOLE_CLK_IN_HZ, + CFG_EARLY_CONSOLE_BAUDRATE); + register_serial_console(&early_console_data.chip); +#endif +} diff --git a/optee_os/core/arch/arm/plat-rockchip/plat_init.S b/optee_os/core/arch/arm/plat-rockchip/plat_init.S new file mode 100644 index 0000000..fb084af --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/plat_init.S @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + */ + +#include +#include +#include + +FUNC plat_cpu_reset_early , : + + /* Enable SMP bit */ + read_actlr r0 + orr r0, r0, #ACTLR_SMP + write_actlr r0 + bx lr +END_FUNC plat_cpu_reset_early + diff --git a/optee_os/core/arch/arm/plat-rockchip/platform.c b/optee_os/core/arch/arm/plat-rockchip/platform.c new file mode 100644 index 0000000..96c343f --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/platform.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include + +int __weak platform_secure_init(void) +{ + return 0; +} + +int __weak platform_secure_ddr_region(int rgn __maybe_unused, + paddr_t st __maybe_unused, + size_t sz __maybe_unused) +{ + MSG("Not protecting region %d: 0x%lx-0x%lx\n", rgn, st, st + sz); + + return 0; +} + +static TEE_Result platform_init(void) +{ + int ret = 0; + + platform_secure_init(); + + /* + * Rockchip SoCs can protect multiple memory regions (mostly 8). + * Region 0 is assigned for Trusted-Firmware memory, so use + * regions 1 for OP-TEE memory, which leaves on all known SoCs + * at least 6 more regions available for other purposes. + */ + ret = platform_secure_ddr_region(1, CFG_TZDRAM_START, CFG_TZDRAM_SIZE); + if (ret < 0) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +service_init(platform_init); diff --git a/optee_os/core/arch/arm/plat-rockchip/platform.h b/optee_os/core/arch/arm/plat-rockchip/platform.h new file mode 100644 index 0000000..7b79b86 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/platform.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Theobroma Systems Design und Consulting GmbH + */ + +#ifndef PLAT_ROCKCHIP_PLATFORM_H +#define PLAT_ROCKCHIP_PLATFORM_H + +int platform_secure_init(void); +int platform_secure_ddr_region(int rgn, paddr_t st, size_t sz); + +#endif diff --git a/optee_os/core/arch/arm/plat-rockchip/platform_config.h b/optee_os/core/arch/arm/plat-rockchip/platform_config.h new file mode 100644 index 0000000..31439e2 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/platform_config.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + * Copyright (C) 2019, Theobroma Systems Design und Consulting GmbH + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#define SIZE_K(n) ((n) * 1024) +#define SIZE_M(n) ((n) * 1024 * 1024) + +#if defined(PLATFORM_FLAVOR_rk322x) + +#define GIC_BASE 0x32010000 +#define GIC_SIZE SIZE_K(64) +#define GICD_BASE (GIC_BASE + 0x1000) +#define GICC_BASE (GIC_BASE + 0x2000) + +#define SGRF_BASE 0x10140000 +#define SGRF_SIZE SIZE_K(64) + +#define DDRSGRF_BASE 0x10150000 +#define DDRSGRF_SIZE SIZE_K(64) + +#define GRF_BASE 0x11000000 +#define GRF_SIZE SIZE_K(64) + +#define UART2_BASE 0x11030000 +#define UART2_SIZE SIZE_K(64) + +#define CRU_BASE 0x110e0000 +#define CRU_SIZE SIZE_K(64) + +/* Internal SRAM */ +#define ISRAM_BASE 0x10080000 +#define ISRAM_SIZE SIZE_K(8) + +#elif defined(PLATFORM_FLAVOR_rk3399) + +#define MMIO_BASE 0xF8000000 + +#define GIC_BASE (MMIO_BASE + 0x06E00000) +#define GIC_SIZE SIZE_M(2) +#define GICC_BASE (MMIO_BASE + 0x07F00000) +#define GICD_BASE GIC_BASE +#define GICR_BASE (GIC_BASE + SIZE_M(1)) + +#define UART0_BASE (MMIO_BASE + 0x07180000) +#define UART0_SIZE SIZE_K(64) + +#define UART1_BASE (MMIO_BASE + 0x07190000) +#define UART1_SIZE SIZE_K(64) + +#define UART2_BASE (MMIO_BASE + 0x071A0000) +#define UART2_SIZE SIZE_K(64) + +#define UART3_BASE (MMIO_BASE + 0x071B0000) +#define UART3_SIZE SIZE_K(64) + +#define SGRF_BASE (MMIO_BASE + 0x07330000) +#define SGRF_SIZE SIZE_K(64) + +#elif defined(PLATFORM_FLAVOR_px30) + +#define GIC_BASE 0xff130000 +#define GIC_SIZE SIZE_K(64) +#define GICD_BASE (GIC_BASE + 0x1000) +#define GICC_BASE (GIC_BASE + 0x2000) + +#define UART1_BASE 0xff158000 +#define UART1_SIZE SIZE_K(64) + +#define UART2_BASE 0xff160000 +#define UART2_SIZE SIZE_K(64) + +#define UART5_BASE 0xff178000 +#define UART5_SIZE SIZE_K(64) + +#define FIREWALL_DDR_BASE 0xff534000 +#define FIREWALL_DDR_SIZE SIZE_K(16) + +#else +#error "Unknown platform flavor" +#endif + +#ifdef CFG_WITH_LPAE +#define MAX_XLAT_TABLES 5 +#endif + +#endif diff --git a/optee_os/core/arch/arm/plat-rockchip/platform_px30.c b/optee_os/core/arch/arm/plat-rockchip/platform_px30.c new file mode 100644 index 0000000..8327ca9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/platform_px30.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Theobroma Systems Design und Consulting GmbH + */ + +#include +#include +#include +#include +#include +#include + +#define FIREWALL_DDR_FW_DDR_RGN(i) ((i) * 0x4) +#define FIREWALL_DDR_FW_DDR_MST(i) (0x20 + (i) * 0x4) +#define FIREWALL_DDR_FW_DDR_CON_REG 0x40 +#define FIREWALL_DDR_FW_DDR_RGN_NUM 8 +#define FIREWALL_DDR_FW_DDR_MST_NUM 6 + +#define RG_MAP_SECURE(top, base) ((((top) - 1) << 16) | (base)) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, FIREWALL_DDR_BASE, FIREWALL_DDR_SIZE); + +int platform_secure_ddr_region(int rgn, paddr_t st, size_t sz) +{ + vaddr_t fw_base = (vaddr_t)phys_to_virt_io(FIREWALL_DDR_BASE, + FIREWALL_DDR_SIZE); + paddr_t ed = st + sz; + uint32_t st_mb = st / SIZE_M(1); + uint32_t ed_mb = ed / SIZE_M(1); + + if (!fw_base) + panic(); + + assert(rgn <= 7); + assert(st < ed); + + /* Check aligned 1MB */ + assert(st % SIZE_M(1) == 0); + assert(ed % SIZE_M(1) == 0); + + DMSG("protecting region %d: 0x%lx-0x%lx\n", rgn, st, ed); + + /* Map top and base */ + io_write32(fw_base + FIREWALL_DDR_FW_DDR_RGN(rgn), + RG_MAP_SECURE(ed_mb, st_mb)); + + /* Enable secure setting */ + io_setbits32(fw_base + FIREWALL_DDR_FW_DDR_CON_REG, BIT(rgn)); + + return 0; +} diff --git a/optee_os/core/arch/arm/plat-rockchip/platform_rk322x.c b/optee_os/core/arch/arm/plat-rockchip/platform_rk322x.c new file mode 100644 index 0000000..9b530b6 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/platform_rk322x.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + * Copyright (C) 2019, Theobroma Systems Design und Consulting GmbH + */ + +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SGRF_BASE, SGRF_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, DDRSGRF_BASE, DDRSGRF_SIZE); + +#define SGRF_SOC_CON(n) ((n) * 4) +#define DDR_SGRF_DDR_CON(n) ((n) * 4) +#define DDR_RGN0_NS BIT32(30) +#define SLAVE_ALL_NS GENMASK_32(31, 16) + +int platform_secure_init(void) +{ + vaddr_t sgrf_base = (vaddr_t)phys_to_virt_io(SGRF_BASE, SGRF_SIZE); + vaddr_t ddrsgrf_base = (vaddr_t)phys_to_virt_io(DDRSGRF_BASE, + DDRSGRF_SIZE); + + /* Set rgn0 non-secure */ + io_write32(ddrsgrf_base + DDR_SGRF_DDR_CON(0), DDR_RGN0_NS); + + /* Initialize all slave non-secure */ + io_write32(sgrf_base + SGRF_SOC_CON(7), SLAVE_ALL_NS); + io_write32(sgrf_base + SGRF_SOC_CON(8), SLAVE_ALL_NS); + io_write32(sgrf_base + SGRF_SOC_CON(9), SLAVE_ALL_NS); + io_write32(sgrf_base + SGRF_SOC_CON(10), SLAVE_ALL_NS); + + return 0; +} diff --git a/optee_os/core/arch/arm/plat-rockchip/platform_rk3399.c b/optee_os/core/arch/arm/plat-rockchip/platform_rk3399.c new file mode 100644 index 0000000..053f566 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/platform_rk3399.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Theobroma Systems Design und Consulting GmbH + */ + +#include +#include +#include +#include +#include +#include + +#define SGRF_DDRRGN_CON0_16(n) ((n) * 4) +#define SGRF_DDR_RGN_0_16_WMSK GENMASK_32(11, 0) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SGRF_BASE, SGRF_SIZE); + +int platform_secure_ddr_region(int rgn, paddr_t st, size_t sz) +{ + vaddr_t sgrf_base = (vaddr_t)phys_to_virt_io(SGRF_BASE, SGRF_SIZE); + paddr_t ed = st + sz; + uint32_t st_mb = st / SIZE_M(1); + uint32_t ed_mb = ed / SIZE_M(1); + + if (!sgrf_base) + panic(); + + assert(rgn <= 7); + assert(st < ed); + + /* Check aligned 1MB */ + assert(st % SIZE_M(1) == 0); + assert(ed % SIZE_M(1) == 0); + + DMSG("protecting region %d: 0x%lx-0x%lx\n", rgn, st, ed); + + /* Set ddr region addr start */ + io_write32(sgrf_base + SGRF_DDRRGN_CON0_16(rgn), + BITS_WITH_WMASK(st_mb, SGRF_DDR_RGN_0_16_WMSK, 0)); + + /* Set ddr region addr end */ + io_write32(sgrf_base + SGRF_DDRRGN_CON0_16(rgn + 8), + BITS_WITH_WMASK((ed_mb - 1), SGRF_DDR_RGN_0_16_WMSK, 0)); + + io_write32(sgrf_base + SGRF_DDRRGN_CON0_16(16), + BIT_WITH_WMSK(rgn)); + + return 0; +} diff --git a/optee_os/core/arch/arm/plat-rockchip/psci_rk322x.c b/optee_os/core/arch/arm/plat-rockchip/psci_rk322x.c new file mode 100644 index 0000000..07feae8 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/psci_rk322x.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct dram_data { + uint32_t cru_mode_con; + uint32_t cru_clksel0; + uint32_t cru_clksel1; + uint32_t cru_clksel10; + uint32_t cru_clksel21; + uint32_t cru_clkgate[CRU_CLKGATE_CON_CNT]; +}; + +static struct dram_data dram_d; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CRU_BASE, CRU_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GRF_BASE, GRF_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, ISRAM_BASE, ISRAM_SIZE); + +static const uint32_t clks_gating_table[CRU_CLKGATE_CON_CNT] = { + /* gate: 0-3 */ + 0xefb8, + 0x0ff7, + 0xfff4, + 0x887f, + /* gate: 4-7 */ + 0x0030, + 0x00f8, + 0x07e0, + 0xc000, + /* gate: 8-11 */ + 0xff84, + 0xb047, + 0x1ca0, + 0x57ff, + /* gate: 12-15 */ + 0x0000, + 0x00ff, + 0x1cc0, + 0x000f, +}; + +static void clks_disable(void) +{ + uint32_t i; + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + for (i = 0; i < CRU_CLKGATE_CON_CNT; i++) { + dram_d.cru_clkgate[i] = io_read32(va_base + CRU_CLKGATE_CON(i)); + io_write32(va_base + CRU_CLKGATE_CON(i), + BITS_WITH_WMASK(clks_gating_table[i], 0xffff, 0)); + } +} + +static void clks_restore(void) +{ + uint32_t i; + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + for (i = 0; i < CRU_CLKGATE_CON_CNT; i++) + io_write32(va_base + CRU_CLKGATE_CON(i), + BITS_WITH_WMASK(dram_d.cru_clkgate[i], 0xffff, 0)); +} + +static void pll_power_down(uint32_t pll) +{ + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + io_write32(va_base + CRU_MODE_CON, PLL_SLOW_MODE(pll)); + io_write32(va_base + CRU_PLL_CON1(pll), PLL_POWER_DOWN); +} + +static void pll_power_up(uint32_t pll) +{ + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + io_write32(va_base + CRU_PLL_CON1(pll), PLL_POWER_UP); +} + +static void pll_wait_lock(uint32_t pll) +{ + uint32_t loop = 0; + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + while (!(io_read32(va_base + CRU_PLL_CON1(pll)) & PLL_LOCK) && + (loop < 500)) { + udelay(2); + loop++; + } + + if (!(io_read32(va_base + CRU_PLL_CON1(pll)) & PLL_LOCK)) { + EMSG("PLL can't lock, index = %" PRIu32, pll); + panic(); + } +} + +/* + * Select clock from external 24MHz OSC(slow mode) and power down plls, + * then set frequency division of relevant bus to 24MHz. + */ +static void plls_power_down(void) +{ + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + dram_d.cru_clksel0 = io_read32(va_base + CRU_CLKSEL_CON(0)); + dram_d.cru_clksel1 = io_read32(va_base + CRU_CLKSEL_CON(1)); + dram_d.cru_clksel10 = io_read32(va_base + CRU_CLKSEL_CON(10)); + dram_d.cru_clksel21 = io_read32(va_base + CRU_CLKSEL_CON(21)); + dram_d.cru_mode_con = io_read32(va_base + CRU_MODE_CON); + + pll_power_down(GPLL_ID); + pll_power_down(CPLL_ID); + pll_power_down(APLL_ID); + + /* core */ + io_write32(va_base + CRU_CLKSEL_CON(0), BITS_WITH_WMASK(0, 0x1f, 0)); + io_write32(va_base + CRU_CLKSEL_CON(1), + BITS_WITH_WMASK(0, 0xf, 0) | BITS_WITH_WMASK(0, 0x7, 4)); + + /* peri aclk, hclk, pclk */ + io_write32(va_base + CRU_CLKSEL_CON(10), + BITS_WITH_WMASK(0, 0x1f, 0) | BITS_WITH_WMASK(0, 0x3, 8) | + BITS_WITH_WMASK(0, 0x7, 12)); + + /* pdbus */ + io_write32(va_base + CRU_CLKSEL_CON(0), BITS_WITH_WMASK(0, 0x1f, 8)); + io_write32(va_base + CRU_CLKSEL_CON(1), + BITS_WITH_WMASK(0, 0x3, 8) | BITS_WITH_WMASK(0, 0x7, 12)); + + /* hdmi cec 32k */ + io_write32(va_base + CRU_CLKSEL_CON(21), + BITS_WITH_WMASK(732, 0x3fff, 0) | + BITS_WITH_WMASK(2, 0x3, 14)); +} + +static void plls_restore(void) +{ + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + /* power up plls */ + pll_power_up(APLL_ID); + pll_power_up(GPLL_ID); + pll_power_up(CPLL_ID); + + udelay(200); + + /* wait lock*/ + pll_wait_lock(APLL_ID); + pll_wait_lock(GPLL_ID); + pll_wait_lock(CPLL_ID); + + /* hdmi cec 32k */ + io_write32(va_base + CRU_CLKSEL_CON(21), + dram_d.cru_clksel21 | BITS_WMSK(0x3fff, 0) | + BITS_WMSK(0x3, 14)); + + /* pdbus */ + io_write32(va_base + CRU_CLKSEL_CON(0), + dram_d.cru_clksel0 | BITS_WMSK(0x1f, 8)); + io_write32(va_base + CRU_CLKSEL_CON(1), + dram_d.cru_clksel1 | BITS_WMSK(0x3, 8) | BITS_WMSK(0x7, 12)); + + /* peri aclk, hclk, pclk */ + io_write32(va_base + CRU_CLKSEL_CON(10), + dram_d.cru_clksel10 | BITS_WMSK(0x1f, 0) | + BITS_WMSK(0x3, 8) | BITS_WMSK(0x7, 12)); + + /* core */ + io_write32(va_base + CRU_CLKSEL_CON(0), + dram_d.cru_clksel0 | BITS_WMSK(0x1f, 0)); + io_write32(va_base + CRU_CLKSEL_CON(1), + dram_d.cru_clksel1 | BITS_WMSK(0xf, 0) | BITS_WMSK(0x7, 4)); + + /* resume plls mode */ + io_write32(va_base + CRU_MODE_CON, + dram_d.cru_mode_con | BITS_WMSK(0x1, PLL_MODE_BIT(APLL_ID))); + io_write32(va_base + CRU_MODE_CON, + dram_d.cru_mode_con | BITS_WMSK(0x1, PLL_MODE_BIT(CPLL_ID))); + io_write32(va_base + CRU_MODE_CON, + dram_d.cru_mode_con | BITS_WMSK(0x1, PLL_MODE_BIT(GPLL_ID))); +} + +static bool wait_core_wfe_i(uint32_t core) +{ + uint32_t wfei_mask, loop = 0; + vaddr_t va_base = (vaddr_t)phys_to_virt_io(GRF_BASE, GRF_SIZE); + + wfei_mask = CORE_WFE_I_MASK(core); + while (!(io_read32(va_base + GRF_CPU_STATUS1) & wfei_mask) && + loop < 500) { + udelay(2); + loop++; + } + + return io_read32(va_base + GRF_CPU_STATUS1) & wfei_mask; +} + +static bool core_held_in_reset(uint32_t core) +{ + uint32_t val; + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + val = io_read32(va_base + CRU_SOFTRST_CON(0)); + + return val & CORE_HELD_IN_RESET(core); +} + +uint32_t psci_version(void) +{ + return PSCI_VERSION_1_0; +} + +int psci_features(uint32_t psci_fid) +{ + switch (psci_fid) { + case PSCI_PSCI_FEATURES: + case PSCI_VERSION: + case PSCI_CPU_ON: + case PSCI_CPU_OFF: + case PSCI_SYSTEM_SUSPEND: + case PSCI_SYSTEM_RESET: + return PSCI_RET_SUCCESS; + default: + return PSCI_RET_NOT_SUPPORTED; + } +} + +int psci_cpu_on(uint32_t core_idx, uint32_t entry, + uint32_t context_id) +{ + bool wfei; + vaddr_t cru_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + vaddr_t isram_base = (vaddr_t)phys_to_virt_io(ISRAM_BASE, ISRAM_SIZE); + + core_idx &= MPIDR_CPU_MASK; + if ((core_idx == 0) || (core_idx >= CFG_TEE_CORE_NB_CORE)) + return PSCI_RET_INVALID_PARAMETERS; + + DMSG("core_id: %" PRIu32, core_idx); + + /* set secondary cores' NS entry addresses */ + boot_set_core_ns_entry(core_idx, entry, context_id); + + /* wait */ + if (!core_held_in_reset(core_idx)) { + wfei = wait_core_wfe_i(core_idx); + if (!wfei) { + EMSG("Can't wait cpu%" PRIu32 " wfei before softrst", + core_idx); + return PSCI_RET_DENIED; + } + } + + /* soft reset core */ + io_write32(cru_base + CRU_SOFTRST_CON(0), CORE_SOFT_RESET(core_idx)); + dsb(); + + udelay(2); + + /* soft release core */ + io_write32(cru_base + CRU_SOFTRST_CON(0), CORE_SOFT_RELEASE(core_idx)); + dsb(); + + /* wait */ + wfei = wait_core_wfe_i(core_idx); + if (!wfei) { + EMSG("Can't wait cpu%" PRIu32 " wfei after softrst", core_idx); + return PSCI_RET_DENIED; + } + + /* set secondary secure entry address and lock tag */ + io_write32(isram_base + BOOT_ADDR_OFFSET, TEE_LOAD_ADDR); + io_write32(isram_base + LOCK_ADDR_OFFSET, LOCK_TAG); + dsb(); + + sev(); + dsb(); + + return PSCI_RET_SUCCESS; +} + +int psci_cpu_off(void) +{ + uint32_t core = get_core_pos(); + + if ((core == 0) || (core >= CFG_TEE_CORE_NB_CORE)) + return PSCI_RET_INVALID_PARAMETERS; + + DMSG("core_id: %" PRIu32, core); + + psci_armv7_cpu_off(); + thread_mask_exceptions(THREAD_EXCP_ALL); + + while (1) + wfi(); + + return PSCI_RET_INTERNAL_FAILURE; +} + +int psci_affinity_info(uint32_t affinity, + uint32_t lowest_affnity_level __unused) +{ + uint32_t core_idx = affinity & MPIDR_CPU_MASK; + uint32_t wfi_mask = CORE_WFI_MASK(core_idx); + vaddr_t va_base = (vaddr_t)phys_to_virt_io(GRF_BASE, GRF_SIZE); + + DMSG("core_id: %" PRIu32 " STATUS: %" PRIx32 " MASK: %" PRIx32, + core_idx, io_read32(va_base + GRF_CPU_STATUS1), wfi_mask); + + return (io_read32(va_base + GRF_CPU_STATUS1) & wfi_mask) ? + PSCI_AFFINITY_LEVEL_OFF : PSCI_AFFINITY_LEVEL_ON; +} + +void psci_system_reset(void) +{ + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + /* PLLs enter slow mode */ + io_write32(va_base + CRU_MODE_CON, PLLS_SLOW_MODE); + dsb(); + + /* Global second reset */ + io_write32(va_base + CRU_SNDRST_VAL_BASE, CRU_SNDRST_VAL); + dsb(); +} + +int psci_system_suspend(uintptr_t entry __unused, + uint32_t context_id __unused, + struct sm_nsec_ctx *nsec __unused) +{ + DMSG("system suspend"); + + clks_disable(); + plls_power_down(); + + cache_op_inner(DCACHE_CLEAN_INV, NULL, 0); + + wfi(); + + plls_restore(); + clks_restore(); + + return PSCI_RET_SUCCESS; +} + +/* When SMP bootup, we release cores one by one */ +static TEE_Result reset_nonboot_cores(void) +{ + vaddr_t va_base = (vaddr_t)phys_to_virt_io(CRU_BASE, CRU_SIZE); + + io_write32(va_base + CRU_SOFTRST_CON(0), NONBOOT_CORES_SOFT_RESET); + + return TEE_SUCCESS; +} + +service_init_late(reset_nonboot_cores); diff --git a/optee_os/core/arch/arm/plat-rockchip/sub.mk b/optee_os/core/arch/arm/plat-rockchip/sub.mk new file mode 100644 index 0000000..74f8cf5 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rockchip/sub.mk @@ -0,0 +1,12 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-y += platform.c +srcs-$(PLATFORM_FLAVOR_px30) += platform_px30.c +srcs-$(PLATFORM_FLAVOR_rk322x) += platform_rk322x.c +srcs-$(PLATFORM_FLAVOR_rk3399) += platform_rk3399.c + +ifeq ($(PLATFORM_FLAVOR),rk322x) +srcs-y += plat_init.S +srcs-y += core_pos_a32.S +srcs-$(CFG_PSCI_ARM32) += psci_rk322x.c +endif diff --git a/optee_os/core/arch/arm/plat-rpi3/conf.mk b/optee_os/core/arch/arm/plat-rpi3/conf.mk new file mode 100644 index 0000000..6bb5651 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rpi3/conf.mk @@ -0,0 +1,29 @@ +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_TEE_CORE_NB_CORE,4) + +CFG_SHMEM_START ?= 0x08000000 +CFG_SHMEM_SIZE ?= 0x00400000 +CFG_TZDRAM_START ?= 0x10100000 +CFG_TZDRAM_SIZE ?= 0x00F00000 +CFG_TEE_RAM_VA_SIZE ?= 0x00700000 + +$(call force,CFG_8250_UART,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) + +CFG_NUM_THREADS ?= 4 +CFG_CRYPTO_WITH_CE ?= n + +CFG_TEE_CORE_EMBED_INTERNAL_TESTS ?= y +CFG_WITH_STACK_CANARIES ?= y +CFG_WITH_STATS ?= y + +arm32-platform-cflags += -Wno-error=cast-align +arm64-platform-cflags += -Wno-error=cast-align + +$(call force,CFG_CRYPTO_SHA256_ARM32_CE,n) +$(call force,CFG_CRYPTO_SHA256_ARM64_CE,n) +$(call force,CFG_CRYPTO_SHA1_ARM32_CE,n) +$(call force,CFG_CRYPTO_SHA1_ARM64_CE,n) +$(call force,CFG_CRYPTO_AES_ARM64_CE,n) diff --git a/optee_os/core/arch/arm/plat-rpi3/main.c b/optee_os/core/arch/arm/plat-rpi3/main.c new file mode 100644 index 0000000..3e9021d --- /dev/null +++ b/optee_os/core/arch/arm/plat-rpi3/main.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Sequitur Labs Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, + CONSOLE_UART_BASE, SERIAL8250_UART_REG_SIZE); + +static struct serial8250_uart_data console_data; + +void console_init(void) +{ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-rpi3/platform_config.h b/optee_os/core/arch/arm/plat-rpi3/platform_config.h new file mode 100644 index 0000000..dfb9e29 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rpi3/platform_config.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Sequitur Labs Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#ifdef ARM64 +#ifdef CFG_WITH_PAGER +#error "Pager not supported for ARM64" +#endif +#endif /* ARM64 */ + +/* 16550 UART */ +#define CONSOLE_UART_BASE 0x3f215040 /* UART0 */ +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 19200000 + +/* + * RPi memory map + * + * No secure memory on RPi... + * + * + * Available to Linux + * 0x0a00_0000 + * TA RAM: 16 MiB | + * 0x0842_0000 | TZDRAM + * TEE RAM: 4 MiB (TEE_RAM_VA_SIZE) | + * 0x0840_0000 [ARM Trusted Firmware ] - + * 0x0840_0000 [TZDRAM_BASE, BL32_LOAD_ADDR] - + * Shared memory: 4 MiB | + * 0x0800_0000 | DRAM0 + * Available to Linux | + * 0x0000_0000 [DRAM0_BASE] - + * + */ + +#define DRAM0_BASE 0x00000000 +#define DRAM0_SIZE 0x40000000 + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-rpi3/sub.mk b/optee_os/core/arch/arm/plat-rpi3/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-rpi3/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-rzg/conf.mk b/optee_os/core/arch/arm/plat-rzg/conf.mk new file mode 100644 index 0000000..c001112 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzg/conf.mk @@ -0,0 +1,43 @@ +PLATFORM_FLAVOR ?= hihope_rzg2m + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_SCIF,y) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) + +# Disable core ASLR for two reasons: +# 1. There is no source for ALSR seed, as RZ/G2 platform +# does not provide DTB to OP-TEE. Also, there is no +# publicly available documentation on integrated +# hardware RNG, so we can't use it either. +# 2. OP-TEE crashes during boot with enabled CFG_CORE_ASLR. +$(call force,CFG_CORE_ASLR,n) + +ifeq ($(PLATFORM_FLAVOR),ek874) +$(call force,CFG_TEE_CORE_NB_CORE,2) +endif +ifeq ($(PLATFORM_FLAVOR),hihope_rzg2m) +$(call force,CFG_TEE_CORE_NB_CORE,6) +# RZ/G2M have 6 cores for 2 clusters, but the number isn't contiguous. +# One cluster has ids 0, 1, other has ids 3, 4, 5, 6. +# CFG_CORE_CLUSTER_SHIFT will process to make the right numbering. +$(call force,CFG_CORE_CLUSTER_SHIFT,1) +endif +ifeq ($(PLATFORM_FLAVOR),hihope_rzg2n) +$(call force,CFG_TEE_CORE_NB_CORE,2) +endif +ifeq ($(PLATFORM_FLAVOR),hihope_rzg2h) +$(call force,CFG_TEE_CORE_NB_CORE,8) +endif + +CFG_TZDRAM_START ?= 0x44100000 +CFG_TZDRAM_SIZE ?= 0x03D00000 +CFG_TEE_RAM_VA_SIZE ?= 0x100000 +ifeq ($(CFG_ARM64_core),y) +supported-ta-targets = ta_arm64 +endif + +CFG_DT ?= y diff --git a/optee_os/core/arch/arm/plat-rzg/link.mk b/optee_os/core/arch/arm/plat-rzg/link.mk new file mode 100644 index 0000000..229c4b7 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzg/link.mk @@ -0,0 +1,3 @@ +include core/arch/arm/kernel/link.mk + +all: $(link-out-dir)/tee.srec diff --git a/optee_os/core/arch/arm/plat-rzg/main.c b/optee_os/core/arch/arm/plat-rzg/main.c new file mode 100644 index 0000000..31dfd08 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzg/main.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, GlobalLogic + * Copyright (c) 2019-2020, Renesas Electronics Corporation + */ + +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, SCIF_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GIC_DIST_REG_SIZE); + +register_dynamic_shm(NSEC_DDR_0_BASE, NSEC_DDR_0_SIZE); +#ifdef NSEC_DDR_1_BASE +register_dynamic_shm(NSEC_DDR_1_BASE, NSEC_DDR_1_SIZE); +#endif +#ifdef NSEC_DDR_2_BASE +register_dynamic_shm(NSEC_DDR_2_BASE, NSEC_DDR_2_SIZE); +#endif +#ifdef NSEC_DDR_3_BASE +register_dynamic_shm(NSEC_DDR_3_BASE, NSEC_DDR_3_SIZE); +#endif + +static struct scif_uart_data console_data __nex_bss; + +void console_init(void) +{ + scif_uart_init(&console_data, CONSOLE_UART_BASE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-rzg/platform_config.h b/optee_os/core/arch/arm/plat-rzg/platform_config.h new file mode 100644 index 0000000..eacc5a4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzg/platform_config.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, GlobalLogic + * Copyright (c) 2020, Renesas Electronics Corporation + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#define GIC_BASE 0xF1000000 +#define GICC_BASE 0xF1020000 +#define GICD_BASE 0xF1010000 + +#define CONSOLE_UART_BASE 0xE6E88000 + +#if defined(PLATFORM_FLAVOR_ek874) +#define NSEC_DDR_0_BASE 0x47E00000U +#define NSEC_DDR_0_SIZE 0x78200000 + +#elif defined(PLATFORM_FLAVOR_hihope_rzg2h) + +#define NSEC_DDR_0_BASE 0x47E00000U +#define NSEC_DDR_0_SIZE 0x78200000 +#define NSEC_DDR_1_BASE 0x500000000U +#define NSEC_DDR_1_SIZE 0x80000000 + +#elif defined(PLATFORM_FLAVOR_hihope_rzg2m) + +#define NSEC_DDR_0_BASE 0x47E00000U +#define NSEC_DDR_0_SIZE 0x78200000 +#define NSEC_DDR_1_BASE 0x600000000U +#define NSEC_DDR_1_SIZE 0x80000000 + +#elif defined(PLATFORM_FLAVOR_hihope_rzg2n) + +#define NSEC_DDR_0_BASE 0x47E00000U +#define NSEC_DDR_0_SIZE 0x78200000 +#define NSEC_DDR_1_BASE 0x480000000U +#define NSEC_DDR_1_SIZE 0x80000000 + +#else +#error "Unknown platform flavor" +#endif + +#define TEE_SHMEM_START (TZDRAM_BASE + TZDRAM_SIZE) +#define TEE_SHMEM_SIZE 0x100000 + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-rzg/sub.mk b/optee_os/core/arch/arm/plat-rzg/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzg/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-rzn1/a7_plat_init.S b/optee_os/core/arch/arm/plat-rzn1/a7_plat_init.S new file mode 100644 index 0000000..4f3ae2c --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/a7_plat_init.S @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Wind River Systems. + * Copyright (c) 2020, Linaro Limited + */ + +/* + * Entry points for the A7 init. + * + * Assumptions: + * - No stack is available when these routines are called. + * - Each routine is called with return address in LR and + * with ARM registers R0, R1, R2, R3 being scratchable. + */ + +#include +#include +#include +#include + +.section .text +.balign 4 +.code 32 + +FUNC plat_cpu_reset_early , : + /* + * SCR = 0x00000020 + * - FW: Disallow NSec to mask FIQ [bit4=0] + * - AW: Allow NSec to manage Imprecise Abort [bit5=1] + * - EA: Imprecise Abort trapped to Abort Mode [bit3=0] + * - FIQ: In Sec world, FIQ trapped to FIQ Mode [bit2=0] + * - IRQ: IRQ always trapped to IRQ Mode [bit1=0] + * - NS: Secure World [bit0=0] + */ + mov r0, #SCR_AW + write_scr r0 + + mov_imm r0, 0x00000000 + write_sctlr r0 + + /* + * ACTRL = 0x00006040 + * - DDI: Disable dual issue [bit28=0] + * - DDVM: Disable Distributed Virtual Memory transactions [bit15=0] + * - L1PCTL: L1 Data prefetch control [bit14:13=2b11] + * - L1RADIS: L1 Data Cache read-allocate mode disable [bit12=0] + * - L2RADIS: L2 Data Cache read-allocate mode disable [bit11=0] + * - DODMBS: Disable optimized data memory barrier behavior [bit10=0] + * - SMP: Enables coherent requests to the processor [bit6=0] + */ + mov_imm r0, 0x00006040 + write_actlr r0 + + /* + * NSACR = 0x00000C00 + * - NS_SMP: Non-secure mode cannot change ACTRL.SMP (bit18=0) + * - NSASEDIS/NSD32DIS/CP10/CP11: Non-secure mode can use SIMD/VFP + * (bit15:14=2b00, bit11:10=2b11) + */ + mov_imm r0, 0x00000C00 + write_nsacr r0 + + mov pc, lr +END_FUNC plat_cpu_reset_early diff --git a/optee_os/core/arch/arm/plat-rzn1/conf.mk b/optee_os/core/arch/arm/plat-rzn1/conf.mk new file mode 100644 index 0000000..5dd7534 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/conf.mk @@ -0,0 +1,28 @@ +PLATFORM_FLAVOR ?= rzn1 + +include core/arch/arm/cpu/cortex-a7.mk + +$(call force,CFG_ARM32_core,y) +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_BOOT_SECONDARY_REQUEST,y) +$(call force,CFG_SECONDARY_INIT_CNTFRQ,y) +$(call force,CFG_PSCI_ARM32,y) +$(call force,CFG_16550_UART,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_PAGER,n) +$(call force,CFG_GIC,y) +$(call force,CFG_SM_PLATFORM_HANDLER,y) +$(call force,CFG_TA_FLOAT_SUPPORT,n) + +ta-targets = ta_arm32 + +CFG_TZDRAM_START ?= 0x88000000 +CFG_TZDRAM_SIZE ?= 0x00A00000 +CFG_SHMEM_START ?= 0x87C00000 +CFG_SHMEM_SIZE ?= 0x00400000 +CFG_TEE_RAM_VA_SIZE ?= 0x00200000 + +CFG_NUM_THREADS ?= 4 +CFG_NS_ENTRY_ADDR ?= 0x87A00000 + +CFG_BOOT_CM3 ?= y diff --git a/optee_os/core/arch/arm/plat-rzn1/main.c b/optee_os/core/arch/arm/plat-rzn1/main.c new file mode 100644 index 0000000..ac6d88f --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/main.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Schneider Electric + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SYSCTRL_PWRCTRL_CM3 (SYSCTRL_BASE + 0x174) +#define SYSCTRL_PWRSTAT_CM3 (SYSCTRL_BASE + 0x178) + +#define SYSCTRL_PWRCTRL_CM3_CLKEN_A BIT(0) +#define SYSCTRL_PWRCTRL_CM3_RSTN_A BIT(1) +#define SYSCTRL_PWRCTRL_CM3_MIREQ_A BIT(2) + +#define SYSCTRL_PWRSTAT_CM3_MIRACK_A BIT(0) + +/* Timeout waiting for Master Idle Request Acknowledge */ +#define IDLE_ACK_TIMEOUT_US 1000 + +static struct ns16550_data console_data; + +register_phys_mem(MEM_AREA_IO_SEC, GIC_BASE, CORE_MMU_PGDIR_SIZE); +register_phys_mem(MEM_AREA_IO_SEC, PERIPH_REG_BASE, CORE_MMU_PGDIR_SIZE); +register_ddr(DRAM_BASE, DRAM_SIZE); + +void console_init(void) +{ + ns16550_init(&console_data, CONSOLE_UART_BASE, IO_WIDTH_U32, 2); + register_serial_console(&console_data.chip); +} + +void boot_primary_init_intc(void) +{ + gic_init(GICC_BASE, GICD_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +static TEE_Result rzn1_tz_init(void) +{ + vaddr_t tza_init_reg = 0; + vaddr_t tza_targ_reg = 0; + + tza_init_reg = core_mmu_get_va(FW_STATIC_TZA_INIT, MEM_AREA_IO_SEC, + sizeof(uint32_t)); + tza_targ_reg = core_mmu_get_va(FW_STATIC_TZA_TARG, MEM_AREA_IO_SEC, + sizeof(uint32_t)); + + /* TZ initiator ports */ + io_write32(tza_init_reg, TZ_INIT_CSA_SEC | TZ_INIT_YS_SEC | + TZ_INIT_YC_SEC | TZ_INIT_YD_SEC); + + /* TZ target ports */ + io_write32(tza_targ_reg, TZ_TARG_PC_SEC | TZ_TARG_QB_SEC | + TZ_TARG_QA_SEC | TZ_TARG_UB_SEC | + TZ_TARG_UA_SEC); + + return TEE_SUCCESS; +} + +service_init(rzn1_tz_init); + +#ifdef CFG_BOOT_CM3 +static TEE_Result rzn1_cm3_start(void) +{ + vaddr_t cm3_pwrctrl_reg = 0; + vaddr_t cm3_pwrstat_reg = 0; + uint64_t timeout_ack = timeout_init_us(IDLE_ACK_TIMEOUT_US); + + cm3_pwrctrl_reg = core_mmu_get_va(SYSCTRL_PWRCTRL_CM3, MEM_AREA_IO_SEC, + sizeof(uint32_t)); + cm3_pwrstat_reg = core_mmu_get_va(SYSCTRL_PWRSTAT_CM3, MEM_AREA_IO_SEC, + sizeof(uint32_t)); + + /* Master Idle Request to the interconnect for CM3 */ + io_clrbits32(cm3_pwrctrl_reg, SYSCTRL_PWRCTRL_CM3_MIREQ_A); + + /* Wait for Master Idle Request Acknowledge for CM3 */ + while (!timeout_elapsed(timeout_ack)) + if (!(io_read32(cm3_pwrstat_reg) & + SYSCTRL_PWRSTAT_CM3_MIRACK_A)) + break; + + if (io_read32(cm3_pwrstat_reg) & SYSCTRL_PWRSTAT_CM3_MIRACK_A) + panic(); + + /* Clock Enable for CM3_HCLK & Active low Reset to CM3 */ + io_setbits32(cm3_pwrctrl_reg, SYSCTRL_PWRCTRL_CM3_CLKEN_A); + io_setbits32(cm3_pwrctrl_reg, SYSCTRL_PWRCTRL_CM3_RSTN_A); + + return TEE_SUCCESS; +} + +service_init(rzn1_cm3_start); +#endif diff --git a/optee_os/core/arch/arm/plat-rzn1/platform_config.h b/optee_os/core/arch/arm/plat-rzn1/platform_config.h new file mode 100644 index 0000000..a5a2fb1 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/platform_config.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Schneider Electric + * Copyright (c) 2020, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* DRAM */ +#define DRAM_BASE 0x80000000 +#define DRAM_SIZE 0x40000000 /* 1GB, and support 256MB */ + +/* GIC */ +#define GIC_BASE 0x44100000 +#define GICD_OFFSET 0x1000 +#define GICC_OFFSET 0x2000 +#define GICD_BASE (GIC_BASE + GICD_OFFSET) +#define GICC_BASE (GIC_BASE + GICC_OFFSET) + +/* Peripheral memory map */ +#define PERIPH_REG_BASE 0x40000000 + +/* System Control */ +#define SYSCTRL_BASE 0x4000C000 + +/* UART */ +#define CONSOLE_UART_BASE 0x40060000 + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-rzn1/psci.c b/optee_os/core/arch/arm/plat-rzn1/psci.c new file mode 100644 index 0000000..9e5fb3c --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/psci.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Schneider Electric + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define SYSCTRL_REG_RSTEN (SYSCTRL_BASE + 0x120) +#define SYSCTRL_REG_RSTCTRL (SYSCTRL_BASE + 0x198) +#define SYSCTRL_BOOTADDR_REG (SYSCTRL_BASE + 0x204) + +#define SYSCTRL_REG_RSTEN_MRESET_EN BIT(0) +#define SYSCTRL_REG_RSTEN_SWRST_EN BIT(6) +#define SYSCTRL_REG_RSTCTRL_SWRST_REQ BIT(6) + +int psci_features(uint32_t psci_fid) +{ + switch (psci_fid) { + case PSCI_PSCI_FEATURES: + case PSCI_VERSION: + case PSCI_CPU_ON: + case PSCI_CPU_OFF: + case PSCI_SYSTEM_RESET: + return PSCI_RET_SUCCESS; + default: + return PSCI_RET_NOT_SUPPORTED; + } +} + +uint32_t psci_version(void) +{ + return PSCI_VERSION_1_0; +} + +int psci_cpu_on(uint32_t core_id, uint32_t entry, uint32_t context_id) +{ + vaddr_t sctl_va = core_mmu_get_va(SYSCTRL_BOOTADDR_REG, + MEM_AREA_IO_SEC, + sizeof(uint32_t)); + + if (core_id == 0 || core_id >= CFG_TEE_CORE_NB_CORE) + return PSCI_RET_INVALID_PARAMETERS; + + DMSG("core_id: %" PRIu32, core_id); + + boot_set_core_ns_entry(core_id, entry, context_id); + io_write32(sctl_va, TEE_LOAD_ADDR); + + dsb(); + sev(); + + return PSCI_RET_SUCCESS; +} + +int __noreturn psci_cpu_off(void) +{ + DMSG("core_id: %" PRIu32, get_core_pos()); + + psci_armv7_cpu_off(); + + thread_mask_exceptions(THREAD_EXCP_ALL); + + while (1) + wfi(); +} + +void psci_system_reset(void) +{ + /* Enable software reset */ + io_setbits32(core_mmu_get_va(SYSCTRL_REG_RSTEN, MEM_AREA_IO_SEC, + sizeof(uint32_t)), + SYSCTRL_REG_RSTEN_SWRST_EN | SYSCTRL_REG_RSTEN_MRESET_EN); + + /* Trigger software reset */ + io_setbits32(core_mmu_get_va(SYSCTRL_REG_RSTCTRL, MEM_AREA_IO_SEC, + sizeof(uint32_t)), + SYSCTRL_REG_RSTCTRL_SWRST_REQ); + + dsb(); +} diff --git a/optee_os/core/arch/arm/plat-rzn1/rzn1_regauth.h b/optee_os/core/arch/arm/plat-rzn1/rzn1_regauth.h new file mode 100644 index 0000000..a8572a1 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/rzn1_regauth.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Schneider Electric + * Copyright (c) 2020, Linaro Limited + */ + +#ifndef RZN1_REGAUTH_H +#define RZN1_REGAUTH_H + +struct regauth_t { + uint32_t paddr; + uint32_t size; + uint32_t rmask; + uint32_t wmask; +}; + +static const struct regauth_t regauth[] = { + /* OTP */ + { 0x40007000U, 0x4U, 0x0U, 0x0U }, /* OTPWCTRL */ + /* System Controller */ + { 0x4000C064U, 0x4U, 0xFFFFFFFFU, 0xFFFFFFE0U }, /* PWRCTRL_DDRC */ + { 0x4000C204U, 0x4U, 0x0U, 0x0U }, /* BOOTADDR */ + /* DDR CTRL */ + { 0x4000D16CU, 0x3FCU, 0x0U, 0x0U }, /* DDR_CTL 91-346 */ + { 0x4000E000U, 0x4U, 0xFFFFFFFFU, 0xFFFFFFFEU }, /* UNCCTRL */ + { 0x4000E004U, 0x4U, 0xFFFFFFFFU, 0xFFFFFFFEU }, /* DLLCTRL */ +}; + +#endif /* RZN1_REGAUTH_H */ diff --git a/optee_os/core/arch/arm/plat-rzn1/rzn1_tz.h b/optee_os/core/arch/arm/plat-rzn1/rzn1_tz.h new file mode 100644 index 0000000..9ebc7d2 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/rzn1_tz.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Schneider Electric + * Copyright (c) 2020, Linaro Limited + */ + +#ifndef RZN1_TZ_H +#define RZN1_TZ_H + +#include + +/* TZ config registers */ +#define FW_STATIC_TZA_INIT 0x4000C0D0 +#define FW_STATIC_TZA_TARG 0x4000C0D4 + +/* TZ initiatior ports */ +#define TZ_INIT_CSB_SEC BIT(7) /* CoreSight AHB */ +#define TZ_INIT_CSA_SEC BIT(6) /* CoreSight AXI */ +#define TZ_INIT_YS_SEC BIT(5) /* Cortex-M3 System Bus interface */ +#define TZ_INIT_YC_SEC BIT(4) /* Cortex-M3 ICode interface */ +#define TZ_INIT_YD_SEC BIT(3) /* Cortex-M3 DCode interface */ +#define TZ_INIT_Z_SEC BIT(2) /* Packet Engine */ +#define TZ_INIT_I_SEC BIT(1) /* Peripheral Group */ +#define TZ_INIT_F_SEC BIT(0) /* Peripheral Group */ + +/* TZ target ports */ +#define TZ_TARG_W_SEC BIT(14) /* RTC */ +#define TZ_TARG_PC_SEC BIT(9) /* DDR2/3 Controller */ +#define TZ_TARG_RA_SEC BIT(8) /* CoreSight */ +#define TZ_TARG_QB_SEC BIT(7) /* System Control */ +#define TZ_TARG_QA_SEC BIT(6) /* PG0 */ +#define TZ_TARG_NB_SEC BIT(5) /* Packet Engine */ +#define TZ_TARG_NA_SEC BIT(4) /* Public Key Processor */ +#define TZ_TARG_K_SEC BIT(3) /* Peripheral Group */ +#define TZ_TARG_J_SEC BIT(2) /* Peripheral Group */ +#define TZ_TARG_UB_SEC BIT(1) /* 2MB SRAM */ +#define TZ_TARG_UA_SEC BIT(0) /* 2MB SRAM */ + +#endif /* RZN1_TZ_H */ diff --git a/optee_os/core/arch/arm/plat-rzn1/sm_platform_handler.c b/optee_os/core/arch/arm/plat-rzn1/sm_platform_handler.c new file mode 100644 index 0000000..dc990f4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/sm_platform_handler.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "rzn1_regauth.h" + +#define RZN1_OEM_CONSOLE_PUTC 0x01 +#define RZN1_OEM_SYSREG_AUTH 0x10 + +static const struct regauth_t regauth_pass = {.rmask = ~0U, .wmask = ~0U}; + +static const struct regauth_t *get_regauth(unsigned long paddr) +{ + unsigned int idx = 0; + unsigned int len = ARRAY_SIZE(regauth); + + while (idx < len) { + if (core_is_buffer_inside(paddr, sizeof(uint32_t), + regauth[idx].paddr, + regauth[idx].size)) + return ®auth[idx]; + idx++; + } + + return NULL; +} + +static uint32_t oem_sysreg(uint32_t addr, uint32_t mask, uint32_t *pvalue) +{ + vaddr_t reg = 0; + const struct regauth_t *auth = get_regauth(addr); + + /* Allow operations on registers not in the list */ + if (!auth) + auth = ®auth_pass; + + reg = core_mmu_get_va(addr, MEM_AREA_IO_SEC, sizeof(uint32_t)); + + if (mask) { + /* Write operation */ + mask &= auth->wmask; + if (!reg || !mask) + DMSG("Blocking write of 0x%"PRIx32" to register 0x%" + PRIx32" (0x%"PRIxVA")", *pvalue, addr, reg); + else if (mask == ~0UL) + io_write32(reg, *pvalue); + else + io_mask32(reg, *pvalue, mask); + } else { + /* Read operation */ + if (!reg || !auth->rmask) + DMSG("Blocking read of register 0x%"PRIx32" (0x%" + PRIxVA")", addr, reg); + else + *pvalue = io_read32(reg) & auth->rmask; + } + + return 0; +} + +static enum sm_handler_ret oem_service(struct sm_ctx *ctx __unused, + struct thread_smc_args *args) +{ + switch (OPTEE_SMC_FUNC_NUM(args->a0)) { + case RZN1_OEM_SYSREG_AUTH: + args->a0 = oem_sysreg(args->a1, args->a2, &args->a3); + args->a1 = args->a3; + break; + case RZN1_OEM_CONSOLE_PUTC: + console_putc(args->a1); + break; + default: + return SM_HANDLER_PENDING_SMC; + } + + return SM_HANDLER_SMC_HANDLED; +} + +enum sm_handler_ret sm_platform_handler(struct sm_ctx *ctx) +{ + struct thread_smc_args *args = (void *)&ctx->nsec.r0; + + if (!OPTEE_SMC_IS_FAST_CALL(args->a0)) + return SM_HANDLER_PENDING_SMC; + + switch (OPTEE_SMC_OWNER_NUM(args->a0)) { + case OPTEE_SMC_OWNER_OEM: + return oem_service(ctx, args); + default: + return SM_HANDLER_PENDING_SMC; + } +} diff --git a/optee_os/core/arch/arm/plat-rzn1/sub.mk b/optee_os/core/arch/arm/plat-rzn1/sub.mk new file mode 100644 index 0000000..4fc96f0 --- /dev/null +++ b/optee_os/core/arch/arm/plat-rzn1/sub.mk @@ -0,0 +1,5 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-y += psci.c +srcs-y += sm_platform_handler.c +srcs-y += a7_plat_init.S diff --git a/optee_os/core/arch/arm/plat-sam/conf.mk b/optee_os/core/arch/arm/plat-sam/conf.mk new file mode 100644 index 0000000..d546cbc --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/conf.mk @@ -0,0 +1,110 @@ +PLATFORM_FLAVOR ?= sama5d27_som1_ek + +flavor_dts_file-sama5d2xult = at91-sama5d2_xplained.dts +flavor_dts_file-sama5d2_xplained = at91-sama5d2_xplained.dts +flavor_dts_file-sama5d27_som1_ek = at91-sama5d27_som1_ek.dts +flavor_dts_file-sama5d27_wlsom1_ek = at91-sama5d27_wlsom1_ek.dts + +ifeq ($(PLATFORM_FLAVOR),sama5d2xult) +$(warning "sama5d2xult is deprecated, please use sama5d2_xplained") +endif + +ifeq ($(flavor_dts_file-$(PLATFORM_FLAVOR)),) +$(error Invalid platform flavor $(PLATFORM_FLAVOR)) +endif +CFG_EMBED_DTB_SOURCE_FILE ?= $(flavor_dts_file-$(PLATFORM_FLAVOR)) + +include core/arch/arm/cpu/cortex-a5.mk + +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_ATMEL_UART,y) +$(call force,CFG_ATMEL_SAIC,y) +$(call force,CFG_ATMEL_TCB,y) +$(call force,CFG_NO_SMP,y) +$(call force,CFG_PL310,y) +$(call force,CFG_PL310_LOCKED,y) +$(call force,CFG_AT91_MATRIX,y) +$(call force,CFG_DRIVERS_CLK,y) +$(call force,CFG_DRIVERS_CLK_DT,y) +$(call force,CFG_DRIVERS_CLK_FIXED,y) +$(call force,CFG_DRIVERS_SAM_CLK,y) +$(call force,CFG_DRIVERS_SAMA5D2_CLK,y) +$(call force,CFG_PSCI_ARM32,y) +$(call force,CFG_SM_PLATFORM_HANDLER,y) +$(call force,CFG_CORE_HAS_GENERIC_TIMER,n) + +# These values are forced because of matrix configuration for secure area. +# When modifying these, always update matrix settings in +# matrix_configure_slave_h64mx(). +$(call force,CFG_TZDRAM_START,0x20000000) +$(call force,CFG_TZDRAM_SIZE,0x800000) + +CFG_MMAP_REGIONS ?= 24 + +CFG_SHMEM_START ?= 0x21000000 +CFG_SHMEM_SIZE ?= 0x400000 + +CFG_SCMI_SHMEM_START ?= 0x21400000 +CFG_SCMI_SHMEM_SIZE ?= 0x1000 + +CFG_TEE_RAM_VA_SIZE ?= 0x100000 + +# Device tree related configuration +CFG_DT_ADDR ?= 0x21500000 +CFG_GENERATE_DTB_OVERLAY ?= y + +CFG_WITH_SOFTWARE_PRNG ?= n +CFG_ATMEL_TRNG ?= y +ifeq ($(CFG_ATMEL_TRNG),y) +CFG_HWRNG_PTA ?= y +$(call force,CFG_HWRNG_QUALITY,1024) +endif + +CFG_ATMEL_RSTC ?= y +CFG_ATMEL_SHDWC ?= y + +CFG_ATMEL_PM ?= y + +ifeq ($(CFG_ATMEL_PM),y) +# Suspend mode to be used on PSCI suspend call +# 0 = STANDBY +# 1 = ULP0 +# 2 = ULP0 Fast +# 3 = ULP1 +# 4 = BACKUP +CFG_ATMEL_PM_SUSPEND_MODE ?= 0 + +$(call force,CFG_ATMEL_SHDWC,y) +$(call force,CFG_PM_ARM32,y) +endif + +CFG_WDT ?= y +CFG_WDT_SM_HANDLER ?= y +ifeq ($(CFG_WDT_SM_HANDLER),y) +CFG_WDT_SM_HANDLER_ID := 0x2000500 +endif +CFG_ATMEL_WDT ?= y + +CFG_DRIVERS_RTC ?= y +CFG_RTC_PTA ?= y +CFG_ATMEL_RTC ?= y +CFG_ATMEL_PIOBU ?= y + +ifeq ($(PLATFORM_FLAVOR),sama5d27_wlsom1_ek) +CFG_DRIVERS_GPIO ?= y +CFG_DRIVERS_I2C ?= y +CFG_ATMEL_I2C ?= y +CFG_DRIVERS_PINCTRL ?= y +CFG_ATMEL_PIO ?= y +endif + +# SCMI related configuration +CFG_SCMI_PTA ?= y + +CFG_SCMI_MSG_DRIVERS ?= y +ifeq ($(CFG_SCMI_MSG_DRIVERS),y) +$(call force,CFG_SCMI_MSG_SMT,y) +$(call force,CFG_SCMI_MSG_CLOCK,y) +$(call force,CFG_SCMI_MSG_USE_CLK,y) +$(call force,CFG_SCMI_MSG_SMT_FASTCALL_ENTRY,y) +endif diff --git a/optee_os/core/arch/arm/plat-sam/freq.c b/optee_os/core/arch/arm/plat-sam/freq.c new file mode 100644 index 0000000..b6fd72f --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/freq.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microchip + */ + +#include +#include +#include +#include +#include +#include +#include + +static unsigned long freq; + +static TEE_Result get_freq_from_dt(void) +{ + int node; + struct clk *clk; + const void *fdt = get_embedded_dt(); + + if (!fdt) + panic(); + + node = fdt_node_offset_by_compatible(fdt, -1, "arm,cortex-a5"); + if (!node) + panic(); + + if (clk_dt_get_by_name(fdt, node, "cpu", &clk)) + panic(); + + freq = clk_get_rate(clk); + + return TEE_SUCCESS; +} +early_init_late(get_freq_from_dt); + +unsigned long plat_get_freq(void) +{ + assert(freq); + + return freq; +} diff --git a/optee_os/core/arch/arm/plat-sam/main.c b/optee_os/core/arch/arm/plat-sam/main.c new file mode 100644 index 0000000..503ab0d --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/main.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Timesys Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct atmel_uart_data console_data; +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, + CORE_MMU_PGDIR_SIZE); + +void console_init(void) +{ + atmel_uart_init(&console_data, CONSOLE_UART_BASE); + register_serial_console(&console_data.chip); +} + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AT91C_BASE_MATRIX32, + CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AT91C_BASE_MATRIX64, + CORE_MMU_PGDIR_SIZE); + +vaddr_t matrix32_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(AT91C_BASE_MATRIX32, MEM_AREA_IO_SEC, + 1); + return (vaddr_t)va; + } + return AT91C_BASE_MATRIX32; +} + +vaddr_t matrix64_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(AT91C_BASE_MATRIX64, MEM_AREA_IO_SEC, + 1); + return (vaddr_t)va; + } + return AT91C_BASE_MATRIX64; +} + +static void matrix_configure_slave_h64mx(void) +{ + unsigned int ddr_port; + unsigned int ssr_setting; + unsigned int sasplit_setting; + unsigned int srtop_setting; + + /* + * 0: Bridge from H64MX to AXIMX + * (Internal ROM, Crypto Library, PKCC RAM): Always Secured + */ + + /* 1: H64MX Peripheral Bridge: SDMMC0, SDMMC1 Non-Secure */ + srtop_setting = MATRIX_SRTOP(1, MATRIX_SRTOP_VALUE_128M) + | MATRIX_SRTOP(2, MATRIX_SRTOP_VALUE_128M); + sasplit_setting = MATRIX_SASPLIT(1, MATRIX_SASPLIT_VALUE_128M) + | MATRIX_SASPLIT(2, MATRIX_SASPLIT_VALUE_128M); + ssr_setting = (MATRIX_LANSECH_NS(1) + | MATRIX_LANSECH_NS(2) + | MATRIX_RDNSECH_NS(1) + | MATRIX_RDNSECH_NS(2) + | MATRIX_WRNSECH_NS(1) + | MATRIX_WRNSECH_NS(2)); + matrix_configure_slave_security(matrix64_base(), + H64MX_SLAVE_PERI_BRIDGE, + srtop_setting, + sasplit_setting, + ssr_setting); + + /* + * Matrix DDR configuration is hardcoded here and is difficult to + * generate at runtime. Since this configuration expect the secure + * DRAM to be at start of RAM and 8M of size, enforce it here. + */ + COMPILE_TIME_ASSERT(CFG_TZDRAM_START == AT91C_BASE_DDRCS); + COMPILE_TIME_ASSERT(CFG_TZDRAM_SIZE == 0x800000); + + /* 2 ~ 9 DDR2 Port1 ~ 7: Non-Secure, except op-tee tee/ta memory */ + srtop_setting = MATRIX_SRTOP(0, MATRIX_SRTOP_VALUE_128M); + sasplit_setting = (MATRIX_SASPLIT(0, MATRIX_SASPLIT_VALUE_8M) + | MATRIX_SASPLIT(1, MATRIX_SASPLIT_VALUE_128M) + | MATRIX_SASPLIT(2, MATRIX_SASPLIT_VALUE_128M) + | MATRIX_SASPLIT(3, MATRIX_SASPLIT_VALUE_128M)); + ssr_setting = (MATRIX_LANSECH_S(0) + | MATRIX_LANSECH_NS(1) + | MATRIX_LANSECH_NS(2) + | MATRIX_LANSECH_NS(3) + | MATRIX_RDNSECH_S(0) + | MATRIX_RDNSECH_NS(1) + | MATRIX_RDNSECH_NS(2) + | MATRIX_RDNSECH_NS(3) + | MATRIX_WRNSECH_S(0) + | MATRIX_WRNSECH_NS(1) + | MATRIX_WRNSECH_NS(2) + | MATRIX_WRNSECH_NS(3)); + /* DDR port 0 not used from NWd */ + for (ddr_port = 1; ddr_port < 8; ddr_port++) { + matrix_configure_slave_security(matrix64_base(), + (H64MX_SLAVE_DDR2_PORT_0 + ddr_port), + srtop_setting, + sasplit_setting, + ssr_setting); + } + + /* + * 10: Internal SRAM 128K: + * - First 64K are reserved for suspend code in Secure World + * - Last 64K are for Non-Secure world (used by CAN) + */ + srtop_setting = MATRIX_SRTOP(0, MATRIX_SRTOP_VALUE_128K); + sasplit_setting = MATRIX_SASPLIT(0, MATRIX_SRTOP_VALUE_64K); + ssr_setting = (MATRIX_LANSECH_S(0) | MATRIX_RDNSECH_S(0) | + MATRIX_WRNSECH_S(0)); + matrix_configure_slave_security(matrix64_base(), + H64MX_SLAVE_INTERNAL_SRAM, + srtop_setting, sasplit_setting, + ssr_setting); + + /* 11: Internal SRAM 128K (Cache L2): Default */ + + /* 12: QSPI0: Normal world */ + /* 13: QSPI1: Normal world */ + srtop_setting = MATRIX_SRTOP(0, MATRIX_SRTOP_VALUE_128M); + sasplit_setting = MATRIX_SASPLIT(0, MATRIX_SASPLIT_VALUE_128M); + ssr_setting = MATRIX_LANSECH_NS(0) | MATRIX_RDNSECH_NS(0) | + MATRIX_WRNSECH_NS(0); + + matrix_configure_slave_security(matrix64_base(), H64MX_SLAVE_QSPI0, + srtop_setting, sasplit_setting, + ssr_setting); + matrix_configure_slave_security(matrix64_base(), H64MX_SLAVE_QSPI1, + srtop_setting, sasplit_setting, + ssr_setting); + /* 14: AESB: Default */ +} + +static void matrix_configure_slave_h32mx(void) +{ + unsigned int ssr_setting; + unsigned int sasplit_setting; + unsigned int srtop_setting; + + /* 0: Bridge from H32MX to H64MX: Not Secured */ + /* 1: H32MX Peripheral Bridge 0: Not Secured */ + /* 2: H32MX Peripheral Bridge 1: Not Secured */ + + /* + * 3: External Bus Interface + * EBI CS0 Memory(256M) ----> Slave Region 0, 1 + * EBI CS1 Memory(256M) ----> Slave Region 2, 3 + * EBI CS2 Memory(256M) ----> Slave Region 4, 5 + * EBI CS3 Memory(128M) ----> Slave Region 6 + * NFC Command Registers(128M) -->Slave Region 7 + * NANDFlash(EBI CS3) --> Slave Region 6: Non-Secure + */ + srtop_setting = MATRIX_SRTOP(6, MATRIX_SRTOP_VALUE_128M); + srtop_setting |= MATRIX_SRTOP(7, MATRIX_SRTOP_VALUE_128M); + sasplit_setting = MATRIX_SASPLIT(6, MATRIX_SASPLIT_VALUE_128M); + sasplit_setting |= MATRIX_SASPLIT(7, MATRIX_SASPLIT_VALUE_128M); + ssr_setting = (MATRIX_LANSECH_NS(6) + | MATRIX_RDNSECH_NS(6) + | MATRIX_WRNSECH_NS(6)); + ssr_setting |= (MATRIX_LANSECH_NS(7) + | MATRIX_RDNSECH_NS(7) + | MATRIX_WRNSECH_NS(7)); + matrix_configure_slave_security(matrix32_base(), + H32MX_EXTERNAL_EBI, + srtop_setting, + sasplit_setting, + ssr_setting); + + /* 4: NFC SRAM (4K): Non-Secure */ + srtop_setting = MATRIX_SRTOP(0, MATRIX_SRTOP_VALUE_8K); + sasplit_setting = MATRIX_SASPLIT(0, MATRIX_SASPLIT_VALUE_8K); + ssr_setting = (MATRIX_LANSECH_NS(0) + | MATRIX_RDNSECH_NS(0) + | MATRIX_WRNSECH_NS(0)); + matrix_configure_slave_security(matrix32_base(), + H32MX_NFC_SRAM, + srtop_setting, + sasplit_setting, + ssr_setting); + + /* 5: + * USB Device High Speed Dual Port RAM (DPR): 1M + * USB Host OHCI registers: 1M + * USB Host EHCI registers: 1M + */ + srtop_setting = (MATRIX_SRTOP(0, MATRIX_SRTOP_VALUE_1M) + | MATRIX_SRTOP(1, MATRIX_SRTOP_VALUE_1M) + | MATRIX_SRTOP(2, MATRIX_SRTOP_VALUE_1M)); + sasplit_setting = (MATRIX_SASPLIT(0, MATRIX_SASPLIT_VALUE_1M) + | MATRIX_SASPLIT(1, MATRIX_SASPLIT_VALUE_1M) + | MATRIX_SASPLIT(2, MATRIX_SASPLIT_VALUE_1M)); + ssr_setting = (MATRIX_LANSECH_NS(0) + | MATRIX_LANSECH_NS(1) + | MATRIX_LANSECH_NS(2) + | MATRIX_RDNSECH_NS(0) + | MATRIX_RDNSECH_NS(1) + | MATRIX_RDNSECH_NS(2) + | MATRIX_WRNSECH_NS(0) + | MATRIX_WRNSECH_NS(1) + | MATRIX_WRNSECH_NS(2)); + matrix_configure_slave_security(matrix32_base(), + H32MX_USB, + srtop_setting, + sasplit_setting, + ssr_setting); +} + +static unsigned int security_ps_peri_id[] = { + AT91C_ID_PMC, + AT91C_ID_ARM, + AT91C_ID_PIT, + AT91C_ID_WDT, + AT91C_ID_GMAC, + AT91C_ID_XDMAC0, + AT91C_ID_XDMAC1, + AT91C_ID_ICM, + AT91C_ID_AES, + AT91C_ID_AESB, + AT91C_ID_TDES, + AT91C_ID_SHA, + AT91C_ID_MPDDRC, + AT91C_ID_HSMC, + AT91C_ID_FLEXCOM0, + AT91C_ID_FLEXCOM1, + AT91C_ID_FLEXCOM2, + AT91C_ID_FLEXCOM3, + AT91C_ID_FLEXCOM4, + AT91C_ID_UART0, + AT91C_ID_UART1, + AT91C_ID_UART2, + AT91C_ID_UART3, + AT91C_ID_UART4, + AT91C_ID_TWI0, + AT91C_ID_TWI1, + AT91C_ID_SDMMC0, + AT91C_ID_SDMMC1, + AT91C_ID_SPI0, + AT91C_ID_SPI1, + AT91C_ID_TC0, + AT91C_ID_TC1, + AT91C_ID_PWM, + AT91C_ID_ADC, + AT91C_ID_UHPHS, + AT91C_ID_UDPHS, + AT91C_ID_SSC0, + AT91C_ID_SSC1, + AT91C_ID_LCDC, + AT91C_ID_ISI, + AT91C_ID_TRNG, + AT91C_ID_PDMIC, + AT91C_ID_SFC, + AT91C_ID_QSPI0, + AT91C_ID_QSPI1, + AT91C_ID_I2SC0, + AT91C_ID_I2SC1, + AT91C_ID_CAN0_INT0, + AT91C_ID_CAN1_INT0, + AT91C_ID_CLASSD, + AT91C_ID_SFR, + AT91C_ID_L2CC, + AT91C_ID_CAN0_INT1, + AT91C_ID_CAN1_INT1, + AT91C_ID_GMAC_Q1, + AT91C_ID_GMAC_Q2, + AT91C_ID_SDMMC0_TIMER, + AT91C_ID_SDMMC1_TIMER, + AT91C_ID_SYS, + AT91C_ID_ACC, + AT91C_ID_RXLP, + AT91C_ID_SFRBU, + AT91C_ID_CHIPID, +}; + +static int matrix_init(void) +{ + matrix_write_protect_disable(matrix64_base()); + matrix_write_protect_disable(matrix32_base()); + + matrix_configure_slave_h64mx(); + matrix_configure_slave_h32mx(); + + return matrix_configure_periph_non_secure(security_ps_peri_id, + ARRAY_SIZE(security_ps_peri_id)); +} + +void plat_primary_init_early(void) +{ + matrix_init(); +} + +void boot_primary_init_intc(void) +{ + if (atmel_saic_setup()) + panic("Failed to init interrupts\n"); +} diff --git a/optee_os/core/arch/arm/plat-sam/matrix.c b/optee_os/core/arch/arm/plat-sam/matrix.c new file mode 100644 index 0000000..c0a39e5 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/matrix.c @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: BSD-Source-Code +/* + * Copyright (c) 2013, Atmel Corporation + * Copyright (c) 2017, Timesys Corporation + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the disclaimer below. + * + * Atmel's name may not be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * DISCLAIMER: THIS SOFTWARE IS PROVIDED BY ATMEL "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE + * DISCLAIMED. IN NO EVENT SHALL ATMEL BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MATRIX_AXIMX 1 +#define MATRIX_H64MX 2 +#define MATRIX_H32MX 3 + +#define SECURITY_TYPE_AS 1 +#define SECURITY_TYPE_NS 2 +#define SECURITY_TYPE_PS 3 + +#define WORLD_NON_SECURE 0 +#define WORLD_SECURE 1 + +#define MATRIX_SPSELR_COUNT 3 +#define MATRIX_SLAVE_COUNT 15 + +struct peri_security { + unsigned int peri_id; + unsigned int matrix; + unsigned int security_type; + paddr_t addr; +}; + +static const struct peri_security peri_security_array[] = { + { + .peri_id = AT91C_ID_PMC, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_PMC, + }, + { + .peri_id = AT91C_ID_ARM, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_PIT, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_PITC, + }, + { + .peri_id = AT91C_ID_WDT, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_WDT, + }, + { + .peri_id = AT91C_ID_GMAC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_GMAC, + }, + { + .peri_id = AT91C_ID_XDMAC0, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_XDMAC0, + }, + { + .peri_id = AT91C_ID_XDMAC1, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_XDMAC1, + }, + { + .peri_id = AT91C_ID_ICM, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_ICM, + }, + { + .peri_id = AT91C_ID_AES, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_AES, + }, + { + .peri_id = AT91C_ID_AESB, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_AESB, + }, + { + .peri_id = AT91C_ID_TDES, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_TDES, + }, + { + .peri_id = AT91C_ID_SHA, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SHA, + }, + { + .peri_id = AT91C_ID_MPDDRC, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_MPDDRC, + }, + { + .peri_id = AT91C_ID_MATRIX1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_MATRIX32, + }, + { + .peri_id = AT91C_ID_MATRIX0, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_MATRIX64, + }, + { + .peri_id = AT91C_ID_SECUMOD, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_SECUMOD, + }, + { + .peri_id = AT91C_ID_HSMC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_HSMC, + }, + { + .peri_id = AT91C_ID_PIOA, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_PIOA, + }, + { + .peri_id = AT91C_ID_FLEXCOM0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_FLEXCOM0, + }, + { + .peri_id = AT91C_ID_FLEXCOM1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_FLEXCOM1, + }, + { + .peri_id = AT91C_ID_FLEXCOM2, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_FLEXCOM2, + }, + { + .peri_id = AT91C_ID_FLEXCOM3, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_FLEXCOM3, + }, + { + .peri_id = AT91C_ID_FLEXCOM4, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_FLEXCOM4, + }, + { + .peri_id = AT91C_ID_UART0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_UART0, + }, + { + .peri_id = AT91C_ID_UART1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_UART1, + }, + { + .peri_id = AT91C_ID_UART2, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_UART2, + }, + { + .peri_id = AT91C_ID_UART3, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_UART3, + }, + { + .peri_id = AT91C_ID_UART4, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_UART4, + }, + { + .peri_id = AT91C_ID_TWI0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_TWI0, + }, + { + .peri_id = AT91C_ID_TWI1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_TWI1, + }, + { + .peri_id = AT91C_ID_SDMMC0, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SDHC0, + }, + { + .peri_id = AT91C_ID_SDMMC1, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SDHC1, + }, + { + .peri_id = AT91C_ID_SPI0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SPI0, + }, + { + .peri_id = AT91C_ID_SPI1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SPI1, + }, + { + .peri_id = AT91C_ID_TC0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_TC0, + }, + { + .peri_id = AT91C_ID_TC1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_TC1, + }, + { + .peri_id = AT91C_ID_PWM, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_PWMC, + }, + { + .peri_id = AT91C_ID_ADC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_ADC, + }, + { + .peri_id = AT91C_ID_UHPHS, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_UDPHS, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_UDPHS, + }, + { + .peri_id = AT91C_ID_SSC0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SSC0, + }, + { + .peri_id = AT91C_ID_SSC1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SSC1, + }, + { + .peri_id = AT91C_ID_LCDC, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_LCDC, + }, + { + .peri_id = AT91C_ID_ISI, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_HXISI, + }, + { + .peri_id = AT91C_ID_TRNG, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_TRNG, + }, + { + .peri_id = AT91C_ID_PDMIC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_PDMIC, + }, + { + .peri_id = AT91C_ID_IRQ, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_NS, + }, + { + .peri_id = AT91C_ID_SFC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SFC, + }, + { + .peri_id = AT91C_ID_SECURAM, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_SECURAM, + }, + { + .peri_id = AT91C_ID_QSPI0, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_QSPI0, + }, + { + .peri_id = AT91C_ID_QSPI1, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_QSPI1, + }, + { + .peri_id = AT91C_ID_I2SC0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_I2SC0, + }, + { + .peri_id = AT91C_ID_I2SC1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_I2SC1, + }, + { + .peri_id = AT91C_ID_CAN0_INT0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_CAN1_INT0, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_CLASSD, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_CLASSD, + }, + { + .peri_id = AT91C_ID_SFR, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SFR, + }, + { + .peri_id = AT91C_ID_SAIC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_SAIC, + }, + { + .peri_id = AT91C_ID_AIC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_NS, + .addr = AT91C_BASE_AIC, + }, + { + .peri_id = AT91C_ID_L2CC, + .matrix = MATRIX_H64MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_L2CC, + }, + { + .peri_id = AT91C_ID_CAN0_INT1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_CAN1_INT1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_GMAC_Q1, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_GMAC_Q2, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_PIOB, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_PIOB, + }, + { + .peri_id = AT91C_ID_PIOC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_PIOC, + }, + { + .peri_id = AT91C_ID_PIOD, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_AS, + .addr = AT91C_BASE_PIOD, + }, + { + .peri_id = AT91C_ID_SDMMC0_TIMER, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_SDMMC1_TIMER, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + }, + { + .peri_id = AT91C_ID_SYS, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SYSC, + }, + { + .peri_id = AT91C_ID_ACC, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_ACC, + }, + { + .peri_id = AT91C_ID_RXLP, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_RXLP, + }, + { + .peri_id = AT91C_ID_SFRBU, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_SFRBU, + }, + { + .peri_id = AT91C_ID_CHIPID, + .matrix = MATRIX_H32MX, + .security_type = SECURITY_TYPE_PS, + .addr = AT91C_BASE_CHIPID, + }, +}; + +static void matrix_write(unsigned int base, + unsigned int offset, + const unsigned int value) +{ + io_write32(offset + base, value); +} + +static unsigned int matrix_read(int base, unsigned int offset) +{ + return io_read32(offset + base); +} + +void matrix_write_protect_enable(unsigned int matrix_base) +{ + matrix_write(matrix_base, MATRIX_WPMR, + (MATRIX_WPMR_WPKEY_PASSWD | MATRIX_WPMR_WPEN_ENABLE)); +} + +void matrix_write_protect_disable(unsigned int matrix_base) +{ + matrix_write(matrix_base, MATRIX_WPMR, MATRIX_WPMR_WPKEY_PASSWD); +} + +void matrix_configure_slave_security(unsigned int matrix_base, + unsigned int slave, + unsigned int srtop_setting, + unsigned int srsplit_setting, + unsigned int ssr_setting) +{ + matrix_write(matrix_base, MATRIX_SSR(slave), ssr_setting); + matrix_write(matrix_base, MATRIX_SRTSR(slave), srtop_setting); + matrix_write(matrix_base, MATRIX_SASSR(slave), srsplit_setting); +} + +static const struct peri_security *get_peri_security(unsigned int peri_id) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(peri_security_array); i++) { + if (peri_id == peri_security_array[i].peri_id) + return &peri_security_array[i]; + } + + return NULL; +} + +static int matrix_set_periph_world(unsigned int matrix, unsigned int peri_id, + unsigned int world) +{ + unsigned int base; + unsigned int spselr; + unsigned int idx; + unsigned int bit; + + idx = peri_id / 32; + if (idx > 3) + return -1; + + bit = (0x01 << (peri_id % 32)); + + if (matrix == MATRIX_H32MX) + base = matrix32_base(); + else if (matrix == MATRIX_H64MX) + base = matrix64_base(); + else + return -1; + + spselr = matrix_read(base, MATRIX_SPSELR(idx)); + if (world == WORLD_SECURE) + spselr &= ~bit; + else + spselr |= bit; + matrix_write(base, MATRIX_SPSELR(idx), spselr); + + return 0; +} + +TEE_Result matrix_dt_get_id(const void *fdt, int node, unsigned int *id) +{ + unsigned int i = 0; + paddr_t pbase = 0; + + pbase = fdt_reg_base_address(fdt, node); + if (pbase == DT_INFO_INVALID_REG) + return TEE_ERROR_BAD_PARAMETERS; + + for (i = 0; i < ARRAY_SIZE(peri_security_array); i++) { + if (peri_security_array[i].addr == pbase) { + *id = peri_security_array[i].peri_id; + return TEE_SUCCESS; + } + } + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +int matrix_configure_periph_secure(unsigned int peri_id) +{ + const struct peri_security *psec = NULL; + + psec = get_peri_security(peri_id); + if (!psec) + return -1; + + return matrix_set_periph_world(psec->matrix, peri_id, WORLD_SECURE); +} + +int matrix_configure_periph_non_secure(unsigned int *peri_id_array, + unsigned int size) +{ + unsigned int i; + unsigned int *peri_id_p; + unsigned int matrix; + unsigned int peri_id; + const struct peri_security *peripheral_sec; + int ret; + + if (!peri_id_array || !size) + return -1; + + peri_id_p = peri_id_array; + for (i = 0; i < size; i++) { + peripheral_sec = get_peri_security(*peri_id_p); + if (!peripheral_sec) + return -1; + + if (peripheral_sec->security_type != SECURITY_TYPE_PS) + return -1; + + matrix = peripheral_sec->matrix; + peri_id = *peri_id_p; + ret = matrix_set_periph_world(matrix, peri_id, + WORLD_NON_SECURE); + if (ret) + return -1; + + peri_id_p++; + } + + return 0; +} + +#ifdef CFG_PM_ARM32 +struct matrix_state { + uint32_t spselr[MATRIX_SPSELR_COUNT]; + uint32_t ssr[MATRIX_SLAVE_COUNT]; + uint32_t srtsr[MATRIX_SLAVE_COUNT]; + uint32_t sassr[MATRIX_SLAVE_COUNT]; + uint32_t meier; + uint32_t meimr; +}; + +static struct matrix_state matrix32_state; +static struct matrix_state matrix64_state; + +static void matrix_save_regs(vaddr_t base, struct matrix_state *state) +{ + int idx = 0; + + for (idx = 0; idx < MATRIX_SPSELR_COUNT; idx++) + state->spselr[idx] = matrix_read(base, MATRIX_SPSELR(idx)); + + for (idx = 0; idx < MATRIX_SLAVE_COUNT; idx++) { + state->ssr[idx] = matrix_read(base, MATRIX_SSR(idx)); + state->srtsr[idx] = matrix_read(base, MATRIX_SRTSR(idx)); + state->sassr[idx] = matrix_read(base, MATRIX_SASSR(idx)); + } + + state->meier = matrix_read(base, MATRIX_MEIER); + state->meimr = matrix_read(base, MATRIX_MEIMR); +} + +static void matrix_suspend(void) +{ + matrix_save_regs(matrix32_base(), &matrix32_state); + matrix_save_regs(matrix64_base(), &matrix64_state); +} + +static void matrix_restore_regs(vaddr_t base, struct matrix_state *state) +{ + int idx = 0; + + matrix_write_protect_disable(base); + + for (idx = 0; idx < MATRIX_SPSELR_COUNT; idx++) + matrix_write(base, MATRIX_SPSELR(idx), state->spselr[idx]); + + for (idx = 0; idx < MATRIX_SLAVE_COUNT; idx++) { + matrix_write(base, MATRIX_SSR(idx), state->ssr[idx]); + matrix_write(base, MATRIX_SRTSR(idx), state->srtsr[idx]); + matrix_write(base, MATRIX_SASSR(idx), state->sassr[idx]); + } + + matrix_write(base, MATRIX_MEIER, state->meier); + matrix_write(base, MATRIX_MEIMR, state->meimr); +} + +static void matrix_resume(void) +{ + matrix_restore_regs(matrix32_base(), &matrix32_state); + matrix_restore_regs(matrix64_base(), &matrix64_state); +} + +static TEE_Result matrix_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *hdl __unused) +{ + switch (op) { + case PM_OP_RESUME: + matrix_resume(); + break; + case PM_OP_SUSPEND: + matrix_suspend(); + break; + default: + panic("Invalid PM operation"); + } + + return TEE_SUCCESS; +} + +static TEE_Result matrix_pm_init(void) +{ + /* + * We can't call matrix_register_pm in matrix_init since allocator is + * not ready yet so we just call it later in this driver init callback. + */ + register_pm_driver_cb(matrix_pm, NULL, "sam-matrix"); + + return TEE_SUCCESS; +} +driver_init(matrix_pm_init); + +#endif diff --git a/optee_os/core/arch/arm/plat-sam/matrix.h b/optee_os/core/arch/arm/plat-sam/matrix.h new file mode 100644 index 0000000..2169057 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/matrix.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: BSD-Source-Code */ +/* ---------------------------------------------------------------------------- + * ATMEL Microcontroller Software Support + * ---------------------------------------------------------------------------- + * Copyright (c) 2013, Atmel Corporation + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the disclaimer below. + * + * Atmel's name may not be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * DISCLAIMER: THIS SOFTWARE IS PROVIDED BY ATMEL "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE + * DISCLAIMED. IN NO EVENT SHALL ATMEL BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef MATRIX_H +#define MATRIX_H + +#include + +extern void matrix_write_protect_enable(unsigned int matrix_base); +extern void matrix_write_protect_disable(unsigned int matrix_base); +extern void matrix_configure_slave_security(unsigned int matrix_base, + unsigned int slave, + unsigned int srtop_setting, + unsigned int srsplit_setting, + unsigned int ssr_setting); + +int matrix_configure_periph_non_secure(unsigned int *peri_id_array, + unsigned int size); +int matrix_configure_periph_secure(unsigned int peri_id); +TEE_Result matrix_dt_get_id(const void *fdt, int node, unsigned int *id); + +vaddr_t matrix32_base(void); +vaddr_t matrix64_base(void); + +#endif /* #ifndef MATRIX_H */ diff --git a/optee_os/core/arch/arm/plat-sam/nsec-service/sm_platform_handler.c b/optee_os/core/arch/arm/plat-sam/nsec-service/sm_platform_handler.c new file mode 100644 index 0000000..ff837c1 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/nsec-service/sm_platform_handler.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static enum sm_handler_ret sam_sip_handler(struct thread_smc_args *args) +{ + switch (OPTEE_SMC_FUNC_NUM(args->a0)) { + case SAMA5_SMC_SIP_SFR_SET_USB_SUSPEND: + atmel_sfr_set_usb_suspend(args->a1); + args->a0 = SAMA5_SMC_SIP_RETURN_SUCCESS; + break; + case SAMA5_SMC_SIP_SET_SUSPEND_MODE: + return at91_pm_set_suspend_mode(args); + case SAMA5_SMC_SIP_GET_SUSPEND_MODE: + return at91_pm_get_suspend_mode(args); + case SAMA5_SMC_SIP_SCMI_CALL_ID: + scmi_smt_fastcall_smc_entry(0); + args->a0 = SAMA5_SMC_SIP_RETURN_SUCCESS; + break; + default: + return SM_HANDLER_PENDING_SMC; + } + + return SM_HANDLER_SMC_HANDLED; +} + +enum sm_handler_ret sm_platform_handler(struct sm_ctx *ctx) +{ + uint32_t *nsec_r0 = (uint32_t *)(&ctx->nsec.r0); + uint16_t smc_owner = OPTEE_SMC_OWNER_NUM(*nsec_r0); + + switch (smc_owner) { + case OPTEE_SMC_OWNER_SIP: + return sam_sip_handler((struct thread_smc_args *)nsec_r0); + default: + return SM_HANDLER_PENDING_SMC; + } +} + diff --git a/optee_os/core/arch/arm/plat-sam/nsec-service/smc_ids.h b/optee_os/core/arch/arm/plat-sam/nsec-service/smc_ids.h new file mode 100644 index 0000000..17180a0 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/nsec-service/smc_ids.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Microchip + */ + +#ifndef SMC_IDS_H +#define SMC_IDS_H +#include +#include + +#define SAMA5_SMC_SIP_SCMI_CALL_ID 0x200 + +#define SAMA5_SMC_SIP_SFR_SET_USB_SUSPEND 0x300 + +#define SAMA5_SMC_SIP_SET_SUSPEND_MODE 0x400 +#define SAMA5_SMC_SIP_GET_SUSPEND_MODE 0x401 + +/* SAMA5 SMC return codes */ +#define SAMA5_SMC_SIP_RETURN_SUCCESS 0x0 +#define SAMA5_SMC_SIP_RETURN_EINVAL 0x1 + +#endif /* SMC_IDS_H */ diff --git a/optee_os/core/arch/arm/plat-sam/nsec-service/sub.mk b/optee_os/core/arch/arm/plat-sam/nsec-service/sub.mk new file mode 100644 index 0000000..3c5f232 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/nsec-service/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . + +srcs-y += sm_platform_handler.c diff --git a/optee_os/core/arch/arm/plat-sam/platform_config.h b/optee_os/core/arch/arm/plat-sam/platform_config.h new file mode 100644 index 0000000..9742c61 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/platform_config.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017 Timesys Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include +#include + +#define STACK_ALIGNMENT 64 + +#ifdef CFG_WITH_PAGER +#error "Pager not supported for platform sama5d2" +#endif +#ifdef CFG_WITH_LPAE +#error "LPAE not supported" +#endif + +#if defined(PLATFORM_FLAVOR_sama5d27_wlsom1_ek) +#define CONSOLE_UART_BASE AT91C_BASE_UART0 +#else +#define CONSOLE_UART_BASE AT91C_BASE_UART1 +#endif + +#define PL310_BASE (AT91C_BASE_L2CC) +#define SFR_BASE (AT91C_BASE_SFR) + +/* + * PL310 Auxiliary Control Register + * + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockdown cache lines (bit26=1) + * Round robin replacement policy (bit25=1) + * Force write allocated (default) + * Treats shared accesses (bit22=0, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * Platform flavor specific way config: + * - 16kb way size (bit19:17=3b001) + * Store buffer device limitation disabled (bit11=0) + * Cacheable accesses have high prio (bit10=0) + */ +#define PL310_AUX_CTRL_INIT 0x3E020000 + +/* + * PL310 Prefetch Control Register + * + * Double linefill enabled (bit30=1) + * I/D prefetch enabled (bit29:28=2b11) + * Prefetch drop enabled (bit24=1) + * Incr double linefill enable (bit23=1) + * Prefetch offset = 1 (bit4:0) + */ +#define PL310_PREFETCH_CTRL_INIT 0x71800001 + +/* + * PL310 Power Register + * + * Dynamic clock gating enabled + * Standby mode enabled + */ +#define PL310_POWER_CTRL_INIT 0x00000003 + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-sam/pm/psci.c b/optee_os/core/arch/arm/plat-sam/pm/psci.c new file mode 100644 index 0000000..3bd2471 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/pm/psci.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +int psci_system_suspend(uintptr_t entry, uint32_t context_id __unused, + struct sm_nsec_ctx *nsec) +{ + if (!atmel_pm_suspend_available()) + return PSCI_RET_NOT_SUPPORTED; + + if (atmel_pm_suspend(entry, nsec)) + return PSCI_RET_INTERNAL_FAILURE; + + return PSCI_RET_SUCCESS; +} + +int psci_cpu_suspend(uint32_t power_state, + uintptr_t entry __unused, uint32_t context_id __unused, + struct sm_nsec_ctx *nsec __unused) +{ + uint32_t type = 0; + + if (atmel_pm_suspend_available()) + return PSCI_RET_NOT_SUPPORTED; + + type = (power_state & PSCI_POWER_STATE_TYPE_MASK) >> + PSCI_POWER_STATE_TYPE_SHIFT; + + if (type != PSCI_POWER_STATE_TYPE_STANDBY) { + DMSG("Power state %x not supported", type); + return PSCI_RET_INVALID_PARAMETERS; + } + + atmel_pm_cpu_idle(); + + return PSCI_RET_SUCCESS; +} + +void __noreturn psci_system_off(void) +{ + if (!atmel_shdwc_available()) + panic(); + + atmel_shdwc_shutdown(); +} + +void __noreturn psci_system_reset(void) +{ + if (!atmel_rstc_available()) + panic(); + + atmel_rstc_reset(); +} + +int psci_features(uint32_t psci_fid) +{ + switch (psci_fid) { + case ARM_SMCCC_VERSION: + case PSCI_PSCI_FEATURES: + case PSCI_VERSION: + return PSCI_RET_SUCCESS; + case PSCI_SYSTEM_RESET: + if (atmel_rstc_available()) + return PSCI_RET_SUCCESS; + return PSCI_RET_NOT_SUPPORTED; + case PSCI_SYSTEM_OFF: + if (atmel_shdwc_available()) + return PSCI_RET_SUCCESS; + return PSCI_RET_NOT_SUPPORTED; + case PSCI_CPU_SUSPEND: + case PSCI_SYSTEM_SUSPEND: + if (atmel_pm_suspend_available()) + return PSCI_RET_SUCCESS; + return PSCI_RET_NOT_SUPPORTED; + default: + return PSCI_RET_NOT_SUPPORTED; + } +} + +uint32_t psci_version(void) +{ + return PSCI_VERSION_1_0; +} diff --git a/optee_os/core/arch/arm/plat-sam/pm/sub.mk b/optee_os/core/arch/arm/plat-sam/pm/sub.mk new file mode 100644 index 0000000..c8ae8f9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/pm/sub.mk @@ -0,0 +1 @@ +srcs-$(CFG_PSCI_ARM32) += psci.c diff --git a/optee_os/core/arch/arm/plat-sam/sam_pl310.c b/optee_os/core/arch/arm/plat-sam/sam_pl310.c new file mode 100644 index 0000000..d2cc227 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/sam_pl310.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Timesys Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, PL310_BASE, CORE_MMU_PGDIR_SIZE); + +vaddr_t pl310_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(PL310_BASE, MEM_AREA_IO_SEC, 1); + return (vaddr_t)va; + } + return PL310_BASE; +} + +void arm_cl2_config(vaddr_t pl310_base) +{ + io_write32(pl310_base + PL310_CTRL, 0); + io_write32(sam_sfr_base() + AT91_SFR_L2CC_HRAMC, 0x1); + io_write32(pl310_base + PL310_AUX_CTRL, PL310_AUX_CTRL_INIT); + io_write32(pl310_base + PL310_PREFETCH_CTRL, PL310_PREFETCH_CTRL_INIT); + io_write32(pl310_base + PL310_POWER_CTRL, PL310_POWER_CTRL_INIT); + + /* invalidate all cache ways */ + arm_cl2_invbyway(pl310_base); +} + +void arm_cl2_enable(vaddr_t pl310_base) +{ + /* Enable PL310 ctrl -> only set lsb bit */ + io_write32(pl310_base + PL310_CTRL, 1); +} diff --git a/optee_os/core/arch/arm/plat-sam/sam_sfr.c b/optee_os/core/arch/arm/plat-sam/sam_sfr.c new file mode 100644 index 0000000..3e3523a --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/sam_sfr.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Timesys Corporation. + * Copyright (C) 2021 Microchip + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SFR_BASE, CORE_MMU_PGDIR_SIZE); + +vaddr_t sam_sfr_base(void) +{ + static void *va; + + if (!cpu_mmu_enabled()) + return SFR_BASE; + + if (!va) + va = phys_to_virt(SFR_BASE, MEM_AREA_IO_SEC, 1); + + return (vaddr_t)va; +} + +void atmel_sfr_set_usb_suspend(bool set) +{ + if (set) + io_setbits32(sam_sfr_base() + AT91_SFR_OHCIICR, + AT91_OHCIICR_USB_SUSPEND); + else + io_clrbits32(sam_sfr_base() + AT91_SFR_OHCIICR, + AT91_OHCIICR_USB_SUSPEND); +} + +static TEE_Result atmel_sfr_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + if (fdt_get_status(fdt, node) == DT_STATUS_OK_SEC) + matrix_configure_periph_secure(AT91C_ID_SFR); + + return TEE_SUCCESS; +} + +static const struct dt_device_match atmel_sfr_match_table[] = { + { .compatible = "atmel,sama5d2-sfr" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_sfr_dt_driver) = { + .name = "atmel_sfr", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_sfr_match_table, + .probe = atmel_sfr_probe, +}; diff --git a/optee_os/core/arch/arm/plat-sam/sam_sfr.h b/optee_os/core/arch/arm/plat-sam/sam_sfr.h new file mode 100644 index 0000000..c409102 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/sam_sfr.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Bootlin + */ + +#ifndef SAM_SFR_H +#define SAM_SFR_H + +#include +#include + +/* OHCI INT Configuration Register */ +#define AT91_SFR_OHCIICR 0x10 +/* UTMI Clock Trimming Register */ +#define AT91_SFR_UTMICKTRIM 0x30 +/* Serial number 0 Register */ +#define AT91_SFR_SN0 0x4c +/* Serial number 1 Register */ +#define AT91_SFR_SN1 0x50 +/* AIC Interrupt Redirection Register */ +#define AT91_SFR_AICREDIR 0x54 +/* L2 cache RAM used as an internal SRAM */ +#define AT91_SFR_L2CC_HRAMC 0x58 +/* I2SC Register */ +#define AT91_SFR_I2SCLKSEL 0x90 + +/* Field definitions */ +#define AT91_UTMICKTRIM_FREQ GENMASK_32(1, 0) + +#define AT91_OHCIICR_USB_SUSPEND GENMASK_32(10, 8) + +#define AT91_SFR_AICREDIR_XOR_KEY 0xb6d81c4d +#define AT91_SFR_AICREDIR_KEY_MASK GENMASK_32(31, 1) + +vaddr_t sam_sfr_base(void); + +void atmel_sfr_set_usb_suspend(bool set); + +#endif diff --git a/optee_os/core/arch/arm/plat-sam/sama5d2.h b/optee_os/core/arch/arm/plat-sam/sama5d2.h new file mode 100644 index 0000000..d7d1451 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/sama5d2.h @@ -0,0 +1,274 @@ +/* SPDX-License-Identifier: BSD-Source-Code */ +/* + * Copyright (c) 2015, Atmel Corporation + * Copyright (c) 2017, Timesys Corporation + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the disclaimer below. + * + * Atmel's name may not be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * DISCLAIMER: THIS SOFTWARE IS PROVIDED BY ATMEL "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE + * DISCLAIMED. IN NO EVENT SHALL ATMEL BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SAMA5D2_H +#define SAMA5D2_H + +/* + * Peripheral identifiers/interrupts. + */ +#define AT91C_ID_FIQ 0 /* FIQ Interrupt ID */ +#define AT91C_ID_PMC 1 /* Power Management Controller */ +#define AT91C_ID_ARM 2 /* Performance Monitor Unit */ +#define AT91C_ID_PIT 3 /* Periodic Interval Timer Interrupt */ +#define AT91C_ID_WDT 4 /* Watchdog Timer Interrupt */ +#define AT91C_ID_GMAC 5 /* Ethernet MAC */ +#define AT91C_ID_XDMAC0 6 /* DMA Controller 0 */ +#define AT91C_ID_XDMAC1 7 /* DMA Controller 1 */ +#define AT91C_ID_ICM 8 /* Integrity Check Monitor */ +#define AT91C_ID_AES 9 /* Advanced Encryption Standard */ +#define AT91C_ID_AESB 10 /* AES bridge */ +#define AT91C_ID_TDES 11 /* Triple Data Encryption Standard */ +#define AT91C_ID_SHA 12 /* SHA Signature */ +#define AT91C_ID_MPDDRC 13 /* MPDDR Controller */ +#define AT91C_ID_MATRIX1 14 /* H32MX, 32-bit AHB Matrix */ +#define AT91C_ID_MATRIX0 15 /* H64MX, 64-bit AHB Matrix */ +#define AT91C_ID_SECUMOD 16 /* Secure Module */ +#define AT91C_ID_HSMC 17 /* Multi-bit ECC interrupt */ +#define AT91C_ID_PIOA 18 /* Parallel I/O Controller A */ +#define AT91C_ID_FLEXCOM0 19 /* FLEXCOM0 */ +#define AT91C_ID_FLEXCOM1 20 /* FLEXCOM1 */ +#define AT91C_ID_FLEXCOM2 21 /* FLEXCOM2 */ +#define AT91C_ID_FLEXCOM3 22 /* FLEXCOM3 */ +#define AT91C_ID_FLEXCOM4 23 /* FLEXCOM4 */ +#define AT91C_ID_UART0 24 /* UART0 */ +#define AT91C_ID_UART1 25 /* UART1 */ +#define AT91C_ID_UART2 26 /* UART2 */ +#define AT91C_ID_UART3 27 /* UART3 */ +#define AT91C_ID_UART4 28 /* UART4 */ +#define AT91C_ID_TWI0 29 /* Two-wire Interface 0 */ +#define AT91C_ID_TWI1 30 /* Two-wire Interface 1 */ +#define AT91C_ID_SDMMC0 31 /* SDMMC Controller 0 */ +#define AT91C_ID_SDMMC1 32 /* SDMMC Controller 1 */ +#define AT91C_ID_SPI0 33 /* Serial Peripheral Interface 0 */ +#define AT91C_ID_SPI1 34 /* Serial Peripheral Interface 1 */ +#define AT91C_ID_TC0 35 /* Timer Counter 0 (ch.0,1,2) */ +#define AT91C_ID_TC1 36 /* Timer Counter 1 (ch.3,4,5) */ +/* 37 */ +#define AT91C_ID_PWM 38 /* PWM Controller0 (ch. 0,1,2,3) */ +/* 39 */ +#define AT91C_ID_ADC 40 /* Touch Screen ADC Controller */ +#define AT91C_ID_UHPHS 41 /* USB Host High Speed */ +#define AT91C_ID_UDPHS 42 /* USB Device High Speed */ +#define AT91C_ID_SSC0 43 /* Serial Synchronous Controller 0 */ +#define AT91C_ID_SSC1 44 /* Serial Synchronous Controller 1 */ +#define AT91C_ID_LCDC 45 /* LCD Controller */ +#define AT91C_ID_ISI 46 /* Image Sensor Interface */ +#define AT91C_ID_TRNG 47 /* True Random Number Generator */ +#define AT91C_ID_PDMIC 48 /* PDM Interface Controller */ +#define AT91C_ID_IRQ 49 /* IRQ Interrupt ID */ +#define AT91C_ID_SFC 50 /* Fuse Controller */ +#define AT91C_ID_SECURAM 51 /* Secure RAM */ +#define AT91C_ID_QSPI0 52 /* QSPI0 */ +#define AT91C_ID_QSPI1 53 /* QSPI1 */ +#define AT91C_ID_I2SC0 54 /* Inter-IC Sound Controller 0 */ +#define AT91C_ID_I2SC1 55 /* Inter-IC Sound Controller 1 */ +#define AT91C_ID_CAN0_INT0 56 /* MCAN 0 Interrupt0 */ +#define AT91C_ID_CAN1_INT0 57 /* MCAN 1 Interrupt0 */ +#define AT91C_ID_PTC 58 /* Peripheral Touch Controller */ +#define AT91C_ID_CLASSD 59 /* Audio Class D Amplifier */ +#define AT91C_ID_SFR 60 /* Special Function Register */ +#define AT91C_ID_SAIC 61 /* Secured AIC */ +#define AT91C_ID_AIC 62 /* Advanced Interrupt Controller */ +#define AT91C_ID_L2CC 63 /* L2 Cache Controller */ +#define AT91C_ID_CAN0_INT1 64 /* MCAN 0 Interrupt1 */ +#define AT91C_ID_CAN1_INT1 65 /* MCAN 1 Interrupt1 */ +#define AT91C_ID_GMAC_Q1 66 /* GMAC Queue 1 Interrupt */ +#define AT91C_ID_GMAC_Q2 67 /* GMAC Queue 2 Interrupt */ +#define AT91C_ID_PIOB 68 /* Parallel I/O Controller B */ +#define AT91C_ID_PIOC 69 /* Parallel I/O Controller C */ +#define AT91C_ID_PIOD 70 /* Parallel I/O Controller D */ +#define AT91C_ID_SDMMC0_TIMER 71 /* SDMMC0 Timer */ +#define AT91C_ID_SDMMC1_TIMER 72 /* SDMMC1 Timer */ +/* 73 */ +#define AT91C_ID_SYS 74 /* System Controller Interrupt */ +#define AT91C_ID_ACC 75 /* Analog Comparator */ +#define AT91C_ID_RXLP 76 /* UART Low-Power */ +#define AT91C_ID_SFRBU 77 /* Special Function Register BackUp */ +#define AT91C_ID_CHIPID 78 /* Chip ID */ + +#define AT91C_ID_COUNTS (AT91C_ID_CHIPID + 1) + +/* + * User Peripherals physical base addresses. + */ +#define AT91C_BASE_LCDC 0xf0000000 +#define AT91C_BASE_XDMAC1 0xf0004000 +#define AT91C_BASE_HXISI 0xf0008000 +#define AT91C_BASE_MPDDRC 0xf000c000 +#define AT91C_BASE_XDMAC0 0xf0010000 +#define AT91C_BASE_PMC 0xf0014000 +#define AT91C_BASE_MATRIX64 0xf0018000 /* MATRIX0 */ +#define AT91C_BASE_AESB 0xf001c000 +#define AT91C_BASE_QSPI0 0xf0020000 +#define AT91C_BASE_QSPI1 0xf0024000 +#define AT91C_BASE_SHA 0xf0028000 +#define AT91C_BASE_AES 0xf002c000 + +#define AT91C_BASE_SPI0 0xf8000000 +#define AT91C_BASE_SSC0 0xf8004000 +#define AT91C_BASE_GMAC 0xf8008000 +#define AT91C_BASE_TC0 0xf800c000 +#define AT91C_BASE_TC1 0xf8010000 +#define AT91C_BASE_HSMC 0xf8014000 +#define AT91C_BASE_PDMIC 0xf8018000 +#define AT91C_BASE_UART0 0xf801c000 +#define AT91C_BASE_UART1 0xf8020000 +#define AT91C_BASE_UART2 0xf8024000 +#define AT91C_BASE_TWI0 0xf8028000 +#define AT91C_BASE_PWMC 0xf802c000 +#define AT91C_BASE_SFR 0xf8030000 +#define AT91C_BASE_FLEXCOM0 0xf8034000 +#define AT91C_BASE_FLEXCOM1 0xf8038000 +#define AT91C_BASE_SAIC 0xf803c000 +#define AT91C_BASE_ICM 0xf8040000 +#define AT91C_BASE_SECURAM 0xf8044000 +#define AT91C_BASE_SYSC 0xf8048000 +#define AT91C_BASE_ACC 0xf804a000 +#define AT91C_BASE_RXLP 0xf8049000 +#define AT91C_BASE_SFC 0xf804c000 +#define AT91C_BASE_I2SC0 0xf8050000 +#define AT91C_BASE_CAN0 0xf8054000 + +#define AT91C_BASE_SPI1 0xfc000000 +#define AT91C_BASE_SSC1 0xfc004000 +#define AT91C_BASE_UART3 0xfc008000 +#define AT91C_BASE_UART4 0xfc00c000 +#define AT91C_BASE_FLEXCOM2 0xfc010000 +#define AT91C_BASE_FLEXCOM3 0xfc014000 +#define AT91C_BASE_FLEXCOM4 0xfc018000 +#define AT91C_BASE_TRNG 0xfc01c000 +#define AT91C_BASE_AIC 0xfc020000 +#define AT91C_BASE_TWI1 0xfc028000 +#define AT91C_BASE_UDPHS 0xfc02c000 +#define AT91C_BASE_ADC 0xfc030000 + +#define AT91C_BASE_PIOA 0xfc038000 +#define AT91C_BASE_MATRIX32 0xfc03c000 /* MATRIX1 */ +#define AT91C_BASE_SECUMOD 0xfc040000 +#define AT91C_BASE_TDES 0xfc044000 +#define AT91C_BASE_CLASSD 0xfc048000 +#define AT91C_BASE_I2SC1 0xfc04c000 +#define AT91C_BASE_CAN1 0xfc050000 +#define AT91C_BASE_SFRBU 0xfc05c000 +#define AT91C_BASE_CHIPID 0xfc069000 + +/* + * Address Memory Space + */ +#define AT91C_BASE_INTERNAL_MEM 0x00000000 +#define AT91C_BASE_CS0 0x10000000 +#define AT91C_BASE_DDRCS 0x20000000 +#define AT91C_BASE_DDRCS_AES 0x40000000 +#define AT91C_BASE_CS1 0x60000000 +#define AT91C_BASE_CS2 0x70000000 +#define AT91C_BASE_CS3 0x80000000 +#define AT91C_BASE_QSPI0_AES_MEM 0x90000000 +#define AT91C_BASE_QSPI1_AES_MEM 0x98000000 +#define AT91C_BASE_SDHC0 0xa0000000 +#define AT91C_BASE_SDHC1 0xb0000000 +#define AT91C_BASE_NFC_CMD_REG 0xc0000000 +#define AT91C_BASE_QSPI0_MEM 0xd0000000 +#define AT91C_BASE_QSPI1_MEM 0xd8000000 +#define AT91C_BASE_PERIPH 0xf0000000 + +/* + * Internal Memories + */ +#define AT91C_BASE_ROM 0x00000000 /* ROM */ +#define AT91C_BASE_ECC_ROM 0x00060000 /* ECC ROM */ +#define AT91C_BASE_NFC_SRAM 0x00100000 /* NFC SRAM */ +#define AT91C_BASE_SRAM0 0x00200000 /* SRAM0 */ +#define AT91C_BASE_SRAM1 0x00220000 /* SRAM1 */ +#define AT91C_BASE_UDPHS_SRAM 0x00300000 /* UDPHS RAM */ +#define AT91C_BASE_UHP_OHCI 0x00400000 /* UHP OHCI */ +#define AT91C_BASE_UHP_EHCI 0x00500000 /* UHP EHCI */ +#define AT91C_BASE_AXI_MATRIX 0x00600000 /* AXI Maxtrix */ +#define AT91C_BASE_DAP 0x00700000 /* DAP */ +#define AT91C_BASE_PTC 0x00800000 /* PTC */ +#define AT91C_BASE_L2CC 0x00A00000 /* L2CC */ + +/* + * Other misc defines + */ +#define AT91C_BASE_PMECC (AT91C_BASE_HSMC + 0x70) +#define AT91C_BASE_PMERRLOC (AT91C_BASE_HSMC + 0x500) + +#define AT91_PMECC (AT91C_BASE_PMECC - AT91C_BASE_SYS) +#define AT91_PMERRLOC (AT91C_BASE_PMERRLOC - AT91C_BASE_SYS) + +#define AT91C_BASE_PIOB (AT91C_BASE_PIOA + 0x40) +#define AT91C_BASE_PIOC (AT91C_BASE_PIOB + 0x40) +#define AT91C_BASE_PIOD (AT91C_BASE_PIOC + 0x40) + +/* SYSC spawns */ +#define AT91C_BASE_RSTC AT91C_BASE_SYSC +#define AT91C_BASE_SHDC (AT91C_BASE_SYSC + 0x10) +#define AT91C_BASE_PITC (AT91C_BASE_SYSC + 0x30) +#define AT91C_BASE_WDT (AT91C_BASE_SYSC + 0x40) +#define AT91C_BASE_SCKCR (AT91C_BASE_SYSC + 0x50) +#define AT91C_BASE_RTCC (AT91C_BASE_SYSC + 0xb0) + +#define ATMEL_BASE_SMC (AT91C_BASE_HSMC + 0x700) + +#define AT91C_NUM_PIO 4 +#define AT91C_NUM_TWI 2 + +/* AICREDIR Unlock Key */ +#define AICREDIR_KEY 0xB6D81C4D + +/* + * Matrix Slaves ID + */ +/* MATRIX0(H64MX) Matrix Slaves */ +/* Bridge from H64MX to AXIMX (Internal ROM, Cryto Library, PKCC RAM) */ +#define H64MX_SLAVE_BRIDGE_TO_AXIMX 0 +#define H64MX_SLAVE_PERI_BRIDGE 1 /* H64MX Peripheral Bridge */ +#define H64MX_SLAVE_DDR2_PORT_0 2 /* DDR2 Port0-AESOTF */ +#define H64MX_SLAVE_DDR2_PORT_1 3 /* DDR2 Port1 */ +#define H64MX_SLAVE_DDR2_PORT_2 4 /* DDR2 Port2 */ +#define H64MX_SLAVE_DDR2_PORT_3 5 /* DDR2 Port3 */ +#define H64MX_SLAVE_DDR2_PORT_4 6 /* DDR2 Port4 */ +#define H64MX_SLAVE_DDR2_PORT_5 7 /* DDR2 Port5 */ +#define H64MX_SLAVE_DDR2_PORT_6 8 /* DDR2 Port6 */ +#define H64MX_SLAVE_DDR2_PORT_7 9 /* DDR2 Port7 */ +#define H64MX_SLAVE_INTERNAL_SRAM 10 /* Internal SRAM 128K */ +#define H64MX_SLAVE_CACHE_L2 11 /* Internal SRAM 128K (L2) */ +#define H64MX_SLAVE_QSPI0 12 /* QSPI0 */ +#define H64MX_SLAVE_QSPI1 13 /* QSPI1 */ +#define H64MX_SLAVE_AESB 14 /* AESB */ + +/* MATRIX1(H32MX) Matrix Slaves */ +#define H32MX_BRIDGE_TO_H64MX 0 /* Bridge from H32MX to H64MX */ +#define H32MX_PERI_BRIDGE_0 1 /* H32MX Peripheral Bridge 0 */ +#define H32MX_PERI_BRIDGE_1 2 /* H32MX Peripheral Bridge 1 */ +#define H32MX_EXTERNAL_EBI 3 /* External Bus Interface */ +#define H32MX_NFC_CMD_REG 3 /* NFC command Register */ +#define H32MX_NFC_SRAM 4 /* NFC SRAM */ +#define H32MX_USB 5 + +#endif /* #ifndef SAMA5D2_H */ diff --git a/optee_os/core/arch/arm/plat-sam/scmi_server.c b/optee_os/core/arch/arm/plat-sam/scmi_server.c new file mode 100644 index 0000000..06e2ee6 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/scmi_server.c @@ -0,0 +1,552 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, STMicroelectronics + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include + +static_assert(SMT_BUF_SLOT_SIZE <= CFG_SCMI_SHMEM_SIZE); + +register_phys_mem(MEM_AREA_IO_NSEC, CFG_SCMI_SHMEM_START, CFG_SCMI_SHMEM_SIZE); + +struct channel_resources { + struct scmi_msg_channel *channel; +}; + +static const struct channel_resources scmi_channel[] = { + [0] = { + .channel = &(struct scmi_msg_channel){ + .shm_addr = { .pa = CFG_SCMI_SHMEM_START }, + .shm_size = SMT_BUF_SLOT_SIZE, + }, + }, +}; + +static const struct channel_resources *find_resource(unsigned int channel_id) +{ + assert(channel_id < ARRAY_SIZE(scmi_channel)); + + return scmi_channel + channel_id; +} + +struct scmi_msg_channel *plat_scmi_get_channel(unsigned int channel_id) +{ + const size_t max_id = ARRAY_SIZE(scmi_channel); + unsigned int confined_id = confine_array_index(channel_id, max_id); + + if (channel_id >= max_id) + return NULL; + + return find_resource(confined_id)->channel; +} + +static const char vendor[] = "Microchip"; +static const char sub_vendor[] = ""; + +const char *plat_scmi_vendor_name(void) +{ + return vendor; +} + +const char *plat_scmi_sub_vendor_name(void) +{ + return sub_vendor; +} + +/* Currently supporting only SCMI Base protocol */ +static const uint8_t plat_protocol_list[] = { + SCMI_PROTOCOL_ID_CLOCK, + 0 /* Null termination */ +}; + +size_t plat_scmi_protocol_count(void) +{ + return ARRAY_SIZE(plat_protocol_list) - 1; +} + +const uint8_t *plat_scmi_protocol_list(unsigned int channel_id __unused) +{ + return plat_protocol_list; +} + +struct sama5d2_pmc_clk { + unsigned int scmi_id; + unsigned int pmc_type; + unsigned int pmc_id; +}; + +static const struct sama5d2_pmc_clk pmc_clks[] = { + { + .scmi_id = AT91_SCMI_CLK_CORE_MCK, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_MCK + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_UTMI, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_UTMI + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_MAIN, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_MAIN + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_MCK2, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_MCK2 + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_I2S0_MUX, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_I2S0_MUX + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_I2S1_MUX, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_I2S1_MUX + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_PLLACK, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_PLLACK + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_AUDIOPLLCK, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_AUDIOPLLCK + }, + { + .scmi_id = AT91_SCMI_CLK_CORE_MCK_PRES, + .pmc_type = PMC_TYPE_CORE, + .pmc_id = PMC_MCK_PRES + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_DDRCK, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 2 + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_LCDCK, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 3 + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_UHPCK, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 6 + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_UDPCK, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 7 + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_PCK0, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 8 + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_PCK1, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 9 + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_PCK2, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 10 + }, + { + .scmi_id = AT91_SCMI_CLK_SYSTEM_ISCCK, + .pmc_type = PMC_TYPE_SYSTEM, + .pmc_id = 18 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_MACB0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 5 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_TDES_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 11 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_MATRIX1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 14 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_HSMC_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 17 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_PIOA_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 18 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_FLX0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 19 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_FLX1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 20 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_FLX2_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 21 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_FLX3_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 22 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_FLX4_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 23 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_UART0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 24 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_UART1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 25 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_UART2_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 26 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_UART3_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 27 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_UART4_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 28 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_TWI0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 29 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_TWI1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 30 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SPI0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 33 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SPI1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 34 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_TCB0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 35 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_TCB1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 36 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_PWM_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 38 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_ADC_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 40 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_UHPHS_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 41 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_UDPHS_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 42 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SSC0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 43 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SSC1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 44 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_TRNG_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 47 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_PDMIC_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 48 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SECURAM_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 51 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_I2S0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 54 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_I2S1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 55 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_CAN0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 56 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_CAN1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 57 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_PTC_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 58 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_CLASSD_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 59 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_DMA0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 6 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_DMA1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 7 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_AES_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 9 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_AESB_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 10 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SHA_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 12 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_MPDDR_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 13 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_MATRIX0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 15 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SDMMC0_HCLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 31 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_SDMMC1_HCLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 32 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_LCDC_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 45 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_ISC_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 46 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_QSPI0_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 52 + }, + { + .scmi_id = AT91_SCMI_CLK_PERIPH_QSPI1_CLK, + .pmc_type = PMC_TYPE_PERIPHERAL, + .pmc_id = 53 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_SDMMC0_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 31 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_SDMMC1_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 32 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_TCB0_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 35 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_TCB1_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 36 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_PWM_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 38 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_ISC_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 46 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_PDMIC_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 48 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_I2S0_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 54 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_I2S1_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 55 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_CAN0_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 56 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_CAN1_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 57 + }, + { + .scmi_id = AT91_SCMI_CLK_GCK_CLASSD_GCLK, + .pmc_type = PMC_TYPE_GCK, + .pmc_id = 59 + }, + { + .scmi_id = AT91_SCMI_CLK_PROG_PROG0, + .pmc_type = PMC_TYPE_PROGRAMMABLE, + .pmc_id = 0 + }, + { + .scmi_id = AT91_SCMI_CLK_PROG_PROG1, + .pmc_type = PMC_TYPE_PROGRAMMABLE, + .pmc_id = 1 + }, + { + .scmi_id = AT91_SCMI_CLK_PROG_PROG2, + .pmc_type = PMC_TYPE_PROGRAMMABLE, + .pmc_id = 2 + }, +}; + +static TEE_Result sam_init_scmi_clk(void) +{ + unsigned int i = 0; + struct clk *clk = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + const struct sama5d2_pmc_clk *pmc_clk = NULL; + + for (i = 0; i < ARRAY_SIZE(pmc_clks); i++) { + pmc_clk = &pmc_clks[i]; + res = at91_pmc_clk_get(pmc_clk->pmc_type, pmc_clk->pmc_id, + &clk); + if (res) { + EMSG("Failed to get PMC clock type %u, id %u", + pmc_clk->pmc_type, pmc_clk->pmc_id); + return res; + } + res = scmi_clk_add(clk, 0, pmc_clk->scmi_id); + if (res) { + EMSG("Failed to add PMC SCMI clock id %u", + pmc_clk->scmi_id); + return res; + } + } + + clk = at91_sckc_clk_get(); + if (!clk) + return TEE_ERROR_GENERIC; + + res = scmi_clk_add(clk, 0, AT91_SCMI_CLK_SCKC_SLOWCK_32K); + if (res) { + EMSG("Failed to add slow clock to SCMI clocks"); + return res; + } + + return TEE_SUCCESS; +} + +/* + * Initialize platform SCMI resources + */ +static TEE_Result sam_init_scmi_server(void) +{ + size_t i = 0; + + for (i = 0; i < ARRAY_SIZE(scmi_channel); i++) { + const struct channel_resources *res = scmi_channel + i; + struct scmi_msg_channel *chan = res->channel; + + /* Enforce non-secure shm mapped as device memory */ + chan->shm_addr.va = (vaddr_t)phys_to_virt(chan->shm_addr.pa, + MEM_AREA_IO_NSEC, 1); + assert(chan->shm_addr.va); + + scmi_smt_init_agent_channel(chan); + } + + return sam_init_scmi_clk(); +} + +driver_init_late(sam_init_scmi_server); diff --git a/optee_os/core/arch/arm/plat-sam/sub.mk b/optee_os/core/arch/arm/plat-sam/sub.mk new file mode 100644 index 0000000..a067d68 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/sub.mk @@ -0,0 +1,8 @@ +global-incdirs-y += . +srcs-y += main.c freq.c sam_sfr.c +srcs-$(CFG_AT91_MATRIX) += matrix.c +srcs-$(CFG_PL310) += sam_pl310.c +srcs-$(CFG_SCMI_MSG_DRIVERS) += scmi_server.c + +subdirs-y += pm +subdirs-y += nsec-service diff --git a/optee_os/core/arch/arm/plat-sam/tz_matrix.h b/optee_os/core/arch/arm/plat-sam/tz_matrix.h new file mode 100644 index 0000000..7db2e3d --- /dev/null +++ b/optee_os/core/arch/arm/plat-sam/tz_matrix.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: BSD-Source-Code */ +/* + * Copyright (c) 2013, Atmel Corporation + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the disclaimer below. + * + * Atmel's name may not be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * DISCLAIMER: THIS SOFTWARE IS PROVIDED BY ATMEL "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE + * DISCLAIMED. IN NO EVENT SHALL ATMEL BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef TZ_MATRIX_H +#define TZ_MATRIX_H + +#define MATRIX_MCFG(n) (0x0000 + (n) * 4) /* Master Configuration Register */ +#define MATRIX_SCFG(n) (0x0040 + (n) * 4) /* Slave Configuration Register */ +#define MATRIX_PRAS(n) (0x0080 + (n) * 8) /* Priority Register A for Slave */ +#define MATRIX_PRBS(n) (0x0084 + (n) * 8) /* Priority Register B for Slave */ + +#define MATRIX_MRCR 0x0100 /* Master Remap Control Register */ +#define MATRIX_MEIER 0x0150 /* Master Error Interrupt Enable Register */ +#define MATRIX_MEIDR 0x0154 /* Master Error Interrupt Disable Register */ +#define MATRIX_MEIMR 0x0158 /* Master Error Interrupt Mask Register */ +#define MATRIX_MESR 0x015c /* Master Error Status Register */ + +/* Master n Error Address Register */ +#define MATRIX_MEAR(n) (0x0160 + (n) * 4) + +#define MATRIX_WPMR 0x01E4 /* Write Protect Mode Register */ +#define MATRIX_WPSR 0x01E8 /* Write Protect Status Register */ + +/* Security Slave n Register */ +#define MATRIX_SSR(n) (0x0200 + (n) * 4) +/* Security Area Split Slave n Register */ +#define MATRIX_SASSR(n) (0x0240 + (n) * 4) +/* Security Region Top Slave n Register */ +#define MATRIX_SRTSR(n) (0x0280 + (n) * 4) + +/* Security Peripheral Select n Register */ +#define MATRIX_SPSELR(n) (0x02c0 + (n) * 4) + +/**************************************************************************/ +/* Write Protect Mode Register (MATRIX_WPMR) */ +#define MATRIX_WPMR_WPEN (1 << 0) /* Write Protect Enable */ +#define MATRIX_WPMR_WPEN_DISABLE (0 << 0) +#define MATRIX_WPMR_WPEN_ENABLE (1 << 0) +#define MATRIX_WPMR_WPKEY (PASSWD << 8) /* Write Protect KEY */ +#define MATRIX_WPMR_WPKEY_PASSWD (0x4D4154 << 8) + +/* Security Slave Registers (MATRIX_SSRx) */ +#define MATRIX_LANSECH(n, bit) ((bit) << n) +#define MATRIX_LANSECH_S(n) (0x00 << n) +#define MATRIX_LANSECH_NS(n) (0x01 << n) +#define MATRIX_RDNSECH(n, bit) ((bit) << (n + 8)) +#define MATRIX_RDNSECH_S(n) (0x00 << (n + 8)) +#define MATRIX_RDNSECH_NS(n) (0x01 << (n + 8)) +#define MATRIX_WRNSECH(n, bit) ((bit) << (n + 16)) +#define MATRIX_WRNSECH_S(n) (0x00 << (n + 16)) +#define MATRIX_WRNSECH_NS(n) (0x01 << (n + 16)) + +/* Security Areas Split Slave Registers (MATRIX_SASSRx) */ +#define MATRIX_SASPLIT(n, value) ((value) << (4 * n)) +#define MATRIX_SASPLIT_VALUE_4K 0x00 +#define MATRIX_SASPLIT_VALUE_8K 0x01 +#define MATRIX_SASPLIT_VALUE_16K 0x02 +#define MATRIX_SASPLIT_VALUE_32K 0x03 +#define MATRIX_SASPLIT_VALUE_64K 0x04 +#define MATRIX_SASPLIT_VALUE_128K 0x05 +#define MATRIX_SASPLIT_VALUE_256K 0x06 +#define MATRIX_SASPLIT_VALUE_512K 0x07 +#define MATRIX_SASPLIT_VALUE_1M 0x08 +#define MATRIX_SASPLIT_VALUE_2M 0x09 +#define MATRIX_SASPLIT_VALUE_4M 0x0a +#define MATRIX_SASPLIT_VALUE_8M 0x0b +#define MATRIX_SASPLIT_VALUE_16M 0x0c +#define MATRIX_SASPLIT_VALUE_32M 0x0d +#define MATRIX_SASPLIT_VALUE_64M 0x0e +#define MATRIX_SASPLIT_VALUE_128M 0x0f + +/* Security Region Top Slave Registers (MATRIX_SRTSRx) */ +#define MATRIX_SRTOP(n, value) ((value) << (4 * n)) +#define MATRIX_SRTOP_VALUE_4K 0x00 +#define MATRIX_SRTOP_VALUE_8K 0x01 +#define MATRIX_SRTOP_VALUE_16K 0x02 +#define MATRIX_SRTOP_VALUE_32K 0x03 +#define MATRIX_SRTOP_VALUE_64K 0x04 +#define MATRIX_SRTOP_VALUE_128K 0x05 +#define MATRIX_SRTOP_VALUE_256K 0x06 +#define MATRIX_SRTOP_VALUE_512K 0x07 +#define MATRIX_SRTOP_VALUE_1M 0x08 +#define MATRIX_SRTOP_VALUE_2M 0x09 +#define MATRIX_SRTOP_VALUE_4M 0x0a +#define MATRIX_SRTOP_VALUE_8M 0x0b +#define MATRIX_SRTOP_VALUE_16M 0x0c +#define MATRIX_SRTOP_VALUE_32M 0x0d +#define MATRIX_SRTOP_VALUE_64M 0x0e +#define MATRIX_SRTOP_VALUE_128M 0x0f + +#endif /* #ifndef TZ_MATRIX_H */ diff --git a/optee_os/core/arch/arm/plat-sprd/conf.mk b/optee_os/core/arch/arm/plat-sprd/conf.mk new file mode 100644 index 0000000..8faae60 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sprd/conf.mk @@ -0,0 +1,12 @@ +PLATFORM_FLAVOR ?= sc9860 + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_TEE_CORE_NB_CORE,8) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_GIC,y) +$(call force,CFG_SPRD_UART,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) + +# Overrides default in mk/config.mk with 128 kB +CFG_CORE_HEAP_SIZE ?= 131072 diff --git a/optee_os/core/arch/arm/plat-sprd/console.c b/optee_os/core/arch/arm/plat-sprd/console.c new file mode 100644 index 0000000..15a4b3b --- /dev/null +++ b/optee_os/core/arch/arm/plat-sprd/console.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Spreadtrum Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +static struct sprd_uart_data console_data; + +void console_init(void) +{ + sprd_uart_init(&console_data, CONSOLE_UART_BASE); + register_serial_console(&console_data.chip); +} + +void console_putc(int ch) +{ + struct serial_chip *cons = &console_data.chip; + + cons->ops->putc(cons, ch & 0xff); +} + diff --git a/optee_os/core/arch/arm/plat-sprd/main.c b/optee_os/core/arch/arm/plat-sprd/main.c new file mode 100644 index 0000000..a8e0a12 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sprd/main.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Spreadtrum Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, + ROUNDDOWN(CONSOLE_UART_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(GIC_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(GIC_BASE + GICD_OFFSET, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + diff --git a/optee_os/core/arch/arm/plat-sprd/platform_config.h b/optee_os/core/arch/arm/plat-sprd/platform_config.h new file mode 100644 index 0000000..f6af23d --- /dev/null +++ b/optee_os/core/arch/arm/plat-sprd/platform_config.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Spreadtrum Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#ifdef ARM64 +#ifdef CFG_WITH_PAGER +#error "Pager not supported for ARM64" +#endif +#endif /*ARM64*/ + +#if defined(PLATFORM_FLAVOR_sc9860) + +#define GIC_BASE 0x12000000 +#define UART0_BASE 0x70000000 +#define UART1_BASE 0x70100000 +#define UART2_BASE 0x70200000 +#define UART3_BASE 0x70300000 + +#define CONSOLE_UART_BASE UART1_BASE + +#define DRAM0_BASE 0x80000000 +#define DRAM0_SIZE 0x20000000 + +#define TZDRAM_BASE 0x8f600000 +#define TZDRAM_SIZE (0x02000000 - TEE_SHMEM_SIZE) + +#define TEE_SHMEM_START (TZDRAM_BASE + TZDRAM_SIZE) +#define TEE_SHMEM_SIZE 0x200000 + +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 + +#else +#error "Unknown platform flavor" +#endif + +#define TEE_RAM_VA_SIZE (1024 * 1024) + +#ifdef CFG_TEE_LOAD_ADDR +#define TEE_LOAD_ADDR CFG_TEE_LOAD_ADDR +#else +#define TEE_LOAD_ADDR TEE_RAM_START +#endif +/* + * +------------------+ + * | | TEE_RAM | + * + TZDRAM +---------+ + * | | TA_RAM | + * +--------+---------+ + */ +#define TEE_RAM_PH_SIZE TEE_RAM_VA_SIZE +#define TEE_RAM_START TZDRAM_BASE +#define TA_RAM_START ROUNDUP((TZDRAM_BASE + TEE_RAM_VA_SIZE), \ + CORE_MMU_PGDIR_SIZE) +#define TA_RAM_SIZE ROUNDDOWN((TZDRAM_SIZE - TEE_RAM_VA_SIZE), \ + CORE_MMU_PGDIR_SIZE) + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-sprd/sub.mk b/optee_os/core/arch/arm/plat-sprd/sub.mk new file mode 100644 index 0000000..3a8214b --- /dev/null +++ b/optee_os/core/arch/arm/plat-sprd/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-y += console.c diff --git a/optee_os/core/arch/arm/plat-stm/.gitignore b/optee_os/core/arch/arm/plat-stm/.gitignore new file mode 100644 index 0000000..49b7bb9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm/.gitignore @@ -0,0 +1 @@ +System.map diff --git a/optee_os/core/arch/arm/plat-stm/conf.mk b/optee_os/core/arch/arm/plat-stm/conf.mk new file mode 100644 index 0000000..4e78f36 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm/conf.mk @@ -0,0 +1,45 @@ +PLATFORM_FLAVOR ?= b2260 + +include core/arch/arm/cpu/cortex-a9.mk + +$(call force,CFG_ARM32_core,y) +$(call force,CFG_SECURE_TIME_SOURCE_REE,y) +$(call force,CFG_PL310,y) +$(call force,CFG_CACHE_API,y) +$(call force,CFG_WITH_LPAE,n) +$(call force,CFG_GIC,y) + +CFG_WITH_PAGER ?= n +CFG_BOOT_SYNC_CPU ?= y +CFG_TEE_CORE_EMBED_INTERNAL_TESTS ?= y +CFG_WITH_STATS ?= y +CFG_WITH_SOFTWARE_PRNG ?= n +CFG_STIH_UART ?= y +CFG_ENABLE_SCTLR_RR ?= y + +ifeq ($(PLATFORM_FLAVOR),b2260) +$(call force,CFG_TEE_CORE_NB_CORE,2) +CFG_DDR_START ?= 0x40000000 +CFG_DDR_SIZE ?= 0x40000000 +CFG_STM_RSV_DRAM_STARTBYTES ?= 0 +CFG_CORE_TZSRAM_EMUL_START ?= 0x7fe00000 +CFG_DDR_TEETZ_RESERVED_START ?= 0x7e000000 +CFG_DDR_TEETZ_RESERVED_SIZE ?= 0x01e00000 +CFG_PL310_LOCKED ?= y +else +$(call force,CFG_TEE_CORE_NB_CORE,2) +CFG_DDR_START ?= 0x40000000 +CFG_DDR_SIZE ?= 0x80000000 +CFG_STM_RSV_DRAM_STARTBYTES ?= 0x02000000 +CFG_CORE_TZSRAM_EMUL_START ?= 0x94a00000 +CFG_DDR_TEETZ_RESERVED_START ?= 0x93a00000 +CFG_DDR_TEETZ_RESERVED_SIZE ?= 0x01000000 +CFG_PL310_LOCKED ?= n +endif + +CFG_SHMEM_SIZE ?= 0x00200000 +CFG_TZDRAM_START ?= ($(CFG_DDR_TEETZ_RESERVED_START)) +CFG_TZDRAM_SIZE ?= ($(CFG_DDR_TEETZ_RESERVED_SIZE) - $(CFG_SHMEM_SIZE)) +CFG_TZSRAM_START ?= ($(CFG_CORE_TZSRAM_EMUL_START)) +CFG_TZSRAM_SIZE ?= ($(CFG_CORE_TZSRAM_EMUL_SIZE)) +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) diff --git a/optee_os/core/arch/arm/plat-stm/main.c b/optee_os/core/arch/arm/plat-stm/main.c new file mode 100644 index 0000000..c2bc868 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm/main.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2016, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CPU_IOMEM_BASE, CPU_IOMEM_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, RNG_BASE, RNG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, UART_CONSOLE_BASE, STIH_ASC_REG_SIZE); + +#ifdef DRAM0_BASE +register_ddr(DRAM0_BASE, DRAM0_SIZE); +#endif +#ifdef DRAM1_BASE +register_ddr(DRAM1_BASE, DRAM1_SIZE); +#endif + +static struct stih_asc_pd console_data; + +#if defined(PLATFORM_FLAVOR_b2260) +static bool ns_resources_ready(void) +{ + return true; +} +#else +/* some nonsecure resource might not be ready (uart) */ +static int boot_is_completed; +static bool ns_resources_ready(void) +{ + return !!boot_is_completed; +} + +/* Overriding the default __weak tee_entry_std() */ +TEE_Result tee_entry_std(struct optee_msg_arg *arg, uint32_t num_params) +{ + boot_is_completed = 1; + + return __tee_entry_std(arg, num_params); +} +#endif + +void console_init(void) +{ + stih_asc_init(&console_data, UART_CONSOLE_BASE); +} + +void console_putc(int ch) +{ + + if (ns_resources_ready()) { + struct serial_chip *cons = &console_data.chip; + + if (ch == '\n') + cons->ops->putc(cons, '\r'); + cons->ops->putc(cons, ch); + } +} + +void console_flush(void) +{ + if (ns_resources_ready()) { + struct serial_chip *cons = &console_data.chip; + + if (cons->ops->flush) + cons->ops->flush(cons); + } +} + +vaddr_t pl310_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(PL310_BASE, MEM_AREA_IO_SEC, 1); + return (vaddr_t)va; + } + return PL310_BASE; +} + +void arm_cl2_config(vaddr_t pl310) +{ + /* pl310 off */ + io_write32(pl310 + PL310_CTRL, 0); + + /* config PL310 */ + io_write32(pl310 + PL310_TAG_RAM_CTRL, PL310_TAG_RAM_CTRL_INIT); + io_write32(pl310 + PL310_DATA_RAM_CTRL, PL310_DATA_RAM_CTRL_INIT); + io_write32(pl310 + PL310_AUX_CTRL, PL310_AUX_CTRL_INIT); + io_write32(pl310 + PL310_PREFETCH_CTRL, PL310_PREFETCH_CTRL_INIT); + io_write32(pl310 + PL310_POWER_CTRL, PL310_POWER_CTRL_INIT); + + /* invalidate all pl310 cache ways */ + arm_cl2_invbyway(pl310); +} + +void plat_primary_init_early(void) +{ + int i; + + assert(!cpu_mmu_enabled()); + + io_write32(SCU_BASE + SCU_SAC, SCU_SAC_INIT); + io_write32(SCU_BASE + SCU_NSAC, SCU_NSAC_INIT); + io_write32(SCU_BASE + SCU_FILT_EA, CPU_PORT_FILT_END); + io_write32(SCU_BASE + SCU_FILT_SA, CPU_PORT_FILT_START); + io_write32(SCU_BASE + SCU_CTRL, SCU_CTRL_INIT); + + io_write32(pl310_base() + PL310_ADDR_FILT_END, CPU_PORT_FILT_END); + io_write32(pl310_base() + PL310_ADDR_FILT_START, + CPU_PORT_FILT_START | PL310_CTRL_ENABLE_BIT); + + /* TODO: gic_init scan fails, pre-init all SPIs are nonsecure */ + for (i = 0; i < (31 * 4); i += 4) + io_write32(GIC_DIST_BASE + GIC_DIST_ISR1 + i, 0xFFFFFFFF); +} + +void boot_primary_init_intc(void) +{ + gic_init(GIC_CPU_BASE, GIC_DIST_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} diff --git a/optee_os/core/arch/arm/plat-stm/platform_config.h b/optee_os/core/arch/arm/plat-stm/platform_config.h new file mode 100644 index 0000000..0cf4ddc --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm/platform_config.h @@ -0,0 +1,210 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2016, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include +#include + +/* Below are platform/SoC settings specific to stm platform flavors */ + +#if defined(PLATFORM_FLAVOR_b2260) + +#define CPU_IOMEM_BASE 0x08760000 +#define CPU_IOMEM_SIZE 0x000a0000 +#define CPU_PORT_FILT_START 0x40000000 +#define CPU_PORT_FILT_END 0xC0000000 +#define STXHXXX_LPM_PERIPH_BASE 0x09700000 +#define RNG_BASE 0x08A89000 +#define RNG_SIZE 0x00001000 + +#define ASC_NUM 21 +#define UART_CONSOLE_BASE ST_ASC21_REGS_BASE + +#elif defined(PLATFORM_FLAVOR_cannes) + +#define CPU_IOMEM_BASE 0x08760000 +#define CPU_IOMEM_SIZE 0x000a0000 +#define CPU_PORT_FILT_START 0x40000000 +#define CPU_PORT_FILT_END 0xC0000000 +#define STXHXXX_LPM_PERIPH_BASE 0x09400000 +#define RNG_BASE 0x08A89000 +#define RNG_SIZE 0x00001000 + +#define ASC_NUM 20 +#define UART_CONSOLE_BASE ST_ASC20_REGS_BASE + +#else /* defined(PLATFORM_FLAVOR_xxx) */ + +#error "Unknown platform flavor" + +#endif /* defined(PLATFORM_FLAVOR_xxx) */ + +#define PL310_BASE (CPU_IOMEM_BASE + 0x2000) +#define GIC_DIST_BASE (CPU_IOMEM_BASE + 0x1000) +#define SCU_BASE (CPU_IOMEM_BASE + 0x0000) +#define GIC_CPU_BASE (CPU_IOMEM_BASE + 0x0100) +#define ST_ASC20_REGS_BASE (STXHXXX_LPM_PERIPH_BASE + 0x00130000) +#define ST_ASC21_REGS_BASE (STXHXXX_LPM_PERIPH_BASE + 0x00131000) + +/* Below are settings common to stm platform flavors */ + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 32 + +/* + * CP15 Secure ConTroL Register (SCTLR + * + * - Round-Robin replac. for icache, btac, i/duTLB (bit14: RoundRobin) + */ +#define CPU_SCTLR_INIT 0x00004000 + +/* + * CP15 Auxiliary ConTroL Register (ACTRL) + * + * - core always in full SMP (FW bit0=1, SMP bit6=1) + * - L2 write full line of zero disabled (bit3=0) + * (keep WFLZ low. Will be set once outer L2 is ready) + */ +#define CPU_ACTLR_INIT 0x00000041 + +/* + * CP15 NonSecure Access Control Register (NSACR) + * + * - NSec cannot change ACTRL.SMP (NS_SMP bit18=0) + * - Nsec can lockdown TLB (TL bit17=1) + * - NSec cannot access PLE (PLE bit16=0) + * - NSec can use SIMD/VFP (CP10/CP11) (bit15:14=2b00, bit11:10=2b11) + */ +#define CPU_NSACR_INIT 0x00020C00 + +/* + * CP15 Power Control Register (PCR) + * + * - no change latency, enable clk gating + */ +#define CPU_PCR_INIT 0x00000001 + + +/* + * SCU Secure Access Control / NonSecure Access Control + * + * SAC: Both secure CPU access SCU (bit[3:0]). + * NSAC: Both nonsec cpu access SCU (bit[3:0]), private timers (bit[7:4]) + * and global timers (bit[11:8]). + */ +#if !defined(SCU_SAC_INIT) || !defined(SCU_NSAC_INIT) +#define SCU_CPUS_MASK (SHIFT_U32(1, CFG_TEE_CORE_NB_CORE) - 1) + +#define SCU_SAC_INIT SCU_CPUS_MASK +#define SCU_NSAC_INIT (SHIFT_U32(SCU_CPUS_MASK, SCU_NSAC_SCU_SHIFT) | \ + SHIFT_U32(SCU_CPUS_MASK, SCU_NSAC_PTIMER_SHIFT) | \ + SHIFT_U32(SCU_CPUS_MASK, SCU_NSAC_GTIMER_SHIFT)) +#endif + +/* + * PL310 TAG RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:1 - 2 cycle of read accesses latency + * bit[2:0]:1 - 2 cycle of setup latency + */ +#ifndef PL310_TAG_RAM_CTRL_INIT +#define PL310_TAG_RAM_CTRL_INIT 0x00000111 +#endif + +/* + * PL310 DATA RAM Control Register + * + * bit[10:8]:2 - 3 cycle of write accesses latency + * bit[6:4]:2 - 3 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_DATA_RAM_CTRL_INIT +#define PL310_DATA_RAM_CTRL_INIT 0x00000222 +#endif + +/* + * PL310 Auxiliary Control Register + * + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockown cache lines (bit26=1) + * Pseudo-random replacement policy (bit25=0) + * Force write allocated (default) + * Shared attribute internally ignored (bit22=1, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * Platform fmavor specific way config: + * - way size (bit19:17) + * - way associciativity (bit16) + * Store buffer device limitation enabled (bit11=1) + * Cacheable accesses have high prio (bit10=0) + * Full Line Zero (FLZ) disabled (bit0=0) + */ +#ifndef PL310_AUX_CTRL_INIT +#define PL310_AUX_CTRL_INIT 0x3C480800 +#endif + +/* + * PL310 Prefetch Control Register + * + * Double linefill disabled (bit30=0) + * I/D prefetch enabled (bit29:28=2b11) + * Prefetch drop enabled (bit24=1) + * Incr double linefill disable (bit23=0) + * Prefetch offset = 7 (bit4:0) + */ +#define PL310_PREFETCH_CTRL_INIT 0x31000007 + +/* + * PL310 Power Register + * + * Dynamic clock gating enabled + * Standby mode enabled + */ +#define PL310_POWER_CTRL_INIT 0x00000003 + +/* + * SCU Control Register : CTRL = 0x00000065 + * - ic stanby enable=1 + * - scu standby enable=1 + * - scu enable=1 + */ +#define SCU_CTRL_INIT 0x00000065 + +/* + * Register non-secure DDR chunks for dynamic shared memory: these are + * DDR ranges that do not include OP-TEE secure memory. + * Some Stm platforms may reserve beginning of the DDR for non REE memory. + */ + +#ifdef CFG_DDR_START +/* Carvout out secure RAM range (emulated SRAM is expected near DRAM) */ +#if defined(CFG_WITH_PAGER) && defined(TZSRAM_BASE) +#if TZSRAM_BASE >= CFG_DDR_START +#define STM_SECDDR_BASE MIN_UNSAFE(TZSRAM_BASE, TZDRAM_BASE) +#define STM_SECDDR_END MAX_UNSAFE(TZSRAM_BASE + TZSRAM_SIZE, \ + TZDRAM_BASE + TZDRAM_SIZE) +#endif /*TZSRAM_BASE >= CFG_DDR_START*/ +#endif /*CFG_WITH_PAGER && TZSRAM_BASE*/ + +#ifndef STM_SECDDR_BASE +#define STM_SECDDR_BASE TZDRAM_BASE +#define STM_SECDDR_END (TZDRAM_BASE + TZDRAM_SIZE) +#endif + +#define STM_SECDDR_SIZE (STM_SECDDR_END - STM_SECDDR_BASE) +/* Register the DDR chunks that do not intersect the secure DDR single area */ +#define DRAM0_BASE (CFG_DDR_START + CFG_STM_RSV_DRAM_STARTBYTES) +#define DRAM0_SIZE (STM_SECDDR_BASE - DRAM0_BASE) +#if (STM_SECDDR_END < 0x80000000ULL) +#define DRAM1_BASE STM_SECDDR_END +#define DRAM1_SIZE ((CFG_DDR_START - DRAM1_BASE) + CFG_DDR_SIZE) +#endif +#endif /*CFG_DDR_START*/ + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-stm/rng_support.c b/optee_os/core/arch/arm/plat-stm/rng_support.c new file mode 100644 index 0000000..2a81b7b --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm/rng_support.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2016, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include + +#include "rng_support.h" + +/* Address of the register to read in the RNG IP */ +#define RNG_VAL_OFFSET 0x24 +#define RNG_STATUS_OFFSET 0x20 + +#define RNG_STATUS_ERR0 BIT32(0) +#define RNG_STATUS_ERR1 BIT32(1) +#define RNG_STATUS_FULL BIT32(5) + +static vaddr_t rng_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(RNG_BASE, MEM_AREA_IO_SEC, RNG_SIZE); + return (vaddr_t)va; + } + return RNG_BASE; +} + +static inline int hwrng_waithost_fifo_full(void) +{ + uint32_t status; + + do { + status = io_read32(rng_base() + RNG_STATUS_OFFSET); + } while (!(status & RNG_STATUS_FULL)); + + if (status & (RNG_STATUS_ERR0 | RNG_STATUS_ERR1)) + return 1; + + return 0; +} + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + /* + * Only the HW RNG IP is used to generate the value through the + * HOST interface. + * + * @see the document rng_fspec_revG_120720.pdf for details + * + * - HOST FIFO size = 8x8b (64b) + * - LSB (16b) of the RNG_VAL register allows to read 16b + * - bit5 of the RNG_STATUS register allows to known if the HOST + * FIFO is full or not. + * - bit1,0 of the RNG_STATUS register allows to known if the + * data are valid. + * + * Main principle: + * For performance reason, a local SW fifo is used to store the + * content of the HOST FIFO (max size = 8bytes). When a random + * value is expected, this SW fifo is used to return a stored value. + * When the local SW fifo is empty, it is filled with the HOST FIFO + * according the following sequence: + * + * - wait HOST FIFO full + * o Indicates that max 8-bytes (64b) are available + * o This is mandatory to guarantee that a valid data is + * available. No STATUS bit to indicate that the HOST FIFO + * is empty is provided. + * - check STATUS bits + * - update the local SW fifo with the HOST FIFO + * + * This avoid to wait at each iteration that a valid random value is + * available. _LOCAL_FIFO_SIZE indicates the size of the local SW fifo. + * + */ + + +#define _LOCAL_FIFO_SIZE 8 /* min 2, 4, 6, max 8 */ + + static uint8_t lfifo[_LOCAL_FIFO_SIZE]; /* local fifo */ + static int pos; + + static int nbcall; /* debug purpose - 0 is the initial value*/ + + volatile uint32_t tmpval[_LOCAL_FIFO_SIZE/2]; + int i; + + uint8_t *buffer = buf; + size_t buffer_pos = 0; + + nbcall++; + + while (buffer_pos < len) { + /* Refill our FIFO */ + if (pos == 0) { + if (hwrng_waithost_fifo_full()) + return TEE_ERROR_GENERIC; + + /* + * Read the FIFO according to the number of + * expected elements + */ + for (i = 0; i < _LOCAL_FIFO_SIZE / 2; i++) + tmpval[i] = io_read32(rng_base() + + RNG_VAL_OFFSET) & 0xFFFF; + + /* Update the local SW fifo for next request */ + pos = 0; + for (i = 0; i < _LOCAL_FIFO_SIZE / 2; i++) { + lfifo[pos] = tmpval[i] & 0xFF; + pos++; + lfifo[pos] = (tmpval[i] >> 8) & 0xFF; + pos++; + } + pos = 0; + } + + buffer[buffer_pos++] = lfifo[pos++]; + if (pos == _LOCAL_FIFO_SIZE) + pos = 0; + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/arch/arm/plat-stm/sub.mk b/optee_os/core/arch/arm/plat-stm/sub.mk new file mode 100644 index 0000000..4793bfb --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm/sub.mk @@ -0,0 +1,5 @@ +global-incdirs-y += . + +srcs-y += rng_support.c +srcs-y += tz_a9init.S +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-stm/tz_a9init.S b/optee_os/core/arch/arm/plat-stm/tz_a9init.S new file mode 100644 index 0000000..f6b68ba --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm/tz_a9init.S @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2016, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include + +.section .text +.balign 4 +.code 32 + +/* + * void arm_cl2_enable(vaddr_t pl310_base) - Memory Cache Level2 Enable Function + * + * If PL310 supports FZLW, enable also FZL in A9 core + * + * Use scratables registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + * TODO: to be moved to PL310 code (tz_svce_pl310.S ?) + */ +FUNC arm_cl2_enable , : + /* Enable PL310 ctrl -> only set lsb bit */ + mov r1, #0x1 + str r1, [r0, #PL310_CTRL] + + /* if L2 FLZW enable, enable in L1 */ + ldr r1, [r0, #PL310_AUX_CTRL] + tst r1, #(1 << 0) /* test AUX_CTRL[FLZ] */ + read_actlr r0 + orrne r0, r0, #(1 << 3) /* enable ACTLR[FLZW] */ + write_actlr r0 + + mov pc, lr +END_FUNC arm_cl2_enable + +/* + * Cortex A9 configuration early configuration + * + * Use scratables registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + */ +FUNC plat_cpu_reset_early , : + /* CPSR.A can be modified in any security state. */ + mov_imm r0, SCR_AW + write_scr r0 + + mov_imm r0, CPU_SCTLR_INIT + write_sctlr r0 + + mov_imm r0, CPU_ACTLR_INIT + write_actlr r0 + + mov_imm r0, CPU_NSACR_INIT + write_nsacr r0 + + mov_imm r0, CPU_PCR_INIT + write_pcr r0 + + mov pc, lr +END_FUNC plat_cpu_reset_early + diff --git a/optee_os/core/arch/arm/plat-stm32mp1/boot_api.h b/optee_os/core/arch/arm/plat-stm32mp1/boot_api.h new file mode 100644 index 0000000..62e38b5 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/boot_api.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (C) 2017-2018, STMicroelectronics + */ + +#ifndef __BOOT_API_H__ +#define __BOOT_API_H__ + +/* + * Backup registers mapping + */ + +/* Backup register #4: magic to request core1 boot up */ +#define BCKR_CORE1_MAGIC_NUMBER 4 + +/* Value for BCKR_CORE1_MAGIC_NUMBER entry */ +#define BOOT_API_A7_CORE1_MAGIC_NUMBER 0xca7face1 + +/* Backup register #5: physical address of core1 entry at boot up */ +#define BCKR_CORE1_BRANCH_ADDRESS 5 + +#endif /* __BOOT_API_H__*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/conf.mk b/optee_os/core/arch/arm/plat-stm32mp1/conf.mk new file mode 100644 index 0000000..058199a --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/conf.mk @@ -0,0 +1,377 @@ +# 1GB and 512MB DDR targets do not locate secure DDR at the same place. +flavor_dts_file-157A_DHCOR_AVENGER96 = stm32mp157a-dhcor-avenger96.dts +flavor_dts_file-157A_DK1 = stm32mp157a-dk1.dts +flavor_dts_file-157C_DHCOM_PDK2 = stm32mp157c-dhcom-pdk2.dts +flavor_dts_file-157C_DK2 = stm32mp157c-dk2.dts +flavor_dts_file-157C_ED1 = stm32mp157c-ed1.dts +flavor_dts_file-157C_EV1 = stm32mp157c-ev1.dts + +flavor_dts_file-135F_DK = stm32mp135f-dk.dts + +flavorlist-cryp-512M = $(flavor_dts_file-157C_DK2) \ + $(flavor_dts_file-135F_DK) + +flavorlist-no_cryp-512M = $(flavor_dts_file-157A_DK1) + +flavorlist-cryp-1G = $(flavor_dts_file-157C_DHCOM_PDK2) \ + $(flavor_dts_file-157C_ED1) \ + $(flavor_dts_file-157C_EV1) + +flavorlist-no_cryp-1G = $(flavor_dts_file-157A_DHCOR_AVENGER96) + +flavorlist-no_cryp = $(flavorlist-no_cryp-512M) \ + $(flavorlist-no_cryp-1G) + +flavorlist-512M = $(flavorlist-cryp-512M) \ + $(flavorlist-no_cryp-512M) + +flavorlist-1G = $(flavorlist-cryp-1G) \ + $(flavorlist-no_cryp-1G) + +flavorlist-MP15-HUK-DT = $(flavor_dts_file-157A_DK1) \ + $(flavor_dts_file-157C_DK2) \ + $(flavor_dts_file-157C_ED1) \ + $(flavor_dts_file-157C_EV1) + +flavorlist-MP15 = $(flavor_dts_file-157A_DHCOR_AVENGER96) \ + $(flavor_dts_file-157A_DK1) \ + $(flavor_dts_file-157C_DHCOM_PDK2) \ + $(flavor_dts_file-157C_DK2) \ + $(flavor_dts_file-157C_ED1) \ + $(flavor_dts_file-157C_EV1) + +flavorlist-MP13 = $(flavor_dts_file-135F_DK) + +ifneq ($(PLATFORM_FLAVOR),) +ifeq ($(flavor_dts_file-$(PLATFORM_FLAVOR)),) +$(error Invalid platform flavor $(PLATFORM_FLAVOR)) +endif +CFG_EMBED_DTB_SOURCE_FILE ?= $(flavor_dts_file-$(PLATFORM_FLAVOR)) +endif +CFG_EMBED_DTB_SOURCE_FILE ?= stm32mp157c-dk2.dts + +ifneq ($(filter $(CFG_EMBED_DTB_SOURCE_FILE),$(flavorlist-no_cryp)),) +$(call force,CFG_STM32_CRYP,n) +$(call force,CFG_STM32_SAES,n) +endif + +ifneq ($(filter $(CFG_EMBED_DTB_SOURCE_FILE),$(flavorlist-no_rng)),) +$(call force,CFG_HWRNG_PTA,n) +$(call force,CFG_WITH_SOFTWARE_PRNG,y) +endif + +ifneq ($(filter $(CFG_EMBED_DTB_SOURCE_FILE),$(flavorlist-MP15-HUK-DT)),) +CFG_STM32MP15_HUK ?= y +CFG_STM32_HUK_FROM_DT ?= y +endif + +ifneq ($(filter $(CFG_EMBED_DTB_SOURCE_FILE),$(flavorlist-MP13)),) +$(call force,CFG_STM32MP13,y) +endif + +ifneq ($(filter $(CFG_EMBED_DTB_SOURCE_FILE),$(flavorlist-MP15)),) +$(call force,CFG_STM32MP15,y) +endif + +# CFG_STM32MP1x switches are exclusive. +# - CFG_STM32MP15 is enabled for STM32MP15x-* targets (default) +# - CFG_STM32MP13 is enabled for STM32MP13x-* targets +ifeq ($(CFG_STM32MP13),y) +$(call force,CFG_STM32MP15,n) +else +$(call force,CFG_STM32MP15,y) +$(call force,CFG_STM32MP13,n) +endif +ifeq ($(call cfg-one-enabled,CFG_STM32MP15 CFG_STM32MP13),n) +$(error One of CFG_STM32MP15 CFG_STM32MP13 must be enabled) +endif +ifeq ($(call cfg-all-enabled,CFG_STM32MP15 CFG_STM32MP13),y) +$(error Only one of CFG_STM32MP15 CFG_STM32MP13 must be enabled) +endif + +include core/arch/arm/cpu/cortex-a7.mk + +$(call force,CFG_DRIVERS_CLK,y) +$(call force,CFG_DRIVERS_CLK_DT,y) +$(call force,CFG_DRIVERS_GPIO,y) +$(call force,CFG_DRIVERS_PINCTRL,y) +$(call force,CFG_DRIVERS_REGULATOR,y) +$(call force,CFG_GIC,y) +$(call force,CFG_INIT_CNTVOFF,y) +$(call force,CFG_PSCI_ARM32,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_SM_PLATFORM_HANDLER,y) +$(call force,CFG_STM32_SHARED_IO,y) +$(call force,CFG_REGULATOR_FIXED,y) + +ifeq ($(CFG_STM32MP13),y) +$(call force,CFG_BOOT_SECONDARY_REQUEST,n) +$(call force,CFG_CORE_RESERVED_SHM,n) +$(call force,CFG_DRIVERS_CLK_FIXED,y) +$(call force,CFG_SECONDARY_INIT_CNTFRQ,n) +$(call force,CFG_STM32_GPIO,y) +$(call force,CFG_STM32_VREFBUF,y) +$(call force,CFG_STM32MP_CLK_CORE,y) +$(call force,CFG_STM32MP1_SHARED_RESOURCES,n) +$(call force,CFG_STM32MP13_CLK,y) +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_WITH_NSEC_GPIOS,n) +CFG_EXTERNAL_DT ?= n +CFG_STM32MP_OPP_COUNT ?= 2 +CFG_STM32MP1_SCMI_SHM_SYSRAM ?= y +CFG_WITH_PAGER ?= n +endif # CFG_STM32MP13 + +ifeq ($(CFG_STM32MP15),y) +$(call force,CFG_BOOT_SECONDARY_REQUEST,y) +$(call force,CFG_DRIVERS_CLK_FIXED,n) +$(call force,CFG_SECONDARY_INIT_CNTFRQ,y) +$(call force,CFG_STM32MP1_SHARED_RESOURCES,y) +$(call force,CFG_STM32_SAES,n) +$(call force,CFG_STM32MP15_CLK,y) +CFG_CORE_RESERVED_SHM ?= y +CFG_EXTERNAL_DT ?= y +CFG_STM32_BSEC_SIP ?= y +CFG_TEE_CORE_NB_CORE ?= 2 +CFG_WITH_PAGER ?= y +CFG_WITH_SOFTWARE_PRNG ?= y +endif # CFG_STM32MP15 + +ifeq ($(CFG_WITH_PAGER),y) +CFG_WITH_LPAE ?= n +endif +CFG_WITH_LPAE ?= y +CFG_MMAP_REGIONS ?= 23 +CFG_DTB_MAX_SIZE ?= (256 * 1024) +CFG_CORE_ASLR ?= n + +ifneq ($(CFG_WITH_LPAE),y) +# Without LPAE, default TEE virtual address range is 1MB, we need at least 2MB. +CFG_TEE_RAM_VA_SIZE ?= 0x00200000 +endif + +ifneq ($(filter $(CFG_EMBED_DTB_SOURCE_FILE),$(flavorlist-512M)),) +CFG_TZDRAM_START ?= 0xde000000 +CFG_DRAM_SIZE ?= 0x20000000 +endif + +CFG_DRAM_BASE ?= 0xc0000000 +CFG_DRAM_SIZE ?= 0x40000000 + +# CFG_STM32MP1_SCMI_SHM_BASE and CFG_STM32MP1_SCMI_SHM_SIZE define the +# device memory mapped SRAM used for SCMI message transfers. +# When CFG_STM32MP1_SCMI_SHM_BASE is set to 0, the platform uses OP-TEE +# native shared memory for SCMI communication instead of SRAM. +# +# When CFG_STM32MP1_SCMI_SHM_SYSRAM is enabled, OP-TEE uses the +# last 4KB page of SYSRAM as SCMI shared memory. The switch is default +# disabled. +CFG_STM32MP1_SCMI_SHM_SYSRAM ?= n +ifeq ($(CFG_STM32MP1_SCMI_SHM_SYSRAM),y) +$(call force,CFG_STM32MP1_SCMI_SHM_BASE,0x2ffff000) +else +CFG_STM32MP1_SCMI_SHM_BASE ?= 0 +endif +$(call force,CFG_STM32MP1_SCMI_SHM_SIZE,0x1000) + +ifeq ($(CFG_STM32MP15),y) +CFG_TZDRAM_START ?= 0xfe000000 +ifeq ($(CFG_CORE_RESERVED_SHM),y) +CFG_TZDRAM_SIZE ?= 0x01e00000 +else +CFG_TZDRAM_SIZE ?= 0x02000000 +endif +CFG_TZSRAM_START ?= 0x2ffc0000 +CFG_TZSRAM_SIZE ?= 0x0003f000 +ifeq ($(CFG_CORE_RESERVED_SHM),y) +CFG_SHMEM_START ?= ($(CFG_TZDRAM_START) + $(CFG_TZDRAM_SIZE)) +CFG_SHMEM_SIZE ?= ($(CFG_DRAM_BASE) + $(CFG_DRAM_SIZE) - $(CFG_SHMEM_START)) +endif +else +CFG_TZDRAM_SIZE ?= 0x02000000 +CFG_TZDRAM_START ?= ($(CFG_DRAM_BASE) + $(CFG_DRAM_SIZE) - $(CFG_TZDRAM_SIZE)) +endif #CFG_STM32MP15 + +CFG_STM32_BSEC ?= y +CFG_STM32_CRYP ?= y +CFG_STM32_ETZPC ?= y +CFG_STM32_GPIO ?= y +CFG_STM32_I2C ?= y +CFG_STM32_IWDG ?= y +CFG_STM32_RNG ?= y +CFG_STM32_RSTCTRL ?= y +CFG_STM32_SAES ?= y +CFG_STM32_TAMP ?= y +CFG_STM32_UART ?= y +CFG_STPMIC1 ?= y +CFG_TZC400 ?= y + +CFG_DRIVERS_I2C ?= $(CFG_STM32_I2C) + +CFG_WITH_SOFTWARE_PRNG ?= n +ifneq ($(CFG_WITH_SOFTWARE_PRNG),y) +$(call force,CFG_STM32_RNG,y,Required by HW RNG when CFG_WITH_SOFTWARE_PRNG=n) +endif + +ifeq ($(CFG_STPMIC1),y) +$(call force,CFG_STM32_I2C,y) +$(call force,CFG_STM32_GPIO,y) +endif + +# If any crypto driver is enabled, enable the crypto-framework layer +ifeq ($(call cfg-one-enabled, CFG_STM32_CRYP CFG_STM32_SAES),y) +$(call force,CFG_STM32_CRYPTO_DRIVER,y) +endif + +CFG_DRIVERS_RSTCTRL ?= $(CFG_STM32_RSTCTRL) +$(eval $(call cfg-depends-all,CFG_STM32_RSTCTRL,CFG_DRIVERS_RSTCTRL)) + +CFG_WDT ?= $(CFG_STM32_IWDG) + +# Platform specific configuration +CFG_STM32MP_PANIC_ON_TZC_PERM_VIOLATION ?= y + +# Default enable scmi-msg server if SCP-firmware SCMI server is disabled +ifneq ($(CFG_SCMI_SCPFW),y) +CFG_SCMI_MSG_DRIVERS ?= y +endif + +# SiP/OEM service for non-secure world +CFG_STM32_BSEC_SIP ?= n +CFG_STM32MP1_SCMI_SIP ?= n +ifeq ($(CFG_STM32MP1_SCMI_SIP),y) +$(call force,CFG_SCMI_MSG_DRIVERS,y,Mandated by CFG_STM32MP1_SCMI_SIP) +$(call force,CFG_SCMI_MSG_SMT,y,Mandated by CFG_STM32MP1_SCMI_SIP) +$(call force,CFG_SCMI_MSG_SMT_FASTCALL_ENTRY,y,Mandated by CFG_STM32MP1_SCMI_SIP) +endif + +# Enable BSEC PTA for fuses access management +CFG_STM32_BSEC_PTA ?= y +ifeq ($(CFG_STM32_BSEC_PTA),y) +$(call force,CFG_STM32_BSEC,y,Mandated by CFG_BSEC_PTA) +endif + +# Default enable SCMI PTA support +CFG_SCMI_PTA ?= y +ifeq ($(CFG_SCMI_PTA),y) +ifneq ($(CFG_SCMI_SCPFW),y) +$(call force,CFG_SCMI_MSG_DRIVERS,y,Mandated by CFG_SCMI_PTA) +CFG_SCMI_MSG_SMT_THREAD_ENTRY ?= y +CFG_SCMI_MSG_SHM_MSG ?= y +CFG_SCMI_MSG_SMT ?= y +endif # !CFG_SCMI_SCPFW +endif # CFG_SCMI_PTA + +CFG_SCMI_SCPFW ?= n +ifeq ($(CFG_SCMI_SCPFW),y) +$(call force,CFG_SCMI_SCPFW_PRODUCT,optee-stm32mp1) +endif + +CFG_SCMI_MSG_DRIVERS ?= n +ifeq ($(CFG_SCMI_MSG_DRIVERS),y) +$(call force,CFG_SCMI_MSG_CLOCK,y) +$(call force,CFG_SCMI_MSG_RESET_DOMAIN,y) +CFG_SCMI_MSG_SHM_MSG ?= y +CFG_SCMI_MSG_SMT ?= y +CFG_SCMI_MSG_SMT_THREAD_ENTRY ?= y +$(call force,CFG_SCMI_MSG_VOLTAGE_DOMAIN,y) +endif + +ifneq ($(CFG_WITH_SOFTWARE_PRNG),y) +CFG_HWRNG_PTA ?= y +endif +ifeq ($(CFG_HWRNG_PTA),y) +$(call force,CFG_STM32_RNG,y,Mandated by CFG_HWRNG_PTA) +$(call force,CFG_WITH_SOFTWARE_PRNG,n,Mandated by CFG_HWRNG_PTA) +$(call force,CFG_HWRNG_QUALITY,1024) +endif + +# Provision enough threads to pass xtest +ifneq (,$(filter y,$(CFG_SCMI_PTA) $(CFG_STM32MP1_SCMI_SIP))) +ifeq ($(CFG_WITH_PAGER),y) +CFG_NUM_THREADS ?= 3 +else +CFG_NUM_THREADS ?= 10 +endif +endif + +# Default enable some test facitilites +CFG_ENABLE_EMBEDDED_TESTS ?= y +CFG_WITH_STATS ?= y + +# Enable OTP update with BSEC driver +CFG_STM32_BSEC_WRITE ?= y + +# Default disable some support for pager memory size constraint +ifeq ($(CFG_WITH_PAGER),y) +CFG_TEE_CORE_DEBUG ?= n +CFG_UNWIND ?= n +CFG_LOCKDEP ?= n +CFG_TA_BGET_TEST ?= n +# Default disable early TA compression to support a smaller HEAP size +CFG_EARLY_TA_COMPRESS ?= n +CFG_CORE_HEAP_SIZE ?= 49152 +endif + +# Non-secure UART and GPIO/pinctrl for the output console +CFG_WITH_NSEC_GPIOS ?= y +CFG_WITH_NSEC_UARTS ?= y +# UART instance used for early console (0 disables early console) +CFG_STM32_EARLY_CONSOLE_UART ?= 4 + +# CFG_STM32MP15_HUK enables use of a HUK read from BSEC fuses. +# Disable the HUK by default as it requires a product specific configuration. +# +# Configuration must provide OTP indices where HUK is loaded. +# When CFG_STM32_HUK_FROM_DT is enabled, HUK OTP location is found in the DT. +# When CFG_STM32_HUK_FROM_DT is disabled, configuration sets each HUK location. +# Either with CFG_STM32MP15_HUK_OTP_BASE, in which case the 4 words are used, +# Or with CFG_STM32MP15_HUK_BSEC_KEY_0/1/2/3 each locating one BSEC word. +# +# Configuration must provide the HUK generation scheme. The following switches +# are exclusive and at least one must be eable when CFG_STM32MP15_HUK is enable. +# CFG_STM32MP15_HUK_BSEC_KEY makes platform HUK to be the raw fuses content. +# CFG_STM32MP15_HUK_BSEC_DERIVE_UID makes platform HUK to be the HUK fuses +# content derived with the device UID fuses content. See derivation scheme +# in stm32mp15_huk.c implementation. +CFG_STM32MP15_HUK ?= n +CFG_STM32_HUK_FROM_DT ?= n + +ifeq ($(CFG_STM32MP15_HUK),y) +ifneq ($(CFG_STM32_HUK_FROM_DT),y) +ifneq (,$(CFG_STM32MP15_HUK_OTP_BASE)) +$(call force,CFG_STM32MP15_HUK_BSEC_KEY_0,CFG_STM32MP15_HUK_OTP_BASE) +$(call force,CFG_STM32MP15_HUK_BSEC_KEY_1,(CFG_STM32MP15_HUK_OTP_BASE + 1)) +$(call force,CFG_STM32MP15_HUK_BSEC_KEY_2,(CFG_STM32MP15_HUK_OTP_BASE + 2)) +$(call force,CFG_STM32MP15_HUK_BSEC_KEY_3,(CFG_STM32MP15_HUK_OTP_BASE + 3)) +endif +ifeq (,$(CFG_STM32MP15_HUK_BSEC_KEY_0)) +$(error Missing configuration switch CFG_STM32MP15_HUK_BSEC_KEY_0) +endif +ifeq (,$(CFG_STM32MP15_HUK_BSEC_KEY_1)) +$(error Missing configuration switch CFG_STM32MP15_HUK_BSEC_KEY_1) +endif +ifeq (,$(CFG_STM32MP15_HUK_BSEC_KEY_2)) +$(error Missing configuration switch CFG_STM32MP15_HUK_BSEC_KEY_2) +endif +ifeq (,$(CFG_STM32MP15_HUK_BSEC_KEY_3)) +$(error Missing configuration switch CFG_STM32MP15_HUK_BSEC_KEY_3) +endif +endif # CFG_STM32_HUK_FROM_DT + +CFG_STM32MP15_HUK_BSEC_KEY ?= y +CFG_STM32MP15_HUK_BSEC_DERIVE_UID ?= n +ifneq (y,$(call cfg-one-enabled,CFG_STM32MP15_HUK_BSEC_KEY CFG_STM32MP15_HUK_BSEC_DERIVE_UID)) +$(error CFG_STM32MP15_HUK mandates one of CFG_STM32MP15_HUK_BSEC_KEY CFG_STM32MP15_HUK_BSEC_DERIVE_UID) +else ifeq ($(CFG_STM32MP15_HUK_BSEC_KEY)-$(CFG_STM32MP15_HUK_BSEC_DERIVE_UID),y-y) +$(error CFG_STM32MP15_HUK_BSEC_KEY and CFG_STM32MP15_HUK_BSEC_DERIVE_UID are exclusive) +endif +endif # CFG_STM32MP15_HUK + +CFG_TEE_CORE_DEBUG ?= y +CFG_STM32_DEBUG_ACCESS ?= $(CFG_TEE_CORE_DEBUG) + +# Sanity on choice config switches +ifeq ($(call cfg-all-enabled,CFG_STM32MP15 CFG_STM32MP13),y) +$(error CFG_STM32MP13_CLK and CFG_STM32MP15_CLK are exclusive) +endif diff --git a/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_etzpc.h b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_etzpc.h new file mode 100644 index 0000000..bc0b78f --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_etzpc.h @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (C) 2018-2019, STMicroelectronics + */ + +#ifndef __STM32MP1_ETZPC_H +#define __STM32MP1_ETZPC_H + +/* Define DECPROT IDs for stm32mp1 familly */ +#ifdef CFG_STM32MP15 +#define STM32MP1_ETZPC_STGENC_ID 0 +#define STM32MP1_ETZPC_BKPSRAM_ID 1 +#define STM32MP1_ETZPC_IWDG1_ID 2 +#define STM32MP1_ETZPC_USART1_ID 3 +#define STM32MP1_ETZPC_SPI6_ID 4 +#define STM32MP1_ETZPC_I2C4_ID 5 +#define STM32MP1_ETZPC_GPIOZ_ID 6 +#define STM32MP1_ETZPC_RNG1_ID 7 +#define STM32MP1_ETZPC_HASH1_ID 8 +#define STM32MP1_ETZPC_CRYP1_ID 9 +#define STM32MP1_ETZPC_DDRCTRL_ID 10 +#define STM32MP1_ETZPC_DDRPHYC_ID 11 +#define STM32MP1_ETZPC_I2C6_ID 12 +/* 13-15 Reserved */ +#define STM32MP1_ETZPC_TIM2_ID 16 +#define STM32MP1_ETZPC_TIM3_ID 17 +#define STM32MP1_ETZPC_TIM4_ID 18 +#define STM32MP1_ETZPC_TIM5_ID 19 +#define STM32MP1_ETZPC_TIM6_ID 20 +#define STM32MP1_ETZPC_TIM7_ID 21 +#define STM32MP1_ETZPC_TIM12_ID 22 +#define STM32MP1_ETZPC_TIM13_ID 23 +#define STM32MP1_ETZPC_TIM14_ID 24 +#define STM32MP1_ETZPC_LPTIM1_ID 25 +#define STM32MP1_ETZPC_WWDG1_ID 26 +#define STM32MP1_ETZPC_SPI2_ID 27 +#define STM32MP1_ETZPC_SPI3_ID 28 +#define STM32MP1_ETZPC_SPDIFRX_ID 29 +#define STM32MP1_ETZPC_USART2_ID 30 +#define STM32MP1_ETZPC_USART3_ID 31 +#define STM32MP1_ETZPC_UART4_ID 32 +#define STM32MP1_ETZPC_UART5_ID 33 +#define STM32MP1_ETZPC_I2C1_ID 34 +#define STM32MP1_ETZPC_I2C2_ID 35 +#define STM32MP1_ETZPC_I2C3_ID 36 +#define STM32MP1_ETZPC_I2C5_ID 37 +#define STM32MP1_ETZPC_CEC_ID 38 +#define STM32MP1_ETZPC_DAC_ID 39 +#define STM32MP1_ETZPC_UART7_ID 40 +#define STM32MP1_ETZPC_UART8_ID 41 +/* 42-43 Reserved */ +#define STM32MP1_ETZPC_MDIOS_ID 44 +/* 45-47 Reserved */ +#define STM32MP1_ETZPC_TIM1_ID 48 +#define STM32MP1_ETZPC_TIM8_ID 49 +/* 50 Reserved */ +#define STM32MP1_ETZPC_USART6_ID 51 +#define STM32MP1_ETZPC_SPI1_ID 52 +#define STM32MP1_ETZPC_SPI4_ID 53 +#define STM32MP1_ETZPC_TIM15_ID 54 +#define STM32MP1_ETZPC_TIM16_ID 55 +#define STM32MP1_ETZPC_TIM17_ID 56 +#define STM32MP1_ETZPC_SPI5_ID 57 +#define STM32MP1_ETZPC_SAI1_ID 58 +#define STM32MP1_ETZPC_SAI2_ID 59 +#define STM32MP1_ETZPC_SAI3_ID 60 +#define STM32MP1_ETZPC_DFSDM_ID 61 +#define STM32MP1_ETZPC_TT_FDCAN_ID 62 +/* 63 Reserved */ +#define STM32MP1_ETZPC_LPTIM2_ID 64 +#define STM32MP1_ETZPC_LPTIM3_ID 65 +#define STM32MP1_ETZPC_LPTIM4_ID 66 +#define STM32MP1_ETZPC_LPTIM5_ID 67 +#define STM32MP1_ETZPC_SAI4_ID 68 +#define STM32MP1_ETZPC_VREFBUF_ID 69 +#define STM32MP1_ETZPC_DCMI_ID 70 +#define STM32MP1_ETZPC_CRC2_ID 71 +#define STM32MP1_ETZPC_ADC_ID 72 +#define STM32MP1_ETZPC_HASH2_ID 73 +#define STM32MP1_ETZPC_RNG2_ID 74 +#define STM32MP1_ETZPC_CRYP2_ID 75 +/* 76-79 Reserved */ +#define STM32MP1_ETZPC_SRAM1_ID 80 +#define STM32MP1_ETZPC_SRAM2_ID 81 +#define STM32MP1_ETZPC_SRAM3_ID 82 +#define STM32MP1_ETZPC_SRAM4_ID 83 +#define STM32MP1_ETZPC_RETRAM_ID 84 +#define STM32MP1_ETZPC_OTG_ID 85 +#define STM32MP1_ETZPC_SDMMC3_ID 86 +#define STM32MP1_ETZPC_DLYBSD3_ID 87 +#define STM32MP1_ETZPC_DMA1_ID 88 +#define STM32MP1_ETZPC_DMA2_ID 89 +#define STM32MP1_ETZPC_DMAMUX_ID 90 +#define STM32MP1_ETZPC_FMC_ID 91 +#define STM32MP1_ETZPC_QSPI_ID 92 +#define STM32MP1_ETZPC_DLYBQ_ID 93 +#define STM32MP1_ETZPC_ETH_ID 94 +/* 95 Reserved */ +#define STM32MP1_ETZPC_MAX_ID 96 +#endif /* CFG_STM32MP15 */ + +#ifdef CFG_STM32MP13 +#define STM32MP1_ETZPC_VREFBUF_ID 0 +#define STM32MP1_ETZPC_LPTIM2_ID 1 +#define STM32MP1_ETZPC_LPTIM3_ID 2 +#define STM32MP1_ETZPC_LTDC_ID 3 +#define STM32MP1_ETZPC_DCMIPP_ID 4 +#define STM32MP1_ETZPC_USBPHYCTRL_ID 5 +#define STM32MP1_ETZPC_DDRCTRLPHY_ID 6 +/* 7-11 Reserved */ +#define STM32MP1_ETZPC_IWDG1_ID 12 +#define STM32MP1_ETZPC_STGENC_ID 13 +/* 14-15 Reserved */ +#define STM32MP1_ETZPC_USART1_ID 16 +#define STM32MP1_ETZPC_USART2_ID 17 +#define STM32MP1_ETZPC_SPI4_ID 18 +#define STM32MP1_ETZPC_SPI5_ID 19 +#define STM32MP1_ETZPC_I2C3_ID 20 +#define STM32MP1_ETZPC_I2C4_ID 21 +#define STM32MP1_ETZPC_I2C5_ID 22 +#define STM32MP1_ETZPC_TIM12_ID 23 +#define STM32MP1_ETZPC_TIM13_ID 24 +#define STM32MP1_ETZPC_TIM14_ID 25 +#define STM32MP1_ETZPC_TIM15_ID 26 +#define STM32MP1_ETZPC_TIM16_ID 27 +#define STM32MP1_ETZPC_TIM17_ID 28 +/* 29-31 Reserved */ +#define STM32MP1_ETZPC_ADC1_ID 32 +#define STM32MP1_ETZPC_ADC2_ID 33 +#define STM32MP1_ETZPC_OTG_ID 34 +/* 35-36 Reserved */ +#define STM32MP1_ETZPC_TSC_ID 37 +/* 38-39 Reserved */ +#define STM32MP1_ETZPC_RNG_ID 40 +#define STM32MP1_ETZPC_HASH_ID 41 +#define STM32MP1_ETZPC_CRYP_ID 42 +#define STM32MP1_ETZPC_SAES_ID 43 +#define STM32MP1_ETZPC_PKA_ID 44 +#define STM32MP1_ETZPC_BKPSRAM_ID 45 +/* 46-47 Reserved */ +#define STM32MP1_ETZPC_ETH1_ID 48 +#define STM32MP1_ETZPC_ETH2_ID 49 +#define STM32MP1_ETZPC_SDMMC1_ID 50 +#define STM32MP1_ETZPC_SDMMC2_ID 51 +/* 52 Reserved */ +#define STM32MP1_ETZPC_MCE_ID 53 +#define STM32MP1_ETZPC_FMC_ID 54 +#define STM32MP1_ETZPC_QSPI_ID 55 +/* 56-59 Reserved */ +#define STM32MP1_ETZPC_SRAM1_ID 60 +#define STM32MP1_ETZPC_SRAM2_ID 61 +#define STM32MP1_ETZPC_SRAM3_ID 62 +/* 63 Reserved */ +#define STM32MP1_ETZPC_MAX_ID 64 +#endif /* CFG_STM32MP13 */ +#endif /*__STM32MP1_ETZPC_H*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pmic.c b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pmic.c new file mode 100644 index 0000000..893e5dc --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pmic.c @@ -0,0 +1,782 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2017-2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MODE_STANDBY 8 + +#define PMIC_I2C_TRIALS 1 +#define PMIC_I2C_TIMEOUT_BUSY_MS 5 + +#define PMIC_REGU_SUPPLY_NAME_LEN 12 + +#define PMIC_REGU_COUNT 14 + +enum { + PMIC_REGU_FLAG_MASK_RESET = 0, + PMIC_REGU_FLAG_COUNT +}; + +static_assert(IS_ENABLED(CFG_DRIVERS_REGULATOR)); +static_assert(PMIC_REGU_FLAG_COUNT <= UINT_MAX); + +/* + * struct pmic_regulator_data - Platform specific data + * @flags: Flags for platform property to apply + * @regu_name: Regulator name ID in stpmic1 driver + * @supported_voltages: Supported levels description or NULL is not yet built + */ +struct pmic_regulator_data { + unsigned int flags; + char *regu_name; + struct regulator_voltages *voltages; +}; + +/* Expect a single PMIC instance */ +static struct i2c_handle_s *i2c_handle; +static uint32_t pmic_i2c_addr; +static int pmic_status = -1; + +/* CPU voltage supplier if found */ +static char cpu_supply_name[PMIC_REGU_SUPPLY_NAME_LEN]; + +bool stm32mp_with_pmic(void) +{ + return pmic_status > 0; +} + +static void init_pmic_state(const void *fdt, int pmic_node) +{ + pmic_status = fdt_get_status(fdt, pmic_node); +} + +static bool dt_pmic_is_secure(void) +{ + return stm32mp_with_pmic() && + i2c_handle->dt_status == DT_STATUS_OK_SEC; +} + +static void priv_dt_properties(const void *fdt, int regu_node, + struct pmic_regulator_data *priv) +{ + const char *name = fdt_get_name(fdt, regu_node, NULL); + + assert(name); + priv->regu_name = strdup(name); + if (!priv->regu_name) + panic(); + + if (fdt_getprop(fdt, regu_node, "st,mask-reset", NULL)) + priv->flags |= PMIC_REGU_FLAG_MASK_RESET; +} + +/* + * @flags: Operations expected when entering a low power sequence + * @voltage: Target voltage to apply during low power sequences + */ +struct regu_lp_config { + uint8_t flags; + struct stpmic1_lp_cfg cfg; +}; + +#define REGU_LP_FLAG_LOAD_PWRCTRL BIT(0) +#define REGU_LP_FLAG_ON_IN_SUSPEND BIT(1) +#define REGU_LP_FLAG_OFF_IN_SUSPEND BIT(2) +#define REGU_LP_FLAG_SET_VOLTAGE BIT(3) +#define REGU_LP_FLAG_MODE_STANDBY BIT(4) + +/* + * struct regu_lp_state - Low power configuration for regulators + * @name: low power state identifier string name + * @cfg_count: number of regulator configuration instance in @cfg + * @cfg: regulator configurations for low power state @name + */ +struct regu_lp_state { + const char *name; + size_t cfg_count; + struct regu_lp_config *cfg; +}; + +enum regu_lp_state_id { + REGU_LP_STATE_DISK = 0, + REGU_LP_STATE_STANDBY, + REGU_LP_STATE_MEM, + REGU_LP_STATE_MEM_LOWVOLTAGE, + REGU_LP_STATE_COUNT +}; + +static struct regu_lp_state regu_lp_state[REGU_LP_STATE_COUNT] = { + [REGU_LP_STATE_DISK] = { .name = "standby-ddr-off", }, + [REGU_LP_STATE_STANDBY] = { .name = "standby-ddr-sr", }, + [REGU_LP_STATE_MEM] = { .name = "lp-stop", }, + [REGU_LP_STATE_MEM_LOWVOLTAGE] = { .name = "lplv-stop", }, +}; + +static unsigned int regu_lp_state2idx(const char *name) +{ + unsigned int i = 0; + + for (i = 0; i < ARRAY_SIZE(regu_lp_state); i++) + if (!strcmp(name, regu_lp_state[i].name)) + return i; + + panic(); +} + +static void dt_get_regu_low_power_config(const void *fdt, const char *regu_name, + int regu_node, const char *lp_state) +{ + unsigned int state_idx = regu_lp_state2idx(lp_state); + struct regu_lp_state *state = regu_lp_state + state_idx; + const fdt32_t *cuint = NULL; + int regu_state_node = 0; + struct regu_lp_config *regu_cfg = NULL; + + state->cfg_count++; + state->cfg = realloc(state->cfg, + state->cfg_count * sizeof(*state->cfg)); + if (!state->cfg) + panic(); + + regu_cfg = &state->cfg[state->cfg_count - 1]; + + memset(regu_cfg, 0, sizeof(*regu_cfg)); + + if (stpmic1_regu_has_lp_cfg(regu_name)) { + if (stpmic1_lp_cfg(regu_name, ®u_cfg->cfg)) { + DMSG("Cannot setup low power for regu %s", regu_name); + panic(); + } + /* + * Always copy active configuration (Control register) + * to PWRCTRL Control register, even if regu_state_node + * does not exist. + */ + regu_cfg->flags |= REGU_LP_FLAG_LOAD_PWRCTRL; + } + + /* Parse regulator stte node if any */ + regu_state_node = fdt_subnode_offset(fdt, regu_node, lp_state); + if (regu_state_node <= 0) + return; + + if (fdt_getprop(fdt, regu_state_node, + "regulator-on-in-suspend", NULL)) + regu_cfg->flags |= REGU_LP_FLAG_ON_IN_SUSPEND; + + if (fdt_getprop(fdt, regu_state_node, + "regulator-off-in-suspend", NULL)) + regu_cfg->flags |= REGU_LP_FLAG_OFF_IN_SUSPEND; + + cuint = fdt_getprop(fdt, regu_state_node, + "regulator-suspend-microvolt", NULL); + if (cuint) { + uint32_t mv = fdt32_to_cpu(*cuint) / 1000U; + + if (stpmic1_lp_voltage_cfg(regu_name, mv, ®u_cfg->cfg)) { + DMSG("Cannot set voltage for %s", regu_name); + panic(); + } + regu_cfg->flags |= REGU_LP_FLAG_SET_VOLTAGE; + } + + cuint = fdt_getprop(fdt, regu_state_node, + "regulator-mode", NULL); + if (cuint && fdt32_to_cpu(*cuint) == MODE_STANDBY) + regu_cfg->flags |= REGU_LP_FLAG_MODE_STANDBY; +} + +/* + * int stm32mp_pmic_set_lp_config(char *lp_state) + * + * Load the low power configuration stored in regu_lp_state[]. + */ +void stm32mp_pmic_apply_lp_config(const char *lp_state) +{ + unsigned int state_idx = regu_lp_state2idx(lp_state); + struct regu_lp_state *state = ®u_lp_state[state_idx]; + size_t i = 0; + + if (stpmic1_powerctrl_on()) + panic(); + + for (i = 0; i < state->cfg_count; i++) { + struct stpmic1_lp_cfg *cfg = &state->cfg[i].cfg; + + if ((state->cfg[i].flags & REGU_LP_FLAG_LOAD_PWRCTRL) && + stpmic1_lp_load_unpg(cfg)) + panic(); + + if ((state->cfg[i].flags & REGU_LP_FLAG_ON_IN_SUSPEND) && + stpmic1_lp_on_off_unpg(cfg, 1)) + panic(); + + if ((state->cfg[i].flags & REGU_LP_FLAG_OFF_IN_SUSPEND) && + stpmic1_lp_on_off_unpg(cfg, 0)) + panic(); + + if ((state->cfg[i].flags & REGU_LP_FLAG_SET_VOLTAGE) && + stpmic1_lp_voltage_unpg(cfg)) + panic(); + + if ((state->cfg[i].flags & REGU_LP_FLAG_MODE_STANDBY) && + stpmic1_lp_mode_unpg(cfg, 1)) + panic(); + } +} + +/* Return a libfdt compliant status value */ +static int save_cpu_supply_name(void) +{ + void *fdt = NULL; + int node = 0; + const fdt32_t *cuint = NULL; + const char *name = NULL; + + fdt = get_embedded_dt(); + if (!fdt) + panic(); + + node = fdt_path_offset(fdt, "/cpus/cpu@0"); + if (node < 0) + return -FDT_ERR_NOTFOUND; + + cuint = fdt_getprop(fdt, node, "cpu-supply", NULL); + if (!cuint) + return -FDT_ERR_NOTFOUND; + + node = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(*cuint)); + if (node < 0) + return -FDT_ERR_NOTFOUND; + + name = fdt_get_name(fdt, node, NULL); + assert(strnlen(name, sizeof(cpu_supply_name)) < + sizeof(cpu_supply_name)); + + strncpy(cpu_supply_name, name, sizeof(cpu_supply_name)); + + return 0; +} + +const char *stm32mp_pmic_get_cpu_supply_name(void) +{ + return cpu_supply_name; +} + +/* Preallocate not that much regu references */ +static char *nsec_access_regu_name[PMIC_REGU_COUNT]; + +bool stm32mp_nsec_can_access_pmic_regu(const char *name) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(nsec_access_regu_name); n++) + if (nsec_access_regu_name[n] && + !strcmp(nsec_access_regu_name[n], name)) + return true; + + return false; +} + +static void register_nsec_regu(const char *name_ref) +{ + size_t n = 0; + + assert(!stm32mp_nsec_can_access_pmic_regu(name_ref)); + + for (n = 0; n < ARRAY_SIZE(nsec_access_regu_name); n++) { + if (!nsec_access_regu_name[n]) { + nsec_access_regu_name[n] = strdup(name_ref); + + if (!nsec_access_regu_name[n]) + panic(); + break; + } + } + + assert(stm32mp_nsec_can_access_pmic_regu(name_ref)); +} + +static TEE_Result pmic_set_state(struct regulator *regulator, bool enable) +{ + struct pmic_regulator_data *priv = regulator->priv; + int ret = 0; + + stm32mp_get_pmic(); + + if (enable) + ret = stpmic1_regulator_enable(priv->regu_name); + else + ret = stpmic1_regulator_disable(priv->regu_name); + + stm32mp_put_pmic(); + + if (ret) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result pmic_get_state(struct regulator *regulator, bool *enabled) +{ + struct pmic_regulator_data *priv = regulator->priv; + + stm32mp_get_pmic(); + *enabled = stpmic1_is_regulator_enabled(priv->regu_name); + stm32mp_put_pmic(); + + return TEE_SUCCESS; +} + +static TEE_Result pmic_get_voltage(struct regulator *regulator, int *level_uv) +{ + struct pmic_regulator_data *priv = regulator->priv; + int rc = 0; + + stm32mp_get_pmic(); + rc = stpmic1_regulator_voltage_get(priv->regu_name); + stm32mp_put_pmic(); + + if (rc < 0) + return TEE_ERROR_GENERIC; + + *level_uv = rc * 1000; + + return TEE_SUCCESS; +} + +static TEE_Result pmic_set_voltage(struct regulator *regulator, int level_uv) +{ + struct pmic_regulator_data *priv = regulator->priv; + unsigned int level_mv = level_uv / 1000; + int rc = 0; + + if (level_mv > UINT16_MAX) + return TEE_ERROR_BAD_PARAMETERS; + + stm32mp_get_pmic(); + rc = stpmic1_regulator_voltage_set(priv->regu_name, level_mv); + stm32mp_put_pmic(); + + if (rc) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static int cmp_int_value(const void *a, const void *b) +{ + const int *ia = a; + const int *ib = b; + + return CMP_TRILEAN(*ia, *ib); +} + +static size_t refine_levels_array(struct regulator_voltages *voltages) +{ + int *levels_uv = voltages->entries; + size_t count = voltages->num_levels; + size_t n = 0; + size_t m = 0; + + /* We need to sort the array has STPMIC1 driver does not */ + qsort(levels_uv, count, sizeof(*levels_uv), cmp_int_value); + + /* Remove duplicates and return optimized count */ + for (n = 1; n < count; n++) { + if (levels_uv[m] != levels_uv[n]) { + if (m + 1 != n) + levels_uv[m + 1] = levels_uv[n]; + m++; + } + } + + return m + 1; +} + +static TEE_Result pmic_list_voltages(struct regulator *regulator, + struct regulator_voltages **out_voltages) +{ + struct pmic_regulator_data *priv = regulator->priv; + + if (!priv->voltages) { + struct regulator_voltages *voltages_s = NULL; + struct regulator_voltages *voltages = NULL; + const uint16_t *level_ref = NULL; + size_t level_count = 0; + size_t n = 0; + + /* + * Allocate and build a consised and ordered voltage list + * based on the voltage list provided by stpmic1 driver. + */ + stpmic1_regulator_levels_mv(priv->regu_name, &level_ref, + &level_count); + + voltages = calloc(1, sizeof(*voltages) + + sizeof(*voltages->entries) * level_count); + if (!voltages) + return TEE_ERROR_OUT_OF_MEMORY; + for (n = 0; n < level_count; n++) + voltages->entries[n] = level_ref[n] * 1000; + + voltages->num_levels = level_count; + level_count = refine_levels_array(voltages); + + voltages_s = realloc(voltages, + sizeof(*voltages) + + sizeof(*voltages->entries) * level_count); + if (!voltages_s) { + free(voltages); + return TEE_ERROR_OUT_OF_MEMORY; + } + + voltages_s->type = VOLTAGE_TYPE_FULL_LIST; + voltages_s->num_levels = level_count; + priv->voltages = voltages_s; + } + + *out_voltages = priv->voltages; + + return TEE_SUCCESS; +} + +static TEE_Result pmic_regu_init(struct regulator *regulator, + const void *fdt __unused, int node __unused) +{ + struct pmic_regulator_data *priv = regulator->priv; + struct stpmic1_bo_cfg cfg = { }; + + if (!priv->flags) + return TEE_SUCCESS; + + stm32mp_get_pmic(); + + if (priv->flags & PMIC_REGU_FLAG_MASK_RESET) { + if (stpmic1_bo_mask_reset_cfg(priv->regu_name, &cfg) || + stpmic1_bo_mask_reset_unpg(&cfg)) { + EMSG("Mask reset failed for %s", priv->regu_name); + return TEE_ERROR_GENERIC; + } + } + + if (regulator->flags & REGULATOR_PULL_DOWN) { + if (stpmic1_bo_pull_down_cfg(priv->regu_name, &cfg) || + stpmic1_bo_pull_down_unpg(&cfg)) { + EMSG("Pull down failed for %s", priv->regu_name); + return TEE_ERROR_GENERIC; + } + } + + stm32mp_put_pmic(); + + return TEE_SUCCESS; +} + +static const struct regulator_ops pmic_regu_ops = { + .set_state = pmic_set_state, + .get_state = pmic_get_state, + .set_voltage = pmic_set_voltage, + .get_voltage = pmic_get_voltage, + .supported_voltages = pmic_list_voltages, + .supplied_init = pmic_regu_init, +}; +DECLARE_KEEP_PAGER(pmic_regu_ops); + +static const struct regulator_ops pmic_sw_ops = { + .set_state = pmic_set_state, + .get_state = pmic_get_state, + .supplied_init = pmic_regu_init, +}; +DECLARE_KEEP_PAGER(pmic_sw_ops); + +/* + * STPMIC1 regulator names, used in the DT as regulator node name and + * provider node -supply property, + */ +static const char * const pmic_regu_name_ids[] = { + "buck1", "buck2", "buck3", "buck4", + "ldo1", "ldo2", "ldo3", "ldo4", "ldo5", "ldo6", + "vref_ddr", "boost", "pwr_sw1", "pwr_sw2" +}; + +/* Preallocated regulator instances */ +static struct regulator pmic_regulators[ARRAY_SIZE(pmic_regu_name_ids)]; +static struct pmic_regulator_data pmic_regu_cfg[ARRAY_SIZE(pmic_regu_name_ids)]; + +static TEE_Result release_voltage_lists(void) +{ + size_t n = 0; + + /* Voltage list will be rebuilt at runtime if needed at least once */ + for (n = 0; n < ARRAY_SIZE(pmic_regulators); n++) { + struct pmic_regulator_data *priv = pmic_regulators[n].priv; + + if (priv && priv->voltages) + free(priv->voltages); + } + + return TEE_SUCCESS; +} + +release_init_resource(release_voltage_lists); + +struct regulator *stm32mp_pmic_get_regulator(const char *name) +{ + size_t i = 0; + + if (!name) + return NULL; + + for (i = 0; i < ARRAY_SIZE(pmic_regu_name_ids); i++) + if (!strcmp(pmic_regu_name_ids[i], name) && + pmic_regulators[i].ops) + return pmic_regulators + i; + + return NULL; +} + +static TEE_Result register_pmic_regulator(const void *fdt, + const char *regu_name, int regu_node, + int regulators_node) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct regu_dt_desc desc = { }; + size_t i = 0; + + for (i = 0; i < ARRAY_SIZE(pmic_regu_name_ids); i++) + if (!strcmp(pmic_regu_name_ids[i], regu_name)) + break; + if (i >= ARRAY_SIZE(pmic_regu_name_ids)) { + EMSG("Unknown regulator name %s", regu_name); + panic(); + } + + desc = (struct regu_dt_desc){ + .name = pmic_regu_name_ids[i], + .supply_name = pmic_regu_name_ids[i], + .regulator = pmic_regulators + i, + .priv = pmic_regu_cfg + i, + }; + + priv_dt_properties(fdt, regu_node, pmic_regu_cfg + i); + + /* + * pwr_sw1 and pwr_sw2 are regulator switches hence have no + * set_voltage.get_voltage handler. + */ + if (!strncmp(regu_name, "pwr_sw", 6)) + desc.ops = &pmic_sw_ops; + else + desc.ops = &pmic_regu_ops; + + res = regulator_dt_register(fdt, regu_node, regulators_node, &desc); + if (res) + EMSG("Failed to register %s, error: %#"PRIx32, regu_name, res); + + return res; +} + +static void parse_regulator_fdt_nodes(const void *fdt, int pmic_node) +{ + int regulators_node = 0; + int regu_node = 0; + + regulators_node = fdt_subnode_offset(fdt, pmic_node, "regulators"); + if (regulators_node < 0) + panic(); + + fdt_for_each_subnode(regu_node, fdt, regulators_node) { + int status = fdt_get_status(fdt, regu_node); + const char *regu_name = NULL; + size_t n = 0; + + assert(status >= 0); + if (status == DT_STATUS_DISABLED) + continue; + + regu_name = fdt_get_name(fdt, regu_node, NULL); + + assert(stpmic1_regulator_is_valid(regu_name)); + + if (status & DT_STATUS_OK_NSEC) + register_nsec_regu(regu_name); + + for (n = 0; n < ARRAY_SIZE(regu_lp_state); n++) + dt_get_regu_low_power_config(fdt, regu_name, regu_node, + regu_lp_state[n].name); + + if (register_pmic_regulator(fdt, regu_name, regu_node, + regulators_node)) + panic(); + } + + if (save_cpu_supply_name()) + DMSG("No CPU supply provided"); +} + +/* + * PMIC and resource initialization + */ + +static void initialize_pmic_i2c(const void *fdt, int pmic_node) +{ + const fdt32_t *cuint = NULL; + + cuint = fdt_getprop(fdt, pmic_node, "reg", NULL); + if (!cuint) { + EMSG("PMIC configuration failed on reg property"); + panic(); + } + + pmic_i2c_addr = fdt32_to_cpu(*cuint) << 1; + if (pmic_i2c_addr > UINT16_MAX) { + EMSG("PMIC configuration failed on i2c address translation"); + panic(); + } + + stm32mp_get_pmic(); + + if (!stm32_i2c_is_device_ready(i2c_handle, pmic_i2c_addr, + PMIC_I2C_TRIALS, + PMIC_I2C_TIMEOUT_BUSY_MS)) + panic(); + + stpmic1_bind_i2c(i2c_handle, pmic_i2c_addr); + + stm32mp_put_pmic(); +} + +/* + * Automated suspend/resume at system suspend/resume is expected + * only when the PMIC is secure. If it is non secure, only atomic + * execution context can get/put the PMIC resources. + */ +static TEE_Result pmic_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *pm_handle __unused) +{ + if (op == PM_OP_SUSPEND) + stm32_i2c_suspend(i2c_handle); + else + stm32_i2c_resume(i2c_handle); + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(pmic_pm); + +/* stm32mp_get/put_pmic allows secure atomic sequences to use non secure PMIC */ +void stm32mp_get_pmic(void) +{ + stm32_i2c_resume(i2c_handle); +} + +void stm32mp_put_pmic(void) +{ + stm32_i2c_suspend(i2c_handle); +} + +static void register_non_secure_pmic(void) +{ + /* Allow this function to be called when STPMIC1 not used */ + if (!i2c_handle->base.pa) + return; + + stm32mp_register_non_secure_pinctrl(i2c_handle->pinctrl); + if (i2c_handle->pinctrl_sleep) + stm32mp_register_non_secure_pinctrl(i2c_handle->pinctrl_sleep); + + stm32mp_register_non_secure_periph_iomem(i2c_handle->base.pa); +} + +static void register_secure_pmic(void) +{ + stm32mp_register_secure_pinctrl(i2c_handle->pinctrl); + if (i2c_handle->pinctrl_sleep) + stm32mp_register_secure_pinctrl(i2c_handle->pinctrl_sleep); + + stm32mp_register_secure_periph_iomem(i2c_handle->base.pa); + register_pm_driver_cb(pmic_pm, NULL, "stm32mp1-pmic"); +} + +static TEE_Result initialize_pmic(const void *fdt, int pmic_node) +{ + unsigned long pmic_version = 0; + + init_pmic_state(fdt, pmic_node); + + initialize_pmic_i2c(fdt, pmic_node); + + stm32mp_get_pmic(); + + if (stpmic1_get_version(&pmic_version)) + panic("Failed to access PMIC"); + + DMSG("PMIC version = 0x%02lx", pmic_version); + stm32mp_put_pmic(); + + if (dt_pmic_is_secure()) + register_secure_pmic(); + else + register_non_secure_pmic(); + + parse_regulator_fdt_nodes(fdt, pmic_node); + + return TEE_SUCCESS; +} + +static TEE_Result stm32_pmic_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + struct stm32_i2c_dev *stm32_i2c_dev = NULL; + struct i2c_dev *i2c_dev = NULL; + TEE_Result res = TEE_SUCCESS; + + res = i2c_dt_get_dev(fdt, node, &i2c_dev); + if (res) + return res; + + stm32_i2c_dev = container_of(i2c_dev, struct stm32_i2c_dev, i2c_dev); + i2c_handle = stm32_i2c_dev->handle; + + res = initialize_pmic(fdt, node); + if (res) { + DMSG("Unexpectedly failed to get I2C bus: %#"PRIx32, res); + panic(); + } + + return TEE_SUCCESS; +} + +static const struct dt_device_match stm32_pmic_match_table[] = { + { .compatible = "st,stpmic1" }, + { } +}; + +DEFINE_DT_DRIVER(stm32_pmic_dt_driver) = { + .name = "st,stpmic1", + .match_table = stm32_pmic_match_table, + .probe = stm32_pmic_probe, +}; diff --git a/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pmic.h b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pmic.h new file mode 100644 index 0000000..73794e3 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pmic.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2017-2020, STMicroelectronics + */ + +#ifndef __STM32MP1_PMIC_H__ +#define __STM32MP1_PMIC_H__ + +#include + +#ifdef CFG_STPMIC1 +void stm32mp_pmic_apply_boot_on_config(void); +void stm32mp_pmic_apply_lp_config(const char *lp_state); +void stm32mp_get_pmic(void); +void stm32mp_put_pmic(void); +const char *stm32mp_pmic_get_cpu_supply_name(void); + +/* Get the PMIC regulator related to @name or NULL if not found */ +struct regulator *stm32mp_pmic_get_regulator(const char *name); +#else +static inline void stm32mp_pmic_apply_boot_on_config(void) +{ +} + +static inline void stm32mp_pmic_apply_lp_config(const char *lp_state __unused) +{ +} + +static inline void stm32mp_get_pmic(void) +{ + panic(); +} + +static inline void stm32mp_put_pmic(void) +{ + panic(); +} + +static inline const char *stm32mp_pmic_get_cpu_supply_name(void) +{ + return NULL; +} + +static inline struct regulator * +stm32mp_pmic_get_regulator(const char *name __unused) +{ + return NULL; +} +#endif + +#endif /*__STM32MP1_PMIC_H__*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pwr.c b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pwr.c new file mode 100644 index 0000000..4785c9d --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pwr.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2018-2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PWR_CR3_USB33_EN BIT(24) +#define PWR_CR3_USB33_RDY BIT(26) +#define PWR_CR3_REG18_EN BIT(28) +#define PWR_CR3_REG18_RDY BIT(29) +#define PWR_CR3_REG11_EN BIT(30) +#define PWR_CR3_REG11_RDY BIT(31) + +#define TIMEOUT_US_10MS U(10000) + +struct pwr_regu_desc { + unsigned int level_mv; + uint32_t cr3_enable_mask; + uint32_t cr3_ready_mask; +}; + +static const struct pwr_regu_desc pwr_regulators[PWR_REGU_COUNT] = { + [PWR_REG11] = { + .level_mv = 1100, + .cr3_enable_mask = PWR_CR3_REG11_EN, + .cr3_ready_mask = PWR_CR3_REG11_RDY, + }, + [PWR_REG18] = { + .level_mv = 1800, + .cr3_enable_mask = PWR_CR3_REG18_EN, + .cr3_ready_mask = PWR_CR3_REG18_RDY, + }, + [PWR_USB33] = { + .level_mv = 3300, + .cr3_enable_mask = PWR_CR3_USB33_EN, + .cr3_ready_mask = PWR_CR3_USB33_RDY, + }, +}; + +vaddr_t stm32_pwr_base(void) +{ + static struct io_pa_va base = { .pa = PWR_BASE }; + + return io_pa_or_va_secure(&base, 1); +} + +unsigned int stm32mp1_pwr_regulator_mv(enum pwr_regulator id) +{ + assert(id < PWR_REGU_COUNT); + + return pwr_regulators[id].level_mv; +} + +void stm32mp1_pwr_regulator_set_state(enum pwr_regulator id, bool enable) +{ + uintptr_t cr3 = stm32_pwr_base() + PWR_CR3_OFF; + uint32_t enable_mask = pwr_regulators[id].cr3_enable_mask; + + assert(id < PWR_REGU_COUNT); + + if (enable) { + uint32_t ready_mask = pwr_regulators[id].cr3_ready_mask; + uint64_t to = 0; + + io_setbits32(cr3, enable_mask); + + to = timeout_init_us(10 * 1000); + while (!timeout_elapsed(to)) + if (io_read32(cr3) & ready_mask) + break; + + if (!(io_read32(cr3) & ready_mask)) + panic(); + } else { + io_clrbits32(cr3, enable_mask); + } +} + +bool stm32mp1_pwr_regulator_is_enabled(enum pwr_regulator id) +{ + assert(id < PWR_REGU_COUNT); + + return io_read32(stm32_pwr_base() + PWR_CR3_OFF) & + pwr_regulators[id].cr3_enable_mask; +} + +static TEE_Result stm32mp1_pwr_regu_set_state(struct regulator *regu, + bool enable) +{ + const struct pwr_regu_desc *desc = regu->priv; + uintptr_t cr3 = stm32_pwr_base() + PWR_CR3_OFF; + uint64_t to = 0; + + assert(desc); + + if (enable) { + io_setbits32_stm32shregs(cr3, desc->cr3_enable_mask); + + to = timeout_init_us(TIMEOUT_US_10MS); + while (!timeout_elapsed(to)) + if (io_read32(cr3) & desc->cr3_ready_mask) + break; + + if (!(io_read32(cr3) & desc->cr3_ready_mask)) + return TEE_ERROR_GENERIC; + } else { + io_clrbits32_stm32shregs(cr3, desc->cr3_enable_mask); + } + + return TEE_SUCCESS; +} + +static TEE_Result stm32mp1_pwr_regu_read_state(struct regulator *regu, + bool *enabled) +{ + const struct pwr_regu_desc *desc = regu->priv; + + assert(desc); + + *enabled = io_read32(stm32_pwr_base() + PWR_CR3_OFF) & + desc->cr3_enable_mask; + + return TEE_SUCCESS; +} + +static TEE_Result stm32mp1_pwr_regu_read_voltage(struct regulator *regu, + int *level_uv) +{ + const struct pwr_regu_desc *desc = regu->priv; + + assert(desc); + + *level_uv = (int)desc->level_mv * 1000; + + return TEE_SUCCESS; +} + +static const struct regulator_ops stm32mp1_pwr_regu_ops = { + .set_state = stm32mp1_pwr_regu_set_state, + .get_state = stm32mp1_pwr_regu_read_state, + .get_voltage = stm32mp1_pwr_regu_read_voltage, +}; + +/* Preallocated regulator devices */ +static struct regulator pwr_regu_device[PWR_REGU_COUNT]; + +#define DEFINE_REG(_id, _name, _supply) { \ + .ops = &stm32mp1_pwr_regu_ops, \ + .name = _name, \ + .supply_name = _supply, \ + .priv = (void *)(pwr_regulators + (_id)), \ + .regulator = pwr_regu_device + (_id), \ +} + +static const struct regu_dt_desc stm32mp1_pwr_regu_dt_desc[] = { + [PWR_REG11] = DEFINE_REG(PWR_REG11, "reg11", "vdd"), + [PWR_REG18] = DEFINE_REG(PWR_REG18, "reg18", "vdd"), + [PWR_USB33] = DEFINE_REG(PWR_USB33, "usb33", "vdd_3v3_usbfs"), +}; +DECLARE_KEEP_PAGER(stm32mp1_pwr_regu_dt_desc); + +struct regulator *stm32mp1_pwr_get_regulator(enum pwr_regulator id) +{ + if (id < ARRAY_SIZE(pwr_regu_device)) + return pwr_regu_device + id; + + return NULL; +} + +static TEE_Result stm32mp1_pwr_regu_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const struct regu_dt_desc *dt_desc = stm32mp1_pwr_regu_dt_desc; + int subnode = 0; + + fdt_for_each_subnode(subnode, fdt, node) { + const char *node_name = fdt_get_name(fdt, subnode, NULL); + unsigned int n = 0; + + for (n = 0; n < ARRAY_SIZE(stm32mp1_pwr_regu_dt_desc); n++) + if (!strcmp(dt_desc[n].name, node_name)) + break; + + if (n >= ARRAY_SIZE(stm32mp1_pwr_regu_dt_desc)) { + EMSG("Invalid PWR regulator node %s", node_name); + panic(); + } + + if (IS_ENABLED(CFG_DRIVERS_REGULATOR)) { + res = regulator_dt_register(fdt, subnode, node, + dt_desc + n); + if (res) { + EMSG("Can't register %s: %#"PRIx32, node_name, + res); + panic(); + } + } + } + + return TEE_SUCCESS; +} + +static const struct dt_device_match stm32mp1_pwr_regu_match_table[] = { + { .compatible = "st,stm32mp1,pwr-reg" }, + { } +}; + +DEFINE_DT_DRIVER(stm32mp1_pwr_regu_dt_driver) = { + .name = "stm32mp1-pwr-regu", + .match_table = stm32mp1_pwr_regu_match_table, + .probe = stm32mp1_pwr_regu_probe, +}; diff --git a/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pwr.h b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pwr.h new file mode 100644 index 0000000..65718a0 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_pwr.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2018-2019, STMicroelectronics + */ + +#ifndef __STM32MP1_PWR_H +#define __STM32MP1_PWR_H + +#include +#include +#include + +#define PWR_CR1_OFF 0x00 +#define PWR_CR2_OFF 0x08 +#define PWR_CR3_OFF 0x0c +#define PWR_MPUCR_OFF 0x10 +#define PWR_WKUPCR_OFF 0x20 +#define PWR_MPUWKUPENR_OFF 0x28 + +#define PWR_OFFSET_MASK 0x3fUL + +enum pwr_regulator { + PWR_REG11 = 0, + PWR_REG18, + PWR_USB33, + PWR_REGU_COUNT +}; + +vaddr_t stm32_pwr_base(void); + +unsigned int stm32mp1_pwr_regulator_mv(enum pwr_regulator id); +void stm32mp1_pwr_regulator_set_state(enum pwr_regulator id, bool enable); +bool stm32mp1_pwr_regulator_is_enabled(enum pwr_regulator id); + +/* Returns the registered regulator related to @id or NULL */ +struct regulator *stm32mp1_pwr_get_regulator(enum pwr_regulator id); +#endif /*__STM32MP1_PWR_H*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_syscfg.c b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_syscfg.c new file mode 100644 index 0000000..281ae7c --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/drivers/stm32mp1_syscfg.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2022, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * SYSCFG register offsets (base relative) + */ +#define SYSCFG_CMPCR 0x20U +#define SYSCFG_CMPENSETR 0x24U + +/* + * SYSCFG_CMPCR Register + */ +#define SYSCFG_CMPCR_SW_CTRL BIT(1) +#define SYSCFG_CMPCR_READY BIT(8) +#define SYSCFG_CMPCR_RANSRC GENMASK_32(19, 16) +#define SYSCFG_CMPCR_RANSRC_SHIFT 16 +#define SYSCFG_CMPCR_RAPSRC GENMASK_32(23, 20) +#define SYSCFG_CMPCR_ANSRC_SHIFT 24 + +#define SYSCFG_CMPCR_READY_TIMEOUT_US 1000U + +/* + * SYSCFG_CMPENSETR Register + */ +#define SYSCFG_CMPENSETR_MPU_EN BIT(0) + +static vaddr_t get_syscfg_base(void) +{ + struct io_pa_va base = { .pa = SYSCFG_BASE }; + + return io_pa_or_va(&base, 1); +} + +void stm32mp_syscfg_enable_io_compensation(void) +{ + vaddr_t syscfg_base = get_syscfg_base(); + uint64_t timeout_ref = 0; + + if (clk_enable(stm32mp_rcc_clock_id_to_clk(CK_CSI)) || + clk_enable(stm32mp_rcc_clock_id_to_clk(SYSCFG))) + panic(); + + io_setbits32(syscfg_base + SYSCFG_CMPENSETR, SYSCFG_CMPENSETR_MPU_EN); + + timeout_ref = timeout_init_us(SYSCFG_CMPCR_READY_TIMEOUT_US); + + while (!(io_read32(syscfg_base + SYSCFG_CMPCR) & SYSCFG_CMPCR_READY)) { + if (timeout_elapsed(timeout_ref)) { + EMSG("IO compensation cell not ready"); + /* Allow an almost silent failure here */ + break; + } + } + + io_clrbits32(syscfg_base + SYSCFG_CMPCR, SYSCFG_CMPCR_SW_CTRL); + + DMSG("SYSCFG.cmpcr = %#"PRIx32, io_read32(syscfg_base + SYSCFG_CMPCR)); +} + +void stm32mp_syscfg_disable_io_compensation(void) +{ + vaddr_t syscfg_base = get_syscfg_base(); + uint32_t value = 0; + + value = io_read32(syscfg_base + SYSCFG_CMPCR) >> + SYSCFG_CMPCR_ANSRC_SHIFT; + + io_clrbits32(syscfg_base + SYSCFG_CMPCR, + SYSCFG_CMPCR_RANSRC | SYSCFG_CMPCR_RAPSRC); + + value = io_read32(syscfg_base + SYSCFG_CMPCR) | + (value << SYSCFG_CMPCR_RANSRC_SHIFT); + + io_write32(syscfg_base + SYSCFG_CMPCR, value | SYSCFG_CMPCR_SW_CTRL); + + DMSG("SYSCFG.cmpcr = %#"PRIx32, io_read32(syscfg_base + SYSCFG_CMPCR)); + + io_clrbits32(syscfg_base + SYSCFG_CMPENSETR, SYSCFG_CMPENSETR_MPU_EN); + + clk_disable(stm32mp_rcc_clock_id_to_clk(CK_CSI)); + clk_disable(stm32mp_rcc_clock_id_to_clk(SYSCFG)); +} + +static TEE_Result stm32mp1_iocomp(void) +{ + stm32mp_syscfg_enable_io_compensation(); + + return TEE_SUCCESS; +} +driver_init(stm32mp1_iocomp); diff --git a/optee_os/core/arch/arm/plat-stm32mp1/drivers/sub.mk b/optee_os/core/arch/arm/plat-stm32mp1/drivers/sub.mk new file mode 100644 index 0000000..b07c8f9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/drivers/sub.mk @@ -0,0 +1,3 @@ +srcs-$(CFG_STPMIC1) += stm32mp1_pmic.c +srcs-y += stm32mp1_pwr.c +srcs-y += stm32mp1_syscfg.c diff --git a/optee_os/core/arch/arm/plat-stm32mp1/link.mk b/optee_os/core/arch/arm/plat-stm32mp1/link.mk new file mode 100644 index 0000000..01a9b8e --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/link.mk @@ -0,0 +1,24 @@ +include core/arch/arm/kernel/link.mk + +# Create stm32 formatted images from the native binary images + +define stm32image_cmd + @$(cmd-echo-silent) ' GEN $@' + $(q)./core/arch/arm/plat-stm32mp1/scripts/stm32image.py \ + --load 0 --entry 0 +endef + +all: $(link-out-dir)/tee-header_v2.stm32 +cleanfiles += $(link-out-dir)/tee-header_v2.stm32 +$(link-out-dir)/tee-header_v2.stm32: $(link-out-dir)/tee-header_v2.bin + $(stm32image_cmd) --source $< --dest $@ --bintype 0x20 + +all: $(link-out-dir)/tee-pager_v2.stm32 +cleanfiles += $(link-out-dir)/tee-pager_v2.stm32 +$(link-out-dir)/tee-pager_v2.stm32: $(link-out-dir)/tee-pager_v2.bin + $(stm32image_cmd) --source $< --dest $@ --bintype 0x21 + +all: $(link-out-dir)/tee-pageable_v2.stm32 +cleanfiles += $(link-out-dir)/tee-pageable_v2.stm32 +$(link-out-dir)/tee-pageable_v2.stm32: $(link-out-dir)/tee-pageable_v2.bin + $(stm32image_cmd) --source $< --dest $@ --bintype 0x22 diff --git a/optee_os/core/arch/arm/plat-stm32mp1/link_dummies_paged.c b/optee_os/core/arch/arm/plat-stm32mp1/link_dummies_paged.c new file mode 100644 index 0000000..989d5c2 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/link_dummies_paged.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ +#include +#include +#include + +const struct clk_ops stm32mp1_clk_ops __rodata_dummy; diff --git a/optee_os/core/arch/arm/plat-stm32mp1/main.c b/optee_os/core/arch/arm/plat-stm32mp1/main.c new file mode 100644 index 0000000..eebf0bd --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/main.c @@ -0,0 +1,586 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2022, STMicroelectronics + * Copyright (c) 2016-2018, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, APB1_BASE, APB1_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, APB2_BASE, APB2_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, APB3_BASE, APB3_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, APB4_BASE, APB4_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, APB5_BASE, APB5_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, AHB4_BASE, AHB4_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, AHB5_BASE, AHB5_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, APB1_BASE, APB1_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, APB3_BASE, APB3_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, APB4_BASE, APB4_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, APB5_BASE, APB5_SIZE); +#ifdef CFG_STM32MP13 +register_phys_mem_pgdir(MEM_AREA_IO_SEC, APB6_BASE, APB6_SIZE); +#endif +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AHB4_BASE, AHB4_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, AHB5_BASE, AHB5_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE, GIC_SIZE); + +register_ddr(DDR_BASE, CFG_DRAM_SIZE); + +#define _ID2STR(id) (#id) +#define ID2STR(id) _ID2STR(id) + +static TEE_Result platform_banner(void) +{ + IMSG("Platform stm32mp1: flavor %s - DT %s", + ID2STR(PLATFORM_FLAVOR), + ID2STR(CFG_EMBED_DTB_SOURCE_FILE)); + + return TEE_SUCCESS; +} +service_init(platform_banner); + +/* + * Console + * + * CFG_STM32_EARLY_CONSOLE_UART specifies the ID of the UART used for + * trace console. Value 0 disables the early console. + * + * We cannot use the generic serial_console support since probing + * the console requires the platform clock driver to be already + * up and ready which is done only once service_init are completed. + */ +static struct stm32_uart_pdata console_data; + +void console_init(void) +{ + /* Early console initialization before MMU setup */ + struct uart { + paddr_t pa; + bool secure; + } uarts[] = { + [0] = { .pa = 0 }, + [1] = { .pa = USART1_BASE, .secure = true, }, + [2] = { .pa = USART2_BASE, .secure = false, }, + [3] = { .pa = USART3_BASE, .secure = false, }, + [4] = { .pa = UART4_BASE, .secure = false, }, + [5] = { .pa = UART5_BASE, .secure = false, }, + [6] = { .pa = USART6_BASE, .secure = false, }, + [7] = { .pa = UART7_BASE, .secure = false, }, + [8] = { .pa = UART8_BASE, .secure = false, }, + }; + + COMPILE_TIME_ASSERT(ARRAY_SIZE(uarts) > CFG_STM32_EARLY_CONSOLE_UART); + + if (!uarts[CFG_STM32_EARLY_CONSOLE_UART].pa) + return; + + /* No clock yet bound to the UART console */ + console_data.clock = NULL; + + console_data.secure = uarts[CFG_STM32_EARLY_CONSOLE_UART].secure; + stm32_uart_init(&console_data, uarts[CFG_STM32_EARLY_CONSOLE_UART].pa); + + register_serial_console(&console_data.chip); + + IMSG("Early console on UART#%u", CFG_STM32_EARLY_CONSOLE_UART); +} + +static TEE_Result init_console_from_dt(void) +{ + struct stm32_uart_pdata *pd = NULL; + void *fdt = NULL; + int node = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + fdt = get_embedded_dt(); + res = get_console_node_from_dt(fdt, &node, NULL, NULL); + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + fdt = get_external_dt(); + res = get_console_node_from_dt(fdt, &node, NULL, NULL); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + return TEE_SUCCESS; + if (res != TEE_SUCCESS) + return res; + } + + pd = stm32_uart_init_from_dt_node(fdt, node); + if (!pd) { + IMSG("DTB disables console"); + register_serial_console(NULL); + return TEE_SUCCESS; + } + + /* Replace early console with the new one */ + console_flush(); + console_data = *pd; + register_serial_console(&console_data.chip); + IMSG("DTB enables console (%ssecure)", pd->secure ? "" : "non-"); + free(pd); + + return TEE_SUCCESS; +} + +/* Probe console from DT once clock inits (service init level) are completed */ +service_init_late(init_console_from_dt); + +/* + * GIC init, used also for primary/secondary boot core wake completion + */ +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); + + stm32mp_register_online_cpu(); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); + + stm32mp_register_online_cpu(); +} + +#ifdef CFG_STM32MP13 +#ifdef CFG_STM32_ETZPC +/* Configure ETZPC cell and lock it when resource is secure */ +static void config_lock_decprot(uint32_t decprot_id, + enum etzpc_decprot_attributes decprot_attr) +{ + etzpc_configure_decprot(decprot_id, decprot_attr); + + if (decprot_attr == ETZPC_DECPROT_S_RW) + etzpc_lock_decprot(decprot_id); +} + +static TEE_Result set_etzpc_secure_configuration(void) +{ + config_lock_decprot(STM32MP1_ETZPC_BKPSRAM_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_DDRCTRLPHY_ID, + ETZPC_DECPROT_NS_R_S_W); + + /* Configure ETZPC with peripheral registering */ + config_lock_decprot(STM32MP1_ETZPC_ADC1_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_ADC2_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_CRYP_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_DCMIPP_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_ETH1_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_ETH2_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_FMC_ID, ETZPC_DECPROT_NS_RW); + /* HASH is secure */ + config_lock_decprot(STM32MP1_ETZPC_HASH_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_I2C3_ID, ETZPC_DECPROT_NS_RW); + /* I2C4 is secure */ + config_lock_decprot(STM32MP1_ETZPC_I2C4_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_I2C5_ID, ETZPC_DECPROT_NS_RW); + /* IWDG1 is secure */ + config_lock_decprot(STM32MP1_ETZPC_IWDG1_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_LPTIM2_ID, ETZPC_DECPROT_NS_RW); + /* LPTIM3 is secure */ + config_lock_decprot(STM32MP1_ETZPC_LPTIM3_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_LTDC_ID, ETZPC_DECPROT_NS_RW); + /* MCE is secure */ + config_lock_decprot(STM32MP1_ETZPC_MCE_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_OTG_ID, ETZPC_DECPROT_NS_RW); + /* PKA is secure */ + config_lock_decprot(STM32MP1_ETZPC_PKA_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_QSPI_ID, ETZPC_DECPROT_NS_RW); + /* RNG is secure */ + config_lock_decprot(STM32MP1_ETZPC_RNG_ID, ETZPC_DECPROT_S_RW); + /* SAES is secure */ + config_lock_decprot(STM32MP1_ETZPC_SAES_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_SDMMC1_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_SDMMC2_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_SPI4_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_SPI5_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_SRAM1_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_SRAM2_ID, ETZPC_DECPROT_NS_RW); + /* SRAM3 is secure */ + config_lock_decprot(STM32MP1_ETZPC_SRAM3_ID, ETZPC_DECPROT_S_RW); + /* STGENC is secure */ + config_lock_decprot(STM32MP1_ETZPC_STGENC_ID, ETZPC_DECPROT_S_RW); + /* TIM12 is secure */ + config_lock_decprot(STM32MP1_ETZPC_TIM12_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_TIM13_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_TIM14_ID, ETZPC_DECPROT_NS_RW); + /* TIM15 is secure */ + config_lock_decprot(STM32MP1_ETZPC_TIM15_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_TIM16_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_TIM17_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_USART1_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_USART2_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_USBPHYCTRL_ID, ETZPC_DECPROT_NS_RW); + config_lock_decprot(STM32MP1_ETZPC_VREFBUF_ID, ETZPC_DECPROT_NS_RW); + + return TEE_SUCCESS; +} + +driver_init_late(set_etzpc_secure_configuration); +#endif /* CFG_STM32_ETZPC */ +#endif /* CFG_STM32MP13 */ + +#ifdef CFG_STM32MP15 +/* + * This concerns OP-TEE pager for STM32MP1 to use secure internal + * RAMs to execute. TZSRAM refers the TZSRAM_BASE/TZSRAM_SIZE + * used in boot.c to locate secure unpaged memory. + * + * STM32MP15 variants embed 640kB of contiguous securable SRAMs + * + * +--------------+ <-- SYSRAM_BASE + * | | lower part can be assigned to secure world + * | SYSRAM 256kB | 4kB granule boundary + * | | upper part can be assigned to secure world + * +--------------+ <-- SRAM1_BASE (= SYSRAM_BASE + SYSRAM_SIZE) + | | full range assigned to non-secure world or + * | SRAM1 128kB | to secure world, or to- Cortex-M4 exclusive access + * +--------------+ <-- SRAM2_BASE (= SRAM1_BASE + SRAM1_SIZE) + | | full range assigned to non-secure world or + * | SRAM2 128kB | to secure world, or to- Cortex-M4 exclusive access + * +--------------+ <-- SRAM3_BASE (= SRAM2_BASE + SRAM2_SIZE) + | | full range assigned to non-secure world or + * | SRAM3 64kB | to secure world, or to- Cortex-M4 exclusive access + * +--------------+ <-- SRAM4_BASE (= SRAM3_BASE + SRAM3_SIZE) + | | full range assigned to non-secure world or + * | SRAM4 64kB | to secure world, or to- Cortex-M4 exclusive access + * +--------------+ <-- SRAM4_BASE + SRAM4_SIZE + * + * If SRAMx memories are not used for the companion Cortex-M4 + * processor, OP-TEE can use this memory. + * + * SYSRAM configuration for secure/non-secure boundaries requires the + * secure SYSRAM memory to start at the SYSRAM physical base address and grow + * from there while the non-secure SYSRAM range lies at SYSRAM end addresses + * with a 4KB page granule. + * + * SRAM1, SRAM2, SRAM3 and SRAM4 are independently assigned to secure world, + * to non-secure world or possibly to Cortex-M4 exclusive access. Each + * assignment covers the full related SRAMx memory range. + * + * Using non-secure SYSRAM or one of the SRAMx for SCMI message communication + * can be done using CFG_STM32MP1_SCMI_SHM_BASE/CFG_STM32MP1_SCMI_SHM_SIZE. + * This imposes related memory area is assigned to non-secure world. + + * Using secure internal memories (SYSRAM and/or some SRAMx) with STM32MP15 + * shall meet this constraints known the TZSRAM physical memory range shall + * be contiguous. + */ + +#define SYSRAM_END (SYSRAM_BASE + SYSRAM_SIZE) +#define SYSRAM_SEC_END (SYSRAM_BASE + SYSRAM_SEC_SIZE) +#define SRAMS_END (SRAM4_BASE + SRAM4_SIZE) +#define SRAMS_START SRAM1_BASE +#define TZSRAM_END (CFG_TZSRAM_START + CFG_TZSRAM_SIZE) + +#define SCMI_SHM_IS_IN_SRAMX ((CFG_STM32MP1_SCMI_SHM_BASE >= SRAM1_BASE) && \ + (CFG_STM32MP1_SCMI_SHM_BASE + \ + CFG_STM32MP1_SCMI_SHM_SIZE) <= SRAMS_END) + +#define TZSRAM_FITS_IN_SYSRAM_SEC ((CFG_TZSRAM_START >= SYSRAM_BASE) && \ + (TZSRAM_END <= SYSRAM_SEC_END)) + +#define TZSRAM_FITS_IN_SYSRAM_AND_SRAMS ((CFG_TZSRAM_START >= SYSRAM_BASE) && \ + (CFG_TZSRAM_START < SYSRAM_END) && \ + (TZSRAM_END > SYSRAM_END) && \ + (TZSRAM_END <= SRAMS_END) && \ + (SYSRAM_SIZE == SYSRAM_SEC_SIZE)) + +#define TZSRAM_FITS_IN_SRAMS ((CFG_TZSRAM_START >= SRAMS_START) && \ + (CFG_TZSRAM_START < SRAMS_END) && \ + (TZSRAM_END <= SRAMS_END)) + +#define TZSRAM_IS_IN_DRAM (CFG_TZSRAM_START >= CFG_DRAM_BASE) + +#ifdef CFG_WITH_PAGER +/* + * At build time, we enforce that, when pager is used, + * either TZSRAM fully fits inside SYSRAM secure address range, + * or TZSRAM fully fits inside the full SYSRAM and spread inside SRAMx orderly, + * or TZSRAM fully fits some inside SRAMs address range, + * or TZSRAM is in DDR for debug and test purpose. + */ +static_assert(TZSRAM_FITS_IN_SYSRAM_SEC || TZSRAM_FITS_IN_SYSRAM_AND_SRAMS || + TZSRAM_FITS_IN_SRAMS || TZSRAM_IS_IN_DRAM); +#endif + +#if TZSRAM_FITS_IN_SYSRAM_AND_SRAMS || TZSRAM_FITS_IN_SRAMS || \ + SCMI_SHM_IS_IN_SRAMX +/* At run time we enforce that SRAM1 to SRAM4 are properly assigned if used */ +static TEE_Result init_stm32mp15_secure_srams(void) +{ + if (IS_ENABLED(CFG_WITH_PAGER)) { + if (core_is_buffer_intersect(CFG_TZSRAM_START, CFG_TZSRAM_SIZE, + SRAM1_BASE, SRAM1_SIZE)) + stm32mp_register_secure_periph_iomem(SRAM1_BASE); + + if (core_is_buffer_intersect(CFG_TZSRAM_START, CFG_TZSRAM_SIZE, + SRAM2_BASE, SRAM2_SIZE)) + stm32mp_register_secure_periph_iomem(SRAM2_BASE); + + if (core_is_buffer_intersect(CFG_TZSRAM_START, CFG_TZSRAM_SIZE, + SRAM3_BASE, SRAM3_SIZE)) + stm32mp_register_secure_periph_iomem(SRAM3_BASE); + + if (core_is_buffer_intersect(CFG_TZSRAM_START, CFG_TZSRAM_SIZE, + SRAM4_BASE, SRAM4_SIZE)) + stm32mp_register_secure_periph_iomem(SRAM4_BASE); + } + + if (SCMI_SHM_IS_IN_SRAMX) { + if (core_is_buffer_intersect(CFG_STM32MP1_SCMI_SHM_BASE, + CFG_STM32MP1_SCMI_SHM_SIZE, + SRAM1_BASE, SRAM1_SIZE)) + stm32mp_register_non_secure_periph_iomem(SRAM1_BASE); + + if (core_is_buffer_intersect(CFG_STM32MP1_SCMI_SHM_BASE, + CFG_STM32MP1_SCMI_SHM_SIZE, + SRAM2_BASE, SRAM2_SIZE)) + stm32mp_register_non_secure_periph_iomem(SRAM2_BASE); + + if (core_is_buffer_intersect(CFG_STM32MP1_SCMI_SHM_BASE, + CFG_STM32MP1_SCMI_SHM_SIZE, + SRAM3_BASE, SRAM3_SIZE)) + stm32mp_register_non_secure_periph_iomem(SRAM3_BASE); + + if (core_is_buffer_intersect(CFG_STM32MP1_SCMI_SHM_BASE, + CFG_STM32MP1_SCMI_SHM_SIZE, + SRAM4_BASE, SRAM4_SIZE)) + stm32mp_register_non_secure_periph_iomem(SRAM4_BASE); + } + + return TEE_SUCCESS; +} + +service_init_late(init_stm32mp15_secure_srams); +#endif /* TZSRAM_FITS_IN_SYSRAM_AND_SRAMS || TZSRAM_FITS_IN_SRAMS */ +#endif /* CFG_STM32MP15 && CFG_TZSRAM_START */ + +static TEE_Result init_stm32mp1_drivers(void) +{ + /* Secure internal memories for the platform, once ETZPC is ready */ + etzpc_configure_tzma(0, ETZPC_TZMA_ALL_SECURE); + etzpc_lock_tzma(0); + + etzpc_configure_tzma(1, SYSRAM_SEC_SIZE >> SMALL_PAGE_SHIFT); + etzpc_lock_tzma(1); + + if (SYSRAM_SIZE > SYSRAM_SEC_SIZE) { + size_t nsec_size = SYSRAM_SIZE - SYSRAM_SEC_SIZE; + paddr_t nsec_start = SYSRAM_BASE + SYSRAM_SEC_SIZE; + uint8_t *va = phys_to_virt(nsec_start, MEM_AREA_IO_NSEC, + nsec_size); + + IMSG("Non-secure SYSRAM [%p %p]", va, va + nsec_size - 1); + + /* Clear content from the non-secure part */ + memset(va, 0, nsec_size); + } + + return TEE_SUCCESS; +} + +service_init_late(init_stm32mp1_drivers); + +static TEE_Result init_late_stm32mp1_drivers(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + /* Set access permission to TAM backup registers */ + if (IS_ENABLED(CFG_STM32_TAMP)) { + struct stm32_bkpregs_conf conf = { + .nb_zone1_regs = TAMP_BKP_REGISTER_ZONE1_COUNT, + .nb_zone2_regs = TAMP_BKP_REGISTER_ZONE2_COUNT, + }; + + res = stm32_tamp_set_secure_bkpregs(&conf); + if (res == TEE_ERROR_DEFER_DRIVER_INIT) { + /* TAMP driver was not probed if disabled in the DT */ + res = TEE_SUCCESS; + } + if (res) + panic(); + } + + return TEE_SUCCESS; +} + +driver_init_late(init_late_stm32mp1_drivers); + +vaddr_t stm32_rcc_base(void) +{ + static struct io_pa_va base = { .pa = RCC_BASE }; + + return io_pa_or_va_secure(&base, 1); +} + +vaddr_t get_gicd_base(void) +{ + struct io_pa_va base = { .pa = GIC_BASE + GICD_OFFSET }; + + return io_pa_or_va_secure(&base, 1); +} + +void stm32mp_get_bsec_static_cfg(struct stm32_bsec_static_cfg *cfg) +{ + cfg->base = BSEC_BASE; + cfg->upper_start = STM32MP1_UPPER_OTP_START; + cfg->max_id = STM32MP1_OTP_MAX_ID; +} + +bool __weak stm32mp_with_pmic(void) +{ + return false; +} + +uint32_t may_spin_lock(unsigned int *lock) +{ + if (!lock || !cpu_mmu_enabled()) + return 0; + + return cpu_spin_lock_xsave(lock); +} + +void may_spin_unlock(unsigned int *lock, uint32_t exceptions) +{ + if (!lock || !cpu_mmu_enabled()) + return; + + cpu_spin_unlock_xrestore(lock, exceptions); +} + +static vaddr_t stm32_tamp_base(void) +{ + static struct io_pa_va base = { .pa = TAMP_BASE }; + + return io_pa_or_va_secure(&base, 1); +} + +static vaddr_t bkpreg_base(void) +{ + return stm32_tamp_base() + TAMP_BKP_REGISTER_OFF; +} + +vaddr_t stm32mp_bkpreg(unsigned int idx) +{ + return bkpreg_base() + (idx * sizeof(uint32_t)); +} + +static bool __maybe_unused bank_is_valid(unsigned int bank) +{ + if (IS_ENABLED(CFG_STM32MP15)) + return bank == GPIO_BANK_Z || bank <= GPIO_BANK_K; + + if (IS_ENABLED(CFG_STM32MP13)) + return bank <= GPIO_BANK_I; + + panic(); +} + +unsigned int stm32_get_gpio_bank_offset(unsigned int bank) +{ + assert(bank_is_valid(bank)); + + if (bank == GPIO_BANK_Z) + return 0; + + return bank * GPIO_BANK_OFFSET; +} + +#ifdef CFG_STM32_IWDG +TEE_Result stm32_get_iwdg_otp_config(paddr_t pbase, + struct stm32_iwdg_otp_data *otp_data) +{ + unsigned int idx = 0; + uint32_t otp_id = 0; + size_t bit_len = 0; + uint8_t bit_offset = 0; + uint32_t otp_value = 0; + + switch (pbase) { + case IWDG1_BASE: + idx = 0; + break; + case IWDG2_BASE: + idx = 1; + break; + default: + panic(); + } + + if (stm32_bsec_find_otp_in_nvmem_layout("hw2_otp", &otp_id, &bit_offset, + &bit_len) || + bit_len != 32 || bit_offset != 0) + panic(); + + if (stm32_bsec_read_otp(&otp_value, otp_id)) + panic(); + + otp_data->hw_enabled = otp_value & + BIT(idx + HW2_OTP_IWDG_HW_ENABLE_SHIFT); + otp_data->disable_on_stop = otp_value & + BIT(idx + HW2_OTP_IWDG_FZ_STOP_SHIFT); + otp_data->disable_on_standby = otp_value & + BIT(idx + HW2_OTP_IWDG_FZ_STANDBY_SHIFT); + + return TEE_SUCCESS; +} +#endif /*CFG_STM32_IWDG*/ + +#ifdef CFG_STM32_DEBUG_ACCESS +static TEE_Result init_debug(void) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t conf = stm32_bsec_read_debug_conf(); + struct clk *dbg_clk = stm32mp_rcc_clock_id_to_clk(CK_DBG); + uint32_t state = 0; + + res = stm32_bsec_get_state(&state); + if (res) + return res; + + if (state != BSEC_STATE_SEC_CLOSED && conf) { + if (IS_ENABLED(CFG_WARN_INSECURE)) + IMSG("WARNING: All debug accesses are allowed"); + + res = stm32_bsec_write_debug_conf(conf | BSEC_DEBUG_ALL); + if (res) + return res; + + /* + * Enable DBG clock as used to access coprocessor + * debug registers + */ + clk_enable(dbg_clk); + } + + return TEE_SUCCESS; +} +early_init_late(init_debug); +#endif /* CFG_STM32_DEBUG_ACCESS */ + +/* Some generic resources need to be unpaged */ +DECLARE_KEEP_PAGER(pinctrl_apply_state); diff --git a/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/bsec_svc.c b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/bsec_svc.c new file mode 100644 index 0000000..9f426de --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/bsec_svc.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2016-2020, STMicroelectronics + */ + +#include +#include +#include +#include + +#include "bsec_svc.h" +#include "stm32mp1_smc.h" + +void bsec_main(struct thread_smc_args *args) +{ + TEE_Result result = TEE_ERROR_GENERIC; + uint32_t cmd = args->a1; + uint32_t otp_id = args->a2; + uint32_t in_value = args->a3; + uint32_t *out_value = &args->a1; + uint32_t tmp = 0; + + if (!stm32_bsec_nsec_can_access_otp(otp_id)) { + args->a0 = STM32_SIP_SVC_INVALID_PARAMS; + return; + } + + switch (cmd) { + case STM32_SIP_SVC_BSEC_READ_SHADOW: + FMSG("read shadow @%#"PRIx32, otp_id); + result = stm32_bsec_read_otp(out_value, otp_id); + break; + case STM32_SIP_SVC_BSEC_PROG_OTP: + FMSG("program @%#"PRIx32, otp_id); + result = stm32_bsec_program_otp(in_value, otp_id); + break; + case STM32_SIP_SVC_BSEC_WRITE_SHADOW: + FMSG("write shadow @%#"PRIx32, otp_id); + result = stm32_bsec_write_otp(in_value, otp_id); + break; + case STM32_SIP_SVC_BSEC_READ_OTP: + FMSG("read @%#"PRIx32, otp_id); + result = stm32_bsec_read_otp(&tmp, otp_id); + if (!result) + result = stm32_bsec_shadow_register(otp_id); + if (!result) + result = stm32_bsec_read_otp(out_value, otp_id); + if (!result) + result = stm32_bsec_write_otp(tmp, otp_id); + break; + case STM32_SIP_SVC_BSEC_WRLOCK_OTP: + FMSG("permanent write lock @%#"PRIx32, otp_id); + result = stm32_bsec_permanent_lock_otp(otp_id); + break; + default: + DMSG("Invalid command %#"PRIx32, cmd); + result = TEE_ERROR_BAD_PARAMETERS; + break; + } + + if (!result) + args->a0 = STM32_SIP_SVC_OK; + else if (result == TEE_ERROR_BAD_PARAMETERS) + args->a0 = STM32_SIP_SVC_INVALID_PARAMS; + else + args->a0 = STM32_SIP_SVC_FAILED; +} diff --git a/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/bsec_svc.h b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/bsec_svc.h new file mode 100644 index 0000000..5e57f50 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/bsec_svc.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2016-2020, STMicroelectronics + */ + +#ifndef __STM32MP1_BSEC_SVC_H__ +#define __STM32MP1_BSEC_SVC_H__ + +#include +#include +#include + +#ifdef CFG_STM32_BSEC_SIP +void bsec_main(struct thread_smc_args *args); +#else +static inline void bsec_main(struct thread_smc_args *args) +{ + args->a0 = STM32_SIP_SVC_UNKNOWN_FUNCTION; +} +#endif +#endif /*__STM32MP1_BSEC_SVC_H__*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/stm32mp1_smc.h b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/stm32mp1_smc.h new file mode 100644 index 0000000..2fbcac3 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/stm32mp1_smc.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2016-2022, STMicroelectronics + * Copyright (c) 2018, Linaro Limited + */ +#ifndef __STM32MP1_SMC_H__ +#define __STM32MP1_SMC_H__ + +#include + +/* + * SIP Functions + */ +#define STM32_SIP_SVC_VERSION_MAJOR 0x0 +#define STM32_SIP_SVC_VERSION_MINOR 0x1 + +#define STM32_SIP_SVC_FUNCTION_COUNT 0x3 + +/* STM32 SIP service generic return codes */ +#define STM32_SIP_SVC_OK 0x0 +#define STM32_SIP_SVC_UNKNOWN_FUNCTION OPTEE_SMC_RETURN_UNKNOWN_FUNCTION +#define STM32_SIP_SVC_FAILED 0xfffffffeU +#define STM32_SIP_SVC_INVALID_PARAMS 0xfffffffdU + +/* + * SMC function IDs for STM32 Service queries + * STM32 SMC services use the space between 0x82000000 and 0x8200FFFF + * like this is defined in SMC calling Convention by ARM + * for SiP (Silicon Partner) + * http://infocenter.arm.com/help/topic/com.arm.doc.den0028a/index.html + */ + +/* + * SIP function STM32_SIP_FUNC_CALL_COUNT + * + * Argument a0: (input) SMCC ID + * (output) Count of defined function IDs + */ +#define STM32_SIP_SVC_FUNC_CALL_COUNT 0xff00 + +/* + * Return the following UID if using API specified in this file without + * further extensions: + * 50aa78a7-9bf4-4a14-8a5e-264d5994c214. + * + * Represented in 4 32-bit words in STM32_SIP_UID_0, STM32_SIP_UID_1, + * STM32_SIP_UID_2, STM32_SIP_UID_3. + */ +#define STM32_SIP_SVC_UID_0 0x50aa78a7 +#define STM32_SIP_SVC_UID_1 0x9bf44a14 +#define STM32_SIP_SVC_UID_2 0x8a5e264d +#define STM32_SIP_SVC_UID_3 0x5994c214 + +/* + * SIP function STM32_SIP_SVC_FUNC_UID + * + * Argument a0: (input) SMCC ID + * (output) Lowest 32bit of the stm32mp1 SIP service UUID + * Argument a1: (output) Next 32bit of the stm32mp1 SIP service UUID + * Argument a2: (output) Next 32bit of the stm32mp1 SIP service UUID + * Argument a3: (output) Last 32bit of the stm32mp1 SIP service UUID + */ +#define STM32_SIP_SVC_FUNC_UID 0xff01 + +/* + * SIP function STM32_SIP_FUNC_VERSION + * + * Argument a0: (input) SMCC ID + * (output) STM32 SIP service major + * Argument a1: (output) STM32 SIP service minor + */ +#define STM32_SIP_SVC_FUNC_VERSION 0xff03 + +/* + * SIP functions STM32_SIP_SVC_FUNC_BSEC - Deprecated + * + * Argument a0: (input) SMCCC function ID + * (output) status return code + * Argument a1: (input) Service ID (STM32_SIP_BSEC_xxx) + * Argument a2: (input) OTP index + * (output) OTP read value, if applicable + * Argument a3: (input) OTP value if applicable + */ +#define STM32_SIP_SVC_FUNC_BSEC 0x1003 + +/* Service ID for function ID STM32_SIP_FUNC_BSEC */ +#define STM32_SIP_SVC_BSEC_READ_SHADOW 0x1 +#define STM32_SIP_SVC_BSEC_PROG_OTP 0x2 +#define STM32_SIP_SVC_BSEC_WRITE_SHADOW 0x3 +#define STM32_SIP_SVC_BSEC_READ_OTP 0x4 +/* reserved for STM32_SIP_SVC_SMC_READ_ALL 0x5 */ +/* reserved for STM32_SIP_SVC_SMC_WRITE_ALL 0x6 */ +#define STM32_SIP_SVC_BSEC_WRLOCK_OTP 0x7 + +/* + * SIP function STM32_SIP_SVC_FUNC_SCMI_AGENT0 + * SIP function STM32_SIP_SVC_FUNC_SCMI_AGENT1 + * + * Process SCMI message pending in related SCMI shared memory buffer. + * + * Argument a0: (input) SMCC ID + */ +#define STM32_SIP_SVC_FUNC_SCMI_AGENT0 0x2000 +#define STM32_SIP_SVC_FUNC_SCMI_AGENT1 0x2001 + +#endif /* __STM32MP1_SMC_H__*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/stm32mp1_svc_setup.c b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/stm32mp1_svc_setup.c new file mode 100644 index 0000000..70d6667 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/stm32mp1_svc_setup.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2017-2020, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include + +#include "bsec_svc.h" +#include "stm32mp1_smc.h" + +static enum sm_handler_ret sip_service(struct sm_ctx *ctx __unused, + struct thread_smc_args *args) +{ + switch (OPTEE_SMC_FUNC_NUM(args->a0)) { + case STM32_SIP_SVC_FUNC_CALL_COUNT: + args->a0 = STM32_SIP_SVC_FUNCTION_COUNT; + break; + case STM32_SIP_SVC_FUNC_VERSION: + args->a0 = STM32_SIP_SVC_VERSION_MAJOR; + args->a1 = STM32_SIP_SVC_VERSION_MINOR; + break; + case STM32_SIP_SVC_FUNC_UID: + args->a0 = STM32_SIP_SVC_UID_0; + args->a1 = STM32_SIP_SVC_UID_1; + args->a2 = STM32_SIP_SVC_UID_2; + args->a3 = STM32_SIP_SVC_UID_3; + break; + case STM32_SIP_SVC_FUNC_SCMI_AGENT0: + if (IS_ENABLED(CFG_STM32MP1_SCMI_SIP)) { + scmi_smt_fastcall_smc_entry(0); + args->a0 = STM32_SIP_SVC_OK; + } else { + args->a0 = ARM_SMCCC_RET_NOT_SUPPORTED; + } + break; + case STM32_SIP_SVC_FUNC_SCMI_AGENT1: + if (IS_ENABLED(CFG_STM32MP1_SCMI_SIP)) { + scmi_smt_fastcall_smc_entry(1); + args->a0 = STM32_SIP_SVC_OK; + } else { + args->a0 = ARM_SMCCC_RET_NOT_SUPPORTED; + } + break; + case STM32_SIP_SVC_FUNC_BSEC: + bsec_main(args); + break; + default: + return SM_HANDLER_PENDING_SMC; + } + + return SM_HANDLER_SMC_HANDLED; +} + +enum sm_handler_ret sm_platform_handler(struct sm_ctx *ctx) +{ + struct thread_smc_args *args = (void *)&ctx->nsec.r0; + + if (!OPTEE_SMC_IS_FAST_CALL(args->a0)) + return SM_HANDLER_PENDING_SMC; + + switch (OPTEE_SMC_OWNER_NUM(args->a0)) { + case OPTEE_SMC_OWNER_SIP: + return sip_service(ctx, args); + default: + return SM_HANDLER_PENDING_SMC; + } +} diff --git a/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/sub.mk b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/sub.mk new file mode 100644 index 0000000..6e1abcb --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/nsec-service/sub.mk @@ -0,0 +1,4 @@ +global-incdirs-y += . + +srcs-y += stm32mp1_svc_setup.c +srcs-$(CFG_STM32_BSEC_SIP) += bsec_svc.c diff --git a/optee_os/core/arch/arm/plat-stm32mp1/plat_tzc400.c b/optee_os/core/arch/arm/plat-stm32mp1/plat_tzc400.c new file mode 100644 index 0000000..55f6d41 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/plat_tzc400.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2020, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_STM32MP15 +#define TZC_FILTERS_MASK GENMASK_32(1, 0) +#endif +#ifdef CFG_STM32MP13 +#define TZC_FILTERS_MASK GENMASK_32(0, 0) +#endif + +static enum itr_return tzc_it_handler(struct itr_handler *handler __unused) +{ + EMSG("TZC permission failure"); + tzc_fail_dump(); + + if (IS_ENABLED(CFG_STM32MP_PANIC_ON_TZC_PERM_VIOLATION)) + panic(); + else + tzc_int_clear(); + + return ITRR_HANDLED; +} + +static struct itr_handler tzc_itr_handler = { + .it = STM32MP1_IRQ_TZC, + .handler = tzc_it_handler, +}; +DECLARE_KEEP_PAGER(tzc_itr_handler); + +static bool tzc_region_is_non_secure(unsigned int i, vaddr_t base, size_t size) +{ + struct tzc_region_config region_cfg = { }; + uint32_t ns_cpu_mask = TZC_REGION_ACCESS_RDWR(STM32MP1_TZC_A7_ID); + uint32_t filters_mask = TZC_FILTERS_MASK; + + if (tzc_get_region_config(i, ®ion_cfg)) + panic(); + + return region_cfg.base == base && region_cfg.top == (base + size - 1) && + region_cfg.sec_attr == TZC_REGION_S_NONE && + (region_cfg.ns_device_access & ns_cpu_mask) == ns_cpu_mask && + region_cfg.filters == filters_mask; +} + +static bool tzc_region_is_secure(unsigned int i, vaddr_t base, size_t size) +{ + struct tzc_region_config region_cfg = { }; + uint32_t filters_mask = TZC_FILTERS_MASK; + + if (tzc_get_region_config(i, ®ion_cfg)) + panic(); + + return region_cfg.base == base && region_cfg.top == (base + size - 1) && + region_cfg.sec_attr == TZC_REGION_S_RDWR && + region_cfg.ns_device_access == 0 && + region_cfg.filters == filters_mask; +} + +static TEE_Result init_stm32mp1_tzc(void) +{ + void *base = phys_to_virt(TZC_BASE, MEM_AREA_IO_SEC, 1); + unsigned int region_index = 1; + const uint64_t dram_start = DDR_BASE; + const uint64_t dram_end = dram_start + CFG_DRAM_SIZE; + const uint64_t tzdram_start = CFG_TZDRAM_START; + const uint64_t tzdram_size = CFG_TZDRAM_SIZE; + const uint64_t tzdram_end = tzdram_start + tzdram_size; + + assert(base); + + tzc_init((vaddr_t)base); + tzc_dump_state(); + + /* + * Early boot stage is in charge of configuring memory regions + * OP-TEE hence here only check this complies with static Core + * expectations. + */ + if (dram_start < tzdram_start) { + if (!tzc_region_is_non_secure(region_index, dram_start, + tzdram_start - dram_start)) + panic("Unexpected TZC area on non-secure region"); + + region_index++; + } + + if (!tzc_region_is_secure(region_index, tzdram_start, tzdram_size)) + panic("Unexpected TZC configuration on secure region"); + + if (tzdram_end < dram_end) { + region_index++; + + if (!tzc_region_is_non_secure(region_index, tzdram_end, + dram_end - tzdram_end)) + panic("Unexpected TZC area on non-secure region"); + } + + itr_add(&tzc_itr_handler); + itr_enable(tzc_itr_handler.it); + tzc_set_action(TZC_ACTION_INT); + + return TEE_SUCCESS; +} +driver_init(init_stm32mp1_tzc); diff --git a/optee_os/core/arch/arm/plat-stm32mp1/platform_config.h b/optee_os/core/arch/arm/plat-stm32mp1/platform_config.h new file mode 100644 index 0000000..3c87a37 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/platform_config.h @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2018, STMicroelectronics + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 32 + +/* Translation table */ +#ifdef CFG_WITH_LPAE +#define MAX_XLAT_TABLES 4 +#else +#define MAX_XLAT_TABLES 8 +#endif + +/* SoC interface registers base address ranges */ +#define APB1_BASE 0x40000000 +#define APB1_SIZE 0x0001d000 +#define APB2_BASE 0x44000000 +#define APB2_SIZE 0x00014000 +#define APB3_BASE 0x50020000 +#define APB3_SIZE 0x0000b000 +#define APB4_BASE 0x5a000000 +#define APB4_SIZE 0x00008000 +#define APB5_BASE 0x5c000000 +#define APB5_SIZE 0x0000b000 +#ifdef CFG_STM32MP13 +#define APB6_BASE 0x4c000000 +#define APB6_SIZE 0x0000d000 +#endif + +#define AHB4_BASE 0x50000000 +#define AHB4_SIZE 0x00020000 +#ifdef CFG_STM32MP13 +#define AHB5_BASE 0x54000000 +#define AHB5_SIZE 0x00008000 +#endif +#ifdef CFG_STM32MP15 +#define AHB5_BASE 0x54000000 +#define AHB5_SIZE 0x00005000 +#endif + +/* SoC interface registers base address */ +#define BSEC_BASE 0x5c005000 +#define ETZPC_BASE 0x5c007000 +#define CRYP1_BASE 0x54001000 +#define DDR_BASE 0xc0000000ul +#define GIC_BASE 0xa0021000ul +#define GPIOA_BASE 0x50002000 +#define GPIOB_BASE 0x50003000 +#define GPIOC_BASE 0x50004000 +#define GPIOD_BASE 0x50005000 +#define GPIOE_BASE 0x50006000 +#define GPIOF_BASE 0x50007000 +#define GPIOG_BASE 0x50008000 +#define GPIOH_BASE 0x50009000 +#define GPIOI_BASE 0x5000a000 +#define GPIOJ_BASE 0x5000b000 +#define GPIOK_BASE 0x5000c000 +#define GPIOZ_BASE 0x54004000 +#define HASH1_BASE 0x54002000 +#define I2C4_BASE 0x5c002000 +#define I2C5_BASE 0x40015000 +#define I2C6_BASE 0x5c009000 +#define IWDG1_BASE 0x5c003000 +#define IWDG2_BASE 0x5a002000 +#define PWR_BASE 0x50001000 +#define RCC_BASE 0x50000000 +#ifdef CFG_STM32MP13 +#define RNG1_BASE 0x54004000 +#endif +#ifdef CFG_STM32MP15 +#define RNG1_BASE 0x54003000 +#endif +#define RTC_BASE 0x5c004000 +#define SPI6_BASE 0x5c001000 +#ifdef CFG_STM32MP15 +#define SRAM1_BASE 0x30000000 +#define SRAM2_BASE 0x30020000 +#define SRAM3_BASE 0x30040000 +#define SRAM4_BASE 0x30050000 +#endif +#define SYSCFG_BASE 0x50020000 +#ifdef CFG_STM32MP13 +#define SYSRAM_BASE 0x2ffe0000 +#endif +#ifdef CFG_STM32MP15 +#define SYSRAM_BASE 0x2ffc0000 +#endif +#define TAMP_BASE 0x5c00a000 +#define TZC_BASE 0x5c006000 +#ifdef CFG_STM32MP13 +#define UART1_BASE 0x4c000000 +#define UART2_BASE 0x4c001000 +#endif +#ifdef CFG_STM32MP15 +#define UART1_BASE 0x5c000000 +#define UART2_BASE 0x4000e000 +#endif +#define UART3_BASE 0x4000f000 +#define UART4_BASE 0x40010000 +#define UART5_BASE 0x40011000 +#define UART6_BASE 0x44003000 +#define UART7_BASE 0x40018000 +#define UART8_BASE 0x40019000 + +/* Console configuration */ +#define STM32MP1_DEBUG_USART_BASE UART4_BASE +#define GIC_SPI_UART4 84 + +#define CONSOLE_UART_BASE STM32MP1_DEBUG_USART_BASE +#define CONSOLE_UART_SIZE 1024 + +/* BSEC OTP resources */ +#define STM32MP1_OTP_MAX_ID 0x5FU +#define STM32MP1_UPPER_OTP_START 0x20U + +#define OTP_MAX_SIZE (STM32MP1_OTP_MAX_ID + 1U) + +/* Bit map for BSEC word CFG0_OTP */ +#ifdef CFG_STM32MP13 +#define CFG0_OTP_CLOSED_DEVICE U(0x3F) +#endif +#ifdef CFG_STM32MP15 +#define CFG0_OTP_CLOSED_DEVICE BIT(6) +#endif + +/* Bit map for BSEC word HW2_OTP */ +#define HW2_OTP_IWDG_HW_ENABLE_SHIFT U(3) +#define HW2_OTP_IWDG_FZ_STOP_SHIFT U(5) +#define HW2_OTP_IWDG_FZ_STANDBY_SHIFT U(7) + +/* GIC resources */ +#define GIC_SIZE 0x2000 +#define GICC_OFFSET 0x1000 +#define GICD_OFFSET 0x0000 + +#define GIC_NON_SEC_SGI_0 0 +#define GIC_SEC_SGI_0 8 +#define GIC_SEC_SGI_1 9 + +#define TARGET_CPU0_GIC_MASK BIT(0) +#define TARGET_CPU1_GIC_MASK BIT(1) +#define TARGET_CPUS_GIC_MASK GENMASK_32(CFG_TEE_CORE_NB_CORE - 1, 0) + +/* + * GPIO banks: 11 non secure banks (A to K) and 1 secure bank (Z) + * Bank register's base address is computed from the bank ID listed here. + */ +#define GPIOS_NSEC_COUNT 11 +#define GPIOS_NSEC_BASE GPIOA_BASE +#define GPIOS_NSEC_SIZE (GPIOS_NSEC_COUNT * SMALL_PAGE_SIZE) + +#define STM32MP1_GPIOZ_MAX_COUNT 1 +#define STM32MP1_GPIOZ_PIN_MAX_COUNT 8 + +#define GPIO_BANK_OFFSET 0x1000U + +/* Bank IDs used in GPIO driver API */ +#define GPIO_BANK_A 0U +#define GPIO_BANK_B 1U +#define GPIO_BANK_C 2U +#define GPIO_BANK_D 3U +#define GPIO_BANK_E 4U +#define GPIO_BANK_F 5U +#define GPIO_BANK_G 6U +#define GPIO_BANK_H 7U +#define GPIO_BANK_I 8U +#define GPIO_BANK_J 9U +#define GPIO_BANK_K 10U +#define GPIO_BANK_Z 25U + +/* TAMP resources */ +#define TAMP_BKP_REGISTER_OFF 0x100 +#define TAMP_BKP_REGISTER_COUNT U(32) + +#define TAMP_BKP_REGISTER_ZONE1_COUNT U(10) +#define TAMP_BKP_REGISTER_ZONE2_COUNT U(5) +#define TAMP_BKP_REGISTER_ZONE3_COUNT U(17) + +#if (TAMP_BKP_REGISTER_ZONE1_COUNT + TAMP_BKP_REGISTER_ZONE2_COUNT + \ + TAMP_BKP_REGISTER_ZONE3_COUNT != TAMP_BKP_REGISTER_COUNT) +#error Inconsistent TAMP backup register zone definition +#endif + +/* TZC resources */ +#define STM32MP1_IRQ_TZC 36 + +#define STM32MP1_TZC_A7_ID 0 +#define STM32MP1_TZC_M4_ID 1 +#define STM32MP1_TZC_LCD_ID 3 +#define STM32MP1_TZC_GPU_ID 4 +#define STM32MP1_TZC_MDMA_ID 5 +#define STM32MP1_TZC_DMA_ID 6 +#define STM32MP1_TZC_USB_HOST_ID 7 +#define STM32MP1_TZC_USB_OTG_ID 8 +#define STM32MP1_TZC_SDMMC_ID 9 +#define STM32MP1_TZC_ETH_ID 10 +#define STM32MP1_TZC_DAP_ID 15 + +/* USART/UART resources */ +#define USART1_BASE UART1_BASE +#define USART2_BASE UART2_BASE +#define USART3_BASE UART3_BASE +#define USART6_BASE UART6_BASE + +/* SYSRAM layout */ +#ifdef CFG_STM32MP13 +#define SYSRAM_SIZE 0x20000 +#else /* Assume CFG_STM32MP15 */ +#define SYSRAM_SIZE 0x40000 +#endif +#define SYSRAM_NS_SIZE (SYSRAM_SIZE - SYSRAM_SEC_SIZE) + +/* Non-secure SYSRAM must be above (higher addresses) secure SYSRAM */ +#define STM32MP1_SCMI_SHM_END (CFG_STM32MP1_SCMI_SHM_BASE + \ + CFG_STM32MP1_SCMI_SHM_SIZE) + +#if (CFG_STM32MP1_SCMI_SHM_BASE && \ + (CFG_STM32MP1_SCMI_SHM_BASE >= SYSRAM_BASE) && \ + (STM32MP1_SCMI_SHM_END <= (SYSRAM_BASE + SYSRAM_SIZE))) +#define SYSRAM_SEC_SIZE (CFG_STM32MP1_SCMI_SHM_BASE - SYSRAM_BASE) +#else +#define SYSRAM_SEC_SIZE SYSRAM_SIZE +#endif + +#ifdef CFG_STM32MP15 +#define SRAM1_SIZE 0x20000 +#define SRAM2_SIZE 0x20000 +#define SRAM3_SIZE 0x10000 +#define SRAM4_SIZE 0x10000 +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/pm/psci.c b/optee_os/core/arch/arm/plat-stm32mp1/pm/psci.c new file mode 100644 index 0000000..f39c032 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/pm/psci.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2022, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CONSOLE_FLUSH_DELAY_MS 10 + +/* + * SMP boot support and access to the mailbox + */ + +enum core_state_id { + CORE_OFF = 0, + CORE_RET, + CORE_AWAKE, + CORE_ON, +}; + +static enum core_state_id core_state[CFG_TEE_CORE_NB_CORE]; +static unsigned int __maybe_unused state_lock = SPINLOCK_UNLOCK; + +static uint32_t __maybe_unused lock_state_access(void) +{ + return may_spin_lock(&state_lock); +} + +static void __maybe_unused unlock_state_access(uint32_t exceptions) +{ + may_spin_unlock(&state_lock, exceptions); +} + +int psci_affinity_info(uint32_t affinity, uint32_t lowest_affinity_level) +{ + unsigned int pos = get_core_pos_mpidr(affinity); + + DMSG("core %zu, state %u", pos, core_state[pos]); + + if ((pos >= CFG_TEE_CORE_NB_CORE) || + (lowest_affinity_level > PSCI_AFFINITY_LEVEL_ON)) { + return PSCI_RET_INVALID_PARAMETERS; + } + + switch (core_state[pos]) { + case CORE_OFF: + case CORE_RET: + return PSCI_AFFINITY_LEVEL_OFF; + case CORE_AWAKE: + return PSCI_AFFINITY_LEVEL_ON_PENDING; + case CORE_ON: + return PSCI_AFFINITY_LEVEL_ON; + default: + panic(); + } +} + +#if CFG_TEE_CORE_NB_CORE == 1 +/* + * Function called when a CPU is booted through the OP-TEE. + * All cores shall register when online. + */ +void stm32mp_register_online_cpu(void) +{ + assert(core_state[0] == CORE_OFF); + core_state[0] = CORE_ON; +} +#else +static void __noreturn stm32_pm_cpu_power_down_wfi(void) +{ + dcache_op_level1(DCACHE_OP_CLEAN); + + io_write32(stm32_rcc_base() + RCC_MP_GRSTCSETR, + RCC_MP_GRSTCSETR_MPUP1RST); + + dsb(); + isb(); + wfi(); + panic(); +} + +void stm32mp_register_online_cpu(void) +{ + size_t pos = get_core_pos(); + uint32_t exceptions = lock_state_access(); + + if (pos == 0) { + assert(core_state[pos] == CORE_OFF); + } else { + if (core_state[pos] != CORE_AWAKE) { + core_state[pos] = CORE_OFF; + unlock_state_access(exceptions); + stm32_pm_cpu_power_down_wfi(); + panic(); + } + clk_disable(stm32mp_rcc_clock_id_to_clk(RTCAPB)); + } + + core_state[pos] = CORE_ON; + unlock_state_access(exceptions); +} + +#define GICD_SGIR 0xF00 +static void raise_sgi0_as_secure(void) +{ + dsb_ishst(); + io_write32(get_gicd_base() + GICD_SGIR, + GIC_NON_SEC_SGI_0 | SHIFT_U32(TARGET_CPU1_GIC_MASK, 16)); +} + +static void release_secondary_early_hpen(size_t __unused pos) +{ + /* Need to send SIG#0 over Group0 after individual core 1 reset */ + raise_sgi0_as_secure(); + udelay(20); + + io_write32(stm32mp_bkpreg(BCKR_CORE1_BRANCH_ADDRESS), + TEE_LOAD_ADDR); + io_write32(stm32mp_bkpreg(BCKR_CORE1_MAGIC_NUMBER), + BOOT_API_A7_CORE1_MAGIC_NUMBER); + + dsb_ishst(); + itr_raise_sgi(GIC_SEC_SGI_0, TARGET_CPU1_GIC_MASK); +} + +/* Override default psci_cpu_on() with platform specific sequence */ +int psci_cpu_on(uint32_t core_id, uint32_t entry, uint32_t context_id) +{ + size_t pos = get_core_pos_mpidr(core_id); + uint32_t exceptions = 0; + int rc = 0; + + if (!pos || pos >= CFG_TEE_CORE_NB_CORE) + return PSCI_RET_INVALID_PARAMETERS; + + DMSG("core %zu, ns_entry 0x%" PRIx32 ", state %u", + pos, entry, core_state[pos]); + + exceptions = lock_state_access(); + + switch (core_state[pos]) { + case CORE_ON: + rc = PSCI_RET_ALREADY_ON; + break; + case CORE_AWAKE: + rc = PSCI_RET_ON_PENDING; + break; + case CORE_RET: + rc = PSCI_RET_DENIED; + break; + case CORE_OFF: + core_state[pos] = CORE_AWAKE; + rc = PSCI_RET_SUCCESS; + break; + default: + panic(); + } + + unlock_state_access(exceptions); + + if (rc == PSCI_RET_SUCCESS) { + boot_set_core_ns_entry(pos, entry, context_id); + release_secondary_early_hpen(pos); + } + + return rc; +} + +/* Override default psci_cpu_off() with platform specific sequence */ +int psci_cpu_off(void) +{ + unsigned int pos = get_core_pos(); + uint32_t exceptions = 0; + + if (pos == 0) { + EMSG("PSCI_CPU_OFF not supported for core #0"); + return PSCI_RET_INTERNAL_FAILURE; + } + + DMSG("core %u", pos); + + exceptions = lock_state_access(); + + assert(core_state[pos] == CORE_ON); + core_state[pos] = CORE_OFF; + + unlock_state_access(exceptions); + + /* Enable BKPREG access for the disabled CPU */ + if (clk_enable(stm32mp_rcc_clock_id_to_clk(RTCAPB))) + panic(); + + thread_mask_exceptions(THREAD_EXCP_ALL); + stm32_pm_cpu_power_down_wfi(); + panic(); +} +#endif + +/* Override default psci_system_off() with platform specific sequence */ +void __noreturn psci_system_off(void) +{ + DMSG("core %u", get_core_pos()); + + if (TRACE_LEVEL >= TRACE_DEBUG) { + console_flush(); + mdelay(CONSOLE_FLUSH_DELAY_MS); + } + + if (stm32mp_with_pmic()) { + stm32mp_get_pmic(); + stpmic1_switch_off(); + udelay(100); + } + + panic(); +} + +/* Override default psci_system_reset() with platform specific sequence */ +void __noreturn psci_system_reset(void) +{ + rstctrl_assert(stm32mp_rcc_reset_id_to_rstctrl(MPSYST_R)); + udelay(100); + panic(); +} + +/* Override default psci_cpu_on() with platform supported features */ +int psci_features(uint32_t psci_fid) +{ + switch (psci_fid) { + case ARM_SMCCC_VERSION: + case PSCI_PSCI_FEATURES: + case PSCI_SYSTEM_RESET: + case PSCI_VERSION: + return PSCI_RET_SUCCESS; + case PSCI_CPU_ON: + case PSCI_CPU_OFF: + if (CFG_TEE_CORE_NB_CORE > 1) + return PSCI_RET_SUCCESS; + return PSCI_RET_NOT_SUPPORTED; + case PSCI_SYSTEM_OFF: + if (stm32mp_with_pmic()) + return PSCI_RET_SUCCESS; + return PSCI_RET_NOT_SUPPORTED; + default: + return PSCI_RET_NOT_SUPPORTED; + } +} + +/* Override default psci_version() to enable PSCI_VERSION_1_0 API */ +uint32_t psci_version(void) +{ + return PSCI_VERSION_1_0; +} diff --git a/optee_os/core/arch/arm/plat-stm32mp1/pm/sub.mk b/optee_os/core/arch/arm/plat-stm32mp1/pm/sub.mk new file mode 100644 index 0000000..c8ae8f9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/pm/sub.mk @@ -0,0 +1 @@ +srcs-$(CFG_PSCI_ARM32) += psci.c diff --git a/optee_os/core/arch/arm/plat-stm32mp1/reset.S b/optee_os/core/arch/arm/plat-stm32mp1/reset.S new file mode 100644 index 0000000..8a1ab64 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/reset.S @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2018, STMicroelectronics + */ + +#include +#include +#include + +.section .text +.balign 4 +.code 32 + +#define STM32MP1_NSACR_PRESERVE_MASK (0xfff << 20) + +FUNC plat_cpu_reset_early , : + ldr r0, =SCR_SIF + write_scr r0 + + read_nsacr r0 + mov_imm r1, STM32MP1_NSACR_PRESERVE_MASK + and r0, r0, r1 + write_nsacr r0 + + isb + bx lr +END_FUNC plat_cpu_reset_early diff --git a/optee_os/core/arch/arm/plat-stm32mp1/scmi_server.c b/optee_os/core/arch/arm/plat-stm32mp1/scmi_server.c new file mode 100644 index 0000000..857809e --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/scmi_server.c @@ -0,0 +1,1006 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2022, STMicroelectronics + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TIMEOUT_US_1MS 1000 + +#define SCMI_CLOCK_NAME_SIZE 16 +#define SCMI_RD_NAME_SIZE 16 +#define SCMI_VOLTD_NAME_SIZE 16 + +/* + * struct stm32_scmi_clk - Data for the exposed clock + * @clock_id: Clock identifier in RCC clock driver + * @name: Clock string ID exposed to channel + * @enabled: State of the SCMI clock + */ +struct stm32_scmi_clk { + unsigned long clock_id; + struct clk *clk; + const char *name; + bool enabled; +}; + +/* + * struct stm32_scmi_rd - Data for the exposed reset controller + * @reset_id: Reset identifier in RCC reset driver + * @name: Reset string ID exposed to channel + * @rstctrl: Reset controller device + */ +struct stm32_scmi_rd { + unsigned long reset_id; + const char *name; + struct rstctrl *rstctrl; +}; + +enum voltd_device { + VOLTD_PWR, + VOLTD_PMIC, + VOLTD_VREFBUF, + /* Stub regulator until regulator framework is merged */ + VOLTD_STUB, +}; + +/* + * struct stm32_scmi_voltd - Data for the exposed voltage domains + * @name: Power regulator string ID exposed to channel + * @priv_id: Internal string ID for the regulator + * @priv_dev: Internal ID for the device implementing the regulator + * @regulator: Regulator controller device + * @state: State of the SCMI voltage domain (true: enable, false: disable) + */ +struct stm32_scmi_voltd { + const char *name; + const char *priv_id; + enum voltd_device priv_dev; + struct regulator *regulator; + bool state; +}; + +#if CFG_STM32MP1_SCMI_SHM_BASE +register_phys_mem(MEM_AREA_IO_NSEC, CFG_STM32MP1_SCMI_SHM_BASE, + CFG_STM32MP1_SCMI_SHM_SIZE); + +/* Locate all non-secure SMT message buffers in last page of SYSRAM */ +#define SMT_BUFFER_BASE CFG_STM32MP1_SCMI_SHM_BASE + +#if (SMT_BUFFER_BASE + SMT_BUF_SLOT_SIZE > \ + CFG_STM32MP1_SCMI_SHM_BASE + CFG_STM32MP1_SCMI_SHM_SIZE) +#error "SCMI shared memory mismatch" +#endif +#endif /*CFG_STM32MP1_SCMI_SHM_BASE*/ + +#define CLOCK_CELL(_scmi_id, _id, _name, _init_enabled) \ + [(_scmi_id)] = { \ + .clock_id = (_id), \ + .name = (_name), \ + .enabled = (_init_enabled), \ + } + +#define RESET_CELL(_scmi_id, _id, _name) \ + [(_scmi_id)] = { \ + .reset_id = (_id), \ + .name = (_name), \ + } + +#define VOLTD_CELL(_scmi_id, _dev_id, _priv_id, _name) \ + [(_scmi_id)] = { \ + .priv_id = (_priv_id), \ + .priv_dev = (_dev_id), \ + .name = (_name), \ + } + +#ifdef CFG_STM32MP13 +static struct stm32_scmi_clk stm32_scmi_clock[] = { + CLOCK_CELL(CK_SCMI_HSE, CK_HSE, "ck_hse", true), + CLOCK_CELL(CK_SCMI_HSI, CK_HSI, "ck_hsi", true), + CLOCK_CELL(CK_SCMI_CSI, CK_CSI, "ck_csi", true), + CLOCK_CELL(CK_SCMI_LSE, CK_LSE, "ck_lse", true), + CLOCK_CELL(CK_SCMI_LSI, CK_LSI, "ck_lsi", true), + CLOCK_CELL(CK_SCMI_HSE_DIV2, CK_HSE_DIV2, "clk-hse-div2", true), + CLOCK_CELL(CK_SCMI_PLL2_Q, PLL2_Q, "pll2_q", true), + CLOCK_CELL(CK_SCMI_PLL2_R, PLL2_R, "pll2_r", true), + CLOCK_CELL(CK_SCMI_PLL3_P, PLL3_P, "pll3_p", true), + CLOCK_CELL(CK_SCMI_PLL3_Q, PLL3_Q, "pll3_q", true), + CLOCK_CELL(CK_SCMI_PLL3_R, PLL3_R, "pll3_r", true), + CLOCK_CELL(CK_SCMI_PLL4_P, PLL4_P, "pll4_p", true), + CLOCK_CELL(CK_SCMI_PLL4_Q, PLL4_Q, "pll4_q", true), + CLOCK_CELL(CK_SCMI_PLL4_R, PLL4_R, "pll4_r", true), + CLOCK_CELL(CK_SCMI_MPU, CK_MPU, "ck_mpu", true), + CLOCK_CELL(CK_SCMI_AXI, CK_AXI, "ck_axi", true), + CLOCK_CELL(CK_SCMI_MLAHB, CK_MLAHB, "ck_mlahb", true), + CLOCK_CELL(CK_SCMI_CKPER, CK_PER, "ck_per", true), + CLOCK_CELL(CK_SCMI_PCLK1, PCLK1, "pclk1", true), + CLOCK_CELL(CK_SCMI_PCLK2, PCLK2, "pclk2", true), + CLOCK_CELL(CK_SCMI_PCLK3, PCLK3, "pclk3", true), + CLOCK_CELL(CK_SCMI_PCLK4, PCLK4, "pclk4", true), + CLOCK_CELL(CK_SCMI_PCLK5, PCLK5, "pclk5", true), + CLOCK_CELL(CK_SCMI_PCLK6, PCLK6, "pclk6", true), + CLOCK_CELL(CK_SCMI_CKTIMG1, CK_TIMG1, "timg1_ck", true), + CLOCK_CELL(CK_SCMI_CKTIMG2, CK_TIMG2, "timg2_ck", true), + CLOCK_CELL(CK_SCMI_CKTIMG3, CK_TIMG3, "timg3_ck", true), + CLOCK_CELL(CK_SCMI_RTC, RTC, "ck_rtc", true), + CLOCK_CELL(CK_SCMI_RTCAPB, RTCAPB, "rtcapb", true), + CLOCK_CELL(CK_SCMI_BSEC, BSEC, "bsec", true), +}; +#endif + +#ifdef CFG_STM32MP15 +static struct stm32_scmi_clk stm32_scmi_clock[] = { + CLOCK_CELL(CK_SCMI_HSE, CK_HSE, "ck_hse", true), + CLOCK_CELL(CK_SCMI_HSI, CK_HSI, "ck_hsi", true), + CLOCK_CELL(CK_SCMI_CSI, CK_CSI, "ck_csi", true), + CLOCK_CELL(CK_SCMI_LSE, CK_LSE, "ck_lse", true), + CLOCK_CELL(CK_SCMI_LSI, CK_LSI, "ck_lsi", true), + CLOCK_CELL(CK_SCMI_PLL2_Q, PLL2_Q, "pll2_q", true), + CLOCK_CELL(CK_SCMI_PLL2_R, PLL2_R, "pll2_r", true), + CLOCK_CELL(CK_SCMI_MPU, CK_MPU, "ck_mpu", true), + CLOCK_CELL(CK_SCMI_AXI, CK_AXI, "ck_axi", true), + CLOCK_CELL(CK_SCMI_BSEC, BSEC, "bsec", true), + CLOCK_CELL(CK_SCMI_CRYP1, CRYP1, "cryp1", false), + CLOCK_CELL(CK_SCMI_GPIOZ, GPIOZ, "gpioz", false), + CLOCK_CELL(CK_SCMI_HASH1, HASH1, "hash1", false), + CLOCK_CELL(CK_SCMI_I2C4, I2C4_K, "i2c4_k", false), + CLOCK_CELL(CK_SCMI_I2C6, I2C6_K, "i2c6_k", false), + CLOCK_CELL(CK_SCMI_IWDG1, IWDG1, "iwdg1", false), + CLOCK_CELL(CK_SCMI_RNG1, RNG1_K, "rng1_k", true), + CLOCK_CELL(CK_SCMI_RTC, RTC, "ck_rtc", true), + CLOCK_CELL(CK_SCMI_RTCAPB, RTCAPB, "rtcapb", true), + CLOCK_CELL(CK_SCMI_SPI6, SPI6_K, "spi6_k", false), + CLOCK_CELL(CK_SCMI_USART1, USART1_K, "usart1_k", false), +}; +#endif + +#ifdef CFG_STM32MP13 +static struct stm32_scmi_rd stm32_scmi_reset_domain[] = { + RESET_CELL(RST_SCMI_LTDC, LTDC_R, "ltdc"), + RESET_CELL(RST_SCMI_MDMA, MDMA_R, "mdma"), +}; +#endif + +#ifdef CFG_STM32MP15 +static struct stm32_scmi_rd stm32_scmi_reset_domain[] = { + RESET_CELL(RST_SCMI_SPI6, SPI6_R, "spi6"), + RESET_CELL(RST_SCMI_I2C4, I2C4_R, "i2c4"), + RESET_CELL(RST_SCMI_I2C6, I2C6_R, "i2c6"), + RESET_CELL(RST_SCMI_USART1, USART1_R, "usart1"), + RESET_CELL(RST_SCMI_STGEN, STGEN_R, "stgen"), + RESET_CELL(RST_SCMI_GPIOZ, GPIOZ_R, "gpioz"), + RESET_CELL(RST_SCMI_CRYP1, CRYP1_R, "cryp1"), + RESET_CELL(RST_SCMI_HASH1, HASH1_R, "hash1"), + RESET_CELL(RST_SCMI_RNG1, RNG1_R, "rng1"), + RESET_CELL(RST_SCMI_MDMA, MDMA_R, "mdma"), + RESET_CELL(RST_SCMI_MCU, MCU_R, "mcu"), + RESET_CELL(RST_SCMI_MCU_HOLD_BOOT, MCU_HOLD_BOOT_R, "mcu_hold_boot"), +}; +#endif + +#define PWR_REG11_NAME_ID "0" +#define PWR_REG18_NAME_ID "1" +#define PWR_USB33_NAME_ID "2" + +#define STUB_SDMMC1_IO_NAME_ID "sdmmc1" +#define STUB_SDMMC2_IO_NAME_ID "sdmmc2" + +#ifdef CFG_STM32MP13 +struct stm32_scmi_voltd scmi_voltage_domain[] = { + VOLTD_CELL(VOLTD_SCMI_REG11, VOLTD_PWR, PWR_REG11_NAME_ID, "reg11"), + VOLTD_CELL(VOLTD_SCMI_REG18, VOLTD_PWR, PWR_REG18_NAME_ID, "reg18"), + VOLTD_CELL(VOLTD_SCMI_USB33, VOLTD_PWR, PWR_USB33_NAME_ID, "usb33"), + VOLTD_CELL(VOLTD_SCMI_SDMMC1_IO, VOLTD_STUB, STUB_SDMMC1_IO_NAME_ID, + "sdmmc1"), + VOLTD_CELL(VOLTD_SCMI_SDMMC2_IO, VOLTD_STUB, STUB_SDMMC2_IO_NAME_ID, + "sdmmc2"), + VOLTD_CELL(VOLTD_SCMI_VREFBUF, VOLTD_VREFBUF, "vrefbuf", "vrefbuf"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK1, VOLTD_PMIC, "buck1", "buck1"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK2, VOLTD_PMIC, "buck2", "buck2"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK3, VOLTD_PMIC, "buck3", "buck3"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK4, VOLTD_PMIC, "buck4", "buck4"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO1, VOLTD_PMIC, "ldo1", "ldo1"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO2, VOLTD_PMIC, "ldo2", "ldo2"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO3, VOLTD_PMIC, "ldo3", "ldo3"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO4, VOLTD_PMIC, "ldo4", "ldo4"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO5, VOLTD_PMIC, "ldo5", "ldo5"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO6, VOLTD_PMIC, "ldo6", "ldo6"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_VREFDDR, VOLTD_PMIC, "vref_ddr", + "vref_ddr"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BOOST, VOLTD_PMIC, "boost", "bst_out"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_PWR_SW1, VOLTD_PMIC, "pwr_sw1", + "pwr_sw1"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_PWR_SW2, VOLTD_PMIC, "pwr_sw2", + "pwr_sw2"), +}; +#endif + +#ifdef CFG_STM32MP15 +struct stm32_scmi_voltd scmi_voltage_domain[] = { + VOLTD_CELL(VOLTD_SCMI_REG11, VOLTD_PWR, PWR_REG11_NAME_ID, "reg11"), + VOLTD_CELL(VOLTD_SCMI_REG18, VOLTD_PWR, PWR_REG18_NAME_ID, "reg18"), + VOLTD_CELL(VOLTD_SCMI_USB33, VOLTD_PWR, PWR_USB33_NAME_ID, "usb33"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK1, VOLTD_PMIC, "buck1", "vddcore"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK2, VOLTD_PMIC, "buck2", "vdd_ddr"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK3, VOLTD_PMIC, "buck3", "vdd"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BUCK4, VOLTD_PMIC, "buck4", "v3v3"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO1, VOLTD_PMIC, "ldo1", "v1v8_audio"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO2, VOLTD_PMIC, "ldo2", "v3v3_hdmi"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO3, VOLTD_PMIC, "ldo3", "vtt_ddr"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO4, VOLTD_PMIC, "ldo4", "vdd_usb"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO5, VOLTD_PMIC, "ldo5", "vdda"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_LDO6, VOLTD_PMIC, "ldo6", "v1v2_hdmi"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_VREFDDR, VOLTD_PMIC, "vref_ddr", + "vref_ddr"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_BOOST, VOLTD_PMIC, "boost", "bst_out"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_PWR_SW1, VOLTD_PMIC, "pwr_sw1", + "vbus_otg"), + VOLTD_CELL(VOLTD_SCMI_STPMIC1_PWR_SW2, VOLTD_PMIC, "pwr_sw2", + "vbus_sw"), +}; +#endif + +struct channel_resources { + struct scmi_msg_channel *channel; + struct stm32_scmi_clk *clock; + size_t clock_count; + struct stm32_scmi_rd *rd; + size_t rd_count; + struct stm32_scmi_voltd *voltd; + size_t voltd_count; +}; + +static const struct channel_resources scmi_channel[] = { + [0] = { + .channel = &(struct scmi_msg_channel){ +#ifdef SMT_BUFFER_BASE + .shm_addr = { .pa = SMT_BUFFER_BASE }, + .shm_size = SMT_BUF_SLOT_SIZE, +#endif + }, + .clock = stm32_scmi_clock, + .clock_count = ARRAY_SIZE(stm32_scmi_clock), + .rd = stm32_scmi_reset_domain, + .rd_count = ARRAY_SIZE(stm32_scmi_reset_domain), + .voltd = scmi_voltage_domain, + .voltd_count = ARRAY_SIZE(scmi_voltage_domain), + }, +}; + +static const struct channel_resources *find_resource(unsigned int channel_id) +{ + assert(channel_id < ARRAY_SIZE(scmi_channel)); + + return scmi_channel + channel_id; +} + +struct scmi_msg_channel *plat_scmi_get_channel(unsigned int channel_id) +{ + const size_t max_id = ARRAY_SIZE(scmi_channel); + unsigned int confined_id = confine_array_index(channel_id, max_id); + + if (channel_id >= max_id) + return NULL; + + return find_resource(confined_id)->channel; +} + +static size_t __maybe_unused plat_scmi_protocol_count_paranoid(void) +{ + unsigned int n = 0; + unsigned int count = 0; + const size_t channel_count = ARRAY_SIZE(scmi_channel); + + for (n = 0; n < channel_count; n++) + if (scmi_channel[n].clock_count) + break; + if (n < channel_count) + count++; + + for (n = 0; n < channel_count; n++) + if (scmi_channel[n].rd_count) + break; + if (n < channel_count) + count++; + + for (n = 0; n < channel_count; n++) + if (scmi_channel[n].voltd_count) + break; + if (n < channel_count) + count++; + + return count; +} + +static const char vendor[] = "ST"; +static const char sub_vendor[] = ""; + +const char *plat_scmi_vendor_name(void) +{ + return vendor; +} + +const char *plat_scmi_sub_vendor_name(void) +{ + return sub_vendor; +} + +/* Currently supporting Clocks and Reset Domains */ +static const uint8_t plat_protocol_list[] = { + SCMI_PROTOCOL_ID_CLOCK, + SCMI_PROTOCOL_ID_RESET_DOMAIN, + SCMI_PROTOCOL_ID_VOLTAGE_DOMAIN, + 0 /* Null termination */ +}; + +size_t plat_scmi_protocol_count(void) +{ + const size_t count = ARRAY_SIZE(plat_protocol_list) - 1; + + assert(count == plat_scmi_protocol_count_paranoid()); + + return count; +} + +const uint8_t *plat_scmi_protocol_list(unsigned int channel_id __unused) +{ + assert(plat_scmi_protocol_count_paranoid() == + (ARRAY_SIZE(plat_protocol_list) - 1)); + + return plat_protocol_list; +} + +/* + * Platform SCMI clocks + */ +static struct stm32_scmi_clk *find_clock(unsigned int channel_id, + unsigned int scmi_id) +{ + const struct channel_resources *resource = find_resource(channel_id); + size_t n = 0; + + if (resource) { + for (n = 0; n < resource->clock_count; n++) + if (n == scmi_id) + return &resource->clock[n]; + } + + return NULL; +} + +size_t plat_scmi_clock_count(unsigned int channel_id) +{ + const struct channel_resources *resource = find_resource(channel_id); + + if (!resource) + return 0; + + return resource->clock_count; +} + +const char *plat_scmi_clock_get_name(unsigned int channel_id, + unsigned int scmi_id) +{ + struct stm32_scmi_clk *clock = find_clock(channel_id, scmi_id); + + if (!clock || !stm32mp_nsec_can_access_clock(clock->clock_id)) + return NULL; + + return clock->name; +} + +int32_t plat_scmi_clock_rates_array(unsigned int channel_id, + unsigned int scmi_id, size_t start_index, + unsigned long *array, size_t *nb_elts) +{ + struct stm32_scmi_clk *clock = find_clock(channel_id, scmi_id); + + if (!clock) + return SCMI_NOT_FOUND; + + if (!stm32mp_nsec_can_access_clock(clock->clock_id)) + return SCMI_DENIED; + + /* Exposed clocks are currently fixed rate clocks */ + if (start_index) + return SCMI_INVALID_PARAMETERS; + + if (!array) + *nb_elts = 1; + else if (*nb_elts == 1) + *array = clk_get_rate(clock->clk); + else + return SCMI_GENERIC_ERROR; + + return SCMI_SUCCESS; +} + +unsigned long plat_scmi_clock_get_rate(unsigned int channel_id, + unsigned int scmi_id) +{ + struct stm32_scmi_clk *clock = find_clock(channel_id, scmi_id); + + if (!clock || !stm32mp_nsec_can_access_clock(clock->clock_id)) + return 0; + + return clk_get_rate(clock->clk); +} + +int32_t plat_scmi_clock_get_state(unsigned int channel_id, unsigned int scmi_id) +{ + struct stm32_scmi_clk *clock = find_clock(channel_id, scmi_id); + + if (!clock || !stm32mp_nsec_can_access_clock(clock->clock_id)) + return 0; + + return (int32_t)clock->enabled; +} + +int32_t plat_scmi_clock_set_state(unsigned int channel_id, unsigned int scmi_id, + bool enable_not_disable) +{ + struct stm32_scmi_clk *clock = find_clock(channel_id, scmi_id); + + if (!clock) + return SCMI_NOT_FOUND; + + if (!stm32mp_nsec_can_access_clock(clock->clock_id)) + return SCMI_DENIED; + + if (enable_not_disable) { + if (!clock->enabled) { + FMSG("SCMI clock %u enable", scmi_id); + clk_enable(clock->clk); + clock->enabled = true; + } + } else { + if (clock->enabled) { + FMSG("SCMI clock %u disable", scmi_id); + clk_disable(clock->clk); + clock->enabled = false; + } + } + + return SCMI_SUCCESS; +} + +/* + * Platform SCMI reset domains + */ +static struct stm32_scmi_rd *find_rd(unsigned int channel_id, + unsigned int scmi_id) +{ + const struct channel_resources *resource = find_resource(channel_id); + size_t n = 0; + + if (resource) { + for (n = 0; n < resource->rd_count; n++) + if (n == scmi_id) + return &resource->rd[n]; + } + + return NULL; +} + +const char *plat_scmi_rd_get_name(unsigned int channel_id, unsigned int scmi_id) +{ + const struct stm32_scmi_rd *rd = find_rd(channel_id, scmi_id); + + if (!rd) + return NULL; + + return rd->name; +} + +size_t plat_scmi_rd_count(unsigned int channel_id) +{ + const struct channel_resources *resource = find_resource(channel_id); + + if (!resource) + return 0; + + return resource->rd_count; +} + +int32_t plat_scmi_rd_autonomous(unsigned int channel_id, unsigned int scmi_id, + uint32_t state) +{ + const struct stm32_scmi_rd *rd = find_rd(channel_id, scmi_id); + + if (!rd) + return SCMI_NOT_FOUND; + + if (!rd->rstctrl || !stm32mp_nsec_can_access_reset(rd->reset_id)) + return SCMI_DENIED; + assert(rd->rstctrl); + +#ifdef CFG_STM32MP15 + if (rd->reset_id == MCU_HOLD_BOOT_R) + return SCMI_NOT_SUPPORTED; +#endif + + /* Supports only reset with context loss */ + if (state) + return SCMI_NOT_SUPPORTED; + + FMSG("SCMI reset %u cycle", scmi_id); + + if (rstctrl_assert_to(rd->rstctrl, TIMEOUT_US_1MS)) + return SCMI_HARDWARE_ERROR; + + if (rstctrl_deassert_to(rd->rstctrl, TIMEOUT_US_1MS)) + return SCMI_HARDWARE_ERROR; + + return SCMI_SUCCESS; +} + +int32_t plat_scmi_rd_set_state(unsigned int channel_id, unsigned int scmi_id, + bool assert_not_deassert) +{ + const struct stm32_scmi_rd *rd = find_rd(channel_id, scmi_id); + TEE_Result res = TEE_ERROR_GENERIC; + + if (!rd) + return SCMI_NOT_FOUND; + + if (!rd->rstctrl || !stm32mp_nsec_can_access_reset(rd->reset_id)) + return SCMI_DENIED; + assert(rd->rstctrl); + + if (assert_not_deassert) { + FMSG("SCMI reset %u set", scmi_id); + res = rstctrl_assert(rd->rstctrl); + } else { + FMSG("SCMI reset %u release", scmi_id); + res = rstctrl_deassert(rd->rstctrl); + } + + if (res) + return SCMI_HARDWARE_ERROR; + + return SCMI_SUCCESS; +} + +/* + * Platform SCMI voltage domains + */ +static struct stm32_scmi_voltd *find_voltd(unsigned int channel_id, + unsigned int scmi_id) +{ + const struct channel_resources *resource = find_resource(channel_id); + size_t n = 0; + + if (resource) { + for (n = 0; n < resource->voltd_count; n++) + if (n == scmi_id) + return &resource->voltd[n]; + } + + return NULL; +} + +size_t plat_scmi_voltd_count(unsigned int channel_id) +{ + const struct channel_resources *resource = find_resource(channel_id); + + if (!resource) + return 0; + + return resource->voltd_count; +} + +const char *plat_scmi_voltd_get_name(unsigned int channel_id, + unsigned int scmi_id) +{ + struct stm32_scmi_voltd *voltd = find_voltd(channel_id, scmi_id); + + /* Currently non-secure is allowed to access all PWR regulators */ + if (!voltd) + return NULL; + + return voltd->name; +} + +static enum pwr_regulator pwr_scmi_to_regu_id(struct stm32_scmi_voltd *voltd) +{ + if (!strcmp(voltd->priv_id, PWR_REG11_NAME_ID)) + return PWR_REG11; + if (!strcmp(voltd->priv_id, PWR_REG18_NAME_ID)) + return PWR_REG18; + if (!strcmp(voltd->priv_id, PWR_USB33_NAME_ID)) + return PWR_USB33; + + panic(); +} + +static long stub_get_level_uv(struct stm32_scmi_voltd *voltd) +{ + if (!strcmp(voltd->priv_id, STUB_SDMMC1_IO_NAME_ID) || + !strcmp(voltd->priv_id, STUB_SDMMC2_IO_NAME_ID)) + return 3300000; + + panic(); +} + +static int32_t stub_describe_levels(struct stm32_scmi_voltd *voltd __unused, + size_t start_index, long *microvolt, + size_t *nb_elts) +{ + if (start_index) + return SCMI_INVALID_PARAMETERS; + + if (!microvolt || !*nb_elts) { + *nb_elts = 1; + return SCMI_SUCCESS; + } + + microvolt[0] = stub_get_level_uv(voltd); + *nb_elts = 1; + + return SCMI_SUCCESS; +} + +int32_t plat_scmi_voltd_levels_array(unsigned int channel_id, + unsigned int scmi_id, size_t start_index, + long *levels, size_t *nb_elts) + +{ + struct stm32_scmi_voltd *voltd = find_voltd(channel_id, scmi_id); + + if (!voltd) + return SCMI_NOT_FOUND; + + if (voltd->regulator) { + struct regulator_voltages *voltages = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + int *ref = NULL; + size_t ref_count = 0; + size_t n = 0; + + res = regulator_supported_voltages(voltd->regulator, &voltages); + if (res == TEE_ERROR_NOT_SUPPORTED) + return SCMI_NOT_SUPPORTED; + if (res) + return SCMI_GENERIC_ERROR; + if (!voltages || voltages->type != VOLTAGE_TYPE_FULL_LIST) { + /* + * Triplet min/max/step description. Caller should use + * plat_scmi_voltd_levels_by_step(). + */ + return SCMI_NOT_SUPPORTED; + } + + ref = voltages->entries; + ref_count = voltages->num_levels; + + /* Bound according to regulator registered min/max levels */ + for (n = ref_count; n > 0; n--) + if (voltages->entries[n - 1] > voltd->regulator->max_uv) + ref_count--; + for (n = 0; n < ref_count; n++) + if (voltages->entries[n] >= voltd->regulator->min_uv) + break; + + if (n == ref_count) { + DMSG("Voltage list out of regulator %s level bounds", + regulator_name(voltd->regulator)); + return SCMI_GENERIC_ERROR; + } + + if (start_index >= ref_count) + return SCMI_OUT_OF_RANGE; + + if (!*nb_elts) { + *nb_elts = ref_count - start_index; + return SCMI_SUCCESS; + } + + for (n = 0; n < *nb_elts; n++) + levels[n] = ref[start_index + n]; + + return SCMI_SUCCESS; + } + + switch (voltd->priv_dev) { + case VOLTD_STUB: + return stub_describe_levels(voltd, start_index, levels, + nb_elts); + default: + return SCMI_DENIED; + } +} + +int32_t plat_scmi_voltd_levels_by_step(unsigned int channel_id, + unsigned int scmi_id, long *min_max_step) +{ + struct stm32_scmi_voltd *voltd = find_voltd(channel_id, scmi_id); + + if (!voltd) + return SCMI_NOT_FOUND; + + if (voltd->regulator) { + struct regulator_voltages *voltages = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + int ref_min = 0; + int ref_max = 0; + int ref_step = 0; + + res = regulator_supported_voltages(voltd->regulator, &voltages); + if (res == TEE_ERROR_NOT_SUPPORTED) + return SCMI_NOT_SUPPORTED; + if (res) + return SCMI_GENERIC_ERROR; + if (!voltages || voltages->type != VOLTAGE_TYPE_INCREMENT) { + /* + * Triplet min/max/step description. Caller should use + * plat_scmi_voltd_levels_by_step(). + */ + return SCMI_NOT_SUPPORTED; + } + + ref_min = voltages->entries[0]; + ref_max = voltages->entries[1]; + ref_step = voltages->entries[2]; + + if (ref_min < voltd->regulator->min_uv) { + int diff = voltd->regulator->min_uv - ref_min; + int incr = DIV_ROUND_UP(diff, ref_step); + + ref_min += ref_step * incr; + } + + if (ref_max > voltd->regulator->max_uv) { + int diff = ref_max - voltd->regulator->max_uv; + int decr = diff / ref_step; + + ref_max -= ref_step * decr; + } + + min_max_step[0] = ref_min; + min_max_step[1] = ref_max; + min_max_step[2] = ref_step; + + return SCMI_SUCCESS; + } + + return SCMI_NOT_SUPPORTED; +} + +int32_t plat_scmi_voltd_get_level(unsigned int channel_id, unsigned int scmi_id, + long *level_uv) +{ + struct stm32_scmi_voltd *voltd = find_voltd(channel_id, scmi_id); + + if (!voltd) + return SCMI_INVALID_PARAMETERS; + + if (voltd->regulator) { + *level_uv = regulator_get_voltage(voltd->regulator); + return SCMI_SUCCESS; + } + + switch (voltd->priv_dev) { + case VOLTD_STUB: + *level_uv = stub_get_level_uv(voltd); + return SCMI_SUCCESS; + default: + return SCMI_DENIED; + } +} + +int32_t plat_scmi_voltd_set_level(unsigned int channel_id, unsigned int scmi_id, + long level_uv) +{ + struct stm32_scmi_voltd *voltd = find_voltd(channel_id, scmi_id); + + if (!voltd) + return SCMI_NOT_FOUND; + + if (voltd->regulator) { + TEE_Result res = TEE_ERROR_GENERIC; + + if (level_uv < INT_MIN || level_uv > INT_MAX) + return SCMI_OUT_OF_RANGE; + + res = regulator_set_voltage(voltd->regulator, level_uv); + if (res) + return SCMI_GENERIC_ERROR; + else + return SCMI_SUCCESS; + } + + switch (voltd->priv_dev) { + case VOLTD_STUB: + return SCMI_SUCCESS; + default: + return SCMI_DENIED; + } +} + +int32_t plat_scmi_voltd_get_config(unsigned int channel_id, + unsigned int scmi_id, uint32_t *config) +{ + struct stm32_scmi_voltd *voltd = find_voltd(channel_id, scmi_id); + + if (!voltd) + return SCMI_NOT_FOUND; + + if (voltd->regulator) { + if (voltd->state) + *config = SCMI_VOLTAGE_DOMAIN_CONFIG_ARCH_ON; + else + *config = SCMI_VOLTAGE_DOMAIN_CONFIG_ARCH_OFF; + + return SCMI_SUCCESS; + } + + switch (voltd->priv_dev) { + case VOLTD_STUB: + *config = SCMI_VOLTAGE_DOMAIN_CONFIG_ARCH_ON; + return SCMI_SUCCESS; + default: + return SCMI_DENIED; + } +} + +int32_t plat_scmi_voltd_set_config(unsigned int channel_id, + unsigned int scmi_id, uint32_t config) +{ + struct stm32_scmi_voltd *voltd = find_voltd(channel_id, scmi_id); + + if (!voltd) + return SCMI_NOT_FOUND; + + if (voltd->regulator) { + TEE_Result res = TEE_ERROR_GENERIC; + + if (config == SCMI_VOLTAGE_DOMAIN_CONFIG_ARCH_ON && + !voltd->state) { + res = regulator_enable(voltd->regulator); + if (!res) + voltd->state = true; + } else if (config == SCMI_VOLTAGE_DOMAIN_CONFIG_ARCH_OFF && + voltd->state) { + res = regulator_disable(voltd->regulator); + if (!res) + voltd->state = false; + } else { + res = TEE_ERROR_GENERIC; + } + + if (res) + return SCMI_GENERIC_ERROR; + else + return SCMI_SUCCESS; + } + + switch (voltd->priv_dev) { + case VOLTD_STUB: + return SCMI_SUCCESS; + default: + return SCMI_DENIED; + } +} + +static void get_voltd_regulator(struct stm32_scmi_voltd *voltd) +{ + enum pwr_regulator regu_id = PWR_REGU_COUNT; + + switch (voltd->priv_dev) { + case VOLTD_PWR: + regu_id = pwr_scmi_to_regu_id(voltd); + voltd->regulator = stm32mp1_pwr_get_regulator(regu_id); + break; + case VOLTD_PMIC: + voltd->regulator = stm32mp_pmic_get_regulator(voltd->priv_id); + break; + case VOLTD_VREFBUF: + voltd->regulator = stm32_vrefbuf_regulator(); + break; + case VOLTD_STUB: + break; + default: + break; + } + + if (voltd->regulator && voltd->regulator->flags & REGULATOR_BOOT_ON) + regulator_enable(voltd->regulator); +} + +/* + * Initialize platform SCMI resources + */ +static TEE_Result stm32mp1_init_scmi_server(void) +{ + size_t i = 0; + size_t j = 0; + + for (i = 0; i < ARRAY_SIZE(scmi_channel); i++) { + const struct channel_resources *res = scmi_channel + i; + struct scmi_msg_channel *chan = res->channel; + + if (chan->shm_addr.pa) { + struct io_pa_va *addr = &chan->shm_addr; + + /* Enforce non-secure shm mapped as device memory */ + addr->va = (vaddr_t)phys_to_virt(addr->pa, + MEM_AREA_IO_NSEC, + chan->shm_size); + assert(addr->va); + + scmi_smt_init_agent_channel(chan); + } + + for (j = 0; j < res->clock_count; j++) { + struct stm32_scmi_clk *clk = &res->clock[j]; + + if (!clk->name || + strlen(clk->name) >= SCMI_CLOCK_NAME_SIZE) + panic("SCMI clock name invalid"); + + clk->clk = stm32mp_rcc_clock_id_to_clk(clk->clock_id); + assert(clk->clk); + + /* Sync SCMI clocks with their targeted initial state */ + if (clk->enabled && + stm32mp_nsec_can_access_clock(clk->clock_id)) + clk_enable(clk->clk); + } + + for (j = 0; j < res->rd_count; j++) { + struct stm32_scmi_rd *rd = &res->rd[j]; + struct rstctrl *rstctrl = NULL; + + if (!rd->name || + strlen(rd->name) >= SCMI_RD_NAME_SIZE) + panic("SCMI reset domain name invalid"); + + if (stm32mp_nsec_can_access_clock(rd->reset_id)) + continue; + + rstctrl = stm32mp_rcc_reset_id_to_rstctrl(rd->reset_id); + assert(rstctrl); + if (rstctrl_get_exclusive(rstctrl)) + continue; + + rd->rstctrl = rstctrl; + } + + for (j = 0; j < res->voltd_count; j++) { + struct stm32_scmi_voltd *voltd = &res->voltd[j]; + + if (!voltd->name || + strlen(voltd->name) >= SCMI_VOLTD_NAME_SIZE) + panic("SCMI voltage domain name invalid"); + + get_voltd_regulator(voltd); + } + } + + return TEE_SUCCESS; +} + +driver_init_late(stm32mp1_init_scmi_server); diff --git a/optee_os/core/arch/arm/plat-stm32mp1/scripts/stm32image.py b/optee_os/core/arch/arm/plat-stm32mp1/scripts/stm32image.py new file mode 100755 index 0000000..4067fd7 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/scripts/stm32image.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2017-2018, STMicroelectronics +# +import argparse +import struct +import mmap + +header_size = 256 +hdr_magic_number = 0x324D5453 # magic ='S' 'T' 'M' 0x32 +hdr_header_ver_variant = 0 +hdr_header_ver_minor = 0 +hdr_header_ver_major = 1 +hdr_version_number = 0 +hdr_option_flags = 1 # bit0=1 no signature +hdr_edcsa_algo = 1 + + +def get_size(file): + file.seek(0, 2) # End of the file + size = file.tell() + return size + + +def stm32image_checksum(dest_fd, sizedest): + csum = 0 + if sizedest < header_size: + return 0 + dest_fd.seek(header_size, 0) + length = sizedest - header_size + while length > 0: + csum += ord(dest_fd.read(1)) + length -= 1 + return csum + + +def stm32image_set_header(dest_fd, load, entry, bintype): + sizedest = get_size(dest_fd) + + checksum = stm32image_checksum(dest_fd, sizedest) + + dest_fd.seek(0, 0) + + # Magic number + dest_fd.write(struct.pack(' 0: + mmsrc = mmap.mmap(src_fd.fileno(), 0, access=mmap.ACCESS_READ) + dest_fd.write(mmsrc[:sizesrc]) + mmsrc.close() + + src_fd.close() + + stm32image_set_header(dest_fd, load, entry, bintype) + + dest_fd.close() + + +def int_parse(str): + return int(str, 0) + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--source', + required=True, + help='Source file') + + parser.add_argument('--dest', + required=True, + help='Destination file') + + parser.add_argument('--load', + required=True, type=int_parse, + help='Load address') + + parser.add_argument('--entry', + required=True, type=int_parse, + help='Entry point') + + parser.add_argument('--bintype', + required=True, type=int_parse, + help='Binary identification') + + return parser.parse_args() + + +def main(): + args = get_args() + source_file = args.source + destination_file = args.dest + load_address = args.load + entry_point = args.entry + binary_type = args.bintype + + stm32image_create_header_file(source_file, + destination_file, + load_address, + entry_point, + binary_type) + + +if __name__ == "__main__": + main() diff --git a/optee_os/core/arch/arm/plat-stm32mp1/shared_resources.c b/optee_os/core/arch/arm/plat-stm32mp1/shared_resources.c new file mode 100644 index 0000000..b85a130 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/shared_resources.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2017-2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Once one starts to get the resource registering state, one cannot register + * new resources. This ensures resource state cannot change. + */ +static bool registering_locked; + +/* + * Shared peripherals and resources registration + * + * Each resource assignation is stored in a table. The state defaults + * to PERIPH_UNREGISTERED if the resource is not explicitly assigned. + * + * Resource driver that as not embedded (a.k.a their related CFG_xxx build + * directive is disabled) are assigned to the non-secure world. + * + * Each IO of the GPIOZ IO can be secure or non-secure. + * + * It is the platform responsibility the ensure resource assignation + * matches the access permission firewalls configuration. + */ +enum shres_state { + SHRES_UNREGISTERED = 0, + SHRES_SECURE, + SHRES_NON_SECURE, +}; + +/* Use a byte array to store each resource state */ +static uint8_t shres_state[STM32MP1_SHRES_COUNT] = { +#if !defined(CFG_STM32_IWDG) + [STM32MP1_SHRES_IWDG1] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_UART) + [STM32MP1_SHRES_USART1] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_SPI) + [STM32MP1_SHRES_SPI6] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_I2C) + [STM32MP1_SHRES_I2C4] = SHRES_NON_SECURE, + [STM32MP1_SHRES_I2C6] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_GPIO) + [STM32MP1_SHRES_GPIOZ(0)] = SHRES_NON_SECURE, + [STM32MP1_SHRES_GPIOZ(1)] = SHRES_NON_SECURE, + [STM32MP1_SHRES_GPIOZ(2)] = SHRES_NON_SECURE, + [STM32MP1_SHRES_GPIOZ(3)] = SHRES_NON_SECURE, + [STM32MP1_SHRES_GPIOZ(4)] = SHRES_NON_SECURE, + [STM32MP1_SHRES_GPIOZ(5)] = SHRES_NON_SECURE, + [STM32MP1_SHRES_GPIOZ(6)] = SHRES_NON_SECURE, + [STM32MP1_SHRES_GPIOZ(7)] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_RNG) + [STM32MP1_SHRES_RNG1] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_HASH) + [STM32MP1_SHRES_HASH1] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_CRYP) + [STM32MP1_SHRES_CRYP1] = SHRES_NON_SECURE, +#endif +#if !defined(CFG_STM32_RTC) + [STM32MP1_SHRES_RTC] = SHRES_NON_SECURE, +#endif +}; + +static const char __maybe_unused *shres2str_id_tbl[STM32MP1_SHRES_COUNT] = { + [STM32MP1_SHRES_GPIOZ(0)] = "GPIOZ0", + [STM32MP1_SHRES_GPIOZ(1)] = "GPIOZ1", + [STM32MP1_SHRES_GPIOZ(2)] = "GPIOZ2", + [STM32MP1_SHRES_GPIOZ(3)] = "GPIOZ3", + [STM32MP1_SHRES_GPIOZ(4)] = "GPIOZ4", + [STM32MP1_SHRES_GPIOZ(5)] = "GPIOZ5", + [STM32MP1_SHRES_GPIOZ(6)] = "GPIOZ6", + [STM32MP1_SHRES_GPIOZ(7)] = "GPIOZ7", + [STM32MP1_SHRES_IWDG1] = "IWDG1", + [STM32MP1_SHRES_USART1] = "USART1", + [STM32MP1_SHRES_SPI6] = "SPI6", + [STM32MP1_SHRES_I2C4] = "I2C4", + [STM32MP1_SHRES_RNG1] = "RNG1", + [STM32MP1_SHRES_HASH1] = "HASH1", + [STM32MP1_SHRES_CRYP1] = "CRYP1", + [STM32MP1_SHRES_I2C6] = "I2C6", + [STM32MP1_SHRES_RTC] = "RTC", + [STM32MP1_SHRES_MCU] = "MCU", + [STM32MP1_SHRES_PLL3] = "PLL3", + [STM32MP1_SHRES_MDMA] = "MDMA", + [STM32MP1_SHRES_SRAM1] = "SRAM1", + [STM32MP1_SHRES_SRAM2] = "SRAM2", + [STM32MP1_SHRES_SRAM3] = "SRAM3", + [STM32MP1_SHRES_SRAM4] = "SRAM4", +}; + +static __maybe_unused const char *shres2str_id(enum stm32mp_shres id) +{ + return shres2str_id_tbl[id]; +} + +static const char *shres2str_state_tbl[4] __maybe_unused = { + [SHRES_UNREGISTERED] = "unregistered", + [SHRES_NON_SECURE] = "non-secure", + [SHRES_SECURE] = "secure", +}; + +static __maybe_unused const char *shres2str_state(enum shres_state id) +{ + return shres2str_state_tbl[id]; +} + +/* GPIOZ bank pin count depends on SoC variants */ +/* A light count routine for unpaged context to not depend on DTB support */ +static int gpioz_nbpin = -1; + +static unsigned int get_gpioz_nbpin(void) +{ + if (gpioz_nbpin < 0) + panic(); + + return gpioz_nbpin; +} + +void stm32mp_register_gpioz_pin_count(size_t count) +{ + assert(gpioz_nbpin == -1); + + gpioz_nbpin = count; +} + +static void register_periph(enum stm32mp_shres id, enum shres_state state) +{ + assert(id < STM32MP1_SHRES_COUNT && + (state == SHRES_SECURE || state == SHRES_NON_SECURE)); + + if (registering_locked) + panic(); + + if (shres_state[id] != SHRES_UNREGISTERED && + shres_state[id] != state) { + DMSG("Cannot change %s from %s to %s", + shres2str_id(id), + shres2str_state(shres_state[id]), + shres2str_state(state)); + panic(); + } + + if (shres_state[id] == SHRES_UNREGISTERED) + DMSG("Register %s as %s", + shres2str_id(id), shres2str_state(state)); + + switch (id) { + case STM32MP1_SHRES_GPIOZ(0): + case STM32MP1_SHRES_GPIOZ(1): + case STM32MP1_SHRES_GPIOZ(2): + case STM32MP1_SHRES_GPIOZ(3): + case STM32MP1_SHRES_GPIOZ(4): + case STM32MP1_SHRES_GPIOZ(5): + case STM32MP1_SHRES_GPIOZ(6): + case STM32MP1_SHRES_GPIOZ(7): + if ((id - STM32MP1_SHRES_GPIOZ(0)) >= get_gpioz_nbpin()) { + EMSG("Invalid GPIO %u >= %u", + id - STM32MP1_SHRES_GPIOZ(0), get_gpioz_nbpin()); + panic(); + } + break; + default: + break; + } + + shres_state[id] = state; + + /* Explore clock tree to lock secure clock dependencies */ + if (state == SHRES_SECURE) { + switch (id) { + case STM32MP1_SHRES_GPIOZ(0): + case STM32MP1_SHRES_GPIOZ(1): + case STM32MP1_SHRES_GPIOZ(2): + case STM32MP1_SHRES_GPIOZ(3): + case STM32MP1_SHRES_GPIOZ(4): + case STM32MP1_SHRES_GPIOZ(5): + case STM32MP1_SHRES_GPIOZ(6): + case STM32MP1_SHRES_GPIOZ(7): + stm32mp_register_clock_parents_secure(GPIOZ); + break; + case STM32MP1_SHRES_IWDG1: + stm32mp_register_clock_parents_secure(IWDG1); + break; + case STM32MP1_SHRES_USART1: + stm32mp_register_clock_parents_secure(USART1_K); + break; + case STM32MP1_SHRES_SPI6: + stm32mp_register_clock_parents_secure(SPI6_K); + break; + case STM32MP1_SHRES_I2C4: + stm32mp_register_clock_parents_secure(I2C4_K); + break; + case STM32MP1_SHRES_RNG1: + stm32mp_register_clock_parents_secure(RNG1_K); + break; + case STM32MP1_SHRES_HASH1: + stm32mp_register_clock_parents_secure(HASH1); + break; + case STM32MP1_SHRES_CRYP1: + stm32mp_register_clock_parents_secure(CRYP1); + break; + case STM32MP1_SHRES_I2C6: + stm32mp_register_clock_parents_secure(I2C6_K); + break; + case STM32MP1_SHRES_RTC: + stm32mp_register_clock_parents_secure(RTC); + break; + default: + /* No expected resource dependency */ + break; + } + } +} + +/* Register resource by ID */ +void stm32mp_register_secure_periph(enum stm32mp_shres id) +{ + register_periph(id, SHRES_SECURE); +} + +void stm32mp_register_non_secure_periph(enum stm32mp_shres id) +{ + register_periph(id, SHRES_NON_SECURE); +} + +/* Register resource by IO memory base address */ +static void register_periph_iomem(vaddr_t base, enum shres_state state) +{ + enum stm32mp_shres id = STM32MP1_SHRES_COUNT; + + switch (base) { + case IWDG1_BASE: + id = STM32MP1_SHRES_IWDG1; + break; + case USART1_BASE: + id = STM32MP1_SHRES_USART1; + break; + case SPI6_BASE: + id = STM32MP1_SHRES_SPI6; + break; + case I2C4_BASE: + id = STM32MP1_SHRES_I2C4; + break; + case I2C6_BASE: + id = STM32MP1_SHRES_I2C6; + break; + case RTC_BASE: + id = STM32MP1_SHRES_RTC; + break; + case RNG1_BASE: + id = STM32MP1_SHRES_RNG1; + break; + case CRYP1_BASE: + id = STM32MP1_SHRES_CRYP1; + break; + case HASH1_BASE: + id = STM32MP1_SHRES_HASH1; + break; + case SRAM1_BASE: + id = STM32MP1_SHRES_SRAM1; + break; + case SRAM2_BASE: + id = STM32MP1_SHRES_SRAM2; + break; + case SRAM3_BASE: + id = STM32MP1_SHRES_SRAM3; + break; + case SRAM4_BASE: + id = STM32MP1_SHRES_SRAM4; + break; + + /* Always non-secure resource cases */ +#ifdef CFG_WITH_NSEC_GPIOS + case GPIOA_BASE: + case GPIOB_BASE: + case GPIOC_BASE: + case GPIOD_BASE: + case GPIOE_BASE: + case GPIOF_BASE: + case GPIOG_BASE: + case GPIOH_BASE: + case GPIOI_BASE: + case GPIOJ_BASE: + case GPIOK_BASE: + fallthrough; +#endif +#ifdef CFG_WITH_NSEC_UARTS + case USART2_BASE: + case USART3_BASE: + case UART4_BASE: + case UART5_BASE: + case USART6_BASE: + case UART7_BASE: + case UART8_BASE: + fallthrough; +#endif +#ifdef CFG_WITH_NSEC_I2CS + case I2C5_BASE: + fallthrough; +#endif + case IWDG2_BASE: + /* Allow drivers to register some non-secure resources */ + DMSG("IO for non-secure resource 0x%lx", base); + if (state != SHRES_NON_SECURE) + panic(); + + return; + + default: + panic(); + break; + } + + register_periph(id, state); +} + +void stm32mp_register_secure_periph_iomem(vaddr_t base) +{ + register_periph_iomem(base, SHRES_SECURE); +} + +void stm32mp_register_non_secure_periph_iomem(vaddr_t base) +{ + register_periph_iomem(base, SHRES_NON_SECURE); +} + +/* Register GPIO resource */ +void stm32mp_register_secure_gpio(unsigned int bank, unsigned int pin) +{ + switch (bank) { + case GPIO_BANK_Z: + assert(pin < get_gpioz_nbpin()); + register_periph(STM32MP1_SHRES_GPIOZ(pin), SHRES_SECURE); + break; + default: + EMSG("GPIO bank %u cannot be secured", bank); + panic(); + } +} + +void stm32mp_register_non_secure_gpio(unsigned int bank, unsigned int pin) +{ + switch (bank) { + case GPIO_BANK_Z: + assert(pin < get_gpioz_nbpin()); + register_periph(STM32MP1_SHRES_GPIOZ(pin), SHRES_NON_SECURE); + break; + default: + break; + } +} + +void stm32mp_register_secure_pinctrl(struct pinctrl_state *pinctrl) +{ + unsigned int *bank = NULL; + unsigned int *pin = NULL; + size_t count = 0; + size_t n = 0; + + stm32_gpio_pinctrl_bank_pin(pinctrl, NULL, NULL, &count); + if (!count) + return; + + bank = calloc(count, sizeof(*bank)); + pin = calloc(count, sizeof(*pin)); + if (!bank || !pin) + panic(); + + stm32_gpio_pinctrl_bank_pin(pinctrl, bank, pin, &count); + for (n = 0; n < count; n++) + stm32mp_register_secure_gpio(bank[n], pin[n]); + + free(bank); + free(pin); +} + +void stm32mp_register_non_secure_pinctrl(struct pinctrl_state *pinctrl) +{ + unsigned int *bank = NULL; + unsigned int *pin = NULL; + size_t count = 0; + size_t n = 0; + + stm32_gpio_pinctrl_bank_pin(pinctrl, NULL, NULL, &count); + if (!count) + return; + + bank = calloc(count, sizeof(*bank)); + pin = calloc(count, sizeof(*pin)); + if (!bank || !pin) + panic(); + + stm32_gpio_pinctrl_bank_pin(pinctrl, bank, pin, &count); + for (n = 0; n < count; n++) + stm32mp_register_non_secure_gpio(bank[n], pin[n]); + + free(bank); + free(pin); +} + +static void lock_registering(void) +{ + registering_locked = true; +} + +bool stm32mp_periph_is_secure(enum stm32mp_shres id) +{ + lock_registering(); + + return shres_state[id] == SHRES_SECURE; +} + +bool stm32mp_gpio_bank_is_non_secure(unsigned int bank) +{ + unsigned int not_secure = 0; + unsigned int pin = 0; + + lock_registering(); + + if (bank != GPIO_BANK_Z) + return true; + + for (pin = 0; pin < get_gpioz_nbpin(); pin++) + if (!stm32mp_periph_is_secure(STM32MP1_SHRES_GPIOZ(pin))) + not_secure++; + + return not_secure > 0 && not_secure == get_gpioz_nbpin(); +} + +bool stm32mp_gpio_bank_is_secure(unsigned int bank) +{ + unsigned int secure = 0; + unsigned int pin = 0; + + lock_registering(); + + if (bank != GPIO_BANK_Z) + return false; + + for (pin = 0; pin < get_gpioz_nbpin(); pin++) + if (stm32mp_periph_is_secure(STM32MP1_SHRES_GPIOZ(pin))) + secure++; + + return secure > 0 && secure == get_gpioz_nbpin(); +} + +bool stm32mp_nsec_can_access_clock(unsigned long clock_id) +{ + enum stm32mp_shres shres_id = STM32MP1_SHRES_COUNT; + + /* Oscillators and PLLs are visible from non-secure world */ + COMPILE_TIME_ASSERT(CK_HSE == 0 && + (CK_HSE + 1) == CK_CSI && + (CK_HSE + 2) == CK_LSI && + (CK_HSE + 3) == CK_LSE && + (CK_HSE + 4) == CK_HSI && + (CK_HSE + 5) == CK_HSE_DIV2 && + (PLL1_P + 1) == PLL1_Q && + (PLL1_P + 2) == PLL1_R && + (PLL1_P + 3) == PLL2_P && + (PLL1_P + 4) == PLL2_Q && + (PLL1_P + 5) == PLL2_R && + (PLL1_P + 6) == PLL3_P && + (PLL1_P + 7) == PLL3_Q && + (PLL1_P + 8) == PLL3_R); + + if (clock_id <= CK_HSE_DIV2 || + (clock_id >= PLL1_P && clock_id <= PLL3_R)) + return true; + + switch (clock_id) { + case RTCAPB: + case CK_MPU: + case CK_AXI: + case BSEC: + return true; + case GPIOZ: + return !stm32mp_gpio_bank_is_secure(GPIO_BANK_Z); + case SPI6_K: + shres_id = STM32MP1_SHRES_SPI6; + break; + case I2C4_K: + shres_id = STM32MP1_SHRES_I2C4; + break; + case I2C6_K: + shres_id = STM32MP1_SHRES_I2C6; + break; + case USART1_K: + shres_id = STM32MP1_SHRES_USART1; + break; + case IWDG1: + shres_id = STM32MP1_SHRES_IWDG1; + break; + case CRYP1: + shres_id = STM32MP1_SHRES_CRYP1; + break; + case HASH1: + shres_id = STM32MP1_SHRES_HASH1; + break; + case RNG1_K: + shres_id = STM32MP1_SHRES_RNG1; + break; + case RTC: + shres_id = STM32MP1_SHRES_RTC; + break; + case CK_MCU: + shres_id = STM32MP1_SHRES_MCU; + break; + default: + return false; + } + + return !stm32mp_periph_is_secure(shres_id); +} + +bool stm32mp_nsec_can_access_reset(unsigned int reset_id) +{ + enum stm32mp_shres shres_id = STM32MP1_SHRES_COUNT; + + switch (reset_id) { + case GPIOZ_R: + return stm32mp_gpio_bank_is_non_secure(GPIO_BANK_Z); + case SPI6_R: + shres_id = STM32MP1_SHRES_SPI6; + break; + case I2C4_R: + shres_id = STM32MP1_SHRES_I2C4; + break; + case I2C6_R: + shres_id = STM32MP1_SHRES_I2C6; + break; + case USART1_R: + shres_id = STM32MP1_SHRES_USART1; + break; + case CRYP1_R: + shres_id = STM32MP1_SHRES_CRYP1; + break; + case HASH1_R: + shres_id = STM32MP1_SHRES_HASH1; + break; + case RNG1_R: + shres_id = STM32MP1_SHRES_RNG1; + break; + case MDMA_R: + shres_id = STM32MP1_SHRES_MDMA; + break; + case MCU_R: + case MCU_HOLD_BOOT_R: + shres_id = STM32MP1_SHRES_MCU; + break; + default: + return false; + } + + return !stm32mp_periph_is_secure(shres_id); +} + +static bool mckprot_resource(enum stm32mp_shres id) +{ + switch (id) { + case STM32MP1_SHRES_MCU: + case STM32MP1_SHRES_PLL3: + return true; + default: + return false; + } +} + +#ifdef CFG_STM32_ETZPC +static enum etzpc_decprot_attributes shres2decprot_attr(enum stm32mp_shres id) +{ + if (!stm32mp_periph_is_secure(id)) + return ETZPC_DECPROT_NS_RW; + + if (mckprot_resource(id)) + return ETZPC_DECPROT_MCU_ISOLATION; + + return ETZPC_DECPROT_S_RW; +} + +/* Configure ETZPC cell and lock it when resource is secure */ +static void config_lock_decprot(uint32_t decprot_id, + enum etzpc_decprot_attributes decprot_attr) +{ + etzpc_configure_decprot(decprot_id, decprot_attr); + + if (decprot_attr == ETZPC_DECPROT_S_RW) + etzpc_lock_decprot(decprot_id); +} + +static void set_etzpc_secure_configuration(void) +{ + /* Some peripherals shall be secure */ + config_lock_decprot(STM32MP1_ETZPC_STGENC_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_BKPSRAM_ID, ETZPC_DECPROT_S_RW); + config_lock_decprot(STM32MP1_ETZPC_DDRCTRL_ID, ETZPC_DECPROT_NS_R_S_W); + config_lock_decprot(STM32MP1_ETZPC_DDRPHYC_ID, ETZPC_DECPROT_NS_R_S_W); + + /* Configure ETZPC with peripheral registering */ + config_lock_decprot(STM32MP1_ETZPC_IWDG1_ID, + shres2decprot_attr(STM32MP1_SHRES_IWDG1)); + config_lock_decprot(STM32MP1_ETZPC_USART1_ID, + shres2decprot_attr(STM32MP1_SHRES_USART1)); + config_lock_decprot(STM32MP1_ETZPC_SPI6_ID, + shres2decprot_attr(STM32MP1_SHRES_SPI6)); + config_lock_decprot(STM32MP1_ETZPC_I2C4_ID, + shres2decprot_attr(STM32MP1_SHRES_I2C4)); + config_lock_decprot(STM32MP1_ETZPC_RNG1_ID, + shres2decprot_attr(STM32MP1_SHRES_RNG1)); + config_lock_decprot(STM32MP1_ETZPC_HASH1_ID, + shres2decprot_attr(STM32MP1_SHRES_HASH1)); + config_lock_decprot(STM32MP1_ETZPC_CRYP1_ID, + shres2decprot_attr(STM32MP1_SHRES_CRYP1)); + config_lock_decprot(STM32MP1_ETZPC_I2C6_ID, + shres2decprot_attr(STM32MP1_SHRES_I2C6)); + + config_lock_decprot(STM32MP1_ETZPC_SRAM1_ID, + shres2decprot_attr(STM32MP1_SHRES_SRAM1)); + config_lock_decprot(STM32MP1_ETZPC_SRAM2_ID, + shres2decprot_attr(STM32MP1_SHRES_SRAM2)); + config_lock_decprot(STM32MP1_ETZPC_SRAM3_ID, + shres2decprot_attr(STM32MP1_SHRES_SRAM3)); + config_lock_decprot(STM32MP1_ETZPC_SRAM4_ID, + shres2decprot_attr(STM32MP1_SHRES_SRAM4)); +} +#else +static void set_etzpc_secure_configuration(void) +{ + /* Nothing to do */ +} +#endif + +static void check_rcc_secure_configuration(void) +{ + bool secure = stm32_rcc_is_secure(); + bool mckprot = stm32_rcc_is_mckprot(); + enum stm32mp_shres id = STM32MP1_SHRES_COUNT; + bool have_error = false; + uint32_t state = 0; + + if (stm32_bsec_get_state(&state)) + panic(); + + if (state == BSEC_STATE_SEC_CLOSED && !secure) + panic("Closed device mandates secure RCC"); + + for (id = 0; id < STM32MP1_SHRES_COUNT; id++) { + if (shres_state[id] != SHRES_SECURE) + continue; + + /* SRAMs have no constraints on RCC configuration */ + if (id == STM32MP1_SHRES_SRAM1 || + id == STM32MP1_SHRES_SRAM2 || + id == STM32MP1_SHRES_SRAM3 || + id == STM32MP1_SHRES_SRAM4) + continue; + + if ((mckprot_resource(id) && !mckprot) || !secure) { + EMSG("RCC %s MCKPROT %s and %s (%u) secure", + secure ? "secure" : "non-secure", + mckprot ? "set" : "not set", + shres2str_id(id), id); + have_error = true; + } + } + + if (have_error) + panic(); +} + +static void set_gpio_secure_configuration(void) +{ + unsigned int pin = 0; + + for (pin = 0; pin < get_gpioz_nbpin(); pin++) { + enum stm32mp_shres shres = STM32MP1_SHRES_GPIOZ(pin); + bool secure = stm32mp_periph_is_secure(shres); + + stm32_gpio_set_secure_cfg(GPIO_BANK_Z, pin, secure); + } +} + +static TEE_Result gpioz_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *hdl __unused) +{ + if (op == PM_OP_RESUME) + set_gpio_secure_configuration(); + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(gpioz_pm); + +static TEE_Result stm32mp1_init_final_shres(void) +{ + enum stm32mp_shres id = STM32MP1_SHRES_COUNT; + + lock_registering(); + + for (id = (enum stm32mp_shres)0; id < STM32MP1_SHRES_COUNT; id++) { + uint8_t __maybe_unused *state = &shres_state[id]; + + DMSG("stm32mp %-8s (%2u): %-14s", + shres2str_id(id), id, shres2str_state(*state)); + } + + set_etzpc_secure_configuration(); + if (IS_ENABLED(CFG_STM32_GPIO)) { + set_gpio_secure_configuration(); + register_pm_driver_cb(gpioz_pm, NULL, + "stm32mp1-shared-resources"); + } + check_rcc_secure_configuration(); + + return TEE_SUCCESS; +} +/* Finalize shres after drivers initialization, hence driver_init_late() */ +driver_init_late(stm32mp1_init_final_shres); diff --git a/optee_os/core/arch/arm/plat-stm32mp1/stm32_util.h b/optee_os/core/arch/arm/plat-stm32mp1/stm32_util.h new file mode 100644 index 0000000..4ec5933 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/stm32_util.h @@ -0,0 +1,354 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2018-2022, STMicroelectronics + */ + +#ifndef __STM32_UTIL_H__ +#define __STM32_UTIL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Backup registers and RAM utils */ +vaddr_t stm32mp_bkpreg(unsigned int idx); + +/* + * SYSCFG IO compensation. + * These functions assume non-secure world is suspended. + */ +void stm32mp_syscfg_enable_io_compensation(void); +void stm32mp_syscfg_disable_io_compensation(void); + +/* Platform util for the RCC drivers */ +vaddr_t stm32_rcc_base(void); + +/* Platform util for the GIC */ +vaddr_t get_gicd_base(void); + +/* + * Platform util functions for the GPIO driver + * @bank: Target GPIO bank ID as per DT bindings + * + * Platform shall implement these functions to provide to stm32_gpio + * driver the resource reference for a target GPIO bank. That are + * memory mapped interface base address, interface offset (see below) + * and clock identifier. + * + * stm32_get_gpio_bank_offset() returns a bank offset that is used to + * check DT configuration matches platform implementation of the banks + * description. + */ +unsigned int stm32_get_gpio_bank_offset(unsigned int bank); + +/* Platform util for PMIC support */ +bool stm32mp_with_pmic(void); + +/* Power management service */ +#ifdef CFG_PSCI_ARM32 +void stm32mp_register_online_cpu(void); +#else +static inline void stm32mp_register_online_cpu(void) +{ +} +#endif + +/* + * Generic spinlock function that bypass spinlock if MMU is disabled or + * lock is NULL. + */ +uint32_t may_spin_lock(unsigned int *lock); +void may_spin_unlock(unsigned int *lock, uint32_t exceptions); + +/* Helper from platform RCC clock driver */ +struct clk *stm32mp_rcc_clock_id_to_clk(unsigned long clock_id); + +#ifdef CFG_STM32MP1_SHARED_RESOURCES +/* Return true if @clock_id is shared by secure and non-secure worlds */ +bool stm32mp_nsec_can_access_clock(unsigned long clock_id); +#else /* CFG_STM32MP1_SHARED_RESOURCES */ +static inline bool stm32mp_nsec_can_access_clock(unsigned long clock_id + __unused) +{ + return true; +} +#endif /* CFG_STM32MP1_SHARED_RESOURCES */ + +extern const struct clk_ops stm32mp1_clk_ops; + +#if defined(CFG_STPMIC1) +/* Return true if non-secure world can manipulate regulator @pmic_regu_name */ +bool stm32mp_nsec_can_access_pmic_regu(const char *pmic_regu_name); +#else +static inline bool stm32mp_nsec_can_access_pmic_regu(const char *name __unused) +{ + return false; +} +#endif + +#ifdef CFG_STM32MP1_SHARED_RESOURCES +/* Return true if and only if @reset_id relates to a non-secure peripheral */ +bool stm32mp_nsec_can_access_reset(unsigned int reset_id); +#else /* CFG_STM32MP1_SHARED_RESOURCES */ +static inline bool stm32mp_nsec_can_access_reset(unsigned int reset_id __unused) +{ + return true; +} +#endif /* CFG_STM32MP1_SHARED_RESOURCES */ + +/* Return rstctrl instance related to RCC reset controller DT binding ID */ +struct rstctrl *stm32mp_rcc_reset_id_to_rstctrl(unsigned int binding_id); + +/* + * Structure and API function for BSEC driver to get some platform data. + * + * @base: BSEC interface registers physical base address + * @upper_start: Base ID for the BSEC upper words in the platform + * @max_id: Max value for BSEC word ID for the platform + */ +struct stm32_bsec_static_cfg { + paddr_t base; + unsigned int upper_start; + unsigned int max_id; +}; + +void stm32mp_get_bsec_static_cfg(struct stm32_bsec_static_cfg *cfg); + +/* + * Shared reference counter: increments by 2 on secure increment + * request, decrements by 2 on secure decrement request. Bit #0 + * is set to 1 on non-secure increment request and reset to 0 on + * non-secure decrement request. These counters initialize to + * either 0, 1 or 2 upon their expect default state. + * Counters saturate to UINT_MAX / 2. + */ +#define SHREFCNT_NONSECURE_FLAG 0x1ul +#define SHREFCNT_SECURE_STEP 0x2ul +#define SHREFCNT_MAX (UINT_MAX / 2) + +/* Return 1 if refcnt increments from 0, else return 0 */ +static inline int incr_shrefcnt(unsigned int *refcnt, bool secure) +{ + int rc = !*refcnt; + + if (secure) { + if (*refcnt < SHREFCNT_MAX) { + *refcnt += SHREFCNT_SECURE_STEP; + assert(*refcnt < SHREFCNT_MAX); + } + } else { + *refcnt |= SHREFCNT_NONSECURE_FLAG; + } + + return rc; +} + +/* Return 1 if refcnt decrements to 0, else return 0 */ +static inline int decr_shrefcnt(unsigned int *refcnt, bool secure) +{ + int rc = 0; + + if (secure) { + if (*refcnt < SHREFCNT_MAX) { + if (*refcnt < SHREFCNT_SECURE_STEP) + panic(); + + *refcnt -= SHREFCNT_SECURE_STEP; + rc = !*refcnt; + } + } else { + rc = (*refcnt == SHREFCNT_NONSECURE_FLAG); + *refcnt &= ~SHREFCNT_NONSECURE_FLAG; + } + + return rc; +} + +static inline int incr_refcnt(unsigned int *refcnt) +{ + return incr_shrefcnt(refcnt, true); +} + +static inline int decr_refcnt(unsigned int *refcnt) +{ + return decr_shrefcnt(refcnt, true); +} + +/* + * Shared peripherals and resources registration + * + * Resources listed in enum stm32mp_shres assigned at run-time to the + * non-secure world, to the secure world or shared by both worlds. + * In the later case, there must exist a secure service in OP-TEE + * for the non-secure world to access the resource. + * + * Resources may be a peripheral, a bus, a clock or a memory. + * + * Shared resources driver API functions allows drivers to register the + * resource as secure, non-secure or shared and to get the resource + * assignation state. + */ +#define STM32MP1_SHRES_GPIOZ(i) (STM32MP1_SHRES_GPIOZ_0 + i) + +enum stm32mp_shres { + STM32MP1_SHRES_GPIOZ_0 = 0, + STM32MP1_SHRES_GPIOZ_1, + STM32MP1_SHRES_GPIOZ_2, + STM32MP1_SHRES_GPIOZ_3, + STM32MP1_SHRES_GPIOZ_4, + STM32MP1_SHRES_GPIOZ_5, + STM32MP1_SHRES_GPIOZ_6, + STM32MP1_SHRES_GPIOZ_7, + STM32MP1_SHRES_IWDG1, + STM32MP1_SHRES_USART1, + STM32MP1_SHRES_SPI6, + STM32MP1_SHRES_I2C4, + STM32MP1_SHRES_RNG1, + STM32MP1_SHRES_HASH1, + STM32MP1_SHRES_CRYP1, + STM32MP1_SHRES_I2C6, + STM32MP1_SHRES_RTC, + STM32MP1_SHRES_MCU, + STM32MP1_SHRES_PLL3, + STM32MP1_SHRES_MDMA, + STM32MP1_SHRES_SRAM1, + STM32MP1_SHRES_SRAM2, + STM32MP1_SHRES_SRAM3, + STM32MP1_SHRES_SRAM4, + + STM32MP1_SHRES_COUNT +}; + +#ifdef CFG_STM32MP1_SHARED_RESOURCES +/* Register resource @id as a secure peripheral */ +void stm32mp_register_secure_periph(enum stm32mp_shres id); + +/* Register resource @id as a non-secure peripheral */ +void stm32mp_register_non_secure_periph(enum stm32mp_shres id); + +/* + * Register resource identified by @base as a secure peripheral + * @base: IOMEM physical base address of the resource + */ +void stm32mp_register_secure_periph_iomem(vaddr_t base); + +/* + * Register resource identified by @base as a non-secure peripheral + * @base: IOMEM physical base address of the resource + */ +void stm32mp_register_non_secure_periph_iomem(vaddr_t base); + +/* + * Register GPIO resource as a secure peripheral + * @bank: Bank of the target GPIO + * @pin: Bit position of the target GPIO in the bank + */ +void stm32mp_register_secure_gpio(unsigned int bank, unsigned int pin); + +/* + * Register GPIO resource as a non-secure peripheral + * @bank: Bank of the target GPIO + * @pin: Bit position of the target GPIO in the bank + */ +void stm32mp_register_non_secure_gpio(unsigned int bank, unsigned int pin); + +/* + * Register pin resource of a pin control state as a secure peripheral + * @bank: Bank of the target GPIO + * @pin: Bit position of the target GPIO in the bank + */ +void stm32mp_register_secure_pinctrl(struct pinctrl_state *pinctrl); + +/* + * Register pin resource of a pin control state as a non-secure peripheral + * @bank: Bank of the target GPIO + * @pin: Bit position of the target GPIO in the bank + */ +void stm32mp_register_non_secure_pinctrl(struct pinctrl_state *pinctrl); + +/* Return true if and only if resource @id is registered as secure */ +bool stm32mp_periph_is_secure(enum stm32mp_shres id); + +/* Return true if and only if GPIO bank @bank is registered as secure */ +bool stm32mp_gpio_bank_is_secure(unsigned int bank); + +/* Return true if and only if GPIO bank @bank is registered as non-secure */ +bool stm32mp_gpio_bank_is_non_secure(unsigned int bank); + +/* Register parent clocks of @clock (ID used in clock DT bindings) as secure */ +void stm32mp_register_clock_parents_secure(unsigned long clock_id); + +/* Register number of pins in the GPIOZ bank */ +void stm32mp_register_gpioz_pin_count(size_t count); + +#else /* CFG_STM32MP1_SHARED_RESOURCES */ + +static inline void stm32mp_register_secure_periph(enum stm32mp_shres id + __unused) +{ +} + +static inline void stm32mp_register_non_secure_periph(enum stm32mp_shres id + __unused) +{ +} + +static inline void stm32mp_register_secure_periph_iomem(vaddr_t base __unused) +{ +} + +static inline void stm32mp_register_non_secure_periph_iomem(vaddr_t base + __unused) +{ +} + +static inline void stm32mp_register_secure_gpio(unsigned int bank __unused, + unsigned int pin __unused) +{ +} + +static inline void stm32mp_register_non_secure_gpio(unsigned int bank __unused, + unsigned int pin __unused) +{ +} + +static inline void +stm32mp_register_secure_pinctrl(struct pinctrl_state *pinctrl __unused) +{ +} + +static inline void +stm32mp_register_non_secure_pinctrl(struct pinctrl_state *pinctrl __unused) +{ +} + +static inline bool stm32mp_periph_is_secure(enum stm32mp_shres id __unused) +{ + return true; +} + +static inline bool stm32mp_gpio_bank_is_secure(unsigned int bank __unused) +{ + return true; +} + +static inline bool stm32mp_gpio_bank_is_non_secure(unsigned int bank __unused) +{ + return false; +} + +static inline void stm32mp_register_clock_parents_secure(unsigned long clock_id + __unused) +{ +} + +static inline void stm32mp_register_gpioz_pin_count(size_t count __unused) +{ +} +#endif /* CFG_STM32MP1_SHARED_RESOURCES */ +#endif /*__STM32_UTIL_H__*/ diff --git a/optee_os/core/arch/arm/plat-stm32mp1/sub.mk b/optee_os/core/arch/arm/plat-stm32mp1/sub.mk new file mode 100644 index 0000000..fa6e148 --- /dev/null +++ b/optee_os/core/arch/arm/plat-stm32mp1/sub.mk @@ -0,0 +1,12 @@ +global-incdirs-y += . + +srcs-y += main.c +srcs-y += reset.S +srcs-$(CFG_SCMI_MSG_DRIVERS) += scmi_server.c +srcs-$(CFG_STM32MP1_SHARED_RESOURCES) += shared_resources.c +srcs-$(CFG_TZC400) += plat_tzc400.c +srcs-$(CFG_WITH_PAGER) += link_dummies_paged.c + +subdirs-y += drivers +subdirs-y += nsec-service +subdirs-y += pm diff --git a/optee_os/core/arch/arm/plat-sunxi/conf.mk b/optee_os/core/arch/arm/plat-sunxi/conf.mk new file mode 100644 index 0000000..5c0effc --- /dev/null +++ b/optee_os/core/arch/arm/plat-sunxi/conf.mk @@ -0,0 +1,45 @@ +PLATFORM_FLAVOR ?= bpi_zero + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_8250_UART,y) + +ifeq ($(PLATFORM_FLAVOR),bpi_zero) +include core/arch/arm/cpu/cortex-a7.mk +$(call force,CFG_SUN8I_H2_PLUS,y) +$(call force,CFG_ARM32_core,y) +$(call force,CFG_GIC,y) +$(call force,CFG_WITH_LPAE,n) +$(call force,CFG_WITH_PAGER,n) + +CFG_CRYPTO_SIZE_OPTIMIZATION ?= n +CFG_NUM_THREADS ?= 4 +CFG_TEE_CORE_NB_CORE ?= 4 +CFG_BOOT_SECONDARY_REQUEST ?= y +CFG_PSCI_ARM32 ?= y +CFG_NS_ENTRY_ADDR ?= 0x42000000 +CFG_DT ?= y +CFG_INIT_CNTVOFF ?= y +CFG_SECONDARY_INIT_CNTFRQ ?= y +CFG_TZDRAM_START ?= 0x5c000000 +CFG_TZDRAM_SIZE ?= 0x03e00000 +CFG_SHMEM_START ?= 0x5fe00000 +CFG_SHMEM_SIZE ?= 0x00200000 +endif + +ifeq ($(PLATFORM_FLAVOR),sun50i_a64) +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_ARM64_core,y) + +CFG_TZDRAM_START ?= 0x40000000 +CFG_TZDRAM_SIZE ?= 0x2000000 +CFG_SHMEM_START ?= 0x44000000 +CFG_SHMEM_SIZE ?= 0x00400000 +CFG_TEE_CORE_NB_CORE ?= 4 +CFG_TZC380 ?= y +endif + +ifeq ($(platform-flavor-armv8),1) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +endif + +CFG_WITH_STATS ?= y diff --git a/optee_os/core/arch/arm/plat-sunxi/main.c b/optee_os/core/arch/arm/plat-sunxi/main.c new file mode 100644 index 0000000..7f4215a --- /dev/null +++ b/optee_os/core/arch/arm/plat-sunxi/main.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Allwinner Technology Co., Ltd. + * Copyright (c) 2018, Linaro Limited + * Copyright (c) 2018, Amit Singh Tomar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef GIC_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE, CORE_MMU_PGDIR_SIZE); +#endif + +#ifdef CONSOLE_UART_BASE +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, + CONSOLE_UART_BASE, SUNXI_UART_REG_SIZE); +#endif + +#ifdef SUNXI_TZPC_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SUNXI_TZPC_BASE, SUNXI_TZPC_REG_SIZE); +#define REG_TZPC_SMTA_DECPORT0_STA_REG (0x0004) +#define REG_TZPC_SMTA_DECPORT0_SET_REG (0x0008) +#define REG_TZPC_SMTA_DECPORT0_CLR_REG (0x000C) +#define REG_TZPC_SMTA_DECPORT1_STA_REG (0x0010) +#define REG_TZPC_SMTA_DECPORT1_SET_REG (0x0014) +#define REG_TZPC_SMTA_DECPORT1_CLR_REG (0x0018) +#define REG_TZPC_SMTA_DECPORT2_STA_REG (0x001c) +#define REG_TZPC_SMTA_DECPORT2_SET_REG (0x0020) +#define REG_TZPC_SMTA_DECPORT2_CLR_REG (0x0024) +#endif + +#ifdef SUNXI_CPUCFG_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SUNXI_CPUCFG_BASE, + SUNXI_CPUCFG_REG_SIZE); +#endif + +#ifdef SUNXI_PRCM_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SUNXI_PRCM_BASE, SUNXI_PRCM_REG_SIZE); +#endif + +#ifdef CFG_TZC380 +vaddr_t smc_base(void); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SUNXI_SMC_BASE, TZC400_REG_SIZE); +#define SMC_MASTER_BYPASS 0x18 +#define SMC_MASTER_BYPASS_EN_MASK 0x1 +#endif + +#ifdef SUNXI_TZPC_BASE +static void tzpc_init(void); +#endif + +static struct serial8250_uart_data console_data; + +void console_init(void) +{ + serial8250_uart_init(&console_data, + CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, + CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +#ifdef SUNXI_TZPC_BASE +static void tzpc_init(void) +{ + vaddr_t v = (vaddr_t)phys_to_virt(SUNXI_TZPC_BASE, MEM_AREA_IO_SEC, + SUNXI_TZPC_REG_SIZE); + + DMSG("SMTA_DECPORT0=%x", io_read32(v + REG_TZPC_SMTA_DECPORT0_STA_REG)); + DMSG("SMTA_DECPORT1=%x", io_read32(v + REG_TZPC_SMTA_DECPORT1_STA_REG)); + DMSG("SMTA_DECPORT2=%x", io_read32(v + REG_TZPC_SMTA_DECPORT2_STA_REG)); + + /* Allow all peripherals for normal world */ + io_write32(v + REG_TZPC_SMTA_DECPORT0_SET_REG, 0xbe); + io_write32(v + REG_TZPC_SMTA_DECPORT1_SET_REG, 0xff); + io_write32(v + REG_TZPC_SMTA_DECPORT2_SET_REG, 0x7f); + + DMSG("SMTA_DECPORT0=%x", io_read32(v + REG_TZPC_SMTA_DECPORT0_STA_REG)); + DMSG("SMTA_DECPORT1=%x", io_read32(v + REG_TZPC_SMTA_DECPORT1_STA_REG)); + DMSG("SMTA_DECPORT2=%x", io_read32(v + REG_TZPC_SMTA_DECPORT2_STA_REG)); +} +#else +static inline void tzpc_init(void) +{ +} +#endif /* SUNXI_TZPC_BASE */ + +#ifndef CFG_WITH_ARM_TRUSTED_FW +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} +#endif + +#ifdef ARM32 +void plat_primary_init_early(void) +{ + assert(!cpu_mmu_enabled()); + + tzpc_init(); +} +#endif + +/* + * Allwinner's A64 has TZC380 like controller called SMC that can + * be programmed to protect parts of DRAM from non-secure world. + */ +#ifdef CFG_TZC380 +vaddr_t smc_base(void) +{ + return (vaddr_t)phys_to_virt(SUNXI_SMC_BASE, MEM_AREA_IO_SEC, + TZC400_REG_SIZE); +} + +static TEE_Result smc_init(void) +{ + vaddr_t base = smc_base(); + + if (!base) { + EMSG("smc not mapped"); + panic(); + } + + tzc_init(base); + tzc_configure_region(0, 0x0, TZC_ATTR_REGION_SIZE(TZC_REGION_SIZE_1G) | + TZC_ATTR_REGION_EN_MASK | TZC_ATTR_SP_ALL); + tzc_configure_region(1, 0x0, TZC_ATTR_REGION_SIZE(TZC_REGION_SIZE_32M) | + TZC_ATTR_REGION_EN_MASK | TZC_ATTR_SP_S_RW); + + /* SoC specific bits */ + io_clrbits32(base + SMC_MASTER_BYPASS, SMC_MASTER_BYPASS_EN_MASK); + + return TEE_SUCCESS; +} + +driver_init(smc_init); +#endif /* CFG_TZC380 */ diff --git a/optee_os/core/arch/arm/plat-sunxi/plat_init.S b/optee_os/core/arch/arm/plat-sunxi/plat_init.S new file mode 100644 index 0000000..0b347fc --- /dev/null +++ b/optee_os/core/arch/arm/plat-sunxi/plat_init.S @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + * Copyright (C) 2018, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +FUNC plat_cpu_reset_early , : + /* NSACR configuration */ + read_nsacr r0 + orr r0, r0, #NSACR_CP10 + orr r0, r0, #NSACR_CP11 + orr r0, r0, #NSACR_NS_SMP + write_nsacr r0 + + /* Enable SMP bit */ + read_actlr r0 + orr r0, r0, #ACTLR_SMP + write_actlr r0 + + bx lr +END_FUNC plat_cpu_reset_early diff --git a/optee_os/core/arch/arm/plat-sunxi/platform_config.h b/optee_os/core/arch/arm/plat-sunxi/platform_config.h new file mode 100644 index 0000000..1c682cc --- /dev/null +++ b/optee_os/core/arch/arm/plat-sunxi/platform_config.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Allwinner Technology Co., Ltd. + * Copyright (c) 2018, Linaro Limited + * Copyright (c) 2018, Amit Singh Tomar + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +/* 16550 UART */ +#define CONSOLE_UART_BASE 0x01c28000 /* UART0 */ +#define CONSOLE_UART_CLK_IN_HZ 24000000 +#define CONSOLE_BAUDRATE 115200 +#define SUNXI_UART_REG_SIZE 0x400 + +#if defined(PLATFORM_FLAVOR_bpi_zero) +#define GIC_BASE 0x01c80000 +#define GICC_OFFSET 0x2000 +#define GICD_OFFSET 0x1000 +#define SUNXI_TZPC_BASE 0x01c23400 +#define SUNXI_TZPC_REG_SIZE 0x400 +#define SUNXI_CPUCFG_BASE 0x01f01c00 +#define SUNXI_CPUCFG_REG_SIZE 0x400 +#define SUNXI_PRCM_BASE 0x01f01400 +#define SUNXI_PRCM_REG_SIZE 0x400 +#define PRCM_CPU_SOFT_ENTRY_REG 0x164 +#endif + +#if defined(PLATFORM_FLAVOR_sun50i_a64) +#define SUNXI_SMC_BASE 0x01c1e000 +#endif + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-sunxi/psci.c b/optee_os/core/arch/arm/plat-sunxi/psci.c new file mode 100644 index 0000000..f59f72e --- /dev/null +++ b/optee_os/core/arch/arm/plat-sunxi/psci.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2013, ARM Ltd + * Copyright (c) 2014, Allwinner Technology Co., Ltd. + * Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REG_CPUCFG_RES0 (0x0000) +#define REG_CPUCFG_CPU_RST(cpu) (0x0040 + (cpu) * (0x0040)) +#define REG_CPUCFG_GEN_CTRL (0x0184) +#define REG_CPUCFG_PRIV0 (0x01a4) +#define REG_CPUCFG_DBG_CTRL1 (0x01e4) +#define REG_PRCM_CPU_PWROFF (0x0100) +#define REG_PRCM_CPU_PWR_CLAMP(cpu) (0x0140 + (cpu) * (0x0004)) + +int psci_features(uint32_t psci_fid) +{ + switch (psci_fid) { +#ifdef CFG_BOOT_SECONDARY_REQUEST + case PSCI_CPU_ON: + return 0; +#endif + + default: + return PSCI_RET_NOT_SUPPORTED; + } +} + +#ifdef CFG_BOOT_SECONDARY_REQUEST +int psci_cpu_on(uint32_t core_idx, uint32_t entry, + uint32_t context_id) +{ + vaddr_t base = (vaddr_t)phys_to_virt(SUNXI_PRCM_BASE, MEM_AREA_IO_SEC, + SUNXI_PRCM_REG_SIZE); + vaddr_t cpucfg = (vaddr_t)phys_to_virt(SUNXI_CPUCFG_BASE, + MEM_AREA_IO_SEC, + SUNXI_CPUCFG_REG_SIZE); + uint32_t tmpff; + uint32_t val; + + assert(base); + assert(cpucfg); + + if ((core_idx == 0) || (core_idx >= CFG_TEE_CORE_NB_CORE)) + return PSCI_RET_INVALID_PARAMETERS; + + /* set secondary cores' NS entry addresses */ + boot_set_core_ns_entry(core_idx, entry, context_id); + + val = virt_to_phys((void *)TEE_LOAD_ADDR); + + /* set entry address */ + DMSG("set entry address for CPU %d", core_idx); + io_write32(cpucfg + REG_CPUCFG_PRIV0, val); + + /* assert reset on target CPU */ + DMSG("assert reset on target CPU %d", core_idx); + io_write32(cpucfg + REG_CPUCFG_CPU_RST(core_idx), 0); + + /* invalidate L1 cache */ + DMSG("invalidate L1 cache for CPU %d", core_idx); + io_clrbits32(cpucfg + REG_CPUCFG_GEN_CTRL, BIT32(core_idx)); + + /* lock CPU (Disable external debug access) */ + DMSG("lock CPU %d", core_idx); + io_clrbits32(cpucfg + REG_CPUCFG_DBG_CTRL1, BIT32(core_idx)); + + /* release clamp */ + DMSG("release clamp for CPU %d", core_idx); + tmpff = 0x1ff; + do { + tmpff >>= 1; + io_write32(base + REG_PRCM_CPU_PWR_CLAMP(core_idx), tmpff); + } while (tmpff); + mdelay(10); + + /* clear power gating */ + DMSG("clear power gating for CPU %d", core_idx); + io_clrbits32(base + REG_PRCM_CPU_PWROFF, BIT32(core_idx)); + udelay(1000); + + /* de-assert reset on target CPU */ + DMSG("de-assert reset on target CPU %d", core_idx); + io_write32(cpucfg + REG_CPUCFG_CPU_RST(core_idx), 0x03); + + /* unlock CPU (enable external debug access) */ + DMSG("unlock CPU %d", core_idx); + io_setbits32(cpucfg + REG_CPUCFG_DBG_CTRL1, BIT32(core_idx)); + + return PSCI_RET_SUCCESS; +} + +int __noreturn psci_cpu_off(void) +{ + uint32_t core_id; + vaddr_t base = (vaddr_t)phys_to_virt(SUNXI_PRCM_BASE, MEM_AREA_IO_SEC, + SUNXI_PRCM_REG_SIZE); + vaddr_t cpucfg = (vaddr_t)phys_to_virt(SUNXI_CPUCFG_BASE, + MEM_AREA_IO_SEC, + SUNXI_CPUCFG_REG_SIZE); + + core_id = get_core_pos(); + + DMSG("core_id: %" PRIu32, core_id); + +#ifdef CFG_PSCI_ARM32 + psci_armv7_cpu_off(); +#endif /* CFG_PSCI_ARM32 */ + + assert(base); + assert(cpucfg); + + /* set power gating */ + DMSG("set power gating for cpu %d", core_id); + io_setbits32(base + REG_PRCM_CPU_PWROFF, BIT32(core_id)); + + /* Activate power clamp */ + DMSG("Activate power clamp for cpu %d", core_id); + io_write32(base + REG_PRCM_CPU_PWR_CLAMP(core_id), 0xff); + + while (true) + wfi(); +} +#endif diff --git a/optee_os/core/arch/arm/plat-sunxi/sub.mk b/optee_os/core/arch/arm/plat-sunxi/sub.mk new file mode 100644 index 0000000..6fefa62 --- /dev/null +++ b/optee_os/core/arch/arm/plat-sunxi/sub.mk @@ -0,0 +1,4 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-$(CFG_ARM32_core) += plat_init.S +srcs-$(CFG_ARM32_core) += psci.c diff --git a/optee_os/core/arch/arm/plat-synquacer/conf.mk b/optee_os/core/arch/arm/plat-synquacer/conf.mk new file mode 100644 index 0000000..8e8aaff --- /dev/null +++ b/optee_os/core/arch/arm/plat-synquacer/conf.mk @@ -0,0 +1,22 @@ +PLATFORM_FLAVOR ?= developerbox + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_GIC,y) +$(call force,CFG_PL011,y) + +include core/arch/arm/cpu/cortex-armv8-0.mk +$(call force,CFG_TEE_CORE_NB_CORE,24) +CFG_NUM_THREADS ?= 8 +CFG_TZDRAM_START ?= 0xfc000000 +CFG_TZDRAM_SIZE ?= 0x03c00000 +CFG_SHMEM_START ?= 0xffc00000 +CFG_SHMEM_SIZE ?= 0x00400000 + +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) + +$(call force,CFG_ARM64_core,y) +supported-ta-targets = ta_arm64 + +CFG_CRYPTO_SIZE_OPTIMIZATION ?= n +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) diff --git a/optee_os/core/arch/arm/plat-synquacer/main.c b/optee_os/core/arch/arm/plat-synquacer/main.c new file mode 100644 index 0000000..d94aff3 --- /dev/null +++ b/optee_os/core/arch/arm/plat-synquacer/main.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "synquacer_rng_pta.h" + +static struct pl011_data console_data; + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, + CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE, CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, THERMAL_SENSOR_BASE, + CORE_MMU_PGDIR_SIZE); + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, CONSOLE_UART_CLK_IN_HZ, + CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +void boot_primary_init_intc(void) +{ + gic_init(0, GIC_BASE + GICD_OFFSET); +} + +static enum itr_return timer_itr_cb(struct itr_handler *h __unused) +{ + /* Reset timer for next FIQ */ + generic_timer_handler(TIMER_PERIOD_MS); + + /* Collect entropy on each timer FIQ */ + rng_collect_entropy(); + + return ITRR_HANDLED; +} + +static struct itr_handler timer_itr = { + .it = IT_SEC_TIMER, + .flags = ITRF_TRIGGER_LEVEL, + .handler = timer_itr_cb, +}; + +static TEE_Result init_timer_itr(void) +{ + itr_add(&timer_itr); + itr_enable(IT_SEC_TIMER); + + /* Enable timer FIQ to fetch entropy required during boot */ + generic_timer_start(TIMER_PERIOD_MS); + + return TEE_SUCCESS; +} +driver_init(init_timer_itr); diff --git a/optee_os/core/arch/arm/plat-synquacer/platform_config.h b/optee_os/core/arch/arm/plat-synquacer/platform_config.h new file mode 100644 index 0000000..76a3d4a --- /dev/null +++ b/optee_os/core/arch/arm/plat-synquacer/platform_config.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +#define STACK_ALIGNMENT 64 + +#define GIC_BASE 0x30000000 +#define GICD_OFFSET 0x0 + +/* console uart define */ +#define UART0_BASE 0x2A400000 +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_UART_CLK_IN_HZ 62500000 +#define CONSOLE_BAUDRATE 115200 + +#define THERMAL_SENSOR_BASE 0x54190000 +#define IT_SEC_TIMER 29 +#define TIMER_PERIOD_MS 2 + +#define DRAM0_BASE 0x80000000 + +/* Platform specific defines */ +#if defined(PLATFORM_FLAVOR_developerbox) +#define DRAM0_SIZE 0x80000000 +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-synquacer/rng_pta.c b/optee_os/core/arch/arm/plat-synquacer/rng_pta.c new file mode 100644 index 0000000..16a1e46 --- /dev/null +++ b/optee_os/core/arch/arm/plat-synquacer/rng_pta.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018-2022, Linaro Limited + */ + +/* + * Developerbox doesn't provide a hardware based true random number + * generator. So this pseudo TA provides a good source of entropy using + * noise from 7 thermal sensors. Its suitable for entropy required + * during boot, seeding kernel entropy pool, cryptographic use etc. + * + * Assumption + * ========== + * + * We have assumed the entropy of the sensor is better than 8 bits per + * 14 sensor readings. This entropy estimate is based on our simple + * minimal entropy estimates done on 2.1G bytes of raw samples collected + * from thermal sensors. + * + * We believe our estimate to be conservative and have designed to + * health tests to trigger if a sensor does not achieve at least + * 8 bits in 16 sensor reading (we use 16 rather than 14 to prevent + * spurious failures on edge cases). + * + * Theory of operation + * =================== + * + * This routine uses secure timer interrupt to sample raw thermal sensor + * readings. As thermal sensor refresh rate is every 2ms, so interrupt + * fires every 2ms. It implements continuous health test counting rising + * and falling edges to report if sensors fail to provide entropy. + * + * It uses vetted conditioner as SHA512/256 (approved hash algorithm) + * to condense entropy. As per NIST.SP.800-90B spec, to get full entropy + * from vetted conditioner, we need to supply double of input entropy. + * According to assumption above and requirement for vetted conditioner, + * we need to supply 28 raw sensor readings to get 1 byte of full + * entropy as output. So for 32 bytes of conditioner output, we need to + * supply 896 bytes of raw sensor readings. + * + * Interfaces -> Input + * ------------------- + * + * void rng_collect_entropy(void); + * + * Called as part of secure timer interrupt handler to sample raw + * thermal sensor readings and add entropy to the pool. + * + * Interfaces -> Output + * -------------------- + * + * TEE_Result rng_get_entropy(uint32_t types, + * TEE_Param params[TEE_NUM_PARAMS]); + * + * Invoke command to expose an entropy interface to normal world. + * + * Testing + * ======= + * + * Passes FIPS 140-2 rngtest. + * + * Limitations + * =========== + * + * Output rate is limited to approx. 125 bytes per second. + * + * Our entropy estimation was not reached using any approved or + * published estimation framework such as NIST.SP.800-90B and was tested + * on a very small set of physical samples. Instead we have adopted what + * we believe to be a conservative estimate and partnered it with a + * fairly agressive health check. + * + * Generating the SHA512/256 hash takes 24uS and will be run by an + * interrupt handler that pre-empts the normal world. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "synquacer_rng_pta.h" + +#define PTA_NAME "rng.pta" + +#define THERMAL_SENSOR_BASE0 0x54190800 +#define THERMAL_SENSOR_OFFSET 0x80 +#define NUM_SENSORS 7 +#define NUM_SLOTS ((NUM_SENSORS * 2) - 1) + +#define TEMP_DATA_REG_OFFSET 0x34 + +#define ENTROPY_POOL_SIZE 4096 + +#define SENSOR_DATA_SIZE 128 +#define CONDITIONER_PAYLOAD (SENSOR_DATA_SIZE * NUM_SENSORS) + +/* + * The health test monitors each sensor's least significant bit and counts + * the number of rising and falling edges. It verifies that both counts + * lie within interval of between 12.5% and 37.5% of the samples. + * For true random data with 8 bits of entropy per byte, both counts would + * be close to 25%. + */ +#define MAX_BIT_FLIP_EDGE_COUNT ((3 * SENSOR_DATA_SIZE) / 8) +#define MIN_BIT_FLIP_EDGE_COUNT (SENSOR_DATA_SIZE / 8) + +static uint8_t entropy_pool[ENTROPY_POOL_SIZE] = {0}; +static uint32_t entropy_size; + +static uint8_t sensors_data[NUM_SLOTS][SENSOR_DATA_SIZE] = {0}; +static uint8_t sensors_data_slot_idx; +static uint8_t sensors_data_idx; + +static uint32_t health_test_fail_cnt; +static uint32_t health_test_cnt; + +static unsigned int entropy_lock = SPINLOCK_UNLOCK; + +static void pool_add_entropy(uint8_t *entropy, uint32_t size) +{ + uint32_t copy_size; + + if (entropy_size >= ENTROPY_POOL_SIZE) + return; + + if ((ENTROPY_POOL_SIZE - entropy_size) >= size) + copy_size = size; + else + copy_size = ENTROPY_POOL_SIZE - entropy_size; + + memcpy((entropy_pool + entropy_size), entropy, copy_size); + + entropy_size += copy_size; +} + +static void pool_get_entropy(uint8_t *buf, uint32_t size) +{ + uint32_t off; + + if (size > entropy_size) + return; + + off = entropy_size - size; + + memcpy(buf, &entropy_pool[off], size); + entropy_size -= size; +} + +static bool health_test(uint8_t sensor_id) +{ + uint32_t falling_edge_count = 0, rising_edge_count = 0; + uint32_t lo_edge_count, hi_edge_count; + uint32_t i; + + for (i = 0; i < (SENSOR_DATA_SIZE - 1); i++) { + if ((sensors_data[sensor_id][i] ^ + sensors_data[sensor_id][i + 1]) & 0x1) { + falling_edge_count += (sensors_data[sensor_id][i] & + 0x1); + rising_edge_count += (sensors_data[sensor_id][i + 1] & + 0x1); + } + } + + lo_edge_count = rising_edge_count < falling_edge_count ? + rising_edge_count : falling_edge_count; + hi_edge_count = rising_edge_count < falling_edge_count ? + falling_edge_count : rising_edge_count; + + return (lo_edge_count >= MIN_BIT_FLIP_EDGE_COUNT) && + (hi_edge_count <= MAX_BIT_FLIP_EDGE_COUNT); +} + +static uint8_t pool_check_add_entropy(void) +{ + uint32_t i; + uint8_t entropy_sha512_256[TEE_SHA256_HASH_SIZE]; + uint8_t pool_status = 0; + TEE_Result res; + + for (i = 0; i < NUM_SENSORS; i++) { + /* Check if particular sensor data passes health test */ + if (health_test(sensors_data_slot_idx) == true) { + sensors_data_slot_idx++; + } else { + health_test_fail_cnt++; + memmove(sensors_data[sensors_data_slot_idx], + sensors_data[sensors_data_slot_idx + 1], + (SENSOR_DATA_SIZE * (NUM_SENSORS - i - 1))); + } + } + + health_test_cnt += NUM_SENSORS; + + /* Check if sensors_data have enough pass data for conditioning */ + if (sensors_data_slot_idx >= NUM_SENSORS) { + /* + * Use vetted conditioner SHA512/256 as per + * NIST.SP.800-90B to condition raw data from entropy + * source. + */ + sensors_data_slot_idx -= NUM_SENSORS; + res = hash_sha512_256_compute(entropy_sha512_256, + sensors_data[sensors_data_slot_idx], + CONDITIONER_PAYLOAD); + if (res == TEE_SUCCESS) + pool_add_entropy(entropy_sha512_256, + TEE_SHA256_HASH_SIZE); + } + + if (entropy_size >= ENTROPY_POOL_SIZE) + pool_status = 1; + + return pool_status; +} + +void rng_collect_entropy(void) +{ + uint8_t i, pool_full = 0; + void *vaddr; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + cpu_spin_lock(&entropy_lock); + + for (i = 0; i < NUM_SENSORS; i++) { + vaddr = phys_to_virt_io(THERMAL_SENSOR_BASE0 + + (THERMAL_SENSOR_OFFSET * i) + + TEMP_DATA_REG_OFFSET, + sizeof(uint32_t)); + sensors_data[sensors_data_slot_idx + i][sensors_data_idx] = + (uint8_t)io_read32((vaddr_t)vaddr); + } + + sensors_data_idx++; + + if (sensors_data_idx >= SENSOR_DATA_SIZE) { + pool_full = pool_check_add_entropy(); + sensors_data_idx = 0; + } + + if (pool_full) + generic_timer_stop(); + + cpu_spin_unlock(&entropy_lock); + thread_set_exceptions(exceptions); +} + +static TEE_Result rng_get_entropy(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint8_t *e = NULL; + uint32_t pool_size = 0, rq_size = 0; + uint32_t exceptions; + TEE_Result res = TEE_SUCCESS; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) { + EMSG("bad parameters types: 0x%" PRIx32, types); + return TEE_ERROR_BAD_PARAMETERS; + } + + rq_size = params[0].memref.size; + + if ((rq_size == 0) || (rq_size > ENTROPY_POOL_SIZE)) + return TEE_ERROR_NOT_SUPPORTED; + + e = (uint8_t *)params[0].memref.buffer; + if (!e) + return TEE_ERROR_BAD_PARAMETERS; + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + cpu_spin_lock(&entropy_lock); + + /* + * Report health test failure to normal world in case fail count + * exceeds 1% of pass count. + */ + if (health_test_fail_cnt > ((health_test_cnt + 100) / 100)) { + res = TEE_ERROR_HEALTH_TEST_FAIL; + params[0].memref.size = 0; + health_test_cnt = 0; + health_test_fail_cnt = 0; + goto exit; + } + + pool_size = entropy_size; + + if (pool_size < rq_size) { + params[0].memref.size = pool_size; + pool_get_entropy(e, pool_size); + } else { + params[0].memref.size = rq_size; + pool_get_entropy(e, rq_size); + } + +exit: + /* Enable timer FIQ to fetch entropy */ + generic_timer_start(TIMER_PERIOD_MS); + + cpu_spin_unlock(&entropy_lock); + thread_set_exceptions(exceptions); + + return res; +} + +static TEE_Result rng_get_info(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) { + EMSG("bad parameters types: 0x%" PRIx32, types); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Output RNG rate (per second) */ + params[0].value.a = 125; + + /* + * Quality/entropy per 1024 bit of output data. As we have used + * a vetted conditioner as per NIST.SP.800-90B to provide full + * entropy given our assumption of entropy estimate for raw sensor + * data. + */ + params[0].value.b = 1024; + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *pSessionContext __unused, + uint32_t nCommandID, uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]) +{ + FMSG("command entry point for pseudo-TA \"%s\"", PTA_NAME); + + switch (nCommandID) { + case PTA_CMD_GET_ENTROPY: + return rng_get_entropy(nParamTypes, pParams); + case PTA_CMD_GET_RNG_INFO: + return rng_get_info(nParamTypes, pParams); + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_RNG_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_DEVICE_ENUM, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/arch/arm/plat-synquacer/sub.mk b/optee_os/core/arch/arm/plat-synquacer/sub.mk new file mode 100644 index 0000000..571c058 --- /dev/null +++ b/optee_os/core/arch/arm/plat-synquacer/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-y += rng_pta.c diff --git a/optee_os/core/arch/arm/plat-synquacer/synquacer_rng_pta.h b/optee_os/core/arch/arm/plat-synquacer/synquacer_rng_pta.h new file mode 100644 index 0000000..4360e5d --- /dev/null +++ b/optee_os/core/arch/arm/plat-synquacer/synquacer_rng_pta.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2018-2022, Linaro Limited + */ + +#ifndef __SYNQUACER_RNG_PTA_H +#define __SYNQUACER_RNG_PTA_H + +void rng_collect_entropy(void); + +#endif /* __SYNQUACER_RNG_PTA_H */ diff --git a/optee_os/core/arch/arm/plat-ti/a9_plat_init.S b/optee_os/core/arch/arm/plat-ti/a9_plat_init.S new file mode 100644 index 0000000..067c5d4 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/a9_plat_init.S @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew Davis + */ + +/* + * Entry points for the A9 init. + * It is assumed no stack is available when these routines are called. + * It is assumed each routine is called with return address in LR + * and with ARM registers R0, R1, R2, R3 being scratchable. + */ + +#include +#include +#include +#include +#include + +.arch_extension sec + +.section .text +.balign 4 +.code 32 + +booted: + .word 0 + +/* + * Cortex A9 check for resume + * + * Use scratables registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + */ +FUNC plat_cpu_reset_early , : + /* Check if we are resuming */ + ldr r3, =booted + ldr r2, [r3] + cmp r2, #0 + /* Cold boot, mark our boot flag and return to normal boot */ + moveq r2, #1 + streq r2, [r3] + bxeq lr + /* Otherwise we are resuming */ + b resume_springboard +END_FUNC plat_cpu_reset_early + +LOCAL_FUNC resume_springboard , : +UNWIND( .cantunwind) + /* Setup tmp stack */ + bl __get_core_pos + cmp r0, #CFG_TEE_CORE_NB_CORE + /* Unsupported CPU, park it before it breaks something */ +unhandled_cpu: + wfige + bge unhandled_cpu + add r0, r0, #1 + ldr r1, =stack_tmp_stride + ldr r1, [r1] + mul r1, r0, r1 + ldr r0, =stack_tmp +#if (STACK_TMP_GUARD != 0) + mov_imm r2, STACK_TMP_GUARD + sub r0, r0, r2 +#endif + add sp, r1, r0 + + /* Push our return on the stack as sm_pm_cpu_do_resume expects */ + adr lr, after_resume + push {r4 - r12, lr} + + /* Assumes suspend_regs is flat-mapped */ + ldr r0, =suspend_regs + bl sm_pm_cpu_do_resume + +after_resume: + bl thread_init_per_cpu + + /* r5 contains the non-secure entry address (ARMv7 bootarg #0) */ + mov r0, r5 + bl init_sec_mon + + bl boot_primary_init_intc + + mov r0, #TEESMC_OPTEED_RETURN_ENTRY_DONE + mov r1, #0 + mov r2, #0 + mov r3, #0 + mov r4, #0 + smc #0 + b . /* SMC should not return */ +END_FUNC resume_springboard diff --git a/optee_os/core/arch/arm/plat-ti/api_monitor_index_a15.h b/optee_os/core/arch/arm/plat-ti/api_monitor_index_a15.h new file mode 100644 index 0000000..ea746a6 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/api_monitor_index_a15.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew Davis + */ + +#ifndef API_MONITOR_INDEX_H +#define API_MONITOR_INDEX_H + +#define API_HAL_RET_VALUE_OK 0x00000000 +#define API_HAL_RET_VALUE_SERVICE_UNKNWON 0xFFFFFFFF + +/* Base Index of APIs */ +#define API_MONITOR_BASE_INDEX 0x00000100 + +/* HyperVisor Start */ +#define API_MONITOR_HYP_STARTHYPERVISOR_INDEX (API_MONITOR_BASE_INDEX + 0x00000002) +/* Caches cleaning */ +#define API_MONITOR_CACHES_CLEAN_INDEX (API_MONITOR_BASE_INDEX + 0x00000003) +/* Write the L2 Cache Controller Auxiliary Control */ +#define API_MONITOR_L2ACTLR_SETREGISTER_INDEX (API_MONITOR_BASE_INDEX + 0x00000004) +/* Set the Data and Tag RAM Latency */ +#define API_MONITOR_L2CACHE_SETLATENCY_INDEX (API_MONITOR_BASE_INDEX + 0x00000005) +/* L2 Cache Prefetch Control Register */ +#define API_MONITOR_L2PFR_SETREGISTER_INDEX (API_MONITOR_BASE_INDEX + 0x00000006) +/* Set Auxiliary Control Register */ +#define API_MONITOR_ACTLR_SETREGISTER_INDEX (API_MONITOR_BASE_INDEX + 0x00000007) +/* AMBA IF mode */ +#define API_MONITOR_WUGEN_MPU_SETAMBAIF_INDEX (API_MONITOR_BASE_INDEX + 0x00000008) +/* Timer CNTFRQ register set */ +#define API_MONITOR_TIMER_SETCNTFRQ_INDEX (API_MONITOR_BASE_INDEX + 0x00000009) + +#endif /* API_MONITOR_INDEX_H */ diff --git a/optee_os/core/arch/arm/plat-ti/api_monitor_index_a9.h b/optee_os/core/arch/arm/plat-ti/api_monitor_index_a9.h new file mode 100644 index 0000000..53768cb --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/api_monitor_index_a9.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew Davis + */ + +#ifndef API_MONITOR_INDEX_H +#define API_MONITOR_INDEX_H + +#define API_HAL_RET_VALUE_OK 0x00000000 +#define API_HAL_RET_VALUE_SERVICE_UNKNWON 0xFFFFFFFF + +/* Base for power management related services */ +#define SECURE_SVC_PM 0x70 + +/* Carry out late actions as part of suspend sequence */ +#define SECURE_SVC_PM_LATE_SUSPEND (SECURE_SVC_PM + 1) + +/* Base Index of APIs */ +#define API_MONITOR_BASE_INDEX 0x00000100 + +/* Set the Debug control register */ +#define API_MONITOR_L2CACHE_SETDEBUG_INDEX (API_MONITOR_BASE_INDEX + 0x00000000) +/* Clean and invalidate physical address range */ +#define API_MONITOR_L2CACHE_CLEANINVBYPA_INDEX (API_MONITOR_BASE_INDEX + 0x00000001) +/* Enables/Disables the PL310 Cache */ +#define API_MONITOR_L2CACHE_SETCONTROL_INDEX (API_MONITOR_BASE_INDEX + 0x00000002) +/* Set the Auxiliary Control Register */ +#define API_MONITOR_L2CACHE_SETAUXILIARYCONTROL_INDEX (API_MONITOR_BASE_INDEX + 0x00000009) +/* Set the Data and Tag RAM Latency */ +#define API_MONITOR_L2CACHE_SETLATENCY_INDEX (API_MONITOR_BASE_INDEX + 0x00000012) +/* Set the Pre-fetch Control Register */ +#define API_MONITOR_L2CACHE_SETPREFETCHCONTROL_INDEX (API_MONITOR_BASE_INDEX + 0x00000013) + +#endif /* API_MONITOR_INDEX_H */ diff --git a/optee_os/core/arch/arm/plat-ti/conf.mk b/optee_os/core/arch/arm/plat-ti/conf.mk new file mode 100644 index 0000000..97a41cd --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/conf.mk @@ -0,0 +1,39 @@ +PLATFORM_FLAVOR ?= dra7xx + +CFG_WITH_STATS ?= y +CFG_WITH_SOFTWARE_PRNG ?= n + +ifeq ($(PLATFORM_FLAVOR),dra7xx) +include core/arch/arm/cpu/cortex-a15.mk +$(call force,CFG_TEE_CORE_NB_CORE,2) +CFG_OTP_SUPPORT ?= y +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +endif #dra7xx + +ifeq ($(PLATFORM_FLAVOR),am57xx) +include core/arch/arm/cpu/cortex-a15.mk +$(call force,CFG_TEE_CORE_NB_CORE,2) +CFG_OTP_SUPPORT ?= y +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +endif #am57xx + +ifeq ($(PLATFORM_FLAVOR),am43xx) +include core/arch/arm/cpu/cortex-a9.mk +$(call force, CFG_TEE_CORE_NB_CORE,1) +CFG_WITH_SOFTWARE_PRNG = y +$(call force,CFG_NO_SMP,y) +$(call force,CFG_PL310,y) +$(call force,CFG_PL310_LOCKED,y) +$(call force,CFG_PM_ARM32,y) +$(call force,CFG_SECURE_TIME_SOURCE_REE,y) +endif #am43xx + +$(call force,CFG_8250_UART,y) +$(call force,CFG_ARM32_core,y) +$(call force,CFG_SM_PLATFORM_HANDLER,y) +$(call force,CFG_GIC,y) +ifneq ($(CFG_WITH_SOFTWARE_PRNG),y) +$(call force,CFG_DRA7_RNG,y) +CFG_HWRNG_QUALITY ?= 1024 +CFG_HWRNG_PTA ?= y +endif diff --git a/optee_os/core/arch/arm/plat-ti/main.c b/optee_os/core/arch/arm/plat-ti/main.c new file mode 100644 index 0000000..9d9e75d --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/main.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PLAT_HW_UNIQUE_KEY_LENGTH 32 + +static struct serial8250_uart_data console_data; +static uint8_t plat_huk[PLAT_HW_UNIQUE_KEY_LENGTH]; + +register_phys_mem(MEM_AREA_RAM_SEC, TZDRAM_BASE, TEE_RAM_VA_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SECRAM_BASE, SECRAM_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GICC_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GICD_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, + SERIAL8250_UART_REG_SIZE); + +void boot_primary_init_intc(void) +{ + gic_init(GICC_BASE, GICD_BASE); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +struct plat_nsec_ctx { + uint32_t usr_sp; + uint32_t usr_lr; + uint32_t svc_sp; + uint32_t svc_lr; + uint32_t svc_spsr; + uint32_t abt_sp; + uint32_t abt_lr; + uint32_t abt_spsr; + uint32_t und_sp; + uint32_t und_lr; + uint32_t und_spsr; + uint32_t irq_sp; + uint32_t irq_lr; + uint32_t irq_spsr; + uint32_t fiq_sp; + uint32_t fiq_lr; + uint32_t fiq_spsr; + uint32_t fiq_rx[5]; + uint32_t mon_lr; + uint32_t mon_spsr; +}; + +struct plat_boot_args { + struct plat_nsec_ctx nsec_ctx; + uint8_t huk[PLAT_HW_UNIQUE_KEY_LENGTH]; +}; + +void init_sec_mon(unsigned long nsec_entry) +{ + struct plat_boot_args *plat_boot_args; + struct sm_nsec_ctx *nsec_ctx; + + plat_boot_args = phys_to_virt(nsec_entry, MEM_AREA_IO_SEC, 1); + if (!plat_boot_args) + panic(); + + /* Invalidate cache to fetch data from external memory */ + cache_op_inner(DCACHE_AREA_INVALIDATE, + plat_boot_args, sizeof(*plat_boot_args)); + + /* Initialize secure monitor */ + nsec_ctx = sm_get_nsec_ctx(); + + nsec_ctx->ub_regs.usr_sp = plat_boot_args->nsec_ctx.usr_sp; + nsec_ctx->ub_regs.usr_lr = plat_boot_args->nsec_ctx.usr_lr; + nsec_ctx->ub_regs.irq_spsr = plat_boot_args->nsec_ctx.irq_spsr; + nsec_ctx->ub_regs.irq_sp = plat_boot_args->nsec_ctx.irq_sp; + nsec_ctx->ub_regs.irq_lr = plat_boot_args->nsec_ctx.irq_lr; + nsec_ctx->ub_regs.svc_spsr = plat_boot_args->nsec_ctx.svc_spsr; + nsec_ctx->ub_regs.svc_sp = plat_boot_args->nsec_ctx.svc_sp; + nsec_ctx->ub_regs.svc_lr = plat_boot_args->nsec_ctx.svc_lr; + nsec_ctx->ub_regs.abt_spsr = plat_boot_args->nsec_ctx.abt_spsr; + nsec_ctx->ub_regs.abt_sp = plat_boot_args->nsec_ctx.abt_sp; + nsec_ctx->ub_regs.abt_lr = plat_boot_args->nsec_ctx.abt_lr; + nsec_ctx->ub_regs.und_spsr = plat_boot_args->nsec_ctx.und_spsr; + nsec_ctx->ub_regs.und_sp = plat_boot_args->nsec_ctx.und_sp; + nsec_ctx->ub_regs.und_lr = plat_boot_args->nsec_ctx.und_lr; + nsec_ctx->mon_lr = plat_boot_args->nsec_ctx.mon_lr; + nsec_ctx->mon_spsr = plat_boot_args->nsec_ctx.mon_spsr; + + memcpy(plat_huk, plat_boot_args->huk, sizeof(plat_boot_args->huk)); +} + +void console_init(void) +{ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +#if defined(CFG_OTP_SUPPORT) + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + memcpy(&hwkey->data[0], &plat_huk[0], sizeof(hwkey->data)); + return TEE_SUCCESS; +} + +#endif diff --git a/optee_os/core/arch/arm/plat-ti/platform_config.h b/optee_os/core/arch/arm/plat-ti/platform_config.h new file mode 100644 index 0000000..73f9255 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/platform_config.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#define DRAM0_BASE 0x80000000 +#define DRAM0_SIZE 0x80000000 + +#if defined(PLATFORM_FLAVOR_dra7xx) || defined(PLATFORM_FLAVOR_am57xx) + +/* Location of protected DDR on the DRA7xx platform */ +#define TZDRAM_BASE 0xbdb00000 +#define TZDRAM_SIZE 0x01c00000 + +#ifdef CFG_WITH_PAGER +#define TZSRAM_BASE 0x40300000 +#define TZSRAM_SIZE (256 * 1024) +#endif /* CFG_WITH_PAGER */ + + +#define UART1_BASE 0x4806A000 +#define UART2_BASE 0x4806C000 +#define UART3_BASE 0x48020000 + +#if defined(PLATFORM_FLAVOR_dra7xx) +#define CONSOLE_UART_BASE UART1_BASE +#elif defined(PLATFORM_FLAVOR_am57xx) +#define CONSOLE_UART_BASE UART3_BASE +#else +#error "Unknown platform flavor" +#endif + +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 48000000 + +#define SCU_BASE 0x48210000 +#define GICC_OFFSET 0x2000 +#define GICC_SIZE 0x1000 +#define GICD_OFFSET 0x1000 +#define GICD_SIZE 0x1000 +#define GICC_BASE (SCU_BASE + GICC_OFFSET) +#define GICD_BASE (SCU_BASE + GICD_OFFSET) + +#define WUGEN_MPU_BASE 0x48281000 +#define WUGEN_MPU_SIZE 0x1000 + +#define SECRAM_BASE 0x40200000 +#define SECRAM_SIZE 0x00100000 + +/* RNG */ +#define RNG_BASE 0x48090000 + +#elif defined(PLATFORM_FLAVOR_am43xx) + +/* Location of protected DDR on the AM43xx platform */ +#define TZDRAM_BASE 0xbdb00000 +#define TZDRAM_SIZE 0x01c00000 + +#define UART0_BASE 0x44E09000 +#define UART1_BASE 0x48022000 +#define UART2_BASE 0x48024000 +#define UART3_BASE 0x481A6000 +#define UART4_BASE 0x481A8000 +#define UART5_BASE 0x481AA000 + +#define CONSOLE_UART_BASE UART0_BASE +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 48000000 + +#define SCU_BASE 0x48240000 +#define GICD_OFFSET 0x1000 +#define GICD_SIZE 0x1000 +#define GICC_OFFSET 0x0100 +#define GICC_SIZE 0x0100 +#define PL310_OFFSET 0x2000 +#define PL310_SIZE 0x1000 +#define GICD_BASE (SCU_BASE + GICD_OFFSET) +#define GICC_BASE (SCU_BASE + GICC_OFFSET) +#define PL310_BASE (SCU_BASE + PL310_OFFSET) + +#define SECRAM_BASE 0x402F0000 +#define SECRAM_SIZE 0x00100000 + +/* RNG */ +#define RNG_BASE 0x48310000 + +#else +#error "Unknown platform flavor" +#endif + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#ifdef CFG_WITH_PAGER +/* + * Use TZSRAM for TEE, page out everything else to TZDRAM. + * +--------+----------+ + * | DRAM | SHMEM | + * +--------+----------+ + * | | TA_RAM | + * | TZDRAM +----------+ + * | | PAGE_RAM | + * +--------+----------+ + * | TZSRAM | TEE_RAM | + * +--------+----------+ + */ +#define TEE_RAM_VA_SIZE (1 * 1024 * 1024) +#define TEE_RAM_PH_SIZE TZSRAM_SIZE +#define TEE_RAM_START TZSRAM_BASE +#define TEE_LOAD_ADDR (TEE_RAM_START + 0x1000) + +#else /* CFG_WITH_PAGER */ +/* + * Assumes that either TZSRAM isn't large enough or TZSRAM doesn't exist, + * everything is in TZDRAM. + * +--------+---------+ + * | DRAM | SHMEM | + * +--------+---------+ + * | | TA_RAM | + * | TZDRAM +---------+ + * | | TEE_RAM | + * +--------+---------+ + */ +#define TEE_RAM_VA_SIZE (1 * 1024 * 1024) +#define TEE_RAM_PH_SIZE TEE_RAM_VA_SIZE +#define TEE_RAM_START TZDRAM_BASE +#define TEE_LOAD_ADDR TEE_RAM_START + +#endif /* CFG_WITH_PAGER */ + +#define TA_RAM_START ROUNDUP((TZDRAM_BASE + TEE_RAM_VA_SIZE), \ + CORE_MMU_PGDIR_SIZE) + +#define TA_RAM_SIZE ROUNDDOWN((TZDRAM_SIZE - TEE_RAM_VA_SIZE), \ + CORE_MMU_PGDIR_SIZE) + +/* Full GlobalPlatform test suite requires TEE_SHMEM_SIZE to be at least 2MB */ +#define TEE_SHMEM_START (TZDRAM_BASE + TZDRAM_SIZE) +#define TEE_SHMEM_SIZE (4 * 1024 * 1024) + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-ti/sm_platform_handler_a15.c b/optee_os/core/arch/arm/plat-ti/sm_platform_handler_a15.c new file mode 100644 index 0000000..8cd2a34 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/sm_platform_handler_a15.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew Davis + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "api_monitor_index_a15.h" + +#define WUGEN_MPU_AMBA_IF_MODE 0x80c + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, WUGEN_MPU_BASE, WUGEN_MPU_SIZE); + +static vaddr_t wugen_mpu_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(WUGEN_MPU_BASE, MEM_AREA_IO_SEC, + WUGEN_MPU_SIZE); + return (vaddr_t)va; + } + + return WUGEN_MPU_BASE; +} + +static void write_wugen_mpu_amba_if_mode(uint32_t val) +{ + io_write32(wugen_mpu_base() + WUGEN_MPU_AMBA_IF_MODE, val); +} + +static enum sm_handler_ret ti_sip_handler(struct thread_smc_args *smc_args) +{ + uint16_t sip_func = OPTEE_SMC_FUNC_NUM(smc_args->a0); + + switch (sip_func) { + case API_MONITOR_ACTLR_SETREGISTER_INDEX: + write_actlr(smc_args->a1); + isb(); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_TIMER_SETCNTFRQ_INDEX: + write_cntfrq(smc_args->a1); + isb(); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_WUGEN_MPU_SETAMBAIF_INDEX: + write_wugen_mpu_amba_if_mode(smc_args->a1); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + default: + EMSG("Invalid SIP function code: 0x%04"PRIx16, sip_func); + smc_args->a0 = OPTEE_SMC_RETURN_EBADCMD; + break; + } + + return SM_HANDLER_SMC_HANDLED; +} + +enum sm_handler_ret sm_platform_handler(struct sm_ctx *ctx) +{ + uint32_t *nsec_r0 = (uint32_t *)(&ctx->nsec.r0); + uint16_t smc_owner = OPTEE_SMC_OWNER_NUM(*nsec_r0); + + switch (smc_owner) { + case OPTEE_SMC_OWNER_SIP: + return ti_sip_handler((struct thread_smc_args *)nsec_r0); + default: + return SM_HANDLER_PENDING_SMC; + } +} diff --git a/optee_os/core/arch/arm/plat-ti/sm_platform_handler_a9.c b/optee_os/core/arch/arm/plat-ti/sm_platform_handler_a9.c new file mode 100644 index 0000000..8d628d1 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/sm_platform_handler_a9.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew Davis + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "api_monitor_index_a9.h" + +uint32_t suspend_regs[16]; + +static enum sm_handler_ret ti_sip_handler(struct thread_smc_args *smc_args) +{ + uint16_t sip_func = OPTEE_SMC_FUNC_NUM(smc_args->a0); + + switch (sip_func) { + case SECURE_SVC_PM_LATE_SUSPEND: + sm_pm_cpu_do_suspend(suspend_regs); + cache_op_inner(DCACHE_AREA_CLEAN, suspend_regs, + sizeof(suspend_regs)); + cache_op_outer(DCACHE_AREA_CLEAN, virt_to_phys(suspend_regs), + sizeof(suspend_regs)); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_L2CACHE_SETDEBUG_INDEX: + io_write32(pl310_base() + PL310_DEBUG_CTRL, smc_args->a1); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_L2CACHE_CLEANINVBYPA_INDEX: + arm_cl2_cleaninvbypa(pl310_base(), smc_args->a1, + smc_args->a1 + smc_args->a2); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_L2CACHE_SETCONTROL_INDEX: + io_write32(pl310_base() + PL310_CTRL, smc_args->a1); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_L2CACHE_SETAUXILIARYCONTROL_INDEX: + io_write32(pl310_base() + PL310_AUX_CTRL, smc_args->a1); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_L2CACHE_SETLATENCY_INDEX: + io_write32(pl310_base() + PL310_TAG_RAM_CTRL, smc_args->a1); + io_write32(pl310_base() + PL310_DATA_RAM_CTRL, smc_args->a2); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + case API_MONITOR_L2CACHE_SETPREFETCHCONTROL_INDEX: + io_write32(pl310_base() + PL310_PREFETCH_CTRL, smc_args->a1); + smc_args->a0 = OPTEE_SMC_RETURN_OK; + break; + default: + EMSG("Invalid SIP function code: 0x%04"PRIx16, sip_func); + smc_args->a0 = OPTEE_SMC_RETURN_EBADCMD; + break; + } + + return SM_HANDLER_SMC_HANDLED; +} + +enum sm_handler_ret sm_platform_handler(struct sm_ctx *ctx) +{ + uint32_t *nsec_r0 = (uint32_t *)(&ctx->nsec.r0); + uint16_t smc_owner = OPTEE_SMC_OWNER_NUM(*nsec_r0); + + switch (smc_owner) { + case OPTEE_SMC_OWNER_SIP: + return ti_sip_handler((struct thread_smc_args *)nsec_r0); + default: + return SM_HANDLER_PENDING_SMC; + } +} diff --git a/optee_os/core/arch/arm/plat-ti/sub.mk b/optee_os/core/arch/arm/plat-ti/sub.mk new file mode 100644 index 0000000..0b4c385 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/sub.mk @@ -0,0 +1,6 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-$(CFG_PL310) += ti_pl310.c +srcs-$(PLATFORM_FLAVOR_dra7xx) += sm_platform_handler_a15.c +srcs-$(PLATFORM_FLAVOR_am57xx) += sm_platform_handler_a15.c +srcs-$(PLATFORM_FLAVOR_am43xx) += sm_platform_handler_a9.c a9_plat_init.S diff --git a/optee_os/core/arch/arm/plat-ti/ti_pl310.c b/optee_os/core/arch/arm/plat-ti/ti_pl310.c new file mode 100644 index 0000000..f4a3dd6 --- /dev/null +++ b/optee_os/core/arch/arm/plat-ti/ti_pl310.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew Davis + */ + +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, PL310_BASE, PL310_SIZE); + +vaddr_t pl310_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(PL310_BASE, MEM_AREA_IO_SEC, + PL310_SIZE); + return (vaddr_t)va; + } + + return PL310_BASE; +} + +/* ROM handles initial setup for us */ +void arm_cl2_config(vaddr_t pl310_base) +{ + (void)pl310_base; +} + +/* We provide platform services that expect the cache to be disabled on boot */ +void arm_cl2_enable(vaddr_t pl310_base) +{ + (void)pl310_base; +} diff --git a/optee_os/core/arch/arm/plat-totalcompute/conf.mk b/optee_os/core/arch/arm/plat-totalcompute/conf.mk new file mode 100644 index 0000000..2f6c0ee --- /dev/null +++ b/optee_os/core/arch/arm/plat-totalcompute/conf.mk @@ -0,0 +1,45 @@ +ifneq (,$(filter ${PLATFORM_FLAVOR},tc0 tc1 tc2)) +include core/arch/arm/cpu/cortex-armv8-0.mk +platform-debugger-arm := 1 +endif + +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_GENERIC_BOOT,y) +ifeq ($(CFG_CORE_SEL2_SPMC),y) +$(call force,CFG_GIC,n) +$(call force,CFG_ARM_GICV3,n) +else +$(call force,CFG_GIC,y) +$(call force,CFG_ARM_GICV3,y) +endif +$(call force,CFG_PL011,y) +$(call force,CFG_PM_STUBS,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_ARM64_core,y) + +ifeq ($(platform-debugger-arm),1) +# ARM debugger needs this +platform-cflags-debug-info = -gdwarf-2 +platform-aflags-debug-info = -gdwarf-2 +endif + +$(call force,CFG_CORE_ARM64_PA_BITS,40) + +ifneq (,$(filter ${PLATFORM_FLAVOR},tc0 tc1 tc2)) +CFG_TEE_CORE_NB_CORE = 8 + +ifeq ($(CFG_CORE_SEL1_SPMC),y) +CFG_TZDRAM_START ?= 0xfd000000 +CFG_TZDRAM_SIZE ?= 0x02000000 +else ifeq ($(CFG_CORE_SEL2_SPMC),y) +CFG_TZDRAM_START ?= 0xfd284000 +# TZDRAM size 0x1980000 - 0x4000 manifest size +CFG_TZDRAM_SIZE ?= 0x0197c000 +else +CFG_TZDRAM_START ?= 0xff000000 +CFG_TZDRAM_SIZE ?= 0x01000000 +endif + +CFG_SHMEM_START ?= 0xfce00000 +CFG_SHMEM_SIZE ?= 0x00200000 +endif diff --git a/optee_os/core/arch/arm/plat-totalcompute/fdts/optee_sp_manifest.dts b/optee_os/core/arch/arm/plat-totalcompute/fdts/optee_sp_manifest.dts new file mode 100644 index 0000000..9464fb1 --- /dev/null +++ b/optee_os/core/arch/arm/plat-totalcompute/fdts/optee_sp_manifest.dts @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * This file is a Partition Manifest (PM) for OP-TEE as a Secure Partition (SP) + * + */ + +/dts-v1/; + +/ { + compatible = "arm,ffa-manifest-1.0"; + + /* Properties */ + description = "op-tee"; + ffa-version = <0x00010000>; /* 31:16 - Major, 15:0 - Minor */ + uuid = <0xe0786148 0xe311f8e7 0x02005ebc 0x1bc5d5a5>; + id = <1>; + execution-ctx-count = <8>; + exception-level = <2>; /* S-EL1 */ + execution-state = <0>; /* AARCH64 */ + load-address = <0xfd280000>; + entrypoint-offset = <0x4000>; + xlat-granule = <0>; /* 4KiB */ + boot-order = <0>; + messaging-method = <0x3>; /* Direct request/response supported */ + + device-regions { + compatible = "arm,ffa-manifest-device-regions"; + ap_s_uart { + base-address = <0x00000000 0x2A410000>; + pages-count = <1>; + attributes = <0x3>; /* read-write */ + }; + }; +}; diff --git a/optee_os/core/arch/arm/plat-totalcompute/main.c b/optee_os/core/arch/arm/plat-totalcompute/main.c new file mode 100644 index 0000000..c3445c0 --- /dev/null +++ b/optee_os/core/arch/arm/plat-totalcompute/main.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Arm Limited. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static struct pl011_data console_data __nex_bss; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, PL011_REG_SIZE); +#ifndef CFG_CORE_SEL2_SPMC +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); +#endif + +register_ddr(DRAM0_BASE, DRAM0_SIZE); +register_ddr(DRAM1_BASE, DRAM1_SIZE); + +#ifndef CFG_CORE_SEL2_SPMC +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICC_OFFSET); +} +#endif + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, CONSOLE_UART_CLK_IN_HZ, + CONSOLE_UART_BAUDRATE); + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-totalcompute/platform_config.h b/optee_os/core/arch/arm/plat-totalcompute/platform_config.h new file mode 100644 index 0000000..2b34cd0 --- /dev/null +++ b/optee_os/core/arch/arm/plat-totalcompute/platform_config.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#if (defined(PLATFORM_FLAVOR_tc0) || \ + defined(PLATFORM_FLAVOR_tc1) || \ + defined(PLATFORM_FLAVOR_tc2)) +#ifndef CFG_CORE_SEL2_SPMC +#define GIC_BASE 0x30000000 +#define GICD_OFFSET 0x0 +#define GICC_OFFSET 0x0 +#endif + +#define UART0_BASE 0x2A410000 +#define UART1_BASE 0x2A400000 + +#define CONSOLE_UART_BASE UART0_BASE + +#define DRAM0_BASE 0x80000000 +#define DRAM0_SIZE 0x7d000000 + +#define DRAM1_BASE 0x8080000000ULL +#define DRAM1_SIZE 0x180000000ULL + +#define TZCDRAM_BASE 0xff000000 +#define TZCDRAM_SIZE 0x01000000 + +#else +#error "Unknown platform flavor" +#endif + +#ifdef GIC_BASE +#define GICD_BASE (GIC_BASE + GICD_OFFSET) +#define GICC_BASE (GIC_BASE + GICC_OFFSET) +#endif + +#define CONSOLE_UART_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 7372800 + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-totalcompute/sub.mk b/optee_os/core/arch/arm/plat-totalcompute/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-totalcompute/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-uniphier/conf.mk b/optee_os/core/arch/arm/plat-uniphier/conf.mk new file mode 100644 index 0000000..59e47fb --- /dev/null +++ b/optee_os/core/arch/arm/plat-uniphier/conf.mk @@ -0,0 +1,46 @@ +PLATFORM_FLAVOR ?= ld20 + +include core/arch/arm/cpu/cortex-armv8-0.mk + +ifeq ($(PLATFORM_FLAVOR),ld20) +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +CFG_DRAM0_BASE ?= 0x80000000 +CFG_DRAM0_SIZE ?= 0xc0000000 +CFG_DRAM0_RSV_SIZE ?= 0x02000000 +endif + +ifeq ($(PLATFORM_FLAVOR),ld11) +$(call force,CFG_TEE_CORE_NB_CORE,2) +CFG_DRAM0_BASE ?= 0x80000000 +CFG_DRAM0_SIZE ?= 0x40000000 +CFG_DRAM0_RSV_SIZE ?= 0x02000000 +endif + +CFG_TZDRAM_START ?= (CFG_DRAM0_BASE + 0x01080000) +CFG_TZDRAM_SIZE ?= 0x00E00000 +CFG_SHMEM_START ?= (CFG_DRAM0_BASE + 0x00E00000) +CFG_SHMEM_SIZE ?= 0x00200000 +CFG_TEE_RAM_VA_SIZE ?= 0x00100000 + +# 32-bit flags +core_arm32-platform-aflags += -mfpu=neon + +$(call force,CFG_HWSUPP_MEM_PERM_PXN,y) +$(call force,CFG_GIC,y) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_8250_UART,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_CORE_CLUSTER_SHIFT,1) + +ta-targets = ta_arm32 + +ifeq ($(CFG_ARM64_core),y) +ta-targets += ta_arm64 +else +$(call force,CFG_ARM32_core,y) +endif + +CFG_NUM_THREADS ?= 4 +CFG_CRYPTO_WITH_CE ?= y diff --git a/optee_os/core/arch/arm/plat-uniphier/kern.ld.S b/optee_os/core/arch/arm/plat-uniphier/kern.ld.S new file mode 100644 index 0000000..f031e62 --- /dev/null +++ b/optee_os/core/arch/arm/plat-uniphier/kern.ld.S @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: (BSD-2-Clause AND MIT) */ +#include "../kernel/kern.ld.S" diff --git a/optee_os/core/arch/arm/plat-uniphier/main.c b/optee_os/core/arch/arm/plat-uniphier/main.c new file mode 100644 index 0000000..080dd83 --- /dev/null +++ b/optee_os/core/arch/arm/plat-uniphier/main.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2017, Socionext Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(CONSOLE_UART_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(GIC_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(GIC_BASE + GICD_OFFSET, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +#ifdef DRAM0_BASE +register_ddr(DRAM0_BASE, DRAM0_SIZE); +#endif +#ifdef DRAM1_BASE +register_ddr(DRAM1_BASE, DRAM1_SIZE); +#endif + +static struct serial8250_uart_data console_data; + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + +void console_init(void) +{ + /* Init UART */ + serial8250_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + + /* Register console */ + register_serial_console(&console_data.chip); +} diff --git a/optee_os/core/arch/arm/plat-uniphier/platform_config.h b/optee_os/core/arch/arm/plat-uniphier/platform_config.h new file mode 100644 index 0000000..10f3476 --- /dev/null +++ b/optee_os/core/arch/arm/plat-uniphier/platform_config.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2017, Socionext Inc. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +/* GIC */ +#define GIC_BASE 0x5FE00000 +#define GICD_OFFSET 0 +#define GICC_OFFSET 0x80000 + +/* UART */ +#define UART_CH 0 +#define UART_BASE 0x54006800 +#define CONSOLE_UART_BASE (UART_BASE + 0x100 * UART_CH) +#define CONSOLE_BAUDRATE 115200 +#define CONSOLE_UART_CLK_IN_HZ 58820000 + +/* + * UniPhier memory map + * + * 0xXXXX_XXXX + * Linux kernel and user space | DRAM#0-#x | Normal memory + * 0x8200_0000 [DRAM0_BASE] - - + * unused | | + * 0x81E8_0000 | | + * TA RAM: 13 MiB | TZDRAM | + * 0x8118_0000 | | Secure memory + * TEE RAM: 1 MiB (CFG_TEE_RAM_VA_SIZE) | | + * 0x8108_0000 [CFG_TZDRAM_START] - | + * BL31 runtime: 512 KiB | | + * 0x8100_0000 | - + * Shared memory: 2 MiB (CFG_SHMEM_SIZE) | | + * 0x80E0_0000 [CFG_SHMEM_START] | DRAM#0 | Normal memory + * reserved | | + * 0x8008_0000 | | + * BL2: 512 KiB | | + * 0x8000_0000 [CFG_DRAM0_BASE] - - + */ + +#define DRAM0_BASE (CFG_DRAM0_BASE + CFG_DRAM0_RSV_SIZE) +#define DRAM0_SIZE (CFG_DRAM0_SIZE - CFG_DRAM0_RSV_SIZE) + +#define CFG_TEE_LOAD_ADDR CFG_TZDRAM_START + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-uniphier/sub.mk b/optee_os/core/arch/arm/plat-uniphier/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-uniphier/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-versal/conf.mk b/optee_os/core/arch/arm/plat-versal/conf.mk new file mode 100644 index 0000000..645462d --- /dev/null +++ b/optee_os/core/arch/arm/plat-versal/conf.mk @@ -0,0 +1,88 @@ +PLATFORM_FLAVOR ?= generic + +include core/arch/arm/cpu/cortex-armv8-0.mk + +CFG_MMAP_REGIONS ?= 24 + +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_ARM_GICV3,y) +$(call force,CFG_PL011,y) +$(call force,CFG_GIC,y) + +# Disable core ASLR for two reasons: +# 1. There is no source for ALSR seed, as TF-a does not provide a +# DTB to OP-TEE. Hardware RNG is also not currently supported. +# 2. Xilinx's bootgen can't find the OP-TEE entry point from the TEE.elf file +# used to generate boot.bin. Enabling ASLR requires an update to TF-A. +$(call force,CFG_CORE_ASLR,n) + +CFG_CRYPTO_WITH_CE ?= y +CFG_CORE_DYN_SHM ?= y +CFG_WITH_STATS ?= y +CFG_ARM64_core ?= y + +CFG_TZDRAM_START ?= 0x60000000 +CFG_TZDRAM_SIZE ?= 0x10000000 +CFG_SHMEM_START ?= 0x70000000 +CFG_SHMEM_SIZE ?= 0x10000000 + +ifeq ($(CFG_ARM64_core),y) +$(call force,CFG_CORE_ARM64_PA_BITS,43) +else +$(call force,CFG_ARM32_core,y) +endif + +# GPIO +CFG_VERSAL_GPIO ?= y + +# Debug information +CFG_VERSAL_TRACE_MBOX ?= n +CFG_VERSAL_TRACE_PLM ?= n + +$(call force, CFG_VERSAL_MBOX,y) + +# MBOX configuration +CFG_VERSAL_MBOX_IPI_ID ?= 3 + +$(call force, CFG_VERSAL_RNG_DRV,y) +$(call force, CFG_WITH_SOFTWARE_PRNG,n) + +# TRNG configuration +CFG_VERSAL_TRNG_SEED_LIFE ?= 3 +CFG_VERSAL_TRNG_DF_MUL ?= 2 + +# eFuse and BBRAM driver +$(call force, CFG_VERSAL_NVM,y) + +# Crypto driver +CFG_VERSAL_CRYPTO_DRIVER ?= y +ifeq ($(CFG_VERSAL_CRYPTO_DRIVER),y) +# Disable Fault Mitigation: triggers false positives due to +# the driver's software fallback operations - need further work +CFG_FAULT_MITIGATION ?= n +endif + +# SHA3-384 crypto engine +CFG_VERSAL_SHA3_384 ?= y + +# PM driver +CFG_VERSAL_PM ?= y + +# Physical Unclonable Function +CFG_VERSAL_PUF ?= y + +# Enable Hardware Unique Key driver +CFG_VERSAL_HUK ?= y +# AES-GCM supported key sources for HUK: +# 6 : eFUSE USR 0 +# 7 : eFuse USR 1 +# 11 : PUF KEK +# 12 : AES User Key 0 (devel) +CFG_VERSAL_HUK_KEY ?= 12 +ifneq ($(CFG_VERSAL_HUK_KEY),$(filter 6 7 11 12,$(firstword $(CFG_VERSAL_HUK_KEY)))) +$(error Invalid value: CFG_VERSAL_HUK_KEY=$(CFG_VERSAL_HUK_KEY)) +endif + +CFG_CORE_HEAP_SIZE ?= 262144 diff --git a/optee_os/core/arch/arm/plat-versal/main.c b/optee_os/core/arch/arm/plat-versal/main.c new file mode 100644 index 0000000..37623cb --- /dev/null +++ b/optee_os/core/arch/arm/plat-versal/main.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022 - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define VERSAL_AHWROT_SECURED 0xA5A5A5A5 +#define VERSAL_SHWROT_SECURED 0x96969696 +#define VERSAL_AHWROT_REG 0x14C +#define VERSAL_SHWROT_REG 0x150 + +static struct pl011_data console_data; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(CONSOLE_UART_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + GIC_BASE, CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + GIC_BASE + GICD_OFFSET, CORE_MMU_PGDIR_SIZE); + +register_phys_mem(MEM_AREA_IO_SEC, PLM_RTCA, PLM_RTCA_LEN); + +register_ddr(DRAM0_BASE, DRAM0_SIZE); + +#if defined(DRAM1_BASE) +register_ddr(DRAM1_BASE, DRAM1_SIZE); +register_ddr(DRAM2_BASE, DRAM2_SIZE); +#endif + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +static TEE_Result platform_banner(void) +{ + vaddr_t plm_rtca = (vaddr_t)phys_to_virt(PLM_RTCA, MEM_AREA_IO_SEC, + PLM_RTCA_LEN); + const char __maybe_unused *ahwrot_str = "OFF"; + const char __maybe_unused *shwrot_str = "OFF"; + uint8_t version = 0; + + assert(plm_rtca); + + if (versal_soc_version(&version)) { + EMSG("Failure to retrieve SoC version"); + return TEE_ERROR_GENERIC; + } + + IMSG("Platform Versal:\tSilicon Revision v%"PRIu8, version); + + if (io_read32(plm_rtca + VERSAL_AHWROT_REG) == VERSAL_AHWROT_SECURED) + ahwrot_str = "ON"; + + if (io_read32(plm_rtca + VERSAL_SHWROT_REG) == VERSAL_SHWROT_SECURED) + shwrot_str = "ON"; + + IMSG("Hardware Root of Trust: Asymmetric[%s], Symmetric[%s]", + ahwrot_str, shwrot_str); + + return TEE_SUCCESS; +} + +#if defined(CFG_RPMB_FS) +bool plat_rpmb_key_is_ready(void) +{ + vaddr_t plm_rtca = (vaddr_t)phys_to_virt(PLM_RTCA, MEM_AREA_IO_SEC, + PLM_RTCA_LEN); + + assert(plm_rtca); + + if (io_read32(plm_rtca + VERSAL_AHWROT_REG) == VERSAL_AHWROT_SECURED) + return true; + + if (io_read32(plm_rtca + VERSAL_SHWROT_REG) == VERSAL_SHWROT_SECURED) + return true; + + return false; +} +#endif + +service_init(platform_banner); + +#if defined(CFG_VERSAL_FPGA_DDR_ADDR) +static TEE_Result program_fpga(void) +{ + return versal_write_fpga(CFG_VERSAL_FPGA_DDR_ADDR); +} + +service_init(program_fpga); +#endif diff --git a/optee_os/core/arch/arm/plat-versal/platform_config.h b/optee_os/core/arch/arm/plat-versal/platform_config.h new file mode 100644 index 0000000..8f9b367 --- /dev/null +++ b/optee_os/core/arch/arm/plat-versal/platform_config.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2022 - All Rights Reserved + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define CACHELINE_LEN 64 +#define STACK_ALIGNMENT CACHELINE_LEN + +#if defined(PLATFORM_FLAVOR_generic) + +#define PLM_RTCA 0xF2014000 +#define PLM_RTCA_LEN 0x1000 + +#define GIC_BASE 0xF9000000 +#define UART0_BASE 0xFF000000 +#define UART1_BASE 0xFF010000 + +#define IT_UART0 50 +#define IT_UART1 51 + +#define UART0_CLK_IN_HZ 100000000 +#define UART1_CLK_IN_HZ 100000000 +#define CONSOLE_UART_BASE UART0_BASE +#define IT_CONSOLE_UART IT_UART0 +#define CONSOLE_UART_CLK_IN_HZ UART0_CLK_IN_HZ + +#define DRAM0_BASE 0 +#define DRAM0_SIZE 0x80000000 + +#ifdef ARM64 +/* DDR High area base is only available when compiling for 64 bits */ +#define DRAM1_BASE 0x800000000 +#define DRAM1_SIZE 0x180000000 +#define DRAM2_BASE 0x50000000000 +#define DRAM2_SIZE 0x200000000 +#endif + +#define GICD_OFFSET 0 +#define GICC_OFFSET 0x40000 + +#else +#error "Unknown platform flavor" +#endif + +#ifdef CFG_TEE_LOAD_ADDR +#define TEE_LOAD_ADDR CFG_TEE_LOAD_ADDR +#else +#define TEE_LOAD_ADDR TEE_RAM_START +#endif + +#ifndef UART_BAUDRATE +#define UART_BAUDRATE 115200 +#endif +#ifndef CONSOLE_BAUDRATE +#define CONSOLE_BAUDRATE UART_BAUDRATE +#endif + +#endif /* PLATFORM_CONFIG_H */ diff --git a/optee_os/core/arch/arm/plat-versal/sub.mk b/optee_os/core/arch/arm/plat-versal/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-versal/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/plat-vexpress/conf.mk b/optee_os/core/arch/arm/plat-vexpress/conf.mk new file mode 100644 index 0000000..94a4e62 --- /dev/null +++ b/optee_os/core/arch/arm/plat-vexpress/conf.mk @@ -0,0 +1,156 @@ +PLATFORM_FLAVOR ?= qemu_virt + +ifeq ($(PLATFORM_FLAVOR),qemu_virt) +include core/arch/arm/cpu/cortex-a15.mk +endif +ifeq ($(PLATFORM_FLAVOR),fvp) +include core/arch/arm/cpu/cortex-armv8-0.mk +platform-debugger-arm := 1 +endif +ifeq ($(PLATFORM_FLAVOR),juno) +include core/arch/arm/cpu/cortex-armv8-0.mk +platform-debugger-arm := 1 +# Workaround 808870: Unconditional VLDM instructions might cause an +# alignment fault even though the address is aligned +# Either hard float must be disabled for AArch32 or strict alignment checks +# must be disabled +ifeq ($(CFG_SCTLR_ALIGNMENT_CHECK),y) +$(call force,CFG_TA_ARM32_NO_HARD_FLOAT_SUPPORT,y) +else +$(call force,CFG_SCTLR_ALIGNMENT_CHECK,n) +endif +endif #juno +ifeq ($(PLATFORM_FLAVOR),qemu_armv8a) +include core/arch/arm/cpu/cortex-armv8-0.mk +CFG_ARM64_core ?= y +supported-ta-targets ?= ta_arm64 ta_arm32 +endif + + +ifeq ($(platform-debugger-arm),1) +# ARM debugger needs this +platform-cflags-debug-info = -gdwarf-2 +platform-aflags-debug-info = -gdwarf-2 +endif + +ifeq ($(platform-flavor-armv8),1) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) +endif + +$(call force,CFG_PL011,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) + +ifeq ($(CFG_CORE_TPM_EVENT_LOG),y) +# NOTE: Below values for the TPM event log are implementation +# dependent and used mostly for debugging purposes. +# Care must be taken to properly configure them if used. +CFG_TPM_LOG_BASE_ADDR ?= 0x402c951 +CFG_TPM_MAX_LOG_SIZE ?= 0x200 +endif + +ifneq ($(CFG_ARM64_core),y) +$(call force,CFG_ARM32_core,y) +endif + +CFG_WITH_STATS ?= y +CFG_ENABLE_EMBEDDED_TESTS ?= y + +ifeq ($(CFG_CORE_SEL2_SPMC),y) +$(call force,CFG_CORE_RESERVED_SHM,n) +CFG_GIC ?= n +else +$(call force,CFG_GIC,y) +endif + +ifeq ($(PLATFORM_FLAVOR),fvp) +CFG_TEE_CORE_NB_CORE = 8 +ifeq ($(CFG_CORE_SEL2_SPMC),y) +CFG_TZDRAM_START ?= 0x06281000 +CFG_TZDRAM_SIZE ?= 0x01D80000 +else +CFG_TZDRAM_START ?= 0x06000000 +CFG_TZDRAM_SIZE ?= 0x02000000 +endif +CFG_SHMEM_START ?= 0x83000000 +CFG_SHMEM_SIZE ?= 0x00200000 +# DRAM1 is defined above 4G +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +ifeq ($(CFG_SCMI_SCPFW),y) +$(call force,CFG_SCMI_SCPFW_PRODUCT,optee-fvp) +endif +endif + +ifeq ($(PLATFORM_FLAVOR),juno) +CFG_TEE_CORE_NB_CORE = 6 +CFG_TZDRAM_START ?= 0xff000000 +CFG_TZDRAM_SIZE ?= 0x00ff8000 +CFG_SHMEM_START ?= 0xfee00000 +CFG_SHMEM_SIZE ?= 0x00200000 +# DRAM1 is defined above 4G +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_CORE_ARM64_PA_BITS,36) +CFG_CRYPTO_WITH_CE ?= y +CFG_ARM_SMCCC_TRNG ?= y +CFG_WITH_SOFTWARE_PRNG ?= n +endif + +ifeq ($(PLATFORM_FLAVOR),qemu_virt) +CFG_TEE_CORE_NB_CORE = 4 +# [0e00.0000 0e0f.ffff] is reserved to early boot +CFG_TZDRAM_START ?= 0x0e100000 +CFG_TZDRAM_SIZE ?= 0x00f00000 +CFG_SHMEM_START ?= 0x7fe00000 +CFG_SHMEM_SIZE ?= 0x00200000 +# When Secure Data Path is enable, last MByte of TZDRAM is SDP test memory. +CFG_TEE_SDP_MEM_SIZE ?= 0x00400000 +# Set VA space to 2MB for Kasan offset to match LPAE and 32bit MMU configs +CFG_TEE_RAM_VA_SIZE ?= 0x00200000 +ifeq ($(CFG_CORE_SANITIZE_KADDRESS),y) +# CFG_ASAN_SHADOW_OFFSET is calculated as: +# (&__asan_shadow_start - (TEE_RAM_VA_START / 8) +# This is unfortunately currently not possible to do in make so we have to +# calculate it offline, there's some asserts in +# core/arch/arm/kernel/generic_boot.c to check that we got it right +CFG_ASAN_SHADOW_OFFSET = 0xc6a71c0 +endif +$(call force,CFG_BOOT_SECONDARY_REQUEST,y) +$(call force,CFG_PSCI_ARM32,y) +$(call force,CFG_DT,y) +CFG_DTB_MAX_SIZE ?= 0x100000 +CFG_CORE_ASYNC_NOTIF ?= y +CFG_CORE_ASYNC_NOTIF_GIC_INTID ?= 219 +endif + +ifeq ($(PLATFORM_FLAVOR),qemu_armv8a) +CFG_TEE_CORE_NB_CORE = 4 +ifneq ($(CFG_CORE_SEL2_SPMC),y) +# [0e00.0000 0e0f.ffff] is reserved to early boot +CFG_TZDRAM_START ?= 0x0e100000 +CFG_TZDRAM_SIZE ?= 0x00f00000 +# SHM chosen arbitrary, in a way that it does not interfere +# with initial location of linux kernel, dtb and initrd. +CFG_SHMEM_START ?= 0x42000000 +CFG_SHMEM_SIZE ?= 0x00200000 +# When Secure Data Path is enable, last MByte of TZDRAM is SDP test memory. +CFG_TEE_SDP_MEM_SIZE ?= 0x00400000 +ifeq ($(CFG_CORE_SANITIZE_KADDRESS),y) +# See comment above +CFG_ASAN_SHADOW_OFFSET = 0xc6a71c0 +endif +endif +$(call force,CFG_DT,y) +CFG_DTB_MAX_SIZE ?= 0x100000 +ifeq ($(CFG_SCMI_SCPFW),y) +$(call force,CFG_SCMI_SCPFW_PRODUCT,optee-fvp) +endif +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),qemu_virt qemu_armv8a)) +CFG_DT_DRIVER_EMBEDDED_TEST ?= y +ifeq ($(CFG_DT_DRIVER_EMBEDDED_TEST),y) +$(call force,CFG_EMBED_DTB_SOURCE_FILE,embedded_dtb_test.dts,Mandated for DT tests) +endif +endif + +CFG_PKCS11_TA ?= y diff --git a/optee_os/core/arch/arm/plat-vexpress/juno_core_pos_a32.S b/optee_os/core/arch/arm/plat-vexpress/juno_core_pos_a32.S new file mode 100644 index 0000000..2887172 --- /dev/null +++ b/optee_os/core/arch/arm/plat-vexpress/juno_core_pos_a32.S @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include + +/* For Juno number the two A57s as 4 to 5 and A53s as 0 to 3 */ +FUNC get_core_pos_mpidr , : + /* Calculate CorePos = ((ClusterId ^ 1) * 4) + CoreId */ + and r1, r0, #MPIDR_CPU_MASK + and r0, r0, #MPIDR_CLUSTER_MASK + eor r0, r0, #(1 << MPIDR_CLUSTER_SHIFT) + add r0, r1, r0, LSR #6 + bx lr +END_FUNC get_core_pos_mpidr + diff --git a/optee_os/core/arch/arm/plat-vexpress/juno_core_pos_a64.S b/optee_os/core/arch/arm/plat-vexpress/juno_core_pos_a64.S new file mode 100644 index 0000000..b3a2352 --- /dev/null +++ b/optee_os/core/arch/arm/plat-vexpress/juno_core_pos_a64.S @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#include +#include + +/* For Juno number the two A57s as 4 to 5 and A53s as 0 to 3 */ +FUNC get_core_pos_mpidr , : + /* Calculate CorePos = ((ClusterId ^ 1) * 4) + CoreId */ + and x1, x0, #MPIDR_CPU_MASK + and x0, x0, #MPIDR_CLUSTER_MASK + eor x0, x0, #(1 << MPIDR_CLUSTER_SHIFT) + add x0, x1, x0, LSR #6 + ret +END_FUNC get_core_pos_mpidr diff --git a/optee_os/core/arch/arm/plat-vexpress/main.c b/optee_os/core/arch/arm/plat-vexpress/main.c new file mode 100644 index 0000000..d2ab712 --- /dev/null +++ b/optee_os/core/arch/arm/plat-vexpress/main.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2020, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct pl011_data console_data __nex_bss; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CONSOLE_UART_BASE, PL011_REG_SIZE); +#if defined(PLATFORM_FLAVOR_fvp) +register_phys_mem(MEM_AREA_RAM_SEC, TZCDRAM_BASE, TZCDRAM_SIZE); +#endif +#if defined(PLATFORM_FLAVOR_qemu_virt) +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SECRAM_BASE, SECRAM_COHERENT_SIZE); +#endif +#ifdef DRAM0_BASE +register_ddr(DRAM0_BASE, DRAM0_SIZE); +#endif +#ifdef DRAM1_BASE +register_ddr(DRAM1_BASE, DRAM1_SIZE); +#endif + +#ifdef CFG_GIC +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GIC_DIST_REG_SIZE); + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + +#if !defined(CFG_WITH_ARM_TRUSTED_FW) +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} +#endif +#endif /*CFG_GIC*/ + +#ifdef CFG_CORE_HAFNIUM_INTC +void boot_primary_init_intc(void) +{ + hfic_init(); +} +#endif + +void console_init(void) +{ + pl011_init(&console_data, CONSOLE_UART_BASE, CONSOLE_UART_CLK_IN_HZ, + CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +#if (defined(CFG_GIC) || defined(CFG_CORE_HAFNIUM_INTC)) && \ + defined(IT_CONSOLE_UART) && \ + !defined(CFG_NS_VIRTUALIZATION) && \ + !(defined(CFG_WITH_ARM_TRUSTED_FW) && defined(CFG_ARM_GICV2)) +/* + * This cannot be enabled with TF-A and GICv3 because TF-A then need to + * assign the interrupt number of the UART to OP-TEE (S-EL1). Currently + * there's no way of TF-A to know which interrupts that OP-TEE will serve. + * If TF-A doesn't assign the interrupt we're enabling below to OP-TEE it + * will hang in EL3 since the interrupt will just be delivered again and + * again. + */ + +static void read_console(void) +{ + struct serial_chip *cons = &console_data.chip; + + if (!cons->ops->getchar || !cons->ops->have_rx_data) + return; + + while (cons->ops->have_rx_data(cons)) { + int ch __maybe_unused = cons->ops->getchar(cons); + + DMSG("got 0x%x", ch); + } +} + +static enum itr_return console_itr_cb(struct itr_handler *h __maybe_unused) +{ + if (notif_async_is_started()) { + /* + * Asynchronous notifications are enabled, lets read from + * uart in the bottom half instead. + */ + itr_disable(IT_CONSOLE_UART); + notif_send_async(NOTIF_VALUE_DO_BOTTOM_HALF); + } else { + read_console(); + } + return ITRR_HANDLED; +} + +static struct itr_handler console_itr = { + .it = IT_CONSOLE_UART, + .flags = ITRF_TRIGGER_LEVEL, + .handler = console_itr_cb, +}; +DECLARE_KEEP_PAGER(console_itr); + +static void atomic_console_notif(struct notif_driver *ndrv __unused, + enum notif_event ev __maybe_unused) +{ + DMSG("Asynchronous notifications started, event %d", (int)ev); +} +DECLARE_KEEP_PAGER(atomic_console_notif); + +static void yielding_console_notif(struct notif_driver *ndrv __unused, + enum notif_event ev) +{ + switch (ev) { + case NOTIF_EVENT_DO_BOTTOM_HALF: + read_console(); + itr_enable(IT_CONSOLE_UART); + break; + case NOTIF_EVENT_STOPPED: + DMSG("Asynchronous notifications stopped"); + itr_enable(IT_CONSOLE_UART); + break; + default: + EMSG("Unknown event %d", (int)ev); + } +} + +struct notif_driver console_notif = { + .atomic_cb = atomic_console_notif, + .yielding_cb = yielding_console_notif, +}; + +static TEE_Result init_console_itr(void) +{ + itr_add(&console_itr); + itr_enable(IT_CONSOLE_UART); + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) + notif_register_driver(&console_notif); + return TEE_SUCCESS; +} +driver_init(init_console_itr); +#endif + +#ifdef CFG_TZC400 +register_phys_mem_pgdir(MEM_AREA_IO_SEC, TZC400_BASE, TZC400_REG_SIZE); + +static TEE_Result init_tzc400(void) +{ + void *va; + + DMSG("Initializing TZC400"); + + va = phys_to_virt(TZC400_BASE, MEM_AREA_IO_SEC, TZC400_REG_SIZE); + if (!va) { + EMSG("TZC400 not mapped"); + panic(); + } + + tzc_init((vaddr_t)va); + tzc_dump_state(); + + return TEE_SUCCESS; +} + +service_init(init_tzc400); +#endif /*CFG_TZC400*/ + +#if defined(PLATFORM_FLAVOR_qemu_virt) +static void release_secondary_early_hpen(size_t pos) +{ + struct mailbox { + uint64_t ep; + uint64_t hpen[]; + } *mailbox; + + if (cpu_mmu_enabled()) + mailbox = phys_to_virt(SECRAM_BASE, MEM_AREA_IO_SEC, + SECRAM_COHERENT_SIZE); + else + mailbox = (void *)SECRAM_BASE; + + if (!mailbox) + panic(); + + mailbox->ep = TEE_LOAD_ADDR; + dsb_ishst(); + mailbox->hpen[pos] = 1; + dsb_ishst(); + sev(); +} + +int psci_cpu_on(uint32_t core_id, uint32_t entry, uint32_t context_id) +{ + size_t pos = get_core_pos_mpidr(core_id); + static bool core_is_released[CFG_TEE_CORE_NB_CORE]; + + if (!pos || pos >= CFG_TEE_CORE_NB_CORE) + return PSCI_RET_INVALID_PARAMETERS; + + DMSG("core pos: %zu: ns_entry %#" PRIx32, pos, entry); + + if (core_is_released[pos]) { + EMSG("core %zu already released", pos); + return PSCI_RET_DENIED; + } + core_is_released[pos] = true; + + boot_set_core_ns_entry(pos, entry, context_id); + release_secondary_early_hpen(pos); + + return PSCI_RET_SUCCESS; +} +#endif /*PLATFORM_FLAVOR_qemu_virt*/ diff --git a/optee_os/core/arch/arm/plat-vexpress/platform_config.h b/optee_os/core/arch/arm/plat-vexpress/platform_config.h new file mode 100644 index 0000000..206885c --- /dev/null +++ b/optee_os/core/arch/arm/plat-vexpress/platform_config.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include +#include + +/* Make stacks aligned to data cache line length */ +#define STACK_ALIGNMENT 64 + +#if defined(PLATFORM_FLAVOR_fvp) + +#define GIC_BASE 0x2c000000 +#define UART0_BASE 0x1c090000 +#define UART1_BASE 0x1c0a0000 +#define UART2_BASE 0x1c0b0000 +#define UART3_BASE 0x1c0c0000 +#define TZC400_BASE 0x2a4a0000 + +#define IT_UART1 38 + +#define CONSOLE_UART_BASE UART1_BASE +#define IT_CONSOLE_UART IT_UART1 + +#elif defined(PLATFORM_FLAVOR_juno) + +#define GIC_BASE 0x2c010000 + +/* FPGA UART0 */ +#define UART0_BASE 0x1c090000 +/* FPGA UART1 */ +#define UART1_BASE 0x1c0a0000 +/* SoC UART0 */ +#define UART2_BASE 0x7ff80000 +/* SoC UART1 */ +#define UART3_BASE 0x7ff70000 + + +#define UART0_CLK_IN_HZ 24000000 +#define UART1_CLK_IN_HZ 24000000 +#define UART2_CLK_IN_HZ 7273800 +#define UART3_CLK_IN_HZ 7273800 + + +#define IT_UART3 116 + +#define CONSOLE_UART_BASE UART3_BASE +#define IT_CONSOLE_UART IT_UART3 +#define CONSOLE_UART_CLK_IN_HZ UART3_CLK_IN_HZ + +#elif defined(PLATFORM_FLAVOR_qemu_virt) + +#define GIC_BASE 0x08000000 +#define UART0_BASE 0x09000000 +#define UART1_BASE 0x09040000 +#define PCSC_BASE 0x09100000 + +#define IT_UART1 40 +#define IT_PCSC 37 + +#define CONSOLE_UART_BASE UART1_BASE +#define IT_CONSOLE_UART IT_UART1 + +#elif defined(PLATFORM_FLAVOR_qemu_armv8a) + +#define GIC_BASE 0x08000000 +#define UART0_BASE 0x09000000 +#define UART1_BASE 0x09040000 + +#define IT_UART1 40 + +#define CONSOLE_UART_BASE UART1_BASE +#define IT_CONSOLE_UART IT_UART1 + +#else +#error "Unknown platform flavor" +#endif + +#if defined(PLATFORM_FLAVOR_fvp) +/* + * FVP specifics. + */ + +#define DRAM0_BASE 0x80000000 +#define DRAM0_SIZE 0x7f000000 + +#define DRAM1_BASE 0x880000000UL +#define DRAM1_SIZE 0x180000000UL + +#define TZCDRAM_BASE 0xff000000 +#define TZCDRAM_SIZE 0x01000000 + +#define GICC_OFFSET 0x0 +#define GICD_OFFSET 0x3000000 + +#elif defined(PLATFORM_FLAVOR_juno) +/* + * Juno specifics. + */ + +#define DRAM0_BASE 0x80000000 +#define DRAM0_SIZE 0x7F000000 + +#define DRAM1_BASE 0x880000000UL +#define DRAM1_SIZE 0x180000000UL + +#define GICC_OFFSET 0x1f000 +#define GICD_OFFSET 0 + +#elif defined(PLATFORM_FLAVOR_qemu_virt) +/* + * QEMU virt specifics. + */ + +#define SECRAM_BASE 0x0e000000 +#define SECRAM_COHERENT_SIZE 4096 + +#define GICD_OFFSET 0 +#define GICC_OFFSET 0x10000 + +#elif defined(PLATFORM_FLAVOR_qemu_armv8a) + +#define GICD_OFFSET 0 +#define GICC_OFFSET 0x10000 + +#else +#error "Unknown platform flavor" +#endif + +#ifdef GIC_BASE +#define GICD_BASE (GIC_BASE + GICD_OFFSET) +#define GICC_BASE (GIC_BASE + GICC_OFFSET) +#endif + +#ifndef UART_BAUDRATE +#define UART_BAUDRATE 115200 +#endif +#ifndef CONSOLE_BAUDRATE +#define CONSOLE_BAUDRATE UART_BAUDRATE +#endif + +/* For virtual platforms where there isn't a clock */ +#ifndef CONSOLE_UART_CLK_IN_HZ +#define CONSOLE_UART_CLK_IN_HZ 1 +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-vexpress/sub.mk b/optee_os/core/arch/arm/plat-vexpress/sub.mk new file mode 100644 index 0000000..9b1dd7b --- /dev/null +++ b/optee_os/core/arch/arm/plat-vexpress/sub.mk @@ -0,0 +1,7 @@ +global-incdirs-y += . +srcs-y += main.c +ifeq ($(PLATFORM_FLAVOR_juno),y) +srcs-$(CFG_ARM32_core) += juno_core_pos_a32.S +srcs-$(CFG_ARM64_core) += juno_core_pos_a64.S +endif +srcs-$(CFG_WITH_USER_TA) += vendor_props.c diff --git a/optee_os/core/arch/arm/plat-vexpress/vendor_props.c b/optee_os/core/arch/arm/plat-vexpress/vendor_props.c new file mode 100644 index 0000000..a7061a9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-vexpress/vendor_props.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2020, Linaro Limited. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* + * The data to hash is 48 bytes made up of: + * - 16 bytes: the UUID of the calling TA. + * - 32 bytes: the hardware device ID + * The resulting endorsement seed is 32 bytes. + * + * The output buffer is the "binary" struct defined in + * the "prop_value" union and therefore comprises: + * - 4 bytes: the size of the binary value data (32) + * - 32 bytes: the binary value data (endorsement seed) + * + * Note that this code assumes an endorsement seed + * size == device ID size for convenience. + */ +static TEE_Result get_prop_endorsement(struct ts_session *sess, + void *buf, size_t *blen) +{ + TEE_Result res; + uint32_t ta_endorsement_seed_size = 32; + uint8_t data[sizeof(TEE_UUID) + ta_endorsement_seed_size]; + uint32_t bin[1 + ta_endorsement_seed_size / sizeof(uint32_t)]; + uint32_t *bin_len = (uint32_t *)bin; + uint8_t *bin_val = (uint8_t *)(&bin[1]); + + if (*blen < sizeof(bin)) { + *blen = sizeof(bin); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(bin); + + memcpy(data, &sess->ctx->uuid, sizeof(TEE_UUID)); + + if (tee_otp_get_die_id(&data[sizeof(TEE_UUID)], + ta_endorsement_seed_size)) + return TEE_ERROR_BAD_STATE; + + res = tee_hash_createdigest(TEE_ALG_SHA256, data, sizeof(data), + bin_val, ta_endorsement_seed_size); + if (res != TEE_SUCCESS) + return TEE_ERROR_BAD_STATE; + + *bin_len = ta_endorsement_seed_size; + + return copy_to_user(buf, bin, sizeof(bin)); +} + +static const struct tee_props vendor_propset_array_tee[] = { + { + .name = "com.microsoft.ta.endorsementSeed", + .prop_type = USER_TA_PROP_TYPE_BINARY_BLOCK, + .get_prop_func = get_prop_endorsement + }, +}; + +const struct tee_vendor_props vendor_props_tee = { + .props = vendor_propset_array_tee, + .len = ARRAY_SIZE(vendor_propset_array_tee), +}; diff --git a/optee_os/core/arch/arm/plat-zynq7k/conf.mk b/optee_os/core/arch/arm/plat-zynq7k/conf.mk new file mode 100644 index 0000000..30f2b4d --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynq7k/conf.mk @@ -0,0 +1,22 @@ +PLATFORM_FLAVOR ?= zc702 + +include core/arch/arm/cpu/cortex-a9.mk + +$(call force,CFG_TEE_CORE_NB_CORE,2) +$(call force,CFG_ARM32_core,y) +$(call force,CFG_GIC,y) +$(call force,CFG_CDNS_UART,y) +$(call force,CFG_WITH_SOFTWARE_PRNG,y) +$(call force,CFG_PL310,y) +$(call force,CFG_PL310_LOCKED,y) +$(call force,CFG_SECURE_TIME_SOURCE_REE,y) + +# Xilinx Zynq-7000's Cortex-A9 core has been configured with Non-maskable FIQ +# (NMFI) support. This means that FIQ interrupts cannot be used in system +# designs as atomic contexts cannot mask FIQ out. +$(call force,CFG_CORE_WORKAROUND_ARM_NMFI,y) + +CFG_BOOT_SYNC_CPU ?= y +CFG_BOOT_SECONDARY_REQUEST ?= y +CFG_CRYPTO_SIZE_OPTIMIZATION ?= n +CFG_ENABLE_SCTLR_RR ?= y diff --git a/optee_os/core/arch/arm/plat-zynq7k/main.c b/optee_os/core/arch/arm/plat-zynq7k/main.c new file mode 100644 index 0000000..19f40c9 --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynq7k/main.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * All rights reserved. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct cdns_uart_data console_data; + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, CONSOLE_UART_BASE, + CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_BASE, CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, PL310_BASE, CORE_MMU_PGDIR_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SLCR_BASE, CORE_MMU_PGDIR_SIZE); + +void plat_primary_init_early(void) +{ + /* primary core */ +#if defined(CFG_BOOT_SECONDARY_REQUEST) + /* set secondary entry address and release core */ + io_write32(SECONDARY_ENTRY_DROP, TEE_LOAD_ADDR); + dsb(); + sev(); +#endif + + /* SCU config */ + io_write32(SCU_BASE + SCU_INV_SEC, SCU_INV_CTRL_INIT); + io_write32(SCU_BASE + SCU_SAC, SCU_SAC_CTRL_INIT); + io_write32(SCU_BASE + SCU_NSAC, SCU_NSAC_CTRL_INIT); + + /* SCU enable */ + io_setbits32(SCU_BASE + SCU_CTRL, 0x1); + + /* NS Access control */ + io_write32(SECURITY2_SDIO0, ACCESS_BITS_ALL); + io_write32(SECURITY3_SDIO1, ACCESS_BITS_ALL); + io_write32(SECURITY4_QSPI, ACCESS_BITS_ALL); + io_write32(SECURITY6_APB_SLAVES, ACCESS_BITS_ALL); + + io_write32(SLCR_UNLOCK, SLCR_UNLOCK_MAGIC); + + io_write32(SLCR_TZ_DDR_RAM, ACCESS_BITS_ALL); + io_write32(SLCR_TZ_DMA_NS, ACCESS_BITS_ALL); + io_write32(SLCR_TZ_DMA_IRQ_NS, ACCESS_BITS_ALL); + io_write32(SLCR_TZ_DMA_PERIPH_NS, ACCESS_BITS_ALL); + io_write32(SLCR_TZ_GEM, ACCESS_BITS_ALL); + io_write32(SLCR_TZ_SDIO, ACCESS_BITS_ALL); + io_write32(SLCR_TZ_USB, ACCESS_BITS_ALL); + + io_write32(SLCR_LOCK, SLCR_LOCK_MAGIC); +} + +void console_init(void) +{ + cdns_uart_init(&console_data, CONSOLE_UART_BASE, 0, 0); + register_serial_console(&console_data.chip); +} + +vaddr_t pl310_base(void) +{ + static void *va; + + if (cpu_mmu_enabled()) { + if (!va) + va = phys_to_virt(PL310_BASE, MEM_AREA_IO_SEC, 1); + return (vaddr_t)va; + } + return PL310_BASE; +} + +void arm_cl2_config(vaddr_t pl310_base) +{ + /* Disable PL310 */ + io_write32(pl310_base + PL310_CTRL, 0); + + /* + * Xilinx AR#54190 recommends setting L2C RAM in SLCR + * to 0x00020202 for proper cache operations. + */ + io_write32(SLCR_L2C_RAM, SLCR_L2C_RAM_VALUE); + + io_write32(pl310_base + PL310_TAG_RAM_CTRL, PL310_TAG_RAM_CTRL_INIT); + io_write32(pl310_base + PL310_DATA_RAM_CTRL, PL310_DATA_RAM_CTRL_INIT); + io_write32(pl310_base + PL310_AUX_CTRL, PL310_AUX_CTRL_INIT); + io_write32(pl310_base + PL310_PREFETCH_CTRL, PL310_PREFETCH_CTRL_INIT); + io_write32(pl310_base + PL310_POWER_CTRL, PL310_POWER_CTRL_INIT); + + /* invalidate all cache ways */ + arm_cl2_invbyway(pl310_base); +} + +void arm_cl2_enable(vaddr_t pl310_base) +{ + uint32_t val; + + /* Enable PL310 ctrl -> only set lsb bit */ + io_write32(pl310_base + PL310_CTRL, 1); + + /* if L2 FLZW enable, enable in L1 */ + val = io_read32(pl310_base + PL310_AUX_CTRL); + if (val & 1) + write_actlr(read_actlr() | (1 << 3)); +} + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + +void boot_secondary_init_intc(void) +{ + gic_cpu_init(); +} + +static vaddr_t slcr_access_range[] = { + 0x004, 0x008, /* lock, unlock */ + 0x100, 0x1FF, /* PLL */ + 0x200, 0x2FF, /* Reset */ + 0xA00, 0xAFF /* L2C */ +}; + +static uint32_t write_slcr(uint32_t addr, uint32_t val) +{ + uint32_t i; + + for (i = 0; i < ARRAY_SIZE(slcr_access_range); i += 2) { + if (addr >= slcr_access_range[i] && + addr <= slcr_access_range[i+1]) { + static vaddr_t va; + + if (!va) + va = (vaddr_t)phys_to_virt(SLCR_BASE, + MEM_AREA_IO_SEC, + addr + + sizeof(uint32_t)); + io_write32(va + addr, val); + return OPTEE_SMC_RETURN_OK; + } + } + return OPTEE_SMC_RETURN_EBADADDR; +} + +static uint32_t read_slcr(uint32_t addr, uint32_t *val) +{ + uint32_t i; + + for (i = 0; i < ARRAY_SIZE(slcr_access_range); i += 2) { + if (addr >= slcr_access_range[i] && + addr <= slcr_access_range[i+1]) { + static vaddr_t va; + + if (!va) + va = (vaddr_t)phys_to_virt(SLCR_BASE, + MEM_AREA_IO_SEC, + addr + + sizeof(uint32_t)); + *val = io_read32(va + addr); + return OPTEE_SMC_RETURN_OK; + } + } + return OPTEE_SMC_RETURN_EBADADDR; +} + +/* Overriding the default __weak tee_entry_fast() */ +void tee_entry_fast(struct thread_smc_args *args) +{ + switch (args->a0) { + case ZYNQ7K_SMC_SLCR_WRITE: + args->a0 = write_slcr(args->a1, args->a2); + break; + case ZYNQ7K_SMC_SLCR_READ: + args->a0 = read_slcr(args->a1, &args->a2); + break; + default: + __tee_entry_fast(args); + break; + } +} diff --git a/optee_os/core/arch/arm/plat-zynq7k/plat_init.S b/optee_os/core/arch/arm/plat-zynq7k/plat_init.S new file mode 100644 index 0000000..86b35da --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynq7k/plat_init.S @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Entry points for the A9 inits, A9 revision specific or not. + * It is assume no stack is available when these routines are called. + * It is assume each routine is called with return address in LR + * and with ARM registers R0, R1, R2, R3 being scratchable. + */ + +#include +#include +#include +#include +#include +#include + +#define ZYNQ_SLCR_L2C_RAM 0xF8000A1C + +.section .text +.balign 4 +.code 32 + +/* + * Cortex A9 early configuration + * + * Use registers R0-R3. + * No stack usage. + * LR store return address. + * Trap CPU in case of error. + */ +FUNC plat_cpu_reset_early , : + /* + * Disallow NSec to mask FIQ [bit4: FW=0] + * Allow NSec to manage Imprecise Abort [bit5: AW=1] + * Imprecise Abort trapped to Abort Mode [bit3: EA=0] + * In Sec world, FIQ trapped to FIQ Mode [bit2: FIQ=0] + * IRQ always trapped to IRQ Mode [bit1: IRQ=0] + * Secure World [bit0: NS=0] + */ + mov r0, #SCR_AW + write_scr r0 /* write Secure Configuration Register */ + + /* + * Mandated HW config loaded + * + * SCTLR = 0x00004000 + * - Round-Robin replac. for icache, btac, i/duTLB (bit14: RoundRobin) + * + * ACTRL = 0x00000041 + * - core always in full SMP (FW bit0=1, SMP bit6=1) + * - L2 write full line of zero disabled (bit3=0) + * (keep WFLZ low. Will be set once outer L2 is ready) + * + * NSACR = 0x00020C00 + * - NSec cannot change ACTRL.SMP (NS_SMP bit18=0) + * - Nsec can lockdown TLB (TL bit17=1) + * - NSec cannot access PLE (PLE bit16=0) + * - NSec can use SIMD/VFP (CP10/CP11) (bit15:14=2b00, bit11:10=2b11) + * + * PCR = 0x00000001 + * - no change latency, enable clk gating + */ + mov_imm r0, 0x00004000 + write_sctlr r0 + + mov_imm r0, 0x00000041 + write_actlr r0 + + mov_imm r0, 0x00020C00 + write_nsacr r0 + + mov_imm r0, 0x00000001 + write_pcr r0 + + mov pc, lr +END_FUNC plat_cpu_reset_early diff --git a/optee_os/core/arch/arm/plat-zynq7k/platform_config.h b/optee_os/core/arch/arm/plat-zynq7k/platform_config.h new file mode 100644 index 0000000..41a4888 --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynq7k/platform_config.h @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Wind River Systems. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#define STACK_ALIGNMENT 64 + +/* For Zynq7000 board */ + +#define SCU_BASE 0xF8F00000 +#define PL310_BASE 0xF8F02000 +#define GIC_BASE 0xF8F00000 +#define GICC_OFFSET 0x100 +#define GICD_OFFSET 0x1000 +#define GIC_CPU_BASE (GIC_BASE + GICC_OFFSET) +#define GIC_DIST_BASE (GIC_BASE + GICD_OFFSET) + +#define SLCR_BASE 0xF8000000 +#define SLCR_LOCK 0xF8000004 +#define SLCR_UNLOCK 0xF8000008 +#define SLCR_TZ_DDR_RAM 0xF8000430 +#define SLCR_TZ_DMA_NS 0xF8000440 +#define SLCR_TZ_DMA_IRQ_NS 0xF8000444 +#define SLCR_TZ_DMA_PERIPH_NS 0xF8000448 +#define SLCR_TZ_GEM 0xF8000450 +#define SLCR_TZ_SDIO 0xF8000454 +#define SLCR_TZ_USB 0xF8000458 +#define SLCR_L2C_RAM 0xF8000A1C + +#define SLCR_LOCK_MAGIC 0x0000767B +#define SLCR_UNLOCK_MAGIC 0x0000DF0D + +#define SECURITY2_SDIO0 0xE0200008 +#define SECURITY3_SDIO1 0xE020000C +#define SECURITY4_QSPI 0xE0200010 +#define SECURITY6_APB_SLAVES 0xE0200018 + +#define UART0_BASE 0xE0000000 +#define UART1_BASE 0xE0001000 + +#define CONSOLE_UART_BASE UART1_BASE + +#define TEE_RAM_VA_SIZE (1024 * 1024) + +/* + * PL310 TAG RAM Control Register + * + * bit[10:8]:1 - 2 cycle of write accesses latency + * bit[6:4]:1 - 2 cycle of read accesses latency + * bit[2:0]:1 - 2 cycle of setup latency + */ +#ifndef PL310_TAG_RAM_CTRL_INIT +#define PL310_TAG_RAM_CTRL_INIT 0x00000111 +#endif + +/* + * PL310 DATA RAM Control Register + * + * bit[10:8]:2 - 3 cycle of write accesses latency + * bit[6:4]:2 - 3 cycle of read accesses latency + * bit[2:0]:2 - 3 cycle of setup latency + */ +#ifndef PL310_DATA_RAM_CTRL_INIT +#define PL310_DATA_RAM_CTRL_INIT 0x00000222 +#endif + +/* + * PL310 Auxiliary Control Register + * + * I/Dcache prefetch enabled (bit29:28=2b11) + * NS can access interrupts (bit27=1) + * NS can lockown cache lines (bit26=1) + * Pseudo-random replacement policy (bit25=0) + * Force write allocated (default) + * Shared attribute internally ignored (bit22=1, bit13=0) + * Parity disabled (bit21=0) + * Event monitor disabled (bit20=0) + * Platform fmavor specific way config: + * - 64kb way size (bit19:17=3b011) + * - 8-way associciativity (bit16=0) + * Store buffer device limitation enabled (bit11=1) + * Cacheable accesses have high prio (bit10=0) + * Full Line Zero (FLZ) disabled (bit0=0) + */ +#ifndef PL310_AUX_CTRL_INIT +#define PL310_AUX_CTRL_INIT 0x3C460800 +#endif + +/* + * PL310 Prefetch Control Register + * + * Double linefill disabled (bit30=0) + * I/D prefetch enabled (bit29:28=2b11) + * Prefetch drop enabled (bit24=1) + * Incr double linefill disable (bit23=0) + * Prefetch offset = 7 (bit4:0) + */ +#define PL310_PREFETCH_CTRL_INIT 0x31000007 + +/* + * PL310 Power Register + * + * Dynamic clock gating enabled + * Standby mode enabled + */ +#define PL310_POWER_CTRL_INIT 0x00000003 + +/* + * SCU Invalidate Register + * + * Invalidate all registers + */ +#define SCU_INV_CTRL_INIT 0xFFFFFFFF + +/* + * SCU Access Register + * - both secure CPU access SCU + */ +#define SCU_SAC_CTRL_INIT 0x0000000F + +/* + * SCU NonSecure Access Register + * - both nonsec cpu access SCU, private and global timer + */ +#define SCU_NSAC_CTRL_INIT 0x00000FFF + +/* all bit enabled in access control register */ +#define ACCESS_BITS_ALL 0xFFFFFFFF + +/* recommended value for setting the L2C_RAM register */ +#define SLCR_L2C_RAM_VALUE 0x00020202 + +/* place in OCRAM to write secondary entry to */ +#define SECONDARY_ENTRY_DROP 0xFFFFFFF0 + +/* define the memory areas */ + +#ifdef CFG_WITH_PAGER + +/* + * TEE/TZ RAM layout: + * + * +---------------------------------------+ <- CFG_CORE_TZSRAM_EMUL_START + * | TEE private highly | TEE_RAM | ^ + * | secure memory | | | CFG_CORE_TZSRAM_EMUL_SIZE + * +---------------------------------------+ v + * + * +---------------------------------------+ <- TZDRAM_BASE + * | TEE private secure | TA_RAM | ^ + TZDRAM_SIZE + * | external memory | | v + * +---------------------------------------+ <- TEE_SHMEM_START + * | Non secure | SHM | | + * | shared memory | | | + TEE_SHMEM_SIZE + * +---------------------------------------+ v + * + * TEE_RAM : default 256kByte + * TA_RAM : all what is left in DDR TEE reserved area + * PUB_RAM : default 2MByte + */ + +/* emulated SRAM, 256K at start of secure DDR */ + +#define TZSRAM_BASE 0x3E000000 +#define TZSRAM_SIZE CFG_CORE_TZSRAM_EMUL_SIZE + +/* Location of trusted dram */ + +#define TEE_RAM_START TZSRAM_BASE +#define TEE_RAM_PH_SIZE TZSRAM_SIZE + +#define TZDRAM_BASE 0x3e100000 +#define TZDRAM_SIZE 0x01e00000 + +#define TEE_SHMEM_START 0x3ff00000 +#define TEE_SHMEM_SIZE 0x00100000 + +#define TA_RAM_START TZDRAM_BASE +#define TA_RAM_SIZE TZDRAM_SIZE + +#ifdef CFG_TEE_LOAD_ADDR +#define TEE_LOAD_ADDR CFG_TEE_LOAD_ADDR +#else +#define TEE_LOAD_ADDR TEE_RAM_START +#endif + +#else /* CFG_WITH_PAGER */ + +/* + * TEE/TZ RAM layout: + * + * +---------------------------------------+ <- TZDRAM_BASE + * | TEE private secure | TEE_RAM | ^ + * | external memory +------------------+ | + * | | TA_RAM | | + * +---------------------------------------+ | TZDRAM_SIZE + * | Non secure | SHM | | + * | shared memory | | | + * +---------------------------------------+ v + * + * TEE_RAM : 1MByte + * PUB_RAM : 1MByte + * TA_RAM : all what is left (at least 2MByte !) + */ + +#define TZDRAM_BASE 0x3E000000 +#define TZDRAM_SIZE (0x02000000 - TEE_SHMEM_SIZE) + +#define TEE_RAM_START TZDRAM_BASE +#define TEE_RAM_PH_SIZE (1 * 1024 * 1024) + +#define TA_RAM_START (TZDRAM_BASE + TEE_RAM_PH_SIZE) +#define TA_RAM_SIZE (TZDRAM_SIZE - TEE_RAM_PH_SIZE) + +#define TEE_SHMEM_START (TZDRAM_BASE + TZDRAM_SIZE) +#define TEE_SHMEM_SIZE 0x00100000 + +#ifdef CFG_TEE_LOAD_ADDR +#define TEE_LOAD_ADDR CFG_TEE_LOAD_ADDR +#else +#define TEE_LOAD_ADDR TEE_RAM_START +#endif + +#endif /* CFG_WITH_PAGER */ + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-zynq7k/platform_smc.h b/optee_os/core/arch/arm/plat-zynq7k/platform_smc.h new file mode 100644 index 0000000..b0b8792 --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynq7k/platform_smc.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Wind River System + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef PLATFORM_SMC_H +#define PLATFORM_SMC_H + +#include + +/* + * Read SLCR (System Level Control Register) + * + * Call register usage: + * a0 SMC Function ID, ZYNQ7K_SMC_SLCR_READ + * a1 Register offset + * a2-7 Not used + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 Value read back + * a2-3 Not used + * a4-7 Preserved + * + * OPTEE_SMC_RETURN_EBADCMD on Invalid input offset: + * a0 OPTEE_SMC_RETURN_EBADCMD + * a1 Undefined value + * a2-3 Not used + * a4-7 Preserved + */ +#define ZYNQ7K_SMC_FUNCID_SLCR_READ 0x100 +#define ZYNQ7K_SMC_SLCR_READ \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_FAST_CALL, \ + OPTEE_SMC_OWNER_OEM, ZYNQ7K_SMC_FUNCID_SLCR_READ) + +/* + * Write SLCR (System Level Control Register) + * + * Call register usage: + * a0 SMC Function ID, ZYNQ7K_SMC_SLCR_READ + * a1 Register offset + * a2 Value to write + * a3-7 Not used + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1-3 Not used + * a4-7 Preserved + * + * OPTEE_SMC_RETURN_EBADCMD on Invalid input offset: + * a0 OPTEE_SMC_RETURN_EBADCMD + * a1-3 Not used + * a4-7 Preserved + */ +#define ZYNQ7K_SMC_FUNCID_SLCR_WRITE 0x101 +#define ZYNQ7K_SMC_SLCR_WRITE \ + OPTEE_SMC_CALL_VAL(OPTEE_SMC_32, OPTEE_SMC_FAST_CALL, \ + OPTEE_SMC_OWNER_OEM, ZYNQ7K_SMC_FUNCID_SLCR_WRITE) + +#endif /* PLATFORM_SMC_H */ diff --git a/optee_os/core/arch/arm/plat-zynq7k/sub.mk b/optee_os/core/arch/arm/plat-zynq7k/sub.mk new file mode 100644 index 0000000..652c084 --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynq7k/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +srcs-y += main.c +srcs-y += plat_init.S diff --git a/optee_os/core/arch/arm/plat-zynqmp/conf.mk b/optee_os/core/arch/arm/plat-zynqmp/conf.mk new file mode 100644 index 0000000..60ff011 --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynqmp/conf.mk @@ -0,0 +1,90 @@ +PLATFORM_FLAVOR ?= zcu102 + +include core/arch/arm/cpu/cortex-armv8-0.mk + +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_CDNS_UART,y) +$(call force,CFG_GIC,y) +$(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) +$(call force,CFG_WITH_ARM_TRUSTED_FW,y) + +# Disable core ASLR for two reasons: +# 1. There is no source for ALSR seed, as ATF does not provide a +# DTB to OP-TEE. Hardware RNG is also not currently supported. +# 2. OP-TEE does not boot with enabled CFG_CORE_ASLR. +$(call force,CFG_CORE_ASLR,n) + +ifeq ($(CFG_ARM64_core),y) +# ZynqMP supports up to 40 bits of physical addresses +CFG_CORE_ARM64_PA_BITS ?= 40 +else +$(call force,CFG_ARM32_core,y) +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),zcu102 zcu104 zcu106 zc1751_dc1 zc1751_dc2)) + +CFG_UART_BASE ?= UART0_BASE +CFG_UART_IT ?= IT_UART0 +CFG_UART_CLK_HZ ?= UART0_CLK_IN_HZ + +# ZCU102 features 4 GiB of DDR +ifeq ($(CFG_ARM64_core),y) +CFG_DDR_SIZE ?= 0x100000000 +else +# On 32 bit build limit to 2 GiB of RAM +CFG_DDR_SIZE ?= 0x80000000 +endif +endif + +ifneq (,$(filter $(PLATFORM_FLAVOR),ultra96)) + +CFG_UART_BASE ?= UART1_BASE +CFG_UART_IT ?= IT_UART1 +CFG_UART_CLK_HZ ?= UART1_CLK_IN_HZ + +# Ultra96 features 2 GiB of DDR +CFG_DDR_SIZE ?= 0x80000000 +endif + +# By default use DT address as specified by Xilinx +CFG_DT_ADDR ?= 0x100000 + +CFG_TZDRAM_START ?= 0x60000000 +CFG_TZDRAM_SIZE ?= 0x10000000 +CFG_SHMEM_START ?= 0x70000000 +CFG_SHMEM_SIZE ?= 0x10000000 + +CFG_WITH_STATS ?= y +CFG_CRYPTO_WITH_CE ?= y + +# Enable use of User AES eFuse as device key instead of PUF. +# This is needed when images are encrypted with AES eFuse device key (AES_KEY). +CFG_ZYNQMP_HUK_AES_EFUSE ?= n + +# Configures bitmask which user eFuses should be included in HUK generation. +# Used when (part of) user eFuses are used for HUK seed (i.e. programmed with +# good random values). +# Bit 0 means eFuse USER_0, bit 1 for eFuse USER 1 and so on. +CFG_ZYNQMP_HUK_USER_EFUSE_MASK ?= 0 + +CFG_ZYNQMP_PM ?= $(CFG_ARM64_core) + +ifeq ($(CFG_RPMB_FS),y) +$(call force,CFG_ZYNQMP_HUK,y,Mandated by CFG_RPMB_FS) +endif + +ifeq ($(CFG_ZYNQMP_HUK),y) +$(call force,CFG_ZYNQMP_CSU_AES,y,Mandated by CFG_ZYNQMP_HUK) +ifneq ($(CFG_ZYNQMP_HUK_AES_EFUSE),y) +$(call force,CFG_ZYNQMP_CSU_PUF,y,Mandated by CFG_ZYNQMP_HUK) +endif +endif + +ifeq ($(CFG_ZYNQMP_CSU_AES),y) +$(call force,CFG_ZYNQMP_CSUDMA,y,Mandated by CFG_ZYNQMP_CSU_AES) +$(call force,CFG_DT,y,Mandated by CFG_ZYNQMP_CSU_AES) +endif + +ifneq (,$(filter y, $(CFG_ZYNQMP_CSU_PUF) $(CFG_ZYNQMP_CSUDMA) $(CFG_ZYNQMP_CSU_AES))) +$(call force,CFG_ZYNQMP_CSU,y,Mandated by CFG_ZYNQMP_CSU* clients) +endif diff --git a/optee_os/core/arch/arm/plat-zynqmp/main.c b/optee_os/core/arch/arm/plat-zynqmp/main.c new file mode 100644 index 0000000..dcbf686 --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynqmp/main.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Xilinx Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct cdns_uart_data console_data __nex_bss; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(CONSOLE_UART_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(GIC_BASE, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, + ROUNDDOWN(GIC_BASE + GICD_OFFSET, CORE_MMU_PGDIR_SIZE), + CORE_MMU_PGDIR_SIZE); +#if defined(CFG_ZYNQMP_CSU) +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CSU_BASE, CSU_SIZE); +#endif + +#if CFG_DDR_SIZE > 0x80000000 + +#ifdef CFG_ARM32_core +#error DDR size over 2 GiB is not supported in 32 bit ARM mode +#endif + +register_ddr(DRAM0_BASE, 0x80000000); +register_ddr(DRAM1_BASE, CFG_DDR_SIZE - 0x80000000); +#else +register_ddr(DRAM0_BASE, CFG_DDR_SIZE); +#endif + +void boot_primary_init_intc(void) +{ + gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +} + +void console_init(void) +{ + cdns_uart_init(&console_data, CONSOLE_UART_BASE, + CONSOLE_UART_CLK_IN_HZ, CONSOLE_BAUDRATE); + register_serial_console(&console_data.chip); +} + +#if defined(CFG_RPMB_FS) +bool plat_rpmb_key_is_ready(void) +{ + vaddr_t csu = core_mmu_get_va(CSU_BASE, MEM_AREA_IO_SEC, CSU_SIZE); + struct tee_hw_unique_key hwkey = { }; + uint32_t status = 0; + + if (tee_otp_get_hw_unique_key(&hwkey)) + return false; + + /* + * For security reasons, we don't allow writing the RPMB key using the + * development HUK even though it is unique. + */ + status = io_read32(csu + ZYNQMP_CSU_STATUS_OFFSET); + if (status & ZYNQMP_CSU_STATUS_AUTH) + return true; + + return false; +} +#endif diff --git a/optee_os/core/arch/arm/plat-zynqmp/platform_config.h b/optee_os/core/arch/arm/plat-zynqmp/platform_config.h new file mode 100644 index 0000000..cc9189e --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynqmp/platform_config.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Xilinx Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* Make stacks aligned to data cache line length */ +#define CACHELINE_LEN 64 +#define STACK_ALIGNMENT CACHELINE_LEN + +#ifdef CFG_WITH_PAGER +#error "Pager not supported for zynqmp" +#endif + +/* DDR Low area base */ +#define DRAM0_BASE 0 + +#ifdef ARM64 +/* DDR High area base is only available when compiling for 64 bits */ +#define DRAM1_BASE 0x800000000 +#endif + +#ifdef CFG_CDNS_UART +#define CONSOLE_UART_BASE (CFG_UART_BASE) +#define IT_CONSOLE_UART (CFG_UART_IT) +#define CONSOLE_UART_CLK_IN_HZ (CFG_UART_CLK_HZ) +#endif + +#if defined(PLATFORM_FLAVOR_zc1751_dc1) || \ + defined(PLATFORM_FLAVOR_zc1751_dc2) || \ + defined(PLATFORM_FLAVOR_zcu102) || \ + defined(PLATFORM_FLAVOR_zcu104) || \ + defined(PLATFORM_FLAVOR_zcu106) || \ + defined(PLATFORM_FLAVOR_ultra96) + +#define GIC_BASE 0xF9010000 +#define UART0_BASE 0xFF000000 +#define UART1_BASE 0xFF010000 + +#define IT_UART0 53 +#define IT_UART1 54 + +#define UART0_CLK_IN_HZ 100000000 +#define UART1_CLK_IN_HZ 100000000 + +#define GICD_OFFSET 0 +#define GICC_OFFSET 0x10000 + +#else +#error "Unknown platform flavor" +#endif + +#define CSUDMA_BASE 0xFFC80000 +#define CSUDMA_SIZE 0x1000 +#define CSU_BASE 0xFFCA0000 +#define CSU_SIZE 0x5038 + +#ifdef CFG_TEE_LOAD_ADDR +#define TEE_LOAD_ADDR CFG_TEE_LOAD_ADDR +#else +#define TEE_LOAD_ADDR TEE_RAM_START +#endif + +#ifndef UART_BAUDRATE +#define UART_BAUDRATE 115200 +#endif +#ifndef CONSOLE_BAUDRATE +#define CONSOLE_BAUDRATE UART_BAUDRATE +#endif + +/* For virtual platforms where there isn't a clock */ +#ifndef CONSOLE_UART_CLK_IN_HZ +#define CONSOLE_UART_CLK_IN_HZ 1 +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/arm/plat-zynqmp/sub.mk b/optee_os/core/arch/arm/plat-zynqmp/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/arm/plat-zynqmp/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/arm/sm/pm.c b/optee_os/core/arch/arm/sm/pm.c new file mode 100644 index 0000000..9ce3b3c --- /dev/null +++ b/optee_os/core/arch/arm/sm/pm.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017 NXP + * + * Peng Fan + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if CFG_TEE_CORE_NB_CORE > 4 +#error "Max support 4 cores in one cluster now" +#endif + +void sm_pm_cpu_suspend_save(struct sm_pm_ctx *ctx, uint32_t sp) +{ + struct thread_core_local *p = thread_get_core_local(); + + p->sm_pm_ctx_phys = virt_to_phys((void *)ctx); + + /* The content will be passed to sm_pm_cpu_do_resume as register sp */ + ctx->sp = sp; + ctx->cpu_resume_addr = + virt_to_phys((void *)(vaddr_t)sm_pm_cpu_do_resume); + + sm_pm_cpu_do_suspend(ctx->suspend_regs); + + dcache_op_level1(DCACHE_OP_CLEAN_INV); + +#ifdef CFG_PL310 + arm_cl2_cleanbyway(core_mmu_get_va(PL310_BASE, MEM_AREA_IO_SEC, 1)); +#endif +} diff --git a/optee_os/core/arch/arm/sm/pm_a32.S b/optee_os/core/arch/arm/sm/pm_a32.S new file mode 100644 index 0000000..2b824ef --- /dev/null +++ b/optee_os/core/arch/arm/sm/pm_a32.S @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017 NXP + * + * Peng Fan + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +.section .text + +/* + * int sm_pm_cpu_suspend(uint32_t arg, int (*fn)(uint32_t)) + * @arg will be passed to fn as argument + * return value: 0 - cpu resumed from suspended state. + * -1 - cpu not suspended. + */ +FUNC sm_pm_cpu_suspend, : +UNWIND( .cantunwind) + push {r4 - r12, lr} + mov r5, sp + sub sp, sp, #SM_PM_CTX_SIZE + push {r0, r1} + + mov r1, r5 + add r0, sp, #8 + blx sm_pm_cpu_suspend_save + adr lr, aborted + /* Jump to arch specific suspend */ + pop {r0, pc} +aborted: + /* cpu not suspended */ + add sp, sp, #SM_PM_CTX_SIZE + /* Return -1 to the caller */ + mov r0, #(-1) +suspend_return: + pop {r4 - r12, pc} +END_FUNC sm_pm_cpu_suspend + +FUNC sm_pm_cpu_do_suspend, : +UNWIND( .cantunwind) + push {r4 - r11} + read_midr r4 + ubfx r5, r4, #4, #12 + ldr r4, =CORTEX_A5_PART_NUM + cmp r5, r4 + beq a5_a7_suspend + ldr r4, =CORTEX_A7_PART_NUM + cmp r5, r4 + beq a5_a7_suspend + ldr r4, =CORTEX_A9_PART_NUM + cmp r5, r4 + beq a9_suspend + /* cpu not supported */ + b . + /* A9 needs PCR/DIAG */ +a9_suspend: + read_pcr r4 + read_diag r5 + stmia r0!, {r4 - r5} +a5_a7_suspend: + read_fcseidr r4 + read_tpidruro r5 + stmia r0!, {r4 - r5} + read_dacr r4 +#ifdef CFG_WITH_LPAE +#error "Not supported" +#else + read_ttbr0 r5 + read_ttbr1 r6 + read_ttbcr r7 +#endif + read_sctlr r8 + read_actlr r9 + read_cpacr r10 + read_mvbar r11 + stmia r0!, {r4 - r11} + read_prrr r4 + read_nmrr r5 + read_vbar r6 + read_nsacr r7 + stmia r0, {r4 - r7} + pop {r4 - r11} + bx lr +END_FUNC sm_pm_cpu_do_suspend + +FUNC sm_pm_cpu_resume, : +UNWIND( .cantunwind) + cpsid aif + + /* Call into the runtime address of __get_core_pos */ + adr r0, _core_pos + ldr r1, [r0] + add r0, r0, r1 + blx r0 + + /* + * At this point, MMU is not enabled now. + * 1. Get the runtime physical address of _suspend_sp + * 2. Get the offset from _suspend_sp to &thread_core_local + * 3. Get the runtime physical address of thread_core_local + * Since moving towards non-linear mapping, + * `ldr r0, =thread_core_local` is not used here. + */ + adr r4, _suspend_sp + ldr r5, [r4] + add r4, r4, r5 + + mov_imm r1, THREAD_CORE_LOCAL_SIZE + mla r0, r0, r1, r4 + + ldr r0, [r0, #THREAD_CORE_LOCAL_SM_PM_CTX_PHYS] + /* Need to use r0!, because sm_pm_cpu_do_resume needs it */ + ldmia r0!, {sp, pc} +END_FUNC sm_pm_cpu_resume + +/* + * The following will be located in text section whose attribute is + * marked as readonly, but we only need to read here + * _suspend_sp stores the offset between thread_core_local to _suspend_sp. + * _core_pos stores the offset between __get_core_pos to _core_pos. + */ +.align 2 +.extern thread_core_local +_suspend_sp: + .long thread_core_local - . +.extern __get_core_pos +_core_pos: + .long __get_core_pos - . + +/* + * void sm_pm_cpu_do_resume(paddr suspend_regs) __noreturn; + * Restore the registers stored when sm_pm_cpu_do_suspend + * r0 points to the physical base address of the suspend_regs + * field of struct sm_pm_ctx. + */ +FUNC sm_pm_cpu_do_resume, : +UNWIND( .cantunwind) + read_midr r4 + ubfx r5, r4, #4, #12 + ldr r4, =CORTEX_A5_PART_NUM + cmp r5, r4 + beq a5_a7_resume + ldr r4, =CORTEX_A7_PART_NUM + cmp r5, r4 + beq a5_a7_resume + + /* + * A9 needs PCR/DIAG + */ + ldmia r0!, {r4 - r5} + write_pcr r4 + write_diag r5 + +a5_a7_resume: + /* v7 resume */ + mov ip, #0 + /* Invalidate icache to PoU */ + write_iciallu + /* set reserved context */ + write_contextidr ip + ldmia r0!, {r4 - r5} + write_fcseidr r4 + write_tpidruro r5 + ldmia r0!, {r4 - r11} + /* Invalidate entire TLB */ + write_tlbiall + write_dacr r4 +#ifdef CFG_WITH_LPAE +#error "Not supported -" +#else + write_ttbr0 r5 + write_ttbr1 r6 + write_ttbcr r7 +#endif + + ldmia r0, {r4 - r7} + write_prrr r4 + write_nmrr r5 + write_vbar r6 + write_nsacr r7 + + write_actlr r9 + write_cpacr r10 + write_mvbar r11 + write_bpiall + isb + dsb + /* MMU will be enabled here */ + write_sctlr r8 + isb + mov r0, #0 + b suspend_return +END_FUNC sm_pm_cpu_do_resume + diff --git a/optee_os/core/arch/arm/sm/psci-helper.S b/optee_os/core/arch/arm/sm/psci-helper.S new file mode 100644 index 0000000..feaca88 --- /dev/null +++ b/optee_os/core/arch/arm/sm/psci-helper.S @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017 NXP + */ + +#include +#include +#include +#include + +FUNC psci_disable_smp, : + read_actlr r0 + bic r0, r0, #ACTLR_SMP + write_actlr r0 + isb + bx lr +END_FUNC psci_disable_smp + +FUNC psci_enable_smp, : + read_actlr r0 + orr r0, r0, #ACTLR_SMP + write_actlr r0 + isb + bx lr +END_FUNC psci_enable_smp + +FUNC psci_armv7_cpu_off, : + push {r12, lr} +UNWIND( .save {r12, lr}) + + mov r0, #DCACHE_OP_CLEAN_INV + bl dcache_op_all + + /* Disable Cache */ + read_sctlr r0 + bic r0, r0, #SCTLR_C + write_sctlr r0 + isb + dsb + + mov r0, #DCACHE_OP_CLEAN_INV + bl dcache_op_all + + clrex + + bl psci_disable_smp + + pop {r12, pc} +END_FUNC psci_armv7_cpu_off diff --git a/optee_os/core/arch/arm/sm/psci.c b/optee_os/core/arch/arm/sm/psci.c new file mode 100644 index 0000000..167ef14 --- /dev/null +++ b/optee_os/core/arch/arm/sm/psci.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2016 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Peng Fan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +__weak uint32_t psci_version(void) +{ + return PSCI_VERSION_1_1; +} + +__weak int psci_cpu_suspend(uint32_t power_state __unused, + uintptr_t entry __unused, + uint32_t context_id __unused, + struct sm_nsec_ctx *nsec __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_cpu_off(void) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_cpu_on(uint32_t cpu_id __unused, uint32_t entry __unused, + uint32_t context_id __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_affinity_info(uint32_t affinity __unused, + uint32_t lowest_affnity_level __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_migrate(uint32_t cpu_id __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_migrate_info_type(void) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_migrate_info_up_cpu(void) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak void psci_system_off(void) +{ +} + +__weak void psci_system_reset(void) +{ +} + +__weak int psci_features(uint32_t psci_fid __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_mem_protect(uint32_t enable __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_mem_chk_range(paddr_t base __unused, + size_t length __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_system_reset2(uint32_t reset_type __unused, + uint32_t cookie __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_node_hw_state(uint32_t cpu_id __unused, + uint32_t power_level __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_system_suspend(uintptr_t entry __unused, + uint32_t context_id __unused, + struct sm_nsec_ctx *nsec __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_stat_residency(uint32_t cpu_id __unused, + uint32_t power_state __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +__weak int psci_stat_count(uint32_t cpu_id __unused, + uint32_t power_state __unused) +{ + return PSCI_RET_NOT_SUPPORTED; +} + +void tee_psci_handler(struct thread_smc_args *args, struct sm_nsec_ctx *nsec) +{ + uint32_t smc_fid = args->a0; + uint32_t a1 = args->a1; + uint32_t a2 = args->a2; + uint32_t a3 = args->a3; + + switch (smc_fid) { + case PSCI_VERSION: + args->a0 = psci_version(); + break; + case PSCI_CPU_SUSPEND: + args->a0 = psci_cpu_suspend(a1, a2, a3, nsec); + break; + case PSCI_CPU_OFF: + args->a0 = psci_cpu_off(); + break; + case PSCI_CPU_ON: + args->a0 = psci_cpu_on(a1, a2, a3); + break; + case PSCI_AFFINITY_INFO: + args->a0 = psci_affinity_info(a1, a2); + break; + case PSCI_MIGRATE: + args->a0 = psci_migrate(a1); + break; + case PSCI_MIGRATE_INFO_TYPE: + args->a0 = psci_migrate_info_type(); + break; + case PSCI_MIGRATE_INFO_UP_CPU: + args->a0 = psci_migrate_info_up_cpu(); + break; + case PSCI_SYSTEM_OFF: + psci_system_off(); + while (1) + ; + break; + case PSCI_SYSTEM_RESET: + psci_system_reset(); + while (1) + ; + break; + case PSCI_PSCI_FEATURES: + args->a0 = psci_features(a1); + break; + case PSCI_SYSTEM_RESET2: + args->a0 = psci_system_reset2(a1, a2); + break; + case PSCI_MEM_PROTECT: + args->a0 = psci_mem_protect(a1); + break; + case PSCI_MEM_PROTECT_CHECK_RANGE: + args->a0 = psci_mem_chk_range(a1, a2); + break; + case PSCI_NODE_HW_STATE: + args->a0 = psci_node_hw_state(a1, a2); + break; + case PSCI_SYSTEM_SUSPEND: + args->a0 = psci_system_suspend(a1, a2, nsec); + break; + default: + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; + break; + } +} diff --git a/optee_os/core/arch/arm/sm/sm.c b/optee_os/core/arch/arm/sm/sm.c new file mode 100644 index 0000000..905e028 --- /dev/null +++ b/optee_os/core/arch/arm/sm/sm.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2020, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sm_private.h" + +enum sm_handler_ret __weak sm_platform_handler(struct sm_ctx *ctx __unused) +{ + return SM_HANDLER_PENDING_SMC; +} + +static void smc_arch_handler(struct thread_smc_args *args) +{ + uint32_t smc_fid = args->a0; + uint32_t feature_fid = args->a1; + + switch (smc_fid) { + case ARM_SMCCC_VERSION: + args->a0 = SMCCC_V_1_1; + break; + case ARM_SMCCC_ARCH_FEATURES: + switch (feature_fid) { + case ARM_SMCCC_VERSION: + case ARM_SMCCC_ARCH_SOC_ID: + args->a0 = ARM_SMCCC_RET_SUCCESS; + break; + default: + args->a0 = ARM_SMCCC_RET_NOT_SUPPORTED; + break; + } + break; + case ARM_SMCCC_ARCH_SOC_ID: + args->a0 = ARM_SMCCC_RET_NOT_SUPPORTED; + break; + case ARM_SMCCC_ARCH_WORKAROUND_1: + case ARM_SMCCC_ARCH_WORKAROUND_2: + args->a0 = ARM_SMCCC_RET_NOT_REQUIRED; + break; + default: + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; + break; + } +} + +uint32_t sm_from_nsec(struct sm_ctx *ctx) +{ + uint32_t *nsec_r0 = (uint32_t *)(&ctx->nsec.r0); + struct thread_smc_args *args = (struct thread_smc_args *)nsec_r0; + + /* + * Check that struct sm_ctx has the different parts properly + * aligned since the stack pointer will be updated to point at + * different parts of this struct. + */ + COMPILE_TIME_ASSERT(!(offsetof(struct sm_ctx, sec.r0) % 8)); + COMPILE_TIME_ASSERT(!(offsetof(struct sm_ctx, nsec.r0) % 8)); + COMPILE_TIME_ASSERT(!(sizeof(struct sm_ctx) % 8)); + + if (wdt_sm_handler(args) == SM_HANDLER_SMC_HANDLED) + return SM_EXIT_TO_NON_SECURE; + + if (IS_ENABLED(CFG_SM_PLATFORM_HANDLER) && + sm_platform_handler(ctx) == SM_HANDLER_SMC_HANDLED) + return SM_EXIT_TO_NON_SECURE; + + switch (OPTEE_SMC_OWNER_NUM(args->a0)) { + case OPTEE_SMC_OWNER_STANDARD: + if (IS_ENABLED(CFG_PSCI_ARM32)) { + smc_std_handler(args, &ctx->nsec); + return SM_EXIT_TO_NON_SECURE; + } + break; + case OPTEE_SMC_OWNER_ARCH: + smc_arch_handler(args); + return SM_EXIT_TO_NON_SECURE; + default: + break; + } + + sm_save_unbanked_regs(&ctx->nsec.ub_regs); + sm_restore_unbanked_regs(&ctx->sec.ub_regs); + + memcpy(&ctx->sec.r0, args, sizeof(*args)); + + if (IS_ENABLED(CFG_CORE_WORKAROUND_ARM_NMFI)) { + /* Make sure FIQ is masked when jumping to SMC entry. */ + ctx->sec.mon_spsr |= CPSR_F; + } + + if (OPTEE_SMC_IS_FAST_CALL(ctx->sec.r0)) + ctx->sec.mon_lr = (uint32_t)vector_fast_smc_entry; + else + ctx->sec.mon_lr = (uint32_t)vector_std_smc_entry; + + return SM_EXIT_TO_SECURE; +} diff --git a/optee_os/core/arch/arm/sm/sm_a32.S b/optee_os/core/arch/arm/sm/sm_a32.S new file mode 100644 index 0000000..6181ebb --- /dev/null +++ b/optee_os/core/arch/arm/sm/sm_a32.S @@ -0,0 +1,424 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2020, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SM_CTX_SEC_END (SM_CTX_SEC + SM_CTX_SEC_SIZE) + + .macro save_regs mode + cps \mode + mrs r2, spsr + str r2, [r0], #4 + str sp, [r0], #4 + str lr, [r0], #4 + .endm + +FUNC sm_save_unbanked_regs , : +UNWIND( .cantunwind) + /* User mode registers has to be saved from system mode */ + cps #CPSR_MODE_SYS + str sp, [r0], #4 + str lr, [r0], #4 + + save_regs #CPSR_MODE_IRQ + save_regs #CPSR_MODE_FIQ + save_regs #CPSR_MODE_SVC + save_regs #CPSR_MODE_ABT + save_regs #CPSR_MODE_UND + +#ifdef CFG_SM_NO_CYCLE_COUNTING + read_pmcr r2 + stm r0!, {r2} +#endif + +#ifdef CFG_FTRACE_SUPPORT + read_cntkctl r2 + stm r0!, {r2} +#endif + cps #CPSR_MODE_MON + bx lr +END_FUNC sm_save_unbanked_regs + + .macro restore_regs mode + cps \mode + ldr r2, [r0], #4 + ldr sp, [r0], #4 + ldr lr, [r0], #4 + msr spsr_fsxc, r2 + .endm + +/* Restores the mode specific registers */ +FUNC sm_restore_unbanked_regs , : +UNWIND( .cantunwind) + /* User mode registers has to be saved from system mode */ + cps #CPSR_MODE_SYS + ldr sp, [r0], #4 + ldr lr, [r0], #4 + + restore_regs #CPSR_MODE_IRQ + restore_regs #CPSR_MODE_FIQ + restore_regs #CPSR_MODE_SVC + restore_regs #CPSR_MODE_ABT + restore_regs #CPSR_MODE_UND + +#ifdef CFG_SM_NO_CYCLE_COUNTING + ldm r0!, {r2} + write_pmcr r2 +#endif + +#ifdef CFG_FTRACE_SUPPORT + ldm r0!, {r2} + write_cntkctl r2 +#endif + cps #CPSR_MODE_MON + bx lr +END_FUNC sm_restore_unbanked_regs + +/* + * stack_tmp is used as stack, the top of the stack is reserved to hold + * struct sm_ctx, everything below is for normal stack usage. As several + * different CPU modes are using the same stack it's important that switch + * of CPU mode isn't done until one mode is done. This means FIQ, IRQ and + * Async abort has to be masked while using stack_tmp. + */ +LOCAL_FUNC sm_smc_entry , : +UNWIND( .cantunwind) + srsdb sp!, #CPSR_MODE_MON + push {r0-r7} + + clrex /* Clear the exclusive monitor */ + + /* Find out if we're doing an secure or non-secure entry */ + read_scr r1 + tst r1, #SCR_NS + bne .smc_from_nsec + + /* + * As we're coming from secure world (NS bit cleared) the stack + * pointer points to sm_ctx.sec.r0 at this stage. After the + * instruction below the stack pointer points to sm_ctx. + */ + sub sp, sp, #(SM_CTX_SEC + SM_SEC_CTX_R0) + + /* Save secure context */ + add r0, sp, #SM_CTX_SEC + bl sm_save_unbanked_regs + + /* + * On FIQ exit we're restoring the non-secure context unchanged, on + * all other exits we're shifting r1-r4 from secure context into + * r0-r3 in non-secure context. + */ + add r8, sp, #(SM_CTX_SEC + SM_SEC_CTX_R0) + ldm r8, {r0-r4} + mov_imm r9, TEESMC_OPTEED_RETURN_FIQ_DONE + cmp r0, r9 + addne r8, sp, #(SM_CTX_NSEC + SM_NSEC_CTX_R0) + stmne r8, {r1-r4} + + /* Restore non-secure context */ + add r0, sp, #SM_CTX_NSEC + bl sm_restore_unbanked_regs + +.sm_ret_to_nsec: + /* + * Return to non-secure world + */ + add r0, sp, #(SM_CTX_NSEC + SM_NSEC_CTX_R8) + ldm r0, {r8-r12} + +#ifdef CFG_CORE_WORKAROUND_NSITR_CACHE_PRIME + /* + * Prevent leaking information about which code has been executed. + * This is required to be used together with + * CFG_CORE_WORKAROUND_SPECTRE_BP to protect Cortex A15 CPUs too. + * + * CFG_CORE_WORKAROUND_SPECTRE_BP also invalidates the branch + * predictor on affected CPUs. In the cases where an alternative + * vector has been installed the branch predictor is already + * invalidated so invalidating here again would be redundant, but + * testing for that is more trouble than it's worth. + */ + write_bpiall +#endif + + /* Update SCR */ + read_scr r0 + orr r0, r0, #(SCR_NS | SCR_FIQ) /* Set NS and FIQ bit in SCR */ + write_scr r0 + /* + * isb not needed since we're doing an exception return below + * without dependency to the changes in SCR before that. + */ + + add sp, sp, #(SM_CTX_NSEC + SM_NSEC_CTX_R0) + b .sm_exit + +.smc_from_nsec: + /* + * As we're coming from non-secure world (NS bit set) the stack + * pointer points to sm_ctx.nsec.r0 at this stage. After the + * instruction below the stack pointer points to sm_ctx. + */ + sub sp, sp, #(SM_CTX_NSEC + SM_NSEC_CTX_R0) + + bic r1, r1, #(SCR_NS | SCR_FIQ) /* Clear NS and FIQ bit in SCR */ + write_scr r1 + isb + + add r0, sp, #(SM_CTX_NSEC + SM_NSEC_CTX_R8) + stm r0, {r8-r12} + + mov r0, sp + bl sm_from_nsec + cmp r0, #SM_EXIT_TO_NON_SECURE + beq .sm_ret_to_nsec + + /* + * Continue into secure world + */ + add sp, sp, #(SM_CTX_SEC + SM_SEC_CTX_R0) + +.sm_exit: + pop {r0-r7} + rfefd sp! +END_FUNC sm_smc_entry + +/* + * FIQ handling + * + * Saves CPU context in the same way as sm_smc_entry() above. The CPU + * context will later be restored by sm_smc_entry() when handling a return + * from FIQ. + */ +LOCAL_FUNC sm_fiq_entry , : +UNWIND( .cantunwind) + /* FIQ has a +4 offset for lr compared to preferred return address */ + sub lr, lr, #4 + /* sp points just past struct sm_sec_ctx */ + srsdb sp!, #CPSR_MODE_MON + push {r0-r7} + + clrex /* Clear the exclusive monitor */ + + /* + * As we're coming from non-secure world the stack pointer points + * to sm_ctx.nsec.r0 at this stage. After the instruction below the + * stack pointer points to sm_ctx. + */ + sub sp, sp, #(SM_CTX_NSEC + SM_NSEC_CTX_R0) + + /* Update SCR */ + read_scr r1 + bic r1, r1, #(SCR_NS | SCR_FIQ) /* Clear NS and FIQ bit in SCR */ + write_scr r1 + isb + + /* Save non-secure context */ + add r0, sp, #SM_CTX_NSEC + bl sm_save_unbanked_regs + add r0, sp, #(SM_CTX_NSEC + SM_NSEC_CTX_R8) + stm r0!, {r8-r12} + + /* Set FIQ entry */ + ldr r0, =vector_fiq_entry + str r0, [sp, #(SM_CTX_SEC + SM_SEC_CTX_MON_LR)] + + /* Restore secure context */ + add r0, sp, #SM_CTX_SEC + bl sm_restore_unbanked_regs + + add sp, sp, #(SM_CTX_SEC + SM_SEC_CTX_MON_LR) + + rfefd sp! +END_FUNC sm_fiq_entry + +LOCAL_FUNC sm_vect_table , :, align=32 +UNWIND( .cantunwind) + b . /* Reset */ + b . /* Undefined instruction */ + b sm_smc_entry /* Secure monitor call */ + b . /* Prefetch abort */ + b . /* Data abort */ + b . /* Reserved */ + b . /* IRQ */ + b sm_fiq_entry /* FIQ */ +END_FUNC sm_vect_table + +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP + .macro vector_prologue_spectre + /* + * This depends on SP being 8 byte aligned, that is, the + * lowest three bits in SP are zero. + * + * The idea is to form a specific bit pattern in the lowest + * three bits of SP depending on which entry in the vector + * we enter via. This is done by adding 1 to SP in each + * entry but the last. + */ + add sp, sp, #1 /* 7:111 Reset */ + add sp, sp, #1 /* 6:110 Undefined instruction */ + add sp, sp, #1 /* 5:101 Secure monitor call */ + add sp, sp, #1 /* 4:100 Prefetch abort */ + add sp, sp, #1 /* 3:011 Data abort */ + add sp, sp, #1 /* 2:010 Reserved */ + add sp, sp, #1 /* 1:001 IRQ */ + nop /* 0:000 FIQ */ + .endm + +LOCAL_FUNC sm_vect_table_a15 , :, align=32 +UNWIND( .cantunwind) + vector_prologue_spectre + /* + * Invalidate the branch predictor for the current processor. + * For Cortex-A8 ACTLR[6] has to be set to 1 for BPIALL to be + * effective. + * Note that the BPIALL instruction is not effective in + * invalidating the branch predictor on Cortex-A15. For that CPU, + * set ACTLR[0] to 1 during early processor initialisation, and + * invalidate the branch predictor by performing an ICIALLU + * instruction. See also: + * https://github.com/ARM-software/arm-trusted-firmware/wiki/Arm-Trusted-Firmware-Security-Advisory-TFV-6#variant-2-cve-2017-5715 + */ + write_iciallu + isb + b 1f +END_FUNC sm_vect_table_a15 + + +LOCAL_FUNC sm_vect_table_bpiall , :, align=32 +UNWIND( .cantunwind) + vector_prologue_spectre + /* Invalidate the branch predictor for the current processor. */ + write_bpiall + isb + +1: + /* + * Only two exception does normally occur, smc and fiq. With all + * other exceptions it's good enough to just spinn, the lowest bits + * still tells which exception we're stuck with when attaching a + * debugger. + */ + + /* Test for FIQ, all the lowest bits of SP are supposed to be 0 */ + tst sp, #(BIT(0) | BIT(1) | BIT(2)) + beq sm_fiq_entry + + /* Test for SMC, xor the lowest bits of SP to be 0 */ + eor sp, sp, #(BIT(0) | BIT(2)) + tst sp, #(BIT(0) | BIT(1) | BIT(2)) + beq sm_smc_entry + + /* unhandled exception */ + b . +END_FUNC sm_vect_table_bpiall +#endif /*!CFG_CORE_WORKAROUND_SPECTRE_BP*/ + +/* void sm_init(vaddr_t stack_pointer); */ +FUNC sm_init , : + /* Set monitor stack */ + mrs r1, cpsr + cps #CPSR_MODE_MON + /* Point just beyond sm_ctx.sec */ + sub sp, r0, #(SM_CTX_SIZE - SM_CTX_SEC_END) + +#ifdef CFG_INIT_CNTVOFF + read_scr r0 + orr r0, r0, #SCR_NS /* Set NS bit in SCR */ + write_scr r0 + isb + + /* + * Accessing CNTVOFF: + * If the implementation includes the Virtualization Extensions + * this is a RW register, accessible from Hyp mode, and + * from Monitor mode when SCR.NS is set to 1. + * If the implementation includes the Security Extensions + * but not the Virtualization Extensions, an MCRR or MRRC to + * the CNTVOFF encoding is UNPREDICTABLE if executed in Monitor + * mode, regardless of the value of SCR.NS. + */ + read_id_pfr1 r2 + mov r3, r2 + ands r3, r3, #IDPFR1_GENTIMER_MASK + beq .no_gentimer + ands r2, r2, #IDPFR1_VIRT_MASK + beq .no_gentimer + mov r2, #0 + write_cntvoff r2, r2 + +.no_gentimer: + bic r0, r0, #SCR_NS /* Clr NS bit in SCR */ + write_scr r0 + isb +#endif +#ifdef CFG_SM_NO_CYCLE_COUNTING + read_pmcr r0 + orr r0, #PMCR_DP + write_pmcr r0 +#endif + msr cpsr, r1 + +#ifdef CFG_CORE_WORKAROUND_SPECTRE_BP + /* + * For unrecognized CPUs we fall back to the vector used for + * unaffected CPUs. Cortex A-15 has special treatment compared to + * the other affected Cortex CPUs. + */ + read_midr r1 + ubfx r2, r1, #MIDR_IMPLEMENTER_SHIFT, #MIDR_IMPLEMENTER_WIDTH + cmp r2, #MIDR_IMPLEMENTER_ARM + bne 1f + + ubfx r2, r1, #MIDR_PRIMARY_PART_NUM_SHIFT, \ + #MIDR_PRIMARY_PART_NUM_WIDTH + + movw r3, #CORTEX_A8_PART_NUM + cmp r2, r3 + movwne r3, #CORTEX_A9_PART_NUM + cmpne r2, r3 + movwne r3, #CORTEX_A17_PART_NUM + cmpne r2, r3 + ldreq r0, =sm_vect_table_bpiall + beq 2f + + movw r3, #CORTEX_A15_PART_NUM + cmp r2, r3 + ldreq r0, =sm_vect_table_a15 + beq 2f +#endif + /* Set monitor vector (MVBAR) */ +1: ldr r0, =sm_vect_table +2: write_mvbar r0 + + bx lr +END_FUNC sm_init +DECLARE_KEEP_PAGER sm_init + + +/* struct sm_nsec_ctx *sm_get_nsec_ctx(void); */ +FUNC sm_get_nsec_ctx , : + mrs r1, cpsr + cps #CPSR_MODE_MON + /* + * As we're in secure mode mon_sp points just beyond sm_ctx.sec, + * which allows us to calculate the address of sm_ctx.nsec. + */ + add r0, sp, #(SM_CTX_NSEC - SM_CTX_SEC_END) + msr cpsr, r1 + + bx lr +END_FUNC sm_get_nsec_ctx diff --git a/optee_os/core/arch/arm/sm/sm_private.h b/optee_os/core/arch/arm/sm/sm_private.h new file mode 100644 index 0000000..c224109 --- /dev/null +++ b/optee_os/core/arch/arm/sm/sm_private.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef SM_PRIVATE_H +#define SM_PRIVATE_H + +/* Returns one of SM_EXIT_TO_* exit monitor in secure or non-secure world */ +uint32_t sm_from_nsec(struct sm_ctx *ctx); +#endif /*SM_PRIVATE_H*/ + diff --git a/optee_os/core/arch/arm/sm/std_smc.c b/optee_os/core/arch/arm/sm/std_smc.c new file mode 100644 index 0000000..52ed918 --- /dev/null +++ b/optee_os/core/arch/arm/sm/std_smc.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2016 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Peng Fan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +static const TEE_UUID uuid = { + 0x5f8b97df, 0x2d0d, 0x4ad2, + {0x98, 0xd2, 0x74, 0xf4, 0x38, 0x27, 0x98, 0xbb}, +}; + +void smc_std_handler(struct thread_smc_args *args, struct sm_nsec_ctx *nsec) +{ + uint32_t smc_fid = args->a0; + + if (is_psci_fid(smc_fid)) { + tee_psci_handler(args, nsec); + return; + } + + switch (smc_fid) { + case ARM_STD_SVC_CALL_COUNT: + /* PSCI is the only STD service implemented */ + args->a0 = PSCI_NUM_CALLS; + break; + case ARM_STD_SVC_UID: + args->a0 = uuid.timeLow; + args->a1 = (uuid.timeHiAndVersion << 16) | uuid.timeMid; + args->a2 = (uuid.clockSeqAndNode[3] << 24) | + (uuid.clockSeqAndNode[2] << 16) | + (uuid.clockSeqAndNode[1] << 8) | + uuid.clockSeqAndNode[0]; + args->a3 = (uuid.clockSeqAndNode[7] << 24) | + (uuid.clockSeqAndNode[6] << 16) | + (uuid.clockSeqAndNode[5] << 8) | + uuid.clockSeqAndNode[4]; + break; + case ARM_STD_SVC_VERSION: + args->a0 = STD_SVC_VERSION_MAJOR; + args->a1 = STD_SVC_VERSION_MINOR; + break; + default: + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; + break; + } +} diff --git a/optee_os/core/arch/arm/sm/sub.mk b/optee_os/core/arch/arm/sm/sub.mk new file mode 100644 index 0000000..598bdad --- /dev/null +++ b/optee_os/core/arch/arm/sm/sub.mk @@ -0,0 +1,4 @@ +srcs-y += sm_a32.S +srcs-y += sm.c +srcs-$(CFG_PM_ARM32) += pm.c pm_a32.S +srcs-$(CFG_PSCI_ARM32) += std_smc.c psci.c psci-helper.S diff --git a/optee_os/core/arch/arm/tee/cache.c b/optee_os/core/arch/arm/tee/cache.c new file mode 100644 index 0000000..d8ef9e3 --- /dev/null +++ b/optee_os/core/arch/arm/tee/cache.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include + +/* + * tee_uta_cache_operation - dynamic cache clean/inval request from a TA. + * It follows ARM recommendation: + * https://developer.arm.com/documentation/ddi0246/c/Beicdhde + * Note that this implementation assumes dsb operations are part of + * cache_op_inner(), and outer cache sync are part of cache_op_outer(). + */ +TEE_Result cache_operation(enum utee_cache_operation op, void *va, size_t len) +{ + TEE_Result res; + paddr_t pa; + + pa = virt_to_phys(va); + if (!pa) + return TEE_ERROR_ACCESS_DENIED; + + switch (op) { + case TEE_CACHEFLUSH: +#ifdef CFG_PL310 /* prevent initial L1 clean in case there is no outer L2 */ + /* Clean L1, Flush L2, Flush L1 */ + res = cache_op_inner(DCACHE_AREA_CLEAN, va, len); + if (res != TEE_SUCCESS) + return res; + res = cache_op_outer(DCACHE_AREA_CLEAN_INV, pa, len); + if (res != TEE_SUCCESS) + return res; +#endif + return cache_op_inner(DCACHE_AREA_CLEAN_INV, va, len); + + case TEE_CACHECLEAN: + /* Clean L1, Clean L2 */ + res = cache_op_inner(DCACHE_AREA_CLEAN, va, len); + if (res != TEE_SUCCESS) + return res; + return cache_op_outer(DCACHE_AREA_CLEAN, pa, len); + + case TEE_CACHEINVALIDATE: + /* Inval L2, Inval L1 */ + res = cache_op_outer(DCACHE_AREA_INVALIDATE, pa, len); + if (res != TEE_SUCCESS) + return res; + return cache_op_inner(DCACHE_AREA_INVALIDATE, va, len); + + default: + return TEE_ERROR_NOT_SUPPORTED; + } +} diff --git a/optee_os/core/arch/arm/tee/entry_fast.c b/optee_os/core/arch/arm/tee/entry_fast.c new file mode 100644 index 0000000..5d54dbe --- /dev/null +++ b/optee_os/core/arch/arm/tee/entry_fast.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2021, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_CORE_RESERVED_SHM +static void tee_entry_get_shm_config(struct thread_smc_args *args) +{ + args->a0 = OPTEE_SMC_RETURN_OK; + args->a1 = default_nsec_shm_paddr; + args->a2 = default_nsec_shm_size; + /* Should this be TEESMC cache attributes instead? */ + args->a3 = core_mmu_is_shm_cached(); +} +#endif + +static void tee_entry_fastcall_l2cc_mutex(struct thread_smc_args *args) +{ + TEE_Result ret; +#ifdef ARM32 + paddr_t pa = 0; + + switch (args->a1) { + case OPTEE_SMC_L2CC_MUTEX_GET_ADDR: + ret = tee_get_l2cc_mutex(&pa); + reg_pair_from_64(pa, &args->a2, &args->a3); + break; + case OPTEE_SMC_L2CC_MUTEX_SET_ADDR: + pa = reg_pair_to_64(args->a2, args->a3); + ret = tee_set_l2cc_mutex(&pa); + break; + case OPTEE_SMC_L2CC_MUTEX_ENABLE: + ret = tee_enable_l2cc_mutex(); + break; + case OPTEE_SMC_L2CC_MUTEX_DISABLE: + ret = tee_disable_l2cc_mutex(); + break; + default: + args->a0 = OPTEE_SMC_RETURN_EBADCMD; + return; + } +#else + ret = TEE_ERROR_NOT_SUPPORTED; +#endif + if (ret == TEE_ERROR_NOT_SUPPORTED) + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; + else if (ret) + args->a0 = OPTEE_SMC_RETURN_EBADADDR; + else + args->a0 = OPTEE_SMC_RETURN_OK; +} + +static void tee_entry_exchange_capabilities(struct thread_smc_args *args) +{ + bool res_shm_en = IS_ENABLED(CFG_CORE_RESERVED_SHM); + bool dyn_shm_en __maybe_unused = false; + + /* + * Currently we ignore OPTEE_SMC_NSEC_CAP_UNIPROCESSOR. + * + * The memory mapping of shared memory is defined as normal + * shared memory for SMP systems and normal memory for UP + * systems. Currently we map all memory as shared in secure + * world. + * + * When translation tables are created with shared bit cleared for + * uniprocessor systems we'll need to check + * OPTEE_SMC_NSEC_CAP_UNIPROCESSOR. + */ + + if (args->a1 & ~OPTEE_SMC_NSEC_CAP_UNIPROCESSOR) { + /* Unknown capability. */ + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; + return; + } + + args->a0 = OPTEE_SMC_RETURN_OK; + args->a1 = 0; + + if (res_shm_en) + args->a1 |= OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM; + IMSG("Reserved shared memory is %sabled", res_shm_en ? "en" : "dis"); + +#if defined(CFG_CORE_DYN_SHM) + dyn_shm_en = core_mmu_nsec_ddr_is_defined(); + if (dyn_shm_en) + args->a1 |= OPTEE_SMC_SEC_CAP_DYNAMIC_SHM; +#endif + IMSG("Dynamic shared memory is %sabled", dyn_shm_en ? "en" : "dis"); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + args->a1 |= OPTEE_SMC_SEC_CAP_VIRTUALIZATION; + IMSG("Normal World virtualization support is %sabled", + IS_ENABLED(CFG_NS_VIRTUALIZATION) ? "en" : "dis"); + + args->a1 |= OPTEE_SMC_SEC_CAP_MEMREF_NULL; + + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) { + args->a1 |= OPTEE_SMC_SEC_CAP_ASYNC_NOTIF; + args->a2 = NOTIF_VALUE_MAX; + } + IMSG("Asynchronous notifications are %sabled", + IS_ENABLED(CFG_CORE_ASYNC_NOTIF) ? "en" : "dis"); + + args->a1 |= OPTEE_SMC_SEC_CAP_RPC_ARG; + args->a3 = THREAD_RPC_MAX_NUM_PARAMS; +} + +static void tee_entry_disable_shm_cache(struct thread_smc_args *args) +{ + uint64_t cookie; + + if (!thread_disable_prealloc_rpc_cache(&cookie)) { + args->a0 = OPTEE_SMC_RETURN_EBUSY; + return; + } + + if (!cookie) { + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; + return; + } + + args->a0 = OPTEE_SMC_RETURN_OK; + args->a1 = cookie >> 32; + args->a2 = cookie; +} + +static void tee_entry_enable_shm_cache(struct thread_smc_args *args) +{ + if (thread_enable_prealloc_rpc_cache()) + args->a0 = OPTEE_SMC_RETURN_OK; + else + args->a0 = OPTEE_SMC_RETURN_EBUSY; +} + +static void tee_entry_boot_secondary(struct thread_smc_args *args) +{ +#if defined(CFG_BOOT_SECONDARY_REQUEST) + if (!boot_core_release(args->a1, (paddr_t)(args->a3))) + args->a0 = OPTEE_SMC_RETURN_OK; + else + args->a0 = OPTEE_SMC_RETURN_EBADCMD; +#else + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; +#endif +} + +static void tee_entry_get_thread_count(struct thread_smc_args *args) +{ + args->a0 = OPTEE_SMC_RETURN_OK; + args->a1 = CFG_NUM_THREADS; +} + +#if defined(CFG_NS_VIRTUALIZATION) +static void tee_entry_vm_created(struct thread_smc_args *args) +{ + uint16_t guest_id = args->a1; + + /* Only hypervisor can issue this request */ + if (args->a7 != HYP_CLNT_ID) { + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; + return; + } + + if (virt_guest_created(guest_id)) + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; + else + args->a0 = OPTEE_SMC_RETURN_OK; +} + +static void tee_entry_vm_destroyed(struct thread_smc_args *args) +{ + uint16_t guest_id = args->a1; + + /* Only hypervisor can issue this request */ + if (args->a7 != HYP_CLNT_ID) { + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; + return; + } + + if (virt_guest_destroyed(guest_id)) + args->a0 = OPTEE_SMC_RETURN_ENOTAVAIL; + else + args->a0 = OPTEE_SMC_RETURN_OK; +} +#endif + +/* Note: this function is weak to let platforms add special handling */ +void __weak tee_entry_fast(struct thread_smc_args *args) +{ + __tee_entry_fast(args); +} + +static void get_async_notif_value(struct thread_smc_args *args) +{ + bool value_valid = false; + bool value_pending = false; + + args->a0 = OPTEE_SMC_RETURN_OK; + args->a1 = notif_get_value(&value_valid, &value_pending); + args->a2 = 0; + if (value_valid) + args->a2 |= OPTEE_SMC_ASYNC_NOTIF_VALID; + if (value_pending) + args->a2 |= OPTEE_SMC_ASYNC_NOTIF_PENDING; +} + +static void tee_entry_watchdog(struct thread_smc_args *args) +{ +#if defined(CFG_WDT_SM_HANDLER) + __wdt_sm_handler(args); +#else + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; +#endif +} + +/* + * If tee_entry_fast() is overridden, it's still supposed to call this + * function. + */ +void __tee_entry_fast(struct thread_smc_args *args) +{ + switch (args->a0) { + + /* Generic functions */ + case OPTEE_SMC_CALLS_COUNT: + tee_entry_get_api_call_count(args); + break; + case OPTEE_SMC_CALLS_UID: + tee_entry_get_api_uuid(args); + break; + case OPTEE_SMC_CALLS_REVISION: + tee_entry_get_api_revision(args); + break; + case OPTEE_SMC_CALL_GET_OS_UUID: + tee_entry_get_os_uuid(args); + break; + case OPTEE_SMC_CALL_GET_OS_REVISION: + tee_entry_get_os_revision(args); + break; + + /* OP-TEE specific SMC functions */ +#ifdef CFG_CORE_RESERVED_SHM + case OPTEE_SMC_GET_SHM_CONFIG: + tee_entry_get_shm_config(args); + break; +#endif + case OPTEE_SMC_L2CC_MUTEX: + tee_entry_fastcall_l2cc_mutex(args); + break; + case OPTEE_SMC_EXCHANGE_CAPABILITIES: + tee_entry_exchange_capabilities(args); + break; + case OPTEE_SMC_DISABLE_SHM_CACHE: + tee_entry_disable_shm_cache(args); + break; + case OPTEE_SMC_ENABLE_SHM_CACHE: + tee_entry_enable_shm_cache(args); + break; + case OPTEE_SMC_BOOT_SECONDARY: + tee_entry_boot_secondary(args); + break; + case OPTEE_SMC_GET_THREAD_COUNT: + tee_entry_get_thread_count(args); + break; + +#if defined(CFG_NS_VIRTUALIZATION) + case OPTEE_SMC_VM_CREATED: + tee_entry_vm_created(args); + break; + case OPTEE_SMC_VM_DESTROYED: + tee_entry_vm_destroyed(args); + break; +#endif + + case OPTEE_SMC_ENABLE_ASYNC_NOTIF: + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) { + notif_deliver_atomic_event(NOTIF_EVENT_STARTED); + args->a0 = OPTEE_SMC_RETURN_OK; + } else { + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; + } + break; + case OPTEE_SMC_GET_ASYNC_NOTIF_VALUE: + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) + get_async_notif_value(args); + else + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; + break; + + /* Watchdog entry if handler ID is defined in TOS range */ + case CFG_WDT_SM_HANDLER_ID: + tee_entry_watchdog(args); + break; + + default: + args->a0 = OPTEE_SMC_RETURN_UNKNOWN_FUNCTION; + break; + } +} + +size_t tee_entry_generic_get_api_call_count(void) +{ + /* + * All the different calls handled in this file. If the specific + * target has additional calls it will call this function and + * add the number of calls the target has added. + */ + size_t ret = 12; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + ret += 2; + + return ret; +} + +void __weak tee_entry_get_api_call_count(struct thread_smc_args *args) +{ + args->a0 = tee_entry_generic_get_api_call_count(); +} + +void __weak tee_entry_get_api_uuid(struct thread_smc_args *args) +{ + args->a0 = OPTEE_MSG_UID_0; + args->a1 = OPTEE_MSG_UID_1; + args->a2 = OPTEE_MSG_UID_2; + args->a3 = OPTEE_MSG_UID_3; +} + +void __weak tee_entry_get_api_revision(struct thread_smc_args *args) +{ + args->a0 = OPTEE_MSG_REVISION_MAJOR; + args->a1 = OPTEE_MSG_REVISION_MINOR; +} + +void __weak tee_entry_get_os_uuid(struct thread_smc_args *args) +{ + args->a0 = OPTEE_MSG_OS_OPTEE_UUID_0; + args->a1 = OPTEE_MSG_OS_OPTEE_UUID_1; + args->a2 = OPTEE_MSG_OS_OPTEE_UUID_2; + args->a3 = OPTEE_MSG_OS_OPTEE_UUID_3; +} + +void __weak tee_entry_get_os_revision(struct thread_smc_args *args) +{ + args->a0 = CFG_OPTEE_REVISION_MAJOR; + args->a1 = CFG_OPTEE_REVISION_MINOR; + args->a2 = TEE_IMPL_GIT_SHA1; +} diff --git a/optee_os/core/arch/arm/tee/sub.mk b/optee_os/core/arch/arm/tee/sub.mk new file mode 100644 index 0000000..5523b3b --- /dev/null +++ b/optee_os/core/arch/arm/tee/sub.mk @@ -0,0 +1,8 @@ +ifeq ($(CFG_WITH_USER_TA),y) +srcs-$(CFG_CACHE_API) += svc_cache.c +endif +ifneq ($(CFG_CORE_FFA),y) +srcs-y += entry_fast.c +cppflags-entry_fast.c-y += -DTEE_IMPL_GIT_SHA1=$(TEE_IMPL_GIT_SHA1) +endif +srcs-y += cache.c diff --git a/optee_os/core/arch/arm/tee/svc_cache.c b/optee_os/core/arch/arm/tee/svc_cache.c new file mode 100644 index 0000000..304c4d9 --- /dev/null +++ b/optee_os/core/arch/arm/tee/svc_cache.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include +#include + +TEE_Result syscall_cache_operation(void *va, size_t len, unsigned long op) +{ + struct ts_session *s = ts_get_current_session(); + struct user_ta_ctx *utc = NULL; + TEE_Result res = TEE_SUCCESS; + + if ((to_ta_ctx(s->ctx)->flags & TA_FLAG_CACHE_MAINTENANCE) == 0) + return TEE_ERROR_NOT_SUPPORTED; + + utc = to_user_ta_ctx(s->ctx); + + /* + * TAs are allowed to operate cache maintenance on TA memref parameters + * only, not on the TA private memory. + */ + if (vm_buf_intersects_um_private(&utc->uctx, va, len)) + return TEE_ERROR_ACCESS_DENIED; + + res = vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)va, len); + if (res != TEE_SUCCESS) + return TEE_ERROR_ACCESS_DENIED; + + return cache_operation(op, va, len); +} diff --git a/optee_os/core/arch/riscv/include/encoding.h b/optee_os/core/arch/riscv/include/encoding.h new file mode 100644 index 0000000..5ab5c76 --- /dev/null +++ b/optee_os/core/arch/riscv/include/encoding.h @@ -0,0 +1,4892 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* Copyright (c) 2022 RISC-V International */ + +/* + * This file is auto-generated by running 'make' in + * https://github.com/riscv/riscv-opcodes (86edbf4) + */ + +#ifndef RISCV_CSR_ENCODING_H +#define RISCV_CSR_ENCODING_H + +#define MSTATUS_UIE 0x00000001 +#define MSTATUS_SIE 0x00000002 +#define MSTATUS_HIE 0x00000004 +#define MSTATUS_MIE 0x00000008 +#define MSTATUS_UPIE 0x00000010 +#define MSTATUS_SPIE 0x00000020 +#define MSTATUS_UBE 0x00000040 +#define MSTATUS_MPIE 0x00000080 +#define MSTATUS_SPP 0x00000100 +#define MSTATUS_VS 0x00000600 +#define MSTATUS_MPP 0x00001800 +#define MSTATUS_FS 0x00006000 +#define MSTATUS_XS 0x00018000 +#define MSTATUS_MPRV 0x00020000 +#define MSTATUS_SUM 0x00040000 +#define MSTATUS_MXR 0x00080000 +#define MSTATUS_TVM 0x00100000 +#define MSTATUS_TW 0x00200000 +#define MSTATUS_TSR 0x00400000 +#define MSTATUS32_SD 0x80000000 +#define MSTATUS_UXL 0x0000000300000000 +#define MSTATUS_SXL 0x0000000C00000000 +#define MSTATUS_SBE 0x0000001000000000 +#define MSTATUS_MBE 0x0000002000000000 +#define MSTATUS_GVA 0x0000004000000000 +#define MSTATUS_MPV 0x0000008000000000 +#define MSTATUS64_SD 0x8000000000000000 + +#define MSTATUSH_SBE 0x00000010 +#define MSTATUSH_MBE 0x00000020 +#define MSTATUSH_GVA 0x00000040 +#define MSTATUSH_MPV 0x00000080 + +#define SSTATUS_UIE 0x00000001 +#define SSTATUS_SIE 0x00000002 +#define SSTATUS_UPIE 0x00000010 +#define SSTATUS_SPIE 0x00000020 +#define SSTATUS_UBE 0x00000040 +#define SSTATUS_SPP 0x00000100 +#define SSTATUS_VS 0x00000600 +#define SSTATUS_FS 0x00006000 +#define SSTATUS_XS 0x00018000 +#define SSTATUS_SUM 0x00040000 +#define SSTATUS_MXR 0x00080000 +#define SSTATUS32_SD 0x80000000 +#define SSTATUS_UXL 0x0000000300000000 +#define SSTATUS64_SD 0x8000000000000000 + +#define HSTATUS_VSXL 0x300000000 +#define HSTATUS_VTSR 0x00400000 +#define HSTATUS_VTW 0x00200000 +#define HSTATUS_VTVM 0x00100000 +#define HSTATUS_VGEIN 0x0003f000 +#define HSTATUS_HU 0x00000200 +#define HSTATUS_SPVP 0x00000100 +#define HSTATUS_SPV 0x00000080 +#define HSTATUS_GVA 0x00000040 +#define HSTATUS_VSBE 0x00000020 + +#define USTATUS_UIE 0x00000001 +#define USTATUS_UPIE 0x00000010 + +#define DCSR_XDEBUGVER (3U<<30) +#define DCSR_NDRESET (1<<29) +#define DCSR_FULLRESET (1<<28) +#define DCSR_EBREAKM (1<<15) +#define DCSR_EBREAKH (1<<14) +#define DCSR_EBREAKS (1<<13) +#define DCSR_EBREAKU (1<<12) +#define DCSR_STOPCYCLE (1<<10) +#define DCSR_STOPTIME (1<<9) +#define DCSR_CAUSE (7<<6) +#define DCSR_DEBUGINT (1<<5) +#define DCSR_HALT (1<<3) +#define DCSR_STEP (1<<2) +#define DCSR_PRV (3<<0) + +#define DCSR_CAUSE_NONE 0 +#define DCSR_CAUSE_SWBP 1 +#define DCSR_CAUSE_HWBP 2 +#define DCSR_CAUSE_DEBUGINT 3 +#define DCSR_CAUSE_STEP 4 +#define DCSR_CAUSE_HALT 5 +#define DCSR_CAUSE_GROUP 6 + +#define MCONTROL_TYPE(xlen) (0xfULL<<((xlen)-4)) +#define MCONTROL_DMODE(xlen) (1ULL<<((xlen)-5)) +#define MCONTROL_MASKMAX(xlen) (0x3fULL<<((xlen)-11)) + +#define MCONTROL_SELECT (1<<19) +#define MCONTROL_TIMING (1<<18) +#define MCONTROL_ACTION (0x3f<<12) +#define MCONTROL_CHAIN (1<<11) +#define MCONTROL_MATCH (0xf<<7) +#define MCONTROL_M (1<<6) +#define MCONTROL_H (1<<5) +#define MCONTROL_S (1<<4) +#define MCONTROL_U (1<<3) +#define MCONTROL_EXECUTE (1<<2) +#define MCONTROL_STORE (1<<1) +#define MCONTROL_LOAD (1<<0) + +#define MCONTROL_TYPE_NONE 0 +#define MCONTROL_TYPE_MATCH 2 + +#define MCONTROL_ACTION_DEBUG_EXCEPTION 0 +#define MCONTROL_ACTION_DEBUG_MODE 1 +#define MCONTROL_ACTION_TRACE_START 2 +#define MCONTROL_ACTION_TRACE_STOP 3 +#define MCONTROL_ACTION_TRACE_EMIT 4 + +#define MCONTROL_MATCH_EQUAL 0 +#define MCONTROL_MATCH_NAPOT 1 +#define MCONTROL_MATCH_GE 2 +#define MCONTROL_MATCH_LT 3 +#define MCONTROL_MATCH_MASK_LOW 4 +#define MCONTROL_MATCH_MASK_HIGH 5 + +#define MIP_USIP (1 << IRQ_U_SOFT) +#define MIP_SSIP (1 << IRQ_S_SOFT) +#define MIP_VSSIP (1 << IRQ_VS_SOFT) +#define MIP_MSIP (1 << IRQ_M_SOFT) +#define MIP_UTIP (1 << IRQ_U_TIMER) +#define MIP_STIP (1 << IRQ_S_TIMER) +#define MIP_VSTIP (1 << IRQ_VS_TIMER) +#define MIP_MTIP (1 << IRQ_M_TIMER) +#define MIP_UEIP (1 << IRQ_U_EXT) +#define MIP_SEIP (1 << IRQ_S_EXT) +#define MIP_VSEIP (1 << IRQ_VS_EXT) +#define MIP_MEIP (1 << IRQ_M_EXT) +#define MIP_SGEIP (1 << IRQ_S_GEXT) +#define MIP_LCOFIP (1 << IRQ_LCOF) + +#define MIP_S_MASK (MIP_SSIP | MIP_STIP | MIP_SEIP) +#define MIP_VS_MASK (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP) +#define MIP_HS_MASK (MIP_VS_MASK | MIP_SGEIP) + +#define MIDELEG_FORCED_MASK MIP_HS_MASK + +#define SIP_SSIP MIP_SSIP +#define SIP_STIP MIP_STIP + +#define MENVCFG_FIOM 0x00000001 +#define MENVCFG_CBIE 0x00000030 +#define MENVCFG_CBCFE 0x00000040 +#define MENVCFG_CBZE 0x00000080 +#define MENVCFG_PBMTE 0x4000000000000000 +#define MENVCFG_STCE 0x8000000000000000 + +#define MENVCFGH_PBMTE 0x40000000 +#define MENVCFGH_STCE 0x80000000 + +#define MSTATEEN0_CS 0x00000001 +#define MSTATEEN0_FCSR 0x00000002 +#define MSTATEEN0_HCONTEXT 0x0200000000000000 +#define MSTATEEN0_HENVCFG 0x4000000000000000 +#define MSTATEEN_HSTATEEN 0x8000000000000000 + +#define MSTATEEN0H_HCONTEXT 0x02000000 +#define MSTATEEN0H_HENVCFG 0x40000000 +#define MSTATEENH_HSTATEEN 0x80000000 + +#define MHPMEVENT_VUINH 0x0400000000000000 +#define MHPMEVENT_VSINH 0x0800000000000000 +#define MHPMEVENT_UINH 0x1000000000000000 +#define MHPMEVENT_SINH 0x2000000000000000 +#define MHPMEVENT_MINH 0x4000000000000000 +#define MHPMEVENT_OF 0x8000000000000000 + +#define MHPMEVENTH_VUINH 0x04000000 +#define MHPMEVENTH_VSINH 0x08000000 +#define MHPMEVENTH_UINH 0x10000000 +#define MHPMEVENTH_SINH 0x20000000 +#define MHPMEVENTH_MINH 0x40000000 +#define MHPMEVENTH_OF 0x80000000 + +#define HENVCFG_FIOM 0x00000001 +#define HENVCFG_CBIE 0x00000030 +#define HENVCFG_CBCFE 0x00000040 +#define HENVCFG_CBZE 0x00000080 +#define HENVCFG_PBMTE 0x4000000000000000 +#define HENVCFG_STCE 0x8000000000000000 + +#define HENVCFGH_PBMTE 0x40000000 +#define HENVCFGH_STCE 0x80000000 + +#define HSTATEEN0_CS 0x00000001 +#define HSTATEEN0_FCSR 0x00000002 +#define HSTATEEN0_SCONTEXT 0x0200000000000000 +#define HSTATEEN0_SENVCFG 0x4000000000000000 +#define HSTATEEN_SSTATEEN 0x8000000000000000 + +#define HSTATEEN0H_SCONTEXT 0x02000000 +#define HSTATEEN0H_SENVCFG 0x40000000 +#define HSTATEENH_SSTATEEN 0x80000000 + +#define SENVCFG_FIOM 0x00000001 +#define SENVCFG_CBIE 0x00000030 +#define SENVCFG_CBCFE 0x00000040 +#define SENVCFG_CBZE 0x00000080 + +#define SSTATEEN0_CS 0x00000001 +#define SSTATEEN0_FCSR 0x00000002 + +#define MSECCFG_MML 0x00000001 +#define MSECCFG_MMWP 0x00000002 +#define MSECCFG_RLB 0x00000004 +#define MSECCFG_USEED 0x00000100 +#define MSECCFG_SSEED 0x00000200 + +#define PRV_U 0 +#define PRV_S 1 +#define PRV_M 3 + +#define PRV_HS (PRV_S + 1) + +#define SATP32_MODE 0x80000000 +#define SATP32_ASID 0x7FC00000 +#define SATP32_PPN 0x003FFFFF +#define SATP64_MODE 0xF000000000000000 +#define SATP64_ASID 0x0FFFF00000000000 +#define SATP64_PPN 0x00000FFFFFFFFFFF + +#define SATP_MODE_OFF 0 +#define SATP_MODE_SV32 1 +#define SATP_MODE_SV39 8 +#define SATP_MODE_SV48 9 +#define SATP_MODE_SV57 10 +#define SATP_MODE_SV64 11 + +#define HGATP32_MODE 0x80000000 +#define HGATP32_VMID 0x1FC00000 +#define HGATP32_PPN 0x003FFFFF + +#define HGATP64_MODE 0xF000000000000000 +#define HGATP64_VMID 0x03FFF00000000000 +#define HGATP64_PPN 0x00000FFFFFFFFFFF + +#define HGATP_MODE_OFF 0 +#define HGATP_MODE_SV32X4 1 +#define HGATP_MODE_SV39X4 8 +#define HGATP_MODE_SV48X4 9 +#define HGATP_MODE_SV57X4 10 + +#define PMP_R 0x01 +#define PMP_W 0x02 +#define PMP_X 0x04 +#define PMP_A 0x18 +#define PMP_L 0x80 +#define PMP_SHIFT 2 + +#define PMP_TOR 0x08 +#define PMP_NA4 0x10 +#define PMP_NAPOT 0x18 + +#define IRQ_U_SOFT 0 +#define IRQ_S_SOFT 1 +#define IRQ_VS_SOFT 2 +#define IRQ_M_SOFT 3 +#define IRQ_U_TIMER 4 +#define IRQ_S_TIMER 5 +#define IRQ_VS_TIMER 6 +#define IRQ_M_TIMER 7 +#define IRQ_U_EXT 8 +#define IRQ_S_EXT 9 +#define IRQ_VS_EXT 10 +#define IRQ_M_EXT 11 +#define IRQ_S_GEXT 12 +#define IRQ_COP 12 +#define IRQ_LCOF 13 + +/* page table entry (PTE) fields */ +#define PTE_V 0x001 /* Valid */ +#define PTE_R 0x002 /* Read */ +#define PTE_W 0x004 /* Write */ +#define PTE_X 0x008 /* Execute */ +#define PTE_U 0x010 /* User */ +#define PTE_G 0x020 /* Global */ +#define PTE_A 0x040 /* Accessed */ +#define PTE_D 0x080 /* Dirty */ +#define PTE_SOFT 0x300 /* Reserved for Software */ +#define PTE_RSVD 0x1FC0000000000000 /* Reserved for future standard use */ +#define PTE_PBMT 0x6000000000000000 /* Svpbmt: Page-based memory types */ +#define PTE_N 0x8000000000000000 /* Svnapot: NAPOT translation contiguity */ +#define PTE_ATTR 0xFFC0000000000000 /* All attributes and reserved bits */ + +#define PTE_PPN_SHIFT 10 + +#define PTE_TABLE(PTE) (((PTE) & (PTE_V | PTE_R | PTE_W | PTE_X)) == PTE_V) + +#ifdef __riscv + +#if __riscv_xlen == 64 +# define MSTATUS_SD MSTATUS64_SD +# define SSTATUS_SD SSTATUS64_SD +# define RISCV_PGLEVEL_BITS 9 +# define SATP_MODE SATP64_MODE +#else +# define MSTATUS_SD MSTATUS32_SD +# define SSTATUS_SD SSTATUS32_SD +# define RISCV_PGLEVEL_BITS 10 +# define SATP_MODE SATP32_MODE +#endif +#define RISCV_PGSHIFT 12 +#define RISCV_PGSIZE (1 << RISCV_PGSHIFT) + +#endif + +#endif + +/* Automatically generated by parse_opcodes. */ +#ifndef RISCV_ENCODING_H +#define RISCV_ENCODING_H +#define MATCH_ADD 0x33 +#define MASK_ADD 0xfe00707f +#define MATCH_ADD16 0x40000077 +#define MASK_ADD16 0xfe00707f +#define MATCH_ADD32 0x40002077 +#define MASK_ADD32 0xfe00707f +#define MATCH_ADD64 0xc0001077 +#define MASK_ADD64 0xfe00707f +#define MATCH_ADD8 0x48000077 +#define MASK_ADD8 0xfe00707f +#define MATCH_ADD_UW 0x800003b +#define MASK_ADD_UW 0xfe00707f +#define MATCH_ADDD 0x7b +#define MASK_ADDD 0xfe00707f +#define MATCH_ADDI 0x13 +#define MASK_ADDI 0x707f +#define MATCH_ADDID 0x5b +#define MASK_ADDID 0x707f +#define MATCH_ADDIW 0x1b +#define MASK_ADDIW 0x707f +#define MATCH_ADDW 0x3b +#define MASK_ADDW 0xfe00707f +#define MATCH_AES32DSI 0x2a000033 +#define MASK_AES32DSI 0x3e00707f +#define MATCH_AES32DSMI 0x2e000033 +#define MASK_AES32DSMI 0x3e00707f +#define MATCH_AES32ESI 0x22000033 +#define MASK_AES32ESI 0x3e00707f +#define MATCH_AES32ESMI 0x26000033 +#define MASK_AES32ESMI 0x3e00707f +#define MATCH_AES64DS 0x3a000033 +#define MASK_AES64DS 0xfe00707f +#define MATCH_AES64DSM 0x3e000033 +#define MASK_AES64DSM 0xfe00707f +#define MATCH_AES64ES 0x32000033 +#define MASK_AES64ES 0xfe00707f +#define MATCH_AES64ESM 0x36000033 +#define MASK_AES64ESM 0xfe00707f +#define MATCH_AES64IM 0x30001013 +#define MASK_AES64IM 0xfff0707f +#define MATCH_AES64KS1I 0x31001013 +#define MASK_AES64KS1I 0xff00707f +#define MATCH_AES64KS2 0x7e000033 +#define MASK_AES64KS2 0xfe00707f +#define MATCH_AMOADD_D 0x302f +#define MASK_AMOADD_D 0xf800707f +#define MATCH_AMOADD_W 0x202f +#define MASK_AMOADD_W 0xf800707f +#define MATCH_AMOAND_D 0x6000302f +#define MASK_AMOAND_D 0xf800707f +#define MATCH_AMOAND_W 0x6000202f +#define MASK_AMOAND_W 0xf800707f +#define MATCH_AMOMAX_D 0xa000302f +#define MASK_AMOMAX_D 0xf800707f +#define MATCH_AMOMAX_W 0xa000202f +#define MASK_AMOMAX_W 0xf800707f +#define MATCH_AMOMAXU_D 0xe000302f +#define MASK_AMOMAXU_D 0xf800707f +#define MATCH_AMOMAXU_W 0xe000202f +#define MASK_AMOMAXU_W 0xf800707f +#define MATCH_AMOMIN_D 0x8000302f +#define MASK_AMOMIN_D 0xf800707f +#define MATCH_AMOMIN_W 0x8000202f +#define MASK_AMOMIN_W 0xf800707f +#define MATCH_AMOMINU_D 0xc000302f +#define MASK_AMOMINU_D 0xf800707f +#define MATCH_AMOMINU_W 0xc000202f +#define MASK_AMOMINU_W 0xf800707f +#define MATCH_AMOOR_D 0x4000302f +#define MASK_AMOOR_D 0xf800707f +#define MATCH_AMOOR_W 0x4000202f +#define MASK_AMOOR_W 0xf800707f +#define MATCH_AMOSWAP_D 0x800302f +#define MASK_AMOSWAP_D 0xf800707f +#define MATCH_AMOSWAP_W 0x800202f +#define MASK_AMOSWAP_W 0xf800707f +#define MATCH_AMOXOR_D 0x2000302f +#define MASK_AMOXOR_D 0xf800707f +#define MATCH_AMOXOR_W 0x2000202f +#define MASK_AMOXOR_W 0xf800707f +#define MATCH_AND 0x7033 +#define MASK_AND 0xfe00707f +#define MATCH_ANDI 0x7013 +#define MASK_ANDI 0x707f +#define MATCH_ANDN 0x40007033 +#define MASK_ANDN 0xfe00707f +#define MATCH_AUIPC 0x17 +#define MASK_AUIPC 0x7f +#define MATCH_AVE 0xe0000077 +#define MASK_AVE 0xfe00707f +#define MATCH_BCLR 0x48001033 +#define MASK_BCLR 0xfe00707f +#define MATCH_BCLRI 0x48001013 +#define MASK_BCLRI 0xfc00707f +#define MATCH_BCOMPRESS 0x8006033 +#define MASK_BCOMPRESS 0xfe00707f +#define MATCH_BCOMPRESSW 0x800603b +#define MASK_BCOMPRESSW 0xfe00707f +#define MATCH_BDECOMPRESS 0x48006033 +#define MASK_BDECOMPRESS 0xfe00707f +#define MATCH_BDECOMPRESSW 0x4800603b +#define MASK_BDECOMPRESSW 0xfe00707f +#define MATCH_BEQ 0x63 +#define MASK_BEQ 0x707f +#define MATCH_BEXT 0x48005033 +#define MASK_BEXT 0xfe00707f +#define MATCH_BEXTI 0x48005013 +#define MASK_BEXTI 0xfc00707f +#define MATCH_BFP 0x48007033 +#define MASK_BFP 0xfe00707f +#define MATCH_BFPW 0x4800703b +#define MASK_BFPW 0xfe00707f +#define MATCH_BGE 0x5063 +#define MASK_BGE 0x707f +#define MATCH_BGEU 0x7063 +#define MASK_BGEU 0x707f +#define MATCH_BINV 0x68001033 +#define MASK_BINV 0xfe00707f +#define MATCH_BINVI 0x68001013 +#define MASK_BINVI 0xfc00707f +#define MATCH_BITREV 0xe6000077 +#define MASK_BITREV 0xfe00707f +#define MATCH_BITREVI 0xe8000077 +#define MASK_BITREVI 0xfc00707f +#define MATCH_BLT 0x4063 +#define MASK_BLT 0x707f +#define MATCH_BLTU 0x6063 +#define MASK_BLTU 0x707f +#define MATCH_BMATFLIP 0x60301013 +#define MASK_BMATFLIP 0xfff0707f +#define MATCH_BMATOR 0x8003033 +#define MASK_BMATOR 0xfe00707f +#define MATCH_BMATXOR 0x48003033 +#define MASK_BMATXOR 0xfe00707f +#define MATCH_BNE 0x1063 +#define MASK_BNE 0x707f +#define MATCH_BPICK 0x3077 +#define MASK_BPICK 0x600707f +#define MATCH_BSET 0x28001033 +#define MASK_BSET 0xfe00707f +#define MATCH_BSETI 0x28001013 +#define MASK_BSETI 0xfc00707f +#define MATCH_C_ADD 0x9002 +#define MASK_C_ADD 0xf003 +#define MATCH_C_ADDI 0x1 +#define MASK_C_ADDI 0xe003 +#define MATCH_C_ADDI16SP 0x6101 +#define MASK_C_ADDI16SP 0xef83 +#define MATCH_C_ADDI4SPN 0x0 +#define MASK_C_ADDI4SPN 0xe003 +#define MATCH_C_ADDIW 0x2001 +#define MASK_C_ADDIW 0xe003 +#define MATCH_C_ADDW 0x9c21 +#define MASK_C_ADDW 0xfc63 +#define MATCH_C_AND 0x8c61 +#define MASK_C_AND 0xfc63 +#define MATCH_C_ANDI 0x8801 +#define MASK_C_ANDI 0xec03 +#define MATCH_C_BEQZ 0xc001 +#define MASK_C_BEQZ 0xe003 +#define MATCH_C_BNEZ 0xe001 +#define MASK_C_BNEZ 0xe003 +#define MATCH_C_EBREAK 0x9002 +#define MASK_C_EBREAK 0xffff +#define MATCH_C_FLD 0x2000 +#define MASK_C_FLD 0xe003 +#define MATCH_C_FLDSP 0x2002 +#define MASK_C_FLDSP 0xe003 +#define MATCH_C_FLW 0x6000 +#define MASK_C_FLW 0xe003 +#define MATCH_C_FLWSP 0x6002 +#define MASK_C_FLWSP 0xe003 +#define MATCH_C_FSD 0xa000 +#define MASK_C_FSD 0xe003 +#define MATCH_C_FSDSP 0xa002 +#define MASK_C_FSDSP 0xe003 +#define MATCH_C_FSW 0xe000 +#define MASK_C_FSW 0xe003 +#define MATCH_C_FSWSP 0xe002 +#define MASK_C_FSWSP 0xe003 +#define MATCH_C_J 0xa001 +#define MASK_C_J 0xe003 +#define MATCH_C_JAL 0x2001 +#define MASK_C_JAL 0xe003 +#define MATCH_C_JALR 0x9002 +#define MASK_C_JALR 0xf07f +#define MATCH_C_JR 0x8002 +#define MASK_C_JR 0xf07f +#define MATCH_C_LD 0x6000 +#define MASK_C_LD 0xe003 +#define MATCH_C_LDSP 0x6002 +#define MASK_C_LDSP 0xe003 +#define MATCH_C_LI 0x4001 +#define MASK_C_LI 0xe003 +#define MATCH_C_LQ 0x2000 +#define MASK_C_LQ 0xe003 +#define MATCH_C_LQSP 0x2002 +#define MASK_C_LQSP 0xe003 +#define MATCH_C_LUI 0x6001 +#define MASK_C_LUI 0xe003 +#define MATCH_C_LW 0x4000 +#define MASK_C_LW 0xe003 +#define MATCH_C_LWSP 0x4002 +#define MASK_C_LWSP 0xe003 +#define MATCH_C_MV 0x8002 +#define MASK_C_MV 0xf003 +#define MATCH_C_NOP 0x1 +#define MASK_C_NOP 0xef83 +#define MATCH_C_OR 0x8c41 +#define MASK_C_OR 0xfc63 +#define MATCH_C_SD 0xe000 +#define MASK_C_SD 0xe003 +#define MATCH_C_SDSP 0xe002 +#define MASK_C_SDSP 0xe003 +#define MATCH_C_SLLI 0x2 +#define MASK_C_SLLI 0xe003 +#define MATCH_C_SQ 0xa000 +#define MASK_C_SQ 0xe003 +#define MATCH_C_SQSP 0xa002 +#define MASK_C_SQSP 0xe003 +#define MATCH_C_SRAI 0x8401 +#define MASK_C_SRAI 0xec03 +#define MATCH_C_SRLI 0x8001 +#define MASK_C_SRLI 0xec03 +#define MATCH_C_SUB 0x8c01 +#define MASK_C_SUB 0xfc63 +#define MATCH_C_SUBW 0x9c01 +#define MASK_C_SUBW 0xfc63 +#define MATCH_C_SW 0xc000 +#define MASK_C_SW 0xe003 +#define MATCH_C_SWSP 0xc002 +#define MASK_C_SWSP 0xe003 +#define MATCH_C_XOR 0x8c21 +#define MASK_C_XOR 0xfc63 +#define MATCH_CBO_CLEAN 0x10200f +#define MASK_CBO_CLEAN 0xfff07fff +#define MATCH_CBO_FLUSH 0x20200f +#define MASK_CBO_FLUSH 0xfff07fff +#define MATCH_CBO_INVAL 0x200f +#define MASK_CBO_INVAL 0xfff07fff +#define MATCH_CBO_ZERO 0x40200f +#define MASK_CBO_ZERO 0xfff07fff +#define MATCH_CLMUL 0xa001033 +#define MASK_CLMUL 0xfe00707f +#define MATCH_CLMULH 0xa003033 +#define MASK_CLMULH 0xfe00707f +#define MATCH_CLMULR 0xa002033 +#define MASK_CLMULR 0xfe00707f +#define MATCH_CLO16 0xaeb00077 +#define MASK_CLO16 0xfff0707f +#define MATCH_CLO32 0xafb00077 +#define MASK_CLO32 0xfff0707f +#define MATCH_CLO8 0xae300077 +#define MASK_CLO8 0xfff0707f +#define MATCH_CLRS16 0xae800077 +#define MASK_CLRS16 0xfff0707f +#define MATCH_CLRS32 0xaf800077 +#define MASK_CLRS32 0xfff0707f +#define MATCH_CLRS8 0xae000077 +#define MASK_CLRS8 0xfff0707f +#define MATCH_CLZ 0x60001013 +#define MASK_CLZ 0xfff0707f +#define MATCH_CLZ16 0xae900077 +#define MASK_CLZ16 0xfff0707f +#define MATCH_CLZ32 0xaf900077 +#define MASK_CLZ32 0xfff0707f +#define MATCH_CLZ8 0xae100077 +#define MASK_CLZ8 0xfff0707f +#define MATCH_CLZW 0x6000101b +#define MASK_CLZW 0xfff0707f +#define MATCH_CMIX 0x6001033 +#define MASK_CMIX 0x600707f +#define MATCH_CMOV 0x6005033 +#define MASK_CMOV 0x600707f +#define MATCH_CMPEQ16 0x4c000077 +#define MASK_CMPEQ16 0xfe00707f +#define MATCH_CMPEQ8 0x4e000077 +#define MASK_CMPEQ8 0xfe00707f +#define MATCH_CPOP 0x60201013 +#define MASK_CPOP 0xfff0707f +#define MATCH_CPOPW 0x6020101b +#define MASK_CPOPW 0xfff0707f +#define MATCH_CRAS16 0x44000077 +#define MASK_CRAS16 0xfe00707f +#define MATCH_CRAS32 0x44002077 +#define MASK_CRAS32 0xfe00707f +#define MATCH_CRC32_B 0x61001013 +#define MASK_CRC32_B 0xfff0707f +#define MATCH_CRC32_D 0x61301013 +#define MASK_CRC32_D 0xfff0707f +#define MATCH_CRC32_H 0x61101013 +#define MASK_CRC32_H 0xfff0707f +#define MATCH_CRC32_W 0x61201013 +#define MASK_CRC32_W 0xfff0707f +#define MATCH_CRC32C_B 0x61801013 +#define MASK_CRC32C_B 0xfff0707f +#define MATCH_CRC32C_D 0x61b01013 +#define MASK_CRC32C_D 0xfff0707f +#define MATCH_CRC32C_H 0x61901013 +#define MASK_CRC32C_H 0xfff0707f +#define MATCH_CRC32C_W 0x61a01013 +#define MASK_CRC32C_W 0xfff0707f +#define MATCH_CRSA16 0x46000077 +#define MASK_CRSA16 0xfe00707f +#define MATCH_CRSA32 0x46002077 +#define MASK_CRSA32 0xfe00707f +#define MATCH_CSRRC 0x3073 +#define MASK_CSRRC 0x707f +#define MATCH_CSRRCI 0x7073 +#define MASK_CSRRCI 0x707f +#define MATCH_CSRRS 0x2073 +#define MASK_CSRRS 0x707f +#define MATCH_CSRRSI 0x6073 +#define MASK_CSRRSI 0x707f +#define MATCH_CSRRW 0x1073 +#define MASK_CSRRW 0x707f +#define MATCH_CSRRWI 0x5073 +#define MASK_CSRRWI 0x707f +#define MATCH_CTZ 0x60101013 +#define MASK_CTZ 0xfff0707f +#define MATCH_CTZW 0x6010101b +#define MASK_CTZW 0xfff0707f +#define MATCH_DIV 0x2004033 +#define MASK_DIV 0xfe00707f +#define MATCH_DIVU 0x2005033 +#define MASK_DIVU 0xfe00707f +#define MATCH_DIVUW 0x200503b +#define MASK_DIVUW 0xfe00707f +#define MATCH_DIVW 0x200403b +#define MASK_DIVW 0xfe00707f +#define MATCH_DRET 0x7b200073 +#define MASK_DRET 0xffffffff +#define MATCH_EBREAK 0x100073 +#define MASK_EBREAK 0xffffffff +#define MATCH_ECALL 0x73 +#define MASK_ECALL 0xffffffff +#define MATCH_FADD_D 0x2000053 +#define MASK_FADD_D 0xfe00007f +#define MATCH_FADD_H 0x4000053 +#define MASK_FADD_H 0xfe00007f +#define MATCH_FADD_Q 0x6000053 +#define MASK_FADD_Q 0xfe00007f +#define MATCH_FADD_S 0x53 +#define MASK_FADD_S 0xfe00007f +#define MATCH_FCLASS_D 0xe2001053 +#define MASK_FCLASS_D 0xfff0707f +#define MATCH_FCLASS_H 0xe4001053 +#define MASK_FCLASS_H 0xfff0707f +#define MATCH_FCLASS_Q 0xe6001053 +#define MASK_FCLASS_Q 0xfff0707f +#define MATCH_FCLASS_S 0xe0001053 +#define MASK_FCLASS_S 0xfff0707f +#define MATCH_FCVT_D_H 0x42200053 +#define MASK_FCVT_D_H 0xfff0007f +#define MATCH_FCVT_D_L 0xd2200053 +#define MASK_FCVT_D_L 0xfff0007f +#define MATCH_FCVT_D_LU 0xd2300053 +#define MASK_FCVT_D_LU 0xfff0007f +#define MATCH_FCVT_D_Q 0x42300053 +#define MASK_FCVT_D_Q 0xfff0007f +#define MATCH_FCVT_D_S 0x42000053 +#define MASK_FCVT_D_S 0xfff0007f +#define MATCH_FCVT_D_W 0xd2000053 +#define MASK_FCVT_D_W 0xfff0007f +#define MATCH_FCVT_D_WU 0xd2100053 +#define MASK_FCVT_D_WU 0xfff0007f +#define MATCH_FCVT_H_D 0x44100053 +#define MASK_FCVT_H_D 0xfff0007f +#define MATCH_FCVT_H_L 0xd4200053 +#define MASK_FCVT_H_L 0xfff0007f +#define MATCH_FCVT_H_LU 0xd4300053 +#define MASK_FCVT_H_LU 0xfff0007f +#define MATCH_FCVT_H_Q 0x44300053 +#define MASK_FCVT_H_Q 0xfff0007f +#define MATCH_FCVT_H_S 0x44000053 +#define MASK_FCVT_H_S 0xfff0007f +#define MATCH_FCVT_H_W 0xd4000053 +#define MASK_FCVT_H_W 0xfff0007f +#define MATCH_FCVT_H_WU 0xd4100053 +#define MASK_FCVT_H_WU 0xfff0007f +#define MATCH_FCVT_L_D 0xc2200053 +#define MASK_FCVT_L_D 0xfff0007f +#define MATCH_FCVT_L_H 0xc4200053 +#define MASK_FCVT_L_H 0xfff0007f +#define MATCH_FCVT_L_Q 0xc6200053 +#define MASK_FCVT_L_Q 0xfff0007f +#define MATCH_FCVT_L_S 0xc0200053 +#define MASK_FCVT_L_S 0xfff0007f +#define MATCH_FCVT_LU_D 0xc2300053 +#define MASK_FCVT_LU_D 0xfff0007f +#define MATCH_FCVT_LU_H 0xc4300053 +#define MASK_FCVT_LU_H 0xfff0007f +#define MATCH_FCVT_LU_Q 0xc6300053 +#define MASK_FCVT_LU_Q 0xfff0007f +#define MATCH_FCVT_LU_S 0xc0300053 +#define MASK_FCVT_LU_S 0xfff0007f +#define MATCH_FCVT_Q_D 0x46100053 +#define MASK_FCVT_Q_D 0xfff0007f +#define MATCH_FCVT_Q_H 0x46200053 +#define MASK_FCVT_Q_H 0xfff0007f +#define MATCH_FCVT_Q_L 0xd6200053 +#define MASK_FCVT_Q_L 0xfff0007f +#define MATCH_FCVT_Q_LU 0xd6300053 +#define MASK_FCVT_Q_LU 0xfff0007f +#define MATCH_FCVT_Q_S 0x46000053 +#define MASK_FCVT_Q_S 0xfff0007f +#define MATCH_FCVT_Q_W 0xd6000053 +#define MASK_FCVT_Q_W 0xfff0007f +#define MATCH_FCVT_Q_WU 0xd6100053 +#define MASK_FCVT_Q_WU 0xfff0007f +#define MATCH_FCVT_S_D 0x40100053 +#define MASK_FCVT_S_D 0xfff0007f +#define MATCH_FCVT_S_H 0x40200053 +#define MASK_FCVT_S_H 0xfff0007f +#define MATCH_FCVT_S_L 0xd0200053 +#define MASK_FCVT_S_L 0xfff0007f +#define MATCH_FCVT_S_LU 0xd0300053 +#define MASK_FCVT_S_LU 0xfff0007f +#define MATCH_FCVT_S_Q 0x40300053 +#define MASK_FCVT_S_Q 0xfff0007f +#define MATCH_FCVT_S_W 0xd0000053 +#define MASK_FCVT_S_W 0xfff0007f +#define MATCH_FCVT_S_WU 0xd0100053 +#define MASK_FCVT_S_WU 0xfff0007f +#define MATCH_FCVT_W_D 0xc2000053 +#define MASK_FCVT_W_D 0xfff0007f +#define MATCH_FCVT_W_H 0xc4000053 +#define MASK_FCVT_W_H 0xfff0007f +#define MATCH_FCVT_W_Q 0xc6000053 +#define MASK_FCVT_W_Q 0xfff0007f +#define MATCH_FCVT_W_S 0xc0000053 +#define MASK_FCVT_W_S 0xfff0007f +#define MATCH_FCVT_WU_D 0xc2100053 +#define MASK_FCVT_WU_D 0xfff0007f +#define MATCH_FCVT_WU_H 0xc4100053 +#define MASK_FCVT_WU_H 0xfff0007f +#define MATCH_FCVT_WU_Q 0xc6100053 +#define MASK_FCVT_WU_Q 0xfff0007f +#define MATCH_FCVT_WU_S 0xc0100053 +#define MASK_FCVT_WU_S 0xfff0007f +#define MATCH_FDIV_D 0x1a000053 +#define MASK_FDIV_D 0xfe00007f +#define MATCH_FDIV_H 0x1c000053 +#define MASK_FDIV_H 0xfe00007f +#define MATCH_FDIV_Q 0x1e000053 +#define MASK_FDIV_Q 0xfe00007f +#define MATCH_FDIV_S 0x18000053 +#define MASK_FDIV_S 0xfe00007f +#define MATCH_FENCE 0xf +#define MASK_FENCE 0x707f +#define MATCH_FENCE_I 0x100f +#define MASK_FENCE_I 0x707f +#define MATCH_FEQ_D 0xa2002053 +#define MASK_FEQ_D 0xfe00707f +#define MATCH_FEQ_H 0xa4002053 +#define MASK_FEQ_H 0xfe00707f +#define MATCH_FEQ_Q 0xa6002053 +#define MASK_FEQ_Q 0xfe00707f +#define MATCH_FEQ_S 0xa0002053 +#define MASK_FEQ_S 0xfe00707f +#define MATCH_FLD 0x3007 +#define MASK_FLD 0x707f +#define MATCH_FLE_D 0xa2000053 +#define MASK_FLE_D 0xfe00707f +#define MATCH_FLE_H 0xa4000053 +#define MASK_FLE_H 0xfe00707f +#define MATCH_FLE_Q 0xa6000053 +#define MASK_FLE_Q 0xfe00707f +#define MATCH_FLE_S 0xa0000053 +#define MASK_FLE_S 0xfe00707f +#define MATCH_FLH 0x1007 +#define MASK_FLH 0x707f +#define MATCH_FLQ 0x4007 +#define MASK_FLQ 0x707f +#define MATCH_FLT_D 0xa2001053 +#define MASK_FLT_D 0xfe00707f +#define MATCH_FLT_H 0xa4001053 +#define MASK_FLT_H 0xfe00707f +#define MATCH_FLT_Q 0xa6001053 +#define MASK_FLT_Q 0xfe00707f +#define MATCH_FLT_S 0xa0001053 +#define MASK_FLT_S 0xfe00707f +#define MATCH_FLW 0x2007 +#define MASK_FLW 0x707f +#define MATCH_FMADD_D 0x2000043 +#define MASK_FMADD_D 0x600007f +#define MATCH_FMADD_H 0x4000043 +#define MASK_FMADD_H 0x600007f +#define MATCH_FMADD_Q 0x6000043 +#define MASK_FMADD_Q 0x600007f +#define MATCH_FMADD_S 0x43 +#define MASK_FMADD_S 0x600007f +#define MATCH_FMAX_D 0x2a001053 +#define MASK_FMAX_D 0xfe00707f +#define MATCH_FMAX_H 0x2c001053 +#define MASK_FMAX_H 0xfe00707f +#define MATCH_FMAX_Q 0x2e001053 +#define MASK_FMAX_Q 0xfe00707f +#define MATCH_FMAX_S 0x28001053 +#define MASK_FMAX_S 0xfe00707f +#define MATCH_FMIN_D 0x2a000053 +#define MASK_FMIN_D 0xfe00707f +#define MATCH_FMIN_H 0x2c000053 +#define MASK_FMIN_H 0xfe00707f +#define MATCH_FMIN_Q 0x2e000053 +#define MASK_FMIN_Q 0xfe00707f +#define MATCH_FMIN_S 0x28000053 +#define MASK_FMIN_S 0xfe00707f +#define MATCH_FMSUB_D 0x2000047 +#define MASK_FMSUB_D 0x600007f +#define MATCH_FMSUB_H 0x4000047 +#define MASK_FMSUB_H 0x600007f +#define MATCH_FMSUB_Q 0x6000047 +#define MASK_FMSUB_Q 0x600007f +#define MATCH_FMSUB_S 0x47 +#define MASK_FMSUB_S 0x600007f +#define MATCH_FMUL_D 0x12000053 +#define MASK_FMUL_D 0xfe00007f +#define MATCH_FMUL_H 0x14000053 +#define MASK_FMUL_H 0xfe00007f +#define MATCH_FMUL_Q 0x16000053 +#define MASK_FMUL_Q 0xfe00007f +#define MATCH_FMUL_S 0x10000053 +#define MASK_FMUL_S 0xfe00007f +#define MATCH_FMV_D_X 0xf2000053 +#define MASK_FMV_D_X 0xfff0707f +#define MATCH_FMV_H_X 0xf4000053 +#define MASK_FMV_H_X 0xfff0707f +#define MATCH_FMV_W_X 0xf0000053 +#define MASK_FMV_W_X 0xfff0707f +#define MATCH_FMV_X_D 0xe2000053 +#define MASK_FMV_X_D 0xfff0707f +#define MATCH_FMV_X_H 0xe4000053 +#define MASK_FMV_X_H 0xfff0707f +#define MATCH_FMV_X_W 0xe0000053 +#define MASK_FMV_X_W 0xfff0707f +#define MATCH_FNMADD_D 0x200004f +#define MASK_FNMADD_D 0x600007f +#define MATCH_FNMADD_H 0x400004f +#define MASK_FNMADD_H 0x600007f +#define MATCH_FNMADD_Q 0x600004f +#define MASK_FNMADD_Q 0x600007f +#define MATCH_FNMADD_S 0x4f +#define MASK_FNMADD_S 0x600007f +#define MATCH_FNMSUB_D 0x200004b +#define MASK_FNMSUB_D 0x600007f +#define MATCH_FNMSUB_H 0x400004b +#define MASK_FNMSUB_H 0x600007f +#define MATCH_FNMSUB_Q 0x600004b +#define MASK_FNMSUB_Q 0x600007f +#define MATCH_FNMSUB_S 0x4b +#define MASK_FNMSUB_S 0x600007f +#define MATCH_FSD 0x3027 +#define MASK_FSD 0x707f +#define MATCH_FSGNJ_D 0x22000053 +#define MASK_FSGNJ_D 0xfe00707f +#define MATCH_FSGNJ_H 0x24000053 +#define MASK_FSGNJ_H 0xfe00707f +#define MATCH_FSGNJ_Q 0x26000053 +#define MASK_FSGNJ_Q 0xfe00707f +#define MATCH_FSGNJ_S 0x20000053 +#define MASK_FSGNJ_S 0xfe00707f +#define MATCH_FSGNJN_D 0x22001053 +#define MASK_FSGNJN_D 0xfe00707f +#define MATCH_FSGNJN_H 0x24001053 +#define MASK_FSGNJN_H 0xfe00707f +#define MATCH_FSGNJN_Q 0x26001053 +#define MASK_FSGNJN_Q 0xfe00707f +#define MATCH_FSGNJN_S 0x20001053 +#define MASK_FSGNJN_S 0xfe00707f +#define MATCH_FSGNJX_D 0x22002053 +#define MASK_FSGNJX_D 0xfe00707f +#define MATCH_FSGNJX_H 0x24002053 +#define MASK_FSGNJX_H 0xfe00707f +#define MATCH_FSGNJX_Q 0x26002053 +#define MASK_FSGNJX_Q 0xfe00707f +#define MATCH_FSGNJX_S 0x20002053 +#define MASK_FSGNJX_S 0xfe00707f +#define MATCH_FSH 0x1027 +#define MASK_FSH 0x707f +#define MATCH_FSL 0x4001033 +#define MASK_FSL 0x600707f +#define MATCH_FSLW 0x400103b +#define MASK_FSLW 0x600707f +#define MATCH_FSQ 0x4027 +#define MASK_FSQ 0x707f +#define MATCH_FSQRT_D 0x5a000053 +#define MASK_FSQRT_D 0xfff0007f +#define MATCH_FSQRT_H 0x5c000053 +#define MASK_FSQRT_H 0xfff0007f +#define MATCH_FSQRT_Q 0x5e000053 +#define MASK_FSQRT_Q 0xfff0007f +#define MATCH_FSQRT_S 0x58000053 +#define MASK_FSQRT_S 0xfff0007f +#define MATCH_FSR 0x4005033 +#define MASK_FSR 0x600707f +#define MATCH_FSRI 0x4005013 +#define MASK_FSRI 0x400707f +#define MATCH_FSRIW 0x400501b +#define MASK_FSRIW 0x600707f +#define MATCH_FSRW 0x400503b +#define MASK_FSRW 0x600707f +#define MATCH_FSUB_D 0xa000053 +#define MASK_FSUB_D 0xfe00007f +#define MATCH_FSUB_H 0xc000053 +#define MASK_FSUB_H 0xfe00007f +#define MATCH_FSUB_Q 0xe000053 +#define MASK_FSUB_Q 0xfe00007f +#define MATCH_FSUB_S 0x8000053 +#define MASK_FSUB_S 0xfe00007f +#define MATCH_FSW 0x2027 +#define MASK_FSW 0x707f +#define MATCH_GORC 0x28005033 +#define MASK_GORC 0xfe00707f +#define MATCH_GORCI 0x28005013 +#define MASK_GORCI 0xfc00707f +#define MATCH_GORCIW 0x2800501b +#define MASK_GORCIW 0xfe00707f +#define MATCH_GORCW 0x2800503b +#define MASK_GORCW 0xfe00707f +#define MATCH_GREV 0x68005033 +#define MASK_GREV 0xfe00707f +#define MATCH_GREVI 0x68005013 +#define MASK_GREVI 0xfc00707f +#define MATCH_GREVIW 0x6800501b +#define MASK_GREVIW 0xfe00707f +#define MATCH_GREVW 0x6800503b +#define MASK_GREVW 0xfe00707f +#define MATCH_HFENCE_GVMA 0x62000073 +#define MASK_HFENCE_GVMA 0xfe007fff +#define MATCH_HFENCE_VVMA 0x22000073 +#define MASK_HFENCE_VVMA 0xfe007fff +#define MATCH_HINVAL_GVMA 0x66000073 +#define MASK_HINVAL_GVMA 0xfe007fff +#define MATCH_HINVAL_VVMA 0x26000073 +#define MASK_HINVAL_VVMA 0xfe007fff +#define MATCH_HLV_B 0x60004073 +#define MASK_HLV_B 0xfff0707f +#define MATCH_HLV_BU 0x60104073 +#define MASK_HLV_BU 0xfff0707f +#define MATCH_HLV_D 0x6c004073 +#define MASK_HLV_D 0xfff0707f +#define MATCH_HLV_H 0x64004073 +#define MASK_HLV_H 0xfff0707f +#define MATCH_HLV_HU 0x64104073 +#define MASK_HLV_HU 0xfff0707f +#define MATCH_HLV_W 0x68004073 +#define MASK_HLV_W 0xfff0707f +#define MATCH_HLV_WU 0x68104073 +#define MASK_HLV_WU 0xfff0707f +#define MATCH_HLVX_HU 0x64304073 +#define MASK_HLVX_HU 0xfff0707f +#define MATCH_HLVX_WU 0x68304073 +#define MASK_HLVX_WU 0xfff0707f +#define MATCH_HSV_B 0x62004073 +#define MASK_HSV_B 0xfe007fff +#define MATCH_HSV_D 0x6e004073 +#define MASK_HSV_D 0xfe007fff +#define MATCH_HSV_H 0x66004073 +#define MASK_HSV_H 0xfe007fff +#define MATCH_HSV_W 0x6a004073 +#define MASK_HSV_W 0xfe007fff +#define MATCH_INSB 0xac000077 +#define MASK_INSB 0xff80707f +#define MATCH_JAL 0x6f +#define MASK_JAL 0x7f +#define MATCH_JALR 0x67 +#define MASK_JALR 0x707f +#define MATCH_KABS16 0xad100077 +#define MASK_KABS16 0xfff0707f +#define MATCH_KABS32 0xad200077 +#define MASK_KABS32 0xfff0707f +#define MATCH_KABS8 0xad000077 +#define MASK_KABS8 0xfff0707f +#define MATCH_KABSW 0xad400077 +#define MASK_KABSW 0xfff0707f +#define MATCH_KADD16 0x10000077 +#define MASK_KADD16 0xfe00707f +#define MATCH_KADD32 0x10002077 +#define MASK_KADD32 0xfe00707f +#define MATCH_KADD64 0x90001077 +#define MASK_KADD64 0xfe00707f +#define MATCH_KADD8 0x18000077 +#define MASK_KADD8 0xfe00707f +#define MATCH_KADDH 0x4001077 +#define MASK_KADDH 0xfe00707f +#define MATCH_KADDW 0x1077 +#define MASK_KADDW 0xfe00707f +#define MATCH_KCRAS16 0x14000077 +#define MASK_KCRAS16 0xfe00707f +#define MATCH_KCRAS32 0x14002077 +#define MASK_KCRAS32 0xfe00707f +#define MATCH_KCRSA16 0x16000077 +#define MASK_KCRSA16 0xfe00707f +#define MATCH_KCRSA32 0x16002077 +#define MASK_KCRSA32 0xfe00707f +#define MATCH_KDMABB 0xd2001077 +#define MASK_KDMABB 0xfe00707f +#define MATCH_KDMABB16 0xd8001077 +#define MASK_KDMABB16 0xfe00707f +#define MATCH_KDMABT 0xe2001077 +#define MASK_KDMABT 0xfe00707f +#define MATCH_KDMABT16 0xe8001077 +#define MASK_KDMABT16 0xfe00707f +#define MATCH_KDMATT 0xf2001077 +#define MASK_KDMATT 0xfe00707f +#define MATCH_KDMATT16 0xf8001077 +#define MASK_KDMATT16 0xfe00707f +#define MATCH_KDMBB 0xa001077 +#define MASK_KDMBB 0xfe00707f +#define MATCH_KDMBB16 0xda001077 +#define MASK_KDMBB16 0xfe00707f +#define MATCH_KDMBT 0x1a001077 +#define MASK_KDMBT 0xfe00707f +#define MATCH_KDMBT16 0xea001077 +#define MASK_KDMBT16 0xfe00707f +#define MATCH_KDMTT 0x2a001077 +#define MASK_KDMTT 0xfe00707f +#define MATCH_KDMTT16 0xfa001077 +#define MASK_KDMTT16 0xfe00707f +#define MATCH_KHM16 0x86000077 +#define MASK_KHM16 0xfe00707f +#define MATCH_KHM8 0x8e000077 +#define MASK_KHM8 0xfe00707f +#define MATCH_KHMBB 0xc001077 +#define MASK_KHMBB 0xfe00707f +#define MATCH_KHMBB16 0xdc001077 +#define MASK_KHMBB16 0xfe00707f +#define MATCH_KHMBT 0x1c001077 +#define MASK_KHMBT 0xfe00707f +#define MATCH_KHMBT16 0xec001077 +#define MASK_KHMBT16 0xfe00707f +#define MATCH_KHMTT 0x2c001077 +#define MASK_KHMTT 0xfe00707f +#define MATCH_KHMTT16 0xfc001077 +#define MASK_KHMTT16 0xfe00707f +#define MATCH_KHMX16 0x96000077 +#define MASK_KHMX16 0xfe00707f +#define MATCH_KHMX8 0x9e000077 +#define MASK_KHMX8 0xfe00707f +#define MATCH_KMABB 0x5a001077 +#define MASK_KMABB 0xfe00707f +#define MATCH_KMABB32 0x5a002077 +#define MASK_KMABB32 0xfe00707f +#define MATCH_KMABT 0x6a001077 +#define MASK_KMABT 0xfe00707f +#define MATCH_KMABT32 0x6a002077 +#define MASK_KMABT32 0xfe00707f +#define MATCH_KMADA 0x48001077 +#define MASK_KMADA 0xfe00707f +#define MATCH_KMADRS 0x6c001077 +#define MASK_KMADRS 0xfe00707f +#define MATCH_KMADRS32 0x6c002077 +#define MASK_KMADRS32 0xfe00707f +#define MATCH_KMADS 0x5c001077 +#define MASK_KMADS 0xfe00707f +#define MATCH_KMADS32 0x5c002077 +#define MASK_KMADS32 0xfe00707f +#define MATCH_KMAR64 0x94001077 +#define MASK_KMAR64 0xfe00707f +#define MATCH_KMATT 0x7a001077 +#define MASK_KMATT 0xfe00707f +#define MATCH_KMATT32 0x7a002077 +#define MASK_KMATT32 0xfe00707f +#define MATCH_KMAXDA 0x4a001077 +#define MASK_KMAXDA 0xfe00707f +#define MATCH_KMAXDA32 0x4a002077 +#define MASK_KMAXDA32 0xfe00707f +#define MATCH_KMAXDS 0x7c001077 +#define MASK_KMAXDS 0xfe00707f +#define MATCH_KMAXDS32 0x7c002077 +#define MASK_KMAXDS32 0xfe00707f +#define MATCH_KMDA 0x38001077 +#define MASK_KMDA 0xfe00707f +#define MATCH_KMDA32 0x38002077 +#define MASK_KMDA32 0xfe00707f +#define MATCH_KMMAC 0x60001077 +#define MASK_KMMAC 0xfe00707f +#define MATCH_KMMAC_U 0x70001077 +#define MASK_KMMAC_U 0xfe00707f +#define MATCH_KMMAWB 0x46001077 +#define MASK_KMMAWB 0xfe00707f +#define MATCH_KMMAWB2 0xce001077 +#define MASK_KMMAWB2 0xfe00707f +#define MATCH_KMMAWB2_U 0xde001077 +#define MASK_KMMAWB2_U 0xfe00707f +#define MATCH_KMMAWB_U 0x56001077 +#define MASK_KMMAWB_U 0xfe00707f +#define MATCH_KMMAWT 0x66001077 +#define MASK_KMMAWT 0xfe00707f +#define MATCH_KMMAWT2 0xee001077 +#define MASK_KMMAWT2 0xfe00707f +#define MATCH_KMMAWT2_U 0xfe001077 +#define MASK_KMMAWT2_U 0xfe00707f +#define MATCH_KMMAWT_U 0x76001077 +#define MASK_KMMAWT_U 0xfe00707f +#define MATCH_KMMSB 0x42001077 +#define MASK_KMMSB 0xfe00707f +#define MATCH_KMMSB_U 0x52001077 +#define MASK_KMMSB_U 0xfe00707f +#define MATCH_KMMWB2 0x8e001077 +#define MASK_KMMWB2 0xfe00707f +#define MATCH_KMMWB2_U 0x9e001077 +#define MASK_KMMWB2_U 0xfe00707f +#define MATCH_KMMWT2 0xae001077 +#define MASK_KMMWT2 0xfe00707f +#define MATCH_KMMWT2_U 0xbe001077 +#define MASK_KMMWT2_U 0xfe00707f +#define MATCH_KMSDA 0x4c001077 +#define MASK_KMSDA 0xfe00707f +#define MATCH_KMSDA32 0x4c002077 +#define MASK_KMSDA32 0xfe00707f +#define MATCH_KMSR64 0x96001077 +#define MASK_KMSR64 0xfe00707f +#define MATCH_KMSXDA 0x4e001077 +#define MASK_KMSXDA 0xfe00707f +#define MATCH_KMSXDA32 0x4e002077 +#define MASK_KMSXDA32 0xfe00707f +#define MATCH_KMXDA 0x3a001077 +#define MASK_KMXDA 0xfe00707f +#define MATCH_KMXDA32 0x3a002077 +#define MASK_KMXDA32 0xfe00707f +#define MATCH_KSLL16 0x64000077 +#define MASK_KSLL16 0xfe00707f +#define MATCH_KSLL32 0x64002077 +#define MASK_KSLL32 0xfe00707f +#define MATCH_KSLL8 0x6c000077 +#define MASK_KSLL8 0xfe00707f +#define MATCH_KSLLI16 0x75000077 +#define MASK_KSLLI16 0xff00707f +#define MATCH_KSLLI32 0x84002077 +#define MASK_KSLLI32 0xfe00707f +#define MATCH_KSLLI8 0x7c800077 +#define MASK_KSLLI8 0xff80707f +#define MATCH_KSLLIW 0x36001077 +#define MASK_KSLLIW 0xfe00707f +#define MATCH_KSLLW 0x26001077 +#define MASK_KSLLW 0xfe00707f +#define MATCH_KSLRA16 0x56000077 +#define MASK_KSLRA16 0xfe00707f +#define MATCH_KSLRA16_U 0x66000077 +#define MASK_KSLRA16_U 0xfe00707f +#define MATCH_KSLRA32 0x56002077 +#define MASK_KSLRA32 0xfe00707f +#define MATCH_KSLRA32_U 0x66002077 +#define MASK_KSLRA32_U 0xfe00707f +#define MATCH_KSLRA8 0x5e000077 +#define MASK_KSLRA8 0xfe00707f +#define MATCH_KSLRA8_U 0x6e000077 +#define MASK_KSLRA8_U 0xfe00707f +#define MATCH_KSLRAW 0x6e001077 +#define MASK_KSLRAW 0xfe00707f +#define MATCH_KSLRAW_U 0x7e001077 +#define MASK_KSLRAW_U 0xfe00707f +#define MATCH_KSTAS16 0xc4002077 +#define MASK_KSTAS16 0xfe00707f +#define MATCH_KSTAS32 0xc0002077 +#define MASK_KSTAS32 0xfe00707f +#define MATCH_KSTSA16 0xc6002077 +#define MASK_KSTSA16 0xfe00707f +#define MATCH_KSTSA32 0xc2002077 +#define MASK_KSTSA32 0xfe00707f +#define MATCH_KSUB16 0x12000077 +#define MASK_KSUB16 0xfe00707f +#define MATCH_KSUB32 0x12002077 +#define MASK_KSUB32 0xfe00707f +#define MATCH_KSUB64 0x92001077 +#define MASK_KSUB64 0xfe00707f +#define MATCH_KSUB8 0x1a000077 +#define MASK_KSUB8 0xfe00707f +#define MATCH_KSUBH 0x6001077 +#define MASK_KSUBH 0xfe00707f +#define MATCH_KSUBW 0x2001077 +#define MASK_KSUBW 0xfe00707f +#define MATCH_KWMMUL 0x62001077 +#define MASK_KWMMUL 0xfe00707f +#define MATCH_KWMMUL_U 0x72001077 +#define MASK_KWMMUL_U 0xfe00707f +#define MATCH_LB 0x3 +#define MASK_LB 0x707f +#define MATCH_LBU 0x4003 +#define MASK_LBU 0x707f +#define MATCH_LD 0x3003 +#define MASK_LD 0x707f +#define MATCH_LDU 0x7003 +#define MASK_LDU 0x707f +#define MATCH_LH 0x1003 +#define MASK_LH 0x707f +#define MATCH_LHU 0x5003 +#define MASK_LHU 0x707f +#define MATCH_LQ 0x300f +#define MASK_LQ 0x707f +#define MATCH_LR_D 0x1000302f +#define MASK_LR_D 0xf9f0707f +#define MATCH_LR_W 0x1000202f +#define MASK_LR_W 0xf9f0707f +#define MATCH_LUI 0x37 +#define MASK_LUI 0x7f +#define MATCH_LW 0x2003 +#define MASK_LW 0x707f +#define MATCH_LWU 0x6003 +#define MASK_LWU 0x707f +#define MATCH_MADDR32 0xc4001077 +#define MASK_MADDR32 0xfe00707f +#define MATCH_MAX 0xa006033 +#define MASK_MAX 0xfe00707f +#define MATCH_MAXU 0xa007033 +#define MASK_MAXU 0xfe00707f +#define MATCH_MAXW 0xf2000077 +#define MASK_MAXW 0xfe00707f +#define MATCH_MIN 0xa004033 +#define MASK_MIN 0xfe00707f +#define MATCH_MINU 0xa005033 +#define MASK_MINU 0xfe00707f +#define MATCH_MINW 0xf0000077 +#define MASK_MINW 0xfe00707f +#define MATCH_MRET 0x30200073 +#define MASK_MRET 0xffffffff +#define MATCH_MSUBR32 0xc6001077 +#define MASK_MSUBR32 0xfe00707f +#define MATCH_MUL 0x2000033 +#define MASK_MUL 0xfe00707f +#define MATCH_MULH 0x2001033 +#define MASK_MULH 0xfe00707f +#define MATCH_MULHSU 0x2002033 +#define MASK_MULHSU 0xfe00707f +#define MATCH_MULHU 0x2003033 +#define MASK_MULHU 0xfe00707f +#define MATCH_MULR64 0xf0001077 +#define MASK_MULR64 0xfe00707f +#define MATCH_MULSR64 0xe0001077 +#define MASK_MULSR64 0xfe00707f +#define MATCH_MULW 0x200003b +#define MASK_MULW 0xfe00707f +#define MATCH_OR 0x6033 +#define MASK_OR 0xfe00707f +#define MATCH_ORI 0x6013 +#define MASK_ORI 0x707f +#define MATCH_ORN 0x40006033 +#define MASK_ORN 0xfe00707f +#define MATCH_PACK 0x8004033 +#define MASK_PACK 0xfe00707f +#define MATCH_PACKH 0x8007033 +#define MASK_PACKH 0xfe00707f +#define MATCH_PACKU 0x48004033 +#define MASK_PACKU 0xfe00707f +#define MATCH_PACKUW 0x4800403b +#define MASK_PACKUW 0xfe00707f +#define MATCH_PACKW 0x800403b +#define MASK_PACKW 0xfe00707f +#define MATCH_PAUSE 0x100000f +#define MASK_PAUSE 0xffffffff +#define MATCH_PBSAD 0xfc000077 +#define MASK_PBSAD 0xfe00707f +#define MATCH_PBSADA 0xfe000077 +#define MASK_PBSADA 0xfe00707f +#define MATCH_PKBB16 0xe001077 +#define MASK_PKBB16 0xfe00707f +#define MATCH_PKBB32 0xe002077 +#define MASK_PKBB32 0xfe00707f +#define MATCH_PKBT16 0x1e001077 +#define MASK_PKBT16 0xfe00707f +#define MATCH_PKBT32 0x1e002077 +#define MASK_PKBT32 0xfe00707f +#define MATCH_PKTB16 0x3e001077 +#define MASK_PKTB16 0xfe00707f +#define MATCH_PKTB32 0x3e002077 +#define MASK_PKTB32 0xfe00707f +#define MATCH_PKTT16 0x2e001077 +#define MASK_PKTT16 0xfe00707f +#define MATCH_PKTT32 0x2e002077 +#define MASK_PKTT32 0xfe00707f +#define MATCH_PREFETCH_I 0x6013 +#define MASK_PREFETCH_I 0x1f07fff +#define MATCH_PREFETCH_R 0x106013 +#define MASK_PREFETCH_R 0x1f07fff +#define MATCH_PREFETCH_W 0x306013 +#define MASK_PREFETCH_W 0x1f07fff +#define MATCH_RADD16 0x77 +#define MASK_RADD16 0xfe00707f +#define MATCH_RADD32 0x2077 +#define MASK_RADD32 0xfe00707f +#define MATCH_RADD64 0x80001077 +#define MASK_RADD64 0xfe00707f +#define MATCH_RADD8 0x8000077 +#define MASK_RADD8 0xfe00707f +#define MATCH_RADDW 0x20001077 +#define MASK_RADDW 0xfe00707f +#define MATCH_RCRAS16 0x4000077 +#define MASK_RCRAS16 0xfe00707f +#define MATCH_RCRAS32 0x4002077 +#define MASK_RCRAS32 0xfe00707f +#define MATCH_RCRSA16 0x6000077 +#define MASK_RCRSA16 0xfe00707f +#define MATCH_RCRSA32 0x6002077 +#define MASK_RCRSA32 0xfe00707f +#define MATCH_REM 0x2006033 +#define MASK_REM 0xfe00707f +#define MATCH_REMU 0x2007033 +#define MASK_REMU 0xfe00707f +#define MATCH_REMUW 0x200703b +#define MASK_REMUW 0xfe00707f +#define MATCH_REMW 0x200603b +#define MASK_REMW 0xfe00707f +#define MATCH_ROL 0x60001033 +#define MASK_ROL 0xfe00707f +#define MATCH_ROLW 0x6000103b +#define MASK_ROLW 0xfe00707f +#define MATCH_ROR 0x60005033 +#define MASK_ROR 0xfe00707f +#define MATCH_RORI 0x60005013 +#define MASK_RORI 0xfc00707f +#define MATCH_RORIW 0x6000501b +#define MASK_RORIW 0xfe00707f +#define MATCH_RORW 0x6000503b +#define MASK_RORW 0xfe00707f +#define MATCH_RSTAS16 0xb4002077 +#define MASK_RSTAS16 0xfe00707f +#define MATCH_RSTAS32 0xb0002077 +#define MASK_RSTAS32 0xfe00707f +#define MATCH_RSTSA16 0xb6002077 +#define MASK_RSTSA16 0xfe00707f +#define MATCH_RSTSA32 0xb2002077 +#define MASK_RSTSA32 0xfe00707f +#define MATCH_RSUB16 0x2000077 +#define MASK_RSUB16 0xfe00707f +#define MATCH_RSUB32 0x2002077 +#define MASK_RSUB32 0xfe00707f +#define MATCH_RSUB64 0x82001077 +#define MASK_RSUB64 0xfe00707f +#define MATCH_RSUB8 0xa000077 +#define MASK_RSUB8 0xfe00707f +#define MATCH_RSUBW 0x22001077 +#define MASK_RSUBW 0xfe00707f +#define MATCH_SB 0x23 +#define MASK_SB 0x707f +#define MATCH_SC_D 0x1800302f +#define MASK_SC_D 0xf800707f +#define MATCH_SC_W 0x1800202f +#define MASK_SC_W 0xf800707f +#define MATCH_SCLIP16 0x84000077 +#define MASK_SCLIP16 0xff00707f +#define MATCH_SCLIP32 0xe4000077 +#define MASK_SCLIP32 0xfe00707f +#define MATCH_SCLIP8 0x8c000077 +#define MASK_SCLIP8 0xff80707f +#define MATCH_SCMPLE16 0x1c000077 +#define MASK_SCMPLE16 0xfe00707f +#define MATCH_SCMPLE8 0x1e000077 +#define MASK_SCMPLE8 0xfe00707f +#define MATCH_SCMPLT16 0xc000077 +#define MASK_SCMPLT16 0xfe00707f +#define MATCH_SCMPLT8 0xe000077 +#define MASK_SCMPLT8 0xfe00707f +#define MATCH_SD 0x3023 +#define MASK_SD 0x707f +#define MATCH_SEXT_B 0x60401013 +#define MASK_SEXT_B 0xfff0707f +#define MATCH_SEXT_H 0x60501013 +#define MASK_SEXT_H 0xfff0707f +#define MATCH_SFENCE_INVAL_IR 0x18100073 +#define MASK_SFENCE_INVAL_IR 0xffffffff +#define MATCH_SFENCE_VMA 0x12000073 +#define MASK_SFENCE_VMA 0xfe007fff +#define MATCH_SFENCE_W_INVAL 0x18000073 +#define MASK_SFENCE_W_INVAL 0xffffffff +#define MATCH_SH 0x1023 +#define MASK_SH 0x707f +#define MATCH_SH1ADD 0x20002033 +#define MASK_SH1ADD 0xfe00707f +#define MATCH_SH1ADD_UW 0x2000203b +#define MASK_SH1ADD_UW 0xfe00707f +#define MATCH_SH2ADD 0x20004033 +#define MASK_SH2ADD 0xfe00707f +#define MATCH_SH2ADD_UW 0x2000403b +#define MASK_SH2ADD_UW 0xfe00707f +#define MATCH_SH3ADD 0x20006033 +#define MASK_SH3ADD 0xfe00707f +#define MATCH_SH3ADD_UW 0x2000603b +#define MASK_SH3ADD_UW 0xfe00707f +#define MATCH_SHA256SIG0 0x10201013 +#define MASK_SHA256SIG0 0xfff0707f +#define MATCH_SHA256SIG1 0x10301013 +#define MASK_SHA256SIG1 0xfff0707f +#define MATCH_SHA256SUM0 0x10001013 +#define MASK_SHA256SUM0 0xfff0707f +#define MATCH_SHA256SUM1 0x10101013 +#define MASK_SHA256SUM1 0xfff0707f +#define MATCH_SHA512SIG0 0x10601013 +#define MASK_SHA512SIG0 0xfff0707f +#define MATCH_SHA512SIG0H 0x5c000033 +#define MASK_SHA512SIG0H 0xfe00707f +#define MATCH_SHA512SIG0L 0x54000033 +#define MASK_SHA512SIG0L 0xfe00707f +#define MATCH_SHA512SIG1 0x10701013 +#define MASK_SHA512SIG1 0xfff0707f +#define MATCH_SHA512SIG1H 0x5e000033 +#define MASK_SHA512SIG1H 0xfe00707f +#define MATCH_SHA512SIG1L 0x56000033 +#define MASK_SHA512SIG1L 0xfe00707f +#define MATCH_SHA512SUM0 0x10401013 +#define MASK_SHA512SUM0 0xfff0707f +#define MATCH_SHA512SUM0R 0x50000033 +#define MASK_SHA512SUM0R 0xfe00707f +#define MATCH_SHA512SUM1 0x10501013 +#define MASK_SHA512SUM1 0xfff0707f +#define MATCH_SHA512SUM1R 0x52000033 +#define MASK_SHA512SUM1R 0xfe00707f +#define MATCH_SHFL 0x8001033 +#define MASK_SHFL 0xfe00707f +#define MATCH_SHFLI 0x8001013 +#define MASK_SHFLI 0xfe00707f +#define MATCH_SHFLW 0x800103b +#define MASK_SHFLW 0xfe00707f +#define MATCH_SINVAL_VMA 0x16000073 +#define MASK_SINVAL_VMA 0xfe007fff +#define MATCH_SLL 0x1033 +#define MASK_SLL 0xfe00707f +#define MATCH_SLL16 0x54000077 +#define MASK_SLL16 0xfe00707f +#define MATCH_SLL32 0x54002077 +#define MASK_SLL32 0xfe00707f +#define MATCH_SLL8 0x5c000077 +#define MASK_SLL8 0xfe00707f +#define MATCH_SLLD 0x107b +#define MASK_SLLD 0xfe00707f +#define MATCH_SLLI 0x1013 +#define MASK_SLLI 0xf800707f +#define MATCH_SLLI16 0x74000077 +#define MASK_SLLI16 0xff00707f +#define MATCH_SLLI32 0x74002077 +#define MASK_SLLI32 0xfe00707f +#define MATCH_SLLI8 0x7c000077 +#define MASK_SLLI8 0xff80707f +#define MATCH_SLLI_UW 0x800101b +#define MASK_SLLI_UW 0xfc00707f +#define MATCH_SLLID 0x105b +#define MASK_SLLID 0xfc00707f +#define MATCH_SLLIW 0x101b +#define MASK_SLLIW 0xfe00707f +#define MATCH_SLLW 0x103b +#define MASK_SLLW 0xfe00707f +#define MATCH_SLO 0x20001033 +#define MASK_SLO 0xfe00707f +#define MATCH_SLOI 0x20001013 +#define MASK_SLOI 0xfc00707f +#define MATCH_SLOIW 0x2000101b +#define MASK_SLOIW 0xfe00707f +#define MATCH_SLOW 0x2000103b +#define MASK_SLOW 0xfe00707f +#define MATCH_SLT 0x2033 +#define MASK_SLT 0xfe00707f +#define MATCH_SLTI 0x2013 +#define MASK_SLTI 0x707f +#define MATCH_SLTIU 0x3013 +#define MASK_SLTIU 0x707f +#define MATCH_SLTU 0x3033 +#define MASK_SLTU 0xfe00707f +#define MATCH_SM3P0 0x10801013 +#define MASK_SM3P0 0xfff0707f +#define MATCH_SM3P1 0x10901013 +#define MASK_SM3P1 0xfff0707f +#define MATCH_SM4ED 0x30000033 +#define MASK_SM4ED 0x3e00707f +#define MATCH_SM4KS 0x34000033 +#define MASK_SM4KS 0x3e00707f +#define MATCH_SMAL 0x5e001077 +#define MASK_SMAL 0xfe00707f +#define MATCH_SMALBB 0x88001077 +#define MASK_SMALBB 0xfe00707f +#define MATCH_SMALBT 0x98001077 +#define MASK_SMALBT 0xfe00707f +#define MATCH_SMALDA 0x8c001077 +#define MASK_SMALDA 0xfe00707f +#define MATCH_SMALDRS 0x9a001077 +#define MASK_SMALDRS 0xfe00707f +#define MATCH_SMALDS 0x8a001077 +#define MASK_SMALDS 0xfe00707f +#define MATCH_SMALTT 0xa8001077 +#define MASK_SMALTT 0xfe00707f +#define MATCH_SMALXDA 0x9c001077 +#define MASK_SMALXDA 0xfe00707f +#define MATCH_SMALXDS 0xaa001077 +#define MASK_SMALXDS 0xfe00707f +#define MATCH_SMAQA 0xc8000077 +#define MASK_SMAQA 0xfe00707f +#define MATCH_SMAQA_SU 0xca000077 +#define MASK_SMAQA_SU 0xfe00707f +#define MATCH_SMAR64 0x84001077 +#define MASK_SMAR64 0xfe00707f +#define MATCH_SMAX16 0x82000077 +#define MASK_SMAX16 0xfe00707f +#define MATCH_SMAX32 0x92002077 +#define MASK_SMAX32 0xfe00707f +#define MATCH_SMAX8 0x8a000077 +#define MASK_SMAX8 0xfe00707f +#define MATCH_SMBB16 0x8001077 +#define MASK_SMBB16 0xfe00707f +#define MATCH_SMBT16 0x18001077 +#define MASK_SMBT16 0xfe00707f +#define MATCH_SMBT32 0x18002077 +#define MASK_SMBT32 0xfe00707f +#define MATCH_SMDRS 0x68001077 +#define MASK_SMDRS 0xfe00707f +#define MATCH_SMDRS32 0x68002077 +#define MASK_SMDRS32 0xfe00707f +#define MATCH_SMDS 0x58001077 +#define MASK_SMDS 0xfe00707f +#define MATCH_SMDS32 0x58002077 +#define MASK_SMDS32 0xfe00707f +#define MATCH_SMIN16 0x80000077 +#define MASK_SMIN16 0xfe00707f +#define MATCH_SMIN32 0x90002077 +#define MASK_SMIN32 0xfe00707f +#define MATCH_SMIN8 0x88000077 +#define MASK_SMIN8 0xfe00707f +#define MATCH_SMMUL 0x40001077 +#define MASK_SMMUL 0xfe00707f +#define MATCH_SMMUL_U 0x50001077 +#define MASK_SMMUL_U 0xfe00707f +#define MATCH_SMMWB 0x44001077 +#define MASK_SMMWB 0xfe00707f +#define MATCH_SMMWB_U 0x54001077 +#define MASK_SMMWB_U 0xfe00707f +#define MATCH_SMMWT 0x64001077 +#define MASK_SMMWT 0xfe00707f +#define MATCH_SMMWT_U 0x74001077 +#define MASK_SMMWT_U 0xfe00707f +#define MATCH_SMSLDA 0xac001077 +#define MASK_SMSLDA 0xfe00707f +#define MATCH_SMSLXDA 0xbc001077 +#define MASK_SMSLXDA 0xfe00707f +#define MATCH_SMSR64 0x86001077 +#define MASK_SMSR64 0xfe00707f +#define MATCH_SMTT16 0x28001077 +#define MASK_SMTT16 0xfe00707f +#define MATCH_SMTT32 0x28002077 +#define MASK_SMTT32 0xfe00707f +#define MATCH_SMUL16 0xa0000077 +#define MASK_SMUL16 0xfe00707f +#define MATCH_SMUL8 0xa8000077 +#define MASK_SMUL8 0xfe00707f +#define MATCH_SMULX16 0xa2000077 +#define MASK_SMULX16 0xfe00707f +#define MATCH_SMULX8 0xaa000077 +#define MASK_SMULX8 0xfe00707f +#define MATCH_SMXDS 0x78001077 +#define MASK_SMXDS 0xfe00707f +#define MATCH_SMXDS32 0x78002077 +#define MASK_SMXDS32 0xfe00707f +#define MATCH_SQ 0x4023 +#define MASK_SQ 0x707f +#define MATCH_SRA 0x40005033 +#define MASK_SRA 0xfe00707f +#define MATCH_SRA16 0x50000077 +#define MASK_SRA16 0xfe00707f +#define MATCH_SRA16_U 0x60000077 +#define MASK_SRA16_U 0xfe00707f +#define MATCH_SRA32 0x50002077 +#define MASK_SRA32 0xfe00707f +#define MATCH_SRA32_U 0x60002077 +#define MASK_SRA32_U 0xfe00707f +#define MATCH_SRA8 0x58000077 +#define MASK_SRA8 0xfe00707f +#define MATCH_SRA8_U 0x68000077 +#define MASK_SRA8_U 0xfe00707f +#define MATCH_SRA_U 0x24001077 +#define MASK_SRA_U 0xfe00707f +#define MATCH_SRAD 0x4000507b +#define MASK_SRAD 0xfe00707f +#define MATCH_SRAI 0x40005013 +#define MASK_SRAI 0xf800707f +#define MATCH_SRAI16 0x70000077 +#define MASK_SRAI16 0xff00707f +#define MATCH_SRAI16_U 0x71000077 +#define MASK_SRAI16_U 0xff00707f +#define MATCH_SRAI32 0x70002077 +#define MASK_SRAI32 0xfe00707f +#define MATCH_SRAI32_U 0x80002077 +#define MASK_SRAI32_U 0xfe00707f +#define MATCH_SRAI8 0x78000077 +#define MASK_SRAI8 0xff80707f +#define MATCH_SRAI8_U 0x78800077 +#define MASK_SRAI8_U 0xff80707f +#define MATCH_SRAI_U 0xd4001077 +#define MASK_SRAI_U 0xfc00707f +#define MATCH_SRAID 0x4000505b +#define MASK_SRAID 0xfc00707f +#define MATCH_SRAIW 0x4000501b +#define MASK_SRAIW 0xfe00707f +#define MATCH_SRAIW_U 0x34001077 +#define MASK_SRAIW_U 0xfe00707f +#define MATCH_SRAW 0x4000503b +#define MASK_SRAW 0xfe00707f +#define MATCH_SRET 0x10200073 +#define MASK_SRET 0xffffffff +#define MATCH_SRL 0x5033 +#define MASK_SRL 0xfe00707f +#define MATCH_SRL16 0x52000077 +#define MASK_SRL16 0xfe00707f +#define MATCH_SRL16_U 0x62000077 +#define MASK_SRL16_U 0xfe00707f +#define MATCH_SRL32 0x52002077 +#define MASK_SRL32 0xfe00707f +#define MATCH_SRL32_U 0x62002077 +#define MASK_SRL32_U 0xfe00707f +#define MATCH_SRL8 0x5a000077 +#define MASK_SRL8 0xfe00707f +#define MATCH_SRL8_U 0x6a000077 +#define MASK_SRL8_U 0xfe00707f +#define MATCH_SRLD 0x507b +#define MASK_SRLD 0xfe00707f +#define MATCH_SRLI 0x5013 +#define MASK_SRLI 0xf800707f +#define MATCH_SRLI16 0x72000077 +#define MASK_SRLI16 0xff00707f +#define MATCH_SRLI16_U 0x73000077 +#define MASK_SRLI16_U 0xff00707f +#define MATCH_SRLI32 0x72002077 +#define MASK_SRLI32 0xfe00707f +#define MATCH_SRLI32_U 0x82002077 +#define MASK_SRLI32_U 0xfe00707f +#define MATCH_SRLI8 0x7a000077 +#define MASK_SRLI8 0xff80707f +#define MATCH_SRLI8_U 0x7a800077 +#define MASK_SRLI8_U 0xff80707f +#define MATCH_SRLID 0x505b +#define MASK_SRLID 0xfc00707f +#define MATCH_SRLIW 0x501b +#define MASK_SRLIW 0xfe00707f +#define MATCH_SRLW 0x503b +#define MASK_SRLW 0xfe00707f +#define MATCH_SRO 0x20005033 +#define MASK_SRO 0xfe00707f +#define MATCH_SROI 0x20005013 +#define MASK_SROI 0xfc00707f +#define MATCH_SROIW 0x2000501b +#define MASK_SROIW 0xfe00707f +#define MATCH_SROW 0x2000503b +#define MASK_SROW 0xfe00707f +#define MATCH_STAS16 0xf4002077 +#define MASK_STAS16 0xfe00707f +#define MATCH_STAS32 0xf0002077 +#define MASK_STAS32 0xfe00707f +#define MATCH_STSA16 0xf6002077 +#define MASK_STSA16 0xfe00707f +#define MATCH_STSA32 0xf2002077 +#define MASK_STSA32 0xfe00707f +#define MATCH_SUB 0x40000033 +#define MASK_SUB 0xfe00707f +#define MATCH_SUB16 0x42000077 +#define MASK_SUB16 0xfe00707f +#define MATCH_SUB32 0x42002077 +#define MASK_SUB32 0xfe00707f +#define MATCH_SUB64 0xc2001077 +#define MASK_SUB64 0xfe00707f +#define MATCH_SUB8 0x4a000077 +#define MASK_SUB8 0xfe00707f +#define MATCH_SUBD 0x4000007b +#define MASK_SUBD 0xfe00707f +#define MATCH_SUBW 0x4000003b +#define MASK_SUBW 0xfe00707f +#define MATCH_SUNPKD810 0xac800077 +#define MASK_SUNPKD810 0xfff0707f +#define MATCH_SUNPKD820 0xac900077 +#define MASK_SUNPKD820 0xfff0707f +#define MATCH_SUNPKD830 0xaca00077 +#define MASK_SUNPKD830 0xfff0707f +#define MATCH_SUNPKD831 0xacb00077 +#define MASK_SUNPKD831 0xfff0707f +#define MATCH_SUNPKD832 0xad300077 +#define MASK_SUNPKD832 0xfff0707f +#define MATCH_SW 0x2023 +#define MASK_SW 0x707f +#define MATCH_SWAP8 0xad800077 +#define MASK_SWAP8 0xfff0707f +#define MATCH_UCLIP16 0x85000077 +#define MASK_UCLIP16 0xff00707f +#define MATCH_UCLIP32 0xf4000077 +#define MASK_UCLIP32 0xfe00707f +#define MATCH_UCLIP8 0x8d000077 +#define MASK_UCLIP8 0xff80707f +#define MATCH_UCMPLE16 0x3c000077 +#define MASK_UCMPLE16 0xfe00707f +#define MATCH_UCMPLE8 0x3e000077 +#define MASK_UCMPLE8 0xfe00707f +#define MATCH_UCMPLT16 0x2c000077 +#define MASK_UCMPLT16 0xfe00707f +#define MATCH_UCMPLT8 0x2e000077 +#define MASK_UCMPLT8 0xfe00707f +#define MATCH_UKADD16 0x30000077 +#define MASK_UKADD16 0xfe00707f +#define MATCH_UKADD32 0x30002077 +#define MASK_UKADD32 0xfe00707f +#define MATCH_UKADD64 0xb0001077 +#define MASK_UKADD64 0xfe00707f +#define MATCH_UKADD8 0x38000077 +#define MASK_UKADD8 0xfe00707f +#define MATCH_UKADDH 0x14001077 +#define MASK_UKADDH 0xfe00707f +#define MATCH_UKADDW 0x10001077 +#define MASK_UKADDW 0xfe00707f +#define MATCH_UKCRAS16 0x34000077 +#define MASK_UKCRAS16 0xfe00707f +#define MATCH_UKCRAS32 0x34002077 +#define MASK_UKCRAS32 0xfe00707f +#define MATCH_UKCRSA16 0x36000077 +#define MASK_UKCRSA16 0xfe00707f +#define MATCH_UKCRSA32 0x36002077 +#define MASK_UKCRSA32 0xfe00707f +#define MATCH_UKMAR64 0xb4001077 +#define MASK_UKMAR64 0xfe00707f +#define MATCH_UKMSR64 0xb6001077 +#define MASK_UKMSR64 0xfe00707f +#define MATCH_UKSTAS16 0xe4002077 +#define MASK_UKSTAS16 0xfe00707f +#define MATCH_UKSTAS32 0xe0002077 +#define MASK_UKSTAS32 0xfe00707f +#define MATCH_UKSTSA16 0xe6002077 +#define MASK_UKSTSA16 0xfe00707f +#define MATCH_UKSTSA32 0xe2002077 +#define MASK_UKSTSA32 0xfe00707f +#define MATCH_UKSUB16 0x32000077 +#define MASK_UKSUB16 0xfe00707f +#define MATCH_UKSUB32 0x32002077 +#define MASK_UKSUB32 0xfe00707f +#define MATCH_UKSUB64 0xb2001077 +#define MASK_UKSUB64 0xfe00707f +#define MATCH_UKSUB8 0x3a000077 +#define MASK_UKSUB8 0xfe00707f +#define MATCH_UKSUBH 0x16001077 +#define MASK_UKSUBH 0xfe00707f +#define MATCH_UKSUBW 0x12001077 +#define MASK_UKSUBW 0xfe00707f +#define MATCH_UMAQA 0xcc000077 +#define MASK_UMAQA 0xfe00707f +#define MATCH_UMAR64 0xa4001077 +#define MASK_UMAR64 0xfe00707f +#define MATCH_UMAX16 0x92000077 +#define MASK_UMAX16 0xfe00707f +#define MATCH_UMAX32 0xa2002077 +#define MASK_UMAX32 0xfe00707f +#define MATCH_UMAX8 0x9a000077 +#define MASK_UMAX8 0xfe00707f +#define MATCH_UMIN16 0x90000077 +#define MASK_UMIN16 0xfe00707f +#define MATCH_UMIN32 0xa0002077 +#define MASK_UMIN32 0xfe00707f +#define MATCH_UMIN8 0x98000077 +#define MASK_UMIN8 0xfe00707f +#define MATCH_UMSR64 0xa6001077 +#define MASK_UMSR64 0xfe00707f +#define MATCH_UMUL16 0xb0000077 +#define MASK_UMUL16 0xfe00707f +#define MATCH_UMUL8 0xb8000077 +#define MASK_UMUL8 0xfe00707f +#define MATCH_UMULX16 0xb2000077 +#define MASK_UMULX16 0xfe00707f +#define MATCH_UMULX8 0xba000077 +#define MASK_UMULX8 0xfe00707f +#define MATCH_UNSHFL 0x8005033 +#define MASK_UNSHFL 0xfe00707f +#define MATCH_UNSHFLI 0x8005013 +#define MASK_UNSHFLI 0xfe00707f +#define MATCH_UNSHFLW 0x800503b +#define MASK_UNSHFLW 0xfe00707f +#define MATCH_URADD16 0x20000077 +#define MASK_URADD16 0xfe00707f +#define MATCH_URADD32 0x20002077 +#define MASK_URADD32 0xfe00707f +#define MATCH_URADD64 0xa0001077 +#define MASK_URADD64 0xfe00707f +#define MATCH_URADD8 0x28000077 +#define MASK_URADD8 0xfe00707f +#define MATCH_URADDW 0x30001077 +#define MASK_URADDW 0xfe00707f +#define MATCH_URCRAS16 0x24000077 +#define MASK_URCRAS16 0xfe00707f +#define MATCH_URCRAS32 0x24002077 +#define MASK_URCRAS32 0xfe00707f +#define MATCH_URCRSA16 0x26000077 +#define MASK_URCRSA16 0xfe00707f +#define MATCH_URCRSA32 0x26002077 +#define MASK_URCRSA32 0xfe00707f +#define MATCH_URSTAS16 0xd4002077 +#define MASK_URSTAS16 0xfe00707f +#define MATCH_URSTAS32 0xd0002077 +#define MASK_URSTAS32 0xfe00707f +#define MATCH_URSTSA16 0xd6002077 +#define MASK_URSTSA16 0xfe00707f +#define MATCH_URSTSA32 0xd2002077 +#define MASK_URSTSA32 0xfe00707f +#define MATCH_URSUB16 0x22000077 +#define MASK_URSUB16 0xfe00707f +#define MATCH_URSUB32 0x22002077 +#define MASK_URSUB32 0xfe00707f +#define MATCH_URSUB64 0xa2001077 +#define MASK_URSUB64 0xfe00707f +#define MATCH_URSUB8 0x2a000077 +#define MASK_URSUB8 0xfe00707f +#define MATCH_URSUBW 0x32001077 +#define MASK_URSUBW 0xfe00707f +#define MATCH_VAADD_VV 0x24002057 +#define MASK_VAADD_VV 0xfc00707f +#define MATCH_VAADD_VX 0x24006057 +#define MASK_VAADD_VX 0xfc00707f +#define MATCH_VAADDU_VV 0x20002057 +#define MASK_VAADDU_VV 0xfc00707f +#define MATCH_VAADDU_VX 0x20006057 +#define MASK_VAADDU_VX 0xfc00707f +#define MATCH_VADC_VIM 0x40003057 +#define MASK_VADC_VIM 0xfe00707f +#define MATCH_VADC_VVM 0x40000057 +#define MASK_VADC_VVM 0xfe00707f +#define MATCH_VADC_VXM 0x40004057 +#define MASK_VADC_VXM 0xfe00707f +#define MATCH_VADD_VI 0x3057 +#define MASK_VADD_VI 0xfc00707f +#define MATCH_VADD_VV 0x57 +#define MASK_VADD_VV 0xfc00707f +#define MATCH_VADD_VX 0x4057 +#define MASK_VADD_VX 0xfc00707f +#define MATCH_VAMOADDEI16_V 0x502f +#define MASK_VAMOADDEI16_V 0xf800707f +#define MATCH_VAMOADDEI32_V 0x602f +#define MASK_VAMOADDEI32_V 0xf800707f +#define MATCH_VAMOADDEI64_V 0x702f +#define MASK_VAMOADDEI64_V 0xf800707f +#define MATCH_VAMOADDEI8_V 0x2f +#define MASK_VAMOADDEI8_V 0xf800707f +#define MATCH_VAMOANDEI16_V 0x6000502f +#define MASK_VAMOANDEI16_V 0xf800707f +#define MATCH_VAMOANDEI32_V 0x6000602f +#define MASK_VAMOANDEI32_V 0xf800707f +#define MATCH_VAMOANDEI64_V 0x6000702f +#define MASK_VAMOANDEI64_V 0xf800707f +#define MATCH_VAMOANDEI8_V 0x6000002f +#define MASK_VAMOANDEI8_V 0xf800707f +#define MATCH_VAMOMAXEI16_V 0xa000502f +#define MASK_VAMOMAXEI16_V 0xf800707f +#define MATCH_VAMOMAXEI32_V 0xa000602f +#define MASK_VAMOMAXEI32_V 0xf800707f +#define MATCH_VAMOMAXEI64_V 0xa000702f +#define MASK_VAMOMAXEI64_V 0xf800707f +#define MATCH_VAMOMAXEI8_V 0xa000002f +#define MASK_VAMOMAXEI8_V 0xf800707f +#define MATCH_VAMOMAXUEI16_V 0xe000502f +#define MASK_VAMOMAXUEI16_V 0xf800707f +#define MATCH_VAMOMAXUEI32_V 0xe000602f +#define MASK_VAMOMAXUEI32_V 0xf800707f +#define MATCH_VAMOMAXUEI64_V 0xe000702f +#define MASK_VAMOMAXUEI64_V 0xf800707f +#define MATCH_VAMOMAXUEI8_V 0xe000002f +#define MASK_VAMOMAXUEI8_V 0xf800707f +#define MATCH_VAMOMINEI16_V 0x8000502f +#define MASK_VAMOMINEI16_V 0xf800707f +#define MATCH_VAMOMINEI32_V 0x8000602f +#define MASK_VAMOMINEI32_V 0xf800707f +#define MATCH_VAMOMINEI64_V 0x8000702f +#define MASK_VAMOMINEI64_V 0xf800707f +#define MATCH_VAMOMINEI8_V 0x8000002f +#define MASK_VAMOMINEI8_V 0xf800707f +#define MATCH_VAMOMINUEI16_V 0xc000502f +#define MASK_VAMOMINUEI16_V 0xf800707f +#define MATCH_VAMOMINUEI32_V 0xc000602f +#define MASK_VAMOMINUEI32_V 0xf800707f +#define MATCH_VAMOMINUEI64_V 0xc000702f +#define MASK_VAMOMINUEI64_V 0xf800707f +#define MATCH_VAMOMINUEI8_V 0xc000002f +#define MASK_VAMOMINUEI8_V 0xf800707f +#define MATCH_VAMOOREI16_V 0x4000502f +#define MASK_VAMOOREI16_V 0xf800707f +#define MATCH_VAMOOREI32_V 0x4000602f +#define MASK_VAMOOREI32_V 0xf800707f +#define MATCH_VAMOOREI64_V 0x4000702f +#define MASK_VAMOOREI64_V 0xf800707f +#define MATCH_VAMOOREI8_V 0x4000002f +#define MASK_VAMOOREI8_V 0xf800707f +#define MATCH_VAMOSWAPEI16_V 0x800502f +#define MASK_VAMOSWAPEI16_V 0xf800707f +#define MATCH_VAMOSWAPEI32_V 0x800602f +#define MASK_VAMOSWAPEI32_V 0xf800707f +#define MATCH_VAMOSWAPEI64_V 0x800702f +#define MASK_VAMOSWAPEI64_V 0xf800707f +#define MATCH_VAMOSWAPEI8_V 0x800002f +#define MASK_VAMOSWAPEI8_V 0xf800707f +#define MATCH_VAMOXOREI16_V 0x2000502f +#define MASK_VAMOXOREI16_V 0xf800707f +#define MATCH_VAMOXOREI32_V 0x2000602f +#define MASK_VAMOXOREI32_V 0xf800707f +#define MATCH_VAMOXOREI64_V 0x2000702f +#define MASK_VAMOXOREI64_V 0xf800707f +#define MATCH_VAMOXOREI8_V 0x2000002f +#define MASK_VAMOXOREI8_V 0xf800707f +#define MATCH_VAND_VI 0x24003057 +#define MASK_VAND_VI 0xfc00707f +#define MATCH_VAND_VV 0x24000057 +#define MASK_VAND_VV 0xfc00707f +#define MATCH_VAND_VX 0x24004057 +#define MASK_VAND_VX 0xfc00707f +#define MATCH_VASUB_VV 0x2c002057 +#define MASK_VASUB_VV 0xfc00707f +#define MATCH_VASUB_VX 0x2c006057 +#define MASK_VASUB_VX 0xfc00707f +#define MATCH_VASUBU_VV 0x28002057 +#define MASK_VASUBU_VV 0xfc00707f +#define MATCH_VASUBU_VX 0x28006057 +#define MASK_VASUBU_VX 0xfc00707f +#define MATCH_VCOMPRESS_VM 0x5e002057 +#define MASK_VCOMPRESS_VM 0xfe00707f +#define MATCH_VCPOP_M 0x40082057 +#define MASK_VCPOP_M 0xfc0ff07f +#define MATCH_VDIV_VV 0x84002057 +#define MASK_VDIV_VV 0xfc00707f +#define MATCH_VDIV_VX 0x84006057 +#define MASK_VDIV_VX 0xfc00707f +#define MATCH_VDIVU_VV 0x80002057 +#define MASK_VDIVU_VV 0xfc00707f +#define MATCH_VDIVU_VX 0x80006057 +#define MASK_VDIVU_VX 0xfc00707f +#define MATCH_VFADD_VF 0x5057 +#define MASK_VFADD_VF 0xfc00707f +#define MATCH_VFADD_VV 0x1057 +#define MASK_VFADD_VV 0xfc00707f +#define MATCH_VFCLASS_V 0x4c081057 +#define MASK_VFCLASS_V 0xfc0ff07f +#define MATCH_VFCVT_F_X_V 0x48019057 +#define MASK_VFCVT_F_X_V 0xfc0ff07f +#define MATCH_VFCVT_F_XU_V 0x48011057 +#define MASK_VFCVT_F_XU_V 0xfc0ff07f +#define MATCH_VFCVT_RTZ_X_F_V 0x48039057 +#define MASK_VFCVT_RTZ_X_F_V 0xfc0ff07f +#define MATCH_VFCVT_RTZ_XU_F_V 0x48031057 +#define MASK_VFCVT_RTZ_XU_F_V 0xfc0ff07f +#define MATCH_VFCVT_X_F_V 0x48009057 +#define MASK_VFCVT_X_F_V 0xfc0ff07f +#define MATCH_VFCVT_XU_F_V 0x48001057 +#define MASK_VFCVT_XU_F_V 0xfc0ff07f +#define MATCH_VFDIV_VF 0x80005057 +#define MASK_VFDIV_VF 0xfc00707f +#define MATCH_VFDIV_VV 0x80001057 +#define MASK_VFDIV_VV 0xfc00707f +#define MATCH_VFIRST_M 0x4008a057 +#define MASK_VFIRST_M 0xfc0ff07f +#define MATCH_VFMACC_VF 0xb0005057 +#define MASK_VFMACC_VF 0xfc00707f +#define MATCH_VFMACC_VV 0xb0001057 +#define MASK_VFMACC_VV 0xfc00707f +#define MATCH_VFMADD_VF 0xa0005057 +#define MASK_VFMADD_VF 0xfc00707f +#define MATCH_VFMADD_VV 0xa0001057 +#define MASK_VFMADD_VV 0xfc00707f +#define MATCH_VFMAX_VF 0x18005057 +#define MASK_VFMAX_VF 0xfc00707f +#define MATCH_VFMAX_VV 0x18001057 +#define MASK_VFMAX_VV 0xfc00707f +#define MATCH_VFMERGE_VFM 0x5c005057 +#define MASK_VFMERGE_VFM 0xfe00707f +#define MATCH_VFMIN_VF 0x10005057 +#define MASK_VFMIN_VF 0xfc00707f +#define MATCH_VFMIN_VV 0x10001057 +#define MASK_VFMIN_VV 0xfc00707f +#define MATCH_VFMSAC_VF 0xb8005057 +#define MASK_VFMSAC_VF 0xfc00707f +#define MATCH_VFMSAC_VV 0xb8001057 +#define MASK_VFMSAC_VV 0xfc00707f +#define MATCH_VFMSUB_VF 0xa8005057 +#define MASK_VFMSUB_VF 0xfc00707f +#define MATCH_VFMSUB_VV 0xa8001057 +#define MASK_VFMSUB_VV 0xfc00707f +#define MATCH_VFMUL_VF 0x90005057 +#define MASK_VFMUL_VF 0xfc00707f +#define MATCH_VFMUL_VV 0x90001057 +#define MASK_VFMUL_VV 0xfc00707f +#define MATCH_VFMV_F_S 0x42001057 +#define MASK_VFMV_F_S 0xfe0ff07f +#define MATCH_VFMV_S_F 0x42005057 +#define MASK_VFMV_S_F 0xfff0707f +#define MATCH_VFMV_V_F 0x5e005057 +#define MASK_VFMV_V_F 0xfff0707f +#define MATCH_VFNCVT_F_F_W 0x480a1057 +#define MASK_VFNCVT_F_F_W 0xfc0ff07f +#define MATCH_VFNCVT_F_X_W 0x48099057 +#define MASK_VFNCVT_F_X_W 0xfc0ff07f +#define MATCH_VFNCVT_F_XU_W 0x48091057 +#define MASK_VFNCVT_F_XU_W 0xfc0ff07f +#define MATCH_VFNCVT_ROD_F_F_W 0x480a9057 +#define MASK_VFNCVT_ROD_F_F_W 0xfc0ff07f +#define MATCH_VFNCVT_RTZ_X_F_W 0x480b9057 +#define MASK_VFNCVT_RTZ_X_F_W 0xfc0ff07f +#define MATCH_VFNCVT_RTZ_XU_F_W 0x480b1057 +#define MASK_VFNCVT_RTZ_XU_F_W 0xfc0ff07f +#define MATCH_VFNCVT_X_F_W 0x48089057 +#define MASK_VFNCVT_X_F_W 0xfc0ff07f +#define MATCH_VFNCVT_XU_F_W 0x48081057 +#define MASK_VFNCVT_XU_F_W 0xfc0ff07f +#define MATCH_VFNMACC_VF 0xb4005057 +#define MASK_VFNMACC_VF 0xfc00707f +#define MATCH_VFNMACC_VV 0xb4001057 +#define MASK_VFNMACC_VV 0xfc00707f +#define MATCH_VFNMADD_VF 0xa4005057 +#define MASK_VFNMADD_VF 0xfc00707f +#define MATCH_VFNMADD_VV 0xa4001057 +#define MASK_VFNMADD_VV 0xfc00707f +#define MATCH_VFNMSAC_VF 0xbc005057 +#define MASK_VFNMSAC_VF 0xfc00707f +#define MATCH_VFNMSAC_VV 0xbc001057 +#define MASK_VFNMSAC_VV 0xfc00707f +#define MATCH_VFNMSUB_VF 0xac005057 +#define MASK_VFNMSUB_VF 0xfc00707f +#define MATCH_VFNMSUB_VV 0xac001057 +#define MASK_VFNMSUB_VV 0xfc00707f +#define MATCH_VFRDIV_VF 0x84005057 +#define MASK_VFRDIV_VF 0xfc00707f +#define MATCH_VFREC7_V 0x4c029057 +#define MASK_VFREC7_V 0xfc0ff07f +#define MATCH_VFREDMAX_VS 0x1c001057 +#define MASK_VFREDMAX_VS 0xfc00707f +#define MATCH_VFREDMIN_VS 0x14001057 +#define MASK_VFREDMIN_VS 0xfc00707f +#define MATCH_VFREDOSUM_VS 0xc001057 +#define MASK_VFREDOSUM_VS 0xfc00707f +#define MATCH_VFREDUSUM_VS 0x4001057 +#define MASK_VFREDUSUM_VS 0xfc00707f +#define MATCH_VFRSQRT7_V 0x4c021057 +#define MASK_VFRSQRT7_V 0xfc0ff07f +#define MATCH_VFRSUB_VF 0x9c005057 +#define MASK_VFRSUB_VF 0xfc00707f +#define MATCH_VFSGNJ_VF 0x20005057 +#define MASK_VFSGNJ_VF 0xfc00707f +#define MATCH_VFSGNJ_VV 0x20001057 +#define MASK_VFSGNJ_VV 0xfc00707f +#define MATCH_VFSGNJN_VF 0x24005057 +#define MASK_VFSGNJN_VF 0xfc00707f +#define MATCH_VFSGNJN_VV 0x24001057 +#define MASK_VFSGNJN_VV 0xfc00707f +#define MATCH_VFSGNJX_VF 0x28005057 +#define MASK_VFSGNJX_VF 0xfc00707f +#define MATCH_VFSGNJX_VV 0x28001057 +#define MASK_VFSGNJX_VV 0xfc00707f +#define MATCH_VFSLIDE1DOWN_VF 0x3c005057 +#define MASK_VFSLIDE1DOWN_VF 0xfc00707f +#define MATCH_VFSLIDE1UP_VF 0x38005057 +#define MASK_VFSLIDE1UP_VF 0xfc00707f +#define MATCH_VFSQRT_V 0x4c001057 +#define MASK_VFSQRT_V 0xfc0ff07f +#define MATCH_VFSUB_VF 0x8005057 +#define MASK_VFSUB_VF 0xfc00707f +#define MATCH_VFSUB_VV 0x8001057 +#define MASK_VFSUB_VV 0xfc00707f +#define MATCH_VFWADD_VF 0xc0005057 +#define MASK_VFWADD_VF 0xfc00707f +#define MATCH_VFWADD_VV 0xc0001057 +#define MASK_VFWADD_VV 0xfc00707f +#define MATCH_VFWADD_WF 0xd0005057 +#define MASK_VFWADD_WF 0xfc00707f +#define MATCH_VFWADD_WV 0xd0001057 +#define MASK_VFWADD_WV 0xfc00707f +#define MATCH_VFWCVT_F_F_V 0x48061057 +#define MASK_VFWCVT_F_F_V 0xfc0ff07f +#define MATCH_VFWCVT_F_X_V 0x48059057 +#define MASK_VFWCVT_F_X_V 0xfc0ff07f +#define MATCH_VFWCVT_F_XU_V 0x48051057 +#define MASK_VFWCVT_F_XU_V 0xfc0ff07f +#define MATCH_VFWCVT_RTZ_X_F_V 0x48079057 +#define MASK_VFWCVT_RTZ_X_F_V 0xfc0ff07f +#define MATCH_VFWCVT_RTZ_XU_F_V 0x48071057 +#define MASK_VFWCVT_RTZ_XU_F_V 0xfc0ff07f +#define MATCH_VFWCVT_X_F_V 0x48049057 +#define MASK_VFWCVT_X_F_V 0xfc0ff07f +#define MATCH_VFWCVT_XU_F_V 0x48041057 +#define MASK_VFWCVT_XU_F_V 0xfc0ff07f +#define MATCH_VFWMACC_VF 0xf0005057 +#define MASK_VFWMACC_VF 0xfc00707f +#define MATCH_VFWMACC_VV 0xf0001057 +#define MASK_VFWMACC_VV 0xfc00707f +#define MATCH_VFWMSAC_VF 0xf8005057 +#define MASK_VFWMSAC_VF 0xfc00707f +#define MATCH_VFWMSAC_VV 0xf8001057 +#define MASK_VFWMSAC_VV 0xfc00707f +#define MATCH_VFWMUL_VF 0xe0005057 +#define MASK_VFWMUL_VF 0xfc00707f +#define MATCH_VFWMUL_VV 0xe0001057 +#define MASK_VFWMUL_VV 0xfc00707f +#define MATCH_VFWNMACC_VF 0xf4005057 +#define MASK_VFWNMACC_VF 0xfc00707f +#define MATCH_VFWNMACC_VV 0xf4001057 +#define MASK_VFWNMACC_VV 0xfc00707f +#define MATCH_VFWNMSAC_VF 0xfc005057 +#define MASK_VFWNMSAC_VF 0xfc00707f +#define MATCH_VFWNMSAC_VV 0xfc001057 +#define MASK_VFWNMSAC_VV 0xfc00707f +#define MATCH_VFWREDOSUM_VS 0xcc001057 +#define MASK_VFWREDOSUM_VS 0xfc00707f +#define MATCH_VFWREDUSUM_VS 0xc4001057 +#define MASK_VFWREDUSUM_VS 0xfc00707f +#define MATCH_VFWSUB_VF 0xc8005057 +#define MASK_VFWSUB_VF 0xfc00707f +#define MATCH_VFWSUB_VV 0xc8001057 +#define MASK_VFWSUB_VV 0xfc00707f +#define MATCH_VFWSUB_WF 0xd8005057 +#define MASK_VFWSUB_WF 0xfc00707f +#define MATCH_VFWSUB_WV 0xd8001057 +#define MASK_VFWSUB_WV 0xfc00707f +#define MATCH_VID_V 0x5008a057 +#define MASK_VID_V 0xfdfff07f +#define MATCH_VIOTA_M 0x50082057 +#define MASK_VIOTA_M 0xfc0ff07f +#define MATCH_VL1RE16_V 0x2805007 +#define MASK_VL1RE16_V 0xfff0707f +#define MATCH_VL1RE32_V 0x2806007 +#define MASK_VL1RE32_V 0xfff0707f +#define MATCH_VL1RE64_V 0x2807007 +#define MASK_VL1RE64_V 0xfff0707f +#define MATCH_VL1RE8_V 0x2800007 +#define MASK_VL1RE8_V 0xfff0707f +#define MATCH_VL2RE16_V 0x22805007 +#define MASK_VL2RE16_V 0xfff0707f +#define MATCH_VL2RE32_V 0x22806007 +#define MASK_VL2RE32_V 0xfff0707f +#define MATCH_VL2RE64_V 0x22807007 +#define MASK_VL2RE64_V 0xfff0707f +#define MATCH_VL2RE8_V 0x22800007 +#define MASK_VL2RE8_V 0xfff0707f +#define MATCH_VL4RE16_V 0x62805007 +#define MASK_VL4RE16_V 0xfff0707f +#define MATCH_VL4RE32_V 0x62806007 +#define MASK_VL4RE32_V 0xfff0707f +#define MATCH_VL4RE64_V 0x62807007 +#define MASK_VL4RE64_V 0xfff0707f +#define MATCH_VL4RE8_V 0x62800007 +#define MASK_VL4RE8_V 0xfff0707f +#define MATCH_VL8RE16_V 0xe2805007 +#define MASK_VL8RE16_V 0xfff0707f +#define MATCH_VL8RE32_V 0xe2806007 +#define MASK_VL8RE32_V 0xfff0707f +#define MATCH_VL8RE64_V 0xe2807007 +#define MASK_VL8RE64_V 0xfff0707f +#define MATCH_VL8RE8_V 0xe2800007 +#define MASK_VL8RE8_V 0xfff0707f +#define MATCH_VLE1024_V 0x10007007 +#define MASK_VLE1024_V 0x1df0707f +#define MATCH_VLE1024FF_V 0x11007007 +#define MASK_VLE1024FF_V 0x1df0707f +#define MATCH_VLE128_V 0x10000007 +#define MASK_VLE128_V 0x1df0707f +#define MATCH_VLE128FF_V 0x11000007 +#define MASK_VLE128FF_V 0x1df0707f +#define MATCH_VLE16_V 0x5007 +#define MASK_VLE16_V 0x1df0707f +#define MATCH_VLE16FF_V 0x1005007 +#define MASK_VLE16FF_V 0x1df0707f +#define MATCH_VLE256_V 0x10005007 +#define MASK_VLE256_V 0x1df0707f +#define MATCH_VLE256FF_V 0x11005007 +#define MASK_VLE256FF_V 0x1df0707f +#define MATCH_VLE32_V 0x6007 +#define MASK_VLE32_V 0x1df0707f +#define MATCH_VLE32FF_V 0x1006007 +#define MASK_VLE32FF_V 0x1df0707f +#define MATCH_VLE512_V 0x10006007 +#define MASK_VLE512_V 0x1df0707f +#define MATCH_VLE512FF_V 0x11006007 +#define MASK_VLE512FF_V 0x1df0707f +#define MATCH_VLE64_V 0x7007 +#define MASK_VLE64_V 0x1df0707f +#define MATCH_VLE64FF_V 0x1007007 +#define MASK_VLE64FF_V 0x1df0707f +#define MATCH_VLE8_V 0x7 +#define MASK_VLE8_V 0x1df0707f +#define MATCH_VLE8FF_V 0x1000007 +#define MASK_VLE8FF_V 0x1df0707f +#define MATCH_VLM_V 0x2b00007 +#define MASK_VLM_V 0xfff0707f +#define MATCH_VLOXEI1024_V 0x1c007007 +#define MASK_VLOXEI1024_V 0x1c00707f +#define MATCH_VLOXEI128_V 0x1c000007 +#define MASK_VLOXEI128_V 0x1c00707f +#define MATCH_VLOXEI16_V 0xc005007 +#define MASK_VLOXEI16_V 0x1c00707f +#define MATCH_VLOXEI256_V 0x1c005007 +#define MASK_VLOXEI256_V 0x1c00707f +#define MATCH_VLOXEI32_V 0xc006007 +#define MASK_VLOXEI32_V 0x1c00707f +#define MATCH_VLOXEI512_V 0x1c006007 +#define MASK_VLOXEI512_V 0x1c00707f +#define MATCH_VLOXEI64_V 0xc007007 +#define MASK_VLOXEI64_V 0x1c00707f +#define MATCH_VLOXEI8_V 0xc000007 +#define MASK_VLOXEI8_V 0x1c00707f +#define MATCH_VLSE1024_V 0x18007007 +#define MASK_VLSE1024_V 0x1c00707f +#define MATCH_VLSE128_V 0x18000007 +#define MASK_VLSE128_V 0x1c00707f +#define MATCH_VLSE16_V 0x8005007 +#define MASK_VLSE16_V 0x1c00707f +#define MATCH_VLSE256_V 0x18005007 +#define MASK_VLSE256_V 0x1c00707f +#define MATCH_VLSE32_V 0x8006007 +#define MASK_VLSE32_V 0x1c00707f +#define MATCH_VLSE512_V 0x18006007 +#define MASK_VLSE512_V 0x1c00707f +#define MATCH_VLSE64_V 0x8007007 +#define MASK_VLSE64_V 0x1c00707f +#define MATCH_VLSE8_V 0x8000007 +#define MASK_VLSE8_V 0x1c00707f +#define MATCH_VLUXEI1024_V 0x14007007 +#define MASK_VLUXEI1024_V 0x1c00707f +#define MATCH_VLUXEI128_V 0x14000007 +#define MASK_VLUXEI128_V 0x1c00707f +#define MATCH_VLUXEI16_V 0x4005007 +#define MASK_VLUXEI16_V 0x1c00707f +#define MATCH_VLUXEI256_V 0x14005007 +#define MASK_VLUXEI256_V 0x1c00707f +#define MATCH_VLUXEI32_V 0x4006007 +#define MASK_VLUXEI32_V 0x1c00707f +#define MATCH_VLUXEI512_V 0x14006007 +#define MASK_VLUXEI512_V 0x1c00707f +#define MATCH_VLUXEI64_V 0x4007007 +#define MASK_VLUXEI64_V 0x1c00707f +#define MATCH_VLUXEI8_V 0x4000007 +#define MASK_VLUXEI8_V 0x1c00707f +#define MATCH_VMACC_VV 0xb4002057 +#define MASK_VMACC_VV 0xfc00707f +#define MATCH_VMACC_VX 0xb4006057 +#define MASK_VMACC_VX 0xfc00707f +#define MATCH_VMADC_VI 0x46003057 +#define MASK_VMADC_VI 0xfe00707f +#define MATCH_VMADC_VIM 0x44003057 +#define MASK_VMADC_VIM 0xfe00707f +#define MATCH_VMADC_VV 0x46000057 +#define MASK_VMADC_VV 0xfe00707f +#define MATCH_VMADC_VVM 0x44000057 +#define MASK_VMADC_VVM 0xfe00707f +#define MATCH_VMADC_VX 0x46004057 +#define MASK_VMADC_VX 0xfe00707f +#define MATCH_VMADC_VXM 0x44004057 +#define MASK_VMADC_VXM 0xfe00707f +#define MATCH_VMADD_VV 0xa4002057 +#define MASK_VMADD_VV 0xfc00707f +#define MATCH_VMADD_VX 0xa4006057 +#define MASK_VMADD_VX 0xfc00707f +#define MATCH_VMAND_MM 0x64002057 +#define MASK_VMAND_MM 0xfc00707f +#define MATCH_VMANDN_MM 0x60002057 +#define MASK_VMANDN_MM 0xfc00707f +#define MATCH_VMAX_VV 0x1c000057 +#define MASK_VMAX_VV 0xfc00707f +#define MATCH_VMAX_VX 0x1c004057 +#define MASK_VMAX_VX 0xfc00707f +#define MATCH_VMAXU_VV 0x18000057 +#define MASK_VMAXU_VV 0xfc00707f +#define MATCH_VMAXU_VX 0x18004057 +#define MASK_VMAXU_VX 0xfc00707f +#define MATCH_VMERGE_VIM 0x5c003057 +#define MASK_VMERGE_VIM 0xfe00707f +#define MATCH_VMERGE_VVM 0x5c000057 +#define MASK_VMERGE_VVM 0xfe00707f +#define MATCH_VMERGE_VXM 0x5c004057 +#define MASK_VMERGE_VXM 0xfe00707f +#define MATCH_VMFEQ_VF 0x60005057 +#define MASK_VMFEQ_VF 0xfc00707f +#define MATCH_VMFEQ_VV 0x60001057 +#define MASK_VMFEQ_VV 0xfc00707f +#define MATCH_VMFGE_VF 0x7c005057 +#define MASK_VMFGE_VF 0xfc00707f +#define MATCH_VMFGT_VF 0x74005057 +#define MASK_VMFGT_VF 0xfc00707f +#define MATCH_VMFLE_VF 0x64005057 +#define MASK_VMFLE_VF 0xfc00707f +#define MATCH_VMFLE_VV 0x64001057 +#define MASK_VMFLE_VV 0xfc00707f +#define MATCH_VMFLT_VF 0x6c005057 +#define MASK_VMFLT_VF 0xfc00707f +#define MATCH_VMFLT_VV 0x6c001057 +#define MASK_VMFLT_VV 0xfc00707f +#define MATCH_VMFNE_VF 0x70005057 +#define MASK_VMFNE_VF 0xfc00707f +#define MATCH_VMFNE_VV 0x70001057 +#define MASK_VMFNE_VV 0xfc00707f +#define MATCH_VMIN_VV 0x14000057 +#define MASK_VMIN_VV 0xfc00707f +#define MATCH_VMIN_VX 0x14004057 +#define MASK_VMIN_VX 0xfc00707f +#define MATCH_VMINU_VV 0x10000057 +#define MASK_VMINU_VV 0xfc00707f +#define MATCH_VMINU_VX 0x10004057 +#define MASK_VMINU_VX 0xfc00707f +#define MATCH_VMNAND_MM 0x74002057 +#define MASK_VMNAND_MM 0xfc00707f +#define MATCH_VMNOR_MM 0x78002057 +#define MASK_VMNOR_MM 0xfc00707f +#define MATCH_VMOR_MM 0x68002057 +#define MASK_VMOR_MM 0xfc00707f +#define MATCH_VMORN_MM 0x70002057 +#define MASK_VMORN_MM 0xfc00707f +#define MATCH_VMSBC_VV 0x4e000057 +#define MASK_VMSBC_VV 0xfe00707f +#define MATCH_VMSBC_VVM 0x4c000057 +#define MASK_VMSBC_VVM 0xfe00707f +#define MATCH_VMSBC_VX 0x4e004057 +#define MASK_VMSBC_VX 0xfe00707f +#define MATCH_VMSBC_VXM 0x4c004057 +#define MASK_VMSBC_VXM 0xfe00707f +#define MATCH_VMSBF_M 0x5000a057 +#define MASK_VMSBF_M 0xfc0ff07f +#define MATCH_VMSEQ_VI 0x60003057 +#define MASK_VMSEQ_VI 0xfc00707f +#define MATCH_VMSEQ_VV 0x60000057 +#define MASK_VMSEQ_VV 0xfc00707f +#define MATCH_VMSEQ_VX 0x60004057 +#define MASK_VMSEQ_VX 0xfc00707f +#define MATCH_VMSGT_VI 0x7c003057 +#define MASK_VMSGT_VI 0xfc00707f +#define MATCH_VMSGT_VX 0x7c004057 +#define MASK_VMSGT_VX 0xfc00707f +#define MATCH_VMSGTU_VI 0x78003057 +#define MASK_VMSGTU_VI 0xfc00707f +#define MATCH_VMSGTU_VX 0x78004057 +#define MASK_VMSGTU_VX 0xfc00707f +#define MATCH_VMSIF_M 0x5001a057 +#define MASK_VMSIF_M 0xfc0ff07f +#define MATCH_VMSLE_VI 0x74003057 +#define MASK_VMSLE_VI 0xfc00707f +#define MATCH_VMSLE_VV 0x74000057 +#define MASK_VMSLE_VV 0xfc00707f +#define MATCH_VMSLE_VX 0x74004057 +#define MASK_VMSLE_VX 0xfc00707f +#define MATCH_VMSLEU_VI 0x70003057 +#define MASK_VMSLEU_VI 0xfc00707f +#define MATCH_VMSLEU_VV 0x70000057 +#define MASK_VMSLEU_VV 0xfc00707f +#define MATCH_VMSLEU_VX 0x70004057 +#define MASK_VMSLEU_VX 0xfc00707f +#define MATCH_VMSLT_VV 0x6c000057 +#define MASK_VMSLT_VV 0xfc00707f +#define MATCH_VMSLT_VX 0x6c004057 +#define MASK_VMSLT_VX 0xfc00707f +#define MATCH_VMSLTU_VV 0x68000057 +#define MASK_VMSLTU_VV 0xfc00707f +#define MATCH_VMSLTU_VX 0x68004057 +#define MASK_VMSLTU_VX 0xfc00707f +#define MATCH_VMSNE_VI 0x64003057 +#define MASK_VMSNE_VI 0xfc00707f +#define MATCH_VMSNE_VV 0x64000057 +#define MASK_VMSNE_VV 0xfc00707f +#define MATCH_VMSNE_VX 0x64004057 +#define MASK_VMSNE_VX 0xfc00707f +#define MATCH_VMSOF_M 0x50012057 +#define MASK_VMSOF_M 0xfc0ff07f +#define MATCH_VMUL_VV 0x94002057 +#define MASK_VMUL_VV 0xfc00707f +#define MATCH_VMUL_VX 0x94006057 +#define MASK_VMUL_VX 0xfc00707f +#define MATCH_VMULH_VV 0x9c002057 +#define MASK_VMULH_VV 0xfc00707f +#define MATCH_VMULH_VX 0x9c006057 +#define MASK_VMULH_VX 0xfc00707f +#define MATCH_VMULHSU_VV 0x98002057 +#define MASK_VMULHSU_VV 0xfc00707f +#define MATCH_VMULHSU_VX 0x98006057 +#define MASK_VMULHSU_VX 0xfc00707f +#define MATCH_VMULHU_VV 0x90002057 +#define MASK_VMULHU_VV 0xfc00707f +#define MATCH_VMULHU_VX 0x90006057 +#define MASK_VMULHU_VX 0xfc00707f +#define MATCH_VMV1R_V 0x9e003057 +#define MASK_VMV1R_V 0xfe0ff07f +#define MATCH_VMV2R_V 0x9e00b057 +#define MASK_VMV2R_V 0xfe0ff07f +#define MATCH_VMV4R_V 0x9e01b057 +#define MASK_VMV4R_V 0xfe0ff07f +#define MATCH_VMV8R_V 0x9e03b057 +#define MASK_VMV8R_V 0xfe0ff07f +#define MATCH_VMV_S_X 0x42006057 +#define MASK_VMV_S_X 0xfff0707f +#define MATCH_VMV_V_I 0x5e003057 +#define MASK_VMV_V_I 0xfff0707f +#define MATCH_VMV_V_V 0x5e000057 +#define MASK_VMV_V_V 0xfff0707f +#define MATCH_VMV_V_X 0x5e004057 +#define MASK_VMV_V_X 0xfff0707f +#define MATCH_VMV_X_S 0x42002057 +#define MASK_VMV_X_S 0xfe0ff07f +#define MATCH_VMXNOR_MM 0x7c002057 +#define MASK_VMXNOR_MM 0xfc00707f +#define MATCH_VMXOR_MM 0x6c002057 +#define MASK_VMXOR_MM 0xfc00707f +#define MATCH_VNCLIP_WI 0xbc003057 +#define MASK_VNCLIP_WI 0xfc00707f +#define MATCH_VNCLIP_WV 0xbc000057 +#define MASK_VNCLIP_WV 0xfc00707f +#define MATCH_VNCLIP_WX 0xbc004057 +#define MASK_VNCLIP_WX 0xfc00707f +#define MATCH_VNCLIPU_WI 0xb8003057 +#define MASK_VNCLIPU_WI 0xfc00707f +#define MATCH_VNCLIPU_WV 0xb8000057 +#define MASK_VNCLIPU_WV 0xfc00707f +#define MATCH_VNCLIPU_WX 0xb8004057 +#define MASK_VNCLIPU_WX 0xfc00707f +#define MATCH_VNMSAC_VV 0xbc002057 +#define MASK_VNMSAC_VV 0xfc00707f +#define MATCH_VNMSAC_VX 0xbc006057 +#define MASK_VNMSAC_VX 0xfc00707f +#define MATCH_VNMSUB_VV 0xac002057 +#define MASK_VNMSUB_VV 0xfc00707f +#define MATCH_VNMSUB_VX 0xac006057 +#define MASK_VNMSUB_VX 0xfc00707f +#define MATCH_VNSRA_WI 0xb4003057 +#define MASK_VNSRA_WI 0xfc00707f +#define MATCH_VNSRA_WV 0xb4000057 +#define MASK_VNSRA_WV 0xfc00707f +#define MATCH_VNSRA_WX 0xb4004057 +#define MASK_VNSRA_WX 0xfc00707f +#define MATCH_VNSRL_WI 0xb0003057 +#define MASK_VNSRL_WI 0xfc00707f +#define MATCH_VNSRL_WV 0xb0000057 +#define MASK_VNSRL_WV 0xfc00707f +#define MATCH_VNSRL_WX 0xb0004057 +#define MASK_VNSRL_WX 0xfc00707f +#define MATCH_VOR_VI 0x28003057 +#define MASK_VOR_VI 0xfc00707f +#define MATCH_VOR_VV 0x28000057 +#define MASK_VOR_VV 0xfc00707f +#define MATCH_VOR_VX 0x28004057 +#define MASK_VOR_VX 0xfc00707f +#define MATCH_VREDAND_VS 0x4002057 +#define MASK_VREDAND_VS 0xfc00707f +#define MATCH_VREDMAX_VS 0x1c002057 +#define MASK_VREDMAX_VS 0xfc00707f +#define MATCH_VREDMAXU_VS 0x18002057 +#define MASK_VREDMAXU_VS 0xfc00707f +#define MATCH_VREDMIN_VS 0x14002057 +#define MASK_VREDMIN_VS 0xfc00707f +#define MATCH_VREDMINU_VS 0x10002057 +#define MASK_VREDMINU_VS 0xfc00707f +#define MATCH_VREDOR_VS 0x8002057 +#define MASK_VREDOR_VS 0xfc00707f +#define MATCH_VREDSUM_VS 0x2057 +#define MASK_VREDSUM_VS 0xfc00707f +#define MATCH_VREDXOR_VS 0xc002057 +#define MASK_VREDXOR_VS 0xfc00707f +#define MATCH_VREM_VV 0x8c002057 +#define MASK_VREM_VV 0xfc00707f +#define MATCH_VREM_VX 0x8c006057 +#define MASK_VREM_VX 0xfc00707f +#define MATCH_VREMU_VV 0x88002057 +#define MASK_VREMU_VV 0xfc00707f +#define MATCH_VREMU_VX 0x88006057 +#define MASK_VREMU_VX 0xfc00707f +#define MATCH_VRGATHER_VI 0x30003057 +#define MASK_VRGATHER_VI 0xfc00707f +#define MATCH_VRGATHER_VV 0x30000057 +#define MASK_VRGATHER_VV 0xfc00707f +#define MATCH_VRGATHER_VX 0x30004057 +#define MASK_VRGATHER_VX 0xfc00707f +#define MATCH_VRGATHEREI16_VV 0x38000057 +#define MASK_VRGATHEREI16_VV 0xfc00707f +#define MATCH_VRSUB_VI 0xc003057 +#define MASK_VRSUB_VI 0xfc00707f +#define MATCH_VRSUB_VX 0xc004057 +#define MASK_VRSUB_VX 0xfc00707f +#define MATCH_VS1R_V 0x2800027 +#define MASK_VS1R_V 0xfff0707f +#define MATCH_VS2R_V 0x22800027 +#define MASK_VS2R_V 0xfff0707f +#define MATCH_VS4R_V 0x62800027 +#define MASK_VS4R_V 0xfff0707f +#define MATCH_VS8R_V 0xe2800027 +#define MASK_VS8R_V 0xfff0707f +#define MATCH_VSADD_VI 0x84003057 +#define MASK_VSADD_VI 0xfc00707f +#define MATCH_VSADD_VV 0x84000057 +#define MASK_VSADD_VV 0xfc00707f +#define MATCH_VSADD_VX 0x84004057 +#define MASK_VSADD_VX 0xfc00707f +#define MATCH_VSADDU_VI 0x80003057 +#define MASK_VSADDU_VI 0xfc00707f +#define MATCH_VSADDU_VV 0x80000057 +#define MASK_VSADDU_VV 0xfc00707f +#define MATCH_VSADDU_VX 0x80004057 +#define MASK_VSADDU_VX 0xfc00707f +#define MATCH_VSBC_VVM 0x48000057 +#define MASK_VSBC_VVM 0xfe00707f +#define MATCH_VSBC_VXM 0x48004057 +#define MASK_VSBC_VXM 0xfe00707f +#define MATCH_VSE1024_V 0x10007027 +#define MASK_VSE1024_V 0x1df0707f +#define MATCH_VSE128_V 0x10000027 +#define MASK_VSE128_V 0x1df0707f +#define MATCH_VSE16_V 0x5027 +#define MASK_VSE16_V 0x1df0707f +#define MATCH_VSE256_V 0x10005027 +#define MASK_VSE256_V 0x1df0707f +#define MATCH_VSE32_V 0x6027 +#define MASK_VSE32_V 0x1df0707f +#define MATCH_VSE512_V 0x10006027 +#define MASK_VSE512_V 0x1df0707f +#define MATCH_VSE64_V 0x7027 +#define MASK_VSE64_V 0x1df0707f +#define MATCH_VSE8_V 0x27 +#define MASK_VSE8_V 0x1df0707f +#define MATCH_VSETIVLI 0xc0007057 +#define MASK_VSETIVLI 0xc000707f +#define MATCH_VSETVL 0x80007057 +#define MASK_VSETVL 0xfe00707f +#define MATCH_VSETVLI 0x7057 +#define MASK_VSETVLI 0x8000707f +#define MATCH_VSEXT_VF2 0x4803a057 +#define MASK_VSEXT_VF2 0xfc0ff07f +#define MATCH_VSEXT_VF4 0x4802a057 +#define MASK_VSEXT_VF4 0xfc0ff07f +#define MATCH_VSEXT_VF8 0x4801a057 +#define MASK_VSEXT_VF8 0xfc0ff07f +#define MATCH_VSLIDE1DOWN_VX 0x3c006057 +#define MASK_VSLIDE1DOWN_VX 0xfc00707f +#define MATCH_VSLIDE1UP_VX 0x38006057 +#define MASK_VSLIDE1UP_VX 0xfc00707f +#define MATCH_VSLIDEDOWN_VI 0x3c003057 +#define MASK_VSLIDEDOWN_VI 0xfc00707f +#define MATCH_VSLIDEDOWN_VX 0x3c004057 +#define MASK_VSLIDEDOWN_VX 0xfc00707f +#define MATCH_VSLIDEUP_VI 0x38003057 +#define MASK_VSLIDEUP_VI 0xfc00707f +#define MATCH_VSLIDEUP_VX 0x38004057 +#define MASK_VSLIDEUP_VX 0xfc00707f +#define MATCH_VSLL_VI 0x94003057 +#define MASK_VSLL_VI 0xfc00707f +#define MATCH_VSLL_VV 0x94000057 +#define MASK_VSLL_VV 0xfc00707f +#define MATCH_VSLL_VX 0x94004057 +#define MASK_VSLL_VX 0xfc00707f +#define MATCH_VSM_V 0x2b00027 +#define MASK_VSM_V 0xfff0707f +#define MATCH_VSMUL_VV 0x9c000057 +#define MASK_VSMUL_VV 0xfc00707f +#define MATCH_VSMUL_VX 0x9c004057 +#define MASK_VSMUL_VX 0xfc00707f +#define MATCH_VSOXEI1024_V 0x1c007027 +#define MASK_VSOXEI1024_V 0x1c00707f +#define MATCH_VSOXEI128_V 0x1c000027 +#define MASK_VSOXEI128_V 0x1c00707f +#define MATCH_VSOXEI16_V 0xc005027 +#define MASK_VSOXEI16_V 0x1c00707f +#define MATCH_VSOXEI256_V 0x1c005027 +#define MASK_VSOXEI256_V 0x1c00707f +#define MATCH_VSOXEI32_V 0xc006027 +#define MASK_VSOXEI32_V 0x1c00707f +#define MATCH_VSOXEI512_V 0x1c006027 +#define MASK_VSOXEI512_V 0x1c00707f +#define MATCH_VSOXEI64_V 0xc007027 +#define MASK_VSOXEI64_V 0x1c00707f +#define MATCH_VSOXEI8_V 0xc000027 +#define MASK_VSOXEI8_V 0x1c00707f +#define MATCH_VSRA_VI 0xa4003057 +#define MASK_VSRA_VI 0xfc00707f +#define MATCH_VSRA_VV 0xa4000057 +#define MASK_VSRA_VV 0xfc00707f +#define MATCH_VSRA_VX 0xa4004057 +#define MASK_VSRA_VX 0xfc00707f +#define MATCH_VSRL_VI 0xa0003057 +#define MASK_VSRL_VI 0xfc00707f +#define MATCH_VSRL_VV 0xa0000057 +#define MASK_VSRL_VV 0xfc00707f +#define MATCH_VSRL_VX 0xa0004057 +#define MASK_VSRL_VX 0xfc00707f +#define MATCH_VSSE1024_V 0x18007027 +#define MASK_VSSE1024_V 0x1c00707f +#define MATCH_VSSE128_V 0x18000027 +#define MASK_VSSE128_V 0x1c00707f +#define MATCH_VSSE16_V 0x8005027 +#define MASK_VSSE16_V 0x1c00707f +#define MATCH_VSSE256_V 0x18005027 +#define MASK_VSSE256_V 0x1c00707f +#define MATCH_VSSE32_V 0x8006027 +#define MASK_VSSE32_V 0x1c00707f +#define MATCH_VSSE512_V 0x18006027 +#define MASK_VSSE512_V 0x1c00707f +#define MATCH_VSSE64_V 0x8007027 +#define MASK_VSSE64_V 0x1c00707f +#define MATCH_VSSE8_V 0x8000027 +#define MASK_VSSE8_V 0x1c00707f +#define MATCH_VSSRA_VI 0xac003057 +#define MASK_VSSRA_VI 0xfc00707f +#define MATCH_VSSRA_VV 0xac000057 +#define MASK_VSSRA_VV 0xfc00707f +#define MATCH_VSSRA_VX 0xac004057 +#define MASK_VSSRA_VX 0xfc00707f +#define MATCH_VSSRL_VI 0xa8003057 +#define MASK_VSSRL_VI 0xfc00707f +#define MATCH_VSSRL_VV 0xa8000057 +#define MASK_VSSRL_VV 0xfc00707f +#define MATCH_VSSRL_VX 0xa8004057 +#define MASK_VSSRL_VX 0xfc00707f +#define MATCH_VSSUB_VV 0x8c000057 +#define MASK_VSSUB_VV 0xfc00707f +#define MATCH_VSSUB_VX 0x8c004057 +#define MASK_VSSUB_VX 0xfc00707f +#define MATCH_VSSUBU_VV 0x88000057 +#define MASK_VSSUBU_VV 0xfc00707f +#define MATCH_VSSUBU_VX 0x88004057 +#define MASK_VSSUBU_VX 0xfc00707f +#define MATCH_VSUB_VV 0x8000057 +#define MASK_VSUB_VV 0xfc00707f +#define MATCH_VSUB_VX 0x8004057 +#define MASK_VSUB_VX 0xfc00707f +#define MATCH_VSUXEI1024_V 0x14007027 +#define MASK_VSUXEI1024_V 0x1c00707f +#define MATCH_VSUXEI128_V 0x14000027 +#define MASK_VSUXEI128_V 0x1c00707f +#define MATCH_VSUXEI16_V 0x4005027 +#define MASK_VSUXEI16_V 0x1c00707f +#define MATCH_VSUXEI256_V 0x14005027 +#define MASK_VSUXEI256_V 0x1c00707f +#define MATCH_VSUXEI32_V 0x4006027 +#define MASK_VSUXEI32_V 0x1c00707f +#define MATCH_VSUXEI512_V 0x14006027 +#define MASK_VSUXEI512_V 0x1c00707f +#define MATCH_VSUXEI64_V 0x4007027 +#define MASK_VSUXEI64_V 0x1c00707f +#define MATCH_VSUXEI8_V 0x4000027 +#define MASK_VSUXEI8_V 0x1c00707f +#define MATCH_VWADD_VV 0xc4002057 +#define MASK_VWADD_VV 0xfc00707f +#define MATCH_VWADD_VX 0xc4006057 +#define MASK_VWADD_VX 0xfc00707f +#define MATCH_VWADD_WV 0xd4002057 +#define MASK_VWADD_WV 0xfc00707f +#define MATCH_VWADD_WX 0xd4006057 +#define MASK_VWADD_WX 0xfc00707f +#define MATCH_VWADDU_VV 0xc0002057 +#define MASK_VWADDU_VV 0xfc00707f +#define MATCH_VWADDU_VX 0xc0006057 +#define MASK_VWADDU_VX 0xfc00707f +#define MATCH_VWADDU_WV 0xd0002057 +#define MASK_VWADDU_WV 0xfc00707f +#define MATCH_VWADDU_WX 0xd0006057 +#define MASK_VWADDU_WX 0xfc00707f +#define MATCH_VWMACC_VV 0xf4002057 +#define MASK_VWMACC_VV 0xfc00707f +#define MATCH_VWMACC_VX 0xf4006057 +#define MASK_VWMACC_VX 0xfc00707f +#define MATCH_VWMACCSU_VV 0xfc002057 +#define MASK_VWMACCSU_VV 0xfc00707f +#define MATCH_VWMACCSU_VX 0xfc006057 +#define MASK_VWMACCSU_VX 0xfc00707f +#define MATCH_VWMACCU_VV 0xf0002057 +#define MASK_VWMACCU_VV 0xfc00707f +#define MATCH_VWMACCU_VX 0xf0006057 +#define MASK_VWMACCU_VX 0xfc00707f +#define MATCH_VWMACCUS_VX 0xf8006057 +#define MASK_VWMACCUS_VX 0xfc00707f +#define MATCH_VWMUL_VV 0xec002057 +#define MASK_VWMUL_VV 0xfc00707f +#define MATCH_VWMUL_VX 0xec006057 +#define MASK_VWMUL_VX 0xfc00707f +#define MATCH_VWMULSU_VV 0xe8002057 +#define MASK_VWMULSU_VV 0xfc00707f +#define MATCH_VWMULSU_VX 0xe8006057 +#define MASK_VWMULSU_VX 0xfc00707f +#define MATCH_VWMULU_VV 0xe0002057 +#define MASK_VWMULU_VV 0xfc00707f +#define MATCH_VWMULU_VX 0xe0006057 +#define MASK_VWMULU_VX 0xfc00707f +#define MATCH_VWREDSUM_VS 0xc4000057 +#define MASK_VWREDSUM_VS 0xfc00707f +#define MATCH_VWREDSUMU_VS 0xc0000057 +#define MASK_VWREDSUMU_VS 0xfc00707f +#define MATCH_VWSUB_VV 0xcc002057 +#define MASK_VWSUB_VV 0xfc00707f +#define MATCH_VWSUB_VX 0xcc006057 +#define MASK_VWSUB_VX 0xfc00707f +#define MATCH_VWSUB_WV 0xdc002057 +#define MASK_VWSUB_WV 0xfc00707f +#define MATCH_VWSUB_WX 0xdc006057 +#define MASK_VWSUB_WX 0xfc00707f +#define MATCH_VWSUBU_VV 0xc8002057 +#define MASK_VWSUBU_VV 0xfc00707f +#define MATCH_VWSUBU_VX 0xc8006057 +#define MASK_VWSUBU_VX 0xfc00707f +#define MATCH_VWSUBU_WV 0xd8002057 +#define MASK_VWSUBU_WV 0xfc00707f +#define MATCH_VWSUBU_WX 0xd8006057 +#define MASK_VWSUBU_WX 0xfc00707f +#define MATCH_VXOR_VI 0x2c003057 +#define MASK_VXOR_VI 0xfc00707f +#define MATCH_VXOR_VV 0x2c000057 +#define MASK_VXOR_VV 0xfc00707f +#define MATCH_VXOR_VX 0x2c004057 +#define MASK_VXOR_VX 0xfc00707f +#define MATCH_VZEXT_VF2 0x48032057 +#define MASK_VZEXT_VF2 0xfc0ff07f +#define MATCH_VZEXT_VF4 0x48022057 +#define MASK_VZEXT_VF4 0xfc0ff07f +#define MATCH_VZEXT_VF8 0x48012057 +#define MASK_VZEXT_VF8 0xfc0ff07f +#define MATCH_WEXT 0xce000077 +#define MASK_WEXT 0xfe00707f +#define MATCH_WEXTI 0xde000077 +#define MASK_WEXTI 0xfe00707f +#define MATCH_WFI 0x10500073 +#define MASK_WFI 0xffffffff +#define MATCH_WRS_NTO 0xd00073 +#define MASK_WRS_NTO 0xffffffff +#define MATCH_WRS_STO 0x1d00073 +#define MASK_WRS_STO 0xffffffff +#define MATCH_XNOR 0x40004033 +#define MASK_XNOR 0xfe00707f +#define MATCH_XOR 0x4033 +#define MASK_XOR 0xfe00707f +#define MATCH_XORI 0x4013 +#define MASK_XORI 0x707f +#define MATCH_XPERM16 0x28006033 +#define MASK_XPERM16 0xfe00707f +#define MATCH_XPERM32 0x28000033 +#define MASK_XPERM32 0xfe00707f +#define MATCH_XPERM4 0x28002033 +#define MASK_XPERM4 0xfe00707f +#define MATCH_XPERM8 0x28004033 +#define MASK_XPERM8 0xfe00707f +#define MATCH_ZUNPKD810 0xacc00077 +#define MASK_ZUNPKD810 0xfff0707f +#define MATCH_ZUNPKD820 0xacd00077 +#define MASK_ZUNPKD820 0xfff0707f +#define MATCH_ZUNPKD830 0xace00077 +#define MASK_ZUNPKD830 0xfff0707f +#define MATCH_ZUNPKD831 0xacf00077 +#define MASK_ZUNPKD831 0xfff0707f +#define MATCH_ZUNPKD832 0xad700077 +#define MASK_ZUNPKD832 0xfff0707f + +#define CSR_FFLAGS 0x1 +#define CSR_FRM 0x2 +#define CSR_FCSR 0x3 +#define CSR_VSTART 0x8 +#define CSR_VXSAT 0x9 +#define CSR_VXRM 0xa +#define CSR_VCSR 0xf +#define CSR_SEED 0x15 +#define CSR_CYCLE 0xc00 +#define CSR_TIME 0xc01 +#define CSR_INSTRET 0xc02 +#define CSR_HPMCOUNTER3 0xc03 +#define CSR_HPMCOUNTER4 0xc04 +#define CSR_HPMCOUNTER5 0xc05 +#define CSR_HPMCOUNTER6 0xc06 +#define CSR_HPMCOUNTER7 0xc07 +#define CSR_HPMCOUNTER8 0xc08 +#define CSR_HPMCOUNTER9 0xc09 +#define CSR_HPMCOUNTER10 0xc0a +#define CSR_HPMCOUNTER11 0xc0b +#define CSR_HPMCOUNTER12 0xc0c +#define CSR_HPMCOUNTER13 0xc0d +#define CSR_HPMCOUNTER14 0xc0e +#define CSR_HPMCOUNTER15 0xc0f +#define CSR_HPMCOUNTER16 0xc10 +#define CSR_HPMCOUNTER17 0xc11 +#define CSR_HPMCOUNTER18 0xc12 +#define CSR_HPMCOUNTER19 0xc13 +#define CSR_HPMCOUNTER20 0xc14 +#define CSR_HPMCOUNTER21 0xc15 +#define CSR_HPMCOUNTER22 0xc16 +#define CSR_HPMCOUNTER23 0xc17 +#define CSR_HPMCOUNTER24 0xc18 +#define CSR_HPMCOUNTER25 0xc19 +#define CSR_HPMCOUNTER26 0xc1a +#define CSR_HPMCOUNTER27 0xc1b +#define CSR_HPMCOUNTER28 0xc1c +#define CSR_HPMCOUNTER29 0xc1d +#define CSR_HPMCOUNTER30 0xc1e +#define CSR_HPMCOUNTER31 0xc1f +#define CSR_VL 0xc20 +#define CSR_VTYPE 0xc21 +#define CSR_VLENB 0xc22 +#define CSR_SSTATUS 0x100 +#define CSR_SEDELEG 0x102 +#define CSR_SIDELEG 0x103 +#define CSR_SIE 0x104 +#define CSR_STVEC 0x105 +#define CSR_SCOUNTEREN 0x106 +#define CSR_SENVCFG 0x10a +#define CSR_SSTATEEN0 0x10c +#define CSR_SSTATEEN1 0x10d +#define CSR_SSTATEEN2 0x10e +#define CSR_SSTATEEN3 0x10f +#define CSR_SSCRATCH 0x140 +#define CSR_SEPC 0x141 +#define CSR_SCAUSE 0x142 +#define CSR_STVAL 0x143 +#define CSR_SIP 0x144 +#define CSR_STIMECMP 0x14d +#define CSR_SATP 0x180 +#define CSR_SCONTEXT 0x5a8 +#define CSR_VSSTATUS 0x200 +#define CSR_VSIE 0x204 +#define CSR_VSTVEC 0x205 +#define CSR_VSSCRATCH 0x240 +#define CSR_VSEPC 0x241 +#define CSR_VSCAUSE 0x242 +#define CSR_VSTVAL 0x243 +#define CSR_VSIP 0x244 +#define CSR_VSTIMECMP 0x24d +#define CSR_VSATP 0x280 +#define CSR_HSTATUS 0x600 +#define CSR_HEDELEG 0x602 +#define CSR_HIDELEG 0x603 +#define CSR_HIE 0x604 +#define CSR_HTIMEDELTA 0x605 +#define CSR_HCOUNTEREN 0x606 +#define CSR_HGEIE 0x607 +#define CSR_HENVCFG 0x60a +#define CSR_HSTATEEN0 0x60c +#define CSR_HSTATEEN1 0x60d +#define CSR_HSTATEEN2 0x60e +#define CSR_HSTATEEN3 0x60f +#define CSR_HTVAL 0x643 +#define CSR_HIP 0x644 +#define CSR_HVIP 0x645 +#define CSR_HTINST 0x64a +#define CSR_HGATP 0x680 +#define CSR_HCONTEXT 0x6a8 +#define CSR_HGEIP 0xe12 +#define CSR_SCOUNTOVF 0xda0 +#define CSR_UTVT 0x7 +#define CSR_UNXTI 0x45 +#define CSR_UINTSTATUS 0x46 +#define CSR_USCRATCHCSW 0x48 +#define CSR_USCRATCHCSWL 0x49 +#define CSR_STVT 0x107 +#define CSR_SNXTI 0x145 +#define CSR_SINTSTATUS 0x146 +#define CSR_SSCRATCHCSW 0x148 +#define CSR_SSCRATCHCSWL 0x149 +#define CSR_MTVT 0x307 +#define CSR_MNXTI 0x345 +#define CSR_MINTSTATUS 0x346 +#define CSR_MSCRATCHCSW 0x348 +#define CSR_MSCRATCHCSWL 0x349 +#define CSR_MSTATUS 0x300 +#define CSR_MISA 0x301 +#define CSR_MEDELEG 0x302 +#define CSR_MIDELEG 0x303 +#define CSR_MIE 0x304 +#define CSR_MTVEC 0x305 +#define CSR_MCOUNTEREN 0x306 +#define CSR_MENVCFG 0x30a +#define CSR_MSTATEEN0 0x30c +#define CSR_MSTATEEN1 0x30d +#define CSR_MSTATEEN2 0x30e +#define CSR_MSTATEEN3 0x30f +#define CSR_MCOUNTINHIBIT 0x320 +#define CSR_MSCRATCH 0x340 +#define CSR_MEPC 0x341 +#define CSR_MCAUSE 0x342 +#define CSR_MTVAL 0x343 +#define CSR_MIP 0x344 +#define CSR_MTINST 0x34a +#define CSR_MTVAL2 0x34b +#define CSR_PMPCFG0 0x3a0 +#define CSR_PMPCFG1 0x3a1 +#define CSR_PMPCFG2 0x3a2 +#define CSR_PMPCFG3 0x3a3 +#define CSR_PMPCFG4 0x3a4 +#define CSR_PMPCFG5 0x3a5 +#define CSR_PMPCFG6 0x3a6 +#define CSR_PMPCFG7 0x3a7 +#define CSR_PMPCFG8 0x3a8 +#define CSR_PMPCFG9 0x3a9 +#define CSR_PMPCFG10 0x3aa +#define CSR_PMPCFG11 0x3ab +#define CSR_PMPCFG12 0x3ac +#define CSR_PMPCFG13 0x3ad +#define CSR_PMPCFG14 0x3ae +#define CSR_PMPCFG15 0x3af +#define CSR_PMPADDR0 0x3b0 +#define CSR_PMPADDR1 0x3b1 +#define CSR_PMPADDR2 0x3b2 +#define CSR_PMPADDR3 0x3b3 +#define CSR_PMPADDR4 0x3b4 +#define CSR_PMPADDR5 0x3b5 +#define CSR_PMPADDR6 0x3b6 +#define CSR_PMPADDR7 0x3b7 +#define CSR_PMPADDR8 0x3b8 +#define CSR_PMPADDR9 0x3b9 +#define CSR_PMPADDR10 0x3ba +#define CSR_PMPADDR11 0x3bb +#define CSR_PMPADDR12 0x3bc +#define CSR_PMPADDR13 0x3bd +#define CSR_PMPADDR14 0x3be +#define CSR_PMPADDR15 0x3bf +#define CSR_PMPADDR16 0x3c0 +#define CSR_PMPADDR17 0x3c1 +#define CSR_PMPADDR18 0x3c2 +#define CSR_PMPADDR19 0x3c3 +#define CSR_PMPADDR20 0x3c4 +#define CSR_PMPADDR21 0x3c5 +#define CSR_PMPADDR22 0x3c6 +#define CSR_PMPADDR23 0x3c7 +#define CSR_PMPADDR24 0x3c8 +#define CSR_PMPADDR25 0x3c9 +#define CSR_PMPADDR26 0x3ca +#define CSR_PMPADDR27 0x3cb +#define CSR_PMPADDR28 0x3cc +#define CSR_PMPADDR29 0x3cd +#define CSR_PMPADDR30 0x3ce +#define CSR_PMPADDR31 0x3cf +#define CSR_PMPADDR32 0x3d0 +#define CSR_PMPADDR33 0x3d1 +#define CSR_PMPADDR34 0x3d2 +#define CSR_PMPADDR35 0x3d3 +#define CSR_PMPADDR36 0x3d4 +#define CSR_PMPADDR37 0x3d5 +#define CSR_PMPADDR38 0x3d6 +#define CSR_PMPADDR39 0x3d7 +#define CSR_PMPADDR40 0x3d8 +#define CSR_PMPADDR41 0x3d9 +#define CSR_PMPADDR42 0x3da +#define CSR_PMPADDR43 0x3db +#define CSR_PMPADDR44 0x3dc +#define CSR_PMPADDR45 0x3dd +#define CSR_PMPADDR46 0x3de +#define CSR_PMPADDR47 0x3df +#define CSR_PMPADDR48 0x3e0 +#define CSR_PMPADDR49 0x3e1 +#define CSR_PMPADDR50 0x3e2 +#define CSR_PMPADDR51 0x3e3 +#define CSR_PMPADDR52 0x3e4 +#define CSR_PMPADDR53 0x3e5 +#define CSR_PMPADDR54 0x3e6 +#define CSR_PMPADDR55 0x3e7 +#define CSR_PMPADDR56 0x3e8 +#define CSR_PMPADDR57 0x3e9 +#define CSR_PMPADDR58 0x3ea +#define CSR_PMPADDR59 0x3eb +#define CSR_PMPADDR60 0x3ec +#define CSR_PMPADDR61 0x3ed +#define CSR_PMPADDR62 0x3ee +#define CSR_PMPADDR63 0x3ef +#define CSR_MSECCFG 0x747 +#define CSR_TSELECT 0x7a0 +#define CSR_TDATA1 0x7a1 +#define CSR_TDATA2 0x7a2 +#define CSR_TDATA3 0x7a3 +#define CSR_TINFO 0x7a4 +#define CSR_TCONTROL 0x7a5 +#define CSR_MCONTEXT 0x7a8 +#define CSR_MSCONTEXT 0x7aa +#define CSR_DCSR 0x7b0 +#define CSR_DPC 0x7b1 +#define CSR_DSCRATCH0 0x7b2 +#define CSR_DSCRATCH1 0x7b3 +#define CSR_MCYCLE 0xb00 +#define CSR_MINSTRET 0xb02 +#define CSR_MHPMCOUNTER3 0xb03 +#define CSR_MHPMCOUNTER4 0xb04 +#define CSR_MHPMCOUNTER5 0xb05 +#define CSR_MHPMCOUNTER6 0xb06 +#define CSR_MHPMCOUNTER7 0xb07 +#define CSR_MHPMCOUNTER8 0xb08 +#define CSR_MHPMCOUNTER9 0xb09 +#define CSR_MHPMCOUNTER10 0xb0a +#define CSR_MHPMCOUNTER11 0xb0b +#define CSR_MHPMCOUNTER12 0xb0c +#define CSR_MHPMCOUNTER13 0xb0d +#define CSR_MHPMCOUNTER14 0xb0e +#define CSR_MHPMCOUNTER15 0xb0f +#define CSR_MHPMCOUNTER16 0xb10 +#define CSR_MHPMCOUNTER17 0xb11 +#define CSR_MHPMCOUNTER18 0xb12 +#define CSR_MHPMCOUNTER19 0xb13 +#define CSR_MHPMCOUNTER20 0xb14 +#define CSR_MHPMCOUNTER21 0xb15 +#define CSR_MHPMCOUNTER22 0xb16 +#define CSR_MHPMCOUNTER23 0xb17 +#define CSR_MHPMCOUNTER24 0xb18 +#define CSR_MHPMCOUNTER25 0xb19 +#define CSR_MHPMCOUNTER26 0xb1a +#define CSR_MHPMCOUNTER27 0xb1b +#define CSR_MHPMCOUNTER28 0xb1c +#define CSR_MHPMCOUNTER29 0xb1d +#define CSR_MHPMCOUNTER30 0xb1e +#define CSR_MHPMCOUNTER31 0xb1f +#define CSR_MHPMEVENT3 0x323 +#define CSR_MHPMEVENT4 0x324 +#define CSR_MHPMEVENT5 0x325 +#define CSR_MHPMEVENT6 0x326 +#define CSR_MHPMEVENT7 0x327 +#define CSR_MHPMEVENT8 0x328 +#define CSR_MHPMEVENT9 0x329 +#define CSR_MHPMEVENT10 0x32a +#define CSR_MHPMEVENT11 0x32b +#define CSR_MHPMEVENT12 0x32c +#define CSR_MHPMEVENT13 0x32d +#define CSR_MHPMEVENT14 0x32e +#define CSR_MHPMEVENT15 0x32f +#define CSR_MHPMEVENT16 0x330 +#define CSR_MHPMEVENT17 0x331 +#define CSR_MHPMEVENT18 0x332 +#define CSR_MHPMEVENT19 0x333 +#define CSR_MHPMEVENT20 0x334 +#define CSR_MHPMEVENT21 0x335 +#define CSR_MHPMEVENT22 0x336 +#define CSR_MHPMEVENT23 0x337 +#define CSR_MHPMEVENT24 0x338 +#define CSR_MHPMEVENT25 0x339 +#define CSR_MHPMEVENT26 0x33a +#define CSR_MHPMEVENT27 0x33b +#define CSR_MHPMEVENT28 0x33c +#define CSR_MHPMEVENT29 0x33d +#define CSR_MHPMEVENT30 0x33e +#define CSR_MHPMEVENT31 0x33f +#define CSR_MVENDORID 0xf11 +#define CSR_MARCHID 0xf12 +#define CSR_MIMPID 0xf13 +#define CSR_MHARTID 0xf14 +#define CSR_MCONFIGPTR 0xf15 +#define CSR_STIMECMPH 0x15d +#define CSR_VSTIMECMPH 0x25d +#define CSR_HTIMEDELTAH 0x615 +#define CSR_HENVCFGH 0x61a +#define CSR_HSTATEEN0H 0x61c +#define CSR_HSTATEEN1H 0x61d +#define CSR_HSTATEEN2H 0x61e +#define CSR_HSTATEEN3H 0x61f +#define CSR_CYCLEH 0xc80 +#define CSR_TIMEH 0xc81 +#define CSR_INSTRETH 0xc82 +#define CSR_HPMCOUNTER3H 0xc83 +#define CSR_HPMCOUNTER4H 0xc84 +#define CSR_HPMCOUNTER5H 0xc85 +#define CSR_HPMCOUNTER6H 0xc86 +#define CSR_HPMCOUNTER7H 0xc87 +#define CSR_HPMCOUNTER8H 0xc88 +#define CSR_HPMCOUNTER9H 0xc89 +#define CSR_HPMCOUNTER10H 0xc8a +#define CSR_HPMCOUNTER11H 0xc8b +#define CSR_HPMCOUNTER12H 0xc8c +#define CSR_HPMCOUNTER13H 0xc8d +#define CSR_HPMCOUNTER14H 0xc8e +#define CSR_HPMCOUNTER15H 0xc8f +#define CSR_HPMCOUNTER16H 0xc90 +#define CSR_HPMCOUNTER17H 0xc91 +#define CSR_HPMCOUNTER18H 0xc92 +#define CSR_HPMCOUNTER19H 0xc93 +#define CSR_HPMCOUNTER20H 0xc94 +#define CSR_HPMCOUNTER21H 0xc95 +#define CSR_HPMCOUNTER22H 0xc96 +#define CSR_HPMCOUNTER23H 0xc97 +#define CSR_HPMCOUNTER24H 0xc98 +#define CSR_HPMCOUNTER25H 0xc99 +#define CSR_HPMCOUNTER26H 0xc9a +#define CSR_HPMCOUNTER27H 0xc9b +#define CSR_HPMCOUNTER28H 0xc9c +#define CSR_HPMCOUNTER29H 0xc9d +#define CSR_HPMCOUNTER30H 0xc9e +#define CSR_HPMCOUNTER31H 0xc9f +#define CSR_MSTATUSH 0x310 +#define CSR_MENVCFGH 0x31a +#define CSR_MSTATEEN0H 0x31c +#define CSR_MSTATEEN1H 0x31d +#define CSR_MSTATEEN2H 0x31e +#define CSR_MSTATEEN3H 0x31f +#define CSR_MHPMEVENT3H 0x723 +#define CSR_MHPMEVENT4H 0x724 +#define CSR_MHPMEVENT5H 0x725 +#define CSR_MHPMEVENT6H 0x726 +#define CSR_MHPMEVENT7H 0x727 +#define CSR_MHPMEVENT8H 0x728 +#define CSR_MHPMEVENT9H 0x729 +#define CSR_MHPMEVENT10H 0x72a +#define CSR_MHPMEVENT11H 0x72b +#define CSR_MHPMEVENT12H 0x72c +#define CSR_MHPMEVENT13H 0x72d +#define CSR_MHPMEVENT14H 0x72e +#define CSR_MHPMEVENT15H 0x72f +#define CSR_MHPMEVENT16H 0x730 +#define CSR_MHPMEVENT17H 0x731 +#define CSR_MHPMEVENT18H 0x732 +#define CSR_MHPMEVENT19H 0x733 +#define CSR_MHPMEVENT20H 0x734 +#define CSR_MHPMEVENT21H 0x735 +#define CSR_MHPMEVENT22H 0x736 +#define CSR_MHPMEVENT23H 0x737 +#define CSR_MHPMEVENT24H 0x738 +#define CSR_MHPMEVENT25H 0x739 +#define CSR_MHPMEVENT26H 0x73a +#define CSR_MHPMEVENT27H 0x73b +#define CSR_MHPMEVENT28H 0x73c +#define CSR_MHPMEVENT29H 0x73d +#define CSR_MHPMEVENT30H 0x73e +#define CSR_MHPMEVENT31H 0x73f +#define CSR_MSECCFGH 0x757 +#define CSR_MCYCLEH 0xb80 +#define CSR_MINSTRETH 0xb82 +#define CSR_MHPMCOUNTER3H 0xb83 +#define CSR_MHPMCOUNTER4H 0xb84 +#define CSR_MHPMCOUNTER5H 0xb85 +#define CSR_MHPMCOUNTER6H 0xb86 +#define CSR_MHPMCOUNTER7H 0xb87 +#define CSR_MHPMCOUNTER8H 0xb88 +#define CSR_MHPMCOUNTER9H 0xb89 +#define CSR_MHPMCOUNTER10H 0xb8a +#define CSR_MHPMCOUNTER11H 0xb8b +#define CSR_MHPMCOUNTER12H 0xb8c +#define CSR_MHPMCOUNTER13H 0xb8d +#define CSR_MHPMCOUNTER14H 0xb8e +#define CSR_MHPMCOUNTER15H 0xb8f +#define CSR_MHPMCOUNTER16H 0xb90 +#define CSR_MHPMCOUNTER17H 0xb91 +#define CSR_MHPMCOUNTER18H 0xb92 +#define CSR_MHPMCOUNTER19H 0xb93 +#define CSR_MHPMCOUNTER20H 0xb94 +#define CSR_MHPMCOUNTER21H 0xb95 +#define CSR_MHPMCOUNTER22H 0xb96 +#define CSR_MHPMCOUNTER23H 0xb97 +#define CSR_MHPMCOUNTER24H 0xb98 +#define CSR_MHPMCOUNTER25H 0xb99 +#define CSR_MHPMCOUNTER26H 0xb9a +#define CSR_MHPMCOUNTER27H 0xb9b +#define CSR_MHPMCOUNTER28H 0xb9c +#define CSR_MHPMCOUNTER29H 0xb9d +#define CSR_MHPMCOUNTER30H 0xb9e +#define CSR_MHPMCOUNTER31H 0xb9f + +#define CAUSE_MISALIGNED_FETCH 0x0 +#define CAUSE_FETCH_ACCESS 0x1 +#define CAUSE_ILLEGAL_INSTRUCTION 0x2 +#define CAUSE_BREAKPOINT 0x3 +#define CAUSE_MISALIGNED_LOAD 0x4 +#define CAUSE_LOAD_ACCESS 0x5 +#define CAUSE_MISALIGNED_STORE 0x6 +#define CAUSE_STORE_ACCESS 0x7 +#define CAUSE_USER_ECALL 0x8 +#define CAUSE_SUPERVISOR_ECALL 0x9 +#define CAUSE_VIRTUAL_SUPERVISOR_ECALL 0xa +#define CAUSE_MACHINE_ECALL 0xb +#define CAUSE_FETCH_PAGE_FAULT 0xc +#define CAUSE_LOAD_PAGE_FAULT 0xd +#define CAUSE_STORE_PAGE_FAULT 0xf +#define CAUSE_FETCH_GUEST_PAGE_FAULT 0x14 +#define CAUSE_LOAD_GUEST_PAGE_FAULT 0x15 +#define CAUSE_VIRTUAL_INSTRUCTION 0x16 +#define CAUSE_STORE_GUEST_PAGE_FAULT 0x17 + +#define INSN_FIELD_RD 0xf80 +#define INSN_FIELD_RT 0xf8000 +#define INSN_FIELD_RS1 0xf8000 +#define INSN_FIELD_RS2 0x1f00000 +#define INSN_FIELD_RS3 0xf8000000 +#define INSN_FIELD_AQRL 0x6000000 +#define INSN_FIELD_AQ 0x4000000 +#define INSN_FIELD_RL 0x2000000 +#define INSN_FIELD_FM 0xf0000000 +#define INSN_FIELD_PRED 0xf000000 +#define INSN_FIELD_SUCC 0xf00000 +#define INSN_FIELD_RM 0x7000 +#define INSN_FIELD_FUNCT3 0x7000 +#define INSN_FIELD_FUNCT2 0x6000000 +#define INSN_FIELD_IMM20 0xfffff000 +#define INSN_FIELD_JIMM20 0xfffff000 +#define INSN_FIELD_IMM12 0xfff00000 +#define INSN_FIELD_CSR 0xfff00000 +#define INSN_FIELD_IMM12HI 0xfe000000 +#define INSN_FIELD_BIMM12HI 0xfe000000 +#define INSN_FIELD_IMM12LO 0xf80 +#define INSN_FIELD_BIMM12LO 0xf80 +#define INSN_FIELD_ZIMM 0xf8000 +#define INSN_FIELD_SHAMT 0x7f00000 +#define INSN_FIELD_SHAMTW 0x1f00000 +#define INSN_FIELD_SHAMTW4 0xf00000 +#define INSN_FIELD_SHAMTD 0x3f00000 +#define INSN_FIELD_BS 0xc0000000 +#define INSN_FIELD_RNUM 0xf00000 +#define INSN_FIELD_RC 0x3e000000 +#define INSN_FIELD_IMM2 0x300000 +#define INSN_FIELD_IMM3 0x700000 +#define INSN_FIELD_IMM4 0xf00000 +#define INSN_FIELD_IMM5 0x1f00000 +#define INSN_FIELD_IMM6 0x3f00000 +#define INSN_FIELD_OPCODE 0x7f +#define INSN_FIELD_FUNCT7 0xfe000000 +#define INSN_FIELD_VD 0xf80 +#define INSN_FIELD_VS3 0xf80 +#define INSN_FIELD_VS1 0xf8000 +#define INSN_FIELD_VS2 0x1f00000 +#define INSN_FIELD_VM 0x2000000 +#define INSN_FIELD_WD 0x4000000 +#define INSN_FIELD_AMOOP 0xf8000000 +#define INSN_FIELD_NF 0xe0000000 +#define INSN_FIELD_SIMM5 0xf8000 +#define INSN_FIELD_ZIMM10 0x3ff00000 +#define INSN_FIELD_ZIMM11 0x7ff00000 +#define INSN_FIELD_C_NZUIMM10 0x1fe0 +#define INSN_FIELD_C_UIMM7LO 0x60 +#define INSN_FIELD_C_UIMM7HI 0x1c00 +#define INSN_FIELD_C_UIMM8LO 0x60 +#define INSN_FIELD_C_UIMM8HI 0x1c00 +#define INSN_FIELD_C_UIMM9LO 0x60 +#define INSN_FIELD_C_UIMM9HI 0x1c00 +#define INSN_FIELD_C_NZIMM6LO 0x7c +#define INSN_FIELD_C_NZIMM6HI 0x1000 +#define INSN_FIELD_C_IMM6LO 0x7c +#define INSN_FIELD_C_IMM6HI 0x1000 +#define INSN_FIELD_C_NZIMM10HI 0x1000 +#define INSN_FIELD_C_NZIMM10LO 0x7c +#define INSN_FIELD_C_NZIMM18HI 0x1000 +#define INSN_FIELD_C_NZIMM18LO 0x7c +#define INSN_FIELD_C_IMM12 0x1ffc +#define INSN_FIELD_C_BIMM9LO 0x7c +#define INSN_FIELD_C_BIMM9HI 0x1c00 +#define INSN_FIELD_C_NZUIMM5 0x7c +#define INSN_FIELD_C_NZUIMM6LO 0x7c +#define INSN_FIELD_C_NZUIMM6HI 0x1000 +#define INSN_FIELD_C_UIMM8SPLO 0x7c +#define INSN_FIELD_C_UIMM8SPHI 0x1000 +#define INSN_FIELD_C_UIMM8SP_S 0x1f80 +#define INSN_FIELD_C_UIMM10SPLO 0x7c +#define INSN_FIELD_C_UIMM10SPHI 0x1000 +#define INSN_FIELD_C_UIMM9SPLO 0x7c +#define INSN_FIELD_C_UIMM9SPHI 0x1000 +#define INSN_FIELD_C_UIMM10SP_S 0x1f80 +#define INSN_FIELD_C_UIMM9SP_S 0x1f80 +#define INSN_FIELD_RS1_P 0x380 +#define INSN_FIELD_RS2_P 0x1c +#define INSN_FIELD_RD_P 0x1c +#define INSN_FIELD_RD_RS1_N0 0xf80 +#define INSN_FIELD_RD_RS1_P 0x380 +#define INSN_FIELD_RD_RS1 0xf80 +#define INSN_FIELD_RD_N2 0xf80 +#define INSN_FIELD_RD_N0 0xf80 +#define INSN_FIELD_RS1_N0 0xf80 +#define INSN_FIELD_C_RS2_N0 0x7c +#define INSN_FIELD_C_RS1_N0 0xf80 +#define INSN_FIELD_C_RS2 0x7c +#endif +#ifdef DECLARE_INSN +DECLARE_INSN(add, MATCH_ADD, MASK_ADD) +DECLARE_INSN(add16, MATCH_ADD16, MASK_ADD16) +DECLARE_INSN(add32, MATCH_ADD32, MASK_ADD32) +DECLARE_INSN(add64, MATCH_ADD64, MASK_ADD64) +DECLARE_INSN(add8, MATCH_ADD8, MASK_ADD8) +DECLARE_INSN(add_uw, MATCH_ADD_UW, MASK_ADD_UW) +DECLARE_INSN(addd, MATCH_ADDD, MASK_ADDD) +DECLARE_INSN(addi, MATCH_ADDI, MASK_ADDI) +DECLARE_INSN(addid, MATCH_ADDID, MASK_ADDID) +DECLARE_INSN(addiw, MATCH_ADDIW, MASK_ADDIW) +DECLARE_INSN(addw, MATCH_ADDW, MASK_ADDW) +DECLARE_INSN(aes32dsi, MATCH_AES32DSI, MASK_AES32DSI) +DECLARE_INSN(aes32dsmi, MATCH_AES32DSMI, MASK_AES32DSMI) +DECLARE_INSN(aes32esi, MATCH_AES32ESI, MASK_AES32ESI) +DECLARE_INSN(aes32esmi, MATCH_AES32ESMI, MASK_AES32ESMI) +DECLARE_INSN(aes64ds, MATCH_AES64DS, MASK_AES64DS) +DECLARE_INSN(aes64dsm, MATCH_AES64DSM, MASK_AES64DSM) +DECLARE_INSN(aes64es, MATCH_AES64ES, MASK_AES64ES) +DECLARE_INSN(aes64esm, MATCH_AES64ESM, MASK_AES64ESM) +DECLARE_INSN(aes64im, MATCH_AES64IM, MASK_AES64IM) +DECLARE_INSN(aes64ks1i, MATCH_AES64KS1I, MASK_AES64KS1I) +DECLARE_INSN(aes64ks2, MATCH_AES64KS2, MASK_AES64KS2) +DECLARE_INSN(amoadd_d, MATCH_AMOADD_D, MASK_AMOADD_D) +DECLARE_INSN(amoadd_w, MATCH_AMOADD_W, MASK_AMOADD_W) +DECLARE_INSN(amoand_d, MATCH_AMOAND_D, MASK_AMOAND_D) +DECLARE_INSN(amoand_w, MATCH_AMOAND_W, MASK_AMOAND_W) +DECLARE_INSN(amomax_d, MATCH_AMOMAX_D, MASK_AMOMAX_D) +DECLARE_INSN(amomax_w, MATCH_AMOMAX_W, MASK_AMOMAX_W) +DECLARE_INSN(amomaxu_d, MATCH_AMOMAXU_D, MASK_AMOMAXU_D) +DECLARE_INSN(amomaxu_w, MATCH_AMOMAXU_W, MASK_AMOMAXU_W) +DECLARE_INSN(amomin_d, MATCH_AMOMIN_D, MASK_AMOMIN_D) +DECLARE_INSN(amomin_w, MATCH_AMOMIN_W, MASK_AMOMIN_W) +DECLARE_INSN(amominu_d, MATCH_AMOMINU_D, MASK_AMOMINU_D) +DECLARE_INSN(amominu_w, MATCH_AMOMINU_W, MASK_AMOMINU_W) +DECLARE_INSN(amoor_d, MATCH_AMOOR_D, MASK_AMOOR_D) +DECLARE_INSN(amoor_w, MATCH_AMOOR_W, MASK_AMOOR_W) +DECLARE_INSN(amoswap_d, MATCH_AMOSWAP_D, MASK_AMOSWAP_D) +DECLARE_INSN(amoswap_w, MATCH_AMOSWAP_W, MASK_AMOSWAP_W) +DECLARE_INSN(amoxor_d, MATCH_AMOXOR_D, MASK_AMOXOR_D) +DECLARE_INSN(amoxor_w, MATCH_AMOXOR_W, MASK_AMOXOR_W) +DECLARE_INSN(and, MATCH_AND, MASK_AND) +DECLARE_INSN(andi, MATCH_ANDI, MASK_ANDI) +DECLARE_INSN(andn, MATCH_ANDN, MASK_ANDN) +DECLARE_INSN(auipc, MATCH_AUIPC, MASK_AUIPC) +DECLARE_INSN(ave, MATCH_AVE, MASK_AVE) +DECLARE_INSN(bclr, MATCH_BCLR, MASK_BCLR) +DECLARE_INSN(bclri, MATCH_BCLRI, MASK_BCLRI) +DECLARE_INSN(bcompress, MATCH_BCOMPRESS, MASK_BCOMPRESS) +DECLARE_INSN(bcompressw, MATCH_BCOMPRESSW, MASK_BCOMPRESSW) +DECLARE_INSN(bdecompress, MATCH_BDECOMPRESS, MASK_BDECOMPRESS) +DECLARE_INSN(bdecompressw, MATCH_BDECOMPRESSW, MASK_BDECOMPRESSW) +DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) +DECLARE_INSN(bext, MATCH_BEXT, MASK_BEXT) +DECLARE_INSN(bexti, MATCH_BEXTI, MASK_BEXTI) +DECLARE_INSN(bfp, MATCH_BFP, MASK_BFP) +DECLARE_INSN(bfpw, MATCH_BFPW, MASK_BFPW) +DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) +DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) +DECLARE_INSN(binv, MATCH_BINV, MASK_BINV) +DECLARE_INSN(binvi, MATCH_BINVI, MASK_BINVI) +DECLARE_INSN(bitrev, MATCH_BITREV, MASK_BITREV) +DECLARE_INSN(bitrevi, MATCH_BITREVI, MASK_BITREVI) +DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) +DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) +DECLARE_INSN(bmatflip, MATCH_BMATFLIP, MASK_BMATFLIP) +DECLARE_INSN(bmator, MATCH_BMATOR, MASK_BMATOR) +DECLARE_INSN(bmatxor, MATCH_BMATXOR, MASK_BMATXOR) +DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) +DECLARE_INSN(bpick, MATCH_BPICK, MASK_BPICK) +DECLARE_INSN(bset, MATCH_BSET, MASK_BSET) +DECLARE_INSN(bseti, MATCH_BSETI, MASK_BSETI) +DECLARE_INSN(c_add, MATCH_C_ADD, MASK_C_ADD) +DECLARE_INSN(c_addi, MATCH_C_ADDI, MASK_C_ADDI) +DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP) +DECLARE_INSN(c_addi4spn, MATCH_C_ADDI4SPN, MASK_C_ADDI4SPN) +DECLARE_INSN(c_addiw, MATCH_C_ADDIW, MASK_C_ADDIW) +DECLARE_INSN(c_addw, MATCH_C_ADDW, MASK_C_ADDW) +DECLARE_INSN(c_and, MATCH_C_AND, MASK_C_AND) +DECLARE_INSN(c_andi, MATCH_C_ANDI, MASK_C_ANDI) +DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) +DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) +DECLARE_INSN(c_ebreak, MATCH_C_EBREAK, MASK_C_EBREAK) +DECLARE_INSN(c_fld, MATCH_C_FLD, MASK_C_FLD) +DECLARE_INSN(c_fldsp, MATCH_C_FLDSP, MASK_C_FLDSP) +DECLARE_INSN(c_flw, MATCH_C_FLW, MASK_C_FLW) +DECLARE_INSN(c_flwsp, MATCH_C_FLWSP, MASK_C_FLWSP) +DECLARE_INSN(c_fsd, MATCH_C_FSD, MASK_C_FSD) +DECLARE_INSN(c_fsdsp, MATCH_C_FSDSP, MASK_C_FSDSP) +DECLARE_INSN(c_fsw, MATCH_C_FSW, MASK_C_FSW) +DECLARE_INSN(c_fswsp, MATCH_C_FSWSP, MASK_C_FSWSP) +DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) +DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) +DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) +DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) +DECLARE_INSN(c_ld, MATCH_C_LD, MASK_C_LD) +DECLARE_INSN(c_ldsp, MATCH_C_LDSP, MASK_C_LDSP) +DECLARE_INSN(c_li, MATCH_C_LI, MASK_C_LI) +DECLARE_INSN(c_lq, MATCH_C_LQ, MASK_C_LQ) +DECLARE_INSN(c_lqsp, MATCH_C_LQSP, MASK_C_LQSP) +DECLARE_INSN(c_lui, MATCH_C_LUI, MASK_C_LUI) +DECLARE_INSN(c_lw, MATCH_C_LW, MASK_C_LW) +DECLARE_INSN(c_lwsp, MATCH_C_LWSP, MASK_C_LWSP) +DECLARE_INSN(c_mv, MATCH_C_MV, MASK_C_MV) +DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP) +DECLARE_INSN(c_or, MATCH_C_OR, MASK_C_OR) +DECLARE_INSN(c_sd, MATCH_C_SD, MASK_C_SD) +DECLARE_INSN(c_sdsp, MATCH_C_SDSP, MASK_C_SDSP) +DECLARE_INSN(c_slli, MATCH_C_SLLI, MASK_C_SLLI) +DECLARE_INSN(c_sq, MATCH_C_SQ, MASK_C_SQ) +DECLARE_INSN(c_sqsp, MATCH_C_SQSP, MASK_C_SQSP) +DECLARE_INSN(c_srai, MATCH_C_SRAI, MASK_C_SRAI) +DECLARE_INSN(c_srli, MATCH_C_SRLI, MASK_C_SRLI) +DECLARE_INSN(c_sub, MATCH_C_SUB, MASK_C_SUB) +DECLARE_INSN(c_subw, MATCH_C_SUBW, MASK_C_SUBW) +DECLARE_INSN(c_sw, MATCH_C_SW, MASK_C_SW) +DECLARE_INSN(c_swsp, MATCH_C_SWSP, MASK_C_SWSP) +DECLARE_INSN(c_xor, MATCH_C_XOR, MASK_C_XOR) +DECLARE_INSN(cbo_clean, MATCH_CBO_CLEAN, MASK_CBO_CLEAN) +DECLARE_INSN(cbo_flush, MATCH_CBO_FLUSH, MASK_CBO_FLUSH) +DECLARE_INSN(cbo_inval, MATCH_CBO_INVAL, MASK_CBO_INVAL) +DECLARE_INSN(cbo_zero, MATCH_CBO_ZERO, MASK_CBO_ZERO) +DECLARE_INSN(clmul, MATCH_CLMUL, MASK_CLMUL) +DECLARE_INSN(clmulh, MATCH_CLMULH, MASK_CLMULH) +DECLARE_INSN(clmulr, MATCH_CLMULR, MASK_CLMULR) +DECLARE_INSN(clo16, MATCH_CLO16, MASK_CLO16) +DECLARE_INSN(clo32, MATCH_CLO32, MASK_CLO32) +DECLARE_INSN(clo8, MATCH_CLO8, MASK_CLO8) +DECLARE_INSN(clrs16, MATCH_CLRS16, MASK_CLRS16) +DECLARE_INSN(clrs32, MATCH_CLRS32, MASK_CLRS32) +DECLARE_INSN(clrs8, MATCH_CLRS8, MASK_CLRS8) +DECLARE_INSN(clz, MATCH_CLZ, MASK_CLZ) +DECLARE_INSN(clz16, MATCH_CLZ16, MASK_CLZ16) +DECLARE_INSN(clz32, MATCH_CLZ32, MASK_CLZ32) +DECLARE_INSN(clz8, MATCH_CLZ8, MASK_CLZ8) +DECLARE_INSN(clzw, MATCH_CLZW, MASK_CLZW) +DECLARE_INSN(cmix, MATCH_CMIX, MASK_CMIX) +DECLARE_INSN(cmov, MATCH_CMOV, MASK_CMOV) +DECLARE_INSN(cmpeq16, MATCH_CMPEQ16, MASK_CMPEQ16) +DECLARE_INSN(cmpeq8, MATCH_CMPEQ8, MASK_CMPEQ8) +DECLARE_INSN(cpop, MATCH_CPOP, MASK_CPOP) +DECLARE_INSN(cpopw, MATCH_CPOPW, MASK_CPOPW) +DECLARE_INSN(cras16, MATCH_CRAS16, MASK_CRAS16) +DECLARE_INSN(cras32, MATCH_CRAS32, MASK_CRAS32) +DECLARE_INSN(crc32_b, MATCH_CRC32_B, MASK_CRC32_B) +DECLARE_INSN(crc32_d, MATCH_CRC32_D, MASK_CRC32_D) +DECLARE_INSN(crc32_h, MATCH_CRC32_H, MASK_CRC32_H) +DECLARE_INSN(crc32_w, MATCH_CRC32_W, MASK_CRC32_W) +DECLARE_INSN(crc32c_b, MATCH_CRC32C_B, MASK_CRC32C_B) +DECLARE_INSN(crc32c_d, MATCH_CRC32C_D, MASK_CRC32C_D) +DECLARE_INSN(crc32c_h, MATCH_CRC32C_H, MASK_CRC32C_H) +DECLARE_INSN(crc32c_w, MATCH_CRC32C_W, MASK_CRC32C_W) +DECLARE_INSN(crsa16, MATCH_CRSA16, MASK_CRSA16) +DECLARE_INSN(crsa32, MATCH_CRSA32, MASK_CRSA32) +DECLARE_INSN(csrrc, MATCH_CSRRC, MASK_CSRRC) +DECLARE_INSN(csrrci, MATCH_CSRRCI, MASK_CSRRCI) +DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS) +DECLARE_INSN(csrrsi, MATCH_CSRRSI, MASK_CSRRSI) +DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW) +DECLARE_INSN(csrrwi, MATCH_CSRRWI, MASK_CSRRWI) +DECLARE_INSN(ctz, MATCH_CTZ, MASK_CTZ) +DECLARE_INSN(ctzw, MATCH_CTZW, MASK_CTZW) +DECLARE_INSN(div, MATCH_DIV, MASK_DIV) +DECLARE_INSN(divu, MATCH_DIVU, MASK_DIVU) +DECLARE_INSN(divuw, MATCH_DIVUW, MASK_DIVUW) +DECLARE_INSN(divw, MATCH_DIVW, MASK_DIVW) +DECLARE_INSN(dret, MATCH_DRET, MASK_DRET) +DECLARE_INSN(ebreak, MATCH_EBREAK, MASK_EBREAK) +DECLARE_INSN(ecall, MATCH_ECALL, MASK_ECALL) +DECLARE_INSN(fadd_d, MATCH_FADD_D, MASK_FADD_D) +DECLARE_INSN(fadd_h, MATCH_FADD_H, MASK_FADD_H) +DECLARE_INSN(fadd_q, MATCH_FADD_Q, MASK_FADD_Q) +DECLARE_INSN(fadd_s, MATCH_FADD_S, MASK_FADD_S) +DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D) +DECLARE_INSN(fclass_h, MATCH_FCLASS_H, MASK_FCLASS_H) +DECLARE_INSN(fclass_q, MATCH_FCLASS_Q, MASK_FCLASS_Q) +DECLARE_INSN(fclass_s, MATCH_FCLASS_S, MASK_FCLASS_S) +DECLARE_INSN(fcvt_d_h, MATCH_FCVT_D_H, MASK_FCVT_D_H) +DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L) +DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU) +DECLARE_INSN(fcvt_d_q, MATCH_FCVT_D_Q, MASK_FCVT_D_Q) +DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S) +DECLARE_INSN(fcvt_d_w, MATCH_FCVT_D_W, MASK_FCVT_D_W) +DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU) +DECLARE_INSN(fcvt_h_d, MATCH_FCVT_H_D, MASK_FCVT_H_D) +DECLARE_INSN(fcvt_h_l, MATCH_FCVT_H_L, MASK_FCVT_H_L) +DECLARE_INSN(fcvt_h_lu, MATCH_FCVT_H_LU, MASK_FCVT_H_LU) +DECLARE_INSN(fcvt_h_q, MATCH_FCVT_H_Q, MASK_FCVT_H_Q) +DECLARE_INSN(fcvt_h_s, MATCH_FCVT_H_S, MASK_FCVT_H_S) +DECLARE_INSN(fcvt_h_w, MATCH_FCVT_H_W, MASK_FCVT_H_W) +DECLARE_INSN(fcvt_h_wu, MATCH_FCVT_H_WU, MASK_FCVT_H_WU) +DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D) +DECLARE_INSN(fcvt_l_h, MATCH_FCVT_L_H, MASK_FCVT_L_H) +DECLARE_INSN(fcvt_l_q, MATCH_FCVT_L_Q, MASK_FCVT_L_Q) +DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S) +DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D) +DECLARE_INSN(fcvt_lu_h, MATCH_FCVT_LU_H, MASK_FCVT_LU_H) +DECLARE_INSN(fcvt_lu_q, MATCH_FCVT_LU_Q, MASK_FCVT_LU_Q) +DECLARE_INSN(fcvt_lu_s, MATCH_FCVT_LU_S, MASK_FCVT_LU_S) +DECLARE_INSN(fcvt_q_d, MATCH_FCVT_Q_D, MASK_FCVT_Q_D) +DECLARE_INSN(fcvt_q_h, MATCH_FCVT_Q_H, MASK_FCVT_Q_H) +DECLARE_INSN(fcvt_q_l, MATCH_FCVT_Q_L, MASK_FCVT_Q_L) +DECLARE_INSN(fcvt_q_lu, MATCH_FCVT_Q_LU, MASK_FCVT_Q_LU) +DECLARE_INSN(fcvt_q_s, MATCH_FCVT_Q_S, MASK_FCVT_Q_S) +DECLARE_INSN(fcvt_q_w, MATCH_FCVT_Q_W, MASK_FCVT_Q_W) +DECLARE_INSN(fcvt_q_wu, MATCH_FCVT_Q_WU, MASK_FCVT_Q_WU) +DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D) +DECLARE_INSN(fcvt_s_h, MATCH_FCVT_S_H, MASK_FCVT_S_H) +DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L) +DECLARE_INSN(fcvt_s_lu, MATCH_FCVT_S_LU, MASK_FCVT_S_LU) +DECLARE_INSN(fcvt_s_q, MATCH_FCVT_S_Q, MASK_FCVT_S_Q) +DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W) +DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU) +DECLARE_INSN(fcvt_w_d, MATCH_FCVT_W_D, MASK_FCVT_W_D) +DECLARE_INSN(fcvt_w_h, MATCH_FCVT_W_H, MASK_FCVT_W_H) +DECLARE_INSN(fcvt_w_q, MATCH_FCVT_W_Q, MASK_FCVT_W_Q) +DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S) +DECLARE_INSN(fcvt_wu_d, MATCH_FCVT_WU_D, MASK_FCVT_WU_D) +DECLARE_INSN(fcvt_wu_h, MATCH_FCVT_WU_H, MASK_FCVT_WU_H) +DECLARE_INSN(fcvt_wu_q, MATCH_FCVT_WU_Q, MASK_FCVT_WU_Q) +DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S) +DECLARE_INSN(fdiv_d, MATCH_FDIV_D, MASK_FDIV_D) +DECLARE_INSN(fdiv_h, MATCH_FDIV_H, MASK_FDIV_H) +DECLARE_INSN(fdiv_q, MATCH_FDIV_Q, MASK_FDIV_Q) +DECLARE_INSN(fdiv_s, MATCH_FDIV_S, MASK_FDIV_S) +DECLARE_INSN(fence, MATCH_FENCE, MASK_FENCE) +DECLARE_INSN(fence_i, MATCH_FENCE_I, MASK_FENCE_I) +DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D) +DECLARE_INSN(feq_h, MATCH_FEQ_H, MASK_FEQ_H) +DECLARE_INSN(feq_q, MATCH_FEQ_Q, MASK_FEQ_Q) +DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S) +DECLARE_INSN(fld, MATCH_FLD, MASK_FLD) +DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D) +DECLARE_INSN(fle_h, MATCH_FLE_H, MASK_FLE_H) +DECLARE_INSN(fle_q, MATCH_FLE_Q, MASK_FLE_Q) +DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S) +DECLARE_INSN(flh, MATCH_FLH, MASK_FLH) +DECLARE_INSN(flq, MATCH_FLQ, MASK_FLQ) +DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D) +DECLARE_INSN(flt_h, MATCH_FLT_H, MASK_FLT_H) +DECLARE_INSN(flt_q, MATCH_FLT_Q, MASK_FLT_Q) +DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S) +DECLARE_INSN(flw, MATCH_FLW, MASK_FLW) +DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D) +DECLARE_INSN(fmadd_h, MATCH_FMADD_H, MASK_FMADD_H) +DECLARE_INSN(fmadd_q, MATCH_FMADD_Q, MASK_FMADD_Q) +DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S) +DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D) +DECLARE_INSN(fmax_h, MATCH_FMAX_H, MASK_FMAX_H) +DECLARE_INSN(fmax_q, MATCH_FMAX_Q, MASK_FMAX_Q) +DECLARE_INSN(fmax_s, MATCH_FMAX_S, MASK_FMAX_S) +DECLARE_INSN(fmin_d, MATCH_FMIN_D, MASK_FMIN_D) +DECLARE_INSN(fmin_h, MATCH_FMIN_H, MASK_FMIN_H) +DECLARE_INSN(fmin_q, MATCH_FMIN_Q, MASK_FMIN_Q) +DECLARE_INSN(fmin_s, MATCH_FMIN_S, MASK_FMIN_S) +DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D) +DECLARE_INSN(fmsub_h, MATCH_FMSUB_H, MASK_FMSUB_H) +DECLARE_INSN(fmsub_q, MATCH_FMSUB_Q, MASK_FMSUB_Q) +DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S) +DECLARE_INSN(fmul_d, MATCH_FMUL_D, MASK_FMUL_D) +DECLARE_INSN(fmul_h, MATCH_FMUL_H, MASK_FMUL_H) +DECLARE_INSN(fmul_q, MATCH_FMUL_Q, MASK_FMUL_Q) +DECLARE_INSN(fmul_s, MATCH_FMUL_S, MASK_FMUL_S) +DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X) +DECLARE_INSN(fmv_h_x, MATCH_FMV_H_X, MASK_FMV_H_X) +DECLARE_INSN(fmv_w_x, MATCH_FMV_W_X, MASK_FMV_W_X) +DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D) +DECLARE_INSN(fmv_x_h, MATCH_FMV_X_H, MASK_FMV_X_H) +DECLARE_INSN(fmv_x_w, MATCH_FMV_X_W, MASK_FMV_X_W) +DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D) +DECLARE_INSN(fnmadd_h, MATCH_FNMADD_H, MASK_FNMADD_H) +DECLARE_INSN(fnmadd_q, MATCH_FNMADD_Q, MASK_FNMADD_Q) +DECLARE_INSN(fnmadd_s, MATCH_FNMADD_S, MASK_FNMADD_S) +DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D) +DECLARE_INSN(fnmsub_h, MATCH_FNMSUB_H, MASK_FNMSUB_H) +DECLARE_INSN(fnmsub_q, MATCH_FNMSUB_Q, MASK_FNMSUB_Q) +DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S) +DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD) +DECLARE_INSN(fsgnj_d, MATCH_FSGNJ_D, MASK_FSGNJ_D) +DECLARE_INSN(fsgnj_h, MATCH_FSGNJ_H, MASK_FSGNJ_H) +DECLARE_INSN(fsgnj_q, MATCH_FSGNJ_Q, MASK_FSGNJ_Q) +DECLARE_INSN(fsgnj_s, MATCH_FSGNJ_S, MASK_FSGNJ_S) +DECLARE_INSN(fsgnjn_d, MATCH_FSGNJN_D, MASK_FSGNJN_D) +DECLARE_INSN(fsgnjn_h, MATCH_FSGNJN_H, MASK_FSGNJN_H) +DECLARE_INSN(fsgnjn_q, MATCH_FSGNJN_Q, MASK_FSGNJN_Q) +DECLARE_INSN(fsgnjn_s, MATCH_FSGNJN_S, MASK_FSGNJN_S) +DECLARE_INSN(fsgnjx_d, MATCH_FSGNJX_D, MASK_FSGNJX_D) +DECLARE_INSN(fsgnjx_h, MATCH_FSGNJX_H, MASK_FSGNJX_H) +DECLARE_INSN(fsgnjx_q, MATCH_FSGNJX_Q, MASK_FSGNJX_Q) +DECLARE_INSN(fsgnjx_s, MATCH_FSGNJX_S, MASK_FSGNJX_S) +DECLARE_INSN(fsh, MATCH_FSH, MASK_FSH) +DECLARE_INSN(fsl, MATCH_FSL, MASK_FSL) +DECLARE_INSN(fslw, MATCH_FSLW, MASK_FSLW) +DECLARE_INSN(fsq, MATCH_FSQ, MASK_FSQ) +DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D) +DECLARE_INSN(fsqrt_h, MATCH_FSQRT_H, MASK_FSQRT_H) +DECLARE_INSN(fsqrt_q, MATCH_FSQRT_Q, MASK_FSQRT_Q) +DECLARE_INSN(fsqrt_s, MATCH_FSQRT_S, MASK_FSQRT_S) +DECLARE_INSN(fsr, MATCH_FSR, MASK_FSR) +DECLARE_INSN(fsri, MATCH_FSRI, MASK_FSRI) +DECLARE_INSN(fsriw, MATCH_FSRIW, MASK_FSRIW) +DECLARE_INSN(fsrw, MATCH_FSRW, MASK_FSRW) +DECLARE_INSN(fsub_d, MATCH_FSUB_D, MASK_FSUB_D) +DECLARE_INSN(fsub_h, MATCH_FSUB_H, MASK_FSUB_H) +DECLARE_INSN(fsub_q, MATCH_FSUB_Q, MASK_FSUB_Q) +DECLARE_INSN(fsub_s, MATCH_FSUB_S, MASK_FSUB_S) +DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW) +DECLARE_INSN(gorc, MATCH_GORC, MASK_GORC) +DECLARE_INSN(gorci, MATCH_GORCI, MASK_GORCI) +DECLARE_INSN(gorciw, MATCH_GORCIW, MASK_GORCIW) +DECLARE_INSN(gorcw, MATCH_GORCW, MASK_GORCW) +DECLARE_INSN(grev, MATCH_GREV, MASK_GREV) +DECLARE_INSN(grevi, MATCH_GREVI, MASK_GREVI) +DECLARE_INSN(greviw, MATCH_GREVIW, MASK_GREVIW) +DECLARE_INSN(grevw, MATCH_GREVW, MASK_GREVW) +DECLARE_INSN(hfence_gvma, MATCH_HFENCE_GVMA, MASK_HFENCE_GVMA) +DECLARE_INSN(hfence_vvma, MATCH_HFENCE_VVMA, MASK_HFENCE_VVMA) +DECLARE_INSN(hinval_gvma, MATCH_HINVAL_GVMA, MASK_HINVAL_GVMA) +DECLARE_INSN(hinval_vvma, MATCH_HINVAL_VVMA, MASK_HINVAL_VVMA) +DECLARE_INSN(hlv_b, MATCH_HLV_B, MASK_HLV_B) +DECLARE_INSN(hlv_bu, MATCH_HLV_BU, MASK_HLV_BU) +DECLARE_INSN(hlv_d, MATCH_HLV_D, MASK_HLV_D) +DECLARE_INSN(hlv_h, MATCH_HLV_H, MASK_HLV_H) +DECLARE_INSN(hlv_hu, MATCH_HLV_HU, MASK_HLV_HU) +DECLARE_INSN(hlv_w, MATCH_HLV_W, MASK_HLV_W) +DECLARE_INSN(hlv_wu, MATCH_HLV_WU, MASK_HLV_WU) +DECLARE_INSN(hlvx_hu, MATCH_HLVX_HU, MASK_HLVX_HU) +DECLARE_INSN(hlvx_wu, MATCH_HLVX_WU, MASK_HLVX_WU) +DECLARE_INSN(hsv_b, MATCH_HSV_B, MASK_HSV_B) +DECLARE_INSN(hsv_d, MATCH_HSV_D, MASK_HSV_D) +DECLARE_INSN(hsv_h, MATCH_HSV_H, MASK_HSV_H) +DECLARE_INSN(hsv_w, MATCH_HSV_W, MASK_HSV_W) +DECLARE_INSN(insb, MATCH_INSB, MASK_INSB) +DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) +DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) +DECLARE_INSN(kabs16, MATCH_KABS16, MASK_KABS16) +DECLARE_INSN(kabs32, MATCH_KABS32, MASK_KABS32) +DECLARE_INSN(kabs8, MATCH_KABS8, MASK_KABS8) +DECLARE_INSN(kabsw, MATCH_KABSW, MASK_KABSW) +DECLARE_INSN(kadd16, MATCH_KADD16, MASK_KADD16) +DECLARE_INSN(kadd32, MATCH_KADD32, MASK_KADD32) +DECLARE_INSN(kadd64, MATCH_KADD64, MASK_KADD64) +DECLARE_INSN(kadd8, MATCH_KADD8, MASK_KADD8) +DECLARE_INSN(kaddh, MATCH_KADDH, MASK_KADDH) +DECLARE_INSN(kaddw, MATCH_KADDW, MASK_KADDW) +DECLARE_INSN(kcras16, MATCH_KCRAS16, MASK_KCRAS16) +DECLARE_INSN(kcras32, MATCH_KCRAS32, MASK_KCRAS32) +DECLARE_INSN(kcrsa16, MATCH_KCRSA16, MASK_KCRSA16) +DECLARE_INSN(kcrsa32, MATCH_KCRSA32, MASK_KCRSA32) +DECLARE_INSN(kdmabb, MATCH_KDMABB, MASK_KDMABB) +DECLARE_INSN(kdmabb16, MATCH_KDMABB16, MASK_KDMABB16) +DECLARE_INSN(kdmabt, MATCH_KDMABT, MASK_KDMABT) +DECLARE_INSN(kdmabt16, MATCH_KDMABT16, MASK_KDMABT16) +DECLARE_INSN(kdmatt, MATCH_KDMATT, MASK_KDMATT) +DECLARE_INSN(kdmatt16, MATCH_KDMATT16, MASK_KDMATT16) +DECLARE_INSN(kdmbb, MATCH_KDMBB, MASK_KDMBB) +DECLARE_INSN(kdmbb16, MATCH_KDMBB16, MASK_KDMBB16) +DECLARE_INSN(kdmbt, MATCH_KDMBT, MASK_KDMBT) +DECLARE_INSN(kdmbt16, MATCH_KDMBT16, MASK_KDMBT16) +DECLARE_INSN(kdmtt, MATCH_KDMTT, MASK_KDMTT) +DECLARE_INSN(kdmtt16, MATCH_KDMTT16, MASK_KDMTT16) +DECLARE_INSN(khm16, MATCH_KHM16, MASK_KHM16) +DECLARE_INSN(khm8, MATCH_KHM8, MASK_KHM8) +DECLARE_INSN(khmbb, MATCH_KHMBB, MASK_KHMBB) +DECLARE_INSN(khmbb16, MATCH_KHMBB16, MASK_KHMBB16) +DECLARE_INSN(khmbt, MATCH_KHMBT, MASK_KHMBT) +DECLARE_INSN(khmbt16, MATCH_KHMBT16, MASK_KHMBT16) +DECLARE_INSN(khmtt, MATCH_KHMTT, MASK_KHMTT) +DECLARE_INSN(khmtt16, MATCH_KHMTT16, MASK_KHMTT16) +DECLARE_INSN(khmx16, MATCH_KHMX16, MASK_KHMX16) +DECLARE_INSN(khmx8, MATCH_KHMX8, MASK_KHMX8) +DECLARE_INSN(kmabb, MATCH_KMABB, MASK_KMABB) +DECLARE_INSN(kmabb32, MATCH_KMABB32, MASK_KMABB32) +DECLARE_INSN(kmabt, MATCH_KMABT, MASK_KMABT) +DECLARE_INSN(kmabt32, MATCH_KMABT32, MASK_KMABT32) +DECLARE_INSN(kmada, MATCH_KMADA, MASK_KMADA) +DECLARE_INSN(kmadrs, MATCH_KMADRS, MASK_KMADRS) +DECLARE_INSN(kmadrs32, MATCH_KMADRS32, MASK_KMADRS32) +DECLARE_INSN(kmads, MATCH_KMADS, MASK_KMADS) +DECLARE_INSN(kmads32, MATCH_KMADS32, MASK_KMADS32) +DECLARE_INSN(kmar64, MATCH_KMAR64, MASK_KMAR64) +DECLARE_INSN(kmatt, MATCH_KMATT, MASK_KMATT) +DECLARE_INSN(kmatt32, MATCH_KMATT32, MASK_KMATT32) +DECLARE_INSN(kmaxda, MATCH_KMAXDA, MASK_KMAXDA) +DECLARE_INSN(kmaxda32, MATCH_KMAXDA32, MASK_KMAXDA32) +DECLARE_INSN(kmaxds, MATCH_KMAXDS, MASK_KMAXDS) +DECLARE_INSN(kmaxds32, MATCH_KMAXDS32, MASK_KMAXDS32) +DECLARE_INSN(kmda, MATCH_KMDA, MASK_KMDA) +DECLARE_INSN(kmda32, MATCH_KMDA32, MASK_KMDA32) +DECLARE_INSN(kmmac, MATCH_KMMAC, MASK_KMMAC) +DECLARE_INSN(kmmac_u, MATCH_KMMAC_U, MASK_KMMAC_U) +DECLARE_INSN(kmmawb, MATCH_KMMAWB, MASK_KMMAWB) +DECLARE_INSN(kmmawb2, MATCH_KMMAWB2, MASK_KMMAWB2) +DECLARE_INSN(kmmawb2_u, MATCH_KMMAWB2_U, MASK_KMMAWB2_U) +DECLARE_INSN(kmmawb_u, MATCH_KMMAWB_U, MASK_KMMAWB_U) +DECLARE_INSN(kmmawt, MATCH_KMMAWT, MASK_KMMAWT) +DECLARE_INSN(kmmawt2, MATCH_KMMAWT2, MASK_KMMAWT2) +DECLARE_INSN(kmmawt2_u, MATCH_KMMAWT2_U, MASK_KMMAWT2_U) +DECLARE_INSN(kmmawt_u, MATCH_KMMAWT_U, MASK_KMMAWT_U) +DECLARE_INSN(kmmsb, MATCH_KMMSB, MASK_KMMSB) +DECLARE_INSN(kmmsb_u, MATCH_KMMSB_U, MASK_KMMSB_U) +DECLARE_INSN(kmmwb2, MATCH_KMMWB2, MASK_KMMWB2) +DECLARE_INSN(kmmwb2_u, MATCH_KMMWB2_U, MASK_KMMWB2_U) +DECLARE_INSN(kmmwt2, MATCH_KMMWT2, MASK_KMMWT2) +DECLARE_INSN(kmmwt2_u, MATCH_KMMWT2_U, MASK_KMMWT2_U) +DECLARE_INSN(kmsda, MATCH_KMSDA, MASK_KMSDA) +DECLARE_INSN(kmsda32, MATCH_KMSDA32, MASK_KMSDA32) +DECLARE_INSN(kmsr64, MATCH_KMSR64, MASK_KMSR64) +DECLARE_INSN(kmsxda, MATCH_KMSXDA, MASK_KMSXDA) +DECLARE_INSN(kmsxda32, MATCH_KMSXDA32, MASK_KMSXDA32) +DECLARE_INSN(kmxda, MATCH_KMXDA, MASK_KMXDA) +DECLARE_INSN(kmxda32, MATCH_KMXDA32, MASK_KMXDA32) +DECLARE_INSN(ksll16, MATCH_KSLL16, MASK_KSLL16) +DECLARE_INSN(ksll32, MATCH_KSLL32, MASK_KSLL32) +DECLARE_INSN(ksll8, MATCH_KSLL8, MASK_KSLL8) +DECLARE_INSN(kslli16, MATCH_KSLLI16, MASK_KSLLI16) +DECLARE_INSN(kslli32, MATCH_KSLLI32, MASK_KSLLI32) +DECLARE_INSN(kslli8, MATCH_KSLLI8, MASK_KSLLI8) +DECLARE_INSN(kslliw, MATCH_KSLLIW, MASK_KSLLIW) +DECLARE_INSN(ksllw, MATCH_KSLLW, MASK_KSLLW) +DECLARE_INSN(kslra16, MATCH_KSLRA16, MASK_KSLRA16) +DECLARE_INSN(kslra16_u, MATCH_KSLRA16_U, MASK_KSLRA16_U) +DECLARE_INSN(kslra32, MATCH_KSLRA32, MASK_KSLRA32) +DECLARE_INSN(kslra32_u, MATCH_KSLRA32_U, MASK_KSLRA32_U) +DECLARE_INSN(kslra8, MATCH_KSLRA8, MASK_KSLRA8) +DECLARE_INSN(kslra8_u, MATCH_KSLRA8_U, MASK_KSLRA8_U) +DECLARE_INSN(kslraw, MATCH_KSLRAW, MASK_KSLRAW) +DECLARE_INSN(kslraw_u, MATCH_KSLRAW_U, MASK_KSLRAW_U) +DECLARE_INSN(kstas16, MATCH_KSTAS16, MASK_KSTAS16) +DECLARE_INSN(kstas32, MATCH_KSTAS32, MASK_KSTAS32) +DECLARE_INSN(kstsa16, MATCH_KSTSA16, MASK_KSTSA16) +DECLARE_INSN(kstsa32, MATCH_KSTSA32, MASK_KSTSA32) +DECLARE_INSN(ksub16, MATCH_KSUB16, MASK_KSUB16) +DECLARE_INSN(ksub32, MATCH_KSUB32, MASK_KSUB32) +DECLARE_INSN(ksub64, MATCH_KSUB64, MASK_KSUB64) +DECLARE_INSN(ksub8, MATCH_KSUB8, MASK_KSUB8) +DECLARE_INSN(ksubh, MATCH_KSUBH, MASK_KSUBH) +DECLARE_INSN(ksubw, MATCH_KSUBW, MASK_KSUBW) +DECLARE_INSN(kwmmul, MATCH_KWMMUL, MASK_KWMMUL) +DECLARE_INSN(kwmmul_u, MATCH_KWMMUL_U, MASK_KWMMUL_U) +DECLARE_INSN(lb, MATCH_LB, MASK_LB) +DECLARE_INSN(lbu, MATCH_LBU, MASK_LBU) +DECLARE_INSN(ld, MATCH_LD, MASK_LD) +DECLARE_INSN(ldu, MATCH_LDU, MASK_LDU) +DECLARE_INSN(lh, MATCH_LH, MASK_LH) +DECLARE_INSN(lhu, MATCH_LHU, MASK_LHU) +DECLARE_INSN(lq, MATCH_LQ, MASK_LQ) +DECLARE_INSN(lr_d, MATCH_LR_D, MASK_LR_D) +DECLARE_INSN(lr_w, MATCH_LR_W, MASK_LR_W) +DECLARE_INSN(lui, MATCH_LUI, MASK_LUI) +DECLARE_INSN(lw, MATCH_LW, MASK_LW) +DECLARE_INSN(lwu, MATCH_LWU, MASK_LWU) +DECLARE_INSN(maddr32, MATCH_MADDR32, MASK_MADDR32) +DECLARE_INSN(max, MATCH_MAX, MASK_MAX) +DECLARE_INSN(maxu, MATCH_MAXU, MASK_MAXU) +DECLARE_INSN(maxw, MATCH_MAXW, MASK_MAXW) +DECLARE_INSN(min, MATCH_MIN, MASK_MIN) +DECLARE_INSN(minu, MATCH_MINU, MASK_MINU) +DECLARE_INSN(minw, MATCH_MINW, MASK_MINW) +DECLARE_INSN(mret, MATCH_MRET, MASK_MRET) +DECLARE_INSN(msubr32, MATCH_MSUBR32, MASK_MSUBR32) +DECLARE_INSN(mul, MATCH_MUL, MASK_MUL) +DECLARE_INSN(mulh, MATCH_MULH, MASK_MULH) +DECLARE_INSN(mulhsu, MATCH_MULHSU, MASK_MULHSU) +DECLARE_INSN(mulhu, MATCH_MULHU, MASK_MULHU) +DECLARE_INSN(mulr64, MATCH_MULR64, MASK_MULR64) +DECLARE_INSN(mulsr64, MATCH_MULSR64, MASK_MULSR64) +DECLARE_INSN(mulw, MATCH_MULW, MASK_MULW) +DECLARE_INSN(or, MATCH_OR, MASK_OR) +DECLARE_INSN(ori, MATCH_ORI, MASK_ORI) +DECLARE_INSN(orn, MATCH_ORN, MASK_ORN) +DECLARE_INSN(pack, MATCH_PACK, MASK_PACK) +DECLARE_INSN(packh, MATCH_PACKH, MASK_PACKH) +DECLARE_INSN(packu, MATCH_PACKU, MASK_PACKU) +DECLARE_INSN(packuw, MATCH_PACKUW, MASK_PACKUW) +DECLARE_INSN(packw, MATCH_PACKW, MASK_PACKW) +DECLARE_INSN(pause, MATCH_PAUSE, MASK_PAUSE) +DECLARE_INSN(pbsad, MATCH_PBSAD, MASK_PBSAD) +DECLARE_INSN(pbsada, MATCH_PBSADA, MASK_PBSADA) +DECLARE_INSN(pkbb16, MATCH_PKBB16, MASK_PKBB16) +DECLARE_INSN(pkbb32, MATCH_PKBB32, MASK_PKBB32) +DECLARE_INSN(pkbt16, MATCH_PKBT16, MASK_PKBT16) +DECLARE_INSN(pkbt32, MATCH_PKBT32, MASK_PKBT32) +DECLARE_INSN(pktb16, MATCH_PKTB16, MASK_PKTB16) +DECLARE_INSN(pktb32, MATCH_PKTB32, MASK_PKTB32) +DECLARE_INSN(pktt16, MATCH_PKTT16, MASK_PKTT16) +DECLARE_INSN(pktt32, MATCH_PKTT32, MASK_PKTT32) +DECLARE_INSN(prefetch_i, MATCH_PREFETCH_I, MASK_PREFETCH_I) +DECLARE_INSN(prefetch_r, MATCH_PREFETCH_R, MASK_PREFETCH_R) +DECLARE_INSN(prefetch_w, MATCH_PREFETCH_W, MASK_PREFETCH_W) +DECLARE_INSN(radd16, MATCH_RADD16, MASK_RADD16) +DECLARE_INSN(radd32, MATCH_RADD32, MASK_RADD32) +DECLARE_INSN(radd64, MATCH_RADD64, MASK_RADD64) +DECLARE_INSN(radd8, MATCH_RADD8, MASK_RADD8) +DECLARE_INSN(raddw, MATCH_RADDW, MASK_RADDW) +DECLARE_INSN(rcras16, MATCH_RCRAS16, MASK_RCRAS16) +DECLARE_INSN(rcras32, MATCH_RCRAS32, MASK_RCRAS32) +DECLARE_INSN(rcrsa16, MATCH_RCRSA16, MASK_RCRSA16) +DECLARE_INSN(rcrsa32, MATCH_RCRSA32, MASK_RCRSA32) +DECLARE_INSN(rem, MATCH_REM, MASK_REM) +DECLARE_INSN(remu, MATCH_REMU, MASK_REMU) +DECLARE_INSN(remuw, MATCH_REMUW, MASK_REMUW) +DECLARE_INSN(remw, MATCH_REMW, MASK_REMW) +DECLARE_INSN(rol, MATCH_ROL, MASK_ROL) +DECLARE_INSN(rolw, MATCH_ROLW, MASK_ROLW) +DECLARE_INSN(ror, MATCH_ROR, MASK_ROR) +DECLARE_INSN(rori, MATCH_RORI, MASK_RORI) +DECLARE_INSN(roriw, MATCH_RORIW, MASK_RORIW) +DECLARE_INSN(rorw, MATCH_RORW, MASK_RORW) +DECLARE_INSN(rstas16, MATCH_RSTAS16, MASK_RSTAS16) +DECLARE_INSN(rstas32, MATCH_RSTAS32, MASK_RSTAS32) +DECLARE_INSN(rstsa16, MATCH_RSTSA16, MASK_RSTSA16) +DECLARE_INSN(rstsa32, MATCH_RSTSA32, MASK_RSTSA32) +DECLARE_INSN(rsub16, MATCH_RSUB16, MASK_RSUB16) +DECLARE_INSN(rsub32, MATCH_RSUB32, MASK_RSUB32) +DECLARE_INSN(rsub64, MATCH_RSUB64, MASK_RSUB64) +DECLARE_INSN(rsub8, MATCH_RSUB8, MASK_RSUB8) +DECLARE_INSN(rsubw, MATCH_RSUBW, MASK_RSUBW) +DECLARE_INSN(sb, MATCH_SB, MASK_SB) +DECLARE_INSN(sc_d, MATCH_SC_D, MASK_SC_D) +DECLARE_INSN(sc_w, MATCH_SC_W, MASK_SC_W) +DECLARE_INSN(sclip16, MATCH_SCLIP16, MASK_SCLIP16) +DECLARE_INSN(sclip32, MATCH_SCLIP32, MASK_SCLIP32) +DECLARE_INSN(sclip8, MATCH_SCLIP8, MASK_SCLIP8) +DECLARE_INSN(scmple16, MATCH_SCMPLE16, MASK_SCMPLE16) +DECLARE_INSN(scmple8, MATCH_SCMPLE8, MASK_SCMPLE8) +DECLARE_INSN(scmplt16, MATCH_SCMPLT16, MASK_SCMPLT16) +DECLARE_INSN(scmplt8, MATCH_SCMPLT8, MASK_SCMPLT8) +DECLARE_INSN(sd, MATCH_SD, MASK_SD) +DECLARE_INSN(sext_b, MATCH_SEXT_B, MASK_SEXT_B) +DECLARE_INSN(sext_h, MATCH_SEXT_H, MASK_SEXT_H) +DECLARE_INSN(sfence_inval_ir, MATCH_SFENCE_INVAL_IR, MASK_SFENCE_INVAL_IR) +DECLARE_INSN(sfence_vma, MATCH_SFENCE_VMA, MASK_SFENCE_VMA) +DECLARE_INSN(sfence_w_inval, MATCH_SFENCE_W_INVAL, MASK_SFENCE_W_INVAL) +DECLARE_INSN(sh, MATCH_SH, MASK_SH) +DECLARE_INSN(sh1add, MATCH_SH1ADD, MASK_SH1ADD) +DECLARE_INSN(sh1add_uw, MATCH_SH1ADD_UW, MASK_SH1ADD_UW) +DECLARE_INSN(sh2add, MATCH_SH2ADD, MASK_SH2ADD) +DECLARE_INSN(sh2add_uw, MATCH_SH2ADD_UW, MASK_SH2ADD_UW) +DECLARE_INSN(sh3add, MATCH_SH3ADD, MASK_SH3ADD) +DECLARE_INSN(sh3add_uw, MATCH_SH3ADD_UW, MASK_SH3ADD_UW) +DECLARE_INSN(sha256sig0, MATCH_SHA256SIG0, MASK_SHA256SIG0) +DECLARE_INSN(sha256sig1, MATCH_SHA256SIG1, MASK_SHA256SIG1) +DECLARE_INSN(sha256sum0, MATCH_SHA256SUM0, MASK_SHA256SUM0) +DECLARE_INSN(sha256sum1, MATCH_SHA256SUM1, MASK_SHA256SUM1) +DECLARE_INSN(sha512sig0, MATCH_SHA512SIG0, MASK_SHA512SIG0) +DECLARE_INSN(sha512sig0h, MATCH_SHA512SIG0H, MASK_SHA512SIG0H) +DECLARE_INSN(sha512sig0l, MATCH_SHA512SIG0L, MASK_SHA512SIG0L) +DECLARE_INSN(sha512sig1, MATCH_SHA512SIG1, MASK_SHA512SIG1) +DECLARE_INSN(sha512sig1h, MATCH_SHA512SIG1H, MASK_SHA512SIG1H) +DECLARE_INSN(sha512sig1l, MATCH_SHA512SIG1L, MASK_SHA512SIG1L) +DECLARE_INSN(sha512sum0, MATCH_SHA512SUM0, MASK_SHA512SUM0) +DECLARE_INSN(sha512sum0r, MATCH_SHA512SUM0R, MASK_SHA512SUM0R) +DECLARE_INSN(sha512sum1, MATCH_SHA512SUM1, MASK_SHA512SUM1) +DECLARE_INSN(sha512sum1r, MATCH_SHA512SUM1R, MASK_SHA512SUM1R) +DECLARE_INSN(shfl, MATCH_SHFL, MASK_SHFL) +DECLARE_INSN(shfli, MATCH_SHFLI, MASK_SHFLI) +DECLARE_INSN(shflw, MATCH_SHFLW, MASK_SHFLW) +DECLARE_INSN(sinval_vma, MATCH_SINVAL_VMA, MASK_SINVAL_VMA) +DECLARE_INSN(sll, MATCH_SLL, MASK_SLL) +DECLARE_INSN(sll16, MATCH_SLL16, MASK_SLL16) +DECLARE_INSN(sll32, MATCH_SLL32, MASK_SLL32) +DECLARE_INSN(sll8, MATCH_SLL8, MASK_SLL8) +DECLARE_INSN(slld, MATCH_SLLD, MASK_SLLD) +DECLARE_INSN(slli, MATCH_SLLI, MASK_SLLI) +DECLARE_INSN(slli16, MATCH_SLLI16, MASK_SLLI16) +DECLARE_INSN(slli32, MATCH_SLLI32, MASK_SLLI32) +DECLARE_INSN(slli8, MATCH_SLLI8, MASK_SLLI8) +DECLARE_INSN(slli_uw, MATCH_SLLI_UW, MASK_SLLI_UW) +DECLARE_INSN(sllid, MATCH_SLLID, MASK_SLLID) +DECLARE_INSN(slliw, MATCH_SLLIW, MASK_SLLIW) +DECLARE_INSN(sllw, MATCH_SLLW, MASK_SLLW) +DECLARE_INSN(slo, MATCH_SLO, MASK_SLO) +DECLARE_INSN(sloi, MATCH_SLOI, MASK_SLOI) +DECLARE_INSN(sloiw, MATCH_SLOIW, MASK_SLOIW) +DECLARE_INSN(slow, MATCH_SLOW, MASK_SLOW) +DECLARE_INSN(slt, MATCH_SLT, MASK_SLT) +DECLARE_INSN(slti, MATCH_SLTI, MASK_SLTI) +DECLARE_INSN(sltiu, MATCH_SLTIU, MASK_SLTIU) +DECLARE_INSN(sltu, MATCH_SLTU, MASK_SLTU) +DECLARE_INSN(sm3p0, MATCH_SM3P0, MASK_SM3P0) +DECLARE_INSN(sm3p1, MATCH_SM3P1, MASK_SM3P1) +DECLARE_INSN(sm4ed, MATCH_SM4ED, MASK_SM4ED) +DECLARE_INSN(sm4ks, MATCH_SM4KS, MASK_SM4KS) +DECLARE_INSN(smal, MATCH_SMAL, MASK_SMAL) +DECLARE_INSN(smalbb, MATCH_SMALBB, MASK_SMALBB) +DECLARE_INSN(smalbt, MATCH_SMALBT, MASK_SMALBT) +DECLARE_INSN(smalda, MATCH_SMALDA, MASK_SMALDA) +DECLARE_INSN(smaldrs, MATCH_SMALDRS, MASK_SMALDRS) +DECLARE_INSN(smalds, MATCH_SMALDS, MASK_SMALDS) +DECLARE_INSN(smaltt, MATCH_SMALTT, MASK_SMALTT) +DECLARE_INSN(smalxda, MATCH_SMALXDA, MASK_SMALXDA) +DECLARE_INSN(smalxds, MATCH_SMALXDS, MASK_SMALXDS) +DECLARE_INSN(smaqa, MATCH_SMAQA, MASK_SMAQA) +DECLARE_INSN(smaqa_su, MATCH_SMAQA_SU, MASK_SMAQA_SU) +DECLARE_INSN(smar64, MATCH_SMAR64, MASK_SMAR64) +DECLARE_INSN(smax16, MATCH_SMAX16, MASK_SMAX16) +DECLARE_INSN(smax32, MATCH_SMAX32, MASK_SMAX32) +DECLARE_INSN(smax8, MATCH_SMAX8, MASK_SMAX8) +DECLARE_INSN(smbb16, MATCH_SMBB16, MASK_SMBB16) +DECLARE_INSN(smbt16, MATCH_SMBT16, MASK_SMBT16) +DECLARE_INSN(smbt32, MATCH_SMBT32, MASK_SMBT32) +DECLARE_INSN(smdrs, MATCH_SMDRS, MASK_SMDRS) +DECLARE_INSN(smdrs32, MATCH_SMDRS32, MASK_SMDRS32) +DECLARE_INSN(smds, MATCH_SMDS, MASK_SMDS) +DECLARE_INSN(smds32, MATCH_SMDS32, MASK_SMDS32) +DECLARE_INSN(smin16, MATCH_SMIN16, MASK_SMIN16) +DECLARE_INSN(smin32, MATCH_SMIN32, MASK_SMIN32) +DECLARE_INSN(smin8, MATCH_SMIN8, MASK_SMIN8) +DECLARE_INSN(smmul, MATCH_SMMUL, MASK_SMMUL) +DECLARE_INSN(smmul_u, MATCH_SMMUL_U, MASK_SMMUL_U) +DECLARE_INSN(smmwb, MATCH_SMMWB, MASK_SMMWB) +DECLARE_INSN(smmwb_u, MATCH_SMMWB_U, MASK_SMMWB_U) +DECLARE_INSN(smmwt, MATCH_SMMWT, MASK_SMMWT) +DECLARE_INSN(smmwt_u, MATCH_SMMWT_U, MASK_SMMWT_U) +DECLARE_INSN(smslda, MATCH_SMSLDA, MASK_SMSLDA) +DECLARE_INSN(smslxda, MATCH_SMSLXDA, MASK_SMSLXDA) +DECLARE_INSN(smsr64, MATCH_SMSR64, MASK_SMSR64) +DECLARE_INSN(smtt16, MATCH_SMTT16, MASK_SMTT16) +DECLARE_INSN(smtt32, MATCH_SMTT32, MASK_SMTT32) +DECLARE_INSN(smul16, MATCH_SMUL16, MASK_SMUL16) +DECLARE_INSN(smul8, MATCH_SMUL8, MASK_SMUL8) +DECLARE_INSN(smulx16, MATCH_SMULX16, MASK_SMULX16) +DECLARE_INSN(smulx8, MATCH_SMULX8, MASK_SMULX8) +DECLARE_INSN(smxds, MATCH_SMXDS, MASK_SMXDS) +DECLARE_INSN(smxds32, MATCH_SMXDS32, MASK_SMXDS32) +DECLARE_INSN(sq, MATCH_SQ, MASK_SQ) +DECLARE_INSN(sra, MATCH_SRA, MASK_SRA) +DECLARE_INSN(sra16, MATCH_SRA16, MASK_SRA16) +DECLARE_INSN(sra16_u, MATCH_SRA16_U, MASK_SRA16_U) +DECLARE_INSN(sra32, MATCH_SRA32, MASK_SRA32) +DECLARE_INSN(sra32_u, MATCH_SRA32_U, MASK_SRA32_U) +DECLARE_INSN(sra8, MATCH_SRA8, MASK_SRA8) +DECLARE_INSN(sra8_u, MATCH_SRA8_U, MASK_SRA8_U) +DECLARE_INSN(sra_u, MATCH_SRA_U, MASK_SRA_U) +DECLARE_INSN(srad, MATCH_SRAD, MASK_SRAD) +DECLARE_INSN(srai, MATCH_SRAI, MASK_SRAI) +DECLARE_INSN(srai16, MATCH_SRAI16, MASK_SRAI16) +DECLARE_INSN(srai16_u, MATCH_SRAI16_U, MASK_SRAI16_U) +DECLARE_INSN(srai32, MATCH_SRAI32, MASK_SRAI32) +DECLARE_INSN(srai32_u, MATCH_SRAI32_U, MASK_SRAI32_U) +DECLARE_INSN(srai8, MATCH_SRAI8, MASK_SRAI8) +DECLARE_INSN(srai8_u, MATCH_SRAI8_U, MASK_SRAI8_U) +DECLARE_INSN(srai_u, MATCH_SRAI_U, MASK_SRAI_U) +DECLARE_INSN(sraid, MATCH_SRAID, MASK_SRAID) +DECLARE_INSN(sraiw, MATCH_SRAIW, MASK_SRAIW) +DECLARE_INSN(sraiw_u, MATCH_SRAIW_U, MASK_SRAIW_U) +DECLARE_INSN(sraw, MATCH_SRAW, MASK_SRAW) +DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) +DECLARE_INSN(srl, MATCH_SRL, MASK_SRL) +DECLARE_INSN(srl16, MATCH_SRL16, MASK_SRL16) +DECLARE_INSN(srl16_u, MATCH_SRL16_U, MASK_SRL16_U) +DECLARE_INSN(srl32, MATCH_SRL32, MASK_SRL32) +DECLARE_INSN(srl32_u, MATCH_SRL32_U, MASK_SRL32_U) +DECLARE_INSN(srl8, MATCH_SRL8, MASK_SRL8) +DECLARE_INSN(srl8_u, MATCH_SRL8_U, MASK_SRL8_U) +DECLARE_INSN(srld, MATCH_SRLD, MASK_SRLD) +DECLARE_INSN(srli, MATCH_SRLI, MASK_SRLI) +DECLARE_INSN(srli16, MATCH_SRLI16, MASK_SRLI16) +DECLARE_INSN(srli16_u, MATCH_SRLI16_U, MASK_SRLI16_U) +DECLARE_INSN(srli32, MATCH_SRLI32, MASK_SRLI32) +DECLARE_INSN(srli32_u, MATCH_SRLI32_U, MASK_SRLI32_U) +DECLARE_INSN(srli8, MATCH_SRLI8, MASK_SRLI8) +DECLARE_INSN(srli8_u, MATCH_SRLI8_U, MASK_SRLI8_U) +DECLARE_INSN(srlid, MATCH_SRLID, MASK_SRLID) +DECLARE_INSN(srliw, MATCH_SRLIW, MASK_SRLIW) +DECLARE_INSN(srlw, MATCH_SRLW, MASK_SRLW) +DECLARE_INSN(sro, MATCH_SRO, MASK_SRO) +DECLARE_INSN(sroi, MATCH_SROI, MASK_SROI) +DECLARE_INSN(sroiw, MATCH_SROIW, MASK_SROIW) +DECLARE_INSN(srow, MATCH_SROW, MASK_SROW) +DECLARE_INSN(stas16, MATCH_STAS16, MASK_STAS16) +DECLARE_INSN(stas32, MATCH_STAS32, MASK_STAS32) +DECLARE_INSN(stsa16, MATCH_STSA16, MASK_STSA16) +DECLARE_INSN(stsa32, MATCH_STSA32, MASK_STSA32) +DECLARE_INSN(sub, MATCH_SUB, MASK_SUB) +DECLARE_INSN(sub16, MATCH_SUB16, MASK_SUB16) +DECLARE_INSN(sub32, MATCH_SUB32, MASK_SUB32) +DECLARE_INSN(sub64, MATCH_SUB64, MASK_SUB64) +DECLARE_INSN(sub8, MATCH_SUB8, MASK_SUB8) +DECLARE_INSN(subd, MATCH_SUBD, MASK_SUBD) +DECLARE_INSN(subw, MATCH_SUBW, MASK_SUBW) +DECLARE_INSN(sunpkd810, MATCH_SUNPKD810, MASK_SUNPKD810) +DECLARE_INSN(sunpkd820, MATCH_SUNPKD820, MASK_SUNPKD820) +DECLARE_INSN(sunpkd830, MATCH_SUNPKD830, MASK_SUNPKD830) +DECLARE_INSN(sunpkd831, MATCH_SUNPKD831, MASK_SUNPKD831) +DECLARE_INSN(sunpkd832, MATCH_SUNPKD832, MASK_SUNPKD832) +DECLARE_INSN(sw, MATCH_SW, MASK_SW) +DECLARE_INSN(swap8, MATCH_SWAP8, MASK_SWAP8) +DECLARE_INSN(uclip16, MATCH_UCLIP16, MASK_UCLIP16) +DECLARE_INSN(uclip32, MATCH_UCLIP32, MASK_UCLIP32) +DECLARE_INSN(uclip8, MATCH_UCLIP8, MASK_UCLIP8) +DECLARE_INSN(ucmple16, MATCH_UCMPLE16, MASK_UCMPLE16) +DECLARE_INSN(ucmple8, MATCH_UCMPLE8, MASK_UCMPLE8) +DECLARE_INSN(ucmplt16, MATCH_UCMPLT16, MASK_UCMPLT16) +DECLARE_INSN(ucmplt8, MATCH_UCMPLT8, MASK_UCMPLT8) +DECLARE_INSN(ukadd16, MATCH_UKADD16, MASK_UKADD16) +DECLARE_INSN(ukadd32, MATCH_UKADD32, MASK_UKADD32) +DECLARE_INSN(ukadd64, MATCH_UKADD64, MASK_UKADD64) +DECLARE_INSN(ukadd8, MATCH_UKADD8, MASK_UKADD8) +DECLARE_INSN(ukaddh, MATCH_UKADDH, MASK_UKADDH) +DECLARE_INSN(ukaddw, MATCH_UKADDW, MASK_UKADDW) +DECLARE_INSN(ukcras16, MATCH_UKCRAS16, MASK_UKCRAS16) +DECLARE_INSN(ukcras32, MATCH_UKCRAS32, MASK_UKCRAS32) +DECLARE_INSN(ukcrsa16, MATCH_UKCRSA16, MASK_UKCRSA16) +DECLARE_INSN(ukcrsa32, MATCH_UKCRSA32, MASK_UKCRSA32) +DECLARE_INSN(ukmar64, MATCH_UKMAR64, MASK_UKMAR64) +DECLARE_INSN(ukmsr64, MATCH_UKMSR64, MASK_UKMSR64) +DECLARE_INSN(ukstas16, MATCH_UKSTAS16, MASK_UKSTAS16) +DECLARE_INSN(ukstas32, MATCH_UKSTAS32, MASK_UKSTAS32) +DECLARE_INSN(ukstsa16, MATCH_UKSTSA16, MASK_UKSTSA16) +DECLARE_INSN(ukstsa32, MATCH_UKSTSA32, MASK_UKSTSA32) +DECLARE_INSN(uksub16, MATCH_UKSUB16, MASK_UKSUB16) +DECLARE_INSN(uksub32, MATCH_UKSUB32, MASK_UKSUB32) +DECLARE_INSN(uksub64, MATCH_UKSUB64, MASK_UKSUB64) +DECLARE_INSN(uksub8, MATCH_UKSUB8, MASK_UKSUB8) +DECLARE_INSN(uksubh, MATCH_UKSUBH, MASK_UKSUBH) +DECLARE_INSN(uksubw, MATCH_UKSUBW, MASK_UKSUBW) +DECLARE_INSN(umaqa, MATCH_UMAQA, MASK_UMAQA) +DECLARE_INSN(umar64, MATCH_UMAR64, MASK_UMAR64) +DECLARE_INSN(umax16, MATCH_UMAX16, MASK_UMAX16) +DECLARE_INSN(umax32, MATCH_UMAX32, MASK_UMAX32) +DECLARE_INSN(umax8, MATCH_UMAX8, MASK_UMAX8) +DECLARE_INSN(umin16, MATCH_UMIN16, MASK_UMIN16) +DECLARE_INSN(umin32, MATCH_UMIN32, MASK_UMIN32) +DECLARE_INSN(umin8, MATCH_UMIN8, MASK_UMIN8) +DECLARE_INSN(umsr64, MATCH_UMSR64, MASK_UMSR64) +DECLARE_INSN(umul16, MATCH_UMUL16, MASK_UMUL16) +DECLARE_INSN(umul8, MATCH_UMUL8, MASK_UMUL8) +DECLARE_INSN(umulx16, MATCH_UMULX16, MASK_UMULX16) +DECLARE_INSN(umulx8, MATCH_UMULX8, MASK_UMULX8) +DECLARE_INSN(unshfl, MATCH_UNSHFL, MASK_UNSHFL) +DECLARE_INSN(unshfli, MATCH_UNSHFLI, MASK_UNSHFLI) +DECLARE_INSN(unshflw, MATCH_UNSHFLW, MASK_UNSHFLW) +DECLARE_INSN(uradd16, MATCH_URADD16, MASK_URADD16) +DECLARE_INSN(uradd32, MATCH_URADD32, MASK_URADD32) +DECLARE_INSN(uradd64, MATCH_URADD64, MASK_URADD64) +DECLARE_INSN(uradd8, MATCH_URADD8, MASK_URADD8) +DECLARE_INSN(uraddw, MATCH_URADDW, MASK_URADDW) +DECLARE_INSN(urcras16, MATCH_URCRAS16, MASK_URCRAS16) +DECLARE_INSN(urcras32, MATCH_URCRAS32, MASK_URCRAS32) +DECLARE_INSN(urcrsa16, MATCH_URCRSA16, MASK_URCRSA16) +DECLARE_INSN(urcrsa32, MATCH_URCRSA32, MASK_URCRSA32) +DECLARE_INSN(urstas16, MATCH_URSTAS16, MASK_URSTAS16) +DECLARE_INSN(urstas32, MATCH_URSTAS32, MASK_URSTAS32) +DECLARE_INSN(urstsa16, MATCH_URSTSA16, MASK_URSTSA16) +DECLARE_INSN(urstsa32, MATCH_URSTSA32, MASK_URSTSA32) +DECLARE_INSN(ursub16, MATCH_URSUB16, MASK_URSUB16) +DECLARE_INSN(ursub32, MATCH_URSUB32, MASK_URSUB32) +DECLARE_INSN(ursub64, MATCH_URSUB64, MASK_URSUB64) +DECLARE_INSN(ursub8, MATCH_URSUB8, MASK_URSUB8) +DECLARE_INSN(ursubw, MATCH_URSUBW, MASK_URSUBW) +DECLARE_INSN(vaadd_vv, MATCH_VAADD_VV, MASK_VAADD_VV) +DECLARE_INSN(vaadd_vx, MATCH_VAADD_VX, MASK_VAADD_VX) +DECLARE_INSN(vaaddu_vv, MATCH_VAADDU_VV, MASK_VAADDU_VV) +DECLARE_INSN(vaaddu_vx, MATCH_VAADDU_VX, MASK_VAADDU_VX) +DECLARE_INSN(vadc_vim, MATCH_VADC_VIM, MASK_VADC_VIM) +DECLARE_INSN(vadc_vvm, MATCH_VADC_VVM, MASK_VADC_VVM) +DECLARE_INSN(vadc_vxm, MATCH_VADC_VXM, MASK_VADC_VXM) +DECLARE_INSN(vadd_vi, MATCH_VADD_VI, MASK_VADD_VI) +DECLARE_INSN(vadd_vv, MATCH_VADD_VV, MASK_VADD_VV) +DECLARE_INSN(vadd_vx, MATCH_VADD_VX, MASK_VADD_VX) +DECLARE_INSN(vamoaddei16_v, MATCH_VAMOADDEI16_V, MASK_VAMOADDEI16_V) +DECLARE_INSN(vamoaddei32_v, MATCH_VAMOADDEI32_V, MASK_VAMOADDEI32_V) +DECLARE_INSN(vamoaddei64_v, MATCH_VAMOADDEI64_V, MASK_VAMOADDEI64_V) +DECLARE_INSN(vamoaddei8_v, MATCH_VAMOADDEI8_V, MASK_VAMOADDEI8_V) +DECLARE_INSN(vamoandei16_v, MATCH_VAMOANDEI16_V, MASK_VAMOANDEI16_V) +DECLARE_INSN(vamoandei32_v, MATCH_VAMOANDEI32_V, MASK_VAMOANDEI32_V) +DECLARE_INSN(vamoandei64_v, MATCH_VAMOANDEI64_V, MASK_VAMOANDEI64_V) +DECLARE_INSN(vamoandei8_v, MATCH_VAMOANDEI8_V, MASK_VAMOANDEI8_V) +DECLARE_INSN(vamomaxei16_v, MATCH_VAMOMAXEI16_V, MASK_VAMOMAXEI16_V) +DECLARE_INSN(vamomaxei32_v, MATCH_VAMOMAXEI32_V, MASK_VAMOMAXEI32_V) +DECLARE_INSN(vamomaxei64_v, MATCH_VAMOMAXEI64_V, MASK_VAMOMAXEI64_V) +DECLARE_INSN(vamomaxei8_v, MATCH_VAMOMAXEI8_V, MASK_VAMOMAXEI8_V) +DECLARE_INSN(vamomaxuei16_v, MATCH_VAMOMAXUEI16_V, MASK_VAMOMAXUEI16_V) +DECLARE_INSN(vamomaxuei32_v, MATCH_VAMOMAXUEI32_V, MASK_VAMOMAXUEI32_V) +DECLARE_INSN(vamomaxuei64_v, MATCH_VAMOMAXUEI64_V, MASK_VAMOMAXUEI64_V) +DECLARE_INSN(vamomaxuei8_v, MATCH_VAMOMAXUEI8_V, MASK_VAMOMAXUEI8_V) +DECLARE_INSN(vamominei16_v, MATCH_VAMOMINEI16_V, MASK_VAMOMINEI16_V) +DECLARE_INSN(vamominei32_v, MATCH_VAMOMINEI32_V, MASK_VAMOMINEI32_V) +DECLARE_INSN(vamominei64_v, MATCH_VAMOMINEI64_V, MASK_VAMOMINEI64_V) +DECLARE_INSN(vamominei8_v, MATCH_VAMOMINEI8_V, MASK_VAMOMINEI8_V) +DECLARE_INSN(vamominuei16_v, MATCH_VAMOMINUEI16_V, MASK_VAMOMINUEI16_V) +DECLARE_INSN(vamominuei32_v, MATCH_VAMOMINUEI32_V, MASK_VAMOMINUEI32_V) +DECLARE_INSN(vamominuei64_v, MATCH_VAMOMINUEI64_V, MASK_VAMOMINUEI64_V) +DECLARE_INSN(vamominuei8_v, MATCH_VAMOMINUEI8_V, MASK_VAMOMINUEI8_V) +DECLARE_INSN(vamoorei16_v, MATCH_VAMOOREI16_V, MASK_VAMOOREI16_V) +DECLARE_INSN(vamoorei32_v, MATCH_VAMOOREI32_V, MASK_VAMOOREI32_V) +DECLARE_INSN(vamoorei64_v, MATCH_VAMOOREI64_V, MASK_VAMOOREI64_V) +DECLARE_INSN(vamoorei8_v, MATCH_VAMOOREI8_V, MASK_VAMOOREI8_V) +DECLARE_INSN(vamoswapei16_v, MATCH_VAMOSWAPEI16_V, MASK_VAMOSWAPEI16_V) +DECLARE_INSN(vamoswapei32_v, MATCH_VAMOSWAPEI32_V, MASK_VAMOSWAPEI32_V) +DECLARE_INSN(vamoswapei64_v, MATCH_VAMOSWAPEI64_V, MASK_VAMOSWAPEI64_V) +DECLARE_INSN(vamoswapei8_v, MATCH_VAMOSWAPEI8_V, MASK_VAMOSWAPEI8_V) +DECLARE_INSN(vamoxorei16_v, MATCH_VAMOXOREI16_V, MASK_VAMOXOREI16_V) +DECLARE_INSN(vamoxorei32_v, MATCH_VAMOXOREI32_V, MASK_VAMOXOREI32_V) +DECLARE_INSN(vamoxorei64_v, MATCH_VAMOXOREI64_V, MASK_VAMOXOREI64_V) +DECLARE_INSN(vamoxorei8_v, MATCH_VAMOXOREI8_V, MASK_VAMOXOREI8_V) +DECLARE_INSN(vand_vi, MATCH_VAND_VI, MASK_VAND_VI) +DECLARE_INSN(vand_vv, MATCH_VAND_VV, MASK_VAND_VV) +DECLARE_INSN(vand_vx, MATCH_VAND_VX, MASK_VAND_VX) +DECLARE_INSN(vasub_vv, MATCH_VASUB_VV, MASK_VASUB_VV) +DECLARE_INSN(vasub_vx, MATCH_VASUB_VX, MASK_VASUB_VX) +DECLARE_INSN(vasubu_vv, MATCH_VASUBU_VV, MASK_VASUBU_VV) +DECLARE_INSN(vasubu_vx, MATCH_VASUBU_VX, MASK_VASUBU_VX) +DECLARE_INSN(vcompress_vm, MATCH_VCOMPRESS_VM, MASK_VCOMPRESS_VM) +DECLARE_INSN(vcpop_m, MATCH_VCPOP_M, MASK_VCPOP_M) +DECLARE_INSN(vdiv_vv, MATCH_VDIV_VV, MASK_VDIV_VV) +DECLARE_INSN(vdiv_vx, MATCH_VDIV_VX, MASK_VDIV_VX) +DECLARE_INSN(vdivu_vv, MATCH_VDIVU_VV, MASK_VDIVU_VV) +DECLARE_INSN(vdivu_vx, MATCH_VDIVU_VX, MASK_VDIVU_VX) +DECLARE_INSN(vfadd_vf, MATCH_VFADD_VF, MASK_VFADD_VF) +DECLARE_INSN(vfadd_vv, MATCH_VFADD_VV, MASK_VFADD_VV) +DECLARE_INSN(vfclass_v, MATCH_VFCLASS_V, MASK_VFCLASS_V) +DECLARE_INSN(vfcvt_f_x_v, MATCH_VFCVT_F_X_V, MASK_VFCVT_F_X_V) +DECLARE_INSN(vfcvt_f_xu_v, MATCH_VFCVT_F_XU_V, MASK_VFCVT_F_XU_V) +DECLARE_INSN(vfcvt_rtz_x_f_v, MATCH_VFCVT_RTZ_X_F_V, MASK_VFCVT_RTZ_X_F_V) +DECLARE_INSN(vfcvt_rtz_xu_f_v, MATCH_VFCVT_RTZ_XU_F_V, MASK_VFCVT_RTZ_XU_F_V) +DECLARE_INSN(vfcvt_x_f_v, MATCH_VFCVT_X_F_V, MASK_VFCVT_X_F_V) +DECLARE_INSN(vfcvt_xu_f_v, MATCH_VFCVT_XU_F_V, MASK_VFCVT_XU_F_V) +DECLARE_INSN(vfdiv_vf, MATCH_VFDIV_VF, MASK_VFDIV_VF) +DECLARE_INSN(vfdiv_vv, MATCH_VFDIV_VV, MASK_VFDIV_VV) +DECLARE_INSN(vfirst_m, MATCH_VFIRST_M, MASK_VFIRST_M) +DECLARE_INSN(vfmacc_vf, MATCH_VFMACC_VF, MASK_VFMACC_VF) +DECLARE_INSN(vfmacc_vv, MATCH_VFMACC_VV, MASK_VFMACC_VV) +DECLARE_INSN(vfmadd_vf, MATCH_VFMADD_VF, MASK_VFMADD_VF) +DECLARE_INSN(vfmadd_vv, MATCH_VFMADD_VV, MASK_VFMADD_VV) +DECLARE_INSN(vfmax_vf, MATCH_VFMAX_VF, MASK_VFMAX_VF) +DECLARE_INSN(vfmax_vv, MATCH_VFMAX_VV, MASK_VFMAX_VV) +DECLARE_INSN(vfmerge_vfm, MATCH_VFMERGE_VFM, MASK_VFMERGE_VFM) +DECLARE_INSN(vfmin_vf, MATCH_VFMIN_VF, MASK_VFMIN_VF) +DECLARE_INSN(vfmin_vv, MATCH_VFMIN_VV, MASK_VFMIN_VV) +DECLARE_INSN(vfmsac_vf, MATCH_VFMSAC_VF, MASK_VFMSAC_VF) +DECLARE_INSN(vfmsac_vv, MATCH_VFMSAC_VV, MASK_VFMSAC_VV) +DECLARE_INSN(vfmsub_vf, MATCH_VFMSUB_VF, MASK_VFMSUB_VF) +DECLARE_INSN(vfmsub_vv, MATCH_VFMSUB_VV, MASK_VFMSUB_VV) +DECLARE_INSN(vfmul_vf, MATCH_VFMUL_VF, MASK_VFMUL_VF) +DECLARE_INSN(vfmul_vv, MATCH_VFMUL_VV, MASK_VFMUL_VV) +DECLARE_INSN(vfmv_f_s, MATCH_VFMV_F_S, MASK_VFMV_F_S) +DECLARE_INSN(vfmv_s_f, MATCH_VFMV_S_F, MASK_VFMV_S_F) +DECLARE_INSN(vfmv_v_f, MATCH_VFMV_V_F, MASK_VFMV_V_F) +DECLARE_INSN(vfncvt_f_f_w, MATCH_VFNCVT_F_F_W, MASK_VFNCVT_F_F_W) +DECLARE_INSN(vfncvt_f_x_w, MATCH_VFNCVT_F_X_W, MASK_VFNCVT_F_X_W) +DECLARE_INSN(vfncvt_f_xu_w, MATCH_VFNCVT_F_XU_W, MASK_VFNCVT_F_XU_W) +DECLARE_INSN(vfncvt_rod_f_f_w, MATCH_VFNCVT_ROD_F_F_W, MASK_VFNCVT_ROD_F_F_W) +DECLARE_INSN(vfncvt_rtz_x_f_w, MATCH_VFNCVT_RTZ_X_F_W, MASK_VFNCVT_RTZ_X_F_W) +DECLARE_INSN(vfncvt_rtz_xu_f_w, MATCH_VFNCVT_RTZ_XU_F_W, MASK_VFNCVT_RTZ_XU_F_W) +DECLARE_INSN(vfncvt_x_f_w, MATCH_VFNCVT_X_F_W, MASK_VFNCVT_X_F_W) +DECLARE_INSN(vfncvt_xu_f_w, MATCH_VFNCVT_XU_F_W, MASK_VFNCVT_XU_F_W) +DECLARE_INSN(vfnmacc_vf, MATCH_VFNMACC_VF, MASK_VFNMACC_VF) +DECLARE_INSN(vfnmacc_vv, MATCH_VFNMACC_VV, MASK_VFNMACC_VV) +DECLARE_INSN(vfnmadd_vf, MATCH_VFNMADD_VF, MASK_VFNMADD_VF) +DECLARE_INSN(vfnmadd_vv, MATCH_VFNMADD_VV, MASK_VFNMADD_VV) +DECLARE_INSN(vfnmsac_vf, MATCH_VFNMSAC_VF, MASK_VFNMSAC_VF) +DECLARE_INSN(vfnmsac_vv, MATCH_VFNMSAC_VV, MASK_VFNMSAC_VV) +DECLARE_INSN(vfnmsub_vf, MATCH_VFNMSUB_VF, MASK_VFNMSUB_VF) +DECLARE_INSN(vfnmsub_vv, MATCH_VFNMSUB_VV, MASK_VFNMSUB_VV) +DECLARE_INSN(vfrdiv_vf, MATCH_VFRDIV_VF, MASK_VFRDIV_VF) +DECLARE_INSN(vfrec7_v, MATCH_VFREC7_V, MASK_VFREC7_V) +DECLARE_INSN(vfredmax_vs, MATCH_VFREDMAX_VS, MASK_VFREDMAX_VS) +DECLARE_INSN(vfredmin_vs, MATCH_VFREDMIN_VS, MASK_VFREDMIN_VS) +DECLARE_INSN(vfredosum_vs, MATCH_VFREDOSUM_VS, MASK_VFREDOSUM_VS) +DECLARE_INSN(vfredusum_vs, MATCH_VFREDUSUM_VS, MASK_VFREDUSUM_VS) +DECLARE_INSN(vfrsqrt7_v, MATCH_VFRSQRT7_V, MASK_VFRSQRT7_V) +DECLARE_INSN(vfrsub_vf, MATCH_VFRSUB_VF, MASK_VFRSUB_VF) +DECLARE_INSN(vfsgnj_vf, MATCH_VFSGNJ_VF, MASK_VFSGNJ_VF) +DECLARE_INSN(vfsgnj_vv, MATCH_VFSGNJ_VV, MASK_VFSGNJ_VV) +DECLARE_INSN(vfsgnjn_vf, MATCH_VFSGNJN_VF, MASK_VFSGNJN_VF) +DECLARE_INSN(vfsgnjn_vv, MATCH_VFSGNJN_VV, MASK_VFSGNJN_VV) +DECLARE_INSN(vfsgnjx_vf, MATCH_VFSGNJX_VF, MASK_VFSGNJX_VF) +DECLARE_INSN(vfsgnjx_vv, MATCH_VFSGNJX_VV, MASK_VFSGNJX_VV) +DECLARE_INSN(vfslide1down_vf, MATCH_VFSLIDE1DOWN_VF, MASK_VFSLIDE1DOWN_VF) +DECLARE_INSN(vfslide1up_vf, MATCH_VFSLIDE1UP_VF, MASK_VFSLIDE1UP_VF) +DECLARE_INSN(vfsqrt_v, MATCH_VFSQRT_V, MASK_VFSQRT_V) +DECLARE_INSN(vfsub_vf, MATCH_VFSUB_VF, MASK_VFSUB_VF) +DECLARE_INSN(vfsub_vv, MATCH_VFSUB_VV, MASK_VFSUB_VV) +DECLARE_INSN(vfwadd_vf, MATCH_VFWADD_VF, MASK_VFWADD_VF) +DECLARE_INSN(vfwadd_vv, MATCH_VFWADD_VV, MASK_VFWADD_VV) +DECLARE_INSN(vfwadd_wf, MATCH_VFWADD_WF, MASK_VFWADD_WF) +DECLARE_INSN(vfwadd_wv, MATCH_VFWADD_WV, MASK_VFWADD_WV) +DECLARE_INSN(vfwcvt_f_f_v, MATCH_VFWCVT_F_F_V, MASK_VFWCVT_F_F_V) +DECLARE_INSN(vfwcvt_f_x_v, MATCH_VFWCVT_F_X_V, MASK_VFWCVT_F_X_V) +DECLARE_INSN(vfwcvt_f_xu_v, MATCH_VFWCVT_F_XU_V, MASK_VFWCVT_F_XU_V) +DECLARE_INSN(vfwcvt_rtz_x_f_v, MATCH_VFWCVT_RTZ_X_F_V, MASK_VFWCVT_RTZ_X_F_V) +DECLARE_INSN(vfwcvt_rtz_xu_f_v, MATCH_VFWCVT_RTZ_XU_F_V, MASK_VFWCVT_RTZ_XU_F_V) +DECLARE_INSN(vfwcvt_x_f_v, MATCH_VFWCVT_X_F_V, MASK_VFWCVT_X_F_V) +DECLARE_INSN(vfwcvt_xu_f_v, MATCH_VFWCVT_XU_F_V, MASK_VFWCVT_XU_F_V) +DECLARE_INSN(vfwmacc_vf, MATCH_VFWMACC_VF, MASK_VFWMACC_VF) +DECLARE_INSN(vfwmacc_vv, MATCH_VFWMACC_VV, MASK_VFWMACC_VV) +DECLARE_INSN(vfwmsac_vf, MATCH_VFWMSAC_VF, MASK_VFWMSAC_VF) +DECLARE_INSN(vfwmsac_vv, MATCH_VFWMSAC_VV, MASK_VFWMSAC_VV) +DECLARE_INSN(vfwmul_vf, MATCH_VFWMUL_VF, MASK_VFWMUL_VF) +DECLARE_INSN(vfwmul_vv, MATCH_VFWMUL_VV, MASK_VFWMUL_VV) +DECLARE_INSN(vfwnmacc_vf, MATCH_VFWNMACC_VF, MASK_VFWNMACC_VF) +DECLARE_INSN(vfwnmacc_vv, MATCH_VFWNMACC_VV, MASK_VFWNMACC_VV) +DECLARE_INSN(vfwnmsac_vf, MATCH_VFWNMSAC_VF, MASK_VFWNMSAC_VF) +DECLARE_INSN(vfwnmsac_vv, MATCH_VFWNMSAC_VV, MASK_VFWNMSAC_VV) +DECLARE_INSN(vfwredosum_vs, MATCH_VFWREDOSUM_VS, MASK_VFWREDOSUM_VS) +DECLARE_INSN(vfwredusum_vs, MATCH_VFWREDUSUM_VS, MASK_VFWREDUSUM_VS) +DECLARE_INSN(vfwsub_vf, MATCH_VFWSUB_VF, MASK_VFWSUB_VF) +DECLARE_INSN(vfwsub_vv, MATCH_VFWSUB_VV, MASK_VFWSUB_VV) +DECLARE_INSN(vfwsub_wf, MATCH_VFWSUB_WF, MASK_VFWSUB_WF) +DECLARE_INSN(vfwsub_wv, MATCH_VFWSUB_WV, MASK_VFWSUB_WV) +DECLARE_INSN(vid_v, MATCH_VID_V, MASK_VID_V) +DECLARE_INSN(viota_m, MATCH_VIOTA_M, MASK_VIOTA_M) +DECLARE_INSN(vl1re16_v, MATCH_VL1RE16_V, MASK_VL1RE16_V) +DECLARE_INSN(vl1re32_v, MATCH_VL1RE32_V, MASK_VL1RE32_V) +DECLARE_INSN(vl1re64_v, MATCH_VL1RE64_V, MASK_VL1RE64_V) +DECLARE_INSN(vl1re8_v, MATCH_VL1RE8_V, MASK_VL1RE8_V) +DECLARE_INSN(vl2re16_v, MATCH_VL2RE16_V, MASK_VL2RE16_V) +DECLARE_INSN(vl2re32_v, MATCH_VL2RE32_V, MASK_VL2RE32_V) +DECLARE_INSN(vl2re64_v, MATCH_VL2RE64_V, MASK_VL2RE64_V) +DECLARE_INSN(vl2re8_v, MATCH_VL2RE8_V, MASK_VL2RE8_V) +DECLARE_INSN(vl4re16_v, MATCH_VL4RE16_V, MASK_VL4RE16_V) +DECLARE_INSN(vl4re32_v, MATCH_VL4RE32_V, MASK_VL4RE32_V) +DECLARE_INSN(vl4re64_v, MATCH_VL4RE64_V, MASK_VL4RE64_V) +DECLARE_INSN(vl4re8_v, MATCH_VL4RE8_V, MASK_VL4RE8_V) +DECLARE_INSN(vl8re16_v, MATCH_VL8RE16_V, MASK_VL8RE16_V) +DECLARE_INSN(vl8re32_v, MATCH_VL8RE32_V, MASK_VL8RE32_V) +DECLARE_INSN(vl8re64_v, MATCH_VL8RE64_V, MASK_VL8RE64_V) +DECLARE_INSN(vl8re8_v, MATCH_VL8RE8_V, MASK_VL8RE8_V) +DECLARE_INSN(vle1024_v, MATCH_VLE1024_V, MASK_VLE1024_V) +DECLARE_INSN(vle1024ff_v, MATCH_VLE1024FF_V, MASK_VLE1024FF_V) +DECLARE_INSN(vle128_v, MATCH_VLE128_V, MASK_VLE128_V) +DECLARE_INSN(vle128ff_v, MATCH_VLE128FF_V, MASK_VLE128FF_V) +DECLARE_INSN(vle16_v, MATCH_VLE16_V, MASK_VLE16_V) +DECLARE_INSN(vle16ff_v, MATCH_VLE16FF_V, MASK_VLE16FF_V) +DECLARE_INSN(vle256_v, MATCH_VLE256_V, MASK_VLE256_V) +DECLARE_INSN(vle256ff_v, MATCH_VLE256FF_V, MASK_VLE256FF_V) +DECLARE_INSN(vle32_v, MATCH_VLE32_V, MASK_VLE32_V) +DECLARE_INSN(vle32ff_v, MATCH_VLE32FF_V, MASK_VLE32FF_V) +DECLARE_INSN(vle512_v, MATCH_VLE512_V, MASK_VLE512_V) +DECLARE_INSN(vle512ff_v, MATCH_VLE512FF_V, MASK_VLE512FF_V) +DECLARE_INSN(vle64_v, MATCH_VLE64_V, MASK_VLE64_V) +DECLARE_INSN(vle64ff_v, MATCH_VLE64FF_V, MASK_VLE64FF_V) +DECLARE_INSN(vle8_v, MATCH_VLE8_V, MASK_VLE8_V) +DECLARE_INSN(vle8ff_v, MATCH_VLE8FF_V, MASK_VLE8FF_V) +DECLARE_INSN(vlm_v, MATCH_VLM_V, MASK_VLM_V) +DECLARE_INSN(vloxei1024_v, MATCH_VLOXEI1024_V, MASK_VLOXEI1024_V) +DECLARE_INSN(vloxei128_v, MATCH_VLOXEI128_V, MASK_VLOXEI128_V) +DECLARE_INSN(vloxei16_v, MATCH_VLOXEI16_V, MASK_VLOXEI16_V) +DECLARE_INSN(vloxei256_v, MATCH_VLOXEI256_V, MASK_VLOXEI256_V) +DECLARE_INSN(vloxei32_v, MATCH_VLOXEI32_V, MASK_VLOXEI32_V) +DECLARE_INSN(vloxei512_v, MATCH_VLOXEI512_V, MASK_VLOXEI512_V) +DECLARE_INSN(vloxei64_v, MATCH_VLOXEI64_V, MASK_VLOXEI64_V) +DECLARE_INSN(vloxei8_v, MATCH_VLOXEI8_V, MASK_VLOXEI8_V) +DECLARE_INSN(vlse1024_v, MATCH_VLSE1024_V, MASK_VLSE1024_V) +DECLARE_INSN(vlse128_v, MATCH_VLSE128_V, MASK_VLSE128_V) +DECLARE_INSN(vlse16_v, MATCH_VLSE16_V, MASK_VLSE16_V) +DECLARE_INSN(vlse256_v, MATCH_VLSE256_V, MASK_VLSE256_V) +DECLARE_INSN(vlse32_v, MATCH_VLSE32_V, MASK_VLSE32_V) +DECLARE_INSN(vlse512_v, MATCH_VLSE512_V, MASK_VLSE512_V) +DECLARE_INSN(vlse64_v, MATCH_VLSE64_V, MASK_VLSE64_V) +DECLARE_INSN(vlse8_v, MATCH_VLSE8_V, MASK_VLSE8_V) +DECLARE_INSN(vluxei1024_v, MATCH_VLUXEI1024_V, MASK_VLUXEI1024_V) +DECLARE_INSN(vluxei128_v, MATCH_VLUXEI128_V, MASK_VLUXEI128_V) +DECLARE_INSN(vluxei16_v, MATCH_VLUXEI16_V, MASK_VLUXEI16_V) +DECLARE_INSN(vluxei256_v, MATCH_VLUXEI256_V, MASK_VLUXEI256_V) +DECLARE_INSN(vluxei32_v, MATCH_VLUXEI32_V, MASK_VLUXEI32_V) +DECLARE_INSN(vluxei512_v, MATCH_VLUXEI512_V, MASK_VLUXEI512_V) +DECLARE_INSN(vluxei64_v, MATCH_VLUXEI64_V, MASK_VLUXEI64_V) +DECLARE_INSN(vluxei8_v, MATCH_VLUXEI8_V, MASK_VLUXEI8_V) +DECLARE_INSN(vmacc_vv, MATCH_VMACC_VV, MASK_VMACC_VV) +DECLARE_INSN(vmacc_vx, MATCH_VMACC_VX, MASK_VMACC_VX) +DECLARE_INSN(vmadc_vi, MATCH_VMADC_VI, MASK_VMADC_VI) +DECLARE_INSN(vmadc_vim, MATCH_VMADC_VIM, MASK_VMADC_VIM) +DECLARE_INSN(vmadc_vv, MATCH_VMADC_VV, MASK_VMADC_VV) +DECLARE_INSN(vmadc_vvm, MATCH_VMADC_VVM, MASK_VMADC_VVM) +DECLARE_INSN(vmadc_vx, MATCH_VMADC_VX, MASK_VMADC_VX) +DECLARE_INSN(vmadc_vxm, MATCH_VMADC_VXM, MASK_VMADC_VXM) +DECLARE_INSN(vmadd_vv, MATCH_VMADD_VV, MASK_VMADD_VV) +DECLARE_INSN(vmadd_vx, MATCH_VMADD_VX, MASK_VMADD_VX) +DECLARE_INSN(vmand_mm, MATCH_VMAND_MM, MASK_VMAND_MM) +DECLARE_INSN(vmandn_mm, MATCH_VMANDN_MM, MASK_VMANDN_MM) +DECLARE_INSN(vmax_vv, MATCH_VMAX_VV, MASK_VMAX_VV) +DECLARE_INSN(vmax_vx, MATCH_VMAX_VX, MASK_VMAX_VX) +DECLARE_INSN(vmaxu_vv, MATCH_VMAXU_VV, MASK_VMAXU_VV) +DECLARE_INSN(vmaxu_vx, MATCH_VMAXU_VX, MASK_VMAXU_VX) +DECLARE_INSN(vmerge_vim, MATCH_VMERGE_VIM, MASK_VMERGE_VIM) +DECLARE_INSN(vmerge_vvm, MATCH_VMERGE_VVM, MASK_VMERGE_VVM) +DECLARE_INSN(vmerge_vxm, MATCH_VMERGE_VXM, MASK_VMERGE_VXM) +DECLARE_INSN(vmfeq_vf, MATCH_VMFEQ_VF, MASK_VMFEQ_VF) +DECLARE_INSN(vmfeq_vv, MATCH_VMFEQ_VV, MASK_VMFEQ_VV) +DECLARE_INSN(vmfge_vf, MATCH_VMFGE_VF, MASK_VMFGE_VF) +DECLARE_INSN(vmfgt_vf, MATCH_VMFGT_VF, MASK_VMFGT_VF) +DECLARE_INSN(vmfle_vf, MATCH_VMFLE_VF, MASK_VMFLE_VF) +DECLARE_INSN(vmfle_vv, MATCH_VMFLE_VV, MASK_VMFLE_VV) +DECLARE_INSN(vmflt_vf, MATCH_VMFLT_VF, MASK_VMFLT_VF) +DECLARE_INSN(vmflt_vv, MATCH_VMFLT_VV, MASK_VMFLT_VV) +DECLARE_INSN(vmfne_vf, MATCH_VMFNE_VF, MASK_VMFNE_VF) +DECLARE_INSN(vmfne_vv, MATCH_VMFNE_VV, MASK_VMFNE_VV) +DECLARE_INSN(vmin_vv, MATCH_VMIN_VV, MASK_VMIN_VV) +DECLARE_INSN(vmin_vx, MATCH_VMIN_VX, MASK_VMIN_VX) +DECLARE_INSN(vminu_vv, MATCH_VMINU_VV, MASK_VMINU_VV) +DECLARE_INSN(vminu_vx, MATCH_VMINU_VX, MASK_VMINU_VX) +DECLARE_INSN(vmnand_mm, MATCH_VMNAND_MM, MASK_VMNAND_MM) +DECLARE_INSN(vmnor_mm, MATCH_VMNOR_MM, MASK_VMNOR_MM) +DECLARE_INSN(vmor_mm, MATCH_VMOR_MM, MASK_VMOR_MM) +DECLARE_INSN(vmorn_mm, MATCH_VMORN_MM, MASK_VMORN_MM) +DECLARE_INSN(vmsbc_vv, MATCH_VMSBC_VV, MASK_VMSBC_VV) +DECLARE_INSN(vmsbc_vvm, MATCH_VMSBC_VVM, MASK_VMSBC_VVM) +DECLARE_INSN(vmsbc_vx, MATCH_VMSBC_VX, MASK_VMSBC_VX) +DECLARE_INSN(vmsbc_vxm, MATCH_VMSBC_VXM, MASK_VMSBC_VXM) +DECLARE_INSN(vmsbf_m, MATCH_VMSBF_M, MASK_VMSBF_M) +DECLARE_INSN(vmseq_vi, MATCH_VMSEQ_VI, MASK_VMSEQ_VI) +DECLARE_INSN(vmseq_vv, MATCH_VMSEQ_VV, MASK_VMSEQ_VV) +DECLARE_INSN(vmseq_vx, MATCH_VMSEQ_VX, MASK_VMSEQ_VX) +DECLARE_INSN(vmsgt_vi, MATCH_VMSGT_VI, MASK_VMSGT_VI) +DECLARE_INSN(vmsgt_vx, MATCH_VMSGT_VX, MASK_VMSGT_VX) +DECLARE_INSN(vmsgtu_vi, MATCH_VMSGTU_VI, MASK_VMSGTU_VI) +DECLARE_INSN(vmsgtu_vx, MATCH_VMSGTU_VX, MASK_VMSGTU_VX) +DECLARE_INSN(vmsif_m, MATCH_VMSIF_M, MASK_VMSIF_M) +DECLARE_INSN(vmsle_vi, MATCH_VMSLE_VI, MASK_VMSLE_VI) +DECLARE_INSN(vmsle_vv, MATCH_VMSLE_VV, MASK_VMSLE_VV) +DECLARE_INSN(vmsle_vx, MATCH_VMSLE_VX, MASK_VMSLE_VX) +DECLARE_INSN(vmsleu_vi, MATCH_VMSLEU_VI, MASK_VMSLEU_VI) +DECLARE_INSN(vmsleu_vv, MATCH_VMSLEU_VV, MASK_VMSLEU_VV) +DECLARE_INSN(vmsleu_vx, MATCH_VMSLEU_VX, MASK_VMSLEU_VX) +DECLARE_INSN(vmslt_vv, MATCH_VMSLT_VV, MASK_VMSLT_VV) +DECLARE_INSN(vmslt_vx, MATCH_VMSLT_VX, MASK_VMSLT_VX) +DECLARE_INSN(vmsltu_vv, MATCH_VMSLTU_VV, MASK_VMSLTU_VV) +DECLARE_INSN(vmsltu_vx, MATCH_VMSLTU_VX, MASK_VMSLTU_VX) +DECLARE_INSN(vmsne_vi, MATCH_VMSNE_VI, MASK_VMSNE_VI) +DECLARE_INSN(vmsne_vv, MATCH_VMSNE_VV, MASK_VMSNE_VV) +DECLARE_INSN(vmsne_vx, MATCH_VMSNE_VX, MASK_VMSNE_VX) +DECLARE_INSN(vmsof_m, MATCH_VMSOF_M, MASK_VMSOF_M) +DECLARE_INSN(vmul_vv, MATCH_VMUL_VV, MASK_VMUL_VV) +DECLARE_INSN(vmul_vx, MATCH_VMUL_VX, MASK_VMUL_VX) +DECLARE_INSN(vmulh_vv, MATCH_VMULH_VV, MASK_VMULH_VV) +DECLARE_INSN(vmulh_vx, MATCH_VMULH_VX, MASK_VMULH_VX) +DECLARE_INSN(vmulhsu_vv, MATCH_VMULHSU_VV, MASK_VMULHSU_VV) +DECLARE_INSN(vmulhsu_vx, MATCH_VMULHSU_VX, MASK_VMULHSU_VX) +DECLARE_INSN(vmulhu_vv, MATCH_VMULHU_VV, MASK_VMULHU_VV) +DECLARE_INSN(vmulhu_vx, MATCH_VMULHU_VX, MASK_VMULHU_VX) +DECLARE_INSN(vmv1r_v, MATCH_VMV1R_V, MASK_VMV1R_V) +DECLARE_INSN(vmv2r_v, MATCH_VMV2R_V, MASK_VMV2R_V) +DECLARE_INSN(vmv4r_v, MATCH_VMV4R_V, MASK_VMV4R_V) +DECLARE_INSN(vmv8r_v, MATCH_VMV8R_V, MASK_VMV8R_V) +DECLARE_INSN(vmv_s_x, MATCH_VMV_S_X, MASK_VMV_S_X) +DECLARE_INSN(vmv_v_i, MATCH_VMV_V_I, MASK_VMV_V_I) +DECLARE_INSN(vmv_v_v, MATCH_VMV_V_V, MASK_VMV_V_V) +DECLARE_INSN(vmv_v_x, MATCH_VMV_V_X, MASK_VMV_V_X) +DECLARE_INSN(vmv_x_s, MATCH_VMV_X_S, MASK_VMV_X_S) +DECLARE_INSN(vmxnor_mm, MATCH_VMXNOR_MM, MASK_VMXNOR_MM) +DECLARE_INSN(vmxor_mm, MATCH_VMXOR_MM, MASK_VMXOR_MM) +DECLARE_INSN(vnclip_wi, MATCH_VNCLIP_WI, MASK_VNCLIP_WI) +DECLARE_INSN(vnclip_wv, MATCH_VNCLIP_WV, MASK_VNCLIP_WV) +DECLARE_INSN(vnclip_wx, MATCH_VNCLIP_WX, MASK_VNCLIP_WX) +DECLARE_INSN(vnclipu_wi, MATCH_VNCLIPU_WI, MASK_VNCLIPU_WI) +DECLARE_INSN(vnclipu_wv, MATCH_VNCLIPU_WV, MASK_VNCLIPU_WV) +DECLARE_INSN(vnclipu_wx, MATCH_VNCLIPU_WX, MASK_VNCLIPU_WX) +DECLARE_INSN(vnmsac_vv, MATCH_VNMSAC_VV, MASK_VNMSAC_VV) +DECLARE_INSN(vnmsac_vx, MATCH_VNMSAC_VX, MASK_VNMSAC_VX) +DECLARE_INSN(vnmsub_vv, MATCH_VNMSUB_VV, MASK_VNMSUB_VV) +DECLARE_INSN(vnmsub_vx, MATCH_VNMSUB_VX, MASK_VNMSUB_VX) +DECLARE_INSN(vnsra_wi, MATCH_VNSRA_WI, MASK_VNSRA_WI) +DECLARE_INSN(vnsra_wv, MATCH_VNSRA_WV, MASK_VNSRA_WV) +DECLARE_INSN(vnsra_wx, MATCH_VNSRA_WX, MASK_VNSRA_WX) +DECLARE_INSN(vnsrl_wi, MATCH_VNSRL_WI, MASK_VNSRL_WI) +DECLARE_INSN(vnsrl_wv, MATCH_VNSRL_WV, MASK_VNSRL_WV) +DECLARE_INSN(vnsrl_wx, MATCH_VNSRL_WX, MASK_VNSRL_WX) +DECLARE_INSN(vor_vi, MATCH_VOR_VI, MASK_VOR_VI) +DECLARE_INSN(vor_vv, MATCH_VOR_VV, MASK_VOR_VV) +DECLARE_INSN(vor_vx, MATCH_VOR_VX, MASK_VOR_VX) +DECLARE_INSN(vredand_vs, MATCH_VREDAND_VS, MASK_VREDAND_VS) +DECLARE_INSN(vredmax_vs, MATCH_VREDMAX_VS, MASK_VREDMAX_VS) +DECLARE_INSN(vredmaxu_vs, MATCH_VREDMAXU_VS, MASK_VREDMAXU_VS) +DECLARE_INSN(vredmin_vs, MATCH_VREDMIN_VS, MASK_VREDMIN_VS) +DECLARE_INSN(vredminu_vs, MATCH_VREDMINU_VS, MASK_VREDMINU_VS) +DECLARE_INSN(vredor_vs, MATCH_VREDOR_VS, MASK_VREDOR_VS) +DECLARE_INSN(vredsum_vs, MATCH_VREDSUM_VS, MASK_VREDSUM_VS) +DECLARE_INSN(vredxor_vs, MATCH_VREDXOR_VS, MASK_VREDXOR_VS) +DECLARE_INSN(vrem_vv, MATCH_VREM_VV, MASK_VREM_VV) +DECLARE_INSN(vrem_vx, MATCH_VREM_VX, MASK_VREM_VX) +DECLARE_INSN(vremu_vv, MATCH_VREMU_VV, MASK_VREMU_VV) +DECLARE_INSN(vremu_vx, MATCH_VREMU_VX, MASK_VREMU_VX) +DECLARE_INSN(vrgather_vi, MATCH_VRGATHER_VI, MASK_VRGATHER_VI) +DECLARE_INSN(vrgather_vv, MATCH_VRGATHER_VV, MASK_VRGATHER_VV) +DECLARE_INSN(vrgather_vx, MATCH_VRGATHER_VX, MASK_VRGATHER_VX) +DECLARE_INSN(vrgatherei16_vv, MATCH_VRGATHEREI16_VV, MASK_VRGATHEREI16_VV) +DECLARE_INSN(vrsub_vi, MATCH_VRSUB_VI, MASK_VRSUB_VI) +DECLARE_INSN(vrsub_vx, MATCH_VRSUB_VX, MASK_VRSUB_VX) +DECLARE_INSN(vs1r_v, MATCH_VS1R_V, MASK_VS1R_V) +DECLARE_INSN(vs2r_v, MATCH_VS2R_V, MASK_VS2R_V) +DECLARE_INSN(vs4r_v, MATCH_VS4R_V, MASK_VS4R_V) +DECLARE_INSN(vs8r_v, MATCH_VS8R_V, MASK_VS8R_V) +DECLARE_INSN(vsadd_vi, MATCH_VSADD_VI, MASK_VSADD_VI) +DECLARE_INSN(vsadd_vv, MATCH_VSADD_VV, MASK_VSADD_VV) +DECLARE_INSN(vsadd_vx, MATCH_VSADD_VX, MASK_VSADD_VX) +DECLARE_INSN(vsaddu_vi, MATCH_VSADDU_VI, MASK_VSADDU_VI) +DECLARE_INSN(vsaddu_vv, MATCH_VSADDU_VV, MASK_VSADDU_VV) +DECLARE_INSN(vsaddu_vx, MATCH_VSADDU_VX, MASK_VSADDU_VX) +DECLARE_INSN(vsbc_vvm, MATCH_VSBC_VVM, MASK_VSBC_VVM) +DECLARE_INSN(vsbc_vxm, MATCH_VSBC_VXM, MASK_VSBC_VXM) +DECLARE_INSN(vse1024_v, MATCH_VSE1024_V, MASK_VSE1024_V) +DECLARE_INSN(vse128_v, MATCH_VSE128_V, MASK_VSE128_V) +DECLARE_INSN(vse16_v, MATCH_VSE16_V, MASK_VSE16_V) +DECLARE_INSN(vse256_v, MATCH_VSE256_V, MASK_VSE256_V) +DECLARE_INSN(vse32_v, MATCH_VSE32_V, MASK_VSE32_V) +DECLARE_INSN(vse512_v, MATCH_VSE512_V, MASK_VSE512_V) +DECLARE_INSN(vse64_v, MATCH_VSE64_V, MASK_VSE64_V) +DECLARE_INSN(vse8_v, MATCH_VSE8_V, MASK_VSE8_V) +DECLARE_INSN(vsetivli, MATCH_VSETIVLI, MASK_VSETIVLI) +DECLARE_INSN(vsetvl, MATCH_VSETVL, MASK_VSETVL) +DECLARE_INSN(vsetvli, MATCH_VSETVLI, MASK_VSETVLI) +DECLARE_INSN(vsext_vf2, MATCH_VSEXT_VF2, MASK_VSEXT_VF2) +DECLARE_INSN(vsext_vf4, MATCH_VSEXT_VF4, MASK_VSEXT_VF4) +DECLARE_INSN(vsext_vf8, MATCH_VSEXT_VF8, MASK_VSEXT_VF8) +DECLARE_INSN(vslide1down_vx, MATCH_VSLIDE1DOWN_VX, MASK_VSLIDE1DOWN_VX) +DECLARE_INSN(vslide1up_vx, MATCH_VSLIDE1UP_VX, MASK_VSLIDE1UP_VX) +DECLARE_INSN(vslidedown_vi, MATCH_VSLIDEDOWN_VI, MASK_VSLIDEDOWN_VI) +DECLARE_INSN(vslidedown_vx, MATCH_VSLIDEDOWN_VX, MASK_VSLIDEDOWN_VX) +DECLARE_INSN(vslideup_vi, MATCH_VSLIDEUP_VI, MASK_VSLIDEUP_VI) +DECLARE_INSN(vslideup_vx, MATCH_VSLIDEUP_VX, MASK_VSLIDEUP_VX) +DECLARE_INSN(vsll_vi, MATCH_VSLL_VI, MASK_VSLL_VI) +DECLARE_INSN(vsll_vv, MATCH_VSLL_VV, MASK_VSLL_VV) +DECLARE_INSN(vsll_vx, MATCH_VSLL_VX, MASK_VSLL_VX) +DECLARE_INSN(vsm_v, MATCH_VSM_V, MASK_VSM_V) +DECLARE_INSN(vsmul_vv, MATCH_VSMUL_VV, MASK_VSMUL_VV) +DECLARE_INSN(vsmul_vx, MATCH_VSMUL_VX, MASK_VSMUL_VX) +DECLARE_INSN(vsoxei1024_v, MATCH_VSOXEI1024_V, MASK_VSOXEI1024_V) +DECLARE_INSN(vsoxei128_v, MATCH_VSOXEI128_V, MASK_VSOXEI128_V) +DECLARE_INSN(vsoxei16_v, MATCH_VSOXEI16_V, MASK_VSOXEI16_V) +DECLARE_INSN(vsoxei256_v, MATCH_VSOXEI256_V, MASK_VSOXEI256_V) +DECLARE_INSN(vsoxei32_v, MATCH_VSOXEI32_V, MASK_VSOXEI32_V) +DECLARE_INSN(vsoxei512_v, MATCH_VSOXEI512_V, MASK_VSOXEI512_V) +DECLARE_INSN(vsoxei64_v, MATCH_VSOXEI64_V, MASK_VSOXEI64_V) +DECLARE_INSN(vsoxei8_v, MATCH_VSOXEI8_V, MASK_VSOXEI8_V) +DECLARE_INSN(vsra_vi, MATCH_VSRA_VI, MASK_VSRA_VI) +DECLARE_INSN(vsra_vv, MATCH_VSRA_VV, MASK_VSRA_VV) +DECLARE_INSN(vsra_vx, MATCH_VSRA_VX, MASK_VSRA_VX) +DECLARE_INSN(vsrl_vi, MATCH_VSRL_VI, MASK_VSRL_VI) +DECLARE_INSN(vsrl_vv, MATCH_VSRL_VV, MASK_VSRL_VV) +DECLARE_INSN(vsrl_vx, MATCH_VSRL_VX, MASK_VSRL_VX) +DECLARE_INSN(vsse1024_v, MATCH_VSSE1024_V, MASK_VSSE1024_V) +DECLARE_INSN(vsse128_v, MATCH_VSSE128_V, MASK_VSSE128_V) +DECLARE_INSN(vsse16_v, MATCH_VSSE16_V, MASK_VSSE16_V) +DECLARE_INSN(vsse256_v, MATCH_VSSE256_V, MASK_VSSE256_V) +DECLARE_INSN(vsse32_v, MATCH_VSSE32_V, MASK_VSSE32_V) +DECLARE_INSN(vsse512_v, MATCH_VSSE512_V, MASK_VSSE512_V) +DECLARE_INSN(vsse64_v, MATCH_VSSE64_V, MASK_VSSE64_V) +DECLARE_INSN(vsse8_v, MATCH_VSSE8_V, MASK_VSSE8_V) +DECLARE_INSN(vssra_vi, MATCH_VSSRA_VI, MASK_VSSRA_VI) +DECLARE_INSN(vssra_vv, MATCH_VSSRA_VV, MASK_VSSRA_VV) +DECLARE_INSN(vssra_vx, MATCH_VSSRA_VX, MASK_VSSRA_VX) +DECLARE_INSN(vssrl_vi, MATCH_VSSRL_VI, MASK_VSSRL_VI) +DECLARE_INSN(vssrl_vv, MATCH_VSSRL_VV, MASK_VSSRL_VV) +DECLARE_INSN(vssrl_vx, MATCH_VSSRL_VX, MASK_VSSRL_VX) +DECLARE_INSN(vssub_vv, MATCH_VSSUB_VV, MASK_VSSUB_VV) +DECLARE_INSN(vssub_vx, MATCH_VSSUB_VX, MASK_VSSUB_VX) +DECLARE_INSN(vssubu_vv, MATCH_VSSUBU_VV, MASK_VSSUBU_VV) +DECLARE_INSN(vssubu_vx, MATCH_VSSUBU_VX, MASK_VSSUBU_VX) +DECLARE_INSN(vsub_vv, MATCH_VSUB_VV, MASK_VSUB_VV) +DECLARE_INSN(vsub_vx, MATCH_VSUB_VX, MASK_VSUB_VX) +DECLARE_INSN(vsuxei1024_v, MATCH_VSUXEI1024_V, MASK_VSUXEI1024_V) +DECLARE_INSN(vsuxei128_v, MATCH_VSUXEI128_V, MASK_VSUXEI128_V) +DECLARE_INSN(vsuxei16_v, MATCH_VSUXEI16_V, MASK_VSUXEI16_V) +DECLARE_INSN(vsuxei256_v, MATCH_VSUXEI256_V, MASK_VSUXEI256_V) +DECLARE_INSN(vsuxei32_v, MATCH_VSUXEI32_V, MASK_VSUXEI32_V) +DECLARE_INSN(vsuxei512_v, MATCH_VSUXEI512_V, MASK_VSUXEI512_V) +DECLARE_INSN(vsuxei64_v, MATCH_VSUXEI64_V, MASK_VSUXEI64_V) +DECLARE_INSN(vsuxei8_v, MATCH_VSUXEI8_V, MASK_VSUXEI8_V) +DECLARE_INSN(vwadd_vv, MATCH_VWADD_VV, MASK_VWADD_VV) +DECLARE_INSN(vwadd_vx, MATCH_VWADD_VX, MASK_VWADD_VX) +DECLARE_INSN(vwadd_wv, MATCH_VWADD_WV, MASK_VWADD_WV) +DECLARE_INSN(vwadd_wx, MATCH_VWADD_WX, MASK_VWADD_WX) +DECLARE_INSN(vwaddu_vv, MATCH_VWADDU_VV, MASK_VWADDU_VV) +DECLARE_INSN(vwaddu_vx, MATCH_VWADDU_VX, MASK_VWADDU_VX) +DECLARE_INSN(vwaddu_wv, MATCH_VWADDU_WV, MASK_VWADDU_WV) +DECLARE_INSN(vwaddu_wx, MATCH_VWADDU_WX, MASK_VWADDU_WX) +DECLARE_INSN(vwmacc_vv, MATCH_VWMACC_VV, MASK_VWMACC_VV) +DECLARE_INSN(vwmacc_vx, MATCH_VWMACC_VX, MASK_VWMACC_VX) +DECLARE_INSN(vwmaccsu_vv, MATCH_VWMACCSU_VV, MASK_VWMACCSU_VV) +DECLARE_INSN(vwmaccsu_vx, MATCH_VWMACCSU_VX, MASK_VWMACCSU_VX) +DECLARE_INSN(vwmaccu_vv, MATCH_VWMACCU_VV, MASK_VWMACCU_VV) +DECLARE_INSN(vwmaccu_vx, MATCH_VWMACCU_VX, MASK_VWMACCU_VX) +DECLARE_INSN(vwmaccus_vx, MATCH_VWMACCUS_VX, MASK_VWMACCUS_VX) +DECLARE_INSN(vwmul_vv, MATCH_VWMUL_VV, MASK_VWMUL_VV) +DECLARE_INSN(vwmul_vx, MATCH_VWMUL_VX, MASK_VWMUL_VX) +DECLARE_INSN(vwmulsu_vv, MATCH_VWMULSU_VV, MASK_VWMULSU_VV) +DECLARE_INSN(vwmulsu_vx, MATCH_VWMULSU_VX, MASK_VWMULSU_VX) +DECLARE_INSN(vwmulu_vv, MATCH_VWMULU_VV, MASK_VWMULU_VV) +DECLARE_INSN(vwmulu_vx, MATCH_VWMULU_VX, MASK_VWMULU_VX) +DECLARE_INSN(vwredsum_vs, MATCH_VWREDSUM_VS, MASK_VWREDSUM_VS) +DECLARE_INSN(vwredsumu_vs, MATCH_VWREDSUMU_VS, MASK_VWREDSUMU_VS) +DECLARE_INSN(vwsub_vv, MATCH_VWSUB_VV, MASK_VWSUB_VV) +DECLARE_INSN(vwsub_vx, MATCH_VWSUB_VX, MASK_VWSUB_VX) +DECLARE_INSN(vwsub_wv, MATCH_VWSUB_WV, MASK_VWSUB_WV) +DECLARE_INSN(vwsub_wx, MATCH_VWSUB_WX, MASK_VWSUB_WX) +DECLARE_INSN(vwsubu_vv, MATCH_VWSUBU_VV, MASK_VWSUBU_VV) +DECLARE_INSN(vwsubu_vx, MATCH_VWSUBU_VX, MASK_VWSUBU_VX) +DECLARE_INSN(vwsubu_wv, MATCH_VWSUBU_WV, MASK_VWSUBU_WV) +DECLARE_INSN(vwsubu_wx, MATCH_VWSUBU_WX, MASK_VWSUBU_WX) +DECLARE_INSN(vxor_vi, MATCH_VXOR_VI, MASK_VXOR_VI) +DECLARE_INSN(vxor_vv, MATCH_VXOR_VV, MASK_VXOR_VV) +DECLARE_INSN(vxor_vx, MATCH_VXOR_VX, MASK_VXOR_VX) +DECLARE_INSN(vzext_vf2, MATCH_VZEXT_VF2, MASK_VZEXT_VF2) +DECLARE_INSN(vzext_vf4, MATCH_VZEXT_VF4, MASK_VZEXT_VF4) +DECLARE_INSN(vzext_vf8, MATCH_VZEXT_VF8, MASK_VZEXT_VF8) +DECLARE_INSN(wext, MATCH_WEXT, MASK_WEXT) +DECLARE_INSN(wexti, MATCH_WEXTI, MASK_WEXTI) +DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI) +DECLARE_INSN(wrs_nto, MATCH_WRS_NTO, MASK_WRS_NTO) +DECLARE_INSN(wrs_sto, MATCH_WRS_STO, MASK_WRS_STO) +DECLARE_INSN(xnor, MATCH_XNOR, MASK_XNOR) +DECLARE_INSN(xor, MATCH_XOR, MASK_XOR) +DECLARE_INSN(xori, MATCH_XORI, MASK_XORI) +DECLARE_INSN(xperm16, MATCH_XPERM16, MASK_XPERM16) +DECLARE_INSN(xperm32, MATCH_XPERM32, MASK_XPERM32) +DECLARE_INSN(xperm4, MATCH_XPERM4, MASK_XPERM4) +DECLARE_INSN(xperm8, MATCH_XPERM8, MASK_XPERM8) +DECLARE_INSN(zunpkd810, MATCH_ZUNPKD810, MASK_ZUNPKD810) +DECLARE_INSN(zunpkd820, MATCH_ZUNPKD820, MASK_ZUNPKD820) +DECLARE_INSN(zunpkd830, MATCH_ZUNPKD830, MASK_ZUNPKD830) +DECLARE_INSN(zunpkd831, MATCH_ZUNPKD831, MASK_ZUNPKD831) +DECLARE_INSN(zunpkd832, MATCH_ZUNPKD832, MASK_ZUNPKD832) +#endif +#ifdef DECLARE_CSR +DECLARE_CSR(fflags, CSR_FFLAGS) +DECLARE_CSR(frm, CSR_FRM) +DECLARE_CSR(fcsr, CSR_FCSR) +DECLARE_CSR(vstart, CSR_VSTART) +DECLARE_CSR(vxsat, CSR_VXSAT) +DECLARE_CSR(vxrm, CSR_VXRM) +DECLARE_CSR(vcsr, CSR_VCSR) +DECLARE_CSR(seed, CSR_SEED) +DECLARE_CSR(cycle, CSR_CYCLE) +DECLARE_CSR(time, CSR_TIME) +DECLARE_CSR(instret, CSR_INSTRET) +DECLARE_CSR(hpmcounter3, CSR_HPMCOUNTER3) +DECLARE_CSR(hpmcounter4, CSR_HPMCOUNTER4) +DECLARE_CSR(hpmcounter5, CSR_HPMCOUNTER5) +DECLARE_CSR(hpmcounter6, CSR_HPMCOUNTER6) +DECLARE_CSR(hpmcounter7, CSR_HPMCOUNTER7) +DECLARE_CSR(hpmcounter8, CSR_HPMCOUNTER8) +DECLARE_CSR(hpmcounter9, CSR_HPMCOUNTER9) +DECLARE_CSR(hpmcounter10, CSR_HPMCOUNTER10) +DECLARE_CSR(hpmcounter11, CSR_HPMCOUNTER11) +DECLARE_CSR(hpmcounter12, CSR_HPMCOUNTER12) +DECLARE_CSR(hpmcounter13, CSR_HPMCOUNTER13) +DECLARE_CSR(hpmcounter14, CSR_HPMCOUNTER14) +DECLARE_CSR(hpmcounter15, CSR_HPMCOUNTER15) +DECLARE_CSR(hpmcounter16, CSR_HPMCOUNTER16) +DECLARE_CSR(hpmcounter17, CSR_HPMCOUNTER17) +DECLARE_CSR(hpmcounter18, CSR_HPMCOUNTER18) +DECLARE_CSR(hpmcounter19, CSR_HPMCOUNTER19) +DECLARE_CSR(hpmcounter20, CSR_HPMCOUNTER20) +DECLARE_CSR(hpmcounter21, CSR_HPMCOUNTER21) +DECLARE_CSR(hpmcounter22, CSR_HPMCOUNTER22) +DECLARE_CSR(hpmcounter23, CSR_HPMCOUNTER23) +DECLARE_CSR(hpmcounter24, CSR_HPMCOUNTER24) +DECLARE_CSR(hpmcounter25, CSR_HPMCOUNTER25) +DECLARE_CSR(hpmcounter26, CSR_HPMCOUNTER26) +DECLARE_CSR(hpmcounter27, CSR_HPMCOUNTER27) +DECLARE_CSR(hpmcounter28, CSR_HPMCOUNTER28) +DECLARE_CSR(hpmcounter29, CSR_HPMCOUNTER29) +DECLARE_CSR(hpmcounter30, CSR_HPMCOUNTER30) +DECLARE_CSR(hpmcounter31, CSR_HPMCOUNTER31) +DECLARE_CSR(vl, CSR_VL) +DECLARE_CSR(vtype, CSR_VTYPE) +DECLARE_CSR(vlenb, CSR_VLENB) +DECLARE_CSR(sstatus, CSR_SSTATUS) +DECLARE_CSR(sedeleg, CSR_SEDELEG) +DECLARE_CSR(sideleg, CSR_SIDELEG) +DECLARE_CSR(sie, CSR_SIE) +DECLARE_CSR(stvec, CSR_STVEC) +DECLARE_CSR(scounteren, CSR_SCOUNTEREN) +DECLARE_CSR(senvcfg, CSR_SENVCFG) +DECLARE_CSR(sstateen0, CSR_SSTATEEN0) +DECLARE_CSR(sstateen1, CSR_SSTATEEN1) +DECLARE_CSR(sstateen2, CSR_SSTATEEN2) +DECLARE_CSR(sstateen3, CSR_SSTATEEN3) +DECLARE_CSR(sscratch, CSR_SSCRATCH) +DECLARE_CSR(sepc, CSR_SEPC) +DECLARE_CSR(scause, CSR_SCAUSE) +DECLARE_CSR(stval, CSR_STVAL) +DECLARE_CSR(sip, CSR_SIP) +DECLARE_CSR(stimecmp, CSR_STIMECMP) +DECLARE_CSR(satp, CSR_SATP) +DECLARE_CSR(scontext, CSR_SCONTEXT) +DECLARE_CSR(vsstatus, CSR_VSSTATUS) +DECLARE_CSR(vsie, CSR_VSIE) +DECLARE_CSR(vstvec, CSR_VSTVEC) +DECLARE_CSR(vsscratch, CSR_VSSCRATCH) +DECLARE_CSR(vsepc, CSR_VSEPC) +DECLARE_CSR(vscause, CSR_VSCAUSE) +DECLARE_CSR(vstval, CSR_VSTVAL) +DECLARE_CSR(vsip, CSR_VSIP) +DECLARE_CSR(vstimecmp, CSR_VSTIMECMP) +DECLARE_CSR(vsatp, CSR_VSATP) +DECLARE_CSR(hstatus, CSR_HSTATUS) +DECLARE_CSR(hedeleg, CSR_HEDELEG) +DECLARE_CSR(hideleg, CSR_HIDELEG) +DECLARE_CSR(hie, CSR_HIE) +DECLARE_CSR(htimedelta, CSR_HTIMEDELTA) +DECLARE_CSR(hcounteren, CSR_HCOUNTEREN) +DECLARE_CSR(hgeie, CSR_HGEIE) +DECLARE_CSR(henvcfg, CSR_HENVCFG) +DECLARE_CSR(hstateen0, CSR_HSTATEEN0) +DECLARE_CSR(hstateen1, CSR_HSTATEEN1) +DECLARE_CSR(hstateen2, CSR_HSTATEEN2) +DECLARE_CSR(hstateen3, CSR_HSTATEEN3) +DECLARE_CSR(htval, CSR_HTVAL) +DECLARE_CSR(hip, CSR_HIP) +DECLARE_CSR(hvip, CSR_HVIP) +DECLARE_CSR(htinst, CSR_HTINST) +DECLARE_CSR(hgatp, CSR_HGATP) +DECLARE_CSR(hcontext, CSR_HCONTEXT) +DECLARE_CSR(hgeip, CSR_HGEIP) +DECLARE_CSR(scountovf, CSR_SCOUNTOVF) +DECLARE_CSR(utvt, CSR_UTVT) +DECLARE_CSR(unxti, CSR_UNXTI) +DECLARE_CSR(uintstatus, CSR_UINTSTATUS) +DECLARE_CSR(uscratchcsw, CSR_USCRATCHCSW) +DECLARE_CSR(uscratchcswl, CSR_USCRATCHCSWL) +DECLARE_CSR(stvt, CSR_STVT) +DECLARE_CSR(snxti, CSR_SNXTI) +DECLARE_CSR(sintstatus, CSR_SINTSTATUS) +DECLARE_CSR(sscratchcsw, CSR_SSCRATCHCSW) +DECLARE_CSR(sscratchcswl, CSR_SSCRATCHCSWL) +DECLARE_CSR(mtvt, CSR_MTVT) +DECLARE_CSR(mnxti, CSR_MNXTI) +DECLARE_CSR(mintstatus, CSR_MINTSTATUS) +DECLARE_CSR(mscratchcsw, CSR_MSCRATCHCSW) +DECLARE_CSR(mscratchcswl, CSR_MSCRATCHCSWL) +DECLARE_CSR(mstatus, CSR_MSTATUS) +DECLARE_CSR(misa, CSR_MISA) +DECLARE_CSR(medeleg, CSR_MEDELEG) +DECLARE_CSR(mideleg, CSR_MIDELEG) +DECLARE_CSR(mie, CSR_MIE) +DECLARE_CSR(mtvec, CSR_MTVEC) +DECLARE_CSR(mcounteren, CSR_MCOUNTEREN) +DECLARE_CSR(menvcfg, CSR_MENVCFG) +DECLARE_CSR(mstateen0, CSR_MSTATEEN0) +DECLARE_CSR(mstateen1, CSR_MSTATEEN1) +DECLARE_CSR(mstateen2, CSR_MSTATEEN2) +DECLARE_CSR(mstateen3, CSR_MSTATEEN3) +DECLARE_CSR(mcountinhibit, CSR_MCOUNTINHIBIT) +DECLARE_CSR(mscratch, CSR_MSCRATCH) +DECLARE_CSR(mepc, CSR_MEPC) +DECLARE_CSR(mcause, CSR_MCAUSE) +DECLARE_CSR(mtval, CSR_MTVAL) +DECLARE_CSR(mip, CSR_MIP) +DECLARE_CSR(mtinst, CSR_MTINST) +DECLARE_CSR(mtval2, CSR_MTVAL2) +DECLARE_CSR(pmpcfg0, CSR_PMPCFG0) +DECLARE_CSR(pmpcfg1, CSR_PMPCFG1) +DECLARE_CSR(pmpcfg2, CSR_PMPCFG2) +DECLARE_CSR(pmpcfg3, CSR_PMPCFG3) +DECLARE_CSR(pmpcfg4, CSR_PMPCFG4) +DECLARE_CSR(pmpcfg5, CSR_PMPCFG5) +DECLARE_CSR(pmpcfg6, CSR_PMPCFG6) +DECLARE_CSR(pmpcfg7, CSR_PMPCFG7) +DECLARE_CSR(pmpcfg8, CSR_PMPCFG8) +DECLARE_CSR(pmpcfg9, CSR_PMPCFG9) +DECLARE_CSR(pmpcfg10, CSR_PMPCFG10) +DECLARE_CSR(pmpcfg11, CSR_PMPCFG11) +DECLARE_CSR(pmpcfg12, CSR_PMPCFG12) +DECLARE_CSR(pmpcfg13, CSR_PMPCFG13) +DECLARE_CSR(pmpcfg14, CSR_PMPCFG14) +DECLARE_CSR(pmpcfg15, CSR_PMPCFG15) +DECLARE_CSR(pmpaddr0, CSR_PMPADDR0) +DECLARE_CSR(pmpaddr1, CSR_PMPADDR1) +DECLARE_CSR(pmpaddr2, CSR_PMPADDR2) +DECLARE_CSR(pmpaddr3, CSR_PMPADDR3) +DECLARE_CSR(pmpaddr4, CSR_PMPADDR4) +DECLARE_CSR(pmpaddr5, CSR_PMPADDR5) +DECLARE_CSR(pmpaddr6, CSR_PMPADDR6) +DECLARE_CSR(pmpaddr7, CSR_PMPADDR7) +DECLARE_CSR(pmpaddr8, CSR_PMPADDR8) +DECLARE_CSR(pmpaddr9, CSR_PMPADDR9) +DECLARE_CSR(pmpaddr10, CSR_PMPADDR10) +DECLARE_CSR(pmpaddr11, CSR_PMPADDR11) +DECLARE_CSR(pmpaddr12, CSR_PMPADDR12) +DECLARE_CSR(pmpaddr13, CSR_PMPADDR13) +DECLARE_CSR(pmpaddr14, CSR_PMPADDR14) +DECLARE_CSR(pmpaddr15, CSR_PMPADDR15) +DECLARE_CSR(pmpaddr16, CSR_PMPADDR16) +DECLARE_CSR(pmpaddr17, CSR_PMPADDR17) +DECLARE_CSR(pmpaddr18, CSR_PMPADDR18) +DECLARE_CSR(pmpaddr19, CSR_PMPADDR19) +DECLARE_CSR(pmpaddr20, CSR_PMPADDR20) +DECLARE_CSR(pmpaddr21, CSR_PMPADDR21) +DECLARE_CSR(pmpaddr22, CSR_PMPADDR22) +DECLARE_CSR(pmpaddr23, CSR_PMPADDR23) +DECLARE_CSR(pmpaddr24, CSR_PMPADDR24) +DECLARE_CSR(pmpaddr25, CSR_PMPADDR25) +DECLARE_CSR(pmpaddr26, CSR_PMPADDR26) +DECLARE_CSR(pmpaddr27, CSR_PMPADDR27) +DECLARE_CSR(pmpaddr28, CSR_PMPADDR28) +DECLARE_CSR(pmpaddr29, CSR_PMPADDR29) +DECLARE_CSR(pmpaddr30, CSR_PMPADDR30) +DECLARE_CSR(pmpaddr31, CSR_PMPADDR31) +DECLARE_CSR(pmpaddr32, CSR_PMPADDR32) +DECLARE_CSR(pmpaddr33, CSR_PMPADDR33) +DECLARE_CSR(pmpaddr34, CSR_PMPADDR34) +DECLARE_CSR(pmpaddr35, CSR_PMPADDR35) +DECLARE_CSR(pmpaddr36, CSR_PMPADDR36) +DECLARE_CSR(pmpaddr37, CSR_PMPADDR37) +DECLARE_CSR(pmpaddr38, CSR_PMPADDR38) +DECLARE_CSR(pmpaddr39, CSR_PMPADDR39) +DECLARE_CSR(pmpaddr40, CSR_PMPADDR40) +DECLARE_CSR(pmpaddr41, CSR_PMPADDR41) +DECLARE_CSR(pmpaddr42, CSR_PMPADDR42) +DECLARE_CSR(pmpaddr43, CSR_PMPADDR43) +DECLARE_CSR(pmpaddr44, CSR_PMPADDR44) +DECLARE_CSR(pmpaddr45, CSR_PMPADDR45) +DECLARE_CSR(pmpaddr46, CSR_PMPADDR46) +DECLARE_CSR(pmpaddr47, CSR_PMPADDR47) +DECLARE_CSR(pmpaddr48, CSR_PMPADDR48) +DECLARE_CSR(pmpaddr49, CSR_PMPADDR49) +DECLARE_CSR(pmpaddr50, CSR_PMPADDR50) +DECLARE_CSR(pmpaddr51, CSR_PMPADDR51) +DECLARE_CSR(pmpaddr52, CSR_PMPADDR52) +DECLARE_CSR(pmpaddr53, CSR_PMPADDR53) +DECLARE_CSR(pmpaddr54, CSR_PMPADDR54) +DECLARE_CSR(pmpaddr55, CSR_PMPADDR55) +DECLARE_CSR(pmpaddr56, CSR_PMPADDR56) +DECLARE_CSR(pmpaddr57, CSR_PMPADDR57) +DECLARE_CSR(pmpaddr58, CSR_PMPADDR58) +DECLARE_CSR(pmpaddr59, CSR_PMPADDR59) +DECLARE_CSR(pmpaddr60, CSR_PMPADDR60) +DECLARE_CSR(pmpaddr61, CSR_PMPADDR61) +DECLARE_CSR(pmpaddr62, CSR_PMPADDR62) +DECLARE_CSR(pmpaddr63, CSR_PMPADDR63) +DECLARE_CSR(mseccfg, CSR_MSECCFG) +DECLARE_CSR(tselect, CSR_TSELECT) +DECLARE_CSR(tdata1, CSR_TDATA1) +DECLARE_CSR(tdata2, CSR_TDATA2) +DECLARE_CSR(tdata3, CSR_TDATA3) +DECLARE_CSR(tinfo, CSR_TINFO) +DECLARE_CSR(tcontrol, CSR_TCONTROL) +DECLARE_CSR(mcontext, CSR_MCONTEXT) +DECLARE_CSR(mscontext, CSR_MSCONTEXT) +DECLARE_CSR(dcsr, CSR_DCSR) +DECLARE_CSR(dpc, CSR_DPC) +DECLARE_CSR(dscratch0, CSR_DSCRATCH0) +DECLARE_CSR(dscratch1, CSR_DSCRATCH1) +DECLARE_CSR(mcycle, CSR_MCYCLE) +DECLARE_CSR(minstret, CSR_MINSTRET) +DECLARE_CSR(mhpmcounter3, CSR_MHPMCOUNTER3) +DECLARE_CSR(mhpmcounter4, CSR_MHPMCOUNTER4) +DECLARE_CSR(mhpmcounter5, CSR_MHPMCOUNTER5) +DECLARE_CSR(mhpmcounter6, CSR_MHPMCOUNTER6) +DECLARE_CSR(mhpmcounter7, CSR_MHPMCOUNTER7) +DECLARE_CSR(mhpmcounter8, CSR_MHPMCOUNTER8) +DECLARE_CSR(mhpmcounter9, CSR_MHPMCOUNTER9) +DECLARE_CSR(mhpmcounter10, CSR_MHPMCOUNTER10) +DECLARE_CSR(mhpmcounter11, CSR_MHPMCOUNTER11) +DECLARE_CSR(mhpmcounter12, CSR_MHPMCOUNTER12) +DECLARE_CSR(mhpmcounter13, CSR_MHPMCOUNTER13) +DECLARE_CSR(mhpmcounter14, CSR_MHPMCOUNTER14) +DECLARE_CSR(mhpmcounter15, CSR_MHPMCOUNTER15) +DECLARE_CSR(mhpmcounter16, CSR_MHPMCOUNTER16) +DECLARE_CSR(mhpmcounter17, CSR_MHPMCOUNTER17) +DECLARE_CSR(mhpmcounter18, CSR_MHPMCOUNTER18) +DECLARE_CSR(mhpmcounter19, CSR_MHPMCOUNTER19) +DECLARE_CSR(mhpmcounter20, CSR_MHPMCOUNTER20) +DECLARE_CSR(mhpmcounter21, CSR_MHPMCOUNTER21) +DECLARE_CSR(mhpmcounter22, CSR_MHPMCOUNTER22) +DECLARE_CSR(mhpmcounter23, CSR_MHPMCOUNTER23) +DECLARE_CSR(mhpmcounter24, CSR_MHPMCOUNTER24) +DECLARE_CSR(mhpmcounter25, CSR_MHPMCOUNTER25) +DECLARE_CSR(mhpmcounter26, CSR_MHPMCOUNTER26) +DECLARE_CSR(mhpmcounter27, CSR_MHPMCOUNTER27) +DECLARE_CSR(mhpmcounter28, CSR_MHPMCOUNTER28) +DECLARE_CSR(mhpmcounter29, CSR_MHPMCOUNTER29) +DECLARE_CSR(mhpmcounter30, CSR_MHPMCOUNTER30) +DECLARE_CSR(mhpmcounter31, CSR_MHPMCOUNTER31) +DECLARE_CSR(mhpmevent3, CSR_MHPMEVENT3) +DECLARE_CSR(mhpmevent4, CSR_MHPMEVENT4) +DECLARE_CSR(mhpmevent5, CSR_MHPMEVENT5) +DECLARE_CSR(mhpmevent6, CSR_MHPMEVENT6) +DECLARE_CSR(mhpmevent7, CSR_MHPMEVENT7) +DECLARE_CSR(mhpmevent8, CSR_MHPMEVENT8) +DECLARE_CSR(mhpmevent9, CSR_MHPMEVENT9) +DECLARE_CSR(mhpmevent10, CSR_MHPMEVENT10) +DECLARE_CSR(mhpmevent11, CSR_MHPMEVENT11) +DECLARE_CSR(mhpmevent12, CSR_MHPMEVENT12) +DECLARE_CSR(mhpmevent13, CSR_MHPMEVENT13) +DECLARE_CSR(mhpmevent14, CSR_MHPMEVENT14) +DECLARE_CSR(mhpmevent15, CSR_MHPMEVENT15) +DECLARE_CSR(mhpmevent16, CSR_MHPMEVENT16) +DECLARE_CSR(mhpmevent17, CSR_MHPMEVENT17) +DECLARE_CSR(mhpmevent18, CSR_MHPMEVENT18) +DECLARE_CSR(mhpmevent19, CSR_MHPMEVENT19) +DECLARE_CSR(mhpmevent20, CSR_MHPMEVENT20) +DECLARE_CSR(mhpmevent21, CSR_MHPMEVENT21) +DECLARE_CSR(mhpmevent22, CSR_MHPMEVENT22) +DECLARE_CSR(mhpmevent23, CSR_MHPMEVENT23) +DECLARE_CSR(mhpmevent24, CSR_MHPMEVENT24) +DECLARE_CSR(mhpmevent25, CSR_MHPMEVENT25) +DECLARE_CSR(mhpmevent26, CSR_MHPMEVENT26) +DECLARE_CSR(mhpmevent27, CSR_MHPMEVENT27) +DECLARE_CSR(mhpmevent28, CSR_MHPMEVENT28) +DECLARE_CSR(mhpmevent29, CSR_MHPMEVENT29) +DECLARE_CSR(mhpmevent30, CSR_MHPMEVENT30) +DECLARE_CSR(mhpmevent31, CSR_MHPMEVENT31) +DECLARE_CSR(mvendorid, CSR_MVENDORID) +DECLARE_CSR(marchid, CSR_MARCHID) +DECLARE_CSR(mimpid, CSR_MIMPID) +DECLARE_CSR(mhartid, CSR_MHARTID) +DECLARE_CSR(mconfigptr, CSR_MCONFIGPTR) +DECLARE_CSR(stimecmph, CSR_STIMECMPH) +DECLARE_CSR(vstimecmph, CSR_VSTIMECMPH) +DECLARE_CSR(htimedeltah, CSR_HTIMEDELTAH) +DECLARE_CSR(henvcfgh, CSR_HENVCFGH) +DECLARE_CSR(hstateen0h, CSR_HSTATEEN0H) +DECLARE_CSR(hstateen1h, CSR_HSTATEEN1H) +DECLARE_CSR(hstateen2h, CSR_HSTATEEN2H) +DECLARE_CSR(hstateen3h, CSR_HSTATEEN3H) +DECLARE_CSR(cycleh, CSR_CYCLEH) +DECLARE_CSR(timeh, CSR_TIMEH) +DECLARE_CSR(instreth, CSR_INSTRETH) +DECLARE_CSR(hpmcounter3h, CSR_HPMCOUNTER3H) +DECLARE_CSR(hpmcounter4h, CSR_HPMCOUNTER4H) +DECLARE_CSR(hpmcounter5h, CSR_HPMCOUNTER5H) +DECLARE_CSR(hpmcounter6h, CSR_HPMCOUNTER6H) +DECLARE_CSR(hpmcounter7h, CSR_HPMCOUNTER7H) +DECLARE_CSR(hpmcounter8h, CSR_HPMCOUNTER8H) +DECLARE_CSR(hpmcounter9h, CSR_HPMCOUNTER9H) +DECLARE_CSR(hpmcounter10h, CSR_HPMCOUNTER10H) +DECLARE_CSR(hpmcounter11h, CSR_HPMCOUNTER11H) +DECLARE_CSR(hpmcounter12h, CSR_HPMCOUNTER12H) +DECLARE_CSR(hpmcounter13h, CSR_HPMCOUNTER13H) +DECLARE_CSR(hpmcounter14h, CSR_HPMCOUNTER14H) +DECLARE_CSR(hpmcounter15h, CSR_HPMCOUNTER15H) +DECLARE_CSR(hpmcounter16h, CSR_HPMCOUNTER16H) +DECLARE_CSR(hpmcounter17h, CSR_HPMCOUNTER17H) +DECLARE_CSR(hpmcounter18h, CSR_HPMCOUNTER18H) +DECLARE_CSR(hpmcounter19h, CSR_HPMCOUNTER19H) +DECLARE_CSR(hpmcounter20h, CSR_HPMCOUNTER20H) +DECLARE_CSR(hpmcounter21h, CSR_HPMCOUNTER21H) +DECLARE_CSR(hpmcounter22h, CSR_HPMCOUNTER22H) +DECLARE_CSR(hpmcounter23h, CSR_HPMCOUNTER23H) +DECLARE_CSR(hpmcounter24h, CSR_HPMCOUNTER24H) +DECLARE_CSR(hpmcounter25h, CSR_HPMCOUNTER25H) +DECLARE_CSR(hpmcounter26h, CSR_HPMCOUNTER26H) +DECLARE_CSR(hpmcounter27h, CSR_HPMCOUNTER27H) +DECLARE_CSR(hpmcounter28h, CSR_HPMCOUNTER28H) +DECLARE_CSR(hpmcounter29h, CSR_HPMCOUNTER29H) +DECLARE_CSR(hpmcounter30h, CSR_HPMCOUNTER30H) +DECLARE_CSR(hpmcounter31h, CSR_HPMCOUNTER31H) +DECLARE_CSR(mstatush, CSR_MSTATUSH) +DECLARE_CSR(menvcfgh, CSR_MENVCFGH) +DECLARE_CSR(mstateen0h, CSR_MSTATEEN0H) +DECLARE_CSR(mstateen1h, CSR_MSTATEEN1H) +DECLARE_CSR(mstateen2h, CSR_MSTATEEN2H) +DECLARE_CSR(mstateen3h, CSR_MSTATEEN3H) +DECLARE_CSR(mhpmevent3h, CSR_MHPMEVENT3H) +DECLARE_CSR(mhpmevent4h, CSR_MHPMEVENT4H) +DECLARE_CSR(mhpmevent5h, CSR_MHPMEVENT5H) +DECLARE_CSR(mhpmevent6h, CSR_MHPMEVENT6H) +DECLARE_CSR(mhpmevent7h, CSR_MHPMEVENT7H) +DECLARE_CSR(mhpmevent8h, CSR_MHPMEVENT8H) +DECLARE_CSR(mhpmevent9h, CSR_MHPMEVENT9H) +DECLARE_CSR(mhpmevent10h, CSR_MHPMEVENT10H) +DECLARE_CSR(mhpmevent11h, CSR_MHPMEVENT11H) +DECLARE_CSR(mhpmevent12h, CSR_MHPMEVENT12H) +DECLARE_CSR(mhpmevent13h, CSR_MHPMEVENT13H) +DECLARE_CSR(mhpmevent14h, CSR_MHPMEVENT14H) +DECLARE_CSR(mhpmevent15h, CSR_MHPMEVENT15H) +DECLARE_CSR(mhpmevent16h, CSR_MHPMEVENT16H) +DECLARE_CSR(mhpmevent17h, CSR_MHPMEVENT17H) +DECLARE_CSR(mhpmevent18h, CSR_MHPMEVENT18H) +DECLARE_CSR(mhpmevent19h, CSR_MHPMEVENT19H) +DECLARE_CSR(mhpmevent20h, CSR_MHPMEVENT20H) +DECLARE_CSR(mhpmevent21h, CSR_MHPMEVENT21H) +DECLARE_CSR(mhpmevent22h, CSR_MHPMEVENT22H) +DECLARE_CSR(mhpmevent23h, CSR_MHPMEVENT23H) +DECLARE_CSR(mhpmevent24h, CSR_MHPMEVENT24H) +DECLARE_CSR(mhpmevent25h, CSR_MHPMEVENT25H) +DECLARE_CSR(mhpmevent26h, CSR_MHPMEVENT26H) +DECLARE_CSR(mhpmevent27h, CSR_MHPMEVENT27H) +DECLARE_CSR(mhpmevent28h, CSR_MHPMEVENT28H) +DECLARE_CSR(mhpmevent29h, CSR_MHPMEVENT29H) +DECLARE_CSR(mhpmevent30h, CSR_MHPMEVENT30H) +DECLARE_CSR(mhpmevent31h, CSR_MHPMEVENT31H) +DECLARE_CSR(mseccfgh, CSR_MSECCFGH) +DECLARE_CSR(mcycleh, CSR_MCYCLEH) +DECLARE_CSR(minstreth, CSR_MINSTRETH) +DECLARE_CSR(mhpmcounter3h, CSR_MHPMCOUNTER3H) +DECLARE_CSR(mhpmcounter4h, CSR_MHPMCOUNTER4H) +DECLARE_CSR(mhpmcounter5h, CSR_MHPMCOUNTER5H) +DECLARE_CSR(mhpmcounter6h, CSR_MHPMCOUNTER6H) +DECLARE_CSR(mhpmcounter7h, CSR_MHPMCOUNTER7H) +DECLARE_CSR(mhpmcounter8h, CSR_MHPMCOUNTER8H) +DECLARE_CSR(mhpmcounter9h, CSR_MHPMCOUNTER9H) +DECLARE_CSR(mhpmcounter10h, CSR_MHPMCOUNTER10H) +DECLARE_CSR(mhpmcounter11h, CSR_MHPMCOUNTER11H) +DECLARE_CSR(mhpmcounter12h, CSR_MHPMCOUNTER12H) +DECLARE_CSR(mhpmcounter13h, CSR_MHPMCOUNTER13H) +DECLARE_CSR(mhpmcounter14h, CSR_MHPMCOUNTER14H) +DECLARE_CSR(mhpmcounter15h, CSR_MHPMCOUNTER15H) +DECLARE_CSR(mhpmcounter16h, CSR_MHPMCOUNTER16H) +DECLARE_CSR(mhpmcounter17h, CSR_MHPMCOUNTER17H) +DECLARE_CSR(mhpmcounter18h, CSR_MHPMCOUNTER18H) +DECLARE_CSR(mhpmcounter19h, CSR_MHPMCOUNTER19H) +DECLARE_CSR(mhpmcounter20h, CSR_MHPMCOUNTER20H) +DECLARE_CSR(mhpmcounter21h, CSR_MHPMCOUNTER21H) +DECLARE_CSR(mhpmcounter22h, CSR_MHPMCOUNTER22H) +DECLARE_CSR(mhpmcounter23h, CSR_MHPMCOUNTER23H) +DECLARE_CSR(mhpmcounter24h, CSR_MHPMCOUNTER24H) +DECLARE_CSR(mhpmcounter25h, CSR_MHPMCOUNTER25H) +DECLARE_CSR(mhpmcounter26h, CSR_MHPMCOUNTER26H) +DECLARE_CSR(mhpmcounter27h, CSR_MHPMCOUNTER27H) +DECLARE_CSR(mhpmcounter28h, CSR_MHPMCOUNTER28H) +DECLARE_CSR(mhpmcounter29h, CSR_MHPMCOUNTER29H) +DECLARE_CSR(mhpmcounter30h, CSR_MHPMCOUNTER30H) +DECLARE_CSR(mhpmcounter31h, CSR_MHPMCOUNTER31H) +#endif +#ifdef DECLARE_CAUSE +DECLARE_CAUSE("misaligned fetch", CAUSE_MISALIGNED_FETCH) +DECLARE_CAUSE("fetch access", CAUSE_FETCH_ACCESS) +DECLARE_CAUSE("illegal instruction", CAUSE_ILLEGAL_INSTRUCTION) +DECLARE_CAUSE("breakpoint", CAUSE_BREAKPOINT) +DECLARE_CAUSE("misaligned load", CAUSE_MISALIGNED_LOAD) +DECLARE_CAUSE("load access", CAUSE_LOAD_ACCESS) +DECLARE_CAUSE("misaligned store", CAUSE_MISALIGNED_STORE) +DECLARE_CAUSE("store access", CAUSE_STORE_ACCESS) +DECLARE_CAUSE("user_ecall", CAUSE_USER_ECALL) +DECLARE_CAUSE("supervisor_ecall", CAUSE_SUPERVISOR_ECALL) +DECLARE_CAUSE("virtual_supervisor_ecall", CAUSE_VIRTUAL_SUPERVISOR_ECALL) +DECLARE_CAUSE("machine_ecall", CAUSE_MACHINE_ECALL) +DECLARE_CAUSE("fetch page fault", CAUSE_FETCH_PAGE_FAULT) +DECLARE_CAUSE("load page fault", CAUSE_LOAD_PAGE_FAULT) +DECLARE_CAUSE("store page fault", CAUSE_STORE_PAGE_FAULT) +DECLARE_CAUSE("fetch guest page fault", CAUSE_FETCH_GUEST_PAGE_FAULT) +DECLARE_CAUSE("load guest page fault", CAUSE_LOAD_GUEST_PAGE_FAULT) +DECLARE_CAUSE("virtual instruction", CAUSE_VIRTUAL_INSTRUCTION) +DECLARE_CAUSE("store guest page fault", CAUSE_STORE_GUEST_PAGE_FAULT) +#endif diff --git a/optee_os/core/arch/riscv/include/kernel/arch_scall.h b/optee_os/core/arch/riscv/include/kernel/arch_scall.h new file mode 100644 index 0000000..8d898d5 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/arch_scall.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#ifndef __KERNEL_ARCH_SCALL_H +#define __KERNEL_ARCH_SCALL_H + +#include +#include +#include + +static inline void scall_get_max_args(struct thread_scall_regs *regs, + size_t *scn, size_t *max_args) +{ + *scn = regs->t0; + *max_args = regs->t1; +} + +static inline void scall_set_retval(struct thread_scall_regs *regs, + uint32_t ret_val) +{ + regs->a0 = ret_val; +} + +static inline void scall_set_sys_return_regs(struct thread_scall_regs *regs, + bool panic, uint32_t panic_code) +{ + regs->a1 = panic; + regs->a2 = panic_code; +} + +#endif /*__KERNEL_ARCH_SCALL_H*/ diff --git a/optee_os/core/arch/riscv/include/kernel/cache_helpers_arch.h b/optee_os/core/arch/riscv/include/kernel/cache_helpers_arch.h new file mode 100644 index 0000000..5f5810f --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/cache_helpers_arch.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef __KERNEL_CACHE_HELPERS_ARCH_H +#define __KERNEL_CACHE_HELPERS_ARCH_H + +#endif /*__KERNEL_CACHE_HELPERS_ARCH_H*/ diff --git a/optee_os/core/arch/riscv/include/kernel/clint.h b/optee_os/core/arch/riscv/include/kernel/clint.h new file mode 100644 index 0000000..56efa25 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/clint.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#ifndef __KERNEL_CLINT_H +#define __KERNEL_CLINT_H + +#include +#include +#include +#include + +#ifdef CFG_RISCV_M_MODE + +/* Machine software-interrupt pending register for a specific hart */ +#define CLINT_MSIP(hart) (CLINT_BASE + (4 * (hart))) +/* Register for setting mtimecmp for a specific hart */ +#define CLINT_MTIMECMP(hart)(CLINT_BASE + 0x4000 + (8 * (hart))) +/* Number of cycles counted from the RTCCLK input */ +#define CLINT_MTIME (CLINT_BASE + 0xbff8) + +static inline void clint_ipi_send(unsigned long hart) +{ + assert(hart < CFG_TEE_CORE_NB_CORE); + io_write32(CLINT_MSIP(hart), 1); +} + +static inline void clint_ipi_clear(unsigned long hart) +{ + assert(hart < CFG_TEE_CORE_NB_CORE); + io_write32(CLINT_MSIP(hart), 0); +} + +static inline void clint_set_mtimecmp(uint64_t timecmp) +{ + /* Each hart has a separate source of timer interrupts */ + io_write64(CLINT_MTIMECMP(get_core_pos()), timecmp); +} + +static inline uint64_t clint_get_mtimecmp(void) +{ + return io_read64(CLINT_MTIMECMP(get_core_pos())); +} + +static inline uint64_t clint_get_mtime(void) +{ + return io_read64(CLINT_MTIME); +} + +static inline void clint_set_mtime(uint64_t mtime) +{ + io_write64(CLINT_MTIME, mtime); +} + +#endif /* CFG_RISCV_M_MODE */ +#endif /* __KERNEL_CLINT_H */ diff --git a/optee_os/core/arch/riscv/include/kernel/delay_arch.h b/optee_os/core/arch/riscv/include/kernel/delay_arch.h new file mode 100644 index 0000000..2ddec10 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/delay_arch.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#ifndef __KERNEL_DELAY_ARCH_H +#define __KERNEL_DELAY_ARCH_H + +#include +#include +#include +#include + +static inline uint64_t timeout_init_us(uint32_t us) +{ + return read_time() + ((uint64_t)us * read_cntfrq()) / 1000000ULL; +} + +static inline bool timeout_elapsed(uint64_t expire) +{ + return read_time() > expire; +} + +#endif /*__KERNEL_DELAY_ARCH_H*/ diff --git a/optee_os/core/arch/riscv/include/kernel/misc_arch.h b/optee_os/core/arch/riscv/include/kernel/misc_arch.h new file mode 100644 index 0000000..208e358 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/misc_arch.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef KERNEL_MISC_ARCH_H +#define KERNEL_MISC_ARCH_H + +#endif /*KERNEL_MISC_ARCH_H*/ diff --git a/optee_os/core/arch/riscv/include/kernel/secure_partition.h b/optee_os/core/arch/riscv/include/kernel/secure_partition.h new file mode 100644 index 0000000..c97a31a --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/secure_partition.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef __KERNEL_SECURE_PARTITION_H +#define __KERNEL_SECURE_PARTITION_H + +#include +#include +#include +#include + +struct sp_ctx { + struct user_mode_ctx uctx; + struct ts_ctx ts_ctx; +}; + +static inline bool is_sp_ctx(struct ts_ctx *ctx __unused) +{ + return false; +} + +static inline struct sp_session *__noprof +to_sp_session(struct ts_session *sess __unused) +{ + assert(is_sp_ctx(sess->ctx)); + return NULL; +} + +static inline struct sp_ctx *to_sp_ctx(struct ts_ctx *ctx __unused) +{ + assert(is_sp_ctx(ctx)); + return NULL; +} + +#endif /* __KERNEL_SECURE_PARTITION_H */ diff --git a/optee_os/core/arch/riscv/include/kernel/stmm_sp.h b/optee_os/core/arch/riscv/include/kernel/stmm_sp.h new file mode 100644 index 0000000..9b5b09d --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/stmm_sp.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef __KERNEL_STMM_SP_H +#define __KERNEL_STMM_SP_H + +#include +#include +#include +#include +#include + +struct stmm_ctx { + struct user_mode_ctx uctx; + struct tee_ta_ctx ta_ctx; +}; + +static inline bool is_stmm_ctx(struct ts_ctx *ctx __unused) +{ + return false; +} + +static inline struct stmm_ctx *to_stmm_ctx(struct ts_ctx *ctx __unused) +{ + assert(is_stmm_ctx(ctx)); + return NULL; +} + +static inline TEE_Result +stmm_init_session(const TEE_UUID *uuid __unused, + struct tee_ta_session *s __unused) +{ + return TEE_ERROR_ITEM_NOT_FOUND; +} + +static inline const TEE_UUID *stmm_get_uuid(void) { return NULL; } + +#endif /*__KERNEL_STMM_SP_H*/ diff --git a/optee_os/core/arch/riscv/include/kernel/tee_l2cc_mutex.h b/optee_os/core/arch/riscv/include/kernel/tee_l2cc_mutex.h new file mode 100644 index 0000000..12b9147 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/tee_l2cc_mutex.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ +#ifndef TEE_L2CC_MUTEX_H +#define TEE_L2CC_MUTEX_H + +#endif /* TEE_L2CC_MUTEX_H */ diff --git a/optee_os/core/arch/riscv/include/kernel/thread_arch.h b/optee_os/core/arch/riscv/include/kernel/thread_arch.h new file mode 100644 index 0000000..e150fa1 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/thread_arch.h @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#ifndef __KERNEL_THREAD_ARCH_H +#define __KERNEL_THREAD_ARCH_H + +#ifndef __ASSEMBLER__ +#include +#include +#include +#endif + +#ifndef __ASSEMBLER__ + +#define THREAD_CORE_LOCAL_ALIGNED __aligned(16) + +struct thread_pauth_keys { +}; + +struct thread_core_local { + unsigned long x[4]; + uint32_t hart_id; + vaddr_t tmp_stack_va_end; + short int curr_thread; + uint32_t flags; + vaddr_t abt_stack_va_end; +#ifdef CFG_TEE_CORE_DEBUG + unsigned int locked_count; /* Number of spinlocks held */ +#endif +#ifdef CFG_CORE_DEBUG_CHECK_STACKS + bool stackcheck_recursion; +#endif +} THREAD_CORE_LOCAL_ALIGNED; + +struct thread_user_vfp_state { +}; + +struct thread_abi_args { + unsigned long a0; /* ABI function ID */ + unsigned long a1; /* Parameter */ + unsigned long a2; /* Parameter */ + unsigned long a3; /* Thread ID when returning from RPC */ + unsigned long a4; /* Not used */ + unsigned long a5; /* Not used */ + unsigned long a6; /* Not used */ + unsigned long a7; /* Hypervisor Client ID */ +}; + +struct thread_abort_regs { + unsigned long ra; + unsigned long sp; + unsigned long gp; + unsigned long tp; + unsigned long t0; + unsigned long t1; + unsigned long t2; + unsigned long s0; + unsigned long s1; + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + unsigned long s2; + unsigned long s3; + unsigned long s4; + unsigned long s5; + unsigned long s6; + unsigned long s7; + unsigned long s8; + unsigned long s9; + unsigned long s10; + unsigned long s11; + unsigned long t3; + unsigned long t4; + unsigned long t5; + unsigned long t6; + unsigned long status; + unsigned long cause; + unsigned long epc; + unsigned long tval; + unsigned long satp; +}; + +struct thread_trap_regs { + unsigned long ra; + unsigned long sp; + unsigned long gp; + unsigned long tp; + unsigned long t0; + unsigned long t1; + unsigned long t2; + unsigned long s0; + unsigned long s1; + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + unsigned long s2; + unsigned long s3; + unsigned long s4; + unsigned long s5; + unsigned long s6; + unsigned long s7; + unsigned long s8; + unsigned long s9; + unsigned long s10; + unsigned long s11; + unsigned long t3; + unsigned long t4; + unsigned long t5; + unsigned long t6; + unsigned long epc; + unsigned long status; + unsigned long satp; +} __aligned(16); + +struct thread_scall_regs { + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + unsigned long t0; + unsigned long t1; + unsigned long ra; + unsigned long sp; + unsigned long status; +} __aligned(16); + +struct thread_ctx_regs { + unsigned long ra; + unsigned long sp; + unsigned long gp; + unsigned long tp; + unsigned long t0; + unsigned long t1; + unsigned long t2; + unsigned long s0; + unsigned long s1; + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + unsigned long s2; + unsigned long s3; + unsigned long s4; + unsigned long s5; + unsigned long s6; + unsigned long s7; + unsigned long s8; + unsigned long s9; + unsigned long s10; + unsigned long s11; + unsigned long t3; + unsigned long t4; + unsigned long t5; + unsigned long t6; + unsigned long status; +}; + +struct user_mode_ctx; + +/* + * These flags should vary according to the privilege mode selected + * to run OP-TEE core on (M/HS/S). For now default to S-Mode. + */ + +#define CSR_XIE_SIE BIT64(IRQ_XSOFT) +#define CSR_XIE_TIE BIT64(IRQ_XTIMER) +#define CSR_XIE_EIE BIT64(IRQ_XEXT) + +#define THREAD_EXCP_FOREIGN_INTR CSR_XIE_EIE +#define THREAD_EXCP_NATIVE_INTR (CSR_XIE_SIE | CSR_XIE_TIE) +#define THREAD_EXCP_ALL (THREAD_EXCP_FOREIGN_INTR |\ + THREAD_EXCP_NATIVE_INTR) + +#ifdef CFG_WITH_VFP +uint32_t thread_kernel_enable_vfp(void); +void thread_kernel_disable_vfp(uint32_t state); +void thread_kernel_save_vfp(void); +void thread_kernel_restore_vfp(void); +void thread_user_enable_vfp(struct thread_user_vfp_state *uvfp); +#else /*CFG_WITH_VFP*/ +static inline void thread_kernel_save_vfp(void) +{ +} + +static inline void thread_kernel_restore_vfp(void) +{ +} +#endif /*CFG_WITH_VFP*/ +#ifdef CFG_WITH_VFP +void thread_user_save_vfp(void); +#else +static inline void thread_user_save_vfp(void) +{ +} +#endif +#ifdef CFG_WITH_VFP +void thread_user_clear_vfp(struct user_mode_ctx *uctx); +#else +static inline void thread_user_clear_vfp(struct user_mode_ctx *uctx __unused) +{ +} +#endif + +vaddr_t thread_get_saved_thread_sp(void); + +static inline void thread_get_user_kcode(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz) +{ + *mobj = NULL; + *offset = 0; + *va = 0; + *sz = 0; +} + +static inline void thread_get_user_kdata(struct mobj **mobj, size_t *offset, + vaddr_t *va, size_t *sz) +{ + *mobj = NULL; + *offset = 0; + *va = 0; + *sz = 0; +} + +bool thread_disable_prealloc_rpc_cache(uint64_t *cookie); +bool thread_enable_prealloc_rpc_cache(void); + +#endif /*__ASSEMBLER__*/ +#endif /*__KERNEL_THREAD_ARCH_H*/ diff --git a/optee_os/core/arch/riscv/include/kernel/thread_private_arch.h b/optee_os/core/arch/riscv/include/kernel/thread_private_arch.h new file mode 100644 index 0000000..96ea6dc --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/thread_private_arch.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#ifndef __KERNEL_THREAD_PRIVATE_ARCH_H +#define __KERNEL_THREAD_PRIVATE_ARCH_H + +#ifndef __ASSEMBLER__ + +#include + +#define STACK_TMP_OFFS 0 + +#if defined(__clang__) && !defined(__OPTIMIZE_SIZE__) +#define STACK_TMP_SIZE (4096 + STACK_TMP_OFFS + CFG_STACK_TMP_EXTRA) +#else +#define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS + CFG_STACK_TMP_EXTRA) +#endif + +#define STACK_THREAD_SIZE (8192 + CFG_STACK_THREAD_EXTRA) + +#if TRACE_LEVEL > 0 +#define STACK_ABT_SIZE 3072 +#else +#define STACK_ABT_SIZE 1024 +#endif + +#define STACK_CHECK_EXTRA 0 + +#define THREAD_RPC_NUM_ARGS 4 + +#define TRAP_MODE_KERNEL 0 +#define TRAP_MODE_USER 1 + +struct thread_user_mode_rec { + unsigned long ctx_regs_ptr; + unsigned long exit_status0_ptr; + unsigned long exit_status1_ptr; + unsigned long pad; + /* + * x[] is used to save registers for user/kernel context-switching + * 0-3: ra-tp + * 4-6: s0-s1 + * 6-15: s2-s11 + */ + unsigned long x[16]; +}; + +extern long thread_user_kcode_offset; + +void thread_trap_handler(long cause, unsigned long epc, + struct thread_trap_regs *regs, + bool user); +/* + * Initializes TVEC for current hart. Called by thread_init_per_cpu() + */ +void thread_init_tvec(void); +void thread_trap_vect(void); +void thread_trap_vect_end(void); + +void thread_return_to_ree(unsigned long arg0, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5); + +void __panic_at_abi_return(void); + +/* + * Assembly function as the first function in a thread. Handles a stdcall, + * a0-a3 holds the parameters. Hands over to __thread_std_abi_entry() when + * everything is set up and does some post processing once + * __thread_std_abi_entry() returns. + */ +void thread_std_abi_entry(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5); +uint32_t __thread_std_abi_entry(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5); +/* + * Called from assembly only, vector_fast_abi_entry(). Handles a fast ABI + * by dispatching it to the registered fast ABI handler. + */ +void thread_handle_fast_abi(struct thread_abi_args *args); + +/* + * Called from assembly only, vector_std_abi_entry(). Handles a std ABI by + * dispatching it to the registered std ABI handler. + */ +uint32_t thread_handle_std_abi(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6, uint32_t a7); + +/* + * Private functions made available for thread_rv.S + */ +int thread_state_suspend(uint32_t flags, unsigned long status, vaddr_t pc); +void thread_resume(struct thread_ctx_regs *regs); +uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, + uint32_t *exit_status0, + uint32_t *exit_status1); +void *thread_get_tmp_sp(void); +void thread_state_free(void); +struct thread_ctx_regs *thread_get_ctx_regs(void); +void thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5); +void thread_resume_from_rpc(uint32_t thread_id, uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3); +void thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]); +void thread_scall_handler(struct thread_scall_regs *regs); +void thread_exit_user_mode(unsigned long a0, unsigned long a1, + unsigned long a2, unsigned long a3, + unsigned long sp, unsigned long pc, + unsigned long status); + +#endif /*__ASSEMBLER__*/ +#endif /*__KERNEL_THREAD_PRIVATE_ARCH_H*/ diff --git a/optee_os/core/arch/riscv/include/kernel/tlb_helpers.h b/optee_os/core/arch/riscv/include/kernel/tlb_helpers.h new file mode 100644 index 0000000..ad82024 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/tlb_helpers.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef TLB_HELPERS_H +#define TLB_HELPERS_H + +#ifndef __ASSEMBLER__ + +void tlbi_all(void); +void tlbi_va_allasid(vaddr_t va); +void tlbi_asid(unsigned long asid); +void tlbi_va_asid(vaddr_t va, uint32_t asid); + +#endif /*!__ASSEMBLER__*/ + +#endif /* TLB_HELPERS_H */ diff --git a/optee_os/core/arch/riscv/include/kernel/user_access_arch.h b/optee_os/core/arch/riscv/include/kernel/user_access_arch.h new file mode 100644 index 0000000..2b9d563 --- /dev/null +++ b/optee_os/core/arch/riscv/include/kernel/user_access_arch.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright (c) 2023, Amazon.com Inc. or its affiliates. All rights Reserved. + */ + +#ifndef __KERNEL_USER_ACCESS_ARCH_H +#define __KERNEL_USER_ACCESS_ARCH_H + +#include + +#ifdef CFG_PAN +/* Enter a section where user mode access is temporarily enabled. */ +static inline void enter_user_access(void) +{ + set_csr(CSR_XSTATUS, CSR_XSTATUS_SUM); +} + +/* Exit from the section where user mode access was temporarily enabled. */ +static inline void exit_user_access(void) +{ + clear_csr(CSR_XSTATUS, CSR_XSTATUS_SUM); +} +#else +static inline void enter_user_access(void) {} +static inline void exit_user_access(void) {} +#endif /* CFG_PAN */ + +#endif /* __KERNEL_USER_ACCESS_ARCH_H */ diff --git a/optee_os/core/arch/riscv/include/mm/core_mmu_arch.h b/optee_os/core/arch/riscv/include/mm/core_mmu_arch.h new file mode 100644 index 0000000..1284d47 --- /dev/null +++ b/optee_os/core/arch/riscv/include/mm/core_mmu_arch.h @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ +#ifndef __CORE_MMU_ARCH_H +#define __CORE_MMU_ARCH_H + +#ifndef __ASSEMBLER__ +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include + +#ifdef TRUSTED_DRAM_BASE +#error TRUSTED_DRAM_BASE is already defined +#endif +#define TRUSTED_DRAM_BASE TDDRAM_BASE +#define TRUSTED_DRAM_SIZE TDDRAM_SIZE + +/* MMU defines */ +#ifdef CFG_RISCV_MMU_MODE +#define RISCV_MMU_MODE CFG_RISCV_MMU_MODE +#else +#ifdef RV64 +#define RISCV_MMU_MODE U(39) +#define RISCV_MMU_ASID_WIDTH 16 +#else +#define RISCV_MMU_MODE U(32) +#define RISCV_MMU_ASID_WIDTH 9 +#endif +#endif + +#if RISCV_MMU_MODE == 48 /*Sv48*/ +#define RISCV_SATP_MODE SATP_MODE_SV48 +#define RISCV_SATP_MODE_SHIFT U(60) +#define RISCV_SATP_ASID_SHIFT U(44) +#define RISCV_SATP_ASID_SIZE U(16) +#define RISCV_SATP_ASID_MASK 0x0FFFF +#define RISCV_MMU_PA_WIDTH U(56) +#define RISCV_MMU_VA_WIDTH U(48) +#elif RISCV_MMU_MODE == 39 /*Sv39*/ +#define RISCV_SATP_MODE SATP_MODE_SV39 +#define RISCV_SATP_ASID_SHIFT U(44) +#define RISCV_SATP_ASID_SIZE U(16) +#define RISCV_SATP_ASID_MASK 0x0FFFF +#define RISCV_MMU_PA_WIDTH U(56) +#define RISCV_MMU_VA_WIDTH U(39) +#define RISCV_SATP_MODE_SHIFT 60 +#elif RISCV_MMU_MODE == 32 /*Sv32*/ +#define RISCV_SATP_MODE SATP_MODE_SV32 +#define RISCV_SATP_MODE_SHIFT U(31) +#define RISCV_SATP_ASID_SHIFT U(22) +#define RISCV_SATP_ASID_SIZE U(9) +#define RISCV_SATP_ASID_MASK 0x01FF +#define RISCV_MMU_PA_WIDTH U(32) +#define RISCV_MMU_VA_WIDTH U(32) +#else +#error unknown or unsupported mmu mode +#endif + +#define RISCV_PTES_PER_PT BIT(RISCV_PGLEVEL_BITS) +#define RISCV_PGLEVELS ((RISCV_MMU_VA_WIDTH - RISCV_PGSHIFT) / \ + RISCV_PGLEVEL_BITS) +#define RISCV_MMU_VPN_MASK (BIT(RISCV_PGLEVEL_BITS) - 1) +#define RISCV_MMU_MAX_PGTS 16 + +#define SMALL_PAGE_SHIFT U(12) + +/* + * Level 0, shift = 12, 4 KiB pages + * Level 1, shift = 21, 2 MiB pages (4 MiB pages in Sv32) + * Level 2, shift = 30, 1 GiB pages + * Level 3, shift = 39, 512 GiB pages + * Level 4, shift = 48, 256 TiB pages + */ +#define CORE_MMU_SHIFT_OF_LEVEL(level) (RISCV_PGLEVEL_BITS * \ + (level) + \ + RISCV_PGSHIFT) + +#define CORE_MMU_USER_CODE_SHIFT SMALL_PAGE_SHIFT +#define CORE_MMU_USER_PARAM_SHIFT SMALL_PAGE_SHIFT + +/* + * In all MMU modes, the CORE_MMU_PGDIR_LEVEL is always 0: + * Sv32: 4 KiB, 4 MiB + * Sv39: 4 KiB, 2 MiB, 1 GiB + * Sv48: 4 KiB, 2 MiB, 1 GiB, 512 GiB + * Sv57: 4 KiB, 2 MiB, 1 GiB, 512 GiB, 256 TiB + */ +#define CORE_MMU_PGDIR_LEVEL U(0) +#define CORE_MMU_PGDIR_SHIFT \ + CORE_MMU_SHIFT_OF_LEVEL(CORE_MMU_PGDIR_LEVEL + 1) + +#define CORE_MMU_BASE_TABLE_LEVEL (RISCV_PGLEVELS - 1) +#define CORE_MMU_BASE_TABLE_SHIFT \ + CORE_MMU_SHIFT_OF_LEVEL(CORE_MMU_BASE_TABLE_LEVEL) + +#ifndef __ASSEMBLER__ + +struct core_mmu_config { + unsigned long satp; + uint32_t map_offset; +}; + +struct core_mmu_user_map { + unsigned long user_map; + uint32_t asid; +}; + +/* Cache maintenance operation type */ +enum cache_op { + DCACHE_CLEAN, + DCACHE_AREA_CLEAN, + DCACHE_INVALIDATE, + DCACHE_AREA_INVALIDATE, + ICACHE_INVALIDATE, + ICACHE_AREA_INVALIDATE, + DCACHE_CLEAN_INV, + DCACHE_AREA_CLEAN_INV, +}; + +static inline void core_mmu_table_write_barrier(void) +{ + /* Invoke memory barrier */ + mb(); +} + +TEE_Result cache_op_inner(enum cache_op op, void *va, size_t len); + +static inline bool core_mmu_check_max_pa(paddr_t pa) +{ + return pa <= (BIT64(RISCV_MMU_PA_WIDTH) - 1); +} + +static inline unsigned int core_mmu_get_va_width(void) +{ + return RISCV_MMU_VA_WIDTH; +} + +static inline bool core_mmu_level_in_range(unsigned int level) +{ + return level <= CORE_MMU_BASE_TABLE_LEVEL; +} +#endif /*__ASSEMBLER__*/ + +#endif /* __CORE_MMU_ARCH_H */ diff --git a/optee_os/core/arch/riscv/include/mm/generic_ram_layout.h b/optee_os/core/arch/riscv/include/mm/generic_ram_layout.h new file mode 100644 index 0000000..4903ade --- /dev/null +++ b/optee_os/core/arch/riscv/include/mm/generic_ram_layout.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + * Copyright 2022 NXP + */ + +#ifndef __MM_GENERIC_RAM_LAYOUT_H +#define __MM_GENERIC_RAM_LAYOUT_H + +#include + +/* + * Generic RAM layout configuration directives + * + * Mandatory directives: + * CFG_TDDRAM_START + * CFG_TDDRAM_SIZE + * CFG_SHMEM_START + * CFG_SHMEM_SIZE + * + * Optional directives: + * CFG_TEE_LOAD_ADDR If defined sets TEE_LOAD_ADDR. If not, TEE_LOAD_ADDR + * is set by the platform or defaults to TEE_RAM_START. + * CFG_TEE_RAM_VA_SIZE Some platforms may have specific needs + * + * Optional directives when pager is enabled: + * CFG_TDSRAM_START If no set, emulated at CFG_TDDRAM_START + * CFG_TDSRAM_SIZE Default to CFG_CORE_TDSRAM_EMUL_SIZE + * + * Optional directive when CFG_SECURE_DATA_PATH is enabled: + * CFG_TEE_SDP_MEM_SIZE If CFG_TEE_SDP_MEM_BASE is not defined, SDP test + * memory byte size can be set by CFG_TEE_SDP_MEM_SIZE. + * + * This header file produces the following generic macros upon the mandatory + * and optional configuration directives listed above: + * + * TEE_RAM_START TEE core RAM physical base address + * TEE_RAM_VA_SIZE TEE core virtual memory address range size + * TEE_RAM_PH_SIZE TEE core physical RAM byte size + * TA_RAM_START TA contexts/pagestore RAM physical base address + * TA_RAM_SIZE TA contexts/pagestore RAM byte size + * TEE_SHMEM_START Non-secure static shared memory physical base address + * TEE_SHMEM_SIZE Non-secure static shared memory byte size + * + * TDDRAM_BASE Main/external secure RAM base address + * TDDRAM_SIZE Main/external secure RAM byte size + * TDSRAM_BASE On-chip secure RAM base address, required by pager. + * TDSRAM_SIZE On-chip secure RAM byte size, required by pager. + * + * TEE_LOAD_ADDR Only defined here if CFG_TEE_LOAD_ADDR is defined. + * Otherwise we expect the platform_config.h to define it + * unless which LEE_LOAD_ADDR defaults to TEE_RAM_START. + * + * TEE_RAM_VA_SIZE Set to CFG_TEE_RAM_VA_SIZE or defaults to + * CORE_MMU_PGDIR_SIZE. + * + * TEE_SDP_TEST_MEM_BASE Define if a SDP memory pool is required and none set. + * Always defined in the inner top (high addresses) + * of CFG_TDDRAM_START/_SIZE. + * TEE_SDP_TEST_MEM_SIZE Set to CFG_TEE_SDP_MEM_SIZE or a default size. + * + * ---------------------------------------------------------------------------- + * TEE RAM layout without CFG_WITH_PAGER + *_ + * +----------------------------------+ <-- CFG_TDDRAM_START + * | TEE core secure RAM (TEE_RAM) | + * +----------------------------------+ + * | Trusted Application RAM (TA_RAM) | + * +----------------------------------+ + * | SDP test memory (optional) | + * +----------------------------------+ <-- CFG_TDDRAM_START + CFG_TDDRAM_SIZE + * + * +----------------------------------+ <-- CFG_SHMEM_START + * | Non-secure static SHM | + * +----------------------------------+ <-- CFG_SHMEM_START + CFG_SHMEM_SIZE + * + * ---------------------------------------------------------------------------- + * TEE RAM layout with CFG_WITH_PAGER=y and undefined CFG_TDSRAM_START/_SIZE + * + * +----------------------------------+ <-- CFG_TDDRAM_START + * | TEE core secure RAM (TEE_RAM) | | | CFG_CORE_TDSRAM_EMUL_SIZE + * +----------------------------------+ --|-' + * | reserved (for kasan) | | TEE_RAM_VA_SIZE + * +----------------------------------+ --' + * | TA RAM / Pagestore (TA_RAM) | + * +----------------------------------+ <---- align with CORE_MMU_PGDIR_SIZE + * +----------------------------------+ <-- + * | SDP test memory (optional) | | CFG_TEE_SDP_MEM_SIZE + * +----------------------------------+ <-+ CFG_TDDRAM_START + CFG_TDDRAM_SIZE + * + * +----------------------------------+ <-- CFG_SHMEM_START + * | Non-secure static SHM | | + * +----------------------------------+ v CFG_SHMEM_SIZE + * + * ---------------------------------------------------------------------------- + * TEE RAM layout with CFG_WITH_PAGER=y and define CFG_TDSRAM_START/_SIZE + * + * +----------------------------------+ <-- CFG_TDSRAM_START + * | TEE core secure RAM (TEE_RAM) | | CFG_TDSRAM_SIZE + * +----------------------------------+ --' + * + * +----------------------------------+ <- CFG_TDDRAM_START + * | TA RAM / Pagestore (TA_RAM) | + * |----------------------------------+ <---- align with CORE_MMU_PGDIR_SIZE + * |----------------------------------+ <-- + * | SDP test memory (optional) | | CFG_TEE_SDP_MEM_SIZE + * +----------------------------------+ <-+ CFG_TDDRAM_START + CFG_TDDRAM_SIZE + * + * +----------------------------------+ <-- CFG_SHMEM_START + * | Non-secure static SHM | | + * +----------------------------------+ v CFG_SHMEM_SIZE + */ + +#ifdef CFG_TEE_LOAD_ADDR +#define TEE_LOAD_ADDR CFG_TEE_LOAD_ADDR +#else +/* Platform specific platform_config.h may set TEE_LOAD_ADDR */ +#endif + +#ifdef CFG_TEE_RAM_VA_SIZE +#define TEE_RAM_VA_SIZE CFG_TEE_RAM_VA_SIZE +#else +#define TEE_RAM_VA_SIZE CORE_MMU_PGDIR_SIZE +#endif + +#ifdef CFG_SHMEM_SIZE +#define TEE_SHMEM_SIZE CFG_SHMEM_SIZE +#endif + +#ifdef CFG_SHMEM_START +#define TEE_SHMEM_START CFG_SHMEM_START +#ifndef CFG_SHMEM_SIZE +#error CFG_SHMEM_START mandates CFG_SHMEM_SIZE +#endif +#endif + +#if defined(CFG_TDSRAM_START) +#define TDSRAM_BASE CFG_TDSRAM_START +#define TDSRAM_SIZE CFG_TDSRAM_SIZE +#endif + +#ifdef CFG_TDDRAM_START +#if !defined(CFG_WITH_PAGER) || defined(CFG_TDSRAM_START) +#define TDDRAM_BASE CFG_TDDRAM_START +#define TDDRAM_SIZE CFG_TDDRAM_SIZE +#else +#define TDSRAM_BASE CFG_TDDRAM_START +#define TDSRAM_SIZE CFG_CORE_TDSRAM_EMUL_SIZE +#define TDDRAM_BASE ROUNDUP(TDSRAM_BASE + TDSRAM_SIZE, \ + TEE_RAM_VA_SIZE) +#define TDDRAM_SIZE (CFG_TDDRAM_START + (CFG_TDDRAM_SIZE - \ + TDDRAM_BASE)) +#endif + +#ifdef CFG_WITH_PAGER +#define TEE_RAM_START TDSRAM_BASE +#define TEE_RAM_PH_SIZE TDSRAM_SIZE +#define TA_RAM_START ROUNDUP(TDDRAM_BASE, CORE_MMU_PGDIR_SIZE) +#else +#define TEE_RAM_START TDDRAM_BASE +#define TEE_RAM_PH_SIZE TEE_RAM_VA_SIZE +#define TA_RAM_START ROUNDUP(TDDRAM_BASE + TEE_RAM_VA_SIZE, \ + SMALL_PAGE_SIZE) +#endif /*CFG_WITH_PAGER*/ + +#define TA_RAM_SIZE (ROUNDDOWN(TDDRAM_BASE + (TDDRAM_SIZE - \ + TEE_SDP_TEST_MEM_SIZE), \ + SMALL_PAGE_SIZE) - TA_RAM_START) +#endif /*CFG_TDDRAM_START*/ + +/* + * Secure data path test memory pool + * - If SDP is disabled, no SDP test memory needed. + * - If SDP is enabled, if CFG_TEE_SDP_MEM_BASE, SDP test pool is not needed. + * - If SDP is enabled and CFG_TEE_SDP_MEM_BASE not defined, a SDP test pool + * is defined at the end of the secure RAM. CFG_TEE_SDP_MEM_SIZE can set + * its size otherwise it defaults to 4MB. + */ +#if !defined(CFG_SECURE_DATA_PATH) || defined(CFG_TEE_SDP_MEM_BASE) +#define TEE_SDP_TEST_MEM_SIZE 0 +#else +#ifdef CFG_TEE_SDP_MEM_SIZE +#define TEE_SDP_TEST_MEM_SIZE CFG_TEE_SDP_MEM_SIZE +#else +#define TEE_SDP_TEST_MEM_SIZE SIZE_4M +#endif +#define TEE_SDP_TEST_MEM_BASE (CFG_TDDRAM_START + (CFG_TDDRAM_SIZE - \ + TEE_SDP_TEST_MEM_SIZE)) +#endif + +#endif /*__MM_GENERIC_RAM_LAYOUT_H*/ diff --git a/optee_os/core/arch/riscv/include/riscv.h b/optee_os/core/arch/riscv/include/riscv.h new file mode 100644 index 0000000..ecaba09 --- /dev/null +++ b/optee_os/core/arch/riscv/include/riscv.h @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#ifndef RISCV_H +#define RISCV_H + +#include +#include +#include +#include +#include + +#define RISCV_XLEN_BITS (__riscv_xlen) +#define RISCV_XLEN_BYTES (__riscv_xlen / 8) + +/* Bind registers to their ABI names */ +#define REG_RA 1 +#define REG_SP 2 +#define REG_GP 3 +#define REG_TP 4 +#define REG_T0 5 +#define REG_T2 7 +#define REG_S0 8 +#define REG_S1 9 +#define REG_A0 10 +#define REG_A1 11 +#define REG_A2 12 +#define REG_A3 13 +#define REG_A5 15 +#define REG_A7 17 +#define REG_S2 18 +#define REG_S11 27 +#define REG_T3 28 +#define REG_T6 31 + +#if defined(CFG_RISCV_M_MODE) +#define CSR_MODE_OFFSET PRV_M +#define XRET mret +#elif defined(CFG_RISCV_S_MODE) +#define CSR_MODE_OFFSET PRV_S +#define XRET sret +#endif + +#define CSR_MODE_BITS SHIFT_U64(CSR_MODE_OFFSET, 8) + +#define CSR_XSTATUS (CSR_MODE_BITS | 0x000) +#define CSR_XIE (CSR_MODE_BITS | 0x004) +#define CSR_XTVEC (CSR_MODE_BITS | 0x005) +#define CSR_XSCRATCH (CSR_MODE_BITS | 0x040) +#define CSR_XEPC (CSR_MODE_BITS | 0x041) +#define CSR_XCAUSE (CSR_MODE_BITS | 0x042) +#define CSR_XTVAL (CSR_MODE_BITS | 0x043) +#define CSR_XIP (CSR_MODE_BITS | 0x044) + +#define IRQ_XSOFT (CSR_MODE_OFFSET + 0) +#define IRQ_XTIMER (CSR_MODE_OFFSET + 4) +#define IRQ_XEXT (CSR_MODE_OFFSET + 8) + +#define CSR_XIE_SIE BIT64(IRQ_XSOFT) +#define CSR_XIE_TIE BIT64(IRQ_XTIMER) +#define CSR_XIE_EIE BIT64(IRQ_XEXT) + +#define CSR_XSTATUS_IE BIT(CSR_MODE_OFFSET + 0) +#define CSR_XSTATUS_PIE BIT(CSR_MODE_OFFSET + 4) +#define CSR_XSTATUS_SPP BIT(8) +#define CSR_XSTATUS_SUM BIT(18) +#define CSR_XSTATUS_MXR BIT(19) + +#ifndef __ASSEMBLER__ + +#define read_csr(csr) \ + ({ \ + unsigned long __tmp; \ + asm volatile ("csrr %0, %1" : "=r"(__tmp) : "i"(csr)); \ + __tmp; \ + }) + +#define write_csr(csr, val) \ + ({ \ + asm volatile ("csrw %0, %1" : : "i"(csr), "rK"(val)); \ + }) + +#define swap_csr(csr, val) \ + ({ \ + unsigned long __tmp; \ + asm volatile ("csrrw %0, %1, %2" \ + : "=r"(__tmp) : "i"(csr), "rK"(val)); \ + __tmp; \ + }) + +#define set_csr(csr, bit) \ + ({ \ + unsigned long __tmp; \ + asm volatile ("csrrs %0, %1, %2" \ + : "=r"(__tmp) : "i"(csr), "rK"(bit)); \ + __tmp; \ + }) + +#define clear_csr(csr, bit) \ + ({ \ + unsigned long __tmp; \ + asm volatile ("csrrc %0, %1, %2" \ + : "=r"(__tmp) : "i"(csr), "rK"(bit)); \ + __tmp; \ + }) + +#define rdtime() read_csr(CSR_TIME) +#define rdcycle() read_csr(CSR_CYCLE) +#define rdinstret() read_csr(CSR_INSTRET) + +static inline __noprof void mb(void) +{ + asm volatile ("fence" : : : "memory"); +} + +static inline __noprof unsigned long read_tp(void) +{ + unsigned long tp; + + asm volatile("mv %0, tp" : "=&r"(tp)); + return tp; +} + +static inline __noprof unsigned long read_fp(void) +{ + unsigned long fp = 0; + + asm volatile ("mv %0, s0" : "=r" (fp)); + + return fp; +} + +static inline __noprof unsigned long read_pc(void) +{ + unsigned long pc = 0; + + asm volatile ("auipc %0, 0" : "=r" (pc)); + + return pc; +} + +static inline __noprof void wfi(void) +{ + asm volatile ("wfi"); +} + +static inline __noprof void flush_tlb(void) +{ + asm volatile("sfence.vma zero, zero"); +} + +static inline __noprof void flush_tlb_entry(unsigned long va) +{ + asm volatile ("sfence.vma %0" : : "r" (va) : "memory"); +} + +/* supervisor address translation and protection */ +static inline __noprof unsigned long read_satp(void) +{ + unsigned long satp; + + asm volatile("csrr %0, satp" : "=r" (satp)); + + return satp; +} + +static inline __noprof void write_satp(unsigned long satp) +{ + asm volatile("csrw satp, %0" : : "r" (satp)); +} + +/* machine trap-vector base-address register */ +static inline __noprof unsigned long read_mtvec(void) +{ + unsigned long mtvec; + + asm volatile("csrr %0, mtvec" : "=r" (mtvec)); + + return mtvec; +} + +static inline __noprof void write_mtvec(unsigned long mtvec) +{ + asm volatile("csrw mtvec, %0" : : "r" (mtvec)); +} + +/* supervisor trap-vector base-address register */ +static inline __noprof unsigned long read_stvec(void) +{ + unsigned long stvec; + + asm volatile("csrr %0, stvec" : "=r" (stvec)); + + return stvec; +} + +static inline __noprof void write_stvec(unsigned long stvec) +{ + asm volatile("csrw stvec, %0" : : "r" (stvec)); +} + +/* machine status register */ +static inline __noprof unsigned long read_mstatus(void) +{ + unsigned long mstatus; + + asm volatile("csrr %0, mstatus" : "=r" (mstatus)); + + return mstatus; +} + +static inline __noprof void write_mstatus(unsigned long mstatus) +{ + asm volatile("csrw mstatus, %0" : : "r" (mstatus)); +} + +/* supervisor status register */ +static inline __noprof unsigned long read_sstatus(void) +{ + unsigned long sstatus; + + asm volatile("csrr %0, sstatus" : "=r" (sstatus)); + + return sstatus; +} + +static inline __noprof void write_sstatus(unsigned long sstatus) +{ + asm volatile("csrw sstatus, %0" : : "r" (sstatus)); +} + +static inline __noprof void set_sstatus(unsigned long sstatus) +{ + unsigned long x; + + asm volatile ("csrrs %0, sstatus, %1" : "=r"(x) : "rK"(sstatus)); +} + +/* machine exception delegation */ +static inline __noprof unsigned long read_medeleg(void) +{ + unsigned long medeleg; + + asm volatile("csrr %0, medeleg" : "=r" (medeleg)); + + return medeleg; +} + +static inline __noprof void write_medeleg(unsigned long medeleg) +{ + asm volatile("csrw medeleg, %0" : : "r" (medeleg)); +} + +/* machine interrupt delegation */ +static inline __noprof unsigned long read_mideleg(void) +{ + unsigned long mideleg; + + asm volatile("csrr %0, mideleg" : "=r" (mideleg)); + + return mideleg; +} + +static inline __noprof void write_mideleg(unsigned long mideleg) +{ + asm volatile("csrw mideleg, %0" : : "r" (mideleg)); +} + +/* machine interrupt-enable register */ +static inline __noprof unsigned long read_mie(void) +{ + unsigned long mie; + + asm volatile("csrr %0, mie" : "=r" (mie)); + + return mie; +} + +static inline __noprof void write_mie(unsigned long mie) +{ + asm volatile("csrw mie, %0" : : "r" (mie)); +} + +/* supervisor interrupt-enable register */ +static inline __noprof unsigned long read_sie(void) +{ + unsigned long sie; + + asm volatile("csrr %0, sie" : "=r" (sie)); + + return sie; +} + +static inline __noprof void write_sie(unsigned long sie) +{ + asm volatile("csrw sie, %0" : : "r" (sie)); +} + +/* machine exception program counter */ +static inline __noprof unsigned long read_mepc(void) +{ + unsigned long mepc; + + asm volatile("csrr %0, mepc" : "=r" (mepc)); + + return mepc; +} + +static inline __noprof void write_mepc(unsigned long mepc) +{ + asm volatile("csrw mepc, %0" : : "r" (mepc)); +} + +/* supervisor exception program counter */ +static inline __noprof unsigned long read_sepc(void) +{ + unsigned long sepc; + + asm volatile("csrr %0, sepc" : "=r" (sepc)); + + return sepc; +} + +static inline __noprof void write_sepc(unsigned long sepc) +{ + asm volatile("csrw sepc, %0" : : "r" (sepc)); +} + +/* machine scratch register */ +static inline __noprof unsigned long read_mscratch(void) +{ + unsigned long mscratch; + + asm volatile("csrr %0, mscratch" : "=r" (mscratch)); + + return mscratch; +} + +static inline __noprof void write_mscratch(unsigned long mscratch) +{ + asm volatile("csrw mscratch, %0" : : "r" (mscratch)); +} + +/* supervisor scratch register */ +static inline __noprof unsigned long read_sscratch(void) +{ + unsigned long sscratch; + + asm volatile("csrr %0, sscratch" : "=r" (sscratch)); + + return sscratch; +} + +static inline __noprof void write_sscratch(unsigned long sscratch) +{ + asm volatile("csrw sscratch, %0" : : "r" (sscratch)); +} + +/* trap-return instructions */ +static inline __noprof void mret(void) +{ + asm volatile("mret"); +} + +static inline __noprof void sret(void) +{ + asm volatile("sret"); +} + +static inline __noprof void uret(void) +{ + asm volatile("uret"); +} + +__noprof uint64_t read_time(void); + +static inline __noprof uint64_t barrier_read_counter_timer(void) +{ + mb(); /* Get timer value after pending operations have completed */ + return read_time(); +} + +static inline __noprof uint32_t read_cntfrq(void) +{ + return CFG_RISCV_MTIME_RATE; +} + +#endif /*__ASSEMBLER__*/ + +#endif /*RISCV_H*/ diff --git a/optee_os/core/arch/riscv/include/riscv_macros.S b/optee_os/core/arch/riscv/include/riscv_macros.S new file mode 100644 index 0000000..ff40b87 --- /dev/null +++ b/optee_os/core/arch/riscv/include/riscv_macros.S @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + * Copyright (c) 2015, Linaro Limited + */ + + .altmacro + + + /* + * This helper macro concatenates instr_prefix, instr_suffix, to + * create a l(w,d)/s(w,d) instruction. + */ + .macro __do_reg instr_prefix, base_reg, base_offs, reg + \instr_prefix x\reg, \base_offs(\base_reg) + .endm + + /* + * This helper macro uses recursion to create a loop with a single + * load/store. + */ + .macro _do_regs instr_prefix, reg_bytes, base_reg, base_offs, \ + from_regnum, to_regnum + + .if (\to_regnum - \from_regnum + 1) > 1 + _do_regs \instr_prefix, \reg_bytes, \base_reg, \ + %(\base_offs + 1 * \reg_bytes), \ + %(\from_regnum + 1), \to_regnum + .endif + + __do_reg \instr_prefix, \base_reg, \base_offs, \from_regnum + .endm + + /* + * Stores registers x[from_regnum]..x[to_regnum] at + * [base_reg, #base_offs] + */ + .macro store_xregs base_reg, base_offs, from_regnum, to_regnum + _do_regs STR, RISCV_XLEN_BYTES, \base_reg, \base_offs, \ + \from_regnum, \to_regnum + .endm + + /* + * Loads registers x[from_regnum]..x[to_regnum] at + * [base_reg, #base_offs] + */ + .macro load_xregs base_reg, base_offs, from_regnum, to_regnum + _do_regs LDR, RISCV_XLEN_BYTES, \base_reg, \base_offs, \ + \from_regnum, \to_regnum + .endm + + /* + * Multiplication macro for RISC-V harts without M extension. + */ + .macro mult, reg_op0, reg_op1, reg_res + li \reg_res, 0 + mv a0, \reg_op0 + mv a1, \reg_op1 + mv a2, a0 + li a0, 0 + 1: + andi a3, a1, 1 + beqz a3, 2f + add a0, a0, a2 + 2: + srli a1, a1, 1 + slli a2, a2, 1 + bnez a1, 1b + add \reg_res, \reg_res, a0 + .endm + + .macro panic_at_abi_return +#if defined(CFG_TEE_CORE_DEBUG) + jal __panic_at_abi_return +#else + j . +#endif + .endm diff --git a/optee_os/core/arch/riscv/include/sbi.h b/optee_os/core/arch/riscv/include/sbi.h new file mode 100644 index 0000000..6ea6064 --- /dev/null +++ b/optee_os/core/arch/riscv/include/sbi.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef SBI_H +#define SBI_H + +#if defined(CFG_RISCV_SBI) + +/* SBI return error codes */ +#define SBI_SUCCESS 0 +#define SBI_ERR_FAILURE -1 +#define SBI_ERR_NOT_SUPPORTED -2 +#define SBI_ERR_INVALID_PARAM -3 +#define SBI_ERR_DENIED -4 +#define SBI_ERR_INVALID_ADDRESS -5 +#define SBI_ERR_ALREADY_AVAILABLE -6 +#define SBI_ERR_ALREADY_STARTED -7 +#define SBI_ERR_ALREADY_STOPPED -8 + +/* SBI Extension IDs */ +#define SBI_EXT_0_1_CONSOLE_PUTCHAR 0x01, 0 +#define SBI_EXT_HSM 0x48534D +#define SBI_EXT_TEE 0x544545 + +/* SBI function IDs for HSM extension */ +#define SBI_EXT_HSM_HART_START U(0) +#define SBI_EXT_HSM_HART_STOP U(1) +#define SBI_EXT_HSM_HART_GET_STATUS U(2) +#define SBI_EXT_HSM_HART_SUSPEND U(3) + +#ifndef __ASSEMBLER__ + +#include +#include +#include +#include +#include +#include + +void sbi_console_putchar(int ch); +int sbi_boot_hart(uint32_t hart_id, paddr_t start_addr, unsigned long arg); + +#endif /*__ASSEMBLER__*/ +#endif /*defined(CFG_RISCV_SBI)*/ +#endif /*SBI_H*/ diff --git a/optee_os/core/arch/riscv/include/sub.mk b/optee_os/core/arch/riscv/include/sub.mk new file mode 100644 index 0000000..7ec09f6 --- /dev/null +++ b/optee_os/core/arch/riscv/include/sub.mk @@ -0,0 +1 @@ +global-incdirs-y += . diff --git a/optee_os/core/arch/riscv/include/tee/entry_fast.h b/optee_os/core/arch/riscv/include/tee/entry_fast.h new file mode 100644 index 0000000..08ca181 --- /dev/null +++ b/optee_os/core/arch/riscv/include/tee/entry_fast.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2023 NXP + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_ENTRY_FAST_H +#define TEE_ENTRY_FAST_H + +#include + +/* These functions are overridable by the specific target */ +void tee_entry_get_api_call_count(struct thread_abi_args *args); +void tee_entry_get_api_uuid(struct thread_abi_args *args); +void tee_entry_get_api_revision(struct thread_abi_args *args); +void tee_entry_get_os_uuid(struct thread_abi_args *args); +void tee_entry_get_os_revision(struct thread_abi_args *args); + +/* + * Returns the number of calls recognized by tee_entry(). Used by the + * specific target to calculate the total number of supported calls when + * overriding tee_entry_get_api_call_count(). + */ +size_t tee_entry_generic_get_api_call_count(void); + +/* + * Fast call entry, __weak, overridable. If overridden should call + * __tee_entry_fast() at the end in order to handle the standard functions. + */ +void tee_entry_fast(struct thread_abi_args *args); +void __tee_entry_fast(struct thread_abi_args *args); + +#endif /* TEE_ENTRY_FAST_H */ diff --git a/optee_os/core/arch/riscv/include/tee/optee_abi.h b/optee_os/core/arch/riscv/include/tee/optee_abi.h new file mode 100644 index 0000000..e65bebc --- /dev/null +++ b/optee_os/core/arch/riscv/include/tee/optee_abi.h @@ -0,0 +1,705 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2023 NXP + * Copyright (c) 2015-2021, Linaro Limited + */ +#ifndef OPTEE_ABI_H +#define OPTEE_ABI_H + +#include + +/* + * This file should be kept in sync between secure domain and non-secure + * domain kernel driver. + * This file depends on optee_msg.h being included to expand the ABI id + * macros below. + */ + +#define OPTEE_ABI_32 U(0) +#define OPTEE_ABI_64 U(0x40000000) +#define OPTEE_ABI_FAST_CALL U(0x80000000) +#define OPTEE_ABI_STD_CALL U(0) + +#define OPTEE_ABI_OWNER_MASK U(0x3F) +#define OPTEE_ABI_OWNER_SHIFT U(24) + +#define OPTEE_ABI_FUNC_MASK U(0xFFFF) + +#define OPTEE_ABI_IS_FAST_CALL(abi_val) ((abi_val) & OPTEE_ABI_FAST_CALL) +#define OPTEE_ABI_IS_64(abi_val) ((abi_val) & OPTEE_ABI_64) +#define OPTEE_ABI_FUNC_NUM(abi_val) ((abi_val) & OPTEE_ABI_FUNC_MASK) +#define OPTEE_ABI_OWNER_NUM(abi_val) \ + (((abi_val) >> OPTEE_ABI_OWNER_SHIFT) & OPTEE_ABI_OWNER_MASK) + +#define OPTEE_ABI_CALL_VAL(type, calling_convention, owner, func_num) \ + ((type) | (calling_convention) | \ + (((owner) & OPTEE_ABI_OWNER_MASK) << \ + OPTEE_ABI_OWNER_SHIFT) |\ + ((func_num) & OPTEE_ABI_FUNC_MASK)) + +#define OPTEE_ABI_STD_CALL_VAL(func_num) \ + OPTEE_ABI_CALL_VAL(OPTEE_ABI_32, OPTEE_ABI_STD_CALL, \ + OPTEE_ABI_OWNER_TRUSTED_OS, (func_num)) +#define OPTEE_ABI_FAST_CALL_VAL(func_num) \ + OPTEE_ABI_CALL_VAL(OPTEE_ABI_32, OPTEE_ABI_FAST_CALL, \ + OPTEE_ABI_OWNER_TRUSTED_OS, (func_num)) + +#define OPTEE_ABI_OWNER_ARCH U(0) +#define OPTEE_ABI_OWNER_CPU U(1) +#define OPTEE_ABI_OWNER_SIP U(2) +#define OPTEE_ABI_OWNER_OEM U(3) +#define OPTEE_ABI_OWNER_STANDARD U(4) +#define OPTEE_ABI_OWNER_TRUSTED_APP U(48) +#define OPTEE_ABI_OWNER_TRUSTED_OS U(50) + +#define OPTEE_ABI_OWNER_TRUSTED_OS_OPTEED U(62) +#define OPTEE_ABI_OWNER_TRUSTED_OS_API U(63) + +/* + * Function specified by ABI Calling convention. + */ +#define OPTEE_ABI_FUNCID_CALLS_COUNT U(0xFF00) +#define OPTEE_ABI_CALLS_COUNT \ + OPTEE_ABI_CALL_VAL(OPTEE_ABI_32, OPTEE_ABI_FAST_CALL, \ + OPTEE_ABI_OWNER_TRUSTED_OS_API, \ + OPTEE_ABI_FUNCID_CALLS_COUNT) + +/* + * Normal cached memory (write-back), shareable for SMP systems and not + * shareable for UP systems. + */ +#define OPTEE_ABI_SHM_CACHED U(1) + +/* + * a0..a7 is used as register names in the descriptions below. + */ + +/* + * Function specified by ABI Calling convention + * + * Return the following UID if using API specified in this file + * without further extensions: + * 384fb3e0-e7f8-11e3-af63-0002a5d5c51b. + * see also OPTEE_MSG_UID_* in optee_msg.h + */ +#define OPTEE_ABI_FUNCID_CALLS_UID OPTEE_MSG_FUNCID_CALLS_UID +#define OPTEE_ABI_CALLS_UID \ + OPTEE_ABI_CALL_VAL(OPTEE_ABI_32, OPTEE_ABI_FAST_CALL, \ + OPTEE_ABI_OWNER_TRUSTED_OS_API, \ + OPTEE_ABI_FUNCID_CALLS_UID) + +/* + * Function specified by ABI Calling convention + * + * Returns 2.0 if using API specified in this file without further extensions. + * see also OPTEE_MSG_REVISION_* in optee_msg.h + */ +#define OPTEE_ABI_FUNCID_CALLS_REVISION OPTEE_MSG_FUNCID_CALLS_REVISION +#define OPTEE_ABI_CALLS_REVISION \ + OPTEE_ABI_CALL_VAL(OPTEE_ABI_32, OPTEE_ABI_FAST_CALL, \ + OPTEE_ABI_OWNER_TRUSTED_OS_API, \ + OPTEE_ABI_FUNCID_CALLS_REVISION) + +/* + * Get UUID of Trusted OS. + * + * Used by non-secure world to figure out which Trusted OS is installed. + * Note that returned UUID is the UUID of the Trusted OS, not of the API. + * + * Returns UUID in a0-4 in the same way as OPTEE_ABI_CALLS_UID + * described above. + */ +#define OPTEE_ABI_FUNCID_GET_OS_UUID OPTEE_MSG_FUNCID_GET_OS_UUID +#define OPTEE_ABI_CALL_GET_OS_UUID \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_GET_OS_UUID) + +/* + * Get revision of Trusted OS. + * + * Used by non-secure world to figure out which version of the Trusted OS + * is installed. Note that the returned revision is the revision of the + * Trusted OS, not of the API. + * + * Returns revision in a0-1 in the same way as OPTEE_ABI_CALLS_REVISION + * described above. May optionally return a 32-bit build identifier in a2, + * with zero meaning unspecified. + */ +#define OPTEE_ABI_FUNCID_GET_OS_REVISION OPTEE_MSG_FUNCID_GET_OS_REVISION +#define OPTEE_ABI_CALL_GET_OS_REVISION \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_GET_OS_REVISION) + +/* + * Call with struct optee_msg_arg as argument + * + * When called with OPTEE_ABI_CALL_WITH_RPC_ARG or + * OPTEE_ABI_CALL_WITH_REGD_ARG in a0 there is one RPC struct optee_msg_arg + * following after the first struct optee_msg_arg. The RPC struct + * optee_msg_arg has reserved space for the number of RPC parameters as + * returned by OPTEE_ABI_EXCHANGE_CAPABILITIES. + * + * When calling these functions normal world has a few responsibilities: + * 1. It must be able to handle eventual RPCs + * 2. Non-secure interrupts should not be masked + * 3. If asynchronous notifications has been negotiated successfully, then + * the interrupt for asynchronous notifications should be unmasked + * during this call. + * + * Call register usage, OPTEE_ABI_CALL_WITH_ARG and + * OPTEE_ABI_CALL_WITH_RPC_ARG: + * a0 ABI Function ID, OPTEE_ABI_CALL_WITH_ARG or OPTEE_ABI_CALL_WITH_RPC_ARG + * a1 Upper 32 bits of a 64-bit physical pointer to a struct optee_msg_arg + * a2 Lower 32 bits of a 64-bit physical pointer to a struct optee_msg_arg + * a3 Cache settings, not used if physical pointer is in a predefined shared + * memory area else per OPTEE_ABI_SHM_* + * a4-6 Not used + * a7 Hypervisor Client ID register + * + * Call register usage, OPTEE_ABI_CALL_WITH_REGD_ARG: + * a0 ABI Function ID, OPTEE_ABI_CALL_WITH_REGD_ARG + * a1 Upper 32 bits of a 64-bit shared memory cookie + * a2 Lower 32 bits of a 64-bit shared memory cookie + * a3 Offset of the struct optee_msg_arg in the shared memory with the + * supplied cookie + * a4-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 Return value, OPTEE_ABI_RETURN_* + * a1-3 Not used + * a4-7 Preserved + * + * OPTEE_ABI_RETURN_ETHREAD_LIMIT return register usage: + * a0 Return value, OPTEE_ABI_RETURN_ETHREAD_LIMIT + * a1-3 Preserved + * a4-7 Preserved + * + * RPC return register usage: + * a0 Return value, OPTEE_ABI_RETURN_IS_RPC(val) + * a1-2 RPC parameters + * a3-7 Resume information, must be preserved + * + * Possible return values: + * OPTEE_ABI_RETURN_UNKNOWN_FUNCTION Trusted OS does not recognize this + * function. + * OPTEE_ABI_RETURN_OK Call completed, result updated in + * the previously supplied struct + * optee_msg_arg. + * OPTEE_ABI_RETURN_ETHREAD_LIMIT Number of Trusted OS threads exceeded, + * try again later. + * OPTEE_ABI_RETURN_EBADADDR Bad physical pointer to struct + * optee_msg_arg. + * OPTEE_ABI_RETURN_EBADCMD Bad/unknown cmd in struct optee_msg_arg + * OPTEE_ABI_RETURN_IS_RPC() Call suspended by RPC call to normal + * world. + */ +#define OPTEE_ABI_FUNCID_CALL_WITH_ARG OPTEE_MSG_FUNCID_CALL_WITH_ARG +#define OPTEE_ABI_CALL_WITH_ARG \ + OPTEE_ABI_STD_CALL_VAL(OPTEE_ABI_FUNCID_CALL_WITH_ARG) +#define OPTEE_ABI_CALL_WITH_RPC_ARG \ + OPTEE_ABI_STD_CALL_VAL(OPTEE_ABI_FUNCID_CALL_WITH_RPC_ARG) +#define OPTEE_ABI_CALL_WITH_REGD_ARG \ + OPTEE_ABI_STD_CALL_VAL(OPTEE_ABI_FUNCID_CALL_WITH_REGD_ARG) + +/* + * Get Shared Memory Config + * + * Returns the Secure/Non-secure shared memory config. + * + * Call register usage: + * a0 ABI Function ID, OPTEE_ABI_GET_SHM_CONFIG + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Have config return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1 Physical address of start of SHM + * a2 Size of SHM + * a3 Cache settings of memory, as defined by the + * OPTEE_ABI_SHM_* values above + * a4-7 Preserved + * + * Not available register usage: + * a0 OPTEE_ABI_RETURN_ENOTAVAIL + * a1-3 Not used + * a4-7 Preserved + */ +#define OPTEE_ABI_FUNCID_GET_SHM_CONFIG 7 +#define OPTEE_ABI_GET_SHM_CONFIG \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_GET_SHM_CONFIG) + +/* + * Configures L2CC mutex + * + * Disables, enables usage of L2CC mutex. Returns or sets physical address + * of L2CC mutex. + * + * Call register usage: + * a0 ABI Function ID, OPTEE_ABI_L2CC_MUTEX + * a1 OPTEE_ABI_L2CC_MUTEX_GET_ADDR Get physical address of mutex + * OPTEE_ABI_L2CC_MUTEX_SET_ADDR Set physical address of mutex + * OPTEE_ABI_L2CC_MUTEX_ENABLE Enable usage of mutex + * OPTEE_ABI_L2CC_MUTEX_DISABLE Disable usage of mutex + * a2 if a1 == OPTEE_ABI_L2CC_MUTEX_SET_ADDR, upper 32bit of a 64bit + * physical address of mutex + * a3 if a1 == OPTEE_ABI_L2CC_MUTEX_SET_ADDR, lower 32bit of a 64bit + * physical address of mutex + * a3-6 Not used + * a7 Hypervisor Client ID register + * + * Have config return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1 Preserved + * a2 if a1 == OPTEE_ABI_L2CC_MUTEX_GET_ADDR, upper 32bit of a 64bit + * physical address of mutex + * a3 if a1 == OPTEE_ABI_L2CC_MUTEX_GET_ADDR, lower 32bit of a 64bit + * physical address of mutex + * a3-7 Preserved + * + * Error return register usage: + * a0 OPTEE_ABI_RETURN_ENOTAVAIL Physical address not available + * OPTEE_ABI_RETURN_EBADADDR Bad supplied physical address + * OPTEE_ABI_RETURN_EBADCMD Unsupported value in a1 + * a1-7 Preserved + */ +#define OPTEE_ABI_L2CC_MUTEX_GET_ADDR U(0) +#define OPTEE_ABI_L2CC_MUTEX_SET_ADDR U(1) +#define OPTEE_ABI_L2CC_MUTEX_ENABLE U(2) +#define OPTEE_ABI_L2CC_MUTEX_DISABLE U(3) +#define OPTEE_ABI_FUNCID_L2CC_MUTEX U(8) +#define OPTEE_ABI_L2CC_MUTEX \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_L2CC_MUTEX) + +/* + * Exchanges capabilities between normal world and secure world + * + * Call register usage: + * a0 ABI Function ID, OPTEE_ABI_EXCHANGE_CAPABILITIES + * a1 bitfield of normal world capabilities OPTEE_ABI_NSEC_CAP_* + * a2-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1 bitfield of secure world capabilities OPTEE_ABI_SEC_CAP_* + * a2 The maximum secure world notification number + * a3 Bit[7:0]: Number of parameters needed for RPC to be supplied + * as the second MSG arg struct for + * OPTEE_ABI_CALL_WITH_ARG + * Bit[31:8]: Reserved (MBZ) + * a3-7 Preserved + * + * Error return register usage: + * a0 OPTEE_ABI_RETURN_ENOTAVAIL, can't use the capabilities from normal world + * a1 bitfield of secure world capabilities OPTEE_ABI_SEC_CAP_* + * a2-7 Preserved + */ +/* Normal world works as a uniprocessor system */ +#define OPTEE_ABI_NSEC_CAP_UNIPROCESSOR BIT(0) +/* Secure world has reserved shared memory for normal world to use */ +#define OPTEE_ABI_SEC_CAP_HAVE_RESERVED_SHM BIT(0) +/* Secure world can communicate via previously unregistered shared memory */ +#define OPTEE_ABI_SEC_CAP_UNREGISTERED_SHM BIT(1) +/* + * Secure world supports commands "register/unregister shared memory", + * secure world accepts command buffers located in any parts of non-secure RAM + */ +#define OPTEE_ABI_SEC_CAP_DYNAMIC_SHM BIT(2) +/* Secure world is built with virtualization support */ +#define OPTEE_ABI_SEC_CAP_VIRTUALIZATION BIT(3) +/* Secure world supports Shared Memory with a NULL reference */ +#define OPTEE_ABI_SEC_CAP_MEMREF_NULL BIT(4) +/* Secure world supports asynchronous notification of normal world */ +#define OPTEE_ABI_SEC_CAP_ASYNC_NOTIF BIT(5) +/* Secure world supports pre-allocating RPC arg struct */ +#define OPTEE_ABI_SEC_CAP_RPC_ARG BIT(6) + +#define OPTEE_ABI_FUNCID_EXCHANGE_CAPABILITIES U(9) +#define OPTEE_ABI_EXCHANGE_CAPABILITIES \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_EXCHANGE_CAPABILITIES) + +/* + * Disable and empties cache of shared memory objects + * + * Secure world can cache frequently used shared memory objects, for + * example objects used as RPC arguments. When secure world is idle this + * function returns one shared memory reference to free. To disable the + * cache and free all cached objects this function has to be called until + * it returns OPTEE_ABI_RETURN_ENOTAVAIL. + * + * Call register usage: + * a0 ABI Function ID, OPTEE_ABI_DISABLE_SHM_CACHE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1 Upper 32 bits of a 64-bit Shared memory cookie + * a2 Lower 32 bits of a 64-bit Shared memory cookie + * a3-7 Preserved + * + * Cache empty return register usage: + * a0 OPTEE_ABI_RETURN_ENOTAVAIL + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_ABI_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_ABI_FUNCID_DISABLE_SHM_CACHE U(10) +#define OPTEE_ABI_DISABLE_SHM_CACHE \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_DISABLE_SHM_CACHE) + +/* + * Enable cache of shared memory objects + * + * Secure world can cache frequently used shared memory objects, for + * example objects used as RPC arguments. When secure world is idle this + * function returns OPTEE_ABI_RETURN_OK and the cache is enabled. If + * secure world isn't idle OPTEE_ABI_RETURN_EBUSY is returned. + * + * Call register usage: + * a0 ABI Function ID, OPTEE_ABI_ENABLE_SHM_CACHE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_ABI_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_ABI_FUNCID_ENABLE_SHM_CACHE U(11) +#define OPTEE_ABI_ENABLE_SHM_CACHE \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_ENABLE_SHM_CACHE) + +/* + * Release of secondary cores + * + * OP-TEE in secure world is in charge of the release process of secondary + * cores. The Rich OS issue the this request to ask OP-TEE to boot up the + * secondary cores, go through the OP-TEE per-core initialization, and then + * switch to the Non-seCure world with the Rich OS provided entry address. + * The secondary cores enter Non-Secure world in SVC mode, with Thumb, FIQ, + * IRQ and Abort bits disabled. + * + * Call register usage: + * a0 ABI Function ID, OPTEE_ABI_BOOT_SECONDARY + * a1 Index of secondary core to boot + * a2 Upper 32 bits of a 64-bit Non-Secure world entry physical address + * a3 Lower 32 bits of a 64-bit Non-Secure world entry physical address + * a4-7 Not used + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1-7 Preserved + * + * Error return: + * a0 OPTEE_ABI_RETURN_EBADCMD Core index out of range + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_ABI_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_ABI_FUNCID_BOOT_SECONDARY U(12) +#define OPTEE_ABI_BOOT_SECONDARY \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_BOOT_SECONDARY) + +/* + * Inform OP-TEE about a new virtual machine + * + * Hypervisor issues this call during virtual machine (guest) creation. + * OP-TEE records client id of new virtual machine and prepares + * to receive requests from it. This call is available only if OP-TEE + * was built with virtualization support. + * + * Call requests usage: + * a0 ABI Function ID, OPTEE_ABI_VM_CREATED + * a1 Hypervisor Client ID of newly created virtual machine + * a2-6 Not used + * a7 Hypervisor Client ID register. Must be 0, because only hypervisor + * can issue this call + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1-7 Preserved + * + * Error return: + * a0 OPTEE_ABI_RETURN_ENOTAVAIL OP-TEE have no resources for + * another VM + * a1-7 Preserved + * + */ +#define OPTEE_ABI_FUNCID_VM_CREATED U(13) +#define OPTEE_ABI_VM_CREATED \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_VM_CREATED) + +/* + * Inform OP-TEE about shutdown of a virtual machine + * + * Hypervisor issues this call during virtual machine (guest) destruction. + * OP-TEE will clean up all resources associated with this VM. This call is + * available only if OP-TEE was built with virtualization support. + * + * Call requests usage: + * a0 ABI Function ID, OPTEE_ABI_VM_DESTROYED + * a1 Hypervisor Client ID of virtual machine being shut down + * a2-6 Not used + * a7 Hypervisor Client ID register. Must be 0, because only hypervisor + * can issue this call + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1-7 Preserved + * + */ +#define OPTEE_ABI_FUNCID_VM_DESTROYED U(14) +#define OPTEE_ABI_VM_DESTROYED \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_VM_DESTROYED) + +/* + * Query OP-TEE about number of supported threads + * + * Normal World OS or Hypervisor issues this call to find out how many + * threads OP-TEE supports. That is how many standard calls can be issued + * in parallel before OP-TEE will return OPTEE_ABI_RETURN_ETHREAD_LIMIT. + * + * Call requests usage: + * a0 ABI Function ID, OPTEE_ABI_GET_THREAD_COUNT + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1 Number of threads + * a2-7 Preserved + * + * Error return: + * a0 OPTEE_ABI_RETURN_UNKNOWN_FUNCTION Requested call is not implemented + * a1-7 Preserved + */ +#define OPTEE_ABI_FUNCID_GET_THREAD_COUNT U(15) +#define OPTEE_ABI_GET_THREAD_COUNT \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_GET_THREAD_COUNT) + +/* + * Inform OP-TEE that normal world is able to receive asynchronous + * notifications. + * + * Call requests usage: + * a0 ABI Function ID, OPTEE_ABI_ENABLE_ASYNC_NOTIF + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1-7 Preserved + * + * Not supported return register usage: + * a0 OPTEE_ABI_RETURN_ENOTAVAIL + * a1-7 Preserved + */ +#define OPTEE_ABI_FUNCID_ENABLE_ASYNC_NOTIF 16 +#define OPTEE_ABI_ENABLE_ASYNC_NOTIF \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_ENABLE_ASYNC_NOTIF) + +/* + * Retrieve a value of notifications pending since the last call of this + * function. + * + * OP-TEE keeps a record of all posted values. When an interrupt is + * received which indicates that there are posted values this function + * should be called until all pended values have been retrieved. When a + * value is retrieved, it's cleared from the record in secure world. + * + * It is expected that this function is called from an interrupt handler + * in normal world. + * + * Call requests usage: + * a0 ABI Function ID, OPTEE_ABI_GET_ASYNC_NOTIF_VALUE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_ABI_RETURN_OK + * a1 value + * a2 Bit[0]: OPTEE_ABI_ASYNC_NOTIF_VALUE_VALID if the value in a1 is + * valid, else 0 if no values were pending + * a2 Bit[1]: OPTEE_ABI_ASYNC_NOTIF_VALUE_PENDING if another value is + * pending, else 0. + * Bit[31:2]: MBZ + * a3-7 Preserved + * + * Not supported return register usage: + * a0 OPTEE_ABI_RETURN_ENOTAVAIL + * a1-7 Preserved + */ +#define OPTEE_ABI_ASYNC_NOTIF_VALID BIT(0) +#define OPTEE_ABI_ASYNC_NOTIF_PENDING BIT(1) + +/* + * Notification that OP-TEE expects a yielding call to do some bottom half + * work in a driver. + */ +#define OPTEE_ABI_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF 0 + +#define OPTEE_ABI_FUNCID_GET_ASYNC_NOTIF_VALUE 17 +#define OPTEE_ABI_GET_ASYNC_NOTIF_VALUE \ + OPTEE_ABI_FAST_CALL_VAL(OPTEE_ABI_FUNCID_GET_ASYNC_NOTIF_VALUE) + +/* See OPTEE_ABI_CALL_WITH_RPC_ARG above */ +#define OPTEE_ABI_FUNCID_CALL_WITH_RPC_ARG U(18) + +/* See OPTEE_ABI_CALL_WITH_REGD_ARG above */ +#define OPTEE_ABI_FUNCID_CALL_WITH_REGD_ARG U(19) + +/* + * Resume from RPC (for example after processing a foreign interrupt) + * + * Call register usage: + * a0 ABI Function ID, OPTEE_ABI_CALL_RETURN_FROM_RPC + * a1-3 Value of a1-3 when OPTEE_ABI_CALL_WITH_ARG returned + * OPTEE_ABI_RETURN_RPC in a0 + * + * Return register usage is the same as for OPTEE_ABI_*CALL_WITH_ARG above. + * + * Possible return values + * OPTEE_ABI_RETURN_UNKNOWN_FUNCTION Trusted OS does not recognize this + * function. + * OPTEE_ABI_RETURN_OK Original call completed, result + * updated in the previously supplied. + * struct optee_msg_arg + * OPTEE_ABI_RETURN_RPC Call suspended by RPC call to normal + * world. + * OPTEE_ABI_RETURN_ERESUME Resume failed, the opaque resume + * information was corrupt. + */ +#define OPTEE_ABI_FUNCID_RETURN_FROM_RPC U(3) +#define OPTEE_ABI_CALL_RETURN_FROM_RPC \ + OPTEE_ABI_STD_CALL_VAL(OPTEE_ABI_FUNCID_RETURN_FROM_RPC) + +#define OPTEE_ABI_RETURN_RPC_PREFIX_MASK U(0xFFFF0000) +#define OPTEE_ABI_RETURN_RPC_PREFIX U(0xFFFF0000) +#define OPTEE_ABI_RETURN_RPC_FUNC_MASK U(0x0000FFFF) + +#define OPTEE_ABI_RETURN_GET_RPC_FUNC(ret) \ + ((ret) & OPTEE_ABI_RETURN_RPC_FUNC_MASK) + +#define OPTEE_ABI_RPC_VAL(func) ((func) | OPTEE_ABI_RETURN_RPC_PREFIX) + +/* + * Allocate memory for RPC parameter passing. The memory is used to hold a + * struct optee_msg_arg. + * + * "Call" register usage: + * a0 This value, OPTEE_ABI_RETURN_RPC_ALLOC + * a1 Size in bytes of required argument memory + * a2 Not used + * a3 Resume information, must be preserved + * a4-5 Not used + * a6-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 ABI Function ID, OPTEE_ABI_CALL_RETURN_FROM_RPC. + * a1 Upper 32 bits of 64-bit physical pointer to allocated + * memory, (a1 == 0 && a2 == 0) if size was 0 or if memory can't + * be allocated. + * a2 Lower 32 bits of 64-bit physical pointer to allocated + * memory, (a1 == 0 && a2 == 0) if size was 0 or if memory can't + * be allocated + * a3 Preserved + * a4 Upper 32 bits of 64-bit Shared memory cookie used when freeing + * the memory or doing an RPC + * a5 Lower 32 bits of 64-bit Shared memory cookie used when freeing + * the memory or doing an RPC + * a6-7 Preserved + */ +#define OPTEE_ABI_RPC_FUNC_ALLOC U(0) +#define OPTEE_ABI_RETURN_RPC_ALLOC \ + OPTEE_ABI_RPC_VAL(OPTEE_ABI_RPC_FUNC_ALLOC) + +/* + * Free memory previously allocated by OPTEE_ABI_RETURN_RPC_ALLOC + * + * "Call" register usage: + * a0 This value, OPTEE_ABI_RETURN_RPC_FREE + * a1 Upper 32 bits of 64-bit shared memory cookie belonging to this + * argument memory + * a2 Lower 32 bits of 64-bit shared memory cookie belonging to this + * argument memory + * a3-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 ABI Function ID, OPTEE_ABI_CALL_RETURN_FROM_RPC. + * a1-2 Not used + * a3-7 Preserved + */ +#define OPTEE_ABI_RPC_FUNC_FREE U(2) +#define OPTEE_ABI_RETURN_RPC_FREE \ + OPTEE_ABI_RPC_VAL(OPTEE_ABI_RPC_FUNC_FREE) + +/* + * Deliver a foreign interrupt in normal world. + * + * "Call" register usage: + * a0 OPTEE_ABI_RETURN_RPC_FOREIGN_INTR + * a1-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 ABI Function ID, OPTEE_ABI_CALL_RETURN_FROM_RPC. + * a1-7 Preserved + */ +#define OPTEE_ABI_RPC_FUNC_FOREIGN_INTR U(4) +#define OPTEE_ABI_RETURN_RPC_FOREIGN_INTR \ + OPTEE_ABI_RPC_VAL(OPTEE_ABI_RPC_FUNC_FOREIGN_INTR) + +/* + * Do an RPC request. The supplied struct optee_msg_arg tells which + * request to do and the parameters for the request. The following fields + * are used (the rest are unused): + * - cmd the Request ID + * - ret return value of the request, filled in by normal world + * - num_params number of parameters for the request + * - params the parameters + * - param_attrs attributes of the parameters + * + * "Call" register usage: + * a0 OPTEE_ABI_RETURN_RPC_CMD + * a1 Upper 32 bits of a 64-bit Shared memory cookie holding a + * struct optee_msg_arg, must be preserved, only the data should + * be updated + * a2 Lower 32 bits of a 64-bit Shared memory cookie holding a + * struct optee_msg_arg, must be preserved, only the data should + * be updated + * a3-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 ABI Function ID, OPTEE_ABI_CALL_RETURN_FROM_RPC. + * a1-2 Not used + * a3-7 Preserved + */ +#define OPTEE_ABI_RPC_FUNC_CMD U(5) +#define OPTEE_ABI_RETURN_RPC_CMD \ + OPTEE_ABI_RPC_VAL(OPTEE_ABI_RPC_FUNC_CMD) + +/* Returned in a0 */ +#define OPTEE_ABI_RETURN_UNKNOWN_FUNCTION U(0xFFFFFFFF) + +/* Returned in a0 only from Trusted OS functions */ +#define OPTEE_ABI_RETURN_OK U(0x0) +#define OPTEE_ABI_RETURN_ETHREAD_LIMIT U(0x1) +#define OPTEE_ABI_RETURN_EBUSY U(0x2) +#define OPTEE_ABI_RETURN_ERESUME U(0x3) +#define OPTEE_ABI_RETURN_EBADADDR U(0x4) +#define OPTEE_ABI_RETURN_EBADCMD U(0x5) +#define OPTEE_ABI_RETURN_ENOMEM U(0x6) +#define OPTEE_ABI_RETURN_ENOTAVAIL U(0x7) +#define OPTEE_ABI_RETURN_IS_RPC(_ret) ({\ + typeof(_ret) (ret) = (_ret); \ + (((ret) != OPTEE_ABI_RETURN_UNKNOWN_FUNCTION) && \ + ((((ret) & OPTEE_ABI_RETURN_RPC_PREFIX_MASK) == \ + OPTEE_ABI_RETURN_RPC_PREFIX))); }) + +#endif /* OPTEE_ABI_H */ diff --git a/optee_os/core/arch/riscv/include/tee/teeabi_opteed.h b/optee_os/core/arch/riscv/include/tee/teeabi_opteed.h new file mode 100644 index 0000000..50f8465 --- /dev/null +++ b/optee_os/core/arch/riscv/include/tee/teeabi_opteed.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2023 NXP + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEEABI_OPTEED_H +#define TEEABI_OPTEED_H + +/* + * This file specify ABI function IDs used when returning from TEE to the + * secure monitor if applicable. + */ + +/* + * Issued when returning from initial entry. + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_ENTRY_DONE + * a1 Pointer to entry vector + */ +#define TEEABI_OPTEED_FUNCID_RETURN_ENTRY_DONE 0 +#define TEEABI_OPTEED_RETURN_ENTRY_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_ENTRY_DONE) + +/* + * Issued when returning from "cpu_on" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_ON_DONE + * a1 0 on success and anything else to indicate error condition + */ +#define TEEABI_OPTEED_FUNCID_RETURN_ON_DONE 1 +#define TEEABI_OPTEED_RETURN_ON_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_ON_DONE) + +/* + * Issued when returning from "cpu_off" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_OFF_DONE + * a1 0 on success and anything else to indicate error condition + */ +#define TEEABI_OPTEED_FUNCID_RETURN_OFF_DONE 2 +#define TEEABI_OPTEED_RETURN_OFF_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_OFF_DONE) + +/* + * Issued when returning from "cpu_suspend" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_SUSPEND_DONE + * a1 0 on success and anything else to indicate error condition + */ +#define TEEABI_OPTEED_FUNCID_RETURN_SUSPEND_DONE 3 +#define TEEABI_OPTEED_RETURN_SUSPEND_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_SUSPEND_DONE) + +/* + * Issued when returning from "cpu_resume" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_RESUME_DONE + * a1 0 on success and anything else to indicate error condition + */ +#define TEEABI_OPTEED_FUNCID_RETURN_RESUME_DONE 4 +#define TEEABI_OPTEED_RETURN_RESUME_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_RESUME_DONE) + +/* + * Issued when returning from "std_abi" or "fast_abi" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_CALL_DONE + * a1-4 Return value 0-3 which will passed to non-secure domain in a0-3 + */ +#define TEEABI_OPTEED_FUNCID_RETURN_CALL_DONE 5 +#define TEEABI_OPTEED_RETURN_CALL_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_CALL_DONE) + +/* + * Issued when returning from "fiq" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_FIQ_DONE + */ +#define TEEABI_OPTEED_FUNCID_RETURN_FIQ_DONE 6 +#define TEEABI_OPTEED_RETURN_FIQ_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_FIQ_DONE) + +/* + * Issued when returning from "system_off" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_SYSTEM_OFF_DONE + */ +#define TEEABI_OPTEED_FUNCID_RETURN_SYSTEM_OFF_DONE 7 +#define TEEABI_OPTEED_RETURN_SYSTEM_OFF_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_SYSTEM_OFF_DONE) + +/* + * Issued when returning from "system_reset" vector + * + * Register usage: + * a0 ABI Function ID, TEEABI_OPTEED_RETURN_SYSTEM_RESET_DONE + */ +#define TEEABI_OPTEED_FUNCID_RETURN_SYSTEM_RESET_DONE 8 +#define TEEABI_OPTEED_RETURN_SYSTEM_RESET_DONE \ + TEEABI_OPTEED_RV(TEEABI_OPTEED_FUNCID_RETURN_SYSTEM_RESET_DONE) + +#endif /*TEEABI_OPTEED_H*/ diff --git a/optee_os/core/arch/riscv/include/tee/teeabi_opteed_macros.h b/optee_os/core/arch/riscv/include/tee/teeabi_opteed_macros.h new file mode 100644 index 0000000..6345bd6 --- /dev/null +++ b/optee_os/core/arch/riscv/include/tee/teeabi_opteed_macros.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2023 NXP + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEEABI_OPTEED_MACROS_H +#define TEEABI_OPTEED_MACROS_H + +#define TEEABI_OPTEED_RV(func_num) \ + OPTEE_ABI_CALL_VAL(OPTEE_ABI_32, OPTEE_ABI_FAST_CALL, \ + OPTEE_ABI_OWNER_TRUSTED_OS_OPTEED, (func_num)) + +#endif /*TEEABI_OPTEED_MACROS_H*/ diff --git a/optee_os/core/arch/riscv/kernel/abort.c b/optee_os/core/arch/riscv/kernel/abort.c new file mode 100644 index 0000000..f0dcf89 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/abort.c @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + * Copyright (c) 2015-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum fault_type { + FAULT_TYPE_USER_MODE_PANIC, + FAULT_TYPE_USER_MODE_VFP, + FAULT_TYPE_PAGE_FAULT, + FAULT_TYPE_IGNORE, +}; + +#ifdef CFG_UNWIND + +/* Kernel mode unwind */ +static void __print_stack_unwind(struct abort_info *ai) +{ + struct unwind_state_riscv state = { + .fp = ai->regs->s0, + .pc = ai->regs->epc, + }; + + print_stack_riscv(&state, thread_stack_start(), thread_stack_size()); +} + +#else /* CFG_UNWIND */ +static void __print_stack_unwind(struct abort_info *ai __unused) +{ +} +#endif /* CFG_UNWIND */ + +static __maybe_unused const char *abort_type_to_str(uint32_t abort_type) +{ + if (abort_type == ABORT_TYPE_DATA) + return "data"; + if (abort_type == ABORT_TYPE_PREFETCH) + return "prefetch"; + return "undef"; +} + +static __maybe_unused const char * +fault_to_str(uint32_t abort_type, uint32_t fault_descr) +{ + /* fault_descr is only valid for data or prefetch abort */ + if (abort_type != ABORT_TYPE_DATA && abort_type != ABORT_TYPE_PREFETCH) + return ""; + + switch (core_mmu_get_fault_type(fault_descr)) { + case CORE_MMU_FAULT_ALIGNMENT: + return " (alignment fault)"; + case CORE_MMU_FAULT_TRANSLATION: + return " (translation fault)"; + case CORE_MMU_FAULT_READ_PERMISSION: + return " (read permission fault)"; + case CORE_MMU_FAULT_WRITE_PERMISSION: + return " (write permission fault)"; + case CORE_MMU_FAULT_TAG_CHECK: + return " (tag check fault)"; + default: + return ""; + } +} + +static __maybe_unused void +__print_abort_info(struct abort_info *ai __maybe_unused, + const char *ctx __maybe_unused) +{ + __maybe_unused size_t core_pos = 0; + + if (abort_is_user_exception(ai)) + core_pos = thread_get_tsd()->abort_core; + else + core_pos = get_core_pos(); + + EMSG_RAW(""); + EMSG_RAW("%s %s-abort at address 0x%" PRIxVA "%s", + ctx, abort_type_to_str(ai->abort_type), ai->va, + fault_to_str(ai->abort_type, ai->fault_descr)); + EMSG_RAW("cpu\t#%zu", core_pos); + EMSG_RAW("cause\t%016" PRIxPTR " epc\t%016" PRIxPTR, + ai->regs->cause, ai->regs->epc); + EMSG_RAW("tval\t%016" PRIxPTR " status\t%016" PRIxPTR, + ai->regs->tval, ai->regs->status); + EMSG_RAW("ra\t%016" PRIxPTR " sp\t%016" PRIxPTR, + ai->regs->ra, ai->regs->sp); + EMSG_RAW("gp\t%016" PRIxPTR " tp\t%016" PRIxPTR, + ai->regs->gp, ai->regs->tp); + EMSG_RAW("t0\t%016" PRIxPTR " t1\t%016" PRIxPTR, + ai->regs->t0, ai->regs->t1); + EMSG_RAW("t2\t%016" PRIxPTR " s0\t%016" PRIxPTR, + ai->regs->t2, ai->regs->s0); + EMSG_RAW("s1\t%016" PRIxPTR " a0\t%016" PRIxPTR, + ai->regs->s1, ai->regs->a0); + EMSG_RAW("a1\t%016" PRIxPTR " a2\t%016" PRIxPTR, + ai->regs->a1, ai->regs->a2); + EMSG_RAW("a3\t%016" PRIxPTR " a4\t%016" PRIxPTR, + ai->regs->a3, ai->regs->a4); + EMSG_RAW("a5\t%016" PRIxPTR " a5\t%016" PRIxPTR, + ai->regs->a5, ai->regs->a5); + EMSG_RAW("a6\t%016" PRIxPTR " a7\t%016" PRIxPTR, + ai->regs->a6, ai->regs->a7); + EMSG_RAW("s2\t%016" PRIxPTR " s3\t%016" PRIxPTR, + ai->regs->s2, ai->regs->s3); + EMSG_RAW("s4\t%016" PRIxPTR " s5\t%016" PRIxPTR, + ai->regs->s4, ai->regs->s5); + EMSG_RAW("s6\t%016" PRIxPTR " s7\t%016" PRIxPTR, + ai->regs->s6, ai->regs->s7); + EMSG_RAW("s8\t%016" PRIxPTR " s9\t%016" PRIxPTR, + ai->regs->s8, ai->regs->s9); + EMSG_RAW("s10\t%016" PRIxPTR " s11\t%016" PRIxPTR, + ai->regs->s10, ai->regs->s11); + EMSG_RAW("t3\t%016" PRIxPTR " t4\t%016" PRIxPTR, + ai->regs->t3, ai->regs->t4); + EMSG_RAW("t5\t%016" PRIxPTR " t6\t%016" PRIxPTR, + ai->regs->t5, ai->regs->t6); +} + +/* + * Print abort info and (optionally) stack dump to the console + * @ai kernel-mode abort info. + * @stack_dump true to show a stack trace + */ +static void __abort_print(struct abort_info *ai, bool stack_dump) +{ + assert(!abort_is_user_exception(ai)); + + __print_abort_info(ai, "Core"); + + if (stack_dump) { + trace_printf_helper_raw(TRACE_ERROR, true, + "TEE load address @ %#"PRIxVA, + VCORE_START_VA); + __print_stack_unwind(ai); + } +} + +void abort_print(struct abort_info *ai) +{ + __abort_print(ai, false); +} + +void abort_print_error(struct abort_info *ai) +{ + __abort_print(ai, true); +} + +/* This function must be called from a normal thread */ +void abort_print_current_ts(void) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + struct abort_info ai = { }; + struct ts_session *s = ts_get_current_session(); + + ai.abort_type = tsd->abort_type; + ai.fault_descr = tsd->abort_descr; + ai.va = tsd->abort_va; + ai.pc = tsd->abort_regs.epc; + ai.regs = &tsd->abort_regs; + + if (ai.abort_type != ABORT_TYPE_USER_MODE_PANIC) + __print_abort_info(&ai, "User mode"); + + s->ctx->ops->dump_state(s->ctx); + +#if defined(CFG_FTRACE_SUPPORT) + if (s->ctx->ops->dump_ftrace) { + s->fbuf = NULL; + s->ctx->ops->dump_ftrace(s->ctx); + } +#endif +} + +static void save_abort_info_in_tsd(struct abort_info *ai) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + + tsd->abort_type = ai->abort_type; + tsd->abort_descr = ai->fault_descr; + tsd->abort_va = ai->va; + tsd->abort_regs = *ai->regs; + tsd->abort_core = get_core_pos(); +} + +static void set_abort_info(uint32_t abort_type __unused, + struct thread_abort_regs *regs, + struct abort_info *ai) +{ + ai->fault_descr = regs->cause; + switch (ai->fault_descr) { + case CAUSE_MISALIGNED_FETCH: + case CAUSE_FETCH_ACCESS: + case CAUSE_FETCH_PAGE_FAULT: + case CAUSE_FETCH_GUEST_PAGE_FAULT: + ai->abort_type = ABORT_TYPE_PREFETCH; + break; + case CAUSE_MISALIGNED_LOAD: + case CAUSE_LOAD_ACCESS: + case CAUSE_MISALIGNED_STORE: + case CAUSE_STORE_ACCESS: + case CAUSE_LOAD_PAGE_FAULT: + case CAUSE_STORE_PAGE_FAULT: + case CAUSE_LOAD_GUEST_PAGE_FAULT: + case CAUSE_STORE_GUEST_PAGE_FAULT: + ai->abort_type = ABORT_TYPE_DATA; + break; + default: + ai->abort_type = ABORT_TYPE_UNDEF; + } + + ai->va = regs->tval; + ai->pc = regs->epc; + ai->regs = regs; +} + +static void handle_user_mode_panic(struct abort_info *ai) +{ + /* + * It was a user exception, stop user execution and return + * to TEE Core. + */ + ai->regs->a0 = TEE_ERROR_TARGET_DEAD; + ai->regs->a1 = true; + ai->regs->a2 = 0xdeadbeef; + ai->regs->ra = (vaddr_t)thread_unwind_user_mode; + ai->regs->sp = thread_get_saved_thread_sp(); + ai->regs->status = read_csr(CSR_XSTATUS); + + thread_exit_user_mode(ai->regs->a0, ai->regs->a1, ai->regs->a2, + ai->regs->a3, ai->regs->sp, ai->regs->ra, + ai->regs->status); +} + +#ifdef CFG_WITH_VFP +static void handle_user_mode_vfp(void) +{ + struct ts_session *s = ts_get_current_session(); + + thread_user_enable_vfp(&to_user_mode_ctx(s->ctx)->vfp); +} +#endif /*CFG_WITH_VFP*/ + +#ifdef CFG_WITH_USER_TA + +/* Returns true if the exception originated from user mode */ +bool abort_is_user_exception(struct abort_info *ai) +{ + return (ai->regs->status & CSR_XSTATUS_SPP) == 0; +} + +#else /*CFG_WITH_USER_TA*/ +bool abort_is_user_exception(struct abort_info *ai __unused) +{ + return false; +} +#endif /*CFG_WITH_USER_TA*/ + +#if defined(CFG_WITH_VFP) && defined(CFG_WITH_USER_TA) +static bool is_vfp_fault(struct abort_info *ai) +{ + /* Implement */ + return false; +} +#else /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ +static bool is_vfp_fault(struct abort_info *ai __unused) +{ + return false; +} +#endif /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ + +static enum fault_type get_fault_type(struct abort_info *ai) +{ + if (abort_is_user_exception(ai)) { + if (is_vfp_fault(ai)) + return FAULT_TYPE_USER_MODE_VFP; + return FAULT_TYPE_USER_MODE_PANIC; + } + + if (thread_is_from_abort_mode()) { + abort_print_error(ai); + panic("[abort] abort in abort handler (trap CPU)"); + } + + if (ai->abort_type == ABORT_TYPE_UNDEF) { + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] undefined abort (trap CPU)"); + } + + switch (core_mmu_get_fault_type(ai->fault_descr)) { + case CORE_MMU_FAULT_ALIGNMENT: + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] alignment fault! (trap CPU)"); + break; + + case CORE_MMU_FAULT_ACCESS_BIT: + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] access bit fault! (trap CPU)"); + break; + + case CORE_MMU_FAULT_DEBUG_EVENT: + if (!abort_is_user_exception(ai)) + abort_print(ai); + DMSG("[abort] Ignoring debug event!"); + return FAULT_TYPE_IGNORE; + + case CORE_MMU_FAULT_TRANSLATION: + case CORE_MMU_FAULT_WRITE_PERMISSION: + case CORE_MMU_FAULT_READ_PERMISSION: + return FAULT_TYPE_PAGE_FAULT; + + case CORE_MMU_FAULT_ASYNC_EXTERNAL: + if (!abort_is_user_exception(ai)) + abort_print(ai); + DMSG("[abort] Ignoring async external abort!"); + return FAULT_TYPE_IGNORE; + + case CORE_MMU_FAULT_TAG_CHECK: + if (abort_is_user_exception(ai)) + return FAULT_TYPE_USER_MODE_PANIC; + abort_print_error(ai); + panic("[abort] Tag check fault! (trap CPU)"); + break; + + case CORE_MMU_FAULT_OTHER: + default: + if (!abort_is_user_exception(ai)) + abort_print(ai); + DMSG("[abort] Unhandled fault!"); + return FAULT_TYPE_IGNORE; + } +} + +void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs) +{ + struct abort_info ai; + + set_abort_info(abort_type, regs, &ai); + + switch (get_fault_type(&ai)) { + case FAULT_TYPE_IGNORE: + break; + case FAULT_TYPE_USER_MODE_PANIC: + DMSG("[abort] abort in User mode (TA will panic)"); + save_abort_info_in_tsd(&ai); +#ifdef CFG_WITH_VFP + vfp_disable(); +#endif + handle_user_mode_panic(&ai); + break; +#ifdef CFG_WITH_VFP + case FAULT_TYPE_USER_MODE_VFP: + handle_user_mode_vfp(); + break; +#endif + case FAULT_TYPE_PAGE_FAULT: + default: + if (thread_get_id_may_fail() < 0) { + abort_print_error(&ai); + panic("abort outside thread context"); + } + + if (!abort_is_user_exception(&ai)) { + abort_print_error(&ai); + panic("unhandled page fault abort"); + } + DMSG("[abort] abort in User mode (TA will panic)"); + save_abort_info_in_tsd(&ai); +#ifdef CFG_WITH_VFP + vfp_disable(); +#endif + handle_user_mode_panic(&ai); + break; + } +} diff --git a/optee_os/core/arch/riscv/kernel/arch_scall.c b/optee_os/core/arch/riscv/kernel/arch_scall.c new file mode 100644 index 0000000..546ea68 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/arch_scall.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright 2022-2023 NXP + * Copyright (c) 2014-2022, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TA_CONTEXT_MAX_SIZE (RISCV_XLEN_BYTES * 32) + +#ifdef CFG_UNWIND + +/* Get register values pushed onto the stack by _utee_panic() */ +static void save_panic_regs_rv_ta(struct thread_specific_data *tsd, + unsigned long *pushed) +{ + TEE_Result res = TEE_SUCCESS; + unsigned long s0 = 0; + unsigned long epc = 0; +#if defined(RV32) + unsigned long *stack_s0 = &pushed[2]; + unsigned long *stack_epc = &pushed[3]; +#elif defined(RV64) + unsigned long *stack_s0 = &pushed[0]; + unsigned long *stack_epc = &pushed[1]; +#endif + + res = GET_USER_SCALAR(s0, stack_s0); + if (res) + s0 = 0; + + res = GET_USER_SCALAR(epc, stack_epc); + if (res) + epc = 0; + + tsd->abort_regs = (struct thread_abort_regs){ + .sp = (unsigned long)pushed, + .s0 = s0, + .epc = epc, + }; +} + +void scall_save_panic_stack(struct thread_scall_regs *regs) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + struct ts_session *s = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(s->ctx); + + if (vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE, + (uaddr_t)regs->a1, + TA_CONTEXT_MAX_SIZE)) { + TAMSG_RAW(""); + TAMSG_RAW("Can't unwind invalid user stack 0x%"PRIxUA, + (uaddr_t)regs->a1); + return; + } + + tsd->abort_type = ABORT_TYPE_USER_MODE_PANIC; + tsd->abort_descr = 0; + tsd->abort_va = 0; + + save_panic_regs_rv_ta(tsd, (unsigned long *)regs->a1); +} + +#else /* CFG_UNWIND */ +void scall_save_panic_stack(struct thread_scall_regs *regs __unused) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + + tsd->abort_type = ABORT_TYPE_USER_MODE_PANIC; +} +#endif /* CFG_UNWIND */ diff --git a/optee_os/core/arch/riscv/kernel/arch_scall_rv.S b/optee_os/core/arch/riscv/kernel/arch_scall_rv.S new file mode 100644 index 0000000..2ec90ae --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/arch_scall_rv.S @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ +#include "tee_syscall_numbers.h" +#include "trace_levels.h" +#include +#include +#include +#include +#include +#include + +/* + * uint32_t scall_do_call(struct thread_scall_regs *regs, syscall_t func); + * + * Called from scall_handle_user_ta() + */ +FUNC scall_do_call , : + addi sp, sp, -16 + + /* Save scall regs to t0 */ + mv t0, a0 + + /* Save func to t1 */ + mv t1, a1 + + /* Push return address to stack */ + store_xregs sp, 0, 1 + + /* Load arguments to function */ + load_xregs a0, THREAD_SCALL_REG_A0, 10, 17 + + /* Call the syscall function */ + jalr t1 + + /* Pop return address from stack */ + load_xregs sp, 0, 1 + + addi sp, sp, 16 + ret +END_FUNC scall_do_call + +/* + * void syscall_sys_return(uint32_t ret); + */ +FUNC syscall_sys_return , : + li a1, 0 /* panic = false */ + li a2, 0 /* panic_code = 0 */ + mv a3, t0 /* pointer to struct thread_scall_regs */ + j scall_sys_return_helper +END_FUNC syscall_sys_return + +/* + * void syscall_panic(uint32_t code); + */ +FUNC syscall_panic , : + li a1, 1 /* panic = true */ + mv a2, a0 /* code */ + li a0, TEE_ERROR_TARGET_DEAD + mv a3, t0 /* pointer to struct thread_scall_regs */ + j scall_sys_return_helper +END_FUNC syscall_panic diff --git a/optee_os/core/arch/riscv/kernel/asm-defines.c b/optee_os/core/arch/riscv/kernel/asm-defines.c new file mode 100644 index 0000000..df8f84b --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/asm-defines.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINES +{ + /* struct thread_ctx */ + DEFINE(THREAD_CTX_KERN_SP, offsetof(struct thread_ctx, kern_sp)); + DEFINE(THREAD_CTX_STACK_VA_END, offsetof(struct thread_ctx, + stack_va_end)); + DEFINE(THREAD_CTX_SIZE, sizeof(struct thread_ctx)); + + /* struct thread_core_local */ + DEFINE(THREAD_CORE_LOCAL_SIZE, sizeof(struct thread_core_local)); + DEFINE(THREAD_CORE_LOCAL_HART_ID, + offsetof(struct thread_core_local, hart_id)); + DEFINE(THREAD_CORE_LOCAL_TMP_STACK_VA_END, + offsetof(struct thread_core_local, tmp_stack_va_end)); + DEFINE(THREAD_CORE_LOCAL_CURR_THREAD, + offsetof(struct thread_core_local, curr_thread)); + DEFINE(THREAD_CORE_LOCAL_FLAGS, + offsetof(struct thread_core_local, flags)); + DEFINE(THREAD_CORE_LOCAL_ABT_STACK_VA_END, + offsetof(struct thread_core_local, abt_stack_va_end)); + DEFINE(THREAD_CORE_LOCAL_X10, offsetof(struct thread_core_local, x[0])); + + DEFINE(STACK_TMP_GUARD, STACK_CANARY_SIZE / 2 + STACK_TMP_OFFS); + + /* struct thread_ctx_regs */ + DEFINE(THREAD_CTX_REG_STATUS, offsetof(struct thread_ctx_regs, status)); + DEFINE(THREAD_CTX_REG_RA, offsetof(struct thread_ctx_regs, ra)); + DEFINE(THREAD_CTX_REG_SP, offsetof(struct thread_ctx_regs, sp)); + DEFINE(THREAD_CTX_REG_T0, offsetof(struct thread_ctx_regs, t0)); + DEFINE(THREAD_CTX_REG_S0, offsetof(struct thread_ctx_regs, s0)); + DEFINE(THREAD_CTX_REG_A0, offsetof(struct thread_ctx_regs, a0)); + DEFINE(THREAD_CTX_REG_S2, offsetof(struct thread_ctx_regs, s2)); + DEFINE(THREAD_CTX_REG_T3, offsetof(struct thread_ctx_regs, t3)); + DEFINE(THREAD_CTX_REGS_SIZE, sizeof(struct thread_ctx_regs)); + + /* struct thread_user_mode_rec */ + DEFINE(THREAD_USER_MODE_REC_CTX_REGS_PTR, + offsetof(struct thread_user_mode_rec, ctx_regs_ptr)); + DEFINE(THREAD_USER_MODE_REC_X1, + offsetof(struct thread_user_mode_rec, x[0])); + DEFINE(THREAD_USER_MODE_REC_X4, + offsetof(struct thread_user_mode_rec, x[3])); + DEFINE(THREAD_USER_MODE_REC_X8, + offsetof(struct thread_user_mode_rec, x[4])); + DEFINE(THREAD_USER_MODE_REC_X18, + offsetof(struct thread_user_mode_rec, x[6])); + DEFINE(THREAD_USER_MODE_REC_SIZE, sizeof(struct thread_user_mode_rec)); + + /* struct thread_trap_regs */ + DEFINE(THREAD_TRAP_REG_SP, offsetof(struct thread_trap_regs, sp)); + DEFINE(THREAD_TRAP_REG_RA, offsetof(struct thread_trap_regs, ra)); + DEFINE(THREAD_TRAP_REG_GP, offsetof(struct thread_trap_regs, gp)); + DEFINE(THREAD_TRAP_REG_TP, offsetof(struct thread_trap_regs, tp)); + DEFINE(THREAD_TRAP_REG_T0, offsetof(struct thread_trap_regs, t0)); + DEFINE(THREAD_TRAP_REG_S0, offsetof(struct thread_trap_regs, s0)); + DEFINE(THREAD_TRAP_REG_A0, offsetof(struct thread_trap_regs, a0)); + DEFINE(THREAD_TRAP_REG_T3, offsetof(struct thread_trap_regs, t3)); + DEFINE(THREAD_TRAP_REG_EPC, offsetof(struct thread_trap_regs, epc)); + DEFINE(THREAD_TRAP_REG_STATUS, + offsetof(struct thread_trap_regs, status)); + DEFINE(THREAD_TRAP_REGS_SIZE, sizeof(struct thread_trap_regs)); + + /* struct thread_scall_regs */ + DEFINE(THREAD_SCALL_REG_STATUS, + offsetof(struct thread_scall_regs, status)); + DEFINE(THREAD_SCALL_REG_RA, offsetof(struct thread_scall_regs, ra)); + DEFINE(THREAD_SCALL_REG_SP, offsetof(struct thread_scall_regs, sp)); + DEFINE(THREAD_SCALL_REG_A0, offsetof(struct thread_scall_regs, a0)); + DEFINE(THREAD_SCALL_REGS_SIZE, sizeof(struct thread_scall_regs)); + + /* struct core_mmu_config */ + DEFINE(CORE_MMU_CONFIG_SIZE, sizeof(struct core_mmu_config)); + DEFINE(CORE_MMU_CONFIG_SATP, + offsetof(struct core_mmu_config, satp)); + + /* struct thread_abi_args */ + DEFINE(THREAD_ABI_ARGS_A0, offsetof(struct thread_abi_args, a0)); + DEFINE(THREAD_ABI_ARGS_SIZE, sizeof(struct thread_abi_args)); +} diff --git a/optee_os/core/arch/riscv/kernel/boot.c b/optee_os/core/arch/riscv/kernel/boot.c new file mode 100644 index 0000000..16c424c --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/boot.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PADDR_INVALID ULONG_MAX + +paddr_t start_addr; +unsigned long boot_args[4]; + +uint32_t sem_cpu_sync[CFG_TEE_CORE_NB_CORE]; + +#if defined(CFG_DT) +static int mark_tddram_as_reserved(struct dt_descriptor *dt) +{ + return add_res_mem_dt_node(dt, "optee_core", CFG_TDDRAM_START, + CFG_TDDRAM_SIZE); +} + +static void update_external_dt(void) +{ + struct dt_descriptor *dt = get_external_dt_desc(); + + if (!dt || !dt->blob) + return; + + if (mark_tddram_as_reserved(dt)) + panic("Failed to config secure memory"); +} +#else /*CFG_DT*/ +static void update_external_dt(void) +{ +} +#endif /*!CFG_DT*/ + +void init_sec_mon(unsigned long nsec_entry __maybe_unused) +{ + assert(nsec_entry == PADDR_INVALID); + /* Do nothing as we don't have a secure monitor */ +} + +#ifdef CFG_RISCV_S_MODE +static void start_secondary_cores(void) +{ + size_t i = 0; + size_t pos = get_core_pos(); + + for (i = 0; i < CFG_TEE_CORE_NB_CORE; i++) + if (i != pos && IS_ENABLED(CFG_RISCV_SBI) && + sbi_boot_hart(i, start_addr, i)) + EMSG("Error starting secondary hart %zu", i); +} +#endif + +static void init_runtime(void) +{ + malloc_add_pool(__heap1_start, __heap1_end - __heap1_start); + + IMSG_RAW("\n"); +} + +void init_tee_runtime(void) +{ + core_mmu_init_ta_ram(); + call_preinitcalls(); + call_initcalls(); +} + +static void init_primary(unsigned long nsec_entry) +{ + thread_init_core_local_stacks(); + + /* + * Mask asynchronous exceptions before switch to the thread vector + * as the thread handler requires those to be masked while + * executing with the temporary stack. The thread subsystem also + * asserts that the foreign interrupts are blocked when using most of + * its functions. + */ + thread_set_exceptions(THREAD_EXCP_ALL); + + init_runtime(); + thread_init_boot_thread(); + thread_init_primary(); + thread_init_per_cpu(); + init_sec_mon(nsec_entry); +} + +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak void plat_primary_init_early(void) +{ +} + +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak void boot_primary_init_intc(void) +{ +} + +/* May be overridden in plat-$(PLATFORM)/main.c */ +__weak void boot_secondary_init_intc(void) +{ +} + +void boot_init_primary_early(unsigned long pageable_part __unused, + unsigned long nsec_entry __unused) +{ + unsigned long e = PADDR_INVALID; + + init_primary(e); +} + +void boot_init_primary_late(unsigned long fdt, + unsigned long tos_fw_config __unused) +{ + init_external_dt(fdt); + update_external_dt(); + + IMSG("OP-TEE version: %s", core_v_str); + if (IS_ENABLED(CFG_WARN_INSECURE)) { + IMSG("WARNING: This OP-TEE configuration might be insecure!"); + IMSG("WARNING: Please check https://optee.readthedocs.io/en/latest/architecture/porting_guidelines.html"); + } + IMSG("Primary CPU initializing"); + boot_primary_init_intc(); + init_tee_runtime(); + call_finalcalls(); + IMSG("Primary CPU initialized"); + +#ifdef CFG_RISCV_S_MODE + start_secondary_cores(); +#endif +} + +static void init_secondary_helper(unsigned long nsec_entry) +{ + size_t pos = get_core_pos(); + + IMSG("Secondary CPU %zu initializing", pos); + + /* + * Mask asynchronous exceptions before switch to the thread vector + * as the thread handler requires those to be masked while + * executing with the temporary stack. The thread subsystem also + * asserts that the foreign interrupts are blocked when using most of + * its functions. + */ + thread_set_exceptions(THREAD_EXCP_ALL); + + thread_init_per_cpu(); + init_sec_mon(nsec_entry); + boot_secondary_init_intc(); + + IMSG("Secondary CPU %zu initialized", pos); +} + +void boot_init_secondary(unsigned long nsec_entry __unused) +{ + init_secondary_helper(PADDR_INVALID); +} diff --git a/optee_os/core/arch/riscv/kernel/cache_helpers_rv.S b/optee_os/core/arch/riscv/kernel/cache_helpers_rv.S new file mode 100644 index 0000000..ea67739 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/cache_helpers_rv.S @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#include +#include + +/* + * On the below data cache management, we rely on FENCE instruction. + * The FENCE instruction is used to order device I/O and memory accesses + * as viewed by other RISC-V harts and external devices or coprocessors. + * "fence" below is a pseudo-instruction of "fence iorw, iorw" which + * performs Fence on all memory and I/O. + */ + +/* void dcache_cleaninv_range(void *addr, size_t size); */ +FUNC dcache_cleaninv_range , : + fence + ret +END_FUNC dcache_cleaninv_range + +/* void dcache_clean_range(void *addr, size_t size); */ +FUNC dcache_clean_range , : + fence + ret +END_FUNC dcache_clean_range + +/* void dcache_inv_range(void *addr, size_t size); */ +FUNC dcache_inv_range , : + fence + ret +END_FUNC dcache_inv_range + +/* void dcache_op_all(unsigned long op_type); */ +FUNC dcache_op_all , : + fence + ret +END_FUNC dcache_op_all + +/* void icache_inv_all(void); */ +FUNC icache_inv_all , : + /* + * FENCE.I instruction provides explicit synchronization + * between writes to instruction memory and instruction + * fetches on the same hart. This implies instruction cache + * management operations as result of executing this instruction. + */ + fence.i + ret +END_FUNC icache_inv_all + +/* void icache_inv_range(void *addr, size_t size); */ +FUNC icache_inv_range , : + /* + * RISC-V does not have an instruction to flush a range + * of the I$, therefore, flush it entirely as invoking + * icache_inv_all(). + */ + fence.i + ret +END_FUNC icache_inv_range diff --git a/optee_os/core/arch/riscv/kernel/entry.S b/optee_os/core/arch/riscv/kernel/entry.S new file mode 100644 index 0000000..4e451e0 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/entry.S @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +.section .data +.balign 4 + +#ifdef CFG_BOOT_SYNC_CPU +.equ SEM_CPU_READY, 1 +#endif + + /* + * Setup sp to point to the top of the tmp stack for the current CPU: + * sp is assigned: + * stack_tmp + (hartid + 1) * stack_tmp_stride - STACK_TMP_GUARD + */ +.macro set_sp + /* Unsupported CPU, park it before it breaks something */ + li t1, CFG_TEE_CORE_NB_CORE + csrr t0, CSR_XSCRATCH + bge t0, t1, unhandled_cpu + addi t0, t0, 1 + lw t1, stack_tmp_stride + /* + * t0 = (hartid + 1) + * t1 = value of stack_tmp_stride + * value of stack_tmp_rel = stack_tmp - stack_tmp_rel - STACK_TMP_GUARD + * sp = stack_tmp + (hartid + 1) * stack_tmp_stride - STACK_TMP_GUARD + * = stack_tmp_rel + (value of stack_tmp_rel) + (t0 * t1) + */ + mul t1, t0, t1 + la t2, stack_tmp_rel + lw t0, 0(t2) + add t0, t0, t2 + add sp, t1, t0 +.endm + +.macro cpu_is_ready +#ifdef CFG_BOOT_SYNC_CPU + csrr t0, CSR_XSCRATCH + la t1, sem_cpu_sync + slli t0, t0, 2 + add t1, t1, t0 + li t2, SEM_CPU_READY + sw t2, 0(t1) + fence +#endif +.endm + +.macro set_tp + csrr a0, CSR_XSCRATCH + li a1, THREAD_CORE_LOCAL_SIZE + la tp, thread_core_local + mul a2, a1, a0 + add tp, tp, a2 + sw a0, THREAD_CORE_LOCAL_HART_ID(tp) +.endm + +.macro set_satp + la a1, boot_mmu_config + LDR a0, CORE_MMU_CONFIG_SATP(a1) + csrw CSR_SATP, a0 + sfence.vma zero, zero +.endm + +.macro wait_primary +#ifdef CFG_BOOT_SYNC_CPU + la t0, sem_cpu_sync + li t2, SEM_CPU_READY +1: + fence w, w + lw t1, 0(t0) + bne t1, t2, 1b +#endif +.endm + +.macro wait_secondary +#ifdef CFG_BOOT_SYNC_CPU + la t0, sem_cpu_sync + li t1, CFG_TEE_CORE_NB_CORE + li t2, SEM_CPU_READY +1: + addi t1, t1, -1 + beqz t1, 3f + addi t0, t0, 4 +2: + fence + lw t1, 0(t0) + bne t1, t2, 2b + j 1b +3: +#endif +.endm + +#ifdef CFG_BOOT_SYNC_CPU +#define flush_cpu_semaphores \ + la t0, sem_cpu_sync_start + la t1, sem_cpu_sync_end + fence +#else +#define flush_cpu_semaphores +#endif + +.macro bootargs_entry + /* + * Save boot arguments + */ + la t0, boot_args + /* Save boot hart */ + STR a0, REGOFF(0)(t0) + /* Save FDT address */ + STR a1, REGOFF(1)(t0) +.endm + +FUNC _start , : + /* + * Register usage: + * a0 - if non-NULL holds the hart ID + * a1 - if non-NULL holds the system DTB address + * + * CSR_XSCRATCH - saved a0 + * s1 - saved a1 + */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop +#ifdef CFG_RISCV_M_MODE + csrr a0, CSR_MHARTID +#endif + csrw CSR_XSCRATCH, a0 +#if defined(CFG_DT_ADDR) + li s1, CFG_DT_ADDR +#else + mv s1, a1 /* Save device tree address into s1 */ +#endif + bnez a0, reset_secondary + jal reset_primary + j . +END_FUNC _start + +LOCAL_FUNC reset_primary , : , .identity_map +UNWIND( .cantunwind) + + bootargs_entry + + /* + * Zero bss + */ + lla t0, __bss_start + lla t1, __bss_end + beq t0, t1, 1f +0: + STR zero, (t0) + add t0, t0, RISCV_XLEN_BYTES + bne t0, t1, 0b +1: +#ifdef CFG_RISCV_S_MODE + lla t0, _start + lla t1, start_addr + STR t0, (t1) +#endif + + csrw CSR_SATP, zero + set_sp + set_tp + + jal thread_init_thread_core_local + jal plat_primary_init_early + jal console_init + + mv a0, x0 + la a1, boot_mmu_config + jal core_init_mmu_map + + set_satp + + jal boot_init_primary_early + + /* + * Before entering boot_init_primary_late(), we do these two steps: + * 1. Save current sp to s2, and set sp as threads[0].stack_va_end + * 2. Clear the flag which indicates usage of the temporary stack in the + * current hart's thread_core_local structure. + */ + mv s2, sp + la a0, threads + LDR a0, THREAD_CTX_STACK_VA_END(a0) + mv sp, a0 + jal thread_get_core_local + mv s3, a0 + STR x0, THREAD_CORE_LOCAL_FLAGS(s3) + + mv a0, s1 /* s1 contains saved device tree address */ + mv a1, x0 /* unused */ + jal boot_init_primary_late + + /* + * After returning from boot_init_primary_late(), the flag and sp are + * restored. + */ + li a0, THREAD_CLF_TMP + STR a0, THREAD_CORE_LOCAL_FLAGS(s3) + mv sp, s2 + + cpu_is_ready + flush_cpu_semaphores + wait_secondary + + jal thread_clr_boot_thread + + li a0, TEEABI_OPTEED_RETURN_ENTRY_DONE + la a1, thread_vector_table + j thread_return_to_ree +END_FUNC reset_primary + +LOCAL_FUNC reset_secondary , : , .identity_map +UNWIND( .cantunwind) + wait_primary + csrw CSR_SATP, zero + set_sp + set_tp + set_satp + cpu_is_ready + + jal boot_init_secondary + j . +END_FUNC reset_secondary + +LOCAL_FUNC unhandled_cpu , : + wfi + j unhandled_cpu +END_FUNC unhandled_cpu + +#ifdef CFG_BOOT_SYNC_CPU +LOCAL_DATA sem_cpu_sync_start , : + .word sem_cpu_sync +END_DATA sem_cpu_sync_start + +LOCAL_DATA sem_cpu_sync_end , : + .word sem_cpu_sync + (CFG_TEE_CORE_NB_CORE << 2) +END_DATA sem_cpu_sync_end +#endif + +LOCAL_DATA stack_tmp_rel , : + .word stack_tmp - stack_tmp_rel - STACK_TMP_GUARD +END_DATA stack_tmp_rel + +LOCAL_DATA stack_tmp_stride_rel , : + .word stack_tmp_stride - stack_tmp_stride_rel +END_DATA stack_tmp_stride_rel + + .balign 8 +LOCAL_DATA boot_mmu_config , : /* struct core_mmu_config */ + .skip CORE_MMU_CONFIG_SIZE +END_DATA boot_mmu_config diff --git a/optee_os/core/arch/riscv/kernel/idle.c b/optee_os/core/arch/riscv/kernel/idle.c new file mode 100644 index 0000000..e508372 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/idle.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 NXP + */ + +#include +#include + +void cpu_idle(void) +{ + /* ensure memory operations were complete */ + mb(); + /* stall the hart */ + wfi(); +} diff --git a/optee_os/core/arch/riscv/kernel/kern.ld.S b/optee_os/core/arch/riscv/kernel/kern.ld.S new file mode 100644 index 0000000..e429de3 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/kern.ld.S @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: (BSD-2-Clause AND MIT) */ +/* + * Copyright 2022-2023 NXP + */ + +/* + * Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright (c) 2008-2010 Travis Geiselbrecht + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +/* + * Note: + * Clang 11 (ld.lld) generates non-relocatable reference when using ROUNDDOWN() + * from , which does not work with ASLR. + */ +#define LD_ROUNDDOWN(x, y) ((x) - ((x) % (y))) + +OUTPUT_FORMAT(CFG_KERN_LINKER_FORMAT) +OUTPUT_ARCH(CFG_KERN_LINKER_ARCH) + +ENTRY(_start) +SECTIONS +{ + . = TEE_LOAD_ADDR; + /* Ensure text section is page aligned */ + ASSERT(!(TEE_LOAD_ADDR & (SMALL_PAGE_SIZE - 1)), + "text start should align to 4Kb") + + __text_start = .; + + /* + * Memory between TEE_LOAD_ADDR and page aligned rounded down + * value will be mapped with unpaged "text" section attributes: + * likely to be read-only/executable. + */ + __flatmap_rx_start = LD_ROUNDDOWN(__text_start, SMALL_PAGE_SIZE); + + .text : { + KEEP(*(.text._start)) + __identity_map_init_start = .; + __text_data_start = .; + *(.identity_map.data) + __text_data_end = .; + *(.identity_map .identity_map.*) + __identity_map_init_end = .; + *(.text .text.*) + *(.sram.text.glue_7* .gnu.linkonce.t.*) + . = ALIGN(8); + } + __text_end = .; + +#ifdef CFG_CORE_RODATA_NOEXEC + . = ALIGN(SMALL_PAGE_SIZE); +#endif + __flatmap_rx_size = . - __flatmap_rx_start; + __flatmap_ro_start = .; + + .rodata : ALIGN(8) { + __rodata_start = .; + *(.gnu.linkonce.r.*) + *(.rodata .rodata.*) +#ifndef CFG_CORE_ASLR + . = ALIGN(8); + KEEP(*(SORT(.scattered_array*))); +#endif + . = ALIGN(8); + __rodata_end = .; + } + + .got : { *(.got.plt) *(.got) } + .note.gnu.property : { *(.note.gnu.property) } + .plt : { *(.plt) } + + .ctors : ALIGN(8) { + __ctor_list = .; + KEEP(*(.ctors .ctors.* .init_array .init_array.*)) + __ctor_end = .; + } + .dtors : ALIGN(8) { + __dtor_list = .; + KEEP(*(.dtors .dtors.* .fini_array .fini_array.*)) + __dtor_end = .; + } + + /* Start page aligned read-write memory */ +#ifdef CFG_CORE_RWDATA_NOEXEC + . = ALIGN(SMALL_PAGE_SIZE); +#endif + __flatmap_ro_size = . - __flatmap_ro_start; + __flatmap_rw_start = .; + + .data : ALIGN(8) { + /* writable data */ + __data_start_rom = .; + /* in one segment binaries, the rom data address is on top + of the ram data address */ + __data_start = .; + *(.data .data.* .gnu.linkonce.d.*) + . = ALIGN(8); + /* + * To allow the linker relax accesses to global symbols, + * those need to be within imm12 (signed 12-bit) offsets + * from __global_pointer$. + */ + PROVIDE(__global_pointer$ = . + 0x800 ); + } + + /* uninitialized data */ + .bss : { + __data_end = .; + __bss_start = .; + *(.bss .bss.*) + *(.gnu.linkonce.b.*) + *(COMMON) + . = ALIGN(8); + __bss_end = .; + } + + .heap1 (NOLOAD) : { + /* + * We're keeping track of the padding added before the + * .nozi section so we can do something useful with + * this otherwise wasted memory. + */ + __heap1_start = .; + . += CFG_CORE_HEAP_SIZE; + . = ALIGN(4 * 1024); + __heap1_end = .; + } + /* + * Uninitialized data that shouldn't be zero initialized at + * runtime. + */ + .nozi (NOLOAD) : { + __nozi_start = .; + KEEP(*(.nozi .nozi.*)) + . = ALIGN(16); + __nozi_end = .; + __nozi_stack_start = .; + KEEP(*(.nozi_stack .nozi_stack.*)) + . = ALIGN(8); + __nozi_stack_end = .; + } + +#ifdef CFG_CORE_SANITIZE_KADDRESS + . = TEE_RAM_START + (TEE_RAM_VA_SIZE * 8) / 9 - 8; + . = ALIGN(8); + .asan_shadow : { + __asan_shadow_start = .; + . += TEE_RAM_VA_SIZE / 9; + __asan_shadow_end = .; + __asan_shadow_size = __asan_shadow_end - __asan_shadow_start; + } +#endif /*CFG_CORE_SANITIZE_KADDRESS*/ + + __end = .; + __init_size = __data_end - TEE_LOAD_ADDR; + + /* + * Guard against moving the location counter backwards in the assignment + * below. + */ + ASSERT(. <= (TEE_RAM_START + TEE_RAM_VA_SIZE), + "TEE_RAM_VA_SIZE is too small") + . = TEE_RAM_START + TEE_RAM_VA_SIZE; + + _end_of_ram = .; + + __flatmap_rw_size = _end_of_ram - __flatmap_rw_start; + __get_tee_init_end = .; + + /* + * These regions will not become a normal part of the dumped + * binary, instead some are interpreted by the dump script and + * converted into suitable format for OP-TEE itself to use. + */ + .dynamic : { *(.dynamic) } + .hash : { *(.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + + .rel : { + *(.rel.*) + } + .rela : { + *(.rela.*) + } +#ifndef CFG_CORE_ASLR + ASSERT(SIZEOF(.rel) == 0, "Relocation entries not expected") + ASSERT(SIZEOF(.rela) == 0, "Relocation entries not expected") +#endif + + /DISCARD/ : { + /* Strip unnecessary stuff */ + *(.comment .note .eh_frame .interp) + /* Strip meta variables */ + *(__keep_meta_vars*) + } + +} + +/* Unpaged read-only memories */ +__vcore_unpg_rx_start = __flatmap_rx_start; +__vcore_unpg_ro_start = __flatmap_ro_start; +#ifdef CFG_CORE_RODATA_NOEXEC +__vcore_unpg_rx_size = __flatmap_rx_size; +__vcore_unpg_ro_size = __flatmap_ro_size; +#else +__vcore_unpg_rx_size = __flatmap_rx_size + __flatmap_ro_size; +__vcore_unpg_ro_size = 0; +#endif +__vcore_unpg_rx_end = __vcore_unpg_rx_start + __vcore_unpg_rx_size; +__vcore_unpg_ro_end = __vcore_unpg_ro_start + __vcore_unpg_ro_size; + +/* Unpaged read-write memory */ +__vcore_unpg_rw_start = __flatmap_rw_start; +__vcore_unpg_rw_size = __flatmap_rw_size; +__vcore_unpg_rw_end = __vcore_unpg_rw_start + __vcore_unpg_rw_size; + +#ifdef CFG_CORE_SANITIZE_KADDRESS +__asan_map_start = (__asan_shadow_start / SMALL_PAGE_SIZE) * + SMALL_PAGE_SIZE; +__asan_map_end = ((__asan_shadow_end - 1) / SMALL_PAGE_SIZE) * + SMALL_PAGE_SIZE + SMALL_PAGE_SIZE; +__asan_map_size = __asan_map_end - __asan_map_start; +#endif /*CFG_CORE_SANITIZE_KADDRESS*/ diff --git a/optee_os/core/arch/riscv/kernel/link.mk b/optee_os/core/arch/riscv/kernel/link.mk new file mode 100644 index 0000000..3d1000d --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/link.mk @@ -0,0 +1,143 @@ +link-out-dir = $(out-dir)/core + +link-script = $(if $(wildcard $(platform-dir)/kern.ld.S), \ + $(platform-dir)/kern.ld.S, \ + $(arch-dir)/kernel/kern.ld.S) +link-script-pp = $(link-out-dir)/kern.ld +link-script-dep = $(link-out-dir)/.kern.ld.d + +link-ldflags-common += $(call ld-option,--no-warn-rwx-segments) + +link-ldflags = $(LDFLAGS) +ifeq ($(CFG_CORE_ASLR),y) +link-ldflags += -pie -Bsymbolic -z norelro $(ldflag-apply-dynamic-relocs) +endif + +link-ldflags += -T $(link-script-pp) -Map=$(link-out-dir)/tee.map +link-ldflags += --sort-section=alignment +link-ldflags += --fatal-warnings +link-ldflags += --gc-sections +link-ldflags += $(link-ldflags-common) + +link-ldadd = $(LDADD) +link-ldadd += $(ldflags-external) +link-ldadd += $(libdeps) +link-objs := $(objs) + +ldargs-tee.elf := $(link-ldflags) $(link-objs) $(link-out-dir)/version.o \ + $(link-ldadd) $(libgcccore) + +link-script-cppflags := \ + $(filter-out $(CPPFLAGS_REMOVE) $(cppflags-remove), \ + $(nostdinccore) $(CPPFLAGS) \ + $(addprefix -I,$(incdirscore) $(link-out-dir)) \ + $(cppflagscore)) + +ldargs-all_objs := -T $(link-script) --no-check-sections \ + $(link-ldflags-common) \ + $(link-objs) $(link-ldadd) $(libgcccore) +cleanfiles += $(link-out-dir)/all_objs.o +$(link-out-dir)/all_objs.o: $(objs) $(libdeps) $(MAKEFILE_LIST) + @$(cmd-echo-silent) ' LD $@' + $(q)$(LDcore) $(ldargs-all_objs) -o $@ + +-include $(link-script-dep) + +link-script-extra-deps += $(conf-file) +cleanfiles += $(link-script-pp) $(link-script-dep) +$(link-script-pp): $(link-script) $(link-script-extra-deps) + @$(cmd-echo-silent) ' CPP $@' + @mkdir -p $(dir $@) + $(q)$(CPPcore) -P -MT $@ -MD -MF $(link-script-dep) \ + $(link-script-cppflags) $< -o $@ + +define update-buildcount + @$(cmd-echo-silent) ' UPD $(1)' + $(q)if [ ! -f $(1) ]; then \ + mkdir -p $(dir $(1)); \ + echo 1 >$(1); \ + else \ + expr 0`cat $(1)` + 1 >$(1); \ + fi +endef + +# filter-out to workaround objdump warning +version-o-cflags = $(filter-out -g3,$(core-platform-cflags) \ + $(platform-cflags) $(cflagscore)) +# SOURCE_DATE_EPOCH defined for reproducible builds +ifneq ($(SOURCE_DATE_EPOCH),) +date-opts = -d @$(SOURCE_DATE_EPOCH) +endif +DATE_STR = `LC_ALL=C date -u $(date-opts)` +BUILD_COUNT_STR = `cat $(link-out-dir)/.buildcount` +CORE_CC_VERSION = `$(CCcore) -v 2>&1 | grep "version " | sed 's/ *$$//'` +define gen-version-o + $(call update-buildcount,$(link-out-dir)/.buildcount) + @$(cmd-echo-silent) ' GEN $(link-out-dir)/version.o' + $(q)echo -e "const char core_v_str[] =" \ + "\"$(TEE_IMPL_VERSION) \"" \ + "\"($(CORE_CC_VERSION)) \"" \ + "\"#$(BUILD_COUNT_STR) \"" \ + "\"$(DATE_STR) \"" \ + "\"$(CFG_KERN_LINKER_ARCH)\";\n" \ + | $(CCcore) $(version-o-cflags) \ + -xc - -c -o $(link-out-dir)/version.o +endef + +$(link-out-dir)/version.o: + $(call gen-version-o) + +-include $(link-out-dir)/.tee.elf.cmd +define check-link-objs +$(if $(strip $(filter-out $(link-objs), $(old-link-objs)) + $(filter-out $(old-link-objs), $(link-objs))), FORCE_LINK := FORCE) +endef +#$(eval $(call check-link-objs)) + +all: $(link-out-dir)/tee.elf +cleanfiles += $(link-out-dir)/tee.elf $(link-out-dir)/tee.map +cleanfiles += $(link-out-dir)/version.o +cleanfiles += $(link-out-dir)/.buildcount +cleanfiles += $(link-out-dir)/.tee.elf.cmd +$(link-out-dir)/tee.elf: $(link-objs) $(libdeps) $(link-script-pp) $(FORCE_LINK) + $(call gen-version-o) + @echo "old-link-objs := $(link-objs)" >$(link-out-dir)/.tee.elf.cmd + @$(cmd-echo-silent) ' LD $@' + $(q)$(LDcore) $(ldargs-tee.elf) -o $@ + +all: $(link-out-dir)/tee.dmp +cleanfiles += $(link-out-dir)/tee.dmp +$(link-out-dir)/tee.dmp: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' OBJDUMP $@' + $(q)$(OBJDUMPcore) -l -x -d $< > $@ + +all: $(link-out-dir)/tee.bin +cleanfiles += $(link-out-dir)/tee.bin +$(link-out-dir)/tee.bin: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' GEN $@' + $(q)$(OBJCOPYcore) -O binary $< $@ + +all: $(link-out-dir)/tee.symb_sizes +cleanfiles += $(link-out-dir)/tee.symb_sizes +$(link-out-dir)/tee.symb_sizes: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' GEN $@' + $(q)$(NMcore) --print-size --reverse-sort --size-sort $< > $@ + +cleanfiles += $(link-out-dir)/tee.mem_usage +ifneq ($(filter mem_usage,$(MAKECMDGOALS)),) +mem_usage: $(link-out-dir)/tee.mem_usage + +$(link-out-dir)/tee.mem_usage: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' GEN $@' + $(q)$(PYTHON3) ./scripts/mem_usage.py $< > $@ +endif + +cleanfiles += $(link-out-dir)/tee-raw.bin +$(link-out-dir)/tee-raw.bin: $(link-out-dir)/tee.elf + @$(cmd-echo-silent) ' GEN $@' + $(q)$(OBJCOPYcore) -O binary $< $@ + +cleanfiles += $(link-out-dir)/tee.srec +$(link-out-dir)/tee.srec: $(link-out-dir)/tee-raw.bin + @$(cmd-echo-silent) ' SREC $@' + $(q)$(OBJCOPYcore) -I binary -O srec $(SRECFLAGS) $< $@ diff --git a/optee_os/core/arch/riscv/kernel/sbi.c b/optee_os/core/arch/riscv/kernel/sbi.c new file mode 100644 index 0000000..8d82040 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/sbi.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 NXP + */ + +#include +#include + +struct sbiret { + long error; + long value; +}; + +#define _sbi_ecall(ext, fid, arg0, arg1, arg2, arg3, arg4, arg5, ...) ({ \ + register unsigned long a0 asm("a0") = (unsigned long)arg0; \ + register unsigned long a1 asm("a1") = (unsigned long)arg1; \ + register unsigned long a2 asm("a2") = (unsigned long)arg2; \ + register unsigned long a3 asm("a3") = (unsigned long)arg3; \ + register unsigned long a4 asm("a4") = (unsigned long)arg4; \ + register unsigned long a5 asm("a5") = (unsigned long)arg5; \ + register unsigned long a6 asm("a6") = (unsigned long)fid; \ + register unsigned long a7 asm("a7") = (unsigned long)ext; \ + asm volatile ("ecall" \ + : "+r" (a0), "+r" (a1) \ + : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r"(a6), "r"(a7) \ + : "memory"); \ + (struct sbiret){ .error = a0, .value = a1 }; \ +}) + +#define sbi_ecall(...) _sbi_ecall(__VA_ARGS__, 0, 0, 0, 0, 0, 0, 0) + +void sbi_console_putchar(int ch) +{ + sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, (unsigned long)ch); +} + +int sbi_boot_hart(uint32_t hart_id, paddr_t start_addr, unsigned long arg) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_START, hart_id, start_addr, arg); + + return ret.error; +} diff --git a/optee_os/core/arch/riscv/kernel/sbi_console.c b/optee_os/core/arch/riscv/kernel/sbi_console.c new file mode 100644 index 0000000..18c628d --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/sbi_console.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_RISCV_SBI_CONSOLE + +struct sbi_console_data { + struct serial_chip chip; +}; + +static struct sbi_console_data console_data __nex_bss; +static unsigned int sbi_console_global_lock __nex_bss = SPINLOCK_UNLOCK; + +static void sbi_console_lock_global(void) +{ + cpu_spin_lock(&sbi_console_global_lock); +} + +static void sbi_console_unlock_global(void) +{ + cpu_spin_unlock(&sbi_console_global_lock); +} + +static void sbi_console_flush(struct serial_chip *chip __unused) +{ +} + +static void sbi_console_putc(struct serial_chip *chip __unused, + int ch) +{ + sbi_console_lock_global(); + sbi_console_putchar(ch); + sbi_console_unlock_global(); +} + +static const struct serial_ops sbi_console_ops = { + .flush = sbi_console_flush, + .putc = sbi_console_putc, +}; + +static void sbi_console_init(struct sbi_console_data *pd) +{ + pd->chip.ops = &sbi_console_ops; +} + +void console_init(void) +{ + sbi_console_init(&console_data); + register_serial_console(&console_data.chip); +} + +#endif /*CFG_RISCV_SBI_CONSOLE*/ + diff --git a/optee_os/core/arch/riscv/kernel/spinlock.S b/optee_os/core/arch/riscv/kernel/spinlock.S new file mode 100644 index 0000000..83aa4fd --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/spinlock.S @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 NXP + */ + +#include +#include +#include + +/* void __cpu_spin_lock(unsigned int *lock) */ +FUNC __cpu_spin_lock , : + addi sp, sp, -(RISCV_XLEN_BYTES * 2) + STR s0, 0(sp) + STR ra, 8(sp) + mv s0, a0 +1: + mv a0, s0 + jal __cpu_spin_trylock + addiw a0, a0, 0 + bnez a0, 1b + LDR ra, 8(sp) + LDR s0, 0(sp) + addi sp, sp, (RISCV_XLEN_BYTES * 2) + ret +END_FUNC __cpu_spin_lock + + +/* void __cpu_spin_unlock(unsigned int *lock)*/ +FUNC __cpu_spin_unlock , : + fence rw, w + amoswap.w x0, x0, 0(a0) + ret +END_FUNC __cpu_spin_unlock + +/* unsigned int __cpu_spin_trylock(unsigned int *lock) */ +FUNC __cpu_spin_trylock , : + li t0, SPINLOCK_LOCK + amoswap.w a0, t0, 0(a0) + fence r,rw + ret +END_FUNC __cpu_spin_trylock + diff --git a/optee_os/core/arch/riscv/kernel/sub.mk b/optee_os/core/arch/riscv/kernel/sub.mk new file mode 100644 index 0000000..683e958 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/sub.mk @@ -0,0 +1,30 @@ +srcs-y += spinlock.S +srcs-y += cache_helpers_rv.S +srcs-y += idle.c +srcs-$(CFG_RISCV_TIME_SOURCE_RDTIME) += tee_time_rdtime.c +srcs-$(CFG_RISCV_SBI) += sbi.c +srcs-$(CFG_RISCV_SBI_CONSOLE) += sbi_console.c +srcs-y += boot.c +srcs-y += entry.S +srcs-y += abort.c +srcs-y += thread_rv.S +srcs-y += thread_arch.c +srcs-y += arch_scall_rv.S +srcs-y += arch_scall.c +srcs-$(CFG_UNWIND) += unwind_rv.c +srcs-y += thread_optee_abi.c +srcs-y += thread_optee_abi_rv.S +asm-defines-y += asm-defines.c + +ifeq ($(CFG_SYSCALL_FTRACE),y) +# We would not like to profile thread.c file as it provide common APIs +# that are needed for ftrace framework to trace syscalls. So profiling +# this file could create an incorrect cyclic behaviour. +cflags-remove-thread_arch.c-y += -pg +# Tracing abort dump files corrupts the stack trace. So exclude them +# from profiling. +cflags-remove-abort.c-y += -pg +ifeq ($(CFG_UNWIND),y) +cflags-remove-unwind_rv.c-y += -pg +endif +endif diff --git a/optee_os/core/arch/riscv/kernel/tee_time_rdtime.c b/optee_os/core/arch/riscv/kernel/tee_time_rdtime.c new file mode 100644 index 0000000..90caf90 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/tee_time_rdtime.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include + +__noprof uint64_t read_time(void) +{ + uint64_t time = 0; + uint32_t hi __maybe_unused = 0; + uint32_t lo __maybe_unused = 0; + +#ifdef CFG_RISCV_M_MODE + time = clint_get_mtime(); +#endif /*CFG_RISCV_M_MODE*/ + +#ifdef CFG_RISCV_S_MODE +#ifdef RV32 + do { + hi = read_csr(timeh); + lo = read_csr(time); + } while (hi != read_csr(timeh)); + + time = SHIFT_U64(hi, 32) | lo; +#else /*RV64*/ + time = rdtime(); +#endif /*RV32*/ +#endif /*CFG_RISCV_S_MODE*/ + + return time; +} + +static TEE_Result riscv_get_sys_time(TEE_Time *time) +{ + uint64_t tm = read_time(); + uint64_t rate = read_cntfrq(); + + time->seconds = tm / rate; + time->millis = (tm % rate) / (rate / TEE_TIME_MILLIS_BASE); + + return TEE_SUCCESS; +} + +static const struct time_source riscv_time_source_rdtime = { + .name = "risc-v rdtime", + .protection_level = 1000, + .get_sys_time = riscv_get_sys_time, +}; + +REGISTER_TIME_SOURCE(riscv_time_source_rdtime) diff --git a/optee_os/core/arch/riscv/kernel/thread_arch.c b/optee_os/core/arch/riscv/kernel/thread_arch.c new file mode 100644 index 0000000..9f40ad4 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/thread_arch.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + * Copyright (c) 2016-2022, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020-2021, Arm Limited + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * This function is called as a guard after each ABI call which is not + * supposed to return. + */ +void __noreturn __panic_at_abi_return(void) +{ + panic(); +} + +/* This function returns current masked exception bits. */ +uint32_t __nostackcheck thread_get_exceptions(void) +{ + uint32_t xie = read_csr(CSR_XIE) & THREAD_EXCP_ALL; + + return xie ^ THREAD_EXCP_ALL; +} + +void __nostackcheck thread_set_exceptions(uint32_t exceptions) +{ + /* Foreign interrupts must not be unmasked while holding a spinlock */ + if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) + assert_have_no_spinlock(); + + /* + * In ARM, the bits in DAIF register are used to mask the exceptions. + * While in RISC-V, the bits in CSR XIE are used to enable(unmask) + * corresponding interrupt sources. To not modify the function of + * thread_set_exceptions(), we should "invert" the bits in "exceptions". + * The corresponding bits in "exceptions" will be inverted so they will + * be cleared when we write the final value into CSR XIE. So that we + * can mask those exceptions. + */ + exceptions &= THREAD_EXCP_ALL; + exceptions ^= THREAD_EXCP_ALL; + + barrier(); + write_csr(CSR_XIE, exceptions); + barrier(); +} + +uint32_t __nostackcheck thread_mask_exceptions(uint32_t exceptions) +{ + uint32_t state = thread_get_exceptions(); + + thread_set_exceptions(state | (exceptions & THREAD_EXCP_ALL)); + return state; +} + +void __nostackcheck thread_unmask_exceptions(uint32_t state) +{ + thread_set_exceptions(state & THREAD_EXCP_ALL); +} + +static void thread_lazy_save_ns_vfp(void) +{ + static_assert(!IS_ENABLED(CFG_WITH_VFP)); +} + +static void thread_lazy_restore_ns_vfp(void) +{ + static_assert(!IS_ENABLED(CFG_WITH_VFP)); +} + +static void setup_unwind_user_mode(struct thread_scall_regs *regs) +{ + regs->ra = (uintptr_t)thread_unwind_user_mode; + regs->status = read_csr(CSR_XSTATUS); + regs->sp = thread_get_saved_thread_sp(); +} + +static void thread_unhandled_trap(struct thread_trap_regs *regs __unused, + unsigned long cause __unused) +{ + DMSG("Unhandled trap xepc:0x%016lx xcause:0x%016lx xtval:0x%016lx", + read_csr(CSR_XEPC), read_csr(CSR_XCAUSE), read_csr(CSR_XTVAL)); + panic(); +} + +void thread_scall_handler(struct thread_scall_regs *regs) +{ + struct ts_session *sess = NULL; + uint32_t state = 0; + + /* Enable native interrupts */ + state = thread_get_exceptions(); + thread_unmask_exceptions(state & ~THREAD_EXCP_NATIVE_INTR); + + thread_user_save_vfp(); + + sess = ts_get_current_session(); + + /* Restore foreign interrupts which are disabled on exception entry */ + thread_restore_foreign_intr(); + + assert(sess && sess->handle_scall); + + if (!sess->handle_scall(regs)) { + setup_unwind_user_mode(regs); + thread_exit_user_mode(regs->a0, regs->a1, regs->a2, + regs->a3, regs->sp, regs->ra, + regs->status); + } +} + +static void copy_scall_to_trap(struct thread_scall_regs *scall_regs, + struct thread_trap_regs *trap_regs) +{ + trap_regs->a0 = scall_regs->a0; + trap_regs->a1 = scall_regs->a1; + trap_regs->a2 = scall_regs->a2; + trap_regs->a3 = scall_regs->a3; + trap_regs->a4 = scall_regs->a4; + trap_regs->a5 = scall_regs->a5; + trap_regs->a6 = scall_regs->a6; + trap_regs->a7 = scall_regs->a7; + trap_regs->t0 = scall_regs->t0; + trap_regs->t1 = scall_regs->t1; +} + +static void copy_trap_to_scall(struct thread_trap_regs *trap_regs, + struct thread_scall_regs *scall_regs) +{ + *scall_regs = (struct thread_scall_regs) { + .status = trap_regs->status, + .ra = trap_regs->ra, + .a0 = trap_regs->a0, + .a1 = trap_regs->a1, + .a2 = trap_regs->a2, + .a3 = trap_regs->a3, + .a4 = trap_regs->a4, + .a5 = trap_regs->a5, + .a6 = trap_regs->a6, + .a7 = trap_regs->a7, + .t0 = trap_regs->t0, + .t1 = trap_regs->t1, + }; +} + +static void thread_user_ecall_handler(struct thread_trap_regs *trap_regs) +{ + struct thread_scall_regs scall_regs; + struct thread_core_local *l = thread_get_core_local(); + int ct = l->curr_thread; + + copy_trap_to_scall(trap_regs, &scall_regs); + thread_scall_handler(&scall_regs); + copy_scall_to_trap(&scall_regs, trap_regs); + /* + * Save kernel sp we'll had at the beginning of this function. + * This is when this TA has called another TA because + * __thread_enter_user_mode() also saves the stack pointer in this + * field. + */ + threads[ct].kern_sp = (unsigned long)(trap_regs + 1); + /* + * We are returning to U-Mode, on return, the program counter + * is set to xsepc (pc=xepc), we add 4 (size of an instruction) + * to continue to next instruction. + */ + trap_regs->epc += 4; +} + +static void copy_trap_to_abort(struct thread_trap_regs *trap_regs, + struct thread_abort_regs *abort_regs) +{ + *abort_regs = (struct thread_abort_regs) { + .status = trap_regs->status, + .ra = trap_regs->ra, + .sp = trap_regs->sp, + .gp = trap_regs->gp, + .tp = trap_regs->tp, + .t0 = trap_regs->t0, + .t1 = trap_regs->t1, + .t2 = trap_regs->t2, + .s0 = trap_regs->s0, + .s1 = trap_regs->s1, + .a0 = trap_regs->a0, + .a1 = trap_regs->a1, + .a2 = trap_regs->a2, + .a3 = trap_regs->a3, + .a4 = trap_regs->a4, + .a5 = trap_regs->a5, + .a6 = trap_regs->a6, + .a7 = trap_regs->a7, + .s2 = trap_regs->s2, + .s3 = trap_regs->s3, + .s4 = trap_regs->s4, + .s5 = trap_regs->s5, + .s6 = trap_regs->s6, + .s7 = trap_regs->s7, + .s8 = trap_regs->s8, + .s9 = trap_regs->s9, + .s10 = trap_regs->s10, + .s11 = trap_regs->s11, + .t3 = trap_regs->t3, + .t4 = trap_regs->t4, + .t5 = trap_regs->t5, + .t6 = trap_regs->t6, + }; +} + +static void thread_abort_handler(struct thread_trap_regs *trap_regs, + unsigned long cause) +{ + struct thread_abort_regs abort_regs = { }; + + assert(cause == read_csr(CSR_XCAUSE)); + copy_trap_to_abort(trap_regs, &abort_regs); + abort_regs.cause = read_csr(CSR_XCAUSE); + abort_regs.epc = read_csr(CSR_XEPC); + abort_regs.tval = read_csr(CSR_XTVAL); + abort_regs.satp = read_csr(CSR_SATP); + abort_handler(cause, &abort_regs); +} + +static void thread_exception_handler(unsigned long cause, + struct thread_trap_regs *regs) +{ + switch (cause) { + case CAUSE_USER_ECALL: + thread_user_ecall_handler(regs); + break; + default: + thread_abort_handler(regs, cause); + break; + } +} + +static void thread_irq_handler(void) +{ + interrupt_main_handler(); +} + +static void thread_interrupt_handler(unsigned long cause, + struct thread_trap_regs *regs) +{ + switch (cause & LONG_MAX) { + case IRQ_XTIMER: + clear_csr(CSR_XIE, CSR_XIE_TIE); + break; + case IRQ_XSOFT: + thread_unhandled_trap(regs, cause); + break; + case IRQ_XEXT: + thread_irq_handler(); + break; + default: + thread_unhandled_trap(regs, cause); + } +} + +void thread_trap_handler(long cause, unsigned long epc __unused, + struct thread_trap_regs *regs, + bool user __maybe_unused) +{ + /* + * The Interrupt bit (XLEN-1) in the cause register is set + * if the trap was caused by an interrupt. + */ + if (cause < 0) + thread_interrupt_handler(cause, regs); + /* + * Otherwise, cause is never written by the implementation, + * though it may be explicitly written by software. + */ + else + thread_exception_handler(cause, regs); +} + +static void init_regs(struct thread_ctx *thread, uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6, uint32_t a7, void *pc) +{ + thread->regs.ra = (uintptr_t)pc; + + /* Set up xstatus */ + thread->regs.status = read_csr(CSR_XSTATUS); + + /* Reinitialize stack pointer */ + thread->regs.sp = thread->stack_va_end; + + /* + * Copy arguments into context. This will make the + * arguments appear in a0-a7 when thread is started. + */ + thread->regs.a0 = a0; + thread->regs.a1 = a1; + thread->regs.a2 = a2; + thread->regs.a3 = a3; + thread->regs.a4 = a4; + thread->regs.a5 = a5; + thread->regs.a6 = a6; + thread->regs.a7 = a7; +} + +static void __thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6, uint32_t a7, + void *pc) +{ + struct thread_core_local *l = thread_get_core_local(); + bool found_thread = false; + size_t n = 0; + + assert(l->curr_thread == THREAD_ID_INVALID); + + thread_lock_global(); + + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].state == THREAD_STATE_FREE) { + threads[n].state = THREAD_STATE_ACTIVE; + found_thread = true; + break; + } + } + + thread_unlock_global(); + + if (!found_thread) + return; + + l->curr_thread = n; + + threads[n].flags = 0; + init_regs(threads + n, a0, a1, a2, a3, a4, a5, a6, a7, pc); + + thread_lazy_save_ns_vfp(); + + l->flags &= ~THREAD_CLF_TMP; + + thread_resume(&threads[n].regs); + /*NOTREACHED*/ + panic(); +} + +void thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, + uint32_t a4, uint32_t a5) +{ + __thread_alloc_and_run(a0, a1, a2, a3, a4, a5, 0, 0, + thread_std_abi_entry); +} + +static void copy_a0_to_a3(struct thread_ctx_regs *regs, uint32_t a0, + uint32_t a1, uint32_t a2, uint32_t a3) +{ + regs->a0 = a0; + regs->a1 = a1; + regs->a2 = a2; + regs->a3 = a3; +} + +static bool is_from_user(unsigned long status) +{ + return (status & CSR_XSTATUS_SPP) == 0; +} + +#ifdef CFG_SYSCALL_FTRACE +static void __noprof ftrace_suspend(void) +{ + struct ts_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); + + if (s && s->fbuf) + s->fbuf->syscall_trace_suspended = true; +} + +static void __noprof ftrace_resume(void) +{ + struct ts_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); + + if (s && s->fbuf) + s->fbuf->syscall_trace_suspended = false; +} +#else +static void __maybe_unused __noprof ftrace_suspend(void) +{ +} + +static void __noprof ftrace_resume(void) +{ +} +#endif + +static bool is_user_mode(struct thread_ctx_regs *regs) +{ + return is_from_user((uint32_t)regs->status); +} + +vaddr_t thread_get_saved_thread_sp(void) +{ + struct thread_core_local *l = thread_get_core_local(); + int ct = l->curr_thread; + + assert(ct != THREAD_ID_INVALID); + return threads[ct].kern_sp; +} + +void thread_resume_from_rpc(uint32_t thread_id, uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3) +{ + size_t n = thread_id; + struct thread_core_local *l = thread_get_core_local(); + bool found_thread = false; + + assert(l->curr_thread == THREAD_ID_INVALID); + + thread_lock_global(); + + if (n < CFG_NUM_THREADS && threads[n].state == THREAD_STATE_SUSPENDED) { + threads[n].state = THREAD_STATE_ACTIVE; + found_thread = true; + } + + thread_unlock_global(); + + if (!found_thread) + return; + + l->curr_thread = n; + + if (threads[n].have_user_map) { + core_mmu_set_user_map(&threads[n].user_map); + if (threads[n].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) + tee_ta_ftrace_update_times_resume(); + } + + if (is_user_mode(&threads[n].regs)) + tee_ta_update_session_utime_resume(); + + /* + * Return from RPC to request service of a foreign interrupt must not + * get parameters from non-secure world. + */ + if (threads[n].flags & THREAD_FLAGS_COPY_ARGS_ON_RETURN) { + copy_a0_to_a3(&threads[n].regs, a0, a1, a2, a3); + threads[n].flags &= ~THREAD_FLAGS_COPY_ARGS_ON_RETURN; + } + + thread_lazy_save_ns_vfp(); + + if (threads[n].have_user_map) + ftrace_resume(); + + l->flags &= ~THREAD_CLF_TMP; + thread_resume(&threads[n].regs); + /*NOTREACHED*/ + panic(); +} + +void thread_state_free(void) +{ + struct thread_core_local *l = thread_get_core_local(); + int ct = l->curr_thread; + + assert(ct != THREAD_ID_INVALID); + + thread_lazy_restore_ns_vfp(); + + thread_lock_global(); + + assert(threads[ct].state == THREAD_STATE_ACTIVE); + threads[ct].state = THREAD_STATE_FREE; + threads[ct].flags = 0; + l->curr_thread = THREAD_ID_INVALID; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + thread_unlock_global(); +} + +int thread_state_suspend(uint32_t flags, unsigned long status, vaddr_t pc) +{ + struct thread_core_local *l = thread_get_core_local(); + int ct = l->curr_thread; + + assert(ct != THREAD_ID_INVALID); + + if (core_mmu_user_mapping_is_active()) + ftrace_suspend(); + + thread_check_canaries(); + + if (is_from_user(status)) { + thread_user_save_vfp(); + tee_ta_update_session_utime_suspend(); + tee_ta_gprof_sample_pc(pc); + } + thread_lazy_restore_ns_vfp(); + + thread_lock_global(); + + assert(threads[ct].state == THREAD_STATE_ACTIVE); + threads[ct].flags |= flags; + threads[ct].regs.status = status; + threads[ct].regs.ra = pc; + threads[ct].state = THREAD_STATE_SUSPENDED; + + threads[ct].have_user_map = core_mmu_user_mapping_is_active(); + if (threads[ct].have_user_map) { + if (threads[ct].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) + tee_ta_ftrace_update_times_suspend(); + core_mmu_get_user_map(&threads[ct].user_map); + core_mmu_set_user_map(NULL); + } + + l->curr_thread = THREAD_ID_INVALID; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + + thread_unlock_global(); + + return ct; +} + +bool thread_init_stack(uint32_t thread_id, vaddr_t sp) +{ + if (thread_id >= CFG_NUM_THREADS) + return false; + threads[thread_id].stack_va_end = sp; + return true; +} + +static void init_user_kcode(void) +{ +} + +void thread_init_primary(void) +{ + /* Initialize canaries around the stacks */ + thread_init_canaries(); + + init_user_kcode(); +} + +static vaddr_t get_trap_vect(void) +{ + return (vaddr_t)thread_trap_vect; +} + +void thread_init_tvec(void) +{ + unsigned long tvec = (unsigned long)get_trap_vect(); + + static_assert(sizeof(struct thread_trap_regs) % 16 == 0); + write_csr(CSR_XTVEC, tvec); + assert(read_csr(CSR_XTVEC) == tvec); +} + +void thread_init_per_cpu(void) +{ + thread_init_tvec(); + /* + * We may receive traps from now, therefore, zeroize xSCRATCH such + * that thread_trap_vect() can distinguish between user traps + * and kernel traps. + */ + write_csr(CSR_XSCRATCH, 0); +#ifndef CFG_PAN + /* + * Allow access to user pages. When CFG_PAN is enabled, the SUM bit will + * be set and clear at runtime when necessary. + */ + set_csr(CSR_XSTATUS, CSR_XSTATUS_SUM); +#endif +} + +static void set_ctx_regs(struct thread_ctx_regs *regs, unsigned long a0, + unsigned long a1, unsigned long a2, unsigned long a3, + unsigned long user_sp, unsigned long entry_func, + unsigned long status, + struct thread_pauth_keys *keys __unused) +{ + *regs = (struct thread_ctx_regs){ + .a0 = a0, + .a1 = a1, + .a2 = a2, + .a3 = a3, + .sp = user_sp, + .ra = entry_func, + .status = status + }; +} + +uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, + unsigned long a2, unsigned long a3, + unsigned long user_sp, + unsigned long entry_func, + bool is_32bit __unused, + uint32_t *exit_status0, + uint32_t *exit_status1) +{ + unsigned long status = 0; + uint32_t exceptions = 0; + uint32_t rc = 0; + struct thread_ctx_regs *regs = NULL; + + tee_ta_update_session_utime_resume(); + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + regs = thread_get_ctx_regs(); + status = read_csr(CSR_XSTATUS); + status |= CSR_XSTATUS_PIE; /* Previous interrupt is enabled */ + status = set_field_u64(status, CSR_XSTATUS_SPP, PRV_U); + set_ctx_regs(regs, a0, a1, a2, a3, user_sp, entry_func, status, NULL); + rc = __thread_enter_user_mode(regs, exit_status0, exit_status1); + thread_unmask_exceptions(exceptions); + + return rc; +} diff --git a/optee_os/core/arch/riscv/kernel/thread_optee_abi.c b/optee_os/core/arch/riscv/kernel/thread_optee_abi.c new file mode 100644 index 0000000..d94da93 --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/thread_optee_abi.c @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2021, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool thread_prealloc_rpc_cache; +static unsigned int thread_rpc_pnum; + +static_assert(NOTIF_VALUE_DO_BOTTOM_HALF == + OPTEE_ABI_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF); + +void thread_handle_fast_abi(struct thread_abi_args *args) +{ + thread_check_canaries(); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && + virt_set_guest(args->a7)) { + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; + goto out; + } + + tee_entry_fast(args); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + +out: + /* Fast handlers must not unmask any exceptions */ + assert(thread_get_exceptions() == THREAD_EXCP_ALL); +} + +uint32_t thread_handle_std_abi(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4, uint32_t a5, + uint32_t a6 __unused, uint32_t a7 __maybe_unused) +{ + uint32_t rv = OPTEE_ABI_RETURN_OK; + + thread_check_canaries(); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && virt_set_guest(a7)) + return OPTEE_ABI_RETURN_ENOTAVAIL; + + /* + * thread_resume_from_rpc() and thread_alloc_and_run() only return + * on error. Successful return is done via thread_exit() or + * thread_rpc(). + */ + if (a0 == OPTEE_ABI_CALL_RETURN_FROM_RPC) { + thread_resume_from_rpc(a3, a1, a2, a4, a5); + rv = OPTEE_ABI_RETURN_ERESUME; + } else { + thread_alloc_and_run(a0, a1, a2, a3, 0, 0); + rv = OPTEE_ABI_RETURN_ETHREAD_LIMIT; + } + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_unset_guest(); + + return rv; +} + +/** + * Free physical memory previously allocated with thread_rpc_alloc_arg() + * + * @cookie: cookie received when allocating the buffer + */ +static void thread_rpc_free_arg(uint64_t cookie) +{ + if (cookie) { + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { + OPTEE_ABI_RETURN_RPC_FREE + }; + + reg_pair_from_64(cookie, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + } +} + +static uint32_t get_msg_arg(struct mobj *mobj, size_t offset, + size_t *num_params, struct optee_msg_arg **arg, + struct optee_msg_arg **rpc_arg) +{ + void *p = NULL; + size_t sz = 0; + + if (!mobj) + return OPTEE_ABI_RETURN_EBADADDR; + + p = mobj_get_va(mobj, offset, sizeof(struct optee_msg_arg)); + if (!p || !IS_ALIGNED_WITH_TYPE(p, struct optee_msg_arg)) + return OPTEE_ABI_RETURN_EBADADDR; + + *arg = p; + *num_params = READ_ONCE((*arg)->num_params); + if (*num_params > OPTEE_MSG_MAX_NUM_PARAMS) + return OPTEE_ABI_RETURN_EBADADDR; + + sz = OPTEE_MSG_GET_ARG_SIZE(*num_params); + if (!mobj_get_va(mobj, offset, sz)) + return OPTEE_ABI_RETURN_EBADADDR; + + if (rpc_arg) { + size_t rpc_sz = 0; + + rpc_sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + p = mobj_get_va(mobj, offset + sz, rpc_sz); + if (!p) + return OPTEE_ABI_RETURN_EBADADDR; + *rpc_arg = p; + } + + return OPTEE_ABI_RETURN_OK; +} + +static void clear_prealloc_rpc_cache(struct thread_ctx *thr) +{ + thread_rpc_free_arg(mobj_get_cookie(thr->rpc_mobj)); + mobj_put(thr->rpc_mobj); + thr->rpc_arg = NULL; + thr->rpc_mobj = NULL; +} + +static uint32_t call_entry_std(struct optee_msg_arg *arg, size_t num_params, + struct optee_msg_arg *rpc_arg) +{ + struct thread_ctx *thr = threads + thread_get_id(); + uint32_t rv = 0; + + if (rpc_arg) { + /* + * In case the prealloc RPC arg cache is enabled, clear the + * cached object for this thread. + * + * Normally it doesn't make sense to have the prealloc RPC + * arg cache enabled together with a supplied RPC arg + * struct. But if it is we must use the supplied struct and + * at the same time make sure to not break anything. + */ + if (IS_ENABLED(CFG_PREALLOC_RPC_CACHE) && + thread_prealloc_rpc_cache) + clear_prealloc_rpc_cache(thr); + thr->rpc_arg = rpc_arg; + } + + if (tee_entry_std(arg, num_params)) + rv = OPTEE_ABI_RETURN_EBADCMD; + else + rv = OPTEE_ABI_RETURN_OK; + + thread_rpc_shm_cache_clear(&thr->shm_cache); + if (rpc_arg) + thr->rpc_arg = NULL; + + if (rv == OPTEE_ABI_RETURN_OK && + !(IS_ENABLED(CFG_PREALLOC_RPC_CACHE) && thread_prealloc_rpc_cache)) + clear_prealloc_rpc_cache(thr); + + return rv; +} + +static uint32_t std_entry_with_parg(paddr_t parg, bool with_rpc_arg) +{ + size_t sz = sizeof(struct optee_msg_arg); + struct optee_msg_arg *rpc_arg = NULL; + struct optee_msg_arg *arg = NULL; + struct mobj *mobj = NULL; + size_t num_params = 0; + uint32_t rv = 0; + + /* Check if this region is in static shared space */ + if (core_pbuf_is(CORE_MEM_NSEC_SHM, parg, sz)) { + if (!IS_ALIGNED_WITH_TYPE(parg, struct optee_msg_arg)) + goto bad_addr; + + arg = phys_to_virt(parg, MEM_AREA_NSEC_SHM, + sizeof(struct optee_msg_arg)); + if (!arg) + goto bad_addr; + + num_params = READ_ONCE(arg->num_params); + if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) + return OPTEE_ABI_RETURN_EBADADDR; + + sz = OPTEE_MSG_GET_ARG_SIZE(num_params); + if (with_rpc_arg) { + rpc_arg = (void *)((uint8_t *)arg + sz); + sz += OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + } + if (!core_pbuf_is(CORE_MEM_NSEC_SHM, parg, sz)) + goto bad_addr; + + return call_entry_std(arg, num_params, rpc_arg); + } + + if (parg & SMALL_PAGE_MASK) + goto bad_addr; + /* + * mobj_mapped_shm_alloc checks if parg resides in nonsec + * ddr. + */ + mobj = mobj_mapped_shm_alloc(&parg, 1, 0, 0); + if (!mobj) + goto bad_addr; + if (with_rpc_arg) + rv = get_msg_arg(mobj, 0, &num_params, &arg, &rpc_arg); + else + rv = get_msg_arg(mobj, 0, &num_params, &arg, NULL); + if (!rv) + rv = call_entry_std(arg, num_params, rpc_arg); + mobj_put(mobj); + return rv; + +bad_addr: + EMSG("Bad arg address 0x%"PRIxPA, parg); + return OPTEE_ABI_RETURN_EBADADDR; +} + +static uint32_t std_entry_with_regd_arg(uint64_t cookie, size_t offset) +{ + struct optee_msg_arg *rpc_arg = NULL; + struct optee_msg_arg *arg = NULL; + size_t num_params = 0; + struct mobj *mobj = NULL; + uint32_t rv = 0; + + mobj = mobj_reg_shm_get_by_cookie(cookie); + if (!mobj) { + EMSG("Bad arg cookie 0x%"PRIx64, cookie); + return OPTEE_ABI_RETURN_EBADADDR; + } + + if (mobj_inc_map(mobj)) { + rv = OPTEE_ABI_RETURN_ENOMEM; + goto out; + } + + rv = get_msg_arg(mobj, offset, &num_params, &arg, &rpc_arg); + if (!rv) + rv = call_entry_std(arg, num_params, rpc_arg); + + mobj_dec_map(mobj); +out: + mobj_put(mobj); + + return rv; +} + +static uint32_t std_abi_entry(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3 __unused) +{ + const bool with_rpc_arg = true; + + switch (a0) { + case OPTEE_ABI_CALL_WITH_ARG: + return std_entry_with_parg(reg_pair_to_64(a1, a2), + !with_rpc_arg); + case OPTEE_ABI_CALL_WITH_RPC_ARG: + return std_entry_with_parg(reg_pair_to_64(a1, a2), + with_rpc_arg); + case OPTEE_ABI_CALL_WITH_REGD_ARG: + return std_entry_with_regd_arg(reg_pair_to_64(a1, a2), a3); + default: + EMSG("Unknown ABI 0x%"PRIx32, a0); + return OPTEE_ABI_RETURN_EBADCMD; + } +} + +/* + * Helper routine for the assembly function thread_std_abi_entry() + * + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +uint32_t __weak __thread_std_abi_entry(uint32_t a0, uint32_t a1, uint32_t a2, + uint32_t a3, uint32_t a4 __unused, + uint32_t a5 __unused) +{ + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_on_stdcall(); + + return std_abi_entry(a0, a1, a2, a3); +} + +bool thread_disable_prealloc_rpc_cache(uint64_t *cookie) +{ + bool rv = false; + size_t n = 0; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + thread_lock_global(); + + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].state != THREAD_STATE_FREE) { + rv = false; + goto out; + } + } + + rv = true; + + if (IS_ENABLED(CFG_PREALLOC_RPC_CACHE)) { + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].rpc_arg) { + *cookie = mobj_get_cookie(threads[n].rpc_mobj); + mobj_put(threads[n].rpc_mobj); + threads[n].rpc_arg = NULL; + threads[n].rpc_mobj = NULL; + goto out; + } + } + } + + *cookie = 0; + thread_prealloc_rpc_cache = false; +out: + thread_unlock_global(); + thread_unmask_exceptions(exceptions); + return rv; +} + +bool thread_enable_prealloc_rpc_cache(void) +{ + bool rv = false; + size_t n = 0; + uint32_t exceptions = 0; + + if (!IS_ENABLED(CFG_PREALLOC_RPC_CACHE)) + return true; + + exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + thread_lock_global(); + + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (threads[n].state != THREAD_STATE_FREE) { + rv = false; + goto out; + } + } + + rv = true; + thread_prealloc_rpc_cache = true; +out: + thread_unlock_global(); + thread_unmask_exceptions(exceptions); + return rv; +} + +static struct mobj *rpc_shm_mobj_alloc(paddr_t pa, size_t sz, uint64_t cookie) +{ + /* Check if this region is in static shared space */ + if (core_pbuf_is(CORE_MEM_NSEC_SHM, pa, sz)) + return mobj_shm_alloc(pa, sz, cookie); + + if (IS_ENABLED(CFG_CORE_DYN_SHM) && + !(pa & SMALL_PAGE_MASK) && sz <= SMALL_PAGE_SIZE) + return mobj_mapped_shm_alloc(&pa, 1, 0, cookie); + + return NULL; +} + +/** + * Allocates data for struct optee_msg_arg. + * + * @size: size in bytes of struct optee_msg_arg + * + * @returns mobj that describes allocated buffer or NULL on error + */ +static struct mobj *thread_rpc_alloc_arg(size_t size) +{ + paddr_t pa; + uint64_t co; + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { + OPTEE_ABI_RETURN_RPC_ALLOC, size + }; + struct mobj *mobj = NULL; + + thread_rpc(rpc_args); + + /* Registers 1 and 2 passed from normal world */ + pa = reg_pair_to_64(rpc_args[0], rpc_args[1]); + /* Registers 4 and 5 passed from normal world */ + co = reg_pair_to_64(rpc_args[2], rpc_args[3]); + + if (!IS_ALIGNED_WITH_TYPE(pa, struct optee_msg_arg)) + goto err; + + mobj = rpc_shm_mobj_alloc(pa, size, co); + if (!mobj) + goto err; + + return mobj; +err: + thread_rpc_free_arg(co); + mobj_put(mobj); + return NULL; +} + +static bool set_rmem(struct optee_msg_param *param, + struct thread_param *tpm) +{ + param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + + OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; + param->u.rmem.offs = tpm->u.memref.offs; + param->u.rmem.size = tpm->u.memref.size; + if (tpm->u.memref.mobj) { + param->u.rmem.shm_ref = mobj_get_cookie(tpm->u.memref.mobj); + if (!param->u.rmem.shm_ref) + return false; + } else { + param->u.rmem.shm_ref = 0; + } + + return true; +} + +static bool set_tmem(struct optee_msg_param *param, + struct thread_param *tpm) +{ + paddr_t pa = 0; + uint64_t shm_ref = 0; + struct mobj *mobj = tpm->u.memref.mobj; + + param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + + OPTEE_MSG_ATTR_TYPE_TMEM_INPUT; + if (mobj) { + shm_ref = mobj_get_cookie(mobj); + if (!shm_ref) + return false; + if (mobj_get_pa(mobj, tpm->u.memref.offs, 0, &pa)) + return false; + } + + param->u.tmem.size = tpm->u.memref.size; + param->u.tmem.buf_ptr = pa; + param->u.tmem.shm_ref = shm_ref; + + return true; +} + +static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, + struct thread_param *params, void **arg_ret, + uint64_t *carg_ret) +{ + struct thread_ctx *thr = threads + thread_get_id(); + struct optee_msg_arg *arg = thr->rpc_arg; + size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); + + if (num_params > THREAD_RPC_MAX_NUM_PARAMS) + return TEE_ERROR_BAD_PARAMETERS; + + if (!arg) { + struct mobj *mobj = thread_rpc_alloc_arg(sz); + + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + + arg = mobj_get_va(mobj, 0, sz); + if (!arg) { + thread_rpc_free_arg(mobj_get_cookie(mobj)); + return TEE_ERROR_OUT_OF_MEMORY; + } + + thr->rpc_arg = arg; + thr->rpc_mobj = mobj; + } + + memset(arg, 0, OPTEE_MSG_GET_ARG_SIZE(num_params)); + arg->cmd = cmd; + arg->num_params = num_params; + arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ + + for (size_t n = 0; n < num_params; n++) { + switch (params[n].attr) { + case THREAD_PARAM_ATTR_NONE: + arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; + break; + case THREAD_PARAM_ATTR_VALUE_IN: + case THREAD_PARAM_ATTR_VALUE_OUT: + case THREAD_PARAM_ATTR_VALUE_INOUT: + arg->params[n].attr = params[n].attr - + THREAD_PARAM_ATTR_VALUE_IN + + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; + arg->params[n].u.value.a = params[n].u.value.a; + arg->params[n].u.value.b = params[n].u.value.b; + arg->params[n].u.value.c = params[n].u.value.c; + break; + case THREAD_PARAM_ATTR_MEMREF_IN: + case THREAD_PARAM_ATTR_MEMREF_OUT: + case THREAD_PARAM_ATTR_MEMREF_INOUT: + if (!params[n].u.memref.mobj || + mobj_matches(params[n].u.memref.mobj, + CORE_MEM_NSEC_SHM)) { + if (!set_tmem(arg->params + n, params + n)) + return TEE_ERROR_BAD_PARAMETERS; + } else if (mobj_matches(params[n].u.memref.mobj, + CORE_MEM_REG_SHM)) { + if (!set_rmem(arg->params + n, params + n)) + return TEE_ERROR_BAD_PARAMETERS; + } else { + return TEE_ERROR_BAD_PARAMETERS; + } + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + } + + *arg_ret = arg; + *carg_ret = mobj_get_cookie(thr->rpc_mobj); + + return TEE_SUCCESS; +} + +static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, + struct thread_param *params) +{ + for (size_t n = 0; n < num_params; n++) { + switch (params[n].attr) { + case THREAD_PARAM_ATTR_VALUE_OUT: + case THREAD_PARAM_ATTR_VALUE_INOUT: + params[n].u.value.a = arg->params[n].u.value.a; + params[n].u.value.b = arg->params[n].u.value.b; + params[n].u.value.c = arg->params[n].u.value.c; + break; + case THREAD_PARAM_ATTR_MEMREF_OUT: + case THREAD_PARAM_ATTR_MEMREF_INOUT: + /* + * rmem.size and tmem.size is the same type and + * location. + */ + params[n].u.memref.size = arg->params[n].u.rmem.size; + break; + default: + break; + } + } + + return arg->ret; +} + +uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, + struct thread_param *params) +{ + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_ABI_RETURN_RPC_CMD }; + void *arg = NULL; + uint64_t carg = 0; + uint32_t ret = 0; + + /* The source CRYPTO_RNG_SRC_JITTER_RPC is safe to use here */ + plat_prng_add_jitter_entropy(CRYPTO_RNG_SRC_JITTER_RPC, + &thread_rpc_pnum); + + ret = get_rpc_arg(cmd, num_params, params, &arg, &carg); + if (ret) + return ret; + + reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + + return get_rpc_arg_res(arg, num_params, params); +} + +/** + * Free physical memory previously allocated with thread_rpc_alloc() + * + * @cookie: cookie received when allocating the buffer + * @bt: must be the same as supplied when allocating + * @mobj: mobj that describes allocated buffer + * + * This function also frees corresponding mobj. + */ +static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) +{ + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_ABI_RETURN_RPC_CMD }; + void *arg = NULL; + uint64_t carg = 0; + struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); + uint32_t ret = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, + &arg, &carg); + + mobj_put(mobj); + + if (!ret) { + reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + } +} + +static struct mobj *get_rpc_alloc_res(struct optee_msg_arg *arg, + unsigned int bt, size_t size) +{ + struct mobj *mobj = NULL; + uint64_t cookie = 0; + size_t sz = 0; + paddr_t p = 0; + + if (arg->ret || arg->num_params != 1) + goto err; + + if (arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT && + arg->params[0].attr != (OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | + OPTEE_MSG_ATTR_NONCONTIG)) + goto err; + + p = arg->params[0].u.tmem.buf_ptr; + sz = READ_ONCE(arg->params[0].u.tmem.size); + cookie = arg->params[0].u.tmem.shm_ref; + if (sz < size) + goto err; + + if (arg->params[0].attr == OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT) + mobj = rpc_shm_mobj_alloc(p, sz, cookie); + else + mobj = msg_param_mobj_from_noncontig(p, sz, cookie, true); + + if (!mobj) { + thread_rpc_free(bt, cookie, mobj); + goto err; + } + + assert(mobj_is_nonsec(mobj)); + return mobj; +err: + EMSG("RPC allocation failed. Non-secure world result: ret=%#" + PRIx32" ret_origin=%#"PRIx32, arg->ret, arg->ret_origin); + return NULL; +} + +/** + * Allocates shared memory buffer via RPC + * + * @size: size in bytes of shared memory buffer + * @align: required alignment of buffer + * @bt: buffer type OPTEE_RPC_SHM_TYPE_* + * + * Returns a pointer to MOBJ for the memory on success, or NULL on failure. + */ +static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) +{ + uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_ABI_RETURN_RPC_CMD }; + void *arg = NULL; + uint64_t carg = 0; + struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); + uint32_t ret = get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, + &arg, &carg); + + if (ret) + return NULL; + + reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); + thread_rpc(rpc_args); + + return get_rpc_alloc_res(arg, bt, size); +} + +struct mobj *thread_rpc_alloc_payload(size_t size) +{ + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); +} + +struct mobj *thread_rpc_alloc_kernel_payload(size_t size) +{ + /* + * Error out early since kernel private dynamic shared memory + * allocations don't currently use the `OPTEE_MSG_ATTR_NONCONTIG` bit + * and therefore cannot be larger than a page. + */ + if (IS_ENABLED(CFG_CORE_DYN_SHM) && size > SMALL_PAGE_SIZE) + return NULL; + + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); +} + +void thread_rpc_free_kernel_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, mobj_get_cookie(mobj), mobj); +} + +void thread_rpc_free_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), + mobj); +} + +struct mobj *thread_rpc_alloc_global_payload(size_t size) +{ + return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); +} + +void thread_rpc_free_global_payload(struct mobj *mobj) +{ + thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, mobj_get_cookie(mobj), + mobj); +} diff --git a/optee_os/core/arch/riscv/kernel/thread_optee_abi_rv.S b/optee_os/core/arch/riscv/kernel/thread_optee_abi_rv.S new file mode 100644 index 0000000..6ba598c --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/thread_optee_abi_rv.S @@ -0,0 +1,215 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2023 NXP + * Copyright (c) 2023 Andes Technology Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Implement based on the transport method used to communicate between + * untrusted domain and trusted domain. It could be an SBI/ECALL-based to + * a security monitor running in M-Mode and panic or messaging-based across + * domains where we return to a messaging callback which parses and handles + * messages. + * + * void thread_return_to_ree(unsigned long arg0, unsigned long arg1, + * unsigned long arg2, unsigned long arg3, + * unsigned long arg4, unsigned long arg5); + */ +FUNC thread_return_to_ree , : + /* Caller should provide arguments in a0~a5 */ +#if defined(CFG_RISCV_SBI) + li a7, SBI_EXT_TEE /* extension ID */ + li a6, 0 /* function ID (unused) */ + ecall +#else + /* Other protocol */ +#endif + /* ABI to REE should not return */ + panic_at_abi_return +END_FUNC thread_return_to_ree + +FUNC thread_std_abi_entry , : + jal __thread_std_abi_entry + + /* Save return value */ + mv s0, a0 + + /* Disable all interrupts */ + csrc CSR_XSTATUS, CSR_XSTATUS_IE + + /* Switch to temporary stack */ + jal thread_get_tmp_sp + mv sp, a0 + + /* + * We are returning from thread_alloc_and_run() + * set thread state as free + */ + jal thread_state_free + + /* Restore __thread_std_abi_entry() return value */ + mv a1, s0 + li a2, 0 + li a3, 0 + li a4, 0 + li a0, TEEABI_OPTEED_RETURN_CALL_DONE + + /* Return to untrusted domain */ + jal thread_return_to_ree +END_FUNC thread_std_abi_entry + +/* void thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]) */ +FUNC thread_rpc , : + /* Use stack for temporary storage */ + addi sp, sp, -REGOFF(4) + + /* Read xSTATUS */ + csrr a1, CSR_XSTATUS + + /* Mask all maskable exceptions before switching to temporary stack */ + csrc CSR_XSTATUS, CSR_XSTATUS_IE + + /* Save return address xSTATUS and pointer to rv */ + STR a0, REGOFF(0)(sp) + STR a1, REGOFF(1)(sp) + STR s0, REGOFF(2)(sp) + STR ra, REGOFF(3)(sp) + addi s0, sp, REGOFF(4) + + /* Save thread state */ + jal thread_get_ctx_regs + store_xregs a0, THREAD_CTX_REG_SP, REG_SP + store_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 + store_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 + + /* Get to tmp stack */ + jal thread_get_tmp_sp + + /* Get pointer to rv */ + LDR s1, REGOFF(0)(sp) + + /* xSTATUS to restore */ + LDR a1, REGOFF(1)(sp) + /* Switch to tmp stack */ + mv sp, a0 + + /* Early load rv[] into s2-s4 */ + lw s2, 0(s1) + lw s3, 4(s1) + lw s4, 8(s1) + + li a0, THREAD_FLAGS_COPY_ARGS_ON_RETURN + la a2, .thread_rpc_return + jal thread_state_suspend + + mv a4, a0 /* thread index */ + mv a1, s2 /* rv[0] */ + mv a2, s3 /* rv[1] */ + mv a3, s4 /* rv[2] */ + li a0, TEEABI_OPTEED_RETURN_CALL_DONE + + /* Return to untrusted domain */ + jal thread_return_to_ree +.thread_rpc_return: + /* + * Jumps here from thread_resume() above when RPC has returned. + * At this point has the stack pointer been restored to the value + * stored in THREAD_CTX above. + */ + + /* Get pointer to rv[] */ + LDR a4, REGOFF(0)(sp) + + /* Store a0-a3 into rv[] */ + sw a0, 0(a4) + sw a1, 4(a4) + sw a2, 8(a4) + sw a3, 12(a4) + + /* Pop saved XSTATUS from stack */ + LDR s0, REGOFF(1)(sp) + csrw CSR_XSTATUS, s0 + + /* Pop return address and s0 from stack */ + LDR ra, REGOFF(3)(sp) + LDR s0, REGOFF(2)(sp) + + addi sp, sp, REGOFF(4) + ret +END_FUNC thread_rpc + +LOCAL_FUNC vector_std_abi_entry, : , .identity_map + jal thread_handle_std_abi + /* + * Normally thread_handle_std_abi() should return via + * thread_exit(), thread_rpc(), but if thread_handle_std_abi() + * hasn't switched stack (error detected) it will do a normal "C" + * return. + */ + /* Restore thread_handle_std_abi() return value */ + mv a1, a0 + li a2, 0 + li a3, 0 + li a4, 0 + li a0, TEEABI_OPTEED_RETURN_CALL_DONE + + /* Return to untrusted domain */ + j thread_return_to_ree +END_FUNC vector_std_abi_entry + +LOCAL_FUNC vector_fast_abi_entry , : , .identity_map + addi sp, sp, -THREAD_ABI_ARGS_SIZE + store_xregs sp, THREAD_ABI_ARGS_A0, REG_A0, REG_A7 + mv a0, sp + jal thread_handle_fast_abi + load_xregs sp, THREAD_ABI_ARGS_A0, REG_A1, REG_A7 + addi sp, sp, THREAD_ABI_ARGS_SIZE + + li a0, TEEABI_OPTEED_RETURN_CALL_DONE + /* Return to untrusted domain */ + j thread_return_to_ree +END_FUNC vector_fast_abi_entry + +LOCAL_FUNC vector_fiq_entry , : , .identity_map + /* Secure Monitor received a FIQ and passed control to us. */ + jal interrupt_main_handler + + li a0, TEEABI_OPTEED_RETURN_FIQ_DONE + /* Return to untrusted domain */ + j thread_return_to_ree +END_FUNC vector_fiq_entry + +/* + * Vector table supplied to M-mode secure monitor (e.g., openSBI) at + * initialization. + * + * Note that M-mode secure monitor depends on the layout of this vector table, + * any change in layout has to be synced with M-mode secure monitor. + */ +FUNC thread_vector_table , : , .identity_map, , nobti + .option push + .option norvc + j vector_std_abi_entry + j vector_fast_abi_entry + j . + j . + j . + j . + j vector_fiq_entry + j . + j . + .option pop +END_FUNC thread_vector_table +DECLARE_KEEP_PAGER thread_vector_table diff --git a/optee_os/core/arch/riscv/kernel/thread_rv.S b/optee_os/core/arch/riscv/kernel/thread_rv.S new file mode 100644 index 0000000..e97a5cd --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/thread_rv.S @@ -0,0 +1,264 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +.macro get_thread_ctx res, tmp0 + lw \tmp0, THREAD_CORE_LOCAL_CURR_THREAD(tp) + la \res, threads +1: + beqz \tmp0, 2f + addi \res, \res, THREAD_CTX_SIZE + addi \tmp0, \tmp0, -1 + bnez \tmp0, 1b +2: +.endm + +.macro save_regs, mode + addi sp, sp, -THREAD_TRAP_REGS_SIZE +.if \mode == TRAP_MODE_USER + + /* Save user thread pointer and load kernel thread pointer */ + store_xregs sp, THREAD_TRAP_REG_TP, REG_TP + addi tp, sp, THREAD_TRAP_REGS_SIZE + /* Now tp is at struct thread_user_mode_rec, which has kernel tp */ + load_xregs tp, THREAD_USER_MODE_REC_X4, REG_TP + + store_xregs sp, THREAD_TRAP_REG_GP, REG_GP + + /* + * Set the scratch register to 0 such in case of a recursive + * exception thread_trap_vect() knows that it is emitted from kernel. + */ + csrrw gp, CSR_XSCRATCH, zero + store_xregs sp, THREAD_TRAP_REG_SP, REG_GP +.option push +.option norelax + la gp, __global_pointer$ +.option pop +.endif + store_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 + store_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 + store_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 + store_xregs sp, THREAD_TRAP_REG_RA, REG_RA +#if defined(CFG_UNWIND) + /* To unwind stack we need s0, which is frame pointer. */ + store_xregs sp, THREAD_TRAP_REG_S0, REG_S0 +#endif + + csrr t0, CSR_XSTATUS + store_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 + + csrr a0, CSR_XCAUSE + csrr a1, CSR_XEPC + + store_xregs sp, THREAD_TRAP_REG_EPC, REG_A1 + + mv a2, sp + + /* a0 = cause + * a1 = epc + * a2 = sp + * a3 = user + * thread_trap_handler(cause, epc, sp, user) + */ +.endm + +.macro restore_regs, mode + load_xregs sp, THREAD_TRAP_REG_EPC, REG_T0 + csrw CSR_XEPC, t0 + + load_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 + csrw CSR_XSTATUS, t0 + + load_xregs sp, THREAD_TRAP_REG_RA, REG_RA + load_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 + load_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 + load_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 +#if defined(CFG_UNWIND) + /* To unwind stack we need s0, which is frame pointer. */ + load_xregs sp, THREAD_TRAP_REG_S0, REG_S0 +#endif + +.if \mode == TRAP_MODE_USER + addi gp, sp, THREAD_TRAP_REGS_SIZE + csrw CSR_XSCRATCH, gp + + load_xregs sp, THREAD_TRAP_REG_TP, REG_TP + load_xregs sp, THREAD_TRAP_REG_GP, REG_GP + load_xregs sp, THREAD_TRAP_REG_SP, REG_SP + +.else + addi sp, sp, THREAD_TRAP_REGS_SIZE +.endif +.endm + +/* size_t __get_core_pos(void); */ +FUNC __get_core_pos , : , .identity_map + lw a0, THREAD_CORE_LOCAL_HART_ID(tp) + ret +END_FUNC __get_core_pos + +FUNC thread_trap_vect , : + csrrw sp, CSR_XSCRATCH, sp + bnez sp, 0f + csrrw sp, CSR_XSCRATCH, sp + j trap_from_kernel +0: + j trap_from_user +thread_trap_vect_end: +END_FUNC thread_trap_vect + +LOCAL_FUNC trap_from_kernel, : + save_regs TRAP_MODE_KERNEL + li a3, 0 + jal thread_trap_handler + restore_regs TRAP_MODE_KERNEL + XRET +END_FUNC trap_from_kernel + +LOCAL_FUNC trap_from_user, : + save_regs TRAP_MODE_USER + li a3, 1 + jal thread_trap_handler + restore_regs TRAP_MODE_USER + XRET +END_FUNC trap_from_user + +/* + * void thread_unwind_user_mode(uint32_t ret, uint32_t exit_status0, + * uint32_t exit_status1); + * See description in thread.h + */ +FUNC thread_unwind_user_mode , : + + /* Store the exit status */ + load_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A3, REG_A5 + sw a1, (a4) + sw a2, (a5) + + /* Save user callee regs */ + store_xregs a3, THREAD_CTX_REG_S0, REG_S0, REG_S1 + store_xregs a3, THREAD_CTX_REG_S2, REG_S2, REG_S11 + store_xregs a3, THREAD_CTX_REG_SP, REG_SP, REG_TP + + /* Restore kernel callee regs */ + mv a1, sp + + load_xregs a1, THREAD_USER_MODE_REC_X1, REG_RA, REG_TP + load_xregs a1, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 + load_xregs a1, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 + + add sp, sp, THREAD_USER_MODE_REC_SIZE + + /* + * Zeroize xSCRATCH to indicate to thread_trap_vect() + * that we are executing in kernel. + */ + csrw CSR_XSCRATCH, zero + + /* Return from the call of thread_enter_user_mode() */ + ret +END_FUNC thread_unwind_user_mode + +/* + * void thread_exit_user_mode(unsigned long a0, unsigned long a1, + * unsigned long a2, unsigned long a3, + * unsigned long sp, unsigned long pc, + * unsigned long status); + */ +FUNC thread_exit_user_mode , : + /* Set kernel stack pointer */ + mv sp, a4 + + /* Set xSTATUS */ + csrw CSR_XSTATUS, a6 + + /* Set return address thread_unwind_user_mode() */ + mv ra, a5 + ret +END_FUNC thread_exit_user_mode + +/* + * uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, + * uint32_t *exit_status0, + * uint32_t *exit_status1); + */ +FUNC __thread_enter_user_mode , : + /* + * Create and fill in the struct thread_user_mode_rec + */ + addi sp, sp, -THREAD_USER_MODE_REC_SIZE + store_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A0, REG_A2 + store_xregs sp, THREAD_USER_MODE_REC_X1, REG_RA, REG_TP + store_xregs sp, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 + store_xregs sp, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 + + /* + * Save the kernel stack pointer in the thread context + */ + + /* Get pointer to current thread context */ + get_thread_ctx s0, s1 + + /* + * Save kernel stack pointer to ensure that + * thread_exit_user_mode() uses correct stack pointer. + */ + + store_xregs s0, THREAD_CTX_KERN_SP, REG_SP + /* + * Save kernel stack pointer in xSCRATCH to ensure that + * thread_trap_vect() uses correct stack pointer. + */ + csrw CSR_XSCRATCH, sp + + /* Set user status */ + load_xregs a0, THREAD_CTX_REG_STATUS, REG_S0 + csrw CSR_XSTATUS, s0 + + /* + * Save the values for a1 and a2 in struct thread_core_local to be + * restored later just before the xRET. + */ + store_xregs tp, THREAD_CORE_LOCAL_X10, REG_A1, REG_A2 + + /* Load the rest of the general purpose registers */ + load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_TP + load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 + load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 + load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 + load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 + load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 + + /* Set exception program counter */ + csrw CSR_XEPC, ra + + /* Jump into user mode */ + XRET +END_FUNC __thread_enter_user_mode + +/* void thread_resume(struct thread_ctx_regs *regs) */ +FUNC thread_resume , : + /* + * Restore all registers assuming that GP + * and TP were not changed. + */ + load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_SP + load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 + load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 + load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 + load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 + load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 + store_xregs tp, THREAD_CORE_LOCAL_X10, REG_A0, REG_A1 + ret +END_FUNC thread_resume diff --git a/optee_os/core/arch/riscv/kernel/unwind_rv.c b/optee_os/core/arch/riscv/kernel/unwind_rv.c new file mode 100644 index 0000000..5c3a17b --- /dev/null +++ b/optee_os/core/arch/riscv/kernel/unwind_rv.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BSD-2-Clause +/*- + * Copyright (c) 2023 Andes Technology Corporation + * Copyright (c) 2015 Linaro Limited + * Copyright (c) 2015 The FreeBSD Foundation + */ + +#include +#include +#include +#include +#include + +#if defined(CFG_UNWIND) && (TRACE_LEVEL > 0) +void print_kernel_stack(void) +{ + struct unwind_state_riscv state = { }; + vaddr_t stack_start = 0; + vaddr_t stack_end = 0; + + state.pc = read_pc(); + state.fp = read_fp(); + + trace_printf_helper_raw(TRACE_ERROR, true, + "TEE load address @ %#"PRIxVA, VCORE_START_VA); + get_stack_hard_limits(&stack_start, &stack_end); + print_stack_riscv(&state, stack_start, stack_end - stack_start); +} +#endif diff --git a/optee_os/core/arch/riscv/mm/core_mmu_arch.c b/optee_os/core/arch/riscv/mm/core_mmu_arch.c new file mode 100644 index 0000000..dd6ba4c --- /dev/null +++ b/optee_os/core/arch/riscv/mm/core_mmu_arch.c @@ -0,0 +1,736 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef RV64 +#error implement +#endif + +#ifndef DEBUG_XLAT_TABLE +#define DEBUG_XLAT_TABLE 0 +#endif + +#if DEBUG_XLAT_TABLE +#define debug_print(...) DMSG_RAW(__VA_ARGS__) +#else +#define debug_print(...) ((void)0) +#endif + +static bitstr_t bit_decl(g_asid, RISCV_MMU_ASID_WIDTH) __nex_bss; +static unsigned int g_asid_spinlock __nex_bss = SPINLOCK_UNLOCK; + +struct mmu_pte { + unsigned long entry; +}; + +struct mmu_pgt { + struct mmu_pte entries[RISCV_PTES_PER_PT]; +}; + +#define RISCV_MMU_PGT_SIZE (sizeof(struct mmu_pgt)) + +static struct mmu_pgt root_pgt + __aligned(RISCV_PGSIZE) + __section(".nozi.mmu.root_pgt"); + +static struct mmu_pgt pool_pgts[RISCV_MMU_MAX_PGTS] + __aligned(RISCV_PGSIZE) __section(".nozi.mmu.pool_pgts"); + +static struct mmu_pgt user_pgts[CFG_NUM_THREADS] + __aligned(RISCV_PGSIZE) __section(".nozi.mmu.usr_pgts"); + +static int user_va_idx __nex_data = -1; + +struct mmu_partition { + struct mmu_pgt *root_pgt; + struct mmu_pgt *pool_pgts; + struct mmu_pgt *user_pgts; + unsigned int pgts_used; + unsigned int asid; +}; + +static struct mmu_partition default_partition __nex_data = { + .root_pgt = &root_pgt, + .pool_pgts = pool_pgts, + .user_pgts = user_pgts, + .pgts_used = 0, + .asid = 0 +}; + +static struct mmu_pte *core_mmu_table_get_entry(struct mmu_pgt *pgt, + unsigned int idx) +{ + return &pgt->entries[idx & RISCV_MMU_VPN_MASK]; +} + +static void core_mmu_entry_set(struct mmu_pte *pte, uint64_t val) +{ + pte->entry = val; +} + +static uint64_t core_mmu_entry_get(struct mmu_pte *pte) +{ + return pte->entry; +} + +static bool core_mmu_entry_is_valid(struct mmu_pte *pte) +{ + return pte->entry & PTE_V; +} + +static bool core_mmu_entry_is_invalid(struct mmu_pte *pte) +{ + return !core_mmu_entry_is_valid(pte); +} + +static bool core_mmu_entry_is_leaf(struct mmu_pte *pte) +{ + /* A leaf has one or more RWX bits set */ + return pte->entry & (PTE_R | PTE_W | PTE_X); +} + +static bool __maybe_unused core_mmu_entry_is_branch(struct mmu_pte *pte) +{ + return !core_mmu_entry_is_leaf(pte); +} + +static unsigned long core_mmu_pte_create(unsigned long ppn, uint8_t perm) +{ + return SHIFT_U64(ppn, PTE_PPN_SHIFT) | PTE_V | perm; +} + +static unsigned long core_mmu_ptp_create(unsigned long ppn) +{ + /* set perms to 0 since core_mmu_pte_create() already adds PTE_V */ + return core_mmu_pte_create(ppn, 0); +} + +static unsigned long core_mmu_pte_ppn(struct mmu_pte *pte) +{ + return pte->entry >> PTE_PPN_SHIFT; +} + +static unsigned long pa_to_ppn(paddr_t pa) +{ + return pa >> RISCV_PGSHIFT; +} + +static paddr_t pte_to_pa(struct mmu_pte *pte) +{ + return SHIFT_U64(core_mmu_pte_ppn(pte), RISCV_PGSHIFT); +} + +static unsigned long core_mmu_pgt_to_satp(unsigned long asid, + struct mmu_pgt *pgt) +{ + unsigned long satp = 0; + unsigned long pgt_ppn = virt_to_phys(pgt) >> RISCV_PGSHIFT; + + assert(asid & g_asid == asid); + satp |= SHIFT_U64(asid, RISCV_SATP_ASID_SHIFT); + satp |= SHIFT_U64(RISCV_SATP_MODE, RISCV_SATP_MODE_SHIFT); + satp |= pgt_ppn; + + return satp; +} + +static unsigned long pte_to_mattr(unsigned level __maybe_unused, + struct mmu_pte *pte) +{ + unsigned long mattr = TEE_MATTR_SECURE; + unsigned long entry = core_mmu_entry_get(pte); + + if (entry & PTE_V) { + if (!(entry & (PTE_R | PTE_W | PTE_X))) + return TEE_MATTR_TABLE; + + mattr |= TEE_MATTR_VALID_BLOCK; + } + + if (entry & PTE_U) { + if (entry & PTE_R) + mattr |= TEE_MATTR_UR | TEE_MATTR_PR; + if (entry & PTE_W) + mattr |= TEE_MATTR_UW | TEE_MATTR_PW; + if (entry & PTE_X) + mattr |= TEE_MATTR_UX | TEE_MATTR_PX; + } else { + if (entry & PTE_R) + mattr |= TEE_MATTR_PR; + if (entry & PTE_W) + mattr |= TEE_MATTR_PW; + if (entry & PTE_X) + mattr |= TEE_MATTR_PX; + } + + if (entry & PTE_G) + mattr |= TEE_MATTR_GLOBAL; + + return mattr; +} + +static uint8_t mattr_to_perms(unsigned level __maybe_unused, + uint32_t attr) +{ + unsigned long perms = 0; + + if (attr & TEE_MATTR_TABLE) + return PTE_V; + + if (attr & TEE_MATTR_VALID_BLOCK) + perms |= PTE_V; + + if (attr & TEE_MATTR_UR) + perms |= PTE_R | PTE_U; + if (attr & TEE_MATTR_UW) + perms |= PTE_W | PTE_U; + if (attr & TEE_MATTR_UX) + perms |= PTE_X | PTE_U; + + if (attr & TEE_MATTR_PR) + perms |= PTE_R; + if (attr & TEE_MATTR_PW) + perms |= PTE_W | PTE_R; + if (attr & TEE_MATTR_PX) + perms |= PTE_X | PTE_R; + + if (attr & TEE_MATTR_GLOBAL) + perms |= PTE_G; + + return perms; +} + +static unsigned int core_mmu_pgt_idx(vaddr_t va, unsigned int level) +{ + unsigned int idx = va >> CORE_MMU_SHIFT_OF_LEVEL(level); + + return idx & RISCV_MMU_VPN_MASK; +} + +static struct mmu_partition *core_mmu_get_prtn(void) +{ + return &default_partition; +} + +static struct mmu_pgt *core_mmu_get_root_pgt_va(struct mmu_partition *prtn) +{ + return prtn->root_pgt; +} + +static struct mmu_pgt *core_mmu_get_ta_pgt_va(struct mmu_partition *prtn) +{ + return &prtn->user_pgts[thread_get_id()]; +} + +static struct mmu_pgt *core_mmu_pgt_alloc(struct mmu_partition *prtn) +{ + struct mmu_pgt *pgt = NULL; + + if (prtn->pgts_used >= RISCV_MMU_MAX_PGTS) { + debug_print("%u pgts exhausted", RISCV_MMU_MAX_PGTS); + panic(); + return NULL; + } + + pgt = &prtn->pool_pgts[prtn->pgts_used++]; + + memset(pgt, 0, RISCV_MMU_PGT_SIZE); + + debug_print("pgts used %u / %u", prtn->pgts_used, RISCV_MMU_MAX_PGTS); + + return pgt; +} + +static void core_init_mmu_prtn_ta_core(struct mmu_partition *prtn __unused, + unsigned int core __unused) +{ + /* + * user_va_idx is the index in CORE_MMU_BASE_TABLE_LEVEL. + * The entry holds pointer to the user mapping table in next level + * that changes per core. Therefore, nothing to do. + */ +} + +static void core_init_mmu_prtn_ta(struct mmu_partition *prtn) +{ + unsigned int core = 0; + + assert(user_va_idx != -1); + + memset(prtn->user_pgts, 0, CFG_NUM_THREADS * RISCV_MMU_PGT_SIZE); + for (core = 0; core < CFG_TEE_CORE_NB_CORE; core++) + core_init_mmu_prtn_ta_core(prtn, core); +} + +static void core_init_mmu_prtn_tee(struct mmu_partition *prtn, + struct tee_mmap_region *mm) +{ + size_t n = 0; + void *pgt = core_mmu_get_root_pgt_va(prtn); + + memset(pgt, 0, RISCV_MMU_PGT_SIZE); + memset(prtn->pool_pgts, 0, RISCV_MMU_MAX_PGTS * RISCV_MMU_PGT_SIZE); + + for (n = 0; !core_mmap_is_end_of_table(mm + n); n++) + if (!core_mmu_is_dynamic_vaspace(mm + n)) + core_mmu_map_region(prtn, mm + n); +} + +void tlbi_va_range(vaddr_t va, size_t len, + size_t granule) +{ + assert(granule == CORE_MMU_PGDIR_SIZE || granule == SMALL_PAGE_SIZE); + assert(!(va & (granule - 1)) && !(len & (granule - 1))); + + /* + * Ensure operations are completed or observed before proceeding + * with TLB invalidation. + */ + mb(); + while (len) { + tlbi_va_allasid(va); + len -= granule; + va += granule; + } + /* + * After invalidating TLB entries, a memory barrier is required + * to ensure that the page table entries become visible to other harts + * before subsequent memory accesses are performed. + */ + mb(); +} + +void tlbi_va_range_asid(vaddr_t va, size_t len, + size_t granule, uint32_t asid) +{ + assert(granule == CORE_MMU_PGDIR_SIZE || granule == SMALL_PAGE_SIZE); + assert(!(va & (granule - 1)) && !(len & (granule - 1))); + + /* + * A memory barrier is necessary here to ensure the consistency + * and correctness of memory accesses. + */ + mb(); + while (len) { + tlbi_va_asid(va, asid); + len -= granule; + va += granule; + } + /* Enforce ordering of memory operations and ensure that all + * preceding memory operations are completed after TLB + * invalidation. + */ + mb(); +} + +TEE_Result cache_op_inner(enum cache_op op, void *va, size_t len) +{ + switch (op) { + case DCACHE_CLEAN: + dcache_op_all(DCACHE_OP_CLEAN); + break; + case DCACHE_AREA_CLEAN: + dcache_clean_range(va, len); + break; + case DCACHE_INVALIDATE: + dcache_op_all(DCACHE_OP_INV); + break; + case DCACHE_AREA_INVALIDATE: + dcache_inv_range(va, len); + break; + case ICACHE_INVALIDATE: + icache_inv_all(); + break; + case ICACHE_AREA_INVALIDATE: + icache_inv_range(va, len); + break; + case DCACHE_CLEAN_INV: + dcache_op_all(DCACHE_OP_CLEAN_INV); + break; + case DCACHE_AREA_CLEAN_INV: + dcache_cleaninv_range(va, len); + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + return TEE_SUCCESS; +} + +unsigned int asid_alloc(void) +{ + uint32_t exceptions = cpu_spin_lock_xsave(&g_asid_spinlock); + unsigned int r = 0; + int i = 0; + + bit_ffc(g_asid, RISCV_MMU_ASID_WIDTH, &i); + if (i == -1) { + r = 0; + } else { + bit_set(g_asid, i); + r = i + 1; + } + + cpu_spin_unlock_xrestore(&g_asid_spinlock, exceptions); + + return r; +} + +void asid_free(unsigned int asid) +{ + uint32_t exceptions = cpu_spin_lock_xsave(&g_asid_spinlock); + + if (asid) { + int i = asid - 1; + + assert(i < RISCV_MMU_ASID_WIDTH && bit_test(&g_asid, i)); + bit_clear(g_asid, i); + } + + cpu_spin_unlock_xrestore(&g_asid_spinlock, exceptions); +} + +bool arch_va2pa_helper(void *va, paddr_t *pa) +{ + vaddr_t vaddr = (vaddr_t)va; + struct mmu_pgt *pgt = NULL; + struct mmu_pte *pte = NULL; + int level = 0; + unsigned int idx = 0; + struct mmu_partition *prtn = core_mmu_get_prtn(); + + assert(pa); + + pgt = core_mmu_get_root_pgt_va(prtn); + + for (level = CORE_MMU_BASE_TABLE_LEVEL; level >= 0; level--) { + idx = core_mmu_pgt_idx(vaddr, level); + pte = core_mmu_table_get_entry(pgt, idx); + + if (core_mmu_entry_is_invalid(pte)) { + return false; + } else if (core_mmu_entry_is_leaf(pte)) { + *pa = pte_to_pa(pte) | + (vaddr & (BIT64(RISCV_PGSHIFT) - 1)); + return true; + } + + pgt = phys_to_virt(pte_to_pa(pte), + MEM_AREA_TEE_RAM_RW_DATA, sizeof(*pgt)); + } + + return false; +} + +bool cpu_mmu_enabled(void) +{ + return read_satp(); +} + +bool core_mmu_find_table(struct mmu_partition *prtn, vaddr_t va, + unsigned int max_level, + struct core_mmu_table_info *tbl_info) +{ + struct mmu_pgt *pgt = NULL; + struct mmu_pte *pte = NULL; + unsigned int level = CORE_MMU_BASE_TABLE_LEVEL; + unsigned int idx = 0; + unsigned int deepest_level = max_level; + vaddr_t va_base = 0; + bool ret = false; + + if (max_level == UINT_MAX) + deepest_level = 0; + + if (!prtn) + prtn = core_mmu_get_prtn(); + + pgt = core_mmu_get_root_pgt_va(prtn); + + while (true) { + idx = core_mmu_pgt_idx(va - va_base, level); + pte = core_mmu_table_get_entry(pgt, idx); + if (level == deepest_level || level == 0 || + core_mmu_entry_is_invalid(pte) || + core_mmu_entry_is_leaf(pte)) { + core_mmu_set_info_table(tbl_info, level, va_base, pgt); + ret = true; + goto out; + } + pgt = phys_to_virt(pte_to_pa(pte), + MEM_AREA_TEE_RAM_RW_DATA, sizeof(*pgt)); + if (!pgt) + goto out; + va_base += SHIFT_U64(idx, CORE_MMU_SHIFT_OF_LEVEL(level)); + level--; + } +out: + return ret; +} + +bool core_mmu_entry_to_finer_grained(struct core_mmu_table_info *tbl_info, + unsigned int idx, bool secure __unused) +{ + struct mmu_pgt *pgt = NULL; + struct mmu_pte *pte = NULL; + struct mmu_partition *prtn = core_mmu_get_prtn(); + unsigned long ptp = 0; + + if (!core_mmu_level_in_range(tbl_info->level)) + return false; + + pgt = tbl_info->table; + pte = core_mmu_table_get_entry(pgt, idx); + + if (core_mmu_entry_is_invalid(pte)) { + pgt = core_mmu_pgt_alloc(prtn); + if (!pgt) + return false; + + ptp = core_mmu_ptp_create(pa_to_ppn((paddr_t)pgt)); + core_mmu_entry_set(pte, ptp); + } + + return true; +} + +void core_mmu_set_info_table(struct core_mmu_table_info *tbl_info, + unsigned int level, vaddr_t va_base, void *table) +{ + tbl_info->level = level; + tbl_info->next_level = level - 1; + tbl_info->table = table; + tbl_info->va_base = va_base; + tbl_info->shift = CORE_MMU_SHIFT_OF_LEVEL(level); + assert(level < RISCV_PGLEVELS); + tbl_info->num_entries = RISCV_PTES_PER_PT; +} + +void core_mmu_get_entry_primitive(const void *table, size_t level, + size_t idx, paddr_t *pa, uint32_t *attr) +{ + struct mmu_pgt *pgt = (struct mmu_pgt *)table; + struct mmu_pte *pte = core_mmu_table_get_entry(pgt, idx); + + if (core_mmu_entry_is_valid(pte)) { + if (pa) + *pa = pte_to_pa(pte); + if (attr) + *attr = pte_to_mattr(level, pte); + } else { + if (pa) + *pa = 0; + if (attr) + *attr = 0; + } +} + +void core_mmu_set_entry_primitive(void *table, size_t level, size_t idx, + paddr_t pa, uint32_t attr) +{ + struct mmu_pgt *pgt = (struct mmu_pgt *)table; + struct mmu_pte *pte = core_mmu_table_get_entry(pgt, idx); + uint8_t perms = mattr_to_perms(level, attr); + + core_mmu_entry_set(pte, core_mmu_pte_create(pa_to_ppn(pa), perms)); +} + +static void set_user_va_idx(struct mmu_partition *prtn) +{ + struct mmu_pgt *pgt = NULL; + struct mmu_pte *pte = NULL; + unsigned int idx = 0; + + pgt = core_mmu_get_root_pgt_va(prtn); + + for (idx = 1 ; idx < RISCV_PTES_PER_PT; idx++) { + pte = core_mmu_table_get_entry(pgt, idx); + if (core_mmu_entry_is_invalid(pte)) { + user_va_idx = idx; + return; + } + } + if (user_va_idx < 0) + panic(); +} + +static struct mmu_pte * +core_mmu_get_user_mapping_entry(struct mmu_partition *prtn) +{ + struct mmu_pgt *pgt = core_mmu_get_root_pgt_va(prtn); + + assert(user_va_idx != -1); + + return core_mmu_table_get_entry(pgt, user_va_idx); +} + +void core_mmu_set_user_map(struct core_mmu_user_map *map) +{ + unsigned long satp = 0; + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + struct mmu_partition *prtn = core_mmu_get_prtn(); + struct mmu_pte *pte = NULL; + unsigned long ptp = 0; + + satp = read_satp(); + /* Clear ASID */ + satp &= ~SHIFT_U64(RISCV_SATP_ASID_MASK, RISCV_SATP_ASID_SHIFT); + pte = core_mmu_get_user_mapping_entry(prtn); + if (map && map->user_map) { + ptp = core_mmu_ptp_create(pa_to_ppn((paddr_t)map->user_map)); + core_mmu_entry_set(pte, ptp); + core_mmu_table_write_barrier(); + satp |= SHIFT_U64(map->asid, RISCV_SATP_ASID_SHIFT); + write_satp(satp); + } else { + core_mmu_entry_set(pte, 0); + core_mmu_table_write_barrier(); + } + + tlbi_all(); + thread_unmask_exceptions(exceptions); +} + +void core_mmu_get_user_va_range(vaddr_t *base, size_t *size) +{ + assert(user_va_idx != -1); + + if (base) + *base = SHIFT_U64(user_va_idx, CORE_MMU_BASE_TABLE_SHIFT); + + if (size) + *size = BIT64(CORE_MMU_BASE_TABLE_SHIFT); +} + +void core_mmu_get_user_pgdir(struct core_mmu_table_info *pgd_info) +{ + vaddr_t va_range_base = 0; + struct mmu_partition *prtn = core_mmu_get_prtn(); + struct mmu_pgt *pgt = core_mmu_get_ta_pgt_va(prtn); + + core_mmu_get_user_va_range(&va_range_base, NULL); + core_mmu_set_info_table(pgd_info, CORE_MMU_PGDIR_LEVEL + 1, + va_range_base, pgt); +} + +void core_mmu_create_user_map(struct user_mode_ctx *uctx, + struct core_mmu_user_map *map) +{ + struct core_mmu_table_info tbl_info = { }; + + core_mmu_get_user_pgdir(&tbl_info); + memset(tbl_info.table, 0, RISCV_MMU_PGT_SIZE); + core_mmu_populate_user_map(&tbl_info, uctx); + map->user_map = virt_to_phys(tbl_info.table); + map->asid = uctx->vm_info.asid; +} + +void core_mmu_get_user_map(struct core_mmu_user_map *map) +{ + struct mmu_partition *prtn = core_mmu_get_prtn(); + struct mmu_pte *pte = core_mmu_get_user_mapping_entry(prtn); + + map->user_map = pte_to_pa(pte); + + if (map->user_map) + map->asid = (read_satp() >> RISCV_SATP_ASID_SHIFT) & + RISCV_SATP_ASID_MASK; + else + map->asid = 0; +} + +bool core_mmu_user_mapping_is_active(void) +{ + struct mmu_partition *prtn = core_mmu_get_prtn(); + bool ret = false; + struct mmu_pte *pte = NULL; + uint32_t exceptions = 0; + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + pte = core_mmu_get_user_mapping_entry(prtn); + ret = core_mmu_entry_is_valid(pte); + thread_unmask_exceptions(exceptions); + + return ret; +} + +void core_init_mmu_prtn(struct mmu_partition *prtn, struct tee_mmap_region *mm) +{ + core_init_mmu_prtn_tee(prtn, mm); + core_init_mmu_prtn_ta(prtn); +} + +void core_init_mmu(struct tee_mmap_region *mm) +{ + uint64_t max_va = 0; + size_t n = 0; + + static_assert((RISCV_MMU_MAX_PGTS * RISCV_MMU_PGT_SIZE) == + sizeof(pool_pgts)); + + /* Initialize default pagetables */ + core_init_mmu_prtn_tee(&default_partition, mm); + + for (n = 0; !core_mmap_is_end_of_table(mm + n); n++) { + vaddr_t va_end = mm[n].va + mm[n].size - 1; + + if (va_end > max_va) + max_va = va_end; + } + + set_user_va_idx(&default_partition); + + core_init_mmu_prtn_ta(&default_partition); + + assert(max_va < BIT64(RISCV_MMU_VA_WIDTH)); +} + +void core_init_mmu_regs(struct core_mmu_config *cfg) +{ + struct mmu_partition *prtn = core_mmu_get_prtn(); + + cfg->satp = core_mmu_pgt_to_satp(prtn->asid, + core_mmu_get_root_pgt_va(prtn)); +} + +enum core_mmu_fault core_mmu_get_fault_type(uint32_t fault_descr) +{ + switch (fault_descr) { + case CAUSE_MISALIGNED_FETCH: + case CAUSE_MISALIGNED_LOAD: + case CAUSE_MISALIGNED_STORE: + return CORE_MMU_FAULT_ALIGNMENT; + case CAUSE_STORE_ACCESS: + case CAUSE_LOAD_ACCESS: + return CORE_MMU_FAULT_ACCESS_BIT; + case CAUSE_FETCH_PAGE_FAULT: + case CAUSE_LOAD_PAGE_FAULT: + case CAUSE_STORE_PAGE_FAULT: + case CAUSE_FETCH_GUEST_PAGE_FAULT: + case CAUSE_LOAD_GUEST_PAGE_FAULT: + case CAUSE_STORE_GUEST_PAGE_FAULT: + return CORE_MMU_FAULT_TRANSLATION; + case CAUSE_BREAKPOINT: + return CORE_MMU_FAULT_DEBUG_EVENT; + default: + return CORE_MMU_FAULT_OTHER; + } +} diff --git a/optee_os/core/arch/riscv/mm/sub.mk b/optee_os/core/arch/riscv/mm/sub.mk new file mode 100644 index 0000000..17b374b --- /dev/null +++ b/optee_os/core/arch/riscv/mm/sub.mk @@ -0,0 +1,10 @@ +srcs-y += core_mmu_arch.c +srcs-y += tlb_helpers_rv.S + +ifeq ($(CFG_SYSCALL_FTRACE),y) +# We would not like to profile MMU APIs as these are used to switch TA +# context which may cause undesired behaviour as ftrace requires TA context +# to be active. Moreover profiling code uses some of MMU APIs to check +# if TA context is active or not. +cflags-remove-core_mmu_arch.c-y += -pg +endif diff --git a/optee_os/core/arch/riscv/mm/tlb_helpers_rv.S b/optee_os/core/arch/riscv/mm/tlb_helpers_rv.S new file mode 100644 index 0000000..ee8da22 --- /dev/null +++ b/optee_os/core/arch/riscv/mm/tlb_helpers_rv.S @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright 2022 NXP + */ + +#include + +/* void tlbi_all(void); */ +FUNC tlbi_all , : + sfence.vma zero, zero + ret +END_FUNC tlbi_all + +/* void tlbi_va_allasid(vaddr_t va); */ +FUNC tlbi_va_allasid , : + sfence.vma a0, zero + ret +END_FUNC tlbi_va_allasid + +/* void tlbi_asid(unsigned int asid); */ +FUNC tlbi_asid , : + sfence.vma zero, a0 + ret +END_FUNC tlbi_asid + +/* void tlbi_va_asid(vaddr_t va, uint32_t asid); */ +FUNC tlbi_va_asid , : + sfence.vma a0, a1 + ret +END_FUNC tlbi_va_asid diff --git a/optee_os/core/arch/riscv/plat-spike/conf.mk b/optee_os/core/arch/riscv/plat-spike/conf.mk new file mode 100644 index 0000000..27108ac --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/conf.mk @@ -0,0 +1,77 @@ +CFG_RV64_core ?= y +$(call force,CFG_WITH_USER_TA,n) +$(call force,CFG_WITH_SOFTWARE_PRNG,y) +$(call force,CFG_CORE_FFA,n) +$(call force,CFG_CORE_DYN_SHM,n) +$(call force,CFG_SECURE_PARTITION,n) +$(call force,CFG_PAGED_USER_TA,n) +$(call force,CFG_WITH_PAGER,n) +$(call force,CFG_TEE_TA_LOG_LEVEL,0) +$(call force,CFG_EMBEDDED_TS,n) +$(call force,CFG_CORE_ASLR,n) +$(call force,CFG_LIBUTILS_WITH_ISOC,y) +$(call force,CFG_UNWIND,n) +$(call force,CFG_DT,n) +$(call force,CFG_NS_VIRTUALIZATION,n) +$(call force,CFG_WITH_VFP,n) +$(call force,CFG_WITH_STATS,n) +$(call force,CFG_WITH_STMM_SP,n) +$(call force,CFG_WITH_STACK_CANARIES,n) +$(call force,CFG_TEE_TA_MALLOC_DEBUG,n) +$(call force,CFG_COMPAT_GP10_DES,n) +$(call force,CFG_TEE_CORE_TA_TRACE,n) +$(call force,CFG_TEE_CORE_NB_CORE,4) +$(call force,CFG_TEE_CORE_MALLOC_DEBUG,n) +$(call force,CFG_TEE_CORE_DEBUG,n) +$(call force,CFG_SYSTEM_PTA,n) +$(call force,CFG_SYSCALL_FTRACE,n) +$(call force,CFG_CORE_DEBUG_CHECK_STACKS,n) +$(call force,CFG_BOOT_SECONDARY_REQUEST,n) +$(call force,CFG_CORE_DUMP_OOM,n) +$(call force,CFG_CORE_LARGE_PHYS_ADDR,n) +$(call force,CFG_CORE_RESERVED_SHM,y) +$(call force,CFG_CORE_RODATA_NOEXEC,n) +$(call force,CFG_CORE_RWDATA_NOEXEC,y) +$(call force,CFG_CORE_SANITIZE_KADDRESS,n) +$(call force,CFG_CORE_SANITIZE_UNDEFINED,n) +$(call force,CFG_CORE_THREAD_SHIFT,0) +$(call force,CFG_CORE_TPM_EVENT_LOG,n) +$(call force,CFG_DEBUG_INFO,n) +$(call force,CFG_DEVICE_ENUM_PTA,n) +$(call force,CFG_DRIVERS_CLK,n) +$(call force,CFG_DRIVERS_CLK_DT,n) +$(call force,CFG_DRIVERS_CLK_FIXED,n) +$(call force,CFG_EARLY_TA,n) +$(call force,CFG_EARLY_TA_COMPRESS,n) +$(call force,CFG_EMBED_DTB,n) +$(call force,CFG_ENABLE_EMBEDDED_TESTS,n) +$(call force,CFG_ENABLE_SCTLR_Z,n) +$(call force,CFG_EXTERNAL_DTB_OVERLAY,n) +$(call force,CFG_FTRACE_SUPPORT,n) +$(call force,CFG_GENERATE_DTB_OVERLAY,n) +$(call force,CFG_GIC,n) +$(call force,CFG_GP_SOCKETS,n) +$(call force,CFG_LOCKDEP,n) +$(call force,CFG_TEE_CORE_EMBED_INTERNAL_TESTS,n) +$(call force,CFG_MMAP_REGIONS,13) +$(call force,CFG_NUM_THREADS,4) +$(call force,CFG_PAGED_USER_TA,n) +$(call force,CFG_REE_FS,n) +$(call force,CFG_REE_FS_ALLOW_RESET,n) +$(call force,CFG_REE_FS_TA,n) +$(call force,CFG_REE_FS_TA_BUFFERED,n) +$(call force,CFG_RPMB_FS,n) +$(call force,CFG_SCMI_PTA,n) +$(call force,CFG_SCTLR_ALIGNMENT_CHECK,n) +$(call force,CFG_SECSTOR_TA,n) +$(call force,CFG_SHOW_CONF_ON_BOOT,n) +$(call force,CFG_SYSCALL_WRAPPERS_MCOUNT,n) +$(call force,CFG_BOOT_SYNC_CPU,y) +$(call force,CFG_WARN_INSECURE,y) +$(call force,CFG_RISCV_TIME_SOURCE_RDTIME,y) +CFG_RISCV_MTIME_RATE ?= 10000000 +CFG_TDDRAM_START ?= 0xbdb00000 +CFG_TDDRAM_SIZE ?= 0x00f00000 +CFG_SHMEM_START ?= 0x7fe00000 +CFG_SHMEM_SIZE ?= 0x00200000 +CFG_TEE_RAM_VA_SIZE ?= 0x00400000 diff --git a/optee_os/core/arch/riscv/plat-spike/drivers/htif.c b/optee_os/core/arch/riscv/plat-spike/drivers/htif.c new file mode 100644 index 0000000..c7203e5 --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/drivers/htif.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 NXP + */ + +#include +#include +#include +#include +#include + +#include "htif.h" + +static unsigned int htif_global_lock __nex_bss = SPINLOCK_UNLOCK; + +#ifdef HTIF_BASE +register_phys_mem(MEM_AREA_IO_NSEC, HTIF_BASE, + ROUNDUP(HTIF_REG_SIZE, CORE_MMU_PGDIR_SIZE)); +#endif + +void htif_lock_global(void) +{ + cpu_spin_lock(&htif_global_lock); +} + +void htif_unlock_global(void) +{ + cpu_spin_unlock(&htif_global_lock); +} + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct htif_console_data *pd = + container_of(chip, struct htif_console_data, chip); + + return io_pa_or_va(&pd->base, HTIF_REG_SIZE); +} + +static void __maybe_unused tohost_cmd(vaddr_t base, uint64_t dev, + uint64_t cmd, uint64_t data) +{ + while (io_read64(base)) + barrier(); + + io_write64(base, SHIFT_U64(dev, 56) | SHIFT_U64(cmd, 48) | data); +} + +static void htif_console_putc(struct serial_chip *chip, + int ch __maybe_unused) +{ +#ifdef RV64 + vaddr_t base = 0; + + htif_lock_global(); + base = chip_to_base(chip); + tohost_cmd(base, HTIF_DEV_CONSOLE, HTIF_CMD_WRITE, ch); + htif_unlock_global(); +#else +#warning HTIF is not supported on RV32 +#endif +} + +static void htif_console_flush(struct serial_chip *chip __unused) +{ +} + +static const struct serial_ops htif_console_ops = { + .flush = htif_console_flush, + .putc = htif_console_putc, +}; +DECLARE_KEEP_PAGER(htif_console_ops); + +void htif_console_init(struct htif_console_data *pd, paddr_t pbase) +{ + pd->base.pa = pbase; + pd->chip.ops = &htif_console_ops; +} diff --git a/optee_os/core/arch/riscv/plat-spike/drivers/htif.h b/optee_os/core/arch/riscv/plat-spike/drivers/htif.h new file mode 100644 index 0000000..83935af --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/drivers/htif.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef __DRIVERS_HTIF_H__ +#define __DRIVERS_HTIF_H__ + +#include +#include + +#define HTIF_CMD_WRITE 1 +#define HTIF_DEV_CONSOLE 1 +#define HTIF_REG_SIZE (2 * RISCV_XLEN_BYTES) + +struct htif_console_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void htif_lock_global(void); +void htif_unlock_global(void); +void htif_console_init(struct htif_console_data *pd, paddr_t pbase); + +#endif /*__DRIVERS_HTIF_H__*/ diff --git a/optee_os/core/arch/riscv/plat-spike/drivers/sub.mk b/optee_os/core/arch/riscv/plat-spike/drivers/sub.mk new file mode 100644 index 0000000..deb63e0 --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/drivers/sub.mk @@ -0,0 +1 @@ +srcs-$(CFG_RISCV_M_MODE) += htif.c diff --git a/optee_os/core/arch/riscv/plat-spike/kern.ld.S b/optee_os/core/arch/riscv/plat-spike/kern.ld.S new file mode 100644 index 0000000..e437dc0 --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/kern.ld.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: (BSD-2-Clause) */ +/* + * Copyright 2022 NXP + */ + +#include "../kernel/kern.ld.S" + +SECTIONS +{ + .htif HTIF_BASE: + { + tohost = .; + fromhost = tohost + 8; + } + +} diff --git a/optee_os/core/arch/riscv/plat-spike/main.c b/optee_os/core/arch/riscv/plat-spike/main.c new file mode 100644 index 0000000..40de072 --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/main.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 NXP + */ + +#include +#include +#include +#include + +#include "drivers/htif.h" + +#ifdef CFG_RISCV_M_MODE +static struct htif_console_data console_data __nex_bss; + +void console_init(void) +{ +#ifdef HTIF_BASE + htif_console_init(&console_data, HTIF_BASE); + register_serial_console(&console_data.chip); +#endif /*HTIF_BASE*/ +} +#endif /*CFG_RISCV_M_MODE*/ + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + memset(&hwkey->data[0], 0, sizeof(hwkey->data)); + return TEE_SUCCESS; +} diff --git a/optee_os/core/arch/riscv/plat-spike/platform_config.h b/optee_os/core/arch/riscv/plat-spike/platform_config.h new file mode 100644 index 0000000..7a3ff43 --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/platform_config.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + * + * Brief Spike platform configuration. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +#ifndef HTIF_BASE +#define HTIF_BASE 0x40008000 +#endif + +/* CLINT */ +#ifndef CLINT_BASE +#define CLINT_BASE 0x02000000 +#endif + +#endif diff --git a/optee_os/core/arch/riscv/plat-spike/sub.mk b/optee_os/core/arch/riscv/plat-spike/sub.mk new file mode 100644 index 0000000..4bec00d --- /dev/null +++ b/optee_os/core/arch/riscv/plat-spike/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += . +subdirs-y += drivers +srcs-y += main.c diff --git a/optee_os/core/arch/riscv/plat-virt/conf.mk b/optee_os/core/arch/riscv/plat-virt/conf.mk new file mode 100644 index 0000000..98b703f --- /dev/null +++ b/optee_os/core/arch/riscv/plat-virt/conf.mk @@ -0,0 +1,38 @@ +$(call force,CFG_RV64_core,y) + +$(call force,CFG_CORE_LARGE_PHYS_ADDR,y) +$(call force,CFG_TEE_CORE_DEBUG,n) +$(call force,CFG_CORE_DYN_SHM,n) + +# Crypto flags +$(call force,CFG_WITH_SOFTWARE_PRNG,y) + +# Protection flags +$(call force,CFG_CORE_ASLR,n) +$(call force,CFG_WITH_STACK_CANARIES,n) +$(call force,CFG_CORE_SANITIZE_KADDRESS,n) + +# Hart-related flags +$(call force,CFG_TEE_CORE_NB_CORE,1) +$(call force,CFG_NUM_THREADS,1) +$(call force,CFG_BOOT_SYNC_CPU,y) + +# RISC-V-specific flags +rv64-platform-isa ?= rv64imafdc_zicsr_zifencei + +$(call force,CFG_RISCV_PLIC,y) +$(call force,CFG_SBI_CONSOLE,n) +$(call force,CFG_16550_UART,y) +$(call force,CFG_RISCV_TIME_SOURCE_RDTIME,y) +CFG_RISCV_MTIME_RATE ?= 10000000 + +# TA-related flags +supported-ta-targets = ta_rv64 + +# Memory layout flags +CFG_TDDRAM_START ?= 0x8e000000 +CFG_TDDRAM_SIZE ?= 0x00f00000 +$(call force,CFG_CORE_RESERVED_SHM,y) +CFG_SHMEM_START ?= 0x88f00000 +CFG_SHMEM_SIZE ?= 0x00200000 +CFG_TEE_RAM_VA_SIZE ?= 0x00200000 diff --git a/optee_os/core/arch/riscv/plat-virt/main.c b/optee_os/core/arch/riscv/plat-virt/main.c new file mode 100644 index 0000000..edfbf4c --- /dev/null +++ b/optee_os/core/arch/riscv/plat-virt/main.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include + +static struct ns16550_data console_data __nex_bss; + +register_ddr(DRAM_BASE, DRAM_SIZE); + +register_phys_mem_pgdir(MEM_AREA_IO_NSEC, UART0_BASE, + CORE_MMU_PGDIR_SIZE); + +#ifdef CFG_RISCV_PLIC +void boot_primary_init_intc(void) +{ + plic_init(PLIC_BASE); +} + +void boot_secondary_init_intc(void) +{ + plic_hart_init(); +} +#endif /* CFG_RISCV_PLIC */ + +void console_init(void) +{ + ns16550_init(&console_data, UART0_BASE, IO_WIDTH_U8, 0); + register_serial_console(&console_data.chip); +} + +void interrupt_main_handler(void) +{ + if (IS_ENABLED(CFG_RISCV_PLIC)) + plic_it_handle(); +} diff --git a/optee_os/core/arch/riscv/plat-virt/platform_config.h b/optee_os/core/arch/riscv/plat-virt/platform_config.h new file mode 100644 index 0000000..064cca2 --- /dev/null +++ b/optee_os/core/arch/riscv/plat-virt/platform_config.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + * + * Brief Qemu Virt platform configuration. + */ + +#ifndef PLATFORM_CONFIG_H +#define PLATFORM_CONFIG_H + +#include + +/* The stack pointer is always kept 16-byte aligned */ +#define STACK_ALIGNMENT 16 + +/* DRAM */ +#ifndef DRAM_BASE +#define DRAM_BASE 0x80000000 +#define DRAM_SIZE 0x10000000 +#endif + +/* CLINT */ +#ifndef CLINT_BASE +#define CLINT_BASE 0x02000000 +#endif + +/* PLIC */ +#ifndef PLIC_BASE +#define PLIC_BASE 0x0c000000 +#define PLIC_REG_SIZE 0x600000 +#define PLIC_NUM_SOURCES 0x5f +#endif + +/* UART */ +#ifndef UART0_BASE +#define UART0_BASE 0x10000000 +#endif +#define UART0_IRQ 0x0a + +/* RTC */ +#ifndef RTC_BASE +#define RTC_BASE 0x101000 +#endif +#define RTC_IRQ 0x0b + +/* VIRTIO MMIOs */ +#define NUM_VIRTIO_MMIOS 8 + +#ifndef VIRTIO_MMIO1 +#define VIRTIO_MMIO1 0x10001000 +#define VIRTIO_MMIO1_IRQ 0x01 +#endif + +#ifndef VIRTIO_MMIO2 +#define VIRTIO_MMIO2 0x10002000 +#define VIRTIO_MMIO2_IRQ 0x02 +#endif + +#ifndef VIRTIO_MMIO3 +#define VIRTIO_MMIO3 0x10003000 +#define VIRTIO_MMIO3_IRQ 0x03 +#endif + +#ifndef VIRTIO_MMIO4 +#define VIRTIO_MMIO4 0x10004000 +#define VIRTIO_MMIO4_IRQ 0x04 +#endif + +#ifndef VIRTIO_MMIO5 +#define VIRTIO_MMIO5 0x10005000 +#define VIRTIO_MMIO5_IRQ 0x05 +#endif + +#ifndef VIRTIO_MMIO6 +#define VIRTIO_MMIO6 0x10006000 +#define VIRTIO_MMIO6_IRQ 0x06 +#endif + +#ifndef VIRTIO_MMIO7 +#define VIRTIO_MMIO7 0x10007000 +#define VIRTIO_MMIO7_IRQ 0x07 +#endif + +#ifndef VIRTIO_MMIO8 +#define VIRTIO_MMIO8 0x10008000 +#define VIRTIO_MMIO8_IRQ 0x08 +#endif + +#ifdef CFG_RISCV_MTIME_RATE +#define RISCV_MTIME_RATE CFG_RISCV_MTIME_RATE +#else +#define RISCV_MTIME_RATE 1000000 +#endif + +#endif /*PLATFORM_CONFIG_H*/ diff --git a/optee_os/core/arch/riscv/plat-virt/sub.mk b/optee_os/core/arch/riscv/plat-virt/sub.mk new file mode 100644 index 0000000..8ddc2fd --- /dev/null +++ b/optee_os/core/arch/riscv/plat-virt/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += . +srcs-y += main.c diff --git a/optee_os/core/arch/riscv/riscv.mk b/optee_os/core/arch/riscv/riscv.mk new file mode 100644 index 0000000..d9765c9 --- /dev/null +++ b/optee_os/core/arch/riscv/riscv.mk @@ -0,0 +1,237 @@ +# Setup compiler for the core module +ifeq ($(CFG_RV64_core),y) +arch-bits-core := 64 +else +arch-bits-core := 32 +endif +CROSS_COMPILE_core := $(CROSS_COMPILE$(arch-bits-core)) +COMPILER_core := $(COMPILER) + +include mk/$(COMPILER_core).mk + +# Defines the cc-option macro using the compiler set for the core module +include mk/cc-option.mk + +CFG_MMAP_REGIONS ?= 13 +CFG_RESERVED_VASPACE_SIZE ?= (1024 * 1024 * 10) + +ifeq ($(CFG_RV64_core),y) +CFG_KERN_LINKER_FORMAT ?= elf64-littleriscv +CFG_KERN_LINKER_ARCH ?= riscv +else +ifeq ($(CFG_RV32_core),y) +CFG_KERN_LINKER_FORMAT ?= elf32-littleriscv +CFG_KERN_LINKER_ARCH ?= riscv +else +$(error Error: CFG_RV64_core or CFG_RV32_core should be defined) +endif +endif + +CFG_CORE_RWDATA_NOEXEC ?= y +CFG_CORE_RODATA_NOEXEC ?= n +ifeq ($(CFG_CORE_RODATA_NOEXEC),y) +$(call force,CFG_CORE_RWDATA_NOEXEC,y) +endif + +CFG_MAX_CACHE_LINE_SHIFT ?= 6 + +# CFG_WITH_LPAE is ARM-related flag, however, it is used by core code. +# In order to maintain the code logic, we set it when CFG_CORE_LARGE_PHYS_ADDR is set. +# Platform configuration should accordingly set CFG_CORE_LARGE_PHYS_ADDR or not. +ifeq ($(CFG_CORE_LARGE_PHYS_ADDR),y) +$(call force,CFG_WITH_LPAE,y) +endif + +CFG_RISCV_SBI ?= n +CFG_RISCV_M_MODE ?= y +ifeq ($(CFG_RISCV_M_MODE),y) +ifeq ($(CFG_RISCV_S_MODE),y) +$(error CFG_RISCV_M_MODE and CFG_RISCV_S_MODE cannot be both 'y') +else +$(call force,CFG_RISCV_S_MODE,n) +$(call force,CFG_RISCV_SBI,n) +endif +endif +ifeq ($(CFG_RISCV_S_MODE),y) +$(call force,CFG_RISCV_M_MODE,n) +endif +ifneq (y,$(call cfg-one-enabled,CFG_RISCV_M_MODE M CFG_RISCV_S_MODE)) +$(error Either CFG_RISCV_M_MODE or CFG_RISCV_S_MODE must be 'y') +endif + +ifeq ($(CFG_RISCV_SBI_CONSOLE),y) +$(call force,CFG_RISCV_SBI,y) +endif + +# Disable unsupported and other arch-specific flags +$(call force,CFG_CORE_FFA,n) +$(call force,CFG_SECURE_PARTITION,n) +$(call force,CFG_PAGED_USER_TA,n) +$(call force,CFG_WITH_PAGER,n) +$(call force,CFG_GIC,n) +$(call force,CFG_ARM_GICV3,n) +$(call force,CFG_WITH_VFP,n) +$(call force,CFG_WITH_STMM_SP,n) +$(call force,CFG_TA_BTI,n) + +core-platform-cppflags += -I$(arch-dir)/include +core-platform-subdirs += \ + $(addprefix $(arch-dir)/, kernel mm tee) $(platform-dir) + +# Default values for "-mcmodel", "-march", and "-abi" compiler flags. +# Platform-specific overrides are in core/arch/riscv/plat-*/conf.mk. +riscv-platform-mcmodel ?= medany +rv64-platform-isa ?= rv64imafd +rv64-platform-abi ?= lp64d +rv32-platform-isa ?= rv32imafd +rv32-platform-abi ?= ilp32d + +rv64-platform-cflags += -mcmodel=$(riscv-platform-mcmodel) +rv64-platform-cflags += -march=$(rv64-platform-isa) -mabi=$(rv64-platform-abi) +rv64-platform-cflags += -Wno-missing-include-dirs +rv32-platform-cflags += -mcmodel=$(riscv-platform-mcmodel) +rv32-platform-cflags += -march=$(rv32-platform-isa) -mabi=$(rv32-platform-abi) + +rv64-platform-cppflags += -DRV64=1 -D__LP64__=1 +rv32-platform-cppflags += -DRV32=1 -D__ILP32__=1 + +platform-cflags-generic ?= -ffunction-sections -fdata-sections -pipe +platform-aflags-generic ?= -pipe + +rv64-platform-cflags-generic := -mstrict-align $(call cc-option,) + +# Optimize for size by default, usually gives good performance too +CFG_CC_OPT_LEVEL ?= 0 +platform-cflags-optimization ?= -O$(CFG_CC_OPT_LEVEL) + +CFG_DEBUG_INFO ?= y +ifeq ($(CFG_DEBUG_INFO),y) +platform-cflags-debug-info ?= -g3 +platform-aflags-debug-info ?= -g +endif + +core-platform-cflags += $(platform-cflags-optimization) +core-platform-cflags += $(platform-cflags-generic) +core-platform-cflags += $(platform-cflags-debug-info) + +core-platform-aflags += $(platform-aflags-generic) +core-platform-aflags += $(platform-aflags-debug-info) + +ifeq ($(CFG_CORE_ASLR),y) +core-platform-cflags += -fpie +endif + +ifeq ($(CFG_UNWIND),y) +core-platform-cppflags += -fno-omit-frame-pointer +core-platform-cflags += -fno-omit-frame-pointer +endif + +ifeq ($(CFG_RV64_core),y) +core-platform-cppflags += $(rv64-platform-cppflags) +core-platform-cflags += $(rv64-platform-cflags) +core-platform-cflags += $(rv64-platform-cflags-generic) +core-platform-cflags += $(rv64-platform-cflags-no-hard-float) +core-platform-aflags += $(rv64-platform-aflags) +else +core-platform-cppflags += $(rv32-platform-cppflags) +core-platform-cflags += $(rv32-platform-cflags) +core-platform-cflags += $(rv32-platform-cflags-no-hard-float) +ifeq ($(CFG_UNWIND),y) +core-platform-cflags += -funwind-tables +endif +core-platform-aflags += $(core_rv32-platform-aflags) +core-platform-aflags += $(rv32-platform-aflags) +endif + +# Provide default supported-ta-targets if not set by the platform config +ifeq (,$(supported-ta-targets)) +supported-ta-targets = ta_rv32 +ifeq ($(CFG_RV64_core),y) +supported-ta-targets += ta_rv64 +endif +endif + +ta-targets := $(if $(CFG_USER_TA_TARGETS),$(filter $(supported-ta-targets),$(CFG_USER_TA_TARGETS)),$(supported-ta-targets)) +unsup-targets := $(filter-out $(ta-targets),$(CFG_USER_TA_TARGETS)) +ifneq (,$(unsup-targets)) +$(error CFG_USER_TA_TARGETS contains unsupported value(s): $(unsup-targets). Valid values: $(supported-ta-targets)) +endif + +ifneq ($(filter ta_rv32,$(ta-targets)),) +# Variables for ta-target/sm "ta_rv32" +CFG_RV32_ta_rv32 := y +arch-bits-ta_rv32 := 32 +ta_rv32-platform-cppflags += $(rv32-platform-cppflags) +ta_rv32-platform-cflags += $(rv32-platform-cflags) +ta_rv32-platform-cflags += $(platform-cflags-optimization) +ta_rv32-platform-cflags += $(platform-cflags-debug-info) +ta_rv32-platform-cflags += -fpic + +ifeq ($(CFG_UNWIND),y) +ta_rv32-platform-cflags += -fno-omit-frame-pointer +ta_rv32-platform-cflags += -funwind-tables +endif +ta_rv32-platform-aflags += $(platform-aflags-generic) +ta_rv32-platform-aflags += $(platform-aflags-debug-info) +ta_rv32-platform-aflags += $(rv32-platform-aflags) + +ta_rv32-platform-cxxflags += -fpic +ta_rv32-platform-cxxflags += $(rv32-platform-cxxflags) +ta_rv32-platform-cxxflags += $(platform-cflags-optimization) +ta_rv32-platform-cxxflags += $(platform-cflags-debug-info) + +ta-mk-file-export-vars-ta_rv32 += CFG_RV32_ta_rv32 +ta-mk-file-export-vars-ta_rv32 += ta_rv32-platform-cppflags +ta-mk-file-export-vars-ta_rv32 += ta_rv32-platform-cflags +ta-mk-file-export-vars-ta_rv32 += ta_rv32-platform-aflags +ta-mk-file-export-vars-ta_rv32 += ta_rv32-platform-cxxflags + +ta-mk-file-export-add-ta_rv32 += CROSS_COMPILE ?= riscv32-unknown-linux-gnu-_nl_ +ta-mk-file-export-add-ta_rv32 += CROSS_COMPILE32 ?= $$(CROSS_COMPILE)_nl_ +ta-mk-file-export-add-ta_rv32 += CROSS_COMPILE_ta_rv32 ?= $$(CROSS_COMPILE32)_nl_ +ta-mk-file-export-add-ta_rv32 += COMPILER ?= gcc_nl_ +ta-mk-file-export-add-ta_rv32 += COMPILER_ta_rv32 ?= $$(COMPILER)_nl_ +ta-mk-file-export-add-ta_rv32 += PYTHON3 ?= python3_nl_ +endif + +ifneq ($(filter ta_rv64,$(ta-targets)),) +# Variables for ta-target/sm "ta_rv64" +CFG_RV64_ta_rv64 := y +arch-bits-ta_rv64 := 64 +ta_rv64-platform-cppflags += $(rv64-platform-cppflags) +ta_rv64-platform-cflags += $(rv64-platform-cflags) +ta_rv64-platform-cflags += $(platform-cflags-optimization) +ta_rv64-platform-cflags += $(platform-cflags-debug-info) +ta_rv64-platform-cflags += -fpic +ta_rv64-platform-cflags += $(rv64-platform-cflags-generic) +ifeq ($(CFG_UNWIND),y) +ta_rv64-platform-cflags += -fno-omit-frame-pointer +endif +ifeq ($(rv64-platform-hard-float-enabled),y) +ta_rv64-platform-cflags += $(rv64-platform-cflags-hard-float) +else +ta_rv64-platform-cflags += $(rv64-platform-cflags-no-hard-float) +endif +ta_rv64-platform-aflags += $(platform-aflags-generic) +ta_rv64-platform-aflags += $(platform-aflags-debug-info) +ta_rv64-platform-aflags += $(rv64-platform-aflags) + +ta_rv64-platform-cxxflags += -fpic +ta_rv64-platform-cxxflags += $(platform-cflags-optimization) +ta_rv64-platform-cxxflags += $(platform-cflags-debug-info) + +ta-mk-file-export-vars-ta_rv64 += CFG_RV64_ta_rv64 +ta-mk-file-export-vars-ta_rv64 += ta_rv64-platform-cppflags +ta-mk-file-export-vars-ta_rv64 += ta_rv64-platform-cflags +ta-mk-file-export-vars-ta_rv64 += ta_rv64-platform-aflags +ta-mk-file-export-vars-ta_rv64 += ta_rv64-platform-cxxflags + +ta-mk-file-export-add-ta_rv64 += CROSS_COMPILE64 ?= $$(CROSS_COMPILE)_nl_ +ta-mk-file-export-add-ta_rv64 += CROSS_COMPILE_ta_rv64 ?= $$(CROSS_COMPILE64)_nl_ +ta-mk-file-export-add-ta_rv64 += COMPILER ?= gcc_nl_ +ta-mk-file-export-add-ta_rv64 += COMPILER_ta_rv64 ?= $$(COMPILER)_nl_ +ta-mk-file-export-add-ta_rv64 += PYTHON3 ?= python3_nl_ +endif + +# Set cross compiler prefix for each TA target +$(foreach sm, $(ta-targets), $(eval CROSS_COMPILE_$(sm) ?= $(CROSS_COMPILE$(arch-bits-$(sm))))) diff --git a/optee_os/core/arch/riscv/tee/entry_fast.c b/optee_os/core/arch/riscv/tee/entry_fast.c new file mode 100644 index 0000000..99faa15 --- /dev/null +++ b/optee_os/core/arch/riscv/tee/entry_fast.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2023 NXP + * Copyright (c) 2015-2021, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_CORE_RESERVED_SHM +static void tee_entry_get_shm_config(struct thread_abi_args *args) +{ + args->a0 = OPTEE_ABI_RETURN_OK; + args->a1 = default_nsec_shm_paddr; + args->a2 = default_nsec_shm_size; + /* Should this be TEEABI cache attributes instead? */ + args->a3 = core_mmu_is_shm_cached(); +} +#endif + +static void tee_entry_fastcall_l2cc_mutex(struct thread_abi_args *args) +{ + args->a0 = OPTEE_ABI_RETURN_UNKNOWN_FUNCTION; +} + +static void tee_entry_exchange_capabilities(struct thread_abi_args *args) +{ + bool res_shm_en = IS_ENABLED(CFG_CORE_RESERVED_SHM); + bool dyn_shm_en __maybe_unused = false; + + /* + * Currently we ignore OPTEE_ABI_NSEC_CAP_UNIPROCESSOR. + * + * The memory mapping of shared memory is defined as normal + * shared memory for SMP systems and normal memory for UP + * systems. Currently we map all memory as shared in secure + * world. + * + * When translation tables are created with shared bit cleared for + * uniprocessor systems we'll need to check + * OPTEE_ABI_NSEC_CAP_UNIPROCESSOR. + */ + + if (args->a1 & ~OPTEE_ABI_NSEC_CAP_UNIPROCESSOR) { + /* Unknown capability. */ + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; + return; + } + + args->a0 = OPTEE_ABI_RETURN_OK; + args->a1 = 0; + + if (res_shm_en) + args->a1 |= OPTEE_ABI_SEC_CAP_HAVE_RESERVED_SHM; + IMSG("Reserved shared memory is %sabled", res_shm_en ? "en" : "dis"); + +#if defined(CFG_CORE_DYN_SHM) + dyn_shm_en = core_mmu_nsec_ddr_is_defined(); + if (dyn_shm_en) + args->a1 |= OPTEE_ABI_SEC_CAP_DYNAMIC_SHM; +#endif + IMSG("Dynamic shared memory is %sabled", dyn_shm_en ? "en" : "dis"); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + args->a1 |= OPTEE_ABI_SEC_CAP_VIRTUALIZATION; + IMSG("Normal World virtualization support is %sabled", + IS_ENABLED(CFG_NS_VIRTUALIZATION) ? "en" : "dis"); + + args->a1 |= OPTEE_ABI_SEC_CAP_MEMREF_NULL; + + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) { + args->a1 |= OPTEE_ABI_SEC_CAP_ASYNC_NOTIF; + args->a2 = NOTIF_VALUE_MAX; + } + IMSG("Asynchronous notifications are %sabled", + IS_ENABLED(CFG_CORE_ASYNC_NOTIF) ? "en" : "dis"); + + args->a1 |= OPTEE_ABI_SEC_CAP_RPC_ARG; + args->a3 = THREAD_RPC_MAX_NUM_PARAMS; +} + +static void tee_entry_disable_shm_cache(struct thread_abi_args *args) +{ + uint64_t cookie; + + if (!thread_disable_prealloc_rpc_cache(&cookie)) { + args->a0 = OPTEE_ABI_RETURN_EBUSY; + return; + } + + if (!cookie) { + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; + return; + } + + args->a0 = OPTEE_ABI_RETURN_OK; + args->a1 = cookie >> 32; + args->a2 = cookie; +} + +static void tee_entry_enable_shm_cache(struct thread_abi_args *args) +{ + if (thread_enable_prealloc_rpc_cache()) + args->a0 = OPTEE_ABI_RETURN_OK; + else + args->a0 = OPTEE_ABI_RETURN_EBUSY; +} + +static void tee_entry_boot_secondary(struct thread_abi_args *args) +{ +#if defined(CFG_BOOT_SECONDARY_REQUEST) + if (!boot_core_release(args->a1, (paddr_t)(args->a3))) + args->a0 = OPTEE_ABI_RETURN_OK; + else + args->a0 = OPTEE_ABI_RETURN_EBADCMD; +#else + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; +#endif +} + +static void tee_entry_get_thread_count(struct thread_abi_args *args) +{ + args->a0 = OPTEE_ABI_RETURN_OK; + args->a1 = CFG_NUM_THREADS; +} + +#if defined(CFG_NS_VIRTUALIZATION) +static void tee_entry_vm_created(struct thread_abi_args *args) +{ + uint16_t guest_id = args->a1; + + /* Only hypervisor can issue this request */ + if (args->a7 != HYP_CLNT_ID) { + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; + return; + } + + if (virt_guest_created(guest_id)) + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; + else + args->a0 = OPTEE_ABI_RETURN_OK; +} + +static void tee_entry_vm_destroyed(struct thread_abi_args *args) +{ + uint16_t guest_id = args->a1; + + /* Only hypervisor can issue this request */ + if (args->a7 != HYP_CLNT_ID) { + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; + return; + } + + if (virt_guest_destroyed(guest_id)) + args->a0 = OPTEE_ABI_RETURN_ENOTAVAIL; + else + args->a0 = OPTEE_ABI_RETURN_OK; +} +#endif + +/* Note: this function is weak to let platforms add special handling */ +void __weak tee_entry_fast(struct thread_abi_args *args) +{ + __tee_entry_fast(args); +} + +static void get_async_notif_value(struct thread_abi_args *args) +{ + bool value_valid = false; + bool value_pending = false; + + args->a0 = OPTEE_ABI_RETURN_OK; + args->a1 = notif_get_value(&value_valid, &value_pending); + args->a2 = 0; + if (value_valid) + args->a2 |= OPTEE_ABI_ASYNC_NOTIF_VALID; + if (value_pending) + args->a2 |= OPTEE_ABI_ASYNC_NOTIF_PENDING; +} + +/* + * If tee_entry_fast() is overridden, it's still supposed to call this + * function. + */ +void __tee_entry_fast(struct thread_abi_args *args) +{ + switch (args->a0) { + /* Generic functions */ + case OPTEE_ABI_CALLS_COUNT: + tee_entry_get_api_call_count(args); + break; + case OPTEE_ABI_CALLS_UID: + tee_entry_get_api_uuid(args); + break; + case OPTEE_ABI_CALLS_REVISION: + tee_entry_get_api_revision(args); + break; + case OPTEE_ABI_CALL_GET_OS_UUID: + tee_entry_get_os_uuid(args); + break; + case OPTEE_ABI_CALL_GET_OS_REVISION: + tee_entry_get_os_revision(args); + break; + + /* OP-TEE specific ABI functions */ +#ifdef CFG_CORE_RESERVED_SHM + case OPTEE_ABI_GET_SHM_CONFIG: + tee_entry_get_shm_config(args); + break; +#endif + case OPTEE_ABI_L2CC_MUTEX: + tee_entry_fastcall_l2cc_mutex(args); + break; + case OPTEE_ABI_EXCHANGE_CAPABILITIES: + tee_entry_exchange_capabilities(args); + break; + case OPTEE_ABI_DISABLE_SHM_CACHE: + tee_entry_disable_shm_cache(args); + break; + case OPTEE_ABI_ENABLE_SHM_CACHE: + tee_entry_enable_shm_cache(args); + break; + case OPTEE_ABI_BOOT_SECONDARY: + tee_entry_boot_secondary(args); + break; + case OPTEE_ABI_GET_THREAD_COUNT: + tee_entry_get_thread_count(args); + break; + +#if defined(CFG_NS_VIRTUALIZATION) + case OPTEE_ABI_VM_CREATED: + tee_entry_vm_created(args); + break; + case OPTEE_ABI_VM_DESTROYED: + tee_entry_vm_destroyed(args); + break; +#endif + + case OPTEE_ABI_ENABLE_ASYNC_NOTIF: + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) { + notif_deliver_atomic_event(NOTIF_EVENT_STARTED); + args->a0 = OPTEE_ABI_RETURN_OK; + } else { + args->a0 = OPTEE_ABI_RETURN_UNKNOWN_FUNCTION; + } + break; + case OPTEE_ABI_GET_ASYNC_NOTIF_VALUE: + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) + get_async_notif_value(args); + else + args->a0 = OPTEE_ABI_RETURN_UNKNOWN_FUNCTION; + break; + + default: + args->a0 = OPTEE_ABI_RETURN_UNKNOWN_FUNCTION; + break; + } +} + +size_t tee_entry_generic_get_api_call_count(void) +{ + /* + * All the different calls handled in this file. If the specific + * target has additional calls it will call this function and + * add the number of calls the target has added. + */ + size_t ret = 12; + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + ret += 2; + + return ret; +} + +void __weak tee_entry_get_api_call_count(struct thread_abi_args *args) +{ + args->a0 = tee_entry_generic_get_api_call_count(); +} + +void __weak tee_entry_get_api_uuid(struct thread_abi_args *args) +{ + args->a0 = OPTEE_MSG_UID_0; + args->a1 = OPTEE_MSG_UID_1; + args->a2 = OPTEE_MSG_UID_2; + args->a3 = OPTEE_MSG_UID_3; +} + +void __weak tee_entry_get_api_revision(struct thread_abi_args *args) +{ + args->a0 = OPTEE_MSG_REVISION_MAJOR; + args->a1 = OPTEE_MSG_REVISION_MINOR; +} + +void __weak tee_entry_get_os_uuid(struct thread_abi_args *args) +{ + args->a0 = OPTEE_MSG_OS_OPTEE_UUID_0; + args->a1 = OPTEE_MSG_OS_OPTEE_UUID_1; + args->a2 = OPTEE_MSG_OS_OPTEE_UUID_2; + args->a3 = OPTEE_MSG_OS_OPTEE_UUID_3; +} + +void __weak tee_entry_get_os_revision(struct thread_abi_args *args) +{ + args->a0 = CFG_OPTEE_REVISION_MAJOR; + args->a1 = CFG_OPTEE_REVISION_MINOR; + args->a2 = TEE_IMPL_GIT_SHA1; +} diff --git a/optee_os/core/arch/riscv/tee/sub.mk b/optee_os/core/arch/riscv/tee/sub.mk new file mode 100644 index 0000000..958c87a --- /dev/null +++ b/optee_os/core/arch/riscv/tee/sub.mk @@ -0,0 +1,2 @@ +srcs-y += entry_fast.c +cppflags-entry_fast.c-y += -DTEE_IMPL_GIT_SHA1=$(TEE_IMPL_GIT_SHA1) diff --git a/optee_os/core/core.mk b/optee_os/core/core.mk new file mode 100644 index 0000000..b023e46 --- /dev/null +++ b/optee_os/core/core.mk @@ -0,0 +1,176 @@ +include mk/cleanvars.mk + +# Set current submodule (used for module specific flags compile result etc) +sm := core +sm-$(sm) := y + +arch-dir := core/arch/$(ARCH) +platform-dir := $(arch-dir)/plat-$(PLATFORM) +include $(platform-dir)/conf.mk +include mk/config.mk +# $(ARCH).mk also sets the compiler for the core module +include core/arch/$(ARCH)/$(ARCH).mk + +PLATFORM_$(PLATFORM) := y +PLATFORM_FLAVOR_$(PLATFORM_FLAVOR) := y + +$(eval $(call cfg-depends-all,CFG_PAGED_USER_TA,CFG_WITH_PAGER CFG_WITH_USER_TA)) +include core/crypto.mk + +ifeq ($(CFG_SCMI_SCPFW),y) +include core/lib/scmi-server/conf.mk +endif + +cppflags$(sm) += -D__KERNEL__ + +cppflags$(sm) += -Icore/include +cppflags$(sm) += -include $(conf-file) +cppflags$(sm) += -I$(out-dir)/core/include +cppflags$(sm) += $(core-platform-cppflags) +cflags$(sm) += $(core-platform-cflags) + +core-stackp-cflags-$(CFG_CORE_STACK_PROTECTOR) := -fstack-protector +core-stackp-cflags-$(CFG_CORE_STACK_PROTECTOR_STRONG) := -fstack-protector-strong +core-stackp-cflags-$(CFG_CORE_STACK_PROTECTOR_ALL) := -fstack-protector-all +cflags$(sm) += $(core-stackp-cflags-y) + +ifeq ($(CFG_CORE_SANITIZE_UNDEFINED),y) +cflags$(sm) += -fsanitize=undefined +endif +ifeq ($(CFG_CORE_SANITIZE_KADDRESS),y) +ifeq ($(CFG_ASAN_SHADOW_OFFSET),) +$(error error: CFG_CORE_SANITIZE_KADDRESS not supported by platform (flavor)) +endif +ifeq ($(COMPILER),clang) +$(error error: CFG_CORE_SANITIZE_KADDRESS not supported with Clang) +endif +cflags_kasan += -fsanitize=kernel-address \ + -fasan-shadow-offset=$(CFG_ASAN_SHADOW_OFFSET)\ + --param asan-stack=1 --param asan-globals=1 \ + --param asan-instrumentation-with-call-threshold=0 +cflags$(sm) += $(cflags_kasan) +endif +ifeq ($(CFG_CORE_DEBUG_CHECK_STACKS),y) +finstrument-functions := $(call cc-option,-finstrument-functions) +ifeq (,$(finstrument-functions)) +$(error -finstrument-functions not supported) +endif +cflags$(sm) += $(finstrument-functions) +endif +ifeq ($(CFG_SYSCALL_FTRACE),y) +cflags$(sm) += -pg +endif +aflags$(sm) += $(core-platform-aflags) + +cppflags$(sm) += -DTRACE_LEVEL=$(CFG_TEE_CORE_LOG_LEVEL) +ifeq ($(CFG_TEE_CORE_MALLOC_DEBUG),y) +cppflags$(sm) += -DENABLE_MDBG=1 +endif +ifneq ($(CFG_TEE_CORE_DEBUG),y) +cppflags$(sm) += -DNDEBUG +endif + +cppflags$(sm) += -Ildelf/include +cppflags$(sm) += -Ilib/libutee/include + +ifeq ($(filter y, $(CFG_CORE_DYN_SHM) $(CFG_CORE_RESERVED_SHM)),) +$(error error: No shared memory configured) +endif + +# Tell all libraries and sub-directories (included below) that we have a +# configuration file + +conf-file := $(out-dir)/include/generated/conf.h +conf-mk-file := $(out-dir)/conf.mk +conf-cmake-file := $(out-dir)/conf.cmake +$(conf-file): $(conf-mk-file) + +cleanfiles += $(conf-file) +cleanfiles += $(conf-mk-file) +cleanfiles += $(conf-cmake-file) + +$(conf-file): FORCE + $(call check-conf-h) + +$(conf-mk-file): FORCE + $(call check-conf-mk) + +$(conf-cmake-file): FORCE + $(call check-conf-cmake) + +# +# Do libraries +# + +# Set a prefix to avoid conflicts with user TAs that will use the same +# source but with different flags below +base-prefix := $(sm)- +libname = utils +libdir = lib/libutils +include mk/lib.mk + +# CFG_CRYPTOLIB_NAME must not be changed beyond this line +CFG_CRYPTOLIB_NAME_$(CFG_CRYPTOLIB_NAME) := y + +ifeq ($(CFG_CRYPTOLIB_NAME),tomcrypt) +# We're compiling mbedtls too, but with a limited configuration which only +# provides the MPI routines +libname = mbedtls +libdir = lib/libmbedtls +include mk/lib.mk +endif #tomcrypt + +ifeq ($(CFG_CRYPTOLIB_NAME),mbedtls) +$(call force,CFG_CRYPTO_RSASSA_NA1,n,not supported by mbedtls) +libname = tomcrypt +libdir = core/lib/libtomcrypt +base-prefix := +include mk/lib.mk +base-prefix := $(sm)- +endif + +ifeq ($(firstword $(subst /, ,$(CFG_CRYPTOLIB_DIR))),core) +# If a library can be compiled for both core and user space a base-prefix +# is needed in order to avoid conflicts in the output. However, if the +# library resides under core then it can't be compiled to user space. +base-prefix := +endif + +libname = $(CFG_CRYPTOLIB_NAME) +libdir = $(CFG_CRYPTOLIB_DIR) +include mk/lib.mk + +base-prefix := + +libname = fdt +libdir = core/lib/libfdt +include mk/lib.mk + +ifeq ($(CFG_ZLIB),y) +libname = zlib +libdir = core/lib/zlib +include mk/lib.mk +endif + +libname = unw +libdir = lib/libunw +include mk/lib.mk + +ifeq ($(CFG_SCMI_SCPFW),y) +libname = scmi-server +libdir = core/lib/scmi-server +include mk/lib.mk +endif + +# +# Do main source +# + +subdirs = $(core-platform-subdirs) core +include mk/subdir.mk + +include mk/compile.mk + +include $(if $(wildcard $(platform-dir)/link.mk), \ + $(platform-dir)/link.mk, \ + core/arch/$(ARCH)/kernel/link.mk) diff --git a/optee_os/core/crypto.mk b/optee_os/core/crypto.mk new file mode 100644 index 0000000..ac651ee --- /dev/null +++ b/optee_os/core/crypto.mk @@ -0,0 +1,312 @@ +CFG_CRYPTO ?= y +# Select small code size in the crypto library if applicable (for instance +# LibTomCrypt has -DLTC_SMALL_CODE) +# Note: the compiler flag -Os is not set here but by CFG_CC_OPT_LEVEL +CFG_CRYPTO_SIZE_OPTIMIZATION ?= y + +ifeq (y,$(CFG_CRYPTO)) + +############################################################### +# Platform crypto-driver configuration. It has a higher priority over the +# generic crypto configuration below. +############################################################### +CRYPTO_MAKEFILES := $(sort $(wildcard core/drivers/crypto/*/crypto.mk)) +include $(CRYPTO_MAKEFILES) + +# Ciphers +CFG_CRYPTO_AES ?= y +CFG_CRYPTO_DES ?= y +CFG_CRYPTO_SM4 ?= y + +# Cipher block modes +CFG_CRYPTO_ECB ?= y +CFG_CRYPTO_CBC ?= y +CFG_CRYPTO_CTR ?= y +CFG_CRYPTO_CTS ?= y +CFG_CRYPTO_XTS ?= y + +# Message authentication codes +CFG_CRYPTO_HMAC ?= y +CFG_CRYPTO_CMAC ?= y +CFG_CRYPTO_CBC_MAC ?= y +# Instead of calling the AES CBC encryption function for each 16 byte block of +# input, bundle a maximum of N blocks when possible. A maximum of N*16 bytes of +# temporary data are allocated on the heap. +# Minimum value is 1. +CFG_CRYPTO_CBC_MAC_BUNDLE_BLOCKS ?= 64 + +# Hashes +CFG_CRYPTO_MD5 ?= y +CFG_CRYPTO_SHA1 ?= y +CFG_CRYPTO_SHA224 ?= y +CFG_CRYPTO_SHA256 ?= y +CFG_CRYPTO_SHA384 ?= y +CFG_CRYPTO_SHA512 ?= y +CFG_CRYPTO_SHA512_256 ?= y +CFG_CRYPTO_SM3 ?= y +CFG_CRYPTO_SHA3_224 ?= y +CFG_CRYPTO_SHA3_256 ?= y +CFG_CRYPTO_SHA3_384 ?= y +CFG_CRYPTO_SHA3_512 ?= y + +# Extendable-Output Functions (XOF) +CFG_CRYPTO_SHAKE128 ?= y +CFG_CRYPTO_SHAKE256 ?= y + +# Asymmetric ciphers +CFG_CRYPTO_DSA ?= y +CFG_CRYPTO_RSA ?= y +CFG_CRYPTO_DH ?= y +# ECC includes ECDSA and ECDH +CFG_CRYPTO_ECC ?= y +CFG_CRYPTO_SM2_PKE ?= y +CFG_CRYPTO_SM2_DSA ?= y +CFG_CRYPTO_SM2_KEP ?= y +CFG_CRYPTO_ED25519 ?= y +CFG_CRYPTO_X25519 ?= y + +# Authenticated encryption +CFG_CRYPTO_CCM ?= y +CFG_CRYPTO_GCM ?= y +# Default uses the OP-TEE internal AES-GCM implementation +CFG_CRYPTO_AES_GCM_FROM_CRYPTOLIB ?= n + +endif + +# PRNG configuration +# If CFG_WITH_SOFTWARE_PRNG is enabled, crypto provider provided +# software PRNG implementation is used. +# Otherwise, you need to implement hw_get_random_bytes() for your platform +CFG_WITH_SOFTWARE_PRNG ?= y + +ifeq ($(CFG_WITH_PAGER),y) +ifneq ($(CFG_CRYPTO_SHA256),y) +$(warning Warning: Enabling CFG_CRYPTO_SHA256 [required by CFG_WITH_PAGER]) +CFG_CRYPTO_SHA256:=y +endif +endif + +$(eval $(call cryp-enable-all-depends,CFG_WITH_SOFTWARE_PRNG, AES ECB SHA256)) + +ifeq ($(CFG_CRYPTO_WITH_CE82),y) +$(call force,CFG_CRYPTO_WITH_CE,y,required with CFG_CRYPTO_WITH_CE82) +CFG_CRYPTO_SHA512_ARM_CE ?= $(CFG_CRYPTO_SHA512) +CFG_CORE_CRYPTO_SHA512_ACCEL ?= $(CFG_CRYPTO_SHA512_ARM_CE) +CFG_CRYPTO_SHA3_ARM_CE ?= $(call cfg-one-enabled, CFG_CRYPTO_SHA3_224 \ + CFG_CRYPTO_SHA3_256 CFG_CRYPTO_SHA3_384 \ + CFG_CRYPTO_SHA3_512 CFG_CRYPTO_SHAKE128 \ + CFG_CRYPTO_SHAKE256) +CFG_CORE_CRYPTO_SHA3_ACCEL ?= $(CFG_CRYPTO_SHA3_ARM_CE) +CFG_CRYPTO_SM3_ARM_CE ?= $(CFG_CRYPTO_SM3) +CFG_CORE_CRYPTO_SM3_ACCEL ?= $(CFG_CRYPTO_SM3_ARM_CE) + +# CFG_CRYPTO_SM4_ARM_CE defines whether we use SM4E to optimize SM4 +CFG_CRYPTO_SM4_ARM_CE ?= $(CFG_CRYPTO_SM4) +CFG_CORE_CRYPTO_SM4_ACCEL ?= $(CFG_CRYPTO_SM4_ARM_CE) +endif + +ifeq ($(CFG_CRYPTO_WITH_CE),y) + +$(call force,CFG_AES_GCM_TABLE_BASED,n,conflicts with CFG_CRYPTO_WITH_CE) + +# CFG_HWSUPP_PMULT_64 defines whether the CPU supports polynomial multiplies +# of 64-bit values (Aarch64: PMULL/PMULL2 with the 1Q specifier; Aarch32: +# VMULL.P64). These operations are part of the Cryptographic Extensions, so +# assume they are implicitly contained in CFG_CRYPTO_WITH_CE=y. +CFG_HWSUPP_PMULT_64 ?= y + +CFG_CRYPTO_SHA256_ARM_CE ?= $(CFG_CRYPTO_SHA256) +CFG_CORE_CRYPTO_SHA256_ACCEL ?= $(CFG_CRYPTO_SHA256_ARM_CE) +CFG_CRYPTO_SHA1_ARM_CE ?= $(CFG_CRYPTO_SHA1) +CFG_CORE_CRYPTO_SHA1_ACCEL ?= $(CFG_CRYPTO_SHA1_ARM_CE) +CFG_CRYPTO_AES_ARM_CE ?= $(CFG_CRYPTO_AES) +CFG_CORE_CRYPTO_AES_ACCEL ?= $(CFG_CRYPTO_AES_ARM_CE) + +# CFG_CRYPTO_SM4_ARM_AESE defines whether we use AESE to optimize SM4 +CFG_CRYPTO_SM4_ARM_AESE ?= $(CFG_CRYPTO_SM4) +CFG_CORE_CRYPTO_SM4_ACCEL ?= $(CFG_CRYPTO_SM4_ARM_AESE) +else #CFG_CRYPTO_WITH_CE + +CFG_AES_GCM_TABLE_BASED ?= y + +endif #!CFG_CRYPTO_WITH_CE + + +# Cryptographic extensions can only be used safely when OP-TEE knows how to +# preserve the VFP context +ifeq ($(CFG_CRYPTO_SHA256_ARM32_CE),y) +$(call force,CFG_WITH_VFP,y,required by CFG_CRYPTO_SHA256_ARM32_CE) +endif +ifeq ($(CFG_CRYPTO_SHA256_ARM64_CE),y) +$(call force,CFG_WITH_VFP,y,required by CFG_CRYPTO_SHA256_ARM64_CE) +endif +ifeq ($(CFG_CRYPTO_SHA1_ARM_CE),y) +$(call force,CFG_WITH_VFP,y,required by CFG_CRYPTO_SHA1_ARM_CE) +endif +ifeq ($(CFG_CRYPTO_AES_ARM_CE),y) +$(call force,CFG_WITH_VFP,y,required by CFG_CRYPTO_AES_ARM_CE) +endif +ifeq ($(CFG_CORE_CRYPTO_SM4_ACCEL),y) +$(call force,CFG_WITH_VFP,y,required by CFG_CORE_CRYPTO_SM4_ACCEL) +endif +cryp-enable-all-depends = $(call cfg-enable-all-depends,$(strip $(1)),$(foreach v,$(2),CFG_CRYPTO_$(v))) +$(eval $(call cryp-enable-all-depends,CFG_REE_FS, AES ECB CTR HMAC SHA256 GCM)) +$(eval $(call cryp-enable-all-depends,CFG_RPMB_FS, AES ECB CTR HMAC SHA256 GCM)) + +# Dependency checks: warn and disable some features if dependencies are not met + +cryp-dep-one = $(call cfg-depends-one,CFG_CRYPTO_$(strip $(1)),$(patsubst %, CFG_CRYPTO_%,$(strip $(2)))) +cryp-dep-all = $(call cfg-depends-all,CFG_CRYPTO_$(strip $(1)),$(patsubst %, CFG_CRYPTO_%,$(strip $(2)))) + +$(eval $(call cryp-dep-one, ECB, AES DES)) +$(eval $(call cryp-dep-one, CBC, AES DES)) +$(eval $(call cryp-dep-one, CTR, AES)) +# CTS is implemented with ECB and CBC +$(eval $(call cryp-dep-all, CTS, AES ECB CBC)) +$(eval $(call cryp-dep-one, XTS, AES)) +$(eval $(call cryp-dep-one, HMAC, AES DES)) +$(eval $(call cryp-dep-one, HMAC, MD5 SHA1 SHA224 SHA256 SHA384 SHA512)) +$(eval $(call cryp-dep-one, CMAC, AES)) +$(eval $(call cryp-dep-one, CBC_MAC, AES DES)) +$(eval $(call cryp-dep-one, CCM, AES)) +$(eval $(call cryp-dep-one, GCM, AES)) +# If no AES cipher mode is left, disable AES +$(eval $(call cryp-dep-one, AES, ECB CBC CTR CTS XTS)) +# If no DES cipher mode is left, disable DES +$(eval $(call cryp-dep-one, DES, ECB CBC)) +# SM2 is Elliptic Curve Cryptography, it uses some generic ECC functions +$(eval $(call cryp-dep-one, SM2_PKE, ECC)) +$(eval $(call cryp-dep-one, SM2_DSA, ECC)) +$(eval $(call cryp-dep-one, SM2_KEP, ECC)) + +############################################################### +# libtomcrypt (LTC) specifics, phase #1 +# LTC is only configured via _CFG_CORE_LTC_ prefixed variables +# +# _CFG_CORE_LTC_xxx_DESC means that LTC will only register the +# descriptor of the algorithm, not provide a +# crypt_xxx_alloc_ctx() function. +############################################################### + +# If LTC is the cryptolib, pull configuration from CFG_CRYPTO_xxx +ifeq ($(CFG_CRYPTOLIB_NAME),tomcrypt) +# dsa_make_params() needs all three SHA-2 algorithms. +# Disable DSA if any is missing. +$(eval $(call cryp-dep-all, DSA, SHA256 SHA384 SHA512)) + +# Assign _CFG_CORE_LTC_xxx based on CFG_CRYPTO_yyy +core-ltc-vars = AES DES +core-ltc-vars += ECB CBC CTR CTS XTS +core-ltc-vars += MD5 SHA1 SHA224 SHA256 SHA384 SHA512 SHA512_256 +core-ltc-vars += SHA3_224 SHA3_256 SHA3_384 SHA3_512 SHAKE128 SHAKE256 +core-ltc-vars += HMAC CMAC CBC_MAC +core-ltc-vars += CCM +ifeq ($(CFG_CRYPTO_AES_GCM_FROM_CRYPTOLIB),y) +core-ltc-vars += GCM +endif +core-ltc-vars += RSA DSA DH ECC +core-ltc-vars += SIZE_OPTIMIZATION +core-ltc-vars += SM2_PKE +core-ltc-vars += SM2_DSA +core-ltc-vars += SM2_KEP +core-ltc-vars += ED25519 X25519 +# Assigned selected CFG_CRYPTO_xxx as _CFG_CORE_LTC_xxx +$(foreach v, $(core-ltc-vars), $(eval _CFG_CORE_LTC_$(v) := $(CFG_CRYPTO_$(v)))) +_CFG_CORE_LTC_MPI := $(CFG_CORE_MBEDTLS_MPI) +_CFG_CORE_LTC_AES_ACCEL := $(CFG_CORE_CRYPTO_AES_ACCEL) +_CFG_CORE_LTC_SHA1_ACCEL := $(CFG_CORE_CRYPTO_SHA1_ACCEL) +_CFG_CORE_LTC_SHA256_ACCEL := $(CFG_CORE_CRYPTO_SHA256_ACCEL) +_CFG_CORE_LTC_SHA512_ACCEL := $(CFG_CORE_CRYPTO_SHA512_ACCEL) +_CFG_CORE_LTC_SHA3_ACCEL := $(CFG_CORE_CRYPTO_SHA3_ACCEL) +endif + +############################################################### +# mbedtls specifics +############################################################### + +ifeq ($(CFG_CRYPTOLIB_NAME),mbedtls) +# mbedtls has to be complemented with some algorithms by LTC +# Specify the algorithms here +_CFG_CORE_LTC_DSA := $(CFG_CRYPTO_DSA) +_CFG_CORE_LTC_MPI := $(CFG_CRYPTO_DSA) +_CFG_CORE_LTC_SHA256_DESC := $(CFG_CRYPTO_DSA) +_CFG_CORE_LTC_SHA384_DESC := $(CFG_CRYPTO_DSA) +_CFG_CORE_LTC_SHA512_DESC := $(CFG_CRYPTO_DSA) +_CFG_CORE_LTC_XTS := $(CFG_CRYPTO_XTS) +_CFG_CORE_LTC_CCM := $(CFG_CRYPTO_CCM) +_CFG_CORE_LTC_AES := $(call cfg-one-enabled, CFG_CRYPTO_XTS CFG_CRYPTO_CCM \ + CFG_CRYPTO_AES) +_CFG_CORE_LTC_AES_ACCEL := $(CFG_CORE_CRYPTO_AES_ACCEL) +_CFG_CORE_LTC_X25519 := $(CFG_CRYPTO_X25519) +_CFG_CORE_LTC_ED25519 := $(CFG_CRYPTO_ED25519) +_CFG_CORE_LTC_SHA3_224 := $(CFG_CRYPTO_SHA3_224) +_CFG_CORE_LTC_SHA3_256 := $(CFG_CRYPTO_SHA3_256) +_CFG_CORE_LTC_SHA3_384 := $(CFG_CRYPTO_SHA3_384) +_CFG_CORE_LTC_SHA3_512 := $(CFG_CRYPTO_SHA3_512) +_CFG_CORE_LTC_SHAKE128 := $(CFG_CRYPTO_SHAKE128) +_CFG_CORE_LTC_SHAKE256 := $(CFG_CRYPTO_SHAKE256) +endif + +############################################################### +# libtomcrypt (LTC) specifics, phase #2 +############################################################### + +_CFG_CORE_LTC_MD5_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_MD5_DESC \ + _CFG_CORE_LTC_MD5) +_CFG_CORE_LTC_SHA1_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_SHA1_DESC \ + _CFG_CORE_LTC_SHA1) +_CFG_CORE_LTC_SHA224_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_SHA224_DESC \ + _CFG_CORE_LTC_SHA224) +_CFG_CORE_LTC_SHA256_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_SHA256_DESC \ + _CFG_CORE_LTC_SHA224 \ + _CFG_CORE_LTC_SHA256) +_CFG_CORE_LTC_SHA384_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_SHA384_DESC \ + _CFG_CORE_LTC_SHA384) +_CFG_CORE_LTC_SHA512_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_SHA512_DESC \ + _CFG_CORE_LTC_SHA512_256 \ + _CFG_CORE_LTC_SHA512) +_CFG_CORE_LTC_AES_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_AES_DESC \ + _CFG_CORE_LTC_AES) + +_CFG_CORE_LTC_SHA3_DESC := $(call cfg-one-enabled, _CFG_CORE_LTC_SHA3_224 \ + _CFG_CORE_LTC_SHA3_256 _CFG_CORE_LTC_SHA3_384 \ + _CFG_CORE_LTC_SHA3_512 _CFG_CORE_LTC_SHAKE128 \ + _CFG_CORE_LTC_SHAKE256) + +# Assign system variables +_CFG_CORE_LTC_CE := $(CFG_CRYPTO_WITH_CE) +_CFG_CORE_LTC_VFP := $(CFG_WITH_VFP) +_CFG_CORE_LTC_BIGNUM_MAX_BITS := $(CFG_CORE_BIGNUM_MAX_BITS) +_CFG_CORE_LTC_PAGER := $(CFG_WITH_PAGER) +ifneq ($(CFG_NUM_THREADS),1) +_CFG_CORE_LTC_OPTEE_THREAD := y +else +_CFG_CORE_LTC_OPTEE_THREAD := n +endif +_CFG_CORE_LTC_HWSUPP_PMULL := $(CFG_HWSUPP_PMULL) + +# Assign aggregated variables +ltc-one-enabled = $(call cfg-one-enabled,$(foreach v,$(1),_CFG_CORE_LTC_$(v))) +_CFG_CORE_LTC_ACIPHER := $(call ltc-one-enabled, RSA DSA DH ECC) +_CFG_CORE_LTC_AUTHENC := $(and $(filter y,$(_CFG_CORE_LTC_AES_DESC)), \ + $(filter y,$(call ltc-one-enabled, CCM GCM))) +_CFG_CORE_LTC_CIPHER := $(call ltc-one-enabled, AES_DESC DES) +_CFG_CORE_LTC_HASH := $(call ltc-one-enabled, MD5 SHA1 SHA224 SHA256 SHA384 \ + SHA512 SHA3_224 SHA3_256 \ + SHA3_384 SHA3_512) +ifeq ($(CFG_CRYPTO_HMAC),y) +_CFG_CORE_LTC_HMAC := $(call ltc-one-enabled, MD5 SHA1 SHA224 SHA256 SHA384 \ + SHA512 SHA3_224 SHA3_256 \ + SHA3_384 SHA3_512) +endif + +_CFG_CORE_LTC_MAC := $(call ltc-one-enabled, HMAC CMAC CBC_MAC) +_CFG_CORE_LTC_CBC := $(call ltc-one-enabled, CBC CBC_MAC) +_CFG_CORE_LTC_ASN1 := $(call ltc-one-enabled, RSA DSA ECC) +_CFG_CORE_LTC_EC25519 := $(call ltc-one-enabled, ED25519 X25519) + +# Enable TEE_ALG_RSASSA_PKCS1_V1_5 algorithm for signing with PKCS#1 v1.5 EMSA +# without ASN.1 around the hash. +ifeq ($(CFG_CRYPTOLIB_NAME),tomcrypt) +CFG_CRYPTO_RSASSA_NA1 ?= y +endif diff --git a/optee_os/core/crypto/aes-cts.c b/optee_os/core/crypto/aes-cts.c new file mode 100644 index 0000000..44142ab --- /dev/null +++ b/optee_os/core/crypto/aes-cts.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + + +/* From libtomcrypt doc: + * Ciphertext stealing is a method of dealing with messages + * in CBC mode which are not a multiple of the block + * length. This is accomplished by encrypting the last + * ciphertext block in ECB mode, and XOR'ing the output + * against the last partial block of plaintext. LibTomCrypt + * does not support this mode directly but it is fairly + * easy to emulate with a call to the cipher's + * ecb encrypt() callback function. + * The more sane way to deal with partial blocks is to pad + * them with zeroes, and then use CBC normally + */ + +/* + * From Global Platform: CTS = CBC-CS3 + */ + +struct cts_ctx { + struct crypto_cipher_ctx ctx; + struct crypto_cipher_ctx *ecb; + struct crypto_cipher_ctx *cbc; + TEE_OperationMode mode; +}; + +static const struct crypto_cipher_ops cts_ops; + +static struct cts_ctx *to_cts_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &cts_ops); + + return container_of(ctx, struct cts_ctx, ctx); +} + +static TEE_Result cts_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2, + size_t key2_len, const uint8_t *iv, size_t iv_len) +{ + TEE_Result res = TEE_SUCCESS; + struct cts_ctx *c = to_cts_ctx(ctx); + + c->mode = mode; + + res = crypto_cipher_init(c->ecb, mode, key1, key1_len, key2, key2_len, + iv, iv_len); + if (res) + return res; + + return crypto_cipher_init(c->cbc, mode, key1, key1_len, key2, key2_len, + iv, iv_len); +} + +/* + * From http://en.wikipedia.org/wiki/Ciphertext_stealing + * CBC ciphertext stealing encryption using a standard + * CBC interface: + * 1. Pad the last partial plaintext block with 0. + * 2. Encrypt the whole padded plaintext using the + * standard CBC mode. + * 3. Swap the last two ciphertext blocks. + * 4. Truncate the ciphertext to the length of the + * original plaintext. + * + * CBC ciphertext stealing decryption using a standard + * CBC interface + * 1. Dn = Decrypt (K, Cn-1). Decrypt the second to last + * ciphertext block. + * 2. Cn = Cn || Tail (Dn, B-M). Pad the ciphertext to the + * nearest multiple of the block size using the last + * B-M bits of block cipher decryption of the + * second-to-last ciphertext block. + * 3. Swap the last two ciphertext blocks. + * 4. Decrypt the (modified) ciphertext using the standard + * CBC mode. + * 5. Truncate the plaintext to the length of the original + * ciphertext. + */ +static TEE_Result cbc_cts_update(void *cbc_ctx, void *ecb_ctx, + TEE_OperationMode mode, bool last_block, + const uint8_t *data, size_t len, uint8_t *dst) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t tmp2_block[64] = { 0 }; + uint8_t tmp_block[64] = { 0 }; + int len_last_block = 0; + int block_size = 16; + int nb_blocks = 0; + + if (!last_block) + return tee_do_cipher_update(cbc_ctx, TEE_ALG_AES_CBC_NOPAD, + mode, last_block, data, len, dst); + + /* Compute the last block length and check constraints */ + nb_blocks = (len + block_size - 1) / block_size; + if (nb_blocks < 2) + return TEE_ERROR_BAD_STATE; + len_last_block = len % block_size; + if (len_last_block == 0) + len_last_block = block_size; + + if (mode == TEE_MODE_ENCRYPT) { + memcpy(tmp_block, + data + ((nb_blocks - 1) * block_size), + len_last_block); + memset(tmp_block + len_last_block, + 0, + block_size - len_last_block); + + res = tee_do_cipher_update(cbc_ctx, TEE_ALG_AES_CBC_NOPAD, + mode, 0, data, + (nb_blocks - 1) * block_size, dst); + if (res != TEE_SUCCESS) + return res; + + memcpy(dst + (nb_blocks - 1) * block_size, + dst + (nb_blocks - 2) * block_size, + len_last_block); + + res = tee_do_cipher_update(cbc_ctx, TEE_ALG_AES_CBC_NOPAD, + mode, 0, tmp_block, block_size, + dst + (nb_blocks - 2) * block_size); + if (res != TEE_SUCCESS) + return res; + } else { + /* 1. Decrypt the second to last ciphertext block */ + res = tee_do_cipher_update(ecb_ctx, TEE_ALG_AES_ECB_NOPAD, + mode, 0, + data + (nb_blocks - 2) * block_size, + block_size, tmp2_block); + if (res != TEE_SUCCESS) + return res; + + /* 2. Cn = Cn || Tail (Dn, B-M) */ + memcpy(tmp_block, data + ((nb_blocks - 1) * block_size), + len_last_block); + memcpy(tmp_block + len_last_block, tmp2_block + len_last_block, + block_size - len_last_block); + + /* 3. Swap the last two ciphertext blocks */ + /* done by passing the correct buffers in step 4. */ + + /* 4. Decrypt the (modified) ciphertext */ + if (nb_blocks > 2) { + res = tee_do_cipher_update(cbc_ctx, + TEE_ALG_AES_CBC_NOPAD, mode, + 0, data, + (nb_blocks - 2) * + block_size, dst); + if (res != TEE_SUCCESS) + return res; + } + + res = tee_do_cipher_update(cbc_ctx, TEE_ALG_AES_CBC_NOPAD, + mode, 0, tmp_block, block_size, + dst + + ((nb_blocks - 2) * block_size)); + if (res != TEE_SUCCESS) + return res; + + res = tee_do_cipher_update(cbc_ctx, TEE_ALG_AES_CBC_NOPAD, + mode, 0, data + + ((nb_blocks - 2) * block_size), + block_size, tmp_block); + if (res != TEE_SUCCESS) + return res; + + /* 5. Truncate the plaintext */ + memcpy(dst + (nb_blocks - 1) * block_size, tmp_block, + len_last_block); + } + return TEE_SUCCESS; +} + +static TEE_Result cts_update(struct crypto_cipher_ctx *ctx, bool last_block, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct cts_ctx *c = to_cts_ctx(ctx); + + return cbc_cts_update(c->cbc, c->ecb, c->mode, last_block, data, len, + dst); +} + +static void cts_final(struct crypto_cipher_ctx *ctx) +{ + struct cts_ctx *c = to_cts_ctx(ctx); + + crypto_cipher_final(c->cbc); + crypto_cipher_final(c->ecb); +} + +static void cts_free_ctx(struct crypto_cipher_ctx *ctx) +{ + struct cts_ctx *c = to_cts_ctx(ctx); + + crypto_cipher_free_ctx(c->cbc); + crypto_cipher_free_ctx(c->ecb); + free(c); +} + +static void cts_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct cts_ctx *src = to_cts_ctx(src_ctx); + struct cts_ctx *dst = to_cts_ctx(dst_ctx); + + crypto_cipher_copy_state(dst->cbc, src->cbc); + crypto_cipher_copy_state(dst->ecb, src->ecb); + dst->mode = src->mode; +} + +static const struct crypto_cipher_ops cts_ops = { + .init = cts_init, + .update = cts_update, + .final = cts_final, + .free_ctx = cts_free_ctx, + .copy_state = cts_copy_state, +}; + +TEE_Result crypto_aes_cts_alloc_ctx(struct crypto_cipher_ctx **ctx) +{ + TEE_Result res = TEE_SUCCESS; + struct cts_ctx *c = calloc(1, sizeof(*c)); + + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + res = crypto_aes_ecb_alloc_ctx(&c->ecb); + if (res) + goto err; + res = crypto_aes_cbc_alloc_ctx(&c->cbc); + if (res) + goto err; + + c->ctx.ops = &cts_ops; + *ctx = &c->ctx; + + return TEE_SUCCESS; +err: + crypto_cipher_free_ctx(c->ecb); + free(c); + + return res; +} diff --git a/optee_os/core/crypto/aes-gcm-ghash-tbl.c b/optee_os/core/crypto/aes-gcm-ghash-tbl.c new file mode 100644 index 0000000..000ab00 --- /dev/null +++ b/optee_os/core/crypto/aes-gcm-ghash-tbl.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 +/* + * Copyright (c) 2017-2020, Linaro Limited + * + * NIST SP800-38D compliant GCM implementation + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* + * http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf + * + * See also: + * [MGV] http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/ +gcm-revised-spec.pdf + * + * We use the algorithm described as Shoup's method with 4-bit tables in + * [MGV] 4.1, pp. 12-13, to enhance speed without using too much memory. + */ + +/* + * Precompute small multiples of H, that is set + * HH[i] || HL[i] = H times i, + * where i is seen as a field element as in [MGV], ie high-order bits + * correspond to low powers of P. The result is stored in the same way, that + * is the high-order bit of HH corresponds to P^0 and the low-order bit of HL + * corresponds to P^127. + */ +void internal_aes_gcm_ghash_gen_tbl(struct internal_ghash_key *ghash_key, + const struct internal_aes_gcm_key *ek) +{ + int i, j; + uint64_t vl, vh; + unsigned char h[16]; + + memset(h, 0, 16); + crypto_aes_enc_block(ek->data, sizeof(ek->data), ek->rounds, h, h); + + vh = get_be64(h); + vl = get_be64(h + 8); + + /* 8 = 1000 corresponds to 1 in GF(2^128) */ + ghash_key->HL[8] = vl; + ghash_key->HH[8] = vh; + + /* 0 corresponds to 0 in GF(2^128) */ + ghash_key->HH[0] = 0; + ghash_key->HL[0] = 0; + + for (i = 4; i > 0; i >>= 1) { + uint32_t T = (vl & 1) * 0xe1000000U; + + vl = (vh << 63) | (vl >> 1); + vh = (vh >> 1) ^ ((uint64_t)T << 32); + + ghash_key->HL[i] = vl; + ghash_key->HH[i] = vh; + } + + for (i = 2; i <= 8; i *= 2) { + uint64_t *HiL = ghash_key->HL + i; + uint64_t *HiH = ghash_key->HH + i; + + vh = *HiH; + vl = *HiL; + for (j = 1; j < i; j++) { + HiH[j] = vh ^ ghash_key->HH[j]; + HiL[j] = vl ^ ghash_key->HL[j]; + } + } +} + +/* + * Shoup's method for multiplication use this table with + * last4[x] = x times P^128 + * where x and last4[x] are seen as elements of GF(2^128) as in [MGV] + */ +static const uint64_t last4[16] = { + 0x0000, 0x1c20, 0x3840, 0x2460, + 0x7080, 0x6ca0, 0x48c0, 0x54e0, + 0xe100, 0xfd20, 0xd940, 0xc560, + 0x9180, 0x8da0, 0xa9c0, 0xb5e0 +}; + +/* + * Sets output to x times H using the precomputed tables. + * x and output are seen as elements of GF(2^128) as in [MGV]. + */ +void internal_aes_gcm_ghash_mult_tbl(struct internal_ghash_key *ghash_key, + const unsigned char x[16], + unsigned char output[16]) +{ + int i = 0; + unsigned char lo = 0, hi = 0, rem = 0; + uint64_t zh = 0, zl = 0; + + lo = x[15] & 0xf; + + zh = ghash_key->HH[lo]; + zl = ghash_key->HL[lo]; + + for (i = 15; i >= 0; i--) { + lo = x[i] & 0xf; + hi = x[i] >> 4; + + if (i != 15) { + rem = (unsigned char)zl & 0xf; + zl = (zh << 60) | (zl >> 4); + zh = (zh >> 4); + zh ^= (uint64_t)last4[rem] << 48; + zh ^= ghash_key->HH[lo]; + zl ^= ghash_key->HL[lo]; + } + + rem = (unsigned char)zl & 0xf; + zl = (zh << 60) | (zl >> 4); + zh = (zh >> 4); + zh ^= (uint64_t)last4[rem] << 48; + zh ^= ghash_key->HH[hi]; + zl ^= ghash_key->HL[hi]; + } + + put_be64(output, zh); + put_be64(output + 8, zl); +} diff --git a/optee_os/core/crypto/aes-gcm-sw.c b/optee_os/core/crypto/aes-gcm-sw.c new file mode 100644 index 0000000..9b40400 --- /dev/null +++ b/optee_os/core/crypto/aes-gcm-sw.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +void internal_aes_gcm_set_key(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek) +{ +#ifdef CFG_AES_GCM_TABLE_BASED + internal_aes_gcm_ghash_gen_tbl(&state->ghash_key, ek); +#else + crypto_aes_enc_block(ek->data, sizeof(ek->data), ek->rounds, + state->ctr, state->ghash_key.hash_subkey); +#endif +} + +static void ghash_update_block(struct internal_aes_gcm_state *state, + const void *data) +{ + void *y = state->hash_state; + + internal_aes_gcm_xor_block(y, data); +#ifdef CFG_AES_GCM_TABLE_BASED + internal_aes_gcm_ghash_mult_tbl(&state->ghash_key, y, y); +#else + internal_aes_gcm_gfmul(state->ghash_key.hash_subkey, y, y); +#endif +} + +void internal_aes_gcm_ghash_update(struct internal_aes_gcm_state *state, + const void *head, const void *data, + size_t num_blocks) +{ + size_t n = 0; + + if (head) + ghash_update_block(state, head); + + if (data) + for (n = 0; n < num_blocks; n++) + ghash_update_block(state, + (const uint8_t *)data + + n * TEE_AES_BLOCK_SIZE); +} + +static void encrypt_block(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *enc_key, + const uint64_t src[2], uint64_t dst[2]) +{ + void *buf_cryp = state->buf_cryp; + + internal_aes_gcm_xor_block(buf_cryp, src); + internal_aes_gcm_ghash_update(state, buf_cryp, NULL, 0); + memcpy(dst, buf_cryp, sizeof(state->buf_cryp)); + + crypto_aes_enc_block(enc_key->data, sizeof(enc_key->data), + enc_key->rounds, state->ctr, state->buf_cryp); + internal_aes_gcm_inc_ctr(state); +} + +static void encrypt_pl(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + const uint8_t *src, size_t num_blocks, uint8_t *dst) +{ + size_t n = 0; + + if (IS_ALIGNED_WITH_TYPE(src, uint64_t)) { + for (n = 0; n < num_blocks; n++) { + const void *s = src + n * TEE_AES_BLOCK_SIZE; + void *d = dst + n * TEE_AES_BLOCK_SIZE; + + encrypt_block(state, ek, s, d); + } + } else { + for (n = 0; n < num_blocks; n++) { + uint64_t tmp[2] = { 0 }; + void *d = dst + n * TEE_AES_BLOCK_SIZE; + + memcpy(tmp, src + n * TEE_AES_BLOCK_SIZE, sizeof(tmp)); + encrypt_block(state, ek, tmp, d); + } + } +} + +static void decrypt_block(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *enc_key, + const uint64_t src[2], uint64_t dst[2]) +{ + void *buf_cryp = state->buf_cryp; + + crypto_aes_enc_block(enc_key->data, sizeof(enc_key->data), + enc_key->rounds, state->ctr, buf_cryp); + internal_aes_gcm_inc_ctr(state); + + internal_aes_gcm_xor_block(buf_cryp, src); + internal_aes_gcm_ghash_update(state, src, NULL, 0); + memcpy(dst, buf_cryp, sizeof(state->buf_cryp)); +} + +static void decrypt_pl(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + const uint8_t *src, size_t num_blocks, uint8_t *dst) +{ + size_t n = 0; + + if (IS_ALIGNED_WITH_TYPE(src, uint64_t)) { + for (n = 0; n < num_blocks; n++) { + const void *s = src + n * TEE_AES_BLOCK_SIZE; + void *d = dst + n * TEE_AES_BLOCK_SIZE; + + decrypt_block(state, ek, s, d); + } + } else { + for (n = 0; n < num_blocks; n++) { + uint64_t tmp[2] = { 0 }; + void *d = dst + n * TEE_AES_BLOCK_SIZE; + + memcpy(tmp, src + n * TEE_AES_BLOCK_SIZE, sizeof(tmp)); + decrypt_block(state, ek, tmp, d); + } + } +} + +void +internal_aes_gcm_update_payload_blocks(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + TEE_OperationMode m, const void *src, + size_t num_blocks, void *dst) +{ + assert(!state->buf_pos && num_blocks); + + if (m == TEE_MODE_ENCRYPT) + encrypt_pl(state, ek, src, num_blocks, dst); + else + decrypt_pl(state, ek, src, num_blocks, dst); +} diff --git a/optee_os/core/crypto/aes-gcm.c b/optee_os/core/crypto/aes-gcm.c new file mode 100644 index 0000000..423b6f3 --- /dev/null +++ b/optee_os/core/crypto/aes-gcm.c @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void xor_buf(uint8_t *dst, const uint8_t *src, size_t len) +{ + size_t n; + + for (n = 0; n < len; n++) + dst[n] ^= src[n]; +} + + +static void ghash_update_pad_zero(struct internal_aes_gcm_state *state, + const uint8_t *data, size_t len) +{ + size_t n = len / TEE_AES_BLOCK_SIZE; + uint64_t block[2]; + + if (n) { + if (internal_aes_gcm_ptr_is_block_aligned(data)) { + internal_aes_gcm_ghash_update(state, NULL, data, n); + } else { + size_t m; + + for (m = 0; m < n; m++) { + + memcpy(block, data + m * sizeof(block), + sizeof(block)); + internal_aes_gcm_ghash_update(state, NULL, + (void *)block, 1); + } + } + } + + if (len - n * TEE_AES_BLOCK_SIZE) { + memset(block, 0, sizeof(block)); + memcpy(block, data + n * TEE_AES_BLOCK_SIZE, + len - n * TEE_AES_BLOCK_SIZE); + internal_aes_gcm_ghash_update(state, block, NULL, 0); + } +} + +static void ghash_update_lengths(struct internal_aes_gcm_state *state, + uint32_t l1, uint32_t l2) +{ + uint64_t len_fields[2] = { + TEE_U64_TO_BIG_ENDIAN(l1 * 8), + TEE_U64_TO_BIG_ENDIAN(l2 * 8) + }; + + COMPILE_TIME_ASSERT(sizeof(len_fields) == TEE_AES_BLOCK_SIZE); + internal_aes_gcm_ghash_update(state, (uint8_t *)len_fields, NULL, 0); +} + +static TEE_Result __gcm_init(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + TEE_OperationMode mode, const void *nonce, + size_t nonce_len, size_t tag_len) +{ + COMPILE_TIME_ASSERT(sizeof(state->ctr) == TEE_AES_BLOCK_SIZE); + + if (tag_len > sizeof(state->buf_tag)) + return TEE_ERROR_BAD_PARAMETERS; + + memset(state, 0, sizeof(*state)); + + state->tag_len = tag_len; + internal_aes_gcm_set_key(state, ek); + + if (nonce_len == (96 / 8)) { + memcpy(state->ctr, nonce, nonce_len); + internal_aes_gcm_inc_ctr(state); + } else { + ghash_update_pad_zero(state, nonce, nonce_len); + ghash_update_lengths(state, 0, nonce_len); + + memcpy(state->ctr, state->hash_state, sizeof(state->ctr)); + memset(state->hash_state, 0, sizeof(state->hash_state)); + } + + crypto_aes_enc_block(ek->data, sizeof(ek->data), ek->rounds, + state->ctr, state->buf_tag); + internal_aes_gcm_inc_ctr(state); + if (mode == TEE_MODE_ENCRYPT) { + /* + * Encryption uses the pre-encrypted xor-buffer to encrypt + * while decryption encrypts the xor-buffer when needed + * instead. + * + * The reason for this is that the combined encryption and + * ghash implementation does both operations intertwined. + * In the decrypt case the xor-buffer is needed at the end + * of processing each block, while the encryption case + * needs xor-buffer before processing each block. + * + * In a pure software implementation we wouldn't have any + * use for this kind of optimization, but since this + * AES-GCM implementation is aimed at being combined with + * accelerated routines it's more convenient to always have + * this optimization activated. + */ + crypto_aes_enc_block(ek->data, sizeof(ek->data), ek->rounds, + state->ctr, state->buf_cryp); + internal_aes_gcm_inc_ctr(state); + } + + return TEE_SUCCESS; +} + +TEE_Result internal_aes_gcm_init(struct internal_aes_gcm_ctx *ctx, + TEE_OperationMode mode, const void *key, + size_t key_len, const void *nonce, + size_t nonce_len, size_t tag_len) +{ + TEE_Result res = TEE_SUCCESS; + struct internal_aes_gcm_key *ek = &ctx->key; + + res = crypto_aes_expand_enc_key(key, key_len, ek->data, + sizeof(ek->data), &ek->rounds); + if (res) + return res; + + return __gcm_init(&ctx->state, ek, mode, nonce, nonce_len, tag_len); +} + +static TEE_Result __gcm_update_aad(struct internal_aes_gcm_state *state, + const void *data, size_t len) +{ + const uint8_t *d = data; + size_t l = len; + const uint8_t *head = NULL; + size_t n; + + if (state->payload_bytes) + return TEE_ERROR_BAD_PARAMETERS; + + state->aad_bytes += len; + + while (l) { + if (state->buf_pos || + !internal_aes_gcm_ptr_is_block_aligned(d) || + l < TEE_AES_BLOCK_SIZE) { + n = MIN(TEE_AES_BLOCK_SIZE - state->buf_pos, l); + memcpy(state->buf_hash + state->buf_pos, d, n); + state->buf_pos += n; + + if (state->buf_pos != TEE_AES_BLOCK_SIZE) + return TEE_SUCCESS; + + state->buf_pos = 0; + head = state->buf_hash; + d += n; + l -= n; + } + + if (internal_aes_gcm_ptr_is_block_aligned(d)) + n = l / TEE_AES_BLOCK_SIZE; + else + n = 0; + + internal_aes_gcm_ghash_update(state, head, d, n); + l -= n * TEE_AES_BLOCK_SIZE; + d += n * TEE_AES_BLOCK_SIZE; + } + + return TEE_SUCCESS; +} + +TEE_Result internal_aes_gcm_update_aad(struct internal_aes_gcm_ctx *ctx, + const void *data, size_t len) +{ + return __gcm_update_aad(&ctx->state, data, len); +} + +static TEE_Result +__gcm_update_payload(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + TEE_OperationMode mode, const void *src, + size_t len, void *dst) +{ + size_t n; + const uint8_t *s = src; + uint8_t *d = dst; + size_t l = len; + + if (!state->payload_bytes && state->buf_pos) { + /* AAD part done, finish up the last bits. */ + memset(state->buf_hash + state->buf_pos, 0, + TEE_AES_BLOCK_SIZE - state->buf_pos); + internal_aes_gcm_ghash_update(state, state->buf_hash, NULL, 0); + state->buf_pos = 0; + } + + state->payload_bytes += len; + + while (l) { + if (state->buf_pos || l < TEE_AES_BLOCK_SIZE) { + n = MIN(TEE_AES_BLOCK_SIZE - state->buf_pos, l); + + if (!state->buf_pos && mode == TEE_MODE_DECRYPT) + crypto_aes_enc_block(ek->data, sizeof(ek->data), + ek->rounds, state->ctr, + state->buf_cryp); + + xor_buf(state->buf_cryp + state->buf_pos, s, n); + memcpy(d, state->buf_cryp + state->buf_pos, n); + if (mode == TEE_MODE_ENCRYPT) + memcpy(state->buf_hash + state->buf_pos, + state->buf_cryp + state->buf_pos, n); + else + memcpy(state->buf_hash + state->buf_pos, s, n); + + state->buf_pos += n; + + if (state->buf_pos != TEE_AES_BLOCK_SIZE) + return TEE_SUCCESS; + + internal_aes_gcm_ghash_update(state, state->buf_hash, + NULL, 0); + state->buf_pos = 0; + d += n; + s += n; + l -= n; + + if (mode == TEE_MODE_ENCRYPT) + crypto_aes_enc_block(ek->data, sizeof(ek->data), + ek->rounds, state->ctr, + state->buf_cryp); + internal_aes_gcm_inc_ctr(state); + } else { + n = l / TEE_AES_BLOCK_SIZE; + internal_aes_gcm_update_payload_blocks(state, ek, mode, + s, n, d); + s += n * TEE_AES_BLOCK_SIZE; + d += n * TEE_AES_BLOCK_SIZE; + l -= n * TEE_AES_BLOCK_SIZE; + } + } + + return TEE_SUCCESS; +} + +TEE_Result internal_aes_gcm_update_payload(struct internal_aes_gcm_ctx *ctx, + TEE_OperationMode mode, + const void *src, size_t len, + void *dst) +{ + return __gcm_update_payload(&ctx->state, &ctx->key, mode, src, len, + dst); +} + +static TEE_Result operation_final(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *enc_key, + TEE_OperationMode m, const uint8_t *src, + size_t len, uint8_t *dst) +{ + TEE_Result res; + + res = __gcm_update_payload(state, enc_key, m, src, len, dst); + if (res) + return res; + + if (state->buf_pos) { + memset(state->buf_hash + state->buf_pos, 0, + sizeof(state->buf_hash) - state->buf_pos); + internal_aes_gcm_ghash_update(state, state->buf_hash, NULL, 0); + } + + ghash_update_lengths(state, state->aad_bytes, state->payload_bytes); + /* buf_tag was filled in with the first counter block aes_gcm_init() */ + xor_buf(state->buf_tag, state->hash_state, state->tag_len); + + return TEE_SUCCESS; +} + +static TEE_Result __gcm_enc_final(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *enc_key, + const void *src, size_t len, void *dst, + void *tag, size_t *tag_len) +{ + TEE_Result res; + + if (*tag_len < state->tag_len) + return TEE_ERROR_SHORT_BUFFER; + + res = operation_final(state, enc_key, TEE_MODE_ENCRYPT, src, len, dst); + if (res) + return res; + + memcpy(tag, state->buf_tag, state->tag_len); + *tag_len = state->tag_len; + + return TEE_SUCCESS; +} + +TEE_Result internal_aes_gcm_enc_final(struct internal_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + void *tag, size_t *tag_len) +{ + return __gcm_enc_final(&ctx->state, &ctx->key, src, len, dst, tag, + tag_len); +} + +static TEE_Result __gcm_dec_final(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *enc_key, + const void *src, size_t len, void *dst, + const void *tag, size_t tag_len) +{ + TEE_Result res; + + if (tag_len != state->tag_len) + return TEE_ERROR_MAC_INVALID; + + res = operation_final(state, enc_key, TEE_MODE_DECRYPT, src, len, dst); + if (res) + return res; + + if (consttime_memcmp(state->buf_tag, tag, tag_len)) + return TEE_ERROR_MAC_INVALID; + + return TEE_SUCCESS; +} + +TEE_Result internal_aes_gcm_dec_final(struct internal_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + const void *tag, size_t tag_len) +{ + return __gcm_dec_final(&ctx->state, &ctx->key, src, len, dst, tag, + tag_len); +} + +void internal_aes_gcm_inc_ctr(struct internal_aes_gcm_state *state) +{ + uint64_t c = 0; + + c = TEE_U64_FROM_BIG_ENDIAN(state->ctr[1]) + 1; + state->ctr[1] = TEE_U64_TO_BIG_ENDIAN(c); + if (!c) { + c = TEE_U64_FROM_BIG_ENDIAN(state->ctr[0]) + 1; + state->ctr[0] = TEE_U64_TO_BIG_ENDIAN(c); + } +} + +void internal_aes_gcm_dec_ctr(struct internal_aes_gcm_state *state) +{ + uint64_t c = 0; + + c = TEE_U64_FROM_BIG_ENDIAN(state->ctr[1]) - 1; + state->ctr[1] = TEE_U64_TO_BIG_ENDIAN(c); + if (c == UINT64_MAX) { + c = TEE_U64_FROM_BIG_ENDIAN(state->ctr[0]) - 1; + state->ctr[0] = TEE_U64_TO_BIG_ENDIAN(c); + } +} + +TEE_Result internal_aes_gcm_enc(const struct internal_aes_gcm_key *enc_key, + const void *nonce, size_t nonce_len, + const void *aad, size_t aad_len, + const void *src, size_t len, void *dst, + void *tag, size_t *tag_len) +{ + TEE_Result res; + struct internal_aes_gcm_state state; + + res = __gcm_init(&state, enc_key, TEE_MODE_ENCRYPT, nonce, nonce_len, + *tag_len); + if (res) + return res; + + if (aad) { + res = __gcm_update_aad(&state, aad, aad_len); + if (res) + return res; + } + + return __gcm_enc_final(&state, enc_key, src, len, dst, tag, tag_len); +} + +TEE_Result internal_aes_gcm_dec(const struct internal_aes_gcm_key *enc_key, + const void *nonce, size_t nonce_len, + const void *aad, size_t aad_len, + const void *src, size_t len, void *dst, + const void *tag, size_t tag_len) +{ + TEE_Result res; + struct internal_aes_gcm_state state; + + res = __gcm_init(&state, enc_key, TEE_MODE_DECRYPT, nonce, nonce_len, + tag_len); + if (res) + return res; + + if (aad) { + res = __gcm_update_aad(&state, aad, aad_len); + if (res) + return res; + } + + return __gcm_dec_final(&state, enc_key, src, len, dst, tag, tag_len); +} + + +#ifndef CFG_CRYPTO_AES_GCM_FROM_CRYPTOLIB +#include +#include + +struct aes_gcm_ctx { + struct crypto_authenc_ctx aec; + struct internal_aes_gcm_ctx ctx; +}; + +static const struct crypto_authenc_ops aes_gcm_ops; + +static struct aes_gcm_ctx * +to_aes_gcm_ctx(struct crypto_authenc_ctx *aec) +{ + assert(aec->ops == &aes_gcm_ops); + + return container_of(aec, struct aes_gcm_ctx, aec); +} + +TEE_Result crypto_aes_gcm_alloc_ctx(struct crypto_authenc_ctx **ctx_ret) +{ + struct aes_gcm_ctx *ctx = calloc(1, sizeof(*ctx)); + + if (!ctx) + return TEE_ERROR_OUT_OF_MEMORY; + ctx->aec.ops = &aes_gcm_ops; + + *ctx_ret = &ctx->aec; + + return TEE_SUCCESS; +} + +static void aes_gcm_free_ctx(struct crypto_authenc_ctx *aec) +{ + free(to_aes_gcm_ctx(aec)); +} + +static void aes_gcm_copy_state(struct crypto_authenc_ctx *dst_ctx, + struct crypto_authenc_ctx *src_ctx) +{ + to_aes_gcm_ctx(dst_ctx)->ctx = to_aes_gcm_ctx(src_ctx)->ctx; +} + +static TEE_Result aes_gcm_init(struct crypto_authenc_ctx *aec, + TEE_OperationMode mode, + const uint8_t *key, size_t key_len, + const uint8_t *nonce, size_t nonce_len, + size_t tag_len, size_t aad_len __unused, + size_t payload_len __unused) +{ + return internal_aes_gcm_init(&to_aes_gcm_ctx(aec)->ctx, mode, key, + key_len, nonce, nonce_len, tag_len); +} + +static TEE_Result aes_gcm_update_aad(struct crypto_authenc_ctx *aec, + const uint8_t *data, size_t len) +{ + return internal_aes_gcm_update_aad(&to_aes_gcm_ctx(aec)->ctx, data, + len); +} + +static TEE_Result aes_gcm_update_payload(struct crypto_authenc_ctx *aec, + TEE_OperationMode m, + const uint8_t *src, size_t len, + uint8_t *dst) +{ + return internal_aes_gcm_update_payload(&to_aes_gcm_ctx(aec)->ctx, + m, src, len, dst); +} + +static TEE_Result aes_gcm_enc_final(struct crypto_authenc_ctx *aec, + const uint8_t *src, size_t len, + uint8_t *dst, uint8_t *tag, size_t *tag_len) +{ + return internal_aes_gcm_enc_final(&to_aes_gcm_ctx(aec)->ctx, src, len, + dst, tag, tag_len); +} + +static TEE_Result aes_gcm_dec_final(struct crypto_authenc_ctx *aec, + const uint8_t *src, size_t len, + uint8_t *dst, const uint8_t *tag, + size_t tag_len) +{ + return internal_aes_gcm_dec_final(&to_aes_gcm_ctx(aec)->ctx, src, len, + dst, tag, tag_len); +} + +static void aes_gcm_final(struct crypto_authenc_ctx *aec __unused) +{ +} + +static const struct crypto_authenc_ops aes_gcm_ops = { + .init = aes_gcm_init, + .update_aad = aes_gcm_update_aad, + .update_payload = aes_gcm_update_payload, + .enc_final = aes_gcm_enc_final, + .dec_final = aes_gcm_dec_final, + .final = aes_gcm_final, + .free_ctx = aes_gcm_free_ctx, + .copy_state = aes_gcm_copy_state, +}; + +/* + * internal_aes_gcm_gfmul() is based on ghash_gfmul() from + * https://github.com/openbsd/src/blob/master/sys/crypto/gmac.c + */ +void internal_aes_gcm_gfmul(const uint64_t X[2], const uint64_t Y[2], + uint64_t product[2]) +{ + uint64_t y[2] = { 0 }; + uint64_t z[2] = { 0 }; + const uint8_t *x = (const uint8_t *)X; + uint32_t mul = 0; + size_t n = 0; + + y[0] = TEE_U64_FROM_BIG_ENDIAN(Y[0]); + y[1] = TEE_U64_FROM_BIG_ENDIAN(Y[1]); + + for (n = 0; n < TEE_AES_BLOCK_SIZE * 8; n++) { + /* update Z */ + if (x[n >> 3] & (1 << (~n & 7))) + internal_aes_gcm_xor_block(z, y); + + /* update Y */ + mul = y[1] & 1; + y[1] = (y[0] << 63) | (y[1] >> 1); + y[0] = (y[0] >> 1) ^ (0xe100000000000000 * mul); + } + + product[0] = TEE_U64_TO_BIG_ENDIAN(z[0]); + product[1] = TEE_U64_TO_BIG_ENDIAN(z[1]); +} +#endif /*!CFG_CRYPTO_AES_GCM_FROM_CRYPTOLIB*/ diff --git a/optee_os/core/crypto/cbc-mac.c b/optee_os/core/crypto/cbc-mac.c new file mode 100644 index 0000000..c38b058 --- /dev/null +++ b/optee_os/core/crypto/cbc-mac.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +/* + * This is implemented here as being the plain text which is encoded with IV=0. + * Result of the CBC-MAC is the last 16-bytes cipher. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CBCMAC_MAX_BLOCK_LEN 16 + +struct crypto_cbc_mac_ctx { + struct crypto_mac_ctx ctx; + void *cbc_ctx; + uint32_t cbc_algo; + uint8_t block[CBCMAC_MAX_BLOCK_LEN]; + uint8_t digest[CBCMAC_MAX_BLOCK_LEN]; + unsigned char current_block_len; + unsigned char block_len; + bool is_computed; + bool pkcs5_pad; +}; + +static const struct crypto_mac_ops crypto_cbc_mac_ops; + +static struct crypto_cbc_mac_ctx *to_cbc_mac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx && ctx->ops == &crypto_cbc_mac_ops); + + return container_of(ctx, struct crypto_cbc_mac_ctx, ctx); +} + +static TEE_Result crypto_cbc_mac_init(struct crypto_mac_ctx *ctx, + const uint8_t *key, size_t len) +{ + struct crypto_cbc_mac_ctx *mc = to_cbc_mac_ctx(ctx); + + memset(mc->block, 0, sizeof(mc->block)); + memset(mc->digest, 0, sizeof(mc->digest)); + mc->current_block_len = 0; + mc->is_computed = false; + + /* IV should be zero and mc->block happens to be zero at this stage */ + return crypto_cipher_init(mc->cbc_ctx, TEE_MODE_ENCRYPT, key, len, + NULL, 0, mc->block, mc->block_len); +} + +static TEE_Result crypto_cbc_mac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + size_t nblocks = 0; + size_t out_len = 0; + uint8_t *out_tmp = NULL; + uint8_t *out = NULL; + TEE_Result res = TEE_SUCCESS; + struct crypto_cbc_mac_ctx *mc = to_cbc_mac_ctx(ctx); + + if ((mc->current_block_len > 0) && + (len + mc->current_block_len >= mc->block_len)) { + size_t pad_len = mc->block_len - mc->current_block_len; + + memcpy(mc->block + mc->current_block_len, data, pad_len); + data += pad_len; + len -= pad_len; + res = crypto_cipher_update(mc->cbc_ctx, TEE_MODE_ENCRYPT, + false, mc->block, mc->block_len, + mc->digest); + if (res) + return res; + mc->is_computed = 1; + mc->current_block_len = 0; + } + + nblocks = MIN(len / mc->block_len, + (size_t)CFG_CRYPTO_CBC_MAC_BUNDLE_BLOCKS); + if (nblocks > 1) + out_tmp = malloc(nblocks * mc->block_len); + + while (len >= mc->block_len) { + nblocks = MIN(len / mc->block_len, + (size_t)CFG_CRYPTO_CBC_MAC_BUNDLE_BLOCKS); + + if (nblocks > 1 && out_tmp) { + out_len = nblocks * mc->block_len; + out = out_tmp; + } else { + out_len = mc->block_len; + out = mc->digest; + nblocks = 1; + } + + res = crypto_cipher_update(mc->cbc_ctx, TEE_MODE_ENCRYPT, + false, data, out_len, out); + if (res) + goto out; + mc->is_computed = 1; + data += out_len; + len -= out_len; + if (nblocks > 1 && len < mc->block_len) { + assert(out_tmp); + /* Copy last block of output */ + memcpy(mc->digest, out_tmp + out_len - mc->block_len, + mc->block_len); + } + } + + if (len > 0) { + assert(mc->current_block_len + len < mc->block_len); + memcpy(mc->block + mc->current_block_len, data, len); + mc->current_block_len += len; + } + +out: + free(out_tmp); + return res; +} + +static TEE_Result crypto_cbc_mac_final(struct crypto_mac_ctx *ctx, + uint8_t *digest, size_t digest_len) +{ + struct crypto_cbc_mac_ctx *mc = to_cbc_mac_ctx(ctx); + + if (mc->pkcs5_pad) { + /* + * Padding is in whole bytes. The value of each added + * byte is the number of bytes that are added, i.e. N + * bytes, each of value N are added + */ + size_t pad_len = mc->block_len - mc->current_block_len; + + memset(mc->block + mc->current_block_len, pad_len, pad_len); + mc->current_block_len = 0; + if (crypto_cbc_mac_update(ctx, mc->block, mc->block_len)) + return TEE_ERROR_BAD_STATE; + } + + if (!mc->is_computed || mc->current_block_len) + return TEE_ERROR_BAD_STATE; + + memcpy(digest, mc->digest, MIN(digest_len, mc->block_len)); + crypto_cipher_final(mc->cbc_ctx); + + return TEE_SUCCESS; +} + +static void crypto_cbc_mac_free_ctx(struct crypto_mac_ctx *ctx) +{ + struct crypto_cbc_mac_ctx *mc = to_cbc_mac_ctx(ctx); + + crypto_cipher_free_ctx(mc->cbc_ctx); + free(mc); +} + +static void crypto_cbc_mac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct crypto_cbc_mac_ctx *dst = to_cbc_mac_ctx(dst_ctx); + struct crypto_cbc_mac_ctx *src = to_cbc_mac_ctx(src_ctx); + + assert(dst->block_len == src->block_len); + assert(dst->pkcs5_pad == src->pkcs5_pad); + assert(dst->cbc_algo == src->cbc_algo); + + crypto_cipher_copy_state(dst->cbc_ctx, src->cbc_ctx); + memcpy(dst->block, src->block, sizeof(dst->block)); + memcpy(dst->digest, src->digest, sizeof(dst->digest)); + dst->current_block_len = src->current_block_len; + dst->is_computed = src->is_computed; +} + +static const struct crypto_mac_ops crypto_cbc_mac_ops = { + .init = crypto_cbc_mac_init, + .update = crypto_cbc_mac_update, + .final = crypto_cbc_mac_final, + .free_ctx = crypto_cbc_mac_free_ctx, + .copy_state = crypto_cbc_mac_copy_state, +}; + +static TEE_Result crypto_cbc_mac_alloc_ctx(struct crypto_mac_ctx **ctx_ret, + uint32_t cbc_algo, bool pkcs5_pad) +{ + TEE_Result res; + void *cbc_ctx = NULL; + struct crypto_cbc_mac_ctx *ctx = NULL; + size_t block_size = 0; + + res = crypto_cipher_get_block_size(cbc_algo, &block_size); + if (res) + return res; + + res = crypto_cipher_alloc_ctx(&cbc_ctx, cbc_algo); + if (res) + return res; + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) { + crypto_cipher_free_ctx(cbc_ctx); + return TEE_ERROR_OUT_OF_MEMORY; + } + + ctx->cbc_ctx = cbc_ctx; + ctx->cbc_algo = cbc_algo; + ctx->pkcs5_pad = pkcs5_pad; + ctx->block_len = block_size; + ctx->ctx.ops = &crypto_cbc_mac_ops; + *ctx_ret = &ctx->ctx; + + return TEE_SUCCESS; +} + +TEE_Result crypto_aes_cbc_mac_nopad_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return crypto_cbc_mac_alloc_ctx(ctx, TEE_ALG_AES_CBC_NOPAD, false); +} + +TEE_Result crypto_aes_cbc_mac_pkcs5_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return crypto_cbc_mac_alloc_ctx(ctx, TEE_ALG_AES_CBC_NOPAD, true); +} + +TEE_Result crypto_des_cbc_mac_nopad_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return crypto_cbc_mac_alloc_ctx(ctx, TEE_ALG_DES_CBC_NOPAD, false); +} + +TEE_Result crypto_des_cbc_mac_pkcs5_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return crypto_cbc_mac_alloc_ctx(ctx, TEE_ALG_DES_CBC_NOPAD, true); +} + +TEE_Result crypto_des3_cbc_mac_nopad_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return crypto_cbc_mac_alloc_ctx(ctx, TEE_ALG_DES3_CBC_NOPAD, false); +} + +TEE_Result crypto_des3_cbc_mac_pkcs5_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return crypto_cbc_mac_alloc_ctx(ctx, TEE_ALG_DES3_CBC_NOPAD, true); +} diff --git a/optee_os/core/crypto/crypto.c b/optee_os/core/crypto/crypto.c new file mode 100644 index 0000000..5b16ad4 --- /dev/null +++ b/optee_os/core/crypto/crypto.c @@ -0,0 +1,956 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + * Copyright 2020 NXP + * Copyright 2021, SumUp Service GmbH + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +TEE_Result crypto_hash_alloc_ctx(void **ctx, uint32_t algo) +{ + TEE_Result res = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_hash_ctx *c = NULL; + + /* + * Use default cryptographic implementation if no matching + * drvcrypt device. + */ + res = drvcrypt_hash_alloc_ctx(&c, algo); + + if (res == TEE_ERROR_NOT_IMPLEMENTED) { + switch (algo) { + case TEE_ALG_MD5: + res = crypto_md5_alloc_ctx(&c); + break; + case TEE_ALG_SHA1: + res = crypto_sha1_alloc_ctx(&c); + break; + case TEE_ALG_SHA224: + res = crypto_sha224_alloc_ctx(&c); + break; + case TEE_ALG_SHA256: + res = crypto_sha256_alloc_ctx(&c); + break; + case TEE_ALG_SHA384: + res = crypto_sha384_alloc_ctx(&c); + break; + case TEE_ALG_SHA512: + res = crypto_sha512_alloc_ctx(&c); + break; + case TEE_ALG_SHA3_224: + res = crypto_sha3_224_alloc_ctx(&c); + break; + case TEE_ALG_SHA3_256: + res = crypto_sha3_256_alloc_ctx(&c); + break; + case TEE_ALG_SHA3_384: + res = crypto_sha3_384_alloc_ctx(&c); + break; + case TEE_ALG_SHA3_512: + res = crypto_sha3_512_alloc_ctx(&c); + break; + case TEE_ALG_SHAKE128: + res = crypto_shake128_alloc_ctx(&c); + break; + case TEE_ALG_SHAKE256: + res = crypto_shake256_alloc_ctx(&c); + break; + case TEE_ALG_SM3: + res = crypto_sm3_alloc_ctx(&c); + break; + default: + break; + } + } + + if (!res) + *ctx = c; + + return res; +} + +static const struct crypto_hash_ops *hash_ops(void *ctx) +{ + struct crypto_hash_ctx *c = ctx; + + assert(c && c->ops); + + return c->ops; +} + +void crypto_hash_free_ctx(void *ctx) +{ + if (ctx) + hash_ops(ctx)->free_ctx(ctx); +} + +void crypto_hash_copy_state(void *dst_ctx, void *src_ctx) +{ + hash_ops(dst_ctx)->copy_state(dst_ctx, src_ctx); +} + +TEE_Result crypto_hash_init(void *ctx) +{ + return hash_ops(ctx)->init(ctx); +} + +TEE_Result crypto_hash_update(void *ctx, const uint8_t *data, size_t len) +{ + return hash_ops(ctx)->update(ctx, data, len); +} + +TEE_Result crypto_hash_final(void *ctx, uint8_t *digest, size_t len) +{ + return hash_ops(ctx)->final(ctx, digest, len); +} + +TEE_Result crypto_cipher_alloc_ctx(void **ctx, uint32_t algo) +{ + TEE_Result res = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_cipher_ctx *c = NULL; + + /* + * Use default cryptographic implementation if no matching + * drvcrypt device. + */ + res = drvcrypt_cipher_alloc_ctx(&c, algo); + + if (res == TEE_ERROR_NOT_IMPLEMENTED) { + switch (algo) { + case TEE_ALG_AES_ECB_NOPAD: + res = crypto_aes_ecb_alloc_ctx(&c); + break; + case TEE_ALG_AES_CBC_NOPAD: + res = crypto_aes_cbc_alloc_ctx(&c); + break; + case TEE_ALG_AES_CTR: + res = crypto_aes_ctr_alloc_ctx(&c); + break; + case TEE_ALG_AES_CTS: + res = crypto_aes_cts_alloc_ctx(&c); + break; + case TEE_ALG_AES_XTS: + res = crypto_aes_xts_alloc_ctx(&c); + break; + case TEE_ALG_DES_ECB_NOPAD: + res = crypto_des_ecb_alloc_ctx(&c); + break; + case TEE_ALG_DES3_ECB_NOPAD: + res = crypto_des3_ecb_alloc_ctx(&c); + break; + case TEE_ALG_DES_CBC_NOPAD: + res = crypto_des_cbc_alloc_ctx(&c); + break; + case TEE_ALG_DES3_CBC_NOPAD: + res = crypto_des3_cbc_alloc_ctx(&c); + break; + case TEE_ALG_SM4_ECB_NOPAD: + res = crypto_sm4_ecb_alloc_ctx(&c); + break; + case TEE_ALG_SM4_CBC_NOPAD: + res = crypto_sm4_cbc_alloc_ctx(&c); + break; + case TEE_ALG_SM4_CTR: + res = crypto_sm4_ctr_alloc_ctx(&c); + break; + case TEE_ALG_SM4_XTS: + res = crypto_sm4_xts_alloc_ctx(&c); + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + } + + if (!res) + *ctx = c; + + return res; +} + +static const struct crypto_cipher_ops *cipher_ops(void *ctx) +{ + struct crypto_cipher_ctx *c = ctx; + + assert(c && c->ops); + + return c->ops; +} + +void crypto_cipher_free_ctx(void *ctx) +{ + if (ctx) + cipher_ops(ctx)->free_ctx(ctx); +} + +void crypto_cipher_copy_state(void *dst_ctx, void *src_ctx) +{ + cipher_ops(dst_ctx)->copy_state(dst_ctx, src_ctx); +} + +TEE_Result crypto_cipher_init(void *ctx, TEE_OperationMode mode, + const uint8_t *key1, size_t key1_len, + const uint8_t *key2, size_t key2_len, + const uint8_t *iv, size_t iv_len) +{ + if (mode != TEE_MODE_DECRYPT && mode != TEE_MODE_ENCRYPT) + return TEE_ERROR_BAD_PARAMETERS; + + return cipher_ops(ctx)->init(ctx, mode, key1, key1_len, key2, key2_len, + iv, iv_len); +} + +TEE_Result crypto_cipher_update(void *ctx, TEE_OperationMode mode __unused, + bool last_block, const uint8_t *data, + size_t len, uint8_t *dst) +{ + return cipher_ops(ctx)->update(ctx, last_block, data, len, dst); +} + +void crypto_cipher_final(void *ctx) +{ + cipher_ops(ctx)->final(ctx); +} + +TEE_Result crypto_cipher_get_block_size(uint32_t algo, size_t *size) +{ + uint32_t class = TEE_ALG_GET_CLASS(algo); + + if (class != TEE_OPERATION_CIPHER && class != TEE_OPERATION_MAC && + class != TEE_OPERATION_AE) + return TEE_ERROR_BAD_PARAMETERS; + + switch (TEE_ALG_GET_MAIN_ALG(algo)) { + case TEE_MAIN_ALGO_AES: + *size = TEE_AES_BLOCK_SIZE; + return TEE_SUCCESS; + case TEE_MAIN_ALGO_DES: + case TEE_MAIN_ALGO_DES3: + *size = TEE_DES_BLOCK_SIZE; + return TEE_SUCCESS; + case TEE_MAIN_ALGO_SM4: + *size = TEE_SM4_BLOCK_SIZE; + return TEE_SUCCESS; + default: + return TEE_ERROR_NOT_SUPPORTED; + } +} + +TEE_Result crypto_mac_alloc_ctx(void **ctx, uint32_t algo) +{ + TEE_Result res = TEE_SUCCESS; + struct crypto_mac_ctx *c = NULL; + + /* + * Use default cryptographic implementation if no matching + * drvcrypt device. + */ + res = drvcrypt_mac_alloc_ctx(&c, algo); + + if (res == TEE_ERROR_NOT_IMPLEMENTED) { + switch (algo) { + case TEE_ALG_HMAC_MD5: + res = crypto_hmac_md5_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA1: + res = crypto_hmac_sha1_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA224: + res = crypto_hmac_sha224_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA256: + res = crypto_hmac_sha256_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA384: + res = crypto_hmac_sha384_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA512: + res = crypto_hmac_sha512_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA3_224: + res = crypto_hmac_sha3_224_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA3_256: + res = crypto_hmac_sha3_256_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA3_384: + res = crypto_hmac_sha3_384_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SHA3_512: + res = crypto_hmac_sha3_512_alloc_ctx(&c); + break; + case TEE_ALG_HMAC_SM3: + res = crypto_hmac_sm3_alloc_ctx(&c); + break; + case TEE_ALG_AES_CBC_MAC_NOPAD: + res = crypto_aes_cbc_mac_nopad_alloc_ctx(&c); + break; + case TEE_ALG_AES_CBC_MAC_PKCS5: + res = crypto_aes_cbc_mac_pkcs5_alloc_ctx(&c); + break; + case TEE_ALG_DES_CBC_MAC_NOPAD: + res = crypto_des_cbc_mac_nopad_alloc_ctx(&c); + break; + case TEE_ALG_DES_CBC_MAC_PKCS5: + res = crypto_des_cbc_mac_pkcs5_alloc_ctx(&c); + break; + case TEE_ALG_DES3_CBC_MAC_NOPAD: + res = crypto_des3_cbc_mac_nopad_alloc_ctx(&c); + break; + case TEE_ALG_DES3_CBC_MAC_PKCS5: + res = crypto_des3_cbc_mac_pkcs5_alloc_ctx(&c); + break; + case TEE_ALG_DES3_CMAC: + res = crypto_des3_cmac_alloc_ctx(&c); + break; + case TEE_ALG_AES_CMAC: + res = crypto_aes_cmac_alloc_ctx(&c); + break; + default: + return TEE_ERROR_NOT_SUPPORTED; + } + } + + if (!res) + *ctx = c; + + return res; +} + +static const struct crypto_mac_ops *mac_ops(void *ctx) +{ + struct crypto_mac_ctx *c = ctx; + + assert(c && c->ops); + + return c->ops; +} + +void crypto_mac_free_ctx(void *ctx) +{ + if (ctx) + mac_ops(ctx)->free_ctx(ctx); +} + +void crypto_mac_copy_state(void *dst_ctx, void *src_ctx) +{ + mac_ops(dst_ctx)->copy_state(dst_ctx, src_ctx); +} + +TEE_Result crypto_mac_init(void *ctx, const uint8_t *key, size_t len) +{ + return mac_ops(ctx)->init(ctx, key, len); +} + +TEE_Result crypto_mac_update(void *ctx, const uint8_t *data, size_t len) +{ + if (!len) + return TEE_SUCCESS; + + return mac_ops(ctx)->update(ctx, data, len); +} + +TEE_Result crypto_mac_final(void *ctx, uint8_t *digest, size_t digest_len) +{ + return mac_ops(ctx)->final(ctx, digest, digest_len); +} + +TEE_Result crypto_authenc_alloc_ctx(void **ctx, uint32_t algo) +{ + TEE_Result res = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_authenc_ctx *c = NULL; + + /* + * Use default authenc implementation if no matching + * drvcrypt device. + */ + res = drvcrypt_authenc_alloc_ctx(&c, algo); + + if (res == TEE_ERROR_NOT_IMPLEMENTED) { + switch (algo) { +#if defined(CFG_CRYPTO_CCM) + case TEE_ALG_AES_CCM: + res = crypto_aes_ccm_alloc_ctx(&c); + break; +#endif +#if defined(CFG_CRYPTO_GCM) + case TEE_ALG_AES_GCM: + res = crypto_aes_gcm_alloc_ctx(&c); + break; +#endif + default: + break; + } + } + + if (!res) + *ctx = c; + + return res; +} + +static const struct crypto_authenc_ops *ae_ops(void *ctx) +{ + struct crypto_authenc_ctx *c = ctx; + + assert(c && c->ops); + + return c->ops; +} + +TEE_Result crypto_authenc_init(void *ctx, TEE_OperationMode mode, + const uint8_t *key, size_t key_len, + const uint8_t *nonce, size_t nonce_len, + size_t tag_len, size_t aad_len, + size_t payload_len) +{ + return ae_ops(ctx)->init(ctx, mode, key, key_len, nonce, nonce_len, + tag_len, aad_len, payload_len); +} + +TEE_Result crypto_authenc_update_aad(void *ctx, TEE_OperationMode mode __unused, + const uint8_t *data, size_t len) +{ + return ae_ops(ctx)->update_aad(ctx, data, len); +} + + +TEE_Result crypto_authenc_update_payload(void *ctx, TEE_OperationMode mode, + const uint8_t *src_data, + size_t src_len, uint8_t *dst_data, + size_t *dst_len) +{ + if (*dst_len < src_len) + return TEE_ERROR_SHORT_BUFFER; + *dst_len = src_len; + + return ae_ops(ctx)->update_payload(ctx, mode, src_data, src_len, + dst_data); +} + +TEE_Result crypto_authenc_enc_final(void *ctx, const uint8_t *src_data, + size_t src_len, uint8_t *dst_data, + size_t *dst_len, uint8_t *dst_tag, + size_t *dst_tag_len) +{ + if (*dst_len < src_len) + return TEE_ERROR_SHORT_BUFFER; + *dst_len = src_len; + + return ae_ops(ctx)->enc_final(ctx, src_data, src_len, dst_data, + dst_tag, dst_tag_len); +} + +TEE_Result crypto_authenc_dec_final(void *ctx, const uint8_t *src_data, + size_t src_len, uint8_t *dst_data, + size_t *dst_len, const uint8_t *tag, + size_t tag_len) +{ + if (*dst_len < src_len) + return TEE_ERROR_SHORT_BUFFER; + *dst_len = src_len; + + return ae_ops(ctx)->dec_final(ctx, src_data, src_len, dst_data, tag, + tag_len); +} + +void crypto_authenc_final(void *ctx) +{ + ae_ops(ctx)->final(ctx); +} + +void crypto_authenc_free_ctx(void *ctx) +{ + if (ctx) + ae_ops(ctx)->free_ctx(ctx); +} + +void crypto_authenc_copy_state(void *dst_ctx, void *src_ctx) +{ + ae_ops(dst_ctx)->copy_state(dst_ctx, src_ctx); +} + +#if !defined(CFG_CRYPTO_RSA) && !defined(CFG_CRYPTO_DSA) && \ + !defined(CFG_CRYPTO_DH) && !defined(CFG_CRYPTO_ECC) +struct bignum *crypto_bignum_allocate(size_t size_bits __unused) +{ + return NULL; +} + +TEE_Result crypto_bignum_bin2bn(const uint8_t *from __unused, + size_t fromsize __unused, + struct bignum *to __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +size_t crypto_bignum_num_bytes(struct bignum *a __unused) +{ + return 0; +} + +size_t crypto_bignum_num_bits(struct bignum *a __unused) +{ + return 0; +} + +/* + * crypto_bignum_allocate() and crypto_bignum_bin2bn() failing should be + * enough to guarantee that the functions calling this function aren't + * called, but just in case add a panic() here to avoid unexpected + * behavoir. + */ +static void bignum_cant_happen(void) +{ + volatile bool b = true; + + /* Avoid warning about function does not return */ + if (b) + panic(); +} + +void crypto_bignum_bn2bin(const struct bignum *from __unused, + uint8_t *to __unused) +{ + bignum_cant_happen(); +} + +void crypto_bignum_copy(struct bignum *to __unused, + const struct bignum *from __unused) +{ + bignum_cant_happen(); +} + +void crypto_bignum_free(struct bignum **a) +{ + if (a && *a) + panic(); +} + +void crypto_bignum_clear(struct bignum *a __unused) +{ + bignum_cant_happen(); +} + +/* return -1 if ab */ +int32_t crypto_bignum_compare(struct bignum *a __unused, + struct bignum *b __unused) +{ + bignum_cant_happen(); + return -1; +} +#endif + +#if !defined(CFG_CRYPTO_RSA) +TEE_Result crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *s __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result +crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *s __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +void crypto_acipher_free_rsa_public_key(struct rsa_public_key *s __unused) +{ +} + +void crypto_acipher_free_rsa_keypair(struct rsa_keypair *s __unused) +{ +} + +TEE_Result crypto_acipher_gen_rsa_key(struct rsa_keypair *key __unused, + size_t key_size __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key __unused, + const uint8_t *src __unused, + size_t src_len __unused, + uint8_t *dst __unused, + size_t *dst_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key __unused, + const uint8_t *src __unused, + size_t src_len __unused, + uint8_t *dst __unused, + size_t *dst_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_rsaes_decrypt(uint32_t algo __unused, + struct rsa_keypair *key __unused, + const uint8_t *label __unused, + size_t label_len __unused, + const uint8_t *src __unused, + size_t src_len __unused, + uint8_t *dst __unused, + size_t *dst_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_rsaes_encrypt(uint32_t algo __unused, + struct rsa_public_key *key __unused, + const uint8_t *label __unused, + size_t label_len __unused, + const uint8_t *src __unused, + size_t src_len __unused, + uint8_t *dst __unused, + size_t *dst_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_rsassa_sign(uint32_t algo __unused, + struct rsa_keypair *key __unused, + int salt_len __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + uint8_t *sig __unused, + size_t *sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_rsassa_verify(uint32_t algo __unused, + struct rsa_public_key *key __unused, + int salt_len __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + const uint8_t *sig __unused, + size_t sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /*!CFG_CRYPTO_RSA*/ + +#if !defined(CFG_CRYPTO_DSA) +TEE_Result crypto_acipher_alloc_dsa_keypair(struct dsa_keypair *s __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result +crypto_acipher_alloc_dsa_public_key(struct dsa_public_key *s __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_gen_dsa_key(struct dsa_keypair *key __unused, + size_t key_size __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_dsa_sign(uint32_t algo __unused, + struct dsa_keypair *key __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + uint8_t *sig __unused, + size_t *sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_dsa_verify(uint32_t algo __unused, + struct dsa_public_key *key __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + const uint8_t *sig __unused, + size_t sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /*!CFG_CRYPTO_DSA*/ + +#if !defined(CFG_CRYPTO_DH) +TEE_Result crypto_acipher_alloc_dh_keypair(struct dh_keypair *s __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_gen_dh_key(struct dh_keypair *key __unused, + struct bignum *q __unused, + size_t xbits __unused, + size_t key_size __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result +crypto_acipher_dh_shared_secret(struct dh_keypair *private_key __unused, + struct bignum *public_key __unused, + struct bignum *secret __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /*!CFG_CRYPTO_DH*/ + +TEE_Result crypto_acipher_alloc_ecc_public_key(struct ecc_public_key *key, + uint32_t key_type, + size_t key_size_bits) +{ + TEE_Result res = TEE_ERROR_NOT_IMPLEMENTED; + + /* + * Use default cryptographic implementation if no matching + * drvcrypt device. + */ + res = drvcrypt_asym_alloc_ecc_public_key(key, key_type, key_size_bits); + if (res == TEE_ERROR_NOT_IMPLEMENTED) + res = crypto_asym_alloc_ecc_public_key(key, key_type, + key_size_bits); + + return res; +} + +TEE_Result crypto_acipher_alloc_ecc_keypair(struct ecc_keypair *key, + uint32_t key_type, + size_t key_size_bits) +{ + TEE_Result res = TEE_ERROR_NOT_IMPLEMENTED; + + /* + * Use default cryptographic implementation if no matching + * drvcrypt device. + */ + res = drvcrypt_asym_alloc_ecc_keypair(key, key_type, key_size_bits); + if (res == TEE_ERROR_NOT_IMPLEMENTED) + res = crypto_asym_alloc_ecc_keypair(key, key_type, + key_size_bits); + + return res; +} + +void crypto_acipher_free_ecc_public_key(struct ecc_public_key *key) +{ + assert(key->ops && key->ops->free); + + key->ops->free(key); +} + +TEE_Result crypto_acipher_gen_ecc_key(struct ecc_keypair *key, + size_t key_size_bits) +{ + assert(key->ops && key->ops->generate); + + return key->ops->generate(key, key_size_bits); +} + +TEE_Result crypto_acipher_ecc_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + assert(key->ops); + + if (!key->ops->sign) + return TEE_ERROR_NOT_IMPLEMENTED; + + return key->ops->sign(algo, key, msg, msg_len, sig, sig_len); +} + +TEE_Result crypto_acipher_ecc_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + assert(key->ops); + + if (!key->ops->verify) + return TEE_ERROR_NOT_IMPLEMENTED; + + return key->ops->verify(algo, key, msg, msg_len, sig, sig_len); +} + +TEE_Result crypto_acipher_ecc_shared_secret(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, + unsigned long *secret_len) +{ + assert(private_key->ops); + + if (!private_key->ops->shared_secret) + return TEE_ERROR_NOT_IMPLEMENTED; + + return private_key->ops->shared_secret(private_key, public_key, secret, + secret_len); +} + +TEE_Result crypto_acipher_sm2_pke_decrypt(struct ecc_keypair *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + assert(key->ops); + + if (!key->ops->decrypt) + return TEE_ERROR_NOT_IMPLEMENTED; + + return key->ops->decrypt(key, src, src_len, dst, dst_len); +} + +TEE_Result crypto_acipher_sm2_pke_encrypt(struct ecc_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + assert(key->ops); + + if (!key->ops->encrypt) + return TEE_ERROR_NOT_IMPLEMENTED; + + return key->ops->encrypt(key, src, src_len, dst, dst_len); +} + +#if !defined(CFG_CRYPTO_SM2_KEP) +TEE_Result crypto_acipher_sm2_kep_derive(struct ecc_keypair *my_key __unused, + struct ecc_keypair *my_eph_key + __unused, + struct ecc_public_key *peer_key + __unused, + struct ecc_public_key *peer_eph_key + __unused, + struct sm2_kep_parms *p __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif + +#if !defined(CFG_CRYPTO_X25519) +TEE_Result crypto_acipher_alloc_x25519_keypair(struct montgomery_keypair *key + __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_gen_x25519_key(struct montgomery_keypair + *key __unused, + size_t key_size __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_x25519_shared_secret(struct montgomery_keypair + *private_key __unused, + void *public_key __unused, + void *secret __unused, + unsigned long + *secret_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif + +#if !defined(CFG_CRYPTO_X448) +TEE_Result crypto_acipher_alloc_x448_keypair(struct montgomery_keypair *key + __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_gen_x448_key(struct montgomery_keypair *key __unused, + size_t key_size __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_x448_shared_secret(struct montgomery_keypair + *private_key __unused, + void *public_key __unused, + void *secret __unused, + unsigned long + *secret_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif + +#if !defined(CFG_CRYPTO_ED25519) +TEE_Result crypto_acipher_alloc_ed25519_keypair(struct ed25519_keypair *key + __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result +crypto_acipher_alloc_ed25519_public_key(struct ed25519_public_key *key __unused, + size_t key_size __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_gen_ed25519_key(struct ed25519_keypair *key __unused, + size_t key_size __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_ed25519_sign(struct ed25519_keypair *key __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + uint8_t *sig __unused, + size_t *sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result +crypto_acipher_ed25519_verify(struct ed25519_public_key *key __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + const uint8_t *sig __unused, + size_t sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result crypto_acipher_ed25519ctx_sign(struct ed25519_keypair *key __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + uint8_t *sig __unused, + size_t *sig_len __unused, + bool ph_flag __unused, + const uint8_t *ctx __unused, + size_t ctxlen __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result +crypto_acipher_ed25519ctx_verify(struct ed25519_public_key *key __unused, + const uint8_t *msg __unused, + size_t msg_len __unused, + const uint8_t *sig __unused, + size_t sig_len __unused, + bool ph_flag __unused, + const uint8_t *ctx __unused, + size_t ctxlen __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif + +__weak TEE_Result crypto_storage_obj_del(struct tee_obj *obj __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} diff --git a/optee_os/core/crypto/rng_fortuna.c b/optee_os/core/crypto/rng_fortuna.c new file mode 100644 index 0000000..f9317b6 --- /dev/null +++ b/optee_os/core/crypto/rng_fortuna.c @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* Copyright (c) 2018, Linaro Limited */ + +/* + * This is an implementation of the Fortuna cryptographic PRNG as defined in + * https://www.schneier.com/academic/paperfiles/fortuna.pdf + * There's one small exception, see comment in restart_pool() below. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define NUM_POOLS 32 +#define BLOCK_SIZE 16 +#define KEY_SIZE 32 +#define CIPHER_ALGO TEE_ALG_AES_ECB_NOPAD +#define HASH_ALGO TEE_ALG_SHA256 +#define MIN_POOL_SIZE 64 +#define MAX_EVENT_DATA_LEN 32U +#define RING_BUF_DATA_SIZE 4U + +/* + * struct fortuna_state - state of the Fortuna PRNG + * @ctx: Cipher context used to produce the random numbers + * @counter: Counter which is encrypted to produce the random numbers + * @pool0_length: Amount of data added to pool0 + * @pool_ctx: One hash context for each pool + * @reseed_ctx: Hash context used while reseeding + * @reseed_count: Number of time we've reseeded the PRNG, used to tell + * which pools should be used in the reseed process + * @next_reseed_time: If we have a secure time, the earliest next time we + * may reseed + * + * To minimize the delay in crypto_rng_add_event() there's @pool_spin_lock + * which protects everything needed by this function. + * + * @next_reseed_time is used as a rate limiter for reseeding. + */ +static struct fortuna_state { + void *ctx; + uint64_t counter[2]; + unsigned int pool0_length; + void *pool_ctx[NUM_POOLS]; + void *reseed_ctx; + uint32_t reseed_count; +#ifndef CFG_SECURE_TIME_SOURCE_REE + TEE_Time next_reseed_time; +#endif +} state; + +static struct mutex state_mu = MUTEX_INITIALIZER; + +static struct { + struct { + uint8_t snum; + uint8_t pnum; + uint8_t dlen; + uint8_t data[RING_BUF_DATA_SIZE]; + } elem[8]; + unsigned int begin; + unsigned int end; +} ring_buffer; + +unsigned int ring_buffer_spin_lock; + +static void inc_counter(uint64_t counter[2]) +{ + counter[0]++; + if (!counter[0]) + counter[1]++; +} + +static TEE_Result hash_init(void *ctx) +{ + return crypto_hash_init(ctx); +} + +static TEE_Result hash_update(void *ctx, const void *data, size_t dlen) +{ + return crypto_hash_update(ctx, data, dlen); +} + +static TEE_Result hash_final(void *ctx, uint8_t digest[KEY_SIZE]) +{ + return crypto_hash_final(ctx, digest, KEY_SIZE); +} + +static TEE_Result key_from_data(void *ctx, const void *data, size_t dlen, + uint8_t key[KEY_SIZE]) +{ + TEE_Result res; + + res = hash_init(ctx); + if (res) + return res; + res = hash_update(ctx, data, dlen); + if (res) + return res; + return hash_final(ctx, key); +} + +static TEE_Result cipher_init(void *ctx, uint8_t key[KEY_SIZE]) +{ + return crypto_cipher_init(ctx, TEE_MODE_ENCRYPT, + key, KEY_SIZE, NULL, 0, NULL, 0); +} + +static void fortuna_done(void) +{ + size_t n; + + for (n = 0; n < NUM_POOLS; n++) { + crypto_hash_free_ctx(state.pool_ctx[n]); + state.pool_ctx[n] = NULL; + } + crypto_hash_free_ctx(state.reseed_ctx); + state.reseed_ctx = NULL; + crypto_cipher_free_ctx(state.ctx); + state.ctx = NULL; +} + +TEE_Result crypto_rng_init(const void *data, size_t dlen) +{ + TEE_Result res; + uint8_t key[KEY_SIZE]; + void *ctx; + size_t n; + + COMPILE_TIME_ASSERT(sizeof(state.counter) == BLOCK_SIZE); + + if (state.ctx) + return TEE_ERROR_BAD_STATE; + + memset(&state, 0, sizeof(state)); + + for (n = 0; n < NUM_POOLS; n++) { + res = crypto_hash_alloc_ctx(&state.pool_ctx[n], HASH_ALGO); + if (res) + goto err; + res = crypto_hash_init(state.pool_ctx[n]); + if (res) + goto err; + } + + res = crypto_hash_alloc_ctx(&state.reseed_ctx, HASH_ALGO); + if (res) + goto err; + + res = key_from_data(state.reseed_ctx, data, dlen, key); + if (res) + return res; + + res = crypto_cipher_alloc_ctx(&ctx, CIPHER_ALGO); + if (res) + return res; + res = cipher_init(ctx, key); + if (res) + return res; + inc_counter(state.counter); + state.ctx = ctx; + return TEE_SUCCESS; +err: + fortuna_done(); + return res; +} + +static void push_ring_buffer(uint8_t snum, uint8_t pnum, const void *data, + size_t dlen) +{ + uint8_t dl = MIN(RING_BUF_DATA_SIZE, dlen); + unsigned int next_begin; + uint32_t old_itr_status; + + /* Spinlock to serialize writers */ + old_itr_status = cpu_spin_lock_xsave(&ring_buffer_spin_lock); + + next_begin = (ring_buffer.begin + 1) % ARRAY_SIZE(ring_buffer.elem); + if (next_begin == atomic_load_uint(&ring_buffer.end)) + goto out; /* buffer is full */ + + ring_buffer.elem[next_begin].snum = snum; + ring_buffer.elem[next_begin].pnum = pnum; + ring_buffer.elem[next_begin].dlen = dl; + memcpy(ring_buffer.elem[next_begin].data, data, dl); + + atomic_store_uint(&ring_buffer.begin, next_begin); + +out: + cpu_spin_unlock_xrestore(&ring_buffer_spin_lock, old_itr_status); +} + +static size_t pop_ring_buffer(uint8_t *snum, uint8_t *pnum, + uint8_t data[RING_BUF_DATA_SIZE]) +{ + unsigned int next_end; + size_t dlen; + + if (atomic_load_uint(&ring_buffer.begin) == ring_buffer.end) + return 0; + + next_end = (ring_buffer.end + 1) % ARRAY_SIZE(ring_buffer.elem); + + *snum = ring_buffer.elem[ring_buffer.end].snum; + *pnum = ring_buffer.elem[ring_buffer.end].pnum; + dlen = MIN(ring_buffer.elem[ring_buffer.end].dlen, RING_BUF_DATA_SIZE); + assert(ring_buffer.elem[ring_buffer.end].dlen == dlen); + memcpy(data, ring_buffer.elem[ring_buffer.end].data, dlen); + + atomic_store_uint(&ring_buffer.end, next_end); + + return dlen; +} + +static TEE_Result add_event(uint8_t snum, uint8_t pnum, + const void *data, size_t dlen) +{ + TEE_Result res; + size_t dl = MIN(MAX_EVENT_DATA_LEN, dlen); + uint8_t v[] = { snum, dl }; + + if (pnum >= NUM_POOLS) + return TEE_ERROR_BAD_PARAMETERS; + + res = hash_update(state.pool_ctx[pnum], v, sizeof(v)); + if (res) + return res; + res = hash_update(state.pool_ctx[pnum], data, dl); + if (res) + return res; + if (!pnum) { + unsigned int l; + + if (!ADD_OVERFLOW(state.pool0_length, dl, &l)) + state.pool0_length = l; + } + + return TEE_SUCCESS; +} + +static TEE_Result drain_ring_buffer(void) +{ + while (true) { + TEE_Result res; + uint8_t snum; + uint8_t pnum; + uint8_t data[RING_BUF_DATA_SIZE]; + size_t dlen; + + dlen = pop_ring_buffer(&snum, &pnum, data); + if (!dlen) + return TEE_SUCCESS; + + res = add_event(snum, pnum, data, dlen); + if (res) + return res; + } +} + +static unsigned int get_next_pnum(unsigned int *pnum) +{ + unsigned int nval; + unsigned int oval = atomic_load_uint(pnum); + + while (true) { + nval = (oval + 1) % NUM_POOLS; + + if (atomic_cas_uint(pnum, &oval, nval)) { + /* + * *pnum is normally initialized to 0 and we'd like + * to start feeding pool number 0 as that's the + * most important one. + * + * If we where to take just *pnum and increase it + * later multiple updaters could end up with the + * same number. + * + * By increasing first we get the number unique for + * next update and by subtracting one (using + * modulus) we get the number for this update. + */ + return (nval + NUM_POOLS - 1) % NUM_POOLS; + } + /* + * At this point atomic_cas_uint() has updated oval to the + * current *pnum. + */ + } +} + +void crypto_rng_add_event(enum crypto_rng_src sid, unsigned int *pnum, + const void *data, size_t dlen) +{ + unsigned int pn = get_next_pnum(pnum); + uint8_t snum = sid >> 1; + + if (CRYPTO_RNG_SRC_IS_QUICK(sid)) { + push_ring_buffer(snum, pn, data, dlen); + } else { + mutex_lock(&state_mu); + add_event(snum, pn, data, dlen); + drain_ring_buffer(); + mutex_unlock(&state_mu); + } +} + +/* GenerateBlocks */ +static TEE_Result generate_blocks(void *block, size_t nblocks) +{ + uint8_t *b = block; + size_t n; + + for (n = 0; n < nblocks; n++) { + TEE_Result res = crypto_cipher_update(state.ctx, + TEE_MODE_ENCRYPT, false, + (void *)state.counter, + BLOCK_SIZE, + b + n * BLOCK_SIZE); + + /* + * Make sure to increase the counter before returning an + * eventual errors, we must never re-use the counter with + * the same key. + */ + inc_counter(state.counter); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +/* GenerateRandomData */ +static TEE_Result generate_random_data(void *buf, size_t blen) +{ + TEE_Result res; + + res = generate_blocks(buf, blen / BLOCK_SIZE); + if (res) + return res; + if (blen % BLOCK_SIZE) { + uint8_t block[BLOCK_SIZE]; + uint8_t *b = (uint8_t *)buf + ROUNDDOWN(blen, BLOCK_SIZE); + + res = generate_blocks(block, 1); + if (res) + return res; + memcpy(b, block, blen % BLOCK_SIZE); + } + + return TEE_SUCCESS; +} + +#ifdef CFG_SECURE_TIME_SOURCE_REE +static bool reseed_rate_limiting(void) +{ + /* + * There's no point in checking REE time for reseed rate limiting, + * and also it makes it less complicated if we can avoid doing RPC + * here. + */ + return false; +} +#else +static bool reseed_rate_limiting(void) +{ + TEE_Result res; + TEE_Time time; + const TEE_Time time_100ms = { 0, 100 }; + + res = tee_time_get_sys_time(&time); + /* + * Failure to read time must result in allowing reseed or we could + * block reseeding forever. + */ + if (res) + return false; + + if (TEE_TIME_LT(time, state.next_reseed_time)) + return true; + + /* Time to reseed, calculate next time reseed is OK */ + TEE_TIME_ADD(time, time_100ms, state.next_reseed_time); + return false; +} +#endif + +static TEE_Result restart_pool(void *pool_ctx, uint8_t pool_digest[KEY_SIZE]) +{ + TEE_Result res = hash_final(pool_ctx, pool_digest); + + if (res) + return res; + + res = hash_init(pool_ctx); + if (res) + return res; + + /* + * Restart the pool with the digest of the old pool. This is an + * extension to Fortuna. In the original Fortuna all pools was + * restarted from scratch. This extension is one more defense + * against spamming of the pools with known data which could lead + * to the spammer knowing the state of the pools. + * + * This extra precaution could be useful since OP-TEE sometimes + * have very few sources of good entropy and at the same time has + * sources that could quite easily be predicted by an attacker. + */ + return hash_update(pool_ctx, pool_digest, KEY_SIZE); +} + +static bool reseed_from_pool(uint32_t reseed_count, size_t pool_num) +{ + /* + * Specification says: use pool if + * 2^pool_num is a divisor of reseed_count + * + * in order to avoid an expensive modulus operation we're + * optimizing this below. + */ + return !pool_num || !((reseed_count >> (pool_num - 1)) & 1); +} + +static TEE_Result maybe_reseed(void) +{ + TEE_Result res; + size_t n; + uint8_t pool_digest[KEY_SIZE]; + + if (state.pool0_length < MIN_POOL_SIZE) + return TEE_SUCCESS; + + if (reseed_rate_limiting()) + return TEE_SUCCESS; + + state.reseed_count++; + + res = hash_init(state.reseed_ctx); + if (res) + return res; + + for (n = 0; + n < NUM_POOLS && reseed_from_pool(state.reseed_count, n); n++) { + res = restart_pool(state.pool_ctx[n], pool_digest); + if (res) + return res; + if (!n) + state.pool0_length = 0; + + res = hash_update(state.reseed_ctx, pool_digest, KEY_SIZE); + if (res) + return res; + } + res = hash_final(state.reseed_ctx, pool_digest); + if (res) + return res; + + crypto_cipher_final(state.ctx); + res = crypto_cipher_init(state.ctx, TEE_MODE_ENCRYPT, + pool_digest, KEY_SIZE, NULL, 0, NULL, 0); + if (res) + return res; + inc_counter(state.counter); + + return TEE_SUCCESS; +} + +static TEE_Result fortuna_read(void *buf, size_t blen) +{ + TEE_Result res; + + if (!state.ctx) + return TEE_ERROR_BAD_STATE; + + mutex_lock(&state_mu); + + res = maybe_reseed(); + if (res) + goto out; + + if (blen) { + uint8_t new_key[KEY_SIZE]; + + res = generate_random_data(buf, blen); + if (res) + goto out; + + res = generate_blocks(new_key, KEY_SIZE / BLOCK_SIZE); + if (res) + goto out; + crypto_cipher_final(state.ctx); + res = cipher_init(state.ctx, new_key); + if (res) + goto out; + } + + res = drain_ring_buffer(); +out: + if (res) + fortuna_done(); + mutex_unlock(&state_mu); + + return res; +} + +TEE_Result crypto_rng_read(void *buf, size_t blen) +{ + size_t offs = 0; + + while (true) { + TEE_Result res; + size_t n; + + /* Draw at most 1 MiB of random on a single key */ + n = MIN(blen - offs, SIZE_1M); + if (!n) + return TEE_SUCCESS; + res = fortuna_read((uint8_t *)buf + offs, n); + if (res) + return res; + offs += n; + } +} diff --git a/optee_os/core/crypto/rng_hw.c b/optee_os/core/crypto/rng_hw.c new file mode 100644 index 0000000..4d94ea7 --- /dev/null +++ b/optee_os/core/crypto/rng_hw.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* Copyright (c) 2018, Linaro Limited */ + +#include +#include +#include +#include +#include +#include + +/* This is a HW RNG, no need for seeding */ +TEE_Result crypto_rng_init(const void *data __unused, size_t dlen __unused) +{ + return TEE_SUCCESS; +} + +/* This is a HW RNG, no need to add entropy */ +void crypto_rng_add_event(enum crypto_rng_src sid __unused, + unsigned int *pnum __unused, + const void *data __unused, + size_t dlen __unused) +{ +} + +TEE_Result crypto_rng_read(void *buf, size_t blen) +{ + if (!buf) + return TEE_ERROR_BAD_PARAMETERS; + + return hw_get_random_bytes(buf, blen); +} diff --git a/optee_os/core/crypto/signed_hdr.c b/optee_os/core/crypto/signed_hdr.c new file mode 100644 index 0000000..40c2997 --- /dev/null +++ b/optee_os/core/crypto/signed_hdr.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct shdr *shdr_alloc_and_copy(size_t offs, const void *img, size_t img_size) +{ + size_t shdr_size; + struct shdr *shdr; + vaddr_t img_va = (vaddr_t)img; + vaddr_t tmp = 0; + size_t end = 0; + + if (ADD_OVERFLOW(offs, sizeof(struct shdr), &end) || end > img_size) + return NULL; + + shdr_size = SHDR_GET_SIZE((const struct shdr *)(img_va + offs)); + if (ADD_OVERFLOW(offs, shdr_size, &end) || end > img_size) + return NULL; + + if (ADD_OVERFLOW(img_va, shdr_size, &tmp)) + return NULL; + + shdr = malloc(shdr_size); + if (!shdr) + return NULL; + memcpy(shdr, (const uint8_t *)img + offs, shdr_size); + + /* Check that the data wasn't modified before the copy was completed */ + if (shdr_size != SHDR_GET_SIZE(shdr)) { + free(shdr); + return NULL; + } + + return shdr; +} + +static bool is_weak_hash_algo(uint32_t algo) +{ + return algo == TEE_ALG_MD5 || algo == TEE_ALG_SHA1 || + algo == TEE_ALG_MD5SHA1; +} + +TEE_Result shdr_verify_signature(const struct shdr *shdr) +{ + struct rsa_public_key key = { }; + TEE_Result res = TEE_SUCCESS; + uint32_t e = TEE_U32_TO_BIG_ENDIAN(ta_pub_key_exponent); + struct ftmn ftmn = { }; + unsigned int err_incr = 2; + size_t hash_size = 0; + size_t hash_algo = 0; + + if (shdr->magic != SHDR_MAGIC) + goto err; + + if (TEE_ALG_GET_MAIN_ALG(shdr->algo) != TEE_MAIN_ALGO_RSA) + goto err; + + hash_algo = TEE_DIGEST_HASH_TO_ALGO(shdr->algo); + if (is_weak_hash_algo(hash_algo)) + goto err; + + res = tee_alg_get_digest_size(hash_algo, &hash_size); + if (res) + goto err; + if (hash_size != shdr->hash_size) + goto err; + + res = crypto_acipher_alloc_rsa_public_key(&key, + ta_pub_key_modulus_size * 8); + if (res) + goto err; + + res = crypto_bignum_bin2bn((uint8_t *)&e, sizeof(e), key.e); + if (res) + goto err; + res = crypto_bignum_bin2bn(ta_pub_key_modulus, ta_pub_key_modulus_size, + key.n); + if (res) + goto err; + + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + crypto_acipher_rsassa_verify, shdr->algo, &key, + shdr->hash_size, SHDR_GET_HASH(shdr), shdr->hash_size, + SHDR_GET_SIG(shdr), shdr->sig_size); + if (!res) { + ftmn_checkpoint(&ftmn, FTMN_INCR0); + goto out; + } + err_incr = 1; +err: + res = TEE_ERROR_SECURITY; + FTMN_SET_CHECK_RES_NOT_ZERO(&ftmn, err_incr * FTMN_INCR0, res); +out: + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR0, FTMN_STEP_COUNT(2), res); + crypto_acipher_free_rsa_public_key(&key); + return res; +} + +static const struct shdr_subkey_attr * +find_attr(const struct shdr_subkey *subkey, uint32_t id) +{ + size_t n = 0; + + for (n = 0; n < subkey->attr_count; n++) + if (subkey->attrs[n].id == id) + return subkey->attrs + n; + + return NULL; +} + +static TEE_Result load_rsa_key(const struct shdr_subkey *subkey, + struct rsa_public_key **key_pp) +{ + const uint8_t *base = (const uint8_t *)subkey; + const struct shdr_subkey_attr *pub_exp = NULL; + const struct shdr_subkey_attr *modulus = NULL; + struct rsa_public_key *key = NULL; + TEE_Result res = TEE_SUCCESS; + + pub_exp = find_attr(subkey, TEE_ATTR_RSA_PUBLIC_EXPONENT); + if (!pub_exp) + return TEE_ERROR_SECURITY; + modulus = find_attr(subkey, TEE_ATTR_RSA_MODULUS); + if (!modulus) + return TEE_ERROR_SECURITY; + + key = calloc(1, sizeof(*key)); + if (!key) + return TEE_ERROR_OUT_OF_MEMORY; + res = crypto_acipher_alloc_rsa_public_key(key, modulus->size * 8); + if (res) + goto err_key; + + res = crypto_bignum_bin2bn(base + pub_exp->offs, pub_exp->size, key->e); + if (res) + goto err; + res = crypto_bignum_bin2bn(base + modulus->offs, modulus->size, key->n); + if (res) + goto err; + + *key_pp = key; + return TEE_SUCCESS; +err: + crypto_acipher_free_rsa_public_key(key); +err_key: + free(key); + return TEE_ERROR_SECURITY; +} + +static TEE_Result check_attrs(const struct shdr_subkey *subkey, size_t img_size) +{ + const struct shdr_subkey_attr *attrs = subkey->attrs; + size_t end = 0; + size_t n = 0; + + if (MUL_OVERFLOW(subkey->attr_count, sizeof(*attrs), &end) || + ADD_OVERFLOW(end, sizeof(*subkey), &end) || + end > img_size) + return TEE_ERROR_SECURITY; + + for (n = 0; n < subkey->attr_count; n++) + if (ADD_OVERFLOW(attrs[n].offs, attrs[n].size, &end) || + end > img_size) + return TEE_ERROR_SECURITY; + + return TEE_SUCCESS; +} + +static TEE_Result calc_next_uuid(uint8_t uuid[sizeof(TEE_UUID)], + const uint8_t my_uuid[sizeof(TEE_UUID)], + const void *ns_name, size_t name_size) +{ + TEE_Result res = TEE_ERROR_SECURITY; + void *ctx = NULL; + struct { + uint8_t digest[TEE_SHA1_HASH_SIZE]; + TEE_UUID uuid; + char name_str[]; + } *tmp = NULL; + + if (!name_size) { + memcpy(uuid, my_uuid, sizeof(TEE_UUID)); + return TEE_SUCCESS; + } + + /* + * RFC 4122 requires a SHA-1 digest for UUID v5. Use SHA-512 + * instead for better collision resistance. + */ + if (crypto_hash_alloc_ctx(&ctx, TEE_ALG_SHA512)) + return TEE_ERROR_SECURITY; + + tmp = mempool_alloc(mempool_default, sizeof(*tmp) + name_size); + if (!tmp) + goto out_ctx; + memcpy(tmp->name_str, ns_name, name_size); + + if (crypto_hash_init(ctx) || + crypto_hash_update(ctx, my_uuid, sizeof(TEE_UUID)) || + crypto_hash_update(ctx, (const void *)tmp->name_str, + strnlen(tmp->name_str, name_size)) || + crypto_hash_final(ctx, tmp->digest, sizeof(tmp->digest))) + goto out_mempool; + + tee_uuid_from_octets(&tmp->uuid, tmp->digest); + /* + * Set the four most significant bits (bits 12 through 15) of the + * time_hi_and_version field to 5. + */ + tmp->uuid.timeHiAndVersion &= ~SHIFT_U32(0xf, 12); + tmp->uuid.timeHiAndVersion |= SHIFT_U32(5, 12); + /* + * Set the two most significant bits (bits 6 and 7) of the + * clock_seq_hi_and_reserved to zero and one, respectively. + */ + tmp->uuid.clockSeqAndNode[0] &= ~BIT(6); + tmp->uuid.clockSeqAndNode[0] |= BIT(7); + + tee_uuid_to_octets(uuid, &tmp->uuid); + res = TEE_SUCCESS; + +out_mempool: + mempool_free(mempool_default, tmp); +out_ctx: + crypto_hash_free_ctx(ctx); + + return res; +} + +TEE_Result shdr_load_pub_key(const struct shdr *shdr, size_t offs, + const uint8_t *ns_img, size_t ns_img_size, + const uint8_t next_uuid[sizeof(TEE_UUID)], + uint32_t max_depth, struct shdr_pub_key *key) +{ + struct shdr_subkey *subkey = NULL; + TEE_Result res = TEE_SUCCESS; + void *digest = NULL; + uint8_t *img = NULL; + void *ctx = NULL; + size_t end = 0; + + if (shdr->img_type != SHDR_SUBKEY) + return TEE_ERROR_SECURITY; + + if (shdr->img_size < sizeof(*subkey)) + return TEE_ERROR_SECURITY; + + if (ADD_OVERFLOW(shdr->img_size, offs, &end) || end > ns_img_size) + return TEE_ERROR_SECURITY; + + img = mempool_alloc(mempool_default, shdr->img_size + shdr->hash_size); + if (!img) + return TEE_ERROR_OUT_OF_MEMORY; + memcpy(img + shdr->hash_size, ns_img + offs, shdr->img_size); + subkey = (void *)(img + shdr->hash_size); + digest = img; + + if (crypto_hash_alloc_ctx(&ctx, TEE_DIGEST_HASH_TO_ALGO(shdr->algo))) { + res = TEE_ERROR_SECURITY; + goto out_mempool; + } + + if (crypto_hash_init(ctx) || + crypto_hash_update(ctx, (const void *)shdr, sizeof(*shdr)) || + crypto_hash_update(ctx, (const void *)subkey, shdr->img_size) || + crypto_hash_final(ctx, digest, shdr->hash_size) || + memcmp(digest, SHDR_GET_HASH(shdr), shdr->hash_size)) { + res = TEE_ERROR_SECURITY; + goto out_ctx; + } + + res = check_attrs(subkey, shdr->img_size); + if (res) + goto out_ctx; + + if (subkey->max_depth >= max_depth) { + res = TEE_ERROR_SECURITY; + goto out_ctx; + } + if (next_uuid && memcmp(next_uuid, subkey->uuid, sizeof(TEE_UUID))) { + res = TEE_ERROR_SECURITY; + goto out_ctx; + } + + key->max_depth = subkey->max_depth; + key->name_size = subkey->name_size; + memcpy(key->uuid, subkey->uuid, sizeof(TEE_UUID)); + if (ADD_OVERFLOW(key->name_size, offs + shdr->img_size, &end) || + end > ns_img_size) { + res = TEE_ERROR_SECURITY; + goto out_ctx; + } + res = calc_next_uuid(key->next_uuid, key->uuid, + ns_img + offs + shdr->img_size, key->name_size); + if (res) + goto out_ctx; + + key->main_algo = TEE_ALG_GET_MAIN_ALG(subkey->algo); + switch (key->main_algo) { + case TEE_MAIN_ALGO_RSA: + res = load_rsa_key(subkey, &key->pub_key.rsa); + break; + default: + res = TEE_ERROR_SECURITY; + break; + } + +out_ctx: + crypto_hash_free_ctx(ctx); +out_mempool: + mempool_free(mempool_default, img); + return res; +} + +void shdr_free_pub_key(struct shdr_pub_key *key) +{ + if (key) { + switch (key->main_algo) { + case TEE_MAIN_ALGO_RSA: + crypto_acipher_free_rsa_public_key(key->pub_key.rsa); + free(key->pub_key.rsa); + break; + default: + panic(); + } + } +} + +TEE_Result shdr_verify_signature2(struct shdr_pub_key *key, + const struct shdr *shdr) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int err_incr = 2; + struct ftmn ftmn = { }; + size_t hash_size = 0; + size_t hash_algo = 0; + + if (shdr->magic != SHDR_MAGIC) + goto err; + + if (TEE_ALG_GET_MAIN_ALG(shdr->algo) != key->main_algo) + goto err; + + hash_algo = TEE_DIGEST_HASH_TO_ALGO(shdr->algo); + if (is_weak_hash_algo(hash_algo)) + goto err; + + if (tee_alg_get_digest_size(hash_algo, &hash_size) || + hash_size != shdr->hash_size) + goto err; + + switch (key->main_algo) { + case TEE_MAIN_ALGO_RSA: + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + crypto_acipher_rsassa_verify, shdr->algo, + key->pub_key.rsa, shdr->hash_size, + SHDR_GET_HASH(shdr), shdr->hash_size, + SHDR_GET_SIG(shdr), shdr->sig_size); + break; + default: + panic(); + } + + if (!res) { + ftmn_checkpoint(&ftmn, FTMN_INCR0); + goto out; + } + err_incr = 1; +err: + res = TEE_ERROR_SECURITY; + FTMN_SET_CHECK_RES_NOT_ZERO(&ftmn, err_incr * FTMN_INCR0, res); +out: + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR0, FTMN_STEP_COUNT(2), res); + return res; +} diff --git a/optee_os/core/crypto/sm2-kdf.c b/optee_os/core/crypto/sm2-kdf.c new file mode 100644 index 0000000..81f0f0c --- /dev/null +++ b/optee_os/core/crypto/sm2-kdf.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * GM/T 0003.1‒2012 Part 4 Sections 5.4.2 and 5.4.3 + * GM/T 0003.1‒2012 Part 5 Sections 5.4.2 and 5.4.3 + * Key derivation function based on the SM3 hash function + */ +TEE_Result sm2_kdf(const uint8_t *Z, size_t Z_len, uint8_t *t, size_t tlen) +{ + TEE_Result res = TEE_SUCCESS; + size_t remain = tlen; + uint32_t count = 1; + uint32_t be_count = 0; + void *ctx = NULL; + uint8_t *out = t; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + return res; + + while (remain) { + uint8_t tmp[TEE_SM3_HASH_SIZE] = { }; + uint8_t *buf = NULL; + + if (remain >= TEE_SM3_HASH_SIZE) + buf = out; + else + buf = tmp; + + put_be32(&be_count, count); + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, Z, Z_len); + if (res) + goto out; + res = crypto_hash_update(ctx, (const uint8_t *)&be_count, + sizeof(be_count)); + if (res) + goto out; + res = crypto_hash_final(ctx, buf, TEE_SM3_HASH_SIZE); + if (res) + goto out; + + if (remain < TEE_SM3_HASH_SIZE) { + memcpy(out, tmp, remain); + break; + } + + out += TEE_SM3_HASH_SIZE; + remain -= TEE_SM3_HASH_SIZE; + count++; + } +out: + crypto_hash_free_ctx(ctx); + return res; +} + diff --git a/optee_os/core/crypto/sm3-hash.c b/optee_os/core/crypto/sm3-hash.c new file mode 100644 index 0000000..189b08a --- /dev/null +++ b/optee_os/core/crypto/sm3-hash.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + * Copyright (C) 2019 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sm3.h" + +struct sm3_hash_ctx { + struct crypto_hash_ctx hash_ctx; + struct sm3_context sm3_ctx; +}; + +static const struct crypto_hash_ops sm3_hash_ops; + +static struct sm3_hash_ctx *to_hash_ctx(struct crypto_hash_ctx *ctx) +{ + assert(ctx && ctx->ops == &sm3_hash_ops); + + return container_of(ctx, struct sm3_hash_ctx, hash_ctx); +} + +static TEE_Result op_sm3_hash_init(struct crypto_hash_ctx *ctx) +{ + sm3_init(&to_hash_ctx(ctx)->sm3_ctx); + + return TEE_SUCCESS; +} + +static TEE_Result op_sm3_hash_update(struct crypto_hash_ctx *ctx, + const uint8_t *data, size_t len) +{ + sm3_update(&to_hash_ctx(ctx)->sm3_ctx, data, len); + + return TEE_SUCCESS; +} + +static TEE_Result op_sm3_hash_final(struct crypto_hash_ctx *ctx, + uint8_t *digest, + size_t len) +{ + struct sm3_hash_ctx *hc = to_hash_ctx(ctx); + size_t hash_size = TEE_SM3_HASH_SIZE; + uint8_t block_digest[TEE_SM3_HASH_SIZE] = { 0 }; + uint8_t *tmp_digest = NULL; + + if (len == 0) + return TEE_ERROR_BAD_PARAMETERS; + + if (hash_size > len) + tmp_digest = block_digest; /* use a tempory buffer */ + else + tmp_digest = digest; + + sm3_final(&hc->sm3_ctx, tmp_digest); + + if (hash_size > len) + memcpy(digest, tmp_digest, len); + + return TEE_SUCCESS; +} + +static void op_sm3_hash_free_ctx(struct crypto_hash_ctx *ctx) +{ + struct sm3_hash_ctx *hc = to_hash_ctx(ctx); + + memzero_explicit(&hc->sm3_ctx, sizeof(hc->sm3_ctx)); + free(hc); +} + +static void op_sm3_hash_copy_state(struct crypto_hash_ctx *dst_ctx, + struct crypto_hash_ctx *src_ctx) +{ + struct sm3_hash_ctx *src = to_hash_ctx(src_ctx); + struct sm3_hash_ctx *dst = to_hash_ctx(dst_ctx); + + dst->sm3_ctx = src->sm3_ctx; +} + +static const struct crypto_hash_ops sm3_hash_ops = { + .init = op_sm3_hash_init, + .update = op_sm3_hash_update, + .final = op_sm3_hash_final, + .free_ctx = op_sm3_hash_free_ctx, + .copy_state = op_sm3_hash_copy_state, +}; + +TEE_Result crypto_sm3_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + struct sm3_hash_ctx *hc = NULL; + + hc = calloc(1, sizeof(*hc)); + if (!hc) + return TEE_ERROR_OUT_OF_MEMORY; + + hc->hash_ctx.ops = &sm3_hash_ops; + + *ctx = &hc->hash_ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/crypto/sm3-hmac.c b/optee_os/core/crypto/sm3-hmac.c new file mode 100644 index 0000000..abe2ac8 --- /dev/null +++ b/optee_os/core/crypto/sm3-hmac.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sm3.h" + +struct sm3_hmac_ctx { + struct crypto_mac_ctx mac_ctx; + struct sm3_context sm3_ctx; +}; + +static const struct crypto_mac_ops sm3_hmac_ops; + +static struct sm3_hmac_ctx *to_hmac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx && ctx->ops == &sm3_hmac_ops); + + return container_of(ctx, struct sm3_hmac_ctx, mac_ctx); +} + +static TEE_Result op_sm3_hmac_init(struct crypto_mac_ctx *ctx, + const uint8_t *key, size_t len) +{ + sm3_hmac_init(&to_hmac_ctx(ctx)->sm3_ctx, key, len); + + return TEE_SUCCESS; +} + +static TEE_Result op_sm3_hmac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + sm3_hmac_update(&to_hmac_ctx(ctx)->sm3_ctx, data, len); + + return TEE_SUCCESS; +} + +static TEE_Result op_sm3_hmac_final(struct crypto_mac_ctx *ctx, uint8_t *digest, + size_t len) +{ + struct sm3_hmac_ctx *c = to_hmac_ctx(ctx); + size_t hmac_size = TEE_SM3_HASH_SIZE; + uint8_t block_digest[TEE_SM3_HASH_SIZE] = { 0 }; + uint8_t *tmp_digest = NULL; + + if (len == 0) + return TEE_ERROR_BAD_PARAMETERS; + + if (hmac_size > len) + tmp_digest = block_digest; /* use a tempory buffer */ + else + tmp_digest = digest; + + sm3_hmac_final(&c->sm3_ctx, tmp_digest); + + if (hmac_size > len) + memcpy(digest, tmp_digest, len); + + return TEE_SUCCESS; +} + +static void op_sm3_hmac_free_ctx(struct crypto_mac_ctx *ctx) +{ + struct sm3_hmac_ctx *c = to_hmac_ctx(ctx); + + memzero_explicit(&c->sm3_ctx, sizeof(c->sm3_ctx)); + free(c); +} + +static void op_sm3_hmac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct sm3_hmac_ctx *src = to_hmac_ctx(src_ctx); + struct sm3_hmac_ctx *dst = to_hmac_ctx(dst_ctx); + + dst->sm3_ctx = src->sm3_ctx; +} + +static const struct crypto_mac_ops sm3_hmac_ops = { + .init = op_sm3_hmac_init, + .update = op_sm3_hmac_update, + .final = op_sm3_hmac_final, + .free_ctx = op_sm3_hmac_free_ctx, + .copy_state = op_sm3_hmac_copy_state, +}; + +TEE_Result crypto_hmac_sm3_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + struct sm3_hmac_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->mac_ctx.ops = &sm3_hmac_ops; + + *ctx = &c->mac_ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/crypto/sm3.c b/optee_os/core/crypto/sm3.c new file mode 100644 index 0000000..26bba10 --- /dev/null +++ b/optee_os/core/crypto/sm3.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ +/* + * SM3 Hash algorithm + * thanks to Xyssl + * author:goldboar + * email:goldboar@163.com + * 2011-10-26 + */ + +#include +#include +#include +#include + +#include "sm3.h" + +#define SM3_BLOCK_SIZE 64 + +#define GET_UINT32_BE(n, b, i) \ + do { \ + (n) = ((uint32_t)(b)[(i)] << 24) | \ + ((uint32_t)(b)[(i) + 1] << 16) | \ + ((uint32_t)(b)[(i) + 2] << 8) | \ + ((uint32_t)(b)[(i) + 3]); \ + } while (0) + +#define PUT_UINT32_BE(n, b, i) \ + do { \ + (b)[(i)] = (uint8_t)((n) >> 24); \ + (b)[(i) + 1] = (uint8_t)((n) >> 16); \ + (b)[(i) + 2] = (uint8_t)((n) >> 8); \ + (b)[(i) + 3] = (uint8_t)((n)); \ + } while (0) + +void sm3_init(struct sm3_context *ctx) +{ + ctx->total[0] = 0; + ctx->total[1] = 0; + + ctx->state[0] = 0x7380166F; + ctx->state[1] = 0x4914B2B9; + ctx->state[2] = 0x172442D7; + ctx->state[3] = 0xDA8A0600; + ctx->state[4] = 0xA96F30BC; + ctx->state[5] = 0x163138AA; + ctx->state[6] = 0xE38DEE4D; + ctx->state[7] = 0xB0FB0E4E; +} + +static void __maybe_unused sm3_process(struct sm3_context *ctx, + const uint8_t data[64]) +{ + uint32_t SS1, SS2, TT1, TT2, W[68], W1[64]; + uint32_t A, B, C, D, E, F, G, H; + uint32_t T[64]; + uint32_t Temp1, Temp2, Temp3, Temp4, Temp5; + int j; + + for (j = 0; j < 16; j++) + T[j] = 0x79CC4519; + for (j = 16; j < 64; j++) + T[j] = 0x7A879D8A; + + GET_UINT32_BE(W[0], data, 0); + GET_UINT32_BE(W[1], data, 4); + GET_UINT32_BE(W[2], data, 8); + GET_UINT32_BE(W[3], data, 12); + GET_UINT32_BE(W[4], data, 16); + GET_UINT32_BE(W[5], data, 20); + GET_UINT32_BE(W[6], data, 24); + GET_UINT32_BE(W[7], data, 28); + GET_UINT32_BE(W[8], data, 32); + GET_UINT32_BE(W[9], data, 36); + GET_UINT32_BE(W[10], data, 40); + GET_UINT32_BE(W[11], data, 44); + GET_UINT32_BE(W[12], data, 48); + GET_UINT32_BE(W[13], data, 52); + GET_UINT32_BE(W[14], data, 56); + GET_UINT32_BE(W[15], data, 60); + +#define FF0(x, y, z) ((x) ^ (y) ^ (z)) +#define FF1(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) + +#define GG0(x, y, z) ((x) ^ (y) ^ (z)) +#define GG1(x, y, z) (((x) & (y)) | ((~(x)) & (z))) + +#define SHL(x, n) ((x) << (n)) +#define ROTL(x, n) (SHL((x), (n) & 0x1F) | ((x) >> (32 - ((n) & 0x1F)))) + +#define P0(x) ((x) ^ ROTL((x), 9) ^ ROTL((x), 17)) +#define P1(x) ((x) ^ ROTL((x), 15) ^ ROTL((x), 23)) + + for (j = 16; j < 68; j++) { + /* + * W[j] = P1( W[j-16] ^ W[j-9] ^ ROTL(W[j-3],15)) ^ + * ROTL(W[j - 13],7 ) ^ W[j-6]; + */ + + Temp1 = W[j - 16] ^ W[j - 9]; + Temp2 = ROTL(W[j - 3], 15); + Temp3 = Temp1 ^ Temp2; + Temp4 = P1(Temp3); + Temp5 = ROTL(W[j - 13], 7) ^ W[j - 6]; + W[j] = Temp4 ^ Temp5; + } + + for (j = 0; j < 64; j++) + W1[j] = W[j] ^ W[j + 4]; + + A = ctx->state[0]; + B = ctx->state[1]; + C = ctx->state[2]; + D = ctx->state[3]; + E = ctx->state[4]; + F = ctx->state[5]; + G = ctx->state[6]; + H = ctx->state[7]; + + for (j = 0; j < 16; j++) { + SS1 = ROTL(ROTL(A, 12) + E + ROTL(T[j], j), 7); + SS2 = SS1 ^ ROTL(A, 12); + TT1 = FF0(A, B, C) + D + SS2 + W1[j]; + TT2 = GG0(E, F, G) + H + SS1 + W[j]; + D = C; + C = ROTL(B, 9); + B = A; + A = TT1; + H = G; + G = ROTL(F, 19); + F = E; + E = P0(TT2); + } + + for (j = 16; j < 64; j++) { + SS1 = ROTL(ROTL(A, 12) + E + ROTL(T[j], j), 7); + SS2 = SS1 ^ ROTL(A, 12); + TT1 = FF1(A, B, C) + D + SS2 + W1[j]; + TT2 = GG1(E, F, G) + H + SS1 + W[j]; + D = C; + C = ROTL(B, 9); + B = A; + A = TT1; + H = G; + G = ROTL(F, 19); + F = E; + E = P0(TT2); + } + + ctx->state[0] ^= A; + ctx->state[1] ^= B; + ctx->state[2] ^= C; + ctx->state[3] ^= D; + ctx->state[4] ^= E; + ctx->state[5] ^= F; + ctx->state[6] ^= G; + ctx->state[7] ^= H; +} + +static void sm3_process_blocks(struct sm3_context *ctx, const uint8_t *input, + unsigned int block_count) +{ +#ifdef CFG_CRYPTO_SM3_ARM_CE + if (block_count) + crypto_accel_sm3_compress(ctx->state, input, block_count); +#else + unsigned int n = 0; + + for (n = 0; n < block_count; n++) + sm3_process(ctx, input + n * SM3_BLOCK_SIZE); +#endif +} + +void sm3_update(struct sm3_context *ctx, const uint8_t *input, size_t ilen) +{ + unsigned int block_count = 0; + size_t fill = 0; + size_t left = 0; + + if (!ilen) + return; + + left = ctx->total[0] & 0x3F; + fill = 64 - left; + + ctx->total[0] += ilen; + + if (ctx->total[0] < ilen) + ctx->total[1]++; + + if (left && ilen >= fill) { + memcpy(ctx->buffer + left, input, fill); + sm3_process_blocks(ctx, ctx->buffer, 1); + input += fill; + ilen -= fill; + left = 0; + } + + block_count = ilen / SM3_BLOCK_SIZE; + sm3_process_blocks(ctx, input, block_count); + ilen -= block_count * SM3_BLOCK_SIZE; + input += block_count * SM3_BLOCK_SIZE; + + if (ilen > 0) + memcpy(ctx->buffer + left, input, ilen); +} + +static const uint8_t sm3_padding[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +void sm3_final(struct sm3_context *ctx, uint8_t output[32]) +{ + uint32_t last, padn; + uint32_t high, low; + uint8_t msglen[8]; + + high = (ctx->total[0] >> 29) | (ctx->total[1] << 3); + low = ctx->total[0] << 3; + + PUT_UINT32_BE(high, msglen, 0); + PUT_UINT32_BE(low, msglen, 4); + + last = ctx->total[0] & 0x3F; + padn = (last < 56) ? (56 - last) : (120 - last); + + sm3_update(ctx, sm3_padding, padn); + sm3_update(ctx, msglen, 8); + + PUT_UINT32_BE(ctx->state[0], output, 0); + PUT_UINT32_BE(ctx->state[1], output, 4); + PUT_UINT32_BE(ctx->state[2], output, 8); + PUT_UINT32_BE(ctx->state[3], output, 12); + PUT_UINT32_BE(ctx->state[4], output, 16); + PUT_UINT32_BE(ctx->state[5], output, 20); + PUT_UINT32_BE(ctx->state[6], output, 24); + PUT_UINT32_BE(ctx->state[7], output, 28); +} + +void sm3(const uint8_t *input, size_t ilen, uint8_t output[32]) +{ + struct sm3_context ctx = { }; + + sm3_init(&ctx); + sm3_update(&ctx, input, ilen); + sm3_final(&ctx, output); + + memzero_explicit(&ctx, sizeof(ctx)); +} + +void sm3_hmac_init(struct sm3_context *ctx, const uint8_t *key, size_t keylen) +{ + size_t i; + uint8_t sum[32]; + + if (keylen > 64) { + sm3(key, keylen, sum); + keylen = 32; + key = sum; + } + + memset(ctx->ipad, 0x36, 64); + memset(ctx->opad, 0x5C, 64); + + for (i = 0; i < keylen; i++) { + ctx->ipad[i] ^= key[i]; + ctx->opad[i] ^= key[i]; + } + + sm3_init(ctx); + sm3_update(ctx, ctx->ipad, 64); + + memzero_explicit(sum, sizeof(sum)); +} + +void sm3_hmac_update(struct sm3_context *ctx, const uint8_t *input, size_t ilen) +{ + sm3_update(ctx, input, ilen); +} + +void sm3_hmac_final(struct sm3_context *ctx, uint8_t output[32]) +{ + uint8_t tmpbuf[32]; + + sm3_final(ctx, tmpbuf); + sm3_init(ctx); + sm3_update(ctx, ctx->opad, 64); + sm3_update(ctx, tmpbuf, 32); + sm3_final(ctx, output); + + memzero_explicit(tmpbuf, sizeof(tmpbuf)); +} + +void sm3_hmac(const uint8_t *key, size_t keylen, const uint8_t *input, + size_t ilen, uint8_t output[32]) +{ + struct sm3_context ctx; + + sm3_hmac_init(&ctx, key, keylen); + sm3_hmac_update(&ctx, input, ilen); + sm3_hmac_final(&ctx, output); + + memzero_explicit(&ctx, sizeof(ctx)); +} diff --git a/optee_os/core/crypto/sm3.h b/optee_os/core/crypto/sm3.h new file mode 100644 index 0000000..7bf9afd --- /dev/null +++ b/optee_os/core/crypto/sm3.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ +/** + * \file sm3.h + * thanks to Xyssl + * author:goldboar + * email:goldboar@163.com + * 2011-10-26 + */ +#ifndef CORE_CRYPTO_SM3_H +#define CORE_CRYPTO_SM3_H + +#include +#include + +struct sm3_context { + uint32_t total[2]; /* number of bytes processed */ + uint32_t state[8]; /* intermediate digest state */ + uint8_t buffer[64]; /* data block being processed */ + uint8_t ipad[64]; /* HMAC: inner padding */ + uint8_t opad[64]; /* HMAC: outer padding */ +}; + +void sm3_init(struct sm3_context *ctx); +void sm3_update(struct sm3_context *ctx, const uint8_t *input, size_t ilen); +void sm3_final(struct sm3_context *ctx, uint8_t output[32]); +void sm3(const uint8_t *input, size_t ilen, uint8_t output[32]); + +void sm3_hmac_init(struct sm3_context *ctx, const uint8_t *key, size_t keylen); +void sm3_hmac_update(struct sm3_context *ctx, const uint8_t *input, + size_t ilen); +void sm3_hmac_final(struct sm3_context *ctx, uint8_t output[32]); +void sm3_hmac(const uint8_t *key, size_t keylen, const uint8_t *input, + size_t ilen, uint8_t output[32]); + +#endif /* CORE_CRYPTO_SM3_H */ diff --git a/optee_os/core/crypto/sm4-cbc.c b/optee_os/core/crypto/sm4-cbc.c new file mode 100644 index 0000000..4b96a47 --- /dev/null +++ b/optee_os/core/crypto/sm4-cbc.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sm4.h" + +struct sm4_cbc_ctx { + struct crypto_cipher_ctx ctx; + struct sm4_context state; + uint8_t iv[16]; +}; + +static const struct crypto_cipher_ops sm4_cbc_ops; + +static struct sm4_cbc_ctx *to_sm4_cbc_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &sm4_cbc_ops); + + return container_of(ctx, struct sm4_cbc_ctx, ctx); +} + +static TEE_Result sm4_cbc_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv, size_t iv_len) +{ + struct sm4_cbc_ctx *c = to_sm4_cbc_ctx(ctx); + + if (key1_len != 16 || iv_len != sizeof(c->iv)) + return TEE_ERROR_BAD_PARAMETERS; + + if (mode == TEE_MODE_ENCRYPT) + sm4_setkey_enc(&c->state, key1); + else + sm4_setkey_dec(&c->state, key1); + + memcpy(c->iv, iv, sizeof(c->iv)); + + return TEE_SUCCESS; +} + +static TEE_Result sm4_cbc_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct sm4_cbc_ctx *c = to_sm4_cbc_ctx(ctx); + + sm4_crypt_cbc(&c->state, len, c->iv, data, dst); + + return TEE_SUCCESS; +} + +static void sm4_cbc_final(struct crypto_cipher_ctx *ctx) +{ + struct sm4_cbc_ctx *c = to_sm4_cbc_ctx(ctx); + + memzero_explicit(&c->state, sizeof(c->state)); + memzero_explicit(&c->iv, sizeof(c->iv)); +} + +static void sm4_cbc_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_sm4_cbc_ctx(ctx)); +} + +static void sm4_cbc_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct sm4_cbc_ctx *src = to_sm4_cbc_ctx(src_ctx); + struct sm4_cbc_ctx *dst = to_sm4_cbc_ctx(dst_ctx); + + dst->state = src->state; + memcpy(dst->iv, src->iv, sizeof(src->iv)); +} + +static const struct crypto_cipher_ops sm4_cbc_ops = { + .init = sm4_cbc_init, + .update = sm4_cbc_update, + .final = sm4_cbc_final, + .free_ctx = sm4_cbc_free_ctx, + .copy_state = sm4_cbc_copy_state, +}; + +TEE_Result crypto_sm4_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct sm4_cbc_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &sm4_cbc_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/crypto/sm4-ctr.c b/optee_os/core/crypto/sm4-ctr.c new file mode 100644 index 0000000..0b17400 --- /dev/null +++ b/optee_os/core/crypto/sm4-ctr.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sm4.h" + +struct sm4_ctr_ctx { + struct crypto_cipher_ctx ctx; + struct sm4_context state; + uint8_t ctr[16]; +}; + +static const struct crypto_cipher_ops sm4_ctr_ops; + +static struct sm4_ctr_ctx *to_sm4_ctr_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &sm4_ctr_ops); + + return container_of(ctx, struct sm4_ctr_ctx, ctx); +} + +static TEE_Result sm4_ctr_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode __unused, + const uint8_t *key1, size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv, size_t iv_len) +{ + struct sm4_ctr_ctx *c = to_sm4_ctr_ctx(ctx); + + if (key1_len != 16 || iv_len != sizeof(c->ctr)) + return TEE_ERROR_BAD_PARAMETERS; + + sm4_setkey_enc(&c->state, key1); + memcpy(c->ctr, iv, sizeof(c->ctr)); + + return TEE_SUCCESS; +} + +static TEE_Result sm4_ctr_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct sm4_ctr_ctx *c = to_sm4_ctr_ctx(ctx); + + sm4_crypt_ctr(&c->state, len, c->ctr, data, dst); + + return TEE_SUCCESS; +} + +static void sm4_ctr_final(struct crypto_cipher_ctx *ctx) +{ + struct sm4_ctr_ctx *c = to_sm4_ctr_ctx(ctx); + + memzero_explicit(&c->state, sizeof(c->state)); + memzero_explicit(&c->ctr, sizeof(c->ctr)); +} + +static void sm4_ctr_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_sm4_ctr_ctx(ctx)); +} + +static void sm4_ctr_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct sm4_ctr_ctx *src = to_sm4_ctr_ctx(src_ctx); + struct sm4_ctr_ctx *dst = to_sm4_ctr_ctx(dst_ctx); + + dst->state = src->state; + memcpy(dst->ctr, src->ctr, sizeof(src->ctr)); +} + +static const struct crypto_cipher_ops sm4_ctr_ops = { + .init = sm4_ctr_init, + .update = sm4_ctr_update, + .final = sm4_ctr_final, + .free_ctx = sm4_ctr_free_ctx, + .copy_state = sm4_ctr_copy_state, +}; + +TEE_Result crypto_sm4_ctr_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct sm4_ctr_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &sm4_ctr_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/crypto/sm4-ecb.c b/optee_os/core/crypto/sm4-ecb.c new file mode 100644 index 0000000..2cd394d --- /dev/null +++ b/optee_os/core/crypto/sm4-ecb.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "sm4.h" + +struct sm4_ecb_ctx { + struct crypto_cipher_ctx ctx; + struct sm4_context state; +}; + +static const struct crypto_cipher_ops sm4_ecb_ops; + +static struct sm4_ecb_ctx *to_sm4_ecb_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &sm4_ecb_ops); + + return container_of(ctx, struct sm4_ecb_ctx, ctx); +} + +static TEE_Result sm4_ecb_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct sm4_ecb_ctx *c = to_sm4_ecb_ctx(ctx); + + if (key1_len != 16) + return TEE_ERROR_BAD_STATE; + + if (mode == TEE_MODE_ENCRYPT) + sm4_setkey_enc(&c->state, key1); + else + sm4_setkey_dec(&c->state, key1); + + return TEE_SUCCESS; +} + +static TEE_Result sm4_ecb_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct sm4_ecb_ctx *c = to_sm4_ecb_ctx(ctx); + + sm4_crypt_ecb(&c->state, len, data, dst); + + return TEE_SUCCESS; +} + +static void sm4_ecb_final(struct crypto_cipher_ctx *ctx) +{ + struct sm4_ecb_ctx *c = to_sm4_ecb_ctx(ctx); + + memzero_explicit(&c->state, sizeof(c->state)); +} + +static void sm4_ecb_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_sm4_ecb_ctx(ctx)); +} + +static void sm4_ecb_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct sm4_ecb_ctx *src = to_sm4_ecb_ctx(src_ctx); + struct sm4_ecb_ctx *dst = to_sm4_ecb_ctx(dst_ctx); + + dst->state = src->state; +} + +static const struct crypto_cipher_ops sm4_ecb_ops = { + .init = sm4_ecb_init, + .update = sm4_ecb_update, + .final = sm4_ecb_final, + .free_ctx = sm4_ecb_free_ctx, + .copy_state = sm4_ecb_copy_state, +}; + +TEE_Result crypto_sm4_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct sm4_ecb_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &sm4_ecb_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/crypto/sm4-xts.c b/optee_os/core/crypto/sm4-xts.c new file mode 100644 index 0000000..a46c150 --- /dev/null +++ b/optee_os/core/crypto/sm4-xts.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "sm4.h" + +struct sm4_xts_ctx { + struct crypto_cipher_ctx ctx; + struct sm4_context state; + struct sm4_context state_ek; + struct sm4_context state_dk; + uint8_t iv[16]; +}; + +static const struct crypto_cipher_ops sm4_xts_ops; + +static struct sm4_xts_ctx *to_sm4_xts_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &sm4_xts_ops); + + return container_of(ctx, struct sm4_xts_ctx, ctx); +} + +static TEE_Result sm4_xts_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2, + size_t key2_len, const uint8_t *iv, + size_t iv_len) +{ + struct sm4_xts_ctx *c = to_sm4_xts_ctx(ctx); + + if (key1_len != 16 || key2_len != 16 || iv_len != sizeof(c->iv)) + return TEE_ERROR_BAD_STATE; + + if (iv) + memcpy(c->iv, iv, sizeof(c->iv)); + + if (mode == TEE_MODE_ENCRYPT) + sm4_setkey_enc(&c->state, key1); + else + sm4_setkey_dec(&c->state, key1); + + sm4_setkey_enc(&c->state_ek, key2); + sm4_setkey_dec(&c->state_dk, key2); + + return TEE_SUCCESS; +} + +static TEE_Result sm4_xts_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, const uint8_t *data, + size_t len, uint8_t *dst) +{ + struct sm4_xts_ctx *c = to_sm4_xts_ctx(ctx); + + sm4_crypt_xts(&c->state, &c->state_ek, &c->state_dk, + len, c->iv, data, dst); + + return TEE_SUCCESS; +} + +static void sm4_xts_final(struct crypto_cipher_ctx *ctx) +{ + struct sm4_xts_ctx *c = to_sm4_xts_ctx(ctx); + + memzero_explicit(&c->state, sizeof(c->state)); + memzero_explicit(&c->state_ek, sizeof(c->state_ek)); + memzero_explicit(&c->state_dk, sizeof(c->state_dk)); + memzero_explicit(&c->iv, sizeof(c->iv)); +} + +static void sm4_xts_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_sm4_xts_ctx(ctx)); +} + +static void sm4_xts_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct sm4_xts_ctx *src = to_sm4_xts_ctx(src_ctx); + struct sm4_xts_ctx *dst = to_sm4_xts_ctx(dst_ctx); + + dst->state = src->state; + dst->state_ek = src->state_ek; + dst->state_dk = src->state_dk; + memcpy(dst->iv, src->iv, sizeof(src->iv)); +} + +static const struct crypto_cipher_ops sm4_xts_ops = { + .init = sm4_xts_init, + .update = sm4_xts_update, + .final = sm4_xts_final, + .free_ctx = sm4_xts_free_ctx, + .copy_state = sm4_xts_copy_state, +}; + +TEE_Result crypto_sm4_xts_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct sm4_xts_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &sm4_xts_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/crypto/sm4.c b/optee_os/core/crypto/sm4.c new file mode 100644 index 0000000..8f1dd0d --- /dev/null +++ b/optee_os/core/crypto/sm4.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright Copyright (c) 2019 Huawei Technologies Co., Ltd + */ +/* + * SM4 Encryption algorithm (SMS4 algorithm) + * GM/T 0002-2012 Chinese National Standard ref:http://www.oscca.gov.cn/ + * thanks to Xyssl + * thnaks and refers to http://hi.baidu.com/numax/blog/item/80addfefddfb93e4cf1b3e61.html + * author:goldboar + * email:goldboar@163.com + * 2012-4-20 + */ + +#include "sm4.h" +#include +#include + +#define GET_UINT32_BE(n, b, i) \ + do { \ + (n) = ((uint32_t)(b)[(i)] << 24) | \ + ((uint32_t)(b)[(i) + 1] << 16) | \ + ((uint32_t)(b)[(i) + 2] << 8) | \ + ((uint32_t)(b)[(i) + 3]); \ + } while (0) + +#define PUT_UINT32_BE(n, b, i) \ + do { \ + (b)[(i)] = (uint8_t)((n) >> 24); \ + (b)[(i) + 1] = (uint8_t)((n) >> 16); \ + (b)[(i) + 2] = (uint8_t)((n) >> 8); \ + (b)[(i) + 3] = (uint8_t)((n)); \ + } while (0) + +#define SHL(x, n) (((x) & 0xFFFFFFFF) << (n)) +#define ROTL(x, n) (SHL((x), (n)) | ((x) >> (32 - (n)))) + +#define SWAP(a, b) { uint32_t t = a; a = b; b = t; t = 0; } + +/* + * Expanded SM4 S-boxes + */ +static const uint8_t SboxTable[16][16] = { + {0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, + 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05}, + {0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, + 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99}, + {0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, + 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62}, + {0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, + 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6}, + {0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, + 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8}, + {0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, + 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35}, + {0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, + 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87}, + {0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, + 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e}, + {0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, + 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1}, + {0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, + 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3}, + {0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, + 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f}, + {0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, + 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51}, + {0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, + 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8}, + {0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, + 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0}, + {0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, + 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84}, + {0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, + 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48} +}; + +/* System parameter */ +static const uint32_t FK[4] = { + 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc +}; + +/* Fixed parameter */ +static const uint32_t CK[32] = { + 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269, + 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9, + 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249, + 0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9, + 0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229, + 0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299, + 0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209, + 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279 +}; + +static uint8_t sm4Sbox(uint8_t inch) +{ + uint8_t *tab = (uint8_t *)SboxTable; + + return tab[inch]; +} + +static uint32_t sm4Lt(uint32_t ka) +{ + uint32_t bb = 0; + uint8_t a[4]; + uint8_t b[4]; + + PUT_UINT32_BE(ka, a, 0); + b[0] = sm4Sbox(a[0]); + b[1] = sm4Sbox(a[1]); + b[2] = sm4Sbox(a[2]); + b[3] = sm4Sbox(a[3]); + GET_UINT32_BE(bb, b, 0); + + return bb ^ ROTL(bb, 2) ^ ROTL(bb, 10) ^ ROTL(bb, 18) ^ ROTL(bb, 24); +} + +static uint32_t sm4F(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t rk) +{ + return x0 ^ sm4Lt(x1 ^ x2 ^ x3 ^ rk); +} + +static uint32_t sm4CalciRK(uint32_t ka) +{ + uint32_t bb = 0; + uint8_t a[4]; + uint8_t b[4]; + + PUT_UINT32_BE(ka, a, 0); + b[0] = sm4Sbox(a[0]); + b[1] = sm4Sbox(a[1]); + b[2] = sm4Sbox(a[2]); + b[3] = sm4Sbox(a[3]); + GET_UINT32_BE(bb, b, 0); + + return bb ^ ROTL(bb, 13) ^ ROTL(bb, 23); +} + +static void sm4_setkey(uint32_t SK[32], const uint8_t key[16]) +{ + uint32_t MK[4]; + uint32_t k[36]; + uint32_t i = 0; + + GET_UINT32_BE(MK[0], key, 0); + GET_UINT32_BE(MK[1], key, 4); + GET_UINT32_BE(MK[2], key, 8); + GET_UINT32_BE(MK[3], key, 12); + + k[0] = MK[0] ^ FK[0]; + k[1] = MK[1] ^ FK[1]; + k[2] = MK[2] ^ FK[2]; + k[3] = MK[3] ^ FK[3]; + + for (i = 0; i < 32; i++) { + k[i + 4] = k[i] ^ sm4CalciRK(k[i + 1] ^ k[i + 2] ^ k[i + 3] ^ + CK[i]); + SK[i] = k[i + 4]; + } +} + +static void sm4_one_round(uint32_t sk[32], const uint8_t input[16], + uint8_t output[16]) +{ + uint32_t i = 0; + uint32_t ulbuf[36]; + + memset(ulbuf, 0, sizeof(ulbuf)); + + GET_UINT32_BE(ulbuf[0], input, 0); + GET_UINT32_BE(ulbuf[1], input, 4); + GET_UINT32_BE(ulbuf[2], input, 8); + GET_UINT32_BE(ulbuf[3], input, 12); + + for (i = 0; i < 32; i++) + ulbuf[i + 4] = sm4F(ulbuf[i], ulbuf[i + 1], ulbuf[i + 2], + ulbuf[i + 3], sk[i]); + + PUT_UINT32_BE(ulbuf[35], output, 0); + PUT_UINT32_BE(ulbuf[34], output, 4); + PUT_UINT32_BE(ulbuf[33], output, 8); + PUT_UINT32_BE(ulbuf[32], output, 12); +} + +void sm4_setkey_enc(struct sm4_context *ctx, const uint8_t key[16]) +{ + ctx->mode = SM4_ENCRYPT; + sm4_setkey(ctx->sk, key); +} + +void sm4_setkey_dec(struct sm4_context *ctx, const uint8_t key[16]) +{ + int i; + + ctx->mode = SM4_DECRYPT; + sm4_setkey(ctx->sk, key); + + for (i = 0; i < 16; i++) + SWAP(ctx->sk[i], ctx->sk[31 - i]); +} + +void sm4_crypt_ecb(struct sm4_context *ctx, size_t length, const uint8_t *input, + uint8_t *output) +{ + assert(!(length % 16)); + + while (length > 0) { + sm4_one_round(ctx->sk, input, output); + input += 16; + output += 16; + length -= 16; + } +} + +void sm4_crypt_cbc(struct sm4_context *ctx, size_t length, uint8_t iv[16], + const uint8_t *input, uint8_t *output) +{ + int i; + uint8_t temp[16]; + + assert(!(length % 16)); + + if (ctx->mode == SM4_ENCRYPT) { + while (length > 0) { + for (i = 0; i < 16; i++) + output[i] = (uint8_t)(input[i] ^ iv[i]); + sm4_one_round(ctx->sk, output, output); + memcpy(iv, output, 16); + input += 16; + output += 16; + length -= 16; + } + } else { + /* SM4_DECRYPT */ + while (length > 0) { + memcpy(temp, input, 16); + sm4_one_round(ctx->sk, input, output); + for (i = 0; i < 16; i++) + output[i] = (uint8_t)(output[i] ^ iv[i]); + memcpy(iv, temp, 16); + input += 16; + output += 16; + length -= 16; + } + } +} + +void sm4_crypt_ctr(struct sm4_context *ctx, size_t length, uint8_t ctr[16], + const uint8_t *input, uint8_t *output) +{ + int i; + uint8_t temp[16]; + + assert(!(length % 16)); + + while (length > 0) { + memcpy(temp, ctr, 16); + sm4_one_round(ctx->sk, ctr, ctr); + for (i = 0; i < 16; i++) + output[i] = (uint8_t)(input[i] ^ ctr[i]); + memcpy(ctr, temp, 16); + for (i = 16; i > 0; i--) + if (++ctr[i - 1]) + break; + input += 16; + output += 16; + length -= 16; + } +} + +static void xts_multi(unsigned char *in, unsigned char *out) +{ + uint8_t tt = 0; + uint8_t t = 0; + int i = 0; + + for (i = 0; i < 16; i++) { + tt = in[i] >> 7; + out[i] = ((in[i] << 1) | t) & 0xFF; + t = tt; + } + + out[0] ^= (0x87 & (0 - tt)); +} + +static void xor_128(const uint8_t a[16], const uint8_t b[16], uint8_t c[16]) +{ + int i = 0; + + for (i = 0; i < 16; i++) + c[i] = a[i] ^ b[i]; +} + +void sm4_crypt_xts(struct sm4_context *ctx, struct sm4_context *ctx_ek, + struct sm4_context *ctx_dk, size_t len, uint8_t *iv, + const uint8_t *input, uint8_t *output) +{ + uint8_t tweak[16] = { }; + uint8_t tweak1[16] = { }; + uint8_t ct[16] = { }; + size_t i = 0; + + assert(len >= 16); + + sm4_one_round(ctx_ek->sk, iv, tweak); + + if (ctx->mode == SM4_DECRYPT && (len % 16)) + len -= 16; + + while (len >= 16) { + xor_128(input, tweak, ct); + sm4_one_round(ctx->sk, ct, ct); + xor_128(ct, tweak, output); + + xts_multi(tweak, tweak); + len -= 16; + if (len == 0) { + sm4_one_round(ctx_dk->sk, tweak, iv); + return; + } + input += 16; + output += 16; + } + + if (ctx->mode == SM4_ENCRYPT) { + memcpy(ct, output - 16, 16); + for (i = 0; i < len; i++) { + output[i] = ct[i]; + ct[i] = input[i]; + } + + xor_128(ct, tweak, ct); + sm4_one_round(ctx->sk, ct, ct); + xor_128(ct, tweak, ct); + memcpy(output - 16, ct, 16); + } else { + xts_multi(tweak, tweak1); + xor_128(input, tweak1, ct); + sm4_one_round(ctx->sk, ct, ct); + xor_128(ct, tweak1, ct); + + for (i = 0; i < len; ++i) { + output[16 + i] = ct[i]; + ct[i] = input[16 + i]; + } + xor_128(ct, tweak, ct); + sm4_one_round(ctx->sk, ct, ct); + xor_128(ct, tweak, output); + } +} diff --git a/optee_os/core/crypto/sm4.h b/optee_os/core/crypto/sm4.h new file mode 100644 index 0000000..21d075a --- /dev/null +++ b/optee_os/core/crypto/sm4.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ +#ifndef CORE_CRYPTO_SM4_H +#define CORE_CRYPTO_SM4_H + +#include +#include + +#define SM4_ENCRYPT 1 +#define SM4_DECRYPT 0 + +struct sm4_context { + int mode; /* SM4_ENCRYPT/SM4_DECRYPT */ + uint32_t sk[32]; /* SM4 subkeys */ +}; + +void sm4_setkey_enc(struct sm4_context *ctx, const uint8_t key[16]); +void sm4_setkey_dec(struct sm4_context *ctx, const uint8_t key[16]); +void sm4_crypt_ecb(struct sm4_context *ctx, size_t length, const uint8_t *input, + uint8_t *output); +void sm4_crypt_cbc(struct sm4_context *ctx, size_t length, uint8_t iv[16], + const uint8_t *input, uint8_t *output); +void sm4_crypt_ctr(struct sm4_context *ctx, size_t length, uint8_t ctr[16], + const uint8_t *input, uint8_t *output); +void sm4_crypt_xts(struct sm4_context *ctx, struct sm4_context *ctx_ek, + struct sm4_context *ctx_dk, size_t length, uint8_t *iv, + const uint8_t *input, uint8_t *output); + +#endif /* CORE_CRYPTO_SM4_H */ diff --git a/optee_os/core/crypto/sm4_accel.c b/optee_os/core/crypto/sm4_accel.c new file mode 100644 index 0000000..4c3d9f7 --- /dev/null +++ b/optee_os/core/crypto/sm4_accel.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) Hisilicon Technologies Co., Ltd. 2023. All rights reserved. + * + * SM4 optimization for ARMv8 + */ + +#include "sm4.h" +#include +#include +#include + +void sm4_setkey_enc(struct sm4_context *ctx, const uint8_t key[16]) +{ + ctx->mode = SM4_ENCRYPT; + crypto_accel_sm4_setkey_enc(ctx->sk, key); +} + +void sm4_setkey_dec(struct sm4_context *ctx, const uint8_t key[16]) +{ + ctx->mode = SM4_DECRYPT; + crypto_accel_sm4_setkey_dec(ctx->sk, key); +} + +void sm4_crypt_ecb(struct sm4_context *ctx, size_t length, const uint8_t *input, + uint8_t *output) +{ + assert(!(length % 16)); + + crypto_accel_sm4_ecb_enc(output, input, ctx->sk, length); +} + +void sm4_crypt_cbc(struct sm4_context *ctx, size_t length, uint8_t iv[16], + const uint8_t *input, uint8_t *output) +{ + assert(!(length % 16)); + + if (ctx->mode == SM4_ENCRYPT) + crypto_accel_sm4_cbc_enc(output, input, ctx->sk, length, iv); + else + /* SM4_DECRYPT */ + crypto_accel_sm4_cbc_dec(output, input, ctx->sk, length, iv); +} + +void sm4_crypt_ctr(struct sm4_context *ctx, size_t length, uint8_t ctr[16], + const uint8_t *input, uint8_t *output) +{ + assert(!(length % 16)); + + crypto_accel_sm4_ctr_enc(output, input, ctx->sk, length, ctr); +} + +void sm4_crypt_xts(struct sm4_context *ctx, struct sm4_context *ctx_ek, + struct sm4_context *ctx_dk __unused, size_t len, uint8_t *iv, + const uint8_t *input, uint8_t *output) +{ + assert(len >= 16); + + if (ctx->mode == SM4_ENCRYPT) + crypto_accel_sm4_xts_enc(output, input, ctx->sk, ctx_ek->sk, + len, iv); + else + crypto_accel_sm4_xts_dec(output, input, ctx->sk, ctx_ek->sk, + len, iv); +} diff --git a/optee_os/core/crypto/sub.mk b/optee_os/core/crypto/sub.mk new file mode 100644 index 0000000..223e5d6 --- /dev/null +++ b/optee_os/core/crypto/sub.mk @@ -0,0 +1,45 @@ +srcs-y += crypto.c + +ifeq (y-y,$(CFG_CRYPTO_AES)-$(CFG_CRYPTO_GCM)) +srcs-y += aes-gcm.c +ifneq ($(CFG_CRYPTO_WITH_CE),y) +srcs-y += aes-gcm-sw.c +ifeq ($(CFG_AES_GCM_TABLE_BASED),y) +srcs-y += aes-gcm-ghash-tbl.c +endif +endif +endif + +srcs-$(CFG_WITH_USER_TA) += signed_hdr.c + +ifeq ($(CFG_WITH_SOFTWARE_PRNG),y) +srcs-y += rng_fortuna.c +else +srcs-y += rng_hw.c +endif + +ifneq ($(CFG_CRYPTO_CBC_MAC_FROM_CRYPTOLIB),y) +srcs-$(CFG_CRYPTO_CBC_MAC) += cbc-mac.c +endif +ifneq ($(CFG_CRYPTO_CTS_FROM_CRYPTOLIB),y) +srcs-$(CFG_CRYPTO_CTS) += aes-cts.c +endif +ifneq (,$(filter y,$(CFG_CRYPTO_SM2_PKE) $(CFG_CRYPTO_SM2_KEP))) +srcs-y += sm2-kdf.c +endif +ifeq ($(CFG_CRYPTO_SM3),y) +srcs-y += sm3.c +srcs-y += sm3-hash.c +srcs-$(CFG_CRYPTO_HMAC) += sm3-hmac.c +endif +ifeq ($(CFG_CRYPTO_SM4),y) +ifeq ($(CFG_CORE_CRYPTO_SM4_ACCEL)-$(CFG_ARM64_core),y-y) +srcs-$(CFG_ARM64_core) += sm4_accel.c +else +srcs-y += sm4.c +endif +srcs-$(CFG_CRYPTO_ECB) += sm4-ecb.c +srcs-$(CFG_CRYPTO_CBC) += sm4-cbc.c +srcs-$(CFG_CRYPTO_CTR) += sm4-ctr.c +srcs-$(CFG_CRYPTO_XTS) += sm4-xts.c +endif diff --git a/optee_os/core/drivers/amlogic_uart.c b/optee_os/core/drivers/amlogic_uart.c new file mode 100644 index 0000000..70dfcf4 --- /dev/null +++ b/optee_os/core/drivers/amlogic_uart.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020 Carlo Caione + */ + +#include +#include +#include +#include +#include + +/* Registers */ +#define AML_UART_WFIFO 0x0000 +#define AML_UART_RFIFO 0x0004 +#define AML_UART_CONTROL 0x0008 +#define AML_UART_STATUS 0x000C +#define AML_UART_MISC 0x0010 +#define AML_UART_SIZE 0x0014 + +/* AML_UART_STATUS bits */ +#define AML_UART_RX_EMPTY BIT(20) +#define AML_UART_TX_FULL BIT(21) +#define AML_UART_TX_EMPTY BIT(22) + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct amlogic_uart_data *pd = + container_of(chip, struct amlogic_uart_data, chip); + + return io_pa_or_va(&pd->base, AML_UART_SIZE); +} + +static void amlogic_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + AML_UART_STATUS) & AML_UART_TX_EMPTY)) + ; +} + +static int amlogic_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + if (io_read32(base + AML_UART_STATUS) & AML_UART_RX_EMPTY) + return -1; + + return io_read32(base + AML_UART_RFIFO) & 0xff; +} + +static void amlogic_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + while (io_read32(base + AML_UART_STATUS) & AML_UART_TX_FULL) + ; + + io_write32(base + AML_UART_WFIFO, ch); +} + +static const struct serial_ops amlogic_uart_ops = { + .flush = amlogic_uart_flush, + .getchar = amlogic_uart_getchar, + .putc = amlogic_uart_putc, +}; + +void amlogic_uart_init(struct amlogic_uart_data *pd, paddr_t base) +{ + pd->base.pa = base; + pd->chip.ops = &amlogic_uart_ops; + + /* + * Do nothing, debug uart (AO) shared with normal world, everything for + * uart initialization is done in bootloader. + */ +} diff --git a/optee_os/core/drivers/atmel_piobu.c b/optee_os/core/drivers/atmel_piobu.c new file mode 100644 index 0000000..1f6aa11 --- /dev/null +++ b/optee_os/core/drivers/atmel_piobu.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microchip + * + * Driver for AT91 PIOBU + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SECUMOD_MAX_PINS 8 +#define SECUMOD_PIN_MASK (BIT(SECUMOD_MAX_PINS) - 1) +#define SECUMOD_PIN_SHIFT 16 +#define SECUMOD_PIN_VAL(pin) BIT(SECUMOD_PIN_SHIFT + (pin)) + +#define DT_GPIO_CELLS 2 + +#define SECUMOD_CR 0x0 +#define SECUMOD_CR_KEY_SHIFT 16 +#define SECUMOD_CR_KEY SHIFT_U32(0x89CA, SECUMOD_CR_KEY_SHIFT) +#define SECUMOD_CR_BACKUP BIT(0) +#define SECUMOD_CR_NORMAL BIT(1) + +#define SECUMOD_SR 0x8 +#define SECUMOD_SR_JTAG BIT(3) +#define SECUMOD_SR_TST_PIN BIT(2) + +#define SECUMOD_SCR 0x10 + +#define SECUMOD_PIOBU(x) (0x18 + (x) * 0x4) +#define SECUMOD_PIOBU_AFV_MASK GENMASK_32(3, 0) +#define SECUMOD_PIOBU_RFV_SHIFT 4 +#define SECUMOD_PIOBU_OUTPUT BIT(8) +#define SECUMOD_PIOBU_SOD BIT(9) +#define SECUMOD_PIOBU_PDS BIT(10) +#define SECUMOD_PIOBU_PULLUP_SHIFT 12 +#define SECUMOD_PIOBU_SWITCH_SHIFT 15 + +#define SECUMOD_JTAGCR 0x68 +#define SECUMOD_JTAGCR_FNTRST 0x1 + +#define SECUMOD_BMPR 0x7C +#define SECUMOD_NMPR 0x80 +#define SECUMOD_NIEPR 0x84 +#define SECUMOD_NIDPR 0x88 +#define SECUMOD_NIMPR 0x8C +#define SECUMOD_WKPR 0x90 + +static vaddr_t secumod_base; +static uint32_t gpio_protected; +static struct gpio_chip secumod_chip; + +/* + * Get value from GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin from which value needs to be read + * Return target GPIO pin level. + */ +static enum gpio_level secumod_gpio_get_value(struct gpio_chip *chip __unused, + unsigned int gpio_pin) +{ + vaddr_t piobu_addr = 0; + uint32_t piobu = 0; + + assert(gpio_pin < SECUMOD_MAX_PINS && + !(gpio_protected & BIT32(gpio_pin))); + + piobu_addr = secumod_base + SECUMOD_PIOBU(gpio_pin); + piobu = io_read32(piobu_addr); + + if (piobu & SECUMOD_PIOBU_PDS) + return GPIO_LEVEL_HIGH; + else + return GPIO_LEVEL_LOW; +} + +/* + * Set value for GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin to which value needs to be written + * value: Level state for the target pin + */ +static void secumod_gpio_set_value(struct gpio_chip *chip __unused, + unsigned int gpio_pin, enum gpio_level value) +{ + vaddr_t piobu_addr = 0; + + assert(gpio_pin < SECUMOD_MAX_PINS && + !(gpio_protected & BIT32(gpio_pin))); + + piobu_addr = secumod_base + SECUMOD_PIOBU(gpio_pin); + + if (value == GPIO_LEVEL_HIGH) + io_setbits32(piobu_addr, SECUMOD_PIOBU_SOD); + else + io_clrbits32(piobu_addr, SECUMOD_PIOBU_SOD); +} + +/* + * Get direction from GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin from which direction needs to be read + */ +static enum gpio_dir secumod_gpio_get_direction(struct gpio_chip *chip __unused, + unsigned int gpio_pin) +{ + vaddr_t piobu_addr = 0; + uint32_t piobu = 0; + + assert(gpio_pin < SECUMOD_MAX_PINS && + !(gpio_protected & BIT32(gpio_pin))); + + piobu_addr = secumod_base + SECUMOD_PIOBU(gpio_pin); + piobu = io_read32(piobu_addr); + + if (piobu & SECUMOD_PIOBU_OUTPUT) + return GPIO_DIR_OUT; + else + return GPIO_DIR_IN; +} + +/* + * Set direction for GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin on which direction needs to be set + * direction: direction which needs to be set on pin + */ +static void secumod_gpio_set_direction(struct gpio_chip *chip __unused, + unsigned int gpio_pin, + enum gpio_dir direction) +{ + vaddr_t piobu_addr = 0; + + assert(gpio_pin < SECUMOD_MAX_PINS && + !(gpio_protected & BIT32(gpio_pin))); + + piobu_addr = secumod_base + SECUMOD_PIOBU(gpio_pin); + + if (direction == GPIO_DIR_OUT) + io_setbits32(piobu_addr, SECUMOD_PIOBU_OUTPUT); + else + io_clrbits32(piobu_addr, SECUMOD_PIOBU_OUTPUT); +} + +/* + * Get interrupt from GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin from which interrupt value needs to be read + */ +static enum gpio_interrupt +secumod_gpio_get_interrupt(struct gpio_chip *chip __unused, + unsigned int gpio_pin) +{ + vaddr_t nimpr_addr = secumod_base + SECUMOD_NIMPR; + uint32_t data = 0; + + assert(gpio_pin < SECUMOD_MAX_PINS && + !(gpio_protected & BIT32(gpio_pin))); + + data = io_read32(nimpr_addr); + + if (data & SECUMOD_PIN_VAL(gpio_pin)) + return GPIO_INTERRUPT_ENABLE; + else + return GPIO_INTERRUPT_DISABLE; +} + +/* + * Set interrupt event for GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin on which interrupt value needs to be set + * interrupt: interrupt value which needs to be set on pin + */ +static void secumod_gpio_set_interrupt(struct gpio_chip *chip __unused, + unsigned int gpio_pin, + enum gpio_interrupt interrupt) +{ + vaddr_t niepr_addr = secumod_base + SECUMOD_NIEPR; + + assert(gpio_pin < SECUMOD_MAX_PINS && + !(gpio_protected & BIT32(gpio_pin))); + + if (interrupt == GPIO_INTERRUPT_ENABLE) + io_setbits32(niepr_addr, SECUMOD_PIN_VAL(gpio_pin)); + else + io_clrbits32(niepr_addr, SECUMOD_PIN_VAL(gpio_pin)); +} + +static const struct gpio_ops atmel_piobu_ops = { + .get_direction = secumod_gpio_get_direction, + .set_direction = secumod_gpio_set_direction, + .get_value = secumod_gpio_get_value, + .set_value = secumod_gpio_set_value, + .get_interrupt = secumod_gpio_get_interrupt, + .set_interrupt = secumod_gpio_set_interrupt, +}; + +static TEE_Result secumod_dt_get(struct dt_pargs *pargs, void *data, + struct gpio **out_gpio) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct gpio *gpio = NULL; + struct gpio_chip *chip = data; + + res = gpio_dt_alloc_pin(pargs, &gpio); + if (res) + return res; + + if (gpio_protected & BIT32(gpio->pin)) { + free(gpio); + return TEE_ERROR_GENERIC; + } + + gpio->chip = chip; + *out_gpio = gpio; + + return TEE_SUCCESS; +} + +static enum itr_return secumod_it_handler(struct itr_handler *handler __unused) +{ + int i = 0; + struct optee_rtc_time tm = { }; + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t sr = io_read32(secumod_base + SECUMOD_SR); + + for (i = 0; i < SECUMOD_MAX_PINS; i++) { + if (sr & SECUMOD_PIN_VAL(i)) + EMSG("Detected tampering on pin %d", i); + } + + if (sr & SECUMOD_SR_JTAG) + EMSG("JTAG tampering attempt"); + + if (sr & SECUMOD_SR_TST_PIN) + EMSG("Test pin tampering attempt"); + + res = atmel_rtc_get_tamper_timestamp(&tm); + if (!res) { + EMSG("Date of tampering: %02"PRIu32"/%02"PRIu32"/%02"PRIu32"", + tm.tm_mday, tm.tm_mon, tm.tm_year); + EMSG("Time of tampering: %02"PRIu32":%02"PRIu32":%02"PRIu32"", + tm.tm_hour, tm.tm_min, tm.tm_sec); + } + + io_write32(secumod_base + SECUMOD_SCR, + SECUMOD_PIN_MASK << SECUMOD_PIN_SHIFT); + + panic("Tampering detected, system halted"); + + return ITRR_HANDLED; +} + +static struct itr_handler secumod_itr_handler = { + .it = AT91C_ID_SECUMOD, + .handler = secumod_it_handler, +}; + +static void secumod_interrupt_init(void) +{ + itr_add_type_prio(&secumod_itr_handler, IRQ_TYPE_LEVEL_HIGH, 7); + itr_enable(secumod_itr_handler.it); +} + +static void secumod_cfg_input_pio(uint8_t gpio_pin, uint32_t config) +{ + vaddr_t piobu_addr = 0; + uint8_t afv = 0; + uint8_t rfv = 0; + uint8_t pull_mode = PIOBU_PIN_PULL_NONE; + uint8_t def_level = PIOBU_PIN_DEF_LEVEL_LOW; + + assert(gpio_pin < SECUMOD_MAX_PINS); + + piobu_addr = secumod_base + SECUMOD_PIOBU(gpio_pin); + + /* Set GPIO as input */ + io_clrbits32(piobu_addr, SECUMOD_PIOBU_OUTPUT); + + afv = PIOBU_PIN_AFV(config); + rfv = PIOBU_PIN_RFV(config); + pull_mode = PIOBU_PIN_PULL_MODE(config); + def_level = PIOBU_PIN_DEF_LEVEL(config); + + io_write32(piobu_addr, afv | rfv << SECUMOD_PIOBU_RFV_SHIFT | + pull_mode << SECUMOD_PIOBU_PULLUP_SHIFT | + def_level << SECUMOD_PIOBU_SWITCH_SHIFT); + + /* Enable Tampering Interrupt */ + secumod_gpio_set_interrupt(&secumod_chip, gpio_pin, + GPIO_INTERRUPT_ENABLE); + + /* Enable Intrusion Detection */ + io_setbits32(secumod_base + SECUMOD_NMPR, SECUMOD_PIN_VAL(gpio_pin)); + + /* Enable Wakeup */ + if (PIOBU_PIN_WAKEUP(config)) + io_setbits32(secumod_base + SECUMOD_WKPR, + SECUMOD_PIN_VAL(gpio_pin)); + + gpio_protected |= BIT32(gpio_pin); +} + +static void secumod_hw_init(const void *fdt, int node) +{ + int i = 0; + int len = 0; + uint8_t gpio_pin = 0; + uint32_t config = 0; + const uint32_t *prop = NULL; + + /* Disable JTAG Reset and Debug */ + io_write32(secumod_base + SECUMOD_JTAGCR, SECUMOD_JTAGCR_FNTRST); + + /* Switch IOs to normal mode */ + io_write32(secumod_base + SECUMOD_CR, SECUMOD_CR_KEY | + SECUMOD_CR_NORMAL); + + /* Clear all detection intrusion in normal mode */ + io_write32(secumod_base + SECUMOD_NMPR, 0); + + /* Clear Alarms */ + io_write32(secumod_base + SECUMOD_SCR, + SECUMOD_PIN_MASK << SECUMOD_PIN_SHIFT); + + /* Configure each IOs */ + prop = fdt_getprop(fdt, node, "gpios", &len); + if (!prop) + return; + + len /= sizeof(uint32_t); + for (i = 0; i < len; i += DT_GPIO_CELLS) { + gpio_pin = fdt32_to_cpu(prop[i]); + config = fdt32_to_cpu(prop[i + 1]); + + secumod_cfg_input_pio(gpio_pin, config); + } +} + +#ifdef CFG_PM_ARM32 +static TEE_Result piobu_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *hdl __unused) +{ + switch (op) { + case PM_OP_RESUME: + io_write32(secumod_base + SECUMOD_CR, SECUMOD_CR_KEY | + SECUMOD_CR_NORMAL); + break; + case PM_OP_SUSPEND: + /* Enter backup mode before suspending */ + io_write32(secumod_base + SECUMOD_CR, SECUMOD_CR_KEY | + SECUMOD_CR_BACKUP); + break; + default: + panic("Invalid PM operation"); + } + + return TEE_SUCCESS; +} + +static void piobu_register_pm(void) +{ + register_pm_driver_cb(piobu_pm, NULL, "piobu"); +} +#else +static void piobu_register_pm(void) {} +#endif + +static TEE_Result atmel_secumod_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + size_t size = 0; + + if (secumod_base) + return TEE_ERROR_GENERIC; + + if (dt_map_dev(fdt, node, &secumod_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + secumod_hw_init(fdt, node); + secumod_interrupt_init(); + + secumod_chip.ops = &atmel_piobu_ops; + + piobu_register_pm(); + + assert(gpio_ops_is_valid(&atmel_piobu_ops)); + + return gpio_register_provider(fdt, node, secumod_dt_get, &secumod_chip); +} + +static const struct dt_device_match atmel_secumod_match_table[] = { + { .compatible = "atmel,sama5d2-secumod" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_secumod_dt_driver) = { + .name = "atmel_secumod", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_secumod_match_table, + .probe = atmel_secumod_probe, +}; diff --git a/optee_os/core/drivers/atmel_rstc.c b/optee_os/core/drivers/atmel_rstc.c new file mode 100644 index 0000000..5fb81d4 --- /dev/null +++ b/optee_os/core/drivers/atmel_rstc.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AT91_RSTC_CR 0x0 +#define AT91_RSTC_CR_KEY SHIFT_U32(0xA5, 24) +#define AT91_RSTC_CR_PROCRST BIT32(0) +#define AT91_RSTC_CR_PERRST BIT32(2) + +static vaddr_t rstc_base; + +bool atmel_rstc_available(void) +{ + return rstc_base != 0; +} + +void __noreturn atmel_rstc_reset(void) +{ + uint32_t val = AT91_RSTC_CR_KEY | AT91_RSTC_CR_PROCRST | + AT91_RSTC_CR_PERRST; + + io_write32(rstc_base + AT91_RSTC_CR, val); + + /* + * After the previous write, the CPU will reset so we will never hit + * this loop. + */ + while (true) + ; +} + +static TEE_Result atmel_rstc_probe(const void *fdt, int node, + const void *compat_data __unused) + +{ + size_t size = 0; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_BAD_PARAMETERS; + + matrix_configure_periph_secure(AT91C_ID_SYS); + + if (dt_map_dev(fdt, node, &rstc_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static const struct dt_device_match atmel_rstc_match_table[] = { + { .compatible = "atmel,sama5d3-rstc" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_rstc_dt_driver) = { + .name = "atmel_rstc", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_rstc_match_table, + .probe = atmel_rstc_probe, +}; diff --git a/optee_os/core/drivers/atmel_rtc.c b/optee_os/core/drivers/atmel_rtc.c new file mode 100644 index 0000000..3f79bdf --- /dev/null +++ b/optee_os/core/drivers/atmel_rtc.c @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microchip + * + * Driver for AT91 RTC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTC_VAL(reg, val) (((val) >> RTC_## reg ## _SHIFT) & \ + RTC_## reg ##_MASK) + +#define RTC_SET_VAL(reg, val) SHIFT_U32((val) & RTC_## reg ##_MASK, \ + RTC_## reg ## _SHIFT) + +#define RTC_CR 0x0 +#define RTC_CR_UPDCAL BIT(1) +#define RTC_CR_UPDTIM BIT(0) + +#define RTC_MR 0x4 +#define RTC_MR_HR_MODE BIT(0) +#define RTC_MR_PERSIAN BIT(1) +#define RTC_MR_UTC BIT(2) +#define RTC_MR_NEGPPM BIT(4) +#define RTC_MR_CORR_SHIFT 8 +#define RTC_MR_CORR_MASK GENMASK_32(6, 0) +#define RTC_MR_CORR(val) RTC_VAL(val, MR_CORR) +#define RTC_MR_HIGHPPM BIT(15) + +#define RTC_TIMR 0x8 +#define RTC_CALR 0xC + +#define RTC_SR 0x18 +#define RTC_SR_ACKUPD BIT(0) +#define RTC_SR_SEC BIT(2) + +#define RTC_SCCR 0x1C +#define RTC_SCCR_ACKCLR BIT(0) +#define RTC_SCCR_SECCLR BIT(2) + +#define RTC_VER 0x2C +#define RTC_VER_NVTIM BIT(0) +#define RTC_VER_NVCAL BIT(1) + +#define RTC_TSTR0 0xB0 +#define RTC_TSDR0 0xB4 + +#define RTC_TSSR0 0xB8 +#define RTC_TSSR_DET_OFFSET 16 +#define RTC_TSSR_DET_COUNT 8 +#define RTC_TSSR_TST_PIN BIT(2) +#define RTC_TSSR_JTAG BIT(3) + +/* Layout of Time registers */ +#define RTC_TIME_BACKUP BIT(31) +#define RTC_TIME_HOUR_SHIFT 16 +#define RTC_TIME_HOUR_MASK GENMASK_32(5, 0) +#define RTC_TIME_MIN_SHIFT 8 +#define RTC_TIME_MIN_MASK GENMASK_32(6, 0) +#define RTC_TIME_SEC_SHIFT 0 +#define RTC_TIME_SEC_MASK GENMASK_32(6, 0) + +/* Layout of Calendar registers */ +#define RTC_CAL_DATE_SHIFT 24 +#define RTC_CAL_DATE_MASK GENMASK_32(5, 0) +#define RTC_CAL_DAY_SHIFT 21 +#define RTC_CAL_DAY_MASK GENMASK_32(2, 0) +#define RTC_CAL_MONTH_SHIFT 16 +#define RTC_CAL_MONTH_MASK GENMASK_32(4, 0) +#define RTC_CAL_YEAR_SHIFT 8 +#define RTC_CAL_YEAR_MASK GENMASK_32(7, 0) +#define RTC_CAL_CENT_SHIFT 0 +#define RTC_CAL_CENT_MASK GENMASK_32(6, 0) + +#define ATMEL_RTC_CORR_DIVIDEND 3906000 +#define ATMEL_RTC_CORR_LOW_RATIO 20 + +static vaddr_t rtc_base; + +static uint8_t bcd_decode(uint8_t dcb_val) +{ + return (dcb_val & 0xF) + (dcb_val >> 4) * 10; +} + +static uint8_t bcd_encode(uint32_t value) +{ + return ((value / 10) << 4) + value % 10; +} + +static uint32_t atmel_rtc_read(unsigned int offset) +{ + return io_read32(rtc_base + offset); +} + +static void atmel_rtc_write(unsigned int offset, uint32_t val) +{ + return io_write32(rtc_base + offset, val); +} + +static void atmel_decode_date(unsigned int time_reg, unsigned int cal_reg, + struct optee_rtc_time *tm) +{ + uint32_t time = 0; + uint32_t date = 0; + + /* Must read twice in case it changes */ + do { + time = atmel_rtc_read(time_reg); + date = atmel_rtc_read(cal_reg); + } while ((time != atmel_rtc_read(time_reg)) || + (date != atmel_rtc_read(cal_reg))); + + tm->tm_wday = bcd_decode(RTC_VAL(CAL_DAY, date)) - 1; + tm->tm_mday = bcd_decode(RTC_VAL(CAL_DATE, date)); + tm->tm_mon = bcd_decode(RTC_VAL(CAL_MONTH, date)) - 1; + tm->tm_year = bcd_decode(RTC_VAL(CAL_CENT, date)) * 100; + tm->tm_year += bcd_decode(RTC_VAL(CAL_YEAR, date)); + + tm->tm_hour = bcd_decode(RTC_VAL(TIME_HOUR, time)); + tm->tm_min = bcd_decode(RTC_VAL(TIME_MIN, time)); + tm->tm_sec = bcd_decode(RTC_VAL(TIME_SEC, time)); +} + +static TEE_Result atmel_rtc_get_time(struct rtc *rtc __unused, + struct optee_rtc_time *tm) +{ + atmel_decode_date(RTC_TIMR, RTC_CALR, tm); + + return TEE_SUCCESS; +} + +TEE_Result atmel_rtc_get_tamper_timestamp(struct optee_rtc_time *tm) +{ + if (!rtc_base) + return TEE_ERROR_NOT_SUPPORTED; + + atmel_decode_date(RTC_TSTR0, RTC_TSDR0, tm); + + return TEE_SUCCESS; +} + +static TEE_Result atmel_rtc_set_time(struct rtc *rtc __unused, + struct optee_rtc_time *tm) +{ + uint32_t cr = 0; + uint32_t sr = 0; + uint32_t err = 0; + + /* First, wait for UPDCAL/UPDTIM to be 0 */ + do { + cr = atmel_rtc_read(RTC_CR); + } while (cr & (RTC_CR_UPDCAL | RTC_CR_UPDTIM)); + + /* Stop Time/Calendar for update */ + atmel_rtc_write(RTC_CR, cr | RTC_CR_UPDCAL | RTC_CR_UPDTIM); + + do { + sr = atmel_rtc_read(RTC_SR); + } while (!(sr & RTC_SR_ACKUPD)); + + atmel_rtc_write(RTC_SCCR, RTC_SCCR_ACKCLR); + + atmel_rtc_write(RTC_TIMR, + RTC_SET_VAL(TIME_SEC, bcd_encode(tm->tm_sec)) | + RTC_SET_VAL(TIME_MIN, bcd_encode(tm->tm_min)) | + RTC_SET_VAL(TIME_HOUR, bcd_encode(tm->tm_hour))); + + atmel_rtc_write(RTC_CALR, + RTC_SET_VAL(CAL_CENT, + bcd_encode(tm->tm_year / 100)) | + RTC_SET_VAL(CAL_YEAR, bcd_encode(tm->tm_year % 100)) | + RTC_SET_VAL(CAL_MONTH, bcd_encode(tm->tm_mon + 1)) | + RTC_SET_VAL(CAL_DAY, bcd_encode(tm->tm_wday + 1)) | + RTC_SET_VAL(CAL_DATE, bcd_encode(tm->tm_mday))); + + err = atmel_rtc_read(RTC_VER); + if (err) { + if (err & RTC_VER_NVTIM) + DMSG("Invalid time programmed"); + if (err & RTC_VER_NVCAL) + DMSG("Invalid date programmed"); + + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Restart Time/Calendar */ + atmel_rtc_write(RTC_CR, cr); + + return TEE_SUCCESS; +} + +static TEE_Result atmel_rtc_get_offset(struct rtc *rtc __unused, long *offset) +{ + uint32_t mr = atmel_rtc_read(RTC_MR); + long val = RTC_VAL(MR_CORR, mr); + + if (!val) { + *offset = 0; + return TEE_SUCCESS; + } + + val++; + + if (!(mr & RTC_MR_HIGHPPM)) + val *= ATMEL_RTC_CORR_LOW_RATIO; + + val = UDIV_ROUND_NEAREST(ATMEL_RTC_CORR_DIVIDEND, val); + + if (!(mr & RTC_MR_NEGPPM)) + val = -val; + + *offset = val; + + return TEE_SUCCESS; +} + +static TEE_Result atmel_rtc_set_offset(struct rtc *rtc __unused, long offset) +{ + long corr = 0; + uint32_t mr = 0; + + if (offset > ATMEL_RTC_CORR_DIVIDEND / 2) + return TEE_ERROR_BAD_PARAMETERS; + if (offset < -ATMEL_RTC_CORR_DIVIDEND / 2) + return TEE_ERROR_BAD_PARAMETERS; + + mr = atmel_rtc_read(RTC_MR); + mr &= ~(RTC_MR_NEGPPM | RTC_MR_CORR_MASK | RTC_MR_HIGHPPM); + + if (offset > 0) + mr |= RTC_MR_NEGPPM; + else + offset = -offset; + + /* offset less than 764 ppb, disable correction */ + if (offset < 764) { + atmel_rtc_write(RTC_MR, mr & ~RTC_MR_NEGPPM); + + return TEE_SUCCESS; + } + + /* + * 29208 ppb is the perfect cutoff between low range and high range + * low range values are never better than high range value after that. + */ + if (offset < 29208) { + corr = UDIV_ROUND_NEAREST(ATMEL_RTC_CORR_DIVIDEND, + offset * ATMEL_RTC_CORR_LOW_RATIO); + } else { + corr = UDIV_ROUND_NEAREST(ATMEL_RTC_CORR_DIVIDEND, offset); + mr |= RTC_MR_HIGHPPM; + } + + corr = MIN(corr, 128); + + mr |= ((corr - 1) & RTC_MR_CORR_MASK) << RTC_MR_CORR_SHIFT; + + atmel_rtc_write(RTC_MR, mr); + + return TEE_SUCCESS; +} + +static const struct rtc_ops atmel_rtc_ops = { + .get_time = atmel_rtc_get_time, + .set_time = atmel_rtc_set_time, + .get_offset = atmel_rtc_get_offset, + .set_offset = atmel_rtc_set_offset, +}; + +static struct rtc atmel_rtc = { + .ops = &atmel_rtc_ops, + .range_min = { 1900, 1, 1, 0, 0, 0, 0 }, + .range_max = { 2099, 12, 31, 23, 59, 59, 0 }, +}; + +static TEE_Result atmel_rtc_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + size_t size = 0; + + if (rtc_base) + return TEE_ERROR_GENERIC; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_BAD_PARAMETERS; + + matrix_configure_periph_secure(AT91C_ID_SYS); + + if (dt_map_dev(fdt, node, &rtc_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + atmel_rtc_write(RTC_CR, 0); + /* Enable 24 hours Gregorian mode (this is a clear bits operation !) */ + io_clrbits32(rtc_base + RTC_MR, RTC_MR_PERSIAN | RTC_MR_UTC | + RTC_MR_HR_MODE); + + rtc_register(&atmel_rtc); + + return TEE_SUCCESS; +} + +static const struct dt_device_match atmel_rtc_match_table[] = { + { .compatible = "atmel,sama5d2-rtc" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_rtc_dt_driver) = { + .name = "atmel_rtc", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_rtc_match_table, + .probe = atmel_rtc_probe, +}; + diff --git a/optee_os/core/drivers/atmel_saic.c b/optee_os/core/drivers/atmel_saic.c new file mode 100644 index 0000000..5989056 --- /dev/null +++ b/optee_os/core/drivers/atmel_saic.c @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AT91_AIC_MAX_PRIO 8 + +#define SAMA5D2_AIC_MAX_IRQS 77 + +#define SAMA5D2_AIC_MAX_IRQS32 ((SAMA5D2_AIC_MAX_IRQS + 31) / 32) + +struct saic_data { + struct itr_chip chip; + vaddr_t base; + size_t nr_irqs; + uint32_t external[SAMA5D2_AIC_MAX_IRQS32]; +}; + +static struct saic_data saic; + +static void saic_register_pm(void); + +static void saic_write_reg(uint32_t reg, uint32_t val) +{ + io_write32(saic.base + reg, val); +} + +static uint32_t saic_read_reg(uint32_t reg) +{ + return io_read32(saic.base + reg); +} + +void interrupt_main_handler(void) +{ + uint32_t irqnr = saic_read_reg(AT91_AIC_IVR); + + interrupt_call_handlers(&saic.chip, irqnr); + saic_write_reg(AT91_AIC_EOICR, 0); +} + +static void saic_select_it(size_t it) +{ + assert(!(it & ~AT91_AIC_SSR_ITSEL_MASK)); + + saic_write_reg(AT91_AIC_SSR, it); +} + +static void saic_configure_it(size_t it, uint32_t src_type, uint32_t priority) +{ + saic_select_it(it); + saic_write_reg(AT91_AIC_SMR, src_type | priority); +} + +static bool is_external_it(size_t it) +{ + uint32_t it_grp = it / 32; + uint32_t it_off = it % 32; + + if (it >= saic.nr_irqs) + panic(); + + return saic.external[it_grp] & BIT32(it_off); +} + +static TEE_Result saic_get_src_type(uint32_t dt_level, size_t it, + uint32_t *src_type) +{ + switch (dt_level) { + case IRQ_TYPE_EDGE_RISING: + *src_type = AT91_AIC_SMR_POS_EDGE; + break; + case IRQ_TYPE_EDGE_FALLING: + if (!is_external_it(it)) + return TEE_ERROR_BAD_PARAMETERS; + + *src_type = AT91_AIC_SMR_NEG_EDGE; + break; + case IRQ_TYPE_LEVEL_HIGH: + *src_type = AT91_AIC_SMR_HIGH_LEVEL; + break; + case IRQ_TYPE_LEVEL_LOW: + if (!is_external_it(it)) + return TEE_ERROR_BAD_PARAMETERS; + + *src_type = AT91_AIC_SMR_LEVEL; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static void saic_add(struct itr_chip *chip __unused, size_t it, + uint32_t type, uint32_t prio) +{ + uint32_t src_type = AT91_AIC_SMR_HIGH_LEVEL; + + if (it >= saic.nr_irqs) + panic(); + + if (saic_get_src_type(type, it, &src_type)) + panic("Invalid interrupt specifier"); + + saic_configure_it(it, src_type, prio); +} + +static void saic_enable(struct itr_chip *chip __unused, size_t it) +{ + saic_select_it(it); + saic_write_reg(AT91_AIC_IECR, 1); +} + +static void saic_disable(struct itr_chip *chip __unused, size_t it) +{ + saic_select_it(it); + saic_write_reg(AT91_AIC_IDCR, 1); +} + +static const struct itr_ops saic_ops = { + .add = saic_add, + .mask = saic_disable, + .unmask = saic_enable, + .enable = saic_enable, + .disable = saic_disable, +}; + +static int saic_dt_get_irq(const uint32_t *properties, int len, + uint32_t *type, uint32_t *prio) +{ + int it = DT_INFO_INVALID_INTERRUPT; + uint32_t src_type = 0; + uint32_t priority = 0; + uint32_t irq_type = 0; + + len /= sizeof(uint32_t); + + if (len != 3) + return DT_INFO_INVALID_INTERRUPT; + + it = fdt32_to_cpu(properties[0]); + if (it >= (int)saic.nr_irqs) + return DT_INFO_INVALID_INTERRUPT; + + irq_type = fdt32_to_cpu(properties[1]); + if (saic_get_src_type(irq_type, it, &src_type)) + return DT_INFO_INVALID_INTERRUPT; + + priority = fdt32_to_cpu(properties[2]); + if (priority >= AT91_AIC_MAX_PRIO) + return DT_INFO_INVALID_INTERRUPT; + + if (type) + *type = irq_type; + + if (prio) + *prio = priority; + + return it; +} + +static struct saic_data saic = { + .chip = { + .ops = &saic_ops, + .dt_get_irq = &saic_dt_get_irq, + }, +}; + +static void saic_clear_aicredir(void) +{ + vaddr_t sfr_base = sam_sfr_base(); + uint32_t aicredir_val = 0; + + aicredir_val = io_read32(sfr_base + AT91_SFR_SN1); + aicredir_val ^= AT91_SFR_AICREDIR_XOR_KEY; + aicredir_val &= AT91_SFR_AICREDIR_KEY_MASK; + + /* + * We explicitly don't want to redirect secure interrupts to non secure + * AIC. By default, AT91Bootstrap does so on some platforms. + */ + io_write32(sfr_base + AT91_SFR_AICREDIR, aicredir_val); +} + +static void saic_init_external(const void *fdt, int node) +{ + int i = 0; + int len = 0; + int it_grp = 0; + int it_off = 0; + size_t it = 0; + const uint32_t *external = NULL; + + external = fdt_getprop(fdt, node, "atmel,external-irqs", &len); + if (!external) + return; + + len /= sizeof(uint32_t); + for (i = 0; i < len; i++) { + it = fdt32_to_cpu(external[i]); + + DMSG("IRQ %zu is external", it); + + if (it >= saic.nr_irqs) + panic(); + + it_grp = it / 32; + it_off = it % 32; + + saic.external[it_grp] |= BIT32(it_off); + } +} + +static void saic_init_hw(void) +{ + unsigned int i = 0; + + saic_clear_aicredir(); + + /* Disable write protect if any */ + saic_write_reg(AT91_AIC_WPMR, AT91_AIC_WPKEY); + + /* Pop the (potential) interrupt stack (8 priority) */ + for (i = 0; i < 8; i++) + saic_write_reg(AT91_AIC_EOICR, 0); + + /* Disable and clear all interrupts initially */ + for (i = 0; i < saic.nr_irqs; i++) { + saic_write_reg(AT91_AIC_IDCR, 1); + saic_write_reg(AT91_AIC_ICCR, 1); + /* Set interrupt vector to hold interrupt number */ + saic_select_it(i); + saic_write_reg(AT91_AIC_SVR, i); + } + + saic_write_reg(AT91_AIC_SPU, 0xffffffff); + + /* Disable AIC debugging */ + saic_write_reg(AT91_AIC_DCR, 0); +} + +TEE_Result atmel_saic_setup(void) +{ + int node = -1; + int ret = 0; + size_t size = 0; + const void *fdt = get_embedded_dt(); + + /* There is only 1 SAIC controller */ + if (saic.base) + return TEE_ERROR_GENERIC; + + node = fdt_node_offset_by_compatible(fdt, -1, "atmel,sama5d2-saic"); + if (node < 0) + return TEE_ERROR_GENERIC; + + ret = dt_map_dev(fdt, node, &saic.base, &size, DT_MAP_AUTO); + if (ret) { + EMSG("Failed to map SAIC\n"); + return TEE_ERROR_GENERIC; + } + + saic.chip.ops = &saic_ops; + saic.nr_irqs = SAMA5D2_AIC_MAX_IRQS; + + saic_init_external(fdt, node); + saic_init_hw(); + + interrupt_main_init(&saic.chip); + saic_register_pm(); + + return TEE_SUCCESS; +} + +#ifdef CFG_PM_ARM32 + +static struct { + uint8_t smr[SAMA5D2_AIC_MAX_IRQS]; +} saic_state; + +static void saic_resume(void) +{ + uint8_t it = 0; + + saic_init_hw(); + + for (it = 0; it < SAMA5D2_AIC_MAX_IRQS; it++) { + saic_select_it(it); + saic_write_reg(AT91_AIC_SMR, saic_state.smr[it]); + } +} + +static void saic_suspend(void) +{ + uint8_t it = 0; + + for (it = 0; it < SAMA5D2_AIC_MAX_IRQS; it++) { + saic_select_it(it); + saic_state.smr[it] = saic_read_reg(AT91_AIC_SMR); + } +} + +static TEE_Result saic_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *hdl __unused) +{ + switch (op) { + case PM_OP_RESUME: + saic_resume(); + break; + case PM_OP_SUSPEND: + saic_suspend(); + break; + default: + panic("Invalid PM operation"); + } + + return TEE_SUCCESS; +} + +static void saic_register_pm(void) +{ + register_pm_core_service_cb(saic_pm, NULL, "saic"); +} +#else +static void saic_register_pm(void) +{ +} +#endif diff --git a/optee_os/core/drivers/atmel_shdwc.c b/optee_os/core/drivers/atmel_shdwc.c new file mode 100644 index 0000000..b77f934 --- /dev/null +++ b/optee_os/core/drivers/atmel_shdwc.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015 Atmel Corporation, + * Nicolas Ferre + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define SHDW_WK_PIN(reg, cfg) ((reg) & \ + AT91_SHDW_WKUPIS((cfg)->wkup_pin_input)) +#define SHDW_RTCWK(reg, cfg) (((reg) >> ((cfg)->sr_rtcwk_shift)) & 0x1) +#define SHDW_RTTWK(reg, cfg) (((reg) >> ((cfg)->sr_rttwk_shift)) & 0x1) +#define SHDW_RTCWKEN(cfg) BIT((cfg)->mr_rtcwk_shift) +#define SHDW_RTTWKEN(cfg) BIT((cfg)->mr_rttwk_shift) + +#define SLOW_CLK_FREQ 32768ULL +#define DBC_PERIOD_US(x) DIV_ROUND_UP((1000000ULL * (x)), SLOW_CLK_FREQ) + +static vaddr_t shdwc_base; +static vaddr_t mpddrc_base; + +bool atmel_shdwc_available(void) +{ + return shdwc_base != 0; +} + +void __noreturn atmel_shdwc_shutdown(void) +{ + vaddr_t pmc_base = at91_pmc_get_base(); + + /* + * Mask exception before entering assembly which does not expect to be + * interrupted. + */ + thread_mask_exceptions(THREAD_EXCP_ALL); + + __atmel_shdwc_shutdown(mpddrc_base, shdwc_base, pmc_base); + + /* We are going to shutdown the CPU so we will never hit this loop */ + while (true) + ; +} + +static const unsigned long long sdwc_dbc_period[] = { + 0, 3, 32, 512, 4096, 32768, +}; + +static uint32_t at91_shdwc_debouncer_value(uint32_t in_period_us) +{ + int i = 0; + int max_idx = ARRAY_SIZE(sdwc_dbc_period) - 1; + uint64_t period_us = 0; + uint64_t max_period_us = DBC_PERIOD_US(sdwc_dbc_period[max_idx]); + + if (in_period_us > max_period_us) { + DMSG("debouncer period %"PRIu32" too big, using %"PRIu64" us", + in_period_us, max_period_us); + return max_idx; + } + + for (i = max_idx - 1; i > 0; i--) { + period_us = DBC_PERIOD_US(sdwc_dbc_period[i]); + if (in_period_us > period_us) + break; + } + + return i + 1; +} + +static uint32_t at91_shdwc_get_wakeup_input(const void *fdt, int np) +{ + const uint32_t *prop = NULL; + uint32_t wk_input_mask = 0; + uint32_t wuir = 0; + uint32_t wk_input = 0; + int child = 0; + int len = 0; + + fdt_for_each_subnode(child, fdt, np) { + prop = fdt_getprop(fdt, child, "reg", &len); + if (!prop || len != sizeof(uint32_t)) { + DMSG("reg property is missing for node %s", + fdt_get_name(fdt, child, NULL)); + continue; + } + wk_input = fdt32_to_cpu(*prop); + wk_input_mask = BIT32(wk_input); + if (!(wk_input_mask & AT91_SHDW_WKUPEN_MASK)) { + DMSG("wake-up input %"PRId32" out of bounds ignore", + wk_input); + continue; + } + wuir |= wk_input_mask; + + if (fdt_getprop(fdt, child, "atmel,wakeup-active-high", NULL)) + wuir |= AT91_SHDW_WKUPT(wk_input); + } + + return wuir; +} + +static void at91_shdwc_dt_configure(const void *fdt, int np) +{ + const uint32_t *prop = NULL; + uint32_t mode = 0; + uint32_t tmp = 0; + uint32_t input = 0; + int len = 0; + + prop = fdt_getprop(fdt, np, "debounce-delay-us", &len); + if (prop && len == sizeof(uint32_t)) { + tmp = fdt32_to_cpu(*prop); + mode |= AT91_SHDW_WKUPDBC(at91_shdwc_debouncer_value(tmp)); + } + + if (fdt_getprop(fdt, np, "atmel,wakeup-rtc-timer", &len)) + mode |= AT91_SHDW_RTCWKEN; + + io_write32(shdwc_base + AT91_SHDW_MR, mode); + + input = at91_shdwc_get_wakeup_input(fdt, np); + io_write32(shdwc_base + AT91_SHDW_WUIR, input); +} + +static TEE_Result atmel_shdwc_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + int ddr_node = 0; + size_t size = 0; + uint32_t ddr = AT91_DDRSDRC_MD_LPDDR2; + + /* + * Assembly code relies on the fact that there is only one CPU to avoid + * any other one to invalidate TLB/I-Cache. + */ + COMPILE_TIME_ASSERT(CFG_TEE_CORE_NB_CORE == 1); + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_BAD_PARAMETERS; + + matrix_configure_periph_secure(AT91C_ID_SYS); + + if (dt_map_dev(fdt, node, &shdwc_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + ddr_node = fdt_node_offset_by_compatible(fdt, -1, + "atmel,sama5d3-ddramc"); + if (ddr_node < 0) + return TEE_ERROR_GENERIC; + + if (dt_map_dev(fdt, ddr_node, &mpddrc_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + ddr = io_read32(mpddrc_base + AT91_DDRSDRC_MDR) & AT91_DDRSDRC_MD; + if (ddr != AT91_DDRSDRC_MD_LPDDR2 && ddr != AT91_DDRSDRC_MD_LPDDR3) + mpddrc_base = 0; + + at91_shdwc_dt_configure(fdt, node); + + return sama5d2_pm_init(fdt, shdwc_base); +} + +static const struct dt_device_match atmel_shdwc_match_table[] = { + { .compatible = "atmel,sama5d2-shdwc" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_shdwc_dt_driver) = { + .name = "atmel_shdwc", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_shdwc_match_table, + .probe = atmel_shdwc_probe, +}; diff --git a/optee_os/core/drivers/atmel_shdwc_a32.S b/optee_os/core/drivers/atmel_shdwc_a32.S new file mode 100644 index 0000000..c3ebca1 --- /dev/null +++ b/optee_os/core/drivers/atmel_shdwc_a32.S @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015 Atmel Corporation, + * Nicolas Ferre + * Copyright (c) 2021, Microchip + */ + + +#include +#include +#include +#include +#include + +#include "at91_pmc.h" + +/* + * Code size of shutdown assembly must fit in a single Cortex-A5 cache + * line which is 8 words long (32 bytes) since SDRAM might be disabled and thus + * not accessible to fetch code or data from it. + */ +.macro check_fit_in_cacheline since + .if (. - \since) > 32 + .error "Shutdown assembly code exceeds cache line size" + .endif +.endm + +/** + * Shutdown the CPU + * + * This function is in assembly to be sure the code fits in a single cache line. + * We are going to power down the SDRAM and thus we can't fetch code from it + * once powered down. + * + * r0 = mpddrc_base + * r1 = shdwc_base + * r2 = pmc_base + */ +FUNC __atmel_shdwc_shutdown , : + + mov_imm r3, AT91_DDRSDRC_LPDDR2_PWOFF + mov_imm r4, (AT91_SHDW_KEY | AT91_SHDW_SHDW) + + /* + * Read values from both shutdown controller and PMC to ensure the + * translations will be in the TLB. + */ + ldr r6, [r1, #AT91_SHDW_CR] + ldr r6, [r2, #AT91_PMC_MCKR] + + /* Power down SDRAM0 if mpddrc_base is set */ + tst r0, #0 + beq 1f + +/* Align to cache line to ensure the rest of code fits in a single line */ +.balign 32 +__atmel_shdwc_shutdown_sdram_disabled: + str r3, [r0, #AT91_DDRSDRC_LPR] + + /* Switch the master clock source to slow clock. */ +1: + ldr r6, [r2, #AT91_PMC_MCKR] + bic r6, r6, #AT91_PMC_CSS + str r6, [r2, #AT91_PMC_MCKR] + + /* Wait for clock switch. */ +2: + ldr r6, [r2, #AT91_PMC_SR] + tst r6, #AT91_PMC_MCKRDY + beq 2b + + /* Shutdown CPU */ + str r4, [r1, #AT91_SHDW_CR] + + check_fit_in_cacheline __atmel_shdwc_shutdown_sdram_disabled + + /* We should never reach this since we shut down the CPU */ + b . +END_FUNC __atmel_shdwc_shutdown diff --git a/optee_os/core/drivers/atmel_tcb.c b/optee_os/core/drivers/atmel_tcb.c new file mode 100644 index 0000000..0e87453 --- /dev/null +++ b/optee_os/core/drivers/atmel_tcb.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TCB_CHAN(chan) ((chan) * 0x40) + +#define TCB_CCR(chan) (0x0 + TCB_CHAN(chan)) +#define TCB_CCR_SWTRG 0x4 +#define TCB_CCR_CLKEN 0x1 + +#define TCB_CMR(chan) (0x4 + TCB_CHAN(chan)) +#define TCB_CMR_WAVE BIT32(15) +#define TCB_CMR_TIMER_CLOCK5 4 +#define TCB_CMR_XC1 6 +#define TCB_CMR_ACPA_SET BIT32(16) +#define TCB_CMR_ACPC_CLEAR SHIFT_U32(2, 18) + +#define TCB_CV(chan) (0x10 + TCB_CHAN(chan)) + +#define TCB_RA(chan) (0x14 + TCB_CHAN(chan)) +#define TCB_RB(chan) (0x18 + TCB_CHAN(chan)) +#define TCB_RC(chan) (0x1c + TCB_CHAN(chan)) + +#define TCB_IER(chan) (0x24 + TCB_CHAN(chan)) +#define TCB_IER_COVFS 0x1 + +#define TCB_SR(chan) (0x20 + TCB_CHAN(chan)) +#define TCB_SR_COVFS 0x1 + +#define TCB_IDR(chan) (0x28 + TCB_CHAN(chan)) + +#define TCB_BCR 0xc0 +#define TCB_BCR_SYNC 0x1 + +#define TCB_BMR 0xc4 +#define TCB_BMR_TC1XC1S_TIOA0 SHIFT_U32(2, 2) + +#define TCB_WPMR 0xe4 +#define TCB_WPMR_WAKEY 0x54494d + +static const char * const tcb_clocks[] = { "t0_clk", "gclk", "slow_clk" }; +static vaddr_t tcb_base; +static uint32_t tcb_rate; + +static TEE_Result atmel_tcb_enable_clocks(const void *fdt, int node) +{ + unsigned int i = 0; + struct clk *clk = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + for (i = 0; i < ARRAY_SIZE(tcb_clocks); i++) { + res = clk_dt_get_by_name(fdt, node, tcb_clocks[i], &clk); + if (res) + return res; + + clk_enable(clk); + } + + return TEE_SUCCESS; +} + +static TEE_Result atmel_tcb_get_sys_time(TEE_Time *time) +{ + uint64_t cv0 = 0; + uint64_t cv1 = 0; + + if (!tcb_base) + return TEE_ERROR_BAD_STATE; + + do { + cv1 = io_read32(tcb_base + TCB_CV(1)); + cv0 = io_read32(tcb_base + TCB_CV(0)); + } while (io_read32(tcb_base + TCB_CV(1)) != cv1); + + cv0 |= cv1 << 32; + + time->seconds = cv0 / tcb_rate; + time->millis = (cv0 % tcb_rate) / (tcb_rate / TEE_TIME_MILLIS_BASE); + + return TEE_SUCCESS; +} + +static const struct time_source atmel_tcb_time_source = { + .name = "atmel_tcb", + .protection_level = 1000, + .get_sys_time = atmel_tcb_get_sys_time, +}; + +REGISTER_TIME_SOURCE(atmel_tcb_time_source) + +static void atmel_tcb_configure(void) +{ + /* Disable write protection */ + io_write32(tcb_base + TCB_WPMR, TCB_WPMR_WAKEY); + + /* Disable all irqs for both channel 0 & 1 */ + io_write32(tcb_base + TCB_IDR(0), 0xff); + io_write32(tcb_base + TCB_IDR(1), 0xff); + + /* + * In order to avoid wrapping, use a 64 bit counter by chaining + * two channels. We use the slow_clk which runs at 32K and is sufficient + * for the millisecond precision, this will wrap in approximately + * 17851025 years so no worries here. + * + * Channel 0 is configured to generate a clock on TIOA0 which is cleared + * when reaching 0x80000000 and set when reaching 0. + */ + io_write32(tcb_base + TCB_CMR(0), + TCB_CMR_TIMER_CLOCK5 | TCB_CMR_WAVE | TCB_CMR_ACPA_SET | + TCB_CMR_ACPC_CLEAR); + io_write32(tcb_base + TCB_RC(0), 0x80000000); + io_write32(tcb_base + TCB_RA(0), 0x1); + io_write32(tcb_base + TCB_CCR(0), TCB_CCR_CLKEN); + + /* Channel 1 is configured to use TIOA0 as input */ + io_write32(tcb_base + TCB_CMR(1), TCB_CMR_XC1 | TCB_CMR_WAVE); + io_write32(tcb_base + TCB_CCR(1), TCB_CCR_CLKEN); + + /* Set XC1 input to be TIOA0 (ie output of Channel 0) */ + io_write32(tcb_base + TCB_BMR, TCB_BMR_TC1XC1S_TIOA0); + + /* Sync & start all timers */ + io_write32(tcb_base + TCB_BCR, TCB_BCR_SYNC); + + /* Enable write protection */ + io_write32(tcb_base + TCB_WPMR, TCB_WPMR_WAKEY | 1); +} + +static TEE_Result atmel_tcb_check(void) +{ + if (!tcb_base) + panic("Missing TCB base ! Check the device-tree"); + + return TEE_SUCCESS; +} + +boot_final(atmel_tcb_check); + +static TEE_Result atmel_tcb_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + size_t size = 0; + struct clk *clk = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + unsigned int peri_id = AT91C_ID_TC0; + + /* Enable all TCB clocks */ + res = atmel_tcb_enable_clocks(fdt, node); + if (res) + return res; + + if (tcb_base) + return TEE_SUCCESS; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_SUCCESS; + + res = clk_dt_get_by_name(fdt, node, "slow_clk", &clk); + if (res) + return res; + + res = matrix_dt_get_id(fdt, node, &peri_id); + if (res) + return res; + + if (dt_map_dev(fdt, node, &tcb_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + matrix_configure_periph_secure(peri_id); + + tcb_rate = clk_get_rate(clk); + assert(tcb_rate); + + atmel_tcb_configure(); + + return TEE_SUCCESS; +} + +static const struct dt_device_match atmel_tcb_match_table[] = { + { .compatible = "atmel,sama5d2-tcb" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_tcb_dt_driver) = { + .name = "atmel_tcb", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_tcb_match_table, + .probe = atmel_tcb_probe, +}; diff --git a/optee_os/core/drivers/atmel_trng.c b/optee_os/core/drivers/atmel_trng.c new file mode 100644 index 0000000..0b18c30 --- /dev/null +++ b/optee_os/core/drivers/atmel_trng.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Registers */ +#define TRNG_CTRL 0x0 +#define TRNG_CTRL_WAKEY_OFFSET 8 +#define TRNG_CTRL_WAKEY_VALUE 0x524E47 + +#define TRNG_IER 0x10 +#define TRNG_ISR 0x1C +#define TRNG_ODATA 0x50 + +static unsigned int trng_lock = SPINLOCK_UNLOCK; +static vaddr_t trng_base; + +static uint32_t atmel_trng_read32(void) +{ + uint32_t exceptions = 0; + uint32_t value = 0; + + exceptions = cpu_spin_lock_xsave(&trng_lock); + + while (!io_read32(trng_base + TRNG_ISR)) + ; + + value = io_read32(trng_base + TRNG_ODATA); + + cpu_spin_unlock_xrestore(&trng_lock, exceptions); + + return value; +} + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + uint8_t *rngbuf = buf; + uint32_t val = 0; + size_t len_to_copy = 0; + + assert(trng_base); + + while (len) { + val = atmel_trng_read32(); + len_to_copy = MIN(len, sizeof(uint32_t)); + memcpy(rngbuf, &val, len_to_copy); + rngbuf += len_to_copy; + len -= len_to_copy; + } + + return TEE_SUCCESS; +} + +/* This is a true RNG, no need for seeding */ +void plat_rng_init(void) +{ +} + +static void atmel_trng_reset(void) +{ + uint32_t ctrl_val = TRNG_CTRL_WAKEY_VALUE << TRNG_CTRL_WAKEY_OFFSET; + + /* Disable TRNG */ + io_setbits32(trng_base + TRNG_CTRL, ctrl_val); + /* Enable interrupt */ + io_setbits32(trng_base + TRNG_IER, 1); + /* Enable TRNG */ + io_setbits32(trng_base + TRNG_CTRL, ctrl_val | 1); +} + +static TEE_Result trng_node_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + int status = fdt_get_status(fdt, node); + size_t size = 0; + struct clk *clk = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + if (status != DT_STATUS_OK_SEC) + return TEE_ERROR_GENERIC; + + matrix_configure_periph_secure(AT91C_ID_TRNG); + + res = clk_dt_get_by_index(fdt, node, 0, &clk); + if (res) + return res; + + if (dt_map_dev(fdt, node, &trng_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + clk_enable(clk); + + atmel_trng_reset(); + + return TEE_SUCCESS; +} + +static const struct dt_device_match atmel_trng_match_table[] = { + { .compatible = "atmel,at91sam9g45-trng" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_trng_dt_driver) = { + .name = "atmel_trng", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_trng_match_table, + .probe = trng_node_probe, +}; diff --git a/optee_os/core/drivers/atmel_uart.c b/optee_os/core/drivers/atmel_uart.c new file mode 100644 index 0000000..7454bdb --- /dev/null +++ b/optee_os/core/drivers/atmel_uart.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Timesys Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +/* Register definitions */ +#define ATMEL_UART_CR 0x0000 /* Control Register */ +#define ATMEL_UART_MR 0x0004 /* Mode Register */ +#define ATMEL_UART_IER 0x0008 /* Interrupt Enable Register */ +#define ATMEL_UART_IDR 0x000c /* Interrupt Disable Register */ +#define ATMEL_UART_IMR 0x0010 /* Interrupt Mask Register */ +#define ATMEL_UART_SR 0x0014 /* Status Register */ + #define ATMEL_SR_RXRDY BIT(0) /* Receiver Ready */ + #define ATMEL_SR_TXRDY BIT(1) /* Transmitter Ready */ + #define ATMEL_SR_TXEMPTY BIT(1) /* Transmitter Ready */ +#define ATMEL_UART_RHR 0x0018 /* Receive Holding Register */ +#define ATMEL_UART_THR 0x001c /* Transmit Holding Register */ +#define ATMEL_UART_BRGR 0x0020 /* Baud Rate Generator Register */ +#define ATMEL_UART_CMPR 0x0024 /* Comparison Register */ +#define ATMEL_UART_RTOR 0x0028 /* Receiver Time-out Register */ +#define ATMEL_UART_WPMR 0x00e4 /* Write Protect Mode Register */ +#define ATMEL_UART_SIZE 0x00e8 /* ATMEL_UART_WPMR + sizeof(uint32_t) */ + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct atmel_uart_data *pd = + container_of(chip, struct atmel_uart_data, chip); + + return io_pa_or_va(&pd->base, ATMEL_UART_SIZE); +} + +static void atmel_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + ATMEL_UART_SR) & ATMEL_SR_TXEMPTY)) + ; +} + +static int atmel_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (io_read32(base + ATMEL_UART_SR) & ATMEL_SR_RXRDY) + ; + + return io_read32(base + ATMEL_UART_RHR); +} + +static void atmel_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + ATMEL_UART_SR) & ATMEL_SR_TXRDY)) + ; + + io_write32(base + ATMEL_UART_THR, ch); +} + +static const struct serial_ops atmel_uart_ops = { + .flush = atmel_uart_flush, + .getchar = atmel_uart_getchar, + .putc = atmel_uart_putc, +}; + +void atmel_uart_init(struct atmel_uart_data *pd, paddr_t base) +{ + pd->base.pa = base; + pd->chip.ops = &atmel_uart_ops; + + /* + * Do nothing, debug uart share with normal world, + * everything for uart initialization is done in bootloader. + */ +} diff --git a/optee_os/core/drivers/atmel_wdt.c b/optee_os/core/drivers/atmel_wdt.c new file mode 100644 index 0000000..b2aa545 --- /dev/null +++ b/optee_os/core/drivers/atmel_wdt.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define WDT_CR 0x0 +#define WDT_CR_KEY SHIFT_U32(0xA5, 24) +#define WDT_CR_WDRSTT BIT(0) + +#define WDT_MR 0x4 +#define WDT_MR_WDV GENMASK_32(11, 0) +#define WDT_MR_WDV_SET(val) ((val) & WDT_MR_WDV) +#define WDT_MR_WDFIEN BIT(12) +#define WDT_MR_WDRSTEN BIT(13) +#define WDT_MR_WDDIS BIT(15) +#define WDT_MR_WDD_SHIFT 16 +#define WDT_MR_WDD_MASK GENMASK_32(11, 0) +#define WDT_MR_WDD SHIFT_U32(WDT_MR_WDD_MASK, WDT_MR_WDD_SHIFT) +#define WDT_MR_WDD_SET(val) \ + SHIFT_U32(((val) & WDT_MR_WDD_MASK), WDT_MR_WDD_SHIFT) +#define WDT_MR_WDDBGHLT BIT(28) +#define WDT_MR_WDIDLEHLT BIT(29) + +#define WDT_SR 0x8 +#define WDT_SR_DUNF BIT(0) +#define WDT_SR_DERR BIT(1) + +/* + * The watchdog is clocked by a 32768Hz clock/128 and the counter is on + * 12 bits. + */ +#define SLOW_CLOCK_FREQ (32768) +#define WDT_CLOCK_FREQ (SLOW_CLOCK_FREQ / 128) +#define WDT_MIN_TIMEOUT 1 +#define WDT_MAX_TIMEOUT (BIT(12) / WDT_CLOCK_FREQ) + +#define WDT_DEFAULT_TIMEOUT WDT_MAX_TIMEOUT + +/* + * We must wait at least 3 clocks period before accessing registers MR and CR. + * Ensure that we see at least 4 edges + */ +#define WDT_REG_ACCESS_UDELAY (1000000ULL / SLOW_CLOCK_FREQ * 4) + +#define SEC_TO_WDT(sec) (((sec) * WDT_CLOCK_FREQ) - 1) + +#define WDT_ENABLED(mr) (!((mr) & WDT_MR_WDDIS)) + +struct atmel_wdt { + struct wdt_chip chip; + vaddr_t base; + unsigned long rate; + uint32_t mr; + bool enabled; +}; + +static void atmel_wdt_write_sleep(struct atmel_wdt *wdt, uint32_t reg, + uint32_t val) +{ + udelay(WDT_REG_ACCESS_UDELAY); + + io_write32(wdt->base + reg, val); +} + +static TEE_Result atmel_wdt_settimeout(struct wdt_chip *chip, + unsigned long timeout) +{ + struct atmel_wdt *wdt = container_of(chip, struct atmel_wdt, chip); + + wdt->mr &= ~WDT_MR_WDV; + wdt->mr |= WDT_MR_WDV_SET(SEC_TO_WDT(timeout)); + + /* WDV and WDD can only be updated when the watchdog is running */ + if (WDT_ENABLED(wdt->mr)) + atmel_wdt_write_sleep(wdt, WDT_MR, wdt->mr); + + return TEE_SUCCESS; +} + +static void atmel_wdt_ping(struct wdt_chip *chip) +{ + struct atmel_wdt *wdt = container_of(chip, struct atmel_wdt, chip); + + atmel_wdt_write_sleep(wdt, WDT_CR, WDT_CR_KEY | WDT_CR_WDRSTT); +} + +static void atmel_wdt_start(struct atmel_wdt *wdt) +{ + wdt->mr &= ~WDT_MR_WDDIS; + atmel_wdt_write_sleep(wdt, WDT_MR, wdt->mr); +} + +static void atmel_wdt_enable(struct wdt_chip *chip) +{ + struct atmel_wdt *wdt = container_of(chip, struct atmel_wdt, chip); + + wdt->enabled = true; + atmel_wdt_start(wdt); +} + +static void atmel_wdt_stop(struct atmel_wdt *wdt) +{ + wdt->mr |= WDT_MR_WDDIS; + atmel_wdt_write_sleep(wdt, WDT_MR, wdt->mr); +} + +static void atmel_wdt_disable(struct wdt_chip *chip) +{ + struct atmel_wdt *wdt = container_of(chip, struct atmel_wdt, chip); + + wdt->enabled = false; + atmel_wdt_stop(wdt); +} + +static enum itr_return atmel_wdt_itr_cb(struct itr_handler *h) +{ + struct atmel_wdt *wdt = h->data; + uint32_t sr = io_read32(wdt->base + WDT_SR); + + if (sr & WDT_SR_DUNF) + DMSG("Watchdog Underflow !"); + if (sr & WDT_SR_DERR) + DMSG("Watchdog Error !"); + + panic("Watchdog interrupt"); + + return ITRR_HANDLED; +} + +static TEE_Result atmel_wdt_init(struct wdt_chip *chip __unused, + unsigned long *min_timeout, + unsigned long *max_timeout) +{ + *min_timeout = WDT_MIN_TIMEOUT; + *max_timeout = WDT_MAX_TIMEOUT; + + return TEE_SUCCESS; +} + +static const struct wdt_ops atmel_wdt_ops = { + .init = atmel_wdt_init, + .start = atmel_wdt_enable, + .stop = atmel_wdt_disable, + .ping = atmel_wdt_ping, + .set_timeout = atmel_wdt_settimeout, +}; + +static void atmel_wdt_init_hw(struct atmel_wdt *wdt) +{ + uint32_t mr = 0; + + /* + * If we are resuming, we disabled the watchdog on suspend but the + * bootloader might have enabled the watchdog. If so, disable it + * properly. + */ + if (!WDT_ENABLED(wdt->mr)) { + mr = io_read32(wdt->base + WDT_MR); + if (WDT_ENABLED(mr)) + io_write32(wdt->base + WDT_MR, mr | WDT_MR_WDDIS); + } + + /* Enable interrupt, and disable watchdog in debug and idle */ + wdt->mr |= WDT_MR_WDFIEN | WDT_MR_WDDBGHLT | WDT_MR_WDIDLEHLT; + /* Enable watchdog reset */ + wdt->mr |= WDT_MR_WDRSTEN; + wdt->mr |= WDT_MR_WDD_SET(SEC_TO_WDT(WDT_MAX_TIMEOUT)); + wdt->mr |= WDT_MR_WDV_SET(SEC_TO_WDT(WDT_DEFAULT_TIMEOUT)); + + /* + * If the watchdog was enabled, write the configuration which will ping + * the watchdog. + */ + if (WDT_ENABLED(wdt->mr)) + io_write32(wdt->base + WDT_MR, wdt->mr); +} + +#ifdef CFG_PM_ARM32 +static TEE_Result atmel_wdt_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *hdl) +{ + struct atmel_wdt *wdt = hdl->handle; + + switch (op) { + case PM_OP_RESUME: + atmel_wdt_init_hw(wdt); + if (wdt->enabled) + atmel_wdt_start(wdt); + break; + case PM_OP_SUSPEND: + if (wdt->enabled) + atmel_wdt_stop(wdt); + break; + default: + panic("Invalid PM operation"); + } + + return TEE_SUCCESS; +} + +static void atmel_wdt_register_pm(struct atmel_wdt *wdt) +{ + register_pm_driver_cb(atmel_wdt_pm, wdt, "atmel_wdt"); +} +#else +static void atmel_wdt_register_pm(struct atmel_wdt *wdt __unused) +{ +} +#endif + +static TEE_Result wdt_node_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + size_t size = 0; + struct atmel_wdt *wdt; + uint32_t irq_type = 0; + uint32_t irq_prio = 0; + int it = DT_INFO_INVALID_INTERRUPT; + struct itr_handler *it_hdlr; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_BAD_PARAMETERS; + + matrix_configure_periph_secure(AT91C_ID_WDT); + + wdt = calloc(1, sizeof(*wdt)); + if (!wdt) + return TEE_ERROR_OUT_OF_MEMORY; + + wdt->chip.ops = &atmel_wdt_ops; + + it = dt_get_irq_type_prio(fdt, node, &irq_type, &irq_prio); + if (it == DT_INFO_INVALID_INTERRUPT) + goto err_free_wdt; + + it_hdlr = itr_alloc_add_type_prio(it, &atmel_wdt_itr_cb, 0, wdt, + irq_type, irq_prio); + if (!it_hdlr) + goto err_free_wdt; + + if (dt_map_dev(fdt, node, &wdt->base, &size, DT_MAP_AUTO) < 0) + goto err_free_itr_handler; + + /* Get current state of the watchdog */ + wdt->mr = io_read32(wdt->base + WDT_MR) & WDT_MR_WDDIS; + + atmel_wdt_init_hw(wdt); + itr_enable(it); + atmel_wdt_register_pm(wdt); + + return watchdog_register(&wdt->chip); + +err_free_itr_handler: + itr_free(it_hdlr); +err_free_wdt: + free(wdt); + + return TEE_ERROR_GENERIC; +} + +static const struct dt_device_match atmel_wdt_match_table[] = { + { .compatible = "atmel,sama5d4-wdt" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_wdt_dt_driver) = { + .name = "atmel_wdt", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_wdt_match_table, + .probe = wdt_node_probe, +}; diff --git a/optee_os/core/drivers/bcm_gpio.c b/optee_os/core/drivers/bcm_gpio.c new file mode 100644 index 0000000..cd7d9b3 --- /dev/null +++ b/optee_os/core/drivers/bcm_gpio.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ +#include +#include +#include +#include +#include +#include +#include + +#define IPROC_GPIO_DATA_IN_OFFSET 0x00 +#define IPROC_GPIO_DATA_OUT_OFFSET 0x04 +#define IPROC_GPIO_OUT_EN_OFFSET 0x08 +#define IPROC_GPIO_INT_MSK_OFFSET 0x18 + +#define GPIO_BANK_SIZE 0x200 +#define NGPIOS_PER_BANK 32 +#define GPIO_BANK(pin) ((pin) / NGPIOS_PER_BANK) + +#define IPROC_GPIO_REG(pin, reg) ((reg) + \ + GPIO_BANK(pin) * GPIO_BANK_SIZE) + +#define IPROC_GPIO_SHIFT(pin) ((pin) % NGPIOS_PER_BANK) + +#define GPIO_BANK_CNT 5 +#define SEC_GPIO_SIZE 0x4 +#define IPROC_GPIO_SEC_CFG_REG(pin) \ + (((GPIO_BANK_CNT - 1) - GPIO_BANK(pin)) * SEC_GPIO_SIZE) + +static SLIST_HEAD(, bcm_gpio_chip) gclist = SLIST_HEAD_INITIALIZER(gclist); + +struct bcm_gpio_chip *bcm_gpio_pin_to_chip(unsigned int pin) +{ + struct bcm_gpio_chip *gc = NULL; + + SLIST_FOREACH(gc, &gclist, link) + if ((pin >= gc->gpio_base) && + (pin < (gc->gpio_base + gc->ngpios))) + return gc; + return NULL; +} + +static bool __maybe_unused gpio_is_range_overlap(unsigned int start, + unsigned int end) +{ + struct bcm_gpio_chip *gc = NULL; + + SLIST_FOREACH(gc, &gclist, link) + if ((start < (gc->gpio_base + gc->ngpios)) && + (end > gc->gpio_base)) + return true; + return false; +} + +static void iproc_set_bit(unsigned int reg, unsigned int gpio) +{ + unsigned int offset = IPROC_GPIO_REG(gpio, reg); + unsigned int shift = IPROC_GPIO_SHIFT(gpio); + struct bcm_gpio_chip *gc = bcm_gpio_pin_to_chip(gpio); + + assert(gc); + io_setbits32(gc->base + offset, BIT(shift)); +} + +static void iproc_clr_bit(unsigned int reg, unsigned int gpio) +{ + unsigned int offset = IPROC_GPIO_REG(gpio, reg); + unsigned int shift = IPROC_GPIO_SHIFT(gpio); + struct bcm_gpio_chip *gc = bcm_gpio_pin_to_chip(gpio); + + assert(gc); + io_clrbits32(gc->base + offset, BIT(shift)); +} + +static void iproc_gpio_set(struct gpio_chip *chip __unused, unsigned int gpio, + enum gpio_level val) +{ + if (val == GPIO_LEVEL_HIGH) + iproc_set_bit(IPROC_GPIO_DATA_OUT_OFFSET, gpio); + else + iproc_clr_bit(IPROC_GPIO_DATA_OUT_OFFSET, gpio); +} + +static enum gpio_level iproc_gpio_get(struct gpio_chip *chip __unused, + unsigned int gpio) +{ + unsigned int offset = IPROC_GPIO_REG(gpio, IPROC_GPIO_DATA_IN_OFFSET); + unsigned int shift = IPROC_GPIO_SHIFT(gpio); + struct bcm_gpio_chip *gc = bcm_gpio_pin_to_chip(gpio); + + assert(gc); + + if (io_read32(gc->base + offset) & BIT(shift)) + return GPIO_LEVEL_HIGH; + else + return GPIO_LEVEL_LOW; +} + +static void iproc_gpio_set_dir(struct gpio_chip *chip __unused, + unsigned int gpio, enum gpio_dir dir) +{ + if (dir == GPIO_DIR_OUT) + iproc_set_bit(IPROC_GPIO_OUT_EN_OFFSET, gpio); + else + iproc_clr_bit(IPROC_GPIO_OUT_EN_OFFSET, gpio); +} + +static enum gpio_dir iproc_gpio_get_dir(struct gpio_chip *chip __unused, + unsigned int gpio) +{ + unsigned int offset = IPROC_GPIO_REG(gpio, IPROC_GPIO_OUT_EN_OFFSET); + unsigned int shift = IPROC_GPIO_SHIFT(gpio); + struct bcm_gpio_chip *gc = bcm_gpio_pin_to_chip(gpio); + + assert(gc); + + if (io_read32(gc->base + offset) & BIT(shift)) + return GPIO_DIR_OUT; + else + return GPIO_DIR_IN; +} + +static enum gpio_interrupt iproc_gpio_get_itr(struct gpio_chip *chip __unused, + unsigned int gpio) +{ + unsigned int offset = IPROC_GPIO_REG(gpio, IPROC_GPIO_INT_MSK_OFFSET); + unsigned int shift = IPROC_GPIO_SHIFT(gpio); + struct bcm_gpio_chip *gc = bcm_gpio_pin_to_chip(gpio); + + assert(gc); + + if (io_read32(gc->base + offset) & BIT(shift)) + return GPIO_INTERRUPT_ENABLE; + else + return GPIO_INTERRUPT_DISABLE; +} + +static void iproc_gpio_set_itr(struct gpio_chip *chip __unused, + unsigned int gpio, enum gpio_interrupt ena_dis) +{ + if (ena_dis == GPIO_INTERRUPT_ENABLE) + iproc_set_bit(IPROC_GPIO_OUT_EN_OFFSET, gpio); + else + iproc_clr_bit(IPROC_GPIO_OUT_EN_OFFSET, gpio); +} + +static const struct gpio_ops bcm_gpio_ops = { + .get_direction = iproc_gpio_get_dir, + .set_direction = iproc_gpio_set_dir, + .get_value = iproc_gpio_get, + .set_value = iproc_gpio_set, + .get_interrupt = iproc_gpio_get_itr, + .set_interrupt = iproc_gpio_set_itr, +}; +DECLARE_KEEP_PAGER(bcm_gpio_ops); + +void iproc_gpio_set_secure(int gpiopin) +{ + vaddr_t regaddr = 0; + unsigned int shift = IPROC_GPIO_SHIFT(gpiopin); + vaddr_t baseaddr = + (vaddr_t)phys_to_virt(CHIP_SECURE_GPIO_CONTROL0_BASE, + MEM_AREA_IO_SEC, + IPROC_GPIO_SEC_CFG_REG(gpiopin) + + sizeof(uint32_t)); + + regaddr = baseaddr + IPROC_GPIO_SEC_CFG_REG(gpiopin); + + io_clrbits32(regaddr, BIT(shift)); +} + +static void iproc_gpio_init(struct bcm_gpio_chip *gc, unsigned int paddr, + unsigned int gpio_base, unsigned int ngpios) +{ + assert(!gpio_is_range_overlap(gpio_base, gpio_base + gc->ngpios)); + + gc->base = core_mmu_get_va(paddr, MEM_AREA_IO_SEC, 1); + gc->chip.ops = &bcm_gpio_ops; + gc->gpio_base = gpio_base; + gc->ngpios = ngpios; + + SLIST_INSERT_HEAD(&gclist, gc, link); + + DMSG("gpio chip for <%u - %u>", gpio_base, gpio_base + ngpios); +} + +static TEE_Result bcm_gpio_init(void) +{ + struct bcm_gpio_chip *gc = NULL; + +#ifdef SECURE_GPIO_BASE0 + gc = malloc(sizeof(*gc)); + if (gc == NULL) + return TEE_ERROR_OUT_OF_MEMORY; + + iproc_gpio_init(gc, SECURE_GPIO_BASE0, GPIO_NUM_START0, NUM_GPIOS0); +#endif +#ifdef SECURE_GPIO_BASE1 + gc = malloc(sizeof(*gc)); + if (gc == NULL) + return TEE_ERROR_OUT_OF_MEMORY; + + iproc_gpio_init(gc, SECURE_GPIO_BASE1, GPIO_NUM_START1, NUM_GPIOS1); +#endif + return TEE_SUCCESS; +} +driver_init(bcm_gpio_init); diff --git a/optee_os/core/drivers/bcm_hwrng.c b/optee_os/core/drivers/bcm_hwrng.c new file mode 100644 index 0000000..d9351c1 --- /dev/null +++ b/optee_os/core/drivers/bcm_hwrng.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* Registers */ +#define RNG_CTRL_OFFSET 0x00 +#define RNG_CTRL_MASK 0x00001fff +#define RNG_CTRL_DISABLE 0x00000000 +#define RNG_CTRL_ENABLE 0x00000001 + +#define RNG_SOFT_RESET_OFFSET 0x04 +#define RNG_SOFT_RESET_MASK 0x00000001 + +#define RNG_FIFO_DATA_OFFSET 0x20 + +#define RNG_FIFO_COUNT_OFFSET 0x24 + +#define RNG_FIFO_COUNT_MASK 0x000000ff +#define RNG_TIMEOUT_US 10000 + +static vaddr_t bcm_hwrng_base; + +static void bcm_hwrng_reset(void) +{ + /* Disable RBG */ + io_clrsetbits32(bcm_hwrng_base + RNG_CTRL_OFFSET, + RNG_CTRL_MASK, RNG_CTRL_DISABLE); + /* Reset RNG and RBG */ + io_setbits32(bcm_hwrng_base + + RNG_SOFT_RESET_OFFSET, RNG_SOFT_RESET_MASK); + io_clrbits32(bcm_hwrng_base + + RNG_SOFT_RESET_OFFSET, RNG_SOFT_RESET_MASK); + /* Enable RBG */ + io_clrsetbits32(bcm_hwrng_base + RNG_CTRL_OFFSET, + RNG_CTRL_MASK, RNG_CTRL_ENABLE); +} + +uint32_t bcm_hwrng_read_rng(uint32_t *p_out, uint32_t words_to_read) +{ + uint32_t available_words = 0; + uint32_t num_words = 0; + uint32_t i = 0; + uint64_t timeout = timeout_init_us(RNG_TIMEOUT_US); + + assert(bcm_hwrng_base); + + do { + available_words = io_read32(bcm_hwrng_base + + RNG_FIFO_COUNT_OFFSET); + available_words = available_words & RNG_FIFO_COUNT_MASK; + } while (!available_words && !timeout_elapsed(timeout)); + + if ((available_words > 0) && (words_to_read > 0)) { + num_words = MIN(available_words, words_to_read); + for (i = 0; i < num_words; i++) + p_out[i] = io_read32(bcm_hwrng_base + + RNG_FIFO_DATA_OFFSET); + } + + return num_words; +} + +static TEE_Result bcm_hwrng_init(void) +{ + bcm_hwrng_base = (vaddr_t)phys_to_virt(HWRNG_BASE, MEM_AREA_IO_SEC, + HWRNG_END - HWRNG_BASE); + + bcm_hwrng_reset(); + + DMSG("bcm_hwrng init done\n"); + return TEE_SUCCESS; +} + +driver_init(bcm_hwrng_init); diff --git a/optee_os/core/drivers/bcm_sotp.c b/optee_os/core/drivers/bcm_sotp.c new file mode 100644 index 0000000..75199e4 --- /dev/null +++ b/optee_os/core/drivers/bcm_sotp.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define SOTP_PROG_CONTROL 0x00 +#define SOTP_WRDATA_0 0x04 +#define SOTP_WRDATA_1 0x08 +#define SOTP_ADDR 0x0c +#define SOTP_CTRL_0 0x10 +#define SOTP_STAT_0 0x18 +#define SOTP_STATUS_1 0x1c +#define SOTP_RDDATA_0 0x20 +#define SOTP_RDDATA_1 0x24 +#define SOTP_REGS_SOTP_CHIP_STATES 0x28 +#define SOTP_REGS_OTP_WR_LOCK 0x38 +#define SOTP_CHIP_CTRL 0x4c + +#define SOTP_PROG_CONTROL__OTP_CPU_MODE_EN BIT(15) +#define SOTP_PROG_CONTROL__OTP_DISABLE_ECC BIT(9) +#define SOTP_ADDR__OTP_ROW_ADDR_R 6 +#define SOTP_PROG_CONTROL__OTP_ECC_WREN BIT(8) +#define SOTP_CTRL_0__START 1 +#define SOTP_STATUS_0__FDONE BIT(3) +#define SOTP_STATUS_1__CMD_DONE BIT(1) +#define SOTP_STATUS_1__ECC_DET BIT(17) + +#define SOTP_READ 0 +#define SOTP_ADDR_MASK 0x3ff +#define SOTP_TIMEOUT_US 300 + +#define SOTP_PROG_WORD 10 +#define SOTP_STATUS__PROGOK BIT(2) +#define SOTP_PROG_ENABLE 2 + +#define SOTP_ROW_DATA_MASK UINT32_MAX +#define SOTP_ECC_ERR_BITS_MASK GENMASK_64(40, 32) + +#define SOTP_CHIP_CTRL_SW_OVERRIDE_CHIP_STATES 4 +#define SOTP_CHIP_CTRL_SW_MANU_PROG 5 +#define SOTP_CHIP_CTRL_SW_CID_PROG 6 +#define SOTP_CHIP_CTRL_SW_AB_DEVICE 8 +#define SOTP_CHIP_CTRL_SW_AB_DEV_MODE 9 +#define CHIP_STATE_UNPROGRAMMED 0x1 +#define CHIP_STATE_UNASSIGNED 0x2 +#define CHIP_STATE_DEFAULT (CHIP_STATE_UNASSIGNED | \ + CHIP_STATE_UNPROGRAMMED) + +static vaddr_t bcm_sotp_base; + +static TEE_Result otp_status_done_wait(vaddr_t addr, uint32_t bit) +{ + uint64_t timeout = timeout_init_us(SOTP_TIMEOUT_US); + + while (!(io_read32(addr) & bit)) + if (timeout_elapsed(timeout)) + return TEE_ERROR_BUSY; + return TEE_SUCCESS; +} + +TEE_Result bcm_iproc_sotp_mem_read(uint32_t row_addr, bool sotp_add_ecc, + uint64_t *rdata) +{ + uint64_t read_data = 0; + uint32_t reg_val = 0; + TEE_Result ret = TEE_SUCCESS; + + assert(bcm_sotp_base); + /* Check for FDONE status */ + ret = otp_status_done_wait((bcm_sotp_base + SOTP_STAT_0), + SOTP_STATUS_0__FDONE); + if (ret) { + EMSG("FDONE status done wait failed and returned %#"PRIx32, + ret); + return ret; + } + + /* Enable OTP access by CPU */ + io_setbits32((bcm_sotp_base + SOTP_PROG_CONTROL), + SOTP_PROG_CONTROL__OTP_CPU_MODE_EN); + + /* ROWS does not support ECC */ + if (row_addr <= SOTP_NO_ECC_ROWS) + sotp_add_ecc = false; + + if (sotp_add_ecc) { + io_clrbits32((bcm_sotp_base + SOTP_PROG_CONTROL), + SOTP_PROG_CONTROL__OTP_DISABLE_ECC); + } else { + io_setbits32((bcm_sotp_base + SOTP_PROG_CONTROL), + SOTP_PROG_CONTROL__OTP_DISABLE_ECC); + } + + /* 10 bit row address */ + reg_val = (row_addr & SOTP_ADDR_MASK) << SOTP_ADDR__OTP_ROW_ADDR_R; + io_write32((bcm_sotp_base + SOTP_ADDR), reg_val); + reg_val = SOTP_READ; + io_write32((bcm_sotp_base + SOTP_CTRL_0), reg_val); + + /* Start bit to tell SOTP to send command to the OTP controller */ + io_setbits32((bcm_sotp_base + SOTP_CTRL_0), SOTP_CTRL_0__START); + + /* Wait for SOTP command done to be set */ + ret = otp_status_done_wait((bcm_sotp_base + SOTP_STAT_0), + SOTP_STATUS_1__CMD_DONE); + if (ret) { + EMSG("FDONE cmd done wait failed and returned %#"PRIx32, ret); + return ret; + } + + DMSG("CMD Done"); + + /* Clr Start bit after command done */ + io_clrbits32((bcm_sotp_base + SOTP_CTRL_0), SOTP_CTRL_0__START); + read_data = io_read32(bcm_sotp_base + SOTP_RDDATA_1); + read_data = ((read_data & 0x1ff) << 32); + read_data |= io_read32(bcm_sotp_base + SOTP_RDDATA_0); + + reg_val = io_read32(bcm_sotp_base + SOTP_STATUS_1); + /* No ECC check till SOTP_NO_ECC_ROWS */ + if (row_addr > SOTP_NO_ECC_ROWS && + reg_val & SOTP_STATUS_1__ECC_DET) { + EMSG("SOTP ECC ERROR Detected ROW %"PRIu32, row_addr); + read_data = SOTP_ECC_ERR_DETECT; + } + + /* Command done is cleared */ + io_setbits32((bcm_sotp_base + SOTP_STATUS_1), SOTP_STATUS_1__CMD_DONE); + io_clrbits32((bcm_sotp_base + SOTP_PROG_CONTROL), + SOTP_PROG_CONTROL__OTP_CPU_MODE_EN); + DMSG("read done"); + + *rdata = read_data; + return ret; +} + +TEE_Result bcm_iproc_sotp_mem_write(uint32_t row_addr, bool sotp_add_ecc, + uint64_t wdata) +{ + uint32_t chip_state = 0; + uint32_t chip_ctrl_default = 0; + uint32_t chip_ctrl = 0; + uint32_t loop = 0; + uint8_t prog_array[4] = { 0x0F, 0x04, 0x08, 0x0D }; + TEE_Result ret = TEE_SUCCESS; + + assert(bcm_sotp_base); + + chip_state = io_read32(bcm_sotp_base + SOTP_REGS_SOTP_CHIP_STATES); + + if (chip_state & CHIP_STATE_DEFAULT) { + chip_ctrl_default = io_read32(bcm_sotp_base + SOTP_CHIP_CTRL); + DMSG("SOTP: enable special prog mode"); + + chip_ctrl = BIT(SOTP_CHIP_CTRL_SW_OVERRIDE_CHIP_STATES) | + BIT(SOTP_CHIP_CTRL_SW_MANU_PROG) | + BIT(SOTP_CHIP_CTRL_SW_CID_PROG) | + BIT(SOTP_CHIP_CTRL_SW_AB_DEVICE); + + io_write32(bcm_sotp_base + SOTP_CHIP_CTRL, chip_ctrl); + } + + /* Check for FDONE status */ + ret = otp_status_done_wait(bcm_sotp_base + SOTP_STAT_0, + SOTP_STATUS_0__FDONE); + if (ret) { + EMSG("FDONE status done wait failed and returned %#"PRIx32, + ret); + return ret; + } + + /* Enable OTP access by CPU */ + io_setbits32(bcm_sotp_base + SOTP_PROG_CONTROL, + SOTP_PROG_CONTROL__OTP_CPU_MODE_EN); + + if (row_addr <= SOTP_NO_ECC_ROWS) { + if (sotp_add_ecc) { + io_setbits32(bcm_sotp_base + SOTP_PROG_CONTROL, + SOTP_PROG_CONTROL__OTP_ECC_WREN); + } else { + io_clrbits32(bcm_sotp_base + SOTP_PROG_CONTROL, + SOTP_PROG_CONTROL__OTP_ECC_WREN); + } + } else { + io_clrbits32(bcm_sotp_base + SOTP_PROG_CONTROL, + SOTP_PROG_CONTROL__OTP_ECC_WREN); + } + + io_write32(bcm_sotp_base + SOTP_CTRL_0, SOTP_PROG_ENABLE << 1); + + /* + * In order to avoid unintentional writes/programming of the OTP array, + * the OTP Controller must be put into programming mode before it will + * accept program commands. This is done by writing 0xF, 0x4, 0x8, 0xD + * with program commands prior to starting the actual programming + * sequence. + */ + for (loop = 0; loop < ARRAY_SIZE(prog_array); loop++) { + io_write32(bcm_sotp_base + SOTP_WRDATA_0, prog_array[loop]); + + /* Bit to tell SOTP to send command to the OTP controller */ + io_setbits32(bcm_sotp_base + SOTP_CTRL_0, SOTP_CTRL_0__START); + + /* Wait for SOTP command done to be set */ + ret = otp_status_done_wait(bcm_sotp_base + SOTP_STATUS_1, + SOTP_STATUS_1__CMD_DONE); + if (ret) { + EMSG("FDONE cmd done wait failed and returned %"PRIx32, + ret); + return ret; + } + + /* Command done is cleared w1c */ + io_setbits32(bcm_sotp_base + SOTP_STATUS_1, + SOTP_STATUS_1__CMD_DONE); + + /* Clear Start bit after command done */ + io_clrbits32(bcm_sotp_base + SOTP_CTRL_0, SOTP_CTRL_0__START); + } + + /* Check for PROGOK */ + ret = otp_status_done_wait(bcm_sotp_base + SOTP_STAT_0, + SOTP_STATUS__PROGOK); + if (ret) { + EMSG("PROGOK cmd wait failed and returned %#"PRIx32, ret); + return ret; + } + + /* Set 10 bit row address */ + io_write32(bcm_sotp_base + SOTP_ADDR, + (row_addr & SOTP_ADDR_MASK) << SOTP_ADDR__OTP_ROW_ADDR_R); + + /* Set SOTP Row data */ + io_write32(bcm_sotp_base + SOTP_WRDATA_0, wdata & SOTP_ROW_DATA_MASK); + + /* Set SOTP ECC and error bits */ + io_write32(bcm_sotp_base + SOTP_WRDATA_1, + (wdata & SOTP_ECC_ERR_BITS_MASK) >> 32); + + /* Set prog_word command */ + io_write32(bcm_sotp_base + SOTP_CTRL_0, SOTP_PROG_WORD << 1); + + /* Start bit to tell SOTP to send command to the OTP controller */ + io_setbits32(bcm_sotp_base + SOTP_CTRL_0, SOTP_CTRL_0__START); + + /* Wait for SOTP command done to be set */ + ret = otp_status_done_wait(bcm_sotp_base + SOTP_STATUS_1, + SOTP_STATUS_1__CMD_DONE); + if (ret) { + EMSG("CMD DONE wait failed and returned %#"PRIx32, ret); + return ret; + } + + /* Command done is cleared w1c */ + io_setbits32(bcm_sotp_base + SOTP_STATUS_1, SOTP_STATUS_1__CMD_DONE); + + /* disable OTP access by CPU */ + io_clrbits32(bcm_sotp_base + SOTP_PROG_CONTROL, + SOTP_PROG_CONTROL__OTP_CPU_MODE_EN); + + /* Clr Start bit after command done */ + io_clrbits32(bcm_sotp_base + SOTP_CTRL_0, SOTP_CTRL_0__START); + + if (chip_state & CHIP_STATE_DEFAULT) + io_write32(bcm_sotp_base + SOTP_CHIP_CTRL, chip_ctrl_default); + + return TEE_SUCCESS; +} + +static TEE_Result bcm_sotp_init(void) +{ + bcm_sotp_base = (vaddr_t)phys_to_virt(SOTP_BASE, MEM_AREA_IO_SEC, 1); + + DMSG("bcm_sotp init done"); + return TEE_SUCCESS; +} + +service_init(bcm_sotp_init); diff --git a/optee_os/core/drivers/bnxt/bnxt.c b/optee_os/core/drivers/bnxt/bnxt.c new file mode 100644 index 0000000..ae613dc --- /dev/null +++ b/optee_os/core/drivers/bnxt/bnxt.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define BNXT_REG_CTRL_BASE 0x3040000 +#define BNXT_REG_ECO_RESERVED 0x3042400 +#define BNXT_FLASH_ACCESS_DONE_BIT 2 +#define NIC400_BNXT_IDM_IO_CONTROL_DIRECT 0x60e00408 +#define BNXT_INDIRECT_BASE 0x60800000 +#define BNXT_INDIRECT_ADDR_MASK 0x3fffffu +#define BNXT_INDIRECT_BASE_MASK (~BNXT_INDIRECT_ADDR_MASK) +#define BNXT_INDIRECT_WINDOW_SIZE (BNXT_INDIRECT_ADDR_MASK + 1) +#define BNXT_REG_CTRL_BPE_MODE_REG 0x0 +#define BNXT_REG_CTRL_BPE_MODE_FASTBOOT_MODE_BIT 2 +#define BNXT_REG_CTRL_BPE_MODE_CM3_RST_BIT 1 +#define BNXT_REG_CTRL_BPE_STAT_REG 0x4 +#define BNXT_REG_CTRL_FSTBOOT_PTR_REG 0x8 +#define BNXT_ERROR_MASK 0xf0000000 +#define BNXT_CTRL_ADDR(x) (BNXT_REG_CTRL_BASE + (x)) +#define BNXT_HANDSHAKE_TIMEOUT_MS 1000 + +#define KONG_REG_CTRL_MODE_REG 0x03900000 +#define KONG_REG_CTRL_MODE_CPUHALT_N_BIT 0 + +#define BNXT_STICKY_BYTE_POR 0x04030088 +#define BNXT_STICKY_BYTE_POR_MHB_BIT 4 + +#define BNXT_HEALTH_CHECK_REG 0x03100008 + +enum bnxt_handshake_sts { + BNXT_HANDSHAKE_SUCCESS = 0, + BNXT_HANDSHAKE_WAIT_ERROR, + BNXT_HANDSHAKE_WAIT_TIMEOUT +}; + +static vaddr_t bnxt_access_window_virt_addr; +static vaddr_t bnxt_indirect_dest_addr; + +static void bnxt_prepare_access_window(uint32_t addr) +{ + addr &= BNXT_INDIRECT_BASE_MASK; + io_write32(bnxt_access_window_virt_addr, addr); +} + +static vaddr_t bnxt_indirect_tgt_addr(uint32_t addr) +{ + addr &= BNXT_INDIRECT_ADDR_MASK; + return (vaddr_t)(bnxt_indirect_dest_addr + addr); +} + +uint32_t bnxt_write32_multiple(uintptr_t dst, + uintptr_t src, + uint32_t num_entries, + int src_4byte_increment) +{ + uint32_t i = 0; + vaddr_t target = 0; + + if (num_entries == 0) + return 0; + + /* Only write up to the next window boundary */ + if ((dst & BNXT_INDIRECT_BASE_MASK) != + ((dst + num_entries * sizeof(uint32_t)) & BNXT_INDIRECT_BASE_MASK)) + num_entries = (((dst + BNXT_INDIRECT_WINDOW_SIZE) & + BNXT_INDIRECT_BASE_MASK) - + dst) / + sizeof(uint32_t); + + bnxt_prepare_access_window(dst); + target = bnxt_indirect_tgt_addr(dst); + for (i = 0; i < num_entries; i++) { + io_write32(target, *(uint32_t *)src); + target += sizeof(uint32_t); + if (src_4byte_increment) + src += sizeof(uint32_t); + } + + return num_entries; +} + +static uint32_t bnxt_read(uint32_t addr) +{ + bnxt_prepare_access_window(addr); + return io_read32(bnxt_indirect_tgt_addr(addr)); +} + +static uint32_t bnxt_read_ctrl(uint32_t offset) +{ + return bnxt_read(BNXT_CTRL_ADDR(offset)); +} + +static void bnxt_write(uint32_t addr, uint32_t value) +{ + bnxt_prepare_access_window(addr); + io_write32(bnxt_indirect_tgt_addr(addr), value); +} + +static void bnxt_write_ctrl(uint32_t offset, uint32_t value) +{ + bnxt_write(BNXT_CTRL_ADDR(offset), value); +} + +void bnxt_handshake_clear(void) +{ + uint32_t value = bnxt_read(BNXT_REG_ECO_RESERVED); + + value = value & ~BIT(BNXT_FLASH_ACCESS_DONE_BIT); + bnxt_write(BNXT_REG_ECO_RESERVED, value); +} + +static int bnxt_handshake_done(void) +{ + uint32_t value = 0; + + value = bnxt_read(BNXT_REG_ECO_RESERVED); + value &= BIT(BNXT_FLASH_ACCESS_DONE_BIT); + + return value != 0; +} + +uint32_t bnxt_wait_handshake(uint32_t max_timeout) +{ + int ret = 0; + uint32_t status = 0; + uint32_t timeout = 0; + + /* If no timeout given we go with max timeout */ + if (max_timeout == 0) + max_timeout = BNXT_HANDSHAKE_TIMEOUT_MS; + + timeout = max_timeout; + + DMSG("Waiting for ChiMP handshake..."); + do { + if (bnxt_handshake_done()) { + ret = BNXT_HANDSHAKE_SUCCESS; + break; + } + /* No need to wait if ChiMP reported an error */ + status = bnxt_read_ctrl(BNXT_REG_CTRL_BPE_STAT_REG); + if (status & BNXT_ERROR_MASK) { + EMSG("ChiMP error 0x%x. Wait aborted", status); + ret = BNXT_HANDSHAKE_WAIT_ERROR; + break; + } + mdelay(1); + } while (--timeout); + + if (!bnxt_handshake_done()) { + if (timeout == 0) { + ret = BNXT_HANDSHAKE_WAIT_TIMEOUT; + EMSG("Timeout waiting for ChiMP handshake"); + } + } else { + ret = BNXT_HANDSHAKE_SUCCESS; + DMSG("ChiMP handshake successful"); + } + + return ret; +} + +void bnxt_chimp_halt(void) +{ + uint32_t value = 0; + + value = bnxt_read_ctrl(BNXT_REG_CTRL_BPE_MODE_REG); + value |= BIT(BNXT_REG_CTRL_BPE_MODE_CM3_RST_BIT); + bnxt_write_ctrl(BNXT_REG_CTRL_BPE_MODE_REG, value); +} + +void bnxt_kong_halt(void) +{ + uint32_t value = 0; + + value = bnxt_read(KONG_REG_CTRL_MODE_REG); + value &= ~BIT(KONG_REG_CTRL_MODE_CPUHALT_N_BIT); + bnxt_write(KONG_REG_CTRL_MODE_REG, value); +} + +int bnxt_fastboot(uintptr_t addr) +{ + uint32_t value = 0; + + value = bnxt_read(BNXT_STICKY_BYTE_POR); + value |= BIT(BNXT_STICKY_BYTE_POR_MHB_BIT); + bnxt_write(BNXT_STICKY_BYTE_POR, value); + + /* Set the fastboot address and type */ + bnxt_write_ctrl(BNXT_REG_CTRL_FSTBOOT_PTR_REG, addr); + + /* Set fastboot mode & take BNXT CPU1 out of reset */ + value = bnxt_read_ctrl(BNXT_REG_CTRL_BPE_MODE_REG); + value |= BIT(BNXT_REG_CTRL_BPE_MODE_FASTBOOT_MODE_BIT); + value &= ~BIT(BNXT_REG_CTRL_BPE_MODE_CM3_RST_BIT); + bnxt_write_ctrl(BNXT_REG_CTRL_BPE_MODE_REG, value); + + return 0; +} + +uint32_t bnxt_health_status(void) +{ + return bnxt_read(BNXT_HEALTH_CHECK_REG); +} + +static TEE_Result bnxt_init(void) +{ + bnxt_access_window_virt_addr = + (vaddr_t)phys_to_virt(NIC400_BNXT_IDM_IO_CONTROL_DIRECT, + MEM_AREA_IO_SEC, sizeof(uint32_t)); + bnxt_indirect_dest_addr = + (vaddr_t)phys_to_virt(BNXT_INDIRECT_BASE, + MEM_AREA_IO_SEC, + BNXT_INDIRECT_WINDOW_SIZE); + return TEE_SUCCESS; +} +driver_init(bnxt_init); diff --git a/optee_os/core/drivers/bnxt/bnxt_fw.c b/optee_os/core/drivers/bnxt/bnxt_fw.c new file mode 100644 index 0000000..d4c9bab --- /dev/null +++ b/optee_os/core/drivers/bnxt/bnxt_fw.c @@ -0,0 +1,697 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* + * These macros are the offsets where images reside on sec mem + */ +#define BNXT_BUFFER_SEC_MEM 0x8ae00000 +#define BNXT_FW_SEC_MEM_SRC BNXT_BUFFER_SEC_MEM +#define BNXT_FW_SEC_MEM_CFG (BNXT_BUFFER_SEC_MEM + 0x100000) +#define TEMP_MEM (BNXT_BUFFER_SEC_MEM + 0x180000) + +#define BNXT_CRASH_SEC_MEM 0x8b000000 +#define BNXT_CRASH_LEN 0x2000000 + +#define BNXT_CONFIG_NS3_DEST 0x03a00000 +#define BNXT_BSPD_CFG_OFFSET 0x51b0 +#define BNXT_CONFIG_NS3_BSPD_DEST (BNXT_CONFIG_NS3_DEST + \ + BNXT_BSPD_CFG_OFFSET) +#define BNXT_BSPD_CFG_SIZE 0x200 + +#define BNXT_CRASH_DUMP_INFO_NS3_BASE 0x3a5ff00 + +#define SZ_1K 0x400 + +#define BUFFER_PADDING SZ_1K + +#define INC_SRC_ADDR 1 + +#define EOF -1 + +#define BCM_BNXT_FASTBOOT_MASK 0x3u +#define BCM_BNXT_FASTBOOT_TYPE_1 1 + +#define ADDR_IS_4BYTE_ALIGNED(addr) IS_ALIGNED(addr, 4) + +#define SECTION_IS_LOADABLE(section_ptr) \ + ((section_ptr)->flags_src_offset & SECTION_FLAGS_IS_LOADABLE) +#define SECTION_IS_ZIPPED(section_ptr) \ + ((section_ptr)->flags_src_offset & SECTION_FLAGS_IS_ZIPPED) +#define SECTION_IS_TOBE_COPIED(section_ptr) \ + ((section_ptr)->flags_src_offset & \ + (SECTION_FLAGS_IS_EXEC_INSTR | SECTION_FLAGS_IS_DATA)) +#define SECTION_IS_TOBE_ZEROED(section_ptr) \ + ((section_ptr)->flags_src_offset & SECTION_FLAGS_IS_BSS) +#define SECTION_IS_4BYTE_ALIGNED(section_ptr) \ + ADDR_IS_4BYTE_ALIGNED((section_ptr)->dest_addr) + +#define SECTION_SRC_OFFSET(section_ptr) \ + ((section_ptr)->flags_src_offset & SECTION_SRC_OFFFSET_MASK) + +/* -------------------------------------------------------------------------- */ + +/* Section header for each image block */ +struct ape_section_hdr_s { + /* Destination address that this section is to be copied to */ + uint32_t dest_addr; + + /* + * bit[0:23] source offset address that this image copy from + * bit[24:31] flags + */ + uint32_t flags_src_offset; +#define SECTION_FLAGS_MASK 0xff000000 + /* Session is compressed (zipped) */ +#define SECTION_FLAGS_IS_ZIPPED 0x01000000 + /* Session contains CRC */ +#define SECTION_FLAGS_IS_CRC 0x02000000 + /* Session contains executable code (e.g. .text) */ +#define SECTION_FLAGS_IS_EXEC_INSTR 0x04000000 + /* Session contains initialized data (e.g. .data) */ +#define SECTION_FLAGS_IS_DATA 0x08000000 + /* Session contains zero initialized data (e.g. .bss) */ +#define SECTION_FLAGS_IS_BSS 0x10000000 + /* Loadable section mask */ +#define SECTION_FLAGS_IS_LOADABLE (SECTION_FLAGS_IS_EXEC_INSTR | \ + SECTION_FLAGS_IS_DATA | \ + SECTION_FLAGS_IS_BSS) +#define SECTION_SRC_OFFFSET_MASK 0x00ffffff + + /* Original image length, dword (4byte) length */ + uint32_t org_data_len; + + /* Compressed image length (if FlAGS_IS_ZIPPED is set) */ + uint32_t zip_data_len; + + /* + * checksum value for this image block, if FLAGS_IS_CRC then + * this is CRC checksum; otherwise it is a simple summation + */ + uint32_t checksum; +}; + +struct version_s { + uint8_t version[16]; /* Null-terminated file version string */ +}; + +struct ver_ext_offset_s { + uint8_t version[12]; /* Null-terminated file version string */ + uint32_t ext_hdr_offset; +}; + +union version_and_offset_u { + struct version_s version1; + struct ver_ext_offset_s version2; +}; + +struct ape_bin_hdr_s { + /* APE binary header signature; expects APE_BIN_HDR_SIGNATURE */ + uint32_t signature; +#define APE_BIN_HDR_SIGNATURE 0x1a4d4342 /* "BCM"+0x1a */ + /* Reserved for ChiMP's use */ + uint8_t flags; + uint8_t code_type; + uint8_t device; + uint8_t media; + union version_and_offset_u ver; + uint8_t build; + uint8_t revision; + uint8_t minor_ver; + uint8_t major_ver; + uint32_t entry_address; + uint8_t reserved; + uint8_t header_dword_size; + uint8_t num_total_sections; + uint8_t num_loadable_sections; + uint32_t checksum; +} __packed __aligned(1); + +#define APE_BIN_HDR_SIZE sizeof(struct ape_bin_hdr_s) +#define APE_SECTION_HDR_SIZE sizeof(struct ape_section_hdr_s) + +/* MAX number of image sections that will be accepted */ +#define APE_IMG_MAX_SECTIONS 16 + +#define APE_IMG_LOAD_DEBUG 0 + +/* -------------------------------------------------------------------------- */ + +struct ape_mem_region_s { + uint32_t c_base; /* ChiMP's view of address */ + uint32_t h_base; /* Host's view of address */ + uint32_t size; /* Size in bytes */ +}; + +/* Memory map into various scratchpad memories */ +static struct ape_mem_region_s ape_mem_regions[] = { + /* CHIMP scratchpad */ + {0x00100000, 0x03100000, 1024 * SZ_1K}, + + /* APE scratchpad */ + {0x61000000, 0x03300000, 1152 * SZ_1K}, + + /* BONO scratchpad */ + {0x61600000, 0x03a00000, 512 * SZ_1K}, + + /* KONG scratchpad */ + {0x61400000, 0x03800000, 512 * SZ_1K}, + + /* Keep this last!! */ + {0, 0, 0} +}; + +/* Nitro crash address configuration related macros */ +#define BNXT_CRASH_INFO_SIGNATURE 0x20524444 +#define BNXT_CRASH_INFO_VALID 0x1 +#define MAX_CRASH_ADDR_ITEM 8 + +struct nitro_crash_addr_item { + uint32_t info; + uint32_t size; + uint32_t addr_hi; + uint32_t addr_lo; +}; + +struct nitro_crash_addr_info { + /* CRC of the struct content, starting at next field. */ + uint32_t crc; + uint32_t signature; + uint32_t version; + struct nitro_crash_addr_item table[MAX_CRASH_ADDR_ITEM]; +}; + +static inline void memcpy32_helper(uintptr_t src, + uintptr_t dst, + uint32_t entries, + int inc_src_addr) +{ + uint32_t copied_entries = 0; + + while (entries) { + copied_entries = bnxt_write32_multiple(dst, src, entries, + inc_src_addr); + + if (copied_entries < entries) { + dst += copied_entries * sizeof(uint32_t); + src += (inc_src_addr) ? + (copied_entries * sizeof(uint32_t)) : 0; + entries -= copied_entries; + } else { + entries = 0; + } + } +} + +static uint32_t ape_host_view_addr_get(uint32_t bnxt_view_addr, uint32_t size) +{ + struct ape_mem_region_s *region = ape_mem_regions; + uint32_t addr = 0; + + for (; region->size != 0; region++) { + if (bnxt_view_addr < region->c_base) + continue; + + if (bnxt_view_addr >= (region->c_base + region->size)) + continue; + + if (size > (region->c_base + region->size - bnxt_view_addr)) { + EMSG("ERROR: 0x%x + 0x%x spans memory boundary", + bnxt_view_addr, size); + break; + } + + addr = bnxt_view_addr - region->c_base; + addr += region->h_base; + break; + } + + return addr; +} + +static uint32_t ape_hdr_crc_calc(const struct ape_bin_hdr_s *hdr) +{ + uint32_t crc = 0; + uint32_t dummy = 0; + + /* Compute the CRC up to, but not including, the checksum field */ + crc = CRC32(CRC32_INIT_VAL, + (const char *)hdr, + (uintptr_t)(&hdr->checksum) - (uintptr_t)hdr); + + /* Compute the CRC with the checksum field zeroed out */ + crc = CRC32(~crc, (const char *)&dummy, sizeof(uint32_t)); + + /* + * Compute the remainder part of the image header, i.e., the + * section headers + */ + crc = CRC32(~crc, + (const char *)((uintptr_t)hdr + APE_BIN_HDR_SIZE), + hdr->num_total_sections * APE_SECTION_HDR_SIZE); + + return crc; +} + +static int ape_bin_hdr_valid(const struct ape_bin_hdr_s *hdr) +{ + uint32_t checksum = 0; + + if (!hdr) { + EMSG("ERROR: no APE image header"); + return BNXT_FAILURE; + } + + if (hdr->signature != APE_BIN_HDR_SIGNATURE) { + EMSG("ERROR: bad APE image signature"); + return BNXT_FAILURE; + } + + if (hdr->num_total_sections > APE_IMG_MAX_SECTIONS) { + EMSG("ERROR: too many sections in APE image"); + return BNXT_FAILURE; + } + + checksum = ape_hdr_crc_calc(hdr); + if (hdr->checksum != checksum) { + EMSG("ERROR: bad APE header checksum (exp: %x, act: %x)", + hdr->checksum, checksum); + return BNXT_FAILURE; + } + + return BNXT_SUCCESS; +} + +static int get_char(uint8_t *inbuf, size_t *inbuf_idx, size_t inbuf_size) +{ + int c = 0; + + if (*inbuf_idx >= inbuf_size) + return EOF; + + c = inbuf[*inbuf_idx]; + *inbuf_idx += 1; + + return c; +} + +static void put_char(uint8_t *outbuf, + size_t *outbuf_idx, + size_t outbuf_size, + uint8_t ch) +{ + if (*outbuf_idx >= outbuf_size) + return; + + outbuf[*outbuf_idx] = ch; + *outbuf_idx += 1; +} + +static size_t ape_section_uncompress(uint8_t *inbuf, + size_t inbuf_size, + uint8_t *outbuf, + size_t outbuf_size) +{ + int i = 0, j = 0, k = 0, r = 0, c = 0; + uint32_t flags = 0; + size_t exp_size = 0, codesize = 0; + size_t inbuf_idx = 0, outbuf_idx = 0; +#define CODE_8U_MASK 0xff00u /* 8 code units count mask (8 bits) */ +#define CODE_END_MASK 0x100u /* End of code units mask */ +#define CODE_IS_UNENCODED_MASK 1 /* Unencoded code unit mask */ +#define CODE_POS_MASK 0xe0u /* Encoded unit position mask and */ +#define CODE_POS_SHIFT 3 /* Bit shift */ +#define CODE_LEN_MASK 0x1fu /* Encoded unit length mask */ +#define NS 2048 /* Size of ring buffer */ +#define F 34 /* Upper limit for match_length */ +#define THRESHOLD 2 /* Encode string into position and + * length, if match_length is + * greater than this. + */ + /* + * Ring buffer of size NS, with an extra F-1 bytes to facilitate + * string comparisons. + */ + uint8_t text_buf[NS + F - 1]; + + inbuf_idx = 0; + outbuf_idx = 0; + + for (i = 0; i < NS - F; i++) + text_buf[i] = ' '; + + r = NS - F; + + for (;;) { + if (((flags >>= 1) & CODE_END_MASK) == 0) { + c = get_char(inbuf, &inbuf_idx, inbuf_size); + if (c == EOF) + break; + ++exp_size; + + if (exp_size > inbuf_size) + break; + + /* Use higher byte cleverly to count to eight */ + flags = c | CODE_8U_MASK; + } + + if (flags & CODE_IS_UNENCODED_MASK) { + /* Not encoded; simply copy the unit */ + c = get_char(inbuf, &inbuf_idx, inbuf_size); + if (c == EOF) + break; + + ++exp_size; + if (exp_size > inbuf_size) + break; + + put_char(outbuf, &outbuf_idx, outbuf_size, c); + text_buf[r++] = c; + r &= (NS - 1); + ++codesize; + } else { + /* Encoded; get the position and length & duplicate */ + i = get_char(inbuf, &inbuf_idx, inbuf_size); + if (i == EOF) + break; + + ++exp_size; + if (exp_size > inbuf_size) + break; + + j = get_char(inbuf, &inbuf_idx, inbuf_size); + if (j == EOF) + break; + + ++exp_size; + if (exp_size > inbuf_size) + break; + + i |= ((j & CODE_POS_MASK) << CODE_POS_SHIFT); + j = ((j & CODE_LEN_MASK) + THRESHOLD); + + for (k = 0; k <= j; k++) { + c = text_buf[((i + k) & (NS - 1))]; + put_char(outbuf, &outbuf_idx, outbuf_size, c); + text_buf[r++] = c; + r &= (NS - 1); + ++codesize; + } + } + } + + return codesize; +} + +static int ape_section_copy(struct ape_bin_hdr_s *bin_hdr, + struct ape_section_hdr_s *section) +{ + uintptr_t src = 0; + uintptr_t dst = 0; + uint32_t checksum = 0; + uint32_t i = 0; + size_t size = 0; + uint8_t *section_data = NULL; + size_t work_buff_size = 0; + void *work_buff = NULL; + int rc = BNXT_FAILURE; + + if (SECTION_IS_ZIPPED(section)) { + work_buff_size = section->org_data_len + BUFFER_PADDING; + work_buff = (void *)phys_to_virt(TEMP_MEM, MEM_AREA_RAM_SEC, + work_buff_size); + if (!work_buff) { + EMSG("ERROR: buffer allocation"); + return BNXT_FAILURE; + } + + section_data = (uint8_t *)((uintptr_t)bin_hdr + + SECTION_SRC_OFFSET(section)); + size = ape_section_uncompress(section_data, + section->zip_data_len, + work_buff, + work_buff_size); + if (size >= work_buff_size) { + EMSG("ERROR: section uncompress"); + goto ape_section_copy_exit; + } + if (size < section->org_data_len) { + EMSG("ERROR: decompressed data size mismatch "); + EMSG("(exp: %d, act: %ld)", + section->org_data_len, size); + goto ape_section_copy_exit; + } + src = (uintptr_t)work_buff; + } else { + src = (uintptr_t)bin_hdr + SECTION_SRC_OFFSET(section); + } + + size = section->org_data_len; + + if (section->flags_src_offset & SECTION_FLAGS_IS_CRC) { + checksum = CRC32(CRC32_INIT_VAL, (const char *)src, size); + } else { + checksum = 0; + for (i = 0; i < size / sizeof(uint32_t); i++) + checksum += ((uint32_t *)src)[i]; + } + if (checksum != section->checksum) { + EMSG("ERROR: checksum mismatch (exp: %x, act: %x)", + section->checksum, checksum); + goto ape_section_copy_exit; + } + + dst = ape_host_view_addr_get(section->dest_addr, size); + if (dst == 0) { + EMSG("ERROR: ChiMP-to-host address conversion of %x", + section->dest_addr); + goto ape_section_copy_exit; + } + + /* Copy the section */ + size = size / sizeof(uint32_t); + memcpy32_helper(src, dst, size, INC_SRC_ADDR); + + rc = BNXT_SUCCESS; + +ape_section_copy_exit: + return rc; +} + +static int ape_section_zero(struct ape_section_hdr_s *section) +{ + uint32_t dst = 0; + uint32_t size = section->org_data_len; + uint32_t zero = 0; + + if (section->org_data_len == 0) + return BNXT_SUCCESS; + + /* Convert ChiMP's view of the address in the image to the host view */ + dst = ape_host_view_addr_get(section->dest_addr, size); + if (dst == 0) { + EMSG("ERROR: ChiMP-to-host address conversion of %x", + section->dest_addr); + return BNXT_FAILURE; + } + + /* + * Zero the section; we simply copy zeros and do not increment the + * source buffer address. + */ + size = size / sizeof(uint32_t); + memcpy32_helper((uintptr_t)&zero, dst, size, !INC_SRC_ADDR); + + return BNXT_SUCCESS; +} + +static int bnxt_load(vaddr_t img_buffer) +{ + struct ape_bin_hdr_s *bin_hdr = NULL; + struct ape_section_hdr_s *section = NULL; + int sidx = 0; + int rc = BNXT_SUCCESS; + + bin_hdr = (struct ape_bin_hdr_s *)img_buffer; + section = (struct ape_section_hdr_s *)(img_buffer + + APE_BIN_HDR_SIZE); + + if (ape_bin_hdr_valid(bin_hdr) != BNXT_SUCCESS) + return BNXT_FAILURE; + + for (sidx = 0; sidx < bin_hdr->num_total_sections; sidx++, section++) { + if (!SECTION_IS_LOADABLE(section)) + continue; + + if (!ADDR_IS_4BYTE_ALIGNED(section->dest_addr)) { + EMSG("ERROR: unaligned section dest address 0x%x", + section->dest_addr); + rc = BNXT_FAILURE; + break; + } + + if (!ADDR_IS_4BYTE_ALIGNED(SECTION_SRC_OFFSET(section))) { + EMSG("ERROR: unaligned section src offset (0x%x)", + SECTION_SRC_OFFSET(section)); + rc = BNXT_FAILURE; + break; + } + + if (section->org_data_len % sizeof(uint32_t)) { + EMSG("ERROR: section size (%d) not divisible by 4", + section->org_data_len); + rc = BNXT_FAILURE; + break; + } + + if (SECTION_IS_TOBE_COPIED(section)) { + rc = ape_section_copy(bin_hdr, section); + if (rc != BNXT_SUCCESS) + break; + } else if (SECTION_IS_TOBE_ZEROED(section)) { + rc = ape_section_zero(section); + if (rc != BNXT_SUCCESS) + break; + } + } + + /* Set up boot mode and take BNXT out of reset */ + if (rc == BNXT_SUCCESS) { + bnxt_fastboot((bin_hdr->entry_address & + ~BCM_BNXT_FASTBOOT_MASK) | + BCM_BNXT_FASTBOOT_TYPE_1); + } + + return rc; +} + +static TEE_Result bnxt_crash_config(uintptr_t info_dst, + uint32_t crash_area_start, + uint32_t crash_len) +{ + struct nitro_crash_addr_item *item = NULL; + uintptr_t dst = 0; + struct nitro_crash_addr_info *info = NULL; + uintptr_t src = 0; + uint32_t crc = 0; + size_t size = 0; + + /* + * First we write into local memory to calculate CRC before + * updating into Nitro memory + */ + info = malloc(sizeof(struct nitro_crash_addr_info)); + if (!info) { + EMSG("ERROR: buffer allocation"); + return TEE_ERROR_OUT_OF_MEMORY; + } + + memset(info, 0, sizeof(struct nitro_crash_addr_info)); + + info->signature = BNXT_CRASH_INFO_SIGNATURE; + info->version = 0x01000000 | MAX_CRASH_ADDR_ITEM; + + /* As of now only one item is filled */ + item = &info->table[0]; + item->info = 0; + item->size = crash_len | BNXT_CRASH_INFO_VALID; + item->addr_hi = 0; + item->addr_lo = crash_area_start; + + /* Checksum calculation */ + crc = CRC32(CRC32_INIT_VAL, + (const char *)info + sizeof(uint32_t), + sizeof(struct nitro_crash_addr_info) - sizeof(uint32_t)); + info->crc = crc; + + /* First we write the contents and then set valid bit */ + item->size &= ~BNXT_CRASH_INFO_VALID; + + size = sizeof(struct nitro_crash_addr_info) / sizeof(uint32_t); + dst = info_dst; + src = (uintptr_t)info; + memcpy32_helper(src, dst, size, INC_SRC_ADDR); + + /* Set the valid bit */ + item->size |= BNXT_CRASH_INFO_VALID; + dst = info_dst + offsetof(struct nitro_crash_addr_info, table) + + offsetof(struct nitro_crash_addr_item, size); + bnxt_write32_multiple(dst, (uintptr_t)&item->size, 1, 1); + + free(info); + + return TEE_SUCCESS; +} + +TEE_Result bnxt_load_fw(int chip_type) +{ + uint32_t size = 0; + uintptr_t dst = 0; + uintptr_t src = 0; + struct bnxt_images_info bnxt_src_image_info; + vaddr_t sec_mem_dest = (vaddr_t)phys_to_virt(BNXT_BUFFER_SEC_MEM, + MEM_AREA_RAM_SEC, 1); + + memset(&bnxt_src_image_info, 0, sizeof(struct bnxt_images_info)); + + if (get_bnxt_images_info(&bnxt_src_image_info, + chip_type, sec_mem_dest) != BNXT_SUCCESS) + return TEE_ERROR_ITEM_NOT_FOUND; + + bnxt_handshake_clear(); + bnxt_kong_halt(); + bnxt_chimp_halt(); + + /* Copy the configs */ + src = (uintptr_t)bnxt_src_image_info.bnxt_cfg_vaddr; + dst = (uintptr_t)BNXT_CONFIG_NS3_DEST; + size = bnxt_src_image_info.bnxt_cfg_len; + size = size / sizeof(uint32_t); + memcpy32_helper(src, dst, size, INC_SRC_ADDR); + + /* Copy bspd config */ + src = (uintptr_t)bnxt_src_image_info.bnxt_bspd_cfg_vaddr; + size = bnxt_src_image_info.bnxt_bspd_cfg_len; + dst = (uintptr_t)BNXT_CONFIG_NS3_BSPD_DEST; + + size = size / sizeof(uint32_t); + memcpy32_helper(src, dst, size, INC_SRC_ADDR); + + /* Fill the bnxt crash dump info */ + bnxt_crash_config((uintptr_t)BNXT_CRASH_DUMP_INFO_NS3_BASE, + BNXT_CRASH_SEC_MEM, + BNXT_CRASH_LEN); + + /* Load bnxt firmware and fastboot */ + bnxt_load(bnxt_src_image_info.bnxt_fw_vaddr); + + return TEE_SUCCESS; +} + +TEE_Result bnxt_copy_crash_dump(uint8_t *d, uint32_t offset, uint32_t len) +{ + size_t crash_len = 0; + void *s = NULL; + + if (ADD_OVERFLOW(offset, len, &crash_len) || + crash_len > BNXT_CRASH_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + s = phys_to_virt(BNXT_CRASH_SEC_MEM + offset, MEM_AREA_RAM_SEC, len); + + cache_op_inner(DCACHE_AREA_INVALIDATE, s, len); + + memcpy(d, s, len); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/bnxt/bnxt_images.c b/optee_os/core/drivers/bnxt/bnxt_images.c new file mode 100644 index 0000000..18584c7 --- /dev/null +++ b/optee_os/core/drivers/bnxt/bnxt_images.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include + +#define BNXT_FW_NS3_IMAGE_SIG 0xFF12345A +#define BNXT_NS3_CFG_IMAGE_SIG 0xCF54321A + +#define BNXT_BSPD_CFG_LEN 512 + +#define QSPI_BASE QSPI_MEM_BASE +#define QSPI_BNXT_IMG (QSPI_BASE + 0x400000) +#define QSPI_BSPD_ADDR (QSPI_BASE + 0x700000) + +#define BCM_NS3 1 + +static void set_bnxt_images_info(struct bnxt_images_info *bnxt_info, + int chip_type, vaddr_t src, vaddr_t dst); + +static struct bnxt_img_header { + uint32_t bnxt_fw_ns3_sig; + uint32_t bnxt_fw_ns3_size; + uint32_t bnxt_ns3_cfg_sig; + uint32_t bnxt_ns3_cfg_size; +} *img_header; + +static int verify_header(vaddr_t mem) +{ + img_header = (struct bnxt_img_header *)mem; + + if (img_header->bnxt_fw_ns3_sig == BNXT_FW_NS3_IMAGE_SIG && + img_header->bnxt_ns3_cfg_sig == BNXT_NS3_CFG_IMAGE_SIG) + return BNXT_SUCCESS; + return BNXT_FAILURE; +} + +static void set_bnxt_images_info(struct bnxt_images_info *bnxt_info, + int chip_type, vaddr_t src, vaddr_t dst) +{ + uint32_t len = 0; + struct bnxt_img_header *dst_header = NULL; + uint32_t fw_image_offset = sizeof(struct bnxt_img_header); + + img_header = (struct bnxt_img_header *)src; + if (dst) { + dst_header = (struct bnxt_img_header *)dst; + memcpy(dst_header, img_header, sizeof(*img_header)); + dst += sizeof(*img_header); + + if (chip_type != BCM_NS3) { + dst_header->bnxt_fw_ns3_size = 0; + dst_header->bnxt_ns3_cfg_size = 0; + } + } + + if (chip_type == BCM_NS3) { + len = img_header->bnxt_fw_ns3_size; + bnxt_info->bnxt_fw_vaddr = src + fw_image_offset; + bnxt_info->bnxt_fw_len = len; + if (dst) { + memcpy((void *)dst, (void *)(src + fw_image_offset), + len); + dst += len; + } + + fw_image_offset += len; + + len = img_header->bnxt_ns3_cfg_size; + bnxt_info->bnxt_cfg_vaddr = src + fw_image_offset; + bnxt_info->bnxt_cfg_len = len; + if (dst) { + memcpy((void *)dst, (void *)(src + fw_image_offset), + len); + } + } +} + +int get_bnxt_images_info(struct bnxt_images_info *bnxt_info, int chip_type, + vaddr_t ddr_dest) +{ + vaddr_t flash_dev_vaddr = 0; + + bnxt_info->bnxt_bspd_cfg_len = BNXT_BSPD_CFG_LEN; + + /* First verify if images are on sec mem */ + if (verify_header(ddr_dest + BNXT_IMG_SECMEM_OFFSET) == BNXT_SUCCESS) { + DMSG("Images found on sec memory"); + + bnxt_info->bnxt_bspd_cfg_vaddr = ddr_dest; + + set_bnxt_images_info(bnxt_info, chip_type, + ddr_dest + BNXT_IMG_SECMEM_OFFSET, 0); + } else { + flash_dev_vaddr = (vaddr_t) + phys_to_virt(QSPI_BNXT_IMG, MEM_AREA_IO_NSEC, + sizeof(struct bnxt_img_header)); + + if (verify_header(flash_dev_vaddr) != BNXT_SUCCESS) { + EMSG("failed to load fw images"); + return BNXT_FAILURE; + } + + DMSG("Images loading from flash memory"); + bnxt_info->bnxt_bspd_cfg_vaddr = + (vaddr_t)phys_to_virt(QSPI_BSPD_ADDR, + MEM_AREA_IO_NSEC, + BNXT_BSPD_CFG_LEN); + memcpy((void *)ddr_dest, (void *)bnxt_info->bnxt_bspd_cfg_vaddr, + BNXT_BSPD_CFG_LEN); + + set_bnxt_images_info(bnxt_info, chip_type, flash_dev_vaddr, + ddr_dest + BNXT_IMG_SECMEM_OFFSET); + } + + return BNXT_SUCCESS; +} diff --git a/optee_os/core/drivers/bnxt/sub.mk b/optee_os/core/drivers/bnxt/sub.mk new file mode 100644 index 0000000..a0021a9 --- /dev/null +++ b/optee_os/core/drivers/bnxt/sub.mk @@ -0,0 +1,3 @@ +srcs-y += bnxt.c +srcs-y += bnxt_fw.c +srcs-y += bnxt_images.c diff --git a/optee_os/core/drivers/cbmem_console.c b/optee_os/core/drivers/cbmem_console.c new file mode 100644 index 0000000..2e526c6 --- /dev/null +++ b/optee_os/core/drivers/cbmem_console.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023, Google Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CURSOR_MASK (BIT(28) - 1) +#define OVERFLOW BIT(31) + +struct cbmem_console { + uint32_t size; + uint32_t cursor; + uint8_t body[0]; +}; + +struct cbmem_console_data { + paddr_t base; + struct cbmem_console *console; + struct serial_chip chip; + uint32_t size; +}; + +/* + * Structures describing coreboot's in-memory descriptor tables. See + * https://github.com/coreboot/coreboot/blob/ea2a38be323173075db3b13729a4006ea1fef72d/src/commonlib/include/commonlib/coreboot_tables.h + * for canonical implementation. + */ + +struct cb_header { + uint8_t signature[4]; + uint32_t header_bytes; + uint32_t header_checksum; + uint32_t table_bytes; + uint32_t table_checksum; + uint32_t table_entries; +}; + +#define CB_TAG_CBMEM_CONSOLE 0x17 + +struct cb_entry { + uint32_t tag; + uint32_t size; + uint64_t value; +}; + +static struct cbmem_console_data cbmem_console; + +static void cbmem_console_flush(struct serial_chip *chip __unused) +{ +} + +static int cbmem_console_getchar(struct serial_chip *chip __unused) +{ + return 0; +} + +static bool cbmem_console_have_rx_data(struct serial_chip *chip __unused) +{ + return false; +} + +static void cbmem_console_putc(struct serial_chip *chip, int ch) +{ + struct cbmem_console_data *pd = + container_of(chip, struct cbmem_console_data, chip); + struct cbmem_console *c = pd->console; + + if (!pd->size) + return; + + if ((c->cursor & CURSOR_MASK) + 1 >= pd->size) { + c->cursor &= ~CURSOR_MASK; + c->cursor |= OVERFLOW; + c->body[0] = (uint8_t)(ch & 0xFF); + } else { + c->body[c->cursor & CURSOR_MASK] = (uint8_t)(ch & 0xFF); + c->cursor++; + } +} + +static const struct serial_ops cbmem_console_ops = { + .flush = cbmem_console_flush, + .getchar = cbmem_console_getchar, + .have_rx_data = cbmem_console_have_rx_data, + .putc = cbmem_console_putc, +}; +DECLARE_KEEP_PAGER(cbmem_console_ops); + +static paddr_t get_cbmem_console_from_coreboot_table(paddr_t table_addr, + size_t table_size) +{ + struct cb_header *header = NULL; + void *ptr = NULL; + uint32_t i = 0; + struct cb_entry *entry = NULL; + paddr_t cbmem_console_base = 0; + void *base = NULL; + + base = core_mmu_add_mapping(MEM_AREA_RAM_NSEC, table_addr, table_size); + if (!base) + return 0; + + header = (struct cb_header *)base; + if (memcmp(header->signature, "LBIO", 4)) + goto done; + + if (header->header_bytes + header->table_bytes > table_size) + goto done; + + ptr = (uint8_t *)base + header->header_bytes; + for (i = 0; i < header->table_entries; ++i) { + entry = (struct cb_entry *)ptr; + if ((uint8_t *)ptr >= (uint8_t *)base + table_size - + sizeof(struct cb_entry)) { + goto done; + } + + switch (get_le32(&entry->tag)) { + case CB_TAG_CBMEM_CONSOLE: + cbmem_console_base = get_le64(&entry->value); + goto done; + default: + /* We skip all but one tag type. */ + break; + } + + ptr = (uint8_t *)ptr + get_le32(&entry->size); + } + +done: + core_mmu_remove_mapping(MEM_AREA_RAM_NSEC, base, table_size); + return cbmem_console_base; +} + +bool cbmem_console_init_from_dt(void *fdt) +{ + int offset = 0; + paddr_t cb_addr = 0; + size_t cb_size = 0; + paddr_t cbmem_console_base = 0; + + if (!fdt) + return false; + + offset = fdt_path_offset(fdt, "/firmware/coreboot"); + if (offset < 0) + return false; + + cb_addr = fdt_reg_base_address(fdt, offset); + cb_size = fdt_reg_size(fdt, offset); + + cbmem_console_base = get_cbmem_console_from_coreboot_table(cb_addr, + cb_size); + if (!cbmem_console_base) + return false; + + cbmem_console.base = cbmem_console_base; + cbmem_console.console = (struct cbmem_console *) + core_mmu_add_mapping(MEM_AREA_RAM_NSEC, cbmem_console_base, + sizeof(struct cbmem_console)); + if (!cbmem_console.console) + return false; + + /* + * Copy the size now to prevent non-secure world from spoofing + * it later. + */ + cbmem_console.size = cbmem_console.console->size; + cbmem_console.chip.ops = &cbmem_console_ops; + + register_serial_console(&cbmem_console.chip); + return true; +} diff --git a/optee_os/core/drivers/cdns_uart.c b/optee_os/core/drivers/cdns_uart.c new file mode 100644 index 0000000..b348758 --- /dev/null +++ b/optee_os/core/drivers/cdns_uart.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Xilinx Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include + +#define CDNS_UART_CONTROL 0 +#define CDNS_UART_MODE 4 +#define CDNS_UART_IEN 8 +#define CDNS_UART_IRQ_STATUS 0x14 +#define CDNS_UART_CHANNEL_STATUS 0x2c +#define CDNS_UART_FIFO 0x30 +#define CDNS_UART_SIZE 0x34 + +#define CDNS_UART_CONTROL_RXRES BIT(0) +#define CDNS_UART_CONTROL_TXRES BIT(1) +#define CDNS_UART_CONTROL_RXEN BIT(2) +#define CDNS_UART_CONTROL_TXEN BIT(4) + +#define CDNS_UART_MODE_8BIT (0 << 1) +#define CDNS_UART_MODE_PARITY_NONE (0x4 << 3) +#define CDNS_UART_MODE_1STP (0 << 6) + +#define CDNS_UART_CHANNEL_STATUS_TFUL BIT(4) +#define CDNS_UART_CHANNEL_STATUS_TEMPTY BIT(3) +#define CDNS_UART_CHANNEL_STATUS_REMPTY BIT(1) + +#define CDNS_UART_IRQ_RXTRIG BIT(0) +#define CDNS_UART_IRQ_RXTOUT BIT(8) + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct cdns_uart_data *pd = + container_of(chip, struct cdns_uart_data, chip); + + return io_pa_or_va(&pd->base, CDNS_UART_SIZE); +} + +static void cdns_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + CDNS_UART_CHANNEL_STATUS) & + CDNS_UART_CHANNEL_STATUS_TEMPTY)) + ; +} + +static bool cdns_uart_have_rx_data(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + return !(io_read32(base + CDNS_UART_CHANNEL_STATUS) & + CDNS_UART_CHANNEL_STATUS_REMPTY); +} + +static int cdns_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!cdns_uart_have_rx_data(chip)) + ; + return io_read32(base + CDNS_UART_FIFO) & 0xff; +} + +static void cdns_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + /* Wait until there is space in the FIFO */ + while (io_read32(base + CDNS_UART_CHANNEL_STATUS) & + CDNS_UART_CHANNEL_STATUS_TFUL) + ; + + /* Send the character */ + io_write32(base + CDNS_UART_FIFO, ch); +} + + +static const struct serial_ops cdns_uart_ops = { + .flush = cdns_uart_flush, + .getchar = cdns_uart_getchar, + .have_rx_data = cdns_uart_have_rx_data, + .putc = cdns_uart_putc, +}; +DECLARE_KEEP_PAGER(cdns_uart_ops); + +/* + * we rely on the bootloader having set up the HW correctly, we just enable + * transmitter/receiver here, just in case. + */ +void cdns_uart_init(struct cdns_uart_data *pd, paddr_t base, uint32_t uart_clk, + uint32_t baud_rate) +{ + pd->base.pa = base; + pd->chip.ops = &cdns_uart_ops; + + if (!uart_clk || !baud_rate) + return; + + /* Enable UART and RX/TX */ + io_write32(base + CDNS_UART_CONTROL, + CDNS_UART_CONTROL_RXEN | CDNS_UART_CONTROL_TXEN); + + cdns_uart_flush(&pd->chip); +} diff --git a/optee_os/core/drivers/clk/clk-stm32-core.c b/optee_os/core/drivers/clk/clk-stm32-core.c new file mode 100644 index 0000000..536197f --- /dev/null +++ b/optee_os/core/drivers/clk/clk-stm32-core.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2022 - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-stm32-core.h" + +#define RCC_MP_ENCLRR_OFFSET 0x4 + +#define TIMEOUT_US_200MS U(200000) +#define TIMEOUT_US_1S U(1000000) + +static struct clk_stm32_priv *stm32_clock_data; + +struct clk_stm32_priv *clk_stm32_get_priv(void) +{ + return stm32_clock_data; +} + +uintptr_t clk_stm32_get_rcc_base(void) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + + return priv->base; +} + +/* STM32 MUX API */ +size_t stm32_mux_get_parent(uint32_t mux_id) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct mux_cfg *mux = &priv->muxes[mux_id]; + uint32_t mask = MASK_WIDTH_SHIFT(mux->width, mux->shift); + + return (io_read32(priv->base + mux->offset) & mask) >> mux->shift; +} + +TEE_Result stm32_mux_set_parent(uint16_t mux_id, uint8_t sel) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct mux_cfg *mux = &priv->muxes[mux_id]; + uint32_t mask = MASK_WIDTH_SHIFT(mux->width, mux->shift); + uintptr_t address = priv->base + mux->offset; + + io_clrsetbits32(address, mask, (sel << mux->shift) & mask); + + if (mux->ready != MUX_NO_RDY) + return stm32_gate_wait_ready((uint16_t)mux->ready, true); + + return TEE_SUCCESS; +} + +/* STM32 GATE API */ +static void stm32_gate_endisable(uint16_t gate_id, bool enable) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct gate_cfg *gate = &priv->gates[gate_id]; + uintptr_t addr = priv->base + gate->offset; + + if (enable) { + if (gate->set_clr) + io_write32(addr, BIT(gate->bit_idx)); + else + io_setbits32_stm32shregs(addr, BIT(gate->bit_idx)); + } else { + if (gate->set_clr) + io_write32(addr + RCC_MP_ENCLRR_OFFSET, + BIT(gate->bit_idx)); + else + io_clrbits32_stm32shregs(addr, BIT(gate->bit_idx)); + } +} + +void stm32_gate_disable(uint16_t gate_id) +{ + stm32_gate_endisable(gate_id, false); +} + +void stm32_gate_enable(uint16_t gate_id) +{ + stm32_gate_endisable(gate_id, true); +} + +bool stm32_gate_is_enabled(uint16_t gate_id) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct gate_cfg *gate = &priv->gates[gate_id]; + uintptr_t addr = priv->base + gate->offset; + + return (io_read32(addr) & BIT(gate->bit_idx)) != 0U; +} + +TEE_Result stm32_gate_wait_ready(uint16_t gate_id, bool ready_on) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct gate_cfg *gate = &priv->gates[gate_id]; + uintptr_t address = priv->base + gate->offset; + uint32_t mask_rdy = BIT(gate->bit_idx); + uint64_t timeout = timeout_init_us(TIMEOUT_US_1S); + uint32_t mask = 0U; + + if (ready_on) + mask = BIT(gate->bit_idx); + + while ((io_read32(address) & mask_rdy) != mask) + if (timeout_elapsed(timeout)) + break; + + if ((io_read32(address) & mask_rdy) != mask) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +/* STM32 GATE READY clock operators */ +static TEE_Result stm32_gate_ready_endisable(uint16_t gate_id, bool enable, + bool wait_rdy) +{ + stm32_gate_endisable(gate_id, enable); + + if (wait_rdy) + return stm32_gate_wait_ready(gate_id + 1, enable); + + return TEE_SUCCESS; +} + +TEE_Result stm32_gate_rdy_enable(uint16_t gate_id) +{ + return stm32_gate_ready_endisable(gate_id, true, true); +} + +TEE_Result stm32_gate_rdy_disable(uint16_t gate_id) +{ + return stm32_gate_ready_endisable(gate_id, false, true); +} + +/* STM32 DIV API */ +static unsigned int _get_table_div(const struct div_table_cfg *table, + unsigned int val) +{ + const struct div_table_cfg *clkt = NULL; + + for (clkt = table; clkt->div; clkt++) + if (clkt->val == val) + return clkt->div; + + return 0; +} + +static unsigned int _get_table_val(const struct div_table_cfg *table, + unsigned int div) +{ + const struct div_table_cfg *clkt = NULL; + + for (clkt = table; clkt->div; clkt++) + if (clkt->div == div) + return clkt->val; + + return 0; +} + +static unsigned int _get_div(const struct div_table_cfg *table, + unsigned int val, unsigned long flags, + uint8_t width) +{ + if (flags & CLK_DIVIDER_ONE_BASED) + return val; + + if (flags & CLK_DIVIDER_POWER_OF_TWO) + return BIT(val); + + if (flags & CLK_DIVIDER_MAX_AT_ZERO) + return (val != 0U) ? val : BIT(width); + + if (table) + return _get_table_div(table, val); + + return val + 1U; +} + +static unsigned int _get_val(const struct div_table_cfg *table, + unsigned int div, unsigned long flags, + uint8_t width) +{ + if (flags & CLK_DIVIDER_ONE_BASED) + return div; + + if (flags & CLK_DIVIDER_POWER_OF_TWO) + return __builtin_ffs(div) - 1; + + if (flags & CLK_DIVIDER_MAX_AT_ZERO) + return (div != 0U) ? div : BIT(width); + + if (table) + return _get_table_val(table, div); + + return div - 1U; +} + +static bool _is_valid_table_div(const struct div_table_cfg *table, + unsigned int div) +{ + const struct div_table_cfg *clkt = NULL; + + for (clkt = table; clkt->div; clkt++) + if (clkt->div == div) + return true; + + return false; +} + +static bool _is_valid_div(const struct div_table_cfg *table, + unsigned int div, unsigned long flags) +{ + if (flags & CLK_DIVIDER_POWER_OF_TWO) + return IS_POWER_OF_TWO(div); + + if (table) + return _is_valid_table_div(table, div); + + return true; +} + +static int divider_get_val(unsigned long rate, unsigned long parent_rate, + const struct div_table_cfg *table, uint8_t width, + unsigned long flags) +{ + unsigned int div = 0U; + unsigned int value = 0U; + + div = UDIV_ROUND_NEAREST((uint64_t)parent_rate, rate); + + if (!_is_valid_div(table, div, flags)) + return -1; + + value = _get_val(table, div, flags, width); + + return MIN(value, MASK_WIDTH_SHIFT(width, 0)); +} + +uint32_t stm32_div_get_value(int div_id) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct div_cfg *divider = &priv->div[div_id]; + uint32_t val = 0; + + val = io_read32(priv->base + divider->offset) >> divider->shift; + val &= MASK_WIDTH_SHIFT(divider->width, 0); + + return val; +} + +TEE_Result stm32_div_set_value(uint32_t div_id, uint32_t value) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct div_cfg *divider = NULL; + uintptr_t address = 0; + uint32_t mask = 0; + + if (div_id >= priv->nb_div) + panic(); + + divider = &priv->div[div_id]; + address = priv->base + divider->offset; + + mask = MASK_WIDTH_SHIFT(divider->width, divider->shift); + io_clrsetbits32(address, mask, (value << divider->shift) & mask); + + if (divider->ready == DIV_NO_RDY) + return TEE_SUCCESS; + + return stm32_gate_wait_ready((uint16_t)divider->ready, true); +} + +static unsigned long stm32_div_get_rate(int div_id, unsigned long prate) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct div_cfg *divider = &priv->div[div_id]; + uint32_t val = stm32_div_get_value(div_id); + unsigned int div = 0U; + + div = _get_div(divider->table, val, divider->flags, divider->width); + if (!div) + return prate; + + return ROUNDUP_DIV((uint64_t)prate, div); +} + +TEE_Result stm32_div_set_rate(int div_id, unsigned long rate, + unsigned long prate) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + const struct div_cfg *divider = &priv->div[div_id]; + int value = 0; + + value = divider_get_val(rate, prate, divider->table, + divider->width, divider->flags); + + if (value < 0) + return TEE_ERROR_GENERIC; + + return stm32_div_set_value(div_id, value); +} + +/* STM32 MUX clock operators */ +static size_t clk_stm32_mux_get_parent(struct clk *clk) +{ + struct clk_stm32_mux_cfg *cfg = clk->priv; + + return stm32_mux_get_parent(cfg->mux_id); +} + +static TEE_Result clk_stm32_mux_set_parent(struct clk *clk, size_t pidx) +{ + struct clk_stm32_mux_cfg *cfg = clk->priv; + + return stm32_mux_set_parent(cfg->mux_id, pidx); +} + +const struct clk_ops clk_stm32_mux_ops = { + .get_parent = clk_stm32_mux_get_parent, + .set_parent = clk_stm32_mux_set_parent, +}; + +/* STM32 GATE clock operators */ +static TEE_Result clk_stm32_gate_enable(struct clk *clk) +{ + struct clk_stm32_gate_cfg *cfg = clk->priv; + + stm32_gate_enable(cfg->gate_id); + + return TEE_SUCCESS; +} + +static void clk_stm32_gate_disable(struct clk *clk) +{ + struct clk_stm32_gate_cfg *cfg = clk->priv; + + stm32_gate_disable(cfg->gate_id); +} + +const struct clk_ops clk_stm32_gate_ops = { + .enable = clk_stm32_gate_enable, + .disable = clk_stm32_gate_disable, +}; + +static TEE_Result clk_stm32_gate_ready_enable(struct clk *clk) +{ + struct clk_stm32_gate_cfg *cfg = clk->priv; + + return stm32_gate_rdy_enable(cfg->gate_id); +} + +static void clk_stm32_gate_ready_disable(struct clk *clk) +{ + struct clk_stm32_gate_cfg *cfg = clk->priv; + + if (stm32_gate_rdy_disable(cfg->gate_id)) + panic(); +} + +const struct clk_ops clk_stm32_gate_ready_ops = { + .enable = clk_stm32_gate_ready_enable, + .disable = clk_stm32_gate_ready_disable, +}; + +/* STM32 DIV clock operators */ +unsigned long clk_stm32_divider_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_stm32_div_cfg *cfg = clk->priv; + + return stm32_div_get_rate(cfg->div_id, parent_rate); +} + +TEE_Result clk_stm32_divider_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate) +{ + struct clk_stm32_div_cfg *cfg = clk->priv; + + return stm32_div_set_rate(cfg->div_id, rate, parent_rate); +} + +const struct clk_ops clk_stm32_divider_ops = { + .get_rate = clk_stm32_divider_get_rate, + .set_rate = clk_stm32_divider_set_rate, +}; + +/* STM32 COMPOSITE clock operators */ +size_t clk_stm32_composite_get_parent(struct clk *clk) +{ + struct clk_stm32_composite_cfg *cfg = clk->priv; + + if (cfg->mux_id == NO_MUX) { + /* It could be a normal case */ + return 0; + } + + return stm32_mux_get_parent(cfg->mux_id); +} + +TEE_Result clk_stm32_composite_set_parent(struct clk *clk, size_t pidx) +{ + struct clk_stm32_composite_cfg *cfg = clk->priv; + + if (cfg->mux_id == NO_MUX) + panic(); + + return stm32_mux_set_parent(cfg->mux_id, pidx); +} + +unsigned long clk_stm32_composite_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_stm32_composite_cfg *cfg = clk->priv; + + if (cfg->div_id == NO_DIV) + return parent_rate; + + return stm32_div_get_rate(cfg->div_id, parent_rate); +} + +TEE_Result clk_stm32_composite_set_rate(struct clk *clk, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_stm32_composite_cfg *cfg = clk->priv; + + if (cfg->div_id == NO_DIV) + return TEE_SUCCESS; + + return stm32_div_set_rate(cfg->div_id, rate, parent_rate); +} + +TEE_Result clk_stm32_composite_gate_enable(struct clk *clk) +{ + struct clk_stm32_composite_cfg *cfg = clk->priv; + + stm32_gate_enable(cfg->gate_id); + + return TEE_SUCCESS; +} + +void clk_stm32_composite_gate_disable(struct clk *clk) +{ + struct clk_stm32_composite_cfg *cfg = clk->priv; + + stm32_gate_disable(cfg->gate_id); +} + +const struct clk_ops clk_stm32_composite_ops = { + .get_parent = clk_stm32_composite_get_parent, + .set_parent = clk_stm32_composite_set_parent, + .get_rate = clk_stm32_composite_get_rate, + .set_rate = clk_stm32_composite_set_rate, + .enable = clk_stm32_composite_gate_enable, + .disable = clk_stm32_composite_gate_disable, +}; + +TEE_Result clk_stm32_set_parent_by_index(struct clk *clk, size_t pidx) +{ + struct clk *parent = clk_get_parent_by_index(clk, pidx); + TEE_Result res = TEE_ERROR_GENERIC; + + if (parent) + res = clk_set_parent(clk, parent); + + return res; +} + +int clk_stm32_parse_fdt_by_name(const void *fdt, int node, const char *name, + uint32_t *tab, uint32_t *nb) +{ + const fdt32_t *cell = NULL; + int len = 0; + uint32_t i = 0; + + cell = fdt_getprop(fdt, node, name, &len); + if (cell) + for (i = 0; i < ((uint32_t)len / sizeof(uint32_t)); i++) + tab[i] = fdt32_to_cpu(cell[i]); + + *nb = (uint32_t)len / sizeof(uint32_t); + + return 0; +} + +TEE_Result clk_stm32_init(struct clk_stm32_priv *priv, uintptr_t base) +{ + stm32_clock_data = priv; + + priv->base = base; + + return TEE_SUCCESS; +} + +static unsigned long fixed_factor_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct fixed_factor_cfg *d = clk->priv; + + unsigned long long rate = (unsigned long long)parent_rate * d->mult; + + if (d->div == 0U) + panic("error division by zero"); + + return (unsigned long)(rate / d->div); +}; + +const struct clk_ops clk_fixed_factor_ops = { + .get_rate = fixed_factor_get_rate, +}; + +static unsigned long clk_fixed_get_rate(struct clk *clk, + unsigned long parent_rate __unused) +{ + struct clk_fixed_rate_cfg *cfg = clk->priv; + + return cfg->rate; +} + +const struct clk_ops clk_fixed_clk_ops = { + .get_rate = clk_fixed_get_rate, +}; + +struct clk *stm32mp_rcc_clock_id_to_clk(unsigned long clock_id) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + + if (clock_id > priv->nb_clk_refs) + return NULL; + + return priv->clk_refs[clock_id]; +} + +static TEE_Result stm32mp_clk_dt_get_clk(struct dt_pargs *pargs, + void *data __unused, + struct clk **out_clk) +{ + unsigned long clock_id = pargs->args[0]; + struct clk *clk = NULL; + + if (pargs->args_count != 1) + return TEE_ERROR_BAD_PARAMETERS; + + clk = stm32mp_rcc_clock_id_to_clk(clock_id); + if (!clk) + return TEE_ERROR_BAD_PARAMETERS; + + *out_clk = clk; + + return TEE_SUCCESS; +} + +static void clk_stm32_register_clocks(struct clk_stm32_priv *priv) +{ + unsigned int i = 0; + + for (i = 0; i < priv->nb_clk_refs; i++) { + struct clk *clk = priv->clk_refs[i]; + + if (!clk) + continue; + + refcount_set(&clk->enabled_count, 0); + + if (clk_register(clk)) + panic(); + } + + /* Critical clocks management */ + for (i = 0; i < priv->nb_clk_refs; i++) { + struct clk *clk = priv->clk_refs[i]; + + if (!clk) + continue; + + if (priv->is_critical && priv->is_critical(clk)) + clk_enable(clk); + } +} + +void stm32mp_clk_provider_probe_final(const void *fdt, int node, + struct clk_stm32_priv *priv) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + clk_stm32_register_clocks(priv); + + res = clk_dt_register_clk_provider(fdt, node, stm32mp_clk_dt_get_clk, + priv); + if (res) + panic("Couldn't register clock provider"); +} diff --git a/optee_os/core/drivers/clk/clk-stm32-core.h b/optee_os/core/drivers/clk/clk-stm32-core.h new file mode 100644 index 0000000..c540d6c --- /dev/null +++ b/optee_os/core/drivers/clk/clk-stm32-core.h @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright (C) STMicroelectronics 2022 - All Rights Reserved + */ + +#ifndef CLK_STM32_CORE_H +#define CLK_STM32_CORE_H + +#include + +struct mux_cfg { + uint16_t offset; + uint8_t shift; + uint8_t width; + uint8_t ready; +}; + +struct gate_cfg { + uint16_t offset; + uint8_t bit_idx; + uint8_t set_clr; +}; + +struct div_table_cfg { + unsigned int val; + unsigned int div; +}; + +struct div_cfg { + uint16_t offset; + uint8_t shift; + uint8_t width; + uint8_t flags; + uint8_t ready; + const struct div_table_cfg *table; +}; + +struct clk_stm32_priv { + uintptr_t base; + size_t nb_clk_refs; + struct clk **clk_refs; + const struct mux_cfg *muxes; + const uint32_t nb_muxes; + const struct gate_cfg *gates; + const uint32_t nb_gates; + const struct div_cfg *div; + const uint32_t nb_div; + bool (*is_critical)(struct clk *clk); + void *pdata; +}; + +struct clk_fixed_rate_cfg { + unsigned long rate; +}; + +struct fixed_factor_cfg { + unsigned int mult; + unsigned int div; +}; + +struct clk_gate_cfg { + uint32_t offset; + uint8_t bit_idx; +}; + +struct clk_stm32_mux_cfg { + int mux_id; +}; + +struct clk_stm32_gate_cfg { + int gate_id; +}; + +struct clk_stm32_div_cfg { + int div_id; +}; + +struct clk_stm32_composite_cfg { + int gate_id; + int div_id; + int mux_id; +}; + +struct clk_stm32_timer_cfg { + uint32_t apbdiv; + uint32_t timpre; +}; + +struct clk_stm32_gate_ready_cfg { + int gate_id; + int gate_rdy_id; +}; + +/* Define for divider clocks */ +#define CLK_DIVIDER_ONE_BASED BIT(0) +#define CLK_DIVIDER_POWER_OF_TWO BIT(1) +#define CLK_DIVIDER_ALLOW_ZERO BIT(2) +#define CLK_DIVIDER_HIWORD_MASK BIT(3) +#define CLK_DIVIDER_ROUND_CLOSEST BIT(4) +#define CLK_DIVIDER_READ_ONLY BIT(5) +#define CLK_DIVIDER_MAX_AT_ZERO BIT(6) +#define CLK_DIVIDER_BIG_ENDIAN BIT(7) + +#define DIV_NO_RDY UINT8_MAX +#define MUX_NO_RDY UINT8_MAX + +#define MASK_WIDTH_SHIFT(_width, _shift) \ + GENMASK_32(((_width) + (_shift) - 1U), (_shift)) + +/* Define for composite clocks */ +#define NO_MUX INT32_MAX +#define NO_DIV INT32_MAX +#define NO_GATE INT32_MAX + +void stm32_gate_enable(uint16_t gate_id); +void stm32_gate_disable(uint16_t gate_id); +bool stm32_gate_is_enabled(uint16_t gate_id); +TEE_Result stm32_gate_wait_ready(uint16_t gate_id, bool ready_on); +TEE_Result stm32_gate_rdy_enable(uint16_t gate_id); +TEE_Result stm32_gate_rdy_disable(uint16_t gate_id); + +size_t stm32_mux_get_parent(uint32_t mux_id); +TEE_Result stm32_mux_set_parent(uint16_t pid, uint8_t sel); + +TEE_Result stm32_div_set_rate(int div_id, unsigned long rate, + unsigned long prate); + +uint32_t stm32_div_get_value(int div_id); +TEE_Result stm32_div_set_value(uint32_t div_id, uint32_t value); + +int clk_stm32_parse_fdt_by_name(const void *fdt, int node, const char *name, + uint32_t *tab, uint32_t *nb); + +unsigned long clk_stm32_divider_get_rate(struct clk *clk, + unsigned long parent_rate); + +TEE_Result clk_stm32_divider_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate); + +size_t clk_stm32_composite_get_parent(struct clk *clk); +TEE_Result clk_stm32_composite_set_parent(struct clk *clk, size_t pidx); +unsigned long clk_stm32_composite_get_rate(struct clk *clk, + unsigned long parent_rate); +TEE_Result clk_stm32_composite_set_rate(struct clk *clk, unsigned long rate, + unsigned long parent_rate); +TEE_Result clk_stm32_composite_gate_enable(struct clk *clk); +void clk_stm32_composite_gate_disable(struct clk *clk); + +TEE_Result clk_stm32_set_parent_by_index(struct clk *clk, size_t pidx); + +extern const struct clk_ops clk_fixed_factor_ops; +extern const struct clk_ops clk_fixed_clk_ops; +extern const struct clk_ops clk_stm32_gate_ops; +extern const struct clk_ops clk_stm32_gate_ready_ops; +extern const struct clk_ops clk_stm32_divider_ops; +extern const struct clk_ops clk_stm32_mux_ops; +extern const struct clk_ops clk_stm32_composite_ops; + +#define PARENT(x...) { x } + +#define STM32_FIXED_RATE(_name, _rate)\ + struct clk _name = {\ + .ops = &clk_fixed_clk_ops,\ + .priv = &(struct clk_fixed_rate_cfg) {\ + .rate = (_rate),\ + },\ + .name = #_name,\ + .flags = 0,\ + .num_parents = 0,\ + } + +#define STM32_FIXED_FACTOR(_name, _parent, _flags, _mult, _div)\ + struct clk _name = {\ + .ops = &clk_fixed_factor_ops,\ + .priv = &(struct fixed_factor_cfg) {\ + .mult = _mult,\ + .div = _div,\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = 1,\ + .parents = { (_parent) },\ + } + +#define STM32_GATE(_name, _parent, _flags, _gate_id)\ + struct clk _name = {\ + .ops = &clk_stm32_gate_ops,\ + .priv = &(struct clk_stm32_gate_cfg) {\ + .gate_id = _gate_id,\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = 1,\ + .parents = { (_parent) },\ + } + +#define STM32_DIVIDER(_name, _parent, _flags, _div_id)\ + struct clk _name = {\ + .ops = &clk_stm32_divider_ops,\ + .priv = &(struct clk_stm32_div_cfg) {\ + .div_id = (_div_id),\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = 1,\ + .parents = { (_parent) },\ + } + +#define STM32_MUX(_name, _nb_parents, _parents, _flags, _mux_id)\ + struct clk _name = {\ + .ops = &clk_stm32_mux_ops,\ + .priv = &(struct clk_stm32_mux_cfg) {\ + .mux_id = (_mux_id),\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = (_nb_parents),\ + .parents = _parents,\ + } + +#define STM32_GATE_READY(_name, _parent, _flags, _gate_id)\ + struct clk _name = {\ + .ops = &clk_stm32_gate_ready_ops,\ + .priv = &(struct clk_stm32_gate_cfg) {\ + .gate_id = _gate_id,\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = 1,\ + .parents = { _parent },\ + } + +#define STM32_COMPOSITE(_name, _nb_parents, _parents, _flags,\ + _gate_id, _div_id, _mux_id)\ + struct clk _name = {\ + .ops = &clk_stm32_composite_ops,\ + .priv = &(struct clk_stm32_composite_cfg) {\ + .gate_id = (_gate_id),\ + .div_id = (_div_id),\ + .mux_id = (_mux_id),\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = (_nb_parents),\ + .parents = _parents,\ + } + +struct clk_stm32_priv *clk_stm32_get_priv(void); +uintptr_t clk_stm32_get_rcc_base(void); + +TEE_Result clk_stm32_init(struct clk_stm32_priv *priv, uintptr_t base); + +void stm32mp_clk_provider_probe_final(const void *fdt, int node, + struct clk_stm32_priv *priv); + +#endif /* CLK_STM32_CORE_H */ diff --git a/optee_os/core/drivers/clk/clk-stm32mp13.c b/optee_os/core/drivers/clk/clk-stm32mp13.c new file mode 100644 index 0000000..0a3d6c7 --- /dev/null +++ b/optee_os/core/drivers/clk/clk-stm32mp13.c @@ -0,0 +1,2663 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright (C) STMicroelectronics 2022 - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-stm32-core.h" + +#define MAX_HSI_HZ 64000000 +#define USB_PHY_48_MHZ 48000000 + +#define TIMEOUT_US_200MS U(200000) +#define HSIDIV_TIMEOUT TIMEOUT_US_200MS + +#define MAX_OPP CFG_STM32MP_OPP_COUNT + +#define RCC_PLL_NAME_SIZE 12 + +struct stm32_osci_dt_cfg { + unsigned long freq; + bool bypass; + bool digbyp; + bool css; + uint32_t drive; +}; + +enum pll_mn { + PLL_CFG_M, + PLL_CFG_N, + PLL_DIV_MN_NB +}; + +enum pll_pqr { + PLL_CFG_P, + PLL_CFG_Q, + PLL_CFG_R, + PLL_DIV_PQR_NB +}; + +enum pll_csg { + PLL_CSG_MOD_PER, + PLL_CSG_INC_STEP, + PLL_CSG_SSCG_MODE, + PLL_CSG_NB +}; + +struct stm32_pll_vco { + uint32_t status; + uint32_t src; + uint32_t div_mn[PLL_DIV_MN_NB]; + uint32_t frac; + bool csg_enabled; + uint32_t csg[PLL_CSG_NB]; +}; + +struct stm32_pll_output { + uint32_t output[PLL_DIV_PQR_NB]; +}; + +struct stm32_pll_dt_cfg { + struct stm32_pll_vco vco; + struct stm32_pll_output output; +}; + +struct stm32_clk_opp_cfg { + uint32_t frq; + uint32_t src; + uint32_t div; + struct stm32_pll_dt_cfg pll_cfg; +}; + +struct stm32_clk_opp_dt_cfg { + struct stm32_clk_opp_cfg mpu_opp[MAX_OPP]; + struct stm32_clk_opp_cfg axi_opp[MAX_OPP]; + struct stm32_clk_opp_cfg mlahbs_opp[MAX_OPP]; +}; + +struct stm32_clk_platdata { + uintptr_t rcc_base; + uint32_t nosci; + struct stm32_osci_dt_cfg *osci; + uint32_t npll; + struct stm32_pll_dt_cfg *pll; + struct stm32_clk_opp_dt_cfg *opp; + uint32_t nclksrc; + uint32_t *clksrc; + uint32_t nclkdiv; + uint32_t *clkdiv; +}; + +/* + * GATE CONFIG + */ + +/* Warning GATE_XXX_RDY must follow GATE_XXX */ +enum enum_gate_cfg { + GATE_LSE, + GATE_LSE_RDY, + GATE_LSI, + GATE_LSI_RDY, + GATE_HSI, + GATE_HSI_RDY, + GATE_CSI, + GATE_CSI_RDY, + GATE_HSE, + GATE_HSE_RDY, + GATE_PLL1, + GATE_PLL1_RDY, + GATE_PLL2, + GATE_PLL2_RDY, + GATE_PLL3, + GATE_PLL3_RDY, + GATE_PLL4, + GATE_PLL4_RDY, + GATE_HSIDIVRDY, + GATE_MPUSRCRDY, + GATE_AXISSRCRDY, + GATE_MCUSSRCRDY, + GATE_PLL12SRCRDY, + GATE_PLL3SRCRDY, + GATE_PLL4SRCRDY, + GATE_MPUDIVRDY, + GATE_AXIDIVRDY, + GATE_MLAHBDIVRDY, + GATE_APB1DIVRDY, + GATE_APB2DIVRDY, + GATE_APB3DIVRDY, + GATE_APB4DIVRDY, + GATE_APB5DIVRDY, + GATE_APB6DIVRDY, + GATE_RTCCK, + GATE_MCO1, + GATE_MCO2, + GATE_DBGCK, + GATE_TRACECK, + GATE_PLL1_DIVP, + GATE_PLL1_DIVQ, + GATE_PLL1_DIVR, + GATE_PLL2_DIVP, + GATE_PLL2_DIVQ, + GATE_PLL2_DIVR, + GATE_PLL3_DIVP, + GATE_PLL3_DIVQ, + GATE_PLL3_DIVR, + GATE_PLL4_DIVP, + GATE_PLL4_DIVQ, + GATE_PLL4_DIVR, + GATE_DDRC1, + GATE_DDRC1LP, + GATE_DDRPHYC, + GATE_DDRPHYCLP, + GATE_DDRCAPB, + GATE_DDRCAPBLP, + GATE_AXIDCG, + GATE_DDRPHYCAPB, + GATE_DDRPHYCAPBLP, + GATE_TIM2, + GATE_TIM3, + GATE_TIM4, + GATE_TIM5, + GATE_TIM6, + GATE_TIM7, + GATE_LPTIM1, + GATE_SPI2, + GATE_SPI3, + GATE_USART3, + GATE_UART4, + GATE_UART5, + GATE_UART7, + GATE_UART8, + GATE_I2C1, + GATE_I2C2, + GATE_SPDIF, + GATE_TIM1, + GATE_TIM8, + GATE_SPI1, + GATE_USART6, + GATE_SAI1, + GATE_SAI2, + GATE_DFSDM, + GATE_ADFSDM, + GATE_FDCAN, + GATE_LPTIM2, + GATE_LPTIM3, + GATE_LPTIM4, + GATE_LPTIM5, + GATE_VREF, + GATE_DTS, + GATE_PMBCTRL, + GATE_HDP, + GATE_SYSCFG, + GATE_DCMIPP, + GATE_DDRPERFM, + GATE_IWDG2APB, + GATE_USBPHY, + GATE_STGENRO, + GATE_LTDC, + GATE_RTCAPB, + GATE_TZC, + GATE_ETZPC, + GATE_IWDG1APB, + GATE_BSEC, + GATE_STGENC, + GATE_USART1, + GATE_USART2, + GATE_SPI4, + GATE_SPI5, + GATE_I2C3, + GATE_I2C4, + GATE_I2C5, + GATE_TIM12, + GATE_TIM13, + GATE_TIM14, + GATE_TIM15, + GATE_TIM16, + GATE_TIM17, + GATE_DMA1, + GATE_DMA2, + GATE_DMAMUX1, + GATE_DMA3, + GATE_DMAMUX2, + GATE_ADC1, + GATE_ADC2, + GATE_USBO, + GATE_TSC, + GATE_GPIOA, + GATE_GPIOB, + GATE_GPIOC, + GATE_GPIOD, + GATE_GPIOE, + GATE_GPIOF, + GATE_GPIOG, + GATE_GPIOH, + GATE_GPIOI, + GATE_PKA, + GATE_SAES, + GATE_CRYP1, + GATE_HASH1, + GATE_RNG1, + GATE_BKPSRAM, + GATE_AXIMC, + GATE_MCE, + GATE_ETH1CK, + GATE_ETH1TX, + GATE_ETH1RX, + GATE_ETH1MAC, + GATE_FMC, + GATE_QSPI, + GATE_SDMMC1, + GATE_SDMMC2, + GATE_CRC1, + GATE_USBH, + GATE_ETH2CK, + GATE_ETH2TX, + GATE_ETH2RX, + GATE_ETH2MAC, + GATE_MDMA, + GATE_NB +}; + +#define GATE_CFG(_id, _offset, _bit_idx, _offset_clr)\ + [(_id)] = {\ + .offset = (_offset),\ + .bit_idx = (_bit_idx),\ + .set_clr = (_offset_clr),\ + } + +static const struct gate_cfg gates_mp13[GATE_NB] = { + GATE_CFG(GATE_LSE, RCC_BDCR, 0, 0), + GATE_CFG(GATE_LSE_RDY, RCC_BDCR, 2, 0), + GATE_CFG(GATE_RTCCK, RCC_BDCR, 20, 0), + GATE_CFG(GATE_LSI, RCC_RDLSICR, 0, 0), + GATE_CFG(GATE_LSI_RDY, RCC_RDLSICR, 1, 0), + GATE_CFG(GATE_HSI, RCC_OCENSETR, 0, 1), + GATE_CFG(GATE_HSI_RDY, RCC_OCRDYR, 0, 0), + GATE_CFG(GATE_CSI, RCC_OCENSETR, 4, 1), + GATE_CFG(GATE_CSI_RDY, RCC_OCRDYR, 4, 0), + GATE_CFG(GATE_HSE, RCC_OCENSETR, 8, 1), + GATE_CFG(GATE_HSE_RDY, RCC_OCRDYR, 8, 0), + GATE_CFG(GATE_HSIDIVRDY, RCC_OCRDYR, 2, 0), + GATE_CFG(GATE_MPUSRCRDY, RCC_MPCKSELR, 31, 0), + GATE_CFG(GATE_AXISSRCRDY, RCC_ASSCKSELR, 31, 0), + GATE_CFG(GATE_MCUSSRCRDY, RCC_MSSCKSELR, 31, 0), + GATE_CFG(GATE_PLL12SRCRDY, RCC_RCK12SELR, 31, 0), + GATE_CFG(GATE_PLL3SRCRDY, RCC_RCK3SELR, 31, 0), + GATE_CFG(GATE_PLL4SRCRDY, RCC_RCK4SELR, 31, 0), + GATE_CFG(GATE_MPUDIVRDY, RCC_MPCKDIVR, 31, 0), + GATE_CFG(GATE_AXIDIVRDY, RCC_AXIDIVR, 31, 0), + GATE_CFG(GATE_MLAHBDIVRDY, RCC_MLAHBDIVR, 31, 0), + GATE_CFG(GATE_APB1DIVRDY, RCC_APB1DIVR, 31, 0), + GATE_CFG(GATE_APB2DIVRDY, RCC_APB2DIVR, 31, 0), + GATE_CFG(GATE_APB3DIVRDY, RCC_APB3DIVR, 31, 0), + GATE_CFG(GATE_APB4DIVRDY, RCC_APB4DIVR, 31, 0), + GATE_CFG(GATE_APB5DIVRDY, RCC_APB5DIVR, 31, 0), + GATE_CFG(GATE_APB6DIVRDY, RCC_APB6DIVR, 31, 0), + GATE_CFG(GATE_MCO1, RCC_MCO1CFGR, 12, 0), + GATE_CFG(GATE_MCO2, RCC_MCO2CFGR, 12, 0), + GATE_CFG(GATE_DBGCK, RCC_DBGCFGR, 8, 0), + GATE_CFG(GATE_TRACECK, RCC_DBGCFGR, 9, 0), + GATE_CFG(GATE_PLL1, RCC_PLL1CR, 0, 0), + GATE_CFG(GATE_PLL1_RDY, RCC_PLL1CR, 1, 0), + GATE_CFG(GATE_PLL1_DIVP, RCC_PLL1CR, 4, 0), + GATE_CFG(GATE_PLL1_DIVQ, RCC_PLL1CR, 5, 0), + GATE_CFG(GATE_PLL1_DIVR, RCC_PLL1CR, 6, 0), + GATE_CFG(GATE_PLL2, RCC_PLL2CR, 0, 0), + GATE_CFG(GATE_PLL2_RDY, RCC_PLL2CR, 1, 0), + GATE_CFG(GATE_PLL2_DIVP, RCC_PLL2CR, 4, 0), + GATE_CFG(GATE_PLL2_DIVQ, RCC_PLL2CR, 5, 0), + GATE_CFG(GATE_PLL2_DIVR, RCC_PLL2CR, 6, 0), + GATE_CFG(GATE_PLL3, RCC_PLL3CR, 0, 0), + GATE_CFG(GATE_PLL3_RDY, RCC_PLL3CR, 1, 0), + GATE_CFG(GATE_PLL3_DIVP, RCC_PLL3CR, 4, 0), + GATE_CFG(GATE_PLL3_DIVQ, RCC_PLL3CR, 5, 0), + GATE_CFG(GATE_PLL3_DIVR, RCC_PLL3CR, 6, 0), + GATE_CFG(GATE_PLL4, RCC_PLL4CR, 0, 0), + GATE_CFG(GATE_PLL4_RDY, RCC_PLL4CR, 1, 0), + GATE_CFG(GATE_PLL4_DIVP, RCC_PLL4CR, 4, 0), + GATE_CFG(GATE_PLL4_DIVQ, RCC_PLL4CR, 5, 0), + GATE_CFG(GATE_PLL4_DIVR, RCC_PLL4CR, 6, 0), + GATE_CFG(GATE_DDRC1, RCC_DDRITFCR, 0, 0), + GATE_CFG(GATE_DDRC1LP, RCC_DDRITFCR, 1, 0), + GATE_CFG(GATE_DDRPHYC, RCC_DDRITFCR, 4, 0), + GATE_CFG(GATE_DDRPHYCLP, RCC_DDRITFCR, 5, 0), + GATE_CFG(GATE_DDRCAPB, RCC_DDRITFCR, 6, 0), + GATE_CFG(GATE_DDRCAPBLP, RCC_DDRITFCR, 7, 0), + GATE_CFG(GATE_AXIDCG, RCC_DDRITFCR, 8, 0), + GATE_CFG(GATE_DDRPHYCAPB, RCC_DDRITFCR, 9, 0), + GATE_CFG(GATE_DDRPHYCAPBLP, RCC_DDRITFCR, 10, 0), + GATE_CFG(GATE_TIM2, RCC_MP_APB1ENSETR, 0, 1), + GATE_CFG(GATE_TIM3, RCC_MP_APB1ENSETR, 1, 1), + GATE_CFG(GATE_TIM4, RCC_MP_APB1ENSETR, 2, 1), + GATE_CFG(GATE_TIM5, RCC_MP_APB1ENSETR, 3, 1), + GATE_CFG(GATE_TIM6, RCC_MP_APB1ENSETR, 4, 1), + GATE_CFG(GATE_TIM7, RCC_MP_APB1ENSETR, 5, 1), + GATE_CFG(GATE_LPTIM1, RCC_MP_APB1ENSETR, 9, 1), + GATE_CFG(GATE_SPI2, RCC_MP_APB1ENSETR, 11, 1), + GATE_CFG(GATE_SPI3, RCC_MP_APB1ENSETR, 12, 1), + GATE_CFG(GATE_USART3, RCC_MP_APB1ENSETR, 15, 1), + GATE_CFG(GATE_UART4, RCC_MP_APB1ENSETR, 16, 1), + GATE_CFG(GATE_UART5, RCC_MP_APB1ENSETR, 17, 1), + GATE_CFG(GATE_UART7, RCC_MP_APB1ENSETR, 18, 1), + GATE_CFG(GATE_UART8, RCC_MP_APB1ENSETR, 19, 1), + GATE_CFG(GATE_I2C1, RCC_MP_APB1ENSETR, 21, 1), + GATE_CFG(GATE_I2C2, RCC_MP_APB1ENSETR, 22, 1), + GATE_CFG(GATE_SPDIF, RCC_MP_APB1ENSETR, 26, 1), + GATE_CFG(GATE_TIM1, RCC_MP_APB2ENSETR, 0, 1), + GATE_CFG(GATE_TIM8, RCC_MP_APB2ENSETR, 1, 1), + GATE_CFG(GATE_SPI1, RCC_MP_APB2ENSETR, 8, 1), + GATE_CFG(GATE_USART6, RCC_MP_APB2ENSETR, 13, 1), + GATE_CFG(GATE_SAI1, RCC_MP_APB2ENSETR, 16, 1), + GATE_CFG(GATE_SAI2, RCC_MP_APB2ENSETR, 17, 1), + GATE_CFG(GATE_DFSDM, RCC_MP_APB2ENSETR, 20, 1), + GATE_CFG(GATE_ADFSDM, RCC_MP_APB2ENSETR, 21, 1), + GATE_CFG(GATE_FDCAN, RCC_MP_APB2ENSETR, 24, 1), + GATE_CFG(GATE_LPTIM2, RCC_MP_APB3ENSETR, 0, 1), + GATE_CFG(GATE_LPTIM3, RCC_MP_APB3ENSETR, 1, 1), + GATE_CFG(GATE_LPTIM4, RCC_MP_APB3ENSETR, 2, 1), + GATE_CFG(GATE_LPTIM5, RCC_MP_APB3ENSETR, 3, 1), + GATE_CFG(GATE_VREF, RCC_MP_APB3ENSETR, 13, 1), + GATE_CFG(GATE_DTS, RCC_MP_APB3ENSETR, 16, 1), + GATE_CFG(GATE_PMBCTRL, RCC_MP_APB3ENSETR, 17, 1), + GATE_CFG(GATE_HDP, RCC_MP_APB3ENSETR, 20, 1), + GATE_CFG(GATE_SYSCFG, RCC_MP_S_APB3ENSETR, 0, 1), + GATE_CFG(GATE_DCMIPP, RCC_MP_APB4ENSETR, 1, 1), + GATE_CFG(GATE_DDRPERFM, RCC_MP_APB4ENSETR, 8, 1), + GATE_CFG(GATE_IWDG2APB, RCC_MP_APB4ENSETR, 15, 1), + GATE_CFG(GATE_USBPHY, RCC_MP_APB4ENSETR, 16, 1), + GATE_CFG(GATE_STGENRO, RCC_MP_APB4ENSETR, 20, 1), + GATE_CFG(GATE_LTDC, RCC_MP_S_APB4ENSETR, 0, 1), + GATE_CFG(GATE_RTCAPB, RCC_MP_APB5ENSETR, 8, 1), + GATE_CFG(GATE_TZC, RCC_MP_APB5ENSETR, 11, 1), + GATE_CFG(GATE_ETZPC, RCC_MP_APB5ENSETR, 13, 1), + GATE_CFG(GATE_IWDG1APB, RCC_MP_APB5ENSETR, 15, 1), + GATE_CFG(GATE_BSEC, RCC_MP_APB5ENSETR, 16, 1), + GATE_CFG(GATE_STGENC, RCC_MP_APB5ENSETR, 20, 1), + GATE_CFG(GATE_USART1, RCC_MP_APB6ENSETR, 0, 1), + GATE_CFG(GATE_USART2, RCC_MP_APB6ENSETR, 1, 1), + GATE_CFG(GATE_SPI4, RCC_MP_APB6ENSETR, 2, 1), + GATE_CFG(GATE_SPI5, RCC_MP_APB6ENSETR, 3, 1), + GATE_CFG(GATE_I2C3, RCC_MP_APB6ENSETR, 4, 1), + GATE_CFG(GATE_I2C4, RCC_MP_APB6ENSETR, 5, 1), + GATE_CFG(GATE_I2C5, RCC_MP_APB6ENSETR, 6, 1), + GATE_CFG(GATE_TIM12, RCC_MP_APB6ENSETR, 7, 1), + GATE_CFG(GATE_TIM13, RCC_MP_APB6ENSETR, 8, 1), + GATE_CFG(GATE_TIM14, RCC_MP_APB6ENSETR, 9, 1), + GATE_CFG(GATE_TIM15, RCC_MP_APB6ENSETR, 10, 1), + GATE_CFG(GATE_TIM16, RCC_MP_APB6ENSETR, 11, 1), + GATE_CFG(GATE_TIM17, RCC_MP_APB6ENSETR, 12, 1), + GATE_CFG(GATE_DMA1, RCC_MP_AHB2ENSETR, 0, 1), + GATE_CFG(GATE_DMA2, RCC_MP_AHB2ENSETR, 1, 1), + GATE_CFG(GATE_DMAMUX1, RCC_MP_AHB2ENSETR, 2, 1), + GATE_CFG(GATE_DMA3, RCC_MP_AHB2ENSETR, 3, 1), + GATE_CFG(GATE_DMAMUX2, RCC_MP_AHB2ENSETR, 4, 1), + GATE_CFG(GATE_ADC1, RCC_MP_AHB2ENSETR, 5, 1), + GATE_CFG(GATE_ADC2, RCC_MP_AHB2ENSETR, 6, 1), + GATE_CFG(GATE_USBO, RCC_MP_AHB2ENSETR, 8, 1), + GATE_CFG(GATE_TSC, RCC_MP_AHB4ENSETR, 15, 1), + GATE_CFG(GATE_GPIOA, RCC_MP_S_AHB4ENSETR, 0, 1), + GATE_CFG(GATE_GPIOB, RCC_MP_S_AHB4ENSETR, 1, 1), + GATE_CFG(GATE_GPIOC, RCC_MP_S_AHB4ENSETR, 2, 1), + GATE_CFG(GATE_GPIOD, RCC_MP_S_AHB4ENSETR, 3, 1), + GATE_CFG(GATE_GPIOE, RCC_MP_S_AHB4ENSETR, 4, 1), + GATE_CFG(GATE_GPIOF, RCC_MP_S_AHB4ENSETR, 5, 1), + GATE_CFG(GATE_GPIOG, RCC_MP_S_AHB4ENSETR, 6, 1), + GATE_CFG(GATE_GPIOH, RCC_MP_S_AHB4ENSETR, 7, 1), + GATE_CFG(GATE_GPIOI, RCC_MP_S_AHB4ENSETR, 8, 1), + GATE_CFG(GATE_PKA, RCC_MP_AHB5ENSETR, 2, 1), + GATE_CFG(GATE_SAES, RCC_MP_AHB5ENSETR, 3, 1), + GATE_CFG(GATE_CRYP1, RCC_MP_AHB5ENSETR, 4, 1), + GATE_CFG(GATE_HASH1, RCC_MP_AHB5ENSETR, 5, 1), + GATE_CFG(GATE_RNG1, RCC_MP_AHB5ENSETR, 6, 1), + GATE_CFG(GATE_BKPSRAM, RCC_MP_AHB5ENSETR, 8, 1), + GATE_CFG(GATE_AXIMC, RCC_MP_AHB5ENSETR, 16, 1), + GATE_CFG(GATE_MCE, RCC_MP_AHB6ENSETR, 1, 1), + GATE_CFG(GATE_ETH1CK, RCC_MP_AHB6ENSETR, 7, 1), + GATE_CFG(GATE_ETH1TX, RCC_MP_AHB6ENSETR, 8, 1), + GATE_CFG(GATE_ETH1RX, RCC_MP_AHB6ENSETR, 9, 1), + GATE_CFG(GATE_ETH1MAC, RCC_MP_AHB6ENSETR, 10, 1), + GATE_CFG(GATE_FMC, RCC_MP_AHB6ENSETR, 12, 1), + GATE_CFG(GATE_QSPI, RCC_MP_AHB6ENSETR, 14, 1), + GATE_CFG(GATE_SDMMC1, RCC_MP_AHB6ENSETR, 16, 1), + GATE_CFG(GATE_SDMMC2, RCC_MP_AHB6ENSETR, 17, 1), + GATE_CFG(GATE_CRC1, RCC_MP_AHB6ENSETR, 20, 1), + GATE_CFG(GATE_USBH, RCC_MP_AHB6ENSETR, 24, 1), + GATE_CFG(GATE_ETH2CK, RCC_MP_AHB6ENSETR, 27, 1), + GATE_CFG(GATE_ETH2TX, RCC_MP_AHB6ENSETR, 28, 1), + GATE_CFG(GATE_ETH2RX, RCC_MP_AHB6ENSETR, 29, 1), + GATE_CFG(GATE_ETH2MAC, RCC_MP_AHB6ENSETR, 30, 1), + GATE_CFG(GATE_MDMA, RCC_MP_S_AHB6ENSETR, 0, 1), +}; + +/* + * MUX CONFIG + */ +#define MUXRDY_CFG(_id, _offset, _shift, _witdh, _rdy)\ + [(_id)] = {\ + .offset = (_offset),\ + .shift = (_shift),\ + .width = (_witdh),\ + .ready = (_rdy),\ + } + +#define MUX_CFG(_id, _offset, _shift, _witdh)\ + MUXRDY_CFG(_id, _offset, _shift, _witdh, MUX_NO_RDY) + +static const struct mux_cfg parent_mp13[MUX_NB] = { + MUXRDY_CFG(MUX_MPU, RCC_MPCKSELR, 0, 2, GATE_MPUSRCRDY), + MUXRDY_CFG(MUX_AXI, RCC_ASSCKSELR, 0, 3, GATE_AXISSRCRDY), + MUXRDY_CFG(MUX_MLAHB, RCC_MSSCKSELR, 0, 2, GATE_MCUSSRCRDY), + MUXRDY_CFG(MUX_PLL12, RCC_RCK12SELR, 0, 2, GATE_PLL12SRCRDY), + MUXRDY_CFG(MUX_PLL3, RCC_RCK3SELR, 0, 2, GATE_PLL3SRCRDY), + MUXRDY_CFG(MUX_PLL4, RCC_RCK4SELR, 0, 2, GATE_PLL4SRCRDY), + MUX_CFG(MUX_ADC1, RCC_ADC12CKSELR, 0, 2), + MUX_CFG(MUX_ADC2, RCC_ADC12CKSELR, 2, 2), + MUX_CFG(MUX_CKPER, RCC_CPERCKSELR, 0, 2), + MUX_CFG(MUX_DCMIPP, RCC_DCMIPPCKSELR, 0, 2), + MUX_CFG(MUX_ETH1, RCC_ETH12CKSELR, 0, 2), + MUX_CFG(MUX_ETH2, RCC_ETH12CKSELR, 8, 2), + MUX_CFG(MUX_FDCAN, RCC_FDCANCKSELR, 0, 2), + MUX_CFG(MUX_FMC, RCC_FMCCKSELR, 0, 2), + MUX_CFG(MUX_I2C12, RCC_I2C12CKSELR, 0, 3), + MUX_CFG(MUX_I2C3, RCC_I2C345CKSELR, 0, 3), + MUX_CFG(MUX_I2C4, RCC_I2C345CKSELR, 3, 3), + MUX_CFG(MUX_I2C5, RCC_I2C345CKSELR, 6, 3), + MUX_CFG(MUX_LPTIM1, RCC_LPTIM1CKSELR, 0, 3), + MUX_CFG(MUX_LPTIM2, RCC_LPTIM23CKSELR, 0, 3), + MUX_CFG(MUX_LPTIM3, RCC_LPTIM23CKSELR, 3, 3), + MUX_CFG(MUX_LPTIM45, RCC_LPTIM45CKSELR, 0, 3), + MUX_CFG(MUX_MCO1, RCC_MCO1CFGR, 0, 3), + MUX_CFG(MUX_MCO2, RCC_MCO2CFGR, 0, 3), + MUX_CFG(MUX_QSPI, RCC_QSPICKSELR, 0, 2), + MUX_CFG(MUX_RNG1, RCC_RNG1CKSELR, 0, 2), + MUX_CFG(MUX_RTC, RCC_BDCR, 16, 2), + MUX_CFG(MUX_SAES, RCC_SAESCKSELR, 0, 2), + MUX_CFG(MUX_SAI1, RCC_SAI1CKSELR, 0, 3), + MUX_CFG(MUX_SAI2, RCC_SAI2CKSELR, 0, 3), + MUX_CFG(MUX_SDMMC1, RCC_SDMMC12CKSELR, 0, 3), + MUX_CFG(MUX_SDMMC2, RCC_SDMMC12CKSELR, 3, 3), + MUX_CFG(MUX_SPDIF, RCC_SPDIFCKSELR, 0, 2), + MUX_CFG(MUX_SPI1, RCC_SPI2S1CKSELR, 0, 3), + MUX_CFG(MUX_SPI23, RCC_SPI2S23CKSELR, 0, 3), + MUX_CFG(MUX_SPI4, RCC_SPI45CKSELR, 0, 3), + MUX_CFG(MUX_SPI5, RCC_SPI45CKSELR, 3, 3), + MUX_CFG(MUX_STGEN, RCC_STGENCKSELR, 0, 2), + MUX_CFG(MUX_UART1, RCC_UART12CKSELR, 0, 3), + MUX_CFG(MUX_UART2, RCC_UART12CKSELR, 3, 3), + MUX_CFG(MUX_UART35, RCC_UART35CKSELR, 0, 3), + MUX_CFG(MUX_UART4, RCC_UART4CKSELR, 0, 3), + MUX_CFG(MUX_UART6, RCC_UART6CKSELR, 0, 3), + MUX_CFG(MUX_UART78, RCC_UART78CKSELR, 0, 3), + MUX_CFG(MUX_USBO, RCC_USBCKSELR, 4, 1), + MUX_CFG(MUX_USBPHY, RCC_USBCKSELR, 0, 2), +}; + +/* + * DIV CONFIG + */ +static const struct div_table_cfg axi_div_table[] = { + { 0, 1 }, { 1, 2 }, { 2, 3 }, { 3, 4 }, + { 4, 4 }, { 5, 4 }, { 6, 4 }, { 7, 4 }, + { 0 }, +}; + +static const struct div_table_cfg mlahb_div_table[] = { + { 0, 1 }, { 1, 2 }, { 2, 4 }, { 3, 8 }, + { 4, 16 }, { 5, 32 }, { 6, 64 }, { 7, 128 }, + { 8, 256 }, { 9, 512 }, { 10, 512}, { 11, 512 }, + { 12, 512 }, { 13, 512 }, { 14, 512}, { 15, 512 }, + { 0 }, +}; + +static const struct div_table_cfg apb_div_table[] = { + { 0, 1 }, { 1, 2 }, { 2, 4 }, { 3, 8 }, + { 4, 16 }, { 5, 16 }, { 6, 16 }, { 7, 16 }, + { 0 }, +}; + +#define DIVRDY_CFG(_id, _offset, _shift, _width, _flags, _table, _ready)\ + [(_id)] = {\ + .offset = (_offset),\ + .shift = (_shift),\ + .width = (_width),\ + .flags = (_flags),\ + .table = (_table),\ + .ready = (_ready),\ + } + +#define DIV_CFG(_id, _offset, _shift, _width, _flags, _table)\ + DIVRDY_CFG(_id, _offset, _shift, _width, _flags, _table, DIV_NO_RDY) + +static const struct div_cfg dividers_mp13[] = { + DIVRDY_CFG(DIV_MPU, RCC_MPCKDIVR, 0, 4, 0, NULL, + GATE_MPUDIVRDY), + DIVRDY_CFG(DIV_AXI, RCC_AXIDIVR, 0, 3, 0, axi_div_table, + GATE_AXIDIVRDY), + DIVRDY_CFG(DIV_MLAHB, RCC_MLAHBDIVR, 0, 4, 0, mlahb_div_table, + GATE_MLAHBDIVRDY), + DIVRDY_CFG(DIV_APB1, RCC_APB1DIVR, 0, 3, 0, apb_div_table, + GATE_APB1DIVRDY), + DIVRDY_CFG(DIV_APB2, RCC_APB2DIVR, 0, 3, 0, apb_div_table, + GATE_APB2DIVRDY), + DIVRDY_CFG(DIV_APB3, RCC_APB3DIVR, 0, 3, 0, apb_div_table, + GATE_APB3DIVRDY), + DIVRDY_CFG(DIV_APB4, RCC_APB4DIVR, 0, 3, 0, apb_div_table, + GATE_APB4DIVRDY), + DIVRDY_CFG(DIV_APB5, RCC_APB5DIVR, 0, 3, 0, apb_div_table, + GATE_APB5DIVRDY), + DIVRDY_CFG(DIV_APB6, RCC_APB6DIVR, 0, 3, 0, apb_div_table, + GATE_APB6DIVRDY), + DIVRDY_CFG(DIV_HSI, RCC_HSICFGR, 0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL, + GATE_HSIDIVRDY), + DIV_CFG(DIV_PLL1DIVP, RCC_PLL1CFGR2, 0, 7, 0, NULL), + DIV_CFG(DIV_PLL2DIVP, RCC_PLL2CFGR2, 0, 7, 0, NULL), + DIV_CFG(DIV_PLL2DIVQ, RCC_PLL2CFGR2, 8, 7, 0, NULL), + DIV_CFG(DIV_PLL2DIVR, RCC_PLL2CFGR2, 16, 7, 0, NULL), + DIV_CFG(DIV_PLL3DIVP, RCC_PLL3CFGR2, 0, 7, 0, NULL), + DIV_CFG(DIV_PLL3DIVQ, RCC_PLL3CFGR2, 8, 7, 0, NULL), + DIV_CFG(DIV_PLL3DIVR, RCC_PLL3CFGR2, 16, 7, 0, NULL), + DIV_CFG(DIV_PLL4DIVP, RCC_PLL4CFGR2, 0, 7, 0, NULL), + DIV_CFG(DIV_PLL4DIVQ, RCC_PLL4CFGR2, 8, 7, 0, NULL), + DIV_CFG(DIV_PLL4DIVR, RCC_PLL4CFGR2, 16, 7, 0, NULL), + DIV_CFG(DIV_RTC, RCC_RTCDIVR, 0, 6, 0, NULL), + DIV_CFG(DIV_MCO1, RCC_MCO1CFGR, 4, 4, 0, NULL), + DIV_CFG(DIV_MCO2, RCC_MCO2CFGR, 4, 4, 0, NULL), + DIV_CFG(DIV_TRACE, RCC_DBGCFGR, 0, 3, CLK_DIVIDER_POWER_OF_TWO, NULL), + DIV_CFG(DIV_ETH1PTP, RCC_ETH12CKSELR, 4, 4, 0, NULL), + DIV_CFG(DIV_ETH2PTP, RCC_ETH12CKSELR, 12, 4, 0, NULL), +}; + +enum stm32_osc { + OSC_HSI, + OSC_HSE, + OSC_CSI, + OSC_LSI, + OSC_LSE, + NB_OSCILLATOR +}; + +struct stm32_osc_cfg { + int osc_id; +}; + +struct clk_stm32_bypass { + uint16_t offset; + uint8_t bit_byp; + uint8_t bit_digbyp; +}; + +struct clk_stm32_css { + uint16_t offset; + uint8_t bit_css; +}; + +struct clk_stm32_drive { + uint16_t offset; + uint8_t drv_shift; + uint8_t drv_width; + uint8_t drv_default; +}; + +struct clk_oscillator_data { + const char *name; + unsigned long frequency; + uint16_t gate_id; + struct clk_stm32_bypass *bypass; + struct clk_stm32_css *css; + struct clk_stm32_drive *drive; +}; + +#define BYPASS(_offset, _bit_byp, _bit_digbyp) (&(struct clk_stm32_bypass){\ + .offset = (_offset),\ + .bit_byp = (_bit_byp),\ + .bit_digbyp = (_bit_digbyp),\ +}) + +#define CSS(_offset, _bit_css) (&(struct clk_stm32_css){\ + .offset = (_offset),\ + .bit_css = (_bit_css),\ +}) + +#define DRIVE(_offset, _shift, _width, _default) (&(struct clk_stm32_drive){\ + .offset = (_offset),\ + .drv_shift = (_shift),\ + .drv_width = (_width),\ + .drv_default = (_default),\ +}) + +#define OSCILLATOR(idx_osc, _name, _gate_id, _bypass, _css, _drive) \ + [(idx_osc)] = (struct clk_oscillator_data){\ + .name = (_name),\ + .gate_id = (_gate_id),\ + .bypass = (_bypass),\ + .css = (_css),\ + .drive = (_drive),\ + } + +static struct clk_oscillator_data stm32mp13_osc_data[NB_OSCILLATOR] = { + OSCILLATOR(OSC_HSI, "clk-hsi", GATE_HSI, + NULL, NULL, NULL), + + OSCILLATOR(OSC_LSI, "clk-lsi", GATE_LSI, + NULL, NULL, NULL), + + OSCILLATOR(OSC_CSI, "clk-csi", GATE_CSI, + NULL, NULL, NULL), + + OSCILLATOR(OSC_LSE, "clk-lse", GATE_LSE, + BYPASS(RCC_BDCR, 1, 3), + CSS(RCC_BDCR, 8), + DRIVE(RCC_BDCR, 4, 2, 2)), + + OSCILLATOR(OSC_HSE, "clk-hse", GATE_HSE, + BYPASS(RCC_OCENSETR, 10, 7), + CSS(RCC_OCENSETR, 11), + NULL), +}; + +static struct clk_oscillator_data *clk_oscillator_get_data(int osc_id) +{ + assert(osc_id >= 0 && osc_id < (int)ARRAY_SIZE(stm32mp13_osc_data)); + + return &stm32mp13_osc_data[osc_id]; +} + +static unsigned long clk_stm32_get_rate_oscillateur(int osc_id) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + struct stm32_clk_platdata *pdata = priv->pdata; + struct stm32_osci_dt_cfg *osci = &pdata->osci[osc_id]; + + return osci->freq; +} + +static void clk_oscillator_set_bypass(struct clk_stm32_priv *priv, + struct clk_oscillator_data *osc_data, + bool digbyp, bool bypass) +{ + struct clk_stm32_bypass *bypass_data = osc_data->bypass; + uintptr_t address = 0; + + if (!bypass_data) + return; + + address = priv->base + bypass_data->offset; + + if (digbyp) + io_setbits32(address, BIT(bypass_data->bit_digbyp)); + + if (bypass || digbyp) + io_setbits32(address, BIT(bypass_data->bit_byp)); +} + +static void clk_oscillator_set_css(struct clk_stm32_priv *priv, + struct clk_oscillator_data *osc_data, + bool css) +{ + struct clk_stm32_css *css_data = osc_data->css; + uintptr_t address = 0; + + if (!css_data) + return; + + address = priv->base + css_data->offset; + + if (css) + io_setbits32(address, BIT(css_data->bit_css)); +} + +static void clk_oscillator_set_drive(struct clk_stm32_priv *priv, + struct clk_oscillator_data *osc_data, + uint8_t lsedrv) +{ + struct clk_stm32_drive *drive_data = osc_data->drive; + uintptr_t address = 0; + uint32_t mask = 0; + uint32_t value = 0; + + if (!drive_data) + return; + + address = priv->base + drive_data->offset; + + mask = (BIT(drive_data->drv_width) - 1U) << drive_data->drv_shift; + + /* + * Warning: not recommended to switch directly from "high drive" + * to "medium low drive", and vice-versa. + */ + value = (io_read32(address) & mask) >> drive_data->drv_shift; + + while (value != lsedrv) { + if (value > lsedrv) + value--; + else + value++; + + io_clrsetbits32(address, mask, value << drive_data->drv_shift); + } +} + +static void stm32_enable_oscillator_hse(struct clk_stm32_priv *priv, + struct stm32_clk_platdata *pdata) +{ + struct clk_oscillator_data *osc_data = clk_oscillator_get_data(OSC_HSE); + struct stm32_osci_dt_cfg *osci = &pdata->osci[OSC_HSE]; + + if (osci->freq == 0U) + return; + + clk_oscillator_set_bypass(priv, osc_data, osci->digbyp, osci->bypass); + + /* Enable clock and wait ready bit */ + if (stm32_gate_rdy_enable(osc_data->gate_id)) { + EMSG("timeout to enable hse clock"); + panic(); + } + + clk_oscillator_set_css(priv, osc_data, osci->css); +} + +static void stm32_enable_oscillator_lse(struct clk_stm32_priv *priv, + struct stm32_clk_platdata *pdata) +{ + struct clk_oscillator_data *osc_data = clk_oscillator_get_data(OSC_LSE); + struct stm32_osci_dt_cfg *osci = &pdata->osci[OSC_LSE]; + + if (osci->freq == 0U) + return; + + clk_oscillator_set_bypass(priv, osc_data, osci->digbyp, osci->bypass); + + clk_oscillator_set_drive(priv, osc_data, osci->drive); + + /* Enable lse clock, but don't wait ready bit */ + stm32_gate_enable(osc_data->gate_id); +} + +static void +stm32_enable_oscillator_lsi(struct clk_stm32_priv *priv __maybe_unused, + struct stm32_clk_platdata *pdata) +{ + struct clk_oscillator_data *osc_data = clk_oscillator_get_data(OSC_LSI); + struct stm32_osci_dt_cfg *osci = &pdata->osci[OSC_LSI]; + + if (osci->freq == 0U) + return; + + /* Enable clock and wait ready bit */ + if (stm32_gate_rdy_enable(osc_data->gate_id)) { + EMSG("timeout to enable lsi clock"); + panic(); + } +} + +static void +stm32_enable_oscillator_csi(struct clk_stm32_priv *priv __maybe_unused, + struct stm32_clk_platdata *pdata) +{ + struct clk_oscillator_data *osc_data = clk_oscillator_get_data(OSC_CSI); + struct stm32_osci_dt_cfg *osci = &pdata->osci[OSC_CSI]; + + if (osci->freq == 0U) + return; + + /* Enable clock and wait ready bit */ + if (stm32_gate_rdy_enable(osc_data->gate_id)) { + EMSG("timeout to enable csi clock"); + panic(); + } +} + +static int stm32_clk_oscillators_lse_set_css(struct clk_stm32_priv *priv, + struct stm32_clk_platdata *pdata) + +{ + struct clk_oscillator_data *osc_data = clk_oscillator_get_data(OSC_LSE); + struct stm32_osci_dt_cfg *osci = &pdata->osci[OSC_LSE]; + + clk_oscillator_set_css(priv, osc_data, osci->css); + + return 0; +} + +static int +stm32_clk_oscillators_wait_lse_ready(struct clk_stm32_priv *priv __maybe_unused, + struct stm32_clk_platdata *pdata) +{ + struct clk_oscillator_data *osc_data = clk_oscillator_get_data(OSC_LSE); + struct stm32_osci_dt_cfg *osci = &pdata->osci[OSC_LSE]; + + if (osci->freq == 0U) + return 0; + + if (stm32_gate_wait_ready(osc_data->gate_id, true)) + return -1; + + return 0; +} + +static void stm32_clk_oscillators_enable(struct clk_stm32_priv *priv, + struct stm32_clk_platdata *pdata) +{ + stm32_enable_oscillator_hse(priv, pdata); + stm32_enable_oscillator_lse(priv, pdata); + stm32_enable_oscillator_lsi(priv, pdata); + stm32_enable_oscillator_csi(priv, pdata); +} + +enum stm32_pll_id { + PLL1_ID, + PLL2_ID, + PLL3_ID, + PLL4_ID, + PLL_NB +}; + +enum stm32mp1_plltype { + PLL_800, + PLL_1600, + PLL_2000, + PLL_TYPE_NB +}; + +#define RCC_OFFSET_PLLXCR 0 +#define RCC_OFFSET_PLLXCFGR1 4 +#define RCC_OFFSET_PLLXCFGR2 8 +#define RCC_OFFSET_PLLXFRACR 12 +#define RCC_OFFSET_PLLXCSGR 16 + +struct stm32_clk_pll { + enum stm32mp1_plltype plltype; + uint16_t gate_id; + uint16_t mux_id; + uint16_t reg_pllxcr; +}; + +struct stm32mp1_pll { + uint8_t refclk_min; + uint8_t refclk_max; +}; + +/* Define characteristic of PLL according type */ +static const struct stm32mp1_pll stm32mp1_pll[PLL_TYPE_NB] = { + [PLL_800] = { + .refclk_min = 4, + .refclk_max = 16, + }, + [PLL_1600] = { + .refclk_min = 8, + .refclk_max = 16, + }, + [PLL_2000] = { + .refclk_min = 8, + .refclk_max = 16, + } +}; + +#define CLK_PLL_CFG(_idx, _type, _gate_id, _mux_id, _reg)\ + [(_idx)] = {\ + .gate_id = (_gate_id),\ + .mux_id = (_mux_id),\ + .plltype = (_type),\ + .reg_pllxcr = (_reg),\ + } + +static const struct stm32_clk_pll stm32_mp13_clk_pll[PLL_NB] = { + CLK_PLL_CFG(PLL1_ID, PLL_2000, GATE_PLL1, MUX_PLL12, RCC_PLL1CR), + CLK_PLL_CFG(PLL2_ID, PLL_1600, GATE_PLL2, MUX_PLL12, RCC_PLL2CR), + CLK_PLL_CFG(PLL3_ID, PLL_800, GATE_PLL3, MUX_PLL3, RCC_PLL3CR), + CLK_PLL_CFG(PLL4_ID, PLL_800, GATE_PLL4, MUX_PLL4, RCC_PLL4CR), +}; + +static const struct stm32_clk_pll *clk_stm32_pll_data(unsigned int idx) +{ + return &stm32_mp13_clk_pll[idx]; +} + +/* Clock TREE configuration */ + +static unsigned int stm32_clk_configure_clk_get_binding_id(uint32_t data) +{ + return (data & CLK_ID_MASK) >> CLK_ID_SHIFT; +} + +static int stm32_clk_configure_clk(struct clk_stm32_priv *priv __maybe_unused, + uint32_t data) +{ + int sel = (data & CLK_SEL_MASK) >> CLK_SEL_SHIFT; + int enable = (data & CLK_ON_MASK) >> CLK_ON_SHIFT; + int clk_id = 0; + int ret = 0; + int mux = -1; + int gate = -1; + + clk_id = stm32_clk_configure_clk_get_binding_id(data); + + switch (clk_id) { + case CK_MCO1: + mux = MUX_MCO1; + gate = GATE_MCO1; + break; + + case CK_MCO2: + mux = MUX_MCO2; + gate = GATE_MCO2; + break; + default: + ret = -1; + break; + } + + if (ret != 0) + return ret; + + if (stm32_mux_set_parent(mux, sel)) + return -1; + + if (enable) + stm32_gate_enable(gate); + else + stm32_gate_disable(gate); + + return 0; +} + +static int stm32_clk_configure_mux(__unused struct clk_stm32_priv *priv, + uint32_t data) +{ + int mux = (data & MUX_ID_MASK) >> MUX_ID_SHIFT; + int sel = (data & MUX_SEL_MASK) >> MUX_SEL_SHIFT; + + if (mux == MUX_RTC) { + /* Mux RTC clock only is selector is valid and RTC not yet + * enabled + */ + if (sel == 0) + return 0; + + if (stm32_gate_is_enabled(GATE_RTCCK)) + return 0; + } + + if (stm32_mux_set_parent(mux, sel)) + return -1; + + return 0; +} + +static TEE_Result +stm32_clk_configure_div(struct clk_stm32_priv *priv __maybe_unused, + uint32_t data) +{ + int div_id = (data & DIV_ID_MASK) >> DIV_ID_SHIFT; + int div_n = (data & DIV_DIVN_MASK) >> DIV_DIVN_SHIFT; + + return stm32_div_set_value(div_id, div_n); +} + +static int stm32_clk_dividers_configure(struct clk_stm32_priv *priv) +{ + struct stm32_clk_platdata *pdata = priv->pdata; + unsigned int i = 0; + + for (i = 0; i < pdata->nclkdiv; i++) { + if (stm32_clk_configure_div(priv, pdata->clkdiv[i])) + return -1; + } + + return 0; +} + +static int stm32_clk_source_configure(struct clk_stm32_priv *priv) +{ + struct stm32_clk_platdata *pdata = priv->pdata; + bool ckper_disabled = false; + int ret = 0; + size_t i = 0; + + for (i = 0; i < pdata->nclksrc; i++) { + uint32_t val = pdata->clksrc[i]; + uint32_t cmd = 0; + uint32_t cmd_data = 0; + + if (val == (uint32_t)CLK_CKPER_DISABLED) { + ckper_disabled = true; + continue; + } + + cmd = (val & CMD_MASK) >> CMD_SHIFT; + cmd_data = val & ~CMD_MASK; + + switch (cmd) { + case CMD_MUX: + ret = stm32_clk_configure_mux(priv, cmd_data); + break; + + case CMD_CLK: + ret = stm32_clk_configure_clk(priv, cmd_data); + break; + default: + ret = -1; + break; + } + + if (ret != 0) + return ret; + } + + /* + * CKPER is source for some peripheral clocks + * (FMC-NAND / QPSI-NOR) and switching source is allowed + * only if previous clock is still ON + * => deactivate CKPER only after switching clock + */ + if (ckper_disabled) { + ret = stm32_clk_configure_mux(priv, + CLK_CKPER_DISABLED & CMD_MASK); + if (ret != 0) + return ret; + } + + return 0; +} + +static unsigned long clk_stm32_pll_get_oscillator_rate(int sel) +{ + const int osc[] = { OSC_HSI, OSC_HSE, OSC_CSI }; + + assert(sel >= 0 && sel < (int)ARRAY_SIZE(osc)); + + return clk_stm32_get_rate_oscillateur(osc[sel]); +} + +static int clk_stm32_pll_compute_cfgr1(const struct stm32_clk_pll *pll, + struct stm32_pll_vco *vco, + uint32_t *value) +{ + int sel = (vco->src & MUX_SEL_MASK) >> MUX_SEL_SHIFT; + uint32_t divm = vco->div_mn[PLL_CFG_M]; + uint32_t divn = vco->div_mn[PLL_CFG_N]; + unsigned long refclk = 0UL; + + refclk = clk_stm32_pll_get_oscillator_rate(sel) / (divm + 1U); + + if ((refclk < (stm32mp1_pll[pll->plltype].refclk_min * 1000000U)) || + (refclk > (stm32mp1_pll[pll->plltype].refclk_max * 1000000U))) + return -1; + + *value = 0; + + if (pll->plltype == PLL_800 && refclk >= 8000000U) + *value = 1U << RCC_PLLNCFGR1_IFRGE_SHIFT; + + *value |= (divn << RCC_PLLNCFGR1_DIVN_SHIFT) & RCC_PLLNCFGR1_DIVN_MASK; + *value |= (divm << RCC_PLLNCFGR1_DIVM_SHIFT) & RCC_PLLNCFGR1_DIVM_MASK; + + return 0; +} + +static uint32_t clk_stm32_pll_compute_cfgr2(struct stm32_pll_output *out) +{ + uint32_t value = 0; + + value |= (out->output[PLL_CFG_P] << RCC_PLLNCFGR2_DIVP_SHIFT) & + RCC_PLLNCFGR2_DIVP_MASK; + value |= (out->output[PLL_CFG_Q] << RCC_PLLNCFGR2_DIVQ_SHIFT) & + RCC_PLLNCFGR2_DIVQ_MASK; + value |= (out->output[PLL_CFG_R] << RCC_PLLNCFGR2_DIVR_SHIFT) & + RCC_PLLNCFGR2_DIVR_MASK; + + return value; +} + +/* + * Check if PLL1 can be configured on the fly. + * @result (-1) => config on the fly is not possible. + * (0) => config on the fly is possible. + * (+1) => same parameters, no need to reconfigure. + * Return value is 0 if no error. + */ +static int clk_stm32_is_pll_config_on_the_fly(struct clk_stm32_priv *priv, + const struct stm32_clk_pll *pll, + struct stm32_pll_dt_cfg *pll_conf, + int *result) +{ + uintptr_t pll_base = priv->base + pll->reg_pllxcr; + struct stm32_pll_vco *vco = &pll_conf->vco; + struct stm32_pll_output *out = &pll_conf->output; + uint32_t fracr = 0; + uint32_t value = 0; + int ret = 0; + size_t sel = 0; + + ret = clk_stm32_pll_compute_cfgr1(pll, vco, &value); + if (ret != 0) + return ret; + + sel = (vco->src & MUX_SEL_MASK) >> MUX_SEL_SHIFT; + if (sel != stm32_mux_get_parent(pll->mux_id)) { + /* Clock source of the PLL is different */ + *result = -1; + return 0; + } + + if (io_read32(pll_base + RCC_OFFSET_PLLXCFGR1) != value) { + /* Different DIVN/DIVM, can't config on the fly */ + *result = -1; + return 0; + } + + *result = 1; + + fracr = vco->frac << RCC_PLLNFRACR_FRACV_SHIFT; + fracr |= RCC_PLLNCFGR1_DIVM_MASK; + value = clk_stm32_pll_compute_cfgr2(out); + + if ((io_read32(pll_base + RCC_OFFSET_PLLXFRACR) == fracr) && + (io_read32(pll_base + RCC_OFFSET_PLLXCFGR2) == value)) { + /* Same parameters, no need to config */ + *result = 1; + } else { + *result = 0; + } + + return 0; +} + +static int stm32_clk_hsidiv_configure(struct clk_stm32_priv *priv) +{ + struct stm32_clk_platdata *pdata = priv->pdata; + struct stm32_osci_dt_cfg *osci = &pdata->osci[OSC_HSI]; + + return stm32_div_set_rate(DIV_HSI, osci->freq, MAX_HSI_HZ); +} + +static void clk_stm32_pll_config_vco(struct clk_stm32_priv *priv, + const struct stm32_clk_pll *pll, + struct stm32_pll_vco *vco) +{ + uintptr_t pll_base = priv->base + pll->reg_pllxcr; + uint32_t value = 0; + + if (clk_stm32_pll_compute_cfgr1(pll, vco, &value) != 0) { + EMSG("Invalid Vref clock"); + panic(); + } + + /* Write N / M / IFREGE fields */ + io_write32(pll_base + RCC_OFFSET_PLLXCFGR1, value); + + /* Fractional configuration */ + io_write32(pll_base + RCC_OFFSET_PLLXFRACR, 0); + + /* Frac must be enabled only once its configuration is loaded */ + io_write32(pll_base + RCC_OFFSET_PLLXFRACR, + vco->frac << RCC_PLLNFRACR_FRACV_SHIFT); + + io_setbits32(pll_base + RCC_OFFSET_PLLXFRACR, RCC_PLLNFRACR_FRACLE); +} + +static void clk_stm32_pll_config_csg(struct clk_stm32_priv *priv, + const struct stm32_clk_pll *pll, + struct stm32_pll_vco *vco) +{ + uintptr_t pll_base = priv->base + pll->reg_pllxcr; + uint32_t mod_per = 0; + uint32_t inc_step = 0; + uint32_t sscg_mode = 0; + uint32_t value = 0; + + if (!vco->csg_enabled) + return; + + mod_per = vco->csg[PLL_CSG_MOD_PER]; + inc_step = vco->csg[PLL_CSG_INC_STEP]; + sscg_mode = vco->csg[PLL_CSG_SSCG_MODE]; + + value |= (mod_per << RCC_PLLNCSGR_MOD_PER_SHIFT) & + RCC_PLLNCSGR_MOD_PER_MASK; + value |= (inc_step << RCC_PLLNCSGR_INC_STEP_SHIFT) & + RCC_PLLNCSGR_INC_STEP_MASK; + value |= (sscg_mode << RCC_PLLNCSGR_SSCG_MODE_SHIFT) & + RCC_PLLNCSGR_SSCG_MODE_MASK; + + io_write32(pll_base + RCC_OFFSET_PLLXCSGR, value); + io_setbits32(pll_base + RCC_OFFSET_PLLXCR, RCC_PLLNCR_SSCG_CTRL); +} + +static void clk_stm32_pll_config_out(struct clk_stm32_priv *priv, + const struct stm32_clk_pll *pll, + struct stm32_pll_output *out) +{ + uintptr_t pll_base = priv->base + pll->reg_pllxcr; + uint32_t value = 0; + + value = clk_stm32_pll_compute_cfgr2(out); + + io_write32(pll_base + RCC_OFFSET_PLLXCFGR2, value); +} + +static struct stm32_pll_dt_cfg *clk_stm32_pll_get_pdata(int pll_idx) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + struct stm32_clk_platdata *pdata = priv->pdata; + + return &pdata->pll[pll_idx]; +} + +static int clk_stm32_pll_init_switch_to_hsi_clk_system(int mux_sys) +{ + int sel = 0; + + if (mux_sys == -1) + return -1; + + /* Make a backup to the current parent */ + sel = stm32_mux_get_parent(mux_sys); + + /* Switch to HSI */ + if (stm32_mux_set_parent(mux_sys, 0)) + return -1; + + return sel; +} + +static uint32_t +clk_stm32_pll_backup_output_diven(const struct stm32_clk_pll *pll) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + uintptr_t addr = priv->base + pll->reg_pllxcr; + + return io_read32(addr + RCC_OFFSET_PLLXCR) & + (RCC_PLLNCR_DIVPEN | RCC_PLLNCR_DIVQEN | + RCC_PLLNCR_DIVREN); +} + +static void clk_stm32_pll_restore_output_diven(const struct stm32_clk_pll *pll, + uint32_t value) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + uintptr_t addr = priv->base + pll->reg_pllxcr; + const uint32_t mask = RCC_PLLNCR_DIVPEN | RCC_PLLNCR_DIVQEN | + RCC_PLLNCR_DIVREN; + + io_clrsetbits32(addr, mask, value & mask); +} + +static int clk_stm32_pll_init(struct clk_stm32_priv *priv, int pll_idx, + struct stm32_pll_dt_cfg *pll_conf) +{ + const struct stm32_clk_pll *pll = clk_stm32_pll_data(pll_idx); + int config_on_the_fly = -1; + int ret = 0; + uint8_t sel = 0; + uint32_t save_div_pqr_en = 0; + int mux_system[] = { MUX_MPU, MUX_AXI, MUX_MLAHB, -1 }; + int mux_sys = mux_system[pll_idx]; + + ret = clk_stm32_is_pll_config_on_the_fly(priv, pll, pll_conf, + &config_on_the_fly); + if (ret != 0) + return ret; + + /* Backup status of DIV DIVPEN / DIVQEN / DIVREN */ + save_div_pqr_en = clk_stm32_pll_backup_output_diven(pll); + + if (config_on_the_fly == -1) { + /* Make a backup to the current parent and switch to HSI */ + sel = clk_stm32_pll_init_switch_to_hsi_clk_system(mux_sys); + + /* Stop the PLL before */ + if (stm32_gate_is_enabled(pll->gate_id)) { + io_clrbits32(priv->base + pll->reg_pllxcr, + RCC_PLLNCR_DIVPEN | RCC_PLLNCR_DIVQEN | + RCC_PLLNCR_DIVREN); + + if (stm32_gate_rdy_disable(pll->gate_id)) + return -1; + } + + /* Configure PLLs source */ + ret = stm32_clk_configure_mux(priv, pll_conf->vco.src); + if (ret) + return ret; + + clk_stm32_pll_config_vco(priv, pll, &pll_conf->vco); + } + + if (config_on_the_fly != 1) { + clk_stm32_pll_config_out(priv, pll, &pll_conf->output); + clk_stm32_pll_config_csg(priv, pll, &pll_conf->vco); + } + + if (!stm32_gate_is_enabled(pll->gate_id)) { + if (stm32_gate_rdy_enable(pll->gate_id)) + return -1; + + clk_stm32_pll_restore_output_diven(pll, save_div_pqr_en); + } + + if ((config_on_the_fly == -1) && (mux_sys != -1)) { + /* Restore to backup parent */ + if (stm32_mux_set_parent(mux_sys, sel)) + return -1; + } + + return 0; +} + +static int stm32_clk_pll_configure(struct clk_stm32_priv *priv) +{ + struct stm32_pll_dt_cfg *pll_conf = NULL; + size_t i = 0; + const int plls[] = { PLL1_ID, PLL3_ID, PLL4_ID }; + + for (i = 0; i < ARRAY_SIZE(plls); i++) { + pll_conf = clk_stm32_pll_get_pdata(plls[i]); + + if (pll_conf->vco.status) { + int err = 0; + + err = clk_stm32_pll_init(priv, plls[i], pll_conf); + if (err) + return err; + } + } + + return 0; +} + +static int stm32mp1_init_clock_tree(struct clk_stm32_priv *priv, + struct stm32_clk_platdata *pdata) +{ + int ret = 0; + + /* + * Switch ON oscillators found in device-tree. + * Note: HSI already ON after BootROM stage. + */ + stm32_clk_oscillators_enable(priv, pdata); + + ret = stm32_clk_hsidiv_configure(priv); + if (ret != 0) + return ret; + + ret = stm32_clk_dividers_configure(priv); + if (ret != 0) + panic(); + + ret = stm32_clk_pll_configure(priv); + if (ret != 0) + panic(); + + /* Wait LSE ready before to use it */ + ret = stm32_clk_oscillators_wait_lse_ready(priv, pdata); + if (ret != 0) + panic(); + + /* Configure with expected clock source */ + ret = stm32_clk_source_configure(priv); + if (ret != 0) + panic(); + + /* Configure LSE CSS after RTC source configuration */ + ret = stm32_clk_oscillators_lse_set_css(priv, pdata); + if (ret != 0) + panic(); + + /* Software Self-Refresh mode (SSR) during DDR initilialization */ + io_clrsetbits32(priv->base + RCC_DDRITFCR, RCC_DDRITFCR_DDRCKMOD_MASK, + RCC_DDRITFCR_DDRCKMOD_SSR << + RCC_DDRITFCR_DDRCKMOD_SHIFT); + + return 0; +} + +static int clk_stm32_parse_oscillator_fdt(const void *fdt, int node, + const char *name, + struct stm32_osci_dt_cfg *osci) +{ + int subnode = 0; + + fdt_for_each_subnode(subnode, fdt, node) { + const char *cchar = NULL; + const fdt32_t *cuint = NULL; + int ret = 0; + + cchar = fdt_get_name(fdt, subnode, &ret); + if (!cchar) + return ret; + + if (strncmp(cchar, name, (size_t)ret) || + fdt_get_status(fdt, subnode) == DT_STATUS_DISABLED) + continue; + + cuint = fdt_getprop(fdt, subnode, "clock-frequency", &ret); + if (!cuint) + panic(); + + osci->freq = fdt32_to_cpu(*cuint); + + if (fdt_getprop(fdt, subnode, "st,bypass", NULL)) + osci->bypass = true; + + if (fdt_getprop(fdt, subnode, "st,digbypass", NULL)) + osci->digbyp = true; + + if (fdt_getprop(fdt, subnode, "st,css", NULL)) + osci->css = true; + + osci->drive = fdt_read_uint32_default(fdt, subnode, "st,drive", + LSEDRV_MEDIUM_HIGH); + + return 0; + } + + return -FDT_ERR_NOTFOUND; +} + +static int stm32_clk_parse_fdt_all_oscillator(const void *fdt, + int node __maybe_unused, + struct stm32_clk_platdata *pdata) +{ + int fdt_err = 0; + size_t i = 0; + int osc_node = 0; + + osc_node = fdt_path_offset(fdt, "/clocks"); + if (osc_node < 0) + return -FDT_ERR_NOTFOUND; + + for (i = 0; i < NB_OSCILLATOR; i++) { + struct stm32_osci_dt_cfg *osci = &pdata->osci[i]; + struct clk_oscillator_data *osc_data = NULL; + + osc_data = clk_oscillator_get_data(i); + + fdt_err = clk_stm32_parse_oscillator_fdt(fdt, osc_node, + osc_data->name, osci); + if (fdt_err < 0) + panic(); + } + + return 0; +} + +static int clk_stm32_load_vco_config_fdt(const void *fdt, int subnode, + struct stm32_pll_vco *vco) +{ + int ret = 0; + + ret = fdt_read_uint32_array(fdt, subnode, "divmn", vco->div_mn, + PLL_DIV_MN_NB); + if (ret != 0) + return ret; + + ret = fdt_read_uint32_array(fdt, subnode, "csg", vco->csg, + PLL_CSG_NB); + + vco->csg_enabled = (ret == 0); + + if (ret == -FDT_ERR_NOTFOUND) + ret = 0; + + if (ret != 0) + return ret; + + vco->status = RCC_PLLNCR_DIVPEN | RCC_PLLNCR_DIVQEN | + RCC_PLLNCR_DIVREN | RCC_PLLNCR_PLLON; + + vco->frac = fdt_read_uint32_default(fdt, subnode, "frac", 0); + + vco->src = fdt_read_uint32_default(fdt, subnode, "src", UINT32_MAX); + + return 0; +} + +static int clk_stm32_load_output_config_fdt(const void *fdt, int subnode, + struct stm32_pll_output *output) +{ + return fdt_read_uint32_array(fdt, subnode, "st,pll_div_pqr", + output->output, (int)PLL_DIV_PQR_NB); +} + +static int clk_stm32_parse_pll_fdt(const void *fdt, int subnode, + struct stm32_pll_dt_cfg *pll) +{ + const fdt32_t *cuint = NULL; + int subnode_pll = 0; + int subnode_vco = 0; + int err = 0; + + cuint = fdt_getprop(fdt, subnode, "st,pll", NULL); + if (!cuint) + return -FDT_ERR_NOTFOUND; + + subnode_pll = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(*cuint)); + if (subnode_pll < 0) + return -FDT_ERR_NOTFOUND; + + cuint = fdt_getprop(fdt, subnode_pll, "st,pll_vco", NULL); + if (!cuint) + return -FDT_ERR_NOTFOUND; + + subnode_vco = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(*cuint)); + if (subnode_vco < 0) + return -FDT_ERR_NOTFOUND; + + err = clk_stm32_load_vco_config_fdt(fdt, subnode_vco, &pll->vco); + if (err != 0) + return err; + + err = clk_stm32_load_output_config_fdt(fdt, subnode_pll, &pll->output); + if (err != 0) + return err; + + return 0; +} + +static int stm32_clk_parse_fdt_all_pll(const void *fdt, int node, + struct stm32_clk_platdata *pdata) +{ + size_t i = 0; + + for (i = PLL1_ID; i < pdata->npll; i++) { + struct stm32_pll_dt_cfg *pll = pdata->pll + i; + char name[RCC_PLL_NAME_SIZE] = { 0 }; + int subnode = 0; + int err = 0; + + snprintf(name, sizeof(name), "st,pll@%d", i); + + subnode = fdt_subnode_offset(fdt, node, name); + if (subnode < 0) + continue; + + err = clk_stm32_parse_pll_fdt(fdt, subnode, pll); + if (err != 0) + panic(); + } + + return 0; +} + +static int stm32_clk_parse_fdt_opp(const void *fdt, int node, + const char *opp_name, + struct stm32_clk_opp_cfg *opp_cfg) +{ + int subnode = 0; + int nb_opp = 0; + int ret = 0; + + node = fdt_subnode_offset(fdt, node, opp_name); + if (node == -FDT_ERR_NOTFOUND) + return 0; + if (node < 0) + return node; + + fdt_for_each_subnode(subnode, fdt, node) { + if (nb_opp >= MAX_OPP) { + EMSG("%d MAX opp in %s", MAX_OPP, opp_name); + panic(); + } + + opp_cfg->frq = fdt_read_uint32_default(fdt, subnode, + "hz", + UINT32_MAX); + + opp_cfg->src = fdt_read_uint32_default(fdt, subnode, + "st,clksrc", + UINT32_MAX); + + opp_cfg->div = fdt_read_uint32_default(fdt, subnode, + "st,clkdiv", + UINT32_MAX); + + ret = clk_stm32_parse_pll_fdt(fdt, subnode, &opp_cfg->pll_cfg); + if (ret) + return ret; + + opp_cfg++; + nb_opp++; + } + + return 0; +} + +static int stm32_clk_parse_fdt_all_opp(const void *fdt, int node, + struct stm32_clk_platdata *pdata) +{ + struct stm32_clk_opp_dt_cfg *opp = pdata->opp; + int ret = 0; + + node = fdt_subnode_offset(fdt, node, "st,clk_opp"); + /* No opp are defined */ + if (node == -FDT_ERR_NOTFOUND) + return 0; + if (node < 0) + return node; + + ret = stm32_clk_parse_fdt_opp(fdt, node, "st,ck_mpu", opp->mpu_opp); + if (ret) + return ret; + + ret = stm32_clk_parse_fdt_opp(fdt, node, "st,ck_axi", opp->axi_opp); + if (ret) + return ret; + + ret = stm32_clk_parse_fdt_opp(fdt, node, "st,ck_mlahbs", + opp->mlahbs_opp); + if (ret) + return ret; + + return 0; +} + +static int stm32_clk_parse_fdt(const void *fdt, int node, + struct stm32_clk_platdata *pdata) +{ + int err = 0; + + err = stm32_clk_parse_fdt_all_oscillator(fdt, node, pdata); + if (err != 0) + return err; + + err = stm32_clk_parse_fdt_all_pll(fdt, node, pdata); + if (err != 0) + return err; + + err = stm32_clk_parse_fdt_all_opp(fdt, node, pdata); + if (err != 0) + return err; + + err = clk_stm32_parse_fdt_by_name(fdt, node, "st,clkdiv", pdata->clkdiv, + &pdata->nclkdiv); + if (err != 0) + return err; + + err = clk_stm32_parse_fdt_by_name(fdt, node, "st,clksrc", pdata->clksrc, + &pdata->nclksrc); + if (err != 0) + return err; + + return 0; +} + +struct clk_stm32_pll_cfg { + uint32_t reg_pllxcr; + int gate_id; + int mux_id; +}; + +static size_t clk_stm32_pll_get_parent(struct clk *clk) +{ + struct clk_stm32_pll_cfg *cfg = clk->priv; + + return stm32_mux_get_parent(cfg->mux_id); +} + +static unsigned long clk_stm32_pll_get_rate(struct clk *clk, + unsigned long prate) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + struct clk_stm32_pll_cfg *cfg = clk->priv; + uintptr_t pll_base = priv->base + cfg->reg_pllxcr; + uint32_t cfgr1 = 0; + uint32_t fracr = 0; + uint32_t divm = 0; + uint32_t divn = 0; + unsigned long fvco = 0UL; + + cfgr1 = io_read32(pll_base + RCC_OFFSET_PLLXCFGR1); + fracr = io_read32(pll_base + RCC_OFFSET_PLLXFRACR); + + divm = (cfgr1 & (RCC_PLLNCFGR1_DIVM_MASK)) >> RCC_PLLNCFGR1_DIVM_SHIFT; + divn = cfgr1 & RCC_PLLNCFGR1_DIVN_MASK; + + /* + * With FRACV : + * Fvco = Fck_ref * ((DIVN + 1) + FRACV / 2^13) / (DIVM + 1) + * Without FRACV + * Fvco = Fck_ref * ((DIVN + 1) / (DIVM + 1) + */ + if ((fracr & RCC_PLLNFRACR_FRACLE) != 0U) { + uint32_t fracv = (fracr & RCC_PLLNFRACR_FRACV_MASK) >> + RCC_PLLNFRACR_FRACV_SHIFT; + unsigned long long numerator = 0UL; + unsigned long long denominator = 0UL; + + numerator = (((unsigned long long)divn + 1U) << 13) + fracv; + numerator = prate * numerator; + denominator = ((unsigned long long)divm + 1U) << 13; + fvco = (unsigned long)(numerator / denominator); + } else { + fvco = (unsigned long)(prate * (divn + 1U) / (divm + 1U)); + } + + return fvco; +}; + +static bool clk_stm32_pll_is_enabled(struct clk *clk) +{ + struct clk_stm32_pll_cfg *cfg = clk->priv; + + return stm32_gate_is_enabled(cfg->gate_id); +} + +static TEE_Result clk_stm32_pll_enable(struct clk *clk) +{ + struct clk_stm32_pll_cfg *cfg = clk->priv; + + if (clk_stm32_pll_is_enabled(clk)) + return TEE_SUCCESS; + + return stm32_gate_rdy_enable(cfg->gate_id); +} + +static void clk_stm32_pll_disable(struct clk *clk) +{ + struct clk_stm32_pll_cfg *cfg = clk->priv; + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + uintptr_t pll_base = priv->base + cfg->reg_pllxcr; + + if (!clk_stm32_pll_is_enabled(clk)) + return; + + /* Stop all output */ + io_clrbits32(pll_base, RCC_PLLNCR_DIVPEN | RCC_PLLNCR_DIVQEN | + RCC_PLLNCR_DIVREN); + + stm32_gate_rdy_disable(cfg->gate_id); +} + +static const struct clk_ops clk_stm32_pll_ops = { + .get_parent = clk_stm32_pll_get_parent, + .get_rate = clk_stm32_pll_get_rate, + .enable = clk_stm32_pll_enable, + .disable = clk_stm32_pll_disable, +}; + +static struct +stm32_clk_opp_cfg *clk_stm32_get_opp_config(struct stm32_clk_opp_cfg *opp_cfg, + unsigned long rate) +{ + unsigned int i = 0; + + for (i = 0; i < MAX_OPP; i++, opp_cfg++) { + if (opp_cfg->frq == 0UL) + break; + + if (opp_cfg->frq == rate) + return opp_cfg; + } + + return NULL; +} + +static TEE_Result clk_stm32_pll1_set_rate(struct clk *clk __maybe_unused, + unsigned long rate, + unsigned long prate __maybe_unused) +{ + const struct stm32_clk_pll *pll = clk_stm32_pll_data(PLL1_ID); + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + struct stm32_clk_platdata *pdata = priv->pdata; + struct stm32_pll_dt_cfg *pll_conf = NULL; + struct stm32_clk_opp_cfg *opp = NULL; + int config_on_the_fly = -1; + int err = 0; + size_t sel = stm32_mux_get_parent(MUX_MPU); + + opp = clk_stm32_get_opp_config(pdata->opp->mpu_opp, rate); + if (!opp) + return TEE_ERROR_GENERIC; + + pll_conf = &opp->pll_cfg; + + err = clk_stm32_is_pll_config_on_the_fly(priv, pll, pll_conf, + &config_on_the_fly); + if (err) + return TEE_ERROR_GENERIC; + + if (config_on_the_fly == 1) + return TEE_SUCCESS; + + if (config_on_the_fly == -1) { + /* Switch to HSI and stop PLL1 before reconfiguration */ + if (stm32_mux_set_parent(MUX_MPU, 0)) + return TEE_ERROR_GENERIC; + + stm32_gate_disable(GATE_PLL1_DIVP); + stm32_gate_rdy_disable(GATE_PLL1); + clk_stm32_pll_config_vco(priv, pll, &pll_conf->vco); + } + + clk_stm32_pll_config_out(priv, pll, &pll_conf->output); + if (stm32_gate_rdy_enable(GATE_PLL1)) { + EMSG("timeout to enable PLL1 clock"); + panic(); + } + stm32_gate_enable(GATE_PLL1_DIVP); + + /* Restore MPU source */ + if (stm32_mux_set_parent(MUX_MPU, sel)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static const struct clk_ops clk_stm32_pll1_ops = { + .set_rate = clk_stm32_pll1_set_rate, + .get_parent = clk_stm32_pll_get_parent, + .get_rate = clk_stm32_pll_get_rate, + .enable = clk_stm32_pll_enable, + .disable = clk_stm32_pll_disable, +}; + +static const struct clk_ops clk_stm32_pll1p_ops = { + .get_rate = clk_stm32_composite_get_rate, + .enable = clk_stm32_composite_gate_enable, + .disable = clk_stm32_composite_gate_disable, +}; + +static const struct clk_ops clk_stm32_mpu_ops = { + .get_parent = clk_stm32_composite_get_parent, + .set_parent = clk_stm32_composite_set_parent, +}; + +static const struct clk_ops clk_stm32_axi_ops = { + .get_parent = clk_stm32_composite_get_parent, + .set_parent = clk_stm32_composite_set_parent, + .set_rate = clk_stm32_composite_set_rate, + .get_rate = clk_stm32_composite_get_rate, +}; + +const struct clk_ops clk_stm32_mlahb_ops = { + .get_parent = clk_stm32_composite_get_parent, + .set_parent = clk_stm32_composite_set_parent, + .set_rate = clk_stm32_composite_set_rate, + .get_rate = clk_stm32_composite_get_rate, +}; + +#define APB_DIV_MASK GENMASK_32(2, 0) +#define TIM_PRE_MASK BIT(0) + +static unsigned long ck_timer_get_rate_ops(struct clk *clk, unsigned long prate) +{ + struct clk_stm32_priv *priv = clk_stm32_get_priv(); + struct clk_stm32_timer_cfg *cfg = clk->priv; + uint32_t prescaler, timpre; + uintptr_t rcc_base = priv->base; + + prescaler = io_read32(rcc_base + cfg->apbdiv) & APB_DIV_MASK; + + timpre = io_read32(rcc_base + cfg->timpre) & TIM_PRE_MASK; + + if (prescaler == 0U) + return prate; + + return prate * (timpre + 1U) * 2U; +}; + +const struct clk_ops ck_timer_ops = { + .get_rate = ck_timer_get_rate_ops, +}; + +#define STM32_TIMER(_name, _parent, _flags, _apbdiv, _timpre)\ + struct clk _name = {\ + .ops = &ck_timer_ops,\ + .priv = &(struct clk_stm32_timer_cfg) {\ + .apbdiv = (_apbdiv),\ + .timpre = (_timpre),\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = 1,\ + .parents = { _parent },\ + } + +#define STM32_KCLK(_name, _nb_parents, _parents, _flags, _gate_id, _mux_id)\ + struct clk _name = {\ + .ops = &clk_stm32_composite_ops,\ + .priv = &(struct clk_stm32_composite_cfg) {\ + .gate_id = (_gate_id),\ + .div_id = (NO_DIV),\ + .mux_id = (_mux_id),\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = (_nb_parents),\ + .parents = _parents,\ + } + +#define STM32_PLL_VCO(_name, _nb_parents, _parents, _flags, _reg,\ + _gate_id, _mux_id)\ + struct clk _name = {\ + .ops = &clk_stm32_pll_ops,\ + .priv = &(struct clk_stm32_pll_cfg) {\ + .reg_pllxcr = (_reg),\ + .gate_id = (_gate_id),\ + .mux_id = (_mux_id),\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = (_nb_parents),\ + .parents = _parents,\ + } + +#define STM32_PLL_OUPUT(_name, _nb_parents, _parents, _flags,\ + _gate_id, _div_id, _mux_id)\ + struct clk _name = {\ + .ops = &clk_stm32_composite_ops,\ + .priv = &(struct clk_stm32_composite_cfg) {\ + .gate_id = (_gate_id),\ + .div_id = (_div_id),\ + .mux_id = (_mux_id),\ + },\ + .name = #_name,\ + .flags = (_flags),\ + .num_parents = (_nb_parents),\ + .parents = _parents,\ + } + +/* Oscillator clocks */ +static STM32_GATE_READY(ck_hsi, NULL, 0, GATE_HSI); +static STM32_GATE_READY(ck_hse, NULL, 0, GATE_HSE); +static STM32_GATE_READY(ck_csi, NULL, 0, GATE_CSI); +static STM32_GATE_READY(ck_lsi, NULL, 0, GATE_LSI); +static STM32_GATE_READY(ck_lse, NULL, 0, GATE_LSE); + +static STM32_FIXED_FACTOR(ck_i2sckin, NULL, 0, 1, 1); +static STM32_FIXED_FACTOR(ck_hse_div2, &ck_hse, 0, 1, 2); + +static STM32_FIXED_RATE(ck_off, 0UL); +static STM32_FIXED_RATE(ck_usb_phy_48Mhz, USB_PHY_48_MHZ); + +/* PLL1 clocks */ +static struct clk ck_pll1_vco = { + .ops = &clk_stm32_pll1_ops, + .priv = &(struct clk_stm32_pll_cfg) { + .reg_pllxcr = RCC_PLL1CR, + .gate_id = GATE_PLL1, + .mux_id = MUX_PLL12, + }, + .name = "ck_pll1_vco", + .flags = 0, + .num_parents = 2, + .parents = { &ck_hsi, &ck_hse }, +}; + +static struct clk ck_pll1p = { + .ops = &clk_stm32_pll1p_ops, + .priv = &(struct clk_stm32_composite_cfg) { + .gate_id = GATE_PLL1_DIVP, + .div_id = DIV_PLL1DIVP, + .mux_id = NO_MUX, + }, + .name = "ck_pll1p", + .flags = 0, + .num_parents = 1, + .parents = { &ck_pll1_vco }, +}; + +const struct clk_ops clk_stm32_pll1p_div_ops = { + .get_rate = clk_stm32_divider_get_rate, +}; + +static struct clk ck_pll1p_div = { + .ops = &clk_stm32_pll1p_div_ops, + .priv = &(struct clk_stm32_div_cfg) { + .div_id = DIV_MPU, + }, + .name = "ck_pll1p_div", + .flags = 0, + .num_parents = 1, + .parents = { &ck_pll1p }, +}; + +/* Other PLLs */ +static STM32_PLL_VCO(ck_pll2_vco, 2, PARENT(&ck_hsi, &ck_hse), + 0, RCC_PLL2CR, GATE_PLL2, MUX_PLL12); + +static STM32_PLL_VCO(ck_pll3_vco, 3, + PARENT(&ck_hsi, &ck_hse, &ck_csi), + 0, RCC_PLL3CR, GATE_PLL3, MUX_PLL3); + +static STM32_PLL_VCO(ck_pll4_vco, 4, + PARENT(&ck_hsi, &ck_hse, &ck_csi, &ck_i2sckin), + 0, RCC_PLL4CR, GATE_PLL4, MUX_PLL4); + +static STM32_PLL_OUPUT(ck_pll2p, 1, PARENT(&ck_pll2_vco), 0, + GATE_PLL2_DIVP, DIV_PLL2DIVP, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll2q, 1, PARENT(&ck_pll2_vco), 0, + GATE_PLL2_DIVQ, DIV_PLL2DIVQ, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll2r, 1, PARENT(&ck_pll2_vco), 0, + GATE_PLL2_DIVR, DIV_PLL2DIVR, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll3p, 1, PARENT(&ck_pll3_vco), 0, + GATE_PLL3_DIVP, DIV_PLL3DIVP, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll3q, 1, PARENT(&ck_pll3_vco), 0, + GATE_PLL3_DIVQ, DIV_PLL3DIVQ, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll3r, 1, PARENT(&ck_pll3_vco), 0, + GATE_PLL3_DIVR, DIV_PLL3DIVR, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll4p, 1, PARENT(&ck_pll4_vco), 0, + GATE_PLL4_DIVP, DIV_PLL4DIVP, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll4q, 1, PARENT(&ck_pll4_vco), 0, + GATE_PLL4_DIVQ, DIV_PLL4DIVQ, NO_MUX); + +static STM32_PLL_OUPUT(ck_pll4r, 1, PARENT(&ck_pll4_vco), 0, + GATE_PLL4_DIVR, DIV_PLL4DIVR, NO_MUX); + +/* System clocks */ +static struct clk ck_mpu = { + .ops = &clk_stm32_mpu_ops, + .priv = &(struct clk_stm32_composite_cfg) { + .mux_id = MUX_MPU, + }, + .name = "ck_mpu", + .flags = 0, + .num_parents = 4, + .parents = { &ck_hsi, &ck_hse, &ck_pll1p, &ck_pll1p_div }, +}; + +static struct clk ck_axi = { + .ops = &clk_stm32_axi_ops, + .priv = &(struct clk_stm32_composite_cfg) { + .mux_id = MUX_AXI, + .div_id = DIV_AXI, + }, + .name = "ck_axi", + .flags = 0, + .num_parents = 3, + .parents = { &ck_hsi, &ck_hse, &ck_pll2p }, +}; + +static struct clk ck_mlahb = { + .ops = &clk_stm32_mlahb_ops, + .priv = &(struct clk_stm32_composite_cfg) { + .mux_id = MUX_MLAHB, + .div_id = DIV_MLAHB, + }, + .name = "ck_mlahb", + .flags = 0, + .num_parents = 4, + .parents = { &ck_hsi, &ck_hse, &ck_csi, &ck_pll3p }, +}; + +static STM32_MUX(ck_per, 4, PARENT(&ck_hsi, &ck_csi, &ck_hse, &ck_off), + 0, MUX_CKPER); + +/* Bus clocks */ +static STM32_DIVIDER(ck_pclk1, &ck_mlahb, 0, DIV_APB1); +static STM32_DIVIDER(ck_pclk2, &ck_mlahb, 0, DIV_APB2); +static STM32_DIVIDER(ck_pclk3, &ck_mlahb, 0, DIV_APB3); +static STM32_DIVIDER(ck_pclk4, &ck_axi, 0, DIV_APB4); +static STM32_DIVIDER(ck_pclk5, &ck_axi, 0, DIV_APB5); +static STM32_DIVIDER(ck_pclk6, &ck_mlahb, 0, DIV_APB6); + +/* Timer Clocks */ +static STM32_TIMER(ck_timg1, &ck_pclk1, 0, RCC_APB1DIVR, RCC_TIMG1PRER); +static STM32_TIMER(ck_timg2, &ck_pclk2, 0, RCC_APB2DIVR, RCC_TIMG2PRER); +static STM32_TIMER(ck_timg3, &ck_pclk6, 0, RCC_APB6DIVR, RCC_TIMG3PRER); + +/* Peripheral and Kernel Clocks */ +static STM32_GATE(ck_ddrc1, &ck_axi, 0, GATE_DDRC1); +static STM32_GATE(ck_ddrc1lp, &ck_axi, 0, GATE_DDRC1LP); +static STM32_GATE(ck_ddrphyc, &ck_pll2r, 0, GATE_DDRPHYC); +static STM32_GATE(ck_ddrphyclp, &ck_pll2r, 0, GATE_DDRPHYCLP); +static STM32_GATE(ck_ddrcapb, &ck_pclk4, 0, GATE_DDRCAPB); +static STM32_GATE(ck_ddrcapblp, &ck_pclk4, 0, GATE_DDRCAPBLP); +static STM32_GATE(ck_axidcg, &ck_axi, 0, GATE_AXIDCG); +static STM32_GATE(ck_ddrphycapb, &ck_pclk4, 0, 0); +static STM32_GATE(ck_ddrphycapblp, &ck_pclk4, 0, GATE_DDRPHYCAPBLP); +static STM32_GATE(ck_syscfg, &ck_pclk3, 0, GATE_SYSCFG); +static STM32_GATE(ck_ddrperfm, &ck_pclk4, 0, GATE_DDRPERFM); +static STM32_GATE(ck_iwdg2, &ck_pclk4, 0, GATE_IWDG2APB); +static STM32_GATE(ck_rtcapb, &ck_pclk5, 0, GATE_RTCAPB); +static STM32_GATE(ck_tzc, &ck_pclk5, 0, GATE_TZC); +static STM32_GATE(ck_etzpcb, &ck_pclk5, 0, GATE_ETZPC); +static STM32_GATE(ck_iwdg1apb, &ck_pclk5, 0, GATE_IWDG1APB); +static STM32_GATE(ck_bsec, &ck_pclk5, 0, GATE_BSEC); +static STM32_GATE(ck_tim12_k, &ck_timg3, 0, GATE_TIM12); +static STM32_GATE(ck_tim15_k, &ck_timg3, 0, GATE_TIM15); +static STM32_GATE(ck_gpioa, &ck_mlahb, 0, GATE_GPIOA); +static STM32_GATE(ck_gpiob, &ck_mlahb, 0, GATE_GPIOB); +static STM32_GATE(ck_gpioc, &ck_mlahb, 0, GATE_GPIOC); +static STM32_GATE(ck_gpiod, &ck_mlahb, 0, GATE_GPIOD); +static STM32_GATE(ck_gpioe, &ck_mlahb, 0, GATE_GPIOE); +static STM32_GATE(ck_gpiof, &ck_mlahb, 0, GATE_GPIOF); +static STM32_GATE(ck_gpiog, &ck_mlahb, 0, GATE_GPIOG); +static STM32_GATE(ck_gpioh, &ck_mlahb, 0, GATE_GPIOH); +static STM32_GATE(ck_gpioi, &ck_mlahb, 0, GATE_GPIOI); +static STM32_GATE(ck_pka, &ck_axi, 0, GATE_PKA); +static STM32_GATE(ck_cryp1, &ck_pclk5, 0, GATE_CRYP1); +static STM32_GATE(ck_hash1, &ck_pclk5, 0, GATE_HASH1); +static STM32_GATE(ck_bkpsram, &ck_pclk5, 0, GATE_BKPSRAM); +static STM32_GATE(ck_dbg, &ck_axi, 0, GATE_DBGCK); +static STM32_GATE(ck_mce, &ck_axi, 0, GATE_MCE); +static STM32_GATE(ck_tim2_k, &ck_timg1, 0, GATE_TIM2); +static STM32_GATE(ck_tim3_k, &ck_timg1, 0, GATE_TIM3); +static STM32_GATE(ck_tim4_k, &ck_timg1, 0, GATE_TIM4); +static STM32_GATE(ck_tim5_k, &ck_timg1, 0, GATE_TIM5); +static STM32_GATE(ck_tim6_k, &ck_timg1, 0, GATE_TIM6); +static STM32_GATE(ck_tim7_k, &ck_timg1, 0, GATE_TIM7); +static STM32_GATE(ck_tim13_k, &ck_timg3, 0, GATE_TIM13); +static STM32_GATE(ck_tim14_k, &ck_timg3, 0, GATE_TIM14); +static STM32_GATE(ck_tim1_k, &ck_timg2, 0, GATE_TIM1); +static STM32_GATE(ck_tim8_k, &ck_timg2, 0, GATE_TIM8); +static STM32_GATE(ck_tim16_k, &ck_timg3, 0, GATE_TIM16); +static STM32_GATE(ck_tim17_k, &ck_timg3, 0, GATE_TIM17); +static STM32_GATE(ck_ltdc_px, &ck_pll4q, 0, GATE_LTDC); +static STM32_GATE(ck_dma1, &ck_mlahb, 0, GATE_DMA1); +static STM32_GATE(ck_dma2, &ck_mlahb, 0, GATE_DMA2); +static STM32_GATE(ck_mdma, &ck_axi, 0, GATE_MDMA); +static STM32_GATE(ck_eth1mac, &ck_axi, 0, GATE_ETH1MAC); +static STM32_GATE(ck_usbh, &ck_axi, 0, GATE_USBH); +static STM32_GATE(ck_vref, &ck_pclk3, 0, GATE_VREF); +static STM32_GATE(ck_tmpsens, &ck_pclk3, 0, GATE_DTS); +static STM32_GATE(ck_pmbctrl, &ck_pclk3, 0, GATE_HDP); +static STM32_GATE(ck_hdp, &ck_pclk3, 0, GATE_PMBCTRL); +static STM32_GATE(ck_stgenro, &ck_pclk4, 0, GATE_DCMIPP); +static STM32_GATE(ck_dmamux1, &ck_axi, 0, GATE_DMAMUX1); +static STM32_GATE(ck_dmamux2, &ck_axi, 0, GATE_DMAMUX2); +static STM32_GATE(ck_dma3, &ck_axi, 0, GATE_DMAMUX2); +static STM32_GATE(ck_tsc, &ck_axi, 0, GATE_TSC); +static STM32_GATE(ck_aximc, &ck_axi, 0, GATE_AXIMC); +static STM32_GATE(ck_crc1, &ck_axi, 0, GATE_ETH1TX); +static STM32_GATE(ck_eth1tx, &ck_axi, 0, GATE_ETH1TX); +static STM32_GATE(ck_eth1rx, &ck_axi, 0, GATE_ETH1RX); +static STM32_GATE(ck_eth2tx, &ck_axi, 0, GATE_ETH2TX); +static STM32_GATE(ck_eth2rx, &ck_axi, 0, GATE_ETH2RX); +static STM32_GATE(ck_eth2mac, &ck_axi, 0, GATE_ETH2MAC); + +/* Kernel Clocks */ +static STM32_KCLK(ck_usbphy_k, 3, + PARENT(&ck_hse, &ck_pll4r, &ck_hse_div2), + 0, GATE_USBPHY, MUX_USBPHY); + +static STM32_KCLK(ck_usbo_k, 2, + PARENT(&ck_pll4r, &ck_usb_phy_48Mhz), 0, + GATE_USBO, MUX_USBO); + +static STM32_KCLK(ck_stgen_k, 2, + PARENT(&ck_hsi, &ck_hse), 0, GATE_STGENC, MUX_STGEN); + +static STM32_KCLK(ck_usart1_k, 6, + PARENT(&ck_pclk6, &ck_pll3q, &ck_hsi, + &ck_csi, &ck_pll4q, &ck_hse), + 0, GATE_USART1, MUX_UART1); + +static STM32_KCLK(ck_usart2_k, 6, + PARENT(&ck_pclk6, &ck_pll3q, &ck_hsi, &ck_csi, &ck_pll4q, + &ck_hse), + 0, GATE_USART2, MUX_UART2); + +static STM32_KCLK(ck_i2c4_k, 4, + PARENT(&ck_pclk6, &ck_pll4r, &ck_hsi, &ck_csi), + 0, GATE_I2C4, MUX_I2C4); + +static STM32_KCLK(ck_rtc, 4, + PARENT(&ck_off, &ck_lse, &ck_lsi, &ck_hse), + 0, GATE_RTCCK, MUX_RTC); + +static STM32_KCLK(ck_saes_k, 4, + PARENT(&ck_axi, &ck_per, &ck_pll4r, &ck_lsi), + 0, GATE_SAES, MUX_SAES); + +static STM32_KCLK(ck_rng1_k, 4, + PARENT(&ck_csi, &ck_pll4r, &ck_lse, &ck_lsi), + 0, GATE_RNG1, MUX_RNG1); + +static STM32_KCLK(ck_sdmmc1_k, 4, + PARENT(&ck_axi, &ck_pll3r, &ck_pll4p, &ck_hsi), + 0, GATE_SDMMC1, MUX_SDMMC1); + +static STM32_KCLK(ck_sdmmc2_k, 4, + PARENT(&ck_axi, &ck_pll3r, &ck_pll4p, &ck_hsi), + 0, GATE_SDMMC2, MUX_SDMMC2); + +static STM32_KCLK(ck_usart3_k, 5, + PARENT(&ck_pclk1, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse), + 0, GATE_USART3, MUX_UART35); + +static STM32_KCLK(ck_uart4_k, 5, + PARENT(&ck_pclk1, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse), + 0, GATE_UART4, MUX_UART4); + +static STM32_KCLK(ck_uart5_k, 5, + PARENT(&ck_pclk1, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse), + 0, GATE_UART5, MUX_UART35); + +static STM32_KCLK(ck_uart7_k, 5, + PARENT(&ck_pclk1, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse), + 0, GATE_UART7, MUX_UART78); + +static STM32_KCLK(ck_uart8_k, 5, + PARENT(&ck_pclk1, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse), + 0, GATE_UART8, MUX_UART78); + +static STM32_KCLK(ck_usart6_k, 5, + PARENT(&ck_pclk2, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse), + 0, GATE_USART6, MUX_UART6); + +static STM32_KCLK(ck_fmc_k, 4, + PARENT(&ck_axi, &ck_pll3r, &ck_pll4p, &ck_per), + 0, GATE_FMC, MUX_FMC); + +static STM32_KCLK(ck_qspi_k, 4, + PARENT(&ck_axi, &ck_pll3r, &ck_pll4p, &ck_per), + 0, GATE_QSPI, MUX_QSPI); + +static STM32_KCLK(ck_lptim1_k, 6, + PARENT(&ck_pclk1, &ck_pll4p, &ck_pll3q, &ck_lse, &ck_lsi, + &ck_per), + 0, GATE_LPTIM1, MUX_LPTIM1); + +static STM32_KCLK(ck_spi2_k, 5, + PARENT(&ck_pll4p, &ck_pll3q, &ck_i2sckin, &ck_per, &ck_pll3r), + 0, GATE_SPI2, MUX_SPI23); + +static STM32_KCLK(ck_spi3_k, 5, + PARENT(&ck_pll4p, &ck_pll3q, &ck_i2sckin, &ck_per, &ck_pll3r), + 0, GATE_SPI3, MUX_SPI23); + +static STM32_KCLK(ck_spdif_k, 3, + PARENT(&ck_pll4p, &ck_pll3q, &ck_hsi), + 0, GATE_SPDIF, MUX_SPDIF); + +static STM32_KCLK(ck_spi1_k, 5, + PARENT(&ck_pll4p, &ck_pll3q, &ck_i2sckin, &ck_per, &ck_pll3r), + 0, GATE_SPI1, MUX_SPI1); + +static STM32_KCLK(ck_spi4_k, 6, + PARENT(&ck_pclk6, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse, + &ck_i2sckin), + 0, GATE_SPI4, MUX_SPI4); + +static STM32_KCLK(ck_spi5_k, 5, + PARENT(&ck_pclk6, &ck_pll4q, &ck_hsi, &ck_csi, &ck_hse), + 0, GATE_SPI5, MUX_SPI5); + +static STM32_KCLK(ck_sai1_k, 5, + PARENT(&ck_pll4q, &ck_pll3q, &ck_i2sckin, &ck_per, &ck_pll3r), + 0, GATE_SAI1, MUX_SAI1); + +static STM32_KCLK(ck_sai2_k, 6, + PARENT(&ck_pll4q, &ck_pll3q, &ck_i2sckin, &ck_per, &ck_off, + &ck_pll3r), + 0, GATE_SAI2, MUX_SAI2); + +static STM32_KCLK(ck_dfsdm_k, 5, + PARENT(&ck_pll4q, &ck_pll3q, &ck_i2sckin, &ck_per, &ck_pll3r), + 0, GATE_DFSDM, MUX_SAI1); + +static STM32_KCLK(ck_fdcan_k, 4, + PARENT(&ck_hse, &ck_pll3q, &ck_pll4q, &ck_pll4r), + 0, GATE_FDCAN, MUX_FDCAN); + +static STM32_KCLK(ck_i2c1_k, 4, + PARENT(&ck_pclk1, &ck_pll4r, &ck_hsi, &ck_csi), + 0, GATE_I2C1, MUX_I2C12); + +static STM32_KCLK(ck_i2c2_k, 4, + PARENT(&ck_pclk1, &ck_pll4r, &ck_hsi, &ck_csi), + 0, GATE_I2C2, MUX_I2C12); + +static STM32_KCLK(ck_adfsdm_k, 5, + PARENT(&ck_pll4q, &ck_pll3q, &ck_i2sckin, &ck_per, &ck_pll3r), + 0, GATE_ADFSDM, MUX_SAI1); + +static STM32_KCLK(ck_lptim2_k, 5, + PARENT(&ck_pclk3, &ck_pll4q, &ck_per, &ck_lse, &ck_lsi), + 0, GATE_LPTIM2, MUX_LPTIM2); + +static STM32_KCLK(ck_lptim3_k, 5, + PARENT(&ck_pclk3, &ck_pll4q, &ck_per, &ck_lse, &ck_lsi), + 0, GATE_LPTIM3, MUX_LPTIM3); + +static STM32_KCLK(ck_lptim4_k, 6, + PARENT(&ck_pclk3, &ck_pll4p, &ck_pll3q, &ck_lse, &ck_lsi, + &ck_per), + 0, GATE_LPTIM4, MUX_LPTIM45); + +static STM32_KCLK(ck_lptim5_k, 6, + PARENT(&ck_pclk3, &ck_pll4p, &ck_pll3q, &ck_lse, &ck_lsi, + &ck_per), + 0, GATE_LPTIM5, MUX_LPTIM45); + +static STM32_KCLK(ck_i2c3_k, 4, + PARENT(&ck_pclk6, &ck_pll4r, &ck_hsi, &ck_csi), + 0, GATE_I2C3, MUX_I2C3); + +static STM32_KCLK(ck_i2c5_k, 4, + PARENT(&ck_pclk6, &ck_pll4r, &ck_hsi, &ck_csi), + 0, GATE_I2C5, MUX_I2C5); + +static STM32_KCLK(ck_dcmipp_k, 4, + PARENT(&ck_axi, &ck_pll2q, &ck_pll4p, &ck_per), + 0, GATE_DCMIPP, MUX_DCMIPP); + +static STM32_KCLK(ck_adc1_k, 3, PARENT(&ck_pll4r, &ck_per, &ck_pll3q), + 0, GATE_ADC1, MUX_ADC1); + +static STM32_KCLK(ck_adc2_k, 3, PARENT(&ck_pll4r, &ck_per, &ck_pll3q), + 0, GATE_ADC2, MUX_ADC2); + +static STM32_KCLK(ck_eth1ck_k, 2, PARENT(&ck_pll4p, &ck_pll3q), + 0, GATE_ETH1CK, MUX_ETH1); + +static STM32_KCLK(ck_eth2ck_k, 2, PARENT(&ck_pll4p, &ck_pll3q), + 0, GATE_ETH2CK, MUX_ETH2); + +static STM32_COMPOSITE(ck_mco1, 5, + PARENT(&ck_hsi, &ck_hse, &ck_csi, &ck_lsi, &ck_lse), + 0, GATE_MCO1, DIV_MCO1, MUX_MCO1); + +static STM32_COMPOSITE(ck_mco2, 6, + PARENT(&ck_mpu, &ck_axi, &ck_mlahb, + &ck_pll4p, &ck_hse, &ck_hsi), + 0, GATE_MCO2, DIV_MCO2, MUX_MCO2); + +static STM32_COMPOSITE(ck_trace, 1, PARENT(&ck_axi), + 0, GATE_TRACECK, DIV_TRACE, NO_MUX); + +enum { + USB_PHY_48 = STM32MP1_LAST_CLK, + PLL1P_DIV, + CK_OFF, + I2S_CKIN, + STM32MP13_ALL_CLK_NB +}; + +static struct clk *stm32mp13_clk_provided[STM32MP13_ALL_CLK_NB] = { + [CK_HSE] = &ck_hse, + [CK_CSI] = &ck_csi, + [CK_LSI] = &ck_lsi, + [CK_LSE] = &ck_lse, + [CK_HSI] = &ck_hsi, + [CK_HSE_DIV2] = &ck_hse_div2, + [PLL1] = &ck_pll1_vco, + [PLL2] = &ck_pll2_vco, + [PLL3] = &ck_pll3_vco, + [PLL4] = &ck_pll4_vco, + [PLL1_P] = &ck_pll1p, + [PLL2_P] = &ck_pll2p, + [PLL2_Q] = &ck_pll2q, + [PLL2_R] = &ck_pll2r, + [PLL3_P] = &ck_pll3p, + [PLL3_Q] = &ck_pll3q, + [PLL3_R] = &ck_pll3r, + [PLL4_P] = &ck_pll4p, + [PLL4_Q] = &ck_pll4q, + [PLL4_R] = &ck_pll4r, + [PLL1P_DIV] = &ck_pll1p_div, + [CK_MPU] = &ck_mpu, + [CK_AXI] = &ck_axi, + [CK_MLAHB] = &ck_mlahb, + [CK_PER] = &ck_per, + [PCLK1] = &ck_pclk1, + [PCLK2] = &ck_pclk2, + [PCLK3] = &ck_pclk3, + [PCLK4] = &ck_pclk4, + [PCLK5] = &ck_pclk5, + [PCLK6] = &ck_pclk6, + [CK_TIMG1] = &ck_timg1, + [CK_TIMG2] = &ck_timg2, + [CK_TIMG3] = &ck_timg3, + [DDRC1] = &ck_ddrc1, + [DDRC1LP] = &ck_ddrc1lp, + [DDRPHYC] = &ck_ddrphyc, + [DDRPHYCLP] = &ck_ddrphyclp, + [DDRCAPB] = &ck_ddrcapb, + [DDRCAPBLP] = &ck_ddrcapblp, + [AXIDCG] = &ck_axidcg, + [DDRPHYCAPB] = &ck_ddrphycapb, + [DDRPHYCAPBLP] = &ck_ddrphycapblp, + [SYSCFG] = &ck_syscfg, + [DDRPERFM] = &ck_ddrperfm, + [IWDG2] = &ck_iwdg2, + [USBPHY_K] = &ck_usbphy_k, + [USBO_K] = &ck_usbo_k, + [RTCAPB] = &ck_rtcapb, + [TZC] = &ck_tzc, + [TZPC] = &ck_etzpcb, + [IWDG1] = &ck_iwdg1apb, + [BSEC] = &ck_bsec, + [STGEN_K] = &ck_stgen_k, + [USART1_K] = &ck_usart1_k, + [USART2_K] = &ck_usart2_k, + [I2C4_K] = &ck_i2c4_k, + [TIM12_K] = &ck_tim12_k, + [TIM15_K] = &ck_tim15_k, + [RTC] = &ck_rtc, + [GPIOA] = &ck_gpioa, + [GPIOB] = &ck_gpiob, + [GPIOC] = &ck_gpioc, + [GPIOD] = &ck_gpiod, + [GPIOE] = &ck_gpioe, + [GPIOF] = &ck_gpiof, + [GPIOG] = &ck_gpiog, + [GPIOH] = &ck_gpioh, + [GPIOI] = &ck_gpioi, + [PKA] = &ck_pka, + [SAES_K] = &ck_saes_k, + [CRYP1] = &ck_cryp1, + [HASH1] = &ck_hash1, + [RNG1_K] = &ck_rng1_k, + [BKPSRAM] = &ck_bkpsram, + [SDMMC1_K] = &ck_sdmmc1_k, + [SDMMC2_K] = &ck_sdmmc2_k, + [CK_DBG] = &ck_dbg, + [MCE] = &ck_mce, + [TIM2_K] = &ck_tim2_k, + [TIM3_K] = &ck_tim3_k, + [TIM4_K] = &ck_tim4_k, + [TIM5_K] = &ck_tim5_k, + [TIM6_K] = &ck_tim6_k, + [TIM7_K] = &ck_tim7_k, + [TIM13_K] = &ck_tim13_k, + [TIM14_K] = &ck_tim14_k, + [TIM1_K] = &ck_tim1_k, + [TIM8_K] = &ck_tim8_k, + [TIM16_K] = &ck_tim16_k, + [TIM17_K] = &ck_tim17_k, + [LTDC_PX] = &ck_ltdc_px, + [DMA1] = &ck_dma1, + [DMA2] = &ck_dma2, + [MDMA] = &ck_mdma, + [ETH1MAC] = &ck_eth1mac, + [USBH] = &ck_usbh, + [VREF] = &ck_vref, + [TMPSENS] = &ck_tmpsens, + [PMBCTRL] = &ck_pmbctrl, + [HDP] = &ck_hdp, + [STGENRO] = &ck_stgenro, + [DMAMUX1] = &ck_dmamux1, + [DMAMUX2] = &ck_dmamux2, + [DMA3] = &ck_dma3, + [TSC] = &ck_tsc, + [AXIMC] = &ck_aximc, + [CRC1] = &ck_crc1, + [ETH1TX] = &ck_eth1tx, + [ETH1RX] = &ck_eth1rx, + [ETH2TX] = &ck_eth2tx, + [ETH2RX] = &ck_eth2rx, + [ETH2MAC] = &ck_eth2mac, + [USART3_K] = &ck_usart3_k, + [UART4_K] = &ck_uart4_k, + [UART5_K] = &ck_uart5_k, + [UART7_K] = &ck_uart7_k, + [UART8_K] = &ck_uart8_k, + [USART6_K] = &ck_usart6_k, + [FMC_K] = &ck_fmc_k, + [QSPI_K] = &ck_qspi_k, + [LPTIM1_K] = &ck_lptim1_k, + [SPI2_K] = &ck_spi2_k, + [SPI3_K] = &ck_spi3_k, + [SPDIF_K] = &ck_spdif_k, + [SPI1_K] = &ck_spi1_k, + [SPI4_K] = &ck_spi4_k, + [SPI5_K] = &ck_spi5_k, + [SAI1_K] = &ck_sai1_k, + [SAI2_K] = &ck_sai2_k, + [DFSDM_K] = &ck_dfsdm_k, + [FDCAN_K] = &ck_fdcan_k, + [I2C1_K] = &ck_i2c1_k, + [I2C2_K] = &ck_i2c2_k, + [ADFSDM_K] = &ck_adfsdm_k, + [LPTIM2_K] = &ck_lptim2_k, + [LPTIM3_K] = &ck_lptim3_k, + [LPTIM4_K] = &ck_lptim4_k, + [LPTIM5_K] = &ck_lptim5_k, + [I2C3_K] = &ck_i2c3_k, + [I2C5_K] = &ck_i2c5_k, + [DCMIPP_K] = &ck_dcmipp_k, + [ADC1_K] = &ck_adc1_k, + [ADC2_K] = &ck_adc2_k, + [ETH1CK_K] = &ck_eth1ck_k, + [ETH2CK_K] = &ck_eth2ck_k, + [CK_MCO1] = &ck_mco1, + [CK_MCO2] = &ck_mco2, + [CK_TRACE] = &ck_trace, + [CK_OFF] = &ck_off, + [USB_PHY_48] = &ck_usb_phy_48Mhz, + [I2S_CKIN] = &ck_i2sckin, +}; + +static bool clk_stm32_clock_is_critical(struct clk *clk __maybe_unused) +{ + struct clk *clk_criticals[] = { + &ck_hsi, + &ck_hse, + &ck_csi, + &ck_lsi, + &ck_lse, + &ck_pll2r, + &ck_mpu, + &ck_ddrc1, + &ck_ddrc1lp, + &ck_ddrphyc, + &ck_ddrphyclp, + &ck_ddrcapb, + &ck_ddrcapblp, + &ck_axidcg, + &ck_ddrphycapb, + &ck_ddrphycapblp, + &ck_rtcapb, + &ck_tzc, + &ck_etzpcb, + &ck_iwdg1apb, + &ck_bsec, + &ck_stgen_k, + &ck_bkpsram, + &ck_mce, + &ck_mco1, + &ck_rng1_k, + &ck_mlahb, + }; + size_t i = 0; + + for (i = 0; i < ARRAY_SIZE(clk_criticals); i++) { + struct clk *clk_critical = clk_criticals[i]; + + if (clk == clk_critical) + return true; + } + + return false; +} + +static void clk_stm32_init_oscillators(const void *fdt, int node) +{ + size_t i = 0; + const char *name[6] = { "clk-hse", "clk-hsi", "clk-lse", + "clk-lsi", "clk-csi", "clk-i2sin" }; + struct clk *clks[6] = { &ck_hse, &ck_hsi, &ck_lse, + &ck_lsi, &ck_csi, &ck_i2sckin }; + + for (i = 0; i < ARRAY_SIZE(clks); i++) { + struct clk *clk = NULL; + + clk_dt_get_by_name(fdt, node, name[i], &clk); + + clks[i]->parents[0] = clk; + } +} + +static struct stm32_pll_dt_cfg mp13_pll[PLL_NB]; +static struct stm32_clk_opp_dt_cfg mp13_clk_opp; +static struct stm32_osci_dt_cfg mp13_osci[NB_OSCILLATOR]; +static uint32_t mp13_clksrc[MUX_NB]; +static uint32_t mp13_clkdiv[DIV_NB]; + +static struct stm32_clk_platdata stm32mp13_clock_pdata = { + .osci = mp13_osci, + .nosci = NB_OSCILLATOR, + .pll = mp13_pll, + .opp = &mp13_clk_opp, + .npll = PLL_NB, + .clksrc = mp13_clksrc, + .nclksrc = MUX_NB, + .clkdiv = mp13_clkdiv, + .nclkdiv = DIV_NB, +}; + +static struct clk_stm32_priv stm32mp13_clock_data = { + .muxes = parent_mp13, + .nb_muxes = ARRAY_SIZE(parent_mp13), + .gates = gates_mp13, + .nb_gates = ARRAY_SIZE(gates_mp13), + .div = dividers_mp13, + .nb_div = ARRAY_SIZE(dividers_mp13), + .pdata = &stm32mp13_clock_pdata, + .nb_clk_refs = STM32MP13_ALL_CLK_NB, + .clk_refs = stm32mp13_clk_provided, + .is_critical = clk_stm32_clock_is_critical, +}; + +static TEE_Result stm32mp13_clk_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int fdt_rc = 0; + int rc = 0; + struct clk_stm32_priv *priv = &stm32mp13_clock_data; + struct stm32_clk_platdata *pdata = &stm32mp13_clock_pdata; + + fdt_rc = stm32_clk_parse_fdt(fdt, node, pdata); + if (fdt_rc) { + EMSG("Failed to parse clock node: %d", fdt_rc); + return TEE_ERROR_GENERIC; + } + + res = clk_stm32_init(priv, stm32_rcc_base()); + if (res) + return res; + + rc = stm32mp1_init_clock_tree(priv, pdata); + if (rc) + return TEE_ERROR_GENERIC; + + clk_stm32_init_oscillators(fdt, node); + + stm32mp_clk_provider_probe_final(fdt, node, priv); + + return TEE_SUCCESS; +} + +CLK_DT_DECLARE(stm32mp13_clk, "st,stm32mp13-rcc", stm32mp13_clk_probe); diff --git a/optee_os/core/drivers/clk/clk-stm32mp15.c b/optee_os/core/drivers/clk/clk-stm32mp15.c new file mode 100644 index 0000000..a5b7270 --- /dev/null +++ b/optee_os/core/drivers/clk/clk-stm32mp15.c @@ -0,0 +1,1569 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0+) +/* + * Copyright (C) 2018-2022, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Identifiers for root oscillators */ +enum stm32mp_osc_id { + OSC_HSI, + OSC_HSE, + OSC_CSI, + OSC_LSI, + OSC_LSE, + OSC_I2S_CKIN, + OSC_USB_PHY_48, + NB_OSC, + _UNKNOWN_OSC_ID = 0xffU +}; + +/* Identifiers for parent clocks */ +enum stm32mp1_parent_id { + _HSI, + _HSE, + _CSI, + _LSI, + _LSE, + _I2S_CKIN, + _USB_PHY_48, + _HSI_KER, + _HSE_KER, + _HSE_KER_DIV2, + _HSE_RTC, + _CSI_KER, + _PLL1_P, + _PLL1_Q, + _PLL1_R, + _PLL2_P, + _PLL2_Q, + _PLL2_R, + _PLL3_P, + _PLL3_Q, + _PLL3_R, + _PLL4_P, + _PLL4_Q, + _PLL4_R, + _ACLK, + _PCLK1, + _PCLK2, + _PCLK3, + _PCLK4, + _PCLK5, + _HCLK5, + _HCLK6, + _HCLK2, + _CK_PER, + _CK_MPU, + _CK_MCU, + _PARENT_NB, + _UNKNOWN_ID = 0xff, +}; + +/* + * Identifiers for parent clock selectors. + * This enum lists only the parent clocks we are interested in. + */ +enum stm32mp1_parent_sel { + _STGEN_SEL, + _I2C35_SEL, + _I2C46_SEL, + _SPI6_SEL, + _USART1_SEL, + _RNG1_SEL, + _UART6_SEL, + _UART24_SEL, + _UART35_SEL, + _UART78_SEL, + _AXISS_SEL, + _MCUSS_SEL, + _USBPHY_SEL, + _USBO_SEL, + _RTC_SEL, + _MPU_SEL, + _PARENT_SEL_NB, + _UNKNOWN_SEL = 0xff, +}; + +static const uint8_t parent_id_clock_id[_PARENT_NB] = { + [_HSE] = CK_HSE, + [_HSI] = CK_HSI, + [_CSI] = CK_CSI, + [_LSE] = CK_LSE, + [_LSI] = CK_LSI, + [_I2S_CKIN] = _UNKNOWN_ID, + [_USB_PHY_48] = _UNKNOWN_ID, + [_HSI_KER] = CK_HSI, + [_HSE_KER] = CK_HSE, + [_HSE_KER_DIV2] = CK_HSE_DIV2, + [_HSE_RTC] = _UNKNOWN_ID, + [_CSI_KER] = CK_CSI, + [_PLL1_P] = PLL1_P, + [_PLL1_Q] = PLL1_Q, + [_PLL1_R] = PLL1_R, + [_PLL2_P] = PLL2_P, + [_PLL2_Q] = PLL2_Q, + [_PLL2_R] = PLL2_R, + [_PLL3_P] = PLL3_P, + [_PLL3_Q] = PLL3_Q, + [_PLL3_R] = PLL3_R, + [_PLL4_P] = PLL4_P, + [_PLL4_Q] = PLL4_Q, + [_PLL4_R] = PLL4_R, + [_ACLK] = CK_AXI, + [_PCLK1] = CK_AXI, + [_PCLK2] = CK_AXI, + [_PCLK3] = CK_AXI, + [_PCLK4] = CK_AXI, + [_PCLK5] = CK_AXI, + [_HCLK5] = CK_AXI, + [_HCLK6] = CK_AXI, + [_HCLK2] = CK_AXI, + [_CK_PER] = CK_PER, + [_CK_MPU] = CK_MPU, + [_CK_MCU] = CK_MCU, +}; + +static enum stm32mp1_parent_id osc_id2parent_id(enum stm32mp_osc_id osc_id) +{ + assert(osc_id >= OSC_HSI && osc_id < NB_OSC); + COMPILE_TIME_ASSERT((int)OSC_HSI == (int)_HSI && + (int)OSC_HSE == (int)_HSE && + (int)OSC_CSI == (int)_CSI && + (int)OSC_LSI == (int)_LSI && + (int)OSC_LSE == (int)_LSE && + (int)OSC_I2S_CKIN == (int)_I2S_CKIN && + (int)OSC_USB_PHY_48 == (int)_USB_PHY_48); + + return (enum stm32mp1_parent_id)osc_id; +} + +static enum stm32mp1_parent_id clock_id2parent_id(unsigned long id) +{ + size_t n = 0; + + COMPILE_TIME_ASSERT(STM32MP1_LAST_CLK < _UNKNOWN_ID); + + for (n = 0; n < ARRAY_SIZE(parent_id_clock_id); n++) + if (parent_id_clock_id[n] == id) + return (enum stm32mp1_parent_id)n; + + return _UNKNOWN_ID; +} + +/* Identifiers for PLLs and their configuration resources */ +enum stm32mp1_pll_id { + _PLL1, + _PLL2, + _PLL3, + _PLL4, + _PLL_NB +}; + +enum stm32mp1_div_id { + _DIV_P, + _DIV_Q, + _DIV_R, + _DIV_NB, +}; + +enum stm32mp1_plltype { + PLL_800, + PLL_1600, + PLL_TYPE_NB +}; + +/* + * Clock generic gates clocks which state is controlled by a single RCC bit + * + * @offset: RCC register byte offset from RCC base where clock is controlled + * @bit: Bit position in the RCC 32bit register + * @clock_id: Identifier used for the clock in the clock driver API + * @set_clr: Non-null if and only-if RCC register is a CLEAR/SET register + * (CLEAR register is at offset RCC_MP_ENCLRR_OFFSET from SET register) + * @secure: One of N_S or SEC, defined below + * @sel: _UNKNOWN_ID (fixed parent) or reference to parent clock selector + * (8bit storage of ID from enum stm32mp1_parent_sel) + * @fixed: _UNKNOWN_ID (selectable paranet) or reference to parent clock + * (8bit storage of ID from enum stm32mp1_parent_id) + */ +struct stm32mp1_clk_gate { + uint16_t offset; + uint8_t bit; + uint8_t clock_id; + uint8_t set_clr; + uint8_t secure; + uint8_t sel; /* Relates to enum stm32mp1_parent_sel */ + uint8_t fixed; /* Relates to enum stm32mp1_parent_id */ +}; + +/* Parent clock selection: select register info, parent clocks references */ +struct stm32mp1_clk_sel { + uint16_t offset; + uint8_t src; + uint8_t msk; + uint8_t nb_parent; + const uint8_t *parent; +}; + +#define REFCLK_SIZE 4 +/* PLL control: type, control register offsets, up-to-4 selectable parent */ +struct stm32mp1_clk_pll { + enum stm32mp1_plltype plltype; + uint16_t rckxselr; + uint16_t pllxcfgr1; + uint16_t pllxcfgr2; + uint16_t pllxfracr; + uint16_t pllxcr; + uint16_t pllxcsgr; + enum stm32mp_osc_id refclk[REFCLK_SIZE]; +}; + +#define N_S 0 /* Non-secure can access RCC interface */ +#define SEC 1 /* RCC[TZEN] protects RCC interface */ + +/* Clocks with selectable source and not set/clr register access */ +#define _CLK_SELEC(_sec, _offset, _bit, _clock_id, _parent_sel) \ + { \ + .offset = (_offset), \ + .bit = (_bit), \ + .clock_id = (_clock_id), \ + .set_clr = 0, \ + .secure = (_sec), \ + .sel = (_parent_sel), \ + .fixed = _UNKNOWN_ID, \ + } + +/* Clocks with fixed source and not set/clr register access */ +#define _CLK_FIXED(_sec, _offset, _bit, _clock_id, _parent) \ + { \ + .offset = (_offset), \ + .bit = (_bit), \ + .clock_id = (_clock_id), \ + .set_clr = 0, \ + .secure = (_sec), \ + .sel = _UNKNOWN_SEL, \ + .fixed = (_parent), \ + } + +/* Clocks with selectable source and set/clr register access */ +#define _CLK_SC_SELEC(_sec, _offset, _bit, _clock_id, _parent_sel) \ + { \ + .offset = (_offset), \ + .bit = (_bit), \ + .clock_id = (_clock_id), \ + .set_clr = 1, \ + .secure = (_sec), \ + .sel = (_parent_sel), \ + .fixed = _UNKNOWN_ID, \ + } + +/* Clocks with fixed source and set/clr register access */ +#define _CLK_SC_FIXED(_sec, _offset, _bit, _clock_id, _parent) \ + { \ + .offset = (_offset), \ + .bit = (_bit), \ + .clock_id = (_clock_id), \ + .set_clr = 1, \ + .secure = (_sec), \ + .sel = _UNKNOWN_SEL, \ + .fixed = (_parent), \ + } + +/* + * Clocks with selectable source and set/clr register access + * and enable bit position defined by a label (argument _bit) + */ +#define _CLK_SC2_SELEC(_sec, _offset, _bit, _clock_id, _parent_sel) \ + { \ + .offset = (_offset), \ + .clock_id = (_clock_id), \ + .bit = _offset ## _ ## _bit ## _POS, \ + .set_clr = 1, \ + .secure = (_sec), \ + .sel = (_parent_sel), \ + .fixed = _UNKNOWN_ID, \ + } +#define _CLK_SC2_FIXED(_sec, _offset, _bit, _clock_id, _parent) \ + { \ + .offset = (_offset), \ + .clock_id = (_clock_id), \ + .bit = _offset ## _ ## _bit ## _POS, \ + .set_clr = 1, \ + .secure = (_sec), \ + .sel = _UNKNOWN_SEL, \ + .fixed = (_parent), \ + } + +#define _CLK_PARENT(idx, _offset, _src, _mask, _parent) \ + [(idx)] = { \ + .offset = (_offset), \ + .src = (_src), \ + .msk = (_mask), \ + .parent = (_parent), \ + .nb_parent = ARRAY_SIZE(_parent) \ + } + +#define _CLK_PLL(_idx, _type, _off1, _off2, _off3, _off4, \ + _off5, _off6, _p1, _p2, _p3, _p4) \ + [(_idx)] = { \ + .plltype = (_type), \ + .rckxselr = (_off1), \ + .pllxcfgr1 = (_off2), \ + .pllxcfgr2 = (_off3), \ + .pllxfracr = (_off4), \ + .pllxcr = (_off5), \ + .pllxcsgr = (_off6), \ + .refclk[0] = (_p1), \ + .refclk[1] = (_p2), \ + .refclk[2] = (_p3), \ + .refclk[3] = (_p4), \ + } + +#define NB_GATES ARRAY_SIZE(stm32mp1_clk_gate) + +static const struct stm32mp1_clk_gate stm32mp1_clk_gate[] = { + _CLK_FIXED(SEC, RCC_DDRITFCR, 0, DDRC1, _ACLK), + _CLK_FIXED(SEC, RCC_DDRITFCR, 1, DDRC1LP, _ACLK), + _CLK_FIXED(SEC, RCC_DDRITFCR, 2, DDRC2, _ACLK), + _CLK_FIXED(SEC, RCC_DDRITFCR, 3, DDRC2LP, _ACLK), + _CLK_FIXED(SEC, RCC_DDRITFCR, 4, DDRPHYC, _PLL2_R), + _CLK_FIXED(SEC, RCC_DDRITFCR, 5, DDRPHYCLP, _PLL2_R), + _CLK_FIXED(SEC, RCC_DDRITFCR, 6, DDRCAPB, _PCLK4), + _CLK_FIXED(SEC, RCC_DDRITFCR, 7, DDRCAPBLP, _PCLK4), + _CLK_FIXED(SEC, RCC_DDRITFCR, 8, AXIDCG, _ACLK), + _CLK_FIXED(SEC, RCC_DDRITFCR, 9, DDRPHYCAPB, _PCLK4), + _CLK_FIXED(SEC, RCC_DDRITFCR, 10, DDRPHYCAPBLP, _PCLK4), + + _CLK_SC2_SELEC(SEC, RCC_MP_APB5ENSETR, SPI6EN, SPI6_K, _SPI6_SEL), + _CLK_SC2_SELEC(SEC, RCC_MP_APB5ENSETR, I2C4EN, I2C4_K, _I2C46_SEL), + _CLK_SC2_SELEC(SEC, RCC_MP_APB5ENSETR, I2C6EN, I2C6_K, _I2C46_SEL), + _CLK_SC2_SELEC(SEC, RCC_MP_APB5ENSETR, USART1EN, USART1_K, _USART1_SEL), + _CLK_SC2_FIXED(SEC, RCC_MP_APB5ENSETR, RTCAPBEN, RTCAPB, _PCLK5), + _CLK_SC2_FIXED(SEC, RCC_MP_APB5ENSETR, TZC1EN, TZC1, _PCLK5), + _CLK_SC2_FIXED(SEC, RCC_MP_APB5ENSETR, TZC2EN, TZC2, _PCLK5), + _CLK_SC2_FIXED(SEC, RCC_MP_APB5ENSETR, TZPCEN, TZPC, _PCLK5), + _CLK_SC2_FIXED(SEC, RCC_MP_APB5ENSETR, IWDG1APBEN, IWDG1, _PCLK5), + _CLK_SC2_FIXED(SEC, RCC_MP_APB5ENSETR, BSECEN, BSEC, _PCLK5), + _CLK_SC2_SELEC(SEC, RCC_MP_APB5ENSETR, STGENEN, STGEN_K, _STGEN_SEL), + + _CLK_SC2_FIXED(SEC, RCC_MP_AHB5ENSETR, GPIOZEN, GPIOZ, _HCLK5), + _CLK_SC2_FIXED(SEC, RCC_MP_AHB5ENSETR, CRYP1EN, CRYP1, _HCLK5), + _CLK_SC2_FIXED(SEC, RCC_MP_AHB5ENSETR, HASH1EN, HASH1, _HCLK5), + _CLK_SC2_SELEC(SEC, RCC_MP_AHB5ENSETR, RNG1EN, RNG1_K, _RNG1_SEL), + _CLK_SC2_FIXED(SEC, RCC_MP_AHB5ENSETR, BKPSRAMEN, BKPSRAM, _HCLK5), + + _CLK_SC2_FIXED(SEC, RCC_MP_TZAHB6ENSETR, MDMA, MDMA, _HCLK6), + + _CLK_SELEC(SEC, RCC_BDCR, RCC_BDCR_RTCCKEN_POS, RTC, _RTC_SEL), + + /* Non-secure clocks */ +#ifdef CFG_WITH_NSEC_I2CS + _CLK_SC2_SELEC(N_S, RCC_MP_APB1ENSETR, I2C5EN, I2C5_K, _I2C35_SEL), +#endif + +#ifdef CFG_WITH_NSEC_GPIOS + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 0, GPIOA, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 1, GPIOB, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 2, GPIOC, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 3, GPIOD, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 4, GPIOE, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 5, GPIOF, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 6, GPIOG, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 7, GPIOH, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 8, GPIOI, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 9, GPIOJ, _UNKNOWN_ID), + _CLK_SC_FIXED(N_S, RCC_MP_AHB4ENSETR, 10, GPIOK, _UNKNOWN_ID), +#endif + _CLK_SC_FIXED(N_S, RCC_MP_APB1ENSETR, 6, TIM12_K, _PCLK1), +#ifdef CFG_WITH_NSEC_UARTS + _CLK_SC_SELEC(N_S, RCC_MP_APB1ENSETR, 14, USART2_K, _UART24_SEL), + _CLK_SC_SELEC(N_S, RCC_MP_APB1ENSETR, 15, USART3_K, _UART35_SEL), + _CLK_SC_SELEC(N_S, RCC_MP_APB1ENSETR, 16, UART4_K, _UART24_SEL), + _CLK_SC_SELEC(N_S, RCC_MP_APB1ENSETR, 17, UART5_K, _UART35_SEL), + _CLK_SC_SELEC(N_S, RCC_MP_APB1ENSETR, 18, UART7_K, _UART78_SEL), + _CLK_SC_SELEC(N_S, RCC_MP_APB1ENSETR, 19, UART8_K, _UART78_SEL), +#endif + _CLK_SC_FIXED(N_S, RCC_MP_APB2ENSETR, 2, TIM15_K, _PCLK2), +#ifdef CFG_WITH_NSEC_UARTS + _CLK_SC_SELEC(N_S, RCC_MP_APB2ENSETR, 13, USART6_K, _UART6_SEL), +#endif + _CLK_SC_FIXED(N_S, RCC_MP_APB3ENSETR, 11, SYSCFG, _UNKNOWN_ID), + _CLK_SC_SELEC(N_S, RCC_MP_APB4ENSETR, 8, DDRPERFM, _UNKNOWN_SEL), + _CLK_SC_SELEC(N_S, RCC_MP_APB4ENSETR, 15, IWDG2, _UNKNOWN_SEL), + + _CLK_SELEC(N_S, RCC_DBGCFGR, 8, CK_DBG, _UNKNOWN_SEL), +}; +DECLARE_KEEP_PAGER(stm32mp1_clk_gate); + +const uint8_t stm32mp1_clk_on[] = { + CK_HSE, CK_CSI, CK_LSI, CK_LSE, CK_HSI, CK_HSE_DIV2, + PLL1_P, PLL1_Q, PLL1_R, PLL2_P, PLL2_Q, PLL2_R, PLL3_P, PLL3_Q, PLL3_R, + CK_AXI, CK_MPU, CK_MCU, +}; + +/* Parents for secure aware clocks in the xxxSELR value ordering */ +static const uint8_t stgen_parents[] = { + _HSI_KER, _HSE_KER +}; + +#ifdef CFG_WITH_NSEC_I2CS +static const uint8_t i2c35_parents[] = { + _PCLK1, _PLL4_R, _HSI_KER, _CSI_KER +}; +#endif + +static const uint8_t i2c46_parents[] = { + _PCLK5, _PLL3_Q, _HSI_KER, _CSI_KER +}; + +static const uint8_t spi6_parents[] = { + _PCLK5, _PLL4_Q, _HSI_KER, _CSI_KER, _HSE_KER, _PLL3_Q +}; + +static const uint8_t usart1_parents[] = { + _PCLK5, _PLL3_Q, _HSI_KER, _CSI_KER, _PLL4_Q, _HSE_KER +}; + +static const uint8_t rng1_parents[] = { + _CSI, _PLL4_R, _LSE, _LSI +}; + +static const uint8_t mpu_parents[] = { + _HSI, _HSE, _PLL1_P, _PLL1_P /* specific div */ +}; + +/* Parents for (some) non-secure clocks */ +#ifdef CFG_WITH_NSEC_UARTS +static const uint8_t uart6_parents[] = { + _PCLK2, _PLL4_Q, _HSI_KER, _CSI_KER, _HSE_KER +}; + +static const uint8_t uart234578_parents[] = { + _PCLK1, _PLL4_Q, _HSI_KER, _CSI_KER, _HSE_KER +}; +#endif + +static const uint8_t axiss_parents[] = { + _HSI, _HSE, _PLL2_P +}; + +static const uint8_t mcuss_parents[] = { + _HSI, _HSE, _CSI, _PLL3_P +}; + +static const uint8_t rtc_parents[] = { + _UNKNOWN_ID, _LSE, _LSI, _HSE_RTC +}; + +static const struct stm32mp1_clk_sel stm32mp1_clk_sel[_PARENT_SEL_NB] = { + /* Secure aware clocks */ + _CLK_PARENT(_STGEN_SEL, RCC_STGENCKSELR, 0, 0x3, stgen_parents), + _CLK_PARENT(_I2C46_SEL, RCC_I2C46CKSELR, 0, 0x7, i2c46_parents), + _CLK_PARENT(_SPI6_SEL, RCC_SPI6CKSELR, 0, 0x7, spi6_parents), + _CLK_PARENT(_USART1_SEL, RCC_UART1CKSELR, 0, 0x7, usart1_parents), + _CLK_PARENT(_RNG1_SEL, RCC_RNG1CKSELR, 0, 0x3, rng1_parents), + _CLK_PARENT(_RTC_SEL, RCC_BDCR, 16, 0x3, rtc_parents), + _CLK_PARENT(_MPU_SEL, RCC_MPCKSELR, 0, 0x3, mpu_parents), + /* Always non-secure clocks (maybe used in some way in secure world) */ +#ifdef CFG_WITH_NSEC_I2CS + _CLK_PARENT(_I2C35_SEL, RCC_I2C35CKSELR, 0, 0x7, i2c35_parents), +#endif +#ifdef CFG_WITH_NSEC_UARTS + _CLK_PARENT(_UART6_SEL, RCC_UART6CKSELR, 0, 0x7, uart6_parents), + _CLK_PARENT(_UART24_SEL, RCC_UART24CKSELR, 0, 0x7, uart234578_parents), + _CLK_PARENT(_UART35_SEL, RCC_UART35CKSELR, 0, 0x7, uart234578_parents), + _CLK_PARENT(_UART78_SEL, RCC_UART78CKSELR, 0, 0x7, uart234578_parents), +#endif + _CLK_PARENT(_AXISS_SEL, RCC_ASSCKSELR, 0, 0x3, axiss_parents), + _CLK_PARENT(_MCUSS_SEL, RCC_MSSCKSELR, 0, 0x3, mcuss_parents), +}; + +/* PLLNCFGR2 register divider by output */ +static const uint8_t pllncfgr2[_DIV_NB] = { + [_DIV_P] = RCC_PLLNCFGR2_DIVP_SHIFT, + [_DIV_Q] = RCC_PLLNCFGR2_DIVQ_SHIFT, + [_DIV_R] = RCC_PLLNCFGR2_DIVR_SHIFT, +}; + +static const struct stm32mp1_clk_pll stm32mp1_clk_pll[_PLL_NB] = { + _CLK_PLL(_PLL1, PLL_1600, + RCC_RCK12SELR, RCC_PLL1CFGR1, RCC_PLL1CFGR2, + RCC_PLL1FRACR, RCC_PLL1CR, RCC_PLL1CSGR, + OSC_HSI, OSC_HSE, _UNKNOWN_OSC_ID, _UNKNOWN_OSC_ID), + _CLK_PLL(_PLL2, PLL_1600, + RCC_RCK12SELR, RCC_PLL2CFGR1, RCC_PLL2CFGR2, + RCC_PLL2FRACR, RCC_PLL2CR, RCC_PLL2CSGR, + OSC_HSI, OSC_HSE, _UNKNOWN_OSC_ID, _UNKNOWN_OSC_ID), + _CLK_PLL(_PLL3, PLL_800, + RCC_RCK3SELR, RCC_PLL3CFGR1, RCC_PLL3CFGR2, + RCC_PLL3FRACR, RCC_PLL3CR, RCC_PLL3CSGR, + OSC_HSI, OSC_HSE, OSC_CSI, _UNKNOWN_OSC_ID), + _CLK_PLL(_PLL4, PLL_800, + RCC_RCK4SELR, RCC_PLL4CFGR1, RCC_PLL4CFGR2, + RCC_PLL4FRACR, RCC_PLL4CR, RCC_PLL4CSGR, + OSC_HSI, OSC_HSE, OSC_CSI, OSC_I2S_CKIN), +}; + +/* Prescaler table lookups for clock computation */ +/* div = /1 /2 /4 /8 / 16 /64 /128 /512 */ +static const uint8_t stm32mp1_mcu_div[16] = { + 0, 1, 2, 3, 4, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9, 9 +}; + +/* div = /1 /2 /4 /8 /16 : same divider for PMU and APBX */ +#define stm32mp1_mpu_div stm32mp1_mpu_apbx_div +#define stm32mp1_apbx_div stm32mp1_mpu_apbx_div +static const uint8_t stm32mp1_mpu_apbx_div[8] = { + 0, 1, 2, 3, 4, 4, 4, 4 +}; + +/* div = /1 /2 /3 /4 */ +static const uint8_t stm32mp1_axi_div[8] = { + 1, 2, 3, 4, 4, 4, 4, 4 +}; + +static const char __maybe_unused *const stm32mp1_clk_parent_name[_PARENT_NB] = { + [_HSI] = "HSI", + [_HSE] = "HSE", + [_CSI] = "CSI", + [_LSI] = "LSI", + [_LSE] = "LSE", + [_I2S_CKIN] = "I2S_CKIN", + [_HSI_KER] = "HSI_KER", + [_HSE_KER] = "HSE_KER", + [_HSE_KER_DIV2] = "HSE_KER_DIV2", + [_HSE_RTC] = "HSE_RTC", + [_CSI_KER] = "CSI_KER", + [_PLL1_P] = "PLL1_P", + [_PLL1_Q] = "PLL1_Q", + [_PLL1_R] = "PLL1_R", + [_PLL2_P] = "PLL2_P", + [_PLL2_Q] = "PLL2_Q", + [_PLL2_R] = "PLL2_R", + [_PLL3_P] = "PLL3_P", + [_PLL3_Q] = "PLL3_Q", + [_PLL3_R] = "PLL3_R", + [_PLL4_P] = "PLL4_P", + [_PLL4_Q] = "PLL4_Q", + [_PLL4_R] = "PLL4_R", + [_ACLK] = "ACLK", + [_PCLK1] = "PCLK1", + [_PCLK2] = "PCLK2", + [_PCLK3] = "PCLK3", + [_PCLK4] = "PCLK4", + [_PCLK5] = "PCLK5", + [_HCLK2] = "HCLK2", + [_HCLK5] = "HCLK5", + [_HCLK6] = "HCLK6", + [_CK_PER] = "CK_PER", + [_CK_MPU] = "CK_MPU", + [_CK_MCU] = "CK_MCU", + [_USB_PHY_48] = "USB_PHY_48", +}; + +/* + * Oscillator frequency in Hz. This array shall be initialized + * according to platform. + */ +static unsigned long stm32mp1_osc[NB_OSC]; + +static unsigned long osc_frequency(enum stm32mp_osc_id idx) +{ + if (idx >= ARRAY_SIZE(stm32mp1_osc)) { + DMSG("clk id %d not found", idx); + return 0; + } + + return stm32mp1_osc[idx]; +} + +static const struct stm32mp1_clk_gate *gate_ref(unsigned int idx) +{ + return &stm32mp1_clk_gate[idx]; +} + +static const struct stm32mp1_clk_sel *clk_sel_ref(unsigned int idx) +{ + return &stm32mp1_clk_sel[idx]; +} + +static const struct stm32mp1_clk_pll *pll_ref(unsigned int idx) +{ + return &stm32mp1_clk_pll[idx]; +} + +static int stm32mp1_clk_get_gated_id(unsigned long id) +{ + unsigned int i = 0; + + for (i = 0; i < NB_GATES; i++) + if (gate_ref(i)->clock_id == id) + return i; + + DMSG("clk id %lu not found", id); + return -1; +} + +static enum stm32mp1_parent_sel stm32mp1_clk_get_sel(int i) +{ + return (enum stm32mp1_parent_sel)gate_ref(i)->sel; +} + +static enum stm32mp1_parent_id stm32mp1_clk_get_fixed_parent(int i) +{ + return (enum stm32mp1_parent_id)gate_ref(i)->fixed; +} + +static int stm32mp1_clk_get_parent(unsigned long id) +{ + const struct stm32mp1_clk_sel *sel = NULL; + enum stm32mp1_parent_id parent_id = 0; + uint32_t p_sel = 0; + int i = 0; + enum stm32mp1_parent_id p = _UNKNOWN_ID; + enum stm32mp1_parent_sel s = _UNKNOWN_SEL; + vaddr_t rcc_base = stm32_rcc_base(); + + parent_id = clock_id2parent_id(id); + if (parent_id != _UNKNOWN_ID) + return (int)parent_id; + + i = stm32mp1_clk_get_gated_id(id); + if (i < 0) + panic(); + + p = stm32mp1_clk_get_fixed_parent(i); + if (p < _PARENT_NB) + return (int)p; + + s = stm32mp1_clk_get_sel(i); + if (s == _UNKNOWN_SEL) + return -1; + if (s >= _PARENT_SEL_NB) + panic(); + + sel = clk_sel_ref(s); + p_sel = (io_read32(rcc_base + sel->offset) >> sel->src) & sel->msk; + if (p_sel < sel->nb_parent) + return (int)sel->parent[p_sel]; + + DMSG("No parent selected for clk %lu", id); + return -1; +} + +static unsigned long stm32mp1_pll_get_fref(const struct stm32mp1_clk_pll *pll) +{ + uint32_t selr = io_read32(stm32_rcc_base() + pll->rckxselr); + uint32_t src = selr & RCC_SELR_REFCLK_SRC_MASK; + + return osc_frequency(pll->refclk[src]); +} + +/* + * pll_get_fvco() : return the VCO or (VCO / 2) frequency for the requested PLL + * - PLL1 & PLL2 => return VCO / 2 with Fpll_y_ck = FVCO / 2 * (DIVy + 1) + * - PLL3 & PLL4 => return VCO with Fpll_y_ck = FVCO / (DIVy + 1) + * => in all cases Fpll_y_ck = pll_get_fvco() / (DIVy + 1) + */ +static unsigned long stm32mp1_pll_get_fvco(const struct stm32mp1_clk_pll *pll) +{ + unsigned long refclk = 0; + unsigned long fvco = 0; + uint32_t cfgr1 = 0; + uint32_t fracr = 0; + uint32_t divm = 0; + uint32_t divn = 0; + + cfgr1 = io_read32(stm32_rcc_base() + pll->pllxcfgr1); + fracr = io_read32(stm32_rcc_base() + pll->pllxfracr); + + divm = (cfgr1 & RCC_PLLNCFGR1_DIVM_MASK) >> RCC_PLLNCFGR1_DIVM_SHIFT; + divn = cfgr1 & RCC_PLLNCFGR1_DIVN_MASK; + + refclk = stm32mp1_pll_get_fref(pll); + + /* + * With FRACV : + * Fvco = Fck_ref * ((DIVN + 1) + FRACV / 2^13) / (DIVM + 1) + * Without FRACV + * Fvco = Fck_ref * ((DIVN + 1) / (DIVM + 1) + */ + if (fracr & RCC_PLLNFRACR_FRACLE) { + unsigned long long numerator = 0; + unsigned long long denominator = 0; + uint32_t fracv = (fracr & RCC_PLLNFRACR_FRACV_MASK) >> + RCC_PLLNFRACR_FRACV_SHIFT; + + numerator = (((unsigned long long)divn + 1U) << 13) + fracv; + numerator = refclk * numerator; + denominator = ((unsigned long long)divm + 1U) << 13; + fvco = (unsigned long)(numerator / denominator); + } else { + fvco = (unsigned long)(refclk * (divn + 1U) / (divm + 1U)); + } + + return fvco; +} + +static unsigned long stm32mp1_read_pll_freq(enum stm32mp1_pll_id pll_id, + enum stm32mp1_div_id div_id) +{ + const struct stm32mp1_clk_pll *pll = pll_ref(pll_id); + unsigned long dfout = 0; + uint32_t cfgr2 = 0; + uint32_t divy = 0; + + if (div_id >= _DIV_NB) + return 0; + + cfgr2 = io_read32(stm32_rcc_base() + pll->pllxcfgr2); + divy = (cfgr2 >> pllncfgr2[div_id]) & RCC_PLLNCFGR2_DIVX_MASK; + + dfout = stm32mp1_pll_get_fvco(pll) / (divy + 1U); + + return dfout; +} + +static unsigned long get_clock_rate(enum stm32mp1_parent_id p) +{ + uint32_t reg = 0; + unsigned long clock = 0; + vaddr_t rcc_base = stm32_rcc_base(); + + switch (p) { + case _CK_MPU: + /* MPU sub system */ + reg = io_read32(rcc_base + RCC_MPCKSELR); + switch (reg & RCC_SELR_SRC_MASK) { + case RCC_MPCKSELR_HSI: + clock = osc_frequency(OSC_HSI); + break; + case RCC_MPCKSELR_HSE: + clock = osc_frequency(OSC_HSE); + break; + case RCC_MPCKSELR_PLL: + clock = stm32mp1_read_pll_freq(_PLL1, _DIV_P); + break; + case RCC_MPCKSELR_PLL_MPUDIV: + reg = io_read32(rcc_base + RCC_MPCKDIVR); + if (reg & RCC_MPUDIV_MASK) + clock = stm32mp1_read_pll_freq(_PLL1, _DIV_P) >> + stm32mp1_mpu_div[reg & RCC_MPUDIV_MASK]; + else + clock = 0; + break; + default: + break; + } + break; + /* AXI sub system */ + case _ACLK: + case _HCLK2: + case _HCLK5: + case _HCLK6: + case _PCLK4: + case _PCLK5: + reg = io_read32(rcc_base + RCC_ASSCKSELR); + switch (reg & RCC_SELR_SRC_MASK) { + case RCC_ASSCKSELR_HSI: + clock = osc_frequency(OSC_HSI); + break; + case RCC_ASSCKSELR_HSE: + clock = osc_frequency(OSC_HSE); + break; + case RCC_ASSCKSELR_PLL: + clock = stm32mp1_read_pll_freq(_PLL2, _DIV_P); + break; + default: + break; + } + + /* System clock divider */ + reg = io_read32(rcc_base + RCC_AXIDIVR); + clock /= stm32mp1_axi_div[reg & RCC_AXIDIV_MASK]; + + switch (p) { + case _PCLK4: + reg = io_read32(rcc_base + RCC_APB4DIVR); + clock >>= stm32mp1_apbx_div[reg & RCC_APBXDIV_MASK]; + break; + case _PCLK5: + reg = io_read32(rcc_base + RCC_APB5DIVR); + clock >>= stm32mp1_apbx_div[reg & RCC_APBXDIV_MASK]; + break; + default: + break; + } + break; + /* MCU sub system */ + case _CK_MCU: + case _PCLK1: + case _PCLK2: + case _PCLK3: + reg = io_read32(rcc_base + RCC_MSSCKSELR); + switch (reg & RCC_SELR_SRC_MASK) { + case RCC_MSSCKSELR_HSI: + clock = osc_frequency(OSC_HSI); + break; + case RCC_MSSCKSELR_HSE: + clock = osc_frequency(OSC_HSE); + break; + case RCC_MSSCKSELR_CSI: + clock = osc_frequency(OSC_CSI); + break; + case RCC_MSSCKSELR_PLL: + clock = stm32mp1_read_pll_freq(_PLL3, _DIV_P); + break; + default: + break; + } + + /* MCU clock divider */ + reg = io_read32(rcc_base + RCC_MCUDIVR); + clock >>= stm32mp1_mcu_div[reg & RCC_MCUDIV_MASK]; + + switch (p) { + case _PCLK1: + reg = io_read32(rcc_base + RCC_APB1DIVR); + clock >>= stm32mp1_apbx_div[reg & RCC_APBXDIV_MASK]; + break; + case _PCLK2: + reg = io_read32(rcc_base + RCC_APB2DIVR); + clock >>= stm32mp1_apbx_div[reg & RCC_APBXDIV_MASK]; + break; + case _PCLK3: + reg = io_read32(rcc_base + RCC_APB3DIVR); + clock >>= stm32mp1_apbx_div[reg & RCC_APBXDIV_MASK]; + break; + case _CK_MCU: + default: + break; + } + break; + case _CK_PER: + reg = io_read32(rcc_base + RCC_CPERCKSELR); + switch (reg & RCC_SELR_SRC_MASK) { + case RCC_CPERCKSELR_HSI: + clock = osc_frequency(OSC_HSI); + break; + case RCC_CPERCKSELR_HSE: + clock = osc_frequency(OSC_HSE); + break; + case RCC_CPERCKSELR_CSI: + clock = osc_frequency(OSC_CSI); + break; + default: + break; + } + break; + case _HSI: + case _HSI_KER: + clock = osc_frequency(OSC_HSI); + break; + case _CSI: + case _CSI_KER: + clock = osc_frequency(OSC_CSI); + break; + case _HSE: + case _HSE_KER: + clock = osc_frequency(OSC_HSE); + break; + case _HSE_KER_DIV2: + clock = osc_frequency(OSC_HSE) >> 1; + break; + case _HSE_RTC: + clock = osc_frequency(OSC_HSE); + clock /= (io_read32(rcc_base + RCC_RTCDIVR) & + RCC_DIVR_DIV_MASK) + 1; + break; + case _LSI: + clock = osc_frequency(OSC_LSI); + break; + case _LSE: + clock = osc_frequency(OSC_LSE); + break; + /* PLL */ + case _PLL1_P: + clock = stm32mp1_read_pll_freq(_PLL1, _DIV_P); + break; + case _PLL1_Q: + clock = stm32mp1_read_pll_freq(_PLL1, _DIV_Q); + break; + case _PLL1_R: + clock = stm32mp1_read_pll_freq(_PLL1, _DIV_R); + break; + case _PLL2_P: + clock = stm32mp1_read_pll_freq(_PLL2, _DIV_P); + break; + case _PLL2_Q: + clock = stm32mp1_read_pll_freq(_PLL2, _DIV_Q); + break; + case _PLL2_R: + clock = stm32mp1_read_pll_freq(_PLL2, _DIV_R); + break; + case _PLL3_P: + clock = stm32mp1_read_pll_freq(_PLL3, _DIV_P); + break; + case _PLL3_Q: + clock = stm32mp1_read_pll_freq(_PLL3, _DIV_Q); + break; + case _PLL3_R: + clock = stm32mp1_read_pll_freq(_PLL3, _DIV_R); + break; + case _PLL4_P: + clock = stm32mp1_read_pll_freq(_PLL4, _DIV_P); + break; + case _PLL4_Q: + clock = stm32mp1_read_pll_freq(_PLL4, _DIV_Q); + break; + case _PLL4_R: + clock = stm32mp1_read_pll_freq(_PLL4, _DIV_R); + break; + /* Other */ + case _USB_PHY_48: + clock = osc_frequency(OSC_USB_PHY_48); + break; + default: + break; + } + + return clock; +} + +static void __clk_enable(const struct stm32mp1_clk_gate *gate) +{ + vaddr_t base = stm32_rcc_base(); + uint32_t bit = BIT(gate->bit); + + if (gate->set_clr) + io_write32(base + gate->offset, bit); + else + io_setbits32_stm32shregs(base + gate->offset, bit); + + FMSG("Clock %u has been enabled", gate->clock_id); +} + +static void __clk_disable(const struct stm32mp1_clk_gate *gate) +{ + vaddr_t base = stm32_rcc_base(); + uint32_t bit = BIT(gate->bit); + + if (gate->set_clr) + io_write32(base + gate->offset + RCC_MP_ENCLRR_OFFSET, bit); + else + io_clrbits32_stm32shregs(base + gate->offset, bit); + + FMSG("Clock %u has been disabled", gate->clock_id); +} + +static long get_timer_rate(long parent_rate, unsigned int apb_bus) +{ + uint32_t timgxpre = 0; + uint32_t apbxdiv = 0; + vaddr_t rcc_base = stm32_rcc_base(); + + switch (apb_bus) { + case 1: + apbxdiv = io_read32(rcc_base + RCC_APB1DIVR) & + RCC_APBXDIV_MASK; + timgxpre = io_read32(rcc_base + RCC_TIMG1PRER) & + RCC_TIMGXPRER_TIMGXPRE; + break; + case 2: + apbxdiv = io_read32(rcc_base + RCC_APB2DIVR) & + RCC_APBXDIV_MASK; + timgxpre = io_read32(rcc_base + RCC_TIMG2PRER) & + RCC_TIMGXPRER_TIMGXPRE; + break; + default: + panic(); + break; + } + + if (apbxdiv == 0) + return parent_rate; + + return parent_rate * (timgxpre + 1) * 2; +} + +static unsigned long _stm32_clock_get_rate(unsigned long id) +{ + enum stm32mp1_parent_id p = _UNKNOWN_ID; + unsigned long rate = 0; + + p = stm32mp1_clk_get_parent(id); + if (p < 0) + return 0; + + rate = get_clock_rate(p); + + if ((id >= TIM2_K) && (id <= TIM14_K)) + rate = get_timer_rate(rate, 1); + + if ((id >= TIM1_K) && (id <= TIM17_K)) + rate = get_timer_rate(rate, 2); + + return rate; +} + +/* + * Get the parent ID of the target parent clock, or -1 if no parent found. + */ +static enum stm32mp1_parent_id get_parent_id_parent(enum stm32mp1_parent_id id) +{ + enum stm32mp1_parent_sel s = _UNKNOWN_SEL; + enum stm32mp1_pll_id pll_id = _PLL_NB; + uint32_t p_sel = 0; + + switch (id) { + case _ACLK: + case _HCLK5: + case _HCLK6: + case _PCLK4: + case _PCLK5: + s = _AXISS_SEL; + break; + case _PLL1_P: + case _PLL1_Q: + case _PLL1_R: + pll_id = _PLL1; + break; + case _PLL2_P: + case _PLL2_Q: + case _PLL2_R: + pll_id = _PLL2; + break; + case _PLL3_P: + case _PLL3_Q: + case _PLL3_R: + pll_id = _PLL3; + break; + case _PLL4_P: + case _PLL4_Q: + case _PLL4_R: + pll_id = _PLL4; + break; + case _PCLK1: + case _PCLK2: + case _HCLK2: + case _CK_PER: + case _CK_MPU: + case _CK_MCU: + case _USB_PHY_48: + /* We do not expected to access these */ + panic(); + break; + default: + /* Other parents have no parent */ + return -1; + } + + if (s != _UNKNOWN_SEL) { + const struct stm32mp1_clk_sel *sel = clk_sel_ref(s); + vaddr_t rcc_base = stm32_rcc_base(); + + p_sel = (io_read32(rcc_base + sel->offset) >> sel->src) & + sel->msk; + + if (p_sel < sel->nb_parent) + return sel->parent[p_sel]; + } else { + const struct stm32mp1_clk_pll *pll = pll_ref(pll_id); + + p_sel = io_read32(stm32_rcc_base() + pll->rckxselr) & + RCC_SELR_REFCLK_SRC_MASK; + + if (pll->refclk[p_sel] != _UNKNOWN_OSC_ID) + return osc_id2parent_id(pll->refclk[p_sel]); + } + + FMSG("No parent found for %s", stm32mp1_clk_parent_name[id]); + return -1; +} + +/* We are only interested in knowing if PLL3 shall be secure or not */ +static void secure_parent_clocks(enum stm32mp1_parent_id parent_id) +{ + enum stm32mp1_parent_id grandparent_id = _UNKNOWN_ID; + + switch (parent_id) { + case _ACLK: + case _HCLK2: + case _HCLK5: + case _HCLK6: + case _PCLK4: + case _PCLK5: + /* Intermediate clock mux or clock, go deeper in clock tree */ + break; + case _HSI: + case _HSI_KER: + case _LSI: + case _CSI: + case _CSI_KER: + case _HSE: + case _HSE_KER: + case _HSE_KER_DIV2: + case _HSE_RTC: + case _LSE: + case _PLL1_P: + case _PLL1_Q: + case _PLL1_R: + case _PLL2_P: + case _PLL2_Q: + case _PLL2_R: + /* Always secure clocks, no need to go further */ + return; + case _PLL3_P: + case _PLL3_Q: + case _PLL3_R: + /* PLL3 is a shared resource, registered and don't go further */ + stm32mp_register_secure_periph(STM32MP1_SHRES_PLL3); + return; + default: + DMSG("Cannot lookup parent clock %s", + stm32mp1_clk_parent_name[parent_id]); + panic(); + } + + grandparent_id = get_parent_id_parent(parent_id); + if (grandparent_id >= 0) + secure_parent_clocks(grandparent_id); +} + +void stm32mp_register_clock_parents_secure(unsigned long clock_id) +{ + enum stm32mp1_parent_id parent_id = stm32mp1_clk_get_parent(clock_id); + + if (parent_id < 0) { + DMSG("No parent for clock %lu", clock_id); + return; + } + + secure_parent_clocks(parent_id); +} + +static const char *stm32mp_osc_node_label[NB_OSC] = { + [OSC_LSI] = "clk-lsi", + [OSC_LSE] = "clk-lse", + [OSC_HSI] = "clk-hsi", + [OSC_HSE] = "clk-hse", + [OSC_CSI] = "clk-csi", + [OSC_I2S_CKIN] = "i2s_ckin", + [OSC_USB_PHY_48] = "ck_usbo_48m" +}; + +static unsigned int clk_freq_prop(const void *fdt, int node) +{ + const fdt32_t *cuint = NULL; + int ret = 0; + + /* Disabled clocks report null rate */ + if (fdt_get_status(fdt, node) == DT_STATUS_DISABLED) + return 0; + + cuint = fdt_getprop(fdt, node, "clock-frequency", &ret); + if (!cuint) + panic(); + + return fdt32_to_cpu(*cuint); +} + +static void get_osc_freq_from_dt(const void *fdt) +{ + enum stm32mp_osc_id idx = _UNKNOWN_OSC_ID; + int clk_node = fdt_path_offset(fdt, "/clocks"); + + if (clk_node < 0) + panic(); + + COMPILE_TIME_ASSERT((int)OSC_HSI == 0); + for (idx = OSC_HSI; idx < NB_OSC; idx++) { + const char *name = stm32mp_osc_node_label[idx]; + int subnode = 0; + + fdt_for_each_subnode(subnode, fdt, clk_node) { + const char *cchar = NULL; + int ret = 0; + + cchar = fdt_get_name(fdt, subnode, &ret); + if (!cchar) + panic(); + + if (strncmp(cchar, name, (size_t)ret) == 0) { + stm32mp1_osc[idx] = clk_freq_prop(fdt, subnode); + + DMSG("Osc %s: %lu Hz", name, stm32mp1_osc[idx]); + break; + } + } + + if (!stm32mp1_osc[idx]) + DMSG("Osc %s: no frequency info", name); + } +} + +static void enable_static_secure_clocks(void) +{ + unsigned int idx = 0; + const unsigned long secure_enable[] = { + DDRC1, DDRC1LP, DDRC2, DDRC2LP, DDRPHYC, DDRPHYCLP, DDRCAPB, + AXIDCG, DDRPHYCAPB, DDRPHYCAPBLP, TZPC, TZC1, TZC2, STGEN_K, + BSEC, + }; + + for (idx = 0; idx < ARRAY_SIZE(secure_enable); idx++) { + clk_enable(stm32mp_rcc_clock_id_to_clk(secure_enable[idx])); + stm32mp_register_clock_parents_secure(secure_enable[idx]); + } + + if (CFG_TEE_CORE_NB_CORE > 1) + clk_enable(stm32mp_rcc_clock_id_to_clk(RTCAPB)); +} + +static void __maybe_unused enable_rcc_tzen(void) +{ + io_setbits32(stm32_rcc_base() + RCC_TZCR, RCC_TZCR_TZEN); +} + +static void __maybe_unused disable_rcc_tzen(void) +{ + IMSG("RCC is non-secure"); + io_clrbits32(stm32_rcc_base() + RCC_TZCR, RCC_TZCR_TZEN); +} + +static TEE_Result stm32mp1_clk_fdt_init(const void *fdt, int node) +{ + unsigned int i = 0; + int len = 0; + int ignored = 0; + + get_osc_freq_from_dt(fdt); + + /* + * OP-TEE core is not in charge of configuring clock parenthood. + * This is expected from an earlier boot stage. Modifying the clock + * tree parenthood here may jeopardize already configured clocks. + * The sequence below ignores such DT directives with a friendly + * debug trace. + */ + if (fdt_getprop(fdt, node, "st,clksrc", &len)) { + DMSG("Ignore source clocks configuration from DT"); + ignored++; + } + if (fdt_getprop(fdt, node, "st,clkdiv", &len)) { + DMSG("Ignore clock divisors configuration from DT"); + ignored++; + } + if (fdt_getprop(fdt, node, "st,pkcs", &len)) { + DMSG("Ignore peripheral clocks tree configuration from DT"); + ignored++; + } + for (i = (enum stm32mp1_pll_id)0; i < _PLL_NB; i++) { + char name[] = "st,pll@X"; + + snprintf(name, sizeof(name), "st,pll@%d", i); + node = fdt_subnode_offset(fdt, node, name); + if (node < 0) + continue; + + if (fdt_getprop(fdt, node, "cfg", &len) || + fdt_getprop(fdt, node, "frac", &len)) { + DMSG("Ignore PLL%u configurations from DT", i); + ignored++; + } + } + + if (ignored != 0) + IMSG("DT clock tree configurations were ignored"); + + return TEE_SUCCESS; +} + +/* + * Conversion between clk references and clock gates and clock on internals + * + * stm32mp1_clk first cells follow stm32mp1_clk_gate[] ordering. + * stm32mp1_clk last cells follow stm32mp1_clk_on[] ordering. + */ +static struct clk stm32mp1_clk[ARRAY_SIZE(stm32mp1_clk_gate) + + ARRAY_SIZE(stm32mp1_clk_on)]; + +#define CLK_ON_INDEX_OFFSET ((int)ARRAY_SIZE(stm32mp1_clk_gate)) + +static bool clk_is_gate(struct clk *clk) +{ + int clk_index = clk - stm32mp1_clk; + + assert(clk_index >= 0 && clk_index < (int)ARRAY_SIZE(stm32mp1_clk)); + return clk_index < CLK_ON_INDEX_OFFSET; +} + +static unsigned long clk_to_clock_id(struct clk *clk) +{ + int gate_index = clk - stm32mp1_clk; + int on_index = gate_index - CLK_ON_INDEX_OFFSET; + + if (clk_is_gate(clk)) + return stm32mp1_clk_gate[gate_index].clock_id; + + return stm32mp1_clk_on[on_index]; +} + +static const struct stm32mp1_clk_gate *clk_to_gate_ref(struct clk *clk) +{ + int gate_index = clk - stm32mp1_clk; + + assert(clk_is_gate(clk)); + + return stm32mp1_clk_gate + gate_index; +} + +static int clock_id_to_gate_index(unsigned long clock_id) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(stm32mp1_clk_gate); n++) + if (stm32mp1_clk_gate[n].clock_id == clock_id) + return n; + + return -1; +} + +static int clock_id_to_always_on_index(unsigned long clock_id) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(stm32mp1_clk_on); n++) + if (stm32mp1_clk_on[n] == clock_id) + return n; + + return -1; +} + +static struct clk *clock_id_to_clk(unsigned long clock_id) +{ + int gate_index = clock_id_to_gate_index(clock_id); + int on_index = clock_id_to_always_on_index(clock_id); + + if (gate_index >= 0) + return stm32mp1_clk + gate_index; + + if (on_index >= 0) + return stm32mp1_clk + CLK_ON_INDEX_OFFSET + on_index; + + return NULL; +} + +struct clk *stm32mp_rcc_clock_id_to_clk(unsigned long clock_id) +{ + return clock_id_to_clk(clock_id); +} + +#if (CFG_TEE_CORE_LOG_LEVEL >= TRACE_DEBUG) && defined(CFG_TEE_CORE_DEBUG) +struct clk_name { + unsigned int clock_id; + const char *name; +}; + +#define CLOCK_NAME(_binding, _name) \ + { .clock_id = (_binding), .name = (_name) } + +/* Store names only for some clocks */ +const struct clk_name exposed_clk_name[] = { + /* Clocks used by platform drivers not yet probed from DT */ + CLOCK_NAME(CK_DBG, "dbg"), + CLOCK_NAME(CK_MCU, "mcu"), + CLOCK_NAME(RTCAPB, "rtcapb"), + CLOCK_NAME(BKPSRAM, "bkpsram"), + CLOCK_NAME(RTC, "rtc"), + CLOCK_NAME(CRYP1, "crpy1"), + CLOCK_NAME(SYSCFG, "syscfg"), + CLOCK_NAME(GPIOA, "gpioa"), + CLOCK_NAME(GPIOB, "gpiob"), + CLOCK_NAME(GPIOC, "gpioc"), + CLOCK_NAME(GPIOD, "gpiod"), + CLOCK_NAME(GPIOE, "gpioe"), + CLOCK_NAME(GPIOF, "gpiof"), + CLOCK_NAME(GPIOG, "gpiog"), + CLOCK_NAME(GPIOH, "gpioh"), + CLOCK_NAME(GPIOI, "gpioi"), + CLOCK_NAME(GPIOJ, "gpioj"), + CLOCK_NAME(GPIOK, "gpiok"), + CLOCK_NAME(GPIOZ, "gpioz"), + /* Clock exposed by SCMI. SCMI clock fmro DT bindings to come... */ + CLOCK_NAME(CK_HSE, "hse"), + CLOCK_NAME(CK_HSI, "hsi"), + CLOCK_NAME(CK_CSI, "csi"), + CLOCK_NAME(CK_LSE, "lse"), + CLOCK_NAME(CK_LSI, "lsi"), + CLOCK_NAME(PLL2_Q, "pll2q"), + CLOCK_NAME(PLL2_R, "pll2r"), + CLOCK_NAME(PLL3_Q, "pll3q"), + CLOCK_NAME(PLL3_R, "pll3r"), + CLOCK_NAME(CRYP1, "cryp1"), + CLOCK_NAME(HASH1, "hash1"), + CLOCK_NAME(I2C4_K, "i2c4"), + CLOCK_NAME(I2C6_K, "i2c6"), + CLOCK_NAME(IWDG1, "iwdg"), + CLOCK_NAME(RNG1_K, "rng1"), + CLOCK_NAME(SPI6_K, "spi6"), + CLOCK_NAME(USART1_K, "usart1"), + CLOCK_NAME(CK_MCU, "mcu"), +}; +DECLARE_KEEP_PAGER(exposed_clk_name); + +static const char *clk_op_get_name(struct clk *clk) +{ + unsigned long clock_id = clk_to_clock_id(clk); + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(exposed_clk_name); n++) + if (exposed_clk_name[n].clock_id == clock_id) + return exposed_clk_name[n].name; + + return NULL; +} +#else +static const char *clk_op_get_name(struct clk *clk __unused) +{ + return NULL; +} +#endif /*CFG_TEE_CORE_LOG_LEVEL*/ + +static unsigned long clk_op_compute_rate(struct clk *clk, + unsigned long parent_rate __unused) +{ + return _stm32_clock_get_rate(clk_to_clock_id(clk)); +} + +static TEE_Result clk_op_enable(struct clk *clk) +{ + if (clk_is_gate(clk)) + __clk_enable(clk_to_gate_ref(clk)); + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(clk_op_enable); + +static void clk_op_disable(struct clk *clk) +{ + if (clk_is_gate(clk)) + __clk_disable(clk_to_gate_ref(clk)); +} +DECLARE_KEEP_PAGER(clk_op_disable); + +/* This variable is weak to break its dependency chain when linked as unpaged */ +const struct clk_ops stm32mp1_clk_ops +__weak __relrodata_unpaged("stm32mp1_clk_ops") = { + .enable = clk_op_enable, + .disable = clk_op_disable, + .get_rate = clk_op_compute_rate, +}; + +static TEE_Result register_stm32mp1_clocks(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(stm32mp1_clk); n++) { + stm32mp1_clk[n].ops = &stm32mp1_clk_ops; + stm32mp1_clk[n].name = clk_op_get_name(stm32mp1_clk + n); + refcount_set(&stm32mp1_clk[n].enabled_count, 0); + + res = clk_register(stm32mp1_clk + n); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +static TEE_Result stm32mp1_clk_dt_get_clk(struct dt_pargs *pargs, + void *data __unused, + struct clk **out_clk) +{ + unsigned long clock_id = pargs->args[0]; + struct clk *clk = NULL; + + if (pargs->args_count != 1) + return TEE_ERROR_BAD_PARAMETERS; + + clk = clock_id_to_clk(clock_id); + if (!clk) + return TEE_ERROR_BAD_PARAMETERS; + + *out_clk = clk; + + return TEE_SUCCESS; +} + +/* Non-null reference for compat data */ +static const uint8_t non_secure_rcc; + +static TEE_Result stm32mp1_clock_provider_probe(const void *fdt, int offs, + const void *compat_data) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + if (compat_data == &non_secure_rcc) + disable_rcc_tzen(); + else + enable_rcc_tzen(); + + res = stm32mp1_clk_fdt_init(fdt, offs); + if (res) { + EMSG("Failed to initialize clocks from DT: %#"PRIx32, res); + panic(); + } + + res = register_stm32mp1_clocks(); + if (res) { + EMSG("Failed to register clocks: %#"PRIx32, res); + panic(); + } + + res = clk_dt_register_clk_provider(fdt, offs, stm32mp1_clk_dt_get_clk, + NULL); + if (res) { + EMSG("Failed to register clock provider: %#"PRIx32, res); + panic(); + } + + enable_static_secure_clocks(); + + return TEE_SUCCESS; +} + +static const struct dt_device_match stm32mp1_clock_match_table[] = { + { .compatible = "st,stm32mp1-rcc", .compat_data = &non_secure_rcc, }, + { .compatible = "st,stm32mp1-rcc-secure", }, + { } +}; + +DEFINE_DT_DRIVER(stm32mp1_clock_dt_driver) = { + .name = "stm32mp1_clock", + .type = DT_DRIVER_CLK, + .match_table = stm32mp1_clock_match_table, + .probe = stm32mp1_clock_provider_probe, +}; diff --git a/optee_os/core/drivers/clk/clk.c b/optee_os/core/drivers/clk/clk.c new file mode 100644 index 0000000..00cb57e --- /dev/null +++ b/optee_os/core/drivers/clk/clk.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Bootlin + */ + +#include +#include +#include +#include +#include +#include +#include + +/* Global clock tree lock */ +static unsigned int clk_lock = SPINLOCK_UNLOCK; + +struct clk *clk_alloc(const char *name, const struct clk_ops *ops, + struct clk **parent_clks, size_t parent_count) +{ + struct clk *clk = NULL; + size_t parent = 0; + + clk = calloc(1, sizeof(*clk) + parent_count * sizeof(clk)); + if (!clk) + return NULL; + + clk->num_parents = parent_count; + for (parent = 0; parent < parent_count; parent++) + clk->parents[parent] = parent_clks[parent]; + + clk->name = name; + clk->ops = ops; + refcount_set(&clk->enabled_count, 0); + + return clk; +} + +void clk_free(struct clk *clk) +{ + free(clk); +} + +static bool __maybe_unused clk_check(struct clk *clk) +{ + if (!clk->ops) + return false; + + if (clk->ops->set_parent && !clk->ops->get_parent) + return false; + + if (clk->num_parents > 1 && !clk->ops->get_parent) + return false; + + return true; +} + +static void clk_compute_rate_no_lock(struct clk *clk) +{ + unsigned long parent_rate = 0; + + if (clk->parent) + parent_rate = clk->parent->rate; + + if (clk->ops->get_rate) + clk->rate = clk->ops->get_rate(clk, parent_rate); + else + clk->rate = parent_rate; +} + +struct clk *clk_get_parent_by_index(struct clk *clk, size_t pidx) +{ + if (pidx >= clk->num_parents) + return NULL; + + return clk->parents[pidx]; +} + +static void clk_init_parent(struct clk *clk) +{ + size_t pidx = 0; + + switch (clk->num_parents) { + case 0: + break; + case 1: + clk->parent = clk->parents[0]; + break; + default: + pidx = clk->ops->get_parent(clk); + assert(pidx < clk->num_parents); + + clk->parent = clk->parents[pidx]; + break; + } +} + +TEE_Result clk_register(struct clk *clk) +{ + assert(clk_check(clk)); + + clk_init_parent(clk); + clk_compute_rate_no_lock(clk); + + DMSG("Registered clock %s, freq %lu", clk->name, clk_get_rate(clk)); + + return TEE_SUCCESS; +} + +static bool clk_is_enabled_no_lock(struct clk *clk) +{ + return refcount_val(&clk->enabled_count) != 0; +} + +bool clk_is_enabled(struct clk *clk) +{ + return clk_is_enabled_no_lock(clk); +} + +static void clk_disable_no_lock(struct clk *clk) +{ + struct clk *parent = NULL; + + if (!refcount_dec(&clk->enabled_count)) + return; + + if (clk->ops->disable) + clk->ops->disable(clk); + + parent = clk_get_parent(clk); + if (parent) + clk_disable_no_lock(parent); +} + +static TEE_Result clk_enable_no_lock(struct clk *clk) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct clk *parent = NULL; + + if (refcount_inc(&clk->enabled_count)) + return TEE_SUCCESS; + + parent = clk_get_parent(clk); + if (parent) { + res = clk_enable_no_lock(parent); + if (res) + return res; + } + + if (clk->ops->enable) { + res = clk->ops->enable(clk); + if (res) { + if (parent) + clk_disable_no_lock(parent); + + return res; + } + } + + refcount_set(&clk->enabled_count, 1); + + return TEE_SUCCESS; +} + +TEE_Result clk_enable(struct clk *clk) +{ + uint32_t exceptions = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + exceptions = cpu_spin_lock_xsave(&clk_lock); + res = clk_enable_no_lock(clk); + cpu_spin_unlock_xrestore(&clk_lock, exceptions); + + return res; +} + +void clk_disable(struct clk *clk) +{ + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&clk_lock); + clk_disable_no_lock(clk); + cpu_spin_unlock_xrestore(&clk_lock, exceptions); +} + +unsigned long clk_get_rate(struct clk *clk) +{ + return clk->rate; +} + +static TEE_Result clk_set_rate_no_lock(struct clk *clk, unsigned long rate) +{ + TEE_Result res = TEE_ERROR_GENERIC; + unsigned long parent_rate = 0; + + if (clk->parent) + parent_rate = clk_get_rate(clk->parent); + + res = clk->ops->set_rate(clk, rate, parent_rate); + if (res) + return res; + + clk_compute_rate_no_lock(clk); + + return TEE_SUCCESS; +} + +TEE_Result clk_set_rate(struct clk *clk, unsigned long rate) +{ + uint32_t exceptions = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + if (!clk->ops->set_rate) + return TEE_ERROR_NOT_SUPPORTED; + + exceptions = cpu_spin_lock_xsave(&clk_lock); + + if (clk->flags & CLK_SET_RATE_GATE && clk_is_enabled_no_lock(clk)) + res = TEE_ERROR_BAD_STATE; + else + res = clk_set_rate_no_lock(clk, rate); + + cpu_spin_unlock_xrestore(&clk_lock, exceptions); + + return res; +} + +struct clk *clk_get_parent(struct clk *clk) +{ + return clk->parent; +} + +static TEE_Result clk_get_parent_idx(struct clk *clk, struct clk *parent, + size_t *pidx) +{ + size_t i = 0; + + for (i = 0; i < clk_get_num_parents(clk); i++) { + if (clk_get_parent_by_index(clk, i) == parent) { + *pidx = i; + return TEE_SUCCESS; + } + } + EMSG("Clock %s is not a parent of clock %s", parent->name, clk->name); + + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result clk_set_parent_no_lock(struct clk *clk, struct clk *parent, + size_t pidx) +{ + TEE_Result res = TEE_ERROR_GENERIC; + bool was_enabled = false; + + /* Requested parent is already the one set */ + if (clk->parent == parent) + return TEE_SUCCESS; + + was_enabled = clk_is_enabled_no_lock(clk); + /* Call is needed to decrement refcount on current parent tree */ + if (was_enabled) + clk_disable_no_lock(clk); + + res = clk->ops->set_parent(clk, pidx); + if (res) + goto out; + + clk->parent = parent; + + /* The parent changed and the rate might also have changed */ + clk_compute_rate_no_lock(clk); + +out: + /* Call is needed to increment refcount on the new parent tree */ + if (was_enabled) { + res = clk_enable_no_lock(clk); + if (res) + panic("Failed to re-enable clock after setting parent"); + } + + return res; +} + +TEE_Result clk_set_parent(struct clk *clk, struct clk *parent) +{ + size_t pidx = 0; + uint32_t exceptions = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + if (clk_get_parent_idx(clk, parent, &pidx) || !clk->ops->set_parent) + return TEE_ERROR_BAD_PARAMETERS; + + exceptions = cpu_spin_lock_xsave(&clk_lock); + if (clk->flags & CLK_SET_PARENT_GATE && clk_is_enabled_no_lock(clk)) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + res = clk_set_parent_no_lock(clk, parent, pidx); +out: + cpu_spin_unlock_xrestore(&clk_lock, exceptions); + + return res; +} + +TEE_Result clk_get_rates_array(struct clk *clk, size_t start_index, + unsigned long *rates, size_t *nb_elts) +{ + if (!clk->ops->get_rates_array) + return TEE_ERROR_NOT_SUPPORTED; + + return clk->ops->get_rates_array(clk, start_index, rates, nb_elts); +} diff --git a/optee_os/core/drivers/clk/clk_dt.c b/optee_os/core/drivers/clk/clk_dt.c new file mode 100644 index 0000000..5197a91 --- /dev/null +++ b/optee_os/core/drivers/clk/clk_dt.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Bootlin + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TEE_Result clk_dt_get_by_name(const void *fdt, int nodeoffset, + const char *name, struct clk **clk) +{ + int clk_id = 0; + + clk_id = fdt_stringlist_search(fdt, nodeoffset, "clock-names", name); + if (clk_id < 0) { + *clk = NULL; + return TEE_ERROR_GENERIC; + } + + return clk_dt_get_by_index(fdt, nodeoffset, clk_id, clk); +} + +static TEE_Result clk_dt_get_by_idx_prop(const char *prop_name, const void *fdt, + int nodeoffset, unsigned int clk_idx, + struct clk **clk) +{ + TEE_Result res = TEE_ERROR_GENERIC; + void *out_clk = NULL; + + res = dt_driver_device_from_node_idx_prop(prop_name, fdt, nodeoffset, + clk_idx, DT_DRIVER_CLK, + &out_clk); + if (!res) + *clk = out_clk; + + return res; +} + +TEE_Result clk_dt_get_by_index(const void *fdt, int nodeoffset, + unsigned int clk_idx, struct clk **clk) +{ + return clk_dt_get_by_idx_prop("clocks", fdt, nodeoffset, clk_idx, clk); +} + +#ifdef CFG_DRIVERS_CLK_EARLY_PROBE +/* Recursively called from parse_clock_property() */ +static TEE_Result clk_probe_clock_provider_node(const void *fdt, int node); + +static TEE_Result parse_clock_property(const void *fdt, int node) +{ + int len = 0; + int idx = 0; + int parent_node = 0; + int clock_cells = 0; + uint32_t phandle = 0; + const uint32_t *prop = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + prop = fdt_getprop(fdt, node, "clocks", &len); + if (!prop) + return TEE_SUCCESS; + + len /= sizeof(uint32_t); + while (idx < len) { + phandle = fdt32_to_cpu(prop[idx]); + + parent_node = fdt_node_offset_by_phandle(fdt, phandle); + if (parent_node < 0) + return TEE_ERROR_GENERIC; + + /* Parent probe should not fail or clock won't be available */ + res = clk_probe_clock_provider_node(fdt, parent_node); + if (res) { + EMSG("Probe parent clock node %s on node %s: %#"PRIx32, + fdt_get_name(fdt, parent_node, NULL), + fdt_get_name(fdt, node, NULL), res); + panic(); + } + + clock_cells = fdt_get_dt_driver_cells(fdt, parent_node, + DT_DRIVER_CLK); + if (clock_cells < 0) + return TEE_ERROR_GENERIC; + + idx += 1 + clock_cells; + } + + return TEE_SUCCESS; +} + +static TEE_Result clk_probe_clock_provider_node(const void *fdt, int node) +{ + int len = 0; + int status = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + status = fdt_get_status(fdt, node); + if (!(status & DT_STATUS_OK_SEC)) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* Check if the node is a clock provider */ + if (!fdt_getprop(fdt, node, "#clock-cells", &len)) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* Check if node has already been probed */ + if (dt_driver_get_provider_by_node(node, DT_DRIVER_CLK)) + return TEE_SUCCESS; + + /* Check if the node has a clock property first to probe parent */ + res = parse_clock_property(fdt, node); + if (res) + return res; + + return dt_driver_probe_device_by_node(fdt, node, DT_DRIVER_CLK); +} + +static void clk_probe_node(const void *fdt, int parent_node) +{ + int child = 0; + int status = 0; + __maybe_unused TEE_Result res = TEE_ERROR_GENERIC; + + fdt_for_each_subnode(child, fdt, parent_node) { + status = fdt_get_status(fdt, child); + if (status == DT_STATUS_DISABLED) + continue; + + res = clk_probe_clock_provider_node(fdt, child); + assert(res == TEE_SUCCESS || res == TEE_ERROR_ITEM_NOT_FOUND); + + clk_probe_node(fdt, child); + } +} + +static void parse_assigned_clock(const void *fdt, int nodeoffset) +{ + int rate_len = 0; + int clock_idx = 0; + struct clk *clk = NULL; + unsigned long rate = 0; + struct clk *parent = NULL; + const uint32_t *rate_prop = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + rate_prop = fdt_getprop(fdt, nodeoffset, "assigned-clock-rates", + &rate_len); + rate_len /= sizeof(uint32_t); + + while (true) { + res = clk_dt_get_by_idx_prop("assigned-clocks", fdt, nodeoffset, + clock_idx, &clk); + if (res) + return; + assert(clk); + + res = clk_dt_get_by_idx_prop("assigned-clock-parents", fdt, + nodeoffset, clock_idx, &parent); + if (parent) { + assert(!res); + if (clk_set_parent(clk, parent)) { + EMSG("Could not set clk %s parent to clock %s", + clk->name, parent->name); + panic(); + } + } + + if (rate_prop && clock_idx < rate_len) { + rate = fdt32_to_cpu(rate_prop[clock_idx]); + if (rate && clk_set_rate(clk, rate) != TEE_SUCCESS) + panic(); + } + + clock_idx++; + } +} + +static void clk_probe_assigned(const void *fdt, int parent_node) +{ + int len = 0; + int child = 0; + int status = 0; + + fdt_for_each_subnode(child, fdt, parent_node) { + clk_probe_assigned(fdt, child); + + status = fdt_get_status(fdt, child); + if (status == DT_STATUS_DISABLED) + continue; + + if (fdt_getprop(fdt, child, "assigned-clocks", &len)) + parse_assigned_clock(fdt, child); + } +} + +static TEE_Result clk_dt_probe(void) +{ + const void *fdt = get_secure_dt(); + + DMSG("Probing clocks from devicetree"); + if (!fdt) + panic(); + + clk_probe_node(fdt, -1); + + clk_probe_assigned(fdt, -1); + + return TEE_SUCCESS; +} +early_init(clk_dt_probe); +#endif diff --git a/optee_os/core/drivers/clk/fixed_clk.c b/optee_os/core/drivers/clk/fixed_clk.c new file mode 100644 index 0000000..2da1df9 --- /dev/null +++ b/optee_os/core/drivers/clk/fixed_clk.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Bootlin + */ + +#include +#include +#include +#include +#include + +struct fixed_clock_data { + unsigned long rate; +}; + +static unsigned long fixed_clk_get_rate(struct clk *clk, + unsigned long parent_rate __unused) +{ + struct fixed_clock_data *d = clk->priv; + + return d->rate; +} + +static const struct clk_ops fixed_clk_clk_ops = { + .get_rate = fixed_clk_get_rate, +}; + +static TEE_Result fixed_clock_probe(const void *fdt, int offs, + const void *compat_data __unused) +{ + const uint32_t *freq = NULL; + const char *name = NULL; + struct clk *clk = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + struct fixed_clock_data *fcd = NULL; + + name = fdt_get_name(fdt, offs, NULL); + if (!name) + name = "fixed-clock"; + + clk = clk_alloc(name, &fixed_clk_clk_ops, NULL, 0); + if (!clk) + return TEE_ERROR_OUT_OF_MEMORY; + + fcd = calloc(1, sizeof(struct fixed_clock_data)); + if (!fcd) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto free_clk; + } + + freq = fdt_getprop(fdt, offs, "clock-frequency", NULL); + if (!freq) { + res = TEE_ERROR_BAD_FORMAT; + goto free_fcd; + } + + fcd->rate = fdt32_to_cpu(*freq); + clk->priv = fcd; + + res = clk_register(clk); + if (res) + goto free_fcd; + + res = clk_dt_register_clk_provider(fdt, offs, clk_dt_get_simple_clk, + clk); + if (!res) + return TEE_SUCCESS; + +free_fcd: + free(fcd); +free_clk: + clk_free(clk); + + return res; +} + +CLK_DT_DECLARE(fixed_clock, "fixed-clock", fixed_clock_probe); diff --git a/optee_os/core/drivers/clk/sam/at91_audio_pll.c b/optee_os/core/drivers/clk/sam/at91_audio_pll.c new file mode 100644 index 0000000..72b6e60 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_audio_pll.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2016 Atmel Corporation, + * Songjun Wu , + * Nicolas Ferre + * Copyright (C) 2017 Free Electrons, + * Quentin Schulz + * + * The Sama5d2 SoC has two audio PLLs (PMC and PAD) that shares the same parent + * (FRAC). FRAC can output between 620 and 700MHz and only multiply the rate of + * its own parent. PMC and PAD can then divide the FRAC rate to best match the + * asked rate. + * + * Traits of FRAC clock: + * enable - clk_enable writes nd, fracr parameters and enables PLL + * rate - rate is adjustable. + * clk->rate = parent->rate * ((nd + 1) + (fracr / 2^22)) + * parent - fixed parent. No clk_set_parent support + * + * Traits of PMC clock: + * enable - clk_enable writes qdpmc, and enables PMC output + * rate - rate is adjustable. + * clk->rate = parent->rate / (qdpmc + 1) + * parent - fixed parent. No clk_set_parent support + * + * Traits of PAD clock: + * enable - clk_enable writes divisors and enables PAD output + * rate - rate is adjustable. + * clk->rate = parent->rate / (qdaudio * div)) + * parent - fixed parent. No clk_set_parent support + */ + +#include +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define AUDIO_PLL_DIV_FRAC BIT(22) +#define AUDIO_PLL_ND_MAX (AT91_PMC_AUDIO_PLL_ND_MASK >> \ + AT91_PMC_AUDIO_PLL_ND_OFFSET) + +#define AUDIO_PLL_QDPAD(qd, div) \ + ((AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV(qd) & \ + AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_MASK) | \ + (AT91_PMC_AUDIO_PLL_QDPAD_DIV(div) & \ + AT91_PMC_AUDIO_PLL_QDPAD_DIV_MASK)) + +#define AUDIO_PLL_QDPMC_MAX (AT91_PMC_AUDIO_PLL_QDPMC_MASK >> \ + AT91_PMC_AUDIO_PLL_QDPMC_OFFSET) + +#define AUDIO_PLL_FOUT_MIN 620000000UL +#define AUDIO_PLL_FOUT_MAX 700000000UL + +struct clk_audio_frac { + vaddr_t base; + uint32_t fracr; + uint8_t nd; +}; + +struct clk_audio_pad { + vaddr_t base; + uint8_t qdaudio; + uint8_t div; +}; + +struct clk_audio_pmc { + vaddr_t base; + uint8_t qdpmc; +}; + +static TEE_Result clk_audio_pll_frac_enable(struct clk *clk) +{ + struct clk_audio_frac *frac = clk->priv; + + io_clrbits32(frac->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_RESETN); + io_setbits32(frac->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_RESETN); + io_clrsetbits32(frac->base + AT91_PMC_AUDIO_PLL1, + AT91_PMC_AUDIO_PLL_FRACR_MASK, frac->fracr); + + /* + * reset and enable have to be done in 2 separated writes + * for AT91_PMC_AUDIO_PLL0 + */ + io_clrsetbits32(frac->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_PLLEN | + AT91_PMC_AUDIO_PLL_ND_MASK, + AT91_PMC_AUDIO_PLL_PLLEN | + AT91_PMC_AUDIO_PLL_ND(frac->nd)); + + return TEE_SUCCESS; +} + +static TEE_Result clk_audio_pll_pad_enable(struct clk *clk) +{ + struct clk_audio_pad *apad_ck = clk->priv; + + io_clrsetbits32(apad_ck->base + AT91_PMC_AUDIO_PLL1, + AT91_PMC_AUDIO_PLL_QDPAD_MASK, + AUDIO_PLL_QDPAD(apad_ck->qdaudio, apad_ck->div)); + io_clrsetbits32(apad_ck->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_PADEN, AT91_PMC_AUDIO_PLL_PADEN); + + return TEE_SUCCESS; +} + +static TEE_Result clk_audio_pll_pmc_enable(struct clk *clk) +{ + struct clk_audio_pmc *apmc_ck = clk->priv; + + io_clrsetbits32(apmc_ck->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_PMCEN | + AT91_PMC_AUDIO_PLL_QDPMC_MASK, + AT91_PMC_AUDIO_PLL_PMCEN | + AT91_PMC_AUDIO_PLL_QDPMC(apmc_ck->qdpmc)); + return TEE_SUCCESS; +} + +static void clk_audio_pll_frac_disable(struct clk *clk) +{ + struct clk_audio_frac *frac = clk->priv; + + io_clrbits32(frac->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_PLLEN); + /* Requires 2 separated writes */ + io_clrbits32(frac->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_RESETN); +} + +static void clk_audio_pll_pad_disable(struct clk *clk) +{ + struct clk_audio_pad *apad_ck = clk->priv; + + io_clrbits32(apad_ck->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_PADEN); +} + +static void clk_audio_pll_pmc_disable(struct clk *clk) +{ + struct clk_audio_pmc *apmc_ck = clk->priv; + + io_clrbits32(apmc_ck->base + AT91_PMC_AUDIO_PLL0, + AT91_PMC_AUDIO_PLL_PMCEN); +} + +static unsigned long clk_audio_pll_fout(unsigned long parent_rate, + unsigned long nd, unsigned long fracr) +{ + unsigned long long fr = (unsigned long long)parent_rate * fracr; + + fr = UDIV_ROUND_NEAREST(fr, AUDIO_PLL_DIV_FRAC); + + return parent_rate * (nd + 1) + fr; +} + +static unsigned long clk_audio_pll_frac_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_audio_frac *frac = clk->priv; + + return clk_audio_pll_fout(parent_rate, frac->nd, frac->fracr); +} + +static unsigned long clk_audio_pll_pad_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_audio_pad *apad_ck = clk->priv; + unsigned long apad_rate = 0; + + if (apad_ck->qdaudio && apad_ck->div) + apad_rate = parent_rate / (apad_ck->qdaudio * apad_ck->div); + + return apad_rate; +} + +static unsigned long clk_audio_pll_pmc_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_audio_pmc *apmc_ck = clk->priv; + + return parent_rate / (apmc_ck->qdpmc + 1); +} + +static TEE_Result clk_audio_pll_frac_compute_frac(unsigned long rate, + unsigned long parent_rate, + unsigned long *nd, + unsigned long *fracr) +{ + unsigned long long tmp = 0; + unsigned long long rem = 0; + + if (!rate || !parent_rate) + return TEE_ERROR_BAD_PARAMETERS; + + tmp = rate; + rem = tmp % parent_rate; + tmp /= parent_rate; + if (!tmp || tmp >= AUDIO_PLL_ND_MAX) + return TEE_ERROR_BAD_PARAMETERS; + + *nd = tmp - 1; + + tmp = rem * AUDIO_PLL_DIV_FRAC; + tmp = UDIV_ROUND_NEAREST(tmp, parent_rate); + if (tmp > AT91_PMC_AUDIO_PLL_FRACR_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + /* we can cast here as we verified the bounds just above */ + *fracr = (unsigned long)tmp; + + return TEE_SUCCESS; +} + +static TEE_Result clk_audio_pll_frac_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate) +{ + struct clk_audio_frac *frac = clk->priv; + unsigned long fracr = 0; + unsigned long nd = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + if (rate < AUDIO_PLL_FOUT_MIN || rate > AUDIO_PLL_FOUT_MAX) + return TEE_ERROR_BAD_PARAMETERS; + + res = clk_audio_pll_frac_compute_frac(rate, parent_rate, &nd, &fracr); + if (res) + return res; + + frac->nd = nd; + frac->fracr = fracr; + + return TEE_SUCCESS; +} + +static TEE_Result clk_audio_pll_pad_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate) +{ + struct clk_audio_pad *apad_ck = clk->priv; + uint8_t tmp_div = 1; + + if (!rate) + return TEE_ERROR_BAD_PARAMETERS; + + tmp_div = parent_rate / rate; + if (tmp_div % 3 == 0) { + apad_ck->qdaudio = tmp_div / 3; + apad_ck->div = 3; + } else { + apad_ck->qdaudio = tmp_div / 2; + apad_ck->div = 2; + } + + return TEE_SUCCESS; +} + +static TEE_Result clk_audio_pll_pmc_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate) +{ + struct clk_audio_pmc *apmc_ck = clk->priv; + + if (!rate) + return TEE_ERROR_BAD_PARAMETERS; + + apmc_ck->qdpmc = parent_rate / rate - 1; + + return TEE_SUCCESS; +} + +static const struct clk_ops audio_pll_frac_ops = { + .enable = clk_audio_pll_frac_enable, + .disable = clk_audio_pll_frac_disable, + .get_rate = clk_audio_pll_frac_get_rate, + .set_rate = clk_audio_pll_frac_set_rate, +}; + +static const struct clk_ops audio_pll_pad_ops = { + .enable = clk_audio_pll_pad_enable, + .disable = clk_audio_pll_pad_disable, + .get_rate = clk_audio_pll_pad_get_rate, + .set_rate = clk_audio_pll_pad_set_rate, +}; + +static const struct clk_ops audio_pll_pmc_ops = { + .enable = clk_audio_pll_pmc_enable, + .disable = clk_audio_pll_pmc_disable, + .get_rate = clk_audio_pll_pmc_get_rate, + .set_rate = clk_audio_pll_pmc_set_rate, +}; + +struct clk * +at91_clk_register_audio_pll_frac(struct pmc_data *pmc, const char *name, + struct clk *parent) +{ + struct clk_audio_frac *frac_ck = NULL; + struct clk *clk = NULL; + + clk = clk_alloc(name, &audio_pll_frac_ops, &parent, 1); + if (!clk) + return NULL; + + frac_ck = calloc(1, sizeof(*frac_ck)); + if (!frac_ck) { + clk_free(clk); + return NULL; + } + + clk->flags = CLK_SET_RATE_GATE; + + frac_ck->base = pmc->base; + + clk->priv = frac_ck; + if (clk_register(clk)) { + clk_free(clk); + free(frac_ck); + return NULL; + } + + return clk; +} + +struct clk * +at91_clk_register_audio_pll_pad(struct pmc_data *pmc, const char *name, + struct clk *parent) +{ + struct clk_audio_pad *apad_ck = NULL; + struct clk *clk = NULL; + + clk = clk_alloc(name, &audio_pll_pad_ops, &parent, 1); + if (!clk) + return NULL; + + apad_ck = calloc(1, sizeof(*apad_ck)); + if (!apad_ck) { + clk_free(clk); + return NULL; + } + + clk->flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE; + + apad_ck->base = pmc->base; + + clk->priv = apad_ck; + if (clk_register(clk)) { + clk_free(clk); + free(apad_ck); + return NULL; + } + + return clk; +} + +struct clk * +at91_clk_register_audio_pll_pmc(struct pmc_data *pmc, const char *name, + struct clk *parent) +{ + struct clk_audio_pmc *apmc_ck = NULL; + struct clk *clk = NULL; + + clk = clk_alloc(name, &audio_pll_pmc_ops, &parent, 1); + if (!clk) + return NULL; + + apmc_ck = calloc(1, sizeof(*apmc_ck)); + if (!apmc_ck) { + clk_free(clk); + return NULL; + } + + clk->flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE; + + apmc_ck->base = pmc->base; + + clk->priv = apmc_ck; + + if (clk_register(clk)) { + clk_free(clk); + free(apmc_ck); + return NULL; + } + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_clk.h b/optee_os/core/drivers/clk/sam/at91_clk.h new file mode 100644 index 0000000..98ff77e --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_clk.h @@ -0,0 +1,243 @@ +/* SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause */ +/* + * include/linux/clk/at91_pmc.h + * + * Copyright (C) 2005 Ivan Kokshaysky + * Copyright (C) SAN People + * Copyright (C) 2021 Microchip + * + * Power Management Controller (PMC) - System peripherals registers. + * Based on AT91RM9200 datasheet revision E. + */ + +#ifndef AT91_CLK_H +#define AT91_CLK_H + +#include +#include + +#include "at91_pmc.h" + +#define ffs(x) __builtin_ffs(x) + +#define field_get(_mask, _reg) \ + ({ \ + typeof(_mask) __mask = _mask; \ + \ + (((_reg) & (__mask)) >> (ffs(__mask) - 1)); \ + }) +#define field_prep(_mask, _val) \ + ({ \ + typeof(_mask) __mask = _mask; \ + \ + (((_val) << (ffs(__mask) - 1)) & (__mask)); \ + }) + +struct clk_range { + unsigned long min; + unsigned long max; +}; + +#define CLK_RANGE(MIN, MAX) {.min = MIN, .max = MAX,} + +struct pmc_clk { + struct clk *clk; + uint8_t id; +}; + +struct pmc_data { + vaddr_t base; + unsigned int ncore; + struct pmc_clk *chws; + unsigned int nsystem; + struct pmc_clk *shws; + unsigned int nperiph; + struct pmc_clk *phws; + unsigned int ngck; + struct pmc_clk *ghws; + unsigned int npck; + struct pmc_clk *pchws; + + struct pmc_clk hwtable[]; +}; + +/* PLL */ +struct clk_pll_layout { + uint32_t pllr_mask; + uint32_t mul_mask; + uint32_t frac_mask; + uint32_t div_mask; + uint32_t endiv_mask; + uint8_t mul_shift; + uint8_t frac_shift; + uint8_t div_shift; + uint8_t endiv_shift; +}; + +struct clk_pcr_layout { + uint32_t offset; + uint32_t cmd; + uint32_t div_mask; + uint32_t gckcss_mask; + uint32_t pid_mask; +}; + +struct clk_pll_charac { + struct clk_range input; + int num_output; + const struct clk_range *output; + uint16_t *icpll; + uint8_t *out; + uint8_t upll : 1; +}; + +extern const struct clk_pll_layout sama5d3_pll_layout; + +/* Master */ +struct clk_master_charac { + struct clk_range output; + uint32_t divisors[5]; + uint8_t have_div3_pres; +}; + +struct clk_master_layout { + uint32_t offset; + uint32_t mask; + uint8_t pres_shift; +}; + +struct clk_programmable_layout { + uint8_t pres_mask; + uint8_t pres_shift; + uint8_t css_mask; + uint8_t have_slck_mck; + uint8_t is_pres_direct; +}; + +extern const struct clk_master_layout at91sam9x5_master_layout; + +vaddr_t at91_pmc_get_base(void); + +TEE_Result at91_pmc_clk_get(unsigned int type, unsigned int idx, + struct clk **clk); + +TEE_Result pmc_clk_get(struct pmc_data *pmc, unsigned int type, + unsigned int idx, struct clk **clk); + +struct clk *at91_sckc_clk_get(void); + +struct pmc_data *pmc_data_allocate(unsigned int ncore, unsigned int nsystem, + unsigned int nperiph, unsigned int ngck, + unsigned int npck); + +TEE_Result clk_dt_pmc_get(struct dt_pargs *args, void *data, struct clk **clk); + +struct clk *pmc_clk_get_by_name(struct pmc_clk *clks, unsigned int nclk, + const char *name); + +/* Main clock */ +struct clk *pmc_register_main_rc_osc(struct pmc_data *pmc, const char *name, + unsigned long freq); + +struct clk *pmc_register_main_osc(struct pmc_data *pmc, const char *name, + struct clk *parent, bool bypass); + +struct clk *at91_clk_register_sam9x5_main(struct pmc_data *pmc, + const char *name, + struct clk **parent_clocks, + unsigned int num_parents); + +/* PLL */ +struct clk * +at91_clk_register_pll(struct pmc_data *pmc, const char *name, + struct clk *parent, uint8_t id, + const struct clk_pll_layout *layout, + const struct clk_pll_charac *charac); + +struct clk * +at91_clk_register_plldiv(struct pmc_data *pmc, const char *name, + struct clk *parent); + +/* UTMI */ +struct clk * +at91_clk_register_utmi(struct pmc_data *pmc, const char *name, + struct clk *parent); + +/* Master */ +struct clk * +at91_clk_register_master_pres(struct pmc_data *pmc, + const char *name, int num_parents, + struct clk **parents, + const struct clk_master_layout *layout, + const struct clk_master_charac *charac, + int chg_pid); + +struct clk * +at91_clk_register_master_div(struct pmc_data *pmc, + const char *name, struct clk *parent, + const struct clk_master_layout *layout, + const struct clk_master_charac *charac); + +/* H32MX */ +struct clk * +at91_clk_register_h32mx(struct pmc_data *pmc, const char *name, + struct clk *parent); + +/* USB */ +struct clk * +at91sam9x5_clk_register_usb(struct pmc_data *pmc, const char *name, + struct clk **parents, uint8_t num_parents); + +/* Programmable */ +struct clk * +at91_clk_register_programmable(struct pmc_data *pmc, + const char *name, struct clk **parents, + uint8_t num_parents, uint8_t id, + const struct clk_programmable_layout *layout); + +struct clk * +at91_clk_register_system(struct pmc_data *pmc, const char *name, + struct clk *parent, uint8_t id); + +struct clk * +at91_clk_register_sam9x5_periph(struct pmc_data *pmc, + const struct clk_pcr_layout *layout, + const char *name, struct clk *parent, + uint32_t id, const struct clk_range *range); + +struct clk * +at91_clk_register_generated(struct pmc_data *pmc, + const struct clk_pcr_layout *layout, + const char *name, struct clk **parents, + uint8_t num_parents, uint8_t id, + const struct clk_range *range, + int chg_pid); + +struct clk * +at91_clk_i2s_mux_register(const char *name, struct clk **parents, + unsigned int num_parents, uint8_t bus_id); + +/* Audio PLL */ +struct clk * +at91_clk_register_audio_pll_frac(struct pmc_data *pmc, const char *name, + struct clk *parent); + +struct clk * +at91_clk_register_audio_pll_pad(struct pmc_data *pmc, const char *name, + struct clk *parent); + +struct clk * +at91_clk_register_audio_pll_pmc(struct pmc_data *pmc, const char *name, + struct clk *parent); + +#ifdef CFG_PM_ARM32 +void pmc_register_id(uint8_t id); +void pmc_register_pck(uint8_t pck); +void pmc_register_pm(void); +#else +static inline void pmc_register_id(uint8_t id __unused) {} +static inline void pmc_register_pck(uint8_t pck __unused) {} +static inline void pmc_register_pm(void) {} +#endif + +#endif diff --git a/optee_os/core/drivers/clk/sam/at91_generated.c b/optee_os/core/drivers/clk/sam/at91_generated.c new file mode 100644 index 0000000..3c4c2af --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_generated.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2015 - 2021 Atmel Corporation, + * Nicolas Ferre + * + * Based on clk-programmable & clk-peripheral drivers by Boris BREZILLON. + */ + +#include +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define GENERATED_MAX_DIV 255 + +struct clk_generated { + vaddr_t base; + struct clk_range range; + uint32_t *mux_table; + uint32_t id; + uint32_t gckdiv; + const struct clk_pcr_layout *layout; + uint8_t parent_id; + int chg_pid; +}; + +static TEE_Result clk_generated_enable(struct clk *clk) +{ + struct clk_generated *gck = clk->priv; + + io_write32(gck->base + gck->layout->offset, + (gck->id & gck->layout->pid_mask)); + io_clrsetbits32(gck->base + gck->layout->offset, + AT91_PMC_PCR_GCKDIV_MASK | gck->layout->gckcss_mask | + gck->layout->cmd | AT91_PMC_PCR_GCKEN, + field_prep(gck->layout->gckcss_mask, gck->parent_id) | + gck->layout->cmd | + ((gck->gckdiv << AT91_PMC_PCR_GCKDIV_SHIFT) & + AT91_PMC_PCR_GCKDIV_MASK) | + AT91_PMC_PCR_GCKEN); + + return TEE_SUCCESS; +} + +static void clk_generated_disable(struct clk *clk) +{ + struct clk_generated *gck = clk->priv; + + io_write32(gck->base + gck->layout->offset, + gck->id & gck->layout->pid_mask); + io_clrsetbits32(gck->base + gck->layout->offset, AT91_PMC_PCR_GCKEN, + gck->layout->cmd); +} + +static unsigned long +clk_generated_get_rate(struct clk *clk, unsigned long parent_rate) +{ + struct clk_generated *gck = clk->priv; + + return UDIV_ROUND_NEAREST(parent_rate, gck->gckdiv + 1); +} + +/* No modification of hardware as we have the flag CLK_SET_PARENT_GATE set */ +static TEE_Result clk_generated_set_parent(struct clk *clk, size_t index) +{ + struct clk_generated *gck = clk->priv; + + if (index >= clk_get_num_parents(clk)) + return TEE_ERROR_BAD_PARAMETERS; + + gck->parent_id = index; + + return TEE_SUCCESS; +} + +static size_t clk_generated_get_parent(struct clk *clk) +{ + struct clk_generated *gck = clk->priv; + + return gck->parent_id; +} + +/* No modification of hardware as we have the flag CLK_SET_RATE_GATE set */ +static TEE_Result clk_generated_set_rate(struct clk *clk, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_generated *gck = clk->priv; + uint32_t div = 1; + + if (!rate) + return TEE_ERROR_BAD_PARAMETERS; + + if (gck->range.max && rate > gck->range.max) + return TEE_ERROR_BAD_PARAMETERS; + + div = UDIV_ROUND_NEAREST(parent_rate, rate); + if (div > GENERATED_MAX_DIV + 1 || !div) + return TEE_ERROR_GENERIC; + + gck->gckdiv = div - 1; + return TEE_SUCCESS; +} + +static const struct clk_ops generated_ops = { + .enable = clk_generated_enable, + .disable = clk_generated_disable, + .get_rate = clk_generated_get_rate, + .get_parent = clk_generated_get_parent, + .set_parent = clk_generated_set_parent, + .set_rate = clk_generated_set_rate, +}; + +/** + * clk_generated_startup - Initialize a given clock to its default parent and + * divisor parameter. + * + * @gck: Generated clock to set the startup parameters for. + * + * Take parameters from the hardware and update local clock configuration + * accordingly. + */ +static void clk_generated_startup(struct clk_generated *gck) +{ + uint32_t tmp = 0; + + io_write32(gck->base + gck->layout->offset, + (gck->id & gck->layout->pid_mask)); + tmp = io_read32(gck->base + gck->layout->offset); + + gck->parent_id = field_get(gck->layout->gckcss_mask, tmp); + gck->gckdiv = (tmp & AT91_PMC_PCR_GCKDIV_MASK) >> + AT91_PMC_PCR_GCKDIV_SHIFT; +} + +struct clk * +at91_clk_register_generated(struct pmc_data *pmc, + const struct clk_pcr_layout *layout, + const char *name, struct clk **parents, + uint8_t num_parents, uint8_t id, + const struct clk_range *range, + int chg_pid) +{ + struct clk_generated *gck = NULL; + struct clk *clk = NULL; + + clk = clk_alloc(name, &generated_ops, parents, num_parents); + if (!clk) + return NULL; + + gck = calloc(1, sizeof(*gck)); + if (!gck) { + clk_free(clk); + return NULL; + } + + clk->flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE; + + gck->id = id; + gck->base = pmc->base; + memcpy(&gck->range, range, sizeof(gck->range)); + gck->chg_pid = chg_pid; + gck->layout = layout; + + clk->priv = gck; + + clk_generated_startup(gck); + + if (clk_register(clk)) { + clk_free(clk); + free(gck); + return NULL; + } + pmc_register_id(id); + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_h32mx.c b/optee_os/core/drivers/clk/sam/at91_h32mx.c new file mode 100644 index 0000000..404aa4f --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_h32mx.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2014 Atmel + * + * Alexandre Belloni + */ + +#include +#include +#include +#include + +#include "at91_clk.h" + +#define H32MX_MAX_FREQ 90000000 + +static unsigned long clk_sama5d4_h32mx_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct pmc_data *pmc = clk->priv; + unsigned int mckr = io_read32(pmc->base + AT91_PMC_MCKR); + + if (mckr & AT91_PMC_H32MXDIV) + return parent_rate / 2; + + if (parent_rate > H32MX_MAX_FREQ) + IMSG("H32MX clock is too fast"); + + return parent_rate; +} + +static TEE_Result clk_sama5d4_h32mx_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate) +{ + struct pmc_data *pmc = clk->priv; + uint32_t mckr = 0; + + if (parent_rate != rate && (parent_rate / 2) != rate) + return TEE_ERROR_BAD_PARAMETERS; + + if ((parent_rate / 2) == rate) + mckr = AT91_PMC_H32MXDIV; + + io_clrsetbits32(pmc->base + AT91_PMC_MCKR, AT91_PMC_H32MXDIV, mckr); + + return TEE_SUCCESS; +} + +static const struct clk_ops h32mx_ops = { + .get_rate = clk_sama5d4_h32mx_get_rate, + .set_rate = clk_sama5d4_h32mx_set_rate, +}; + +struct clk * +at91_clk_register_h32mx(struct pmc_data *pmc, const char *name, + struct clk *parent) +{ + struct clk *clk = NULL; + + clk = clk_alloc(name, &h32mx_ops, &parent, 1); + if (!clk) + return NULL; + + clk->ops = &h32mx_ops; + clk->priv = pmc; + clk->name = name; + clk->flags = CLK_SET_RATE_GATE; + + if (clk_register(clk)) { + clk_free(clk); + return NULL; + } + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_i2s_mux.c b/optee_os/core/drivers/clk/sam/at91_i2s_mux.c new file mode 100644 index 0000000..6d767bd --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_i2s_mux.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2018 Microchip Technology Inc, + * Codrin Ciubotariu + */ + +#include +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +struct clk_i2s_mux { + vaddr_t sfr_base; + uint8_t bus_id; +}; + +static size_t clk_i2s_mux_get_parent(struct clk *clk) +{ + struct clk_i2s_mux *mux = clk->priv; + uint32_t val = io_read32(mux->sfr_base + AT91_SFR_I2SCLKSEL); + + return (val & BIT(mux->bus_id)) >> mux->bus_id; +} + +static TEE_Result clk_i2s_mux_set_parent(struct clk *clk, size_t index) +{ + struct clk_i2s_mux *mux = clk->priv; + + io_clrsetbits32(mux->sfr_base + AT91_SFR_I2SCLKSEL, + BIT(mux->bus_id), index << mux->bus_id); + + return TEE_SUCCESS; +} + +static const struct clk_ops clk_i2s_mux_ops = { + .get_parent = clk_i2s_mux_get_parent, + .set_parent = clk_i2s_mux_set_parent, +}; + +struct clk * +at91_clk_i2s_mux_register(const char *name, struct clk **parents, + unsigned int num_parents, uint8_t bus_id) +{ + struct clk_i2s_mux *i2s_ck = NULL; + struct clk *clk = NULL; + + clk = clk_alloc(name, &clk_i2s_mux_ops, parents, num_parents); + if (!clk) + return NULL; + + i2s_ck = calloc(1, sizeof(*i2s_ck)); + if (!i2s_ck) { + clk_free(clk); + return NULL; + } + + i2s_ck->bus_id = bus_id; + i2s_ck->sfr_base = sam_sfr_base(); + + clk->priv = i2s_ck; + + if (clk_register(clk)) { + clk_free(clk); + free(i2s_ck); + return NULL; + } + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_main.c b/optee_os/core/drivers/clk/sam/at91_main.c new file mode 100644 index 0000000..0f4229d --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_main.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define SLOW_CLOCK_FREQ 32768 +#define MAINF_DIV 16 +#define USEC_PER_SEC 1000000L +#define MAINF_LOOP_MIN_WAIT (USEC_PER_SEC / SLOW_CLOCK_FREQ) + +#define OSC_READY_TIMEOUT_US 1000 + +#define MOR_KEY_MASK (0xFF << 16) + +#define CLK_MAIN_PARENT_SELECT(s) (((s) & \ + (AT91_PMC_MOSCEN | \ + AT91_PMC_OSCBYPASS)) ? 1 : 0) + +/* + * Main RC Oscillator + */ + +struct main_rc_osc { + unsigned long freq; + vaddr_t base; +}; + +static bool pmc_main_rc_osc_ready(struct main_rc_osc *osc) +{ + uint32_t status = io_read32(osc->base + AT91_PMC_SR); + + return status & AT91_PMC_MOSCRCS; +} + +static TEE_Result pmc_main_rc_osc_enable(struct clk *clk) +{ + struct main_rc_osc *osc = clk->priv; + uint32_t mor = io_read32(osc->base + AT91_CKGR_MOR); + + /* Enable the oscillator if not */ + if (!(mor & AT91_PMC_MOSCRCEN)) { + io_clrsetbits32(osc->base + AT91_CKGR_MOR, + MOR_KEY_MASK | AT91_PMC_MOSCRCEN, + AT91_PMC_MOSCRCEN | AT91_PMC_KEY); + } + + while (!pmc_main_rc_osc_ready(osc)) + ; + + return TEE_SUCCESS; +} + +static void pmc_main_rc_osc_disable(struct clk *clk) +{ + struct main_rc_osc *osc = clk->priv; + uint32_t mor = io_read32(osc->base + AT91_CKGR_MOR); + + if (!(mor & AT91_PMC_MOSCRCEN)) + return; + + io_clrsetbits32(osc->base + AT91_CKGR_MOR, + MOR_KEY_MASK | AT91_PMC_MOSCRCEN, AT91_PMC_KEY); +} + +static unsigned long +pmc_main_rc_osc_get_rate(struct clk *clk, unsigned long parent_rate __unused) +{ + struct main_rc_osc *osc = clk->priv; + + return osc->freq; +} + +static const struct clk_ops pmc_main_rc_osc_clk_ops = { + .enable = pmc_main_rc_osc_enable, + .disable = pmc_main_rc_osc_disable, + .get_rate = pmc_main_rc_osc_get_rate, +}; + +struct clk *pmc_register_main_rc_osc(struct pmc_data *pmc, const char *name, + unsigned long freq) +{ + struct clk *clk = NULL; + struct main_rc_osc *osc = NULL; + + clk = clk_alloc(name, &pmc_main_rc_osc_clk_ops, NULL, 0); + if (!clk) + return NULL; + + osc = calloc(1, sizeof(*osc)); + if (!osc) { + clk_free(clk); + return NULL; + } + + osc->freq = freq; + osc->base = pmc->base; + + clk->priv = osc; + + if (clk_register(clk)) { + free(osc); + clk_free(clk); + return NULL; + } + + return clk; +} + +/* + * Main Oscillator + */ +static bool pmc_main_osc_ready(struct pmc_data *pmc) +{ + uint32_t status = io_read32(pmc->base + AT91_PMC_SR); + + return status & AT91_PMC_MOSCS; +} + +static TEE_Result pmc_main_osc_enable(struct clk *clk) +{ + struct pmc_data *pmc = clk->priv; + uint32_t mor = io_read32(pmc->base + AT91_CKGR_MOR); + + mor &= ~MOR_KEY_MASK; + + if (mor & AT91_PMC_OSCBYPASS) + return TEE_SUCCESS; + + if (!(mor & AT91_PMC_MOSCEN)) { + mor |= AT91_PMC_MOSCEN | AT91_PMC_KEY; + io_write32(pmc->base + AT91_CKGR_MOR, mor); + } + + while (!pmc_main_osc_ready(pmc)) + ; + + return TEE_SUCCESS; +} + +static void pmc_main_osc_disable(struct clk *clk) +{ + struct pmc_data *pmc = clk->priv; + uint32_t mor = io_read32(pmc->base + AT91_CKGR_MOR); + + if (mor & AT91_PMC_OSCBYPASS) + return; + + if (!(mor & AT91_PMC_MOSCEN)) + return; + + mor &= ~(AT91_PMC_KEY | AT91_PMC_MOSCEN); + io_write32(pmc->base + AT91_CKGR_MOR, mor | AT91_PMC_KEY); +} + +static const struct clk_ops pmc_main_osc_clk_ops = { + .enable = pmc_main_osc_enable, + .disable = pmc_main_osc_disable, +}; + +struct clk *pmc_register_main_osc(struct pmc_data *pmc, const char *name, + struct clk *parent, bool bypass) +{ + struct clk *clk = NULL; + + clk = clk_alloc(name, &pmc_main_osc_clk_ops, &parent, 1); + if (!clk) + panic(); + + clk->priv = pmc; + + if (bypass) + io_clrsetbits32(pmc->base + AT91_CKGR_MOR, + MOR_KEY_MASK | AT91_PMC_OSCBYPASS, + AT91_PMC_OSCBYPASS | AT91_PMC_KEY); + + if (clk_register(clk)) { + clk_free(clk); + return NULL; + } + + return clk; +} + +/* + * Main Clock + */ +static TEE_Result clk_main_probe_frequency(vaddr_t base) +{ + while (!(io_read32(base + AT91_CKGR_MCFR) & AT91_PMC_MAINRDY)) + ; + + return TEE_SUCCESS; +} + +static unsigned long clk_main_get_rate(vaddr_t base, + unsigned long parent_rate) +{ + uint32_t mcfr = 0; + + if (parent_rate) + return parent_rate; + + IMSG("Main crystal frequency not set, using approximate value"); + mcfr = io_read32(base + AT91_CKGR_MCFR); + if (!(mcfr & AT91_PMC_MAINRDY)) + return 0; + + return ((mcfr & AT91_PMC_MAINF) * SLOW_CLOCK_FREQ) / MAINF_DIV; +} + +static bool clk_sam9x5_main_ready(vaddr_t base) +{ + uint32_t status = io_read32(base + AT91_PMC_SR); + + return status & AT91_PMC_MOSCSELS; +} + +static TEE_Result clk_sam9x5_main_enable(struct clk *clk) +{ + struct pmc_data *pmc = clk->priv; + + while (!clk_sam9x5_main_ready(pmc->base)) + ; + + return clk_main_probe_frequency(pmc->base); +} + +static unsigned long clk_sam9x5_main_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct pmc_data *pmc = clk->priv; + + return clk_main_get_rate(pmc->base, parent_rate); +} + +static TEE_Result clk_sam9x5_main_set_parent(struct clk *clk, size_t index) +{ + struct pmc_data *pmc = clk->priv; + uint32_t tmp = 0; + + if (index > 1) + return TEE_ERROR_BAD_PARAMETERS; + + tmp = io_read32(pmc->base + AT91_CKGR_MOR); + + if (index && !(tmp & AT91_PMC_MOSCSEL)) + tmp = AT91_PMC_MOSCSEL; + else if (!index && (tmp & AT91_PMC_MOSCSEL)) + tmp = 0; + else + return TEE_SUCCESS; + + io_clrsetbits32(pmc->base + AT91_CKGR_MOR, + AT91_PMC_MOSCSEL | MOR_KEY_MASK, + tmp | AT91_PMC_KEY); + + while (!clk_sam9x5_main_ready(pmc->base)) + ; + + return TEE_SUCCESS; +} + +static size_t clk_sam9x5_main_get_parent(struct clk *clk) +{ + struct pmc_data *pmc = clk->priv; + uint32_t status = io_read32(pmc->base + AT91_CKGR_MOR); + + return CLK_MAIN_PARENT_SELECT(status); +} + +static const struct clk_ops sam9x5_main_ops = { + .enable = clk_sam9x5_main_enable, + .get_rate = clk_sam9x5_main_get_rate, + .set_parent = clk_sam9x5_main_set_parent, + .get_parent = clk_sam9x5_main_get_parent, +}; + +struct clk * +at91_clk_register_sam9x5_main(struct pmc_data *pmc, + const char *name, + struct clk **parent_clocks, + unsigned int num_parents) +{ + struct clk *clk = NULL; + + if (!name) + return NULL; + + if (!parent_clocks || !num_parents) + return NULL; + + clk = clk_alloc(name, &sam9x5_main_ops, parent_clocks, num_parents); + if (!clk) + return NULL; + + clk->flags = CLK_SET_PARENT_GATE; + clk->priv = pmc; + + if (clk_register(clk)) { + clk_free(clk); + return NULL; + } + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_master.c b/optee_os/core/drivers/clk/sam/at91_master.c new file mode 100644 index 0000000..1780d48 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_master.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include + +#include "at91_clk.h" + +#define MASTER_PRES_MASK 0x7 +#define MASTER_PRES_MAX MASTER_PRES_MASK +#define MASTER_DIV_SHIFT 8 +#define MASTER_DIV_MASK 0x7 + +struct clk_master { + vaddr_t base; + const struct clk_master_layout *layout; + const struct clk_master_charac *charac; + uint32_t *mux_table; + uint32_t mckr; + int chg_pid; + uint8_t div; +}; + +static bool clk_master_ready(struct clk_master *master) +{ + uint32_t status = io_read32(master->base + AT91_PMC_SR); + + return status & AT91_PMC_MCKRDY; +} + +static TEE_Result clk_master_enable(struct clk *clk) +{ + struct clk_master *master = clk->priv; + + while (!clk_master_ready(master)) + ; + + return TEE_SUCCESS; +} + +static unsigned long clk_master_div_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + uint8_t div = 1; + uint32_t mckr = 0; + unsigned long rate = parent_rate; + struct clk_master *master = clk->priv; + const struct clk_master_layout *layout = master->layout; + const struct clk_master_charac *charac = master->charac; + + mckr = io_read32(master->base + master->layout->offset); + + mckr &= layout->mask; + + div = (mckr >> MASTER_DIV_SHIFT) & MASTER_DIV_MASK; + + rate /= charac->divisors[div]; + + if (rate < charac->output.min) + IMSG("master clk div is underclocked"); + else if (rate > charac->output.max) + IMSG("master clk div is overclocked"); + + return rate; +} + +static const struct clk_ops master_div_ops = { + .enable = clk_master_enable, + .get_rate = clk_master_div_get_rate, +}; + +static unsigned long clk_master_pres_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_master *master = clk->priv; + const struct clk_master_charac *charac = master->charac; + uint32_t val = 0; + unsigned int pres = 0; + + val = io_read32(master->base + master->layout->offset); + + pres = (val >> master->layout->pres_shift) & MASTER_PRES_MASK; + if (pres != 3 || !charac->have_div3_pres) + pres = BIT(pres); + + return UDIV_ROUND_NEAREST(parent_rate, pres); +} + +static size_t clk_master_pres_get_parent(struct clk *clk) +{ + struct clk_master *master = clk->priv; + uint32_t mckr = 0; + + mckr = io_read32(master->base + master->layout->offset); + + return mckr & AT91_PMC_CSS; +} + +static const struct clk_ops master_pres_ops = { + .enable = clk_master_enable, + .get_rate = clk_master_pres_get_rate, + .get_parent = clk_master_pres_get_parent, +}; + +static struct clk * +at91_clk_register_master_internal(struct pmc_data *pmc, + const char *name, int num_parents, + struct clk **parents, + const struct clk_master_layout *layout, + const struct clk_master_charac *charac, + const struct clk_ops *ops, int chg_pid) +{ + struct clk_master *master = NULL; + struct clk *clk = NULL; + + if (!name || !num_parents || !parents) + return NULL; + + clk = clk_alloc(name, ops, parents, num_parents); + if (!clk) + return NULL; + + master = calloc(1, sizeof(*master)); + if (!master) { + clk_free(clk); + return NULL; + } + + master->layout = layout; + master->charac = charac; + master->base = pmc->base; + master->chg_pid = chg_pid; + + clk->priv = master; + clk->flags = CLK_SET_RATE_GATE; + + if (clk_register(clk)) { + clk_free(clk); + free(master); + return NULL; + } + + return clk; +} + +struct clk * +at91_clk_register_master_pres(struct pmc_data *pmc, + const char *name, int num_parents, + struct clk **parents, + const struct clk_master_layout *layout, + const struct clk_master_charac *charac, + int chg_pid) +{ + return at91_clk_register_master_internal(pmc, name, num_parents, + parents, layout, + charac, + &master_pres_ops, chg_pid); +} + +struct clk * +at91_clk_register_master_div(struct pmc_data *pmc, + const char *name, struct clk *parent, + const struct clk_master_layout *layout, + const struct clk_master_charac *charac) +{ + return at91_clk_register_master_internal(pmc, name, 1, + &parent, layout, + charac, + &master_div_ops, -1); +} + +const struct clk_master_layout at91sam9x5_master_layout = { + .mask = 0x373, + .pres_shift = 4, + .offset = AT91_PMC_MCKR, +}; diff --git a/optee_os/core/drivers/clk/sam/at91_peripheral.c b/optee_os/core/drivers/clk/sam/at91_peripheral.c new file mode 100644 index 0000000..7538a6c --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_peripheral.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define PERIPHERAL_ID_MIN 2 +#define PERIPHERAL_ID_MASK 31 +#define PERIPHERAL_MASK(id) BIT((id) & PERIPHERAL_ID_MASK) + +#define PERIPHERAL_MAX_SHIFT 3 + +struct clk_sam9x5_peripheral { + vaddr_t base; + struct clk_range range; + uint32_t id; + uint32_t div; + const struct clk_pcr_layout *layout; + bool auto_div; +}; + +static void clk_sam9x5_peripheral_autodiv(struct clk *clk) +{ + struct clk *parent = NULL; + struct clk_sam9x5_peripheral *periph = clk->priv; + unsigned long parent_rate = 0; + int shift = 0; + + if (!periph->auto_div) + return; + + if (periph->range.max) { + parent = clk_get_parent_by_index(clk, 0); + parent_rate = clk_get_rate(parent); + if (!parent_rate) + return; + + for (shift = 0; shift < PERIPHERAL_MAX_SHIFT; shift++) { + if (parent_rate >> shift <= periph->range.max) + break; + } + } + + periph->auto_div = false; + periph->div = shift; +} + +static TEE_Result clk_sam9x5_peripheral_enable(struct clk *clk) +{ + struct clk_sam9x5_peripheral *periph = clk->priv; + + if (periph->id < PERIPHERAL_ID_MIN) + return TEE_SUCCESS; + + io_write32(periph->base + periph->layout->offset, + (periph->id & periph->layout->pid_mask)); + io_clrsetbits32(periph->base + periph->layout->offset, + periph->layout->div_mask | periph->layout->cmd | + AT91_PMC_PCR_EN, + field_prep(periph->layout->div_mask, periph->div) | + periph->layout->cmd | + AT91_PMC_PCR_EN); + + return TEE_SUCCESS; +} + +static void clk_sam9x5_peripheral_disable(struct clk *clk) +{ + struct clk_sam9x5_peripheral *periph = clk->priv; + + if (periph->id < PERIPHERAL_ID_MIN) + return; + + io_write32(periph->base + periph->layout->offset, + (periph->id & periph->layout->pid_mask)); + io_clrsetbits32(periph->base + periph->layout->offset, + AT91_PMC_PCR_EN | periph->layout->cmd, + periph->layout->cmd); +} + +static unsigned long +clk_sam9x5_peripheral_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_sam9x5_peripheral *periph = clk->priv; + uint32_t status = 0; + + if (periph->id < PERIPHERAL_ID_MIN) + return parent_rate; + + io_write32(periph->base + periph->layout->offset, + periph->id & periph->layout->pid_mask); + status = io_read32(periph->base + periph->layout->offset); + + if (status & AT91_PMC_PCR_EN) { + periph->div = field_get(periph->layout->div_mask, status); + periph->auto_div = false; + } else { + clk_sam9x5_peripheral_autodiv(clk); + } + + return parent_rate >> periph->div; +} + +static TEE_Result clk_sam9x5_peripheral_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate) +{ + unsigned int shift = 0; + struct clk_sam9x5_peripheral *periph = clk->priv; + + if (periph->id < PERIPHERAL_ID_MIN || !periph->range.max) { + if (parent_rate == rate) + return TEE_SUCCESS; + else + return TEE_ERROR_GENERIC; + } + + if (periph->range.max && rate > periph->range.max) + return TEE_ERROR_GENERIC; + + for (shift = 0; shift <= PERIPHERAL_MAX_SHIFT; shift++) { + if (parent_rate >> shift == rate) { + periph->auto_div = false; + periph->div = shift; + return TEE_SUCCESS; + } + } + + return TEE_ERROR_GENERIC; +} + +static const struct clk_ops sam9x5_peripheral_ops = { + .enable = clk_sam9x5_peripheral_enable, + .disable = clk_sam9x5_peripheral_disable, + .get_rate = clk_sam9x5_peripheral_get_rate, + .set_rate = clk_sam9x5_peripheral_set_rate, +}; + +struct clk * +at91_clk_register_sam9x5_periph(struct pmc_data *pmc, + const struct clk_pcr_layout *layout, + const char *name, struct clk *parent, + uint32_t id, const struct clk_range *range) +{ + struct clk_sam9x5_peripheral *periph = NULL; + struct clk *clk = NULL; + + if (!name || !parent) + return NULL; + + clk = clk_alloc(name, &sam9x5_peripheral_ops, &parent, 1); + if (!clk) + return NULL; + + periph = calloc(1, sizeof(*periph)); + if (!periph) { + clk_free(clk); + return NULL; + } + + periph->id = id; + periph->div = 0; + periph->base = pmc->base; + if (layout->div_mask) + periph->auto_div = true; + periph->layout = layout; + periph->range = *range; + + clk->priv = periph; + + if (clk_register(clk)) { + clk_free(clk); + free(periph); + return 0; + } + + clk_sam9x5_peripheral_autodiv(clk); + pmc_register_id(id); + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_pll.c b/optee_os/core/drivers/clk/sam/at91_pll.c new file mode 100644 index 0000000..57e5f88 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_pll.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define PLL_STATUS_MASK(id) BIT(1 + (id)) +#define PLL_REG(id) (AT91_CKGR_PLLAR + ((id) * 4)) +#define PLL_DIV_MASK 0xff +#define PLL_DIV_MAX PLL_DIV_MASK +#define PLL_DIV(reg) ((reg) & PLL_DIV_MASK) +#define PLL_MUL(reg, layout) \ + ({ \ + typeof(layout) __layout = layout; \ + \ + (((reg) >> (__layout)->mul_shift) & (__layout)->mul_mask); \ + }) +#define PLL_MUL_MIN 2 +#define PLL_MUL_MASK(layout) ((layout)->mul_mask) +#define PLL_MUL_MAX(layout) (PLL_MUL_MASK(layout) + 1) +#define PLL_ICPR_SHIFT(id) ((id) * 16) +#define PLL_ICPR_MASK(id) (0xffff << PLL_ICPR_SHIFT(id)) +#define PLL_MAX_COUNT 0x3f +#define PLL_COUNT_SHIFT 8 +#define PLL_OUT_SHIFT 14 +#define PLL_MAX_ID 1 + +struct clk_pll { + vaddr_t base; + uint8_t id; + uint8_t div; + uint8_t range; + uint16_t mul; + const struct clk_pll_layout *layout; + const struct clk_pll_charac *charac; +}; + +static bool clk_pll_ready(vaddr_t base, int id) +{ + unsigned int status = io_read32(base + AT91_PMC_SR); + + return status & PLL_STATUS_MASK(id); +} + +static TEE_Result clk_pll_enable(struct clk *clk) +{ + struct clk_pll *pll = clk->priv; + const struct clk_pll_layout *layout = pll->layout; + const struct clk_pll_charac *charac = pll->charac; + uint8_t id = pll->id; + uint32_t mask = PLL_STATUS_MASK(id); + int offset = PLL_REG(id); + uint8_t out = 0; + unsigned int pllr = 0; + unsigned int status = 0; + uint8_t div = 0; + uint16_t mul = 0; + + pllr = io_read32(pll->base + offset); + div = PLL_DIV(pllr); + mul = PLL_MUL(pllr, layout); + + status = io_read32(pll->base + AT91_PMC_SR); + if ((status & mask) && + (div == pll->div && mul == pll->mul)) + return TEE_SUCCESS; + + if (charac->out) + out = charac->out[pll->range]; + + if (charac->icpll) + io_clrsetbits32(pll->base + AT91_PMC_PLLICPR, PLL_ICPR_MASK(id), + charac->icpll[pll->range] << + PLL_ICPR_SHIFT(id)); + + io_clrsetbits32(pll->base + offset, layout->pllr_mask, + pll->div | (PLL_MAX_COUNT << PLL_COUNT_SHIFT) | + (out << PLL_OUT_SHIFT) | + ((pll->mul & layout->mul_mask) << layout->mul_shift)); + + while (!clk_pll_ready(pll->base, pll->id)) + ; + + return TEE_SUCCESS; +} + +static void clk_pll_disable(struct clk *clk) +{ + struct clk_pll *pll = clk->priv; + unsigned int mask = pll->layout->pllr_mask; + + io_clrsetbits32(pll->base + PLL_REG(pll->id), mask, ~mask); +} + +static unsigned long clk_pll_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_pll *pll = clk->priv; + + if (!pll->div || !pll->mul) + return 0; + + return (parent_rate / pll->div) * (pll->mul + 1); +} + +static long clk_pll_get_best_div_mul(struct clk_pll *pll, unsigned long rate, + unsigned long parent_rate, + uint32_t *div, uint32_t *mul, + uint32_t *index) +{ + const struct clk_pll_layout *layout = pll->layout; + const struct clk_pll_charac *charac = pll->charac; + unsigned long bestremainder = ULONG_MAX; + unsigned long maxdiv = 1; + unsigned long mindiv = 1; + unsigned long tmpdiv = 1; + long bestrate = -1; + unsigned long bestdiv = 0; + unsigned long bestmul = 0; + int i = 0; + + /* Check if parent_rate is a valid input rate */ + if (parent_rate < charac->input.min) + return -1; + + /* + * Calculate minimum divider based on the minimum multiplier, the + * parent_rate and the requested rate. + * Should always be 2 according to the input and output charac + * of the PLL blocks. + */ + mindiv = (parent_rate * PLL_MUL_MIN) / rate; + if (!mindiv) + mindiv = 1; + + if (parent_rate > charac->input.max) { + tmpdiv = DIV_ROUND_UP(parent_rate, charac->input.max); + if (tmpdiv > PLL_DIV_MAX) + return -1; + + if (tmpdiv > mindiv) + mindiv = tmpdiv; + } + + /* + * Calculate the maximum divider which is limited by PLL register + * layout (limited by the MUL or DIV field size). + */ + maxdiv = DIV_ROUND_UP(parent_rate * PLL_MUL_MAX(layout), rate); + if (maxdiv > PLL_DIV_MAX) + maxdiv = PLL_DIV_MAX; + + /* + * Iterate over the acceptable divider values to find the best + * divider/multiplier pair (the one that generates the closest + * rate to the requested one). + */ + for (tmpdiv = mindiv; tmpdiv <= maxdiv; tmpdiv++) { + unsigned long remainder = 0; + unsigned long tmprate = 0; + unsigned long tmpmul = 0; + + /* + * Calculate the multiplier associated with the current + * divider that provide the closest rate to the requested one. + */ + tmpmul = UDIV_ROUND_NEAREST(rate, parent_rate / tmpdiv); + tmprate = (parent_rate / tmpdiv) * tmpmul; + if (tmprate > rate) + remainder = tmprate - rate; + else + remainder = rate - tmprate; + + /* + * Compare the remainder with the best remainder found until + * now and elect a new best multiplier/divider pair if the + * current remainder is smaller than the best one. + */ + if (remainder < bestremainder) { + bestremainder = remainder; + bestdiv = tmpdiv; + bestmul = tmpmul; + bestrate = tmprate; + } + + /* + * We've found a perfect match! + * Stop searching now and use this multiplier/divider pair. + */ + if (!remainder) + break; + } + + /* We haven't found any multiplier/divider pair => return -ERANGE */ + if (bestrate < 0) + return bestrate; + + /* Check if bestrate is a valid output rate */ + for (i = 0; i < charac->num_output; i++) { + if (bestrate >= (long)charac->output[i].min && + bestrate <= (long)charac->output[i].max) + break; + } + + if (i >= charac->num_output) + return -1; + + if (div) + *div = bestdiv; + if (mul) + *mul = bestmul - 1; + if (index) + *index = i; + + return bestrate; +} + +static TEE_Result clk_pll_set_rate(struct clk *clk, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_pll *pll = clk->priv; + long ret = -1; + uint32_t div = 1; + uint32_t mul = 0; + uint32_t index = 0; + + ret = clk_pll_get_best_div_mul(pll, rate, parent_rate, + &div, &mul, &index); + if (ret < 0) + return TEE_ERROR_BAD_PARAMETERS; + + pll->range = index; + pll->div = div; + pll->mul = mul; + + return TEE_SUCCESS; +} + +static const struct clk_ops pll_ops = { + .enable = clk_pll_enable, + .disable = clk_pll_disable, + .get_rate = clk_pll_get_rate, + .set_rate = clk_pll_set_rate, +}; + +struct clk * +at91_clk_register_pll(struct pmc_data *pmc, const char *name, + struct clk *parent, uint8_t id, + const struct clk_pll_layout *layout, + const struct clk_pll_charac *charac) +{ + struct clk *clk = NULL; + struct clk_pll *pll = NULL; + int offset = PLL_REG(id); + uint32_t pllr = 0; + + if (!name || !parent) + return NULL; + + clk = clk_alloc(name, &pll_ops, &parent, 1); + if (!clk) + return NULL; + + if (id > PLL_MAX_ID) + return NULL; + + pll = calloc(1, sizeof(*pll)); + if (!pll) { + clk_free(clk); + return NULL; + } + + pll->id = id; + pll->layout = layout; + pll->charac = charac; + pll->base = pmc->base; + pllr = io_read32(pmc->base + offset); + pll->div = PLL_DIV(pllr); + pll->mul = PLL_MUL(pllr, layout); + + clk->flags = CLK_SET_RATE_GATE; + clk->priv = pll; + + if (clk_register(clk)) { + clk_free(clk); + free(pll); + return NULL; + } + + return clk; +} + +const struct clk_pll_layout sama5d3_pll_layout = { + .pllr_mask = 0x1FFFFFF, + .mul_shift = 18, + .mul_mask = 0x7F, +}; diff --git a/optee_os/core/drivers/clk/sam/at91_plldiv.c b/optee_os/core/drivers/clk/sam/at91_plldiv.c new file mode 100644 index 0000000..eca14ec --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_plldiv.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +static unsigned long clk_plldiv_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct pmc_data *pmc = clk->priv; + unsigned int mckr = io_read32(pmc->base + AT91_PMC_MCKR); + + if (mckr & AT91_PMC_PLLADIV2) + return parent_rate / 2; + + return parent_rate; +} + +static TEE_Result clk_plldiv_set_rate(struct clk *clk, unsigned long rate, + unsigned long parent_rate) +{ + struct pmc_data *pmc = clk->priv; + + if (parent_rate != rate && (parent_rate / 2 != rate)) + return TEE_ERROR_GENERIC; + + io_clrsetbits32(pmc->base + AT91_PMC_MCKR, AT91_PMC_PLLADIV2, + parent_rate != rate ? AT91_PMC_PLLADIV2 : 0); + + return TEE_SUCCESS; +} + +static const struct clk_ops plldiv_ops = { + .get_rate = clk_plldiv_get_rate, + .set_rate = clk_plldiv_set_rate, +}; + +struct clk * +at91_clk_register_plldiv(struct pmc_data *pmc, const char *name, + struct clk *parent) +{ + struct clk *clk = NULL; + + clk = clk_alloc(name, &plldiv_ops, &parent, 1); + if (!clk) + return NULL; + + clk->priv = pmc; + clk->flags = CLK_SET_RATE_GATE; + + if (clk_register(clk)) { + clk_free(clk); + return NULL; + } + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_pmc.c b/optee_os/core/drivers/clk/sam/at91_pmc.c new file mode 100644 index 0000000..da0046d --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_pmc.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define PMC_MAX_IDS 128 +#define PMC_MAX_PCKS 8 + +static struct clk *pmc_clk_get_by_id(struct pmc_clk *clks, unsigned int nclk, + unsigned int id) +{ + unsigned int i = 0; + + for (i = 0; i < nclk; i++) { + if (clks[i].clk && clks[i].id == id) + return clks[i].clk; + } + + return NULL; +} + +struct clk *pmc_clk_get_by_name(struct pmc_clk *clks, unsigned int nclk, + const char *name) +{ + unsigned int i = 0; + + for (i = 0; i < nclk; i++) + if (strcmp(clks[i].clk->name, name) == 0) + return clks[i].clk; + + return NULL; +} + +TEE_Result pmc_clk_get(struct pmc_data *pmc, unsigned int type, + unsigned int idx, struct clk **clk) +{ + unsigned int nclk = 0; + struct pmc_clk *clks = NULL; + + switch (type) { + case PMC_TYPE_CORE: + nclk = pmc->ncore; + clks = pmc->chws; + break; + case PMC_TYPE_SYSTEM: + nclk = pmc->nsystem; + clks = pmc->shws; + break; + case PMC_TYPE_PERIPHERAL: + nclk = pmc->nperiph; + clks = pmc->phws; + break; + case PMC_TYPE_GCK: + nclk = pmc->ngck; + clks = pmc->ghws; + break; + case PMC_TYPE_PROGRAMMABLE: + nclk = pmc->npck; + clks = pmc->pchws; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + *clk = pmc_clk_get_by_id(clks, nclk, idx); + if (!*clk) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; +} + +TEE_Result clk_dt_pmc_get(struct dt_pargs *clkspec, void *data, + struct clk **out_clk) +{ + unsigned int type = clkspec->args[0]; + unsigned int idx = clkspec->args[1]; + struct pmc_data *pmc_data = data; + + if (clkspec->args_count != 2) + return TEE_ERROR_BAD_PARAMETERS; + + return pmc_clk_get(pmc_data, type, idx, out_clk); +} + +struct pmc_data *pmc_data_allocate(unsigned int ncore, unsigned int nsystem, + unsigned int nperiph, unsigned int ngck, + unsigned int npck) +{ + unsigned int num_clks = ncore + nsystem + nperiph + ngck + npck; + unsigned int alloc_size = sizeof(struct pmc_data) + + num_clks * sizeof(struct pmc_clk); + struct pmc_data *pmc_data = NULL; + + pmc_data = calloc(1, alloc_size); + if (!pmc_data) + return NULL; + + pmc_data->ncore = ncore; + pmc_data->chws = pmc_data->hwtable; + + pmc_data->nsystem = nsystem; + pmc_data->shws = pmc_data->chws + ncore; + + pmc_data->nperiph = nperiph; + pmc_data->phws = pmc_data->shws + nsystem; + + pmc_data->ngck = ngck; + pmc_data->ghws = pmc_data->phws + nperiph; + + pmc_data->npck = npck; + pmc_data->pchws = pmc_data->ghws + ngck; + + return pmc_data; +} + +#ifdef CFG_PM_ARM32 +static uint8_t registered_ids[PMC_MAX_IDS]; +static uint8_t registered_pcks[PMC_MAX_PCKS]; + +static struct +{ + uint32_t scsr; + uint32_t pcsr0; + uint32_t uckr; + uint32_t mor; + uint32_t mcfr; + uint32_t pllar; + uint32_t mckr; + uint32_t usb; + uint32_t imr; + uint32_t pcsr1; + uint32_t pcr[PMC_MAX_IDS]; + uint32_t audio_pll0; + uint32_t audio_pll1; + uint32_t pckr[PMC_MAX_PCKS]; +} pmc_cache; + +/* + * As Peripheral ID 0 is invalid on AT91 chips, the identifier is stored + * without alteration in the table, and 0 is for unused clocks. + */ +void pmc_register_id(uint8_t id) +{ + int i = 0; + + for (i = 0; i < PMC_MAX_IDS; i++) { + if (registered_ids[i] == 0) { + registered_ids[i] = id; + return; + } + if (registered_ids[i] == id) + return; + } + + panic("Invalid clock ID"); +} + +/* + * As Programmable Clock 0 is valid on AT91 chips, there is an offset + * of 1 between the stored value and the real clock ID. + */ +void pmc_register_pck(uint8_t pck) +{ + int i = 0; + + for (i = 0; i < PMC_MAX_PCKS; i++) { + if (registered_pcks[i] == 0) { + registered_pcks[i] = pck + 1; + return; + } + if (registered_pcks[i] == pck + 1) + return; + } + + panic("Invalid clock ID"); +} + +static void pmc_suspend(void) +{ + int i = 0; + uint8_t num = 0; + vaddr_t pmc_base = at91_pmc_get_base(); + + pmc_cache.scsr = io_read32(pmc_base + AT91_PMC_SCSR); + pmc_cache.pcsr0 = io_read32(pmc_base + AT91_PMC_PCSR); + pmc_cache.uckr = io_read32(pmc_base + AT91_CKGR_UCKR); + pmc_cache.mor = io_read32(pmc_base + AT91_CKGR_MOR); + pmc_cache.mcfr = io_read32(pmc_base + AT91_CKGR_MCFR); + pmc_cache.pllar = io_read32(pmc_base + AT91_CKGR_PLLAR); + pmc_cache.mckr = io_read32(pmc_base + AT91_PMC_MCKR); + pmc_cache.usb = io_read32(pmc_base + AT91_PMC_USB); + pmc_cache.imr = io_read32(pmc_base + AT91_PMC_IMR); + pmc_cache.pcsr1 = io_read32(pmc_base + AT91_PMC_PCSR1); + + for (i = 0; registered_ids[i]; i++) { + io_write32(pmc_base + AT91_PMC_PCR, + registered_ids[i] & AT91_PMC_PCR_PID_MASK); + pmc_cache.pcr[registered_ids[i]] = io_read32(pmc_base + + AT91_PMC_PCR); + } + for (i = 0; registered_pcks[i]; i++) { + num = registered_pcks[i] - 1; + pmc_cache.pckr[num] = io_read32(pmc_base + AT91_PMC_PCKR(num)); + } +} + +static bool pmc_ready(vaddr_t pmc_base, unsigned int mask) +{ + uint32_t status = 0; + + status = io_read32(pmc_base + AT91_PMC_SR); + + return (status & mask) == mask; +} + +static void pmc_resume(void) +{ + int i = 0; + uint8_t num = 0; + uint32_t tmp = 0; + vaddr_t pmc_base = at91_pmc_get_base(); + uint32_t mask = AT91_PMC_MCKRDY | AT91_PMC_LOCKA; + + tmp = io_read32(pmc_base + AT91_PMC_MCKR); + if (pmc_cache.mckr != tmp) + panic("MCKR was not configured properly by the previous bootstage"); + tmp = io_read32(pmc_base + AT91_CKGR_PLLAR); + if (pmc_cache.pllar != tmp) + panic("PLLAR was not configured properly by the previous bootstage"); + + io_write32(pmc_base + AT91_PMC_SCER, pmc_cache.scsr); + io_write32(pmc_base + AT91_PMC_PCER, pmc_cache.pcsr0); + io_write32(pmc_base + AT91_CKGR_UCKR, pmc_cache.uckr); + io_write32(pmc_base + AT91_CKGR_MOR, pmc_cache.mor); + io_write32(pmc_base + AT91_CKGR_MCFR, pmc_cache.mcfr); + io_write32(pmc_base + AT91_PMC_USB, pmc_cache.usb); + io_write32(pmc_base + AT91_PMC_IMR, pmc_cache.imr); + io_write32(pmc_base + AT91_PMC_PCER1, pmc_cache.pcsr1); + + for (i = 0; registered_ids[i]; i++) { + io_write32(pmc_base + AT91_PMC_PCR, + pmc_cache.pcr[registered_ids[i]] | AT91_PMC_PCR_CMD); + } + for (i = 0; registered_pcks[i]; i++) { + num = registered_pcks[i] - 1; + io_write32(pmc_base + AT91_PMC_PCKR(num), pmc_cache.pckr[num]); + } + + if (pmc_cache.uckr & AT91_PMC_UPLLEN) + mask |= AT91_PMC_LOCKU; + + while (!pmc_ready(pmc_base, mask)) + ; +} + +static TEE_Result pmc_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *hdl __unused) +{ + switch (op) { + case PM_OP_RESUME: + pmc_resume(); + break; + case PM_OP_SUSPEND: + pmc_suspend(); + break; + default: + panic("Invalid PM operation"); + } + + return TEE_SUCCESS; +} + +void pmc_register_pm(void) +{ + /* + * We register the clock as a core service since clocks must be + * re-enable prior to accessing devices + */ + register_pm_core_service_cb(pmc_pm, NULL, "pmc"); +} + +#endif diff --git a/optee_os/core/drivers/clk/sam/at91_pmc.h b/optee_os/core/drivers/clk/sam/at91_pmc.h new file mode 100644 index 0000000..5b4a554 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_pmc.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause */ +/* + * Copyright (C) 2005 Ivan Kokshaysky + * Copyright (C) SAN People + * + * Power Management Controller (PMC) - System peripherals registers. + * Based on AT91RM9200 datasheet revision E. + */ + +#ifndef DRIVERS_CLK_SAM_AT91_PM_H +#define DRIVERS_CLK_SAM_AT91_PM_H + +#include + +#define AT91_PMC_V1 (1) +#define AT91_PMC_V2 (2) + +#define AT91_PMC_SCER 0x00 +#define AT91_PMC_SCDR 0x04 + +#define AT91_PMC_SCSR 0x08 +#define AT91_PMC_PCK BIT(0) +#define AT91RM9200_PMC_UDP BIT(1) +#define AT91RM9200_PMC_MCKUDP BIT(2) +#define AT91RM9200_PMC_UHP BIT(4) +#define AT91SAM926x_PMC_UHP BIT(6) +#define AT91SAM926x_PMC_UDP BIT(7) +#define AT91_PMC_PCK0 BIT(8) +#define AT91_PMC_PCK1 BIT(9) +#define AT91_PMC_PCK2 BIT(10) +#define AT91_PMC_PCK3 BIT(11) +#define AT91_PMC_PCK4 BIT(12) +#define AT91_PMC_HCK0 BIT(16) +#define AT91_PMC_HCK1 BIT(17) + +#define AT91_PMC_PLL_CTRL0 0x0C +#define AT91_PMC_PLL_CTRL0_ENPLL BIT(28) +#define AT91_PMC_PLL_CTRL0_ENPLLCK BIT(29) +#define AT91_PMC_PLL_CTRL0_ENLOCK BIT(31) + +#define AT91_PMC_PLL_CTRL1 0x10 + +#define AT91_PMC_PCER 0x10 +#define AT91_PMC_PCDR 0x14 +#define AT91_PMC_PCSR 0x18 + +#define AT91_PMC_PLL_ACR 0x18 +#define AT91_PMC_PLL_ACR_DEFAULT_UPLL 0x12020010UL +#define AT91_PMC_PLL_ACR_DEFAULT_PLLA 0x00020010UL +#define AT91_PMC_PLL_ACR_UTMIVR BIT(12) +#define AT91_PMC_PLL_ACR_UTMIBG BIT(13) + +#define AT91_CKGR_UCKR 0x1C +#define AT91_PMC_UPLLEN BIT(16) +#define AT91_PMC_UPLLCOUNT (0xf << 20) +#define AT91_PMC_BIASEN BIT(24) +#define AT91_PMC_BIASCOUNT (0xf << 28) + +#define AT91_PMC_PLL_UPDT 0x1C +#define AT91_PMC_PLL_UPDT_UPDATE BIT(8) +#define AT91_PMC_PLL_UPDT_ID BIT(0) +#define AT91_PMC_PLL_UPDT_ID_MSK (0xf) +#define AT91_PMC_PLL_UPDT_STUPTIM (0xff << 16) + +#define AT91_CKGR_MOR 0x20 +#define AT91_PMC_MOSCEN BIT(0) +#define AT91_PMC_OSCBYPASS BIT(1) +#define AT91_PMC_WAITMODE BIT(2) +#define AT91_PMC_MOSCRCEN BIT(3) +#define AT91_PMC_OSCOUNT (0xff << 8) +#define AT91_PMC_KEY_MASK (0xff << 16) +#define AT91_PMC_KEY (0x37 << 16) +#define AT91_PMC_MOSCSEL BIT(24) +#define AT91_PMC_CFDEN BIT(25) + +#define AT91_CKGR_MCFR 0x24 +#define AT91_PMC_MAINF (0xffff << 0) +#define AT91_PMC_MAINRDY BIT(16) + +#define AT91_CKGR_PLLAR 0x28 +#define AT91_CKGR_PLLBR 0x2c +#define AT91_PMC_DIV (0xff << 0) +#define AT91_PMC_PLLCOUNT (0x3f << 8) +#define AT91_PMC_OUT (3 << 14) +#define AT91_PMC_MUL (0x7ff << 16) +#define AT91_PMC_MUL_GET(n) ((n) >> 16 & 0x7ff) +#define AT91_PMC3_MUL (0x7f << 18) +#define AT91_PMC3_MUL_GET(n) ((n) >> 18 & 0x7f) +#define AT91_PMC_USBDIV (3 << 28) +#define AT91_PMC_USBDIV_1 (0 << 28) +#define AT91_PMC_USBDIV_2 BIT(28) +#define AT91_PMC_USBDIV_4 (2 << 28) +#define AT91_PMC_USB96M BIT(28) + +#define AT91_PMC_CPU_CKR 0x28 + +#define AT91_PMC_MCKR 0x30 +#define AT91_PMC_CSS (3 << 0) +#define AT91_PMC_CSS_SLOW (0 << 0) +#define AT91_PMC_CSS_MAIN BIT(0) +#define AT91_PMC_CSS_PLLA (2 << 0) +#define AT91_PMC_CSS_PLLB (3 << 0) +#define AT91_PMC_CSS_UPLL (3 << 0) +#define PMC_PRES_OFFSET 2 +#define AT91_PMC_PRES (7 << PMC_PRES_OFFSET) +#define AT91_PMC_PRES_1 (0 << PMC_PRES_OFFSET) +#define AT91_PMC_PRES_2 BIT(PMC_PRES_OFFSET) +#define AT91_PMC_PRES_4 (2 << PMC_PRES_OFFSET) +#define AT91_PMC_PRES_8 (3 << PMC_PRES_OFFSET) +#define AT91_PMC_PRES_16 (4 << PMC_PRES_OFFSET) +#define AT91_PMC_PRES_32 (5 << PMC_PRES_OFFSET) +#define AT91_PMC_PRES_64 (6 << PMC_PRES_OFFSET) +#define PMC_ALT_PRES_OFFSET 4 +#define AT91_PMC_ALT_PRES (7 << PMC_ALT_PRES_OFFSET) +#define AT91_PMC_ALT_PRES_1 (0 << PMC_ALT_PRES_OFFSET) +#define AT91_PMC_ALT_PRES_2 BIT(PMC_ALT_PRES_OFFSET) +#define AT91_PMC_ALT_PRES_4 (2 << PMC_ALT_PRES_OFFSET) +#define AT91_PMC_ALT_PRES_8 (3 << PMC_ALT_PRES_OFFSET) +#define AT91_PMC_ALT_PRES_16 (4 << PMC_ALT_PRES_OFFSET) +#define AT91_PMC_ALT_PRES_32 (5 << PMC_ALT_PRES_OFFSET) +#define AT91_PMC_ALT_PRES_64 (6 << PMC_ALT_PRES_OFFSET) +#define AT91_PMC_MDIV (3 << 8) +#define AT91RM9200_PMC_MDIV_1 (0 << 8) +#define AT91RM9200_PMC_MDIV_2 BIT(8) +#define AT91RM9200_PMC_MDIV_3 (2 << 8) +#define AT91RM9200_PMC_MDIV_4 (3 << 8) +#define AT91SAM9_PMC_MDIV_1 (0 << 8) +#define AT91SAM9_PMC_MDIV_2 BIT(8) +#define AT91SAM9_PMC_MDIV_4 (2 << 8) +#define AT91SAM9_PMC_MDIV_6 (3 << 8) +#define AT91SAM9_PMC_MDIV_3 (3 << 8) +#define AT91_PMC_PDIV BIT(12) +#define AT91_PMC_PDIV_1 (0 << 12) +#define AT91_PMC_PDIV_2 BIT(12) +#define AT91_PMC_PLLADIV2 BIT(12) +#define AT91_PMC_PLLADIV2_OFF (0 << 12) +#define AT91_PMC_PLLADIV2_ON BIT(12) +#define AT91_PMC_H32MXDIV BIT(24) + +#define AT91_PMC_XTALF 0x34 + +#define AT91_PMC_USB 0x38 +#define AT91_PMC_USBS (0x1 << 0) +#define AT91_PMC_USBS_PLLA (0 << 0) +#define AT91_PMC_USBS_UPLL BIT(0) +#define AT91_PMC_USBS_PLLB BIT(0) +#define AT91_PMC_OHCIUSBDIV (0xF << 8) +#define AT91_PMC_OHCIUSBDIV_1 (0x0 << 8) +#define AT91_PMC_OHCIUSBDIV_2 (0x1 << 8) + +#define AT91_PMC_SMD 0x3c +#define AT91_PMC_SMDS (0x1 << 0) +#define AT91_PMC_SMD_DIV (0x1f << 8) +#define AT91_PMC_SMDDIV(n) (((n) << 8) & AT91_PMC_SMD_DIV) + +#define AT91_PMC_PCKR(n) (0x40 + ((n) * 4)) +#define AT91_PMC_ALT_PCKR_CSS (0x7 << 0) +#define AT91_PMC_CSS_MASTER (4 << 0) +#define AT91_PMC_CSSMCK (0x1 << 8) +#define AT91_PMC_CSSMCK_CSS (0 << 8) +#define AT91_PMC_CSSMCK_MCK BIT(8) + +#define AT91_PMC_IER 0x60 +#define AT91_PMC_IDR 0x64 +#define AT91_PMC_SR 0x68 +#define AT91_PMC_MOSCS BIT(0) +#define AT91_PMC_LOCKA BIT(1) +#define AT91_PMC_LOCKB BIT(2) +#define AT91_PMC_MCKRDY BIT(3) +#define AT91_PMC_LOCKU BIT(6) +#define AT91_PMC_OSCSEL BIT(7) +#define AT91_PMC_PCK0RDY BIT(8) +#define AT91_PMC_PCK1RDY BIT(9) +#define AT91_PMC_PCK2RDY BIT(10) +#define AT91_PMC_PCK3RDY BIT(11) +#define AT91_PMC_MOSCSELS BIT(16) +#define AT91_PMC_MOSCRCS BIT(17) +#define AT91_PMC_CFDEV BIT(18) +#define AT91_PMC_GCKRDY BIT(24) +#define AT91_PMC_MCKXRDY BIT(26) +#define AT91_PMC_IMR 0x6c + +#define AT91_PMC_FSMR 0x70 +#define AT91_PMC_FSTT(n) BIT(n) +#define AT91_PMC_RTTAL BIT(16) +#define AT91_PMC_RTCAL BIT(17) +#define AT91_PMC_USBAL BIT(18) +#define AT91_PMC_SDMMC_CD BIT(19) +#define AT91_PMC_LPM BIT(20) +#define AT91_PMC_RXLP_MCE BIT(24) +#define AT91_PMC_ACC_CE BIT(25) + +#define AT91_PMC_FSPR 0x74 + +#define AT91_PMC_FS_INPUT_MASK 0x7ff + +#define AT91_PMC_PLLICPR 0x80 + +#define AT91_PMC_PROT 0xe4 +#define AT91_PMC_WPEN (0x1 << 0) +#define AT91_PMC_WPKEY (0xffffff << 8) +#define AT91_PMC_PROTKEY (0x504d43 << 8) + +#define AT91_PMC_WPSR 0xe8 +#define AT91_PMC_WPVS (0x1 << 0) +#define AT91_PMC_WPVSRC (0xffff << 8) + +#define AT91_PMC_PLL_ISR0 0xEC + +#define AT91_PMC_PCER1 0x100 +#define AT91_PMC_PCDR1 0x104 +#define AT91_PMC_PCSR1 0x108 + +#define AT91_PMC_PCR 0x10c +#define AT91_PMC_PCR_PID_MASK 0x3f +#define AT91_PMC_PCR_CMD (0x1 << 12) +#define AT91_PMC_PCR_GCKDIV_SHIFT 20 +#define AT91_PMC_PCR_GCKDIV_MASK \ + GENMASK_32(27, AT91_PMC_PCR_GCKDIV_SHIFT) +#define AT91_PMC_PCR_EN (0x1 << 28) +#define AT91_PMC_PCR_GCKEN (0x1 << 29) + +#define AT91_PMC_AUDIO_PLL0 0x14c +#define AT91_PMC_AUDIO_PLL_PLLEN BIT(0) +#define AT91_PMC_AUDIO_PLL_PADEN BIT(1) +#define AT91_PMC_AUDIO_PLL_PMCEN BIT(2) +#define AT91_PMC_AUDIO_PLL_RESETN BIT(3) +#define AT91_PMC_AUDIO_PLL_ND_OFFSET 8 +#define AT91_PMC_AUDIO_PLL_ND_MASK \ + (0x7f << AT91_PMC_AUDIO_PLL_ND_OFFSET) +#define AT91_PMC_AUDIO_PLL_ND(n) \ + SHIFT_U32(n, AT91_PMC_AUDIO_PLL_ND_OFFSET) +#define AT91_PMC_AUDIO_PLL_QDPMC_OFFSET 16 +#define AT91_PMC_AUDIO_PLL_QDPMC_MASK \ + (0x7f << AT91_PMC_AUDIO_PLL_QDPMC_OFFSET) +#define AT91_PMC_AUDIO_PLL_QDPMC(n) \ + SHIFT_U32(n, AT91_PMC_AUDIO_PLL_QDPMC_OFFSET) + +#define AT91_PMC_AUDIO_PLL1 0x150 +#define AT91_PMC_AUDIO_PLL_FRACR_MASK 0x3fffff +#define AT91_PMC_AUDIO_PLL_QDPAD_OFFSET 24 +#define AT91_PMC_AUDIO_PLL_QDPAD_MASK \ + (0x7f << AT91_PMC_AUDIO_PLL_QDPAD_OFFSET) +#define AT91_PMC_AUDIO_PLL_QDPAD(n) \ + SHIFT_U32(n, AT91_PMC_AUDIO_PLL_QDPAD_OFFSET) +#define AT91_PMC_AUDIO_PLL_QDPAD_DIV_OFFSET \ + AT91_PMC_AUDIO_PLL_QDPAD_OFFSET +#define AT91_PMC_AUDIO_PLL_QDPAD_DIV_MASK \ + (0x3 << AT91_PMC_AUDIO_PLL_QDPAD_DIV_OFFSET) +#define AT91_PMC_AUDIO_PLL_QDPAD_DIV(n) \ + SHIFT_U32(n, AT91_PMC_AUDIO_PLL_QDPAD_DIV_OFFSET) +#define AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_OFFSET 26 +#define AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_MAX 0x1f +#define AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_MASK \ + (AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_MAX << \ + AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_OFFSET) +#define AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV(n) \ + SHIFT_U32(n, AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_OFFSET) + +#endif diff --git a/optee_os/core/drivers/clk/sam/at91_programmable.c b/optee_os/core/drivers/clk/sam/at91_programmable.c new file mode 100644 index 0000000..4f24442 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_programmable.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define PROG_ID_MAX 7 + +#define PROG_STATUS_MASK(id) (1 << ((id) + 8)) +#define PROG_PRES(layout, pckr) \ + ({ \ + typeof(layout) __layout = layout; \ + \ + (((pckr) >> (__layout)->pres_shift) & (__layout)->pres_mask); \ + }) +#define PROG_MAX_RM9200_CSS 3 + +struct clk_programmable { + vaddr_t base; + uint8_t id; + const struct clk_programmable_layout *layout; +}; + +static unsigned long clk_programmable_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct clk_programmable *prog = clk->priv; + const struct clk_programmable_layout *layout = prog->layout; + unsigned int pckr = io_read32(prog->base + AT91_PMC_PCKR(prog->id)); + unsigned long rate = 0; + + if (layout->is_pres_direct) + rate = parent_rate / (PROG_PRES(layout, pckr) + 1); + else + rate = parent_rate >> PROG_PRES(layout, pckr); + + return rate; +} + +static TEE_Result clk_programmable_set_parent(struct clk *clk, size_t index) +{ + struct clk_programmable *prog = clk->priv; + const struct clk_programmable_layout *layout = prog->layout; + unsigned int mask = layout->css_mask; + unsigned int pckr = index; + + if (layout->have_slck_mck) + mask |= AT91_PMC_CSSMCK_MCK; + + if (index > layout->css_mask) { + if (index > PROG_MAX_RM9200_CSS && !layout->have_slck_mck) + return TEE_ERROR_BAD_PARAMETERS; + + pckr |= AT91_PMC_CSSMCK_MCK; + } + + io_clrsetbits32(prog->base + AT91_PMC_PCKR(prog->id), mask, pckr); + + return TEE_SUCCESS; +} + +static size_t clk_programmable_get_parent(struct clk *clk) +{ + struct clk_programmable *prog = clk->priv; + const struct clk_programmable_layout *layout = prog->layout; + unsigned int pckr = io_read32(prog->base + AT91_PMC_PCKR(prog->id)); + size_t ret = 0; + + ret = pckr & layout->css_mask; + + if (layout->have_slck_mck && (pckr & AT91_PMC_CSSMCK_MCK) && !ret) + ret = PROG_MAX_RM9200_CSS + 1; + + return ret; +} + +static unsigned int flsi(unsigned int val) +{ + if (val == 0) + return 0; + + return sizeof(unsigned int) * 8 - __builtin_clz(val); +} + +static TEE_Result clk_programmable_set_rate(struct clk *clk, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_programmable *prog = clk->priv; + const struct clk_programmable_layout *layout = prog->layout; + unsigned long div = parent_rate / rate; + int shift = 0; + + if (!div) + return TEE_ERROR_BAD_PARAMETERS; + + if (layout->is_pres_direct) { + shift = div - 1; + + if (shift > layout->pres_mask) + return TEE_ERROR_BAD_PARAMETERS; + } else { + shift = flsi(div) - 1; + + if (div != (1ULL << shift)) + return TEE_ERROR_BAD_PARAMETERS; + + if (shift >= layout->pres_mask) + return TEE_ERROR_BAD_PARAMETERS; + } + + io_clrsetbits32(prog->base + AT91_PMC_PCKR(prog->id), + layout->pres_mask << layout->pres_shift, + shift << layout->pres_shift); + + return TEE_SUCCESS; +} + +static const struct clk_ops programmable_ops = { + .get_rate = clk_programmable_get_rate, + .get_parent = clk_programmable_get_parent, + .set_parent = clk_programmable_set_parent, + .set_rate = clk_programmable_set_rate, +}; + +struct clk * +at91_clk_register_programmable(struct pmc_data *pmc, + const char *name, struct clk **parents, + uint8_t num_parents, uint8_t id, + const struct clk_programmable_layout *layout) +{ + struct clk_programmable *prog = NULL; + struct clk *clk = NULL; + + assert(id <= PROG_ID_MAX); + + clk = clk_alloc(name, &programmable_ops, parents, num_parents); + prog = calloc(1, sizeof(*prog)); + if (!prog || !clk) + return NULL; + + prog->id = id; + prog->layout = layout; + prog->base = pmc->base; + + clk->priv = prog; + clk->flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE; + + if (clk_register(clk)) { + clk_free(clk); + free(prog); + return NULL; + } + + pmc_register_pck(id); + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_sckc.c b/optee_os/core/drivers/clk/sam/at91_sckc.c new file mode 100644 index 0000000..f2fc107 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_sckc.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include "at91_clk.h" +#include +#include + +static struct clk *slow_clk; + +#define SLOW_CLOCK_FREQ 32768 + +struct clk *at91_sckc_clk_get(void) +{ + return slow_clk; +} + +static unsigned long sckc_get_rate(struct clk *clk __unused, + unsigned long parent_rate __unused) +{ + return SLOW_CLOCK_FREQ; +} + +static const struct clk_ops sckc_clk_ops = { + .get_rate = sckc_get_rate, +}; + +static TEE_Result sckc_pmc_setup(const void *fdt __unused, int offs, + const void *data __unused) +{ + struct clk *clk = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + if (slow_clk) + return TEE_ERROR_GENERIC; + + clk = clk_alloc("slowck", &sckc_clk_ops, NULL, 0); + if (!clk) + return TEE_ERROR_OUT_OF_MEMORY; + + res = clk_register(clk); + if (res) { + clk_free(clk); + return res; + } + + res = clk_dt_register_clk_provider(fdt, offs, clk_dt_get_simple_clk, + clk); + if (res) + return res; + + slow_clk = clk; + + return TEE_SUCCESS; +} + +CLK_DT_DECLARE(at91_sckc, "atmel,sama5d4-sckc", sckc_pmc_setup); diff --git a/optee_os/core/drivers/clk/sam/at91_system.c b/optee_os/core/drivers/clk/sam/at91_system.c new file mode 100644 index 0000000..c9f881f --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_system.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define SYSTEM_MAX_ID 31 + +#define SYSTEM_MAX_NAME_SZ 32 + +struct clk_system { + vaddr_t base; + uint8_t id; +}; + +static bool is_pck(int id) +{ + return (id >= 8) && (id <= 15); +} + +static bool clk_system_ready(vaddr_t base, int id) +{ + uint32_t status = io_read32(base + AT91_PMC_SR); + + return status & BIT(id); +} + +static TEE_Result clk_system_enable(struct clk *clk) +{ + struct clk_system *sys = clk->priv; + + io_write32(sys->base + AT91_PMC_SCER, 1 << sys->id); + + if (!is_pck(sys->id)) + return TEE_SUCCESS; + + while (!clk_system_ready(sys->base, sys->id)) + ; + + return TEE_SUCCESS; +} + +static void clk_system_disable(struct clk *clk) +{ + struct clk_system *sys = clk->priv; + + io_write32(sys->base + AT91_PMC_SCDR, 1 << sys->id); +} + +static const struct clk_ops system_ops = { + .enable = clk_system_enable, + .disable = clk_system_disable, +}; + +struct clk * +at91_clk_register_system(struct pmc_data *pmc, const char *name, + struct clk *parent, uint8_t id) +{ + struct clk_system *sys = NULL; + struct clk *clk = NULL; + + if (!parent || id > SYSTEM_MAX_ID) + return NULL; + + clk = clk_alloc(name, &system_ops, &parent, 1); + if (!clk) + return NULL; + + sys = calloc(1, sizeof(*sys)); + if (!sys) { + clk_free(clk); + return NULL; + } + + sys->id = id; + sys->base = pmc->base; + + clk->priv = sys; + + if (clk_register(clk)) { + clk_free(clk); + free(sys); + return NULL; + } + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/at91_usb.c b/optee_os/core/drivers/clk/sam/at91_usb.c new file mode 100644 index 0000000..b5a0db3 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_usb.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#define SAM9X5_USB_DIV_SHIFT 8 +#define SAM9X5_USB_DIV_COUNT BIT32(4) +#define SAM9X5_USB_MAX_DIV (SAM9X5_USB_DIV_COUNT - 1) + +#define SAM9X5_USBS_MASK BIT(0) + +struct at91sam9x5_clk_usb { + vaddr_t base; + uint32_t usbs_mask; +}; + +static unsigned long at91sam9x5_clk_usb_get_rate(struct clk *clk, + unsigned long parent_rate) +{ + struct at91sam9x5_clk_usb *usb = clk->priv; + uint8_t usbdiv = 1; + unsigned int usbr = io_read32(usb->base + AT91_PMC_USB); + + usbdiv = (usbr & AT91_PMC_OHCIUSBDIV) >> SAM9X5_USB_DIV_SHIFT; + + return UDIV_ROUND_NEAREST(parent_rate, (usbdiv + 1)); +} + +static TEE_Result at91sam9x5_clk_usb_set_parent(struct clk *clk, size_t index) +{ + struct at91sam9x5_clk_usb *usb = clk->priv; + + if (index >= clk_get_num_parents(clk)) + return TEE_ERROR_BAD_PARAMETERS; + + io_clrsetbits32(usb->base + AT91_PMC_USB, usb->usbs_mask, index); + + return TEE_SUCCESS; +} + +static size_t at91sam9x5_clk_usb_get_parent(struct clk *clk) +{ + struct at91sam9x5_clk_usb *usb = clk->priv; + unsigned int usbr = io_read32(usb->base + AT91_PMC_USB); + + return usbr & usb->usbs_mask; +} + +static TEE_Result at91sam9x5_clk_usb_set_rate(struct clk *clk, + unsigned long rate, + unsigned long parent_rate) +{ + struct at91sam9x5_clk_usb *usb = clk->priv; + unsigned long div = 1; + + if (!rate) + return TEE_ERROR_BAD_PARAMETERS; + + div = UDIV_ROUND_NEAREST(parent_rate, rate); + if (div > SAM9X5_USB_MAX_DIV + 1 || !div) + return TEE_ERROR_BAD_PARAMETERS; + + io_clrsetbits32(usb->base + AT91_PMC_USB, AT91_PMC_OHCIUSBDIV, + (div - 1) << SAM9X5_USB_DIV_SHIFT); + + return TEE_SUCCESS; +} + +static const struct clk_ops at91sam9x5_usb_ops = { + .get_rate = at91sam9x5_clk_usb_get_rate, + .get_parent = at91sam9x5_clk_usb_get_parent, + .set_parent = at91sam9x5_clk_usb_set_parent, + .set_rate = at91sam9x5_clk_usb_set_rate, +}; + +static struct clk * +_at91sam9x5_clk_register_usb(struct pmc_data *pmc, const char *name, + struct clk **parents, uint8_t num_parents, + uint32_t usbs_mask) +{ + struct at91sam9x5_clk_usb *usb = NULL; + struct clk *clk = NULL; + + clk = clk_alloc(name, &at91sam9x5_usb_ops, parents, num_parents); + if (!clk) + return NULL; + + usb = calloc(1, sizeof(*usb)); + if (!usb) + return NULL; + + usb->base = pmc->base; + usb->usbs_mask = usbs_mask; + + clk->priv = usb; + clk->flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE; + + if (clk_register(clk)) { + clk_free(clk); + free(usb); + return NULL; + } + + return clk; +} + +struct clk * +at91sam9x5_clk_register_usb(struct pmc_data *pmc, const char *name, + struct clk **parents, uint8_t num_parents) +{ + return _at91sam9x5_clk_register_usb(pmc, name, parents, + num_parents, SAM9X5_USBS_MASK); +} diff --git a/optee_os/core/drivers/clk/sam/at91_utmi.c b/optee_os/core/drivers/clk/sam/at91_utmi.c new file mode 100644 index 0000000..d5be945 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/at91_utmi.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (C) 2013 Boris BREZILLON + * Copyright (C) 2021 Microchip + */ + +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +/* + * The purpose of this clock is to generate a 480 MHz signal. A different + * rate can't be configured. + */ +#define UTMI_RATE 480000000 + +struct clk_utmi { + vaddr_t pmc_base; + vaddr_t sfr_base; +}; + +static bool clk_utmi_ready(vaddr_t pmc_base) +{ + uint32_t status = io_read32(pmc_base + AT91_PMC_SR); + + return status & AT91_PMC_LOCKU; +} + +static TEE_Result clk_utmi_enable(struct clk *clk) +{ + struct clk *clk_parent = NULL; + struct clk_utmi *utmi = clk->priv; + unsigned int uckr = AT91_PMC_UPLLEN | AT91_PMC_UPLLCOUNT | + AT91_PMC_BIASEN; + unsigned int utmi_ref_clk_freq = 0; + unsigned long parent_rate = 0; + + /* + * If mainck rate is different from 12 MHz, we have to configure the + * FREQ field of the SFR_UTMICKTRIM register to generate properly + * the utmi clock. + */ + clk_parent = clk_get_parent(clk); + parent_rate = clk_get_rate(clk_parent); + + switch (parent_rate) { + case 12000000: + utmi_ref_clk_freq = 0; + break; + case 16000000: + utmi_ref_clk_freq = 1; + break; + case 24000000: + utmi_ref_clk_freq = 2; + break; + /* + * Not supported on SAMA5D2 but it's not an issue since MAINCK + * maximum value is 24 MHz. + */ + case 48000000: + utmi_ref_clk_freq = 3; + break; + default: + EMSG("UTMICK: unsupported mainck rate"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (utmi->sfr_base) { + io_clrsetbits32(utmi->sfr_base + AT91_SFR_UTMICKTRIM, + AT91_UTMICKTRIM_FREQ, utmi_ref_clk_freq); + } else if (utmi_ref_clk_freq) { + EMSG("UTMICK: sfr node required"); + return TEE_ERROR_BAD_STATE; + } + + io_clrsetbits32(utmi->pmc_base + AT91_CKGR_UCKR, uckr, uckr); + + while (!clk_utmi_ready(utmi->pmc_base)) + ; + + return TEE_SUCCESS; +} + +static void clk_utmi_disable(struct clk *clk) +{ + struct clk_utmi *utmi = clk->priv; + + io_clrbits32(utmi->pmc_base + AT91_CKGR_UCKR, AT91_PMC_UPLLEN); +} + +static unsigned long clk_utmi_get_rate(struct clk *clk __unused, + unsigned long parent_rate __unused) +{ + /* UTMI clk rate is fixed. */ + return UTMI_RATE; +} + +static const struct clk_ops utmi_ops = { + .enable = clk_utmi_enable, + .disable = clk_utmi_disable, + .get_rate = clk_utmi_get_rate, +}; + +struct clk * +at91_clk_register_utmi(struct pmc_data *pmc, const char *name, + struct clk *parent) +{ + struct clk_utmi *utmi = NULL; + struct clk *clk = NULL; + + clk = clk_alloc(name, &utmi_ops, &parent, 1); + if (!clk) + return NULL; + + utmi = calloc(1, sizeof(*utmi)); + if (!utmi) { + clk_free(clk); + return NULL; + } + + utmi->pmc_base = pmc->base; + utmi->sfr_base = sam_sfr_base(); + clk->flags = CLK_SET_RATE_GATE; + + clk->priv = utmi; + + if (clk_register(clk)) { + clk_free(clk); + free(utmi); + return NULL; + } + + return clk; +} diff --git a/optee_os/core/drivers/clk/sam/sama5d2_clk.c b/optee_os/core/drivers/clk/sam/sama5d2_clk.c new file mode 100644 index 0000000..65f8f1b --- /dev/null +++ b/optee_os/core/drivers/clk/sam/sama5d2_clk.c @@ -0,0 +1,612 @@ +// SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause +/* + * Copyright (c) 2021, Microchip + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "at91_clk.h" + +#include + +#define PROGCK_PARENT_COUNT 6 +#define PARENT_SIZE ARRAY_SIZE(sama5d2_systemck) + +struct sam_clk { + const char *n; + uint8_t id; +}; + +static const struct clk_master_charac mck_charac = { + .output = { .min = 124000000, .max = 166000000 }, + .divisors = { 1, 2, 4, 3 }, +}; + +static uint8_t plla_out[1]; + +static uint16_t plla_icpll[1]; + +static const struct clk_range plla_outputs[] = { + { .min = 600000000, .max = 1200000000 }, +}; + +static const struct clk_pll_charac plla_charac = { + .input = { .min = 12000000, .max = 24000000 }, + .num_output = ARRAY_SIZE(plla_outputs), + .output = plla_outputs, + .icpll = plla_icpll, + .out = plla_out, +}; + +static const struct clk_pcr_layout sama5d2_pcr_layout = { + .offset = 0x10c, + .cmd = BIT(12), + .gckcss_mask = GENMASK_32(10, 8), + .pid_mask = GENMASK_32(6, 0), +}; + +static const struct clk_programmable_layout sama5d2_prog_layout = { + .pres_mask = 0xff, + .pres_shift = 4, + .css_mask = 0x7, + .have_slck_mck = 0, + .is_pres_direct = 1, +}; + +static const struct sam_clk sama5d2_systemck[] = { + { .n = "ddrck", .id = 2 }, + { .n = "lcdck", .id = 3 }, + { .n = "uhpck", .id = 6 }, + { .n = "udpck", .id = 7 }, + { .n = "pck0", .id = 8 }, + { .n = "pck1", .id = 9 }, + { .n = "pck2", .id = 10 }, + { .n = "iscck", .id = 18 }, +}; + +static const struct { + struct sam_clk clk; + struct clk_range r; +} sama5d2_peri32ck[] = { + { + .clk = { .n = "macb0_clk", .id = 5 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "tdes_clk", .id = 11 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "matrix1_clk", .id = 14 }, + }, + { + .clk = { .n = "hsmc_clk", .id = 17 }, + }, + { + .clk = { .n = "pioA_clk", .id = 18 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "flx0_clk", .id = 19 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "flx1_clk", .id = 20 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "flx2_clk", .id = 21 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "flx3_clk", .id = 22 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "flx4_clk", .id = 23 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "uart0_clk", .id = 24 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "uart1_clk", .id = 25 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "uart2_clk", .id = 26 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "uart3_clk", .id = 27 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "uart4_clk", .id = 28 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "twi0_clk", .id = 29 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "twi1_clk", .id = 30 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "spi0_clk", .id = 33 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "spi1_clk", .id = 34 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "tcb0_clk", .id = 35 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "tcb1_clk", .id = 36 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "pwm_clk", .id = 38 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "adc_clk", .id = 40 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "uhphs_clk", .id = 41 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "udphs_clk", .id = 42 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "ssc0_clk", .id = 43 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "ssc1_clk", .id = 44 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "trng_clk", .id = 47 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "pdmic_clk", .id = 48 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "securam_clk", .id = 51 }, }, + { + .clk = { .n = "i2s0_clk", .id = 54 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "i2s1_clk", .id = 55 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "can0_clk", .id = 56 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "can1_clk", .id = 57 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "ptc_clk", .id = 58 }, + .r = { .min = 0, .max = 83000000 }, + }, + { + .clk = { .n = "classd_clk", .id = 59 }, + .r = { .min = 0, .max = 83000000 }, + }, +}; + +static const struct sam_clk sama5d2_perick[] = { + { .n = "dma0_clk", .id = 6 }, + { .n = "dma1_clk", .id = 7 }, + { .n = "aes_clk", .id = 9 }, + { .n = "aesb_clk", .id = 10 }, + { .n = "sha_clk", .id = 12 }, + { .n = "mpddr_clk", .id = 13 }, + { .n = "matrix0_clk", .id = 15 }, + { .n = "sdmmc0_hclk", .id = 31 }, + { .n = "sdmmc1_hclk", .id = 32 }, + { .n = "lcdc_clk", .id = 45 }, + { .n = "isc_clk", .id = 46 }, + { .n = "qspi0_clk", .id = 52 }, + { .n = "qspi1_clk", .id = 53 }, +}; + +static const struct { + struct sam_clk clk; + struct clk_range r; + int chg_pid; +} sama5d2_gck[] = { + { + .clk = { .n = "sdmmc0_gclk", .id = 31 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "sdmmc1_gclk", .id = 32 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "tcb0_gclk", .id = 35 }, + .r = { .min = 0, .max = 83000000 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "tcb1_gclk", .id = 36 }, + .r = { .min = 0, .max = 83000000 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "pwm_gclk", .id = 38 }, + .r = { .min = 0, .max = 83000000 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "isc_gclk", .id = 46 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "pdmic_gclk", .id = 48 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "i2s0_gclk", .id = 54 }, + .chg_pid = 5, + }, + { + .clk = { .n = "i2s1_gclk", .id = 55 }, + .chg_pid = 5, + }, + { + .clk = { .n = "can0_gclk", .id = 56 }, + .r = { .min = 0, .max = 80000000 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "can1_gclk", .id = 57 }, + .r = { .min = 0, .max = 80000000 }, + .chg_pid = INT_MIN, + }, + { + .clk = { .n = "classd_gclk", .id = 59 }, + .chg_pid = 5, + .r = { .min = 0, .max = 100000000 }, + }, +}; + +static const struct sam_clk sama5d2_progck[] = { + { .n = "prog0", .id = 0 }, + { .n = "prog1", .id = 1 }, + { .n = "prog2", .id = 2 }, +}; + +static struct pmc_data *pmc; + +vaddr_t at91_pmc_get_base(void) +{ + assert(pmc); + + return pmc->base; +} + +TEE_Result at91_pmc_clk_get(unsigned int type, unsigned int idx, + struct clk **clk) +{ + return pmc_clk_get(pmc, type, idx, clk); +} + +static TEE_Result pmc_setup(const void *fdt, int nodeoffset, + const void *data __unused) +{ + size_t size = 0; + vaddr_t base = 0; + unsigned int i = 0; + int bypass = 0; + const uint32_t *fdt_prop = NULL; + struct pmc_clk *pmc_clk = NULL; + const struct sam_clk *sam_clk = NULL; + struct clk_range range = CLK_RANGE(0, 0); + struct clk *h32mxck = NULL; + struct clk *mckdivck = NULL; + struct clk *plladivck = NULL; + struct clk *usbck = NULL; + struct clk *audiopll_pmcck = NULL; + struct clk *parents[PARENT_SIZE] = {NULL}; + struct clk *main_clk = NULL; + struct clk *utmi_clk = NULL; + struct clk *slow_clk = NULL; + struct clk *clk = NULL; + struct clk *main_rc_osc = NULL; + struct clk *main_osc = NULL; + struct clk *main_xtal_clk = NULL; + struct clk *audiopll_fracck = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + /* + * We want PARENT_SIZE to be MAX(ARRAY_SIZE(sama5d2_systemck),6) + * but using this define won't allow static initialization of parents + * due to dynamic size. + */ + COMPILE_TIME_ASSERT(ARRAY_SIZE(sama5d2_systemck) == PARENT_SIZE); + COMPILE_TIME_ASSERT(PARENT_SIZE >= 6); + + if (dt_map_dev(fdt, nodeoffset, &base, &size, DT_MAP_AUTO) < 0) + panic(); + + if (fdt_get_status(fdt, nodeoffset) == DT_STATUS_OK_SEC) + matrix_configure_periph_secure(AT91C_ID_PMC); + + res = clk_dt_get_by_name(fdt, nodeoffset, "slow_clk", &slow_clk); + if (res) + panic(); + + res = clk_dt_get_by_name(fdt, nodeoffset, "main_xtal", &main_xtal_clk); + if (res) + panic(); + + pmc = pmc_data_allocate(PMC_SAMA5D2_CORE_CLK_COUNT, + ARRAY_SIZE(sama5d2_systemck), + ARRAY_SIZE(sama5d2_perick) + + ARRAY_SIZE(sama5d2_peri32ck), + ARRAY_SIZE(sama5d2_gck), + ARRAY_SIZE(sama5d2_progck)); + if (!pmc) + panic(); + pmc->base = base; + + main_rc_osc = pmc_register_main_rc_osc(pmc, "main_rc_osc", 12000000); + if (!main_rc_osc) + panic(); + + fdt_prop = fdt_getprop(fdt, nodeoffset, "atmel,osc-bypass", NULL); + if (fdt_prop) + bypass = fdt32_to_cpu(*fdt_prop); + + main_osc = pmc_register_main_osc(pmc, "main_osc", main_xtal_clk, + bypass); + if (!main_osc) + panic(); + + parents[0] = main_rc_osc; + parents[1] = main_osc; + main_clk = at91_clk_register_sam9x5_main(pmc, "mainck", parents, 2); + if (!main_clk) + panic(); + + pmc_clk = &pmc->chws[PMC_MAIN]; + pmc_clk->clk = main_clk; + pmc_clk->id = PMC_MAIN; + + clk = at91_clk_register_pll(pmc, "pllack", main_clk, 0, + &sama5d3_pll_layout, &plla_charac); + if (!clk) + panic(); + + plladivck = at91_clk_register_plldiv(pmc, "plladivck", clk); + if (!plladivck) + panic(); + + pmc_clk = &pmc->chws[PMC_PLLACK]; + pmc_clk->clk = plladivck; + pmc_clk->id = PMC_PLLACK; + + audiopll_fracck = at91_clk_register_audio_pll_frac(pmc, + "audiopll_fracck", + main_clk); + if (!audiopll_fracck) + panic(); + + pmc_clk = &pmc->chws[PMC_AUDIOPLL_FRACCK]; + pmc_clk->clk = audiopll_fracck; + pmc_clk->id = PMC_AUDIOPLL_FRACCK; + + clk = at91_clk_register_audio_pll_pad(pmc, "audiopll_padck", + audiopll_fracck); + if (!clk) + panic(); + + audiopll_pmcck = at91_clk_register_audio_pll_pmc(pmc, "audiopll_pmcck", + audiopll_fracck); + if (!audiopll_pmcck) + panic(); + + pmc_clk = &pmc->chws[PMC_AUDIOPLLCK]; + pmc_clk->clk = audiopll_pmcck; + pmc_clk->id = PMC_AUDIOPLLCK; + + utmi_clk = at91_clk_register_utmi(pmc, "utmick", main_clk); + if (!utmi_clk) + panic(); + + pmc_clk = &pmc->chws[PMC_UTMI]; + pmc_clk->clk = utmi_clk; + pmc_clk->id = PMC_UTMI; + + parents[0] = slow_clk; + parents[1] = main_clk; + parents[2] = plladivck; + parents[3] = utmi_clk; + + clk = at91_clk_register_master_pres(pmc, "masterck_pres", 4, + parents, + &at91sam9x5_master_layout, + &mck_charac, INT_MIN); + if (!clk) + panic(); + + pmc_clk = &pmc->chws[PMC_MCK_PRES]; + pmc_clk->clk = clk; + pmc_clk->id = PMC_MCK_PRES; + + mckdivck = at91_clk_register_master_div(pmc, "masterck_div", + clk, + &at91sam9x5_master_layout, + &mck_charac); + if (!mckdivck) + panic(); + + pmc_clk = &pmc->chws[PMC_MCK]; + pmc_clk->clk = mckdivck; + pmc_clk->id = PMC_MCK; + + h32mxck = at91_clk_register_h32mx(pmc, "h32mxck", mckdivck); + if (!h32mxck) + panic(); + + pmc_clk = &pmc->chws[PMC_MCK2]; + pmc_clk->clk = h32mxck; + pmc_clk->id = PMC_MCK2; + + parents[0] = plladivck; + parents[1] = utmi_clk; + usbck = at91sam9x5_clk_register_usb(pmc, "usbck", parents, 2); + if (!usbck) + panic(); + + pmc_clk = &pmc->chws[PMC_USBCK]; + pmc_clk->clk = usbck; + pmc_clk->id = PMC_USBCK; + + parents[0] = slow_clk; + parents[1] = main_clk; + parents[2] = plladivck; + parents[3] = utmi_clk; + parents[4] = mckdivck; + parents[5] = audiopll_pmcck; + for (i = 0; i < ARRAY_SIZE(sama5d2_progck); i++) { + sam_clk = &sama5d2_progck[i]; + clk = at91_clk_register_programmable(pmc, sam_clk->n, + parents, + PROGCK_PARENT_COUNT, i, + &sama5d2_prog_layout); + if (!clk) + panic(); + + pmc_clk = &pmc->pchws[i]; + pmc_clk->clk = clk; + pmc_clk->id = sam_clk->id; + } + + /* This array order must match the one in sama5d2_systemck */ + parents[0] = mckdivck; + parents[1] = mckdivck; + parents[2] = usbck; + parents[3] = usbck; + parents[4] = pmc->pchws[0].clk; + parents[5] = pmc->pchws[1].clk; + parents[6] = pmc->pchws[2].clk; + parents[7] = mckdivck; + for (i = 0; i < ARRAY_SIZE(sama5d2_systemck); i++) { + sam_clk = &sama5d2_systemck[i]; + clk = at91_clk_register_system(pmc, sam_clk->n, + parents[i], + sam_clk->id); + if (!clk) + panic(); + + pmc_clk = &pmc->shws[i]; + pmc_clk->clk = clk; + pmc_clk->id = sam_clk->id; + } + + for (i = 0; i < ARRAY_SIZE(sama5d2_perick); i++) { + sam_clk = &sama5d2_perick[i]; + clk = at91_clk_register_sam9x5_periph(pmc, + &sama5d2_pcr_layout, + sam_clk->n, + mckdivck, + sam_clk->id, + &range); + if (!clk) + panic(); + + pmc_clk = &pmc->phws[i]; + pmc_clk->clk = clk; + pmc_clk->id = sam_clk->id; + } + + for (i = 0; i < ARRAY_SIZE(sama5d2_peri32ck); i++) { + sam_clk = &sama5d2_peri32ck[i].clk; + clk = at91_clk_register_sam9x5_periph(pmc, + &sama5d2_pcr_layout, + sam_clk->n, + h32mxck, + sam_clk->id, + &sama5d2_peri32ck[i].r); + if (!clk) + panic(); + + pmc_clk = &pmc->phws[ARRAY_SIZE(sama5d2_perick) + i]; + pmc_clk->clk = clk; + pmc_clk->id = sam_clk->id; + } + + parents[0] = slow_clk; + parents[1] = main_clk; + parents[2] = plladivck; + parents[3] = utmi_clk; + parents[4] = mckdivck; + parents[5] = audiopll_pmcck; + for (i = 0; i < ARRAY_SIZE(sama5d2_gck); i++) { + sam_clk = &sama5d2_gck[i].clk; + clk = at91_clk_register_generated(pmc, + &sama5d2_pcr_layout, + sam_clk->n, + parents, 6, + sam_clk->id, + &sama5d2_gck[i].r, + sama5d2_gck[i].chg_pid); + if (!clk) + panic(); + + pmc_clk = &pmc->ghws[i]; + pmc_clk->clk = clk; + pmc_clk->id = sam_clk->id; + } + + parents[0] = pmc_clk_get_by_name(pmc->phws, pmc->nperiph, "i2s0_clk"); + parents[1] = pmc_clk_get_by_name(pmc->ghws, pmc->ngck, "i2s0_gclk"); + clk = at91_clk_i2s_mux_register("i2s0_muxclk", parents, 2, 0); + if (!clk) + panic(); + + pmc->chws[PMC_I2S0_MUX].clk = clk; + pmc->chws[PMC_I2S0_MUX].id = PMC_I2S0_MUX; + + parents[0] = pmc_clk_get_by_name(pmc->phws, pmc->nperiph, "i2s1_clk"); + parents[1] = pmc_clk_get_by_name(pmc->ghws, pmc->ngck, "i2s1_gclk"); + clk = at91_clk_i2s_mux_register("i2s1_muxclk", parents, 2, 1); + if (!clk) + panic(); + + pmc->chws[PMC_I2S1_MUX].clk = clk; + pmc->chws[PMC_I2S1_MUX].id = PMC_I2S1_MUX; + + clk_dt_register_clk_provider(fdt, nodeoffset, clk_dt_pmc_get, pmc); + + pmc_register_pm(); + + return TEE_SUCCESS; +} + +CLK_DT_DECLARE(sama5d2_clk, "atmel,sama5d2-pmc", pmc_setup); diff --git a/optee_os/core/drivers/clk/sam/sub.mk b/optee_os/core/drivers/clk/sam/sub.mk new file mode 100644 index 0000000..eca5a70 --- /dev/null +++ b/optee_os/core/drivers/clk/sam/sub.mk @@ -0,0 +1,8 @@ +global-incdirs-y += . + +at91-common = at91_sckc.c at91_main.c at91_pmc.c at91_pll.c at91_plldiv.c +at91-common += at91_utmi.c at91_master.c at91_h32mx.c at91_usb.c +at91-common += at91_programmable.c at91_system.c at91_peripheral.c +at91-common += at91_generated.c at91_i2s_mux.c at91_audio_pll.c + +srcs-$(CFG_DRIVERS_SAMA5D2_CLK) += $(at91-common) sama5d2_clk.c diff --git a/optee_os/core/drivers/clk/sub.mk b/optee_os/core/drivers/clk/sub.mk new file mode 100644 index 0000000..6233483 --- /dev/null +++ b/optee_os/core/drivers/clk/sub.mk @@ -0,0 +1,8 @@ +srcs-y += clk.c +srcs-$(CFG_DRIVERS_CLK_DT) += clk_dt.c +srcs-$(CFG_DRIVERS_CLK_FIXED) += fixed_clk.c +srcs-$(CFG_STM32MP_CLK_CORE) += clk-stm32-core.c +srcs-$(CFG_STM32MP13_CLK) += clk-stm32mp13.c +srcs-$(CFG_STM32MP15_CLK) += clk-stm32mp15.c + +subdirs-$(CFG_DRIVERS_SAM_CLK) += sam \ No newline at end of file diff --git a/optee_os/core/drivers/crypto/aspeed/crypto_ast2600.c b/optee_os/core/drivers/crypto/aspeed/crypto_ast2600.c new file mode 100644 index 0000000..c5e0317 --- /dev/null +++ b/optee_os/core/drivers/crypto/aspeed/crypto_ast2600.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Aspeed Technology Inc. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "hace_ast2600.h" + +#define SCU_RST1 0x40 +#define SCU_RSTCLR1 0x44 +#define SCU_RST_CRYPTO BIT(4) + +#define SCU_CLKGATE1 0x80 +#define SCU_CLKGATECLR1 0x84 +#define SCU_CLKGATE_HACE BIT(13) + +static TEE_Result crypto_ast2600_init(void) +{ + TEE_Result rc = TEE_ERROR_GENERIC; + vaddr_t scu_virt = 0; + + scu_virt = core_mmu_get_va(SCU_BASE, MEM_AREA_IO_NSEC, SMALL_PAGE_SIZE); + if (!scu_virt) + panic(); + + /* ast2600 crypto engines share the same reset control */ + io_write32(scu_virt + SCU_RST1, SCU_RST_CRYPTO); + udelay(100); + io_write32(scu_virt + SCU_RSTCLR1, SCU_RST_CRYPTO); + + if (IS_ENABLED(CFG_CRYPTO_DRV_HASH)) { + io_write32(scu_virt + SCU_CLKGATECLR1, SCU_CLKGATE_HACE); + + rc = ast2600_drvcrypt_register_hash(); + if (rc) { + EMSG("cannot register hash driver, rc=%d", rc); + return rc; + } + } + + return TEE_SUCCESS; +} + +early_init_late(crypto_ast2600_init); diff --git a/optee_os/core/drivers/crypto/aspeed/hace_ast2600.c b/optee_os/core/drivers/crypto/aspeed/hace_ast2600.c new file mode 100644 index 0000000..1055025 --- /dev/null +++ b/optee_os/core/drivers/crypto/aspeed/hace_ast2600.c @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Aspeed Technology Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hace_ast2600.h" + +#define HACE_BASE 0x1e6d0000 + +/* register offsets and bit fields */ +#define HACE_STS 0x1C +#define HACE_STS_HASH_INT BIT(9) +#define HACE_STS_HASH_BUSY BIT(0) +#define HACE_HASH_DATA 0x20 +#define HACE_HASH_DIGEST 0x24 +#define HACE_HASH_HMAC_KEY 0x28 +#define HACE_HASH_DATA_LEN 0x2C +#define HACE_HASH_CMD 0x30 +#define HACE_HASH_CMD_ACCUM BIT(8) +#define HACE_HASH_CMD_ALG_SHA1 BIT(5) +#define HACE_HASH_CMD_ALG_SHA256 (BIT(6) | BIT(4)) +#define HACE_HASH_CMD_ALG_SHA384 (BIT(10) | BIT(6) | BIT(5)) +#define HACE_HASH_CMD_ALG_SHA512 (BIT(6) | BIT(5)) +#define HACE_HASH_CMD_SHA_BE BIT(3) + +/* buffer size based on SHA-512 need */ +#define HASH_BLK_BUFSZ 128 +#define HASH_DGT_BUFSZ 64 + +register_phys_mem(MEM_AREA_IO_SEC, HACE_BASE, SMALL_PAGE_SIZE); + +struct ast2600_hace_ctx { + struct crypto_hash_ctx hash_ctx; + uint32_t cmd; + uint32_t algo; + uint32_t dgt_size; + uint32_t blk_size; + uint32_t pad_size; + uint64_t total[2]; + + /* DMA memory to interact with HACE */ + uint8_t *buf; + uint8_t *digest; +}; + +static vaddr_t hace_virt; +struct mutex hace_mtx = MUTEX_INITIALIZER; + +static const uint32_t iv_sha1[8] = { + 0x01234567, 0x89abcdef, 0xfedcba98, 0x76543210, + 0xf0e1d2c3, 0, 0, 0 +}; + +static const uint32_t iv_sha256[8] = { + 0x67e6096a, 0x85ae67bb, 0x72f36e3c, 0x3af54fa5, + 0x7f520e51, 0x8c68059b, 0xabd9831f, 0x19cde05b +}; + +static const uint32_t iv_sha384[16] = { + 0x5d9dbbcb, 0xd89e05c1, 0x2a299a62, 0x07d57c36, + 0x5a015991, 0x17dd7030, 0xd8ec2f15, 0x39590ef7, + 0x67263367, 0x310bc0ff, 0x874ab48e, 0x11155868, + 0x0d2e0cdb, 0xa78ff964, 0x1d48b547, 0xa44ffabe +}; + +static const uint32_t iv_sha512[16] = { + 0x67e6096a, 0x08c9bcf3, 0x85ae67bb, 0x3ba7ca84, + 0x72f36e3c, 0x2bf894fe, 0x3af54fa5, 0xf1361d5f, + 0x7f520e51, 0xd182e6ad, 0x8c68059b, 0x1f6c3e2b, + 0xabd9831f, 0x6bbd41fb, 0x19cde05b, 0x79217e13 +}; + +static TEE_Result ast2600_hace_process(struct crypto_hash_ctx *ctx, + const uint8_t *data, size_t len) +{ + TEE_Result rc = TEE_ERROR_GENERIC; + uint32_t sts = 0; + uint64_t tref = 0; + paddr_t data_phys = 0; + paddr_t digest_phys = 0; + struct ast2600_hace_ctx *hctx = NULL; + + mutex_lock(&hace_mtx); + + hctx = container_of(ctx, struct ast2600_hace_ctx, hash_ctx); + + sts = io_read32(hace_virt + HACE_STS); + if (sts & HACE_STS_HASH_BUSY) { + rc = TEE_ERROR_BUSY; + goto out; + } + + cache_operation(TEE_CACHEFLUSH, (void *)data, len); + + data_phys = virt_to_phys((void *)data); + digest_phys = virt_to_phys(hctx->digest); + + io_write32(hace_virt + HACE_HASH_DATA, (uint32_t)data_phys); + io_write32(hace_virt + HACE_HASH_DIGEST, (uint32_t)digest_phys); + io_write32(hace_virt + HACE_HASH_HMAC_KEY, (uint32_t)digest_phys); + + io_write32(hace_virt + HACE_HASH_DATA_LEN, len); + io_write32(hace_virt + HACE_HASH_CMD, hctx->cmd); + + /* poll for completion */ + tref = timeout_init_us(1000 + (len >> 3)); + + do { + sts = io_read32(hace_virt + HACE_STS); + if (timeout_elapsed(tref)) { + rc = TEE_ERROR_TARGET_DEAD; + goto out; + } + } while (!(sts & HACE_STS_HASH_INT)); + + io_write32(hace_virt + HACE_STS, HACE_STS_HASH_INT); + + rc = TEE_SUCCESS; +out: + mutex_unlock(&hace_mtx); + + return rc; +} + +static TEE_Result ast2600_hace_init(struct crypto_hash_ctx *ctx) +{ + struct ast2600_hace_ctx *hctx = NULL; + + hctx = container_of(ctx, struct ast2600_hace_ctx, hash_ctx); + + switch (hctx->algo) { + case TEE_ALG_SHA1: + memcpy(hctx->digest, iv_sha1, sizeof(iv_sha1)); + break; + case TEE_ALG_SHA256: + memcpy(hctx->digest, iv_sha256, sizeof(iv_sha256)); + break; + case TEE_ALG_SHA384: + memcpy(hctx->digest, iv_sha384, sizeof(iv_sha384)); + break; + case TEE_ALG_SHA512: + memcpy(hctx->digest, iv_sha512, sizeof(iv_sha512)); + break; + default: + return TEE_ERROR_NOT_SUPPORTED; + } + + hctx->total[0] = 0; + hctx->total[1] = 0; + + cache_operation(TEE_CACHEFLUSH, hctx->digest, HASH_DGT_BUFSZ); + + return TEE_SUCCESS; +} + +static TEE_Result ast2600_hace_update(struct crypto_hash_ctx *ctx, + const uint8_t *data, size_t len) +{ + TEE_Result rc = TEE_ERROR_GENERIC; + uint32_t left = 0; + uint32_t fill = 0; + size_t blk_size = 0; + struct ast2600_hace_ctx *hctx = NULL; + + if (!ctx || !data || !len) + return TEE_ERROR_BAD_PARAMETERS; + + hctx = container_of(ctx, struct ast2600_hace_ctx, hash_ctx); + + blk_size = hctx->blk_size; + + left = hctx->total[0] & (blk_size - 1); + fill = blk_size - left; + + hctx->total[0] += len; + if (hctx->total[0] < len) + hctx->total[1]++; + + if (left && len >= fill) { + memcpy(hctx->buf + left, data, fill); + rc = ast2600_hace_process(ctx, hctx->buf, blk_size); + if (rc) + return rc; + + data += fill; + len -= fill; + left = 0; + } + + while (len >= blk_size) { + memcpy(hctx->buf, data, blk_size); + rc = ast2600_hace_process(ctx, hctx->buf, blk_size); + if (rc) + return rc; + + data += blk_size; + len -= blk_size; + } + + if (len) + memcpy(hctx->buf + left, data, len); + + return TEE_SUCCESS; +} + +static TEE_Result ast2600_hace_final(struct crypto_hash_ctx *ctx, + uint8_t *digest, size_t len) +{ + TEE_Result rc = TEE_ERROR_GENERIC; + uint32_t last = 0; + uint32_t padn = 0; + uint8_t pad[HASH_BLK_BUFSZ * 2] = { }; + uint64_t dbits[2] = { }; + uint64_t dbits_be[2] = { }; + struct ast2600_hace_ctx *hctx = NULL; + size_t length = 0; + + hctx = container_of(ctx, struct ast2600_hace_ctx, hash_ctx); + length = MIN(len, hctx->dgt_size); + + memset(pad, 0, sizeof(pad)); + pad[0] = 0x80; + + dbits[0] = (hctx->total[0] << 3); + dbits_be[0] = get_be64(&dbits[0]); + + dbits[1] = (hctx->total[0] >> 61) | (hctx->total[1] << 3); + dbits_be[1] = get_be64(&dbits[1]); + + last = hctx->total[0] & (hctx->blk_size - 1); + + switch (hctx->algo) { + case TEE_ALG_SHA1: + case TEE_ALG_SHA256: + if (last < 56) + padn = 56 - last; + else + padn = 120 - last; + + rc = ast2600_hace_update(ctx, pad, padn); + if (rc) + return rc; + + rc = ast2600_hace_update(ctx, (uint8_t *)&dbits_be[0], + sizeof(dbits_be[0])); + if (rc) + return rc; + break; + case TEE_ALG_SHA384: + case TEE_ALG_SHA512: + if (last < 112) + padn = 112 - last; + else + padn = 240 - last; + + rc = ast2600_hace_update(ctx, pad, padn); + if (rc) + return rc; + + rc = ast2600_hace_update(ctx, (uint8_t *)&dbits_be[1], + sizeof(dbits_be[1])); + if (rc) + return rc; + + rc = ast2600_hace_update(ctx, (uint8_t *)&dbits_be[0], + sizeof(dbits_be[0])); + if (rc) + return rc; + break; + default: + return TEE_ERROR_NOT_SUPPORTED; + } + + cache_operation(TEE_CACHEINVALIDATE, hctx->digest, HASH_DGT_BUFSZ); + + memcpy(digest, hctx->digest, length); + + return TEE_SUCCESS; +} + +static void ast2600_hace_free(struct crypto_hash_ctx *ctx) +{ + struct ast2600_hace_ctx *hctx = NULL; + + hctx = container_of(ctx, struct ast2600_hace_ctx, hash_ctx); + + free(hctx->buf); + free(hctx->digest); + free(hctx); +} + +static void ast2600_hace_copy_state(struct crypto_hash_ctx *dst_ctx, + struct crypto_hash_ctx *src_ctx) +{ + struct ast2600_hace_ctx *src_hctx = NULL; + struct ast2600_hace_ctx *dst_hctx = NULL; + + src_hctx = container_of(src_ctx, struct ast2600_hace_ctx, hash_ctx); + dst_hctx = container_of(dst_ctx, struct ast2600_hace_ctx, hash_ctx); + + dst_hctx->hash_ctx = src_hctx->hash_ctx; + dst_hctx->cmd = src_hctx->cmd; + dst_hctx->dgt_size = src_hctx->dgt_size; + dst_hctx->blk_size = src_hctx->blk_size; + dst_hctx->pad_size = src_hctx->pad_size; + dst_hctx->total[0] = src_hctx->total[0]; + dst_hctx->total[1] = src_hctx->total[1]; + + cache_operation(TEE_CACHEINVALIDATE, src_hctx->buf, HASH_BLK_BUFSZ); + memcpy(dst_hctx->buf, src_hctx->buf, HASH_BLK_BUFSZ); + cache_operation(TEE_CACHEFLUSH, dst_hctx->buf, HASH_BLK_BUFSZ); + + cache_operation(TEE_CACHEINVALIDATE, src_hctx->digest, HASH_DGT_BUFSZ); + memcpy(dst_hctx->digest, src_hctx->digest, HASH_DGT_BUFSZ); + cache_operation(TEE_CACHEFLUSH, dst_hctx->digest, HASH_DGT_BUFSZ); +} + +static const struct crypto_hash_ops ast2600_hace_ops = { + .init = ast2600_hace_init, + .update = ast2600_hace_update, + .final = ast2600_hace_final, + .free_ctx = ast2600_hace_free, + .copy_state = ast2600_hace_copy_state, +}; + +static TEE_Result ast2600_hace_alloc(struct crypto_hash_ctx **pctx, + uint32_t algo) +{ + struct ast2600_hace_ctx *hctx = calloc(1, sizeof(*hctx)); + + if (!hctx) + return TEE_ERROR_OUT_OF_MEMORY; + hctx->buf = memalign(HASH_BLK_BUFSZ, HASH_BLK_BUFSZ); + if (!hctx->buf) + return TEE_ERROR_OUT_OF_MEMORY; + + hctx->digest = memalign(HASH_DGT_BUFSZ, HASH_DGT_BUFSZ); + if (!hctx->digest) + return TEE_ERROR_OUT_OF_MEMORY; + + hctx->hash_ctx.ops = &ast2600_hace_ops; + hctx->algo = algo; + hctx->cmd = HACE_HASH_CMD_ACCUM | HACE_HASH_CMD_SHA_BE; + + switch (algo) { + case TEE_ALG_SHA1: + hctx->dgt_size = 20; + hctx->blk_size = 64; + hctx->pad_size = 8; + hctx->cmd |= HACE_HASH_CMD_ALG_SHA1; + break; + case TEE_ALG_SHA256: + hctx->dgt_size = 32; + hctx->blk_size = 64; + hctx->pad_size = 8; + hctx->cmd |= HACE_HASH_CMD_ALG_SHA256; + break; + case TEE_ALG_SHA384: + hctx->dgt_size = 48; + hctx->blk_size = 128; + hctx->pad_size = 16; + hctx->cmd |= HACE_HASH_CMD_ALG_SHA384; + break; + case TEE_ALG_SHA512: + hctx->dgt_size = 64; + hctx->blk_size = 128; + hctx->pad_size = 16; + hctx->cmd |= HACE_HASH_CMD_ALG_SHA512; + break; + default: + free(hctx); + return TEE_ERROR_NOT_IMPLEMENTED; + } + + *pctx = &hctx->hash_ctx; + + return TEE_SUCCESS; +} + +TEE_Result ast2600_drvcrypt_register_hash(void) +{ + hace_virt = core_mmu_get_va(HACE_BASE, MEM_AREA_IO_SEC, + SMALL_PAGE_SIZE); + if (!hace_virt) { + EMSG("cannot get HACE virtual address"); + return TEE_ERROR_GENERIC; + } + + return drvcrypt_register_hash(ast2600_hace_alloc); +} diff --git a/optee_os/core/drivers/crypto/aspeed/hace_ast2600.h b/optee_os/core/drivers/crypto/aspeed/hace_ast2600.h new file mode 100644 index 0000000..09da7b6 --- /dev/null +++ b/optee_os/core/drivers/crypto/aspeed/hace_ast2600.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Aspeed Technology Inc. + */ +#ifndef __HACE_AST2600_H__ +#define __HACE_AST2600_H__ + +#include + +#ifdef CFG_CRYPTO_DRV_HASH +TEE_Result ast2600_drvcrypt_register_hash(void); +#else +static inline TEE_Result ast2600_drvcrypt_register_hash(void) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif +#endif diff --git a/optee_os/core/drivers/crypto/aspeed/sub.mk b/optee_os/core/drivers/crypto/aspeed/sub.mk new file mode 100644 index 0000000..69f9920 --- /dev/null +++ b/optee_os/core/drivers/crypto/aspeed/sub.mk @@ -0,0 +1,4 @@ +ifeq ($(PLATFORM_FLAVOR),ast2600) +srcs-y += crypto_ast2600.c +srcs-$(CFG_CRYPTO_DRV_HASH) += hace_ast2600.c +endif diff --git a/optee_os/core/drivers/crypto/caam/acipher/caam_dh.c b/optee_os/core/drivers/crypto/caam/acipher/caam_dh.c new file mode 100644 index 0000000..35fc445 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/caam_dh.c @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Implementation of DH + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_CAAM_64BIT +#define MAX_DESC_KEY_GEN 14 +#define MAX_DESC_SHARED 14 +#else +#define MAX_DESC_KEY_GEN 9 +#define MAX_DESC_SHARED 9 +#endif + +/* + * Definition of the local DH Keypair + */ +struct caam_dh_keypair { + struct caambuf g; /* Generator */ + struct caambuf p; /* Prime Number Modulus */ + struct caambuf x; /* Private key */ + struct caambuf y; /* Public key */ +}; + +/* + * Free local DH keypair + * + * @key DH keypair + */ +static void do_keypair_free(struct caam_dh_keypair *key) +{ + caam_free_buf(&key->g); + caam_free_buf(&key->p); + caam_free_buf(&key->x); + caam_free_buf(&key->y); +} + +/* + * Convert Crypto DH Key p and g bignumbers to local buffers + * (via keypair object). + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + */ +static enum caam_status do_keypair_conv_p_g(struct caam_dh_keypair *outkey, + const struct dh_keypair *inkey) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t p_size = 0; + size_t field_size = 0; + + p_size = crypto_bignum_num_bytes(inkey->p); + + DH_TRACE("DH Convert Key Parameters (p,g) size %zu bytes", p_size); + + /* Prime Number Modulus */ + retstatus = caam_calloc_buf(&outkey->p, p_size); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + crypto_bignum_bn2bin(inkey->p, outkey->p.data); + cache_operation(TEE_CACHECLEAN, outkey->p.data, outkey->p.length); + + /* Generator */ + retstatus = caam_calloc_buf(&outkey->g, p_size); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of g to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->g); + crypto_bignum_bn2bin(inkey->g, outkey->g.data + p_size - field_size); + cache_operation(TEE_CACHECLEAN, outkey->g.data, outkey->g.length); + + return CAAM_NO_ERROR; +} + +/* + * Convert Crypto DH Private Key to a local Private Key (via keypair object) + * + * @outkey [out] Output local keypair + * @inkey Input Private key in TEE Crypto format + */ +static enum caam_status do_keypriv_conv(struct caam_dh_keypair *outkey, + const struct dh_keypair *inkey) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t key_size = inkey->xbits / 8; + size_t p_size = 0; + + if (!key_size) + key_size = crypto_bignum_num_bytes(inkey->x); + + DH_TRACE("DH Convert Private Key size %zu bytes", key_size); + + /* Prime */ + p_size = crypto_bignum_num_bytes(inkey->p); + retstatus = caam_calloc_buf(&outkey->p, p_size); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + crypto_bignum_bn2bin(inkey->p, outkey->p.data); + cache_operation(TEE_CACHECLEAN, outkey->p.data, outkey->p.length); + + /* Private Key X */ + retstatus = caam_calloc_buf(&outkey->x, key_size); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + crypto_bignum_bn2bin(inkey->x, outkey->x.data); + cache_operation(TEE_CACHECLEAN, outkey->x.data, outkey->x.length); + + return CAAM_NO_ERROR; +} + +/* + * Convert Crypto DH Public Key to local Public Key (via a keypair object) + * + * @outkey [out] Output local keypair + * @inkey Input Public key in TEE Crypto format + */ +static enum caam_status do_keypub_conv(struct caam_dh_keypair *outkey, + const struct bignum *inkey) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t key_size = 0; + + key_size = crypto_bignum_num_bytes((struct bignum *)inkey); + DH_TRACE("DH Convert Keypair size %zu bytes", key_size); + + /* Public Key Y */ + retstatus = caam_calloc_buf(&outkey->y, key_size); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + crypto_bignum_bn2bin(inkey, outkey->y.data); + cache_operation(TEE_CACHECLEAN, outkey->y.data, outkey->y.length); + + return CAAM_NO_ERROR; +} + +/* + * Allocate a TEE DH keypair. + * Note: The subprime q is not used but it must be allocated to prevent + * system referencing issues when object is destroyed. + * + * @key Keypair + * @size_bits Key size in bits + */ +static TEE_Result do_allocate_keypair(struct dh_keypair *key, size_t size_bits) +{ + DH_TRACE("Allocate Keypair of %zu bits", size_bits); + + /* Initialize the key fields to NULL */ + memset(key, 0, sizeof(*key)); + + /* Allocate Generator Scalar */ + key->g = crypto_bignum_allocate(size_bits); + if (!key->g) + goto err; + + /* Allocate Prime Number Modulus */ + key->p = crypto_bignum_allocate(size_bits); + if (!key->p) + goto err; + + /* Allocate Private key X */ + key->x = crypto_bignum_allocate(size_bits); + if (!key->x) + goto err; + + /* Allocate Public Key Y */ + key->y = crypto_bignum_allocate(size_bits); + if (!key->y) + goto err; + + /* Allocate Subprime even if not used */ + key->q = crypto_bignum_allocate(size_bits); + if (!key->q) + goto err; + + return TEE_SUCCESS; + +err: + DH_TRACE("Allocation error"); + + crypto_bignum_free(&key->g); + crypto_bignum_free(&key->p); + crypto_bignum_free(&key->x); + crypto_bignum_free(&key->y); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Generates an DH keypair + * Keypair @key contains the input prime p and generator g values + * The function calculates private x and public y, knowing that the + * number of bits of x is either key_size if specified or p size. + * + * @key [in/out] Keypair + * @q Sub Prime (not used) + * @key_size Key size in bits multiple of 8 bits + */ +static TEE_Result do_gen_keypair(struct dh_keypair *key, + struct bignum *q __unused, size_t key_size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_dh_keypair caam_dh_key = { }; + struct caambuf dh_r = { }; + size_t n_bytes = key_size / 8; + size_t l_bytes = 0; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + int counter = 0; + + l_bytes = crypto_bignum_num_bytes(key->p); + if (!l_bytes) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * If @key_size not specified, private key size is + * same as the public key size (same as prime size) + */ + if (!n_bytes) + n_bytes = l_bytes; + + /* + * CAAM private key support is limited to the descriptor PDB + * N maximum value (PDB_DL_KEY_N_MASK) + */ + if (n_bytes > PDB_DL_KEY_N_MASK) + n_bytes = PDB_DL_KEY_N_MASK; + + DH_TRACE("Request %zu bits key -> so do %zu bytes key", key_size, + n_bytes); + + /* Allocate the job used to prepare the operation */ + desc = caam_calloc_desc(MAX_DESC_KEY_GEN); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Allocate Private Key to be generated */ + retstatus = caam_calloc_align_buf(&caam_dh_key.x, n_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + cache_operation(TEE_CACHEFLUSH, caam_dh_key.x.data, + caam_dh_key.x.length); + + /* Allocate Public Key to be generated */ + retstatus = caam_calloc_align_buf(&caam_dh_key.y, l_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + cache_operation(TEE_CACHEFLUSH, caam_dh_key.y.data, + caam_dh_key.y.length); + + /* Allocate Private Key modulus (r) and fill it with one's */ + retstatus = caam_calloc_buf(&dh_r, n_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + memset(dh_r.data, UINT8_MAX, dh_r.length); + cache_operation(TEE_CACHECLEAN, dh_r.data, dh_r.length); + + /* Generator and Prime */ + retstatus = do_keypair_conv_p_g(&caam_dh_key, key); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* + * Build the descriptor using the PDB Public Key generation + * block (PD=0) + */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_DL_KEY_L_SIZE(l_bytes) | + PDB_DL_KEY_N_SIZE(n_bytes)); + caam_desc_add_ptr(desc, caam_dh_key.p.paddr); + caam_desc_add_ptr(desc, dh_r.paddr); + caam_desc_add_ptr(desc, caam_dh_key.g.paddr); + caam_desc_add_ptr(desc, caam_dh_key.x.paddr); + caam_desc_add_ptr(desc, caam_dh_key.y.paddr); + caam_desc_add_word(desc, PK_KEYPAIR_GEN(DL)); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + DH_DUMPDESC(desc); + + /* + * If the Secure Key X doesn't have the correct size + * retry a new generation. + * Retry 10 times before returing an error to not lock the system. + */ + for (counter = 0; counter < 10; counter++) { + memset(&jobctx, 0, sizeof(jobctx)); + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + cache_operation(TEE_CACHEINVALIDATE, caam_dh_key.x.data, + caam_dh_key.x.length); + cache_operation(TEE_CACHEINVALIDATE, caam_dh_key.y.data, + caam_dh_key.y.length); + + /* Copy Private and Public keypair */ + ret = crypto_bignum_bin2bn(caam_dh_key.x.data, + caam_dh_key.x.length, + key->x); + if (ret != TEE_SUCCESS) + goto out; + + if (crypto_bignum_num_bytes(key->x) != n_bytes) { + DH_TRACE("Error X size=%zu expected %zu", + crypto_bignum_num_bytes(key->x), + n_bytes); + DH_DUMPBUF("X", caam_dh_key.x.data, + caam_dh_key.x.length); + DH_DUMPBUF("Y", caam_dh_key.y.data, + caam_dh_key.y.length); + continue; + } + + ret = crypto_bignum_bin2bn(caam_dh_key.y.data, + caam_dh_key.y.length, + key->y); + if (ret != TEE_SUCCESS) + goto out; + + /* Set the Private Key size in bits */ + key->xbits = n_bytes * 8; + + ret = TEE_SUCCESS; + goto out; + } else { + DH_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + goto out; + } + } + +out: + caam_free_desc(&desc); + caam_free_buf(&dh_r); + do_keypair_free(&caam_dh_key); + + return ret; +} + +/* + * Compute the shared secret data from DH Private key and Public Key + * + * @sdata [in/out] DH Shared Secret data + */ +static TEE_Result do_shared_secret(struct drvcrypt_secret_data *sdata) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct dh_keypair *inkeypair = sdata->key_priv; + struct caam_dh_keypair caam_dh_key = { }; + struct caamdmaobj secret = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + uint32_t pdb_sgt_flags = 0; + + DH_TRACE("DH Shared Secret"); + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(MAX_DESC_SHARED); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* + * ReAllocate the secret result buffer with a maximum size + * of the secret size if not cache aligned + */ + ret = caam_dmaobj_output_sgtbuf(&secret, sdata->secret.data, + sdata->secret.length, + sdata->secret.length); + if (ret) + goto out; + + if (secret.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKDH_SECRET; + + caam_dmaobj_cache_push(&secret); + + /* Convert the Private key to local key */ + retstatus = do_keypriv_conv(&caam_dh_key, inkeypair); + if (retstatus != CAAM_NO_ERROR) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Convert the Public key to local key */ + retstatus = do_keypub_conv(&caam_dh_key, sdata->key_pub); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* + * Build the descriptor using PDB Shared Secret + */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, pdb_sgt_flags | + PDB_DL_KEY_L_SIZE(caam_dh_key.y.length) | + PDB_DL_KEY_N_SIZE(caam_dh_key.x.length)); + /* Prime */ + caam_desc_add_ptr(desc, caam_dh_key.p.paddr); + /* Modulus - Not used */ + caam_desc_add_ptr(desc, 0); + /* Public key */ + caam_desc_add_ptr(desc, caam_dh_key.y.paddr); + /* Private key */ + caam_desc_add_ptr(desc, caam_dh_key.x.paddr); + /* Output secret */ + caam_desc_add_ptr(desc, secret.sgtbuf.paddr); + + caam_desc_add_word(desc, SHARED_SECRET(DL)); + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + DH_DUMPDESC(desc); + jobctx.desc = desc; + + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + sdata->secret.length = caam_dmaobj_copy_to_orig(&secret); + + DH_DUMPBUF("Secret", sdata->secret.data, sdata->secret.length); + ret = caam_status_to_tee_result(retstatus); + } else { + DH_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_free_desc(&desc); + do_keypair_free(&caam_dh_key); + caam_dmaobj_free(&secret); + + return ret; +} + +/* + * Registration of the ECC Driver + */ +static struct drvcrypt_dh driver_dh = { + .alloc_keypair = do_allocate_keypair, + .gen_keypair = do_gen_keypair, + .shared_secret = do_shared_secret, +}; + +enum caam_status caam_dh_init(struct caam_jrcfg *caam_jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + vaddr_t jr_base = caam_jrcfg->base + caam_jrcfg->offset; + + if (caam_hal_ctrl_pknum(jr_base) && + drvcrypt_register_dh(&driver_dh) == TEE_SUCCESS) + retstatus = CAAM_NO_ERROR; + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/acipher/caam_dsa.c b/optee_os/core/drivers/crypto/caam/acipher/caam_dsa.c new file mode 100644 index 0000000..d60bb8e --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/caam_dsa.c @@ -0,0 +1,777 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019-2021 NXP + * + * Implementation of DSA functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +#ifdef CFG_CAAM_64BIT +#define MAX_DESC_KEY_GEN 14 +#define MAX_DESC_SIGN 19 +#define MAX_DESC_VERIFY 21 +#else +#define MAX_DESC_KEY_GEN 9 +#define MAX_DESC_SIGN 12 +#define MAX_DESC_VERIFY 13 +#endif + +/* + * Definition of the local DSA Keypair + * Domain Parameters (p, q, g) + * Private Key format (x) + * Public Key format (y) + */ +struct caam_dsa_keypair { + struct caambuf g; /* Generator */ + struct caambuf p; /* Prime Number (L bits) */ + struct caambuf q; /* Subprime Number (N bits) */ + struct caambuf x; /* Private key */ + struct caambuf y; /* Public key */ +}; + +/* + * Free local DSA keypair + * + * @key DSA keypair + */ +static void do_keypair_free(struct caam_dsa_keypair *key) +{ + caam_free_buf(&key->g); + caam_free_buf(&key->p); + caam_free_buf(&key->q); + caam_free_buf(&key->x); + caam_free_buf(&key->y); +} + +/* + * If all DSA parameters p, q and g are present, convert them from bignumbers + * to local buffers (via keypair object). Otherwise generate them. + * + * @outkey [out] Output keypair in local format + * @key Input key in TEE Crypto format + * @l_bytes Prime p size in bytes + * @n_bytes Subprime q size in bytes + */ +static TEE_Result get_keypair_domain_params(struct caam_dsa_keypair *outkey, + const struct dsa_keypair *key, + size_t l_bytes, size_t n_bytes) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t in_q_size = 0; + size_t in_p_size = 0; + size_t in_g_size = 0; + struct prime_data_dsa prime = { }; + + DSA_TRACE("DSA conv key param (p, g) of %zu bytes and (q) of %zu bytes", + l_bytes, n_bytes); + + retstatus = caam_calloc_buf(&outkey->q, n_bytes); + if (retstatus != CAAM_NO_ERROR) + return caam_status_to_tee_result(retstatus); + + retstatus = caam_calloc_buf(&outkey->g, l_bytes); + if (retstatus != CAAM_NO_ERROR) + return caam_status_to_tee_result(retstatus); + + retstatus = caam_calloc_buf(&outkey->p, l_bytes); + if (retstatus != CAAM_NO_ERROR) + return caam_status_to_tee_result(retstatus); + + /* + * Get all inputs parameters size, if one of them is not + * define generate new parameters + */ + in_g_size = crypto_bignum_num_bytes(key->g); + in_p_size = crypto_bignum_num_bytes(key->p); + in_q_size = crypto_bignum_num_bytes(key->q); + + if (!in_q_size || !in_g_size || !in_p_size) { + /* Generate DSA parameters: Generator G and Primes P/Q */ + prime.g = &outkey->g; + prime.p = &outkey->p; + prime.q = &outkey->q; + + retstatus = caam_prime_dsa_gen(&prime); + DSA_TRACE("Generate G and Primes P/Q returned %#x", retstatus); + + if (retstatus != CAAM_NO_ERROR) + return caam_status_to_tee_result(retstatus); + + /* Copy Generated DSA Parameter */ + crypto_bignum_bin2bn(outkey->q.data, outkey->q.length, key->q); + crypto_bignum_bin2bn(outkey->g.data, outkey->g.length, key->g); + crypto_bignum_bin2bn(outkey->p.data, outkey->p.length, key->p); + + } else { + DSA_TRACE("Prime Q is defined"); + + crypto_bignum_bn2bin(key->q, + outkey->q.data + n_bytes - in_q_size); + cache_operation(TEE_CACHECLEAN, outkey->q.data, + outkey->q.length); + + DSA_TRACE("Prime G is defined"); + crypto_bignum_bn2bin(key->g, + outkey->g.data + l_bytes - in_g_size); + cache_operation(TEE_CACHECLEAN, outkey->g.data, + outkey->g.length); + + DSA_TRACE("Prime P is defined"); + crypto_bignum_bn2bin(key->p, + outkey->p.data + l_bytes - in_p_size); + cache_operation(TEE_CACHECLEAN, outkey->p.data, + outkey->p.length); + } + + return TEE_SUCCESS; +} + +/* + * Convert Crypto DSA Private Key to local Keypair Key + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + * @l_bytes Prime p size in bytes + * @n_bytes Subprime q size in bytes + */ +static enum caam_status do_keypriv_conv(struct caam_dsa_keypair *outkey, + const struct dsa_keypair *inkey, + size_t l_bytes, size_t n_bytes) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t field_size = 0; + + DSA_TRACE("DSA Convert Key Private size l=%zu bytes, n=%zu bytes", + l_bytes, n_bytes); + + /* Generator */ + retstatus = caam_calloc_buf(&outkey->g, l_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of g to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->g); + crypto_bignum_bn2bin(inkey->g, outkey->g.data + l_bytes - field_size); + + /* Prime Number Modulus */ + retstatus = caam_calloc_buf(&outkey->p, l_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of p to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->p); + crypto_bignum_bn2bin(inkey->p, outkey->p.data + l_bytes - field_size); + + /* Subprime Number Modulus */ + retstatus = caam_calloc_buf(&outkey->q, n_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of q to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->q); + crypto_bignum_bn2bin(inkey->q, outkey->q.data + n_bytes - field_size); + + /* Private key is only scalar x of n bytes */ + retstatus = caam_calloc_buf(&outkey->x, n_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of x to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->x); + crypto_bignum_bn2bin(inkey->x, outkey->x.data + n_bytes - field_size); + + cache_operation(TEE_CACHECLEAN, outkey->g.data, outkey->g.length); + cache_operation(TEE_CACHECLEAN, outkey->p.data, outkey->p.length); + cache_operation(TEE_CACHECLEAN, outkey->q.data, outkey->q.length); + cache_operation(TEE_CACHECLEAN, outkey->x.data, outkey->x.length); + + return CAAM_NO_ERROR; +} + +/* + * Convert Crypto DSA Public Key to local DSA Keypair Key + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + * @l_bytes Prime p size in bytes + * @n_bytes Subprime q size in bytes + */ +static enum caam_status do_keypub_conv(struct caam_dsa_keypair *outkey, + const struct dsa_public_key *inkey, + size_t l_bytes, size_t n_bytes) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t field_size = 0; + + DSA_TRACE("DSA Convert Public Key size l=%zu bytes, n=%zu bytes", + l_bytes, n_bytes); + + /* Generator */ + retstatus = caam_calloc_buf(&outkey->g, l_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of g to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->g); + crypto_bignum_bn2bin(inkey->g, outkey->g.data + l_bytes - field_size); + + /* Prime Number Modulus */ + retstatus = caam_calloc_buf(&outkey->p, l_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of p to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->p); + crypto_bignum_bn2bin(inkey->p, outkey->p.data + l_bytes - field_size); + + /* Subprime Number Modulus */ + retstatus = caam_calloc_buf(&outkey->q, n_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of q to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->q); + crypto_bignum_bn2bin(inkey->q, outkey->q.data + n_bytes - field_size); + + /* Public key is only scalar y of l bytes */ + retstatus = caam_calloc_buf(&outkey->y, l_bytes); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of y to pad with 0's */ + field_size = crypto_bignum_num_bytes(inkey->y); + crypto_bignum_bn2bin(inkey->y, outkey->y.data + l_bytes - field_size); + + cache_operation(TEE_CACHECLEAN, outkey->g.data, outkey->g.length); + cache_operation(TEE_CACHECLEAN, outkey->p.data, outkey->p.length); + cache_operation(TEE_CACHECLEAN, outkey->q.data, outkey->q.length); + cache_operation(TEE_CACHECLEAN, outkey->y.data, outkey->y.length); + + return CAAM_NO_ERROR; +} + +/* + * Allocate a TEE DSA keypair. + * + * @key Keypair + * @l_bits L bits size (prime p size) + * @n_bits N bits size (subprime q size) + */ +static TEE_Result do_allocate_keypair(struct dsa_keypair *key, size_t l_bits, + size_t n_bits) +{ + DSA_TRACE("DSA allocate Keypair of L=%zu bits and N=%zu bits", l_bits, + n_bits); + + /* Initialize the key fields to NULL */ + memset(key, 0, sizeof(*key)); + + /* Allocate Generator Scalar */ + key->g = crypto_bignum_allocate(l_bits); + if (!key->g) + goto err; + + /* Allocate Prime Number Modulus */ + key->p = crypto_bignum_allocate(l_bits); + if (!key->p) + goto err; + + /* Allocate Prime Number Modulus */ + key->q = crypto_bignum_allocate(n_bits); + if (!key->q) + goto err; + + /* Allocate Private key X */ + key->x = crypto_bignum_allocate(n_bits); + if (!key->x) + goto err; + + /* Allocate Public Key Y */ + key->y = crypto_bignum_allocate(l_bits); + if (!key->y) + goto err; + + return TEE_SUCCESS; + +err: + DSA_TRACE("Allocation error"); + + crypto_bignum_free(&key->g); + crypto_bignum_free(&key->p); + crypto_bignum_free(&key->q); + crypto_bignum_free(&key->x); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Allocate a DSA Public Key + * + * @key Public Key + * @l_bits L bits size (prime p size) + * @n_bits N bits size (subprime q size) + */ +static TEE_Result do_allocate_publickey(struct dsa_public_key *key, + size_t l_bits, size_t n_bits) +{ + DSA_TRACE("DSA Allocate Public of L=%zu bits and N=%zu bits", l_bits, + n_bits); + + /* Initialize the key fields to NULL */ + memset(key, 0, sizeof(*key)); + + /* Allocate Generator Scalar */ + key->g = crypto_bignum_allocate(l_bits); + if (!key->g) + goto err; + + /* Allocate Prime Number Modulus */ + key->p = crypto_bignum_allocate(l_bits); + if (!key->p) + goto err; + + /* Allocate Prime Number Modulus */ + key->q = crypto_bignum_allocate(n_bits); + if (!key->q) + goto err; + + /* Allocate Public Key Y */ + key->y = crypto_bignum_allocate(l_bits); + if (!key->y) + goto err; + + return TEE_SUCCESS; + +err: + DSA_TRACE("Allocation error"); + + crypto_bignum_free(&key->g); + crypto_bignum_free(&key->p); + crypto_bignum_free(&key->q); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Generates an DSA keypair + * Keypair @key contains the input primes p, g and generator g values + * The function computes private x and public y. + * + * @key [in/out] Keypair + * @l_bits L bits size (prime p size) + * @n_bits N bits size (subprime q size) + */ +static TEE_Result do_gen_keypair(struct dsa_keypair *key, size_t l_bits, + size_t n_bits) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_dsa_keypair caam_dsa_key = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + size_t l_bytes = l_bits / 8; + size_t n_bytes = n_bits / 8; + + DSA_TRACE("Generate Key - Private (%zu bits) and Public (%zu bits)", + n_bits, l_bits); + + /* Allocate the job used to prepare the operation */ + desc = caam_calloc_desc(MAX_DESC_KEY_GEN); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Allocate Private Key to be generated */ + retstatus = caam_calloc_align_buf(&caam_dsa_key.x, n_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + cache_operation(TEE_CACHEFLUSH, caam_dsa_key.x.data, + caam_dsa_key.x.length); + + /* Allocate Public Key to be generated */ + retstatus = caam_calloc_align_buf(&caam_dsa_key.y, l_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + cache_operation(TEE_CACHEFLUSH, caam_dsa_key.y.data, + caam_dsa_key.y.length); + + /* Generator and Prime */ + ret = get_keypair_domain_params(&caam_dsa_key, key, l_bytes, n_bytes); + if (ret != TEE_SUCCESS) + goto out; + + /* + * Build the descriptor using the PDB Public Key generation + * block (PD=0) + */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_DL_KEY_L_SIZE(l_bytes) | + PDB_DL_KEY_N_SIZE(n_bytes)); + caam_desc_add_ptr(desc, caam_dsa_key.p.paddr); + caam_desc_add_ptr(desc, caam_dsa_key.q.paddr); + caam_desc_add_ptr(desc, caam_dsa_key.g.paddr); + caam_desc_add_ptr(desc, caam_dsa_key.x.paddr); + caam_desc_add_ptr(desc, caam_dsa_key.y.paddr); + caam_desc_add_word(desc, PK_KEYPAIR_GEN(DL)); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + DSA_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + cache_operation(TEE_CACHEINVALIDATE, caam_dsa_key.x.data, + caam_dsa_key.x.length); + cache_operation(TEE_CACHEINVALIDATE, caam_dsa_key.y.data, + caam_dsa_key.y.length); + + /* Copy Private and Public keypair */ + ret = crypto_bignum_bin2bn(caam_dsa_key.x.data, + caam_dsa_key.x.length, key->x); + if (ret != TEE_SUCCESS) + goto out; + + ret = crypto_bignum_bin2bn(caam_dsa_key.y.data, + caam_dsa_key.y.length, key->y); + if (ret != TEE_SUCCESS) + goto out; + + DSA_DUMPBUF("X", caam_dsa_key.x.data, caam_dsa_key.x.length); + DSA_DUMPBUF("Y", caam_dsa_key.y.data, caam_dsa_key.y.length); + } else { + DSA_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_free_desc(&desc); + do_keypair_free(&caam_dsa_key); + + return ret; +} + +/* + * Signature of DSA message + * Note : the message to sign is already hashed + * + * @sdata [in/out] DSA data to sign / Signature + * @l_bytes L bytes size (prime p size) + * @n_bytes N bytes size (subprime q size) + */ +static TEE_Result do_sign(struct drvcrypt_sign_data *sdata, size_t l_bytes, + size_t n_bytes) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct dsa_keypair *inkey = sdata->key; + struct caam_dsa_keypair dsakey = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + struct caamdmaobj msg = { }; + size_t sign_len = 0; + struct caamdmaobj sign_c = { }; + struct caamdmaobj sign_d = { }; + uint32_t pdb_sgt_flags = 0; + + DSA_TRACE("DSA Signature"); + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(MAX_DESC_SIGN); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Convert the private key to a local key */ + retstatus = do_keypriv_conv(&dsakey, inkey, l_bytes, n_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* Prepare the input message CAAM Descriptor entry */ + ret = caam_dmaobj_input_sgtbuf(&msg, sdata->message.data, + sdata->message.length); + if (ret) + goto out; + + if (msg.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKSIGN_MSG; + + caam_dmaobj_cache_push(&msg); + + DSA_DUMPBUF("Message", sdata->message.data, sdata->message.length); + + /* + * Re-allocate the signature result buffer with a maximum size + * of the roundup to 16 bytes of the secure size in bytes if + * the signature buffer is not aligned or too short. + * + * - 1st Part: size_sec + * - 2nd Part: size_sec roundup to 16 bytes + */ + sign_len = ROUNDUP(sdata->size_sec, 16) + sdata->size_sec; + + ret = caam_dmaobj_output_sgtbuf(&sign_c, sdata->signature.data, + sdata->signature.length, sign_len); + if (ret) + goto out; + + if (sign_c.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKSIGN_SIGN_C; + + /* Prepare the 2nd Part of the signature. Derive from sign_c */ + ret = caam_dmaobj_derive_sgtbuf(&sign_d, &sign_c, sdata->size_sec, + ROUNDUP(sdata->size_sec, 16)); + if (ret) + goto out; + + if (sign_d.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKSIGN_SIGN_D; + + caam_dmaobj_cache_push(&sign_c); + + /* + * Build the descriptor using Predifined ECC curve + */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_DSA_SIGN_N(n_bytes) | + PDB_DSA_SIGN_L(l_bytes) | pdb_sgt_flags); + /* Prime number */ + caam_desc_add_ptr(desc, dsakey.p.paddr); + /* Prime Modulus */ + caam_desc_add_ptr(desc, dsakey.q.paddr); + /* Generator */ + caam_desc_add_ptr(desc, dsakey.g.paddr); + /* Secret key */ + caam_desc_add_ptr(desc, dsakey.x.paddr); + /* Input message */ + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + /* Signature 1st part */ + caam_desc_add_ptr(desc, sign_c.sgtbuf.paddr); + /* Signature 2nd part */ + caam_desc_add_ptr(desc, sign_d.sgtbuf.paddr); + /* Message length */ + caam_desc_add_word(desc, sdata->message.length); + + caam_desc_add_word(desc, DSA_SIGN(DL)); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + ECC_DUMPDESC(desc); + + jobctx.desc = desc; + + retstatus = caam_jr_enqueue(&jobctx, NULL); + if (retstatus == CAAM_NO_ERROR) { + /* Limit the copy to 2 * sdata->size_sec */ + sign_c.orig.length = 2 * sdata->size_sec; + sdata->signature.length = caam_dmaobj_copy_to_orig(&sign_c); + + DSA_DUMPBUF("Signature", sdata->signature.data, + sdata->signature.length); + + ret = caam_status_to_tee_result(retstatus); + } else { + DSA_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_free_desc(&desc); + do_keypair_free(&dsakey); + caam_dmaobj_free(&msg); + caam_dmaobj_free(&sign_c); + caam_dmaobj_free(&sign_d); + + return ret; +} + +/* + * Verification of the Signature of DSA message + * Note the message is already hashed + * + * @sdata [in/out] DSA Signature to verify + * @l_bytes L bytes size (prime p size) + * @n_bytes N bytes size (subprime q size) + */ +static TEE_Result do_verify(struct drvcrypt_sign_data *sdata, size_t l_bytes, + size_t n_bytes) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct dsa_public_key *inkey = sdata->key; + struct caam_dsa_keypair dsakey = { }; + struct caambuf tmp = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + struct caamdmaobj msg = { }; + struct caamdmaobj sign_c = { }; + struct caamdmaobj sign_d = { }; + uint32_t pdb_sgt_flags = 0; + + DSA_TRACE("DSA Verify"); + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(MAX_DESC_VERIFY); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Convert the Public key to local key */ + retstatus = do_keypub_conv(&dsakey, inkey, l_bytes, n_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* Prepare the input message CAAM Descriptor entry */ + ret = caam_dmaobj_input_sgtbuf(&msg, sdata->message.data, + sdata->message.length); + if (ret) + goto out; + + if (msg.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKVERIF_MSG; + + caam_dmaobj_cache_push(&msg); + + /* + * Prepare the 1st Part of the signature + * Handle the full signature in case signature buffer needs to + * be reallocated. + */ + ret = caam_dmaobj_input_sgtbuf(&sign_c, sdata->signature.data, + sdata->signature.length); + if (ret) + goto out; + + if (sign_c.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKVERIF_SIGN_C; + + /* Prepare the 2nd Part of the signature, derive from sign_c */ + ret = caam_dmaobj_derive_sgtbuf(&sign_d, &sign_c, sdata->size_sec, + sdata->size_sec); + if (ret) + goto out; + + if (sign_d.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKVERIF_SIGN_D; + + caam_dmaobj_cache_push(&sign_c); + + /* Allocate a Temporary buffer used by the CAAM */ + retstatus = caam_alloc_align_buf(&tmp, l_bytes); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* + * Build the descriptor using Predifined ECC curve + */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_DSA_VERIF_N(n_bytes) | + PDB_DSA_VERIF_L(l_bytes) | pdb_sgt_flags); + /* Prime number */ + caam_desc_add_ptr(desc, dsakey.p.paddr); + /* Prime Modulus */ + caam_desc_add_ptr(desc, dsakey.q.paddr); + /* Generator */ + caam_desc_add_ptr(desc, dsakey.g.paddr); + /* Public key */ + caam_desc_add_ptr(desc, dsakey.y.paddr); + /* Input message */ + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + /* Signature 1st part */ + caam_desc_add_ptr(desc, sign_c.sgtbuf.paddr); + /* Signature 2nd part */ + caam_desc_add_ptr(desc, sign_d.sgtbuf.paddr); + /* Temporary buffer */ + caam_desc_add_ptr(desc, tmp.paddr); + /* Message length */ + caam_desc_add_word(desc, sdata->message.length); + + caam_desc_add_word(desc, DSA_VERIFY(DL)); + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + DSA_DUMPDESC(desc); + + jobctx.desc = desc; + + cache_operation(TEE_CACHEFLUSH, tmp.data, tmp.length); + + retstatus = caam_jr_enqueue(&jobctx, NULL); + if (retstatus == CAAM_JOB_STATUS && !jobctx.status) { + DSA_TRACE("DSA Verify Status 0x%08" PRIx32, jobctx.status); + ret = TEE_ERROR_SIGNATURE_INVALID; + } else if (retstatus != CAAM_NO_ERROR) { + DSA_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } else { + ret = caam_status_to_tee_result(retstatus); + } + +out: + caam_free_desc(&desc); + do_keypair_free(&dsakey); + caam_free_buf(&tmp); + caam_dmaobj_free(&msg); + caam_dmaobj_free(&sign_c); + caam_dmaobj_free(&sign_d); + + return ret; +} + +/* + * Registration of the DSA Driver + */ +static struct drvcrypt_dsa driver_dsa = { + .alloc_keypair = do_allocate_keypair, + .alloc_publickey = do_allocate_publickey, + .gen_keypair = do_gen_keypair, + .sign = do_sign, + .verify = do_verify, +}; + +enum caam_status caam_dsa_init(struct caam_jrcfg *caam_jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + vaddr_t jr_base = caam_jrcfg->base + caam_jrcfg->offset; + + if (caam_hal_ctrl_pknum(jr_base) && + drvcrypt_register_dsa(&driver_dsa) == TEE_SUCCESS) + retstatus = CAAM_NO_ERROR; + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/acipher/caam_ecc.c b/optee_os/core/drivers/crypto/caam/acipher/caam_ecc.c new file mode 100644 index 0000000..62c113c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/caam_ecc.c @@ -0,0 +1,719 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Implementation of ECC functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_CAAM_64BIT +#define MAX_DESC_KEY_GEN 8 +#define MAX_DESC_SIGN 13 +#define MAX_DESC_VERIFY 15 +#define MAX_DESC_SHARED 10 +#else +#define MAX_DESC_KEY_GEN 6 +#define MAX_DESC_SIGN 9 +#define MAX_DESC_VERIFY 10 +#define MAX_DESC_SHARED 7 +#endif + +/* + * Definition of the local ECC Keypair + * Public Key format (x, y) + * Private Key format (d) + */ +struct caam_ecc_keypair { + struct caambuf xy; + struct caambuf d; +}; + +/* + * Free local ECC keypair + * + * @key ECC keypair + */ +static void do_keypair_free(struct caam_ecc_keypair *key) +{ + caam_free_buf(&key->xy); + caam_free_buf(&key->d); +} + +/* + * Convert Crypto ECC Key to local ECC Public Key + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + * @size_sec Security size in bytes + */ +static enum caam_status do_keypub_conv(struct caam_ecc_keypair *outkey, + const struct ecc_public_key *inkey, + size_t size_sec) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t x_size = 0; + size_t y_size = 0; + + ECC_TRACE("ECC Convert Public Key size %zu bytes", size_sec); + + /* Point (x y) is twice security key size */ + retstatus = caam_calloc_buf(&outkey->xy, 2 * size_sec); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Copy x and y and get the number of bytes to pad with 0's */ + x_size = crypto_bignum_num_bytes(inkey->x); + crypto_bignum_bn2bin(inkey->x, outkey->xy.data + size_sec - x_size); + + y_size = crypto_bignum_num_bytes(inkey->y); + crypto_bignum_bn2bin(inkey->y, outkey->xy.data + 2 * size_sec - y_size); + + cache_operation(TEE_CACHECLEAN, outkey->xy.data, outkey->xy.length); + + return CAAM_NO_ERROR; +} + +/* + * Convert Crypto ECC Key to local ECC Keypair Key + * Don't convert the exponent e not used in decrytion + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + * @size_sec Security size in bytes + */ +static enum caam_status do_keypair_conv(struct caam_ecc_keypair *outkey, + const struct ecc_keypair *inkey, + size_t size_sec) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + size_t d_size = 0; + + ECC_TRACE("ECC Convert Keypair size %zu bytes", size_sec); + + /* Private key is only scalar d of sec_size bytes */ + retstatus = caam_calloc_buf(&outkey->d, size_sec); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + /* Get the number of bytes of d to pad with 0's */ + d_size = crypto_bignum_num_bytes(inkey->d); + crypto_bignum_bn2bin(inkey->d, outkey->d.data + size_sec - d_size); + + cache_operation(TEE_CACHECLEAN, outkey->d.data, outkey->d.length); + + return CAAM_NO_ERROR; +} + +/* + * Convert TEE ECC Curve to CAAM ECC Curve + * + * @tee_curve TEE ECC Curve + */ +static enum caam_ecc_curve get_caam_curve(uint32_t tee_curve) +{ + enum caam_ecc_curve caam_curve = CAAM_ECC_UNKNOWN; + + if (tee_curve > 0 && + tee_curve < CAAM_ECC_MAX + TEE_ECC_CURVE_NIST_P192) { + /* + * Realign TEE Curve assuming NIST_P192 is the first entry in + * the list of supported ECC curves. + */ + caam_curve = tee_curve - TEE_ECC_CURVE_NIST_P192 + + CAAM_ECC_P192; + } + + return caam_curve; +} + +/* + * Allocate a ECC keypair + * + * @key Keypair + * @size_bits Key size in bits + */ +static TEE_Result do_allocate_keypair(struct ecc_keypair *key, + uint32_t type __unused, + size_t size_bits) +{ + ECC_TRACE("Allocate Keypair of %zu bits", size_bits); + + /* Initialize the key fields to NULL */ + memset(key, 0, sizeof(*key)); + + /* Allocate Secure Scalar */ + key->d = crypto_bignum_allocate(size_bits); + if (!key->d) + goto err; + + /* Allocate Public coordinate X */ + key->x = crypto_bignum_allocate(size_bits); + if (!key->x) + goto err; + + /* Allocate Public coordinate Y */ + key->y = crypto_bignum_allocate(size_bits); + if (!key->y) + goto err; + + return TEE_SUCCESS; + +err: + ECC_TRACE("Allocation error"); + + crypto_bignum_free(&key->d); + crypto_bignum_free(&key->x); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Allocate an ECC Public Key + * + * @key Public Key + * @size_bits Key size in bits + */ +static TEE_Result do_allocate_publickey(struct ecc_public_key *key, + uint32_t type __unused, + size_t size_bits) +{ + ECC_TRACE("Allocate Public Key of %zu bits", size_bits); + + /* Initialize the key fields to NULL */ + memset(key, 0, sizeof(*key)); + + /* Allocate Public coordinate X */ + key->x = crypto_bignum_allocate(size_bits); + if (!key->x) + goto err; + + /* Allocate Public coordinate Y */ + key->y = crypto_bignum_allocate(size_bits); + if (!key->y) + goto err; + + return TEE_SUCCESS; + +err: + ECC_TRACE("Allocation error"); + + crypto_bignum_free(&key->x); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Free an ECC public key + * + * @key Public Key + */ +static void do_free_publickey(struct ecc_public_key *key) +{ + crypto_bignum_free(&key->x); + crypto_bignum_free(&key->y); +} + +/* + * Generate ECC keypair + * + * @key [out] Keypair + * @key_size Key size in bits multiple of 8 bits + */ +static TEE_Result do_gen_keypair(struct ecc_keypair *key, size_t key_size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + enum caam_ecc_curve curve = CAAM_ECC_UNKNOWN; + struct caambuf d = { }; + struct caambuf xy = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + + ECC_TRACE("Generate Keypair of %zu bits", key_size); + + /* The key size must be a multiple of 8 bits */ + key_size = ROUNDUP(key_size, 8); + + /* Verify first if the curve is supported */ + curve = get_caam_curve(key->curve); + if (curve == CAAM_ECC_UNKNOWN) + return TEE_ERROR_BAD_PARAMETERS; + + /* Allocate the job used to prepare the operation */ + desc = caam_calloc_desc(MAX_DESC_KEY_GEN); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* + * Allocate secure and public keys in one buffer + * Secure key size = key_size align in bytes + * Public key size = (key_size * 2) align in bytes + */ + retstatus = caam_alloc_align_buf(&d, (key_size / 8) * 3); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* Build the xy buffer to simplify the code */ + xy.data = d.data + key_size / 8; + xy.length = 2 * (key_size / 8); + xy.paddr = d.paddr + key_size / 8; + + /* Build the descriptor using Predifined ECC curve */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_PKGEN_PD1 | PDB_ECC_ECDSEL(curve)); + caam_desc_add_ptr(desc, d.paddr); + caam_desc_add_ptr(desc, xy.paddr); + caam_desc_add_word(desc, PK_KEYPAIR_GEN(ECC)); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + ECC_DUMPDESC(desc); + + jobctx.desc = desc; + cache_operation(TEE_CACHEFLUSH, d.data, d.length); + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + cache_operation(TEE_CACHEINVALIDATE, d.data, d.length); + + /* Copy all keypair parameters */ + ret = crypto_bignum_bin2bn(d.data, key_size / 8, key->d); + if (ret != TEE_SUCCESS) + goto out; + + ret = crypto_bignum_bin2bn(xy.data, xy.length / 2, key->x); + if (ret != TEE_SUCCESS) + goto out; + + ret = crypto_bignum_bin2bn(xy.data + xy.length / 2, + xy.length / 2, key->y); + if (ret != TEE_SUCCESS) + goto out; + + ECC_DUMPBUF("D", d.data, key_size / 8); + ECC_DUMPBUF("X", xy.data, xy.length / 2); + ECC_DUMPBUF("Y", xy.data + xy.length / 2, xy.length / 2); + } else { + ECC_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_free_desc(&desc); + caam_free_buf(&d); + + return ret; +} + +/* + * Signature of ECC message + * Note the message to sign is already hashed + * + * @sdata [in/out] ECC data to sign / Signature + */ +static TEE_Result do_sign(struct drvcrypt_sign_data *sdata) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + enum caam_ecc_curve curve = CAAM_ECC_UNKNOWN; + struct ecc_keypair *inkey = sdata->key; + struct caam_ecc_keypair ecckey = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + struct caamdmaobj msg = { }; + size_t sign_len = 0; + struct caamdmaobj sign_c = { }; + struct caamdmaobj sign_d = { }; + uint32_t pdb_sgt_flags = 0; + + ECC_TRACE("ECC Signature"); + + /* Verify first if the curve is supported */ + curve = get_caam_curve(inkey->curve); + if (curve == CAAM_ECC_UNKNOWN) + return TEE_ERROR_BAD_PARAMETERS; + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(MAX_DESC_SIGN); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Convert the private key to a local key */ + retstatus = do_keypair_conv(&ecckey, inkey, sdata->size_sec); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* Prepare the input message CAAM Descriptor entry */ + ret = caam_dmaobj_input_sgtbuf(&msg, sdata->message.data, + sdata->message.length); + if (ret) + goto out; + + if (msg.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKSIGN_MSG; + + caam_dmaobj_cache_push(&msg); + + ECC_DUMPBUF("Message", sdata->message.data, sdata->message.length); + + /* + * ReAllocate the signature result buffer with a maximum size + * of the roundup to 16 bytes of the secure size in bytes if + * the signature buffer is not aligned or too short. + * + * - 1st Part: size_sec + * - 2nd Part: size_sec roundup to 16 bytes + */ + sign_len = ROUNDUP(sdata->size_sec, 16) + sdata->size_sec; + + ret = caam_dmaobj_output_sgtbuf(&sign_c, sdata->signature.data, + sdata->signature.length, sign_len); + if (ret) + goto out; + + if (sign_c.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKSIGN_SIGN_C; + + /* Derive sign_d from created sign_c DMA object */ + ret = caam_dmaobj_derive_sgtbuf(&sign_d, &sign_c, sdata->size_sec, + ROUNDUP(sdata->size_sec, 16)); + if (ret) + goto out; + + if (sign_d.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKSIGN_SIGN_D; + + caam_dmaobj_cache_push(&sign_c); + + /* Build the descriptor using Predifined ECC curve */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_PKSIGN_PD1 | PDB_ECC_ECDSEL(curve) | + pdb_sgt_flags); + /* Secret key */ + caam_desc_add_ptr(desc, ecckey.d.paddr); + /* Input message */ + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + /* Signature 1st part */ + caam_desc_add_ptr(desc, sign_c.sgtbuf.paddr); + /* Signature 2nd part */ + caam_desc_add_ptr(desc, sign_d.sgtbuf.paddr); + /* Message length */ + caam_desc_add_word(desc, sdata->message.length); + + caam_desc_add_word(desc, DSA_SIGN(ECC)); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + ECC_DUMPDESC(desc); + + jobctx.desc = desc; + + retstatus = caam_jr_enqueue(&jobctx, NULL); + if (retstatus == CAAM_NO_ERROR) { + sign_c.orig.length = 2 * sdata->size_sec; + sdata->signature.length = caam_dmaobj_copy_to_orig(&sign_c); + + ECC_DUMPBUF("Signature", sdata->signature.data, + sdata->signature.length); + + ret = caam_status_to_tee_result(retstatus); + } else { + ECC_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_free_desc(&desc); + do_keypair_free(&ecckey); + caam_dmaobj_free(&msg); + caam_dmaobj_free(&sign_d); + caam_dmaobj_free(&sign_c); + + return ret; +} + +/* + * Verification of the Signature of ECC message + * Note the message is already hashed + * + * @sdata [in/out] ECC Signature to verify + */ +static TEE_Result do_verify(struct drvcrypt_sign_data *sdata) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + enum caam_ecc_curve curve = CAAM_ECC_UNKNOWN; + struct ecc_public_key *inkey = sdata->key; + struct caam_ecc_keypair ecckey = { }; + struct caambuf tmp = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + struct caamdmaobj msg = { }; + struct caamdmaobj sign_c = { }; + struct caamdmaobj sign_d = { }; + uint32_t pdb_sgt_flags = 0; + + ECC_TRACE("ECC Verify"); + + /* Verify first if the curve is supported */ + curve = get_caam_curve(inkey->curve); + if (curve == CAAM_ECC_UNKNOWN) + return TEE_ERROR_BAD_PARAMETERS; + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(MAX_DESC_VERIFY); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Convert the Public key to local key */ + retstatus = do_keypub_conv(&ecckey, inkey, sdata->size_sec); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* Prepare the input message CAAM Descriptor entry */ + ret = caam_dmaobj_input_sgtbuf(&msg, sdata->message.data, + sdata->message.length); + if (ret) + goto out; + + if (msg.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKVERIF_MSG; + + caam_dmaobj_cache_push(&msg); + + /* + * Prepare the 1st Part of the signature + * Handle the full signature in case signature buffer needs to + * be reallocated. + */ + ret = caam_dmaobj_input_sgtbuf(&sign_c, sdata->signature.data, + sdata->signature.length); + if (ret) + goto out; + + if (sign_c.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKVERIF_SIGN_C; + + /* Prepare the 2nd Part of the signature, derived from sign_c */ + ret = caam_dmaobj_derive_sgtbuf(&sign_d, &sign_c, sdata->size_sec, + sdata->size_sec); + if (ret) + goto out; + + if (sign_d.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKVERIF_SIGN_D; + + caam_dmaobj_cache_push(&sign_c); + + /* Allocate a Temporary buffer used by the CAAM */ + retstatus = caam_alloc_align_buf(&tmp, 2 * sdata->size_sec); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* Build the descriptor using Predifined ECC curve */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_PKVERIFY_PD1 | PDB_ECC_ECDSEL(curve) | + pdb_sgt_flags); + /* Public key */ + caam_desc_add_ptr(desc, ecckey.xy.paddr); + /* Input message */ + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + /* Signature 1st part */ + caam_desc_add_ptr(desc, sign_c.sgtbuf.paddr); + /* Signature 2nd part */ + caam_desc_add_ptr(desc, sign_d.sgtbuf.paddr); + /* Temporary buffer */ + caam_desc_add_ptr(desc, tmp.paddr); + /* Message length */ + caam_desc_add_word(desc, sdata->message.length); + + caam_desc_add_word(desc, DSA_VERIFY(ECC)); + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + ECC_DUMPDESC(desc); + + jobctx.desc = desc; + + cache_operation(TEE_CACHEFLUSH, tmp.data, tmp.length); + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_JOB_STATUS && !jobctx.status) { + ECC_TRACE("ECC Verify Status 0x%08" PRIx32, jobctx.status); + ret = TEE_ERROR_SIGNATURE_INVALID; + } else if (retstatus != CAAM_NO_ERROR) { + ECC_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } else { + ret = caam_status_to_tee_result(retstatus); + } + +out: + caam_free_desc(&desc); + do_keypair_free(&ecckey); + caam_free_buf(&tmp); + caam_dmaobj_free(&msg); + caam_dmaobj_free(&sign_c); + caam_dmaobj_free(&sign_d); + + return ret; +} + +/* + * Compute the shared secret data from ECC Private key and Public Key + * + * @sdata [in/out] ECC Shared Secret data + */ +static TEE_Result do_shared_secret(struct drvcrypt_secret_data *sdata) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + enum caam_ecc_curve curve = CAAM_ECC_UNKNOWN; + struct ecc_keypair *inprivkey = sdata->key_priv; + struct ecc_public_key *inpubkey = sdata->key_pub; + struct caam_ecc_keypair ecckey = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + struct caamdmaobj secret = { }; + uint32_t pdb_sgt_flags = 0; + + ECC_TRACE("ECC Shared Secret"); + + /* Verify first if the curve is supported */ + curve = get_caam_curve(inpubkey->curve); + if (curve == CAAM_ECC_UNKNOWN) + return TEE_ERROR_BAD_PARAMETERS; + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(MAX_DESC_SHARED); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Convert the Private key to local key */ + retstatus = do_keypair_conv(&ecckey, inprivkey, sdata->size_sec); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* Convert the Public key to local key */ + retstatus = do_keypub_conv(&ecckey, inpubkey, sdata->size_sec); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + /* + * Re-allocate the secret result buffer with a maximum size + * of the secret size if not cache aligned + */ + ret = caam_dmaobj_output_sgtbuf(&secret, sdata->secret.data, + sdata->secret.length, sdata->size_sec); + if (ret) + goto out; + + if (secret.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_PKDH_SECRET; + + caam_dmaobj_cache_push(&secret); + + /* Build the descriptor using Predifined ECC curve */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_SHARED_SECRET_PD1 | PDB_ECC_ECDSEL(curve) | + pdb_sgt_flags); + /* Public key */ + caam_desc_add_ptr(desc, ecckey.xy.paddr); + /* Private key */ + caam_desc_add_ptr(desc, ecckey.d.paddr); + /* Output secret */ + caam_desc_add_ptr(desc, secret.sgtbuf.paddr); + + caam_desc_add_word(desc, SHARED_SECRET(ECC)); + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + ECC_DUMPDESC(desc); + + jobctx.desc = desc; + + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + sdata->secret.length = caam_dmaobj_copy_to_orig(&secret); + + ECC_DUMPBUF("Secret", sdata->secret.data, sdata->secret.length); + + ret = caam_status_to_tee_result(retstatus); + } else { + ECC_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_free_desc(&desc); + do_keypair_free(&ecckey); + caam_dmaobj_free(&secret); + + return ret; +} + +/* + * Registration of the ECC Driver + */ +static struct drvcrypt_ecc driver_ecc = { + .alloc_keypair = do_allocate_keypair, + .alloc_publickey = do_allocate_publickey, + .free_publickey = do_free_publickey, + .gen_keypair = do_gen_keypair, + .sign = do_sign, + .verify = do_verify, + .shared_secret = do_shared_secret, +}; + +enum caam_status caam_ecc_init(struct caam_jrcfg *caam_jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + vaddr_t jr_base = caam_jrcfg->base + caam_jrcfg->offset; + + if (caam_hal_ctrl_pknum(jr_base)) + if (drvcrypt_register_ecc(&driver_ecc) == TEE_SUCCESS) + retstatus = CAAM_NO_ERROR; + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/acipher/caam_math.c b/optee_os/core/drivers/crypto/caam/acipher/caam_math.c new file mode 100644 index 0000000..1cdddb5 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/caam_math.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * CAAM Mathematical Operation manager. + * Implementation of Mathematical operation using CAAM's MATH function + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * MATH operation A xor B modulus n + * + * @data [in/out] operation data + */ +static TEE_Result do_xor_mod_n(struct drvcrypt_mod_op *data) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + struct caamdmaobj res = { }; + struct caamdmaobj data_a = { }; + struct caamdmaobj data_b = { }; + + RSA_TRACE("(A xor B) mod n"); + + ret = caam_dmaobj_input_sgtbuf(&data_a, data->a.data, data->a.length); + if (ret) + return ret; + + ret = caam_dmaobj_input_sgtbuf(&data_b, data->b.data, data->b.length); + if (ret) + goto out; + + /* + * ReAllocate the result buffer with a maximum size + * of the Key Modulus's size (N) if not cache aligned + */ + ret = caam_dmaobj_output_sgtbuf(&res, data->result.data, + data->result.length, + data->result.length); + if (ret) + goto out; + +#ifdef CFG_CAAM_64BIT +#define XOR_OP_DESC_SIZE 14 +#else +#define XOR_OP_DESC_SIZE 11 +#endif + /* Allocate the job descriptor */ + desc = caam_calloc_desc(XOR_OP_DESC_SIZE); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Load in N Modulus Size */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, LD_IMM(CLASS_1, REG_PKHA_N_SIZE, 4)); + caam_desc_add_word(desc, data->n.length); + + /* Load in A f irst value */ + caam_desc_fifo_load(desc, &data_a, CLASS_1, PKHA_A, NOACTION); + caam_desc_fifo_load(desc, &data_b, CLASS_1, PKHA_B, NOACTION); + + /* Operation B = A xor B mod n */ + caam_desc_add_word(desc, PKHA_F2M_OP(MOD_ADD_A_B, B)); + + /* Store the result */ + caam_desc_fifo_store(desc, &res, PKHA_B); + + caam_dmaobj_cache_push(&data_a); + caam_dmaobj_cache_push(&data_b); + caam_dmaobj_cache_push(&res); + + RSA_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + caam_dmaobj_copy_to_orig(&res); + RSA_DUMPBUF("Output", data->result.data, data->result.length); + ret = caam_status_to_tee_result(retstatus); + } else { + RSA_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_free_desc(&desc); + caam_dmaobj_free(&data_a); + caam_dmaobj_free(&data_b); + caam_dmaobj_free(&res); + + return ret; +} + +/* + * Registration of the MATH Driver + */ +static const struct drvcrypt_math driver_math = { + .xor_mod_n = &do_xor_mod_n, +}; + +enum caam_status caam_math_init(struct caam_jrcfg *caam_jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + vaddr_t jr_base = caam_jrcfg->base + caam_jrcfg->offset; + + if (caam_hal_ctrl_pknum(jr_base)) + if (!drvcrypt_register_math(&driver_math)) + retstatus = CAAM_NO_ERROR; + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/acipher/caam_prime_dsa.c b/optee_os/core/drivers/crypto/caam/acipher/caam_prime_dsa.c new file mode 100644 index 0000000..c02c132 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/caam_prime_dsa.c @@ -0,0 +1,758 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2021 NXP + * + * CAAM DSA Prime Numbering. + * Implementation of Prime Number functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +#define PRIME_DESC_ENTRIES 62 + +/* Define the number max of try to generate valid primes */ +#define DSA_MAX_TRIES_PRIME_Q 50000 +#define DSA_MAX_TRIES_PRIME_P 500 + +#define DSA_TRY_FAIL 0x42 +#define DSA_NOT_PRIME 0x43 +#define DSA_PRIME_TOO_SMALL 0x44 + +struct dsa_hash { + unsigned int op; /* CAAM Hash operation code */ + size_t size; /* Hash digest size */ +}; + +/* + * Build the descriptor generating a DSA prime Q + * Referring to FIPS.186-4, Section A.1.1.2 Generation of the + * Probable Primes p and q Using an Approved Hash Function + * + * @desc [out] Descriptor built + * @seed [out] Resulting seed used to generate prime + * @prime [in/out] Prime generation data + * @hash_func Selected Hash function + */ +static void do_desc_prime_q(uint32_t *desc, struct caambuf *seed, + struct prime_data_dsa *prime, + struct dsa_hash *hash_func) +{ + unsigned int desclen = 0; + unsigned int retry_new_mr_failed = 0; + unsigned int retry_mr_test = 0; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + /* Set the PKHA N and A register size */ + caam_desc_add_word(desc, LD_IMM(CLASS_1, REG_PKHA_N_SIZE, 4)); + caam_desc_add_word(desc, prime->q->length); + caam_desc_add_word(desc, LD_IMM(CLASS_1, REG_PKHA_A_SIZE, 4)); + caam_desc_add_word(desc, prime->q->length); + + caam_desc_add_word(desc, MATH(ADD, ZERO, IMM_DATA, VSOL, 4)); + caam_desc_add_word(desc, DSA_MAX_TRIES_PRIME_Q); + + caam_desc_add_word(desc, MATHI_OP1(SHIFT_L, ONE, 63, REG2, 8)); + + retry_new_mr_failed = caam_desc_get_len(desc); + + /* Decrement the number of try */ + caam_desc_add_word(desc, MATH(SUB, VSOL, ONE, VSOL, 4)); + /* Exceed retry count - exit with DSA_TRY_FAIL error */ + caam_desc_add_word(desc, + HALT_USER(ALL_COND_TRUE, MATH_N, DSA_TRY_FAIL)); + + /* Clear Class 2 SHA */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_CLEAR_WRITTEN, 4)); + caam_desc_add_word(desc, CLR_WR_RST_C2_CHA | CLR_WR_RST_C2_DSZ); + + /* + * Step 5. Generate Random Seed + * + * Seed Length shall be equal or greater than N (Q prime length) + * Seed result push in Message Data + */ + if (seed->length > 16) { + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO_n_SIZE, 4)); + caam_desc_add_word(desc, NFIFO_PAD(BOTH, 0, MSG, RND, 16)); + + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO_n_SIZE, 4)); + caam_desc_add_word(desc, + NFIFO_PAD(BOTH, NFIFO_LC1 | NFIFO_LC2, MSG, + RND, seed->length - 16)); + } else { + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO_n_SIZE, 4)); + caam_desc_add_word(desc, NFIFO_PAD(BOTH, NFIFO_LC1 | NFIFO_LC2, + MSG, RND, seed->length)); + } + + caam_desc_add_word(desc, MOVE(C1_ALIGN, OFIFO, 0, seed->length)); + caam_desc_add_word(desc, FIFO_ST(MSG_DATA, seed->length)); + caam_desc_add_ptr(desc, seed->paddr); + + /* + * Hash the Seed, this is a pseudo U, bits upper N - 1 still present + */ + caam_desc_add_word(desc, HASH_INITFINAL(hash_func->op)); + + /* + * Step 6. U = hash(seed) mod 2^(N-1) + * Step 7. q = 2^(N-1) + U + 1 - (U mod 2) + */ + /* Trash the bits > N - 1, the hash size is >= N */ + caam_desc_add_word(desc, + MOVE_WAIT(C2_CTX_REG, MATH_REG0, + hash_func->size - prime->q->length, 8)); + + /* Get the MSB of U and set the bit N-1 */ + caam_desc_add_word(desc, MATH(OR, REG2, REG0, REG0, 8)); + + /* Move the candidate prime q's MSB into IFIFO */ + caam_desc_add_word(desc, MOVE_WAIT(MATH_REG0, IFIFO, 0, 8)); + + /* + * Move the candidate prime q's intermediate value into IFIFO + */ + caam_desc_add_word(desc, + MOVE_WAIT(C2_CTX_REG, IFIFO, + hash_func->size - prime->q->length + 8, + prime->q->length - 16)); + + /* Get the LSB of U and set the bit 0 */ + caam_desc_add_word(desc, MOVE_WAIT(C2_CTX_REG, MATH_REG0, + hash_func->size - 8, 8)); + caam_desc_add_word(desc, MATH(OR, ONE, REG0, REG0, 8)); + + /* Move the candidate prime q's LSB into IFIFO */ + caam_desc_add_word(desc, MOVE_WAIT(MATH_REG0, IFIFO, 0, 8)); + + /* Move the IFIFO in to PKHA N */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 8)); + caam_desc_add_word(desc, NFIFO_NOPAD(C1, NFIFO_FC1, IFIFO, PKHA_N, 0)); + caam_desc_add_word(desc, prime->q->length); + + /* Store the Prime q here because Miller-Rabin test affect PKHA N */ + caam_desc_add_word(desc, FIFO_ST(PKHA_N, prime->q->length)); + caam_desc_add_ptr(desc, prime->q->paddr); + + /* + * Step 8. Test q prime with 'miller-rabin' test + * + * Load the number of Miller-Rabin test iteration + */ + caam_desc_add_word(desc, MATH(ADD, IMM_DATA, ZERO, SIL, 4)); + if (prime->p->length <= 1024 / 8) + caam_desc_add_word(desc, 40); + else if (prime->p->length >= 3072 / 8) + caam_desc_add_word(desc, 64); + else + caam_desc_add_word(desc, 56); + + retry_mr_test = caam_desc_get_len(desc); + + /* Generate 8 random bytes 'miller-rabin seed' */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 8)); + caam_desc_add_word(desc, NFIFO_PAD(C1, NFIFO_FC1, PKHA_A, RND, 0)); + caam_desc_add_word(desc, prime->q->length); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_word(desc, 0x01); + caam_desc_add_word(desc, PKHA_OP(MR_PRIMER_TEST, B)); + + desclen = caam_desc_get_len(desc); + + /* + * Step 9. If q is not q prime back to step 5 + */ + caam_desc_add_word(desc, JUMP_CNO_LOCAL(ANY_COND_FALSE, + JMP_COND(PKHA_IS_PRIME), + retry_new_mr_failed - desclen)); + caam_desc_add_word(desc, MATH(SUB, SIL, ONE, SIL, 4)); + + desclen = caam_desc_get_len(desc); + /* Test while number of MR test iteration not complete */ + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ALL_COND_FALSE, + JMP_COND(MATH_N) | JMP_COND(MATH_Z), + retry_mr_test - desclen)); + DSA_TRACE("Prime Q descriptor"); + DSA_DUMPDESC(desc); +} + +/* + * Build the descriptor generating the intermediate value X (step 11.3) + * Referring to FIPS.186-4, Section A.1.1.2 Generation of the + * Probable Primes p and q Using an Approved Hash Function + * + * @desc [out] Descriptor built + * @x [out] Value X + * @seed [in/out] Seed to hash and next seed for next loop + * @prime [in/out] Prime generation data + * @hash_func Selected Hash function + * @mod_n Modular value (0xFF filled buffer) + * @desc_p Physical address of the descriptor doing Prime P + */ +static void do_desc_gen_x(uint32_t *desc, struct caambuf *x, + struct caambuf *seed, struct prime_data_dsa *prime, + struct dsa_hash *hash_func, struct caambuf *mod_n, + paddr_t desc_p) +{ + unsigned int desclen = 0; + unsigned int loop_n = 0; + size_t n = 0; + size_t b = 0; + size_t b_offset = 0; + + /* + * Step 3. n = ceil(L / outlen) - 1 + * where outlen is the hash size in bits + * + * Note build descriptor with n = ceil(L / outlen) to + * pre-calculate seed for next run. + */ + n = (prime->p->length + hash_func->size) * 8 - 1; + n /= hash_func->size * 8; + + /* + * Step 4. b = L - 1 - (n * outlen) + * + * Note b determine the number of bits to keep in the last + * Vn computed. + * Calculate b_offset which is the offset in bytes to remove from + * the calculated hash + */ + b = prime->p->length * 8 - 1 - (n - 1) * hash_func->size * 8; + + DSA_TRACE("Prime p => n = %zu | b = %zu", n - 1, b); + b_offset = hash_func->size - (b + 1) / 8; + DSA_TRACE("Vn offset is %zu", b_offset); + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + caam_desc_add_word(desc, SEQ_OUT_PTR(x->length)); + caam_desc_add_ptr(desc, x->paddr); + + caam_desc_add_word(desc, MATHI_OP1(SHIFT_L, ONE, 63, REG2, 8)); + + caam_desc_add_word(desc, MATH(ADD, ZERO, IMM_DATA, REG0, 4)); + caam_desc_add_word(desc, n); + + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, PKHA_N, NOACTION, seed->length)); + caam_desc_add_ptr(desc, mod_n->paddr); + + /* + * Because the Sequence Out Pointer is incremental store, we need + * to build w number in reverse. + * + * Hence, calculate the last seed number of the loop and save it. + * Step 11.9 is automatically done here by incrementing seed number. + */ + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_word(desc, n); + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, PKHA_A, NOACTION, seed->length)); + caam_desc_add_ptr(desc, seed->paddr); + caam_desc_add_word(desc, PKHA_OP(MOD_ADD_A_B, A)); + caam_desc_add_word(desc, FIFO_ST(PKHA_A, seed->length)); + caam_desc_add_ptr(desc, seed->paddr); + + caam_desc_add_word(desc, PKHA_CPY_NSIZE(A0, B1)); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_word(desc, 1); + + caam_desc_add_word(desc, WAIT_COND(ALL_COND_TRUE, NIFP)); + + /* + * Step 11.1 + * For j = 0 to n do + * Vj = hash((seed + offset + j) mod 2^seedlen + * Step 11.2 + * W = V0 + (V1 * 2^outlen) + ... + + * (Vn-1 * 2^((n-1)*outlen)) + + * ((Vn mod 2^b) * 2^(n*outlen)) + */ + loop_n = caam_desc_get_len(desc); + + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_CLEAR_WRITTEN, 4)); + caam_desc_add_word(desc, CLR_WR_IFIFO_NFIFO | CLR_WR_RST_C2_CHA | + CLR_WR_RST_C2_DSZ); + + caam_desc_add_word(desc, HASH_INITFINAL(hash_func->op)); + caam_desc_add_word(desc, LD_NOCLASS_IMM(REG_CHA_CTRL, 4)); + caam_desc_add_word(desc, CCTRL_ULOAD_PKHA_A); + + caam_desc_add_word(desc, + MOVE_WAIT(OFIFO, IFIFO_C2_LC2, 0, seed->length)); + + /* If Math Register 2 is zero bypass the high bit set to one */ + caam_desc_add_word(desc, MATH(SUB, REG2, ONE, NODEST, 8)); + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ANY_COND_TRUE, + JMP_COND(MATH_N) | JMP_COND(MATH_Z), + 8)); + /* + * Step 11.3 + * X = W + 2^(L-1) + * + * Set the high bit to one + * Remark: the DSA key is a modulus 8 bytes, hence no need + * to check if the b_offset is less than 8. + */ + caam_desc_add_word(desc, MOVE_WAIT(C2_CTX_REG, MATH_REG1, b_offset, 8)); + caam_desc_add_word(desc, MATH(OR, REG2, REG1, REG1, 8)); + caam_desc_add_word(desc, MOVE(MATH_REG1, OFIFO, 0, 8)); + + if (hash_func->size - b_offset > 8) + caam_desc_add_word(desc, + MOVE_WAIT(C2_CTX_REG, OFIFO, b_offset + 8, + hash_func->size - b_offset - 8)); + caam_desc_add_word(desc, + FIFO_ST_SEQ(MSG_DATA, hash_func->size - b_offset)); + + /* + * Reset MATH Register 2 to bypass the High Bit set + * operation next loop + */ + caam_desc_add_word(desc, MATH(AND, REG2, ZERO, REG2, 8)); + + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ALL_COND_TRUE, JMP_COND(NONE), 2)); + + /* Bypass High Bit set */ + caam_desc_add_word(desc, + ST_NOIMM_SEQ(CLASS_2, REG_CTX, hash_func->size)); + + caam_desc_add_word(desc, PKHA_CPY_NSIZE(B1, A0)); + caam_desc_add_word(desc, PKHA_OP(MOD_SUB_A_B, A)); + caam_desc_add_word(desc, PKHA_CPY_NSIZE(A0, B1)); + + desclen = caam_desc_get_len(desc); + caam_desc_add_word(desc, JUMP_CNO_LOCAL_DEC(ALL_COND_FALSE, MATH_0, + JMP_COND_MATH(N) | + JMP_COND_MATH(Z), + loop_n - desclen)); + /* Jump to the next descriptor desc */ + caam_desc_add_word(desc, JUMP_NOTLOCAL(CLASS_NO, ALL_COND_TRUE, + JMP_COND(NONE))); + caam_desc_add_ptr(desc, desc_p); + + DSA_TRACE("X descriptor"); + DSA_DUMPDESC(desc); +} + +/* + * Build the descriptor generating the Prime P from value X + * Referring to FIPS.186-4, Section A.1.1.2 Generation of the + * Probable Primes p and q Using an Approved Hash Function + * + * @desc [out] Descriptor built + * @prime [in/out] Prime generation data + * @x Value X + * @mod_n Modular value (0xFF filled buffer) + */ +static void do_desc_prime_p(uint32_t *desc, struct prime_data_dsa *prime, + struct caambuf *x, struct caambuf *mod_n) +{ + unsigned int desclen = 0; + unsigned int retry_mr_test = 0; + size_t index = 0; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, PKHA_N, NOACTION, mod_n->length)); + caam_desc_add_ptr(desc, mod_n->paddr); + + /* + * Step 11.4 + * c = X mod 2q + */ + + /* Calculate 2q and store it in PKHA N */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_A, NOACTION, + prime->q->length)); + caam_desc_add_ptr(desc, prime->q->paddr); + caam_desc_add_word(desc, PKHA_CPY_SSIZE(A0, B0)); + caam_desc_add_word(desc, PKHA_OP(MOD_ADD_A_B, A)); + + caam_desc_add_word(desc, PKHA_CPY_SSIZE(A0, N0)); + + /* c = X mod 2q */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_A, NOACTION, x->length)); + caam_desc_add_ptr(desc, x->paddr); + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ALL_COND_TRUE, JMP_COND(NIFP), 1) | + BIT(24)); + caam_desc_add_word(desc, PKHA_OP(MOD_AMODN, A)); + + /* + * Step 11.5 + * p = X - (c - 1) + */ + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, PKHA_N, NOACTION, mod_n->length)); + caam_desc_add_ptr(desc, mod_n->paddr); + + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_ptr(desc, 1); + caam_desc_add_word(desc, PKHA_OP(MOD_SUB_A_B, B)); + + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_A, NOACTION, x->length)); + caam_desc_add_ptr(desc, x->paddr); + caam_desc_add_word(desc, PKHA_OP(MOD_SUB_A_B, A)); + + /* + * Save the candidate Prime q now because N is going to be + * affected by the Miller-Rabin test + */ + caam_desc_add_word(desc, PKHA_CPY_SSIZE(A0, N0)); + caam_desc_add_word(desc, FIFO_ST(PKHA_N, prime->p->length)); + caam_desc_add_ptr(desc, prime->p->paddr); + caam_desc_add_word(desc, FIFO_ST_SEQ(MSG_DATA, 0)); + + /* + * Step 11.6 + * if (p < 2^(L-1)) then go to step 11.9 + * + */ + caam_desc_add_word(desc, LD_NOCLASS_IMM(REG_CHA_CTRL, 4)); + caam_desc_add_word(desc, CCTRL_ULOAD_PKHA_A); + + /* Keep the MSB from p candidate and check if bit 2^(L-1) is set */ + caam_desc_add_word(desc, MOVE_WAIT(OFIFO, MATH_REG0, 0, 8)); + for (index = 1; index < prime->p->length / 128; index++) + caam_desc_add_word(desc, MOVE(OFIFO, C1_CTX_REG, 0, 128)); + + caam_desc_add_word(desc, MOVE(OFIFO, C1_CTX_REG, 0, 124)); + + caam_desc_add_word(desc, MATHI_OP1(SHIFT_L, ONE, 63, REG2, 8)); + caam_desc_add_word(desc, MATH(AND, REG0, REG2, REG0, 8)); + + caam_desc_add_word(desc, HALT_USER(ALL_COND_TRUE, MATH_Z, + DSA_PRIME_TOO_SMALL)); + + /* + * Step 11.7 + * Test whether or not p is prime + * + * Referring to FIPS.186-4, Table C.1 + * Get the number Miller-Rabin test interation function + * of the prime number size + */ + caam_desc_add_word(desc, MATH(ADD, IMM_DATA, ZERO, REG0, 4)); + if (prime->p->length <= 1024 / 8) + caam_desc_add_word(desc, 40); + else if (prime->p->length >= 3072 / 8) + caam_desc_add_word(desc, 64); + else + caam_desc_add_word(desc, 56); + + retry_mr_test = caam_desc_get_len(desc); + /* Generate 8 random bytes 'miller-rabin seed' */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 8)); + caam_desc_add_word(desc, NFIFO_PAD(C1, NFIFO_FC1, PKHA_A, RND, 0)); + caam_desc_add_word(desc, prime->p->length); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_word(desc, 0x01); + caam_desc_add_word(desc, PKHA_OP(MR_PRIMER_TEST, B)); + + desclen = caam_desc_get_len(desc); + + /* + * Step 11.8 + * if p is not a prime continue to step 11.9 + */ + caam_desc_add_word(desc, HALT_USER(ALL_COND_FALSE, PKHA_IS_PRIME, + DSA_NOT_PRIME)); + + desclen = caam_desc_get_len(desc); + /* Test while number of MR test iteration not complete */ + caam_desc_add_word(desc, JUMP_CNO_LOCAL_DEC(ALL_COND_FALSE, MATH_0, + JMP_COND_MATH(N) | + JMP_COND_MATH(Z), + retry_mr_test - desclen)); + + DSA_TRACE("Prime P descriptor"); + DSA_DUMPDESC(desc); + + /* + * Ensure descriptor is pushed in physical memory because it's + * called from another descriptor. + */ + cache_operation(TEE_CACHECLEAN, desc, DESC_SZBYTES(PRIME_DESC_ENTRIES)); +} + +/* + * Run the Prime Q descriptor. + * + * @desc Descriptor built + */ +static enum caam_status run_prime_q(uint32_t *desc, + struct prime_data_dsa *prime) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + + cache_operation(TEE_CACHEFLUSH, prime->q->data, prime->q->length); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus != CAAM_NO_ERROR) { + DSA_TRACE("Prime Q Status 0x%08" PRIx32 " ret 0x%08" PRIx32, + jobctx.status, retstatus); + retstatus = CAAM_FAILURE; + } else { + cache_operation(TEE_CACHEINVALIDATE, prime->q->data, + prime->q->length); + DSA_DUMPBUF("Prime Q", prime->q->data, prime->q->length); + } + + return retstatus; +} + +/* + * Run the Prime P descriptors. + * + * @desc Descriptor built + * @prime Prime generation data + */ +static enum caam_status run_prime_p(uint32_t *desc, + struct prime_data_dsa *prime) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + size_t counter = 0; + + cache_operation(TEE_CACHEFLUSH, prime->p->data, prime->p->length); + + jobctx.desc = desc; + for (counter = 0; counter < 4 * prime->p->length * 8; counter++) { + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + DSA_TRACE("Prime P try: counter=%zu", counter); + cache_operation(TEE_CACHEINVALIDATE, prime->p->data, + prime->p->length); + DSA_DUMPBUF("Prime P", prime->p->data, + prime->p->length); + + return retstatus; + } + + if (retstatus == CAAM_JOB_STATUS) { + if (JRSTA_GET_HALT_USER(jobctx.status) != + DSA_NOT_PRIME && + JRSTA_GET_HALT_USER(jobctx.status) != + DSA_PRIME_TOO_SMALL) { + DSA_TRACE("Prime P status 0x%08" PRIx32, + jobctx.status); + return CAAM_FAILURE; + } + } + } + + /* This is not a prime, will try with another prime q */ + return CAAM_BAD_PARAM; +} + +/* + * Generate the DSA parameter G (generator) + * Referring to FIPS.186-4, Section A.2.1 Unverifiable Generation of the + * Generator g + * + * @desc Descriptor buffer to use + * @prime [in/out] Prime generation data + * @mod_n Modular value (0xFF filled buffer) + */ +static enum caam_status do_generator(uint32_t *desc, + struct prime_data_dsa *prime, + struct caambuf *mod_n) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + unsigned int desclen = 0; + unsigned int retry_new_h = 0; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, PKHA_N, NOACTION, mod_n->length)); + caam_desc_add_ptr(desc, mod_n->paddr); + + /* + * Step 1. + * e = (p - 1)/q + */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_A, NOACTION, + prime->p->length)); + caam_desc_add_ptr(desc, prime->p->paddr); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_ptr(desc, 1); + /* PKHA B = (p - 1) */ + caam_desc_add_word(desc, PKHA_OP(MOD_SUB_A_B, B)); + + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_A, NOACTION, + prime->q->length)); + caam_desc_add_ptr(desc, prime->q->paddr); + /* PKHA A = 1/q */ + caam_desc_add_word(desc, PKHA_OP(MOD_INV_A, A)); + + /* PKHA E = (p - 1)/q */ + caam_desc_add_word(desc, PKHA_OP(MOD_MUL_A_B, A)); + caam_desc_add_word(desc, PKHA_CPY_SSIZE(A0, E)); + + /* Load N with prime p */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_N, NOACTION, + prime->p->length)); + caam_desc_add_ptr(desc, prime->p->paddr); + + /* + * Step 2. Generate a Random h + * where 1 < h < (p - 1) + * + * To ensure h < (p - 1), generate a random of p length - 2 + */ + retry_new_h = caam_desc_get_len(desc); + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 8)); + caam_desc_add_word(desc, NFIFO_PAD(C1, NFIFO_FC1, PKHA_A, RND, 0)); + caam_desc_add_word(desc, prime->p->length - 2); + + /* + * Step 3. + * g = h^e mod p + */ + caam_desc_add_word(desc, PKHA_OP(MOD_EXP_A_E, A)); + + /* + * Step 4. + * if (g = 1) then go to step 2 + */ + desclen = caam_desc_get_len(desc); + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ALL_COND_TRUE, JMP_COND(PKHA_GCD_1), + retry_new_h - desclen)); + + /* g is good save it */ + caam_desc_add_word(desc, FIFO_ST(PKHA_A, prime->g->length)); + caam_desc_add_ptr(desc, prime->g->paddr); + + DSA_DUMPDESC(desc); + + cache_operation(TEE_CACHEFLUSH, prime->g->data, prime->g->length); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus != CAAM_NO_ERROR) { + DSA_TRACE("Generator G Status 0x%08" PRIx32 " ret 0x%08" PRIx32, + jobctx.status, retstatus); + return CAAM_FAILURE; + } + + cache_operation(TEE_CACHEINVALIDATE, prime->g->data, prime->g->length); + DSA_DUMPBUF("Generator G", prime->g->data, prime->g->length); + + return CAAM_NO_ERROR; +} + +enum caam_status caam_prime_dsa_gen(struct prime_data_dsa *data) +{ + enum caam_status retstatus = CAAM_FAILURE; + uint32_t *desc_all = NULL; + uint32_t *desc_q = NULL; + uint32_t *desc_x = NULL; + uint32_t *desc_p = NULL; + struct caambuf seed = { }; + struct caambuf mod_n = { }; + struct dsa_hash hash_func = { OP_ALGO(SHA256), TEE_SHA256_HASH_SIZE }; + size_t nb_tries = 0; + struct caambuf x = { }; + + /* + * For the now as the DSA Prime p size is limited to 3072, Prime q + * is also limited to 256. Hence the hash function to use is + * SHA-256. + * Ensure here that limit is not crossed because on some i.MX device + * hash is limited to 256. + */ + if (data->q->length > 256) + return CAAM_BAD_PARAM; + + retstatus = caam_calloc_buf(&mod_n, data->p->length); + if (retstatus != CAAM_NO_ERROR) + goto out; + + memset(mod_n.data, 0xFF, mod_n.length); + cache_operation(TEE_CACHECLEAN, mod_n.data, mod_n.length); + + retstatus = caam_calloc_align_buf(&seed, data->q->length); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + retstatus = caam_calloc_buf(&x, data->p->length); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + desc_all = caam_calloc_desc(PRIME_DESC_ENTRIES * 3); + if (!desc_all) { + retstatus = CAAM_OUT_MEMORY; + goto out; + } + + DSA_TRACE("Do primes P %zu bytes, Q %zu bytes", data->p->length, + data->q->length); + + desc_q = desc_all; + desc_x = desc_q + PRIME_DESC_ENTRIES; + desc_p = desc_x + PRIME_DESC_ENTRIES; + + do_desc_prime_q(desc_q, &seed, data, &hash_func); + do_desc_gen_x(desc_x, &x, &seed, data, &hash_func, &mod_n, + virt_to_phys(desc_p)); + do_desc_prime_p(desc_p, data, &x, &mod_n); + + cache_operation(TEE_CACHEFLUSH, data->p->data, data->p->length); + cache_operation(TEE_CACHEFLUSH, seed.data, seed.length); + cache_operation(TEE_CACHEFLUSH, x.data, x.length); + + for (nb_tries = DSA_MAX_TRIES_PRIME_P; nb_tries > 0; nb_tries--) { + retstatus = run_prime_q(desc_q, data); + + if (retstatus == CAAM_NO_ERROR) { + retstatus = run_prime_p(desc_x, data); + if (retstatus == CAAM_NO_ERROR) + break; + } + + if (retstatus == CAAM_FAILURE) { + DSA_TRACE("DSA Prime P/Q Generation failed"); + break; + } + } + + if (retstatus == CAAM_NO_ERROR) + retstatus = do_generator(desc_all, data, &mod_n); + +out: + caam_free_desc(&desc_all); + caam_free_buf(&seed); + caam_free_buf(&x); + caam_free_buf(&mod_n); + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/acipher/caam_prime_rsa.c b/optee_os/core/drivers/crypto/caam/acipher/caam_prime_rsa.c new file mode 100644 index 0000000..d3b16ed --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/caam_prime_rsa.c @@ -0,0 +1,863 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * CAAM Prime Numbering. + * Implementation of Prime Number functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +#define RSA_TRY_FAIL 0x42 +#define RETRY_TOO_SMALL 0x2A + +#define STATUS_GOOD_Q 0xCA + +#define MR_PRIME_SIZE 1536 +#define MAX_RETRY_PRIME_GEN 1000 +#define RSA_MAX_TRIES_PRIMES 100 + +#ifdef CFG_CAAM_64BIT +#define SETUP_RSA_DESC_ENTRIES 20 +#define GEN_RSA_DESC_ENTRIES 62 +#define CHECK_P_Q_DESC_ENTRIES 32 +#else +#define SETUP_RSA_DESC_ENTRIES 17 +#define GEN_RSA_DESC_ENTRIES 58 +#define CHECK_P_Q_DESC_ENTRIES 29 +#endif + +/* + * Predefined const value corresponding to the + * operation sqrt(2) * (2 ^ ((nlen / 2) - 1)) + * Used at step 4.4 + */ +static const uint8_t sqrt_value[] = { + 0xb5, 0x04, 0xf3, 0x33, 0xf9, 0xde, 0x64, 0x84, 0x59, 0x7d, 0x89, 0xb3, + 0x75, 0x4a, 0xbe, 0x9f, 0x1d, 0x6f, 0x60, 0xba, 0x89, 0x3b, 0xa8, 0x4c, + 0xed, 0x17, 0xac, 0x85, 0x83, 0x33, 0x99, 0x15, 0x4a, 0xfc, 0x83, 0x04, + 0x3a, 0xb8, 0xa2, 0xc3, 0xa8, 0xb1, 0xfe, 0x6f, 0xdc, 0x83, 0xdb, 0x39, + 0x0f, 0x74, 0xa8, 0x5e, 0x43, 0x9c, 0x7b, 0x4a, 0x78, 0x04, 0x87, 0x36, + 0x3d, 0xfa, 0x27, 0x68, 0xd2, 0x20, 0x2e, 0x87, 0x42, 0xaf, 0x1f, 0x4e, + 0x53, 0x05, 0x9c, 0x60, 0x11, 0xbc, 0x33, 0x7b, 0xca, 0xb1, 0xbc, 0x91, + 0x16, 0x88, 0x45, 0x8a, 0x46, 0x0a, 0xbc, 0x72, 0x2f, 0x7c, 0x4e, 0x33, + 0xc6, 0xd5, 0xa8, 0xa3, 0x8b, 0xb7, 0xe9, 0xdc, 0xcb, 0x2a, 0x63, 0x43, + 0x31, 0xf3, 0xc8, 0x4d, 0xf5, 0x2f, 0x12, 0x0f, 0x83, 0x6e, 0x58, 0x2e, + 0xea, 0xa4, 0xa0, 0x89, 0x90, 0x40, 0xca, 0x4a, 0x81, 0x39, 0x4a, 0xb6, + 0xd8, 0xfd, 0x0e, 0xfd, 0xf4, 0xd3, 0xa0, 0x2c, 0xeb, 0xc9, 0x3e, 0x0c, + 0x42, 0x64, 0xda, 0xbc, 0xd5, 0x28, 0xb6, 0x51, 0xb8, 0xcf, 0x34, 0x1b, + 0x6f, 0x82, 0x36, 0xc7, 0x01, 0x04, 0xdc, 0x01, 0xfe, 0x32, 0x35, 0x2f, + 0x33, 0x2a, 0x5e, 0x9f, 0x7b, 0xda, 0x1e, 0xbf, 0xf6, 0xa1, 0xbe, 0x3f, + 0xca, 0x22, 0x13, 0x07, 0xde, 0xa0, 0x62, 0x41, 0xf7, 0xaa, 0x81, 0xc2, + 0xc1, 0xfc, 0xbd, 0xde, 0xa2, 0xf7, 0xdc, 0x33, 0x18, 0x83, 0x8a, 0x2e, + 0xaf, 0xf5, 0xf3, 0xb2, 0xd2, 0x4f, 0x4a, 0x76, 0x3f, 0xac, 0xb8, 0x82, + 0xfd, 0xfe, 0x17, 0x0f, 0xd3, 0xb1, 0xf7, 0x80, 0xf9, 0xac, 0xce, 0x41, + 0x79, 0x7f, 0x28, 0x05, 0xc2, 0x46, 0x78, 0x5e, 0x92, 0x95, 0x70, 0x23, + 0x5f, 0xcf, 0x8f, 0x7b, 0xca, 0x3e, 0xa3, 0x3b, 0x4d, 0x7c, 0x60, 0xa5, + 0xe6, 0x33, 0xe3, 0xe1 +}; + +/* + * Speedups for prime searching + * + * These values are products of small primes. Information about the product + * preceeds it. These values have been pre-computed by the CAAM design team. + * + * Per Handbook of Applied Cryptography, Menezes et al, 4.4.1, one can compute + * the percentage of non-primes weeded out by checking for small prime factors + * in the candidates. In the table below, "highest prime" is used for B, and + * "%weeded" is the number of candidates which get through this + * sieve. As you can see, even with relatively few primes, there are + * diminishing returns to using larger numbers of primes. + * + * Percentage weeded: 1 - 1.12/ln B + * + * These can be used to compute GCD(prime, smallprime) before the Miller + * Rabin; this will weed out those candidates with 'small' primes before doing + * the costly modular exponentation inside of Miller-Rabin. (If the result is + * not one, then the candidate has as a factor at least one of the small primes + * in the product). + * + * So, where is the sweet spot for the size of the product versus the size of + * the candidate? Does it depend upon the size of the PKHA multiplier? Hunt + * time for primes takes a long time to actually compute, and what are the + * stats for percentage of candidates that might be weeded out? If not many, + * then there is an extra computation. + */ +struct smallprime { + const size_t length; + const uint8_t *data; +}; + +/* sizes | #primes | highest prime | %weeded */ +/* bits / bytes | | */ +/* 64 / 8 | 15 | 53 | 72 */ +static const uint8_t smallprime_8[] = { + 0xe2, 0x21, 0xf9, 0x7c, 0x30, 0xe9, 0x4e, 0x1d, +}; + +/* 128 / 16 | 25 | 101 | 76 */ +static const uint8_t smallprime_16[] = { + 0x57, 0x97, 0xd4, 0x7c, 0x51, 0x68, 0x15, 0x49, 0xd7, 0x34, 0xe4, 0xfc, + 0x4c, 0x3e, 0xaf, 0x7f, +}; + +/* 256 / 32 | 43 | 193 | 79 */ +static const uint8_t smallprime_32[] = { + 0xdb, 0xf0, 0x5b, 0x6f, 0x56, 0x54, 0xb3, 0xc0, 0xf5, 0x24, 0x35, 0x51, + 0x43, 0x95, 0x86, 0x88, 0x9f, 0x15, 0x58, 0x87, 0x81, 0x9a, 0xed, 0x2a, + 0xc0, 0x5b, 0x93, 0x35, 0x2b, 0xe9, 0x86, 0x77, +}; + +/* 384 / 48 | 59 | 281 | 80 */ +static const uint8_t smallprime_48[] = { + 0x50, 0x12, 0x01, 0xcc, 0x51, 0xa4, 0x92, 0xa5, 0x44, 0xd3, 0x90, 0x0a, + 0xd4, 0xf8, 0xb3, 0x2a, 0x20, 0x3c, 0x85, 0x84, 0x06, 0xa4, 0x45, 0x7c, + 0xab, 0x0b, 0x4f, 0x80, 0x5a, 0xb1, 0x8a, 0xc6, 0xeb, 0x95, 0x72, 0xac, + 0x6e, 0x93, 0x94, 0xfa, 0x52, 0x2b, 0xff, 0xb6, 0xf4, 0x4a, 0xf2, 0xf3, +}; + +/* 512 / 64 | 74 | 379 | 81 */ +static const uint8_t smallprime_64[] = { + 0x10, 0x6a, 0xa9, 0xfb, 0x76, 0x46, 0xfa, 0x6e, 0xb0, 0x81, 0x3c, 0x28, + 0xc5, 0xd5, 0xf0, 0x9f, 0x07, 0x7e, 0xc3, 0xba, 0x23, 0x8b, 0xfb, 0x99, + 0xc1, 0xb6, 0x31, 0xa2, 0x03, 0xe8, 0x11, 0x87, 0x23, 0x3d, 0xb1, 0x17, + 0xcb, 0xc3, 0x84, 0x05, 0x6e, 0xf0, 0x46, 0x59, 0xa4, 0xa1, 0x1d, 0xe4, + 0x9f, 0x7e, 0xcb, 0x29, 0xba, 0xda, 0x8f, 0x98, 0x0d, 0xec, 0xec, 0xe9, + 0x2e, 0x30, 0xc4, 0x8f, +}; + +/* 576 / 72 | 81 | 421 | 82 */ +static const uint8_t smallprime_72[] = { + 0x01, 0x85, 0xdb, 0xeb, 0x2b, 0x8b, 0x11, 0xd3, 0x76, 0x33, 0xe9, 0xdc, + 0x1e, 0xec, 0x54, 0x15, 0x65, 0xc6, 0xce, 0x84, 0x31, 0xd2, 0x27, 0xee, + 0x28, 0xf0, 0x32, 0x8a, 0x60, 0xc9, 0x01, 0x18, 0xae, 0x03, 0x1c, 0xc5, + 0xa7, 0x81, 0xc8, 0x24, 0xd1, 0xf1, 0x6d, 0x25, 0xf4, 0xf0, 0xcc, 0xcf, + 0xf3, 0x5e, 0x97, 0x45, 0x79, 0x07, 0x2e, 0xc8, 0xca, 0xf1, 0xac, 0x8e, + 0xef, 0xd5, 0x56, 0x6f, 0xa1, 0x5f, 0xb9, 0x4f, 0xe3, 0x4f, 0x5d, 0x37, +}; + +/* 768 / 96 | 103 | 569 | 82 */ +static const uint8_t smallprime_96[] = { + 0x25, 0xea, 0xc8, 0x9f, 0x8d, 0x4d, 0xa3, 0x38, 0x33, 0x7b, 0x49, 0x85, + 0x0d, 0x2d, 0x14, 0x89, 0x26, 0x63, 0x17, 0x7b, 0x40, 0x10, 0xaf, 0x3d, + 0xd2, 0x3e, 0xeb, 0x0b, 0x22, 0x8f, 0x38, 0x32, 0xff, 0xce, 0xe2, 0xe5, + 0xcb, 0xd1, 0xac, 0xc9, 0x8f, 0x47, 0xf2, 0x51, 0x87, 0x33, 0x80, 0xae, + 0x10, 0xf0, 0xff, 0xdd, 0x8e, 0x60, 0x2f, 0xfa, 0x21, 0x0f, 0x41, 0xf6, + 0x69, 0xa1, 0x57, 0x0a, 0x93, 0xc1, 0x58, 0xc1, 0xa9, 0xa8, 0x22, 0x7f, + 0xf8, 0x1a, 0x90, 0xc5, 0x63, 0x0e, 0x9c, 0x44, 0x84, 0x5c, 0x75, 0x5c, + 0x7d, 0xf3, 0x5a, 0x7d, 0x43, 0x0c, 0x67, 0x9a, 0x11, 0x57, 0x56, 0x55, +}; + +/* 1024 / 128 | 130 | 739 | 83 */ +static const uint8_t smallprime_128[] = { + 0x02, 0xc8, 0x5f, 0xf8, 0x70, 0xf2, 0x4b, 0xe8, 0x0f, 0x62, 0xb1, 0xba, + 0x6c, 0x20, 0xbd, 0x72, 0xb8, 0x37, 0xef, 0xdf, 0x12, 0x12, 0x06, 0xd8, + 0x7d, 0xb5, 0x6b, 0x7d, 0x69, 0xfa, 0x4c, 0x02, 0x1c, 0x10, 0x7c, 0x3c, + 0xa2, 0x06, 0xfe, 0x8f, 0xa7, 0x08, 0x0e, 0xf5, 0x76, 0xef, 0xfc, 0x82, + 0xf9, 0xb1, 0x0f, 0x57, 0x50, 0x65, 0x6b, 0x77, 0x94, 0xb1, 0x6a, 0xfd, + 0x70, 0x99, 0x6e, 0x91, 0xae, 0xf6, 0xe0, 0xad, 0x15, 0xe9, 0x1b, 0x07, + 0x1a, 0xc9, 0xb2, 0x4d, 0x98, 0xb2, 0x33, 0xad, 0x86, 0xee, 0x05, 0x55, + 0x18, 0xe5, 0x8e, 0x56, 0x63, 0x8e, 0xf1, 0x8b, 0xac, 0x5c, 0x74, 0xcb, + 0x35, 0xbb, 0xb6, 0xe5, 0xda, 0xe2, 0x78, 0x3d, 0xd1, 0xc0, 0xce, 0x7d, + 0xec, 0x4f, 0xc7, 0x0e, 0x51, 0x86, 0xd4, 0x11, 0xdf, 0x36, 0x36, 0x8f, + 0x06, 0x1a, 0xa3, 0x60, 0x11, 0xf3, 0x01, 0x79, +}; + +/* 1088 / 184 | 136 | 787 | 83 */ +static const uint8_t smallprime_184[] = { + 0x16, 0xaf, 0x5c, 0x18, 0xa2, 0xbe, 0xf8, 0xef, 0xf2, 0x27, 0x83, 0x32, + 0x18, 0x2d, 0x0f, 0xbf, 0x00, 0x38, 0xcc, 0x20, 0x51, 0x48, 0xb8, 0x3d, + 0x06, 0xe3, 0xd7, 0xd9, 0x32, 0x82, 0x8b, 0x18, 0xe1, 0x1e, 0x09, 0x40, + 0x28, 0xc7, 0xea, 0xed, 0xa3, 0x39, 0x50, 0x17, 0xe0, 0x7d, 0x8a, 0xe9, + 0xb5, 0x94, 0x06, 0x04, 0x51, 0xd0, 0x5f, 0x93, 0x08, 0x4c, 0xb4, 0x81, + 0x66, 0x3c, 0x94, 0xc6, 0xff, 0x98, 0x0d, 0xde, 0xcc, 0xdb, 0x42, 0xad, + 0x37, 0x09, 0x7f, 0x41, 0xa7, 0x83, 0x7f, 0xc9, 0x5a, 0xfe, 0x3f, 0x18, + 0xad, 0x76, 0xf2, 0x34, 0x83, 0xae, 0x94, 0x2e, 0x0f, 0x0c, 0x0b, 0xc6, + 0xe4, 0x00, 0x16, 0x12, 0x31, 0x89, 0x87, 0x2b, 0xe5, 0x8f, 0x6d, 0xfc, + 0x23, 0x9c, 0xa2, 0x8f, 0xb0, 0xcf, 0xbf, 0x96, 0x4c, 0x8f, 0x27, 0xce, + 0x05, 0xd6, 0xc7, 0x7a, 0x01, 0xf9, 0xd3, 0x32, 0x36, 0xc9, 0xd4, 0x42, + 0xad, 0x69, 0xed, 0x33, +}; + +/* 1536 / 192 | 182 | 1093 | 84 */ +static const uint8_t smallprime_192[] = { + 0x02, 0x1b, 0xf9, 0x49, 0x70, 0x91, 0xb8, 0xc3, 0x68, 0xcc, 0x7c, 0x8e, + 0x00, 0xc1, 0x99, 0x0c, 0x60, 0x27, 0x48, 0x1b, 0x79, 0x21, 0x5a, 0xc8, + 0xa7, 0x51, 0x77, 0x49, 0xa2, 0x15, 0x13, 0x77, 0x9a, 0x99, 0x3d, 0x29, + 0x58, 0xfc, 0xb4, 0x9a, 0x73, 0x68, 0x02, 0x92, 0x68, 0x52, 0x79, 0x94, + 0xc6, 0xcc, 0x19, 0x28, 0xad, 0xd4, 0x12, 0x95, 0x96, 0x76, 0x5f, 0x4c, + 0xc3, 0x14, 0x1a, 0x04, 0x4e, 0xb1, 0xd6, 0x15, 0x78, 0x88, 0x16, 0x67, + 0x57, 0xd8, 0x61, 0x87, 0x81, 0x81, 0x30, 0x62, 0x03, 0x22, 0x67, 0x98, + 0x7d, 0xf0, 0xd4, 0x71, 0x9c, 0xd3, 0x8f, 0x1b, 0x70, 0x85, 0xfc, 0xa5, + 0x33, 0x4b, 0xe3, 0xa6, 0x00, 0x3a, 0x3c, 0xe7, 0xe1, 0x9a, 0xba, 0x55, + 0x3e, 0x80, 0xcc, 0x5a, 0xe4, 0x06, 0x0e, 0xff, 0x6e, 0x18, 0x06, 0x66, + 0x1d, 0xa5, 0xee, 0xb7, 0xd1, 0x42, 0xd3, 0xb2, 0xe4, 0x07, 0x39, 0xf1, + 0x44, 0x3d, 0xee, 0x3a, 0x19, 0x86, 0x37, 0xf0, 0x3c, 0x06, 0x28, 0x45, + 0xea, 0xff, 0x3f, 0xf2, 0x7e, 0xa3, 0x8d, 0x93, 0x44, 0xd8, 0xa9, 0x02, + 0x22, 0x47, 0x2d, 0xf0, 0x7d, 0xfb, 0x5c, 0x9c, 0x8a, 0xda, 0x77, 0xcd, + 0x0d, 0x5b, 0x94, 0xef, 0xf0, 0x21, 0xe0, 0x2e, 0x30, 0x7d, 0x08, 0x01, + 0x03, 0x12, 0xd5, 0x7c, 0xb5, 0xd9, 0x75, 0x76, 0x46, 0x97, 0x84, 0x2d, +}; + +/* 2048 / 256 | 232 | 1471 | 85 */ +static const uint8_t smallprime_256[] = { + 0x24, 0x65, 0xa7, 0xbd, 0x85, 0x01, 0x1e, 0x1c, 0x9e, 0x05, 0x27, 0x92, + 0x9f, 0xff, 0x26, 0x8c, 0x82, 0xef, 0x7e, 0xfa, 0x41, 0x68, 0x63, 0xba, + 0xa5, 0xac, 0xdb, 0x09, 0x71, 0xdb, 0xa0, 0xcc, 0xac, 0x3e, 0xe4, 0x99, + 0x93, 0x45, 0x02, 0x9f, 0x2c, 0xf8, 0x10, 0xb9, 0x9e, 0x40, 0x6a, 0xac, + 0x5f, 0xce, 0x5d, 0xd6, 0x9d, 0x1c, 0x71, 0x7d, 0xae, 0xa5, 0xd1, 0x8a, + 0xb9, 0x13, 0xf4, 0x56, 0x50, 0x56, 0x79, 0xbc, 0x91, 0xc5, 0x7d, 0x46, + 0xd9, 0x88, 0x88, 0x57, 0x86, 0x2b, 0x36, 0xe2, 0xed, 0xe2, 0xe4, 0x73, + 0xc1, 0xf0, 0xab, 0x35, 0x9d, 0xa2, 0x52, 0x71, 0xaf, 0xfe, 0x15, 0xff, + 0x24, 0x0e, 0x29, 0x9d, 0x0b, 0x04, 0xf4, 0xcd, 0x0e, 0x4d, 0x7c, 0x0e, + 0x47, 0xb1, 0xa7, 0xba, 0x00, 0x7d, 0xe8, 0x9a, 0xae, 0x84, 0x8f, 0xd5, + 0xbd, 0xcd, 0x7f, 0x98, 0x15, 0x56, 0x4e, 0xb0, 0x60, 0xae, 0x14, 0xf1, + 0x9c, 0xb5, 0x0c, 0x29, 0x1f, 0x0b, 0xbd, 0x8e, 0xd1, 0xc4, 0xc7, 0xf8, + 0xfc, 0x5f, 0xba, 0x51, 0x66, 0x20, 0x01, 0x93, 0x9b, 0x53, 0x2d, 0x92, + 0xda, 0xc8, 0x44, 0xa8, 0x43, 0x1d, 0x40, 0x0c, 0x83, 0x2d, 0x03, 0x9f, + 0x5f, 0x90, 0x0b, 0x27, 0x8a, 0x75, 0x21, 0x9c, 0x29, 0x86, 0x14, 0x0c, + 0x79, 0x04, 0x5d, 0x77, 0x59, 0x54, 0x08, 0x54, 0xc3, 0x15, 0x04, 0xdc, + 0x56, 0xf1, 0xdf, 0x5e, 0xeb, 0xe7, 0xbe, 0xe4, 0x47, 0x65, 0x8b, 0x91, + 0x7b, 0xf6, 0x96, 0xd6, 0x92, 0x7f, 0x2e, 0x24, 0x28, 0xfb, 0xeb, 0x34, + 0x0e, 0x51, 0x5c, 0xb9, 0x83, 0x5d, 0x63, 0x87, 0x1b, 0xe8, 0xbb, 0xe0, + 0x9c, 0xf1, 0x34, 0x45, 0x79, 0x9f, 0x2e, 0x67, 0x78, 0x81, 0x51, 0x57, + 0x1a, 0x93, 0xb4, 0xc1, 0xee, 0xe5, 0x5d, 0x1b, 0x90, 0x72, 0xe0, 0xb2, + 0xf5, 0xc4, 0x60, 0x7f, +}; + +/* 3072 / 384 | 326 | 2179 | 85 */ +static const uint8_t smallprime_384[] = { + 0x00, 0x4d, 0xc2, 0x0e, 0x27, 0x31, 0x51, 0x23, 0xfd, 0xab, 0xcd, 0x18, + 0xca, 0x81, 0x2e, 0xe0, 0xee, 0x44, 0x49, 0x23, 0x87, 0x38, 0x9e, 0xd6, + 0xc9, 0x16, 0x97, 0x95, 0x89, 0x65, 0xed, 0xc5, 0x3d, 0x89, 0x13, 0xa8, + 0xe6, 0xec, 0x7f, 0x83, 0x6a, 0x8b, 0xd6, 0x03, 0x7e, 0x57, 0xed, 0x0c, + 0x69, 0x30, 0xef, 0x26, 0x49, 0x0d, 0xc3, 0x5d, 0x05, 0xd0, 0x98, 0xa4, + 0x66, 0xad, 0xf8, 0x17, 0x9f, 0x82, 0x99, 0x69, 0xd1, 0x39, 0x55, 0x8f, + 0x16, 0xe9, 0x8b, 0x3f, 0x76, 0xfc, 0x90, 0x62, 0xc1, 0x57, 0x25, 0xce, + 0x09, 0x88, 0xfa, 0xed, 0xca, 0x96, 0x6a, 0x6b, 0x92, 0x5f, 0x9b, 0x9c, + 0x67, 0x03, 0x43, 0xea, 0x7e, 0x84, 0x20, 0x65, 0xbd, 0x26, 0xf2, 0xbf, + 0x29, 0x90, 0x4f, 0xa7, 0xf4, 0x9f, 0x33, 0x49, 0x28, 0x96, 0x33, 0x73, + 0xba, 0x08, 0x95, 0x96, 0x51, 0x3d, 0xac, 0xa7, 0x39, 0x28, 0xcf, 0x30, + 0x5a, 0xdf, 0x8c, 0x24, 0x6e, 0x1d, 0x99, 0xa2, 0x42, 0xd9, 0x23, 0x56, + 0x23, 0xc4, 0x9a, 0xf2, 0x91, 0x45, 0x06, 0xc9, 0x11, 0x21, 0x5e, 0x1e, + 0x49, 0xaf, 0x84, 0x80, 0x3e, 0xd9, 0xa2, 0xca, 0x05, 0x51, 0x72, 0x1f, + 0xe6, 0x31, 0x9b, 0xf2, 0x38, 0xc0, 0x8a, 0xae, 0x6f, 0xd5, 0x01, 0x54, + 0x03, 0xd9, 0xe5, 0x55, 0x09, 0xee, 0x31, 0xc9, 0x60, 0x12, 0xf9, 0x08, + 0x35, 0x18, 0x5f, 0x31, 0xcb, 0xd2, 0xe4, 0x89, 0x83, 0x3c, 0x1d, 0x54, + 0x62, 0xfa, 0x80, 0x53, 0x59, 0x04, 0x86, 0x7b, 0x2c, 0x94, 0x5e, 0x9a, + 0x0c, 0x2f, 0x7a, 0xa3, 0x6e, 0x0a, 0xc0, 0xeb, 0x9b, 0xb4, 0xc1, 0x1b, + 0xf5, 0x80, 0xcf, 0x0d, 0x6d, 0x2a, 0x49, 0xed, 0x1a, 0x2d, 0x74, 0xca, + 0xe0, 0xf4, 0xc3, 0xad, 0xff, 0x61, 0xd6, 0x48, 0xca, 0x6a, 0x12, 0x08, + 0x58, 0xf4, 0xab, 0xb3, 0xb3, 0x12, 0x07, 0xcf, 0x9b, 0x7c, 0x2f, 0xda, + 0x74, 0xf7, 0x72, 0x2b, 0x14, 0x99, 0x17, 0x87, 0x5a, 0xac, 0x9d, 0x61, + 0x53, 0xc9, 0x71, 0x13, 0xfc, 0xd3, 0x74, 0xaf, 0x93, 0xdd, 0x3f, 0xa2, + 0x1a, 0x7d, 0xe5, 0x1f, 0x1a, 0x70, 0xc6, 0x31, 0xba, 0x6c, 0x92, 0x26, + 0x1e, 0x89, 0x54, 0x1a, 0xa4, 0x71, 0x41, 0xf4, 0x4e, 0x07, 0x5a, 0x1c, + 0x52, 0x2a, 0xe5, 0x81, 0x60, 0xda, 0xc8, 0x70, 0xdf, 0xbd, 0x86, 0x06, + 0xe4, 0xec, 0xa0, 0x89, 0x2a, 0xe5, 0x1c, 0x87, 0x34, 0xf5, 0xb7, 0x71, + 0x2b, 0xcd, 0x3d, 0xe3, 0x32, 0x5e, 0xc2, 0x5f, 0x07, 0xd4, 0xef, 0x94, + 0x33, 0x94, 0xd5, 0xe7, 0xb3, 0x84, 0x10, 0x05, 0xa3, 0xbd, 0x1a, 0x3e, + 0x4d, 0x27, 0x06, 0x1d, 0x54, 0xd2, 0x44, 0x58, 0x24, 0xf8, 0x51, 0x17, + 0xd0, 0xf6, 0x97, 0x12, 0x84, 0xa8, 0xc9, 0x7a, 0x42, 0x50, 0xb9, 0x9b, +}; + +/* 4096 / 512 | 417 | 2887 | 86 */ +static const uint8_t smallprime_512[] = { + 0x09, 0x62, 0x07, 0xfc, 0xcb, 0x19, 0xd6, 0x75, 0x8e, 0x37, 0x4b, 0xee, + 0x6c, 0x37, 0x09, 0xaf, 0x0a, 0x54, 0xa9, 0x82, 0xbf, 0x90, 0x14, 0xe4, + 0x50, 0xb7, 0x48, 0x18, 0x13, 0xb7, 0x30, 0x5b, 0x4c, 0x25, 0xf0, 0xe2, + 0xea, 0x6e, 0x2b, 0x56, 0xf9, 0x1e, 0x59, 0x92, 0x14, 0x2d, 0x21, 0x6e, + 0xae, 0xb2, 0xec, 0xe0, 0x05, 0xfa, 0x0d, 0x18, 0xef, 0xeb, 0x78, 0xef, + 0xc3, 0x41, 0xf3, 0x1f, 0x78, 0x3e, 0xe4, 0x4a, 0xc5, 0xef, 0x5d, 0xfe, + 0x35, 0x57, 0x91, 0x28, 0x21, 0x06, 0x15, 0x6c, 0x64, 0xd1, 0x67, 0xa5, + 0x42, 0x1c, 0xfe, 0xc3, 0x3c, 0xbb, 0xd3, 0x88, 0x38, 0x0b, 0xe8, 0x54, + 0x14, 0x9f, 0xb6, 0x5c, 0x08, 0xe7, 0x9c, 0xd0, 0x4e, 0xc4, 0x8b, 0x45, + 0x62, 0x8e, 0xe6, 0x7f, 0x5c, 0x6f, 0xb0, 0x18, 0x18, 0xfa, 0x1f, 0xf7, + 0x32, 0x24, 0x0c, 0x0b, 0xb1, 0xc7, 0xfe, 0xc1, 0x4c, 0x48, 0x23, 0x4c, + 0x6f, 0xc3, 0xe0, 0x75, 0x76, 0x4f, 0x63, 0xc0, 0x26, 0x83, 0x61, 0x83, + 0x1d, 0x89, 0x60, 0xf2, 0x4b, 0x23, 0x7e, 0x96, 0xc2, 0xca, 0xba, 0x4c, + 0x1a, 0x21, 0x23, 0xff, 0x33, 0xa4, 0x9b, 0xca, 0x39, 0x49, 0xe8, 0xab, + 0xad, 0xde, 0x06, 0xda, 0xc5, 0x70, 0x3d, 0x16, 0xdb, 0x76, 0x77, 0xdf, + 0x2b, 0x0c, 0xe2, 0xc7, 0x84, 0x85, 0xeb, 0xd5, 0xe6, 0x9b, 0xd8, 0x0a, + 0x18, 0x48, 0xa9, 0xfe, 0x28, 0x9c, 0xa2, 0xba, 0x66, 0x4a, 0x68, 0x7b, + 0x3f, 0x05, 0x40, 0x15, 0x6e, 0x67, 0xae, 0x67, 0x69, 0xc0, 0x9e, 0x11, + 0xce, 0x56, 0x73, 0x57, 0xf5, 0xa5, 0x76, 0xa4, 0x8e, 0xed, 0xd9, 0x63, + 0x35, 0xe6, 0x28, 0x77, 0xc7, 0x3a, 0x65, 0x40, 0x8b, 0x71, 0x48, 0x4e, + 0xd0, 0xf1, 0x1d, 0x20, 0xd5, 0x1e, 0x8e, 0x54, 0x67, 0xa1, 0xe4, 0xc0, + 0x9b, 0xf7, 0x29, 0xba, 0x16, 0x9f, 0xcf, 0xdb, 0xa8, 0xb5, 0x5c, 0x4c, + 0x5b, 0x68, 0x2f, 0xaa, 0x28, 0x71, 0x9b, 0x9f, 0x49, 0xbf, 0x36, 0x2d, + 0x9f, 0x03, 0xee, 0x6b, 0xde, 0x79, 0x01, 0xe9, 0x40, 0xe2, 0x49, 0xb4, + 0x1c, 0x93, 0xb9, 0xab, 0x05, 0x4a, 0xbc, 0xab, 0x10, 0x9a, 0xf1, 0x2a, + 0xa6, 0x53, 0x5e, 0xd8, 0xf6, 0x23, 0xab, 0xfd, 0x31, 0x2a, 0xaa, 0x08, + 0x4a, 0x74, 0x8f, 0x86, 0x53, 0x83, 0xbc, 0xe3, 0x15, 0xdc, 0x0d, 0x45, + 0xcb, 0x89, 0x50, 0x8d, 0xec, 0xa9, 0x3b, 0xda, 0x22, 0xf0, 0xe7, 0x7a, + 0x4f, 0xea, 0xa2, 0xa7, 0x90, 0xe0, 0x0e, 0x5a, 0xda, 0x9b, 0xbb, 0x9a, + 0xe7, 0xd5, 0xfb, 0x63, 0x54, 0xa2, 0x52, 0xda, 0x7d, 0xc2, 0x6e, 0x6a, + 0xc2, 0xd7, 0xa6, 0x42, 0xea, 0xbf, 0x48, 0x12, 0xe6, 0x4a, 0xe1, 0x95, + 0xbf, 0x29, 0xcc, 0x9e, 0xe0, 0x25, 0x84, 0xb7, 0x74, 0xdc, 0xb1, 0x12, + 0x91, 0x57, 0xbf, 0x52, 0x43, 0x8f, 0xb7, 0xb7, 0xcd, 0x6a, 0x78, 0x24, + 0xa7, 0x41, 0x8b, 0xcc, 0x65, 0x83, 0x05, 0x8e, 0xc2, 0xf0, 0x69, 0x28, + 0xe4, 0x42, 0x62, 0x37, 0x98, 0xb5, 0x03, 0xf6, 0x75, 0x1d, 0xce, 0xe2, + 0xc0, 0x1f, 0x39, 0xac, 0xb0, 0xfb, 0x47, 0x8f, 0x6e, 0x8b, 0x16, 0xa3, + 0x0f, 0xe8, 0x21, 0x9b, 0x8e, 0x67, 0x04, 0xc7, 0x26, 0xb6, 0x03, 0xe1, + 0x00, 0x09, 0xf6, 0x77, 0x76, 0x46, 0x51, 0x41, 0x57, 0x0d, 0x4b, 0x4c, + 0x2a, 0x30, 0xdb, 0x84, 0x02, 0x6f, 0x93, 0x4b, 0x81, 0xf0, 0xd5, 0xe9, + 0x85, 0xc9, 0x75, 0xd6, 0xa9, 0x07, 0x5a, 0x41, 0xd4, 0x17, 0xc6, 0xd9, + 0x93, 0xcb, 0x49, 0x73, 0xcb, 0xe5, 0x12, 0xa6, 0x7d, 0xb3, 0x1f, 0x6a, + 0xec, 0x8c, 0xc3, 0xe9, 0xe5, 0xeb, 0xdc, 0x1e, 0xb7, 0xb4, 0x74, 0x54, + 0x51, 0x52, 0xa1, 0x56, 0xd5, 0xac, 0x58, 0x7d, +}; + +static const struct smallprime smallprimes[] = { + { .data = smallprime_8, .length = sizeof(smallprime_8) }, + { .data = smallprime_16, .length = sizeof(smallprime_16) }, + { .data = smallprime_32, .length = sizeof(smallprime_32) }, + { .data = smallprime_48, .length = sizeof(smallprime_48) }, + { .data = smallprime_64, .length = sizeof(smallprime_64) }, + { .data = smallprime_72, .length = sizeof(smallprime_72) }, + { .data = smallprime_96, .length = sizeof(smallprime_96) }, + { .data = smallprime_128, .length = sizeof(smallprime_128) }, + { .data = smallprime_184, .length = sizeof(smallprime_184) }, + { .data = smallprime_192, .length = sizeof(smallprime_192) }, + { .data = smallprime_256, .length = sizeof(smallprime_256) }, + { .data = smallprime_384, .length = sizeof(smallprime_384) }, + { .data = smallprime_512, .length = sizeof(smallprime_512) }, +}; + +/* + * Search the small prime closed to the given input bytes size + * + * @size Size in bytes + * @prime [out] Output predefined small prime + */ +static void search_smallprime(size_t size, struct caambuf *prime) +{ + size_t nb_elem = ARRAY_SIZE(smallprimes); + size_t idx = 0; + size_t psize = 0; + + for (; idx < nb_elem; idx++) { + psize = smallprimes[idx].length; + + if (psize == size) { + /* Found a predefined prime */ + RSA_TRACE("Found prime idx %zu", idx); + prime->data = (uint8_t *)smallprimes[idx].data; + prime->length = psize; + prime->paddr = virt_to_phys(prime->data); + break; + } + } +} + +/* + * Build the descriptor preparing the CAAM global variables used during the + * prime generation + * + * @desc [out] Descriptor built + * @data Prime generation data + * @small_prime Pre-generated small prime value + * @desc_prime Physical address of the prime generator descriptor + */ +static void do_desc_setup(uint32_t *desc, struct prime_data_rsa *data, + const struct caambuf *small_prime, + const paddr_t desc_prime) +{ + /* + * Referring to FIPS.186-4, B.3.3 (step 4.7) + * Maximum tries = 5 * (nlen / 2) + * Where nlen is the RSA security length in bit + */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + caam_desc_add_word(desc, MATH(ADD, IMM_DATA, ZERO, SOL, 4)); + caam_desc_add_word(desc, 5 * (data->key_size / 2)); + + /* + * Referring to FIPS.186-4, Table C.2 + * Get the number Miller-Rabin test interation function + * of the prime number size + */ + caam_desc_add_word(desc, MATH(ADD, IMM_DATA, ZERO, SIL, 4)); + if (data->p->length > (MR_PRIME_SIZE / 8)) + caam_desc_add_word(desc, 0x4); + else + caam_desc_add_word(desc, 0x5); + + /* + * Preload PKHA A2 with the sqrt_value array (step 4.4) + * Do it once, not at each loop + */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_A2, NOACTION, + data->p->length)); + caam_desc_add_ptr(desc, virt_to_phys((void *)sqrt_value)); + + if (data->era >= 8 && small_prime->paddr) { + /* + * Preload PKHA B2 with small prime predefined + * (preload only prime size requested) + * + * Before Era 8, the PRIME TEST function overwrites PKHA B2 + * hence PKHA B2 must be reloaded if new prime tentative after + * PRIME TEST on Era < 8 + */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_B2, NOACTION, + small_prime->length)); + caam_desc_add_ptr(desc, small_prime->paddr); + } + + /* Set the High order bit used to turn on MSB in prime candidate */ + caam_desc_add_word(desc, MATHI_OP1(SHIFT_L, ONE, 0x3F, REG2, 8)); + + /* Load PKHA N Size with the prime size */ + caam_desc_add_word(desc, LD_IMM(CLASS_1, REG_PKHA_N_SIZE, 4)); + caam_desc_add_word(desc, data->p->length); + + /* + * Set the number of maximum tries because of generated value + * is too small. This value is used to not lock the system + * in prime number generation + */ + caam_desc_add_word(desc, MATH(ADD, ZERO, IMM_DATA, DPOVRD, 4)); + caam_desc_add_word(desc, MAX_RETRY_PRIME_GEN); + + /* Jump to the next descriptor desc */ + caam_desc_add_word(desc, JUMP_NOTLOCAL(CLASS_NO, ALL_COND_TRUE, + JMP_COND(NONE))); + caam_desc_add_ptr(desc, desc_prime); + + RSA_DUMPDESC(desc); + cache_operation(TEE_CACHECLEAN, (void *)sqrt_value, data->p->length); +} + +/* + * Build the descriptor generating a prime + * + * @desc [out] Descriptor built + * @data Prime generation data + * @small_prime Pre-generated small prime value + * @do_prime_q Generate Prime Q + * @desc_next Physical address of the next descriptor (can be NULL) + */ +static void do_desc_prime(uint32_t *desc, struct prime_data_rsa *data, + const struct caambuf *small_prime, bool do_prime_q, + const paddr_t desc_next) +{ + uint32_t desclen = 0; + uint32_t retry_too_small = 0; + uint32_t retry_new_number = 0; + uint32_t retry_new_mr_failed = 0; + uint32_t retry_mr_test = 0; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + /* Setup the number of try counter = MAX (counting down) */ + caam_desc_add_word(desc, MATH(ADD, SOL, ZERO, VSOL, 4)); + + retry_new_mr_failed = caam_desc_get_len(desc); + if (data->era < 8 && small_prime->paddr) { + /* + * Preload PKHA B2 with small prime predefined + * (preload only prime size requested) + */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_B2, NOACTION, + small_prime->length)); + caam_desc_add_ptr(desc, small_prime->paddr); + } + + retry_new_number = caam_desc_get_len(desc); + /* Decrement the number of try */ + caam_desc_add_word(desc, MATH(SUB, VSOL, ONE, VSOL, 4)); + /* Exceed retry count - exit with RSA_TRY_FAIL error */ + caam_desc_add_word(desc, + HALT_USER(ALL_COND_TRUE, MATH_N, RSA_TRY_FAIL)); + + retry_too_small = caam_desc_get_len(desc); + /* Check internal limit on random value generation */ + caam_desc_add_word(desc, MATH(SUB, DPOVRD, ONE, DPOVRD, 4)); + caam_desc_add_word(desc, + HALT_USER(ALL_COND_TRUE, MATH_Z, RETRY_TOO_SMALL)); + + /* + * Step 4.2 - Obtain a string p of (nlen/2) bits + * Step 4.3 - if (p is not odd) then p = p + 1 + */ + /* Generate 16 random bytes load into DECO fifo */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 4)); + caam_desc_add_word(desc, NFIFO_PAD(DECO, NFIFO_LC1, MSG, RND, 16)); + + /* Get the DECO Input fifo 8 MSB and force on high bit */ + caam_desc_add_word(desc, MATH(OR, REG2, IFIFO, REG0, 8)); + /* Get the DECO Input fifo 8 LSB and force it be be odd */ + caam_desc_add_word(desc, MATH(OR, ONE, IFIFO, REG1, 8)); + /* Move the MSB and LSB into IFIFO */ + caam_desc_add_word(desc, MOVE(MATH_REG0, IFIFO, 0, 16)); + /* Send the 8 MSB into PKHA N */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 4)); + caam_desc_add_word(desc, NFIFO_NOPAD(C1, 0, IFIFO, PKHA_N, 8)); + + /* + * Generate the "middle" random bytes and start them + * on their way into PKHA N + */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 8)); + caam_desc_add_word(desc, NFIFO_PAD(C1, 0, PKHA_N, RND, 0)); + caam_desc_add_word(desc, data->p->length - 16); + + /* And send the 8 LSB into PKHA N */ + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 4)); + caam_desc_add_word(desc, NFIFO_NOPAD(C1, NFIFO_FC1, IFIFO, PKHA_N, 8)); + + /* + * Step 4.4 - if ((prime < (sqrt 2)(2^((nlen / 2) - 1)) + * ==> retry_too_small + */ + caam_desc_add_word(desc, PKHA_CPY_SSIZE(A2, B0)); + caam_desc_add_word(desc, PKHA_CPY_SSIZE(B0, A0)); + caam_desc_add_word(desc, PKHA_OP(MOD_AMODN, A)); + caam_desc_add_word(desc, PKHA_CPY_SSIZE(A2, B0)); + caam_desc_add_word(desc, PKHA_F2M_OP(MOD_ADD_A_B, B)); + + desclen = caam_desc_get_len(desc); + caam_desc_add_word(desc, JUMP_CNO_LOCAL(ANY_COND_FALSE, + JMP_COND(PKHA_IS_ZERO), + retry_too_small - desclen)); + + /* + * Step 4.5 - Compute GCD(prime-1, e) and test if = 1 else try + * another candidate + */ + caam_desc_add_word(desc, PKHA_CPY_SSIZE(N0, A0)); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_word(desc, 0x01); + caam_desc_add_word(desc, PKHA_F2M_OP(MOD_ADD_A_B, B)); + caam_desc_add_word(desc, PKHA_CPY_SSIZE(B0, N0)); + + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, PKHA_A, NOACTION, data->e->length)); + caam_desc_add_ptr(desc, data->e->paddr); + caam_desc_add_word(desc, PKHA_OP(GCD_A_N, B)); + + desclen = caam_desc_get_len(desc); + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ANY_COND_FALSE, JMP_COND(PKHA_GCD_1), + retry_new_number - desclen)); + + caam_desc_add_word(desc, PKHA_CPY_SSIZE(N0, A0)); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_word(desc, 0x01); + caam_desc_add_word(desc, PKHA_F2M_OP(MOD_ADD_A_B, B)); + caam_desc_add_word(desc, PKHA_CPY_SSIZE(B0, N0)); + + /* + * Step 4.5.1 - test primality + */ + if (small_prime->paddr) { + caam_desc_add_word(desc, PKHA_CPY_SSIZE(B2, A0)); + caam_desc_add_word(desc, PKHA_OP(GCD_A_N, B)); + desclen = caam_desc_get_len(desc); + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ANY_COND_FALSE, + JMP_COND(PKHA_GCD_1), + retry_new_number - desclen)); + } + + /* Generate 8 random bytes 'miller-rabin seed' */ + /* Load the number of Miller-Rabin test iteration */ + caam_desc_add_word(desc, MATH(ADD, SIL, ZERO, VSIL, 4)); + retry_mr_test = caam_desc_get_len(desc); + caam_desc_add_word(desc, LD_IMM(CLASS_NO, REG_NFIFO, 8)); + caam_desc_add_word(desc, NFIFO_PAD(C1, NFIFO_FC1, PKHA_A, RND, 0)); + caam_desc_add_word(desc, data->p->length); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, PKHA_B, NOACTION, 1)); + caam_desc_add_word(desc, 0x01); + caam_desc_add_word(desc, PKHA_OP(MR_PRIMER_TEST, B)); + + desclen = caam_desc_get_len(desc); + caam_desc_add_word(desc, JUMP_CNO_LOCAL(ANY_COND_FALSE, + JMP_COND(PKHA_IS_PRIME), + retry_new_mr_failed - desclen)); + caam_desc_add_word(desc, MATH(SUB, VSIL, ONE, VSIL, 4)); + + desclen = caam_desc_get_len(desc); + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ALL_COND_FALSE, + JMP_COND(MATH_N) | JMP_COND(MATH_Z), + retry_mr_test - desclen)); + + /* Save prime generated */ + caam_desc_add_word(desc, FIFO_ST(PKHA_N, data->p->length)); + + if (do_prime_q) + caam_desc_add_ptr(desc, data->q->paddr); + else + caam_desc_add_ptr(desc, data->p->paddr); + + if (desc_next) { + /* Jump to the next descriptor desc */ + caam_desc_add_word(desc, JUMP_NOTLOCAL(CLASS_NO, ALL_COND_TRUE, + JMP_COND(NONE))); + caam_desc_add_ptr(desc, desc_next); + } + + RSA_DUMPDESC(desc); +} + +/* + * Build the descriptor to check primes p and q not too closed. + * Check the upper 100 bits with operation: + * |p - q| <= 2^(nlen/2-100) + * + * @desc [out] Descriptor built + * @p Prime P + * @max_n Max N built with 0xFFFF... + * @desc_new_q Physical address to generate a new Q value + */ +static void do_checks_primes(uint32_t *desc, const struct caambuf *p, + const struct caambuf *max_n, + const paddr_t desc_new_q) +{ + const uint8_t check_len = 16; /* Check 128 bits */ + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + /* Load prime p */ + caam_desc_add_word(desc, FIFO_LD(CLASS_1, PKHA_B, NOACTION, p->length)); + caam_desc_add_ptr(desc, p->paddr); + + /* Retrieve Q from PKHA N, previously computed */ + caam_desc_add_word(desc, PKHA_CPY_SSIZE(N0, A0)); + + /* Calculate p - q, need a modulus of size prime p filled with 0xFF */ + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, PKHA_N, NOACTION, max_n->length)); + caam_desc_add_ptr(desc, max_n->paddr); + + /* PKHA_B = p - q */ + caam_desc_add_word(desc, PKHA_OP(MOD_SUB_A_B, B)); + + /* Unload PKHA register B to output Data FIFO */ + caam_desc_add_word(desc, LD_NOCLASS_IMM(REG_CHA_CTRL, 4)); + caam_desc_add_word(desc, CCTRL_ULOAD_PKHA_B); + + /* Get the first 128 bits in MATH 0 */ + caam_desc_add_word(desc, MOVE_WAIT(OFIFO, MATH_REG0, 0, check_len)); + + /* + * We now need to trash the rest of the result. + * We started with 128, 192, or 256 bytes in the OFIFO before we moved + * check_len bytes into MATH registers. + */ + if (p->length > 128 + (size_t)check_len) { + caam_desc_add_word(desc, MOVE(OFIFO, C1_CTX_REG, 0, check_len)); + caam_desc_add_word(desc, MOVE(OFIFO, C1_CTX_REG, 0, + (p->length - 128 - check_len))); + } else if (p->length > check_len) { + caam_desc_add_word(desc, MOVE(OFIFO, C1_CTX_REG, 0, + (p->length - check_len))); + } + + /* + * In MATH registers we have the p - q value modulo 0xFFFFF... + * Check the upper 100 bits are either zero or one meaning + * q is too close to p + */ + /* Check first 64 bits if not 0's check if 1's */ + caam_desc_add_word(desc, MATH(ADD, ZERO, REG0, REG0, 8)); + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ANY_COND_FALSE, JMP_COND(MATH_Z), 6)); + /* First 64 bits are 0's, check next 36 bits */ + caam_desc_add_word(desc, MATH(AND, REG1, IMM_DATA, REG1, 8)); + caam_desc_add_word(desc, UINT32_MAX); + caam_desc_add_word(desc, 0xF0000000); + + /* Next 36 bits are 0 */ + caam_desc_add_word(desc, + JUMP_CNO_LOCAL(ALL_COND_TRUE, JMP_COND(MATH_Z), 10)); + /* Exit status GOOD Q */ + caam_desc_add_word(desc, HALT_USER(ALL_COND_TRUE, NONE, STATUS_GOOD_Q)); + + /* Check if 100 bits are 1's */ + caam_desc_add_word(desc, MATH(ADD, ONE, REG0, REG0, 8)); + /* Not all 1's exit status GOOD Q */ + caam_desc_add_word(desc, + HALT_USER(ANY_COND_FALSE, MATH_Z, STATUS_GOOD_Q)); + /* First 64 bits are 1's, check next 36 bits */ + caam_desc_add_word(desc, MATH(AND, REG1, IMM_DATA, REG1, 8)); + caam_desc_add_word(desc, UINT32_MAX); + caam_desc_add_word(desc, SHIFT_U32(0xF, 28)); + + /* Use only 4 bytes of immediate data even is operation is 8 bytes */ + caam_desc_add_word(desc, MATH(ADD, REG1, IMM_DATA, REG1, 8) | MATH_IFB); + caam_desc_add_word(desc, SHIFT_U32(1, 28)); + + /* Not all 1's exit status GOOD Q */ + caam_desc_add_word(desc, + HALT_USER(ANY_COND_FALSE, MATH_Z, STATUS_GOOD_Q)); + + if (desc_new_q) { + caam_desc_add_word(desc, JUMP_NOTLOCAL(CLASS_NO, ALL_COND_TRUE, + JMP_COND(NONE))); + caam_desc_add_ptr(desc, desc_new_q); + } + + RSA_DUMPDESC(desc); +} + +/* + * Run the Primes descriptor. + * + * @desc Descriptor built + * @prime Prime generation data + */ +static enum caam_status run_primes(uint32_t *desc, struct prime_data_rsa *data) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + + cache_operation(TEE_CACHEFLUSH, data->p->data, data->p->length); + + if (data->q) + cache_operation(TEE_CACHEFLUSH, data->q->data, data->q->length); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (data->q && retstatus == CAAM_JOB_STATUS) { + /* + * Expect to have a retstatus == CAAM_JOB_STATUS, where + * job status == STATUS_GOOD_Q + */ + RSA_TRACE("Check Prime Q Status 0x%08" PRIx32, jobctx.status); + + if (JRSTA_GET_HALT_USER(jobctx.status) == STATUS_GOOD_Q) { + cache_operation(TEE_CACHEINVALIDATE, data->p->data, + data->p->length); + cache_operation(TEE_CACHEINVALIDATE, data->q->data, + data->q->length); + + RSA_DUMPBUF("Prime P", data->p->data, data->p->length); + RSA_DUMPBUF("Prime Q", data->q->data, data->q->length); + retstatus = CAAM_NO_ERROR; + } + } else if (retstatus == CAAM_NO_ERROR && !data->q) { + cache_operation(TEE_CACHEINVALIDATE, data->p->data, + data->p->length); + + RSA_DUMPBUF("Prime", data->p->data, data->p->length); + } else if (retstatus != CAAM_NO_ERROR) { + RSA_TRACE("Prime Status 0x%08" PRIx32, jobctx.status); + } + + return retstatus; +} + +enum caam_status caam_prime_rsa_gen(struct prime_data_rsa *data) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caambuf small_prime = { }; + struct caambuf max_n = { }; + uint32_t *all_descs = NULL; + uint32_t *desc_p = NULL; + uint32_t *desc_q = NULL; + uint32_t *desc_check_p_q = NULL; + paddr_t paddr_desc_p = 0; + paddr_t paddr_desc_q = 0; + paddr_t paddr_desc_check_p_q = 0; + size_t size_all_descs = 0; + size_t nb_tries = RSA_MAX_TRIES_PRIMES; + + /* Allocate the job used to prepare the operation */ + if (data->q) { + size_all_descs = SETUP_RSA_DESC_ENTRIES + + GEN_RSA_DESC_ENTRIES * 2 + + CHECK_P_Q_DESC_ENTRIES; + + retstatus = caam_calloc_buf(&max_n, data->p->length + 1); + if (retstatus != CAAM_NO_ERROR) + goto end_gen_prime; + + /* Set the max_n with 0xFFF... to operate the check P and Q */ + memset(max_n.data, UINT8_MAX, max_n.length); + cache_operation(TEE_CACHECLEAN, max_n.data, max_n.length); + } else { + size_all_descs = SETUP_RSA_DESC_ENTRIES + GEN_RSA_DESC_ENTRIES; + } + + all_descs = caam_calloc_desc(size_all_descs); + if (!all_descs) { + retstatus = CAAM_OUT_MEMORY; + goto end_gen_prime; + } + + /* Descriptor Prime P */ + desc_p = all_descs + SETUP_RSA_DESC_ENTRIES; + paddr_desc_p = virt_to_phys(desc_p); + if (!paddr_desc_p) { + retstatus = CAAM_FAILURE; + goto end_gen_prime; + } + + /* + * Search predefined prime in the small_prime list, if the + * small prime is not found in the list, continue anyway + * but prime will be probably not so strong + */ + search_smallprime(data->p->length, &small_prime); + + RSA_TRACE("Do prime of %zu bytes (security len %zu bits) (ERA=%" PRId8 + ")", + data->p->length, data->key_size, data->era); + + do_desc_setup(all_descs, data, &small_prime, paddr_desc_p); + + if (data->q) { + /* Descriptor Prime Q */ + desc_q = desc_p + GEN_RSA_DESC_ENTRIES; + paddr_desc_q = + paddr_desc_p + DESC_SZBYTES(GEN_RSA_DESC_ENTRIES); + + /* Descriptor Check Primes P & Q */ + desc_check_p_q = desc_q + GEN_RSA_DESC_ENTRIES; + paddr_desc_check_p_q = + paddr_desc_q + DESC_SZBYTES(GEN_RSA_DESC_ENTRIES); + + /* Generate Prime P and Q then check Q not too close than P */ + do_desc_prime(desc_p, data, &small_prime, false, paddr_desc_q); + + do_desc_prime(desc_q, data, &small_prime, true, + paddr_desc_check_p_q); + + do_checks_primes(desc_check_p_q, data->p, &max_n, paddr_desc_q); + } else { + do_desc_prime(desc_p, data, &small_prime, false, 0); + } + + cache_operation(TEE_CACHECLEAN, small_prime.data, data->p->length); + cache_operation(TEE_CACHECLEAN, data->e->data, data->e->length); + cache_operation(TEE_CACHECLEAN, (void *)all_descs, + DESC_SZBYTES(size_all_descs)); + + do { + retstatus = run_primes(all_descs, data); + } while (--nb_tries && retstatus != CAAM_NO_ERROR); + +end_gen_prime: + caam_free_desc(&all_descs); + caam_free_buf(&max_n); + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/acipher/caam_rsa.c b/optee_os/core/drivers/crypto/caam/acipher/caam_rsa.c new file mode 100644 index 0000000..580048b --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/caam_rsa.c @@ -0,0 +1,1580 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * CAAM RSA manager. + * Implementation of RSA functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +/* + * Definition of the maximum bits of Exponent e + * Refer to sp800-56b + */ +#define MAX_BITS_EXP_E 256 + +/* + * Define the maximum number of entries in a descriptor + * function of the encrypt/decrypt and private key format + */ +#ifdef CFG_CAAM_64BIT +#define MAX_DESC_ENC (8 + 4) +#define MAX_DESC_DEC_1 (7 + 2 + 4) +#define MAX_DESC_DEC_2 (11 + 2 + 7) +#define MAX_DESC_DEC_3 (13 + 2 + 10) +/* Define the maximum number of entries in the RSA Finish Key descriptor */ +#define MAX_DESC_KEY_FINISH 24 +#else +#define MAX_DESC_ENC 8 +#define MAX_DESC_DEC_1 (7 + 2) +#define MAX_DESC_DEC_2 (11 + 2) +#define MAX_DESC_DEC_3 (13 + 2) +/* Define the maximum number of entries in the RSA Finish Key descriptor */ +#define MAX_DESC_KEY_FINISH 15 +#endif /* CFG_CAAM_64BIT */ + +static TEE_Result do_caam_encrypt(struct drvcrypt_rsa_ed *rsa_data, + uint32_t operation); +static TEE_Result do_caam_decrypt(struct drvcrypt_rsa_ed *rsa_data, + uint32_t operation); + +/* + * Definition of the local RSA keypair + * Public Key Format: (n, e) + * Private Key Format #1: (n, d) + * Private Key Format #2: (p, q, d) + * Private Key Format #3: (p, q, dp, dq, qp) + */ +struct caam_rsa_keypair { + uint8_t format; /* Define the Private Key Format (1, 2 or 3) */ + struct caambuf n; /* Modulus [n = p * q] */ + struct caambuf e; /* Public Exponent 65537 <= e < 2^256 */ + struct caambuf d; /* Private Exponent [d = 1/e mod LCM(p-1, q-1)] */ + struct caambuf p; /* Private Prime p */ + struct caambuf q; /* Private Prime q */ + struct caambuf dp; /* Private [dp = d mod (p-1)] */ + struct caambuf dq; /* Private [dq = d mod (q-1)] */ + struct caambuf qp; /* Private [qp = 1/q mod p] */ +}; + +#define RSA_PRIVATE_KEY_FORMAT_1 1 +#define RSA_PRIVATE_KEY_FORMAT_2 2 +#define RSA_PRIVATE_KEY_FORMAT_3 3 + +/* CAAM Era version */ +static uint8_t caam_era; + +/* + * Free RSA keypair + * + * @key RSA keypair + */ +static void do_free_keypair(struct rsa_keypair *key) +{ + crypto_bignum_free(&key->e); + crypto_bignum_free(&key->d); + crypto_bignum_free(&key->n); + crypto_bignum_free(&key->p); + crypto_bignum_free(&key->q); + crypto_bignum_free(&key->qp); + crypto_bignum_free(&key->dp); + crypto_bignum_free(&key->dq); +} + +/* + * Free local caam RSA keypair + * + * @key caam RSA keypair + */ +static void do_keypair_free(struct caam_rsa_keypair *key) +{ + caam_free_buf(&key->e); + caam_free_buf(&key->n); + caam_free_buf(&key->d); + + if (key->format > RSA_PRIVATE_KEY_FORMAT_1 && key->p.data) { + key->p.length += key->q.length; + caam_free_buf(&key->p); + } + + if (key->format > RSA_PRIVATE_KEY_FORMAT_2 && key->dp.data) { + key->dp.length += key->dq.length + key->qp.length; + caam_free_buf(&key->dp); + } +} + +/* + * Convert Crypto RSA Key to local RSA Public Key + * Ensure Key is push in physical memory + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + */ +static enum caam_status do_keypub_conv(struct caam_rsa_keypair *outkey, + const struct rsa_public_key *inkey) +{ + enum caam_status retstatus = CAAM_FAILURE; + + RSA_TRACE("RSA Convert Public Key size N=%zu", + crypto_bignum_num_bytes(inkey->n)); + + retstatus = caam_calloc_align_buf(&outkey->e, + crypto_bignum_num_bytes(inkey->e)); + if (retstatus != CAAM_NO_ERROR) + goto exit_conv; + + crypto_bignum_bn2bin(inkey->e, outkey->e.data); + cache_operation(TEE_CACHECLEAN, outkey->e.data, outkey->e.length); + + retstatus = caam_calloc_align_buf(&outkey->n, + crypto_bignum_num_bytes(inkey->n)); + if (retstatus != CAAM_NO_ERROR) + goto exit_conv; + + crypto_bignum_bn2bin(inkey->n, outkey->n.data); + cache_operation(TEE_CACHECLEAN, outkey->n.data, outkey->n.length); + + return CAAM_NO_ERROR; + +exit_conv: + do_keypair_free(outkey); + + return CAAM_OUT_MEMORY; +} + +/* + * Convert Crypto RSA Key additional fields of the key format #3 + * Optional fields (dp, dq, qp) + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + */ +static enum caam_status do_keypair_conv_f3(struct caam_rsa_keypair *outkey, + const struct rsa_keypair *inkey) +{ + enum caam_status retstatus = CAAM_FAILURE; + size_t size_p = 0; + size_t size_q = 0; + size_t size_dp = 0; + size_t size_dq = 0; + size_t size_qp = 0; + + size_p = outkey->p.length; + size_q = outkey->q.length; + size_dp = crypto_bignum_num_bytes(inkey->dp); + size_dq = crypto_bignum_num_bytes(inkey->dq); + size_qp = crypto_bignum_num_bytes(inkey->qp); + + /* Check that dp, dq and qp size not exceed p and q size */ + if (size_dp > size_p || size_dq > size_q || size_qp > size_p) + return CAAM_FAILURE; + + /* + * If one of the parameters dp, dq or qp are not filled, + * returns immediately. This is not an error. + */ + if (!size_dp || !size_dq || !size_qp) + return CAAM_NO_ERROR; + + /* + * CAAM is assuming that: + * - dp and dq are same size as p + * - dq same size as q + * + * Because calculation of dp, dq and qp can be less + * than above assumption, force the dp, dq and qp + * buffer size. + */ + /* Allocate one buffer for the 3 fields */ + retstatus = + caam_calloc_align_buf(&outkey->dp, size_p + size_q + size_p); + if (retstatus != CAAM_NO_ERROR) + return CAAM_OUT_MEMORY; + + /* Field dp */ + outkey->dp.length = size_p; + + /* + * Ensure buffer is copied starting with 0's + * if size_dp != size_p + */ + crypto_bignum_bn2bin(inkey->dp, outkey->dp.data + size_p - size_dp); + + /* Field dq */ + outkey->dq.data = outkey->dp.data + size_p; + outkey->dq.length = size_q; + outkey->dq.paddr = outkey->dp.paddr + size_p; + + /* + * Ensure buffer is copied starting with 0's + * if size_dq != size_q + */ + crypto_bignum_bn2bin(inkey->dq, outkey->dq.data + size_q - size_dq); + + /* Field qp */ + outkey->qp.data = outkey->dq.data + size_q; + outkey->qp.length = size_p; + outkey->qp.paddr = outkey->dq.paddr + size_q; + + /* + * Ensure buffer is copied starting with 0's + * if size_qp != size_p + */ + crypto_bignum_bn2bin(inkey->qp, outkey->qp.data + size_p - size_qp); + + /* Push fields value to the physical memory */ + cache_operation(TEE_CACHECLEAN, outkey->dp.data, + outkey->dp.length + outkey->dq.length + + outkey->qp.length); + + outkey->format = RSA_PRIVATE_KEY_FORMAT_3; + + return CAAM_NO_ERROR; +} + +/* + * Convert Crypto RSA Key additional fields of the key format #2 + * Optional fields (p, q) + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + */ +static enum caam_status do_keypair_conv_f2(struct caam_rsa_keypair *outkey, + const struct rsa_keypair *inkey) +{ + enum caam_status retstatus = CAAM_FAILURE; + size_t size_p = 0; + size_t size_q = 0; + + size_p = crypto_bignum_num_bytes(inkey->p); + size_q = crypto_bignum_num_bytes(inkey->q); + + /* + * If the Prime P or Prime Q are not filled, returns + * immediately. This is not an error. + */ + if (size_p || !size_q) + return CAAM_NO_ERROR; + + /* Allocate one buffer for both */ + retstatus = caam_calloc_align_buf(&outkey->p, size_p + size_q); + if (retstatus != CAAM_NO_ERROR) + return CAAM_OUT_MEMORY; + + /* Field Prime p */ + outkey->p.length = size_p; + crypto_bignum_bn2bin(inkey->p, outkey->p.data); + + /* Field Prime q */ + outkey->q.data = outkey->p.data + size_p; + outkey->q.length = size_q; + outkey->q.paddr = outkey->p.paddr + size_p; + + crypto_bignum_bn2bin(inkey->q, outkey->q.data); + + /* Push fields value to the physical memory */ + cache_operation(TEE_CACHECLEAN, outkey->p.data, size_p + size_q); + + outkey->format = RSA_PRIVATE_KEY_FORMAT_2; + + if (CFG_NXP_CAAM_RSA_KEY_FORMAT > RSA_PRIVATE_KEY_FORMAT_2) { + retstatus = do_keypair_conv_f3(outkey, inkey); + RSA_TRACE("do_keypair_conv_f3 returned 0x%" PRIx32, retstatus); + } + + return retstatus; +} + +/* + * Convert Crypto RSA Key to local RSA Keypair Key + * Ensure Key is push in physical memory + * Don't convert the exponent e not used in decrytion + * + * @outkey [out] Output keypair in local format + * @inkey Input key in TEE Crypto format + */ +static enum caam_status do_keypair_conv(struct caam_rsa_keypair *outkey, + const struct rsa_keypair *inkey) +{ + enum caam_status retstatus = CAAM_FAILURE; + + RSA_TRACE("RSA Convert Keypair size N=%zu", + crypto_bignum_num_bytes(inkey->n)); + + /* Mandatory fields are n and d => Private Key Format #1 */ + retstatus = caam_calloc_align_buf(&outkey->n, + crypto_bignum_num_bytes(inkey->n)); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + crypto_bignum_bn2bin(inkey->n, outkey->n.data); + cache_operation(TEE_CACHECLEAN, outkey->n.data, outkey->n.length); + + retstatus = caam_calloc_align_buf(&outkey->d, + crypto_bignum_num_bytes(inkey->d)); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + crypto_bignum_bn2bin(inkey->d, outkey->d.data); + cache_operation(TEE_CACHECLEAN, outkey->d.data, outkey->d.length); + + outkey->format = RSA_PRIVATE_KEY_FORMAT_1; + + if (CFG_NXP_CAAM_RSA_KEY_FORMAT > RSA_PRIVATE_KEY_FORMAT_1) { + retstatus = do_keypair_conv_f2(outkey, inkey); + RSA_TRACE("do_keypair_conv_f2 returned 0x%" PRIx32, retstatus); + } + + return retstatus; +} + +/* + * Allocate a RSA keypair + * + * @key Keypair + * @size_bits Key size in bits + */ +static TEE_Result do_allocate_keypair(struct rsa_keypair *key, size_t size_bits) +{ + RSA_TRACE("Allocate Keypair of %zu bits", size_bits); + + /* Initialize all input key fields to 0 */ + memset(key, 0, sizeof(*key)); + + /* Allocate the Public Exponent to maximum size */ + key->e = crypto_bignum_allocate(MAX_BITS_EXP_E); + if (!key->e) + goto err_alloc_keypair; + + /* Allocate the Private Exponent [d = 1/e mod LCM(p-1, q-1)] */ + key->d = crypto_bignum_allocate(size_bits); + if (!key->d) + goto err_alloc_keypair; + + /* Allocate the Modulus (size_bits) [n = p * q] */ + key->n = crypto_bignum_allocate(size_bits); + if (!key->n) + goto err_alloc_keypair; + + /* Allocate the prime number p of size (size_bits / 2) */ + key->p = crypto_bignum_allocate(size_bits / 2); + if (!key->p) + goto err_alloc_keypair; + + /* Allocate the prime number q of size (size_bits / 2) */ + key->q = crypto_bignum_allocate(size_bits / 2); + if (!key->q) + goto err_alloc_keypair; + + /* Allocate dp (size_bits / 2) [d mod (p-1)] */ + key->dp = crypto_bignum_allocate(size_bits / 2); + if (!key->dp) + goto err_alloc_keypair; + + /* Allocate dq (size_bits / 2) [d mod (q-1)] */ + key->dq = crypto_bignum_allocate(size_bits / 2); + if (!key->dq) + goto err_alloc_keypair; + + /* Allocate qp (size_bits / 2) [1/q mod p] */ + key->qp = crypto_bignum_allocate(size_bits / 2); + if (!key->qp) + goto err_alloc_keypair; + + return TEE_SUCCESS; + +err_alloc_keypair: + RSA_TRACE("Allocation error"); + + do_free_keypair(key); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Allocate a RSA public key + * + * @key Public Key + * @size_bits Key size in bits + */ +static TEE_Result do_allocate_publickey(struct rsa_public_key *key, + size_t size_bits) +{ + RSA_TRACE("Allocate Public Key of %zu bits", size_bits); + + /* Initialize all input key fields to 0 */ + memset(key, 0, sizeof(*key)); + + /* Allocate the Public Exponent to maximum size */ + key->e = crypto_bignum_allocate(MAX_BITS_EXP_E); + if (!key->e) + goto err_alloc_publickey; + + /* Allocate the Modulus (size_bits) [n = p * q] */ + key->n = crypto_bignum_allocate(size_bits); + if (!key->n) + goto err_alloc_publickey; + + return TEE_SUCCESS; + +err_alloc_publickey: + RSA_TRACE("Allocation error"); + + crypto_bignum_free(&key->e); + crypto_bignum_free(&key->n); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Free a RSA public key + * + * @key Public Key + */ +static void do_free_publickey(struct rsa_public_key *key) +{ + crypto_bignum_free(&key->e); + crypto_bignum_free(&key->n); +} + +/* + * Output the RSA keypair format 3 additional fields in bignumber object + * + * @key [out] Keypair + * @key_size Key size in bits + */ +static TEE_Result gen_keypair_get_f3(struct rsa_keypair *key, + struct caam_rsa_keypair *genkey) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + cache_operation(TEE_CACHEINVALIDATE, genkey->dp.data, + genkey->dp.length + genkey->dq.length + + genkey->qp.length); + + RSA_DUMPBUF("dp", genkey->dp.data, genkey->dp.length); + RSA_DUMPBUF("dq", genkey->dq.data, genkey->dq.length); + RSA_DUMPBUF("qp", genkey->qp.data, genkey->qp.length); + + ret = crypto_bignum_bin2bn(genkey->dp.data, genkey->dp.length, key->dp); + if (ret != TEE_SUCCESS) + return ret; + + ret = crypto_bignum_bin2bn(genkey->dq.data, genkey->dq.length, key->dq); + if (ret != TEE_SUCCESS) + return ret; + + ret = crypto_bignum_bin2bn(genkey->qp.data, genkey->qp.length, key->qp); + return ret; +} + +/* + * Output the RSA keypair format 2 additional fields in big number object + * + * @key [out] Keypair + * @key_size Key size in bits + */ +static TEE_Result gen_keypair_get_f2(struct rsa_keypair *key, + struct caam_rsa_keypair *genkey) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + cache_operation(TEE_CACHEINVALIDATE, genkey->p.data, + genkey->p.length + genkey->q.length); + + ret = crypto_bignum_bin2bn(genkey->p.data, genkey->p.length, key->p); + if (ret != TEE_SUCCESS) + return ret; + + ret = crypto_bignum_bin2bn(genkey->q.data, genkey->q.length, key->q); + + if (ret == TEE_SUCCESS && genkey->format > RSA_PRIVATE_KEY_FORMAT_2) + ret = gen_keypair_get_f3(key, genkey); + + return ret; +} + +/* + * Generates a RSA keypair + * + * @key [out] Keypair + * @key_size Key size in bits + */ +static TEE_Result do_gen_keypair(struct rsa_keypair *key, size_t key_size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_rsa_keypair genkey = { }; + size_t size_d = 0; + size_t size_n = 0; + size_t size_d_gen = 0; + struct caam_jobctx jobctx = { }; + uint32_t *desc = 0; + uint32_t desclen = 0; + struct prime_data_rsa prime = { }; + + RSA_TRACE("Generate Keypair of %zu bits", key_size); + + genkey.format = CFG_NXP_CAAM_RSA_KEY_FORMAT; + + /* Allocate the job used to prepare the operation */ + desc = caam_calloc_desc(MAX_DESC_KEY_FINISH); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto exit_gen_keypair; + } + + /* First allocate primes p and q in one buffer */ + retstatus = caam_calloc_align_buf(&genkey.p, key_size / 8); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_gen_keypair; + } + + /* Prepare q */ + genkey.p.length /= 2; + genkey.q.data = genkey.p.data + genkey.p.length; + genkey.q.length = genkey.p.length; + genkey.q.paddr = genkey.p.paddr + genkey.p.length; + + /* Allocate Public exponent to a caam buffer */ + retstatus = caam_calloc_buf(&genkey.e, crypto_bignum_num_bytes(key->e)); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_gen_keypair; + } + + /* + * Allocate d and n in one buffer. + * Size of d is (key_size + 1) bits - Add a 32 bits word to + * retrieve the length of d generated by CAAM RSA Finalize Key + */ + size_d = sizeof(uint32_t) + key_size / 8 + 1; + size_n = key_size / 8; + + retstatus = caam_calloc_align_buf(&genkey.d, size_d + size_n); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_gen_keypair; + } + + genkey.d.length = size_d; + genkey.n.data = genkey.d.data + size_d; + genkey.n.length = size_n; + genkey.n.paddr = genkey.d.paddr + size_d; + + if (genkey.format > RSA_PRIVATE_KEY_FORMAT_2) { + /* Allocate dp, dq and qp in one buffer */ + retstatus = caam_calloc_align_buf(&genkey.dp, + ((key_size / 8) / 2) * 3); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_gen_keypair; + } + + genkey.dp.length /= 3; + /* Prepare dq and qp */ + genkey.dq.data = genkey.dp.data + genkey.dp.length; + genkey.dq.length = genkey.dp.length; + genkey.dq.paddr = genkey.dp.paddr + genkey.dp.length; + + genkey.qp.data = genkey.dq.data + genkey.dq.length; + genkey.qp.length = genkey.dq.length; + genkey.qp.paddr = genkey.dq.paddr + genkey.dq.length; + } + + crypto_bignum_bn2bin(key->e, genkey.e.data); + + prime.era = caam_era; + prime.key_size = key_size; + prime.e = &genkey.e; + prime.p = &genkey.p; + prime.q = &genkey.q; + + /* Generate prime p and q */ + retstatus = caam_prime_rsa_gen(&prime); + RSA_TRACE("Generate Prime P and Q returned 0x%" PRIx32, retstatus); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_gen_keypair; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + caam_desc_add_word(desc, 0); + caam_desc_add_word(desc, PDB_RSA_KEY_P_SIZE(genkey.p.length)); + caam_desc_add_word(desc, PDB_RSA_KEY_N_SIZE(genkey.n.length) | + PDB_RSA_KEY_E_SIZE(genkey.e.length)); + + caam_desc_add_ptr(desc, genkey.p.paddr); + caam_desc_add_ptr(desc, genkey.q.paddr); + caam_desc_add_ptr(desc, genkey.e.paddr); + caam_desc_add_ptr(desc, genkey.n.paddr); + caam_desc_add_ptr(desc, genkey.d.paddr + sizeof(uint32_t)); + caam_desc_add_ptr(desc, genkey.d.paddr); + + if (genkey.format > RSA_PRIVATE_KEY_FORMAT_2) { + caam_desc_add_ptr(desc, genkey.dp.paddr); + caam_desc_add_ptr(desc, genkey.dq.paddr); + caam_desc_add_ptr(desc, genkey.qp.paddr); + caam_desc_add_word(desc, RSA_FINAL_KEY(ALL)); + + cache_operation(TEE_CACHEFLUSH, genkey.dp.data, + genkey.dp.length + genkey.dq.length + + genkey.qp.length); + + } else { + caam_desc_add_word(desc, RSA_FINAL_KEY(N_D)); + } + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + jobctx.desc = desc; + RSA_DUMPDESC(desc); + + cache_operation(TEE_CACHECLEAN, genkey.e.data, genkey.e.length); + cache_operation(TEE_CACHEFLUSH, genkey.p.data, + genkey.p.length + genkey.q.length); + cache_operation(TEE_CACHEFLUSH, genkey.d.data, + genkey.d.length + genkey.n.length); + + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + cache_operation(TEE_CACHEINVALIDATE, genkey.d.data, + genkey.d.length + genkey.n.length); + + size_d_gen = caam_read_val32(genkey.d.data); + RSA_TRACE("D size %zu", size_d_gen); + RSA_DUMPBUF("N", genkey.n.data, genkey.n.length); + RSA_DUMPBUF("D", genkey.d.data + sizeof(uint32_t), size_d_gen); + + ret = crypto_bignum_bin2bn(genkey.n.data, genkey.n.length, + key->n); + if (ret != TEE_SUCCESS) + goto exit_gen_keypair; + + ret = crypto_bignum_bin2bn(genkey.d.data + sizeof(uint32_t), + size_d_gen, key->d); + if (ret != TEE_SUCCESS) + goto exit_gen_keypair; + + if (genkey.format > RSA_PRIVATE_KEY_FORMAT_1) + ret = gen_keypair_get_f2(key, &genkey); + } else { + RSA_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +exit_gen_keypair: + genkey.d.length += genkey.n.length; + genkey.n.data = NULL; + do_keypair_free(&genkey); + + caam_free_desc(&desc); + + return ret; +} + +/* + * RSA EME-OAEP Decoding operation + * Refer the chapter 7.1.2 Decryption operation of pkcs-1v2-1 specification + * + * @rsa_data [in/out] RSA Data to encode + */ +static TEE_Result do_oaep_decoding(struct drvcrypt_rsa_ed *rsa_data) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caambuf DB = { }; + struct caambuf lHash = { }; + struct caambuf seed = { }; + struct caambuf dbMask = { }; + struct caambuf maskedDB = { }; + struct caambuf maskedSeed = { }; + struct caambuf EM = { }; + size_t db_size = 0; + size_t b01_idx = 0; + size_t db_len = 0; + struct drvcrypt_rsa_mgf mgf_data = { }; + struct drvcrypt_rsa_ed dec_data = { }; + struct drvcrypt_mod_op mod_op = { }; + + RSA_TRACE("RSA OAEP Decoding"); + + /* + * First Decryption of the Cipher to a EM of modulus size + */ + retstatus = caam_calloc_align_buf(&EM, rsa_data->key.n_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_decrypt; + } + + memcpy(&dec_data, rsa_data, sizeof(dec_data)); + dec_data.message.data = EM.data; + dec_data.message.length = EM.length; + + ret = do_caam_decrypt(&dec_data, RSA_DECRYPT(NO)); + + RSA_DUMPBUF("EM", EM.data, EM.length); + + /* + * DB = lHash' || PS || 0x01 || M + * DB length = k - hLen - 1 + * + * PS is a 0's buffer of length h - mLen - 2hLen - 2 + * + * k is the key modulus length + * hLen is the Hash digest length + * mLen is the input RSA message length + */ + /* Calculate the DB size */ + db_size = rsa_data->key.n_size - rsa_data->digest_size - 1; + RSA_TRACE("DB is %zu bytes", db_size); + + /* Allocate the DB buffer */ + retstatus = caam_calloc_align_buf(&DB, db_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_decrypt; + } + + /* + * Step a + * Generate the lHash + */ + /* Allocate the lHash buffer */ + retstatus = caam_calloc_align_buf(&lHash, rsa_data->digest_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_decrypt; + } + + RSA_TRACE("Hash the RSA Label of %zu bytes", rsa_data->label.length); + ret = tee_hash_createdigest(rsa_data->hash_algo, rsa_data->label.data, + rsa_data->label.length, lHash.data, + lHash.length); + RSA_TRACE("Hash the RSA Label returned 0x%08" PRIx32, ret); + if (ret != TEE_SUCCESS) + goto exit_oaep_decrypt; + + RSA_DUMPBUF("lHash", lHash.data, lHash.length); + + /* Allocate the seed buffer */ + retstatus = caam_calloc_align_buf(&seed, rsa_data->digest_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_decrypt; + } + + /* Allocate the dbMask buffer */ + retstatus = caam_calloc_align_buf(&dbMask, db_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_decrypt; + } + + /* + * Step b + * Split the EM string + * EM = Y || maskedSeed || maskedDB + * + * Where: + * Y size = 1 byte + * maskedSeed size = hLen + * maskedDB size = k - hLen - 1 bytes + * + * k is the key modulus length + * hLen is the Hash digest length + * mLen is the input RSA message length + * + * Note Y should have been remove during the + */ + maskedSeed.data = &EM.data[1]; + maskedSeed.length = rsa_data->digest_size; + maskedSeed.paddr = EM.paddr + sizeof(uint8_t); + + maskedDB.data = &EM.data[1 + rsa_data->digest_size]; + maskedDB.length = dbMask.length; + maskedDB.paddr = EM.paddr + sizeof(uint8_t) + rsa_data->digest_size; + + /* + * Step c + * Generate a Mask of the maskedDB + * seedMask = MGF(maskedDB, k - hLen - 1) + * + * Note: Use same buffer for seed and seedMask + */ + mgf_data.hash_algo = rsa_data->hash_algo; + mgf_data.digest_size = rsa_data->digest_size; + mgf_data.seed.data = maskedDB.data; + mgf_data.seed.length = maskedDB.length; + mgf_data.mask.data = seed.data; + mgf_data.mask.length = seed.length; + + ret = rsa_data->mgf(&mgf_data); + if (ret != TEE_SUCCESS) + goto exit_oaep_decrypt; + + /* + * Step d + * seed = maskedSeed xor seedMask + * + * Note: Use same buffer for seed and seedMask + */ + mod_op.n.length = seed.length; + mod_op.a.data = maskedSeed.data; + mod_op.a.length = maskedSeed.length; + mod_op.b.data = seed.data; + mod_op.b.length = seed.length; + mod_op.result.data = seed.data; + mod_op.result.length = seed.length; + + retstatus = drvcrypt_xor_mod_n(&mod_op); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_decrypt; + } + + RSA_DUMPBUF("Seed", seed.data, seed.length); + + /* + * Step e + * Generate a Mask of the seed value + * dbMask = MGF(seed, k - hLen - 1) + */ + mgf_data.seed.data = seed.data; + mgf_data.seed.length = seed.length; + mgf_data.mask.data = dbMask.data; + mgf_data.mask.length = dbMask.length; + + ret = rsa_data->mgf(&mgf_data); + if (ret != TEE_SUCCESS) + goto exit_oaep_decrypt; + + /* + * Step f + * DB = maskedDB xor dbMask + */ + mod_op.n.length = DB.length; + mod_op.a.data = maskedDB.data; + mod_op.a.length = maskedDB.length; + mod_op.b.data = dbMask.data; + mod_op.b.length = dbMask.length; + mod_op.result.data = DB.data; + mod_op.result.length = DB.length; + + retstatus = drvcrypt_xor_mod_n(&mod_op); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_decrypt; + } + + RSA_DUMPBUF("DB", DB.data, DB.length); + + /* + * Step g + * Check the DB generated + * DB = lHash' || PS || 0x01 || M + * + * Error if: + * - lHash' != lHash (First step - Hash the Label) + * - byte 0x01 between PS and M is not present + */ + /* Check Hash values */ + if (memcmp(DB.data, lHash.data, lHash.length)) { + RSA_TRACE("Hash error"); + ret = TEE_ERROR_BAD_PARAMETERS; + goto exit_oaep_decrypt; + } + + /* Find the byte 0x01 separating PS and M */ + for (b01_idx = rsa_data->digest_size; + b01_idx < db_size && !DB.data[b01_idx]; b01_idx++) + ; + + if (b01_idx == db_size) { + RSA_TRACE("byte 0x01 not present"); + ret = TEE_ERROR_BAD_PARAMETERS; + goto exit_oaep_decrypt; + } + + db_len = DB.length - b01_idx - 1; + + if (rsa_data->message.length < db_len) { + rsa_data->message.length = db_len; + ret = TEE_ERROR_SHORT_BUFFER; + goto exit_oaep_decrypt; + } + + rsa_data->message.length = db_len; + memcpy(rsa_data->message.data, &DB.data[b01_idx + 1], + rsa_data->message.length); + + RSA_DUMPBUF("Message decrypted", rsa_data->message.data, + rsa_data->message.length); + ret = TEE_SUCCESS; + +exit_oaep_decrypt: + caam_free_buf(&EM); + caam_free_buf(&DB); + caam_free_buf(&seed); + caam_free_buf(&dbMask); + caam_free_buf(&lHash); + + return ret; +} + +/* + * RSA EME-OAEP Encoding operation + * Refer the chapter 7.1.1 Encryption operation of pkcs-1v2-1 specification + * + * @rsa_data [int/out] RSA Data to encode + */ +static TEE_Result do_oaep_encoding(struct drvcrypt_rsa_ed *rsa_data) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus; + struct caambuf DB = { }; + struct caambuf lHash = { }; + struct caambuf seed = { }; + struct caambuf dbMask = { }; + struct caambuf maskedDB = { }; + struct caambuf maskedSeed = { }; + struct caambuf EM = { }; + size_t db_size = 0; + size_t ps_size = 0; + struct drvcrypt_rsa_mgf mgf_data = { }; + struct drvcrypt_rsa_ed enc_data = { }; + struct drvcrypt_mod_op mod_op = { }; + + RSA_TRACE("RSA OAEP Encoding"); + + /* + * DB = lHash || PS || 0x01 || M + * DB length = k - hLen - 1 + * + * PS is a 0's buffer of length h - mLen - 2hLen - 2 + * + * k is the key modulus length + * hLen is the Hash digest length + * mLen is the input RSA message length + */ + /* Calculate the DB size */ + db_size = rsa_data->key.n_size - rsa_data->digest_size - 1; + RSA_TRACE("DB is %zu bytes", db_size); + + /* Allocate the DB buffer */ + retstatus = caam_calloc_align_buf(&DB, db_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_encrypt; + } + + /* + * Step a + * Generate the lHash + */ + lHash.length = rsa_data->digest_size; + lHash.data = DB.data; + + RSA_TRACE("Hash the RSA Label of %zu bytes", rsa_data->label.length); + ret = tee_hash_createdigest(rsa_data->hash_algo, rsa_data->label.data, + rsa_data->label.length, lHash.data, + lHash.length); + RSA_TRACE("Hash the RSA Label returned 0x%08" PRIx32, ret); + if (ret != TEE_SUCCESS) + goto exit_oaep_encrypt; + RSA_DUMPBUF("lHash", lHash.data, lHash.length); + + /* + * Step b + * Add PS 0's + * Note: DB is already filled with 0's at the allocation + */ + ps_size = rsa_data->key.n_size - rsa_data->message.length - + 2 * rsa_data->digest_size - 2; + RSA_TRACE("PS is %zu bytes", ps_size); + + /* + * Step c + * Set the value 0x01 after the lHash and the PS + * Concatenate result with input message + */ + DB.data[lHash.length + ps_size] = 0x01; + memcpy(&DB.data[lHash.length + ps_size + 1], rsa_data->message.data, + rsa_data->message.length); + + RSA_DUMPBUF("DB", DB.data, DB.length); + + /* + * Step d + * Generate a random seed of hLen + */ + /* Allocate the seed buffer */ + retstatus = caam_calloc_align_buf(&seed, rsa_data->digest_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_encrypt; + } + + /* Allocate the dbMask buffer */ + retstatus = caam_calloc_align_buf(&dbMask, db_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_encrypt; + } + + ret = crypto_rng_read(seed.data, seed.length); + RSA_TRACE("Get seed of %zu bytes (ret = 0x%08" PRIx32 ")", seed.length, + ret); + if (ret != TEE_SUCCESS) + goto exit_oaep_encrypt; + + RSA_DUMPBUF("Seed", seed.data, seed.length); + + /* + * Step e + * Generate a Mask of the seed value + * dbMask = MGF(seed, k - hLen - 1) + */ + mgf_data.hash_algo = rsa_data->hash_algo; + mgf_data.digest_size = rsa_data->digest_size; + mgf_data.seed.data = seed.data; + mgf_data.seed.length = seed.length; + mgf_data.mask.data = dbMask.data; + mgf_data.mask.length = dbMask.length; + + ret = rsa_data->mgf(&mgf_data); + if (ret != TEE_SUCCESS) + goto exit_oaep_encrypt; + + /* + * Step f + * maskedDB = DB xor dbMask + */ + retstatus = caam_calloc_align_buf(&EM, rsa_data->key.n_size); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_oaep_encrypt; + } + + maskedDB.data = &EM.data[1 + rsa_data->digest_size]; + maskedDB.length = dbMask.length; + maskedDB.paddr = EM.paddr + sizeof(uint8_t) + rsa_data->digest_size; + + mod_op.n.length = maskedDB.length; + mod_op.a.data = DB.data; + mod_op.a.length = DB.length; + mod_op.b.data = dbMask.data; + mod_op.b.length = dbMask.length; + mod_op.result.data = maskedDB.data; + mod_op.result.length = maskedDB.length; + + ret = drvcrypt_xor_mod_n(&mod_op); + if (ret != TEE_SUCCESS) + goto exit_oaep_encrypt; + + /* + * Step g + * Generate a Mask of the maskedDB + * seedMask = MGF(maskedDB, hLen) + * + * Note: Use same buffer for seedMask and maskedSeed + */ + maskedSeed.data = &EM.data[1]; + maskedSeed.length = rsa_data->digest_size; + maskedSeed.paddr = EM.paddr + sizeof(uint8_t); + + mgf_data.seed.data = maskedDB.data; + mgf_data.seed.length = maskedDB.length; + mgf_data.mask.data = maskedSeed.data; + mgf_data.mask.length = maskedSeed.length; + ret = rsa_data->mgf(&mgf_data); + if (ret != TEE_SUCCESS) + goto exit_oaep_encrypt; + + /* + * Step h + * maskedSeed = seed xor seedMask + */ + mod_op.n.length = maskedSeed.length; + mod_op.a.data = seed.data; + mod_op.a.length = seed.length; + mod_op.b.data = maskedSeed.data; + mod_op.b.length = maskedSeed.length; + mod_op.result.data = maskedSeed.data; + mod_op.result.length = maskedSeed.length; + + ret = drvcrypt_xor_mod_n(&mod_op); + if (ret != TEE_SUCCESS) + goto exit_oaep_encrypt; + + RSA_DUMPBUF("EM", EM.data, EM.length); + + /* + * Last Encryption of the EM of modulus size to Cipher + */ + memcpy(&enc_data, rsa_data, sizeof(enc_data)); + + enc_data.message.data = EM.data; + enc_data.message.length = EM.length; + + ret = do_caam_encrypt(&enc_data, RSA_ENCRYPT(NO)); + +exit_oaep_encrypt: + caam_free_buf(&DB); + caam_free_buf(&seed); + caam_free_buf(&dbMask); + caam_free_buf(&EM); + + return ret; +} + +/* + * CAAM RSA Encryption of the input message to a cipher + * + * @rsa_data [in/out] RSA Data to encrypt + * @operation CAAM RSA Encryption operation + */ +static TEE_Result do_caam_encrypt(struct drvcrypt_rsa_ed *rsa_data, + uint32_t operation) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_rsa_keypair key = { }; + struct caamdmaobj msg = { }; + struct caamdmaobj cipher = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + uint32_t pdb_sgt_flags = 0; + + RSA_TRACE("RSA Encrypt mode %d", rsa_data->rsa_id); + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(MAX_DESC_ENC); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto exit_encrypt; + } + + /* + * Convert TEE rsa key type to CAAM rsa key type + * Push key value to memory + */ + retstatus = do_keypub_conv(&key, rsa_data->key.key); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_encrypt; + } + + /* + * ReAllocate the cipher result buffer with a maximum size + * of the Key Modulus's size (N) if not cache aligned + */ + ret = caam_dmaobj_output_sgtbuf(&cipher, rsa_data->cipher.data, + rsa_data->cipher.length, key.n.length); + if (ret) + goto exit_encrypt; + + if (cipher.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_RSA_ENC_SGT_G; + + caam_dmaobj_cache_push(&cipher); + + /* Prepare the input message CAAM descriptor entry */ + ret = caam_dmaobj_input_sgtbuf(&msg, rsa_data->message.data, + rsa_data->message.length); + if (ret) + goto exit_encrypt; + + if (msg.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_RSA_ENC_SGT_F; + + caam_dmaobj_cache_push(&msg); + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PDB_RSA_ENC_E_SIZE(key.e.length) | + PDB_RSA_ENC_N_SIZE(key.n.length) | + pdb_sgt_flags); + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + caam_desc_add_ptr(desc, cipher.sgtbuf.paddr); + caam_desc_add_ptr(desc, key.n.paddr); + caam_desc_add_ptr(desc, key.e.paddr); + caam_desc_add_word(desc, PDB_RSA_ENC_F_SIZE(rsa_data->message.length)); + caam_desc_add_word(desc, operation); + + /* Set the descriptor Header with length */ + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + RSA_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + rsa_data->cipher.length = caam_dmaobj_copy_to_orig(&cipher); + + RSA_DUMPBUF("Output", rsa_data->cipher.data, + rsa_data->cipher.length); + ret = caam_status_to_tee_result(retstatus); + } else { + RSA_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +exit_encrypt: + caam_free_desc(&desc); + do_keypair_free(&key); + caam_dmaobj_free(&msg); + caam_dmaobj_free(&cipher); + + return ret; +} + +/* + * CAAM RSA Decryption of the input cipher to a message + * + * @rsa_data [in/out] RSA Data to decrypt + * @operation CAAM RSA Decryption operation + */ +static TEE_Result do_caam_decrypt(struct drvcrypt_rsa_ed *rsa_data, + uint32_t operation) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_rsa_keypair key = { }; + struct caamdmaobj cipher = { }; + struct caamdmaobj msg = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + uint32_t pdb_sgt_flags = 0; + struct caambuf size_msg = { }; + struct caambuf tmp = { }; + + RSA_TRACE("RSA Decrypt mode %d", rsa_data->rsa_id); + + /* + * Convert TEE rsa key type to CAAM rsa key type + * Push key value to memory + */ + retstatus = do_keypair_conv(&key, rsa_data->key.key); + if (retstatus != CAAM_NO_ERROR) { + RSA_TRACE("do_keypair_conv returned 0x%" PRIx32, retstatus); + ret = caam_status_to_tee_result(retstatus); + goto exit_decrypt; + } + + /* + * Allocate the temporary result buffer with a maximum size + * of the Key Modulus's size (N) + */ + ret = caam_dmaobj_output_sgtbuf(&msg, rsa_data->message.data, + rsa_data->message.length, key.n.length); + + if (ret) + goto exit_decrypt; + + if (msg.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_RSA_DEC_SGT_F; + + caam_dmaobj_cache_push(&msg); + + /* Allocate the returned computed size when PKCS V1.5 */ + if (operation == RSA_DECRYPT(PKCS_V1_5)) { + retstatus = caam_alloc_align_buf(&size_msg, 4); + if (retstatus != CAAM_NO_ERROR) + goto exit_decrypt; + + cache_operation(TEE_CACHEFLUSH, size_msg.data, size_msg.length); + } + + /* Prepare the input cipher CAAM descriptor entry */ + ret = caam_dmaobj_input_sgtbuf(&cipher, rsa_data->cipher.data, + rsa_data->cipher.length); + + if (cipher.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_RSA_DEC_SGT_G; + + caam_dmaobj_cache_push(&cipher); + + /* Allocate the job descriptor function of the Private key format */ + switch (key.format) { + case RSA_PRIVATE_KEY_FORMAT_1: + desc = caam_calloc_desc(MAX_DESC_DEC_1); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto exit_decrypt; + } + break; + + case RSA_PRIVATE_KEY_FORMAT_2: + case RSA_PRIVATE_KEY_FORMAT_3: + if (key.format == RSA_PRIVATE_KEY_FORMAT_2) + desc = caam_calloc_desc(MAX_DESC_DEC_2); + else + desc = caam_calloc_desc(MAX_DESC_DEC_3); + + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto exit_decrypt; + } + /* Allocate temporary buffers used by the CAAM */ + retstatus = + caam_alloc_align_buf(&tmp, key.p.length + key.q.length); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto exit_decrypt; + } + + cache_operation(TEE_CACHEFLUSH, tmp.data, tmp.length); + break; + + default: + ret = TEE_ERROR_GENERIC; + goto exit_decrypt; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + /* Build the descriptor function of the Private Key format */ + switch (key.format) { + case RSA_PRIVATE_KEY_FORMAT_1: + caam_desc_add_word(desc, + PDB_RSA_DEC_D_SIZE(key.d.length) | + PDB_RSA_DEC_N_SIZE(key.n.length) | + pdb_sgt_flags); + caam_desc_add_ptr(desc, cipher.sgtbuf.paddr); + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + caam_desc_add_ptr(desc, key.n.paddr); + caam_desc_add_ptr(desc, key.d.paddr); + + break; + + case RSA_PRIVATE_KEY_FORMAT_2: + caam_desc_add_word(desc, + PDB_RSA_DEC_D_SIZE(key.d.length) | + PDB_RSA_DEC_N_SIZE(key.n.length) | + pdb_sgt_flags); + caam_desc_add_ptr(desc, cipher.sgtbuf.paddr); + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + caam_desc_add_ptr(desc, key.d.paddr); + caam_desc_add_ptr(desc, key.p.paddr); + caam_desc_add_ptr(desc, key.q.paddr); + caam_desc_add_ptr(desc, tmp.paddr); + caam_desc_add_ptr(desc, tmp.paddr + key.p.length); + caam_desc_add_word(desc, + PDB_RSA_DEC_Q_SIZE(key.q.length) | + PDB_RSA_DEC_P_SIZE(key.p.length)); + break; + + case RSA_PRIVATE_KEY_FORMAT_3: + caam_desc_add_word(desc, PDB_RSA_DEC_N_SIZE(key.n.length) | + pdb_sgt_flags); + caam_desc_add_ptr(desc, cipher.sgtbuf.paddr); + caam_desc_add_ptr(desc, msg.sgtbuf.paddr); + caam_desc_add_ptr(desc, key.qp.paddr); + caam_desc_add_ptr(desc, key.p.paddr); + caam_desc_add_ptr(desc, key.q.paddr); + caam_desc_add_ptr(desc, key.dp.paddr); + caam_desc_add_ptr(desc, key.dq.paddr); + caam_desc_add_ptr(desc, tmp.paddr); + caam_desc_add_ptr(desc, tmp.paddr + key.p.length); + caam_desc_add_word(desc, + PDB_RSA_DEC_Q_SIZE(key.q.length) | + PDB_RSA_DEC_P_SIZE(key.p.length)); + break; + + default: + ret = TEE_ERROR_GENERIC; + goto exit_decrypt; + } + + /* Set the Decryption operation type */ + caam_desc_add_word(desc, operation | PROT_RSA_DEC_KEYFORM(key.format)); + + if (operation == RSA_DECRYPT(PKCS_V1_5)) { + /* Get the PPKCS1 v1.5 Message length generated */ + caam_desc_add_word(desc, + ST_NOIMM_OFF(CLASS_DECO, REG_MATH0, 4, 4)); + caam_desc_add_ptr(desc, size_msg.paddr); + /* Set the descriptor Header with length */ + desclen = caam_desc_get_len(desc); +#ifdef CFG_CAAM_64BIT + caam_desc_update_hdr(desc, + DESC_HEADER_IDX(desclen, desclen - 1 - 3)); +#else + caam_desc_update_hdr(desc, + DESC_HEADER_IDX(desclen, desclen - 1 - 2)); +#endif /* CFG_CAAM_64BIT */ + } else { + desclen = caam_desc_get_len(desc); + /* Set the descriptor Header with length */ + caam_desc_update_hdr(desc, + DESC_HEADER_IDX(desclen, desclen - 1)); + } + + RSA_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus != CAAM_NO_ERROR) { + RSA_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + goto exit_decrypt; + } + + if (operation == RSA_DECRYPT(NO) && + rsa_data->rsa_id == DRVCRYPT_RSA_NOPAD) { + rsa_data->message.length = caam_dmaobj_copy_ltrim_to_orig(&msg); + } else { + if (operation == RSA_DECRYPT(PKCS_V1_5)) { + /* PKCS 1 v1.5 */ + cache_operation(TEE_CACHEINVALIDATE, size_msg.data, + size_msg.length); + + /* Check if the original buffer size is large enough */ + if (msg.orig.length < caam_read_val32(size_msg.data)) { + rsa_data->message.length = + caam_read_val32(size_msg.data); + ret = TEE_ERROR_SHORT_BUFFER; + goto exit_decrypt; + } + + msg.orig.length = caam_read_val32(size_msg.data); + RSA_TRACE("New length %zu", msg.orig.length); + } + + rsa_data->message.length = caam_dmaobj_copy_to_orig(&msg); + } + + RSA_DUMPBUF("Output", rsa_data->message.data, rsa_data->message.length); + ret = TEE_SUCCESS; + +exit_decrypt: + caam_free_desc(&desc); + do_keypair_free(&key); + caam_free_buf(&size_msg); + caam_dmaobj_free(&msg); + caam_dmaobj_free(&cipher); + + caam_free_buf(&tmp); + + return ret; +} + +/* + * RSA Encryption + * + * @rsa_data [in/out] RSA Data to encrypt / Cipher resulting + */ +static TEE_Result do_encrypt(struct drvcrypt_rsa_ed *rsa_data) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + + switch (rsa_data->rsa_id) { + case DRVCRYPT_RSA_NOPAD: + case DRVCRYPT_RSASSA_PKCS_V1_5: + case DRVCRYPT_RSASSA_PSS: + ret = do_caam_encrypt(rsa_data, RSA_ENCRYPT(NO)); + break; + + case DRVCRYPT_RSA_PKCS_V1_5: + ret = do_caam_encrypt(rsa_data, RSA_ENCRYPT(PKCS_V1_5)); + break; + + case DRVCRYPT_RSA_OAEP: + ret = do_oaep_encoding(rsa_data); + break; + + default: + break; + } + + return ret; +} + +/* + * RSA Decryption + * + * @rsa_data [in/out] RSA Data to decrypt / Message resulting + */ +static TEE_Result do_decrypt(struct drvcrypt_rsa_ed *rsa_data) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + + switch (rsa_data->rsa_id) { + case DRVCRYPT_RSA_NOPAD: + case DRVCRYPT_RSASSA_PKCS_V1_5: + case DRVCRYPT_RSASSA_PSS: + ret = do_caam_decrypt(rsa_data, RSA_DECRYPT(NO)); + break; + + case DRVCRYPT_RSA_PKCS_V1_5: + ret = do_caam_decrypt(rsa_data, RSA_DECRYPT(PKCS_V1_5)); + break; + + case DRVCRYPT_RSA_OAEP: + ret = do_oaep_decoding(rsa_data); + break; + + default: + break; + } + + return ret; +} + +/* + * Registration of the RSA Driver + */ +static const struct drvcrypt_rsa driver_rsa = { + .alloc_keypair = do_allocate_keypair, + .alloc_publickey = do_allocate_publickey, + .free_publickey = do_free_publickey, + .free_keypair = do_free_keypair, + .gen_keypair = do_gen_keypair, + .encrypt = do_encrypt, + .decrypt = do_decrypt, + .optional.ssa_sign = NULL, + .optional.ssa_verify = NULL, +}; + +enum caam_status caam_rsa_init(struct caam_jrcfg *caam_jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + vaddr_t jr_base = caam_jrcfg->base + caam_jrcfg->offset; + + if (caam_hal_ctrl_pknum(jr_base)) { + caam_era = caam_hal_ctrl_era(jr_base); + RSA_TRACE("CAAM Era %d", caam_era); + + if (!drvcrypt_register_rsa(&driver_rsa)) + retstatus = CAAM_NO_ERROR; + } + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/acipher/local.h b/optee_os/core/drivers/crypto/caam/acipher/local.h new file mode 100644 index 0000000..60b6fbc --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/local.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * CAAM Cipher Local header. + */ +#ifndef __LOCAL_H__ +#define __LOCAL_H__ + +#include "caam_common.h" + +/* + * Prime generator structure for RSA + */ +struct prime_data_rsa { + uint8_t era; /* CAAM Era version */ + size_t key_size; /* Key size in bits */ + struct caambuf *e; /* Key exponent e */ + struct caambuf *p; /* Prime p */ + struct caambuf *q; /* Prime q (can be NULL of only p asked) */ +}; + +/* + * Generate prime numbers for RSA + * Algorithm based on the Chapter B.3.3 of the FIPS.184-6 specification + * + * @data [in/out] Prime generation data + */ +enum caam_status caam_prime_rsa_gen(struct prime_data_rsa *data); + +/* + * Prime generator structure for DSA + */ +struct prime_data_dsa { + struct caambuf *g; /* Generator g */ + struct caambuf *p; /* Prime p */ + struct caambuf *q; /* Prime q */ +}; + +/* + * Generate prime numbers for DSA + * Algorithm based on the Chapter A.1.2 of the FIPS.186-4 specification + * + * @data [in/out] Prime generation data + */ +enum caam_status caam_prime_dsa_gen(struct prime_data_dsa *data); + +#endif /* __LOCAL_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/acipher/sub.mk b/optee_os/core/drivers/crypto/caam/acipher/sub.mk new file mode 100644 index 0000000..ada3830 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/acipher/sub.mk @@ -0,0 +1,7 @@ +incdirs-y += ../include + +srcs-$(CFG_NXP_CAAM_RSA_DRV) += caam_rsa.c caam_prime_rsa.c +srcs-$(CFG_NXP_CAAM_DH_DRV) += caam_dh.c +srcs-$(CFG_NXP_CAAM_ECC_DRV) += caam_ecc.c +srcs-$(CFG_NXP_CAAM_DSA_DRV) += caam_dsa.c caam_prime_dsa.c +srcs-$(CFG_NXP_CAAM_MATH_DRV) += caam_math.c \ No newline at end of file diff --git a/optee_os/core/drivers/crypto/caam/blob/caam_blob.c b/optee_os/core/drivers/crypto/caam/blob/caam_blob.c new file mode 100644 index 0000000..5898e40 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/blob/caam_blob.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020 Pengutronix, Rouven Czerwinski + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MKVB_SIZE 32 + +static uint8_t stored_key[MKVB_SIZE]; +static bool mkvb_retrieved; + +enum caam_status caam_blob_mkvb_init(vaddr_t baseaddr) +{ + struct caam_jobctx jobctx = { }; + enum caam_status res = CAAM_NO_ERROR; + struct caambuf buf = { }; + uint32_t *desc = NULL; + + assert(!mkvb_retrieved); + + res = caam_calloc_align_buf(&buf, MKVB_SIZE); + if (res != CAAM_NO_ERROR) + goto out; + + desc = caam_calloc_desc(8); + if (!desc) { + res = CAAM_OUT_MEMORY; + goto out_buf; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, SEQ_OUT_PTR(32)); + caam_desc_add_ptr(desc, buf.paddr); + caam_desc_add_word(desc, BLOB_MSTR_KEY); + BLOB_DUMPDESC(desc); + + cache_operation(TEE_CACHEFLUSH, buf.data, buf.length); + + jobctx.desc = desc; + res = caam_jr_enqueue(&jobctx, NULL); + + if (res != CAAM_NO_ERROR) { + BLOB_TRACE("JR return code: %#"PRIx32, res); + BLOB_TRACE("MKVB failed: Job status %#"PRIx32, jobctx.status); + } else { + cache_operation(TEE_CACHEINVALIDATE, buf.data, MKVB_SIZE); + BLOB_DUMPBUF("MKVB", buf.data, buf.length); + memcpy(&stored_key, buf.data, buf.length); + mkvb_retrieved = true; + } + +out_buf: + caam_free_desc(&desc); + caam_free_buf(&buf); +out: + caam_hal_ctrl_inc_priblob(baseaddr); + + return res; +} + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + COMPILE_TIME_ASSERT(sizeof(hwkey->data) <= sizeof(stored_key)); + + if (!mkvb_retrieved) + return TEE_ERROR_SECURITY; + + memcpy(&hwkey->data, &stored_key, sizeof(hwkey->data)); + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/crypto/caam/blob/caam_dek.c b/optee_os/core/drivers/crypto/caam/blob/caam_dek.c new file mode 100644 index 0000000..cff61a7 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/blob/caam_dek.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019-2021, 2023 NXP + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_PHYS_64BIT +#define BLOB_OPERATE_DESC_ENTRIES 12 +#else +#define BLOB_OPERATE_DESC_ENTRIES 10 +#endif + +/* Secure Memory Access Permission allowed */ +#define SM_GRP_BLOB BIT32(3) /* Export/Import Secure Memory blobs allowed */ + +/* Secure Memory Page(s)/Partition definition for DEK Blob generation */ +static const struct caam_sm_page_desc dek_sm_page = { + .partition = 1, + .page = 3, + .page_count = 1, +}; + +TEE_Result caam_dek_generate(const uint8_t *payload, size_t payload_size, + uint8_t *dek, size_t dek_size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_sm_page_addr dek_sm_addr = { }; + struct caamdmaobj resblob = { }; + struct caam_jobctx jobctx = { }; + uint32_t key_modifier[2] = { }; + uint32_t *desc = NULL; + unsigned int opflags = 0; + + assert(payload && dek); + assert(payload_size && dek_size); + + /* Re-allocate output buffer if alignment needed */ + ret = caam_dmaobj_output_sgtbuf(&resblob, dek, dek_size, dek_size); + if (ret) + return ret; + + /* Allocate page(s) in one Secure Memory partition */ + ret = caam_sm_alloc(&dek_sm_page, &dek_sm_addr); + if (ret != CAAM_NO_ERROR) { + BLOB_TRACE("Secure memory allocation error 0x%" PRIx32, ret); + goto out; + } + + /* Copy input data to encapsulate in Secure Memory allocated */ + memcpy((void *)dek_sm_addr.vaddr, payload, payload_size); + + /* + * Set the partition access rights for the group #1 to be + * a blob export/import + */ + caam_sm_set_access_perm(&dek_sm_page, SM_GRP_BLOB, 0); + + /* + * Create the key modifier: + * 31 16 8 0 + * --------------------------------------------------- + * | Length of the payload | AES - 0x55 | CCM - 0x66 | + * --------------------------------------------------- + */ + key_modifier[0] = SHIFT_U32(payload_size, 16) | SHIFT_U32(0x55, 8) | + SHIFT_U32(0x66, 0); + key_modifier[1] = 0; + + /* Allocate the descriptor */ + desc = caam_calloc_desc(BLOB_OPERATE_DESC_ENTRIES); + if (!desc) { + BLOB_TRACE("CAAM Context Descriptor Allocation error"); + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, LD_IMM_OFF(CLASS_2, REG_KEY, 8, 12)); + caam_desc_add_word(desc, key_modifier[0]); + caam_desc_add_word(desc, key_modifier[1]); + caam_desc_add_word(desc, SEQ_IN_PTR(payload_size)); + caam_desc_add_ptr(desc, dek_sm_addr.paddr); + caam_desc_seq_out(desc, &resblob); + caam_desc_add_word(desc, BLOB_ENCAPS | PROT_BLOB_SEC_MEM | opflags); + + BLOB_DUMPDESC(desc); + + cache_operation(TEE_CACHECLEAN, (void *)payload, payload_size); + caam_dmaobj_cache_push(&resblob); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus) { + BLOB_TRACE("CAAM Status 0x%08" PRIx32 "", jobctx.status); + goto out; + } + + caam_dmaobj_copy_to_orig(&resblob); + + BLOB_TRACE("Done CAAM BLOB from Secure Memory encaps"); + BLOB_DUMPBUF("Blob Output", resblob.orig.data, resblob.orig.length); +out: + caam_sm_free(&dek_sm_page); + caam_free_desc(&desc); + caam_dmaobj_free(&resblob); + + return caam_status_to_tee_result(retstatus); +} diff --git a/optee_os/core/drivers/crypto/caam/blob/sub.mk b/optee_os/core/drivers/crypto/caam/blob/sub.mk new file mode 100644 index 0000000..15e445c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/blob/sub.mk @@ -0,0 +1,4 @@ +srcs-$(CFG_NXP_CAAM_BLOB_DRV) += caam_blob.c +srcs-$(CFG_NXP_CAAM_DEK_DRV) += caam_dek.c + +incdirs-y += ../include diff --git a/optee_os/core/drivers/crypto/caam/caam_ctrl.c b/optee_os/core/drivers/crypto/caam/caam_ctrl.c new file mode 100644 index 0000000..4dc07f6 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/caam_ctrl.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2021 NXP + * + * Brief CAAM Global Controller. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * If the CAAM DMA only supports 32 bits physical addresses, OPTEE must + * be located within the 32 bits address space. + */ +#ifndef CFG_CAAM_64BIT +static_assert((CFG_TZDRAM_START + CFG_TZDRAM_SIZE) < UINT32_MAX); +#endif + +/* Crypto driver initialization */ +static TEE_Result crypto_driver_init(void) +{ + TEE_Result retresult = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jrcfg jrcfg = {}; + + /* Enable the CAAM Clock */ + caam_hal_clk_enable(true); + + /* Set OTP as master key if the platform is closed */ + if (snvs_is_device_closed()) { + retresult = imx_snvs_set_master_otpmk(); + if (retresult && retresult != TEE_ERROR_NOT_IMPLEMENTED) + goto exit_init; + } + + retstatus = caam_hal_cfg_get_conf(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_NOT_SUPPORTED; + goto exit_init; + } + + /* Initialize the CAAM Controller */ + caam_hal_ctrl_init(jrcfg.base); + + /* Initialize the Job Ring to be used */ + retstatus = caam_jr_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the RNG Module */ + retstatus = caam_rng_init(jrcfg.base); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the Hash Module */ + retstatus = caam_hash_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the MATH Module */ + retstatus = caam_math_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the RSA Module */ + retstatus = caam_rsa_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the Cipher Module */ + retstatus = caam_cipher_init(jrcfg.base); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the HMAC Module */ + retstatus = caam_hmac_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the BLOB Module */ + retstatus = caam_blob_mkvb_init(jrcfg.base); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the CMAC Module */ + retstatus = caam_cmac_init(jrcfg.base); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the ECC Module */ + retstatus = caam_ecc_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the DH Module */ + retstatus = caam_dh_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the DSA Module */ + retstatus = caam_dsa_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the Manufacturing Protection Module */ + retstatus = caam_mp_init(jrcfg.base); + if (retstatus != CAAM_NO_ERROR && retstatus != CAAM_NOT_SUPPORTED) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Initialize the secure memory */ + retstatus = caam_sm_init(&jrcfg); + if (retstatus != CAAM_NO_ERROR) { + retresult = TEE_ERROR_GENERIC; + goto exit_init; + } + + /* Everything is OK, register the Power Management handler */ + caam_pwr_init(); + + /* + * Configure Job Rings to NS World + * If the Driver Crypto is not used CFG_NXP_CAAM_RUNTIME_JR is not + * enable, hence relax the JR used for the CAAM configuration to + * the Non-Secure + */ + if (jrcfg.base) + caam_hal_cfg_setup_nsjobring(&jrcfg); + + retresult = TEE_SUCCESS; +exit_init: + if (retresult != TEE_SUCCESS) { + EMSG("CAAM Driver initialization (0x%" PRIx32 ")", retresult); + panic(); + } + + return retresult; +} + +early_init(crypto_driver_init); + +/* Crypto driver late initialization to complete on-going CAAM operations */ +static TEE_Result init_caam_late(void) +{ + enum caam_status ret = CAAM_BUSY; + + ret = caam_jr_complete(); + + if (ret != CAAM_NO_ERROR) { + EMSG("CAAM initialization failed"); + panic(); + } + + return TEE_SUCCESS; +} + +early_init_late(init_caam_late); diff --git a/optee_os/core/drivers/crypto/caam/caam_desc.c b/optee_os/core/drivers/crypto/caam/caam_desc.c new file mode 100644 index 0000000..361e1aa --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/caam_desc.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019, 2021 NXP + * + * Brief Descriptor construction functions. + */ +#include +#include +#include +#include + +struct ptr_addr { +#ifdef CFG_CAAM_BIG_ENDIAN + uint32_t high; + uint32_t low; +#else + uint32_t low; + uint32_t high; +#endif /* CFG_CAAM_BIG_ENDIAN */ +}; + +uint32_t caam_desc_get_len(uint32_t *desc) +{ + return GET_JD_DESCLEN(caam_read_val32((void *)desc)); +} + +void caam_desc_init(uint32_t *desc) +{ + *desc = 0; +} + +void caam_desc_update_hdr(uint32_t *desc, uint32_t word) +{ + /* Update first word of desc */ + caam_write_val32((void *)desc, word); +} + +void caam_desc_add_word(uint32_t *desc, uint32_t word) +{ + uint32_t len = caam_desc_get_len(desc); + uint32_t *last = desc + len; + + /* Add Word at Last */ + caam_write_val32((void *)last, word); + + /* Increase the length */ + caam_write_val32((void *)desc, caam_read_val32((void *)desc) + 1); +} + +void caam_desc_add_ptr(uint32_t *desc, paddr_t ptr) +{ + uint32_t len = caam_desc_get_len(desc); + uint32_t *last = desc + len; + uint32_t inc = 1; + + /* Add Word at Last */ +#ifdef CFG_CAAM_64BIT + struct ptr_addr *ptr_addr = (struct ptr_addr *)(uintptr_t)last; + +#ifdef CFG_ARM64_core + caam_write_val32(&ptr_addr->high, ptr >> 32); +#else + caam_write_val32(&ptr_addr->high, 0); +#endif /* CFG_ARM64_core */ + caam_write_val32(&ptr_addr->low, ptr); + inc++; +#else + caam_write_val32((void *)last, ptr); +#endif /* CFG_CAAM_64BIT */ + + /* Increase the length */ + caam_write_val32((void *)desc, caam_read_val32((void *)desc) + inc); +} + +#ifdef CFG_CAAM_64BIT +void caam_desc_push(struct caam_inring_entry *in_entry, paddr_t paddr) +{ +#ifdef CFG_CAAM_BIG_ENDIAN + put_be64(&in_entry->desc, paddr); +#else + put_le64(&in_entry->desc, paddr); +#endif /* CFG_CAAM_BIG_ENDIAN */ +} + +paddr_t caam_desc_pop(struct caam_outring_entry *out_entry) +{ + const uintptr_t v_desc = (uintptr_t)&out_entry->desc; + const uint32_t *a32 = (const uint32_t *)v_desc; + +#ifdef CFG_CAAM_BIG_ENDIAN + return SHIFT_U64(get_be32(&a32[0]), 32) | get_be32(&a32[1]); +#else + return SHIFT_U64(a32[1], 32) | a32[0]; +#endif /* CFG_CAAM_BIG_ENDIAN */ +} +#else /* CFG_CAAM_64BIT */ +void caam_desc_push(struct caam_inring_entry *in_entry, paddr_t paddr) +{ + caam_write_val32(&in_entry->desc, paddr); +} + +paddr_t caam_desc_pop(struct caam_outring_entry *out_entry) +{ + return caam_read_val32(&out_entry->desc); +} +#endif /* CFG_CAAM_64BIT */ + +uint32_t caam_read_jobstatus(struct caam_outring_entry *out) +{ + return caam_read_val32(&out->status); +} + +void caam_desc_add_dmaobj(uint32_t *desc, struct caamdmaobj *data, + unsigned int pre_op) +{ + uint32_t operation = pre_op; + size_t op_length = 0; + uint32_t op_ext_length = 0; + + if (data->sgtbuf.sgt_type) + operation |= CMD_SGT; + + /* Check the operation length to set extension length or not */ + switch (GET_CMD_TYPE(pre_op)) { + case CMD_FIFO_LOAD_TYPE: + op_length = FIFO_LOAD_LENGTH(data->sgtbuf.length); + op_ext_length = FIFO_LOAD_EXT; + break; + + case CMD_STORE_TYPE: + /* Note: there is extension length for the STORE command */ + op_length = STORE_LENGTH(data->sgtbuf.length); + break; + + case CMD_FIFO_STORE_TYPE: + op_length = FIFO_STORE_LENGTH(data->sgtbuf.length); + op_ext_length = FIFO_STORE_EXT; + break; + + case CMD_KEY_TYPE: + /* Note: there is extension length for the KEY command */ + op_length = KEY_LENGTH(data->sgtbuf.length); + break; + + case CMD_SEQ_OUT_TYPE: + op_length = SEQ_LENGTH(data->sgtbuf.length); + op_ext_length = SEQ_EXT; + break; + + default: + break; + } + + if (op_length == data->sgtbuf.length) + operation |= op_length; + else + operation |= op_ext_length; + + caam_desc_add_word(desc, operation); + caam_desc_add_ptr(desc, data->sgtbuf.paddr); + + if (op_length != data->sgtbuf.length) + caam_desc_add_word(desc, data->sgtbuf.length); +} diff --git a/optee_os/core/drivers/crypto/caam/caam_jr.c b/optee_os/core/drivers/crypto/caam/caam_jr.c new file mode 100644 index 0000000..eeb64f0 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/caam_jr.c @@ -0,0 +1,684 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Job Rings manager. + * Implementation of functions to enqueue/dequeue CAAM Job Descriptor + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Job Free define + */ +#define JR_JOB_FREE 0 + +/* + * Caller information context object + */ +struct caller_info { + struct caam_jobctx *jobctx; /* Caller job context object */ + uint32_t job_id; /* Current Job ID */ + paddr_t pdesc; /* Physical address of the descriptor */ +}; + +/* + * Job Ring module private data + */ +struct jr_privdata { + vaddr_t baseaddr; /* Job Ring base address */ + + vaddr_t ctrladdr; /* CAAM virtual base address */ + paddr_t jroffset; /* Job Ring address offset */ + uint64_t paddr_inrings; /* CAAM physical addr of input queue */ + uint64_t paddr_outrings; /* CAAM physical addr of output queue */ + + uint8_t nb_jobs; /* Number of Job ring entries managed */ + + /* Input Job Ring Variables */ + struct caam_inring_entry *inrings; /* Input JR HW queue */ + unsigned int inlock; /* Input JR spin lock */ + uint16_t inwrite_index; /* SW Index - next JR entry free */ + + /* Output Job Ring Variables */ + struct caam_outring_entry *outrings; /* Output JR HW queue */ + unsigned int outlock; /* Output JR spin lock */ + uint16_t outread_index; /* SW Index - next JR output done */ + + /* Caller Information Variables */ + struct caller_info *callers; /* Job Ring Caller information */ + unsigned int callers_lock; /* Job Ring Caller spin lock */ + + struct itr_handler it_handler; /* Interrupt handler */ +}; + +/* + * Job Ring module private data reference + */ +static struct jr_privdata *jr_privdata; + +/* + * Free module resources + * + * @jr_priv Reference to the module private data + */ +static void do_jr_free(struct jr_privdata *jr_priv) +{ + if (jr_priv) { + caam_free(jr_priv->inrings); + caam_free(jr_priv->outrings); + caam_free(jr_priv->callers); + caam_free(jr_priv); + } +} + +/* + * Allocate module resources + * + * @privdata [out] Allocated Job Ring private data + * @nb_jobs Number of jobs to manage in the queue + */ +static enum caam_status do_jr_alloc(struct jr_privdata **privdata, + uint8_t nb_jobs) +{ + enum caam_status retstatus = CAAM_OUT_MEMORY; + struct jr_privdata *jr_priv = NULL; + + /* Allocate the Job Ring private data */ + jr_priv = caam_calloc(sizeof(*jr_priv)); + + if (!jr_priv) { + JR_TRACE("Private Data allocation error"); + goto end_alloc; + } + + /* Setup the number of jobs */ + jr_priv->nb_jobs = nb_jobs; + + /* Allocate the input and output job ring queues */ + jr_priv->inrings = + caam_calloc_align(nb_jobs * sizeof(struct caam_inring_entry)); + jr_priv->outrings = + caam_calloc_align(nb_jobs * sizeof(struct caam_outring_entry)); + + /* Allocate the callers information */ + jr_priv->callers = caam_calloc(nb_jobs * sizeof(struct caller_info)); + + if (!jr_priv->inrings || !jr_priv->outrings || !jr_priv->callers) { + JR_TRACE("JR resources allocation error"); + goto end_alloc; + } + + /* Initialize the spin locks */ + jr_priv->inlock = SPINLOCK_UNLOCK; + jr_priv->outlock = SPINLOCK_UNLOCK; + jr_priv->callers_lock = SPINLOCK_UNLOCK; + + /* Initialize the queue counter */ + jr_priv->inwrite_index = 0; + jr_priv->outread_index = 0; + + /* + * Ensure that allocated queue initialization is pushed to the physical + * memory + */ + cache_operation(TEE_CACHEFLUSH, jr_priv->inrings, + nb_jobs * sizeof(struct caam_inring_entry)); + cache_operation(TEE_CACHEFLUSH, jr_priv->outrings, + nb_jobs * sizeof(struct caam_outring_entry)); + + retstatus = CAAM_NO_ERROR; +end_alloc: + if (retstatus != CAAM_NO_ERROR) + do_jr_free(jr_priv); + else + *privdata = jr_priv; + + return retstatus; +} + +/* + * Job Ring Interrupt handler + * + * @handler Interrupt Handler structure + */ +static enum itr_return caam_jr_irqhandler(struct itr_handler *handler) +{ + JR_TRACE("Disable the interrupt"); + itr_disable(handler->it); + + /* Send a signal to exit WFE loop */ + sev(); + + return ITRR_HANDLED; +} + +/* + * Returns all jobs completed depending on the input @wait_job_ids mask. + * + * Dequeues all Jobs completed. Call the job context callback + * function. Function returns the bit mask of the expected completed job + * (@wait_job_ids parameter) + * + * @wait_job_ids Expected Jobs to be complete + */ +static uint32_t do_jr_dequeue(uint32_t wait_job_ids) +{ + uint32_t ret_job_id = 0; + struct caller_info *caller = NULL; + struct caam_outring_entry *jr_out = NULL; + struct caam_jobctx *jobctx = NULL; + uint32_t exceptions = 0; + bool found = false; + uint16_t idx_jr = 0; + uint32_t nb_jobs_done = 0; + size_t nb_jobs_inv = 0; + + exceptions = cpu_spin_lock_xsave(&jr_privdata->outlock); + + nb_jobs_done = caam_hal_jr_get_nbjob_done(jr_privdata->baseaddr); + + if (nb_jobs_done == 0) { + cpu_spin_unlock_xrestore(&jr_privdata->outlock, exceptions); + return ret_job_id; + } + + /* Ensure that output ring descriptor entries are not in cache */ + if ((jr_privdata->outread_index + nb_jobs_done) > + jr_privdata->nb_jobs) { + /* + * Invalidate the whole circular job buffer because some + * completed job rings are at the beginning of the buffer + */ + jr_out = jr_privdata->outrings; + nb_jobs_inv = jr_privdata->nb_jobs; + } else { + /* Invalidate only the completed job */ + jr_out = &jr_privdata->outrings[jr_privdata->outread_index]; + nb_jobs_inv = nb_jobs_done; + } + + cache_operation(TEE_CACHEINVALIDATE, jr_out, + sizeof(struct caam_outring_entry) * nb_jobs_inv); + + for (; nb_jobs_done; nb_jobs_done--) { + jr_out = &jr_privdata->outrings[jr_privdata->outread_index]; + + /* + * Lock the caller information array because enqueue is + * also touching it + */ + cpu_spin_lock(&jr_privdata->callers_lock); + for (idx_jr = 0, found = false; idx_jr < jr_privdata->nb_jobs; + idx_jr++) { + /* + * Search for the caller information corresponding to + * the completed JR. + * Don't use the outread_index or inwrite_index because + * completion can be out of order compared to input + * buffer + */ + caller = &jr_privdata->callers[idx_jr]; + if (caam_desc_pop(jr_out) == caller->pdesc) { + jobctx = caller->jobctx; + jobctx->status = caam_read_jobstatus(jr_out); + + /* Update return Job IDs mask */ + if (caller->job_id & wait_job_ids) + ret_job_id |= caller->job_id; + + JR_TRACE("JR id=%" PRId32 + ", context @0x%08" PRIxVA, + caller->job_id, (vaddr_t)jobctx); + /* Clear the Entry Descriptor DMA */ + caller->pdesc = 0; + caller->jobctx = NULL; + caller->job_id = JR_JOB_FREE; + found = true; + JR_TRACE("Free space #%" PRId16 + " in the callers array", + idx_jr); + break; + } + } + cpu_spin_unlock(&jr_privdata->callers_lock); + + /* + * Remove the JR from the output list even if no + * JR caller found + */ + caam_hal_jr_del_job(jr_privdata->baseaddr); + + /* + * Increment index to next JR output entry taking care that + * it is a circular buffer of nb_jobs size. + */ + jr_privdata->outread_index++; + jr_privdata->outread_index %= jr_privdata->nb_jobs; + + if (found && jobctx->callback) { + /* Finally, execute user's callback */ + jobctx->callback(jobctx); + } + } + + cpu_spin_unlock_xrestore(&jr_privdata->outlock, exceptions); + + return ret_job_id; +} + +/* + * Enqueues a new job in the Job Ring input queue. Keep the caller's + * job context in private array. + * + * @jobctx Caller's job context + * @job_id [out] Job ID enqueued + */ +static enum caam_status do_jr_enqueue(struct caam_jobctx *jobctx, + uint32_t *job_id) +{ + enum caam_status retstatus = CAAM_BUSY; + struct caam_inring_entry *cur_inrings = NULL; + struct caller_info *caller = NULL; + uint32_t exceptions = 0; + uint32_t job_mask = 0; + uint8_t idx_jr = 0; + bool found = false; + + exceptions = cpu_spin_lock_xsave(&jr_privdata->inlock); + + caam_hal_clk_enable(true); + + /* + * Stay locked until a job is available + * Check if there is an available JR index in the HW + */ + while (caam_hal_jr_read_nbslot_available(jr_privdata->baseaddr) == 0) { + /* + * WFE will return thanks to a SEV generated by the + * interrupt handler or by a spin_unlock + */ + wfe(); + }; + + /* + * There is a space free in the input ring but it doesn't mean + * that the job pushed is completed. + * Completion is out of order. Look for a free space in the + * caller data to push them and get a job ID for the completion + * + * Lock the caller information array because dequeue is + * also touching it + */ + cpu_spin_lock(&jr_privdata->callers_lock); + for (idx_jr = 0; idx_jr < jr_privdata->nb_jobs; idx_jr++) { + if (jr_privdata->callers[idx_jr].job_id == JR_JOB_FREE) { + JR_TRACE("Found a space #%" PRId8 + " free in the callers array", + idx_jr); + job_mask = 1 << idx_jr; + + /* Store the caller information for the JR completion */ + caller = &jr_privdata->callers[idx_jr]; + caller->job_id = job_mask; + caller->jobctx = jobctx; + caller->pdesc = virt_to_phys((void *)jobctx->desc); + + found = true; + break; + } + } + cpu_spin_unlock(&jr_privdata->callers_lock); + + if (!found) { + JR_TRACE("Error didn't find a free space in the callers array"); + goto end_enqueue; + } + + JR_TRACE("Push id=%" PRId16 ", job (0x%08" PRIx32 + ") context @0x%08" PRIxVA, + jr_privdata->inwrite_index, job_mask, (vaddr_t)jobctx); + + cur_inrings = &jr_privdata->inrings[jr_privdata->inwrite_index]; + + /* Push the descriptor into the JR HW list */ + caam_desc_push(cur_inrings, caller->pdesc); + + /* Ensure that physical memory is up to date */ + cache_operation(TEE_CACHECLEAN, cur_inrings, + sizeof(struct caam_inring_entry)); + + /* + * Increment index to next JR input entry taking care that + * it is a circular buffer of nb_jobs size. + */ + jr_privdata->inwrite_index++; + jr_privdata->inwrite_index %= jr_privdata->nb_jobs; + + /* Ensure that input descriptor is pushed in physical memory */ + cache_operation(TEE_CACHECLEAN, jobctx->desc, + DESC_SZBYTES(caam_desc_get_len(jobctx->desc))); + + /* Inform HW that a new JR is available */ + caam_hal_jr_add_newjob(jr_privdata->baseaddr); + + *job_id = job_mask; + retstatus = CAAM_NO_ERROR; + +end_enqueue: + cpu_spin_unlock_xrestore(&jr_privdata->inlock, exceptions); + + return retstatus; +} + +/* + * Synchronous job completion callback + * + * @jobctx Job context + */ +static void job_done(struct caam_jobctx *jobctx) +{ + jobctx->completion = true; +} + +void caam_jr_cancel(uint32_t job_id) +{ + unsigned int idx = 0; + + cpu_spin_lock(&jr_privdata->callers_lock); + + JR_TRACE("Job cancel 0x%" PRIx32, job_id); + for (idx = 0; idx < jr_privdata->nb_jobs; idx++) { + /* + * Search for the caller information corresponding to + * the job_id mask. + */ + if (jr_privdata->callers[idx].job_id == job_id) { + /* Clear the Entry Descriptor */ + jr_privdata->callers[idx].pdesc = 0; + jr_privdata->callers[idx].jobctx = NULL; + jr_privdata->callers[idx].job_id = JR_JOB_FREE; + return; + } + } + + cpu_spin_unlock(&jr_privdata->callers_lock); +} + +enum caam_status caam_jr_dequeue(uint32_t job_ids, unsigned int timeout_ms) +{ + uint32_t job_complete = 0; + uint32_t nb_loop = 0; + bool infinite = false; + bool it_active = false; + + if (timeout_ms == UINT_MAX) + infinite = true; + else + nb_loop = timeout_ms * 100; + + do { + /* Call the do_jr_dequeue function to dequeue the jobs */ + job_complete = do_jr_dequeue(job_ids); + + /* Check if new job has been submitted and acknowledge it */ + it_active = caam_hal_jr_check_ack_itr(jr_privdata->baseaddr); + + if (job_complete & job_ids) + return CAAM_NO_ERROR; + + /* Check if JR interrupt otherwise wait a bit */ + if (!it_active) + caam_udelay(10); + } while (infinite || (nb_loop--)); + + return CAAM_TIMEOUT; +} + +enum caam_status caam_jr_enqueue(struct caam_jobctx *jobctx, uint32_t *job_id) +{ + enum caam_status retstatus = CAAM_FAILURE; + __maybe_unused int timeout = 10; /* Nb loops to pool job completion */ + + if (!jobctx) + return CAAM_BAD_PARAM; + + JR_DUMPDESC(jobctx->desc); + + if (!jobctx->callback && job_id) { + JR_TRACE("Job Callback not defined whereas asynchronous"); + return CAAM_BAD_PARAM; + } + + if (jobctx->callback && !job_id) { + JR_TRACE("Job Id not defined whereas asynchronous"); + return CAAM_BAD_PARAM; + } + + jobctx->completion = false; + jobctx->status = 0; + + /* + * If parameter job_id is NULL, the job is synchronous, hence use + * the local job_done callback function + */ + if (!jobctx->callback && !job_id) { + jobctx->callback = job_done; + jobctx->context = jobctx; + } + + retstatus = do_jr_enqueue(jobctx, &jobctx->id); + + if (retstatus != CAAM_NO_ERROR) { + JR_TRACE("enqueue job error 0x%08x", retstatus); + return retstatus; + } + + /* + * If parameter job_id is defined, the job is asynchronous, so + * returns with setting the job_id value + */ + if (job_id) { + *job_id = jobctx->id; + return CAAM_PENDING; + } + +#ifdef TIMEOUT_COMPLETION + /* + * Job is synchronous wait until job completion or timeout + */ + while (!jobctx->completion && timeout--) + caam_jr_dequeue(jobctx->id, 100); + + if (timeout <= 0) { + /* Job timeout, cancel it and return in error */ + caam_jr_cancel(jobctx->id); + retstatus = CAAM_TIMEOUT; + } else { + if (JRSTA_SRC_GET(jobctx->status) != JRSTA_SRC(NONE)) + retstatus = CAAM_JOB_STATUS; + else + retstatus = CAAM_NO_ERROR; + } +#else + /* + * Job is synchronous wait until job complete + * Don't use a timeout because there is no HW timer and + * so the timeout is not precise + */ + while (!jobctx->completion) + caam_jr_dequeue(jobctx->id, 100); + + if (JRSTA_SRC_GET(jobctx->status) != JRSTA_SRC(NONE)) + retstatus = CAAM_JOB_STATUS; + else + retstatus = CAAM_NO_ERROR; +#endif + + /* Erase local callback function */ + jobctx->callback = NULL; + + return retstatus; +} + +enum caam_status caam_jr_init(struct caam_jrcfg *jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + + JR_TRACE("Initialization"); + + /* Allocate the Job Ring resources */ + retstatus = do_jr_alloc(&jr_privdata, jrcfg->nb_jobs); + if (retstatus != CAAM_NO_ERROR) + goto end_init; + + jr_privdata->ctrladdr = jrcfg->base; + jr_privdata->jroffset = jrcfg->offset; + + retstatus = + caam_hal_jr_setowner(jrcfg->base, jrcfg->offset, JROWN_ARM_S); + JR_TRACE("JR setowner returned 0x%x", retstatus); + + if (retstatus != CAAM_NO_ERROR) + goto end_init; + + jr_privdata->baseaddr = jrcfg->base + jrcfg->offset; + retstatus = caam_hal_jr_reset(jr_privdata->baseaddr); + if (retstatus != CAAM_NO_ERROR) + goto end_init; + + /* + * Get the physical address of the Input/Output queue + * The HW configuration is 64 bits registers regardless + * the CAAM or CPU addressing mode. + */ + jr_privdata->paddr_inrings = virt_to_phys(jr_privdata->inrings); + jr_privdata->paddr_outrings = virt_to_phys(jr_privdata->outrings); + if (!jr_privdata->paddr_inrings || !jr_privdata->paddr_outrings) { + JR_TRACE("JR bad queue pointers"); + retstatus = CAAM_FAILURE; + goto end_init; + } + + caam_hal_jr_config(jr_privdata->baseaddr, jr_privdata->nb_jobs, + jr_privdata->paddr_inrings, + jr_privdata->paddr_outrings); + + /* + * Prepare the interrupt handler to secure the interrupt even + * if the interrupt is not used + */ + jr_privdata->it_handler.it = jrcfg->it_num; + jr_privdata->it_handler.flags = ITRF_TRIGGER_LEVEL; + jr_privdata->it_handler.handler = caam_jr_irqhandler; + jr_privdata->it_handler.data = jr_privdata; + +#if defined(CFG_NXP_CAAM_RUNTIME_JR) && defined(CFG_CAAM_ITR) + itr_add(&jr_privdata->it_handler); +#endif + caam_hal_jr_enable_itr(jr_privdata->baseaddr); + + retstatus = CAAM_NO_ERROR; + +end_init: + if (retstatus != CAAM_NO_ERROR) + do_jr_free(jr_privdata); + + return retstatus; +} + +enum caam_status caam_jr_halt(void) +{ + enum caam_status retstatus = CAAM_FAILURE; + __maybe_unused uint32_t job_complete = 0; + + retstatus = caam_hal_jr_halt(jr_privdata->baseaddr); + + /* + * All jobs in the input queue have been done, call the + * dequeue function to complete them. + */ + job_complete = do_jr_dequeue(UINT32_MAX); + JR_TRACE("Completion of jobs mask 0x%" PRIx32, job_complete); + + return retstatus; +} + +enum caam_status caam_jr_flush(void) +{ + enum caam_status retstatus = CAAM_FAILURE; + __maybe_unused uint32_t job_complete = 0; + + retstatus = caam_hal_jr_flush(jr_privdata->baseaddr); + + /* + * All jobs in the input queue have been done, call the + * dequeue function to complete them. + */ + job_complete = do_jr_dequeue(UINT32_MAX); + JR_TRACE("Completion of jobs mask 0x%" PRIx32, job_complete); + + return retstatus; +} + +void caam_jr_resume(uint32_t pm_hint) +{ + if (pm_hint == PM_HINT_CONTEXT_STATE) { +#ifndef CFG_NXP_CAAM_RUNTIME_JR + /* + * In case the CAAM is not used the JR used to + * instantiate the RNG has been released to Non-Secure + * hence, need reconfigure the Secure JR and release + * it after RNG instantiation + */ + caam_hal_jr_setowner(jr_privdata->ctrladdr, + jr_privdata->jroffset, JROWN_ARM_S); + + caam_hal_jr_config(jr_privdata->baseaddr, jr_privdata->nb_jobs, + jr_privdata->paddr_inrings, + jr_privdata->paddr_outrings); +#endif /* CFG_NXP_CAAM_RUNTIME_JR */ + + /* Read the current job ring index */ + jr_privdata->inwrite_index = + caam_hal_jr_input_index(jr_privdata->baseaddr); + /* Read the current output ring index */ + jr_privdata->outread_index = + caam_hal_jr_output_index(jr_privdata->baseaddr); + + if (caam_rng_instantiation() != CAAM_NO_ERROR) + panic(); + +#ifndef CFG_NXP_CAAM_RUNTIME_JR + caam_hal_jr_setowner(jr_privdata->ctrladdr, + jr_privdata->jroffset, JROWN_ARM_NS); +#endif /* CFG_NXP_CAAM_RUNTIME_JR */ + } else { + caam_hal_jr_resume(jr_privdata->baseaddr); + } +} + +enum caam_status caam_jr_complete(void) +{ + enum caam_status ret = CAAM_BUSY; + + ret = caam_hal_jr_flush(jr_privdata->baseaddr); + if (ret == CAAM_NO_ERROR) + caam_hal_jr_resume(jr_privdata->baseaddr); + + return ret; +} diff --git a/optee_os/core/drivers/crypto/caam/caam_pwr.c b/optee_os/core/drivers/crypto/caam/caam_pwr.c new file mode 100644 index 0000000..d5a6ed4 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/caam_pwr.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019, 2021, 2023 NXP + * + * Brief CAAM Power state management. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static SLIST_HEAD(, + backup_data) data_list = SLIST_HEAD_INITIALIZER(backup_data); + +void caam_pwr_add_backup(vaddr_t baseaddr, const struct reglist *regs, + size_t nbentries) +{ + struct backup_data *newelem = NULL; + struct backup_data *elem = NULL; + uint32_t idx = 0; + uint32_t nbregs = 0; + + newelem = malloc(sizeof(*newelem)); + if (!newelem) + panic(); + + /* Count the number of registers to save/restore */ + for (idx = 0; idx < nbentries; idx++) + nbregs += regs[idx].nbregs; + + newelem->baseaddr = baseaddr; + newelem->nbentries = nbentries; + newelem->regs = regs; + newelem->val = malloc(nbregs * sizeof(*newelem->val)); + + if (!newelem->val) + panic(); + + /* Add the new backup data element at the end of the list */ + elem = SLIST_FIRST(&data_list); + if (elem) { + while (SLIST_NEXT(elem, next)) + elem = SLIST_NEXT(elem, next); + + SLIST_INSERT_AFTER(elem, newelem, next); + } else { + SLIST_INSERT_HEAD(&data_list, newelem, next); + } +} + +/* Backup all registers present in the data_list */ +static void do_save_regs(void) +{ + struct backup_data *elem = NULL; + const struct reglist *reg = NULL; + uint32_t idx = 0; + uint32_t validx = 0; + uint32_t regidx = 0; + + SLIST_FOREACH(elem, &data_list, next) { + reg = elem->regs; + validx = 0; + for (idx = 0; idx < elem->nbentries; idx++, reg++) { + for (regidx = 0; regidx < reg->nbregs; + regidx++, validx++) { + elem->val[validx] = + io_caam_read32(elem->baseaddr + + reg->offset + + (4 * regidx)); + elem->val[validx] &= ~reg->mask_clr; + + PWR_TRACE("Save @0x%" PRIxPTR "=0x%" PRIx32, + elem->baseaddr + reg->offset + + 4 * regidx, + elem->val[validx]); + } + } + } +} + +/* Restore all registers present in the data_list */ +static void do_restore_regs(void) +{ + struct backup_data *elem = NULL; + const struct reglist *reg = NULL; + uint32_t idx = 0; + uint32_t validx = 0; + uint32_t regidx = 0; + + SLIST_FOREACH(elem, &data_list, next) { + reg = elem->regs; + validx = 0; + for (idx = 0; idx < elem->nbentries; idx++, reg++) { + for (regidx = 0; regidx < reg->nbregs; + regidx++, validx++) { + PWR_TRACE("Restore @0x%" PRIxPTR "=0x%" PRIx32, + elem->baseaddr + reg->offset + + 4 * regidx, + elem->val[validx]); + io_caam_write32(elem->baseaddr + reg->offset + + 4 * regidx, + elem->val[validx] | + reg->mask_set); + } + } + } +} + +/* + * CAAM Power state preparation/entry + * + * @pm_hint Power mode type + */ +static TEE_Result pm_enter(uint32_t pm_hint) +{ + enum caam_status ret = CAAM_BUSY; + + PWR_TRACE("CAAM power mode %" PRIu32 " entry", pm_hint); + + if (pm_hint == PM_HINT_CLOCK_STATE) { + ret = caam_jr_halt(); + } else if (pm_hint == PM_HINT_CONTEXT_STATE) { + do_save_regs(); + ret = caam_jr_flush(); + } + + if (ret == CAAM_NO_ERROR) + return TEE_SUCCESS; + else + return TEE_ERROR_GENERIC; +} + +/* + * CAAM Power state resume + * + * @pm_hint Power mode type + */ +static TEE_Result pm_resume(uint32_t pm_hint) +{ + enum caam_status ret = CAAM_FAILURE; + + PWR_TRACE("CAAM power mode %" PRIu32 " resume", pm_hint); + if (pm_hint == PM_HINT_CONTEXT_STATE) { + caam_hal_clk_enable(true); + do_restore_regs(); + } + + caam_jr_resume(pm_hint); + + ret = caam_mp_resume(pm_hint); + + return caam_status_to_tee_result(ret); +} + +/* + * Power Management Callback function executed when system enter or resume + * from a power mode + * + * @op Operation mode SUSPEND/RESUME + * @pm_hint Power mode type + * @pm_handle Driver private handle (not used) + */ +static TEE_Result +pm_enter_resume(enum pm_op op, uint32_t pm_hint, + const struct pm_callback_handle *pm_handle __unused) +{ + if (op == PM_OP_SUSPEND) + return pm_enter(pm_hint); + else + return pm_resume(pm_hint); +} + +void caam_pwr_init(void) +{ + register_pm_driver_cb(pm_enter_resume, NULL, "caam_pwr"); +} diff --git a/optee_os/core/drivers/crypto/caam/caam_rng.c b/optee_os/core/drivers/crypto/caam/caam_rng.c new file mode 100644 index 0000000..fa81c22 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/caam_rng.c @@ -0,0 +1,570 @@ +// SPDX-License-Identifier: BSD-2-Clause +/** + * Copyright 2017-2021 NXP + * + * Brief CAAM Random Number Generator manager. + * Implementation of RNG functions. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Define the RNG Data buffer size and number + */ +#define RNG_DATABUF_SIZE 1024 +#define RNG_DATABUF_NB 2 + +/* + * Define the number of descriptor entry to generate random data + */ +#define RNG_GEN_DESC_ENTRIES 5 + +/* + * Status of the data generation + */ +enum rngsta { + DATA_EMPTY = 0, /* Data bufer empty */ + DATA_ONGOING, /* Data generation on going */ + DATA_FAILURE, /* Error during data generation */ + DATA_OK, /* Data generation complete with success */ +}; + +/* + * RNG Data generation + */ +struct rngdata { + struct caam_jobctx jobctx; /* Job Ring Context */ + uint32_t job_id; /* Job Id enqueued */ + + uint8_t *data; /* Random Data buffer */ + size_t size; /* Size in bytes of the Random data buffer */ + size_t rdindex; /* Current data index in the buffer */ + + enum rngsta status; /* Status of the data generation */ +}; + +/* + * RNG module private data + */ +struct rng_privdata { + vaddr_t baseaddr; /* RNG base address */ + bool instantiated; /* RNG instantiated */ + struct rngdata databuf[RNG_DATABUF_NB]; /* RNG Data generation */ + uint8_t dataidx; /* Current RNG Data buffer */ +}; + +static struct rng_privdata *rng_privdata; + +/* Allocate and initialize module private data */ +static enum caam_status do_allocate(void) +{ + struct rngdata *rngdata = NULL; + unsigned int idx = 0; + + /* Allocate the Module resources */ + rng_privdata = caam_calloc(sizeof(*rng_privdata)); + if (!rng_privdata) { + RNG_TRACE("Private Data allocation error"); + return CAAM_OUT_MEMORY; + } + + rng_privdata->instantiated = false; + + /* Allocates the RNG Data Buffers */ + for (idx = 0; idx < RNG_DATABUF_NB; idx++) { + rngdata = &rng_privdata->databuf[idx]; + rngdata->data = caam_calloc_align(RNG_DATABUF_SIZE); + if (!rngdata->data) + return CAAM_OUT_MEMORY; + + rngdata->size = RNG_DATABUF_SIZE; + rngdata->jobctx.desc = caam_calloc_desc(RNG_GEN_DESC_ENTRIES); + if (!rngdata->jobctx.desc) + return CAAM_OUT_MEMORY; + } + + return CAAM_NO_ERROR; +} + +/* Free module private data */ +static void do_free(void) +{ + struct rngdata *rng = NULL; + unsigned int idx = 0; + + if (rng_privdata) { + for (idx = 0; idx < RNG_DATABUF_NB; idx++) { + rng = &rng_privdata->databuf[idx]; + + /* Check if there is a Job ongoing to cancel it */ + if (atomic_load_u32(&rng->status) == DATA_ONGOING) + caam_jr_cancel(rng->job_id); + + caam_free_desc(&rng->jobctx.desc); + caam_free(rng->data); + rng->data = NULL; + } + + caam_free(rng_privdata); + rng_privdata = NULL; + } +} + +#ifdef CFG_NXP_CAAM_RNG_DRV +/* + * RNG data generation job ring callback completion + * + * @jobctx RNG data JR Job Context + */ +static void rng_data_done(struct caam_jobctx *jobctx) +{ + struct rngdata *rng = jobctx->context; + + RNG_TRACE("RNG Data id 0x%08" PRIx32 " done with status 0x%" PRIx32, + rng->job_id, jobctx->status); + + if (JRSTA_SRC_GET(jobctx->status) == JRSTA_SRC(NONE)) { + atomic_store_u32(&rng->status, DATA_OK); + + /* Invalidate the data buffer to ensure software gets it */ + cache_operation(TEE_CACHEINVALIDATE, rng->data, rng->size); + } else { + RNG_TRACE("RNG Data completion in error 0x%" PRIx32, + jobctx->status); + atomic_store_u32(&rng->status, DATA_FAILURE); + } + + rng->job_id = 0; + rng->rdindex = 0; +} + +/* + * Prepares the data generation descriptors + * + * @rng Reference to the RNG Data object + */ +static enum caam_status prepare_gen_desc(struct rngdata *rng) +{ + paddr_t paddr = 0; + uint32_t *desc = NULL; + + /* Convert the buffer virtual address to physical address */ + paddr = virt_to_phys(rng->data); + if (!paddr) + return CAAM_FAILURE; + + desc = rng->jobctx.desc; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, RNG_GEN_DATA); + caam_desc_add_word(desc, FIFO_ST(RNG_TO_MEM, rng->size)); + caam_desc_add_ptr(desc, paddr); + + RNG_DUMPDESC(desc); + + /* Prepare the job context */ + rng->jobctx.context = rng; + rng->jobctx.callback = rng_data_done; + return CAAM_NO_ERROR; +} + +/* + * Launches a RNG Data generation + * + * @rng RNG Data context + */ +static enum caam_status do_rng_start(struct rngdata *rng) +{ + enum caam_status ret = CAAM_FAILURE; + + /* Ensure that data buffer is visible from the HW */ + cache_operation(TEE_CACHEFLUSH, rng->data, rng->size); + + rng->job_id = 0; + atomic_store_u32(&rng->status, DATA_EMPTY); + + ret = caam_jr_enqueue(&rng->jobctx, &rng->job_id); + + if (ret == CAAM_PENDING) { + atomic_store_u32(&rng->status, DATA_ONGOING); + ret = CAAM_NO_ERROR; + } else { + RNG_TRACE("RNG Job Ring Error 0x%08x", ret); + atomic_store_u32(&rng->status, DATA_FAILURE); + ret = CAAM_FAILURE; + } + + return ret; +} + +/* Checks if there are random data available */ +static enum caam_status do_check_data(void) +{ + enum caam_status ret = CAAM_FAILURE; + struct rngdata *rng = NULL; + uint32_t wait_jobs = 0; + unsigned int idx = 0; + unsigned int loop = 4; + + /* Check if there is a RNG Job to be run */ + for (idx = 0; idx < RNG_DATABUF_NB; idx++) { + rng = &rng_privdata->databuf[idx]; + if (atomic_load_u32(&rng->status) == DATA_EMPTY) { + RNG_TRACE("Start RNG #%" PRIu32 " data generation", + idx); + ret = do_rng_start(rng); + if (ret != CAAM_NO_ERROR) + return CAAM_FAILURE; + } + } + + /* Check if the current data buffer contains data */ + rng = &rng_privdata->databuf[rng_privdata->dataidx]; + + switch (atomic_load_u32(&rng->status)) { + case DATA_OK: + return CAAM_NO_ERROR; + + case DATA_FAILURE: + return CAAM_FAILURE; + + default: + /* Wait until one of the data buffer completes */ + do { + wait_jobs = 0; + for (idx = 0; idx < RNG_DATABUF_NB; idx++) { + rng = &rng_privdata->databuf[idx]; + wait_jobs |= rng->job_id; + + if (atomic_load_u32(&rng->status) == DATA_OK) { + RNG_TRACE("RNG Data buffer #%" PRIu32 + " ready", + idx); + rng_privdata->dataidx = idx; + return CAAM_NO_ERROR; + } + } + + if (!wait_jobs) { + RNG_TRACE("There are no Data Buffers ongoing"); + return CAAM_FAILURE; + } + + /* Need to wait until one of the jobs completes */ + (void)caam_jr_dequeue(wait_jobs, 100); + } while (loop--); + + break; + } + + return CAAM_FAILURE; +} + +/* + * Return the requested random data + * + * @buf [out] data buffer + * @len number of bytes to returns + */ +static TEE_Result do_rng_read(uint8_t *buf, size_t len) +{ + struct rngdata *rng = NULL; + size_t remlen = len; + uint8_t *rngbuf = buf; + + if (!rng_privdata) { + RNG_TRACE("RNG Driver not initialized"); + return TEE_ERROR_BAD_STATE; + } + + if (!rng_privdata->instantiated) { + RNG_TRACE("RNG Driver not initialized"); + return TEE_ERROR_BAD_STATE; + } + + do { + if (do_check_data() != CAAM_NO_ERROR) { + RNG_TRACE("No Data available or Error"); + return TEE_ERROR_BAD_STATE; + } + + rng = &rng_privdata->databuf[rng_privdata->dataidx]; + RNG_TRACE("Context #%" PRIu8 + " contains %zu data asked %zu (%zu)", + rng_privdata->dataidx, rng->size - rng->rdindex, + remlen, len); + + /* Check that current context data are available */ + if ((rng->size - rng->rdindex) <= remlen) { + /* + * There is no or just enough data available, + * copy all data + */ + RNG_TRACE("Copy all available data"); + memcpy(rngbuf, &rng->data[rng->rdindex], + rng->size - rng->rdindex); + + remlen -= rng->size - rng->rdindex; + rngbuf += rng->size - rng->rdindex; + /* Set the RNG data status as empty */ + atomic_store_u32(&rng->status, DATA_EMPTY); + } else { + /* There is enough data in the current context */ + RNG_TRACE("Copy %zu data", remlen); + memcpy(rngbuf, &rng->data[rng->rdindex], remlen); + rng->rdindex += remlen; + remlen = 0; + } + } while (remlen); + + return TEE_SUCCESS; +} + +/* Initialize the RNG module to generate data */ +static enum caam_status caam_rng_init_data(void) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct rngdata *rng = NULL; + unsigned int idx = 0; + + for (idx = 0; idx < RNG_DATABUF_NB; idx++) { + rng = &rng_privdata->databuf[idx]; + retstatus = prepare_gen_desc(rng); + + if (retstatus != CAAM_NO_ERROR) + break; + } + + return retstatus; +} +#endif /* CFG_NXP_CAAM_RNG_DRV */ + +/* + * Prepares the instantiation descriptor + * + * @nb_sh Number of the State Handle + * @sh_status State Handles status + * @desc Reference to the descriptor + * @desc [out] Descriptor filled + */ +static void prepare_inst_desc(uint32_t nb_sh, uint32_t sh_status, + uint32_t *desc) +{ + bool key_loaded = false; + unsigned int sh_idx = 0; + unsigned int nb_max_sh = nb_sh; + + /* Read the SH and secure key status */ + key_loaded = caam_hal_rng_key_loaded(rng_privdata->baseaddr); + RNG_TRACE("RNG SH Status 0x%08" PRIx32 " - Key Status %" PRId8, + sh_status, key_loaded); + + while (sh_status & BIT(sh_idx)) + sh_idx++; + + RNG_TRACE("Instantiation start at SH%" PRIu32 " (%" PRIu32 ")", sh_idx, + nb_max_sh); + + /* Don't set the descriptor header now */ + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + /* First State Handle to instantiate */ + caam_desc_add_word(desc, RNG_SH_INST(sh_idx)); + + /* Next State Handles */ + for (sh_idx++; sh_idx < nb_max_sh; sh_idx++) { + if (!(sh_status & BIT(sh_idx))) { + /* + * If there is more SH to instantiate, add a wait loop + * followed by a reset of the done status to execute + * next command + */ + caam_desc_add_word(desc, + JUMP_C1_LOCAL(ALL_COND_TRUE, + JMP_COND(NONE), 1)); + caam_desc_add_word(desc, + LD_NOCLASS_IMM(REG_CLEAR_WRITTEN, + sizeof(uint32_t))); + caam_desc_add_word(desc, 0x1); + caam_desc_add_word(desc, RNG_SH_INST(sh_idx)); + } + } + + /* Load the Key if needed */ + if (!key_loaded) { + /* + * Add a wait loop while previous operation not completed, + * followed by a register clear before executing next command + */ + caam_desc_add_word(desc, JUMP_C1_LOCAL(ALL_COND_TRUE, + JMP_COND(NONE), 1)); + caam_desc_add_word(desc, LD_NOCLASS_IMM(REG_CLEAR_WRITTEN, + sizeof(uint32_t))); + caam_desc_add_word(desc, 0x1); + caam_desc_add_word(desc, RNG_GEN_SECKEYS); + } + + RNG_DUMPDESC(desc); +} + +enum caam_status caam_rng_instantiation(void) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = {}; + uint32_t *desc = NULL; + uint32_t sh_status = 0; + uint32_t nb_sh = 0; + uint32_t sh_mask = 0; + uint32_t inc_delay = 0; + + RNG_TRACE("RNG Instantation"); + + /* Check if RNG is already instantiated */ + retstatus = caam_hal_rng_instantiated(rng_privdata->baseaddr); + + /* RNG is already instantiated or an error occurred */ + if (retstatus != CAAM_NOT_INIT) + goto end_inst; + + /* + * RNG needs to be instantiated. Allocate and prepare the + * Job Descriptor + */ + + /* Calculate the State Handles bit mask */ + nb_sh = caam_hal_rng_get_nb_sh(rng_privdata->baseaddr); + sh_mask = GENMASK_32(nb_sh - 1, 0); + + /* + * The maximum size of the descriptor is: + * |----------------------| + * | Header | = 1 + * |----------------------| + * | First instantation | = 1 + * |----------------------|------------------------- + * | wait complete | = 1 + * |----------------------| + * | Clear done status | Repeat (nb_sh - 1) + * | | = 2 + * |----------------------| + * | next SH instantation | = 1 + * |----------------------|------------------------- + * | wait complete | = 1 + * |----------------------| + * | Clear done status | = 2 + * | | + * |----------------------| + * | Generate Secure Keys | = 1 + * |----------------------| + * | Pad with a 0 | = 1 + */ + desc = caam_calloc_desc(2 + (nb_sh - 1) * 4 + 4 + 1); + if (!desc) { + RNG_TRACE("Descriptor Allocation error"); + retstatus = CAAM_OUT_MEMORY; + goto end_inst; + } + + jobctx.desc = desc; + + do { + /* Check if all State Handles are instantiated */ + sh_status = caam_hal_rng_get_sh_status(rng_privdata->baseaddr); + if ((sh_status & sh_mask) == sh_mask) { + RNG_TRACE("RNG All SH are instantiated (0x%08" PRIx32 + ")", + sh_status); + retstatus = CAAM_NO_ERROR; + goto end_inst; + } + + if (sh_status == 0) { + retstatus = caam_hal_rng_kick(rng_privdata->baseaddr, + inc_delay); + RNG_TRACE("RNG Kick (inc=%" PRIu32 ") ret 0x%08x", + inc_delay, retstatus); + if (retstatus != CAAM_NO_ERROR) { + retstatus = CAAM_FAILURE; + goto end_inst; + } + inc_delay += 200; + } + + prepare_inst_desc(nb_sh, sh_status, desc); + + retstatus = caam_jr_enqueue(&jobctx, NULL); + RNG_TRACE("RNG Job returned 0x%08x", retstatus); + + if (retstatus != CAAM_NO_ERROR && + retstatus != CAAM_JOB_STATUS) + goto end_inst; + + if (retstatus == CAAM_JOB_STATUS) { + RNG_TRACE("RNG Job status 0x%08" PRIx32, jobctx.status); + if ((JRSTA_SRC_GET(jobctx.status) != JRSTA_SRC(CCB)) || + (JRSTA_CCB_GET_ERR(jobctx.status) != + (JRSTA_CCB_CHAID_RNG | JRSTA_CCB_ERRID_HW))) + retstatus = CAAM_FAILURE; + else + retstatus = CAAM_NO_ERROR; + } + } while (retstatus == CAAM_NO_ERROR); + +end_inst: + if (retstatus == CAAM_NO_ERROR) + rng_privdata->instantiated = true; + + caam_free_desc(&desc); + + RNG_TRACE("RNG Instantiation return 0x%08x", retstatus); + + return retstatus; +} + +enum caam_status caam_rng_init(vaddr_t ctrl_addr) +{ + enum caam_status retstatus = CAAM_FAILURE; + + RNG_TRACE("Initialization"); + retstatus = do_allocate(); + if (retstatus == CAAM_NO_ERROR) { + rng_privdata->baseaddr = ctrl_addr; + retstatus = caam_rng_instantiation(); + } + +#ifdef CFG_NXP_CAAM_RNG_DRV + if (retstatus == CAAM_NO_ERROR) + retstatus = caam_rng_init_data(); +#endif + + if (retstatus != CAAM_NO_ERROR) + do_free(); + + return retstatus; +} + +#ifdef CFG_NXP_CAAM_RNG_DRV +TEE_Result hw_get_random_bytes(void *buf, size_t blen) +{ + if (!buf) + return TEE_ERROR_BAD_PARAMETERS; + + return do_rng_read(buf, blen); +} + +void plat_rng_init(void) +{ +} +#endif diff --git a/optee_os/core/drivers/crypto/caam/caam_sm.c b/optee_os/core/drivers/crypto/caam/caam_sm.c new file mode 100644 index 0000000..231df87 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/caam_sm.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019, 2023 NXP + */ +#include +#include +#include +#include +#include +#include +#include + +/* + * Secure memory module private data + */ +static struct sm_privdata { + vaddr_t baseaddr; /* Secure memory base address */ + vaddr_t ctrl_addr; /* CAAM base address */ + vaddr_t jr_addr; /* Job Ring base address */ + paddr_t jr_offset; /* Job Ring offset */ +} sm_privdata; + +enum caam_status caam_sm_alloc(const struct caam_sm_page_desc *page_desc, + struct caam_sm_page_addr *page_addr) +{ + enum caam_status ret = CAAM_FAILURE; + + if (!page_desc || !page_addr) + return TEE_ERROR_BAD_PARAMETERS; + + ret = caam_hal_sm_check_page_partition(sm_privdata.jr_addr, page_desc); + if (ret != CAAM_NO_ERROR) { + SM_TRACE("Pages %u to %u or partition %u are out of bounds", + page_desc->page, + page_desc->page + page_desc->page_count - 1, + page_desc->partition); + return ret; + } + + /* Check if partition is already allocated */ + if (!caam_hal_sm_prtn_is_free(sm_privdata.jr_addr, + page_desc->partition)) { + SM_TRACE("Partition %u not available", page_desc->partition); + return CAAM_BUSY; + } + + /* Open secure memory partition to all groups */ + caam_hal_sm_open_access_perm(sm_privdata.jr_addr, page_desc->partition); + caam_hal_sm_set_access_all_group(sm_privdata.jr_addr, + page_desc->partition); + + ret = caam_hal_sm_allocate_page(sm_privdata.jr_addr, page_desc); + if (ret != CAAM_NO_ERROR) { + SM_TRACE("Error allocation Pages %u to %u into partition %u", + page_desc->page, + page_desc->page + page_desc->page_count - 1, + page_desc->partition); + + /* Free all potientiel pages allocated before failure */ + return caam_hal_sm_deallocate_pages(sm_privdata.jr_addr, + page_desc); + } + + page_addr->paddr = caam_hal_ctrl_get_smvaddr(sm_privdata.ctrl_addr, + sm_privdata.jr_offset) + + caam_hal_sm_get_pages_size(sm_privdata.jr_addr, + page_desc->page); + page_addr->vaddr = sm_privdata.baseaddr + + caam_hal_sm_get_pages_size(sm_privdata.jr_addr, + page_desc->page); + + SM_TRACE("Partition %u Pages %u to %u allocated @0x%" PRIxVA + " (phys 0x@%" PRIxPA, + page_desc->partition, page_desc->page, + page_desc->page + page_desc->page_count - 1, page_addr->vaddr, + page_addr->paddr); + + return CAAM_NO_ERROR; +} + +enum caam_status caam_sm_free(const struct caam_sm_page_desc *page_desc) +{ + enum caam_status ret = CAAM_FAILURE; + + SM_TRACE("Free Secure Memory pages %u to %u from partition %u", + page_desc->page, page_desc->page + page_desc->page_count, + page_desc->partition); + + /* + * De-allocate partition. It automatically releases partition's pages + * to the pool of available pages. if the partition if marked as CSP, + * pages will be zeroized. If the partition is marked as PSP, + * partition and pages will not be de-allocated and a PSP will be + * returned. + */ + if (!caam_hal_sm_prtn_is_owned(sm_privdata.jr_addr, + page_desc->partition)) { + SM_TRACE("Partition %u not owned by used JR", + page_desc->partition); + return TEE_ERROR_ACCESS_DENIED; + } + + ret = caam_hal_sm_deallocate_pages(sm_privdata.jr_addr, page_desc); + if (ret) { + SM_TRACE("De-alloc pages %u to %u error 0x%" PRIx32, + page_desc->page, + page_desc->page + page_desc->page_count, ret); + + return ret; + } + + ret = caam_hal_sm_deallocate_partition(sm_privdata.jr_addr, + page_desc->partition); + if (ret) { + SM_TRACE("De-alloc partition %u error 0x%" PRIx32, + page_desc->partition, ret); + return ret; + } + + return CAAM_NO_ERROR; +} + +enum caam_status +caam_sm_set_access_perm(const struct caam_sm_page_desc *page_desc, + unsigned int grp1_perm, unsigned int grp2_perm) +{ + uint32_t grp1 = UINT32_MAX; + uint32_t grp2 = UINT32_MAX; + + if (!page_desc) + return CAAM_BAD_PARAM; + + /* Check if the partition is already owned */ + if (!caam_hal_sm_prtn_is_owned(sm_privdata.jr_addr, + page_desc->partition)) { + SM_TRACE("Partition %d not owned by current JR", + page_desc->partition); + return CAAM_FAILURE; + } + + /* + * Set ourself to access Secure Memory group 1 and/or group 2 + * function if @grp1_perm and/or @grp2_perm not equal 0. + * + * The Access Group is related to the Job Ring owner setting without + * the Secure Bit setting already managed by the Job Ring. + */ + if (grp1_perm) + grp1 = JROWN_ARM_NS; + + if (grp2_perm) + grp2 = JROWN_ARM_NS; + + caam_hal_sm_set_access_group(sm_privdata.jr_addr, page_desc->partition, + grp1, grp2); + caam_hal_sm_set_access_perm(sm_privdata.jr_addr, page_desc->partition, + grp1_perm, grp2_perm); + + return CAAM_NO_ERROR; +} + +enum caam_status caam_sm_init(struct caam_jrcfg *jrcfg) +{ + if (!jrcfg) + return CAAM_FAILURE; + + sm_privdata.ctrl_addr = jrcfg->base; + sm_privdata.jr_addr = jrcfg->base + jrcfg->offset; + sm_privdata.jr_offset = jrcfg->offset; + sm_privdata.baseaddr = caam_hal_sm_get_base(); + + if (!sm_privdata.baseaddr) + return CAAM_FAILURE; + + SM_TRACE("Secure Memory Base address = 0x%" PRIxVA, + sm_privdata.baseaddr); + SM_TRACE("CAAM controller address = 0x%" PRIxVA, sm_privdata.ctrl_addr); + SM_TRACE("CAAM Job Ring address = 0x%" PRIxVA, sm_privdata.jr_addr); + + return CAAM_NO_ERROR; +} diff --git a/optee_os/core/drivers/crypto/caam/cipher/caam_cipher.c b/optee_os/core/drivers/crypto/caam/cipher/caam_cipher.c new file mode 100644 index 0000000..29cb8f4 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/cipher/caam_cipher.c @@ -0,0 +1,901 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Implementation of Cipher functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +/* Local Function declaration */ +static TEE_Result do_update_streaming(struct drvcrypt_cipher_update *dupdate); +static TEE_Result do_update_cipher(struct drvcrypt_cipher_update *dupdate); + +/* + * Constants definition of the AES algorithm + */ +static const struct cipheralg aes_alg[] = { + [TEE_CHAIN_MODE_ECB_NOPAD] = { + .type = OP_ALGO(AES) | ALGO_AAI(AES_ECB), + .size_block = TEE_AES_BLOCK_SIZE, + .size_ctx = 0, + .ctx_offset = 0, + .require_key = NEED_KEY1, + .def_key = { .min = 16, .max = 32, .mod = 8 }, + .update = do_update_cipher, + }, + [TEE_CHAIN_MODE_CBC_NOPAD] = { + .type = OP_ALGO(AES) | ALGO_AAI(AES_CBC), + .size_block = TEE_AES_BLOCK_SIZE, + .size_ctx = 2 * sizeof(uint64_t), + .ctx_offset = 0, + .require_key = NEED_KEY1 | NEED_IV, + .def_key = { .min = 16, .max = 32, .mod = 8 }, + .update = do_update_cipher, + }, + [TEE_CHAIN_MODE_CTR] = { + .type = OP_ALGO(AES) | ALGO_AAI(AES_CTR_MOD128), + .size_block = TEE_AES_BLOCK_SIZE, + .size_ctx = 2 * sizeof(uint64_t), + .ctx_offset = 16, + .require_key = NEED_KEY1 | NEED_IV, + .def_key = { .min = 16, .max = 32, .mod = 8 }, + .update = do_update_streaming, + }, + [TEE_CHAIN_MODE_CTS] = { + .type = 0, + }, + [TEE_CHAIN_MODE_XTS] = { + .type = OP_ALGO(AES) | ALGO_AAI(AES_ECB), + .size_block = TEE_AES_BLOCK_SIZE, + .size_ctx = 0, + .ctx_offset = 0, + .require_key = NEED_KEY1 | NEED_KEY2 | NEED_TWEAK, + .def_key = { .min = 16, .max = 32, .mod = 8 }, + .update = caam_cipher_update_xts, + }, +}; + +/* + * Constants definition of the DES algorithm + */ +static const struct cipheralg des_alg[] = { + [TEE_CHAIN_MODE_ECB_NOPAD] = { + .type = OP_ALGO(DES) | ALGO_AAI(DES_ECB), + .size_block = TEE_DES_BLOCK_SIZE, + .size_ctx = 0, + .ctx_offset = 0, + .require_key = NEED_KEY1, + .def_key = { .min = 8, .max = 8, .mod = 8 }, + .update = do_update_cipher, + }, + [TEE_CHAIN_MODE_CBC_NOPAD] = { + .type = OP_ALGO(DES) | ALGO_AAI(DES_CBC), + .size_block = TEE_DES_BLOCK_SIZE, + .size_ctx = sizeof(uint64_t), + .ctx_offset = 0, + .require_key = NEED_KEY1 | NEED_IV, + .def_key = { .min = 8, .max = 8, .mod = 8 }, + .update = do_update_cipher, + }, +}; + +/* + * Constants definition of the DES3 algorithm + */ +static const struct cipheralg des3_alg[] = { + [TEE_CHAIN_MODE_ECB_NOPAD] = { + .type = OP_ALGO(3DES) | ALGO_AAI(DES_ECB), + .size_block = TEE_DES_BLOCK_SIZE, + .size_ctx = 0, + .ctx_offset = 0, + .require_key = NEED_KEY1, + .def_key = { .min = 16, .max = 24, .mod = 8 }, + .update = do_update_cipher, + }, + [TEE_CHAIN_MODE_CBC_NOPAD] = { + /* Triple-DES CBC No Pad */ + .type = OP_ALGO(3DES) | ALGO_AAI(DES_CBC), + .size_block = TEE_DES_BLOCK_SIZE, + .size_ctx = sizeof(uint64_t), + .ctx_offset = 0, + .require_key = NEED_KEY1 | NEED_IV, + .def_key = { .min = 16, .max = 24, .mod = 8 }, + .update = do_update_cipher, + }, +}; + +/* + * Allocate context data and copy input data into + * + * @dst [out] Destination data to allocate and fill + * @src Source of data to copy + */ +static enum caam_status copy_ctx_data(struct caambuf *dst, + struct drvcrypt_buf *src) +{ + enum caam_status ret = CAAM_OUT_MEMORY; + + if (!dst->data) { + /* Allocate the destination buffer */ + ret = caam_alloc_align_buf(dst, src->length); + if (ret != CAAM_NO_ERROR) + return CAAM_OUT_MEMORY; + } + + /* Do the copy */ + memcpy(dst->data, src->data, dst->length); + + /* Push data to physical memory */ + cache_operation(TEE_CACHEFLUSH, dst->data, dst->length); + + return CAAM_NO_ERROR; +} + +/* + * Verify the input key size with the requirements + * + * @def Key requirements + * @size Key size to verify + */ +static enum caam_status do_check_keysize(const struct caamdefkey *def, + size_t size) +{ + if (size >= def->min && size <= def->max && !(size % def->mod)) + return CAAM_NO_ERROR; + + return CAAM_BAD_PARAM; +} + +enum caam_status caam_cipher_block(struct cipherdata *ctx, bool savectx, + uint8_t keyid, bool encrypt, + struct caamdmaobj *src, + struct caamdmaobj *dst) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + uint32_t *desc = ctx->descriptor; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + if (keyid == NEED_KEY1) { + /* Build the descriptor */ + caam_desc_add_word(desc, LD_KEY_PLAIN(CLASS_1, REG, + ctx->key1.length)); + caam_desc_add_ptr(desc, ctx->key1.paddr); + } else if (keyid == NEED_KEY2) { + /* Build the descriptor */ + caam_desc_add_word(desc, LD_KEY_PLAIN(CLASS_1, REG, + ctx->key2.length)); + caam_desc_add_ptr(desc, ctx->key2.paddr); + } + + /* If there is a context register load it */ + if (ctx->ctx.length && ctx->alg->size_ctx) { + caam_desc_add_word(desc, LD_NOIMM_OFF(CLASS_1, REG_CTX, + ctx->ctx.length, + ctx->alg->ctx_offset)); + caam_desc_add_ptr(desc, ctx->ctx.paddr); + /* Operation with the direction */ + caam_desc_add_word(desc, CIPHER_INIT(ctx->alg->type, encrypt)); + } else { + /* Operation with the direction */ + caam_desc_add_word(desc, + CIPHER_INITFINAL(ctx->alg->type, encrypt)); + } + + /* Load the source data if any */ + if (src) { + caam_desc_fifo_load(desc, src, CLASS_1, MSG, LAST_C1); + caam_dmaobj_cache_push(src); + } + + /* Store the output data if any */ + if (dst) { + caam_desc_fifo_store(desc, dst, MSG_DATA); + caam_dmaobj_cache_push(dst); + } + + if (ctx->ctx.length && ctx->alg->size_ctx) { + if (savectx) { + /* Store the context */ + caam_desc_add_word(desc, + ST_NOIMM_OFF(CLASS_1, REG_CTX, + ctx->ctx.length, + ctx->alg->ctx_offset)); + caam_desc_add_ptr(desc, ctx->ctx.paddr); + } + + /* Ensure Context register data are not in cache */ + cache_operation(TEE_CACHEINVALIDATE, ctx->ctx.data, + ctx->ctx.length); + } + + CIPHER_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus != CAAM_NO_ERROR) { + CIPHER_TRACE("CAAM return 0x%08x Status 0x%08" PRIx32, + retstatus, jobctx.status); + retstatus = CAAM_FAILURE; + } + + return retstatus; +} + +/* + * Checks if the algorithm @algo is supported and returns the + * local algorithm entry in the corresponding cipher array + */ +static const struct cipheralg *get_cipheralgo(uint32_t algo) +{ + unsigned int algo_id = TEE_ALG_GET_MAIN_ALG(algo); + unsigned int algo_md = TEE_ALG_GET_CHAIN_MODE(algo); + const struct cipheralg *ca = NULL; + + CIPHER_TRACE("Algo id:%" PRId32 " md:%" PRId32, algo_id, algo_md); + + switch (algo_id) { + case TEE_MAIN_ALGO_AES: + if (algo_md < ARRAY_SIZE(aes_alg)) + ca = &aes_alg[algo_md]; + break; + + case TEE_MAIN_ALGO_DES: + if (algo_md < ARRAY_SIZE(des_alg)) + ca = &des_alg[algo_md]; + break; + + case TEE_MAIN_ALGO_DES3: + if (algo_md < ARRAY_SIZE(des3_alg)) + ca = &des3_alg[algo_md]; + break; + + default: + break; + } + + if (ca && ca->type) + return ca; + + return NULL; +} + +/* + * Allocate the SW cipher data context + * + * @ctx [out] Caller context variable + * @algo Algorithm ID of the context + */ +static TEE_Result do_allocate(void **ctx, uint32_t algo) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct cipherdata *cipherdata = NULL; + const struct cipheralg *alg = NULL; + + CIPHER_TRACE("Allocate Algo 0x%" PRIX32 " Context (%p)", algo, ctx); + + alg = get_cipheralgo(algo); + if (!alg) { + CIPHER_TRACE("Algorithm not supported"); + return TEE_ERROR_NOT_IMPLEMENTED; + } + + cipherdata = caam_calloc(sizeof(*cipherdata)); + if (!cipherdata) { + CIPHER_TRACE("Allocation Cipher data error"); + return TEE_ERROR_OUT_OF_MEMORY; + } + + /* Allocate the descriptor */ + cipherdata->descriptor = caam_calloc_desc(MAX_DESC_ENTRIES); + if (!cipherdata->descriptor) { + CIPHER_TRACE("Allocation descriptor error"); + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Setup the Algorithm pointer */ + cipherdata->alg = alg; + + /* Initialize the block buffer */ + cipherdata->blockbuf.max = cipherdata->alg->size_block; + + *ctx = cipherdata; + + return TEE_SUCCESS; + +out: + caam_free_desc(&cipherdata->descriptor); + caam_free(cipherdata); + + return ret; +} + +/* + * Free the internal cipher data context + * + * @ctx Caller context variable or NULL + */ +static void do_free_intern(struct cipherdata *ctx) +{ + CIPHER_TRACE("Free Context (%p)", ctx); + + if (ctx) { + /* Free the descriptor */ + caam_free_desc(&ctx->descriptor); + + /* Free the Key 1 */ + caam_free_buf(&ctx->key1); + + /* Free the Key 2 */ + caam_free_buf(&ctx->key2); + + /* Free the Tweak */ + caam_free_buf(&ctx->tweak); + + /* Free the Context Register */ + caam_free_buf(&ctx->ctx); + + /* Free Temporary buffer */ + caam_free_buf(&ctx->blockbuf.buf); + } +} + +void caam_cipher_free(void *ctx) +{ + CIPHER_TRACE("Free Context (%p)", ctx); + + if (ctx) { + do_free_intern(ctx); + caam_free(ctx); + } +} + +void caam_cipher_copy_state(void *dst_ctx, void *src_ctx) +{ + struct cipherdata *dst = dst_ctx; + struct cipherdata *src = src_ctx; + + CIPHER_TRACE("Copy State context (%p) to (%p)", src_ctx, dst_ctx); + + dst->alg = src->alg; + dst->encrypt = src->encrypt; + + if (src->blockbuf.filled) { + struct caambuf srcdata = { + .data = src->blockbuf.buf.data, + .length = src->blockbuf.filled + }; + caam_cpy_block_src(&dst->blockbuf, &srcdata, 0); + } + + if (src->key1.length) { + struct drvcrypt_buf key1 = { + .data = src->key1.data, + .length = src->key1.length + }; + copy_ctx_data(&dst->key1, &key1); + } + + if (src->key2.length) { + struct drvcrypt_buf key2 = { + .data = src->key2.data, + .length = src->key2.length + }; + copy_ctx_data(&dst->key2, &key2); + } + + if (src->ctx.length) { + struct drvcrypt_buf ctx = { + .data = src->ctx.data, + .length = src->ctx.length + }; + cache_operation(TEE_CACHEINVALIDATE, ctx.data, ctx.length); + copy_ctx_data(&dst->ctx, &ctx); + } + + if (src->tweak.length) { + struct drvcrypt_buf tweak = { + .data = src->tweak.data, + .length = src->tweak.length + }; + copy_ctx_data(&dst->tweak, &tweak); + } +} + +TEE_Result caam_cipher_initialize(struct drvcrypt_cipher_init *dinit) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + enum caam_status retstatus = CAAM_FAILURE; + struct cipherdata *cipherdata = dinit->ctx; + const struct cipheralg *alg = NULL; + + CIPHER_TRACE("Action %s", dinit->encrypt ? "Encrypt" : "Decrypt"); + + if (!cipherdata) + return ret; + + alg = cipherdata->alg; + + /* Check if all required keys are defined */ + if (alg->require_key & NEED_KEY1) { + if (!dinit->key1.data || !dinit->key1.length) + goto out; + + retstatus = do_check_keysize(&alg->def_key, dinit->key1.length); + if (retstatus != CAAM_NO_ERROR) { + CIPHER_TRACE("Bad Key 1 size"); + goto out; + } + + /* Copy the key 1 */ + retstatus = copy_ctx_data(&cipherdata->key1, &dinit->key1); + CIPHER_TRACE("Copy Key 1 returned 0x%" PRIx32, retstatus); + + if (retstatus != CAAM_NO_ERROR) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + } + + if (alg->require_key & NEED_KEY2) { + if (!dinit->key2.data || !dinit->key2.length) + goto out; + + retstatus = do_check_keysize(&alg->def_key, dinit->key2.length); + if (retstatus != CAAM_NO_ERROR) { + CIPHER_TRACE("Bad Key 2 size"); + goto out; + } + + /* Copy the key 2 */ + retstatus = copy_ctx_data(&cipherdata->key2, &dinit->key2); + CIPHER_TRACE("Copy Key 2 returned 0x%" PRIx32, retstatus); + + if (retstatus != CAAM_NO_ERROR) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + } + + if (alg->require_key & NEED_IV) { + if (!dinit->iv.data || !dinit->iv.length) + goto out; + + if (dinit->iv.length != alg->size_ctx) { + CIPHER_TRACE("Bad IV size %zu (expected %" PRId32 ")", + dinit->iv.length, alg->size_ctx); + goto out; + } + + CIPHER_TRACE("Allocate CAAM Context Register (%" PRId32 + " bytes)", + alg->size_ctx); + + /* Copy the IV into the context register */ + retstatus = copy_ctx_data(&cipherdata->ctx, &dinit->iv); + CIPHER_TRACE("Copy IV returned 0x%" PRIx32, retstatus); + + if (retstatus != CAAM_NO_ERROR) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + } + + if (alg->require_key & NEED_TWEAK) { + /* This is accepted to start with a NULL Tweak */ + if (dinit->iv.length) { + if (dinit->iv.length != alg->size_block) { + CIPHER_TRACE("Bad tweak 2 size"); + goto out; + } + + /* Copy the tweak */ + retstatus = copy_ctx_data(&cipherdata->tweak, + &dinit->iv); + CIPHER_TRACE("Copy Tweak returned 0x%" PRIx32, + retstatus); + + if (retstatus != CAAM_NO_ERROR) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + } else { + /* Create tweak 0's */ + if (!cipherdata->tweak.data) { + /* + * Allocate the destination buffer and + * fill it with 0's + */ + ret = caam_calloc_align_buf(&cipherdata->tweak, + alg->size_block); + if (ret != CAAM_NO_ERROR) + goto out; + } else { + /* Fill it with 0's */ + memset(cipherdata->tweak.data, 0, + cipherdata->tweak.length); + } + + /* Push data to physical memory */ + cache_operation(TEE_CACHEFLUSH, cipherdata->tweak.data, + cipherdata->tweak.length); + } + } + + /* Save the operation direction */ + cipherdata->encrypt = dinit->encrypt; + cipherdata->blockbuf.filled = 0; + + ret = TEE_SUCCESS; + +out: + /* Free the internal context in case of error */ + if (ret != TEE_SUCCESS) + do_free_intern(cipherdata); + + return ret; +} + +/* + * Update of the cipher operation in streaming mode, meaning + * doing partial intermediate block. + * If there is a context, the context is saved only when a + * full block is done. + * The partial block (if not the last block) is encrypted or + * decrypted to return the result and it's saved to be concatened + * to next data to rebuild a full block. + * + * @dupdate Data update object + */ +static TEE_Result do_update_streaming(struct drvcrypt_cipher_update *dupdate) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct cipherdata *ctx = dupdate->ctx; + struct caamdmaobj src = { }; + struct caamdmaobj dst = { }; + struct caamblock trash_bck = { }; + size_t fullsize = 0; + size_t size_topost = 0; + size_t size_todo = 0; + size_t size_inmade = 0; + size_t size_done = 0; + size_t offset = 0; + + CIPHER_TRACE("Length=%zu - %s", dupdate->src.length, + ctx->encrypt ? "Encrypt" : "Decrypt"); + + /* Calculate the total data to be handled */ + fullsize = ctx->blockbuf.filled + dupdate->src.length; + CIPHER_TRACE("Fullsize %zu", fullsize); + if (fullsize < ctx->alg->size_block) { + size_topost = dupdate->src.length; + goto end_streaming_post; + } else { + size_topost = fullsize % ctx->alg->size_block; + /* Total size that is a cipher block multiple */ + size_todo = fullsize - size_topost; + size_inmade = size_todo - ctx->blockbuf.filled; + } + + CIPHER_TRACE("FullSize %zu - posted %zu - todo %zu", fullsize, + size_topost, size_todo); + + if (size_todo) { + ret = caam_dmaobj_init_input(&src, dupdate->src.data, + dupdate->src.length); + if (ret) + goto end_streaming; + + ret = caam_dmaobj_init_output(&dst, dupdate->dst.data, + dupdate->dst.length, + dupdate->dst.length); + if (ret) + goto end_streaming; + + ret = caam_dmaobj_prepare(&src, &dst, ctx->alg->size_block); + if (ret) + goto end_streaming; + } + + /* + * Check first if there is some data saved to complete the + * buffer. + */ + if (ctx->blockbuf.filled) { + ret = caam_dmaobj_add_first_block(&src, &ctx->blockbuf); + if (ret) + goto end_streaming; + + ret = caam_dmaobj_add_first_block(&dst, &ctx->blockbuf); + if (ret) + goto end_streaming; + + ctx->blockbuf.filled = 0; + } + + size_done = size_todo; + dupdate->dst.length = 0; + for (offset = 0; size_todo; + offset += size_done, size_todo -= size_done) { + CIPHER_TRACE("Do input %zu bytes (%zu), offset %zu", size_done, + size_todo, offset); + + size_done = size_todo; + ret = caam_dmaobj_sgtbuf_inout_build(&src, &dst, &size_done, + offset, + ctx->alg->size_block); + if (ret) + goto end_streaming; + + retstatus = caam_cipher_block(ctx, true, NEED_KEY1, + ctx->encrypt, &src, &dst); + + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto end_streaming; + } + + dupdate->dst.length += caam_dmaobj_copy_to_orig(&dst); + } + + CIPHER_DUMPBUF("Source", dupdate->src.data, dupdate->src.length); + CIPHER_DUMPBUF("Result", dupdate->dst.data, dupdate->dst.length); + +end_streaming_post: + if (size_topost) { + /* + * Save the input data in the block buffer for next operation + * and prepare the source DMA Object with the overall saved + * data to generate destination bytes. + */ + struct caambuf cpysrc = { + .data = dupdate->src.data, + .length = dupdate->src.length + }; + + caam_dmaobj_free(&src); + caam_dmaobj_free(&dst); + CIPHER_TRACE("Save input data %zu bytes (done %zu) - off %zu", + size_topost, size_inmade, offset); + + size_todo = size_topost + ctx->blockbuf.filled; + + /* + * Prepare the destination DMA Object: + * - Use given destination parameter bytes to return + * - If the previous operation saved data, use a trash + * buffer to do the operation but not use unneeded data. + */ + ret = caam_dmaobj_init_output(&dst, + dupdate->dst.data + size_inmade, + size_topost, size_topost); + if (ret) + goto end_streaming; + + ret = caam_dmaobj_prepare(NULL, &dst, ctx->alg->size_block); + if (ret) + goto end_streaming; + + if (ctx->blockbuf.filled) { + /* + * Because there are some bytes to trash, use + * a block buffer that will be added to the + * destination SGT/Buffer structure to do the + * cipher operation. + */ + ret = caam_alloc_align_buf(&trash_bck.buf, + ctx->blockbuf.filled); + if (ret != CAAM_NO_ERROR) { + CIPHER_TRACE("Allocation Trash Block error"); + goto end_streaming; + } + trash_bck.filled = ctx->blockbuf.filled; + + ret = caam_dmaobj_add_first_block(&dst, &trash_bck); + if (ret) + goto end_streaming; + } + + retstatus = caam_cpy_block_src(&ctx->blockbuf, &cpysrc, + size_inmade); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto end_streaming; + } + + ret = caam_dmaobj_init_input(&src, ctx->blockbuf.buf.data, + ctx->blockbuf.filled); + if (ret) + goto end_streaming; + + ret = caam_dmaobj_prepare(&src, NULL, ctx->alg->size_block); + if (ret) + goto end_streaming; + + /* + * Build input and output DMA Object with the same size. + */ + size_done = size_todo; + ret = caam_dmaobj_sgtbuf_inout_build(&src, &dst, &size_done, 0, + size_todo); + if (ret) + goto end_streaming; + + if (size_todo != size_done) { + CIPHER_TRACE("Invalid end streaming size %zu vs %zu", + size_done, size_todo); + ret = TEE_ERROR_GENERIC; + goto end_streaming; + } + + retstatus = caam_cipher_block(ctx, false, NEED_KEY1, + ctx->encrypt, &src, &dst); + + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto end_streaming; + } + + dupdate->dst.length += caam_dmaobj_copy_to_orig(&dst); + + CIPHER_DUMPBUF("Source", ctx->blockbuf.buf.data, + ctx->blockbuf.filled); + CIPHER_DUMPBUF("Result", dupdate->dst.data + size_inmade, + size_topost); + } + + ret = TEE_SUCCESS; + +end_streaming: + caam_dmaobj_free(&src); + caam_dmaobj_free(&dst); + + /* Free Trash block buffer */ + caam_free_buf(&trash_bck.buf); + + return ret; +} + +/* + * Update of the cipher operation with complete block except + * if last block. Last block can be partial block. + * + * @dupdate Data update object + */ +static TEE_Result do_update_cipher(struct drvcrypt_cipher_update *dupdate) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct cipherdata *ctx = dupdate->ctx; + struct caamdmaobj src = { }; + struct caamdmaobj dst = { }; + size_t offset = 0; + size_t size_todo = 0; + size_t size_done = 0; + + CIPHER_TRACE("Length=%zu - %s", dupdate->src.length, + (ctx->encrypt ? "Encrypt" : "Decrypt")); + + /* + * Check the length of the payload/cipher to be at least + * one or n cipher block. + */ + if (dupdate->src.length < ctx->alg->size_block || + dupdate->src.length % ctx->alg->size_block) { + CIPHER_TRACE("Bad payload/cipher size %zu bytes", + dupdate->src.length); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = caam_dmaobj_init_input(&src, dupdate->src.data, + dupdate->src.length); + if (ret) + goto end_cipher; + + ret = caam_dmaobj_init_output(&dst, dupdate->dst.data, + dupdate->dst.length, dupdate->dst.length); + if (ret) + goto end_cipher; + + ret = caam_dmaobj_prepare(&src, &dst, ctx->alg->size_block); + if (ret) + goto end_cipher; + + size_todo = dupdate->src.length; + dupdate->dst.length = 0; + for (offset = 0; size_todo; + offset += size_done, size_todo -= size_done) { + size_done = size_todo; + CIPHER_TRACE("Do input %zu bytes, offset %zu", size_done, + offset); + ret = caam_dmaobj_sgtbuf_inout_build(&src, &dst, &size_done, + offset, + ctx->alg->size_block); + if (ret) + goto end_cipher; + + retstatus = caam_cipher_block(ctx, true, NEED_KEY1, + ctx->encrypt, &src, &dst); + + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto end_cipher; + } + + dupdate->dst.length += caam_dmaobj_copy_to_orig(&dst); + } + + ret = TEE_SUCCESS; + +end_cipher: + caam_dmaobj_free(&src); + caam_dmaobj_free(&dst); + + return ret; +} + +/* + * Update of the cipher operation. Call the algorithm update + * function associated. + * + * @dupdate Data update object + */ +static TEE_Result do_update(struct drvcrypt_cipher_update *dupdate) +{ + struct cipherdata *cipherdata = dupdate->ctx; + + return cipherdata->alg->update(dupdate); +} + +/* + * Finalize of the cipher operation + * + * @ctx Caller context variable or NULL + */ +static void do_final(void *ctx __unused) +{ +} + +/* + * Registration of the Cipher Driver + */ +static struct drvcrypt_cipher driver_cipher = { + .alloc_ctx = do_allocate, + .free_ctx = caam_cipher_free, + .init = caam_cipher_initialize, + .update = do_update, + .final = do_final, + .copy_state = caam_cipher_copy_state, +}; + +/* + * Initialize the Cipher module + * + * @ctrl_addr Controller base address + */ +enum caam_status caam_cipher_init(vaddr_t ctrl_addr __unused) +{ + enum caam_status retstatus = CAAM_FAILURE; + + if (drvcrypt_register_cipher(&driver_cipher) == TEE_SUCCESS) + retstatus = CAAM_NO_ERROR; + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/cipher/caam_cipher_mac.c b/optee_os/core/drivers/crypto/caam/cipher/caam_cipher_mac.c new file mode 100644 index 0000000..f73a6f1 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/cipher/caam_cipher_mac.c @@ -0,0 +1,825 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Implementation of CMAC functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +static TEE_Result do_update_mac(struct drvcrypt_cipher_update *dupdate); +static TEE_Result do_update_cmac(struct drvcrypt_cipher_update *dupdate); + +/* + * Constant definitions of AES MAC algorithms + */ +static const struct cipheralg aes_cbc_mac_alg = { + .type = OP_ALGO(AES) | ALGO_AAI(AES_CBC), + .size_block = TEE_AES_BLOCK_SIZE, + .size_ctx = 2 * sizeof(uint64_t), + .ctx_offset = 0, + .require_key = NEED_KEY1 | NEED_IV, + .def_key = { + .min = 16, + .max = 32, + .mod = 8 + }, + .update = do_update_mac, +}; + +static const struct cipheralg aes_cmac_alg = { + .type = OP_ALGO(AES) | ALGO_AAI(AES_CMAC), + .size_block = TEE_AES_BLOCK_SIZE, + .size_ctx = 4 * sizeof(uint64_t), + .ctx_offset = 0, + .require_key = NEED_KEY1, + .def_key = { + .min = 16, + .max = 32, + .mod = 8 + }, + .update = do_update_cmac, +}; + +/* + * Constant definitions of DES MAC algorithm + */ +static const struct cipheralg des_mac_alg = { + .type = OP_ALGO(DES) | ALGO_AAI(DES_CBC), + .size_block = TEE_DES_BLOCK_SIZE, + .size_ctx = sizeof(uint64_t), + .ctx_offset = 0, + .require_key = NEED_KEY1 | NEED_IV, + .def_key = { + .min = 8, + .max = 8, + .mod = 8 + }, + .update = do_update_mac, +}; + +/* + * Constant definitions of DES3 MAC algorithm + */ +static const struct cipheralg des3_mac_alg = { + .type = OP_ALGO(3DES) | ALGO_AAI(DES_CBC), + .size_block = TEE_DES_BLOCK_SIZE, + .size_ctx = sizeof(uint64_t), + .ctx_offset = 0, + .require_key = NEED_KEY1 | NEED_IV, + .def_key = { + .min = 16, + .max = 24, + .mod = 8 + }, + .update = do_update_mac, +}; + +static const struct crypto_mac_ops cmac_ops; + +/* + * Format the MAC context to keep the reference to the operation driver + */ +struct crypto_mac { + struct crypto_mac_ctx mac_ctx; /* Crypto MAC API context */ + struct cipherdata *ctx; /* CMAC context */ +}; + +/* + * Returns the reference to the driver context + * + * @ctx API context + */ +static struct crypto_mac *to_mac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx && ctx->ops == &cmac_ops); + + return container_of(ctx, struct crypto_mac, mac_ctx); +} + +/* + * Checks if the algorithm @algo is supported and returns the + * local algorithm entry in the corresponding cipher array. + * + * @algo Algorithm ID + */ +static const struct cipheralg *get_macalgo(uint32_t algo) +{ + switch (algo) { + case TEE_ALG_AES_CBC_MAC_NOPAD: + case TEE_ALG_AES_CBC_MAC_PKCS5: + return &aes_cbc_mac_alg; + case TEE_ALG_AES_CMAC: + return &aes_cmac_alg; + case TEE_ALG_DES_CBC_MAC_NOPAD: + case TEE_ALG_DES_CBC_MAC_PKCS5: + return &des_mac_alg; + case TEE_ALG_DES3_CBC_MAC_NOPAD: + case TEE_ALG_DES3_CBC_MAC_PKCS5: + return &des3_mac_alg; + default: + return NULL; + } +} + +/* + * MAC update of the cipher operation of complete block except + * if last block. Last block can be partial block. + * + * @dupdate Data update object + */ +static TEE_Result do_update_mac(struct drvcrypt_cipher_update *dupdate) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + enum caam_status retstatus = CAAM_FAILURE; + struct cipherdata *ctx = dupdate->ctx; + struct caamdmaobj src = { }; + struct caamdmaobj dst = { }; + size_t full_size = 0; + size_t size_topost = 0; + size_t size_todo = 0; + size_t size_done = 0; + size_t size_inmade = 0; + size_t offset = 0; + + CIPHER_TRACE("Length=%zu - %s", dupdate->src.length, + ctx->encrypt ? "Encrypt" : "Decrypt"); + + /* Calculate the total data to be handled */ + full_size = ctx->blockbuf.filled + dupdate->src.length; + if (full_size < ctx->alg->size_block) { + size_topost = dupdate->src.length; + } else { + size_topost = full_size % ctx->alg->size_block; + size_inmade = dupdate->src.length - size_topost; + /* Total size that is a cipher block multiple */ + size_todo = full_size - size_topost; + } + + CIPHER_TRACE("FullSize %zu - posted %zu - todo %zu", full_size, + size_topost, size_todo); + + if (!size_todo) { + /* + * There is no complete block to do: + * - either input size + already saved data < block size + * - or no input data and this is the last block + */ + if (dupdate->last) + memcpy(dupdate->dst.data, ctx->ctx.data, + MIN(dupdate->dst.length, ctx->alg->size_ctx)); + + ret = TEE_SUCCESS; + goto end_mac_post; + } + + if (dupdate->src.length) { + ret = caam_dmaobj_init_input(&src, dupdate->src.data, + dupdate->src.length); + if (ret) + goto end_mac; + + ret = caam_dmaobj_prepare(&src, NULL, ctx->alg->size_block); + if (ret) + goto end_mac; + } + + if (dupdate->last) { + ret = caam_dmaobj_output_sgtbuf(&dst, dupdate->dst.data, + dupdate->dst.length, + dupdate->dst.length); + if (ret) + goto end_mac; + + /* Remove a block of data to do the last block */ + if (size_todo > ctx->alg->size_block) + size_todo -= ctx->alg->size_block; + else + size_todo = 0; + } + + /* Check if there is some data saved to complete the buffer */ + if (ctx->blockbuf.filled) { + ret = caam_dmaobj_add_first_block(&src, &ctx->blockbuf); + if (ret) + goto end_mac; + ctx->blockbuf.filled = 0; + } + + size_done = ctx->alg->size_block; + for (offset = 0; size_todo; + offset += size_done, size_todo -= size_done) { + CIPHER_TRACE("Do input %zu bytes, offset %zu", size_done, + offset); + + ret = caam_dmaobj_sgtbuf_build(&src, &size_done, offset, + ctx->alg->size_block); + if (ret) + goto end_mac; + + if (size_done != ctx->alg->size_block) { + ret = TEE_ERROR_GENERIC; + goto end_mac; + } + + retstatus = caam_cipher_block(ctx, true, NEED_KEY1, true, &src, + NULL); + + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto end_mac; + } + } + + if (dupdate->last) { + CIPHER_TRACE("Do input %zu bytes, offset %zu", size_done, + offset); + + ret = caam_dmaobj_sgtbuf_build(&src, &size_done, offset, + ctx->alg->size_block); + if (ret) + goto end_mac; + + if (size_done != ctx->alg->size_block) { + ret = TEE_ERROR_GENERIC; + goto end_mac; + } + + retstatus = caam_cipher_block(ctx, true, NEED_KEY1, true, &src, + &dst); + + if (retstatus == CAAM_NO_ERROR) + caam_dmaobj_copy_to_orig(&dst); + + ret = caam_status_to_tee_result(retstatus); + } + +end_mac_post: + if (size_topost) { + struct caambuf cpysrc = { + .data = dupdate->src.data, + .length = dupdate->src.length + }; + + CIPHER_TRACE("Save input data %zu bytes of %zu (%zu)", + size_topost, dupdate->src.length, size_inmade); + + retstatus = caam_cpy_block_src(&ctx->blockbuf, &cpysrc, + size_inmade); + ret = caam_status_to_tee_result(retstatus); + } + +end_mac: + caam_dmaobj_free(&src); + caam_dmaobj_free(&dst); + + return ret; +} + +/* + * Build and run the CMAC descriptor (AES only) + * + * @ctx Cipher Data context + * @src Input data + * @dstbuf [out] Output data if last block + * @last Last block flag + */ +static TEE_Result run_cmac_desc(struct cipherdata *ctx, struct caamdmaobj *src, + struct caamdmaobj *dst, bool last) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + + desc = ctx->descriptor; + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + if (ctx->alg->require_key & NEED_KEY1) { + /* Build the descriptor */ + caam_desc_add_word(desc, LD_KEY_PLAIN(CLASS_1, REG, + ctx->key1.length)); + caam_desc_add_ptr(desc, ctx->key1.paddr); + } + + /* If context already allocated, this is an update */ + if (ctx->ctx.length) { + CIPHER_TRACE("%s operation", last ? "Final" : "Update"); + caam_desc_add_word(desc, LD_NOIMM_OFF(CLASS_1, REG_CTX, + ctx->ctx.length, + ctx->alg->ctx_offset)); + caam_desc_add_ptr(desc, ctx->ctx.paddr); + if (last) + caam_desc_add_word(desc, + CIPHER_FINAL(ctx->alg->type, true)); + else + caam_desc_add_word(desc, + CIPHER_UPDATE(ctx->alg->type, true)); + } else if (last) { + CIPHER_TRACE("Init/Final operation"); + + caam_desc_add_word(desc, + CIPHER_INITFINAL(ctx->alg->type, true)); + } else { + CIPHER_TRACE("Init operation"); + + caam_desc_add_word(desc, CIPHER_INIT(ctx->alg->type, true)); + if (!ctx->ctx.data) { + retstatus = caam_alloc_align_buf(&ctx->ctx, + ctx->alg->size_ctx); + if (retstatus != CAAM_NO_ERROR) + return TEE_ERROR_OUT_OF_MEMORY; + } + } + + /* Check first if there is some pending data from previous updates */ + if (ctx->blockbuf.filled) { + /* Add the temporary buffer */ + if (src) + caam_desc_add_word(desc, + FIFO_LD_EXT(CLASS_1, MSG, NOACTION)); + else + caam_desc_add_word(desc, + FIFO_LD_EXT(CLASS_1, MSG, LAST_C1)); + + caam_desc_add_ptr(desc, ctx->blockbuf.buf.paddr); + caam_desc_add_word(desc, ctx->blockbuf.filled); + + /* Clean the circular buffer data to be loaded */ + cache_operation(TEE_CACHECLEAN, ctx->blockbuf.buf.data, + ctx->blockbuf.filled); + } + + if (src) { + caam_desc_fifo_load(desc, src, CLASS_1, MSG, LAST_C1); + caam_dmaobj_cache_push(src); + } else { + if (last && !ctx->blockbuf.filled) { + /* + * Add the input data of 0 bytes to start + * algorithm by setting the input data size + */ + caam_desc_add_word(desc, + FIFO_LD(CLASS_1, MSG, LAST_C1, 0)); + caam_desc_add_ptr(desc, 0); + } + } + + ctx->blockbuf.filled = 0; + + if (last) { + caam_desc_store(desc, dst, CLASS_1, REG_CTX); + caam_dmaobj_cache_push(dst); + } else { + /* Store the context */ + caam_desc_add_word(desc, ST_NOIMM_OFF(CLASS_1, REG_CTX, + ctx->ctx.length, + ctx->alg->ctx_offset)); + caam_desc_add_ptr(desc, ctx->ctx.paddr); + } + + CIPHER_DUMPDESC(desc); + + /* Invalidate Context register */ + if (ctx->ctx.length) + cache_operation(TEE_CACHEINVALIDATE, ctx->ctx.data, + ctx->ctx.length); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + ret = TEE_SUCCESS; + } else { + CIPHER_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + + return ret; +} + +/* + * Update of the CMAC operation of complete block except + * if last block. Last block can be a partial block. + * + * @dupdate Data update object + */ +static TEE_Result do_update_cmac(struct drvcrypt_cipher_update *dupdate) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + enum caam_status retstatus = CAAM_FAILURE; + struct cipherdata *ctx = dupdate->ctx; + size_t full_size = 0; + size_t size_topost = 0; + size_t size_todo = 0; + size_t size_inmade = 0; + size_t size_done = 0; + size_t offset = 0; + struct caamdmaobj src = { }; + struct caamdmaobj dst = { }; + + CIPHER_TRACE("Length=%zu - %s", dupdate->src.length, + dupdate->encrypt ? "Encrypt" : "Decrypt"); + + /* Calculate the total data to be handled */ + full_size = ctx->blockbuf.filled + dupdate->src.length; + if (!dupdate->last) { + /* + * In case there is no data to save and because it's + * not the final operation, ensure that a block of data + * is kept for the final operation. + */ + if (full_size <= ctx->alg->size_block) { + size_topost = dupdate->src.length; + goto end_cmac_post; + } + + size_topost = full_size % ctx->alg->size_block; + + if (!size_topost) + size_topost = ctx->alg->size_block; + + size_inmade = dupdate->src.length - size_topost; + size_todo = full_size - size_topost; + } else { + ret = caam_dmaobj_output_sgtbuf(&dst, dupdate->dst.data, + dupdate->dst.length, + dupdate->dst.length); + if (ret) + goto end_cmac; + + /* + * If there more than one block to do, keep the last + * block to build the CMAC output. + */ + if (full_size > ctx->alg->size_block) { + size_todo = full_size - ctx->alg->size_block; + size_inmade = size_todo - ctx->blockbuf.filled; + } + } + + if (size_inmade) { + ret = caam_dmaobj_init_input(&src, dupdate->src.data, + size_inmade); + if (ret) + goto end_cmac; + + ret = caam_dmaobj_prepare(&src, NULL, ctx->alg->size_block); + if (ret) + goto end_cmac; + } + + CIPHER_TRACE("FullSize %zu - posted %zu - todo %zu", full_size, + size_topost, size_todo); + + for (offset = 0; size_todo; + offset += size_done, size_todo -= size_done) { + /* + * At least one block is to be done. + * At first iteration, we can have less than one block + * data available from previous update operation which + * was not block modulus. + * Remove the previous saved data (blockbuf) from the data to + * take from input data. + * Next iteration, blockbuf will be empty. + */ + size_todo -= ctx->blockbuf.filled; + size_done = size_todo; + + if (size_inmade) { + ret = caam_dmaobj_sgtbuf_build(&src, &size_done, offset, + ctx->alg->size_block); + if (ret) + goto end_cmac; + + /* + * Need to re-adjust the length of the data if the + * posted data block is not empty and the SGT/Buffer + * is part of the full input data to do. + */ + if (ctx->blockbuf.filled && size_done < size_todo) { + size_done -= ctx->blockbuf.filled; + src.sgtbuf.length = size_done; + } + CIPHER_TRACE("Do input %zu bytes, offset %zu", + size_done, offset); + + ret = run_cmac_desc(ctx, &src, NULL, false); + } else { + CIPHER_TRACE("Do saved blockbuf %zu bytes (done = %zu)", + ctx->blockbuf.filled, size_done); + ret = run_cmac_desc(ctx, NULL, NULL, false); + } + + if (ret) + goto end_cmac; + } + + if (dupdate->last) { + if (dupdate->src.length - size_inmade) { + size_done = dupdate->src.length - size_inmade; + ret = caam_dmaobj_sgtbuf_build(&src, &size_done, offset, + ctx->alg->size_block); + if (ret) + goto end_cmac; + + if (size_done != dupdate->src.length - size_inmade) { + ret = TEE_ERROR_GENERIC; + goto end_cmac; + } + + ret = run_cmac_desc(ctx, &src, &dst, true); + } else { + ret = run_cmac_desc(ctx, NULL, &dst, true); + } + + if (!ret) + caam_dmaobj_copy_to_orig(&dst); + } + +end_cmac_post: + if (size_topost) { + struct caambuf srcbuf = { .data = dupdate->src.data, + .length = dupdate->src.length }; + + CIPHER_TRACE("Post %zu of input len %zu made %zu", size_topost, + srcbuf.length, size_inmade); + + retstatus = caam_cpy_block_src(&ctx->blockbuf, &srcbuf, + size_inmade); + ret = caam_status_to_tee_result(retstatus); + } + +end_cmac: + caam_dmaobj_free(&src); + caam_dmaobj_free(&dst); + + return ret; +} + +/* + * Initialization of the CMAC operation. + * + * @ctx Operation software context + * @key Input key to compute + * @len Key length + */ +static TEE_Result do_cmac_init(struct crypto_mac_ctx *ctx, const uint8_t *key, + size_t len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + uint8_t *iv_tmp = NULL; + struct drvcrypt_cipher_init dinit = { }; + struct crypto_mac *mac = to_mac_ctx(ctx); + struct cipherdata *macdata = mac->ctx; + + if (macdata->mode != TEE_CHAIN_MODE_CMAC) { + /* Allocate temporary IV initialize with 0's */ + iv_tmp = caam_calloc(macdata->alg->size_ctx); + if (!iv_tmp) + return TEE_ERROR_OUT_OF_MEMORY; + } else { + /* + * Check if the context register is allocated to free it, + * because in case of CMAC mode, the context register + * is allocated during do_update_cmac() operation if + * necessary. + */ + if (macdata->ctx.data) + caam_free_buf(&macdata->ctx); + } + + macdata->countdata = 0; + + /* Prepare the initialization data */ + dinit.ctx = macdata; + dinit.encrypt = true; + dinit.key1.data = (uint8_t *)key; + dinit.key1.length = len; + dinit.key2.data = NULL; + dinit.key2.length = 0; + dinit.iv.data = iv_tmp; + dinit.iv.length = macdata->alg->size_ctx; + ret = caam_cipher_initialize(&dinit); + + caam_free(iv_tmp); + + return ret; +} + +/* + * Update of the CMAC operation. + * + * @ctx Operation software context + * @data Data to encrypt + * @len Data length + */ +static TEE_Result do_cmac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct crypto_mac *mac = to_mac_ctx(ctx); + struct cipherdata *macdata = mac->ctx; + struct drvcrypt_cipher_update dupdate = { }; + + /* Prepare the update data */ + dupdate.ctx = macdata; + dupdate.encrypt = true; + dupdate.last = false; + dupdate.src.data = (uint8_t *)data; + dupdate.src.length = len; + dupdate.dst.data = NULL; + dupdate.dst.length = 0; + + ret = macdata->alg->update(&dupdate); + + if (!ret && macdata->mode == TEE_CHAIN_MODE_CBC_MAC_PKCS5) + macdata->countdata += len; + + return ret; +} + +/* + * Finalize the CMAC operation + * + * @ctx Operation software context + * @digest [out] Digest buffer + * @len Digest buffer length + */ +static TEE_Result do_cmac_final(struct crypto_mac_ctx *ctx, uint8_t *digest, + size_t len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + uint8_t *pad_src = NULL; + size_t pad_size = 0; + struct crypto_mac *mac = to_mac_ctx(ctx); + struct cipherdata *macdata = mac->ctx; + struct drvcrypt_cipher_update dupdate = { }; + + if (macdata->mode == TEE_CHAIN_MODE_CBC_MAC_PKCS5) { + /* Calculate the last block PAD size */ + pad_size = macdata->alg->size_block - + (macdata->countdata % macdata->alg->size_block); + CIPHER_TRACE("Pad size = %zu", pad_size); + + if (pad_size) { + /* Need to pad the last block */ + pad_src = caam_calloc(pad_size); + if (!pad_src) { + CIPHER_TRACE("Pad src allocation error"); + return TEE_ERROR_OUT_OF_MEMORY; + } + + memset(pad_src, pad_size, pad_size); + } + } + + /* Prepare the update data */ + dupdate.ctx = macdata; + dupdate.encrypt = true; + dupdate.last = true; + dupdate.src.data = pad_src; + dupdate.src.length = pad_size; + dupdate.dst.data = digest; + dupdate.dst.length = MIN(len, macdata->alg->size_block); + + ret = macdata->alg->update(&dupdate); + + caam_free(pad_src); + + return ret; +} + +/* + * Free the software context + * + * @ctx [in/out] Caller context variable + */ +static void do_cmac_free(struct crypto_mac_ctx *ctx) +{ + struct crypto_mac *mac = to_mac_ctx(ctx); + + caam_cipher_free(mac->ctx); + free(mac); +} + +/* + * Copy software CMAC context + * + * @dst_ctx [out] Reference the context destination + * @src_ctx Reference the context source + */ +static void do_cmac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct crypto_mac *mac_src = to_mac_ctx(src_ctx); + struct crypto_mac *mac_dst = to_mac_ctx(dst_ctx); + struct cipherdata *macdata_dst = mac_dst->ctx; + struct cipherdata *macdata_src = mac_src->ctx; + + caam_cipher_copy_state(macdata_dst, macdata_src); + + macdata_dst->countdata = macdata_src->countdata; + macdata_dst->mode = macdata_src->mode; +} + +/* + * Registration of the CMAC driver + */ +static const struct crypto_mac_ops cmac_ops = { + .init = do_cmac_init, + .update = do_cmac_update, + .final = do_cmac_final, + .free_ctx = do_cmac_free, + .copy_state = do_cmac_copy_state, +}; + +/* + * Allocate the software context + * + * @ctx [out] Caller context variable + * @algo Algorithm ID + */ +static TEE_Result caam_cmac_allocate(struct crypto_mac_ctx **ctx, uint32_t algo) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_mac *mac = NULL; + const struct cipheralg *alg = NULL; + struct cipherdata *macdata = NULL; + + CIPHER_TRACE("Allocate Context (%p) algo %" PRIx32, ctx, algo); + + alg = get_macalgo(algo); + if (!alg) { + CIPHER_TRACE("Algorithm not supported"); + return TEE_ERROR_NOT_IMPLEMENTED; + } + + mac = calloc(1, sizeof(*mac)); + if (!mac) + return TEE_ERROR_OUT_OF_MEMORY; + + macdata = caam_calloc(sizeof(*macdata)); + if (!macdata) { + CIPHER_TRACE("Allocation MAC data error"); + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + /* Allocate the descriptor */ + macdata->descriptor = caam_calloc_desc(MAX_DESC_ENTRIES); + if (!macdata->descriptor) { + CIPHER_TRACE("Allocation descriptor error"); + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + /* Setup the algorithm pointer */ + macdata->alg = alg; + + /* Initialize the block buffer */ + macdata->blockbuf.max = alg->size_block; + + /* Keep the MAC mode */ + macdata->mode = TEE_ALG_GET_CHAIN_MODE(algo); + + mac->mac_ctx.ops = &cmac_ops; + mac->ctx = macdata; + + *ctx = &mac->mac_ctx; + + return TEE_SUCCESS; + +err: + if (macdata) + caam_free_desc(&macdata->descriptor); + + caam_free(macdata); + free(mac); + + return ret; +} + +/* + * Initialize the CMAC module + * + * @ctrl_addr Controller base address + */ +enum caam_status caam_cmac_init(vaddr_t ctrl_addr __unused) +{ + if (drvcrypt_register(CRYPTO_CMAC, &caam_cmac_allocate)) + return CAAM_FAILURE; + + return CAAM_NO_ERROR; +} diff --git a/optee_os/core/drivers/crypto/caam/cipher/caam_cipher_xts.c b/optee_os/core/drivers/crypto/caam/cipher/caam_cipher_xts.c new file mode 100644 index 0000000..e3c1bde --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/cipher/caam_cipher_xts.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Implementation of Cipher XTS functions + */ +#include +#include +#include +#include +#include + +#include "local.h" + +/* + * Galois Multiplication + * + * @buf [in/out] buffer to multiply + */ +static void do_galois_mult(struct caambuf *buf) +{ + size_t idx = 0; + uint8_t tmp = 0; + uint8_t tmptmp = 0; + + for (idx = 0; idx < buf->length; idx++) { + tmptmp = buf->data[idx] >> 7; + buf->data[idx] = (buf->data[idx] << 1) | tmp; + tmp = tmptmp; + } + + if (tmptmp) + buf->data[0] ^= 0x87; +} + +/* + * Tweak a cipher block (XTS mode) + * + * @ctx Cipher context + * @enc_tweak [in/out] Encrypted tweak (Galois multiplication) + * @srcbuf Source data to encrypt/decrypt + * @dstbuf [out] Destination data encrypted/decrypted + * @tmp Temporary data buffer + */ +static TEE_Result do_tweak_block(struct cipherdata *ctx, + struct caambuf *enc_tweak, + struct caambuf *srcbuf, struct caambuf *dstbuf, + struct caamdmaobj *tmp) +{ + enum caam_status retstatus = CAAM_FAILURE; + unsigned int idx = 0; + + /* + * TODO: Optimization by using CAAM to do it with MATH op in the + * operation description + */ + for (idx = 0; idx < ctx->alg->size_block; idx++) + tmp->orig.data[idx] = srcbuf->data[idx] ^ enc_tweak->data[idx]; + + retstatus = caam_cipher_block(ctx, false, NEED_KEY1, ctx->encrypt, tmp, + tmp); + + if (retstatus != CAAM_NO_ERROR) + return caam_status_to_tee_result(retstatus); + + caam_dmaobj_copy_to_orig(tmp); + + for (idx = 0; idx < ctx->alg->size_block; idx++) + dstbuf->data[idx] = tmp->orig.data[idx] ^ enc_tweak->data[idx]; + + /* Galois field multiplication of the tweak */ + do_galois_mult(enc_tweak); + + return TEE_SUCCESS; +} + +TEE_Result caam_cipher_update_xts(struct drvcrypt_cipher_update *dupdate) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct cipherdata *ctx = dupdate->ctx; + struct caambuf tmpsrc = { }; + struct caamdmaobj tmpdst = { }; + struct caamdmaobj tweak = { }; + struct caamdmaobj enc_tweak = { }; + struct caambuf srcbuf = { }; + struct caambuf dstbuf = { }; + size_t idx = 0; + size_t fullsize = 0; + size_t lastblk = 0; + paddr_t psrc = 0; + paddr_t pdst = 0; + + CIPHER_TRACE("Algo AES XTS length=%zu - %s", dupdate->src.length, + ctx->encrypt ? "Encrypt" : " Decrypt"); + + psrc = virt_to_phys(dupdate->src.data); + pdst = virt_to_phys(dupdate->dst.data); + + /* Check the payload/cipher physical addresses */ + if (!psrc || !pdst) { + CIPHER_TRACE("Bad Addr (src %#" PRIxPA ") (dst %#" PRIxPA ")", + psrc, pdst); + return TEE_ERROR_GENERIC; + } + + ret = caam_dmaobj_input_sgtbuf(&tweak, ctx->tweak.data, + ctx->tweak.length); + if (ret) + goto out; + + /* + * First operation is to encrypt the tweak with the key #2 + * Allocate the encrypted tweak buffer + */ + ret = caam_dmaobj_output_sgtbuf(&enc_tweak, NULL, 0, ctx->tweak.length); + if (ret) + goto out; + + ret = caam_dmaobj_output_sgtbuf(&tmpdst, NULL, 0, ctx->alg->size_block); + if (ret) + goto out; + + retstatus = caam_cipher_block(ctx, false, NEED_KEY2, true, &tweak, + &enc_tweak); + if (retstatus != CAAM_NO_ERROR) { + CIPHER_TRACE("Tweak encryption error"); + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + caam_dmaobj_copy_to_orig(&enc_tweak); + + /* + * Encrypt or Decrypt input data. + * Check if the last block is partial or not + * - if last block is partial, rebuild a complete + * block using the penultimate complete block + * encryption/decryption. + * - else do all blocks. + */ + + /* Calculate the number of complete block */ + fullsize = dupdate->src.length; + lastblk = fullsize % ctx->alg->size_block; + fullsize -= lastblk; + + /* One full block is needed */ + if (!fullsize) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (lastblk) + fullsize -= ctx->alg->size_block; + + srcbuf.data = dupdate->src.data; + srcbuf.length = ctx->alg->size_block; + srcbuf.paddr = psrc; + + dstbuf.data = dupdate->dst.data; + dstbuf.length = ctx->alg->size_block; + dstbuf.paddr = pdst; + + for (; fullsize > 0; fullsize -= ctx->alg->size_block) { + CIPHER_TRACE("Tweak block fullsize %zu", fullsize); + ret = do_tweak_block(ctx, &enc_tweak.orig, &srcbuf, &dstbuf, + &tmpdst); + + CIPHER_TRACE("Tweak block ret 0x%" PRIx32, ret); + if (ret) + goto out; + + CIPHER_DUMPBUF("Source", srcbuf.data, srcbuf.length); + CIPHER_DUMPBUF("Dest", dstbuf.data, dstbuf.length); + + /* Increment the source and destination block */ + srcbuf.data += ctx->alg->size_block; + srcbuf.paddr += ctx->alg->size_block; + + dstbuf.data += ctx->alg->size_block; + dstbuf.paddr += ctx->alg->size_block; + } + + if (lastblk) { + CIPHER_TRACE("Last block size is %zu", lastblk); + + /* + * Allocate the temporary buffer containing the + * penultimate block computed + */ + retstatus = caam_alloc_align_buf(&tmpsrc, ctx->alg->size_block); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + if (!ctx->encrypt) { + /* + * In case of decryption, need to multiply + * the tweak first + */ + memcpy(tmpsrc.data, enc_tweak.orig.data, + enc_tweak.orig.length); + do_galois_mult(&tmpsrc); + + ret = do_tweak_block(ctx, &tmpsrc, &srcbuf, + &tmpdst.orig, &tmpdst); + } else { + ret = do_tweak_block(ctx, &enc_tweak.orig, &srcbuf, + &tmpdst.orig, &tmpdst); + } + + CIPHER_TRACE("Tweak penultimate block ret 0x%" PRIx32, ret); + + if (ret) + goto out; + + /* Build the last block and create the last destination block */ + for (idx = 0; idx < lastblk; idx++) { + tmpsrc.data[idx] = + srcbuf.data[ctx->alg->size_block + idx]; + dstbuf.data[ctx->alg->size_block + idx] = + tmpdst.orig.data[idx]; + } + + for (; idx < ctx->alg->size_block; idx++) + tmpsrc.data[idx] = tmpdst.orig.data[idx]; + + ret = do_tweak_block(ctx, &enc_tweak.orig, &tmpsrc, &dstbuf, + &tmpdst); + + CIPHER_TRACE("Tweak last block ret 0x%" PRIx32, ret); + if (ret) + goto out; + + CIPHER_DUMPBUF("Source", tmpsrc.data, tmpsrc.length); + CIPHER_DUMPBUF("Dest", dstbuf.data, dstbuf.length); + } + + /* Finalize by decrypting the tweak back */ + retstatus = caam_cipher_block(ctx, false, NEED_KEY2, false, &enc_tweak, + &tweak); + if (retstatus != CAAM_NO_ERROR) { + CIPHER_TRACE("Tweak decryption error"); + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + caam_dmaobj_copy_to_orig(&tweak); + + ret = TEE_SUCCESS; +out: + caam_free_buf(&tmpsrc); + caam_dmaobj_free(&tmpdst); + caam_dmaobj_free(&tweak); + caam_dmaobj_free(&enc_tweak); + + return ret; +} diff --git a/optee_os/core/drivers/crypto/caam/cipher/local.h b/optee_os/core/drivers/crypto/caam/cipher/local.h new file mode 100644 index 0000000..ac0965d --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/cipher/local.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * CAAM Cipher Local header. + */ +#ifndef __LOCAL_H__ +#define __LOCAL_H__ + +#include +#include +#include + +/* + * Definition of the maximum number of CAAM Job descriptor entries + */ +#ifdef CFG_CAAM_64BIT +#define MAX_DESC_ENTRIES 22 +#else +#define MAX_DESC_ENTRIES 16 +#endif + +/* + * Definition of flags tagging which key(s) is required + */ +#define NEED_KEY1 BIT(0) +#define NEED_KEY2 BIT(1) +#define NEED_IV BIT(2) +#define NEED_TWEAK BIT(3) + +/* + * Cipher Algorithm definition + */ +struct cipheralg { + uint32_t type; /* Algo type for operation */ + uint8_t size_block; /* Computing block size */ + uint8_t size_ctx; /* CAAM Context Register size */ + uint8_t ctx_offset; /* CAAM Context Register offset */ + uint8_t require_key; /* Tag defining key(s) required */ + struct caamdefkey def_key; /* Key size accepted */ + + TEE_Result (*update)(struct drvcrypt_cipher_update *dupdate); +}; + +/* + * Full Cipher data SW context + */ +struct cipherdata { + uint32_t *descriptor; /* Job descriptor */ + bool encrypt; /* Encrypt direction */ + struct caambuf key1; /* First Key */ + struct caambuf key2; /* Second Key */ + struct caambuf tweak; /* XTS Tweak */ + struct caambuf ctx; /* CAAM Context Register */ + struct caamblock blockbuf; /* Temporary Block buffer */ + const struct cipheralg *alg; /* Reference to the algo constants */ + + /* Additionnal Data for the MAC */ + unsigned int mode; /* MAC TEE_CHAIN_MODE* */ + size_t countdata; /* MAC Number of input data */ +}; + +/* + * Cipher additionnal data block + */ +enum caam_cipher_block { + CIPHER_BLOCK_NONE = 0, + CIPHER_BLOCK_IN, + CIPHER_BLOCK_OUT, + CIPHER_BLOCK_BOTH, +}; + +/* + * Update of the cipher operation of complete block except + * if last block. Last block can be partial block. + * + * @ctx Cipher context + * @savectx Save or not the context + * @keyid Id of the key to be used during operation + * @encrypt Encrypt or decrypt direction + * @src Source data to encrypt/decrypt + * @dst [out] Destination data encrypted/decrypted + */ +enum caam_status caam_cipher_block(struct cipherdata *ctx, bool savectx, + uint8_t keyid, bool encrypt, + struct caamdmaobj *src, + struct caamdmaobj *dst); + +/* + * Update of the cipher operation in xts mode. + * + * @dupdate Data update object + */ +TEE_Result caam_cipher_update_xts(struct drvcrypt_cipher_update *dupdate); + +/* + * Initialization of the cipher operation + * + * @dinit Data initialization object + */ +TEE_Result caam_cipher_initialize(struct drvcrypt_cipher_init *dinit); + +/* + * Free software context + * + * @ctx Caller context variable + */ +void caam_cipher_free(void *ctx); + +/* + * Copy software Context + * + * @dst_ctx [out] Reference the context destination + * @src_ctx Reference the context source + */ +void caam_cipher_copy_state(void *dst_ctx, void *src_ctx); + +#endif /* __LOCAL_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/cipher/sub.mk b/optee_os/core/drivers/crypto/caam/cipher/sub.mk new file mode 100644 index 0000000..802f99d --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/cipher/sub.mk @@ -0,0 +1,5 @@ +incdirs-y += ../include + +srcs-y += caam_cipher.c +srcs-y += caam_cipher_xts.c +srcs-$(CFG_NXP_CAAM_CMAC_DRV) += caam_cipher_mac.c diff --git a/optee_os/core/drivers/crypto/caam/crypto.mk b/optee_os/core/drivers/crypto/caam/crypto.mk new file mode 100644 index 0000000..8483680 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/crypto.mk @@ -0,0 +1,211 @@ +ifeq ($(CFG_NXP_CAAM),y) +# CAAM Debug: define 3x32 bits value (same bit used to debug a module) +# CFG_DBG_CAAM_TRACE Module print trace +# CFG_DBG_CAAM_DESC Module descriptor dump +# CFG_DBG_CAAM_BUF Module buffer dump +# +# DBG_HAL BIT32(0) // HAL trace +# DBG_CTRL BIT32(1) // Controller trace +# DBG_MEM BIT32(2) // Memory utility trace +# DBG_SGT BIT32(3) // Scatter Gather trace +# DBG_PWR BIT32(4) // Power trace +# DBG_JR BIT32(5) // Job Ring trace +# DBG_RNG BIT32(6) // RNG trace +# DBG_HASH BIT32(7) // Hash trace +# DBG_RSA BIT32(8) // RSA trace +# DBG_CIPHER BIT32(9) // Cipher trace +# DBG_BLOB BIT32(10) // BLOB trace +# DBG_DMAOBJ BIT32(11) // DMA Object Trace +# DBG_ECC BIT32(12) // ECC trace +# DBG_DH BIT32(13) // DH Trace +# DBG_DSA BIT32(14) // DSA trace +# DBG_MP BIT32(15) // MP trace +CFG_DBG_CAAM_TRACE ?= 0x2 +CFG_DBG_CAAM_DESC ?= 0x0 +CFG_DBG_CAAM_BUF ?= 0x0 + +# CAAM default drivers +caam-drivers = RNG BLOB + +# CAAM default drivers connected to the HW crypto API +caam-crypto-drivers = CIPHER HASH HMAC CMAC + +ifneq (,$(filter $(PLATFORM_FLAVOR),ls1012ardb ls1043ardb ls1046ardb)) +$(call force, CFG_CAAM_BIG_ENDIAN,y) +$(call force, CFG_JR_BLOCK_SIZE,0x10000) +$(call force, CFG_JR_INDEX,2) +$(call force, CFG_JR_INT,105) +$(call force, CFG_CAAM_SGT_ALIGN,4) +$(call force, CFG_CAAM_64BIT,y) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +$(call force, CFG_CAAM_ITR,n) +caam-crypto-drivers += RSA DSA ECC DH MATH +else ifneq (,$(filter $(PLATFORM_FLAVOR),ls1088ardb ls2088ardb ls1028ardb)) +$(call force, CFG_CAAM_LITTLE_ENDIAN,y) +$(call force, CFG_JR_BLOCK_SIZE,0x10000) +$(call force, CFG_JR_INDEX,2) +$(call force, CFG_JR_INT,174) +$(call force, CFG_NXP_CAAM_SGT_V2,y) +$(call force, CFG_CAAM_SGT_ALIGN,4) +$(call force, CFG_CAAM_64BIT,y) +$(call force, CFG_CAAM_ITR,n) +caam-crypto-drivers += RSA DSA ECC DH MATH +else ifneq (,$(filter $(PLATFORM_FLAVOR),lx2160aqds lx2160ardb)) +$(call force, CFG_CAAM_LITTLE_ENDIAN,y) +$(call force, CFG_JR_BLOCK_SIZE,0x10000) +$(call force, CFG_JR_INDEX,2) +$(call force, CFG_JR_INT, 174) +$(call force, CFG_NB_JOBS_QUEUE, 80) +$(call force, CFG_NXP_CAAM_SGT_V2,y) +$(call force, CFG_CAAM_SGT_ALIGN,4) +$(call force, CFG_CAAM_64BIT,y) +$(call force, CFG_CAAM_ITR,n) +caam-crypto-drivers += RSA DSA ECC DH MATH +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8qm-flavorlist) $(mx8qx-flavorlist))) +$(call force, CFG_CAAM_SIZE_ALIGN,4) +$(call force, CFG_JR_BLOCK_SIZE,0x10000) +$(call force, CFG_JR_INDEX,3) +$(call force, CFG_JR_INT,486) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +caam-crypto-drivers += RSA DSA ECC DH MATH +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8dxl-flavorlist))) +$(call force, CFG_CAAM_SIZE_ALIGN,4) +$(call force, CFG_JR_BLOCK_SIZE,0x10000) +$(call force, CFG_JR_INDEX,2) +$(call force, CFG_JR_INT,355) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +$(call force, CFG_CAAM_JR_DISABLE_NODE,n) +caam-crypto-drivers += RSA DSA ECC DH MATH +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8mm-flavorlist) $(mx8mn-flavorlist) \ + $(mx8mp-flavorlist) $(mx8mq-flavorlist))) +$(call force, CFG_JR_BLOCK_SIZE,0x1000) +$(call force, CFG_JR_INDEX,2) +$(call force, CFG_JR_INT,146) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +$(call force, CFG_JR_HAB_INDEX,0) +caam-drivers += MP DEK +caam-crypto-drivers += RSA DSA ECC DH MATH +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx8ulp-flavorlist))) +$(call force, CFG_JR_BLOCK_SIZE,0x1000) +$(call force, CFG_JR_INDEX,2) +$(call force, CFG_JR_INT,114) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +$(call force, CFG_CAAM_ITR,n) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx7ulp-flavorlist))) +$(call force, CFG_JR_BLOCK_SIZE,0x1000) +$(call force, CFG_JR_INDEX,0) +$(call force, CFG_JR_INT,137) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +$(call force, CFG_CAAM_ITR,n) +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6ul-flavorlist) $(mx7d-flavorlist) \ + $(mx7s-flavorlist))) +$(call force, CFG_JR_BLOCK_SIZE,0x1000) +$(call force, CFG_JR_INDEX,0) +$(call force, CFG_JR_INT,137) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +caam-drivers += MP +caam-crypto-drivers += RSA DSA ECC DH MATH +else ifneq (,$(filter $(PLATFORM_FLAVOR),$(mx6q-flavorlist) $(mx6qp-flavorlist) \ + $(mx6sx-flavorlist) $(mx6d-flavorlist) $(mx6dl-flavorlist) \ + $(mx6s-flavorlist) $(mx8ulp-flavorlist))) +$(call force, CFG_JR_BLOCK_SIZE,0x1000) +$(call force, CFG_JR_INDEX,0) +$(call force, CFG_JR_INT,137) +$(call force, CFG_NXP_CAAM_SGT_V1,y) +else +$(error Unsupported PLATFORM_FLAVOR "$(PLATFORM_FLAVOR)") +endif + +# Disable the i.MX CAAM driver +$(call force,CFG_IMX_CAAM,n,Mandated by CFG_NXP_CAAM) + +# CAAM buffer alignment size +CFG_CAAM_SIZE_ALIGN ?= 1 + +# Default padding number for SGT allocation +CFG_CAAM_SGT_ALIGN ?= 1 + +# Enable job ring interruption +CFG_CAAM_ITR ?= y + +# Keep the CFG_JR_INDEX as secure at runtime +CFG_NXP_CAAM_RUNTIME_JR ?= y + +# Define the RSA Private Key Format used by the CAAM +# Format #1: (n, d) +# Format #2: (p, q, d) +# Format #3: (p, q, dp, dq, qp) +CFG_NXP_CAAM_RSA_KEY_FORMAT ?= 3 + +# Disable device tree status of the secure job ring +CFG_CAAM_JR_DISABLE_NODE ?= y + +# Enable CAAM non-crypto drivers +$(foreach drv, $(caam-drivers), $(eval CFG_NXP_CAAM_$(drv)_DRV ?= y)) + +# Disable software RNG if CAAM RNG driver is enabled +ifeq ($(CFG_NXP_CAAM_RNG_DRV), y) +$(call force, CFG_WITH_SOFTWARE_PRNG,n,Mandated by CFG_NXP_CAAM_RNG_DRV) +endif + +# DEK driver requires the SM driver to be enabled +ifeq ($(CFG_NXP_CAAM_DEK_DRV), y) +$(call force, CFG_NXP_CAAM_SM_DRV,y,Mandated by CFG_NXP_CAAM_DEK_DRV) +endif + +ifeq ($(CFG_CRYPTO_DRIVER), y) +CFG_CRYPTO_DRIVER_DEBUG ?= 0 + +# Enable CAAM Crypto drivers +$(foreach drv, $(caam-crypto-drivers), $(eval CFG_NXP_CAAM_$(drv)_DRV ?= y)) + +# Enable MAC crypto driver +ifeq ($(call cfg-one-enabled,CFG_NXP_CAAM_HMAC_DRV CFG_NXP_CAAM_CMAC_DRV),y) +$(call force, CFG_CRYPTO_DRV_MAC,y,Mandated by CFG_NXP_CAAM_HMAC/CMAC_DRV) +endif + +# Enable CIPHER crypto driver +ifeq ($(CFG_NXP_CAAM_CIPHER_DRV), y) +$(call force, CFG_CRYPTO_DRV_CIPHER,y,Mandated by CFG_NXP_CAAM_CIPHER_DRV) +endif + +# Enable HASH crypto driver +ifeq ($(CFG_NXP_CAAM_HASH_DRV), y) +$(call force, CFG_CRYPTO_DRV_HASH,y,Mandated by CFG_NXP_CAAM_HASH_DRV) +endif + +# Enable RSA crypto driver +ifeq ($(CFG_NXP_CAAM_RSA_DRV), y) +$(call force, CFG_CRYPTO_DRV_RSA,y,Mandated by CFG_NXP_CAAM_RSA_DRV) +endif + +# Enable ECC crypto driver +ifeq ($(CFG_NXP_CAAM_ECC_DRV), y) +$(call force, CFG_CRYPTO_DRV_ECC,y,Mandated by CFG_NXP_CAAM_ECC_DRV) +endif + +# Enable DSA crypto driver +ifeq ($(CFG_NXP_CAAM_DSA_DRV), y) +$(call force, CFG_CRYPTO_DRV_DSA,y,Mandated by CFG_NXP_CAAM_DSA_DRV) +endif + +# Enable DH crypto driver +ifeq ($(CFG_NXP_CAAM_DH_DRV), y) +$(call force, CFG_CRYPTO_DRV_DH,y,Mandated by CFG_NXP_CAAM_DH_DRV) +endif + +# Enable ACIPHER crypto driver +ifeq ($(call cfg-one-enabled,CFG_CRYPTO_DRV_RSA CFG_CRYPTO_DRV_ECC \ + CFG_CRYPTO_DRV_DSA CFG_CRYPTO_DRV_DH),y) +$(call force, CFG_CRYPTO_DRV_ACIPHER,y,Mandated by CFG_CRYPTO_DRV_{RSA|ECC|DSA|DH}) +endif + +# Disable SM2 as it is not supported by the CAAM driver +ifeq ($(CFG_NXP_CAAM_ECC_DRV),y) +$(call force,CFG_CRYPTO_SM2_PKE,n) +$(call force,CFG_CRYPTO_SM2_KEP,n) +$(call force,CFG_CRYPTO_SM2_DSA,n) +endif + +endif # CFG_CRYPTO_DRIVER +endif # CFG_NXP_CAAM diff --git a/optee_os/core/drivers/crypto/caam/hal/common/hal_cfg.c b/optee_os/core/drivers/crypto/caam/hal/common/hal_cfg.c new file mode 100644 index 0000000..2bdf9e6 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/hal_cfg.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2019, 2021 NXP + * + * Brief CAAM Configuration. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum caam_status caam_hal_cfg_get_conf(struct caam_jrcfg *jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + vaddr_t ctrl_base = 0; + void *fdt = NULL; + + fdt = get_dt(); + + /* + * First get the CAAM Controller base address from the DTB, + * if DTB present and if the CAAM Controller defined in it. + */ + if (fdt) + caam_hal_cfg_get_ctrl_dt(fdt, &ctrl_base); + + if (!ctrl_base) { + ctrl_base = (vaddr_t)core_mmu_add_mapping(MEM_AREA_IO_SEC, + CAAM_BASE, CAAM_SIZE); + if (!ctrl_base) { + EMSG("Unable to map CAAM Registers"); + goto exit_get_conf; + } + } + + jrcfg->base = ctrl_base; + + /* + * Next get the Job Ring reserved for the Secure environment + * into the DTB. If nothing reserved use the default hard coded + * value. + */ + if (fdt) + caam_hal_cfg_get_jobring_dt(fdt, jrcfg); + + if (!jrcfg->offset) { + jrcfg->offset = (CFG_JR_INDEX + 1) * JRX_BLOCK_SIZE; + jrcfg->it_num = CFG_JR_INT; + + if (IS_ENABLED(CFG_NXP_CAAM_RUNTIME_JR) && + !is_embedded_dt(fdt)) { + if (fdt) { + /* Ensure Secure Job Ring is secure in DTB */ + caam_hal_cfg_disable_jobring_dt(fdt, jrcfg); + } + } + } + + jrcfg->nb_jobs = NB_JOBS_QUEUE; + + retstatus = CAAM_NO_ERROR; + +exit_get_conf: + HAL_TRACE("HAL CFG Get CAAM config ret (0x%x)\n", retstatus); + return retstatus; +} + +void __weak caam_hal_cfg_setup_nsjobring(struct caam_jrcfg *jrcfg) +{ + enum caam_status status = CAAM_FAILURE; + paddr_t jr_offset = 0; + uint8_t jrnum = 0; + + for (jrnum = caam_hal_ctrl_jrnum(jrcfg->base); jrnum; jrnum--) { + jr_offset = jrnum * JRX_BLOCK_SIZE; + +#ifdef CFG_NXP_CAAM_RUNTIME_JR + /* + * When the Cryptographic driver is enabled, keep the + * Secure Job Ring don't release it. + * But save the configuration to restore it when + * device reset after suspend. + */ + if (jr_offset == jrcfg->offset) { + caam_hal_jr_prepare_backup(jrcfg->base, jr_offset); + continue; + } +#endif + status = caam_hal_jr_setowner(jrcfg->base, jr_offset, + JROWN_ARM_NS); + if (status == CAAM_NO_ERROR) + caam_hal_jr_prepare_backup(jrcfg->base, jr_offset); + } +} diff --git a/optee_os/core/drivers/crypto/caam/hal/common/hal_cfg_dt.c b/optee_os/core/drivers/crypto/caam/hal/common/hal_cfg_dt.c new file mode 100644 index 0000000..9a84b2c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/hal_cfg_dt.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2019, 2021 NXP + * + * Brief CAAM Configuration. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char *dt_caam_match_table = { + "fsl,sec-v4.0", +}; + +static const char *dt_jr_match_table = { + "fsl,sec-v4.0-job-ring", +}; + +/* + * Finds the Job Ring reserved for the Secure Mode in the DTB + * + * @fdt Reference to the Device Tree + * @status Status mask flag of the node to found + * @find_node [out] Node offset found + */ +static paddr_t find_jr_offset(void *fdt, int status, int *find_node) +{ + paddr_t jr_offset = 0; + int node = fdt_node_offset_by_compatible(fdt, 0, dt_jr_match_table); + + for (; node != -FDT_ERR_NOTFOUND; + node = fdt_node_offset_by_compatible(fdt, node, + dt_jr_match_table)) { + HAL_TRACE("Found Job Ring node status @%" PRId32, node); + if (fdt_get_status(fdt, node) == status) { + HAL_TRACE("Found Job Ring node @%" PRId32, node); + jr_offset = fdt_reg_base_address(fdt, node); + *find_node = node; + break; + } + } + + HAL_TRACE("JR Offset return 0x%" PRIxPTR, jr_offset); + return jr_offset; +} + +void caam_hal_cfg_get_ctrl_dt(void *fdt, vaddr_t *ctrl_base) +{ + size_t size = 0; + int node = 0; + paddr_t pctrl_base = 0; + + *ctrl_base = 0; + /* Get the CAAM Node to get the controller base address */ + node = fdt_node_offset_by_compatible(fdt, 0, dt_caam_match_table); + + if (node < 0) + return; + + /* + * Map CAAM controller base address as Secure IO if not + * already present in the MMU table. + * Then get the virtual address of the CAAM controller + */ + pctrl_base = fdt_reg_base_address(fdt, node); + if (pctrl_base == DT_INFO_INVALID_REG) { + HAL_TRACE("CAAM control base address not defined"); + return; + } + + size = fdt_reg_size(fdt, node); + if (size == DT_INFO_INVALID_REG_SIZE) { + HAL_TRACE("CAAM control base address size not defined"); + return; + } + + *ctrl_base = (vaddr_t)core_mmu_add_mapping(MEM_AREA_IO_SEC, pctrl_base, + size); + if (!*ctrl_base) { + EMSG("CAAM control base MMU PA mapping failure"); + return; + } + + HAL_TRACE("Map Controller 0x%" PRIxVA, *ctrl_base); +} + +void caam_hal_cfg_get_jobring_dt(void *fdt, struct caam_jrcfg *jrcfg) +{ + paddr_t jr_offset = 0; + int jr_it_num = 0; + int node = 0; + + jr_offset = find_jr_offset(fdt, DT_STATUS_OK_SEC, &node); + if (jr_offset) { + if (!is_embedded_dt(fdt)) { + /* Disable JR for Normal World */ + if (dt_enable_secure_status(fdt, node)) { + EMSG("Not able to disable JR DTB entry"); + return; + } + } + + /* Get the job ring interrupt */ + jr_it_num = dt_get_irq(fdt, node); + if (jr_it_num == DT_INFO_INVALID_INTERRUPT) { + EMSG("Job Ring interrupt number not defined in DTB"); + return; + } + + jrcfg->offset = jr_offset; + jrcfg->it_num = jr_it_num; + } +} + +void caam_hal_cfg_disable_jobring_dt(void *fdt, struct caam_jrcfg *jrcfg) +{ + int node = fdt_node_offset_by_compatible(fdt, 0, dt_jr_match_table); + + for (; node != -FDT_ERR_NOTFOUND; + node = fdt_node_offset_by_compatible(fdt, node, + dt_jr_match_table)) { + HAL_TRACE("Found Job Ring node @%" PRId32, node); + if (fdt_reg_base_address(fdt, node) == jrcfg->offset) { + HAL_TRACE("Disable Job Ring node @%" PRId32, node); + if (dt_enable_secure_status(fdt, node)) + panic(); + break; + } + } +} diff --git a/optee_os/core/drivers/crypto/caam/hal/common/hal_ctrl.c b/optee_os/core/drivers/crypto/caam/hal/common/hal_ctrl.c new file mode 100644 index 0000000..8783eec --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/hal_ctrl.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * Brief CAAM Controller Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +uint8_t caam_hal_ctrl_era(vaddr_t baseaddr) +{ + /* Read the number of instance */ + uint32_t val = io_caam_read32(baseaddr + CCBVID); + + return GET_CCBVID_CAAM_ERA(val); +} + +uint8_t caam_hal_ctrl_jrnum(vaddr_t baseaddr) +{ + uint32_t val = 0; + uint8_t jrnum = 0; + + if (caam_hal_ctrl_era(baseaddr) < 10) { + val = io_caam_read32(baseaddr + CHANUM_MS); + jrnum = GET_CHANUM_MS_JRNUM(val); + } else { + val = io_caam_read32(baseaddr + JR_VERSION); + jrnum = GET_JR_VERSION_JRNUM(val); + } + + return jrnum; +} + +uint8_t caam_hal_ctrl_hash_limit(vaddr_t baseaddr) +{ + uint32_t val = 0; + + if (caam_hal_ctrl_era(baseaddr) < 10) { + /* Read the number of instance */ + val = io_caam_read32(baseaddr + CHANUM_LS); + + if (GET_CHANUM_LS_MDNUM(val)) { + /* Hashing is supported */ + val = io_caam_read32(baseaddr + CHAVID_LS); + val &= BM_CHAVID_LS_MDVID; + if (val == CHAVID_LS_MDVID_LP256) + return TEE_MAIN_ALGO_SHA256; + + return TEE_MAIN_ALGO_SHA512; + } + } else { + /* Read the number of instance */ + val = io_caam_read32(baseaddr + MDHA_VERSION); + + if (GET_MDHA_VERSION_MDNUM(val)) { + /* Hashing is supported */ + val &= BM_MDHA_VERSION_MDVID; + if (val == MDHA_VERSION_MDVID_LP256) + return TEE_MAIN_ALGO_SHA256; + + return TEE_MAIN_ALGO_SHA512; + } + } + + return UINT8_MAX; +} + +bool caam_hal_ctrl_splitkey_support(vaddr_t baseaddr) +{ + uint32_t val = io_caam_read32(baseaddr + CTPR_LS); + + return GET_CTPR_LS_SPLIT_KEY(val); +} + +uint8_t caam_hal_ctrl_pknum(vaddr_t baseaddr) +{ + uint32_t val = 0; + uint8_t pknum = 0; + + if (caam_hal_ctrl_era(baseaddr) < 10) { + val = io_caam_read32(baseaddr + CHANUM_LS); + pknum = GET_CHANUM_LS_PKNUM(val); + } else { + val = io_caam_read32(baseaddr + PKHA_VERSION); + pknum = GET_PKHA_VERSION_PKNUM(val); + } + + return pknum; +} + +#define PRIBLOB_MASK GENMASK_32(1, 0) + +void caam_hal_ctrl_inc_priblob(vaddr_t baseaddr) +{ + uint32_t val = 0; + uint32_t blob = 0; + + if (!IS_ENABLED(CFG_CAAM_INC_PRIBLOB)) + return; + + val = io_caam_read32(baseaddr + SCFGR); + val &= PRIBLOB_MASK; + CTRL_TRACE("Reading CAAM PRIBLOB: 0x%"PRIx32, val); + + if (val == 0 || val == 2) + blob = val + 1; + else if (val == 1) + blob = val + 2; + else + panic("Error locking PRIBLOB, PRIBLOB =3"); + + CTRL_TRACE("New CAAM PRIBLOB value: 0x%"PRIx32, blob); + + val = io_caam_read32(baseaddr + SCFGR); + val |= blob; + io_caam_write32(baseaddr + SCFGR, val); + + val = io_caam_read32(baseaddr + SCFGR); + val &= PRIBLOB_MASK; + CTRL_TRACE("Checking: CAAM PRIBLOB: 0x%"PRIx32 " want: 0x%"PRIx32, val, + blob); + if (val != blob) + panic("Written PRIBLOB and read PRIBLOB do not match!"); +} + +#ifdef CFG_NXP_CAAM_MP_DRV +uint8_t caam_hal_ctrl_get_mpcurve(vaddr_t ctrl_addr) +{ + uint32_t val_scfgr = 0; + + /* + * On i.MX8MQ B0, the MP is not usable, hence + * return UINT8_MAX + */ + if (soc_is_imx8mq_b0_layer()) + return UINT8_MAX; + + /* + * Verify if the device is closed or not + * If device is closed, check get the MPCurve + */ + if (snvs_is_device_closed()) { + /* Get the SCFGR content */ + val_scfgr = io_caam_read32(ctrl_addr + SCFGR); + + /* Get the MPCurve field value - 4 bits */ + val_scfgr = (val_scfgr & BM_SCFGR_MPCURVE) >> BS_SCFGR_MPCURVE; + + /* + * If the device is closed and the MPCurve field is 0 + * return UINT8_MAX indicating that there is a problem and the + * MP can not be supported. + */ + if (!val_scfgr) + return UINT8_MAX; + } + + return val_scfgr; +} + +TEE_Result caam_hal_ctrl_read_mpmr(vaddr_t ctrl_addr, struct caambuf *mpmr) +{ + unsigned int i = 0; + uint32_t val = 0; + + if (mpmr->length < MPMR_NB_REG) { + mpmr->length = MPMR_NB_REG; + return TEE_ERROR_SHORT_BUFFER; + } + + /* MPMR endianness is reverted between write and read */ + for (i = 0; i < MPMR_NB_REG; i += 4) { + val = io_caam_read32(ctrl_addr + MPMR + i); + mpmr->data[i] = (uint8_t)(val >> 24); + mpmr->data[i + 1] = (uint8_t)(val >> 16); + mpmr->data[i + 2] = (uint8_t)(val >> 8); + mpmr->data[i + 3] = (uint8_t)val; + } + + mpmr->length = MPMR_NB_REG; + return TEE_SUCCESS; +} + +bool caam_hal_ctrl_is_mp_set(vaddr_t ctrl_addr) +{ + return io_caam_read32(ctrl_addr + SCFGR) & BM_SCFGR_MPMRL; +} + +void caam_hal_ctrl_fill_mpmr(vaddr_t ctrl_addr, struct caambuf *msg_mpmr) +{ + size_t i = 0; + vaddr_t reg = ctrl_addr + MPMR; + bool is_filled = false; + uint32_t val = 0; + size_t min_size = 0; + size_t remain_size = 0; + + /* check if the MPMR is filled */ + is_filled = caam_hal_ctrl_is_mp_set(ctrl_addr); + + DMSG("is_filled = %s", is_filled ? "true" : "false"); + + if (!is_filled) { + /* + * Fill the MPMR with the most significant input value and + * complete with 0's if value too short. + */ + min_size = MIN(msg_mpmr->length, (size_t)MPMR_NB_REG); + remain_size = min_size % 4; + + for (i = 0; i < min_size - remain_size; i += 4, reg += 4) { + val = msg_mpmr->data[i] | msg_mpmr->data[i + 1] << 8 | + msg_mpmr->data[i + 2] << 16 | + msg_mpmr->data[i + 3] << 24; + io_caam_write32(reg, val); + } + + /* Last input bytes value */ + if (remain_size) { + val = 0; + + /* + * Fill the MPMR with the 8 bits values + * until the end of the message length + */ + for (i = 0; i < remain_size; i++) + val |= msg_mpmr->data[i] << (i * 8); + io_caam_write32(reg, val); + reg += 4; + } + + /* Complete with 0's */ + remain_size = (MPMR_NB_REG - ROUNDUP(msg_mpmr->length, 4)) / 4; + for (i = 0; i < remain_size; i++, reg += 4) + io_caam_write32(reg, 0x0); + + /* + * Locks the MPMR for writing and remains locked until + * the next power-on session. + */ + io_caam_write32(ctrl_addr + SCFGR, + io_caam_read32(ctrl_addr + SCFGR) | + BM_SCFGR_MPMRL); + + DMSG("val_scfgr = %#"PRIx32, io_caam_read32(ctrl_addr + SCFGR)); + } +} +#endif /* CFG_NXP_CAAM_MP_DRV */ + +#ifdef CFG_NXP_CAAM_SM_DRV +vaddr_t caam_hal_ctrl_get_smvaddr(vaddr_t ctrl_addr, paddr_t jr_offset) +{ + /* + * The Secure Memory Virtual Base Address contains only the upper + * bits of the base address of Secure Memory in this Job Ring's virtual + * address space. Since the base address of Secure Memory must be on a + * 64 kbyte boundary, the least significant 16 bits are omitted. + */ + return io_caam_read32(ctrl_addr + JRX_SMVBAR(JRX_IDX(jr_offset))) << 16; +} +#endif /* CFG_NXP_CAAM_SM_DRV */ diff --git a/optee_os/core/drivers/crypto/caam/hal/common/hal_jr.c b/optee_os/core/drivers/crypto/caam/hal/common/hal_jr.c new file mode 100644 index 0000000..1d344b4 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/hal_jr.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Job Rings Hardware Abstration Layer. + * Implementation of primitives to access HW + */ +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_NXP_CAAM_RUNTIME_JR +/* + * List of common JR registers to save/restore + */ +static const struct reglist jr_backup[] = { + BACKUP_REG(JRX_IRBAR, 2, 0, 0), + BACKUP_REG(JRX_IRSR, 1, 0, 0), + BACKUP_REG(JRX_ORBAR, 2, 0, 0), + BACKUP_REG(JRX_ORSR, 1, 0, 0), + BACKUP_REG(JRX_JRCFGR_LS, 1, 0, 0), +}; +#endif /* CFG_NXP_CAAM_RUNTIME_JR */ + +enum caam_status caam_hal_jr_reset(vaddr_t baseaddr) +{ + uint16_t timeout = 10000; + uint32_t reg_val = 0; + + /* + * Reset is done in 2 steps: + * - Flush all pending jobs (Set RESET bit) + * - Reset the Job Ring (Set RESET bit second time) + */ + + /* Mask interrupts to poll for reset completion status */ + io_setbits32(baseaddr + JRX_JRCFGR_LS, JRX_JRCFGR_LS_IMSK); + + /* Initiate flush (required prior to reset) */ + io_caam_write32(baseaddr + JRX_JRCR, JRX_JRCR_RESET); + + do { + caam_udelay(100); + reg_val = io_caam_read32(baseaddr + JRX_JRINTR); + reg_val &= BM_JRX_JRINTR_HALT; + } while ((reg_val == JRINTR_HALT_ONGOING) && --timeout); + + if (!timeout || reg_val != JRINTR_HALT_DONE) { + EMSG("Failed to flush job ring\n"); + return CAAM_FAILURE; + } + + /* Initiate reset */ + timeout = 100; + io_caam_write32(baseaddr + JRX_JRCR, JRX_JRCR_RESET); + do { + caam_udelay(100); + reg_val = io_caam_read32(baseaddr + JRX_JRCR); + } while ((reg_val & JRX_JRCR_RESET) && --timeout); + + if (!timeout) { + EMSG("Failed to reset job ring\n"); + return CAAM_FAILURE; + } + + return CAAM_NO_ERROR; +} + +void caam_hal_jr_config(vaddr_t baseaddr, uint8_t nbjobs, uint64_t inrings, + uint64_t outrings) +{ + uint32_t value = 0; + + /* Setup the JR input queue */ +#if defined(CFG_CAAM_64BIT) && defined(CFG_CAAM_LITTLE_ENDIAN) + io_caam_write32(baseaddr + JRX_IRBAR, inrings); + io_caam_write32(baseaddr + JRX_IRBAR + 4, inrings >> 32); +#else + io_caam_write32(baseaddr + JRX_IRBAR, inrings >> 32); + io_caam_write32(baseaddr + JRX_IRBAR + 4, inrings); +#endif + io_caam_write32(baseaddr + JRX_IRSR, nbjobs); + + /* Setup the JR output queue */ +#if defined(CFG_CAAM_64BIT) && defined(CFG_CAAM_LITTLE_ENDIAN) + io_caam_write32(baseaddr + JRX_ORBAR, outrings); + io_caam_write32(baseaddr + JRX_ORBAR + 4, outrings >> 32); +#else + io_caam_write32(baseaddr + JRX_ORBAR, outrings >> 32); + io_caam_write32(baseaddr + JRX_ORBAR + 4, outrings); +#endif + io_caam_write32(baseaddr + JRX_ORSR, nbjobs); + + /* Disable the JR interrupt */ + caam_hal_jr_disable_itr(baseaddr); + + /* + * Configure interrupt and disable it: + * Optimization to generate an interrupt either when there are + * half of the job done + * or when there is a job done and 10 clock cycles elapsed without + * new job completion + */ + value = JRX_JRCFGR_LS_ICTT(10); + value |= JRX_JRCFGR_LS_ICDCT(nbjobs / 2); + value |= JRX_JRCFGR_LS_ICEN; + value |= JRX_JRCFGR_LS_IMSK; + io_caam_write32(baseaddr + JRX_JRCFGR_LS, value); + +#ifdef CFG_NXP_CAAM_RUNTIME_JR + caam_pwr_add_backup(baseaddr, jr_backup, ARRAY_SIZE(jr_backup)); +#endif +} + +uint32_t caam_hal_jr_read_nbslot_available(vaddr_t baseaddr) +{ + return io_caam_read32(baseaddr + JRX_IRSAR); +} + +void caam_hal_jr_add_newjob(vaddr_t baseaddr) +{ + io_caam_write32(baseaddr + JRX_IRJAR, 1); +} + +uint32_t caam_hal_jr_get_nbjob_done(vaddr_t baseaddr) +{ + return io_caam_read32(baseaddr + JRX_ORSFR); +} + +void caam_hal_jr_del_job(vaddr_t baseaddr) +{ + io_caam_write32(baseaddr + JRX_ORJRR, 1); +} + +#ifdef CFG_CAAM_ITR +void caam_hal_jr_disable_itr(vaddr_t baseaddr) +{ + io_setbits32(baseaddr + JRX_JRCFGR_LS, JRX_JRCFGR_LS_IMSK); + io_setbits32(baseaddr + JRX_JRINTR, JRX_JRINTR_JRI); +} + +void caam_hal_jr_enable_itr(vaddr_t baseaddr) +{ + io_mask32(baseaddr + JRX_JRCFGR_LS, ~JRX_JRCFGR_LS_IMSK, + JRX_JRCFGR_LS_IMSK); +} +#else +void caam_hal_jr_disable_itr(vaddr_t baseaddr __unused) {} +void caam_hal_jr_enable_itr(vaddr_t baseaddr __unused) {} +#endif /* CFG_CAAM_ITR */ + +bool caam_hal_jr_check_ack_itr(vaddr_t baseaddr) +{ + uint32_t val = 0; + + val = io_caam_read32(baseaddr + JRX_JRINTR); + + if ((val & JRX_JRINTR_JRI) == JRX_JRINTR_JRI) { + /* Acknowledge interrupt */ + io_setbits32(baseaddr + JRX_JRINTR, JRX_JRINTR_JRI); + return true; + } + + return false; +} + +enum caam_status caam_hal_jr_halt(vaddr_t baseaddr) +{ + uint16_t timeout = 10000; + uint32_t val = 0; + + /* Mask interrupts to poll for completion status */ + io_setbits32(baseaddr + JRX_JRCFGR_LS, JRX_JRCFGR_LS_IMSK); + + /* Request Job ring halt */ + io_caam_write32(baseaddr + JRX_JRCR, JRX_JRCR_PARK); + + /* Check if there is a job running */ + val = io_caam_read32(baseaddr + JRX_IRSR); + if ((caam_hal_jr_read_nbslot_available(baseaddr) == val) && + (io_caam_read32(baseaddr + JRX_CSTA) != JRX_CSTA_BSY)) + return CAAM_NO_ERROR; + + /* Wait until all jobs complete */ + do { + caam_udelay(10); + val = io_caam_read32(baseaddr + JRX_JRINTR); + val &= BM_JRX_JRINTR_HALT; + } while ((val != JRINTR_HALT_DONE) && --timeout); + + if (!timeout) + return CAAM_BUSY; + + return CAAM_NO_ERROR; +} + +enum caam_status caam_hal_jr_flush(vaddr_t baseaddr) +{ + uint16_t timeout = 10000; + uint32_t val = 0; + + /* Mask interrupts to poll for completion status */ + io_setbits32(baseaddr + JRX_JRCFGR_LS, JRX_JRCFGR_LS_IMSK); + + /* Request Job ring to flush input queue */ + io_caam_write32(baseaddr + JRX_JRCR, JRX_JRCR_RESET); + + /* Check if there is a job running */ + val = io_caam_read32(baseaddr + JRX_IRSR); + if ((caam_hal_jr_read_nbslot_available(baseaddr) == val) && + (io_caam_read32(baseaddr + JRX_CSTA) != JRX_CSTA_BSY)) + return CAAM_NO_ERROR; + + /* Wait until all jobs complete */ + do { + caam_udelay(10); + val = io_caam_read32(baseaddr + JRX_JRINTR); + val &= BM_JRX_JRINTR_HALT; + } while ((val == JRINTR_HALT_ONGOING) && --timeout); + + if (!timeout) + return CAAM_BUSY; + + return CAAM_NO_ERROR; +} + +void caam_hal_jr_resume(vaddr_t baseaddr) +{ + io_caam_write32(baseaddr + JRX_JRINTR, JRINTR_HALT_RESUME); + + caam_hal_jr_enable_itr(baseaddr); +} + +uint8_t caam_hal_jr_input_index(vaddr_t baseaddr) +{ + return io_caam_read32(baseaddr + JRX_IRRIR) >> 2; +} + +uint8_t caam_hal_jr_output_index(vaddr_t baseaddr) +{ + return io_caam_read32(baseaddr + JRX_ORWIR) >> 3; +} diff --git a/optee_os/core/drivers/crypto/caam/hal/common/hal_rng.c b/optee_os/core/drivers/crypto/caam/hal/common/hal_rng.c new file mode 100644 index 0000000..c98ee4d --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/hal_rng.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Brief CAAM Random Number Generator Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include +#include +#include +#include + +enum caam_status __weak caam_hal_rng_instantiated(vaddr_t baseaddr) +{ + uint32_t vid = 0; + uint32_t nb_sh = 0; + uint32_t status = 0; + + /* RNG version < 4 and RNG state handle is already instantiated */ + if (caam_hal_ctrl_era(baseaddr) < 10) { + vid = io_caam_read32(baseaddr + CHAVID_LS); + + if (GET_CHAVID_LS_RNGVID(vid) < 4) + return CAAM_NO_ERROR; + } else { + vid = io_caam_read32(baseaddr + RNG_VERSION); + + if (GET_RNG_VERSION_VID(vid) < 4) + return CAAM_NO_ERROR; + } + + /* Get the Number of State Handles */ + nb_sh = caam_hal_rng_get_nb_sh(baseaddr); + + /* Read the RNG Status and checks if all channels are instantiatied */ + status = caam_hal_rng_get_sh_status(baseaddr); + + if (status != GENMASK_32(nb_sh - 1, 0)) + return CAAM_NOT_INIT; + + return CAAM_NO_ERROR; +} + +uint32_t caam_hal_rng_get_nb_sh(vaddr_t baseaddr) +{ + uint32_t reg = 0; + + reg = io_caam_read32(baseaddr + CTPR_MS); + + return GET_CTPR_MS_RNG_I(reg); +} + +uint32_t caam_hal_rng_get_sh_status(vaddr_t baseaddr) +{ + return io_caam_read32(baseaddr + RNG_STA) & (RNG_STA_IF1 | RNG_STA_IF0); +} + +bool caam_hal_rng_key_loaded(vaddr_t baseaddr) +{ + return io_caam_read32(baseaddr + RNG_STA) & RNG_STA_SKVN; +} + +enum caam_status caam_hal_rng_kick(vaddr_t baseaddr, uint32_t inc_delay) +{ + uint32_t val = 0; + uint32_t ent_delay = TRNG_SDCTL_ENT_DLY_MIN + inc_delay; + + if (ent_delay > TRNG_SDCTL_ENT_DLY_MAX) + return CAAM_OUT_OF_BOUND; + + /* + * Switch RNG in program mode + * Setting both RTMCTL:PRGM and RTMCTL:TRNG_ACC causes TRNG to + * properly invalidate the entropy in the entropy register and + * force re-generation + */ + io_setbits32(baseaddr + TRNG_MCTL, TRNG_MCTL_PRGM | TRNG_MCTL_ACC); + + /* + * Configure the RNG Entropy Delay + * Performance-wise, it does not make sense to + * set the delay to a value that is lower + * than the last one that worked (i.e. the state handles + * were instantiated correctly). Thus, instead of wasting + * time trying to set the values controlling the sample + * frequency, the function simply returns. + */ + val = io_caam_read32(baseaddr + TRNG_SDCTL); + val = GET_TRNG_SDCTL_ENT_DLY(val); + + if (ent_delay < val) { + /* + * In this case do the programmation anyway because on some + * device the other registers value can be wrong. + */ + ent_delay = val; + } + + io_caam_write32(baseaddr + TRNG_SDCTL, TRNG_SDCTL_ENT_DLY(ent_delay) | + TRNG_SDCTL_SAMP_SIZE(512)); + + /* min. freq. count, equal to 1/4 of the entropy sample length */ + io_caam_write32(baseaddr + TRNG_FRQMIN, ent_delay >> 2); + + /* max. freq. count, equal to 16 times the entropy sample length */ + io_caam_write32(baseaddr + TRNG_FRQMAX, ent_delay << 4); + + io_caam_write32(baseaddr + TRNG_RTSCMISC, + TRNG_RTSCMISC_RTY_CNT(2) | TRNG_RTSCMISC_LRUN_MAX(32)); + io_caam_write32(baseaddr + TRNG_RTPKRRNG, TRNG_RTPKRRNG_PKR_RNG(570)); + io_caam_write32(baseaddr + TRNG_RTPKRMAX, TRNG_RTPKRMAX_PKR_MAX(1600)); + io_caam_write32(baseaddr + TRNG_RTSCML, + TRNG_RTSCML_MONO_RNG(122) | TRNG_RTSCML_MONO_MAX(317)); + io_caam_write32(baseaddr + TRNG_RTSCR1L, + TRNG_RTSCR1L_RUN1_RNG(80) | TRNG_RTSCR1L_RUN1_MAX(107)); + io_caam_write32(baseaddr + TRNG_RTSCR2L, + TRNG_RTSCR2L_RUN2_RNG(57) | TRNG_RTSCR2L_RUN2_MAX(62)); + io_caam_write32(baseaddr + TRNG_RTSCR3L, + TRNG_RTSCR3L_RUN3_RNG(39) | TRNG_RTSCR3L_RUN3_MAX(39)); + io_caam_write32(baseaddr + TRNG_RTSCR4L, + TRNG_RTSCR4L_RUN4_RNG(27) | TRNG_RTSCR4L_RUN4_MAX(26)); + io_caam_write32(baseaddr + TRNG_RTSCR5L, + TRNG_RTSCR5L_RUN5_RNG(19) | TRNG_RTSCR5L_RUN5_MAX(18)); + io_caam_write32(baseaddr + TRNG_RTSCR6PL, + TRNG_RTSCR5L_RUN5_RNG(18) | TRNG_RTSCR5L_RUN5_MAX(17)); + + val = io_caam_read32(baseaddr + TRNG_MCTL); + /* + * Select raw sampling in both entropy shifter + * and statistical checker + */ + val &= ~BM_TRNG_MCTL_SAMP_MODE; + val |= TRNG_MCTL_SAMP_MODE_RAW_ES_SC; + /* Put RNG4 into run mode with handling CAAM/RNG4-TRNG Errata */ + val &= ~(TRNG_MCTL_PRGM | TRNG_MCTL_ACC); + io_caam_write32(baseaddr + TRNG_MCTL, val); + + /* + * Clear the ERR bit in RTMCTL if set. The TRNG error can occur when + * the RNG clock is not within 1/2x to 8x the system clock. + * This error is possible if ROM code does not initialize the system + * PLLs immediately after PoR. + */ + io_setbits32(baseaddr + TRNG_MCTL, TRNG_MCTL_ERR); + + return CAAM_NO_ERROR; +} diff --git a/optee_os/core/drivers/crypto/caam/hal/common/hal_sm.c b/optee_os/core/drivers/crypto/caam/hal/common/hal_sm.c new file mode 100644 index 0000000..04552c2 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/hal_sm.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019, 2023 NXP + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Issue a Secure Memory Command to the @page and @partition. + * Returns the command status when completed + * + * @jr_base JR virtual base address + * @page Memory Page + * @partition Partition + * @cmd Command to sent + */ +static uint32_t issue_cmd(vaddr_t jr_base, unsigned int page, + unsigned int partition, uint8_t cmd) +{ + uint32_t status = 0; + uint64_t timeout_ref = timeout_init_us(10000); + + assert(jr_base); + + /* Send cmd */ + io_caam_write32(jr_base + SM_SMCR, SM_SMCR_PAGE(page) | + SM_SMCR_PRTN(partition) | + SM_SMCR_CMD(cmd)); + + /* Wait for the command to complete */ + do { + if (timeout_elapsed(timeout_ref)) + break; + status = io_caam_read32(jr_base + SM_SMCSR); + } while (SM_SMCSR_CERR(status) == SM_SMCSR_CERR_NOT_COMPLETED); + + return io_caam_read32(jr_base + SM_SMCSR); +} + +enum caam_status +caam_hal_sm_check_page_partition(vaddr_t jr_base, + const struct caam_sm_page_desc *page_desc) +{ + uint32_t val = 0; + + if (!jr_base || !page_desc) + return CAAM_BAD_PARAM; + + val = io_caam_read32(jr_base + SMVID_MS); + + if (page_desc->page + page_desc->page_count > + GET_SMVID_MS_MAX_NPAG(val) || + page_desc->partition > GET_SMVID_MS_NPRT(val)) + return CAAM_BAD_PARAM; + + return CAAM_NO_ERROR; +} + +size_t caam_hal_sm_get_pages_size(vaddr_t jr_base, unsigned int page) +{ + size_t page_size = 0; + + page_size = GET_SMVID_LS_PSIZ(io_caam_read32(jr_base + SMVID_LS)); + + return SHIFT_U32(1, page_size) * (size_t)page * 1024; +} + +bool caam_hal_sm_prtn_is_free(vaddr_t jr_base, unsigned int partition) +{ + return SM_SMPO_OWNER(io_caam_read32(jr_base + SM_SMPO), partition) == + SMPO_PO_AVAIL; +} + +bool caam_hal_sm_prtn_is_owned(vaddr_t jr_base, unsigned int partition) +{ + return SM_SMPO_OWNER(io_caam_read32(jr_base + SM_SMPO), partition) == + SMPO_PO_OWNED; +} + +void caam_hal_sm_set_access_all_group(vaddr_t jr_base, unsigned int partition) +{ + io_caam_write32(jr_base + SM_SMAG1(partition), UINT32_MAX); + io_caam_write32(jr_base + SM_SMAG2(partition), UINT32_MAX); +} + +void caam_hal_sm_set_access_group(vaddr_t jr_base, unsigned int partition, + uint32_t grp1, uint32_t grp2) +{ + if (!jr_base) + return; + + if (grp1 != UINT32_MAX) + io_caam_write32(jr_base + SM_SMAG1(partition), + SHIFT_U32(1, grp1)); + + if (grp2 != UINT32_MAX) + io_caam_write32(jr_base + SM_SMAG2(partition), + SHIFT_U32(1, grp2)); +} + +void caam_hal_sm_open_access_perm(vaddr_t jr_base, unsigned int partition) +{ + io_caam_write32(jr_base + SM_SMAPR(partition), + SM_SMAPR_GRP1(UINT8_MAX) | SM_SMAPR_GRP2(UINT8_MAX)); +} + +void caam_hal_sm_set_access_perm(vaddr_t jr_base, unsigned int partition, + unsigned int grp1_perm, unsigned int grp2_perm) +{ + io_caam_write32(jr_base + SM_SMAPR(partition), + SM_SMAPR_GRP1(grp1_perm) | SM_SMAPR_GRP2(grp2_perm) | + SM_SMAPR_CSP | SM_SMAPR_SMAP_LCK | SM_SMAPR_SMAG_LCK); +} + +enum caam_status +caam_hal_sm_allocate_page(vaddr_t jr_base, + const struct caam_sm_page_desc *page_desc) +{ + unsigned int page = 0; + uint32_t status = 0; + + if (!jr_base || !page_desc) + return CAAM_BAD_PARAM; + + /* Check if pages are available */ + for (page = page_desc->page; + page < page_desc->page + page_desc->page_count; page++) { + status = issue_cmd(jr_base, page, page_desc->partition, + SM_SMCR_PAGE_INQ); + if (SM_SMCSR_PO(status) != SM_SMCSR_PO_AVAILABLE) + return CAAM_BUSY; + } + + /* Allocate pages to partition */ + for (page = page_desc->page; + page < page_desc->page + page_desc->page_count; page++) { + status = issue_cmd(jr_base, page, page_desc->partition, + SM_SMCR_PAGE_ALLOC); + if (SM_SMCSR_AERR(status) != SM_SMCSR_AERR_NO_ERROR) + return CAAM_FAILURE; + } + + /* Check if pages are available */ + for (page = page_desc->page; + page < page_desc->page + page_desc->page_count; page++) { + status = issue_cmd(jr_base, page, page_desc->partition, + SM_SMCR_PAGE_INQ); + if (SM_SMCSR_PO(status) != SM_SMCSR_PO_OWNED || + SM_SMCSR_PRTN(status) != page_desc->partition) + return CAAM_FAILURE; + } + + return CAAM_NO_ERROR; +} + +enum caam_status caam_hal_sm_deallocate_partition(vaddr_t jr_base, + unsigned int partition) +{ + unsigned int status = 0; + + if (!jr_base) + return CAAM_BAD_PARAM; + + /* De-Allocate partition and so all partition's page */ + status = issue_cmd(jr_base, 0, partition, SM_SMCR_PARTITION_DEALLOC); + if (SM_SMCSR_AERR(status) != SM_SMCSR_AERR_NO_ERROR) + return CAAM_FAILURE; + + return CAAM_NO_ERROR; +} + +enum caam_status +caam_hal_sm_deallocate_pages(vaddr_t jr_base, + const struct caam_sm_page_desc *page_desc) +{ + unsigned int page = 0; + uint32_t status = 0; + + if (!jr_base || !page_desc) + return CAAM_BAD_PARAM; + + for (page = page_desc->page; + page < page_desc->page + page_desc->page_count; page++) { + /* Deallocate page, set partition as not used */ + status = issue_cmd(jr_base, page, 0, SM_SMCR_PAGE_DEALLOC); + if (SM_SMCSR_AERR(status) != SM_SMCSR_AERR_NO_ERROR) + return CAAM_FAILURE; + } + + return CAAM_NO_ERROR; +} + +register_phys_mem(MEM_AREA_IO_SEC, SECMEM_BASE, SECMEM_SIZE); +vaddr_t caam_hal_sm_get_base(void) +{ + vaddr_t sm_base = 0; + void *fdt = NULL; + + fdt = get_dt(); + if (fdt) + caam_hal_sm_get_base_dt(fdt, &sm_base); + + if (!sm_base) + sm_base = core_mmu_get_va(SECMEM_BASE, MEM_AREA_IO_SEC, + SECMEM_SIZE); + + return sm_base; +} diff --git a/optee_os/core/drivers/crypto/caam/hal/common/registers/jr_regs.h b/optee_os/core/drivers/crypto/caam/hal/common/registers/jr_regs.h new file mode 100644 index 0000000..8fa24b0 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/registers/jr_regs.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019 NXP + * + * Brief Job Ring Registers. + */ +#ifndef __JR_REGS_H__ +#define __JR_REGS_H__ + +#include + +/* Job Ring Block Register Size */ +#define JRX_BLOCK_SIZE CFG_JR_BLOCK_SIZE +#define JRX_IDX(offset) (((offset) - JRX_BLOCK_SIZE) / JRX_BLOCK_SIZE) + +/* + * Input Ring + */ +/* Base Address */ +#define JRX_IRBAR 0x0000 +/* Size */ +#define JRX_IRSR 0x000C +/* Slots Available */ +#define JRX_IRSAR 0x0014 +/* Jobs Added */ +#define JRX_IRJAR 0x001C + +/* + * Output Ring + */ +/* Base Address */ +#define JRX_ORBAR 0x0020 +/* Size */ +#define JRX_ORSR 0x002C +/* Jobs Removed */ +#define JRX_ORJRR 0x0034 +/* Slots Full */ +#define JRX_ORSFR 0x003C + +/* Interrupt Status */ +#define JRX_JRINTR 0x004C +#define BM_JRX_JRINTR_HALT SHIFT_U32(0x3, 2) +#define JRINTR_HALT_RESUME SHIFT_U32(0x2, 2) +#define JRINTR_HALT_ONGOING SHIFT_U32(0x1, 2) +#define JRINTR_HALT_DONE SHIFT_U32(0x2, 2) +#define JRX_JRINTR_JRI BIT32(0) + +/* Configuration */ +#define JRX_JRCFGR_LS 0x0054 +#define JRX_JRCFGR_LS_ICTT(val) SHIFT_U32((val) & 0xFFFF, 16) +#define JRX_JRCFGR_LS_ICDCT(val) SHIFT_U32((val) & 0xFF, 8) +#define JRX_JRCFGR_LS_ICEN BIT32(1) +#define JRX_JRCFGR_LS_IMSK BIT32(0) + +/* Input Ring Read Index */ +#define JRX_IRRIR 0x005C + +/* Output Ring Write Index */ +#define JRX_ORWIR 0x0064 + +/* Command */ +#define JRX_JRCR 0x006C +#define JRX_JRCR_PARK BIT32(1) +#define JRX_JRCR_RESET BIT32(0) + +/* CAAM Status register - duplicated */ +#define JRX_CSTA 0x0FD4 +#define JRX_CSTA_TRNG_IDLE BIT32(2) +#define JRX_CSTA_IDLE BIT32(1) +#define JRX_CSTA_BSY BIT32(0) + +#endif /* __JR_REGS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hal/common/registers/rng_regs.h b/optee_os/core/drivers/crypto/caam/hal/common/registers/rng_regs.h new file mode 100644 index 0000000..f96acee --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/registers/rng_regs.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Brief Random Number Generator Registers. + */ +#ifndef __RNG_REGS_H__ +#define __RNG_REGS_H__ + +#include + +/* + * RNG Test Registers + */ +/* Miscellaneous Control */ +#define TRNG_MCTL 0x0600 +#define TRNG_MCTL_PRGM BIT32(16) +#define TRNG_MCTL_ERR BIT32(12) +#define TRNG_MCTL_ACC BIT32(5) +#define BM_TRNG_MCTL_SAMP_MODE SHIFT_U32(0x3, 0) +/* Use raw data in both entropy shifter and statistical checker */ +#define TRNG_MCTL_SAMP_MODE_RAW_ES_SC SHIFT_U32(1, 0) + +/* Seed Control */ +#define TRNG_SDCTL 0x0610 +#define BM_TRNG_SDCTL_ENT_DLY SHIFT_U32(0xFFFF, 16) +#define GET_TRNG_SDCTL_ENT_DLY(val) (((val) & BM_TRNG_SDCTL_ENT_DLY) >> 16) +#define TRNG_SDCTL_ENT_DLY(val) SHIFT_U32(((val) & 0xFFFF), 16) +#define TRNG_SDCTL_SAMP_SIZE(val) ((val) & 0xFFFF) + +#ifdef CFG_MX6SX +#define TRNG_SDCTL_ENT_DLY_MIN 12000 +#else +#define TRNG_SDCTL_ENT_DLY_MIN 3200 +#endif +#define TRNG_SDCTL_ENT_DLY_MAX 12800 + +/* Frequency Count Minimum Limit */ +#define TRNG_FRQMIN 0x0618 +/* Frequency Count Maximum Limit */ +#define TRNG_FRQMAX 0x061C + +/* Statistical Check Miscellaneous */ +#define TRNG_RTSCMISC 0x0604 +#define BM_TRNG_RTSCMISC_RTY_CNT SHIFT_U32(0xF, 16) +#define TRNG_RTSCMISC_RTY_CNT(val) SHIFT_U32(((val) & (0xF)), 16) +#define BM_TRNG_RTSCMISC_LRUN_MAX SHIFT_U32(0xFF, 0) +#define TRNG_RTSCMISC_LRUN_MAX(val) SHIFT_U32(((val) & (0xFF)), 0) + +/* Poker Range */ +#define TRNG_RTPKRRNG 0x0608 +#define BM_TRNG_RTPKRRNG_PKR_RNG SHIFT_U32(0xFFFF, 0) +#define TRNG_RTPKRRNG_PKR_RNG(val) SHIFT_U32(((val) & (0xFFFF)), 0) + +/* Poker Maximum Limit */ +#define TRNG_RTPKRMAX 0x060C +#define BM_TRNG_RTPKRMAX_PKR_MAX SHIFT_U32(0xFFFFFF, 0) +#define TRNG_RTPKRMAX_PKR_MAX(val) SHIFT_U32(((val) & (0xFFFFFF)), 0) + +/* Statistical Check Monobit Limit */ +#define TRNG_RTSCML 0x0620 +#define BM_TRNG_RTSCML_MONO_RNG SHIFT_U32(0xFFFF, 16) +#define TRNG_RTSCML_MONO_RNG(val) SHIFT_U32(((val) & (0xFFFF)), 16) +#define BM_TRNG_RTSCML_MONO_MAX SHIFT_U32(0xFFFF, 0) +#define TRNG_RTSCML_MONO_MAX(val) SHIFT_U32(((val) & (0xFFFF)), 0) + +/* Statistical Check Run Length 1 Limit */ +#define TRNG_RTSCR1L 0x0624 +#define BM_TRNG_RTSCR1L_RUN1_RNG SHIFT_U32(0x7FFF, 16) +#define TRNG_RTSCR1L_RUN1_RNG(val) SHIFT_U32(((val) & (0x7FFF)), 16) +#define BM_TRNG_RTSCR1L_RUN1_MAX SHIFT_U32(0x7FFF, 0) +#define TRNG_RTSCR1L_RUN1_MAX(val) SHIFT_U32(((val) & (0x7FFF)), 0) + +/* Statistical Check Run Length 2 Limit */ +#define TRNG_RTSCR2L 0x0628 +#define BM_TRNG_RTSCR2L_RUN2_RNG SHIFT_U32(0x3FFF, 16) +#define TRNG_RTSCR2L_RUN2_RNG(val) SHIFT_U32(((val) & (0x3FFF)), 16) +#define BM_TRNG_RTSCR2L_RUN2_MAX SHIFT_U32(0x3FFF, 0) +#define TRNG_RTSCR2L_RUN2_MAX(val) SHIFT_U32(((val) & (0x3FFF)), 0) + +/* Statistical Check Run Length 3 Limit */ +#define TRNG_RTSCR3L 0x062C +#define BM_TRNG_RTSCR3L_RUN3_RNG SHIFT_U32(0x1FFF, 16) +#define TRNG_RTSCR3L_RUN3_RNG(val) SHIFT_U32(((val) & (0x1FFF)), 16) +#define BM_TRNG_RTSCR3L_RUN3_MAX SHIFT_U32(0x1FFF, 0) +#define TRNG_RTSCR3L_RUN3_MAX(val) SHIFT_U32(((val) & (0x1FFF)), 0) + +/* Statistical Check Run Length 4 Limit */ +#define TRNG_RTSCR4L 0x0630 +#define BM_TRNG_RTSCR4L_RUN4_RNG SHIFT_U32(0xFFF, 16) +#define TRNG_RTSCR4L_RUN4_RNG(val) SHIFT_U32(((val) & (0xFFF)), 16) +#define BM_TRNG_RTSCR4L_RUN4_MAX SHIFT_U32(0xFFF, 0) +#define TRNG_RTSCR4L_RUN4_MAX(val) SHIFT_U32(((val) & (0xFFF)), 0) + +/* Statistical Check Run Length 5 Limit */ +#define TRNG_RTSCR5L 0x0634 +#define BM_TRNG_RTSCR5L_RUN5_RNG SHIFT_U32(0x7FF, 16) +#define TRNG_RTSCR5L_RUN5_RNG(val) SHIFT_U32(((val) & (0x7FF)), 16) +#define BM_TRNG_RTSCR5L_RUN5_MAX SHIFT_U32(0x7FF, 0) +#define TRNG_RTSCR5L_RUN5_MAX(val) SHIFT_U32(((val) & (0x7FF)), 0) + +/* Statistical Check Run Length 6+ Limit */ +#define TRNG_RTSCR6PL 0x0638 +#define BM_TRNG_RTSCR6PL_RUN6P_RNG SHIFT_U32(0x7FF, 16) +#define TRNG_RTSCR6PL_RUN6P_RNG(val) SHIFT_U32(((val) & (0x7FF)), 16) +#define BM_TRNG_RTSCR6PL_RUN6P_MAX SHIFT_U32(0x7FF, 0) +#define TRNG_RTSCR6PL_RUN6P_MAX(val) SHIFT_U32(((val) & (0x7FF)), 0) + +/* + * RNG Registers + */ +/* Status */ +#define RNG_STA 0x06C0 + +#define RNG_STA_SKVN BIT32(30) +#define RNG_STA_IF1 BIT32(1) +#define RNG_STA_IF0 BIT32(0) + +#endif /* __RNG_REGS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hal/common/registers/sm_regs.h b/optee_os/core/drivers/crypto/caam/hal/common/registers/sm_regs.h new file mode 100644 index 0000000..f8a328c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/registers/sm_regs.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 NXP + */ +#ifndef __SM_REGS_H__ +#define __SM_REGS_H__ + +#include + +/* Access Permission */ +#define SM_SMAPR(prtn) (0x0A04 + (prtn) * 16) +#define SM_SMAPR_GRP1(perm) SHIFT_U32((perm) & 0xF, 0) +#define SM_SMAPR_GRP2(perm) SHIFT_U32((perm) & 0xF, 4) +#define SM_SMAPR_CSP BIT32(15) +#define SM_SMAPR_SMAP_LCK BIT32(13) +#define SM_SMAPR_SMAG_LCK BIT32(12) + +/* Access Group */ +#define SM_SMAG2(prtn) (0x0A08 + (prtn) * 16) +#define SM_SMAG1(prtn) (0x0A0C + (prtn) * 16) + +/* Command */ +#define SM_SMCR 0x0BE4 +#define SM_SMCR_PAGE(page) SHIFT_U32((page) & UINT16_MAX, 16) +#define SM_SMCR_PRTN(prtn) SHIFT_U32((prtn) & 0xF, 8) +#define SM_SMCR_CMD(cmd) SHIFT_U32((cmd) & 0xF, 0) +#define SM_SMCR_PAGE_ALLOC 0x1 +#define SM_SMCR_PAGE_DEALLOC 0x2 +#define SM_SMCR_PARTITION_DEALLOC 0x3 +#define SM_SMCR_PAGE_INQ 0x5 + +/* Command Status */ +#define SM_SMCSR 0x0BEC +#define SM_SMCSR_CERR(val) (((val) >> 14) & 0x3) +#define SM_SMCSR_CERR_NO_ERROR 0x0 +#define SM_SMCSR_CERR_NOT_COMPLETED 0x1 +#define SM_SMCSR_AERR(val) (((val) >> 12) & 0x3) +#define SM_SMCSR_AERR_NO_ERROR 0x0 +#define SM_SMCSR_PO(val) (((val) >> 6) & 0x3) +#define SM_SMCSR_PO_AVAILABLE 0x0 +#define SM_SMCSR_PO_UNKNOWN 0x1 +#define SM_SMCSR_PO_OWNED_BY_OTHER 0x2 +#define SM_SMCSR_PO_OWNED 0x3 +#define SM_SMCSR_PRTN(val) ((val) & 0x3) + +/* Partition Owners */ +#define SM_SMPO 0x0FBC +#define SM_SMPO_PART(prtn) ((prtn) * 2) +#define SM_SMPO_OWNER(val, prtn) (((val) >> SM_SMPO_PART(prtn)) & 0x3) +#define SMPO_PO_AVAIL 0x0 +#define SMPO_PO_OWNED 0x3 + +#endif /* __SM_REGS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hal/common/registers/version_regs.h b/optee_os/core/drivers/crypto/caam/hal/common/registers/version_regs.h new file mode 100644 index 0000000..9691117 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/registers/version_regs.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2020 NXP + * + * Brief Version Registers. + */ +#ifndef __VERSION_REGS_H__ +#define __VERSION_REGS_H__ + +#include + +/* Compile Time Parameters */ +#define CTPR_MS 0x0FA8 +#define BM_CTPR_MS_RNG_I SHIFT_U32(0x7, 8) +#define GET_CTPR_MS_RNG_I(val) (((val) & BM_CTPR_MS_RNG_I) >> 8) + +#define CTPR_LS 0x0FAC +#define BM_CTPR_LS_SPLIT_KEY BIT(14) +#define GET_CTPR_LS_SPLIT_KEY(val) (((val) & BM_CTPR_LS_SPLIT_KEY) >> 14) + +/* Secure Memory Version ID */ +#define SMVID_MS 0x0FD8 +#define BM_SMVID_MS_MAX_NPAG SHIFT_U32(0x3FF, 16) +#define GET_SMVID_MS_MAX_NPAG(val) (((val) & BM_SMVID_MS_MAX_NPAG) >> 16) +#define BM_SMVID_MS_NPRT SHIFT_U32(0xF, 12) +#define GET_SMVID_MS_NPRT(val) (((val) & BM_SMVID_MS_NPRT) >> 12) + +#define SMVID_LS 0x0FDC +#define BM_SMVID_LS_PSIZ SHIFT_U32(0x7, 16) +#define GET_SMVID_LS_PSIZ(val) (((val) & BM_SMVID_LS_PSIZ) >> 16) + +/* CHA Cluster Block Version ID */ +#define CCBVID 0x0FE4 +#define BM_CCBVID_CAAM_ERA SHIFT_U32(0xFF, 24) +#define GET_CCBVID_CAAM_ERA(val) (((val) & BM_CCBVID_CAAM_ERA) >> 24) + +/* CHA Version ID */ +#define CHAVID_LS 0x0FEC +#define BM_CHAVID_LS_RNGVID SHIFT_U32(0xF, 16) +#define GET_CHAVID_LS_RNGVID(val) (((val) & BM_CHAVID_LS_RNGVID) >> 16) +#define BM_CHAVID_LS_MDVID SHIFT_U32(0xF, 12) + +#define CHAVID_LS_MDVID_LP256 SHIFT_U32(0, 12) + +/* CHA Number */ +#define CHANUM_MS 0x0FF0 +#define BM_CHANUM_MS_JRNUM SHIFT_U32(0xF, 28) +#define GET_CHANUM_MS_JRNUM(val) (((val) & BM_CHANUM_MS_JRNUM) >> 28) + +#define CHANUM_LS 0x0FF4 +#define BM_CHANUM_LS_PKNUM SHIFT_U32(0xF, 28) +#define GET_CHANUM_LS_PKNUM(val) (((val) & BM_CHANUM_LS_PKNUM) >> 28) +#define BM_CHANUM_LS_MDNUM SHIFT_U32(0xF, 12) +#define GET_CHANUM_LS_MDNUM(val) (((val) & BM_CHANUM_LS_MDNUM) >> 12) + +/* PKHA Version for Era > 10 */ +#define PKHA_VERSION 0x0E8C +#define BM_PKHA_VERSION_PKNUM 0xFF +#define GET_PKHA_VERSION_PKNUM(val) ((val) & BM_PKHA_VERSION_PKNUM) + +/* MDHA Version for Era > 10 */ +#define MDHA_VERSION 0xE94 +#define BM_MDHA_VERSION_MDNUM 0xFF +#define GET_MDHA_VERSION_MDNUM(val) ((val) & BM_MDHA_VERSION_MDNUM) +#define BM_MDHA_VERSION_MDVID SHIFT_U32(0xFF, 24) + +#define MDHA_VERSION_MDVID_LP256 SHIFT_U32(0, 24) + +/* RNG Version for Era > 10 */ +#define RNG_VERSION 0x0EF8 +#define BM_RNG_VERSION_VID SHIFT_U32(0xFF, 24) +#define GET_RNG_VERSION_VID(val) ((val) & BM_RNG_VERSION_VID) + +/* JR Version for Era > 10 */ +#define JR_VERSION 0x0EF8 +#define BM_JR_VERSION_JRNUM 0xFF +#define GET_JR_VERSION_JRNUM(val) ((val) & BM_JR_VERSION_JRNUM) + +#endif /* __VERSION_REGS_H__ */ + diff --git a/optee_os/core/drivers/crypto/caam/hal/common/sub.mk b/optee_os/core/drivers/crypto/caam/hal/common/sub.mk new file mode 100644 index 0000000..dbab332 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/common/sub.mk @@ -0,0 +1,10 @@ +incdirs-y += ../../include +incdirs-y += ../$(CAAM_HAL_DIR) +incdirs-y += . + +srcs-$(CFG_DT) += hal_cfg_dt.c +srcs-y += hal_cfg.c +srcs-y += hal_rng.c +srcs-y += hal_jr.c +srcs-y += hal_ctrl.c +srcs-$(CFG_NXP_CAAM_SM_DRV) += hal_sm.c diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx6.c b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx6.c new file mode 100644 index 0000000..3333a22 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx6.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Clock functions. + */ +#include +#include +#include +#include + +void caam_hal_clk_enable(bool enable) +{ + vaddr_t ccm_base = (vaddr_t)phys_to_virt(CCM_BASE, MEM_AREA_IO_SEC, + CCM_SIZE); + uint32_t reg = 0; + uint32_t mask = 0; + + reg = io_read32(ccm_base + CCM_CCGR0); + + mask = BM_CCM_CCGR0_CAAM_WRAPPER_IPG | BM_CCM_CCGR0_CAAM_WRAPPER_ACLK | + BM_CCM_CCGR0_CAAM_SECURE_MEM; + + if (enable) + reg |= mask; + else + reg &= ~mask; + + io_write32(ccm_base + CCM_CCGR0, reg); + + if (!soc_is_imx6ul()) { + /* EMI slow clk */ + reg = io_read32(ccm_base + CCM_CCGR6); + mask = BM_CCM_CCGR6_EMI_SLOW; + + if (enable) + reg |= mask; + else + reg &= ~mask; + + io_write32(ccm_base + CCM_CCGR6, reg); + } +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx7.c b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx7.c new file mode 100644 index 0000000..51f29e6 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx7.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Clock functions. + */ +#include +#include +#include +#include + +void caam_hal_clk_enable(bool enable) +{ + vaddr_t ccm_base = (vaddr_t)phys_to_virt(CCM_BASE, MEM_AREA_IO_SEC, 1); + + if (enable) { + io_write32(ccm_base + CCM_CCGRx_SET(CCM_CLOCK_DOMAIN_CAAM), + CCM_CCGRx_ALWAYS_ON(0)); + } else { + io_write32(ccm_base + CCM_CCGRx_CLR(CCM_CLOCK_DOMAIN_CAAM), + CCM_CCGRx_ALWAYS_ON(0)); + } +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx7ulp.c b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx7ulp.c new file mode 100644 index 0000000..9cb1594 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_clk_mx7ulp.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Clock functions. + */ +#include +#include +#include +#include + +void caam_hal_clk_enable(bool enable) +{ + vaddr_t pcc2_base = (vaddr_t)phys_to_virt(PCC2_BASE, MEM_AREA_IO_SEC, + PCC_CAAM + sizeof(uint32_t)); + + if (enable) + io_write32(pcc2_base + PCC_CAAM, PCC_ENABLE_CLOCK); + else + io_write32(pcc2_base + PCC_CAAM, PCC_DISABLE_CLOCK); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_ctrl.c b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_ctrl.c new file mode 100644 index 0000000..d0a5086 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_ctrl.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Controller Hardware Abstration Layer. + * Implementation of primitives to access HW + */ +#include +#include +#include +#include + +/* + * List of control registers to save/restore + */ +static const struct reglist ctrl_backup[] = { + BACKUP_REG(MCFGR, 1, 0, 0), +#ifdef CFG_NXP_CAAM_MP_DRV + BACKUP_REG(SCFGR, 1, BM_SCFGR_MPMRL | BM_SCFGR_MPCURVE, 0), +#else + /* For device not supporting MP (bits not defined) */ + BACKUP_REG(SCFGR, 1, 0, 0), +#endif +}; + +void caam_hal_ctrl_init(vaddr_t baseaddr) +{ + /* Enable DECO watchdogs */ + io_setbits32(baseaddr + MCFGR, MCFGR_WDE); + + /* + * ERRATA: mx6 devices have an issue wherein AXI bus transactions + * may not occur in the correct order. This isn't a problem running + * single descriptors, but can be if running multiple concurrent + * descriptors. Reworking the driver to throttle to single requests + * is impractical, thus the workaround is to limit the AXI pipeline + * to a depth of 1 (default depth is 4) to preclude this situation + * from occurring. + * + * mx7 devices, this bit has no effect. + */ + io_mask32(baseaddr + MCFGR, MCFGR_AXIPIPE(1), BM_MCFGR_AXIPIPE); + + caam_pwr_add_backup(baseaddr, ctrl_backup, ARRAY_SIZE(ctrl_backup)); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_jr.c b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_jr.c new file mode 100644 index 0000000..cda6f75 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/hal_jr.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Job Rings Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include +#include +#include +#include + +/* + * List of JR configuration registers to save/restore + */ +static const struct reglist jrcfg_backup[] = { + BACKUP_REG(JR0MIDR_LS, 1, 0, 0), + BACKUP_REG(JR0MIDR_MS, 1, 0, 0), +}; + +enum caam_status caam_hal_jr_setowner(vaddr_t ctrl_base, paddr_t jr_offset, + enum caam_jr_owner owner) +{ + enum caam_status retstatus = CAAM_FAILURE; + uint32_t val = 0; + uint32_t cfg_ms = 0; + uint32_t cfg_ls = 0; + unsigned int jr_idx = JRX_IDX(jr_offset); + + /* Read the Job Ring Lock bit */ + val = io_caam_read32(ctrl_base + JRxMIDR_MS(jr_idx)); + HAL_TRACE("JR%" PRIu32 "MIDR_MS value 0x%" PRIx32, jr_idx, val); + + /* Prepare the Job Ring MS/LS registers */ + if (owner & JROWNER_SECURE) { + /* Configuration only lock for the Secure JR */ + cfg_ms = JRxMIDR_MS_JROWN_MID(owner & ~JROWNER_SECURE); + cfg_ms |= JRxMIDR_MS_AMTD; +#ifdef CFG_NXP_CAAM_RUNTIME_JR + cfg_ms |= JRxMIDR_MS_LAMTD; + cfg_ms |= JRxMIDR_MS_LMID; +#endif + cfg_ls = JRxMIDR_LS_SEQ_MID(owner & ~JROWNER_SECURE); + cfg_ls |= JRxMIDR_LS_NONSEQ_MID(owner & ~JROWNER_SECURE); + } else { + cfg_ms = JRxMIDR_MS_JROWN_MID(owner) | JRxMIDR_MS_JROWN_NS; + cfg_ls = JRxMIDR_LS_SEQ_MID(owner) | JRxMIDR_LS_SEQ_NS; + cfg_ls |= JRxMIDR_LS_NONSEQ_MID(owner) | JRxMIDR_LS_NONSEQ_NS; + } + + if (val & JRxMIDR_MS_LMID) { + /* + * Configuration already locked, check it is the + * expected configuration. + */ + HAL_TRACE("JR%" PRIu32 "MIDR_MS value 0x%" PRIx32 " (0x%" PRIx32 + ")", + jr_idx, val, cfg_ms); + if ((cfg_ms | JRxMIDR_MS_LMID) == val) { + /* + * Read the LS register and compare with expected + * value + */ + val = io_caam_read32(ctrl_base + JRxMIDR_LS(jr_idx)); + HAL_TRACE("JR%" PRIu32 "MIDR_LS value 0x%" PRIx32 + " (0x%" PRIX32 ")", + jr_idx, val, cfg_ls); + if (val == cfg_ls) + retstatus = CAAM_NO_ERROR; + } + } else { + HAL_TRACE("JR%" PRIu32 "MIDR_LS set value 0x%" PRIx32, jr_idx, + cfg_ls); + HAL_TRACE("JR%" PRIu32 "MIDR_MS set value 0x%" PRIx32, jr_idx, + cfg_ms); + /* Set the configuration */ + io_caam_write32(ctrl_base + JRxMIDR_LS(jr_idx), cfg_ls); + io_caam_write32(ctrl_base + JRxMIDR_MS(jr_idx), cfg_ms); + retstatus = CAAM_NO_ERROR; + } + + return retstatus; +} + +void caam_hal_jr_prepare_backup(vaddr_t ctrl_base, paddr_t jr_offset) +{ + unsigned int jr_idx = JRX_IDX(jr_offset); + + caam_pwr_add_backup(ctrl_base + (jr_idx * JRxMIDR_SIZE), jrcfg_backup, + ARRAY_SIZE(jrcfg_backup)); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_6_7/registers/ctrl_regs.h b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/registers/ctrl_regs.h new file mode 100644 index 0000000..4edc68d --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/registers/ctrl_regs.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Brief Control Registers. + */ +#ifndef __CTRL_REGS_H__ +#define __CTRL_REGS_H__ + +#include + +/* Master Configuration */ +#define MCFGR 0x0004 +#define MCFGR_WDE BIT32(30) +#define MCFGR_AXIPIPE(val) SHIFT_U32(val, 4) +#define BM_MCFGR_AXIPIPE SHIFT_U32(0xF, 4) + +/* Job Ring x MID */ +#define JRxMIDR_SIZE 0x8 +#define JR0MIDR_MS 0x0010 +#define JR0MIDR_LS 0x0014 +#define JRxMIDR_MS(idx) (JR0MIDR_MS + (idx) * JRxMIDR_SIZE) +#define JRxMIDR_LS(idx) (JR0MIDR_LS + (idx) * JRxMIDR_SIZE) + +#define JRxMIDR_MS_LMID BIT32(31) +#define JRxMIDR_MS_LAMTD BIT32(17) +#define JRxMIDR_MS_AMTD BIT32(16) +#if !defined(CFG_MX7ULP) +#define JRxMIDR_MS_JROWN_NS BIT32(3) +#define JRxMIDR_MS_JROWN_MID(val) SHIFT_U32((val) & 0x7, 0) + +#define JRxMIDR_LS_NONSEQ_NS BIT32(19) +#define JRxMIDR_LS_NONSEQ_MID(val) SHIFT_U32((val) & 0x7, 16) +#define JRxMIDR_LS_SEQ_NS BIT32(3) +#define JRxMIDR_LS_SEQ_MID(val) SHIFT_U32((val) & 0x7, 0) +#else +#define JRxMIDR_MS_JROWN_NS BIT32(4) +#define JRxMIDR_MS_JROWN_MID(val) SHIFT_U32((val) & 0xF, 0) + +#define JRxMIDR_LS_NONSEQ_NS BIT32(20) +#define JRxMIDR_LS_NONSEQ_MID(val) SHIFT_U32((val) & 0xF, 16) +#define JRxMIDR_LS_SEQ_NS BIT32(4) +#define JRxMIDR_LS_SEQ_MID(val) SHIFT_U32((val) & 0xF, 0) +#endif + +/* Security Configuration */ +#define SCFGR 0x000C +#define BS_SCFGR_MPCURVE 28 +#define BM_SCFGR_MPCURVE SHIFT_U32(0xF, BS_SCFGR_MPCURVE) +#define BM_SCFGR_MPMRL BIT32(26) + +/* Manufacturing Protection Message */ +#define MPMR 0x0380 +#define MPMR_NB_REG 0x20 + +#endif /* __CTRL_REGS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_6_7/sub.mk b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/sub.mk new file mode 100644 index 0000000..b306fdc --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_6_7/sub.mk @@ -0,0 +1,9 @@ +incdirs-y += ../common +incdirs-y += ../../include +incdirs-y += . + +srcs-$(CFG_MX6) += hal_clk_mx6.c +srcs-$(CFG_MX7) += hal_clk_mx7.c +srcs-$(CFG_MX7ULP) += hal_clk_mx7ulp.c +srcs-y += hal_ctrl.c +srcs-y += hal_jr.c diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_clk.c b/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_clk.c new file mode 100644 index 0000000..eee59d6 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_clk.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018 NXP + * + * Brief CAAM Clock functions. + */ +#include +#include + +void caam_hal_clk_enable(bool enable __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_ctrl.c b/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_ctrl.c new file mode 100644 index 0000000..571194e --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_ctrl.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Controller Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include + +void caam_hal_ctrl_init(vaddr_t baseaddr) +{ + /* Enable DECO watchdogs */ + io_setbits32(baseaddr + MCFGR, MCFGR_WDE); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_jr.c b/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_jr.c new file mode 100644 index 0000000..f188815 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8m/hal_jr.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Job Rings Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include +#include +#include +#include + +/* + * List of JR configuration registers to save/restore + */ +static const struct reglist jrcfg_backup[] = { + BACKUP_REG(JR0DID_MS, 1, 0, 0), + BACKUP_REG(JR0DID_LS, 1, 0, 0), +}; + +enum caam_status caam_hal_jr_setowner(vaddr_t ctrl_base, paddr_t jr_offset, + enum caam_jr_owner owner) +{ + enum caam_status retstatus = CAAM_FAILURE; + uint32_t val = 0; + uint32_t cfg_ms = 0; + uint32_t cfg_ls = 0; + unsigned int jr_idx = JRX_IDX(jr_offset); + + /* Read the Job Ring Lock bit */ + val = io_caam_read32(ctrl_base + JRxDID_MS(jr_idx)); + HAL_TRACE("JR%" PRIu32 "DID_MS value 0x%" PRIx32, jr_idx, val); + + /* Prepare the Job Ring MS/LS registers */ + if (owner & JROWNER_SECURE) { + /* Configuration only locked for the Secure JR */ + cfg_ms = JRxDID_MS_PRIM_DID(owner & ~JROWNER_SECURE); + cfg_ms |= JRxDID_MS_PRIM_TZ | JRxDID_MS_TZ_OWN; + cfg_ms |= JRxDID_MS_AMTD; + cfg_ms |= JRxDID_MS_PRIM_ICID(owner & ~JROWNER_SECURE); +#ifdef CFG_NXP_CAAM_RUNTIME_JR + cfg_ms |= JRxDID_MS_LDID; + cfg_ms |= JRxDID_MS_LAMTD; +#endif + } else { + cfg_ms = JRxDID_MS_PRIM_DID(owner); + cfg_ms |= JRxDID_MS_PRIM_ICID(owner); + } + + if (val & JRxDID_MS_LDID) { + /* + * Configuration already locked, check it is the + * expected configuration. + */ + HAL_TRACE("JR%" PRIu32 "DID_MS value 0x%" PRIx32 " (0x%" PRIx32 + ")", + jr_idx, val, cfg_ms); + if ((cfg_ms | JRxDID_MS_LDID) == val) { + /* Read LS register and compare with expected value */ + val = io_caam_read32(ctrl_base + JRxDID_LS(jr_idx)); + HAL_TRACE("JR%" PRIu32 "DID_LS value 0x%" PRIx32 + " (0x%" PRIx32 ")", + jr_idx, val, cfg_ls); + if (val == cfg_ls) + retstatus = CAAM_NO_ERROR; + } + } else { + HAL_TRACE("JR%" PRIu32 "DID_LS set value 0x%" PRIx32, jr_idx, + cfg_ls); + HAL_TRACE("JR%" PRIu32 "DID_MS set value 0x%" PRIx32, jr_idx, + cfg_ms); + /* Set the configuration */ + io_caam_write32(ctrl_base + JRxDID_LS(jr_idx), cfg_ls); + io_caam_write32(ctrl_base + JRxDID_MS(jr_idx), cfg_ms); + retstatus = CAAM_NO_ERROR; + } + + return retstatus; +} + +void caam_hal_jr_prepare_backup(vaddr_t ctrl_base, paddr_t jr_offset) +{ + unsigned int jr_idx = JRX_IDX(jr_offset); + + caam_pwr_add_backup(ctrl_base + (jr_idx * JRxDID_SIZE), jrcfg_backup, + ARRAY_SIZE(jrcfg_backup)); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8m/registers/ctrl_regs.h b/optee_os/core/drivers/crypto/caam/hal/imx_8m/registers/ctrl_regs.h new file mode 100644 index 0000000..2235d28 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8m/registers/ctrl_regs.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Brief Control Registers. + */ +#ifndef __CTRL_REGS_H__ +#define __CTRL_REGS_H__ + +#include + +/* Master Configuration */ +#define MCFGR 0x0004 +#define MCFGR_WDE BIT32(30) + +/* Job Ring x MID */ +#define JRxDID_SIZE 0x8 +#define JR0DID_MS 0x0010 +#define JR0DID_LS 0x0014 +#define JRxDID_MS(idx) (JR0DID_MS + ((idx) * JRxDID_SIZE)) +#define JRxDID_LS(idx) (JR0DID_LS + ((idx) * JRxDID_SIZE)) + +#define JRxDID_MS_LDID BIT32(31) +#define JRxDID_MS_PRIM_ICID(val) SHIFT_U32(((val) & 0x3FF), 19) +#define JRxDID_MS_LAMTD BIT32(17) +#define JRxDID_MS_AMTD BIT32(16) +#define JRxDID_MS_TZ_OWN BIT32(15) +#define JRxDID_MS_PRIM_TZ BIT32(4) +#define JRxDID_MS_PRIM_DID(val) SHIFT_U32(((val) & 0xF), 0) + +/* Security Configuration */ +#define SCFGR 0x000C +#define BS_SCFGR_MPCURVE 28 +#define BM_SCFGR_MPCURVE SHIFT_U32(0xF, BS_SCFGR_MPCURVE) +#define BM_SCFGR_MPMRL BIT32(26) + +/* Manufacturing Protection Message */ +#define MPMR 0x0380 +#define MPMR_NB_REG U(32) + +/* Secure Memory physical base address */ +#define JRX_SMVBAR(idx) (0x0184 + (idx) * 8) + +#endif /* __CTRL_REGS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8m/sub.mk b/optee_os/core/drivers/crypto/caam/hal/imx_8m/sub.mk new file mode 100644 index 0000000..5231cce --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8m/sub.mk @@ -0,0 +1,7 @@ +incdirs-y += ../common +incdirs-y += ../../include +incdirs-y += . + +srcs-y += hal_clk.c +srcs-y += hal_ctrl.c +srcs-y += hal_jr.c diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_cfg.c b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_cfg.c new file mode 100644 index 0000000..2fff1c1 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_cfg.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2021 NXP + */ +#include + +void caam_hal_cfg_setup_nsjobring(struct caam_jrcfg *jrcfg __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_clk.c b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_clk.c new file mode 100644 index 0000000..fe8bf25 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_clk.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2021 NXP + */ +#include +#include + +void caam_hal_clk_enable(bool enable __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_ctrl.c b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_ctrl.c new file mode 100644 index 0000000..dfec080 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_ctrl.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2021 NXP + */ +#include +#include + +void caam_hal_ctrl_init(vaddr_t baseaddr __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_jr.c b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_jr.c new file mode 100644 index 0000000..f92745f --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_jr.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2021 NXP + */ +#include +#include +#include +#include +#include + +enum caam_status caam_hal_jr_setowner(vaddr_t ctrl_base __unused, + paddr_t jr_offset __unused, + enum caam_jr_owner owner __unused) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + ret = imx_sc_driver_init(); + if (ret != TEE_SUCCESS) + return CAAM_FAILURE; + + ret = imx_sc_rm_enable_jr(CFG_JR_INDEX); + if (ret != TEE_SUCCESS) + return CAAM_FAILURE; + else + return CAAM_NO_ERROR; +} + +void caam_hal_jr_prepare_backup(vaddr_t ctrl_base __unused, + paddr_t jr_offset __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_rng.c b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_rng.c new file mode 100644 index 0000000..a561ab1 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8q/hal_rng.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2021 NXP + */ +#include +#include +#include +#include +#include + +enum caam_status caam_hal_rng_instantiated(vaddr_t baseaddr __unused) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + ret = imx_sc_seco_start_rng(); + if (ret != TEE_SUCCESS) + return CAAM_FAILURE; + else + return CAAM_NO_ERROR; +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8q/registers/ctrl_regs.h b/optee_os/core/drivers/crypto/caam/hal/imx_8q/registers/ctrl_regs.h new file mode 100644 index 0000000..70296b3 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8q/registers/ctrl_regs.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020 NXP + */ +#ifndef __CTRL_REGS_H__ +#define __CTRL_REGS_H__ + +/* Global includes */ +#include + +/* Job Ring x MID */ +#define JRxDID_SIZE 0x8 +#define JR0DID_MS 0x0010 +#define JR0DID_LS 0x0014 +#define JRxDID_MS(idx) (JR0DID_MS + (idx) * (JRxDID_SIZE)) +#define JRxDID_LS(idx) (JR0DID_LS + (idx) * (JRxDID_SIZE)) + +#define JRxDID_MS_LDID BIT32(31) +#define JRxDID_MS_PRIM_ICID(val) SHIFT_U32((val) & (0x3FF), 19) +#define JRxDID_MS_LAMTD BIT32(17) +#define JRxDID_MS_AMTD BIT32(16) +#define JRxDID_MS_TZ_OWN BIT32(15) +#define JRxDID_MS_PRIM_TZ BIT32(4) +#define JRxDID_MS_PRIM_DID(val) SHIFT_U32((val) & (0xF), 0) + +/* Security Configuration */ +#define SCFGR 0x000C +#define BS_SCFGR_MPCURVE 28 +#define BM_SCFGR_MPCURVE SHIFT_U32(0xF, BS_SCFGR_MPCURVE) +#define BM_SCFGR_MPMRL BIT32(26) + +/* Secure Memory Virtual Base Address */ +#define JRX_SMVBAR(idx) (0x0184 + (idx) * (8)) + +/* Manufacturing Protection Message */ +#define MPMR 0x0380 +#define MPMR_NB_REG 0x20 + +#endif /* __CTRL_REGS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8q/sub.mk b/optee_os/core/drivers/crypto/caam/hal/imx_8q/sub.mk new file mode 100644 index 0000000..72f3eab --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8q/sub.mk @@ -0,0 +1,9 @@ +incdirs-y += ../common +incdirs-y += ../../include +incdirs-y += . + +srcs-y += hal_clk.c +srcs-y += hal_ctrl.c +srcs-y += hal_jr.c +srcs-y += hal_cfg.c +srcs-y += hal_rng.c diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_clk.c b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_clk.c new file mode 100644 index 0000000..1f49479 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_clk.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * Brief CAAM Clock functions. + */ +#include +#include +#include +#include + +void caam_hal_clk_enable(bool enable) +{ + vaddr_t pcc3_base = (vaddr_t)phys_to_virt(PCC3_BASE, MEM_AREA_IO_SEC, + PCC3_SIZE); + + if (enable) + io_setbits32(pcc3_base + PCC_CAAM, PCC_ENABLE_CLOCK); + else + io_clrbits32(pcc3_base + PCC_CAAM, PCC_ENABLE_CLOCK); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_ctrl.c b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_ctrl.c new file mode 100644 index 0000000..8ec1fb8 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_ctrl.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * Brief CAAM Controller Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include +#include + +void caam_hal_ctrl_init(vaddr_t baseaddr) +{ + /* Enable DECO watchdogs */ + io_setbits32(baseaddr + MCFGR, MCFGR_WDE); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_jr.c b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_jr.c new file mode 100644 index 0000000..a022bb5 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/hal_jr.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * Brief CAAM Job Rings Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include +#include +#include +#include +#include + +/* + * List of JR configuration registers to save/restore + */ +static const struct reglist jrcfg_backup[] = { + BACKUP_REG(JR0DID_MS, 1, 0, 0), + BACKUP_REG(JR0DID_LS, 1, 0, 0), +}; + +enum caam_status caam_hal_jr_setowner(vaddr_t ctrl_base, paddr_t jr_offset, + enum caam_jr_owner owner) +{ + enum caam_status retstatus = CAAM_FAILURE; + uint32_t val = 0; + uint32_t cfg_ms = 0; + uint32_t cfg_ls = 0; + unsigned int jr_idx = JRX_IDX(jr_offset); + + /* Read the Job Ring Lock bit */ + val = io_caam_read32(ctrl_base + JRxDID_MS(jr_idx)); + HAL_TRACE("JR%" PRIu32 "DID_MS value 0x%" PRIx32, jr_idx, val); + + /* Prepare the Job Ring MS/LS registers */ + if (owner & JROWNER_SECURE) { + /* Configuration only locked for the Secure JR */ + cfg_ms = JRxDID_MS_PRIM_DID(owner & ~JROWNER_SECURE); + cfg_ms |= JRxDID_MS_PRIM_TZ | JRxDID_MS_TZ_OWN; + cfg_ms |= JRxDID_MS_AMTD; + cfg_ms |= JRxDID_MS_PRIM_ICID(owner & ~JROWNER_SECURE); + if (IS_ENABLED(CFG_NXP_CAAM_RUNTIME_JR)) { + cfg_ms |= JRxDID_MS_LDID; + cfg_ms |= JRxDID_MS_LAMTD; + } + } else { + cfg_ms = JRxDID_MS_PRIM_DID(owner); + cfg_ms |= JRxDID_MS_PRIM_ICID(owner); + } + + if (val & JRxDID_MS_LDID) { + /* + * Configuration already locked, check it is the + * expected configuration. + */ + HAL_TRACE("JR%" PRIu32 "DID_MS value 0x%" PRIx32 " (0x%" PRIx32 + ")", + jr_idx, val, cfg_ms); + if ((cfg_ms | JRxDID_MS_LDID) == val) { + /* Read LS register and compare with expected value */ + val = io_caam_read32(ctrl_base + JRxDID_LS(jr_idx)); + HAL_TRACE("JR%" PRIu32 "DID_LS value 0x%" PRIx32 + " (0x%" PRIx32 ")", + jr_idx, val, cfg_ls); + if (val == cfg_ls) + retstatus = CAAM_NO_ERROR; + } + } else { + HAL_TRACE("JR%" PRIu32 "DID_LS set value 0x%" PRIx32, jr_idx, + cfg_ls); + HAL_TRACE("JR%" PRIu32 "DID_MS set value 0x%" PRIx32, jr_idx, + cfg_ms); + /* Set the configuration */ + io_caam_write32(ctrl_base + JRxDID_LS(jr_idx), cfg_ls); + io_caam_write32(ctrl_base + JRxDID_MS(jr_idx), cfg_ms); + retstatus = CAAM_NO_ERROR; + } + + return retstatus; +} + +void caam_hal_jr_prepare_backup(vaddr_t ctrl_base, paddr_t jr_offset) +{ + unsigned int jr_idx = JRX_IDX(jr_offset); + + caam_pwr_add_backup(ctrl_base + (jr_idx * JRxDID_SIZE), jrcfg_backup, + ARRAY_SIZE(jrcfg_backup)); +} diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/registers/ctrl_regs.h b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/registers/ctrl_regs.h new file mode 100644 index 0000000..fae7969 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/registers/ctrl_regs.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + * + * Brief Control Registers. + */ +#ifndef __CTRL_REGS_H__ +#define __CTRL_REGS_H__ + +#include + +/* Master Configuration */ +#define MCFGR 0x0004 +#define MCFGR_WDE BIT32(30) + +/* Job Ring x MID */ +#define JRxDID_SIZE 0x8 +#define JR0DID_MS 0x0010 +#define JR0DID_LS 0x0014 +#define JRxDID_MS(idx) (JR0DID_MS + ((idx) * (JRxDID_SIZE))) +#define JRxDID_LS(idx) (JR0DID_LS + ((idx) * (JRxDID_SIZE))) + +#define JRxDID_MS_LDID BIT32(31) +#define JRxDID_MS_PRIM_ICID(val) SHIFT_U32(((val) & (0x3FF)), 19) +#define JRxDID_MS_LAMTD BIT32(17) +#define JRxDID_MS_AMTD BIT32(16) +#define JRxDID_MS_TZ_OWN BIT32(15) +#define JRxDID_MS_PRIM_TZ BIT32(4) +#define JRxDID_MS_PRIM_DID(val) SHIFT_U32(((val) & (0xF)), 0) + +/* Security Configuration */ +#define SCFGR 0x000C +#define BS_SCFGR_MPCURVE 28 +#define BM_SCFGR_MPCURVE SHIFT_U32(0xF, BS_SCFGR_MPCURVE) +#define BM_SCFGR_MPMRL BIT32(26) + +/* Secure Memory Virtual Base Address */ +#define JRX_SMVBAR(idx) (0x0184 + (idx) * (8)) + +/* Manufacturing Protection Message */ +#define MPMR 0x0380 +#define MPMR_NB_REG 0x20 + +#endif /* __CTRL_REGS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/sub.mk b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/sub.mk new file mode 100644 index 0000000..5231cce --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/imx_8ulp/sub.mk @@ -0,0 +1,7 @@ +incdirs-y += ../common +incdirs-y += ../../include +incdirs-y += . + +srcs-y += hal_clk.c +srcs-y += hal_ctrl.c +srcs-y += hal_jr.c diff --git a/optee_os/core/drivers/crypto/caam/hal/ls/hal_clk.c b/optee_os/core/drivers/crypto/caam/hal/ls/hal_clk.c new file mode 100644 index 0000000..e3e059c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/ls/hal_clk.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 NXP + * + * Brief CAAM Clock functions. + */ +#include +#include + +void caam_hal_clk_enable(bool enable __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/ls/hal_ctrl.c b/optee_os/core/drivers/crypto/caam/hal/ls/hal_ctrl.c new file mode 100644 index 0000000..dab1821 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/ls/hal_ctrl.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 NXP + * + * Brief CAAM Controller Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include + +void caam_hal_ctrl_init(vaddr_t baseaddr __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/ls/hal_jr.c b/optee_os/core/drivers/crypto/caam/hal/ls/hal_jr.c new file mode 100644 index 0000000..7352677 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/ls/hal_jr.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 NXP + * + * Brief CAAM Job Rings Hardware Abstration Layer. + * Implementation of primitives to access HW. + */ +#include +#include +#include +#include +#include + +enum caam_status caam_hal_jr_setowner(vaddr_t ctrl_base, paddr_t jr_offset, + enum caam_jr_owner owner) +{ + uint32_t val = 0; + unsigned int jr_idx = JRX_IDX(jr_offset); + + if (owner == JROWN_ARM_S) { + /* Read the Job Ring Lock bit */ + val = io_caam_read32(ctrl_base + JRxMIDR_MS(jr_idx)); + HAL_TRACE("JR%" PRIu32 "MIDR_MS value 0x%" PRIx32, jr_idx, val); + val |= JRxMIDR_MS_TZ; + + io_caam_write32(ctrl_base + JRxMIDR_MS(jr_idx), val); + } + + return CAAM_NO_ERROR; +} + +void caam_hal_jr_prepare_backup(vaddr_t ctrl_base __unused, + paddr_t jr_offset __unused) +{ +} diff --git a/optee_os/core/drivers/crypto/caam/hal/ls/registers/ctrl_regs.h b/optee_os/core/drivers/crypto/caam/hal/ls/registers/ctrl_regs.h new file mode 100644 index 0000000..2432b95 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/ls/registers/ctrl_regs.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019, 2021 NXP + * + * Brief Control Registers. + */ +#ifndef __CTRL_REGS_H__ +#define __CTRL_REGS_H__ + +#include + +/* Master Configuration */ +#define MCFGR 0x0004 +#define MCFGR_WDE BIT32(30) +#define MCFGR_AXIPIPE(val) SHIFT_U32(val, 4) +#define BM_MCFGR_AXIPIPE SHIFT_U32(0xF, 4) + +/* Job Ring x MID */ +#define JRxMIDR_SIZE 0x8 +#define JR0MIDR_MS 0x0010 +#define JR0MIDR_LS 0x0014 +#define JRxMIDR_MS(idx) (JR0MIDR_MS + (idx) * JRxMIDR_SIZE) +#define JRxMIDR_LS(idx) (JR0MIDR_LS + (idx) * JRxMIDR_SIZE) + +#define JRxMIDR_MS_LMID BIT32(31) +#define JRxMIDR_MS_LAMTD BIT32(17) +#define JRxMIDR_MS_TZ BIT32(15) +#define JRxMIDR_MS_AMTD BIT32(16) +#define JRxMIDR_MS_JROWN_NS BIT32(3) +#define JRxMIDR_MS_JROWN_MID(val) SHIFT_U32((val) & 0x7, 0) + +#define JRxMIDR_LS_NONSEQ_NS BIT32(19) +#define JRxMIDR_LS_NONSEQ_MID(val) SHIFT_U32((val) & 0x7, 16) +#define JRxMIDR_LS_SEQ_NS BIT32(3) +#define JRxMIDR_LS_SEQ_MID(val) SHIFT_U32((val) & 0x7, 0) + +/* Security Configuration */ +#define SCFGR 0x000C + +#endif /* __CTRL_REGS_H__ */ + diff --git a/optee_os/core/drivers/crypto/caam/hal/ls/sub.mk b/optee_os/core/drivers/crypto/caam/hal/ls/sub.mk new file mode 100644 index 0000000..5231cce --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/ls/sub.mk @@ -0,0 +1,7 @@ +incdirs-y += ../common +incdirs-y += ../../include +incdirs-y += . + +srcs-y += hal_clk.c +srcs-y += hal_ctrl.c +srcs-y += hal_jr.c diff --git a/optee_os/core/drivers/crypto/caam/hal/sub.mk b/optee_os/core/drivers/crypto/caam/hal/sub.mk new file mode 100644 index 0000000..86f316a --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hal/sub.mk @@ -0,0 +1,19 @@ +ifeq ($(CFG_LS),y) +CAAM_HAL_DIR = ls +endif +ifeq ($(filter y, $(CFG_MX6) $(CFG_MX7) $(CFG_MX7ULP)),y) +CAAM_HAL_DIR = imx_6_7 +endif +ifeq ($(filter y, $(CFG_MX8MQ) $(CFG_MX8MM) $(CFG_MX8MN) $(CFG_MX8MP)),y) +CAAM_HAL_DIR = imx_8m +endif +ifeq ($(filter y, $(CFG_MX8QM) $(CFG_MX8QX) $(CFG_MX8DXL)),y) +CAAM_HAL_DIR = imx_8q +endif +ifeq ($(filter y, $(CFG_MX8ULP)),y) +CAAM_HAL_DIR = imx_8ulp +endif + + +subdirs-y += common +subdirs-y += $(CAAM_HAL_DIR) diff --git a/optee_os/core/drivers/crypto/caam/hash/caam_hash.c b/optee_os/core/drivers/crypto/caam/hash/caam_hash.c new file mode 100644 index 0000000..fbcf4a4 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hash/caam_hash.c @@ -0,0 +1,719 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019, 2021 NXP + * + * Implementation of Hashing functions. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +static const struct crypto_hash_ops hash_ops; + +/* + * Maximum number of entries in the descriptor + */ +#define MAX_DESC_ENTRIES 20 + +/* + * Constants definition of the hash/HMAC algorithm + */ +static const struct hashalg hash_alg[] = { + { + /* md5 */ + .type = OP_ALGO(MD5), + .size_digest = TEE_MD5_HASH_SIZE, + .size_block = TEE_MD5_HASH_SIZE * 4, + .size_ctx = HASH_MSG_LEN + TEE_MD5_HASH_SIZE, + .size_key = 32, + }, + { + /* sha1 */ + .type = OP_ALGO(SHA1), + .size_digest = TEE_SHA1_HASH_SIZE, + .size_block = TEE_MAX_HASH_SIZE, + .size_ctx = HASH_MSG_LEN + TEE_SHA1_HASH_SIZE, + .size_key = 40, + }, + { + /* sha224 */ + .type = OP_ALGO(SHA224), + .size_digest = TEE_SHA224_HASH_SIZE, + .size_block = TEE_MAX_HASH_SIZE, + .size_ctx = HASH_MSG_LEN + TEE_SHA256_HASH_SIZE, + .size_key = 64, + }, + { + /* sha256 */ + .type = OP_ALGO(SHA256), + .size_digest = TEE_SHA256_HASH_SIZE, + .size_block = TEE_MAX_HASH_SIZE, + .size_ctx = HASH_MSG_LEN + TEE_SHA256_HASH_SIZE, + .size_key = 64, + }, + { + /* sha384 */ + .type = OP_ALGO(SHA384), + .size_digest = TEE_SHA384_HASH_SIZE, + .size_block = TEE_MAX_HASH_SIZE * 2, + .size_ctx = HASH_MSG_LEN + TEE_SHA512_HASH_SIZE, + .size_key = 128, + }, + { + /* sha512 */ + .type = OP_ALGO(SHA512), + .size_digest = TEE_SHA512_HASH_SIZE, + .size_block = TEE_MAX_HASH_SIZE * 2, + .size_ctx = HASH_MSG_LEN + TEE_SHA512_HASH_SIZE, + .size_key = 128, + }, +}; + +/* + * Format the hash context to keep the reference to the + * operation driver + */ +struct crypto_hash { + struct crypto_hash_ctx hash_ctx; /* Crypto Hash API context */ + struct hashctx *ctx; /* Hash Context */ +}; + +/* + * Keep the HW hash limit because after the initialization + * of the module, we don't have the CAAM Controller base address + * to call the function returning the HW capacity. + */ +static uint8_t caam_hash_limit; + +/* + * Returns the reference to the driver context + * + * @ctx API Context + */ +static struct crypto_hash *to_hash_ctx(struct crypto_hash_ctx *ctx) +{ + assert(ctx && ctx->ops == &hash_ops); + + return container_of(ctx, struct crypto_hash, hash_ctx); +} + +/* + * Add the load key in the CAAM descriptor and clean the key buffer. + * + * @desc CAAM Descriptor + * @key Key to load + */ +static void do_desc_load_key(uint32_t *desc, struct caambuf *key) +{ + HASH_TRACE("Insert Key"); + caam_desc_add_word(desc, LD_KEY_SPLIT(key->length)); + caam_desc_add_ptr(desc, key->paddr); + + cache_operation(TEE_CACHECLEAN, key->data, key->length); +} + +/* + * Free the internal hashing data context + * + * @ctx [in/out] Caller context variable + */ +static void do_free_intern(struct hashctx *ctx) +{ + HASH_TRACE("Free Context (%p)", ctx); + + if (ctx) { + /* Free the descriptor */ + caam_free_desc(&ctx->descriptor); + + /* Free the Temporary buffer */ + caam_free_buf(&ctx->blockbuf.buf); + + /* Free the context register */ + caam_free_buf(&ctx->ctx); + + /* Free the HMAC Key */ + caam_free_buf(&ctx->key); + } +} + +/* + * Initialization of the Hash operation + * Call common initialization operation between hash and HMAC + * + * @ctx Operation software context + */ +static TEE_Result do_hash_init(struct crypto_hash_ctx *ctx) +{ + struct crypto_hash *hash = to_hash_ctx(ctx); + + return caam_hash_hmac_init(hash->ctx); +} + +/* + * Update the Hash operation + * Call common update operation between hash and HMAC + * + * @ctx Operation software context + * @data Data to hash + * @len Data length + */ +static TEE_Result do_hash_update(struct crypto_hash_ctx *ctx, + const uint8_t *data, size_t len) +{ + struct crypto_hash *hash = to_hash_ctx(ctx); + + return caam_hash_hmac_update(hash->ctx, data, len); +} + +/* + * Finalize the Hash operation + * Call common final operation between hash and HMAC + * + * @ctx Operation software context + * @digest [out] Hash digest buffer + * @len Digest buffer length + */ +static TEE_Result do_hash_final(struct crypto_hash_ctx *ctx, uint8_t *digest, + size_t len) +{ + struct crypto_hash *hash = to_hash_ctx(ctx); + + return caam_hash_hmac_final(hash->ctx, digest, len); +} + +/* + * Free the SW hashing data context + * Call common free operation between hash and HMAC + * + * @ctx [in/out] Caller context variable + */ +static void do_hash_free(struct crypto_hash_ctx *ctx) +{ + struct crypto_hash *hash = to_hash_ctx(ctx); + + caam_hash_hmac_free(hash->ctx); + + free(hash); +} + +/* + * Copy Software Hashing Context + * Call common copy operation between hash and HMAC + * + * @dst_ctx [out] Reference the context destination + * @src_ctx Reference the context source + */ +static void do_hash_copy_state(struct crypto_hash_ctx *dst_ctx, + struct crypto_hash_ctx *src_ctx) +{ + struct crypto_hash *hash_src = to_hash_ctx(src_ctx); + struct crypto_hash *hash_dst = to_hash_ctx(dst_ctx); + + return caam_hash_hmac_copy_state(hash_dst->ctx, hash_src->ctx); +} + +/* + * Registration of the hash Driver + */ +static const struct crypto_hash_ops hash_ops = { + .init = do_hash_init, + .update = do_hash_update, + .final = do_hash_final, + .free_ctx = do_hash_free, + .copy_state = do_hash_copy_state, +}; + +/* + * Allocate the internal hashing data context + * + * @ctx [out] Caller context variable + * @algo Algorithm ID + */ +static TEE_Result caam_hash_allocate(struct crypto_hash_ctx **ctx, + uint32_t algo) +{ + struct crypto_hash *hash = NULL; + struct hashctx *hash_ctx = NULL; + const struct hashalg *alg = NULL; + TEE_Result ret = TEE_ERROR_GENERIC; + + HASH_TRACE("Allocate Context (%p) algo %" PRId32, ctx, algo); + + *ctx = NULL; + + alg = caam_hash_get_alg(algo); + if (!alg) + return TEE_ERROR_NOT_IMPLEMENTED; + + hash = calloc(1, sizeof(*hash)); + if (!hash) + return TEE_ERROR_OUT_OF_MEMORY; + + hash_ctx = caam_calloc(sizeof(*hash_ctx)); + if (!hash_ctx) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + hash_ctx->alg = alg; + hash->hash_ctx.ops = &hash_ops; + hash->ctx = hash_ctx; + + *ctx = &hash->hash_ctx; + + ret = caam_hash_hmac_allocate(hash_ctx); + if (ret != TEE_SUCCESS) + goto err; + + HASH_TRACE("Allocated Context (%p)", hash_ctx); + + return TEE_SUCCESS; + +err: + free(hash); + + if (hash_ctx) + caam_free(hash_ctx); + + return ret; +} + +TEE_Result caam_hash_hmac_allocate(struct hashctx *ctx) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + HASH_TRACE("Allocate Context (%p)", ctx); + + /* Allocate the descriptor */ + ctx->descriptor = caam_calloc_desc(MAX_DESC_ENTRIES); + if (!ctx->descriptor) { + HASH_TRACE("Allocation context descriptor error"); + return TEE_ERROR_OUT_OF_MEMORY; + } + + /* Initialize the block buffer */ + ctx->blockbuf.filled = 0; + ctx->blockbuf.max = ctx->alg->size_block; + + /* Allocate the CAAM Context register */ + if (caam_calloc_align_buf(&ctx->ctx, ctx->alg->size_ctx) != + CAAM_NO_ERROR) { + HASH_TRACE("Allocation context register error"); + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + /* Allocate the Hash Key */ + if (caam_calloc_align_buf(&ctx->key, ctx->alg->size_key) != + CAAM_NO_ERROR) { + HASH_TRACE("Allocation context key error"); + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + cache_operation(TEE_CACHEFLUSH, ctx->ctx.data, ctx->ctx.length); + + /* Ensure buffer length is 0 */ + ctx->ctx.length = 0; + + return TEE_SUCCESS; + +err: + do_free_intern(ctx); + + return ret; +} + +/* + * Free the SW hashing data context + * + * @ctx Caller context variable + */ +void caam_hash_hmac_free(struct hashctx *ctx) +{ + HASH_TRACE("Free Context (%p)", ctx); + + if (ctx) { + do_free_intern(ctx); + caam_free(ctx); + } +} + +const struct hashalg *caam_hash_get_alg(uint32_t algo) +{ + uint8_t hash_id = TEE_ALG_GET_MAIN_ALG(algo); + unsigned int idx = hash_id - TEE_MAIN_ALGO_MD5; + + if (hash_id > caam_hash_limit || idx > ARRAY_SIZE(hash_alg)) + return NULL; + + return &hash_alg[idx]; +} + +TEE_Result caam_hash_hmac_init(struct hashctx *ctx) +{ + HASH_TRACE("Hash/HMAC Init (%p)", ctx); + if (!ctx) + return TEE_ERROR_BAD_PARAMETERS; + + /* Initialize the block buffer */ + ctx->blockbuf.filled = 0; + ctx->blockbuf.max = ctx->alg->size_block; + + /* Ensure Context length is 0 */ + ctx->ctx.length = 0; + + /* Initialize the HMAC Key */ + ctx->key.length = 0; + + ctx->initialized = true; + + return TEE_SUCCESS; +} + +/* + * Build and run the CAAM Hash descriptor to update (or start) the + * data digest. + * + * @ctx [in/out] Caller context variable + * @src Input data to digest + */ +static TEE_Result do_update_hash(struct hashctx *ctx, struct caamdmaobj *src) +{ + enum caam_status retstatus = CAAM_FAILURE; + const struct hashalg *alg = ctx->alg; + struct caam_jobctx jobctx = { }; + uint32_t *desc = ctx->descriptor; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + + /* There are blocks to hash - Create the Descriptor */ + if (ctx->ctx.length) { + HASH_TRACE("Update Operation"); + /* Algo Operation - Update */ + caam_desc_add_word(desc, HASH_UPDATE(alg->type)); + /* Running context to restore */ + caam_desc_add_word(desc, + LD_NOIMM(CLASS_2, REG_CTX, ctx->ctx.length)); + caam_desc_add_ptr(desc, ctx->ctx.paddr); + } else { + HASH_TRACE("Init Operation"); + + /* Check if there is a key to load it */ + if (ctx->key.length) { + do_desc_load_key(desc, &ctx->key); + + /* Algo Operation - HMAC Init */ + caam_desc_add_word(desc, HMAC_INIT_PRECOMP(alg->type)); + } else { + /* Algo Operation - Init */ + caam_desc_add_word(desc, HASH_INIT(alg->type)); + } + ctx->ctx.length = alg->size_ctx; + } + + if (ctx->blockbuf.filled) { + caam_desc_add_word(desc, FIFO_LD(CLASS_2, MSG, NOACTION, + ctx->blockbuf.filled)); + caam_desc_add_ptr(desc, ctx->blockbuf.buf.paddr); + cache_operation(TEE_CACHECLEAN, ctx->blockbuf.buf.data, + ctx->blockbuf.filled); + } + + caam_desc_fifo_load(desc, src, CLASS_2, MSG, LAST_C2); + caam_dmaobj_cache_push(src); + + ctx->blockbuf.filled = 0; + + if (ctx->ctx.length) { + /* Save the running context */ + caam_desc_add_word(desc, + ST_NOIMM(CLASS_2, REG_CTX, ctx->ctx.length)); + caam_desc_add_ptr(desc, ctx->ctx.paddr); + + /* Ensure Context register data are not in cache */ + cache_operation(TEE_CACHEINVALIDATE, ctx->ctx.data, + ctx->ctx.length); + } + + HASH_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus != CAAM_NO_ERROR) { + HASH_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + return job_status_to_tee_result(jobctx.status); + } + + HASH_DUMPBUF("CTX", ctx->ctx.data, ctx->ctx.length); + + return TEE_SUCCESS; +} + +TEE_Result caam_hash_hmac_update(struct hashctx *ctx, const uint8_t *data, + size_t len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + const struct hashalg *alg = NULL; + size_t fullsize = 0; + size_t size_topost = 0; + size_t size_todo = 0; + size_t size_done = 0; + size_t size_inmade = 0; + struct caamdmaobj src = { }; + size_t offset = 0; + + HASH_TRACE("Hash/HMAC Update (%p) %p - %zu", ctx, data, len); + + if ((!data && len) || !ctx) + return TEE_ERROR_BAD_PARAMETERS; + + alg = ctx->alg; + + if (!ctx->ctx.data) + return TEE_ERROR_GENERIC; + + HASH_TRACE("Update Type 0x%" PRIX32 " - Input @%p-%zu", alg->type, data, + len); + + /* Calculate the total data to be handled */ + fullsize = ctx->blockbuf.filled + len; + size_topost = fullsize % alg->size_block; + size_todo = fullsize - size_topost; + size_inmade = len - size_topost; + HASH_TRACE("FullSize %zu - posted %zu - todo %zu", fullsize, + size_topost, size_todo); + + if (!size_todo) { + ret = TEE_SUCCESS; + + /* All input data must be saved */ + if (size_topost) + size_inmade = 0; + + goto save_posted; + } + + ret = caam_dmaobj_init_input(&src, data, size_inmade); + if (ret) + goto exit_update; + + ret = caam_dmaobj_prepare(&src, NULL, alg->size_block); + if (ret) + goto exit_update; + + size_todo = size_inmade; + + for (offset = 0; offset < size_inmade; + offset += size_done, size_todo -= size_done) { + size_done = size_todo; + HASH_TRACE("Do input %zu bytes, offset %zu", size_done, offset); + + ret = caam_dmaobj_sgtbuf_build(&src, &size_done, offset, + alg->size_block); + if (ret) + goto exit_update; + + /* + * Need to re-adjust the length of the data if the + * posted data block is not empty and the SGT/Buffer + * is part of the full input data to do. + */ + if (ctx->blockbuf.filled && size_done < size_todo) { + size_done -= ctx->blockbuf.filled; + src.sgtbuf.length = size_done; + } + + ret = do_update_hash(ctx, &src); + if (ret) + goto exit_update; + } + +save_posted: + if (size_topost && data) { + struct caambuf srcdata = { + .data = (uint8_t *)data, + .length = len, + }; + + HASH_TRACE("Posted %zu of input len %zu made %zu", size_topost, + len, size_inmade); + + retstatus = caam_cpy_block_src(&ctx->blockbuf, &srcdata, + size_inmade); + ret = caam_status_to_tee_result(retstatus); + } + +exit_update: + caam_dmaobj_free(&src); + + if (ret) + do_free_intern(ctx); + + return ret; +} + +TEE_Result caam_hash_hmac_final(struct hashctx *ctx, uint8_t *digest, + size_t len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + const struct hashalg *alg = NULL; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + struct caamdmaobj dig = { }; + + HASH_TRACE("Hash/HMAC Final (%p)", ctx); + + if (!digest || !len || !ctx) + return TEE_ERROR_BAD_PARAMETERS; + + alg = ctx->alg; + + if (!ctx->ctx.data) + return TEE_ERROR_GENERIC; + + ret = caam_dmaobj_output_sgtbuf(&dig, digest, len, alg->size_digest); + if (ret) + goto out; + + HASH_TRACE("Final Type 0x%" PRIX32 " - Digest %zu", alg->type, len); + + desc = ctx->descriptor; + caam_desc_init(desc); + + /* Set the descriptor Header with length */ + caam_desc_add_word(desc, DESC_HEADER(0)); + + /* Load key if any */ + if (ctx->key.length) + do_desc_load_key(desc, &ctx->key); + + if (ctx->ctx.length) { + HASH_TRACE("Final Operation"); + + if (ctx->key.length) + caam_desc_add_word(desc, HMAC_FINAL_PRECOMP(alg->type)); + else + caam_desc_add_word(desc, HASH_FINAL(alg->type)); + + /* Running context to restore */ + caam_desc_add_word(desc, + LD_NOIMM(CLASS_2, REG_CTX, ctx->ctx.length)); + caam_desc_add_ptr(desc, ctx->ctx.paddr); + + cache_operation(TEE_CACHEINVALIDATE, ctx->ctx.data, + ctx->ctx.length); + HASH_DUMPBUF("CTX", ctx->ctx.data, ctx->ctx.length); + ctx->ctx.length = 0; + } else { + HASH_TRACE("Init/Final Operation"); + if (ctx->key.length) + caam_desc_add_word(desc, + HMAC_INITFINAL_PRECOMP(alg->type)); + else + caam_desc_add_word(desc, HASH_INITFINAL(alg->type)); + } + + HASH_DUMPBUF("Temporary Block", ctx->blockbuf.buf.data, + ctx->blockbuf.filled); + + caam_desc_add_word(desc, FIFO_LD_EXT(CLASS_2, MSG, LAST_C2)); + caam_desc_add_ptr(desc, ctx->blockbuf.buf.paddr); + caam_desc_add_word(desc, ctx->blockbuf.filled); + + if (ctx->blockbuf.filled) + cache_operation(TEE_CACHECLEAN, ctx->blockbuf.buf.data, + ctx->blockbuf.filled); + + ctx->blockbuf.filled = 0; + + /* Save the final digest */ + caam_desc_store(desc, &dig, CLASS_2, REG_CTX); + caam_dmaobj_cache_push(&dig); + + HASH_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + caam_dmaobj_copy_to_orig(&dig); + + HASH_DUMPBUF("Digest", digest, (size_t)alg->size_digest); + + ret = TEE_SUCCESS; + } else { + HASH_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_dmaobj_free(&dig); + + return ret; +} + +void caam_hash_hmac_copy_state(struct hashctx *dst, struct hashctx *src) +{ + HASH_TRACE("Copy State context (%p) to (%p)", src, dst); + + assert(dst && src); + + if (!dst->initialized && caam_hash_hmac_init(dst)) + panic(); + + dst->alg = src->alg; + + if (src->ctx.length) { + cache_operation(TEE_CACHEINVALIDATE, src->ctx.data, + src->ctx.length); + memcpy(dst->ctx.data, src->ctx.data, src->ctx.length); + dst->ctx.length = src->ctx.length; + cache_operation(TEE_CACHECLEAN, dst->ctx.data, dst->ctx.length); + } + + if (src->blockbuf.filled) { + struct caambuf srcdata = { + .data = src->blockbuf.buf.data, + .length = src->blockbuf.filled + }; + + caam_cpy_block_src(&dst->blockbuf, &srcdata, 0); + } + + if (src->key.data) { + memcpy(dst->key.data, src->key.data, src->key.length); + dst->key.length = src->key.length; + } +} + +enum caam_status caam_hash_init(struct caam_jrcfg *caam_jrcfg) +{ + enum caam_status retstatus = CAAM_FAILURE; + vaddr_t jr_base = caam_jrcfg->base + caam_jrcfg->offset; + + caam_hash_limit = caam_hal_ctrl_hash_limit(jr_base); + + if (caam_hash_limit != UINT8_MAX) { + if (drvcrypt_register_hash(&caam_hash_allocate) == TEE_SUCCESS) + retstatus = CAAM_NO_ERROR; + } + + return retstatus; +} diff --git a/optee_os/core/drivers/crypto/caam/hash/caam_hash_mac.c b/optee_os/core/drivers/crypto/caam/hash/caam_hash_mac.c new file mode 100644 index 0000000..423e90f --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hash/caam_hash_mac.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Implementation of Hashing functions. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +#ifdef CFG_PHYS_64BIT +#define KEY_REDUCE_DESC_ENTRIES 10 +#define KEY_COMPUTE_DESC_ENTRIES 10 +#else +#define KEY_REDUCE_DESC_ENTRIES 8 +#define KEY_COMPUTE_DESC_ENTRIES 8 +#endif + +static const struct crypto_mac_ops hmac_ops; + +/* + * Format the MAC context to keep the reference to the operation driver. + */ +struct crypto_mac { + struct crypto_mac_ctx mac_ctx; /* Crypto MAC API context */ + struct hashctx *ctx; /* HMAC context */ +}; + +/* + * Keep the HW hash limit because after the initialization + * of the module, we don't have the CAAM Controller base address + * to call the function returning the HW capacity. + */ +static uint8_t caam_hash_limit; + +/* + * Returns the reference to the driver context + * + * @ctx API Context + */ +static struct crypto_mac *to_mac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx && ctx->ops == &hmac_ops); + + return container_of(ctx, struct crypto_mac, mac_ctx); +} + +/* + * Reduce key to be a hash algorithm block size maximum + * + * @alg Reference to the algorithm definition + * @inkey Key to be reduced + * @outkey [out] key resulting + */ +static enum caam_status do_reduce_key(struct caamdmaobj *reduce_key, + const struct hashalg *alg, + const uint8_t *inkey, size_t len) +{ + enum caam_status retstatus = CAAM_FAILURE; + struct caamdmaobj key = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + + if (caam_dmaobj_input_sgtbuf(&key, inkey, len)) + return CAAM_OUT_MEMORY; + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(KEY_REDUCE_DESC_ENTRIES); + if (!desc) { + retstatus = CAAM_OUT_MEMORY; + goto out; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, HASH_INITFINAL(alg->type)); + + /* Load the input key */ + caam_desc_fifo_load(desc, &key, CLASS_2, MSG, LAST_C2); + /* Store key reduced */ + caam_desc_store(desc, reduce_key, CLASS_2, REG_CTX); + + caam_dmaobj_cache_push(&key); + caam_dmaobj_cache_push(reduce_key); + + HASH_DUMPDESC(desc); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus != CAAM_NO_ERROR) { + HASH_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + retstatus = CAAM_FAILURE; + } + +out: + caam_dmaobj_free(&key); + caam_free_desc(&desc); + + return retstatus; +} + +/* + * Initialization of the HMAC operation. + * Split the input key using the CAAM HW HMAC operation + * Call common initialization operation between hash and HMAC + * + * @ctx Operation software context + * @key Input key to compute + * @len Key length + */ +static TEE_Result do_hmac_init(struct crypto_mac_ctx *ctx, const uint8_t *inkey, + size_t len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct crypto_mac *mac = to_mac_ctx(ctx); + struct hashctx *hmac_ctx = mac->ctx; + const struct hashalg *alg = hmac_ctx->alg; + struct caamdmaobj reduce_key = { }; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + + /* First initialize the context */ + ret = caam_hash_hmac_init(hmac_ctx); + if (ret != TEE_SUCCESS) + return ret; + + HASH_TRACE("split key length %zu", len); + + /* Allocate the job descriptor */ + desc = caam_calloc_desc(KEY_COMPUTE_DESC_ENTRIES); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + hmac_ctx->key.length = alg->size_key; + + if (len > alg->size_block) { + HASH_TRACE("Input key must be reduced"); + + ret = caam_dmaobj_output_sgtbuf(&reduce_key, NULL, 0, + alg->size_digest); + if (ret) { + HASH_TRACE("Reduced Key allocation error"); + goto out; + } + + retstatus = do_reduce_key(&reduce_key, alg, inkey, len); + if (retstatus != CAAM_NO_ERROR) + goto out; + } else { + /* Key size is correct use directly the input key */ + ret = caam_dmaobj_input_sgtbuf(&reduce_key, inkey, len); + if (ret) + goto out; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + /* Load either input key or the reduced input key into key register */ + caam_desc_load_key(desc, &reduce_key, CLASS_2, REG); + /* Split the key */ + caam_desc_add_word(desc, HMAC_INIT_DECRYPT(alg->type)); + caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_2, MSG, LAST_C2, 0)); + /* Store the split key */ + caam_desc_add_word(desc, FIFO_ST(C2_MDHA_SPLIT_KEY_AES_ECB_JKEK, + hmac_ctx->key.length)); + caam_desc_add_ptr(desc, hmac_ctx->key.paddr); + HASH_DUMPDESC(desc); + + caam_dmaobj_cache_push(&reduce_key); + cache_operation(TEE_CACHEFLUSH, hmac_ctx->key.data, + hmac_ctx->key.length); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + HASH_DUMPBUF("Split Key", hmac_ctx->key.data, + hmac_ctx->key.length); + + ret = TEE_SUCCESS; + } else { + HASH_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_dmaobj_free(&reduce_key); + caam_free_desc(&desc); + + return ret; +} + +/* + * Update the HMAC operation + * Call common update operation between hash and HMAC + * + * @ctx Operation Software context + * @data Data to hash + * @len Data length + */ +static TEE_Result do_hmac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + struct crypto_mac *mac = to_mac_ctx(ctx); + + return caam_hash_hmac_update(mac->ctx, data, len); +} + +/* + * Finalize the HMAC operation + * Call common final operation between hash and HMAC + * + * @ctx Operation Software context + * @digest [out] Hash digest buffer + * @len Digest buffer length + */ +static TEE_Result do_hmac_final(struct crypto_mac_ctx *ctx, uint8_t *digest, + size_t len) +{ + struct crypto_mac *mac = to_mac_ctx(ctx); + + return caam_hash_hmac_final(mac->ctx, digest, len); +} + +/* + * Free the software context + * Call common free operation between hash and HMAC + * + * @ctx Caller context variable + */ +static void do_hmac_free(struct crypto_mac_ctx *ctx) +{ + struct crypto_mac *mac = to_mac_ctx(ctx); + + caam_hash_hmac_free(mac->ctx); + + free(mac); +} + +/* + * Copy sofware HMAC context + * Call common copy operation between hash and HMAC + * + * @dst_ctx [out] Reference the context destination + * @src_ctx Reference the context source + */ +static void do_hmac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct crypto_mac *mac_src = to_mac_ctx(src_ctx); + struct crypto_mac *mac_dst = to_mac_ctx(dst_ctx); + + return caam_hash_hmac_copy_state(mac_dst->ctx, mac_src->ctx); +} + +/* + * Registration of the HMAC driver + */ +static const struct crypto_mac_ops hmac_ops = { + .init = do_hmac_init, + .update = do_hmac_update, + .final = do_hmac_final, + .free_ctx = do_hmac_free, + .copy_state = do_hmac_copy_state, +}; + +/* + * Allocate the internal hashing data context + * + * @ctx [out] Caller context reference + * @algo Algorithm ID + */ +static TEE_Result caam_hmac_allocate(struct crypto_mac_ctx **ctx, uint32_t algo) +{ + struct crypto_mac *mac = NULL; + struct hashctx *hmac_ctx = NULL; + const struct hashalg *alg = NULL; + TEE_Result ret = TEE_ERROR_GENERIC; + + HASH_TRACE("Allocate Context (%p) algo %" PRId32, ctx, algo); + + *ctx = NULL; + + alg = caam_hash_get_alg(algo); + if (!alg) + return TEE_ERROR_NOT_IMPLEMENTED; + + mac = calloc(1, sizeof(*mac)); + if (!mac) + return TEE_ERROR_OUT_OF_MEMORY; + + hmac_ctx = caam_calloc(sizeof(*hmac_ctx)); + if (!hmac_ctx) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + hmac_ctx->alg = alg; + mac->mac_ctx.ops = &hmac_ops; + mac->ctx = hmac_ctx; + + *ctx = &mac->mac_ctx; + + ret = caam_hash_hmac_allocate(hmac_ctx); + if (ret != TEE_SUCCESS) + goto err; + + HASH_TRACE("Allocated Context (%p)", hmac_ctx); + + return TEE_SUCCESS; + +err: + free(mac); + + if (hmac_ctx) + caam_free(hmac_ctx); + + return ret; +} + +enum caam_status caam_hmac_init(struct caam_jrcfg *caam_jrcfg) +{ + vaddr_t jr_base = caam_jrcfg->base + caam_jrcfg->offset; + + caam_hash_limit = caam_hal_ctrl_hash_limit(jr_base); + + if (caam_hash_limit != UINT8_MAX && + caam_hal_ctrl_splitkey_support(jr_base)) { + if (drvcrypt_register_hmac(&caam_hmac_allocate)) + return CAAM_FAILURE; + } + + return CAAM_NO_ERROR; +} diff --git a/optee_os/core/drivers/crypto/caam/hash/local.h b/optee_os/core/drivers/crypto/caam/hash/local.h new file mode 100644 index 0000000..01da063 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hash/local.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019-2021 NXP + * + * CAAM hash/HMAC local header. + */ +#ifndef __LOCAL_H__ +#define __LOCAL_H__ + +#include + +/* + * Full hashing/HMAC data SW context + */ +struct hashctx { + uint32_t *descriptor; /* Job descriptor */ + struct caamblock blockbuf; /* Temporary block buffer */ + struct caambuf ctx; /* Hash context used by the CAAM */ + const struct hashalg *alg; /* Reference to the algo constants */ + struct caambuf key; /* HMAC split key */ + bool initialized; /* Context initialization flag */ +}; + +/* + * Hash/HMAC algorithm definition + */ +struct hashalg { + uint32_t type; /* Algo type for operation */ + uint8_t size_digest; /* Digest size */ + uint8_t size_block; /* Computing block size */ + uint8_t size_ctx; /* CAAM context register size (8 + digest size) */ + uint8_t size_key; /* HMAC split key size */ +}; + +/* First part CAAM HW context - message length */ +#define HASH_MSG_LEN 8 + +/* + * Initialization of the hash/HMAC operation + * + * @ctx Operation software context + */ +TEE_Result caam_hash_hmac_init(struct hashctx *ctx); + +/* + * Update the hash/HMAC operation + * + * @ctx Operation software context + * @data Data to hash + * @len Data length + */ +TEE_Result caam_hash_hmac_update(struct hashctx *ctx, const uint8_t *data, + size_t len); + +/* + * Finalize the hash/HMAC operation + * + * @ctx Operation software context + * @digest [out] Hash digest buffer + * @len Digest buffer length + */ +TEE_Result caam_hash_hmac_final(struct hashctx *ctx, uint8_t *digest, + size_t len); + +/* + * Copy sofware hashing context + * + * @dst [out] Reference the destination context + * @src Reference the source context + */ +void caam_hash_hmac_copy_state(struct hashctx *dst, struct hashctx *src); + +/* + * Free the software context + * + * @ctx [in/out] Caller context variable + */ +void caam_hash_hmac_free(struct hashctx *ctx); + +/* + * Get hash/HMAC algorithm definition + * + * @algo Hash algorithm + */ +const struct hashalg *caam_hash_get_alg(uint32_t algo); + +/* + * Allocate the internal hashing data context + * + * @ctx [in/out] Caller context variable + */ +TEE_Result caam_hash_hmac_allocate(struct hashctx *ctx); + +#endif /* __LOCAL_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/hash/sub.mk b/optee_os/core/drivers/crypto/caam/hash/sub.mk new file mode 100644 index 0000000..37f2b6e --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/hash/sub.mk @@ -0,0 +1,4 @@ +incdirs-y += ../include + +srcs-y += caam_hash.c +srcs-$(CFG_NXP_CAAM_HMAC_DRV) += caam_hash_mac.c diff --git a/optee_os/core/drivers/crypto/caam/include/caam_acipher.h b/optee_os/core/drivers/crypto/caam/include/caam_acipher.h new file mode 100644 index 0000000..6cdb85a --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_acipher.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief CAAM Asymmetric Cipher manager header. + */ +#ifndef __CAAM_ACIPHER_H__ +#define __CAAM_ACIPHER_H__ + +#include +#include + +#ifdef CFG_NXP_CAAM_RSA_DRV +/* + * Initialize the RSA module + * + * @caam_jrcfg JR configuration structure + */ +enum caam_status caam_rsa_init(struct caam_jrcfg *caam_jrcfg); +#else +static inline enum caam_status +caam_rsa_init(struct caam_jrcfg *caam_jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_RSA_DRV */ + +#ifdef CFG_NXP_CAAM_DH_DRV +/* + * Initialize the DH module + * + * @caam_jrcfg JR configuration structure + */ +enum caam_status caam_dh_init(struct caam_jrcfg *caam_jrcfg); +#else +static inline enum caam_status +caam_dh_init(struct caam_jrcfg *caam_jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_DH_DRV */ + +#ifdef CFG_NXP_CAAM_MATH_DRV +/* + * Initialize the MATH module + * + * @caam_jrcfg JR configuration structure + */ +enum caam_status caam_math_init(struct caam_jrcfg *caam_jrcfg); +#else +static inline enum caam_status +caam_math_init(struct caam_jrcfg *caam_jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_MATH_DRV */ + +#ifdef CFG_NXP_CAAM_ECC_DRV +/* + * Initialize the Cipher module + * + * @caam_jrcfg JR configuration structure + */ +enum caam_status caam_ecc_init(struct caam_jrcfg *caam_jrcfg); +#else +static inline enum caam_status +caam_ecc_init(struct caam_jrcfg *caam_jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_ECC_DRV */ + +#ifdef CFG_NXP_CAAM_DSA_DRV +/* + * Initialize the DSA module + * + * @caam_jrcfg CAAM job ring configuration + */ +enum caam_status caam_dsa_init(struct caam_jrcfg *caam_jrcfg); +#else +static inline enum caam_status +caam_dsa_init(struct caam_jrcfg *caam_jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_DSA_DRV */ +#endif /* __CAAM_ACIPHER_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_blob.h b/optee_os/core/drivers/crypto/caam/include/caam_blob.h new file mode 100644 index 0000000..768cdaa --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_blob.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020 Pengutronix, Rouven Czerwinski + */ +#ifndef __CAAM_BLOB_H__ +#define __CAAM_BLOB_H__ + +#include + +#ifdef CFG_NXP_CAAM_BLOB_DRV +/* + * Initialize the BLOB module + * + * @ctrl_addr Controller base address + */ +enum caam_status caam_blob_mkvb_init(vaddr_t baseaddr); +#else +static inline enum caam_status caam_blob_mkvb_init(vaddr_t baseaddr __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_BLOB_DRV */ + +#endif /* __CAAM_BLOB_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_cipher.h b/optee_os/core/drivers/crypto/caam/include/caam_cipher.h new file mode 100644 index 0000000..8a92f20 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_cipher.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2020 NXP + * + * Brief CAAM Cipher manager header. + */ +#ifndef __CAAM_CIPHER_H__ +#define __CAAM_CIPHER_H__ + +#include + +#ifdef CFG_NXP_CAAM_CIPHER_DRV +/* + * Initialize the Cipher module + * + * @ctrl_addr Controller base address + */ +enum caam_status caam_cipher_init(vaddr_t ctrl_addr); +#else +static inline enum caam_status caam_cipher_init(vaddr_t ctrl_addr __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_CIPHER_DRV */ +#ifdef CFG_NXP_CAAM_CMAC_DRV +/* + * Initialize the CMAC module + * + * @ctrl_addr Controller base address + */ +enum caam_status caam_cmac_init(vaddr_t ctrl_addr); +#else +static inline enum caam_status caam_cmac_init(vaddr_t ctrl_addr __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_CMAC_DRV */ +#endif /* __CAAM_CIPHER_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_common.h b/optee_os/core/drivers/crypto/caam/include/caam_common.h new file mode 100644 index 0000000..a502851 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_common.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019, 2021 NXP + * + * CAAM driver common include file. + */ + +#ifndef __CAAM_COMMON_H__ +#define __CAAM_COMMON_H__ + +#include +#include +#include +#include + +/* + * Definition of the number of CAAM Jobs to manage in JR queues + */ +#if defined(CFG_NB_JOBS_QUEUE) +#define NB_JOBS_QUEUE CFG_NB_JOBS_QUEUE +#else +#define NB_JOBS_QUEUE 10 +#endif + +/* + * Flag Job Ring Owner is Secure + */ +#define JROWNER_SECURE 0x10 + +/* + * Job Ring Owner. Enumerate Id (expect the Secure Flag) correspond + * to the HW ID. + */ +#if defined(CFG_MX7ULP) +enum caam_jr_owner { + JROWN_ARM_NS = 0x4, /* Non-Secure ARM */ + JROWN_ARM_S = JROWNER_SECURE | 0x4, /* Secure ARM */ +}; +#elif defined(CFG_MX8ULP) +enum caam_jr_owner { + JROWN_ARM_NS = 0x7, /* Non-Secure ARM */ + JROWN_ARM_S = JROWNER_SECURE | 0x7, /* Secure ARM */ +}; +#else +enum caam_jr_owner { + JROWN_ARM_NS = 0x1, /* Non-Secure ARM */ + JROWN_ARM_S = JROWNER_SECURE | 0x1, /* Secure ARM */ +}; +#endif + +#endif /* __CAAM_COMMON_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_desc_ccb_defines.h b/optee_os/core/drivers/crypto/caam/include/caam_desc_ccb_defines.h new file mode 100644 index 0000000..8bfd512 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_desc_ccb_defines.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief Define the CCB Registers to use in the CAAM descriptor + */ +#ifndef __CAAM_DESC_CCB_DEFINES_H__ +#define __CAAM_DESC_CCB_DEFINES_H__ + +/* CCB CHA Control Register */ +#define CCTRL_ULOAD_PKHA_B BIT32(27) +#define CCTRL_ULOAD_PKHA_A BIT32(26) + +/* CCB Clear Written Register */ +#define CLR_WR_IFIFO_NFIFO BIT32(31) +#define CLR_WR_RST_C2_CHA BIT32(28) +#define CLR_WR_RST_C2_DSZ BIT32(18) + +/* CCB NFIFO */ +#define NFIFO_CLASS(cla) SHIFT_U32(NFIFO_CLASS_##cla & 0x3, 30) +#define NFIFO_CLASS_DECO 0x0 +#define NFIFO_CLASS_C1 0x1 +#define NFIFO_CLASS_BOTH 0x3 + +#define NFIFO_LC2 BIT32(29) +#define NFIFO_LC1 BIT32(28) +#define NFIFO_FC1 BIT32(26) + +#define NFIFO_STYPE(src) SHIFT_U32(NFIFO_STYPE_##src & 0x3, 24) +#define NFIFO_STYPE_IFIFO 0x0 +#define NFIFO_STYPE_PAD 0x2 + +#define NFIFO_DTYPE(data) SHIFT_U32(NFIFO_DTYPE_##data & 0xF, 20) +#define NFIFO_DTYPE_MSG 0xF +#define NFIFO_DTYPE_PKHA_N 0x8 +#define NFIFO_DTYPE_PKHA_A 0xC + +#define NFIFO_PTYPE(pad) SHIFT_U32(NFIFO_PTYPE_##pad & 0x7, 16) +#define NFIFO_PTYPE_ZERO 0x0 +#define NFIFO_PTYPE_RND 0x3 + +#define NFIFO_DATA_LENGTH(len) SHIFT_U32((len) & 0xFFF, 0) +#define NFIFO_PAD_LENGTH(len) SHIFT_U32((len) & 0x7F, 0) + +/* + * CCB NFIFO Entry to pad data with pad type + */ +#define NFIFO_PAD(cla, options, data, pad, len) \ + (NFIFO_CLASS(cla) | (options) | NFIFO_STYPE(PAD) | NFIFO_DTYPE(data) | \ + NFIFO_PTYPE(pad) | NFIFO_PAD_LENGTH(len)) + +/* + * CCB NFIFO Entry to move data from src to data + */ +#define NFIFO_NOPAD(cla, options, src, data, len) \ + (NFIFO_CLASS(cla) | (options) | NFIFO_STYPE(src) | NFIFO_DTYPE(data) | \ + NFIFO_PTYPE(ZERO) | NFIFO_DATA_LENGTH(len)) + +#endif /* __CAAM_DESC_CCB_DEFINES_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_desc_defines.h b/optee_os/core/drivers/crypto/caam/include/caam_desc_defines.h new file mode 100644 index 0000000..c6f8f1d --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_desc_defines.h @@ -0,0 +1,716 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief CAAM Descriptor defines. + */ +#ifndef __CAAM_DESC_DEFINES_H__ +#define __CAAM_DESC_DEFINES_H__ + +#include + +/* + * Common Command constants + */ +#define CMD_TYPE(cmd) SHIFT_U32((cmd) & 0x1F, 27) +#define GET_CMD_TYPE(op) ((op) & (SHIFT_U32(0x1F, 27))) +#define CMD_CLASS(val) SHIFT_U32((val) & 0x3, 25) +#define CLASS_NO 0x0 +#define CLASS_1 0x1 +#define CLASS_2 0x2 +#define CLASS_DECO 0x3 + +#define CMD_SGT BIT32(24) +#define CMD_IMM BIT32(23) + +/* + * HEADER Job Descriptor Header format + */ +#define CMD_HDR_JD_TYPE CMD_TYPE(0x16) + +/* Must be ONE */ +#define HDR_JD_ONE BIT32(23) + +/* Start Index if SHR = 0 */ +#define HDR_JD_START_IDX(line) SHIFT_U32((line) & 0x3F, 16) + +/* Descriptor Length */ +#define HDR_JD_DESCLEN(len) SHIFT_U32((len) & 0x7F, 0) +#define GET_JD_DESCLEN(entry) ((entry) & 0x7F) + +/* + * KEY Command fields + */ +#define CMD_KEY_TYPE CMD_TYPE(0x00) + +/* Key Destination */ +#define KEY_DEST(val) SHIFT_U32((KEY_DEST_##val) & 0x3, 16) +#define KEY_DEST_REG 0x0 +#define KEY_DEST_PKHA_E 0x1 +#define KEY_DEST_AFHA_SBOX 0x2 +#define KEY_DEST_MDHA_SPLIT 0x3 + +/* Plaintext Store */ +#define KEY_PTS BIT32(14) + +/* Key Length */ +#define KEY_LENGTH(len) SHIFT_U32((len) & 0x3FF, 0) + +/* + * LOAD Command fields + */ +#define CMD_LOAD_TYPE CMD_TYPE(0x02) + +/* Load Destination */ +#define LOAD_DST(reg) SHIFT_U32((reg) & 0x7F, 16) + +/* Offset in destination register */ +#define LOAD_OFFSET(off) SHIFT_U32((off) & 0xFF, 8) + +/* Length */ +#define LOAD_LENGTH(len) SHIFT_U32((len) & 0xFF, 0) + +/* + * STORE Command fields + */ +#define CMD_STORE_TYPE CMD_TYPE(0x0A) +#define CMD_STORE_SEQ_TYPE CMD_TYPE(0x0B) + +/* Store Source */ +#define STORE_SRC(reg) SHIFT_U32((reg) & 0x7F, 16) + +/* Offset in source register */ +#define STORE_OFFSET(off) SHIFT_U32((off) & 0xFF, 8) + +/* Length */ +#define STORE_LENGTH(len) SHIFT_U32((len) & 0xFF, 0) + +/* + * Define the Load/Store Registers Source and Destination + */ +#define REG_MODE 0x00 +#define REG_KEY_SIZE 0x01 +#define REG_DATA_SIZE 0x02 +#define REG_ICV_SIZE 0x03 +#define REG_DECO_MID_STATUS 0x04 +#define REG_DECO_CTRL2 0x05 +#define REG_CHA_CTRL 0x06 +#define REG_DECO_CTRL 0x06 +#define REG_IRQ_CTRL 0x07 +#define REG_DECO_PROT_OVERWRITE 0x07 +#define REG_CLEAR_WRITTEN 0x08 +#define REG_MATH0 0x08 +#define REG_MATH1 0x09 +#define REG_MATH2 0x0A +#define REG_CHA_INST_SELECT 0x0A +#define REG_AAD_SIZE 0x0B +#define REG_MATH3 0x0B +#define REG_ALT_DATA_SIZE_C1 0x0F +#define REG_PKHA_A_SIZE 0x10 +#define REG_PKHA_B_SIZE 0x11 +#define REG_PKHA_N_SIZE 0x12 +#define REG_PKHA_E_SIZE 0x13 +#define REG_CTX 0x20 +#define REG_MATH0_DW 0x30 +#define REG_MATH1_DW 0x31 +#define REG_MATH2_DW 0x32 +#define REG_MATH3_DW 0x33 +#define REG_MATH0_B 0x38 +#define REG_MATH1_B 0x39 +#define REG_MATH2_B 0x3A +#define REG_MATH3_B 0x3B +#define REG_KEY 0x40 +#define REG_DECO_DESC 0x40 +#define REG_NFIFO_n_SIZE 0x70 +#define REG_NFIFO_MATH 0x73 +#define REG_SIZE 0x74 +#define REG_SIZE_MATH 0x75 +#define REG_IFIFO_SHIFT 0x76 +#define REG_OFIFO_SHIFT 0x77 +#define REG_AUX_FIFO 0x78 +#define REG_NFIFO 0x7A +#define REG_IFIFO 0x7C +#define REG_OFIFO 0x7E + +/* + * FIFO LOAD Command fields + */ +#define CMD_FIFO_LOAD_TYPE CMD_TYPE(0x04) + +/* Extended Length */ +#define FIFO_LOAD_EXT BIT32(22) + +/* Input data */ +#define FIFO_LOAD_INPUT(reg) SHIFT_U32((FIFO_LOAD_##reg) & 0x3F, 16) +#define FIFO_LOAD_ACTION(act) SHIFT_U32((FIFO_LOAD_##act) & 0x3F, 16) + +/* Length */ +#define FIFO_LOAD_MAX 0xFFFF +#define FIFO_LOAD_LENGTH(len) SHIFT_U32((len) & FIFO_LOAD_MAX, 0) + +/* + * Define the FIFO Load Type Input + */ +#define FIFO_LOAD_PKHA_A0 0x00 +#define FIFO_LOAD_PKHA_A1 0x01 +#define FIFO_LOAD_PKHA_A2 0x02 +#define FIFO_LOAD_PKHA_A3 0x03 +#define FIFO_LOAD_PKHA_B0 0x04 +#define FIFO_LOAD_PKHA_B1 0x05 +#define FIFO_LOAD_PKHA_B2 0x06 +#define FIFO_LOAD_PKHA_B3 0x07 +#define FIFO_LOAD_PKHA_N 0x08 +#define FIFO_LOAD_PKHA_A 0x0C +#define FIFO_LOAD_PKHA_B 0x0D +#define FIFO_LOAD_NO_INFO_NFIFO 0x0F +#define FIFO_LOAD_MSG 0x10 +#define FIFO_LOAD_MSG_C1_OUT_C2 0x18 +#define FIFO_LOAD_IV 0x20 +#define FIFO_LOAD_BITDATA 0x2C +#define FIFO_LOAD_AAD 0x30 +#define FIFO_LOAD_ICV 0x38 + +/* Define Action of some FIFO Data */ +#define FIFO_LOAD_NOACTION 0x0 +#define FIFO_LOAD_FLUSH 0x1 +#define FIFO_LOAD_LAST_C1 0x2 +#define FIFO_LOAD_LAST_C2 0x4 + +/* + * FIFO STORE Command fields + */ +#define CMD_FIFO_STORE_TYPE CMD_TYPE(0x0C) +#define CMD_SEQ_FIFO_STORE_TYPE CMD_TYPE(0x0D) + +/* Extended Length */ +#define FIFO_STORE_EXT BIT32(22) + +/* Output data */ +#define FIFO_STORE_OUTPUT(reg) SHIFT_U32((FIFO_STORE_##reg) & 0x3F, 16) + +/* Length */ +#define FIFO_STORE_MAX 0xFFFF +#define FIFO_STORE_LENGTH(len) SHIFT_U32((len) & FIFO_STORE_MAX, 0) + +/* + * Define the FIFO Store Type Output + */ +#define FIFO_STORE_PKHA_A0 0x00 +#define FIFO_STORE_PKHA_A1 0x01 +#define FIFO_STORE_PKHA_A2 0x02 +#define FIFO_STORE_PKHA_A3 0x03 +#define FIFO_STORE_PKHA_B0 0x04 +#define FIFO_STORE_PKHA_B1 0x05 +#define FIFO_STORE_PKHA_B2 0x06 +#define FIFO_STORE_PKHA_B3 0x07 +#define FIFO_STORE_PKHA_N 0x08 +#define FIFO_STORE_PKHA_A 0x0C +#define FIFO_STORE_PKHA_B 0x0D +#define FIFO_STORE_AFHA_SBOX_AES_CCM_JKEK 0x10 +#define FIFO_STORE_AFHA_SBOX_AES_CCM_TKEK 0x11 +#define FIFO_STORE_PKHA_E_AES_CCM_JKEK 0x12 +#define FIFO_STORE_PKHA_E_AES_CCM_TKEK 0x13 +#define FIFO_STORE_KEY_AES_CCM_JKEK 0x14 +#define FIFO_STORE_KEY_AES_CCM_TKEK 0x15 +#define FIFO_STORE_C2_MDHA_SPLIT_KEY_AES_CCM_JKEK 0x16 +#define FIFO_STORE_C2_MDHA_SPLIT_KEY_AES_CCM_TKEK 0x17 +#define FIFO_STORE_AFHA_SBOX_AES_ECB_JKEK 0x20 +#define FIFO_STORE_AFHA_SBOX_AES_ECB_TKEK 0x21 +#define FIFO_STORE_PKHA_E_AES_ECB_JKEK 0x22 +#define FIFO_STORE_PKHA_E_AES_ECB_TKEK 0x23 +#define FIFO_STORE_KEY_AES_ECB_JKEK 0x24 +#define FIFO_STORE_KEY_AES_ECB_TKEK 0x25 +#define FIFO_STORE_C2_MDHA_SPLIT_KEY_AES_ECB_JKEK 0x26 +#define FIFO_STORE_C2_MDHA_SPLIT_KEY_AES_ECB_TKEK 0x27 +#define FIFO_STORE_MSG_DATA 0x30 +#define FIFO_STORE_RNG_TO_MEM 0x34 +#define FIFO_STORE_RNG_STAY_FIFO 0x35 +#define FIFO_STORE_SKIP 0x3F + +/* + * MOVE Command fields + */ +#define CMD_MOVE_TYPE CMD_TYPE(0x0F) + +/* Auxiliary */ +#define MOVE_AUX(val) SHIFT_U32((val) & 0x3, 25) + +/* Wait for completion */ +#define MOVE_WC BIT32(24) + +/* Source */ +#define MOVE_SRC(src) MOVE_SRC_##src +#define MOVE_REG_SRC(reg) SHIFT_U32((reg) & 0xF, 20) +#define MOVE_SRC_C1_CTX_REG MOVE_REG_SRC(0x0) +#define MOVE_SRC_C2_CTX_REG MOVE_REG_SRC(0x1) +#define MOVE_SRC_OFIFO MOVE_REG_SRC(0x2) +#define MOVE_SRC_DESC_BUF MOVE_REG_SRC(0x3) +#define MOVE_SRC_MATH_REG0 MOVE_REG_SRC(0x4) +#define MOVE_SRC_MATH_REG1 MOVE_REG_SRC(0x5) +#define MOVE_SRC_MATH_REG2 MOVE_REG_SRC(0x6) +#define MOVE_SRC_MATH_REG3 MOVE_REG_SRC(0x7) +#define MOVE_SRC_NFIFO_DECO_ALIGN MOVE_REG_SRC(0x8) +#define MOVE_SRC_NFIFO_C1_ALIGN (MOVE_REG_SRC(0x9) | MOVE_AUX(0x1)) +#define MOVE_SRC_NFIFO_C2_ALIGN (MOVE_REG_SRC(0x9) | MOVE_AUX(0x0)) +#define MOVE_SRC_DECO_ALIGN (MOVE_REG_SRC(0xA) | MOVE_AUX(0x0)) +#define MOVE_SRC_C1_ALIGN (MOVE_REG_SRC(0xA) | MOVE_AUX(0x1)) +#define MOVE_SRC_C2_ALIGN (MOVE_REG_SRC(0xA) | MOVE_AUX(0x2)) +#define MOVE_SRC_C1_KEY MOVE_REG_SRC(0xD) +#define MOVE_SRC_C2_KEY MOVE_REG_SRC(0xE) + +/* Destination */ +#define MOVE_DST(dst) SHIFT_U32((MOVE_DST_##dst), 16) +#define MOVE_DST_C1_CTX_REG 0x0 +#define MOVE_DST_C2_CTX_REG 0x1 +#define MOVE_DST_OFIFO 0x2 +#define MOVE_DST_DESC_BUF 0x3 +#define MOVE_DST_MATH_REG0 0x4 +#define MOVE_DST_MATH_REG1 0x5 +#define MOVE_DST_MATH_REG2 0x6 +#define MOVE_DST_MATH_REG3 0x7 +#define MOVE_DST_IFIFO_C1 0x8 +#define MOVE_DST_IFIFO_C2 0x9 +#define MOVE_DST_IFIFO_C2_LC2 ((0x9 << 16 | MOVE_AUX(0x1)) >> 16) +#define MOVE_DST_IFIFO 0xA +#define MOVE_DST_PKHA_A 0xC +#define MOVE_DST_C1_KEY 0xD +#define MOVE_DST_C2_KEY 0xE +#define MOVE_DST_AUX_FIFO 0xF + +/* Offset */ +#define MOVE_OFFSET(off) SHIFT_U32((off) & 0xFF, 8) + +/* Length */ +#define MOVE_LENGTH(len) SHIFT_U32((len) & 0xFF, 0) + +/* + * Operation Command fields + * Algorithm/Protocol/PKHA + */ +#define CMD_OP_TYPE CMD_TYPE(0x10) + +/* Operation Type */ +#define OP_TYPE(type) SHIFT_U32((OP_TYPE_##type) & 0x7, 24) +#define OP_TYPE_UNI 0x0 +#define OP_TYPE_PKHA 0x1 +#define OP_TYPE_CLASS1 0x2 +#define OP_TYPE_CLASS2 0x4 +#define OP_TYPE_DECAPS 0x6 +#define OP_TYPE_ENCAPS 0x7 + +/* Protocol Identifier */ +#define PROTID(id) SHIFT_U32((PROTID_##id) & 0xFF, 16) +#define PROTID_BLOB 0x0D +#define PROTID_MPKEY 0x14 +#define PROTID_PKKEY 0x14 +#define PROTID_MPSIGN 0x15 +#define PROTID_DSASIGN 0x15 +#define PROTID_DSAVERIFY 0x16 +#define PROTID_SHARED_SECRET 0x17 +#define PROTID_RSA_ENC 0x18 +#define PROTID_RSA_DEC 0x19 +#define PROTID_RSA_FINISH_KEY 0x1A + +/* + * RSA Protocol Information + */ +#define PROT_RSA_FMT(format) SHIFT_U32((PROT_RSA_FMT_##format) & 0x1, 12) +#define PROT_RSA_FMT_NO 0 +#define PROT_RSA_FMT_PKCS_V1_5 1 + +#define PROT_RSA_DEC_KEYFORM(format) SHIFT_U32(((format) - 1) & 0x3, 0) + +/* RSA Key Protocol Information */ +#define PROT_RSA_KEY(format) SHIFT_U32((PROT_RSA_KEY_##format) & 0x3, 0) +#define PROT_RSA_KEY_ALL 0 +#define PROT_RSA_KEY_N_D 2 + +/* + * ECC Protocol Information + */ +#define PROT_PK_MSG(type) SHIFT_U32(PROT_PK_MSG_##type, 10) +#define PROT_PK_MSG_HASHED 2 +#define PROT_PK_TYPE(type) SHIFT_U32(PROT_PK_##type, 1) +#define PROT_PK_DL 0 +#define PROT_PK_ECC 1 + +/* + * BLOB Protocol Information + */ +#define PROT_BLOB_FMT_MSTR BIT32(1) +#define PROT_BLOB_TYPE(type) SHIFT_U32(1, PROT_BLOB_TYPE_##type) +#define PROT_BLOB_TYPE_BLACK_KEY 2 +#define PROT_BLOB_SEC_MEM BIT32(3) +#define PROT_BLOB_EKT 8 +#define PROT_BLOB_INFO(aes) SHIFT_U32(PROT_BLOB_AES_##aes, \ + PROT_BLOB_EKT) +#define PROT_BLOB_AES_CCM 1 +#define PROT_BLOB_AES_ECB 0 +#define PROT_BLOB_FORMAT(format) SHIFT_U32(0, PROT_BLOB_FORMAT_##format) +#define PROT_BLOB_FORMAT_NORMAL 0 + +/* + * MP Protocol Information + */ +#define PROT_MP_PUBK_SGT BIT32(31) +#define PROT_MP_CURVE(curve) SHIFT_U32((curve) & 0xF, 17) + +/* + * Algorithm Identifier + */ +#define OP_ALGO(algo) SHIFT_U32((ALGO_##algo) & 0xFF, 16) +#define ALGO_AES 0x10 +#define ALGO_DES 0x20 +#define ALGO_3DES 0x21 +#define ALGO_ARC4 0x30 +#define ALGO_RNG 0x50 +#define ALGO_MD5 0x40 +#define ALGO_SHA1 0x41 +#define ALGO_SHA224 0x42 +#define ALGO_SHA256 0x43 +#define ALGO_SHA384 0x44 +#define ALGO_SHA512 0x45 +#define ALGO_SHA512_224 0x46 +#define ALGO_SHA512_256 0x47 + +/* Algorithm Additional Information */ +#define ALGO_AAI(info) SHIFT_U32((AAI_##info) & 0x1FF, 4) + +/* AES AAI */ +#define AAI_AES_CTR_MOD128 0x00 +#define AAI_AES_CBC 0x10 +#define AAI_AES_ECB 0x20 +#define AAI_AES_CFB 0x30 +#define AAI_AES_OFB 0x40 +#define AAI_AES_CMAC 0x60 +#define AAI_AES_XCBC_MAC 0x70 +#define AAI_AES_CCM 0x80 +#define AAI_AES_GCM 0x90 + +/* DES AAI */ +#define AAI_DES_CBC 0x10 +#define AAI_DES_ECB 0x20 +#define AAI_DES_CFB 0x30 +#define AAI_DES_OFB 0x40 + +/* Digest MD5/SHA AAI */ +#define AAI_DIGEST_HASH 0x00 +#define AAI_DIGEST_HMAC 0x01 +#define AAI_DIGEST_SMAC 0x02 +#define AAI_DIGEST_HMAC_PRECOMP 0x04 + +/* Algorithm State */ +#define ALGO_AS(state) SHIFT_U32((AS_##state) & 0x3, 2) +#define AS_UPDATE 0x0 +#define AS_INIT 0x1 +#define AS_FINAL 0x2 +#define AS_INIT_FINAL 0x3 + +/* Algorithm Encrypt/Decrypt */ +#define ALGO_DECRYPT SHIFT_U32(0x0, 0) +#define ALGO_ENCRYPT SHIFT_U32(0x1, 0) + +/* + * Specific RNG Algorithm bits 12-0 + */ +/* Secure Key */ +#define ALGO_RNG_SK BIT32(12) + +/* State Handle */ +#define ALGO_RNG_SH(sh) SHIFT_U32((sh) & 0x3, 4) + +/* Prediction Resistance */ +#define ALGO_RNG_PR BIT32(1) + +/* State */ +#define AS_RNG_GENERATE 0x0 +#define AS_RNG_INSTANTIATE 0x1 +#define AS_RNG_RESEED 0x2 +#define AS_RNG_UNINSTANTIATE 0x3 + +/* + * JUMP Command fields + */ +#define CMD_JUMP_TYPE CMD_TYPE(0x14) + +/* Jump Select Type */ +#define JMP_JSL BIT32(24) + +/* Jump Type */ +#define JUMP_TYPE(type) SHIFT_U32((JMP_##type) & 0xF, 20) +#define JMP_LOCAL 0x0 +#define JMP_LOCAL_INC 0x1 +#define JMP_SUBROUTINE_CALL 0x2 +#define JMP_LOCAL_DEC 0x3 +#define JMP_NON_LOCAL 0x4 +#define JMP_SUBROUTINE_RET 0x6 +#define JMP_HALT 0x8 +#define JMP_HALT_USER_STATUS 0xC + +/* Test Type */ +#define JUMP_TST_TYPE(type) SHIFT_U32((JMP_TST_##type) & 0x3, 16) +#define JMP_TST_ALL_COND_TRUE 0x0 +#define JMP_TST_ALL_COND_FALSE 0x1 +#define JMP_TST_ANY_COND_TRUE 0x2 +#define JMP_TST_ANY_COND_FALSE 0x3 + +/* Jump Source to increment/decrement */ +#define JMP_SRC(src) SHIFT_U32((JMP_SRC_##src) & 0xF, 12) +#define JMP_SRC_MATH_0 0x0 + +/* Test Condition */ +#define JMP_COND(cond) SHIFT_U32((JMP_COND_##cond) & 0xFF, 8) +#define JMP_COND_MATH(cond) SHIFT_U32((JMP_COND_MATH_##cond) & 0xF, 8) +#define JMP_COND_NONE 0x00 +#define JMP_COND_PKHA_IS_ZERO 0x80 +#define JMP_COND_PKHA_GCD_1 0x40 +#define JMP_COND_PKHA_IS_PRIME 0x20 +#define JMP_COND_MATH_N 0x08 +#define JMP_COND_MATH_Z 0x04 +#define JMP_COND_NIFP 0x04 +#define JMP_COND_MATH_C 0x02 +#define JMP_COND_MATH_NV 0x01 + +/* Local Offset */ +#define JMP_LOCAL_OFFSET(off) SHIFT_U32((off) & 0xFF, 0) + +/* + * MATH Command fields + */ +#define CMD_MATH_TYPE CMD_TYPE(0x15) +#define CMD_MATHI_TYPE CMD_TYPE(0x1D) + +/* Immediate Four Bytes */ +#define MATH_IFB BIT32(26) + +/* Function Mathematical */ +#define MATH_FUNC(func) SHIFT_U32((MATH_FUNC_##func) & 0xF, 20) +#define MATH_FUNC_ADD 0x0 +#define MATH_FUNC_ADD_W_CARRY 0x1 +#define MATH_FUNC_SUB 0x2 +#define MATH_FUNC_SUB_W_BORROW 0x3 +#define MATH_FUNC_OR 0x4 +#define MATH_FUNC_AND 0x5 +#define MATH_FUNC_XOR 0x6 +#define MATH_FUNC_SHIFT_L 0x7 +#define MATH_FUNC_SHIFT_R 0x8 +#define MATH_FUNC_SHLD 0x9 +#define MATH_FUNC_ZBYTE 0xA +#define MATH_FUNC_SWAP_BYTES 0xB + +/* Source 0 */ +#define MATH_SRC0(reg) SHIFT_U32((MATH_SRC0_##reg) & 0xF, 16) +#define MATH_SRC0_REG0 0x0 +#define MATH_SRC0_REG1 0x1 +#define MATH_SRC0_REG2 0x2 +#define MATH_SRC0_IMM_DATA 0x4 +#define MATH_SRC0_DPOVRD 0x7 +#define MATH_SRC0_SIL 0x8 +#define MATH_SRC0_SOL 0x9 +#define MATH_SRC0_VSIL 0xA +#define MATH_SRC0_VSOL 0xB +#define MATH_SRC0_ZERO 0xC +#define MATH_SRC0_ONE 0xF + +/* Source 1 */ +#define MATH_SRC1(reg) SHIFT_U32((MATH_SRC1_##reg) & 0xF, 12) +#define MATH_SRC1_REG0 0x0 +#define MATH_SRC1_REG1 0x1 +#define MATH_SRC1_REG2 0x2 +#define MATH_SRC1_IMM_DATA 0x4 +#define MATH_SRC1_DPOVRD 0x7 +#define MATH_SRC1_VSIL 0x8 +#define MATH_SRC1_VSOL 0x9 +#define MATH_SRC1_IFIFO 0xA +#define MATH_SRC1_OFIFO 0xB +#define MATH_SRC1_ONE 0xC +#define MATH_SRC1_ZERO 0xF + +/* Destination */ +#define MATH_DST(reg) SHIFT_U32((MATH_DST_##reg) & 0xF, 8) +#define MATH_DST_REG0 0x0 +#define MATH_DST_REG1 0x1 +#define MATH_DST_REG2 0x2 +#define MATH_DST_DPOVRD 0x7 +#define MATH_DST_SIL 0x8 +#define MATH_DST_SOL 0x9 +#define MATH_DST_VSIL 0xA +#define MATH_DST_VSOL 0xB +#define MATH_DST_NODEST 0xF + +/* Length */ +#define MATH_LENGTH(len) SHIFT_U32((len) & 0xF, 0) + +/* Immediate Value - MATHI operation */ +#define MATHI_SRC(reg) SHIFT_U32((MATH_SRC0_##reg) & 0xF, 16) +#define MATHI_DST(reg) SHIFT_U32((MATH_DST_##reg) & 0xF, 12) +#define MATHI_IMM_VALUE(val) SHIFT_U32((val) & 0xFF, 4) + +/* + * Sequence Input/Output + */ +#define CMD_SEQ_IN_TYPE CMD_TYPE(0x1E) +#define CMD_SEQ_OUT_TYPE CMD_TYPE(0x1F) + +/* Extended Length */ +#define SEQ_EXT BIT(22) + +/* Length */ +#define SEQ_LENGTH(len) SHIFT_U32((len) & 0xFFFF, 0) + +/* + * PKHA Operation + */ +#define PKHA_ALG SHIFT_U32(0x8, 20) + +#define PKHA_F2M BIT32(17) + +#define PKHA_OUTSEL(dst) SHIFT_U32((PKHA_OUTSEL_##dst) & 0x3, 8) +#define PKHA_OUTSEL_B 0x0 +#define PKHA_OUTSEL_A 0x1 + +#define PKHA_FUNC(func) SHIFT_U32((PKHA_FUNC_##func) & 0x3F, 0) +#define PKHA_FUNC_CPY_NSIZE 0x10 +#define PKHA_FUNC_CPY_SSIZE 0x11 +#define PKHA_FUNC_MOD_ADD_A_B 0x02 +#define PKHA_FUNC_MOD_SUB_A_B 0x03 +#define PKHA_FUNC_MOD_SUB_B_A 0x04 +#define PKHA_FUNC_MOD_MUL_A_B 0x05 +#define PKHA_FUNC_MOD_EXP_A_E 0x06 +#define PKHA_FUNC_MOD_AMODN 0x07 +#define PKHA_FUNC_MOD_INV_A 0x08 +#define PKHA_FUNC_ECC_POINT_ADD_P1_P2 0x09 +#define PKHA_FUNC_ECC_POINT_DBL_P1 0x0A +#define PKHA_FUNC_ECC_POINT_MUL_E_P1 0x0B +#define PKHA_FUNC_MONT_RADIX_R2_MODE_N 0x0C +#define PKHA_FUNC_GCD_A_N 0x0E +#define PKHA_FUNC_MR_PRIMER_TEST 0x0F +#define PKHA_FUNC_MOD_CHECK_POINT 0x1C + +/* PKHA Copy Memory Source and Destination */ +#define PKHA_REG_SRC(reg) SHIFT_U32((PKHA_REG_##reg) & 0x7, 17) +#define PKHA_REG_DST(reg) SHIFT_U32((PKHA_REG_##reg) & 0x3, 10) +#define PKHA_REG_A 0x0 +#define PKHA_REG_B 0x1 +#define PKHA_REG_E 0x2 +#define PKHA_REG_N 0x3 + +#define PKHA_SEG_SRC(seg) SHIFT_U32((seg) & 0x3, 8) +#define PKHA_SEG_DST(seg) SHIFT_U32((seg) & 0x3, 6) + +#define PKHA_CPY_SRC(src) PKHA_CPY_SRC_##src +#define PKHA_CPY_SRC_A0 (PKHA_REG_SRC(A) | PKHA_SEG_SRC(0)) +#define PKHA_CPY_SRC_A1 (PKHA_REG_SRC(A) | PKHA_SEG_SRC(1)) +#define PKHA_CPY_SRC_A2 (PKHA_REG_SRC(A) | PKHA_SEG_SRC(2)) +#define PKHA_CPY_SRC_A3 (PKHA_REG_SRC(A) | PKHA_SEG_SRC(3)) +#define PKHA_CPY_SRC_B0 (PKHA_REG_SRC(B) | PKHA_SEG_SRC(0)) +#define PKHA_CPY_SRC_B1 (PKHA_REG_SRC(B) | PKHA_SEG_SRC(1)) +#define PKHA_CPY_SRC_B2 (PKHA_REG_SRC(B) | PKHA_SEG_SRC(2)) +#define PKHA_CPY_SRC_B3 (PKHA_REG_SRC(B) | PKHA_SEG_SRC(3)) +#define PKHA_CPY_SRC_N0 (PKHA_REG_SRC(N) | PKHA_SEG_SRC(0)) +#define PKHA_CPY_SRC_N1 (PKHA_REG_SRC(N) | PKHA_SEG_SRC(1)) +#define PKHA_CPY_SRC_N2 (PKHA_REG_SRC(N) | PKHA_SEG_SRC(2)) +#define PKHA_CPY_SRC_N3 (PKHA_REG_SRC(N) | PKHA_SEG_SRC(3)) + +#define PKHA_CPY_DST(dst) PKHA_CPY_DST_##dst +#define PKHA_CPY_DST_A0 (PKHA_REG_DST(A) | PKHA_SEG_DST(0)) +#define PKHA_CPY_DST_A1 (PKHA_REG_DST(A) | PKHA_SEG_DST(1)) +#define PKHA_CPY_DST_A2 (PKHA_REG_DST(A) | PKHA_SEG_DST(2)) +#define PKHA_CPY_DST_A3 (PKHA_REG_DST(A) | PKHA_SEG_DST(3)) +#define PKHA_CPY_DST_B0 (PKHA_REG_DST(B) | PKHA_SEG_DST(0)) +#define PKHA_CPY_DST_B1 (PKHA_REG_DST(B) | PKHA_SEG_DST(1)) +#define PKHA_CPY_DST_B2 (PKHA_REG_DST(B) | PKHA_SEG_DST(2)) +#define PKHA_CPY_DST_B3 (PKHA_REG_DST(B) | PKHA_SEG_DST(3)) +#define PKHA_CPY_DST_N0 (PKHA_REG_DST(N) | PKHA_SEG_DST(0)) +#define PKHA_CPY_DST_N1 (PKHA_REG_DST(N) | PKHA_SEG_DST(1)) +#define PKHA_CPY_DST_N2 (PKHA_REG_DST(N) | PKHA_SEG_DST(2)) +#define PKHA_CPY_DST_N3 (PKHA_REG_DST(N) | PKHA_SEG_DST(3)) +#define PKHA_CPY_DST_E (PKHA_REG_DST(E)) + +/* + * Descriptor Protocol Data Block + */ +/* RSA Encryption */ +#define PDB_RSA_ENC_SGT_F SHIFT_U32(1, 31) +#define PDB_RSA_ENC_SGT_G SHIFT_U32(1, 30) +#define PDB_RSA_ENC_E_SIZE(len) SHIFT_U32((len) & 0xFFF, 12) +#define PDB_RSA_ENC_N_SIZE(len) SHIFT_U32((len) & 0xFFF, 0) +#define PDB_RSA_ENC_F_SIZE(len) SHIFT_U32((len) & 0xFFF, 0) + +/* RSA Decryption */ +#define PDB_RSA_DEC_SGT_G SHIFT_U32(1, 31) +#define PDB_RSA_DEC_SGT_F SHIFT_U32(1, 30) +#define PDB_RSA_DEC_D_SIZE(len) SHIFT_U32((len) & 0xFFF, 12) +#define PDB_RSA_DEC_N_SIZE(len) SHIFT_U32((len) & 0xFFF, 0) +#define PDB_RSA_DEC_Q_SIZE(len) SHIFT_U32((len) & 0xFFF, 12) +#define PDB_RSA_DEC_P_SIZE(len) SHIFT_U32((len) & 0xFFF, 0) + +/* RSA Finalize Key */ +#define PDB_RSA_KEY_P_SIZE(len) SHIFT_U32((len) & 0x1FF, 0) +#define PDB_RSA_KEY_E_SIZE(len) SHIFT_U32((len) & 0x3FF, 0) +#define PDB_RSA_KEY_N_SIZE(len) SHIFT_U32((len) & 0x3FF, 16) + +/* Manufacturing Curve Select */ +#define PDB_SGT_MP_SIGN_MSG SHIFT_U32(1, 31) +#define PDB_SGT_MP_SIGN_C SHIFT_U32(1, 29) +#define PDB_SGT_MP_SIGN_D SHIFT_U32(1, 28) +#define PDB_MP_CSEL_P256 0x03 +#define PDB_MP_CSEL_P384 0x04 +#define PDB_MP_CSEL_P521 0x05 + +/* Public Key Generation */ +#define PDB_PKGEN_PD1 SHIFT_U32(1, 25) +/* Public Key Signature */ +#define PDB_PKSIGN_PD1 SHIFT_U32(1, 22) +/* Public Key Verify */ +#define PDB_PKVERIFY_PD1 SHIFT_U32(1, 22) +/* Shared Secret */ +#define PDB_SHARED_SECRET_PD1 SHIFT_U32(1, 25) + +/* DSA Signatures */ +#define PDB_DSA_SIGN_N(len) SHIFT_U32((len) & (0x7F), 0) +#define PDB_DSA_SIGN_L(len) SHIFT_U32((len) & (0x3FF), 7) + +/* SGT Flags Signature */ +#define PDB_SGT_PKSIGN_MSG SHIFT_U32(1, 27) +#define PDB_SGT_PKSIGN_SIGN_C SHIFT_U32(1, 26) +#define PDB_SGT_PKSIGN_SIGN_D SHIFT_U32(1, 25) + +/* DSA Verify */ +#define PDB_DSA_VERIF_N(len) SHIFT_U32((len) & (0x7F), 0) +#define PDB_DSA_VERIF_L(len) SHIFT_U32((len) & (0x3FF), 7) + +/* SGT Flags Verify */ +#define PDB_SGT_PKVERIF_MSG SHIFT_U32(1, 27) +#define PDB_SGT_PKVERIF_SIGN_C SHIFT_U32(1, 26) +#define PDB_SGT_PKVERIF_SIGN_D SHIFT_U32(1, 25) + +/* SGT Flags Shared Secret */ +#define PDB_SGT_PKDH_SECRET SHIFT_U32(1, 27) + +/* DL Keypair Generation */ +#define PDB_DL_KEY_L_SIZE(len) SHIFT_U32((len) & (0x3FF), 7) +#define PDB_DL_KEY_N_MASK 0x7F +#define PDB_DL_KEY_N_SIZE(len) SHIFT_U32((len) & (PDB_DL_KEY_N_MASK), 0) + +/* ECC Domain Selection */ +#define PDB_ECC_ECDSEL(curve) SHIFT_U32((curve) & 0x3F, 7) + +/* Black key padding */ +#define BLACK_KEY_NONCE_SIZE 6 +#define BLACK_KEY_ICV_SIZE 6 + +/* + * ECC Predefined Domain + */ +enum caam_ecc_curve { + CAAM_ECC_P192 = (0x00), + CAAM_ECC_P224, + CAAM_ECC_P256, + CAAM_ECC_P384, + CAAM_ECC_P521, + CAAM_ECC_MAX, + CAAM_ECC_UNKNOWN = (0xFF), +}; + +#endif /* __CAAM_DESC_DEFINES_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_desc_helper.h b/optee_os/core/drivers/crypto/caam/include/caam_desc_helper.h new file mode 100644 index 0000000..bf0de2c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_desc_helper.h @@ -0,0 +1,558 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief CAAM Descriptor interface. + */ +#ifndef __CAAM_DESC_HELPER_H__ +#define __CAAM_DESC_HELPER_H__ + +#include +#include +#include +#include + +/* + * Returns the number of entries of the descriptor + */ +uint32_t caam_desc_get_len(uint32_t *desc); + +/* Descriptor Modification function */ +void caam_desc_init(uint32_t *desc); +void caam_desc_update_hdr(uint32_t *desc, uint32_t word); +void caam_desc_add_ptr(uint32_t *desc, paddr_t ptr); +void caam_desc_add_word(uint32_t *desc, uint32_t word); +void caam_desc_add_dmaobj(uint32_t *desc, struct caamdmaobj *data, + uint32_t pre_operation); + +#define caam_desc_fifo_load(desc, data, cla, dst, act) \ + caam_desc_add_dmaobj(desc, data, FIFO_LD(cla, dst, act, 0)) +#define caam_desc_load_key(desc, data, cla, dst) \ + caam_desc_add_dmaobj(desc, data, LD_KEY_PLAIN(cla, dst, 0)) +#define caam_desc_store(desc, data, cla, src) \ + caam_desc_add_dmaobj(desc, data, ST_NOIMM(cla, src, 0)) +#define caam_desc_fifo_store(desc, data, src) \ + caam_desc_add_dmaobj(desc, data, FIFO_ST(src, 0)) +#define caam_desc_seq_out(desc, data) \ + caam_desc_add_dmaobj(desc, data, SEQ_OUT_PTR(0)) + +/* Push/Pop descriptor rings queue */ +void caam_desc_push(struct caam_inring_entry *in_entry, paddr_t paddr); +paddr_t caam_desc_pop(struct caam_outring_entry *out_entry); + +uint32_t caam_read_jobstatus(struct caam_outring_entry *out); + +/* Debug print function to dump a Descriptor in hex */ +static inline void dump_desc(uint32_t *desc) +{ + size_t idx = 0; + size_t len = 0; + + len = caam_desc_get_len(desc); + + for (idx = 0; idx < len; idx++) + trace_printf(NULL, 0, 0, false, "[%02zu] %08" PRIX32, idx, + desc[idx]); +} + +/* + * Returns the descriptor size in bytes of nbEntries + */ +#define DESC_SZBYTES(nbentries) ((nbentries) * sizeof(uint32_t)) + +/* + * Descriptor Header starting at idx w/o descriptor length + */ +#define DESC_HDR(idx) (CMD_HDR_JD_TYPE | HDR_JD_ONE | HDR_JD_START_IDX(idx)) + +/* + * Descriptor Header starting at index 0 with descriptor length len + */ +#define DESC_HEADER(len) (DESC_HDR(0) | HDR_JD_DESCLEN(len)) + +/* + * Descriptor Header starting at idx with descriptor length len + */ +#define DESC_HEADER_IDX(len, idx) (DESC_HDR(idx) | HDR_JD_DESCLEN(len)) + +/* + * Jump Local of class cla to descriptor offset if test meet the + * condition cond + */ +#define JUMP_LOCAL(cla, test, cond, offset) \ + (CMD_JUMP_TYPE | CMD_CLASS(cla) | JUMP_TYPE(LOCAL) | \ + JUMP_TST_TYPE(test) | (cond) | JMP_LOCAL_OFFSET(offset)) + +/* + * Jump Local of no class to descriptor offset if test meet the + * condition cond + */ +#define JUMP_CNO_LOCAL(test, cond, offset) \ + JUMP_LOCAL(CLASS_NO, test, cond, offset) + +/* + * Jump Local of class 1 to descriptor offset if test meet the + * condition cond + */ +#define JUMP_C1_LOCAL(test, cond, offset) \ + JUMP_LOCAL(CLASS_1, test, cond, offset) + +/* + * First decrement specified source then + * Jump Local of no class to descriptor offset if test meet the + * condition cond + */ +#define JUMP_CNO_LOCAL_DEC(test, src, cond, offset) \ + (CMD_JUMP_TYPE | CMD_CLASS(CLASS_NO) | JUMP_TYPE(LOCAL_DEC) | \ + JUMP_TST_TYPE(test) | JMP_SRC(src) | (cond) | \ + JMP_LOCAL_OFFSET(offset)) + +/* + * Wait until test condition meet and jump next + */ +#define WAIT_COND(test, cond) \ + (JUMP_LOCAL(CLASS_NO, test, JMP_COND(cond), 1) | JMP_JSL) + +/* + * Jump No Local of class cla to descriptor offset if test meet the + * condition cond + */ +#define JUMP_NOTLOCAL(cla, test, cond) \ + (CMD_JUMP_TYPE | CMD_CLASS(cla) | JUMP_TYPE(NON_LOCAL) | \ + JUMP_TST_TYPE(test) | (cond)) + +/* + * User Halt with error if test meet the condition cond + */ +#define HALT_USER(test, cond, error) \ + (CMD_JUMP_TYPE | JUMP_TYPE(HALT_USER_STATUS) | JUMP_TST_TYPE(test) | \ + JMP_COND(cond) | JMP_LOCAL_OFFSET(error)) + +/* + * Load Immediate value of length len to register dst of class cla + */ +#define LD_IMM(cla, dst, len) \ + (CMD_LOAD_TYPE | CMD_CLASS(cla) | CMD_IMM | LOAD_DST(dst) | \ + LOAD_LENGTH(len)) + +/* + * Load Immediate value of length len to register dst of class starting of + * register offset. + */ +#define LD_IMM_OFF(cla, dst, len, off) \ + (CMD_LOAD_TYPE | CMD_CLASS(cla) | CMD_IMM | LOAD_DST(dst) | \ + LOAD_OFFSET(off) | LOAD_LENGTH(len)) + +/* + * Load Immediate value of length len to register dst w/o class + */ +#define LD_NOCLASS_IMM(dst, len) LD_IMM(CLASS_NO, dst, len) + +/* + * Load value of length len to register dst of class cla + */ +#define LD_NOIMM(cla, dst, len) \ + (CMD_LOAD_TYPE | CMD_CLASS(cla) | LOAD_DST(dst) | LOAD_LENGTH(len)) + +/* + * Load value of length len to register dst of class cla starting + * at register offset off + */ +#define LD_NOIMM_OFF(cla, dst, len, off) \ + (CMD_LOAD_TYPE | CMD_CLASS(cla) | LOAD_DST(dst) | LOAD_OFFSET(off) | \ + LOAD_LENGTH(len)) + +/* + * FIFO Load to register dst class cla with action act + */ +#define FIFO_LD(cla, dst, act, len) \ + (CMD_FIFO_LOAD_TYPE | CMD_CLASS(cla) | FIFO_LOAD_INPUT(dst) | \ + FIFO_LOAD_ACTION(act) | FIFO_LOAD_LENGTH(len)) + +/* + * FIFO Load to register dst class cla with action act. + * Pointer is a Scatter/Gather Table + */ +#define FIFO_LD_SGT(cla, dst, act, len) \ + (CMD_FIFO_LOAD_TYPE | CMD_CLASS(cla) | CMD_SGT | \ + FIFO_LOAD_INPUT(dst) | FIFO_LOAD_ACTION(act) | FIFO_LOAD_LENGTH(len)) + +/* + * FIFO Load to register dst class cla with action act. + * Pointer is a Scatter/Gather Table + * The length is externally defined + */ +#define FIFO_LD_SGT_EXT(cla, dst, act) \ + (CMD_FIFO_LOAD_TYPE | CMD_CLASS(cla) | CMD_SGT | FIFO_LOAD_EXT | \ + FIFO_LOAD_INPUT(dst) | FIFO_LOAD_ACTION(act)) + +/* + * FIFO Load to register dst class cla with action act. + * The length is externally defined + */ +#define FIFO_LD_EXT(cla, dst, act) \ + (CMD_FIFO_LOAD_TYPE | FIFO_LOAD_EXT | CMD_CLASS(cla) | \ + FIFO_LOAD_INPUT(dst) | FIFO_LOAD_ACTION(act)) + +/* + * FIFO Load Immediate data length len to register dst class cla + * with action act. + */ +#define FIFO_LD_IMM(cla, dst, act, len) \ + (CMD_FIFO_LOAD_TYPE | CMD_IMM | CMD_CLASS(cla) | \ + FIFO_LOAD_INPUT(dst) | FIFO_LOAD_ACTION(act) | FIFO_LOAD_LENGTH(len)) + +/* + * Store value of length len from register src of class cla + */ +#define ST_NOIMM(cla, src, len) \ + (CMD_STORE_TYPE | CMD_CLASS(cla) | STORE_SRC(src) | STORE_LENGTH(len)) + +/* + * Store value of length len from register src of class cla + * Pointer is a Scatter/Gather Table + */ +#define ST_SGT_NOIMM(cla, src, len) \ + (CMD_STORE_TYPE | CMD_CLASS(cla) | CMD_SGT | STORE_SRC(src) | \ + STORE_LENGTH(len)) + +/* + * Store value of length len from register src of class cla starting + * at register offset off + */ +#define ST_NOIMM_OFF(cla, src, len, off) \ + (CMD_STORE_TYPE | CMD_CLASS(cla) | STORE_SRC(src) | \ + STORE_OFFSET(off) | STORE_LENGTH(len)) + +/* + * Store value of length len from register src of class cla + */ +#define ST_NOIMM_SEQ(cla, src, len) \ + (CMD_STORE_SEQ_TYPE | CMD_CLASS(cla) | STORE_SRC(src) | \ + STORE_LENGTH(len)) + +/* + * FIFO Store from register src of length len + */ +#define FIFO_ST(src, len) \ + (CMD_FIFO_STORE_TYPE | FIFO_STORE_OUTPUT(src) | FIFO_STORE_LENGTH(len)) + +/* + * FIFO Store from register src. + * The length is externally defined + */ +#define FIFO_ST_EXT(src) \ + (CMD_FIFO_STORE_TYPE | FIFO_STORE_EXT | FIFO_STORE_OUTPUT(src)) + +/* + * FIFO Store from register src of length len. + * Pointer is a Scatter/Gather Table + */ +#define FIFO_ST_SGT(src, len) \ + (CMD_FIFO_STORE_TYPE | CMD_SGT | FIFO_STORE_OUTPUT(src) | \ + FIFO_STORE_LENGTH(len)) + +/* + * FIFO Store from register src. + * Pointer is a Scatter/Gather Table + * The length is externally defined + */ +#define FIFO_ST_SGT_EXT(src) \ + (CMD_FIFO_STORE_TYPE | CMD_SGT | FIFO_STORE_EXT | \ + FIFO_STORE_OUTPUT(src)) + +/* + * SEQ FIFO Store from register src of length len + */ +#define FIFO_ST_SEQ(src, len) \ + (CMD_SEQ_FIFO_STORE_TYPE | FIFO_STORE_OUTPUT(src) | \ + FIFO_STORE_LENGTH(len)) + +/* + * RNG State Handle instantation operation for sh ID + */ +#define RNG_SH_INST(sh) \ + (CMD_OP_TYPE | OP_TYPE(CLASS1) | OP_ALGO(RNG) | ALGO_RNG_SH(sh) | \ + ALGO_AS(RNG_INSTANTIATE) | ALGO_RNG_PR) + +/* + * RNG Generates Secure Keys + */ +#define RNG_GEN_SECKEYS \ + (CMD_OP_TYPE | OP_TYPE(CLASS1) | OP_ALGO(RNG) | ALGO_RNG_SK | \ + ALGO_AS(RNG_GENERATE)) + +/* + * RNG Generates Data + */ +#define RNG_GEN_DATA \ + (CMD_OP_TYPE | OP_TYPE(CLASS1) | OP_ALGO(RNG) | ALGO_AS(RNG_GENERATE)) + +/* + * Hash Init Operation of algorithm algo + */ +#define HASH_INIT(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(INIT) | ALGO_ENCRYPT) + +/* + * Hash Update Operation of algorithm algo + */ +#define HASH_UPDATE(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(UPDATE) | \ + ALGO_ENCRYPT) + +/* + * Hash Final Operation of algorithm algo + */ +#define HASH_FINAL(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(FINAL) | ALGO_ENCRYPT) + +/* + * Hash Init and Final Operation of algorithm algo + */ +#define HASH_INITFINAL(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(INIT_FINAL) | \ + ALGO_ENCRYPT) + +/* + * HMAC Init Decryption Operation of algorithm algo + */ +#define HMAC_INIT_DECRYPT(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(INIT) | \ + ALGO_AAI(DIGEST_HMAC) | ALGO_DECRYPT) + +/* + * HMAC Init and Final Operation of algorithm algo with Precomp key + */ +#define HMAC_INITFINAL_PRECOMP(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(INIT_FINAL) | \ + ALGO_AAI(DIGEST_HMAC_PRECOMP) | ALGO_ENCRYPT) + +/* + * HMAC Init Operation of algorithm algo with Precomp key + */ +#define HMAC_INIT_PRECOMP(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(INIT) | \ + ALGO_AAI(DIGEST_HMAC_PRECOMP) | ALGO_ENCRYPT) + +/* + * HMAC Final Operation of algorithm algo with Precomp key + */ +#define HMAC_FINAL_PRECOMP(algo) \ + (CMD_OP_TYPE | OP_TYPE(CLASS2) | (algo) | ALGO_AS(FINAL) | \ + ALGO_AAI(DIGEST_HMAC_PRECOMP) | ALGO_ENCRYPT) + +/* + * Cipher Init and Final Operation of algorithm algo + */ +#define CIPHER_INITFINAL(algo, encrypt) \ + (CMD_OP_TYPE | OP_TYPE(CLASS1) | (algo) | ALGO_AS(INIT_FINAL) | \ + ((encrypt) ? ALGO_ENCRYPT : ALGO_DECRYPT)) + +/* + * Cipher Init Operation of algorithm algo + */ +#define CIPHER_INIT(algo, encrypt) \ + (CMD_OP_TYPE | OP_TYPE(CLASS1) | (algo) | ALGO_AS(INIT) | \ + ((encrypt) ? ALGO_ENCRYPT : ALGO_DECRYPT)) + +/* + * Cipher Update Operation of algorithm algo + */ +#define CIPHER_UPDATE(algo, encrypt) \ + (CMD_OP_TYPE | OP_TYPE(CLASS1) | (algo) | ALGO_AS(UPDATE) | \ + ((encrypt) ? ALGO_ENCRYPT : ALGO_DECRYPT)) + +/* + * Cipher Final Operation of algorithm algo + */ +#define CIPHER_FINAL(algo, encrypt) \ + (CMD_OP_TYPE | OP_TYPE(CLASS1) | (algo) | ALGO_AS(FINAL) | \ + ((encrypt) ? ALGO_ENCRYPT : ALGO_DECRYPT)) + +/* + * Load a class cla key of length len to register dst. + * Key can be stored in plain text. + */ +#define LD_KEY_PLAIN(cla, dst, len) \ + (CMD_KEY_TYPE | CMD_CLASS(cla) | KEY_PTS | KEY_DEST(dst) | \ + KEY_LENGTH(len)) + +/* + * Load a class cla key of length len to register dst. + * Key can be stored in plain text. + * Pointer is a Scatter/Gatter Table + */ +#define LD_KEY_SGT_PLAIN(cla, dst, len) \ + (CMD_KEY_TYPE | CMD_CLASS(cla) | CMD_SGT | KEY_PTS | KEY_DEST(dst) | \ + KEY_LENGTH(len)) + +/* + * Load a split key of length len. + */ +#define LD_KEY_SPLIT(len) \ + (CMD_KEY_TYPE | CMD_CLASS(CLASS_2) | KEY_DEST(MDHA_SPLIT) | \ + KEY_LENGTH(len)) + +/* + * MPPRIVK generation function. + */ +#define MPPRIVK (CMD_OP_TYPE | OP_TYPE(ENCAPS) | PROTID(MPKEY)) + +/* + * MPPUBK generation function. + */ +#define MPPUBK (CMD_OP_TYPE | OP_TYPE(DECAPS) | PROTID(MPKEY)) + +/* + * MPSIGN function. + */ +#define MPSIGN_OP (CMD_OP_TYPE | OP_TYPE(DECAPS) | PROTID(MPSIGN)) + +/* + * Operation Mathematical of length len + * dest = src0 (operation func) src1 + */ +#define MATH(func, src0, src1, dst, len) \ + (CMD_MATH_TYPE | MATH_FUNC(func) | MATH_SRC0(src0) | MATH_SRC1(src1) | \ + MATH_DST(dst) | MATH_LENGTH(len)) + +/* + * Operation Mathematical of length len using an immediate value as operand 1 + * dest = src (operation func) val + */ +#define MATHI_OP1(func, src, val, dst, len) \ + (CMD_MATHI_TYPE | MATH_FUNC(func) | MATHI_SRC(src) | \ + MATHI_IMM_VALUE(val) | MATHI_DST(dst) | MATH_LENGTH(len)) + +/* + * PKHA Copy function from src to dst. Copy number of words specified + * in Source size register + */ +#define PKHA_CPY_SSIZE(src, dst) \ + (CMD_OP_TYPE | OP_TYPE(PKHA) | PKHA_ALG | PKHA_FUNC(CPY_SSIZE) | \ + PKHA_CPY_SRC(src) | PKHA_CPY_DST(dst)) + +/* + * PKHA Copy N-Size function from src to dst. Copy number of words specified + * in PKHA N size register + */ +#define PKHA_CPY_NSIZE(src, dst) \ + (CMD_OP_TYPE | OP_TYPE(PKHA) | PKHA_ALG | PKHA_FUNC(CPY_NSIZE) | \ + PKHA_CPY_SRC(src) | PKHA_CPY_DST(dst)) + +/* + * PKHA Operation op result into dst + */ +#define PKHA_OP(op, dst) \ + (CMD_OP_TYPE | OP_TYPE(PKHA) | PKHA_ALG | PKHA_FUNC(op) | \ + PKHA_OUTSEL(dst)) + +/* + * PKHA Binomial operation op result into dst + */ +#define PKHA_F2M_OP(op, dst) \ + (CMD_OP_TYPE | OP_TYPE(PKHA) | PKHA_ALG | PKHA_F2M | PKHA_FUNC(op) | \ + PKHA_OUTSEL(dst)) + +/* + * Move src to dst + */ +#define MOVE(src, dst, off, len) \ + (CMD_MOVE_TYPE | MOVE_SRC(src) | MOVE_DST(dst) | MOVE_OFFSET(off) | \ + MOVE_LENGTH(len)) + +/* + * Move src to dst and wait until completion + */ +#define MOVE_WAIT(src, dst, off, len) \ + (CMD_MOVE_TYPE | MOVE_WC | MOVE_SRC(src) | MOVE_DST(dst) | \ + MOVE_OFFSET(off) | MOVE_LENGTH(len)) + +/* + * RSA Encryption using format + */ +#define RSA_ENCRYPT(format) \ + (CMD_OP_TYPE | PROTID(RSA_ENC) | PROT_RSA_FMT(format)) + +/* + * RSA Decryption using format + */ +#define RSA_DECRYPT(format) \ + (CMD_OP_TYPE | PROTID(RSA_DEC) | PROT_RSA_FMT(format)) + +/* + * RSA Finalize Key in format + */ +#define RSA_FINAL_KEY(format) \ + (CMD_OP_TYPE | PROTID(RSA_FINISH_KEY) | PROT_RSA_KEY(format)) + +/* + * Public Keypair generation + */ +#define PK_KEYPAIR_GEN(type) \ + (CMD_OP_TYPE | OP_TYPE(UNI) | PROTID(PKKEY) | PROT_PK_TYPE(type)) + +/* + * DSA/ECDSA signature of message hashed + */ +#define DSA_SIGN(type) \ + (CMD_OP_TYPE | OP_TYPE(UNI) | PROTID(DSASIGN) | PROT_PK_MSG(HASHED) | \ + PROT_PK_TYPE(type)) + +/* + * DSA/ECDSA signature verify message hashed + */ +#define DSA_VERIFY(type) \ + (CMD_OP_TYPE | OP_TYPE(UNI) | PROTID(DSAVERIFY) | \ + PROT_PK_MSG(HASHED) | PROT_PK_TYPE(type)) + +/* + * DH/ECC Shared Secret + */ +#define SHARED_SECRET(type) \ + (CMD_OP_TYPE | OP_TYPE(UNI) | PROTID(SHARED_SECRET) | \ + PROT_PK_TYPE(type)) + +/* + * Blob Master Key Verification + */ +#define BLOB_MSTR_KEY \ + (CMD_OP_TYPE | OP_TYPE(ENCAPS) | PROTID(BLOB) | PROT_BLOB_FMT_MSTR) + +/* + * Blob encapsulation + */ +#define BLOB_ENCAPS \ + (CMD_OP_TYPE | OP_TYPE(ENCAPS) | PROTID(BLOB) | \ + PROT_BLOB_FORMAT(NORMAL)) + +/* + * Blob decapsulation + */ +#define BLOB_DECAPS \ + (CMD_OP_TYPE | OP_TYPE(DECAPS) | PROTID(BLOB) | \ + PROT_BLOB_FORMAT(NORMAL)) + +/* + * Black key CCM size + */ +#define BLACK_KEY_CCM_SIZE(size) \ + (ROUNDUP(size, 8) + BLACK_KEY_NONCE_SIZE + BLACK_KEY_ICV_SIZE) + +/* + * Black key ECB size + */ +#define BLACK_KEY_ECB_SIZE(size) ROUNDUP(size, 16) + +/* + * Sequence Inout Pointer of length len + */ +#define SEQ_IN_PTR(len) (CMD_SEQ_IN_TYPE | SEQ_LENGTH(len)) + +/* + * Sequence Output Pointer of length len + */ +#define SEQ_OUT_PTR(len) (CMD_SEQ_OUT_TYPE | SEQ_LENGTH(len)) + +#endif /* __CAAM_DESC_HELPER_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_hal_cfg.h b/optee_os/core/drivers/crypto/caam/include/caam_hal_cfg.h new file mode 100644 index 0000000..2d80562 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_hal_cfg.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Configuration header. + */ +#ifndef __CAAM_HAL_CFG_H__ +#define __CAAM_HAL_CFG_H__ + +#include + +/* + * Returns the Job Ring Configuration to be used by the TEE + * + * @jrcfg [out] Job Ring Configuration + * + * Returns: + * CAAM_NO_ERROR Success + * CAAM_FAILURE An error occurred + */ +enum caam_status caam_hal_cfg_get_conf(struct caam_jrcfg *jrcfg); + +/* + * Setup the Non-Secure Job Ring + * + * @jrcfg Job Ring configuration + */ +void caam_hal_cfg_setup_nsjobring(struct caam_jrcfg *jrcfg); + +#ifdef CFG_DT +/* + * Returns the Job Ring configuration to be used by the TEE + * + * @fdt Device Tree handle + * @ctrl_base [out] CAAM Controller base address + */ +void caam_hal_cfg_get_ctrl_dt(void *fdt, vaddr_t *ctrl_base); + +/* + * Returns the Job Ring configuration to be used by the TEE + * + * @fdt Device Tree handle + * @jrcfg [out] Job Ring configuration + */ +void caam_hal_cfg_get_jobring_dt(void *fdt, struct caam_jrcfg *jrcfg); + +/* + * Disable the DT node related to the Job Ring used by secure world + * + * @fdt Device Tree handle + * @jrcfg Job Ring configuration + */ +void caam_hal_cfg_disable_jobring_dt(void *fdt, struct caam_jrcfg *jrcfg); +#else +static inline void caam_hal_cfg_get_ctrl_dt(void *fdt __unused, + vaddr_t *ctrl_base) +{ + *ctrl_base = 0; +} + +static inline void +caam_hal_cfg_get_jobring_dt(void *fdt __unused, + struct caam_jrcfg *jrcfg) +{ + jrcfg->offset = 0; + jrcfg->it_num = 0; +} + +static inline void +caam_hal_cfg_disable_jobring_dt(void *fdt __unused, + struct caam_jrcfg *jrcfg __unused) +{ +} +#endif /* CFG_DT */ + +#endif /* __CAAM_HAL_CFG_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_hal_clk.h b/optee_os/core/drivers/crypto/caam/include/caam_hal_clk.h new file mode 100644 index 0000000..b9f296e --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_hal_clk.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Clock functions header. + */ +#ifndef __CAAM_HAL_CLK_H__ +#define __CAAM_HAL_CLK_H__ + +#include + +/* + * Enable/disable the CAAM clocks + * + * @enable Enable the clock if true + */ +void caam_hal_clk_enable(bool enable); + +#endif /* __CAAM_HAL_CLK_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_hal_ctrl.h b/optee_os/core/drivers/crypto/caam/include/caam_hal_ctrl.h new file mode 100644 index 0000000..f3cea24 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_hal_ctrl.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2020 NXP + * + * Brief CAAM Controller Hardware Abstration Layer header. + */ +#ifndef __CAAM_HAL_CTRL_H__ +#define __CAAM_HAL_CTRL_H__ + +#include +#include + +/* + * Initializes the CAAM HW Controller + * + * @baseaddr Controller base address + */ +void caam_hal_ctrl_init(vaddr_t baseaddr); + +/* + * Returns the number of Job Ring supported + * + * @baseaddr Controller base address + */ +uint8_t caam_hal_ctrl_jrnum(vaddr_t baseaddr); + +/* + * If Hash operation is supported, returns the Maximum Hash Algorithm + * supported by the HW else UINT8_MAX + * + * @baseaddr Controller base address + */ +uint8_t caam_hal_ctrl_hash_limit(vaddr_t baseaddr); + +/* + * Returns the number of Public Key module supported + * + * @baseaddr Controller base address + */ +uint8_t caam_hal_ctrl_pknum(vaddr_t baseaddr); + +/* + * Returns if the HW support the split key operation. + * + * @baseaddr Controller base address + */ +bool caam_hal_ctrl_splitkey_support(vaddr_t baseaddr); + +/* + * Returns the CAAM Era + * + * @baseaddr Controller base address + */ +uint8_t caam_hal_ctrl_era(vaddr_t baseaddr); + +/* + * Increment the CAAM PRIBLOB field + * + * @baseaddr Controller base address + */ +void caam_hal_ctrl_inc_priblob(vaddr_t baseaddr); + +#ifdef CFG_NXP_CAAM_MP_DRV +/* + * Get the SCFGR content and check the MPCURVE fields. + * The function returns either: + * - UINT8_MAX if the Manafacturing Protection is not supported + * - The MP Curve Value if programmed (4 bits value) + * - 0 if the MP Curve is not programmed + * + * @ctrl_addr Controller base address + */ +uint8_t caam_hal_ctrl_get_mpcurve(vaddr_t ctrl_addr); + +/* + * Read the MPMR content + * + * @ctrl_addr Controller base address + * @mpmr [out] MPMR buffer read + */ +TEE_Result caam_hal_ctrl_read_mpmr(vaddr_t ctrl_addr, struct caambuf *mpmr); + +/* + * Fill the MPMR content then lock the register + * + * @ctrl_addr Controller base address + * @msg_mpmr Buffer with the message and length to fill the MPMR content + */ +void caam_hal_ctrl_fill_mpmr(vaddr_t ctrl_addr, struct caambuf *msg_mpmr); + +/* + * Indicate if the MP is set + * + * @ctrl_addr Controller base address + */ +bool caam_hal_ctrl_is_mp_set(vaddr_t ctrl_addr); +#endif /* CFG_NXP_CAAM_MP_DRV */ + +#ifdef CFG_NXP_CAAM_SM_DRV +/* + * Get the Secure Memory Virtual base address setup in the given job ring + * + * @ctrl_addr Controller base address + * @jr_offset Job ring offset + */ +vaddr_t caam_hal_ctrl_get_smvaddr(vaddr_t ctrl_addr, paddr_t jr_offset); +#endif /* CFG_NXP_CAAM_SM_DRV */ +#endif /* __CAAM_HAL_CTRL_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_hal_jr.h b/optee_os/core/drivers/crypto/caam/include/caam_hal_jr.h new file mode 100644 index 0000000..19cc68b --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_hal_jr.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Job Rings Hardware Abstration Layer header. + */ +#ifndef __CAAM_HAL_JR_H__ +#define __CAAM_HAL_JR_H__ + +#include + +/* + * Configures the Job Ring Owner and lock it. + * If the configuration is already locked, checks the configuration + * set and returns an error if value is not corresponding to the + * expected value. + * + * @ctrl_base Base address of the controller + * @jr_offset Job Ring offset to configure + * @owner Onwer ID to configure + */ +enum caam_status caam_hal_jr_setowner(vaddr_t ctrl_base, paddr_t jr_offset, + enum caam_jr_owner owner); + +/* + * Resets the Job Ring to ensure that all pending jobs are completed + * and no other will be executed + * + * @baseaddr Job Ring Base address + */ +enum caam_status caam_hal_jr_reset(vaddr_t baseaddr); + +/* + * Configures the Job Ring HW queues. + * + * @baseaddr Job Ring Base Address + * @nbjobs Number of job rings supported + * @inrings physical address of the JR input queue + * @outrings physical address of the JR output queue + */ +void caam_hal_jr_config(vaddr_t baseaddr, uint8_t nbjobs, uint64_t inrings, + uint64_t outrings); + +/* + * Returns the number of slots available in the input job ring + * + * @baseaddr Job Ring Base address + */ +uint32_t caam_hal_jr_read_nbslot_available(vaddr_t baseaddr); + +/* + * Indicates to HW that a new job is available + * + * @baseaddr Job Ring Base Address + */ +void caam_hal_jr_add_newjob(vaddr_t baseaddr); + +/* + * Returns the number of job completed and present in the output ring slots + * + * @baseaddr Job Ring Base Address + */ +uint32_t caam_hal_jr_get_nbjob_done(vaddr_t baseaddr); + +/* + * Removes a job from the job ring output queue + * + * @baseaddr Job Ring Base Address + */ +void caam_hal_jr_del_job(vaddr_t baseaddr); + +/* + * Disable and acknwoledge the Job Ring interrupt + * + * @baseaddr Job Ring Base Address + */ +void caam_hal_jr_disable_itr(vaddr_t baseaddr); + +/* + * Enable the Job Ring interrupt + * + * @baseaddr Job Ring Base Address + */ +void caam_hal_jr_enable_itr(vaddr_t baseaddr); + +/* + * If an interrupt is pending, acknowledges it and returns true. + * + * @baseaddr Job Ring Base Address + */ +bool caam_hal_jr_check_ack_itr(vaddr_t baseaddr); + +/* + * Halt the Job Ring processing. Stop fetching input queue and wait + * all running jobs normal completion. + * + * @baseaddr Job Ring Base Address + */ +enum caam_status caam_hal_jr_halt(vaddr_t baseaddr); + +/* + * Wait all Input queue Job Ring processing. + * + * @baseaddr Job Ring Base Address + */ +enum caam_status caam_hal_jr_flush(vaddr_t baseaddr); + +/* + * Resume the Job Ring processing. + * + * @baseaddr Job Ring Base Address + */ +void caam_hal_jr_resume(vaddr_t baseaddr); + +/* + * Returns the next entry free in the JR input queue. + * The HW increments register by 4. Convert it to a software index number + * + * @baseaddr CAAM JR Base Address + */ +uint8_t caam_hal_jr_input_index(vaddr_t baseaddr); + +/* + * Returns the next entry to read from the JR output queue. + * The HW increments register by 8. Convert it to a software index number + * + * @baseaddr CAAM JR Base Address + */ +uint8_t caam_hal_jr_output_index(vaddr_t baseaddr); + +/* + * Let the JR prepare data that need backup + * + * @ctrl_base CAAM JR Base Address + * @jr_offset Job Ring offset to prepare backup for + */ +void caam_hal_jr_prepare_backup(vaddr_t ctrl_base, paddr_t jr_offset); + +#endif /* __CAAM_HAL_JR_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_hal_rng.h b/optee_os/core/drivers/crypto/caam/include/caam_hal_rng.h new file mode 100644 index 0000000..9b87215 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_hal_rng.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019-2021 NXP + * + * Brief CAAM Random Number Generator Hardware Abstration Layer. + * Implementation of primitives to access HW + */ +#ifndef __CAAM_HAL_RNG_H__ +#define __CAAM_HAL_RNG_H__ + +#include +#include + +/* + * Returns if all RNG State Handler already instantiated or not + * + * @baseaddr RNG Base Address + */ +enum caam_status caam_hal_rng_instantiated(vaddr_t baseaddr); + +/* + * Returns the number of RNG State Handle + * + * @baseaddr RNG Base Address + */ +uint32_t caam_hal_rng_get_nb_sh(vaddr_t baseaddr); + +/* + * Returns the RNG Status State Handle + * + * @baseaddr RNG Base Address + */ +uint32_t caam_hal_rng_get_sh_status(vaddr_t baseaddr); + +/* + * Returns true if the RNG Key is loaded, false otherwise + * + * @baseaddr RNG Base Address + */ +bool caam_hal_rng_key_loaded(vaddr_t baseaddr); + +/* + * Configures the RNG entropy delay + * + * @baseaddr RNG Base Address + * @inc_delay Entropy Delay incrementation + */ +enum caam_status caam_hal_rng_kick(vaddr_t baseaddr, uint32_t inc_delay); + +#endif /* __CAAM_HAL_RNG_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_hal_sm.h b/optee_os/core/drivers/crypto/caam/include/caam_hal_sm.h new file mode 100644 index 0000000..171a9bd --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_hal_sm.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019, 2023 NXP + */ +#ifndef __CAAM_HAL_SM_H__ +#define __CAAM_HAL_SM_H__ + +#include +#include + +/* + * Checks if @page and @partition number are valid + * + * @jr_base JR base address + * @sm_page_desc Secure Memory page + */ +enum caam_status +caam_hal_sm_check_page_partition(vaddr_t jr_base, + const struct caam_sm_page_desc *sm_page_desc); + +/* + * Return the Pages Size in KBytes + * + * @jr_base JR base address + * @page Page number + */ +size_t caam_hal_sm_get_pages_size(vaddr_t jr_base, unsigned int page); + +/* + * Return if the partition is free (available) + * + * @jr_base JR base address + * @partition Partition number + */ +bool caam_hal_sm_prtn_is_free(vaddr_t jr_base, unsigned int partition); + +/* + * Return if the partition is owned (by the HW register reader) + * + * @jr_base JR base address + * @partition Partition number + */ +bool caam_hal_sm_prtn_is_owned(vaddr_t jr_base, unsigned int partition); + +/* + * Set the Secure Memory access to all groups + * + * @jr_base JR base address + * @partition Partition number + * @grp1 Group 1 value + * @grp2 Group 2 value + */ +void caam_hal_sm_set_access_all_group(vaddr_t jr_base, unsigned int partition); + +/* + * Set the Secure Memory access to group 1 and/or group 2 + * + * @jr_base JR base address + * @partition Partition number + * @grp1 Group 1 value + * @grp2 Group 2 value + */ +void caam_hal_sm_set_access_group(vaddr_t jr_base, unsigned int partition, + uint32_t grp1, uint32_t grp2); + +/* + * Open all Secure Memory Permissions + * + * @jr_base JR base address + * @partition Partition number + */ +void caam_hal_sm_open_access_perm(vaddr_t jr_base, unsigned int partition); + +/* + * Set the Secure Memory access permission for group 1 and group 2. + * Enable Critical Security and lock configuration + * + * @jr_base JR base address + * @partition Partition number + * @grp1_perm Group 1 Permissions + * @grp2_perm Group 2 Permissions + */ +void caam_hal_sm_set_access_perm(vaddr_t jr_base, unsigned int partition, + unsigned int grp1_perm, + unsigned int grp2_perm); + +/* + * Allocate a @page to the @partition. + * + * @jr_base JR base address + * @sm_page_desc Secure Memory page + */ +enum caam_status +caam_hal_sm_allocate_page(vaddr_t jr_base, + const struct caam_sm_page_desc *sm_page_desc); + +/* + * De-allocate a @partition and all partition's page. + * + * @jr_base JR base address + * @partition Partition number + */ +enum caam_status caam_hal_sm_deallocate_partition(vaddr_t jr_base, + unsigned int partition); + +/* + * De-allocate all pages specified in the @sm struct + * + * @jr_base JR base address + * @sm_page_desc Secure Memory page + */ +enum caam_status +caam_hal_sm_deallocate_pages(vaddr_t jr_base, + const struct caam_sm_page_desc *sm_page_desc); + +/* Return the virtual base address of the Secure Memory registers */ +vaddr_t caam_hal_sm_get_base(void); + +#ifdef CFG_DT +void caam_hal_sm_get_base_dt(void *fdt, vaddr_t *sm_base); +#else +static inline void caam_hal_sm_get_base_dt(void *fdt __unused, vaddr_t *sm_base) +{ + *sm_base = 0; +} +#endif /* CFG_DT */ +#endif /* __CAAM_HAL_SM_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_hash.h b/optee_os/core/drivers/crypto/caam/include/caam_hash.h new file mode 100644 index 0000000..62ef758 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_hash.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief CAAM Hash manager header. + */ +#ifndef __CAAM_HASH_H__ +#define __CAAM_HASH_H__ + +#include + +#ifdef CFG_NXP_CAAM_HASH_DRV +/* + * Initialize the Hash module + * + * @caam_jrcfg JR configuration structure + */ +enum caam_status caam_hash_init(struct caam_jrcfg *caam_jrcfg); +#else +static inline enum caam_status +caam_hash_init(struct caam_jrcfg *caam_jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_HASH_DRV */ + +#ifdef CFG_NXP_CAAM_HMAC_DRV +/* + * Initialize the HMAC module + * + * @caam_jrcfg JR configuration structure + */ +enum caam_status caam_hmac_init(struct caam_jrcfg *caam_jrcfg); +#else +static inline enum caam_status +caam_hmac_init(struct caam_jrcfg *caam_jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_HMAC_DRV */ +#endif /* __CAAM_HASH_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_io.h b/optee_os/core/drivers/crypto/caam/include/caam_io.h new file mode 100644 index 0000000..166875a --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_io.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019, 2021 NXP + * + * Brief Specific Macro used to read/write value with a specific + * format (BE/LE, 32/64 bits) to be updated for future platform + * support. + */ + +#ifndef __CAAM_IO_H__ +#define __CAAM_IO_H__ + +#include + +#ifdef CFG_CAAM_BIG_ENDIAN +/* Big Endian 32 bits Registers access */ +#define io_caam_read32(a) TEE_U32_FROM_BIG_ENDIAN(io_read32(a)) +#define io_caam_write32(a, val) io_write32(a, TEE_U32_TO_BIG_ENDIAN(val)) + +/* Big Endian 32 bits Value access */ +#define caam_read_val32(a) get_be32(a) +#define caam_write_val32(a, v) put_be32(a, v) +#else +/* Little Endian 32 bits Registers access */ +#define io_caam_read32(a) io_read32(a) +#define io_caam_write32(a, val) io_write32(a, val) + +/* Little Endian 32 bits Value access */ +#define caam_read_val32(a) get_le32(a) +#define caam_write_val32(a, v) put_le32(a, v) + +#define caam_read_val64(a) get_le64(a) +#define caam_write_val64(a, v) put_le64(a, v) +#endif + +#endif /* __CAAM_IO_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_jr.h b/optee_os/core/drivers/crypto/caam/include/caam_jr.h new file mode 100644 index 0000000..4456457 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_jr.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM Job Rings module header. + */ +#ifndef __CAAM_JR_H__ +#define __CAAM_JR_H__ + +#include +#include + +/* + * Job context to enqueue/dequeue + */ +struct caam_jobctx { + uint32_t *desc; /* reference to the descriptor */ + uint32_t status; /* executed job status */ + uint32_t id; /* Job identifier */ + bool completion; /* job completion flag */ + void *context; /* caller job context */ + void (*callback)(struct caam_jobctx *ctx); /* job completion callback */ +}; + +/* + * Job Ring module configuration + */ +struct caam_jrcfg { + vaddr_t base; /* CAAM virtual base address */ + paddr_t offset; /* Job Ring address offset */ + int it_num; /* Job Ring interrupt number */ + uint8_t nb_jobs; /* Number of Jobs to managed */ +}; + +/* + * The CAAM physical address is decorrelated from the CPU addressing mode. + * CAAM can manage 32 or 64 bits address depending on its version and the + * device. + */ +/* + * Definition of input and output ring object + */ +#ifdef CFG_CAAM_64BIT +struct caam_inring_entry { + uint64_t desc; /* Physical address of the descriptor */ +}; + +struct caam_outring_entry { + uint64_t desc; /* Physical address of the descriptor */ + uint32_t status; /* Status of the executed job */ +} __packed; +#else +struct caam_inring_entry { + uint32_t desc; /* Physical address of the descriptor */ +}; + +struct caam_outring_entry { + uint32_t desc; /* Physical address of the descriptor */ + uint32_t status; /* Status of the executed job */ +} __packed; +#endif /* CFG_CAAM_64BIT */ + +/* + * Initialization of the CAAM Job Ring module + * + * @jrcfg Job Ring Configuration + */ +enum caam_status caam_jr_init(struct caam_jrcfg *jrcfg); + +/* + * Cancels a job ID. Remove the job from SW Job array + * + * @job_id Job ID + */ +void caam_jr_cancel(uint32_t job_id); + +/* + * Checks if one of the given job IDs in bit mask format + * is completed. If none is completed, wait until timeout expires. + * Endlessly wait if @timeout_ms = UINT_MAX + * + * @job_ids Job IDs Mask + * @timeout_ms Timeout in millisecond + */ +enum caam_status caam_jr_dequeue(uint32_t job_ids, unsigned int timeout_ms); + +/* + * Enqueues a job in the Job Ring input queue and either wait until job + * completion or if job is asynchrnous, returns immediately (if status + * success, the output parameter job_id is filled with the Job Id pushed) + * + * @jobctx Reference to the job context + * @job_id [out] If pointer not NULL, job is asynchronous and parameter is + * the Job Id enqueued + */ +enum caam_status caam_jr_enqueue(struct caam_jobctx *jobctx, uint32_t *job_id); + +/* + * Request the CAAM JR to halt. + * Stop fetching input queue and wait running job completion. + */ +enum caam_status caam_jr_halt(void); + +/* Request the CAAM JR to flush all job running. */ +enum caam_status caam_jr_flush(void); + +/* + * Resume the CAAM JR processing. + * + * @pm_hints Hint on current power transition + */ +void caam_jr_resume(uint32_t pm_hints); + +/* Forces the completion of all CAAM Job to ensure CAAM is not BUSY. */ +enum caam_status caam_jr_complete(void); +#endif /* __CAAM_JR_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_jr_status.h b/optee_os/core/drivers/crypto/caam/include/caam_jr_status.h new file mode 100644 index 0000000..f978622 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_jr_status.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief CAAM Job Ring Status definition header. + */ +#ifndef __JR_STATUS_H__ +#define __JR_STATUS_H__ + +#include + +/* Source */ +#define BM_JRSTA_SRC SHIFT_U32(0xF, 28) + +#define JRSTA_SRC_GET(status) ((status) & BM_JRSTA_SRC) +#define JRSTA_SRC(src) SHIFT_U32(JRSTA_SRC_##src, 28) + +#define JRSTA_SRC_NONE 0x0 +#define JRSTA_SRC_CCB 0x2 +#define JRSTA_SRC_JMP_HALT_USER 0x3 +#define JRSTA_SRC_DECO 0x4 +#define JRSTA_SRC_JR 0x6 +#define JRSTA_SRC_JMP_HALT_COND 0x7 + +#define JRSTA_CCB_GET_ERR(status) ((status) & SHIFT_U32(0xFF, 0)) +#define JRSTA_CCB_CHAID_RNG SHIFT_U32(0x5, 4) +#define JRSTA_CCB_ERRID_HW SHIFT_U32(0xB, 0) +#define JRSTA_DECO_ERRID_FORMAT SHIFT_U32(0x88, 0) +#define JRSTA_DECO_INV_SIGNATURE SHIFT_U32(0x86, 0) + +/* Return the Halt User status else 0 if not a Jump Halt User */ +#define JRSTA_GET_HALT_USER(status) \ + (__extension__({ \ + __typeof__(status) _status = (status); \ + JRSTA_SRC_GET(_status) == JRSTA_SRC(JMP_HALT_USER) ? \ + _status & UINT8_MAX : \ + 0; })) +#endif /* __CAAM_JR_STATUS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_mp.h b/optee_os/core/drivers/crypto/caam/include/caam_mp.h new file mode 100644 index 0000000..ba37a1b --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_mp.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019, 2021, 2023 NXP + */ +#ifndef __CAAM_MP_H__ +#define __CAAM_MP_H__ + +#include "tee_api_types.h" +#include "types_ext.h" + +#ifdef CFG_NXP_CAAM_MP_DRV +/* + * Initialize the MP module and generate the private key + * + * @ctrl_addr Controller base address + */ +enum caam_status caam_mp_init(vaddr_t ctrl_addr); + +/* + * Power Management for MP + * + * @pm_hint Power mode type + */ +enum caam_status caam_mp_resume(uint32_t pm_hint); +#else +static inline enum caam_status caam_mp_init(vaddr_t ctrl_addr __unused) +{ + return CAAM_NO_ERROR; +} + +static inline enum caam_status caam_mp_resume(uint32_t pm_hint __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_MP_DRV */ + +#endif /* __CAAM_MP_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_pwr.h b/optee_os/core/drivers/crypto/caam/include/caam_pwr.h new file mode 100644 index 0000000..b8a365a --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_pwr.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019 NXP + * + * Brief CAAM driver common include file. + * Definition of the structure type to save and restore + * HW registers configuration + */ +#ifndef __CAAM_PWR_H__ +#define __CAAM_PWR_H__ + +#include + +/* + * Definition of the structure type used to list HW registers + * to be saved and restored. + */ +struct reglist { + uint32_t offset; /* Register offset */ + uint32_t nbregs; /* Number of consecutive registers */ + uint32_t mask_clr; /* Clear mask of bit to exclude in restore value */ + uint32_t mask_set; /* Set mask of bit to force in restore value */ +}; + +#define BACKUP_REG(_offset, _nbregs, _mask_clr, _mask_set) \ + { \ + .offset = _offset, .nbregs = _nbregs, .mask_clr = _mask_clr, \ + .mask_set = _mask_set, \ + } +/* + * Definition of the structure type used to store registers to backup + */ +struct backup_data { + vaddr_t baseaddr; /* Register virtual base address */ + size_t nbentries; /* Number of entries in the list */ + const struct reglist *regs; /* Register list */ + uint32_t *val; /* Register value */ + + SLIST_ENTRY(backup_data) next; /* Link to next data */ +}; + +/* + * Add definition of the backup data in the list + * + * @baseaddr Register base address + * @regs Register list + * @nbentries Number of entries in the list + */ +void caam_pwr_add_backup(vaddr_t baseaddr, const struct reglist *regs, + size_t nbentries); + +/* + * Power Initialization function called when all CAAM modules are + * initialized correctly. + * Register the PM callback in the system. + */ +void caam_pwr_init(void); + +#endif /* __CAAM_PWR_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_rng.h b/optee_os/core/drivers/crypto/caam/include/caam_rng.h new file mode 100644 index 0000000..3c5d100 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_rng.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Brief CAAM Random Number Generator manager header. + */ +#ifndef __CAAM_RNG_H__ +#define __CAAM_RNG_H__ + +/* + * Initialize the RNG module to generate data + * + * @ctrl_addr Controller base address + */ +enum caam_status caam_rng_init(vaddr_t ctrl_addr); + +/* Instantiates the RNG State Handles if not already done */ +enum caam_status caam_rng_instantiation(void); + +#endif /* __CAAM_RNG_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_sm.h b/optee_os/core/drivers/crypto/caam/include/caam_sm.h new file mode 100644 index 0000000..20acc48 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_sm.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019, 2023 NXP + */ +#ifndef __CAAM_SM_H__ +#define __CAAM_SM_H__ + +#include "caam_jr.h" +#include "drivers/caam_extension.h" + +/* + * Secure Memory data + */ +struct caam_sm_page_addr { + paddr_t paddr; /* Secure memory base address */ + vaddr_t vaddr; /* Secure memory virtual base address */ +}; + +/* + * Secure Memory Page(s)/Partition definition + */ +struct caam_sm_page_desc { + unsigned int partition; /* Partition number */ + unsigned int page; /* Page number */ + unsigned int page_count; /* Number of pages used */ +}; + +#ifdef CFG_NXP_CAAM_SM_DRV +/* + * CAAM Secure memory module initialization + * + * @jrcfg JR configuration structure + */ +enum caam_status caam_sm_init(struct caam_jrcfg *jrcfg); + +/* + * Allocate page(s) to one partition in the CAAM secure memory. + * Reset the group access and permission access to remove restrictions. + * + * @sm_page_descriptor Secure Memory page + * @sm_page_addr [out] Secure Memory page addresses + */ +enum caam_status +caam_sm_alloc(const struct caam_sm_page_desc *sm_page_descriptor, + struct caam_sm_page_addr *sm_page_addr); + +/* + * Set the Secure Memory group 1 and group 2 access rights to allocated + * partition and lock configuration. + * + * @page_desc Secure Memory page + * @grp1_perm Group 1 Permission value + * @grp2_perm Group 2 Permission value + */ +enum caam_status +caam_sm_set_access_perm(const struct caam_sm_page_desc *page_desc, + unsigned int grp1_perm, unsigned int grp2_perm); + +/* + * Free a Secure Memory pages + * + * @sm_page_descriptor Secure Memory page + */ +TEE_Result caam_sm_free(const struct caam_sm_page_desc *sm_page_descriptor); + +#else +static inline enum caam_status caam_sm_init(struct caam_jrcfg *jrcfg __unused) +{ + return CAAM_NO_ERROR; +} +#endif /* CFG_NXP_CAAM_SM_DRV */ +#endif /* __CAAM_SM_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_status.h b/optee_os/core/drivers/crypto/caam/include/caam_status.h new file mode 100644 index 0000000..9048578 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_status.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019, 2021 NXP + * + * Brief CAAM driver internal status definition + */ + +#ifndef __CAAM_STATUS_H__ +#define __CAAM_STATUS_H__ + +/* + * Internal CAAM Driver status codes + */ +enum caam_status { + CAAM_NO_ERROR = 0, /* No Error */ + CAAM_FAILURE, /* General failure */ + CAAM_NOT_SUPPORTED, /* Feature not supported */ + CAAM_OUT_MEMORY, /* Out of memory */ + CAAM_BAD_PARAM, /* Bad parameters */ + CAAM_SHORT_BUFFER, /* Buffer is too short */ + CAAM_BUSY, /* Operation is not possible, system busy */ + CAAM_PENDING, /* Operation is pending */ + CAAM_TIMEOUT, /* Operation timeout */ + CAAM_OUT_OF_BOUND, /* Value is out of boundary */ + CAAM_JOB_STATUS, /* A job status is available */ + CAAM_NOT_INIT, /* Feature is not initialized */ +}; + +#endif /* __CAAM_STATUS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_trace.h b/optee_os/core/drivers/crypto/caam/include/caam_trace.h new file mode 100644 index 0000000..77772c6 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_trace.h @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019-2021 NXP + * + * Brief CAAM driver trace include file. + * Definition of the internal driver trace macros. + */ + +#ifndef __CAAM_TRACE_H__ +#define __CAAM_TRACE_H__ + +#include +#include + +#define CAAM_DBG_TRACE(var) (CFG_DBG_CAAM_TRACE & DBG_TRACE_##var) +#define CAAM_DBG_DESC(var) (CFG_DBG_CAAM_DESC & DBG_TRACE_##var) +#define CAAM_DBG_BUF(var) (CFG_DBG_CAAM_BUF & DBG_TRACE_##var) + +/* + * Debug Macros function of CAAM Debug Level setting + * CFG_DBG_CAAM_TRACE Module print trace + * CFG_DBG_CAAM_DESC Module descriptor dump + * CFG_DBG_CAAM_BUF Module buffer dump + * + * A module is represented with the same bit in each configuration value. + * Module Bit definition is as follow: + */ +#define DBG_TRACE_HAL BIT32(0) /* HAL trace */ +#define DBG_TRACE_CTRL BIT32(1) /* Controller trace */ +#define DBG_TRACE_MEM BIT32(2) /* Memory utility trace */ +#define DBG_TRACE_SGT BIT32(3) /* Scatter Gather trace */ +#define DBG_TRACE_PWR BIT32(4) /* Power trace */ +#define DBG_TRACE_JR BIT32(5) /* Job Ring trace */ +#define DBG_TRACE_RNG BIT32(6) /* RNG trace */ +#define DBG_TRACE_HASH BIT32(7) /* Hash trace */ +#define DBG_TRACE_RSA BIT32(8) /* RSA trace */ +#define DBG_TRACE_CIPHER BIT32(9) /* Cipher dump Buffer */ +#define DBG_TRACE_BLOB BIT32(10) /* BLOB trace */ +#define DBG_TRACE_DMAOBJ BIT32(11) /* DMA Object trace */ +#define DBG_TRACE_ECC BIT32(12) /* ECC trace */ +#define DBG_TRACE_DH BIT32(13) /* DH trace */ +#define DBG_TRACE_DSA BIT32(14) /* DSA trace */ +#define DBG_TRACE_MP BIT32(15) /* MP trace */ +#define DBG_TRACE_SM BIT32(16) /* Secure Memory trace */ + +/* HAL */ +#if CAAM_DBG_TRACE(HAL) +#define HAL_TRACE DRV_TRACE +#else +#define HAL_TRACE(...) +#endif + +/* Controller */ +#if CAAM_DBG_TRACE(CTRL) +#define CTRL_TRACE DRV_TRACE +#else +#define CTRL_TRACE(...) +#endif + +/* Memory Utility */ +#if CAAM_DBG_TRACE(MEM) +#define MEM_TRACE DRV_TRACE +#else +#define MEM_TRACE(...) +#endif + +/* Scatter Gether Table */ +#if CAAM_DBG_TRACE(SGT) +#define SGT_TRACE DRV_TRACE +#else +#define SGT_TRACE(...) +#endif + +/* Power */ +#if CAAM_DBG_TRACE(PWR) +#define PWR_TRACE DRV_TRACE +#else +#define PWR_TRACE(...) +#endif + +/* Job Ring */ +#if CAAM_DBG_TRACE(JR) +#define JR_TRACE DRV_TRACE +#if CAAM_DBG_DESC(JR) +#define JR_DUMPDESC(desc) \ + do { \ + JR_TRACE("Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define JR_DUMPDESC(desc) +#endif +#else +#define JR_TRACE(...) +#define JR_DUMPDESC(desc) +#endif + +/* RNG */ +#if CAAM_DBG_TRACE(RNG) +#define RNG_TRACE DRV_TRACE +#if CAAM_DBG_DESC(RNG) +#define RNG_DUMPDESC(desc) \ + do { \ + RNG_TRACE("RNG Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define RNG_DUMPDESC(desc) +#endif +#else +#define RNG_TRACE(...) +#define RNG_DUMPDESC(desc) +#endif + +/* Hash */ +#if CAAM_DBG_TRACE(HASH) +#define HASH_TRACE DRV_TRACE +#if CAAM_DBG_DESC(HASH) +#define HASH_DUMPDESC(desc) \ + do { \ + HASH_TRACE("HASH Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define HASH_DUMPDESC(desc) +#endif +#if CAAM_DBG_BUF(HASH) +#define HASH_DUMPBUF DRV_DUMPBUF +#else +#define HASH_DUMPBUF(...) +#endif +#else +#define HASH_TRACE(...) +#define HASH_DUMPDESC(desc) +#define HASH_DUMPBUF(...) +#endif + +/* RSA */ +#if CAAM_DBG_TRACE(RSA) +#define RSA_TRACE DRV_TRACE +#if CAAM_DBG_DESC(RSA) +#define RSA_DUMPDESC(desc) \ + do { \ + RSA_TRACE("RSA Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define RSA_DUMPDESC(desc) +#endif +#if CAAM_DBG_BUF(RSA) +#define RSA_DUMPBUF DRV_DUMPBUF +#else +#define RSA_DUMPBUF(...) +#endif +#else +#define RSA_TRACE(...) +#define RSA_DUMPDESC(desc) +#define RSA_DUMPBUF(...) +#endif + +/* Cipher */ +#if CAAM_DBG_TRACE(CIPHER) +#define CIPHER_TRACE DRV_TRACE +#if CAAM_DBG_DESC(CIPHER) +#define CIPHER_DUMPDESC(desc) \ + do { \ + CIPHER_TRACE("CIPHER Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define CIPHER_DUMPDESC(desc) +#endif +#if CAAM_DBG_BUF(CIPHER) +#define CIPHER_DUMPBUF DRV_DUMPBUF +#else +#define CIPHER_DUMPBUF(...) +#endif +#else +#define CIPHER_TRACE(...) +#define CIPHER_DUMPDESC(desc) +#define CIPHER_DUMPBUF(...) +#endif + +/* DMA Object */ +#if CAAM_DBG_TRACE(DMAOBJ) +#define DMAOBJ_TRACE DRV_TRACE +#else +#define DMAOBJ_TRACE(...) +#endif + +/* ECC */ +#if CAAM_DBG_TRACE(ECC) +#define ECC_TRACE DRV_TRACE +#if CAAM_DBG_DESC(ECC) +#define ECC_DUMPDESC(desc) \ + do { \ + ECC_TRACE("ECC Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define ECC_DUMPDESC(desc) do { } while (0) +#endif +#if CAAM_DBG_BUF(ECC) +#define ECC_DUMPBUF DRV_DUMPBUF +#else +#define ECC_DUMPBUF(...) do { } while (0) +#endif +#else +#define ECC_TRACE(...) do { } while (0) +#define ECC_DUMPDESC(desc) do { } while (0) +#define ECC_DUMPBUF(...) do { } while (0) +#endif + +/* DH */ +#if CAAM_DBG_TRACE(DH) +#define DH_TRACE DRV_TRACE +#if CAAM_DBG_DESC(DH) +#define DH_DUMPDESC(desc) \ + { \ + DH_TRACE("DH Descriptor"); \ + DRV_DUMPDESC(desc); \ + } +#else +#define DH_DUMPDESC(desc) +#endif +#if CAAM_DBG_BUF(DH) +#define DH_DUMPBUF DRV_DUMPBUF +#else +#define DH_DUMPBUF(...) +#endif +#else +#define DH_TRACE(...) +#define DH_DUMPDESC(desc) +#define DH_DUMPBUF(...) +#endif + +/* DSA */ +#if CAAM_DBG_TRACE(DSA) +#define DSA_TRACE DRV_TRACE +#if CAAM_DBG_DESC(DSA) +#define DSA_DUMPDESC(desc) \ + do { \ + MP_TRACE("DSA Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define DSA_DUMPDESC(desc) +#endif +#if CAAM_DBG_BUF(DSA) +#define DSA_DUMPBUF DRV_DUMPBUF +#else +#define DSA_DUMPBUF(...) +#endif +#else +#define DSA_TRACE(...) +#define DSA_DUMPDESC(desc) +#define DSA_DUMPBUF(...) +#endif + +/* MP */ +#if CAAM_DBG_TRACE(MP) +#define MP_TRACE DRV_TRACE +#if CAAM_DBG_DESC(MP) +#define MP_DUMPDESC(desc) \ + do { \ + MP_TRACE("MP Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define MP_DUMPDESC(desc) +#endif +#if CAAM_DBG_BUF(MP) +#define MP_DUMPBUF DRV_DUMPBUF +#else +#define MP_DUMPBUF(...) +#endif +#else +#define MP_TRACE(...) do { } while (0) +#define MP_DUMPDESC(desc) +#define MP_DUMPBUF(...) +#endif + +#if (TRACE_LEVEL >= TRACE_DEBUG) +#define DRV_TRACE(...) \ + trace_printf(__func__, __LINE__, TRACE_DEBUG, true, __VA_ARGS__) +#define DRV_DUMPDESC(desc) dump_desc(desc) + +#define DRV_DUMPBUF(title, buf, len) \ + do { \ + __typeof__(buf) _buf = (buf); \ + __typeof__(len) _len = (len); \ + \ + DRV_TRACE("%s @%p : %zu", title, _buf, _len); \ + dhex_dump(NULL, 0, 0, _buf, _len); \ + } while (0) + +#else +#define DRV_TRACE(...) +#define DRV_DUMPDESC(...) +#define DRV_DUMPBUF(...) +#endif + +/* BLOB */ +#if CAAM_DBG_TRACE(BLOB) +#define BLOB_TRACE DRV_TRACE +#if CAAM_DBG_DESC(BLOB) +#define BLOB_DUMPDESC(desc) \ + do { \ + BLOB_TRACE("BLOB Descriptor"); \ + DRV_DUMPDESC(desc); \ + } while (0) +#else +#define BLOB_DUMPDESC(desc) +#endif +#if CAAM_DBG_BUF(BLOB) +#define BLOB_DUMPBUF DRV_DUMPBUF +#else +#define BLOB_DUMPBUF(...) +#endif +#else +#define BLOB_TRACE(...) +#define BLOB_DUMPDESC(desc) +#define BLOB_DUMPBUF(...) +#endif + +/* Secure Memory */ +#if CAAM_DBG_TRACE(SM) +#define SM_TRACE DRV_TRACE +#else +#define SM_TRACE(...) +#endif + +#endif /* CAAM_TRACE_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_types.h b/optee_os/core/drivers/crypto/caam/include/caam_types.h new file mode 100644 index 0000000..ffa0638 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_types.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020-2021 NXP + * + * CAAM driver data type definition. + */ + +#ifndef __CAAM_TYPES_H__ +#define __CAAM_TYPES_H__ + +#include + +/* + * Definition of a CAAM buffer type + */ +struct caambuf { + uint8_t *data; /* Data buffer */ + paddr_t paddr; /* Physical address of the buffer */ + size_t length; /* Number of bytes in the data buffer */ + uint8_t nocache; /* =1 if buffer is not cacheable, 0 otherwise */ +}; + +/* + * Definition of a CAAM Block buffer. Buffer used to store + * user source data to build a full algorithm block buffer + */ +struct caamblock { + struct caambuf buf; /* Data buffer */ + size_t filled; /* Current length filled in the buffer */ + size_t max; /* Maximum size of the block */ +}; + +/* + * Definition of key size + */ +struct caamdefkey { + uint8_t min; /* Minimum size */ + uint8_t max; /* Maximum size */ + uint8_t mod; /* Key modulus */ +}; + +#endif /* __CAAM_TYPES_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_utils_delay.h b/optee_os/core/drivers/crypto/caam/include/caam_utils_delay.h new file mode 100644 index 0000000..8d41601 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_utils_delay.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019 NXP + * + * Brief Delay management utilities header. + */ +#ifndef __CAAM_UTILS_DELAY_H__ +#define __CAAM_UTILS_DELAY_H__ + +/* + * Wait given microsecond + * + * @time Time in microsecond + */ +void caam_udelay(uint32_t time); + +#endif /* __CAAM_UTILS_DELAY_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_utils_dmaobj.h b/optee_os/core/drivers/crypto/caam/include/caam_utils_dmaobj.h new file mode 100644 index 0000000..518ebb6 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_utils_dmaobj.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020-2021 NXP + * + * CAAM DMA data object utilities include file. + */ +#ifndef __CAAM_UTILS_DMAOBJ_H__ +#define __CAAM_UTILS_DMAOBJ_H__ + +#include +#include +#include + +/* + * CAAM DMA Object type + * @priv Private object data not used externally. + * @orig Original data buffer + * @sgtbuf CAAM SGT/Buffer object + */ +struct caamdmaobj { + void *priv; + struct caambuf orig; + struct caamsgtbuf sgtbuf; +}; + +/* + * Initialize a CAAM DMA object of type input data. + * If necessary, a new CAAM Buffer will be reallocated if given @data is not + * accessible by the CAAM DMA and input data copied into. + * + * @obj [out] CAAM DMA object initialized + * @data Input data pointer + * @length Length in bytes of the input data + */ +TEE_Result caam_dmaobj_init_input(struct caamdmaobj *obj, const void *data, + size_t len); +/* + * Initialize and build the SGT/Buffer Object of a CAAM DMA object of type + * input data. + * Function call the caam_dmaobj_init_input function and if success + * the caam_dmaobj_sgtbuf_build function. If the full size of the input + * data can't be handled in the SGT/Buffer Object, returns in error. + * + * @obj [out] CAAM DMA object initialized + * @data Input data pointer + * @length Length in bytes of the input data + */ +TEE_Result caam_dmaobj_input_sgtbuf(struct caamdmaobj *obj, const void *data, + size_t len); +/* + * Initialize a CAAM DMA object of type output data. + * If necessary, a new CAAM Buffer will be reallocated if given @data is not + * accessible by the CAAM DMA or if the given @length is lower than + * @min_length requested for the CAAM operation. + * + * @obj [out] CAAM DMA object initialized + * @data Output data pointer + * @length Length in bytes of the output data + * @min_length Minimum length in bytes needed for the output data + */ +TEE_Result caam_dmaobj_init_output(struct caamdmaobj *obj, void *data, + size_t length, size_t min_length); + +/* + * Initialize and build the SGT/Buffer Object of a CAAM DMA object of type + * output data. + * Function call the caam_dmaobj_init_output function and if success + * the caam_dmaobj_sgtbuf_build function. If the full size of the output + * data can't be handled in the SGT/Buffer Object, returns in error. + * + * Note: to allocate a output buffer, set @data = NULL and length = 0, the + * buffer size allocated will be the @min_length size. Caution, the field + * orig of the @obj is kept empty. + * + * @obj [out] CAAM DMA object initialized + * @data Output data pointer + * @length Length in bytes of the output data + * @min_length Minimum length in bytes needed for the output data + */ +TEE_Result caam_dmaobj_output_sgtbuf(struct caamdmaobj *obj, void *data, + size_t length, size_t min_length); + +/* + * Push the data to physical memory with a cache clean or flush depending + * on the type of data, respectively input or output. + * + * @obj CAAM DMA object + */ +void caam_dmaobj_cache_push(struct caamdmaobj *obj); + +/* + * Copy the CAAM DMA object buffer to the original data buffer. + * Return the number of bytes copied. + * + * @obj CAAM DMA object + */ +size_t caam_dmaobj_copy_to_orig(struct caamdmaobj *obj); + +/* + * Copy the CAAM DMA object buffer to the original data buffer removing + * non-significant first zeros (left zeros). + * If all DMA object buffer is zero, left only one zero in the destination. + * Return the number of bytes copied. + * + * @obj CAAM DMA object + */ +size_t caam_dmaobj_copy_ltrim_to_orig(struct caamdmaobj *obj); + +/* + * Free the CAAM DMA object. + * If a buffer has been reallocated, free it. + * Free the sgtbuf object. + * + * @obj CAAM DMA object + */ +void caam_dmaobj_free(struct caamdmaobj *obj); + +/* + * Create a CAAM DMA object SGT type with the block buffer @block first and + * the CAAM DMA Object after + * + * @res CAAM DMA object resulting + * @block CAAM Block buffer to add first + * @obj CAAM DMA object to add secondly + */ +TEE_Result caam_dmaobj_add_first_block(struct caamdmaobj *obj, + struct caamblock *block); + +/* + * Derive a CAAM DMA object's sgtbuf object to a new DMA object. + * The @from CAAM DMA object sgtbuf must have to be created first to + * allocate the DMA Buffers. + * + * @obj [out] CAAM DMA object derived + * @from Original CAAM DMA object + * @offset Offset to start from + * @length Length in bytes of the data + */ +TEE_Result caam_dmaobj_derive_sgtbuf(struct caamdmaobj *obj, + const struct caamdmaobj *from, + size_t offset, size_t length); + +/* + * Build the CAAM DMA Object's sgtbuf input and output with the same data + * length. + * First try to build input sgtbuf of maximum @length starting at @offset. + * Then build output sgtbuf with same input data length built start at @offset. + * If output sgtbuf built data length is not the same as the input's one, + * rebuild the input with same output data length. + * + * If the both input and output length are not equal returns an error. + * + * @input CAAM DMA Input object + * @output CAAM DMA Output object + * @length [in/out] maximum length to do/done + * @off Starting offset + * @align Buffer allocation alignment + */ +TEE_Result caam_dmaobj_sgtbuf_inout_build(struct caamdmaobj *input, + struct caamdmaobj *output, + size_t *length, size_t off, + size_t align); + +/* + * Prepare input/output CAAM DMA Object's by allocating the DMA Buffers + * if needed. + * If @input or @output is NULL, allocates DMA buffer of given object. + * Else if both objects are set, allocates DMA buffer of the same + * size for the @input and @output objects. + * Minimum DMA Buffer size allocated is the @min_size value. Even if this + * minimum size allocation failed, returns an error. + * + * @input CAAM DMA object input + * @output CAAM DMA object output + * @min_size Mimimum length to allocate + */ +TEE_Result caam_dmaobj_prepare(struct caamdmaobj *input, + struct caamdmaobj *output, size_t min_size); + +/* + * Build the CAAM DMA Object's sgtbuf object. Try to build a sgtbuf of + * maximum @length starting at @offset. + * Return the @length mapped in the sgtbuf object. + * + * @obj CAAM DMA object + * @length [in/out] maximum length to do/done + * @off Starting offset + * @align Buffer allocation alignment + */ +TEE_Result caam_dmaobj_sgtbuf_build(struct caamdmaobj *obj, size_t *length, + size_t off, size_t align); + +#endif /* __CAAM_UTILS_DMAOBJ_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_utils_mem.h b/optee_os/core/drivers/crypto/caam/include/caam_utils_mem.h new file mode 100644 index 0000000..cb7ce2c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_utils_mem.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief Memory management utilities. + * Primitive to allocate, free memory. + */ + +#ifndef __CAAM_UTILS_MEM_H__ +#define __CAAM_UTILS_MEM_H__ + +#include + +/* + * Allocate normal memory. + * + * @size size in bytes of the memory to allocate + */ +void *caam_alloc(size_t size); + +/* + * Allocate normal memory and initialize it to 0's. + * + * @size size in bytes of the memory to allocate + */ +void *caam_calloc(size_t size); + +/* + * Allocate memory aligned with a cache line and initialize it to 0's. + * + * @size size in bytes of the memory to allocate + */ +void *caam_calloc_align(size_t size); + +/* + * Free allocated memory + * + * @ptr reference to the object to free + */ +void caam_free(void *ptr); + +/* + * Allocate Job descriptor and initialize it to 0's. + * + * @nbentries Number of descriptor entries + */ +uint32_t *caam_calloc_desc(uint8_t nbentries); + +/* + * Free descriptor + * + * @ptr Reference to the descriptor to free + */ +void caam_free_desc(uint32_t **ptr); + +/* + * Allocate internal driver buffer and initialize it with 0s. + * + * @buf [out] buffer allocated + * @size size in bytes of the memory to allocate + */ +enum caam_status caam_calloc_buf(struct caambuf *buf, size_t size); + +/* + * Allocate internal driver buffer. + * + * @buf [out] buffer allocated + * @size size in bytes of the memory to allocate + */ +enum caam_status caam_alloc_buf(struct caambuf *buf, size_t size); + +/* + * Allocate internal driver buffer aligned with a cache line and initialize + * if with 0's. + * + * @buf [out] buffer allocated + * @size size in bytes of the memory to allocate + */ +enum caam_status caam_calloc_align_buf(struct caambuf *buf, size_t size); + +/* + * Allocate internal driver buffer aligned with a cache line. + * + * @buf [out] buffer allocated + * @size size in bytes of the memory to allocate + */ +enum caam_status caam_alloc_align_buf(struct caambuf *buf, size_t size); + +/* + * Free internal driver buffer allocated memory + * + * @buf Driver buffer to free + */ +void caam_free_buf(struct caambuf *buf); + +/* + * Copy source data into the block buffer. Allocate block buffer if + * it's not defined. + * + * @block [in/out] Block buffer information. Return buffer filled. + * @src Source to copy + * @offset Source offset to start + */ +enum caam_status caam_cpy_block_src(struct caamblock *block, + struct caambuf *src, size_t offset); + +/* + * Return the number of Physical Areas used by the buffer @buf. + * If @pabufs is not NULL, function fills it with the Physical Areas used + * to map the buffer @buf. + * + * @buf Data buffer to analyze + * @pabufs[out] If not NULL, list the Physical Areas of the @buf + * + * Returns: + * Number of physical area used + * (-1) if error + */ +int caam_mem_get_pa_area(struct caambuf *buf, struct caambuf **pabufs); + +/* + * Return if the buffer @buf is cacheable or not + * + * @buf Buffer address + * @size Buffer size + */ +bool caam_mem_is_cached_buf(void *buf, size_t size); + +/* + * Copy source data into the destination buffer removing non-significant + * first zeros (left zeros). + * If all source @src buffer is zero, left only one zero in the destination. + * + * @dst [out] Destination buffer + * @src Source to copy + */ +void caam_mem_cpy_ltrim_buf(struct caambuf *dst, struct caambuf *src); + +#endif /* __CAAM_UTILS_MEM_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_utils_sgt.h b/optee_os/core/drivers/crypto/caam/include/caam_utils_sgt.h new file mode 100644 index 0000000..4dd6b68 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_utils_sgt.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019, 2021 NXP + * + * Brief Scatter-Gather Table management utilities header. + */ +#ifndef __CAAM_UTILS_SGT_H__ +#define __CAAM_UTILS_SGT_H__ + +#include +#include + +#define BP_SGT_V2_OFFSET 48 +#define BS_SGT_V2_OFFSET 12 +#define SGT_V2_OFFSET_MAX_VALUE GENMASK_64(BS_SGT_V2_OFFSET - 1, 0) +#define BM_SGT_V2_OFFSET \ + SHIFT_U64(GENMASK_64(BS_SGT_V2_OFFSET - 1, 0), BP_SGT_V2_OFFSET) +#define BV_SGT_V2_OFFSET(_x) SHIFT_U64(((uint64_t)_x), BP_SGT_V2_OFFSET) +#define SGT_V2_ENTRY_OFFSET(_x) \ + ((((uint64_t)_x) & BM_SGT_V2_OFFSET) >> BP_SGT_V2_OFFSET) + +#define BP_SGT_V2_AVAIL_LENGTH 0 +#define BS_SGT_V2_AVAIL_LENGTH 32 +#define SGT_V2_AVAIL_LENGTH_MAX_VALUE GENMASK_64(BS_SGT_V2_AVAIL_LENGTH - 1, 0) +#define BM_SGT_V2_AVAIL_LENGTH \ + SHIFT_U64(SGT_V2_AVAIL_LENGTH_MAX_VALUE, BP_SGT_V2_AVAIL_LENGTH) +#define BV_SGT_V2_AVAIL_LENGTH(_x) \ + SHIFT_U64(((uint64_t)_x), BP_SGT_V2_AVAIL_LENGTH) +#define SGT_V2_ENTRY_AVAIL_LENGTH(_x) \ + ((((uint64_t)_x) & BM_SGT_V2_AVAIL_LENGTH) >> BP_SGT_V2_AVAIL_LENGTH) + +#define BP_SGT_V2_F 63 +#define BM_SGT_V2_F BIT64(BP_SGT_V2_F) +#define BP_SGT_V2_IVP 46 +#define BM_SGT_V2_IVP BIT64(BP_SGT_V2_IVP) + +/* + * Scatter/Gather Table type for input and output data + */ +union caamsgt { + struct { + /* W0 - Address pointer (MS 8 LSBs) */ + uint32_t ptr_ms; + /* W1 - Address pointer (LS 32 bits) */ + uint32_t ptr_ls; + /* W2 - Length 30bits, 1bit Final, 1bit Extension */ + uint32_t len_f_e; + /* W3- Offset in memory buffer (13 LSBs) */ + uint32_t offset; + } v1; + struct { + uint64_t w1; /* Address of the data */ + uint64_t w2; /* Final bit, offset and length */ + } v2; +}; + +/* + * Data buffer encoded in SGT format + */ +struct caamsgtbuf { + union caamsgt *sgt; /* SGT Array */ + struct caambuf *buf; /* Buffer Array */ + unsigned int number; /* Number of SGT/Buf */ + size_t length; /* Total length of the data encoded */ + paddr_t paddr; /* Physical address to use in CAAM descriptor */ + bool sgt_type; /* Define the data format */ +}; + +/* + * Allocate data of type struct caamsgtbuf + * + * @data [out] Data object allocated + */ +enum caam_status caam_sgtbuf_alloc(struct caamsgtbuf *data); + +/* + * Free data of type struct caamsgtbuf + * + * @data Data object to free + */ +void caam_sgtbuf_free(struct caamsgtbuf *data); + +/* + * Cache operation on SGT table + * + * @op Cache operation + * @insgt SGT table + * @length Length of data to maintain + */ +void caam_sgt_cache_op(enum utee_cache_operation op, struct caamsgtbuf *insgt, + size_t length); + +/* + * Set a Scatter Gather Table Entry + * + * @sgt SGT entry + * @paddr Data's physical address + * @len Data's length + * @offset Offset to start in data buffer + * @final_e Final entry in the table if true + */ +void caam_sgt_set_entry(union caamsgt *sgt, vaddr_t paddr, size_t len, + unsigned int offset, bool final_e); + +#define CAAM_SGT_ENTRY(sgt, paddr, len) \ + caam_sgt_set_entry(sgt, paddr, len, 0, false) +#define CAAM_SGT_ENTRY_FINAL(sgt, paddr, len) \ + caam_sgt_set_entry(sgt, paddr, len, 0, true) + +/* + * Build a SGT object with @data buffer. + * If the @data buffer is a buffer mapped on non-contiguous physical areas, + * convert it in SGT entries. + * Fill the CAAM SGT table with the buffer list in @sgt parameter + * + * @sgt [in/out] SGT buffer list and table + */ +void caam_sgt_fill_table(struct caamsgtbuf *sgt); + +/* + * Derive a CAAM SGT table from the @from SGT table starting @offset. + * Allocate the resulting SGT table derived. + * + * @sgt [out] SGT buffer list and table + * @from Input SGT table + * @offset Offset to start + * @length Length of the new SGT data + */ +enum caam_status caam_sgt_derive(struct caamsgtbuf *sgt, + const struct caamsgtbuf *from, size_t offset, + size_t length); + +/* + * Print the details of an SGT entry using the trace macro + * + * @idx [in]Index of the sgt to print + * @sgt [in] SGT buffer list and table + */ +void sgt_entry_trace(unsigned int idx, const struct caamsgtbuf *sgt); + +/* + * Add an @offset to the SGT entry + * + * @sgt [in/out] Sgt entry + * @offset Offset to add + */ +void sgt_entry_offset(union caamsgt *sgt, unsigned int offset); + +#endif /* __CAAM_UTILS_SGT_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/include/caam_utils_status.h b/optee_os/core/drivers/crypto/caam/include/caam_utils_status.h new file mode 100644 index 0000000..8da32dc --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/include/caam_utils_status.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019, 2021 NXP + * + * Brief Status code management utilities header. + */ +#ifndef __CAAM_UTILS_STATUS_H__ +#define __CAAM_UTILS_STATUS_H__ + +#include +#include + +/* + * Convert Job status code to TEE Result + * + * @status Job status code + */ +TEE_Result job_status_to_tee_result(uint32_t status); + +/* + * Convert caam status code to TEE Result + * + * @status caam status code + */ +TEE_Result caam_status_to_tee_result(enum caam_status status); + +#endif /* __CAAM_UTILS_STATUS_H__ */ diff --git a/optee_os/core/drivers/crypto/caam/mp/caam_mp.c b/optee_os/core/drivers/crypto/caam/mp/caam_mp.c new file mode 100644 index 0000000..6447f21 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/mp/caam_mp.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021, 2023 NXP + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MP_SIGN_MAX_MSG_SIZE (4 * 1024) + +#ifdef CFG_PHYS_64BIT +#define MP_PRIV_DESC_ENTRIES 7 +#define MP_PUB_DESC_ENTRIES 7 +#define MP_SIGN_DESC_ENTRIES 13 +#else +#define MP_PRIV_DESC_ENTRIES 6 +#define MP_PUB_DESC_ENTRIES 6 +#define MP_SIGN_DESC_ENTRIES 9 +#endif + +/* + * MP module private data + */ +static struct mp_privdata { + uint8_t curve; /* Protocol Data Block curve selection */ + uint8_t sec_size; /* Security key size in bytes */ + vaddr_t ctrl_addr; /* Base address of the controller */ + enum caam_status mp_status; /* Indicate the MP status */ +} mp_privdata; + +/* + * Generate manufacturing private key. + * The ECDSA private key is securely stored in the MPPKR. + * This register is locked to prevent reading or writing. + * + * @passphrase Passphrase + * @len Passphrase length + */ +static enum caam_status do_mppriv_gen(const char *passphrase, size_t len) +{ + enum caam_status ret = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + + MP_TRACE("MP private key generation"); + + assert(passphrase && len); + + desc = caam_calloc_desc(MP_PRIV_DESC_ENTRIES); + if (!desc) + return CAAM_OUT_MEMORY; + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, PROT_MP_CURVE(mp_privdata.curve)); + caam_desc_add_ptr(desc, virt_to_phys((void *)passphrase)); + caam_desc_add_word(desc, len); + caam_desc_add_word(desc, MPPRIVK); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + MP_DUMPDESC(desc); + + cache_operation(TEE_CACHECLEAN, (void *)passphrase, len); + + jobctx.desc = desc; + ret = caam_jr_enqueue(&jobctx, NULL); + + if (ret != CAAM_NO_ERROR) { + MP_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = CAAM_NOT_SUPPORTED; + } + + caam_free_desc(&desc); + return ret; +} + +TEE_Result caam_mp_export_mpmr(uint8_t *mpmr, size_t *size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct caambuf caam_mpmr = { + .data = mpmr, + .length = *size, + }; + + MP_TRACE("Get MP message"); + + ret = caam_hal_ctrl_read_mpmr(mp_privdata.ctrl_addr, &caam_mpmr); + *size = caam_mpmr.length; + + return ret; +} + +TEE_Result caam_mp_export_publickey(uint8_t *pubkey, size_t *size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + struct caamdmaobj reskey = { }; + uint32_t pdb_sgt_flag = 0; + uint32_t desclen = 0; + uint32_t *desc = NULL; + + /* Check if MP is operational */ + if (mp_privdata.mp_status != CAAM_NO_ERROR) + return caam_status_to_tee_result(mp_privdata.mp_status); + + if (!pubkey || !size) + return TEE_ERROR_BAD_PARAMETERS; + + /* The public key size is twice the private key size */ + if (*size < 2 * mp_privdata.sec_size) { + *size = 2 * mp_privdata.sec_size; + return TEE_ERROR_SHORT_BUFFER; + } + + ret = caam_dmaobj_output_sgtbuf(&reskey, pubkey, *size, + 2 * mp_privdata.sec_size); + if (ret) + return ret; + + if (reskey.sgtbuf.sgt_type) + pdb_sgt_flag = PROT_MP_PUBK_SGT; + + desc = caam_calloc_desc(MP_PUB_DESC_ENTRIES); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, + PROT_MP_CURVE(mp_privdata.curve) | pdb_sgt_flag); + caam_desc_add_ptr(desc, reskey.sgtbuf.paddr); + caam_desc_add_word(desc, reskey.sgtbuf.length); + caam_desc_add_word(desc, MPPUBK); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + MP_DUMPDESC(desc); + + caam_dmaobj_cache_push(&reskey); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + MP_TRACE("MP Public Key generated"); + reskey.orig.length = 2 * mp_privdata.sec_size; + *size = caam_dmaobj_copy_to_orig(&reskey); + + MP_DUMPBUF("MP PubKey", pubkey, *size); + + ret = caam_status_to_tee_result(retstatus); + } else { + MP_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +out: + caam_dmaobj_free(&reskey); + caam_free_desc(&desc); + + return ret; +} + +TEE_Result caam_mp_sign(uint8_t *msg, size_t *msg_size, uint8_t *sig, + size_t *sig_size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct caam_jobctx jobctx = { }; + struct caamdmaobj msg_input = { }; + struct caamdmaobj sign_c = { }; + struct caamdmaobj sign_d = { }; + struct caambuf hash = { }; + uint32_t *desc = NULL; + uint32_t desclen = 0; + uint32_t pdb_sgt_flags = 0; + uint8_t *aligned_msg = NULL; + size_t sign_len = 0; + + MP_TRACE("MP sign operation"); + + /* Check if MP is operational */ + if (mp_privdata.mp_status != CAAM_NO_ERROR) + return caam_status_to_tee_result(mp_privdata.mp_status); + + if (!msg || !msg_size || !sig || !sig_size) + return TEE_ERROR_BAD_PARAMETERS; + + if (*sig_size < 2 * mp_privdata.sec_size) { + *sig_size = 2 * mp_privdata.sec_size; + return TEE_ERROR_SHORT_BUFFER; + } + + if (*msg_size > MP_SIGN_MAX_MSG_SIZE) + return TEE_ERROR_EXCESS_DATA; + + /* Re-allocate the message to a cache-aligned buffer */ + aligned_msg = caam_alloc(*msg_size); + if (!aligned_msg) { + MP_TRACE("Message reallocation error"); + ret = TEE_ERROR_OUT_OF_MEMORY; + goto exit_mpsign; + } + memcpy(aligned_msg, msg, *msg_size); + + /* + * Allocate the hash buffer of the Message + MPMR payload + * Note: Hash is not retrieve, hence no need to do cache + * maintenance + */ + retstatus = caam_alloc_align_buf(&hash, TEE_MAX_HASH_SIZE); + if (retstatus != CAAM_NO_ERROR) { + MP_TRACE("Hash allocation error"); + ret = caam_status_to_tee_result(retstatus); + goto exit_mpsign; + } + + /* + * Re-allocate the signature result buffer with a maximum size + * of the roundup to 16 bytes of the secure size in bytes if + * the signature buffer is not aligned or too short. + * + * - 1st Part: size_sec + * - 2nd Part: size_sec roundup to 16 bytes + */ + sign_len = ROUNDUP(mp_privdata.sec_size, 16) + mp_privdata.sec_size; + + ret = caam_dmaobj_output_sgtbuf(&sign_c, sig, *sig_size, sign_len); + if (ret) + goto exit_mpsign; + + if (sign_c.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_MP_SIGN_C; + + /* Prepare the 2nd Part of the signature. Derived from sign_c */ + ret = caam_dmaobj_derive_sgtbuf(&sign_d, &sign_c, mp_privdata.sec_size, + ROUNDUP(mp_privdata.sec_size, 16)); + if (ret) + goto exit_mpsign; + + if (sign_d.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_MP_SIGN_D; + + ret = caam_dmaobj_input_sgtbuf(&msg_input, aligned_msg, *msg_size); + if (ret) + goto exit_mpsign; + + if (msg_input.sgtbuf.sgt_type) + pdb_sgt_flags |= PDB_SGT_MP_SIGN_MSG; + + desc = caam_calloc_desc(MP_SIGN_DESC_ENTRIES); + if (!desc) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto exit_mpsign; + } + + caam_desc_init(desc); + caam_desc_add_word(desc, DESC_HEADER(0)); + caam_desc_add_word(desc, + PROT_MP_CURVE(mp_privdata.curve) | pdb_sgt_flags); + caam_desc_add_ptr(desc, msg_input.sgtbuf.paddr); + caam_desc_add_ptr(desc, hash.paddr); + caam_desc_add_ptr(desc, sign_c.sgtbuf.paddr); + caam_desc_add_ptr(desc, sign_d.sgtbuf.paddr); + caam_desc_add_word(desc, msg_input.sgtbuf.length); + caam_desc_add_word(desc, MPSIGN_OP); + + desclen = caam_desc_get_len(desc); + caam_desc_update_hdr(desc, DESC_HEADER_IDX(desclen, desclen - 1)); + + MP_DUMPDESC(desc); + + caam_dmaobj_cache_push(&msg_input); + caam_dmaobj_cache_push(&sign_c); + + jobctx.desc = desc; + retstatus = caam_jr_enqueue(&jobctx, NULL); + + if (retstatus == CAAM_NO_ERROR) { + sign_c.orig.length = 2 * mp_privdata.sec_size; + *sig_size = caam_dmaobj_copy_to_orig(&sign_c); + + MP_DUMPBUF("MP Signature", sdata->signature.data, + sdata->signature.length); + + ret = caam_status_to_tee_result(retstatus); + } else { + MP_TRACE("CAAM Status 0x%08" PRIx32, jobctx.status); + ret = job_status_to_tee_result(jobctx.status); + } + +exit_mpsign: + caam_free(aligned_msg); + caam_free_buf(&hash); + caam_free_desc(&desc); + caam_dmaobj_free(&msg_input); + caam_dmaobj_free(&sign_c); + caam_dmaobj_free(&sign_d); + + return ret; +} + +enum caam_status caam_mp_init(vaddr_t ctrl_addr) +{ + /* + * Manufacturing protection secret values for DSA key pair + * generation. + */ + static const char passphrase[] = "manufacturing protection"; + static const char mpmr_data[] = "value to fill the MPMR content"; + enum caam_status retstatus = CAAM_FAILURE; + uint8_t curve = 0; + uint8_t hash_limit = 0; + + struct caambuf msg_mpmr = { + .data = (uint8_t *)mpmr_data, + .length = strlen(mpmr_data) + }; + + mp_privdata.ctrl_addr = ctrl_addr; + mp_privdata.mp_status = CAAM_NOT_INIT; + + curve = caam_hal_ctrl_get_mpcurve(ctrl_addr); + + if (curve == UINT8_MAX) { + mp_privdata.mp_status = CAAM_NOT_SUPPORTED; + return mp_privdata.mp_status; + } + + if (caam_hal_ctrl_is_mp_set(ctrl_addr)) { + mp_privdata.mp_status = CAAM_NO_ERROR; + return CAAM_NO_ERROR; + } + + if (!curve) { + /* Get the device HASH Limit to select the MP Curve */ + hash_limit = caam_hal_ctrl_hash_limit(ctrl_addr); + + switch (hash_limit) { + case TEE_MAIN_ALGO_SHA256: + mp_privdata.curve = PDB_MP_CSEL_P256; + mp_privdata.sec_size = 32; + break; + case TEE_MAIN_ALGO_SHA512: + mp_privdata.curve = PDB_MP_CSEL_P521; + mp_privdata.sec_size = 66; + break; + default: + MP_TRACE("This curve doesn't exist"); + return CAAM_FAILURE; + } + + MP_TRACE("Generating MP Private key"); + retstatus = do_mppriv_gen(passphrase, strlen(passphrase)); + + if (retstatus != CAAM_NO_ERROR) { + MP_TRACE("do_mppriv_gen failed!"); + return retstatus; + } + } else { + /* MP Curve is already programmed. Set the right key size */ + mp_privdata.curve = curve; + + switch (curve) { + case PDB_MP_CSEL_P256: + mp_privdata.sec_size = 32; + break; + case PDB_MP_CSEL_P521: + mp_privdata.sec_size = 66; + break; + default: + MP_TRACE("This curve is not supported"); + return CAAM_FAILURE; + } + } + + /* Fill the MPMR content then lock it */ + caam_hal_ctrl_fill_mpmr(ctrl_addr, &msg_mpmr); + + mp_privdata.mp_status = CAAM_NO_ERROR; + + return CAAM_NO_ERROR; +} + +enum caam_status caam_mp_resume(uint32_t pm_hint) +{ + if (pm_hint == PM_HINT_CONTEXT_STATE) + return caam_mp_init(mp_privdata.ctrl_addr); + + return CAAM_NO_ERROR; +} diff --git a/optee_os/core/drivers/crypto/caam/mp/sub.mk b/optee_os/core/drivers/crypto/caam/mp/sub.mk new file mode 100644 index 0000000..1aa9326 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/mp/sub.mk @@ -0,0 +1,3 @@ +incdirs-y += ../include + +srcs-y += caam_mp.c diff --git a/optee_os/core/drivers/crypto/caam/sub.mk b/optee_os/core/drivers/crypto/caam/sub.mk new file mode 100644 index 0000000..8124501 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/sub.mk @@ -0,0 +1,16 @@ +incdirs-y += include + +subdirs-y += hal +subdirs-y += utils + +srcs-y += caam_pwr.c +srcs-y += caam_ctrl.c +srcs-y += caam_jr.c +srcs-y += caam_rng.c +srcs-y += caam_desc.c +srcs-$(CFG_NXP_CAAM_SM_DRV) += caam_sm.c +subdirs-$(call cfg-one-enabled, CFG_NXP_CAAM_HASH_DRV CFG_NXP_CAAM_HMAC_DRV) += hash +subdirs-$(call cfg-one-enabled, CFG_NXP_CAAM_CIPHER_DRV CFG_NXP_CAAM_CMAC_DRV) += cipher +subdirs-y += acipher +subdirs-y += blob +subdirs-$(CFG_NXP_CAAM_MP_DRV) += mp diff --git a/optee_os/core/drivers/crypto/caam/utils/sub.mk b/optee_os/core/drivers/crypto/caam/utils/sub.mk new file mode 100644 index 0000000..f69195b --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/sub.mk @@ -0,0 +1,9 @@ +incdirs-y += ../include + +srcs-y += utils_mem.c +srcs-y += utils_delay.c +srcs-y += utils_sgt.c +srcs-$(CFG_NXP_CAAM_SGT_V1) += utils_sgt_v1.c +srcs-$(CFG_NXP_CAAM_SGT_V2) += utils_sgt_v2.c +srcs-y += utils_status.c +srcs-y += utils_dmaobj.c diff --git a/optee_os/core/drivers/crypto/caam/utils/utils_delay.c b/optee_os/core/drivers/crypto/caam/utils/utils_delay.c new file mode 100644 index 0000000..84be5c8 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/utils_delay.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018 NXP + * + * Brief Delay management utilities. + * Primitive to delay a delay. + */ +#include +#include + +void caam_udelay(uint32_t time) +{ + uint32_t counter = time * 500; + + /* Implementation of a Software loop assuming CPU clock of 500MHz */ + while (counter--) { + isb(); + dsb(); + }; +} diff --git a/optee_os/core/drivers/crypto/caam/utils/utils_dmaobj.c b/optee_os/core/drivers/crypto/caam/utils/utils_dmaobj.c new file mode 100644 index 0000000..424757c --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/utils_dmaobj.c @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2021, 2023 NXP + * + * CAAM DMA data object utilities. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IS_DMA_OVERFLOW(addr) ((addr) > UINT32_MAX) +#define MAX_BUFFER_ALLOC_SIZE ((size_t)(8 * 1024)) + +/* + * Local defines used to identify Object type as: + * - input or output data + * - SGT object created because buffer is not physical contiguous + * - derived object (not buffer reallocation) + * - allocated origin buffer + */ +#define DMAOBJ_INPUT BIT(0) +#define DMAOBJ_OUTPUT BIT(1) +#define DMAOBJ_ALLOC_ORIG BIT(2) +#define DMAOBJ_DONT_COPY BIT(3) + +/* + * DMA Buffer + * + * @require DMA Buffer size require + * @allocated Size of the buffer allocated + * @remind Size available in the buffer + * @buf CAAM Buffer + */ +struct caamdmabuf { + size_t require; + size_t allocated; + size_t remind; + struct caambuf buf; +}; + +/* + * DMA Object buffer entry + * + * @newbuf True if list entry is a new DMA Buffer + * @nodma_access Buffer is not accessible from CAAM DMA + * @nocopy Buffer doesn't have to be copied back to the origin + * @origbuf Original buffer reference + * @link Pointer to next entry + */ +struct dmaentry { + bool newbuf; + bool nodma_access; + bool nocopy; + + struct caambuf origbuf; + + TAILQ_ENTRY(dmaentry) link; +}; + +/* + * SGT/Buffer Data currently handled + * + * @orig Original buffer reference + * @dma DMA Buffer (new or original) + * @length Buffer length + */ +struct sgtdata { + uint8_t *orig; + uint8_t *dma; + size_t length; +}; + +/* + * CAAM DMA private Object data + * @type Type of DMA Object + * @nb_sgtbuf Number of SGT/Buffer entries allocated + * @dmabuf DMA Buffer allocated + * @sgtdata Reference to SGT/Buffer list in used + * @list List of the DMA Object buffer entry + */ +struct priv_dmaobj { + unsigned int type; + unsigned int nb_sgtbuf; + + struct caamdmabuf dmabuf; + struct sgtdata *sgtdata; + + TAILQ_HEAD(dmalist, dmaentry) list; +}; + +/* + * Memory allocation and free spinlock to ensure that in case + * of big buffer reallocation, memory used is freed + */ +static unsigned int memlock; + +/* + * Try to allocate a DMA Buffer of type input or output data of @size bytes. + * If allocation success, set the DMA Buffer settings, else + * return in error. + * + * @priv CAAM DMA object private data + * @size Size of the DMA Buffer to allocate + */ +static TEE_Result try_allocate_dmabuf(struct priv_dmaobj *priv, size_t size) +{ + enum caam_status retstatus = CAAM_FAILURE; + + if (priv->dmabuf.allocated) { + caam_free_buf(&priv->dmabuf.buf); + priv->dmabuf.allocated = 0; + } + + if (priv->type & DMAOBJ_INPUT) + retstatus = caam_alloc_buf(&priv->dmabuf.buf, size); + else + retstatus = caam_alloc_align_buf(&priv->dmabuf.buf, size); + + DMAOBJ_TRACE("Alloc %s DMA buffer (%zu) ret 0x%" PRIx32, + (priv->type & DMAOBJ_INPUT) ? "Input" : "Output", size, + retstatus); + + if (retstatus == CAAM_NO_ERROR) { + DMAOBJ_TRACE("DMA buffer Allocation Success"); + /* Set the Object's DMA Buffer settings */ + priv->dmabuf.allocated = size; + priv->dmabuf.remind = size; + priv->dmabuf.buf.length = 0; + return TEE_SUCCESS; + } + + DMAOBJ_TRACE("DMA buffer Allocation Failure"); + return TEE_ERROR_OUT_OF_MEMORY; +} + +/* + * Allocate and initialize the CAAM DMA object's private data. + * + * @obj CAAM DMA Object + * @type Type of the CAAM DMA Object (i.e. Input or Output) + */ +static TEE_Result allocate_private(struct caamdmaobj *obj, unsigned int type) +{ + struct priv_dmaobj *priv = NULL; + + priv = caam_calloc(sizeof(*priv)); + if (!priv) + return TEE_ERROR_OUT_OF_MEMORY; + + obj->priv = priv; + + /* Set the object type as input */ + priv->type = type; + + TAILQ_INIT(&priv->list); + + return TEE_SUCCESS; +} + +/* + * Fill the @sgtdata object to record the current input/output data + * handled in the DMA SGT/Buffer object. + * Increment the SGT/Buffer length according + * + * @obj CAAM DMA object + * @sgtdata [out] SGT Data handled + * @entry DMA Object buffer entry + * @dma DMA SGT/Buffer object + * @offset Start offset of the DMA Object buffer + */ +static void add_sgtdata_entry(struct caamdmaobj *obj, struct sgtdata *sgtdata, + struct dmaentry *entry, struct caambuf *dma, + size_t offset) +{ + if (entry->nocopy) { + sgtdata->orig = 0; + sgtdata->length = 0; + sgtdata->dma = 0; + } else { + sgtdata->orig = entry->origbuf.data + offset; + sgtdata->length = dma->length; + sgtdata->dma = dma->data; + } + + obj->sgtbuf.length += dma->length; +} + +/* + * Add a new DMA Buffer entry as first element of the list. + * Return NULL if error, else the new entry in the list + * + * @priv DMA Object private data + * @orig Original buffer reference + */ +static struct dmaentry *dmalist_add_entry_head(struct priv_dmaobj *priv, + struct caambuf *orig) +{ + struct dmaentry *entry = NULL; + + entry = caam_calloc(sizeof(*entry)); + if (entry) { + /* Save the original buffer reference */ + memcpy(&entry->origbuf, orig, sizeof(entry->origbuf)); + DMAOBJ_TRACE("entry %p - insert head entry of %zu bytes", entry, + orig->length); + TAILQ_INSERT_HEAD(&priv->list, entry, link); + } + + return entry; +} + +/* + * Add a new DMA Buffer entry in the list. + * Return NULL if error, else the new entry in the list + * + * @priv DMA Object private data + * @orig Original buffer reference + */ +static struct dmaentry *dmalist_add_entry(struct priv_dmaobj *priv, + struct caambuf *orig) +{ + struct dmaentry *entry = NULL; + + entry = caam_calloc(sizeof(*entry)); + if (entry) { + /* Save the original buffer reference */ + memcpy(&entry->origbuf, orig, sizeof(entry->origbuf)); + DMAOBJ_TRACE("entry %p - insert entry of %zu bytes", entry, + orig->length); + if (TAILQ_EMPTY(&priv->list)) + TAILQ_INSERT_HEAD(&priv->list, entry, link); + else + TAILQ_INSERT_TAIL(&priv->list, entry, link); + } + + return entry; +} + +/* + * Insert and allocate a DMA entry in the list before the given DMA entry. + * Return the allocated DMA entry. + * + * @priv DMA Object private data + * @before DMA entry after the new DMA entry + * @new CAAM buffer of the new DMA entry + */ +static struct dmaentry *dmalist_insert_before_entry(struct priv_dmaobj *priv, + struct dmaentry *before, + struct caambuf *new) +{ + struct dmaentry *entry = NULL; + + entry = caam_calloc(sizeof(*entry)); + if (entry) { + /* Save the original buffer reference */ + memcpy(&entry->origbuf, new, sizeof(entry->origbuf)); + DMAOBJ_TRACE("entry %p - insert entry of %zu bytes", entry, + new->length); + if (TAILQ_FIRST(&priv->list) == before) + TAILQ_INSERT_HEAD(&priv->list, entry, link); + else + TAILQ_INSERT_BEFORE(before, entry, link); + } + + return entry; +} + +/* + * Insert and allocate a DMA entry in the list after the given DMA entry. + * Return the allocated DMA entry. + * + * @priv DMA Object private data + * @after DMA entry before the new DMA entry + * @new CAAM buffer of the new DMA entry + */ +static struct dmaentry *dmalist_insert_after_entry(struct priv_dmaobj *priv, + struct dmaentry *after, + struct caambuf *new) +{ + struct dmaentry *entry = NULL; + + entry = caam_calloc(sizeof(*entry)); + if (entry) { + /* Save the original buffer reference */ + memcpy(&entry->origbuf, new, sizeof(entry->origbuf)); + DMAOBJ_TRACE("entry %p - insert entry of %zu bytes", entry, + new->length); + TAILQ_INSERT_AFTER(&priv->list, after, entry, link); + } + + return entry; +} + +/* + * Apply the cache operation @op to the DMA Object (SGT or buffer) + * + * @op Cache operation + * @obj CAAM DMA object + */ +static inline void dmaobj_cache_operation(enum utee_cache_operation op, + struct caamdmaobj *obj) +{ + if (!obj->sgtbuf.length) + return; + + if (obj->sgtbuf.sgt_type) + caam_sgt_cache_op(op, &obj->sgtbuf, obj->sgtbuf.length); + else if (!obj->sgtbuf.buf->nocache) + cache_operation(op, obj->sgtbuf.buf->data, obj->sgtbuf.length); +} + +/* + * Set the required allocation size for the DMA buffer. + * + * @priv DMA Object private data + * @length Required buffer size + */ +static inline void add_dma_require(struct priv_dmaobj *priv, size_t length) +{ + size_t tmp = 0; + + if (ADD_OVERFLOW(priv->dmabuf.require, length, &tmp)) + priv->dmabuf.require = SIZE_MAX; + else + priv->dmabuf.require = tmp; +} + +/* + * Check if the buffer start/end addresses are aligned on the cache line. + * If not flags as start and/or end addresses not aligned, expect if the + * maximum length @maxlen to use is inside a cache line size. In this case, + * flags to allocate a new buffer. + * + * @priv DMA Object private data + * @maxlen Maximum length to use + */ +static TEE_Result check_buffer_alignment(struct priv_dmaobj *priv, + size_t maxlen) +{ + unsigned int cacheline_size = 0; + struct dmaentry *entry = NULL; + struct dmaentry *new_entry = NULL; + struct caambuf newbuf = {}; + vaddr_t va_start = 0; + vaddr_t va_end = 0; + vaddr_t va_end_align = 0; + vaddr_t va_start_align = 0; + size_t remlen = 0; + size_t acclen = 0; + + cacheline_size = dcache_get_line_size(); + + TAILQ_FOREACH(entry, &priv->list, link) { + DMAOBJ_TRACE("Entry %p: start %p len %zu (%zu >= %zu)", entry, + entry->origbuf.data, entry->origbuf.length, acclen, + maxlen); + + /* No need to continue if we convert the needed length */ + if (acclen >= maxlen) + return TEE_SUCCESS; + + acclen += entry->origbuf.length; + + if (entry->nodma_access || entry->newbuf) + continue; + + if (entry->origbuf.length < cacheline_size) { + /* + * Length of the entry is not aligned on cache size + * Require a full aligned buffer + */ + DMAOBJ_TRACE("Length %zu vs cache line %u", + entry->origbuf.length, cacheline_size); + + entry->newbuf = true; + add_dma_require(priv, entry->origbuf.length); + continue; + } + + va_start = (vaddr_t)entry->origbuf.data; + va_start_align = ROUNDUP(va_start, cacheline_size); + + if (va_start_align != va_start) { + DMAOBJ_TRACE("Start 0x%" PRIxVA " vs align 0x%" PRIxVA, + va_start, va_start_align); + + remlen = entry->origbuf.length - + (va_start_align - va_start); + if (remlen <= cacheline_size) { + /* + * Start address is not aligned and the + * remaining length if after re-alignment + * is not cache size aligned. + * Require a full aligned buffer + */ + DMAOBJ_TRACE("Rem length %zu vs cache line %u", + remlen, cacheline_size); + entry->newbuf = true; + add_dma_require(priv, entry->origbuf.length); + continue; + } + + /* + * Insert a new entry to make buffer on a cache line. + */ + newbuf.data = entry->origbuf.data; + newbuf.length = va_start_align - va_start; + newbuf.paddr = entry->origbuf.paddr; + newbuf.nocache = entry->origbuf.nocache; + + add_dma_require(priv, newbuf.length); + new_entry = dmalist_insert_before_entry(priv, entry, + &newbuf); + if (!new_entry) + return TEE_ERROR_OUT_OF_MEMORY; + + new_entry->newbuf = true; + + /* + * Update current entry with align address and new + * length. + */ + entry->origbuf.data = (uint8_t *)va_start_align; + entry->origbuf.length -= newbuf.length; + entry->origbuf.paddr += newbuf.length; + + /* + * Set current entry to new entry to continue + * the FOREACH loop from this new_entry and then + * verify the rest of the entry modified. + */ + entry = new_entry; + acclen -= entry->origbuf.length; + continue; + } + + va_end = (vaddr_t)entry->origbuf.data + entry->origbuf.length; + va_end_align = ROUNDUP(va_end, cacheline_size); + + if (va_end != va_end_align) { + DMAOBJ_TRACE("End 0x%" PRIxVA " vs align 0x%" PRIxVA, + va_end, va_end_align); + + va_end_align = ROUNDDOWN(va_end, cacheline_size); + remlen = entry->origbuf.length - va_end_align; + + if (remlen <= cacheline_size) { + /* + * End address is not aligned and the remaining + * length if after re-alignment is not cache + * size aligned. + * Require a full aligned buffer + */ + DMAOBJ_TRACE("Rem length %zu vs cache line %u", + remlen, cacheline_size); + entry->newbuf = true; + add_dma_require(priv, entry->origbuf.length); + continue; + } + + /* + * Insert a new entry to make buffer on a cache line. + */ + newbuf.data = (uint8_t *)va_end_align; + newbuf.length = va_end - va_end_align; + newbuf.paddr = entry->origbuf.paddr + newbuf.length; + newbuf.nocache = entry->origbuf.nocache; + + add_dma_require(priv, newbuf.length); + + new_entry = dmalist_insert_after_entry(priv, entry, + &newbuf); + if (!new_entry) + return TEE_ERROR_OUT_OF_MEMORY; + + new_entry->newbuf = true; + + /* Update current entry with new length */ + entry->origbuf.length -= newbuf.length; + + /* + * Set current entry to new entry to continue + * the FOREACH loop from this new_entry and then + * verify the rest of the entry modified. + */ + entry = new_entry; + acclen -= newbuf.length; + continue; + } + } + + return TEE_SUCCESS; +} + +/* + * Go through all the @orig space to extract all physical area used to + * map the buffer. + * If one of the physical area is not accessible by the CAAM DMA, flag it + * to be reallocated with DMA accessible buffer. + * If the DMA Object is an output buffer, check and flag the start/end + * address of the buffer to be aligned on a cache line. + * + * @obj CAAM DMA object + * @orig Original Data + * @maxlen Maximum length to use + */ +static TEE_Result check_buffer_boundary(struct caamdmaobj *obj, + struct caambuf *orig, size_t maxlen) +{ + TEE_Result ret = TEE_ERROR_OUT_OF_MEMORY; + struct priv_dmaobj *priv = obj->priv; + struct dmaentry *entry = NULL; + struct caambuf *pabufs = NULL; + int nb_pa_area = -1; + int idx = 0; + paddr_t last_pa = 0; + size_t remlen = maxlen; + size_t tmp = 0; + + /* + * Get the number of physical areas used by the + * DMA Buffer + */ + nb_pa_area = caam_mem_get_pa_area(orig, &pabufs); + DMAOBJ_TRACE("Number of pa areas = %d (for max length %zu bytes)", + nb_pa_area, remlen); + if (nb_pa_area == -1) + goto out; + + for (idx = 0; idx < nb_pa_area && remlen; idx++) { + DMAOBJ_TRACE("Remaining length = %zu", remlen); + if (ADD_OVERFLOW(pabufs[idx].paddr, pabufs[idx].length, + &last_pa)) + goto out; + + DMAOBJ_TRACE("PA 0x%" PRIxPA " = 0x%" PRIxPA " + %zu", last_pa, + pabufs[idx].paddr, pabufs[idx].length); + + entry = dmalist_add_entry(priv, &pabufs[idx]); + if (!entry) + goto out; + + if (IS_DMA_OVERFLOW(last_pa)) { + entry->nodma_access = true; + if (ADD_OVERFLOW(priv->dmabuf.require, + pabufs[idx].length, &tmp)) + priv->dmabuf.require = SIZE_MAX; + else + priv->dmabuf.require = tmp; + } + + if (remlen > pabufs[idx].length) + remlen -= pabufs[idx].length; + else + remlen = 0; + } + + /* + * Check the buffer alignment if the buffer is cacheable and + * an output buffer. + */ + if (priv->type & DMAOBJ_OUTPUT && !orig->nocache) { + ret = check_buffer_alignment(priv, maxlen); + if (ret) + goto out; + } + + orig->length = maxlen; + + ret = TEE_SUCCESS; +out: + caam_free(pabufs); + return ret; +} + +/* + * Re-map a DMA entry into a CAAM DMA accessible buffer. + * Create the SGT/Buffer entry to be used in the CAAM Descriptor + * Record this entry in the SGT/Buffer Data to get information on current + * working data. + * + * @obj CAAM DMA object + * @entry DMA entry to re-map + * @index Index in the SGT/Buffer table + * @off Start offset of the DMA entry data + */ +static enum caam_status entry_sgtbuf_dmabuf(struct caamdmaobj *obj, + struct dmaentry *entry, + unsigned int index, size_t off) +{ + struct priv_dmaobj *priv = obj->priv; + struct caambuf *sgtbuf = &obj->sgtbuf.buf[index]; + struct caamdmabuf *dmabuf = &priv->dmabuf; + + if (!priv->dmabuf.allocated) + return CAAM_OUT_MEMORY; + + sgtbuf->data = dmabuf->buf.data + dmabuf->buf.length; + sgtbuf->length = MIN(dmabuf->remind, entry->origbuf.length - off); + sgtbuf->paddr = dmabuf->buf.paddr + dmabuf->buf.length; + sgtbuf->nocache = dmabuf->buf.nocache; + dmabuf->remind -= sgtbuf->length; + dmabuf->buf.length += sgtbuf->length; + + if (priv->type & DMAOBJ_INPUT) + memcpy(sgtbuf->data, &entry->origbuf.data[off], sgtbuf->length); + else + entry->newbuf = true; + + add_sgtdata_entry(obj, &priv->sgtdata[index], entry, sgtbuf, off); + + return CAAM_NO_ERROR; +} + +/* + * Create the SGT/Buffer entry mapping the DMA @entry. + * Record these entry in the SGT/buffer Data to get information on current + * working data. + * + * @obj CAAM DMA object + * @entry DMA entry to re-map + * @index Index in the SGT/Buffer table + * @off Start offset of the DMA entry data + */ +static enum caam_status entry_sgtbuf(struct caamdmaobj *obj, + struct dmaentry *entry, unsigned int index, + size_t off) +{ + struct priv_dmaobj *priv = obj->priv; + struct caambuf *sgtbuf = &obj->sgtbuf.buf[index]; + struct sgtdata *sgtdata = &priv->sgtdata[index]; + + memcpy(sgtbuf, &entry->origbuf, sizeof(*sgtbuf)); + sgtbuf->data += off; + sgtbuf->paddr += off; + sgtbuf->length -= off; + + DMAOBJ_TRACE("DMA buffer %p - %zu", sgtbuf->data, sgtbuf->length); + add_sgtdata_entry(obj, sgtdata, entry, sgtbuf, off); + + return CAAM_NO_ERROR; +} + +TEE_Result caam_dmaobj_init_input(struct caamdmaobj *obj, const void *data, + size_t length) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + DMAOBJ_TRACE("Input object with data @%p of %zu bytes", data, length); + + if (!data || !length || !obj) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + obj->orig.paddr = virt_to_phys((void *)data); + if (!obj->orig.paddr) { + DMAOBJ_TRACE("Object virtual address error"); + ret = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + obj->orig.data = (void *)data; + obj->orig.length = length; + if (!caam_mem_is_cached_buf((void *)data, length)) + obj->orig.nocache = 1; + + ret = allocate_private(obj, DMAOBJ_INPUT); + if (ret) + goto err; + + ret = check_buffer_boundary(obj, &obj->orig, obj->orig.length); + + goto out; +err: + caam_dmaobj_free(obj); +out: + DMAOBJ_TRACE("Object returns 0x%" PRIx32, ret); + return ret; +} + +TEE_Result caam_dmaobj_input_sgtbuf(struct caamdmaobj *obj, const void *data, + size_t length) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t size_done = length; + + ret = caam_dmaobj_init_input(obj, data, length); + if (ret) + goto err; + + ret = caam_dmaobj_prepare(obj, NULL, length); + if (ret) + goto err; + + ret = caam_dmaobj_sgtbuf_build(obj, &size_done, 0, length); + if (ret) + goto err; + + if (size_done != length) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + return TEE_SUCCESS; +err: + caam_dmaobj_free(obj); + return ret; +} + +TEE_Result caam_dmaobj_init_output(struct caamdmaobj *obj, void *data, + size_t length, size_t min_length) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct dmaentry *entry = NULL; + struct caambuf newbuf = {}; + + DMAOBJ_TRACE("Output object with data @%p of %zu bytes (%zu)", data, + length, min_length); + + if (!obj) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + ret = allocate_private(obj, DMAOBJ_OUTPUT); + if (ret) + goto err; + + if (data) { + obj->orig.paddr = virt_to_phys((void *)data); + if (!obj->orig.paddr) { + DMAOBJ_TRACE("Object virtual address error"); + ret = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + obj->orig.data = (void *)data; + obj->orig.length = length; + if (!caam_mem_is_cached_buf((void *)data, length)) + obj->orig.nocache = 1; + + ret = check_buffer_boundary(obj, &obj->orig, + MIN(min_length, obj->orig.length)); + if (ret) + goto err; + } + + if (length < min_length || !data) { + DMAOBJ_TRACE("Output buffer too short need %zu bytes (+%zu)", + min_length, min_length - length); + newbuf.length = min_length - length; + + entry = dmalist_add_entry(obj->priv, &newbuf); + if (!entry) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + /* Add the additional size in the DMA buffer length */ + add_dma_require(obj->priv, newbuf.length); + + entry->nocopy = true; + entry->newbuf = true; + } + + ret = TEE_SUCCESS; + goto out; + +err: + caam_dmaobj_free(obj); +out: + DMAOBJ_TRACE("Object returns 0x%" PRIx32, ret); + return ret; +} + +TEE_Result caam_dmaobj_output_sgtbuf(struct caamdmaobj *obj, void *data, + size_t length, size_t min_length) +{ + enum caam_status retstatus = CAAM_FAILURE; + TEE_Result ret = TEE_ERROR_GENERIC; + struct priv_dmaobj *priv = NULL; + size_t size = 0; + struct caambuf buf = {}; + + if (!data && !length && min_length) { + /* + * We are sure that the minimum size of the allocated + * buffer is a cache line, hence we know that + * start/end address are cache aligned. + * If the @min_length is less than a cache line size, we + * can initializing the output buffer with the cache line size + * to prevent end buffer misalignement so reallocate a not used + * buffer. + */ + size = MAX(min_length, dcache_get_line_size()); + + /* Allocate a new cache aligned buffer */ + retstatus = caam_alloc_align_buf(&buf, size); + DMAOBJ_TRACE("New output buffer of %zu bytes ret 0x%" PRIx32, + min_length, retstatus); + if (retstatus != CAAM_NO_ERROR) + return caam_status_to_tee_result(retstatus); + + ret = caam_dmaobj_init_output(obj, buf.data, buf.length, size); + if (ret) + return ret; + + /* Set the correct origin buffer length asked */ + obj->orig.length = min_length; + + /* Flag origin buffer as new allocation to free it */ + priv = obj->priv; + priv->type |= DMAOBJ_ALLOC_ORIG; + } else { + ret = caam_dmaobj_init_output(obj, data, length, min_length); + if (ret) + return ret; + } + + ret = caam_dmaobj_prepare(NULL, obj, min_length); + if (ret) + return ret; + + size = min_length; + ret = caam_dmaobj_sgtbuf_build(obj, &size, 0, min_length); + if (ret) + return ret; + + if (size != min_length) + return TEE_ERROR_OUT_OF_MEMORY; + + return TEE_SUCCESS; +} + +void caam_dmaobj_cache_push(struct caamdmaobj *obj) +{ + struct priv_dmaobj *priv = NULL; + enum utee_cache_operation op = TEE_CACHECLEAN; + + if (!obj || !obj->priv) + return; + + priv = obj->priv; + if (priv->type & DMAOBJ_OUTPUT) + op = TEE_CACHEFLUSH; + + dmaobj_cache_operation(op, obj); +} + +size_t caam_dmaobj_copy_to_orig(struct caamdmaobj *obj) +{ + struct priv_dmaobj *priv = NULL; + unsigned int idx = 0; + size_t length = 0; + size_t dst_rlen = 0; + size_t copy_size = 0; + + if (!obj || !obj->orig.data || !obj->priv) + return 0; + + dmaobj_cache_operation(TEE_CACHEINVALIDATE, obj); + + priv = obj->priv; + + /* + * The maximum data size to copy cannot exceed the output buffer size + * (obj->orig.length) and cannot exceed the data processed by the + * CAAM (obj->sgtbuf.length). + */ + dst_rlen = MIN(obj->orig.length, obj->sgtbuf.length); + + DMAOBJ_TRACE("Copy (len=%zu)", dst_rlen); + + for (idx = 0; idx < obj->sgtbuf.number; idx++) { + struct sgtdata *sgtdata = &priv->sgtdata[idx]; + + copy_size = MIN(dst_rlen, sgtdata->length); + if (sgtdata->orig != sgtdata->dma && sgtdata->orig) { + copy_size = MIN(dst_rlen, sgtdata->length); + memcpy(sgtdata->orig, sgtdata->dma, copy_size); + } + + length += copy_size; + dst_rlen -= copy_size; + } + + return length; +} + +size_t caam_dmaobj_copy_ltrim_to_orig(struct caamdmaobj *obj) +{ + struct priv_dmaobj *priv = NULL; + uint8_t *dst = NULL; + size_t off = 0; + size_t offset = 0; + size_t dst_rlen = 0; + size_t copy_size = 0; + unsigned int idx = 0; + size_t length = 0; + + if (!obj || !obj->orig.data || !obj->priv) + return 0; + + dmaobj_cache_operation(TEE_CACHEINVALIDATE, obj); + + priv = obj->priv; + + /* Parse the SGT data list to discard leading zeros */ + for (idx = 0; idx < obj->sgtbuf.number; idx++) { + struct sgtdata *sgtdata = &priv->sgtdata[idx]; + + if (!sgtdata->orig) + continue; + + for (offset = 0; offset < sgtdata->length; off++, offset++) { + if (sgtdata->dma[offset]) + goto do_copy; + } + } + +do_copy: + if (off < obj->orig.length) + dst_rlen = obj->orig.length - off; + + dst = obj->orig.data; + + DMAOBJ_TRACE("Copy/Move Offset=%zu (len=%zu) TYPE=%d", off, dst_rlen, + obj->sgtbuf.sgt_type); + + if (!dst_rlen) { + dst[0] = 0; + return 1; + } + + /* + * After discarding leading zeros in the SGT data list, start the copy + * operation on the remaining elements of the data list. + * List index must not be re-initialized before entering this loop. + */ + for (; idx < obj->sgtbuf.number; idx++) { + struct sgtdata *sgtdata = &priv->sgtdata[idx]; + + if (!sgtdata->orig) + continue; + + if (offset) { + copy_size = MIN(dst_rlen, sgtdata->length - offset); + memmove(dst, &sgtdata->dma[offset], copy_size); + offset = 0; + } else { + copy_size = MIN(dst_rlen, sgtdata->length); + if (dst != sgtdata->dma) + memmove(dst, sgtdata->dma, copy_size); + } + + dst += copy_size; + dst_rlen -= copy_size; + length += copy_size; + } + + return length; +} + +void caam_dmaobj_free(struct caamdmaobj *obj) +{ + struct priv_dmaobj *priv = NULL; + struct dmaentry *entry = NULL; + struct dmaentry *next = NULL; + uint32_t exceptions = 0; + + if (!obj) + return; + + exceptions = cpu_spin_lock_xsave(&memlock); + priv = obj->priv; + if (!priv) + goto out; + + DMAOBJ_TRACE("Free %s object with data @%p of %zu bytes", + priv->type & DMAOBJ_INPUT ? "Input" : "Output", + obj->orig.data, obj->orig.length); + + TAILQ_FOREACH_SAFE(entry, &priv->list, link, next) { + DMAOBJ_TRACE("Is type 0x%" PRIx8 " newbuf %s", priv->type, + entry->newbuf ? "true" : "false"); + + DMAOBJ_TRACE("Free entry %p", entry); + caam_free(entry); + } + + if (priv->nb_sgtbuf) { + DMAOBJ_TRACE("Free #%d SGT data %p", priv->nb_sgtbuf, + priv->sgtdata); + caam_free(priv->sgtdata); + + obj->sgtbuf.number = priv->nb_sgtbuf; + obj->sgtbuf.sgt_type = (priv->nb_sgtbuf > 1) ? true : false; + } + + if (priv->dmabuf.allocated) { + DMAOBJ_TRACE("Free CAAM DMA buffer"); + caam_free_buf(&priv->dmabuf.buf); + } + + if (priv->type & DMAOBJ_ALLOC_ORIG) { + DMAOBJ_TRACE("Free Allocated origin"); + caam_free_buf(&obj->orig); + } + + DMAOBJ_TRACE("Free private object %p", priv); + caam_free(priv); + +out: + if (obj->sgtbuf.number) { + DMAOBJ_TRACE("Free #%d SGT/Buffer %p", obj->sgtbuf.number, + &obj->sgtbuf); + caam_sgtbuf_free(&obj->sgtbuf); + } + + memset(obj, 0, sizeof(*obj)); + + cpu_spin_unlock_xrestore(&memlock, exceptions); +} + +TEE_Result caam_dmaobj_add_first_block(struct caamdmaobj *obj, + struct caamblock *block) +{ + struct priv_dmaobj *priv = NULL; + struct caambuf newbuf = {}; + struct dmaentry *entry = NULL; + + if (!obj || !obj->priv || !block) + return TEE_ERROR_BAD_PARAMETERS; + + priv = obj->priv; + + /* Save the block buffer reference and insert it at the head list */ + newbuf.data = block->buf.data; + newbuf.length = block->filled; + newbuf.paddr = block->buf.paddr; + newbuf.nocache = block->buf.nocache; + + entry = dmalist_add_entry_head(priv, &newbuf); + + if (!entry) + return TEE_ERROR_OUT_OF_MEMORY; + + /* + * Block buffer added in the output DMA buffer doesn't have to + * be part of the output copy to origin buffer. + */ + if (priv->type & DMAOBJ_OUTPUT) + entry->nocopy = true; + + return TEE_SUCCESS; +} + +TEE_Result caam_dmaobj_derive_sgtbuf(struct caamdmaobj *obj, + const struct caamdmaobj *from, + size_t offset, size_t length) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct priv_dmaobj *priv = NULL; + + DMAOBJ_TRACE("Derive object %p - offset %zu - length %zu bytes", from, + offset, length); + + if (!obj || !from || !length || !from->priv) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (!from->orig.data || !from->orig.length) { + DMAOBJ_TRACE("No data/length to derive from"); + ret = TEE_ERROR_NO_DATA; + goto out; + } + + priv = from->priv; + if (!priv->nb_sgtbuf) { + DMAOBJ_TRACE("From SGT/Buffer not prepared"); + ret = TEE_ERROR_NO_DATA; + goto out; + } + + retstatus = caam_sgt_derive(&obj->sgtbuf, &from->sgtbuf, offset, + length); + + ret = caam_status_to_tee_result(retstatus); + +out: + DMAOBJ_TRACE("Object returns 0x%" PRIx32, ret); + return ret; +} + +/* + * Get the maximum allocation size for the given CAAM DMA object. + * Return the maximum allocation size. + * + * @obj CAAM DMA object + */ +static size_t get_dma_max_alloc_size(struct caamdmaobj *obj) +{ + size_t alloc_size = 0; + struct priv_dmaobj *priv = NULL; + + if (!obj) + return 0; + + priv = obj->priv; + + DMAOBJ_TRACE("DMA buffer size require %zu", priv->dmabuf.require); + alloc_size = MIN(priv->dmabuf.require, MAX_BUFFER_ALLOC_SIZE); + if (alloc_size > 1024) + alloc_size = ROUNDDOWN(alloc_size, 1024); + + return alloc_size; +} + +/* + * Allocate the CAAM DMA buffer. + * First, try to allocate the with the maximum size. If it fails, try to + * allocate with the same size divided by two. Try to allocate until + * minimum size is reached. If the allocation cannot be done with the + * minimum size, return TEE_ERROR_OUT_OF_MEMORY, TEE_SUCCESS otherwise. + * + * @obj CAAM DMA object + * @min_size minimum size allocation + * @size[out] successful allocation size + */ +static TEE_Result try_allocate_dmabuf_max_size(struct caamdmaobj *obj, + size_t min_size, + size_t *size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t alloc_size = 0; + struct priv_dmaobj *priv = NULL; + bool try_alloc = false; + uint32_t exceptions = 0; + + alloc_size = get_dma_max_alloc_size(obj); + if (alloc_size) { + try_alloc = true; + } else { + ret = TEE_SUCCESS; + goto out; + } + + priv = obj->priv; + + exceptions = cpu_spin_lock_xsave(&memlock); + + while (try_alloc) { + ret = try_allocate_dmabuf(priv, alloc_size); + if (!ret) { + try_alloc = false; + } else { + if (alloc_size > min_size) + alloc_size = MAX(min_size, alloc_size / 2); + else + try_alloc = false; + } + } + + cpu_spin_unlock_xrestore(&memlock, exceptions); + +out: + *size = alloc_size; + + return ret; +} + +TEE_Result caam_dmaobj_prepare(struct caamdmaobj *input, + struct caamdmaobj *output, size_t min_size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t alloc_input = 0; + size_t alloc_output = 0; + + if (!input && !output) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((input && !input->priv) || (output && !output->priv)) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + DMAOBJ_TRACE("input=%p - output=%p - min=%zu", input, output, min_size); + + ret = try_allocate_dmabuf_max_size(input, min_size, &alloc_input); + if (ret) + goto out; + + ret = try_allocate_dmabuf_max_size(output, min_size, &alloc_output); + if (ret) + goto out; + +out: + DMAOBJ_TRACE("Allocation (input %zu, output %zu) returns 0x%" PRIx32, + input ? alloc_input : 0, output ? alloc_output : 0, + ret); + + return ret; +} + +TEE_Result caam_dmaobj_sgtbuf_inout_build(struct caamdmaobj *input, + struct caamdmaobj *output, + size_t *length, size_t off, + size_t align) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t len = 0; + + DMAOBJ_TRACE("input=%p/output=%p %zu bytes (offset=%zu, align=%zu)", + input, output, *length, off, align); + + if (!input || !output || !length || !input->priv || !output->priv || + !*length) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * First build the input SGT/Buffer + */ + ret = caam_dmaobj_sgtbuf_build(input, length, off, align); + if (ret) + goto out; + + /* + * Next build the output SGT/Buffer. + * If returned length is not same as input, redo the input + * SGT/Buffer with the same length as the output. + */ + len = *length; + ret = caam_dmaobj_sgtbuf_build(output, &len, off, *length); + if (ret) + goto out; + + if (len != *length) { + DMAOBJ_TRACE("Retry In %zu bytes vs Out %zu bytes", *length, + len); + + /* Redo the input with the output length */ + *length = len; + ret = caam_dmaobj_sgtbuf_build(input, length, off, len); + if (!ret && *length != len) { + DMAOBJ_TRACE("Error In %zu bytes vs Out %zu bytes", + *length, len); + ret = TEE_ERROR_OUT_OF_MEMORY; + } + } + +out: + DMAOBJ_TRACE("Input/Output SGTBUF returns 0x%" PRIx32, ret); + + return ret; +} + +TEE_Result caam_dmaobj_sgtbuf_build(struct caamdmaobj *obj, size_t *length, + size_t off, size_t align) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + enum caam_status retstatus = CAAM_FAILURE; + struct priv_dmaobj *priv = NULL; + struct dmaentry *entry = NULL; + struct dmaentry *start_entry = NULL; + size_t max_length = 0; + size_t acc_length = 0; + size_t offset = off; + unsigned int idx = 0; + unsigned int nb_sgt = 0; + + DMAOBJ_TRACE("obj=%p of %zu bytes (offset=%zu) - align %zu", obj, + *length, off, align); + + if (!obj || !obj->priv || !length || !*length) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + priv = obj->priv; + + max_length = *length; + if (priv->dmabuf.allocated && max_length > priv->dmabuf.allocated && + priv->dmabuf.allocated > align) + max_length = ROUNDDOWN(priv->dmabuf.allocated, align); + + DMAOBJ_TRACE("Prepare SGT/Buffer to do %zu of %zu", max_length, + *length); + + /* Find the first DMA buffer to start with */ + TAILQ_FOREACH(entry, &priv->list, link) { + if (offset < entry->origbuf.length) + break; + + offset -= entry->origbuf.length; + } + + if (!entry) { + DMAOBJ_TRACE("There is no DMA Object available"); + ret = TEE_ERROR_GENERIC; + goto out; + } + + start_entry = entry; + DMAOBJ_TRACE("Start with %p data %p offset %zu", start_entry, + start_entry->origbuf.data, offset); + + acc_length = entry->origbuf.length - offset; + nb_sgt = 1; + + /* Calculate the number of SGT entry */ + for (entry = TAILQ_NEXT(entry, link); entry && acc_length < max_length; + entry = TAILQ_NEXT(entry, link)) { + acc_length += entry->origbuf.length; + nb_sgt++; + } + + DMAOBJ_TRACE("%d of %d SGT/Buffer entries to handle", nb_sgt, + priv->nb_sgtbuf); + if (priv->nb_sgtbuf < nb_sgt) { + if (priv->nb_sgtbuf) { + obj->sgtbuf.number = priv->nb_sgtbuf; + obj->sgtbuf.sgt_type = (priv->nb_sgtbuf > 1); + + caam_sgtbuf_free(&obj->sgtbuf); + caam_free(priv->sgtdata); + priv->nb_sgtbuf = 0; + } + + obj->sgtbuf.number = nb_sgt; + obj->sgtbuf.sgt_type = (nb_sgt > 1) ? true : false; + + /* Allocate a new SGT/Buffer object */ + retstatus = caam_sgtbuf_alloc(&obj->sgtbuf); + DMAOBJ_TRACE("Allocate %d SGT entries ret 0x%" PRIx32, + obj->sgtbuf.number, retstatus); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + + priv->sgtdata = caam_calloc(nb_sgt * sizeof(*priv->sgtdata)); + if (!priv->sgtdata) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + priv->nb_sgtbuf = nb_sgt; + } else { + obj->sgtbuf.number = nb_sgt; + obj->sgtbuf.sgt_type = (nb_sgt > 1) ? true : false; + } + + /* Reset the DMA Buffer index if allocated */ + if (priv->dmabuf.allocated) { + priv->dmabuf.remind = priv->dmabuf.allocated; + priv->dmabuf.buf.length = 0; + } + + obj->sgtbuf.length = 0; + for (entry = start_entry; entry && idx < nb_sgt; + entry = TAILQ_NEXT(entry, link), idx++) { + DMAOBJ_TRACE("entry %p (%d)", entry, idx); + if (entry->nodma_access || entry->newbuf) { + retstatus = entry_sgtbuf_dmabuf(obj, entry, idx, + offset); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + } else { + retstatus = entry_sgtbuf(obj, entry, idx, offset); + if (retstatus != CAAM_NO_ERROR) { + ret = caam_status_to_tee_result(retstatus); + goto out; + } + } + + if (obj->sgtbuf.length >= max_length) { + DMAOBJ_TRACE("Hold-on enough length %zu", max_length); + obj->sgtbuf.length = max_length; + break; + } + offset = 0; + } + + if (obj->sgtbuf.sgt_type) { + /* Build the SGT table based on the physical area list */ + caam_sgt_fill_table(&obj->sgtbuf); + + obj->sgtbuf.paddr = virt_to_phys(obj->sgtbuf.sgt); + } else { + obj->sgtbuf.paddr = obj->sgtbuf.buf->paddr; + } + + *length = obj->sgtbuf.length; + ret = TEE_SUCCESS; +out: + DMAOBJ_TRACE("SGTBUF (%zu) returns 0x%" PRIx32, *length, ret); + return ret; +} diff --git a/optee_os/core/drivers/crypto/caam/utils/utils_mem.c b/optee_os/core/drivers/crypto/caam/utils/utils_mem.c new file mode 100644 index 0000000..e7c23cd --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/utils_mem.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Brief Memory management utilities. + * Primitive to allocate, free memory. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define MEM_TYPE_NORMAL 0 /* Normal allocation */ +#define MEM_TYPE_ZEROED BIT(0) /* Buffer filled with 0's */ +#define MEM_TYPE_ALIGN BIT(1) /* Address and size aligned on a cache line */ + +/* + * Read the first byte at the given @addr to ensure that + * virtual page is mapped before getting its physical address. + * + * @addr: address to read + */ +static inline void touch_page(vaddr_t addr) +{ + io_read8(addr); +} + +/* + * Allocate an area of given size in bytes. Add the memory allocator + * information in the newly allocated area. + * + * @size Size in bytes to allocate + * @type Type of area to allocate (refer to MEM_TYPE_*) + */ +static void *mem_alloc(size_t size, uint8_t type) +{ + void *ptr = NULL; + size_t alloc_size = size; + + MEM_TRACE("alloc %zu bytes of type %" PRIu8, size, type); + + if (type & MEM_TYPE_ALIGN) { + size_t cacheline_size = dcache_get_line_size(); + + if (ROUNDUP_OVERFLOW(alloc_size, CFG_CAAM_SIZE_ALIGN, + &alloc_size)) + return NULL; + + if (ROUNDUP_OVERFLOW(alloc_size, cacheline_size, &alloc_size)) + return NULL; + + ptr = memalign(cacheline_size, alloc_size); + } else { + ptr = malloc(alloc_size); + } + + if (!ptr) { + MEM_TRACE("alloc Error - NULL"); + return NULL; + } + + if (type & MEM_TYPE_ZEROED) + memset(ptr, 0, alloc_size); + + MEM_TRACE("alloc returned %p", ptr); + return ptr; +} + +/* + * Free allocated area + * + * @ptr area to free + */ +static void mem_free(void *ptr) +{ + if (ptr) { + MEM_TRACE("free %p", ptr); + free(ptr); + } +} + +/* + * Allocate internal driver buffer aligned with a cache line. + * + * @buf [out] buffer allocated + * @size size in bytes of the memory to allocate + * @type Type of area to allocate (refer to MEM_TYPE_*) + */ +static enum caam_status mem_alloc_buf(struct caambuf *buf, size_t size, + uint8_t type) +{ + buf->data = mem_alloc(size, type); + + if (!buf->data) + return CAAM_OUT_MEMORY; + + buf->paddr = virt_to_phys(buf->data); + if (!buf->paddr) { + caam_free_buf(buf); + return CAAM_OUT_MEMORY; + } + + buf->length = size; + buf->nocache = 0; + return CAAM_NO_ERROR; +} + +void *caam_alloc(size_t size) +{ + return mem_alloc(size, MEM_TYPE_NORMAL); +} + +void *caam_calloc(size_t size) +{ + return mem_alloc(size, MEM_TYPE_ZEROED); +} + +void *caam_calloc_align(size_t size) +{ + return mem_alloc(size, MEM_TYPE_ZEROED | MEM_TYPE_ALIGN); +} + +void caam_free(void *ptr) +{ + mem_free(ptr); +} + +uint32_t *caam_calloc_desc(uint8_t nbentries) +{ + return mem_alloc(DESC_SZBYTES(nbentries), + MEM_TYPE_ZEROED | MEM_TYPE_ALIGN); +} + +void caam_free_desc(uint32_t **ptr) +{ + mem_free(*ptr); + *ptr = NULL; +} + +enum caam_status caam_alloc_buf(struct caambuf *buf, size_t size) +{ + return mem_alloc_buf(buf, size, MEM_TYPE_NORMAL); +} + +enum caam_status caam_calloc_buf(struct caambuf *buf, size_t size) +{ + return mem_alloc_buf(buf, size, MEM_TYPE_ZEROED); +} + +enum caam_status caam_calloc_align_buf(struct caambuf *buf, size_t size) +{ + return mem_alloc_buf(buf, size, MEM_TYPE_ZEROED | MEM_TYPE_ALIGN); +} + +enum caam_status caam_alloc_align_buf(struct caambuf *buf, size_t size) +{ + return mem_alloc_buf(buf, size, MEM_TYPE_ALIGN); +} + +void caam_free_buf(struct caambuf *buf) +{ + if (buf) { + if (buf->data) { + caam_free(buf->data); + buf->data = NULL; + } + + buf->length = 0; + buf->paddr = 0; + buf->nocache = 0; + } +} + +bool caam_mem_is_cached_buf(void *buf, size_t size) +{ + enum teecore_memtypes mtype = MEM_AREA_MAXTYPE; + bool is_cached = false; + + /* + * First check if the buffer is a known memory area mapped + * with a type listed in the teecore_memtypes enum. + * If not mapped, this is a User Area and so assume + * it cacheable + */ + mtype = core_mmu_get_type_by_pa(virt_to_phys(buf)); + if (mtype == MEM_AREA_MAXTYPE) + is_cached = true; + else + is_cached = core_vbuf_is(CORE_MEM_CACHED, buf, size); + + return is_cached; +} + +enum caam_status caam_cpy_block_src(struct caamblock *block, + struct caambuf *src, size_t offset) +{ + enum caam_status ret = CAAM_FAILURE; + size_t cpy_size = 0; + + if (!src->data) + return CAAM_FAILURE; + + /* Check if the temporary buffer is allocated, else allocate it */ + if (!block->buf.data) { + ret = caam_alloc_align_buf(&block->buf, block->max); + if (ret != CAAM_NO_ERROR) { + MEM_TRACE("Allocation Block buffer error"); + goto end_cpy; + } + } + + /* Calculate the number of bytes to copy in the block buffer */ + MEM_TRACE("Current buffer is %zu (%zu) bytes", block->filled, + block->max); + + cpy_size = block->max - block->filled; + cpy_size = MIN(cpy_size, src->length - offset); + + memcpy(&block->buf.data[block->filled], &src->data[offset], cpy_size); + + block->filled += cpy_size; + + ret = CAAM_NO_ERROR; + +end_cpy: + return ret; +} + +int caam_mem_get_pa_area(struct caambuf *buf, struct caambuf **out_pabufs) +{ + int nb_pa_area = 0; + size_t len = 0; + size_t len_tohandle = 0; + vaddr_t va = 0; + vaddr_t next_va = 0; + paddr_t pa = 0; + paddr_t next_pa = 0; + struct caambuf *pabufs = NULL; + + MEM_TRACE("Get PA Areas of %p-%zu (out %p)", buf->data, buf->length, + out_pabufs); + + if (out_pabufs) { + /* + * Caller asked for the extracted contiguous + * physical areas. + * Allocate maximum possible small pages + */ + if (buf->length > SMALL_PAGE_SIZE) { + nb_pa_area = buf->length / SMALL_PAGE_SIZE + 1; + if (buf->length % SMALL_PAGE_SIZE) + nb_pa_area++; + } else { + nb_pa_area = 2; + } + + pabufs = caam_calloc(nb_pa_area * sizeof(*pabufs)); + if (!pabufs) + return -1; + + MEM_TRACE("Allocate max %d Physical Areas", nb_pa_area); + } + + /* + * Go thru all the VA space to extract the contiguous + * physical areas + */ + va = (vaddr_t)buf->data; + pa = virt_to_phys((void *)va); + if (!pa) + goto err; + + nb_pa_area = 0; + if (pabufs) { + pabufs[nb_pa_area].data = (uint8_t *)va; + pabufs[nb_pa_area].paddr = pa; + pabufs[nb_pa_area].length = 0; + pabufs[nb_pa_area].nocache = buf->nocache; + MEM_TRACE("Add %d PA 0x%" PRIxPA " VA 0x%" PRIxVA, nb_pa_area, + pa, va); + } + + for (len = buf->length; len; len -= len_tohandle) { + len_tohandle = + MIN(SMALL_PAGE_SIZE - (va & SMALL_PAGE_MASK), len); + next_va = va + len_tohandle; + if (pabufs) + pabufs[nb_pa_area].length += len_tohandle; + + /* + * Reaches the end of buffer, exits here because + * the next virtual address is out of scope. + */ + if (len == len_tohandle) + break; + + touch_page(next_va); + next_pa = virt_to_phys((void *)next_va); + if (!next_pa) + goto err; + + if (next_pa != (pa + len_tohandle)) { + nb_pa_area++; + if (pabufs) { + pabufs[nb_pa_area].data = (uint8_t *)next_va; + pabufs[nb_pa_area].paddr = next_pa; + pabufs[nb_pa_area].length = 0; + pabufs[nb_pa_area].nocache = buf->nocache; + } + MEM_TRACE("Add %d PA 0x%" PRIxPA " VA 0x%" PRIxVA, + nb_pa_area, next_pa, next_va); + } + + va = next_va; + pa = next_pa; + } + + if (out_pabufs) + *out_pabufs = pabufs; + + MEM_TRACE("Nb Physical Area %d", nb_pa_area + 1); + return nb_pa_area + 1; + +err: + free(pabufs); + return -1; +} + +void caam_mem_cpy_ltrim_buf(struct caambuf *dst, struct caambuf *src) +{ + size_t offset = 0; + size_t cpy_size = 0; + + /* Calculate the offset to start the copy */ + while (!src->data[offset] && offset < src->length) + offset++; + + if (offset >= src->length) + offset = src->length - 1; + + cpy_size = MIN(dst->length, (src->length - offset)); + MEM_TRACE("Copy %zu of src %zu bytes (offset = %zu)", cpy_size, + src->length, offset); + memcpy(dst->data, &src->data[offset], cpy_size); + + dst->length = cpy_size; +} diff --git a/optee_os/core/drivers/crypto/caam/utils/utils_sgt.c b/optee_os/core/drivers/crypto/caam/utils/utils_sgt.c new file mode 100644 index 0000000..4ebbb25 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/utils_sgt.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019, 2021 NXP + * + * Brief Scatter-Gatter Table management utilities. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Perform cache management clean operation on the SGT table entry + * + * @sgtbuf SGT table to manage + */ +static void caam_sgt_entries_cache_clean(const struct caamsgtbuf *sgtbuf) +{ + cache_operation(TEE_CACHECLEAN, (void *)sgtbuf->sgt, + ROUNDUP(sgtbuf->number, CFG_CAAM_SGT_ALIGN) * + sizeof(*sgtbuf->sgt)); +} + +void caam_sgt_cache_op(enum utee_cache_operation op, struct caamsgtbuf *insgt, + size_t length) +{ + unsigned int idx = 0; + size_t op_size = 0; + size_t rem_length = length; + + caam_sgt_entries_cache_clean(insgt); + + SGT_TRACE("SGT @%p %d entries", insgt, insgt->number); + for (idx = 0; idx < insgt->number && rem_length; idx++) { + op_size = MIN(rem_length, insgt->buf[idx].length); + if (!insgt->buf[idx].nocache) + cache_operation(op, (void *)insgt->buf[idx].data, + op_size); + rem_length -= op_size; + } +} + +void caam_sgt_fill_table(struct caamsgtbuf *sgt) +{ + unsigned int idx = 0; + + SGT_TRACE("Create %d SGT entries", sgt->number); + + for (idx = 0; idx < sgt->number - 1; idx++) { + CAAM_SGT_ENTRY(&sgt->sgt[idx], sgt->buf[idx].paddr, + sgt->buf[idx].length); + sgt_entry_trace(idx, sgt); + } + + CAAM_SGT_ENTRY_FINAL(&sgt->sgt[idx], sgt->buf[idx].paddr, + sgt->buf[idx].length); + sgt_entry_trace(idx, sgt); +} + +enum caam_status caam_sgt_derive(struct caamsgtbuf *sgt, + const struct caamsgtbuf *from, size_t offset, + size_t length) +{ + enum caam_status retstatus = CAAM_FAILURE; + unsigned int idx = 0; + unsigned int st_idx = 0; + size_t off = offset; + size_t rlength = length; + + SGT_TRACE("Derive from %p - offset %zu, %d SGT entries", from, offset, + from->number); + + if (from->length - offset < length) { + SGT_TRACE("From SGT/Buffer too short (%zu)", from->length); + return CAAM_SHORT_BUFFER; + } + + for (; idx < from->number && off >= from->buf[idx].length; idx++) + off -= from->buf[idx].length; + + st_idx = idx; + sgt->number = 1; + rlength -= MIN(rlength, from->buf[idx].length - off); + + for (idx++; idx < from->number && rlength; idx++) { + rlength -= MIN(rlength, from->buf[idx].length); + sgt->number++; + } + + sgt->sgt_type = (sgt->number > 1) ? true : false; + + /* Allocate a new SGT/Buffer object */ + retstatus = caam_sgtbuf_alloc(sgt); + SGT_TRACE("Allocate %d SGT entries ret 0x%" PRIx32, sgt->number, + retstatus); + if (retstatus != CAAM_NO_ERROR) + return retstatus; + + memcpy(sgt->buf, &from->buf[st_idx], sgt->number * sizeof(*sgt->buf)); + + if (sgt->sgt_type) { + memcpy(sgt->sgt, &from->sgt[st_idx], + sgt->number * sizeof(*sgt->sgt)); + + /* Set the offset of the first sgt entry */ + sgt_entry_offset(sgt->sgt, off); + + /* + * Push the SGT Table into memory now because + * derived objects are not pushed. + */ + caam_sgt_entries_cache_clean(sgt); + + sgt->paddr = virt_to_phys(sgt->sgt); + } else { + sgt->paddr = sgt->buf->paddr + off; + } + + sgt->length = length; + + return CAAM_NO_ERROR; +} + +void caam_sgtbuf_free(struct caamsgtbuf *data) +{ + if (data->sgt_type) + caam_free(data->sgt); + else + caam_free(data->buf); + + data->sgt = NULL; + data->buf = NULL; +} + +enum caam_status caam_sgtbuf_alloc(struct caamsgtbuf *data) +{ + unsigned int nb_sgt = 0; + + if (!data || !data->number) + return CAAM_BAD_PARAM; + + if (data->sgt_type) { + nb_sgt = ROUNDUP(data->number, CFG_CAAM_SGT_ALIGN); + data->sgt = caam_calloc(nb_sgt * (sizeof(union caamsgt) + + sizeof(struct caambuf))); + data->buf = (void *)(((uint8_t *)data->sgt) + + (nb_sgt * sizeof(union caamsgt))); + } else { + data->buf = caam_calloc(data->number * sizeof(struct caambuf)); + data->sgt = NULL; + } + + if (!data->buf || (!data->sgt && data->sgt_type)) { + caam_sgtbuf_free(data); + return CAAM_OUT_MEMORY; + } + + return CAAM_NO_ERROR; +} diff --git a/optee_os/core/drivers/crypto/caam/utils/utils_sgt_v1.c b/optee_os/core/drivers/crypto/caam/utils/utils_sgt_v1.c new file mode 100644 index 0000000..577585b --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/utils_sgt_v1.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * Scatter-gather entry management code for version 1 + */ + +#include +#include +#include + +#define ENTRY_LEN(len) (((uint32_t)len) & GENMASK_32(29, 0)) +#define BS_ENTRY_FINAL BIT32(30) + +void sgt_entry_trace(unsigned int idx __maybe_unused, + const struct caamsgtbuf *sgt __maybe_unused) +{ + SGT_TRACE("SGT[%d] (%p)", idx, &sgt->sgt[idx]); + SGT_TRACE("SGT[%d]->data = %p", idx, sgt->buf[idx].data); + SGT_TRACE("SGT[%d]->length = %zu", idx, sgt->buf[idx].length); + SGT_TRACE("SGT[%d]->paddr = 0x%" PRIxPA, idx, sgt->buf[idx].paddr); + SGT_TRACE("SGT[%d]->ptr_ms = %" PRIx32, idx, sgt->sgt[idx].v1.ptr_ms); + SGT_TRACE("SGT[%d]->ptr_ls = %" PRIx32, idx, sgt->sgt[idx].v1.ptr_ls); + SGT_TRACE("SGT[%d]->len_f_e = %" PRIx32, idx, + sgt->sgt[idx].v1.len_f_e); + SGT_TRACE("SGT[%d]->offset = %" PRIx32, idx, sgt->sgt[idx].v1.offset); +} + +void sgt_entry_offset(union caamsgt *sgt, unsigned int offset) +{ + uint32_t len_f_e = 0; + + len_f_e = caam_read_val32(&sgt->v1.len_f_e); + + /* Set the new length and keep the Final bit if set */ + len_f_e = (ENTRY_LEN(len_f_e) - offset) | (len_f_e & BS_ENTRY_FINAL); + + caam_write_val32(&sgt->v1.len_f_e, len_f_e); + caam_write_val32(&sgt->v1.offset, offset); +} + +void caam_sgt_set_entry(union caamsgt *sgt, paddr_t paddr, size_t len, + unsigned int offset, bool final_e) +{ + unsigned int len_f_e = 0; + + caam_write_val32(&sgt->v1.ptr_ls, paddr); +#if defined(CFG_CAAM_64BIT) && defined(CFG_ARM64_core) + caam_write_val32(&sgt->v1.ptr_ms, paddr >> 32); +#else + caam_write_val32(&sgt->v1.ptr_ms, 0); +#endif + + len_f_e = ENTRY_LEN(len); + if (final_e) + len_f_e |= BS_ENTRY_FINAL; + + caam_write_val32(&sgt->v1.len_f_e, len_f_e); + caam_write_val32(&sgt->v1.offset, offset); +} diff --git a/optee_os/core/drivers/crypto/caam/utils/utils_sgt_v2.c b/optee_os/core/drivers/crypto/caam/utils/utils_sgt_v2.c new file mode 100644 index 0000000..5d7fd39 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/utils_sgt_v2.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * Scatter-gather entry management code for version 2 + */ + +#include +#include +#include + +void sgt_entry_trace(unsigned int idx __maybe_unused, + const struct caamsgtbuf *sgt __maybe_unused) +{ + SGT_TRACE("SGT[%d] (%p)", idx, &sgt->sgt[idx]); + SGT_TRACE("SGT[%d]->data = %p", idx, sgt->buf[idx].data); + SGT_TRACE("SGT[%d]->length = %zu", idx, sgt->buf[idx].length); + SGT_TRACE("SGT[%d]->paddr = 0x%" PRIxPA, idx, sgt->buf[idx].paddr); + SGT_TRACE("SGT[%d]->w1 = %" PRIx64, idx, sgt->sgt[idx].v2.w1); + SGT_TRACE("SGT[%d]->w2 = %" PRIx64, idx, sgt->sgt[idx].v2.w2); +} + +void sgt_entry_offset(union caamsgt *sgt, unsigned int offset) +{ + uint64_t w2 = 0; + uint64_t len = 0; + uint64_t off = 0; + + w2 = caam_read_val64(&sgt->v2.w2); + + /* + * Compute the new offset reading the one present and adding the + * input + */ + off = SGT_V2_ENTRY_OFFSET(w2); + off += offset; + + /* Reading length and computing new value by subtracting the offset */ + len = SGT_V2_ENTRY_AVAIL_LENGTH(w2); + len = (offset > len) ? 0 : len - offset; + + /* Clear the offset and length fields */ + w2 &= ~(BM_SGT_V2_OFFSET | BM_SGT_V2_AVAIL_LENGTH); + + /* Update offset and field */ + w2 |= BV_SGT_V2_OFFSET(offset) | BV_SGT_V2_AVAIL_LENGTH(len); + + caam_write_val64(&sgt->v2.w2, w2); +} + +void caam_sgt_set_entry(union caamsgt *sgt, paddr_t paddr, size_t len, + unsigned int offset, bool final_e) +{ + uint64_t w2 = 0; + + /* Write the address to set */ + caam_write_val64(&sgt->v2.w1, paddr); + + /* Compute the second word */ + w2 = (final_e ? BM_SGT_V2_F : 0) | BV_SGT_V2_OFFSET(offset) | + BM_SGT_V2_IVP | BV_SGT_V2_AVAIL_LENGTH(len); + + caam_write_val64(&sgt->v2.w2, w2); +} diff --git a/optee_os/core/drivers/crypto/caam/utils/utils_status.c b/optee_os/core/drivers/crypto/caam/utils/utils_status.c new file mode 100644 index 0000000..1bade52 --- /dev/null +++ b/optee_os/core/drivers/crypto/caam/utils/utils_status.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019, 2021 NXP + * + * Brief Status management utilities. + */ +#include +#include +#include + +TEE_Result job_status_to_tee_result(uint32_t status) +{ + /* + * Add all status code that can be translated + * to a TEE_Result other than TEE_ERROR_GENERIC + */ + switch (JRSTA_SRC_GET(status)) { + case JRSTA_SRC(NONE): + return TEE_SUCCESS; + + case JRSTA_SRC(DECO): + if (JRSTA_CCB_GET_ERR(status) == JRSTA_DECO_ERRID_FORMAT) + return TEE_ERROR_BAD_PARAMETERS; + + if (JRSTA_CCB_GET_ERR(status) == JRSTA_DECO_INV_SIGNATURE) + return TEE_ERROR_SIGNATURE_INVALID; + + break; + + default: + break; + } + + return TEE_ERROR_GENERIC; +} + +TEE_Result caam_status_to_tee_result(enum caam_status status) +{ + switch (status) { + case CAAM_NO_ERROR: + return TEE_SUCCESS; + + case CAAM_OUT_MEMORY: + return TEE_ERROR_OUT_OF_MEMORY; + + case CAAM_BAD_PARAM: + return TEE_ERROR_BAD_PARAMETERS; + + case CAAM_SHORT_BUFFER: + return TEE_ERROR_SHORT_BUFFER; + + case CAAM_NOT_SUPPORTED: + return TEE_ERROR_NOT_SUPPORTED; + + default: + break; + } + + return TEE_ERROR_GENERIC; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/dh.c b/optee_os/core/drivers/crypto/crypto_api/acipher/dh.c new file mode 100644 index 0000000..2414517 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/dh.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Crypto DH interface implementation to enable HW driver. + */ +#include +#include +#include +#include + +TEE_Result crypto_acipher_alloc_dh_keypair(struct dh_keypair *key, + size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_dh *dh = NULL; + + if (!key || !size_bits) { + CRYPTO_TRACE("Param error key @%#" PRIxPTR " size %zu bits", + (uintptr_t)key, size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + dh = drvcrypt_get_ops(CRYPTO_DH); + if (dh) + ret = dh->alloc_keypair(key, size_bits); + + CRYPTO_TRACE("DH Keypair (%zu bits) alloc ret = 0x%" PRIx32, size_bits, + ret); + return ret; +} + +TEE_Result crypto_acipher_gen_dh_key(struct dh_keypair *key, struct bignum *q, + size_t xbits, size_t key_size) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_dh *dh = NULL; + + if (!key) { + CRYPTO_TRACE("Parameters error key is NULL"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (key_size != 8 * crypto_bignum_num_bytes(key->p)) + return TEE_ERROR_BAD_PARAMETERS; + + dh = drvcrypt_get_ops(CRYPTO_DH); + if (dh) + ret = dh->gen_keypair(key, q, xbits); + + CRYPTO_TRACE("DH Keypair (%zu bits) generate ret = 0x%" PRIx32, + key_size, ret); + + return ret; +} + +TEE_Result crypto_acipher_dh_shared_secret(struct dh_keypair *private_key, + struct bignum *public_key, + struct bignum *secret) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_dh *dh = NULL; + struct drvcrypt_secret_data sdata = { }; + uint8_t *secret_buf = NULL; + + if (!private_key || !public_key || !secret) { + CRYPTO_TRACE("Input parameters reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + dh = drvcrypt_get_ops(CRYPTO_DH); + if (dh) { + /* Allocate the binary Secret buffer */ + sdata.secret.length = crypto_bignum_num_bytes(private_key->p); + secret_buf = malloc(sdata.secret.length); + if (!secret_buf) + return TEE_ERROR_OUT_OF_MEMORY; + + /* Prepare the Secret structure data */ + sdata.key_priv = private_key; + sdata.key_pub = public_key; + sdata.secret.data = secret_buf; + + ret = dh->shared_secret(&sdata); + if (ret == TEE_SUCCESS) + ret = crypto_bignum_bin2bn(secret_buf, + sdata.secret.length, secret); + + free(secret_buf); + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Shared Secret returned 0x%" PRIx32, ret); + + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/dsa.c b/optee_os/core/drivers/crypto/crypto_api/acipher/dsa.c new file mode 100644 index 0000000..780b8be --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/dsa.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Crypto DSA interface implementation to enable HW driver. + */ +#include +#include +#include + +/* + * Get the recommended L and N bits parameters corresponding + * respectively to the size of the Primes P and G (and so + * the Public Key and Private Key). + * + * Refer the NIST.FIPS 186-4 section 4.2 + * + * @size_bits Maximum key size bits + * @l_bits [out] L size in bits + * @n_bits [out] N size in bits + */ +static TEE_Result get_keys_size(size_t size_bits, size_t *l_bits, + size_t *n_bits) +{ + if (size_bits <= 1024) + *n_bits = 160; + else if (size_bits <= 3072) + *n_bits = 256; + else + return TEE_ERROR_NOT_IMPLEMENTED; + + *l_bits = size_bits; + + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_alloc_dsa_keypair(struct dsa_keypair *key, + size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_dsa *dsa = NULL; + size_t l_bits = 0; + size_t n_bits = 0; + + if (!key || !size_bits) { + CRYPTO_TRACE("Param error key @0x%" PRIxPTR " size %zu bits", + (uintptr_t)key, size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = get_keys_size(size_bits, &l_bits, &n_bits); + if (ret == TEE_SUCCESS) { + dsa = drvcrypt_get_ops(CRYPTO_DSA); + if (dsa) + ret = dsa->alloc_keypair(key, l_bits, n_bits); + else + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("DSA Keypair (%zu bits) alloc ret = 0x%" PRIx32, size_bits, + ret); + return ret; +} + +TEE_Result crypto_acipher_alloc_dsa_public_key(struct dsa_public_key *key, + size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_dsa *dsa = NULL; + size_t l_bits = 0; + size_t n_bits = 0; + + if (!key || !size_bits) { + CRYPTO_TRACE("Param error key @0x%" PRIxPTR " size %zu bits", + (uintptr_t)key, size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = get_keys_size(size_bits, &l_bits, &n_bits); + if (ret == TEE_SUCCESS) { + dsa = drvcrypt_get_ops(CRYPTO_DSA); + if (dsa) + ret = dsa->alloc_publickey(key, l_bits, n_bits); + else + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("DSA Public Key (%zu bits) alloc ret = 0x%" PRIx32, + size_bits, ret); + return ret; +} + +TEE_Result crypto_acipher_gen_dsa_key(struct dsa_keypair *key, size_t key_size) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_dsa *dsa = NULL; + size_t l_bits = 0; + size_t n_bits = 0; + + if (!key || !key_size) { + CRYPTO_TRACE("Param error key @0x%" PRIxPTR " size %zu bits", + (uintptr_t)key, key_size); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = get_keys_size(key_size, &l_bits, &n_bits); + if (ret == TEE_SUCCESS) { + dsa = drvcrypt_get_ops(CRYPTO_DSA); + if (dsa) + ret = dsa->gen_keypair(key, l_bits, n_bits); + else + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("DSA Keypair (%zu bits) generate ret = 0x%" PRIx32, + key_size, ret); + + return ret; +} + +TEE_Result crypto_acipher_dsa_sign(uint32_t algo, struct dsa_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_dsa *dsa = NULL; + struct drvcrypt_sign_data sdata = { }; + size_t l_bytes = 0; + size_t n_bytes = 0; + + if (!key || !msg || !sig_len) { + CRYPTO_TRACE("Input parameters reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* + * Verify the signature length function of the key size + * + * Prime number sizes are not stored but deducted from bignum size. + * This requires prime numbers p and q to have their MSB set otherwise + * crypto_bignum_num_bytes() will return a wrong size. + */ + n_bytes = crypto_bignum_num_bytes(key->q); + l_bytes = crypto_bignum_num_bytes(key->p); + if (*sig_len < 2 * n_bytes) { + CRYPTO_TRACE("Length (%zu) too short expected %zu bytes", + *sig_len, 2 * n_bytes); + *sig_len = 2 * n_bytes; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!sig) { + CRYPTO_TRACE("Parameter \"sig\" reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + dsa = drvcrypt_get_ops(CRYPTO_DSA); + if (dsa) { + sdata.algo = algo; + sdata.key = key; + sdata.size_sec = n_bytes; + sdata.message.data = (uint8_t *)msg; + sdata.message.length = msg_len; + sdata.signature.data = sig; + sdata.signature.length = *sig_len; + + ret = dsa->sign(&sdata, l_bytes, n_bytes); + + /* Set the signature length */ + *sig_len = sdata.signature.length; + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Sign algo (0x%" PRIx32 ") returned 0x%" PRIx32, algo, + ret); + + return ret; +} + +TEE_Result crypto_acipher_dsa_verify(uint32_t algo, struct dsa_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_dsa *dsa = NULL; + struct drvcrypt_sign_data sdata = { }; + size_t l_bytes = 0; + size_t n_bytes = 0; + + if (!key || !msg || !sig) { + CRYPTO_TRACE("Input parameters reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* + * Verify the signature length function of the key size + * + * Prime number sizes are not stored but deducted from bignum size. + * This requires prime numbers p and q to have their MSB set otherwise + * crypto_bignum_num_bytes() will return a wrong size. + */ + n_bytes = crypto_bignum_num_bytes(key->q); + l_bytes = crypto_bignum_num_bytes(key->p); + if (sig_len != 2 * n_bytes) { + CRYPTO_TRACE("Length (%zu) is invalid expected %zu bytes", + sig_len, 2 * n_bytes); + return TEE_ERROR_SIGNATURE_INVALID; + } + + dsa = drvcrypt_get_ops(CRYPTO_DSA); + if (dsa) { + sdata.algo = algo; + sdata.key = key; + sdata.size_sec = n_bytes; + sdata.message.data = (uint8_t *)msg; + sdata.message.length = msg_len; + sdata.signature.data = (uint8_t *)sig; + sdata.signature.length = sig_len; + + ret = dsa->verify(&sdata, l_bytes, n_bytes); + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Verify algo (0x%" PRIx32 ") returned 0x%" PRIx32, algo, + ret); + + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/ecc.c b/optee_os/core/drivers/crypto/crypto_api/acipher/ecc.c new file mode 100644 index 0000000..6ec0e10 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/ecc.c @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2021 NXP + * + * Crypto ECC interface implementation to enable HW driver. + */ +#include +#include +#include +#include + +/* + * Returns the key size in bytes for the given ECC curve + * + * @curve ECC Curve ID + */ +static size_t get_ecc_key_size_bytes(uint32_t curve) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + return 24; + + case TEE_ECC_CURVE_NIST_P224: + return 28; + + case TEE_ECC_CURVE_NIST_P256: + case TEE_ECC_CURVE_SM2: + return 32; + + case TEE_ECC_CURVE_NIST_P384: + return 48; + + case TEE_ECC_CURVE_NIST_P521: + return 66; + + default: + return 0; + } +} + +/* + * Returns the key size in bits for the given ECC curve + * + * @curve ECC Curve ID + */ + +static size_t get_ecc_key_size_bits(uint32_t curve) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + case TEE_ECC_CURVE_NIST_P224: + case TEE_ECC_CURVE_NIST_P256: + case TEE_ECC_CURVE_NIST_P384: + case TEE_ECC_CURVE_SM2: + return get_ecc_key_size_bytes(curve) * 8; + + case TEE_ECC_CURVE_NIST_P521: + return 521; + + default: + return 0; + } +} + +/* + * Verify if the cryptographic algorithm @algo is valid for + * the ECC curve + * + * @curve ECC curve + * @algo Cryptographic algorithm + */ +static bool algo_is_valid(uint32_t curve, uint32_t algo) +{ + unsigned int algo_op = TEE_ALG_GET_CLASS(algo); + unsigned int algo_id = TEE_ALG_GET_MAIN_ALG(algo); + unsigned int algo_curve = TEE_ALG_GET_DIGEST_HASH(algo); + + /* Check first the algo operation and id */ + if ((algo_op == TEE_OPERATION_ASYMMETRIC_SIGNATURE && + algo_id == TEE_MAIN_ALGO_ECDSA) || + (algo_op == TEE_OPERATION_KEY_DERIVATION && + algo_id == TEE_MAIN_ALGO_ECDH)) { + if (curve == algo_curve) { + CRYPTO_TRACE("Algo 0x%" PRIx32 " curve 0x%" PRIx32 + " is valid", algo, curve); + return true; + } + } + + if (algo_op == TEE_OPERATION_ASYMMETRIC_SIGNATURE && + algo_id == TEE_MAIN_ALGO_SM2_DSA_SM3) { + if (curve == TEE_ECC_CURVE_SM2) + return true; + } + + CRYPTO_TRACE("Algo 0x%" PRIx32 " curve 0x%" PRIx32 " is not valid", + algo, curve); + + return false; +} + +/* + * Free an ECC public key + * + * @key Public Key + */ +static void ecc_free_public_key(struct ecc_public_key *key) +{ + struct drvcrypt_ecc *ecc = NULL; + + if (key) { + ecc = drvcrypt_get_ops(CRYPTO_ECC); + if (ecc) { + CRYPTO_TRACE("ECC Public Key free"); + ecc->free_publickey(key); + } + } +} + +/* + * Generates an ECC keypair + * + * @key Keypair + * @size_bits Key size in bits + */ +static TEE_Result ecc_generate_keypair(struct ecc_keypair *key, + size_t size_bits __maybe_unused) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_ecc *ecc = NULL; + size_t key_size_bits = 0; + + /* Check input parameters */ + if (!key) { + CRYPTO_TRACE("Parameters error key is NULL"); + return TEE_ERROR_BAD_PARAMETERS; + } + + key_size_bits = get_ecc_key_size_bits(key->curve); + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + if (ecc) + ret = ecc->gen_keypair(key, key_size_bits); + + CRYPTO_TRACE("ECC Keypair (%zu bits) generate ret = 0x%" PRIx32, + key_size_bits, ret); + + return ret; +} + +/* + * Sign the message with the ECC Key given by the Keypair + * + * @algo ECC algorithm + * @key ECC Keypair + * @msg Message to sign + * @msg_len Length of the message (bytes) + * @sig Signature + * @sig_len [in/out] Length of the signature (bytes) + */ +static TEE_Result ecc_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, uint8_t *sig, + size_t *sig_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_ecc *ecc = NULL; + struct drvcrypt_sign_data sdata = { }; + size_t size_bytes = 0; + + /* Verify first the input parameters */ + if (!key || !msg || !sig_len) { + CRYPTO_TRACE("Input parameters reference error"); + return ret; + } + + if (!algo_is_valid(key->curve, algo)) + return ret; + + size_bytes = get_ecc_key_size_bytes(key->curve); + if (!size_bytes) + return TEE_ERROR_BAD_PARAMETERS; + + /* Verify the signature length function of the key size */ + if (*sig_len < 2 * size_bytes) { + CRYPTO_TRACE("Length (%zu) too short expected %zu bytes", + *sig_len, 2 * size_bytes); + *sig_len = 2 * size_bytes; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!sig) { + CRYPTO_TRACE("Parameter \"sig\" reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + if (ecc) { + /* + * Prepare the Signature structure data + */ + sdata.algo = algo; + sdata.key = key; + sdata.size_sec = size_bytes; + sdata.message.data = (uint8_t *)msg; + sdata.message.length = msg_len; + sdata.signature.data = (uint8_t *)sig; + sdata.signature.length = *sig_len; + + ret = ecc->sign(&sdata); + + /* Set the signature length */ + *sig_len = sdata.signature.length; + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Sign algo (0x%" PRIx32 ") returned 0x%" PRIx32, algo, + ret); + + return ret; +} + +/* + * Verify if signature is signed with the given public key. + * + * @algo ECC algorithm + * @key ECC Public key + * @msg Message to sign + * @msg_len Length of the message (bytes) + * @sig Signature + * @sig_len Length of the signature (bytes) + */ +static TEE_Result ecc_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_ecc *ecc = NULL; + struct drvcrypt_sign_data sdata = { }; + size_t size_bytes = 0; + + /* Verify first the input parameters */ + if (!key || !msg || !sig) { + CRYPTO_TRACE("Input parameters reference error"); + return ret; + } + + if (!algo_is_valid(key->curve, algo)) + return ret; + + size_bytes = get_ecc_key_size_bytes(key->curve); + if (!size_bytes) + return TEE_ERROR_BAD_PARAMETERS; + + /* Verify the signature length against key size */ + if (sig_len != 2 * size_bytes) { + CRYPTO_TRACE("Length (%zu) is invalid expected %zu bytes", + sig_len, 2 * size_bytes); + return TEE_ERROR_SIGNATURE_INVALID; + } + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + if (ecc) { + sdata.algo = algo; + sdata.key = key; + sdata.size_sec = size_bytes; + sdata.message.data = (uint8_t *)msg; + sdata.message.length = msg_len; + sdata.signature.data = (uint8_t *)sig; + sdata.signature.length = sig_len; + + ret = ecc->verify(&sdata); + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Verify algo (0x%" PRIx32 ") returned 0x%" PRIx32, algo, + ret); + + return ret; +} + +/* + * Compute the shared secret data from ECC Private key and Public Key + * + * @private_key ECC Private key + * @public_key ECC Public key + * @secret Secret + * @secret_len Length of the secret (bytes) + */ +static TEE_Result ecc_shared_secret(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, unsigned long *secret_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_ecc *ecc = NULL; + struct drvcrypt_secret_data sdata = { }; + size_t size_bytes = 0; + + /* Verify first the input parameters */ + if (!private_key || !public_key || !secret_len) { + CRYPTO_TRACE("Input parameters reference error"); + return ret; + } + + if (private_key->curve != public_key->curve) { + CRYPTO_TRACE("Private Key curve (%d) != Public Key curve (%d)", + private_key->curve, public_key->curve); + return ret; + } + + size_bytes = get_ecc_key_size_bytes(public_key->curve); + if (!size_bytes) + return ret; + + if (*secret_len < size_bytes) { + *secret_len = size_bytes; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!secret) { + CRYPTO_TRACE("Parameter \"secret\" reference error"); + return ret; + } + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + if (ecc) { + /* + * Prepare the Secret structure data + */ + sdata.key_priv = private_key; + sdata.key_pub = public_key; + sdata.size_sec = size_bytes; + sdata.secret.data = secret; + sdata.secret.length = *secret_len; + + ret = ecc->shared_secret(&sdata); + + /* Set the secret length */ + *secret_len = sdata.secret.length; + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Shared Secret returned 0x%" PRIx32, ret); + + return ret; +} + +static TEE_Result ecc_sm2_encrypt(struct ecc_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_ecc_ed cdata = { }; + struct drvcrypt_ecc *ecc = NULL; + size_t ciphertext_len = 0; + size_t size_bytes = 0; + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + + size_bytes = get_ecc_key_size_bytes(key->curve); + if (!size_bytes) { + CRYPTO_TRACE("Curve 0x%08"PRIx32" not supported", key->curve); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Uncompressed form indicator */ + dst[0] = 0x04; + + ciphertext_len = 2 * size_bytes + src_len + TEE_SM3_HASH_SIZE; + + cdata.key = key; + cdata.size_sec = size_bytes; + cdata.plaintext.data = (uint8_t *)src; + cdata.plaintext.length = src_len; + cdata.ciphertext.data = dst + 1; + cdata.ciphertext.length = ciphertext_len; + + ret = ecc->encrypt(&cdata); + + if (!ret || ret == TEE_ERROR_SHORT_BUFFER) + *dst_len = cdata.ciphertext.length + 1; + + return ret; +} + +static TEE_Result ecc_encrypt(struct ecc_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + struct drvcrypt_ecc *ecc = NULL; + + if (!key || !src || !dst) { + CRYPTO_TRACE("Input parameters reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + if (!ecc || !ecc->encrypt) + return TEE_ERROR_NOT_IMPLEMENTED; + + switch (key->curve) { + case TEE_ECC_CURVE_SM2: + return ecc_sm2_encrypt(key, src, src_len, dst, dst_len); + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +static TEE_Result ecc_sm2_decrypt(struct ecc_keypair *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_ecc_ed cdata = { }; + struct drvcrypt_ecc *ecc = NULL; + uint8_t *ciphertext = NULL; + size_t ciphertext_len = 0; + size_t size_bytes = 0; + size_t plaintext_len = 0; + /* Point Compression */ + uint8_t pc = 0; + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + + size_bytes = get_ecc_key_size_bytes(key->curve); + if (!size_bytes) { + CRYPTO_TRACE("Curve 0x%08"PRIx32" not supported", key->curve); + return TEE_ERROR_BAD_PARAMETERS; + } + + pc = src[0]; + switch (pc) { + case 0x02: + case 0x03: + /* Compressed form */ + return TEE_ERROR_NOT_SUPPORTED; + case 0x04: + /* Uncompressed form */ + ciphertext = (uint8_t *)src + 1; + ciphertext_len = src_len - 1; + break; + case 0x06: + case 0x07: + /* Hybrid form */ + return TEE_ERROR_NOT_SUPPORTED; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + if (SUB_OVERFLOW(ciphertext_len, 2 * size_bytes + TEE_SM3_HASH_SIZE, + &plaintext_len)) + return TEE_ERROR_BAD_PARAMETERS; + + cdata.key = key; + cdata.size_sec = size_bytes; + cdata.ciphertext.data = ciphertext; + cdata.ciphertext.length = ciphertext_len; + cdata.plaintext.data = dst; + cdata.plaintext.length = plaintext_len; + + ret = ecc->decrypt(&cdata); + + /* Set the plaintext length */ + if (!ret || ret == TEE_ERROR_SHORT_BUFFER) + *dst_len = cdata.plaintext.length; + + return ret; +} + +static TEE_Result ecc_decrypt(struct ecc_keypair *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + struct drvcrypt_ecc *ecc = NULL; + + if (!key || !src || !dst) { + CRYPTO_TRACE("Input parameters reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ecc = drvcrypt_get_ops(CRYPTO_ECC); + if (!ecc || !ecc->decrypt) + return TEE_ERROR_NOT_IMPLEMENTED; + + switch (key->curve) { + case TEE_ECC_CURVE_SM2: + return ecc_sm2_decrypt(key, src, src_len, dst, dst_len); + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +static const struct crypto_ecc_keypair_ops ecc_keypair_ops = { + .generate = ecc_generate_keypair, + .sign = ecc_sign, + .shared_secret = ecc_shared_secret, + .decrypt = ecc_decrypt, +}; + +TEE_Result drvcrypt_asym_alloc_ecc_keypair(struct ecc_keypair *key, + uint32_t type, size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_ecc *ecc = NULL; + + if (!key || !size_bits) { + CRYPTO_TRACE("Bad parameters (key @%p)(size %zu bits)", key, + size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + switch (type) { + case TEE_TYPE_ECDSA_KEYPAIR: + case TEE_TYPE_ECDH_KEYPAIR: + case TEE_TYPE_SM2_PKE_KEYPAIR: + case TEE_TYPE_SM2_DSA_KEYPAIR: + ecc = drvcrypt_get_ops(CRYPTO_ECC); + break; + default: + break; + } + + if (ecc) + ret = ecc->alloc_keypair(key, type, size_bits); + + if (!ret) { + key->ops = &ecc_keypair_ops; + + /* ecc->alloc_keypair() can not get type to set curve */ + switch (type) { + case TEE_TYPE_SM2_PKE_KEYPAIR: + case TEE_TYPE_SM2_DSA_KEYPAIR: + key->curve = TEE_ECC_CURVE_SM2; + break; + default: + break; + } + } + + CRYPTO_TRACE("ECC Keypair (%zu bits) alloc ret = 0x%" PRIx32, size_bits, + ret); + return ret; +} + +static const struct crypto_ecc_public_ops ecc_public_key_ops = { + .free = ecc_free_public_key, + .verify = ecc_verify, + .encrypt = ecc_encrypt, +}; + +TEE_Result drvcrypt_asym_alloc_ecc_public_key(struct ecc_public_key *key, + uint32_t type, size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_ecc *ecc = NULL; + + if (!key || !size_bits) { + CRYPTO_TRACE("Bad parameters (key @%p)(size %zu bits)", key, + size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + switch (type) { + case TEE_TYPE_ECDSA_PUBLIC_KEY: + case TEE_TYPE_ECDH_PUBLIC_KEY: + case TEE_TYPE_SM2_PKE_PUBLIC_KEY: + case TEE_TYPE_SM2_DSA_PUBLIC_KEY: + ecc = drvcrypt_get_ops(CRYPTO_ECC); + break; + default: + break; + } + + if (ecc) + ret = ecc->alloc_publickey(key, type, size_bits); + + if (!ret) { + key->ops = &ecc_public_key_ops; + + /* ecc->alloc_publickey() can not get type to set curve */ + switch (type) { + case TEE_TYPE_SM2_PKE_PUBLIC_KEY: + case TEE_TYPE_SM2_DSA_PUBLIC_KEY: + key->curve = TEE_ECC_CURVE_SM2; + break; + default: + break; + } + } + + CRYPTO_TRACE("ECC Public Key (%zu bits) alloc ret = 0x%" PRIx32, + size_bits, ret); + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/local.h b/optee_os/core/drivers/crypto/crypto_api/acipher/local.h new file mode 100644 index 0000000..2700328 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/local.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2020 NXP + * + * Definition of the functions shared locally. + */ +#ifndef __LOCAL_H__ +#define __LOCAL_H__ + +#include + +/* + * Mask Generation function. Use a Hash operation + * to generate an output @mask from an input @seed + * + * @mgf_data [in/out] MGF data + */ +TEE_Result drvcrypt_rsa_mgf1(struct drvcrypt_rsa_mgf *mgf_data); + +/* + * PKCS#1 - Signature of RSA message and encodes the signature. + * + * @ssa_data [in/out] RSA data to sign / Signature + */ +TEE_Result drvcrypt_rsassa_sign(struct drvcrypt_rsa_ssa *ssa_data); + +/* + * PKCS#1 - Verification the encoded signature of RSA message. + * + * @ssa_data RSA Encoded signature data + */ +TEE_Result drvcrypt_rsassa_verify(struct drvcrypt_rsa_ssa *ssa_data); + +#endif /* __LOCAL_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/rsa.c b/optee_os/core/drivers/crypto/crypto_api/acipher/rsa.c new file mode 100644 index 0000000..24b26c2 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/rsa.c @@ -0,0 +1,504 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * Crypto RSA interface implementation to enable HW driver. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +TEE_Result crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *key, + size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + + struct drvcrypt_rsa *rsa = NULL; + + if (!key || !size_bits) { + CRYPTO_TRACE("Parameters error (key @%p) (size %zu bits)", key, + size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) + ret = rsa->alloc_keypair(key, size_bits); + + CRYPTO_TRACE("RSA Keypair (%zu bits) alloc ret = 0x%" PRIx32, size_bits, + ret); + return ret; +} + +TEE_Result crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *key, + size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_rsa *rsa = NULL; + + if (!key || !size_bits) { + CRYPTO_TRACE("Parameters error (key @%p) (size %zu bits)", key, + size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) + ret = rsa->alloc_publickey(key, size_bits); + + CRYPTO_TRACE("RSA Public Key (%zu bits) alloc ret = 0x%" PRIx32, + size_bits, ret); + return ret; +} + +void crypto_acipher_free_rsa_public_key(struct rsa_public_key *key) +{ + struct drvcrypt_rsa *rsa = NULL; + + if (key) { + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + CRYPTO_TRACE("RSA Public Key free"); + rsa->free_publickey(key); + } + } +} + +void crypto_acipher_free_rsa_keypair(struct rsa_keypair *key) +{ + struct drvcrypt_rsa *rsa = NULL; + + if (key) { + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + CRYPTO_TRACE("RSA Keypair free"); + rsa->free_keypair(key); + } + } +} + +TEE_Result crypto_acipher_gen_rsa_key(struct rsa_keypair *key, size_t size_bits) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_rsa *rsa = NULL; + + if (!key || !size_bits) { + CRYPTO_TRACE("Parameters error (key @%p) (size %zu bits) ", + key, size_bits); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) + ret = rsa->gen_keypair(key, size_bits); + + CRYPTO_TRACE("RSA Keypair (%zu bits) generate ret = 0x%" PRIx32, + size_bits, ret); + + return ret; +} + +TEE_Result crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key, + const uint8_t *cipher, + size_t cipher_len, uint8_t *msg, + size_t *msg_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_rsa *rsa = NULL; + struct drvcrypt_rsa_ed rsa_data = { }; + + if (!key || !msg || !cipher || !msg_len) { + CRYPTO_TRACE("Parameters error (key @%p)\n" + "(msg @%p size %zu bytes)\n" + "(cipher @0%p size %zu bytes)", + key, msg, msg_len ? *msg_len : 0, + cipher, cipher_len); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa_data.key.key = key; + rsa_data.key.isprivate = true; + rsa_data.key.n_size = crypto_bignum_num_bytes(key->n); + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + rsa_data.rsa_id = DRVCRYPT_RSA_NOPAD; + rsa_data.message.data = msg; + rsa_data.message.length = *msg_len; + rsa_data.cipher.data = (uint8_t *)cipher; + rsa_data.cipher.length = cipher_len; + + ret = rsa->decrypt(&rsa_data); + + *msg_len = rsa_data.message.length; + } + + CRYPTO_TRACE("RSA Decrypt NO PAD ret = 0x%" PRIx32, ret); + + return ret; +} + +TEE_Result crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key, + const uint8_t *msg, size_t msg_len, + uint8_t *cipher, size_t *cipher_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_rsa *rsa = NULL; + struct drvcrypt_rsa_ed rsa_data = { }; + + if (!key || !msg || !cipher_len) { + CRYPTO_TRACE("Parameters error (key @%p)\n" + "(msg @%p size %zu bytes)\n" + "(cipher @%p size %zu bytes)", + key, msg, msg_len, + cipher, cipher_len ? *cipher_len : 0); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa_data.key.key = key; + rsa_data.key.isprivate = false; + rsa_data.key.n_size = crypto_bignum_num_bytes(key->n); + + if (rsa_data.key.n_size > *cipher_len) { + CRYPTO_TRACE("Cipher length (%zu) too short expected %zu bytes", + *cipher_len, rsa_data.key.n_size); + *cipher_len = rsa_data.key.n_size; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!cipher) { + CRYPTO_TRACE("Parameter \"cipher\" reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the encryption data parameters */ + rsa_data.rsa_id = DRVCRYPT_RSA_NOPAD; + rsa_data.message.data = (uint8_t *)msg; + rsa_data.message.length = msg_len; + rsa_data.cipher.data = cipher; + rsa_data.cipher.length = *cipher_len; + + ret = rsa->encrypt(&rsa_data); + + /* Set the cipher size */ + *cipher_len = rsa_data.cipher.length; + } + + CRYPTO_TRACE("RSA Encrypt NO PAD ret = 0x%" PRIx32, ret); + + return ret; +} + +TEE_Result crypto_acipher_rsaes_decrypt(uint32_t algo, struct rsa_keypair *key, + const uint8_t *label, size_t label_len, + const uint8_t *cipher, + size_t cipher_len, uint8_t *msg, + size_t *msg_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_rsa *rsa = NULL; + struct drvcrypt_rsa_ed rsa_data = { }; + + if (!key || !msg || !cipher || !msg_len || (!label && label_len)) { + CRYPTO_TRACE("Parameters error (key @%p)\n" + "(msg @%p size %zu bytes)\n" + "(cipher @%p size %zu bytes)\n" + "(label @%p size %zu bytes)", + key, msg, msg_len ? *msg_len : 0, + cipher, cipher_len, label, label_len); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the encryption data parameters */ + if (algo == TEE_ALG_RSAES_PKCS1_V1_5) { + rsa_data.rsa_id = DRVCRYPT_RSA_PKCS_V1_5; + } else { + rsa_data.rsa_id = DRVCRYPT_RSA_OAEP; + rsa_data.hash_algo = TEE_INTERNAL_HASH_TO_ALGO(algo); + + ret = tee_alg_get_digest_size(rsa_data.hash_algo, + &rsa_data.digest_size); + if (ret != TEE_SUCCESS) + return ret; + + rsa_data.mgf = &drvcrypt_rsa_mgf1; + } + + rsa_data.key.key = key; + rsa_data.key.isprivate = true; + rsa_data.key.n_size = crypto_bignum_num_bytes(key->n); + + rsa_data.message.data = msg; + rsa_data.message.length = *msg_len; + rsa_data.cipher.data = (uint8_t *)cipher; + rsa_data.cipher.length = cipher_len; + rsa_data.label.data = + ((label_len > 0) ? (uint8_t *)label : NULL); + rsa_data.label.length = label_len; + rsa_data.algo = algo; + + ret = rsa->decrypt(&rsa_data); + + /* Set the message size */ + *msg_len = rsa_data.message.length; + } + + CRYPTO_TRACE("RSAES Decrypt ret = 0x%" PRIx32, ret); + + return ret; +} + +TEE_Result crypto_acipher_rsaes_encrypt(uint32_t algo, + struct rsa_public_key *key, + const uint8_t *label, size_t label_len, + const uint8_t *msg, size_t msg_len, + uint8_t *cipher, size_t *cipher_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct drvcrypt_rsa *rsa = NULL; + struct drvcrypt_rsa_ed rsa_data = { }; + + if (!key || !msg || !cipher_len || (!label && label_len)) { + CRYPTO_TRACE("Parameters error (key @%p\n" + "(msg @%p size %zu bytes)\n" + "(cipher @%p size %zu bytes)\n" + "(label @%p size %zu bytes)", + key, msg, msg_len, + cipher, cipher_len ? *cipher_len : 0, + label, label_len); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa_data.key.key = key; + rsa_data.key.isprivate = false; + rsa_data.key.n_size = crypto_bignum_num_bytes(key->n); + + if (rsa_data.key.n_size > *cipher_len) { + CRYPTO_TRACE("Cipher length (%zu) too short expected %zu bytes", + *cipher_len, rsa_data.key.n_size); + *cipher_len = rsa_data.key.n_size; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!cipher) { + CRYPTO_TRACE("Parameter \"cipher\" reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the encryption data parameters */ + if (algo == TEE_ALG_RSAES_PKCS1_V1_5) { + rsa_data.rsa_id = DRVCRYPT_RSA_PKCS_V1_5; + + /* Message length <= (modulus_size - 11) */ + if (msg_len > rsa_data.key.n_size - 11) + return TEE_ERROR_BAD_PARAMETERS; + + } else { + rsa_data.rsa_id = DRVCRYPT_RSA_OAEP; + rsa_data.hash_algo = TEE_INTERNAL_HASH_TO_ALGO(algo); + + /* Message length <= (modulus_size - 2 * hLength - 2) */ + ret = tee_alg_get_digest_size(rsa_data.hash_algo, + &rsa_data.digest_size); + if (ret != TEE_SUCCESS) + return ret; + + if (2 * rsa_data.digest_size >= rsa_data.key.n_size - 2) + return TEE_ERROR_BAD_PARAMETERS; + + if (msg_len > + rsa_data.key.n_size - 2 * rsa_data.digest_size - 2) + return TEE_ERROR_BAD_PARAMETERS; + + rsa_data.mgf = &drvcrypt_rsa_mgf1; + } + + rsa_data.message.data = (uint8_t *)msg; + rsa_data.message.length = msg_len; + rsa_data.cipher.data = cipher; + rsa_data.cipher.length = rsa_data.key.n_size; + rsa_data.label.data = (label_len > 0) ? (uint8_t *)label : NULL; + rsa_data.label.length = label_len; + rsa_data.algo = algo; + + ret = rsa->encrypt(&rsa_data); + + /* Set the cipher size */ + *cipher_len = rsa_data.cipher.length; + } + + CRYPTO_TRACE("RSAES Encrypt ret = 0x%" PRIx32, ret); + + return ret; +} + +TEE_Result crypto_acipher_rsassa_sign(uint32_t algo, struct rsa_keypair *key, + int salt_len, const uint8_t *msg, + size_t msg_len, uint8_t *sig, + size_t *sig_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_rsa *rsa = NULL; + struct drvcrypt_rsa_ssa rsa_ssa = { }; + + if (!key || !msg || !sig_len) { + CRYPTO_TRACE("Input parameters reference error"); + return ret; + } + + if (algo != TEE_ALG_RSASSA_PKCS1_V1_5) { + /* Prepare the Digest */ + rsa_ssa.hash_algo = TEE_DIGEST_HASH_TO_ALGO(algo); + + /* Check if the message length is digest hash size */ + ret = tee_alg_get_digest_size(TEE_DIGEST_HASH_TO_ALGO(algo), + &rsa_ssa.digest_size); + if (ret != TEE_SUCCESS) + return ret; + + if (msg_len != rsa_ssa.digest_size) { + CRYPTO_TRACE("Msg length (%zu expected %zu)", msg_len, + rsa_ssa.digest_size); + return TEE_ERROR_BAD_PARAMETERS; + } + } else { + rsa_ssa.hash_algo = 0; + rsa_ssa.digest_size = 0; + } + + /* Prepare the Key */ + rsa_ssa.key.key = key; + rsa_ssa.key.isprivate = true; + rsa_ssa.key.n_size = crypto_bignum_num_bytes(key->n); + + if (rsa_ssa.key.n_size > *sig_len) { + CRYPTO_TRACE("Sign length (%zu) too short must be %zu bytes", + *sig_len, rsa_ssa.key.n_size); + *sig_len = rsa_ssa.key.n_size; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!sig) { + CRYPTO_TRACE("Parameter \"sig\" reference error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the Encoded Signature structure data */ + rsa_ssa.algo = algo; + rsa_ssa.message.data = (uint8_t *)msg; + rsa_ssa.message.length = msg_len; + rsa_ssa.signature.data = (uint8_t *)sig; + rsa_ssa.signature.length = rsa_ssa.key.n_size; + rsa_ssa.salt_len = salt_len; + rsa_ssa.mgf = &drvcrypt_rsa_mgf1; + + ret = TEE_ERROR_NOT_IMPLEMENTED; + if (rsa->optional.ssa_sign) + ret = rsa->optional.ssa_sign(&rsa_ssa); + + if (ret == TEE_ERROR_NOT_IMPLEMENTED) + ret = drvcrypt_rsassa_sign(&rsa_ssa); + + /* Set the signature length */ + *sig_len = rsa_ssa.signature.length; + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Encode signature algo (0x%" PRIx32 + ") returned 0x%" PRIx32, + algo, ret); + return ret; +} + +TEE_Result crypto_acipher_rsassa_verify(uint32_t algo, + struct rsa_public_key *key, + int salt_len, const uint8_t *msg, + size_t msg_len, const uint8_t *sig, + size_t sig_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_rsa *rsa = NULL; + struct drvcrypt_rsa_ssa rsa_ssa = { }; + + if (!key || !msg || !sig) { + CRYPTO_TRACE("Input parameters reference error"); + goto out; + } + + if (algo != TEE_ALG_RSASSA_PKCS1_V1_5) { + /* Prepare the Digest */ + rsa_ssa.hash_algo = TEE_DIGEST_HASH_TO_ALGO(algo); + + /* Check if the message length is digest hash size */ + ret = tee_alg_get_digest_size(TEE_DIGEST_HASH_TO_ALGO(algo), + &rsa_ssa.digest_size); + if (ret != TEE_SUCCESS) + goto out; + + if (msg_len != rsa_ssa.digest_size) { + CRYPTO_TRACE("Input msg length (%zu expected %zu)", + msg_len, rsa_ssa.digest_size); + ret = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + } else { + rsa_ssa.hash_algo = 0; + rsa_ssa.digest_size = 0; + } + + /* Prepare the Key */ + rsa_ssa.key.key = key; + rsa_ssa.key.isprivate = false; + rsa_ssa.key.n_size = crypto_bignum_num_bytes(key->n); + + if (rsa_ssa.key.n_size > sig_len) { + CRYPTO_TRACE("Signature length expected %zu", + rsa_ssa.key.n_size); + ret = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the Encoded Signature structure data */ + rsa_ssa.algo = algo; + rsa_ssa.message.data = (uint8_t *)msg; + rsa_ssa.message.length = msg_len; + rsa_ssa.signature.data = (uint8_t *)sig; + rsa_ssa.signature.length = sig_len; + rsa_ssa.salt_len = salt_len; + rsa_ssa.mgf = &drvcrypt_rsa_mgf1; + + ret = TEE_ERROR_NOT_IMPLEMENTED; + if (rsa->optional.ssa_verify) + ret = rsa->optional.ssa_verify(&rsa_ssa); + + if (ret == TEE_ERROR_NOT_IMPLEMENTED) + ret = drvcrypt_rsassa_verify(&rsa_ssa); + + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + CRYPTO_TRACE("Signature verif algo (0x%" PRIx32 ") returned 0x%" PRIx32, + algo, ret); + +out: + FTMN_CALLEE_DONE(ret); + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/rsamgf.c b/optee_os/core/drivers/crypto/crypto_api/acipher/rsamgf.c new file mode 100644 index 0000000..7b559a6 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/rsamgf.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * RSA Mask Generation function implementation. + */ +#include +#include +#include +#include + +#include "local.h" + +TEE_Result drvcrypt_rsa_mgf1(struct drvcrypt_rsa_mgf *mgf_data) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + void *ctx = NULL; + size_t lastBlock_size = 0; + size_t nbBlock = 0; + uint32_t counter = 0; + uint32_t swapcount = 0; + uint8_t *cur_mask = mgf_data->mask.data; + uint8_t *tmpdigest = NULL; + + CRYPTO_TRACE("Generate Mask (%zu bytes) with seed of %zu bytes", + mgf_data->mask.length, mgf_data->seed.length); + + /* Calculate the number of complet hash digest */ + lastBlock_size = mgf_data->mask.length % mgf_data->digest_size; + if (lastBlock_size) { + /* Allocate a digest buffer for the last block */ + tmpdigest = calloc(1, mgf_data->digest_size); + if (!tmpdigest) + return TEE_ERROR_OUT_OF_MEMORY; + } + + /* Allocate the Hash Context */ + ret = crypto_hash_alloc_ctx(&ctx, mgf_data->hash_algo); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + nbBlock = (mgf_data->mask.length - lastBlock_size) / + mgf_data->digest_size; + + CRYPTO_TRACE("Nb Loop (%zu bytes) = %zu, last Block = %zu bytes", + mgf_data->digest_size, nbBlock, lastBlock_size); + + for (; counter < nbBlock; + counter++, cur_mask += mgf_data->digest_size) { + swapcount = TEE_U32_TO_BIG_ENDIAN(counter); + + ret = crypto_hash_init(ctx); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + ret = crypto_hash_update(ctx, mgf_data->seed.data, + mgf_data->seed.length); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + ret = crypto_hash_update(ctx, (uint8_t *)&swapcount, + sizeof(swapcount)); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + ret = crypto_hash_final(ctx, cur_mask, mgf_data->digest_size); + if (ret != TEE_SUCCESS) + goto exit_mgf; + } + + if (lastBlock_size) { + CRYPTO_TRACE("Last Block = %zu bytes", lastBlock_size); + + swapcount = TEE_U32_TO_BIG_ENDIAN(counter); + + ret = crypto_hash_init(ctx); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + ret = crypto_hash_update(ctx, mgf_data->seed.data, + mgf_data->seed.length); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + ret = crypto_hash_update(ctx, (uint8_t *)&swapcount, + sizeof(swapcount)); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + ret = crypto_hash_final(ctx, tmpdigest, mgf_data->digest_size); + if (ret != TEE_SUCCESS) + goto exit_mgf; + + memcpy(cur_mask, tmpdigest, lastBlock_size); + } + + ret = TEE_SUCCESS; + +exit_mgf: + crypto_hash_free_ctx(ctx); + free(tmpdigest); + + CRYPTO_TRACE("return 0x%08" PRIx32, ret); + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/rsassa.c b/optee_os/core/drivers/crypto/crypto_api/acipher/rsassa.c new file mode 100644 index 0000000..c37670c --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/rsassa.c @@ -0,0 +1,932 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * RSA Signature Software common implementation. + * Functions preparing and/or verifying the signature + * encoded string. + * + * PKCS #1 v2.1: RSA Cryptography Standard + * https://www.ietf.org/rfc/rfc3447.txt + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +/* + * PKCS#1 V1.5 - Encode the message in Distinguished Encoding Rules + * (DER) format. + * Refer to EMSA-PKCS1-v1_5 chapter of the PKCS#1 v2.1 + * + * @ssa_data RSA data to encode + * @EM [out] Encoded Message + */ +static TEE_Result emsa_pkcs1_v1_5_encode(struct drvcrypt_rsa_ssa *ssa_data, + struct drvcrypt_buf *EM) +{ + const struct drvcrypt_oid *hash_oid = NULL; + size_t ps_size = 0; + uint8_t *buf = NULL; + + hash_oid = drvcrypt_get_alg_hash_oid(ssa_data->hash_algo); + if (!hash_oid) + return TEE_ERROR_NOT_SUPPORTED; + + /* + * Calculate the PS size + * EM Size (modulus size) - 3 bytes - DigestInfo DER format size + */ + ps_size = ssa_data->key.n_size - 3; + ps_size -= ssa_data->digest_size; + ps_size -= 10 + hash_oid->asn1_length; + + CRYPTO_TRACE("PS size = %zu (n %zu)", ps_size, ssa_data->key.n_size); + + /* + * EM = 0x00 || 0x01 || PS || 0x00 || T + * + * where T represent the message DigestInfo in DER: + * DigestInfo ::= SEQUENCE { + * digestAlgorithm AlgorithmIdentifier, + * digest OCTET STRING + * } + * + * T Length = digest length + oid length + * EM Length = T Length + 11 + PS Length + */ + buf = EM->data; + + /* Set the EM first byte to 0x00 */ + *buf++ = 0x00; + + /* Set the EM second byte to 0x01 */ + *buf++ = 0x01; + + /* Fill PS with 0xFF */ + memset(buf, UINT8_MAX, ps_size); + buf += ps_size; + + /* Set the Byte after PS to 0x00 */ + *buf++ = 0x00; + + /* + * Create the DigestInfo DER Sequence + * + * DigestInfo ::= SEQUENCE { + * digestAlgorithm AlgorithmIdentifier, + * digest OCTET STRING + * } + * + */ + /* SEQUENCE { */ + *buf++ = DRVCRYPT_ASN1_SEQUENCE | DRVCRYPT_ASN1_CONSTRUCTED; + *buf++ = 0x08 + hash_oid->asn1_length + ssa_data->digest_size; + + /* digestAlgorithm AlgorithmIdentifier */ + *buf++ = DRVCRYPT_ASN1_SEQUENCE | DRVCRYPT_ASN1_CONSTRUCTED; + *buf++ = 0x04 + hash_oid->asn1_length; + *buf++ = DRVCRYPT_ASN1_OID; + *buf++ = hash_oid->asn1_length; + + /* digest OCTET STRING */ + memcpy(buf, hash_oid->asn1, hash_oid->asn1_length); + buf += hash_oid->asn1_length; + *buf++ = DRVCRYPT_ASN1_NULL; + *buf++ = 0x00; + *buf++ = DRVCRYPT_ASN1_OCTET_STRING; + *buf++ = ssa_data->digest_size; + /* } */ + + memcpy(buf, ssa_data->message.data, ssa_data->digest_size); + + CRYPTO_DUMPBUF("Encoded Message", EM->data, (size_t)EM->length); + + return TEE_SUCCESS; +} + +/* + * PKCS#1 V1.5 - Encode the message in Distinguished Encoding Rules + * (DER) format. + * Refer to EMSA-PKCS1-v1_5 chapter of the PKCS#1 v2.1 + * + * @ssa_data RSA data to encode + * @EM [out] Encoded Message + */ +static TEE_Result +emsa_pkcs1_v1_5_encode_noasn1(struct drvcrypt_rsa_ssa *ssa_data, + struct drvcrypt_buf *EM) +{ + size_t ps_size = 0; + uint8_t *buf = NULL; + + /* + * Calculate the PS size + * EM Size (modulus size) - 3 bytes - Message Length + */ + ps_size = ssa_data->key.n_size - 3; + + if (ps_size < ssa_data->message.length) + return TEE_ERROR_BAD_PARAMETERS; + + ps_size -= ssa_data->message.length; + + CRYPTO_TRACE("PS size = %zu (n %zu)", ps_size, ssa_data->key.n_size); + + /* + * EM = 0x00 || 0x01 || PS || 0x00 || T + * + * T Length = message length + * EM Length = T Length + PS Length + */ + buf = EM->data; + + /* Set the EM first byte to 0x00 */ + *buf++ = 0x00; + + /* Set the EM second byte to 0x01 */ + *buf++ = 0x01; + + /* Fill PS with 0xFF */ + memset(buf, UINT8_MAX, ps_size); + buf += ps_size; + + /* Set the Byte after PS to 0x00 */ + *buf++ = 0x00; + + memcpy(buf, ssa_data->message.data, ssa_data->message.length); + + CRYPTO_DUMPBUF("Encoded Message", EM->data, EM->length); + + return TEE_SUCCESS; +} + +/* + * PKCS#1 V1.5 - Signature of RSA message and encodes the signature. + * Refer to RSASSA-PKCS1-v1_5 chapter of the PKCS#1 v2.1 + * + * @ssa_data [in/out] RSA data to sign / Signature + */ +static TEE_Result rsassa_pkcs1_v1_5_sign(struct drvcrypt_rsa_ssa *ssa_data) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_buf EM = { }; + struct drvcrypt_rsa_ed rsa_data = { }; + struct drvcrypt_rsa *rsa = NULL; + + EM.length = ssa_data->key.n_size; + EM.data = malloc(EM.length); + if (!EM.data) + return TEE_ERROR_OUT_OF_MEMORY; + + /* Encode the Message */ + if (ssa_data->algo != TEE_ALG_RSASSA_PKCS1_V1_5) + ret = emsa_pkcs1_v1_5_encode(ssa_data, &EM); + else + ret = emsa_pkcs1_v1_5_encode_noasn1(ssa_data, &EM); + + if (ret != TEE_SUCCESS) + goto out; + + /* + * RSA Encrypt/Decrypt are doing the same operation except + * that decrypt takes a RSA Private key in parameter + */ + rsa_data.key.key = ssa_data->key.key; + rsa_data.key.isprivate = true; + rsa_data.key.n_size = ssa_data->key.n_size; + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (!rsa) { + ret = TEE_ERROR_NOT_IMPLEMENTED; + goto out; + } + + /* Prepare the decryption data parameters */ + rsa_data.rsa_id = DRVCRYPT_RSASSA_PKCS_V1_5; + rsa_data.message.data = ssa_data->signature.data; + rsa_data.message.length = ssa_data->signature.length; + rsa_data.cipher.data = EM.data; + rsa_data.cipher.length = EM.length; + rsa_data.hash_algo = ssa_data->hash_algo; + rsa_data.algo = ssa_data->algo; + + ret = rsa->decrypt(&rsa_data); + + /* Set the message decrypted size */ + ssa_data->signature.length = rsa_data.message.length; + +out: + free(EM.data); + + return ret; +} + +/* + * PKCS#1 V1.5 - Verification of the RSA message's signature. + * Refer to RSASSA-PKCS1-v1_5 chapter of the PKCS#1 v2.1 + * + * @ssa_data [int/out] RSA data signed and encoded signature + */ +static TEE_Result rsassa_pkcs1_v1_5_verify(struct drvcrypt_rsa_ssa *ssa_data) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + struct drvcrypt_buf EM = { }; + struct drvcrypt_buf EM_gen = { }; + struct drvcrypt_rsa_ed rsa_data = { }; + struct drvcrypt_rsa *rsa = NULL; + + EM.length = ssa_data->key.n_size; + EM.data = malloc(EM.length); + + if (!EM.data) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto end_verify; + } + + EM_gen.length = ssa_data->key.n_size; + EM_gen.data = malloc(EM.length); + + if (!EM_gen.data) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto end_verify; + } + + /* + * RSA Encrypt/Decrypt are doing the same operation except + * that the encrypt takes a RSA Public key in parameter + */ + rsa_data.key.key = ssa_data->key.key; + rsa_data.key.isprivate = false; + rsa_data.key.n_size = ssa_data->key.n_size; + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the encryption data parameters */ + rsa_data.rsa_id = DRVCRYPT_RSASSA_PKCS_V1_5; + rsa_data.message.data = ssa_data->signature.data; + rsa_data.message.length = ssa_data->signature.length; + rsa_data.cipher.data = EM.data; + rsa_data.cipher.length = EM.length; + rsa_data.hash_algo = ssa_data->hash_algo; + ret = rsa->encrypt(&rsa_data); + + /* Set the cipher size */ + EM.length = rsa_data.cipher.length; + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + + if (ret != TEE_SUCCESS) + goto end_verify; + + /* Encode the Message */ + if (ssa_data->algo != TEE_ALG_RSASSA_PKCS1_V1_5) + ret = emsa_pkcs1_v1_5_encode(ssa_data, &EM_gen); + else + ret = emsa_pkcs1_v1_5_encode_noasn1(ssa_data, &EM_gen); + + if (ret != TEE_SUCCESS) + goto end_verify; + + /* Check if EM decrypted and EM re-generated are identical */ + ret = TEE_ERROR_SIGNATURE_INVALID; + if (EM.length == EM_gen.length) { + if (!memcmp(EM.data, EM_gen.data, EM.length)) + ret = TEE_SUCCESS; + } + +end_verify: + free(EM.data); + free(EM_gen.data); + + return ret; +} + +/* + * PSS - Encode the message using a Probabilistic Signature Scheme (PSS) + * Refer to EMSA-PSS (encoding) chapter of the PKCS#1 v2.1 + * + * @ssa_data RSA data to encode + * @emBits EM size in bits + * @EM [out] Encoded Message + */ +static TEE_Result emsa_pss_encode(struct drvcrypt_rsa_ssa *ssa_data, + size_t emBits, struct drvcrypt_buf *EM) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct drvcrypt_rsa_mgf mgf_data = { }; + struct drvcrypt_buf hash = { }; + struct drvcrypt_buf dbMask = { }; + struct drvcrypt_buf DB = { }; + size_t db_size = 0; + size_t ps_size = 0; + size_t msg_size = 0; + uint8_t *buf = NULL; + uint8_t *msg_db = NULL; + uint8_t *salt = NULL; + struct drvcrypt_mod_op mod_op = { }; + + /* + * Build EM = maskedDB || H || 0xbc + * + * where + * maskedDB = DB xor dbMask + * DB = PS || 0x01 || salt + * dbMask = MGF(H, emLen - hLen - 1) + * + * H = Hash(M') + * M' = (0x)00 00 00 00 00 00 00 00 || mHash || salt + * + * PS size = emLen - sLen - hLen - 2 (may be = 0) + */ + + /* + * Calculate the M' and DB size to allocate a temporary buffer + * used for both object + */ + ps_size = EM->length - ssa_data->digest_size - ssa_data->salt_len - 2; + db_size = EM->length - ssa_data->digest_size - 1; + msg_size = 8 + ssa_data->digest_size + ssa_data->salt_len; + + CRYPTO_TRACE("PS Len = %zu, DB Len = %zu, M' Len = %zu", ps_size, + db_size, msg_size); + + msg_db = malloc(MAX(db_size, msg_size)); + if (!msg_db) + return TEE_ERROR_OUT_OF_MEMORY; + + if (ssa_data->salt_len) { + salt = malloc(ssa_data->salt_len); + + if (!salt) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto end_pss_encode; + } + } + + /* + * Step 4 and 5 + * Generate the M' = (0x)00 00 00 00 00 00 00 00 || mHash || salt + * + * where + * mHash is the input message (already hash) + * salt is a random number of salt_len (input data) can be empty + */ + buf = msg_db; + + memset(buf, 0, 8); + buf += 8; + + memcpy(buf, ssa_data->message.data, ssa_data->message.length); + buf += ssa_data->message.length; + + /* Get salt random number if salt length not 0 */ + if (ssa_data->salt_len) { + ret = crypto_rng_read(salt, ssa_data->salt_len); + CRYPTO_TRACE("Get salt of %zu bytes (ret = 0x%08" PRIx32 ")", + ssa_data->salt_len, ret); + if (ret != TEE_SUCCESS) + goto end_pss_encode; + + memcpy(buf, salt, ssa_data->salt_len); + } + + /* + * Step 6 + * Hash the M' generated new message + * H = hash(M') + */ + hash.data = &EM->data[db_size]; + hash.length = ssa_data->digest_size; + + ret = tee_hash_createdigest(ssa_data->hash_algo, msg_db, msg_size, + hash.data, hash.length); + + CRYPTO_TRACE("H = hash(M') returned 0x%08" PRIx32, ret); + if (ret != TEE_SUCCESS) + goto end_pss_encode; + + CRYPTO_DUMPBUF("H = hash(M')", hash.data, hash.length); + + /* + * Step 7 and 8 + * DB = PS || 0x01 || salt + */ + buf = msg_db; + if (ps_size) + memset(buf, 0, ps_size); + buf += ps_size; + *buf++ = 0x01; + + if (ssa_data->salt_len) + memcpy(buf, salt, ssa_data->salt_len); + + DB.data = msg_db; + DB.length = db_size; + + CRYPTO_DUMPBUF("DB", DB.data, DB.length); + + /* + * Step 9 + * Generate a Mask of the seed value + * dbMask = MGF(H, emLen - hLen - 1) + * + * Note: Will use the same buffer for the dbMask and maskedDB + * maskedDB is in the EM output + */ + dbMask.data = EM->data; + dbMask.length = db_size; + + mgf_data.hash_algo = ssa_data->hash_algo; + mgf_data.digest_size = ssa_data->digest_size; + mgf_data.seed.data = hash.data; + mgf_data.seed.length = hash.length; + mgf_data.mask.data = dbMask.data; + mgf_data.mask.length = dbMask.length; + ret = ssa_data->mgf(&mgf_data); + + CRYPTO_TRACE("dbMask = MGF(H, emLen - hLen - 1) returned 0x%08" PRIx32, + ret); + if (ret != TEE_SUCCESS) + goto end_pss_encode; + + CRYPTO_DUMPBUF("dbMask", dbMask.data, dbMask.length); + + /* + * Step 10 + * maskedDB = DB xor dbMask + */ + mod_op.n.length = dbMask.length; + mod_op.a.data = DB.data; + mod_op.a.length = DB.length; + mod_op.b.data = dbMask.data; + mod_op.b.length = dbMask.length; + mod_op.result.data = dbMask.data; + mod_op.result.length = dbMask.length; + + ret = drvcrypt_xor_mod_n(&mod_op); + CRYPTO_TRACE("maskedDB = DB xor dbMask returned 0x%08" PRIx32, ret); + if (ret != TEE_SUCCESS) + goto end_pss_encode; + + CRYPTO_DUMPBUF("maskedDB", dbMask.data, dbMask.length); + + /* + * Step 11 + * Set the leftmost 8emLen - emBits of the leftmost octet + * in maskedDB to 0' + */ + EM->data[0] &= (UINT8_MAX >> ((EM->length * 8) - emBits)); + + /* + * Step 12 + * EM = maskedDB || H || 0xbc + */ + EM->data[EM->length - 1] = 0xbc; + + CRYPTO_DUMPBUF("EM", EM->data, EM->length); + + ret = TEE_SUCCESS; +end_pss_encode: + free(msg_db); + free(salt); + + return ret; +} + +/* + * PSS - Verify the message using a Probabilistic Signature Scheme (PSS) + * Refer to EMSA-PSS (verification) chapter of the PKCS#1 v2.1 + * + * @ssa_data RSA data to encode + * @emBits EM size in bits + * @EM [out] Encoded Message + */ +static TEE_Result emsa_pss_verify(struct drvcrypt_rsa_ssa *ssa_data, + size_t emBits, struct drvcrypt_buf *EM) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct drvcrypt_rsa_mgf mgf_data = { }; + struct drvcrypt_buf hash = { }; + struct drvcrypt_buf hash_gen = { }; + size_t db_size = 0; + size_t ps_size = 0; + size_t msg_size = 0; + uint8_t *msg_db = NULL; + uint8_t *salt = NULL; + uint8_t *buf = NULL; + struct drvcrypt_mod_op mod_op = { }; + + /* + * EM = maskedDB || H || 0xbc + * + * where + * maskedDB = DB xor dbMask + * DB = PS || 0x01 || salt + * dbMask = MGF(H, emLen - hLen - 1) + * + * H = Hash(M') + * M' = (0x)00 00 00 00 00 00 00 00 || mHash || salt + * + * PS size = emLen - sLen - hLen - 2 (may be = 0) + */ + + /* + * Calculate the M' and DB size to allocate a temporary buffer + * used for both object + */ + ps_size = EM->length - ssa_data->digest_size - ssa_data->salt_len - 2; + db_size = EM->length - ssa_data->digest_size - 1; + msg_size = 8 + ssa_data->digest_size + ssa_data->salt_len; + + CRYPTO_TRACE("PS Len = %zu, DB Len = %zu, M' Len = %zu", ps_size, + db_size, msg_size); + + msg_db = malloc(MAX(db_size, msg_size)); + if (!msg_db) + return TEE_ERROR_OUT_OF_MEMORY; + + /* + * Step 4 + * Check if rightmost octet of EM is 0xbc + */ + if (EM->data[EM->length - 1] != 0xbc) { + CRYPTO_TRACE("rigthmost octet != 0xbc (0x%" PRIX8 ")", + EM->data[EM->length - 1]); + ret = TEE_ERROR_SIGNATURE_INVALID; + goto end_pss_verify; + } + + /* + * Step 6 + * Check if the leftmost 8emLen - emBits of the leftmost octet + * in maskedDB are 0's + */ + if (EM->data[0] & ~(UINT8_MAX >> (EM->length * 8 - emBits))) { + CRYPTO_TRACE("Error leftmost octet maskedDB not 0's"); + CRYPTO_TRACE("EM[0] = 0x%" PRIX8 + " - EM Len = %zu, emBits = %zu", + EM->data[0], EM->length, emBits); + ret = TEE_ERROR_SIGNATURE_INVALID; + goto end_pss_verify; + } + + hash.data = &EM->data[db_size]; + hash.length = ssa_data->digest_size; + + /* + * Step 7 + * dbMask = MGF(H, emLen - hLen - 1) + * + * Note: Will use the same buffer for the dbMask and DB + */ + mgf_data.hash_algo = ssa_data->hash_algo; + mgf_data.digest_size = ssa_data->digest_size; + mgf_data.seed.data = hash.data; + mgf_data.seed.length = hash.length; + mgf_data.mask.data = msg_db; + mgf_data.mask.length = db_size; + ret = ssa_data->mgf(&mgf_data); + + CRYPTO_TRACE("dbMask = MGF(H, emLen - hLen - 1) returned 0x%08" PRIx32, + ret); + if (ret != TEE_SUCCESS) + goto end_pss_verify; + + CRYPTO_DUMPBUF("dbMask", msg_db, db_size); + + /* + * Step 8 + * DB = maskedDB xor dbMask + * + * + * Note: maskedDB is in the EM input + */ + mod_op.n.length = db_size; + mod_op.a.data = EM->data; + mod_op.a.length = db_size; + mod_op.b.data = msg_db; + mod_op.b.length = db_size; + mod_op.result.data = msg_db; + mod_op.result.length = db_size; + + ret = drvcrypt_xor_mod_n(&mod_op); + CRYPTO_TRACE("DB = maskedDB xor dbMask returned 0x%08" PRIx32, ret); + if (ret != TEE_SUCCESS) + goto end_pss_verify; + + /* + * Step 9 + * Set the leftmost 8emLen - emBits of the leftmost octet in + * DB to zero + */ + *msg_db &= UINT8_MAX >> (EM->length * 8 - emBits); + + CRYPTO_DUMPBUF("DB", msg_db, db_size); + + /* + * Step 10 + * Expected to have + * DB = PS || 0x01 || salt + * + * PS must be 0 + * PS size = emLen - sLen - hLen - 2 (may be = 0) + */ + buf = msg_db; + while (buf < msg_db + ps_size) { + if (*buf++ != 0) { + ret = TEE_ERROR_SIGNATURE_INVALID; + goto end_pss_verify; + } + } + + if (*buf++ != 0x01) { + ret = TEE_ERROR_SIGNATURE_INVALID; + goto end_pss_verify; + } + + /* + * Step 11 + * Get the salt value + */ + if (ssa_data->salt_len) { + salt = malloc(ssa_data->salt_len); + if (!salt) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto end_pss_verify; + } + + memcpy(salt, buf, ssa_data->salt_len); + } + + /* + * Step 12 + * Generate the M' = (0x)00 00 00 00 00 00 00 00 || mHash || salt + * + * where + * mHash is the input message (already hash) + * salt is a random number of salt_len (input data) can be empty + */ + buf = msg_db; + + memset(buf, 0, 8); + buf += 8; + + memcpy(buf, ssa_data->message.data, ssa_data->message.length); + buf += ssa_data->message.length; + + if (ssa_data->salt_len) + memcpy(buf, salt, ssa_data->salt_len); + + /* + * Step 13 + * Hash the M' generated new message + * H' = hash(M') + * + * Note: reuse the msg_db buffer as Hash result + */ + hash_gen.data = msg_db; + hash_gen.length = ssa_data->digest_size; + + ret = tee_hash_createdigest(ssa_data->hash_algo, msg_db, msg_size, + hash_gen.data, hash_gen.length); + + CRYPTO_TRACE("H' = hash(M') returned 0x%08" PRIx32, ret); + if (ret != TEE_SUCCESS) + goto end_pss_verify; + + CRYPTO_DUMPBUF("H' = hash(M')", hash_gen.data, hash_gen.length); + + /* + * Step 14 + * Compare H and H' + */ + ret = TEE_ERROR_SIGNATURE_INVALID; + if (!memcmp(hash_gen.data, hash.data, hash_gen.length)) + ret = TEE_SUCCESS; + +end_pss_verify: + free(msg_db); + free(salt); + + return ret; +} + +/* + * PSS - Signature of RSA message and encodes the signature. + * Refer to RSASSA-PSS chapter of the PKCS#1 v2.1 + * + * @ssa_data [in/out] RSA data to sign / Signature + */ +static TEE_Result rsassa_pss_sign(struct drvcrypt_rsa_ssa *ssa_data) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct rsa_keypair *key = NULL; + struct drvcrypt_buf EM = { }; + size_t modBits = 0; + struct drvcrypt_rsa_ed rsa_data = { }; + struct drvcrypt_rsa *rsa = NULL; + + key = ssa_data->key.key; + + /* Get modulus length in bits */ + modBits = crypto_bignum_num_bits(key->n); + if (modBits <= 0) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * EM Length = (modBits - 1) / 8 + * if (modBits - 1) is not divisible by 8, one more byte is needed + */ + modBits--; + EM.length = ROUNDUP(modBits, 8) / 8; + + if (EM.length < ssa_data->digest_size + ssa_data->salt_len + 2) + return TEE_ERROR_BAD_PARAMETERS; + + EM.data = malloc(EM.length); + if (!EM.data) + return TEE_ERROR_OUT_OF_MEMORY; + + CRYPTO_TRACE("modBits = %zu, hence EM Length = %zu", modBits + 1, + EM.length); + + /* Encode the Message */ + ret = emsa_pss_encode(ssa_data, modBits, &EM); + CRYPTO_TRACE("EMSA PSS Encode returned 0x%08" PRIx32, ret); + + /* + * RSA Encrypt/Decrypt are doing the same operation + * expect that the decrypt takes a RSA Private key in parameter + */ + if (ret == TEE_SUCCESS) { + rsa_data.key.key = ssa_data->key.key; + rsa_data.key.isprivate = true; + rsa_data.key.n_size = ssa_data->key.n_size; + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the decryption data parameters */ + rsa_data.rsa_id = DRVCRYPT_RSASSA_PSS; + rsa_data.message.data = ssa_data->signature.data; + rsa_data.message.length = ssa_data->signature.length; + rsa_data.cipher.data = EM.data; + rsa_data.cipher.length = EM.length; + rsa_data.algo = ssa_data->algo; + + ret = rsa->decrypt(&rsa_data); + + /* Set the message decrypted size */ + ssa_data->signature.length = rsa_data.message.length; + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + } + } + free(EM.data); + + return ret; +} + +/* + * PSS - Signature verification of RSA message. + * Refer to RSASSA-PSS chapter of the PKCS#1 v2.1 + * + * @ssa_data [in/out] RSA Signature vs. message to verify + */ +static TEE_Result rsassa_pss_verify(struct drvcrypt_rsa_ssa *ssa_data) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct rsa_public_key *key = NULL; + struct drvcrypt_buf EM = { }; + size_t modBits = 0; + struct drvcrypt_rsa_ed rsa_data = { }; + struct drvcrypt_rsa *rsa = NULL; + + key = ssa_data->key.key; + + /* Get modulus length in bits */ + modBits = crypto_bignum_num_bits(key->n); + if (modBits <= 0) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * EM Length = (modBits - 1) / 8 + * if (modBits - 1) is not divisible by 8, one more byte is needed + */ + modBits--; + EM.length = ROUNDUP(modBits, 8) / 8; + + if (EM.length < ssa_data->digest_size + ssa_data->salt_len + 2) + return TEE_ERROR_BAD_PARAMETERS; + + EM.data = malloc(EM.length); + if (!EM.data) + return TEE_ERROR_OUT_OF_MEMORY; + + CRYPTO_TRACE("modBits = %zu, hence EM Length = %zu", modBits + 1, + EM.length); + + /* + * RSA Encrypt/Decrypt are doing the same operation + * expect that the encrypt takes a RSA Public key in parameter + */ + rsa_data.key.key = ssa_data->key.key; + rsa_data.key.isprivate = false; + rsa_data.key.n_size = ssa_data->key.n_size; + + rsa = drvcrypt_get_ops(CRYPTO_RSA); + if (rsa) { + /* Prepare the encryption data parameters */ + rsa_data.rsa_id = DRVCRYPT_RSASSA_PSS; + rsa_data.message.data = ssa_data->signature.data; + rsa_data.message.length = ssa_data->signature.length; + rsa_data.cipher.data = EM.data; + rsa_data.cipher.length = EM.length; + rsa_data.algo = ssa_data->algo; + + ret = rsa->encrypt(&rsa_data); + + /* Set the cipher size */ + EM.length = rsa_data.cipher.length; + } else { + ret = TEE_ERROR_NOT_IMPLEMENTED; + goto end_pss_verify; + } + + if (ret == TEE_SUCCESS) { + /* Verify the Message */ + ret = emsa_pss_verify(ssa_data, modBits, &EM); + CRYPTO_TRACE("EMSA PSS Verify returned 0x%08" PRIx32, ret); + } else { + CRYPTO_TRACE("RSA NO PAD returned 0x%08" PRIx32, ret); + ret = TEE_ERROR_SIGNATURE_INVALID; + } + +end_pss_verify: + free(EM.data); + + return ret; +} + +TEE_Result drvcrypt_rsassa_sign(struct drvcrypt_rsa_ssa *ssa_data) +{ + switch (ssa_data->algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5: + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + return rsassa_pkcs1_v1_5_sign(ssa_data); + + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + return rsassa_pss_sign(ssa_data); + + default: + break; + } + + return TEE_ERROR_BAD_PARAMETERS; +} + +TEE_Result drvcrypt_rsassa_verify(struct drvcrypt_rsa_ssa *ssa_data) +{ + switch (ssa_data->algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5: + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + return rsassa_pkcs1_v1_5_verify(ssa_data); + + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + return rsassa_pss_verify(ssa_data); + + default: + break; + } + + return TEE_ERROR_BAD_PARAMETERS; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/acipher/sub.mk b/optee_os/core/drivers/crypto/crypto_api/acipher/sub.mk new file mode 100644 index 0000000..c6a7378 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/acipher/sub.mk @@ -0,0 +1,4 @@ +srcs-$(CFG_CRYPTO_DRV_RSA) += rsa.c rsamgf.c rsassa.c +srcs-$(CFG_CRYPTO_DRV_ECC) += ecc.c +srcs-$(CFG_CRYPTO_DRV_DH) += dh.c +srcs-$(CFG_CRYPTO_DRV_DSA) += dsa.c diff --git a/optee_os/core/drivers/crypto/crypto_api/authenc/authenc.c b/optee_os/core/drivers/crypto/crypto_api/authenc/authenc.c new file mode 100644 index 0000000..6563e68 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/authenc/authenc.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + * + * Crypto authenc interface implementation to enable HW driver. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const struct crypto_authenc_ops authenc_ops; + +/* + * Returns the reference to the driver context + * + * @ctx Reference the API context pointer + */ +static struct crypto_authenc *to_authenc_ctx(struct crypto_authenc_ctx *ctx) +{ + assert(ctx && ctx->ops == &authenc_ops); + + return container_of(ctx, struct crypto_authenc, authenc_ctx); +} + +/* + * Free authenc context + * + * @ctx Reference the API context pointer + */ +static void authenc_free_ctx(struct crypto_authenc_ctx *ctx) +{ + struct crypto_authenc *authenc = to_authenc_ctx(ctx); + + if (authenc->op && authenc->op->free_ctx) + authenc->op->free_ctx(authenc->ctx); + + free(authenc); +} + +/* + * Copy authenc context + * + * @dst_ctx [out] Reference the API context pointer destination + * @src_ctx Reference the API context pointer source + */ +static void authenc_copy_state(struct crypto_authenc_ctx *dst_ctx, + struct crypto_authenc_ctx *src_ctx) +{ + struct crypto_authenc *authenc_src = to_authenc_ctx(src_ctx); + struct crypto_authenc *authenc_dst = to_authenc_ctx(dst_ctx); + + if (authenc_src->op && authenc_src->op->copy_state) + authenc_src->op->copy_state(authenc_dst->ctx, authenc_src->ctx); +} + +/* + * Initialization of the authenc operation + * + * @ctx Reference the API context pointer + * @mode Operation mode + * @key Key + * @key_len Length of the key + * @nonce Nonce + * @nonce_len Length of the nonce + * @tag_len Length of the requested tag + * @aad_len Length of the associated authenticated data + * @payload_len Length of payload + */ +static TEE_Result authenc_init(struct crypto_authenc_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key, + size_t key_len, const uint8_t *nonce, + size_t nonce_len, size_t tag_len, size_t aad_len, + size_t payload_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_authenc *authenc = to_authenc_ctx(ctx); + + if ((!key && key_len) || (!nonce && nonce_len)) { + CRYPTO_TRACE("One of the key is not correct"); + CRYPTO_TRACE("key @%p-%zu bytes", key, key_len); + CRYPTO_TRACE("nonce @%p-%zu bytes", nonce, nonce_len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (authenc->op && authenc->op->init) { + struct drvcrypt_authenc_init dinit = { + .ctx = authenc->ctx, + .encrypt = (mode == TEE_MODE_ENCRYPT), + .key.data = (uint8_t *)key, + .key.length = key_len, + .nonce.data = (uint8_t *)nonce, + .nonce.length = nonce_len, + .tag_len = tag_len, + .aad_len = aad_len, + .payload_len = payload_len, + }; + + ret = authenc->op->init(&dinit); + } + + CRYPTO_TRACE("authenc ret 0x%" PRIx32, ret); + return ret; +} + +/* + * Update Additional Authenticated Data part of the authenc operation + * + * @ctx Reference the API context pointer + * @data Data to authenticate without encrypt/decrypt (AAD) + * @len AAD length in bytes + */ +static TEE_Result authenc_update_aad(struct crypto_authenc_ctx *ctx, + const uint8_t *data, size_t len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_authenc *authenc = to_authenc_ctx(ctx); + + if (!data && len) { + CRYPTO_TRACE("Bad data @%p-%zu bytes", data, len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (authenc->op && authenc->op->update_aad) { + struct drvcrypt_authenc_update_aad dupdate = { + .ctx = authenc->ctx, + .aad.data = (uint8_t *)data, + .aad.length = len, + }; + + ret = authenc->op->update_aad(&dupdate); + } + + CRYPTO_TRACE("authenc ret 0x%" PRIx32, ret); + return ret; +} + +/* + * Update payload part of the authenc operation + * + * @ctx Reference the API context pointer + * @data Data to authenticate and encrypt/decrypt + * @len Length of the input data and output result + * @dst [out] Output data of the operation + */ +static TEE_Result authenc_update_payload(struct crypto_authenc_ctx *ctx, + TEE_OperationMode mode, + const uint8_t *data, + size_t len, uint8_t *dst) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_authenc *authenc = to_authenc_ctx(ctx); + + if (!dst) { + CRYPTO_TRACE("Destination buffer error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!data && len) { + CRYPTO_TRACE("Bad data @%p-%zu bytes", data, len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (authenc->op && authenc->op->update_payload) { + struct drvcrypt_authenc_update_payload dupdate = { + .ctx = authenc->ctx, + .encrypt = (mode == TEE_MODE_ENCRYPT), + .src.data = (uint8_t *)data, + .src.length = len, + .dst.data = dst, + .dst.length = len, + }; + + ret = authenc->op->update_payload(&dupdate); + } + + CRYPTO_TRACE("authenc ret 0x%" PRIx32, ret); + return ret; +} + +/* + * Last block for the authenc encrypt and get tag operation + * + * @ctx Reference the API context pointer + * @data Data to authenticate and encrypt (can be NULL) + * @len Length of the input data and output result (can be 0) + * @dst [out] Output data of the operation + * @tag [out] Output tag of the operation + * @tag_len [in/out] in: size of the dst_tag buffer + * out: size of the computed tag + */ +static TEE_Result authenc_enc_final(struct crypto_authenc_ctx *ctx, + const uint8_t *data, size_t len, + uint8_t *dst, uint8_t *tag, + size_t *tag_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_authenc *authenc = to_authenc_ctx(ctx); + + if (!dst && len) { + CRYPTO_TRACE("Bad output @%p-%zu bytes", dst, len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!data && len) { + CRYPTO_TRACE("Bad input @%p-%zu bytes", data, len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (authenc->op && authenc->op->enc_final) { + struct drvcrypt_authenc_final dfinal = { + .ctx = authenc->ctx, + .src.data = (uint8_t *)data, + .src.length = len, + .dst.data = dst, + .dst.length = len, + .tag.data = tag, + .tag.length = *tag_len + }; + + ret = authenc->op->enc_final(&dfinal); + if (ret == TEE_SUCCESS) + *tag_len = dfinal.tag.length; + } + + CRYPTO_TRACE("authenc ret 0x%" PRIx32, ret); + return ret; +} + +/* + * Last block for the authenc decrypt and check tag operation + * + * @ctx Reference the API context pointer + * @src_data Data to authenticate and encrypt (can be NULL) + * @len Length of the input data and output result (can be 0) + * @dst [out] Output data of the operation + * @tag Tag to check at end of operation + * @tag_len Length of @tag + */ +static TEE_Result authenc_dec_final(struct crypto_authenc_ctx *ctx, + const uint8_t *data, size_t len, + uint8_t *dst, const uint8_t *tag, + size_t tag_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_authenc *authenc = to_authenc_ctx(ctx); + + if (!dst && len) { + CRYPTO_TRACE("Bad output @%p-%zu bytes", dst, len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!data && len) { + CRYPTO_TRACE("Bad data @%p-%zu bytes", data, len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (authenc->op && authenc->op->dec_final) { + struct drvcrypt_authenc_final dfinal = { + .ctx = authenc->ctx, + .src.data = (uint8_t *)data, + .src.length = len, + .dst.data = dst, + .dst.length = len, + .tag.data = (uint8_t *)tag, + .tag.length = tag_len + }; + + ret = authenc->op->dec_final(&dfinal); + } + + CRYPTO_TRACE("authenc ret 0x%" PRIx32, ret); + return ret; +} + +/* + * Finalize the authenc operation + * + * @ctx Reference the API context pointer + */ +static void authenc_final(struct crypto_authenc_ctx *ctx) +{ + struct crypto_authenc *authenc = to_authenc_ctx(ctx); + + if (authenc->op && authenc->op->final) + authenc->op->final(authenc->ctx); +} + +static const struct crypto_authenc_ops authenc_ops = { + .init = authenc_init, + .update_aad = authenc_update_aad, + .update_payload = authenc_update_payload, + .enc_final = authenc_enc_final, + .dec_final = authenc_dec_final, + .final = authenc_final, + .free_ctx = authenc_free_ctx, + .copy_state = authenc_copy_state, +}; + +TEE_Result drvcrypt_authenc_alloc_ctx(struct crypto_authenc_ctx **ctx, + uint32_t algo) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_authenc *authenc = NULL; + + CRYPTO_TRACE("authenc alloc_ctx algo 0x%" PRIx32, algo); + + assert(ctx); + + authenc = calloc(1, sizeof(*authenc)); + if (!authenc) + return TEE_ERROR_OUT_OF_MEMORY; + + authenc->op = drvcrypt_get_ops(CRYPTO_AUTHENC); + if (authenc->op && authenc->op->alloc_ctx) + ret = authenc->op->alloc_ctx(&authenc->ctx, algo); + + if (ret != TEE_SUCCESS) { + free(authenc); + } else { + authenc->authenc_ctx.ops = &authenc_ops; + *ctx = &authenc->authenc_ctx; + } + + CRYPTO_TRACE("authenc alloc_ctx ret 0x%" PRIx32, ret); + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/authenc/sub.mk b/optee_os/core/drivers/crypto/crypto_api/authenc/sub.mk new file mode 100644 index 0000000..1065a67 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/authenc/sub.mk @@ -0,0 +1 @@ +srcs-y += authenc.c diff --git a/optee_os/core/drivers/crypto/crypto_api/cipher/cipher.c b/optee_os/core/drivers/crypto/crypto_api/cipher/cipher.c new file mode 100644 index 0000000..3d9d548 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/cipher/cipher.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * Crypto Cipher interface implementation to enable HW driver. + */ +#include +#include +#include +#include +#include +#include +#include + +static const struct crypto_cipher_ops cipher_ops; + +/* + * Returns the reference to the driver context + * + * @ctx Reference the API context pointer + */ +static struct crypto_cipher *to_cipher_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &cipher_ops); + + return container_of(ctx, struct crypto_cipher, cipher_ctx); +} + +/* + * Free cipher context + * + * @ctx Reference the API context pointer + */ +static void cipher_free_ctx(struct crypto_cipher_ctx *ctx) +{ + struct crypto_cipher *cipher = to_cipher_ctx(ctx); + + if (cipher->op && cipher->op->free_ctx) + cipher->op->free_ctx(cipher->ctx); + + free(cipher); +} + +/* + * Copy cipher context + * + * @dst_ctx [out] Reference the API context pointer destination + * @src_ctx Reference the API context pointer source + */ +static void cipher_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct crypto_cipher *cipher_src = to_cipher_ctx(src_ctx); + struct crypto_cipher *cipher_dst = to_cipher_ctx(dst_ctx); + + if (cipher_src->op && cipher_src->op->copy_state) + cipher_src->op->copy_state(cipher_dst->ctx, cipher_src->ctx); +} + +/* + * Initialization of the cipher operation + * + * @ctx Reference the API context pointer + * @mode Operation mode + * @key1 First Key + * @key1_len Length of the first key + * @key2 Second Key + * @key2_len Length of the second key + * @iv Initial Vector + * @iv_len Length of the IV + */ +static TEE_Result cipher_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2, + size_t key2_len, const uint8_t *iv, size_t iv_len) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_cipher *cipher = to_cipher_ctx(ctx); + + if ((!key1 && key1_len) || (!key2 && key2_len) || (!iv && iv_len)) { + CRYPTO_TRACE("One of the key is not correct"); + CRYPTO_TRACE("key1 @%p-%zu bytes", key1, key1_len); + CRYPTO_TRACE("key2 @%p-%zu bytes", key1, key1_len); + CRYPTO_TRACE("iv @%p-%zu bytes", iv, iv_len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (cipher->op && cipher->op->init) { + struct drvcrypt_cipher_init dinit = { + .ctx = cipher->ctx, + .encrypt = (mode == TEE_MODE_ENCRYPT), + .key1.data = (uint8_t *)key1, + .key1.length = key1_len, + .key2.data = (uint8_t *)key2, + .key2.length = key2_len, + .iv.data = (uint8_t *)iv, + .iv.length = iv_len, + }; + + ret = cipher->op->init(&dinit); + } + + CRYPTO_TRACE("cipher ret 0x%" PRIX32, ret); + return ret; +} + +/* + * Update of the cipher operation + * + * @ctx Reference the API context pointer + * @last_block True if last block to handle + * @data Data to encrypt/decrypt + * @len Length of the input data and output result + * @dst [out] Output data of the operation + */ +static TEE_Result cipher_update(struct crypto_cipher_ctx *ctx, bool last_block, + const uint8_t *data, size_t len, uint8_t *dst) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_cipher *cipher = to_cipher_ctx(ctx); + + if (!dst) { + CRYPTO_TRACE("Destination buffer error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!data && len) { + CRYPTO_TRACE("Bad data data @%p-%zu bytes", data, len); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (cipher->op && cipher->op->update) { + struct drvcrypt_cipher_update dupdate = { + .ctx = cipher->ctx, + .last = last_block, + .src.data = (uint8_t *)data, + .src.length = len, + .dst.data = dst, + .dst.length = len, + }; + + ret = cipher->op->update(&dupdate); + } + + CRYPTO_TRACE("cipher ret 0x%" PRIX32, ret); + return ret; +} + +/* + * Finalize the cipher operation + * + * @ctx Reference the API context pointer + */ +static void cipher_final(struct crypto_cipher_ctx *ctx) +{ + struct crypto_cipher *cipher = to_cipher_ctx(ctx); + + if (cipher->op && cipher->op->final) + cipher->op->final(cipher->ctx); +} + +static const struct crypto_cipher_ops cipher_ops = { + .init = cipher_init, + .update = cipher_update, + .final = cipher_final, + .free_ctx = cipher_free_ctx, + .copy_state = cipher_copy_state, +}; + +TEE_Result drvcrypt_cipher_alloc_ctx(struct crypto_cipher_ctx **ctx, + uint32_t algo) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + struct crypto_cipher *cipher = NULL; + + CRYPTO_TRACE("Cipher alloc_ctx algo 0x%" PRIX32, algo); + + assert(ctx); + + cipher = calloc(1, sizeof(*cipher)); + if (!cipher) + return TEE_ERROR_OUT_OF_MEMORY; + + cipher->op = drvcrypt_get_ops(CRYPTO_CIPHER); + if (cipher->op && cipher->op->alloc_ctx) + ret = cipher->op->alloc_ctx(&cipher->ctx, algo); + + if (ret != TEE_SUCCESS) { + free(cipher); + } else { + cipher->cipher_ctx.ops = &cipher_ops; + *ctx = &cipher->cipher_ctx; + } + + CRYPTO_TRACE("Cipher alloc_ctx ret 0x%" PRIX32, ret); + + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/cipher/sub.mk b/optee_os/core/drivers/crypto/crypto_api/cipher/sub.mk new file mode 100644 index 0000000..e17a654 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/cipher/sub.mk @@ -0,0 +1 @@ +srcs-y += cipher.c diff --git a/optee_os/core/drivers/crypto/crypto_api/drvcrypt.c b/optee_os/core/drivers/crypto/crypto_api/drvcrypt.c new file mode 100644 index 0000000..94bb9f9 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/drvcrypt.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief This driver interfaces TEE Cryptographic API crypto_* + */ +#include +#include +#include + +static void *crypt_algo[CRYPTO_MAX_ALGO]; + +TEE_Result drvcrypt_register(enum drvcrypt_algo_id algo_id, void *ops) +{ + if (!crypt_algo[algo_id]) { + CRYPTO_TRACE("Registering module id %d with 0x%p", algo_id, + ops); + crypt_algo[algo_id] = ops; + return TEE_SUCCESS; + } + + CRYPTO_TRACE("Fail to register module id %d with 0x%p", algo_id, ops); + return TEE_ERROR_GENERIC; +} + +void drvcrypt_register_change(enum drvcrypt_algo_id algo_id, void *ops) +{ + CRYPTO_TRACE("Change registered module id %d with 0x%p", algo_id, ops); + crypt_algo[algo_id] = ops; +} + +void *drvcrypt_get_ops(enum drvcrypt_algo_id algo_id) +{ + return crypt_algo[algo_id]; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/hash/hash.c b/optee_os/core/drivers/crypto/crypto_api/hash/hash.c new file mode 100644 index 0000000..51db02f --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/hash/hash.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019 NXP + * + * Brief Crypto Hash interface implementation to enable HW driver. + */ +#include +#include +#include +#include + +TEE_Result drvcrypt_hash_alloc_ctx(struct crypto_hash_ctx **ctx, uint32_t algo) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + hw_hash_allocate hash_alloc = NULL; + + CRYPTO_TRACE("hash alloc_ctx algo 0x%" PRIX32, algo); + + assert(ctx); + + hash_alloc = drvcrypt_get_ops(CRYPTO_HASH); + + if (hash_alloc) + ret = hash_alloc(ctx, algo); + + CRYPTO_TRACE("hash alloc_ctx ret 0x%" PRIX32, ret); + + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/hash/sub.mk b/optee_os/core/drivers/crypto/crypto_api/hash/sub.mk new file mode 100644 index 0000000..8336568 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/hash/sub.mk @@ -0,0 +1 @@ +srcs-y += hash.c diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt.h new file mode 100644 index 0000000..b248f99 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief Crypto Driver exported constants and interfaces. + */ +#ifndef __DRVCRYPT_H__ +#define __DRVCRYPT_H__ + +#include +#include +#include + +/* + * Debug Macros function of Crypto Driver Debug Level setting + * The CFG_CRYPTO_DRV_DBG is a bit mask 32 bits value defined + * as followed: + */ +#define DRV_DBG_TRACE BIT32(0) /* Driver trace */ +#define DRV_DBG_BUF BIT32(1) /* Driver dump Buffer */ + +#if (CFG_CRYPTO_DRIVER_DEBUG & DRV_DBG_TRACE) +#define CRYPTO_TRACE DMSG +#else +#define CRYPTO_TRACE(...) +#endif +#if (CFG_CRYPTO_DRIVER_DEBUG & DRV_DBG_BUF) +#define CRYPTO_DUMPBUF(title, buf, len) \ + do { \ + __typeof__(buf) _buf = (buf); \ + __typeof__(len) _len = (len); \ + CRYPTO_TRACE("%s @%p: %zu", title, _buf, _len); \ + dhex_dump(NULL, 0, 0, _buf, _len); \ + } while (0) +#else +#define CRYPTO_DUMPBUF(...) +#endif + +/* + * Definition of a crypto buffer type + */ +struct drvcrypt_buf { + uint8_t *data; + size_t length; +}; + +/* + * Crypto Library Algorithm enumeration + */ +enum drvcrypt_algo_id { + CRYPTO_HASH = 0, /* Hash driver */ + CRYPTO_HMAC, /* HMAC driver */ + CRYPTO_CMAC, /* CMAC driver */ + CRYPTO_RSA, /* Asymmetric RSA driver */ + CRYPTO_MATH, /* Mathematical driver */ + CRYPTO_CIPHER, /* Cipher driver */ + CRYPTO_ECC, /* Asymmetric ECC driver */ + CRYPTO_X25519, /* Asymmetric X25519 driver */ + CRYPTO_X448, /* Asymmetric X448 driver */ + CRYPTO_DH, /* Asymmetric DH driver */ + CRYPTO_DSA, /* Asymmetric DSA driver */ + CRYPTO_AUTHENC, /* Authenticated Encryption driver */ + CRYPTO_MAX_ALGO /* Maximum number of algo supported */ +}; + +/* + * Register the Cryptographic's operation in the table of modules + * + * @algo_id ID of the Cryptographic module + * @ops Operation (function/structure) to register + */ +TEE_Result drvcrypt_register(enum drvcrypt_algo_id algo_id, void *ops); + +/* + * Modify the Cryptographic algorithm in the table of modules + * + * @algo_id ID of the Cryptographic module + * @ops Operation (function/structure) to register + */ +void drvcrypt_register_change(enum drvcrypt_algo_id algo_id, void *ops); + +/* + * Return the Cryptographic's operation (function/structure) registered in + * the table of modules. + * + * @algo_id ID of the Cryptographic module + */ +void *drvcrypt_get_ops(enum drvcrypt_algo_id algo_id); + +#endif /* __DRVCRYPT_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_acipher.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_acipher.h new file mode 100644 index 0000000..2a009c9 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_acipher.h @@ -0,0 +1,273 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2021 NXP + * + * Brief Asymmetric Cipher interface calling the HW crypto driver. + */ +#ifndef __DRVCRYPT_ACIPHER_H__ +#define __DRVCRYPT_ACIPHER_H__ + +#include +#include + +/* + * Assymetric Cipher RSA Algorithm enumerate + */ +enum drvcrypt_rsa_id { + DRVCRYPT_RSA_NOPAD = 0, /* RSA Algo mode NO PAD */ + DRVCRYPT_RSA_OAEP, /* RSA Algo mode OAEP */ + DRVCRYPT_RSA_PKCS_V1_5, /* RSA Algo mode PKCSv1.5 */ + DRVCRYPT_RSASSA_PKCS_V1_5, /* RSA Signature Algo mode PKCSv1.5 */ + DRVCRYPT_RSASSA_PSS, /* RSA Signature Algo mode PSS */ +}; + +/* + * RSA Key object + */ +struct drvcrypt_rsakey { + void *key; /* Public or Private key */ + size_t n_size; /* Size in bytes of the Modulus N */ + bool isprivate; /* True if private key */ +}; + +/* + * RSA Mask Generation data + */ +struct drvcrypt_rsa_mgf { + uint32_t hash_algo; /* HASH Algorithm */ + size_t digest_size; /* Hash Digest Size */ + struct drvcrypt_buf seed; /* Seed to generate mask */ + struct drvcrypt_buf mask; /* Mask generated */ +}; + +/* + * RSA Encoded Signature data + */ +struct drvcrypt_rsa_ssa { + uint32_t algo; /* Operation algorithm */ + uint32_t hash_algo; /* HASH Algorithm */ + size_t digest_size; /* Hash Digest Size */ + struct drvcrypt_rsakey key; /* Public or Private Key */ + struct drvcrypt_buf message; /* Message to sign or signed */ + struct drvcrypt_buf signature; /* Signature of the message */ + size_t salt_len; /* Signature Salt length */ + + /* RSA Mask Generation function */ + TEE_Result (*mgf)(struct drvcrypt_rsa_mgf *mgf_data); +}; + +/* + * RSA Encrypt/Decrypt data + */ +struct drvcrypt_rsa_ed { + uint32_t algo; /* Operation algorithm */ + enum drvcrypt_rsa_id rsa_id; /* RSA Algorithm Id */ + uint32_t hash_algo; /* HASH Algorithm */ + size_t digest_size; /* Hash Digest Size */ + struct drvcrypt_rsakey key; /* Public or Private key */ + struct drvcrypt_buf message; /* Message to encrypt or decrypted */ + struct drvcrypt_buf cipher; /* Cipher encrypted or to decrypt */ + struct drvcrypt_buf label; /* Additional Label (RSAES) */ + + /* RSA Mask Generation function */ + TEE_Result (*mgf)(struct drvcrypt_rsa_mgf *mgf_data); +}; + +/* + * Crypto Library RSA driver operations + */ +struct drvcrypt_rsa { + /* Allocates the RSA keypair */ + TEE_Result (*alloc_keypair)(struct rsa_keypair *key, size_t size_bits); + /* Allocates the RSA public key */ + TEE_Result (*alloc_publickey)(struct rsa_public_key *key, + size_t size_bits); + /* Free RSA public key */ + void (*free_publickey)(struct rsa_public_key *key); + /* Free RSA keypair */ + void (*free_keypair)(struct rsa_keypair *key); + /* Generates the RSA keypair */ + TEE_Result (*gen_keypair)(struct rsa_keypair *key, size_t size_bits); + + /* RSA Encryption */ + TEE_Result (*encrypt)(struct drvcrypt_rsa_ed *rsa_data); + /* RSA Decryption */ + TEE_Result (*decrypt)(struct drvcrypt_rsa_ed *rsa_data); + + struct { + /* RSA Sign a message and encode the signature */ + TEE_Result (*ssa_sign)(struct drvcrypt_rsa_ssa *ssa_data); + /* RSA Encoded Signature Verification */ + TEE_Result (*ssa_verify)(struct drvcrypt_rsa_ssa *ssa_data); + } optional; +}; + +/* + * Register a RSA processing driver in the crypto API + * + * @ops - Driver operations in the HW layer + */ +static inline TEE_Result drvcrypt_register_rsa(const struct drvcrypt_rsa *ops) +{ + return drvcrypt_register(CRYPTO_RSA, (void *)ops); +} + +/* + * Signature data + */ +struct drvcrypt_sign_data { + uint32_t algo; /* Operation algorithm */ + void *key; /* Public or Private Key */ + size_t size_sec; /* Security size in bytes */ + struct drvcrypt_buf message; /* Message to sign or signed */ + struct drvcrypt_buf signature; /* Signature of the message */ +}; + +/* + * Shared Secret data + */ +struct drvcrypt_secret_data { + void *key_priv; /* Private Key */ + void *key_pub; /* Public Key */ + size_t size_sec; /* Security size in bytes */ + struct drvcrypt_buf secret; /* Shared secret */ +}; + +/* + * Encrypt/Decrypt data + */ +struct drvcrypt_ecc_ed { + uint32_t algo; /* Operation algorithm */ + void *key; /* Public or Private Key */ + size_t size_sec; /* Security size in bytes */ + struct drvcrypt_buf plaintext; /* Clear text message */ + struct drvcrypt_buf ciphertext; /* Encrypted message */ +}; + +/* + * Crypto ECC driver operations + */ +struct drvcrypt_ecc { + /* Allocates the ECC keypair */ + TEE_Result (*alloc_keypair)(struct ecc_keypair *key, uint32_t type, + size_t size_bits); + /* Allocates the ECC public key */ + TEE_Result (*alloc_publickey)(struct ecc_public_key *key, uint32_t type, + size_t size_bits); + /* Free ECC public key */ + void (*free_publickey)(struct ecc_public_key *key); + /* Generates the ECC keypair */ + TEE_Result (*gen_keypair)(struct ecc_keypair *key, size_t size_bits); + /* ECC Sign a message and returns the signature */ + TEE_Result (*sign)(struct drvcrypt_sign_data *sdata); + /* ECC Verify a message's signature */ + TEE_Result (*verify)(struct drvcrypt_sign_data *sdata); + /* ECC Shared Secret */ + TEE_Result (*shared_secret)(struct drvcrypt_secret_data *sdata); + /* ECC Encrypt */ + TEE_Result (*encrypt)(struct drvcrypt_ecc_ed *cdata); + /* ECC Decrypt */ + TEE_Result (*decrypt)(struct drvcrypt_ecc_ed *cdata); +}; + +/* + * Register an ECC processing driver in the crypto API + * + * @ops - Driver operations in the HW layer + */ +static inline TEE_Result drvcrypt_register_ecc(struct drvcrypt_ecc *ops) +{ + return drvcrypt_register(CRYPTO_ECC, (void *)ops); +} + +/* + * Crypto Library DH driver operations + */ +struct drvcrypt_dh { + /* Allocates the DH keypair */ + TEE_Result (*alloc_keypair)(struct dh_keypair *key, size_t size_bits); + /* Generates the DH keypair */ + TEE_Result (*gen_keypair)(struct dh_keypair *key, struct bignum *q, + size_t size_bits); + /* DH Shared Secret */ + TEE_Result (*shared_secret)(struct drvcrypt_secret_data *sdata); +}; + +/* + * Register a DH processing driver in the crypto API + * + * @ops - Driver operations in the HW layer + */ +static inline TEE_Result drvcrypt_register_dh(struct drvcrypt_dh *ops) +{ + return drvcrypt_register(CRYPTO_DH, (void *)ops); +} + +/* + * Crypto Library DSA driver operations + */ +struct drvcrypt_dsa { + /* Allocates the DSA keypair */ + TEE_Result (*alloc_keypair)(struct dsa_keypair *key, size_t l_bits, + size_t n_bits); + /* Allocates the DSA public key */ + TEE_Result (*alloc_publickey)(struct dsa_public_key *key, size_t l_bits, + size_t n_bits); + /* Generates the DSA keypair */ + TEE_Result (*gen_keypair)(struct dsa_keypair *key, size_t l_bits, + size_t n_bits); + /* DSA Sign a message and returns the signature */ + TEE_Result (*sign)(struct drvcrypt_sign_data *sdata, size_t l_bytes, + size_t n_bytes); + /* DSA Verify a message's signature */ + TEE_Result (*verify)(struct drvcrypt_sign_data *sdata, size_t l_bytes, + size_t n_bytes); +}; + +/* + * Register a DSA processing driver in the crypto API + * + * @ops - Driver operations in the HW layer + */ +static inline TEE_Result drvcrypt_register_dsa(struct drvcrypt_dsa *ops) +{ + return drvcrypt_register(CRYPTO_DSA, (void *)ops); +} + +/* + * Crypto Library Montgomery driver operations + */ + +struct drvcrypt_montgomery { + /* Allocates the Montgomery key pair */ + TEE_Result (*alloc_keypair)(struct montgomery_keypair *key, + size_t size_bits); + /* Generates the Montgomery key pair */ + TEE_Result (*gen_keypair)(struct montgomery_keypair *key, + size_t key_size); + /* Montgomery Shared Secret */ + TEE_Result (*shared_secret)(struct drvcrypt_secret_data *sdata); +}; + +/* + * Register a X25519 processing driver in the crypto API + * + * @ops - Driver operations in the HW layer + */ +static inline TEE_Result drvcrypt_register_x25519(struct drvcrypt_montgomery + *ops) +{ + return drvcrypt_register(CRYPTO_X25519, (void *)ops); +} + +/* + * Register a X448 processing driver in the crypto API + * + * @ops - Driver operations in the HW layer + */ +static inline TEE_Result drvcrypt_register_x448(struct drvcrypt_montgomery *ops) +{ + return drvcrypt_register(CRYPTO_X448, (void *)ops); +} + +#endif /* __DRVCRYPT_ACIPHER_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_asn1_oid.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_asn1_oid.h new file mode 100644 index 0000000..7d62819 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_asn1_oid.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2020 NXP + * + * Brief Definition of the cryptographic algorthim's OID in the + * ASN1 String format. + * Definition of the ASN1 DER tags. + * + * Computer Security Objects Register + * http://csrc.nist.gov/groups/ST/crypto_apps_infra/csor/algorithms.html + */ +#ifndef __ASN1_OID_H__ +#define __ASN1_OID_H__ + +#include +#include + +/* + * ASN1 Tags + */ +#define DRVCRYPT_ASN1_CONSTRUCTED 0x20 +#define DRVCRYPT_ASN1_SEQUENCE 0x10 +#define DRVCRYPT_ASN1_OID 0x06 +#define DRVCRYPT_ASN1_NULL 0x05 +#define DRVCRYPT_ASN1_OCTET_STRING 0x04 + +/* + * OID Top Level = first two Node (Standard and Registration-authority) + * + * iso(1) member-body(2) + * iso(1) identified-organization(3) + * joint-iso-itu-t(2) country(16) + */ +#define DRVCRYPT_OID_ISO_MEMBER_BODY "\x2a" +#define DRVCRYPT_OID_ISO_ID_ORG "\x2b" +#define DRVCRYPT_OID_ISO_ITU_COUNTRY "\x60" + +/* + * ISO Member body + * + * us(840) + * us(840) rsadsi(113549) + */ +#define DRVCRYPT_OID_MB_US "\x86\x48" +#define DRVCRYPT_OID_MB_US_RSADSI DRVCRYPT_OID_MB_US "\x86\xF7\x0D" + +/* + * ISO Identified organization + * + * oiw(14) + * oiw(14) secsig(3) + */ +#define DRVCRYPT_OID_IO_OIW "\x0e" +#define DRVCRYPT_OID_IO_OIW_SECSIG DRVCRYPT_OID_IO_OIW "\x03" + +/* + * ISO ITU OID + * + * organization(1) + * organization(1) gov(101) + */ +#define DRVCRYPT_OID_ITU_ORG "\x01" +#define DRVCRYPT_OID_ITU_ORG_GOV DRVCRYPT_OID_ITU_ORG "\x65" + +/* + * Digest Algorithm + * + * digestAlgorithm(2) + * csor(3) nistalgotrithm(4) + */ +#define DRVCRYPT_OID_DIGEST "\x02" +#define DRVCRYPT_OID_DIGEST_CSOR_NIST "\x03\x04" + +/* + * Definition of the Hash OID String + * + * id-md5 OBJECT IDENTIFIER ::= { + * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 + * } + * id-sha1 OBJECT IDENTIFIER ::= { + * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) 26 + * } + * id-sha224 OBJECT IDENTIFIER ::= { + * joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) + * csor(3) nistalgorithm(4) hashalgs(2) 4 + * } + * id-sha256 OBJECT IDENTIFIER ::= { + * joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) + * csor(3) nistalgorithm(4) hashalgs(2) 1 + * } + * id-sha384 OBJECT IDENTIFIER ::= { + * joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) + * csor(3) nistalgorithm(4) hashalgs(2) 2 + * } + * id-sha512 OBJECT IDENTIFIER ::= { + * joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) + * csor(3) nistalgorithm(4) hashalgs(2) 3 + * } + * + */ +#define DRVCRYPT_OID_ID_MD5 \ + DRVCRYPT_OID_ISO_MEMBER_BODY DRVCRYPT_OID_MB_US_RSADSI \ + DRVCRYPT_OID_DIGEST "\x05" + +#define DRVCRYPT_OID_ID_SHA1 \ + DRVCRYPT_OID_ISO_ID_ORG DRVCRYPT_OID_IO_OIW_SECSIG DRVCRYPT_OID_DIGEST \ + "\x1a" + +#define DRVCRYPT_OID_ID_SHA224 \ + DRVCRYPT_OID_ISO_ITU_COUNTRY DRVCRYPT_OID_MB_US \ + DRVCRYPT_OID_ITU_ORG_GOV DRVCRYPT_OID_DIGEST_CSOR_NIST \ + DRVCRYPT_OID_DIGEST "\x04" + +#define DRVCRYPT_OID_ID_SHA256 \ + DRVCRYPT_OID_ISO_ITU_COUNTRY DRVCRYPT_OID_MB_US \ + DRVCRYPT_OID_ITU_ORG_GOV DRVCRYPT_OID_DIGEST_CSOR_NIST \ + DRVCRYPT_OID_DIGEST "\x01" + +#define DRVCRYPT_OID_ID_SHA384 \ + DRVCRYPT_OID_ISO_ITU_COUNTRY DRVCRYPT_OID_MB_US \ + DRVCRYPT_OID_ITU_ORG_GOV DRVCRYPT_OID_DIGEST_CSOR_NIST \ + DRVCRYPT_OID_DIGEST "\x02" + +#define DRVCRYPT_OID_ID_SHA512 \ + DRVCRYPT_OID_ISO_ITU_COUNTRY DRVCRYPT_OID_MB_US \ + DRVCRYPT_OID_ITU_ORG_GOV DRVCRYPT_OID_DIGEST_CSOR_NIST \ + DRVCRYPT_OID_DIGEST "\x03" + +#define DRVCRYPT_OID_LEN(_id) (sizeof(_id) - 1) + +/* + * Definition of the ASN1 OID structure + */ +struct drvcrypt_oid { + const char *asn1; /* OID ASN1 string */ + const size_t asn1_length; /* OID ASN1 string length */ +}; + +/* + * Hash OID constant array + */ +extern const struct drvcrypt_oid drvcrypt_hash_oid[]; + +/* + * Return the Hash OID value registered in the Hash OID table. + * + * @algo Hash algorithm identifier + */ +const struct drvcrypt_oid *drvcrypt_get_alg_hash_oid(uint32_t algo); + +#endif /* __ASN1_OID_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_authenc.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_authenc.h new file mode 100644 index 0000000..3944869 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_authenc.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + * + * Authenticated Encryption interface calling the crypto driver + */ +#ifndef __DRVCRYPT_AUTHENC_H__ +#define __DRVCRYPT_AUTHENC_H__ + +#include +#include + +/* + * Authenticated Encryption operation context + */ +struct crypto_authenc { + struct crypto_authenc_ctx authenc_ctx; /* Crypto authenc API context */ + void *ctx; /* Authenc context */ + struct drvcrypt_authenc *op; /* Reference to the operation */ +}; + +/* + * Authenticated Encryption algorithm initialization data + */ +struct drvcrypt_authenc_init { + void *ctx; /* Software context */ + bool encrypt; /* Encrypt or decrypt direction */ + struct drvcrypt_buf key; /* First key */ + struct drvcrypt_buf nonce; /* Nonce */ + size_t tag_len; /* Tag length */ + size_t aad_len; /* Additional Authenticated Data length */ + size_t payload_len; /* Payload length */ +}; + +/* + * Authenticated Encryption algorithm update_aad data + */ +struct drvcrypt_authenc_update_aad { + void *ctx; /* Software context */ + bool encrypt; /* Encrypt or decrypt direction */ + struct drvcrypt_buf aad; /* Additional Authenticated Data buffer */ +}; + +/* + * Authenticated Encryption algorithm update_aad data + */ +struct drvcrypt_authenc_update_payload { + void *ctx; /* Software context */ + bool encrypt; /* Encrypt or decrypt direction */ + struct drvcrypt_buf src; /* Buffer source (message or cipher) */ + struct drvcrypt_buf dst; /* Buffer destination (cipher or message) */ +}; + +/* + * Authenticated Encryption algorithm final data + */ +struct drvcrypt_authenc_final { + void *ctx; /* Software context */ + bool encrypt; /* Encrypt or decrypt direction */ + struct drvcrypt_buf src; /* Buffer source (message or cipher) */ + struct drvcrypt_buf dst; /* Buffer destination (cipher or message) */ + struct drvcrypt_buf tag; /* Tag buffer */ +}; + +/* + * Crypto library authenc driver operations + */ +struct drvcrypt_authenc { + /* Allocate context */ + TEE_Result (*alloc_ctx)(void **ctx, uint32_t algo); + /* Free context */ + void (*free_ctx)(void *ctx); + /* Initialize the authenc operation */ + TEE_Result (*init)(struct drvcrypt_authenc_init *dinit); + /* Update the authenc operation with associated data */ + TEE_Result (*update_aad)(struct drvcrypt_authenc_update_aad *dupdate); + /* Update the authenc operation with payload data */ + TEE_Result (*update_payload)(struct drvcrypt_authenc_update_payload *d); + /* Update (or not) with payload data and get tag for encrypt op. */ + TEE_Result (*enc_final)(struct drvcrypt_authenc_final *dfinal); + /* Update (or not) with payload data and verify tag for decrypt op. */ + TEE_Result (*dec_final)(struct drvcrypt_authenc_final *dfinal); + /* Finalize the authenc operation */ + void (*final)(void *ctx); + /* Copy authenc context */ + void (*copy_state)(void *dst_ctx, void *src_ctx); +}; + +/* + * Register an authenc processing driver in the crypto API + * + * @ops - Driver operations + */ +static inline TEE_Result drvcrypt_register_authenc(struct drvcrypt_authenc *ops) +{ + return drvcrypt_register(CRYPTO_AUTHENC, (void *)ops); +} + +#endif /* __DRVCRYPT_AUTHENC_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_cipher.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_cipher.h new file mode 100644 index 0000000..a609b15 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_cipher.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2020 NXP + * + * Cipher interface calling the crypto driver. + */ +#ifndef __DRVCRYPT_CIPHER_H__ +#define __DRVCRYPT_CIPHER_H__ + +#include +#include + +/* + * Cipher operation context + */ +struct crypto_cipher { + struct crypto_cipher_ctx cipher_ctx; /* Crypto cipher API context */ + void *ctx; /* Cipher context */ + struct drvcrypt_cipher *op; /* Reference to the operation */ +}; + +/* + * Cipher algorithm initialization data + */ +struct drvcrypt_cipher_init { + void *ctx; /* Software context */ + bool encrypt; /* Encrypt or decrypt direction */ + struct drvcrypt_buf key1; /* First key */ + struct drvcrypt_buf key2; /* Second key */ + struct drvcrypt_buf iv; /* Initial vector */ +}; + +/* + * Cipher algorithm update data + */ +struct drvcrypt_cipher_update { + void *ctx; /* Software context */ + bool encrypt; /* Encrypt or decrypt direction */ + bool last; /* Last block to handle */ + struct drvcrypt_buf src; /* Buffer source (message or cipher) */ + struct drvcrypt_buf dst; /* Buffer dest (message or cipher) */ +}; + +/* + * Crypto library cipher driver operations + */ +struct drvcrypt_cipher { + /* Allocate context */ + TEE_Result (*alloc_ctx)(void **ctx, uint32_t algo); + /* Free context */ + void (*free_ctx)(void *ctx); + /* Initialize the cipher operation */ + TEE_Result (*init)(struct drvcrypt_cipher_init *dinit); + /* Update the cipher operation */ + TEE_Result (*update)(struct drvcrypt_cipher_update *dupdate); + /* Finalize the cipher operation */ + void (*final)(void *ctx); + /* Copy cipher context */ + void (*copy_state)(void *dst_ctx, void *src_ctx); +}; + +/* + * Register a cipher processing driver in the crypto API + * + * @ops - Driver operations + */ +static inline TEE_Result drvcrypt_register_cipher(struct drvcrypt_cipher *ops) +{ + return drvcrypt_register(CRYPTO_CIPHER, (void *)ops); +} + +#endif /* __DRVCRYPT_CIPHER_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_hash.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_hash.h new file mode 100644 index 0000000..2be4573 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_hash.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2020 NXP + * + * Brief Hash interface calling the HW crypto driver. + */ +#ifndef __DRVCRYPT_HASH_H__ +#define __DRVCRYPT_HASH_H__ + +#include +#include +#include + +/* + * Crypto Library Hash driver allocation function prototype + */ +typedef TEE_Result (*hw_hash_allocate)(struct crypto_hash_ctx **ctx, + uint32_t algo); + +/* + * Register a hash processing driver in the crypto API + * + * @allocate - Callback for driver context allocation in the crypto layer + */ +static inline TEE_Result drvcrypt_register_hash(hw_hash_allocate allocate) +{ + return drvcrypt_register(CRYPTO_HASH, (void *)allocate); +} + +#endif /* __DRVCRYPT_HASH_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_mac.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_mac.h new file mode 100644 index 0000000..c000366 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_mac.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019-2020 NXP + * + * MAC interface calling the HW crypto driver. + */ +#ifndef __DRVCRYPT_MAC_H__ +#define __DRVCRYPT_MAC_H__ + +#include +#include +#include + +/* + * Crypto library HMAC driver allocation function prototype + */ +typedef TEE_Result (*drvcrypt_mac_allocate)(struct crypto_mac_ctx **ctx, + uint32_t algo); + +/* + * Register a HMAC processing driver in the crypto API + * + * @allocate - Callback for driver context allocation in the crypto layer + */ +static inline TEE_Result drvcrypt_register_hmac(drvcrypt_mac_allocate allocate) +{ + return drvcrypt_register(CRYPTO_HMAC, (void *)allocate); +} + +/* + * Register a CMAC processing driver in the crypto API + * + * @allocate - Callback for driver context allocation in the crypto layer + */ +static inline TEE_Result drvcrypt_register_cmac(drvcrypt_mac_allocate allocate) +{ + return drvcrypt_register(CRYPTO_CMAC, (void *)allocate); +} +#endif /* __DRVCRYPT_MAC_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_math.h b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_math.h new file mode 100644 index 0000000..3069a29 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/include/drvcrypt_math.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2020 NXP + * + * Brief Cryptographic library using the HW crypto driver. + * Mathematical operation using HW if available. + */ +#ifndef __DRVCRYPT_MATH_H__ +#define __DRVCRYPT_MATH_H__ + +#include + +/* + * Binary Modular operation data + */ +struct drvcrypt_mod_op { + struct drvcrypt_buf n; /* Modulus N */ + struct drvcrypt_buf a; /* Operand A */ + struct drvcrypt_buf b; /* Operand B */ + struct drvcrypt_buf result; /* Result of operation */ +}; + +/* + * Operation (A xor B) mod N + * + * @data [in/out] Data operation + */ +TEE_Result drvcrypt_xor_mod_n(struct drvcrypt_mod_op *data); + +/* + * Crypto Library Binaries Modular driver operations + */ +struct drvcrypt_math { + /* (A xor B) mod N */ + TEE_Result (*xor_mod_n)(struct drvcrypt_mod_op *op_data); +}; + +/* + * Register a mathematical processing driver in the crypto API + * + * @ops - Driver operations in the HW layer + */ +static inline TEE_Result drvcrypt_register_math(const struct drvcrypt_math *ops) +{ + return drvcrypt_register(CRYPTO_MATH, (void *)ops); +} +#endif /* __DRVCRYPT_MATH_H__ */ diff --git a/optee_os/core/drivers/crypto/crypto_api/mac/mac.c b/optee_os/core/drivers/crypto/crypto_api/mac/mac.c new file mode 100644 index 0000000..a233ab0 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/mac/mac.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * Crypto MAC interface implementation to enable HW driver. + */ +#include +#include +#include +#include +#include + +TEE_Result drvcrypt_mac_alloc_ctx(struct crypto_mac_ctx **ctx, uint32_t algo) +{ + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + drvcrypt_mac_allocate mac_alloc = NULL; + unsigned int algo_id = TEE_ALG_GET_MAIN_ALG(algo); + + CRYPTO_TRACE("mac alloc_ctx algo 0x%" PRIX32, algo); + + assert(ctx); + + if (algo_id >= TEE_MAIN_ALGO_MD5 && algo_id <= TEE_MAIN_ALGO_SHA512) + mac_alloc = drvcrypt_get_ops(CRYPTO_HMAC); + else + mac_alloc = drvcrypt_get_ops(CRYPTO_CMAC); + + if (mac_alloc) + ret = mac_alloc(ctx, algo); + + CRYPTO_TRACE("mac alloc_ctx ret 0x%" PRIX32, ret); + + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/mac/sub.mk b/optee_os/core/drivers/crypto/crypto_api/mac/sub.mk new file mode 100644 index 0000000..68b265b --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/mac/sub.mk @@ -0,0 +1 @@ +srcs-y += mac.c diff --git a/optee_os/core/drivers/crypto/crypto_api/math/modulus.c b/optee_os/core/drivers/crypto/crypto_api/math/modulus.c new file mode 100644 index 0000000..4080af7 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/math/modulus.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * Brief Cryptographic library using the NXP CAAM driver. + * Mathematical Modulus operation implementation. + */ +#include +#include +#include +#include + +TEE_Result drvcrypt_xor_mod_n(struct drvcrypt_mod_op *data) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct drvcrypt_math *math = NULL; + + if (!data->a.data || !data->a.length || !data->b.data || + !data->b.length || !data->result.data || !data->result.length || + !data->n.length) + return TEE_ERROR_BAD_PARAMETERS; + + if (data->result.length < data->n.length) + return TEE_ERROR_BAD_PARAMETERS; + + math = drvcrypt_get_ops(CRYPTO_MATH); + if (math) { + /* Operation done by Math driver */ + ret = math->xor_mod_n(data); + } else { + /* Operation done by Software */ + size_t min = 0, idx = 0; + + /* Calculate the minimum size to do A xor B */ + min = MIN(data->a.length, data->b.length); + min = MIN(min, data->n.length); + + for (; idx < min; idx++) + data->result.data[idx] = + data->a.data[idx] ^ data->b.data[idx]; + + if (min < data->n.length) { + /* Complete result to make a N modulus number */ + if (data->a.length > min) { + memcpy(&data->result.data[idx], + &data->a.data[idx], + data->n.length - min); + } else if (data->b.length > min) { + memcpy(&data->result.data[idx], + &data->b.data[idx], + data->n.length - min); + } else { + memset(&data->result.data[idx], 0, + data->n.length - min); + } + } + + ret = TEE_SUCCESS; + } + + return ret; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/math/sub.mk b/optee_os/core/drivers/crypto/crypto_api/math/sub.mk new file mode 100644 index 0000000..e718dff --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/math/sub.mk @@ -0,0 +1 @@ +srcs-y += modulus.c diff --git a/optee_os/core/drivers/crypto/crypto_api/oid/hash_oid.c b/optee_os/core/drivers/crypto/crypto_api/oid/hash_oid.c new file mode 100644 index 0000000..1d7f12b --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/oid/hash_oid.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2020 NXP + * + * Brief Definition of the Hash's OID + */ + +/* Driver Crypto includes */ +#include +#include + +/* + * Hash OID values + */ +const struct drvcrypt_oid drvcrypt_hash_oid[] = { + /* empty entry */ + { NULL, 0 }, + /* MD5 */ + { DRVCRYPT_OID_ID_MD5, DRVCRYPT_OID_LEN(DRVCRYPT_OID_ID_MD5) }, + /* SHA1 */ + { DRVCRYPT_OID_ID_SHA1, DRVCRYPT_OID_LEN(DRVCRYPT_OID_ID_SHA1) }, + /* SHA224 */ + { DRVCRYPT_OID_ID_SHA224, DRVCRYPT_OID_LEN(DRVCRYPT_OID_ID_SHA224) }, + /* SHA256 */ + { DRVCRYPT_OID_ID_SHA256, DRVCRYPT_OID_LEN(DRVCRYPT_OID_ID_SHA256) }, + /* SHA384 */ + { DRVCRYPT_OID_ID_SHA384, DRVCRYPT_OID_LEN(DRVCRYPT_OID_ID_SHA384) }, + /* SHA512 */ + { DRVCRYPT_OID_ID_SHA512, DRVCRYPT_OID_LEN(DRVCRYPT_OID_ID_SHA512) }, +}; + +const struct drvcrypt_oid *drvcrypt_get_alg_hash_oid(uint32_t algo) +{ + uint32_t main_alg = TEE_ALG_GET_MAIN_ALG(algo); + + if (main_alg < ARRAY_SIZE(drvcrypt_hash_oid)) + return &drvcrypt_hash_oid[main_alg]; + + return NULL; +} diff --git a/optee_os/core/drivers/crypto/crypto_api/oid/sub.mk b/optee_os/core/drivers/crypto/crypto_api/oid/sub.mk new file mode 100644 index 0000000..1b051f3 --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/oid/sub.mk @@ -0,0 +1 @@ +srcs-y += hash_oid.c diff --git a/optee_os/core/drivers/crypto/crypto_api/sub.mk b/optee_os/core/drivers/crypto/crypto_api/sub.mk new file mode 100644 index 0000000..0577e3f --- /dev/null +++ b/optee_os/core/drivers/crypto/crypto_api/sub.mk @@ -0,0 +1,10 @@ +srcs-y += drvcrypt.c + +subdirs-y += math + +subdirs-$(CFG_CRYPTO_DRV_HASH) += hash +subdirs-$(CFG_CRYPTO_DRV_ACIPHER) += acipher +subdirs-$(CFG_CRYPTO_DRV_ACIPHER) += oid +subdirs-$(CFG_CRYPTO_DRV_CIPHER) += cipher +subdirs-$(CFG_CRYPTO_DRV_MAC) += mac +subdirs-$(CFG_CRYPTO_DRV_AUTHENC) += authenc diff --git a/optee_os/core/drivers/crypto/se050/adaptors/apis/apdu.c b/optee_os/core/drivers/crypto/se050/adaptors/apis/apdu.c new file mode 100644 index 0000000..b2d0af5 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/apis/apdu.c @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include + +sss_status_t se050_factory_reset(pSe05xSession_t ctx) +{ + if (!ctx) + return kStatus_SSS_Fail; + + if (Se05x_API_DeleteAll_Iterative(ctx) == SM_OK) + return kStatus_SSS_Success; + + return kStatus_SSS_Fail; +} + +bool se050_key_exists(uint32_t key_id, pSe05xSession_t ctx) +{ + SE05x_Result_t inuse = kSE05x_Result_FAILURE; + smStatus_t status = SM_OK; + + if (!ctx) + return false; + + status = Se05x_API_CheckObjectExists(ctx, key_id, &inuse); + if (status != SM_OK) + return false; + + if (inuse == kSE05x_Result_SUCCESS) + return true; + + return false; +} + +static sss_status_t set_rsa_public(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + struct se050_rsa_keypub *keypub, + size_t key_bit_len) +{ + SE05x_TransientType_t type = kSE05x_TransientType_Transient; + SE05x_RSAKeyFormat_t rsa_format = kSE05x_RSAKeyFormat_RAW; + smStatus_t status = SM_OK; + + if (k_object->isPersistant) + type = kSE05x_TransientType_Persistent; + + switch (k_object->cipherType) { + case kSSS_CipherType_RSA: + rsa_format = kSE05x_RSAKeyFormat_RAW; + break; + case kSSS_CipherType_RSA_CRT: + rsa_format = kSE05x_RSAKeyFormat_CRT; + break; + default: + return kStatus_SSS_Fail; + } + + status = Se05x_API_WriteRSAKey(s_ctx, policy, k_object->keyId, + (U16)key_bit_len, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + keypub->e, keypub->e_len, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_Public, + rsa_format); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + keypub->n, keypub->n_len, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + rsa_format); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +static sss_status_t set_rsa_private_rsa(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + struct se050_rsa_keypair *keypair, + size_t key_bit_len) +{ + SE05x_TransientType_t type = kSE05x_TransientType_Transient; + smStatus_t status = SM_OK; + + if (k_object->isPersistant) + type = kSE05x_TransientType_Persistent; + + status = Se05x_API_WriteRSAKey(s_ctx, policy, k_object->keyId, + (U16)key_bit_len, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + keypair->d, keypair->d_len, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_Pair, + kSE05x_RSAKeyFormat_RAW); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + keypair->n, keypair->n_len, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_RAW); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +static sss_status_t set_rsa_private_rsa_crt(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + struct se050_rsa_keypair *keypair, + size_t key_bit_len) +{ + SE05x_TransientType_t type = kSE05x_TransientType_Transient; + smStatus_t status = SM_OK; + + if (k_object->isPersistant) + type = kSE05x_TransientType_Persistent; + + status = Se05x_API_WriteRSAKey(s_ctx, policy, k_object->keyId, + (U16)key_bit_len, + keypair->p, + keypair->p_len, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_Private, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + keypair->q, + keypair->q_len, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + keypair->dp, + keypair->dp_len, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + keypair->dq, + keypair->dq_len, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + keypair->qp, + keypair->qp_len, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +static sss_status_t set_rsa_keypair_rsa(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + struct se050_rsa_keypair *keypair, + size_t key_bit_len) +{ + SE05x_TransientType_t type = kSE05x_TransientType_Transient; + smStatus_t status = SM_OK; + + if (k_object->isPersistant) + type = kSE05x_TransientType_Persistent; + + status = Se05x_API_WriteRSAKey(s_ctx, policy, k_object->keyId, + (U16)key_bit_len, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + keypair->e, keypair->e_len, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_Pair, + kSE05x_RSAKeyFormat_RAW); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + keypair->d, keypair->d_len, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_RAW); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + keypair->n, keypair->n_len, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_RAW); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +static sss_status_t set_rsa_keypair_rsa_crt(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + struct se050_rsa_keypair *keypair, + size_t key_bit_len) +{ + SE05x_TransientType_t type = kSE05x_TransientType_Transient; + smStatus_t status = SM_OK; + + if (k_object->isPersistant) + type = kSE05x_TransientType_Persistent; + + status = Se05x_API_WriteRSAKey(s_ctx, policy, k_object->keyId, + (U16)key_bit_len, + keypair->p, keypair->p_len, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_Pair, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + keypair->q, keypair->q_len, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + keypair->dp, keypair->dp_len, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + keypair->dq, keypair->dq_len, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + keypair->qp, keypair->qp_len, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + keypair->e, keypair->e_len, + SE05X_RSA_NO_priv, + SE05X_RSA_NO_pubMod, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + status = Se05x_API_WriteRSAKey(s_ctx, NULL, k_object->keyId, + 0, + SE05X_RSA_NO_p, + SE05X_RSA_NO_q, + SE05X_RSA_NO_dp, + SE05X_RSA_NO_dq, + SE05X_RSA_NO_qInv, + SE05X_RSA_NO_pubExp, + SE05X_RSA_NO_priv, + keypair->n, keypair->n_len, + (SE05x_INS_t)type, + kSE05x_KeyPart_NA, + kSE05x_RSAKeyFormat_CRT); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +sss_status_t se050_key_store_set_rsa_key_bin(sss_se05x_key_store_t *store, + sss_se05x_object_t *k_object, + struct se050_rsa_keypair *keypair, + struct se050_rsa_keypub *keypub, + size_t key_bit_len) +{ + Se05xPolicy_t policy = { }; + + if (!store || !store->session || !k_object) + return kStatus_SSS_Fail; + + if (se050_key_exists(k_object->keyId, &store->session->s_ctx)) + key_bit_len = 0; + + switch (k_object->objectType) { + case kSSS_KeyPart_Public: + return set_rsa_public(&store->session->s_ctx, + &policy, k_object, + keypub, key_bit_len); + case kSSS_KeyPart_Private: + if (k_object->cipherType == kSSS_CipherType_RSA) + return set_rsa_private_rsa(&store->session->s_ctx, + &policy, k_object, + keypair, key_bit_len); + + if (k_object->cipherType == kSSS_CipherType_RSA_CRT) + return set_rsa_private_rsa_crt(&store->session->s_ctx, + &policy, k_object, + keypair, key_bit_len); + return kStatus_SSS_Fail; + case kSSS_KeyPart_Pair: + if (k_object->cipherType == kSSS_CipherType_RSA) + return set_rsa_keypair_rsa(&store->session->s_ctx, + &policy, k_object, + keypair, key_bit_len); + + if (k_object->cipherType == kSSS_CipherType_RSA_CRT) + return set_rsa_keypair_rsa_crt(&store->session->s_ctx, + &policy, k_object, + keypair, key_bit_len); + return kStatus_SSS_Fail; + default: + return kStatus_SSS_Fail; + } +} + +sss_status_t se050_get_free_memory(pSe05xSession_t ctx, uint16_t *p, + SE05x_MemoryType_t type) +{ + if (p && ctx && Se05x_API_GetFreeMemory(ctx, type, p) == SM_OK) + return kStatus_SSS_Success; + + return kStatus_SSS_Fail; +} + +sss_status_t se050_scp03_send_rotate_cmd(pSe05xSession_t ctx, + struct s050_scp_rotate_cmd *cmd) +{ + uint8_t rsp[64] = { 0 }; + size_t rsp_len = sizeof(rsp); + tlvHeader_t hdr = { + .hdr = { + [0] = 0x80, + [1] = 0xd8, + [2] = 0, + [3] = PUT_KEYS_KEY_IDENTIFIER, + }, + }; + smStatus_t st = SM_NOT_OK; + + if (!ctx || !cmd) + return kStatus_SSS_Fail; + + hdr.hdr[2] = cmd->cmd[0]; + st = DoAPDUTxRx_s_Case4(ctx, &hdr, cmd->cmd, cmd->cmd_len, + rsp, &rsp_len); + + if ((rsp_len - 1 > sizeof(rsp)) || rsp_len < 2) + return kStatus_SSS_Fail; + + st = (rsp[rsp_len - 2] << 8) + rsp[rsp_len - 1]; + if (st != SM_OK) + return kStatus_SSS_Fail; + + if (!memcmp(rsp, cmd->kcv, cmd->kcv_len)) + return kStatus_SSS_Success; + + return kStatus_SSS_Fail; +} + +static uint8_t *alloc_pubkey_buf(struct se050_ecc_keypub *keypub, size_t *len) +{ + size_t pubkey_len = 0; + uint8_t *pubkey = NULL; + uint8_t *buf = NULL; + + pubkey_len = keypub->x_len + keypub->y_len + 1; + buf = malloc(pubkey_len); + if (!buf) + return NULL; + + *buf = 0x04; + pubkey = buf + 1; + memcpy(pubkey, keypub->x, keypub->x_len); + memcpy(pubkey + keypub->x_len, keypub->y, keypub->y_len); + *len = pubkey_len; + + return buf; +} + +sss_status_t se050_ecc_gen_shared_secret(pSe05xSession_t ctx, uint32_t kid, + struct se050_ecc_keypub *keypub, + uint8_t *secret, size_t *len) +{ + smStatus_t status = SM_OK; + uint8_t *buf = NULL; + size_t pubkey_len = 0; + + if (!keypub || !secret || !len) + return kStatus_SSS_Fail; + + buf = alloc_pubkey_buf(keypub, &pubkey_len); + if (!buf) + return kStatus_SSS_Fail; + + status = Se05x_API_ECGenSharedSecret(ctx, kid, + buf, pubkey_len, secret, len); + free(buf); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +static sss_status_t set_ecc_public(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + SE05x_TransientType_t type, + struct se050_ecc_keypub *keypub) +{ + size_t pubkey_len = 0; + smStatus_t status = SM_NOT_OK; + uint8_t *buf = NULL; + + buf = alloc_pubkey_buf(keypub, &pubkey_len); + if (!buf) + return kStatus_SSS_Fail; + + k_object->curve_id = keypub->curve; + status = Se05x_API_WriteECKey(s_ctx, policy, SE05x_MaxAttemps_UNLIMITED, + k_object->keyId, + keypub->curve, + NULL, + 0, + buf, + pubkey_len, + (SE05x_INS_t)type, + kSE05x_KeyPart_Public); + free(buf); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +static sss_status_t set_ecc_private(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + SE05x_TransientType_t type, + struct se050_ecc_keypair *keypair) +{ + smStatus_t status = SM_NOT_OK; + + k_object->curve_id = keypair->pub.curve; + status = Se05x_API_WriteECKey(s_ctx, policy, SE05x_MaxAttemps_UNLIMITED, + k_object->keyId, + keypair->pub.curve, + keypair->d, + keypair->d_len, + NULL, + 0, + (SE05x_INS_t)type, + kSE05x_KeyPart_Private); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +static sss_status_t set_ecc_pair(Se05xSession_t *s_ctx, + Se05xPolicy_t *policy, + sss_se05x_object_t *k_object, + SE05x_TransientType_t type, + struct se050_ecc_keypair *keypair) +{ + size_t pubkey_len = 0; + smStatus_t status = SM_NOT_OK; + uint8_t *buf = NULL; + + buf = alloc_pubkey_buf(&keypair->pub, &pubkey_len); + if (!buf) + return kStatus_SSS_Fail; + + k_object->curve_id = keypair->pub.curve; + status = Se05x_API_WriteECKey(s_ctx, policy, SE05x_MaxAttemps_UNLIMITED, + k_object->keyId, + keypair->pub.curve, + keypair->d, + keypair->d_len, + buf, + pubkey_len, + (SE05x_INS_t)type, + kSE05x_KeyPart_Pair); + free(buf); + if (status != SM_OK) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +sss_status_t se050_key_store_set_ecc_key_bin(sss_se05x_key_store_t *store, + sss_se05x_object_t *k_object, + struct se050_ecc_keypair *keypair, + struct se050_ecc_keypub *keypub) +{ + SE05x_TransientType_t type = kSE05x_TransientType_Transient; + Se05xPolicy_t policy = { }; + + if (!store || !store->session || !k_object) + return kStatus_SSS_Fail; + + if (k_object->isPersistant) + type = kSE05x_TransientType_Persistent; + + switch (k_object->objectType) { + case kSSS_KeyPart_Public: + if (!keypub) + return kStatus_SSS_Fail; + + return set_ecc_public(&store->session->s_ctx, + &policy, k_object, type, keypub); + case kSSS_KeyPart_Private: + if (!keypair) + return kStatus_SSS_Fail; + + return set_ecc_private(&store->session->s_ctx, + &policy, k_object, type, keypair); + case kSSS_KeyPart_Pair: + if (!keypair) + return kStatus_SSS_Fail; + + return set_ecc_pair(&store->session->s_ctx, + &policy, k_object, type, keypair); + default: + return kStatus_SSS_Fail; + } +} + +sss_status_t se050_key_store_get_ecc_key_bin(sss_se05x_key_store_t *store, + sss_se05x_object_t *k_object, + uint8_t *key, size_t *key_len) +{ + smStatus_t status = SM_NOT_OK; + uint8_t *buf = NULL; + size_t buflen = 0; + + if (!store || !store->session || !k_object || !key || !key_len) + return kStatus_SSS_Fail; + + switch (k_object->cipherType) { + case kSSS_CipherType_EC_NIST_P: + case kSSS_CipherType_EC_NIST_K: + case kSSS_CipherType_EC_BRAINPOOL: + case kSSS_CipherType_EC_BARRETO_NAEHRIG: + case kSSS_CipherType_EC_MONTGOMERY: + case kSSS_CipherType_EC_TWISTED_ED: + add_ecc_header(key, key_len, &buf, &buflen, k_object->curve_id); + status = Se05x_API_ReadObject(&store->session->s_ctx, + k_object->keyId, 0, 0, + buf, key_len); + if (status != SM_OK) + return kStatus_SSS_Fail; + + *key_len += buflen; + buflen = *key_len; + get_ecc_raw_data(key, *key_len, &buf, &buflen, + k_object->curve_id); + + /* return only the binary data */ + *key_len = buflen; + memcpy(key, buf, buflen); + return kStatus_SSS_Success; + default: + return kStatus_SSS_Fail; + } +} diff --git a/optee_os/core/drivers/crypto/se050/adaptors/apis/sss.c b/optee_os/core/drivers/crypto/se050/adaptors/apis/sss.c new file mode 100644 index 0000000..197ee6e --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/apis/sss.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include + +static const sss_policy_u asym_key = { + .type = KPolicy_Asym_Key, + .auth_obj_id = 0, + .policy = { + .asymmkey = { + .can_Sign = 1, + .can_Verify = 1, + .can_Encrypt = 1, + .can_Decrypt = 1, + .can_KD = 1, + .can_Wrap = 1, + .can_Write = 1, + .can_Gen = 1, + .can_Import_Export = 1, + .can_KA = 1, + .can_Read = 1, + .can_Attest = 1, + } + } +}; + +static const sss_policy_u common = { + .type = KPolicy_Common, + .auth_obj_id = 0, + .policy = { + .common = { + .can_Delete = 1, + .can_Read = 1, + .can_Write = 1, + .req_Sm = 1, + }, + }, +}; + +sss_policy_t se050_asym_policy = { + .nPolicies = 2, + .policies = { &asym_key, &common }, +}; + +sss_status_t se050_rotate_scp03_keys(struct sss_se05x_ctx *ctx) +{ + struct s050_scp_rotate_cmd cmd = { }; + sss_status_t status = kStatus_SSS_Fail; + struct se050_scp_key cur_keys = { }; + struct se050_scp_key new_keys = { }; + SE_Connect_Ctx_t *connect_ctx = NULL; + sss_se05x_session_t *session = NULL; + + if (!ctx) + return kStatus_SSS_Fail; + + if (IS_ENABLED(CFG_CORE_SE05X_SCP03_PROVISION_WITH_FACTORY_KEYS)) { + /* Public */ + status = se050_scp03_get_keys(&new_keys, SCP03_OFID); + if (status != kStatus_SSS_Success) + return status; + } else { + /* Secret */ + status = se050_scp03_subkey_derive(&new_keys); + if (status != kStatus_SSS_Success) + return status; + } + + status = se050_scp03_get_current_keys(&cur_keys); + if (status != kStatus_SSS_Success) + return status; + + if (IS_ENABLED(CFG_CORE_SE05X_DISPLAY_SCP03_KEYS)) { + IMSG("scp03: current keys"); + nLog_au8("scp03", 0xff, "dek: ", + cur_keys.dek, SE050_SCP03_KEY_SZ); + nLog_au8("scp03", 0xff, "mac: ", + cur_keys.mac, SE050_SCP03_KEY_SZ); + nLog_au8("scp03", 0xff, "enc: ", + cur_keys.enc, SE050_SCP03_KEY_SZ); + IMSG("scp03: proposed new keys"); + nLog_au8("scp03", 0xff, "dek: ", + new_keys.dek, SE050_SCP03_KEY_SZ); + nLog_au8("scp03", 0xff, "mac: ", + new_keys.mac, SE050_SCP03_KEY_SZ); + nLog_au8("scp03", 0xff, "enc: ", + new_keys.enc, SE050_SCP03_KEY_SZ); + } + + if (!consttime_memcmp(new_keys.enc, cur_keys.enc, SE050_SCP03_KEY_SZ) && + !consttime_memcmp(new_keys.mac, cur_keys.mac, SE050_SCP03_KEY_SZ) && + !consttime_memcmp(new_keys.dek, cur_keys.dek, SE050_SCP03_KEY_SZ)) + return kStatus_SSS_Success; + + connect_ctx = &ctx->open_ctx; + session = &ctx->session; + + status = se050_scp03_prepare_rotate_cmd(ctx, &cmd, &new_keys); + if (status != kStatus_SSS_Success) + return status; + + sss_se05x_refresh_session(se050_session, NULL); + sss_se05x_session_close(session); + + /* re-open session with same keys */ + connect_ctx->skip_select_applet = 1; + status = sss_se05x_session_open(session, kType_SSS_SE_SE05x, 0, + kSSS_ConnectionType_Encrypted, + connect_ctx); + if (status != kStatus_SSS_Success) { + se050_scp03_set_disable(); + EMSG("scp03 re-open failed, session lost"); + return kStatus_SSS_Fail; + } + + status = se050_scp03_send_rotate_cmd(&session->s_ctx, &cmd); + if (status != kStatus_SSS_Success) { + EMSG("scp03 keys not updated"); + return kStatus_SSS_Fail; + } + + sss_host_session_close(&ctx->host_session); + sss_se05x_session_close(se050_session); + memset(ctx, 0, sizeof(*ctx)); + + /* open session with new keys */ + if (IS_ENABLED(CFG_CORE_SE05X_SCP03_PROVISION_WITH_FACTORY_KEYS)) + se050_scp03_set_enable(SCP03_OFID); + else + se050_scp03_set_enable(SCP03_DERIVED); + + if (se050_core_early_init(&new_keys)) { + se050_scp03_set_disable(); + EMSG("scp03 keys rejected, session lost"); + return kStatus_SSS_Fail; + } + + return kStatus_SSS_Success; +} + +sss_status_t se050_enable_scp03(sss_se05x_session_t *session) +{ + struct se050_scp_key keys = { }; + sss_status_t status = kStatus_SSS_Success; + enum se050_scp03_ksrc key_src[] = { SCP03_DERIVED, SCP03_CFG, + SCP03_OFID }; + size_t i = 0; + + if (se050_scp03_enabled()) + return kStatus_SSS_Success; + + for (i = 0; i < ARRAY_SIZE(key_src); i++) { + status = se050_scp03_get_keys(&keys, key_src[i]); + if (status != kStatus_SSS_Success) + continue; + + if (session->subsystem) + sss_se05x_session_close(session); + + if (!se050_core_early_init(&keys)) { + se050_scp03_set_enable(key_src[i]); + goto out; + } + + sss_host_session_close(&se050_ctx.host_session); + } + + return kStatus_SSS_Fail; +out: + if (IS_ENABLED(CFG_CORE_SE05X_SCP03_PROVISION_ON_INIT)) + return se050_rotate_scp03_keys(&se050_ctx); + + return kStatus_SSS_Success; +} + +sss_status_t se050_session_open(struct sss_se05x_ctx *ctx, + struct se050_scp_key *current_keys) +{ + sss_status_t status = kStatus_SSS_Fail; + SE_Connect_Ctx_t *connect_ctx = NULL; + sss_se05x_session_t *session = NULL; + + if (!ctx) + return kStatus_SSS_Fail; + + connect_ctx = &ctx->open_ctx; + session = &ctx->session; + connect_ctx->connType = kType_SE_Conn_Type_T1oI2C; + connect_ctx->portName = NULL; + + if (!current_keys) { + return sss_se05x_session_open(session, kType_SSS_SE_SE05x, 0, + kSSS_ConnectionType_Plain, + connect_ctx); + } + + status = se050_configure_host(&ctx->host_session, + &ctx->host_ks, + &ctx->open_ctx, + &ctx->se05x_auth, + kSSS_AuthType_SCP03, + current_keys); + if (status != kStatus_SSS_Success) + return status; + + return sss_se05x_session_open(session, kType_SSS_SE_SE05x, 0, + kSSS_ConnectionType_Encrypted, + connect_ctx); +} + +sss_status_t se050_key_store_and_object_init(struct sss_se05x_ctx *ctx) +{ + if (!ctx) + return kStatus_SSS_Fail; + + return sss_se05x_key_store_context_init(&ctx->ks, &ctx->session); +} diff --git a/optee_os/core/drivers/crypto/se050/adaptors/apis/user.c b/optee_os/core/drivers/crypto/se050/adaptors/apis/user.c new file mode 100644 index 0000000..e3a4eaa --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/apis/user.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include + +static sss_status_t alloc_scp_key_to_auth(sss_object_t *k_object, + sss_key_store_t *k_store, + uint32_t key_id) +{ + sss_status_t status = kStatus_SSS_Fail; + + if (!k_object || !k_store) + return kStatus_SSS_Fail; + + status = sss_host_key_object_init(k_object, k_store); + if (status != kStatus_SSS_Success) + return status; + + return sss_host_key_object_allocate_handle(k_object, key_id, + kSSS_KeyPart_Default, + kSSS_CipherType_AES, 16, + kKeyObject_Mode_Transient); +} + +static sss_status_t prepare_host_scp(NXSCP03_AuthCtx_t *scp, + struct se050_auth_ctx *auth, + sss_key_store_t *k_store, + struct se050_scp_key *keys, + uint32_t oid) +{ + sss_status_t status = kStatus_SSS_Fail; + NXSCP03_StaticCtx_t *pStatic_ctx = NULL; + NXSCP03_DynCtx_t *pDyn_ctx = NULL; + size_t len = 0; + + if (!scp || !auth || !k_store) + return kStatus_SSS_Fail; + + pStatic_ctx = &auth->static_ctx; + pDyn_ctx = &auth->dynamic_ctx; + + scp->pStatic_ctx = pStatic_ctx; + scp->pDyn_ctx = pDyn_ctx; + pStatic_ctx->keyVerNo = 0x0B; + + status = alloc_scp_key_to_auth(&pStatic_ctx->Enc, k_store, oid++); + if (status != kStatus_SSS_Success) + return status; + + len = sizeof(keys->enc); + status = sss_host_key_store_set_key(k_store, &pStatic_ctx->Enc, + keys->enc, len, len * 8, NULL, 0); + if (status != kStatus_SSS_Success) + return status; + + status = alloc_scp_key_to_auth(&pStatic_ctx->Mac, k_store, oid++); + if (status != kStatus_SSS_Success) + return status; + + len = sizeof(keys->mac); + status = sss_host_key_store_set_key(k_store, &pStatic_ctx->Mac, + keys->mac, len, len * 8, NULL, 0); + if (status != kStatus_SSS_Success) + return status; + + status = alloc_scp_key_to_auth(&pStatic_ctx->Dek, k_store, oid++); + if (status != kStatus_SSS_Success) + return status; + + len = sizeof(keys->dek); + status = sss_host_key_store_set_key(k_store, &pStatic_ctx->Dek, + keys->dek, len, len * 8, NULL, 0); + if (status != kStatus_SSS_Success) + return status; + + status = alloc_scp_key_to_auth(&pDyn_ctx->Enc, k_store, oid++); + if (status != kStatus_SSS_Success) + return status; + + status = alloc_scp_key_to_auth(&pDyn_ctx->Mac, k_store, oid++); + if (status != kStatus_SSS_Success) + return status; + + return alloc_scp_key_to_auth(&pDyn_ctx->Rmac, k_store, oid++); +} + +sss_status_t se050_configure_host(sss_user_impl_session_t *host_session, + sss_key_store_t *host_ks, + SE_Connect_Ctx_t *open_ctx, + struct se050_auth_ctx *auth, + SE_AuthType_t auth_type, + struct se050_scp_key *keys) +{ + sss_status_t status = kStatus_SSS_Fail; + uint32_t host_oid = 0; + + if (!host_session || !host_ks || !open_ctx || !auth) + return kStatus_SSS_Fail; + + if (host_session->subsystem != kType_SSS_SubSystem_NONE) + goto prepare; + + status = sss_host_session_open(host_session, kType_SSS_Software, 0, + kSSS_ConnectionType_Plain, NULL); + if (status != kStatus_SSS_Success) + return status; + + status = sss_host_key_store_context_init(host_ks, host_session); + if (status != kStatus_SSS_Success) + goto error; + + status = sss_host_key_store_allocate(host_ks, host_oid++); + if (status != kStatus_SSS_Success) + goto error; +prepare: + status = prepare_host_scp(&open_ctx->auth.ctx.scp03, auth, host_ks, + keys, host_oid); + if (status != kStatus_SSS_Success) + goto error; + + open_ctx->auth.authType = auth_type; + return status; + +error: + sss_host_session_close(host_session); + return status; +} + +TEE_Result se050_host_key_store_get_key(sss_key_store_t *ks __unused, + sss_object_t *ko, uint8_t *data, + size_t *byte_len, size_t *bit_len) +{ + sss_user_impl_object_t *key_object = (sss_user_impl_object_t *)ko; + + if (!ko) + return TEE_ERROR_GENERIC; + + if (*byte_len < sizeof(key_object->key)) + return TEE_ERROR_EXCESS_DATA; + + memcpy(data, key_object->key, sizeof(key_object->key)); + *byte_len = sizeof(key_object->key); + *bit_len = 8 * sizeof(key_object->key); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/crypto/se050/adaptors/include/se050.h b/optee_os/core/drivers/crypto/se050/adaptors/include/se050.h new file mode 100644 index 0000000..e5f8f66 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/include/se050.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#ifndef SE050_H_ +#define SE050_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Supported Devices*/ +#define SE050A1_ID 0xA204 +#define SE050A2_ID 0xA205 +#define SE050B1_ID 0xA202 +#define SE050B2_ID 0xA203 +#define SE050C1_ID 0xA200 +#define SE050C2_ID 0xA201 +#define SE050DV_ID 0xA1F4 +#define SE051A2_ID 0xA565 +#define SE051C2_ID 0xA564 +#define SE050F2_ID 0xA77E +#define SE050E_ID 0xA921 +#define SE051A_ID 0xA920 +#define SE051C_ID 0xA8FA +#define SE051W_ID 0xA739 +#define SE050F_ID 0xA92A + +TEE_Result se050_core_early_init(struct se050_scp_key *keys); + +extern sss_se05x_key_store_t *se050_kstore; +extern sss_se05x_session_t *se050_session; +extern struct sss_se05x_ctx se050_ctx; + +static inline uint32_t se050_get_oefid(void) +{ + return SHIFT_U32(se050_ctx.se_info.oefid[0], 8) | + SHIFT_U32(se050_ctx.se_info.oefid[1], 0); +} + +#endif /* SE050_H_ */ diff --git a/optee_os/core/drivers/crypto/se050/adaptors/include/se050_apdu_apis.h b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_apdu_apis.h new file mode 100644 index 0000000..766d12e --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_apdu_apis.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#ifndef SE050_APDU_APIS_H_ +#define SE050_APDU_APIS_H_ + +#include + +struct s050_scp_rotate_cmd; + +sss_status_t se050_factory_reset(pSe05xSession_t ctx); + +bool se050_key_exists(uint32_t k_id, pSe05xSession_t ctx); + +struct se050_rsa_keypair { + uint8_t *e; + size_t e_len; + uint8_t *d; + size_t d_len; + uint8_t *n; + size_t n_len; + + uint8_t *p; + size_t p_len; + uint8_t *q; + size_t q_len; + uint8_t *qp; + size_t qp_len; + uint8_t *dp; + size_t dp_len; + uint8_t *dq; + size_t dq_len; +}; + +struct se050_rsa_keypub { + uint8_t *e; + size_t e_len; + uint8_t *n; + size_t n_len; +}; + +sss_status_t se050_key_store_set_rsa_key_bin(sss_se05x_key_store_t *k_store, + sss_se05x_object_t *k_object, + struct se050_rsa_keypair *k_pair, + struct se050_rsa_keypub *k_pub, + size_t k_bit_len); + +sss_status_t se050_get_free_memory(pSe05xSession_t ctx, uint16_t *t, + SE05x_MemoryType_t type); + +sss_status_t se050_scp03_send_rotate_cmd(pSe05xSession_t ctx, + struct s050_scp_rotate_cmd *cmd); + +struct se050_ecc_keypub { + uint8_t *x; + size_t x_len; + uint8_t *y; + size_t y_len; + uint32_t curve; +}; + +struct se050_ecc_keypair { + struct se050_ecc_keypub pub; + uint8_t *d; + size_t d_len; +}; + +sss_status_t se050_key_store_set_ecc_key_bin(sss_se05x_key_store_t *store, + sss_se05x_object_t *object, + struct se050_ecc_keypair *pair, + struct se050_ecc_keypub *pub); + +sss_status_t se050_key_store_get_ecc_key_bin(sss_se05x_key_store_t *store, + sss_se05x_object_t *object, + uint8_t *key, size_t *len); + +sss_status_t se050_ecc_gen_shared_secret(pSe05xSession_t ctx, uint32_t id, + struct se050_ecc_keypub *pub, + uint8_t *secret, size_t *len); + +#endif /* SE050_APDU_APIS_H_ */ diff --git a/optee_os/core/drivers/crypto/se050/adaptors/include/se050_sss_apis.h b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_sss_apis.h new file mode 100644 index 0000000..6b20a8f --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_sss_apis.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#ifndef SE050_SSS_APIS_H_ +#define SE050_SSS_APIS_H_ + +#include +#include + +extern sss_policy_t se050_asym_policy; +struct se050_scp_key; + +struct sss_se05x_ctx { + SE_Connect_Ctx_t open_ctx; + sss_se05x_session_t session; + sss_se05x_key_store_t ks; + + struct se050_auth_ctx { + NXSCP03_StaticCtx_t static_ctx; + NXSCP03_DynCtx_t dynamic_ctx; + } se05x_auth; + + sss_user_impl_session_t host_session; + sss_key_store_t host_ks; + + struct se05x_se_info { + uint8_t applet[3]; + uint8_t oefid[2]; + } se_info; +}; + +sss_status_t se050_key_store_and_object_init(struct sss_se05x_ctx *ctx); +sss_status_t se050_enable_scp03(sss_se05x_session_t *session); +sss_status_t se050_rotate_scp03_keys(struct sss_se05x_ctx *ctx); +sss_status_t se050_session_open(struct sss_se05x_ctx *ctx, + struct se050_scp_key *key); +#endif /* SE050_SSS_APIS_H_ */ diff --git a/optee_os/core/drivers/crypto/se050/adaptors/include/se050_user_apis.h b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_user_apis.h new file mode 100644 index 0000000..ad83388 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_user_apis.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#ifndef SE050_USER_APIS_H_ +#define SE050_USER_APIS_H_ + +#include +#include +#include +#include +#include + +sss_status_t se050_configure_host(sss_user_impl_session_t *host_session, + sss_key_store_t *host_ks, + SE_Connect_Ctx_t *open_ctx, + struct se050_auth_ctx *auth_ctx, + SE_AuthType_t auth_type, + struct se050_scp_key *keys); + +TEE_Result se050_host_key_store_get_key(sss_key_store_t *ks __unused, + sss_object_t *ko, uint8_t *data, + size_t *byte_len, size_t *bit_len); +#endif /* SE050_USER_APIS_H_ */ diff --git a/optee_os/core/drivers/crypto/se050/adaptors/include/se050_utils.h b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_utils.h new file mode 100644 index 0000000..7a0a427 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/include/se050_utils.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#ifndef SE050_UTILS_H_ +#define SE050_UTILS_H_ + +#include +#include + +#define SE050_SCP03_KEY_SZ 16 + +struct se050_scp_key { + uint8_t enc[SE050_SCP03_KEY_SZ]; + uint8_t mac[SE050_SCP03_KEY_SZ]; + uint8_t dek[SE050_SCP03_KEY_SZ]; +}; + +struct s050_scp_rotate_cmd { + uint8_t cmd[128]; + size_t cmd_len; + uint8_t kcv[16]; + size_t kcv_len; +}; + +#define OID_MIN ((uint32_t)(0x00000001)) +#define OID_MAX ((uint32_t)(OID_MIN + 0x7BFFFFFE)) + +#define SE050_KEY_WATERMARK 0x57721566 +#define WATERMARKED(x) \ + ((uint64_t)(((uint64_t)SE050_KEY_WATERMARK) << 32) + (x)) + +sss_status_t se050_get_oid(uint32_t *val); + +struct rsa_keypair; +struct ecc_keypair; + +uint32_t se050_rsa_keypair_from_nvm(struct rsa_keypair *key); +uint32_t se050_ecc_keypair_from_nvm(struct ecc_keypair *key); +uint64_t se050_generate_private_key(uint32_t oid); + +void se050_refcount_init_ctx(uint8_t **cnt); +int se050_refcount_final_ctx(uint8_t *cnt); + +sss_status_t se050_get_se_info(sss_se05x_session_t *session, bool display); + +enum se050_scp03_ksrc { SCP03_CFG, SCP03_DERIVED, SCP03_OFID }; +void se050_scp03_set_enable(enum se050_scp03_ksrc ksrc); +void se050_scp03_set_disable(void); +bool se050_scp03_enabled(void); +sss_status_t se050_scp03_get_current_keys(struct se050_scp_key *keys); +sss_status_t se050_scp03_get_keys(struct se050_scp_key *keys, + enum se050_scp03_ksrc); +sss_status_t se050_scp03_subkey_derive(struct se050_scp_key *keys); +sss_status_t se050_scp03_prepare_rotate_cmd(struct sss_se05x_ctx *ctx, + struct s050_scp_rotate_cmd *cmd, + struct se050_scp_key *keys); +#endif /* SE050_UTILS_H_ */ diff --git a/optee_os/core/drivers/crypto/se050/adaptors/sub.mk b/optee_os/core/drivers/crypto/se050/adaptors/sub.mk new file mode 100644 index 0000000..87c9ea6 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/sub.mk @@ -0,0 +1,12 @@ +cflags-y += -Wno-strict-aliasing +include ${CFG_NXP_SE05X_PLUG_AND_TRUST}/cflags.mk + +incdirs_ext-y += ${CFG_NXP_SE05X_PLUG_AND_TRUST}/optee_lib/include +incdirs-y += ./include + +srcs-y += utils/scp_config.c +srcs-y += utils/utils.c +srcs-y += utils/info.c +srcs-y += apis/apdu.c +srcs-y += apis/user.c +srcs-y += apis/sss.c diff --git a/optee_os/core/drivers/crypto/se050/adaptors/utils/info.c b/optee_os/core/drivers/crypto/se050/adaptors/utils/info.c new file mode 100644 index 0000000..aa9759c --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/utils/info.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Force the output until the P&T stack fixes its verbosity */ +#define LOG_MAU8_I(msg, buf, len) nLog_au8("Info", 0xff, msg, buf, len) +#define LOG_I(format, ...) nLog("Info", 0xff, format, ##__VA_ARGS__) +#define LOG_E(format, ...) nLog("Info", NX_LEVEL_ERROR, format, ##__VA_ARGS__) +#define LOG_MAU8_E(msg, buf, len) \ + nLog_au8("Info", NX_LEVEL_ERROR, msg, buf, len) + +static sss_status_t jcop4_get_id(void *ctx, bool display) +{ + char jcop_platform_id[17] = { 0 }; + smStatus_t ret = SM_OK; + unsigned char cmd[] = { + 0x80, /* CLA '80' / '00' GlobalPlatform / ISO / IEC */ + 0xCA, /* INS 'CA' GET DATA(IDENTIFY) */ + 0x00, /* P1 '00' High order tag value */ + 0xFE, /* P2 'FE' Low order tag value - proprietary data */ + 0x02, /* Lc '02' Length of data field */ + 0xDF, + 0x28, /* Data 'DF28' Card identification data */ + 0x00 /* Le '00' Length of response data */ + }; + struct msg_rsp { + uint8_t vTag_value_proprietary_data; + uint8_t vLength_of_following_data; + uint8_t vTag_card_identification_data[0x02]; + uint8_t vLength_of_card_identification_data; + uint8_t vTag_configuration_ID; + uint8_t vLength_configuration_ID; + uint8_t vConfiguration_ID[0x0C]; + uint8_t vTag_patch_ID; + uint8_t vLength_patch_ID; + uint8_t vPatch_ID[0x08]; + uint8_t vTag_platform_build_ID1; + uint8_t vLength_platform_build_ID; + uint8_t vPlatform_build_ID[0x18]; + uint8_t vTag_FIPS_mode; + uint8_t vLength_FIPS_mode; + uint8_t vFIPS_mode; + uint8_t vTag_pre_perso_state; + uint8_t vLength_pre_perso_state; + uint8_t vBit_mask_of_pre_perso_state; + uint8_t vTag_ROM_ID; + uint8_t vLength_ROM_ID; + uint8_t vROM_ID[0x08]; + uint8_t vStatus_Word_SW_[0x02]; + } rsp = { 0 }; + uint8_t *p = (uint8_t *)&rsp; + uint32_t len = sizeof(struct msg_rsp); + uint16_t dummy = sizeof(struct msg_rsp); + + ret = GP_Select(ctx, p, 0, p, &dummy); + if (ret != SM_OK) { + LOG_E("Could not select ISD."); + return kStatus_SSS_Fail; + } + + ret = smCom_TransceiveRaw(ctx, cmd, sizeof(cmd), p, &len); + if (ret != SM_OK || len != sizeof(rsp)) { + LOG_MAU8_E("Error reading JCOP ID", p, sizeof(rsp)); + return kStatus_SSS_Fail; + } + + memcpy(se050_ctx.se_info.oefid, &rsp.vConfiguration_ID[2], 2); + if (!display) + return kStatus_SSS_Success; + + LOG_I("SE050 JCOP4 Information:"); + LOG_I("%s = 0x%02X", "Tag value - proprietary data 0xFE", + rsp.vTag_value_proprietary_data); + LOG_I("%s = 0x%02X", "Length of following data 0x45", + rsp.vLength_of_following_data); + LOG_MAU8_I("Tag card identification data", + rsp.vTag_card_identification_data, + sizeof(rsp.vTag_card_identification_data)); + LOG_I("%s = 0x%02X", "Length of card identification data", + rsp.vLength_of_card_identification_data); + LOG_I("%s = 0x%02X", "Tag configuration ID (Must be 0x01)", + rsp.vTag_configuration_ID); + LOG_I("%s = 0x%02X", "Length configuration ID 0x0C", + rsp.vLength_configuration_ID); + LOG_MAU8_I("Configuration ID", + rsp.vConfiguration_ID, sizeof(rsp.vConfiguration_ID)); + LOG_MAU8_I("OEF ID", &rsp.vConfiguration_ID[2], 2); + LOG_I("%s = 0x%02X", "Tag patch ID (Must be 0x02)", rsp.vTag_patch_ID); + LOG_I("%s = 0x%02X", "Length patch ID 0x08", rsp.vLength_patch_ID); + LOG_MAU8_I("Patch ID", rsp.vPatch_ID, sizeof(rsp.vPatch_ID)); + LOG_I("%s = 0x%02X", "Tag platform build ID1 (Must be 0x03)", + rsp.vTag_platform_build_ID1); + LOG_I("%s = 0x%02X", "Length platform build ID 0x18", + rsp.vLength_platform_build_ID); + LOG_MAU8_I("Platform build ID", + rsp.vPlatform_build_ID, sizeof(rsp.vPlatform_build_ID)); + memcpy(jcop_platform_id, rsp.vPlatform_build_ID, 16); + + LOG_I("%s = %s", "JCOP Platform ID", jcop_platform_id); + LOG_I("%s = 0x%02X", "Tag FIPS mode (Must be 0x05)", + rsp.vTag_FIPS_mode); + LOG_I("%s = 0x%02X", "Length FIPS mode 0x01", rsp.vLength_FIPS_mode); + LOG_I("%s = 0x%02X", "FIPS mode var", rsp.vFIPS_mode); + LOG_I("%s = 0x%02X", "Tag pre-perso state (Must be 0x07)", + rsp.vTag_pre_perso_state); + LOG_I("%s = 0x%02X", "Length pre-perso state 0x01", + rsp.vLength_pre_perso_state); + LOG_I("%s = 0x%02X", "Bit mask of pre-perso state var", + rsp.vBit_mask_of_pre_perso_state); + + LOG_I("%s = 0x%02X", "Tag ROM ID (Must be 0x08)", rsp.vTag_ROM_ID); + LOG_I("%s = 0x%02X", "Length ROM ID 0x08", rsp.vLength_ROM_ID); + LOG_MAU8_I("ROM ID", rsp.vROM_ID, sizeof(rsp.vROM_ID)); + LOG_MAU8_I("Status Word (SW)", rsp.vStatus_Word_SW_, + sizeof(rsp.vStatus_Word_SW_)); + + return kStatus_SSS_Success; +} + +#define ITEM(__x) { \ + .name = #__x, \ + .val = (kSE05x_AppletConfig_##__x), \ + } + +static void show_config(uint16_t cfg) +{ + struct items { + uint16_t val; + const char *name; + } features[] = { + ITEM(ECDAA), ITEM(ECDSA_ECDH_ECDHE), ITEM(EDDSA), ITEM(DH_MONT), + ITEM(HMAC), ITEM(RSA_PLAIN), ITEM(RSA_CRT), ITEM(AES), + ITEM(DES), ITEM(PBKDF), ITEM(TLS), ITEM(MIFARE), ITEM(I2CM), + }; + size_t i = 0; + + for (i = 0; i < ARRAY_SIZE(features); i++) { + LOG_I("\t%s%s", cfg & features[i].val ? "with\t" : "without\t", + features[i].name); + } +} + +static sss_status_t applet_get_id(sss_se05x_session_t *session, bool display) +{ + SE05x_Result_t result = kSE05x_Result_NA; + smStatus_t ret = SM_OK; + uint8_t uid[SE050_MODULE_UNIQUE_ID_LEN] = { 0 }; + size_t uidLen = sizeof(uid); + uint8_t applet_version[7] = { 0 }; + size_t applet_versionLen = sizeof(applet_version); + + ret = Se05x_API_CheckObjectExists(&session->s_ctx, + kSE05x_AppletResID_UNIQUE_ID, + &result); + if (ret != SM_OK) + return kStatus_SSS_Fail; + + ret = Se05x_API_ReadObject(&session->s_ctx, + kSE05x_AppletResID_UNIQUE_ID, 0, + (uint16_t)uidLen, uid, &uidLen); + if (ret != SM_OK) + return kStatus_SSS_Fail; + + /* + * VersionInfo is a 7 - byte value consisting of: + * - 1 - byte Major applet version + * - 1 - byte Minor applet version + * - 1 - byte patch applet version + * - 2 - byte AppletConfig, indicating the supported applet features + * - 2-byte Secure Box version: major version (MSB) concatenated with + * minor version (LSB). + */ + ret = Se05x_API_GetVersion(&session->s_ctx, applet_version, + &applet_versionLen); + if (ret != SM_OK) { + LOG_E("Failed Se05x_API_GetVersion"); + return kStatus_SSS_Fail; + } + + memcpy(se050_ctx.se_info.applet, applet_version, 3); + if (!display) + return kStatus_SSS_Success; + + LOG_MAU8_I("Applet ID", uid, uidLen); + + LOG_I("Applet Major = %d", applet_version[0]); + LOG_I("Applet Minor = %d", applet_version[1]); + LOG_I("Applet patch = %d", applet_version[2]); + LOG_I("AppletConfig = %02X%02X", applet_version[3], applet_version[4]); + show_config(applet_version[3] << 8 | applet_version[4]); + LOG_I("Internal = %02X%02X", applet_version[5], applet_version[6]); + + return kStatus_SSS_Success; +} + +sss_status_t se050_get_se_info(sss_se05x_session_t *session, bool display) +{ + sss_status_t ret = kStatus_SSS_Fail; + __maybe_unused uint32_t oefid = 0; + + if (session) { + ret = applet_get_id(session, display); + if (ret != kStatus_SSS_Success) { + EMSG("Can't retrieve Applet information"); + return ret; + } + + ret = jcop4_get_id(session->s_ctx.conn_ctx, display); + if (ret != kStatus_SSS_Success) { + EMSG("Can't retrieve JCOP information"); + return ret; + } + +#ifdef CFG_CORE_SE05X_OEFID + /* validate the requested OEFID against the runtime detected */ + oefid = SHIFT_U32(se050_ctx.se_info.oefid[0], 8) | + SHIFT_U32(se050_ctx.se_info.oefid[1], 0); + + if (oefid != CFG_CORE_SE05X_OEFID) { + EMSG("OEFID configuration error, 0x%x != 0x%"PRIx32, + CFG_CORE_SE05X_OEFID, oefid); + return kStatus_SSS_Fail; + } +#endif + return kStatus_SSS_Success; + } + + return kStatus_SSS_Fail; +} diff --git a/optee_os/core/drivers/crypto/se050/adaptors/utils/scp_config.c b/optee_os/core/drivers/crypto/se050/adaptors/utils/scp_config.c new file mode 100644 index 0000000..493dede --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/utils/scp_config.c @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static enum se050_scp03_ksrc scp03_ksrc; +static bool scp03_enabled; + +#define SE050A1 0 +#define SE050A2 1 +#define SE050B1 2 +#define SE050B2 3 +#define SE050C1 4 +#define SE050C2 5 +#define SE050DV 6 +#define SE051A2 7 +#define SE051C2 8 +#define SE050F2 9 +#define SE050E 10 +#define SE051A 11 +#define SE051C 12 +#define SE051W 13 +#define SE050F 14 + +static const struct se050_scp_key se050_default_keys[] = { + [SE050A1] = { + .enc = { 0x34, 0xae, 0x09, 0x67, 0xe3, 0x29, 0xe9, 0x51, + 0x8e, 0x72, 0x65, 0xd5, 0xad, 0xcc, 0x01, 0xc2 }, + .mac = { 0x52, 0xb2, 0x53, 0xca, 0xdf, 0x47, 0x2b, 0xdb, + 0x3d, 0x0f, 0xb3, 0x8e, 0x09, 0x77, 0x00, 0x99 }, + .dek = { 0xac, 0xc9, 0x14, 0x31, 0xfe, 0x26, 0x81, 0x1b, + 0x5e, 0xcb, 0xc8, 0x45, 0x62, 0x0d, 0x83, 0x44 }, + }, + [SE050A2] = { + .enc = { 0x46, 0xa9, 0xc4, 0x8c, 0x34, 0xef, 0xe3, 0x44, + 0xa5, 0x22, 0xe6, 0x67, 0x44, 0xf8, 0x99, 0x6a }, + .mac = { 0x12, 0x03, 0xff, 0x61, 0xdf, 0xbc, 0x9c, 0x86, + 0x19, 0x6a, 0x22, 0x74, 0xae, 0xf4, 0xed, 0x28 }, + .dek = { 0xf7, 0x56, 0x1c, 0x6f, 0x48, 0x33, 0x61, 0x19, + 0xee, 0x39, 0x43, 0x9a, 0xab, 0x34, 0x09, 0x8e }, + }, + [SE050B1] = { + .enc = { 0xd4, 0x99, 0xbc, 0x90, 0xde, 0xa5, 0x42, 0xcf, + 0x78, 0xd2, 0x5e, 0x13, 0xd6, 0x4c, 0xbb, 0x1f }, + .mac = { 0x08, 0x15, 0x55, 0x96, 0x43, 0xfb, 0x79, 0xeb, + 0x85, 0x01, 0xa0, 0xdc, 0x83, 0x3d, 0x90, 0x1f }, + .dek = { 0xbe, 0x7d, 0xdf, 0xb4, 0x06, 0xe8, 0x1a, 0xe4, + 0xe9, 0x66, 0x5a, 0x9f, 0xed, 0x64, 0x26, 0x7c }, + }, + [SE050B2] = { + .enc = { 0x5f, 0xa4, 0x3d, 0x82, 0x02, 0xd2, 0x5e, 0x9a, + 0x85, 0xb1, 0xfe, 0x7e, 0x2d, 0x26, 0x47, 0x8d }, + .mac = { 0x10, 0x5c, 0xea, 0x22, 0x19, 0xf5, 0x2b, 0xd1, + 0x67, 0xa0, 0x74, 0x63, 0xc6, 0x93, 0x79, 0xc3 }, + .dek = { 0xd7, 0x02, 0x81, 0x57, 0xf2, 0xad, 0x37, 0x2c, + 0x74, 0xbe, 0x96, 0x9b, 0xcc, 0x39, 0x06, 0x27 }, + }, + [SE050C1] = { + .enc = { 0x85, 0x2b, 0x59, 0x62, 0xe9, 0xcc, 0xe5, 0xd0, + 0xbe, 0x74, 0x6b, 0x83, 0x3b, 0xcc, 0x62, 0x87 }, + .mac = { 0xdb, 0x0a, 0xa3, 0x19, 0xa4, 0x08, 0x69, 0x6c, + 0x8e, 0x10, 0x7a, 0xb4, 0xe3, 0xc2, 0x6b, 0x47 }, + .dek = { 0x4c, 0x2f, 0x75, 0xc6, 0xa2, 0x78, 0xa4, 0xae, + 0xe5, 0xc9, 0xaf, 0x7c, 0x50, 0xee, 0xa8, 0x0c }, + }, + [SE050C2] = { + .enc = { 0xbd, 0x1d, 0xe2, 0x0a, 0x81, 0xea, 0xb2, 0xbf, + 0x3b, 0x70, 0x9a, 0x9d, 0x69, 0xa3, 0x12, 0x54 }, + .mac = { 0x9a, 0x76, 0x1b, 0x8d, 0xba, 0x6b, 0xed, 0xf2, + 0x27, 0x41, 0xe4, 0x5d, 0x8d, 0x42, 0x36, 0xf5 }, + .dek = { 0x9b, 0x99, 0x3b, 0x60, 0x0f, 0x1c, 0x64, 0xf5, + 0xad, 0xc0, 0x63, 0x19, 0x2a, 0x96, 0xc9, 0x47 }, + }, + [SE050DV] = { + .enc = { 0x35, 0xc2, 0x56, 0x45, 0x89, 0x58, 0xa3, 0x4f, + 0x61, 0x36, 0x15, 0x5f, 0x82, 0x09, 0xd6, 0xcd }, + .mac = { 0xaf, 0x17, 0x7d, 0x5d, 0xbd, 0xf7, 0xc0, 0xd5, + 0xc1, 0x0a, 0x05, 0xb9, 0xf1, 0x60, 0x7f, 0x78 }, + .dek = { 0xa1, 0xbc, 0x84, 0x38, 0xbf, 0x77, 0x93, 0x5b, + 0x36, 0x1a, 0x44, 0x25, 0xfe, 0x79, 0xfa, 0x29 }, + }, + [SE051A2] = { + .enc = { 0x84, 0x0a, 0x5d, 0x51, 0x79, 0x55, 0x11, 0xc9, + 0xce, 0xf0, 0xc9, 0x6f, 0xd2, 0xcb, 0xf0, 0x41 }, + .mac = { 0x64, 0x6b, 0xc2, 0xb8, 0xc3, 0xa4, 0xd9, 0xc1, + 0xfa, 0x8d, 0x71, 0x16, 0xbe, 0x04, 0xfd, 0xfe }, + .dek = { 0x03, 0xe6, 0x69, 0x9a, 0xca, 0x94, 0x26, 0xd9, + 0xc3, 0x89, 0x22, 0xf8, 0x91, 0x4c, 0xe5, 0xf7 }, + }, + [SE051C2] = { + .enc = { 0x88, 0xdb, 0xcd, 0x65, 0x82, 0x0d, 0x2a, 0xa0, + 0x6f, 0xfa, 0xb9, 0x2a, 0xa8, 0xe7, 0x93, 0x64 }, + .mac = { 0xa8, 0x64, 0x4e, 0x2a, 0x04, 0xd9, 0xe9, 0xc8, + 0xc0, 0xea, 0x60, 0x86, 0x68, 0x29, 0x99, 0xe5 }, + .dek = { 0x8a, 0x38, 0x72, 0x38, 0x99, 0x88, 0x18, 0x44, + 0xe2, 0xc1, 0x51, 0x3d, 0xac, 0xd9, 0xf8, 0x0d }, + }, + [SE050F2] = { + .enc = { 0x91, 0x88, 0xda, 0x8c, 0xf3, 0x69, 0xcf, 0xa9, + 0xa0, 0x08, 0x91, 0x62, 0x7b, 0x65, 0x34, 0x5a }, + .mac = { 0xcb, 0x20, 0xf8, 0x09, 0xc7, 0xa0, 0x39, 0x32, + 0xbc, 0x20, 0x3b, 0x0a, 0x01, 0x81, 0x6c, 0x81 }, + .dek = { 0x27, 0x8e, 0x61, 0x9d, 0x83, 0x51, 0x8e, 0x14, + 0xc6, 0xf1, 0xe4, 0xfa, 0x96, 0x8b, 0xe5, 0x1c }, + }, + [SE050E] = { + .enc = { 0xd2, 0xdb, 0x63, 0xe7, 0xa0, 0xa5, 0xae, 0xd7, + 0x2a, 0x64, 0x60, 0xc4, 0xdf, 0xdc, 0xaf, 0x64 }, + .mac = { 0x73, 0x8d, 0x5b, 0x79, 0x8e, 0xd2, 0x41, 0xb0, + 0xb2, 0x47, 0x68, 0x51, 0x4b, 0xfb, 0xa9, 0x5b }, + .dek = { 0x67, 0x02, 0xda, 0xc3, 0x09, 0x42, 0xb2, 0xc8, + 0x5e, 0x7f, 0x47, 0xb4, 0x2c, 0xed, 0x4e, 0x7f }, + }, + [SE051A] = { + .enc = { 0x88, 0xea, 0x9f, 0xa6, 0x86, 0xf3, 0xcf, 0x2f, + 0xfc, 0xaf, 0x4b, 0x1c, 0xba, 0x93, 0xe4, 0x42 }, + .mac = { 0x4f, 0x16, 0x3f, 0x59, 0xf0, 0x74, 0x31, 0xf4, + 0x3e, 0xe2, 0xee, 0x18, 0x34, 0xa5, 0x23, 0x34 }, + .dek = { 0xd4, 0x76, 0xcf, 0x47, 0xaa, 0x27, 0xb5, 0x4a, + 0xb3, 0xdb, 0xeb, 0xe7, 0x65, 0x6d, 0x67, 0x70 }, + }, + [SE051C] = { + .enc = { 0xbf, 0xc2, 0xdb, 0xe1, 0x82, 0x8e, 0x03, 0x5d, + 0x3e, 0x7f, 0xa3, 0x6b, 0x90, 0x2a, 0x05, 0xc6 }, + .mac = { 0xbe, 0xf8, 0x5b, 0xd7, 0xba, 0x04, 0x97, 0xd6, + 0x28, 0x78, 0x1c, 0xe4, 0x7b, 0x18, 0x8c, 0x96 }, + .dek = { 0xd8, 0x73, 0xf3, 0x16, 0xbe, 0x29, 0x7f, 0x2f, + 0xc9, 0xc0, 0xe4, 0x5f, 0x54, 0x71, 0x06, 0x99 } + }, + [SE051W] = { + .enc = { 0x18, 0xb3, 0xb4, 0xe3, 0x40, 0xc0, 0x80, 0xd9, + 0x9b, 0xeb, 0xb8, 0xb8, 0x64, 0x4b, 0x8c, 0x52 }, + .mac = { 0x3d, 0x0c, 0xfa, 0xc8, 0x7b, 0x96, 0x7c, 0x00, + 0xe3, 0x3b, 0xa4, 0x96, 0x61, 0x38, 0x38, 0xa2 }, + .dek = { 0x68, 0x06, 0x83, 0xf9, 0x4e, 0x6b, 0xcb, 0x94, + 0x73, 0xec, 0xc1, 0x56, 0x7a, 0x1b, 0xd1, 0x09 }, + }, + [SE050F] = { + .enc = { 0xB5, 0x0E, 0x1F, 0x12, 0xB8, 0x1F, 0xE5, 0x3B, + 0x6C, 0x3B, 0x53, 0x87, 0x91, 0x2A, 0x1A, 0x5A, }, + .mac = { 0x71, 0x93, 0x69, 0x59, 0xD3, 0x7F, 0x2B, 0x22, + 0xC5, 0xA0, 0xC3, 0x49, 0x19, 0xA2, 0xBC, 0x1F, }, + .dek = { 0x86, 0x95, 0x93, 0x23, 0x98, 0x54, 0xDC, 0x0D, + 0x86, 0x99, 0x00, 0x50, 0x0C, 0xA7, 0x9C, 0x15, }, + }, +}; + +static sss_status_t get_id_from_ofid(uint32_t ofid, uint32_t *id) +{ + switch (ofid) { + case SE050A1_ID: + *id = SE050A1; + break; + case SE050A2_ID: + *id = SE050A2; + break; + case SE050B1_ID: + *id = SE050B1; + break; + case SE050B2_ID: + *id = SE050B2; + break; + case SE050C1_ID: + *id = SE050C1; + break; + case SE050C2_ID: + *id = SE050C2; + break; + case SE050DV_ID: + *id = SE050DV; + break; + case SE051A2_ID: + *id = SE051A2; + break; + case SE051C2_ID: + *id = SE051C2; + break; + case SE050F2_ID: + *id = SE050F2; + break; + case SE050E_ID: + *id = SE050E; + break; + case SE051A_ID: + *id = SE051A; + break; + case SE051C_ID: + *id = SE051C; + break; + case SE051W_ID: + *id = SE051W; + break; + case SE050F_ID: + *id = SE050F; + break; + default: + return kStatus_SSS_Fail; + } + + return kStatus_SSS_Success; +} + +static sss_status_t encrypt_key_and_get_kcv(uint8_t *enc, uint8_t *kc, + uint8_t *key, + struct sss_se05x_ctx *ctx, + uint32_t id) +{ + static const uint8_t ones[] = { [0 ... AES_KEY_LEN_nBYTE - 1] = 1 }; + uint8_t enc_len = AES_KEY_LEN_nBYTE; + uint8_t kc_len = AES_KEY_LEN_nBYTE; + sss_status_t st = kStatus_SSS_Fail; + sss_object_t *dek_object = NULL; + sss_se05x_symmetric_t symm = { }; + sss_se05x_object_t ko = { }; + uint8_t dek[AES_KEY_LEN_nBYTE] = { 0 }; + size_t dek_len = sizeof(dek); + size_t dek_bit_len = dek_len * 8; + + st = sss_se05x_key_object_init(&ko, &ctx->ks); + if (st != kStatus_SSS_Success) + return kStatus_SSS_Fail; + + st = sss_se05x_key_object_allocate_handle(&ko, id, + kSSS_KeyPart_Default, + kSSS_CipherType_AES, + AES_KEY_LEN_nBYTE, + kKeyObject_Mode_Transient); + if (st != kStatus_SSS_Success) + return kStatus_SSS_Fail; + + st = sss_se05x_key_store_set_key(&ctx->ks, &ko, key, AES_KEY_LEN_nBYTE, + AES_KEY_LEN_nBYTE * 8, NULL, 0); + if (st != kStatus_SSS_Success) + goto out; + + st = sss_se05x_symmetric_context_init(&symm, &ctx->session, &ko, + kAlgorithm_SSS_AES_ECB, + kMode_SSS_Encrypt); + if (st != kStatus_SSS_Success) + goto out; + + st = sss_se05x_cipher_one_go(&symm, NULL, 0, ones, kc, kc_len); + if (st != kStatus_SSS_Success) + goto out; + + dek_object = &ctx->open_ctx.auth.ctx.scp03.pStatic_ctx->Dek; + if (se050_host_key_store_get_key(&ctx->host_ks, dek_object, + dek, &dek_len, &dek_bit_len)) + goto out; + + st = sss_se05x_key_store_set_key(&ctx->ks, &ko, dek, AES_KEY_LEN_nBYTE, + AES_KEY_LEN_nBYTE * 8, NULL, 0); + if (st != kStatus_SSS_Success) + goto out; + + st = sss_se05x_cipher_one_go(&symm, NULL, 0, key, enc, enc_len); +out: + if (symm.keyObject) + sss_se05x_symmetric_context_free(&symm); + + sss_se05x_key_object_free(&ko); + + Se05x_API_DeleteSecureObject(&ctx->session.s_ctx, id); + + return st; +} + +static sss_status_t prepare_key_data(uint8_t *key, uint8_t *cmd, + struct sss_se05x_ctx *ctx, uint32_t id) +{ + uint8_t kc[AES_KEY_LEN_nBYTE] = { 0 }; + sss_status_t status = kStatus_SSS_Fail; + + cmd[0] = PUT_KEYS_KEY_TYPE_CODING_AES; + cmd[1] = AES_KEY_LEN_nBYTE + 1; + cmd[2] = AES_KEY_LEN_nBYTE; + cmd[3 + AES_KEY_LEN_nBYTE] = CRYPTO_KEY_CHECK_LEN; + + status = encrypt_key_and_get_kcv(&cmd[3], kc, key, ctx, id); + if (status != kStatus_SSS_Success) + return status; + + memcpy(&cmd[3 + AES_KEY_LEN_nBYTE + 1], kc, CRYPTO_KEY_CHECK_LEN); + + return kStatus_SSS_Success; +} + +sss_status_t se050_scp03_prepare_rotate_cmd(struct sss_se05x_ctx *ctx, + struct s050_scp_rotate_cmd *cmd, + struct se050_scp_key *keys) + +{ + sss_status_t status = kStatus_SSS_Fail; + size_t kcv_len = 0; + size_t cmd_len = 0; + uint8_t key_version = 0; + uint8_t *key[] = { + [0] = keys->enc, + [1] = keys->mac, + [2] = keys->dek, + }; + uint32_t oid = 0; + size_t i = 0; + + key_version = ctx->open_ctx.auth.ctx.scp03.pStatic_ctx->keyVerNo; + cmd->cmd[cmd_len] = key_version; + cmd_len += 1; + + cmd->kcv[kcv_len] = key_version; + kcv_len += 1; + + for (i = 0; i < ARRAY_SIZE(key); i++) { + status = se050_get_oid(&oid); + if (status != kStatus_SSS_Success) + return kStatus_SSS_Fail; + + status = prepare_key_data(key[i], &cmd->cmd[cmd_len], ctx, oid); + if (status != kStatus_SSS_Success) + return kStatus_SSS_Fail; + + memcpy(&cmd->kcv[kcv_len], + &cmd->cmd[cmd_len + 3 + AES_KEY_LEN_nBYTE + 1], + CRYPTO_KEY_CHECK_LEN); + + cmd_len += 3 + AES_KEY_LEN_nBYTE + 1 + CRYPTO_KEY_CHECK_LEN; + kcv_len += CRYPTO_KEY_CHECK_LEN; + } + + cmd->cmd_len = cmd_len; + cmd->kcv_len = kcv_len; + + return kStatus_SSS_Success; +} + +static sss_status_t get_ofid_key(struct se050_scp_key *keys) +{ + uint32_t oefid = SHIFT_U32(se050_ctx.se_info.oefid[0], 8) | + SHIFT_U32(se050_ctx.se_info.oefid[1], 0); + sss_status_t status = kStatus_SSS_Success; + uint32_t id = 0; + + status = get_id_from_ofid(oefid, &id); + if (status != kStatus_SSS_Success) + return status; + + memcpy(keys, &se050_default_keys[id], sizeof(*keys)); + return kStatus_SSS_Success; +} + +static sss_status_t get_config_key(struct se050_scp_key *keys __maybe_unused) +{ +#ifdef CFG_CORE_SE05X_SCP03_CURRENT_DEK + struct se050_scp_key current_keys = { + .dek = { CFG_CORE_SE05X_SCP03_CURRENT_DEK }, + .mac = { CFG_CORE_SE05X_SCP03_CURRENT_MAC }, + .enc = { CFG_CORE_SE05X_SCP03_CURRENT_ENC }, + }; + + memcpy(keys, ¤t_keys, sizeof(*keys)); + return kStatus_SSS_Success; +#else + return kStatus_SSS_Fail; +#endif +} + +static const char * __maybe_unused get_scp03_ksrc_name(enum se050_scp03_ksrc k) +{ + switch (k) { + case SCP03_DERIVED: + return "derived"; + case SCP03_CFG: + return "built-in"; + case SCP03_OFID: + return "factory"; + default: + panic(); + } + + return NULL; +} + +sss_status_t se050_scp03_subkey_derive(struct se050_scp_key *keys) +{ + struct { + const char *name; + uint8_t *data; + } key[3] = { + [0] = { .name = "dek", .data = keys->dek }, + [1] = { .name = "mac", .data = keys->mac }, + [2] = { .name = "enc", .data = keys->enc }, + }; + uint8_t msg[SE050_SCP03_KEY_SZ + 3] = { 0 }; + size_t i = 0; + + if (IS_ENABLED(CFG_CORE_SCP03_ONLY)) { + memset(msg, 0x55, sizeof(msg)); + } else { + /* add some randomness */ + if (tee_otp_get_die_id(msg + 3, SE050_SCP03_KEY_SZ)) + return kStatus_SSS_Fail; + } + + for (i = 0; i < ARRAY_SIZE(key); i++) { + memcpy(msg, key[i].name, 3); + if (huk_subkey_derive(HUK_SUBKEY_SE050, msg, sizeof(msg), + key[i].data, SE050_SCP03_KEY_SZ)) + return kStatus_SSS_Fail; + } + + return kStatus_SSS_Success; +} + +bool se050_scp03_enabled(void) +{ + return scp03_enabled; +} + +void se050_scp03_set_enable(enum se050_scp03_ksrc ksrc) +{ + scp03_enabled = true; + scp03_ksrc = ksrc; + + IMSG("SE05X SCP03 using %s keys", get_scp03_ksrc_name(ksrc)); +} + +void se050_scp03_set_disable(void) +{ + scp03_enabled = false; +} + +sss_status_t se050_scp03_get_keys(struct se050_scp_key *keys, + enum se050_scp03_ksrc ksrc) +{ + switch (ksrc) { + case SCP03_CFG: + return get_config_key(keys); + case SCP03_DERIVED: + return se050_scp03_subkey_derive(keys); + case SCP03_OFID: + return get_ofid_key(keys); + default: + return kStatus_SSS_Fail; + } +} + +sss_status_t se050_scp03_get_current_keys(struct se050_scp_key *keys) +{ + if (se050_scp03_enabled()) + return se050_scp03_get_keys(keys, scp03_ksrc); + + return kStatus_SSS_Fail; +} diff --git a/optee_os/core/drivers/crypto/se050/adaptors/utils/utils.c b/optee_os/core/drivers/crypto/se050/adaptors/utils/utils.c new file mode 100644 index 0000000..88fcad9 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/adaptors/utils/utils.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include + +/* exp: minimal amount of transient memory required to generate an RSA key */ +#define TRANSIENT_MEMORY_THRESHOLD 0x140 + +#define NBR_OID ((uint32_t)(OID_MAX - OID_MIN)) +#define IS_WATERMARKED(x) (((x) & WATERMARKED(0)) == WATERMARKED(0)) + +static void delete_transient_objects(void) +{ + SE05x_AttestationType_t att = kSE05x_AttestationType_None; + SE05x_SecureObjectType_t type = kSE05x_SecObjTyp_HMAC_KEY; + uint8_t more = kSE05x_MoreIndicator_NA; + smStatus_t status = SM_NOT_OK; + Se05xSession_t *ctx = NULL; + uint8_t *list = NULL; + size_t len = 1024; + uint16_t offset = 0; + uint8_t mode = 0; + uint32_t id = 0; + size_t i = 0; + + if (!se050_session) + return; + + ctx = &se050_session->s_ctx; + + list = calloc(1, len); + if (!list) + return; + do { + status = Se05x_API_ReadIDList(ctx, offset, 0xFF, &more, + list, &len); + if (status != SM_OK) + break; + + offset = len; + for (i = 0; i < len; i += 4) { + id = (list[i + 0] << (3 * 8)) | + (list[i + 1] << (2 * 8)) | + (list[i + 2] << (1 * 8)) | + (list[i + 3] << (0 * 8)); + + if (id >= OID_MAX || id == 0) + continue; + + status = Se05x_API_ReadType(ctx, id, &type, &mode, att); + if (status != SM_OK) + continue; + + if (mode == kKeyObject_Mode_Transient) + Se05x_API_DeleteSecureObject(ctx, id); + } + } while (more == kSE05x_MoreIndicator_MORE); + + free(list); +} + +static sss_status_t generate_oid(uint32_t *val) +{ + uint32_t oid = OID_MIN; + uint32_t random = 0; + size_t i = 0; + + for (i = 0; i < NBR_OID; i++) { + if (crypto_rng_read(&random, sizeof(random))) + return kStatus_SSS_Fail; + + oid = OID_MIN + (random & OID_MAX); + + /* + * The less significant byte different than zero prevents bignum + * conversions from discarding it + */ + if (oid > OID_MAX || !(oid & 0xFF)) + continue; + + if (!se050_key_exists(oid, &se050_session->s_ctx)) { + *val = oid; + return kStatus_SSS_Success; + } + } + + return kStatus_SSS_Fail; +} + +sss_status_t se050_get_oid(uint32_t *val) +{ + sss_status_t status = kStatus_SSS_Success; + uint16_t mem_t = 0; + + if (!val) + return kStatus_SSS_Fail; + + status = se050_get_free_memory(&se050_session->s_ctx, &mem_t, + kSE05x_MemoryType_TRANSIENT_DESELECT); + if (status != kStatus_SSS_Success) + return kStatus_SSS_Fail; + + if (mem_t < TRANSIENT_MEMORY_THRESHOLD) + delete_transient_objects(); + + return generate_oid(val); +} + +static uint32_t se050_key(uint64_t key) +{ + uint32_t oid = (uint32_t)key; + + if (!IS_WATERMARKED(key)) + return 0; + + /* oid > OID_MAX could have been created by an external client */ + if (oid < OID_MIN) + return 0; + + return oid; +} + +uint32_t se050_rsa_keypair_from_nvm(struct rsa_keypair *key) +{ + uint64_t key_id = 0; + + if (!key) + return 0; + + if (crypto_bignum_num_bytes(key->d) != sizeof(uint64_t)) + return 0; + + crypto_bignum_bn2bin(key->d, (uint8_t *)&key_id); + + return se050_key(key_id); +} + +uint32_t se050_ecc_keypair_from_nvm(struct ecc_keypair *key) +{ + uint64_t key_id = 0; + + if (!key) + return 0; + + if (crypto_bignum_num_bytes(key->d) != sizeof(uint64_t)) + return 0; + + crypto_bignum_bn2bin(key->d, (uint8_t *)&key_id); + + return se050_key(key_id); +} + +uint64_t se050_generate_private_key(uint32_t oid) +{ + return WATERMARKED(oid); +} + +void se050_refcount_init_ctx(uint8_t **cnt) +{ + if (!*cnt) { + *cnt = calloc(1, sizeof(uint8_t)); + if (*cnt) + **cnt = 1; + else + EMSG("can't allocate refcount"); + } else { + **cnt = **cnt + 1; + } +} + +int se050_refcount_final_ctx(uint8_t *cnt) +{ + if (!cnt) + return 1; + + if (!*cnt) { + free(cnt); + return 1; + } + + *cnt = *cnt - 1; + + return 0; +} diff --git a/optee_os/core/drivers/crypto/se050/core/apdu.c b/optee_os/core/drivers/crypto/se050/core/apdu.c new file mode 100644 index 0000000..c526e0d --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/apdu.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2021 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include + +TEE_Result crypto_se_do_apdu(enum crypto_apdu_type type, + uint8_t *hdr, size_t hdr_len, + uint8_t *src_data, size_t src_len, + uint8_t *dst_data, size_t *dst_len) +{ + sss_status_t status = kStatus_SSS_Fail; + + status = sss_se05x_do_apdu(&se050_session->s_ctx, type, + hdr, hdr_len, src_data, src_len, + dst_data, dst_len); + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/crypto/se050/core/cipher.c b/optee_os/core/drivers/crypto/se050/core/cipher.c new file mode 100644 index 0000000..ce0a1e5 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/cipher.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static TEE_Result do_init(struct drvcrypt_cipher_init *dinit) +{ + struct crypto_cipher_ctx *ctx = dinit->ctx; + TEE_OperationMode mode = TEE_MODE_DECRYPT; + + if (dinit->encrypt) + mode = TEE_MODE_ENCRYPT; + + return ctx->ops->init(dinit->ctx, mode, + dinit->key1.data, dinit->key1.length, + dinit->key2.data, dinit->key2.length, + dinit->iv.data, dinit->iv.length); +} + +static TEE_Result do_update(struct drvcrypt_cipher_update *dupdate) +{ + struct crypto_cipher_ctx *ctx = dupdate->ctx; + + return ctx->ops->update(ctx, dupdate->last, dupdate->src.data, + dupdate->src.length, dupdate->dst.data); +} + +static void do_final(void *context) +{ + struct crypto_cipher_ctx *ctx = context; + + ctx->ops->final(ctx); +} + +static void do_free(void *context) +{ + struct crypto_cipher_ctx *ctx = context; + + ctx->ops->free_ctx(ctx); +} + +static void do_copy_state(void *out, void *in) +{ + struct crypto_cipher_ctx *dst_ctx = out; + struct crypto_cipher_ctx *src_ctx = in; + + src_ctx->ops->copy_state(dst_ctx, src_ctx); +} + +static TEE_Result do_allocate(void **ctx, uint32_t algo) +{ + switch (algo) { + case TEE_ALG_AES_CTR: + return se050_aes_ctr_allocate(ctx); + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +static struct drvcrypt_cipher driver_cipher = { + .alloc_ctx = do_allocate, + .free_ctx = do_free, + .init = do_init, + .update = do_update, + .final = do_final, + .copy_state = do_copy_state, +}; + +static TEE_Result se050_cipher_init(void) +{ + return drvcrypt_register_cipher(&driver_cipher); +} + +driver_init_late(se050_cipher_init); diff --git a/optee_os/core/drivers/crypto/se050/core/ctr.c b/optee_os/core/drivers/crypto/se050/core/ctr.c new file mode 100644 index 0000000..1b161fa --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/ctr.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include + +struct se050_aes_ctr_ctx { + struct crypto_cipher_ctx ctx; + sss_se05x_symmetric_t aes_ctx; + sss_se05x_object_t key_obj; + uint8_t *cnt; /* shared reference counter for duplicated ciphers */ + int nc_off; + unsigned char counter[TEE_AES_BLOCK_SIZE]; + unsigned char block[TEE_AES_BLOCK_SIZE]; +}; + +static struct se050_aes_ctr_ctx *to_aes_ctr_ctx(struct crypto_cipher_ctx *ctx) +{ + return container_of(ctx, struct se050_aes_ctr_ctx, ctx); +} + +static TEE_Result se050_aes_ctr_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode __unused, + const uint8_t *key1, + size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv, size_t iv_len __unused) +{ + struct se050_aes_ctr_ctx *c = to_aes_ctr_ctx(ctx); + sss_status_t st = kStatus_SSS_Success; + uint32_t oid = 0; + + if (c->key_obj.keyId) + goto init; + + memcpy(c->counter, iv, sizeof(c->counter)); + + st = sss_se05x_key_object_init(&c->key_obj, se050_kstore); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = se050_get_oid(&oid); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = sss_se05x_key_object_allocate_handle(&c->key_obj, oid, + kSSS_KeyPart_Default, + kSSS_CipherType_AES, 0, + kKeyObject_Mode_Transient); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = sss_se05x_key_store_set_key(se050_kstore, &c->key_obj, + key1, key1_len, key1_len * 8, NULL, 0); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = sss_se05x_symmetric_context_init(&c->aes_ctx, se050_session, + &c->key_obj, + kAlgorithm_SSS_AES_ECB, + kMode_SSS_Encrypt); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; +init: + st = sss_se05x_cipher_init(&c->aes_ctx, (uint8_t *)NULL, 0); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result se050_aes_ctr_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct se050_aes_ctr_ctx *c = to_aes_ctr_ctx(ctx); + sss_status_t st = kStatus_SSS_Success; + size_t dst_len = len; + int i = 0; + int n = 0; + int j = 0; + + n = c->nc_off; + while (len--) { + dst_len = sizeof(c->counter); + if (n == 0) { + st = sss_se05x_cipher_update(&c->aes_ctx, + c->counter, 16, + c->block, &dst_len); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_STATE; + + for (i = 16; i > 0; i--) + if (++c->counter[i - 1] != 0) + break; + } + j = *data++; + *dst++ = (unsigned char)(j ^ c->block[n]); + n = (n + 1) & 0x0F; + } + + c->nc_off = n; + return TEE_SUCCESS; +} + +static void do_final(struct crypto_cipher_ctx *ctx) +{ + struct se050_aes_ctr_ctx *c = to_aes_ctr_ctx(ctx); + + if (!se050_refcount_final_ctx(c->cnt)) { + memset(c->block, 0, sizeof(c->block)); + return; + } + + if (c->key_obj.keyId) + sss_se05x_key_store_erase_key(se050_kstore, &c->key_obj); + + sss_se05x_symmetric_context_free(&c->aes_ctx); +} + +static void do_free(struct crypto_cipher_ctx *ctx) +{ + free(to_aes_ctr_ctx(ctx)); +} + +static void do_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct se050_aes_ctr_ctx *src = to_aes_ctr_ctx(src_ctx); + struct se050_aes_ctr_ctx *dst = to_aes_ctr_ctx(dst_ctx); + + se050_refcount_init_ctx(&src->cnt); + memcpy(dst, src, sizeof(*dst)); +} + +static struct crypto_cipher_ops aes_ctr_ops = { + .update = se050_aes_ctr_update, + .copy_state = do_copy_state, + .init = se050_aes_ctr_init, + .free_ctx = do_free, + .final = do_final, +}; + +TEE_Result se050_aes_ctr_allocate(void **ctx) +{ + struct se050_aes_ctr_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &aes_ctr_ops; + *ctx = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/crypto/se050/core/die_id.c b/optee_os/core/drivers/crypto/se050/core/die_id.c new file mode 100644 index 0000000..cf4fc92 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/die_id.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include + +int tee_otp_get_die_id(uint8_t *buffer, size_t len) +{ + uint8_t die_id[SE050_MODULE_UNIQUE_ID_LEN] = { 0 }; + size_t die_id_len = sizeof(die_id); + sss_status_t status = kStatus_SSS_Fail; + + status = sss_se05x_session_prop_get_au8(se050_session, + kSSS_SessionProp_UID, + die_id, &die_id_len); + if (status != kStatus_SSS_Success) + return -1; + + if (tee_hash_createdigest(TEE_ALG_SHA256, die_id, die_id_len, buffer, + len)) + return -1; + + return 0; +} diff --git a/optee_os/core/drivers/crypto/se050/core/ecc.c b/optee_os/core/drivers/crypto/se050/core/ecc.c new file mode 100644 index 0000000..c55ab8b --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/ecc.c @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const struct crypto_ecc_keypair_ops *pair_ops; +static const struct crypto_ecc_public_ops *pub_ops; + +static bool oefid_key_supported(size_t bits) +{ + switch (se050_get_oefid()) { + case SE050F_ID: + return bits >= 224; + default: + return true; + } +} + +static bool oefid_algo_supported(uint32_t algo) +{ + switch (se050_get_oefid()) { + case SE050F_ID: + switch (algo) { + case TEE_ALG_ECDSA_SHA224: + case TEE_ALG_ECDSA_SHA256: + case TEE_ALG_ECDSA_SHA384: + case TEE_ALG_ECDSA_SHA512: + return true; + default: + return false; + } + default: + return true; + } +} + +static uint32_t algo_tee2se050(uint32_t algo) +{ + switch (algo) { + case TEE_ALG_ECDSA_SHA1: + return kAlgorithm_SSS_ECDSA_SHA1; + case TEE_ALG_ECDSA_SHA224: + return kAlgorithm_SSS_ECDSA_SHA224; + case TEE_ALG_ECDSA_SHA256: + return kAlgorithm_SSS_ECDSA_SHA256; + case TEE_ALG_ECDSA_SHA384: + return kAlgorithm_SSS_ECDSA_SHA384; + case TEE_ALG_ECDSA_SHA512: + return kAlgorithm_SSS_ECDSA_SHA512; + default: + EMSG("algorithm %#"PRIx32" not enabled", algo); + return kAlgorithm_None; + } +} + +static uint32_t cipher_tee2se050(uint32_t curve) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + case TEE_ECC_CURVE_NIST_P224: + case TEE_ECC_CURVE_NIST_P256: + case TEE_ECC_CURVE_NIST_P384: + case TEE_ECC_CURVE_NIST_P521: + return kSSS_CipherType_EC_NIST_P; + default: + EMSG("cipher %#"PRIx32" not enabled", curve); + return kSSS_CipherType_NONE; + } +} + +static uint32_t curve_tee2se050(uint32_t curve) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + return kSE05x_ECCurve_NIST_P192; + case TEE_ECC_CURVE_NIST_P224: + return kSE05x_ECCurve_NIST_P224; + case TEE_ECC_CURVE_NIST_P256: + return kSE05x_ECCurve_NIST_P256; + case TEE_ECC_CURVE_NIST_P384: + return kSE05x_ECCurve_NIST_P384; + case TEE_ECC_CURVE_NIST_P521: + return kSE05x_ECCurve_NIST_P521; + default: + EMSG("curve %#"PRIx32" not enabled", curve); + return kSE05x_ECCurve_NA; + } +} + +static uint32_t curve_se0502tee(uint32_t curve) +{ + switch (curve) { + case kSE05x_ECCurve_NIST_P192: + return TEE_ECC_CURVE_NIST_P192; + case kSE05x_ECCurve_NIST_P224: + return TEE_ECC_CURVE_NIST_P224; + case kSE05x_ECCurve_NIST_P256: + return TEE_ECC_CURVE_NIST_P256; + case kSE05x_ECCurve_NIST_P384: + return TEE_ECC_CURVE_NIST_P384; + case kSE05x_ECCurve_NIST_P521: + return TEE_ECC_CURVE_NIST_P521; + default: + EMSG("curve %#"PRIx32" not enabled", curve); + return TEE_CRYPTO_ELEMENT_NONE; + } +} + +static bool bn_alloc_max(struct bignum **s) +{ + *s = crypto_bignum_allocate(4096); + + return *s; +} + +static TEE_Result ecc_get_key_size(uint32_t curve, uint32_t algo, + size_t *bytes, size_t *bits) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + *bits = 192; + *bytes = 24; + break; + case TEE_ECC_CURVE_NIST_P224: + *bits = 224; + *bytes = 28; + break; + case TEE_ECC_CURVE_NIST_P256: + *bits = 256; + *bytes = 32; + break; + case TEE_ECC_CURVE_NIST_P384: + *bits = 384; + *bytes = 48; + break; + case TEE_ECC_CURVE_NIST_P521: + *bits = 521; + *bytes = 66; + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + + if (!oefid_key_supported(*bits)) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (!oefid_algo_supported(algo)) + return TEE_ERROR_NOT_IMPLEMENTED; + + return TEE_SUCCESS; +} + +static TEE_Result ecc_prepare_msg(uint32_t algo, const uint8_t *msg, + size_t *msg_len, uint8_t **msg_padded) +{ + struct { + uint32_t algo; + size_t len; + } map[] = { + { kAlgorithm_SSS_ECDSA_SHA1, TEE_SHA1_HASH_SIZE }, + { kAlgorithm_SSS_ECDSA_SHA224, TEE_SHA224_HASH_SIZE }, + { kAlgorithm_SSS_ECDSA_SHA256, TEE_SHA256_HASH_SIZE }, + { kAlgorithm_SSS_ECDSA_SHA384, TEE_SHA384_HASH_SIZE }, + { kAlgorithm_SSS_ECDSA_SHA512, TEE_SHA512_HASH_SIZE }, + }; + size_t i = 0; + + for (i = 0; i < ARRAY_SIZE(map); i++) { + if (algo == map[i].algo) + break; + } + + if (i >= ARRAY_SIZE(map)) + return TEE_ERROR_BAD_PARAMETERS; + + if (*msg_len >= map[i].len) { + /* truncate */ + *msg_len = map[i].len; + return TEE_SUCCESS; + } + + /* pad */ + *msg_padded = calloc(1, map[i].len); + if (!*msg_padded) + return TEE_ERROR_OUT_OF_MEMORY; + + memcpy(*msg_padded, msg, *msg_len); + *msg_len = map[i].len; + + return TEE_SUCCESS; +} + +static TEE_Result set_binary_data(struct bignum *b, size_t key_len, uint8_t **p, + size_t *len) +{ + size_t a = crypto_bignum_num_bytes(b); + uint8_t leading_zeros = 0; + uint8_t *q = NULL; + + if (!a) + return TEE_ERROR_GENERIC; + + if (a != key_len) { + leading_zeros = key_len - a; + a = key_len; + } + + q = calloc(1, a); + if (!q) + return TEE_ERROR_OUT_OF_MEMORY; + + crypto_bignum_bn2bin(b, q + leading_zeros); + *len = a; + *p = q; + + return TEE_SUCCESS; +} + +static TEE_Result se050_inject_public_key(sss_se05x_object_t *k_object, + struct ecc_public_key *key, + size_t key_len) +{ + struct se050_ecc_keypub key_bin = { }; + TEE_Result ret = TEE_ERROR_GENERIC; + sss_status_t st = kStatus_SSS_Fail; + uint32_t oid = 0; + + st = sss_se05x_key_object_init(k_object, se050_kstore); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * This function might return an error if the curve already + * exists in the secure element. An actual error creating the + * curve will be caught when attempting to set the key. + */ + sss_se05x_key_store_create_curve(&se050_session->s_ctx, + curve_tee2se050(key->curve)); + + st = se050_get_oid(&oid); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = sss_se05x_key_object_allocate_handle(k_object, oid, + kSSS_KeyPart_Public, + cipher_tee2se050(key->curve), + 0, + kKeyObject_Mode_Transient); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + ret = set_binary_data(key->x, key_len, &key_bin.x, &key_bin.x_len); + if (ret != TEE_SUCCESS) + return ret; + + ret = set_binary_data(key->y, key_len, &key_bin.y, &key_bin.y_len); + if (ret != TEE_SUCCESS) { + free(key_bin.x); + return ret; + } + + key_bin.curve = curve_tee2se050(key->curve); + st = se050_key_store_set_ecc_key_bin(se050_kstore, k_object, NULL, + &key_bin); + free(key_bin.x); + free(key_bin.y); + if (st != kStatus_SSS_Success) { + EMSG("Can't inject transient key, curve: %#"PRIx32, key->curve); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result se050_inject_keypair(sss_se05x_object_t *k_object, + struct ecc_keypair *key, + size_t key_len) +{ + struct se050_ecc_keypair key_bin = { }; + sss_status_t st = kStatus_SSS_Fail; + TEE_Result ret = TEE_ERROR_GENERIC; + uint32_t key_id = 0; + uint32_t oid = 0; + + st = sss_se05x_key_object_init(k_object, se050_kstore); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + key_id = se050_ecc_keypair_from_nvm(key); + if (key_id) { + st = sss_se05x_key_object_get_handle(k_object, key_id); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; + } + + /* + * This function might return an error if the curve already + * exists in the secure element. An actual error creating the + * curve will be caught when attempting to set the key. + */ + sss_se05x_key_store_create_curve(&se050_session->s_ctx, + curve_tee2se050(key->curve)); + + st = se050_get_oid(&oid); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = sss_se05x_key_object_allocate_handle(k_object, oid, + kSSS_KeyPart_Pair, + cipher_tee2se050(key->curve), + 0, + kKeyObject_Mode_Transient); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + ret = set_binary_data(key->d, key_len, &key_bin.d, &key_bin.d_len); + if (ret != TEE_SUCCESS) + return ret; + + ret = set_binary_data(key->x, key_len, + &key_bin.pub.x, &key_bin.pub.x_len); + if (ret != TEE_SUCCESS) { + free(key_bin.d); + return ret; + } + + ret = set_binary_data(key->y, key_len, + &key_bin.pub.y, &key_bin.pub.y_len); + if (ret != TEE_SUCCESS) { + free(key_bin.d); + free(key_bin.pub.x); + return ret; + } + + key_bin.pub.curve = curve_tee2se050(key->curve); + st = se050_key_store_set_ecc_key_bin(se050_kstore, k_object, &key_bin, + NULL); + free(key_bin.d); + free(key_bin.pub.x); + free(key_bin.pub.y); + if (st != kStatus_SSS_Success) { + EMSG("Can't inject transient key, curve: %#"PRIx32, key->curve); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result gen_fallback(struct ecc_keypair *key, size_t len) +{ + if (!IS_ENABLED(CFG_NXP_SE05X_ECC_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + DMSG("se050: debug: ECC software fallback: KEYGEN"); + return pair_ops->generate(key, len); +} + +static TEE_Result shared_secret_fallback(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, size_t *secret_len) +{ + const struct crypto_ecc_keypair_ops *ops = NULL; + + if (!IS_ENABLED(CFG_NXP_SE05X_ECC_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (se050_ecc_keypair_from_nvm(private_key)) + return TEE_ERROR_NOT_IMPLEMENTED; + + ops = crypto_asym_get_ecc_keypair_ops(TEE_TYPE_ECDH_KEYPAIR); + if (!ops) + return TEE_ERROR_NOT_IMPLEMENTED; + + DMSG("se050: debug: ECC software fallback: ECDH"); + return ops->shared_secret(private_key, public_key, + secret, (unsigned long *)secret_len); +} + +static TEE_Result verify_fallback(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + if (!IS_ENABLED(CFG_NXP_SE05X_ECC_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + DMSG("se050: debug: ECC software fallback: VERIFY"); + return pub_ops->verify(algo, key, msg, msg_len, sig, sig_len); +} + +static TEE_Result sign_fallback(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + if (!IS_ENABLED(CFG_NXP_SE05X_ECC_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (se050_ecc_keypair_from_nvm(key)) + return TEE_ERROR_NOT_IMPLEMENTED; + + DMSG("se050: debug: ECC software fallback: SIGN"); + return pair_ops->sign(algo, key, msg, msg_len, sig, sig_len); +} + +static TEE_Result shared_secret(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, size_t *secret_len) +{ + struct se050_ecc_keypub key = { }; + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_derive_key_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result ret = TEE_SUCCESS; + size_t key_bits = 0; + size_t key_bytes = 0; + + if (private_key->curve != public_key->curve) + return TEE_ERROR_BAD_PARAMETERS; + + ret = ecc_get_key_size(private_key->curve, 0, &key_bytes, &key_bits); + if (ret) { + if (ret != TEE_ERROR_NOT_IMPLEMENTED) + return ret; + + return shared_secret_fallback(private_key, public_key, + secret, secret_len); + } + + ret = se050_inject_keypair(&kobject, private_key, key_bytes); + if (ret != TEE_SUCCESS) + return TEE_ERROR_BAD_PARAMETERS; + + st = sss_se05x_derive_key_context_init(&ctx, se050_session, &kobject, + kAlgorithm_SSS_ECDH, + kMode_SSS_ComputeSharedSecret); + if (st != kStatus_SSS_Success) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + /* prepare the public key (must be in raw format) */ + ret = set_binary_data(public_key->x, key_bytes, &key.x, &key.x_len); + if (ret != TEE_SUCCESS) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + ret = set_binary_data(public_key->y, key_bytes, &key.y, &key.y_len); + if (ret != TEE_SUCCESS) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + st = se050_ecc_gen_shared_secret(&se050_session->s_ctx, + kobject.keyId, &key, + secret, secret_len); + + if (st != kStatus_SSS_Success) + ret = TEE_ERROR_BAD_PARAMETERS; +exit: + if (!se050_ecc_keypair_from_nvm(private_key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + + free(key.x); + free(key.y); + + return ret; +} + +static TEE_Result sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + uint8_t *sig_der = NULL; + size_t sig_der_len = 0; + size_t key_bytes = 0; + size_t key_bits = 0; + uint8_t *p = NULL; + + res = ecc_get_key_size(key->curve, algo, &key_bytes, &key_bits); + if (res) { + if (res != TEE_ERROR_NOT_IMPLEMENTED) + goto exit; + + return sign_fallback(algo, key, msg, msg_len, sig, sig_len); + } + + /* allocate temporary buffer to retrieve the signature in DER format */ + sig_der_len = 2 * key_bytes + DER_SIGNATURE_SZ; + + sig_der = malloc(sig_der_len); + if (!sig_der) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto exit; + } + + /* truncate or pad the message as needed */ + res = ecc_prepare_msg(algo_tee2se050(algo), msg, &msg_len, &p); + if (res != TEE_SUCCESS) + goto exit; + + res = se050_inject_keypair(&kobject, key, key_bytes); + if (res != TEE_SUCCESS) + goto exit; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + algo_tee2se050(algo), + kMode_SSS_Sign); + if (st != kStatus_SSS_Success) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + st = sss_se05x_asymmetric_sign_digest(&ctx, p ? p : (uint8_t *)msg, + msg_len, sig_der, &sig_der_len); + if (st != kStatus_SSS_Success) { + EMSG("curve: %#"PRIx32, key->curve); + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + sss_se05x_signature_der2bin(sig_der, &sig_der_len); + + if (sig_der_len > *sig_len) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + memcpy(sig, sig_der, sig_der_len); + *sig_len = sig_der_len; +exit: + if (!se050_ecc_keypair_from_nvm(key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + + sss_se05x_asymmetric_context_free(&ctx); + + free(sig_der); + free(p); + + return res; +} + +static TEE_Result verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + uint8_t *signature = NULL; + size_t signature_len = sig_len + DER_SIGNATURE_SZ; + size_t key_bytes = 0; + size_t key_bits = 0; + uint8_t *p = NULL; + + res = ecc_get_key_size(key->curve, algo, &key_bytes, &key_bits); + if (res) { + if (res != TEE_ERROR_NOT_IMPLEMENTED) + goto exit; + + return verify_fallback(algo, key, msg, msg_len, sig, sig_len); + } + + /* truncate or pad the message as needed */ + res = ecc_prepare_msg(algo_tee2se050(algo), msg, &msg_len, &p); + if (res != TEE_SUCCESS) + goto exit; + + res = se050_inject_public_key(&kobject, key, key_bytes); + if (res != TEE_SUCCESS) + goto exit; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + algo_tee2se050(algo), + kMode_SSS_Verify); + if (st != kStatus_SSS_Success) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + signature = calloc(1, signature_len); + if (!signature) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto exit; + } + + st = sss_se05x_signature_bin2der(signature, &signature_len, + (uint8_t *)sig, sig_len); + if (st != kStatus_SSS_Success) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + st = sss_se05x_asymmetric_verify_digest(&ctx, p ? p : (uint8_t *)msg, + msg_len, (uint8_t *)signature, + signature_len); + if (st != kStatus_SSS_Success) + res = TEE_ERROR_SIGNATURE_INVALID; +exit: + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + sss_se05x_asymmetric_context_free(&ctx); + free(p); + free(signature); + + return res; +} + +static TEE_Result gen_keypair(struct ecc_keypair *key, size_t key_size) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_object_t k_object = { }; + TEE_Result ret = TEE_SUCCESS; + uint8_t kf[512] = { }; + uint32_t oid = 0; + uint64_t kid = 0; + size_t bytes = 0; + size_t bits = 0; + + ret = ecc_get_key_size(key->curve, 0, &bytes, &bits); + if (ret) { + if (ret != TEE_ERROR_NOT_IMPLEMENTED) + return ret; + + return gen_fallback(key, key_size); + } + + st = sss_se05x_key_object_init(&k_object, se050_kstore); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + st = se050_get_oid(&oid); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = sss_se05x_key_object_allocate_handle(&k_object, oid, + kSSS_KeyPart_Pair, + cipher_tee2se050(key->curve), + 0, + kKeyObject_Mode_Persistent); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + st = sss_se05x_key_store_generate_key(se050_kstore, &k_object, bits, + &se050_asym_policy); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + bytes = sizeof(kf); + st = se050_key_store_get_ecc_key_bin(se050_kstore, &k_object, kf, + &bytes); + if (st != kStatus_SSS_Success) { + sss_se05x_key_store_erase_key(se050_kstore, &k_object); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* skip the DER tag */ + crypto_bignum_bin2bn(kf + 1, bytes / 2, key->x); + crypto_bignum_bin2bn(kf + 1 + bytes / 2, bytes / 2, key->y); + + kid = se050_generate_private_key(oid); + crypto_bignum_bin2bn((uint8_t *)&kid, sizeof(kid), key->d); + key->curve = curve_se0502tee(k_object.curve_id); + if (key->curve != TEE_CRYPTO_ELEMENT_NONE) + return TEE_SUCCESS; + + EMSG("ecc key generation failed"); + sss_se05x_key_store_erase_key(se050_kstore, &k_object); + + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result do_gen_keypair(struct ecc_keypair *key, size_t size_bytes) +{ + return gen_keypair(key, size_bytes); +} + +static TEE_Result do_shared_secret(struct drvcrypt_secret_data *sdata) +{ + return shared_secret(sdata->key_priv, + sdata->key_pub, + sdata->secret.data, + &sdata->secret.length); +} + +static TEE_Result do_sign(struct drvcrypt_sign_data *sdata) +{ + return sign(sdata->algo, + sdata->key, + sdata->message.data, + sdata->message.length, + sdata->signature.data, + &sdata->signature.length); +} + +static TEE_Result do_verify(struct drvcrypt_sign_data *sdata) +{ + return verify(sdata->algo, + sdata->key, + sdata->message.data, + sdata->message.length, + sdata->signature.data, + sdata->signature.length); +} + +static TEE_Result do_alloc_keypair(struct ecc_keypair *s, uint32_t type, + size_t size_bits __unused) +{ + /* This driver only supports ECDH/ECDSA */ + if (type != TEE_TYPE_ECDSA_KEYPAIR && + type != TEE_TYPE_ECDH_KEYPAIR) + return TEE_ERROR_NOT_IMPLEMENTED; + + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->d)) + goto err; + if (!bn_alloc_max(&s->x)) + goto err; + if (!bn_alloc_max(&s->y)) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->d); + crypto_bignum_free(&s->x); + crypto_bignum_free(&s->y); + return TEE_ERROR_OUT_OF_MEMORY; +} + +static TEE_Result do_alloc_publickey(struct ecc_public_key *s, uint32_t type, + size_t size_bits __unused) +{ + /* This driver only supports ECDH/ECDSA */ + if (type != TEE_TYPE_ECDSA_PUBLIC_KEY && + type != TEE_TYPE_ECDH_PUBLIC_KEY) + return TEE_ERROR_NOT_IMPLEMENTED; + + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->x)) + goto err; + if (!bn_alloc_max(&s->y)) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->x); + crypto_bignum_free(&s->y); + return TEE_ERROR_OUT_OF_MEMORY; +} + +static void do_free_publickey(struct ecc_public_key *s) +{ + if (!s) + return; + + crypto_bignum_free(&s->x); + crypto_bignum_free(&s->y); +} + +static struct drvcrypt_ecc driver_ecc = { + .alloc_keypair = do_alloc_keypair, + .alloc_publickey = do_alloc_publickey, + .free_publickey = do_free_publickey, + .gen_keypair = do_gen_keypair, + .sign = do_sign, + .verify = do_verify, + .shared_secret = do_shared_secret, +}; + +static TEE_Result ecc_init(void) +{ + pub_ops = crypto_asym_get_ecc_public_ops(TEE_TYPE_ECDSA_PUBLIC_KEY); + if (!pub_ops) + return TEE_ERROR_GENERIC; + + pair_ops = crypto_asym_get_ecc_keypair_ops(TEE_TYPE_ECDSA_KEYPAIR); + if (!pair_ops) + return TEE_ERROR_GENERIC; + + /* This driver supports both ECDH and ECDSA */ + assert((pub_ops == + crypto_asym_get_ecc_public_ops(TEE_TYPE_ECDH_PUBLIC_KEY)) && + (pair_ops == + crypto_asym_get_ecc_keypair_ops(TEE_TYPE_ECDH_KEYPAIR))); + + return drvcrypt_register_ecc(&driver_ecc); +} + +driver_init_late(ecc_init); diff --git a/optee_os/core/drivers/crypto/se050/core/include/se050_cipher_algorithms.h b/optee_os/core/drivers/crypto/se050/core/include/se050_cipher_algorithms.h new file mode 100644 index 0000000..2fa3b33 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/include/se050_cipher_algorithms.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#ifndef SE050_CIPHER_ALGORITHMS_H_ +#define SE050_CIPHER_ALGORITHMS_H_ + +#include + +#if defined(CFG_NXP_SE05X_CTR_DRV) +TEE_Result se050_aes_ctr_allocate(void **ctx); +#else +static inline TEE_Result se050_aes_ctr_allocate(void **ctx __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif + +#endif /* SE050_CIPHER_ALGORITHMS_H_ */ diff --git a/optee_os/core/drivers/crypto/se050/core/rng.c b/optee_os/core/drivers/crypto/se050/core/rng.c new file mode 100644 index 0000000..bedcff8 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/rng.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include + +static TEE_Result do_rng_read(void *buf, size_t blen) +{ + sss_status_t status = kStatus_SSS_Success; + sss_se05x_rng_context_t rng = { }; + + sss_se05x_rng_context_init(&rng, se050_session); + status = sss_se05x_rng_get_random(&rng, buf, blen); + sss_se05x_rng_context_free(&rng); + + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +void plat_rng_init(void) +{ +} + +TEE_Result hw_get_random_bytes(void *buf, size_t blen) +{ + if (!buf) + return TEE_ERROR_BAD_PARAMETERS; + + return do_rng_read(buf, blen); +} diff --git a/optee_os/core/drivers/crypto/se050/core/rsa.c b/optee_os/core/drivers/crypto/se050/core/rsa.c new file mode 100644 index 0000000..475d2b9 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/rsa.c @@ -0,0 +1,933 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static sss_cipher_type_t oefid_cipher_type(void) +{ + switch (se050_get_oefid()) { + case SE050F_ID: + return kSSS_CipherType_RSA_CRT; + default: + return kSSS_CipherType_RSA; + } +} + +static bool oefid_keylen_supported(size_t bits) +{ + switch (se050_get_oefid()) { + case SE050F_ID: + return bits >= 2048; + default: + return true; + } +} + +static bool rsa_keypair_has_crt(struct rsa_keypair *key) +{ + if (key->p && crypto_bignum_num_bytes(key->p) && + key->q && crypto_bignum_num_bytes(key->q) && + key->qp && crypto_bignum_num_bytes(key->qp) && + key->dp && crypto_bignum_num_bytes(key->dp) && + key->dq && crypto_bignum_num_bytes(key->dq)) + return true; + + return false; +} + +static bool keypair_supported(struct rsa_keypair *key, sss_cipher_type_t ctype) +{ + if (se050_rsa_keypair_from_nvm(key)) + return true; + + if (ctype == kSSS_CipherType_RSA_CRT) + return rsa_keypair_has_crt(key); + + return true; +} + +static uint32_t tee2se050(uint32_t algo) +{ + switch (algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + return kAlgorithm_SSS_RSASSA_PKCS1_V1_5_SHA1; + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + return kAlgorithm_SSS_RSASSA_PKCS1_V1_5_SHA224; + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + return kAlgorithm_SSS_RSASSA_PKCS1_V1_5_SHA256; + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + return kAlgorithm_SSS_RSASSA_PKCS1_V1_5_SHA384; + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + return kAlgorithm_SSS_RSASSA_PKCS1_V1_5_SHA512; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + return kAlgorithm_SSS_RSASSA_PKCS1_PSS_MGF1_SHA1; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + return kAlgorithm_SSS_RSASSA_PKCS1_PSS_MGF1_SHA224; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + return kAlgorithm_SSS_RSASSA_PKCS1_PSS_MGF1_SHA256; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + return kAlgorithm_SSS_RSASSA_PKCS1_PSS_MGF1_SHA384; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + return kAlgorithm_SSS_RSASSA_PKCS1_PSS_MGF1_SHA512; + case TEE_ALG_RSAES_PKCS1_V1_5: + return kAlgorithm_SSS_RSAES_PKCS1_V1_5_SHA256; + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1: + return kAlgorithm_SSS_RSAES_PKCS1_OAEP_SHA1; + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224: + return kAlgorithm_SSS_RSAES_PKCS1_OAEP_SHA224; + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256: + return kAlgorithm_SSS_RSAES_PKCS1_OAEP_SHA256; + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384: + return kAlgorithm_SSS_RSAES_PKCS1_OAEP_SHA384; + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512: + return kAlgorithm_SSS_RSAES_PKCS1_OAEP_SHA512; + case TEE_ALG_RSA_NOPAD: + return kAlgorithm_SSS_RSASSA_NO_PADDING; +#ifdef CFG_CRYPTO_RSASSA_NA1 + case TEE_ALG_RSASSA_PKCS1_V1_5: + return kAlgorithm_SSS_RSASSA_PKCS1_V1_5_NO_HASH; +#endif + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5SHA1: + default: + return kAlgorithm_None; + } +} + +static bool bn_alloc_max(struct bignum **s) +{ + *s = crypto_bignum_allocate(4096); + + return *s; +} + +static TEE_Result set_binary_data(struct bignum *b, uint8_t **p, size_t *len) +{ + *len = crypto_bignum_num_bytes(b); + if (*len) { + *p = (uint8_t *)calloc(1, *len); + if (!*p) + return TEE_ERROR_OUT_OF_MEMORY; + crypto_bignum_bn2bin(b, *p); + } + return TEE_SUCCESS; +} + +static TEE_Result se050_inject_public_key(sss_se05x_object_t *k_object, + struct rsa_public_key *key) +{ + sss_status_t st = kStatus_SSS_Fail; + struct se050_rsa_keypub key_bin = { }; + uint32_t oid = 0; + + st = sss_se05x_key_object_init(k_object, se050_kstore); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + st = se050_get_oid(&oid); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + /* + * Keys 2048 and above MUST to be placed on persistent storage even + * though the keys will be deleted after the operation. This is a + * memory restriction in the secure element. + */ + st = sss_se05x_key_object_allocate_handle(k_object, oid, + kSSS_KeyPart_Public, + oefid_cipher_type(), 0, + kKeyObject_Mode_Persistent); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + set_binary_data(key->e, &key_bin.e, &key_bin.e_len); + set_binary_data(key->n, &key_bin.n, &key_bin.n_len); + st = se050_key_store_set_rsa_key_bin(se050_kstore, k_object, NULL, + &key_bin, key_bin.n_len * 8); + free(key_bin.n); + free(key_bin.e); + + if (st != kStatus_SSS_Success) { + sss_se05x_key_store_erase_key(se050_kstore, k_object); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result se050_inject_keypair(sss_se05x_object_t *k_object, + struct rsa_keypair *key) +{ + sss_status_t st = kStatus_SSS_Fail; + struct se050_rsa_keypair key_bin = { }; + uint32_t key_id = 0; + uint32_t oid = 0; + + st = sss_se05x_key_object_init(k_object, se050_kstore); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + key_id = se050_rsa_keypair_from_nvm(key); + if (key_id) { + st = sss_se05x_key_object_get_handle(k_object, key_id); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + return TEE_SUCCESS; + } + + st = se050_get_oid(&oid); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + /* Keys 2048 and above need to be placed on persistent storage */ + st = sss_se05x_key_object_allocate_handle(k_object, oid, + kSSS_KeyPart_Pair, + oefid_cipher_type(), 0, + kKeyObject_Mode_Persistent); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + set_binary_data(key->e, &key_bin.e, &key_bin.e_len); + set_binary_data(key->d, &key_bin.d, &key_bin.d_len); + set_binary_data(key->n, &key_bin.n, &key_bin.n_len); + set_binary_data(key->p, &key_bin.p, &key_bin.p_len); + set_binary_data(key->q, &key_bin.q, &key_bin.q_len); + set_binary_data(key->qp, &key_bin.qp, &key_bin.qp_len); + set_binary_data(key->dp, &key_bin.dp, &key_bin.dp_len); + set_binary_data(key->dq, &key_bin.dq, &key_bin.dq_len); + st = se050_key_store_set_rsa_key_bin(se050_kstore, k_object, + &key_bin, NULL, + crypto_bignum_num_bits(key->n)); + free(key_bin.e); + free(key_bin.d); + free(key_bin.n); + free(key_bin.p); + free(key_bin.q); + free(key_bin.qp); + free(key_bin.dp); + free(key_bin.dq); + + if (st != kStatus_SSS_Success) { + sss_se05x_key_store_erase_key(se050_kstore, k_object); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result decrypt_es(uint32_t algo, struct rsa_keypair *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + uint8_t *buf = NULL; + size_t buf_len = src_len; + + res = se050_inject_keypair(&kobject, key); + if (res) + return res; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + tee2se050(algo), + kMode_SSS_Decrypt); + if (st != kStatus_SSS_Success) { + if (!se050_rsa_keypair_from_nvm(key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* we don't know the size of the decrypted data, just the upper limit */ + buf = mempool_calloc(mempool_default, 1, buf_len); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + st = sss_se05x_asymmetric_decrypt(&ctx, src, src_len, buf, &buf_len); + if (st != kStatus_SSS_Success) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (buf_len > *dst_len) { + *dst_len = buf_len; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + *dst_len = buf_len; + memcpy(dst, buf, buf_len); +out: + if (!se050_rsa_keypair_from_nvm(key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + + sss_se05x_asymmetric_context_free(&ctx); + mempool_free(mempool_default, buf); + + return res; +} + +static TEE_Result encrypt_es(uint32_t algo, struct rsa_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + + if (*dst_len < crypto_bignum_num_bytes(key->n)) { + *dst_len = crypto_bignum_num_bytes(key->n); + return TEE_ERROR_SHORT_BUFFER; + } + + if (se050_inject_public_key(&kobject, key)) + return TEE_ERROR_BAD_PARAMETERS; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + tee2se050(algo), + kMode_SSS_Encrypt); + if (st != kStatus_SSS_Success) { + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + return TEE_ERROR_BAD_PARAMETERS; + } + + st = sss_se05x_asymmetric_encrypt(&ctx, src, src_len, dst, dst_len); + if (st != kStatus_SSS_Success) + res = TEE_ERROR_BAD_PARAMETERS; + + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + sss_se05x_asymmetric_context_free(&ctx); + + return res; +} + +static TEE_Result decrypt_nopad(struct rsa_keypair *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + uint8_t *buf = NULL; + size_t offset = 0; + size_t blen = 0; + size_t rsa_len = 0; + + res = se050_inject_keypair(&kobject, key); + if (res) + return res; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + kAlgorithm_SSS_RSASSA_NO_PADDING, + kMode_SSS_Decrypt); + if (st != kStatus_SSS_Success) { + if (!se050_rsa_keypair_from_nvm(key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + return TEE_ERROR_BAD_PARAMETERS; + } + + blen = CFG_CORE_BIGNUM_MAX_BITS / 8; + buf = mempool_calloc(mempool_default, 1, blen); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + rsa_len = crypto_bignum_num_bytes(key->n); + memcpy(buf + rsa_len - src_len, src, src_len); + + st = sss_se05x_asymmetric_decrypt(&ctx, buf, rsa_len, buf, &blen); + if (st != kStatus_SSS_Success) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* Remove the zero-padding (leave one zero if buff is all zeroes) */ + offset = 0; + while ((offset < blen - 1) && (buf[offset] == 0)) + offset++; + + if (*dst_len < blen - offset) { + *dst_len = blen - offset; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + *dst_len = blen - offset; + memcpy(dst, buf + offset, *dst_len); +out: + mempool_free(mempool_default, buf); + if (!se050_rsa_keypair_from_nvm(key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + + sss_se05x_asymmetric_context_free(&ctx); + + return res; +} + +static TEE_Result encrypt_nopad(struct rsa_public_key *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + uint8_t *buf = NULL; + size_t offset = 0; + size_t blen = 0; + size_t rsa_len = 0; + + if (se050_inject_public_key(&kobject, key)) + return TEE_ERROR_BAD_PARAMETERS; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + kAlgorithm_SSS_RSASSA_NO_PADDING, + kMode_SSS_Encrypt); + if (st != kStatus_SSS_Success) { + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + return TEE_ERROR_BAD_PARAMETERS; + } + + blen = CFG_CORE_BIGNUM_MAX_BITS / 8; + buf = mempool_calloc(mempool_default, 1, blen); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + rsa_len = crypto_bignum_num_bytes(key->n); + memcpy(buf + rsa_len - src_len, src, src_len); + + st = sss_se05x_asymmetric_encrypt(&ctx, buf, rsa_len, buf, &blen); + if (st != kStatus_SSS_Success) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* Remove the zero-padding (leave one zero if buff is all zeroes) */ + offset = 0; + while ((offset < blen - 1) && (buf[offset] == 0)) + offset++; + + if (*dst_len < blen - offset) { + *dst_len = blen - offset; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + *dst_len = blen - offset; + memcpy(dst, buf + offset, *dst_len); +out: + mempool_free(mempool_default, buf); + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + sss_se05x_asymmetric_context_free(&ctx); + + return res; +} + +static TEE_Result sign_ssa(uint32_t algo, struct rsa_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + + if (*sig_len < crypto_bignum_num_bytes(key->n)) { + *sig_len = crypto_bignum_num_bytes(key->n); + return TEE_ERROR_SHORT_BUFFER; + } + + res = se050_inject_keypair(&kobject, key); + if (res) + return res; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + tee2se050(algo), kMode_SSS_Sign); + if (st != kStatus_SSS_Success) { + if (!se050_rsa_keypair_from_nvm(key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + return TEE_ERROR_BAD_PARAMETERS; + } + + st = sss_se05x_asymmetric_sign_digest(&ctx, (uint8_t *)msg, msg_len, + sig, sig_len); + if (st != kStatus_SSS_Success) + res = TEE_ERROR_BAD_PARAMETERS; + + if (!se050_rsa_keypair_from_nvm(key)) + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + + sss_se05x_asymmetric_context_free(&ctx); + + return res; +} + +static TEE_Result verify_ssa(uint32_t algo, struct rsa_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_asymmetric_t ctx = { }; + sss_se05x_object_t kobject = { }; + TEE_Result res = TEE_SUCCESS; + + if (se050_inject_public_key(&kobject, key)) + return TEE_ERROR_BAD_PARAMETERS; + + st = sss_se05x_asymmetric_context_init(&ctx, se050_session, &kobject, + tee2se050(algo), + kMode_SSS_Verify); + if (st != kStatus_SSS_Success) { + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + return TEE_ERROR_BAD_PARAMETERS; + } + + st = sss_se05x_asymmetric_verify_digest(&ctx, (uint8_t *)msg, msg_len, + (uint8_t *)sig, sig_len); + if (st != kStatus_SSS_Success) + res = TEE_ERROR_SIGNATURE_INVALID; + + sss_se05x_key_store_erase_key(se050_kstore, &kobject); + sss_se05x_asymmetric_context_free(&ctx); + + return res; +} + +static TEE_Result do_alloc_keypair(struct rsa_keypair *s, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->e)) + return TEE_ERROR_OUT_OF_MEMORY; + if (!bn_alloc_max(&s->d)) + goto err; + if (!bn_alloc_max(&s->n)) + goto err; + if (!bn_alloc_max(&s->p)) + goto err; + if (!bn_alloc_max(&s->q)) + goto err; + if (!bn_alloc_max(&s->qp)) + goto err; + if (!bn_alloc_max(&s->dp)) + goto err; + if (!bn_alloc_max(&s->dq)) + goto err; + + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->e); + crypto_bignum_free(&s->d); + crypto_bignum_free(&s->n); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->q); + crypto_bignum_free(&s->qp); + crypto_bignum_free(&s->dp); + crypto_bignum_free(&s->dq); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +static TEE_Result do_alloc_publickey(struct rsa_public_key *s, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->e)) + return TEE_ERROR_OUT_OF_MEMORY; + if (!bn_alloc_max(&s->n)) { + crypto_bignum_free(&s->e); + return TEE_ERROR_OUT_OF_MEMORY; + } + + return TEE_SUCCESS; +} + +static void do_free_publickey(struct rsa_public_key *s) +{ + if (s) { + crypto_bignum_free(&s->n); + crypto_bignum_free(&s->e); + } +} + +static void do_free_keypair(struct rsa_keypair *s) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_object_t k_object = { }; + uint32_t key_id = 0; + + if (!s) + return; + + key_id = se050_rsa_keypair_from_nvm(s); + if (key_id) { + st = sss_se05x_key_object_get_handle(&k_object, key_id); + if (st == kStatus_SSS_Success) + sss_se05x_key_store_erase_key(se050_kstore, &k_object); + } + + crypto_bignum_free(&s->e); + crypto_bignum_free(&s->d); + crypto_bignum_free(&s->n); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->q); + crypto_bignum_free(&s->qp); + crypto_bignum_free(&s->dp); + crypto_bignum_free(&s->dq); +} + +static TEE_Result do_gen_keypair(struct rsa_keypair *key, size_t kb) +{ + sss_status_t st = kStatus_SSS_Fail; + sss_se05x_object_t k_object = { }; + uint32_t oid = 0; + uint64_t kid = 0; + uint8_t k[2048] = { 0 }; + uint8_t *n = NULL; + uint8_t *e = NULL; + size_t n_len = 0; + size_t e_len = 0; + size_t k_len = sizeof(k); + + if (!oefid_keylen_supported(kb)) { + if (!IS_ENABLED(CFG_NXP_SE05X_RSA_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + DMSG("se050: debug: RSA software fallback: KEYGEN"); + return sw_crypto_acipher_gen_rsa_key(key, kb); + } + + st = sss_se05x_key_object_init(&k_object, se050_kstore); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + st = se050_get_oid(&oid); + if (st != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + st = sss_se05x_key_object_allocate_handle(&k_object, oid, + kSSS_KeyPart_Pair, + oefid_cipher_type(), 0, + kKeyObject_Mode_Persistent); + if (st != kStatus_SSS_Success) + return TEE_ERROR_BAD_PARAMETERS; + + st = sss_se05x_key_store_generate_key(se050_kstore, &k_object, kb, + &se050_asym_policy); + if (st != kStatus_SSS_Success) + goto error; + + st = sss_se05x_key_store_get_key(se050_kstore, &k_object, k, &k_len, + &kb); + if (st != kStatus_SSS_Success) + goto error; + + st = sss_util_asn1_rsa_parse_public(k, k_len, &n, &n_len, &e, &e_len); + if (st != kStatus_SSS_Success) + goto error; + + crypto_bignum_bin2bn(n, n_len, key->n); + crypto_bignum_bin2bn(e, e_len, key->e); + kid = se050_generate_private_key(oid); + crypto_bignum_bin2bn((uint8_t *)&kid, sizeof(kid), (key->d)); + crypto_bignum_bin2bn((uint8_t *)&oid, sizeof(oid), key->p); + crypto_bignum_bin2bn((uint8_t *)&oid, sizeof(oid), key->q); + crypto_bignum_bin2bn((uint8_t *)&oid, sizeof(oid), key->qp); + crypto_bignum_bin2bn((uint8_t *)&oid, sizeof(oid), key->dp); + crypto_bignum_bin2bn((uint8_t *)&oid, sizeof(oid), key->dq); + free(n); + free(e); + + return TEE_SUCCESS; +error: + sss_se05x_key_store_erase_key(se050_kstore, &k_object); + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result encrypt_fallback(struct drvcrypt_rsa_ed *p) +{ + if (!IS_ENABLED(CFG_NXP_SE05X_RSA_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + switch (p->rsa_id) { + case DRVCRYPT_RSA_NOPAD: + DMSG("se050: debug: RSA software fallback: ENCRYPT_NOPAD"); + return sw_crypto_acipher_rsanopad_encrypt(p->key.key, + p->message.data, + p->message.length, + p->cipher.data, + &p->cipher.length); + + case DRVCRYPT_RSA_OAEP: + case DRVCRYPT_RSA_PKCS_V1_5: + case DRVCRYPT_RSASSA_PKCS_V1_5: + case DRVCRYPT_RSASSA_PSS: + default: + DMSG("se050: debug: RSA software fallback: ENCRYPT_ES"); + return sw_crypto_acipher_rsaes_encrypt(p->algo, + p->key.key, + p->label.data, + p->label.length, + p->message.data, + p->message.length, + p->cipher.data, + &p->cipher.length); + } +} + +static TEE_Result do_encrypt(struct drvcrypt_rsa_ed *rsa_data) +{ + if (!oefid_keylen_supported(rsa_data->key.n_size * 8)) + return encrypt_fallback(rsa_data); + + switch (rsa_data->rsa_id) { + case DRVCRYPT_RSA_NOPAD: + case DRVCRYPT_RSASSA_PSS: + case DRVCRYPT_RSASSA_PKCS_V1_5: + return encrypt_nopad(rsa_data->key.key, + rsa_data->message.data, + rsa_data->message.length, + rsa_data->cipher.data, + &rsa_data->cipher.length); + + case DRVCRYPT_RSA_PKCS_V1_5: + return encrypt_es(TEE_ALG_RSAES_PKCS1_V1_5, + rsa_data->key.key, + rsa_data->message.data, + rsa_data->message.length, + rsa_data->cipher.data, + &rsa_data->cipher.length); + + case DRVCRYPT_RSA_OAEP: + if (rsa_data->hash_algo != TEE_ALG_SHA1) + return encrypt_fallback(rsa_data); + + return encrypt_es(TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1, + rsa_data->key.key, + rsa_data->message.data, + rsa_data->message.length, + rsa_data->cipher.data, + &rsa_data->cipher.length); + + default: + break; + } + + return encrypt_fallback(rsa_data); +} + +static TEE_Result decrypt_fallback(struct drvcrypt_rsa_ed *p) +{ + if (!IS_ENABLED(CFG_NXP_SE05X_RSA_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (se050_rsa_keypair_from_nvm(p->key.key)) + return TEE_ERROR_NOT_IMPLEMENTED; + + switch (p->rsa_id) { + case DRVCRYPT_RSA_NOPAD: + DMSG("se050: debug: RSA software fallback: DECRYPT_NOPAD"); + return sw_crypto_acipher_rsanopad_decrypt(p->key.key, + p->cipher.data, + p->cipher.length, + p->message.data, + &p->message.length); + + case DRVCRYPT_RSA_OAEP: + case DRVCRYPT_RSA_PKCS_V1_5: + case DRVCRYPT_RSASSA_PKCS_V1_5: + case DRVCRYPT_RSASSA_PSS: + default: + DMSG("se050: debug: RSA software fallback: DECRYPT_ES"); + return sw_crypto_acipher_rsaes_decrypt(p->algo, + p->key.key, + p->label.data, + p->label.length, + p->cipher.data, + p->cipher.length, + p->message.data, + &p->message.length); + } +} + +static TEE_Result do_decrypt(struct drvcrypt_rsa_ed *rsa_data) +{ + if (!oefid_keylen_supported(rsa_data->key.n_size * 8)) + return decrypt_fallback(rsa_data); + + if (!keypair_supported(rsa_data->key.key, oefid_cipher_type())) + return decrypt_fallback(rsa_data); + + switch (rsa_data->rsa_id) { + case DRVCRYPT_RSA_NOPAD: + case DRVCRYPT_RSASSA_PSS: + case DRVCRYPT_RSASSA_PKCS_V1_5: + return decrypt_nopad(rsa_data->key.key, + rsa_data->cipher.data, + rsa_data->cipher.length, + rsa_data->message.data, + &rsa_data->message.length); + + case DRVCRYPT_RSA_PKCS_V1_5: + return decrypt_es(TEE_ALG_RSAES_PKCS1_V1_5, + rsa_data->key.key, + rsa_data->cipher.data, + rsa_data->cipher.length, + rsa_data->message.data, + &rsa_data->message.length); + + case DRVCRYPT_RSA_OAEP: + if (rsa_data->hash_algo != TEE_ALG_SHA1) + return decrypt_fallback(rsa_data); + + return decrypt_es(TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1, + rsa_data->key.key, + rsa_data->cipher.data, + rsa_data->cipher.length, + rsa_data->message.data, + &rsa_data->message.length); + + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static TEE_Result sign_ssa_fallback(struct drvcrypt_rsa_ssa *p) +{ + if (!IS_ENABLED(CFG_NXP_SE05X_RSA_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (se050_rsa_keypair_from_nvm(p->key.key)) + return TEE_ERROR_NOT_IMPLEMENTED; + + DMSG("se050: debug: RSA software fallback: SIGN"); + return sw_crypto_acipher_rsassa_sign(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + &p->signature.length); +} + +static TEE_Result do_ssa_sign(struct drvcrypt_rsa_ssa *ssa_data) +{ + if (!oefid_keylen_supported(ssa_data->key.n_size * 8)) + return sign_ssa_fallback(ssa_data); + + if (!keypair_supported(ssa_data->key.key, oefid_cipher_type())) + return sign_ssa_fallback(ssa_data); + + /* PKCS1_PSS_MGF1 padding limitations */ + switch (ssa_data->algo) { + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + if (ssa_data->key.n_size * 8 <= 512) + return sign_ssa_fallback(ssa_data); + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + if (ssa_data->key.n_size * 8 <= 1024) + return sign_ssa_fallback(ssa_data); + break; + default: + break; + } + + return sign_ssa(ssa_data->algo, + ssa_data->key.key, + ssa_data->message.data, + ssa_data->message.length, + ssa_data->signature.data, + &ssa_data->signature.length); +} + +static TEE_Result verify_ssa_fallback(struct drvcrypt_rsa_ssa *p) +{ + if (!IS_ENABLED(CFG_NXP_SE05X_RSA_DRV_FALLBACK)) + return TEE_ERROR_NOT_IMPLEMENTED; + + DMSG("se050: debug: RSA software fallback: VERIFY"); + return sw_crypto_acipher_rsassa_verify(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + p->signature.length); +} + +static TEE_Result do_ssa_verify(struct drvcrypt_rsa_ssa *ssa_data) +{ + if (!oefid_keylen_supported(ssa_data->key.n_size * 8)) + return verify_ssa_fallback(ssa_data); + + /* PKCS1_PSS_MGF1 padding limitations */ + switch (ssa_data->algo) { + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + if (ssa_data->key.n_size * 8 <= 512) + return verify_ssa_fallback(ssa_data); + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + if (ssa_data->key.n_size * 8 <= 1024) + return verify_ssa_fallback(ssa_data); + break; + default: + break; + } + + return verify_ssa(ssa_data->algo, + ssa_data->key.key, + ssa_data->message.data, + ssa_data->message.length, + ssa_data->signature.data, + ssa_data->signature.length); +} + +static const struct drvcrypt_rsa driver_rsa = { + .alloc_keypair = do_alloc_keypair, + .alloc_publickey = do_alloc_publickey, + .free_publickey = do_free_publickey, + .free_keypair = do_free_keypair, + .gen_keypair = do_gen_keypair, + .encrypt = do_encrypt, + .decrypt = do_decrypt, + .optional.ssa_sign = do_ssa_sign, + .optional.ssa_verify = do_ssa_verify, +}; + +static TEE_Result rsa_init(void) +{ + return drvcrypt_register_rsa(&driver_rsa); +} + +driver_init_late(rsa_init); diff --git a/optee_os/core/drivers/crypto/se050/core/scp03.c b/optee_os/core/drivers/crypto/se050/core/scp03.c new file mode 100644 index 0000000..153e5c8 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/scp03.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include + +TEE_Result crypto_se_enable_scp03(bool rotate_keys) +{ + sss_status_t status = kStatus_SSS_Success; + + status = se050_enable_scp03(se050_session); + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + if (rotate_keys) { + if (IS_ENABLED(CFG_CORE_SE05X_SCP03_PROVISION)) { + status = se050_rotate_scp03_keys(&se050_ctx); + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; + } + return TEE_ERROR_BAD_PARAMETERS; + } + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/crypto/se050/core/storage.c b/optee_os/core/drivers/crypto/se050/core/storage.c new file mode 100644 index 0000000..0f937b8 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/storage.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +TEE_Result crypto_storage_obj_del(struct tee_obj *o) +{ + sss_status_t status = kStatus_SSS_Success; + uint32_t val = SE050_KEY_WATERMARK; + TEE_Result ret = TEE_ERROR_GENERIC; + sss_se05x_object_t k_object = { }; + uint8_t *data = NULL; + uint8_t *p = NULL; + bool found = false; + size_t len = 0; + + if (!o) + return TEE_ERROR_BAD_PARAMETERS; + + len = o->info.dataSize; + + /* Supported keys (ECC/RSA) require less than 4KB of storage */ + if (len > SMALL_PAGE_SIZE || len <= sizeof(uint64_t)) + return TEE_SUCCESS; + + data = calloc(1, len); + if (!data) + return TEE_ERROR_OUT_OF_MEMORY; + + /* Read the object into memory */ + ret = o->pobj->fops->read(o->fh, o->info.dataPosition, + data, NULL, &len); + if (ret) { + EMSG("se05x: can not read the object prior removal"); + free(data); + goto out; + } + + /* Scan the object for the watermark */ + p = data; + while (len >= sizeof(uint32_t) && !found) { + if (memcmp(p, &val, sizeof(val)) != 0) { + p++; + len--; + continue; + } + found = true; + } + + if (!found) { + free(data); + return TEE_SUCCESS; + } + + /* Retrieve the object identifier */ + p = p - 4; + memcpy((void *)&val, p, sizeof(val)); + free(data); + + if (val < OID_MIN || val > OID_MAX) + return TEE_SUCCESS; + + status = sss_se05x_key_object_init(&k_object, se050_kstore); + if (status != kStatus_SSS_Success) { + ret = TEE_ERROR_BAD_STATE; + goto out; + } + + status = sss_se05x_key_object_get_handle(&k_object, val); + if (status != kStatus_SSS_Success) { + EMSG("se05x: can not communicate with the secure element"); + ret = TEE_ERROR_BAD_STATE; + goto out; + } + + status = sss_se05x_key_store_erase_key(se050_kstore, &k_object); + if (status != kStatus_SSS_Success) { + EMSG("se05x: can not communicate with the secure element"); + ret = TEE_ERROR_BAD_STATE; + goto out; + } + +out: + /* + * Users can delete the SE05X NVM objects during boot using a built + * time configuration flag (CFG_CORE_SE05X_INIT_NVM). + * + * This could cause the deletion of the secure storage objects holding + * references to those IDs via crypto_storage_obj_del() to fail, leaving + * broken links in the file system. + * + * Therefore we only permit this call to block the deletion upon an + * additional specific config. + */ + if (ret && IS_ENABLED(CFG_CORE_SE05X_BLOCK_OBJ_DEL_ON_ERROR)) + return ret; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/crypto/se050/core/sub.mk b/optee_os/core/drivers/crypto/se050/core/sub.mk new file mode 100644 index 0000000..615579e --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/core/sub.mk @@ -0,0 +1,15 @@ +include ${CFG_NXP_SE05X_PLUG_AND_TRUST}/cflags.mk + +incdirs_ext-y += ${CFG_NXP_SE05X_PLUG_AND_TRUST}/optee_lib/include +incdirs-y += ../adaptors/include +incdirs-y += include + +srcs-y += storage.c +srcs-$(CFG_NXP_SE05X_RSA_DRV) += rsa.c +srcs-$(CFG_NXP_SE05X_ECC_DRV) += ecc.c +srcs-$(CFG_NXP_SE05X_CTR_DRV) += ctr.c +srcs-$(CFG_NXP_SE05X_DIEID_DRV) += die_id.c +srcs-$(CFG_NXP_SE05X_RNG_DRV) += rng.c +srcs-$(CFG_NXP_SE05X_CIPHER_DRV) += cipher.c +srcs-$(CFG_NXP_SE05X_SCP03_DRV) += scp03.c +srcs-$(CFG_NXP_SE05X_APDU_DRV) += apdu.c diff --git a/optee_os/core/drivers/crypto/se050/crypto.mk b/optee_os/core/drivers/crypto/se050/crypto.mk new file mode 100644 index 0000000..473cd82 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/crypto.mk @@ -0,0 +1,104 @@ +ifeq ($(CFG_NXP_SE05X),y) +# Enable the crypto driver +$(call force,CFG_CRYPTO_DRIVER,y) +CFG_CRYPTO_DRIVER_DEBUG ?= 0 + +# SE050 initialization +# Some secure elements can only be accessed over an SCP03 enabled session. +# Some of the NXP SE05X devices fall in this category (i.e NXP SE050F). +# Only enable this configuration to support those systems. +CFG_CORE_SCP03_ONLY ?= n +# Rotate the SCP03 keys during SCP03 init (does not require user intervention). +# CAUTION: the provisioning configuration chosen might require a stable HUK. +CFG_CORE_SE05X_SCP03_PROVISION_ON_INIT ?= n +# Rotate the SCP03 keys via PTA (request from Normal World). +CFG_CORE_SE05X_SCP03_PROVISION ?= n +# The Provision request will rotate the SCP03 keys back to its factory settings. +CFG_CORE_SE05X_SCP03_PROVISION_WITH_FACTORY_KEYS ?= n +# CAUTION: Leaks the SCP03 keys that are going to be programmed on the device's +# NVM during a provisioning operation. +CFG_CORE_SE05X_DISPLAY_SCP03_KEYS ?= n +# Displays the SE050 device information on the console at boot (i.e. OEFID) +CFG_CORE_SE05X_DISPLAY_INFO ?= y +# Enables SCP03 protocol during boot (does not require user intervention) +CFG_CORE_SE05X_SCP03_EARLY ?= y +# CAUTION: Deletes all persistent storage (keys/certs) from the SE05X at boot +CFG_CORE_SE05X_INIT_NVM ?= n +# Prevents the deletion of the secure storage object holding a reference to a +# Secure Element (SE) Non Volatile Memory object unless there is explicit +# confirmation from the SE that the NVM object has been removed. +CFG_CORE_SE05X_BLOCK_OBJ_DEL_ON_ERROR ?= n +# Select the SE05X applet version for aligning the built-in features ++CFG_CORE_SE05X_VER ?= 03_XX + +# I2C bus baudrate (depends on SoC) +CFG_CORE_SE05X_BAUDRATE ?= 3400000 +# I2C bus [0..2] (depends on board) +CFG_CORE_SE05X_I2C_BUS ?= 2 +# I2C access via REE after TEE boot +CFG_CORE_SE05X_I2C_TRAMPOLINE ?= y + +# Extra stacks required to support the Plug and Trust external library +ifeq ($(shell test $(CFG_STACK_THREAD_EXTRA) -lt 8192; echo $$?), 0) +$(error Error: SE050 requires CFG_STACK_THREAD_EXTRA at least 8192) +endif +ifeq ($(shell test $(CFG_STACK_TMP_EXTRA) -lt 8192; echo $$?), 0) +$(error Error: SE050 requires CFG_STACK_TMP_EXTRA at least 8192) +endif + +# SE05X Die Identifier +CFG_NXP_SE05X_DIEID_DRV ?= y + +# Allow a secure client to enable the SCP03 session +CFG_NXP_SE05X_SCP03_DRV ?= y +ifeq ($(CFG_NXP_SE05X_SCP03_DRV),y) +$(call force,CFG_SCP03_PTA,y,Mandated by CFG_NXP_SE05X_SCP03) +endif + +# Allow a secure client to send APDU raw frames +CFG_NXP_SE05X_APDU_DRV ?= y +ifeq ($(CFG_NXP_SE05X_APDU_DRV),y) +$(call force,CFG_APDU_PTA,y,Mandated by CFG_NXP_SE05X_APDU) +endif + +# Random Number Generator +CFG_NXP_SE05X_RNG_DRV ?= y +ifeq ($(CFG_NXP_SE05X_RNG_DRV),y) +$(call force,CFG_WITH_SOFTWARE_PRNG,n) +endif + +se050-one-enabled = $(call cfg-one-enabled, \ + $(foreach v,$(1), CFG_NXP_SE05X_$(v)_DRV)) +# Asymmetric ciphers +CFG_NXP_SE05X_RSA_DRV ?= y +CFG_NXP_SE05X_RSA_DRV_FALLBACK ?= n +CFG_NXP_SE05X_ECC_DRV ?= y +CFG_NXP_SE05X_ECC_DRV_FALLBACK ?= n +$(call force,CFG_NXP_SE05X_ACIPHER_DRV,$(call se050-one-enabled,RSA ECC)) + +# Asymmetric driver +ifeq ($(CFG_NXP_SE05X_ACIPHER_DRV),y) +$(call force,CFG_CRYPTO_DRV_ACIPHER,y,Mandated by CFG_NXP_SE05X_ACIPHER_DRV) +endif + +# Asymmetric ciphers configuration +# - RSA +ifeq ($(CFG_NXP_SE05X_RSA_DRV),y) +$(call force,CFG_CRYPTO_DRV_RSA,y) +CFG_CRYPTO_RSASSA_NA1 ?= y +endif +# - ECC +ifeq ($(CFG_NXP_SE05X_ECC_DRV),y) +$(call force,CFG_CRYPTO_DRV_ECC,y) +endif + +# Symmetric ciphers +CFG_NXP_SE05X_CTR_DRV ?= y +$(call force,CFG_NXP_SE05X_CIPHER_DRV,$(call se050-one-enabled,CTR)) + +# Symmetric driver +ifeq ($(CFG_NXP_SE05X_CIPHER_DRV),y) +$(call force,CFG_CRYPTO_DRV_CIPHER,y,Mandated by CFG_NXP_SE05X_CIPHER_DRV) +endif + +endif # CFG_NXP_SE05X diff --git a/optee_os/core/drivers/crypto/se050/glue/i2c.c b/optee_os/core/drivers/crypto/se050/glue/i2c.c new file mode 100644 index 0000000..aa05aec --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/glue/i2c.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include + +static TEE_Result (*transfer)(struct rpc_i2c_request *req, size_t *bytes); + +static int i2c_transfer(uint8_t *buffer, int len, enum rpc_i2c_mode mode) +{ + struct rpc_i2c_request request = { + .bus = CFG_CORE_SE05X_I2C_BUS, + .chip = SMCOM_I2C_ADDRESS >> 1, + .mode = mode, + .buffer = buffer, + .buffer_len = len, + .flags = 0, + }; + size_t bytes = 0; + int retry = 5; + + do { + if ((*transfer)(&request, &bytes) == TEE_SUCCESS) + return bytes; + } while (--retry); + + return -1; +} + +int glue_i2c_read(uint8_t *buffer, int len) +{ + return i2c_transfer(buffer, len, RPC_I2C_MODE_READ); +} + +int glue_i2c_write(uint8_t *buffer, int len) +{ + return i2c_transfer(buffer, len, RPC_I2C_MODE_WRITE); +} + +int glue_i2c_init(void) +{ + if (transfer == rpc_io_i2c_transfer) + return 0; + + transfer = native_i2c_transfer; + + return native_i2c_init(); +} + +static TEE_Result load_trampoline(void) +{ + if (IS_ENABLED(CFG_CORE_SE05X_I2C_TRAMPOLINE)) + transfer = rpc_io_i2c_transfer; + + return TEE_SUCCESS; +} + +boot_final(load_trampoline); diff --git a/optee_os/core/drivers/crypto/se050/glue/i2c_imx.c b/optee_os/core/drivers/crypto/se050/glue/i2c_imx.c new file mode 100644 index 0000000..4a4add3 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/glue/i2c_imx.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include + +TEE_Result native_i2c_transfer(struct rpc_i2c_request *req, + size_t *bytes) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + if (req->mode == RPC_I2C_MODE_READ) + ret = imx_i2c_read(req->bus, req->chip, req->buffer, + req->buffer_len); + else + ret = imx_i2c_write(req->bus, req->chip, req->buffer, + req->buffer_len); + + if (!ret) + *bytes = req->buffer_len; + + return ret; +} + +int native_i2c_init(void) +{ + if (imx_i2c_init(CFG_CORE_SE05X_I2C_BUS, CFG_CORE_SE05X_BAUDRATE)) + return -1; + + if (imx_i2c_probe(CFG_CORE_SE05X_I2C_BUS, SMCOM_I2C_ADDRESS >> 1)) + return -1; + + return 0; +} diff --git a/optee_os/core/drivers/crypto/se050/glue/i2c_stm32.c b/optee_os/core/drivers/crypto/se050/glue/i2c_stm32.c new file mode 100644 index 0000000..c897d56 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/glue/i2c_stm32.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022 - All Rights Reserved + * Author: Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static_assert(CFG_CORE_SE05X_I2C_BUS < 10); +static struct i2c_handle_s i2c; + +TEE_Result native_i2c_transfer(struct rpc_i2c_request *req, size_t *bytes) +{ + if (req->mode == RPC_I2C_MODE_READ) { + if (stm32_i2c_master_receive(&i2c, req->chip << 1, req->buffer, + req->buffer_len, 25)) + return TEE_ERROR_GENERIC; + } else { + if (stm32_i2c_master_transmit(&i2c, req->chip << 1, req->buffer, + req->buffer_len, 25)) + return TEE_ERROR_GENERIC; + } + + *bytes = req->buffer_len; + + return TEE_SUCCESS; +} + +static int dt_i2c_bus_config(struct stm32_i2c_init_s *init, + struct pinctrl_state **pinctrl_active, + struct pinctrl_state **pinctrl_sleep) +{ + const fdt32_t *cuint = NULL; + const char *path = NULL; + char bus[6] = { }; + void *fdt = NULL; + int node = 0; + + fdt = get_embedded_dt(); + if (!fdt) + return -FDT_ERR_NOTFOUND; + + snprintf(bus, sizeof(bus), "i2c%d", CFG_CORE_SE05X_I2C_BUS); + + path = fdt_get_alias(fdt, bus); + if (!path) + return -FDT_ERR_NOTFOUND; + + node = fdt_path_offset(fdt, path); + if (node < 0) + return -FDT_ERR_NOTFOUND; + + cuint = fdt_getprop(fdt, node, "clock-frequency", NULL); + if (cuint && fdt32_to_cpu(*cuint) != CFG_CORE_SE05X_BAUDRATE) + IMSG("SE05X ignoring CFG_CORE_SE05X_BAUDRATE, use DTB"); + else if (I2C_STANDARD_RATE != CFG_CORE_SE05X_BAUDRATE) + IMSG("SE05x ignoring CFG_CORE_SE05X_BAUDRATE, use built-in"); + + return stm32_i2c_get_setup_from_fdt(fdt, node, init, pinctrl_active, + pinctrl_sleep); +} + +int native_i2c_init(void) +{ + struct stm32_i2c_init_s i2c_init = { }; + + /* No need to re-initialize */ + if (i2c.base.pa) + return 0; + + /* Support only one device on the platform */ + if (dt_i2c_bus_config(&i2c_init, &i2c.pinctrl, &i2c.pinctrl_sleep)) + return -1; + + /* Probe the device */ + i2c_init.own_address1 = SMCOM_I2C_ADDRESS; + i2c_init.digital_filter_coef = 0; + i2c_init.analog_filter = true; + + stm32_i2c_resume(&i2c); + + return stm32_i2c_init(&i2c, &i2c_init); +} diff --git a/optee_os/core/drivers/crypto/se050/glue/include/i2c_native.h b/optee_os/core/drivers/crypto/se050/glue/include/i2c_native.h new file mode 100644 index 0000000..19238c4 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/glue/include/i2c_native.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2022 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#ifndef I2C_NATIVE_H_ +#define I2C_NATIVE_H_ + +#include + +TEE_Result native_i2c_transfer(struct rpc_i2c_request *req, + size_t *bytes); +int native_i2c_init(void); + +#endif diff --git a/optee_os/core/drivers/crypto/se050/glue/user.c b/optee_os/core/drivers/crypto/se050/glue/user.c new file mode 100644 index 0000000..9c762de --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/glue/user.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ +#include +#include +#include +#include +#include +#include + +sss_status_t glue_mac_context_init(void **mac, const uint8_t *key, size_t len) +{ + if (crypto_mac_alloc_ctx(mac, TEE_ALG_AES_CMAC)) + return kStatus_SSS_Fail; + + if (crypto_mac_init(*mac, key, len)) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +void glue_mac_context_free(void *mac) +{ + crypto_mac_free_ctx(mac); +} + +sss_status_t glue_mac_update(void *mac, const uint8_t *msg, size_t len) +{ + if (crypto_mac_update(mac, msg, len)) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +sss_status_t glue_mac_final(void *mac, uint8_t *buf, size_t len) +{ + if (crypto_mac_final(mac, buf, len)) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +sss_status_t glue_mac_one_go(void *mac, const uint8_t *msg, size_t msg_len, + uint8_t *buf, size_t mac_len) +{ + if (crypto_mac_update(mac, msg, msg_len)) + return kStatus_SSS_Fail; + + if (crypto_mac_final(mac, buf, mac_len)) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +sss_status_t glue_symmetric_context_init(void **cipher) +{ + if (crypto_cipher_alloc_ctx(cipher, TEE_ALG_AES_CBC_NOPAD)) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} + +sss_status_t glue_cipher_one_go(void *cipher, TEE_OperationMode mode, + uint8_t *iv, size_t iv_len, + uint8_t *key, size_t key_len, + const uint8_t *src, uint8_t *dst, size_t len) +{ + if (crypto_cipher_init(cipher, mode, key, key_len, NULL, 0, iv, iv_len)) + return kStatus_SSS_Fail; + + if (crypto_cipher_update(cipher, 0, true, src, len, dst)) + return kStatus_SSS_Fail; + + crypto_cipher_final(cipher); + + return kStatus_SSS_Success; +} + +void glue_context_free(void *cipher) +{ + crypto_cipher_free_ctx(cipher); +} + +sss_status_t glue_rng_get_random(uint8_t *data, size_t len) +{ + if (IS_ENABLED(CFG_NXP_SE05X_RNG_DRV)) + return kStatus_SSS_InvalidArgument; + + if (crypto_rng_read(data, len)) + return kStatus_SSS_Fail; + + return kStatus_SSS_Success; +} diff --git a/optee_os/core/drivers/crypto/se050/session.c b/optee_os/core/drivers/crypto/se050/session.c new file mode 100644 index 0000000..cb6e93e --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/session.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include + +sss_se05x_key_store_t *se050_kstore; +sss_se05x_session_t *se050_session; +struct sss_se05x_ctx se050_ctx; + +TEE_Result se050_core_early_init(struct se050_scp_key *keys) +{ + sss_status_t status = kStatus_SSS_Success; + + status = se050_session_open(&se050_ctx, keys); + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + if (IS_ENABLED(CFG_CORE_SE05X_INIT_NVM)) { + status = se050_factory_reset(&se050_ctx.session.s_ctx); + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + } + + if (se050_ctx.session.subsystem == kType_SSS_SubSystem_NONE) + return TEE_ERROR_GENERIC; + + status = se050_key_store_and_object_init(&se050_ctx); + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + se050_session = (sss_se05x_session_t *)((void *)&se050_ctx.session); + se050_kstore = (sss_se05x_key_store_t *)((void *)&se050_ctx.ks); + + return TEE_SUCCESS; +} + +static TEE_Result update_se_info(void) +{ + sss_status_t status = kStatus_SSS_Success; + + status = se050_get_se_info(se050_session, + IS_ENABLED(CFG_CORE_SE05X_DISPLAY_INFO)); + + /* the session must be closed after accessing the board information */ + sss_se05x_session_close(se050_session); + se050_scp03_set_disable(); + + if (status != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + if (IS_ENABLED(CFG_CORE_SCP03_ONLY)) + return TEE_SUCCESS; + + return se050_core_early_init(NULL); +} + +static TEE_Result enable_scp03(void) +{ + if (se050_enable_scp03(se050_session) != kStatus_SSS_Success) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result se050_early_init_default(void) +{ + if (se050_core_early_init(NULL)) { + EMSG("Failed to open the default session"); + panic(); + } + + if (update_se_info()) { + EMSG("Failed to read the secure element configuration"); + panic(); + } + + if (IS_ENABLED(CFG_CORE_SE05X_SCP03_EARLY)) { + if (enable_scp03()) { + EMSG("Failed to open the SCP03 session"); + panic(); + } + } + + return TEE_SUCCESS; +} + +static TEE_Result se050_early_init_scp03(void) +{ + /* Initialize session */ + se050_session = (sss_se05x_session_t *)((void *)&se050_ctx.session); + se050_kstore = (sss_se05x_key_store_t *)((void *)&se050_ctx.ks); + +#ifdef CFG_CORE_SE05X_OEFID + se050_ctx.se_info.oefid[0] = CFG_CORE_SE05X_OEFID >> 8; + se050_ctx.se_info.oefid[1] = CFG_CORE_SE05X_OEFID & 0xff; +#endif + if (enable_scp03()) { + EMSG("Failed to enable SCP03 session"); + panic(); + } + + if (update_se_info()) { + EMSG("Failed to read the secure element configuration"); + panic(); + } + + if (enable_scp03()) { + EMSG("Failed to re-open the SCP03 session"); + panic(); + } + + return TEE_SUCCESS; +} + +static TEE_Result se050_session_init(void) +{ + if (IS_ENABLED(CFG_CORE_SCP03_ONLY)) + return se050_early_init_scp03(); + + return se050_early_init_default(); +} + +driver_init(se050_session_init); diff --git a/optee_os/core/drivers/crypto/se050/sub.mk b/optee_os/core/drivers/crypto/se050/sub.mk new file mode 100644 index 0000000..cb02572 --- /dev/null +++ b/optee_os/core/drivers/crypto/se050/sub.mk @@ -0,0 +1,15 @@ +include ${CFG_NXP_SE05X_PLUG_AND_TRUST}/cflags.mk + +incdirs_ext-y += ${CFG_NXP_SE05X_PLUG_AND_TRUST}/optee_lib/include +incdirs-y += adaptors/include +incdirs-y += glue/include + +subdirs-y += adaptors +subdirs-y += core +subdirs_ext-y += ${CFG_NXP_SE05X_PLUG_AND_TRUST} + +srcs-y += session.c +srcs-$(CFG_IMX_I2C) += glue/i2c_imx.c +srcs-$(CFG_STM32_I2C) += glue/i2c_stm32.c +srcs-y += glue/i2c.c +srcs-y += glue/user.c diff --git a/optee_os/core/drivers/crypto/stm32/authenc.c b/optee_os/core/drivers/crypto/stm32/authenc.c new file mode 100644 index 0000000..4ff320c --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/authenc.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "stm32_cryp.h" + +#define MAX_TAG_SIZE 16U + +struct stm32_ae_ctx { + struct crypto_authenc_ctx a_ctx; + struct stm32_cryp_context cryp; + enum stm32_cryp_algo_mode algo; + uint8_t tag_mask[MAX_TAG_SIZE]; +}; + +static void xor_vec(uint8_t *r, uint8_t *a, uint8_t *b, size_t len) +{ + size_t i = 0; + + for (i = 0; i < len; i++) + r[i] = a[i] ^ b[i]; +} + +static struct stm32_ae_ctx *to_stm32_ae_ctx(struct crypto_authenc_ctx *ctx) +{ + assert(ctx); + + return container_of(ctx, struct stm32_ae_ctx, a_ctx); +} + +static TEE_Result stm32_ae_gcm_generate_iv(struct stm32_ae_ctx *c, + uint32_t *iv, + struct drvcrypt_authenc_init *dinit) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t tag1[MAX_TAG_SIZE] = { 0 }; + uint8_t tag2[MAX_TAG_SIZE] = { 0 }; + uint32_t j0[MAX_TAG_SIZE / sizeof(uint32_t)] = { 0 }; + uint8_t dummy_iv[MAX_TAG_SIZE] = { 0 }; + struct stm32_cryp_context ctx = { }; + uint8_t *data_out = NULL; + + if (dinit->nonce.length == 12) { + memcpy(iv, dinit->nonce.data, dinit->nonce.length); + iv[3] = TEE_U32_TO_BIG_ENDIAN(2); + return TEE_SUCCESS; + } + + /* Calculate GHASH(dinit->nonce.data) */ + dummy_iv[15] = 2; + + res = stm32_cryp_init(&ctx, true, STM32_CRYP_MODE_AES_GCM, + dinit->key.data, dinit->key.length, + dummy_iv, sizeof(dummy_iv)); + if (res) + return res; + + res = stm32_cryp_final(&ctx, tag1, sizeof(tag1)); + if (res) + return res; + + memset(&ctx, 0, sizeof(ctx)); + res = stm32_cryp_init(&ctx, true, STM32_CRYP_MODE_AES_GCM, + dinit->key.data, dinit->key.length, + dummy_iv, sizeof(dummy_iv)); + if (res) + return res; + + data_out = malloc(dinit->nonce.length); + if (!data_out) + return TEE_ERROR_OUT_OF_MEMORY; + + res = stm32_cryp_update_load(&ctx, dinit->nonce.data, data_out, + dinit->nonce.length); + free(data_out); + + if (res) + return res; + + res = stm32_cryp_final(&ctx, tag2, sizeof(tag2)); + if (res) + return res; + + xor_vec((uint8_t *)j0, tag1, tag2, sizeof(tag1)); + + memcpy(iv, j0, sizeof(j0)); + iv[3] = TEE_U32_TO_BIG_ENDIAN(TEE_U32_FROM_BIG_ENDIAN(iv[3]) + 1); + + /* Compute first mask=AES_ECB(J0_real) into tag1 */ + memset(&ctx, 0, sizeof(ctx)); + res = stm32_cryp_init(&ctx, false, STM32_CRYP_MODE_AES_ECB, + dinit->key.data, dinit->key.length, + NULL, 0); + if (res) + return res; + + res = stm32_cryp_update(&ctx, true, (uint8_t *)j0, tag1, + sizeof(tag1)); + if (res) + return res; + + /* Compute second mask=AES_ECB(J0_used_by_HW) into tag2 */ + memset(&ctx, 0, sizeof(ctx)); + j0[3] = TEE_U32_TO_BIG_ENDIAN(1); + res = stm32_cryp_init(&ctx, false, STM32_CRYP_MODE_AES_ECB, + dinit->key.data, dinit->key.length, + NULL, 0); + if (res) + return res; + + res = stm32_cryp_update(&ctx, true, (uint8_t *)j0, tag2, + sizeof(tag2)); + if (res) + return res; + + /* + * Save the mask we will apply in {enc,dec}_final() to the + * (wrongly) computed tag to get the expected one. + */ + xor_vec(c->tag_mask, tag1, tag2, sizeof(c->tag_mask)); + + return TEE_SUCCESS; +} + +static void stm32_ae_ccm_generate_b0(uint8_t *b0, + struct drvcrypt_authenc_init *dinit) +{ + size_t m = dinit->tag_len; + size_t l = 15 - dinit->nonce.length; + size_t payload_len = dinit->payload_len; + size_t i = 15; + + /* The tag_len should be 4, 6, 8, 10, 12, 14 or 16 */ + assert(m >= 4 && m <= 16 && (m % 2) == 0); + + memset(b0, 0, TEE_AES_BLOCK_SIZE); + /* Flags: (Adata << 6) | (M' << 3) | L' */ + b0[0] = ((dinit->aad_len ? 1 : 0) << 6) | + (((m - 2) / 2) << 3) | + (l - 1); + + /* Nonce */ + memcpy(b0 + 1, dinit->nonce.data, dinit->nonce.length); + + /* Payload length */ + for (i = 15; i >= 15 - l + 1; i--, payload_len >>= 8) + b0[i] = payload_len & 0xFF; +} + +static TEE_Result stm32_ae_ccm_push_b1(struct stm32_ae_ctx *c, + struct drvcrypt_authenc_init *dinit) +{ + uint8_t b1[TEE_AES_BLOCK_SIZE] = { 0 }; + size_t len = 0; + + if (dinit->aad_len == 0) + return TEE_SUCCESS; + + if (dinit->aad_len < 0x100) { + b1[1] = dinit->aad_len; + len = 2; + } else if (dinit->aad_len < 0xFF00) { + b1[0] = dinit->aad_len / 0x100; + b1[1] = dinit->aad_len % 0x100; + len = 2; + } else if (dinit->aad_len <= UINT32_MAX) { + b1[0] = 0xFF; + b1[1] = 0xFE; + b1[2] = dinit->aad_len & GENMASK_32(7, 0); + b1[3] = (dinit->aad_len & GENMASK_32(15, 8)) >> 8; + b1[4] = (dinit->aad_len & GENMASK_32(23, 16)) >> 16; + b1[5] = (dinit->aad_len & GENMASK_32(31, 24)) >> 24; + len = 6; + } else { + b1[0] = 0xFF; + b1[1] = 0xFF; + b1[2] = dinit->aad_len & GENMASK_64(7, 0); + b1[3] = (dinit->aad_len & GENMASK_64(15, 8)) >> 8; + b1[4] = (dinit->aad_len & GENMASK_64(23, 16)) >> 16; + b1[5] = (dinit->aad_len & GENMASK_64(31, 24)) >> 24; + b1[6] = (dinit->aad_len & GENMASK_64(39, 32)) >> 32; + b1[7] = (dinit->aad_len & GENMASK_64(47, 40)) >> 40; + b1[8] = (dinit->aad_len & GENMASK_64(55, 48)) >> 48; + b1[9] = (dinit->aad_len & GENMASK_64(63, 56)) >> 56; + len = 10; + } + + return stm32_cryp_update_assodata(&c->cryp, b1, len); +} + +static TEE_Result stm32_ae_initialize(struct drvcrypt_authenc_init *dinit) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t iv[4] = { 0 }; + struct stm32_ae_ctx *c = to_stm32_ae_ctx(dinit->ctx); + + if (c->algo == STM32_CRYP_MODE_AES_GCM) { + res = stm32_ae_gcm_generate_iv(c, iv, dinit); + if (res) + return res; + } else if (c->algo == STM32_CRYP_MODE_AES_CCM) { + stm32_ae_ccm_generate_b0((uint8_t *)iv, dinit); + } + + res = stm32_cryp_init(&c->cryp, !dinit->encrypt, c->algo, + dinit->key.data, dinit->key.length, iv, + sizeof(iv)); + if (res) + return res; + + if (c->algo == STM32_CRYP_MODE_AES_CCM) + return stm32_ae_ccm_push_b1(c, dinit); + + return TEE_SUCCESS; +} + +static TEE_Result +stm32_ae_update_aad(struct drvcrypt_authenc_update_aad *dupdate) +{ + struct stm32_ae_ctx *c = to_stm32_ae_ctx(dupdate->ctx); + + return stm32_cryp_update_assodata(&c->cryp, dupdate->aad.data, + dupdate->aad.length); +} + +static TEE_Result +stm32_ae_update_payload(struct drvcrypt_authenc_update_payload *dupdate) +{ + struct stm32_ae_ctx *c = to_stm32_ae_ctx(dupdate->ctx); + size_t len = MIN(dupdate->src.length, dupdate->dst.length); + + return stm32_cryp_update_load(&c->cryp, dupdate->src.data, + dupdate->dst.data, len); +} + +static TEE_Result stm32_ae_encdec_final(struct stm32_ae_ctx *c, uint8_t *tag, + size_t tag_size) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t t[MAX_TAG_SIZE] = { 0 }; + + res = stm32_cryp_final(&c->cryp, t, sizeof(t)); + if (res) + return res; + + xor_vec(tag, t, c->tag_mask, tag_size); + + return TEE_SUCCESS; +} + +static TEE_Result stm32_ae_enc_final(struct drvcrypt_authenc_final *dfinal) +{ + TEE_Result res = TEE_SUCCESS; + struct stm32_ae_ctx *c = to_stm32_ae_ctx(dfinal->ctx); + size_t len = MIN(dfinal->src.length, dfinal->dst.length); + + res = stm32_cryp_update_load(&c->cryp, dfinal->src.data, + dfinal->dst.data, len); + if (res) + return res; + + return stm32_ae_encdec_final(c, dfinal->tag.data, dfinal->tag.length); +} + +static TEE_Result stm32_ae_dec_final(struct drvcrypt_authenc_final *dfinal) +{ + TEE_Result res = TEE_SUCCESS; + struct stm32_ae_ctx *c = to_stm32_ae_ctx(dfinal->ctx); + size_t len = MIN(dfinal->src.length, dfinal->dst.length); + unsigned char tag_buf[MAX_TAG_SIZE] = { 0 }; + + res = stm32_cryp_update_load(&c->cryp, dfinal->src.data, + dfinal->dst.data, len); + if (res) + return res; + + res = stm32_ae_encdec_final(c, tag_buf, sizeof(tag_buf)); + if (res) + return res; + + if (consttime_memcmp(tag_buf, dfinal->tag.data, dfinal->tag.length)) + return TEE_ERROR_MAC_INVALID; + + return TEE_SUCCESS; +} + +static void stm32_ae_final(void *ctx __unused) +{ +} + +static void stm32_ae_free(void *ctx) +{ + struct stm32_ae_ctx *c = to_stm32_ae_ctx(ctx); + + free(c); +} + +static void stm32_ae_copy_state(void *dst_ctx, void *src_ctx) +{ + struct stm32_ae_ctx *src = to_stm32_ae_ctx(src_ctx); + struct stm32_ae_ctx *dst = to_stm32_ae_ctx(dst_ctx); + + memcpy(dst, src, sizeof(*dst)); +} + +static TEE_Result alloc_ctx(void **ctx, enum stm32_cryp_algo_mode algo) +{ + struct stm32_ae_ctx *c = calloc(1, sizeof(*c)); + + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->algo = algo; + *ctx = &c->a_ctx; + + return TEE_SUCCESS; +} + +/* + * Allocate the SW authenc data context + * + * @ctx [out] Caller context variable + * @algo Algorithm ID of the context + */ +static TEE_Result stm32_ae_allocate(void **ctx, uint32_t algo) +{ + /* Convert TEE_ALGO id to CRYP id */ + switch (algo) { + case TEE_ALG_AES_CCM: + return alloc_ctx(ctx, STM32_CRYP_MODE_AES_CCM); + case TEE_ALG_AES_GCM: + return alloc_ctx(ctx, STM32_CRYP_MODE_AES_GCM); + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +/* + * Registration of the Authenc Driver + */ +static struct drvcrypt_authenc driver_authenc = { + .alloc_ctx = &stm32_ae_allocate, + .free_ctx = &stm32_ae_free, + .init = &stm32_ae_initialize, + .update_aad = &stm32_ae_update_aad, + .update_payload = &stm32_ae_update_payload, + .enc_final = &stm32_ae_enc_final, + .dec_final = &stm32_ae_dec_final, + .final = &stm32_ae_final, + .copy_state = &stm32_ae_copy_state, +}; + +TEE_Result stm32_register_authenc(void) +{ + return drvcrypt_register_authenc(&driver_authenc); +} diff --git a/optee_os/core/drivers/crypto/stm32/cipher.c b/optee_os/core/drivers/crypto/stm32/cipher.c new file mode 100644 index 0000000..ec66c4f --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/cipher.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "stm32_cryp.h" +#include "stm32_saes.h" + +#define DES3_KEY_SIZE 24 + +struct cryp_ctx { + struct stm32_cryp_context ctx; + enum stm32_cryp_algo_mode algo; +}; + +struct saes_ctx { + struct stm32_saes_context ctx; + enum stm32_saes_chaining_mode algo; +}; + +/* + * Internal peripheral context + * SAES and CRYP are registered under the same ID in the crypto framework. + * Therefore, only one of them can be registered. + */ + +union ip_ctx { + struct saes_ctx saes; + struct cryp_ctx cryp; +}; + +/* Internal Peripheral cipher ops*/ +struct ip_cipher_ops { + TEE_Result (*init)(union ip_ctx *ctx, bool is_decrypt, + const uint8_t *key, size_t key_len, + const uint8_t *iv, size_t iv_len); + TEE_Result (*update)(union ip_ctx *ctx, bool last_block, uint8_t *src, + uint8_t *dst, size_t len); +}; + +struct stm32_cipher_ctx { + struct crypto_cipher_ctx c_ctx; + union ip_ctx ip_ctx; + const struct ip_cipher_ops *ops; +}; + +static TEE_Result cryp_init(union ip_ctx *ip_ctx, bool is_decrypt, + const uint8_t *key, size_t key_len, + const uint8_t *iv, size_t iv_len) +{ + uint8_t temp_key[DES3_KEY_SIZE] = { }; + + if (!IS_ENABLED(CFG_STM32_CRYP)) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (key_len == 16 && + (ip_ctx->cryp.algo == STM32_CRYP_MODE_TDES_ECB || + ip_ctx->cryp.algo == STM32_CRYP_MODE_TDES_CBC)) { + /* Manage DES2: i.e. K=K1.K2.K1 */ + memcpy(temp_key, key, key_len); + memcpy(temp_key + key_len, key, key_len / 2); + key_len = DES3_KEY_SIZE; + key = temp_key; + } + + return stm32_cryp_init(&ip_ctx->cryp.ctx, is_decrypt, ip_ctx->cryp.algo, + key, key_len, iv, iv_len); +} + +static TEE_Result cryp_update(union ip_ctx *ip_ctx, bool last_block, + uint8_t *src, uint8_t *dst, size_t len) +{ + if (!IS_ENABLED(CFG_STM32_CRYP)) + return TEE_ERROR_NOT_IMPLEMENTED; + + return stm32_cryp_update(&ip_ctx->cryp.ctx, last_block, src, dst, len); +} + +static TEE_Result saes_init(union ip_ctx *ip_ctx, bool is_decrypt, + const uint8_t *key, size_t key_len, + const uint8_t *iv, size_t iv_len) +{ + enum stm32_saes_key_selection key_sel = STM32_SAES_KEY_SOFT; + + if (!IS_ENABLED(CFG_STM32_SAES)) + return TEE_ERROR_NOT_IMPLEMENTED; + + return stm32_saes_init(&ip_ctx->saes.ctx, is_decrypt, ip_ctx->saes.algo, + key_sel, key, key_len, iv, iv_len); +} + +static TEE_Result saes_update(union ip_ctx *ip_ctx, bool last_block, + uint8_t *src, uint8_t *dst, size_t len) +{ + if (!IS_ENABLED(CFG_STM32_SAES)) + return TEE_ERROR_NOT_IMPLEMENTED; + + return stm32_saes_update(&ip_ctx->saes.ctx, last_block, src, dst, len); +} + +const struct ip_cipher_ops cryp_ops = { + .init = cryp_init, + .update = cryp_update, +}; + +const struct ip_cipher_ops saes_ops = { + .init = saes_init, + .update = saes_update, +}; + +static struct stm32_cipher_ctx * +to_stm32_cipher_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx); + + return container_of(ctx, struct stm32_cipher_ctx, c_ctx); +} + +static TEE_Result stm32_cipher_initialize(struct drvcrypt_cipher_init *dinit) +{ + struct stm32_cipher_ctx *c = to_stm32_cipher_ctx(dinit->ctx); + + return c->ops->init(&c->ip_ctx, !dinit->encrypt, dinit->key1.data, + dinit->key1.length, dinit->iv.data, + dinit->iv.length); +} + +static TEE_Result stm32_cipher_update(struct drvcrypt_cipher_update *dupdate) +{ + struct stm32_cipher_ctx *c = to_stm32_cipher_ctx(dupdate->ctx); + size_t len = MIN(dupdate->src.length, dupdate->dst.length); + + return c->ops->update(&c->ip_ctx, dupdate->last, dupdate->src.data, + dupdate->dst.data, len); +} + +static void stm32_cipher_final(void *ctx __unused) +{ +} + +static void stm32_cipher_free(void *ctx) +{ + struct stm32_cipher_ctx *c = to_stm32_cipher_ctx(ctx); + + free(c); +} + +static void stm32_cipher_copy_state(void *dst_ctx, void *src_ctx) +{ + struct stm32_cipher_ctx *src = to_stm32_cipher_ctx(src_ctx); + struct stm32_cipher_ctx *dst = to_stm32_cipher_ctx(dst_ctx); + + memcpy(dst, src, sizeof(*dst)); +} + +static TEE_Result alloc_cryp_ctx(void **ctx, enum stm32_cryp_algo_mode algo) +{ + struct stm32_cipher_ctx *c = calloc(1, sizeof(*c)); + + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + FMSG("Using CRYP %d", algo); + c->ip_ctx.cryp.algo = algo; + c->ops = &cryp_ops; + *ctx = &c->c_ctx; + + return TEE_SUCCESS; +} + +static TEE_Result alloc_saes_ctx(void **ctx, enum stm32_saes_chaining_mode algo) +{ + struct stm32_cipher_ctx *c = calloc(1, sizeof(*c)); + + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + FMSG("Using SAES %d", algo); + c->ip_ctx.saes.algo = algo; + c->ops = &saes_ops; + *ctx = &c->c_ctx; + + return TEE_SUCCESS; +} + +/* + * Allocate the SW cipher data context for CRYP peripheral. + * + * @ctx [out] Caller context variable + * @algo Algorithm ID of the context + */ +static TEE_Result stm32_cryp_cipher_allocate(void **ctx, uint32_t algo) +{ + /* + * Convert TEE_ALGO id to internal id + */ + switch (algo) { + case TEE_ALG_DES_ECB_NOPAD: + return alloc_cryp_ctx(ctx, STM32_CRYP_MODE_DES_ECB); + case TEE_ALG_DES_CBC_NOPAD: + return alloc_cryp_ctx(ctx, STM32_CRYP_MODE_DES_CBC); + case TEE_ALG_DES3_ECB_NOPAD: + return alloc_cryp_ctx(ctx, STM32_CRYP_MODE_TDES_ECB); + case TEE_ALG_DES3_CBC_NOPAD: + return alloc_cryp_ctx(ctx, STM32_CRYP_MODE_TDES_CBC); + case TEE_ALG_AES_ECB_NOPAD: + return alloc_cryp_ctx(ctx, STM32_CRYP_MODE_AES_ECB); + case TEE_ALG_AES_CBC_NOPAD: + return alloc_cryp_ctx(ctx, STM32_CRYP_MODE_AES_CBC); + case TEE_ALG_AES_CTR: + return alloc_cryp_ctx(ctx, STM32_CRYP_MODE_AES_CTR); + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +/* + * Allocate the SW cipher data context for SAES peripheral. + * + * @ctx [out] Caller context variable + * @algo Algorithm ID of the context + */ +static TEE_Result stm32_saes_cipher_allocate(void **ctx, uint32_t algo) +{ + /* + * Convert TEE_ALGO id to internal id + */ + switch (algo) { + case TEE_ALG_AES_ECB_NOPAD: + return alloc_saes_ctx(ctx, STM32_SAES_MODE_ECB); + case TEE_ALG_AES_CBC_NOPAD: + return alloc_saes_ctx(ctx, STM32_SAES_MODE_CBC); + case TEE_ALG_AES_CTR: + return alloc_saes_ctx(ctx, STM32_SAES_MODE_CTR); + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +static struct drvcrypt_cipher driver_cipher_cryp = { + .alloc_ctx = &stm32_cryp_cipher_allocate, + .free_ctx = &stm32_cipher_free, + .init = &stm32_cipher_initialize, + .update = &stm32_cipher_update, + .final = &stm32_cipher_final, + .copy_state = &stm32_cipher_copy_state, +}; + +static struct drvcrypt_cipher driver_cipher_saes = { + .alloc_ctx = &stm32_saes_cipher_allocate, + .free_ctx = &stm32_cipher_free, + .init = &stm32_cipher_initialize, + .update = &stm32_cipher_update, + .final = &stm32_cipher_final, + .copy_state = &stm32_cipher_copy_state, +}; + +TEE_Result stm32_register_cipher(enum stm32_cipher_ip_id cipher_ip) +{ + if (cipher_ip == SAES_IP) + return drvcrypt_register_cipher(&driver_cipher_saes); + else if (cipher_ip == CRYP_IP) + return drvcrypt_register_cipher(&driver_cipher_cryp); + else + return TEE_ERROR_BAD_PARAMETERS; +} diff --git a/optee_os/core/drivers/crypto/stm32/common.h b/optee_os/core/drivers/crypto/stm32/common.h new file mode 100644 index 0000000..7ae7f29 --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/common.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + */ + +#ifndef __DRIVERS_CRYPTO_STM32_COMMON_H +#define __DRIVERS_CRYPTO_STM32_COMMON_H + +#include + +enum stm32_cipher_ip_id { + CRYP_IP, + SAES_IP, +}; + +/* + * Crypto algorithm common macro used in stm32_saes and stm32_cryp driver + */ + +#define INT8_BIT U(8) +#define AES_BLOCK_SIZE_BIT U(128) +#define AES_BLOCK_SIZE (AES_BLOCK_SIZE_BIT / INT8_BIT) +#define AES_BLOCK_NB_U32 (AES_BLOCK_SIZE / sizeof(uint32_t)) +#define DES_BLOCK_SIZE_BIT U(64) +#define DES_BLOCK_SIZE (DES_BLOCK_SIZE_BIT / INT8_BIT) +#define DES_BLOCK_NB_U32 (DES_BLOCK_SIZE / sizeof(uint32_t)) +#define MAX_BLOCK_SIZE_BIT AES_BLOCK_SIZE_BIT +#define MAX_BLOCK_SIZE AES_BLOCK_SIZE +#define MAX_BLOCK_NB_U32 AES_BLOCK_NB_U32 +#define AES_KEYSIZE_128 U(16) +#define AES_KEYSIZE_192 U(24) +#define AES_KEYSIZE_256 U(32) +#define AES_IVSIZE U(16) + +TEE_Result stm32_register_authenc(void); +TEE_Result stm32_register_cipher(enum stm32_cipher_ip_id); + +#endif /* __DRIVERS_CRYPTO_STM32_COMMON_H */ diff --git a/optee_os/core/drivers/crypto/stm32/crypto.mk b/optee_os/core/drivers/crypto/stm32/crypto.mk new file mode 100644 index 0000000..3f3aa88 --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/crypto.mk @@ -0,0 +1,21 @@ +# CFG_STM32_CRYPTO_DRIVER, when enabled, embeds +# STM32 HW cryptographic support and OP-TEE Crypto Driver. +# CFG_STM32_CRYP, when enabled, embeds +# STM32 CRYP module support, +# CIPHER Crypto Driver, +# AUTHENC Crypto Driver. + +ifeq ($(CFG_STM32_CRYPTO_DRIVER),y) + +$(call force,CFG_CRYPTO_DRIVER,y) +CFG_CRYPTO_DRIVER_DEBUG ?= 0 + +ifeq ($(call cfg-one-enabled, CFG_STM32_CRYP CFG_STM32_SAES),y) +$(call force,CFG_CRYPTO_DRV_CIPHER,y,Mandated by CFG_STM32_CRYP) +endif + +ifeq ($(CFG_STM32_CRYP),y) +$(call force,CFG_CRYPTO_DRV_AUTHENC,y,Mandated by CFG_STM32_CRYP) +endif + +endif # CFG_STM32_CRYPTO_DRIVER diff --git a/optee_os/core/drivers/crypto/stm32/stm32_cryp.c b/optee_os/core/drivers/crypto/stm32/stm32_cryp.c new file mode 100644 index 0000000..70ad4e3 --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/stm32_cryp.c @@ -0,0 +1,1301 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "stm32_cryp.h" +#include "common.h" + +/* CRYP control register */ +#define _CRYP_CR 0x0U +/* CRYP status register */ +#define _CRYP_SR 0x04U +/* CRYP data input register */ +#define _CRYP_DIN 0x08U +/* CRYP data output register */ +#define _CRYP_DOUT 0x0CU +/* CRYP DMA control register */ +#define _CRYP_DMACR 0x10U +/* CRYP interrupt mask set/clear register */ +#define _CRYP_IMSCR 0x14U +/* CRYP raw interrupt status register */ +#define _CRYP_RISR 0x18U +/* CRYP masked interrupt status register */ +#define _CRYP_MISR 0x1CU +/* CRYP key registers */ +#define _CRYP_K0LR 0x20U +#define _CRYP_K0RR 0x24U +#define _CRYP_K1LR 0x28U +#define _CRYP_K1RR 0x2CU +#define _CRYP_K2LR 0x30U +#define _CRYP_K2RR 0x34U +#define _CRYP_K3LR 0x38U +#define _CRYP_K3RR 0x3CU +/* CRYP initialization vector registers */ +#define _CRYP_IV0LR 0x40U +#define _CRYP_IV0RR 0x44U +#define _CRYP_IV1LR 0x48U +#define _CRYP_IV1RR 0x4CU +/* CRYP context swap GCM-CCM registers */ +#define _CRYP_CSGCMCCM0R 0x50U +#define _CRYP_CSGCMCCM1R 0x54U +#define _CRYP_CSGCMCCM2R 0x58U +#define _CRYP_CSGCMCCM3R 0x5CU +#define _CRYP_CSGCMCCM4R 0x60U +#define _CRYP_CSGCMCCM5R 0x64U +#define _CRYP_CSGCMCCM6R 0x68U +#define _CRYP_CSGCMCCM7R 0x6CU +/* CRYP context swap GCM registers */ +#define _CRYP_CSGCM0R 0x70U +#define _CRYP_CSGCM1R 0x74U +#define _CRYP_CSGCM2R 0x78U +#define _CRYP_CSGCM3R 0x7CU +#define _CRYP_CSGCM4R 0x80U +#define _CRYP_CSGCM5R 0x84U +#define _CRYP_CSGCM6R 0x88U +#define _CRYP_CSGCM7R 0x8CU +/* CRYP hardware configuration register */ +#define _CRYP_HWCFGR 0x3F0U +/* CRYP HW version register */ +#define _CRYP_VERR 0x3F4U +/* CRYP identification */ +#define _CRYP_IPIDR 0x3F8U +/* CRYP HW magic ID */ +#define _CRYP_MID 0x3FCU + +#define CRYP_TIMEOUT_US 1000000U +#define TIMEOUT_US_1MS 1000U + +/* CRYP control register fields */ +#define _CRYP_CR_RESET_VALUE 0x0U +#define _CRYP_CR_NPBLB_MSK GENMASK_32(23, 20) +#define _CRYP_CR_NPBLB_OFF 20U +#define _CRYP_CR_GCM_CCMPH_MSK GENMASK_32(17, 16) +#define _CRYP_CR_GCM_CCMPH_OFF 16U +#define _CRYP_CR_GCM_CCMPH_INIT 0U +#define _CRYP_CR_GCM_CCMPH_HEADER 1U +#define _CRYP_CR_GCM_CCMPH_PAYLOAD 2U +#define _CRYP_CR_GCM_CCMPH_FINAL 3U +#define _CRYP_CR_CRYPEN BIT(15) +#define _CRYP_CR_FFLUSH BIT(14) +#define _CRYP_CR_KEYSIZE_MSK GENMASK_32(9, 8) +#define _CRYP_CR_KEYSIZE_OFF 8U +#define _CRYP_CR_KSIZE_128 0U +#define _CRYP_CR_KSIZE_192 1U +#define _CRYP_CR_KSIZE_256 2U +#define _CRYP_CR_DATATYPE_MSK GENMASK_32(7, 6) +#define _CRYP_CR_DATATYPE_OFF 6U +#define _CRYP_CR_DATATYPE_NONE 0U +#define _CRYP_CR_DATATYPE_HALF_WORD 1U +#define _CRYP_CR_DATATYPE_BYTE 2U +#define _CRYP_CR_DATATYPE_BIT 3U +#define _CRYP_CR_ALGOMODE_MSK (BIT(19) | GENMASK_32(5, 3)) +#define _CRYP_CR_ALGOMODE_OFF 3U +#define _CRYP_CR_ALGOMODE_TDES_ECB 0x0U +#define _CRYP_CR_ALGOMODE_TDES_CBC 0x1U +#define _CRYP_CR_ALGOMODE_DES_ECB 0x2U +#define _CRYP_CR_ALGOMODE_DES_CBC 0x3U +#define _CRYP_CR_ALGOMODE_AES_ECB 0x4U +#define _CRYP_CR_ALGOMODE_AES_CBC 0x5U +#define _CRYP_CR_ALGOMODE_AES_CTR 0x6U +#define _CRYP_CR_ALGOMODE_AES 0x7U +#define _CRYP_CR_ALGOMODE_AES_GCM BIT(16) +#define _CRYP_CR_ALGOMODE_AES_CCM (BIT(16) | BIT(0)) +#define _CRYP_CR_ALGODIR BIT(2) +#define _CRYP_CR_ALGODIR_ENC 0U +#define _CRYP_CR_ALGODIR_DEC BIT(2) + +/* CRYP status register fields */ +#define _CRYP_SR_BUSY BIT(4) +#define _CRYP_SR_OFFU BIT(3) +#define _CRYP_SR_OFNE BIT(2) +#define _CRYP_SR_IFNF BIT(1) +#define _CRYP_SR_IFEM BIT(0) + +/* CRYP DMA control register fields */ +#define _CRYP_DMACR_DOEN BIT(1) +#define _CRYP_DMACR_DIEN BIT(0) + +/* CRYP interrupt fields */ +#define _CRYP_I_OUT BIT(1) +#define _CRYP_I_IN BIT(0) + +/* CRYP hardware configuration register fields */ +#define _CRYP_HWCFGR_CFG1_MSK GENMASK_32(3, 0) +#define _CRYP_HWCFGR_CFG1_OFF 0U +#define _CRYP_HWCFGR_CFG2_MSK GENMASK_32(7, 4) +#define _CRYP_HWCFGR_CFG2_OFF 4U +#define _CRYP_HWCFGR_CFG3_MSK GENMASK_32(11, 8) +#define _CRYP_HWCFGR_CFG3_OFF 8U +#define _CRYP_HWCFGR_CFG4_MSK GENMASK_32(15, 12) +#define _CRYP_HWCFGR_CFG4_OFF 12U + +/* CRYP HW version register */ +#define _CRYP_VERR_MSK GENMASK_32(7, 0) +#define _CRYP_VERR_OFF 0U + +/* + * Macro to manage bit manipulation when we work on a local variable + * before writing only once to the hardware register. + */ +#define CLRBITS(v, bits) ((v) &= ~(bits)) +#define SETBITS(v, bits) ((v) |= (bits)) + +#define IS_ALGOMODE(cr, mod) \ + (((cr) & _CRYP_CR_ALGOMODE_MSK) == (_CRYP_CR_ALGOMODE_##mod << \ + _CRYP_CR_ALGOMODE_OFF)) + +#define SET_ALGOMODE(mod, cr) \ + clrsetbits(&(cr), _CRYP_CR_ALGOMODE_MSK, (_CRYP_CR_ALGOMODE_##mod << \ + _CRYP_CR_ALGOMODE_OFF)) + +#define GET_ALGOMODE(cr) \ + (((cr) & _CRYP_CR_ALGOMODE_MSK) >> _CRYP_CR_ALGOMODE_OFF) + +static struct stm32_cryp_platdata cryp_pdata; +static struct mutex cryp_lock = MUTEX_INITIALIZER; + +static void clrsetbits(uint32_t *v, uint32_t mask, uint32_t bits) +{ + *v = (*v & ~mask) | bits; +} + +static bool algo_mode_needs_iv(uint32_t cr) +{ + return !IS_ALGOMODE(cr, TDES_ECB) && !IS_ALGOMODE(cr, DES_ECB) && + !IS_ALGOMODE(cr, AES_ECB); +} + +static bool algo_mode_is_ecb_cbc(uint32_t cr) +{ + return GET_ALGOMODE(cr) < _CRYP_CR_ALGOMODE_AES_CTR; +} + +static bool algo_mode_is_aes(uint32_t cr) +{ + return ((cr & _CRYP_CR_ALGOMODE_MSK) >> _CRYP_CR_ALGOMODE_OFF) >= + _CRYP_CR_ALGOMODE_AES_ECB; +} + +static bool is_decrypt(uint32_t cr) +{ + return (cr & _CRYP_CR_ALGODIR) == _CRYP_CR_ALGODIR_DEC; +} + +static bool is_encrypt(uint32_t cr) +{ + return !is_decrypt(cr); +} + +static bool does_need_npblb(uint32_t cr) +{ + return (IS_ALGOMODE(cr, AES_GCM) && is_encrypt(cr)) || + (IS_ALGOMODE(cr, AES_CCM) && is_decrypt(cr)); +} + +static TEE_Result wait_sr_bits(vaddr_t base, uint32_t bits) +{ + uint64_t timeout_ref = timeout_init_us(CRYP_TIMEOUT_US); + + while ((io_read32(base + _CRYP_SR) & bits) != bits) + if (timeout_elapsed(timeout_ref)) + break; + + if ((io_read32(base + _CRYP_SR) & bits) != bits) + return TEE_ERROR_BUSY; + + return TEE_SUCCESS; +} + +static TEE_Result wait_end_busy(vaddr_t base) +{ + uint64_t timeout_ref = timeout_init_us(CRYP_TIMEOUT_US); + + while (io_read32(base + _CRYP_SR) & _CRYP_SR_BUSY) + if (timeout_elapsed(timeout_ref)) + break; + + if (io_read32(base + _CRYP_SR) & _CRYP_SR_BUSY) + return TEE_ERROR_BUSY; + + return TEE_SUCCESS; +} + +static TEE_Result wait_end_enable(vaddr_t base) +{ + uint64_t timeout_ref = timeout_init_us(CRYP_TIMEOUT_US); + + while (io_read32(base + _CRYP_CR) & _CRYP_CR_CRYPEN) + if (timeout_elapsed(timeout_ref)) + break; + + if (io_read32(base + _CRYP_CR) & _CRYP_CR_CRYPEN) + return TEE_ERROR_BUSY; + + return TEE_SUCCESS; +} + +static TEE_Result __must_check write_align_block(struct stm32_cryp_context *ctx, + uint32_t *data) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = 0; + + res = wait_sr_bits(ctx->base, _CRYP_SR_IFNF); + if (res) + return res; + + for (i = 0; i < ctx->block_u32; i++) { + /* No need to htobe() as we configure the HW to swap bytes */ + io_write32(ctx->base + _CRYP_DIN, data[i]); + } + + return TEE_SUCCESS; +} + +static TEE_Result __must_check write_block(struct stm32_cryp_context *ctx, + uint8_t *data) +{ + if (!IS_ALIGNED_WITH_TYPE(data, uint32_t)) { + uint32_t data_u32[MAX_BLOCK_NB_U32] = { 0 }; + + memcpy(data_u32, data, ctx->block_u32 * sizeof(uint32_t)); + return write_align_block(ctx, data_u32); + } + + return write_align_block(ctx, (void *)data); +} + +static TEE_Result __must_check read_align_block(struct stm32_cryp_context *ctx, + uint32_t *data) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = 0; + + res = wait_sr_bits(ctx->base, _CRYP_SR_OFNE); + if (res) + return res; + + for (i = 0; i < ctx->block_u32; i++) { + /* No need to htobe() as we configure the HW to swap bytes */ + data[i] = io_read32(ctx->base + _CRYP_DOUT); + } + + return TEE_SUCCESS; +} + +static TEE_Result __must_check read_block(struct stm32_cryp_context *ctx, + uint8_t *data) +{ + if (!IS_ALIGNED_WITH_TYPE(data, uint32_t)) { + TEE_Result res = TEE_SUCCESS; + uint32_t data_u32[MAX_BLOCK_NB_U32] = { 0 }; + + res = read_align_block(ctx, data_u32); + if (res) + return res; + + memcpy(data, data_u32, ctx->block_u32 * sizeof(uint32_t)); + + return TEE_SUCCESS; + } + + return read_align_block(ctx, (void *)data); +} + +static void cryp_end(struct stm32_cryp_context *ctx, TEE_Result prev_error) +{ + if (prev_error) { + if (rstctrl_assert_to(cryp_pdata.reset, TIMEOUT_US_1MS)) + panic(); + if (rstctrl_deassert_to(cryp_pdata.reset, TIMEOUT_US_1MS)) + panic(); + } + + /* Disable the CRYP peripheral */ + io_clrbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); +} + +static void cryp_write_iv(struct stm32_cryp_context *ctx) +{ + if (algo_mode_needs_iv(ctx->cr)) { + unsigned int i = 0; + + /* Restore the _CRYP_IVRx */ + for (i = 0; i < ctx->block_u32; i++) + io_write32(ctx->base + _CRYP_IV0LR + i * + sizeof(uint32_t), ctx->iv[i]); + } +} + +static void cryp_save_suspend(struct stm32_cryp_context *ctx) +{ + unsigned int i = 0; + + if (IS_ALGOMODE(ctx->cr, AES_GCM) || IS_ALGOMODE(ctx->cr, AES_CCM)) + for (i = 0; i < ARRAY_SIZE(ctx->pm_gcmccm); i++) + ctx->pm_gcmccm[i] = io_read32(ctx->base + + _CRYP_CSGCMCCM0R + + i * sizeof(uint32_t)); + + if (IS_ALGOMODE(ctx->cr, AES_GCM)) + for (i = 0; i < ARRAY_SIZE(ctx->pm_gcm); i++) + ctx->pm_gcm[i] = io_read32(ctx->base + _CRYP_CSGCM0R + + i * sizeof(uint32_t)); +} + +static void cryp_restore_suspend(struct stm32_cryp_context *ctx) +{ + unsigned int i = 0; + + if (IS_ALGOMODE(ctx->cr, AES_GCM) || IS_ALGOMODE(ctx->cr, AES_CCM)) + for (i = 0; i < ARRAY_SIZE(ctx->pm_gcmccm); i++) + io_write32(ctx->base + _CRYP_CSGCMCCM0R + + i * sizeof(uint32_t), ctx->pm_gcmccm[i]); + + if (IS_ALGOMODE(ctx->cr, AES_GCM)) + for (i = 0; i < ARRAY_SIZE(ctx->pm_gcm); i++) + io_write32(ctx->base + _CRYP_CSGCM0R + + i * sizeof(uint32_t), ctx->pm_gcm[i]); +} + +static void cryp_write_key(struct stm32_cryp_context *ctx) +{ + vaddr_t reg = 0; + int i = 0; + uint32_t algo = GET_ALGOMODE(ctx->cr); + + if (algo == _CRYP_CR_ALGOMODE_DES_ECB || + algo == _CRYP_CR_ALGOMODE_DES_CBC) + reg = ctx->base + _CRYP_K1RR; + else + reg = ctx->base + _CRYP_K3RR; + + for (i = ctx->key_size / sizeof(uint32_t) - 1; + i >= 0; + i--, reg -= sizeof(uint32_t)) + io_write32(reg, ctx->key[i]); +} + +static TEE_Result cryp_prepare_key(struct stm32_cryp_context *ctx) +{ + TEE_Result res = TEE_SUCCESS; + + /* + * For AES ECB/CBC decryption, key preparation mode must be selected + * to populate the key. + */ + if (is_decrypt(ctx->cr) && (IS_ALGOMODE(ctx->cr, AES_ECB) || + IS_ALGOMODE(ctx->cr, AES_CBC))) { + /* Select Algomode "prepare key" */ + io_clrsetbits32(ctx->base + _CRYP_CR, _CRYP_CR_ALGOMODE_MSK, + _CRYP_CR_ALGOMODE_AES << _CRYP_CR_ALGOMODE_OFF); + + cryp_write_key(ctx); + + /* Enable CRYP */ + io_setbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); + + res = wait_end_busy(ctx->base); + if (res) + return res; + + /* Reset 'real' algomode */ + io_clrsetbits32(ctx->base + _CRYP_CR, _CRYP_CR_ALGOMODE_MSK, + ctx->cr & _CRYP_CR_ALGOMODE_MSK); + } else { + cryp_write_key(ctx); + } + + return TEE_SUCCESS; +} + +static TEE_Result save_context(struct stm32_cryp_context *ctx) +{ + /* Device should not be in a processing phase */ + if (io_read32(ctx->base + _CRYP_SR) & _CRYP_SR_BUSY) + return TEE_ERROR_BAD_STATE; + + /* Disable the CRYP peripheral */ + io_clrbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); + + /* Save CR */ + ctx->cr = io_read32(ctx->base + _CRYP_CR); + + cryp_save_suspend(ctx); + + /* If algo mode needs to save current IV */ + if (algo_mode_needs_iv(ctx->cr)) { + unsigned int i = 0; + + /* Save IV */ + for (i = 0; i < ctx->block_u32; i++) + ctx->iv[i] = io_read32(ctx->base + _CRYP_IV0LR + i * + sizeof(uint32_t)); + } + + return TEE_SUCCESS; +} + +/* To resume the processing of a message */ +static TEE_Result restore_context(struct stm32_cryp_context *ctx) +{ + TEE_Result res = TEE_SUCCESS; + + /* IP should be disabled */ + if (io_read32(ctx->base + _CRYP_CR) & _CRYP_CR_CRYPEN) { + DMSG("Device is still enabled"); + return TEE_ERROR_BAD_STATE; + } + + /* Restore the _CRYP_CR */ + io_write32(ctx->base + _CRYP_CR, ctx->cr); + + /* Write key and, in case of AES_CBC or AES_ECB decrypt, prepare it */ + res = cryp_prepare_key(ctx); + if (res) + return res; + + cryp_restore_suspend(ctx); + + cryp_write_iv(ctx); + + /* Flush internal fifo */ + io_setbits32(ctx->base + _CRYP_CR, _CRYP_CR_FFLUSH); + + /* Enable the CRYP peripheral */ + io_setbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); + + return TEE_SUCCESS; +} + +/* + * Translate a byte index in an array of BE uint32_t into the index of same + * byte in the corresponding LE uint32_t array. + */ +static size_t be_index(size_t index) +{ + return (index & ~0x3) + 3 - (index & 0x3); +} + +static TEE_Result ccm_first_context(struct stm32_cryp_context *ctx) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t b0[AES_BLOCK_NB_U32] = { 0 }; + uint8_t *iv = (uint8_t *)ctx->iv; + size_t l = 0; + size_t i = 15; + + /* IP should be disabled */ + if (io_read32(ctx->base + _CRYP_CR) & _CRYP_CR_CRYPEN) + return TEE_ERROR_BAD_STATE; + + /* Write the _CRYP_CR */ + io_write32(ctx->base + _CRYP_CR, ctx->cr); + + /* Write key */ + res = cryp_prepare_key(ctx); + if (res) + return res; + + /* Save full IV that will be b0 */ + memcpy(b0, iv, sizeof(b0)); + + /* + * Update IV to become CTR0/1 before setting it. + * IV is saved as LE uint32_t[4] as expected by hardware, + * but CCM RFC defines bytes to update in a BE array. + */ + /* Set flag bits to 0 (5 higher bits), keep 3 low bits */ + iv[be_index(0)] &= 0x7; + /* Get size of length field (can be from 2 to 8) */ + l = iv[be_index(0)] + 1; + /* Set Q to 0 */ + for (i = 15; i >= 15 - l + 1; i--) + iv[be_index(i)] = 0; + /* Save CTR0 */ + memcpy(ctx->ctr0_ccm, iv, sizeof(b0)); + /* Increment Q */ + iv[be_index(15)] |= 0x1; + + cryp_write_iv(ctx); + + /* Enable the CRYP peripheral */ + io_setbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); + + res = write_align_block(ctx, b0); + + return res; +} + +static TEE_Result do_from_init_to_phase(struct stm32_cryp_context *ctx, + uint32_t new_phase) +{ + TEE_Result res = TEE_SUCCESS; + + /* + * We didn't run the init phase yet + * CCM need a specific restore_context phase for the init phase + */ + if (IS_ALGOMODE(ctx->cr, AES_CCM)) + res = ccm_first_context(ctx); + else + res = restore_context(ctx); + + if (res) + return res; + + res = wait_end_enable(ctx->base); + if (res) + return res; + + /* Move to 'new_phase' */ + io_clrsetbits32(ctx->base + _CRYP_CR, _CRYP_CR_GCM_CCMPH_MSK, + new_phase << _CRYP_CR_GCM_CCMPH_OFF); + + /* Enable the CRYP peripheral (init disabled it) */ + io_setbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); + + return TEE_SUCCESS; +} + +static TEE_Result do_from_header_to_phase(struct stm32_cryp_context *ctx, + uint32_t new_phase) +{ + TEE_Result res = TEE_SUCCESS; + + res = restore_context(ctx); + if (res) + return res; + + if (ctx->extra_size) { + /* Manage unaligned header data before moving to next phase */ + memset((uint8_t *)ctx->extra + ctx->extra_size, 0, + ctx->block_u32 * sizeof(uint32_t) - ctx->extra_size); + + res = write_align_block(ctx, ctx->extra); + if (res) + return res; + + ctx->assoc_len += (ctx->extra_size) * INT8_BIT; + ctx->extra_size = 0; + } + + /* Move to 'new_phase' */ + io_clrsetbits32(ctx->base + _CRYP_CR, _CRYP_CR_GCM_CCMPH_MSK, + new_phase << _CRYP_CR_GCM_CCMPH_OFF); + + return TEE_SUCCESS; +} + +/** + * @brief Start a AES computation. + * @param ctx: CRYP process context + * @param is_dec: true if decryption, false if encryption + * @param algo: define the algo mode + * @param key: pointer to key + * @param key_size: key size + * @param iv: pointer to initialization vector (unused if algo is ECB) + * @param iv_size: iv size + * @note this function doesn't access to hardware but stores in ctx the values + * + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_cryp_init(struct stm32_cryp_context *ctx, bool is_dec, + enum stm32_cryp_algo_mode algo, + const void *key, size_t key_size, const void *iv, + size_t iv_size) +{ + unsigned int i = 0; + const uint32_t *iv_u32 = NULL; + uint32_t local_iv[4] = { 0 }; + const uint32_t *key_u32 = NULL; + uint32_t local_key[8] = { 0 }; + + ctx->assoc_len = 0; + ctx->load_len = 0; + ctx->extra_size = 0; + ctx->lock = &cryp_lock; + + ctx->base = io_pa_or_va(&cryp_pdata.base, 1); + ctx->cr = _CRYP_CR_RESET_VALUE; + + /* We want buffer to be u32 aligned */ + if (IS_ALIGNED_WITH_TYPE(key, uint32_t)) { + key_u32 = key; + } else { + memcpy(local_key, key, key_size); + key_u32 = local_key; + } + + if (IS_ALIGNED_WITH_TYPE(iv, uint32_t)) { + iv_u32 = iv; + } else { + memcpy(local_iv, iv, iv_size); + iv_u32 = local_iv; + } + + if (is_dec) + SETBITS(ctx->cr, _CRYP_CR_ALGODIR); + else + CLRBITS(ctx->cr, _CRYP_CR_ALGODIR); + + /* Save algo mode */ + switch (algo) { + case STM32_CRYP_MODE_TDES_ECB: + SET_ALGOMODE(TDES_ECB, ctx->cr); + break; + case STM32_CRYP_MODE_TDES_CBC: + SET_ALGOMODE(TDES_CBC, ctx->cr); + break; + case STM32_CRYP_MODE_DES_ECB: + SET_ALGOMODE(DES_ECB, ctx->cr); + break; + case STM32_CRYP_MODE_DES_CBC: + SET_ALGOMODE(DES_CBC, ctx->cr); + break; + case STM32_CRYP_MODE_AES_ECB: + SET_ALGOMODE(AES_ECB, ctx->cr); + break; + case STM32_CRYP_MODE_AES_CBC: + SET_ALGOMODE(AES_CBC, ctx->cr); + break; + case STM32_CRYP_MODE_AES_CTR: + SET_ALGOMODE(AES_CTR, ctx->cr); + break; + case STM32_CRYP_MODE_AES_GCM: + SET_ALGOMODE(AES_GCM, ctx->cr); + break; + case STM32_CRYP_MODE_AES_CCM: + SET_ALGOMODE(AES_CCM, ctx->cr); + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + /* + * We will use HW Byte swap (_CRYP_CR_DATATYPE_BYTE) for data. + * So we won't need to + * TEE_U32_TO_BIG_ENDIAN(data) before write to DIN register + * nor + * TEE_U32_FROM_BIG_ENDIAN after reading from DOUT register. + */ + clrsetbits(&ctx->cr, _CRYP_CR_DATATYPE_MSK, + _CRYP_CR_DATATYPE_BYTE << _CRYP_CR_DATATYPE_OFF); + + /* + * Configure keysize for AES algorithms + * And save block size + */ + if (algo_mode_is_aes(ctx->cr)) { + switch (key_size) { + case AES_KEYSIZE_128: + clrsetbits(&ctx->cr, _CRYP_CR_KEYSIZE_MSK, + _CRYP_CR_KSIZE_128 << _CRYP_CR_KEYSIZE_OFF); + break; + case AES_KEYSIZE_192: + clrsetbits(&ctx->cr, _CRYP_CR_KEYSIZE_MSK, + _CRYP_CR_KSIZE_192 << _CRYP_CR_KEYSIZE_OFF); + break; + case AES_KEYSIZE_256: + clrsetbits(&ctx->cr, _CRYP_CR_KEYSIZE_MSK, + _CRYP_CR_KSIZE_256 << _CRYP_CR_KEYSIZE_OFF); + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + /* And set block size */ + ctx->block_u32 = AES_BLOCK_NB_U32; + } else { + /* And set DES/TDES block size */ + ctx->block_u32 = DES_BLOCK_NB_U32; + } + + /* Save key in HW order */ + ctx->key_size = key_size; + for (i = 0; i < key_size / sizeof(uint32_t); i++) + ctx->key[i] = TEE_U32_TO_BIG_ENDIAN(key_u32[i]); + + /* Save IV */ + if (algo_mode_needs_iv(ctx->cr)) { + if (!iv || iv_size != ctx->block_u32 * sizeof(uint32_t)) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * We save IV in the byte order expected by the + * IV registers + */ + for (i = 0; i < ctx->block_u32; i++) + ctx->iv[i] = TEE_U32_TO_BIG_ENDIAN(iv_u32[i]); + } + + /* Reset suspend registers */ + memset(ctx->pm_gcmccm, 0, sizeof(ctx->pm_gcmccm)); + memset(ctx->pm_gcm, 0, sizeof(ctx->pm_gcm)); + + return TEE_SUCCESS; +} + +/** + * @brief Update (or start) a AES authenticate process of + * associated data (CCM or GCM). + * @param ctx: CRYP process context + * @param data: pointer to associated data + * @param data_size: data size + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_cryp_update_assodata(struct stm32_cryp_context *ctx, + uint8_t *data, size_t data_size) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = 0; + uint32_t previous_phase = 0; + + /* If no associated data, nothing to do */ + if (!data || !data_size) + return TEE_SUCCESS; + + mutex_lock(ctx->lock); + + previous_phase = (ctx->cr & _CRYP_CR_GCM_CCMPH_MSK) >> + _CRYP_CR_GCM_CCMPH_OFF; + + switch (previous_phase) { + case _CRYP_CR_GCM_CCMPH_INIT: + res = do_from_init_to_phase(ctx, _CRYP_CR_GCM_CCMPH_HEADER); + break; + case _CRYP_CR_GCM_CCMPH_HEADER: + /* + * Function update_assodata was already called. + * We only need to restore the context. + */ + res = restore_context(ctx); + break; + default: + assert(0); + res = TEE_ERROR_BAD_STATE; + } + + if (res) + goto out; + + /* Manage if remaining data from a previous update_assodata call */ + if (ctx->extra_size && + (ctx->extra_size + data_size >= + ctx->block_u32 * sizeof(uint32_t))) { + uint32_t block[MAX_BLOCK_NB_U32] = { 0 }; + + memcpy(block, ctx->extra, ctx->extra_size); + memcpy((uint8_t *)block + ctx->extra_size, data, + ctx->block_u32 * sizeof(uint32_t) - ctx->extra_size); + + res = write_align_block(ctx, block); + if (res) + goto out; + + i += ctx->block_u32 * sizeof(uint32_t) - ctx->extra_size; + ctx->extra_size = 0; + ctx->assoc_len += ctx->block_u32 * sizeof(uint32_t) * INT8_BIT; + } + + while (data_size - i >= ctx->block_u32 * sizeof(uint32_t)) { + res = write_block(ctx, data + i); + if (res) + goto out; + + /* Process next block */ + i += ctx->block_u32 * sizeof(uint32_t); + ctx->assoc_len += ctx->block_u32 * sizeof(uint32_t) * INT8_BIT; + } + + /* + * Manage last block if not a block size multiple: + * Save remaining data to manage them later (potentially with new + * associated data). + */ + if (i < data_size) { + memcpy((uint8_t *)ctx->extra + ctx->extra_size, data + i, + data_size - i); + ctx->extra_size += data_size - i; + } + + res = save_context(ctx); +out: + if (res) + cryp_end(ctx, res); + + mutex_unlock(ctx->lock); + + return res; +} + +/** + * @brief Update (or start) a AES authenticate and de/encrypt with + * payload data (CCM or GCM). + * @param ctx: CRYP process context + * @param data_in: pointer to payload + * @param data_out: pointer where to save de/encrypted payload + * @param data_size: payload size + * + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_cryp_update_load(struct stm32_cryp_context *ctx, + uint8_t *data_in, uint8_t *data_out, + size_t data_size) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = 0; + uint32_t previous_phase = 0; + + if (!data_in || !data_size) + return TEE_SUCCESS; + + mutex_lock(ctx->lock); + + previous_phase = (ctx->cr & _CRYP_CR_GCM_CCMPH_MSK) >> + _CRYP_CR_GCM_CCMPH_OFF; + + switch (previous_phase) { + case _CRYP_CR_GCM_CCMPH_INIT: + res = do_from_init_to_phase(ctx, _CRYP_CR_GCM_CCMPH_PAYLOAD); + break; + case _CRYP_CR_GCM_CCMPH_HEADER: + res = do_from_header_to_phase(ctx, _CRYP_CR_GCM_CCMPH_PAYLOAD); + break; + case _CRYP_CR_GCM_CCMPH_PAYLOAD: + /* new update_load call, we only need to restore context */ + res = restore_context(ctx); + break; + default: + assert(0); + res = TEE_ERROR_BAD_STATE; + } + + if (res) + goto out; + + /* Manage if incomplete block from a previous update_load call */ + if (ctx->extra_size && + (ctx->extra_size + data_size >= + ctx->block_u32 * sizeof(uint32_t))) { + uint32_t block_out[MAX_BLOCK_NB_U32] = { 0 }; + + memcpy((uint8_t *)ctx->extra + ctx->extra_size, data_in + i, + ctx->block_u32 * sizeof(uint32_t) - ctx->extra_size); + + res = write_align_block(ctx, ctx->extra); + if (res) + goto out; + + res = read_align_block(ctx, block_out); + if (res) + goto out; + + memcpy(data_out + i, (uint8_t *)block_out + ctx->extra_size, + ctx->block_u32 * sizeof(uint32_t) - ctx->extra_size); + + i += ctx->block_u32 * sizeof(uint32_t) - ctx->extra_size; + ctx->extra_size = 0; + + ctx->load_len += ctx->block_u32 * sizeof(uint32_t) * INT8_BIT; + } + + while (data_size - i >= ctx->block_u32 * sizeof(uint32_t)) { + res = write_block(ctx, data_in + i); + if (res) + goto out; + + res = read_block(ctx, data_out + i); + if (res) + goto out; + + /* Process next block */ + i += ctx->block_u32 * sizeof(uint32_t); + ctx->load_len += ctx->block_u32 * sizeof(uint32_t) * INT8_BIT; + } + + res = save_context(ctx); + if (res) + goto out; + + /* + * Manage last block if not a block size multiple + * We saved context, + * Complete block with 0 and send to CRYP to get {en,de}crypted data + * Store data to resend as last block in final() + * or to complete next update_load() to get correct tag. + */ + if (i < data_size) { + uint32_t block_out[MAX_BLOCK_NB_U32] = { 0 }; + size_t prev_extra_size = ctx->extra_size; + + /* Re-enable the CRYP peripheral */ + io_setbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); + + memcpy((uint8_t *)ctx->extra + ctx->extra_size, data_in + i, + data_size - i); + ctx->extra_size += data_size - i; + memset((uint8_t *)ctx->extra + ctx->extra_size, 0, + ctx->block_u32 * sizeof(uint32_t) - ctx->extra_size); + + res = write_align_block(ctx, ctx->extra); + if (res) + goto out; + + res = read_align_block(ctx, block_out); + if (res) + goto out; + + memcpy(data_out + i, (uint8_t *)block_out + prev_extra_size, + data_size - i); + + /* Disable the CRYP peripheral */ + io_clrbits32(ctx->base + _CRYP_CR, _CRYP_CR_CRYPEN); + } + +out: + if (res) + cryp_end(ctx, res); + + mutex_unlock(ctx->lock); + + return res; +} + +/** + * @brief Get authentication tag for AES authenticated algorithms (CCM or GCM). + * @param ctx: CRYP process context + * @param tag: pointer where to save the tag + * @param data_size: tag size + * + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_cryp_final(struct stm32_cryp_context *ctx, uint8_t *tag, + size_t tag_size) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t tag_u32[4] = { 0 }; + uint32_t previous_phase = 0; + + mutex_lock(ctx->lock); + + previous_phase = (ctx->cr & _CRYP_CR_GCM_CCMPH_MSK) >> + _CRYP_CR_GCM_CCMPH_OFF; + + switch (previous_phase) { + case _CRYP_CR_GCM_CCMPH_INIT: + res = do_from_init_to_phase(ctx, _CRYP_CR_GCM_CCMPH_FINAL); + break; + case _CRYP_CR_GCM_CCMPH_HEADER: + res = do_from_header_to_phase(ctx, _CRYP_CR_GCM_CCMPH_FINAL); + break; + case _CRYP_CR_GCM_CCMPH_PAYLOAD: + res = restore_context(ctx); + if (res) + break; + + /* Manage if incomplete block from a previous update_load() */ + if (ctx->extra_size) { + uint32_t block_out[MAX_BLOCK_NB_U32] = { 0 }; + size_t sz = ctx->block_u32 * sizeof(uint32_t) - + ctx->extra_size; + + if (does_need_npblb(ctx->cr)) { + io_clrsetbits32(ctx->base + _CRYP_CR, + _CRYP_CR_NPBLB_MSK, + sz << _CRYP_CR_NPBLB_OFF); + } + + memset((uint8_t *)ctx->extra + ctx->extra_size, 0, sz); + + res = write_align_block(ctx, ctx->extra); + if (res) + break; + + /* Don't care {en,de}crypted data, already saved */ + res = read_align_block(ctx, block_out); + if (res) + break; + + ctx->load_len += (ctx->extra_size * INT8_BIT); + ctx->extra_size = 0; + } + + /* Move to final phase */ + io_clrsetbits32(ctx->base + _CRYP_CR, _CRYP_CR_GCM_CCMPH_MSK, + _CRYP_CR_GCM_CCMPH_FINAL << + _CRYP_CR_GCM_CCMPH_OFF); + break; + default: + assert(0); + res = TEE_ERROR_BAD_STATE; + } + + if (res) + goto out; + + if (IS_ALGOMODE(ctx->cr, AES_GCM)) { + /* No need to htobe() as we configure the HW to swap bytes */ + io_write32(ctx->base + _CRYP_DIN, 0U); + io_write32(ctx->base + _CRYP_DIN, ctx->assoc_len); + io_write32(ctx->base + _CRYP_DIN, 0U); + io_write32(ctx->base + _CRYP_DIN, ctx->load_len); + } else if (IS_ALGOMODE(ctx->cr, AES_CCM)) { + /* No need to htobe() in this phase */ + res = write_align_block(ctx, ctx->ctr0_ccm); + if (res) + goto out; + } + + res = read_align_block(ctx, tag_u32); + if (res) + goto out; + + memcpy(tag, tag_u32, MIN(sizeof(tag_u32), tag_size)); + +out: + cryp_end(ctx, res); + mutex_unlock(ctx->lock); + + return res; +} + +/** + * @brief Update (or start) a de/encrypt process. + * @param ctx: CRYP process context + * @param last_block: true if last payload data block + * @param data_in: pointer to payload + * @param data_out: pointer where to save de/encrypted payload + * @param data_size: payload size + * + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_cryp_update(struct stm32_cryp_context *ctx, bool last_block, + uint8_t *data_in, uint8_t *data_out, + size_t data_size) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = 0; + + mutex_lock(ctx->lock); + + /* + * In CBC and ECB encryption we need to manage specifically last + * 2 blocks if total size in not aligned to a block size. + * Currently return TEE_ERROR_NOT_IMPLEMENTED. Moreover as we need to + * know last 2 blocks, if unaligned and call with less than two blocks, + * return TEE_ERROR_BAD_STATE. + */ + if (last_block && algo_mode_is_ecb_cbc(ctx->cr) && + is_encrypt(ctx->cr) && + (ROUNDDOWN(data_size, ctx->block_u32 * sizeof(uint32_t)) != + data_size)) { + if (data_size < ctx->block_u32 * sizeof(uint32_t) * 2) { + /* + * If CBC, size of the last part should be at + * least 2*BLOCK_SIZE + */ + EMSG("Unexpected last block size"); + res = TEE_ERROR_BAD_STATE; + goto out; + } + /* + * Moreover the ECB/CBC specific padding for encrypt is not + * yet implemented, and not used in OPTEE + */ + res = TEE_ERROR_NOT_IMPLEMENTED; + goto out; + } + + /* Manage remaining CTR mask from previous update call */ + if (IS_ALGOMODE(ctx->cr, AES_CTR) && ctx->extra_size) { + unsigned int j = 0; + uint8_t *mask = (uint8_t *)ctx->extra; + + for (j = 0; j < ctx->extra_size && i < data_size; j++, i++) + data_out[i] = data_in[i] ^ mask[j]; + + if (j != ctx->extra_size) { + /* + * We didn't consume all saved mask, + * but no more data. + */ + + /* We save remaining mask and its new size */ + memmove(ctx->extra, ctx->extra + j, + ctx->extra_size - j); + ctx->extra_size -= j; + + /* + * We don't need to save HW context we didn't + * modify HW state. + */ + res = TEE_SUCCESS; + goto out; + } + + /* All extra mask consumed */ + ctx->extra_size = 0; + } + + res = restore_context(ctx); + if (res) + goto out; + + while (data_size - i >= ctx->block_u32 * sizeof(uint32_t)) { + /* + * We only write/read one block at a time + * but CRYP use a in (and out) FIFO of 8 * uint32_t + */ + res = write_block(ctx, data_in + i); + if (res) + goto out; + + res = read_block(ctx, data_out + i); + if (res) + goto out; + + /* Process next block */ + i += ctx->block_u32 * sizeof(uint32_t); + } + + /* Manage last block if not a block size multiple */ + if (i < data_size) { + uint32_t block_in[MAX_BLOCK_NB_U32] = { 0 }; + uint32_t block_out[MAX_BLOCK_NB_U32] = { 0 }; + + if (!IS_ALGOMODE(ctx->cr, AES_CTR)) { + /* + * Other algorithm than CTR can manage only multiple + * of block_size. + */ + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * For CTR we save the generated mask to use it at next + * update call. + */ + memcpy(block_in, data_in + i, data_size - i); + + res = write_align_block(ctx, block_in); + if (res) + goto out; + + res = read_align_block(ctx, block_out); + if (res) + goto out; + + memcpy(data_out + i, block_out, data_size - i); + + /* Save mask for possibly next call */ + ctx->extra_size = ctx->block_u32 * sizeof(uint32_t) - + (data_size - i); + memcpy(ctx->extra, (uint8_t *)block_out + data_size - i, + ctx->extra_size); + } + + if (!last_block) + res = save_context(ctx); + +out: + /* If last block or error, end of CRYP process */ + if (last_block || res) + cryp_end(ctx, res); + + mutex_unlock(ctx->lock); + + return res; +} + +static TEE_Result stm32_cryp_probe(const void *fdt, int node, + const void *compt_data __unused) +{ + TEE_Result res = TEE_SUCCESS; + struct dt_node_info dt_cryp = { }; + struct rstctrl *rstctrl = NULL; + struct clk *clk = NULL; + + fdt_fill_device_info(fdt, &dt_cryp, node); + + if (dt_cryp.reg == DT_INFO_INVALID_REG || + dt_cryp.reg_size == DT_INFO_INVALID_REG_SIZE) + panic(); + + res = clk_dt_get_by_index(fdt, node, 0, &clk); + if (res) + return res; + + res = rstctrl_dt_get_by_index(fdt, node, 0, &rstctrl); + if (res) + return res; + + cryp_pdata.clock = clk; + cryp_pdata.reset = rstctrl; + cryp_pdata.base.pa = dt_cryp.reg; + + io_pa_or_va_secure(&cryp_pdata.base, dt_cryp.reg_size); + if (!cryp_pdata.base.va) + panic(); + + stm32mp_register_secure_periph_iomem(cryp_pdata.base.pa); + + if (clk_enable(cryp_pdata.clock)) + panic(); + + if (rstctrl_assert_to(cryp_pdata.reset, TIMEOUT_US_1MS)) + panic(); + + if (rstctrl_deassert_to(cryp_pdata.reset, TIMEOUT_US_1MS)) + panic(); + + if (IS_ENABLED(CFG_CRYPTO_DRV_AUTHENC)) { + res = stm32_register_authenc(); + if (res) { + EMSG("Failed to register to authenc: %#"PRIx32, res); + panic(); + } + } + + if (IS_ENABLED(CFG_CRYPTO_DRV_CIPHER)) { + res = stm32_register_cipher(CRYP_IP); + if (res) { + EMSG("Failed to register to cipher: %#"PRIx32, res); + panic(); + } + } + + return TEE_SUCCESS; +} + +static const struct dt_device_match stm32_cryp_match_table[] = { + { .compatible = "st,stm32mp1-cryp" }, + { } +}; + +DEFINE_DT_DRIVER(stm32_cryp_dt_driver) = { + .name = "stm32-cryp", + .match_table = stm32_cryp_match_table, + .probe = stm32_cryp_probe, +}; diff --git a/optee_os/core/drivers/crypto/stm32/stm32_cryp.h b/optee_os/core/drivers/crypto/stm32/stm32_cryp.h new file mode 100644 index 0000000..a8f609c --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/stm32_cryp.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + */ + +#ifndef STM32_CRYP_H +#define STM32_CRYP_H + +#include +#include +#include +#include +#include +#include +#include + +/* + * Platform data related to CRYP instance + * @base - IO memory base address + * @clk - CRYP clock reference + * @rstctrl - CRYP reset controller reference + */ +struct stm32_cryp_platdata { + struct io_pa_va base; + struct clk *clock; + struct rstctrl *reset; +}; + +enum stm32_cryp_algo_mode { + STM32_CRYP_MODE_TDES_ECB, + STM32_CRYP_MODE_TDES_CBC, + STM32_CRYP_MODE_DES_ECB, + STM32_CRYP_MODE_DES_CBC, + STM32_CRYP_MODE_AES_ECB, + STM32_CRYP_MODE_AES_CBC, + STM32_CRYP_MODE_AES_CTR, + STM32_CRYP_MODE_AES_GCM, + STM32_CRYP_MODE_AES_CCM, +}; + +/* + * Full CRYP context. + * Store CRYP internal state to be able to compute any supported algorithm. + */ +struct stm32_cryp_context { + vaddr_t base; + uint32_t cr; + struct mutex *lock; /* Protect CRYP HW instance access */ + uint32_t assoc_len; + uint32_t load_len; + uint32_t key[8]; /* In HW byte order */ + size_t key_size; + size_t block_u32; + uint32_t iv[4]; /* In HW byte order */ + uint32_t pm_gcmccm[8]; + union { + uint32_t pm_gcm[8]; + uint32_t ctr0_ccm[4]; + }; + uint32_t extra[4]; + size_t extra_size; +}; + +TEE_Result stm32_cryp_init(struct stm32_cryp_context *ctx, bool is_decrypt, + enum stm32_cryp_algo_mode mode, + const void *key, size_t key_size, const void *iv, + size_t iv_size); +TEE_Result stm32_cryp_update(struct stm32_cryp_context *ctx, bool last_block, + uint8_t *data_in, uint8_t *data_out, + size_t data_size); +TEE_Result stm32_cryp_update_assodata(struct stm32_cryp_context *ctx, + uint8_t *data, size_t data_size); +TEE_Result stm32_cryp_update_load(struct stm32_cryp_context *ctx, + uint8_t *data_in, uint8_t *data_out, + size_t data_size); +TEE_Result stm32_cryp_final(struct stm32_cryp_context *ctx, uint8_t *tag, + size_t tag_size); +#endif diff --git a/optee_os/core/drivers/crypto/stm32/stm32_saes.c b/optee_os/core/drivers/crypto/stm32/stm32_saes.c new file mode 100644 index 0000000..75f8bf8 --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/stm32_saes.c @@ -0,0 +1,1427 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2023, STMicroelectronics - All Rights Reserved + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "stm32_saes.h" + +/* SAES control register */ +#define _SAES_CR U(0x0) +/* SAES status register */ +#define _SAES_SR U(0x04) +/* SAES data input register */ +#define _SAES_DINR U(0x08) +/* SAES data output register */ +#define _SAES_DOUTR U(0x0c) +/* SAES key registers [0-3] */ +#define _SAES_KEYR0 U(0x10) +#define _SAES_KEYR1 U(0x14) +#define _SAES_KEYR2 U(0x18) +#define _SAES_KEYR3 U(0x1c) +/* SAES initialization vector registers [0-3] */ +#define _SAES_IVR0 U(0x20) +#define _SAES_IVR1 U(0x24) +#define _SAES_IVR2 U(0x28) +#define _SAES_IVR3 U(0x2c) +/* SAES key registers [4-7] */ +#define _SAES_KEYR4 U(0x30) +#define _SAES_KEYR5 U(0x34) +#define _SAES_KEYR6 U(0x38) +#define _SAES_KEYR7 U(0x3c) +/* SAES suspend registers [0-7] */ +#define _SAES_SUSPR0 U(0x40) +#define _SAES_SUSPR1 U(0x44) +#define _SAES_SUSPR2 U(0x48) +#define _SAES_SUSPR3 U(0x4c) +#define _SAES_SUSPR4 U(0x50) +#define _SAES_SUSPR5 U(0x54) +#define _SAES_SUSPR6 U(0x58) +#define _SAES_SUSPR7 U(0x5c) +/* SAES Interrupt Enable Register */ +#define _SAES_IER U(0x300) +/* SAES Interrupt Status Register */ +#define _SAES_ISR U(0x304) +/* SAES Interrupt Clear Register */ +#define _SAES_ICR U(0x308) + +/* SAES control register fields */ +#define _SAES_CR_RESET_VALUE U(0x0) +#define _SAES_CR_IPRST BIT(31) +#define _SAES_CR_KEYSEL_MASK GENMASK_32(30, 28) +#define _SAES_CR_KEYSEL_SHIFT U(28) +#define _SAES_CR_KEYSEL_SOFT U(0x0) +#define _SAES_CR_KEYSEL_DHUK U(0x1) +#define _SAES_CR_KEYSEL_BHK U(0x2) +#define _SAES_CR_KEYSEL_BHU_XOR_BH_K U(0x4) +#define _SAES_CR_KEYSEL_TEST U(0x7) +#define _SAES_CR_KSHAREID_MASK GENMASK_32(27, 26) +#define _SAES_CR_KSHAREID_SHIFT U(26) +#define _SAES_CR_KSHAREID_CRYP U(0x0) +#define _SAES_CR_KEYMOD_MASK GENMASK_32(25, 24) +#define _SAES_CR_KEYMOD_SHIFT U(24) +#define _SAES_CR_KEYMOD_NORMAL U(0x0) +#define _SAES_CR_KEYMOD_WRAPPED U(0x1) +#define _SAES_CR_KEYMOD_SHARED U(0x2) +#define _SAES_CR_NPBLB_MASK GENMASK_32(23, 20) +#define _SAES_CR_NPBLB_SHIFT U(20) +#define _SAES_CR_KEYPROT BIT(19) +#define _SAES_CR_KEYSIZE BIT(18) +#define _SAES_CR_GCMPH_MASK GENMASK_32(14, 13) +#define _SAES_CR_GCMPH_SHIFT U(13) +#define _SAES_CR_GCMPH_INIT U(0) +#define _SAES_CR_GCMPH_HEADER U(1) +#define _SAES_CR_GCMPH_PAYLOAD U(2) +#define _SAES_CR_GCMPH_FINAL U(3) +#define _SAES_CR_DMAOUTEN BIT(12) +#define _SAES_CR_DMAINEN BIT(11) +#define _SAES_CR_CHMOD_MASK (BIT(16) | GENMASK_32(6, 5)) +#define _SAES_CR_CHMOD_SHIFT U(5) +#define _SAES_CR_CHMOD_ECB U(0x0) +#define _SAES_CR_CHMOD_CBC U(0x1) +#define _SAES_CR_CHMOD_CTR U(0x2) +#define _SAES_CR_CHMOD_GCM U(0x3) +#define _SAES_CR_CHMOD_GMAC U(0x3) +#define _SAES_CR_CHMOD_CCM U(0x800) +#define _SAES_CR_MODE_MASK GENMASK_32(4, 3) +#define _SAES_CR_MODE_SHIFT U(3) +#define _SAES_CR_MODE_ENC U(0) +#define _SAES_CR_MODE_KEYPREP U(1) +#define _SAES_CR_MODE_DEC U(2) +#define _SAES_CR_DATATYPE_MASK GENMASK_32(2, 1) +#define _SAES_CR_DATATYPE_SHIFT U(1) +#define _SAES_CR_DATATYPE_NONE U(0) +#define _SAES_CR_DATATYPE_HALF_WORD U(1) +#define _SAES_CR_DATATYPE_BYTE U(2) +#define _SAES_CR_DATATYPE_BIT U(3) +#define _SAES_CR_EN BIT(0) + +/* SAES status register fields */ +#define _SAES_SR_KEYVALID BIT(7) +#define _SAES_SR_BUSY BIT(3) +#define _SAES_SR_WRERR BIT(2) +#define _SAES_SR_RDERR BIT(1) +#define _SAES_SR_CCF BIT(0) + +/* SAES interrupt registers fields */ +#define _SAES_I_RNG_ERR BIT(3) +#define _SAES_I_KEY_ERR BIT(2) +#define _SAES_I_RW_ERR BIT(1) +#define _SAES_I_CC BIT(0) + +#define SAES_TIMEOUT_US U(100000) +#define TIMEOUT_US_1MS U(1000) +#define SAES_RESET_DELAY U(2) + +#define IS_CHAINING_MODE(mode, cr) \ + (((cr) & _SAES_CR_CHMOD_MASK) == (_SAES_CR_CHMOD_##mode << \ + _SAES_CR_CHMOD_SHIFT)) + +#define SET_CHAINING_MODE(mode, cr) \ + set_field_u32(cr, _SAES_CR_CHMOD_MASK, _SAES_CR_CHMOD_##mode) + +static struct mutex saes_lock = MUTEX_INITIALIZER; +static struct stm32_saes_platdata { + vaddr_t base; + struct clk *clk; + struct rstctrl *reset; +} saes_pdata; + +static bool does_chaining_mode_need_iv(uint32_t cr) +{ + return !IS_CHAINING_MODE(ECB, cr); +} + +static bool is_encrypt(uint32_t cr) +{ + return (cr & _SAES_CR_MODE_MASK) == + SHIFT_U32(_SAES_CR_MODE_ENC, _SAES_CR_MODE_SHIFT); +} + +static bool is_decrypt(uint32_t cr) +{ + return (cr & _SAES_CR_MODE_MASK) == + SHIFT_U32(_SAES_CR_MODE_DEC, _SAES_CR_MODE_SHIFT); +} + +static bool does_need_npblb(uint32_t cr) +{ + return (IS_CHAINING_MODE(GCM, cr) && is_encrypt(cr)) || + (IS_CHAINING_MODE(CCM, cr) && is_decrypt(cr)); +} + +static bool can_suspend(uint32_t cr) +{ + return !IS_CHAINING_MODE(GCM, cr); +} + +static void write_aligned_block(vaddr_t base, uint32_t *data) +{ + unsigned int i = 0; + + /* SAES is configured to swap bytes as expected */ + for (i = 0; i < AES_BLOCK_NB_U32; i++) + io_write32(base + _SAES_DINR, data[i]); +} + +static void write_block(vaddr_t base, uint8_t *data) +{ + if (IS_ALIGNED_WITH_TYPE(data, uint32_t)) { + write_aligned_block(base, (void *)data); + } else { + uint32_t data_u32[AES_BLOCK_NB_U32] = { }; + + memcpy(data_u32, data, sizeof(data_u32)); + write_aligned_block(base, data_u32); + } +} + +static void read_aligned_block(vaddr_t base, uint32_t *data) +{ + unsigned int i = 0; + + /* SAES is configured to swap bytes as expected */ + for (i = 0; i < AES_BLOCK_NB_U32; i++) + data[i] = io_read32(base + _SAES_DOUTR); +} + +static void read_block(vaddr_t base, uint8_t *data) +{ + if (IS_ALIGNED_WITH_TYPE(data, uint32_t)) { + read_aligned_block(base, (void *)data); + } else { + uint32_t data_u32[AES_BLOCK_NB_U32] = { }; + + read_aligned_block(base, data_u32); + + memcpy(data, data_u32, sizeof(data_u32)); + } +} + +static TEE_Result wait_computation_completed(vaddr_t base) +{ + uint64_t timeout_ref = timeout_init_us(SAES_TIMEOUT_US); + + while ((io_read32(base + _SAES_SR) & _SAES_SR_CCF) != _SAES_SR_CCF) + if (timeout_elapsed(timeout_ref)) + break; + + if ((io_read32(base + _SAES_SR) & _SAES_SR_CCF) != _SAES_SR_CCF) { + DMSG("CCF timeout"); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +static void clear_computation_completed(uintptr_t base) +{ + io_setbits32(base + _SAES_ICR, _SAES_I_CC); +} + +static TEE_Result wait_key_valid(vaddr_t base) +{ + uint64_t timeout_ref = timeout_init_us(SAES_TIMEOUT_US); + + while (!(io_read32(base + _SAES_SR) & _SAES_SR_KEYVALID)) + if (timeout_elapsed(timeout_ref)) + break; + + if (!(io_read32(base + _SAES_SR) & _SAES_SR_KEYVALID)) { + DMSG("CCF timeout"); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +static TEE_Result saes_start(struct stm32_saes_context *ctx) +{ + uint64_t timeout_ref = 0; + + /* Reset SAES */ + io_setbits32(ctx->base + _SAES_CR, _SAES_CR_IPRST); + io_clrbits32(ctx->base + _SAES_CR, _SAES_CR_IPRST); + + timeout_ref = timeout_init_us(SAES_TIMEOUT_US); + while (io_read32(ctx->base + _SAES_SR) & _SAES_SR_BUSY) + if (timeout_elapsed(timeout_ref)) + break; + + if (io_read32(ctx->base + _SAES_SR) & _SAES_SR_BUSY) { + DMSG("busy timeout"); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +static void saes_end(struct stm32_saes_context *ctx, int prev_error) +{ + if (prev_error) { + /* Reset SAES */ + io_setbits32(ctx->base + _SAES_CR, _SAES_CR_IPRST); + io_clrbits32(ctx->base + _SAES_CR, _SAES_CR_IPRST); + } + + /* Disable the SAES peripheral */ + io_clrbits32(ctx->base + _SAES_CR, _SAES_CR_EN); +} + +static void saes_write_iv(struct stm32_saes_context *ctx) +{ + /* If chaining mode need to restore IV */ + if (does_chaining_mode_need_iv(ctx->cr)) { + unsigned int i = 0; + + for (i = 0; i < AES_IVSIZE / sizeof(uint32_t); i++) { + io_write32(ctx->base + _SAES_IVR0 + i * + sizeof(uint32_t), ctx->iv[i]); + } + } +} + +static void saes_save_suspend(struct stm32_saes_context *ctx) +{ + size_t i = 0; + + for (i = 0; i < 8; i++) + ctx->susp[i] = io_read32(ctx->base + _SAES_SUSPR0 + + i * sizeof(uint32_t)); +} + +static void saes_restore_suspend(struct stm32_saes_context *ctx) +{ + size_t i = 0; + + for (i = 0; i < 8; i++) + io_write32(ctx->base + _SAES_SUSPR0 + i * sizeof(uint32_t), + ctx->susp[i]); +} + +static void saes_write_key(struct stm32_saes_context *ctx) +{ + /* Restore the _SAES_KEYRx if SOFTWARE key */ + if ((ctx->cr & _SAES_CR_KEYSEL_MASK) == + SHIFT_U32(_SAES_CR_KEYSEL_SOFT, _SAES_CR_KEYSEL_SHIFT)) { + size_t i = 0; + + for (i = 0; i < AES_KEYSIZE_128 / sizeof(uint32_t); i++) + io_write32(ctx->base + _SAES_KEYR0 + i * + sizeof(uint32_t), + ctx->key[i]); + + if ((ctx->cr & _SAES_CR_KEYSIZE) == _SAES_CR_KEYSIZE) { + for (i = 0; + i < (AES_KEYSIZE_256 / 2) / sizeof(uint32_t); + i++) { + io_write32(ctx->base + _SAES_KEYR4 + i * + sizeof(uint32_t), + ctx->key[i + 4]); + } + } + } +} + +static TEE_Result saes_prepare_key(struct stm32_saes_context *ctx) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + /* Disable the SAES peripheral */ + io_clrbits32(ctx->base + _SAES_CR, _SAES_CR_EN); + + /* Set key size */ + if ((ctx->cr & _SAES_CR_KEYSIZE)) + io_setbits32(ctx->base + _SAES_CR, _SAES_CR_KEYSIZE); + else + io_clrbits32(ctx->base + _SAES_CR, _SAES_CR_KEYSIZE); + + saes_write_key(ctx); + + res = wait_key_valid(ctx->base); + if (res) + return res; + + /* + * For ECB/CBC decryption, key preparation mode must be selected + * to populate the key. + */ + if ((IS_CHAINING_MODE(ECB, ctx->cr) || + IS_CHAINING_MODE(CBC, ctx->cr)) && is_decrypt(ctx->cr)) { + /* Select Mode 2 */ + io_clrsetbits32(ctx->base + _SAES_CR, _SAES_CR_MODE_MASK, + SHIFT_U32(_SAES_CR_MODE_KEYPREP, + _SAES_CR_MODE_SHIFT)); + + /* Enable SAES */ + io_setbits32(ctx->base + _SAES_CR, _SAES_CR_EN); + + res = wait_computation_completed(ctx->base); + if (res) + return res; + + clear_computation_completed(ctx->base); + + /* Set Mode 3 */ + io_clrsetbits32(ctx->base + _SAES_CR, _SAES_CR_MODE_MASK, + SHIFT_U32(_SAES_CR_MODE_DEC, + _SAES_CR_MODE_SHIFT)); + } + + return TEE_SUCCESS; +} + +static TEE_Result save_context(struct stm32_saes_context *ctx) +{ + if ((io_read32(ctx->base + _SAES_SR) & _SAES_SR_CCF)) { + /* Device should not be in a processing phase */ + return TEE_ERROR_BAD_STATE; + } + + /* Save CR */ + ctx->cr = io_read32(ctx->base + _SAES_CR); + + if (!can_suspend(ctx->cr)) + return TEE_SUCCESS; + + saes_save_suspend(ctx); + + /* If chaining mode need to save current IV */ + if (does_chaining_mode_need_iv(ctx->cr)) { + uint8_t i = 0; + + /* Save IV */ + for (i = 0; i < AES_IVSIZE / sizeof(uint32_t); i++) { + ctx->iv[i] = io_read32(ctx->base + _SAES_IVR0 + i * + sizeof(uint32_t)); + } + } + + /* Disable the SAES peripheral */ + io_clrbits32(ctx->base + _SAES_CR, _SAES_CR_EN); + + return TEE_SUCCESS; +} + +/* To resume the processing of a message */ +static TEE_Result restore_context(struct stm32_saes_context *ctx) +{ + TEE_Result res = TEE_SUCCESS; + + /* SAES shall be disabled */ + if ((io_read32(ctx->base + _SAES_CR) & _SAES_CR_EN)) { + DMSG("Device is still enabled"); + return TEE_ERROR_BAD_STATE; + } + + /* Reset internal state */ + io_setbits32(ctx->base + _SAES_CR, _SAES_CR_IPRST); + + /* Restore configuration register */ + io_write32(ctx->base + _SAES_CR, ctx->cr); + + /* Write key and, in case of CBC or ECB decrypt, prepare it */ + res = saes_prepare_key(ctx); + if (res) + return res; + + saes_restore_suspend(ctx); + + saes_write_iv(ctx); + + /* Enable the SAES peripheral */ + io_setbits32(ctx->base + _SAES_CR, _SAES_CR_EN); + + return TEE_SUCCESS; +} + +static TEE_Result do_from_init_to_phase(struct stm32_saes_context *ctx, + uint32_t new_phase) +{ + TEE_Result res = TEE_SUCCESS; + + /* We didn't run the init phase yet */ + res = restore_context(ctx); + if (res) + return res; + + res = wait_computation_completed(ctx->base); + if (res) + return res; + + clear_computation_completed(ctx->base); + + /* Move to 'new_phase' */ + io_clrsetbits32(ctx->base + _SAES_CR, _SAES_CR_GCMPH_MASK, + SHIFT_U32(new_phase, _SAES_CR_GCMPH_SHIFT)); + + /* Enable the SAES peripheral (init disabled it) */ + io_setbits32(ctx->base + _SAES_CR, _SAES_CR_EN); + + return TEE_SUCCESS; +} + +static TEE_Result do_from_header_to_phase(struct stm32_saes_context *ctx, + uint32_t new_phase) +{ + TEE_Result res = TEE_SUCCESS; + + if (can_suspend(ctx->cr)) { + res = restore_context(ctx); + if (res) + return res; + } + + if (ctx->extra_size) { + /* Manage unaligned header data before moving to next phase */ + memset((uint8_t *)ctx->extra + ctx->extra_size, 0, + AES_BLOCK_SIZE - ctx->extra_size); + + write_aligned_block(ctx->base, ctx->extra); + + res = wait_computation_completed(ctx->base); + if (res) + return res; + + clear_computation_completed(ctx->base); + + ctx->assoc_len += ctx->extra_size * INT8_BIT; + ctx->extra_size = U(0); + } + + /* Move to 'new_phase' */ + io_clrsetbits32(ctx->base + _SAES_CR, _SAES_CR_GCMPH_MASK, + SHIFT_U32(new_phase, _SAES_CR_GCMPH_SHIFT)); + + return TEE_SUCCESS; +} + +/** + * @brief Start an AES computation. + * @param ctx: SAES process context + * @param is_dec: true if decryption, false if encryption + * @param ch_mode: define the chaining mode + * @param key_select: define where the key comes from + * @param key: pointer to key (if key_select is KEY_SOFT, else unused) + * @param key_size: key size + * @param iv: pointer to initialization vector (unused if ch_mode is ECB) + * @param iv_size: iv size + * @note this function doesn't access to hardware but stores in ctx the values + * + * @retval TEE_SUCCESS if OK or a TEE_Result compliant code. + */ +TEE_Result stm32_saes_init(struct stm32_saes_context *ctx, bool is_dec, + enum stm32_saes_chaining_mode ch_mode, + enum stm32_saes_key_selection key_select, + const void *key, size_t key_size, const void *iv, + size_t iv_size) +{ + const uint32_t *key_u32 = NULL; + const uint32_t *iv_u32 = NULL; + uint32_t local_key[8] = { }; + uint32_t local_iv[4] = { }; + unsigned int i = 0; + + if (!ctx) + return TEE_ERROR_BAD_PARAMETERS; + + *ctx = (struct stm32_saes_context){ + .lock = &saes_lock, + .base = saes_pdata.base, + .cr = _SAES_CR_RESET_VALUE + }; + + /* We want buffer to be u32 aligned */ + if (IS_ALIGNED_WITH_TYPE(key, uint32_t)) { + key_u32 = key; + } else { + memcpy(local_key, key, key_size); + key_u32 = local_key; + } + + if (IS_ALIGNED_WITH_TYPE(iv, uint32_t)) { + iv_u32 = iv; + } else { + memcpy(local_iv, iv, iv_size); + iv_u32 = local_iv; + } + + if (is_dec) + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_MODE_MASK, + _SAES_CR_MODE_DEC); + else + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_MODE_MASK, + _SAES_CR_MODE_ENC); + + /* Save chaining mode */ + switch (ch_mode) { + case STM32_SAES_MODE_ECB: + ctx->cr |= SET_CHAINING_MODE(ECB, ctx->cr); + break; + case STM32_SAES_MODE_CBC: + ctx->cr |= SET_CHAINING_MODE(CBC, ctx->cr); + break; + case STM32_SAES_MODE_CTR: + ctx->cr |= SET_CHAINING_MODE(CTR, ctx->cr); + break; + case STM32_SAES_MODE_GCM: + ctx->cr |= SET_CHAINING_MODE(GCM, ctx->cr); + break; + case STM32_SAES_MODE_CCM: + ctx->cr |= SET_CHAINING_MODE(CCM, ctx->cr); + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + /* + * We will use HW Byte swap (_SAES_CR_DATATYPE_BYTE) for data. + * So we won't need to + * TEE_U32_TO_BIG_ENDIAN(data) before write to DINR + * nor + * TEE_U32_FROM_BIG_ENDIAN after reading from DOUTR. + * + * But note that wrap key only accept _SAES_CR_DATATYPE_NONE. + */ + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_DATATYPE_MASK, + _SAES_CR_DATATYPE_BYTE); + + /* Configure keysize */ + switch (key_size) { + case AES_KEYSIZE_128: + ctx->cr &= ~_SAES_CR_KEYSIZE; + break; + case AES_KEYSIZE_256: + ctx->cr |= _SAES_CR_KEYSIZE; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Configure key */ + switch (key_select) { + case STM32_SAES_KEY_SOFT: + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_KEYSEL_MASK, + _SAES_CR_KEYSEL_SOFT); + /* Save key */ + switch (key_size) { + case AES_KEYSIZE_128: + /* First 16 bytes == 4 u32 */ + for (i = 0; i < AES_KEYSIZE_128 / sizeof(uint32_t); + i++) { + ctx->key[i] = + TEE_U32_TO_BIG_ENDIAN(key_u32[3 - i]); + /* + * /!\ we save the key in HW byte order + * and word order: key[i] is for _SAES_KEYRi. + */ + } + break; + case AES_KEYSIZE_256: + for (i = 0; i < AES_KEYSIZE_256 / sizeof(uint32_t); + i++) { + ctx->key[i] = + TEE_U32_TO_BIG_ENDIAN(key_u32[7 - i]); + /* + * /!\ we save the key in HW byte order + * and word order: key[i] is for _SAES_KEYRi. + */ + } + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + break; + case STM32_SAES_KEY_DHU: + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_KEYSEL_MASK, + _SAES_CR_KEYSEL_DHUK); + break; + case STM32_SAES_KEY_BH: + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_KEYSEL_MASK, + _SAES_CR_KEYSEL_BHK); + break; + case STM32_SAES_KEY_BHU_XOR_BH: + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_KEYSEL_MASK, + _SAES_CR_KEYSEL_BHU_XOR_BH_K); + break; + case STM32_SAES_KEY_WRAPPED: + ctx->cr |= set_field_u32(ctx->cr, _SAES_CR_KEYSEL_MASK, + _SAES_CR_KEYSEL_SOFT); + break; + + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Save IV */ + if (ch_mode != STM32_SAES_MODE_ECB) { + if (!iv || iv_size != AES_IVSIZE) + return TEE_ERROR_BAD_PARAMETERS; + + for (i = 0; i < AES_IVSIZE / sizeof(uint32_t); i++) + ctx->iv[i] = TEE_U32_TO_BIG_ENDIAN(iv_u32[3 - i]); + } + + /* Reset suspend registers */ + memset(ctx->susp, 0, sizeof(ctx->susp)); + + return saes_start(ctx); +} + +/** + * @brief Update (or start) an AES authentificate process of + * associated data (CCM or GCM). + * @param ctx: SAES process context + * @param data: pointer to associated data + * @param data_size: data size + * + * @retval 0 if OK. + */ +TEE_Result stm32_saes_update_assodata(struct stm32_saes_context *ctx, + uint8_t *data, size_t data_size) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = 0; + uint32_t previous_phase = 0; + + if (!ctx) + return TEE_ERROR_BAD_PARAMETERS; + + /* If no associated data, nothing to do */ + if (!data || !data_size) + return TEE_SUCCESS; + + mutex_lock(ctx->lock); + + previous_phase = (ctx->cr & _SAES_CR_GCMPH_MASK) >> + _SAES_CR_GCMPH_SHIFT; + + switch (previous_phase) { + case _SAES_CR_GCMPH_INIT: + res = do_from_init_to_phase(ctx, _SAES_CR_GCMPH_HEADER); + break; + case _SAES_CR_GCMPH_HEADER: + /* + * Function update_assodata() was already called. + * We only need to restore the context. + */ + if (can_suspend(ctx->cr)) + res = restore_context(ctx); + + break; + default: + DMSG("out of order call"); + res = TEE_ERROR_BAD_STATE; + } + + if (res) + goto out; + + /* Manage if remaining data from a previous update_assodata() call */ + if (ctx->extra_size && + ((ctx->extra_size + data_size) >= AES_BLOCK_SIZE)) { + uint32_t block[AES_BLOCK_NB_U32] = { }; + + memcpy(block, ctx->extra, ctx->extra_size); + memcpy((uint8_t *)block + ctx->extra_size, data, + AES_BLOCK_SIZE - ctx->extra_size); + + write_aligned_block(ctx->base, block); + + res = wait_computation_completed(ctx->base); + if (res) + goto out; + + clear_computation_completed(ctx->base); + + i += AES_BLOCK_SIZE - ctx->extra_size; + ctx->extra_size = 0; + ctx->assoc_len += AES_BLOCK_SIZE_BIT; + } + + while (data_size - i >= AES_BLOCK_SIZE) { + write_block(ctx->base, data + i); + + res = wait_computation_completed(ctx->base); + if (res) + goto out; + + clear_computation_completed(ctx->base); + + /* Process next block */ + i += AES_BLOCK_SIZE; + ctx->assoc_len += AES_BLOCK_SIZE_BIT; + } + + /* + * Manage last block if not a block size multiple: + * Save remaining data to manage them later (potentially with new + * associated data). + */ + if (i < data_size) { + memcpy((uint8_t *)ctx->extra + ctx->extra_size, data + i, + data_size - i); + ctx->extra_size += data_size - i; + } + + res = save_context(ctx); +out: + if (res) + saes_end(ctx, res); + + mutex_unlock(ctx->lock); + + return res; +} + +/** + * @brief Update (or start) an AES authenticate and de/encrypt with + * payload data (CCM or GCM). + * @param ctx: SAES process context + * @param last_block: true if last payload data block + * @param data_in: pointer to payload + * @param data_out: pointer where to save de/encrypted payload + * @param data_size: payload size + * + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_saes_update_load(struct stm32_saes_context *ctx, + bool last_block, uint8_t *data_in, + uint8_t *data_out, size_t data_size) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = 0; + uint32_t previous_phase = 0; + + if (!ctx) + return TEE_ERROR_BAD_PARAMETERS; + + /* If there is no data, nothing to do */ + if (!data_in || !data_size) + return TEE_SUCCESS; + + mutex_lock(ctx->lock); + + previous_phase = ((ctx->cr & _SAES_CR_GCMPH_MASK) >> + _SAES_CR_GCMPH_SHIFT); + + switch (previous_phase) { + case _SAES_CR_GCMPH_INIT: + res = do_from_init_to_phase(ctx, _SAES_CR_GCMPH_PAYLOAD); + break; + case _SAES_CR_GCMPH_HEADER: + res = do_from_header_to_phase(ctx, _SAES_CR_GCMPH_PAYLOAD); + break; + case _SAES_CR_GCMPH_PAYLOAD: + /* new update_load call, we only need to restore context */ + if (can_suspend(ctx->cr)) + res = restore_context(ctx); + + break; + default: + DMSG("out of order call"); + res = TEE_ERROR_BAD_STATE; + } + + if (res) + goto out; + + while (i < ROUNDDOWN(data_size, AES_BLOCK_SIZE)) { + write_block(ctx->base, data_in + i); + + res = wait_computation_completed(ctx->base); + if (res) + goto out; + + read_block(ctx->base, data_out + i); + + clear_computation_completed(ctx->base); + + /* Process next block */ + i += AES_BLOCK_SIZE; + ctx->load_len += AES_BLOCK_SIZE_BIT; + } + + /* Manage last block if not a block size multiple */ + if (last_block && i < data_size) { + uint32_t block_in[AES_BLOCK_NB_U32] = { }; + uint32_t block_out[AES_BLOCK_NB_U32] = { }; + + memcpy(block_in, data_in + i, data_size - i); + + if (does_need_npblb(ctx->cr)) { + uint32_t npblb = AES_BLOCK_SIZE - (data_size - i); + + io_clrsetbits32(ctx->base + _SAES_CR, + _SAES_CR_NPBLB_MASK, + SHIFT_U32(npblb, _SAES_CR_NPBLB_SHIFT)); + } + + write_aligned_block(ctx->base, block_in); + + res = wait_computation_completed(ctx->base); + if (res) + goto out; + + read_aligned_block(ctx->base, block_out); + + clear_computation_completed(ctx->base); + + memcpy(data_out + i, block_out, data_size - i); + + ctx->load_len += (data_size - i) * INT8_BIT; + } + + res = save_context(ctx); +out: + if (res) + saes_end(ctx, res); + + mutex_unlock(ctx->lock); + + return res; +} + +/** + * @brief Get authentication tag for AES authenticated algorithms (CCM or GCM). + * @param ctx: SAES process context + * @param tag: pointer where to save the tag + * @param data_size: tag size + * + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_saes_final(struct stm32_saes_context *ctx, uint8_t *tag, + size_t tag_size) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t tag_u32[4] = { }; + uint32_t previous_phase = 0; + + if (!ctx) + return TEE_ERROR_BAD_PARAMETERS; + + mutex_lock(ctx->lock); + + previous_phase = (ctx->cr & _SAES_CR_GCMPH_MASK) >> + _SAES_CR_GCMPH_SHIFT; + + switch (previous_phase) { + case _SAES_CR_GCMPH_INIT: + res = do_from_init_to_phase(ctx, _SAES_CR_GCMPH_FINAL); + break; + case _SAES_CR_GCMPH_HEADER: + res = do_from_header_to_phase(ctx, _SAES_CR_GCMPH_FINAL); + break; + case _SAES_CR_GCMPH_PAYLOAD: + if (can_suspend(ctx->cr)) + res = restore_context(ctx); + + /* Move to final phase */ + io_clrsetbits32(ctx->base + _SAES_CR, _SAES_CR_GCMPH_MASK, + SHIFT_U32(_SAES_CR_GCMPH_FINAL, + _SAES_CR_GCMPH_SHIFT)); + break; + default: + DMSG("out of order call"); + res = TEE_ERROR_BAD_STATE; + } + if (res) + goto out; + + if (IS_CHAINING_MODE(GCM, ctx->cr)) { + /* SAES is configured to swap bytes as expected */ + io_write32(ctx->base + _SAES_DINR, 0); + io_write32(ctx->base + _SAES_DINR, ctx->assoc_len); + io_write32(ctx->base + _SAES_DINR, 0); + io_write32(ctx->base + _SAES_DINR, ctx->load_len); + } + + res = wait_computation_completed(ctx->base); + if (res) + goto out; + + read_aligned_block(ctx->base, tag_u32); + + clear_computation_completed(ctx->base); + + memcpy(tag, tag_u32, MIN(sizeof(tag_u32), tag_size)); + +out: + saes_end(ctx, res); + mutex_unlock(ctx->lock); + + return res; +} + +/** + * @brief Update (or start) an AES de/encrypt process (ECB, CBC or CTR). + * @param ctx: SAES process context + * @param last_block: true if last payload data block + * @param data_in: pointer to payload + * @param data_out: pointer where to save de/encrypted payload + * @param data_size: payload size + * + * @retval TEE_SUCCESS if OK. + */ +TEE_Result stm32_saes_update(struct stm32_saes_context *ctx, bool last_block, + uint8_t *data_in, uint8_t *data_out, + size_t data_size) +{ + TEE_Result res = TEE_SUCCESS; + unsigned int i = U(0); + + if (!ctx) + return TEE_ERROR_BAD_PARAMETERS; + + mutex_lock(ctx->lock); + + /* + * CBC encryption requires the 2 last blocks to be aligned with AES + * block size. + */ + if (last_block && IS_CHAINING_MODE(CBC, ctx->cr) && + is_encrypt(ctx->cr) && + (ROUNDDOWN(data_size, AES_BLOCK_SIZE) != data_size)) { + if (data_size < AES_BLOCK_SIZE * 2) { + /* + * If CBC, size of the last part should be at + * least 2*AES_BLOCK_SIZE + */ + EMSG("Unexpected last block size"); + res = TEE_ERROR_BAD_STATE; + goto out; + } + /* + * Do not support padding if the total size is not aligned with + * the size of a block. + */ + res = TEE_ERROR_NOT_IMPLEMENTED; + goto out; + } + + /* Manage remaining CTR mask from previous update call */ + if (IS_CHAINING_MODE(CTR, ctx->cr) && ctx->extra_size) { + unsigned int j = 0; + uint8_t *mask = (uint8_t *)ctx->extra; + + for (i = 0, j = 0; j < ctx->extra_size && i < data_size; + j++, i++) + data_out[i] = data_in[i] ^ mask[j]; + + if (j != ctx->extra_size) { + /* + * We didn't consume all saved mask, + * but no more data. + */ + + /* We save remaining mask and its new size */ + memmove(ctx->extra, ctx->extra + j, + ctx->extra_size - j); + ctx->extra_size -= j; + + /* + * We don't need to save HW context we didn't + * modify HW state. + */ + res = TEE_SUCCESS; + goto out; + } + /* All extra mask consumed */ + ctx->extra_size = 0; + } + + res = restore_context(ctx); + if (res) + goto out; + + while (data_size - i >= AES_BLOCK_SIZE) { + write_block(ctx->base, data_in + i); + + res = wait_computation_completed(ctx->base); + if (res) + goto out; + + read_block(ctx->base, data_out + i); + + clear_computation_completed(ctx->base); + + /* Process next block */ + i += AES_BLOCK_SIZE; + } + + /* Manage last block if not a block size multiple */ + if (i < data_size) { + if (IS_CHAINING_MODE(CTR, ctx->cr)) { + /* + * For CTR we save the generated mask to use it at next + * update call. + */ + uint32_t block_in[AES_BLOCK_NB_U32] = { }; + uint32_t block_out[AES_BLOCK_NB_U32] = { }; + + memcpy(block_in, data_in + i, data_size - i); + + write_aligned_block(ctx->base, block_in); + + res = wait_computation_completed(ctx->base); + if (res) + goto out; + + read_aligned_block(ctx->base, block_out); + + clear_computation_completed(ctx->base); + + memcpy(data_out + i, block_out, data_size - i); + + /* Save mask for possibly next call */ + ctx->extra_size = AES_BLOCK_SIZE - (data_size - i); + memcpy(ctx->extra, (uint8_t *)block_out + data_size - i, + ctx->extra_size); + } else { + /* CBC and ECB can manage only multiple of block_size */ + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + } + + if (!last_block) + res = save_context(ctx); + +out: + /* If last block or error, end of SAES process */ + if (last_block || res) + saes_end(ctx, res); + + mutex_unlock(ctx->lock); + + return res; +} + +static void xor_block(uint8_t *b1, uint8_t *b2, size_t size) +{ + size_t i = 0; + + for (i = 0; i < size; i++) + b1[i] ^= b2[i]; +} + +static TEE_Result stm32_saes_cmac_prf_128(struct stm32_saes_context *ctx, + enum stm32_saes_key_selection key_sel, + const void *key, size_t key_size, + uint8_t *data, size_t data_size, + uint8_t *out) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t block[AES_BLOCK_SIZE] = { }; + uint8_t k1[AES_BLOCK_SIZE] = { }; + uint8_t k2[AES_BLOCK_SIZE] = { }; + uint8_t l[AES_BLOCK_SIZE] = { }; + size_t processed = 0; + uint8_t bit = 0; + int i = 0; + + if (!ctx) + return TEE_ERROR_BAD_PARAMETERS; + + /* Get K1 and K2 */ + res = stm32_saes_init(ctx, false, STM32_SAES_MODE_ECB, key_sel, + key, key_size, NULL, 0); + if (res) + return res; + + res = stm32_saes_update(ctx, true, l, l, sizeof(l)); + if (res) + return res; + + /* MSB(L) == 0 => K1 = L << 1 */ + bit = 0; + for (i = sizeof(l) - 1; i >= 0; i--) { + k1[i] = (l[i] << 1) | bit; + bit = (l[i] & 0x80) >> 7; + } + /* MSB(L) == 1 => K1 = (L << 1) XOR const_Rb */ + if ((l[0] & 0x80)) + k1[sizeof(k1) - 1] = k1[sizeof(k1) - 1] ^ 0x87; + + /* MSB(K1) == 0 => K2 = K1 << 1 */ + bit = 0; + for (i = sizeof(k1) - 1; i >= 0; i--) { + k2[i] = (k1[i] << 1) | bit; + bit = (k1[i] & 0x80) >> 7; + } + + /* MSB(K1) == 1 => K2 = (K1 << 1) XOR const_Rb */ + if ((k1[0] & 0x80)) + k2[sizeof(k2) - 1] = k2[sizeof(k2) - 1] ^ 0x87; + + if (data_size > AES_BLOCK_SIZE) { + uint8_t *data_out = NULL; + + /* All block but last in CBC mode */ + res = stm32_saes_init(ctx, false, STM32_SAES_MODE_CBC, + key_sel, key, key_size, block, + sizeof(block)); + if (res) + return res; + + processed = ROUNDDOWN(data_size - 1, AES_BLOCK_SIZE); + data_out = malloc(processed); + if (!data_out) + return TEE_ERROR_OUT_OF_MEMORY; + + res = stm32_saes_update(ctx, true, data, data_out, processed); + if (!res) { + /* Copy last out block or keep block as { 0 } */ + memcpy(block, data_out + processed - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + } + + free(data_out); + + if (res) + return res; + } + + /* Manage last block */ + xor_block(block, data + processed, data_size - processed); + if (data_size - processed == AES_BLOCK_SIZE) { + xor_block(block, k1, AES_BLOCK_SIZE); + } else { + /* xor with padding = 0b100... */ + block[data_size - processed] ^= 0x80; + xor_block(block, k2, AES_BLOCK_SIZE); + } + + /* + * AES last block. + * We need to use same chaining mode to keep same key if DHUK is + * selected so we reuse l as a zero initialized IV. + */ + memset(l, 0, sizeof(l)); + res = stm32_saes_init(ctx, false, STM32_SAES_MODE_CBC, key_sel, key, + key_size, l, sizeof(l)); + if (res) + return res; + + return stm32_saes_update(ctx, true, block, out, AES_BLOCK_SIZE); +} + +TEE_Result stm32_saes_kdf(struct stm32_saes_context *ctx, + enum stm32_saes_key_selection key_sel, + const void *key, size_t key_size, + const void *input, size_t input_size, + uint8_t *subkey, size_t subkey_size) + +{ + TEE_Result res = TEE_SUCCESS; + uint32_t index = 0; + uint32_t index_be = 0; + uint8_t *data = NULL; + size_t data_index = 0; + size_t subkey_index = 0; + size_t data_size = input_size + sizeof(index_be); + uint8_t cmac[AES_BLOCK_SIZE] = { }; + + if (!ctx || !input || !input_size) + return TEE_ERROR_BAD_PARAMETERS; + + /* For each K(i) we will add an index */ + data = malloc(data_size); + if (!data) + return TEE_ERROR_OUT_OF_MEMORY; + + data_index = 0; + index_be = TEE_U32_TO_BIG_ENDIAN(index); + memcpy(data + data_index, &index_be, sizeof(index_be)); + data_index += sizeof(index_be); + memcpy(data + data_index, input, input_size); + data_index += input_size; + + /* K(i) computation. */ + index = 0; + while (subkey_index < subkey_size) { + index++; + index_be = TEE_U32_TO_BIG_ENDIAN(index); + memcpy(data, &index_be, sizeof(index_be)); + + res = stm32_saes_cmac_prf_128(ctx, key_sel, key, key_size, + data, data_size, cmac); + if (res) + goto out; + + memcpy(subkey + subkey_index, cmac, + MIN(subkey_size - subkey_index, sizeof(cmac))); + subkey_index += sizeof(cmac); + } + +out: + free(data); + if (res) + memzero_explicit(subkey, subkey_size); + + return res; +} + +/* Implement hardware HUK derivation using SAES resources */ +TEE_Result huk_subkey_derive(enum huk_subkey_usage usage, + const void *const_data, size_t const_data_len, + uint8_t *subkey, size_t subkey_len) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *input = NULL; + size_t input_index = 0; + size_t subkey_bitlen = 0; + struct stm32_saes_context ctx = { }; + uint8_t separator = 0; + + /* Check if driver is probed */ + if (!saes_pdata.base) { + return __huk_subkey_derive(usage, const_data, const_data_len, + subkey, subkey_len); + } + + input = malloc(const_data_len + sizeof(separator) + sizeof(usage) + + sizeof(subkey_bitlen) + AES_BLOCK_SIZE); + if (!input) + return TEE_ERROR_OUT_OF_MEMORY; + + input_index = 0; + if (const_data) { + memcpy(input + input_index, const_data, const_data_len); + input_index += const_data_len; + + memcpy(input + input_index, &separator, sizeof(separator)); + input_index += sizeof(separator); + } + + memcpy(input + input_index, &usage, sizeof(usage)); + input_index += sizeof(usage); + + /* + * We should add the subkey_len in bits at end of input. + * And we choose to put in a MSB first uint32_t. + */ + subkey_bitlen = TEE_U32_TO_BIG_ENDIAN(subkey_len * INT8_BIT); + memcpy(input + input_index, &subkey_bitlen, sizeof(subkey_bitlen)); + input_index += sizeof(subkey_bitlen); + + /* + * We get K(0) to avoid some key control attack + * and store it at end of input. + */ + res = stm32_saes_cmac_prf_128(&ctx, STM32_SAES_KEY_DHU, NULL, + AES_KEYSIZE_128, + input, input_index, + input + input_index); + if (res) + goto out; + + /* We just added K(0) to input */ + input_index += AES_BLOCK_SIZE; + + res = stm32_saes_kdf(&ctx, STM32_SAES_KEY_DHU, NULL, AES_KEYSIZE_128, + input, input_index, subkey, subkey_len); + +out: + free(input); + return res; +} + +static TEE_Result stm32_saes_parse_fdt(struct stm32_saes_platdata *pdata, + const void *fdt, int node) +{ + struct dt_node_info dt_saes = { }; + TEE_Result res = TEE_ERROR_GENERIC; + + dt_saes.reg = fdt_reg_base_address(fdt, node); + dt_saes.reg_size = fdt_reg_size(fdt, node); + + if (dt_saes.reg == DT_INFO_INVALID_REG || + dt_saes.reg_size == DT_INFO_INVALID_REG_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + res = clk_dt_get_by_index(fdt, node, 0, &pdata->clk); + if (res != TEE_SUCCESS) + return res; + + res = rstctrl_dt_get_by_index(fdt, node, 0, &pdata->reset); + if (res != TEE_SUCCESS && res != TEE_ERROR_ITEM_NOT_FOUND) + return res; + + pdata->base = (vaddr_t)phys_to_virt(dt_saes.reg, MEM_AREA_IO_SEC, + dt_saes.reg_size); + if (!pdata->base) + panic(); + + return TEE_SUCCESS; +} + +static TEE_Result stm32_saes_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_SUCCESS; + + assert(!saes_pdata.base); + + res = stm32_saes_parse_fdt(&saes_pdata, fdt, node); + if (res) + return res; + + if (clk_enable(saes_pdata.clk)) + panic(); + + if (saes_pdata.reset) { + /* External reset of SAES */ + if (rstctrl_assert_to(saes_pdata.reset, TIMEOUT_US_1MS)) + panic(); + + udelay(SAES_RESET_DELAY); + + if (rstctrl_deassert_to(saes_pdata.reset, TIMEOUT_US_1MS)) + panic(); + } else { + /* Internal reset of SAES */ + io_setbits32(saes_pdata.base + _SAES_CR, _SAES_CR_IPRST); + udelay(SAES_RESET_DELAY); + io_clrbits32(saes_pdata.base + _SAES_CR, _SAES_CR_IPRST); + } + + if (IS_ENABLED(CFG_CRYPTO_DRV_CIPHER)) { + res = stm32_register_cipher(SAES_IP); + if (res) { + EMSG("Failed to register to cipher: %#"PRIx32, res); + panic(); + } + } + + return TEE_SUCCESS; +} + +static const struct dt_device_match saes_match_table[] = { + { .compatible = "st,stm32mp13-saes" }, + { } +}; + +DEFINE_DT_DRIVER(stm32_saes_dt_driver) = { + .name = "stm32-saes", + .match_table = saes_match_table, + .probe = &stm32_saes_probe, +}; diff --git a/optee_os/core/drivers/crypto/stm32/stm32_saes.h b/optee_os/core/drivers/crypto/stm32/stm32_saes.h new file mode 100644 index 0000000..8a35706 --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/stm32_saes.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021-2023, STMicroelectronics - All Rights Reserved + */ + +#ifndef STM32_SAES_H +#define STM32_SAES_H + +#include +#include +#include +#include +#include +#include +#include + +enum stm32_saes_chaining_mode { + STM32_SAES_MODE_ECB, + STM32_SAES_MODE_CBC, + STM32_SAES_MODE_CTR, + STM32_SAES_MODE_GCM, + STM32_SAES_MODE_CCM, +}; + +enum stm32_saes_key_selection { + STM32_SAES_KEY_SOFT, + STM32_SAES_KEY_DHU, /* Derived HW unique key */ + STM32_SAES_KEY_BH, /* Boot HW key */ + STM32_SAES_KEY_BHU_XOR_BH, /* XOR of DHUK and BHK */ + STM32_SAES_KEY_WRAPPED +}; + +struct stm32_saes_context { + vaddr_t base; + uint32_t cr; + struct mutex *lock; /* Save the HW instance mutex */ + uint32_t assoc_len; + uint32_t load_len; + uint32_t key[8]; /* In HW byte order */ + uint32_t iv[4]; /* In HW byte order */ + uint32_t susp[8]; + uint32_t extra[4]; + size_t extra_size; +}; + +TEE_Result stm32_saes_init(struct stm32_saes_context *ctx, bool is_decrypt, + enum stm32_saes_chaining_mode ch_mode, + enum stm32_saes_key_selection key_select, + const void *key, size_t key_len, const void *iv, + size_t iv_len); +TEE_Result stm32_saes_update(struct stm32_saes_context *ctx, bool last_block, + uint8_t *data_in, uint8_t *data_out, + size_t data_len); +TEE_Result stm32_saes_update_assodata(struct stm32_saes_context *ctx, + uint8_t *data, size_t data_len); +TEE_Result stm32_saes_update_load(struct stm32_saes_context *ctx, + bool last_block, uint8_t *data_in, + uint8_t *data_out, size_t data_len); +TEE_Result stm32_saes_final(struct stm32_saes_context *ctx, uint8_t *tag, + size_t tag_len); + +TEE_Result stm32_saes_kdf(struct stm32_saes_context *ctx, + enum stm32_saes_key_selection key_sel, + const void *key, size_t key_size, + const void *input, size_t input_size, + uint8_t *subkey, size_t subkey_size); + +#endif diff --git a/optee_os/core/drivers/crypto/stm32/sub.mk b/optee_os/core/drivers/crypto/stm32/sub.mk new file mode 100644 index 0000000..153e52c --- /dev/null +++ b/optee_os/core/drivers/crypto/stm32/sub.mk @@ -0,0 +1,4 @@ +srcs-$(CFG_STM32_CRYP) += stm32_cryp.c +srcs-$(CFG_STM32_SAES) += stm32_saes.c +srcs-$(CFG_CRYPTO_DRV_CIPHER) += cipher.c +srcs-$(CFG_CRYPTO_DRV_AUTHENC) += authenc.c diff --git a/optee_os/core/drivers/crypto/sub.mk b/optee_os/core/drivers/crypto/sub.mk new file mode 100644 index 0000000..3c26eda --- /dev/null +++ b/optee_os/core/drivers/crypto/sub.mk @@ -0,0 +1,13 @@ +global-incdirs-$(CFG_CRYPTO_DRIVER) += crypto_api/include + +subdirs-$(CFG_CRYPTO_DRIVER) += crypto_api + +subdirs-$(CFG_NXP_CAAM) += caam + +subdirs-$(CFG_NXP_SE05X) += se050 + +subdirs-$(CFG_STM32_CRYPTO_DRIVER) += stm32 + +subdirs-$(CFG_ASPEED_CRYPTO_DRIVER) += aspeed + +subdirs-$(CFG_VERSAL_CRYPTO_DRIVER) += versal diff --git a/optee_os/core/drivers/crypto/versal/authenc.c b/optee_os/core/drivers/crypto/versal/authenc.c new file mode 100644 index 0000000..8776dac --- /dev/null +++ b/optee_os/core/drivers/crypto/versal/authenc.c @@ -0,0 +1,870 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022. + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * This driver does not queue/pad non-aligned data. + * + * Allow debug information for future PLM work: if the PLM can not implement + * the required changes, we might be able to do it in OP-TEE. + */ +#define DEBUG_VERSAL_AES 0 + +#define GCM_TAG_LEN 16 + +#define XSECURE_AES_KEY_SIZE_128 0 /* Key Length = 32 bytes = 256 bits */ +#define XSECURE_AES_KEY_SIZE_256 2 /* Key Length = 16 bytes = 128 bits */ + +#define XSECURE_ENCRYPT 0 +#define XSECURE_DECRYPT 1 + +enum versal_aes_err { + AES_GCM_TAG_MISMATCH = 0x40, + AES_KEY_CLEAR_ERROR, + AES_DPA_CM_NOT_SUPPORTED, + AES_KAT_WRITE_KEY_FAILED_ERROR, + AES_KAT_DECRYPT_INIT_FAILED_ERROR, + AES_KAT_GCM_TAG_MISMATCH_ERROR, + AES_KAT_DATA_MISMATCH_ERROR, + AES_KAT_FAILED_ERROR, + AESDPACM_KAT_WRITE_KEY_FAILED_ERROR, + AESDPACM_KAT_KEYLOAD_FAILED_ERROR, + AESDPACM_SSS_CFG_FAILED_ERROR, + AESDPACM_KAT_FAILED_ERROR, + AESDPACM_KAT_CHECK1_FAILED_ERROR, + AESDPACM_KAT_CHECK2_FAILED_ERROR, + AESDPACM_KAT_CHECK3_FAILED_ERROR, + AESDPACM_KAT_CHECK4_FAILED_ERROR, + AESDPACM_KAT_CHECK5_FAILED_ERROR, + AES_INVALID_PARAM, + AESKAT_INVALID_PARAM, + AES_STATE_MISMATCH_ERROR, + AES_DEVICE_KEY_NOT_ALLOWED, +}; + +#define VERSAL_AES_ERROR(m) { . error = (m), .name = TO_STR(m) } + +static const char *versal_aes_error(uint8_t err) +{ + const struct { + enum versal_aes_err error; + const char *name; + } elist[] = { + VERSAL_AES_ERROR(AES_GCM_TAG_MISMATCH), + VERSAL_AES_ERROR(AES_KEY_CLEAR_ERROR), + VERSAL_AES_ERROR(AES_DPA_CM_NOT_SUPPORTED), + VERSAL_AES_ERROR(AES_KAT_WRITE_KEY_FAILED_ERROR), + VERSAL_AES_ERROR(AES_KAT_DECRYPT_INIT_FAILED_ERROR), + VERSAL_AES_ERROR(AES_KAT_GCM_TAG_MISMATCH_ERROR), + VERSAL_AES_ERROR(AES_KAT_DATA_MISMATCH_ERROR), + VERSAL_AES_ERROR(AES_KAT_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_WRITE_KEY_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_KEYLOAD_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_SSS_CFG_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_CHECK1_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_CHECK2_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_CHECK3_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_CHECK4_FAILED_ERROR), + VERSAL_AES_ERROR(AESDPACM_KAT_CHECK5_FAILED_ERROR), + VERSAL_AES_ERROR(AES_INVALID_PARAM), + VERSAL_AES_ERROR(AESKAT_INVALID_PARAM), + VERSAL_AES_ERROR(AES_STATE_MISMATCH_ERROR), + VERSAL_AES_ERROR(AES_DEVICE_KEY_NOT_ALLOWED), + }; + + if (err >= AES_GCM_TAG_MISMATCH && err <= AES_DEVICE_KEY_NOT_ALLOWED) { + if (elist[err - AES_GCM_TAG_MISMATCH].name) + return elist[err - AES_GCM_TAG_MISMATCH].name; + + return "Invalid"; + } + + return "Unknown"; +} + +enum aes_key_src { + XSECURE_AES_BBRAM_KEY = 0, /* BBRAM Key */ + XSECURE_AES_BBRAM_RED_KEY, /* BBRAM Red Key */ + XSECURE_AES_BH_KEY, /* BH Key */ + XSECURE_AES_BH_RED_KEY, /* BH Red Key */ + XSECURE_AES_EFUSE_KEY, /* eFUSE Key */ + XSECURE_AES_EFUSE_RED_KEY, /* eFUSE Red Key */ + XSECURE_AES_EFUSE_USER_KEY_0, /* eFUSE User Key 0 */ + XSECURE_AES_EFUSE_USER_KEY_1, /* eFUSE User Key 1 */ + XSECURE_AES_EFUSE_USER_RED_KEY_0, /* eFUSE User Red Key 0 */ + XSECURE_AES_EFUSE_USER_RED_KEY_1, /* eFUSE User Red Key 1 */ + XSECURE_AES_KUP_KEY, /* KUP key */ + XSECURE_AES_PUF_KEY, /* PUF key */ + XSECURE_AES_USER_KEY_0, /* User Key 0 */ + XSECURE_AES_USER_KEY_1, /* User Key 1 */ + XSECURE_AES_USER_KEY_2, /* User Key 2 */ + XSECURE_AES_USER_KEY_3, /* User Key 3 */ + XSECURE_AES_USER_KEY_4, /* User Key 4 */ + XSECURE_AES_USER_KEY_5, /* User Key 5 */ + XSECURE_AES_USER_KEY_6, /* User Key 6 */ + XSECURE_AES_USER_KEY_7, /* User Key 7 */ + XSECURE_AES_EXPANDED_KEYS, /* Expanded keys */ + XSECURE_AES_ALL_KEYS, /* AES All keys */ +}; + +struct versal_payload { + struct versal_mbox_mem input_cmd; + struct versal_mbox_mem src; + struct versal_mbox_mem dst; + bool encrypt; +}; + +struct versal_aad { + struct versal_mbox_mem mem; +}; + +struct versal_node { + struct versal_payload payload; + struct versal_aad aad; + bool is_aad; + STAILQ_ENTRY(versal_node) link; +}; + +struct versal_init { + uint32_t key_len; + uint32_t operation; + struct versal_mbox_mem key; + struct versal_mbox_mem nonce; + struct versal_mbox_mem init_buf; +}; + +struct versal_ae_ctx { + struct crypto_authenc_ctx a_ctx; +}; + +struct versal_context_node { + struct versal_ae_ctx *ctx; + STAILQ_ENTRY(versal_context_node) link; +}; + +enum engine_state { + READY = 1, INIT = 2, FINALIZED = 3, +}; + +static struct mutex engine_lock = MUTEX_INITIALIZER; + +static struct versal_engine { + enum aes_key_src key_src; + enum engine_state state; + struct versal_init init; + struct refcount refc; + struct mutex *lock; /* protect the HW instance */ + STAILQ_HEAD(authenc_context_list, versal_context_node) context_list; + STAILQ_HEAD(authenc_replay_list, versal_node) replay_list; +} engine = { + .key_src = XSECURE_AES_USER_KEY_0, + .lock = &engine_lock, +}; + +static struct versal_ae_ctx *to_versal_ctx(struct crypto_authenc_ctx *ctx) +{ + assert(ctx); + + return container_of(ctx, struct versal_ae_ctx, a_ctx); +} + +static TEE_Result replay_init(void) +{ + struct versal_cmd_args arg = { }; + uint32_t err = 0; + + if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) { + EMSG("AES_INIT error"); + return TEE_ERROR_GENERIC; + } + + arg.data[arg.dlen++] = engine.init.key_len; + arg.data[arg.dlen++] = engine.key_src; + arg.ibuf[0].mem = engine.init.key; + + if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) { + EMSG("AES_WRITE_KEY error"); + return TEE_ERROR_GENERIC; + } + + memset(&arg, 0, sizeof(arg)); + + arg.ibuf[0].mem = engine.init.init_buf; + arg.ibuf[1].mem = engine.init.nonce; + arg.ibuf[1].only_cache = true; + + if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) { + EMSG("AES_OP_INIT error"); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +static TEE_Result replay_aad(struct versal_aad *p) +{ + struct versal_cmd_args arg = { }; + uint32_t err = 0; + + arg.data[arg.dlen++] = p->mem.len % 16 ? p->mem.alloc_len : p->mem.len; + arg.ibuf[0].mem = p->mem; + + if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) { + EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err)); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +static TEE_Result replay_payload(struct versal_payload *p) +{ + enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE; + struct versal_cmd_args arg = { }; + uint32_t err = 0; + + arg.ibuf[0].mem = p->input_cmd; + arg.ibuf[1].mem = p->dst; + arg.ibuf[2].mem = p->src; + + if (p->encrypt) + id = VERSAL_AES_ENCRYPT_UPDATE; + + if (versal_crypto_request(id, &arg, &err)) { + EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err)); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +static TEE_Result do_replay(void) +{ + struct versal_node *node = NULL; + TEE_Result ret = TEE_SUCCESS; + + ret = replay_init(); + if (ret) + return ret; + + STAILQ_FOREACH(node, &engine.replay_list, link) { + if (node->is_aad) { + ret = replay_aad(&node->aad); + if (ret) + return ret; + } else { + ret = replay_payload(&node->payload); + if (ret) + return ret; + } + } + + /* Engine has been init */ + engine.state = INIT; + + return TEE_SUCCESS; +} + +static bool engine_in_use(void) +{ + if (STAILQ_EMPTY(&engine.context_list)) + return false; + + return true; +} + +static bool context_allowed(struct crypto_authenc_ctx *ctx) +{ + struct versal_context_node *node = NULL; + + STAILQ_FOREACH(node, &engine.context_list, link) { + if (node->ctx == to_versal_ctx(ctx)) + return true; + } + + return false; +} + +static TEE_Result do_init(struct drvcrypt_authenc_init *dinit) +{ + uint32_t key_len = XSECURE_AES_KEY_SIZE_128; + struct versal_context_node *node = NULL; + struct versal_aes_init *init = NULL; + struct versal_mbox_mem init_buf = { }; + struct versal_mbox_mem key = { }; + struct versal_mbox_mem nonce = { }; + struct versal_cmd_args arg = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + + if (engine_in_use()) { + EMSG("Versal: AES-GCM Engine busy"); + return TEE_ERROR_BUSY; + } + + if (dinit->key.length != 32 && dinit->key.length != 16) + return TEE_ERROR_BAD_PARAMETERS; + + if (dinit->key.length == 32) + key_len = XSECURE_AES_KEY_SIZE_256; + + if (engine.state != READY) + return TEE_ERROR_BAD_STATE; + + /* Initialize the AES engine */ + if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) { + EMSG("AES_INIT error: %s", versal_aes_error(err)); + return TEE_ERROR_GENERIC; + } + + /* Write the key */ + versal_mbox_alloc(dinit->key.length, dinit->key.data, &key); + + arg.data[arg.dlen++] = key_len; + arg.data[arg.dlen++] = engine.key_src; + arg.ibuf[0].mem = key; + + if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) { + EMSG("AES_WRITE_KEY error: %s", versal_aes_error(err)); + ret = TEE_ERROR_GENERIC; + goto error; + } + + memset(&arg, 0, sizeof(arg)); + + /* Send the initialization structure */ + versal_mbox_alloc(sizeof(*init), NULL, &init_buf); + versal_mbox_alloc(dinit->nonce.length, dinit->nonce.data, &nonce); + + init = init_buf.buf; + init->iv_addr = virt_to_phys(nonce.buf); + init->operation = dinit->encrypt ? XSECURE_ENCRYPT : XSECURE_DECRYPT; + init->key_src = engine.key_src; + init->key_len = key_len; + + arg.ibuf[0].mem = init_buf; + arg.ibuf[1].mem = nonce; + arg.ibuf[1].only_cache = true; + + if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) { + EMSG("AES_OP_INIT error: %s", versal_aes_error(err)); + ret = TEE_ERROR_GENERIC; + goto error; + } + + node = calloc(1, sizeof(*node)); + if (!node) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto error; + } + + /* Save key context */ + engine.init.operation = dinit->encrypt ? + XSECURE_ENCRYPT : XSECURE_DECRYPT; + engine.init.key_len = key_len; + engine.init.init_buf = init_buf; + engine.init.nonce = nonce; + engine.init.key = key; + + /* Active context */ + node->ctx = to_versal_ctx(dinit->ctx); + STAILQ_INSERT_TAIL(&engine.context_list, node, link); + + /* Engine has been init*/ + engine.state = INIT; + + return TEE_SUCCESS; +error: + free(key.buf); + free(init_buf.buf); + free(nonce.buf); + + return ret; +} + +static TEE_Result do_update_aad(struct drvcrypt_authenc_update_aad *dupdate) +{ + struct versal_cmd_args arg = { }; + struct versal_mbox_mem p = { }; + TEE_Result ret = TEE_SUCCESS; + struct versal_node *node = NULL; + uint32_t err = 0; + + /* This context has not been inited */ + if (!context_allowed(dupdate->ctx)) + return TEE_ERROR_BUSY; + + /* There is a copy of the context: don't allow updates, only finalize */ + if (refcount_val(&engine.refc) > 1) + return TEE_ERROR_BUSY; + + /* There was a copy of the context and it was finalized, then replay */ + if (engine.state == FINALIZED) + do_replay(); + + versal_mbox_alloc(dupdate->aad.length, dupdate->aad.data, &p); + + arg.data[arg.dlen++] = p.len % 16 ? p.alloc_len : p.len; + arg.ibuf[0].mem = p; + +#if DEBUG_VERSAL_AES + IMSG("versal: aad length - requested: %zu, sent to plm: %"PRIu32, + dupdate->aad.length, arg.data[0]); +#endif + if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) { + EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err)); + ret = TEE_ERROR_GENERIC; + goto error; + } + + node = calloc(1, sizeof(*node)); + if (!node) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto error; + } + + /* Save the context */ + node->aad.mem = p; + node->is_aad = true; + STAILQ_INSERT_TAIL(&engine.replay_list, node, link); + + return TEE_SUCCESS; +error: + free(p.buf); + return ret; +} + +static TEE_Result +update_payload(struct drvcrypt_authenc_update_payload *dupdate, bool is_last) +{ + enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE; + struct versal_aes_input_param *input = NULL; + struct versal_mbox_mem input_cmd = { }; + struct versal_mbox_mem p = { }; + struct versal_mbox_mem q = { }; + TEE_Result ret = TEE_SUCCESS; + struct versal_cmd_args arg = { }; + struct versal_node *node = NULL; + uint32_t err = 0; + + if (!context_allowed(dupdate->ctx)) + return TEE_ERROR_BUSY; + + if (!dupdate->src.length || dupdate->src.length % 4) { + EMSG("Versal AES payload length not word aligned (len = %zu)", + dupdate->src.length); + return TEE_ERROR_BAD_PARAMETERS; + } + + versal_mbox_alloc(dupdate->src.length, dupdate->src.data, &p); + versal_mbox_alloc(dupdate->dst.length, NULL, &q); + versal_mbox_alloc(sizeof(*input), NULL, &input_cmd); + + input = input_cmd.buf; + input->input_addr = virt_to_phys(p.buf); + input->input_len = p.len % 16 ? p.alloc_len : p.len; + input->is_last = is_last; + + arg.ibuf[0].mem = input_cmd; + arg.ibuf[1].mem = q; + arg.ibuf[2].mem = p; + + if (dupdate->encrypt) + id = VERSAL_AES_ENCRYPT_UPDATE; + +#if DEBUG_VERSAL_AES + IMSG("versal: payload length - requested %zu, sent to plm: %"PRIu32, + dupdate->src.length, input->input_len); + IMSG("versal: destination length - %zu ", dupdate->dst.length); +#endif + if (versal_crypto_request(id, &arg, &err)) { + EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err)); + ret = TEE_ERROR_GENERIC; + goto out; + } + + if (dupdate->dst.data) + memcpy(dupdate->dst.data, q.buf, dupdate->dst.length); + + if (!is_last) { + node = calloc(1, sizeof(*node)); + if (!node) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + node->is_aad = false; + node->payload.dst = q; + node->payload.src = p; + node->payload.input_cmd = input_cmd; + node->payload.encrypt = dupdate->encrypt; + STAILQ_INSERT_TAIL(&engine.replay_list, node, link); + + return TEE_SUCCESS; + } +out: + free(p.buf); + free(q.buf); + free(input_cmd.buf); + + return ret; +} + +static TEE_Result do_update_payload(struct drvcrypt_authenc_update_payload *p) +{ + TEE_Result ret = TEE_SUCCESS; + + if (!context_allowed(p->ctx)) + return TEE_ERROR_BUSY; + + /* + * If there is a copy, we don't allow updates until one of the copies + * has been deleted + */ + if (refcount_val(&engine.refc) > 1) + return TEE_ERROR_BUSY; + + /* + * If there was a copy and it was finalized, we need to replay before + * we can update; do not clear the list so the state can be copied + */ + if (engine.state == FINALIZED) { + ret = do_replay(); + if (ret) + return ret; + } + + return update_payload(p, false); +} + +static TEE_Result do_enc_final(struct drvcrypt_authenc_final *dfinal) +{ + struct drvcrypt_authenc_update_payload last = { }; + struct versal_cmd_args arg = { }; + struct versal_mbox_mem p = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + + if (!context_allowed(dfinal->ctx)) + return TEE_ERROR_BUSY; + + if (engine.state == FINALIZED) { + DMSG("Operation was already finalized"); + ret = do_replay(); + if (ret) + return ret; + } + + if (engine.state != INIT) + panic(); + + last.ctx = dfinal->ctx; + last.dst = dfinal->dst; + last.encrypt = true; + last.src = dfinal->src; + + ret = update_payload(&last, true); + if (ret) + return ret; + + memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length); + + versal_mbox_alloc(GCM_TAG_LEN, NULL, &p); + + arg.ibuf[0].mem = p; + if (versal_crypto_request(VERSAL_AES_ENCRYPT_FINAL, &arg, &err)) { + EMSG("AES_ENCRYPT_FINAL error: %s", versal_aes_error(err)); + ret = TEE_ERROR_GENERIC; + goto out; + } + + memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN); + dfinal->tag.length = GCM_TAG_LEN; +out: + free(p.buf); + + if (refcount_val(&engine.refc) > 1) + engine.state = FINALIZED; + else + engine.state = READY; + + return ret; +} + +static TEE_Result do_dec_final(struct drvcrypt_authenc_final *dfinal) +{ + struct drvcrypt_authenc_update_payload last = { }; + struct versal_cmd_args arg = { }; + struct versal_mbox_mem p = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + + if (!context_allowed(dfinal->ctx)) + return TEE_ERROR_BUSY; + + if (engine.state == FINALIZED) { + DMSG("Operation was already finalized"); + ret = do_replay(); + if (ret) + return ret; + } + + if (engine.state != INIT) + panic(); + + last.encrypt = false; + last.ctx = dfinal->ctx; + last.dst = dfinal->dst; + last.src = dfinal->src; + + ret = update_payload(&last, true); + if (ret) + return ret; + + versal_mbox_alloc(dfinal->tag.length, dfinal->tag.data, &p); + arg.ibuf[0].mem = p; + + if (versal_crypto_request(VERSAL_AES_DECRYPT_FINAL, &arg, &err)) { + EMSG("AES_DECRYPT_FINAL error: %s", versal_aes_error(err)); + ret = TEE_ERROR_GENERIC; + goto out; + } + + memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length); + memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN); + dfinal->tag.length = GCM_TAG_LEN; +out: + free(p.buf); + + if (refcount_val(&engine.refc) > 1) + engine.state = FINALIZED; + else + engine.state = READY; + + return ret; +} + +static void do_final(void *ctx __unused) +{ +} + +static void do_free(void *ctx) +{ + struct versal_ae_ctx *c = to_versal_ctx(ctx); + struct versal_node *next = NULL; + struct versal_node *node = NULL; + struct versal_context_node *ctx_next = NULL; + struct versal_context_node *ctx_node = NULL; + bool release = false; + + if (refcount_dec(&engine.refc)) { + /* this is a final release */ + release = true; + refcount_set(&engine.refc, 1); + engine.state = READY; + free(engine.init.init_buf.buf); + free(engine.init.nonce.buf); + free(engine.init.key.buf); + memset(&engine.init, 0, sizeof(engine.init)); + STAILQ_FOREACH_SAFE(node, &engine.replay_list, link, next) { + STAILQ_REMOVE(&engine.replay_list, node, + versal_node, link); + if (node->is_aad) { + free(node->aad.mem.buf); + } else { + free(node->payload.dst.buf); + free(node->payload.src.buf); + free(node->payload.input_cmd.buf); + } + free(node); + } + } + + STAILQ_FOREACH_SAFE(ctx_node, &engine.context_list, link, ctx_next) { + if (c == ctx_node->ctx) { + STAILQ_REMOVE(&engine.context_list, ctx_node, + versal_context_node, link); + free(ctx_node); + } + } + + if (release && engine_in_use()) + panic(); + + free(c); +} + +static void do_copy_state(void *dst_ctx, void *src_ctx) +{ + struct versal_context_node *node = NULL; + + STAILQ_FOREACH(node, &engine.context_list, link) { + if (node->ctx != to_versal_ctx(src_ctx)) + continue; + /* + * The running context has been copied: from now on we can only + * finalize any of the contexts until there is only one active + * again. + */ + node = calloc(1, sizeof(*node)); + if (!node) + panic(); + + node->ctx = to_versal_ctx(dst_ctx); + STAILQ_INSERT_TAIL(&engine.context_list, node, link); + + /* number of active contexts */ + refcount_inc(&engine.refc); + + return; + } + + panic(); +} + +static TEE_Result do_allocate(void **ctx, uint32_t algo) +{ + struct versal_ae_ctx *c = NULL; + + if (algo != TEE_ALG_AES_GCM) + return TEE_ERROR_NOT_IMPLEMENTED; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + *ctx = &c->a_ctx; + + return TEE_SUCCESS; +} + +static TEE_Result +do_update_payload_locked(struct drvcrypt_authenc_update_payload *p) +{ + TEE_Result ret = TEE_SUCCESS; + + mutex_lock(engine.lock); + ret = do_update_payload(p); + mutex_unlock(engine.lock); + return ret; +} + +static TEE_Result +do_update_aad_locked(struct drvcrypt_authenc_update_aad *p) +{ + TEE_Result ret = TEE_SUCCESS; + + mutex_lock(engine.lock); + ret = do_update_aad(p); + mutex_unlock(engine.lock); + return ret; +} + +static void do_copy_state_locked(void *dst, void *src) +{ + mutex_lock(engine.lock); + do_copy_state(dst, src); + mutex_unlock(engine.lock); +} + +static TEE_Result do_enc_final_locked(struct drvcrypt_authenc_final *p) +{ + TEE_Result ret = TEE_SUCCESS; + + mutex_lock(engine.lock); + ret = do_enc_final(p); + mutex_unlock(engine.lock); + return ret; +} + +static TEE_Result do_dec_final_locked(struct drvcrypt_authenc_final *p) +{ + TEE_Result ret = TEE_SUCCESS; + + mutex_lock(engine.lock); + ret = do_dec_final(p); + mutex_unlock(engine.lock); + return ret; +} + +static void do_free_locked(void *p) +{ + mutex_lock(engine.lock); + do_free(p); + mutex_unlock(engine.lock); +} + +static TEE_Result do_init_locked(struct drvcrypt_authenc_init *p) +{ + TEE_Result ret = TEE_SUCCESS; + + mutex_lock(engine.lock); + ret = do_init(p); + mutex_unlock(engine.lock); + return ret; +} + +static struct drvcrypt_authenc versal_authenc = { + .update_payload = do_update_payload_locked, + .update_aad = do_update_aad_locked, + .copy_state = do_copy_state_locked, + .enc_final = do_enc_final_locked, + .dec_final = do_dec_final_locked, + .free_ctx = do_free_locked, + .alloc_ctx = do_allocate, + .init = do_init_locked, + .final = do_final, +}; + +static TEE_Result enable_secure_status(void) +{ + /* Once Linux has support, we need to reserve the device */ + return TEE_SUCCESS; +} + +static TEE_Result versal_register_authenc(void) +{ + TEE_Result ret = TEE_SUCCESS; + + ret = drvcrypt_register_authenc(&versal_authenc); + if (ret) + return ret; + + if (engine.key_src < XSECURE_AES_USER_KEY_0 || + engine.key_src > XSECURE_AES_USER_KEY_7) + return TEE_ERROR_GENERIC; + + engine.state = READY; + STAILQ_INIT(&engine.replay_list); + STAILQ_INIT(&engine.context_list); + refcount_set(&engine.refc, 1); + + return enable_secure_status(); +} + +driver_init_late(versal_register_authenc); diff --git a/optee_os/core/drivers/crypto/versal/crypto.mk b/optee_os/core/drivers/crypto/versal/crypto.mk new file mode 100644 index 0000000..2dc92c2 --- /dev/null +++ b/optee_os/core/drivers/crypto/versal/crypto.mk @@ -0,0 +1,11 @@ +ifeq ($(CFG_VERSAL_CRYPTO_DRIVER),y) +# Enable the crypto driver +$(call force,CFG_CRYPTO_DRIVER,y) + +CFG_CRYPTO_DRIVER_DEBUG ?= 0 +$(call force,CFG_CRYPTO_DRV_ACIPHER,y) +$(call force,CFG_CRYPTO_DRV_ECC,y) +$(call force,CFG_CRYPTO_DRV_RSA,y) +$(call force,CFG_CRYPTO_DRV_AUTHENC,y) + +endif diff --git a/optee_os/core/drivers/crypto/versal/ecc.c b/optee_os/core/drivers/crypto/versal/ecc.c new file mode 100644 index 0000000..64a919e --- /dev/null +++ b/optee_os/core/drivers/crypto/versal/ecc.c @@ -0,0 +1,469 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022. + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* AMD/Xilinx Versal's Known Answer Tests */ +#define XSECURE_ECDSA_KAT_NIST_P384 0 +#define XSECURE_ECDSA_KAT_NIST_P521 2 + +/* Software based ECDSA operations */ +static const struct crypto_ecc_keypair_ops *pair_ops; +static const struct crypto_ecc_public_ops *pub_ops; + +enum versal_ecc_err { + KAT_KEY_NOTVALID_ERROR = 0xC0, + KAT_FAILED_ERROR, + NON_SUPPORTED_CURVE, + KEY_ZERO, + KEY_WRONG_ORDER, + KEY_NOT_ON_CURVE, + BAD_SIGN, + GEN_SIGN_INCORRECT_HASH_LEN, + VER_SIGN_INCORRECT_HASH_LEN, + GEN_SIGN_BAD_RAND_NUM, + GEN_KEY_ERR, + INVALID_PARAM, + VER_SIGN_R_ZERO, + VER_SIGN_S_ZERO, + VER_SIGN_R_ORDER_ERROR, + VER_SIGN_S_ORDER_ERROR, + KAT_INVLD_CRV_ERROR, +}; + +#define VERSAL_ECC_ERROR(m) { .error = (m), .name = TO_STR(m) } + +static const char *versal_ecc_error(uint8_t err) +{ + struct { + enum versal_ecc_err error; + const char *name; + } elist[] = { + VERSAL_ECC_ERROR(KAT_KEY_NOTVALID_ERROR), + VERSAL_ECC_ERROR(KAT_FAILED_ERROR), + VERSAL_ECC_ERROR(NON_SUPPORTED_CURVE), + VERSAL_ECC_ERROR(KEY_ZERO), + VERSAL_ECC_ERROR(KEY_WRONG_ORDER), + VERSAL_ECC_ERROR(KEY_NOT_ON_CURVE), + VERSAL_ECC_ERROR(BAD_SIGN), + VERSAL_ECC_ERROR(GEN_SIGN_INCORRECT_HASH_LEN), + VERSAL_ECC_ERROR(VER_SIGN_INCORRECT_HASH_LEN), + VERSAL_ECC_ERROR(GEN_SIGN_BAD_RAND_NUM), + VERSAL_ECC_ERROR(GEN_KEY_ERR), + VERSAL_ECC_ERROR(INVALID_PARAM), + VERSAL_ECC_ERROR(VER_SIGN_R_ZERO), + VERSAL_ECC_ERROR(VER_SIGN_S_ZERO), + VERSAL_ECC_ERROR(VER_SIGN_R_ORDER_ERROR), + VERSAL_ECC_ERROR(VER_SIGN_S_ORDER_ERROR), + VERSAL_ECC_ERROR(KAT_INVLD_CRV_ERROR), + }; + + if (err <= KAT_INVLD_CRV_ERROR && err >= KAT_KEY_NOTVALID_ERROR) { + if (elist[err - KAT_KEY_NOTVALID_ERROR].name) + return elist[err - KAT_KEY_NOTVALID_ERROR].name; + + return "Invalid"; + } + + return "Unknown"; +} + +static TEE_Result ecc_get_key_size(uint32_t curve, size_t *bytes, size_t *bits) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P384: + *bits = 384; + *bytes = 48; + break; + case TEE_ECC_CURVE_NIST_P521: + *bits = 521; + *bytes = 66; + break; + default: + return TEE_ERROR_NOT_SUPPORTED; + } + + return TEE_SUCCESS; +} + +static void memcpy_swp(uint8_t *to, const uint8_t *from, size_t len) +{ + size_t i = 0; + + for (i = 0; i < len; i++) + to[i] = from[len - 1 - i]; +} + +static void crypto_bignum_bn2bin_eswap(uint32_t curve, + struct bignum *from, uint8_t *to) +{ + uint8_t pad[66] = { 0 }; + size_t len = crypto_bignum_num_bytes(from); + size_t bytes = 0; + size_t bits = 0; + + if (ecc_get_key_size(curve, &bytes, &bits)) + panic(); + + crypto_bignum_bn2bin(from, pad + bytes - len); + memcpy_swp(to, pad, bytes); +} + +static TEE_Result ecc_prepare_msg(uint32_t algo, const uint8_t *msg, + size_t msg_len, struct versal_mbox_mem *p) +{ + uint8_t swp[TEE_SHA512_HASH_SIZE + 2] = { 0 }; + size_t len = 0; + + if (msg_len > TEE_SHA512_HASH_SIZE + 2) + return TEE_ERROR_BAD_PARAMETERS; + + if (algo == TEE_ALG_ECDSA_SHA384) + len = TEE_SHA384_HASH_SIZE; + else if (algo == TEE_ALG_ECDSA_SHA512) + len = TEE_SHA512_HASH_SIZE + 2; + else + return TEE_ERROR_NOT_SUPPORTED; + + /* Swap the hash/message and pad if necessary */ + memcpy_swp(swp, msg, msg_len); + return versal_mbox_alloc(len, swp, p); +} + +static TEE_Result verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result ret = TEE_SUCCESS; + struct versal_ecc_verify_param *cmd = NULL; + struct versal_cmd_args arg = { }; + struct versal_mbox_mem x = { }; + struct versal_mbox_mem s = { }; + struct versal_mbox_mem p = { }; + struct versal_mbox_mem cmd_buf = { }; + uint32_t err = 0; + size_t bytes = 0; + size_t bits = 0; + + if (sig_len % 2) + return TEE_ERROR_SIGNATURE_INVALID; + + ret = ecc_get_key_size(key->curve, &bytes, &bits); + if (ret != TEE_SUCCESS) { + if (ret != TEE_ERROR_NOT_SUPPORTED) + return ret; + + /* Fallback to software */ + return pub_ops->verify(algo, key, msg, msg_len, sig, sig_len); + } + + ret = ecc_prepare_msg(algo, msg, msg_len, &p); + if (ret) + return ret; + + versal_mbox_alloc(bytes * 2, NULL, &x); + crypto_bignum_bn2bin_eswap(key->curve, key->x, x.buf); + crypto_bignum_bn2bin_eswap(key->curve, key->y, + (uint8_t *)x.buf + bytes); + /* Validate the public key for the curve */ + arg.data[0] = key->curve; + arg.dlen = 1; + arg.ibuf[0].mem = x; + if (versal_crypto_request(VERSAL_ELLIPTIC_VALIDATE_PUBLIC_KEY, + &arg, &err)) { + EMSG("Versal ECC: %s", versal_ecc_error(err)); + ret = TEE_ERROR_GENERIC; + goto out; + } + memset(&arg, 0, sizeof(arg)); + + versal_mbox_alloc(sig_len, NULL, &s); + /* Swap the {R,S} components */ + memcpy_swp(s.buf, sig, sig_len / 2); + memcpy_swp((uint8_t *)s.buf + sig_len / 2, sig + sig_len / 2, + sig_len / 2); + versal_mbox_alloc(sizeof(*cmd), NULL, &cmd_buf); + + cmd = cmd_buf.buf; + cmd->signature_addr = virt_to_phys(s.buf); + cmd->pub_key_addr = virt_to_phys(x.buf); + cmd->hash_addr = virt_to_phys(p.buf); + cmd->hash_len = p.len; + cmd->curve = key->curve; + + arg.ibuf[0].mem = cmd_buf; + arg.ibuf[1].mem = p; + arg.ibuf[1].only_cache = true; + arg.ibuf[2].mem = x; + arg.ibuf[3].mem = s; + + if (versal_crypto_request(VERSAL_ELLIPTIC_VERIFY_SIGN, &arg, &err)) { + EMSG("Versal ECC: %s", versal_ecc_error(err)); + ret = TEE_ERROR_GENERIC; + } +out: + free(p.buf); + free(x.buf); + free(s.buf); + free(cmd); + + return ret; +} + +static TEE_Result sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + struct versal_ecc_sign_param *cmd = NULL; + struct versal_mbox_mem cmd_buf = { }; + struct ecc_keypair ephemeral = { }; + struct versal_cmd_args arg = { }; + struct versal_mbox_mem p = { }; + struct versal_mbox_mem k = { }; + struct versal_mbox_mem d = { }; + struct versal_mbox_mem s = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + size_t bytes = 0; + size_t bits = 0; + + ret = ecc_get_key_size(key->curve, &bytes, &bits); + if (ret != TEE_SUCCESS) { + if (ret != TEE_ERROR_NOT_SUPPORTED) + return ret; + + /* Fallback to software */ + return pair_ops->sign(algo, key, msg, msg_len, sig, sig_len); + } + + /* Hash and update the length */ + ret = ecc_prepare_msg(algo, msg, msg_len, &p); + if (ret) + return ret; + + /* Ephemeral private key */ + ret = drvcrypt_asym_alloc_ecc_keypair(&ephemeral, + TEE_TYPE_ECDSA_KEYPAIR, bits); + if (ret) { + EMSG("Versal, can't allocate the ephemeral key"); + return ret; + } + + ephemeral.curve = key->curve; + ret = crypto_acipher_gen_ecc_key(&ephemeral, bits); + if (ret) { + EMSG("Versal, can't generate the ephemeral key"); + return ret; + } + + versal_mbox_alloc(bytes, NULL, &k); + crypto_bignum_bn2bin_eswap(key->curve, ephemeral.d, k.buf); + crypto_bignum_free(&ephemeral.d); + crypto_bignum_free(&ephemeral.x); + crypto_bignum_free(&ephemeral.y); + + /* Private key*/ + versal_mbox_alloc(bytes, NULL, &d); + crypto_bignum_bn2bin_eswap(key->curve, key->d, d.buf); + + /* Signature */ + versal_mbox_alloc(*sig_len, NULL, &s); + + /* IPI command */ + versal_mbox_alloc(sizeof(*cmd), NULL, &cmd_buf); + + cmd = cmd_buf.buf; + cmd->priv_key_addr = virt_to_phys(d.buf); + cmd->epriv_key_addr = virt_to_phys(k.buf); + cmd->hash_addr = virt_to_phys(p.buf); + cmd->hash_len = p.len; + cmd->curve = key->curve; + + arg.ibuf[0].mem = cmd_buf; + arg.ibuf[1].mem = s; + arg.ibuf[2].mem = k; + arg.ibuf[3].mem = d; + arg.ibuf[4].mem = p; + + if (versal_crypto_request(VERSAL_ELLIPTIC_GENERATE_SIGN, &arg, &err)) { + EMSG("Versal ECC: %s", versal_ecc_error(err)); + ret = TEE_ERROR_GENERIC; + goto out; + } + + *sig_len = 2 * bytes; + + /* Swap the {R,S} components */ + memcpy_swp(sig, s.buf, *sig_len / 2); + memcpy_swp(sig + *sig_len / 2, (uint8_t *)s.buf + *sig_len / 2, + *sig_len / 2); +out: + free(cmd); + free(k.buf); + free(p.buf); + free(s.buf); + free(d.buf); + + return ret; +} + +static TEE_Result shared_secret(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, size_t *secret_len) +{ + return pair_ops->shared_secret(private_key, public_key, + secret, secret_len); +} + +static TEE_Result do_shared_secret(struct drvcrypt_secret_data *sdata) +{ + return shared_secret(sdata->key_priv, + sdata->key_pub, + sdata->secret.data, + &sdata->secret.length); +} + +static TEE_Result do_sign(struct drvcrypt_sign_data *sdata) +{ + return sign(sdata->algo, + sdata->key, + sdata->message.data, + sdata->message.length, + sdata->signature.data, + &sdata->signature.length); +} + +static TEE_Result do_verify(struct drvcrypt_sign_data *sdata) +{ + return verify(sdata->algo, + sdata->key, + sdata->message.data, + sdata->message.length, + sdata->signature.data, + sdata->signature.length); +} + +static TEE_Result do_gen_keypair(struct ecc_keypair *s, size_t size_bits) +{ + /* + * Versal requires little endian so need to memcpy_swp on Versal IP ops. + * We chose not to do it here because some tests might be using + * their own keys + */ + return pair_ops->generate(s, size_bits); +} + +static TEE_Result do_alloc_keypair(struct ecc_keypair *s, + uint32_t type, size_t size_bits) +{ + TEE_Result ret = TEE_SUCCESS; + + /* This driver only supports ECDH/ECDSA */ + if (type != TEE_TYPE_ECDSA_KEYPAIR && + type != TEE_TYPE_ECDH_KEYPAIR) + return TEE_ERROR_NOT_IMPLEMENTED; + + ret = crypto_asym_alloc_ecc_keypair(s, TEE_TYPE_ECDSA_KEYPAIR, + size_bits); + if (ret) + return TEE_ERROR_NOT_IMPLEMENTED; + + /* + * Ignore the software operations, the crypto API will populate + * this interface. + */ + s->ops = NULL; + + return TEE_SUCCESS; +} + +static TEE_Result do_alloc_publickey(struct ecc_public_key *s, + uint32_t type, size_t size_bits) +{ + TEE_Result ret = TEE_SUCCESS; + + /* This driver only supports ECDH/ECDSA */ + if (type != TEE_TYPE_ECDSA_PUBLIC_KEY && + type != TEE_TYPE_ECDH_PUBLIC_KEY) + return TEE_ERROR_NOT_IMPLEMENTED; + + ret = crypto_asym_alloc_ecc_public_key(s, TEE_TYPE_ECDSA_PUBLIC_KEY, + size_bits); + if (ret) + return TEE_ERROR_NOT_IMPLEMENTED; + + /* + * Ignore the software operations, the crypto API will populate + * this interface. + */ + s->ops = NULL; + + return TEE_SUCCESS; +} + +static void do_free_publickey(struct ecc_public_key *s) +{ + return pub_ops->free(s); +} + +static struct drvcrypt_ecc driver_ecc = { + .shared_secret = do_shared_secret, + .alloc_publickey = do_alloc_publickey, + .free_publickey = do_free_publickey, + .alloc_keypair = do_alloc_keypair, + .gen_keypair = do_gen_keypair, + .verify = do_verify, + .sign = do_sign, +}; + +static TEE_Result ecc_init(void) +{ + struct versal_cmd_args arg = { }; + uint32_t err = 0; + + arg.data[arg.dlen++] = XSECURE_ECDSA_KAT_NIST_P384; + if (versal_crypto_request(VERSAL_ELLIPTIC_KAT, &arg, &err)) { + EMSG("Versal KAG NIST_P384: %s", versal_ecc_error(err)); + return TEE_ERROR_GENERIC; + } + + /* Clean previous request */ + arg.dlen = 0; + + arg.data[arg.dlen++] = XSECURE_ECDSA_KAT_NIST_P521; + if (versal_crypto_request(VERSAL_ELLIPTIC_KAT, &arg, &err)) { + EMSG("Versal KAG NIST_P521 %s", versal_ecc_error(err)); + return TEE_ERROR_GENERIC; + } + + pair_ops = crypto_asym_get_ecc_keypair_ops(TEE_TYPE_ECDSA_KEYPAIR); + if (!pair_ops) + return TEE_ERROR_GENERIC; + + pub_ops = crypto_asym_get_ecc_public_ops(TEE_TYPE_ECDSA_PUBLIC_KEY); + if (!pub_ops) + return TEE_ERROR_GENERIC; + + /* This driver supports both ECDH and ECDSA */ + assert((pub_ops == + crypto_asym_get_ecc_public_ops(TEE_TYPE_ECDH_PUBLIC_KEY)) && + (pair_ops == + crypto_asym_get_ecc_keypair_ops(TEE_TYPE_ECDH_KEYPAIR))); + + return drvcrypt_register_ecc(&driver_ecc); +} + +driver_init(ecc_init); diff --git a/optee_os/core/drivers/crypto/versal/include/ipi.h b/optee_os/core/drivers/crypto/versal/include/ipi.h new file mode 100644 index 0000000..2c92a1a --- /dev/null +++ b/optee_os/core/drivers/crypto/versal/include/ipi.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2022 + * Author: Jorge Ramirez + */ + +#ifndef IPI_H +#define IPI_H + +#include + +struct versal_rsa_input_param { + uint64_t key_addr; + uint64_t data_addr; + uint32_t key_len; +}; + +struct versal_rsa_sign_param { + uint64_t sign_addr; + uint64_t hash_addr; + uint32_t hash_len; +}; + +struct versal_ecc_sign_param { + uint64_t hash_addr; + uint64_t priv_key_addr; + uint64_t epriv_key_addr; + uint32_t curve; + uint32_t hash_len; +}; + +struct versal_ecc_verify_param { + uint64_t hash_addr; + uint64_t pub_key_addr; + uint64_t signature_addr; + uint32_t curve; + uint32_t hash_len; +}; + +enum versal_aes_operation { VERSAL_AES_ENCRYPT, VERSAL_AES_DECRYPT }; + +struct versal_aes_init { + uint64_t iv_addr; + uint32_t operation; + uint32_t key_src; + uint32_t key_len; +}; + +struct versal_aes_input_param { + uint64_t input_addr; + uint32_t input_len; + uint32_t is_last; +}; + +enum versal_crypto_api { + VERSAL_FEATURES = 0U, + VERSAL_RSA_SIGN_VERIFY, + VERSAL_RSA_PUBLIC_ENCRYPT, + VERSAL_RSA_PRIVATE_DECRYPT, + VERSAL_RSA_KAT, + VERSAL_SHA3_UPDATE = 32U, + VERSAL_SHA3_KAT, + VERSAL_ELLIPTIC_GENERATE_PUBLIC_KEY = 64U, + VERSAL_ELLIPTIC_GENERATE_SIGN, + VERSAL_ELLIPTIC_VALIDATE_PUBLIC_KEY, + VERSAL_ELLIPTIC_VERIFY_SIGN, + VERSAL_ELLIPTIC_KAT, + VERSAL_AES_INIT = 96U, + VERSAL_AES_OP_INIT, + VERSAL_AES_UPDATE_AAD, + VERSAL_AES_ENCRYPT_UPDATE, + VERSAL_AES_ENCRYPT_FINAL, + VERSAL_AES_DECRYPT_UPDATE, + VERSAL_AES_DECRYPT_FINAL, + VERSAL_AES_KEY_ZERO, + VERSAL_AES_WRITE_KEY, + VERSAL_AES_LOCK_USER_KEY, + VERSAL_AES_KEK_DECRYPT, + VERSAL_AES_SET_DPA_CM, + VERSAL_AES_DECRYPT_KAT, + VERSAL_AES_DECRYPT_CM_KAT, + VERSAL_CRYPTO_API_MAX +}; + +#define VERSAL_MAX_IPI_REGS 6 + +struct versal_cmd_args { + uint32_t data[VERSAL_MAX_IPI_REGS]; + size_t dlen; + struct versal_ipi_buf ibuf[VERSAL_MAX_IPI_BUF]; +}; + +TEE_Result versal_crypto_request(enum versal_crypto_api id, + struct versal_cmd_args *arg, uint32_t *err); +#endif + diff --git a/optee_os/core/drivers/crypto/versal/ipi.c b/optee_os/core/drivers/crypto/versal/ipi.c new file mode 100644 index 0000000..ba2b636 --- /dev/null +++ b/optee_os/core/drivers/crypto/versal/ipi.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022 + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define SEC_MODULE_SHIFT 8 +#define SEC_MODULE_ID 5 + +#define CRYPTO_API_ID(__x) ((SEC_MODULE_ID << SEC_MODULE_SHIFT) | (__x)) + +static TEE_Result versal_sha3_request(enum versal_crypto_api id, + struct versal_cmd_args *arg) +{ + struct versal_ipi_cmd cmd = { }; + uint32_t a = 0; + uint32_t b = 0; + + cmd.data[0] = CRYPTO_API_ID(id); + + if (arg->data[0]) { + /* write */ + reg_pair_from_64(virt_to_phys(arg->ibuf[0].mem.buf), &b, &a); + cmd.data[1] = a; + cmd.data[2] = b; + cmd.data[3] = arg->data[0]; + + cmd.ibuf[0].mem = arg->ibuf[0].mem; + } else { + /* read */ + reg_pair_from_64(virt_to_phys(arg->ibuf[0].mem.buf), &b, &a); + cmd.data[4] = a; + cmd.data[5] = b; + + cmd.ibuf[0].mem = arg->ibuf[0].mem; + } + + return versal_mbox_notify(&cmd, NULL, NULL); +} + +static TEE_Result versal_aes_update_aad_request(enum versal_crypto_api id, + struct versal_cmd_args *arg) +{ + struct versal_ipi_cmd cmd = { }; + uint32_t a = 0; + uint32_t b = 0; + + reg_pair_from_64(virt_to_phys(arg->ibuf[0].mem.buf), &b, &a); + + cmd.data[0] = CRYPTO_API_ID(id); + cmd.data[1] = a; + cmd.data[2] = b; + cmd.data[3] = arg->data[0]; + + cmd.ibuf[0].mem = arg->ibuf[0].mem; + + return versal_mbox_notify(&cmd, NULL, NULL); +} + +TEE_Result versal_crypto_request(enum versal_crypto_api id, + struct versal_cmd_args *arg, uint32_t *err) +{ + struct versal_ipi_cmd cmd = { }; + uint32_t a = 0; + uint32_t b = 0; + size_t i = 0; + + if (id == VERSAL_SHA3_UPDATE) + return versal_sha3_request(id, arg); + + if (id == VERSAL_AES_UPDATE_AAD) + return versal_aes_update_aad_request(id, arg); + + cmd.data[0] = CRYPTO_API_ID(id); + for (i = 1; i < arg->dlen + 1; i++) + cmd.data[i] = arg->data[i - 1]; + + /* src */ + if (!arg->ibuf[0].mem.buf) + goto notify; + + reg_pair_from_64(virt_to_phys(arg->ibuf[0].mem.buf), &b, &a); + cmd.data[i++] = a; + cmd.data[i++] = b; + + /* dst */ + if (!arg->ibuf[1].mem.buf) + goto cache; + + if (arg->ibuf[1].only_cache) + goto cache; + + reg_pair_from_64(virt_to_phys(arg->ibuf[1].mem.buf), &b, &a); + cmd.data[i++] = a; + cmd.data[i++] = b; +cache: + for (i = 0; i < VERSAL_MAX_IPI_BUF; i++) + cmd.ibuf[i].mem = arg->ibuf[i].mem; +notify: + return versal_mbox_notify(&cmd, NULL, err); +} diff --git a/optee_os/core/drivers/crypto/versal/rsa.c b/optee_os/core/drivers/crypto/versal/rsa.c new file mode 100644 index 0000000..bbfe979 --- /dev/null +++ b/optee_os/core/drivers/crypto/versal/rsa.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022. + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RSA_MAX_PRIV_EXP_LEN 512 +#define RSA_MAX_PUB_EXP_LEN 4 +#define RSA_MAX_MOD_LEN 512 + +static void crypto_bignum_bn2bin_pad(size_t size, + struct bignum *from, uint8_t *to) +{ + size_t len = crypto_bignum_num_bytes(from); + + crypto_bignum_bn2bin(from, to + size - len); +} + +static TEE_Result do_encrypt(struct drvcrypt_rsa_ed *rsa_data) +{ + struct rsa_public_key *p = rsa_data->key.key; + struct versal_rsa_input_param *cmd = NULL; + struct versal_mbox_mem cmd_buf = { }; + struct versal_mbox_mem cipher = { }; + struct versal_mbox_mem key = { }; + struct versal_mbox_mem msg = { }; + struct versal_cmd_args arg = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + + switch (rsa_data->rsa_id) { + case DRVCRYPT_RSA_PKCS_V1_5: + return sw_crypto_acipher_rsaes_encrypt(rsa_data->algo, + rsa_data->key.key, + rsa_data->label.data, + rsa_data->label.length, + rsa_data->message.data, + rsa_data->message.length, + rsa_data->cipher.data, + &rsa_data->cipher.length); + case DRVCRYPT_RSA_OAEP: + return sw_crypto_acipher_rsaes_encrypt(rsa_data->algo, + rsa_data->key.key, + rsa_data->label.data, + rsa_data->label.length, + rsa_data->message.data, + rsa_data->message.length, + rsa_data->cipher.data, + &rsa_data->cipher.length); + case DRVCRYPT_RSA_NOPAD: + return sw_crypto_acipher_rsanopad_encrypt(rsa_data->key.key, + rsa_data->message.data, + rsa_data->message.length, + rsa_data->cipher.data, + &rsa_data->cipher.length); + case DRVCRYPT_RSASSA_PKCS_V1_5: + assert(rsa_data->hash_algo != TEE_ALG_SHA1); + assert(rsa_data->key.n_size != 128); + break; + case DRVCRYPT_RSASSA_PSS: + default: + assert(0); + } + + versal_mbox_alloc(RSA_MAX_MOD_LEN + RSA_MAX_PUB_EXP_LEN, NULL, &key); + crypto_bignum_bn2bin_pad(rsa_data->key.n_size, p->n, key.buf); + crypto_bignum_bn2bin_pad(RSA_MAX_PUB_EXP_LEN, + p->e, (uint8_t *)key.buf + RSA_MAX_MOD_LEN); + + versal_mbox_alloc(rsa_data->message.length, rsa_data->message.data, + &msg); + versal_mbox_alloc(rsa_data->cipher.length, NULL, &cipher); + versal_mbox_alloc(sizeof(*cmd), NULL, &cmd_buf); + + cmd = cmd_buf.buf; + cmd->key_len = rsa_data->key.n_size; + cmd->data_addr = virt_to_phys(msg.buf); + cmd->key_addr = virt_to_phys(key.buf); + + arg.ibuf[0].mem = cmd_buf; + arg.ibuf[1].mem = cipher; + arg.ibuf[2].mem = msg; + arg.ibuf[3].mem = key; + + if (versal_crypto_request(VERSAL_RSA_PUBLIC_ENCRYPT, &arg, &err)) { + EMSG("Versal RSA: encrypt: error 0x%x [id:0x%x, len:%zu]", + err, rsa_data->rsa_id, rsa_data->key.n_size); + + if (rsa_data->rsa_id == DRVCRYPT_RSASSA_PKCS_V1_5) + ret = TEE_ERROR_SIGNATURE_INVALID; + else + ret = TEE_ERROR_GENERIC; + } + + if (!ret) { + rsa_data->cipher.length = rsa_data->key.n_size; + memcpy(rsa_data->cipher.data, cipher.buf, rsa_data->key.n_size); + } + + free(cipher.buf); + free(cmd); + free(msg.buf); + free(key.buf); + + return ret; +} + +static TEE_Result do_decrypt(struct drvcrypt_rsa_ed *rsa_data) +{ + struct versal_rsa_input_param *cmd = NULL; + struct rsa_keypair *p = rsa_data->key.key; + struct versal_mbox_mem cmd_buf = { }; + struct versal_mbox_mem cipher = { }; + struct versal_mbox_mem key = { }; + struct versal_mbox_mem msg = { }; + struct versal_cmd_args arg = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + + switch (rsa_data->rsa_id) { + case DRVCRYPT_RSA_PKCS_V1_5: + return sw_crypto_acipher_rsaes_decrypt(rsa_data->algo, + rsa_data->key.key, + rsa_data->label.data, + rsa_data->label.length, + rsa_data->cipher.data, + rsa_data->cipher.length, + rsa_data->message.data, + &rsa_data->message.length); + case DRVCRYPT_RSA_OAEP: + return sw_crypto_acipher_rsaes_decrypt(rsa_data->algo, + rsa_data->key.key, + rsa_data->label.data, + rsa_data->label.length, + rsa_data->cipher.data, + rsa_data->cipher.length, + rsa_data->message.data, + &rsa_data->message.length); + case DRVCRYPT_RSA_NOPAD: + return sw_crypto_acipher_rsanopad_decrypt(rsa_data->key.key, + rsa_data->cipher.data, + rsa_data->cipher.length, + rsa_data->message.data, + &rsa_data->message.length); + case DRVCRYPT_RSASSA_PKCS_V1_5: + assert(rsa_data->hash_algo != TEE_ALG_SHA1); + assert(rsa_data->key.n_size != 128); + break; + case DRVCRYPT_RSASSA_PSS: + default: + assert(0); + } + + versal_mbox_alloc(RSA_MAX_MOD_LEN + RSA_MAX_PRIV_EXP_LEN, NULL, &key); + crypto_bignum_bn2bin_pad(rsa_data->key.n_size, p->n, key.buf); + crypto_bignum_bn2bin_pad(rsa_data->key.n_size, p->d, + (uint8_t *)key.buf + RSA_MAX_MOD_LEN); + + versal_mbox_alloc(rsa_data->cipher.length, rsa_data->cipher.data, + &cipher); + versal_mbox_alloc(rsa_data->message.length, NULL, &msg); + versal_mbox_alloc(sizeof(*cmd), NULL, &cmd_buf); + + cmd = cmd_buf.buf; + cmd->key_len = rsa_data->key.n_size; + cmd->data_addr = virt_to_phys(cipher.buf); + cmd->key_addr = virt_to_phys(key.buf); + + arg.ibuf[0].mem = cmd_buf; + arg.ibuf[1].mem = msg; + arg.ibuf[2].mem = cipher; + arg.ibuf[3].mem = key; + + if (versal_crypto_request(VERSAL_RSA_PRIVATE_DECRYPT, &arg, &err)) { + EMSG("Versal RSA: decrypt: error 0x%x [id:0x%x, len:%zu]", + err, rsa_data->rsa_id, rsa_data->key.n_size); + ret = TEE_ERROR_GENERIC; + } + + if (!ret) { + rsa_data->message.length = rsa_data->key.n_size; + memcpy(rsa_data->message.data, msg.buf, rsa_data->key.n_size); + } + + free(cipher.buf); + free(cmd); + free(key.buf); + free(msg.buf); + + return ret; +} + +static TEE_Result do_ssa_sign(struct drvcrypt_rsa_ssa *p) +{ + switch (p->algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + if (p->key.n_size != 128) { + /* use DRVCRYPT_RSASSA_PKCS_V1_5, decrypt */ + return TEE_ERROR_NOT_IMPLEMENTED; + } + return sw_crypto_acipher_rsassa_sign(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + &p->signature.length); + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + return sw_crypto_acipher_rsassa_sign(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + &p->signature.length); + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + return sw_crypto_acipher_rsassa_sign(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + &p->signature.length); + default: + break; + } + + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result do_ssa_verify(struct drvcrypt_rsa_ssa *p) +{ + switch (p->algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + if (p->key.n_size != 128) { + /* use DRVCRYPT_RSASSA_PKCS_V1_5, encrypt */ + return TEE_ERROR_NOT_IMPLEMENTED; + } + return sw_crypto_acipher_rsassa_verify(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + p->signature.length); + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + return sw_crypto_acipher_rsassa_verify(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + p->signature.length); + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + return sw_crypto_acipher_rsassa_verify(p->algo, + p->key.key, + p->salt_len, + p->message.data, + p->message.length, + p->signature.data, + p->signature.length); + default: + break; + } + + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result do_gen_keypair(struct rsa_keypair *s, size_t size_bits) +{ + return sw_crypto_acipher_gen_rsa_key(s, size_bits); +} + +static TEE_Result do_alloc_keypair(struct rsa_keypair *s, size_t size_bits) +{ + return sw_crypto_acipher_alloc_rsa_keypair(s, size_bits); +} + +static TEE_Result do_alloc_publickey(struct rsa_public_key *key, size_t bits) +{ + return sw_crypto_acipher_alloc_rsa_public_key(key, bits); +} + +static void do_free_publickey(struct rsa_public_key *s) +{ + sw_crypto_acipher_free_rsa_public_key(s); +} + +static void do_free_keypair(struct rsa_keypair *s) +{ + sw_crypto_acipher_free_rsa_keypair(s); +} + +static struct drvcrypt_rsa driver_rsa = { + .alloc_publickey = do_alloc_publickey, + .free_publickey = do_free_publickey, + .alloc_keypair = do_alloc_keypair, + .optional.ssa_verify = do_ssa_verify, + .optional.ssa_sign = do_ssa_sign, + .free_keypair = do_free_keypair, + .gen_keypair = do_gen_keypair, + .encrypt = do_encrypt, + .decrypt = do_decrypt, +}; + +static TEE_Result rsa_init(void) +{ + struct versal_cmd_args arg = { }; + + if (versal_crypto_request(VERSAL_RSA_KAT, &arg, NULL)) + return TEE_ERROR_GENERIC; + + return drvcrypt_register_rsa(&driver_rsa); +} + +driver_init(rsa_init); diff --git a/optee_os/core/drivers/crypto/versal/sub.mk b/optee_os/core/drivers/crypto/versal/sub.mk new file mode 100644 index 0000000..5618469 --- /dev/null +++ b/optee_os/core/drivers/crypto/versal/sub.mk @@ -0,0 +1,6 @@ +incdirs-y += include + +srcs-y += ipi.c +srcs-y += authenc.c +srcs-y += ecc.c +srcs-y += rsa.c diff --git a/optee_os/core/drivers/dra7_rng.c b/optee_os/core/drivers/dra7_rng.c new file mode 100644 index 0000000..c3af335 --- /dev/null +++ b/optee_os/core/drivers/dra7_rng.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNG_OUTPUT_L 0x0000 +#define RNG_OUTPUT_H 0x0004 +#define RNG_STATUS 0x0008 +# define RNG_READY BIT(0) +# define SHUTDOWN_OFLO BIT(1) +#define RNG_INTMASK 0x000C +#define RNG_INTACK 0x0010 +#define RNG_CONTROL 0x0014 +# define ENABLE_TRNG BIT(10) +#define RNG_CONFIG 0x0018 +#define RNG_ALARMCNT 0x001C +#define RNG_FROENABLE 0x0020 +#define RNG_FRODETUNE 0x0024 +#define RNG_ALARMMASK 0x0028 +#define RNG_ALARMSTOP 0x002C +#define RNG_LFSR_L 0x0030 +#define RNG_LFSR_M 0x0034 +#define RNG_LFSR_H 0x0038 +#define RNG_COUNT 0x003C +#define RNG_OPTIONS 0x0078 +#define RNG_EIP_REV 0x007C +#define RNG_MMR_STATUS_EN 0x1FD8 +#define RNG_REV 0x1FE0 +#define RNG_SYS_CONFIG_REG 0x1FE4 +# define RNG_AUTOIDLE BIT(0) +#define RNG_MMR_STATUS_SET 0x1FEC +#define RNG_SOFT_RESET_REG 0x1FF0 +# define RNG_SOFT_RESET BIT(0) +#define RNG_IRQ_EOI_REG 0x1FF4 +#define RNG_IRQSTATUS 0x1FF8 + +#define RNG_CONTROL_STARTUP_CYCLES_SHIFT 16 +#define RNG_CONTROL_STARTUP_CYCLES_MASK GENMASK_32(31, 16) + +#define RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT 16 +#define RNG_CONFIG_MAX_REFIL_CYCLES_MASK GENMASK_32(31, 16) +#define RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT 0 +#define RNG_CONFIG_MIN_REFIL_CYCLES_MASK GENMASK_32(7, 0) + +#define RNG_ALARMCNT_ALARM_TH_SHIFT 0 +#define RNG_ALARMCNT_ALARM_TH_MASK GENMASK_32(7, 0) +#define RNG_ALARMCNT_SHUTDOWN_TH_SHIFT 16 +#define RNG_ALARMCNT_SHUTDOWN_TH_MASK GENMASK_32(20, 16) + +#define RNG_CONTROL_STARTUP_CYCLES 0xff +#define RNG_CONFIG_MIN_REFIL_CYCLES 0x21 +#define RNG_CONFIG_MAX_REFIL_CYCLES 0x22 +#define RNG_ALARM_THRESHOLD 0xff +#define RNG_SHUTDOWN_THRESHOLD 0x4 + +#define RNG_FRO_MASK GENMASK_32(23, 0) + +#define RNG_REG_SIZE 0x2000 + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, RNG_BASE, RNG_REG_SIZE); + +static unsigned int rng_lock = SPINLOCK_UNLOCK; +static vaddr_t rng; + +static void dra7_rng_read64(uint32_t *low_word, uint32_t *high_word) +{ + /* Is the result ready (available)? */ + while (!(io_read32(rng + RNG_STATUS) & RNG_READY)) { + /* Is the shutdown threshold reached? */ + if (io_read32(rng + RNG_STATUS) & SHUTDOWN_OFLO) { + uint32_t alarm = io_read32(rng + RNG_ALARMSTOP); + uint32_t tune = io_read32(rng + RNG_FRODETUNE); + + /* Clear the alarm events */ + io_write32(rng + RNG_ALARMMASK, 0x0); + io_write32(rng + RNG_ALARMSTOP, 0x0); + /* De-tune offending FROs */ + io_write32(rng + RNG_FRODETUNE, tune ^ alarm); + /* Re-enable the shut down FROs */ + io_write32(rng + RNG_FROENABLE, RNG_FRO_MASK); + /* Clear the shutdown overflow event */ + io_write32(rng + RNG_INTACK, SHUTDOWN_OFLO); + + DMSG("Fixed FRO shutdown\n"); + } + } + /* Read random value */ + *low_word = io_read32(rng + RNG_OUTPUT_L); + *high_word = io_read32(rng + RNG_OUTPUT_H); + /* Acknowledge read complete */ + io_write32(rng + RNG_INTACK, RNG_READY); +} + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + static union { + uint32_t val[2]; + uint8_t byte[8]; + } fifo; + static size_t fifo_pos; + uint8_t *buffer = buf; + size_t buffer_pos = 0; + + while (buffer_pos < len) { + uint32_t exceptions = cpu_spin_lock_xsave(&rng_lock); + + /* Refill our FIFO */ + if (fifo_pos == 0) + dra7_rng_read64(&fifo.val[0], &fifo.val[1]); + + buffer[buffer_pos++] = fifo.byte[fifo_pos++]; + fifo_pos %= 8; + + cpu_spin_unlock_xrestore(&rng_lock, exceptions); + } + + return TEE_SUCCESS; +} + +static TEE_Result dra7_rng_init(void) +{ + uint32_t val; + + rng = (vaddr_t)phys_to_virt(RNG_BASE, MEM_AREA_IO_SEC, RNG_REG_SIZE); + + /* Execute a software reset */ + io_write32(rng + RNG_SOFT_RESET_REG, RNG_SOFT_RESET); + + /* Wait for the software reset completion by polling */ + while (io_read32(rng + RNG_SOFT_RESET_REG) & RNG_SOFT_RESET) + ; + + /* Switch to low-power operating mode */ + io_write32(rng + RNG_SYS_CONFIG_REG, RNG_AUTOIDLE); + + /* + * Select the number of clock input cycles to the + * FROs between two samples + */ + val = 0; + + /* Ensure initial latency */ + val |= RNG_CONFIG_MIN_REFIL_CYCLES << + RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT; + val |= RNG_CONFIG_MAX_REFIL_CYCLES << + RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT; + io_write32(rng + RNG_CONFIG, val); + + /* Configure the desired FROs */ + io_write32(rng + RNG_FRODETUNE, 0x0); + + /* Enable all FROs */ + io_write32(rng + RNG_FROENABLE, 0xffffff); + + /* + * Select the maximum number of samples after + * which if a repeating pattern is still detected, an + * alarm event is generated + */ + val = RNG_ALARM_THRESHOLD << RNG_ALARMCNT_ALARM_TH_SHIFT; + + /* + * Set the shutdown threshold to the number of FROs + * allowed to be shut downed + */ + val |= RNG_SHUTDOWN_THRESHOLD << RNG_ALARMCNT_SHUTDOWN_TH_SHIFT; + io_write32(rng + RNG_ALARMCNT, val); + + /* Enable the RNG module */ + val = RNG_CONTROL_STARTUP_CYCLES << RNG_CONTROL_STARTUP_CYCLES_SHIFT; + val |= ENABLE_TRNG; + io_write32(rng + RNG_CONTROL, val); + + IMSG("DRA7x TRNG initialized"); + + return TEE_SUCCESS; +} +driver_init(dra7_rng_init); diff --git a/optee_os/core/drivers/gic.c b/optee_os/core/drivers/gic.c new file mode 100644 index 0000000..6d5b777 --- /dev/null +++ b/optee_os/core/drivers/gic.c @@ -0,0 +1,606 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2017, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Offsets from gic.gicc_base */ +#define GICC_CTLR (0x000) +#define GICC_PMR (0x004) +#define GICC_IAR (0x00C) +#define GICC_EOIR (0x010) + +#define GICC_CTLR_ENABLEGRP0 (1 << 0) +#define GICC_CTLR_ENABLEGRP1 (1 << 1) +#define GICD_CTLR_ENABLEGRP1S (1 << 2) +#define GICC_CTLR_FIQEN (1 << 3) + +/* Offsets from gic.gicd_base */ +#define GICD_CTLR (0x000) +#define GICD_TYPER (0x004) +#define GICD_IGROUPR(n) (0x080 + (n) * 4) +#define GICD_ISENABLER(n) (0x100 + (n) * 4) +#define GICD_ICENABLER(n) (0x180 + (n) * 4) +#define GICD_ISPENDR(n) (0x200 + (n) * 4) +#define GICD_ICPENDR(n) (0x280 + (n) * 4) +#define GICD_IPRIORITYR(n) (0x400 + (n) * 4) +#define GICD_ITARGETSR(n) (0x800 + (n) * 4) +#define GICD_IGROUPMODR(n) (0xd00 + (n) * 4) +#define GICD_SGIR (0xF00) + +#define GICD_CTLR_ENABLEGRP0 (1 << 0) +#define GICD_CTLR_ENABLEGRP1 (1 << 1) + +/* Number of Private Peripheral Interrupt */ +#define NUM_PPI 32 + +/* Number of Software Generated Interrupt */ +#define NUM_SGI 16 + +/* Number of Non-secure Software Generated Interrupt */ +#define NUM_NS_SGI 8 + +/* Number of interrupts in one register */ +#define NUM_INTS_PER_REG 32 + +/* Number of targets in one register */ +#define NUM_TARGETS_PER_REG 4 + +/* Accessors to access ITARGETSRn */ +#define ITARGETSR_FIELD_BITS 8 +#define ITARGETSR_FIELD_MASK 0xff + +#define GICD_TYPER_IT_LINES_NUM_MASK 0x1f +#define GICC_IAR_IT_ID_MASK 0x3ff +#define GICC_IAR_CPU_ID_MASK 0x7 +#define GICC_IAR_CPU_ID_SHIFT 10 + +struct gic_data { + vaddr_t gicc_base; + vaddr_t gicd_base; + size_t max_it; + struct itr_chip chip; +}; + +static struct gic_data gic_data __nex_bss; + +static void gic_op_add(struct itr_chip *chip, size_t it, uint32_t type, + uint32_t prio); +static void gic_op_enable(struct itr_chip *chip, size_t it); +static void gic_op_disable(struct itr_chip *chip, size_t it); +static void gic_op_raise_pi(struct itr_chip *chip, size_t it); +static void gic_op_raise_sgi(struct itr_chip *chip, size_t it, + uint8_t cpu_mask); +static void gic_op_set_affinity(struct itr_chip *chip, size_t it, + uint8_t cpu_mask); + +static const struct itr_ops gic_ops = { + .add = gic_op_add, + .mask = gic_op_disable, + .unmask = gic_op_enable, + .enable = gic_op_enable, + .disable = gic_op_disable, + .raise_pi = gic_op_raise_pi, + .raise_sgi = gic_op_raise_sgi, + .set_affinity = gic_op_set_affinity, +}; +DECLARE_KEEP_PAGER(gic_ops); + +static size_t probe_max_it(vaddr_t gicc_base __maybe_unused, vaddr_t gicd_base) +{ + int i; + uint32_t old_ctlr; + size_t ret = 0; + size_t max_regs = io_read32(gicd_base + GICD_TYPER) & + GICD_TYPER_IT_LINES_NUM_MASK; + + /* + * Probe which interrupt number is the largest. + */ +#if defined(CFG_ARM_GICV3) + old_ctlr = read_icc_ctlr(); + write_icc_ctlr(0); +#else + old_ctlr = io_read32(gicc_base + GICC_CTLR); + io_write32(gicc_base + GICC_CTLR, 0); +#endif + for (i = max_regs; i >= 0; i--) { + uint32_t old_reg; + uint32_t reg; + int b; + + old_reg = io_read32(gicd_base + GICD_ISENABLER(i)); + io_write32(gicd_base + GICD_ISENABLER(i), 0xffffffff); + reg = io_read32(gicd_base + GICD_ISENABLER(i)); + io_write32(gicd_base + GICD_ICENABLER(i), ~old_reg); + for (b = NUM_INTS_PER_REG - 1; b >= 0; b--) { + if (BIT32(b) & reg) { + ret = i * NUM_INTS_PER_REG + b; + goto out; + } + } + } +out: +#if defined(CFG_ARM_GICV3) + write_icc_ctlr(old_ctlr); +#else + io_write32(gicc_base + GICC_CTLR, old_ctlr); +#endif + return ret; +} + +void gic_cpu_init(void) +{ + struct gic_data *gd = &gic_data; + +#if defined(CFG_ARM_GICV3) + assert(gd->gicd_base); +#else + assert(gd->gicd_base && gd->gicc_base); +#endif + + /* per-CPU interrupts config: + * ID0-ID7(SGI) for Non-secure interrupts + * ID8-ID15(SGI) for Secure interrupts. + * All PPI config as Non-secure interrupts. + */ + io_write32(gd->gicd_base + GICD_IGROUPR(0), 0xffff00ff); + + /* Set the priority mask to permit Non-secure interrupts, and to + * allow the Non-secure world to adjust the priority mask itself + */ +#if defined(CFG_ARM_GICV3) + write_icc_pmr(0x80); + write_icc_igrpen1(1); +#else + io_write32(gd->gicc_base + GICC_PMR, 0x80); + + /* Enable GIC */ + io_write32(gd->gicc_base + GICC_CTLR, + GICC_CTLR_ENABLEGRP0 | GICC_CTLR_ENABLEGRP1 | + GICC_CTLR_FIQEN); +#endif +} + +static int gic_dt_get_irq(const uint32_t *properties, int count, uint32_t *type, + uint32_t *prio) +{ + int it_num = DT_INFO_INVALID_INTERRUPT; + + if (type) + *type = IRQ_TYPE_NONE; + + if (prio) + *prio = 0; + + if (!properties || count < 2) + return DT_INFO_INVALID_INTERRUPT; + + it_num = fdt32_to_cpu(properties[1]); + + switch (fdt32_to_cpu(properties[0])) { + case 1: + it_num += 16; + break; + case 0: + it_num += 32; + break; + default: + it_num = DT_INFO_INVALID_INTERRUPT; + } + + return it_num; +} + +static void gic_init_base_addr(paddr_t gicc_base_pa, paddr_t gicd_base_pa) +{ + struct gic_data *gd = &gic_data; + vaddr_t gicc_base = 0; + vaddr_t gicd_base = 0; + + assert(cpu_mmu_enabled()); + + gicd_base = core_mmu_get_va(gicd_base_pa, MEM_AREA_IO_SEC, + GIC_DIST_REG_SIZE); + if (!gicd_base) + panic(); + + if (!IS_ENABLED(CFG_ARM_GICV3)) { + gicc_base = core_mmu_get_va(gicc_base_pa, MEM_AREA_IO_SEC, + GIC_CPU_REG_SIZE); + if (!gicc_base) + panic(); + } + + gd->gicc_base = gicc_base; + gd->gicd_base = gicd_base; + gd->max_it = probe_max_it(gicc_base, gicd_base); + gd->chip.ops = &gic_ops; + + if (IS_ENABLED(CFG_DT)) + gd->chip.dt_get_irq = gic_dt_get_irq; +} + +void gic_init(paddr_t gicc_base_pa, paddr_t gicd_base_pa) +{ + struct gic_data __maybe_unused *gd = &gic_data; + size_t __maybe_unused n = 0; + + gic_init_base_addr(gicc_base_pa, gicd_base_pa); + + /* GIC configuration is initialized from TF-A when embedded */ +#ifndef CFG_WITH_ARM_TRUSTED_FW + for (n = 0; n <= gd->max_it / NUM_INTS_PER_REG; n++) { + /* Disable interrupts */ + io_write32(gd->gicd_base + GICD_ICENABLER(n), 0xffffffff); + + /* Make interrupts non-pending */ + io_write32(gd->gicd_base + GICD_ICPENDR(n), 0xffffffff); + + /* Mark interrupts non-secure */ + if (n == 0) { + /* per-CPU inerrupts config: + * ID0-ID7(SGI) for Non-secure interrupts + * ID8-ID15(SGI) for Secure interrupts. + * All PPI config as Non-secure interrupts. + */ + io_write32(gd->gicd_base + GICD_IGROUPR(n), 0xffff00ff); + } else { + io_write32(gd->gicd_base + GICD_IGROUPR(n), 0xffffffff); + } + } + + /* Set the priority mask to permit Non-secure interrupts, and to + * allow the Non-secure world to adjust the priority mask itself + */ +#if defined(CFG_ARM_GICV3) + write_icc_pmr(0x80); + write_icc_igrpen1(1); + io_setbits32(gd->gicd_base + GICD_CTLR, GICD_CTLR_ENABLEGRP1S); +#else + io_write32(gd->gicc_base + GICC_PMR, 0x80); + + /* Enable GIC */ + io_write32(gd->gicc_base + GICC_CTLR, GICC_CTLR_FIQEN | + GICC_CTLR_ENABLEGRP0 | GICC_CTLR_ENABLEGRP1); + io_setbits32(gd->gicd_base + GICD_CTLR, + GICD_CTLR_ENABLEGRP0 | GICD_CTLR_ENABLEGRP1); +#endif +#endif /*CFG_WITH_ARM_TRUSTED_FW*/ + + interrupt_main_init(&gic_data.chip); +} + +static void gic_it_add(struct gic_data *gd, size_t it) +{ + size_t idx = it / NUM_INTS_PER_REG; + uint32_t mask = 1 << (it % NUM_INTS_PER_REG); + + assert(gd == &gic_data); + + /* Disable the interrupt */ + io_write32(gd->gicd_base + GICD_ICENABLER(idx), mask); + /* Make it non-pending */ + io_write32(gd->gicd_base + GICD_ICPENDR(idx), mask); + /* Assign it to group0 */ + io_clrbits32(gd->gicd_base + GICD_IGROUPR(idx), mask); +#if defined(CFG_ARM_GICV3) + /* Assign it to group1S */ + io_setbits32(gd->gicd_base + GICD_IGROUPMODR(idx), mask); +#endif +} + +static void gic_it_set_cpu_mask(struct gic_data *gd, size_t it, + uint8_t cpu_mask) +{ + size_t idx __maybe_unused = it / NUM_INTS_PER_REG; + uint32_t mask __maybe_unused = 1 << (it % NUM_INTS_PER_REG); + uint32_t target, target_shift; + vaddr_t itargetsr = gd->gicd_base + + GICD_ITARGETSR(it / NUM_TARGETS_PER_REG); + + assert(gd == &gic_data); + + /* Assigned to group0 */ + assert(!(io_read32(gd->gicd_base + GICD_IGROUPR(idx)) & mask)); + + /* Route it to selected CPUs */ + target = io_read32(itargetsr); + target_shift = (it % NUM_TARGETS_PER_REG) * ITARGETSR_FIELD_BITS; + target &= ~(ITARGETSR_FIELD_MASK << target_shift); + target |= cpu_mask << target_shift; + DMSG("cpu_mask: writing 0x%x to 0x%" PRIxVA, target, itargetsr); + io_write32(itargetsr, target); + DMSG("cpu_mask: 0x%x", io_read32(itargetsr)); +} + +static void gic_it_set_prio(struct gic_data *gd, size_t it, uint8_t prio) +{ + size_t idx __maybe_unused = it / NUM_INTS_PER_REG; + uint32_t mask __maybe_unused = 1 << (it % NUM_INTS_PER_REG); + + assert(gd == &gic_data); + + /* Assigned to group0 */ + assert(!(io_read32(gd->gicd_base + GICD_IGROUPR(idx)) & mask)); + + /* Set prio it to selected CPUs */ + DMSG("prio: writing 0x%x to 0x%" PRIxVA, + prio, gd->gicd_base + GICD_IPRIORITYR(0) + it); + io_write8(gd->gicd_base + GICD_IPRIORITYR(0) + it, prio); +} + +static void gic_it_enable(struct gic_data *gd, size_t it) +{ + size_t idx = it / NUM_INTS_PER_REG; + uint32_t mask = 1 << (it % NUM_INTS_PER_REG); + vaddr_t base = gd->gicd_base; + + assert(gd == &gic_data); + + /* Assigned to group0 */ + assert(!(io_read32(base + GICD_IGROUPR(idx)) & mask)); + + /* Enable the interrupt */ + io_write32(base + GICD_ISENABLER(idx), mask); +} + +static void gic_it_disable(struct gic_data *gd, size_t it) +{ + size_t idx = it / NUM_INTS_PER_REG; + uint32_t mask = 1 << (it % NUM_INTS_PER_REG); + + assert(gd == &gic_data); + + /* Assigned to group0 */ + assert(!(io_read32(gd->gicd_base + GICD_IGROUPR(idx)) & mask)); + + /* Disable the interrupt */ + io_write32(gd->gicd_base + GICD_ICENABLER(idx), mask); +} + +static void gic_it_set_pending(struct gic_data *gd, size_t it) +{ + size_t idx = it / NUM_INTS_PER_REG; + uint32_t mask = BIT32(it % NUM_INTS_PER_REG); + + assert(gd == &gic_data); + + /* Should be Peripheral Interrupt */ + assert(it >= NUM_SGI); + + /* Raise the interrupt */ + io_write32(gd->gicd_base + GICD_ISPENDR(idx), mask); +} + +static void gic_it_raise_sgi(struct gic_data *gd __maybe_unused, size_t it, + uint8_t cpu_mask, uint8_t group) +{ +#if defined(CFG_ARM_GICV3) + /* Only support sending SGI to the cores in the same cluster now */ + uint32_t mask_id = it & 0xf; + uint32_t mask_cpu = cpu_mask & 0xff; + uint64_t mpidr = read_mpidr(); + uint64_t mask_aff1 = (mpidr & MPIDR_AFF1_MASK) >> MPIDR_AFF1_SHIFT; + uint64_t mask_aff2 = (mpidr & MPIDR_AFF2_MASK) >> MPIDR_AFF2_SHIFT; + uint64_t mask_aff3 = (mpidr & MPIDR_AFF3_MASK) >> MPIDR_AFF3_SHIFT; + uint64_t mask = (mask_cpu | + SHIFT_U64(mask_aff1, 16) | + SHIFT_U64(mask_id, 24) | + SHIFT_U64(mask_aff2, 32) | + SHIFT_U64(mask_aff3, 48)); + + /* Raise the interrupt */ + if (group) + write_icc_asgi1r(mask); + else + write_icc_sgi1r(mask); +#else + uint32_t mask_id = it & 0xf; + uint32_t mask_group = group & 0x1; + uint32_t mask_cpu = cpu_mask & 0xff; + uint32_t mask = (mask_id | SHIFT_U32(mask_group, 15) | + SHIFT_U32(mask_cpu, 16)); + + /* Raise the interrupt */ + io_write32(gd->gicd_base + GICD_SGIR, mask); +#endif +} + +static uint32_t gic_read_iar(struct gic_data *gd __maybe_unused) +{ + assert(gd == &gic_data); + +#if defined(CFG_ARM_GICV3) + return read_icc_iar1(); +#else + return io_read32(gd->gicc_base + GICC_IAR); +#endif +} + +static void gic_write_eoir(struct gic_data *gd __maybe_unused, uint32_t eoir) +{ + assert(gd == &gic_data); + +#if defined(CFG_ARM_GICV3) + write_icc_eoir1(eoir); +#else + io_write32(gd->gicc_base + GICC_EOIR, eoir); +#endif +} + +static bool gic_it_is_enabled(struct gic_data *gd, size_t it) +{ + size_t idx = it / NUM_INTS_PER_REG; + uint32_t mask = 1 << (it % NUM_INTS_PER_REG); + + assert(gd == &gic_data); + return !!(io_read32(gd->gicd_base + GICD_ISENABLER(idx)) & mask); +} + +static bool __maybe_unused gic_it_get_group(struct gic_data *gd, size_t it) +{ + size_t idx = it / NUM_INTS_PER_REG; + uint32_t mask = 1 << (it % NUM_INTS_PER_REG); + + assert(gd == &gic_data); + return !!(io_read32(gd->gicd_base + GICD_IGROUPR(idx)) & mask); +} + +static uint32_t __maybe_unused gic_it_get_target(struct gic_data *gd, size_t it) +{ + size_t reg_idx = it / NUM_TARGETS_PER_REG; + uint32_t target_shift = (it % NUM_TARGETS_PER_REG) * + ITARGETSR_FIELD_BITS; + uint32_t target_mask = ITARGETSR_FIELD_MASK << target_shift; + uint32_t target = io_read32(gd->gicd_base + GICD_ITARGETSR(reg_idx)); + + assert(gd == &gic_data); + return (target & target_mask) >> target_shift; +} + +void gic_dump_state(void) +{ + struct gic_data *gd = &gic_data; + int i = 0; + +#if defined(CFG_ARM_GICV3) + DMSG("GICC_CTLR: 0x%x", read_icc_ctlr()); +#else + DMSG("GICC_CTLR: 0x%x", io_read32(gd->gicc_base + GICC_CTLR)); +#endif + DMSG("GICD_CTLR: 0x%x", io_read32(gd->gicd_base + GICD_CTLR)); + + for (i = 0; i <= (int)gd->max_it; i++) { + if (gic_it_is_enabled(gd, i)) { + DMSG("irq%d: enabled, group:%d, target:%x", i, + gic_it_get_group(gd, i), gic_it_get_target(gd, i)); + } + } +} + +static void __maybe_unused gic_native_itr_handler(void) +{ + struct gic_data *gd = &gic_data; + uint32_t iar = 0; + uint32_t id = 0; + + iar = gic_read_iar(gd); + id = iar & GICC_IAR_IT_ID_MASK; + + if (id <= gd->max_it) + interrupt_call_handlers(&gd->chip, id); + else + DMSG("ignoring interrupt %" PRIu32, id); + + gic_write_eoir(gd, iar); +} + +#ifndef CFG_CORE_WORKAROUND_ARM_NMFI +/* Override interrupt_main_handler() with driver implementation */ +void interrupt_main_handler(void) +{ + gic_native_itr_handler(); +} +#endif /*CFG_CORE_WORKAROUND_ARM_NMFI*/ + +static void gic_op_add(struct itr_chip *chip, size_t it, + uint32_t type __unused, + uint32_t prio __unused) +{ + struct gic_data *gd = container_of(chip, struct gic_data, chip); + + assert(gd == &gic_data); + + if (it > gd->max_it) + panic(); + + gic_it_add(gd, it); + /* Set the CPU mask to deliver interrupts to any online core */ + gic_it_set_cpu_mask(gd, it, 0xff); + gic_it_set_prio(gd, it, 0x1); +} + +static void gic_op_enable(struct itr_chip *chip, size_t it) +{ + struct gic_data *gd = container_of(chip, struct gic_data, chip); + + assert(gd == &gic_data); + + if (it > gd->max_it) + panic(); + + gic_it_enable(gd, it); +} + +static void gic_op_disable(struct itr_chip *chip, size_t it) +{ + struct gic_data *gd = container_of(chip, struct gic_data, chip); + + assert(gd == &gic_data); + + if (it > gd->max_it) + panic(); + + gic_it_disable(gd, it); +} + +static void gic_op_raise_pi(struct itr_chip *chip, size_t it) +{ + struct gic_data *gd = container_of(chip, struct gic_data, chip); + + assert(gd == &gic_data); + + if (it > gd->max_it) + panic(); + + gic_it_set_pending(gd, it); +} + +static void gic_op_raise_sgi(struct itr_chip *chip, size_t it, + uint8_t cpu_mask) +{ + struct gic_data *gd = container_of(chip, struct gic_data, chip); + + assert(gd == &gic_data); + + /* Should be Software Generated Interrupt */ + assert(it < NUM_SGI); + + if (it > gd->max_it) + panic(); + + if (it < NUM_NS_SGI) + gic_it_raise_sgi(gd, it, cpu_mask, 1); + else + gic_it_raise_sgi(gd, it, cpu_mask, 0); +} + +static void gic_op_set_affinity(struct itr_chip *chip, size_t it, + uint8_t cpu_mask) +{ + struct gic_data *gd = container_of(chip, struct gic_data, chip); + + assert(gd == &gic_data); + + if (it > gd->max_it) + panic(); + + gic_it_set_cpu_mask(gd, it, cpu_mask); +} diff --git a/optee_os/core/drivers/gpio/gpio.c b/optee_os/core/drivers/gpio/gpio.c new file mode 100644 index 0000000..a3e4f23 --- /dev/null +++ b/optee_os/core/drivers/gpio/gpio.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include + +TEE_Result gpio_dt_alloc_pin(struct dt_pargs *pargs, struct gpio **out_gpio) +{ + struct gpio *gpio = NULL; + + if (pargs->args_count != 2) + return TEE_ERROR_BAD_PARAMETERS; + + gpio = calloc(1, sizeof(struct gpio)); + if (!gpio) + return TEE_ERROR_OUT_OF_MEMORY; + + gpio->pin = pargs->args[0]; + gpio->dt_flags = pargs->args[1]; + + *out_gpio = gpio; + + return TEE_SUCCESS; +} + +static char *gpio_get_dt_prop_name(const char *gpio_name) +{ + char *prop_name = NULL; + int max_len = strlen(gpio_name) + strlen("-gpios") + 1; + + prop_name = calloc(1, max_len); + if (!prop_name) + return NULL; + + snprintf(prop_name, max_len, "%s-gpios", gpio_name); + + return prop_name; +} + +TEE_Result gpio_dt_get_by_index(const void *fdt, int nodeoffset, + unsigned int index, const char *gpio_name, + struct gpio **gpio) +{ + TEE_Result res = TEE_ERROR_GENERIC; + char *prop_name = NULL; + void *out_gpio = NULL; + + prop_name = gpio_get_dt_prop_name(gpio_name); + if (!prop_name) + return TEE_ERROR_OUT_OF_MEMORY; + + res = dt_driver_device_from_node_idx_prop(prop_name, fdt, nodeoffset, + index, DT_DRIVER_GPIO, + &out_gpio); + free(prop_name); + if (!res) + *gpio = out_gpio; + + return res; +} diff --git a/optee_os/core/drivers/gpio/sub.mk b/optee_os/core/drivers/gpio/sub.mk new file mode 100644 index 0000000..9b91fbc --- /dev/null +++ b/optee_os/core/drivers/gpio/sub.mk @@ -0,0 +1 @@ +srcs-y += gpio.c diff --git a/optee_os/core/drivers/hfic.c b/optee_os/core/drivers/hfic.c new file mode 100644 index 0000000..1099410 --- /dev/null +++ b/optee_os/core/drivers/hfic.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +struct hfic_data { + struct itr_chip chip; +}; + +static struct hfic_data hfic_data __nex_bss; + +static void hfic_op_add(struct itr_chip *chip __unused, size_t it __unused, + uint32_t type __unused, uint32_t prio __unused) +{ +} + +static void hfic_op_enable(struct itr_chip *chip __unused, size_t it) +{ + uint32_t res __maybe_unused = 0; + + res = thread_hvc(HF_INTERRUPT_ENABLE, it, HF_ENABLE, + HF_INTERRUPT_TYPE_IRQ); + assert(!res); +} + +static void hfic_op_disable(struct itr_chip *chip __unused, size_t it) +{ + uint32_t res __maybe_unused = 0; + + res = thread_hvc(HF_INTERRUPT_ENABLE, it, HF_DISABLE, + HF_INTERRUPT_TYPE_IRQ); + assert(!res); +} + +static const struct itr_ops hfic_ops = { + .add = hfic_op_add, + .mask = hfic_op_disable, + .unmask = hfic_op_enable, + .enable = hfic_op_enable, + .disable = hfic_op_disable, +}; + +void hfic_init(void) +{ + hfic_data.chip.ops = &hfic_ops; + interrupt_main_init(&hfic_data.chip); +} + +/* Override interrupt_main_handler() with driver implementation */ +void interrupt_main_handler(void) +{ + uint32_t id = 0; + uint32_t res __maybe_unused = 0; + + id = thread_hvc(HF_INTERRUPT_GET, 0, 0, 0); + if (id == HF_INVALID_INTID) { + DMSG("ignoring invalid interrupt %#"PRIx32, id); + return; + } + + interrupt_call_handlers(&hfic_data.chip, id); + + res = thread_hvc(HF_INTERRUPT_DEACTIVATE, id, id, 0); + assert(!res); +} diff --git a/optee_os/core/drivers/hi16xx_rng.c b/optee_os/core/drivers/hi16xx_rng.c new file mode 100644 index 0000000..a3ba2cd --- /dev/null +++ b/optee_os/core/drivers/hi16xx_rng.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +/* Driver for the internal Random Number Generator of HiSilicon P660/Hi16xx */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* ALG sub-controller registers */ + +#define ALG_SC_RNG_RESET_DREQ 0xAB4 /* RNG reset cancel */ +# define ALG_SC_SRST_DREQ_RNG BIT(0) + +/* RNG registers */ + +#define RNG_SEED 0x0 /* Initial seed */ +#define RNG_CTRL 0x4 /* Control register */ +# define RNG_SEED_SEL BIT(2) /* Re-seed source: 1: ring osc., 0: LFSR */ +# define RNG_RING_EN BIT(1) /* Enable ring oscillator */ +# define RNG_EN BIT(0) /* Enable RNG */ +#define RNG_NUM 0x10 /* Random number output */ +#define RNG_PHY_SEED 0x14 /* Ring oscillator output */ + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, ALG_SC_BASE, ALG_SC_REG_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, RNG_BASE, RNG_REG_SIZE); + +static unsigned int rng_lock = SPINLOCK_UNLOCK; + +static TEE_Result hi16xx_rng_init(void) +{ + vaddr_t alg = (vaddr_t)phys_to_virt(ALG_SC_BASE, MEM_AREA_IO_SEC, + ALG_SC_REG_SIZE); + vaddr_t rng = (vaddr_t)phys_to_virt(RNG_BASE, MEM_AREA_IO_SEC, + RNG_REG_SIZE); + TEE_Time time; + + /* ALG sub-controller must allow RNG out of reset */ + io_write32(alg + ALG_SC_RNG_RESET_DREQ, ALG_SC_SRST_DREQ_RNG); + + /* Set initial seed */ + tee_time_get_sys_time(&time); + io_write32(rng + RNG_SEED, time.seconds * 1000 + time.millis); + + /* + * Enable RNG and configure it to re-seed automatically from the + * internal ring oscillator + */ + io_write32(rng + RNG_CTRL, RNG_EN | RNG_RING_EN | RNG_SEED_SEL); + + IMSG("Hi16xx RNG initialized"); + return TEE_SUCCESS; +} + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + static vaddr_t r; + static int pos; + static union { + uint32_t val; + uint8_t byte[4]; + } random; + size_t buffer_pos = 0; + uint8_t *buffer = buf; + uint32_t exceptions; + + exceptions = cpu_spin_lock_xsave(&rng_lock); + if (!r) + r = (vaddr_t)phys_to_virt(RNG_BASE, MEM_AREA_IO_SEC, 1) + + RNG_NUM; + cpu_spin_unlock_xrestore(&rng_lock, exceptions); + + while (buffer_pos < len) { + exceptions = cpu_spin_lock_xsave(&rng_lock); + + /* Refill our FIFO */ + if (pos == 0) + random.val = io_read32(r); + + buffer[buffer_pos++] = random.byte[pos++]; + if (pos == 4) + pos = 0; + + cpu_spin_unlock_xrestore(&rng_lock, exceptions); + } + + return TEE_SUCCESS; +} + +driver_init(hi16xx_rng_init); diff --git a/optee_os/core/drivers/hi16xx_uart.c b/optee_os/core/drivers/hi16xx_uart.c new file mode 100644 index 0000000..6f987c3 --- /dev/null +++ b/optee_os/core/drivers/hi16xx_uart.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ +#include +#include +#include +#include +#include +#include + +/* Register offsets */ + +#define UART_RBR 0x00 /* RX data buffer register */ +#define UART_THR 0x00 /* TX data buffer register */ +#define UART_DLL 0x00 /* Lower-bit frequency divider register */ + +#define UART_IEL 0x04 /* Interrupt enable register */ +#define UART_DLH 0x04 /* Upper-bit frequency divider register */ + +#define UART_FCR 0x08 /* FIFO control register */ + +#define UART_LCR 0x0C /* Line control register */ + +#define UART_LSR 0x14 /* Line status register */ + +#define UART_USR 0x7C /* Status register */ + +/* + * Line control register + */ + +/* Data length selection */ +#define UART_LCR_DLS5 0x0 /* 5 bits */ +#define UART_LCR_DLS6 0x1 /* 6 bits */ +#define UART_LCR_DLS7 0x2 /* 7 bits */ +#define UART_LCR_DLS8 0x3 /* 8 bits */ + +/* Enable access to UART_DLL and UART_DLH */ +#define UART_LCR_DLAB 0x80 + +/* + * FIFO control register + */ + +#define UART_FCR_FIFO_EN 0x1 /* Enable FIFO (depth: 32 bytes) */ +#define UART_FCR_RX_FIFO_RST 0x2 /* Clear receive FIFO (auto reset) */ +#define UART_FCR_TX_FIFO_RST 0x4 /* Clear send FIFO (auto reset) */ + + +/* + * Status register + */ + +#define UART_USR_BUSY_BIT 0 /* 0: idle/non-activated, 1: busy */ +#define UART_USR_TFNF_BIT 1 /* Transmit FIFO not full bit */ +#define UART_USR_TFE_BIT 2 /* Transmit FIFO empty bit */ +#define UART_USR_RFNE_BIT 3 /* Receive FIFO not empty bit */ +#define UART_USR_RFF_BIT 4 /* Receive FIFO full bit */ + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct hi16xx_uart_data *pd = + container_of(chip, struct hi16xx_uart_data, chip); + + return io_pa_or_va(&pd->base, HI16XX_UART_REG_SIZE); +} + +static void hi16xx_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + UART_USR) & UART_USR_TFE_BIT)) + ; +} + +static void hi16xx_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + /* Wait until TX FIFO is empty */ + while (!(io_read32(base + UART_USR) & UART_USR_TFE_BIT)) + ; + + /* Put character into TX FIFO */ + io_write32(base + UART_THR, ch & 0xFF); +} + +static bool hi16xx_uart_have_rx_data(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + return (io_read32(base + UART_USR) & UART_USR_RFNE_BIT); +} + +static int hi16xx_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!hi16xx_uart_have_rx_data(chip)) + ; + return io_read32(base + UART_RBR) & 0xFF; +} + +static const struct serial_ops hi16xx_uart_ops = { + .flush = hi16xx_uart_flush, + .getchar = hi16xx_uart_getchar, + .have_rx_data = hi16xx_uart_have_rx_data, + .putc = hi16xx_uart_putc, +}; +DECLARE_KEEP_PAGER(hi16xx_uart_ops); + +void hi16xx_uart_init(struct hi16xx_uart_data *pd, paddr_t base, + uint32_t uart_clk, uint32_t baud_rate) +{ + uint16_t freq_div = uart_clk / (16 * baud_rate); + + pd->base.pa = base; + pd->chip.ops = &hi16xx_uart_ops; + + /* Enable (and clear) FIFOs */ + io_write32(base + UART_FCR, UART_FCR_FIFO_EN); + + /* Enable access to _DLL and _DLH */ + io_write32(base + UART_LCR, UART_LCR_DLAB); + + /* Calculate and set UART_DLL */ + io_write32(base + UART_DLL, freq_div & 0xFF); + + /* Calculate and set UART_DLH */ + io_write32(base + UART_DLH, (freq_div >> 8) & 0xFF); + + /* Clear _DLL/_DLH access bit, set data size (8 bits), parity etc. */ + io_write32(base + UART_LCR, UART_LCR_DLS8); + + /* Disable interrupt mode */ + io_write32(base + UART_IEL, 0); + + hi16xx_uart_flush(&pd->chip); +} + diff --git a/optee_os/core/drivers/i2c/atmel_i2c.c b/optee_os/core/drivers/i2c/atmel_i2c.c new file mode 100644 index 0000000..df44f31 --- /dev/null +++ b/optee_os/core/drivers/i2c/atmel_i2c.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021-2023 Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TWIHS_CR 0x0 +#define TWIHS_CR_SWRST BIT(7) +#define TWIHS_CR_SVDIS BIT(5) +#define TWIHS_CR_MSEN BIT(2) +#define TWIHS_CR_STOP BIT(1) +#define TWIHS_CR_START BIT(0) + +#define TWIHS_MMR 0x4 +#define TWIHS_MMR_DADR_SHIFT 16 +#define TWIHS_MMR_DADR_MASK 0x7F +#define TWIHS_MMR_MREAD BIT32(12) + +#define TWIHS_CWGR 0x10 +#define TWIHS_CWGR_HOLD_SHIFT 24 +#define TWIHS_CWGR_HOLD_MAX 0x1F +#define TWIHS_CWGR_CKDIV_SHIFT 16 +#define TWIHS_CWGR_CKDIV_MAX 0x7 +#define TWIHS_CWGR_CHDIV_SHIFT 8 + +#define TWIHS_CKSRC BIT32(20) + +#define TWIHS_SR 0x20 +#define TWIHS_SR_NACK BIT32(8) +#define TWIHS_SR_TXRDY BIT32(2) +#define TWIHS_SR_RXRDY BIT32(1) +#define TWIHS_SR_TXCOMP BIT32(0) + +#define TWIHS_RHR 0x30 +#define TWIHS_THR 0x34 + +#define TWIHS_WPMR 0xE4 +#define TWIHS_WPMR_WPKEY SHIFT_U32(0x545749, 8) + +#define I2C_BUS_FREQ 400000 + +struct atmel_i2c { + uint32_t sda_hold_time; + vaddr_t base; + struct clk *clk; + struct i2c_ctrl i2c_ctrl; +}; + +static struct atmel_i2c *atmel_i2c_from_i2c_ctrl(struct i2c_ctrl *i2c_ctrl) +{ + return container_of(i2c_ctrl, struct atmel_i2c, i2c_ctrl); +} + +static TEE_Result atmel_i2c_send_one_byte(struct atmel_i2c *i2c, uint8_t byte) +{ + uint32_t sr = 0; + + io_write32(i2c->base + TWIHS_THR, byte); + + while (true) { + sr = io_read32(i2c->base + TWIHS_SR); + if (sr & TWIHS_SR_NACK) { + EMSG("I2C received NACK while writing"); + return TEE_ERROR_GENERIC; + } + if (sr & TWIHS_SR_TXRDY) + break; + } + + return TEE_SUCCESS; +} + +static void atmel_i2c_wait_txcomp(struct atmel_i2c *i2c) +{ + uint32_t sr = 0; + + while (true) { + sr = io_read32(i2c->base + TWIHS_SR); + if (sr & TWIHS_SR_TXCOMP) + return; + } +} + +static void atmel_i2c_send_start(struct atmel_i2c *i2c) +{ + io_write32(i2c->base + TWIHS_CR, TWIHS_CR_START); +} + +static void atmel_i2c_send_stop(struct atmel_i2c *i2c) +{ + io_write32(i2c->base + TWIHS_CR, TWIHS_CR_STOP); +} + +static TEE_Result atmel_i2c_write_data_no_stop(struct i2c_dev *i2c_dev, + const uint8_t *buf, size_t len) +{ + size_t i = 0; + TEE_Result res = TEE_ERROR_GENERIC; + struct atmel_i2c *i2c = atmel_i2c_from_i2c_ctrl(i2c_dev->ctrl); + uint32_t mmr = SHIFT_U32(i2c_dev->addr, TWIHS_MMR_DADR_SHIFT); + + io_write32(i2c->base + TWIHS_MMR, mmr); + + for (i = 0; i < len; i++) { + res = atmel_i2c_send_one_byte(i2c, buf[i]); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +static TEE_Result atmel_i2c_write_data(struct i2c_dev *i2c_dev, + const uint8_t *buf, size_t len) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct atmel_i2c *i2c = atmel_i2c_from_i2c_ctrl(i2c_dev->ctrl); + + res = atmel_i2c_write_data_no_stop(i2c_dev, buf, len); + if (res) + return res; + + atmel_i2c_send_stop(i2c); + atmel_i2c_wait_txcomp(i2c); + + return TEE_SUCCESS; +} + +static TEE_Result atmel_i2c_recv_one_byte(struct atmel_i2c *i2c, + uint8_t *byte) +{ + uint32_t sr = 0; + + while (true) { + sr = io_read32(i2c->base + TWIHS_SR); + if (sr & TWIHS_SR_NACK) { + EMSG("I2C received NACK while reading"); + return TEE_ERROR_GENERIC; + } + if (sr & TWIHS_SR_RXRDY) + break; + } + + *byte = io_read32(i2c->base + TWIHS_RHR); + + return TEE_SUCCESS; +} + +static TEE_Result atmel_i2c_read_data(struct i2c_dev *i2c_dev, uint8_t *buf, + size_t len) +{ + size_t i = 0; + TEE_Result res = TEE_ERROR_GENERIC; + struct atmel_i2c *i2c = atmel_i2c_from_i2c_ctrl(i2c_dev->ctrl); + uint32_t mmr = TWIHS_MMR_MREAD | SHIFT_U32(i2c_dev->addr, + TWIHS_MMR_DADR_SHIFT); + + io_write32(i2c->base + TWIHS_MMR, mmr); + + atmel_i2c_send_start(i2c); + + for (i = 0; i < len; i++) { + if (i == len - 1) + atmel_i2c_send_stop(i2c); + + res = atmel_i2c_recv_one_byte(i2c, &buf[i]); + if (res) + return res; + } + + atmel_i2c_wait_txcomp(i2c); + + return TEE_SUCCESS; +} + +static TEE_Result atmel_i2c_smbus(struct i2c_dev *i2c_dev, + enum i2c_smbus_dir dir, + enum i2c_smbus_protocol proto __unused, + uint8_t cmd_code, + uint8_t buf[I2C_SMBUS_MAX_BUF_SIZE], + size_t len) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + /* Send command code first */ + res = atmel_i2c_write_data_no_stop(i2c_dev, &cmd_code, 1); + if (res) + return res; + + if (dir == I2C_SMBUS_READ) + return atmel_i2c_read_data(i2c_dev, buf, len); + else + return atmel_i2c_write_data(i2c_dev, buf, len); +} + +static unsigned int flsi(unsigned int val) +{ + if (val == 0) + return 0; + + return sizeof(unsigned int) * 8 - __builtin_clz(val); +} + +static TEE_Result atmel_i2c_init_clk(struct atmel_i2c *i2c) +{ + long div = 0; + long hold = 0; + uint32_t cwgr = 0; + uint32_t cxdiv = 0; + uint32_t ckdiv = 0; + unsigned long clk = clk_get_rate(i2c->clk); + + /* + * Since we will configure both CHDIV and CLDIV with the same value + * use 2 * clk + */ + div = UDIV_ROUND_NEAREST(clk, 2 * I2C_BUS_FREQ) - 3; + if (div < 0) + div = 0; + + /* CHDIV/CLDIV are on 8 bits, CKDIV on 3 bits */ + ckdiv = flsi(div >> 8); + if (ckdiv > TWIHS_CWGR_CKDIV_MAX) { + EMSG("CKDIV value too large"); + return TEE_ERROR_BAD_PARAMETERS; + } + cxdiv = div >> ckdiv; + + if (i2c->sda_hold_time) { + /* hold_time = (HOLD + 3) x tperipheral clock */ + hold = UDIV_ROUND_NEAREST(i2c->sda_hold_time * clk, 1000000000); + hold -= 3; + if (hold < 0 || hold > TWIHS_CWGR_HOLD_MAX) { + EMSG("Incorrect hold value"); + return TEE_ERROR_BAD_PARAMETERS; + } + + cwgr |= hold << TWIHS_CWGR_HOLD_SHIFT; + } + + cwgr |= ckdiv << TWIHS_CWGR_CKDIV_SHIFT; + /* CHDIV == CLDIV */ + cwgr |= cxdiv << TWIHS_CWGR_CHDIV_SHIFT; + cwgr |= cxdiv; + io_write32(i2c->base + TWIHS_CWGR, cwgr); + + return TEE_SUCCESS; +} + +static TEE_Result atmel_i2c_init_hw(struct atmel_i2c *i2c) +{ + /* Unlock TWIHS IP */ + io_write32(i2c->base + TWIHS_WPMR, TWIHS_WPMR_WPKEY); + + /* Configure master mode */ + io_write32(i2c->base + TWIHS_CR, TWIHS_CR_SWRST); + + io_write32(i2c->base + TWIHS_CR, TWIHS_CR_SVDIS); + io_write32(i2c->base + TWIHS_CR, TWIHS_CR_MSEN); + + return atmel_i2c_init_clk(i2c); +} + +static const struct i2c_ctrl_ops atmel_i2c_ops = { + .read = atmel_i2c_read_data, + .write = atmel_i2c_write_data, + .smbus = atmel_i2c_smbus, +}; + +static TEE_Result atmel_i2c_get_dt_i2c(struct dt_pargs *args, void *data, + struct i2c_dev **out_device) +{ + struct i2c_dev *i2c_dev = NULL; + struct i2c_ctrl *i2c_ctrl = data; + + i2c_dev = i2c_create_dev(i2c_ctrl, args->fdt, args->phandle_node); + if (!i2c_dev) + return TEE_ERROR_OUT_OF_MEMORY; + + *out_device = i2c_dev; + + return TEE_SUCCESS; +} + +static TEE_Result atmel_i2c_node_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + size_t size = 0; + const uint32_t *cuint = 0; + unsigned int matrix_id = 0; + struct i2c_ctrl *i2c_ctrl = NULL; + struct atmel_i2c *atmel_i2c = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + int status = fdt_get_status(fdt, node); + + if (status != DT_STATUS_OK_SEC) + return TEE_SUCCESS; + + atmel_i2c = calloc(1, sizeof(struct atmel_i2c)); + if (!atmel_i2c) + return TEE_ERROR_OUT_OF_MEMORY; + + i2c_ctrl = &atmel_i2c->i2c_ctrl; + i2c_ctrl->ops = &atmel_i2c_ops; + + res = clk_dt_get_by_index(fdt, node, 0, &atmel_i2c->clk); + if (res) + goto err_free; + + res = matrix_dt_get_id(fdt, node, &matrix_id); + if (res) + goto err_free; + + if (dt_map_dev(fdt, node, &atmel_i2c->base, &size, DT_MAP_AUTO) < 0) { + res = TEE_ERROR_GENERIC; + goto err_free; + } + + matrix_configure_periph_secure(matrix_id); + + cuint = fdt_getprop(fdt, node, "i2c-sda-hold-time-ns", NULL); + if (cuint) + atmel_i2c->sda_hold_time = fdt32_to_cpu(*cuint); + + clk_enable(atmel_i2c->clk); + + res = atmel_i2c_init_hw(atmel_i2c); + if (res) + goto err_clk_disable; + + res = i2c_register_provider(fdt, node, atmel_i2c_get_dt_i2c, i2c_ctrl); + if (res) + goto err_clk_disable; + + return TEE_SUCCESS; + +err_clk_disable: + clk_disable(atmel_i2c->clk); +err_free: + free(atmel_i2c); + + return res; +} + +static const struct dt_device_match atmel_i2c_match_table[] = { + { .compatible = "atmel,sama5d2-i2c" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_i2c_dt_driver) = { + .name = "atmel_i2c", + .type = DT_DRIVER_NOTYPE, + .match_table = atmel_i2c_match_table, + .probe = atmel_i2c_node_probe, +}; diff --git a/optee_os/core/drivers/i2c/i2c.c b/optee_os/core/drivers/i2c/i2c.c new file mode 100644 index 0000000..37ca1f2 --- /dev/null +++ b/optee_os/core/drivers/i2c/i2c.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2023 Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +struct i2c_dev *i2c_create_dev(struct i2c_ctrl *i2c_ctrl, const void *fdt, + int node) +{ + struct i2c_dev *i2c_dev = NULL; + paddr_t addr = fdt_reg_base_address(fdt, node); + + if (addr == DT_INFO_INVALID_REG) + return NULL; + + i2c_dev = calloc(1, sizeof(struct i2c_dev)); + if (!i2c_dev) + return NULL; + + i2c_dev->addr = addr; + i2c_dev->ctrl = i2c_ctrl; + + return i2c_dev; +} diff --git a/optee_os/core/drivers/i2c/sub.mk b/optee_os/core/drivers/i2c/sub.mk new file mode 100644 index 0000000..e2a28bc --- /dev/null +++ b/optee_os/core/drivers/i2c/sub.mk @@ -0,0 +1,2 @@ +srcs-y += i2c.c +srcs-$(CFG_ATMEL_I2C) += atmel_i2c.c \ No newline at end of file diff --git a/optee_os/core/drivers/imx/dcp/dcp.c b/optee_os/core/drivers/imx/dcp/dcp.c new file mode 100644 index 0000000..34324b9 --- /dev/null +++ b/optee_os/core/drivers/imx/dcp/dcp.c @@ -0,0 +1,776 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const uint8_t sha1_null_msg[] = { + 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, + 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09, +}; + +static const uint8_t sha256_null_msg[] = { + 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, + 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, + 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55, +}; + +static vaddr_t dcp_base; +static bool driver_initialized; +static unsigned int clk_refcount; +static unsigned int key_store_spinlock = SPINLOCK_UNLOCK; +static unsigned int clock_spinlock = SPINLOCK_UNLOCK; +static struct dcp_align_buf hw_context_buffer; + +static struct mutex lock_channel[DCP_NB_CHANNELS] = { + [DCP_CHANN0] = MUTEX_INITIALIZER, + [DCP_CHANN1] = MUTEX_INITIALIZER, + [DCP_CHANN2] = MUTEX_INITIALIZER, + [DCP_CHANN3] = MUTEX_INITIALIZER, +}; + +static const struct dcp_hashalg hash_alg[2] = { + [DCP_SHA1] = { + .type = DCP_CONTROL1_HASH_SELECT_SHA1, + .size = TEE_SHA1_HASH_SIZE, + }, + [DCP_SHA256] = { + .type = DCP_CONTROL1_HASH_SELECT_SHA256, + .size = TEE_SHA256_HASH_SIZE, + }, +}; + +/* + * Enable/disable DCP clock. + * + * @enable Enable the clock if true, disable if false. + */ +static void dcp_clk_enable(bool enable) +{ + vaddr_t ccm_base = core_mmu_get_va(CCM_BASE, MEM_AREA_IO_SEC, + CCM_CCGR0 + sizeof(uint32_t)); + uint32_t clock_except = cpu_spin_lock_xsave(&clock_spinlock); + + if (enable) { + if (clk_refcount > 0) { + clk_refcount++; + goto out; + } else { + clk_refcount++; + io_setbits32(ccm_base + CCM_CCGR0, DCP_CLK_ENABLE_MASK); + } + } else { + assert(clk_refcount != 0); + + clk_refcount--; + if (clk_refcount > 0) + goto out; + else + io_clrbits32(ccm_base + CCM_CCGR0, DCP_CLK_ENABLE_MASK); + } +out: + cpu_spin_unlock_xrestore(&clock_spinlock, clock_except); +} + +/* + * Lock the given channel with a mutex. + * + * @chan DCP channel to lock + */ +static TEE_Result dcp_lock_known_channel(enum dcp_channel chan) +{ + if (mutex_trylock(&lock_channel[chan])) + return TEE_SUCCESS; + else + return TEE_ERROR_BUSY; +} + +/* + * Lock a DCP channel + * + * @channel Pointer on operation channel parameter + */ +static TEE_Result dcp_lock_channel(enum dcp_channel *channel) +{ + TEE_Result ret = TEE_ERROR_BUSY; + enum dcp_channel chan = DCP_CHANN0; + + for (chan = DCP_CHANN0; chan < DCP_NB_CHANNELS; chan++) { + ret = dcp_lock_known_channel(chan); + if (ret == TEE_SUCCESS) { + *channel = chan; + return ret; + } + } + + EMSG("All channels are busy"); + + return ret; +} + +/* + * Unlock the given channel. + * + * @chan DCP channel to unlock + */ +static void dcp_unlock_channel(enum dcp_channel chan) +{ + mutex_unlock(&lock_channel[chan]); +} + +/* + * Start the DCP operation. + * + * @dcp_data Structure containing dcp_descriptor configuration and channel to + * use. + */ +static TEE_Result dcp_run(struct dcp_data *dcp_data) +{ + TEE_Result ret = TEE_SUCCESS; + unsigned int timeout = 0; + uint32_t val = 0; + + dcp_data->desc.next = 0; + cache_operation(TEE_CACHEFLUSH, &dcp_data->desc, + sizeof(dcp_data->desc)); + + /* Enable clock if it's not done */ + dcp_clk_enable(true); + + /* Clear DCP_STAT IRQ field for the channel used by the operation */ + io_clrbits32(dcp_base + DCP_STAT, BIT32(dcp_data->channel)); + + /* Clear CH_N_STAT to clear IRQ and error codes */ + io_write32(dcp_base + DCP_CH_N_STAT(dcp_data->channel), 0x0); + + /* Update descriptor structure to be processed for the channel */ + io_write32(dcp_base + DCP_CH_N_CMDPTR(dcp_data->channel), + virt_to_phys(&dcp_data->desc)); + + /* Increment the semaphore to start the transfer */ + io_write32(dcp_base + DCP_CH_N_SEMA(dcp_data->channel), 0x1); + + for (timeout = 0; timeout < DCP_MAX_TIMEOUT; timeout++) { + dcp_udelay(10); + val = io_read32(dcp_base + DCP_STAT); + if (val & BIT(dcp_data->channel)) + break; + } + + if (timeout == DCP_MAX_TIMEOUT) { + EMSG("Timeout elapsed before operation"); + ret = TEE_ERROR_GENERIC; + goto out; + } + + val = io_read32(dcp_base + DCP_CH_N_STAT(dcp_data->channel)); + if (val & DCP_CH_STAT_ERROR_MASK) { + EMSG("Error operation, 0x%" PRIx32, val); + ret = TEE_ERROR_GENERIC; + } + +out: + dcp_clk_enable(false); + + return ret; +} + +static TEE_Result dcp_cmac_subkey_generation(struct dcp_cipher_init *init, + uint8_t *k1, uint8_t *k2) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct dcp_cipher_data data = { }; + uint8_t l[16] = { }; + uint8_t tmp[16] = { }; + uint8_t const_zero[16] = { }; + uint8_t const_rb[16] = { [15] = 0x87 }; + + ret = dcp_cipher_do_init(&data, init); + if (ret != TEE_SUCCESS) + return ret; + + ret = dcp_cipher_do_update(&data, const_zero, l, sizeof(l)); + if (ret != TEE_SUCCESS) + goto out; + + if ((l[0] & BIT(7)) == 0) { + dcp_left_shift_buffer(l, k1, 16); + } else { + dcp_left_shift_buffer(l, tmp, 16); + dcp_xor(tmp, const_rb, k1, 16); + } + + if ((k1[0] & BIT(7)) == 0) { + dcp_left_shift_buffer(k1, k2, 16); + } else { + dcp_left_shift_buffer(k1, tmp, 16); + dcp_xor(tmp, const_rb, k2, 16); + } + + ret = TEE_SUCCESS; +out: + dcp_cipher_do_final(&data); + + return ret; +} + +TEE_Result dcp_store_key(uint32_t *key, unsigned int index) +{ + uint32_t val = 0; + unsigned int i = 0; + uint32_t key_store_except = 0; + + if (!key) + return TEE_ERROR_BAD_PARAMETERS; + + if (index > DCP_SRAM_KEY_NB_SUBWORD - 1) { + EMSG("Bad parameters, index must be < %u", + DCP_SRAM_KEY_NB_SUBWORD); + return TEE_ERROR_BAD_PARAMETERS; + } + + key_store_except = cpu_spin_lock_xsave(&key_store_spinlock); + + dcp_clk_enable(true); + + val = DCP_SRAM_KEY_INDEX(index); + io_write32(dcp_base + DCP_KEY, val); + + /* + * Key is stored as four uint32 values, starting with subword0 + * (least-significant word) + */ + for (i = 0; i < DCP_SRAM_KEY_NB_SUBWORD; i++) { + val = TEE_U32_TO_BIG_ENDIAN(key[i]); + io_write32(dcp_base + DCP_KEYDATA, val); + } + + dcp_clk_enable(false); + + cpu_spin_unlock_xrestore(&key_store_spinlock, key_store_except); + + return TEE_SUCCESS; +} + +TEE_Result dcp_cmac(struct dcp_cipher_init *init, uint8_t *input, + size_t input_size, uint8_t *output) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + uint8_t key1[DCP_AES128_KEY_SIZE] = { }; + uint8_t key2[DCP_AES128_KEY_SIZE] = { }; + unsigned int nb_blocks = 0; + bool block_complete = false; + struct dcp_cipher_data data = { }; + uint8_t y[DCP_AES128_BLOCK_SIZE] = { }; + uint8_t x[DCP_AES128_BLOCK_SIZE] = { }; + uint8_t last[DCP_AES128_BLOCK_SIZE] = { }; + unsigned int i = 0; + uint8_t offset = 0; + + if (!output || !init) + return TEE_ERROR_BAD_PARAMETERS; + + if (!input && input_size) + return TEE_ERROR_BAD_PARAMETERS; + + ret = dcp_cipher_do_init(&data, init); + if (ret != TEE_SUCCESS) { + ret = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* Generate CMAC subkeys */ + ret = dcp_cmac_subkey_generation(init, key1, key2); + if (ret != TEE_SUCCESS) + goto out; + + /* Get number of block */ + nb_blocks = ROUNDUP(input_size, DCP_AES128_BLOCK_SIZE) / + DCP_AES128_BLOCK_SIZE; + + block_complete = nb_blocks && !(input_size % DCP_AES128_BLOCK_SIZE); + if (nb_blocks == 0) + nb_blocks = 1; + + for (i = 0; i < nb_blocks - 1; i++) { + dcp_xor(x, input + offset, y, DCP_AES128_BLOCK_SIZE); + ret = dcp_cipher_do_update(&data, y, x, + DCP_AES128_BLOCK_SIZE); + if (ret) + goto out; + offset += DCP_AES128_BLOCK_SIZE; + } + + /* Process the last block */ + memcpy(last, input + offset, input_size - offset); + + if (block_complete) { + dcp_xor(last, key1, last, DCP_AES128_BLOCK_SIZE); + } else { + dcp_cmac_padding(last, input_size % DCP_AES128_BLOCK_SIZE); + dcp_xor(last, key2, last, DCP_AES128_BLOCK_SIZE); + } + + dcp_xor(x, last, y, DCP_AES128_BLOCK_SIZE); + ret = dcp_cipher_do_update(&data, y, x, + DCP_AES128_BLOCK_SIZE); + if (ret) + goto out; + + memcpy(output, x, DCP_AES128_BLOCK_SIZE); + +out: + dcp_cipher_do_final(&data); + + return ret; +} + +TEE_Result dcp_cipher_do_init(struct dcp_cipher_data *data, + struct dcp_cipher_init *init) +{ + struct dcp_descriptor *desc = NULL; + TEE_Result ret = TEE_ERROR_GENERIC; + + if (!init || !data) + return TEE_ERROR_BAD_PARAMETERS; + + ret = dcp_lock_channel(&data->dcp_data.channel); + if (ret != TEE_SUCCESS) + return ret; + + desc = &data->dcp_data.desc; + + desc->ctrl0 = DCP_CONTROL0_DECR_SEMAPHORE | DCP_CONTROL0_ENABLE_CIPHER | + DCP_CONTROL0_INTERRUPT_ENABLE; + desc->ctrl1 = DCP_CONTROL1_CIPHER_SELECT_AES128; + + if (init->op == DCP_ENCRYPT) + desc->ctrl0 |= DCP_CONTROL0_CIPHER_ENCRYPT; + + if (init->key_mode == DCP_OTP) { + desc->ctrl0 &= ~DCP_CONTROL0_OTP_KEY; + desc->ctrl1 |= DCP_CONTROL1_KEY_SELECT_OTP_CRYPTO; + } else if (init->key_mode == DCP_PAYLOAD) { + desc->ctrl0 |= DCP_CONTROL0_PAYLOAD_KEY; + if (!init->key) + return TEE_ERROR_BAD_PARAMETERS; + memcpy(data->key, init->key, DCP_AES128_KEY_SIZE); + } else { + desc->ctrl1 |= SHIFT_U32(init->key_mode, 8); + } + + if (init->mode == DCP_CBC) { + desc->ctrl0 |= DCP_CONTROL0_CIPHER_INIT; + desc->ctrl1 |= DCP_CONTROL1_CIPHER_MODE_CBC; + if (!init->iv) + return TEE_ERROR_BAD_PARAMETERS; + memcpy(data->iv, init->iv, DCP_AES128_IV_SIZE); + } + + /* Allocate aligned buffer for dcp iv and key */ + ret = dcp_calloc_align_buf(&data->payload, + DCP_AES128_IV_SIZE + DCP_AES128_KEY_SIZE); + if (ret != TEE_SUCCESS) + return ret; + + desc->src_buffer = 0; + desc->dest_buffer = 0; + desc->status = 0; + desc->buff_size = 0; + desc->next = virt_to_phys(desc); + + data->initialized = true; + + return ret; +} + +TEE_Result dcp_cipher_do_update(struct dcp_cipher_data *data, + const uint8_t *src, uint8_t *dst, size_t size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct dcp_align_buf output = { }; + struct dcp_align_buf input = { }; + struct dcp_descriptor *desc = NULL; + + if (!data || !src || !dst) + return TEE_ERROR_BAD_PARAMETERS; + + if (!data->initialized) { + EMSG("Error, please call dcp_aes_do_init() before"); + return TEE_ERROR_BAD_STATE; + } + + if (size % DCP_AES128_BLOCK_SIZE) { + EMSG("Input size has to be a multiple of %zu bytes", + DCP_AES128_BLOCK_SIZE); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = dcp_calloc_align_buf(&output, size); + if (ret != TEE_SUCCESS) + goto out; + + ret = dcp_calloc_align_buf(&input, size); + if (ret != TEE_SUCCESS) + goto out; + + desc = &data->dcp_data.desc; + + /* Copy input data */ + memcpy(input.data, src, size); + + /* Copy key and IV */ + memcpy(data->payload.data, data->key, DCP_AES128_KEY_SIZE); + data->payload_size = DCP_AES128_KEY_SIZE; + if (desc->ctrl0 & DCP_CONTROL0_CIPHER_INIT) { + memcpy(data->payload.data + DCP_AES128_KEY_SIZE, data->iv, + DCP_AES128_IV_SIZE); + data->payload_size += DCP_AES128_IV_SIZE; + } + + desc->src_buffer = input.paddr; + desc->dest_buffer = output.paddr; + desc->payload = data->payload.paddr; + desc->buff_size = size; + + cache_operation(TEE_CACHECLEAN, data->payload.data, + data->payload_size); + cache_operation(TEE_CACHECLEAN, input.data, size); + cache_operation(TEE_CACHEINVALIDATE, output.data, size); + + ret = dcp_run(&data->dcp_data); + if (ret) + goto out; + + cache_operation(TEE_CACHEINVALIDATE, output.data, size); + + desc->ctrl0 &= ~DCP_CONTROL0_CIPHER_INIT; + + memcpy(dst, output.data, size); +out: + dcp_free(&output); + dcp_free(&input); + + return ret; +} + +void dcp_cipher_do_final(struct dcp_cipher_data *data) +{ + if (data) + data->initialized = false; + + dcp_free(&data->payload); + dcp_unlock_channel(data->dcp_data.channel); +} + +TEE_Result dcp_sha_do_init(struct dcp_hash_data *hashdata) +{ + struct dcp_descriptor *desc = NULL; + TEE_Result ret = TEE_ERROR_GENERIC; + + if (!hashdata) { + EMSG("Bad parameters, hashdata is NULL"); + return TEE_ERROR_BAD_PARAMETERS; + } + + desc = &hashdata->dcp_data.desc; + + /* DCP descriptor init */ + desc->status = 0; + desc->payload = 0; + desc->dest_buffer = 0; + desc->ctrl0 = DCP_CONTROL0_ENABLE_HASH | DCP_CONTROL0_INTERRUPT_ENABLE | + DCP_CONTROL0_DECR_SEMAPHORE | DCP_CONTROL0_HASH_INIT; + desc->ctrl1 = hash_alg[hashdata->alg].type; + desc->buff_size = 0; + desc->next = 0; + desc->src_buffer = 0; + + ret = dcp_lock_channel(&hashdata->dcp_data.channel); + if (ret != TEE_SUCCESS) { + EMSG("Channel is busy, can't start operation now"); + return ret; + } + + /* Allocate context data */ + ret = dcp_calloc_align_buf(&hashdata->ctx, DCP_SHA_BLOCK_SIZE); + if (ret != TEE_SUCCESS) + return ret; + + hashdata->initialized = true; + hashdata->ctx_size = 0; + + return ret; +} + +TEE_Result dcp_sha_do_update(struct dcp_hash_data *hashdata, + const uint8_t *data, size_t len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + struct dcp_descriptor *desc = NULL; + struct dcp_align_buf input = { }; + uint32_t offset = 0; + uint32_t nb_blocks = 0; + size_t size_todo = 0; + size_t size_left = 0; + size_t size_total = 0; + + if (!hashdata || !data || !len) + return TEE_ERROR_BAD_PARAMETERS; + + if (!hashdata->initialized) { + EMSG("hashdata is uninitialized"); + return TEE_ERROR_BAD_STATE; + } + + /* Get number of blocks */ + if (ADD_OVERFLOW(hashdata->ctx_size, len, &size_total)) + return TEE_ERROR_BAD_PARAMETERS; + + nb_blocks = size_total / DCP_SHA_BLOCK_SIZE; + size_todo = nb_blocks * DCP_SHA_BLOCK_SIZE; + size_left = len - size_todo + hashdata->ctx_size; + desc = &hashdata->dcp_data.desc; + + if (size_todo) { + /* Allocate buffer as input */ + ret = dcp_calloc_align_buf(&input, size_todo); + if (ret != TEE_SUCCESS) + return ret; + + /* Copy previous data if any */ + offset = size_todo - hashdata->ctx_size; + memcpy(input.data, hashdata->ctx.data, hashdata->ctx_size); + memcpy(input.data + hashdata->ctx_size, data, offset); + hashdata->ctx_size = 0; + + desc->src_buffer = input.paddr; + desc->buff_size = size_todo; + + cache_operation(TEE_CACHECLEAN, input.data, size_todo); + + ret = dcp_run(&hashdata->dcp_data); + desc->ctrl0 &= ~DCP_CONTROL0_HASH_INIT; + + dcp_free(&input); + } else { + size_left = len; + offset = 0; + ret = TEE_SUCCESS; + } + + /* Save any data left */ + memcpy(hashdata->ctx.data + hashdata->ctx_size, data + offset, + size_left); + hashdata->ctx_size += size_left; + + return ret; +} + +TEE_Result dcp_sha_do_final(struct dcp_hash_data *hashdata, uint8_t *digest, + size_t digest_size) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t payload_size = 0; + struct dcp_descriptor *desc = NULL; + struct dcp_align_buf payload = { }; + + if (!hashdata || !digest) + return TEE_ERROR_BAD_PARAMETERS; + + if (!hashdata->initialized) { + EMSG("hashdata is uninitialized"); + return TEE_ERROR_BAD_STATE; + } + + if (digest_size < hash_alg[hashdata->alg].size) { + EMSG("Digest buffer size is to small, should be %" PRId32, + hash_alg[hashdata->alg].size); + return TEE_ERROR_BAD_PARAMETERS; + } + + desc = &hashdata->dcp_data.desc; + payload_size = hash_alg[hashdata->alg].size; + + /* Handle the case where the input message is NULL */ + if ((desc->ctrl0 & DCP_CONTROL0_HASH_INIT) && hashdata->ctx_size == 0) { + if (hashdata->alg == DCP_SHA1) + memcpy(digest, sha1_null_msg, payload_size); + if (hashdata->alg == DCP_SHA256) + memcpy(digest, sha256_null_msg, payload_size); + ret = TEE_SUCCESS; + } else { + /* Allocate buffer for the digest */ + ret = dcp_calloc_align_buf(&payload, payload_size); + if (ret != TEE_SUCCESS) + return ret; + + /* Set work packet for last iteration */ + desc->ctrl0 |= DCP_CONTROL0_HASH_TERM; + desc->src_buffer = hashdata->ctx.paddr; + desc->buff_size = hashdata->ctx_size; + desc->payload = payload.paddr; + + cache_operation(TEE_CACHECLEAN, hashdata->ctx.data, + hashdata->ctx_size); + cache_operation(TEE_CACHEINVALIDATE, payload.data, + payload_size); + + ret = dcp_run(&hashdata->dcp_data); + + /* Copy the result */ + cache_operation(TEE_CACHEINVALIDATE, payload.data, + payload_size); + /* DCP payload result is flipped */ + dcp_reverse(payload.data, digest, payload_size); + + dcp_free(&payload); + } + + dcp_free(&hashdata->ctx); + + /* Reset hashdata strcuture */ + hashdata->initialized = false; + + dcp_unlock_channel(hashdata->dcp_data.channel); + + return ret; +} + +void dcp_disable_unique_key(void) +{ + dcp_clk_enable(true); + io_setbits32(dcp_base + DCP_CAPABILITY0, + DCP_CAPABILITY0_DISABLE_UNIQUE_KEY); + dcp_clk_enable(false); +} + +#ifdef CFG_DT +static const char *const dt_ctrl_match_table[] = { + "fsl,imx28-dcp", + "fsl,imx6sl-dcp", +}; + +/* + * Fetch DCP base address from DT + * + * @base [out] DCP base address + */ +static TEE_Result dcp_pbase(paddr_t *base) +{ + void *fdt = NULL; + int node = -1; + unsigned int i = 0; + + fdt = get_dt(); + if (!fdt) { + EMSG("DTB no present"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + for (i = 0; i < ARRAY_SIZE(dt_ctrl_match_table); i++) { + node = fdt_node_offset_by_compatible(fdt, 0, + dt_ctrl_match_table[i]); + if (node >= 0) + break; + } + + if (node < 0) { + EMSG("DCP node not found err = %d", node); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + if (fdt_get_status(fdt, node) == DT_STATUS_DISABLED) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* Force secure-status = "okay" and status="disabled" */ + if (dt_enable_secure_status(fdt, node)) { + EMSG("Not able to set DCP Control DTB entry secure"); + return TEE_ERROR_NOT_SUPPORTED; + } + + *base = fdt_reg_base_address(fdt, node); + if (*base == DT_INFO_INVALID_REG) { + EMSG("Unable to get the DCP Base address"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + return TEE_SUCCESS; +} +#endif /* CFG_DT */ + +TEE_Result dcp_init(void) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + paddr_t pbase = 0; + + if (driver_initialized) + return TEE_SUCCESS; + + dcp_clk_enable(true); + + ret = dcp_pbase(&pbase); + if (ret != TEE_SUCCESS) + pbase = DCP_BASE; + + dcp_base = core_mmu_get_va(pbase, MEM_AREA_IO_SEC, DCP_CONTEXT + + sizeof(uint32_t)); + if (!dcp_base) { + EMSG("Unable to get DCP physical address"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + /* Context switching buffer memory allocation */ + ret = dcp_calloc_align_buf(&hw_context_buffer, DCP_CONTEXT_BUFFER_SIZE); + if (ret != TEE_SUCCESS) { + EMSG("hw_context_buffer allocation failed"); + return ret; + } + + /* + * Reset the DCP before initialization. Depending on the SoC lifecycle + * state, the DCP needs to be reset to reload the OTP master key from + * the SNVS. + */ + io_write32(dcp_base + DCP_CTRL_SET, DCP_CTRL_SFTRST | DCP_CTRL_CLKGATE); + + /* + * Initialize control register. + * Enable normal DCP operation (SFTRST & CLKGATE bits set to 0) + */ + io_write32(dcp_base + DCP_CTRL_CLR, DCP_CTRL_SFTRST | DCP_CTRL_CLKGATE); + + io_write32(dcp_base + DCP_CTRL_SET, + DCP_CTRL_GATHER_RESIDUAL_WRITES | + DCP_CTRL_ENABLE_CONTEXT_SWITCHING); + + /* Enable all DCP channels */ + io_write32(dcp_base + DCP_CHANNELCTRL, + DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK); + + /* Clear DCP_STAT register */ + io_write32(dcp_base + DCP_STAT_CLR, DCP_STAT_CLEAR); + + /* Copy context switching buffer address in DCP_CONTEXT register */ + io_write32(dcp_base + DCP_CONTEXT, (uint32_t)hw_context_buffer.paddr); + + driver_initialized = true; + + dcp_clk_enable(false); + + return ret; +} diff --git a/optee_os/core/drivers/imx/dcp/dcp_huk.c b/optee_os/core/drivers/imx/dcp/dcp_huk.c new file mode 100644 index 0000000..ca79ff7 --- /dev/null +++ b/optee_os/core/drivers/imx/dcp/dcp_huk.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020 NXP + */ + +#include +#include +#include +#include + +#define HUK_MESSAGE_NULL_BYTE 0 +#define NB_ITERATION_HUK 1 +#define HUK_SIZE_BITS 128 + +/* State of the generated HUK */ +enum dcp_huk_state { + DCP_HUK_EMPTY = 0, + DCP_HUK_GENERATED, + DCP_HUK_ERROR, +}; + +/* Information about HUK */ +static struct { + enum dcp_huk_state state; + uint8_t data[HW_UNIQUE_KEY_LENGTH]; +} dcp_huk = { .state = DCP_HUK_EMPTY }; + +/* + * Generate Hardware Unique Key using the Data Co-Processor (DCP) AES128-CMAC + * cryptographic operation + * Follow dcp_aes_cmac() message format + * + * @hwkey [out] Hardware Unique Key private data + */ +static TEE_Result dcp_generate_huk(struct tee_hw_unique_key *hwkey) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct dcp_cipher_init init = { + .key_mode = DCP_OTP, + .mode = DCP_ECB, + .op = DCP_ENCRYPT, + }; + uint8_t content[DCP_AES128_BLOCK_SIZE] = { NB_ITERATION_HUK, + 'h', + 'u', + 'k', + HUK_MESSAGE_NULL_BYTE, + 'o', + 'p', + 't', + 'e', + 'e', + 'o', + 's', + 'd', + 'c', + 'p', + HUK_SIZE_BITS }; + + res = dcp_cmac(&init, content, DCP_AES128_BLOCK_SIZE, hwkey->data); + + dcp_disable_unique_key(); + + return res; +} + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + if (!hwkey) { + EMSG("HUK generation failed, hwkey structure is NULL"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = dcp_init(); + if (ret != TEE_SUCCESS) { + dcp_huk.state = DCP_HUK_ERROR; + return ret; + } + + if (dcp_huk.state == DCP_HUK_EMPTY) { + ret = dcp_generate_huk(hwkey); + if (ret != TEE_SUCCESS) { + dcp_huk.state = DCP_HUK_ERROR; + } else { + memcpy(dcp_huk.data, hwkey->data, HW_UNIQUE_KEY_LENGTH); + dcp_huk.state = DCP_HUK_GENERATED; + } + } else if (dcp_huk.state == DCP_HUK_GENERATED) { + memcpy(hwkey->data, dcp_huk.data, HW_UNIQUE_KEY_LENGTH); + ret = TEE_SUCCESS; + } else { + ret = TEE_ERROR_GENERIC; + } + + return ret; +} diff --git a/optee_os/core/drivers/imx/dcp/dcp_utils.c b/optee_os/core/drivers/imx/dcp/dcp_utils.c new file mode 100644 index 0000000..9d21d7a --- /dev/null +++ b/optee_os/core/drivers/imx/dcp/dcp_utils.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020 NXP + */ +#include +#include +#include +#include +#include +#include +#include + +TEE_Result dcp_calloc_align_buf(struct dcp_align_buf *buf, size_t size) +{ + if (!buf) { + EMSG("Error, buf is null"); + return TEE_ERROR_BAD_PARAMETERS; + } + + buf->data = alloc_cache_aligned(size); + if (!buf->data) + return TEE_ERROR_OUT_OF_MEMORY; + + buf->paddr = virt_to_phys(buf->data); + + if (!buf->paddr) { + dcp_free(buf); + return TEE_ERROR_OUT_OF_MEMORY; + } + + buf->size = size; + + return TEE_SUCCESS; +} + +void dcp_free(struct dcp_align_buf *buf) +{ + free(buf->data); +} + +void dcp_left_shift_buffer(uint8_t *input, uint8_t *result, size_t buffer_size) +{ + unsigned int i = 0; + uint8_t overflow = 0; + + /* For each byte */ + for (i = 0; i < buffer_size; i++) { + /* Left shift a bytes by one */ + result[buffer_size - 1 - i] = + input[buffer_size - 1 - i] << 1 | overflow; + + overflow = input[buffer_size - 1 - i] >> 7; + } +} + +void dcp_udelay(uint32_t time) +{ + uint32_t counter = time * 500; + + /* Implementation of a Software loop assuming CPU clock of 500MHz */ + while (counter--) { + isb(); + dsb(); + }; +} + +void dcp_reverse(uint8_t *in, uint8_t *out, size_t size) +{ + unsigned int i = 0; + + for (i = 0; i < size; i++) + out[i] = in[size - 1 - i]; +} + +void dcp_xor(uint8_t *a, uint8_t *b, uint8_t *out, size_t size) +{ + unsigned int i = 0; + + for (i = 0; i < size; i++) + out[i] = a[i] ^ b[i]; +} + +void dcp_cmac_padding(uint8_t *block, size_t len) +{ + unsigned int i = 0; + + for (i = len; i < DCP_AES128_BLOCK_SIZE; i++) { + if (i == len) + block[i] = BIT(7); + else + block[i] = 0x0; + } +} diff --git a/optee_os/core/drivers/imx/dcp/include/dcp_utils.h b/optee_os/core/drivers/imx/dcp/include/dcp_utils.h new file mode 100644 index 0000000..c96c501 --- /dev/null +++ b/optee_os/core/drivers/imx/dcp/include/dcp_utils.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020 NXP + */ +#ifndef __DCP_UTILS_H__ +#define __DCP_UTILS_H__ + +#include +#include + +/* Adjust index value for writing in register */ +#define DCP_SRAM_KEY_INDEX(idx) SHIFT_U32(idx, 4) + +/* Calculate context switching buffer offset */ +#define DCP_CONTEXT_SW_OFFSET(chann) ((DCP_NB_CHANNELS - 1 - (chann)) * 52) + +/* + * Allocate internal driver buffer aligned with a cache line and initialize it + * with 0s + * + * @buf [out] Buffer to allocate + * @size Size in bytes of the memory to allocate + */ +TEE_Result dcp_calloc_align_buf(struct dcp_align_buf *buf, size_t size); + +/* + * Free allocated memory + * + * @buf buffer to free. + */ +void dcp_free(struct dcp_align_buf *buf); + +/* + * Left shifting a multi bytes buffer by one bit + * + * @result [out] Buffer containing the result of the operation + * @input Input buffer for the operation + * @buffer_size Size of the buffer in bytes + */ +void dcp_left_shift_buffer(uint8_t *result, uint8_t *input, size_t buffer_size); + +/* + * Wait given microsecond + * + * @time Time in microsecond + */ +void dcp_udelay(uint32_t time); + +/* + * Copies elements from a buffer to another one. These elements are copied in + * reverse order. + * + * @in input buffer + * @out output buffer + * @size bytes to copy + */ +void dcp_reverse(uint8_t *in, uint8_t *out, size_t size); + +/* + * Operate a XOR between two same size buffers + * + * @a Input buffer to XOR + * @b Input buffer to XOR + * @out Result of the XOR operation + * @size Size of input and output buffers + */ +void dcp_xor(uint8_t *a, uint8_t *b, uint8_t *out, size_t size); + +/* + * CMAC padding. + * The padding result is the concatenation of the input buffer block and a + * single '1' followed by the minimum number of '0's to get a 128 bits + * block. + * + * @block Block to pad + * @len Length of the padding + */ +void dcp_cmac_padding(uint8_t *block, size_t len); + +#endif /* __DCP_UTILS_H__ */ diff --git a/optee_os/core/drivers/imx/dcp/include/local.h b/optee_os/core/drivers/imx/dcp/include/local.h new file mode 100644 index 0000000..9d6b312 --- /dev/null +++ b/optee_os/core/drivers/imx/dcp/include/local.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020 NXP + */ +#ifndef __LOCAL_H__ +#define __LOCAL_H__ + +#define DCP_CONTEXT_BUFFER_SIZE 208 +#define DCP_MAX_TIMEOUT 10 +#define DCP_SRAM_KEY_NB_SUBWORD 4 +#define DCP_CLK_ENABLE_MASK GENMASK_32(11, 10) + +#endif /* __LOCAL_H__ */ diff --git a/optee_os/core/drivers/imx/dcp/sub.mk b/optee_os/core/drivers/imx/dcp/sub.mk new file mode 100644 index 0000000..ddc13b7 --- /dev/null +++ b/optee_os/core/drivers/imx/dcp/sub.mk @@ -0,0 +1,5 @@ +srcs-y += dcp_huk.c +srcs-y += dcp.c +srcs-y += dcp_utils.c + +incdirs-y += include diff --git a/optee_os/core/drivers/imx/mu/imx_mu.c b/optee_os/core/drivers/imx/mu/imx_mu.c new file mode 100644 index 0000000..017b97a --- /dev/null +++ b/optee_os/core/drivers/imx/mu/imx_mu.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 NXP + */ +#include +#include +#include +#include +#include +#include + +#include "imx_mu_platform.h" + +#define RX_TIMEOUT (100 * 1000) + +static unsigned int mu_spinlock = SPINLOCK_UNLOCK; + +__weak void imx_mu_plat_init(vaddr_t base __unused) +{ +} + +__weak TEE_Result imx_mu_plat_send(vaddr_t base __unused, + unsigned int index __unused, + uint32_t msg __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +__weak TEE_Result imx_mu_plat_receive(vaddr_t base __unused, + unsigned int index __unused, + uint32_t *msg __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +/* + * Receive a message via the MU + * + * @base: virtual base address of the MU controller + * @[out]msg: message received + */ +static TEE_Result imx_mu_receive_msg(vaddr_t base, struct imx_mu_msg *msg) +{ + TEE_Result res = TEE_ERROR_GENERIC; + unsigned int count = 0; + uint32_t response = 0; + unsigned int nb_channel = 0; + uint64_t tout_rx = timeout_init_us(RX_TIMEOUT); + + assert(base && msg); + + do { + res = imx_mu_plat_receive(base, 0, &response); + if (timeout_elapsed(tout_rx)) + break; + } while (res == TEE_ERROR_NO_DATA); + + if (res) + return res; + + memcpy(&msg->header, &response, sizeof(response)); + + /* Check the size of the message to receive */ + if (msg->header.size > IMX_MU_MSG_SIZE) { + EMSG("Size of the message is > than IMX_MU_MSG_SIZE"); + return TEE_ERROR_BAD_FORMAT; + } + + nb_channel = imx_mu_plat_get_rx_channel(base); + + for (count = 1; count < msg->header.size; count++) { + res = imx_mu_plat_receive(base, count % nb_channel, + &msg->data.u32[count - 1]); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +/* + * Send a message via the MU + * + * @base: virtual base address of the MU controller + * @[in]msg: message to send + */ +static TEE_Result imx_mu_send_msg(vaddr_t base, struct imx_mu_msg *msg) +{ + TEE_Result res = TEE_ERROR_GENERIC; + unsigned int count = 0; + unsigned int nb_channel = 0; + uint32_t word = 0; + + assert(base && msg); + + if (msg->header.size > IMX_MU_MSG_SIZE) { + EMSG("msg->size is > than IMX_MU_MSG_SIZE"); + return TEE_ERROR_BAD_FORMAT; + } + + memcpy(&word, &msg->header, sizeof(uint32_t)); + res = imx_mu_plat_send(base, 0, word); + if (res) + return res; + + nb_channel = imx_mu_plat_get_tx_channel(base); + + for (count = 1; count < msg->header.size; count++) { + res = imx_mu_plat_send(base, count % nb_channel, + msg->data.u32[count - 1]); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +void imx_mu_init(vaddr_t base) +{ + uint32_t exceptions = 0; + + if (!base) { + EMSG("Bad MU base address"); + return; + } + + exceptions = cpu_spin_lock_xsave(&mu_spinlock); + + imx_mu_plat_init(base); + + cpu_spin_unlock_xrestore(&mu_spinlock, exceptions); +} + +TEE_Result imx_mu_call(vaddr_t base, struct imx_mu_msg *msg, + bool wait_for_answer) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t exceptions = 0; + + if (!base || !msg) + return TEE_ERROR_BAD_PARAMETERS; + + exceptions = cpu_spin_lock_xsave(&mu_spinlock); + + res = imx_mu_send_msg(base, msg); + if (res == TEE_SUCCESS && wait_for_answer) + res = imx_mu_receive_msg(base, msg); + + cpu_spin_unlock_xrestore(&mu_spinlock, exceptions); + + return res; +} diff --git a/optee_os/core/drivers/imx/mu/imx_mu_8q.c b/optee_os/core/drivers/imx/mu/imx_mu_8q.c new file mode 100644 index 0000000..6089339 --- /dev/null +++ b/optee_os/core/drivers/imx/mu/imx_mu_8q.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2020-2022 NXP + */ +#include +#include +#include +#include +#include + +#include "imx_mu_platform.h" + +#define MU_ATR(n) (0x0 + (n) * (4)) +#define MU_ARR(n) (0x10 + (n) * (4)) +#define MU_ASR_OFFSET 0x20 +#define MU_ACR_OFFSET 0x24 + +#define MU_SR_RF(n) SHIFT_U32(1, 27 - (n)) +#define MU_SR_TE(n) SHIFT_U32(1, 23 - (n)) + +#define MU_CR_GIE_MASK GENMASK_32(31, 28) +#define MU_CR_RIE_MASK GENMASK_32(27, 24) +#define MU_CR_TIE_MASK GENMASK_32(23, 20) +#define MU_CR_GIR_MASK GENMASK_32(19, 16) +#define MU_CR_F_MASK GENMASK_32(2, 0) + +#define MU_MAX_CHANNEL 4 + +static TEE_Result mu_wait_for(vaddr_t addr, uint32_t mask) +{ + uint64_t timeout = timeout_init_us(1000); + + while (!(io_read32(addr) & mask)) + if (timeout_elapsed(timeout)) + break; + + if (io_read32(addr) & mask) + return TEE_SUCCESS; + else + return TEE_ERROR_BUSY; + + return TEE_SUCCESS; +} + +unsigned int imx_mu_plat_get_rx_channel(vaddr_t base __unused) +{ + return MU_MAX_CHANNEL; +} + +unsigned int imx_mu_plat_get_tx_channel(vaddr_t base __unused) +{ + return MU_MAX_CHANNEL; +} + +void imx_mu_plat_init(vaddr_t base) +{ + io_clrbits32(base + MU_ACR_OFFSET, + MU_CR_GIE_MASK | MU_CR_RIE_MASK | MU_CR_TIE_MASK | + MU_CR_GIR_MASK | MU_CR_F_MASK); +} + +TEE_Result imx_mu_plat_send(vaddr_t base, unsigned int index, uint32_t msg) +{ + assert(index < MU_MAX_CHANNEL); + + /* Wait TX register to be empty */ + if (mu_wait_for(base + MU_ASR_OFFSET, MU_SR_TE(index))) + return TEE_ERROR_BUSY; + + /* Write message in TX register */ + io_write32(base + MU_ATR(index), msg); + + return TEE_SUCCESS; +} + +TEE_Result imx_mu_plat_receive(vaddr_t base, unsigned int index, uint32_t *msg) +{ + assert(index < MU_MAX_CHANNEL); + + /* Wait RX register to be full */ + if (mu_wait_for(base + MU_ASR_OFFSET, MU_SR_RF(index))) + return TEE_ERROR_NO_DATA; + + /* Read message in RX register */ + *msg = io_read32(base + MU_ARR(index)); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/imx/mu/imx_mu_8ulp.c b/optee_os/core/drivers/imx/mu/imx_mu_8ulp.c new file mode 100644 index 0000000..da09dde --- /dev/null +++ b/optee_os/core/drivers/imx/mu/imx_mu_8ulp.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + */ +#include +#include +#include +#include +#include + +#include "imx_mu_platform.h" + +#define MU_PAR 0x004 +#define MU_TCR 0x120 +#define MU_TSR 0x124 +#define MU_RCR 0x128 +#define MU_RSR 0x12C +#define MU_TR(n) (0x200 + 0x4 * (n)) +#define MU_RR(n) (0x280 + 0x4 * (n)) +#define MU_TSR_TE(n) BIT32(n) +#define MU_RSR_RF(n) BIT32(n) + +#define RR_NUM_MASK GENMASK_32(15, 8) +#define RR_NUM_SHIFT 8 +#define TR_NUM_MASK GENMASK_32(7, 0) + +static TEE_Result mu_wait_for(vaddr_t addr, uint32_t mask) +{ + uint64_t timeout = timeout_init_us(1000); + + while (!(io_read32(addr) & mask)) + if (timeout_elapsed(timeout)) + break; + + if (io_read32(addr) & mask) + return TEE_SUCCESS; + else + return TEE_ERROR_BUSY; + + return TEE_SUCCESS; +} + +unsigned int imx_mu_plat_get_rx_channel(vaddr_t base) +{ + return (io_read32(base + MU_PAR) & RR_NUM_MASK) >> RR_NUM_SHIFT; +} + +unsigned int imx_mu_plat_get_tx_channel(vaddr_t base) +{ + return io_read32(base + MU_PAR) & TR_NUM_MASK; +} + +TEE_Result imx_mu_plat_send(vaddr_t base, unsigned int index, uint32_t msg) +{ + assert(index < imx_mu_plat_get_tx_channel(base)); + + /* Wait TX register to be empty */ + if (mu_wait_for(base + MU_TSR, MU_TSR_TE(index))) + return TEE_ERROR_BUSY; + + io_write32(base + MU_TR(index), msg); + + return TEE_SUCCESS; +} + +TEE_Result imx_mu_plat_receive(vaddr_t base, unsigned int index, uint32_t *msg) +{ + assert(index < imx_mu_plat_get_rx_channel(base)); + + /* Wait RX register to be full */ + if (mu_wait_for(base + MU_RSR, MU_RSR_RF(index))) + return TEE_ERROR_NO_DATA; + + *msg = io_read32(base + MU_RR(index)); + + return TEE_SUCCESS; +} + +void imx_mu_plat_init(vaddr_t base) +{ + /* Reset status registers */ + io_write32(base + MU_TCR, 0x0); + io_write32(base + MU_RCR, 0x0); +} diff --git a/optee_os/core/drivers/imx/mu/imx_mu_platform.h b/optee_os/core/drivers/imx/mu/imx_mu_platform.h new file mode 100644 index 0000000..2342f47 --- /dev/null +++ b/optee_os/core/drivers/imx/mu/imx_mu_platform.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 NXP + */ + +#ifndef __IMX_MU_PLATFORM_H__ +#define __IMX_MU_PLATFORM_H__ + +/* + * Return the number of reception channels + */ +unsigned int imx_mu_plat_get_rx_channel(vaddr_t base); + +/* + * Return the number of transmission channels + */ +unsigned int imx_mu_plat_get_tx_channel(vaddr_t base); + +/* + * Send a 32bits word via the MU + * + * @base: virtual base address of the MU controller + * @index: MU channel index + * @[in]msg: word to send + */ +TEE_Result imx_mu_plat_send(vaddr_t base, unsigned int index, uint32_t msg); + +/* + * Get the 32bits word received by the MU + * + * @base: virtual base address of the MU controller + * @index: MU channel index + * @[out]msg: word received + */ +TEE_Result imx_mu_plat_receive(vaddr_t base, unsigned int index, uint32_t *msg); + +/* + * Initialize the MU interface + * + * @base: virtual base address of the MU controller + */ +void imx_mu_plat_init(vaddr_t base); +#endif /* __IMX_MU_PLATFORM_H__ */ diff --git a/optee_os/core/drivers/imx/mu/sub.mk b/optee_os/core/drivers/imx/mu/sub.mk new file mode 100644 index 0000000..368c22c --- /dev/null +++ b/optee_os/core/drivers/imx/mu/sub.mk @@ -0,0 +1,5 @@ +srcs-y += imx_mu.c +srcs-$(call cfg-one-enabled,CFG_MX8ULP CFG_MX93) += imx_mu_8ulp.c +ifeq ($(filter y, $(CFG_MX8QM) $(CFG_MX8QX) $(CFG_MX8DXL)),y) +srcs-y += imx_mu_8q.c +endif diff --git a/optee_os/core/drivers/imx/sub.mk b/optee_os/core/drivers/imx/sub.mk new file mode 100644 index 0000000..5b4df0b --- /dev/null +++ b/optee_os/core/drivers/imx/sub.mk @@ -0,0 +1,2 @@ +subdirs-$(CFG_IMX_DCP) += dcp +subdirs-$(CFG_IMX_MU) += mu \ No newline at end of file diff --git a/optee_os/core/drivers/imx_caam.c b/optee_os/core/drivers/imx_caam.c new file mode 100644 index 0000000..747957a --- /dev/null +++ b/optee_os/core/drivers/imx_caam.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019 Bryan O'Donoghue + * Copyright 2019, 2023 NXP + * + * Bryan O'Donoghue + */ + +#include +#include +#include + +struct imx_caam_job_ring { + uint32_t jrmidr_ms; + uint32_t jrmidr_ls; +}; + +#define CAAM_NUM_JOB_RINGS 4 + +/* CAAM ownersip definition bits */ +#define JROWN_NS BIT(3) +#define JROWN_MID 0x01 + +/* A basic sub-set of the CAAM */ +struct imx_caam_ctrl { + uint32_t res0; + uint32_t mcfgr; + uint32_t res1; + uint32_t scfgr; + struct imx_caam_job_ring jr[CAAM_NUM_JOB_RINGS]; +}; + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CAAM_BASE, CORE_MMU_PGDIR_SIZE); + +static TEE_Result init_caam(void) +{ + struct imx_caam_ctrl *caam; + uint32_t reg; + int i; + + caam = (struct imx_caam_ctrl *) + core_mmu_get_va(CAAM_BASE, MEM_AREA_IO_SEC, + sizeof(struct imx_caam_ctrl)); + if (!caam) + return TEE_ERROR_GENERIC; + + /* + * Set job-ring ownership to non-secure by default. + * A Linux kernel that runs after OP-TEE will run in normal-world + * so we want to enable that kernel to have total ownership of the + * CAAM job-rings. + * + * It is possible to use CAAM job-rings inside of OP-TEE i.e. in + * secure world code but, to do that OP-TEE and kernel should agree + * via a DTB which job-rings are owned by OP-TEE and which are + * owned by Kernel, something that the OP-TEE CAAM driver should + * set up. + * + * This code below simply sets a default for the case where no + * runtime OP-TEE CAAM code will be run + */ + for (i = 0; i < CAAM_NUM_JOB_RINGS; i++) { + reg = io_read32((vaddr_t)&caam->jr[i].jrmidr_ms); + reg |= JROWN_NS | JROWN_MID; + io_write32((vaddr_t)&caam->jr[i].jrmidr_ms, reg); + } + + return TEE_SUCCESS; +} + +driver_init(init_caam); diff --git a/optee_os/core/drivers/imx_csu.c b/optee_os/core/drivers/imx_csu.c new file mode 100644 index 0000000..fc06697 --- /dev/null +++ b/optee_os/core/drivers/imx_csu.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2023 NXP + * + */ + +#include +#include +#include +#include +#include +#include +#include + +struct csu_setting { + int csu_index; + uint32_t value; +}; + +const struct csu_setting csu_setting_imx6[] = { + {13, 0xFF0033}, /* Protect ROMCP */ + {16, 0x330033}, /* Protect TZASC */ + {26, 0xFF0033}, /* Protect OCRAM */ + {(-1), 0}, +}; + +struct csu_sa_setting { + uint32_t access_value; + uint32_t lock_value; +}; + +struct csu_config { + const struct csu_sa_setting * const sa; + const struct csu_setting * const csl; +}; + +const struct csu_setting csu_setting_imx6ul[] = { + {13, 0xFF0033}, /* Protect ROMCP */ + {16, 0x3300FF}, /* Protect TZASC */ + {39, 0x3300FF}, /* Protect OCRAM */ + {(-1), 0}, +}; + +const struct csu_setting csu_setting_imx6ull[] = { + { 13, 0xFF0033 }, /* Protect ROMCP */ + { 16, 0x3300FF }, /* Protect TZASC */ + { 34, 0xFF0033 }, /* Protect DCP */ + { 39, 0x3300FF }, /* Protect OCRAM */ + { (-1), 0 }, +}; + +const struct csu_setting csu_setting_imx6sl[] = { + { 13, 0x3F0033 }, /* Protect DCP/ROMCP */ + { 16, 0xFF0033 }, /* Protect TZASC */ + { 26, 0xFF0033 }, /* Protect OCRAM */ + { (-1), 0 }, +}; + +const struct csu_setting csu_setting_imx6sx[] = { + {13, 0xFF0033}, /* Protect ROMCP */ + {15, 0xFF0033}, /* Protect RDC */ + {16, 0x3300FF}, /* Protect TZASC */ + {34, 0x3300FF}, /* Protect OCRAM */ + {(-1), 0}, +}; + +const struct csu_setting csu_setting_imx7ds[] = { + {14, 0x3300FF}, /* Protect RDC */ + {15, 0xFF0033}, /* Protect CSU */ + {28, 0xFF0033}, /* Protect TZASC */ + {59, 0x3300FF}, /* Protect OCRAM_S */ + {(-1), 0}, +}; + +/* Set all masters to non-secure except the Cortex-A7 */ +const struct csu_sa_setting csu_sa_imx6ul = { 0x10554550, 0x20aa8aa2 }; +const struct csu_sa_setting csu_sa_imx7ds = { 0x15554554, 0x2aaa8aaa }; + +const struct csu_config csu_imx6 = { NULL, csu_setting_imx6 }; +const struct csu_config csu_imx6ul = { &csu_sa_imx6ul, csu_setting_imx6ul }; +const struct csu_config csu_imx6ull = { NULL, csu_setting_imx6ull }; +const struct csu_config csu_imx6sl = { NULL, csu_setting_imx6sl }; +const struct csu_config csu_imx6sx = { NULL, csu_setting_imx6sx }; +const struct csu_config csu_imx7ds = { &csu_sa_imx7ds, csu_setting_imx7ds }; + +static void rngb_configure(vaddr_t csu_base) +{ + int csu_index = 0; + + if (soc_is_imx6sl() || soc_is_imx6sll()) + csu_index = 16; + else if (soc_is_imx6ull()) + csu_index = 34; + else + return; + + /* Protect RNGB */ + io_mask32(csu_base + csu_index * 4, 0x330000, 0xFF0000); +} + +static TEE_Result csu_configure(void) +{ + vaddr_t csu_base; + vaddr_t offset; + const struct csu_config *csu_config = NULL; + const struct csu_setting *csu_setting = NULL; + + csu_base = core_mmu_get_va(CSU_BASE, MEM_AREA_IO_SEC, 1); + if (!csu_base) + panic(); + + if (soc_is_imx6sx()) + csu_config = &csu_imx6sx; + else if (soc_is_imx6ul()) + csu_config = &csu_imx6ul; + else if (soc_is_imx6ull()) + csu_config = &csu_imx6ull; + else if (soc_is_imx6sll() || soc_is_imx6sl()) + csu_config = &csu_imx6sl; + else if (soc_is_imx6()) + csu_config = &csu_imx6; + else if (soc_is_imx7ds()) + csu_config = &csu_imx7ds; + else + return TEE_SUCCESS; + + /* first grant all peripherals */ + for (offset = CSU_CSL_START; offset < CSU_CSL_END; offset += 4) + io_write32(csu_base + offset, CSU_ACCESS_ALL); + + csu_setting = csu_config->csl; + + while (csu_setting->csu_index >= 0) { + io_write32(csu_base + (csu_setting->csu_index * 4), + csu_setting->value); + + csu_setting++; + } + + if (IS_ENABLED(CFG_IMX_RNGB)) + rngb_configure(csu_base); + + /* lock the settings */ + for (offset = CSU_CSL_START; offset < CSU_CSL_END; offset += 4) { + io_write32(csu_base + offset, + io_read32(csu_base + offset) | CSU_SETTING_LOCK); + } + + if (csu_config->sa) { + io_write32(csu_base + CSU_SA, csu_config->sa->access_value); + io_setbits32(csu_base + CSU_SA, csu_config->sa->lock_value); + } + + return TEE_SUCCESS; +} + +static TEE_Result +pm_enter_resume(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *pm_handle __unused) +{ + if (op == PM_OP_RESUME) + csu_configure(); + + return TEE_SUCCESS; +} + +static TEE_Result csu_init(void) +{ + csu_configure(); + register_pm_driver_cb(pm_enter_resume, NULL, "imx-csu"); + + return TEE_SUCCESS; +} + +driver_init(csu_init); diff --git a/optee_os/core/drivers/imx_ele.c b/optee_os/core/drivers/imx_ele.c new file mode 100644 index 0000000..bfa2f19 --- /dev/null +++ b/optee_os/core/drivers/imx_ele.c @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ELE_BASE_ADDR MU_BASE +#define ELE_BASE_SIZE MU_SIZE + +#define ELE_VERSION_BASELINE 0x06 +#define ELE_VERSION_HSM 0x07 +#define ELE_COMMAND_SUCCEED 0xd6 +#define ELE_REQUEST_TAG 0x17 +#define ELE_RESPONSE_TAG 0xe1 + +#define ELE_CMD_SESSION_OPEN 0x10 +#define ELE_CMD_SESSION_CLOSE 0x11 +#define ELE_CMD_SESSION_DEVICE_INFO 0x16 +#define ELE_CMD_RNG_GET 0xCD +#define ELE_CMD_TRNG_STATE 0xA4 +#define ELE_CMD_GET_INFO 0xDA +#define ELE_CMD_DERIVE_KEY 0xA9 + +#define IMX_ELE_TRNG_STATUS_READY 0x3 + +#define ELE_MU_ID 0x2 +#define ELE_MU_IRQ 0x0 + +#if defined(CFG_MX8ULP) +#define ELE_MU_DID 0x7 +#define CACHELINE_SIZE 64 +#elif defined(CFG_MX93) +#define ELE_MU_DID 0x3 +#define CACHELINE_SIZE 64 +#else +#error "Platform DID is not defined" +#endif + +#define SIZE_MSG_32(_msg) size_msg_32(sizeof(_msg)) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, ELE_BASE_ADDR, ELE_BASE_SIZE); + +struct get_info_msg_rsp { + uint32_t rsp_code; + uint16_t soc_id; + uint16_t soc_rev; + uint16_t lifecycle; + uint16_t sssm_state; + uint32_t uid[4]; + uint32_t sha256_rom_patch[8]; + uint32_t sha256_fw[8]; +} __packed; + +struct session_get_device_info_rsp { + uint32_t rsp_code; + uint32_t user_sab_id; + uint32_t chip_uid[4]; + uint16_t chip_life_cycle; + uint16_t chip_monotonic_counter; + uint32_t ele_version; + uint32_t ele_version_ext; + uint8_t fips_mode; + uint8_t reserved[3]; + uint32_t crc; +} __packed; + +struct response_code { + uint8_t status; + uint8_t rating; + uint16_t rating_extension; +} __packed; + +/* + * Print ELE response status and rating + * + * @rsp response code structure + */ +static void print_rsp_code(struct response_code rsp __maybe_unused) +{ + DMSG("Response status %#"PRIx8", rating %#"PRIx8" (ext %#"PRIx16")", + rsp.status, rsp.rating, rsp.rating_extension); +} + +/* + * Print ELE message header + * + * @hdr message header + */ +static void print_msg_header(struct imx_mu_msg_header hdr __maybe_unused) +{ + DMSG("Header ver %#"PRIx8", size %"PRId8", tag %#"PRIx8", cmd %#"PRIx8, + hdr.version, hdr.size, hdr.tag, hdr.command); +} + +/* + * Print full ELE message content + * + * @msg message + */ +static void dump_message(const struct imx_mu_msg *msg __maybe_unused) +{ + size_t i = 0; + size_t size __maybe_unused = msg->header.size; + uint32_t *data __maybe_unused = (uint32_t *)msg; + + DMSG("Dump of message %p(%zu)", data, size); + for (i = 0; i < size; i++) + DMSG("word %zu: %#"PRIx32, i, data[i]); +} + +/* + * Return the number of 32 bits words of the given message. + * + * @cmd command size in byte + */ +static size_t size_msg_32(size_t cmd) +{ + /* Roundup and add header size */ + return ROUNDUP_DIV(cmd, sizeof(uint32_t)) + 1; +} + +/* + * The CRC for the message is computed xor-ing all the words of the message: + * the header and all the words except the word storing the CRC. + * + * @msg MU message to hash + */ +static uint32_t compute_crc(const struct imx_mu_msg *msg) +{ + uint32_t crc = 0; + uint8_t i = 0; + uint32_t *payload = (uint32_t *)msg; + + assert(msg); + + for (i = 0; i < msg->header.size - 1; i++) + crc ^= payload[i]; + + return crc; +} + +/* + * Compute message CRC and update CRC in message header. + * + * @msg MU message to hash + */ +static void update_crc(struct imx_mu_msg *msg) +{ + assert(msg); + /* + * The CRC field is the last element of array. The size of the header + * is also subtracted from CRC computation. + */ + msg->data.u32[msg->header.size - 2] = compute_crc(msg); +} + +/* + * Return the given MU base address, depending on the MMU state. + * + * @pa MU physical base address + * @sz MU size + */ +static vaddr_t imx_ele_init(paddr_t pa, size_t sz) +{ + static bool is_initialized; + vaddr_t va = 0; + + assert(pa && sz); + + if (cpu_mmu_enabled()) + va = core_mmu_get_va(pa, MEM_AREA_IO_SEC, sz); + else + va = (vaddr_t)pa; + + if (!is_initialized) { + imx_mu_init(va); + is_initialized = true; + } + + return va; +} + +/* + * Extract response codes from the given word + * + * @word 32 bits word MU response + */ +static struct response_code get_response_code(uint32_t word) +{ + struct response_code rsp = { + .rating_extension = (word & GENMASK_32(31, 16)) >> 16, + .rating = (word & GENMASK_32(15, 8)) >> 8, + .status = (word & GENMASK_32(7, 0)) >> 0, + }; + + return rsp; +} + +/* + * Initiate a communication with the EdgeLock Enclave. It sends a message + * and expects an answer. + * + * @msg MU message + */ +static TEE_Result imx_ele_call(struct imx_mu_msg *msg) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct response_code rsp = { }; + vaddr_t va = 0; + + assert(msg); + + if (msg->header.tag != ELE_REQUEST_TAG) { + EMSG("Request has invalid tag: %#"PRIx8" instead of %#"PRIx8, + msg->header.tag, ELE_REQUEST_TAG); + return TEE_ERROR_BAD_PARAMETERS; + } + + va = imx_ele_init(ELE_BASE_ADDR, ELE_BASE_SIZE); + if (!va) { + EMSG("Fail to get base address"); + return TEE_ERROR_GENERIC; + } + + res = imx_mu_call(va, msg, true); + if (res) { + EMSG("Failed to transmit message: %#"PRIx32, res); + print_msg_header(msg->header); + dump_message(msg); + return res; + } + + rsp = get_response_code(msg->data.u32[0]); + + if (msg->header.tag != ELE_RESPONSE_TAG) { + EMSG("Response has invalid tag: %#"PRIx8" instead of %#"PRIx8, + msg->header.tag, ELE_RESPONSE_TAG); + print_msg_header(msg->header); + return TEE_ERROR_GENERIC; + } + + if (rsp.status != ELE_COMMAND_SUCCEED) { + EMSG("Command has failed"); + print_rsp_code(rsp); + return TEE_ERROR_GENERIC; + } + + /* The rating can be different in success and failing cases */ + if (rsp.rating != 0) { + EMSG("Command has invalid rating: %#"PRIx8, rsp.rating); + print_rsp_code(rsp); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +/* + * Get device information from EdgeLock Enclave + * + * @session_handle EdgeLock Enclave session handle + * @rsp Device info + */ +static TEE_Result +imx_ele_session_get_device_info(uint32_t session_handle, + struct session_get_device_info_rsp *rsp) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct session_get_device_info_cmd { + uint32_t session_handle; + } cmd = { + .session_handle = session_handle, + }; + struct imx_mu_msg msg = { + .header.version = ELE_VERSION_HSM, + .header.size = SIZE_MSG_32(cmd), + .header.tag = ELE_REQUEST_TAG, + .header.command = ELE_CMD_SESSION_DEVICE_INFO, + }; + + assert(rsp); + + memcpy(msg.data.u8, &cmd, sizeof(cmd)); + + res = imx_ele_call(&msg); + if (res) + return res; + + memcpy(rsp, msg.data.u32, sizeof(*rsp)); + + if (compute_crc(&msg) != rsp->crc) + return TEE_ERROR_CORRUPT_OBJECT; + + return TEE_SUCCESS; +} + +/* + * Open a session with EdgeLock Enclave. It returns a session handle. + * + * @session_handle EdgeLock Enclave session handle + */ +static TEE_Result imx_ele_session_open(uint32_t *session_handle) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct open_session_cmd { + uint8_t mu_id; + uint8_t interrupt_num; + uint8_t tz; + uint8_t did; + uint8_t priority; + uint8_t op_mode; + uint16_t reserved; + } __packed cmd = { + .mu_id = ELE_MU_ID, + .interrupt_num = ELE_MU_IRQ, + .tz = 0, + .did = ELE_MU_DID, + .priority = 0, + .op_mode = 0, + .reserved = 0, + }; + struct open_session_rsp { + uint32_t rsp_code; + uint32_t session_handle; + } rsp = { }; + struct imx_mu_msg msg = { + .header.version = ELE_VERSION_HSM, + .header.size = SIZE_MSG_32(cmd), + .header.tag = ELE_REQUEST_TAG, + .header.command = ELE_CMD_SESSION_OPEN, + }; + + assert(session_handle); + + memcpy(msg.data.u8, &cmd, sizeof(cmd)); + + res = imx_ele_call(&msg); + if (res) + return res; + + memcpy(&rsp, msg.data.u8, sizeof(rsp)); + + *session_handle = rsp.session_handle; + + return TEE_SUCCESS; +} + +/* + * Close a session with EdgeLock Enclave. + * + * @session_handle EdgeLock Enclave session handle + */ +static TEE_Result imx_ele_session_close(uint32_t session_handle) +{ + struct close_session_cmd { + uint32_t session_handle; + } cmd = { + .session_handle = session_handle, + }; + struct imx_mu_msg msg = { + .header.version = ELE_VERSION_HSM, + .header.size = SIZE_MSG_32(cmd), + .header.tag = ELE_REQUEST_TAG, + .header.command = ELE_CMD_SESSION_CLOSE, + }; + + memcpy(msg.data.u8, &cmd, sizeof(cmd)); + + return imx_ele_call(&msg); +} + +/* + * Get the current state of the ELE TRNG + */ +static TEE_Result imx_ele_rng_get_trng_state(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct rng_get_trng_state_msg_rsp { + uint32_t rsp_code; + uint8_t trng_state; + uint8_t csal_state; + } __packed rsp = { }; + struct imx_mu_msg msg = { + .header.version = ELE_VERSION_BASELINE, + .header.size = 1, + .header.tag = ELE_REQUEST_TAG, + .header.command = ELE_CMD_TRNG_STATE, + }; + + res = imx_ele_call(&msg); + if (res) + return res; + + memcpy(&rsp, msg.data.u8, sizeof(rsp)); + + if (rsp.trng_state != IMX_ELE_TRNG_STATUS_READY) + return TEE_ERROR_BUSY; + else + return TEE_SUCCESS; +} + +unsigned long plat_get_aslr_seed(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint64_t timeout = timeout_init_us(10 * 1000); + struct rng_get_random_cmd { + uint32_t addr_msb; + uint32_t addr_lsb; + uint32_t size; + uint32_t crc; + } cmd = { }; + struct imx_mu_msg msg = { + .header.version = ELE_VERSION_HSM, + .header.size = SIZE_MSG_32(cmd), + .header.tag = ELE_REQUEST_TAG, + .header.command = ELE_CMD_RNG_GET, + }; + unsigned long aslr __aligned(CACHELINE_SIZE) = 0; + + /* + * This function can only be called when the MMU is off. No + * virtual/physical address translation is performed, nor cache + * maintenance. + */ + assert(!cpu_mmu_enabled()); + + reg_pair_from_64((uint64_t)&aslr, &cmd.addr_msb, &cmd.addr_lsb); + cmd.size = sizeof(aslr); + + /* + * Check the current TRNG state of the ELE. The TRNG must be + * started with a command earlier in the boot to allow the TRNG + * to generate enough entropy. + */ + while (imx_ele_rng_get_trng_state() == TEE_ERROR_BUSY) + if (timeout_elapsed(timeout)) + panic("ELE RNG is busy"); + + memcpy(msg.data.u8, &cmd, sizeof(cmd)); + update_crc(&msg); + + res = imx_ele_call(&msg); + if (res) + panic("Cannot retrieve random data from ELE"); + + return aslr; +} + +int tee_otp_get_die_id(uint8_t *buffer, size_t len) +{ + uint32_t session_handle = 0; + /* + * The die ID must be cached because some board configuration prevents + * the MU to be used by OPTEE at runtime. + */ + static struct session_get_device_info_rsp rsp; + + if (rsp.rsp_code) + goto out; + + if (imx_ele_session_open(&session_handle)) + goto err; + + if (imx_ele_session_get_device_info(session_handle, &rsp)) + goto err; + + if (imx_ele_session_close(session_handle)) + goto err; + +out: + /* + * In the device info array return by the ELE, the words 2, 3, 4 and 5 + * are the device UID. + */ + memcpy(buffer, rsp.chip_uid, MIN(sizeof(rsp.chip_uid), len)); + + return 0; +err: + panic("Fail to get the device UID"); +} + +#if defined(CFG_MX93) +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const char pattern[16] __aligned(CACHELINE_SIZE) = "TEE_for_HUK_ELE"; + static uint8_t key[CACHELINE_SIZE] __aligned(CACHELINE_SIZE); + static bool is_fetched; + uint32_t msb = 0; + uint32_t lsb = 0; + struct key_derive_cmd { + uint32_t key_addr_msb; + uint32_t key_addr_lsb; + uint32_t ctx_addr_msb; + uint32_t ctx_addr_lsb; + uint16_t key_size; + uint16_t ctx_size; + uint32_t crc; + } __packed cmd = { }; + struct imx_mu_msg msg = { + .header.version = ELE_VERSION_BASELINE, + .header.size = SIZE_MSG_32(cmd), + .header.tag = ELE_REQUEST_TAG, + .header.command = ELE_CMD_DERIVE_KEY, + }; + + if (is_fetched) + goto out; + + /* + * Intermediate msb and lsb values are needed. Directly using + * key_addr_msb and key_addr_lsb might be unaligned because of the + * __packed attribute of key_derive_cmd {} + */ + reg_pair_from_64((uint64_t)virt_to_phys(key), &msb, &lsb); + + cmd.key_addr_lsb = lsb; + cmd.key_addr_msb = msb; + cmd.key_size = HW_UNIQUE_KEY_LENGTH; + + reg_pair_from_64((uint64_t)virt_to_phys((void *)pattern), &msb, &lsb); + + cmd.ctx_addr_lsb = lsb; + cmd.ctx_addr_msb = msb; + cmd.ctx_size = sizeof(pattern); + + memcpy(msg.data.u8, &cmd, sizeof(cmd)); + update_crc(&msg); + + cache_operation(TEE_CACHEFLUSH, key, HW_UNIQUE_KEY_LENGTH); + cache_operation(TEE_CACHECLEAN, (void *)pattern, sizeof(pattern)); + + res = imx_ele_call(&msg); + if (res) + panic("failed to get the huk"); + + cache_operation(TEE_CACHEINVALIDATE, key, HW_UNIQUE_KEY_LENGTH); + is_fetched = true; +out: + memcpy(hwkey->data, key, + MIN(sizeof(key), (size_t)HW_UNIQUE_KEY_LENGTH)); + + return TEE_SUCCESS; +} +#endif /* CFG_MX93 */ diff --git a/optee_os/core/drivers/imx_i2c.c b/optee_os/core/drivers/imx_i2c.c new file mode 100644 index 0000000..8ef093b --- /dev/null +++ b/optee_os/core/drivers/imx_i2c.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * (c) 2020 Jorge Ramirez , Foundries Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define I2C_CLK_RATE 24000000 /* Bits per second */ + +/* Utility macros (__x identifies the bus [1 .. 3]) */ +#define I2C_CFG_SCL(__x) (IOMUXC_I2C1_SCL_CFG_OFF + ((__x) - 1) * 0x8) +#define I2C_CFG_SDA(__x) (IOMUXC_I2C1_SDA_CFG_OFF + ((__x) - 1) * 0x8) +#define I2C_MUX_SCL(__x) (IOMUXC_I2C1_SCL_MUX_OFF + ((__x) - 1) * 0x8) +#define I2C_MUX_SDA(__x) (IOMUXC_I2C1_SDA_MUX_OFF + ((__x) - 1) * 0x8) +#if defined(CFG_MX8MM) || defined(CFG_MX8MQ) || defined(CFG_MX8MP) || \ + defined(CFG_MX8MN) +/* IOMUX */ +#define I2C_INP_SCL(__x) 0 /* Not implemented */ +#define I2C_INP_SDA(__x) 0 /* Not implemented */ +#define I2C_INP_VAL(__x) 0 /* Not implemented */ +#define I2C_MUX_VAL(__x) 0x010 +#define I2C_CFG_VAL(__x) 0x1c3 +/* Clock */ +#define I2C_CLK_CGRBM(__x) 0 /* Not implemented */ +#define I2C_CLK_CGR6BM(__x) 0 +#define I2C_CLK_CGR(__x) CCM_CCRG_I2C##__x +#elif defined(CFG_MX6ULL) +/* IOMUX */ +#define I2C_INP_SCL(__x) (IOMUXC_I2C1_SCL_INP_OFF + ((__x) - 1) * 0x8) +#define I2C_INP_SDA(__x) (IOMUXC_I2C1_SDA_INP_OFF + ((__x) - 1) * 0x8) +#define I2C_INP_VAL(__x) (((__x) == 1) ? 0x1 : 0x2) +#define I2C_MUX_VAL(__x) 0x012 +#define I2C_CFG_VAL(__x) 0x1b8b0 +/* Clock */ +#define I2C_CLK_CGRBM(__x) BM_CCM_CCGR2_I2C##__x##_SERIAL +#define I2C_CLK_CGR6BM(__x) BM_CCM_CCGR6_I2C##__x##_SERIAL +#define I2C_CLK_CGR(__x) (((__x) == 4) ? CCM_CCGR6 : CCM_CCGR2) +#else +#error IMX_I2C driver not supported on this platform +#endif + +#if !defined(CFG_MX8MP) +static struct io_pa_va i2c_bus[4] = { +#if !defined(CFG_DT) || defined(CFG_EXTERNAL_DTB_OVERLAY) +#if defined(I2C1_BASE) + [0] = { .pa = I2C1_BASE, }, +#endif +#if defined(I2C2_BASE) + [1] = { .pa = I2C2_BASE, }, +#endif +#if defined(I2C3_BASE) + [2] = { .pa = I2C3_BASE, }, +#endif +#if defined(I2C4_BASE) + [3] = { .pa = I2C4_BASE, }, +#endif +#endif +}; +#else +static struct io_pa_va i2c_bus[6] = { +#if !defined(CFG_DT) || defined(CFG_EXTERNAL_DTB_OVERLAY) +#if defined(I2C1_BASE) + [0] = { .pa = I2C1_BASE, }, +#endif +#if defined(I2C2_BASE) + [1] = { .pa = I2C2_BASE, }, +#endif +#if defined(I2C3_BASE) + [2] = { .pa = I2C3_BASE, }, +#endif +#if defined(I2C4_BASE) + [3] = { .pa = I2C4_BASE, }, +#endif +#if defined(I2C5_BASE) + [4] = { .pa = I2C5_BASE, }, +#endif +#if defined(I2C6_BASE) + [5] = { .pa = I2C6_BASE, }, +#endif + +#endif +}; +#endif + +static struct imx_i2c_clk { + struct io_pa_va base; + uint32_t i2c[ARRAY_SIZE(i2c_bus)]; + uint32_t cgrbm[ARRAY_SIZE(i2c_bus)]; +} i2c_clk = { + .base.pa = CCM_BASE, + .i2c = { I2C_CLK_CGR(1), I2C_CLK_CGR(2), I2C_CLK_CGR(3), I2C_CLK_CGR(4), }, + .cgrbm = { I2C_CLK_CGRBM(1), I2C_CLK_CGRBM(2), I2C_CLK_CGRBM(3), I2C_CLK_CGR6BM(4),}, +}; + +static struct imx_i2c_mux { + struct io_pa_va base; + struct imx_i2c_mux_regs { + uint32_t scl_mux; + uint32_t scl_cfg; + uint32_t scl_inp; + uint32_t sda_mux; + uint32_t sda_cfg; + uint32_t sda_inp; + } i2c[ARRAY_SIZE(i2c_bus)]; +} i2c_mux = { + .base.pa = IOMUXC_BASE, + .i2c = {{ .scl_mux = I2C_MUX_SCL(1), .scl_cfg = I2C_CFG_SCL(1), + .scl_inp = I2C_INP_SCL(1), .sda_mux = I2C_MUX_SDA(1), + .sda_cfg = I2C_CFG_SDA(1), .sda_inp = I2C_INP_SDA(1), }, + { .scl_mux = I2C_MUX_SCL(2), .scl_cfg = I2C_CFG_SCL(2), + .scl_inp = I2C_INP_SCL(2), .sda_mux = I2C_MUX_SDA(2), + .sda_cfg = I2C_CFG_SDA(2), .sda_inp = I2C_INP_SDA(2), }, + { .scl_mux = I2C_MUX_SCL(3), .scl_cfg = I2C_CFG_SCL(3), + .scl_inp = I2C_INP_SCL(3), .sda_mux = I2C_MUX_SDA(3), + .sda_cfg = I2C_CFG_SDA(3), .sda_inp = I2C_INP_SDA(3), }, + { .scl_mux = I2C_MUX_SCL(4), .scl_cfg = I2C_CFG_SCL(4), + .scl_inp = I2C_INP_SCL(4), .sda_mux = I2C_MUX_SDA(4), + .sda_cfg = I2C_CFG_SDA(4), .sda_inp = I2C_INP_SDA(4), },}, +}; + +#define I2DR 0x10 +#define I2SR 0x0C +#define I2CR 0x08 +#define IFDR 0x04 + +#define I2CR_IEN BIT(7) +#define I2CR_IIEN BIT(6) +#define I2CR_MSTA BIT(5) +#define I2CR_MTX BIT(4) +#define I2CR_TX_NO_AK BIT(3) +#define I2CR_RSTA BIT(2) + +#define I2SR_ICF BIT(7) +#define I2SR_IBB BIT(5) +#define I2SR_IAL BIT(4) +#define I2SR_IIF BIT(1) +#define I2SR_RX_NO_AK BIT(0) + +static uint8_t i2c_io_read8(uint8_t bid, uint32_t address) +{ + return io_read8(i2c_bus[bid].va + address); +} + +static void i2c_io_write8(uint8_t bid, uint32_t address, uint8_t data) +{ + return io_write8(i2c_bus[bid].va + address, data); +} + +static bool bus_is_idle(uint32_t sr) +{ + return (sr & I2SR_IBB) == 0; +} + +static bool bus_is_busy(uint32_t sr) +{ + return !bus_is_idle(sr); +} + +static bool isr_active(uint32_t sr) +{ + return (sr & I2SR_IIF) == I2SR_IIF; +} + +static struct ifdr_pair { + uint32_t divider; + uint8_t prescaler; +} ifdr_table[] = { + { 22, 0x20 }, { 24, 0x21 }, { 26, 0x22 }, { 28, 0x23 }, + { 30, 0x00 }, { 32, 0x24 }, { 36, 0x25 }, { 40, 0x26 }, + { 42, 0x03 }, { 44, 0x27 }, { 48, 0x28 }, { 52, 0x05 }, + { 56, 0x29 }, { 60, 0x06 }, { 64, 0x2A }, { 72, 0x2B }, + { 80, 0x2C }, { 88, 0x09 }, { 96, 0x2D }, { 104, 0x0A }, + { 112, 0x2E }, { 128, 0x2F }, { 144, 0x0C }, { 160, 0x30 }, + { 192, 0x31 }, { 224, 0x32 }, { 240, 0x0F }, { 256, 0x33 }, + { 288, 0x10 }, { 320, 0x34 }, { 384, 0x35 }, { 448, 0x36 }, + { 480, 0x13 }, { 512, 0x37 }, { 576, 0x14 }, { 640, 0x38 }, + { 768, 0x39 }, { 896, 0x3A }, { 960, 0x17 }, { 1024, 0x3B }, + { 1152, 0x18 }, { 1280, 0x3C }, { 1536, 0x3D }, { 1792, 0x3E }, + { 1920, 0x1B }, { 2048, 0x3F }, { 2304, 0x1C }, { 2560, 0x1D }, + { 3072, 0x1E }, { 3840, 0x1F } +}; + +static void i2c_set_prescaler(uint8_t bid, uint32_t bps) +{ + struct ifdr_pair *p = ifdr_table; + struct ifdr_pair *q = p + ARRAY_SIZE(ifdr_table) - 1; + uint32_t div = (I2C_CLK_RATE + bps - 1) / bps; + + if (div < p->divider) + q = p; + else if (div > q->divider) + p = q; + + while (p != q) { + if (div <= p->divider) + break; + p++; + } + + i2c_io_write8(bid, IFDR, p->prescaler); +} + +static void i2c_set_bus_speed(uint8_t bid, int bps) +{ + vaddr_t addr = i2c_clk.base.va; + uint32_t val = 0; + +#if defined(CFG_MX8MM) || defined(CFG_MX8MQ) || defined(CFG_MX8MP) || \ + defined(CFG_MX8MN) + addr += CCM_CCGRx_SET(i2c_clk.i2c[bid]); + val = CCM_CCGRx_ALWAYS_ON(0); +#elif defined(CFG_MX6ULL) + addr += i2c_clk.i2c[bid]; + val = i2c_clk.cgrbm[bid] | io_read32(addr); +#else +#error IMX_I2C driver not supported on this platform +#endif + io_write32(addr, val); + i2c_set_prescaler(bid, bps); +} + +static TEE_Result i2c_sync_bus(uint8_t bid, bool (*match)(uint32_t), + uint32_t *status) +{ + uint64_t tref = timeout_init_us(100000); + uint32_t sr = 0; + + while (!timeout_elapsed(tref)) { + sr = i2c_io_read8(bid, I2SR); + if (sr & I2SR_IAL) { + EMSG("bus arbitration lost"); + i2c_io_write8(bid, I2SR, sr & ~I2SR_IAL); + return TEE_ERROR_COMMUNICATION; + } + if ((*match)(sr)) { + if (status) + *status = sr; + return TEE_SUCCESS; + } + } + + return TEE_ERROR_BUSY; +} + +static TEE_Result i2c_idle_bus(uint8_t bid) +{ + uint8_t tmp = i2c_io_read8(bid, I2CR) & ~I2CR_MSTA; + TEE_Result ret = TEE_SUCCESS; + + i2c_io_write8(bid, I2CR, tmp); + ret = i2c_sync_bus(bid, &bus_is_idle, NULL); + i2c_io_write8(bid, I2SR, 0); + + return ret; +} + +static TEE_Result i2c_write_byte(uint8_t bid, uint8_t byte) +{ + TEE_Result ret = TEE_SUCCESS; + uint32_t status = 0; + + i2c_io_write8(bid, I2DR, byte); + ret = i2c_sync_bus(bid, &isr_active, &status); + i2c_io_write8(bid, I2SR, 0); + + if (!ret && (status & I2SR_RX_NO_AK)) + return TEE_ERROR_BAD_STATE; + + return ret; +} + +static TEE_Result i2c_read_byte(uint8_t bid, uint8_t *p) +{ + TEE_Result ret = TEE_SUCCESS; + + *p = i2c_io_read8(bid, I2DR); + ret = i2c_sync_bus(bid, &isr_active, NULL); + i2c_io_write8(bid, I2SR, 0); + + return ret; +} + +static TEE_Result i2c_write_data(uint8_t bid, const uint8_t *buf, int len) +{ + TEE_Result ret = TEE_SUCCESS; + uint32_t tmp = 0; + + if (!len) + return TEE_SUCCESS; + + tmp = i2c_io_read8(bid, I2CR) | I2CR_MTX | I2CR_TX_NO_AK; + i2c_io_write8(bid, I2CR, tmp); + + while (len--) { + ret = i2c_write_byte(bid, *buf++); + if (ret) + return ret; + } + + return ret; +} + +static TEE_Result i2c_read_data(uint8_t bid, uint8_t *buf, int len) +{ + TEE_Result ret = TEE_SUCCESS; + uint8_t dummy = 0; + uint32_t tmp = 0; + + if (!len) + return TEE_SUCCESS; + + tmp = i2c_io_read8(bid, I2CR) & ~I2CR_MTX; + tmp = (len == 1) ? tmp | I2CR_TX_NO_AK : tmp & ~I2CR_TX_NO_AK; + i2c_io_write8(bid, I2CR, tmp); + i2c_io_read8(bid, I2DR); + + ret = i2c_read_byte(bid, &dummy); + if (ret) + return ret; + + /* + * A data transfer ends when the master signals a stop; for a master + * receiver to terminate a transfer it must inform the slave transmiter + * by not acknowledging the last data byte. This is done by setting the + * transmit acknowledge bit before reading the next-to-last byte. + */ + do { + if (len == 2) { + tmp = i2c_io_read8(bid, I2CR) | I2CR_TX_NO_AK; + i2c_io_write8(bid, I2CR, tmp); + } + + ret = i2c_read_byte(bid, buf++); + if (ret) + return ret; + } while (len--); + + return ret; +} + +static TEE_Result i2c_init_transfer(uint8_t bid, uint8_t chip) +{ + TEE_Result ret = TEE_SUCCESS; + uint32_t tmp = 0; + + ret = i2c_idle_bus(bid); + if (ret) + return ret; + + /* Enable the interface */ + tmp = !(i2c_io_read8(bid, I2CR) & I2CR_IEN); + if (tmp) { + i2c_io_write8(bid, I2CR, I2CR_IEN); + udelay(50); + } + i2c_io_write8(bid, I2SR, 0); + + tmp = i2c_io_read8(bid, I2CR) | I2CR_MSTA; + i2c_io_write8(bid, I2CR, tmp); + + /* Wait until the bus is active */ + ret = i2c_sync_bus(bid, &bus_is_busy, NULL); + if (ret) + return ret; + + /* Slave address on the bus */ + return i2c_write_data(bid, &chip, 1); +} + +TEE_Result imx_i2c_read(uint8_t bid, uint8_t chip, uint8_t *buf, int len) +{ + TEE_Result ret = TEE_SUCCESS; + + if (bid >= ARRAY_SIZE(i2c_bus)) + return TEE_ERROR_BAD_PARAMETERS; + + if ((len && !buf) || chip > 0x7F) + return TEE_ERROR_BAD_PARAMETERS; + + if (!i2c_bus[bid].va) + return TEE_ERROR_BAD_PARAMETERS; + + ret = i2c_init_transfer(bid, chip << 1 | BIT(0)); + if (!ret) + ret = i2c_read_data(bid, buf, len); + + if (i2c_idle_bus(bid)) + IMSG("bus not idle"); + + return ret; +} + +TEE_Result imx_i2c_write(uint8_t bid, uint8_t chip, const uint8_t *buf, int len) +{ + TEE_Result ret = TEE_SUCCESS; + + if (bid >= ARRAY_SIZE(i2c_bus)) + return TEE_ERROR_BAD_PARAMETERS; + + if ((len && !buf) || chip > 0x7F) + return TEE_ERROR_BAD_PARAMETERS; + + if (!i2c_bus[bid].va) + return TEE_ERROR_BAD_PARAMETERS; + + ret = i2c_init_transfer(bid, chip << 1); + if (!ret) + ret = i2c_write_data(bid, buf, len); + + if (i2c_idle_bus(bid)) + IMSG("bus not idle"); + + return ret; +} + +TEE_Result imx_i2c_probe(uint8_t bid, uint8_t chip) +{ + if (bid >= ARRAY_SIZE(i2c_bus)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!i2c_bus[bid].va) + return TEE_ERROR_BAD_PARAMETERS; + + if (chip > 0x7F) + return TEE_ERROR_BAD_PARAMETERS; + + return imx_i2c_write(bid, chip, NULL, 0); +} + +/* + * I2C bus initialization: configure the IOMUX and enable the clock. + * @bid: Bus ID: (0=I2C1), (1=I2C2), (2=I2C3), (3=I2C4). + * @bps: Bus baud rate, in bits per second. + */ +TEE_Result imx_i2c_init(uint8_t bid, int bps) +{ + struct imx_i2c_mux *mux = &i2c_mux; + + if (bid >= ARRAY_SIZE(i2c_bus)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!bps) + return TEE_ERROR_BAD_PARAMETERS; + + if (!i2c_bus[bid].va) + return TEE_ERROR_BAD_PARAMETERS; + + io_write32(mux->base.va + mux->i2c[bid].scl_mux, I2C_MUX_VAL(bid)); + io_write32(mux->base.va + mux->i2c[bid].scl_cfg, I2C_CFG_VAL(bid)); + if (mux->i2c[bid].scl_inp) + io_write32(mux->base.va + mux->i2c[bid].scl_inp, + I2C_INP_VAL(bid + 1)); + + io_write32(mux->base.va + mux->i2c[bid].sda_mux, I2C_MUX_VAL(bid)); + io_write32(mux->base.va + mux->i2c[bid].sda_cfg, I2C_CFG_VAL(bid)); + if (mux->i2c[bid].sda_inp) + io_write32(mux->base.va + mux->i2c[bid].sda_inp, + I2C_INP_VAL(bid + 2)); + + /* Baud rate in bits per second */ + i2c_set_bus_speed(bid, bps); + + return TEE_SUCCESS; +} + +static TEE_Result get_va(paddr_t pa, vaddr_t *va) +{ + *va = (vaddr_t)core_mmu_add_mapping(MEM_AREA_IO_SEC, pa, 0x10000); + if (!*va) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +#if defined(CFG_DT) && !defined(CFG_EXTERNAL_DTB_OVERLAY) +static const char *const dt_i2c_match_table[] = { + "fsl,imx21-i2c", +}; + +static TEE_Result i2c_mapped(const char *i2c_match) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + void *fdt = get_dt(); + size_t size = 0; + size_t i = 0; + int off = 0; + + if (!fdt) + return TEE_ERROR_NOT_SUPPORTED; + + for (i = 0; i < ARRAY_SIZE(i2c_bus); i++) { + off = fdt_node_offset_by_compatible(fdt, off, i2c_match); + if (off < 0) + break; + + if (!(fdt_get_status(fdt, off) & DT_STATUS_OK_SEC)) { + EMSG("i2c%zu not enabled", i + 1); + continue; + } + + if (dt_map_dev(fdt, off, &i2c_bus[i].va, &size, + DT_MAP_AUTO) < 0) { + EMSG("i2c%zu not enabled", i + 1); + continue; + } + + i2c_bus[i].pa = virt_to_phys((void *)i2c_bus[i].va); + ret = TEE_SUCCESS; + } + + return ret; +} + +static TEE_Result i2c_map_controller(void) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t i = 0; + + for (i = 0; i < ARRAY_SIZE(dt_i2c_match_table); i++) { + ret = i2c_mapped(dt_i2c_match_table[i]); + if (!ret || ret == TEE_ERROR_NOT_SUPPORTED) + return ret; + } + + return ret; +} +#else +static TEE_Result i2c_map_controller(void) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(i2c_bus); n++) { + if (i2c_bus[n].pa) { + if (get_va(i2c_bus[n].pa, &i2c_bus[n].va)) + EMSG("i2c%zu not enabled", n + 1); + else + ret = TEE_SUCCESS; + } else { + IMSG("i2c%zu not enabled", n + 1); + } + } + + return ret; +} +#endif + +static TEE_Result i2c_init(void) +{ + if (get_va(i2c_clk.base.pa, &i2c_clk.base.va)) + return TEE_ERROR_GENERIC; + + if (get_va(i2c_mux.base.pa, &i2c_mux.base.va)) + return TEE_ERROR_GENERIC; + + return i2c_map_controller(); +} + +early_init(i2c_init); diff --git a/optee_os/core/drivers/imx_lpuart.c b/optee_os/core/drivers/imx_lpuart.c new file mode 100644 index 0000000..374b1f3 --- /dev/null +++ b/optee_os/core/drivers/imx_lpuart.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2019 NXP + */ + +#include +#include +#include +#include +#include +#include +#include + +#define STAT 0x14 +#define DATA 0x1C +#define UART_SIZE 0x20 +#define STAT_TDRE BIT(23) +#define STAT_RDRF BIT(21) +#define STAT_OR BIT(19) + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct imx_uart_data *pd = + container_of(chip, struct imx_uart_data, chip); + + return io_pa_or_va(&pd->base, UART_SIZE); +} + +static int imx_lpuart_getchar(struct serial_chip *chip) +{ + int ch = 0; + vaddr_t base = chip_to_base(chip); + + while (io_read32(base + STAT) & STAT_RDRF) + ; + + ch = io_read32(base + DATA) & 0x3ff; + + if (io_read32(base + STAT) & STAT_OR) + io_write32(base + STAT, STAT_OR); + + return ch; +} + +static void imx_lpuart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + STAT) & STAT_TDRE)) + ; + + io_write32(base + DATA, ch); +} + +static const struct serial_ops imx_lpuart_ops = { + .getchar = imx_lpuart_getchar, + .putc = imx_lpuart_putc, +}; +DECLARE_KEEP_PAGER(imx_lpuart_ops); + +void imx_uart_init(struct imx_uart_data *pd, paddr_t base) +{ + pd->base.pa = base; + pd->chip.ops = &imx_lpuart_ops; + + /* + * Do nothing, debug uart(sc lpuart) shared with normal world, + * everything for uart initialization is done in bootloader. + */ +} + +#ifdef CFG_DT +static struct serial_chip *imx_lpuart_dev_alloc(void) +{ + struct imx_uart_data *pd = calloc(1, sizeof(*pd)); + + if (!pd) + return NULL; + + return &pd->chip; +} + +static int imx_lpuart_dev_init(struct serial_chip *chip, const void *fdt, + int offs, const char *parms) +{ + struct imx_uart_data *pd = + container_of(chip, struct imx_uart_data, chip); + vaddr_t vbase = 0; + paddr_t pbase = 0; + size_t size = 0; + + if (parms && parms[0]) + IMSG("imx_lpuart: device parameters ignored (%s)", parms); + + if (dt_map_dev(fdt, offs, &vbase, &size, DT_MAP_AUTO) < 0) + return -1; + + pbase = virt_to_phys((void *)vbase); + imx_uart_init(pd, pbase); + + return 0; +} + +static void imx_lpuart_dev_free(struct serial_chip *chip) +{ + struct imx_uart_data *pd = + container_of(chip, struct imx_uart_data, chip); + + free(pd); +} + +static const struct serial_driver imx_lpuart_driver = { + .dev_alloc = imx_lpuart_dev_alloc, + .dev_init = imx_lpuart_dev_init, + .dev_free = imx_lpuart_dev_free, +}; + +static const struct dt_device_match imx_match_table[] = { + { .compatible = "fsl,imx7ulp-lpuart" }, + { .compatible = "fsl,imx8qm-lpuart" }, + { 0 } +}; + +DEFINE_DT_DRIVER(imx_dt_driver) = { + .name = "imx_lpuart", + .type = DT_DRIVER_UART, + .match_table = imx_match_table, + .driver = &imx_lpuart_driver, +}; + +#endif /* CFG_DT */ diff --git a/optee_os/core/drivers/imx_ocotp.c b/optee_os/core/drivers/imx_ocotp.c new file mode 100644 index 0000000..93d0bea --- /dev/null +++ b/optee_os/core/drivers/imx_ocotp.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define OCOTP_CTRL 0x0 +#define OCOTP_CTRL_ERROR BIT32(9) +#define OCOTP_CTRL_BUSY BIT32(8) + +#if defined(CFG_MX6) || defined(CFG_MX7ULP) +#define OCOTP_SHADOW_OFFSET(_b, _w) ((_b) * (0x80) + (_w) * (0x10) + 0x400) +#else +#define OCOTP_SHADOW_OFFSET(_b, _w) ((_b) * (0x40) + (_w) * (0x10) + 0x400) +#endif + +struct ocotp_instance { + unsigned char nb_banks; + unsigned char nb_words; + TEE_Result (*get_die_id)(uint64_t *ret_uid); +}; + +static vaddr_t g_base_addr; +static struct mutex fuse_read = MUTEX_INITIALIZER; +static const struct ocotp_instance *g_ocotp; + +#if defined(CFG_MX6) +static void ocotp_clock_enable(void) +{ + vaddr_t va = core_mmu_get_va(CCM_BASE, MEM_AREA_IO_SEC, CCM_SIZE); + + io_setbits32(va + CCM_CCGR2, BM_CCM_CCGR2_OCOTP_CTRL); +} +#elif defined(CFG_MX7) +static void ocotp_clock_enable(void) +{ + vaddr_t va = core_mmu_get_va(CCM_BASE, MEM_AREA_IO_SEC, CCM_SIZE); + + io_setbits32(va + CCM_CCGRx_SET(CCM_CLOCK_DOMAIN_OCOTP), + CCM_CCGRx_ALWAYS_ON(0)); +} +#elif defined(CFG_MX8M) +static void ocotp_clock_enable(void) +{ + vaddr_t va = core_mmu_get_va(CCM_BASE, MEM_AREA_IO_SEC, CCM_SIZE); + + io_setbits32(va + CCM_CCGRx_SET(CCM_CCRG_OCOTP), + CCM_CCGRx_ALWAYS_ON(0)); +} +#elif defined(CFG_MX7ULP) +/* The i.MX7ULP has the OCOTP always powered on */ +static inline void ocotp_clock_enable(void) { } +#else +#error "Platform not supported" +#endif + +static TEE_Result ocotp_ctrl_wait_for(uint32_t mask) +{ + unsigned int loop = 0; + uint32_t reg = 0; + + assert(g_base_addr); + + /* 20us delay assuming the CPU clock running at 500MHz */ + for (loop = 10000; loop > 0; loop--) { + reg = io_read32(g_base_addr + OCOTP_CTRL) & mask; + if (!reg) + return TEE_SUCCESS; + dsb(); + isb(); + } + + return TEE_ERROR_BUSY; +} + +TEE_Result imx_ocotp_read(unsigned int bank, unsigned int word, uint32_t *val) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + + if (!val) + return TEE_ERROR_BAD_PARAMETERS; + + if (bank > g_ocotp->nb_banks || word > g_ocotp->nb_words) + return TEE_ERROR_BAD_PARAMETERS; + + assert(g_base_addr && g_ocotp); + + mutex_lock(&fuse_read); + + ocotp_clock_enable(); + + /* Clear error bit */ + io_clrbits32(g_base_addr + OCOTP_CTRL, OCOTP_CTRL_ERROR); + + /* Wait for busy flag to be cleared */ + ret = ocotp_ctrl_wait_for(OCOTP_CTRL_BUSY); + if (ret) { + EMSG("OCOTP is busy"); + goto out; + } + + /* Read shadow register */ + *val = io_read32(g_base_addr + OCOTP_SHADOW_OFFSET(bank, word)); + + DMSG("OCOTP Bank %d Word %d Fuse 0x%" PRIx32, bank, word, *val); +out: + mutex_unlock(&fuse_read); + + return ret; +} + +static TEE_Result ocotp_get_die_id_mx7ulp(uint64_t *ret_uid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t val = 0; + uint64_t uid = 0; + + res = imx_ocotp_read(1, 6, &val); + if (res) + goto out; + uid = val & GENMASK_32(15, 0); + + res = imx_ocotp_read(1, 5, &val); + if (res) + goto out; + uid = SHIFT_U64(uid, 16) | (val & GENMASK_32(15, 0)); + + res = imx_ocotp_read(1, 4, &val); + if (res) + goto out; + uid = SHIFT_U64(uid, 16) | (val & GENMASK_32(15, 0)); + + res = imx_ocotp_read(1, 3, &val); + if (res) + goto out; + uid = SHIFT_U64(uid, 16) | (val & GENMASK_32(15, 0)); + +out: + if (res == TEE_SUCCESS) + *ret_uid = uid; + + return res; +} + +static TEE_Result ocotp_get_die_id_mx(uint64_t *ret_uid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t val = 0; + uint64_t uid = 0; + + res = imx_ocotp_read(0, 2, &val); + if (res) + goto out; + uid = val; + + res = imx_ocotp_read(0, 1, &val); + if (res) + goto out; + uid = SHIFT_U64(uid, 32) | val; + +out: + if (res == TEE_SUCCESS) + *ret_uid = uid; + + return res; +} + +static const struct ocotp_instance ocotp_imx6q = { + .nb_banks = 16, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx6sl = { + .nb_banks = 8, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx6sll = { + .nb_banks = 16, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx6sx = { + .nb_banks = 16, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx6ul = { + .nb_banks = 16, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx6ull = { + .nb_banks = 8, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx7d = { + .nb_banks = 8, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx7ulp = { + .nb_banks = 32, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx7ulp, +}; + +static const struct ocotp_instance ocotp_imx8m = { + .nb_banks = 32, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +static const struct ocotp_instance ocotp_imx8mp = { + .nb_banks = 48, + .nb_words = 8, + .get_die_id = ocotp_get_die_id_mx, +}; + +int tee_otp_get_die_id(uint8_t *buffer, size_t len) +{ + size_t max_size_uid = IMX_UID_SIZE; + uint64_t uid = 0; + + assert(buffer); + assert(g_base_addr && g_ocotp); + + if (g_ocotp->get_die_id(&uid)) + goto err; + + memcpy(buffer, &uid, MIN(max_size_uid, len)); + return 0; + +err: + EMSG("Error while getting die ID"); + return -1; +} + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, OCOTP_BASE, CORE_MMU_PGDIR_SIZE); +static TEE_Result imx_ocotp_init(void) +{ + g_base_addr = core_mmu_get_va(OCOTP_BASE, MEM_AREA_IO_SEC, OCOTP_SIZE); + if (!g_base_addr) + return TEE_ERROR_GENERIC; + + if (soc_is_imx6sdl() || soc_is_imx6dq() || soc_is_imx6dqp()) { + g_ocotp = &ocotp_imx6q; + } else if (soc_is_imx6sl()) { + g_ocotp = &ocotp_imx6sl; + } else if (soc_is_imx6sll()) { + g_ocotp = &ocotp_imx6sll; + } else if (soc_is_imx6sx()) { + g_ocotp = &ocotp_imx6sx; + } else if (soc_is_imx6ul()) { + g_ocotp = &ocotp_imx6ul; + } else if (soc_is_imx6ull()) { + g_ocotp = &ocotp_imx6ull; + } else if (soc_is_imx7ds()) { + g_ocotp = &ocotp_imx7d; + } else if (soc_is_imx7ulp()) { + g_ocotp = &ocotp_imx7ulp; + } else if (soc_is_imx8mm() || soc_is_imx8mn() || soc_is_imx8mq()) { + g_ocotp = &ocotp_imx8m; + } else if (soc_is_imx8mp()) { + g_ocotp = &ocotp_imx8mp; + } else { + g_ocotp = NULL; + return TEE_ERROR_NOT_SUPPORTED; + } + + return TEE_SUCCESS; +} +service_init(imx_ocotp_init); diff --git a/optee_os/core/drivers/imx_rngb.c b/optee_os/core/drivers/imx_rngb.c new file mode 100644 index 0000000..3b020fc --- /dev/null +++ b/optee_os/core/drivers/imx_rngb.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * (c) 2021 Jorge Ramirez , Foundries Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNG_VER 0x00 +#define RNG_CMD 0x04 +#define RNG_CR 0x08 +#define RNG_SR 0x0C +#define RNG_ESR 0x10 +#define RNG_OUT 0x14 + +#define RNG_CMD_CLR_INT BIT(4) +#define RNG_CMD_CLR_ERR BIT(5) + +#define RNG_CR_AR BIT(4) +#define RNG_CR_MASK_DONE BIT(5) +#define RNG_CR_MASK_ERROR BIT(6) + +#define RNG_SR_ST_DONE BIT(4) +#define RNG_SR_SEED_DONE BIT(5) +#define RNG_SR_ERROR BIT(16) +#define RNG_SR_FIFO_LEVEL_SHIFT 8 +#define RNG_SR_FIFO_LEVEL_MASK GENMASK_32(11, RNG_SR_FIFO_LEVEL_SHIFT) + +#define RNG_VER_TYPE_SHIFT 28 +#define RNG_VER_TYPE_MASK GENMASK_32(31, RNG_VER_TYPE_SHIFT) + +#define RNG_ESR_STATUS_STAT_ERR BIT(3) + +#define RNG_TYPE_RNGA 0 +#define RNG_TYPE_RNGB 1 +#define RNG_TYPE_RNGC 2 + +#define SEED_TIMEOUT 2000000 +#define IRQ_TIMEOUT 1000000 + +#define WORDS_IN_FIFO(__rng_sr) \ + (((__rng_sr) & RNG_SR_FIFO_LEVEL_MASK) >> RNG_SR_FIFO_LEVEL_SHIFT) + +#define RNG_TYPE(__rng_vr) \ + (((__rng_vr) & RNG_VER_TYPE_MASK) >> RNG_VER_TYPE_SHIFT) + +static struct imx_rng { + struct io_pa_va base; + size_t size; + bool ready; + uint32_t error; +} rngb = { + .base.pa = RNGB_BASE, + .size = 0x4000, +}; + +static void wait_for_irq(struct imx_rng *rng) +{ + uint64_t tref = timeout_init_us(IRQ_TIMEOUT); + uint32_t status = 0; + + do { + rng->error = io_read32(rng->base.va + RNG_ESR); + status = io_read32(rng->base.va + RNG_SR); + + if (timeout_elapsed(tref)) + panic(); + + } while ((status & (RNG_SR_SEED_DONE | RNG_SR_ST_DONE)) == 0); +} + +static void irq_clear(struct imx_rng *rng) +{ + io_setbits32(rng->base.va + RNG_CR, + RNG_CR_MASK_DONE | RNG_CR_MASK_ERROR); + io_setbits32(rng->base.va + RNG_CMD, + RNG_CMD_CLR_INT | RNG_CMD_CLR_ERR); +} + +static void irq_unmask(struct imx_rng *rng) +{ + io_clrbits32(rng->base.va + RNG_CR, + RNG_CR_MASK_DONE | RNG_CR_MASK_ERROR); +} + +static void rng_seed(struct imx_rng *rng) +{ + uint64_t tref = timeout_init_us(SEED_TIMEOUT); + + irq_clear(rng); + do { + irq_unmask(rng); + /* configure continuous auto-reseed */ + io_setbits32(rng->base.va + RNG_CR, RNG_CR_AR); + wait_for_irq(rng); + irq_clear(rng); + + if (timeout_elapsed(tref)) + panic(); + } while (rng->error); +} + +static TEE_Result map_controller_static(void) +{ + rngb.base.va = (vaddr_t)core_mmu_add_mapping(MEM_AREA_IO_SEC, + rngb.base.pa, rngb.size); + if (!rngb.base.va) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +#if !defined(CFG_DT) +static TEE_Result map_controller(void) +{ + return map_controller_static(); +} +#else +static const char *const rng_match_table[] = { + "fsl,imx25-rngb", +}; + +static TEE_Result map_controller(void) +{ + void *fdt = get_dt(); + unsigned int i = 0; + int off = -1; + + if (!fdt) + return map_controller_static(); + + for (i = 0; i < ARRAY_SIZE(rng_match_table); i++) { + off = fdt_node_offset_by_compatible(fdt, 0, rng_match_table[i]); + if (off >= 0) + break; + } + + if (off < 0) + return map_controller_static(); + + if (dt_enable_secure_status(fdt, off)) + return TEE_ERROR_NOT_SUPPORTED; + + if (dt_map_dev(fdt, off, &rngb.base.va, &rngb.size, DT_MAP_AUTO) < 0) + return TEE_ERROR_NOT_SUPPORTED; + + rngb.base.pa = virt_to_phys((void *)rngb.base.va); + + return TEE_SUCCESS; +} +#endif + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + uint32_t *rngbuf = buf; + uint32_t status = 0; + uint32_t val = 0; + + if (!rngb.ready) + return TEE_ERROR_BAD_STATE; + + while (len) { + status = io_read32(rngb.base.va + RNG_SR); + if (status & RNG_SR_ERROR) + return TEE_ERROR_BAD_STATE; + + if (WORDS_IN_FIFO(status)) { + val = io_read32(rngb.base.va + RNG_OUT); + if (len > sizeof(uint32_t)) { + len = len - sizeof(uint32_t); + memcpy(rngbuf, &val, sizeof(uint32_t)); + rngbuf++; + } else { + memcpy(rngbuf, &val, len); + len = 0; + } + } + } + + return TEE_SUCCESS; +} + +void plat_rng_init(void) +{ +} + +static TEE_Result rngb_init(void) +{ + uint32_t type = 0; + + if (map_controller()) + panic(); + + type = RNG_TYPE(io_read32(rngb.base.va + RNG_VER)); + if (type != RNG_TYPE_RNGB && type != RNG_TYPE_RNGC) + panic(); + + rng_seed(&rngb); + rngb.ready = true; + + return TEE_SUCCESS; +} + +driver_init(rngb_init); diff --git a/optee_os/core/drivers/imx_sc_api.c b/optee_os/core/drivers/imx_sc_api.c new file mode 100644 index 0000000..5a51252 --- /dev/null +++ b/optee_os/core/drivers/imx_sc_api.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2016 Freescale Semiconductor, Inc. + * Copyright 2017-2022 NXP + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNG_INIT_RETRY 100 + +#define SC_RPC_VERSION 1 +#define SC_RPC_MAX_MSG 8 + +/* Defines for struct sc_rpc_msg svc field */ +#define SC_RPC_SVC_PM 2 +#define SC_RPC_SVC_RM 3 +#define SC_RPC_SVC_SECO 9 + +/* Define for PM function calls */ +enum sc_pm_func { + SC_PM_FUNC_SET_RESOURCE_POWER_MODE = 3 +}; + +/* Defines for RM function calls */ +enum sc_rm_func { + SC_RM_FUNC_GET_PARTITION = 5, + SC_RM_FUNC_ASSIGN_RESOURCE = 8 +}; + +/* Define for SECO function calls */ +enum sc_seco_func { + SC_SECO_FUNC_START_RNG = 22 +}; + +/* Internal SCFW API error codes */ +enum sc_error { + SC_ERR_NONE = 0, /* Success */ + SC_ERR_VERSION, /* Incompatible API version */ + SC_ERR_CONFIG, /* Configuration error */ + SC_ERR_PARM, /* Bad parameter */ + SC_ERR_NOACCESS, /* Permission error (no access) */ + SC_ERR_LOCKED, /* Permission error (locked) */ + SC_ERR_UNAVAILABLE, /* Unavailable (out of resources) */ + SC_ERR_NOTFOUND, /* Not found */ + SC_ERR_NOPOWER, /* No power */ + SC_ERR_IPC, /* Generic IPC error */ + SC_ERR_BUSY, /* Resource is currently busy/active */ + SC_ERR_FAIL, /* General I/O failure */ + SC_ERR_LAST +}; + +/* RNG SECO states */ +enum sc_seco_rng_status { + SC_SECO_RNG_STAT_UNAVAILABLE = 0, + SC_SECO_RNG_STAT_INPROGRESS, + SC_SECO_RNG_STAT_READY +}; + +/* Resources IDs */ +enum sc_resource { + SC_RES_CAAM_JR1 = 500, + SC_RES_CAAM_JR2, + SC_RES_CAAM_JR3, + SC_RES_CAAM_JR1_OUT = 514, + SC_RES_CAAM_JR2_OUT, + SC_RES_CAAM_JR3_OUT, + SC_RES_CAAM_JR0 = 519, + SC_RES_CAAM_JR0_OUT, + SC_RES_LAST = 546 +}; + +/* Power modes */ +enum sc_power_mode { + SC_PM_PW_MODE_OFF = 0, + SC_PM_PW_MODE_STBY, + SC_PM_PW_MODE_LP, + SC_PM_PW_MODE_ON +}; + +static vaddr_t secure_ipc_addr; + +register_phys_mem(MEM_AREA_IO_SEC, SC_IPC_BASE_SECURE, SC_IPC_SIZE); + +/* + * Get the partition ID of secure world + * + * @partition Partition ID + */ +static TEE_Result sc_rm_get_partition(uint8_t *partition) +{ + TEE_Result res = TEE_ERROR_GENERIC; + enum sc_error err = SC_ERR_LAST; + struct imx_mu_msg msg = { + .header.version = SC_RPC_VERSION, + .header.size = 1, + .header.tag = SC_RPC_SVC_RM, + .header.command = SC_RM_FUNC_GET_PARTITION, + }; + + res = imx_mu_call(secure_ipc_addr, &msg, true); + if (res != TEE_SUCCESS) { + EMSG("Communication error"); + return res; + } + + err = msg.header.command; + if (err != SC_ERR_NONE) { + EMSG("Unable to get partition ID, sc_error: %d", err); + return TEE_ERROR_GENERIC; + } + + *partition = IMX_MU_DATA_U8(&msg, 0); + + return TEE_SUCCESS; +} + +/* + * Set the given power mode of a resource + * + * @resource ID of the resource + * @mode Power mode to apply + */ +static TEE_Result sc_pm_set_resource_power_mode(enum sc_resource resource, + enum sc_power_mode mode) +{ + TEE_Result res = TEE_ERROR_GENERIC; + enum sc_error scu_error = SC_ERR_LAST; + struct imx_mu_msg msg = { + .header.version = SC_RPC_VERSION, + .header.size = 2, + .header.tag = SC_RPC_SVC_PM, + .header.command = SC_PM_FUNC_SET_RESOURCE_POWER_MODE, + }; + + IMX_MU_DATA_U16(&msg, 0) = (uint16_t)resource; + IMX_MU_DATA_U8(&msg, 2) = (uint8_t)mode; + + res = imx_mu_call(secure_ipc_addr, &msg, true); + if (res != TEE_SUCCESS) { + EMSG("Communication error"); + return res; + } + + scu_error = msg.header.command; + if (scu_error != SC_ERR_NONE) { + EMSG("Unable to set resource power mode sc_error: %d", + scu_error); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +/* + * Assign ownership of a resource to the secure partition + * + * @resource Resource to assign + */ +static TEE_Result sc_rm_assign_resource(enum sc_resource resource) +{ + TEE_Result res = TEE_ERROR_GENERIC; + enum sc_error err = SC_ERR_LAST; + uint8_t secure_partition = 0; + struct imx_mu_msg msg = { + .header.version = SC_RPC_VERSION, + .header.size = 2, + .header.tag = SC_RPC_SVC_RM, + .header.command = SC_RM_FUNC_ASSIGN_RESOURCE, + }; + + res = sc_rm_get_partition(&secure_partition); + if (res != TEE_SUCCESS) { + EMSG("Cannot get secure partition ID"); + return res; + } + + IMX_MU_DATA_U16(&msg, 0) = (uint16_t)resource; + IMX_MU_DATA_U8(&msg, 2) = secure_partition; + + res = imx_mu_call(secure_ipc_addr, &msg, true); + if (res != TEE_SUCCESS) { + EMSG("Communication error"); + return res; + } + + err = msg.header.command; + if (err != SC_ERR_NONE) { + EMSG("Unable to assign resource, sc_error: %d", err); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +TEE_Result imx_sc_rm_enable_jr(unsigned int jr_index) +{ + TEE_Result res = TEE_ERROR_GENERIC; + enum sc_resource jr_res = SC_RES_LAST; + enum sc_resource jr_out_res = SC_RES_LAST; + + switch (jr_index) { + case 0: + jr_res = SC_RES_CAAM_JR0; + jr_out_res = SC_RES_CAAM_JR0_OUT; + break; + + case 1: + jr_res = SC_RES_CAAM_JR1; + jr_out_res = SC_RES_CAAM_JR1_OUT; + break; + + case 2: + jr_res = SC_RES_CAAM_JR2; + jr_out_res = SC_RES_CAAM_JR2_OUT; + break; + + case 3: + jr_res = SC_RES_CAAM_JR3; + jr_out_res = SC_RES_CAAM_JR3_OUT; + break; + + default: + EMSG("Wrong JR Index, should be 0, 1, 2 or 3"); + return TEE_ERROR_GENERIC; + } + + /* Assign JR resources to secure world */ + res = sc_rm_assign_resource(jr_res); + if (res != TEE_SUCCESS) { + EMSG("Assign SC_R_CAAM_JR%u resource failed", jr_index); + return res; + } + + res = sc_rm_assign_resource(jr_out_res); + if (res != TEE_SUCCESS) { + EMSG("Assign SC_R_CAAM_JR%u_OUT resource failed", jr_index); + return res; + } + + /* Power ON JR resources */ + res = sc_pm_set_resource_power_mode(jr_res, SC_PM_PW_MODE_ON); + if (res != TEE_SUCCESS) { + EMSG("POWER ON SC_R_CAAM_JR%u resource failed", jr_index); + return res; + } + + res = sc_pm_set_resource_power_mode(jr_out_res, SC_PM_PW_MODE_ON); + if (res != TEE_SUCCESS) { + EMSG("POWER ON SC_R_CAAM_JR%u_OUT resource failed", jr_index); + return res; + } + + return TEE_SUCCESS; +} + +TEE_Result imx_sc_seco_start_rng(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + enum sc_error err = SC_ERR_LAST; + enum sc_seco_rng_status status = SC_SECO_RNG_STAT_UNAVAILABLE; + unsigned int retry = 0; + struct imx_mu_msg msg = { + .header.version = SC_RPC_VERSION, + .header.size = 1, + .header.tag = SC_RPC_SVC_SECO, + .header.command = SC_SECO_FUNC_START_RNG, + }; + + for (retry = RNG_INIT_RETRY; retry; retry--) { + res = imx_mu_call(secure_ipc_addr, &msg, true); + if (res != TEE_SUCCESS) { + EMSG("Configuration error"); + return res; + } + + err = msg.header.command; + if (err != SC_ERR_NONE) { + EMSG("RNG status: %d", err); + return TEE_ERROR_GENERIC; + } + + status = IMX_MU_DATA_U32(&msg, 0); + + if (status == SC_SECO_RNG_STAT_READY) + return TEE_SUCCESS; + } + + return TEE_ERROR_GENERIC; +} + +TEE_Result imx_sc_driver_init(void) +{ + vaddr_t va = 0; + + va = core_mmu_get_va(SC_IPC_BASE_SECURE, MEM_AREA_IO_SEC, SC_IPC_SIZE); + if (!va) + return TEE_ERROR_GENERIC; + + imx_mu_init(va); + secure_ipc_addr = va; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/imx_scu.c b/optee_os/core/drivers/imx_scu.c new file mode 100644 index 0000000..738a60f --- /dev/null +++ b/optee_os/core/drivers/imx_scu.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2019, 2023 NXP + * + */ + +#include +#include +#include +#include +#include + +/* Invalidate all registers */ +#define SCU_INV_CTRL_INIT 0xFFFFFFFF +/* Both secure CPU access SCU */ +#define SCU_SAC_CTRL_INIT 0x0000000F +/* Both non-secure CPU access SCU, private and global timer */ +#define SCU_NSAC_CTRL_INIT 0x00000FFF + +static TEE_Result scu_init(void) +{ + vaddr_t scu_base = core_mmu_get_va(SCU_BASE, MEM_AREA_IO_SEC, + SCU_SIZE); + + if (!scu_base) + return TEE_ERROR_GENERIC; + + /* SCU config */ + io_write32(scu_base + SCU_INV_SEC, SCU_INV_CTRL_INIT); + io_write32(scu_base + SCU_SAC, SCU_SAC_CTRL_INIT); + io_write32(scu_base + SCU_NSAC, SCU_NSAC_CTRL_INIT); + + /* SCU enable */ + io_write32(scu_base + SCU_CTRL, io_read32(scu_base + SCU_CTRL) | 0x1); + + return TEE_SUCCESS; +} +driver_init(scu_init); diff --git a/optee_os/core/drivers/imx_snvs.c b/optee_os/core/drivers/imx_snvs.c new file mode 100644 index 0000000..7f05614 --- /dev/null +++ b/optee_os/core/drivers/imx_snvs.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2020 Pengutronix + * Rouven Czerwinski + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define SNVS_HPLR 0x00 +#define SNVS_HPCOMR 0x04 +#define SNVS_HPSR 0x14 +#define SNVS_LPLR 0x34 +#define SNVS_LPCR 0x38 +#define SNVS_LPMKCR 0x3C + +#define HPSR_SSM_ST_MASK GENMASK_32(11, 8) +#define HPSR_SSM_ST_SHIFT 8 + +#define SNVS_HPSR_SYS_SECURITY_CFG_OFFSET 12 +#define SNVS_HPSR_SYS_SECURITY_CFG GENMASK_32(14, 12) + +#define SNVS_HPSR_OTPMK_SYND GENMASK_32(24, 16) +#define SNVS_HPSR_OTPMK_ZERO BIT(27) + +#define SNVS_HPLR_MKS_SL BIT32(9) + +#define SNVS_LPLR_MKS_HL BIT32(9) + +#define SNVS_HPCOMR_MKS_EN BIT32(13) +#define SNVS_HPCOMR_NPSWA_EN BIT32(31) + +#define SNVS_LPMKCR_MKCR_MKS_SEL GENMASK_32(1, 0) + +#define SNVS_LPCR_TOP_MASK BIT(6) +#define SNVS_LPCR_DP_EN_MASK BIT(5) +#define SNVS_LPCR_SRTC_ENV_MASK BIT(1) + +enum snvs_ssm_mode { + SNVS_SSM_MODE_INIT, + SNVS_SSM_MODE_HARD_FAIL, + SNVS_SSM_MODE_SOFT_FAIL = 3, + SNVS_SSM_MODE_INIT_INTERMEDIATE = 8, + SNVS_SSM_MODE_CHECK, + SNVS_SSM_MODE_NON_SECURE = 11, + SNVS_SSM_MODE_TRUSTED = 13, + SNVS_SSM_MODE_SECURE = 15, +}; + +enum snvs_security_cfg { + SNVS_SECURITY_CFG_FAB, + SNVS_SECURITY_CFG_OPEN, + SNVS_SECURITY_CFG_CLOSED, + SNVS_SECURITY_CFG_FIELD_RETURN, +}; + +/* + * Return true if the master key is OTPMK, false otherwise. + */ +static bool is_otpmk_selected(void) +{ + uint32_t hp_mks = 0; + vaddr_t base = core_mmu_get_va(SNVS_BASE, MEM_AREA_IO_SEC, SNVS_SIZE); + + hp_mks = io_read32(base + SNVS_HPCOMR); + + /* + * The master key selection might be done by the MASTER_KEY_SEL field + * of LPMKCR instead. + */ + if (hp_mks & SNVS_HPCOMR_MKS_EN) { + uint32_t lp_mks = io_read32(base + SNVS_LPMKCR); + + if (lp_mks & SNVS_LPMKCR_MKCR_MKS_SEL) + return false; + } + + return true; +} + +/* + * Return true if the master key selection is locked, false otherwise. + */ +static bool is_mks_locked(void) +{ + vaddr_t base = core_mmu_get_va(SNVS_BASE, MEM_AREA_IO_SEC, SNVS_SIZE); + + return io_read32(base + SNVS_HPLR) & SNVS_HPLR_MKS_SL || + io_read32(base + SNVS_LPLR) & SNVS_LPLR_MKS_HL; +} + +/* Set the Master key to use OTPMK and lock it. */ +static void set_mks_otpmk(void) +{ + vaddr_t base = core_mmu_get_va(SNVS_BASE, MEM_AREA_IO_SEC, SNVS_SIZE); + + io_setbits32(base + SNVS_HPCOMR, SNVS_HPCOMR_MKS_EN); + io_clrbits32(base + SNVS_LPMKCR, SNVS_LPMKCR_MKCR_MKS_SEL); + io_clrbits32(base + SNVS_HPLR, SNVS_HPLR_MKS_SL); + io_setbits32(base + SNVS_LPLR, SNVS_LPLR_MKS_HL); +} + +/* + * Return true if OTPMK is valid, false otherwise. + */ +static bool is_otpmk_valid(void) +{ + vaddr_t base = core_mmu_get_va(SNVS_BASE, MEM_AREA_IO_SEC, SNVS_SIZE); + uint32_t status = io_read32(base + SNVS_HPSR); + + return !(status & (SNVS_HPSR_OTPMK_ZERO | SNVS_HPSR_OTPMK_SYND)); +} + +static enum snvs_security_cfg snvs_get_security_cfg(void) +{ + uint32_t val = 0; + vaddr_t base = core_mmu_get_va(SNVS_BASE, MEM_AREA_IO_SEC, + SNVS_SIZE); + + val = (io_read32(base + SNVS_HPSR) & SNVS_HPSR_SYS_SECURITY_CFG) >> + SNVS_HPSR_SYS_SECURITY_CFG_OFFSET; + + switch (val) { + case 0b000: + return SNVS_SECURITY_CFG_FAB; + case 0b001: + return SNVS_SECURITY_CFG_OPEN; + case 0b011: + return SNVS_SECURITY_CFG_CLOSED; + default: + return SNVS_SECURITY_CFG_FIELD_RETURN; + } +} + +bool snvs_is_device_closed(void) +{ + return (snvs_get_security_cfg() == SNVS_SECURITY_CFG_CLOSED); +} + +#ifdef CFG_RPMB_FS +static enum snvs_ssm_mode snvs_get_ssm_mode(void) +{ + vaddr_t snvs = core_mmu_get_va(SNVS_BASE, MEM_AREA_IO_SEC, + SNVS_HPSR + sizeof(uint32_t)); + uint32_t val = 0; + + val = io_read32(snvs + SNVS_HPSR); + val &= HPSR_SSM_ST_MASK; + val = val >> HPSR_SSM_ST_SHIFT; + DMSG("HPSR: SSM ST Mode: 0x%01"PRIx32, val); + return val; +} + +bool plat_rpmb_key_is_ready(void) +{ + enum snvs_ssm_mode mode = SNVS_SSM_MODE_INIT; + bool ssm_secure = false; + + mode = snvs_get_ssm_mode(); + ssm_secure = (mode == SNVS_SSM_MODE_TRUSTED || + mode == SNVS_SSM_MODE_SECURE); + + /* + * On i.MX6SDL and i.MX6DQ, the security cfg always returns + * SNVS_SECURITY_CFG_FAB (000), therefore we ignore the security + * configuration for this SoC. + */ + if (soc_is_imx6sdl() || soc_is_imx6dq()) + return ssm_secure; + + return ssm_secure && snvs_is_device_closed(); +} +#endif /* CFG_RPMB_FS */ + +TEE_Result imx_snvs_set_master_otpmk(void) +{ + if (!is_otpmk_valid()) + return TEE_ERROR_BAD_STATE; + + if (is_mks_locked()) { + if (is_otpmk_selected()) + return TEE_SUCCESS; + + return TEE_ERROR_BAD_STATE; + } + + set_mks_otpmk(); + + return TEE_SUCCESS; +} + +void imx_snvs_shutdown(void) +{ + vaddr_t base = core_mmu_get_va(SNVS_BASE, MEM_AREA_IO_SEC, SNVS_SIZE); + + io_write32(base + SNVS_LPCR, + SNVS_LPCR_TOP_MASK | + SNVS_LPCR_DP_EN_MASK | + SNVS_LPCR_SRTC_ENV_MASK); +} diff --git a/optee_os/core/drivers/imx_uart.c b/optee_os/core/drivers/imx_uart.c new file mode 100644 index 0000000..6d7bd29 --- /dev/null +++ b/optee_os/core/drivers/imx_uart.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * All rights reserved. + * Copyright 2018-2019 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* Register definitions */ +#define URXD 0x0 /* Receiver Register */ +#define UTXD 0x40 /* Transmitter Register */ +#define UCR1 0x80 /* Control Register 1 */ +#define UCR2 0x84 /* Control Register 2 */ +#define UCR3 0x88 /* Control Register 3 */ +#define UCR4 0x8c /* Control Register 4 */ +#define UFCR 0x90 /* FIFO Control Register */ +#define USR1 0x94 /* Status Register 1 */ +#define USR2 0x98 /* Status Register 2 */ +#define UESC 0x9c /* Escape Character Register */ +#define UTIM 0xa0 /* Escape Timer Register */ +#define UBIR 0xa4 /* BRM Incremental Register */ +#define UBMR 0xa8 /* BRM Modulator Register */ +#define UBRC 0xac /* Baud Rate Count Register */ +#define UTS 0xb4 /* UART Test Register (mx31) */ +#define USIZE 0xb8 /* UTS + sizeof(uint32_t) */ + +/* UART Control Register Bit Fields.*/ +#define URXD_CHARRDY (1<<15) +#define URXD_ERR (1<<14) +#define URXD_OVRRUN (1<<13) +#define URXD_FRMERR (1<<12) +#define URXD_BRK (1<<11) +#define URXD_PRERR (1<<10) +#define URXD_RX_DATA (0xFF) +#define UCR1_ADEN (1<<15) /* Auto dectect interrupt */ +#define UCR1_ADBR (1<<14) /* Auto detect baud rate */ +#define UCR1_TRDYEN (1<<13) /* Transmitter ready interrupt enable */ +#define UCR1_IDEN (1<<12) /* Idle condition interrupt */ +#define UCR1_RRDYEN (1<<9) /* Recv ready interrupt enable */ +#define UCR1_RDMAEN (1<<8) /* Recv ready DMA enable */ +#define UCR1_IREN (1<<7) /* Infrared interface enable */ +#define UCR1_TXMPTYEN (1<<6) /* Transimitter empty interrupt enable */ +#define UCR1_RTSDEN (1<<5) /* RTS delta interrupt enable */ +#define UCR1_SNDBRK (1<<4) /* Send break */ +#define UCR1_TDMAEN (1<<3) /* Transmitter ready DMA enable */ +#define UCR1_UARTCLKEN (1<<2) /* UART clock enabled */ +#define UCR1_DOZE (1<<1) /* Doze */ +#define UCR1_UARTEN (1<<0) /* UART enabled */ + +#define UTS_FRCPERR (1<<13) /* Force parity error */ +#define UTS_LOOP (1<<12) /* Loop tx and rx */ +#define UTS_TXEMPTY (1<<6) /* TxFIFO empty */ +#define UTS_RXEMPTY (1<<5) /* RxFIFO empty */ +#define UTS_TXFULL (1<<4) /* TxFIFO full */ +#define UTS_RXFULL (1<<3) /* RxFIFO full */ +#define UTS_SOFTRST (1<<0) /* Software reset */ + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct imx_uart_data *pd = + container_of(chip, struct imx_uart_data, chip); + + return io_pa_or_va(&pd->base, USIZE); +} + +static void imx_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + + while (!(io_read32(base + UTS) & UTS_TXEMPTY)) + if (!(io_read32(base + UCR1) & UCR1_UARTEN)) + return; +} + +static int imx_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (io_read32(base + UTS) & UTS_RXEMPTY) + ; + + return (io_read32(base + URXD) & URXD_RX_DATA); +} + +static void imx_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + /* Wait until there's space in the TX FIFO */ + while (io_read32(base + UTS) & UTS_TXFULL) + if (!(io_read32(base + UCR1) & UCR1_UARTEN)) + return; + + io_write32(base + UTXD, ch); +} + +static const struct serial_ops imx_uart_ops = { + .flush = imx_uart_flush, + .getchar = imx_uart_getchar, + .putc = imx_uart_putc, +}; +DECLARE_KEEP_PAGER(imx_uart_ops); + +void imx_uart_init(struct imx_uart_data *pd, paddr_t base) +{ + pd->base.pa = base; + pd->chip.ops = &imx_uart_ops; + + /* + * Do nothing, debug uart(uart0) share with normal world, + * everything for uart0 initialization is done in bootloader. + */ +} + +#ifdef CFG_DT +static struct serial_chip *imx_uart_dev_alloc(void) +{ + struct imx_uart_data *pd = calloc(1, sizeof(*pd)); + + if (!pd) + return NULL; + + return &pd->chip; +} + +static int imx_uart_dev_init(struct serial_chip *chip, const void *fdt, + int offs, const char *parms) +{ + struct imx_uart_data *pd = + container_of(chip, struct imx_uart_data, chip); + vaddr_t vbase = 0; + paddr_t pbase = 0; + size_t size = 0; + + if (parms && parms[0]) + IMSG("imx_uart: device parameters ignored (%s)", parms); + + if (dt_map_dev(fdt, offs, &vbase, &size, DT_MAP_AUTO) < 0) + return -1; + + pbase = virt_to_phys((void *)vbase); + imx_uart_init(pd, pbase); + + return 0; +} + +static void imx_uart_dev_free(struct serial_chip *chip) +{ + struct imx_uart_data *pd = + container_of(chip, struct imx_uart_data, chip); + + free(pd); +} + +static const struct serial_driver imx_uart_driver = { + .dev_alloc = imx_uart_dev_alloc, + .dev_init = imx_uart_dev_init, + .dev_free = imx_uart_dev_free, +}; + +static const struct dt_device_match imx_match_table[] = { + { .compatible = "fsl,imx6q-uart" }, + { 0 } +}; + +DEFINE_DT_DRIVER(imx_dt_driver) = { + .name = "imx_uart", + .type = DT_DRIVER_UART, + .match_table = imx_match_table, + .driver = &imx_uart_driver, +}; + +#endif /* CFG_DT */ diff --git a/optee_os/core/drivers/imx_wdog.c b/optee_os/core/drivers/imx_wdog.c new file mode 100644 index 0000000..f1642a2 --- /dev/null +++ b/optee_os/core/drivers/imx_wdog.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2019 NXP + * + * Peng Fan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool ext_reset_output __maybe_unused; +static vaddr_t wdog_base; + +void imx_wdog_restart(bool external_reset __maybe_unused) +{ + uint32_t val = 0; + + if (!wdog_base) { + EMSG("No wdog mapped\n"); + panic(); + } + +#ifdef CFG_MX7ULP + val = io_read32(wdog_base + WDOG_CS); + + io_write32(wdog_base + WDOG_CNT, UNLOCK); + /* Enable wdog */ + io_write32(wdog_base + WDOG_CS, val | WDOG_CS_EN); + + io_write32(wdog_base + WDOG_CNT, UNLOCK); + io_write32(wdog_base + WDOG_TOVAL, 1000); + io_write32(wdog_base + WDOG_CNT, REFRESH); +#else + if (external_reset && ext_reset_output) + val = 0x14; + else + val = 0x24; + + DMSG("val %x\n", val); + + io_write16(wdog_base + WDT_WCR, val); + dsb(); + + if (io_read16(wdog_base + WDT_WCR) & WDT_WCR_WDE) { + io_write16(wdog_base + WDT_WSR, WDT_SEQ1); + io_write16(wdog_base + WDT_WSR, WDT_SEQ2); + } + + io_write16(wdog_base + WDT_WCR, val); + io_write16(wdog_base + WDT_WCR, val); +#endif + + while (1) + ; +} +DECLARE_KEEP_PAGER(imx_wdog_restart); + +#if defined(CFG_DT) && !defined(CFG_EXTERNAL_DTB_OVERLAY) +static const char * const dt_wdog_match_table[] = { + "fsl,imx21-wdt", + "fsl,imx7ulp-wdt", +}; + +static TEE_Result imx_wdog_base(vaddr_t *wdog_vbase) +{ + const char *match = NULL; + void *fdt = NULL; + vaddr_t vbase = 0; + int found_off = 0; + size_t sz = 0; + int off = 0; + int st = 0; + uint32_t i = 0; + + fdt = get_dt(); + if (!fdt) { + EMSG("No DTB\n"); + return TEE_ERROR_NOT_SUPPORTED; + } + + /* search the first usable wdog */ + for (i = 0; i < ARRAY_SIZE(dt_wdog_match_table); i++) { + match = dt_wdog_match_table[i]; + off = 0; + while (off >= 0) { + off = fdt_node_offset_by_compatible(fdt, off, match); + if (off > 0) { + st = fdt_get_status(fdt, off); + if (st & DT_STATUS_OK_SEC) { + DMSG("Wdog found at %u", off); + found_off = off; + break; + } + } + } + if (found_off) + break; + else + DMSG("%s not found in DTB", dt_wdog_match_table[i]); + } + + if (!found_off) { + EMSG("No Watchdog found in DTB\n"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + ext_reset_output = dt_have_prop(fdt, found_off, + "fsl,ext-reset-output"); + + if (dt_map_dev(fdt, found_off, &vbase, &sz, DT_MAP_AUTO) < 0) { + EMSG("Failed to map Watchdog\n"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + *wdog_vbase = vbase; + + return TEE_SUCCESS; +} +#else +register_phys_mem_pgdir(MEM_AREA_IO_SEC, WDOG_BASE, CORE_MMU_PGDIR_SIZE); +static TEE_Result imx_wdog_base(vaddr_t *wdog_vbase) +{ + *wdog_vbase = (vaddr_t)phys_to_virt(WDOG_BASE, MEM_AREA_IO_SEC, 1); +#if defined(CFG_IMX_WDOG_EXT_RESET) + ext_reset_output = true; +#endif + return TEE_SUCCESS; +} +#endif + +static TEE_Result imx_wdog_init(void) +{ + return imx_wdog_base(&wdog_base); +} +driver_init(imx_wdog_init); diff --git a/optee_os/core/drivers/lpc_uart.c b/optee_os/core/drivers/lpc_uart.c new file mode 100644 index 0000000..a1e9120 --- /dev/null +++ b/optee_os/core/drivers/lpc_uart.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, HiSilicon Limited + */ + +#include +#include +#include +#include +#include +#include + +static paddr_t chip_to_base(struct serial_chip *chip) +{ + struct lpc_uart_data *pd = + container_of(chip, struct lpc_uart_data, chip); + + return io_pa_or_va(&pd->base, LPC_SIZE); +} + +static void lpc_byte_read(paddr_t addr, uint8_t *data) +{ + uint32_t status = 0; + uint32_t cnt = 0; + + io_write32(LPC_CMD_REG_OFFSET + addr, LPC_SINGLE_READ); + + io_write32(LPC_OP_LEN_REG_OFFSET + addr, 1); + io_write32(LPC_ADDR_REG_OFFSET + addr, UART_BASE + UART_LSR); + + io_write32(LPC_START_REG_OFFSET + addr, 1); + + status = io_read32(LPC_IRQ_ST_REG_OFFSET + addr); + while (!(status & LPC_IRQ_ST_ON)) { + if (cnt > UART_SEND_LOOP_MAX) + return; + cnt++; + status = io_read32(LPC_IRQ_ST_REG_OFFSET + addr); + } + + io_write32(LPC_IRQ_ST_REG_OFFSET + addr, LPC_IRQ_ST_ON); + + if (io_read32(LPC_OP_STATUS_REG_OFFSET + addr) & LPC_IRQ_ST_ON) + *data = io_read32(LPC_RDATA_REG_OFFSET + addr); +} + +static void lpc_byte_write(paddr_t addr, uint8_t data) +{ + uint32_t status = 0; + uint32_t cnt = 0; + + io_write32(LPC_CMD_REG_OFFSET + addr, LPC_SINGLE_WRITE); + io_write32(LPC_OP_LEN_REG_OFFSET + addr, 1); + io_write32(LPC_WDATA_REG_OFFSET + addr, data); + + io_write32(LPC_ADDR_REG_OFFSET + addr, UART_BASE + UART_THR); + io_write32(LPC_START_REG_OFFSET + addr, 1); + + status = io_read32(LPC_IRQ_ST_REG_OFFSET + addr); + while (!(status & LPC_IRQ_ST_ON)) { + if (cnt > UART_SEND_LOOP_MAX) + return; + cnt++; + status = io_read32(LPC_IRQ_ST_REG_OFFSET + addr); + } + + io_write32(LPC_IRQ_ST_REG_OFFSET + addr, LPC_IRQ_ST_ON); +} + +static void lpc_uart_core_putc(paddr_t base, int ch) +{ + uint8_t var = '\0'; + uint32_t i = 0; + + for (i = 0; i < UART_SEND_LOOP_MAX; i++) { + lpc_byte_read(base, &var); + if ((var & LPC_RADTA_LEN) == LPC_RADTA_LEN) + break; + } + + lpc_byte_write(base, ch); + + for (i = 0; i < UART_SEND_LOOP_MAX; i++) { + lpc_byte_read(base, &var); + if ((var & LPC_RADTA_LEN) == LPC_RADTA_LEN) + break; + } +} + +static void lpc_uart_putc(struct serial_chip *chip, int ch) +{ + paddr_t base = chip_to_base(chip); + + lpc_uart_core_putc(base, ch); +} + +static const struct serial_ops lpc_uart_ops = { + .putc = lpc_uart_putc, +}; +DECLARE_KEEP_PAGER(lpc_uart_ops); + +void lpc_uart_init(struct lpc_uart_data *pd, paddr_t base, + uint32_t uart_clk __unused, uint32_t baud_rate __unused) +{ + pd->base.pa = base; + pd->chip.ops = &lpc_uart_ops; +} + diff --git a/optee_os/core/drivers/ls_dspi.c b/optee_os/core/drivers/ls_dspi.c new file mode 100644 index 0000000..a7055e0 --- /dev/null +++ b/optee_os/core/drivers/ls_dspi.c @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * Driver for DSPI Controller + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* SPI register offset */ +#define DSPI_MCR 0x0 /* Module Configuration Register */ +#define DSPI_TCR 0x8 /* Transfer Count Register */ +#define DSPI_CTAR0 \ + 0xC /* Clock and Transfer Attributes Register (in Master mode) */ +#define DSPI_CTAR1 \ + 0x10 /* Clock and Transfer Attributes Register (in Master mode) */ +#define DSPI_SR 0x2C /* Status Register */ +#define DSPI_RSER 0x30 /* DMA/Interrupt Request Select and Enable Register */ +#define DSPI_PUSHR 0x34 /* PUSH TX FIFO Register In Master Mode */ +#define DSPI_POPR 0x38 /* POP RX FIFO Register */ +#define DSPI_TXFR0 0x3C /* Transmit FIFO Registers */ +#define DSPI_TXFR1 0x40 /* Transmit FIFO Registers */ +#define DSPI_TXFR2 0x44 /* Transmit FIFO Registers */ +#define DSPI_TXFR3 0x48 /* Transmit FIFO Registers */ +#define DSPI_RXFR0 0x7C /* Receive FIFO Registers */ +#define DSPI_RXFR1 0x80 /* Receive FIFO Registers */ +#define DSPI_RXFR2 0x84 /* Receive FIFO Registers */ +#define DSPI_RXFR3 0x88 /* Receive FIFO Registers */ +#define DSPI_CTARE0 0x11C /* Clock and Transfer Attributes Register Extended */ +#define DSPI_CTARE1 0x120 /* Clock and Transfer Attributes Register Extended */ +#define DSPI_SREX 0x13C /* Status Register Extended */ + +/* Module configuration */ +#define DSPI_MCR_MSTR 0x80000000 /* Master/Slave Mode Select [0] */ +#define DSPI_MCR_CSCK 0x40000000 /* Continuous SCK Enable [1] */ +#define DSPI_MCR_DCONF(x) (((x) & 0x03) << 28) /* SPI Configuration [2-3] */ +#define DSPI_MCR_ROOE \ + 0x01000000 /* Receive FIFO Overflow Overwrite Enable[7] */ +#define DSPI_MCR_PCSIS(x) \ + (1 << (16 + (x))) /* Peripheral Chip Select x Inactive State [12-15] */ +#define DSPI_MCR_PCSIS_MASK (0xff << 16) +#define DSPI_MCR_MDIS 0x00004000 /* Module Disable [17] */ +#define DSPI_MCR_DTXF 0x00002000 /* Disable Transmit FIFO [18] */ +#define DSPI_MCR_DRXF 0x00001000 /* Disable Receive FIFO [19] */ +#define DSPI_MCR_CTXF 0x00000800 /* Clear TX FIFO [20] */ +#define DSPI_MCR_CRXF 0x00000400 /* Clear RX FIFO [21] */ +#define DPSI_XSPI 0x00000008 /* Extended SPI Mode [28] */ +#define DSPI_MCR_PES 0x00000002 /* Parity Error Stop [30] */ +#define DSPI_MCR_HALT 0x00000001 /* Halt [31] */ +#define DPSI_ENABLE 0x0 +#define DSPI_DISABLE 0x1 + +/* Transfer count */ +#define DSPI_TCR_SPI_TCNT(x) (((x) & 0x0000FFFF) << 16) + +/* Status */ +#define DSPI_SR_TXRXS 0x40000000 /* TX and RX Status [1] */ +#define DSPI_SR_TXCTR(x) \ + (((x) & 0x0000F000) >> 12) /* TX FIFO Counter [16-19] */ +#define DSPI_SR_RXCTR(x) \ + (((x) & 0x000000F0) >> 4) /* RX FIFO Counter [24-27] */ + +#define DSPI_DATA_8BIT SHIFT_U32(8, 0) +#define DSPI_DATA_16BIT SHIFT_U32(0xF, 0) + +#define DSPI_TFR_CONT (0x80000000) +#define DSPI_TFR_CTAS(x) (((x) & 0x07) << 12) +#define DSPI_TFR_PCS(x) (((1 << (x)) & 0x0000003f) << 16) +#define DSPI_IDLE_DATA 0x0 + +/* tx/rx data wait timeout value, unit: us */ +#define DSPI_TXRX_WAIT_TIMEOUT 1000000 + +/* Transfer Fifo */ +#define DSPI_TFR_TXDATA(x) (((x) & 0x0000FFFF)) + +/* Bit definitions and macros for DRFR */ +#define DSPI_RFR_RXDATA(x) (((x) & 0x0000FFFF)) + +/* CTAR register pre-configure mask */ +#define DSPI_CTAR_SET_MODE_MASK \ + (DSPI_CTAR_FMSZ(15) | DSPI_CTAR_PCS_SCK(3) | DSPI_CTAR_PA_SCK(3) | \ + DSPI_CTAR_P_DT(3) | DSPI_CTAR_CS_SCK(15) | DSPI_CTAR_A_SCK(15) | \ + DSPI_CTAR_A_DT(15)) + +/* default SCK frequency, unit: HZ */ +#define PLATFORM_CLK 650000000 +#define DSPI_DEFAULT_SCK_FREQ 10000000 +#define DSPI_CLK_DIV 4 /* prescaler divisor */ +#define DSPI_CLK (PLATFORM_CLK / DSPI_CLK_DIV) /* DSPI clock */ +#define CS_SPEED_MAX_HZ 1000000 /* Slave max speed */ + +/* + * Calculate the divide scaler value between expected SCK frequency + * and input clk frequency + * req_pbr: pre-scaler value of baud rate for slave + * req_br: scaler value of baud rate for slave + * speed_hz: speed value of slave + * clkrate: clock value of slave + */ +static TEE_Result dspi_convert_hz_to_baud(unsigned int *req_pbr, + unsigned int *req_br, + unsigned int speed_hz, + unsigned int clkrate) +{ + /* Valid pre-scaler values for baud rate*/ + static const unsigned int pbr_val[4] = { 2, 3, 5, 7 }; + /* Valid baud rate scaler values*/ + static const unsigned int brs_val[16] = { 2, 4, 6, 8, + 16, 32, 64, 128, + 256, 512, 1024, 2048, + 4096, 8192, 16384, 32768 }; + unsigned int tmp_val = 0; + unsigned int curr_val = 0; + unsigned int i = 0; + unsigned int j = 0; + + tmp_val = clkrate / speed_hz; + + for (i = 0; i < ARRAY_SIZE(pbr_val); i++) { + for (j = 0; j < ARRAY_SIZE(brs_val); j++) { + curr_val = pbr_val[i] * brs_val[j]; + if (curr_val >= tmp_val) { + *req_pbr = i; + *req_br = j; + return TEE_SUCCESS; + } + } + } + + EMSG("Can not find valid baud rate, speed_hz is %d, ", speed_hz); + EMSG("clkrate is %d, using max prescaler value", clkrate); + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +/* + * Configure speed of slave + * dspi_data: DSPI controller chip instance + * speed: speed of slave + */ +static void dspi_setup_speed(struct ls_dspi_data *dspi_data, + unsigned int speed) +{ + TEE_Result status = TEE_ERROR_GENERIC; + unsigned int bus_setup = 0; + unsigned int bus_clock = 0; + unsigned int req_i = 0; + unsigned int req_j = 0; + + bus_clock = dspi_data->bus_clk_hz; + + DMSG("DSPI set_speed: expected SCK speed %u, bus_clk %u", speed, + bus_clock); + + bus_setup = io_read32(dspi_data->base + DSPI_CTAR0); + bus_setup &= ~(DSPI_CTAR_BRD | DSPI_CTAR_BRP(0x3) | DSPI_CTAR_BR(0xf)); + + status = dspi_convert_hz_to_baud(&req_i, &req_j, speed, bus_clock); + + /* In case of failure scenario with max speed, setting default speed */ + if (status == TEE_ERROR_ITEM_NOT_FOUND) { + speed = dspi_data->speed_hz; + status = dspi_convert_hz_to_baud(&req_i, &req_j, + speed, bus_clock); + } + + if (status == TEE_SUCCESS) { + bus_setup |= (DSPI_CTAR_BRP(req_i) | DSPI_CTAR_BR(req_j)); + io_write32(dspi_data->base + DSPI_CTAR0, bus_setup); + dspi_data->speed_hz = speed; + } else { + EMSG("Unable to set speed"); + } +} + +/* + * Transferred data to TX FIFO + * dspi_data: DSPI controller chip instance + */ +static void dspi_tx(struct ls_dspi_data *dspi_data, uint32_t ctrl, + uint16_t data) +{ + int timeout = DSPI_TXRX_WAIT_TIMEOUT; + uint32_t dspi_val_addr = dspi_data->base + DSPI_PUSHR; + uint32_t dspi_val = ctrl | data; + + /* wait for empty entries in TXFIFO or timeout */ + while (DSPI_SR_TXCTR(io_read32(dspi_data->base + DSPI_SR)) >= 4 && + timeout--) + udelay(1); + + if (timeout >= 0) + io_write32(dspi_val_addr, dspi_val); + else + EMSG("waiting timeout!"); +} + +/* + * Read data from RX FIFO + * dspi_data: DSPI controller chip instance + */ +static uint16_t dspi_rx(struct ls_dspi_data *dspi_data) +{ + int timeout = DSPI_TXRX_WAIT_TIMEOUT; + uint32_t dspi_val_addr = dspi_data->base + DSPI_POPR; + + /* wait for valid entries in RXFIFO or timeout */ + while (DSPI_SR_RXCTR(io_read32(dspi_data->base + DSPI_SR)) == 0 && + timeout--) + udelay(1); + + if (timeout >= 0) + return (uint16_t)DSPI_RFR_RXDATA(io_read32(dspi_val_addr)); + + EMSG("waiting timeout!"); + + return 0xFFFF; +} + +/* + * Transfer and Receive 8-bit data + * chip: spi_chip instance + * wdata: TX data queue + * rdata: RX data queue + * num_pkts: number of data packets + */ +static enum spi_result ls_dspi_txrx8(struct spi_chip *chip, uint8_t *wdata, + uint8_t *rdata, size_t num_pkts) +{ + uint8_t *spi_rd = NULL; + uint8_t *spi_wr = NULL; + uint32_t ctrl = 0; + struct ls_dspi_data *data = container_of(chip, struct ls_dspi_data, + chip); + unsigned int cs = data->slave_cs; + + spi_wr = wdata; + spi_rd = rdata; + + /* + * Assert PCSn signals between transfers + * select which CTAR register and slave to be used for TX + * CTAS selects which CTAR to be used, here we are using CTAR0 + * PCS (peripheral chip select) is selecting the slave. + */ + ctrl = DSPI_TFR_CTAS(data->ctar_sel) | DSPI_TFR_PCS(cs); + if (data->slave_mode & SPI_CONT) + ctrl |= DSPI_TFR_CONT; + + if (data->slave_data_size_bits != 8) { + EMSG("data_size_bits should be 8, not %u", + data->slave_data_size_bits); + return SPI_ERR_CFG; + } + + while (num_pkts) { + if (wdata && rdata) { + dspi_tx(data, ctrl, *spi_wr++); + *spi_rd++ = dspi_rx(data); + } else if (wdata) { + dspi_tx(data, ctrl, *spi_wr++); + dspi_rx(data); + } else if (rdata) { + dspi_tx(data, ctrl, DSPI_IDLE_DATA); + *spi_rd++ = dspi_rx(data); + } + num_pkts = num_pkts - 1; + } + + return SPI_OK; +} + +/* + * Transfer and Receive 16-bit data + * chip: spi_chip instance + * wdata: TX data queue + * rdata: RX data queue + * num_pkts: number of data packets + */ +static enum spi_result ls_dspi_txrx16(struct spi_chip *chip, uint16_t *wdata, + uint16_t *rdata, size_t num_pkts) +{ + uint32_t ctrl = 0; + uint16_t *spi_rd = NULL; + uint16_t *spi_wr = NULL; + struct ls_dspi_data *data = container_of(chip, struct ls_dspi_data, + chip); + unsigned int cs = data->slave_cs; + + spi_wr = wdata; + spi_rd = rdata; + + /* + * Assert PCSn signals between transfers + * select which CTAR register and slave to be used for TX + * CTAS selects which CTAR to be used, here we are using CTAR0 + * PCS (peripheral chip select) is selecting the slave. + */ + ctrl = DSPI_TFR_CTAS(data->ctar_sel) | DSPI_TFR_PCS(cs); + if (data->slave_mode & SPI_CONT) + ctrl |= DSPI_TFR_CONT; + + if (data->slave_data_size_bits != 16) { + EMSG("data_size_bits should be 16, not %u", + data->slave_data_size_bits); + return SPI_ERR_CFG; + } + + while (num_pkts) { + if (wdata && rdata) { + dspi_tx(data, ctrl, *spi_wr++); + *spi_rd++ = dspi_rx(data); + } else if (wdata) { + dspi_tx(data, ctrl, *spi_wr++); + dspi_rx(data); + } else if (rdata) { + dspi_tx(data, ctrl, DSPI_IDLE_DATA); + *spi_rd++ = dspi_rx(data); + } + num_pkts = num_pkts - 1; + } + + return SPI_OK; +} + +/* + * Statrt DSPI module + * chip: spi_chip instance + */ +static void ls_dspi_start(struct spi_chip *chip) +{ + struct ls_dspi_data *data = container_of(chip, struct ls_dspi_data, + chip); + + DMSG("Start DSPI Module"); + io_clrbits32(data->base + DSPI_MCR, DSPI_MCR_HALT); +} + +/* + * Stop DSPI module + * chip: spi_chip instance + */ +static void ls_dspi_end(struct spi_chip *chip) +{ + struct ls_dspi_data *data = container_of(chip, struct ls_dspi_data, + chip); + + /* De-assert PCSn if in CONT mode */ + if (data->slave_mode & SPI_CONT) { + unsigned int cs = data->slave_cs; + unsigned int ctrl = DSPI_TFR_CTAS(data->ctar_sel) | + DSPI_TFR_PCS(cs); + + /* Dummy read to deassert */ + dspi_tx(data, ctrl, DSPI_IDLE_DATA); + dspi_rx(data); + } + + DMSG("Stop DSPI Module"); + io_setbits32(data->base + DSPI_MCR, DSPI_MCR_HALT); +} + +/* + * Clear RX and TX FIFO + * dspi_data: DSPI controller chip instance + */ +static void dspi_flush_fifo(struct ls_dspi_data *dspi_data) +{ + unsigned int mcr_val = 0; + + mcr_val = io_read32(dspi_data->base + DSPI_MCR); + /* flush RX and TX FIFO */ + mcr_val |= (DSPI_MCR_CTXF | DSPI_MCR_CRXF); + + io_write32(dspi_data->base + DSPI_MCR, mcr_val); +} + +/* + * Flush DSPI module + * chip: spi_chip instance + */ +static void ls_dspi_flush(struct spi_chip *chip) +{ + struct ls_dspi_data *data = container_of(chip, struct ls_dspi_data, + chip); + + dspi_flush_fifo(data); +} + +/* + * Configure active state of slave + * dspi_data: DSPI controller chip instance + * cs: chip select value of slave + * state: slave mode + */ +static void dspi_set_cs_active_state(struct ls_dspi_data *dspi_data, + unsigned int cs, unsigned int state) +{ + DMSG("Set CS active state cs=%d state=%d", cs, state); + + if (state & SPI_CS_HIGH) + /* CSx inactive state is low */ + io_clrbits32(dspi_data->base + DSPI_MCR, DSPI_MCR_PCSIS(cs)); + else + /* CSx inactive state is high */ + io_setbits32(dspi_data->base + DSPI_MCR, DSPI_MCR_PCSIS(cs)); +} + +/* + * Configure transfer state of slave + * dspi_data: DSPI controller chip instance + * state: slave mode + */ +static void dspi_set_transfer_state(struct ls_dspi_data *dspi_data, + unsigned int state) +{ + unsigned int bus_setup = 0; + + DMSG("Set transfer state=%d bits=%d", state, + dspi_data->slave_data_size_bits); + + bus_setup = io_read32(dspi_data->base + DSPI_CTAR0); + bus_setup &= ~DSPI_CTAR_SET_MODE_MASK; + bus_setup |= dspi_data->ctar_val; + bus_setup &= ~(DSPI_CTAR_CPOL | DSPI_CTAR_CPHA | DSPI_CTAR_LSBFE); + + if (state & SPI_CPOL) + bus_setup |= DSPI_CTAR_CPOL; + if (state & SPI_CPHA) + bus_setup |= DSPI_CTAR_CPHA; + if (state & SPI_LSB_FIRST) + bus_setup |= DSPI_CTAR_LSBFE; + + if (dspi_data->slave_data_size_bits == 8) + bus_setup |= DSPI_CTAR_FMSZ(7); + else if (dspi_data->slave_data_size_bits == 16) + bus_setup |= DSPI_CTAR_FMSZ(15); + + if (dspi_data->ctar_sel == 0) + io_write32(dspi_data->base + DSPI_CTAR0, bus_setup); + else + io_write32(dspi_data->base + DSPI_CTAR1, bus_setup); +} + +/* + * Configure speed of slave + * dspi_data: DSPI controller chip instance + * speed_max_hz: maximum speed for slave + */ +static void dspi_set_speed(struct ls_dspi_data *dspi_data, + unsigned int speed_max_hz) +{ + dspi_setup_speed(dspi_data, speed_max_hz); +} + +/* + * Configure slave for DSPI controller + * dspi_data: DSPI controller chip instance + * cs: chip select value of slave + * speed_max_hz: maximum speed of slave + * state: slave mode + */ +static void dspi_config_slave_state(struct ls_dspi_data *dspi_data, + unsigned int cs, unsigned int speed_max_hz, + unsigned int state) +{ + unsigned int sr_val = 0; + + /* configure speed */ + dspi_set_speed(dspi_data, speed_max_hz); + + /* configure transfer state */ + dspi_set_transfer_state(dspi_data, state); + + /* configure active state of CSX */ + dspi_set_cs_active_state(dspi_data, cs, state); + + /* clear FIFO */ + dspi_flush_fifo(dspi_data); + + /* check module TX and RX status */ + sr_val = io_read32(dspi_data->base + DSPI_SR); + if ((sr_val & DSPI_SR_TXRXS) != DSPI_SR_TXRXS) + EMSG("DSPI RX/TX not ready"); +} + +/* + * Configure master for DSPI controller + * dspi_data: DSPI controller chip instance + * mcr_val: value of master configuration register + */ +static void dspi_set_master_state(struct ls_dspi_data *dspi_data, + unsigned int mcr_val) +{ + DMSG("Set master state val=0x%x", mcr_val); + io_write32(dspi_data->base + DSPI_MCR, mcr_val); +} + +/* + * Configure DSPI controller + * chip: spi_chip instance + */ +static void ls_dspi_configure(struct spi_chip *chip) +{ + struct ls_dspi_data *data = container_of(chip, struct ls_dspi_data, + chip); + unsigned int mcr_cfg_val = 0; + + mcr_cfg_val = DSPI_MCR_MSTR | DSPI_MCR_PCSIS_MASK | DSPI_MCR_CRXF | + DSPI_MCR_CTXF; + + /* Configure Master */ + dspi_set_master_state(data, mcr_cfg_val); + + /* Configure DSPI slave */ + dspi_config_slave_state(data, data->slave_cs, data->slave_speed_max_hz, + data->slave_mode); +} + +/* + * Extract information for DSPI Controller from the DTB + * dspi_data: DSPI controller chip instance + */ +static TEE_Result get_info_from_device_tree(struct ls_dspi_data *dspi_data) +{ + const fdt32_t *bus_num = NULL; + const fdt32_t *chip_select_num = NULL; + size_t size = 0; + int node = 0; + vaddr_t ctrl_base = 0; + void *fdt = NULL; + + /* + * First get the DSPI Controller base address from the DTB + * if DTB present and if the DSPI Controller defined in it. + */ + fdt = get_dt(); + if (!fdt) { + EMSG("Unable to get DTB, DSPI init failed\n"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + node = 0; + while (node != -FDT_ERR_NOTFOUND) { + node = fdt_node_offset_by_compatible(fdt, node, + "fsl,lx2160a-dspi"); + if (!(fdt_get_status(fdt, node) & DT_STATUS_OK_SEC)) + continue; + + bus_num = fdt_getprop(fdt, node, "bus-num", NULL); + if (bus_num && dspi_data->slave_bus == + (unsigned int)fdt32_to_cpu(*bus_num)) { + if (dt_map_dev(fdt, node, &ctrl_base, &size, + DT_MAP_AUTO) < 0) { + EMSG("Unable to get virtual address"); + return TEE_ERROR_GENERIC; + } + break; + } + } + + dspi_data->base = ctrl_base; + dspi_data->bus_clk_hz = DSPI_CLK; + + chip_select_num = fdt_getprop(fdt, node, "spi-num-chipselects", NULL); + if (chip_select_num) + dspi_data->num_chipselect = (int)fdt32_to_cpu(*chip_select_num); + else + return TEE_ERROR_ITEM_NOT_FOUND; + + dspi_data->speed_hz = DSPI_DEFAULT_SCK_FREQ; + + return TEE_SUCCESS; +} + +static const struct spi_ops ls_dspi_ops = { + .configure = ls_dspi_configure, + .start = ls_dspi_start, + .txrx8 = ls_dspi_txrx8, + .txrx16 = ls_dspi_txrx16, + .end = ls_dspi_end, + .flushfifo = ls_dspi_flush, +}; +DECLARE_KEEP_PAGER(ls_dspi_ops); + +TEE_Result ls_dspi_init(struct ls_dspi_data *dspi_data) +{ + TEE_Result status = TEE_ERROR_GENERIC; + + /* + * First get the DSPI Controller base address from the DTB, + * if DTB present and if the DSPI Controller defined in it. + */ + if (dspi_data) + status = get_info_from_device_tree(dspi_data); + if (status == TEE_SUCCESS) + /* generic DSPI chip handle */ + dspi_data->chip.ops = &ls_dspi_ops; + else + EMSG("Unable to get info from device tree"); + + return status; +} diff --git a/optee_os/core/drivers/ls_gpio.c b/optee_os/core/drivers/ls_gpio.c new file mode 100644 index 0000000..5d4a297 --- /dev/null +++ b/optee_os/core/drivers/ls_gpio.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * Driver for GPIO Controller + * + */ + +#include +#include +#include +#include +#include +#include +#include + +static const char * const gpio_controller_map[] = { + "/soc/gpio@2300000", + "/soc/gpio@2310000", + "/soc/gpio@2320000", + "/soc/gpio@2330000" +}; + +/* + * Get value from GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin from which value needs to be read + */ +static enum gpio_level ls_gpio_get_value(struct gpio_chip *chip, + unsigned int gpio_pin) +{ + vaddr_t gpio_data_addr = 0; + uint32_t data = 0; + struct ls_gpio_chip_data *gc_data = container_of(chip, + struct ls_gpio_chip_data, + chip); + + assert(gpio_pin <= MAX_GPIO_PINS); + + gpio_data_addr = gc_data->gpio_base + GPIODAT; + data = io_read32(gpio_data_addr); + + if (data & PIN_SHIFT(gpio_pin)) + return GPIO_LEVEL_HIGH; + else + return GPIO_LEVEL_LOW; +} + +/* + * Set value for GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin to which value needs to be write + * value: value needs to be written to the pin + */ +static void ls_gpio_set_value(struct gpio_chip *chip, unsigned int gpio_pin, + enum gpio_level value) +{ + vaddr_t gpio_data_addr = 0; + struct ls_gpio_chip_data *gc_data = container_of(chip, + struct ls_gpio_chip_data, + chip); + + assert(gpio_pin <= MAX_GPIO_PINS); + + gpio_data_addr = gc_data->gpio_base + GPIODAT; + + if (value == GPIO_LEVEL_HIGH) + /* if value is high then set pin value */ + io_setbits32(gpio_data_addr, PIN_SHIFT(gpio_pin)); + else + /* if value is low then clear pin value */ + io_clrbits32(gpio_data_addr, PIN_SHIFT(gpio_pin)); +} + +/* + * Get direction from GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin from which direction needs to be read + */ +static enum gpio_dir ls_gpio_get_direction(struct gpio_chip *chip, + unsigned int gpio_pin) +{ + vaddr_t gpio_dir_addr = 0; + uint32_t data = 0; + struct ls_gpio_chip_data *gc_data = container_of(chip, + struct ls_gpio_chip_data, + chip); + + assert(gpio_pin <= MAX_GPIO_PINS); + + gpio_dir_addr = gc_data->gpio_base + GPIODIR; + data = io_read32(gpio_dir_addr); + + if (data & PIN_SHIFT(gpio_pin)) + return GPIO_DIR_OUT; + else + return GPIO_DIR_IN; +} + +/* + * Set direction for GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin on which direction needs to be set + * direction: direction which needs to be set on pin + */ +static void ls_gpio_set_direction(struct gpio_chip *chip, unsigned int gpio_pin, + enum gpio_dir direction) +{ + vaddr_t gpio_dir_addr = 0; + struct ls_gpio_chip_data *gc_data = container_of(chip, + struct ls_gpio_chip_data, + chip); + + assert(gpio_pin <= MAX_GPIO_PINS); + + gpio_dir_addr = gc_data->gpio_base + GPIODIR; + + if (direction == GPIO_DIR_OUT) + io_setbits32(gpio_dir_addr, PIN_SHIFT(gpio_pin)); + else + io_clrbits32(gpio_dir_addr, PIN_SHIFT(gpio_pin)); +} + +/* + * Get interrupt from GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin from which interrupt value needs to be read + */ +static enum gpio_interrupt gpio_get_interrupt(struct gpio_chip *chip, + unsigned int gpio_pin) +{ + vaddr_t gpio_ier_addr = 0; + uint32_t data = 0; + struct ls_gpio_chip_data *gc_data = container_of(chip, + struct ls_gpio_chip_data, + chip); + + assert(gpio_pin <= MAX_GPIO_PINS); + + gpio_ier_addr = gc_data->gpio_base + GPIOIER; + data = io_read32(gpio_ier_addr); + + if (data & PIN_SHIFT(gpio_pin)) + return GPIO_INTERRUPT_ENABLE; + else + return GPIO_INTERRUPT_DISABLE; +} + +/* + * Set interrupt event for GPIO controller + * chip: pointer to GPIO controller chip instance + * gpio_pin: pin on which interrupt value needs to be set + * interrupt: interrupt valie which needs to be set on pin + */ +static void gpio_set_interrupt(struct gpio_chip *chip, unsigned int gpio_pin, + enum gpio_interrupt interrupt) +{ + vaddr_t gpio_ier_addr = 0; + struct ls_gpio_chip_data *gc_data = container_of(chip, + struct ls_gpio_chip_data, + chip); + + assert(gpio_pin <= MAX_GPIO_PINS); + + gpio_ier_addr = gc_data->gpio_base + GPIOIER; + + if (interrupt == GPIO_INTERRUPT_ENABLE) + io_setbits32(gpio_ier_addr, PIN_SHIFT(gpio_pin)); + else + io_clrbits32(gpio_ier_addr, PIN_SHIFT(gpio_pin)); +} + +/* + * Extract information for GPIO Controller from the DTB + * gpio_data: GPIO controller chip instance + */ +static TEE_Result get_info_from_device_tree(struct ls_gpio_chip_data *gpio_data) +{ + size_t size = 0; + int node = 0; + vaddr_t ctrl_base = 0; + void *fdt = NULL; + + /* + * First get the GPIO Controller base address from the DTB + * if DTB present and if the GPIO Controller defined in it. + */ + fdt = get_embedded_dt(); + if (!fdt) { + EMSG("Unable to get the Embedded DTB, GPIO init failed\n"); + return TEE_ERROR_GENERIC; + } + + node = fdt_path_offset(fdt, gpio_controller_map + [gpio_data->gpio_controller]); + if (node > 0) { + if (dt_map_dev(fdt, node, &ctrl_base, &size, + DT_MAP_AUTO) < 0) { + EMSG("Unable to get virtual address"); + return TEE_ERROR_GENERIC; + } + } else { + EMSG("Unable to get gpio offset node"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + gpio_data->gpio_base = ctrl_base; + + return TEE_SUCCESS; +} + +static const struct gpio_ops ls_gpio_ops = { + .get_direction = ls_gpio_get_direction, + .set_direction = ls_gpio_set_direction, + .get_value = ls_gpio_get_value, + .set_value = ls_gpio_set_value, + .get_interrupt = gpio_get_interrupt, + .set_interrupt = gpio_set_interrupt, +}; +DECLARE_KEEP_PAGER(ls_gpio_ops); + +TEE_Result ls_gpio_init(struct ls_gpio_chip_data *gpio_data) +{ + TEE_Result status = TEE_ERROR_GENERIC; + + /* + * First get the GPIO Controller base address from the DTB, + * if DTB present and if the GPIO Controller defined in it. + */ + status = get_info_from_device_tree(gpio_data); + if (status == TEE_SUCCESS) { + /* set GPIO Input Buffer Enable register */ + io_setbits32(gpio_data->gpio_base + GPIOIBE, UINT32_MAX); + + /* generic GPIO chip handle */ + gpio_data->chip.ops = &ls_gpio_ops; + } else { + EMSG("Unable to get info from device tree"); + } + + return status; +} diff --git a/optee_os/core/drivers/ls_i2c.c b/optee_os/core/drivers/ls_i2c.c new file mode 100644 index 0000000..87d7381 --- /dev/null +++ b/optee_os/core/drivers/ls_i2c.c @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + * + * I2C driver for I2C Controller + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char * const i2c_controller_map[] = { + "/soc/i2c@2000000", "/soc/i2c@2010000", "/soc/i2c@2020000", + "/soc/i2c@2030000", "/soc/i2c@2040000", "/soc/i2c@2050000", + "/soc/i2c@2060000", "/soc/i2c@2070000" +}; + +/* + * I2C divisor and ibfd register values when glitch filter is enabled + * In case of duplicate SCL divisor value, the ibfd value with high MUL value + * has been selected. A higher MUL value results in a lower sampling rate of + * the I2C signals. This gives the I2C module greater immunity against glitches + * in the I2C signals. + */ +static const struct i2c_clock_divisor_pair clk_div_glitch_enabled[] = { + { 34, 0x0 }, { 36, 0x1 }, { 38, 0x2 }, { 40, 0x3 }, + { 42, 0x4 }, { 44, 0x8 }, { 48, 0x9 }, { 52, 0xA }, + { 54, 0x7 }, { 56, 0xB }, { 60, 0xC }, { 64, 0x10 }, + { 68, 0x40 }, { 72, 0x41 }, { 76, 0x42 }, { 80, 0x43 }, + { 84, 0x44 }, { 88, 0x48 }, { 96, 0x49 }, { 104, 0x4A }, + { 108, 0x47 }, { 112, 0x4B }, { 120, 0x4C }, { 128, 0x50 }, + { 136, 0x80 }, { 144, 0x81 }, { 152, 0x82 }, { 160, 0x83 }, + { 168, 0x84 }, { 176, 0x88 }, { 192, 0x89 }, { 208, 0x8A }, + { 216, 0x87 }, { 224, 0x8B }, { 240, 0x8C }, { 256, 0x90 }, + { 288, 0x91 }, { 320, 0x92 }, { 336, 0x8F }, { 352, 0x93 }, + { 384, 0x98 }, { 416, 0x95 }, { 448, 0x99 }, { 480, 0x96 }, + { 512, 0x9A }, { 576, 0x9B }, { 640, 0xA0 }, { 704, 0x9D }, + { 768, 0xA1 }, { 832, 0x9E }, { 896, 0xA2 }, { 960, 0x67 }, + { 1024, 0xA3 }, { 1152, 0xA4 }, { 1280, 0xA8 }, { 1536, 0xA9 }, + { 1792, 0xAA }, { 1920, 0xA7 }, { 2048, 0xAB }, { 2304, 0xAC }, + { 2560, 0xB0 }, { 3072, 0xB1 }, { 3584, 0xB2 }, { 3840, 0xAF }, + { 4096, 0xB3 }, { 4608, 0xB4 }, { 5120, 0xB8 }, { 6144, 0xB9 }, + { 7168, 0xBA }, { 7680, 0xB7 }, { 8192, 0xBB }, { 9216, 0xBC }, + { 10240, 0xBD }, { 12288, 0xBE }, { 15360, 0xBF } +}; + +/* + * I2C divisor and ibfd register values when glitch filter is disabled. + * In case of duplicate SCL divisor value, the ibfd value with high MUL value + * has been selected. A higher MUL value results in a lower sampling rate of + * the I2C signals. This gives the I2C module greater immunity against glitches + * in the I2C signals. + */ +static const struct i2c_clock_divisor_pair clk_div_glitch_disabled[] = { + { 20, 0x0 }, { 22, 0x1 }, { 24, 0x2 }, { 26, 0x3 }, + { 28, 0x8 }, { 30, 0x5 }, { 32, 0x9 }, { 34, 0x6 }, + { 36, 0x0A }, { 40, 0x40 }, { 44, 0x41 }, { 48, 0x42 }, + { 52, 0x43 }, { 56, 0x48 }, { 60, 0x45 }, { 64, 0x49 }, + { 68, 0x46 }, { 72, 0x4A }, { 80, 0x80 }, { 88, 0x81 }, + { 96, 0x82 }, { 104, 0x83 }, { 112, 0x88 }, { 120, 0x85 }, + { 128, 0x89 }, { 136, 0x86 }, { 144, 0x8A }, { 160, 0x8B }, + { 176, 0x8C }, { 192, 0x90 }, { 208, 0x56 }, { 224, 0x91 }, + { 240, 0x1F }, { 256, 0x92 }, { 272, 0x8F }, { 288, 0x93 }, + { 320, 0x98 }, { 352, 0x95 }, { 384, 0x99 }, { 416, 0x96 }, + { 448, 0x9A }, { 480, 0x5F }, { 512, 0x9B }, { 576, 0x9C }, + { 640, 0xA0 }, { 768, 0xA1 }, { 896, 0xA2 }, { 960, 0x9F }, + { 1024, 0xA3 }, { 1152, 0xA4 }, { 1280, 0xA8 }, { 1536, 0xA9 }, + { 1792, 0xAA }, { 1920, 0xA7 }, { 2048, 0xAB }, { 2304, 0xAC }, + { 2560, 0xAD }, { 3072, 0xB1 }, { 3584, 0xB2 }, { 3840, 0xAF }, + { 4096, 0xB3 }, { 4608, 0xB4 }, { 5120, 0xB8 }, { 6144, 0xB9 }, + { 7168, 0xBA }, { 7680, 0xB7 }, { 8192, 0xBB }, { 9216, 0xBC }, + { 10240, 0xBD }, { 12288, 0xBE }, { 15360, 0xBF } +}; + +void i2c_reset(vaddr_t base) +{ + struct i2c_regs *regs = (struct i2c_regs *)base; + + io_setbits8((vaddr_t)®s->ibcr, I2C_IBCR_MDIS); + io_setbits8((vaddr_t)®s->ibsr, I2C_IBSR_IBAL | I2C_IBSR_IBIF); + io_clrbits8((vaddr_t)®s->ibcr, I2C_IBCR_IBIE | I2C_IBCR_DMAEN); + io_clrbits8((vaddr_t)®s->ibic, I2C_IBIC_BIIE); +} + +/* + * Get I2c Bus Frequency Divider Register value based on clock_divisor + * and if the glitch is enabled or not in I2c controller. + * base Base address of I2C controller + * clock_divisor Clock Divisor + */ +static uint8_t i2c_get_ibfd(vaddr_t base, uint16_t clock_divisor) +{ + struct i2c_regs *regs = (struct i2c_regs *)base; + const struct i2c_clock_divisor_pair *dpair = NULL; + size_t dpair_sz = 0; + unsigned int n = 0; + + if (io_read8((vaddr_t)®s->ibdbg) & I2C_IBDBG_GLFLT_EN) { + dpair = clk_div_glitch_enabled; + dpair_sz = ARRAY_SIZE(clk_div_glitch_enabled); + } else { + dpair = clk_div_glitch_disabled; + dpair_sz = ARRAY_SIZE(clk_div_glitch_disabled); + } + + for (n = 0; n < dpair_sz - 1; n++) + if (clock_divisor < dpair[n].divisor) + break; + + return dpair[n].ibfd; +} + +TEE_Result i2c_init(struct ls_i2c_data *i2c_data) +{ + struct i2c_regs *regs = NULL; + uint16_t clock_divisor = 0; + uint8_t ibfd = 0; /* I2c Bus Frequency Divider Register */ + size_t size = 0; + int node = 0; + vaddr_t ctrl_base = 0; + void *fdt = NULL; + + /* + * First get the I2C Controller base address from the DTB + * if DTB present and if the I2C Controller defined in it. + */ + fdt = get_embedded_dt(); + if (!fdt) { + EMSG("Unable to get the Embedded DTB, I2C init failed\n"); + return TEE_ERROR_GENERIC; + } + + node = fdt_path_offset(fdt, + i2c_controller_map[i2c_data->i2c_controller]); + if (node > 0) { + if (dt_map_dev(fdt, node, &ctrl_base, &size, + DT_MAP_AUTO) < 0) { + EMSG("Unable to get virtual address"); + return TEE_ERROR_GENERIC; + } + } else { + EMSG("Unable to get I2C offset node"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + i2c_data->base = ctrl_base; + + regs = (struct i2c_regs *)ctrl_base; + + clock_divisor = (i2c_data->i2c_bus_clock + i2c_data->speed - 1) / + i2c_data->speed; + ibfd = i2c_get_ibfd(ctrl_base, clock_divisor); + + io_write8((vaddr_t)®s->ibfd, ibfd); + + i2c_reset(ctrl_base); + + return TEE_SUCCESS; +} + +/* + * Check if I2C bus is busy with previous transaction or not. + * regs pointer to I2c controller registers + * test_busy this flag tells if we need to check the busy bit in IBSR reg + */ +static TEE_Result i2c_bus_test_bus_busy(struct i2c_regs *regs, bool test_busy) +{ + unsigned int n = 0; + uint8_t reg = 0; + + for (n = 0; n < I2C_NUM_RETRIES; n++) { + reg = io_read8((vaddr_t)®s->ibsr); + + if (reg & I2C_IBSR_IBAL) { + io_write8((vaddr_t)®s->ibsr, reg); + return TEE_ERROR_BUSY; + } + + if (test_busy && (reg & I2C_IBSR_IBB)) + break; + + if (!test_busy && !(reg & I2C_IBSR_IBB)) + break; + + mdelay(1); + } + + if (n == I2C_NUM_RETRIES) + return TEE_ERROR_BUSY; + + return TEE_SUCCESS; +} + +/* + * Check if data transfer to/from i2c controller is complete. + * regs pointer to I2c controller registers + * test_rx_ack this flag tells if we need to check RXAK bit in IBSR reg + */ +static TEE_Result i2c_transfer_complete(struct i2c_regs *regs, bool test_rx_ack) +{ + unsigned int n = 0; + uint8_t reg = 0; + + for (n = 0; n < I2C_NUM_RETRIES; n++) { + reg = io_read8((vaddr_t)®s->ibsr); + + if (reg & I2C_IBSR_IBIF) { + /* Write 1 to clear the IBIF field */ + io_write8((vaddr_t)®s->ibsr, reg); + break; + } + mdelay(1); + } + + if (n == I2C_NUM_RETRIES) + return TEE_ERROR_BUSY; + + if (test_rx_ack && (reg & I2C_IBSR_RXAK)) + return TEE_ERROR_NO_DATA; + + if (reg & I2C_IBSR_TCF) + return TEE_SUCCESS; + + return TEE_ERROR_GENERIC; +} + +/* + * Read data from I2c controller. + * regs pointer to I2c controller registers + * slave_address slave address from which to read + * operation pointer to i2c_operation struct + * is_last_operation if current operation is last operation + */ +static TEE_Result i2c_read(struct i2c_regs *regs, unsigned int slave_address, + struct i2c_operation *operation, + bool is_last_operation) +{ + TEE_Result res = TEE_ERROR_GENERIC; + unsigned int n = 0; + + /* Write Slave Address */ + io_write8((vaddr_t)®s->ibdr, (slave_address << 0x1) | BIT(0)); + res = i2c_transfer_complete(regs, I2C_BUS_TEST_RX_ACK); + if (res) + return res; + + /* select Receive mode. */ + io_clrbits8((vaddr_t)®s->ibcr, I2C_IBCR_TXRX); + if (operation->length_in_bytes > 1) { + /* Set No ACK = 0 */ + io_clrbits8((vaddr_t)®s->ibcr, I2C_IBCR_NOACK); + } + + /* Perform a dummy read to initiate the receive operation. */ + io_read8((vaddr_t)®s->ibdr); + + for (n = 0; n < operation->length_in_bytes; n++) { + res = i2c_transfer_complete(regs, I2C_BUS_NO_TEST_RX_ACK); + if (res) + return res; + if (n == (operation->length_in_bytes - 2)) { + /* Set No ACK = 1 */ + io_setbits8((vaddr_t)®s->ibcr, I2C_IBCR_NOACK); + } else if (n == (operation->length_in_bytes - 1)) { + if (!is_last_operation) { + /* select Transmit mode (for repeat start) */ + io_setbits8((vaddr_t)®s->ibcr, + I2C_IBCR_TXRX); + } else { + /* Generate Stop Signal */ + io_clrbits8((vaddr_t)®s->ibcr, + (I2C_IBCR_MSSL | I2C_IBCR_TXRX)); + res = i2c_bus_test_bus_busy(regs, + I2C_BUS_TEST_IDLE); + if (res) + return res; + } + } + operation->buffer[n] = io_read8((vaddr_t)®s->ibdr); + } + + return TEE_SUCCESS; +} + +/* + * Write data to I2c controller + * regs pointer to I2c controller registers + * slave_address slave address from which to read + * operation pointer to i2c_operation struct + */ +static TEE_Result i2c_write(struct i2c_regs *regs, unsigned int slave_address, + struct i2c_operation *operation) +{ + TEE_Result res = TEE_ERROR_GENERIC; + unsigned int n = 0; + + /* Write Slave Address */ + io_write8((vaddr_t)®s->ibdr, + (slave_address << 0x1) & ~(BIT(0))); + res = i2c_transfer_complete(regs, I2C_BUS_TEST_RX_ACK); + if (res) + return res; + + /* Write Data */ + for (n = 0; n < operation->length_in_bytes; n++) { + io_write8((vaddr_t)®s->ibdr, operation->buffer[n]); + res = i2c_transfer_complete(regs, I2C_BUS_TEST_RX_ACK); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +/* + * Generate Stop Signal and disable I2C controller. + * regs pointer to I2c controller registers + */ +static TEE_Result i2c_stop(struct i2c_regs *regs) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t reg = 0; + + reg = io_read8((vaddr_t)®s->ibsr); + if (reg & I2C_IBSR_IBB) { + /* Generate Stop Signal */ + io_clrbits8((vaddr_t)®s->ibcr, + I2C_IBCR_MSSL | I2C_IBCR_TXRX); + res = i2c_bus_test_bus_busy(regs, I2C_BUS_TEST_IDLE); + if (res) + return res; + } + + /* Disable I2c Controller */ + io_setbits8((vaddr_t)®s->ibcr, I2C_IBCR_MDIS); + + return TEE_SUCCESS; +} + +/* + * Generate Start Signal and set I2C controller in transmit mode. + * regs pointer to I2c controller registers + */ +static TEE_Result i2c_start(struct i2c_regs *regs) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + io_setbits8((vaddr_t)®s->ibsr, I2C_IBSR_IBAL | I2C_IBSR_IBIF); + io_clrbits8((vaddr_t)®s->ibcr, I2C_IBCR_MDIS); + + /* Wait controller to be stable */ + mdelay(1); + + /* Generate Start Signal */ + io_setbits8((vaddr_t)®s->ibcr, I2C_IBCR_MSSL); + res = i2c_bus_test_bus_busy(regs, I2C_BUS_TEST_BUSY); + if (res) + return res; + + /* Select Transmit Mode. set No ACK = 1 */ + io_setbits8((vaddr_t)®s->ibcr, I2C_IBCR_TXRX | I2C_IBCR_NOACK); + + return TEE_SUCCESS; +} + +TEE_Result i2c_bus_xfer(vaddr_t base, unsigned int slave_address, + struct i2c_operation *i2c_operation, + unsigned int operation_count) +{ + unsigned int n = 0; + struct i2c_regs *regs = (struct i2c_regs *)base; + struct i2c_operation *operation = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + bool is_last_operation = false; + + res = i2c_bus_test_bus_busy(regs, I2C_BUS_TEST_IDLE); + if (res) + goto out; + + res = i2c_start(regs); + if (res) + goto out; + + for (n = 0, operation = i2c_operation; + n < operation_count; n++, operation++) { + if (n == (operation_count - 1)) + is_last_operation = true; + + /* Send repeat start after first transmit/receive */ + if (n) { + io_setbits8((vaddr_t)®s->ibcr, I2C_IBCR_RSTA); + res = i2c_bus_test_bus_busy(regs, I2C_BUS_TEST_BUSY); + if (res) + goto out; + } + + /* Read/write data */ + if (operation->flags & I2C_FLAG_READ) + res = i2c_read(regs, slave_address, operation, + is_last_operation); + else + res = i2c_write(regs, slave_address, operation); + if (res) + goto out; + } + +out: + i2c_stop(regs); + + return res; +} diff --git a/optee_os/core/drivers/ls_sec_mon.c b/optee_os/core/drivers/ls_sec_mon.c new file mode 100644 index 0000000..f296aac --- /dev/null +++ b/optee_os/core/drivers/ls_sec_mon.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microsoft + * + * Driver for the NXP LX2160A-series Security Monitor (SecMon). + */ + +#include +#include +#include +#include +#include +#include +#include + +/** + * struct ls_sec_mon_registers - Memory map of the SecMon registers. + * hplr; HP Lock Register. + * @hpcomr: HP Command Register. + * @rsvd0: Reserved. + * @hpsicr: HP Security Interrupt Control Register. + * @hpsvcr: HP Security Violation Control Register. + * @hpsr: HP Status Register. + * @hpsvsr: HP Security Violation Status Register. + * @hphacivr: HP High Assurance Counter IV Register. + * @hphacr: HP High Assurance Counter Register. + * @rsvd1[0x4]: Reserved. + * @lplr: LP Lock Register. + * @lpcr: LP Control Register. + * @lpmkcr: LP Master Key Control Register. + * @lpsvcr: LP Security Violation Control Register. + * @rsvd2: Reserved. + * @lptdcr: LP Tamper Detectors Configuration. + * @lpsr: LP Status Register. + * @rsvd3[0x3]: Reserved. + * @lpsmcmr: LP Secure Monotonic Counter MSB Register. + * @lpsmclr: LP Secure Monotonic Counter LSB Register. + * @lppgdr: LP Power Glitch Detector Register. + * @rsvd4: Reserved. + * @lpzmkr[0x8]: LP Zeroizable Master Key Registers. + * @lpgpr[0x4]: LP General Purpose Registers. + * @rsvd5[0x2d2]: Reserved. + * @hpvidr1: HP Version ID Register 1. + * @hpvidr2: HP Version ID Register 2. + */ +static struct ls_sec_mon_registers { + uint32_t hplr; /* 0x000 */ + uint32_t hpcomr; /* 0x004 */ + uint32_t rsvd0; /* 0x008 */ + uint32_t hpsicr; /* 0x00C */ + uint32_t hpsvcr; /* 0x010 */ + uint32_t hpsr; /* 0x014 */ + uint32_t hpsvsr; /* 0x018 */ + uint32_t hphacivr; /* 0x01C */ + uint32_t hphacr; /* 0x020 */ + uint32_t rsvd1[0x4]; /* 0x024 */ + uint32_t lplr; /* 0x034 */ + uint32_t lpcr; /* 0x038 */ + uint32_t lpmkcr; /* 0x03C */ + uint32_t lpsvcr; /* 0x040 */ + uint32_t rsvd2; /* 0x044 */ + uint32_t lptdcr; /* 0x048 */ + uint32_t lpsr; /* 0x04C */ + uint32_t rsvd3[0x3]; /* 0x050 */ + uint32_t lpsmcmr; /* 0x05C */ + uint32_t lpsmclr; /* 0x060 */ + uint32_t lppgdr; /* 0x064 */ + uint32_t rsvd4; /* 0x068 */ + uint32_t lpzmkr[0x8]; /* 0x06C */ + uint32_t lpgpr[0x4]; /* 0x090 */ + uint32_t rsvd5[0x2d2]; /* 0x0B0 */ + uint32_t hpvidr1; /* 0xBF8 */ + uint32_t hpvidr2; /* 0xBFC */ +} *sec_mon_regs; + +/** + * ls_sec_mon_init() - Initialize the SecMon driver and assign the sec_mon_regs + * pointer to the SecMon base address detailed in the device + * tree. + * + * Return: 0 if successful or > 0 on error. + */ +static TEE_Result ls_sec_mon_init(void) +{ + void *fdt = NULL; + size_t size = 0; + uint32_t node = 0; + vaddr_t ctrl_base = 0; + + fdt = get_embedded_dt(); + if (!fdt) { + EMSG("Unable to find the device tree"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + node = fdt_node_offset_by_compatible(fdt, node, "fsl,lx2160a-sec-mon"); + if (node <= 0) { + EMSG("Unable to find the SecMon device tree node"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + if (dt_map_dev(fdt, node, &ctrl_base, &size, DT_MAP_AUTO) < 0) { + EMSG("Unable to get the SecMon virtual address"); + return TEE_ERROR_GENERIC; + } + + sec_mon_regs = (struct ls_sec_mon_registers *)ctrl_base; + + return TEE_SUCCESS; +} + +TEE_Result ls_sec_mon_read(struct ls_sec_mon_data *data) +{ + if (!sec_mon_regs) { + EMSG("SecMon driver is not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!data) { + EMSG("Given buffer is uninitialized"); + return TEE_ERROR_BAD_PARAMETERS; + } + + data->hplr = io_read32((vaddr_t)&sec_mon_regs->hplr); + data->hpcomr = io_read32((vaddr_t)&sec_mon_regs->hpcomr); + data->hpsicr = io_read32((vaddr_t)&sec_mon_regs->hpsicr); + data->hpsvcr = io_read32((vaddr_t)&sec_mon_regs->hpsvcr); + data->hpsr = io_read32((vaddr_t)&sec_mon_regs->hpsr); + data->hpsvsr = io_read32((vaddr_t)&sec_mon_regs->hpsvsr); + data->hphacivr = io_read32((vaddr_t)&sec_mon_regs->hphacivr); + data->hphacr = io_read32((vaddr_t)&sec_mon_regs->hphacr); + data->lplr = io_read32((vaddr_t)&sec_mon_regs->lplr); + data->lpcr = io_read32((vaddr_t)&sec_mon_regs->lpcr); + data->lpmkcr = io_read32((vaddr_t)&sec_mon_regs->lpmkcr); + data->lpsvcr = io_read32((vaddr_t)&sec_mon_regs->lpsvcr); + data->lptdcr = io_read32((vaddr_t)&sec_mon_regs->lptdcr); + data->lpsr = io_read32((vaddr_t)&sec_mon_regs->lpsr); + data->lpsmcmr = io_read32((vaddr_t)&sec_mon_regs->lpsmcmr); + data->lpsmclr = io_read32((vaddr_t)&sec_mon_regs->lpsmclr); + data->lppgdr = io_read32((vaddr_t)&sec_mon_regs->lppgdr); + data->hpvidr1 = io_read32((vaddr_t)&sec_mon_regs->hpvidr1); + data->hpvidr2 = io_read32((vaddr_t)&sec_mon_regs->hpvidr2); + + for (uint32_t i = 0; i < ARRAY_SIZE(data->lpzmkr); ++i) + data->lpzmkr[i] = io_read32((vaddr_t)&sec_mon_regs->lpzmkr[i]); + + for (uint32_t i = 0; i < ARRAY_SIZE(data->lpgpr); ++i) + data->lpgpr[i] = io_read32((vaddr_t)&sec_mon_regs->lpgpr[i]); + + return TEE_SUCCESS; +} + +TEE_Result ls_sec_mon_status(void) +{ + if (!sec_mon_regs) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +driver_init(ls_sec_mon_init); diff --git a/optee_os/core/drivers/ls_sfp.c b/optee_os/core/drivers/ls_sfp.c new file mode 100644 index 0000000..4c22040 --- /dev/null +++ b/optee_os/core/drivers/ls_sfp.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microsoft + * + * Driver for the NXP LX2160A-series Security Fuse Processor (SFP). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * struct ls_sfp_registers - Memory map of the SFP registers. + * @rsvd0[0x8]: Reserved. + * @ingr: Instruction Register. + * @svhesr: Secret Value Hamming Error Status Registers. + * @sfpcr: SFP Configuration Register. + * @rsvd1[0x3]: Reserved. + * @version: SFP Version Register. + * @rsvd2[0x71]: Reserved. + * @ospr0: OEM Security Policy Register 0. + * @ospr1: OEM Security Policy Register 1. + * @dcvr0: Debug Challenge Value Register 0. + * @dcvr1: Debug Challenge Value Register 1. + * @drvr0: Debug Response Value Register 0. + * @drvr1: Debug Response Value Register 1 + * @fswpr: Factory Section Write Protect Register. + * @fuidr0: Factory Unique ID Register 0. + * @fuidr1: Factory Unique ID Register 1. + * @isbccr: ISBC Configuration Register. + * @fspfr[0x3]: Factory Scratch Pad Fuse Registers. + * @otpmkr[0x8]: One Time Programmable Master Key. + * @srkhr[0x8]: Super Root Key Hash. + * @ouidr[0x5]: OEM Unique ID Scratch Pad Fuse Registers. + */ +static struct ls_sfp_registers { + uint32_t rsvd0[0x8]; /* 0x000 */ + uint32_t ingr; /* 0x020 */ + uint32_t svhesr; /* 0x024 */ + uint32_t sfpcr; /* 0x028 */ + uint32_t rsvd1[0x3]; /* 0x02C */ + uint32_t version; /* 0x038 */ + uint32_t rsvd2[0x71]; /* 0x03C */ + uint32_t ospr0; /* 0x200 */ + uint32_t ospr1; /* 0x204 */ + uint32_t dcvr0; /* 0x208 */ + uint32_t dcvr1; /* 0x20C */ + uint32_t drvr0; /* 0x210 */ + uint32_t drvr1; /* 0x214 */ + uint32_t fswpr; /* 0x218 */ + uint32_t fuidr0; /* 0x21C */ + uint32_t fuidr1; /* 0x220 */ + uint32_t isbccr; /* 0x224 */ + uint32_t fspfr[0x3]; /* 0x228 */ + uint32_t otpmkr[0x8]; /* 0x234 */ + uint32_t srkhr[0x8]; /* 0x254 */ + uint32_t ouidr[0x5]; /* 0x274 */ +} *sfp_regs; + +/** + * struct ls_gpio_info - Data struct containing GPIO specific information. + * @gpio_pin: GPIO pin number. + * @gpio_chip: GPIO controller instance data. + */ +static struct ls_gpio_info { + uint32_t gpio_pin; + struct ls_gpio_chip_data gpio_chip; +} gpio_info; + +/** + * ls_sfp_init() - Get SFP info from the embedded device tree and initialize + * sfp_regs and gpio_info. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +static TEE_Result ls_sfp_init(void) +{ + size_t size = 0; + int node = 0; + int rc = 0; + int povdd_node = 0; + int prop_len = 0; + vaddr_t ctrl_base = 0; + struct ls_gpio_chip_data *gc = NULL; + const char *fdt_prop_gpio = "povdd-gpio-controller"; + const char *fdt_prop_pin = "povdd-gpio-pin"; + const fdt32_t *gpio_val = NULL; + const fdt32_t *pin_val = NULL; + void *fdt = get_embedded_dt(); + + if (!fdt) { + EMSG("Unable to get the Embedded DTB, SFP init failed"); + return TEE_ERROR_GENERIC; + } + + node = fdt_node_offset_by_compatible(fdt, node, "fsl,lx2160a-sfp"); + if (node <= 0) { + EMSG("Unable to find SFP FDT node - rc = 0x%#"PRIx32, rc); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + rc = dt_map_dev(fdt, node, &ctrl_base, &size, DT_MAP_AUTO); + if (rc < 0) { + EMSG("Unable to get SFP virtual address - rc = 0x%#"PRIx32, rc); + return TEE_ERROR_GENERIC; + } + + povdd_node = fdt_path_offset(fdt, "/povdd"); + if (povdd_node <= 0) { + EMSG("Unable to find POVDD FDT node - rc = 0x%#"PRIx32, + povdd_node); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + sfp_regs = (struct ls_sfp_registers *)ctrl_base; + + gpio_val = fdt_getprop(fdt, povdd_node, fdt_prop_gpio, &prop_len); + if (!gpio_val) { + EMSG("Missing %s from POVDD FDT node", fdt_prop_gpio); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + pin_val = fdt_getprop(fdt, povdd_node, fdt_prop_pin, &prop_len); + if (!pin_val) { + EMSG("Missing %s from POVDD FDT node", fdt_prop_pin); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + gc = &gpio_info.gpio_chip; + gc->gpio_controller = (uint8_t)fdt32_to_cpu(*gpio_val); + gpio_info.gpio_pin = fdt32_to_cpu(*pin_val); + + return ls_gpio_init(gc); +} + +/** + * ls_sfp_program_fuses() - Write to fuses and verify that the correct value was + * written. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +static TEE_Result ls_sfp_program_fuses(void) +{ + TEE_Result ret = TEE_SUCCESS; + struct gpio_chip *gc = NULL; + uint32_t pin = gpio_info.gpio_pin; + vaddr_t sfp_ingr_va = (vaddr_t)&sfp_regs->ingr; + uint64_t timeout = 0; + + /* Enable POVDD */ + gc = &gpio_info.gpio_chip.chip; + + DMSG("Set GPIO %"PRIu32" pin %"PRIu32" to HIGH", + (uint32_t)gpio_info.gpio_chip.gpio_controller, pin); + gc->ops->set_direction(gc, pin, GPIO_DIR_OUT); + gc->ops->set_value(gc, pin, GPIO_LEVEL_HIGH); + + if (gc->ops->get_value(gc, pin) != GPIO_LEVEL_HIGH) { + EMSG("Error setting POVDD to HIGH"); + return TEE_ERROR_GENERIC; + } + + /* TA_PROG_SFP ramp rate requires up to 5ms for stability */ + mdelay(5); + + /* Send SFP write command */ + io_write32(sfp_ingr_va, SFP_INGR_PROGFB_CMD); + + /* Wait until fuse programming is successful */ + timeout = timeout_init_us(SFP_INGR_FUSE_TIMEOUT_US); + while (io_read32(sfp_ingr_va) & SFP_INGR_PROGFB_CMD) { + if (timeout_elapsed(timeout)) { + EMSG("SFP fusing timed out"); + ret = TEE_ERROR_GENERIC; + break; + } + } + + /* Disable POVDD */ + DMSG("Set GPIO %"PRIu8" pin %"PRIu32" to LOW", + gpio_info.gpio_chip.gpio_controller, pin); + gc->ops->set_value(gc, pin, GPIO_LEVEL_LOW); + gc->ops->set_direction(gc, pin, GPIO_DIR_IN); + + if (ret) + return ret; + + /* Check for SFP fuse programming error */ + if (io_read32(sfp_ingr_va) & SFP_INGR_ERROR_MASK) { + EMSG("Error writing SFP fuses"); + return TEE_ERROR_GENERIC; + } + + DMSG("Programmed fuse successfully"); + + return TEE_SUCCESS; +} + +TEE_Result ls_sfp_read(struct ls_sfp_data *data) +{ + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!data) + return TEE_ERROR_BAD_PARAMETERS; + + data->ingr = io_read32((vaddr_t)&sfp_regs->ingr); + data->svhesr = io_read32((vaddr_t)&sfp_regs->svhesr); + data->sfpcr = io_read32((vaddr_t)&sfp_regs->sfpcr); + data->version = io_read32((vaddr_t)&sfp_regs->version); + data->ospr0 = io_read32((vaddr_t)&sfp_regs->ospr0); + data->ospr1 = io_read32((vaddr_t)&sfp_regs->ospr1); + data->dcvr0 = io_read32((vaddr_t)&sfp_regs->dcvr0); + data->dcvr1 = io_read32((vaddr_t)&sfp_regs->dcvr1); + data->drvr0 = io_read32((vaddr_t)&sfp_regs->drvr0); + data->drvr1 = io_read32((vaddr_t)&sfp_regs->drvr1); + data->fswpr = io_read32((vaddr_t)&sfp_regs->fswpr); + data->fuidr0 = io_read32((vaddr_t)&sfp_regs->fuidr0); + data->fuidr1 = io_read32((vaddr_t)&sfp_regs->fuidr1); + data->isbccr = io_read32((vaddr_t)&sfp_regs->isbccr); + + for (uint32_t i = 0; i < ARRAY_SIZE(sfp_regs->fspfr); ++i) + data->fspfr[i] = io_read32((vaddr_t)&sfp_regs->fspfr[i]); + + for (uint32_t i = 0; i < ARRAY_SIZE(sfp_regs->otpmkr); ++i) + data->otpmkr[i] = io_read32((vaddr_t)&sfp_regs->otpmkr[i]); + + for (uint32_t i = 0; i < ARRAY_SIZE(sfp_regs->srkhr); ++i) + data->srkhr[i] = io_read32((vaddr_t)&sfp_regs->srkhr[i]); + + for (uint32_t i = 0; i < ARRAY_SIZE(sfp_regs->ouidr); ++i) + data->ouidr[i] = io_read32((vaddr_t)&sfp_regs->ouidr[i]); + + return TEE_SUCCESS; +} + +TEE_Result ls_sfp_get_debug_level(uint32_t *dblev) +{ + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!dblev) + return TEE_ERROR_BAD_PARAMETERS; + + *dblev = io_read32((vaddr_t)&sfp_regs->ospr1) & SFP_OSPR1_DBLEV_MASK; + + return TEE_SUCCESS; +} + +TEE_Result ls_sfp_get_its(uint32_t *its) +{ + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!its) + return TEE_ERROR_BAD_PARAMETERS; + + *its = (io_read32((vaddr_t)&sfp_regs->ospr0) & SFP_OSPR0_ITS_MASK) >> + SFP_OSPR0_ITS_OFFSET; + + return TEE_SUCCESS; +} + +TEE_Result ls_sfp_get_ouid(uint32_t index, uint32_t *ouid) +{ + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!ouid) + return TEE_ERROR_BAD_PARAMETERS; + + if (index >= ARRAY_SIZE(sfp_regs->ouidr)) { + DMSG("Index greater or equal to ouid: %"PRIu32" >= %zu", + index, ARRAY_SIZE(sfp_regs->ouidr)); + return TEE_ERROR_BAD_PARAMETERS; + } + + *ouid = io_read32((vaddr_t)&sfp_regs->ouidr[index]); + + return TEE_SUCCESS; +} + +TEE_Result ls_sfp_get_sb(uint32_t *sb) +{ + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!sb) + return TEE_ERROR_BAD_PARAMETERS; + + *sb = (io_read32((vaddr_t)&sfp_regs->sfpcr) & SFP_SFPCR_SB_MASK) >> + SFP_SFPCR_SB_OFFSET; + + return TEE_SUCCESS; +} + +TEE_Result ls_sfp_get_srkh(uint32_t index, uint32_t *srkh) +{ + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!srkh) + return TEE_ERROR_BAD_PARAMETERS; + + if (index >= ARRAY_SIZE(sfp_regs->srkhr)) { + DMSG("Index greater or equal to srkhr: %"PRIu32" >= %zu", + index, ARRAY_SIZE(sfp_regs->srkhr)); + return TEE_ERROR_BAD_PARAMETERS; + } + + *srkh = io_read32((vaddr_t)&sfp_regs->srkhr[index]); + + return TEE_SUCCESS; +} + +TEE_Result ls_sfp_set_debug_level(uint32_t dblev) +{ + uint32_t ospr1 = 0; + + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (!dblev) + return TEE_SUCCESS; + + ospr1 = io_read32((vaddr_t)&sfp_regs->ospr1); + if (ospr1 & SFP_OSPR1_DBLEV_MASK) { + DMSG("Debug level has already been fused"); + return TEE_ERROR_SECURITY; + } + + io_write32((vaddr_t)&sfp_regs->ospr1, ospr1 | dblev); + + return ls_sfp_program_fuses(); +} + +TEE_Result ls_sfp_set_its_wp(void) +{ + uint32_t ospr0 = 0; + + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + ospr0 = io_read32((vaddr_t)&sfp_regs->ospr0); + if (ospr0 & (SFP_OSPR0_WP_MASK | SFP_OSPR0_ITS_MASK)) { + DMSG("SFP is already fused"); + return TEE_ERROR_SECURITY; + } + + ospr0 |= SFP_OSPR0_WP_MASK | SFP_OSPR0_ITS_MASK; + io_write32((vaddr_t)&sfp_regs->ospr0, ospr0); + + return ls_sfp_program_fuses(); +} + +TEE_Result ls_sfp_set_ouid(uint32_t index, uint32_t ouid) +{ + if (!sfp_regs) { + EMSG("SFP driver not initialized"); + return TEE_ERROR_GENERIC; + } + + if (index >= ARRAY_SIZE(sfp_regs->ouidr)) { + DMSG("Index greater or equal to ouid: %"PRIu32" >= %"PRIu32, + index, ARRAY_SIZE(sfp_regs->ouidr)); + return TEE_ERROR_BAD_PARAMETERS; + } + + io_write32((vaddr_t)&sfp_regs->ouidr[index], ouid); + + return ls_sfp_program_fuses(); +} + +TEE_Result ls_sfp_status(void) +{ + if (!sfp_regs) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +driver_init(ls_sfp_init); diff --git a/optee_os/core/drivers/mvebu_uart.c b/optee_os/core/drivers/mvebu_uart.c new file mode 100644 index 0000000..75ba73b --- /dev/null +++ b/optee_os/core/drivers/mvebu_uart.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017 Marvell International Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* MVEBU UART Registers */ +#define UART_RX_REG 0x00 +#define UART_TX_REG 0x04 +#define UART_CTRL_REG 0x08 +#define UART_STATUS_REG 0x0c +#define UART_BAUD_REG 0x10 +#define UART_POSSR_REG 0x14 +#define UART_SIZE 0x18 + +/* Line Status Register bits */ +#define UARTLSR_TXFIFOFULL (1 << 11) /* Tx Fifo Full */ +#define UARTLSR_TXFIFOEMPTY (1 << 13) +#define UARTLSR_TXEMPTY (1 << 6) +#define UART_RX_READY (1 << 4) + +/* UART Control Register bits */ +#define UART_CTRL_RXFIFO_RESET (1 << 14) +#define UART_CTRL_TXFIFO_RESET (1 << 15) + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct mvebu_uart_data *pd = + container_of(chip, struct mvebu_uart_data, chip); + + return io_pa_or_va(&pd->base, UART_SIZE); +} + +static void mvebu_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + /* + * Wait for the transmit FIFO to be empty. + * It can happen that Linux initializes the OP-TEE driver with the + * console UART disabled; avoid an infinite loop by checking the UART + * enabled flag. Checking it in the loop makes the code safe against + * asynchronous disable. + */ + while (!(io_read32(base + UART_STATUS_REG) & UARTLSR_TXFIFOEMPTY)) + ; +} + +static bool mvebu_uart_have_rx_data(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + return (io_read32(base + UART_STATUS_REG) & UART_RX_READY); +} + +static int mvebu_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!mvebu_uart_have_rx_data(chip)) + ; + return io_read32(base + UART_RX_REG) & 0xff; +} + +static void mvebu_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + uint32_t tmp; + /* wait for space in tx FIFO */ + do { + tmp = io_read32(base + UART_STATUS_REG); + tmp &= UARTLSR_TXFIFOFULL; + } while (tmp == UARTLSR_TXFIFOFULL); + + io_write32(base + UART_TX_REG, ch); +} + +static const struct serial_ops mvebu_uart_ops = { + .flush = mvebu_uart_flush, + .getchar = mvebu_uart_getchar, + .have_rx_data = mvebu_uart_have_rx_data, + .putc = mvebu_uart_putc, +}; +DECLARE_KEEP_PAGER(mvebu_uart_ops); + +void mvebu_uart_init(struct mvebu_uart_data *pd, paddr_t pbase, + uint32_t uart_clk, uint32_t baud_rate) +{ + vaddr_t base; + uint32_t dll = 0; + + pd->base.pa = pbase; + pd->chip.ops = &mvebu_uart_ops; + + base = io_pa_or_va(&pd->base, UART_SIZE); + + dll = (uart_clk / (baud_rate << 4)) & 0x3FF; + + /* init UART */ + io_clrsetbits32(base + UART_BAUD_REG, 0x3FF, dll); + + /* set UART to default 16x scheme */ + io_write32(base + UART_POSSR_REG, 0); + + /* reset FIFO */ + io_write32(base + UART_CTRL_REG, + UART_CTRL_RXFIFO_RESET | UART_CTRL_TXFIFO_RESET); + + /* No Parity, 1 stop */ + io_write32(base + UART_CTRL_REG, 0); + + mvebu_uart_flush(&pd->chip); +} diff --git a/optee_os/core/drivers/ns16550.c b/optee_os/core/drivers/ns16550.c new file mode 100644 index 0000000..d0e52ce --- /dev/null +++ b/optee_os/core/drivers/ns16550.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2017, 2020, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +/* uart register defines */ +#define UART_RBR 0x0 +#define UART_THR 0x0 +#define UART_IER 0x1 +#define UART_FCR 0x2 +#define UART_LCR 0x3 +#define UART_MCR 0x4 +#define UART_LSR 0x5 +#define UART_MSR 0x6 +#define UART_SPR 0x7 + +/* uart status register bits */ +#define UART_LSR_DR 0x01 /* DATA Ready */ +#define UART_LSR_THRE 0x20 /* Transmit-hold-register empty */ + +static vaddr_t chip_to_base_and_data(struct serial_chip *chip, + struct ns16550_data **pd) +{ + *pd = container_of(chip, struct ns16550_data, chip); + + return io_pa_or_va(&(*pd)->base, NS16550_UART_REG_SIZE); +} + +static void ns16550_flush(struct serial_chip *chip) +{ + struct ns16550_data *pd = NULL; + vaddr_t base = chip_to_base_and_data(chip, &pd); + + while ((serial_in(base + (UART_LSR << pd->reg_shift), pd->io_width) & + UART_LSR_THRE) == 0) + ; +} + +static void ns16550_putc(struct serial_chip *chip, int ch) +{ + struct ns16550_data *pd = NULL; + vaddr_t base = chip_to_base_and_data(chip, &pd); + + ns16550_flush(chip); + + /* write out charset to Transmit-hold-register */ + serial_out(base + (UART_THR << pd->reg_shift), pd->io_width, ch); +} + +static bool ns16550_have_rx_data(struct serial_chip *chip) +{ + struct ns16550_data *pd = NULL; + vaddr_t base = chip_to_base_and_data(chip, &pd); + + return serial_in(base + (UART_LSR << pd->reg_shift), pd->io_width) & + UART_LSR_DR; +} + +static int ns16550_getchar(struct serial_chip *chip) +{ + struct ns16550_data *pd = NULL; + vaddr_t base = chip_to_base_and_data(chip, &pd); + + while (!ns16550_have_rx_data(chip)) { + /* Data is not ready, waiting again */ + ; + } + + return serial_in(base + (UART_RBR << pd->reg_shift), pd->io_width) & + 0xFF; +} + +static const struct serial_ops ns16550_ops = { + .flush = ns16550_flush, + .putc = ns16550_putc, + .getchar = ns16550_getchar, + .have_rx_data = ns16550_have_rx_data, +}; +DECLARE_KEEP_PAGER(ns16550_ops); + +void ns16550_init(struct ns16550_data *pd, paddr_t base, uint8_t io_width, + uint8_t reg_shift) +{ + pd->base.pa = base; + pd->io_width = io_width; + pd->reg_shift = reg_shift; + pd->chip.ops = &ns16550_ops; + + /* + * Do nothing, uart driver shared with normal world, + * everything for uart driver initialization is done in bootloader. + */ +} diff --git a/optee_os/core/drivers/pinctrl/atmel_pio.c b/optee_os/core/drivers/pinctrl/atmel_pio.c new file mode 100644 index 0000000..98b181f --- /dev/null +++ b/optee_os/core/drivers/pinctrl/atmel_pio.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Microchip. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PIO_GROUP_COUNT 4 +#define PIO_GROUP_OFFSET 0x40 +#define PIO_REG(reg, group) ((reg) + ((group) * PIO_GROUP_OFFSET)) +/* Mask register */ +#define PIO_MSKR(group) PIO_REG(0x0, (group)) +/* Configuration register */ +#define PIO_CFGR(group) PIO_REG(0x4, (group)) +#define PIO_CFGR_FUNC GENMASK(2, 0) +#define PIO_CFGR_PUEN BIT(9) +#define PIO_CFGR_PDEN BIT(10) + +/* Non-Secure configuration register */ +#define PIO_SIONR(group) PIO_REG(0x30, (group)) +/* Secure configuration register */ +#define PIO_SIOSR(group) PIO_REG(0x34, (group)) + +#define DT_GET_PIN_NO(val) ((val) & 0xFF) +#define DT_GET_FUNC(val) (((val) >> 16) & 0xF) + +struct atmel_pio { + vaddr_t base; +}; + +struct atmel_pio_pin_conf { + uint32_t pin_mask; + uint32_t pin_cfg; + uint8_t pio_group; + struct atmel_pio *pio; +}; + +static void pio_write(struct atmel_pio *pio, unsigned int offset, uint32_t val) +{ + io_write32(pio->base + offset, val); +} + +static TEE_Result pio_conf_apply(struct pinconf *conf) +{ + struct atmel_pio_pin_conf *pio_conf = conf->priv; + struct atmel_pio *pio = pio_conf->pio; + + DMSG("Apply cfg %#" PRIx32 " on group %" PRIu8 ", pins %#" PRIx32, + pio_conf->pin_cfg, pio_conf->pio_group, pio_conf->pin_mask); + + pio_write(pio, PIO_SIOSR(pio_conf->pio_group), pio_conf->pin_mask); + pio_write(pio, PIO_MSKR(pio_conf->pio_group), pio_conf->pin_mask); + pio_write(pio, PIO_CFGR(pio_conf->pio_group), pio_conf->pin_cfg); + + return TEE_SUCCESS; +} + +static void pio_conf_free(struct pinconf *conf) +{ + free(conf); +} + +static const struct pinctrl_ops pio_pinctrl_ops = { + .conf_apply = pio_conf_apply, + .conf_free = pio_conf_free, +}; + +static TEE_Result pio_pinctrl_dt_get(struct dt_pargs *pargs, void *data, + struct pinconf **out_pinconf) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int i = 0; + int func = 0; + int group = 0; + int pin_no = 0; + uint32_t cfg = 0; + int prop_count = 0; + int pio_group = -1; + uint32_t pinmux = 0; + uint32_t pin_mask = 0; + bitstr_t *cfg_modes = NULL; + const uint32_t *prop = NULL; + struct pinconf *pinconf = NULL; + struct atmel_pio *atmel_pio = data; + struct atmel_pio_pin_conf *pio_conf = NULL; + + prop = fdt_getprop(pargs->fdt, pargs->phandle_node, "pinmux", + &prop_count); + if (!prop) + return TEE_ERROR_ITEM_NOT_FOUND; + + prop_count /= sizeof(uint32_t); + for (i = 0; i < prop_count; i++) { + pinmux = fdt32_to_cpu(prop[i]); + + pin_no = DT_GET_PIN_NO(pinmux); + func = DT_GET_FUNC(pinmux); + + group = pin_no / 32; + if (pio_group == -1) { + pio_group = group; + } else { + if (group != pio_group) { + EMSG("Unexpected group %d vs %d", group, + pio_group); + return TEE_ERROR_GENERIC; + } + } + + pin_mask |= BIT(pin_no % 32); + } + + cfg = func; + + res = pinctrl_parse_dt_pin_modes(pargs->fdt, pargs->phandle_node, + &cfg_modes); + if (res) + return res; + + for (i = 0; i < PINCTRL_DT_PROP_MAX; i++) { + if (!bit_test(cfg_modes, i)) + continue; + + switch (i) { + case PINCTRL_DT_PROP_BIAS_PULL_UP: + cfg |= PIO_CFGR_PUEN; + cfg &= ~PIO_CFGR_PDEN; + break; + case PINCTRL_DT_PROP_BIAS_PULL_DOWN: + cfg |= PIO_CFGR_PDEN; + cfg &= ~PIO_CFGR_PUEN; + break; + case PINCTRL_DT_PROP_BIAS_DISABLE: + break; + default: + EMSG("Unhandled config %u", i); + break; + } + } + + free(cfg_modes); + + pinconf = calloc(1, sizeof(*pinconf) + sizeof(*pio_conf)); + if (!pinconf) + return TEE_ERROR_OUT_OF_MEMORY; + + pio_conf = (struct atmel_pio_pin_conf *)(pinconf + 1); + + pio_conf->pin_mask = pin_mask; + pio_conf->pin_cfg = cfg; + pio_conf->pio = atmel_pio; + pio_conf->pio_group = pio_group; + pinconf->priv = pio_conf; + pinconf->ops = &pio_pinctrl_ops; + + *out_pinconf = pinconf; + + return TEE_SUCCESS; +} + +static void pio_init_hw(struct atmel_pio *pio) +{ + int i = 0; + + /* Set all IOs as non-secure */ + for (i = 0; i < PIO_GROUP_COUNT; i++) + pio_write(pio, PIO_SIONR(PIO_GROUP_COUNT), GENMASK_32(31, 0)); +} + +static TEE_Result pio_node_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + size_t size = 0; + struct clk *clk = NULL; + struct atmel_pio *pio = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_BAD_STATE; + + pio = calloc(1, sizeof(*pio)); + if (!pio) + return TEE_ERROR_OUT_OF_MEMORY; + + res = clk_dt_get_by_index(fdt, node, 0, &clk); + if (res) + goto free_pio; + + if (dt_map_dev(fdt, node, &pio->base, &size, DT_MAP_AUTO) < 0) + goto free_pio; + + res = clk_enable(clk); + if (res) + goto free_pio; + + matrix_configure_periph_secure(AT91C_ID_PIOA); + matrix_configure_periph_secure(AT91C_ID_PIOB); + matrix_configure_periph_secure(AT91C_ID_PIOC); + matrix_configure_periph_secure(AT91C_ID_PIOD); + + pio_init_hw(pio); + + res = pinctrl_register_provider(fdt, node, pio_pinctrl_dt_get, pio); + if (res) + goto disable_clock; + + return TEE_SUCCESS; + +disable_clock: + clk_disable(clk); +free_pio: + free(pio); + + return res; +} + +static const struct dt_device_match atmel_pio_match_table[] = { + { .compatible = "atmel,sama5d2-pinctrl" }, + { } +}; + +DEFINE_DT_DRIVER(atmel_pio_dt_driver) = { + .name = "atmel_pio", + .type = DT_DRIVER_PINCTRL, + .match_table = atmel_pio_match_table, + .probe = pio_node_probe, +}; diff --git a/optee_os/core/drivers/pinctrl/pinctrl.c b/optee_os/core/drivers/pinctrl/pinctrl.c new file mode 100644 index 0000000..2ebeee0 --- /dev/null +++ b/optee_os/core/drivers/pinctrl/pinctrl.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static const char * const pin_modes[PINCTRL_DT_PROP_MAX] = { + [PINCTRL_DT_PROP_BIAS_DISABLE] = "bias-disable", + [PINCTRL_DT_PROP_BIAS_PULL_UP] = "bias-pull-up", + [PINCTRL_DT_PROP_BIAS_PULL_DOWN] = "bias-pull-down", +}; + +TEE_Result pinctrl_parse_dt_pin_modes(const void *fdt, int node, + bitstr_t **modes) +{ + unsigned int i = 0; + bitstr_t *modes_ptr = NULL; + + modes_ptr = bit_alloc(PINCTRL_DT_PROP_MAX); + if (!modes_ptr) + return TEE_ERROR_OUT_OF_MEMORY; + + for (i = 0; i < ARRAY_SIZE(pin_modes); i++) + if (fdt_getprop(fdt, node, pin_modes[i], NULL)) + bit_set(modes_ptr, i); + + *modes = modes_ptr; + + return TEE_SUCCESS; +} + +TEE_Result pinctrl_apply_state(struct pinctrl_state *state) +{ + unsigned int i = 0; + struct pinconf *conf = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + for (i = 0; i < state->conf_count; i++) { + conf = state->confs[i]; + + res = conf->ops->conf_apply(conf); + if (res) { + EMSG("Failed to apply pin conf"); + return res; + } + } + + return TEE_SUCCESS; +} + +void pinctrl_free_state(struct pinctrl_state *state) +{ + unsigned int i = 0; + + for (i = 0; i < state->conf_count; i++) + state->confs[i]->ops->conf_free(state->confs[i]); + + free(state); +} + +TEE_Result pinctrl_get_state_by_idx(const void *fdt, int nodeoffset, + unsigned int pinctrl_index, + struct pinctrl_state **state_ret) +{ + int bw = 0; + unsigned int conf_id = 0; + const uint32_t *prop = NULL; + unsigned int conf_count = 0; + /* Enough char to hold "pinctrl-" */ + char prop_name[8 + 20 + 1] = { }; + struct pinctrl_state *state = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + bw = snprintf(prop_name, sizeof(prop_name), "pinctrl-%d", + pinctrl_index); + if (bw >= (int)sizeof(prop_name)) + return TEE_ERROR_OVERFLOW; + + prop = fdt_getprop(fdt, nodeoffset, prop_name, (int *)&conf_count); + if (!prop) + return TEE_ERROR_ITEM_NOT_FOUND; + + conf_count /= sizeof(uint32_t); + state = calloc(1, sizeof(struct pinctrl_state) + + conf_count * sizeof(struct pinconf *)); + if (!state) + return TEE_ERROR_OUT_OF_MEMORY; + + state->conf_count = conf_count; + for (conf_id = 0; conf_id < conf_count; conf_id++) { + void *pinconf = NULL; + + res = dt_driver_device_from_node_idx_prop(prop_name, fdt, + nodeoffset, conf_id, + DT_DRIVER_PINCTRL, + &pinconf); + if (res) { + free(state); + return res; + } + + state->confs[conf_id] = pinconf; + } + + *state_ret = state; + + return TEE_SUCCESS; +} + +TEE_Result pinctrl_get_state_by_name(const void *fdt, int nodeoffset, + const char *name, + struct pinctrl_state **state) +{ + int pinctrl_index = 0; + + if (!name) + name = "default"; + + pinctrl_index = fdt_stringlist_search(fdt, nodeoffset, "pinctrl-names", + name); + if (pinctrl_index < 0) { + *state = NULL; + if (pinctrl_index == -FDT_ERR_NOTFOUND) + return TEE_ERROR_ITEM_NOT_FOUND; + else + return TEE_ERROR_GENERIC; + } + + return pinctrl_get_state_by_idx(fdt, nodeoffset, pinctrl_index, state); +} diff --git a/optee_os/core/drivers/pinctrl/sub.mk b/optee_os/core/drivers/pinctrl/sub.mk new file mode 100644 index 0000000..90b07de --- /dev/null +++ b/optee_os/core/drivers/pinctrl/sub.mk @@ -0,0 +1,2 @@ +srcs-$(CFG_DRIVERS_PINCTRL) += pinctrl.c +srcs-$(CFG_ATMEL_PIO) += atmel_pio.c diff --git a/optee_os/core/drivers/pl011.c b/optee_os/core/drivers/pl011.c new file mode 100644 index 0000000..0fc03d6 --- /dev/null +++ b/optee_os/core/drivers/pl011.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UART_DR 0x00 /* data register */ +#define UART_RSR_ECR 0x04 /* receive status or error clear */ +#define UART_DMAWM 0x08 /* DMA watermark configure */ +#define UART_TIMEOUT 0x0C /* Timeout period */ +/* reserved space */ +#define UART_FR 0x18 /* flag register */ +#define UART_ILPR 0x20 /* IrDA low-poer */ +#define UART_IBRD 0x24 /* integer baud register */ +#define UART_FBRD 0x28 /* fractional baud register */ +#define UART_LCR_H 0x2C /* line control register */ +#define UART_CR 0x30 /* control register */ +#define UART_IFLS 0x34 /* interrupt FIFO level select */ +#define UART_IMSC 0x38 /* interrupt mask set/clear */ +#define UART_RIS 0x3C /* raw interrupt register */ +#define UART_MIS 0x40 /* masked interrupt register */ +#define UART_ICR 0x44 /* interrupt clear register */ +#define UART_DMACR 0x48 /* DMA control register */ + +/* flag register bits */ +#define UART_FR_RTXDIS (1 << 13) +#define UART_FR_TERI (1 << 12) +#define UART_FR_DDCD (1 << 11) +#define UART_FR_DDSR (1 << 10) +#define UART_FR_DCTS (1 << 9) +#define UART_FR_RI (1 << 8) +#define UART_FR_TXFE (1 << 7) +#define UART_FR_RXFF (1 << 6) +#define UART_FR_TXFF (1 << 5) +#define UART_FR_RXFE (1 << 4) +#define UART_FR_BUSY (1 << 3) +#define UART_FR_DCD (1 << 2) +#define UART_FR_DSR (1 << 1) +#define UART_FR_CTS (1 << 0) + +/* transmit/receive line register bits */ +#define UART_LCRH_SPS (1 << 7) +#define UART_LCRH_WLEN_8 (3 << 5) +#define UART_LCRH_WLEN_7 (2 << 5) +#define UART_LCRH_WLEN_6 (1 << 5) +#define UART_LCRH_WLEN_5 (0 << 5) +#define UART_LCRH_FEN (1 << 4) +#define UART_LCRH_STP2 (1 << 3) +#define UART_LCRH_EPS (1 << 2) +#define UART_LCRH_PEN (1 << 1) +#define UART_LCRH_BRK (1 << 0) + +/* control register bits */ +#define UART_CR_CTSEN (1 << 15) +#define UART_CR_RTSEN (1 << 14) +#define UART_CR_OUT2 (1 << 13) +#define UART_CR_OUT1 (1 << 12) +#define UART_CR_RTS (1 << 11) +#define UART_CR_DTR (1 << 10) +#define UART_CR_RXE (1 << 9) +#define UART_CR_TXE (1 << 8) +#define UART_CR_LPE (1 << 7) +#define UART_CR_OVSFACT (1 << 3) +#define UART_CR_UARTEN (1 << 0) + +#define UART_IMSC_RTIM (1 << 6) +#define UART_IMSC_RXIM (1 << 4) + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct pl011_data *pd = + container_of(chip, struct pl011_data, chip); + + return io_pa_or_va(&pd->base, PL011_REG_SIZE); +} + +static void pl011_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + /* + * Wait for the transmit FIFO to be empty. + * It can happen that Linux initializes the OP-TEE driver with the + * console UART disabled; avoid an infinite loop by checking the UART + * enabled flag. Checking it in the loop makes the code safe against + * asynchronous disable. + */ + while ((io_read32(base + UART_CR) & UART_CR_UARTEN) && + !(io_read32(base + UART_FR) & UART_FR_TXFE)) + ; +} + +static bool pl011_have_rx_data(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + return !(io_read32(base + UART_FR) & UART_FR_RXFE); +} + +static int pl011_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!pl011_have_rx_data(chip)) + ; + return io_read32(base + UART_DR) & 0xff; +} + +static void pl011_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + /* Wait until there is space in the FIFO or device is disabled */ + while (io_read32(base + UART_FR) & UART_FR_TXFF) + ; + + /* Send the character */ + io_write32(base + UART_DR, ch); +} + +static const struct serial_ops pl011_ops = { + .flush = pl011_flush, + .getchar = pl011_getchar, + .have_rx_data = pl011_have_rx_data, + .putc = pl011_putc, +}; +DECLARE_KEEP_PAGER(pl011_ops); + +void pl011_init(struct pl011_data *pd, paddr_t pbase, uint32_t uart_clk, + uint32_t baud_rate) +{ + vaddr_t base; + + pd->base.pa = pbase; + pd->chip.ops = &pl011_ops; + + base = io_pa_or_va(&pd->base, PL011_REG_SIZE); + + /* Clear all errors */ + io_write32(base + UART_RSR_ECR, 0); + /* Disable everything */ + io_write32(base + UART_CR, 0); + + if (baud_rate) { + uint32_t divisor = (uart_clk * 4) / baud_rate; + + io_write32(base + UART_IBRD, divisor >> 6); + io_write32(base + UART_FBRD, divisor & 0x3f); + } + + /* Configure TX to 8 bits, 1 stop bit, no parity, fifo disabled. */ + io_write32(base + UART_LCR_H, UART_LCRH_WLEN_8); + + /* Enable interrupts for receive and receive timeout */ + io_write32(base + UART_IMSC, UART_IMSC_RXIM | UART_IMSC_RTIM); + + /* Enable UART and RX/TX */ + io_write32(base + UART_CR, UART_CR_UARTEN | UART_CR_TXE | UART_CR_RXE); + + pl011_flush(&pd->chip); +} + +#ifdef CFG_DT + +static struct serial_chip *pl011_dev_alloc(void) +{ + struct pl011_data *pd = nex_calloc(1, sizeof(*pd)); + + if (!pd) + return NULL; + return &pd->chip; +} + +static int pl011_dev_init(struct serial_chip *chip, const void *fdt, int offs, + const char *parms) +{ + struct pl011_data *pd = container_of(chip, struct pl011_data, chip); + vaddr_t vbase; + paddr_t pbase; + size_t size; + + if (parms && parms[0]) + IMSG("pl011: device parameters ignored (%s)", parms); + + if (dt_map_dev(fdt, offs, &vbase, &size, DT_MAP_AUTO) < 0) + return -1; + + if (size != 0x1000) { + EMSG("pl011: unexpected register size: %zx", size); + return -1; + } + + pbase = virt_to_phys((void *)vbase); + pl011_init(pd, pbase, 0, 0); + + return 0; +} + +static void pl011_dev_free(struct serial_chip *chip) +{ + struct pl011_data *pd = container_of(chip, struct pl011_data, chip); + + nex_free(pd); +} + +static const struct serial_driver pl011_driver = { + .dev_alloc = pl011_dev_alloc, + .dev_init = pl011_dev_init, + .dev_free = pl011_dev_free, +}; + +static const struct dt_device_match pl011_match_table[] = { + { .compatible = "arm,pl011" }, + { 0 } +}; + +DEFINE_DT_DRIVER(pl011_dt_driver) = { + .name = "pl011", + .type = DT_DRIVER_UART, + .match_table = pl011_match_table, + .driver = &pl011_driver, +}; + +#endif /* CFG_DT */ diff --git a/optee_os/core/drivers/pl022_spi.c b/optee_os/core/drivers/pl022_spi.c new file mode 100644 index 0000000..91e54b2 --- /dev/null +++ b/optee_os/core/drivers/pl022_spi.c @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* SPI register offsets */ +#define SSPCR0 0x000 +#define SSPCR1 0x004 +#define SSPDR 0x008 +#define SSPSR 0x00C +#define SSPCPSR 0x010 +#define SSPIMSC 0x014 +#define SSPRIS 0x018 +#define SSPMIS 0x01C +#define SSPICR 0x020 +#define SSPDMACR 0x024 + +#ifdef PLATFORM_hikey +/* HiKey extensions */ +#define SSPTXFIFOCR 0x028 +#define SSPRXFIFOCR 0x02C +#define SSPB2BTRANS 0x030 +#endif + +/* test registers */ +#define SSPTCR 0x080 +#define SSPITIP 0x084 +#define SSPITOP 0x088 +#define SSPTDR 0x08C + +#define SSPPeriphID0 0xFE0 +#define SSPPeriphID1 0xFE4 +#define SSPPeriphID2 0xFE8 +#define SSPPeriphID3 0xFEC + +#define SSPPCellID0 0xFF0 +#define SSPPCellID1 0xFF4 +#define SSPPCellID2 0xFF8 +#define SSPPCellID3 0xFFC + +/* SPI register masks */ +#define SSPCR0_SCR SHIFT_U32(0xFF, 8) +#define SSPCR0_SPH SHIFT_U32(1, 7) +#define SSPCR0_SPH1 SHIFT_U32(1, 7) +#define SSPCR0_SPH0 SHIFT_U32(0, 7) +#define SSPCR0_SPO SHIFT_U32(1, 6) +#define SSPCR0_SPO1 SHIFT_U32(1, 6) +#define SSPCR0_SPO0 SHIFT_U32(0, 6) +#define SSPCR0_FRF SHIFT_U32(3, 4) +#define SSPCR0_FRF_SPI SHIFT_U32(0, 4) +#define SSPCR0_DSS SHIFT_U32(0xFF, 0) +#define SSPCR0_DSS_16BIT SHIFT_U32(0xF, 0) +#define SSPCR0_DSS_8BIT SHIFT_U32(7, 0) + +#define SSPCR1_SOD SHIFT_U32(1, 3) +#define SSPCR1_SOD_ENABLE SHIFT_U32(1, 3) +#define SSPCR1_SOD_DISABLE SHIFT_U32(0, 3) +#define SSPCR1_MS SHIFT_U32(1, 2) +#define SSPCR1_MS_SLAVE SHIFT_U32(1, 2) +#define SSPCR1_MS_MASTER SHIFT_U32(0, 2) +#define SSPCR1_SSE SHIFT_U32(1, 1) +#define SSPCR1_SSE_ENABLE SHIFT_U32(1, 1) +#define SSPCR1_SSE_DISABLE SHIFT_U32(0, 1) +#define SSPCR1_LBM SHIFT_U32(1, 0) +#define SSPCR1_LBM_YES SHIFT_U32(1, 0) +#define SSPCR1_LBM_NO SHIFT_U32(0, 0) + +#define SSPDR_DATA SHIFT_U32(0xFFFF, 0) + +#define SSPSR_BSY SHIFT_U32(1, 4) +#define SSPSR_RNF SHIFT_U32(1, 3) +#define SSPSR_RNE SHIFT_U32(1, 2) +#define SSPSR_TNF SHIFT_U32(1, 1) +#define SSPSR_TFE SHIFT_U32(1, 0) + +#define SSPCPSR_CPSDVR SHIFT_U32(0xFF, 0) + +#define SSPIMSC_TXIM SHIFT_U32(1, 3) +#define SSPIMSC_RXIM SHIFT_U32(1, 2) +#define SSPIMSC_RTIM SHIFT_U32(1, 1) +#define SSPIMSC_RORIM SHIFT_U32(1, 0) + +#define SSPRIS_TXRIS SHIFT_U32(1, 3) +#define SSPRIS_RXRIS SHIFT_U32(1, 2) +#define SSPRIS_RTRIS SHIFT_U32(1, 1) +#define SSPRIS_RORRIS SHIFT_U32(1, 0) + +#define SSPMIS_TXMIS SHIFT_U32(1, 3) +#define SSPMIS_RXMIS SHIFT_U32(1, 2) +#define SSPMIS_RTMIS SHIFT_U32(1, 1) +#define SSPMIS_RORMIS SHIFT_U32(1, 0) + +#define SSPICR_RTIC SHIFT_U32(1, 1) +#define SSPICR_RORIC SHIFT_U32(1, 0) + +#define SSPDMACR_TXDMAE SHIFT_U32(1, 1) +#define SSPDMACR_RXDMAE SHIFT_U32(1, 0) + +#define SSPPeriphID0_PartNumber0 SHIFT_U32(0xFF, 0) /* 0x22 */ +#define SSPPeriphID1_Designer0 SHIFT_U32(0xF, 4) /* 0x1 */ +#define SSPPeriphID1_PartNumber1 SHIFT_U32(0xF, 0) /* 0x0 */ +#define SSPPeriphID2_Revision SHIFT_U32(0xF, 4) +#define SSPPeriphID2_Designer1 SHIFT_U32(0xF, 0) /* 0x4 */ +#define SSPPeriphID3_Configuration SHIFT_U32(0xFF, 0) /* 0x00 */ + +#define SSPPCellID_0 SHIFT_U32(0xFF, 0) /* 0x0D */ +#define SSPPCellID_1 SHIFT_U32(0xFF, 0) /* 0xF0 */ +#define SSPPPCellID_2 SHIFT_U32(0xFF, 0) /* 0x05 */ +#define SSPPPCellID_3 SHIFT_U32(0xFF, 0) /* 0xB1 */ + +#define MASK_32 0xFFFFFFFF +#define MASK_28 0xFFFFFFF +#define MASK_24 0xFFFFFF +#define MASK_20 0xFFFFF +#define MASK_16 0xFFFF +#define MASK_12 0xFFF +#define MASK_8 0xFF +#define MASK_4 0xF +/* SPI register masks */ + +#define SSP_CPSDVR_MAX 254 +#define SSP_CPSDVR_MIN 2 +#define SSP_SCR_MAX 255 +#define SSP_SCR_MIN 0 +#define SSP_DATASIZE_MAX 16 + +static enum spi_result pl022_txrx8(struct spi_chip *chip, uint8_t *wdat, + uint8_t *rdat, size_t num_pkts) +{ + size_t i = 0; + size_t j = 0; + struct pl022_data *pd = container_of(chip, struct pl022_data, chip); + + + if (pd->data_size_bits != 8) { + EMSG("data_size_bits should be 8, not %u", + pd->data_size_bits); + return SPI_ERR_CFG; + } + + if (wdat) + while (i < num_pkts) { + if (io_read8(pd->base + SSPSR) & SSPSR_TNF) { + /* tx 1 packet */ + io_write8(pd->base + SSPDR, wdat[i++]); + } + + if (rdat) + if (io_read8(pd->base + SSPSR) & SSPSR_RNE) { + /* rx 1 packet */ + rdat[j++] = io_read8(pd->base + SSPDR); + } + } + + /* Capture remaining rdat not read above */ + if (rdat) { + while ((j < num_pkts) && + (io_read8(pd->base + SSPSR) & SSPSR_RNE)) { + /* rx 1 packet */ + rdat[j++] = io_read8(pd->base + SSPDR); + } + + if (j < num_pkts) { + EMSG("Packets requested %zu, received %zu", + num_pkts, j); + return SPI_ERR_PKTCNT; + } + } + + return SPI_OK; +} + +static enum spi_result pl022_txrx16(struct spi_chip *chip, uint16_t *wdat, + uint16_t *rdat, size_t num_pkts) +{ + size_t i = 0; + size_t j = 0; + struct pl022_data *pd = container_of(chip, struct pl022_data, chip); + + if (pd->data_size_bits != 16) { + EMSG("data_size_bits should be 16, not %u", + pd->data_size_bits); + return SPI_ERR_CFG; + } + + if (wdat) + while (i < num_pkts) { + if (io_read8(pd->base + SSPSR) & SSPSR_TNF) { + /* tx 1 packet */ + io_write16(pd->base + SSPDR, wdat[i++]); + } + + if (rdat) + if (io_read8(pd->base + SSPSR) & SSPSR_RNE) { + /* rx 1 packet */ + rdat[j++] = io_read16(pd->base + SSPDR); + } + } + + /* Capture remaining rdat not read above */ + if (rdat) { + while ((j < num_pkts) && + (io_read8(pd->base + SSPSR) & SSPSR_RNE)) { + /* rx 1 packet */ + rdat[j++] = io_read16(pd->base + SSPDR); + } + + if (j < num_pkts) { + EMSG("Packets requested %zu, received %zu", + num_pkts, j); + return SPI_ERR_PKTCNT; + } + } + + return SPI_OK; +} + +static void pl022_print_peri_id(struct pl022_data *pd __maybe_unused) +{ + DMSG("Expected: 0x 22 10 ?4 00"); + DMSG("Read: 0x %02x %02x %02x %02x", + io_read8(pd->base + SSPPeriphID0), + io_read8(pd->base + SSPPeriphID1), + io_read8(pd->base + SSPPeriphID2), + io_read8(pd->base + SSPPeriphID3)); +} + +static void pl022_print_cell_id(struct pl022_data *pd __maybe_unused) +{ + DMSG("Expected: 0x 0d f0 05 b1"); + DMSG("Read: 0x %02x %02x %02x %02x", + io_read8(pd->base + SSPPCellID0), + io_read8(pd->base + SSPPCellID1), + io_read8(pd->base + SSPPCellID2), + io_read8(pd->base + SSPPCellID3)); +} + +static void pl022_sanity_check(struct pl022_data *pd) +{ + assert(pd); + assert(pd->chip.ops); + assert(pd->cs_control <= PL022_CS_CTRL_MANUAL); + switch (pd->cs_control) { + case PL022_CS_CTRL_AUTO_GPIO: + assert(pd->cs_data.gpio_data.chip); + assert(pd->cs_data.gpio_data.chip->ops); + break; + case PL022_CS_CTRL_CB: + assert(pd->cs_data.cs_cb); + break; + default: + break; + } + assert(pd->clk_hz); + assert(pd->speed_hz && pd->speed_hz <= pd->clk_hz/2); + assert(pd->mode <= SPI_MODE3); + assert(pd->data_size_bits == 8 || pd->data_size_bits == 16); + + #ifdef PLATFORM_hikey + DMSG("SSPB2BTRANS: Expected: 0x2. Read: 0x%x", + io_read8(pd->base + SSPB2BTRANS)); + #endif + pl022_print_peri_id(pd); + pl022_print_cell_id(pd); +} + +static inline uint32_t pl022_calc_freq(struct pl022_data *pd, + uint8_t cpsdvr, uint8_t scr) +{ + return pd->clk_hz / (cpsdvr * (1 + scr)); +} + +static void pl022_control_cs(struct spi_chip *chip, enum gpio_level value) +{ + struct pl022_data *pd = container_of(chip, struct pl022_data, chip); + + switch (pd->cs_control) { + case PL022_CS_CTRL_AUTO_GPIO: + if (io_read8(pd->base + SSPSR) & SSPSR_BSY) + DMSG("pl022 busy - do NOT set CS!"); + while (io_read8(pd->base + SSPSR) & SSPSR_BSY) + ; + DMSG("pl022 done - set CS!"); + + pd->cs_data.gpio_data.chip->ops->set_value(NULL, + pd->cs_data.gpio_data.pin_num, value); + break; + case PL022_CS_CTRL_CB: + pd->cs_data.cs_cb(value); + break; + default: + break; + } +} + +static void pl022_calc_clk_divisors(struct pl022_data *pd, + uint8_t *cpsdvr, uint8_t *scr) +{ + unsigned int freq1 = 0; + unsigned int freq2 = 0; + uint8_t tmp_cpsdvr1; + uint8_t tmp_scr1; + uint8_t tmp_cpsdvr2 = 0; + uint8_t tmp_scr2 = 0; + + for (tmp_scr1 = SSP_SCR_MIN; tmp_scr1 < SSP_SCR_MAX; tmp_scr1++) { + for (tmp_cpsdvr1 = SSP_CPSDVR_MIN; tmp_cpsdvr1 < SSP_CPSDVR_MAX; + tmp_cpsdvr1++) { + freq1 = pl022_calc_freq(pd, tmp_cpsdvr1, tmp_scr1); + if (freq1 == pd->speed_hz) + goto done; + else if (freq1 < pd->speed_hz) + goto stage2; + } + } + +stage2: + for (tmp_cpsdvr2 = SSP_CPSDVR_MIN; tmp_cpsdvr2 < SSP_CPSDVR_MAX; + tmp_cpsdvr2++) { + for (tmp_scr2 = SSP_SCR_MIN; tmp_scr2 < SSP_SCR_MAX; + tmp_scr2++) { + freq2 = pl022_calc_freq(pd, tmp_cpsdvr2, tmp_scr2); + if (freq2 <= pd->speed_hz) + goto done; + } + } + +done: + if (freq1 >= freq2) { + *cpsdvr = tmp_cpsdvr1; + *scr = tmp_scr1; + DMSG("speed: requested: %u, closest1: %u", + pd->speed_hz, freq1); + } else { + *cpsdvr = tmp_cpsdvr2; + *scr = tmp_scr2; + DMSG("speed: requested: %u, closest2: %u", + pd->speed_hz, freq2); + } + DMSG("CPSDVR: %u (0x%x), SCR: %u (0x%x)", + *cpsdvr, *cpsdvr, *scr, *scr); +} + +static void pl022_flush_fifo(struct spi_chip *chip) +{ + uint32_t __maybe_unused rdat; + struct pl022_data *pd = container_of(chip, struct pl022_data, chip); + do { + while (io_read32(pd->base + SSPSR) & SSPSR_RNE) { + rdat = io_read32(pd->base + SSPDR); + DMSG("rdat: 0x%x", rdat); + } + } while (io_read32(pd->base + SSPSR) & SSPSR_BSY); +} + +static void pl022_configure(struct spi_chip *chip) +{ + uint16_t mode; + uint16_t data_size; + uint8_t cpsdvr; + uint8_t scr; + uint8_t lbm; + struct pl022_data *pd = container_of(chip, struct pl022_data, chip); + + pl022_sanity_check(pd); + + switch (pd->cs_control) { + case PL022_CS_CTRL_AUTO_GPIO: + DMSG("Use auto GPIO CS control"); + DMSG("Mask/disable interrupt for CS GPIO"); + pd->cs_data.gpio_data.chip->ops->set_interrupt(NULL, + pd->cs_data.gpio_data.pin_num, + GPIO_INTERRUPT_DISABLE); + DMSG("Set CS GPIO dir to out"); + pd->cs_data.gpio_data.chip->ops->set_direction(NULL, + pd->cs_data.gpio_data.pin_num, + GPIO_DIR_OUT); + break; + case PL022_CS_CTRL_CB: + DMSG("Use registered CS callback"); + break; + case PL022_CS_CTRL_MANUAL: + DMSG("Use manual CS control"); + break; + default: + EMSG("Invalid CS control type: %d", pd->cs_control); + panic(); + } + + DMSG("Pull CS high"); + pl022_control_cs(chip, GPIO_LEVEL_HIGH); + + pl022_calc_clk_divisors(pd, &cpsdvr, &scr); + + /* configure ssp based on platform settings */ + switch (pd->mode) { + case SPI_MODE0: + DMSG("SPI mode 0"); + mode = SSPCR0_SPO0 | SSPCR0_SPH0; + break; + case SPI_MODE1: + DMSG("SPI mode 1"); + mode = SSPCR0_SPO0 | SSPCR0_SPH1; + break; + case SPI_MODE2: + DMSG("SPI mode 2"); + mode = SSPCR0_SPO1 | SSPCR0_SPH0; + break; + case SPI_MODE3: + DMSG("SPI mode 3"); + mode = SSPCR0_SPO1 | SSPCR0_SPH1; + break; + default: + EMSG("Invalid SPI mode: %u", pd->mode); + panic(); + } + + switch (pd->data_size_bits) { + case 8: + DMSG("Data size: 8"); + data_size = SSPCR0_DSS_8BIT; + break; + case 16: + DMSG("Data size: 16"); + data_size = SSPCR0_DSS_16BIT; + break; + default: + EMSG("Unsupported data size: %u bits", pd->data_size_bits); + panic(); + } + + if (pd->loopback) { + DMSG("Starting in loopback mode!"); + lbm = SSPCR1_LBM_YES; + } else { + DMSG("Starting in regular (non-loopback) mode!"); + lbm = SSPCR1_LBM_NO; + } + + DMSG("Set Serial Clock Rate (SCR), SPI mode (phase and clock)"); + DMSG("Set frame format (SPI) and data size (8- or 16-bit)"); + io_mask16(pd->base + SSPCR0, SHIFT_U32(scr, 8) | mode | SSPCR0_FRF_SPI | + data_size, MASK_16); + + DMSG("Set master mode, disable SSP, set loopback mode"); + io_mask8(pd->base + SSPCR1, SSPCR1_SOD_DISABLE | SSPCR1_MS_MASTER | + SSPCR1_SSE_DISABLE | lbm, MASK_4); + + DMSG("Set clock prescale"); + io_mask8(pd->base + SSPCPSR, cpsdvr, SSPCPSR_CPSDVR); + + DMSG("Disable interrupts"); + io_mask8(pd->base + SSPIMSC, 0, MASK_4); + + DMSG("Clear interrupts"); + io_mask8(pd->base + SSPICR, SSPICR_RORIC | SSPICR_RTIC, + SSPICR_RORIC | SSPICR_RTIC); + + DMSG("Empty FIFO before starting"); + pl022_flush_fifo(chip); +} + +static void pl022_start(struct spi_chip *chip) +{ + struct pl022_data *pd = container_of(chip, struct pl022_data, chip); + + DMSG("Enable SSP"); + io_mask8(pd->base + SSPCR1, SSPCR1_SSE_ENABLE, SSPCR1_SSE); + + pl022_control_cs(chip, GPIO_LEVEL_LOW); +} + +static void pl022_end(struct spi_chip *chip) +{ + struct pl022_data *pd = container_of(chip, struct pl022_data, chip); + + pl022_control_cs(chip, GPIO_LEVEL_HIGH); + + DMSG("Disable SSP"); + io_mask8(pd->base + SSPCR1, SSPCR1_SSE_DISABLE, SSPCR1_SSE); +} + +static const struct spi_ops pl022_ops = { + .configure = pl022_configure, + .start = pl022_start, + .txrx8 = pl022_txrx8, + .txrx16 = pl022_txrx16, + .end = pl022_end, + .flushfifo = pl022_flush_fifo, +}; +DECLARE_KEEP_PAGER(pl022_ops); + +void pl022_init(struct pl022_data *pd) +{ + assert(pd); + pd->chip.ops = &pl022_ops; +} diff --git a/optee_os/core/drivers/pl061_gpio.c b/optee_os/core/drivers/pl061_gpio.c new file mode 100644 index 0000000..19c62d1 --- /dev/null +++ b/optee_os/core/drivers/pl061_gpio.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#ifndef PLAT_PL061_MAX_GPIOS +# define PLAT_PL061_MAX_GPIOS 32 +#endif /* PLAT_PL061_MAX_GPIOS */ + +#define MAX_GPIO_DEVICES ((PLAT_PL061_MAX_GPIOS + \ + (GPIOS_PER_PL061 - 1)) / GPIOS_PER_PL061) + +#define GPIOS_PER_PL061 8 + +/* gpio register offsets */ +#define GPIODIR 0x400 +#define GPIOIS 0x404 +#define GPIOIBE 0x408 +#define GPIOIEV 0x40C +#define GPIOIE 0x410 +#define GPIORIS 0x414 +#define GPIOMIS 0x418 +#define GPIOIC 0x41C +#define GPIOAFSEL 0x420 + +/* gpio register masks */ +#define GPIOIE_ENABLED SHIFT_U32(1, 0) +#define GPIOIE_MASKED SHIFT_U32(0, 0) +#define GPIOAFSEL_HW SHIFT_U32(1, 0) +#define GPIOAFSEL_SW SHIFT_U32(0, 0) +#define GPIODIR_OUT SHIFT_U32(1, 0) +#define GPIODIR_IN SHIFT_U32(0, 0) + +static vaddr_t pl061_reg_base[MAX_GPIO_DEVICES]; + +static enum gpio_dir pl061_get_direction(struct gpio_chip *chip __unused, + unsigned int gpio_pin) +{ + vaddr_t base_addr; + uint8_t data; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + data = io_read8(base_addr + GPIODIR); + if (data & BIT(offset)) + return GPIO_DIR_OUT; + return GPIO_DIR_IN; +} + +static void pl061_set_direction(struct gpio_chip *chip __unused, + unsigned int gpio_pin, enum gpio_dir direction) +{ + vaddr_t base_addr; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + if (direction == GPIO_DIR_OUT) + io_setbits8(base_addr + GPIODIR, BIT(offset)); + else + io_clrbits8(base_addr + GPIODIR, BIT(offset)); +} + +/* + * The offset of GPIODATA register is 0. + * The values read from GPIODATA are determined for each bit, by the mask bit + * derived from the address used to access the data register, PADDR[9:2]. + * Bits that are 1 in the address mask cause the corresponding bits in GPIODATA + * to be read, and bits that are 0 in the address mask cause the corresponding + * bits in GPIODATA to be read as 0, regardless of their value. + */ +static enum gpio_level pl061_get_value(struct gpio_chip *chip __unused, + unsigned int gpio_pin) +{ + vaddr_t base_addr; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + if (io_read8(base_addr + BIT(offset + 2))) + return GPIO_LEVEL_HIGH; + return GPIO_LEVEL_LOW; +} + +/* + * In order to write GPIODATA, the corresponding bits in the mask, resulting + * from the address bus, PADDR[9:2], must be HIGH. Otherwise the bit values + * remain unchanged by the write. + */ +static void pl061_set_value(struct gpio_chip *chip __unused, + unsigned int gpio_pin, enum gpio_level value) +{ + vaddr_t base_addr; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + if (value == GPIO_LEVEL_HIGH) + io_write8(base_addr + BIT(offset + 2), BIT(offset)); + else + io_write8(base_addr + BIT(offset + 2), 0); +} + +static enum gpio_interrupt pl061_get_interrupt(struct gpio_chip *chip __unused, + unsigned int gpio_pin) +{ + vaddr_t base_addr; + uint8_t data; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + data = io_read8(base_addr + GPIOIE); + if (data & BIT(offset)) + return GPIO_INTERRUPT_ENABLE; + return GPIO_INTERRUPT_DISABLE; +} + +static void pl061_set_interrupt(struct gpio_chip *chip __unused, + unsigned int gpio_pin, + enum gpio_interrupt ena_dis) +{ + vaddr_t base_addr; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + if (ena_dis == GPIO_INTERRUPT_ENABLE) + io_setbits8(base_addr + GPIOIE, BIT(offset)); + else + io_clrbits8(base_addr + GPIOIE, BIT(offset)); +} + +/* + * Register the PL061 GPIO controller with a base address and the offset + * of start pin in this GPIO controller. + * This function is called after pl061_init(). + */ +void pl061_register(vaddr_t base_addr, unsigned int gpio_dev) +{ + assert(gpio_dev < MAX_GPIO_DEVICES); + + pl061_reg_base[gpio_dev] = base_addr; +} + +static const struct gpio_ops pl061_ops = { + .get_direction = pl061_get_direction, + .set_direction = pl061_set_direction, + .get_value = pl061_get_value, + .set_value = pl061_set_value, + .get_interrupt = pl061_get_interrupt, + .set_interrupt = pl061_set_interrupt, +}; +DECLARE_KEEP_PAGER(pl061_ops); + +/* + * Initialize PL061 GPIO controller + */ +void pl061_init(struct pl061_data *pd) +{ + COMPILE_TIME_ASSERT(PLAT_PL061_MAX_GPIOS > 0); + + assert(pd); + pd->chip.ops = &pl061_ops; +} + +enum pl061_mode_control pl061_get_mode_control(unsigned int gpio_pin) +{ + vaddr_t base_addr; + uint8_t data; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + data = io_read8(base_addr + GPIOAFSEL); + if (data & BIT(offset)) + return PL061_MC_HW; + return PL061_MC_SW; +} + +void pl061_set_mode_control(unsigned int gpio_pin, + enum pl061_mode_control hw_sw) +{ + vaddr_t base_addr; + unsigned int offset; + + assert(gpio_pin < PLAT_PL061_MAX_GPIOS); + + base_addr = pl061_reg_base[gpio_pin / GPIOS_PER_PL061]; + offset = gpio_pin % GPIOS_PER_PL061; + if (hw_sw == PL061_MC_HW) + io_setbits8(base_addr + GPIOAFSEL, BIT(offset)); + else + io_clrbits8(base_addr + GPIOAFSEL, BIT(offset)); +} diff --git a/optee_os/core/drivers/plic.c b/optee_os/core/drivers/plic.c new file mode 100644 index 0000000..be14656 --- /dev/null +++ b/optee_os/core/drivers/plic.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PLIC_PRIORITY_OFFSET 0 +#define PLIC_PENDING_OFFSET 0x1000 +#define PLIC_ENABLE_OFFSET 0x2000 +#define PLIC_THRESHOLD_OFFSET 0x200000 +#define PLIC_CLAIM_OFFSET 0x200004 + +#define PLIC_PRIORITY_SHIFT_PER_SOURCE U(2) +#define PLIC_PENDING_SHIFT_PER_SOURCE U(0) + +#define PLIC_ENABLE_SHIFT_PER_TARGET U(7) +#define PLIC_THRESHOLD_SHIFT_PER_TARGET U(12) +#define PLIC_CLAIM_SHIFT_PER_TARGET U(12) + +#define PLIC_PRIORITY(base, source) \ + ((base) + PLIC_PRIORITY_OFFSET + \ + SHIFT_U32(source, PLIC_PRIORITY_SHIFT_PER_SOURCE) \ + ) +#define PLIC_PENDING(base, source) \ + ((base) + PLIC_PENDING_OFFSET + \ + (4 * ((source) / 32)) \ + ) +#define PLIC_ENABLE(base, source, context) \ + ((base) + PLIC_ENABLE_OFFSET + \ + SHIFT_U32(context, PLIC_ENABLE_SHIFT_PER_TARGET) +\ + (4 * ((source) / 32)) \ + ) +#define PLIC_THRESHOLD(base, context) \ + ((base) + PLIC_THRESHOLD_OFFSET + \ + SHIFT_U32(context, PLIC_THRESHOLD_SHIFT_PER_TARGET) \ + ) +#define PLIC_COMPLETE(base, context) \ + ((base) + PLIC_CLAIM_OFFSET + \ + SHIFT_U32(context, PLIC_CLAIM_SHIFT_PER_TARGET) \ + ) +#define PLIC_CLAIM(base, context) PLIC_COMPLETE(base, context) + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, PLIC_BASE, PLIC_REG_SIZE); + +struct plic_data { + vaddr_t plic_base; + size_t max_it; + struct itr_chip chip; +}; + +static struct plic_data plic_data __nex_bss; + +/* + * We assume that each hart has M-mode and S-mode, so the contexts look like: + * PLIC context 0 is hart 0 M-mode + * PLIC context 1 is hart 0 S-mode + * PLIC context 2 is hart 1 M-mode + * PLIC context 3 is hart 1 S-mode + * ... + */ +static uint32_t plic_get_context(void) +{ + size_t hartid = get_core_pos(); + bool smode = IS_ENABLED(CFG_RISCV_S_MODE) ? true : false; + + return hartid * 2 + smode; +} + +static bool __maybe_unused +plic_is_pending(struct plic_data *pd, uint32_t source) +{ + return io_read32(PLIC_PENDING(pd->plic_base, source)) & + BIT(source % 32); +} + +static void plic_set_pending(struct plic_data *pd, uint32_t source) +{ + io_setbits32(PLIC_PENDING(pd->plic_base, source), BIT(source % 32)); +} + +static void plic_enable_interrupt(struct plic_data *pd, uint32_t source) +{ + uint32_t context = plic_get_context(); + + io_setbits32(PLIC_ENABLE(pd->plic_base, source, context), + BIT(source & 0x1f)); +} + +static uint32_t __maybe_unused +plic_get_interrupt_enable(struct plic_data *pd, uint32_t source) +{ + uint32_t context = plic_get_context(); + + return io_read32(PLIC_ENABLE(pd->plic_base, source, context)) & + BIT(source & 0x1f); +} + +static void plic_disable_interrupt(struct plic_data *pd, uint32_t source) +{ + uint32_t context = plic_get_context(); + + io_clrbits32(PLIC_ENABLE(pd->plic_base, source, context), + BIT(source & 0x1f)); +} + +static uint32_t __maybe_unused plic_get_threshold(struct plic_data *pd) +{ + uint32_t context = plic_get_context(); + + return io_read32(PLIC_THRESHOLD(pd->plic_base, context)); +} + +static void plic_set_threshold(struct plic_data *pd, uint32_t threshold) +{ + uint32_t context = plic_get_context(); + + io_write32(PLIC_THRESHOLD(pd->plic_base, context), threshold); +} + +static uint32_t __maybe_unused +plic_get_priority(struct plic_data *pd, uint32_t source) +{ + return io_read32(PLIC_PRIORITY(pd->plic_base, source)); +} + +static void plic_set_priority(struct plic_data *pd, uint32_t source, + uint32_t priority) +{ + io_write32(PLIC_PRIORITY(pd->plic_base, source), priority); +} + +static uint32_t plic_claim_interrupt(struct plic_data *pd) +{ + uint32_t context = plic_get_context(); + + return io_read32(PLIC_CLAIM(pd->plic_base, context)); +} + +static void plic_complete_interrupt(struct plic_data *pd, uint32_t source) +{ + uint32_t context = plic_get_context(); + + io_write32(PLIC_CLAIM(pd->plic_base, context), source); +} + +static void plic_op_add(struct itr_chip *chip, size_t it, + uint32_t type __unused, + uint32_t prio) +{ + struct plic_data *pd = container_of(chip, struct plic_data, chip); + + if (it > pd->max_it) + panic(); + + plic_disable_interrupt(pd, it); + plic_set_priority(pd, it, prio); +} + +static void plic_op_enable(struct itr_chip *chip, size_t it) +{ + struct plic_data *pd = container_of(chip, struct plic_data, chip); + + if (it > pd->max_it) + panic(); + + plic_enable_interrupt(pd, it); +} + +static void plic_op_disable(struct itr_chip *chip, size_t it) +{ + struct plic_data *pd = container_of(chip, struct plic_data, chip); + + if (it > pd->max_it) + panic(); + + plic_disable_interrupt(pd, it); +} + +static void plic_op_raise_pi(struct itr_chip *chip, size_t it) +{ + struct plic_data *pd = container_of(chip, struct plic_data, chip); + + if (it > pd->max_it) + panic(); + + plic_set_pending(pd, it); +} + +static void plic_op_raise_sgi(struct itr_chip *chip __unused, + size_t it __unused, uint8_t cpu_mask __unused) +{ +} + +static void plic_op_set_affinity(struct itr_chip *chip __unused, + size_t it __unused, uint8_t cpu_mask __unused) +{ +} + +static int plic_dt_get_irq(const uint32_t *properties __unused, + int count __unused, uint32_t *type __unused, + uint32_t *prio __unused) +{ + return DT_INFO_INVALID_INTERRUPT; +} + +static size_t probe_max_it(vaddr_t plic_base __unused) +{ + return PLIC_NUM_SOURCES; +} + +static const struct itr_ops plic_ops = { + .add = plic_op_add, + .mask = plic_op_disable, + .unmask = plic_op_enable, + .enable = plic_op_enable, + .disable = plic_op_disable, + .raise_pi = plic_op_raise_pi, + .raise_sgi = plic_op_raise_sgi, + .set_affinity = plic_op_set_affinity, +}; + +static void plic_init_base_addr(struct plic_data *pd, paddr_t plic_base_pa) +{ + vaddr_t plic_base = 0; + + assert(cpu_mmu_enabled()); + + plic_base = core_mmu_get_va(plic_base_pa, MEM_AREA_IO_SEC, + PLIC_REG_SIZE); + if (!plic_base) + panic(); + + pd->plic_base = plic_base; + pd->max_it = probe_max_it(plic_base); + pd->chip.ops = &plic_ops; + + if (IS_ENABLED(CFG_DT)) + pd->chip.dt_get_irq = plic_dt_get_irq; +} + +void plic_hart_init(void) +{ + /* TODO: To be called by secondary harts */ +} + +void plic_init(paddr_t plic_base_pa) +{ + struct plic_data *pd = &plic_data; + size_t n = 0; + + plic_init_base_addr(pd, plic_base_pa); + + for (n = 0; n <= pd->max_it; n++) { + plic_disable_interrupt(pd, n); + plic_set_priority(pd, n, 1); + } + + plic_set_threshold(pd, 0); + + interrupt_main_init(&plic_data.chip); +} + +void plic_it_handle(void) +{ + struct plic_data *pd = &plic_data; + uint32_t id = plic_claim_interrupt(pd); + + if (id <= pd->max_it) + interrupt_call_handlers(&pd->chip, id); + else + DMSG("ignoring interrupt %" PRIu32, id); + + plic_complete_interrupt(pd, id); +} diff --git a/optee_os/core/drivers/pm/imx/gpcv2.c b/optee_os/core/drivers/pm/imx/gpcv2.c new file mode 100644 index 0000000..ec21e68 --- /dev/null +++ b/optee_os/core/drivers/pm/imx/gpcv2.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2017, 2023 NXP + * + * Peng Fan + */ + +#include +#include +#include + +#include "local.h" + +#define GPC_PGC_C1 0x840 +#define GPC_PGC_PCG_MASK BIT(0) + +#define GPC_CPU_PGC_SW_PUP_REQ 0xf0 +#define GPC_PU_PGC_SW_PUP_REQ 0xf8 +#define GPC_CPU_PGC_SW_PDN_REQ 0xfc +#define GPC_PU_PGC_SW_PDN_REQ 0x104 +#define GPC_PGC_SW_PDN_PUP_REQ_CORE1_MASK BIT(1) + +static void imx_gpcv2_set_core_pgc(bool enable, uint32_t offset) +{ + vaddr_t va = core_mmu_get_va(GPC_BASE, MEM_AREA_IO_SEC, GPC_SIZE); + + if (enable) + io_setbits32(va + offset, GPC_PGC_PCG_MASK); + else + io_clrbits32(va + offset, GPC_PGC_PCG_MASK); +} + +void imx_gpcv2_set_core1_pup_by_software(void) +{ + vaddr_t va = core_mmu_get_va(GPC_BASE, MEM_AREA_IO_SEC, GPC_SIZE); + uint64_t timeout = timeout_init_us(10 * 1000); + + imx_gpcv2_set_core_pgc(true, GPC_PGC_C1); + + io_setbits32(va + GPC_CPU_PGC_SW_PUP_REQ, + GPC_PGC_SW_PDN_PUP_REQ_CORE1_MASK); + + while ((io_read32(va + GPC_CPU_PGC_SW_PUP_REQ) & + GPC_PGC_SW_PDN_PUP_REQ_CORE1_MASK)) { + if (timeout_elapsed(timeout)) + return; + } + + imx_gpcv2_set_core_pgc(false, GPC_PGC_C1); +} diff --git a/optee_os/core/drivers/pm/imx/local.h b/optee_os/core/drivers/pm/imx/local.h new file mode 100644 index 0000000..ee2a0bf --- /dev/null +++ b/optee_os/core/drivers/pm/imx/local.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2023 NXP + */ + +#include + +/* + * Read the SRC GPR ARG register for the given core number + * @cpu Core number + */ +uint32_t imx_get_src_gpr_arg(unsigned int cpu); + +/* + * Set the SRC GPR ARG register for the given core number + * @cpu Core number + * @val Register value to set + */ +void imx_set_src_gpr_arg(unsigned int cpu, uint32_t val); + +/* + * Read the SRC GPR ENTRY register for the given core number + * @cpu Core number + */ +uint32_t imx_get_src_gpr_entry(unsigned int cpu); + +/* + * Set the SRC GPR ENTRY register for the given core number + * @cpu Core number + * @val Register value to set + */ +void imx_set_src_gpr_entry(unsigned int cpu, uint32_t val); + +/* + * Release the given core + * @cpu Core number + */ +void imx_src_release_secondary_core(unsigned int cpu); + +/* + * Shutdown the given core + * @cpu Core number + */ +void imx_src_shutdown_core(unsigned int cpu); + +/* + * GPC Core 1 power down + */ +void imx_gpcv2_set_core1_pup_by_software(void); diff --git a/optee_os/core/drivers/pm/imx/psci.c b/optee_os/core/drivers/pm/imx/psci.c new file mode 100644 index 0000000..a0e21dd --- /dev/null +++ b/optee_os/core/drivers/pm/imx/psci.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2016 Freescale Semiconductor, Inc. + * Copyright 2023 NXP + * + * Peng Fan + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "local.h" + +#define IOMUXC_GPR5_OFFSET 0x14 +#define ARM_WFI_STAT_MASK(n) BIT(n) + +int psci_features(uint32_t psci_fid) +{ + switch (psci_fid) { + case ARM_SMCCC_VERSION: + case PSCI_PSCI_FEATURES: + case PSCI_VERSION: + case PSCI_CPU_OFF: +#ifdef CFG_BOOT_SECONDARY_REQUEST + case PSCI_CPU_ON: +#endif + case PSCI_AFFINITY_INFO: + case PSCI_SYSTEM_OFF: + case PSCI_SYSTEM_RESET: + case PSCI_SYSTEM_RESET2: + return PSCI_RET_SUCCESS; + default: + return PSCI_RET_NOT_SUPPORTED; + } +} + +uint32_t psci_version(void) +{ + return PSCI_VERSION_1_0; +} + +#ifdef CFG_BOOT_SECONDARY_REQUEST +int psci_cpu_on(uint32_t core_idx, uint32_t entry, uint32_t context_id) +{ + if (core_idx == 0 || core_idx >= CFG_TEE_CORE_NB_CORE) + return PSCI_RET_INVALID_PARAMETERS; + + /* set secondary cores' NS entry addresses */ + boot_set_core_ns_entry(core_idx, entry, context_id); + imx_set_src_gpr_entry(core_idx, virt_to_phys((void *)TEE_LOAD_ADDR)); + +#ifdef CFG_MX7 + imx_gpcv2_set_core1_pup_by_software(); + imx_src_release_secondary_core(core_idx); +#else + imx_src_release_secondary_core(core_idx); + imx_set_src_gpr_arg(core_idx, 0); +#endif /* CFG_MX7 */ + + IMSG("psci on ok"); + + return PSCI_RET_SUCCESS; +} + +int __noreturn psci_cpu_off(void) +{ + uint32_t core_id = get_core_pos(); + + IMSG("core_id: %" PRIu32, core_id); + + psci_armv7_cpu_off(); + + imx_set_src_gpr_arg(core_id, UINT32_MAX); + + thread_mask_exceptions(THREAD_EXCP_ALL); + + while (true) + wfi(); +} + +int psci_affinity_info(uint32_t affinity, + uint32_t lowest_affnity_level __unused) +{ + vaddr_t base = core_mmu_get_va(IOMUXC_BASE, MEM_AREA_IO_SEC, + IOMUXC_SIZE); + uint32_t cpu = affinity; + bool wfi = true; + + if (!soc_is_imx7ds()) + wfi = io_read32(base + IOMUXC_GPR5_OFFSET) & + ARM_WFI_STAT_MASK(cpu); + + if (imx_get_src_gpr_arg(cpu) == 0 || !wfi) + return PSCI_AFFINITY_LEVEL_ON; + + DMSG("cpu: %" PRIu32 "GPR: %" PRIx32, cpu, imx_get_src_gpr_arg(cpu)); + + while (imx_get_src_gpr_arg(cpu) != UINT_MAX) + ; + + imx_src_shutdown_core(cpu); + imx_set_src_gpr_arg(cpu, 0); + + return PSCI_AFFINITY_LEVEL_OFF; +} +#endif + +void __noreturn psci_system_off(void) +{ +#ifndef CFG_MX7ULP + imx_snvs_shutdown(); +#endif + dsb(); + + while (1) + ; +} + +void __noreturn psci_system_reset(void) +{ + imx_wdog_restart(true); +} + +int __noreturn psci_system_reset2(uint32_t reset_type __unused, + uint32_t cookie __unused) +{ + /* force WDOG reset */ + imx_wdog_restart(false); +} diff --git a/optee_os/core/drivers/pm/imx/src.c b/optee_os/core/drivers/pm/imx/src.c new file mode 100644 index 0000000..d6cb46a --- /dev/null +++ b/optee_os/core/drivers/pm/imx/src.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019, 2023 NXP + */ + +#include +#include +#include +#include + +#include "local.h" + +#define SRC_SCR 0x000 +#define SRC_A7RCR0 0x004 +#define SRC_A7RCR1 0x008 +#if defined(CFG_MX7) +#define SRC_GPR1 0x074 +#else +#define SRC_GPR1 0x020 +#endif + +#define SRC_SCR_CORE1_RST_BIT(_cpu) BIT32(14 + (_cpu) - 1) +#define SRC_SCR_CORE1_ENABLE_BIT(_cpu) BIT32(22 + (_cpu) - 1) +#define SRC_A7RCR0_A7_CORE_RESET0_BIT(_cpu) BIT32((_cpu) - 1) +#define SRC_A7RCR1_A7_CORE1_ENABLE_BIT(_cpu) BIT32(1 + (_cpu) - 1) + +#define ENTRY_OFFSET(_cpu) ((_cpu) * 8) +#define ARG_OFFSET(_cpu) (ENTRY_OFFSET(_cpu) + 4) + +register_phys_mem(MEM_AREA_IO_SEC, SRC_BASE, SRC_SIZE); + +uint32_t imx_get_src_gpr_arg(unsigned int cpu) +{ + vaddr_t va = core_mmu_get_va(SRC_BASE, MEM_AREA_IO_SEC, SRC_SIZE); + + return io_read32(va + SRC_GPR1 + ARG_OFFSET(cpu)); +} + +void imx_set_src_gpr_arg(unsigned int cpu, uint32_t val) +{ + vaddr_t va = core_mmu_get_va(SRC_BASE, MEM_AREA_IO_SEC, SRC_SIZE); + + io_write32(va + SRC_GPR1 + ARG_OFFSET(cpu), val); +} + +uint32_t imx_get_src_gpr_entry(unsigned int cpu) +{ + vaddr_t va = core_mmu_get_va(SRC_BASE, MEM_AREA_IO_SEC, SRC_SIZE); + + return io_read32(va + SRC_GPR1 + ENTRY_OFFSET(cpu)); +} + +void imx_set_src_gpr_entry(unsigned int cpu, uint32_t val) +{ + vaddr_t va = core_mmu_get_va(SRC_BASE, MEM_AREA_IO_SEC, SRC_SIZE); + + io_write32(va + SRC_GPR1 + ENTRY_OFFSET(cpu), val); +} + +void imx_src_release_secondary_core(unsigned int cpu) +{ + vaddr_t va = core_mmu_get_va(SRC_BASE, MEM_AREA_IO_SEC, SRC_SIZE); + + if (soc_is_imx7ds()) + io_setbits32(va + SRC_A7RCR1, + SRC_A7RCR1_A7_CORE1_ENABLE_BIT(cpu)); + else + io_setbits32(va + SRC_SCR, SRC_SCR_CORE1_ENABLE_BIT(cpu) | + SRC_SCR_CORE1_RST_BIT(cpu)); +} + +void imx_src_shutdown_core(unsigned int cpu) +{ + vaddr_t va = core_mmu_get_va(SRC_BASE, MEM_AREA_IO_SEC, SRC_SIZE); + + if (soc_is_imx7ds()) { + io_clrbits32(va + SRC_A7RCR1, + SRC_A7RCR1_A7_CORE1_ENABLE_BIT(cpu)); + } else { + uint32_t mask = io_read32(va + SRC_SCR); + + mask &= ~SRC_SCR_CORE1_ENABLE_BIT(cpu); + mask |= SRC_SCR_CORE1_RST_BIT(cpu); + io_write32(va + SRC_SCR, mask); + } +} diff --git a/optee_os/core/drivers/pm/imx/sub.mk b/optee_os/core/drivers/pm/imx/sub.mk new file mode 100644 index 0000000..c3860cb --- /dev/null +++ b/optee_os/core/drivers/pm/imx/sub.mk @@ -0,0 +1,3 @@ +srcs-y += psci.c +srcs-$(CFG_MX7) += gpcv2.c +srcs-$(CFG_MX6)$(CFG_MX7) += src.c diff --git a/optee_os/core/drivers/pm/sam/at91_pm.c b/optee_os/core/drivers/pm/sam/at91_pm.c new file mode 100644 index 0000000..86fb049 --- /dev/null +++ b/optee_os/core/drivers/pm/sam/at91_pm.c @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "at91_pm.h" + +#if CFG_ATMEL_PM_SUSPEND_MODE < AT91_PM_STANDBY || \ + CFG_ATMEL_PM_SUSPEND_MODE > AT91_PM_BACKUP +#error Invalid suspend mode, please check CFG_ATMEL_PM_SUSPEND_MODE +#endif + +#define AT91_SECUMOD_SYSR 0x04 +#define AT91_SECUMOD_RAMRDY 0x14 +#define AT91_SECUMOD_RAMRDY_READY BIT(0) + +static struct at91_pm_data soc_pm; + +/* Backup canary */ +static uint32_t canary = 0xA5A5A5A5; + +/* Backup mode information used by at91bootstrap */ +static struct at91bootstrap_bu { + uint32_t suspended; + uint32_t reserved; + uint32_t *canary; + uint32_t resume; +} *at91bootstrap_bu; + +static vaddr_t at91_suspend_sram_base; +static void (*at91_suspend_sram_fn)(struct at91_pm_data *); + +enum sm_handler_ret at91_pm_set_suspend_mode(struct thread_smc_args *args) +{ + unsigned int mode = args->a1; + + /* + * We don't expect this function to be called simultaneously while we + * are entering suspend/resume function. On sama5d2, this is not a + * problem since this SoC is a single core one but in order to prevent + * any other SoC support to be added without handling this concurrency, + * check that we are compiled for a single core. + */ + COMPILE_TIME_ASSERT(CFG_TEE_CORE_NB_CORE == 1); + + if (mode > AT91_PM_BACKUP) { + args->a0 = SAMA5_SMC_SIP_RETURN_EINVAL; + return SM_HANDLER_SMC_HANDLED; + } + DMSG("Setting suspend mode to %u", mode); + + args->a0 = SAMA5_SMC_SIP_RETURN_SUCCESS; + soc_pm.mode = mode; + + return SM_HANDLER_SMC_HANDLED; +} + +enum sm_handler_ret at91_pm_get_suspend_mode(struct thread_smc_args *args) +{ + args->a1 = soc_pm.mode; + args->a0 = SAMA5_SMC_SIP_RETURN_SUCCESS; + + return SM_HANDLER_SMC_HANDLED; +} + +static void at91_pm_copy_suspend_to_sram(void) +{ + memcpy((void *)at91_suspend_sram_base, &at91_pm_suspend_in_sram, + at91_pm_suspend_in_sram_sz); + + cache_op_inner(DCACHE_AREA_CLEAN, (void *)at91_suspend_sram_base, + at91_pm_suspend_in_sram_sz); + cache_op_inner(ICACHE_AREA_INVALIDATE, at91_suspend_sram_fn, + at91_pm_suspend_in_sram_sz); +} + +void atmel_pm_cpu_idle(void) +{ + uint32_t lpr0 = 0; + uint32_t saved_lpr0 = 0; + + saved_lpr0 = io_read32(soc_pm.ramc + AT91_DDRSDRC_LPR); + lpr0 = saved_lpr0 & ~AT91_DDRSDRC_LPCB; + lpr0 |= AT91_DDRSDRC_LPCB_POWER_DOWN; + + io_write32(soc_pm.ramc + AT91_DDRSDRC_LPR, lpr0); + + cpu_idle(); + + io_write32(soc_pm.ramc + AT91_DDRSDRC_LPR, saved_lpr0); +} + +static void at91_sama5d2_config_shdwc_ws(vaddr_t shdwc, uint32_t *mode, + uint32_t *polarity) +{ + uint32_t val = 0; + + /* SHDWC.WUIR */ + val = io_read32(shdwc + AT91_SHDW_WUIR); + *mode |= val & AT91_SHDW_WKUPEN_MASK; + *polarity |= (val >> AT91_SHDW_WKUPT_SHIFT) & AT91_SHDW_WKUPT_MASK; +} + +static int at91_sama5d2_config_pmc_ws(vaddr_t pmc, uint32_t mode, + uint32_t polarity) +{ + io_write32(pmc + AT91_PMC_FSMR, mode); + io_write32(pmc + AT91_PMC_FSPR, polarity); + + return 0; +} + +struct wakeup_source_info { + unsigned int pmc_fsmr_bit; + unsigned int shdwc_mr_bit; + bool set_polarity; +}; + +static const struct wakeup_source_info ws_info[] = { + { .pmc_fsmr_bit = AT91_PMC_FSTT(10), .set_polarity = true }, + { .pmc_fsmr_bit = AT91_PMC_RTCAL, .shdwc_mr_bit = BIT(17) }, + { .pmc_fsmr_bit = AT91_PMC_USBAL }, + { .pmc_fsmr_bit = AT91_PMC_SDMMC_CD }, +}; + +struct wakeup_src { + const char *compatible; + const struct wakeup_source_info *info; +}; + +static const struct wakeup_src sama5d2_ws_ids[] = { + { .compatible = "atmel,sama5d2-gem", .info = &ws_info[0] }, + { .compatible = "atmel,at91rm9200-rtc", .info = &ws_info[1] }, + { .compatible = "atmel,sama5d3-udc", .info = &ws_info[2] }, + { .compatible = "atmel,at91rm9200-ohci", .info = &ws_info[2] }, + { .compatible = "usb-ohci", .info = &ws_info[2] }, + { .compatible = "atmel,at91sam9g45-ehci", .info = &ws_info[2] }, + { .compatible = "usb-ehci", .info = &ws_info[2] }, + { .compatible = "atmel,sama5d2-sdhci", .info = &ws_info[3] } +}; + +static bool dev_is_wakeup_source(const void *fdt, int node) +{ + return fdt_get_property(fdt, node, "wakeup-source", NULL); +} + +static int at91_pm_config_ws_ulp1(bool set) +{ + const struct wakeup_source_info *wsi = NULL; + const struct wakeup_src *wsrc = NULL; + unsigned int polarity = 0; + unsigned int mode = 0; + unsigned int val = 0; + unsigned int src = 0; + int node = 0; + + if (!set) { + io_write32(soc_pm.pmc + AT91_PMC_FSMR, mode); + return TEE_SUCCESS; + } + + at91_sama5d2_config_shdwc_ws(soc_pm.shdwc, &mode, &polarity); + + val = io_read32(soc_pm.shdwc + AT91_SHDW_MR); + + /* Loop through defined wakeup sources. */ + for (src = 0; src < ARRAY_SIZE(sama5d2_ws_ids); src++) { + wsrc = &sama5d2_ws_ids[src]; + wsi = wsrc->info; + + node = fdt_node_offset_by_compatible(soc_pm.fdt, -1, + wsrc->compatible); + while (node >= 0) { + if (dev_is_wakeup_source(soc_pm.fdt, node)) { + /* Check if enabled on SHDWC. */ + if (wsi->shdwc_mr_bit && + !(val & wsi->shdwc_mr_bit)) + goto next_node; + + mode |= wsi->pmc_fsmr_bit; + if (wsi->set_polarity) + polarity |= wsi->pmc_fsmr_bit; + } +next_node: + node = fdt_node_offset_by_compatible(soc_pm.fdt, node, + wsrc->compatible); + } + } + + if (!mode) { + EMSG("AT91: PM: no ULP1 wakeup sources found!"); + return TEE_ERROR_BAD_STATE; + } + + at91_sama5d2_config_pmc_ws(soc_pm.pmc, mode, polarity); + + return TEE_SUCCESS; +} + +/* + * Verify that all the clocks are correct before entering + * slow-clock mode. + */ +static bool at91_pm_verify_clocks(void) +{ + int i = 0; + uint32_t scsr = 0; + + scsr = io_read32(soc_pm.pmc + AT91_PMC_SCSR); + + /* USB must not be using PLLB */ + if ((scsr & (AT91SAM926x_PMC_UHP | AT91SAM926x_PMC_UDP)) != 0) { + EMSG("AT91: PM - Suspend-to-RAM with USB still active"); + return false; + } + + /* PCK0..PCK3 must be disabled, or configured to use clk32k */ + for (i = 0; i < 4; i++) { + uint32_t css = 0; + + if ((scsr & (AT91_PMC_PCK0 << i)) == 0) + continue; + css = io_read32(soc_pm.pmc + AT91_PMC_PCKR(i)) & AT91_PMC_CSS; + if (css != AT91_PMC_CSS_SLOW) { + EMSG("AT91: PM - Suspend-to-RAM with PCK%d src %"PRId32, + i, css); + return false; + } + } + + return true; +} + +static TEE_Result at91_write_backup_data(void) +{ + uint32_t val = 0; + + while (true) { + val = io_read32(soc_pm.secumod + AT91_SECUMOD_RAMRDY); + if (val & AT91_SECUMOD_RAMRDY_READY) + break; + } + + io_write32((vaddr_t)&at91bootstrap_bu->suspended, 1); + io_write32((vaddr_t)&at91bootstrap_bu->canary, virt_to_phys(&canary)); + io_write32((vaddr_t)&at91bootstrap_bu->resume, + virt_to_phys((void *)(vaddr_t)at91_pm_cpu_resume)); + + return TEE_SUCCESS; +} + +static TEE_Result at91_enter_backup(void) +{ + int ret = -1; + TEE_Result res = TEE_ERROR_GENERIC; + + res = at91_write_backup_data(); + if (res) + return res; + + pm_change_state(PM_OP_SUSPEND, 0); + ret = sm_pm_cpu_suspend((uint32_t)&soc_pm, + (void *)at91_suspend_sram_fn); + if (ret < 0) { + DMSG("Suspend failed"); + res = TEE_ERROR_BAD_STATE; + } else { + res = TEE_SUCCESS; + } + + pm_change_state(PM_OP_RESUME, 0); + if (res) + return res; + + /* SRAM content is lost after resume */ + at91_pm_copy_suspend_to_sram(); + + return TEE_SUCCESS; +} + +TEE_Result atmel_pm_suspend(uintptr_t entry, struct sm_nsec_ctx *nsec) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + DMSG("Entering suspend mode %d", soc_pm.mode); + + if (soc_pm.mode >= AT91_PM_ULP0) { + if (!at91_pm_verify_clocks()) + return TEE_ERROR_BAD_STATE; + } + + if (soc_pm.mode == AT91_PM_ULP1) + at91_pm_config_ws_ulp1(true); + + sm_save_unbanked_regs(&nsec->ub_regs); + + if (soc_pm.mode == AT91_PM_BACKUP) { + res = at91_enter_backup(); + } else { + at91_suspend_sram_fn(&soc_pm); + res = TEE_SUCCESS; + } + + if (soc_pm.mode == AT91_PM_ULP1) + at91_pm_config_ws_ulp1(false); + + sm_restore_unbanked_regs(&nsec->ub_regs); + + /* + * If the system went to backup mode, register state was lost and must + * be restored by jumping to the user provided entry point + */ + if (res == TEE_SUCCESS && soc_pm.mode == AT91_PM_BACKUP) + nsec->mon_lr = entry; + + DMSG("Exiting suspend mode %d, res %"PRIx32, soc_pm.mode, res); + + return res; +} + +static TEE_Result at91_pm_dt_dram_init(const void *fdt) +{ + int node = -1; + size_t size = 0; + + node = fdt_node_offset_by_compatible(fdt, -1, "atmel,sama5d3-ddramc"); + if (node < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (dt_map_dev(fdt, node, &soc_pm.ramc, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result at91_pm_backup_init(const void *fdt) +{ + int node = -1; + size_t size = 0; + + node = fdt_node_offset_by_compatible(fdt, -1, "atmel,sama5d2-sfrbu"); + if (node < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (dt_map_dev(fdt, node, &soc_pm.sfrbu, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + if (fdt_get_status(fdt, node) == DT_STATUS_OK_SEC) + matrix_configure_periph_secure(AT91C_ID_SFRBU); + + return TEE_SUCCESS; +} + +static TEE_Result at91_pm_sram_init(const void *fdt) +{ + int node = -1; + size_t size = 0; + paddr_t at91_suspend_sram_pbase; + size_t suspend_sz = at91_pm_suspend_in_sram_sz; + + node = fdt_node_offset_by_compatible(fdt, -1, "atmel,sama5d2-sram"); + if (node < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_GENERIC; + + if (dt_map_dev(fdt, node, &at91_suspend_sram_base, &size, + DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + at91_suspend_sram_pbase = virt_to_phys((void *)at91_suspend_sram_base); + + /* Map the secure ram suspend code to be executable */ + at91_suspend_sram_fn = core_mmu_add_mapping(MEM_AREA_TEE_RAM, + at91_suspend_sram_pbase, + suspend_sz); + if (!at91_suspend_sram_fn) { + EMSG("Failed to remap sram as executable"); + return TEE_ERROR_GENERIC; + } + + at91_pm_copy_suspend_to_sram(); + + return TEE_SUCCESS; +} + +static TEE_Result at91_securam_init(const void *fdt) +{ + int node = -1; + size_t size = 0; + struct clk *clk = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + node = fdt_node_offset_by_compatible(fdt, -1, "atmel,sama5d2-securam"); + if (node < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_GENERIC; + + if (dt_map_dev(fdt, node, &soc_pm.securam, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + res = clk_dt_get_by_index(fdt, node, 0, &clk); + if (res) + return res; + + if (clk_enable(clk)) + return TEE_ERROR_GENERIC; + + if (size < sizeof(struct at91bootstrap_bu)) + return TEE_ERROR_SHORT_BUFFER; + + at91bootstrap_bu = (void *)soc_pm.securam; + + node = fdt_node_offset_by_compatible(fdt, -1, "atmel,sama5d2-secumod"); + if (node < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) + return TEE_ERROR_GENERIC; + + if (dt_map_dev(fdt, node, &soc_pm.secumod, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result sama5d2_pm_init_all(const void *fdt, vaddr_t shdwc) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + soc_pm.fdt = fdt; + soc_pm.shdwc = shdwc; + soc_pm.pmc = at91_pmc_get_base(); + if (!soc_pm.pmc) + return TEE_ERROR_GENERIC; + + soc_pm.mode = CFG_ATMEL_PM_SUSPEND_MODE; + + res = at91_securam_init(fdt); + if (res) + return res; + + res = at91_pm_dt_dram_init(fdt); + if (res) + return res; + + res = at91_pm_backup_init(fdt); + if (res) + return res; + + res = at91_pm_sram_init(fdt); + if (res) + return res; + + return TEE_SUCCESS; +} + +TEE_Result sama5d2_pm_init(const void *fdt, vaddr_t shdwc) +{ + if (sama5d2_pm_init_all(fdt, shdwc)) + panic("Failed to setup PM for sama5d2"); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/pm/sam/at91_pm.h b/optee_os/core/drivers/pm/sam/at91_pm.h new file mode 100644 index 0000000..47b2d72 --- /dev/null +++ b/optee_os/core/drivers/pm/sam/at91_pm.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Microchip + */ + +#ifndef AT91_PM_H +#define AT91_PM_H + +#define AT91_PM_STANDBY 0x00 +#define AT91_PM_ULP0 0x01 +#define AT91_PM_ULP0_FAST 0x02 +#define AT91_PM_ULP1 0x03 +#define AT91_PM_BACKUP 0x04 + +#ifndef __ASSEMBLER__ + +#include +#include +#include +#include + +struct at91_pm_data { + vaddr_t shdwc; + vaddr_t securam; + vaddr_t secumod; + vaddr_t sfrbu; + vaddr_t pmc; + vaddr_t ramc; + unsigned int mode; + const void *fdt; +}; + +void at91_pm_suspend_in_sram(struct at91_pm_data *pm_data); +void at91_pm_cpu_resume(void); +extern uint32_t at91_pm_suspend_in_sram_sz; + +void at91_pm_resume(struct at91_pm_data *pm_data); + +#endif /* __ASSEMBLER__ */ + +#endif /* AT91_PM_H */ diff --git a/optee_os/core/drivers/pm/sam/pm-defines.c b/optee_os/core/drivers/pm/sam/pm-defines.c new file mode 100644 index 0000000..94e31b8 --- /dev/null +++ b/optee_os/core/drivers/pm/sam/pm-defines.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Microchip + */ + +#include "at91_pm.h" +#include +#include +#include + +DEFINES +{ + DEFINE(PM_DATA_PMC, offsetof(struct at91_pm_data, pmc)); + DEFINE(PM_DATA_RAMC0, offsetof(struct at91_pm_data, ramc)); + DEFINE(PM_DATA_MODE, offsetof(struct at91_pm_data, mode)); + DEFINE(PM_DATA_SHDWC, offsetof(struct at91_pm_data, shdwc)); + DEFINE(PM_DATA_SFRBU, offsetof(struct at91_pm_data, sfrbu)); +} diff --git a/optee_os/core/drivers/pm/sam/pm_resume.S b/optee_os/core/drivers/pm/sam/pm_resume.S new file mode 100644 index 0000000..367fee1 --- /dev/null +++ b/optee_os/core/drivers/pm/sam/pm_resume.S @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include + +FUNC at91_pm_cpu_resume, : +UNWIND( .cantunwind) + cps #CPSR_MODE_MON + + blx plat_cpu_reset_early + + b sm_pm_cpu_resume +END_FUNC at91_pm_cpu_resume diff --git a/optee_os/core/drivers/pm/sam/pm_suspend.S b/optee_os/core/drivers/pm/sam/pm_suspend.S new file mode 100644 index 0000000..9367d23 --- /dev/null +++ b/optee_os/core/drivers/pm/sam/pm_suspend.S @@ -0,0 +1,520 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * This file was imported from Linux arch/arm/mach-at91/pm_suspend.S and + * relicensed with dual GPL-2.0/BSD-2-Clause with Microchip agreement. + * + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include + +#include "at91_pm.h" + +#define SRAMC_SELF_FRESH_ACTIVE 0x01 +#define SRAMC_SELF_FRESH_EXIT 0x00 + +pmc .req r0 +tmp1 .req r4 +tmp2 .req r5 +tmp3 .req r6 + +/* + * Wait until master clock is ready (after switching master clock source) + */ +.macro wait_mckrdy +1: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_MCKRDY + beq 1b +.endm + +/* + * Wait until master oscillator has stabilized. + */ +.macro wait_moscrdy +1: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_MOSCS + beq 1b +.endm + +/* + * Wait for main oscillator selection is done + */ +.macro wait_moscsels +1: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_MOSCSELS + beq 1b +.endm + +/* + * Put the processor to enter the idle state + */ +.macro at91_cpu_idle + + mov tmp1, #AT91_PMC_PCK + str tmp1, [pmc, #AT91_PMC_SCDR] + + dsb + + wfi @ Wait For Interrupt + +.endm + +.section .text.psci.suspend + +.arm + + +#define SUSPEND_FUNC(__name) \ +__name: + +#define SUSPEND_END_FUNC(__name) \ + .size __name, .-__name + +.macro check_fit_in_sram since + .if (. - \since) > 0x10000 + .error "Suspend assembly code exceeds dedicated SRAM size" + .endif +.endm + +/* + * void at91_suspend_sram_fn(struct at91_pm_data*) + * @input param: + * @r0: base address of struct at91_pm_data + */ +.align 3 +.global at91_pm_suspend_in_sram +SUSPEND_FUNC(at91_pm_suspend_in_sram) + /* Save registers on stack */ + stmfd sp!, {r4 - r12, lr} + + /* Drain write buffer */ + mov tmp1, #0 + mcr p15, 0, tmp1, c7, c10, 4 + + ldr tmp1, [r0, #PM_DATA_PMC] + str tmp1, .pmc_base + ldr tmp1, [r0, #PM_DATA_RAMC0] + str tmp1, .sramc_base + ldr tmp1, [r0, #PM_DATA_MODE] + str tmp1, .pm_mode + /* Both ldrne below are here to preload their address in the TLB */ + ldr tmp1, [r0, #PM_DATA_SHDWC] + str tmp1, .shdwc + cmp tmp1, #0 + ldrne tmp2, [tmp1, #0] + ldr tmp1, [r0, #PM_DATA_SFRBU] + str tmp1, .sfrbu + cmp tmp1, #0 + ldrne tmp2, [tmp1, #0x10] + + /* Active the self-refresh mode */ + mov r0, #SRAMC_SELF_FRESH_ACTIVE + bl at91_sramc_self_refresh + + ldr r0, .pm_mode + cmp r0, #AT91_PM_STANDBY + beq standby + cmp r0, #AT91_PM_BACKUP + beq backup_mode + + bl at91_ulp_mode + b exit_suspend + +standby: + /* Wait for interrupt */ + ldr pmc, .pmc_base + at91_cpu_idle + b exit_suspend + +backup_mode: + bl at91_backup_mode + b exit_suspend + +exit_suspend: + /* Exit the self-refresh mode */ + mov r0, #SRAMC_SELF_FRESH_EXIT + bl at91_sramc_self_refresh + + /* Restore registers, and return */ + ldmfd sp!, {r4 - r12, pc} +SUSPEND_END_FUNC(at91_pm_suspend_in_sram) + +SUSPEND_FUNC(at91_backup_mode) + /* Switch the master clock source to slow clock. */ + ldr pmc, .pmc_base + ldr tmp1, [pmc, #AT91_PMC_MCKR] + bic tmp1, tmp1, #AT91_PMC_CSS + str tmp1, [pmc, #AT91_PMC_MCKR] + + wait_mckrdy + + /*BUMEN*/ + ldr r0, .sfrbu + mov tmp1, #0x1 + str tmp1, [r0, #0x10] + + /* Shutdown */ + ldr r0, .shdwc + mov tmp1, #0xA5000000 + add tmp1, tmp1, #0x1 + str tmp1, [r0, #0] +SUSPEND_END_FUNC(at91_backup_mode) + +.macro at91_pm_ulp0_mode + ldr pmc, .pmc_base + ldr tmp2, .pm_mode + + /* Check if ULP0 fast variant has been requested. */ + cmp tmp2, #AT91_PM_ULP0_FAST + bne 0f + + /* Set highest prescaler for power saving */ + ldr tmp1, [pmc, #AT91_PMC_MCKR] + bic tmp1, tmp1, #AT91_PMC_PRES + orr tmp1, tmp1, #AT91_PMC_PRES_64 + str tmp1, [pmc, #AT91_PMC_MCKR] + wait_mckrdy + b 1f + +0: + /* Turn off the crystal oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + bic tmp1, tmp1, #AT91_PMC_MOSCEN + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + /* Save RC oscillator state */ + ldr tmp1, [pmc, #AT91_PMC_SR] + str tmp1, .saved_osc_status + tst tmp1, #AT91_PMC_MOSCRCS + bne 1f + + /* Turn off RC oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + bic tmp1, tmp1, #AT91_PMC_MOSCRCEN + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + /* Wait main RC disabled done */ +2: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_MOSCRCS + bne 2b + + /* Wait for interrupt */ +1: at91_cpu_idle + + /* Check if ULP0 fast variant has been requested. */ + cmp tmp2, #AT91_PM_ULP0_FAST + bne 5f + + /* Set lowest prescaler for fast resume. */ + ldr tmp1, [pmc, #AT91_PMC_MCKR] + bic tmp1, tmp1, #AT91_PMC_PRES + str tmp1, [pmc, #AT91_PMC_MCKR] + wait_mckrdy + b 6f + +5: /* Restore RC oscillator state */ + ldr tmp1, .saved_osc_status + tst tmp1, #AT91_PMC_MOSCRCS + beq 4f + + /* Turn on RC oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + orr tmp1, tmp1, #AT91_PMC_MOSCRCEN + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + /* Wait main RC stabilization */ +3: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_MOSCRCS + beq 3b + + /* Turn on the crystal oscillator */ +4: ldr tmp1, [pmc, #AT91_CKGR_MOR] + orr tmp1, tmp1, #AT91_PMC_MOSCEN + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + wait_moscrdy +6: +.endm + +/** + * Note: This procedure only applies on the platform which uses + * the external crystal oscillator as a main clock source. + */ +.macro at91_pm_ulp1_mode + ldr pmc, .pmc_base + + /* Save RC oscillator state and check if it is enabled. */ + ldr tmp1, [pmc, #AT91_PMC_SR] + str tmp1, .saved_osc_status + tst tmp1, #AT91_PMC_MOSCRCS + bne 2f + + /* Enable RC oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + orr tmp1, tmp1, #AT91_PMC_MOSCRCEN + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + /* Wait main RC stabilization */ +1: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_MOSCRCS + beq 1b + + /* Switch the main clock source to 12-MHz RC oscillator */ +2: ldr tmp1, [pmc, #AT91_CKGR_MOR] + bic tmp1, tmp1, #AT91_PMC_MOSCSEL + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + wait_moscsels + + /* Disable the crystal oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + bic tmp1, tmp1, #AT91_PMC_MOSCEN + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + /* Switch the master clock source to main clock */ + ldr tmp1, [pmc, #AT91_PMC_MCKR] + bic tmp1, tmp1, #AT91_PMC_CSS + orr tmp1, tmp1, #AT91_PMC_CSS_MAIN + str tmp1, [pmc, #AT91_PMC_MCKR] + + wait_mckrdy + + /* Enter the ULP1 mode by set WAITMODE bit in CKGR_MOR */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + orr tmp1, tmp1, #AT91_PMC_WAITMODE + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + /* Quirk for SAM9X60's PMC */ + nop + nop + + wait_mckrdy + + /* Enable the crystal oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + orr tmp1, tmp1, #AT91_PMC_MOSCEN + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + wait_moscrdy + + /* Switch the master clock source to slow clock */ + ldr tmp1, [pmc, #AT91_PMC_MCKR] + bic tmp1, tmp1, #AT91_PMC_CSS + str tmp1, [pmc, #AT91_PMC_MCKR] + + wait_mckrdy + + /* Switch main clock source to crystal oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + orr tmp1, tmp1, #AT91_PMC_MOSCSEL + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + wait_moscsels + + /* Switch the master clock source to main clock */ + ldr tmp1, [pmc, #AT91_PMC_MCKR] + bic tmp1, tmp1, #AT91_PMC_CSS + orr tmp1, tmp1, #AT91_PMC_CSS_MAIN + str tmp1, [pmc, #AT91_PMC_MCKR] + + wait_mckrdy + + /* Restore RC oscillator state */ + ldr tmp1, .saved_osc_status + tst tmp1, #AT91_PMC_MOSCRCS + bne 3f + + /* Disable RC oscillator */ + ldr tmp1, [pmc, #AT91_CKGR_MOR] + bic tmp1, tmp1, #AT91_PMC_MOSCRCEN + bic tmp1, tmp1, #AT91_PMC_KEY_MASK + orr tmp1, tmp1, #AT91_PMC_KEY + str tmp1, [pmc, #AT91_CKGR_MOR] + + /* Wait RC oscillator disable done */ +4: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_MOSCRCS + bne 4b + +3: +.endm + +.macro at91_plla_disable + /* Save PLLA setting and disable it */ + ldr tmp1, [pmc, #AT91_CKGR_PLLAR] + str tmp1, .saved_pllar + + /* Disable PLLA. */ + mov tmp1, #AT91_PMC_PLLCOUNT + orr tmp1, tmp1, #(1 << 29) /* bit 29 always set */ + str tmp1, [pmc, #AT91_CKGR_PLLAR] +2: +.endm + +.macro at91_plla_enable + ldr tmp2, .saved_pllar + + /* Restore PLLA setting */ + str tmp2, [pmc, #AT91_CKGR_PLLAR] + + /* Enable PLLA. */ + tst tmp2, #(AT91_PMC_MUL & 0xff0000) + bne 1f + tst tmp2, #(AT91_PMC_MUL & ~0xff0000) + beq 2f + +1: ldr tmp1, [pmc, #AT91_PMC_SR] + tst tmp1, #AT91_PMC_LOCKA + beq 1b +2: +.endm + +SUSPEND_FUNC(at91_ulp_mode) + ldr pmc, .pmc_base + ldr tmp3, .pm_mode + + /* Save Master clock setting */ + ldr tmp1, [pmc, #AT91_PMC_MCKR] + str tmp1, .saved_mckr + + /* + * Set master clock source to: + * - MAINCK if using ULP0 fast variant + * - slow clock, otherwise + */ + bic tmp1, tmp1, #AT91_PMC_CSS + cmp tmp3, #AT91_PM_ULP0_FAST + bne save_mck + orr tmp1, tmp1, #AT91_PMC_CSS_MAIN +save_mck: + str tmp1, [pmc, #AT91_PMC_MCKR] + + wait_mckrdy + + at91_plla_disable + + cmp tmp3, #AT91_PM_ULP1 + beq ulp1_mode + + at91_pm_ulp0_mode + b ulp_exit + +ulp1_mode: + at91_pm_ulp1_mode + b ulp_exit + +ulp_exit: + ldr pmc, .pmc_base + + at91_plla_enable + + /* + * Restore master clock setting + */ + ldr tmp2, .saved_mckr + str tmp2, [pmc, #AT91_PMC_MCKR] + + wait_mckrdy + + mov pc, lr +SUSPEND_END_FUNC(at91_ulp_mode) + +/* + * void at91_sramc_self_refresh(unsigned int is_active) + * + * @input param: + * @r0: 1 - active self-refresh mode + * 0 - exit self-refresh mode + * register usage: + * @r2: base address of the sram controller + */ + +SUSPEND_FUNC(at91_sramc_self_refresh) + ldr r2, .sramc_base + + /* + * DDR Memory controller + */ + tst r0, #SRAMC_SELF_FRESH_ACTIVE + beq ddrc_exit_sf + + /* LPDDR1 --> force DDR2 mode during self-refresh */ + ldr r3, [r2, #AT91_DDRSDRC_MDR] + str r3, .saved_sam9_mdr + bic r3, r3, #~AT91_DDRSDRC_MD + cmp r3, #AT91_DDRSDRC_MD_LOW_POWER_DDR + ldreq r3, [r2, #AT91_DDRSDRC_MDR] + biceq r3, r3, #AT91_DDRSDRC_MD + orreq r3, r3, #AT91_DDRSDRC_MD_DDR2 + streq r3, [r2, #AT91_DDRSDRC_MDR] + + /* Active DDRC self-refresh mode */ + ldr r3, [r2, #AT91_DDRSDRC_LPR] + str r3, .saved_sam9_lpr + bic r3, r3, #AT91_DDRSDRC_LPCB + orr r3, r3, #AT91_DDRSDRC_LPCB_SELF_REFRESH + str r3, [r2, #AT91_DDRSDRC_LPR] + + b exit_sramc_sf + +ddrc_exit_sf: + /* Restore MDR in case of LPDDR1 */ + ldr r3, .saved_sam9_mdr + str r3, [r2, #AT91_DDRSDRC_MDR] + /* Restore LPR on AT91 with DDRAM */ + ldr r3, .saved_sam9_lpr + str r3, [r2, #AT91_DDRSDRC_LPR] + +exit_sramc_sf: + mov pc, lr +SUSPEND_END_FUNC(at91_sramc_self_refresh) + +.pmc_base: + .word 0 +.sramc_base: + .word 0 +.shdwc: + .word 0 +.sfrbu: + .word 0 +.pm_mode: + .word 0 +.saved_mckr: + .word 0 +.saved_pllar: + .word 0 +.saved_sam9_lpr: + .word 0 +.saved_sam9_mdr: + .word 0 +.saved_osc_status: + .word 0 + +.global at91_pm_suspend_in_sram_sz +at91_pm_suspend_in_sram_sz: + .word .-at91_pm_suspend_in_sram + +check_fit_in_sram at91_pm_suspend_in_sram diff --git a/optee_os/core/drivers/pm/sam/sub.mk b/optee_os/core/drivers/pm/sam/sub.mk new file mode 100644 index 0000000..66a5228 --- /dev/null +++ b/optee_os/core/drivers/pm/sam/sub.mk @@ -0,0 +1,2 @@ +srcs-y += at91_pm.c pm_suspend.S pm_resume.S +asm-defines-y += pm-defines.c \ No newline at end of file diff --git a/optee_os/core/drivers/pm/sub.mk b/optee_os/core/drivers/pm/sub.mk new file mode 100644 index 0000000..b52e455 --- /dev/null +++ b/optee_os/core/drivers/pm/sub.mk @@ -0,0 +1,2 @@ +subdirs-$(CFG_ATMEL_PM) += sam +subdirs-$(CFG_IMX_PM) += imx diff --git a/optee_os/core/drivers/regulator/regulator.c b/optee_os/core/drivers/regulator/regulator.c new file mode 100644 index 0000000..247e231 --- /dev/null +++ b/optee_os/core/drivers/regulator/regulator.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static SLIST_HEAD(, regulator) regulator_device_list = + SLIST_HEAD_INITIALIZER(regulator); + +static void lock_regulator(struct regulator *regulator) +{ + /* + * Regulator operation may occur at runtime and during specific + * system power transition: power off, PM suspend and resume. + * These operate upon fastcall entries, under PSCI services + * execution, where non-secure world is not operational. In these + * cases we cannot take a mutex and will expect the mutex is + * unlocked. + */ + if (thread_get_id_may_fail() == THREAD_ID_INVALID) { + assert(!regulator->lock.state); + return; + } + + mutex_lock(®ulator->lock); +} + +static void unlock_regulator(struct regulator *regulator) +{ + if (thread_get_id_may_fail() == THREAD_ID_INVALID) { + /* Path for PM sequences when with local Monitor */ + return; + } + + mutex_unlock(®ulator->lock); +} + +static TEE_Result set_state(struct regulator *regulator, bool on_not_off) +{ + if (!regulator->ops->set_state) + return TEE_SUCCESS; + + return regulator->ops->set_state(regulator, on_not_off); +} + +static TEE_Result regulator_refcnt_enable(struct regulator *regulator) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + FMSG("%s", regulator_name(regulator)); + + if (regulator->supply) { + res = regulator_enable(regulator->supply); + if (res) + return res; + } + + lock_regulator(regulator); + + if (!regulator->refcount) { + res = set_state(regulator, true); + if (res) { + EMSG("regul %s set state failed with %#"PRIx32, + regulator_name(regulator), res); + + unlock_regulator(regulator); + + if (regulator->supply && + regulator_disable(regulator->supply)) + panic(); + + return res; + } + } + + regulator->refcount++; + if (!regulator->refcount) + panic(); + + FMSG("%s refcount: %u", regulator_name(regulator), regulator->refcount); + + unlock_regulator(regulator); + + return TEE_SUCCESS; +} + +TEE_Result regulator_enable(struct regulator *regulator) +{ + assert(regulator); + FMSG("%s", regulator_name(regulator)); + + if (regulator_is_always_on(regulator)) + return TEE_SUCCESS; + + return regulator_refcnt_enable(regulator); +} + +static TEE_Result regulator_refcnt_disable(struct regulator *regulator) +{ + FMSG("%s", regulator_name(regulator)); + + lock_regulator(regulator); + + if (regulator->refcount == 1) { + TEE_Result res = set_state(regulator, false); + + if (res) { + EMSG("regul %s set state failed with %#"PRIx32, + regulator_name(regulator), res); + unlock_regulator(regulator); + return res; + } + } + + if (!regulator->refcount) { + EMSG("Unbalanced %s", regulator_name(regulator)); + panic(); + } + + regulator->refcount--; + + FMSG("%s refcount: %u", regulator_name(regulator), regulator->refcount); + + unlock_regulator(regulator); + + if (regulator->supply && regulator_disable(regulator->supply)) { + /* We can't leave this unbalanced */ + EMSG("Can't disable %s", regulator_name(regulator->supply)); + panic(); + } + + return TEE_SUCCESS; +} + +TEE_Result regulator_disable(struct regulator *regulator) +{ + assert(regulator); + FMSG("%s", regulator_name(regulator)); + + if (regulator_is_always_on(regulator)) + return TEE_SUCCESS; + + return regulator_refcnt_disable(regulator); +} + +bool regulator_is_enabled(struct regulator *regulator) +{ + TEE_Result res = TEE_SUCCESS; + bool enabled = false; + + if (!regulator->ops->get_state) + return true; + + lock_regulator(regulator); + res = regulator->ops->get_state(regulator, &enabled); + unlock_regulator(regulator); + + if (res) + EMSG("regul %s get state failed with %#"PRIx32, + regulator_name(regulator), res); + + return !res && enabled; +} + +TEE_Result regulator_set_voltage(struct regulator *regulator, int level_uv) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + assert(regulator); + FMSG("%s %duV", regulator_name(regulator), level_uv); + + if (level_uv < regulator->min_uv || level_uv > regulator->max_uv) + return TEE_ERROR_BAD_PARAMETERS; + + if (level_uv == regulator->cur_uv) + return TEE_SUCCESS; + + if (!regulator->ops->set_voltage) + return TEE_ERROR_NOT_SUPPORTED; + + lock_regulator(regulator); + res = regulator->ops->set_voltage(regulator, level_uv); + unlock_regulator(regulator); + + if (res) { + EMSG("regul %s set volt failed with %#"PRIx32, + regulator_name(regulator), res); + return res; + } + + regulator->cur_uv = level_uv; + + return TEE_SUCCESS; +} + +TEE_Result regulator_supported_voltages(struct regulator *regulator, + struct regulator_voltages **voltages) +{ + assert(regulator && voltages); + + if (regulator->ops->supported_voltages) { + TEE_Result res = TEE_ERROR_GENERIC; + + res = regulator->ops->supported_voltages(regulator, voltages); + if (res == TEE_SUCCESS) + return TEE_SUCCESS; + if (res != TEE_ERROR_NOT_SUPPORTED) + return res; + } + + *voltages = ®ulator->voltages_fallback.desc; + + return TEE_SUCCESS; +} + +TEE_Result regulator_register(struct regulator *regulator) +{ + TEE_Result res = TEE_SUCCESS; + int min_uv = 0; + int max_uv = 0; + int uv = 0; + + if (!regulator || !regulator->ops || + regulator->flags & ~REGULATOR_FLAGS_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + regulator_get_range(regulator, &min_uv, &max_uv); + if (min_uv > max_uv) + return TEE_ERROR_BAD_PARAMETERS; + + /* Sanitize regulator effective level */ + if (regulator->ops->get_voltage) { + res = regulator->ops->get_voltage(regulator, &uv); + if (res) + return res; + } else { + uv = min_uv; + } + regulator->cur_uv = uv; + + if (uv < min_uv || uv > max_uv) { + res = regulator_set_voltage(regulator, min_uv); + if (res) + return res; + } + + /* Unbalanced enable refcount to keep always-on regulators enabled */ + if (regulator_is_always_on(regulator)) { + res = regulator_refcnt_enable(regulator); + if (res) + return res; + } + + /* Preset voltage list in case ops::supported_voltages is NULL */ + if (regulator->min_uv == regulator->max_uv) { + regulator->voltages_fallback.desc.type = VOLTAGE_TYPE_FULL_LIST; + regulator->voltages_fallback.desc.num_levels = 1; + regulator->voltages_fallback.levels[0] = regulator->min_uv; + } else { + regulator->voltages_fallback.desc.type = VOLTAGE_TYPE_INCREMENT; + regulator->voltages_fallback.levels[0] = regulator->min_uv; + regulator->voltages_fallback.levels[1] = regulator->max_uv; + regulator->voltages_fallback.levels[2] = 1; + } + + SLIST_INSERT_HEAD(®ulator_device_list, regulator, link); + + return TEE_SUCCESS; +} + +/* + * Log regulators state + */ +void regulator_print_state(const char *message __maybe_unused) +{ + struct regulator *regulator = NULL; + + DMSG("Regulator state: %s", message); + DMSG("name use\ten\tuV\tmin\tmax\tflags\tsupply"); + + SLIST_FOREACH(regulator, ®ulator_device_list, link) + DMSG("%8s %u\t%d\t%d\t%d\t%d\t%#x\t%s\n", + regulator->name, regulator->refcount, + regulator_is_enabled(regulator), + regulator_get_voltage(regulator), + regulator->min_uv, regulator->max_uv, regulator->flags, + regulator->supply ? regulator_name(regulator->supply) : + ""); +} + +/* + * Clean-up regulators that are not used. + */ +static TEE_Result regulator_core_cleanup(void) +{ + struct regulator *regulator = NULL; + + SLIST_FOREACH(regulator, ®ulator_device_list, link) { + if (!regulator->refcount) { + DMSG("disable %s", regulator_name(regulator)); + lock_regulator(regulator); + set_state(regulator, false /* disable */); + unlock_regulator(regulator); + } + } + + regulator_print_state(__func__); + + return TEE_SUCCESS; +} + +release_init_resource(regulator_core_cleanup); diff --git a/optee_os/core/drivers/regulator/regulator_dt.c b/optee_os/core/drivers/regulator/regulator_dt.c new file mode 100644 index 0000000..4265419 --- /dev/null +++ b/optee_os/core/drivers/regulator/regulator_dt.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022-2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * struct regulator_property - DT binding boolean property names + * @name: Property name in the regulator DT node + * @flag: Mask of the related REGULATOR_* boolean property + */ +struct regulator_property { + const char *name; + unsigned int flag; +}; + +static struct regulator_property flag_prop[] = { + { + .name = "regulator-always-on", + .flag = REGULATOR_ALWAYS_ON, + }, + { + .name = "regulator-pull-down", + .flag = REGULATOR_PULL_DOWN, + }, + { + .name = "regulator-boot-on", + .flag = REGULATOR_BOOT_ON, + }, +}; + +/* + * struct pending_regu - Regulators waiting for their supply to be ready + * + * @fdt: DT to work on + * @node: Node of the regulator in @fdt + * @supply_phandle: Phandle in @fdt of the regulator supply, or 0 if no supply + * @regulator_allocated: True if framework allocates and frees @regulator + * @regulator: Regulator device instance + * @link: Link in pending regulators list + * + * When calling regulator_dt_register(), either the regulator depends on a + * supply that is not initialized, or this dependency is resolved (there is + * no supply or the supply is ready to use). + * + * In the former case, the regulator is placed in a pending regulator list. + * Each time a new regulator is successfully registered, we process the + * pending regulator list in case some pending regulators find their + * supply and finalize their registration and initialization. + * + * In the latter case, the regulator registration and initialization + * are processed. + */ +struct pending_regu { + const void *fdt; + int node; + int supply_phandle; + bool regulator_allocated; + struct regulator *regulator; + SLIST_ENTRY(pending_regu) link; +}; + +static SLIST_HEAD(, pending_regu) pending_regu_list = + SLIST_HEAD_INITIALIZER(pending_regu); + +/* Helper to find the phandle of a regulator supply */ +static TEE_Result get_supply_phandle(const void *fdt, int node, + const char *supply_name, + uint32_t *supply_phandle) +{ + char *supply_prop = NULL; + size_t prop_len = 0; + const fdt32_t *cuint = NULL; + int len = 0; + + prop_len = strlen(supply_name) + strlen("-supply") + 1; + supply_prop = calloc(1, prop_len); + if (!supply_prop) + return TEE_ERROR_OUT_OF_MEMORY; + + len = snprintf(supply_prop, prop_len, "%s-supply", supply_name); + assert(len > 0 && (size_t)len == prop_len - 1); + + cuint = fdt_getprop(fdt, node, supply_prop, &len); + free(supply_prop); + if (!cuint || (size_t)len != sizeof(*cuint)) { + if (len != -FDT_ERR_NOTFOUND) + return TEE_ERROR_GENERIC; + + *supply_phandle = 0; + + return TEE_SUCCESS; + } + + *supply_phandle = fdt32_to_cpu(*cuint); + assert(*supply_phandle); + + return TEE_SUCCESS; +} + +TEE_Result regulator_dt_get_supply(const void *fdt, int node, + const char *supply_name, + struct regulator **regulator) +{ + struct dt_driver_provider *provider = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t supply_phandle = 0; + + res = get_supply_phandle(fdt, node, supply_name, &supply_phandle); + if (res) + return res; + + provider = dt_driver_get_provider_by_phandle(supply_phandle, + DT_DRIVER_REGULATOR); + if (!provider) + return TEE_ERROR_DEFER_DRIVER_INIT; + + *regulator = dt_driver_provider_priv_data(provider); + assert(*regulator); + + return TEE_SUCCESS; +} + +/* Helper function to register a regulator provider instance */ +static TEE_Result regulator_register_provider(const void *fdt, int node, + struct regulator *regulator) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t phandle = 0; + + phandle = fdt_get_phandle(fdt, node); + switch (phandle) { + case 0: + /* We can ignore regulators without any phandle */ + return TEE_SUCCESS; + case (uint32_t)-1: + DMSG("Failed to find provider phandle"); + return TEE_ERROR_GENERIC; + default: + res = dt_driver_register_provider(fdt, node, NULL, regulator, + DT_DRIVER_REGULATOR); + if (res) + EMSG("Can't register regulator provider %s: %#"PRIx32, + regulator_name(regulator), res); + + return res; + } +} + +static TEE_Result register_final(const void *fdt, int node, + struct regulator *regulator) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + FMSG("Regulator: finalize %s registering", regulator_name(regulator)); + + res = regulator_register(regulator); + if (res) + return res; + + if (regulator->ops->supplied_init) { + res = regulator->ops->supplied_init(regulator, fdt, node); + if (res) + return res; + } + + return regulator_register_provider(fdt, node, regulator); +} + +/* + * Pending regulators list: stores all regulator devices registered by their + * driver but not yet available to consumers as their dependency on their + * regulator supply is not yet resolved (supply has not been initialized yet). + */ + +static void __maybe_unused print_pending_regulators(void) +{ + struct pending_regu *pending = NULL; + + SLIST_FOREACH(pending, &pending_regu_list, link) + DMSG("Pending regulator %s", + regulator_name(pending->regulator)); +} + +/* + * Returns true if at least 1 regulator found its supply and finalized its + * registration. + */ +static bool process_pending_list(void) +{ + struct dt_driver_provider *p = NULL; + struct pending_regu *pending = NULL; + struct pending_regu *next = NULL; + bool supplied = false; + + SLIST_FOREACH_SAFE(pending, &pending_regu_list, link, next) { + p = dt_driver_get_provider_by_phandle(pending->supply_phandle, + DT_DRIVER_REGULATOR); + if (!p) + continue; + + pending->regulator->supply = dt_driver_provider_priv_data(p); + + if (register_final(pending->fdt, pending->node, + pending->regulator)) + panic(); + + SLIST_REMOVE(&pending_regu_list, pending, pending_regu, link); + free(pending); + + supplied = true; + } + + return supplied; +} + +/* + * Attempt to register pending regulators once their supply is found. + * Return true if pending regulator list is empty upon processing. + */ +static bool resolve_pending_list(void) +{ + while (process_pending_list()) + ; + + return SLIST_EMPTY(&pending_regu_list); +} + +/* Adds a regulator to the pending list: those waiting for their supply */ +static TEE_Result add_to_pending_list(const void *fdt, int node, + struct regulator *regulator, + uint32_t supply_phandle, + bool regulator_allocated) +{ + struct pending_regu *pending = NULL; + + pending = calloc(1, sizeof(*pending)); + if (!pending) + return TEE_ERROR_OUT_OF_MEMORY; + + *pending = (struct pending_regu){ + .fdt = fdt, + .node = node, + .supply_phandle = supply_phandle, + .regulator = regulator, + .regulator_allocated = regulator_allocated, + }; + + SLIST_INSERT_HEAD(&pending_regu_list, pending, link); + + return TEE_SUCCESS; +} + +static TEE_Result parse_dt(const void *fdt, int node, + struct regulator *regulator) +{ + struct regulator_property *fp = NULL; + const fdt32_t *cuint = NULL; + int len = 0; + + FMSG("Regulator: parse DT node %s", fdt_get_name(fdt, node, NULL)); + + cuint = fdt_getprop(fdt, node, "regulator-name", NULL); + if (cuint) { + /* Replace name with the one found from the DT node */ + char *name = (char *)cuint; + + free(regulator->name); + regulator->name = strdup(name); + if (!regulator->name) + return TEE_ERROR_OUT_OF_MEMORY; + } + + for (fp = flag_prop; fp < (flag_prop + ARRAY_SIZE(flag_prop)); fp++) + if (fdt_getprop(fdt, node, fp->name, NULL)) + regulator->flags |= fp->flag; + + cuint = fdt_getprop(fdt, node, "regulator-min-microvolt", &len); + if (cuint && len == sizeof(*cuint)) + regulator->min_uv = fdt32_to_cpu(*cuint); + else if (cuint || len != -FDT_ERR_NOTFOUND) + panic(); + + cuint = fdt_getprop(fdt, node, "regulator-max-microvolt", &len); + if (cuint && len == sizeof(*cuint)) { + regulator->max_uv = fdt32_to_cpu(*cuint); + + if (regulator->max_uv < regulator->min_uv) { + EMSG("Regulator %s max_uv %d < %d", + regulator_name(regulator), regulator->max_uv, + regulator->min_uv); + + return TEE_ERROR_GENERIC; + } + } else if (cuint || len != -FDT_ERR_NOTFOUND) { + panic(); + } else { + regulator->max_uv = INT_MAX; + } + + return TEE_SUCCESS; +} + +/* + * API function to register a DRIVER_REGULATOR provider instance. + * The registration can be deferred if the regulator supply (if any) + * is not yet registered, in which case the regulator is placed in + * a regulator pending list. + */ +TEE_Result regulator_dt_register(const void *fdt, int node, int provider_node, + const struct regu_dt_desc *desc) +{ + TEE_Result res = TEE_ERROR_OUT_OF_MEMORY; + struct regulator *regulator = NULL; + uint32_t supply_phandle = 0; + char *name = NULL; + + assert(desc); + + if (!desc->regulator) { + regulator = calloc(1, sizeof(*regulator)); + if (!regulator) + return TEE_ERROR_OUT_OF_MEMORY; + } else { + regulator = desc->regulator; + memset(regulator, 0, sizeof(*regulator)); + } + + if (desc->name) { + /* Will be freed if overridden by DT node content */ + name = strdup(desc->name); + if (!name) + goto err_free; + } + + *regulator = (struct regulator){ + .name = name, + .ops = desc->ops, + .priv = desc->priv, + }; + + res = parse_dt(fdt, node, regulator); + if (res) + goto err_free; + + if (desc->supply_name) { + res = get_supply_phandle(fdt, provider_node, desc->supply_name, + &supply_phandle); + if (res) + goto err_free; + } + + if (supply_phandle) { + res = add_to_pending_list(fdt, node, regulator, supply_phandle, + !desc->regulator); + if (res) + goto err_free; + } else { + res = register_final(fdt, node, regulator); + if (res) + goto err_free; + } + + resolve_pending_list(); + + return TEE_SUCCESS; + +err_free: + /* This function cannot return TEE_ERROR_DEFER_DRIVER_INIT */ + assert(res != TEE_ERROR_DEFER_DRIVER_INIT); + + free(regulator->name); + if (!desc->regulator) + free(regulator); + + return res; +} + +static TEE_Result release_regulator_pending_lists(void) +{ + struct pending_regu *pending = NULL; + struct pending_regu *next = NULL; + + if (!SLIST_EMPTY(&pending_regu_list)) + DMSG("Some regulators were not supplied:"); + + SLIST_FOREACH_SAFE(pending, &pending_regu_list, link, next) { + DMSG(" Unsupplied regulator %s", + regulator_name(pending->regulator)); + + SLIST_REMOVE(&pending_regu_list, pending, pending_regu, link); + if (pending->regulator_allocated) + free(pending->regulator); + free(pending); + } + + return TEE_SUCCESS; +} + +release_init_resource(release_regulator_pending_lists); diff --git a/optee_os/core/drivers/regulator/regulator_fixed.c b/optee_os/core/drivers/regulator/regulator_fixed.c new file mode 100644 index 0000000..0037cad --- /dev/null +++ b/optee_os/core/drivers/regulator/regulator_fixed.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022-2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static_assert(GPIO_LEVEL_HIGH == 1 && GPIO_LEVEL_LOW == 0); + +/* + * struct regulator_gpio - GPIO controlled regulator + * @regulator: Preallocated regulator instance + * @enable_gpio: GPIO for the enable state of the regulator or NULL if always on + * @enable_delay: Time (in microsecond) for the regulator to get enabled + * @off_on_delay: Min time (in microsecond) between enable and disable request + * @off_on_us: Timestamp of the last disable request + */ +struct regulator_fixed { + struct regulator regulator; + struct gpio *enable_gpio; + unsigned int enable_delay; + unsigned int off_on_delay; + uint64_t off_on_us; +}; + +static struct regulator_fixed *regulator_priv(struct regulator *regulator) +{ + return container_of(regulator, struct regulator_fixed, regulator); +} + +static TEE_Result fixed_set_state(struct regulator *regulator, bool enabled) +{ + struct regulator_fixed *regu = regulator_priv(regulator); + + if (regu->enable_gpio) { + if (enabled) { + while (!timeout_elapsed(regu->off_on_us)) + udelay(1); + gpio_set_value(regu->enable_gpio, GPIO_LEVEL_HIGH); + udelay(regu->enable_delay); + } else { + regu->off_on_us = timeout_init_us(regu->off_on_delay); + gpio_set_value(regu->enable_gpio, GPIO_LEVEL_LOW); + } + } + + return TEE_SUCCESS; +} + +static TEE_Result fixed_get_state(struct regulator *regulator, bool *enabled) +{ + struct regulator_fixed *regu = regulator_priv(regulator); + + if (regu->enable_gpio) + *enabled = gpio_get_value(regu->enable_gpio); + else + *enabled = true; + + return TEE_SUCCESS; +} + +static const struct regulator_ops fixed_regulator_ops = { + .set_state = fixed_set_state, + .get_state = fixed_get_state, +}; + +static TEE_Result get_enable_gpio(const void *fdt, int node, + struct regulator_fixed *regu) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const fdt32_t *cuint = NULL; + struct gpio *gpio = NULL; + void *gpio_ref = &gpio; + + res = dt_driver_device_from_node_idx_prop("gpios", fdt, node, 0, + DT_DRIVER_GPIO, gpio_ref); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = dt_driver_device_from_node_idx_prop("gpio", fdt, node, 0, + DT_DRIVER_GPIO, + gpio_ref); + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + regu->enable_gpio = NULL; + + return TEE_SUCCESS; + } + if (res) + return res; + + /* Override active level phandle flag, as per DT bindings */ + if (dt_have_prop(fdt, node, "enable-active-high")) + gpio->dt_flags &= ~GPIO_ACTIVE_LOW; + else + gpio->dt_flags |= GPIO_ACTIVE_LOW; + + /* Override open drain/open source phandle flag, as per DT bindings */ + if (dt_have_prop(fdt, node, "gpio-open-drain")) + gpio->dt_flags |= GPIO_LINE_OPEN_DRAIN; + else + gpio->dt_flags &= ~GPIO_LINE_OPEN_DRAIN; + + cuint = fdt_getprop(fdt, node, "startup-delay-us", NULL); + if (cuint) + regu->enable_delay = fdt32_to_cpu(*cuint); + + cuint = fdt_getprop(fdt, node, "off-on-delay-us", NULL); + if (cuint) + regu->off_on_delay = fdt32_to_cpu(*cuint); + + gpio_set_direction(gpio, GPIO_DIR_OUT); + + regu->enable_gpio = gpio; + + return TEE_SUCCESS; +} + +static TEE_Result fixed_regulator_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + struct regulator_fixed *regu = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + struct regu_dt_desc desc = { }; + const char *supply_name = NULL; + const char *type = NULL; + char *regu_name = NULL; + + regu_name = (char *)fdt_get_name(fdt, node, NULL); + + type = fdt_getprop(fdt, node, "regulator-type", NULL); + if (type && strcmp(type, "voltage")) { + EMSG("Regulator gpio node %s: type %s not supported", + regu_name, type); + return TEE_ERROR_GENERIC; + } + + regu = calloc(1, sizeof(*regu)); + if (!regu) + return TEE_ERROR_OUT_OF_MEMORY; + + res = get_enable_gpio(fdt, node, regu); + if (res) + goto err; + + if (fdt_getprop(fdt, node, "vin-supply", NULL)) + supply_name = "vin"; + + desc = (struct regu_dt_desc){ + .name = regu_name, + .ops = &fixed_regulator_ops, + .supply_name = supply_name, + .regulator = ®u->regulator, + }; + + res = regulator_dt_register(fdt, node, node, &desc); + if (res) { + EMSG("Can't register regulator %s: %#"PRIx32, regu_name, res); + goto err; + } + + return TEE_SUCCESS; + +err: + free(regu); + + return res; +} + +static const struct dt_device_match regulator_match_table[] = { + { .compatible = "regulator-fixed" }, + { } +}; + +DEFINE_DT_DRIVER(fixed_regulator_dt_driver) = { + .name = "regulator-fixed", + .match_table = regulator_match_table, + .probe = fixed_regulator_probe, +}; diff --git a/optee_os/core/drivers/regulator/stm32_vrefbuf.c b/optee_os/core/drivers/regulator/stm32_vrefbuf.c new file mode 100644 index 0000000..fc2b638 --- /dev/null +++ b/optee_os/core/drivers/regulator/stm32_vrefbuf.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* STM32 VREFBUF registers */ +#define VREFBUF_CSR U(0) + +/* STM32 VREFBUF CSR bitfields */ + +/* VRS bit 3 is unused because the voltage is not specified */ +#define VREFBUF_CSR_VRS GENMASK_32(5, 4) +#define VREFBUF_CSR_VRS_SHIFT U(4) +#define INV_VRS(x) ((~(x)) & VREFBUF_CSR_VRS) + +#define VREFBUF_CSR_VRR BIT(3) +#define VREFBUF_CSR_HIZ BIT(1) +#define VREFBUF_CSR_ENVR BIT(0) + +#define TIMEOUT_US_10MS U(10 * 1000) +#define TIMEOUT_US_1MS U(1 * 1000) + +#define VREFBUF_LEVELS_COUNT U(4) + +/* + * struct vrefbuf_compat - Compatibility data + * @voltages: Voltage levels supported + */ +struct vrefbuf_compat { + int voltages[VREFBUF_LEVELS_COUNT]; +}; + +/* + * struct vrefbuf_regul - VREFBUF regulator + * @base: IO memory base address + * @clock: VREFBUF access bus clock + * @regulator: Preallocated instance for the regulator + * @compat: Compatibility data + * @voltages_desc: Supported voltage level description + * @voltages_level: Supplorted levels, must follow @voltages_desc + */ +struct vrefbuf_regul { + vaddr_t base; + struct clk *clock; + uint64_t disable_timeout; + struct regulator regulator; + const struct vrefbuf_compat *compat; + struct regulator_voltages voltages_desc; + int supported_levels[VREFBUF_LEVELS_COUNT]; +}; + +static const struct vrefbuf_compat stm32mp15_vrefbuf_compat = { + .voltages = { + /* Matches resp. VRS = 011b, 010b, 001b, 000b */ + 1500000, 1800000, 2048000, 2500000, + }, +}; + +static const struct vrefbuf_compat stm32mp13_vrefbuf_compat = { + .voltages = { + /* Matches resp. VRS = 011b, 010b, 001b, 000b */ + 1650000, 1800000, 2048000, 2500000, + }, +}; + +/* Expect at most 1 instance */ +static struct vrefbuf_regul *stm32_vrefbuf; + +struct regulator *stm32_vrefbuf_regulator(void) +{ + if (!stm32_vrefbuf) + return NULL; + + return &stm32_vrefbuf->regulator; +} + +static struct vrefbuf_regul *regulator_to_vr(struct regulator *regulator) +{ + return container_of(regulator, struct vrefbuf_regul, regulator); +} + +static TEE_Result vrefbuf_wait_ready(struct vrefbuf_regul *vr) +{ + uint32_t val = 0; + + if (IO_READ32_POLL_TIMEOUT(vr->base + VREFBUF_CSR, val, + val & VREFBUF_CSR_VRR, 0, TIMEOUT_US_10MS)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result vrefbuf_set_state(struct regulator *regulator, bool enable) +{ + struct vrefbuf_regul *vr = regulator_to_vr(regulator); + TEE_Result res = TEE_ERROR_GENERIC; + + res = clk_enable(vr->clock); + if (res) + return res; + + if (enable) { + io_clrbits32(vr->base + VREFBUF_CSR, VREFBUF_CSR_HIZ); + + /* + * If first enable after boot or if it was disabled since + * less than 1ms, then wait for 1ms in pull down mode to + * avoid an overshoot. + */ + if (!vr->disable_timeout || + !timeout_elapsed(vr->disable_timeout)) + udelay(1000); + + io_setbits32(vr->base + VREFBUF_CSR, VREFBUF_CSR_ENVR); + + if (vrefbuf_wait_ready(vr) != TEE_SUCCESS) { + clk_disable(vr->clock); + + return TEE_ERROR_GENERIC; + } + } else { + io_clrbits32(vr->base + VREFBUF_CSR, VREFBUF_CSR_ENVR); + + vr->disable_timeout = timeout_init_us(TIMEOUT_US_1MS); + } + + clk_disable(vr->clock); + + return TEE_SUCCESS; +} + +static TEE_Result vrefbuf_get_state(struct regulator *regulator, bool *enabled) +{ + struct vrefbuf_regul *vr = regulator_to_vr(regulator); + TEE_Result res = TEE_ERROR_GENERIC; + + res = clk_enable(vr->clock); + if (res) + return res; + + *enabled = io_read32(vr->base + VREFBUF_CSR) & VREFBUF_CSR_VRR; + + clk_disable(vr->clock); + + return TEE_SUCCESS; +} + +static TEE_Result vrefbuf_get_voltage(struct regulator *regulator, + int *level_uv) +{ + struct vrefbuf_regul *vr = regulator_to_vr(regulator); + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t index = 0; + + res = clk_enable(vr->clock); + if (res) + return res; + + index = io_read32(vr->base + VREFBUF_CSR) & VREFBUF_CSR_VRS; + index = INV_VRS(index) >> VREFBUF_CSR_VRS_SHIFT; + + clk_disable(vr->clock); + + *level_uv = vr->compat->voltages[index]; + + return TEE_SUCCESS; +} + +static TEE_Result vrefbuf_set_voltage(struct regulator *regulator, int level_uv) +{ + struct vrefbuf_regul *vr = regulator_to_vr(regulator); + TEE_Result res = TEE_ERROR_GENERIC; + size_t i = 0; + + for (i = 0 ; i < ARRAY_SIZE(vr->compat->voltages) ; i++) { + if (vr->compat->voltages[i] == level_uv) { + uint32_t val = INV_VRS(i << VREFBUF_CSR_VRS_SHIFT); + + res = clk_enable(vr->clock); + if (res) + return res; + + io_clrsetbits32(vr->base + VREFBUF_CSR, VREFBUF_CSR_VRS, + val); + + clk_disable(vr->clock); + + return TEE_SUCCESS; + } + } + + EMSG("Failed to set voltage on vrefbuf"); + + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result vrefbuf_list_voltages(struct regulator *regulator __unused, + struct regulator_voltages **voltages) +{ + struct vrefbuf_regul *vr = regulator_to_vr(regulator); + + if (!vr->voltages_desc.type) { + size_t num_levels = ARRAY_SIZE(vr->compat->voltages); + unsigned int index_high = num_levels - 1; + unsigned int index_low = 0; + unsigned int count = 0; + unsigned int n = 0; + + for (n = 0; n <= index_high; n++) + if (vr->compat->voltages[n] >= regulator->min_uv) + break; + if (n > index_high) + return TEE_ERROR_GENERIC; + index_low = n; + + for (n = index_high; n >= index_low; n--) + if (vr->compat->voltages[n] <= regulator->max_uv) + break; + if (n < index_low) + return TEE_ERROR_GENERIC; + index_high = n; + + count = index_high - index_low + 1; + + vr->voltages_desc.type = VOLTAGE_TYPE_FULL_LIST; + vr->voltages_desc.num_levels = count; + for (n = 0; n < count; n++) + vr->supported_levels[n] = + vr->compat->voltages[index_low + n]; + } + + *voltages = &vr->voltages_desc; + + return TEE_SUCCESS; +} + +static TEE_Result stm32_vrefbuf_pm(enum pm_op op, unsigned int pm_hint __unused, + const struct pm_callback_handle *hdl) +{ + struct regulator *regulator = hdl->handle; + struct vrefbuf_regul *vr = regulator_to_vr(regulator); + vaddr_t csr_va = vr->base + VREFBUF_CSR; + TEE_Result res = TEE_ERROR_GENERIC; + /* Context to save/restore on PM suspend/resume */ + static uint32_t pm_val; + + assert(op == PM_OP_SUSPEND || op == PM_OP_RESUME); + + res = clk_enable(vr->clock); + if (res) + return res; + + if (op == PM_OP_SUSPEND) { + pm_val = io_read32(csr_va); + + if (pm_val & VREFBUF_CSR_ENVR && vrefbuf_wait_ready(vr)) { + clk_disable(vr->clock); + + return TEE_ERROR_GENERIC; + } + } else { + io_clrsetbits32(csr_va, VREFBUF_CSR_VRS, pm_val); + + if (pm_val & VREFBUF_CSR_ENVR) { + vr->disable_timeout = 0; + vrefbuf_set_state(&vr->regulator, true); + } + } + + clk_disable(vr->clock); + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(stm32_vrefbuf_pm); + +static TEE_Result stm32_vrefbuf_init(struct regulator *regulator, + const void *fdt __unused, + int node __unused) +{ + register_pm_driver_cb(stm32_vrefbuf_pm, regulator, "stm32-vrefbuf"); + + return TEE_SUCCESS; +} + +static const struct regulator_ops vrefbuf_ops = { + .set_state = vrefbuf_set_state, + .get_state = vrefbuf_get_state, + .set_voltage = vrefbuf_set_voltage, + .get_voltage = vrefbuf_get_voltage, + .supported_voltages = vrefbuf_list_voltages, + .supplied_init = stm32_vrefbuf_init, +}; + +static TEE_Result stm32_vrefbuf_regulator_probe(const void *fdt, int node, + const void *compat_data) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct vrefbuf_regul *vr = NULL; + struct regu_dt_desc desc = { }; + char *regu_name = NULL; + struct clk *clk = NULL; + paddr_t reg_base = 0; + size_t reg_size = 0; + + assert(!stm32_vrefbuf); + + res = clk_dt_get_by_index(fdt, node, 0, &clk); + if (res) + return res; + + vr = calloc(1, sizeof(*vr)); + if (!vr) + panic(); + + vr->compat = compat_data; + + regu_name = strdup(fdt_get_name(fdt, node, NULL)); + if (!regu_name) + panic(); + + reg_base = fdt_reg_base_address(fdt, node); + reg_size = fdt_reg_size(fdt, node); + if (reg_base == DT_INFO_INVALID_REG || + reg_size == DT_INFO_INVALID_REG_SIZE) + panic(); + + vr->base = (vaddr_t)phys_to_virt(reg_base, MEM_AREA_IO_SEC, reg_size); + if (!vr->base) + panic(); + + vr->clock = clk; + + desc = (struct regu_dt_desc){ + .name = regu_name, + .ops = &vrefbuf_ops, + .supply_name = "vdda", + .regulator = &vr->regulator, + }; + + res = regulator_dt_register(fdt, node, node, &desc); + if (res) + panic(); + + stm32_vrefbuf = vr; + + return TEE_SUCCESS; +} + +static const struct dt_device_match stm32_vrefbuf_match_table[] = { + { + .compatible = "st,stm32-vrefbuf", + .compat_data = &stm32mp15_vrefbuf_compat, + }, + { + .compatible = "st,stm32mp13-vrefbuf", + .compat_data = &stm32mp13_vrefbuf_compat + }, + { } +}; + +DEFINE_DT_DRIVER(stm32_vrefbuf_regulator_dt_driver) = { + .name = "stm32-vrefbuf-regulator", + .match_table = stm32_vrefbuf_match_table, + .probe = &stm32_vrefbuf_regulator_probe, +}; diff --git a/optee_os/core/drivers/regulator/sub.mk b/optee_os/core/drivers/regulator/sub.mk new file mode 100644 index 0000000..1562241 --- /dev/null +++ b/optee_os/core/drivers/regulator/sub.mk @@ -0,0 +1,4 @@ +srcs-y += regulator.c +srcs-$(CFG_DT) += regulator_dt.c +srcs-$(CFG_REGULATOR_FIXED) += regulator_fixed.c +srcs-$(CFG_STM32_VREFBUF) += stm32_vrefbuf.c diff --git a/optee_os/core/drivers/rstctrl/rstctrl.c b/optee_os/core/drivers/rstctrl/rstctrl.c new file mode 100644 index 0000000..fa453d4 --- /dev/null +++ b/optee_os/core/drivers/rstctrl/rstctrl.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +/* Global reset controller access lock */ + +TEE_Result rstctrl_get_exclusive(struct rstctrl *rstctrl) +{ + uint32_t exceptions = 0; + TEE_Result res = TEE_ERROR_ACCESS_CONFLICT; + static unsigned int rstctrl_lock = SPINLOCK_UNLOCK; + + exceptions = cpu_spin_lock_xsave(&rstctrl_lock); + + if (!rstctrl->exclusive) { + rstctrl->exclusive = true; + res = TEE_SUCCESS; + } + + cpu_spin_unlock_xrestore(&rstctrl_lock, exceptions); + + return res; +} + +void rstctrl_put_exclusive(struct rstctrl *rstctrl) +{ + assert(rstctrl->exclusive); + + WRITE_ONCE(rstctrl->exclusive, false); +} + +TEE_Result rstctrl_dt_get_by_name(const void *fdt, int nodeoffset, + const char *name, struct rstctrl **rstctrl) +{ + int index = 0; + + index = fdt_stringlist_search(fdt, nodeoffset, "reset-names", name); + if (index < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + return rstctrl_dt_get_by_index(fdt, nodeoffset, index, rstctrl); +} diff --git a/optee_os/core/drivers/rstctrl/stm32_rstctrl.c b/optee_os/core/drivers/rstctrl/stm32_rstctrl.c new file mode 100644 index 0000000..50ff4ec --- /dev/null +++ b/optee_os/core/drivers/rstctrl/stm32_rstctrl.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2022, Linaro Limited + * Copyright (c) 2018-2022, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RESET_ID_MASK GENMASK_32(31, 5) +#define RESET_ID_SHIFT 5 +#define RESET_BIT_POS_MASK GENMASK_32(4, 0) +#define RESET_OFFSET_MAX 1024 + +/* Exposed rstctrl instance */ +struct stm32_rstline { + unsigned int id; + struct rstctrl rstctrl; + SLIST_ENTRY(stm32_rstline) link; +}; + +static SLIST_HEAD(, stm32_rstline) stm32_rst_list = + SLIST_HEAD_INITIALIZER(stm32_rst_list); + +static size_t reset_id2reg_offset(unsigned int id) +{ + size_t offset = (id & RESET_ID_MASK) >> RESET_ID_SHIFT; + + assert(offset < RESET_OFFSET_MAX); + return offset * sizeof(uint32_t); +} + +static uint8_t reset_id2reg_bit_pos(unsigned int reset_id) +{ + uint8_t pos = reset_id & RESET_BIT_POS_MASK; + + assert(pos < 32); + return pos; +} + +static struct stm32_rstline *to_rstline(struct rstctrl *rstctrl) +{ + assert(rstctrl); + + return container_of(rstctrl, struct stm32_rstline, rstctrl); +} + +static TEE_Result reset_assert(struct rstctrl *rstctrl, unsigned int to_us) +{ + unsigned int id = to_rstline(rstctrl)->id; + vaddr_t rcc_base = stm32_rcc_base(); + uint32_t bit_mask = 0; + size_t offset = 0; + +#ifdef CFG_STM32MP15 + switch (id) { + case MCU_HOLD_BOOT_R: + /* + * The RCC_MP_GCR is a read/write register. + * Assert the MCU HOLD_BOOT means clear the BOOT_MCU bit + */ + io_clrbits32(rcc_base + RCC_MP_GCR, RCC_MP_GCR_BOOT_MCU); + + return TEE_SUCCESS; + case MCU_R: + /* MCU reset can only be written */ + to_us = 0; + break; + default: + break; + } +#endif + + offset = reset_id2reg_offset(id); + bit_mask = BIT(reset_id2reg_bit_pos(id)); + + io_write32(rcc_base + offset, bit_mask); + + if (to_us) { + uint64_t timeout_ref = timeout_init_us(to_us); + + while (!(io_read32(rcc_base + offset) & bit_mask)) + if (timeout_elapsed(timeout_ref)) + break; + + if (!(io_read32(rcc_base + offset) & bit_mask)) + return TEE_ERROR_SECURITY; + } + + return TEE_SUCCESS; +} + +static TEE_Result reset_deassert(struct rstctrl *rstctrl, unsigned int to_us) +{ + unsigned int id = to_rstline(rstctrl)->id; + vaddr_t rcc_base = stm32_rcc_base(); + uint32_t bit_mask = 0; + size_t offset = 0; + +#ifdef CFG_STM32MP15 + switch (id) { + case MCU_HOLD_BOOT_R: + /* + * The RCC_MP_GCR is a read/write register. + * Deassert the MCU HOLD_BOOT means set the BOOT_MCU the bit + */ + io_setbits32(rcc_base + RCC_MP_GCR, RCC_MP_GCR_BOOT_MCU); + + return TEE_SUCCESS; + case MCU_R: + /* MCU reset deasserts by its own */ + return TEE_SUCCESS; + default: + break; + } +#endif + + offset = reset_id2reg_offset(id) + RCC_MP_RSTCLRR_OFFSET; + bit_mask = BIT(reset_id2reg_bit_pos(id)); + + io_write32(rcc_base + offset, bit_mask); + + if (to_us) { + uint64_t timeout_ref = timeout_init_us(to_us); + + while ((io_read32(rcc_base + offset) & bit_mask)) + if (timeout_elapsed(timeout_ref)) + break; + + if (io_read32(rcc_base + offset) & bit_mask) + return TEE_ERROR_SECURITY; + } + + return TEE_SUCCESS; +} + +static struct rstctrl_ops stm32_rstctrl_ops = { + .assert_level = reset_assert, + .deassert_level = reset_deassert, +}; + +static struct stm32_rstline *find_rstctrl_device(unsigned int control_id) +{ + struct stm32_rstline *stm32_rstline = NULL; + + SLIST_FOREACH(stm32_rstline, &stm32_rst_list, link) + if (stm32_rstline->id == control_id) + break; + + return stm32_rstline; +} + +static struct stm32_rstline *find_or_allocate_rstline(unsigned int binding_id) +{ + struct stm32_rstline *stm32_rstline = find_rstctrl_device(binding_id); + + if (stm32_rstline) + return stm32_rstline; + + stm32_rstline = calloc(1, sizeof(*stm32_rstline)); + if (stm32_rstline) { + stm32_rstline->rstctrl.ops = &stm32_rstctrl_ops; + stm32_rstline->id = binding_id; + SLIST_INSERT_HEAD(&stm32_rst_list, stm32_rstline, link); + } + + return stm32_rstline; +} + +struct rstctrl *stm32mp_rcc_reset_id_to_rstctrl(unsigned int binding_id) +{ + struct stm32_rstline *rstline = find_or_allocate_rstline(binding_id); + + assert(rstline); + return &rstline->rstctrl; +} + +static TEE_Result stm32_rstctrl_get_dev(struct dt_pargs *arg, + void *priv_data __unused, + struct rstctrl **out_device) +{ + struct stm32_rstline *stm32_rstline = NULL; + uintptr_t control_id = 0; + + if (arg->args_count != 1) + return TEE_ERROR_BAD_PARAMETERS; + + control_id = arg->args[0]; + + stm32_rstline = find_or_allocate_rstline(control_id); + if (!stm32_rstline) + return TEE_ERROR_OUT_OF_MEMORY; + + *out_device = &stm32_rstline->rstctrl; + + return TEE_SUCCESS; +} + +static TEE_Result stm32_rstctrl_provider_probe(const void *fdt, int offs, + const void *compat_data __unused) +{ + struct dt_node_info info = { }; + + assert(rstctrl_ops_is_valid(&stm32_rstctrl_ops)); + + fdt_fill_device_info(fdt, &info, offs); + + assert(info.reg == RCC_BASE && + info.reg_size != DT_INFO_INVALID_REG_SIZE); + + return rstctrl_register_provider(fdt, offs, stm32_rstctrl_get_dev, + NULL); +} + +static const struct dt_device_match stm32_rstctrl_match_table[] = { + { .compatible = "st,stm32mp1-rcc" }, + { .compatible = "st,stm32mp1-rcc-secure" }, + { .compatible = "st,stm32mp13-rcc" }, + { } +}; + +DEFINE_DT_DRIVER(stm32_rstctrl_dt_driver) = { + .name = "stm32_rstctrl", + .type = DT_DRIVER_RSTCTRL, + .match_table = stm32_rstctrl_match_table, + .probe = stm32_rstctrl_provider_probe, +}; diff --git a/optee_os/core/drivers/rstctrl/sub.mk b/optee_os/core/drivers/rstctrl/sub.mk new file mode 100644 index 0000000..0466b62 --- /dev/null +++ b/optee_os/core/drivers/rstctrl/sub.mk @@ -0,0 +1,2 @@ +srcs-y += rstctrl.c +srcs-$(CFG_STM32_RSTCTRL) += stm32_rstctrl.c diff --git a/optee_os/core/drivers/rtc/rtc.c b/optee_os/core/drivers/rtc/rtc.c new file mode 100644 index 0000000..97d3f83 --- /dev/null +++ b/optee_os/core/drivers/rtc/rtc.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microchip + */ + +#include +#include +#include + +struct rtc *rtc_device; + +void rtc_register(struct rtc *rtc) +{ + /* One RTC is supported only */ + assert(!rtc_device); + + /* RTC should *at least* allow to get the time */ + assert(rtc && rtc->ops && rtc->ops->get_time); + + rtc_device = rtc; +} diff --git a/optee_os/core/drivers/rtc/sub.mk b/optee_os/core/drivers/rtc/sub.mk new file mode 100644 index 0000000..87723ed --- /dev/null +++ b/optee_os/core/drivers/rtc/sub.mk @@ -0,0 +1 @@ +srcs-$(CFG_DRIVERS_RTC) += rtc.c diff --git a/optee_os/core/drivers/scif.c b/optee_os/core/drivers/scif.c new file mode 100644 index 0000000..32c382d --- /dev/null +++ b/optee_os/core/drivers/scif.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, GlobalLogic + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +#define SCIF_SCSCR (0x08) +#define SCIF_SCFSR (0x10) +#define SCIF_SCFTDR (0x0C) +#define SCIF_SCFCR (0x18) +#define SCIF_SCFDR (0x1C) + +#define SCSCR_TE BIT(5) +#define SCFSR_TDFE BIT(5) +#define SCFSR_TEND BIT(6) + +#define SCFDR_T_SHIFT 8 + +#define SCIF_TX_FIFO_SIZE 16 + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct scif_uart_data *pd = + container_of(chip, struct scif_uart_data, chip); + + return io_pa_or_va(&pd->base, SCIF_REG_SIZE); +} + +static void scif_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read16(base + SCIF_SCFSR) & SCFSR_TEND)) + ; +} + +static void scif_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + /* Wait until there is space in the FIFO */ + while ((io_read16(base + SCIF_SCFDR) >> SCFDR_T_SHIFT) >= + SCIF_TX_FIFO_SIZE) + ; + io_write8(base + SCIF_SCFTDR, ch); + io_clrbits16(base + SCIF_SCFSR, SCFSR_TEND | SCFSR_TDFE); +} + +static const struct serial_ops scif_uart_ops = { + .flush = scif_uart_flush, + .putc = scif_uart_putc, +}; +DECLARE_KEEP_PAGER(scif_uart_ops); + +void scif_uart_init(struct scif_uart_data *pd, paddr_t pbase) +{ + vaddr_t base; + + pd->base.pa = pbase; + pd->chip.ops = &scif_uart_ops; + + base = io_pa_or_va(&pd->base, SCIF_REG_SIZE); + + /* Set Transmit Enable in Control register */ + io_setbits16(base + SCIF_SCSCR, SCSCR_TE); + + scif_uart_flush(&pd->chip); +} diff --git a/optee_os/core/drivers/scmi-msg/base.c b/optee_os/core/drivers/scmi-msg/base.c new file mode 100644 index 0000000..f43c6db --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/base.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include + +#include "base.h" +#include "common.h" + +static bool message_id_is_supported(unsigned int message_id); + +static void report_version(struct scmi_msg *msg) +{ + struct scmi_protocol_version_p2a return_values = { + .status = SCMI_SUCCESS, + .version = SCMI_PROTOCOL_VERSION_BASE, + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void report_attributes(struct scmi_msg *msg) +{ + size_t protocol_count = plat_scmi_protocol_count(); + struct scmi_protocol_attributes_p2a return_values = { + .status = SCMI_SUCCESS, + /* Null agent count since agent discovery is not supported */ + .attributes = SCMI_BASE_PROTOCOL_ATTRIBUTES(protocol_count, 0), + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void report_message_attributes(struct scmi_msg *msg) +{ + struct scmi_protocol_message_attributes_a2p *in_args = (void *)msg->in; + struct scmi_protocol_message_attributes_p2a return_values = { + .status = SCMI_SUCCESS, + /* For this protocol, attributes shall be zero */ + .attributes = 0, + }; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (!message_id_is_supported(in_args->message_id)) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void discover_vendor(struct scmi_msg *msg) +{ + const char *name = plat_scmi_vendor_name(); + struct scmi_base_discover_vendor_p2a return_values = { + .status = SCMI_SUCCESS, + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + COPY_NAME_IDENTIFIER(return_values.vendor_identifier, name); + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void discover_sub_vendor(struct scmi_msg *msg) +{ + const char *name = plat_scmi_sub_vendor_name(); + struct scmi_base_discover_sub_vendor_p2a return_values = { + .status = SCMI_SUCCESS, + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + COPY_NAME_IDENTIFIER(return_values.sub_vendor_identifier, name); + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void discover_implementation_version(struct scmi_msg *msg) +{ + struct scmi_protocol_version_p2a return_values = { + .status = SCMI_SUCCESS, + .version = SCMI_IMPL_VERSION, + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static unsigned int count_protocols_in_list(const uint8_t *protocol_list) +{ + unsigned int count = 0; + + if (protocol_list) + while (protocol_list[count]) + count++; + + return count; +} + +static void discover_list_protocols(struct scmi_msg *msg) +{ + const struct scmi_base_discover_list_protocols_a2p *a2p = NULL; + struct scmi_base_discover_list_protocols_p2a p2a = { + .status = SCMI_SUCCESS, + }; + const uint8_t *list = NULL; + unsigned int count = 0; + + if (msg->in_size != sizeof(*a2p)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + a2p = (void *)msg->in; + + list = plat_scmi_protocol_list(msg->channel_id); + count = count_protocols_in_list(list); + if (count > a2p->skip) + count = MIN(count - a2p->skip, msg->out_size - sizeof(p2a)); + else + count = 0; + + p2a.num_protocols = count; + + memcpy(msg->out, &p2a, sizeof(p2a)); + memcpy(msg->out + sizeof(p2a), list + a2p->skip, count); + msg->out_size_out = sizeof(p2a) + ROUNDUP(count, sizeof(uint32_t)); +} + +static const scmi_msg_handler_t scmi_base_handler_table[] = { + [SCMI_PROTOCOL_VERSION] = report_version, + [SCMI_PROTOCOL_ATTRIBUTES] = report_attributes, + [SCMI_PROTOCOL_MESSAGE_ATTRIBUTES] = report_message_attributes, + [SCMI_BASE_DISCOVER_VENDOR] = discover_vendor, + [SCMI_BASE_DISCOVER_SUB_VENDOR] = discover_sub_vendor, + [SCMI_BASE_DISCOVER_IMPLEMENTATION_VERSION] = + discover_implementation_version, + [SCMI_BASE_DISCOVER_LIST_PROTOCOLS] = discover_list_protocols, +}; + +static bool message_id_is_supported(unsigned int message_id) +{ + return message_id < ARRAY_SIZE(scmi_base_handler_table) && + scmi_base_handler_table[message_id]; +} + +scmi_msg_handler_t scmi_msg_get_base_handler(struct scmi_msg *msg) +{ + const size_t array_size = ARRAY_SIZE(scmi_base_handler_table); + unsigned int message_id = 0; + + if (msg->message_id >= array_size) { + DMSG("Base handle not found %u", msg->message_id); + return NULL; + } + + message_id = confine_array_index(msg->message_id, array_size); + + return scmi_base_handler_table[message_id]; +} diff --git a/optee_os/core/drivers/scmi-msg/base.h b/optee_os/core/drivers/scmi-msg/base.h new file mode 100644 index 0000000..731443a --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/base.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef SCMI_MSG_BASE_H +#define SCMI_MSG_BASE_H + +#include +#include + +#include "common.h" + +#define SCMI_PROTOCOL_VERSION_BASE 0x20000 + +#define SCMI_DEFAULT_STRING_LENGTH 16 + +enum scmi_base_message_id { + SCMI_BASE_DISCOVER_VENDOR = 0x003, + SCMI_BASE_DISCOVER_SUB_VENDOR = 0x004, + SCMI_BASE_DISCOVER_IMPLEMENTATION_VERSION = 0x005, + SCMI_BASE_DISCOVER_LIST_PROTOCOLS = 0x006, + SCMI_BASE_DISCOVER_AGENT = 0x007, + SCMI_BASE_NOTIFY_ERRORS = 0x008, +}; + +/* + * PROTOCOL_ATTRIBUTES + */ + +#define SCMI_BASE_PROTOCOL_ATTRS_NUM_PROTOCOLS_POS 0 +#define SCMI_BASE_PROTOCOL_ATTRS_NUM_AGENTS_POS 8 + +#define SCMI_BASE_PROTOCOL_ATTRS_NUM_PROTOCOLS_MASK 0xFF +#define SCMI_BASE_PROTOCOL_ATTRS_NUM_AGENTS_MASK 0xFF00 + +#define SCMI_BASE_PROTOCOL_ATTRIBUTES(NUM_PROTOCOLS, NUM_AGENTS) \ + ((((NUM_PROTOCOLS) << SCMI_BASE_PROTOCOL_ATTRS_NUM_PROTOCOLS_POS) & \ + SCMI_BASE_PROTOCOL_ATTRS_NUM_PROTOCOLS_MASK) | \ + (((NUM_AGENTS) << SCMI_BASE_PROTOCOL_ATTRS_NUM_AGENTS_POS) & \ + SCMI_BASE_PROTOCOL_ATTRS_NUM_AGENTS_MASK)) + +/* + * BASE_DISCOVER_VENDOR + */ +struct scmi_base_discover_vendor_p2a { + int32_t status; + char vendor_identifier[SCMI_DEFAULT_STRING_LENGTH]; +}; + +/* + * BASE_DISCOVER_SUB_VENDOR + */ +struct scmi_base_discover_sub_vendor_p2a { + int32_t status; + char sub_vendor_identifier[SCMI_DEFAULT_STRING_LENGTH]; +}; + +/* + * BASE_DISCOVER_IMPLEMENTATION_VERSION + * No special structure right now, see protocol_version. + */ + +/* + * BASE_DISCOVER_LIST_PROTOCOLS + */ +struct scmi_base_discover_list_protocols_a2p { + uint32_t skip; +}; + +struct scmi_base_discover_list_protocols_p2a { + int32_t status; + uint32_t num_protocols; + uint32_t protocols[]; +}; + +/* + * scmi_msg_get_base_handler - Return a handler for a base message + * @msg - message to process + * Return a function handler for the message or NULL + */ +scmi_msg_handler_t scmi_msg_get_base_handler(struct scmi_msg *msg); + +#endif /* SCMI_MSG_BASE_H */ diff --git a/optee_os/core/drivers/scmi-msg/clock.c b/optee_os/core/drivers/scmi-msg/clock.c new file mode 100644 index 0000000..dce76fa --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/clock.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ +#include +#include +#include +#include +#include +#include + +#include "clock.h" +#include "common.h" + +static bool message_id_is_supported(unsigned int message_id); + +size_t __weak plat_scmi_clock_count(unsigned int channel_id __unused) +{ + return 0; +} + +const char __weak *plat_scmi_clock_get_name(unsigned int channel_id __unused, + unsigned int scmi_id __unused) +{ + return NULL; +} + +int32_t __weak plat_scmi_clock_rates_array(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + size_t start_index __unused, + unsigned long *rates __unused, + size_t *nb_elts __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_clock_rates_by_step(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + unsigned long *steps __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +unsigned long __weak plat_scmi_clock_get_rate(unsigned int channel_id __unused, + unsigned int scmi_id __unused) +{ + return 0; +} + +int32_t __weak plat_scmi_clock_set_rate(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + unsigned long rate __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_clock_get_state(unsigned int channel_id __unused, + unsigned int scmi_id __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_clock_set_state(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + bool enable_not_disable __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +static void report_version(struct scmi_msg *msg) +{ + struct scmi_protocol_version_p2a return_values = { + .status = SCMI_SUCCESS, + .version = SCMI_PROTOCOL_VERSION_CLOCK, + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void report_attributes(struct scmi_msg *msg) +{ + size_t clk_count = plat_scmi_clock_count(msg->channel_id); + struct scmi_protocol_attributes_p2a return_values = { + .status = SCMI_SUCCESS, + .attributes = SCMI_CLOCK_PROTOCOL_ATTRIBUTES(1, clk_count), + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void report_message_attributes(struct scmi_msg *msg) +{ + struct scmi_protocol_message_attributes_a2p *in_args = (void *)msg->in; + struct scmi_protocol_message_attributes_p2a return_values = { + .status = SCMI_SUCCESS, + /* For this protocol, attributes shall be zero */ + .attributes = 0, + }; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (!message_id_is_supported(in_args->message_id)) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void scmi_clock_attributes(struct scmi_msg *msg) +{ + const struct scmi_clock_attributes_a2p *in_args = (void *)msg->in; + struct scmi_clock_attributes_p2a return_values = { + .status = SCMI_SUCCESS, + }; + const char *name = NULL; + unsigned int clock_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->clock_id >= plat_scmi_clock_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + clock_id = confine_array_index(in_args->clock_id, + plat_scmi_clock_count(msg->channel_id)); + + name = plat_scmi_clock_get_name(msg->channel_id, clock_id); + if (!name) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + COPY_NAME_IDENTIFIER(return_values.clock_name, name); + + return_values.attributes = plat_scmi_clock_get_state(msg->channel_id, + clock_id); + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void scmi_clock_rate_get(struct scmi_msg *msg) +{ + const struct scmi_clock_rate_get_a2p *in_args = (void *)msg->in; + unsigned long rate = 0; + struct scmi_clock_rate_get_p2a return_values = { }; + unsigned int clock_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->clock_id >= plat_scmi_clock_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + clock_id = confine_array_index(in_args->clock_id, + plat_scmi_clock_count(msg->channel_id)); + + rate = plat_scmi_clock_get_rate(msg->channel_id, clock_id); + + reg_pair_from_64(rate, return_values.rate + 1, return_values.rate); + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void scmi_clock_rate_set(struct scmi_msg *msg) +{ + const struct scmi_clock_rate_set_a2p *in_args = (void *)msg->in; + uint64_t rate_64 = 0; + unsigned long rate = 0; + int32_t status = 0; + unsigned int clock_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->clock_id >= plat_scmi_clock_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + clock_id = confine_array_index(in_args->clock_id, + plat_scmi_clock_count(msg->channel_id)); + + rate_64 = reg_pair_to_64(in_args->rate[1], in_args->rate[0]); + rate = rate_64; + + status = plat_scmi_clock_set_rate(msg->channel_id, clock_id, rate); + + scmi_status_response(msg, status); +} + +static void scmi_clock_config_set(struct scmi_msg *msg) +{ + const struct scmi_clock_config_set_a2p *in_args = (void *)msg->in; + int32_t status = SCMI_GENERIC_ERROR; + bool enable = false; + unsigned int clock_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->clock_id >= plat_scmi_clock_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + clock_id = confine_array_index(in_args->clock_id, + plat_scmi_clock_count(msg->channel_id)); + + enable = in_args->attributes & SCMI_CLOCK_CONFIG_SET_ENABLE_MASK; + + status = plat_scmi_clock_set_state(msg->channel_id, clock_id, enable); + + scmi_status_response(msg, status); +} + +#define SCMI_RATES_BY_ARRAY(_nb_rates, _rem_rates) \ + SCMI_CLOCK_DESCRIBE_RATES_NUM_RATES_FLAGS((_nb_rates), \ + SCMI_CLOCK_RATE_FORMAT_LIST, \ + (_rem_rates)) +#define SCMI_RATES_BY_STEP \ + SCMI_CLOCK_DESCRIBE_RATES_NUM_RATES_FLAGS(3, \ + SCMI_CLOCK_RATE_FORMAT_RANGE, \ + 0) + +#define RATE_DESC_SIZE sizeof(struct scmi_clock_rate) + +static void write_rate_desc_array_in_buffer(char *dest, unsigned long *rates, + size_t nb_elt) +{ + uint32_t *out = NULL; + size_t n = 0; + + assert(IS_ALIGNED_WITH_TYPE(dest, uint32_t)); + out = (uint32_t *)(uintptr_t)dest; + + for (n = 0; n < nb_elt; n++) { + uint64_t rate = rates[n]; + + reg_pair_from_64(rate, out + 2 * n + 1, out + 2 * n); + } +} + +static void scmi_clock_describe_rates(struct scmi_msg *msg) +{ + const struct scmi_clock_describe_rates_a2p *in_args = (void *)msg->in; + struct scmi_clock_describe_rates_p2a p2a = { }; + size_t nb_rates = 0; + int32_t status = SCMI_GENERIC_ERROR; + unsigned int clock_id = 0; + unsigned int out_count = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->clock_id >= plat_scmi_clock_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + clock_id = confine_array_index(in_args->clock_id, + plat_scmi_clock_count(msg->channel_id)); + + /* Platform may support array rate description */ + status = plat_scmi_clock_rates_array(msg->channel_id, clock_id, 0, NULL, + &nb_rates); + if (status == SCMI_SUCCESS) { + unsigned int rate_index = in_args->rate_index; + unsigned int remaining = 0; + size_t avail_sz = msg->out_size - sizeof(p2a); + char *out_rates = msg->out + sizeof(p2a); + + if (avail_sz < RATE_DESC_SIZE && nb_rates) { + status = SCMI_PROTOCOL_ERROR; + goto out; + } + + while (avail_sz >= RATE_DESC_SIZE && rate_index < nb_rates) { + unsigned long rate = 0; + size_t cnt = 1; + + status = plat_scmi_clock_rates_array(msg->channel_id, + clock_id, + rate_index, + &rate, &cnt); + if (status) + goto out; + + write_rate_desc_array_in_buffer(out_rates, &rate, cnt); + avail_sz -= RATE_DESC_SIZE; + out_rates += RATE_DESC_SIZE; + rate_index++; + } + + out_count = rate_index - in_args->rate_index; + remaining = nb_rates - in_args->rate_index; + p2a.num_rates_flags = SCMI_RATES_BY_ARRAY(out_count, remaining); + } else if (status == SCMI_NOT_SUPPORTED) { + unsigned long triplet[3] = { 0, 0, 0 }; + + if (msg->out_size < sizeof(p2a) + 3 * RATE_DESC_SIZE) { + status = SCMI_PROTOCOL_ERROR; + goto out; + } + + /* Platform may support min/max/step triplet description */ + status = plat_scmi_clock_rates_by_step(msg->channel_id, + clock_id, triplet); + if (status) + goto out; + + write_rate_desc_array_in_buffer(msg->out + sizeof(p2a), + triplet, 3); + + out_count = 3; + p2a.num_rates_flags = SCMI_RATES_BY_STEP; + } else { + /* Fallthrough generic exit sequence below with error status */ + } + +out: + if (status) { + scmi_status_response(msg, status); + } else { + p2a.status = SCMI_SUCCESS; + memcpy(msg->out, &p2a, sizeof(p2a)); + msg->out_size_out = sizeof(p2a) + out_count * RATE_DESC_SIZE; + } +} + +static const scmi_msg_handler_t scmi_clock_handler_table[] = { + [SCMI_PROTOCOL_VERSION] = report_version, + [SCMI_PROTOCOL_ATTRIBUTES] = report_attributes, + [SCMI_PROTOCOL_MESSAGE_ATTRIBUTES] = report_message_attributes, + [SCMI_CLOCK_ATTRIBUTES] = scmi_clock_attributes, + [SCMI_CLOCK_DESCRIBE_RATES] = scmi_clock_describe_rates, + [SCMI_CLOCK_RATE_SET] = scmi_clock_rate_set, + [SCMI_CLOCK_RATE_GET] = scmi_clock_rate_get, + [SCMI_CLOCK_CONFIG_SET] = scmi_clock_config_set, +}; + +static bool message_id_is_supported(unsigned int message_id) +{ + return message_id < ARRAY_SIZE(scmi_clock_handler_table) && + scmi_clock_handler_table[message_id]; +} + +scmi_msg_handler_t scmi_msg_get_clock_handler(struct scmi_msg *msg) +{ + const size_t array_size = ARRAY_SIZE(scmi_clock_handler_table); + unsigned int message_id = 0; + + if (msg->message_id >= array_size) { + DMSG("Clock handle not found %u", msg->message_id); + return NULL; + } + + message_id = confine_array_index(msg->message_id, array_size); + + return scmi_clock_handler_table[message_id]; +} diff --git a/optee_os/core/drivers/scmi-msg/clock.h b/optee_os/core/drivers/scmi-msg/clock.h new file mode 100644 index 0000000..a20678c --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/clock.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef SCMI_MSG_CLOCK_H +#define SCMI_MSG_CLOCK_H + +#include +#include + +#include "common.h" + +#define SCMI_PROTOCOL_VERSION_CLOCK 0x20000 + +/* + * Identifiers of the SCMI Clock Management Protocol commands + */ +enum scmi_clock_command_id { + SCMI_CLOCK_ATTRIBUTES = 0x003, + SCMI_CLOCK_DESCRIBE_RATES = 0x004, + SCMI_CLOCK_RATE_SET = 0x005, + SCMI_CLOCK_RATE_GET = 0x006, + SCMI_CLOCK_CONFIG_SET = 0x007, +}; + +/* Protocol attributes */ +#define SCMI_CLOCK_CLOCK_COUNT_MASK GENMASK_32(15, 0) +#define SCMI_CLOCK_MAX_PENDING_TRANSITIONS_MASK GENMASK_32(23, 16) + +#define SCMI_CLOCK_PROTOCOL_ATTRIBUTES(_max_pending, _clk_count) \ + ((((_max_pending) << 16) & SCMI_CLOCK_MAX_PENDING_TRANSITIONS_MASK) | \ + (((_clk_count) & SCMI_CLOCK_CLOCK_COUNT_MASK))) + +struct scmi_clock_attributes_a2p { + uint32_t clock_id; +}; + +#define SCMI_CLOCK_NAME_LENGTH_MAX 16 + +struct scmi_clock_attributes_p2a { + int32_t status; + uint32_t attributes; + char clock_name[SCMI_CLOCK_NAME_LENGTH_MAX]; +}; + +/* + * Clock Rate Get + */ + +struct scmi_clock_rate_get_a2p { + uint32_t clock_id; +}; + +struct scmi_clock_rate_get_p2a { + int32_t status; + uint32_t rate[2]; +}; + +/* + * Clock Rate Set + */ + +/* If set, set the new clock rate asynchronously */ +#define SCMI_CLOCK_RATE_SET_ASYNC_POS 0 +/* If set, do not send a delayed asynchronous response */ +#define SCMI_CLOCK_RATE_SET_NO_DELAYED_RESPONSE_POS 1 +/* Round up, if set, otherwise round down */ +#define SCMI_CLOCK_RATE_SET_ROUND_UP_POS 2 +/* If set, the platform chooses the appropriate rounding mode */ +#define SCMI_CLOCK_RATE_SET_ROUND_AUTO_POS 3 + +#define SCMI_CLOCK_RATE_SET_ASYNC_MASK \ + BIT(SCMI_CLOCK_RATE_SET_ASYNC_POS) +#define SCMI_CLOCK_RATE_SET_NO_DELAYED_RESPONSE_MASK \ + BIT(SCMI_CLOCK_RATE_SET_NO_DELAYED_RESPONSE_POS) +#define SCMI_CLOCK_RATE_SET_ROUND_UP_MASK \ + BIT(SCMI_CLOCK_RATE_SET_ROUND_UP_POS) +#define SCMI_CLOCK_RATE_SET_ROUND_AUTO_MASK \ + BIT(SCMI_CLOCK_RATE_SET_ROUND_AUTO_POS) + +struct scmi_clock_rate_set_a2p { + uint32_t flags; + uint32_t clock_id; + uint32_t rate[2]; +}; + +struct scmi_clock_rate_set_p2a { + int32_t status; +}; + +/* + * Clock Config Set + */ + +#define SCMI_CLOCK_CONFIG_SET_ENABLE_POS 0 + +#define SCMI_CLOCK_CONFIG_SET_ENABLE_MASK \ + BIT(SCMI_CLOCK_CONFIG_SET_ENABLE_POS) + +struct scmi_clock_config_set_a2p { + uint32_t clock_id; + uint32_t attributes; +}; + +struct scmi_clock_config_set_p2a { + int32_t status; +}; + +/* + * Clock Describe Rates + */ + +#define SCMI_CLOCK_RATE_FORMAT_RANGE 1 +#define SCMI_CLOCK_RATE_FORMAT_LIST 0 + +#define SCMI_CLOCK_DESCRIBE_RATES_REMAINING_MASK GENMASK_32(31, 16) +#define SCMI_CLOCK_DESCRIBE_RATES_REMAINING_POS 16 + +#define SCMI_CLOCK_DESCRIBE_RATES_FORMAT_MASK BIT(12) +#define SCMI_CLOCK_DESCRIBE_RATES_FORMAT_POS 12 + +#define SCMI_CLOCK_DESCRIBE_RATES_COUNT_MASK GENMASK_32(11, 0) + +#define SCMI_CLOCK_DESCRIBE_RATES_NUM_RATES_FLAGS(_count, _fmt, _rem_rates) \ + ( \ + ((_count) & SCMI_CLOCK_DESCRIBE_RATES_COUNT_MASK) | \ + (((_rem_rates) << SCMI_CLOCK_DESCRIBE_RATES_REMAINING_POS) & \ + SCMI_CLOCK_DESCRIBE_RATES_REMAINING_MASK) | \ + (((_fmt) << SCMI_CLOCK_DESCRIBE_RATES_FORMAT_POS) & \ + SCMI_CLOCK_DESCRIBE_RATES_FORMAT_MASK) \ + ) + +struct scmi_clock_rate { + uint32_t low; + uint32_t high; +}; + +struct scmi_clock_describe_rates_a2p { + uint32_t clock_id; + uint32_t rate_index; +}; + +struct scmi_clock_describe_rates_p2a { + int32_t status; + uint32_t num_rates_flags; + struct scmi_clock_rate rates[]; +}; + +#ifdef CFG_SCMI_MSG_CLOCK +/* + * scmi_msg_get_clock_handler - Return a handler for a clock message + * @msg - message to process + * Return a function handler for the message or NULL + */ +scmi_msg_handler_t scmi_msg_get_clock_handler(struct scmi_msg *msg); +#else +static inline +scmi_msg_handler_t scmi_msg_get_clock_handler(struct scmi_msg *msg __unused) +{ + return NULL; +} +#endif +#endif /* SCMI_MSG_CLOCK_H */ diff --git a/optee_os/core/drivers/scmi-msg/clock_generic.c b/optee_os/core/drivers/scmi-msg/clock_generic.c new file mode 100644 index 0000000..1b2d1e6 --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/clock_generic.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Microchip + */ + +#include +#include +#include +#include +#include +#include + +#include "clock.h" + +/** + * struct scmi_clk - Binds an SCMI channel/clock to a core clk reference + * @clk: Core clock reference + * @channel_id: SCMI server channel handle exposing the clock + * @scmi_id: SCMI clock domain ID + * @enabled: SCMI clock state + * @link: Reference in SCMI server clock list + */ +struct scmi_clk { + struct clk *clk; + unsigned int channel_id; + unsigned int scmi_id; + bool enabled; + SLIST_ENTRY(scmi_clk) link; +}; + +static bool scmi_clk_init_done; +static SLIST_HEAD(, scmi_clk) scmi_clk_list = + SLIST_HEAD_INITIALIZER(scmi_clk_list); + +size_t plat_scmi_clock_count(unsigned int channel_id) +{ + unsigned int count = 0; + unsigned int max_id = 0; + struct scmi_clk *clk = NULL; + + SLIST_FOREACH(clk, &scmi_clk_list, link) { + if (clk->channel_id == channel_id) { + count++; + max_id = MAX(max_id, clk->scmi_id); + } + } + + if (!count) + return 0; + + /* IDs are starting from 0 so we need to return max_id + 1 for count */ + return max_id + 1; +} + +static struct scmi_clk *clk_scmi_get_by_id(unsigned int channel_id, + unsigned int scmi_id) +{ + struct scmi_clk *clk = NULL; + + SLIST_FOREACH(clk, &scmi_clk_list, link) + if (clk->channel_id == channel_id && clk->scmi_id == scmi_id) + return clk; + + return NULL; +} + +const char *plat_scmi_clock_get_name(unsigned int channel_id, + unsigned int scmi_id) +{ + struct scmi_clk *clk = NULL; + + clk = clk_scmi_get_by_id(channel_id, scmi_id); + if (!clk) + return "dummy"; + + return clk_get_name(clk->clk); +} + +int32_t plat_scmi_clock_rates_array(unsigned int channel_id, + unsigned int scmi_id, + size_t start_index, + unsigned long *rates, + size_t *nb_elts) +{ + struct scmi_clk *clk = NULL; + + clk = clk_scmi_get_by_id(channel_id, scmi_id); + if (!clk) + return SCMI_DENIED; + + if (clk_get_rates_array(clk->clk, start_index, rates, nb_elts)) + return SCMI_GENERIC_ERROR; + + return SCMI_SUCCESS; +} + +unsigned long plat_scmi_clock_get_rate(unsigned int channel_id, + unsigned int scmi_id) +{ + struct scmi_clk *clk = NULL; + + clk = clk_scmi_get_by_id(channel_id, scmi_id); + if (!clk) + return 0; + + return clk_get_rate(clk->clk); +} + +int32_t plat_scmi_clock_set_rate(unsigned int channel_id, + unsigned int scmi_id, + unsigned long rate) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct scmi_clk *clk = NULL; + + clk = clk_scmi_get_by_id(channel_id, scmi_id); + if (!clk) + return SCMI_DENIED; + + res = clk_set_rate(clk->clk, rate); + if (res) + return SCMI_GENERIC_ERROR; + + return SCMI_SUCCESS; +} + +int32_t plat_scmi_clock_get_state(unsigned int channel_id, + unsigned int scmi_id) +{ + struct scmi_clk *clk = NULL; + + clk = clk_scmi_get_by_id(channel_id, scmi_id); + if (!clk) + return false; + + return clk->enabled; +} + +int32_t plat_scmi_clock_set_state(unsigned int channel_id, + unsigned int scmi_id, + bool enable_not_disable) +{ + struct scmi_clk *clk = NULL; + + clk = clk_scmi_get_by_id(channel_id, scmi_id); + if (!clk) { + if (enable_not_disable) + return SCMI_DENIED; + else + return SCMI_SUCCESS; + } + + if (enable_not_disable) { + if (!clk->enabled) { + if (clk_enable(clk->clk)) + return SCMI_GENERIC_ERROR; + clk->enabled = true; + } + } else { + if (clk->enabled) { + clk_disable(clk->clk); + clk->enabled = false; + } + } + + return SCMI_SUCCESS; +} + +static TEE_Result clk_check_scmi_id(struct clk *new_clk, + unsigned int channel_id, + unsigned int scmi_id) +{ + struct scmi_clk *clk = NULL; + + SLIST_FOREACH(clk, &scmi_clk_list, link) { + if (clk->channel_id == channel_id && clk->scmi_id == scmi_id) { + EMSG("SCMI channel %u, clock %u already registered", + channel_id, scmi_id); + return TEE_ERROR_BAD_PARAMETERS; + } + } + + if (strlen(clk_get_name(new_clk)) >= SCMI_CLOCK_NAME_LENGTH_MAX) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; +} + +TEE_Result scmi_clk_add(struct clk *clk, unsigned int channel_id, + unsigned int scmi_id) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct scmi_clk *scmi_clk = NULL; + + if (scmi_clk_init_done) + return TEE_ERROR_BAD_STATE; + + res = clk_check_scmi_id(clk, channel_id, scmi_id); + if (res) + return res; + + scmi_clk = calloc(1, sizeof(*scmi_clk)); + if (!scmi_clk) + return TEE_ERROR_OUT_OF_MEMORY; + + scmi_clk->clk = clk; + scmi_clk->channel_id = channel_id; + scmi_clk->scmi_id = scmi_id; + scmi_clk->enabled = false; + + SLIST_INSERT_HEAD(&scmi_clk_list, scmi_clk, link); + + return TEE_SUCCESS; +} + +static TEE_Result scmi_clk_init_fini(void) +{ + scmi_clk_init_done = true; + + return TEE_SUCCESS; +} + +release_init_resource(scmi_clk_init_fini); diff --git a/optee_os/core/drivers/scmi-msg/common.h b/optee_os/core/drivers/scmi-msg/common.h new file mode 100644 index 0000000..e251e3b --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/common.h @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ +#ifndef SCMI_MSG_COMMON_H +#define SCMI_MSG_COMMON_H + +#include +#include +#include +#include +#include + +#define SCMI_VERSION 0x30000 +#define SCMI_IMPL_VERSION 0 + +/* + * Secure copy of input payload: we expect small messages, at most the legacy + * SMT messages that are 128 bytes (Linux kernel) including SMT header. + */ +#define SCMI_SEC_PAYLOAD_SIZE 92 + +/* + * Copy name identifier in target buffer following the SCMI specification + * that state name identifier shall be a null terminated string. + */ +#define COPY_NAME_IDENTIFIER(_dst_array, _name) \ + do { \ + assert(strlen(_name) < sizeof(_dst_array)); \ + strncpy((_dst_array), (_name), sizeof(_dst_array)); \ + } while (0) + +/* Common command identifiers shared by all procotols */ +enum scmi_common_message_id { + SCMI_PROTOCOL_VERSION = 0x000, + SCMI_PROTOCOL_ATTRIBUTES = 0x001, + SCMI_PROTOCOL_MESSAGE_ATTRIBUTES = 0x002 +}; + +/* Common platform-to-agent (p2a) PROTOCOL_VERSION structure */ +struct scmi_protocol_version_p2a { + int32_t status; + uint32_t version; +}; + +/* Generic platform-to-agent (p2a) PROTOCOL_ATTRIBUTES structure */ +struct scmi_protocol_attributes_p2a { + int32_t status; + uint32_t attributes; +}; + +/* Generic agent-to-platform (a2p) PROTOCOL_MESSAGE_ATTRIBUTES structure */ +struct scmi_protocol_message_attributes_a2p { + uint32_t message_id; +}; + +/* Generic platform-to-agent (p2a) PROTOCOL_MESSAGE_ATTRIBUTES structure */ +struct scmi_protocol_message_attributes_p2a { + int32_t status; + uint32_t attributes; +}; + +/* + * struct scmi_msg - SCMI message context + * + * @channel_id: SCMI channel ID, safely set from secure world + * @protocol_id: SCMI protocol ID for the related message, set by caller agent + * @message_id: SCMI message ID for the related message, set by caller agent + * @in: Address of the incoming message payload copied in secure memory + * @in_size: Byte length of the incoming message payload, set by caller agent + * @out: Address of of the output message payload message in non-secure memory + * @out_size: Byte length of the provisionned output buffer + * @out_size_out: Byte length of the output message payload + */ +struct scmi_msg { + unsigned int channel_id; + unsigned int protocol_id; + unsigned int message_id; + char *in; + size_t in_size; + char *out; + size_t out_size; + size_t out_size_out; +}; + +/* + * Type scmi_msg_handler_t is used by procotol drivers to safely find + * the handler function for the incoming message ID. + */ +typedef void (*scmi_msg_handler_t)(struct scmi_msg *msg); + +/* + * Process Read, process and write response for input SCMI message + * + * @msg: SCMI message context + */ +void scmi_process_message(struct scmi_msg *msg); + +/* + * Write SCMI response payload to output message shared memory + * + * @msg: SCMI message context + * @payload: Output message payload + * @size: Byte size of output message payload + */ +void scmi_write_response(struct scmi_msg *msg, void *payload, size_t size); + +/* + * Write status only SCMI response payload to output message shared memory + * + * @msg: SCMI message context + * @status: SCMI status value returned to caller + */ +void scmi_status_response(struct scmi_msg *msg, int32_t status); + +/* + * Claim access to channel + * @channel: SCMI channel reference + * Return true upon success or false if the channel is already busy + */ +bool scmi_msg_claim_channel(struct scmi_msg_channel *channel); + +/* + * Release access channel + * @channel: SCMI channel reference + */ +void scmi_msg_release_channel(struct scmi_msg_channel *channel); + +/* + * Entry for processing a channel using SMT shared memory protocol + * @channel_id: SCMI channel identifier provided by client + * @payload_buf: Secure buffer where to copy input message + */ +void scmi_entry_smt(unsigned int channel_id, uint32_t *payload_buf); + +/* + * Entry for processing a channel using SMT shared memory protocol + * + * @channel_id: SCMI channel identifier provided by client + * @in_buf: Shared buffer storing input SCMI message + * @in_size: Byte size of @in_buf, including MSG header and message payload + * @out_buf: Shared buffer storing input SCMI message + * @out_size: [in] @out_buf max byte size + * [out] @out_buf output byte size (MSG header and message payload) + * @sec_buf: Secure buffer where to copy input message + */ +TEE_Result scmi_entry_msg(unsigned int channel_id, void *in_buf, size_t in_size, + void *out_buf, size_t *out_size, uint32_t *sec_buf); +#endif /* SCMI_MSG_COMMON_H */ diff --git a/optee_os/core/drivers/scmi-msg/entry.c b/optee_os/core/drivers/scmi-msg/entry.c new file mode 100644 index 0000000..0fd0968 --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/entry.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019-2022, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base.h" +#include "clock.h" +#include "common.h" +#include "reset_domain.h" +#include "voltage_domain.h" + +/* Provision input message payload buffers for each supported entry channel */ +#define SCMI_PAYLOAD_U32_MAX (SCMI_SEC_PAYLOAD_SIZE / sizeof(uint32_t)) + +static uint32_t threaded_payload[CFG_NUM_THREADS][SCMI_PAYLOAD_U32_MAX] +__maybe_unused; + +static uint32_t interrupt_payload[CFG_TEE_CORE_NB_CORE][SCMI_PAYLOAD_U32_MAX] +__maybe_unused; + +static uint32_t fastcall_payload[CFG_TEE_CORE_NB_CORE][SCMI_PAYLOAD_U32_MAX] +__maybe_unused; + +/* SMP protection on channel->busy field */ +static unsigned int smt_channels_lock; + +/* If channel is not busy, set busy and return true, otherwise return false */ +bool scmi_msg_claim_channel(struct scmi_msg_channel *channel) +{ + uint32_t exceptions = cpu_spin_lock_xsave(&smt_channels_lock); + bool channel_is_busy = channel->busy; + + if (!channel_is_busy) + channel->busy = true; + + cpu_spin_unlock_xrestore(&smt_channels_lock, exceptions); + + return !channel_is_busy; +} + +void scmi_msg_release_channel(struct scmi_msg_channel *channel) +{ + channel->busy = false; +} + +void scmi_status_response(struct scmi_msg *msg, int32_t status) +{ + assert(msg->out && msg->out_size >= sizeof(int32_t)); + + memcpy(msg->out, &status, sizeof(int32_t)); + msg->out_size_out = sizeof(int32_t); +} + +void scmi_write_response(struct scmi_msg *msg, void *payload, size_t size) +{ + if (msg->out_size < size) { + DMSG("SCMI resp. payload %zu > %zu bytes", size, msg->out_size); + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + } else { + memcpy(msg->out, payload, size); + msg->out_size_out = size; + } +} + +void scmi_process_message(struct scmi_msg *msg) +{ + scmi_msg_handler_t handler = NULL; + + switch (msg->protocol_id) { + case SCMI_PROTOCOL_ID_BASE: + handler = scmi_msg_get_base_handler(msg); + break; + case SCMI_PROTOCOL_ID_CLOCK: + handler = scmi_msg_get_clock_handler(msg); + break; + case SCMI_PROTOCOL_ID_RESET_DOMAIN: + handler = scmi_msg_get_rd_handler(msg); + break; + case SCMI_PROTOCOL_ID_VOLTAGE_DOMAIN: + handler = scmi_msg_get_voltd_handler(msg); + break; + default: + break; + } + + if (handler) { + handler(msg); + return; + } + + DMSG("Channel %u Protocol %#x Message %#x: not supported", + msg->channel_id, msg->protocol_id, msg->message_id); + + scmi_status_response(msg, SCMI_NOT_SUPPORTED); +} + +#ifdef CFG_SCMI_MSG_SMT_FASTCALL_ENTRY +void scmi_smt_fastcall_smc_entry(unsigned int channel_id) +{ + assert(!plat_scmi_get_channel(channel_id)->threaded); + + scmi_entry_smt(channel_id, fastcall_payload[get_core_pos()]); +} +#endif + +#ifdef CFG_SCMI_MSG_SMT_INTERRUPT_ENTRY +void scmi_smt_interrupt_entry(unsigned int channel_id) +{ + assert(!plat_scmi_get_channel(channel_id)->threaded); + + scmi_entry_smt(channel_id, interrupt_payload[get_core_pos()]); +} +#endif + +#ifdef CFG_SCMI_MSG_SMT_THREAD_ENTRY +void scmi_smt_threaded_entry(unsigned int channel_id) +{ + assert(plat_scmi_get_channel(channel_id)->threaded); + + scmi_entry_smt(channel_id, threaded_payload[thread_get_id()]); +} +#endif + +#ifdef CFG_SCMI_MSG_SHM_MSG +TEE_Result scmi_msg_threaded_entry(unsigned int channel_id, + void *in_buf, size_t in_size, + void *out_buf, size_t *out_size) +{ + assert(plat_scmi_get_channel(channel_id)->threaded); + + return scmi_entry_msg(channel_id, in_buf, in_size, out_buf, out_size, + threaded_payload[thread_get_id()]); +} +#endif diff --git a/optee_os/core/drivers/scmi-msg/reset_domain.c b/optee_os/core/drivers/scmi-msg/reset_domain.c new file mode 100644 index 0000000..e3999ec --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/reset_domain.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "reset_domain.h" + +static bool message_id_is_supported(unsigned int message_id); + +size_t __weak plat_scmi_rd_count(unsigned int channel_id __unused) +{ + return 0; +} + +const char __weak *plat_scmi_rd_get_name(unsigned int channel_id __unused, + unsigned int scmi_id __unused) +{ + return NULL; +} + +int32_t __weak plat_scmi_rd_autonomous(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + unsigned int state __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_rd_set_state(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + bool assert_not_deassert __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +static void report_version(struct scmi_msg *msg) +{ + struct scmi_protocol_version_p2a return_values = { + .status = SCMI_SUCCESS, + .version = SCMI_PROTOCOL_VERSION_RESET_DOMAIN, + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void report_attributes(struct scmi_msg *msg) +{ + struct scmi_protocol_attributes_p2a return_values = { + .status = SCMI_SUCCESS, + .attributes = plat_scmi_rd_count(msg->channel_id), + }; + + if (msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void report_message_attributes(struct scmi_msg *msg) +{ + struct scmi_protocol_message_attributes_a2p *in_args = (void *)msg->in; + struct scmi_protocol_message_attributes_p2a return_values = { + .status = SCMI_SUCCESS, + /* For this protocol, attributes shall be zero */ + .attributes = 0, + }; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (!message_id_is_supported(in_args->message_id)) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void reset_domain_attributes(struct scmi_msg *msg) +{ + struct scmi_reset_domain_attributes_a2p *in_args = (void *)msg->in; + struct scmi_reset_domain_attributes_p2a return_values = { }; + const char *name = NULL; + unsigned int domain_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->domain_id >= plat_scmi_rd_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_rd_count(msg->channel_id)); + + name = plat_scmi_rd_get_name(msg->channel_id, domain_id); + if (!name) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + COPY_NAME_IDENTIFIER(return_values.name, name); + return_values.status = SCMI_SUCCESS; + return_values.flags = 0; /* Async and Notif are not supported */ + return_values.latency = SCMI_RESET_DOMAIN_ATTR_UNK_LAT; + + scmi_write_response(msg, &return_values, sizeof(return_values)); +} + +static void reset_request(struct scmi_msg *msg) +{ + struct scmi_reset_domain_request_a2p *in_args = (void *)msg->in; + struct scmi_reset_domain_request_p2a out_args = { + .status = SCMI_SUCCESS, + }; + unsigned int domain_id = 0; + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_rd_count(msg->channel_id)); + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->domain_id >= plat_scmi_rd_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + if (in_args->flags & SCMI_RESET_DOMAIN_AUTO) + out_args.status = plat_scmi_rd_autonomous(msg->channel_id, + domain_id, + in_args->reset_state); + else if (in_args->flags & SCMI_RESET_DOMAIN_EXPLICIT) + out_args.status = plat_scmi_rd_set_state(msg->channel_id, + domain_id, true); + else + out_args.status = plat_scmi_rd_set_state(msg->channel_id, + domain_id, false); + + if (out_args.status) + scmi_status_response(msg, out_args.status); + else + scmi_write_response(msg, &out_args, sizeof(out_args)); +} + +static const scmi_msg_handler_t scmi_rd_handler_table[] = { + [SCMI_PROTOCOL_VERSION] = report_version, + [SCMI_PROTOCOL_ATTRIBUTES] = report_attributes, + [SCMI_PROTOCOL_MESSAGE_ATTRIBUTES] = report_message_attributes, + [SCMI_RESET_DOMAIN_ATTRIBUTES] = reset_domain_attributes, + [SCMI_RESET_DOMAIN_REQUEST] = reset_request, +}; + +static bool message_id_is_supported(unsigned int message_id) +{ + return message_id < ARRAY_SIZE(scmi_rd_handler_table) && + scmi_rd_handler_table[message_id]; +} + +scmi_msg_handler_t scmi_msg_get_rd_handler(struct scmi_msg *msg) +{ + const size_t array_size = ARRAY_SIZE(scmi_rd_handler_table); + unsigned int message_id = 0; + + if (msg->message_id >= array_size) { + DMSG("Reset domain handle not found %u", msg->message_id); + return NULL; + } + + message_id = confine_array_index(msg->message_id, array_size); + + return scmi_rd_handler_table[message_id]; +} diff --git a/optee_os/core/drivers/scmi-msg/reset_domain.h b/optee_os/core/drivers/scmi-msg/reset_domain.h new file mode 100644 index 0000000..e86b9e3 --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/reset_domain.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ +#ifndef SCMI_MSG_RESET_DOMAIN_H +#define SCMI_MSG_RESET_DOMAIN_H + +#include +#include +#include +#include +#include + +#include "common.h" + +#define SCMI_PROTOCOL_VERSION_RESET_DOMAIN 0x10000 + +#define SCMI_RESET_STATE_ARCH BIT(31) +#define SCMI_RESET_STATE_IMPL 0 + +/* + * Identifiers of the SCMI Reset Domain Management Protocol commands + */ +enum scmi_reset_domain_command_id { + SCMI_RESET_DOMAIN_ATTRIBUTES = 0x03, + SCMI_RESET_DOMAIN_REQUEST = 0x04, + SCMI_RESET_DOMAIN_NOTIFY = 0x05, +}; + +/* + * Identifiers of the SCMI Reset Domain Management Protocol responses + */ +enum scmi_reset_domain_response_id { + SCMI_RESET_ISSUED = 0x00, + SCMI_RESET_COMPLETE = 0x04, +}; + +/* + * PROTOCOL_ATTRIBUTES + */ + +#define SCMI_RESET_DOMAIN_COUNT_MASK GENMASK_32(15, 0) + +struct scmi_reset_domain_protocol_attributes_p2a { + int32_t status; + uint32_t attributes; +}; + +/* Value for scmi_reset_domain_attributes_p2a:flags */ +#define SCMI_RESET_DOMAIN_ATTR_ASYNC BIT(31) +#define SCMI_RESET_DOMAIN_ATTR_NOTIF BIT(30) + +/* Value for scmi_reset_domain_attributes_p2a:latency */ +#define SCMI_RESET_DOMAIN_ATTR_UNK_LAT 0x7fffffff +#define SCMI_RESET_DOMAIN_ATTR_MAX_LAT 0x7ffffffe + +/* Macro for scmi_reset_domain_attributes_p2a:name */ +#define SCMI_RESET_DOMAIN_ATTR_NAME_SZ 16 + +struct scmi_reset_domain_attributes_a2p { + uint32_t domain_id; +}; + +struct scmi_reset_domain_attributes_p2a { + int32_t status; + uint32_t flags; + uint32_t latency; + char name[SCMI_RESET_DOMAIN_ATTR_NAME_SZ]; +}; + +/* + * RESET + */ + +/* Values for scmi_reset_domain_request_a2p:flags */ +#define SCMI_RESET_DOMAIN_ASYNC BIT(2) +#define SCMI_RESET_DOMAIN_EXPLICIT BIT(1) +#define SCMI_RESET_DOMAIN_AUTO BIT(0) + +struct scmi_reset_domain_request_a2p { + uint32_t domain_id; + uint32_t flags; + uint32_t reset_state; +}; + +struct scmi_reset_domain_request_p2a { + int32_t status; +}; + +/* + * RESET_NOTIFY + */ + +/* Values for scmi_reset_notify_p2a:flags */ +#define SCMI_RESET_DOMAIN_DO_NOTIFY BIT(0) + +struct scmi_reset_domain_notify_a2p { + uint32_t domain_id; + uint32_t notify_enable; +}; + +struct scmi_reset_domain_notify_p2a { + int32_t status; +}; + +/* + * RESET_COMPLETE + */ + +struct scmi_reset_domain_complete_p2a { + int32_t status; + uint32_t domain_id; +}; + +/* + * RESET_ISSUED + */ + +struct scmi_reset_domain_issued_p2a { + uint32_t domain_id; + uint32_t reset_state; +}; + +#ifdef CFG_SCMI_MSG_RESET_DOMAIN +/* + * scmi_msg_get_rd_handler - Return a handler for a reset domain message + * @msg - message to process + * Return a function handler for the message or NULL + */ +scmi_msg_handler_t scmi_msg_get_rd_handler(struct scmi_msg *msg); +#else +static inline +scmi_msg_handler_t scmi_msg_get_rd_handler(struct scmi_msg *msg __unused) +{ + return NULL; +} +#endif +#endif /* SCMI_MSG_RESET_DOMAIN_H */ diff --git a/optee_os/core/drivers/scmi-msg/shm_msg.c b/optee_os/core/drivers/scmi-msg/shm_msg.c new file mode 100644 index 0000000..3fe8363 --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/shm_msg.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2022, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" + +/** + * struct msg_header - MSG formatted header for MSG base shared memory transfer + * + * @message_header: 32bit header used in MSG shared memory protocol + * @payload: SCMI message payload data + */ +struct msg_header { + uint32_t message_header; + uint32_t payload[]; +}; + +/* Bit fields packed in msg_header::message_header */ +#define MSG_ID_MASK GENMASK_32(7, 0) +#define MSG_ID(_hdr) ((_hdr) & MSG_ID_MASK) + +#define MSG_TYPE_MASK GENMASK_32(9, 8) +#define MSG_TYPE(_hdr) (((_hdr) & MSG_TYPE_MASK) >> 8) + +#define MSG_PROT_ID_MASK GENMASK_32(17, 10) +#define MSG_PROT_ID(_hdr) (((_hdr) & MSG_PROT_ID_MASK) >> 10) + +/* + * Creates a SCMI message instance in secure memory and push it in the SCMI + * message drivers. Message structure contains SCMI protocol meta-data and + * references to input payload in secure memory and output message buffer + * in shared memory. + */ +TEE_Result scmi_entry_msg(unsigned int channel_id, void *in_buf, size_t in_size, + void *out_buf, size_t *out_size, uint32_t *sec_buf) +{ + struct scmi_msg_channel *channel = plat_scmi_get_channel(channel_id); + struct msg_header *hdr = NULL; + struct scmi_msg msg = { }; + uint32_t msg_header = 0; + + if (!channel) { + DMSG("Invalid channel ID %u", channel_id); + return TEE_ERROR_BAD_PARAMETERS; + } + + assert(in_buf && out_buf && out_size && sec_buf); + + if (in_size < sizeof(struct msg_header) || + in_size - sizeof(struct msg_header) > SCMI_SEC_PAYLOAD_SIZE || + !IS_ALIGNED_WITH_TYPE(in_buf, uint32_t) || + *out_size < sizeof(struct msg_header) || + !IS_ALIGNED_WITH_TYPE(out_buf, uint32_t)) { + DMSG("Invalid SCMI buffer references %zu@%p / %zu@%p", + in_size, in_buf, *out_size, out_buf); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!scmi_msg_claim_channel(channel)) { + DMSG("SCMI channel %u busy", channel_id); + return TEE_ERROR_BUSY; + } + + /* Copy SCMI protocol data and message payload in secure memory */ + hdr = (struct msg_header *)in_buf; + msg_header = READ_ONCE(hdr->message_header); + + msg.protocol_id = MSG_PROT_ID(msg_header); + msg.message_id = MSG_ID(msg_header); + msg.channel_id = channel_id; + + msg.in = (char *)sec_buf; + msg.in_size = in_size - sizeof(struct msg_header); + memcpy(msg.in, hdr->payload, msg.in_size); + + /* Prepare output message buffer references */ + hdr = (struct msg_header *)out_buf; + + msg.out = (char *)hdr->payload; + msg.out_size = *out_size - sizeof(struct msg_header); + + scmi_process_message(&msg); + + /* Update SCMI protocol data and output shared buffer size */ + hdr->message_header = msg_header; + *out_size = msg.out_size_out + sizeof(struct msg_header); + + scmi_msg_release_channel(channel); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/scmi-msg/smt.c b/optee_os/core/drivers/scmi-msg/smt.c new file mode 100644 index 0000000..880212b --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/smt.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019-2022, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" + +/** + * struct smt_header - SMT formatted header for SMT base shared memory transfer + * + * @status: Bit flags, see SMT_STATUS_* + * @flags: Bit flags, see SMT_FLAG_* + * @length: Byte size of message payload (variable) + ::message_header (32bit) + * payload: SCMI message payload data + */ +struct smt_header { + uint32_t reserved0; + uint32_t status; + uint64_t reserved1; + uint32_t flags; + uint32_t length; /* message_header + payload */ + uint32_t message_header; + uint32_t payload[]; +}; + +/* Flag set in smt_header::status when SMT does not contain pending message */ +#define SMT_STATUS_FREE BIT(0) +/* Flag set in smt_header::status when SMT reports an error */ +#define SMT_STATUS_ERROR BIT(1) + +/* Flag set in smt_header::flags when SMT uses interrupts */ +#define SMT_FLAG_INTR_ENABLED BIT(1) + +/* Bit fields packed in smt_header::message_header */ +#define SMT_MSG_ID_MASK GENMASK_32(7, 0) +#define SMT_HDR_MSG_ID(_hdr) ((_hdr) & SMT_MSG_ID_MASK) + +#define SMT_MSG_TYPE_MASK GENMASK_32(9, 8) +#define SMT_HDR_TYPE_ID(_hdr) (((_hdr) & SMT_MSG_TYPE_MASK) >> 8) + +#define SMT_MSG_PROT_ID_MASK GENMASK_32(17, 10) +#define SMT_HDR_PROT_ID(_hdr) (((_hdr) & SMT_MSG_PROT_ID_MASK) >> 10) + +static struct smt_header *channel_to_smt_hdr(struct scmi_msg_channel *channel) +{ + if (!channel) + return NULL; + + return (struct smt_header *)io_pa_or_va(&channel->shm_addr, + sizeof(struct smt_header)); +} + +/* + * Creates a SCMI message instance in secure memory and push it in the SCMI + * message drivers. Message structure contains SCMI protocol meta-data and + * references to input payload in secure memory and output message buffer + * in shared memory. + */ +void scmi_entry_smt(unsigned int channel_id, uint32_t *payload_buf) +{ + struct scmi_msg_channel *channel = NULL; + struct smt_header *smt_hdr = NULL; + size_t in_payload_size = 0; + uint32_t smt_status = 0; + struct scmi_msg msg = { }; + bool error = true; + + channel = plat_scmi_get_channel(channel_id); + if (!channel) { + DMSG("Invalid channel ID %u", channel_id); + return; + } + + smt_hdr = channel_to_smt_hdr(channel); + if (!smt_hdr) { + DMSG("No shared buffer for channel ID %u", channel_id); + return; + } + + if (!scmi_msg_claim_channel(channel)) { + DMSG("SCMI channel %u busy", channel_id); + goto out; + } + + smt_status = READ_ONCE(smt_hdr->status); + + in_payload_size = READ_ONCE(smt_hdr->length) - + sizeof(smt_hdr->message_header); + + if (in_payload_size > SCMI_SEC_PAYLOAD_SIZE) { + DMSG("SCMI payload too big %u", in_payload_size); + goto out; + } + + if (smt_status & (SMT_STATUS_ERROR | SMT_STATUS_FREE)) { + DMSG("SCMI channel bad status 0x%x", + smt_hdr->status & (SMT_STATUS_ERROR | SMT_STATUS_FREE)); + goto out; + } + + /* Fill message */ + msg.in = (char *)payload_buf; + msg.in_size = in_payload_size; + msg.out = (char *)smt_hdr->payload; + msg.out_size = channel->shm_size - sizeof(*smt_hdr); + + assert(msg.out && msg.out_size >= sizeof(int32_t)); + + /* Here the payload is copied in secure memory */ + memcpy(msg.in, smt_hdr->payload, in_payload_size); + + msg.protocol_id = SMT_HDR_PROT_ID(smt_hdr->message_header); + msg.message_id = SMT_HDR_MSG_ID(smt_hdr->message_header); + msg.channel_id = channel_id; + + scmi_process_message(&msg); + + /* Update message length with the length of the response message */ + smt_hdr->length = msg.out_size_out + sizeof(smt_hdr->message_header); + + scmi_msg_release_channel(channel); + error = false; + +out: + if (error) { + DMSG("SCMI error"); + smt_hdr->status |= SMT_STATUS_ERROR | SMT_STATUS_FREE; + } else { + smt_hdr->status |= SMT_STATUS_FREE; + } +} + +/* Init a SMT header for a shared memory buffer: state it a free/no-error */ +void scmi_smt_init_agent_channel(struct scmi_msg_channel *channel) +{ + struct smt_header *smt_header = channel_to_smt_hdr(channel); + + static_assert(SCMI_SEC_PAYLOAD_SIZE + sizeof(struct smt_header) <= + SMT_BUF_SLOT_SIZE && + IS_ALIGNED(SCMI_SEC_PAYLOAD_SIZE, sizeof(uint32_t))); + assert(smt_header); + + memset(smt_header, 0, sizeof(*smt_header)); + smt_header->status = SMT_STATUS_FREE; +} + +void scmi_smt_set_shared_buffer(struct scmi_msg_channel *channel, void *base) +{ + paddr_t p_base = 0; + + if (base) { + assert(!channel->shm_addr.va && !channel->shm_addr.pa); + p_base = virt_to_phys(base); + assert(p_base); + } + + channel->shm_addr.va = (vaddr_t)base; + channel->shm_addr.pa = p_base; +} diff --git a/optee_os/core/drivers/scmi-msg/sub.mk b/optee_os/core/drivers/scmi-msg/sub.mk new file mode 100644 index 0000000..9a97d5c --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/sub.mk @@ -0,0 +1,8 @@ +srcs-y += base.c +srcs-$(CFG_SCMI_MSG_CLOCK) += clock.c +srcs-$(CFG_SCMI_MSG_USE_CLK) += clock_generic.c +srcs-y += entry.c +srcs-$(CFG_SCMI_MSG_RESET_DOMAIN) += reset_domain.c +srcs-$(CFG_SCMI_MSG_SHM_MSG) += shm_msg.c +srcs-$(CFG_SCMI_MSG_SMT) += smt.c +srcs-$(CFG_SCMI_MSG_VOLTAGE_DOMAIN) += voltage_domain.c diff --git a/optee_os/core/drivers/scmi-msg/voltage_domain.c b/optee_os/core/drivers/scmi-msg/voltage_domain.c new file mode 100644 index 0000000..a4a211e --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/voltage_domain.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2020, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "voltage_domain.h" + +static bool message_id_is_supported(unsigned int message_id); + +size_t __weak plat_scmi_voltd_count(unsigned int channel_id __unused) +{ + return 0; +} + +const char __weak *plat_scmi_voltd_get_name(unsigned int channel_id __unused, + unsigned int scmi_id __unused) +{ + return NULL; +} + +int32_t __weak plat_scmi_voltd_levels_array(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + size_t start_index __unused, + long *levels __unused, + size_t *nb_elts __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_voltd_levels_by_step(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + long *steps __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_voltd_get_level(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + long *level __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_voltd_set_level(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + long microvolt __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_voltd_get_config(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + uint32_t *config __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +int32_t __weak plat_scmi_voltd_set_config(unsigned int channel_id __unused, + unsigned int scmi_id __unused, + uint32_t config __unused) +{ + return SCMI_NOT_SUPPORTED; +} + +static void report_version(struct scmi_msg *msg) +{ + struct scmi_protocol_version_p2a out_args = { + .status = SCMI_SUCCESS, + .version = SCMI_PROTOCOL_VERSION_VOLTAGE_DOMAIN, + }; + + if (IS_ENABLED(CFG_SCMI_MSG_STRICT_ABI) && msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &out_args, sizeof(out_args)); +} + +static void report_attributes(struct scmi_msg *msg) +{ + size_t domain_count = plat_scmi_voltd_count(msg->channel_id); + struct scmi_protocol_attributes_p2a out_args = { + .status = SCMI_SUCCESS, + .attributes = domain_count, + }; + + assert(!(domain_count & ~SCMI_VOLTAGE_DOMAIN_COUNT_MASK)); + + if (IS_ENABLED(CFG_SCMI_MSG_STRICT_ABI) && msg->in_size) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + scmi_write_response(msg, &out_args, sizeof(out_args)); +} + +static void report_message_attributes(struct scmi_msg *msg) +{ + struct scmi_protocol_message_attributes_a2p *in_args = (void *)msg->in; + struct scmi_protocol_message_attributes_p2a out_args = { + .status = SCMI_SUCCESS, + /* For this protocol, attributes shall be zero */ + .attributes = 0, + }; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (!message_id_is_supported(in_args->message_id)) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + scmi_write_response(msg, &out_args, sizeof(out_args)); +} + +static void scmi_voltd_domain_attributes(struct scmi_msg *msg) +{ + const struct scmi_voltd_attributes_a2p *in_args = (void *)msg->in; + struct scmi_voltd_attributes_p2a out_args = { + .status = SCMI_SUCCESS, + }; + const char *name = NULL; + unsigned int domain_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->domain_id >= plat_scmi_voltd_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_voltd_count(msg->channel_id)); + + name = plat_scmi_voltd_get_name(msg->channel_id, domain_id); + if (!name) { + scmi_status_response(msg, SCMI_NOT_FOUND); + return; + } + + COPY_NAME_IDENTIFIER(out_args.name, name); + + scmi_write_response(msg, &out_args, sizeof(out_args)); +} + +#define LEVELS_BY_ARRAY(_nb_rates, _rem_rates) \ + SCMI_VOLTAGE_DOMAIN_LEVELS_FLAGS((_nb_rates), \ + SCMI_VOLTD_LEVELS_FORMAT_LIST, \ + (_rem_rates)) + +#define LEVELS_BY_STEP \ + SCMI_VOLTAGE_DOMAIN_LEVELS_FLAGS(3, SCMI_VOLTD_LEVELS_FORMAT_RANGE, 0) + +#define LEVEL_DESC_SIZE sizeof(int32_t) + +static void scmi_voltd_describe_levels(struct scmi_msg *msg) +{ + const struct scmi_voltd_describe_levels_a2p *in_args = (void *)msg->in; + struct scmi_voltd_describe_levels_p2a out_args = { }; + int32_t status = SCMI_GENERIC_ERROR; + unsigned int out_count = 0; + unsigned int domain_id = 0; + int32_t *out_levels = NULL; + size_t nb_levels = 0; + + if (msg->in_size != sizeof(*in_args)) { + status = SCMI_PROTOCOL_ERROR; + goto out; + } + + if (in_args->domain_id >= plat_scmi_voltd_count(msg->channel_id)) { + status = SCMI_INVALID_PARAMETERS; + goto out; + } + + if (msg->out_size < sizeof(out_args)) { + status = SCMI_INVALID_PARAMETERS; + goto out; + } + assert(IS_ALIGNED_WITH_TYPE(msg->out + sizeof(out_args), int32_t)); + out_levels = (int32_t *)(uintptr_t)(msg->out + sizeof(out_args)); + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_voltd_count(msg->channel_id)); + + /* Platform may support array rate description */ + status = plat_scmi_voltd_levels_array(msg->channel_id, domain_id, 0, + NULL, &nb_levels); + if (status == SCMI_SUCCESS) { + size_t avail_sz = msg->out_size - sizeof(out_args); + unsigned int level_index = in_args->level_index; + unsigned int remaining = 0; + + if (avail_sz < LEVEL_DESC_SIZE && nb_levels) { + status = SCMI_PROTOCOL_ERROR; + goto out; + } + + while (avail_sz >= LEVEL_DESC_SIZE && level_index < nb_levels) { + long plat_level = 0; + size_t cnt = 1; + + status = plat_scmi_voltd_levels_array(msg->channel_id, + domain_id, + level_index, + &plat_level, + &cnt); + if (status) + goto out; + + *out_levels = plat_level; + + avail_sz -= LEVEL_DESC_SIZE; + out_levels++; + level_index++; + } + + remaining = nb_levels - in_args->level_index; + out_count = level_index - in_args->level_index; + out_args.flags = LEVELS_BY_ARRAY(out_count, remaining); + } else if (status == SCMI_NOT_SUPPORTED) { + long triplet[3] = { 0, 0, 0 }; + + if (msg->out_size < sizeof(out_args) + 3 * LEVEL_DESC_SIZE) { + status = SCMI_PROTOCOL_ERROR; + goto out; + } + + /* Platform may support min/max/step triplet description */ + status = plat_scmi_voltd_levels_by_step(msg->channel_id, + domain_id, triplet); + if (status) + goto out; + + out_levels[0] = triplet[0]; + out_levels[1] = triplet[1]; + out_levels[2] = triplet[2]; + + out_count = 3; + out_args.flags = LEVELS_BY_STEP; + } + +out: + if (status) { + scmi_status_response(msg, status); + } else { + out_args.status = SCMI_SUCCESS; + memcpy(msg->out, &out_args, sizeof(out_args)); + msg->out_size_out = sizeof(out_args) + + out_count * LEVEL_DESC_SIZE; + } +} + +static void scmi_voltd_config_set(struct scmi_msg *msg) +{ + const struct scmi_voltd_config_set_a2p *in_args = (void *)msg->in; + unsigned int domain_id = 0; + unsigned long config = 0; + int32_t status = SCMI_GENERIC_ERROR; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->domain_id >= plat_scmi_voltd_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_voltd_count(msg->channel_id)); + + config = in_args->config & SCMI_VOLTAGE_DOMAIN_CONFIG_MASK; + + status = plat_scmi_voltd_set_config(msg->channel_id, domain_id, config); + + scmi_status_response(msg, status); +} + +static void scmi_voltd_config_get(struct scmi_msg *msg) +{ + const struct scmi_voltd_config_get_a2p *in_args = (void *)msg->in; + struct scmi_voltd_config_get_p2a out_args = { }; + unsigned int domain_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->domain_id >= plat_scmi_voltd_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_voltd_count(msg->channel_id)); + + if (plat_scmi_voltd_get_config(msg->channel_id, domain_id, + &out_args.config)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + scmi_write_response(msg, &out_args, sizeof(out_args)); +} + +static void scmi_voltd_level_set(struct scmi_msg *msg) +{ + const struct scmi_voltd_level_set_a2p *in_args = (void *)msg->in; + int32_t status = SCMI_GENERIC_ERROR; + unsigned int domain_id = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->domain_id >= plat_scmi_voltd_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_voltd_count(msg->channel_id)); + + status = plat_scmi_voltd_set_level(msg->channel_id, domain_id, + in_args->voltage_level); + + scmi_status_response(msg, status); +} + +static void scmi_voltd_level_get(struct scmi_msg *msg) +{ + const struct scmi_voltd_level_get_a2p *in_args = (void *)msg->in; + struct scmi_voltd_level_get_p2a out_args = { + .status = SCMI_SUCCESS, + }; + unsigned int domain_id = 0; + long level = 0; + + if (msg->in_size != sizeof(*in_args)) { + scmi_status_response(msg, SCMI_PROTOCOL_ERROR); + return; + } + + if (in_args->domain_id >= plat_scmi_voltd_count(msg->channel_id)) { + scmi_status_response(msg, SCMI_INVALID_PARAMETERS); + return; + } + + domain_id = confine_array_index(in_args->domain_id, + plat_scmi_voltd_count(msg->channel_id)); + + out_args.status = plat_scmi_voltd_get_level(msg->channel_id, domain_id, + &level); + out_args.voltage_level = level; + + scmi_write_response(msg, &out_args, sizeof(out_args)); +} + +static const scmi_msg_handler_t handler_array[] = { + [SCMI_PROTOCOL_VERSION] = report_version, + [SCMI_PROTOCOL_ATTRIBUTES] = report_attributes, + [SCMI_PROTOCOL_MESSAGE_ATTRIBUTES] = report_message_attributes, + [SCMI_VOLTAGE_DOMAIN_ATTRIBUTES] = scmi_voltd_domain_attributes, + [SCMI_VOLTAGE_DESCRIBE_LEVELS] = scmi_voltd_describe_levels, + [SCMI_VOLTAGE_CONFIG_SET] = scmi_voltd_config_set, + [SCMI_VOLTAGE_CONFIG_GET] = scmi_voltd_config_get, + [SCMI_VOLTAGE_LEVEL_SET] = scmi_voltd_level_set, + [SCMI_VOLTAGE_LEVEL_GET] = scmi_voltd_level_get, +}; + +static bool message_id_is_supported(unsigned int id) +{ + return id < ARRAY_SIZE(handler_array) && handler_array[id]; +} + +scmi_msg_handler_t scmi_msg_get_voltd_handler(struct scmi_msg *msg) +{ + const size_t array_size = ARRAY_SIZE(handler_array); + unsigned int message_id = 0; + + if (msg->message_id >= array_size) { + DMSG("Voltage domain handle not found %u", msg->message_id); + return NULL; + } + + message_id = confine_array_index(msg->message_id, array_size); + + return handler_array[message_id]; +} diff --git a/optee_os/core/drivers/scmi-msg/voltage_domain.h b/optee_os/core/drivers/scmi-msg/voltage_domain.h new file mode 100644 index 0000000..be53cb6 --- /dev/null +++ b/optee_os/core/drivers/scmi-msg/voltage_domain.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef SCMI_MSG_VOLTAGE_DOMAIN_H +#define SCMI_MSG_VOLTAGE_DOMAIN_H + +#include +#include + +#include "common.h" + +#define SCMI_PROTOCOL_VERSION_VOLTAGE_DOMAIN 0x30000 + +/* + * Identifiers of the SCMI Clock Management Protocol commands + */ +enum scmi_voltd_command_id { + SCMI_VOLTAGE_DOMAIN_ATTRIBUTES = 0x3, + SCMI_VOLTAGE_DESCRIBE_LEVELS = 0x4, + SCMI_VOLTAGE_CONFIG_SET = 0x5, + SCMI_VOLTAGE_CONFIG_GET = 0x6, + SCMI_VOLTAGE_LEVEL_SET = 0x7, + SCMI_VOLTAGE_LEVEL_GET = 0x8, +}; + +#define SCMI_VOLTAGE_DOMAIN_COUNT_MASK GENMASK_32(15, 0) + +struct scmi_voltd_protocol_attrs_p2a { + int32_t status; + uint32_t attributes; +}; + +struct scmi_voltd_attributes_a2p { + uint32_t domain_id; +}; + +#define SCMI_VOLTAGE_DOMAIN_NAME_MAX 16 + +struct scmi_voltd_attributes_p2a { + int32_t status; + uint32_t attributes; + char name[SCMI_VOLTAGE_DOMAIN_NAME_MAX]; +}; + +struct scmi_voltd_describe_levels_a2p { + uint32_t domain_id; + uint32_t level_index; +}; + +#define SCMI_VOLTD_LEVELS_REMAINING_MASK GENMASK_32(31, 16) +#define SCMI_VOLTD_LEVELS_REMAINING_POS 16 + +#define SCMI_VOLTD_LEVELS_FORMAT_RANGE 1 +#define SCMI_VOLTD_LEVELS_FORMAT_LIST 0 +#define SCMI_VOLTD_LEVELS_FORMAT_MASK BIT(12) +#define SCMI_VOLTD_LEVELS_FORMAT_POS 12 + +#define SCMI_VOLTD_LEVELS_COUNT_MASK GENMASK_32(11, 0) + +#define SCMI_VOLTAGE_DOMAIN_LEVELS_FLAGS(_count, _fmt, _rem_rates) \ + ( \ + ((_count) & SCMI_VOLTD_LEVELS_COUNT_MASK) | \ + (((_rem_rates) << SCMI_VOLTD_LEVELS_REMAINING_POS) & \ + SCMI_VOLTD_LEVELS_REMAINING_MASK) | \ + (((_fmt) << SCMI_VOLTD_LEVELS_FORMAT_POS) & \ + SCMI_VOLTD_LEVELS_FORMAT_MASK) \ + ) + +struct scmi_voltd_level { + int32_t mircovolt; +}; + +struct scmi_voltd_describe_levels_p2a { + int32_t status; + uint32_t flags; + struct scmi_voltd_level voltage[]; +}; + +struct scmi_voltd_level_set_a2p { + uint32_t domain_id; + uint32_t flags; + int32_t voltage_level; +}; + +struct scmi_voltd_level_set_p2a { + uint32_t status; +}; + +struct scmi_voltd_level_get_a2p { + uint32_t domain_id; +}; + +struct scmi_voltd_level_get_p2a { + int32_t status; + int32_t voltage_level; +}; + +#define SCMI_VOLTAGE_DOMAIN_CONFIG_MASK GENMASK_32(3, 0) + +struct scmi_voltd_config_set_a2p { + uint32_t domain_id; + uint32_t config; +}; + +struct scmi_voltd_config_set_p2a { + uint32_t status; +}; + +struct scmi_voltd_config_get_a2p { + uint32_t domain_id; +}; + +struct scmi_voltd_config_get_p2a { + int32_t status; + uint32_t config; +}; + +#ifdef CFG_SCMI_MSG_VOLTAGE_DOMAIN +/* + * scmi_msg_get_voltd_handler - Return a handler for a voltage domain message + * @msg - message to process + * Return a function handler for the message or NULL + */ +scmi_msg_handler_t scmi_msg_get_voltd_handler(struct scmi_msg *msg); +#else +static inline +scmi_msg_handler_t scmi_msg_get_voltd_handler(struct scmi_msg *msg __unused) +{ + return NULL; +} +#endif +#endif /* SCMI_MSG_CLOCK_H */ diff --git a/optee_os/core/drivers/serial8250_uart.c b/optee_os/core/drivers/serial8250_uart.c new file mode 100644 index 0000000..dddd3e6 --- /dev/null +++ b/optee_os/core/drivers/serial8250_uart.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* uart register defines */ +#define UART_RHR 0x0 +#define UART_THR 0x0 +#define UART_IER 0x4 +#define UART_ISR 0x8 +#define UART_FCR 0x8 +#define UART_LCR 0xc +#define UART_MCR 0x10 +#define UART_LSR 0x14 +#define UART_MSR 0x18 +#define UART_SPR 0x1c + +/* uart status register bits */ +#define LSR_TEMT 0x40 /* Transmitter empty */ +#define LSR_THRE 0x20 /* Transmit-hold-register empty */ +#define LSR_EMPTY (LSR_TEMT | LSR_THRE) +#define LSR_DR 0x01 /* DATA Ready */ + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct serial8250_uart_data *pd = + container_of(chip, struct serial8250_uart_data, chip); + + return io_pa_or_va(&pd->base, SERIAL8250_UART_REG_SIZE); +} + +static void serial8250_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (1) { + uint32_t state = io_read32(base + UART_LSR); + + /* Wait until transmit FIFO is empty */ + if ((state & LSR_EMPTY) == LSR_EMPTY) + break; + } +} + +static bool serial8250_uart_have_rx_data(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + return (io_read32(base + UART_LSR) & LSR_DR); +} + +static int serial8250_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!serial8250_uart_have_rx_data(chip)) { + /* Transmit FIFO is empty, waiting again */ + ; + } + return io_read32(base + UART_RHR) & 0xff; +} + +static void serial8250_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + serial8250_uart_flush(chip); + + /* Write out character to transmit FIFO */ + io_write32(base + UART_THR, ch); +} + +static const struct serial_ops serial8250_uart_ops = { + .flush = serial8250_uart_flush, + .getchar = serial8250_uart_getchar, + .have_rx_data = serial8250_uart_have_rx_data, + .putc = serial8250_uart_putc, +}; +DECLARE_KEEP_PAGER(serial8250_uart_ops); + +void serial8250_uart_init(struct serial8250_uart_data *pd, paddr_t base, + uint32_t __unused uart_clk, + uint32_t __unused baud_rate) + +{ + pd->base.pa = base; + pd->chip.ops = &serial8250_uart_ops; + + /* + * do nothing, debug uart(uart0) share with normal world, + * everything for uart0 is ready now. + */ +} + +#ifdef CFG_DT + +static struct serial_chip *serial8250_uart_dev_alloc(void) +{ + struct serial8250_uart_data *pd = calloc(1, sizeof(*pd)); + + if (!pd) + return NULL; + return &pd->chip; +} + +static int serial8250_uart_dev_init(struct serial_chip *chip, + const void *fdt, + int offs, + const char *parms) +{ + struct serial8250_uart_data *pd = + container_of(chip, struct serial8250_uart_data, chip); + vaddr_t vbase; + paddr_t pbase; + size_t size; + + if (parms && parms[0]) + IMSG("serial8250_uart: device parameters ignored (%s)", parms); + + if (dt_map_dev(fdt, offs, &vbase, &size, DT_MAP_AUTO) < 0) + return -1; + + if (size < SERIAL8250_UART_REG_SIZE) { + EMSG("serial8250_uart: register size too small: %zx", size); + return -1; + } + + pbase = virt_to_phys((void *)vbase); + serial8250_uart_init(pd, pbase, 0, 0); + + return 0; +} + +static void serial8250_uart_dev_free(struct serial_chip *chip) +{ + struct serial8250_uart_data *pd = + container_of(chip, struct serial8250_uart_data, chip); + + free(pd); +} + +static const struct serial_driver serial8250_driver = { + .dev_alloc = serial8250_uart_dev_alloc, + .dev_init = serial8250_uart_dev_init, + .dev_free = serial8250_uart_dev_free, +}; + +static const struct dt_device_match serial8250_match_table[] = { + { .compatible = "snps,dw-apb-uart" }, + { 0 } +}; + +DEFINE_DT_DRIVER(serial8250_dt_driver) = { + .name = "serial8250_uart", + .type = DT_DRIVER_UART, + .match_table = serial8250_match_table, + .driver = &serial8250_driver, +}; + +#endif /* CFG_DT */ diff --git a/optee_os/core/drivers/smccc_trng.c b/optee_os/core/drivers/smccc_trng.c new file mode 100644 index 0000000..4d28ff0 --- /dev/null +++ b/optee_os/core/drivers/smccc_trng.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2022 Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Arm SMCCC TRNG firmware interface specification: + * https://developer.arm.com/documentation/den0098/ + */ +#define ARM_SMCCC_TRNG_VERSION 0x84000050 +#define ARM_SMCCC_TRNG_FEATURES 0x84000051 +#define ARM_SMCCC_TRNG_GET_UUID 0x84000052 +#define ARM_SMCCC_TRNG_RND_32 0x84000053 +#define ARM_SMCCC_TRNG_RND_64 0xc4000053 + +#define ARM_SMCCC_RET_TRNG_SUCCESS U(0) +#define ARM_SMCCC_RET_TRNG_NOT_SUPPORTED ((unsigned long)-1) +#define ARM_SMCCC_RET_TRNG_INVALID_PARAMETER ((unsigned long)-2) +#define ARM_SMCCC_RET_TRNG_NO_ENTROPY ((unsigned long)-3) + +#define TRNG_MAJOR_MASK GENMASK_32(30, 16) +#define TRNG_MAJOR_SHIFT 16 +#define TRNG_MINOR_MASK GENMASK_32(15, 0) +#define TRNG_MINOR_SHIFT 0 +#define TRNG_MAKE_VERSION(major, minor) \ + ((SHIFT_U32(major, TRNG_MAJOR_SHIFT) & TRNG_MAJOR_MASK) | \ + (SHIFT_U32(minor, TRNG_MINOR_SHIFT) & TRNG_MINOR_MASK)) + +#define TRNG_VERSION_1_0 TRNG_MAKE_VERSION(1, 0) + +#define TRNG_MAX_RND_64 (192 / 8) +#define TRNG_MAX_RND_32 (96 / 8) + +/* Function ID discovered for getting random bytes or 0 if not supported */ +static uint32_t trng_rnd_fid; + +static bool smccc_trng_is_supported(void) +{ + struct thread_smc_args args = { }; + static bool inited; + + if (inited) + return trng_rnd_fid != 0; + + inited = true; + + /* + * TRNG ABI requires caller to check that Arm SMCCC version is + * larger or equal to v1.1 + */ + args.a0 = ARM_SMCCC_VERSION; + thread_smccc(&args); + if (args.a0 & BIT32(31) || args.a0 < SMCCC_V_1_1) + return false; + + /* + * Check TRNG version, if successful we're guaranteed to have at least + * the ARM_SMCCC_TRNG_FEATURES fid. + */ + args.a0 = ARM_SMCCC_TRNG_VERSION; + thread_smccc(&args); + if (args.a0 & BIT32(31) || args.a0 < TRNG_VERSION_1_0) + return false; + +#ifdef ARM64 + args.a0 = ARM_SMCCC_TRNG_FEATURES; + args.a1 = ARM_SMCCC_TRNG_RND_64; + thread_smccc(&args); + if (args.a0 == ARM_SMCCC_RET_SUCCESS) { + trng_rnd_fid = ARM_SMCCC_TRNG_RND_64; + return true; + } +#endif + + args.a0 = ARM_SMCCC_TRNG_FEATURES; + args.a1 = ARM_SMCCC_TRNG_RND_32; + thread_smccc(&args); + if (args.a0 == ARM_SMCCC_RET_TRNG_SUCCESS) { + trng_rnd_fid = ARM_SMCCC_TRNG_RND_32; + return true; + } + + return false; +} + +static void read_bytes(unsigned long val, size_t byte_count, uint8_t **buf, + size_t *rem) +{ + size_t count = MIN(byte_count, *rem); + size_t n = 0; + + for (n = 0; n < count; n++) + (*buf)[n] = val >> (n * 8); + + *buf += count; + *rem -= count; +} + +static void read_samples(struct thread_smc_args *args, uint8_t *buf, size_t len) +{ + uint8_t *ptr = buf; + size_t rem = len; + size_t byte_count = 4; + +#ifdef ARM64 + if (trng_rnd_fid == ARM_SMCCC_TRNG_RND_64) + byte_count = 8; +#endif + + read_bytes(args->a3, byte_count, &ptr, &rem); + read_bytes(args->a2, byte_count, &ptr, &rem); + read_bytes(args->a1, byte_count, &ptr, &rem); +} + +static TEE_Result __maybe_unused smccc_trng_read(void *buf, size_t len) +{ + struct thread_smc_args args = { }; + uint8_t *ptr = buf; + size_t rem = len; + size_t max_burst = 0; + + if (!smccc_trng_is_supported()) + return TEE_ERROR_NOT_SUPPORTED; + + if (trng_rnd_fid == ARM_SMCCC_TRNG_RND_64) + max_burst = TRNG_MAX_RND_64; + else + max_burst = TRNG_MAX_RND_32; + + while (rem) { + size_t burst = MIN(rem, max_burst); + + args.a0 = trng_rnd_fid; + args.a1 = burst * 8; + + thread_smccc(&args); + + switch (args.a0) { + case ARM_SMCCC_RET_TRNG_SUCCESS: + read_samples(&args, ptr, burst); + rem -= burst; + ptr += burst; + break; + case ARM_SMCCC_RET_TRNG_NO_ENTROPY: + break; + default: + return TEE_ERROR_GENERIC; + } + } + + return TEE_SUCCESS; +} + +static void __maybe_unused smccc_trng_print_info(void) +{ + struct thread_smc_args args = { }; + unsigned int __maybe_unused major = 0; + unsigned int __maybe_unused minor = 0; + + if (!IS_ENABLED(CFG_TEE_CORE_DEBUG)) + return; + + args.a0 = ARM_SMCCC_TRNG_VERSION; + thread_smccc(&args); + assert((args.a0 & BIT32(31)) == 0); + major = (args.a0 & TRNG_MAJOR_MASK) >> TRNG_MAJOR_SHIFT; + minor = (args.a0 & TRNG_MINOR_MASK) >> TRNG_MINOR_SHIFT; + + args.a0 = ARM_SMCCC_TRNG_GET_UUID; + thread_smccc(&args); + assert(args.a0 != ARM_SMCCC_RET_TRNG_NOT_SUPPORTED); + + DMSG("SMCCC TRNG v%u.%u, UUID %08lx-%04lx-%04lx-%04lx-%04lx%08lx\n", + major, minor, (unsigned long)args.a0, (unsigned long)args.a1 >> 16, + (unsigned long)args.a1 & GENMASK_32(16, 0), + (unsigned long)args.a2 >> 16, + (unsigned long)args.a2 & GENMASK_32(16, 0), + (unsigned long)args.a3); +} + +void plat_rng_init(void) +{ + if (!smccc_trng_is_supported()) + panic("SMCCC TRNG not supported"); + + smccc_trng_print_info(); + + if (IS_ENABLED(CFG_WITH_SOFTWARE_PRNG)) { + /* If CFG_WITH_SOFTWARE_PRNG is enabled, seed PRNG with TRNG */ + uint8_t seed[32] = { 0 }; + + if (smccc_trng_read(seed, sizeof(seed))) + panic("SMCCC TRNG not supported"); + + if (crypto_rng_init(seed, sizeof(seed))) + panic(); + } +} + +/* If CFG_WITH_SOFTWARE_PRNG is disabled, TRNG is our HW RNG */ +#ifndef CFG_WITH_SOFTWARE_PRNG +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + return smccc_trng_read(buf, len); +} +#endif diff --git a/optee_os/core/drivers/sp805_wdt.c b/optee_os/core/drivers/sp805_wdt.c new file mode 100644 index 0000000..048620c --- /dev/null +++ b/optee_os/core/drivers/sp805_wdt.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Broadcom + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static vaddr_t chip_to_base(struct wdt_chip *chip) +{ + struct sp805_wdt_data *pd = + container_of(chip, struct sp805_wdt_data, chip); + + return io_pa_or_va(&pd->base, WDT_SIZE); +} + +static TEE_Result sp805_setload(struct wdt_chip *chip, unsigned long timeout) +{ + struct sp805_wdt_data *pd = + container_of(chip, struct sp805_wdt_data, chip); + uint32_t load = 0; + + /* + * sp805 runs counter with given value twice, after the end of first + * counter it gives an interrupt and then starts counter again. If + * interrupt already occurred then it resets the system. This is why + * load is half of what should be required. + */ + if (MUL_OVERFLOW(timeout, pd->clk_rate, &load)) + return TEE_ERROR_SECURITY; + + load = (load / 2) - 1; + if (load < WDT_LOAD_MIN) + load = WDT_LOAD_MIN; + + pd->load_val = load; + return TEE_SUCCESS; +} + +static void sp805_config(struct wdt_chip *chip, bool enable) +{ + struct sp805_wdt_data *pd = + container_of(chip, struct sp805_wdt_data, chip); + vaddr_t base = chip_to_base(chip); + + io_write32(base + WDT_LOCK_OFFSET, WDT_UNLOCK_KEY); + io_write32(base + WDT_LOAD_OFFSET, pd->load_val); + io_write32(base + WDT_INTCLR_OFFSET, WDT_INT_CLR); + + if (enable) + io_write32(base + WDT_CONTROL_OFFSET, + WDT_INT_EN | WDT_RESET_EN); + + io_write32(base + WDT_LOCK_OFFSET, WDT_LOCK_KEY); + + /* Flush posted writes. */ + (void)io_read32(base + WDT_LOCK_OFFSET); +} + +static void sp805_ping(struct wdt_chip *chip) +{ + sp805_config(chip, false); +} + +static void sp805_enable(struct wdt_chip *chip) +{ + sp805_config(chip, true); +} + +static void sp805_disable(struct wdt_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + io_write32(base + WDT_LOCK_OFFSET, WDT_UNLOCK_KEY); + io_write32(base + WDT_CONTROL_OFFSET, 0); + io_write32(base + WDT_LOCK_OFFSET, WDT_LOCK_KEY); + + /* Flush posted writes. */ + (void)io_read32(base + WDT_LOCK_OFFSET); +} + +static enum itr_return wdt_itr_cb(struct itr_handler *h) +{ + struct wdt_chip *chip = h->data; + struct sp805_wdt_data *pd = + container_of(chip, struct sp805_wdt_data, chip); + + if (pd->itr_handler) + pd->itr_handler(chip); + + return ITRR_HANDLED; +} +DECLARE_KEEP_PAGER(wdt_itr_cb); + +TEE_Result sp805_register_itr_handler(struct sp805_wdt_data *pd, + uint32_t itr_num, uint32_t itr_flags, + sp805_itr_handler_func_t itr_handler) +{ + struct itr_handler *wdt_itr; + + assert(!pd->chip.wdt_itr); + + wdt_itr = itr_alloc_add(itr_num, wdt_itr_cb, + itr_flags, &pd->chip); + if (!wdt_itr) + return TEE_ERROR_OUT_OF_MEMORY; + + pd->itr_handler = itr_handler; + pd->chip.wdt_itr = wdt_itr; + + itr_enable(wdt_itr->it); + + return TEE_SUCCESS; +} + +static const struct wdt_ops sp805_wdt_ops = { + .start = sp805_enable, + .stop = sp805_disable, + .ping = sp805_ping, + .set_timeout = sp805_setload, +}; +DECLARE_KEEP_PAGER(sp805_wdt_ops); + +TEE_Result sp805_wdt_init(struct sp805_wdt_data *pd, paddr_t base, + uint32_t clk_rate, uint32_t timeout) +{ + assert(pd); + pd->base.pa = base; + pd->clk_rate = clk_rate; + pd->chip.ops = &sp805_wdt_ops; + return sp805_setload(&pd->chip, timeout); +} diff --git a/optee_os/core/drivers/sprd_uart.c b/optee_os/core/drivers/sprd_uart.c new file mode 100644 index 0000000..a30fd4e --- /dev/null +++ b/optee_os/core/drivers/sprd_uart.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Spreadtrum Communications Inc. + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/* Register definitions */ +#define UART_TXD 0x0000 +#define UART_RXD 0x0004 +#define UART_STS1 0x000C /* data number in TX and RX fifo */ +#define UART_SIZE 0x0010 + +/* Register Bit Fields*/ +#define STS1_RXF_CNT_MASK 0x00ff /* Rx FIFO data counter mask */ +#define STS1_TXF_CNT_MASK 0xff00 /* Tx FIFO data counter mask */ + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct sprd_uart_data *pd = + container_of(chip, struct sprd_uart_data, chip); + + return io_pa_or_va(&pd->base, UART_SIZE); +} + +static void sprd_uart_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (io_read32(base + UART_STS1) & STS1_TXF_CNT_MASK) + ; +} + +static bool sprd_uart_have_rx_data(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + return !!(io_read32(base + UART_STS1) & STS1_RXF_CNT_MASK); +} + +static void sprd_uart_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + sprd_uart_flush(chip); + io_write32(base + UART_TXD, ch); +} + +static int sprd_uart_getchar(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!sprd_uart_have_rx_data(chip)) + ; + + return io_read32(base + UART_RXD) & 0xff; +} + +static const struct serial_ops sprd_uart_ops = { + .flush = sprd_uart_flush, + .getchar = sprd_uart_getchar, + .have_rx_data = sprd_uart_have_rx_data, + .putc = sprd_uart_putc, +}; +DECLARE_KEEP_PAGER(sprd_uart_ops); + +void sprd_uart_init(struct sprd_uart_data *pd, paddr_t base) +{ + pd->base.pa = base; + pd->chip.ops = &sprd_uart_ops; +} diff --git a/optee_os/core/drivers/stih_asc.c b/optee_os/core/drivers/stih_asc.c new file mode 100644 index 0000000..72ebffc --- /dev/null +++ b/optee_os/core/drivers/stih_asc.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ +#include +#include +#include +#include + +#define ASC_BAUDRATE 0x00 +#define ASC_TXBUFFER 0x04 +#define ASC_STATUS 0x14 + +#define ASC_STATUS_TX_EMPTY BIT(1) +#define ASC_STATUS_TX_HALF_EMPTY BIT(2) + +static vaddr_t chip_to_base(struct serial_chip *chip) +{ + struct stih_asc_pd *pd = + container_of(chip, struct stih_asc_pd, chip); + + return io_pa_or_va(&pd->base, STIH_ASC_REG_SIZE); +} + +static void stih_asc_flush(struct serial_chip *chip) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + ASC_STATUS) & ASC_STATUS_TX_EMPTY)) + ; +} + +static void stih_asc_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = chip_to_base(chip); + + while (!(io_read32(base + ASC_STATUS) & ASC_STATUS_TX_HALF_EMPTY)) + ; + + io_write32(base + ASC_TXBUFFER, ch); +} + +static const struct serial_ops stih_asc_ops = { + .flush = stih_asc_flush, + .putc = stih_asc_putc, +}; +DECLARE_KEEP_PAGER(stih_asc_ops); + +void stih_asc_init(struct stih_asc_pd *pd, vaddr_t base) +{ + pd->base.pa = base; + pd->chip.ops = &stih_asc_ops; +} diff --git a/optee_os/core/drivers/stm32_bsec.c b/optee_os/core/drivers/stm32_bsec.c new file mode 100644 index 0000000..4a87969 --- /dev/null +++ b/optee_os/core/drivers/stm32_bsec.c @@ -0,0 +1,956 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2017-2021, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_STM32MP13 +#define DT_BSEC_COMPAT "st,stm32mp13-bsec" +#endif +#ifdef CFG_STM32MP15 +#define DT_BSEC_COMPAT "st,stm32mp15-bsec" +#endif + +#define BSEC_OTP_MASK GENMASK_32(4, 0) +#define BSEC_OTP_BANK_SHIFT U(5) + +/* Permanent lock bitmasks */ +#define DATA_LOWER_OTP_PERLOCK_BIT U(3) +#define DATA_UPPER_OTP_PERLOCK_BIT U(1) + +/* BSEC register offset */ +#define BSEC_OTP_CONF_OFF U(0x000) +#define BSEC_OTP_CTRL_OFF U(0x004) +#define BSEC_OTP_WRDATA_OFF U(0x008) +#define BSEC_OTP_STATUS_OFF U(0x00C) +#define BSEC_OTP_LOCK_OFF U(0x010) +#define BSEC_DEN_OFF U(0x014) +#define BSEC_FEN_OFF U(0x018) +#define BSEC_DISTURBED_OFF U(0x01C) +#define BSEC_DISTURBED1_OFF U(0x020) +#define BSEC_DISTURBED2_OFF U(0x024) +#define BSEC_ERROR_OFF U(0x034) +#define BSEC_ERROR1_OFF U(0x038) +#define BSEC_ERROR2_OFF U(0x03C) +#define BSEC_WRLOCK_OFF U(0x04C) +#define BSEC_WRLOCK1_OFF U(0x050) +#define BSEC_WRLOCK2_OFF U(0x054) +#define BSEC_SPLOCK_OFF U(0x064) +#define BSEC_SPLOCK1_OFF U(0x068) +#define BSEC_SPLOCK2_OFF U(0x06C) +#define BSEC_SWLOCK_OFF U(0x07C) +#define BSEC_SWLOCK1_OFF U(0x080) +#define BSEC_SWLOCK2_OFF U(0x084) +#define BSEC_SRLOCK_OFF U(0x094) +#define BSEC_SRLOCK1_OFF U(0x098) +#define BSEC_SRLOCK2_OFF U(0x09C) +#define BSEC_JTAG_IN_OFF U(0x0AC) +#define BSEC_JTAG_OUT_OFF U(0x0B0) +#define BSEC_SCRATCH_OFF U(0x0B4) +#define BSEC_OTP_DATA_OFF U(0x200) +#define BSEC_IPHW_CFG_OFF U(0xFF0) +#define BSEC_IPVR_OFF U(0xFF4) +#define BSEC_IP_ID_OFF U(0xFF8) +#define BSEC_IP_MAGIC_ID_OFF U(0xFFC) + +/* BSEC_CONFIGURATION Register */ +#define BSEC_CONF_POWER_UP_MASK BIT(0) +#define BSEC_CONF_POWER_UP_SHIFT U(0) +#define BSEC_CONF_FRQ_MASK GENMASK_32(2, 1) +#define BSEC_CONF_FRQ_SHIFT U(1) +#define BSEC_CONF_PRG_WIDTH_MASK GENMASK_32(6, 3) +#define BSEC_CONF_PRG_WIDTH_SHIFT U(3) +#define BSEC_CONF_TREAD_MASK GENMASK_32(8, 7) +#define BSEC_CONF_TREAD_SHIFT U(7) + +/* BSEC_CONTROL Register */ +#define BSEC_READ U(0x000) +#define BSEC_WRITE U(0x100) +#define BSEC_LOCK U(0x200) + +/* BSEC_STATUS Register */ +#define BSEC_MODE_SECURED BIT(0) +#define BSEC_MODE_INVALID BIT(2) +#define BSEC_MODE_BUSY BIT(3) +#define BSEC_MODE_PROGFAIL BIT(4) +#define BSEC_MODE_PWR BIT(5) +#define BSEC_MODE_CLOSED BIT(8) + +/* BSEC_DEBUG bitfields */ +#ifdef CFG_STM32MP13 +#define BSEC_DEN_ALL_MSK (GENMASK_32(11, 10) | GENMASK_32(8, 1)) +#endif +#ifdef CFG_STM32MP15 +#define BSEC_DEN_ALL_MSK GENMASK_32(11, 1) +#endif + +/* + * OTP Lock services definition + * Value must corresponding to the bit position in the register + */ +#define BSEC_LOCK_UPPER_OTP U(0x00) +#define BSEC_LOCK_DEBUG U(0x02) +#define BSEC_LOCK_PROGRAM U(0x04) + +/* Timeout when polling on status */ +#define BSEC_TIMEOUT_US U(10000) + +struct bsec_dev { + struct io_pa_va base; + unsigned int upper_base; + unsigned int max_id; + uint32_t *nsec_access; +}; + +/* Only 1 instance of BSEC is expected per platform */ +static struct bsec_dev bsec_dev; + +/* BSEC access protection */ +static unsigned int lock = SPINLOCK_UNLOCK; + +static uint32_t bsec_lock(void) +{ + return may_spin_lock(&lock); +} + +static void bsec_unlock(uint32_t exceptions) +{ + may_spin_unlock(&lock, exceptions); +} + +static uint32_t otp_max_id(void) +{ + return bsec_dev.max_id; +} + +static uint32_t otp_upper_base(void) +{ + return bsec_dev.upper_base; +} + +static uint32_t otp_bank_offset(uint32_t otp_id) +{ + assert(otp_id <= otp_max_id()); + + return ((otp_id & ~BSEC_OTP_MASK) >> BSEC_OTP_BANK_SHIFT) * + sizeof(uint32_t); +} + +static vaddr_t bsec_base(void) +{ + return io_pa_or_va_secure(&bsec_dev.base, BSEC_IP_MAGIC_ID_OFF + 1); +} + +static uint32_t bsec_status(void) +{ + return io_read32(bsec_base() + BSEC_OTP_STATUS_OFF); +} + +static bool state_is_invalid_mode(void) +{ + return bsec_status() & BSEC_MODE_INVALID; +} + +static bool state_is_secured_mode(void) +{ + return bsec_status() & BSEC_MODE_SECURED; +} + +static bool state_is_closed_mode(void) +{ + uint32_t otp_cfg = 0; + uint32_t close_mode = 0; + TEE_Result res = TEE_ERROR_GENERIC; + size_t __maybe_unused sz = 0; + uint8_t __maybe_unused offset = 0; + + if (IS_ENABLED(CFG_STM32MP13)) + return bsec_status() & BSEC_MODE_CLOSED; + + res = stm32_bsec_find_otp_in_nvmem_layout("cfg0_otp", &otp_cfg, + &offset, &sz); + if (res || sz != 8 || offset) + panic("CFG0 OTP not found or invalid"); + + if (stm32_bsec_read_otp(&close_mode, otp_cfg)) + panic("Unable to read OTP"); + + return close_mode & CFG0_OTP_CLOSED_DEVICE; +} + +/* + * Check that BSEC interface does not report an error + * @otp_id : OTP number + * @check_disturbed: check only error (false) or all sources (true) + * Return a TEE_Result compliant value + */ +static TEE_Result check_no_error(uint32_t otp_id, bool check_disturbed) +{ + uint32_t bit = BIT(otp_id & BSEC_OTP_MASK); + uint32_t bank = otp_bank_offset(otp_id); + + if (io_read32(bsec_base() + BSEC_ERROR_OFF + bank) & bit) + return TEE_ERROR_GENERIC; + + if (check_disturbed && + io_read32(bsec_base() + BSEC_DISTURBED_OFF + bank) & bit) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result power_up_safmem(void) +{ + uint64_t timeout_ref = 0; + + io_mask32(bsec_base() + BSEC_OTP_CONF_OFF, BSEC_CONF_POWER_UP_MASK, + BSEC_CONF_POWER_UP_MASK); + + timeout_ref = timeout_init_us(BSEC_TIMEOUT_US); + while (!timeout_elapsed(timeout_ref)) + if (bsec_status() & BSEC_MODE_PWR) + break; + + if (bsec_status() & BSEC_MODE_PWR) + return TEE_SUCCESS; + + return TEE_ERROR_GENERIC; +} + +static TEE_Result power_down_safmem(void) +{ + uint64_t timeout_ref = 0; + + io_mask32(bsec_base() + BSEC_OTP_CONF_OFF, 0, BSEC_CONF_POWER_UP_MASK); + + timeout_ref = timeout_init_us(BSEC_TIMEOUT_US); + while (!timeout_elapsed(timeout_ref)) + if (!(bsec_status() & BSEC_MODE_PWR)) + break; + + if (!(bsec_status() & BSEC_MODE_PWR)) + return TEE_SUCCESS; + + return TEE_ERROR_GENERIC; +} + +TEE_Result stm32_bsec_shadow_register(uint32_t otp_id) +{ + TEE_Result result = 0; + uint32_t exceptions = 0; + uint64_t timeout_ref = 0; + bool locked = false; + + /* Check if shadowing of OTP is locked, informative only */ + result = stm32_bsec_read_sr_lock(otp_id, &locked); + if (result) + return result; + + if (locked) + DMSG("BSEC shadow warning: OTP locked"); + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + exceptions = bsec_lock(); + + result = power_up_safmem(); + if (result) + goto out; + + io_write32(bsec_base() + BSEC_OTP_CTRL_OFF, otp_id | BSEC_READ); + + timeout_ref = timeout_init_us(BSEC_TIMEOUT_US); + while (!timeout_elapsed(timeout_ref)) + if (!(bsec_status() & BSEC_MODE_BUSY)) + break; + + if (bsec_status() & BSEC_MODE_BUSY) + result = TEE_ERROR_BUSY; + else + result = check_no_error(otp_id, true /* check-disturbed */); + + power_down_safmem(); + +out: + bsec_unlock(exceptions); + + return result; +} + +TEE_Result stm32_bsec_read_otp(uint32_t *value, uint32_t otp_id) +{ + if (otp_id > otp_max_id()) + return TEE_ERROR_BAD_PARAMETERS; + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + *value = io_read32(bsec_base() + BSEC_OTP_DATA_OFF + + (otp_id * sizeof(uint32_t))); + + return TEE_SUCCESS; +} + +TEE_Result stm32_bsec_shadow_read_otp(uint32_t *otp_value, uint32_t otp_id) +{ + TEE_Result result = 0; + + result = stm32_bsec_shadow_register(otp_id); + if (result) { + EMSG("BSEC %"PRIu32" Shadowing Error %#"PRIx32, otp_id, result); + return result; + } + + result = stm32_bsec_read_otp(otp_value, otp_id); + if (result) + EMSG("BSEC %"PRIu32" Read Error %#"PRIx32, otp_id, result); + + return result; +} + +TEE_Result stm32_bsec_write_otp(uint32_t value, uint32_t otp_id) +{ + TEE_Result result = 0; + uint32_t exceptions = 0; + vaddr_t otp_data_base = bsec_base() + BSEC_OTP_DATA_OFF; + bool locked = false; + + /* Check if write of OTP is locked, informative only */ + result = stm32_bsec_read_sw_lock(otp_id, &locked); + if (result) + return result; + + if (locked) + DMSG("BSEC write warning: OTP locked"); + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + exceptions = bsec_lock(); + + io_write32(otp_data_base + (otp_id * sizeof(uint32_t)), value); + + bsec_unlock(exceptions); + + return TEE_SUCCESS; +} + +#ifdef CFG_STM32_BSEC_WRITE +TEE_Result stm32_bsec_program_otp(uint32_t value, uint32_t otp_id) +{ + TEE_Result result = 0; + uint32_t exceptions = 0; + uint64_t timeout_ref = 0; + bool locked = false; + + /* Check if shadowing of OTP is locked, informative only */ + result = stm32_bsec_read_sp_lock(otp_id, &locked); + if (result) + return result; + + if (locked) + DMSG("BSEC program warning: OTP locked"); + + if (io_read32(bsec_base() + BSEC_OTP_LOCK_OFF) & BIT(BSEC_LOCK_PROGRAM)) + DMSG("BSEC program warning: GPLOCK activated"); + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + exceptions = bsec_lock(); + + result = power_up_safmem(); + if (result) + goto out; + + io_write32(bsec_base() + BSEC_OTP_WRDATA_OFF, value); + io_write32(bsec_base() + BSEC_OTP_CTRL_OFF, otp_id | BSEC_WRITE); + + timeout_ref = timeout_init_us(BSEC_TIMEOUT_US); + while (!timeout_elapsed(timeout_ref)) + if (!(bsec_status() & BSEC_MODE_BUSY)) + break; + + if (bsec_status() & BSEC_MODE_BUSY) + result = TEE_ERROR_BUSY; + else if (bsec_status() & BSEC_MODE_PROGFAIL) + result = TEE_ERROR_BAD_PARAMETERS; + else + result = check_no_error(otp_id, true /* check-disturbed */); + + power_down_safmem(); + +out: + bsec_unlock(exceptions); + + return result; +} + +TEE_Result stm32_bsec_permanent_lock_otp(uint32_t otp_id) +{ + TEE_Result result = 0; + uint32_t data = 0; + uint32_t addr = 0; + uint32_t exceptions = 0; + vaddr_t base = bsec_base(); + uint64_t timeout_ref = 0; + uint32_t upper_base = otp_upper_base(); + + if (otp_id > otp_max_id()) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * 2 bits per words for lower OTPs: 2:1 Redundancy + * 1 bit per word for upper OTPs : ECC support + * e.g with 32 lower and 64 upper OTPs: + * OTP word to be ADDR[6:0] WRDATA[31:0] + * locked + * 0 0x00 0x0000 0003 + * 1 0x00 0x0000 000C + * ... ... ... + * 7 0x00 0x0000 C000 + * 8 0x01 0x0000 0003 + * ... ... ... + * 31 0x03 0x0000 C000 + * 32 0x04 0x0000 0001 + * 33 0x04 0x0000 0002 + * 95 0x07 0x0000 8000 + */ + if (otp_id < upper_base) { + addr = otp_id / 8U; + data = DATA_LOWER_OTP_PERLOCK_BIT << ((otp_id * 2U) & 0xF); + } else { + addr = upper_base / 8U + (otp_id - upper_base) / 16U; + data = DATA_UPPER_OTP_PERLOCK_BIT << (otp_id & 0xF); + } + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + exceptions = bsec_lock(); + + result = power_up_safmem(); + if (result) + goto out; + + io_write32(base + BSEC_OTP_WRDATA_OFF, data); + io_write32(base + BSEC_OTP_CTRL_OFF, addr | BSEC_WRITE | BSEC_LOCK); + + timeout_ref = timeout_init_us(BSEC_TIMEOUT_US); + while (!timeout_elapsed(timeout_ref)) + if (!(bsec_status() & BSEC_MODE_BUSY)) + break; + + if (bsec_status() & BSEC_MODE_BUSY) + result = TEE_ERROR_BUSY; + else if (bsec_status() & BSEC_MODE_PROGFAIL) + result = TEE_ERROR_BAD_PARAMETERS; + else + result = check_no_error(otp_id, false /* not-disturbed */); + +#ifdef CFG_STM32MP13 + io_write32(base + BSEC_OTP_CTRL_OFF, addr | BSEC_READ | BSEC_LOCK); +#endif + + power_down_safmem(); + +out: + bsec_unlock(exceptions); + + return result; +} +#endif /*CFG_STM32_BSEC_WRITE*/ + +TEE_Result stm32_bsec_write_debug_conf(uint32_t value) +{ + TEE_Result result = TEE_ERROR_GENERIC; + uint32_t exceptions = 0; + + assert(!(value & ~BSEC_DEN_ALL_MSK)); + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + exceptions = bsec_lock(); + + io_clrsetbits32(bsec_base() + BSEC_DEN_OFF, BSEC_DEN_ALL_MSK, value); + + if (stm32_bsec_read_debug_conf() == value) + result = TEE_SUCCESS; + + bsec_unlock(exceptions); + + return result; +} + +uint32_t stm32_bsec_read_debug_conf(void) +{ + return io_read32(bsec_base() + BSEC_DEN_OFF) & BSEC_DEN_ALL_MSK; +} + +static TEE_Result set_bsec_lock(uint32_t otp_id, size_t lock_offset) +{ + uint32_t bank = otp_bank_offset(otp_id); + uint32_t otp_mask = BIT(otp_id & BSEC_OTP_MASK); + vaddr_t lock_addr = bsec_base() + bank + lock_offset; + uint32_t exceptions = 0; + + if (otp_id > STM32MP1_OTP_MAX_ID) + return TEE_ERROR_BAD_PARAMETERS; + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + exceptions = bsec_lock(); + + io_write32(lock_addr, otp_mask); + + bsec_unlock(exceptions); + + return TEE_SUCCESS; +} + +TEE_Result stm32_bsec_set_sr_lock(uint32_t otp_id) +{ + return set_bsec_lock(otp_id, BSEC_SRLOCK_OFF); +} + +TEE_Result stm32_bsec_set_sw_lock(uint32_t otp_id) +{ + return set_bsec_lock(otp_id, BSEC_SWLOCK_OFF); +} + +TEE_Result stm32_bsec_set_sp_lock(uint32_t otp_id) +{ + return set_bsec_lock(otp_id, BSEC_SPLOCK_OFF); +} + +static TEE_Result read_bsec_lock(uint32_t otp_id, bool *locked, + size_t lock_offset) +{ + uint32_t bank = otp_bank_offset(otp_id); + uint32_t otp_mask = BIT(otp_id & BSEC_OTP_MASK); + vaddr_t lock_addr = bsec_base() + bank + lock_offset; + + if (otp_id > STM32MP1_OTP_MAX_ID) + return TEE_ERROR_BAD_PARAMETERS; + + if (state_is_invalid_mode()) + return TEE_ERROR_SECURITY; + + *locked = (io_read32(lock_addr) & otp_mask) != 0; + + return TEE_SUCCESS; +} + +TEE_Result stm32_bsec_read_sr_lock(uint32_t otp_id, bool *locked) +{ + return read_bsec_lock(otp_id, locked, BSEC_SRLOCK_OFF); +} + +TEE_Result stm32_bsec_read_sw_lock(uint32_t otp_id, bool *locked) +{ + return read_bsec_lock(otp_id, locked, BSEC_SWLOCK_OFF); +} + +TEE_Result stm32_bsec_read_sp_lock(uint32_t otp_id, bool *locked) +{ + return read_bsec_lock(otp_id, locked, BSEC_SPLOCK_OFF); +} + +TEE_Result stm32_bsec_read_permanent_lock(uint32_t otp_id, bool *locked) +{ + return read_bsec_lock(otp_id, locked, BSEC_WRLOCK_OFF); +} + +static size_t nsec_access_array_size(void) +{ + size_t upper_count = otp_max_id() - otp_upper_base() + 1; + + return ROUNDUP_DIV(upper_count, BSEC_BITS_PER_WORD); +} + +static bool nsec_access_granted(unsigned int index) +{ + uint32_t *array = bsec_dev.nsec_access; + + return array && + (index / BSEC_BITS_PER_WORD) < nsec_access_array_size() && + array[index / BSEC_BITS_PER_WORD] & + BIT(index % BSEC_BITS_PER_WORD); +} + +bool stm32_bsec_can_access_otp(uint32_t otp_id) +{ + return (otp_id <= otp_max_id()) && !state_is_invalid_mode(); +} + +bool stm32_bsec_nsec_can_access_otp(uint32_t otp_id) +{ + return otp_id < otp_upper_base() || + nsec_access_granted(otp_id - otp_upper_base()); +} + +/* + * struct nvmem_layout - NVMEM cell description + * @name: Name of the nvmem node in the DT + * @otp_id: BSEC base index for the OTP words + * @bit_offset: Bit offset in the OTP word + * @bit_len: Bit size of the OTP word + * @phandle: Associated phandle in embedded DTB + */ +struct nvmem_layout { + char *name; + uint32_t otp_id; + uint8_t bit_offset; + size_t bit_len; + uint32_t phandle; +}; + +static struct nvmem_layout *nvmem_layout; +static size_t nvmem_layout_count; + +static TEE_Result stm32_bsec_otp_setting(size_t i, + uint32_t *otp_id, + uint8_t *otp_bit_offset, + size_t *otp_bit_len) +{ + if (otp_id) + *otp_id = nvmem_layout[i].otp_id; + + if (otp_bit_offset) + *otp_bit_offset = nvmem_layout[i].bit_offset; + + if (otp_bit_len) + *otp_bit_len = nvmem_layout[i].bit_len; + + DMSG("nvmem[%zu] = %s at BSEC word %" PRIu32 " bits [%" PRIu8 " %zu]", + i, nvmem_layout[i].name, nvmem_layout[i].otp_id, + nvmem_layout[i].bit_offset, nvmem_layout[i].bit_len); + + return TEE_SUCCESS; +} + +TEE_Result stm32_bsec_find_otp_in_nvmem_layout(const char *name, + uint32_t *otp_id, + uint8_t *otp_bit_offset, + size_t *otp_bit_len) +{ + size_t i = 0; + + if (!name) + return TEE_ERROR_BAD_PARAMETERS; + + for (i = 0; i < nvmem_layout_count; i++) { + if (!nvmem_layout[i].name || strcmp(name, nvmem_layout[i].name)) + continue; + + return stm32_bsec_otp_setting(i, otp_id, otp_bit_offset, + otp_bit_len); + } + + DMSG("nvmem %s failed", name); + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +TEE_Result stm32_bsec_find_otp_by_phandle(const uint32_t phandle, + uint32_t *otp_id, + uint8_t *otp_bit_offset, + size_t *otp_bit_len) +{ + size_t i = 0; + + if (!phandle) + return TEE_ERROR_GENERIC; + + for (i = 0; i < nvmem_layout_count; i++) { + if (nvmem_layout[i].phandle != phandle) + continue; + + return stm32_bsec_otp_setting(i, otp_id, otp_bit_offset, + otp_bit_len); + } + + DMSG("nvmem %u not found", phandle); + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +TEE_Result stm32_bsec_get_state(enum stm32_bsec_sec_state *state) +{ + if (!state) + return TEE_ERROR_BAD_PARAMETERS; + + if (state_is_invalid_mode() || !state_is_secured_mode()) { + *state = BSEC_STATE_INVALID; + } else { + if (state_is_closed_mode()) + *state = BSEC_STATE_SEC_CLOSED; + else + *state = BSEC_STATE_SEC_OPEN; + } + + return TEE_SUCCESS; +} + +static void enable_nsec_access(unsigned int otp_id) +{ + unsigned int idx = (otp_id - otp_upper_base()) / BSEC_BITS_PER_WORD; + + if (otp_id < otp_upper_base()) + return; + + if (otp_id > otp_max_id() || stm32_bsec_shadow_register(otp_id)) + panic(); + + bsec_dev.nsec_access[idx] |= BIT(otp_id % BSEC_BITS_PER_WORD); +} + +static void bsec_dt_otp_nsec_access(void *fdt, int bsec_node) +{ + int bsec_subnode = 0; + + bsec_dev.nsec_access = calloc(nsec_access_array_size(), + sizeof(*bsec_dev.nsec_access)); + if (!bsec_dev.nsec_access) + panic(); + + fdt_for_each_subnode(bsec_subnode, fdt, bsec_node) { + unsigned int reg_offset = 0; + unsigned int reg_size = 0; + unsigned int otp_id = 0; + unsigned int i = 0; + size_t size = 0; + + reg_offset = fdt_reg_base_address(fdt, bsec_subnode); + reg_size = fdt_reg_size(fdt, bsec_subnode); + + assert(reg_offset != DT_INFO_INVALID_REG && + reg_size != DT_INFO_INVALID_REG_SIZE); + + otp_id = reg_offset / sizeof(uint32_t); + + if (otp_id < STM32MP1_UPPER_OTP_START) { + unsigned int otp_end = + ROUNDUP_DIV(reg_offset + reg_size, + sizeof(uint32_t)); + + if (otp_end > STM32MP1_UPPER_OTP_START) { + /* + * OTP crosses Lower/Upper boundary, consider + * only the upper part. + */ + otp_id = STM32MP1_UPPER_OTP_START; + reg_size -= (STM32MP1_UPPER_OTP_START * + sizeof(uint32_t)) - reg_offset; + reg_offset = STM32MP1_UPPER_OTP_START * + sizeof(uint32_t); + + DMSG("OTP crosses Lower/Upper boundary"); + } else { + continue; + } + } + + /* Handle different kinds of non-secure accesses */ + if (fdt_getprop(fdt, bsec_subnode, + "st,non-secure-otp-provisioning", NULL)) { + bool locked = false; + bool locked_2 = false; + + /* Check if write of OTP is locked */ + if (stm32_bsec_read_permanent_lock(otp_id, &locked)) + panic("Cannot read permanent lock"); + + /* + * Check if fuses of the subnode + * have the same lock status + */ + for (i = 1; i < (reg_size / sizeof(uint32_t)); i++) { + if (stm32_bsec_read_permanent_lock(otp_id + i, + &locked_2)) + panic("Cannot read permanent lock"); + + if (locked != locked_2) { + EMSG("Inconsistent status OTP ID %u", + otp_id + i); + locked = true; + } + } + + if (locked) { + DMSG("BSEC: OTP locked"); + continue; + } + } else if (!fdt_getprop(fdt, bsec_subnode, "st,non-secure-otp", + NULL)) { + continue; + } + + if ((reg_offset % sizeof(uint32_t)) || + (reg_size % sizeof(uint32_t))) + panic("Unaligned non-secure OTP"); + + size = reg_size / sizeof(uint32_t); + + if (otp_id + size > OTP_MAX_SIZE) + panic("OTP range oversized"); + + for (i = otp_id; i < otp_id + size; i++) + enable_nsec_access(i); + } +} + +static void save_dt_nvmem_layout(void *fdt, int bsec_node) +{ + int cell_max = 0; + int cell_cnt = 0; + int node = 0; + + fdt_for_each_subnode(node, fdt, bsec_node) + cell_max++; + if (!cell_max) + return; + + nvmem_layout = calloc(cell_max, sizeof(*nvmem_layout)); + if (!nvmem_layout) + panic(); + + fdt_for_each_subnode(node, fdt, bsec_node) { + unsigned int reg_offset = 0; + unsigned int reg_length = 0; + const char *string = NULL; + const char *s = NULL; + int len = 0; + struct nvmem_layout *layout_cell = &nvmem_layout[cell_cnt]; + uint32_t bits[2] = { }; + + string = fdt_get_name(fdt, node, &len); + if (!string || !len) + continue; + + layout_cell->phandle = fdt_get_phandle(fdt, node); + assert(layout_cell->phandle != (uint32_t)-1); + + reg_offset = fdt_reg_base_address(fdt, node); + reg_length = fdt_reg_size(fdt, node); + + if (reg_offset == DT_INFO_INVALID_REG || + reg_length == DT_INFO_INVALID_REG_SIZE) { + DMSG("Malformed nvmem %s: ignored", string); + continue; + } + + layout_cell->otp_id = reg_offset / sizeof(uint32_t); + layout_cell->bit_offset = (reg_offset % sizeof(uint32_t)) * + CHAR_BIT; + layout_cell->bit_len = reg_length * CHAR_BIT; + + if (!fdt_read_uint32_array(fdt, node, "bits", bits, 2)) { + layout_cell->bit_offset += bits[0]; + layout_cell->bit_len = bits[1]; + } + + s = strchr(string, '@'); + if (s) + len = s - string; + + layout_cell->name = strndup(string, len); + if (!layout_cell->name) + panic(); + cell_cnt++; + DMSG("nvmem[%d] = %s at BSEC word %" PRIu32 + " bits [%" PRIu8 " %zu]", + cell_cnt, layout_cell->name, layout_cell->otp_id, + layout_cell->bit_offset, layout_cell->bit_len); + } + + if (cell_cnt != cell_max) { + nvmem_layout = realloc(nvmem_layout, + cell_cnt * sizeof(*nvmem_layout)); + if (!nvmem_layout) + panic(); + } + + nvmem_layout_count = cell_cnt; +} + +static void initialize_bsec_from_dt(void) +{ + void *fdt = NULL; + int node = 0; + struct dt_node_info bsec_info = { }; + + fdt = get_embedded_dt(); + node = fdt_node_offset_by_compatible(fdt, 0, DT_BSEC_COMPAT); + if (node < 0) + panic(); + + fdt_fill_device_info(fdt, &bsec_info, node); + + if (bsec_info.reg != bsec_dev.base.pa || + !(bsec_info.status & DT_STATUS_OK_SEC)) + panic(); + + bsec_dt_otp_nsec_access(fdt, node); + + save_dt_nvmem_layout(fdt, node); +} + +static TEE_Result bsec_pm(enum pm_op op, uint32_t pm_hint __unused, + const struct pm_callback_handle *hdl __unused) +{ + static uint32_t debug_conf; + + assert(op == PM_OP_SUSPEND || op == PM_OP_RESUME); + + if (op == PM_OP_SUSPEND) + debug_conf = stm32_bsec_read_debug_conf(); + else + stm32_bsec_write_debug_conf(debug_conf); + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(bsec_pm); + +static TEE_Result initialize_bsec(void) +{ + struct stm32_bsec_static_cfg cfg = { }; + + stm32mp_get_bsec_static_cfg(&cfg); + + bsec_dev.base.pa = cfg.base; + bsec_dev.upper_base = cfg.upper_start; + bsec_dev.max_id = cfg.max_id; + + if (state_is_invalid_mode()) + panic(); + + initialize_bsec_from_dt(); + + register_pm_core_service_cb(bsec_pm, NULL, "stm32_bsec"); + + return TEE_SUCCESS; +} + +early_init(initialize_bsec); diff --git a/optee_os/core/drivers/stm32_etzpc.c b/optee_os/core/drivers/stm32_etzpc.c new file mode 100644 index 0000000..19c7d70 --- /dev/null +++ b/optee_os/core/drivers/stm32_etzpc.c @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2015-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2019, STMicroelectronics + */ + +/* + * STM32 ETPZC acts as a firewall on stm32mp SoC peripheral interfaces and + * internal memories. The driver expects a single instance of the controller + * in the platform. + * + * The driver API is defined in header file stm32_etzpc.h. + * + * Driver registers a PM callback for restoration of the access permissions + * when it resumes. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Devicetree compatibility */ +#define ETZPC_COMPAT "st,stm32-etzpc" + +/* ID Registers */ +#define ETZPC_TZMA0_SIZE 0x000U +#define ETZPC_DECPROT0 0x010U +#define ETZPC_DECPROT_LOCK0 0x030U +#define ETZPC_HWCFGR 0x3F0U +#define ETZPC_VERR 0x3F4U + +/* ID Registers fields */ +#define ETZPC_TZMA0_SIZE_LOCK BIT(31) +#define ETZPC_DECPROT0_MASK GENMASK_32(1, 0) +#define ETZPC_HWCFGR_NUM_TZMA_MASK GENMASK_32(7, 0) +#define ETZPC_HWCFGR_NUM_TZMA_SHIFT 0 +#define ETZPC_HWCFGR_NUM_PER_SEC_MASK GENMASK_32(15, 8) +#define ETZPC_HWCFGR_NUM_PER_SEC_SHIFT 8 +#define ETZPC_HWCFGR_NUM_AHB_SEC_MASK GENMASK_32(23, 16) +#define ETZPC_HWCFGR_NUM_AHB_SEC_SHIFT 16 +#define ETZPC_HWCFGR_CHUNCKS1N4_MASK GENMASK_32(31, 24) +#define ETZPC_HWCFGR_CHUNCKS1N4_SHIFT 24 + +#define DECPROT_SHIFT 1 +#define IDS_PER_DECPROT_REGS 16U +#define IDS_PER_DECPROT_LOCK_REGS 32U + +/* + * Implementation uses uint8_t to store each securable DECPROT configuration + * and uint16_t to store each securable TZMA configuration. When resuming + * from deep suspend, the DECPROT configurations are restored. + */ +#define PERIPH_PM_LOCK_BIT BIT(7) +#define PERIPH_PM_ATTR_MASK GENMASK_32(2, 0) +#define TZMA_PM_LOCK_BIT BIT(15) +#define TZMA_PM_VALUE_MASK GENMASK_32(9, 0) + +/* + * @base - iobase for interface base address + * @num_tzma - number of TZMA zone, read from the hardware + * @num_ahb_sec - number of securable AHB master zone, read from the hardware + * @num_per_sec - number of securable AHB & APB periphs, read from the hardware + * @periph_cfg - Backup for restoring DECPROT when resuming (PERIH_PM_*) + * @tzma_cfg - Backup for restoring TZMA when resuming (TZMA_PM_*) + */ +struct etzpc_instance { + struct io_pa_va base; + unsigned int num_tzma; + unsigned int num_per_sec; + unsigned int num_ahb_sec; + uint8_t *periph_cfg; + uint16_t *tzma_cfg; +}; + +/* Only 1 instance of the ETZPC is expected per platform */ +static struct etzpc_instance etzpc_dev; + +static vaddr_t etzpc_base(void) +{ + return io_pa_or_va_secure(&etzpc_dev.base, 1); +} + +static bool __maybe_unused valid_decprot_id(unsigned int id) +{ + return id < etzpc_dev.num_per_sec; +} + +static bool __maybe_unused valid_tzma_id(unsigned int id) +{ + return id < etzpc_dev.num_tzma; +} + +void etzpc_configure_decprot(uint32_t decprot_id, + enum etzpc_decprot_attributes decprot_attr) +{ + size_t offset = 4U * (decprot_id / IDS_PER_DECPROT_REGS); + uint32_t shift = (decprot_id % IDS_PER_DECPROT_REGS) << DECPROT_SHIFT; + uint32_t masked_decprot = (uint32_t)decprot_attr & ETZPC_DECPROT0_MASK; + vaddr_t base = etzpc_base(); + + assert(valid_decprot_id(decprot_id)); + + io_clrsetbits32(base + ETZPC_DECPROT0 + offset, + ETZPC_DECPROT0_MASK << shift, + masked_decprot << shift); + + /* Save for PM */ + assert((decprot_attr & ~PERIPH_PM_ATTR_MASK) == 0); + COMPILE_TIME_ASSERT(ETZPC_DECPROT_MAX <= UINT8_MAX); + + etzpc_dev.periph_cfg[decprot_id] &= ~PERIPH_PM_ATTR_MASK; + etzpc_dev.periph_cfg[decprot_id] |= (uint8_t)decprot_attr; +} + +enum etzpc_decprot_attributes etzpc_get_decprot(uint32_t decprot_id) +{ + size_t offset = 4U * (decprot_id / IDS_PER_DECPROT_REGS); + uint32_t shift = (decprot_id % IDS_PER_DECPROT_REGS) << DECPROT_SHIFT; + vaddr_t base = etzpc_base(); + uint32_t value = 0; + + assert(valid_decprot_id(decprot_id)); + + value = (io_read32(base + ETZPC_DECPROT0 + offset) >> shift) & + ETZPC_DECPROT0_MASK; + + return (enum etzpc_decprot_attributes)value; +} + +void etzpc_lock_decprot(uint32_t decprot_id) +{ + size_t offset = 4U * (decprot_id / IDS_PER_DECPROT_LOCK_REGS); + uint32_t mask = BIT(decprot_id % IDS_PER_DECPROT_LOCK_REGS); + vaddr_t base = etzpc_base(); + + assert(valid_decprot_id(decprot_id)); + + io_write32(base + offset + ETZPC_DECPROT_LOCK0, mask); + + /* Save for PM */ + etzpc_dev.periph_cfg[decprot_id] |= PERIPH_PM_LOCK_BIT; +} + +bool etzpc_get_lock_decprot(uint32_t decprot_id) +{ + size_t offset = 4U * (decprot_id / IDS_PER_DECPROT_LOCK_REGS); + uint32_t mask = BIT(decprot_id % IDS_PER_DECPROT_LOCK_REGS); + vaddr_t base = etzpc_base(); + + assert(valid_decprot_id(decprot_id)); + + return io_read32(base + offset + ETZPC_DECPROT_LOCK0) & mask; +} + +void etzpc_configure_tzma(uint32_t tzma_id, uint16_t tzma_value) +{ + size_t offset = sizeof(uint32_t) * tzma_id; + vaddr_t base = etzpc_base(); + + assert(valid_tzma_id(tzma_id)); + + io_write32(base + ETZPC_TZMA0_SIZE + offset, tzma_value); + + /* Save for PM */ + assert((tzma_value & ~TZMA_PM_VALUE_MASK) == 0); + etzpc_dev.tzma_cfg[tzma_id] &= ~TZMA_PM_VALUE_MASK; + etzpc_dev.tzma_cfg[tzma_id] |= tzma_value; +} + +uint16_t etzpc_get_tzma(uint32_t tzma_id) +{ + size_t offset = sizeof(uint32_t) * tzma_id; + vaddr_t base = etzpc_base(); + + assert(valid_tzma_id(tzma_id)); + + return io_read32(base + ETZPC_TZMA0_SIZE + offset); +} + +void etzpc_lock_tzma(uint32_t tzma_id) +{ + size_t offset = sizeof(uint32_t) * tzma_id; + vaddr_t base = etzpc_base(); + + assert(valid_tzma_id(tzma_id)); + + io_setbits32(base + ETZPC_TZMA0_SIZE + offset, ETZPC_TZMA0_SIZE_LOCK); + + /* Save for PM */ + etzpc_dev.tzma_cfg[tzma_id] |= TZMA_PM_LOCK_BIT; +} + +bool etzpc_get_lock_tzma(uint32_t tzma_id) +{ + size_t offset = sizeof(uint32_t) * tzma_id; + vaddr_t base = etzpc_base(); + + assert(valid_tzma_id(tzma_id)); + + return io_read32(base + ETZPC_TZMA0_SIZE + offset) & + ETZPC_TZMA0_SIZE_LOCK; +} + +static TEE_Result etzpc_pm(enum pm_op op, unsigned int pm_hint __unused, + const struct pm_callback_handle *pm_handle) +{ + struct etzpc_instance *dev = NULL; + unsigned int n = 0; + + if (op != PM_OP_RESUME) + return TEE_SUCCESS; + + dev = (struct etzpc_instance *)PM_CALLBACK_GET_HANDLE(pm_handle); + + for (n = 0; n < dev->num_per_sec; n++) { + unsigned int attr = dev->periph_cfg[n] & PERIPH_PM_ATTR_MASK; + + etzpc_configure_decprot(n, (enum etzpc_decprot_attributes)attr); + + if (dev->periph_cfg[n] & PERIPH_PM_LOCK_BIT) + etzpc_lock_decprot(n); + } + + for (n = 0; n < dev->num_tzma; n++) { + uint16_t value = dev->tzma_cfg[n] & TZMA_PM_VALUE_MASK; + + etzpc_configure_tzma(n, value); + + if (dev->tzma_cfg[n] & TZMA_PM_LOCK_BIT) + etzpc_lock_tzma(n); + } + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(etzpc_pm); + +static void init_pm(struct etzpc_instance *dev) +{ + unsigned int n = 0; + + dev->periph_cfg = calloc(dev->num_per_sec, sizeof(*dev->periph_cfg)); + dev->tzma_cfg = calloc(dev->num_tzma, sizeof(*dev->tzma_cfg)); + if (!dev->periph_cfg || !dev->tzma_cfg) + panic(); + + for (n = 0; n < dev->num_per_sec; n++) { + dev->periph_cfg[n] = (uint8_t)etzpc_get_decprot(n); + if (etzpc_get_lock_decprot(n)) + dev->periph_cfg[n] |= PERIPH_PM_LOCK_BIT; + } + + for (n = 0; n < dev->num_tzma; n++) { + dev->tzma_cfg[n] = (uint8_t)etzpc_get_tzma(n); + if (etzpc_get_lock_tzma(n)) + dev->tzma_cfg[n] |= TZMA_PM_LOCK_BIT; + } + + register_pm_core_service_cb(etzpc_pm, dev, "stm32-etzpc"); +} + +struct etzpc_hwcfg { + unsigned int num_tzma; + unsigned int num_per_sec; + unsigned int num_ahb_sec; + unsigned int chunk_size; +}; + +static void get_hwcfg(struct etzpc_hwcfg *hwcfg) +{ + uint32_t reg = io_read32(etzpc_base() + ETZPC_HWCFGR); + + hwcfg->num_tzma = (reg & ETZPC_HWCFGR_NUM_TZMA_MASK) >> + ETZPC_HWCFGR_NUM_TZMA_SHIFT; + hwcfg->num_per_sec = (reg & ETZPC_HWCFGR_NUM_PER_SEC_MASK) >> + ETZPC_HWCFGR_NUM_PER_SEC_SHIFT; + hwcfg->num_ahb_sec = (reg & ETZPC_HWCFGR_NUM_AHB_SEC_MASK) >> + ETZPC_HWCFGR_NUM_AHB_SEC_SHIFT; + hwcfg->chunk_size = (reg & ETZPC_HWCFGR_CHUNCKS1N4_MASK) >> + ETZPC_HWCFGR_CHUNCKS1N4_SHIFT; +} + +static void init_device_from_hw_config(struct etzpc_instance *dev, + paddr_t pbase) +{ + struct etzpc_hwcfg hwcfg = { }; + + assert(!dev->base.pa && cpu_mmu_enabled()); + dev->base.pa = pbase; + dev->base.va = (vaddr_t)phys_to_virt(dev->base.pa, MEM_AREA_IO_SEC, 1); + assert(etzpc_base()); + + get_hwcfg(&hwcfg); + dev->num_tzma = hwcfg.num_tzma; + dev->num_per_sec = hwcfg.num_per_sec; + dev->num_ahb_sec = hwcfg.num_ahb_sec; + + DMSG("ETZPC revison 0x02%" PRIu8 ", per_sec %u, ahb_sec %u, tzma %u", + io_read8(etzpc_base() + ETZPC_VERR), + hwcfg.num_per_sec, hwcfg.num_ahb_sec, hwcfg.num_tzma); + + init_pm(dev); +} + +static TEE_Result stm32_etzpc_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + paddr_t pbase = 0; + int subnode = 0; + + pbase = fdt_reg_base_address(fdt, node); + if (pbase == DT_INFO_INVALID_REG) + panic(); + + init_device_from_hw_config(&etzpc_dev, pbase); + + fdt_for_each_subnode(subnode, fdt, node) { + res = dt_driver_maybe_add_probe_node(fdt, subnode); + if (res) { + EMSG("Failed to add node %s to probe list: %#"PRIx32, + fdt_get_name(fdt, subnode, NULL), res); + panic(); + } + } + + return TEE_SUCCESS; +} + +static const struct dt_device_match etzpc_match_table[] = { + { .compatible = "st,stm32-etzpc" }, + { } +}; + +DEFINE_DT_DRIVER(etzpc_dt_driver) = { + .name = "stm32-etzpc", + .match_table = etzpc_match_table, + .probe = stm32_etzpc_probe, +}; diff --git a/optee_os/core/drivers/stm32_gpio.c b/optee_os/core/drivers/stm32_gpio.c new file mode 100644 index 0000000..6450c6f --- /dev/null +++ b/optee_os/core/drivers/stm32_gpio.c @@ -0,0 +1,972 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2017-2023, STMicroelectronics + * + * STM32 GPIO driver is used as pin controller for stm32mp SoCs. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef CFG_DRIVERS_GPIO +#error stm32_gpio driver expects CFG_DRIVERS_GPIO +#endif + +#define GPIO_PIN_MAX 15 + +#define GPIO_MODER_OFFSET 0x00 +#define GPIO_OTYPER_OFFSET 0x04 +#define GPIO_OSPEEDR_OFFSET 0x08 +#define GPIO_PUPDR_OFFSET 0x0c +#define GPIO_IDR_OFFSET 0x10 +#define GPIO_ODR_OFFSET 0x14 +#define GPIO_BSRR_OFFSET 0x18 +#define GPIO_AFRL_OFFSET 0x20 +#define GPIO_AFRH_OFFSET 0x24 +#define GPIO_SECR_OFFSET 0x30 + +#define GPIO_ALT_LOWER_LIMIT 0x8 + +#define GPIO_MODE_MASK GENMASK_32(1, 0) +#define GPIO_OSPEED_MASK GENMASK_32(1, 0) +#define GPIO_PUPD_PULL_MASK GENMASK_32(1, 0) +#define GPIO_ALTERNATE_MASK GENMASK_32(3, 0) + +#define DT_GPIO_BANK_SHIFT 12 +#define DT_GPIO_BANK_MASK GENMASK_32(16, 12) +#define DT_GPIO_PIN_SHIFT 8 +#define DT_GPIO_PIN_MASK GENMASK_32(11, 8) +#define DT_GPIO_MODE_MASK GENMASK_32(7, 0) + +#define DT_GPIO_BANK_NAME0 "GPIOA" + +#define GPIO_MODE_INPUT U(0x0) +#define GPIO_MODE_OUTPUT U(0x1) +#define GPIO_MODE_ALTERNATE U(0x2) +#define GPIO_MODE_ANALOG U(0x3) + +#define GPIO_OTYPE_PUSH_PULL U(0x0) +#define GPIO_OTYPE_OPEN_DRAIN U(0x1) + +#define GPIO_OSPEED_LOW U(0x0) +#define GPIO_OSPEED_MEDIUM U(0x1) +#define GPIO_OSPEED_HIGH U(0x2) +#define GPIO_OSPEED_VERY_HIGH U(0x3) + +#define GPIO_PUPD_NO_PULL U(0x0) +#define GPIO_PUPD_PULL_UP U(0x1) +#define GPIO_PUPD_PULL_DOWN U(0x2) + +#define GPIO_OD_LEVEL_LOW U(0x0) +#define GPIO_OD_LEVEL_HIGH U(0x1) + +/* + * GPIO configuration description structured as single 16bit word + * for efficient save/restore when GPIO pin suspends or resumes. + * + * @mode: One of GPIO_MODE_* + * @otype: One of GPIO_OTYPE_* + * @ospeed: One of GPIO_OSPEED_* + * @pupd: One of GPIO_PUPD_* + * @od: One of GPIO_OD_* + * @af: Alternate function numerical ID between 0 and 15 + */ +struct gpio_cfg { + uint16_t mode: 2; + uint16_t otype: 1; + uint16_t ospeed: 2; + uint16_t pupd: 2; + uint16_t od: 1; + uint16_t af: 4; +}; + +/* + * Description of a pin and its muxing + * + * @bank: GPIO bank identifier as assigned by the platform + * @pin: Pin number in the GPIO bank + * @cfg: Pin configuration + */ +struct stm32_pinctrl { + uint8_t bank; + uint8_t pin; + struct gpio_cfg cfg; +}; + +/* + * struct stm32_pinctrl_array - Array of pins in a pin control state + * @count: Number of cells in @pinctrl + * @pinctrl: Pin control configuration + */ +struct stm32_pinctrl_array { + size_t count; + struct stm32_pinctrl pinctrl[]; +}; + +/** + * struct stm32_gpio_bank - GPIO bank instance + * + * @base: base address of the GPIO controller registers. + * @clock: clock identifier. + * @gpio_chip: GPIO chip reference for that GPIO bank + * @ngpios: number of GPIOs. + * @bank_id: Id of the bank. + * @lock: lock protecting the GPIO bank access. + * @sec_support: True if bank supports pin security protection, otherwise false + * @seccfgr: Secure configuration register value. + * @link: Link in bank list + */ +struct stm32_gpio_bank { + vaddr_t base; + struct clk *clock; + struct gpio_chip gpio_chip; + unsigned int ngpios; + unsigned int bank_id; + unsigned int lock; + STAILQ_ENTRY(stm32_gpio_bank) link; +}; + +/* + * Compatibility information of supported banks + * @gpioz True if bank is a GPIOZ bank + */ +struct bank_compat { + bool gpioz; +}; + +static unsigned int gpio_lock; + +static STAILQ_HEAD(, stm32_gpio_bank) bank_list = + STAILQ_HEAD_INITIALIZER(bank_list); + +static bool is_stm32_gpio_chip(struct gpio_chip *chip); + +static struct stm32_gpio_bank *gpio_chip_to_bank(struct gpio_chip *chip) +{ + return container_of(chip, struct stm32_gpio_bank, gpio_chip); +} + +static enum gpio_level stm32_gpio_get_level(struct gpio_chip *chip, + unsigned int gpio_pin) +{ + struct stm32_gpio_bank *bank = gpio_chip_to_bank(chip); + enum gpio_level level = GPIO_LEVEL_HIGH; + unsigned int reg_offset = 0; + unsigned int mode = 0; + + assert(gpio_pin < bank->ngpios); + + if (clk_enable(bank->clock)) + panic(); + + mode = (io_read32(bank->base + GPIO_MODER_OFFSET) >> (gpio_pin << 1)) & + GPIO_MODE_MASK; + + switch (mode) { + case GPIO_MODE_INPUT: + reg_offset = GPIO_IDR_OFFSET; + break; + case GPIO_MODE_OUTPUT: + reg_offset = GPIO_ODR_OFFSET; + break; + default: + panic(); + } + + if (io_read32(bank->base + reg_offset) & BIT(gpio_pin)) + level = GPIO_LEVEL_HIGH; + else + level = GPIO_LEVEL_LOW; + + clk_disable(bank->clock); + + return level; +} + +static void stm32_gpio_set_level(struct gpio_chip *chip, unsigned int gpio_pin, + enum gpio_level level) +{ + struct stm32_gpio_bank *bank = gpio_chip_to_bank(chip); + + assert(gpio_pin < bank->ngpios); + + if (clk_enable(bank->clock)) + panic(); + + assert(((io_read32(bank->base + GPIO_MODER_OFFSET) >> + (gpio_pin << 1)) & GPIO_MODE_MASK) == GPIO_MODE_OUTPUT); + + if (level == GPIO_LEVEL_HIGH) + io_write32(bank->base + GPIO_BSRR_OFFSET, BIT(gpio_pin)); + else + io_write32(bank->base + GPIO_BSRR_OFFSET, BIT(gpio_pin + 16)); + + clk_disable(bank->clock); +} + +static enum gpio_dir stm32_gpio_get_direction(struct gpio_chip *chip, + unsigned int gpio_pin) +{ + struct stm32_gpio_bank *bank = gpio_chip_to_bank(chip); + uint32_t mode = 0; + + assert(gpio_pin < bank->ngpios); + + if (clk_enable(bank->clock)) + panic(); + + mode = (io_read32(bank->base + GPIO_MODER_OFFSET) >> (gpio_pin << 1)) & + GPIO_MODE_MASK; + + clk_disable(bank->clock); + + switch (mode) { + case GPIO_MODE_INPUT: + return GPIO_DIR_IN; + case GPIO_MODE_OUTPUT: + return GPIO_DIR_OUT; + default: + panic(); + } +} + +static void stm32_gpio_set_direction(struct gpio_chip *chip, + unsigned int gpio_pin, + enum gpio_dir direction) +{ + struct stm32_gpio_bank *bank = gpio_chip_to_bank(chip); + uint32_t exceptions = 0; + uint32_t mode = 0; + + assert(gpio_pin < bank->ngpios); + + if (direction == GPIO_DIR_IN) + mode = GPIO_MODE_INPUT; + else + mode = GPIO_MODE_OUTPUT; + + if (clk_enable(bank->clock)) + panic(); + exceptions = cpu_spin_lock_xsave(&gpio_lock); + io_clrsetbits32(bank->base + GPIO_MODER_OFFSET, + SHIFT_U32(GPIO_MODE_MASK, gpio_pin << 1), + SHIFT_U32(mode, gpio_pin << 1)); + cpu_spin_unlock_xrestore(&gpio_lock, exceptions); + clk_disable(bank->clock); +} + +static void stm32_gpio_put_gpio(struct gpio_chip *chip __maybe_unused, + struct gpio *gpio) +{ + assert(is_stm32_gpio_chip(chip)); + free(gpio); +} + +static const struct gpio_ops stm32_gpio_ops = { + .get_direction = stm32_gpio_get_direction, + .set_direction = stm32_gpio_set_direction, + .get_value = stm32_gpio_get_level, + .set_value = stm32_gpio_set_level, + .put = stm32_gpio_put_gpio, +}; + +static bool __maybe_unused is_stm32_gpio_chip(struct gpio_chip *chip) +{ + return chip && chip->ops == &stm32_gpio_ops; +} + +static struct stm32_gpio_bank *stm32_gpio_get_bank(unsigned int bank_id) +{ + struct stm32_gpio_bank *bank = NULL; + + STAILQ_FOREACH(bank, &bank_list, link) + if (bank_id == bank->bank_id) + return bank; + + panic(); +} + +/* Save to output @cfg the current GPIO (@bank_id/@pin) configuration */ +static void __maybe_unused get_gpio_cfg(uint32_t bank_id, uint32_t pin, + struct gpio_cfg *cfg) +{ + struct stm32_gpio_bank *bank = stm32_gpio_get_bank(bank_id); + + if (clk_enable(bank->clock)) + panic(); + + /* + * Save GPIO configuration bits spread over the few bank registers. + * 1bit fields are accessed at bit position being the pin index. + * 2bit fields are accessed at bit position being twice the pin index. + * 4bit fields are accessed at bit position being fourth the pin index + * but accessed from 2 32bit registers at incremental addresses. + */ + cfg->mode = (io_read32(bank->base + GPIO_MODER_OFFSET) >> (pin << 1)) & + GPIO_MODE_MASK; + + cfg->otype = (io_read32(bank->base + GPIO_OTYPER_OFFSET) >> pin) & 1; + + cfg->ospeed = (io_read32(bank->base + GPIO_OSPEEDR_OFFSET) >> + (pin << 1)) & GPIO_OSPEED_MASK; + + cfg->pupd = (io_read32(bank->base + GPIO_PUPDR_OFFSET) >> (pin << 1)) & + GPIO_PUPD_PULL_MASK; + + cfg->od = (io_read32(bank->base + GPIO_ODR_OFFSET) >> (pin << 1)) & 1; + + if (pin < GPIO_ALT_LOWER_LIMIT) + cfg->af = (io_read32(bank->base + GPIO_AFRL_OFFSET) >> + (pin << 2)) & GPIO_ALTERNATE_MASK; + else + cfg->af = (io_read32(bank->base + GPIO_AFRH_OFFSET) >> + ((pin - GPIO_ALT_LOWER_LIMIT) << 2)) & + GPIO_ALTERNATE_MASK; + + clk_disable(bank->clock); +} + +/* Apply GPIO (@bank/@pin) configuration described by @cfg */ +static void set_gpio_cfg(uint32_t bank_id, uint32_t pin, struct gpio_cfg *cfg) +{ + struct stm32_gpio_bank *bank = stm32_gpio_get_bank(bank_id); + uint32_t exceptions = 0; + + if (clk_enable(bank->clock)) + panic(); + exceptions = cpu_spin_lock_xsave(&gpio_lock); + + /* Load GPIO MODE value, 2bit value shifted by twice the pin number */ + io_clrsetbits32(bank->base + GPIO_MODER_OFFSET, + SHIFT_U32(GPIO_MODE_MASK, pin << 1), + SHIFT_U32(cfg->mode, pin << 1)); + + /* Load GPIO Output TYPE value, 1bit shifted by pin number value */ + io_clrsetbits32(bank->base + GPIO_OTYPER_OFFSET, BIT(pin), + SHIFT_U32(cfg->otype, pin)); + + /* Load GPIO Output Speed confguration, 2bit value */ + io_clrsetbits32(bank->base + GPIO_OSPEEDR_OFFSET, + SHIFT_U32(GPIO_OSPEED_MASK, pin << 1), + SHIFT_U32(cfg->ospeed, pin << 1)); + + /* Load GPIO pull configuration, 2bit value */ + io_clrsetbits32(bank->base + GPIO_PUPDR_OFFSET, BIT(pin), + SHIFT_U32(cfg->pupd, pin << 1)); + + /* Load pin mux Alternate Function configuration, 4bit value */ + if (pin < GPIO_ALT_LOWER_LIMIT) { + io_clrsetbits32(bank->base + GPIO_AFRL_OFFSET, + SHIFT_U32(GPIO_ALTERNATE_MASK, pin << 2), + SHIFT_U32(cfg->af, pin << 2)); + } else { + size_t shift = (pin - GPIO_ALT_LOWER_LIMIT) << 2; + + io_clrsetbits32(bank->base + GPIO_AFRH_OFFSET, + SHIFT_U32(GPIO_ALTERNATE_MASK, shift), + SHIFT_U32(cfg->af, shift)); + } + + /* Load GPIO Output direction confuguration, 1bit */ + io_clrsetbits32(bank->base + GPIO_ODR_OFFSET, BIT(pin), cfg->od << pin); + + cpu_spin_unlock_xrestore(&gpio_lock, exceptions); + clk_disable(bank->clock); +} + +/* Count pins described in the DT node and get related data if possible */ +static int get_pinctrl_from_fdt(const void *fdt, int node, + struct stm32_pinctrl *pinctrl, size_t count) +{ + const fdt32_t *cuint = NULL; + const fdt32_t *slewrate = NULL; + int len = 0; + uint32_t i = 0; + uint32_t speed = GPIO_OSPEED_LOW; + uint32_t pull = GPIO_PUPD_NO_PULL; + size_t found = 0; + + cuint = fdt_getprop(fdt, node, "pinmux", &len); + if (!cuint) + return -FDT_ERR_NOTFOUND; + + slewrate = fdt_getprop(fdt, node, "slew-rate", NULL); + if (slewrate) + speed = fdt32_to_cpu(*slewrate); + + if (fdt_getprop(fdt, node, "bias-pull-up", NULL)) + pull = GPIO_PUPD_PULL_UP; + if (fdt_getprop(fdt, node, "bias-pull-down", NULL)) + pull = GPIO_PUPD_PULL_DOWN; + + for (i = 0; i < ((uint32_t)len / sizeof(uint32_t)); i++) { + uint32_t pincfg = 0; + uint32_t bank = 0; + uint32_t pin = 0; + uint32_t mode = 0; + uint32_t alternate = 0; + uint32_t odata = 0; + bool opendrain = false; + + pincfg = fdt32_to_cpu(*cuint); + cuint++; + + bank = (pincfg & DT_GPIO_BANK_MASK) >> DT_GPIO_BANK_SHIFT; + + pin = (pincfg & DT_GPIO_PIN_MASK) >> DT_GPIO_PIN_SHIFT; + + mode = pincfg & DT_GPIO_MODE_MASK; + + switch (mode) { + case 0: + mode = GPIO_MODE_INPUT; + break; + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + alternate = mode - 1U; + mode = GPIO_MODE_ALTERNATE; + break; + case 17: + mode = GPIO_MODE_ANALOG; + break; + default: + mode = GPIO_MODE_OUTPUT; + break; + } + + if (fdt_getprop(fdt, node, "drive-open-drain", NULL)) + opendrain = true; + + if (fdt_getprop(fdt, node, "output-high", NULL) && + mode == GPIO_MODE_INPUT) { + mode = GPIO_MODE_OUTPUT; + odata = 1; + } + + if (fdt_getprop(fdt, node, "output-low", NULL) && + mode == GPIO_MODE_INPUT) { + mode = GPIO_MODE_OUTPUT; + odata = 0; + } + + if (found < count) { + struct stm32_pinctrl *ref = &pinctrl[found]; + + ref->bank = (uint8_t)bank; + ref->pin = (uint8_t)pin; + ref->cfg.mode = mode; + if (opendrain) + ref->cfg.otype = GPIO_OTYPE_OPEN_DRAIN; + else + ref->cfg.otype = GPIO_OTYPE_PUSH_PULL; + ref->cfg.ospeed = speed; + ref->cfg.pupd = pull; + ref->cfg.od = odata; + ref->cfg.af = alternate; + } + + found++; + } + + return (int)found; +} + +static TEE_Result stm32_gpio_get_dt(struct dt_pargs *pargs, void *data, + struct gpio **out_gpio) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct stm32_gpio_bank *bank = data; + struct gpio *gpio = NULL; + unsigned int shift_1b = 0; + unsigned int shift_2b = 0; + uint32_t exceptions = 0; + uint32_t otype = 0; + uint32_t pupd = 0; + uint32_t mode = 0; + + res = gpio_dt_alloc_pin(pargs, &gpio); + if (res) + return res; + + if (gpio->pin >= bank->ngpios) { + DMSG("Invalid GPIO reference"); + free(gpio); + return TEE_ERROR_GENERIC; + } + + shift_1b = gpio->pin; + shift_2b = SHIFT_U32(gpio->pin, 1); + + if (gpio->dt_flags & GPIO_PULL_UP) + pupd = GPIO_PUPD_PULL_UP; + else if (gpio->dt_flags & GPIO_PULL_DOWN) + pupd = GPIO_PUPD_PULL_DOWN; + else + pupd = GPIO_PUPD_NO_PULL; + + if (gpio->dt_flags & GPIO_LINE_OPEN_DRAIN) + otype = GPIO_OTYPE_OPEN_DRAIN; + else + otype = GPIO_OTYPE_PUSH_PULL; + + if (clk_enable(bank->clock)) + panic(); + exceptions = cpu_spin_lock_xsave(&gpio_lock); + + io_clrsetbits32(bank->base + GPIO_MODER_OFFSET, + SHIFT_U32(GPIO_MODE_MASK, shift_2b), + SHIFT_U32(mode, shift_2b)); + + io_clrsetbits32(bank->base + GPIO_OTYPER_OFFSET, + SHIFT_U32(GPIO_OTYPE_OPEN_DRAIN, shift_1b), + SHIFT_U32(otype, shift_1b)); + + io_clrsetbits32(bank->base + GPIO_PUPDR_OFFSET, + SHIFT_U32(GPIO_PUPD_PULL_MASK, shift_2b), + SHIFT_U32(pupd, shift_2b)); + + cpu_spin_unlock_xrestore(&gpio_lock, exceptions); + clk_disable(bank->clock); + + gpio->chip = &bank->gpio_chip; + + *out_gpio = gpio; + + return TEE_SUCCESS; +} + +/* Get bank ID from bank node property st,bank-name or panic on failure */ +static unsigned int dt_get_bank_id(const void *fdt, int node) +{ + const int dt_name_len = strlen(DT_GPIO_BANK_NAME0); + const fdt32_t *cuint = NULL; + int len = 0; + + /* Parse "st,bank-name" to get its id (eg: GPIOA -> 0) */ + cuint = fdt_getprop(fdt, node, "st,bank-name", &len); + if (!cuint || (len != dt_name_len + 1)) + panic("Missing/wrong st,bank-name property"); + + if (strncmp((const char *)cuint, DT_GPIO_BANK_NAME0, dt_name_len - 1) || + strcmp((const char *)cuint, DT_GPIO_BANK_NAME0) < 0) + panic("Wrong st,bank-name property"); + + return (unsigned int)strcmp((const char *)cuint, DT_GPIO_BANK_NAME0); +} + +/* + * Return whether or not the GPIO bank related to a DT node is already + * registered in the GPIO bank link. + */ +static bool bank_is_registered(const void *fdt, int node) +{ + unsigned int bank_id = dt_get_bank_id(fdt, node); + struct stm32_gpio_bank *bank = NULL; + + STAILQ_FOREACH(bank, &bank_list, link) + if (bank->bank_id == bank_id) + return true; + + return false; +} + +/* Get GPIO bank information from the DT */ +static TEE_Result dt_stm32_gpio_bank(const void *fdt, int node, + const void *compat_data, + int range_offset, + struct stm32_gpio_bank **out_bank) +{ + const struct bank_compat *compat = compat_data; + TEE_Result res = TEE_ERROR_GENERIC; + struct stm32_gpio_bank *bank = NULL; + const fdt32_t *cuint = NULL; + struct io_pa_va pa_va = { }; + struct clk *clk = NULL; + size_t blen = 0; + paddr_t pa = 0; + int len = 0; + int i = 0; + + assert(out_bank); + + /* Probe deferrable devices first */ + res = clk_dt_get_by_index(fdt, node, 0, &clk); + if (res) + return res; + + bank = calloc(1, sizeof(*bank)); + if (!bank) + return TEE_ERROR_OUT_OF_MEMORY; + + /* + * Do not rely *only* on the "reg" property to get the address, + * but consider also the "ranges" translation property + */ + pa = fdt_reg_base_address(fdt, node); + if (pa == DT_INFO_INVALID_REG) + panic("missing reg property"); + + pa_va.pa = pa + range_offset; + + blen = fdt_reg_size(fdt, node); + if (blen == DT_INFO_INVALID_REG_SIZE) + panic("missing reg size property"); + + DMSG("Bank name %s", fdt_get_name(fdt, node, NULL)); + bank->base = io_pa_or_va_secure(&pa_va, blen); + bank->bank_id = dt_get_bank_id(fdt, node); + bank->clock = clk; + bank->gpio_chip.ops = &stm32_gpio_ops; + + /* Parse gpio-ranges with its 4 parameters */ + cuint = fdt_getprop(fdt, node, "gpio-ranges", &len); + len /= sizeof(*cuint); + if (len % 4) + panic("wrong gpio-ranges syntax"); + + /* Get the last defined gpio line (offset + nb of pins) */ + for (i = 0; i < len / 4; i++) { + bank->ngpios = MAX(bank->ngpios, + (unsigned int)(fdt32_to_cpu(*(cuint + 1)) + + fdt32_to_cpu(*(cuint + 3)))); + cuint += 4; + } + + if (compat->gpioz) + stm32mp_register_gpioz_pin_count(bank->ngpios); + + *out_bank = bank; + return TEE_SUCCESS; +} + +static void set_bank_gpio_non_secure(struct stm32_gpio_bank *bank) +{ + unsigned int pin = 0; + + for (pin = 0; pin <= bank->ngpios; pin++) + stm32_gpio_set_secure_cfg(bank->bank_id, pin, false); +} + +/* Parse a pinctrl node to register the GPIO banks it describes */ +static TEE_Result dt_stm32_gpio_pinctrl(const void *fdt, int node, + const void *compat_data) +{ + TEE_Result res = TEE_SUCCESS; + const fdt32_t *cuint = NULL; + int range_offs = 0; + int b_node = 0; + int len = 0; + + /* Read the ranges property (for regs memory translation) */ + cuint = fdt_getprop(fdt, node, "ranges", &len); + if (!cuint) + panic("missing ranges property"); + + len /= sizeof(*cuint); + if (len == 3) + range_offs = fdt32_to_cpu(*(cuint + 1)) - fdt32_to_cpu(*cuint); + + fdt_for_each_subnode(b_node, fdt, node) { + cuint = fdt_getprop(fdt, b_node, "gpio-controller", &len); + if (cuint) { + /* + * We found a property "gpio-controller" in the node: + * the node is a GPIO bank description, add it to the + * bank list. + */ + struct stm32_gpio_bank *bank = NULL; + + if (fdt_get_status(fdt, b_node) == DT_STATUS_DISABLED || + bank_is_registered(fdt, b_node)) + continue; + + res = dt_stm32_gpio_bank(fdt, b_node, compat_data, + range_offs, &bank); + if (res) + return res; + + /* Registering a provider should not defer probe */ + res = gpio_register_provider(fdt, b_node, + stm32_gpio_get_dt, bank); + if (res) + panic(); + + STAILQ_INSERT_TAIL(&bank_list, bank, link); + + if (IS_ENABLED(CFG_STM32MP13)) + set_bank_gpio_non_secure(bank); + } else { + if (len != -FDT_ERR_NOTFOUND) + panic(); + } + } + + return TEE_SUCCESS; +} + + +int stm32_get_gpio_count(void *fdt, int pinctrl_node, unsigned int bank) +{ + int node = 0; + const fdt32_t *cuint = NULL; + + fdt_for_each_subnode(node, fdt, pinctrl_node) { + if (!fdt_getprop(fdt, node, "gpio-controller", NULL)) + continue; + + cuint = fdt_getprop(fdt, node, "reg", NULL); + if (!cuint) + continue; + + if (fdt32_to_cpu(*cuint) != stm32_get_gpio_bank_offset(bank)) + continue; + + cuint = fdt_getprop(fdt, node, "ngpios", NULL); + if (!cuint) + panic(); + + return (int)fdt32_to_cpu(*cuint); + } + + return -1; +} + +void stm32_gpio_set_secure_cfg(unsigned int bank_id, unsigned int pin, + bool secure) +{ + struct stm32_gpio_bank *bank = stm32_gpio_get_bank(bank_id); + uint32_t exceptions = 0; + + if (clk_enable(bank->clock)) + panic(); + exceptions = cpu_spin_lock_xsave(&gpio_lock); + + if (secure) + io_setbits32(bank->base + GPIO_SECR_OFFSET, BIT(pin)); + else + io_clrbits32(bank->base + GPIO_SECR_OFFSET, BIT(pin)); + + cpu_spin_unlock_xrestore(&gpio_lock, exceptions); + clk_disable(bank->clock); +} + +#ifdef CFG_DRIVERS_PINCTRL +static TEE_Result stm32_pinctrl_conf_apply(struct pinconf *conf) +{ + struct stm32_pinctrl_array *ref = conf->priv; + struct stm32_pinctrl *p = ref->pinctrl; + size_t pin_count = ref->count; + size_t n = 0; + + for (n = 0; n < pin_count; n++) + set_gpio_cfg(p[n].bank, p[n].pin, &p[n].cfg); + + return TEE_SUCCESS; +} + +static void stm32_pinctrl_conf_free(struct pinconf *conf) +{ + free(conf); +} + +static const struct pinctrl_ops stm32_pinctrl_ops = { + .conf_apply = stm32_pinctrl_conf_apply, + .conf_free = stm32_pinctrl_conf_free, +}; + +DECLARE_KEEP_PAGER(stm32_pinctrl_ops); + +void stm32_gpio_pinctrl_bank_pin(struct pinctrl_state *pinctrl, + unsigned int *bank, unsigned int *pin, + unsigned int *count) +{ + size_t conf_index = 0; + size_t pin_count = 0; + size_t n = 0; + + assert(count); + if (!pinctrl) + goto out; + + for (conf_index = 0; conf_index < pinctrl->conf_count; conf_index++) { + struct pinconf *pinconf = pinctrl->confs[conf_index]; + struct stm32_pinctrl_array *ref = pinconf->priv; + + /* Consider only the stm32_gpio pins */ + if (pinconf->ops != &stm32_pinctrl_ops) + continue; + + if (bank || pin) { + for (n = 0; n < ref->count; n++) { + if (bank && pin_count < *count) + bank[pin_count] = ref->pinctrl[n].bank; + if (pin && pin_count < *count) + pin[pin_count] = ref->pinctrl[n].pin; + pin_count++; + } + } else { + pin_count += ref->count; + } + } + +out: + *count = pin_count; +} + +void stm32_pinctrl_set_secure_cfg(struct pinctrl_state *pinctrl, bool secure) +{ + size_t conf_index = 0; + + if (!pinctrl) + return; + + for (conf_index = 0; conf_index < pinctrl->conf_count; conf_index++) { + struct pinconf *pinconf = pinctrl->confs[conf_index]; + struct stm32_pinctrl_array *ref = pinconf->priv; + struct stm32_pinctrl *pc = NULL; + size_t n = 0; + + for (n = 0; n < ref->count; n++) { + if (pinconf->ops != &stm32_pinctrl_ops) + continue; + + pc = ref->pinctrl + n; + stm32_gpio_set_secure_cfg(pc->bank, pc->pin, secure); + } + } +} + +/* Allocate and return a pinctrl configuration from a DT reference */ +static TEE_Result stm32_pinctrl_dt_get(struct dt_pargs *pargs, + void *data __unused, + struct pinconf **out_pinconf) +{ + struct conf { + struct pinconf pinconf; + struct stm32_pinctrl_array array_ref; + } *loc_conf = NULL; + struct stm32_pinctrl *pinctrl = NULL; + struct pinconf *pinconf = NULL; + const void *fdt = NULL; + size_t pin_count = 0; + int pinctrl_node = 0; + int pinmux_node = 0; + int count = 0; + + pinctrl_node = pargs->phandle_node; + fdt = pargs->fdt; + assert(fdt && pinctrl_node); + + fdt_for_each_subnode(pinmux_node, fdt, pinctrl_node) { + if (fdt_getprop(fdt, pinmux_node, "pinmux", &count)) + pin_count += (size_t)count / sizeof(uint32_t); + else if (count != -FDT_ERR_NOTFOUND) + panic(); + } + + loc_conf = calloc(1, sizeof(*loc_conf) + sizeof(*pinctrl) * pin_count); + if (!loc_conf) + return TEE_ERROR_OUT_OF_MEMORY; + + pinconf = &loc_conf->pinconf; + pinconf->ops = &stm32_pinctrl_ops; + pinconf->priv = &loc_conf->array_ref; + + loc_conf->array_ref.count = pin_count; + pinctrl = loc_conf->array_ref.pinctrl; + + count = 0; + fdt_for_each_subnode(pinmux_node, fdt, pinctrl_node) { + int found = 0; + + found = get_pinctrl_from_fdt(fdt, pinmux_node, pinctrl + count, + pin_count - count); + if (found <= 0 && found > ((int)pin_count - count)) { + /* We can't recover from an error here so let's panic */ + panic(); + } + + count += found; + } + + *out_pinconf = pinconf; + + return TEE_SUCCESS; +} +#endif /*CFG_DRIVERS_PINCTRL*/ + +static TEE_Result stm32_pinctrl_probe(const void *fdt, int node, + const void *compat_data) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + /* Register GPIO banks described in this pin control node */ + res = dt_stm32_gpio_pinctrl(fdt, node, compat_data); + if (res) + return res; + +#ifdef CFG_DRIVERS_PINCTRL + res = pinctrl_register_provider(fdt, node, stm32_pinctrl_dt_get, + (void *)compat_data); + if (res) + return res; +#endif + + return TEE_SUCCESS; +} + +static const struct dt_device_match stm32_pinctrl_match_table[] = { + { + .compatible = "st,stm32mp135-pinctrl", + .compat_data = &(struct bank_compat){ }, + + }, + { + .compatible = "st,stm32mp157-pinctrl", + .compat_data = &(struct bank_compat){ }, + }, + { + .compatible = "st,stm32mp157-z-pinctrl", + .compat_data = &(struct bank_compat){ .gpioz = true, }, + }, + { } +}; + +DEFINE_DT_DRIVER(stm32_pinctrl_dt_driver) = { + .name = "stm32_gpio-pinctrl", + .type = DT_DRIVER_PINCTRL, + .match_table = stm32_pinctrl_match_table, + .probe = stm32_pinctrl_probe, +}; diff --git a/optee_os/core/drivers/stm32_i2c.c b/optee_os/core/drivers/stm32_i2c.c new file mode 100644 index 0000000..563dfc9 --- /dev/null +++ b/optee_os/core/drivers/stm32_i2c.c @@ -0,0 +1,1689 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright (c) 2017-2019, STMicroelectronics + * + * The driver API is defined in header file stm32_i2c.h. + * + * I2C bus driver does not register to the PM framework. It is the + * responsibility of the bus owner to call the related STM32 I2C driver + * API functions when bus suspends or resumes. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* STM32 I2C registers offsets */ +#define I2C_CR1 0x00U +#define I2C_CR2 0x04U +#define I2C_OAR1 0x08U +#define I2C_OAR2 0x0CU +#define I2C_TIMINGR 0x10U +#define I2C_TIMEOUTR 0x14U +#define I2C_ISR 0x18U +#define I2C_ICR 0x1CU +#define I2C_PECR 0x20U +#define I2C_RXDR 0x24U +#define I2C_TXDR 0x28U +#define I2C_SIZE 0x2CU + +/* Bit definition for I2C_CR1 register */ +#define I2C_CR1_PE BIT(0) +#define I2C_CR1_TXIE BIT(1) +#define I2C_CR1_RXIE BIT(2) +#define I2C_CR1_ADDRIE BIT(3) +#define I2C_CR1_NACKIE BIT(4) +#define I2C_CR1_STOPIE BIT(5) +#define I2C_CR1_TCIE BIT(6) +#define I2C_CR1_ERRIE BIT(7) +#define I2C_CR1_DNF GENMASK_32(11, 8) +#define I2C_CR1_ANFOFF BIT(12) +#define I2C_CR1_SWRST BIT(13) +#define I2C_CR1_TXDMAEN BIT(14) +#define I2C_CR1_RXDMAEN BIT(15) +#define I2C_CR1_SBC BIT(16) +#define I2C_CR1_NOSTRETCH BIT(17) +#define I2C_CR1_WUPEN BIT(18) +#define I2C_CR1_GCEN BIT(19) +#define I2C_CR1_SMBHEN BIT(22) +#define I2C_CR1_SMBDEN BIT(21) +#define I2C_CR1_ALERTEN BIT(22) +#define I2C_CR1_PECEN BIT(23) + +/* Bit definition for I2C_CR2 register */ +#define I2C_CR2_SADD GENMASK_32(9, 0) +#define I2C_CR2_RD_WRN BIT(10) +#define I2C_CR2_RD_WRN_OFFSET 10U +#define I2C_CR2_ADD10 BIT(11) +#define I2C_CR2_HEAD10R BIT(12) +#define I2C_CR2_START BIT(13) +#define I2C_CR2_STOP BIT(14) +#define I2C_CR2_NACK BIT(15) +#define I2C_CR2_NBYTES GENMASK_32(23, 16) +#define I2C_CR2_NBYTES_OFFSET 16U +#define I2C_CR2_RELOAD BIT(24) +#define I2C_CR2_AUTOEND BIT(25) +#define I2C_CR2_PECBYTE BIT(26) + +/* Bit definition for I2C_OAR1 register */ +#define I2C_OAR1_OA1 GENMASK_32(9, 0) +#define I2C_OAR1_OA1MODE BIT(10) +#define I2C_OAR1_OA1EN BIT(15) + +/* Bit definition for I2C_OAR2 register */ +#define I2C_OAR2_OA2 GENMASK_32(7, 1) +#define I2C_OAR2_OA2MSK GENMASK_32(10, 8) +#define I2C_OAR2_OA2NOMASK 0 +#define I2C_OAR2_OA2MASK01 BIT(8) +#define I2C_OAR2_OA2MASK02 BIT(9) +#define I2C_OAR2_OA2MASK03 GENMASK_32(9, 8) +#define I2C_OAR2_OA2MASK04 BIT(10) +#define I2C_OAR2_OA2MASK05 (BIT(8) | BIT(10)) +#define I2C_OAR2_OA2MASK06 (BIT(9) | BIT(10)) +#define I2C_OAR2_OA2MASK07 GENMASK_32(10, 8) +#define I2C_OAR2_OA2EN BIT(15) + +/* Bit definition for I2C_TIMINGR register */ +#define I2C_TIMINGR_SCLL GENMASK_32(7, 0) +#define I2C_TIMINGR_SCLH GENMASK_32(15, 8) +#define I2C_TIMINGR_SDADEL GENMASK_32(19, 16) +#define I2C_TIMINGR_SCLDEL GENMASK_32(23, 20) +#define I2C_TIMINGR_PRESC GENMASK_32(31, 28) +#define I2C_TIMINGR_SCLL_MAX (I2C_TIMINGR_SCLL + 1) +#define I2C_TIMINGR_SCLH_MAX ((I2C_TIMINGR_SCLH >> 8) + 1) +#define I2C_TIMINGR_SDADEL_MAX ((I2C_TIMINGR_SDADEL >> 16) + 1) +#define I2C_TIMINGR_SCLDEL_MAX ((I2C_TIMINGR_SCLDEL >> 20) + 1) +#define I2C_TIMINGR_PRESC_MAX ((I2C_TIMINGR_PRESC >> 28) + 1) +#define I2C_SET_TIMINGR_SCLL(n) ((n) & \ + (I2C_TIMINGR_SCLL_MAX - 1)) +#define I2C_SET_TIMINGR_SCLH(n) (((n) & \ + (I2C_TIMINGR_SCLH_MAX - 1)) << 8) +#define I2C_SET_TIMINGR_SDADEL(n) (((n) & \ + (I2C_TIMINGR_SDADEL_MAX - 1)) << 16) +#define I2C_SET_TIMINGR_SCLDEL(n) (((n) & \ + (I2C_TIMINGR_SCLDEL_MAX - 1)) << 20) +#define I2C_SET_TIMINGR_PRESC(n) (((n) & \ + (I2C_TIMINGR_PRESC_MAX - 1)) << 28) + +/* Bit definition for I2C_TIMEOUTR register */ +#define I2C_TIMEOUTR_TIMEOUTA GENMASK_32(11, 0) +#define I2C_TIMEOUTR_TIDLE BIT(12) +#define I2C_TIMEOUTR_TIMOUTEN BIT(15) +#define I2C_TIMEOUTR_TIMEOUTB GENMASK_32(27, 16) +#define I2C_TIMEOUTR_TEXTEN BIT(31) + +/* Bit definition for I2C_ISR register */ +#define I2C_ISR_TXE BIT(0) +#define I2C_ISR_TXIS BIT(1) +#define I2C_ISR_RXNE BIT(2) +#define I2C_ISR_ADDR BIT(3) +#define I2C_ISR_NACKF BIT(4) +#define I2C_ISR_STOPF BIT(5) +#define I2C_ISR_TC BIT(6) +#define I2C_ISR_TCR BIT(7) +#define I2C_ISR_BERR BIT(8) +#define I2C_ISR_ARLO BIT(9) +#define I2C_ISR_OVR BIT(10) +#define I2C_ISR_PECERR BIT(11) +#define I2C_ISR_TIMEOUT BIT(12) +#define I2C_ISR_ALERT BIT(13) +#define I2C_ISR_BUSY BIT(15) +#define I2C_ISR_DIR BIT(16) +#define I2C_ISR_ADDCODE GENMASK_32(23, 17) + +/* Bit definition for I2C_ICR register */ +#define I2C_ICR_ADDRCF BIT(3) +#define I2C_ICR_NACKCF BIT(4) +#define I2C_ICR_STOPCF BIT(5) +#define I2C_ICR_BERRCF BIT(8) +#define I2C_ICR_ARLOCF BIT(9) +#define I2C_ICR_OVRCF BIT(10) +#define I2C_ICR_PECCF BIT(11) +#define I2C_ICR_TIMOUTCF BIT(12) +#define I2C_ICR_ALERTCF BIT(13) + +/* Max data size for a single I2C transfer */ +#define MAX_NBYTE_SIZE 255U + +#define I2C_NSEC_PER_SEC 1000000000UL +#define I2C_TIMEOUT_BUSY_MS 25 +#define I2C_TIMEOUT_BUSY_US (I2C_TIMEOUT_BUSY_MS * 1000) +#define I2C_TIMEOUT_RXNE_MS 5 + +#define I2C_TIMEOUT_DEFAULT_MS 100 + +#define CR2_RESET_MASK (I2C_CR2_SADD | I2C_CR2_HEAD10R | \ + I2C_CR2_NBYTES | I2C_CR2_RELOAD | \ + I2C_CR2_RD_WRN) + +#define TIMINGR_CLEAR_MASK (I2C_TIMINGR_SCLL | I2C_TIMINGR_SCLH | \ + I2C_TIMINGR_SDADEL | \ + I2C_TIMINGR_SCLDEL | I2C_TIMINGR_PRESC) + +/* + * I2C transfer modes + * I2C_RELOAD: Enable Reload mode + * I2C_AUTOEND_MODE: Enable automatic end mode + * I2C_SOFTEND_MODE: Enable software end mode + */ +#define I2C_RELOAD_MODE I2C_CR2_RELOAD +#define I2C_AUTOEND_MODE I2C_CR2_AUTOEND +#define I2C_SOFTEND_MODE 0x0 + +/* + * Start/restart/stop I2C transfer requests. + * + * I2C_NO_STARTSTOP: Don't Generate stop and start condition + * I2C_GENERATE_STOP: Generate stop condition (size should be set to 0) + * I2C_GENERATE_START_READ: Generate Restart for read request. + * I2C_GENERATE_START_WRITE: Generate Restart for write request + */ +#define I2C_NO_STARTSTOP 0x0 +#define I2C_GENERATE_STOP (BIT(31) | I2C_CR2_STOP) +#define I2C_GENERATE_START_READ (BIT(31) | I2C_CR2_START | \ + I2C_CR2_RD_WRN) +#define I2C_GENERATE_START_WRITE (BIT(31) | I2C_CR2_START) + +/* Memory address byte sizes */ +#define I2C_MEMADD_SIZE_8BIT 1 +#define I2C_MEMADD_SIZE_16BIT 2 + +/* Effective rate cannot be lower than 80% target rate */ +#define RATE_MIN(rate) (((rate) * 80U) / 100U) + +/* + * struct i2c_spec_s - Private I2C timing specifications. + * @rate: I2C bus speed (Hz) + * @fall_max: Max fall time of both SDA and SCL signals (ns) + * @rise_max: Max rise time of both SDA and SCL signals (ns) + * @hddat_min: Min data hold time (ns) + * @vddat_max: Max data valid time (ns) + * @sudat_min: Min data setup time (ns) + * @l_min: Min low period of the SCL clock (ns) + * @h_min: Min high period of the SCL clock (ns) + */ +struct i2c_spec_s { + uint32_t rate; + uint32_t fall_max; + uint32_t rise_max; + uint32_t hddat_min; + uint32_t vddat_max; + uint32_t sudat_min; + uint32_t l_min; + uint32_t h_min; +}; + +/* + * struct i2c_timing_s - Private I2C output parameters. + * @scldel: Data setup time + * @sdadel: Data hold time + * @sclh: SCL high period (master mode) + * @sclh: SCL low period (master mode) + * @is_saved: True if relating to a configuration candidate + */ +struct i2c_timing_s { + uint8_t scldel; + uint8_t sdadel; + uint8_t sclh; + uint8_t scll; + bool is_saved; +}; + +/* This table must be sorted in increasing value for field @rate */ +static const struct i2c_spec_s i2c_specs[] = { + /* Standard - 100KHz */ + { + .rate = I2C_STANDARD_RATE, + .fall_max = 300, + .rise_max = 1000, + .hddat_min = 0, + .vddat_max = 3450, + .sudat_min = 250, + .l_min = 4700, + .h_min = 4000, + }, + /* Fast - 400KHz */ + { + .rate = I2C_FAST_RATE, + .fall_max = 300, + .rise_max = 300, + .hddat_min = 0, + .vddat_max = 900, + .sudat_min = 100, + .l_min = 1300, + .h_min = 600, + }, + /* FastPlus - 1MHz */ + { + .rate = I2C_FAST_PLUS_RATE, + .fall_max = 100, + .rise_max = 120, + .hddat_min = 0, + .vddat_max = 450, + .sudat_min = 50, + .l_min = 500, + .h_min = 260, + }, +}; + +/* + * I2C request parameters + * @dev_addr: I2C address of the target device + * @mode: Communication mode, one of I2C_MODE_(MASTER|MEM) + * @mem_addr: Target memory cell accessed in device (memory mode) + * @mem_addr_size: Byte size of the memory cell address (memory mode) + * @timeout_ms: Timeout in millisenconds for the request + */ +struct i2c_request { + uint32_t dev_addr; + enum i2c_mode_e mode; + uint32_t mem_addr; + uint32_t mem_addr_size; + unsigned int timeout_ms; +}; + +static vaddr_t get_base(struct i2c_handle_s *hi2c) +{ + return io_pa_or_va_secure(&hi2c->base, hi2c->reg_size); +} + +static void notif_i2c_timeout(struct i2c_handle_s *hi2c) +{ + hi2c->i2c_err |= I2C_ERROR_TIMEOUT; + hi2c->i2c_state = I2C_STATE_READY; +} + +static const struct i2c_spec_s *get_specs(uint32_t rate) +{ + size_t i = 0; + + for (i = 0; i < ARRAY_SIZE(i2c_specs); i++) + if (rate <= i2c_specs[i].rate) + return i2c_specs + i; + + return NULL; +} + +static void save_cfg(struct i2c_handle_s *hi2c, struct i2c_cfg *cfg) +{ + vaddr_t base = get_base(hi2c); + + clk_enable(hi2c->clock); + + cfg->cr1 = io_read32(base + I2C_CR1); + cfg->cr2 = io_read32(base + I2C_CR2); + cfg->oar1 = io_read32(base + I2C_OAR1); + cfg->oar2 = io_read32(base + I2C_OAR2); + cfg->timingr = io_read32(base + I2C_TIMINGR); + + clk_disable(hi2c->clock); +} + +static void restore_cfg(struct i2c_handle_s *hi2c, struct i2c_cfg *cfg) +{ + vaddr_t base = get_base(hi2c); + + clk_enable(hi2c->clock); + + io_clrbits32(base + I2C_CR1, I2C_CR1_PE); + io_write32(base + I2C_TIMINGR, cfg->timingr & TIMINGR_CLEAR_MASK); + io_write32(base + I2C_OAR1, cfg->oar1); + io_write32(base + I2C_CR2, cfg->cr2); + io_write32(base + I2C_OAR2, cfg->oar2); + io_write32(base + I2C_CR1, cfg->cr1 & ~I2C_CR1_PE); + io_setbits32(base + I2C_CR1, cfg->cr1 & I2C_CR1_PE); + + clk_disable(hi2c->clock); +} + +static void __maybe_unused dump_cfg(struct i2c_cfg *cfg __maybe_unused) +{ + DMSG("CR1: %#"PRIx32, cfg->cr1); + DMSG("CR2: %#"PRIx32, cfg->cr2); + DMSG("OAR1: %#"PRIx32, cfg->oar1); + DMSG("OAR2: %#"PRIx32, cfg->oar2); + DMSG("TIM: %#"PRIx32, cfg->timingr); +} + +static void __maybe_unused dump_i2c(struct i2c_handle_s *hi2c) +{ + vaddr_t __maybe_unused base = get_base(hi2c); + + clk_enable(hi2c->clock); + + DMSG("CR1: %#"PRIx32, io_read32(base + I2C_CR1)); + DMSG("CR2: %#"PRIx32, io_read32(base + I2C_CR2)); + DMSG("OAR1: %#"PRIx32, io_read32(base + I2C_OAR1)); + DMSG("OAR2: %#"PRIx32, io_read32(base + I2C_OAR2)); + DMSG("TIM: %#"PRIx32, io_read32(base + I2C_TIMINGR)); + + clk_disable(hi2c->clock); +} + +/* + * Compute the I2C device timings + * + * @init: Ref to the initialization configuration structure + * @clock_src: I2C clock source frequency (Hz) + * @timing: Pointer to the final computed timing result + * Return 0 on success or a negative value + */ +static int i2c_compute_timing(struct stm32_i2c_init_s *init, + unsigned long clock_src, uint32_t *timing) +{ + const struct i2c_spec_s *specs = NULL; + uint32_t speed_freq = 0; + uint32_t i2cbus = UDIV_ROUND_NEAREST(I2C_NSEC_PER_SEC, speed_freq); + uint32_t i2cclk = UDIV_ROUND_NEAREST(I2C_NSEC_PER_SEC, clock_src); + uint32_t p_prev = I2C_TIMINGR_PRESC_MAX; + uint32_t af_delay_min = 0; + uint32_t af_delay_max = 0; + uint32_t dnf_delay = 0; + uint32_t tsync = 0; + uint32_t clk_min = 0; + uint32_t clk_max = 0; + int clk_error_prev = 0; + uint16_t p = 0; + uint16_t l = 0; + uint16_t a = 0; + uint16_t h = 0; + unsigned int sdadel_min = 0; + unsigned int sdadel_max = 0; + unsigned int scldel_min = 0; + unsigned int delay = 0; + int s = -1; + struct i2c_timing_s solutions[I2C_TIMINGR_PRESC_MAX] = { 0 }; + + specs = get_specs(init->bus_rate); + if (!specs) { + DMSG("I2C speed out of bound: %"PRId32"Hz", init->bus_rate); + return -1; + } + + speed_freq = specs->rate; + i2cbus = UDIV_ROUND_NEAREST(I2C_NSEC_PER_SEC, speed_freq); + clk_error_prev = INT_MAX; + + if (init->rise_time > specs->rise_max || + init->fall_time > specs->fall_max) { + DMSG("I2C rise{%"PRId32">%"PRId32"}/fall{%"PRId32">%"PRId32"}", + init->rise_time, specs->rise_max, + init->fall_time, specs->fall_max); + return -1; + } + + if (init->digital_filter_coef > STM32_I2C_DIGITAL_FILTER_MAX) { + DMSG("DNF out of bound %"PRId8"/%d", + init->digital_filter_coef, STM32_I2C_DIGITAL_FILTER_MAX); + return -1; + } + + /* Analog and Digital Filters */ + if (init->analog_filter) { + af_delay_min = STM32_I2C_ANALOG_FILTER_DELAY_MIN; + af_delay_max = STM32_I2C_ANALOG_FILTER_DELAY_MAX; + } + dnf_delay = init->digital_filter_coef * i2cclk; + + sdadel_min = specs->hddat_min + init->fall_time; + delay = af_delay_min - ((init->digital_filter_coef + 3) * i2cclk); + if (SUB_OVERFLOW(sdadel_min, delay, &sdadel_min)) + sdadel_min = 0; + + sdadel_max = specs->vddat_max - init->rise_time; + delay = af_delay_max - ((init->digital_filter_coef + 4) * i2cclk); + if (SUB_OVERFLOW(sdadel_max, delay, &sdadel_max)) + sdadel_max = 0; + + scldel_min = init->rise_time + specs->sudat_min; + + DMSG("I2C SDADEL(min/max): %u/%u, SCLDEL(Min): %u", + sdadel_min, sdadel_max, scldel_min); + + /* Compute possible values for PRESC, SCLDEL and SDADEL */ + for (p = 0; p < I2C_TIMINGR_PRESC_MAX; p++) { + for (l = 0; l < I2C_TIMINGR_SCLDEL_MAX; l++) { + uint32_t scldel = (l + 1) * (p + 1) * i2cclk; + + if (scldel < scldel_min) + continue; + + for (a = 0; a < I2C_TIMINGR_SDADEL_MAX; a++) { + uint32_t sdadel = (a * (p + 1) + 1) * i2cclk; + + if ((sdadel >= sdadel_min) && + (sdadel <= sdadel_max) && + (p != p_prev)) { + solutions[p].scldel = l; + solutions[p].sdadel = a; + solutions[p].is_saved = true; + p_prev = p; + break; + } + } + + if (p_prev == p) + break; + } + } + + if (p_prev == I2C_TIMINGR_PRESC_MAX) { + DMSG("I2C no Prescaler solution"); + return -1; + } + + tsync = af_delay_min + dnf_delay + (2 * i2cclk); + clk_max = I2C_NSEC_PER_SEC / RATE_MIN(specs->rate); + clk_min = I2C_NSEC_PER_SEC / specs->rate; + + /* + * Among prescaler possibilities discovered above figures out SCL Low + * and High Period. Provided: + * - SCL Low Period has to be higher than Low Period of the SCL Clock + * defined by I2C Specification. I2C Clock has to be lower than + * (SCL Low Period - Analog/Digital filters) / 4. + * - SCL High Period has to be lower than High Period of the SCL Clock + * defined by I2C Specification. + * - I2C Clock has to be lower than SCL High Period. + */ + for (p = 0; p < I2C_TIMINGR_PRESC_MAX; p++) { + uint32_t prescaler = (p + 1) * i2cclk; + + if (!solutions[p].is_saved) + continue; + + for (l = 0; l < I2C_TIMINGR_SCLL_MAX; l++) { + uint32_t tscl_l = ((l + 1) * prescaler) + tsync; + + if (tscl_l < specs->l_min || + i2cclk >= ((tscl_l - af_delay_min - dnf_delay) / 4)) + continue; + + for (h = 0; h < I2C_TIMINGR_SCLH_MAX; h++) { + uint32_t tscl_h = ((h + 1) * prescaler) + tsync; + uint32_t tscl = tscl_l + tscl_h + + init->rise_time + + init->fall_time; + + if (tscl >= clk_min && tscl <= clk_max && + tscl_h >= specs->h_min && i2cclk < tscl_h) { + int clk_error = tscl - i2cbus; + + if (clk_error < 0) + clk_error = -clk_error; + + if (clk_error < clk_error_prev) { + clk_error_prev = clk_error; + solutions[p].scll = l; + solutions[p].sclh = h; + s = p; + } + } + } + } + } + + if (s < 0) { + DMSG("I2C no solution at all"); + return -1; + } + + /* Finalize timing settings */ + *timing = I2C_SET_TIMINGR_PRESC(s) | + I2C_SET_TIMINGR_SCLDEL(solutions[s].scldel) | + I2C_SET_TIMINGR_SDADEL(solutions[s].sdadel) | + I2C_SET_TIMINGR_SCLH(solutions[s].sclh) | + I2C_SET_TIMINGR_SCLL(solutions[s].scll); + + DMSG("I2C TIMINGR (PRESC/SCLDEL/SDADEL): %i/%"PRIu8"/%"PRIu8, + s, solutions[s].scldel, solutions[s].sdadel); + DMSG("I2C TIMINGR (SCLH/SCLL): %"PRIu8"/%"PRIu8, + solutions[s].sclh, solutions[s].scll); + DMSG("I2C TIMINGR: 0x%"PRIx32, *timing); + + return 0; +} + +/* i2c_specs[] must be sorted by increasing rate */ +static bool __maybe_unused i2c_specs_is_consistent(void) +{ + size_t i = 0; + + COMPILE_TIME_ASSERT(ARRAY_SIZE(i2c_specs)); + + for (i = 1; i < ARRAY_SIZE(i2c_specs); i++) + if (i2c_specs[i - 1].rate >= i2c_specs[i].rate) + return false; + + return true; +} + +/* + * @brief From requested rate, get the closest I2C rate without exceeding it, + * within I2C specification values defined in @i2c_specs. + * @param rate: The requested rate. + * @retval Found rate, else the lowest value supported by platform. + */ +static uint32_t get_lower_rate(uint32_t rate) +{ + size_t i = 0; + + for (i = ARRAY_SIZE(i2c_specs); i > 0; i--) + if (rate > i2c_specs[i - 1].rate) + return i2c_specs[i - 1].rate; + + return i2c_specs[0].rate; +} + +/* + * Setup the I2C device timings + * + * @hi2c: I2C handle structure + * @init: Ref to the initialization configuration structure + * @timing: Output TIMINGR register configuration value + * @retval 0 if OK, negative value else + */ +static int i2c_setup_timing(struct i2c_handle_s *hi2c, + struct stm32_i2c_init_s *init, + uint32_t *timing) +{ + int rc = 0; + unsigned long clock_src = 0; + + assert(i2c_specs_is_consistent()); + + clock_src = clk_get_rate(hi2c->clock); + if (!clock_src) { + DMSG("Null I2C clock rate"); + return -1; + } + + /* + * If the timing has already been computed, and the frequency is the + * same as when it was computed, then use the saved timing. + */ + if (clock_src == hi2c->saved_frequency) { + *timing = hi2c->saved_timing; + return 0; + } + + do { + rc = i2c_compute_timing(init, clock_src, timing); + if (rc) { + DMSG("Failed to compute I2C timings"); + if (init->bus_rate > I2C_STANDARD_RATE) { + init->bus_rate = get_lower_rate(init->bus_rate); + IMSG("Downgrade I2C speed to %"PRIu32"Hz)", + init->bus_rate); + } else { + break; + } + } + } while (rc); + + if (rc) { + DMSG("Impossible to compute I2C timings"); + return rc; + } + + DMSG("I2C Freq(%"PRIu32"Hz), Clk Source(%lu)", + init->bus_rate, clock_src); + DMSG("I2C Rise(%"PRId32") and Fall(%"PRId32") Time", + init->rise_time, init->fall_time); + DMSG("I2C Analog Filter(%s), DNF(%"PRIu8")", + init->analog_filter ? "On" : "Off", init->digital_filter_coef); + + hi2c->saved_timing = *timing; + hi2c->saved_frequency = clock_src; + + return 0; +} + +/* + * Configure I2C Analog noise filter. + * @hi2c: I2C handle structure + * @analog_filter_on: True if enabling analog filter, false otherwise + * Return 0 on success or a negative value + */ +static int i2c_config_analog_filter(struct i2c_handle_s *hi2c, + bool analog_filter_on) +{ + vaddr_t base = get_base(hi2c); + + if (hi2c->i2c_state != I2C_STATE_READY) + return -1; + + hi2c->i2c_state = I2C_STATE_BUSY; + + /* Disable the selected I2C peripheral */ + io_clrbits32(base + I2C_CR1, I2C_CR1_PE); + + /* Reset I2Cx ANOFF bit */ + io_clrbits32(base + I2C_CR1, I2C_CR1_ANFOFF); + + /* Set analog filter bit if filter is disabled */ + if (!analog_filter_on) + io_setbits32(base + I2C_CR1, I2C_CR1_ANFOFF); + + /* Enable the selected I2C peripheral */ + io_setbits32(base + I2C_CR1, I2C_CR1_PE); + + hi2c->i2c_state = I2C_STATE_READY; + + return 0; +} + +TEE_Result stm32_i2c_get_setup_from_fdt(void *fdt, int node, + struct stm32_i2c_init_s *init, + struct pinctrl_state **pinctrl, + struct pinctrl_state **pinctrl_sleep) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const fdt32_t *cuint = NULL; + struct dt_node_info info = { .status = 0 }; + int __maybe_unused count = 0; + + /* Default STM32 specific configs caller may need to overwrite */ + memset(init, 0, sizeof(*init)); + + fdt_fill_device_info(fdt, &info, node); + assert(info.reg != DT_INFO_INVALID_REG && + info.reg_size != DT_INFO_INVALID_REG_SIZE); + + init->dt_status = info.status; + init->pbase = info.reg; + init->reg_size = info.reg_size; + + res = clk_dt_get_by_index(fdt, node, 0, &init->clock); + if (res) + return res; + + cuint = fdt_getprop(fdt, node, "i2c-scl-rising-time-ns", NULL); + if (cuint) + init->rise_time = fdt32_to_cpu(*cuint); + else + init->rise_time = STM32_I2C_RISE_TIME_DEFAULT; + + cuint = fdt_getprop(fdt, node, "i2c-scl-falling-time-ns", NULL); + if (cuint) + init->fall_time = fdt32_to_cpu(*cuint); + else + init->fall_time = STM32_I2C_FALL_TIME_DEFAULT; + + cuint = fdt_getprop(fdt, node, "clock-frequency", NULL); + if (cuint) { + init->bus_rate = fdt32_to_cpu(*cuint); + + if (init->bus_rate > I2C_FAST_PLUS_RATE) { + DMSG("Invalid bus speed (%"PRIu32" > %i)", + init->bus_rate, I2C_FAST_PLUS_RATE); + return TEE_ERROR_GENERIC; + } + } else { + init->bus_rate = I2C_STANDARD_RATE; + } + + if (pinctrl) { + res = pinctrl_get_state_by_name(fdt, node, "default", pinctrl); + if (res) + return res; + } + + if (pinctrl_sleep) { + res = pinctrl_get_state_by_name(fdt, node, "sleep", + pinctrl_sleep); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = TEE_SUCCESS; + if (res) + return res; + } + + return TEE_SUCCESS; +} + +int stm32_i2c_init(struct i2c_handle_s *hi2c, + struct stm32_i2c_init_s *init_data) +{ + int rc = 0; + uint32_t timing = 0; + vaddr_t base = 0; + uint32_t val = 0; + + rc = i2c_setup_timing(hi2c, init_data, &timing); + if (rc) + return rc; + + clk_enable(hi2c->clock); + + base = get_base(hi2c); + hi2c->i2c_state = I2C_STATE_BUSY; + + /* Disable the selected I2C peripheral */ + io_clrbits32(base + I2C_CR1, I2C_CR1_PE); + + /* Configure I2Cx: Frequency range */ + io_write32(base + I2C_TIMINGR, timing & TIMINGR_CLEAR_MASK); + + /* Disable Own Address1 before set the Own Address1 configuration */ + io_write32(base + I2C_OAR1, 0); + + /* Configure I2Cx: Own Address1 and ack own address1 mode */ + if (init_data->addr_mode_10b_not_7b) + io_write32(base + I2C_OAR1, + I2C_OAR1_OA1EN | I2C_OAR1_OA1MODE | + init_data->own_address1); + else + io_write32(base + I2C_OAR1, + I2C_OAR1_OA1EN | init_data->own_address1); + + /* Configure I2Cx: Addressing Master mode */ + io_write32(base + I2C_CR2, 0); + if (init_data->addr_mode_10b_not_7b) + io_setbits32(base + I2C_CR2, I2C_CR2_ADD10); + + /* + * Enable the AUTOEND by default, and enable NACK + * (should be disabled only during Slave process). + */ + io_setbits32(base + I2C_CR2, I2C_CR2_AUTOEND | I2C_CR2_NACK); + + /* Disable Own Address2 before set the Own Address2 configuration */ + io_write32(base + I2C_OAR2, 0); + + /* Configure I2Cx: Dual mode and Own Address2 */ + if (init_data->dual_address_mode) + io_write32(base + I2C_OAR2, + I2C_OAR2_OA2EN | init_data->own_address2 | + (init_data->own_address2_masks << 8)); + + /* Configure I2Cx: Generalcall and NoStretch mode */ + val = 0; + if (init_data->general_call_mode) + val |= I2C_CR1_GCEN; + if (init_data->no_stretch_mode) + val |= I2C_CR1_NOSTRETCH; + io_write32(base + I2C_CR1, val); + + /* Enable the selected I2C peripheral */ + io_setbits32(base + I2C_CR1, I2C_CR1_PE); + + hi2c->i2c_err = I2C_ERROR_NONE; + hi2c->i2c_state = I2C_STATE_READY; + + rc = i2c_config_analog_filter(hi2c, init_data->analog_filter); + if (rc) + DMSG("I2C analog filter error %d", rc); + + if (IS_ENABLED(CFG_STM32MP13)) + stm32_pinctrl_set_secure_cfg(hi2c->pinctrl, true); + + clk_disable(hi2c->clock); + + return rc; +} + +/* I2C transmit (TX) data register flush sequence */ +static void i2c_flush_txdr(struct i2c_handle_s *hi2c) +{ + vaddr_t base = get_base(hi2c); + + /* + * If a pending TXIS flag is set, + * write a dummy data in TXDR to clear it. + */ + if (io_read32(base + I2C_ISR) & I2C_ISR_TXIS) + io_write32(base + I2C_TXDR, 0); + + /* Flush TX register if not empty */ + if ((io_read32(base + I2C_ISR) & I2C_ISR_TXE) == 0) + io_setbits32(base + I2C_ISR, I2C_ISR_TXE); +} + +/* + * Wait for a single target I2C_ISR bit to reach an awaited value (0 or 1) + * + * @hi2c: I2C handle structure + * @bit_mask: Bit mask for the target single bit position to consider + * @awaited_value: Awaited value of the target bit in I2C_ISR, 0 or 1 + * @timeout_ref: Expriation timeout reference + * Return 0 on success and a non-zero value on timeout + */ +static int wait_isr_event(struct i2c_handle_s *hi2c, uint32_t bit_mask, + unsigned int awaited_value, uint64_t timeout_ref) +{ + vaddr_t isr = get_base(hi2c) + I2C_ISR; + + assert(IS_POWER_OF_TWO(bit_mask) && !(awaited_value & ~1U)); + + /* May timeout while TEE thread is suspended */ + while (!timeout_elapsed(timeout_ref)) + if (!!(io_read32(isr) & bit_mask) == awaited_value) + break; + + if (!!(io_read32(isr) & bit_mask) == awaited_value) + return 0; + + notif_i2c_timeout(hi2c); + return -1; +} + +/* Handle Acknowledge-Failed sequence detection during an I2C Communication */ +static int i2c_ack_failed(struct i2c_handle_s *hi2c, uint64_t timeout_ref) +{ + vaddr_t base = get_base(hi2c); + + if ((io_read32(base + I2C_ISR) & I2C_ISR_NACKF) == 0U) + return 0; + + /* + * Wait until STOP Flag is reset. Use polling method. + * AutoEnd should be initiate after AF. + * Timeout may elpased while TEE thread is suspended. + */ + while (!timeout_elapsed(timeout_ref)) + if (io_read32(base + I2C_ISR) & I2C_ISR_STOPF) + break; + + if ((io_read32(base + I2C_ISR) & I2C_ISR_STOPF) == 0) { + notif_i2c_timeout(hi2c); + return -1; + } + + io_write32(base + I2C_ICR, I2C_ISR_NACKF); + + io_write32(base + I2C_ICR, I2C_ISR_STOPF); + + i2c_flush_txdr(hi2c); + + io_clrbits32(base + I2C_CR2, CR2_RESET_MASK); + + hi2c->i2c_err |= I2C_ERROR_ACKF; + hi2c->i2c_state = I2C_STATE_READY; + + return -1; +} + +/* Wait TXIS bit is 1 in I2C_ISR register */ +static int i2c_wait_txis(struct i2c_handle_s *hi2c, uint64_t timeout_ref) +{ + while (!timeout_elapsed(timeout_ref)) { + if (io_read32(get_base(hi2c) + I2C_ISR) & I2C_ISR_TXIS) + break; + if (i2c_ack_failed(hi2c, timeout_ref)) + return -1; + } + + if (io_read32(get_base(hi2c) + I2C_ISR) & I2C_ISR_TXIS) + return 0; + + if (i2c_ack_failed(hi2c, timeout_ref)) + return -1; + + notif_i2c_timeout(hi2c); + return -1; +} + +/* Wait STOPF bit is 1 in I2C_ISR register */ +static int i2c_wait_stop(struct i2c_handle_s *hi2c, uint64_t timeout_ref) +{ + while (!timeout_elapsed(timeout_ref)) { + if (io_read32(get_base(hi2c) + I2C_ISR) & I2C_ISR_STOPF) + break; + + if (i2c_ack_failed(hi2c, timeout_ref)) + return -1; + } + + if (io_read32(get_base(hi2c) + I2C_ISR) & I2C_ISR_STOPF) + return 0; + + if (i2c_ack_failed(hi2c, timeout_ref)) + return -1; + + notif_i2c_timeout(hi2c); + return -1; +} + +/* + * Load I2C_CR2 register for a I2C transfer + * + * @hi2c: I2C handle structure + * @dev_addr: Slave address to be transferred + * @size: Number of bytes to be transferred + * @i2c_mode: One of I2C_{RELOAD|AUTOEND|SOFTEND}_MODE: Enable Reload mode. + * @startstop: One of I2C_NO_STARTSTOP, I2C_GENERATE_STOP, + * I2C_GENERATE_START_{READ|WRITE} + */ +static void i2c_transfer_config(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint32_t size, uint32_t i2c_mode, + uint32_t startstop) +{ + uint32_t clr_value = I2C_CR2_SADD | I2C_CR2_NBYTES | I2C_CR2_RELOAD | + I2C_CR2_AUTOEND | I2C_CR2_START | I2C_CR2_STOP | + (I2C_CR2_RD_WRN & + (startstop >> (31U - I2C_CR2_RD_WRN_OFFSET))); + uint32_t set_value = (dev_addr & I2C_CR2_SADD) | + ((size << I2C_CR2_NBYTES_OFFSET) & + I2C_CR2_NBYTES) | + i2c_mode | startstop; + + io_clrsetbits32(get_base(hi2c) + I2C_CR2, clr_value, set_value); +} + +/* + * Master sends target device address followed by internal memory + * address for a memory write request. + * Function returns 0 on success or a negative value. + */ +static int i2c_request_mem_write(struct i2c_handle_s *hi2c, + struct i2c_request *request, + uint64_t timeout_ref) +{ + vaddr_t base = get_base(hi2c); + + i2c_transfer_config(hi2c, request->dev_addr, request->mem_addr_size, + I2C_RELOAD_MODE, I2C_GENERATE_START_WRITE); + + if (i2c_wait_txis(hi2c, timeout_ref)) + return -1; + + if (request->mem_addr_size == I2C_MEMADD_SIZE_8BIT) { + /* Send memory address */ + io_write8(base + I2C_TXDR, request->mem_addr & 0x00FFU); + } else { + /* Send MSB of memory address */ + io_write8(base + I2C_TXDR, (request->mem_addr & 0xFF00U) >> 8); + + if (i2c_wait_txis(hi2c, timeout_ref)) + return -1; + + /* Send LSB of memory address */ + io_write8(base + I2C_TXDR, request->mem_addr & 0x00FFU); + } + + if (wait_isr_event(hi2c, I2C_ISR_TCR, 1, timeout_ref)) + return -1; + + return 0; +} + +/* + * Master sends target device address followed by internal memory + * address to prepare a memory read request. + * Function returns 0 on success or a negative value. + */ +static int i2c_request_mem_read(struct i2c_handle_s *hi2c, + struct i2c_request *request, + uint64_t timeout_ref) +{ + vaddr_t base = get_base(hi2c); + + i2c_transfer_config(hi2c, request->dev_addr, request->mem_addr_size, + I2C_SOFTEND_MODE, I2C_GENERATE_START_WRITE); + + if (i2c_wait_txis(hi2c, timeout_ref)) + return -1; + + if (request->mem_addr_size == I2C_MEMADD_SIZE_8BIT) { + /* Send memory address */ + io_write8(base + I2C_TXDR, request->mem_addr & 0x00FFU); + } else { + /* Send MSB of memory address */ + io_write8(base + I2C_TXDR, (request->mem_addr & 0xFF00U) >> 8); + + if (i2c_wait_txis(hi2c, timeout_ref)) + return -1; + + /* Send LSB of memory address */ + io_write8(base + I2C_TXDR, request->mem_addr & 0x00FFU); + } + + if (wait_isr_event(hi2c, I2C_ISR_TC, 1, timeout_ref)) + return -1; + + return 0; +} + +/* + * Write an amount of data in blocking mode + * + * @hi2c: Reference to struct i2c_handle_s + * @request: I2C request parameters + * @p_data: Pointer to data buffer + * @size: Amount of data to be sent + * Return 0 on success or a negative value + */ +static int do_write(struct i2c_handle_s *hi2c, struct i2c_request *request, + uint8_t *p_data, uint16_t size) +{ + uint64_t timeout_ref = 0; + vaddr_t base = get_base(hi2c); + int rc = -1; + uint8_t *p_buff = p_data; + size_t xfer_size = 0; + size_t xfer_count = size; + + if (request->mode != I2C_MODE_MASTER && request->mode != I2C_MODE_MEM) + return -1; + + if (hi2c->i2c_state != I2C_STATE_READY) + return -1; + + if (!p_data || !size) + return -1; + + clk_enable(hi2c->clock); + + timeout_ref = timeout_init_us(I2C_TIMEOUT_BUSY_MS * 1000); + if (wait_isr_event(hi2c, I2C_ISR_BUSY, 0, timeout_ref)) + goto bail; + + hi2c->i2c_state = I2C_STATE_BUSY_TX; + hi2c->i2c_err = I2C_ERROR_NONE; + timeout_ref = timeout_init_us(request->timeout_ms * 1000); + + if (request->mode == I2C_MODE_MEM) { + /* In memory mode, send slave address and memory address */ + if (i2c_request_mem_write(hi2c, request, timeout_ref)) + goto bail; + + if (xfer_count > MAX_NBYTE_SIZE) { + xfer_size = MAX_NBYTE_SIZE; + i2c_transfer_config(hi2c, request->dev_addr, xfer_size, + I2C_RELOAD_MODE, I2C_NO_STARTSTOP); + } else { + xfer_size = xfer_count; + i2c_transfer_config(hi2c, request->dev_addr, xfer_size, + I2C_AUTOEND_MODE, I2C_NO_STARTSTOP); + } + } else { + /* In master mode, send slave address */ + if (xfer_count > MAX_NBYTE_SIZE) { + xfer_size = MAX_NBYTE_SIZE; + i2c_transfer_config(hi2c, request->dev_addr, xfer_size, + I2C_RELOAD_MODE, + I2C_GENERATE_START_WRITE); + } else { + xfer_size = xfer_count; + i2c_transfer_config(hi2c, request->dev_addr, xfer_size, + I2C_AUTOEND_MODE, + I2C_GENERATE_START_WRITE); + } + } + + do { + if (i2c_wait_txis(hi2c, timeout_ref)) + goto bail; + + io_write8(base + I2C_TXDR, *p_buff); + p_buff++; + xfer_count--; + xfer_size--; + + if (xfer_count && !xfer_size) { + /* Wait until TCR flag is set */ + if (wait_isr_event(hi2c, I2C_ISR_TCR, 1, timeout_ref)) + goto bail; + + if (xfer_count > MAX_NBYTE_SIZE) { + xfer_size = MAX_NBYTE_SIZE; + i2c_transfer_config(hi2c, request->dev_addr, + xfer_size, + I2C_RELOAD_MODE, + I2C_NO_STARTSTOP); + } else { + xfer_size = xfer_count; + i2c_transfer_config(hi2c, request->dev_addr, + xfer_size, + I2C_AUTOEND_MODE, + I2C_NO_STARTSTOP); + } + } + + } while (xfer_count > 0U); + + /* + * No need to Check TC flag, with AUTOEND mode the stop + * is automatically generated. + * Wait until STOPF flag is reset. + */ + if (i2c_wait_stop(hi2c, timeout_ref)) + goto bail; + + io_write32(base + I2C_ICR, I2C_ISR_STOPF); + + io_clrbits32(base + I2C_CR2, CR2_RESET_MASK); + + hi2c->i2c_state = I2C_STATE_READY; + + rc = 0; + +bail: + clk_disable(hi2c->clock); + + return rc; +} + +int stm32_i2c_mem_write(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint32_t mem_addr, uint32_t mem_addr_size, + uint8_t *p_data, size_t size, unsigned int timeout_ms) +{ + struct i2c_request request = { + .dev_addr = dev_addr, + .mode = I2C_MODE_MEM, + .mem_addr = mem_addr, + .mem_addr_size = mem_addr_size, + .timeout_ms = timeout_ms, + }; + + return do_write(hi2c, &request, p_data, size); +} + +int stm32_i2c_master_transmit(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint8_t *p_data, size_t size, + unsigned int timeout_ms) +{ + struct i2c_request request = { + .dev_addr = dev_addr, + .mode = I2C_MODE_MASTER, + .timeout_ms = timeout_ms, + }; + + return do_write(hi2c, &request, p_data, size); +} + +int stm32_i2c_read_write_membyte(struct i2c_handle_s *hi2c, uint16_t dev_addr, + unsigned int mem_addr, uint8_t *p_data, + bool write) +{ + uint64_t timeout_ref = 0; + uintptr_t base = get_base(hi2c); + int rc = -1; + uint8_t *p_buff = p_data; + uint32_t event_mask = 0; + + if (hi2c->i2c_state != I2C_STATE_READY || !p_data) + return -1; + + clk_enable(hi2c->clock); + + timeout_ref = timeout_init_us(I2C_TIMEOUT_BUSY_US); + if (wait_isr_event(hi2c, I2C_ISR_BUSY, 0, timeout_ref)) + goto bail; + + hi2c->i2c_state = write ? I2C_STATE_BUSY_TX : I2C_STATE_BUSY_RX; + hi2c->i2c_err = I2C_ERROR_NONE; + + i2c_transfer_config(hi2c, dev_addr, I2C_MEMADD_SIZE_8BIT, + write ? I2C_RELOAD_MODE : I2C_SOFTEND_MODE, + I2C_GENERATE_START_WRITE); + + timeout_ref = timeout_init_us(I2C_TIMEOUT_BUSY_US); + if (i2c_wait_txis(hi2c, timeout_ref)) + goto bail; + + io_write8(base + I2C_TXDR, mem_addr); + + if (write) + event_mask = I2C_ISR_TCR; + else + event_mask = I2C_ISR_TC; + + timeout_ref = timeout_init_us(I2C_TIMEOUT_BUSY_US); + if (wait_isr_event(hi2c, event_mask, 1, timeout_ref)) + goto bail; + + i2c_transfer_config(hi2c, dev_addr, I2C_MEMADD_SIZE_8BIT, + I2C_AUTOEND_MODE, + write ? I2C_NO_STARTSTOP : I2C_GENERATE_START_READ); + + timeout_ref = timeout_init_us(I2C_TIMEOUT_BUSY_US); + if (write) { + if (i2c_wait_txis(hi2c, timeout_ref)) + goto bail; + + io_write8(base + I2C_TXDR, *p_buff); + } else { + if (wait_isr_event(hi2c, I2C_ISR_RXNE, 1, timeout_ref)) + goto bail; + + *p_buff = io_read8(base + I2C_RXDR); + } + + timeout_ref = timeout_init_us(I2C_TIMEOUT_BUSY_US); + if (i2c_wait_stop(hi2c, timeout_ref)) + goto bail; + + io_write32(base + I2C_ICR, I2C_ISR_STOPF); + io_clrbits32(base + I2C_CR2, CR2_RESET_MASK); + + hi2c->i2c_state = I2C_STATE_READY; + + rc = 0; + +bail: + clk_disable(hi2c->clock); + + return rc; +} + +/* + * Read an amount of data in blocking mode + * + * @hi2c: Reference to struct i2c_handle_s + * @request: I2C request parameters + * @p_data: Pointer to data buffer + * @size: Amount of data to be sent + * Return 0 on success or a negative value + */ +static int do_read(struct i2c_handle_s *hi2c, struct i2c_request *request, + uint8_t *p_data, uint32_t size) +{ + vaddr_t base = get_base(hi2c); + uint64_t timeout_ref = 0; + int rc = -1; + uint8_t *p_buff = p_data; + size_t xfer_count = size; + size_t xfer_size = 0; + + if (request->mode != I2C_MODE_MASTER && request->mode != I2C_MODE_MEM) + return -1; + + if (hi2c->i2c_state != I2C_STATE_READY) + return -1; + + if (!p_data || !size) + return -1; + + clk_enable(hi2c->clock); + + timeout_ref = timeout_init_us(I2C_TIMEOUT_BUSY_MS * 1000); + if (wait_isr_event(hi2c, I2C_ISR_BUSY, 0, timeout_ref)) + goto bail; + + hi2c->i2c_state = I2C_STATE_BUSY_RX; + hi2c->i2c_err = I2C_ERROR_NONE; + timeout_ref = timeout_init_us(request->timeout_ms * 1000); + + if (request->mode == I2C_MODE_MEM) { + /* Send memory address */ + if (i2c_request_mem_read(hi2c, request, timeout_ref)) + goto bail; + } + + /* + * Send slave address. + * Set NBYTES to write and reload if xfer_count > MAX_NBYTE_SIZE + * and generate RESTART. + */ + if (xfer_count > MAX_NBYTE_SIZE) { + xfer_size = MAX_NBYTE_SIZE; + i2c_transfer_config(hi2c, request->dev_addr, xfer_size, + I2C_RELOAD_MODE, I2C_GENERATE_START_READ); + } else { + xfer_size = xfer_count; + i2c_transfer_config(hi2c, request->dev_addr, xfer_size, + I2C_AUTOEND_MODE, I2C_GENERATE_START_READ); + } + + do { + if (wait_isr_event(hi2c, I2C_ISR_RXNE, 1, + timeout_init_us(I2C_TIMEOUT_RXNE_MS * 1000))) + goto bail; + + *p_buff = io_read8(base + I2C_RXDR); + p_buff++; + xfer_size--; + xfer_count--; + + if (xfer_count && !xfer_size) { + if (wait_isr_event(hi2c, I2C_ISR_TCR, 1, timeout_ref)) + goto bail; + + if (xfer_count > MAX_NBYTE_SIZE) { + xfer_size = MAX_NBYTE_SIZE; + i2c_transfer_config(hi2c, request->dev_addr, + xfer_size, + I2C_RELOAD_MODE, + I2C_NO_STARTSTOP); + } else { + xfer_size = xfer_count; + i2c_transfer_config(hi2c, request->dev_addr, + xfer_size, + I2C_AUTOEND_MODE, + I2C_NO_STARTSTOP); + } + } + } while (xfer_count > 0U); + + /* + * No need to Check TC flag, with AUTOEND mode the stop + * is automatically generated. + * Wait until STOPF flag is reset. + */ + if (i2c_wait_stop(hi2c, timeout_ref)) + goto bail; + + /* Clear the NACK generated at the end of the transfer */ + if ((io_read32(get_base(hi2c) + I2C_ISR) & I2C_ISR_NACKF)) + io_write32(get_base(hi2c) + I2C_ICR, I2C_ICR_NACKCF); + + io_write32(base + I2C_ICR, I2C_ISR_STOPF); + + io_clrbits32(base + I2C_CR2, CR2_RESET_MASK); + + hi2c->i2c_state = I2C_STATE_READY; + + rc = 0; + +bail: + clk_disable(hi2c->clock); + + return rc; +} + +int stm32_i2c_mem_read(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint32_t mem_addr, uint32_t mem_addr_size, + uint8_t *p_data, size_t size, unsigned int timeout_ms) +{ + struct i2c_request request = { + .dev_addr = dev_addr, + .mode = I2C_MODE_MEM, + .mem_addr = mem_addr, + .mem_addr_size = mem_addr_size, + .timeout_ms = timeout_ms, + }; + + return do_read(hi2c, &request, p_data, size); +} + +int stm32_i2c_master_receive(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint8_t *p_data, size_t size, + unsigned int timeout_ms) +{ + struct i2c_request request = { + .dev_addr = dev_addr, + .mode = I2C_MODE_MASTER, + .timeout_ms = timeout_ms, + }; + + return do_read(hi2c, &request, p_data, size); +} + +static struct i2c_handle_s *stm32_i2c_dev_to_handle(struct i2c_dev *i2c_dev) +{ + struct stm32_i2c_dev *dev = container_of(i2c_dev, struct stm32_i2c_dev, + i2c_dev); + + return dev->handle; +} + +static TEE_Result stm32_i2c_read_data(struct i2c_dev *i2c_dev, uint8_t *buf, + size_t len) +{ + struct i2c_handle_s *i2c_handle = stm32_i2c_dev_to_handle(i2c_dev); + int rc = 0; + + rc = stm32_i2c_master_receive(i2c_handle, i2c_dev->addr, buf, len, + I2C_TIMEOUT_DEFAULT_MS); + if (!rc) + return TEE_SUCCESS; + else + return TEE_ERROR_GENERIC; +} + +static TEE_Result stm32_i2c_write_data(struct i2c_dev *i2c_dev, + const uint8_t *buf, size_t len) +{ + struct i2c_handle_s *i2c_handle = stm32_i2c_dev_to_handle(i2c_dev); + uint8_t *buf2 = (uint8_t *)buf; + int rc = 0; + + rc = stm32_i2c_master_transmit(i2c_handle, i2c_dev->addr, buf2, len, + I2C_TIMEOUT_DEFAULT_MS); + if (!rc) + return TEE_SUCCESS; + else + return TEE_ERROR_GENERIC; +} + +static const struct i2c_ctrl_ops stm32_i2c_ops = { + .read = stm32_i2c_read_data, + .write = stm32_i2c_write_data, +}; + +bool stm32_i2c_is_device_ready(struct i2c_handle_s *hi2c, uint32_t dev_addr, + unsigned int trials, unsigned int timeout_ms) +{ + vaddr_t base = get_base(hi2c); + unsigned int i2c_trials = 0U; + bool rc = false; + + if (hi2c->i2c_state != I2C_STATE_READY) + return rc; + + clk_enable(hi2c->clock); + + if (io_read32(base + I2C_ISR) & I2C_ISR_BUSY) + goto bail; + + hi2c->i2c_state = I2C_STATE_BUSY; + hi2c->i2c_err = I2C_ERROR_NONE; + + do { + uint64_t timeout_ref = 0; + vaddr_t isr = base + I2C_ISR; + + /* Generate Start */ + if ((io_read32(base + I2C_OAR1) & I2C_OAR1_OA1MODE) == 0) + io_write32(base + I2C_CR2, + ((dev_addr & I2C_CR2_SADD) | + I2C_CR2_START | I2C_CR2_AUTOEND) & + ~I2C_CR2_RD_WRN); + else + io_write32(base + I2C_CR2, + ((dev_addr & I2C_CR2_SADD) | + I2C_CR2_START | I2C_CR2_ADD10) & + ~I2C_CR2_RD_WRN); + + /* + * No need to Check TC flag, with AUTOEND mode the stop + * is automatically generated. + * Wait until STOPF flag is set or a NACK flag is set. + */ + timeout_ref = timeout_init_us(timeout_ms * 1000); + while (!timeout_elapsed(timeout_ref)) + if (io_read32(isr) & (I2C_ISR_STOPF | I2C_ISR_NACKF)) + break; + + if ((io_read32(isr) & (I2C_ISR_STOPF | I2C_ISR_NACKF)) == 0) { + notif_i2c_timeout(hi2c); + goto bail; + } + + if ((io_read32(base + I2C_ISR) & I2C_ISR_NACKF) == 0U) { + if (wait_isr_event(hi2c, I2C_ISR_STOPF, 1, timeout_ref)) + goto bail; + + io_write32(base + I2C_ICR, I2C_ISR_STOPF); + + hi2c->i2c_state = I2C_STATE_READY; + + rc = true; + goto bail; + } + + if (wait_isr_event(hi2c, I2C_ISR_STOPF, 1, timeout_ref)) + goto bail; + + io_write32(base + I2C_ICR, I2C_ISR_NACKF); + io_write32(base + I2C_ICR, I2C_ISR_STOPF); + + if (i2c_trials == trials) { + io_setbits32(base + I2C_CR2, I2C_CR2_STOP); + + if (wait_isr_event(hi2c, I2C_ISR_STOPF, 1, timeout_ref)) + goto bail; + + io_write32(base + I2C_ICR, I2C_ISR_STOPF); + } + + i2c_trials++; + } while (i2c_trials < trials); + + notif_i2c_timeout(hi2c); + +bail: + clk_disable(hi2c->clock); + + return rc; +} + +void stm32_i2c_resume(struct i2c_handle_s *hi2c) +{ + if (hi2c->i2c_state == I2C_STATE_READY) + return; + + if ((hi2c->i2c_state != I2C_STATE_RESET) && + (hi2c->i2c_state != I2C_STATE_SUSPENDED)) + panic(); + + if (pinctrl_apply_state(hi2c->pinctrl)) + panic(); + + if (hi2c->i2c_state == I2C_STATE_RESET) { + /* There is no valid I2C configuration to be loaded yet */ + return; + } + + restore_cfg(hi2c, &hi2c->sec_cfg); + + if (IS_ENABLED(CFG_STM32MP13)) + stm32_pinctrl_set_secure_cfg(hi2c->pinctrl, true); + + hi2c->i2c_state = I2C_STATE_READY; +} + +void stm32_i2c_suspend(struct i2c_handle_s *hi2c) +{ + if (hi2c->i2c_state == I2C_STATE_SUSPENDED) + return; + + if (hi2c->i2c_state != I2C_STATE_READY) + panic(); + + save_cfg(hi2c, &hi2c->sec_cfg); + + if (hi2c->pinctrl_sleep && pinctrl_apply_state(hi2c->pinctrl_sleep)) + panic(); + + hi2c->i2c_state = I2C_STATE_SUSPENDED; +} + +static TEE_Result stm32_get_i2c_dev(struct dt_pargs *args, void *data, + struct i2c_dev **out_device) +{ + struct stm32_i2c_dev *stm32_i2c_dev = NULL; + paddr_t addr = 0; + + addr = fdt_reg_base_address(args->fdt, args->phandle_node); + if (addr == DT_INFO_INVALID_REG) { + DMSG("Can't get device I2C address"); + return TEE_ERROR_GENERIC; + } + + stm32_i2c_dev = calloc(1, sizeof(*stm32_i2c_dev)); + if (!stm32_i2c_dev) + return TEE_ERROR_OUT_OF_MEMORY; + + stm32_i2c_dev->handle = data; + stm32_i2c_dev->i2c_dev.addr = addr; + stm32_i2c_dev->i2c_ctrl.ops = &stm32_i2c_ops; + stm32_i2c_dev->i2c_dev.ctrl = &stm32_i2c_dev->i2c_ctrl; + + *out_device = &stm32_i2c_dev->i2c_dev; + + return TEE_SUCCESS; +} + +static TEE_Result stm32_i2c_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_SUCCESS; + int subnode = 0; + struct i2c_handle_s *i2c_handle_p = NULL; + struct stm32_i2c_init_s init_data = { }; + struct pinctrl_state *pinctrl_active = NULL; + struct pinctrl_state *pinctrl_idle = NULL; + + res = stm32_i2c_get_setup_from_fdt((void *)fdt, node, &init_data, + &pinctrl_active, &pinctrl_idle); + if (res) + return res; + + i2c_handle_p = calloc(1, sizeof(struct i2c_handle_s)); + if (!i2c_handle_p) + return TEE_ERROR_OUT_OF_MEMORY; + + i2c_handle_p->dt_status = init_data.dt_status; + i2c_handle_p->reg_size = init_data.reg_size; + i2c_handle_p->clock = init_data.clock; + i2c_handle_p->base.pa = init_data.pbase; + i2c_handle_p->base.va = io_pa_or_va(&i2c_handle_p->base, + init_data.reg_size); + assert(i2c_handle_p->base.va); + i2c_handle_p->clock = init_data.clock; + i2c_handle_p->i2c_state = I2C_STATE_RESET; + i2c_handle_p->pinctrl = pinctrl_active; + i2c_handle_p->pinctrl_sleep = pinctrl_idle; + + init_data.analog_filter = true; + init_data.digital_filter_coef = 0; + + res = stm32_i2c_init(i2c_handle_p, &init_data); + if (res) + panic("Couldn't initialise I2C"); + + res = i2c_register_provider(fdt, node, stm32_get_i2c_dev, i2c_handle_p); + if (res) + panic("Couldn't register I2C provider"); + + fdt_for_each_subnode(subnode, fdt, node) { + res = dt_driver_maybe_add_probe_node(fdt, subnode); + if (res) { + EMSG("Failed on node %s with %#"PRIx32, + fdt_get_name(fdt, subnode, NULL), res); + panic(); + } + } + + return res; +} + +static const struct dt_device_match stm32_i2c_match_table[] = { + { .compatible = "st,stm32mp15-i2c" }, + { .compatible = "st,stm32mp13-i2c" }, + { .compatible = "st,stm32mp15-i2c-non-secure" }, + { } +}; + +DEFINE_DT_DRIVER(stm32_i2c_dt_driver) = { + .name = "stm32_i2c", + .match_table = stm32_i2c_match_table, + .probe = stm32_i2c_probe, + .type = DT_DRIVER_I2C +}; diff --git a/optee_os/core/drivers/stm32_iwdg.c b/optee_os/core/drivers/stm32_iwdg.c new file mode 100644 index 0000000..fd2899f --- /dev/null +++ b/optee_os/core/drivers/stm32_iwdg.c @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2017-2022, STMicroelectronics - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* IWDG Compatibility */ +#define IWDG_TIMEOUT_US U(1000) +#define IWDG_CNT_MASK GENMASK_32(11, 0) + +/* IWDG registers offsets */ +#define IWDG_KR_OFFSET U(0x00) +#define IWDG_PR_OFFSET U(0x04) +#define IWDG_RLR_OFFSET U(0x08) +#define IWDG_SR_OFFSET U(0x0C) +#define IWDG_EWCR_OFFSET U(0x14) + +#define IWDG_KR_ACCESS_KEY U(0x5555) +#define IWDG_KR_RELOAD_KEY U(0xAAAA) +#define IWDG_KR_START_KEY U(0xCCCC) + +/* Use a fixed prescaler divider of 256 */ +#define IWDG_PRESCALER_256 U(256) +#define IWDG_PR_DIV_256 U(0x06) +#define IWDG_PR_DIV_MASK GENMASK_32(3, 0) + +#define IWDG_SR_PVU BIT(0) +#define IWDG_SR_RVU BIT(1) +#define IWDG_SR_WVU BIT(2) +#define IWDG_SR_EWU BIT(3) +#define IWDG_SR_UPDATE_MASK (IWDG_SR_PVU | IWDG_SR_RVU | IWDG_SR_WVU | \ + IWDG_SR_EWU) + +#define IWDG_EWCR_EWIE BIT(15) +#define IWDG_EWCR_EWIC BIT(14) + +/* + * Values for struct stm32_iwdg_device::flags + * IWDG_FLAGS_HW_ENABLED Watchdog is enabled by BootROM + * IWDG_FLAGS_DISABLE_ON_STOP Watchdog is freezed in SoC STOP mode + * IWDG_FLAGS_DISABLE_ON_STANDBY Watchdog is freezed in SoC STANDBY mode + * IWDG_FLAGS_NON_SECURE Instance is assigned to non-secure world + * IWDG_FLAGS_ENABLED Watchdog has been enabled + */ +#define IWDG_FLAGS_HW_ENABLED BIT(0) +#define IWDG_FLAGS_DISABLE_ON_STOP BIT(1) +#define IWDG_FLAGS_DISABLE_ON_STANDBY BIT(2) +#define IWDG_FLAGS_NON_SECURE BIT(3) +#define IWDG_FLAGS_ENABLED BIT(4) + +/* + * IWDG watch instance data + * @base - IWDG interface IOMEM base address + * @clock - Bus clock + * @clk_lsi - IWDG source clock + * @flags - Property flags for the IWDG instance + * @timeout - Watchdog elaspure timeout + * @wdt_chip - Wathcdog chip instance + * @link - Link in registered watchdog instance list + */ +struct stm32_iwdg_device { + struct io_pa_va base; + struct clk *clock; + struct clk *clk_lsi; + uint32_t flags; + unsigned long timeout; + struct wdt_chip wdt_chip; + SLIST_ENTRY(stm32_iwdg_device) link; +}; + +static unsigned int iwdg_lock = SPINLOCK_UNLOCK; + +static SLIST_HEAD(iwdg_dev_list_head, stm32_iwdg_device) iwdg_dev_list = + SLIST_HEAD_INITIALIZER(iwdg_dev_list_head); + +static vaddr_t get_base(struct stm32_iwdg_device *iwdg) +{ + return io_pa_or_va(&iwdg->base, 1); +} + +static bool is_assigned_to_nsec(struct stm32_iwdg_device *iwdg) +{ + return iwdg->flags & IWDG_FLAGS_NON_SECURE; +} + +static bool is_enable(struct stm32_iwdg_device *iwdg) +{ + return iwdg->flags & IWDG_FLAGS_ENABLED; +} + +/* Return counter value to related to input timeout in seconds, or 0 on error */ +static uint32_t iwdg_timeout_cnt(struct stm32_iwdg_device *iwdg, + unsigned long to_sec) +{ + uint64_t reload = (uint64_t)to_sec * clk_get_rate(iwdg->clk_lsi); + uint64_t cnt = (reload / IWDG_PRESCALER_256) - 1; + + /* Be safe and expect any counter to be above 2 */ + if (cnt > IWDG_CNT_MASK || cnt < 3) + return 0; + + return cnt; +} + +/* Wait IWDG programming completes */ +static TEE_Result iwdg_wait_sync(struct stm32_iwdg_device *iwdg) +{ + uint64_t timeout_ref = timeout_init_us(IWDG_TIMEOUT_US); + vaddr_t iwdg_base = get_base(iwdg); + + while (io_read32(iwdg_base + IWDG_SR_OFFSET) & IWDG_SR_UPDATE_MASK) + if (timeout_elapsed(timeout_ref)) + break; + + if (!(io_read32(iwdg_base + IWDG_SR_OFFSET) & IWDG_SR_UPDATE_MASK)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result configure_timeout(struct stm32_iwdg_device *iwdg) +{ + TEE_Result res = TEE_ERROR_GENERIC; + vaddr_t iwdg_base = get_base(iwdg); + uint32_t rlr_value = 0; + + assert(is_enable(iwdg)); + + rlr_value = iwdg_timeout_cnt(iwdg, iwdg->timeout); + if (!rlr_value) + return TEE_ERROR_GENERIC; + + clk_enable(iwdg->clock); + + io_write32(iwdg_base + IWDG_KR_OFFSET, IWDG_KR_ACCESS_KEY); + io_write32(iwdg_base + IWDG_PR_OFFSET, IWDG_PR_DIV_256); + io_write32(iwdg_base + IWDG_RLR_OFFSET, rlr_value); + io_write32(iwdg_base + IWDG_KR_OFFSET, IWDG_KR_RELOAD_KEY); + + res = iwdg_wait_sync(iwdg); + + clk_disable(iwdg->clock); + + return res; +} + +static void iwdg_start(struct stm32_iwdg_device *iwdg) +{ + clk_enable(iwdg->clock); + io_write32(get_base(iwdg) + IWDG_KR_OFFSET, IWDG_KR_START_KEY); + clk_disable(iwdg->clock); + + iwdg->flags |= IWDG_FLAGS_ENABLED; +} + +static void iwdg_refresh(struct stm32_iwdg_device *iwdg) +{ + clk_enable(iwdg->clock); + io_write32(get_base(iwdg) + IWDG_KR_OFFSET, IWDG_KR_RELOAD_KEY); + clk_disable(iwdg->clock); +} + +/* Operators for watchdog OP-TEE interface */ +static struct stm32_iwdg_device *wdt_chip_to_iwdg(struct wdt_chip *chip) +{ + return container_of(chip, struct stm32_iwdg_device, wdt_chip); +} + +static void iwdg_wdt_start(struct wdt_chip *chip) +{ + struct stm32_iwdg_device *iwdg = wdt_chip_to_iwdg(chip); + + iwdg_start(iwdg); + + if (configure_timeout(iwdg)) + panic(); +} + +static void iwdg_wdt_refresh(struct wdt_chip *chip) +{ + struct stm32_iwdg_device *iwdg = wdt_chip_to_iwdg(chip); + + iwdg_refresh(iwdg); +} + +static TEE_Result iwdg_wdt_set_timeout(struct wdt_chip *chip, + unsigned long timeout) +{ + struct stm32_iwdg_device *iwdg = wdt_chip_to_iwdg(chip); + + if (!iwdg_timeout_cnt(iwdg, timeout)) + return TEE_ERROR_BAD_PARAMETERS; + + iwdg->timeout = timeout; + + if (is_enable(iwdg)) { + TEE_Result res = TEE_ERROR_GENERIC; + + res = configure_timeout(iwdg); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +static const struct wdt_ops stm32_iwdg_ops = { + .start = iwdg_wdt_start, + .ping = iwdg_wdt_refresh, + .set_timeout = iwdg_wdt_set_timeout, +}; +DECLARE_KEEP_PAGER(stm32_iwdg_ops); + +/* Refresh all registered watchdogs */ +void stm32_iwdg_refresh(void) +{ + struct stm32_iwdg_device *iwdg = NULL; + uint32_t exceptions = cpu_spin_lock_xsave(&iwdg_lock); + + SLIST_FOREACH(iwdg, &iwdg_dev_list, link) + iwdg_refresh(iwdg); + + cpu_spin_unlock_xrestore(&iwdg_lock, exceptions); +} + +/* Driver initialization */ +static TEE_Result stm32_iwdg_parse_fdt(struct stm32_iwdg_device *iwdg, + const void *fdt, int node) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct dt_node_info dt_info = { }; + const fdt32_t *cuint = NULL; + + fdt_fill_device_info(fdt, &dt_info, node); + + if (dt_info.reg == DT_INFO_INVALID_REG || + dt_info.reg_size == DT_INFO_INVALID_REG_SIZE) + panic(); + + res = clk_dt_get_by_name(fdt, node, "pclk", &iwdg->clock); + if (res) + return res; + + res = clk_dt_get_by_name(fdt, node, "lsi", &iwdg->clk_lsi); + if (res) + return res; + + if (dt_info.status == DT_STATUS_OK_NSEC) + iwdg->flags |= IWDG_FLAGS_NON_SECURE; + + /* Get IOMEM address */ + iwdg->base.pa = dt_info.reg; + + if (iwdg->flags & IWDG_FLAGS_NON_SECURE) + io_pa_or_va_nsec(&iwdg->base, dt_info.reg_size); + else + io_pa_or_va_secure(&iwdg->base, dt_info.reg_size); + + assert(iwdg->base.va); + + /* Get and check timeout value */ + cuint = fdt_getprop(fdt, node, "timeout-sec", NULL); + if (!cuint) + return TEE_ERROR_BAD_PARAMETERS; + + iwdg->timeout = (int)fdt32_to_cpu(*cuint); + if (!iwdg->timeout) + return TEE_ERROR_BAD_PARAMETERS; + + if (!iwdg_timeout_cnt(iwdg, iwdg->timeout)) { + EMSG("Timeout %lu not applicable", iwdg->timeout); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* DT can specify low power cases */ + if (!fdt_getprop(fdt, node, "stm32,enable-on-stop", NULL)) + iwdg->flags |= IWDG_FLAGS_DISABLE_ON_STOP; + + if (!fdt_getprop(fdt, node, "stm32,enable-on-standby", NULL)) + iwdg->flags |= IWDG_FLAGS_DISABLE_ON_STANDBY; + + return TEE_SUCCESS; +} + +/* Platform should override this function to provide IWDG fuses configuration */ +TEE_Result __weak stm32_get_iwdg_otp_config(paddr_t pbase __unused, + struct stm32_iwdg_otp_data *otp_d) +{ + otp_d->hw_enabled = false; + otp_d->disable_on_stop = false; + otp_d->disable_on_standby = false; + + return TEE_SUCCESS; +} + +static TEE_Result stm32_iwdg_setup(struct stm32_iwdg_device *iwdg, + const void *fdt, int node) +{ + struct stm32_iwdg_otp_data otp_data = { }; + TEE_Result res = TEE_SUCCESS; + + res = stm32_iwdg_parse_fdt(iwdg, fdt, node); + if (res) + return res; + + res = stm32_get_iwdg_otp_config(iwdg->base.pa, &otp_data); + if (res) + return res; + + if (otp_data.hw_enabled) + iwdg->flags |= IWDG_FLAGS_HW_ENABLED; + if (otp_data.disable_on_stop) + iwdg->flags |= IWDG_FLAGS_DISABLE_ON_STOP; + if (otp_data.disable_on_standby) + iwdg->flags |= IWDG_FLAGS_DISABLE_ON_STANDBY; + + /* Enable watchdog source clock once for all */ + clk_enable(iwdg->clk_lsi); + + if (otp_data.hw_enabled) { + iwdg->flags |= IWDG_FLAGS_ENABLED; + + /* Configure timeout if watchdog is already enabled */ + res = configure_timeout(iwdg); + if (res) + return res; + + iwdg_refresh(iwdg); + } + + return TEE_SUCCESS; +} + +static TEE_Result stm32_iwdg_register(struct stm32_iwdg_device *iwdg) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + if (is_assigned_to_nsec(iwdg)) { + stm32mp_register_non_secure_periph_iomem(iwdg->base.pa); + } else { + stm32mp_register_secure_periph_iomem(iwdg->base.pa); + + /* Expose watchdog runtime service only to secure IWDG */ + iwdg->wdt_chip.ops = &stm32_iwdg_ops; + + res = watchdog_register(&iwdg->wdt_chip); + if (res) + return res; + } + + SLIST_INSERT_HEAD(&iwdg_dev_list, iwdg, link); + + return TEE_SUCCESS; +} + +static TEE_Result stm32_iwdg_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + struct stm32_iwdg_device *iwdg = NULL; + TEE_Result res = TEE_SUCCESS; + + iwdg = calloc(1, sizeof(*iwdg)); + if (!iwdg) + return TEE_ERROR_OUT_OF_MEMORY; + + res = stm32_iwdg_setup(iwdg, fdt, node); + if (res) + goto err; + + res = stm32_iwdg_register(iwdg); + if (res) + goto err; + + return TEE_SUCCESS; + +err: + free(iwdg); + return res; +} + +static const struct dt_device_match stm32_iwdg_match_table[] = { + { .compatible = "st,stm32mp1-iwdg" }, + { } +}; + +DEFINE_DT_DRIVER(stm32_iwdg_dt_driver) = { + .name = "stm32-iwdg", + .match_table = stm32_iwdg_match_table, + .probe = stm32_iwdg_probe, +}; diff --git a/optee_os/core/drivers/stm32_rng.c b/optee_os/core/drivers/stm32_rng.c new file mode 100644 index 0000000..ef0f362 --- /dev/null +++ b/optee_os/core/drivers/stm32_rng.c @@ -0,0 +1,553 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2018-2023, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNG_CR U(0x00) +#define RNG_SR U(0x04) +#define RNG_DR U(0x08) + +#define RNG_CR_RNGEN BIT(2) +#define RNG_CR_IE BIT(3) +#define RNG_CR_CED BIT(5) +#define RNG_CR_CLKDIV GENMASK_32(19, 16) +#define RNG_CR_CLKDIV_SHIFT U(16) +#define RNG_CR_CONDRST BIT(30) + +#define RNG_SR_DRDY BIT(0) +#define RNG_SR_CECS BIT(1) +#define RNG_SR_SECS BIT(2) +#define RNG_SR_CEIS BIT(5) +#define RNG_SR_SEIS BIT(6) + +#if TRACE_LEVEL > TRACE_DEBUG +#define RNG_READY_TIMEOUT_US U(100000) +#else +#define RNG_READY_TIMEOUT_US U(10000) +#endif +#define RNG_RESET_TIMEOUT_US U(1000) + +#define RNG_FIFO_BYTE_DEPTH U(16) + +#define RNG_NIST_CONFIG_A U(0x0F00D00) +#define RNG_NIST_CONFIG_B U(0x1801000) +#define RNG_NIST_CONFIG_MASK GENMASK_32(25, 8) + +#define RNG_MAX_NOISE_CLK_FREQ U(3000000) + +struct stm32_rng_driver_data { + bool has_cond_reset; +}; + +struct stm32_rng_instance { + struct io_pa_va base; + struct clk *clock; + struct rstctrl *rstctrl; + const struct stm32_rng_driver_data *ddata; + unsigned int lock; + bool release_post_boot; + bool clock_error; + bool error_conceal; + uint64_t error_to_ref; +}; + +/* Expect at most a single RNG instance */ +static struct stm32_rng_instance *stm32_rng; + +static vaddr_t get_base(void) +{ + assert(stm32_rng); + + return io_pa_or_va(&stm32_rng->base, 1); +} + +/* + * Extracts from the STM32 RNG specification when RNG supports CONDRST. + * + * When a noise source (or seed) error occurs, the RNG stops generating + * random numbers and sets to “1” both SEIS and SECS bits to indicate + * that a seed error occurred. (...) + * + * 1. Software reset by writing CONDRST at 1 and at 0 (see bitfield + * description for details). This step is needed only if SECS is set. + * Indeed, when SEIS is set and SECS is cleared it means RNG performed + * the reset automatically (auto-reset). + * 2. If SECS was set in step 1 (no auto-reset) wait for CONDRST + * to be cleared in the RNG_CR register, then confirm that SEIS is + * cleared in the RNG_SR register. Otherwise just clear SEIS bit in + * the RNG_SR register. + * 3. If SECS was set in step 1 (no auto-reset) wait for SECS to be + * cleared by RNG. The random number generation is now back to normal. + */ +static void conceal_seed_error_cond_reset(void) +{ + struct stm32_rng_instance *dev = stm32_rng; + vaddr_t rng_base = get_base(); + + if (!dev->error_conceal) { + uint32_t sr = io_read32(rng_base + RNG_SR); + + if (sr & RNG_SR_SECS) { + /* Conceal by resetting the subsystem (step 1.) */ + io_setbits32(rng_base + RNG_CR, RNG_CR_CONDRST); + io_clrbits32(rng_base + RNG_CR, RNG_CR_CONDRST); + + /* Arm timeout for error_conceal sequence */ + dev->error_to_ref = + timeout_init_us(RNG_READY_TIMEOUT_US); + dev->error_conceal = true; + } else { + /* RNG auto-reset (step 2.) */ + io_clrbits32(rng_base + RNG_SR, RNG_SR_SEIS); + } + } else { + /* Measure time before possible reschedule */ + bool timed_out = timeout_elapsed(dev->error_to_ref); + + /* Wait CONDRST is cleared (step 2.) */ + if (io_read32(rng_base + RNG_CR) & RNG_CR_CONDRST) { + if (timed_out) + panic(); + + /* Wait subsystem reset cycle completes */ + return; + } + + /* Check SEIS is cleared (step 2.) */ + if (io_read32(rng_base + RNG_SR) & RNG_SR_SEIS) + panic(); + + /* Wait SECS is cleared (step 3.) */ + if (io_read32(rng_base + RNG_SR) & RNG_SR_SECS) { + if (timed_out) + panic(); + + /* Wait subsystem reset cycle completes */ + return; + } + + dev->error_conceal = false; + } +} + +/* + * Extracts from the STM32 RNG specification, when CONDRST is not supported + * + * When a noise source (or seed) error occurs, the RNG stops generating + * random numbers and sets to “1” both SEIS and SECS bits to indicate + * that a seed error occurred. (...) + * + * The following sequence shall be used to fully recover from a seed + * error after the RNG initialization: + * 1. Clear the SEIS bit by writing it to “0”. + * 2. Read out 12 words from the RNG_DR register, and discard each of + * them in order to clean the pipeline. + * 3. Confirm that SEIS is still cleared. Random number generation is + * back to normal. + */ +static void conceal_seed_error_sw_reset(void) +{ + vaddr_t rng_base = get_base(); + size_t i = 0; + + io_clrbits32(rng_base + RNG_SR, RNG_SR_SEIS); + + for (i = 12; i != 0; i--) + (void)io_read32(rng_base + RNG_DR); + + if (io_read32(rng_base + RNG_SR) & RNG_SR_SEIS) + panic("RNG noise"); +} + +static void conceal_seed_error(void) +{ + if (stm32_rng->ddata->has_cond_reset) + conceal_seed_error_cond_reset(); + else + conceal_seed_error_sw_reset(); +} + +static TEE_Result read_available(vaddr_t rng_base, uint8_t *out, size_t *size) +{ + struct stm32_rng_instance *dev = stm32_rng; + uint8_t *buf = NULL; + size_t req_size = 0; + size_t len = 0; + + if (dev->error_conceal || io_read32(rng_base + RNG_SR) & RNG_SR_SEIS) + conceal_seed_error(); + + if (!(io_read32(rng_base + RNG_SR) & RNG_SR_DRDY)) { + FMSG("RNG not ready"); + return TEE_ERROR_NO_DATA; + } + + if (io_read32(rng_base + RNG_SR) & RNG_SR_SEIS) { + FMSG("RNG noise error"); + return TEE_ERROR_NO_DATA; + } + + buf = out; + req_size = MIN(RNG_FIFO_BYTE_DEPTH, *size); + len = req_size; + + /* RNG is ready: read up to 4 32bit words */ + while (len) { + uint32_t data32 = 0; + size_t sz = MIN(len, sizeof(uint32_t)); + + if (!(io_read32(rng_base + RNG_SR) & RNG_SR_DRDY)) + break; + data32 = io_read32(rng_base + RNG_DR); + + /* Late seed error case: DR being 0 is an error status */ + if (!data32) { + conceal_seed_error(); + return TEE_ERROR_NO_DATA; + } + + memcpy(buf, &data32, sz); + buf += sz; + len -= sz; + } + + *size = req_size - len; + + return TEE_SUCCESS; +} + +static uint32_t stm32_rng_clock_freq_restrain(void) +{ + struct stm32_rng_instance *dev = stm32_rng; + unsigned long clock_rate = 0; + uint32_t clock_div = 0; + + clock_rate = clk_get_rate(dev->clock); + + /* + * Get the exponent to apply on the CLKDIV field in RNG_CR register + * No need to handle the case when clock-div > 0xF as it is physically + * impossible + */ + while ((clock_rate >> clock_div) > RNG_MAX_NOISE_CLK_FREQ) + clock_div++; + + DMSG("RNG clk rate : %lu", clk_get_rate(dev->clock) >> clock_div); + + return clock_div; +} + +static TEE_Result init_rng(void) +{ + vaddr_t rng_base = get_base(); + uint64_t timeout_ref = 0; + uint32_t cr_ced_mask = 0; + + if (!stm32_rng->clock_error) + cr_ced_mask = RNG_CR_CED; + + /* Clean error indications */ + io_write32(rng_base + RNG_SR, 0); + + if (stm32_rng->ddata->has_cond_reset) { + uint32_t clock_div = stm32_rng_clock_freq_restrain(); + + /* Update configuration fields */ + io_clrsetbits32(rng_base + RNG_CR, RNG_NIST_CONFIG_MASK, + RNG_NIST_CONFIG_B | RNG_CR_CONDRST | + cr_ced_mask); + io_clrsetbits32(rng_base + RNG_CR, RNG_CR_CLKDIV, + clock_div << RNG_CR_CLKDIV_SHIFT); + + /* No need to wait for RNG_CR_CONDRST toggle as we enable clk */ + io_clrsetbits32(rng_base + RNG_CR, RNG_CR_CONDRST, + RNG_CR_RNGEN); + } else { + io_setbits32(rng_base + RNG_CR, RNG_CR_RNGEN | cr_ced_mask); + } + + timeout_ref = timeout_init_us(RNG_READY_TIMEOUT_US); + while (!(io_read32(rng_base + RNG_SR) & RNG_SR_DRDY)) + if (timeout_elapsed(timeout_ref)) + break; + + if (!(io_read32(rng_base + RNG_SR) & RNG_SR_DRDY)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result stm32_rng_read(uint8_t *out, size_t size) +{ + TEE_Result rc = TEE_ERROR_GENERIC; + bool burst_timeout = false; + uint64_t timeout_ref = 0; + uint32_t exceptions = 0; + uint8_t *out_ptr = out; + vaddr_t rng_base = 0; + size_t out_size = 0; + + if (!stm32_rng) { + DMSG("No RNG"); + return TEE_ERROR_NOT_SUPPORTED; + } + + clk_enable(stm32_rng->clock); + rng_base = get_base(); + + /* Arm timeout */ + timeout_ref = timeout_init_us(RNG_READY_TIMEOUT_US); + burst_timeout = false; + + while (out_size < size) { + /* Read by chunks of the size the RNG FIFO depth */ + size_t sz = size - out_size; + + exceptions = may_spin_lock(&stm32_rng->lock); + + rc = read_available(rng_base, out_ptr, &sz); + + /* Raise timeout only if we failed to get some samples */ + assert(!rc || rc == TEE_ERROR_NO_DATA); + if (rc) + burst_timeout = timeout_elapsed(timeout_ref); + + may_spin_unlock(&stm32_rng->lock, exceptions); + + if (burst_timeout) { + rc = TEE_ERROR_GENERIC; + goto out; + } + + if (!rc) { + out_size += sz; + out_ptr += sz; + /* Re-arm timeout */ + timeout_ref = timeout_init_us(RNG_READY_TIMEOUT_US); + burst_timeout = false; + } + } + +out: + assert(!rc || rc == TEE_ERROR_GENERIC); + clk_disable(stm32_rng->clock); + + return rc; +} + +#ifdef CFG_WITH_SOFTWARE_PRNG +/* Override weak plat_rng_init with platform handler to seed PRNG */ +void plat_rng_init(void) +{ + uint8_t seed[RNG_FIFO_BYTE_DEPTH] = { }; + + if (stm32_rng_read(seed, sizeof(seed))) + panic(); + + if (crypto_rng_init(seed, sizeof(seed))) + panic(); + + DMSG("PRNG seeded with RNG"); +} +#else +TEE_Result hw_get_random_bytes(void *out, size_t size) +{ + return stm32_rng_read(out, size); +} + +void plat_rng_init(void) +{ +} +#endif + +static TEE_Result stm32_rng_pm_resume(uint32_t pm_cr) +{ + vaddr_t base = get_base(); + + /* Clean error indications */ + io_write32(base + RNG_SR, 0); + + if (stm32_rng->ddata->has_cond_reset) { + /* + * Correct configuration in bits [29:4] must be set in the same + * access that set RNG_CR_CONDRST bit. Else config setting is + * not taken into account. + */ + io_write32(base + RNG_CR, pm_cr | RNG_CR_CONDRST); + + io_clrsetbits32(base + RNG_CR, RNG_CR_CONDRST, RNG_CR_RNGEN); + } else { + io_write32(base + RNG_CR, RNG_CR_RNGEN | pm_cr); + } + + return TEE_SUCCESS; +} + +static TEE_Result +stm32_rng_pm(enum pm_op op, unsigned int pm_hint __unused, + const struct pm_callback_handle *pm_handle __unused) +{ + static uint32_t pm_cr; + TEE_Result res = TEE_ERROR_GENERIC; + + assert(stm32_rng && (op == PM_OP_SUSPEND || op == PM_OP_RESUME)); + + res = clk_enable(stm32_rng->clock); + if (res) + return res; + + if (op == PM_OP_SUSPEND) + pm_cr = io_read32(get_base() + RNG_CR); + else + res = stm32_rng_pm_resume(pm_cr); + + clk_disable(stm32_rng->clock); + + return res; +} +DECLARE_KEEP_PAGER(stm32_rng_pm); + +static TEE_Result stm32_rng_parse_fdt(const void *fdt, int node) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct dt_node_info dt_rng = { }; + + fdt_fill_device_info(fdt, &dt_rng, node); + if (dt_rng.reg == DT_INFO_INVALID_REG) + return TEE_ERROR_BAD_PARAMETERS; + + stm32_rng->base.pa = dt_rng.reg; + stm32_rng->base.va = io_pa_or_va_secure(&stm32_rng->base, + dt_rng.reg_size); + assert(stm32_rng->base.va); + + res = rstctrl_dt_get_by_index(fdt, node, 0, &stm32_rng->rstctrl); + if (res != TEE_SUCCESS && res != TEE_ERROR_ITEM_NOT_FOUND) + return res; + + res = clk_dt_get_by_index(fdt, node, 0, &stm32_rng->clock); + if (res) + return res; + + if (fdt_getprop(fdt, node, "clock-error-detect", NULL)) + stm32_rng->clock_error = true; + + /* Release device if not used at runtime or for pm transitions */ + stm32_rng->release_post_boot = IS_ENABLED(CFG_WITH_SOFTWARE_PRNG) && + !IS_ENABLED(CFG_PM); + + return TEE_SUCCESS; +} + +static TEE_Result stm32_rng_probe(const void *fdt, int offs, + const void *compat_data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + /* Expect a single RNG instance */ + assert(!stm32_rng); + + stm32_rng = calloc(1, sizeof(*stm32_rng)); + if (!stm32_rng) + panic(); + + res = stm32_rng_parse_fdt(fdt, offs); + if (res) + goto err; + + stm32_rng->ddata = compat_data; + assert(stm32_rng->ddata); + + res = clk_enable(stm32_rng->clock); + if (res) + goto err; + + if (stm32_rng->rstctrl && + rstctrl_assert_to(stm32_rng->rstctrl, RNG_RESET_TIMEOUT_US)) { + res = TEE_ERROR_GENERIC; + goto err_clk; + } + + if (stm32_rng->rstctrl && + rstctrl_deassert_to(stm32_rng->rstctrl, RNG_RESET_TIMEOUT_US)) { + res = TEE_ERROR_GENERIC; + goto err_clk; + } + + res = init_rng(); + if (res) + goto err_clk; + + clk_disable(stm32_rng->clock); + + if (stm32_rng->release_post_boot) + stm32mp_register_non_secure_periph_iomem(stm32_rng->base.pa); + else + stm32mp_register_secure_periph_iomem(stm32_rng->base.pa); + + register_pm_core_service_cb(stm32_rng_pm, &stm32_rng, "rng-service"); + + return TEE_SUCCESS; + +err_clk: + clk_disable(stm32_rng->clock); +err: + free(stm32_rng); + stm32_rng = NULL; + + return res; +} + +static const struct stm32_rng_driver_data mp13_data[] = { + { .has_cond_reset = true }, +}; + +static const struct stm32_rng_driver_data mp15_data[] = { + { .has_cond_reset = false }, +}; +DECLARE_KEEP_PAGER(mp15_data); + +static const struct dt_device_match rng_match_table[] = { + { .compatible = "st,stm32-rng", .compat_data = &mp15_data }, + { .compatible = "st,stm32mp13-rng", .compat_data = &mp13_data }, + { } +}; + +DEFINE_DT_DRIVER(stm32_rng_dt_driver) = { + .name = "stm32_rng", + .match_table = rng_match_table, + .probe = stm32_rng_probe, +}; + +static TEE_Result stm32_rng_release(void) +{ + if (stm32_rng && stm32_rng->release_post_boot) { + DMSG("Release RNG driver"); + free(stm32_rng); + stm32_rng = NULL; + } + + return TEE_SUCCESS; +} + +release_init_resource(stm32_rng_release); diff --git a/optee_os/core/drivers/stm32_shared_io.c b/optee_os/core/drivers/stm32_shared_io.c new file mode 100644 index 0000000..fa20b47 --- /dev/null +++ b/optee_os/core/drivers/stm32_shared_io.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2022, STMicroelectronics + */ + +#include +#include +#include +#include + +static unsigned int shregs_lock = SPINLOCK_UNLOCK; + +static uint32_t lock_stm32shregs(void) +{ + return may_spin_lock(&shregs_lock); +} + +static void unlock_stm32shregs(uint32_t exceptions) +{ + may_spin_unlock(&shregs_lock, exceptions); +} + +void io_mask32_stm32shregs(vaddr_t va, uint32_t value, uint32_t mask) +{ + uint32_t exceptions = lock_stm32shregs(); + + io_mask32(va, value, mask); + + unlock_stm32shregs(exceptions); +} + +void io_clrsetbits32_stm32shregs(vaddr_t va, uint32_t clr, uint32_t set) +{ + uint32_t exceptions = lock_stm32shregs(); + + io_clrsetbits32(va, clr, set); + + unlock_stm32shregs(exceptions); +} diff --git a/optee_os/core/drivers/stm32_tamp.c b/optee_os/core/drivers/stm32_tamp.c new file mode 100644 index 0000000..6a34bd1 --- /dev/null +++ b/optee_os/core/drivers/stm32_tamp.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2021-2022, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* STM32 Registers */ +#define _TAMP_CR1 0x00U +#define _TAMP_CR2 0x04U +#define _TAMP_CR3 0x08U +#define _TAMP_FLTCR 0x0CU +#define _TAMP_ATCR1 0x10U +#define _TAMP_ATSEEDR 0x14U +#define _TAMP_ATOR 0x18U +#define _TAMP_ATCR2 0x1CU +#define _TAMP_SECCFGR 0x20U +#define _TAMP_SMCR 0x20U +#define _TAMP_PRIVCFGR 0x24U +#define _TAMP_IER 0x2CU +#define _TAMP_SR 0x30U +#define _TAMP_MISR 0x34U +#define _TAMP_SMISR 0x38U +#define _TAMP_SCR 0x3CU +#define _TAMP_COUNTR 0x40U +#define _TAMP_COUNT2R 0x44U +#define _TAMP_OR 0x50U +#define _TAMP_ERCFGR 0X54U +#define _TAMP_HWCFGR2 0x3ECU +#define _TAMP_HWCFGR1 0x3F0U +#define _TAMP_VERR 0x3F4U +#define _TAMP_IPIDR 0x3F8U +#define _TAMP_SIDR 0x3FCU + +/* _TAMP_SECCFGR bit fields */ +#define _TAMP_SECCFGR_BKPRWSEC_MASK GENMASK_32(7, 0) +#define _TAMP_SECCFGR_BKPRWSEC_SHIFT 0U +#define _TAMP_SECCFGR_CNT2SEC BIT(14) +#define _TAMP_SECCFGR_CNT1SEC BIT(15) +#define _TAMP_SECCFGR_BKPWSEC_MASK GENMASK_32(23, 16) +#define _TAMP_SECCFGR_BKPWSEC_SHIFT 16U +#define _TAMP_SECCFGR_BHKLOCK BIT(30) +#define _TAMP_SECCFGR_TAMPSEC BIT(31) +#define _TAMP_SECCFGR_BUT_BKP_MASK (GENMASK_32(31, 30) | \ + GENMASK_32(15, 14)) + +/* _TAMP_SMCR bit fields */ +#define _TAMP_SMCR_BKPRWDPROT_MASK GENMASK_32(7, 0) +#define _TAMP_SMCR_BKPRWDPROT_SHIFT 0U +#define _TAMP_SMCR_BKPWDPROT_MASK GENMASK_32(23, 16) +#define _TAMP_SMCR_BKPWDPROT_SHIFT 16U +#define _TAMP_SMCR_DPROT BIT(31) +/* + * _TAMP_PRIVCFGR bit fields + */ +#define _TAMP_PRIVCFG_CNT2PRIV BIT(14) +#define _TAMP_PRIVCFG_CNT1PRIV BIT(15) +#define _TAMP_PRIVCFG_BKPRWPRIV BIT(29) +#define _TAMP_PRIVCFG_BKPWPRIV BIT(30) +#define _TAMP_PRIVCFG_TAMPPRIV BIT(31) +#define _TAMP_PRIVCFGR_MASK (GENMASK_32(31, 29) | \ + GENMASK_32(15, 14)) + +/* + * _TAMP_PRIVCFGR bit fields + */ +#define _TAMP_PRIVCFG_CNT2PRIV BIT(14) +#define _TAMP_PRIVCFG_CNT1PRIV BIT(15) +#define _TAMP_PRIVCFG_BKPRWPRIV BIT(29) +#define _TAMP_PRIVCFG_BKPWPRIV BIT(30) +#define _TAMP_PRIVCFG_TAMPPRIV BIT(31) +#define _TAMP_PRIVCFGR_MASK (GENMASK_32(31, 29) | \ + GENMASK_32(15, 14)) + +/* _TAMP_HWCFGR2 bit fields */ +#define _TAMP_HWCFGR2_TZ GENMASK_32(11, 8) +#define _TAMP_HWCFGR2_OR GENMASK_32(7, 0) + +/* _TAMP_HWCFGR1 bit fields */ +#define _TAMP_HWCFGR1_BKPREG GENMASK_32(7, 0) +#define _TAMP_HWCFGR1_TAMPER GENMASK_32(11, 8) +#define _TAMP_HWCFGR1_ACTIVE GENMASK_32(15, 12) +#define _TAMP_HWCFGR1_INTERN GENMASK_32(31, 16) +#define _TAMP_HWCFGR1_ITAMP_MAX_ID 16U +#define _TAMP_HWCFGR1_ITAMP(id) BIT((id) - INT_TAMP1 + 16U) + +/* _TAMP_VERR bit fields */ +#define _TAMP_VERR_MINREV GENMASK_32(3, 0) +#define _TAMP_VERR_MAJREV GENMASK_32(7, 4) + +/* + * TAMP instance data + * @base - IOMEM base address + * @clock - TAMP clock + * @it - TAMP interrupt number + * @hwconf1 - Copy of TAMP HWCONF1 register content + * @hwconf2 - Copy of TAMP HWCONF2 register content + * @compat - Reference to compat data passed at driver initialization + */ +struct stm32_tamp_instance { + struct io_pa_va base; + struct clk *clock; + int it; + uint32_t hwconf1; + uint32_t hwconf2; + struct stm32_tamp_compat *compat; +}; + +/* + * Compatibility capabilities + * TAMP_HAS_REGISTER_SECCFG - Supports SECCFGR, otherwise supports SMCR register + * TAMP_HAS_REGISTER_PRIVCFG - Supports PRIVCFGR configuration register + */ +#define TAMP_HAS_REGISTER_SECCFG BIT(0) +#define TAMP_HAS_REGISTER_PRIVCFGR BIT(1) + +/* + * @nb_monotonic_counter - Number of monotic counter supported + * @tags - Bit flags TAMP_HAS_* for compatibily management + */ +struct stm32_tamp_compat { + int nb_monotonic_counter; + uint32_t tags; +}; + +/* Expects at most a single instance */ +static struct stm32_tamp_instance *stm32_tamp_device; + +TEE_Result stm32_tamp_set_secure_bkpregs(struct stm32_bkpregs_conf *bkr_conf) +{ + struct stm32_tamp_instance *tamp = stm32_tamp_device; + vaddr_t base = 0; + uint32_t first_z2 = 0; + uint32_t first_z3 = 0; + + if (!tamp) + return TEE_ERROR_DEFER_DRIVER_INIT; + + if (!bkr_conf) + return TEE_ERROR_BAD_PARAMETERS; + + base = io_pa_or_va(&tamp->base, 1); + + first_z2 = bkr_conf->nb_zone1_regs; + first_z3 = bkr_conf->nb_zone1_regs + bkr_conf->nb_zone2_regs; + + if ((first_z2 > (tamp->hwconf1 & _TAMP_HWCFGR1_BKPREG)) || + (first_z3 > (tamp->hwconf1 & _TAMP_HWCFGR1_BKPREG))) + return TEE_ERROR_BAD_PARAMETERS; + + if (tamp->compat && (tamp->compat->tags & TAMP_HAS_REGISTER_SECCFG)) { + io_clrsetbits32(base + _TAMP_SECCFGR, + _TAMP_SECCFGR_BKPRWSEC_MASK, + (first_z2 << _TAMP_SECCFGR_BKPRWSEC_SHIFT) & + _TAMP_SECCFGR_BKPRWSEC_MASK); + + io_clrsetbits32(base + _TAMP_SECCFGR, + _TAMP_SECCFGR_BKPWSEC_MASK, + (first_z3 << _TAMP_SECCFGR_BKPWSEC_SHIFT) & + _TAMP_SECCFGR_BKPWSEC_MASK); + } else { + io_clrsetbits32(base + _TAMP_SMCR, + _TAMP_SMCR_BKPRWDPROT_MASK, + (first_z2 << _TAMP_SMCR_BKPRWDPROT_SHIFT) & + _TAMP_SMCR_BKPRWDPROT_MASK); + + io_clrsetbits32(base + _TAMP_SMCR, + _TAMP_SMCR_BKPWDPROT_MASK, + (first_z3 << _TAMP_SMCR_BKPWDPROT_SHIFT) & + _TAMP_SMCR_BKPWDPROT_MASK); + } + + return TEE_SUCCESS; +} + +static void stm32_tamp_set_secure(struct stm32_tamp_instance *tamp, + uint32_t mode) +{ + vaddr_t base = io_pa_or_va(&tamp->base, 1); + + if (tamp->compat && (tamp->compat->tags & TAMP_HAS_REGISTER_SECCFG)) { + io_clrsetbits32(base + _TAMP_SECCFGR, + _TAMP_SECCFGR_BUT_BKP_MASK, + mode & _TAMP_SECCFGR_BUT_BKP_MASK); + } else { + /* + * Note: MP15 doesn't use SECCFG register and + * inverts the secure bit. + */ + if (mode & _TAMP_SECCFGR_TAMPSEC) + io_clrbits32(base + _TAMP_SMCR, _TAMP_SMCR_DPROT); + else + io_setbits32(base + _TAMP_SMCR, _TAMP_SMCR_DPROT); + } +} + +static void stm32_tamp_set_privilege(struct stm32_tamp_instance *tamp, + uint32_t mode) +{ + vaddr_t base = io_pa_or_va(&tamp->base, 1); + + if (tamp->compat && (tamp->compat->tags & TAMP_HAS_REGISTER_PRIVCFGR)) + io_clrsetbits32(base + _TAMP_PRIVCFGR, _TAMP_PRIVCFGR_MASK, + mode & _TAMP_PRIVCFGR_MASK); +} + +static TEE_Result stm32_tamp_parse_fdt(struct stm32_tamp_instance *tamp, + const void *fdt, int node, + const void *compat) +{ + struct dt_node_info dt_tamp = { }; + + fdt_fill_device_info(fdt, &dt_tamp, node); + + if (dt_tamp.reg == DT_INFO_INVALID_REG || + dt_tamp.reg_size == DT_INFO_INVALID_REG_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + tamp->compat = (struct stm32_tamp_compat *)compat; + tamp->it = dt_tamp.interrupt; + tamp->base.pa = dt_tamp.reg; + io_pa_or_va_secure(&tamp->base, dt_tamp.reg_size); + + return clk_dt_get_by_index(fdt, node, 0, &tamp->clock); +} + +static TEE_Result stm32_tamp_probe(const void *fdt, int node, + const void *compat_data) +{ + struct stm32_tamp_instance *tamp = NULL; + uint32_t __maybe_unused revision = 0; + TEE_Result res = TEE_SUCCESS; + vaddr_t base = 0; + + tamp = calloc(1, sizeof(*tamp)); + if (!tamp) + return TEE_ERROR_OUT_OF_MEMORY; + + res = stm32_tamp_parse_fdt(tamp, fdt, node, compat_data); + if (res) + goto err; + + clk_enable(tamp->clock); + + base = io_pa_or_va(&tamp->base, 1); + + tamp->hwconf1 = io_read32(base + _TAMP_HWCFGR1); + tamp->hwconf2 = io_read32(base + _TAMP_HWCFGR2); + + revision = io_read32(base + _TAMP_VERR); + FMSG("STM32 TAMPER V%"PRIx32".%"PRIu32, + (revision & _TAMP_VERR_MAJREV) >> 4, revision & _TAMP_VERR_MINREV); + + if (!(tamp->hwconf2 & _TAMP_HWCFGR2_TZ)) { + EMSG("TAMP doesn't support TrustZone"); + res = TEE_ERROR_NOT_SUPPORTED; + goto err_clk; + } + + /* + * Enforce secure only access to protected TAMP registers. + * Allow non-secure access to monotonic counter. + */ + stm32_tamp_set_secure(tamp, _TAMP_SECCFGR_TAMPSEC); + + /* + * Enforce privilege only access to TAMP registers, backup + * registers and monotonic counter. + */ + stm32_tamp_set_privilege(tamp, _TAMP_PRIVCFG_TAMPPRIV | + _TAMP_PRIVCFG_BKPRWPRIV | + _TAMP_PRIVCFG_BKPWPRIV); + + stm32_tamp_device = tamp; + + return TEE_SUCCESS; + +err_clk: + clk_disable(tamp->clock); +err: + free(tamp); + return res; +} + +static const struct stm32_tamp_compat mp13_compat = { + .nb_monotonic_counter = 2, + .tags = TAMP_HAS_REGISTER_SECCFG | TAMP_HAS_REGISTER_PRIVCFGR, +}; + +static const struct stm32_tamp_compat mp15_compat = { + .nb_monotonic_counter = 1, + .tags = 0, +}; + +static const struct dt_device_match stm32_tamp_match_table[] = { + { .compatible = "st,stm32mp13-tamp", .compat_data = &mp13_compat }, + { .compatible = "st,stm32-tamp", .compat_data = &mp15_compat }, + { } +}; + +DEFINE_DT_DRIVER(stm32_tamp_dt_driver) = { + .name = "stm32-tamp", + .match_table = stm32_tamp_match_table, + .probe = stm32_tamp_probe, +}; diff --git a/optee_os/core/drivers/stm32_uart.c b/optee_os/core/drivers/stm32_uart.c new file mode 100644 index 0000000..3862b60 --- /dev/null +++ b/optee_os/core/drivers/stm32_uart.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2018, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UART_REG_CR1 0x00 /* Control register 1 */ +#define UART_REG_CR2 0x04 /* Control register 2 */ +#define UART_REG_CR3 0x08 /* Control register 3 */ +#define UART_REG_BRR 0x0c /* Baud rate register */ +#define UART_REG_RQR 0x18 /* Request register */ +#define UART_REG_ISR 0x1c /* Interrupt & status reg. */ +#define UART_REG_ICR 0x20 /* Interrupt flag clear reg. */ +#define UART_REG_RDR 0x24 /* Receive data register */ +#define UART_REG_TDR 0x28 /* Transmit data register */ +#define UART_REG_PRESC 0x2c /* Prescaler register */ + +#define PUTC_TIMEOUT_US 1000 +#define FLUSH_TIMEOUT_US 16000 + +/* + * Uart Interrupt & status register bits + * + * Bit 5 RXNE: Read data register not empty/RXFIFO not empty + * Bit 6 TC: Transmission complete + * Bit 7 TXE/TXFNF: Transmit data register empty/TXFIFO not full + * Bit 27 TXFE: TXFIFO threshold reached + */ +#define USART_ISR_RXNE_RXFNE BIT(5) +#define USART_ISR_TC BIT(6) +#define USART_ISR_TXE_TXFNF BIT(7) +#define USART_ISR_TXFE BIT(27) + +static vaddr_t loc_chip_to_base(struct serial_chip *chip) +{ + struct stm32_uart_pdata *pd = NULL; + + pd = container_of(chip, struct stm32_uart_pdata, chip); + + return io_pa_or_va(&pd->base, 1); +} + +static void loc_flush(struct serial_chip *chip) +{ + vaddr_t base = loc_chip_to_base(chip); + uint64_t timeout = timeout_init_us(FLUSH_TIMEOUT_US); + + while (!(io_read32(base + UART_REG_ISR) & USART_ISR_TXFE)) + if (timeout_elapsed(timeout)) + return; +} + +static void loc_putc(struct serial_chip *chip, int ch) +{ + vaddr_t base = loc_chip_to_base(chip); + uint64_t timeout = timeout_init_us(PUTC_TIMEOUT_US); + + while (!(io_read32(base + UART_REG_ISR) & USART_ISR_TXE_TXFNF)) + if (timeout_elapsed(timeout)) + return; + + io_write32(base + UART_REG_TDR, ch); +} + +static bool loc_have_rx_data(struct serial_chip *chip) +{ + vaddr_t base = loc_chip_to_base(chip); + + return io_read32(base + UART_REG_ISR) & USART_ISR_RXNE_RXFNE; +} + +static int loc_getchar(struct serial_chip *chip) +{ + vaddr_t base = loc_chip_to_base(chip); + + while (!loc_have_rx_data(chip)) + ; + + return io_read32(base + UART_REG_RDR) & 0xff; +} + +static const struct serial_ops stm32_uart_serial_ops = { + .flush = loc_flush, + .putc = loc_putc, + .have_rx_data = loc_have_rx_data, + .getchar = loc_getchar, + +}; +DECLARE_KEEP_PAGER(stm32_uart_serial_ops); + +void stm32_uart_init(struct stm32_uart_pdata *pd, vaddr_t base) +{ + pd->base.pa = base; + pd->chip.ops = &stm32_uart_serial_ops; +} + +static void register_secure_uart(struct stm32_uart_pdata *pd) +{ + size_t __maybe_unused n = 0; + + stm32mp_register_secure_periph_iomem(pd->base.pa); + stm32mp_register_secure_pinctrl(pd->pinctrl); + if (pd->pinctrl_sleep) + stm32mp_register_secure_pinctrl(pd->pinctrl_sleep); +} + +static void register_non_secure_uart(struct stm32_uart_pdata *pd) +{ + size_t __maybe_unused n = 0; + + stm32mp_register_non_secure_periph_iomem(pd->base.pa); + stm32mp_register_non_secure_pinctrl(pd->pinctrl); + if (pd->pinctrl_sleep) + stm32mp_register_non_secure_pinctrl(pd->pinctrl_sleep); +} + +struct stm32_uart_pdata *stm32_uart_init_from_dt_node(void *fdt, int node) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct stm32_uart_pdata *pd = NULL; + struct dt_node_info info = { }; + + fdt_fill_device_info(fdt, &info, node); + + if (info.status == DT_STATUS_DISABLED) + return NULL; + + assert(info.reg != DT_INFO_INVALID_REG && + info.reg_size != DT_INFO_INVALID_REG_SIZE); + + pd = calloc(1, sizeof(*pd)); + if (!pd) + panic(); + + pd->chip.ops = &stm32_uart_serial_ops; + pd->base.pa = info.reg; + pd->secure = (info.status == DT_STATUS_OK_SEC); + + res = clk_dt_get_by_index(fdt, node, 0, &pd->clock); + if (res) { + EMSG("Failed to get clock: %#"PRIx32, res); + panic(); + } + + res = clk_enable(pd->clock); + if (res) + panic(); + + assert(cpu_mmu_enabled()); + pd->base.va = (vaddr_t)phys_to_virt(pd->base.pa, + pd->secure ? MEM_AREA_IO_SEC : + MEM_AREA_IO_NSEC, info.reg_size); + + res = pinctrl_get_state_by_name(fdt, node, "default", &pd->pinctrl); + if (res) + panic(); + + res = pinctrl_get_state_by_name(fdt, node, "sleep", &pd->pinctrl_sleep); + if (res && res != TEE_ERROR_ITEM_NOT_FOUND) + panic(); + + res = pinctrl_apply_state(pd->pinctrl); + if (res) + panic(); + + if (pd->secure) + register_secure_uart(pd); + else + register_non_secure_uart(pd); + + return pd; +} diff --git a/optee_os/core/drivers/stm32mp15_huk.c b/optee_os/core/drivers/stm32mp15_huk.c new file mode 100644 index 0000000..e0550d9 --- /dev/null +++ b/optee_os/core/drivers/stm32mp15_huk.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + * Copyright (c) 2022, Foundries.io Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HUK_NB_OTP (HW_UNIQUE_KEY_LENGTH / sizeof(uint32_t)) + +static bool stm32mp15_huk_init; + +static TEE_Result stm32mp15_read_uid(uint32_t *uid) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + uint32_t *q = uid; + uint32_t otp_idx = 0; + size_t __maybe_unused sz = 0; + uint8_t __maybe_unused offset = 0; + + ret = stm32_bsec_find_otp_in_nvmem_layout("uid_otp", &otp_idx, &offset, + &sz); + if (ret) + return ret; + assert(sz == 3 * 32); + assert(offset == 0); + + /* + * Shadow memory for UID words might not be locked: to guarante that + * the final values are read we must lock them. + */ + if (stm32_bsec_set_sw_lock(otp_idx) || + stm32_bsec_shadow_read_otp(q++, otp_idx)) + return TEE_ERROR_GENERIC; + + if (stm32_bsec_set_sw_lock(otp_idx + 1) || + stm32_bsec_shadow_read_otp(q++, otp_idx + 1)) + return TEE_ERROR_GENERIC; + + if (stm32_bsec_set_sw_lock(otp_idx + 2) || + stm32_bsec_shadow_read_otp(q++, otp_idx + 2)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result stm32mp15_read_otp(uint32_t otp, uint32_t *key, bool *locked) +{ + bool tmp = true; + uint32_t state = 0; + + if (stm32_bsec_get_state(&state)) + panic(); + + if (state != BSEC_STATE_SEC_CLOSED) { + /* + * When the device is not closed, the shadow memory for these + * words might not be locked: check and report them + */ + if (stm32_bsec_read_permanent_lock(otp, &tmp)) + return TEE_ERROR_GENERIC; + + if (tmp && stm32_bsec_read_sw_lock(otp, &tmp)) + return TEE_ERROR_GENERIC; + } + + if (stm32_bsec_shadow_read_otp(key, otp)) + return TEE_ERROR_GENERIC; + + *locked = *locked && tmp; + + return TEE_SUCCESS; +} + +/* + * AES-GCM: nonce must be unique per message and key. + * + * This function always uses the same key - once its locked - with the same + * unique message hence the nonce can be any constant. + */ +static TEE_Result aes_gcm_encrypt_uid(uint8_t *key, size_t key_len, + uint8_t *out, size_t *out_len) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + const uint8_t nonce[12] = { 0x55 }; + uint32_t uid[4] = { 0 }; + uint8_t tag[16] = { 0 }; + size_t nonce_len = sizeof(nonce); + size_t tag_len = sizeof(tag); + size_t uid_len = sizeof(uid); + void *ctx = NULL; + + ret = stm32mp15_read_uid(uid); + if (ret) + goto out; + + ret = crypto_authenc_alloc_ctx(&ctx, TEE_ALG_AES_GCM); + if (ret) + goto out; + + ret = crypto_authenc_init(ctx, TEE_MODE_ENCRYPT, key, key_len, nonce, + nonce_len, TEE_AES_BLOCK_SIZE, 0, uid_len); + if (ret) + goto out_free_ctx; + + ret = crypto_authenc_enc_final(ctx, (uint8_t *)uid, sizeof(uid), + out, out_len, tag, &tag_len); + if (ret) + goto out_free_ctx; + + crypto_authenc_final(ctx); +out_free_ctx: + crypto_authenc_free_ctx(ctx); +out: + if (ret) + memzero_explicit(out, *out_len); + + return ret; +} + +static __maybe_unused TEE_Result pos_from_dt(uint32_t otp_id[HUK_NB_OTP]) +{ + TEE_Result ret = TEE_SUCCESS; + uint32_t otp_start = 0; + size_t sz = 0; + uint8_t offset = 0; + size_t i = 0; + + ret = stm32_bsec_find_otp_in_nvmem_layout("huk-otp", &otp_start, + &offset, &sz); + if (ret) + return ret; + + if (sz != (HW_UNIQUE_KEY_LENGTH * CHAR_BIT) || offset != 0) + return TEE_ERROR_SECURITY; + + for (i = 0; i < HUK_NB_OTP; i++) + otp_id[i] = otp_start + i; + + return TEE_SUCCESS; +} + +static TEE_Result get_otp_pos(uint32_t otp_id[HUK_NB_OTP]) +{ +#ifdef CFG_STM32_HUK_FROM_DT + return pos_from_dt(otp_id); +#else /* CFG_STM32_HUK_FROM_DT */ + + static_assert(CFG_STM32MP15_HUK_BSEC_KEY_0 < STM32MP1_OTP_MAX_ID); + static_assert(CFG_STM32MP15_HUK_BSEC_KEY_1 < STM32MP1_OTP_MAX_ID); + static_assert(CFG_STM32MP15_HUK_BSEC_KEY_2 < STM32MP1_OTP_MAX_ID); + static_assert(CFG_STM32MP15_HUK_BSEC_KEY_3 < STM32MP1_OTP_MAX_ID); + + otp_id[0] = CFG_STM32MP15_HUK_BSEC_KEY_0; + otp_id[1] = CFG_STM32MP15_HUK_BSEC_KEY_1; + otp_id[2] = CFG_STM32MP15_HUK_BSEC_KEY_2; + otp_id[3] = CFG_STM32MP15_HUK_BSEC_KEY_3; + + return TEE_SUCCESS; +#endif /* CFG_STM32_HUK_FROM_DT */ +} + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + uint32_t otp_key[HUK_NB_OTP] = { }; + uint32_t otp_id[HUK_NB_OTP] = { }; + size_t len = HW_UNIQUE_KEY_LENGTH; + TEE_Result ret = TEE_SUCCESS; + uint32_t *key = otp_key; + bool lock = true; + size_t i = 0; + + ret = get_otp_pos(otp_id); + if (ret) + return ret; + + for (i = 0; i < HUK_NB_OTP; i++) { + ret = stm32mp15_read_otp(otp_id[i], key++, &lock); + if (ret) + goto out; + } + + if (IS_ENABLED(CFG_STM32MP15_HUK_BSEC_KEY)) { + static_assert(sizeof(otp_key) == HW_UNIQUE_KEY_LENGTH); + memcpy(hwkey->data, otp_key, HW_UNIQUE_KEY_LENGTH); + ret = TEE_SUCCESS; + goto out; + } + + if (IS_ENABLED(CFG_STM32MP15_HUK_BSEC_DERIVE_UID)) { + ret = aes_gcm_encrypt_uid((uint8_t *)otp_key, len, hwkey->data, + &len); + if (len != HW_UNIQUE_KEY_LENGTH) + ret = TEE_ERROR_GENERIC; + goto out; + } + + panic(); + +out: + memzero_explicit(otp_key, HW_UNIQUE_KEY_LENGTH); + + if (!ret && !stm32mp15_huk_init) { + stm32mp15_huk_init = true; + IMSG("HUK %slocked", lock ? "" : "un"); + DHEXDUMP(hwkey->data, HW_UNIQUE_KEY_LENGTH); + } + + return ret; +} + diff --git a/optee_os/core/drivers/stpmic1.c b/optee_os/core/drivers/stpmic1.c new file mode 100644 index 0000000..444b983 --- /dev/null +++ b/optee_os/core/drivers/stpmic1.c @@ -0,0 +1,1027 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2016-2020, STMicroelectronics - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define VOLTAGE_INDEX_INVALID ((unsigned int)~0) + +struct regul_struct { + const char *dt_node_name; + const uint16_t *voltage_table; + uint8_t voltage_table_size; + uint8_t control_reg; + uint8_t low_power_reg; + uint8_t enable_pos; + uint8_t pull_down_reg; + uint8_t pull_down_pos; + uint8_t mask_reset_reg; + uint8_t mask_reset_pos; +}; + +static struct i2c_handle_s *pmic_i2c_handle; +static uint16_t pmic_i2c_addr; + +/* Voltage tables in mV */ +static const uint16_t buck1_voltage_table[] = { + 725, + 725, + 725, + 725, + 725, + 725, + 750, + 775, + 800, + 825, + 850, + 875, + 900, + 925, + 950, + 975, + 1000, + 1025, + 1050, + 1075, + 1100, + 1125, + 1150, + 1175, + 1200, + 1225, + 1250, + 1275, + 1300, + 1325, + 1350, + 1375, + 1400, + 1425, + 1450, + 1475, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, + 1500, +}; + +static const uint16_t buck2_voltage_table[] = { + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1050, + 1050, + 1100, + 1100, + 1150, + 1150, + 1200, + 1200, + 1250, + 1250, + 1300, + 1300, + 1350, + 1350, + 1400, + 1400, + 1450, + 1450, + 1500, +}; + +static const uint16_t buck3_voltage_table[] = { + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1100, + 1100, + 1100, + 1100, + 1200, + 1200, + 1200, + 1200, + 1300, + 1300, + 1300, + 1300, + 1400, + 1400, + 1400, + 1400, + 1500, + 1600, + 1700, + 1800, + 1900, + 2000, + 2100, + 2200, + 2300, + 2400, + 2500, + 2600, + 2700, + 2800, + 2900, + 3000, + 3100, + 3200, + 3300, + 3400, +}; + +static const uint16_t buck4_voltage_table[] = { + 600, + 625, + 650, + 675, + 700, + 725, + 750, + 775, + 800, + 825, + 850, + 875, + 900, + 925, + 950, + 975, + 1000, + 1025, + 1050, + 1075, + 1100, + 1125, + 1150, + 1175, + 1200, + 1225, + 1250, + 1275, + 1300, + 1300, + 1350, + 1350, + 1400, + 1400, + 1450, + 1450, + 1500, + 1600, + 1700, + 1800, + 1900, + 2000, + 2100, + 2200, + 2300, + 2400, + 2500, + 2600, + 2700, + 2800, + 2900, + 3000, + 3100, + 3200, + 3300, + 3400, + 3500, + 3600, + 3700, + 3800, + 3900, +}; + +static const uint16_t ldo1_voltage_table[] = { + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1800, + 1900, + 2000, + 2100, + 2200, + 2300, + 2400, + 2500, + 2600, + 2700, + 2800, + 2900, + 3000, + 3100, + 3200, + 3300, +}; + +static const uint16_t ldo2_voltage_table[] = { + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1800, + 1900, + 2000, + 2100, + 2200, + 2300, + 2400, + 2500, + 2600, + 2700, + 2800, + 2900, + 3000, + 3100, + 3200, + 3300, +}; + +static const uint16_t ldo3_voltage_table[] = { + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1800, + 1900, + 2000, + 2100, + 2200, + 2300, + 2400, + 2500, + 2600, + 2700, + 2800, + 2900, + 3000, + 3100, + 3200, + 3300, + 3300, + 3300, + 3300, + 3300, + 3300, + 3300, + 500, /* VOUT2/2 (Sink/source mode) */ + 0xFFFF, /* VREFDDR */ +}; + +static const uint16_t ldo5_voltage_table[] = { + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1700, + 1800, + 1900, + 2000, + 2100, + 2200, + 2300, + 2400, + 2500, + 2600, + 2700, + 2800, + 2900, + 3000, + 3100, + 3200, + 3300, + 3400, + 3500, + 3600, + 3700, + 3800, + 3900, +}; + +static const uint16_t ldo6_voltage_table[] = { + 900, + 1000, + 1100, + 1200, + 1300, + 1400, + 1500, + 1600, + 1700, + 1800, + 1900, + 2000, + 2100, + 2200, + 2300, + 2400, + 2500, + 2600, + 2700, + 2800, + 2900, + 3000, + 3100, + 3200, + 3300, +}; + +static const uint16_t ldo4_voltage_table[] = { + 3300, +}; + +static const uint16_t vref_ddr_voltage_table[] = { + 3300, +}; + +static const uint16_t fixed_5v_voltage_table[] = { + 5000, +}; + +/* Table of Regulators in PMIC SoC */ +static const struct regul_struct regulators_table[] = { + { + .dt_node_name = "buck1", + .voltage_table = buck1_voltage_table, + .voltage_table_size = ARRAY_SIZE(buck1_voltage_table), + .control_reg = BUCK1_CONTROL_REG, + .low_power_reg = BUCK1_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .pull_down_reg = BUCK_PULL_DOWN_REG, + .pull_down_pos = BUCK1_PULL_DOWN_SHIFT, + .mask_reset_reg = MASK_RESET_BUCK_REG, + .mask_reset_pos = BUCK1_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "buck2", + .voltage_table = buck2_voltage_table, + .voltage_table_size = ARRAY_SIZE(buck2_voltage_table), + .control_reg = BUCK2_CONTROL_REG, + .low_power_reg = BUCK2_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .pull_down_reg = BUCK_PULL_DOWN_REG, + .pull_down_pos = BUCK2_PULL_DOWN_SHIFT, + .mask_reset_reg = MASK_RESET_BUCK_REG, + .mask_reset_pos = BUCK2_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "buck3", + .voltage_table = buck3_voltage_table, + .voltage_table_size = ARRAY_SIZE(buck3_voltage_table), + .control_reg = BUCK3_CONTROL_REG, + .low_power_reg = BUCK3_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .pull_down_reg = BUCK_PULL_DOWN_REG, + .pull_down_pos = BUCK3_PULL_DOWN_SHIFT, + .mask_reset_reg = MASK_RESET_BUCK_REG, + .mask_reset_pos = BUCK3_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "buck4", + .voltage_table = buck4_voltage_table, + .voltage_table_size = ARRAY_SIZE(buck4_voltage_table), + .control_reg = BUCK4_CONTROL_REG, + .low_power_reg = BUCK4_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .pull_down_reg = BUCK_PULL_DOWN_REG, + .pull_down_pos = BUCK4_PULL_DOWN_SHIFT, + .mask_reset_reg = MASK_RESET_BUCK_REG, + .mask_reset_pos = BUCK4_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "ldo1", + .voltage_table = ldo1_voltage_table, + .voltage_table_size = ARRAY_SIZE(ldo1_voltage_table), + .control_reg = LDO1_CONTROL_REG, + .low_power_reg = LDO1_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .mask_reset_reg = MASK_RESET_LDO_REG, + .mask_reset_pos = LDO1_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "ldo2", + .voltage_table = ldo2_voltage_table, + .voltage_table_size = ARRAY_SIZE(ldo2_voltage_table), + .control_reg = LDO2_CONTROL_REG, + .low_power_reg = LDO2_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .mask_reset_reg = MASK_RESET_LDO_REG, + .mask_reset_pos = LDO2_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "ldo3", + .voltage_table = ldo3_voltage_table, + .voltage_table_size = ARRAY_SIZE(ldo3_voltage_table), + .control_reg = LDO3_CONTROL_REG, + .low_power_reg = LDO3_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .mask_reset_reg = MASK_RESET_LDO_REG, + .mask_reset_pos = LDO3_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "ldo4", + .voltage_table = ldo4_voltage_table, + .voltage_table_size = ARRAY_SIZE(ldo4_voltage_table), + .control_reg = LDO4_CONTROL_REG, + .low_power_reg = LDO4_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .mask_reset_reg = MASK_RESET_LDO_REG, + .mask_reset_pos = LDO4_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "ldo5", + .voltage_table = ldo5_voltage_table, + .voltage_table_size = ARRAY_SIZE(ldo5_voltage_table), + .control_reg = LDO5_CONTROL_REG, + .low_power_reg = LDO5_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .mask_reset_reg = MASK_RESET_LDO_REG, + .mask_reset_pos = LDO5_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "ldo6", + .voltage_table = ldo6_voltage_table, + .voltage_table_size = ARRAY_SIZE(ldo6_voltage_table), + .control_reg = LDO6_CONTROL_REG, + .low_power_reg = LDO6_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .mask_reset_reg = MASK_RESET_LDO_REG, + .mask_reset_pos = LDO6_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "vref_ddr", + .voltage_table = vref_ddr_voltage_table, + .voltage_table_size = ARRAY_SIZE(vref_ddr_voltage_table), + .control_reg = VREF_DDR_CONTROL_REG, + .low_power_reg = VREF_DDR_PWRCTRL_REG, + .enable_pos = LDO_BUCK_ENABLE_POS, + .mask_reset_reg = MASK_RESET_LDO_REG, + .mask_reset_pos = VREF_DDR_MASK_RESET_SHIFT, + }, + { + .dt_node_name = "boost", + .voltage_table = fixed_5v_voltage_table, + .voltage_table_size = ARRAY_SIZE(fixed_5v_voltage_table), + .control_reg = USB_CONTROL_REG, + .enable_pos = BOOST_ENABLED_POS, + }, + { + .dt_node_name = "pwr_sw1", + .voltage_table = fixed_5v_voltage_table, + .voltage_table_size = ARRAY_SIZE(fixed_5v_voltage_table), + .control_reg = USB_CONTROL_REG, + .enable_pos = USBSW_OTG_SWITCH_ENABLED_POS, + }, + { + .dt_node_name = "pwr_sw2", + .voltage_table = fixed_5v_voltage_table, + .voltage_table_size = ARRAY_SIZE(fixed_5v_voltage_table), + .control_reg = USB_CONTROL_REG, + .enable_pos = SWIN_SWOUT_ENABLED_POS, + }, +}; + +static const struct regul_struct *get_regulator_data(const char *name) +{ + unsigned int i = 0; + + for (i = 0; i < ARRAY_SIZE(regulators_table); i++) + if (strcmp(name, regulators_table[i].dt_node_name) == 0) + return ®ulators_table[i]; + + DMSG("Regulator %s not found", name); + return NULL; +} + +bool stpmic1_regulator_is_valid(const char *name) +{ + return get_regulator_data(name); +} + +void stpmic1_regulator_levels_mv(const char *name, + const uint16_t **levels, + size_t *levels_count) +{ + const struct regul_struct *regul = get_regulator_data(name); + + assert(regul); + + if (levels_count) { + *levels_count = regul->voltage_table_size; + + /* Remove last cell 0xFFFF that is not a voltage level */ + if (regul->voltage_table == ldo3_voltage_table) + (*levels_count)--; + } + + if (levels) + *levels = regul->voltage_table; +} + +static size_t voltage_to_index(const char *name, uint16_t millivolts) +{ + const struct regul_struct *regul = get_regulator_data(name); + unsigned int i = 0; + + assert(regul->voltage_table); + for (i = 0; i < regul->voltage_table_size; i++) + if (regul->voltage_table[i] == millivolts) + return i; + + return VOLTAGE_INDEX_INVALID; +} + +int stpmic1_powerctrl_on(void) +{ + return stpmic1_register_update(MAIN_CONTROL_REG, PWRCTRL_PIN_VALID, + PWRCTRL_PIN_VALID); +} + +int stpmic1_switch_off(void) +{ + return stpmic1_register_update(MAIN_CONTROL_REG, 1, + SOFTWARE_SWITCH_OFF_ENABLED); +} + +int stpmic1_regulator_enable(const char *name) +{ + const struct regul_struct *regul = get_regulator_data(name); + + return stpmic1_register_update(regul->control_reg, + BIT(regul->enable_pos), + BIT(regul->enable_pos)); +} + +int stpmic1_regulator_disable(const char *name) +{ + const struct regul_struct *regul = get_regulator_data(name); + + return stpmic1_register_update(regul->control_reg, 0, + BIT(regul->enable_pos)); +} + +bool stpmic1_is_regulator_enabled(const char *name) +{ + const struct regul_struct *regul = get_regulator_data(name); + uint8_t val = 0; + + if (stpmic1_register_read(regul->control_reg, &val)) + panic(); + + return val & BIT(regul->enable_pos); +} + +/* Voltage can be set for buck or ldo (except ldo4) regulators */ +static uint8_t find_plat_mask(const char *name) +{ + if (!strncmp(name, "buck", 4)) + return BUCK_VOLTAGE_MASK; + + if (!strncmp(name, "ldo", 3) && strcmp(name, "ldo4")) + return LDO_VOLTAGE_MASK; + + return 0; +} + +int stpmic1_regulator_voltage_set(const char *name, uint16_t millivolts) +{ + size_t voltage_index = voltage_to_index(name, millivolts); + const struct regul_struct *regul = get_regulator_data(name); + uint8_t mask = 0; + + if (voltage_index == VOLTAGE_INDEX_INVALID) + return -1; + + mask = find_plat_mask(name); + if (!mask) + return 0; + + return stpmic1_register_update(regul->control_reg, + voltage_index << LDO_BUCK_VOLTAGE_SHIFT, + mask); +} + +int stpmic1_regulator_mask_reset_set(const char *name) +{ + const struct regul_struct *regul = get_regulator_data(name); + + if (regul->control_reg == USB_CONTROL_REG) { + DMSG("No reset for USB control"); + return -1; + } + + return stpmic1_register_update(regul->mask_reset_reg, + BIT(regul->mask_reset_pos), + LDO_BUCK_RESET_MASK << + regul->mask_reset_pos); +} + +int stpmic1_bo_enable_cfg(const char *name, struct stpmic1_bo_cfg *cfg) +{ + const struct regul_struct *regul = get_regulator_data(name); + + cfg->ctrl_reg = regul->control_reg; + cfg->enable_pos = regul->enable_pos; + + return 0; +} + +int stpmic1_bo_enable_unpg(struct stpmic1_bo_cfg *cfg) +{ + return stpmic1_register_update(cfg->ctrl_reg, + BIT(cfg->enable_pos), + BIT(cfg->enable_pos)); +} + +/* Returns 1 if no configuration are expected applied at runtime, 0 otherwise */ +int stpmic1_bo_voltage_cfg(const char *name, uint16_t min_millivolt, + struct stpmic1_bo_cfg *cfg) +{ + size_t min_index = voltage_to_index(name, min_millivolt); + const struct regul_struct *regul = get_regulator_data(name); + uint8_t mask = 0; + + if (min_index == VOLTAGE_INDEX_INVALID) + panic(); + + mask = find_plat_mask(name); + if (!mask) + return 1; + + cfg->ctrl_reg = regul->control_reg; + cfg->min_value = min_index << LDO_BUCK_VOLTAGE_SHIFT; + cfg->mask = mask; + + return 0; +} + +int stpmic1_bo_voltage_unpg(struct stpmic1_bo_cfg *cfg) +{ + uint8_t value = 0; + + assert(cfg->ctrl_reg); + + if (stpmic1_register_read(cfg->ctrl_reg, &value)) + return -1; + + if ((value & cfg->mask) >= cfg->min_value) + return 0; + + return stpmic1_register_update(cfg->ctrl_reg, cfg->min_value, + cfg->mask); +} + +int stpmic1_bo_pull_down_cfg(const char *name, struct stpmic1_bo_cfg *cfg) +{ + const struct regul_struct *regul = get_regulator_data(name); + + if (!regul->pull_down_reg) { + DMSG("No pull down for regu %s", name); + panic(); + } + + cfg->pd_reg = regul->pull_down_reg; + cfg->pd_value = BIT(regul->pull_down_pos); + cfg->pd_mask = LDO_BUCK_PULL_DOWN_MASK << regul->pull_down_pos; + + return 0; +} + +int stpmic1_bo_pull_down_unpg(struct stpmic1_bo_cfg *cfg) +{ + assert(cfg->pd_reg); + + return stpmic1_register_update(cfg->pd_reg, cfg->pd_value, + cfg->pd_mask); +} + +int stpmic1_bo_mask_reset_cfg(const char *name, struct stpmic1_bo_cfg *cfg) +{ + const struct regul_struct *regul = get_regulator_data(name); + + if (!regul->mask_reset_reg) { + DMSG("No reset mask for regu %s", name); + panic(); + } + + cfg->mrst_reg = regul->mask_reset_reg; + cfg->mrst_value = BIT(regul->mask_reset_pos); + cfg->mrst_mask = LDO_BUCK_RESET_MASK << regul->mask_reset_pos; + + return 0; +} + +int stpmic1_bo_mask_reset_unpg(struct stpmic1_bo_cfg *cfg) +{ + assert(cfg->mrst_reg); + + return stpmic1_register_update(cfg->mrst_reg, cfg->mrst_value, + cfg->mrst_mask); +} + +int stpmic1_regulator_voltage_get(const char *name) +{ + const struct regul_struct *regul = get_regulator_data(name); + uint8_t value = 0; + uint8_t mask = 0; + + mask = find_plat_mask(name); + if (!mask) + return 0; + + if (stpmic1_register_read(regul->control_reg, &value)) + return -1; + + value = (value & mask) >> LDO_BUCK_VOLTAGE_SHIFT; + + if (value > regul->voltage_table_size) + return -1; + + return regul->voltage_table[value]; +} + +int stpmic1_lp_copy_reg(const char *name) +{ + const struct regul_struct *regul = get_regulator_data(name); + uint8_t val = 0; + int status = 0; + + if (!regul->low_power_reg) + return -1; + + status = stpmic1_register_read(regul->control_reg, &val); + if (status) + return status; + + return stpmic1_register_write(regul->low_power_reg, val); +} + +bool stpmic1_regu_has_lp_cfg(const char *name) +{ + return get_regulator_data(name)->low_power_reg; +} + +int stpmic1_lp_cfg(const char *name, struct stpmic1_lp_cfg *cfg) +{ + const struct regul_struct *regul = get_regulator_data(name); + + if (!regul->low_power_reg) + return -1; + + cfg->ctrl_reg = regul->control_reg; + cfg->lp_reg = regul->low_power_reg; + + return 0; +} + +int stpmic1_lp_load_unpg(struct stpmic1_lp_cfg *cfg) +{ + uint8_t val = 0; + int status = 0; + + assert(cfg->lp_reg); + + status = stpmic1_register_read(cfg->ctrl_reg, &val); + if (!status) + status = stpmic1_register_write(cfg->lp_reg, val); + + return status; +} + +int stpmic1_lp_reg_on_off(const char *name, uint8_t enable) +{ + const struct regul_struct *regul = get_regulator_data(name); + + if (!regul->low_power_reg) + return -1; + + return stpmic1_register_update(regul->low_power_reg, enable, + LDO_BUCK_ENABLE_MASK); +} + +int stpmic1_lp_on_off_unpg(struct stpmic1_lp_cfg *cfg, int enable) +{ + assert(cfg->lp_reg && (enable == 0 || enable == 1)); + + return stpmic1_register_update(cfg->lp_reg, enable, + LDO_BUCK_ENABLE_MASK); +} + +int stpmic1_lp_set_mode(const char *name, uint8_t hplp) +{ + const struct regul_struct *regul = get_regulator_data(name); + + assert(regul->low_power_reg && (hplp == 0 || hplp == 1)); + + return stpmic1_register_update(regul->low_power_reg, + hplp << LDO_BUCK_HPLP_POS, + BIT(LDO_BUCK_HPLP_POS)); +} + +int stpmic1_lp_mode_unpg(struct stpmic1_lp_cfg *cfg, unsigned int mode) +{ + assert(cfg->lp_reg && (mode == 0 || mode == 1)); + return stpmic1_register_update(cfg->lp_reg, + mode << LDO_BUCK_HPLP_POS, + BIT(LDO_BUCK_HPLP_POS)); +} + +int stpmic1_lp_set_voltage(const char *name, uint16_t millivolts) +{ + size_t voltage_index = voltage_to_index(name, millivolts); + const struct regul_struct *regul = get_regulator_data(name); + uint8_t mask = 0; + + assert(voltage_index != VOLTAGE_INDEX_INVALID); + + mask = find_plat_mask(name); + if (!mask) + return 0; + + return stpmic1_register_update(regul->low_power_reg, voltage_index << 2, + mask); +} + +/* Returns 1 if no configuration are expected applied at runtime, 0 otherwise */ +int stpmic1_lp_voltage_cfg(const char *name, uint16_t millivolts, + struct stpmic1_lp_cfg *cfg) + +{ + size_t voltage_index = voltage_to_index(name, millivolts); + uint8_t mask = 0; + + mask = find_plat_mask(name); + if (!mask) + return 1; + + assert(voltage_index != VOLTAGE_INDEX_INVALID && + cfg->lp_reg == get_regulator_data(name)->low_power_reg); + + cfg->value = voltage_index << 2; + cfg->mask = mask; + + return 0; +} + +int stpmic1_lp_voltage_unpg(struct stpmic1_lp_cfg *cfg) +{ + assert(cfg->lp_reg); + + return stpmic1_register_update(cfg->lp_reg, cfg->value, cfg->mask); +} + +int stpmic1_register_read(uint8_t register_id, uint8_t *value) +{ + struct i2c_handle_s *i2c = pmic_i2c_handle; + + return stm32_i2c_read_write_membyte(i2c, pmic_i2c_addr, + register_id, value, + false /* !write */); +} + +int stpmic1_register_write(uint8_t register_id, uint8_t value) +{ + struct i2c_handle_s *i2c = pmic_i2c_handle; + uint8_t val = value; + + return stm32_i2c_read_write_membyte(i2c, pmic_i2c_addr, + register_id, &val, + true /* write */); +} + +int stpmic1_register_update(uint8_t register_id, uint8_t value, uint8_t mask) +{ + int status = 0; + uint8_t val = 0; + + status = stpmic1_register_read(register_id, &val); + if (status) + return status; + + val = (val & ~mask) | (value & mask); + + return stpmic1_register_write(register_id, val); +} + +void stpmic1_bind_i2c(struct i2c_handle_s *i2c_handle, uint16_t i2c_addr) +{ + pmic_i2c_handle = i2c_handle; + pmic_i2c_addr = i2c_addr; +} + +void stpmic1_dump_regulators(void) +{ + size_t i = 0; + char __maybe_unused const *name = NULL; + + for (i = 0; i < ARRAY_SIZE(regulators_table); i++) { + if (!regulators_table[i].control_reg) + continue; + + name = regulators_table[i].dt_node_name; + DMSG("PMIC regul %s: %sable, %dmV", + name, stpmic1_is_regulator_enabled(name) ? "en" : "dis", + stpmic1_regulator_voltage_get(name)); + } +} + +int stpmic1_get_version(unsigned long *version) +{ + uint8_t read_val = 0; + + if (stpmic1_register_read(VERSION_STATUS_REG, &read_val)) + return -1; + + *version = read_val; + return 0; +} diff --git a/optee_os/core/drivers/sub.mk b/optee_os/core/drivers/sub.mk new file mode 100644 index 0000000..9e7625a --- /dev/null +++ b/optee_os/core/drivers/sub.mk @@ -0,0 +1,92 @@ +incdirs-$(CFG_VERSAL_SHA3_384) += crypto/versal/include + +srcs-$(CFG_CDNS_UART) += cdns_uart.c +srcs-$(CFG_PL011) += pl011.c +srcs-$(CFG_TZC400) += tzc400.c +srcs-$(CFG_TZC380) += tzc380.c +srcs-$(CFG_GIC) += gic.c +srcs-$(CFG_CORE_HAFNIUM_INTC) += hfic.c +srcs-$(CFG_PL061) += pl061_gpio.c +srcs-$(CFG_PL022) += pl022_spi.c +srcs-$(CFG_SP805_WDT) += sp805_wdt.c +srcs-$(CFG_8250_UART) += serial8250_uart.c +srcs-$(CFG_16550_UART) += ns16550.c +srcs-$(CFG_IMX_SNVS) += imx_snvs.c +srcs-$(CFG_IMX_UART) += imx_uart.c +srcs-$(CFG_IMX_I2C) += imx_i2c.c +srcs-$(CFG_IMX_LPUART) += imx_lpuart.c +srcs-$(CFG_IMX_WDOG) += imx_wdog.c +srcs-$(CFG_SPRD_UART) += sprd_uart.c +srcs-$(CFG_HI16XX_UART) += hi16xx_uart.c +srcs-$(CFG_HI16XX_RNG) += hi16xx_rng.c +srcs-$(CFG_LPC_UART) += lpc_uart.c +srcs-$(CFG_SCIF) += scif.c +srcs-$(CFG_DRA7_RNG) += dra7_rng.c +srcs-$(CFG_STIH_UART) += stih_asc.c +srcs-$(CFG_ATMEL_UART) += atmel_uart.c +srcs-$(CFG_ATMEL_TRNG) += atmel_trng.c +srcs-$(CFG_ATMEL_RSTC) += atmel_rstc.c +srcs-$(CFG_ATMEL_SHDWC) += atmel_shdwc.c atmel_shdwc_a32.S +srcs-$(CFG_ATMEL_SAIC) += atmel_saic.c +srcs-$(CFG_ATMEL_WDT) += atmel_wdt.c +srcs-$(CFG_ATMEL_RTC) += atmel_rtc.c +srcs-$(CFG_ATMEL_PIOBU) += atmel_piobu.c +srcs-$(CFG_ATMEL_TCB) += atmel_tcb.c +srcs-$(CFG_AMLOGIC_UART) += amlogic_uart.c +srcs-$(CFG_MVEBU_UART) += mvebu_uart.c +srcs-$(CFG_STM32_BSEC) += stm32_bsec.c +srcs-$(CFG_STM32_ETZPC) += stm32_etzpc.c +srcs-$(CFG_STM32_GPIO) += stm32_gpio.c +srcs-$(CFG_STM32_IWDG) += stm32_iwdg.c +srcs-$(CFG_STM32_I2C) += stm32_i2c.c +srcs-$(CFG_STM32_RNG) += stm32_rng.c +srcs-$(CFG_STM32_SHARED_IO) += stm32_shared_io.c +srcs-$(CFG_STM32_TAMP) += stm32_tamp.c +srcs-$(CFG_STM32_UART) += stm32_uart.c +srcs-$(CFG_STPMIC1) += stpmic1.c +srcs-$(CFG_BCM_HWRNG) += bcm_hwrng.c +srcs-$(CFG_BCM_SOTP) += bcm_sotp.c +srcs-$(CFG_BCM_GPIO) += bcm_gpio.c +srcs-$(CFG_LS_I2C) += ls_i2c.c +srcs-$(CFG_LS_GPIO) += ls_gpio.c +srcs-$(CFG_LS_DSPI) += ls_dspi.c +srcs-$(CFG_LS_SEC_MON) += ls_sec_mon.c +srcs-$(CFG_LS_SFP) += ls_sfp.c +srcs-$(CFG_IMX_RNGB) += imx_rngb.c +srcs-$(CFG_IMX_OCOTP) += imx_ocotp.c +srcs-$(CFG_IMX_CAAM) += imx_caam.c +srcs-$(CFG_IMX_SCU) += imx_scu.c +srcs-$(CFG_IMX_CSU) += imx_csu.c +srcs-$(CFG_XIPHERA_TRNG) += xiphera_trng.c +srcs-$(CFG_IMX_SC) += imx_sc_api.c +srcs-$(CFG_IMX_ELE) += imx_ele.c +srcs-$(CFG_ZYNQMP_CSU_PUF) += zynqmp_csu_puf.c +srcs-$(CFG_ZYNQMP_CSUDMA) += zynqmp_csudma.c +srcs-$(CFG_ZYNQMP_CSU_AES) += zynqmp_csu_aes.c +srcs-$(CFG_ZYNQMP_PM) += zynqmp_pm.c +srcs-$(CFG_ZYNQMP_HUK) += zynqmp_huk.c +srcs-$(CFG_ARM_SMCCC_TRNG) += smccc_trng.c +srcs-$(CFG_VERSAL_GPIO) += versal_gpio.c +srcs-$(CFG_VERSAL_MBOX) += versal_mbox.c +srcs-$(CFG_VERSAL_PM) += versal_pm.c +srcs-$(CFG_STM32MP15_HUK) += stm32mp15_huk.c +srcs-$(CFG_VERSAL_RNG_DRV) += versal_trng.c +srcs-$(CFG_VERSAL_NVM) += versal_nvm.c +srcs-$(CFG_VERSAL_SHA3_384) += versal_sha3_384.c +srcs-$(CFG_VERSAL_PUF) += versal_puf.c +srcs-$(CFG_VERSAL_HUK) += versal_huk.c +srcs-$(CFG_CBMEM_CONSOLE) += cbmem_console.c +srcs-$(CFG_RISCV_PLIC) += plic.c +subdirs-y += crypto +subdirs-$(CFG_BNXT_FW) += bnxt +subdirs-$(CFG_DRIVERS_CLK) += clk +subdirs-$(CFG_DRIVERS_GPIO) += gpio +subdirs-$(CFG_DRIVERS_I2C) += i2c +subdirs-$(CFG_DRIVERS_PINCTRL) += pinctrl +subdirs-$(CFG_DRIVERS_REGULATOR) += regulator +subdirs-$(CFG_DRIVERS_RSTCTRL) += rstctrl +subdirs-$(CFG_SCMI_MSG_DRIVERS) += scmi-msg +subdirs-y += imx +subdirs-y += pm +subdirs-y += wdt +subdirs-y += rtc diff --git a/optee_os/core/drivers/tzc380.c b/optee_os/core/drivers/tzc380.c new file mode 100644 index 0000000..4403391 --- /dev/null +++ b/optee_os/core/drivers/tzc380.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2017-2020 NXP + * All rights reserved. + * + * Peng Fan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Implementation defined values used to validate inputs later. + * Filters : max of 4 ; 0 to 3 + * Regions : max of 9 ; 0 to 8 + * Address width : Values between 32 to 64 + */ +struct tzc_instance { + vaddr_t base; + uint8_t addr_width; + uint8_t num_regions; +}; + +static struct tzc_instance tzc; + +static uint32_t tzc_read_build_config(vaddr_t base) +{ + return io_read32(base + BUILD_CONFIG_OFF); +} + +static void tzc_write_action(vaddr_t base, enum tzc_action action) +{ + io_write32(base + ACTION_OFF, action); +} + +static uint32_t tzc_read_action(vaddr_t base) +{ + return io_read32(base + ACTION_OFF); +} + +static void tzc_write_region_base_low(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_SETUP_LOW_OFF(region), val); +} + +static void tzc_write_region_base_high(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_SETUP_HIGH_OFF(region), val); +} + +static uint32_t tzc_read_region_attributes(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_ATTRIBUTES_OFF(region)); +} + +static void tzc_write_region_attributes(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_ATTRIBUTES_OFF(region), val); +} + +void tzc_init(vaddr_t base) +{ + uint32_t tzc_build; + + assert(base); + tzc.base = base; + + /* Save values we will use later. */ + tzc_build = tzc_read_build_config(tzc.base); + tzc.addr_width = ((tzc_build >> BUILD_CONFIG_AW_SHIFT) & + BUILD_CONFIG_AW_MASK) + 1; + tzc.num_regions = ((tzc_build >> BUILD_CONFIG_NR_SHIFT) & + BUILD_CONFIG_NR_MASK) + 1; +} + +/* + * There are two modes of operation for the region security + * permissions, with or without security inversion. + * Check TZC380 "2.2.5 Region security permissions" for + * more details. + */ +void tzc_security_inversion_en(vaddr_t base) +{ + io_write32(base + SECURITY_INV_EN_OFF, 1); +} + +/* + * Enable a single region. Sometimes we could not use tzc_configure_region + * to enable the region, when security inversion is on. + * When need security inversion, we need to first configure + * region address and attribute, then configure security inversion, + * then enable the regions. + */ +void tzc_region_enable(uint8_t region) +{ + uint32_t val; + + val = tzc_read_region_attributes(tzc.base, region); + val |= TZC_ATTR_REGION_EN_MASK; + tzc_write_region_attributes(tzc.base, region, val); +} + +/* + * Dump info when TZC380 catchs an unallowed access with TZC + * interrupt enabled. + */ +void tzc_fail_dump(void) +{ + vaddr_t base __maybe_unused = core_mmu_get_va(tzc.base, + MEM_AREA_IO_SEC, + TZC400_REG_SIZE); + + EMSG("Fail address Low 0x%" PRIx32, + io_read32(base + FAIL_ADDRESS_LOW_OFF)); + EMSG("Fail address High 0x%" PRIx32, + io_read32(base + FAIL_ADDRESS_HIGH_OFF)); + EMSG("Fail Control 0x%" PRIx32, io_read32(base + FAIL_CONTROL_OFF)); + EMSG("Fail Id 0x%" PRIx32, io_read32(base + FAIL_ID)); +} + +void tzc_int_clear(void) +{ + vaddr_t base = core_mmu_get_va(tzc.base, MEM_AREA_IO_SEC, + TZC400_REG_SIZE); + + io_write32(base + INT_CLEAR, 0); +} + +static uint32_t addr_low(vaddr_t addr) +{ + return (uint32_t)addr; +} + +static uint32_t addr_high(vaddr_t addr __maybe_unused) +{ +#if (UINTPTR_MAX == UINT64_MAX) + return addr >> 32; +#else + return 0; +#endif +} + + +/* + * `tzc_configure_region` is used to program regions into the TrustZone + * controller. + */ +void tzc_configure_region(uint8_t region, vaddr_t region_base, uint32_t attr) +{ + assert(tzc.base); + + assert(region < tzc.num_regions); + + /* + * For region 0, this high/low/size/en field is Read Only (RO). + * So should not configure those field for region 0. + */ + if (region) { + tzc_write_region_base_low(tzc.base, region, + addr_low(region_base)); + tzc_write_region_base_high(tzc.base, region, + addr_high(region_base)); + tzc_write_region_attributes(tzc.base, region, attr); + } else { + tzc_write_region_attributes(tzc.base, region, + attr & TZC_ATTR_SP_MASK); + } +} + +void tzc_set_action(enum tzc_action action) +{ + assert(tzc.base); + + /* + * - Currently no handler is provided to trap an error via interrupt + * or exception. + * - The interrupt action has not been tested. + */ + tzc_write_action(tzc.base, action); +} + +uint32_t tzc_get_action(void) +{ + assert(tzc.base); + + return tzc_read_action(tzc.base); +} + +int tzc_auto_configure(vaddr_t addr, vaddr_t size, uint32_t attr, + uint8_t region) +{ + uint64_t sub_region_size = 0; + uint64_t area = 0; + uint8_t lregion = region; + uint64_t region_size = 0; + vaddr_t sub_address = 0; + vaddr_t address = addr; + uint64_t lsize = size; + uint32_t mask = 0; + int i = 0; + uint8_t pow = 0; + + assert(tzc.base); + + /* + * TZC380 RM + * For region_attributes_ registers, region_size: + * Note: The AXI address width, that is AXI_ADDRESS_MSB+1, controls the + * upper limit value of the field. + */ + pow = tzc.addr_width; + + while (lsize != 0 && pow >= 15) { + region_size = 1ULL << pow; + + /* Case region fits alignment and covers requested area */ + if ((address % region_size == 0) && + ((address + lsize) % region_size == 0)) { + tzc_configure_region(lregion, address, + TZC_ATTR_REGION_SIZE(pow - 1) | + TZC_ATTR_REGION_EN_MASK | + attr); + lregion++; + address += region_size; + lsize -= region_size; + pow = tzc.addr_width; + continue; + } + + /* Cover area using several subregions */ + sub_region_size = region_size / 8; + if (address % sub_region_size == 0 && + lsize > 2 * sub_region_size) { + sub_address = (address / region_size) * region_size; + mask = 0; + for (i = 0; i < 8; i++) { + area = (i + 1) * sub_region_size; + if (sub_address + area <= address || + sub_address + area > address + lsize) { + mask |= TZC_ATTR_SUBREGION_DIS(i); + } else { + address += sub_region_size; + lsize -= sub_region_size; + } + } + tzc_configure_region(lregion, sub_address, + TZC_ATTR_REGION_SIZE(pow - 1) | + TZC_ATTR_REGION_EN_MASK | + mask | attr); + lregion++; + pow = tzc.addr_width; + continue; + } + pow--; + } + assert(lsize == 0); + assert(address == addr + size); + return lregion; +} + +/* + * `region_lockdown` is used to lockdown the TZC380 configuration to prevent + * unintended overwrites of the configuration. Returns TEE_ERROR_SECURITY in + * case the lockdown fails. + */ +TEE_Result tzc_regions_lockdown(void) +{ + uint32_t val = 0; + uint32_t check = 0; + + val = LOCKDOWN_RANGE_ENABLE | (tzc.num_regions - 1); + io_write32(tzc.base + LOCKDOWN_RANGE_OFF, val); + check = io_read32(tzc.base + LOCKDOWN_RANGE_OFF); + if (check != val) + return TEE_ERROR_SECURITY; + + val = LOCKDOWN_SELECT_RANGE_ENABLE; + io_write32(tzc.base + LOCKDOWN_SELECT_OFF, val); + check = io_read32(tzc.base + LOCKDOWN_SELECT_OFF); + if (check != val) + return TEE_ERROR_SECURITY; + + return TEE_SUCCESS; +} + +#if TRACE_LEVEL >= TRACE_DEBUG + +static uint32_t tzc_read_region_base_low(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_SETUP_LOW_OFF(region)); +} + +static uint32_t tzc_read_region_base_high(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_SETUP_HIGH_OFF(region)); +} + +#define REGION_MAX 16 +void tzc_dump_state(void) +{ + uint32_t n; + uint32_t temp_32reg, temp_32reg_h; + + DMSG("TZC380 configuration:"); + DMSG("security_inversion_en %x", + io_read32(tzc.base + SECURITY_INV_EN_OFF)); + for (n = 0; n <= REGION_MAX; n++) { + temp_32reg = tzc_read_region_attributes(tzc.base, n); + if (!(temp_32reg & TZC_ATTR_REGION_EN_MASK)) + continue; + + DMSG(""); + DMSG("region %d", n); + temp_32reg = tzc_read_region_base_low(tzc.base, n); + temp_32reg_h = tzc_read_region_base_high(tzc.base, n); + DMSG("region_base: 0x%08x%08x", temp_32reg_h, temp_32reg); + temp_32reg = tzc_read_region_attributes(tzc.base, n); + DMSG("region sp: %x", temp_32reg >> TZC_ATTR_SP_SHIFT); + DMSG("region size: %x", (temp_32reg & TZC_REGION_SIZE_MASK) >> + TZC_REGION_SIZE_SHIFT); + } + DMSG("Lockdown select: %"PRIx32, + io_read32(tzc.base + LOCKDOWN_SELECT_OFF)); + DMSG("Lockdown range: %"PRIx32, + io_read32(tzc.base + LOCKDOWN_RANGE_OFF)); + DMSG("Action register: %"PRIx32, tzc_get_action()); + DMSG("exit"); +} + +#endif /* CFG_TRACE_LEVEL >= TRACE_DEBUG */ diff --git a/optee_os/core/drivers/tzc400.c b/optee_os/core/drivers/tzc400.c new file mode 100644 index 0000000..9d8c0a0 --- /dev/null +++ b/optee_os/core/drivers/tzc400.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: (BSD-2-Clause AND BSD-3-Clause) +/* + * Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + /* + * Copyright (c) 2014, ARM Limited and Contributors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of ARM nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* + * Implementation defined values used to validate inputs later. + * Filters : max of 4 ; 0 to 3 + * Regions : max of 9 ; 0 to 8 + * Address width : Values between 32 to 64 + */ +struct tzc_instance { + vaddr_t base; + uint8_t addr_width; + uint8_t num_filters; + uint8_t num_regions; +}; + +static struct tzc_instance tzc; + + +static uint32_t tzc_read_build_config(vaddr_t base) +{ + return io_read32(base + BUILD_CONFIG_OFF); +} + +static uint32_t tzc_read_gate_keeper(vaddr_t base) +{ + return io_read32(base + GATE_KEEPER_OFF); +} + +static void tzc_write_gate_keeper(vaddr_t base, uint32_t val) +{ + io_write32(base + GATE_KEEPER_OFF, val); +} + +static void tzc_write_action(vaddr_t base, enum tzc_action action) +{ + io_write32(base + ACTION_OFF, action); +} + +static uint32_t tzc_read_region_base_low(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_BASE_LOW_OFF + REGION_NUM_OFF(region)); +} + +static void tzc_write_region_base_low(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_BASE_LOW_OFF + REGION_NUM_OFF(region), val); +} + +static uint32_t tzc_read_region_base_high(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_BASE_HIGH_OFF + REGION_NUM_OFF(region)); +} + +static void tzc_write_region_base_high(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_BASE_HIGH_OFF + REGION_NUM_OFF(region), val); +} + +static uint32_t tzc_read_region_top_low(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_TOP_LOW_OFF + REGION_NUM_OFF(region)); +} + +static void tzc_write_region_top_low(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_TOP_LOW_OFF + REGION_NUM_OFF(region), val); +} + +static uint32_t tzc_read_region_top_high(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_TOP_HIGH_OFF + REGION_NUM_OFF(region)); +} + +static void tzc_write_region_top_high(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_TOP_HIGH_OFF + REGION_NUM_OFF(region), val); +} + +static uint32_t tzc_read_region_attributes(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_ATTRIBUTES_OFF + REGION_NUM_OFF(region)); +} + +static void tzc_write_region_attributes(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_ATTRIBUTES_OFF + REGION_NUM_OFF(region), val); +} + +static uint32_t tzc_read_region_id_access(vaddr_t base, uint32_t region) +{ + return io_read32(base + REGION_ID_ACCESS_OFF + REGION_NUM_OFF(region)); +} + +static void tzc_write_region_id_access(vaddr_t base, uint32_t region, + uint32_t val) +{ + io_write32(base + REGION_ID_ACCESS_OFF + REGION_NUM_OFF(region), val); +} + +static uint32_t tzc_read_component_id(vaddr_t base) +{ + uint32_t id; + + id = io_read8(base + CID0_OFF); + id |= SHIFT_U32(io_read8(base + CID1_OFF), 8); + id |= SHIFT_U32(io_read8(base + CID2_OFF), 16); + id |= SHIFT_U32(io_read8(base + CID3_OFF), 24); + + return id; +} + +static uint32_t tzc_get_gate_keeper(vaddr_t base, uint8_t filter) +{ + uint32_t tmp; + + tmp = (tzc_read_gate_keeper(base) >> GATE_KEEPER_OS_SHIFT) & + GATE_KEEPER_OS_MASK; + + return (tmp >> filter) & GATE_KEEPER_FILTER_MASK; +} + +/* This function is not MP safe. */ +static void tzc_set_gate_keeper(vaddr_t base, uint8_t filter, uint32_t val) +{ + uint32_t tmp; + + /* Upper half is current state. Lower half is requested state. */ + tmp = (tzc_read_gate_keeper(base) >> GATE_KEEPER_OS_SHIFT) & + GATE_KEEPER_OS_MASK; + + if (val) + tmp |= (1 << filter); + else + tmp &= ~(1 << filter); + + tzc_write_gate_keeper(base, (tmp & GATE_KEEPER_OR_MASK) << + GATE_KEEPER_OR_SHIFT); + + /* Wait here until we see the change reflected in the TZC status. */ + while (((tzc_read_gate_keeper(base) >> GATE_KEEPER_OS_SHIFT) & + GATE_KEEPER_OS_MASK) != tmp) + ; +} + + +void tzc_init(vaddr_t base) +{ + uint32_t tzc_id, tzc_build; + + assert(base); + tzc.base = base; + + /* + * We expect to see a tzc400. Check component ID. The TZC-400 TRM shows + * component ID is expected to be "0xB105F00D". + */ + tzc_id = tzc_read_component_id(tzc.base); + if (tzc_id != TZC400_COMPONENT_ID) { + EMSG("TZC : Wrong device ID (0x%" PRIx32 ")", tzc_id); + panic(); + } + + /* Save values we will use later. */ + tzc_build = tzc_read_build_config(tzc.base); + tzc.num_filters = ((tzc_build >> BUILD_CONFIG_NF_SHIFT) & + BUILD_CONFIG_NF_MASK) + 1; + tzc.addr_width = ((tzc_build >> BUILD_CONFIG_AW_SHIFT) & + BUILD_CONFIG_AW_MASK) + 1; + tzc.num_regions = ((tzc_build >> BUILD_CONFIG_NR_SHIFT) & + BUILD_CONFIG_NR_MASK) + 1; +} + +static uint32_t addr_low(vaddr_t addr) +{ + return (uint32_t)addr; +} + +static uint32_t addr_high(vaddr_t addr __unused) +{ +#if (UINTPTR_MAX == UINT64_MAX) + return (addr >> 32); +#else + return 0; +#endif +} + + +/* + * `tzc_configure_region` is used to program regions into the TrustZone + * controller. A region can be associated with more than one filter. The + * associated filters are passed in as a bitmap (bit0 = filter0). + * NOTE: + * The region 0 covers the whole address space and is enabled on all filters, + * this cannot be changed. It is, however, possible to change some region 0 + * permissions. + */ +void tzc_configure_region(uint8_t region, const struct tzc_region_config *cfg) +{ + assert(tzc.base && cfg); + + /* Do range checks on filters and regions. */ + assert(((cfg->filters >> tzc.num_filters) == 0) && + (region < tzc.num_regions)); + + /* + * Do address range check based on TZC configuration. A 64bit address is + * the max and expected case. + */ +#if (UINTPTR_MAX == UINT64_MAX) + assert(((cfg->top <= (UINT64_MAX >> (64 - tzc.addr_width))) && + (cfg->base < cfg->top))); +#endif + /* region_base and (region_top + 1) must be 4KB aligned */ + assert(((cfg->base | (cfg->top + 1)) & (4096 - 1)) == 0); + + assert(cfg->sec_attr <= TZC_REGION_S_RDWR); + + /* + * Inputs look ok, start programming registers. + * All the address registers are 32 bits wide and have a LOW and HIGH + * component used to construct a up to a 64bit address. + */ + tzc_write_region_base_low(tzc.base, region, addr_low(cfg->base)); + tzc_write_region_base_high(tzc.base, region, addr_high(cfg->base)); + + tzc_write_region_top_low(tzc.base, region, addr_low(cfg->top)); + tzc_write_region_top_high(tzc.base, region, addr_high(cfg->top)); + + /* Assign the region to a filter and set secure attributes */ + tzc_write_region_attributes(tzc.base, region, + (cfg->sec_attr << REG_ATTR_SEC_SHIFT) | + cfg->filters); + + /* + * Specify which non-secure devices have permission to access this + * region. + */ + tzc_write_region_id_access(tzc.base, region, cfg->ns_device_access); +} + +TEE_Result tzc_get_region_config(uint8_t region, struct tzc_region_config *cfg) +{ + uint32_t val32 = 0; + + if (region >= tzc.num_regions) + return TEE_ERROR_GENERIC; + + cfg->base = reg_pair_to_64(tzc_read_region_base_high(tzc.base, region), + tzc_read_region_base_low(tzc.base, region)); + cfg->top = reg_pair_to_64(tzc_read_region_top_high(tzc.base, region), + tzc_read_region_top_low(tzc.base, region)); + + cfg->ns_device_access = tzc_read_region_id_access(tzc.base, region); + + val32 = tzc_read_region_attributes(tzc.base, region); + cfg->sec_attr = val32 >> REG_ATTR_SEC_SHIFT; + cfg->filters = val32 & REG_ATTR_F_EN_MASK; + + return TEE_SUCCESS; +} + +void tzc_set_action(enum tzc_action action) +{ + assert(tzc.base); + + /* + * - Currently no handler is provided to trap an error via interrupt + * or exception. + * - The interrupt action has not been tested. + */ + tzc_write_action(tzc.base, action); +} + + +void tzc_enable_filters(void) +{ + uint32_t state; + uint32_t filter; + + assert(tzc.base); + + for (filter = 0; filter < tzc.num_filters; filter++) { + state = tzc_get_gate_keeper(tzc.base, filter); + if (state) { + /* + * The TZC filter is already configured. Changing the + * programmer's view in an active system can cause + * unpredictable behavior therefore panic for now rather + * than try to determine whether this is safe in this + * instance. See: + * http://infocenter.arm.com/help/index.jsp?\ + * topic=/com.arm.doc.ddi0504c/CJHHECBF.html + */ + EMSG("TZC : Filter %d Gatekeeper already enabled", + filter); + panic(); + } + tzc_set_gate_keeper(tzc.base, filter, 1); + } +} + + +void tzc_disable_filters(void) +{ + uint32_t filter; + + assert(tzc.base); + + /* + * We don't do the same state check as above as the Gatekeepers are + * disabled after reset. + */ + for (filter = 0; filter < tzc.num_filters; filter++) + tzc_set_gate_keeper(tzc.base, filter, 0); +} + +static bool __maybe_unused write_not_read(unsigned int filter) +{ + return io_read32(tzc.base + FAIL_CONTROL(filter)) & + FAIL_CONTROL_DIRECTION_WRITE; +} + +static bool __maybe_unused nonsecure_not_secure(unsigned int filter) +{ + return io_read32(tzc.base + FAIL_CONTROL(filter)) & + FAIL_CONTROL_NONSECURE; +} + +static bool __maybe_unused priv_not_unpriv(unsigned int filter) +{ + return io_read32(tzc.base + FAIL_CONTROL(filter)) & + FAIL_CONTROL_PRIVILEGED; +} + +static void dump_fail_filter(unsigned int filter) +{ + uint64_t __maybe_unused addr = 0; + uint32_t status = io_read32(tzc.base + INT_STATUS); + + if (!(status & BIT(filter + INT_STATUS_OVERLAP_SHIFT)) && + !(status & BIT(filter + INT_STATUS_OVERRUN_SHIFT)) && + !(status & BIT(filter + INT_STATUS_STATUS_SHIFT))) + return; + + if (status & BIT(filter + INT_STATUS_OVERLAP_SHIFT)) + EMSG("Overlap violation on filter %u", filter); + + if (status & BIT(filter + INT_STATUS_OVERRUN_SHIFT)) + EMSG("Overrun violation on filter %u", filter); + + if (status & BIT(filter + INT_STATUS_STATUS_SHIFT)) + EMSG("Permission violation on filter %u", filter); + + addr = reg_pair_to_64(io_read32(tzc.base + FAIL_ADDRESS_HIGH(filter)), + io_read32(tzc.base + FAIL_ADDRESS_LOW(filter))); + + EMSG("Violation @0x%"PRIx64", %ssecure %sprivileged %s, AXI ID %"PRIx32, + addr, + nonsecure_not_secure(filter) ? "non-" : "", + priv_not_unpriv(filter) ? "" : "un", + write_not_read(filter) ? "write" : "read", + io_read32(tzc.base + FAIL_ID(filter))); +} + +/* + * Dump info when TZC400 catches an unallowed access with TZC + * interrupt enabled. + */ +void tzc_fail_dump(void) +{ + unsigned int filter = 0; + + for (filter = 0; filter < tzc.num_filters; filter++) + dump_fail_filter(filter); +} + +void tzc_int_clear(void) +{ + assert(tzc.base); + + io_setbits32(tzc.base + INT_CLEAR, GENMASK_32(tzc.num_filters - 1, 0)); +} + +#if TRACE_LEVEL >= TRACE_DEBUG + +#define REGION_MAX 8 +static const __maybe_unused char * const tzc_attr_msg[] = { + "TZC_REGION_S_NONE", + "TZC_REGION_S_RD", + "TZC_REGION_S_WR", + "TZC_REGION_S_RDWR" +}; + +void tzc_dump_state(void) +{ + uint32_t n; + uint32_t temp_32reg, temp_32reg_h; + unsigned int filter = 0; + + for (n = 0; n <= REGION_MAX; n++) { + temp_32reg = tzc_read_region_attributes(tzc.base, n); + if (!(temp_32reg & REG_ATTR_F_EN_MASK)) + continue; + + DMSG("region %d", n); + temp_32reg = tzc_read_region_base_low(tzc.base, n); + temp_32reg_h = tzc_read_region_base_high(tzc.base, n); + DMSG("region_base: 0x%08x%08x", temp_32reg_h, temp_32reg); + temp_32reg = tzc_read_region_top_low(tzc.base, n); + temp_32reg_h = tzc_read_region_top_high(tzc.base, n); + DMSG("region_top: 0x%08x%08x", temp_32reg_h, temp_32reg); + temp_32reg = tzc_read_region_attributes(tzc.base, n); + DMSG("secure rw: %s", + tzc_attr_msg[temp_32reg >> REG_ATTR_SEC_SHIFT]); + + for (filter = 0; filter < tzc.num_filters; filter++) + if (temp_32reg & BIT(filter)) + DMSG("filter %u enable", filter); + } +} + +#endif /* CFG_TRACE_LEVEL >= TRACE_DEBUG */ diff --git a/optee_os/core/drivers/versal_gpio.c b/optee_os/core/drivers/versal_gpio.c new file mode 100644 index 0000000..514ee40 --- /dev/null +++ b/optee_os/core/drivers/versal_gpio.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (C) 2002 - 2021 Xilinx, Inc. All rights reserved. + * Copyright (c) 2022 Foundries.io Ltd. (jorge@foundries.io) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "drivers/versal_gpio.h" + +#define VERSAL_GPIO_LEN 0x10000 + +#define DATA_LSW_OFFSET(__bank) (0x000 + 0x08 * (__bank)) +#define DATA_MSW_OFFSET(__bank) (0x004 + 0x08 * (__bank)) +#define DATA_RO_OFFSET(__bank) (0x060 + 0x04 * (__bank)) +#define DIRM_OFFSET(__bank) (0x204 + 0x40 * (__bank)) +#define OUTEN_OFFSET(__bank) (0x208 + 0x40 * (__bank)) + +#define VERSAL_GPIO_MID_PIN 16 +#define VERSAL_GPIO_UPPER_MASK 0xFFFF0000 + +/* Max pins in the PMC_GPIO devices + * 00 - 025, Bank 0 + * 26 - 051, Bank 1 + * 52 - 083, Bank 3 + * 84 - 115, Bank 4 + */ +#define VERSAL_GPIO_PMC_BASE 0xf1020000 +#define VERSAL_GPIO_PMC_NR_GPIOS 116 +#define VERSAL_GPIO_PMC_MAX_BANK 5 + +static const struct versal_gpio_platform_data versal_gpio_pmc_def = { + .max_bank = VERSAL_GPIO_PMC_MAX_BANK, + .ngpio = VERSAL_GPIO_PMC_NR_GPIOS, + .label = "versal_pmc_gpio", + .bank_min[0] = 0, + .bank_max[0] = 25, + .bank_min[1] = 26, + .bank_max[1] = 51, + .bank_min[3] = 52, + .bank_max[3] = 83, + .bank_min[4] = 84, + .bank_max[4] = 115, +}; + +/* Max pins in the PS_GPIO devices + * 00 - 25, Bank 0 + * 26 - 57, Bank 3 + */ +#define VERSAL_GPIO_PS_BASE 0xff0b0000 +#define VERSAL_GPIO_PS_NR_GPIOS 58 +#define VERSAL_GPIO_PS_MAX_BANK 4 + +static const struct versal_gpio_platform_data versal_gpio_ps_def = { + .max_bank = VERSAL_GPIO_PS_MAX_BANK, + .ngpio = VERSAL_GPIO_PS_NR_GPIOS, + .label = "versal_ps_gpio", + .bank_min[0] = 0, + .bank_max[0] = 25, + .bank_min[3] = 26, + .bank_max[3] = 57, +}; + +static void versal_gpio_get_pin(struct versal_gpio_chip *chip, uint32_t gpio, + uint32_t *bank, uint32_t *pin) +{ + struct versal_gpio_platdata *platdata = &chip->plat; + uint32_t bnk = 0; + + assert(gpio < platdata->p_data->ngpio); + + for (bnk = 0; bnk < platdata->p_data->max_bank; bnk++) { + if (gpio < platdata->p_data->bank_min[bnk]) + continue; + + if (gpio > platdata->p_data->bank_max[bnk]) + continue; + + *bank = bnk; + *pin = gpio - platdata->p_data->bank_min[bnk]; + + return; + } + + EMSG("GPIO_%d not found", gpio); + panic(); +} + +static enum gpio_level versal_gpio_get_value(struct versal_gpio_chip *chip, + uint32_t gpio) +{ + uint32_t bank = 0; + uint32_t pin = 0; + + versal_gpio_get_pin(chip, gpio, &bank, &pin); + + return (io_read32(chip->base + DATA_RO_OFFSET(bank)) >> pin) & 1; +} + +static void versal_gpio_set_value(struct versal_gpio_chip *chip, uint32_t gpio, + enum gpio_level val) +{ + uint32_t bank = 0; + uint32_t off = 0; + uint32_t pin = 0; + + versal_gpio_get_pin(chip, gpio, &bank, &pin); + + if (bank >= VERSAL_GPIO_MID_PIN) { + bank -= VERSAL_GPIO_MID_PIN; + off = DATA_MSW_OFFSET(bank); + } else { + off = DATA_LSW_OFFSET(bank); + } + + /* + * get the 32 bit value to be written to the mask/data register where + * the upper 16 bits is the mask and lower 16 bits is the data + */ + val = !!val; + val = ~BIT32(pin + VERSAL_GPIO_MID_PIN) & + (SHIFT_U32(val, pin) | VERSAL_GPIO_UPPER_MASK); + + io_write32(chip->base + off, val); +} + +static void versal_gpio_set_direction(struct versal_gpio_chip *chip, + uint32_t gpio, enum gpio_dir direction) +{ + uint32_t bank = 0; + uint32_t reg = 0; + uint32_t pin = 0; + + versal_gpio_get_pin(chip, gpio, &bank, &pin); + + if (direction == GPIO_DIR_OUT) { + /* set the GPIO pin as output */ + reg = io_read32(chip->base + DIRM_OFFSET(bank)); + reg |= BIT(pin); + io_write32(chip->base + DIRM_OFFSET(bank), reg); + + /* configure the output enable reg for the pin */ + reg = io_read32(chip->base + OUTEN_OFFSET(bank)); + + reg |= BIT(pin); + io_write32(chip->base + OUTEN_OFFSET(bank), reg); + + /* set the state of the pin */ + versal_gpio_set_value(chip, gpio, GPIO_LEVEL_LOW); + } else { + /* bnk 0 pins 7 and 8 cannot be used as inputs */ + assert(!(bank == 0 && (pin == 7 || pin == 8))); + + reg = io_read32(chip->base + DIRM_OFFSET(bank)); + reg &= ~BIT(pin); + io_write32(chip->base + DIRM_OFFSET(bank), reg); + } +} + +static enum gpio_dir versal_gpio_get_direction(struct versal_gpio_chip *chip, + uint32_t gpio) +{ + uint32_t pin = 0; + uint32_t bank = 0; + + versal_gpio_get_pin(chip, gpio, &bank, &pin); + + if (io_read32(chip->base + DIRM_OFFSET(bank)) & BIT(pin)) + return GPIO_DIR_OUT; + + return GPIO_DIR_IN; +} + +static enum gpio_level do_get_value(struct gpio_chip *chip, uint32_t gpio) +{ + struct versal_gpio_chip *p = container_of(chip, + struct versal_gpio_chip, chip); + return versal_gpio_get_value(p, gpio); +} + +static void do_set_value(struct gpio_chip *chip, uint32_t gpio, + enum gpio_level val) +{ + struct versal_gpio_chip *p = container_of(chip, + struct versal_gpio_chip, chip); + return versal_gpio_set_value(p, gpio, val); +} + +static void do_set_dir(struct gpio_chip *chip, uint32_t gpio, + enum gpio_dir direction) +{ + struct versal_gpio_chip *p = container_of(chip, + struct versal_gpio_chip, chip); + return versal_gpio_set_direction(p, gpio, direction); +} + +static enum gpio_dir do_get_dir(struct gpio_chip *chip, uint32_t gpio) +{ + struct versal_gpio_chip *p = container_of(chip, + struct versal_gpio_chip, chip); + return versal_gpio_get_direction(p, gpio); +} + +static const struct gpio_ops versal_gpio_ops = { + .get_direction = do_get_dir, + .set_direction = do_set_dir, + .get_value = do_get_value, + .set_value = do_set_value, + .get_interrupt = NULL, + .set_interrupt = NULL, +}; + +TEE_Result versal_gpio_pmc_init(struct versal_gpio_chip *chip) +{ + if (chip->base) + return TEE_SUCCESS; + + chip->plat.p_data = &versal_gpio_pmc_def; + chip->plat.base = VERSAL_GPIO_PMC_BASE; + chip->chip.ops = &versal_gpio_ops; + + chip->base = (vaddr_t)core_mmu_add_mapping(MEM_AREA_IO_SEC, + VERSAL_GPIO_PMC_BASE, + VERSAL_GPIO_LEN); + if (!chip->base) { + EMSG("Failed to map gpio"); + chip->chip.ops = NULL; + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +TEE_Result versal_gpio_ps_init(struct versal_gpio_chip *chip) +{ + if (chip->base) + return TEE_SUCCESS; + + chip->plat.p_data = &versal_gpio_ps_def; + chip->plat.base = VERSAL_GPIO_PS_BASE; + chip->chip.ops = &versal_gpio_ops; + + chip->base = (vaddr_t)core_mmu_add_mapping(MEM_AREA_IO_SEC, + VERSAL_GPIO_PS_BASE, + VERSAL_GPIO_LEN); + if (!chip->base) { + EMSG("Failed to map gpio"); + chip->chip.ops = NULL; + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/versal_huk.c b/optee_os/core/drivers/versal_huk.c new file mode 100644 index 0000000..1260c02 --- /dev/null +++ b/optee_os/core/drivers/versal_huk.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Foundries.io Ltd. + * Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct { + uint8_t key[HW_UNIQUE_KEY_LENGTH]; + bool ready; +} huk; + +#define MODULE_SHIFT 8 +#define MODULE_ID 5 +#define API_ID(__x) ((MODULE_ID << MODULE_SHIFT) | (__x)) + +#define VERSAL_AES_KEY_SIZE_256 2 +#define VERSAL_AES_GCM_ENCRYPT 0 + +enum versal_aes_key_src { + VERSAL_AES_BBRAM_KEY = 0, + VERSAL_AES_BBRAM_RED_KEY, + VERSAL_AES_BH_KEY, + VERSAL_AES_BH_RED_KEY, + VERSAL_AES_EFUSE_KEY, + VERSAL_AES_EFUSE_RED_KEY, + VERSAL_AES_EFUSE_USER_KEY_0, + VERSAL_AES_EFUSE_USER_KEY_1, + VERSAL_AES_EFUSE_USER_RED_KEY_0, + VERSAL_AES_EFUSE_USER_RED_KEY_1, + VERSAL_AES_KUP_KEY, + VERSAL_AES_PUF_KEY, + VERSAL_AES_USER_KEY_0, + VERSAL_AES_USER_KEY_1, + VERSAL_AES_USER_KEY_2, + VERSAL_AES_USER_KEY_3, + VERSAL_AES_USER_KEY_4, + VERSAL_AES_USER_KEY_5, + VERSAL_AES_USER_KEY_6, + VERSAL_AES_USER_KEY_7, + VERSAL_AES_EXPANDED_KEYS, + VERSAL_AES_ALL_KEYS, +}; + +enum versal_crypto_api { + VERSAL_AES_INIT = 96U, + VERSAL_AES_OP_INIT, + VERSAL_AES_UPDATE_AAD, + VERSAL_AES_ENCRYPT_UPDATE, + VERSAL_AES_ENCRYPT_FINAL, + VERSAL_AES_DECRYPT_UPDATE, + VERSAL_AES_DECRYPT_FINAL, + VERSAL_AES_KEY_ZERO, + VERSAL_AES_WRITE_KEY, + VERSAL_AES_LOCK_USER_KEY, + VERSAL_AES_KEK_DECRYPT, + VERSAL_AES_SET_DPA_CM, + VERSAL_AES_DECRYPT_KAT, + VERSAL_AES_DECRYPT_CM_KAT, +}; + +struct versal_aes_input_param { + uint64_t input_addr; + uint32_t input_len; + uint32_t is_last; +}; + +struct versal_aes_init { + uint64_t iv_addr; + uint32_t operation; + uint32_t key_src; + uint32_t key_len; +}; + +/* + * The PLM is little endian. When programming the keys in uint32_t the driver + * will BE swap the values. + * + * This way the test key below corresponds to the byte array 0xf8, 0x78, 0xb8, + * 0x38, 0xd8, 0x58, 0x98, 0x18, 0xe8, 0x68, .... + * + * NOTICE: This hardcoded value in DEVEL_KEY could have just been zeroes as done + * in the weak implementation found in otp_stubs.c. + */ +#define DEVEL_KEY { \ + 0xf878b838, 0xd8589818, 0xe868a828, 0xc8488808, \ + 0xf070b030, 0xd0509010, 0xe060a020, 0xc0408000, \ + } + +#define AAD { \ + 0x67, 0xe2, 0x1c, 0xf3, 0xcb, 0x29, 0xe0, 0xdc, 0xbc, 0x4d, \ + 0x8b, 0x1d, 0x0c, 0xc5, 0x33, 0x4b, \ + } + +#define NONCE { \ + 0xd2, 0x45, 0x0e, 0x07, 0xea, 0x5d, 0xe0, 0x42, 0x6c, 0x0f, \ + 0xa1, 0x33, \ + } + +static bool versal_persistent_key(enum versal_aes_key_src src, bool *secure) +{ + struct versal_efuse_puf_sec_ctrl_bits puf_ctrl = { }; + struct versal_efuse_sec_ctrl_bits ctrl = { }; + struct versal_puf_data puf_data = { }; + struct versal_puf_cfg cfg = { + .global_var_filter = VERSAL_PUF_GLBL_VAR_FLTR_OPTION, + .read_option = VERSAL_PUF_READ_FROM_EFUSE_CACHE, + .puf_operation = VERSAL_PUF_REGEN_ON_DEMAND, + .shutter_value = VERSAL_PUF_SHUTTER_VALUE, + .reg_mode = VERSAL_PUF_SYNDROME_MODE_4K, + }; + + switch (src) { + case VERSAL_AES_EFUSE_USER_KEY_0: + if (versal_efuse_read_sec_ctrl(&ctrl)) + panic(); + + *secure = ctrl.user_key0_wr_lk; + return true; + + case VERSAL_AES_EFUSE_USER_KEY_1: + if (versal_efuse_read_sec_ctrl(&ctrl)) + panic(); + + *secure = ctrl.user_key1_wr_lk; + return true; + + case VERSAL_AES_PUF_KEY: + if (versal_efuse_read_puf_sec_ctrl(&puf_ctrl)) + panic(); + + if (versal_puf_regenerate(&puf_data, &cfg)) + panic(); + + *secure = puf_ctrl.puf_syn_lk; + return true; + + case VERSAL_AES_USER_KEY_0: + *secure = false; + return false; + + default: + EMSG("Trying to use an invalid key for the HUK"); + panic(); + } + + return false; +} + +/* Encrypt using an AES-GCM key selectable with CFG_VERSAL_HUK_KEY */ +static TEE_Result aes_gcm_encrypt(uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len) +{ + struct versal_aes_input_param *input = NULL; + struct versal_aes_init *init = NULL; + struct versal_mbox_mem input_cmd = { }; + struct versal_mbox_mem init_buf = { }; + struct versal_mbox_mem p = { }; + struct versal_mbox_mem q = { }; + uint32_t key_data[8] = DEVEL_KEY; + uint8_t nce_data[12] = NONCE; + uint8_t aad_data[16] = AAD; + size_t nce_len = sizeof(nce_data); + size_t key_len = sizeof(key_data); + size_t aad_len = sizeof(aad_data); + TEE_Result ret = TEE_SUCCESS; + struct versal_ipi_cmd cmd = { }; + bool secure = false; + size_t i = 0; + uint32_t key_id = CFG_VERSAL_HUK_KEY; + + if (key_id > VERSAL_AES_ALL_KEYS) + return TEE_ERROR_BAD_PARAMETERS; + + cmd.data[0] = API_ID(VERSAL_AES_INIT); + if (versal_mbox_notify(&cmd, NULL, NULL)) { + EMSG("AES_INIT error"); + return TEE_ERROR_GENERIC; + } + + if (!versal_persistent_key(key_id, &secure)) { + for (i = 0; i < ARRAY_SIZE(key_data); i++) + key_data[i] = TEE_U32_BSWAP(key_data[i]); + + versal_mbox_alloc(key_len, key_data, &p); + cmd.data[0] = API_ID(VERSAL_AES_WRITE_KEY); + cmd.data[1] = VERSAL_AES_KEY_SIZE_256; + cmd.data[2] = key_id; + reg_pair_from_64(virt_to_phys(p.buf), + &cmd.data[4], &cmd.data[3]); + cmd.ibuf[0].mem = p; + if (versal_mbox_notify(&cmd, NULL, NULL)) { + EMSG("AES_WRITE_KEY error"); + ret = TEE_ERROR_GENERIC; + } + free(p.buf); + memset(&cmd, 0, sizeof(cmd)); + if (ret) + return ret; + } + + /* Trace indication that it is safe to generate a RPMB key */ + IMSG("Using %s HUK", secure ? "Production" : "Development"); + + versal_mbox_alloc(sizeof(*init), NULL, &init_buf); + versal_mbox_alloc(nce_len, nce_data, &p); + init = init_buf.buf; + init->operation = VERSAL_AES_GCM_ENCRYPT; + init->key_len = VERSAL_AES_KEY_SIZE_256; + init->iv_addr = virt_to_phys(p.buf); + init->key_src = key_id; + cmd.data[0] = API_ID(VERSAL_AES_OP_INIT); + reg_pair_from_64(virt_to_phys(init), &cmd.data[2], &cmd.data[1]); + cmd.ibuf[0].mem = init_buf; + cmd.ibuf[1].mem = p; + if (versal_mbox_notify(&cmd, NULL, NULL)) { + EMSG("AES_OP_INIT error"); + ret = TEE_ERROR_GENERIC; + } + free(init); + free(p.buf); + memset(&cmd, 0, sizeof(cmd)); + if (ret) + return ret; + + versal_mbox_alloc(aad_len, aad_data, &p); + cmd.data[0] = API_ID(VERSAL_AES_UPDATE_AAD); + reg_pair_from_64(virt_to_phys(p.buf), &cmd.data[2], &cmd.data[1]); + if (p.len % 16) + cmd.data[3] = p.alloc_len; + else + cmd.data[3] = p.len; + cmd.ibuf[0].mem = p; + if (versal_mbox_notify(&cmd, NULL, NULL)) { + EMSG("AES_UPDATE_AAD error"); + ret = TEE_ERROR_GENERIC; + } + free(p.buf); + memset(&cmd, 0, sizeof(cmd)); + if (ret) + return ret; + + versal_mbox_alloc(sizeof(*input), NULL, &input_cmd); + versal_mbox_alloc(src_len, src, &p); + versal_mbox_alloc(dst_len, NULL, &q); + input = input_cmd.buf; + input->input_addr = virt_to_phys(p.buf); + input->input_len = p.len; + input->is_last = true; + cmd.data[0] = API_ID(VERSAL_AES_ENCRYPT_UPDATE); + reg_pair_from_64(virt_to_phys(input), &cmd.data[2], &cmd.data[1]); + reg_pair_from_64(virt_to_phys(q.buf), &cmd.data[4], &cmd.data[3]); + cmd.ibuf[0].mem = input_cmd; + cmd.ibuf[1].mem = p; + cmd.ibuf[2].mem = q; + if (versal_mbox_notify(&cmd, NULL, NULL)) { + EMSG("AES_UPDATE_PAYLOAD error"); + ret = TEE_ERROR_GENERIC; + } + memcpy(dst, q.buf, dst_len); + free(input); + free(p.buf); + free(q.buf); + memset(&cmd, 0, sizeof(cmd)); + if (ret) + return ret; + + versal_mbox_alloc(16, NULL, &p); + cmd.data[0] = API_ID(VERSAL_AES_ENCRYPT_FINAL); + reg_pair_from_64(virt_to_phys(p.buf), &cmd.data[2], &cmd.data[1]); + if (versal_mbox_notify(&cmd, NULL, NULL)) { + EMSG("AES_ENCRYPT_FINAL error"); + ret = TEE_ERROR_GENERIC; + } + free(p.buf); + memzero_explicit(&cmd, sizeof(cmd)); + + return ret; +} + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + uint32_t dna[EFUSE_DNA_LEN / sizeof(uint32_t)] = { }; + uint8_t enc_data[64] = { }; + uint8_t sha[48] = { }; + TEE_Result ret = TEE_SUCCESS; + + if (huk.ready) + goto out; + + if (versal_efuse_read_dna(dna, sizeof(dna))) + return TEE_ERROR_GENERIC; + + if (versal_sha3_384((uint8_t *)dna, sizeof(dna), sha, sizeof(sha))) { + ret = TEE_ERROR_GENERIC; + goto cleanup; + } + + if (aes_gcm_encrypt(sha, sizeof(sha), enc_data, sizeof(enc_data))) { + ret = TEE_ERROR_GENERIC; + goto cleanup; + } + + if (tee_hash_createdigest(TEE_ALG_SHA256, enc_data, sizeof(enc_data), + huk.key, sizeof(huk.key))) { + ret = TEE_ERROR_GENERIC; + goto cleanup; + } + +cleanup: + memzero_explicit(enc_data, sizeof(enc_data)); + memzero_explicit(dna, sizeof(dna)); + memzero_explicit(sha, sizeof(sha)); + + if (ret) + return ret; + + huk.ready = true; +out: + memcpy(hwkey->data, huk.key, HW_UNIQUE_KEY_LENGTH); + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/versal_mbox.c b/optee_os/core/drivers/versal_mbox.c new file mode 100644 index 0000000..2f27627 --- /dev/null +++ b/optee_os/core/drivers/versal_mbox.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2022 Foundries.io Ltd + * Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include "drivers/versal_mbox.h" + +#define PM_SIP_SVC 0xc2000000 + +/* IPI targets */ +#define IPI_ID_PMC 1 +#define IPI_ID_0 2 +#define IPI_ID_RPU0 3 +#define IPI_ID_RPU1 4 +#define IPI_ID_3 5 +#define IPI_ID_4 6 +#define IPI_ID_5 7 + +/* Buffers */ +#define IPI_BUFFER_BASEADDR 0xFF3F0000 +#define IPI_BUFFER_APU_ID_0_BASE (IPI_BUFFER_BASEADDR + 0x400) +#define IPI_BUFFER_APU_ID_3_BASE (IPI_BUFFER_BASEADDR + 0xA00) +#define IPI_BUFFER_APU_ID_4_BASE (IPI_BUFFER_BASEADDR + 0xC00) +#define IPI_BUFFER_APU_ID_5_BASE (IPI_BUFFER_BASEADDR + 0xE00) +#define IPI_BUFFER_PMC_BASE (IPI_BUFFER_BASEADDR + 0x200) +#define IPI_BUFFER_TARGET_APU_OFFSET 0x80 +#define IPI_BUFFER_TARGET_PMC_OFFSET 0x40 +#define IPI_BUFFER_REQ_OFFSET 0x0 +#define IPI_BUFFER_RESP_OFFSET 0x20 + +#define IPI_BUFFER_LOCAL_OFFSET IPI_BUFFER_TARGET_APU_OFFSET +#define IPI_BUFFER_REMOTE_OFFSET IPI_BUFFER_TARGET_PMC_OFFSET + +#define IPI_BLOCK 1 +#define IPI_NON_BLOCK 0 + +/* Mailbox api */ +enum versal_ipi_api_id { + IPI_MAILBOX_OPEN = 0x1000, + IPI_MAILBOX_RELEASE, + IPI_MAILBOX_STATUS_ENQUIRY, + IPI_MAILBOX_NOTIFY, + IPI_MAILBOX_ACK, + IPI_MAILBOX_ENABLE_IRQ, + IPI_MAILBOX_DISABLE_IRQ +}; + +static struct versal_ipi { + uint32_t lcl; + const uint32_t rmt; + paddr_t buf; + /* Exclusive access to the IPI shared buffer */ + struct mutex lock; + void *rsp; + void *req; +} ipi = { + .buf = IPI_BUFFER_APU_ID_3_BASE, + .rmt = IPI_ID_PMC, + .lcl = IPI_ID_3, +}; + +static const char *const nvm_id[] = { + [0] = "API_FEATURES", + [1] = "BBRAM_WRITE_AES_KEY", + [2] = "BBRAM_ZEROIZE", + [3] = "BBRAM_WRITE_USER_DATA", + [4] = "BBRAM_READ_USER_DATA", + [5] = "BBRAM_LOCK_WRITE_USER_DATA", + [6] = "EFUSE_WRITE", + [7] = "EFUSE_WRITE_PUF", + [8] = "EFUSE_PUF_USER_FUSE_WRITE", + [9] = "EFUSE_READ_IV", + [10] = "EFUSE_READ_REVOCATION_ID", + [11] = "EFUSE_READ_OFFCHIP_REVOCATION_ID", + [12] = "EFUSE_READ_USER_FUSES", + [13] = "EFUSE_READ_MISC_CTRL", + [14] = "EFUSE_READ_SEC_CTRL", + [15] = "EFUSE_READ_SEC_MISC1", + [16] = "EFUSE_READ_BOOT_ENV_CTRL", + [17] = "EFUSE_READ_PUF_SEC_CTRL", + [18] = "EFUSE_READ_PPK_HASH", + [19] = "EFUSE_READ_DEC_EFUSE_ONLY", + [20] = "EFUSE_READ_DNA", + [21] = "EFUSE_READ_PUF_USER_FUSES", + [22] = "EFUSE_READ_PUF", + [23] = "EFUSE_INVALID", +}; + +static const char *const crypto_id[] = { + [0] = "FEATURES", + [1] = "RSA_SIGN_VERIFY", + [2] = "RSA_PUBLIC_ENCRYPT", + [3] = "RSA_PRIVATE_DECRYPT", + [4] = "RSA_KAT", + [32] = "SHA3_UPDATE", + [33] = "SHA3_KAT", + [64] = "ELLIPTIC_GENERATE_PUBLIC_KEY", + [65] = "ELLIPTIC_GENERATE_SIGN", + [66] = "ELLIPTIC_VALIDATE_PUBLIC_KEY", + [67] = "ELLIPTIC_VERIFY_SIGN", + [68] = "ELLIPTIC_KAT", + [96] = "AES_INIT", + [97] = "AES_OP_INIT", + [98] = "AES_UPDATE_AAD", + [99] = "AES_ENCRYPT_UPDATE", + [100] = "AES_ENCRYPT_FINAL", + [101] = "AES_DECRYPT_UPDATE", + [102] = "AES_DECRYPT_FINAL", + [103] = "AES_KEY_ZERO", + [104] = "AES_WRITE_KEY", + [105] = "AES_LOCK_USER_KEY", + [106] = "AES_KEK_DECRYPT", + [107] = "AES_SET_DPA_CM", + [108] = "AES_DECRYPT_KAT", + [109] = "AES_DECRYPT_CM_KAT", + [110] = "MAX", +}; + +static const char *const puf_id[] = { + [0] = "PUF_API_FEATURES", + [1] = "PUF_REGISTRATION", + [2] = "PUF_REGENERATION", + [3] = "PUF_CLEAR_PUF_ID", +}; + +static const char *const module[] = { + [5] = "CRYPTO", + [7] = "FPGA", + [11] = "NVM", + [12] = "PUF", +}; + +static const char *const fpga_id[] = { + [1] = "LOAD", +}; + +static void versal_mbox_call_trace(uint32_t call) +{ + uint32_t mid = call >> 8 & 0xff; + uint32_t api = call & 0xff; + const char *val = NULL; + + switch (mid) { + case 5: + if (api < ARRAY_SIZE(crypto_id)) + val = crypto_id[api]; + + break; + case 7: + if (api < ARRAY_SIZE(fpga_id)) + val = fpga_id[api]; + + break; + case 11: + if (api < ARRAY_SIZE(nvm_id)) + val = nvm_id[api]; + + break; + case 12: + if (api < ARRAY_SIZE(puf_id)) + val = puf_id[api]; + + break; + default: + break; + } + + IMSG("--- mbox: service: %s\t call: %s", module[mid], + val ? val : "Invalid"); +}; + +static TEE_Result mbox_call(enum versal_ipi_api_id id, uint32_t blocking_call) +{ + struct thread_smc_args args = { + .a0 = PM_SIP_SVC | id, + .a1 = reg_pair_to_64(0, ipi.lcl), + .a2 = reg_pair_to_64(0, ipi.rmt), + .a3 = reg_pair_to_64(0, blocking_call), + }; + + thread_smccc(&args); + + /* Give the PLM time to access the console */ + if (IS_ENABLED(CFG_VERSAL_TRACE_PLM)) + mdelay(500); + + if (args.a0) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result versal_mbox_write_req(struct versal_ipi_cmd *cmd) +{ + size_t i = 0; + + for (i = 0; i < VERSAL_MAX_IPI_BUF; i++) { + if (!cmd->ibuf[i].mem.buf) + continue; + + if (!IS_ALIGNED((uintptr_t)cmd->ibuf[i].mem.buf, + CACHELINE_LEN)) { + EMSG("address not aligned: buffer %zu - %p", i, + cmd->ibuf[i].mem.buf); + return TEE_ERROR_GENERIC; + } + + if (!IS_ALIGNED(cmd->ibuf[i].mem.alloc_len, CACHELINE_LEN)) { + EMSG("length not aligned: buffer %zu - %zu", + i, cmd->ibuf[i].mem.alloc_len); + return TEE_ERROR_GENERIC; + } + + cache_operation(TEE_CACHEFLUSH, cmd->ibuf[i].mem.buf, + cmd->ibuf[i].mem.alloc_len); + } + + memcpy(ipi.req, cmd->data, sizeof(cmd->data)); + + /* Cache operation on the IPI buffer is safe */ + cache_operation(TEE_CACHEFLUSH, ipi.req, sizeof(cmd->data)); + + return TEE_SUCCESS; +} + +static TEE_Result versal_mbox_read_rsp(struct versal_ipi_cmd *cmd, + struct versal_ipi_cmd *rsp, + uint32_t *status) +{ + size_t i = 0; + + /* Cache operation on the IPI buffer is safe */ + cache_operation(TEE_CACHEINVALIDATE, ipi.rsp, sizeof(rsp->data)); + + *status = *(uint32_t *)ipi.rsp; + + if (*status) + return TEE_ERROR_GENERIC; + + if (rsp) + memcpy(rsp->data, ipi.rsp, sizeof(rsp->data)); + + for (i = 0; i < VERSAL_MAX_IPI_BUF; i++) { + if (!cmd->ibuf[i].mem.buf) + continue; + + if (!IS_ALIGNED((uintptr_t)cmd->ibuf[i].mem.buf, + CACHELINE_LEN)) { + EMSG("address not aligned: buffer %zu - %p", + i, cmd->ibuf[i].mem.buf); + return TEE_ERROR_GENERIC; + } + + if (!IS_ALIGNED(cmd->ibuf[i].mem.alloc_len, CACHELINE_LEN)) { + EMSG("length not aligned: buffer %zu - %zu", + i, cmd->ibuf[i].mem.alloc_len); + return TEE_ERROR_GENERIC; + } + + cache_operation(TEE_CACHEINVALIDATE, + cmd->ibuf[i].mem.buf, + cmd->ibuf[i].mem.alloc_len); + } + + return TEE_SUCCESS; +} + +TEE_Result versal_mbox_alloc(size_t len, const void *init, + struct versal_mbox_mem *mem) +{ + mem->buf = memalign(CACHELINE_LEN, ROUNDUP(len, CACHELINE_LEN)); + if (!mem->buf) + panic(); + + memset(mem->buf, 0, ROUNDUP(len, CACHELINE_LEN)); + + if (init) + memcpy(mem->buf, init, len); + + mem->alloc_len = ROUNDUP(len, CACHELINE_LEN); + mem->len = len; + + return TEE_SUCCESS; +} + +TEE_Result versal_mbox_notify(struct versal_ipi_cmd *cmd, + struct versal_ipi_cmd *rsp, uint32_t *err) +{ + TEE_Result ret = TEE_SUCCESS; + uint32_t remote_status = 0; + + mutex_lock(&ipi.lock); + + ret = versal_mbox_write_req(cmd); + if (ret) { + EMSG("Can't write the request command"); + goto out; + } + + if (IS_ENABLED(CFG_VERSAL_TRACE_MBOX)) + versal_mbox_call_trace(cmd->data[0]); + + ret = mbox_call(IPI_MAILBOX_NOTIFY, IPI_BLOCK); + if (ret) { + EMSG("IPI error"); + goto out; + } + + ret = versal_mbox_read_rsp(cmd, rsp, &remote_status); + if (ret) + EMSG("Can't read the remote response"); + + if (remote_status) { + if (err) + *err = remote_status; + /* + * Check the remote code (FSBL repository) in xplmi_status.h + * and the relevant service error (ie, xsecure_error.h) for + * detailed information. + */ + DMSG("PLM: plm status = 0x%" PRIx32 ", lib_status = 0x%" PRIx32, + (remote_status & 0xFFFF0000) >> 16, + (remote_status & 0x0000FFFF)); + + ret = TEE_ERROR_GENERIC; + } +out: + mutex_unlock(&ipi.lock); + + return ret; +} + +static TEE_Result versal_mbox_init(void) +{ + switch (CFG_VERSAL_MBOX_IPI_ID) { + case 0: + ipi.buf = IPI_BUFFER_APU_ID_0_BASE; + ipi.lcl = IPI_ID_0; + break; + case 3: + break; + case 4: + ipi.buf = IPI_BUFFER_APU_ID_4_BASE; + ipi.lcl = IPI_ID_4; + break; + case 5: + ipi.buf = IPI_BUFFER_APU_ID_5_BASE; + ipi.lcl = IPI_ID_5; + break; + default: + EMSG("Invalid IPI requested"); + return TEE_ERROR_GENERIC; + } + + ipi.req = core_mmu_add_mapping(MEM_AREA_RAM_SEC, + ipi.buf + IPI_BUFFER_REMOTE_OFFSET + + IPI_BUFFER_REQ_OFFSET, + sizeof(struct versal_ipi_cmd)); + + ipi.rsp = core_mmu_add_mapping(MEM_AREA_RAM_SEC, + ipi.buf + IPI_BUFFER_REMOTE_OFFSET + + IPI_BUFFER_RESP_OFFSET, + sizeof(struct versal_ipi_cmd)); + if (!ipi.req || !ipi.rsp) + panic(); + + mutex_init(&ipi.lock); + + return mbox_call(IPI_MAILBOX_OPEN, IPI_BLOCK); +} +early_init(versal_mbox_init); diff --git a/optee_os/core/drivers/versal_nvm.c b/optee_os/core/drivers/versal_nvm.c new file mode 100644 index 0000000..b17ae63 --- /dev/null +++ b/optee_os/core/drivers/versal_nvm.c @@ -0,0 +1,1005 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2022 Foundries.io Ltd + * Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "drivers/versal_nvm.h" + +#define NVM_WORD_LEN 4 + +/* Protocol API with the remote processor */ +#define NVM_MODULE_SHIFT 8 +#define NVM_MODULE 11 +#define NVM_API_ID(_id) ((NVM_MODULE << NVM_MODULE_SHIFT) | (_id)) + +#define __aligned_efuse __aligned(CACHELINE_LEN) + +/* Internal */ +struct versal_efuse_puf_fuse_addr { + uint64_t data_addr; + uint32_t start_row; + uint32_t num_rows; + uint8_t env_monitor_dis; + uint8_t prgm_puf_fuse; + uint8_t pad[46]; +}; + +/* + * Max size of the buffer needed for the remote processor to DMA efuse _data_ + * to/from + */ +#define EFUSE_MAX_LEN (EFUSE_MAX_USER_FUSES * sizeof(uint32_t)) + +enum versal_nvm_api_id { + API_FEATURES = 0, + BBRAM_WRITE_AES_KEY = 1, + BBRAM_ZEROIZE = 2, + BBRAM_WRITE_USER_DATA = 3, + BBRAM_READ_USER_DATA = 4, + BBRAM_LOCK_WRITE_USER_DATA = 5, + EFUSE_WRITE = 6, + EFUSE_WRITE_PUF = 7, + EFUSE_PUF_USER_FUSE_WRITE = 8, + EFUSE_READ_IV = 9, + EFUSE_READ_REVOCATION_ID = 10, + EFUSE_READ_OFFCHIP_REVOCATION_ID = 11, + EFUSE_READ_USER_FUSES = 12, + EFUSE_READ_MISC_CTRL = 13, + EFUSE_READ_SEC_CTRL = 14, + EFUSE_READ_SEC_MISC1 = 15, + EFUSE_READ_BOOT_ENV_CTRL = 16, + EFUSE_READ_PUF_SEC_CTRL = 17, + EFUSE_READ_PPK_HASH = 18, + EFUSE_READ_DEC_EFUSE_ONLY = 19, + EFUSE_READ_DNA = 20, + EFUSE_READ_PUF_USER_FUSES = 21, + EFUSE_READ_PUF = 22, + EFUSE_INVALID = 23, +}; + +/* uint64_t are memory addresses */ +struct versal_efuse_data { + uint64_t env_mon_dis_flag; + uint64_t aes_key_addr; + uint64_t ppk_hash_addr; + uint64_t dec_only_addr; + uint64_t sec_ctrl_addr; + uint64_t misc_ctrl_addr; + uint64_t revoke_id_addr; + uint64_t iv_addr; + uint64_t user_fuse_addr; + uint64_t glitch_cfg_addr; + uint64_t boot_env_ctrl_addr; + uint64_t misc1_ctrl_addr; + uint64_t offchip_id_addr; + uint8_t pad[24]; +}; + +/* Helper read and write requests (not part of the protocol) */ +struct versal_nvm_buf { + size_t len; + void *buf; +}; + +struct versal_nvm_read_req { + enum versal_nvm_api_id efuse_id; + enum versal_nvm_revocation_id revocation_id; + enum versal_nvm_offchip_id offchip_id; + enum versal_nvm_ppk_type ppk_type; + enum versal_nvm_iv_type iv_type; + struct versal_nvm_buf ibuf[VERSAL_MAX_IPI_BUF]; +}; + +struct versal_bbram_data { + size_t aes_key_len; + uint32_t user_data; +}; + +struct versal_nvm_write_req { + struct versal_efuse_data data; + struct versal_bbram_data bbram; + struct versal_nvm_buf ibuf[VERSAL_MAX_IPI_BUF]; + enum versal_nvm_api_id efuse_id; +}; + +static TEE_Result +prepare_cmd(struct versal_ipi_cmd *cmd, enum versal_nvm_api_id efuse, + struct versal_nvm_buf *ibufs, uint32_t *arg) +{ + uint32_t a = 0; + uint32_t b = 0; + size_t i = 0; + + cmd->data[i++] = NVM_API_ID(efuse); + if (arg) + cmd->data[i++] = *arg; + + if (!ibufs[0].buf) + return TEE_SUCCESS; + + reg_pair_from_64(virt_to_phys(ibufs[0].buf), &b, &a); + + cmd->data[i++] = a; + cmd->data[i++] = b; + + for (i = 0; i < VERSAL_MAX_IPI_BUF; i++) { + cmd->ibuf[i].mem.alloc_len = ibufs[i].len; + cmd->ibuf[i].mem.buf = ibufs[i].buf; + } + + return TEE_SUCCESS; +} + +static TEE_Result efuse_req(enum versal_nvm_api_id efuse, + struct versal_nvm_buf *ibufs, uint32_t *arg) +{ + struct versal_ipi_cmd cmd = { }; + TEE_Result ret = TEE_SUCCESS; + + ret = prepare_cmd(&cmd, efuse, ibufs, arg); + if (ret) + return ret; + + ret = versal_mbox_notify(&cmd, NULL, NULL); + if (ret) + EMSG("Mailbox error"); + + return ret; +} + +static TEE_Result versal_alloc_read_buffer(struct versal_nvm_read_req *req) +{ + assert(req); + req->ibuf[0].len = 1024; + req->ibuf[0].buf = alloc_cache_aligned(req->ibuf[0].len); + if (!req->ibuf[0].buf) + return TEE_ERROR_OUT_OF_MEMORY; + + return TEE_SUCCESS; +} + +static void versal_free_read_buffer(struct versal_nvm_read_req *req) +{ + assert(req); + free(req->ibuf[0].buf); +} + +static void *versal_get_read_buffer(struct versal_nvm_read_req *req) +{ + assert(req); + return req->ibuf[0].buf; +} + +static TEE_Result versal_nvm_read(struct versal_nvm_read_req *req) +{ + uint32_t *arg = NULL; + uint32_t val = 0; + + if (!req) + return TEE_ERROR_GENERIC; + + switch (req->efuse_id) { + case EFUSE_READ_DNA: + case EFUSE_READ_DEC_EFUSE_ONLY: + case EFUSE_READ_PUF_SEC_CTRL: + case EFUSE_READ_BOOT_ENV_CTRL: + case EFUSE_READ_SEC_CTRL: + case EFUSE_READ_MISC_CTRL: + case EFUSE_READ_SEC_MISC1: + case EFUSE_READ_USER_FUSES: + case EFUSE_READ_PUF_USER_FUSES: + case EFUSE_READ_PUF: + break; + case EFUSE_READ_OFFCHIP_REVOCATION_ID: + val = req->offchip_id; + arg = &val; + break; + case EFUSE_READ_REVOCATION_ID: + val = req->revocation_id; + arg = &val; + break; + case EFUSE_READ_IV: + val = req->iv_type; + arg = &val; + break; + case EFUSE_READ_PPK_HASH: + val = req->ppk_type; + arg = &val; + break; + case BBRAM_READ_USER_DATA: + break; + default: + return TEE_ERROR_GENERIC; + } + + return efuse_req(req->efuse_id, req->ibuf, arg); +} + +static TEE_Result versal_nvm_write(struct versal_nvm_write_req *req) +{ + uint32_t *arg = NULL; + uint32_t val = 0; + + switch (req->efuse_id) { + case BBRAM_WRITE_AES_KEY: + val = req->bbram.aes_key_len; + arg = &val; + break; + case BBRAM_WRITE_USER_DATA: + val = req->bbram.user_data; + arg = &val; + break; + case EFUSE_PUF_USER_FUSE_WRITE: + case EFUSE_WRITE_PUF: + case EFUSE_WRITE: + break; + default: + return TEE_ERROR_GENERIC; + } + + return efuse_req(req->efuse_id, req->ibuf, arg); +} + +TEE_Result versal_efuse_read_user_data(uint32_t *buf, size_t len, + uint32_t first, size_t num) +{ + struct versal_efuse_user_data cfg __aligned_efuse = { + .start = first, + .num = num, + }; + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_USER_FUSES, + }; + void *rsp = NULL; + + if (first + num > EFUSE_MAX_USER_FUSES || len < num * sizeof(uint32_t)) + return TEE_ERROR_BAD_PARAMETERS; + + rsp = alloc_cache_aligned(1024); + if (!rsp) + return TEE_ERROR_OUT_OF_MEMORY; + + req.ibuf[0].buf = &cfg; + req.ibuf[0].len = sizeof(cfg); + req.ibuf[1].buf = rsp; + req.ibuf[1].len = 1024; + + cfg.addr = virt_to_phys((void *)rsp); + + if (versal_nvm_read(&req)) { + free(rsp); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, rsp, num * sizeof(uint32_t)); + free(rsp); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_dna(uint32_t *buf, size_t len) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_DNA, + }; + + if (len < EFUSE_DNA_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), EFUSE_DNA_LEN); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_iv(uint32_t *buf, size_t len, + enum versal_nvm_iv_type type) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_IV, + .iv_type = type, + }; + + if (len < EFUSE_IV_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), EFUSE_IV_LEN); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_ppk(uint32_t *buf, size_t len, + enum versal_nvm_ppk_type type) +{ + struct versal_nvm_read_req req = { + req.efuse_id = EFUSE_READ_PPK_HASH, + .ppk_type = type, + }; + + if (len < EFUSE_PPK_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) + return TEE_ERROR_GENERIC; + + memcpy(buf, versal_get_read_buffer(&req), EFUSE_PPK_LEN); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_write_user_data(uint32_t *buf, size_t len, + uint32_t first, size_t num) +{ + uint32_t lbuf[EFUSE_MAX_USER_FUSES] __aligned_efuse = { 0 }; + struct versal_efuse_user_data cfg __aligned_efuse = { + .addr = (uintptr_t)lbuf, + .start = first, + .num = num, + }; + struct versal_nvm_write_req __aligned_efuse req = { + .data.user_fuse_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + size_t i = 0; + + if (first + num > EFUSE_MAX_USER_FUSES || len < num * sizeof(uint32_t)) + return TEE_ERROR_BAD_PARAMETERS; + + req.data.user_fuse_addr = virt_to_phys((void *)req.data.user_fuse_addr); + cfg.addr = virt_to_phys(lbuf); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + req.ibuf[2].buf = lbuf; + req.ibuf[2].len = sizeof(lbuf); + + for (i = 0; i < cfg.num; i++) + lbuf[i] = buf[i]; + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_aes_keys(struct versal_efuse_aes_keys *keys) +{ + struct versal_efuse_aes_keys cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.aes_key_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, keys, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_ppk_hash(struct versal_efuse_ppk_hash *hash) +{ + struct versal_efuse_ppk_hash cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.ppk_hash_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, hash, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_iv(struct versal_efuse_ivs *p) +{ + struct versal_efuse_ivs cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.iv_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_dec_only(struct versal_efuse_dec_only *p) +{ + struct versal_efuse_dec_only cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.dec_only_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_sec(struct versal_efuse_sec_ctrl_bits *p) +{ + struct versal_efuse_sec_ctrl_bits cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.sec_ctrl_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_misc(struct versal_efuse_misc_ctrl_bits *p) +{ + struct versal_efuse_misc_ctrl_bits cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.misc_ctrl_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_glitch_cfg(struct versal_efuse_glitch_cfg_bits *p) +{ + struct versal_efuse_glitch_cfg_bits cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.glitch_cfg_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_boot_env(struct versal_efuse_boot_env_ctrl_bits + *p) +{ + struct versal_efuse_boot_env_ctrl_bits cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.boot_env_ctrl_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_sec_misc1(struct versal_efuse_sec_misc1_bits *p) +{ + struct versal_efuse_sec_misc1_bits cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.misc1_ctrl_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_offchip_ids(struct versal_efuse_offchip_ids *p) +{ + struct versal_efuse_offchip_ids cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.offchip_id_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + memcpy(&cfg, p, sizeof(cfg)); + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_revoke_ppk(enum versal_nvm_ppk_type type) +{ + struct versal_efuse_misc_ctrl_bits cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.misc_ctrl_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + + req.data.misc_ctrl_addr = virt_to_phys((void *)req.data.misc_ctrl_addr); + if (type == EFUSE_PPK0) + cfg.ppk0_invalid = 1; + else if (type == EFUSE_PPK1) + cfg.ppk1_invalid = 1; + else if (type == EFUSE_PPK2) + cfg.ppk2_invalid = 1; + else + return TEE_ERROR_BAD_PARAMETERS; + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_write_revoke_id(uint32_t id) +{ + struct versal_efuse_revoke_ids cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .data.revoke_id_addr = virt_to_phys(&cfg), + .data.env_mon_dis_flag = 1, + .efuse_id = EFUSE_WRITE, + }; + uint32_t row = 0; + uint32_t bit = 0; + + row = id >> (NVM_WORD_LEN + 1); + bit = id & (NVM_WORD_LEN - 1); + + cfg.revoke_id[row] = BIT(bit); + cfg.prgm_revoke_id = 1; + + req.ibuf[0].buf = &req.data; + req.ibuf[0].len = sizeof(req.data); + req.ibuf[1].buf = &cfg; + req.ibuf[1].len = sizeof(cfg); + + return versal_nvm_write(&req); +} + +TEE_Result versal_efuse_read_revoke_id(uint32_t *buf, size_t len, + enum versal_nvm_revocation_id id) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_REVOCATION_ID, + .revocation_id = id, + }; + + if (len < EFUSE_REVOCATION_ID_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), EFUSE_REVOCATION_ID_LEN); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_misc_ctrl(struct versal_efuse_misc_ctrl_bits *buf) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_MISC_CTRL, + }; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), sizeof(*buf)); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_sec_ctrl(struct versal_efuse_sec_ctrl_bits *buf) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_SEC_CTRL, + }; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), sizeof(*buf)); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_sec_misc1(struct versal_efuse_sec_misc1_bits *buf) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_SEC_MISC1, + }; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), sizeof(*buf)); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result +versal_efuse_read_boot_env_ctrl(struct versal_efuse_boot_env_ctrl_bits *buf) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_BOOT_ENV_CTRL, + }; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), sizeof(*buf)); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_offchip_revoke_id(uint32_t *buf, size_t len, + enum versal_nvm_offchip_id id) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_OFFCHIP_REVOCATION_ID, + .offchip_id = id, + }; + + if (len < EFUSE_OFFCHIP_REVOCATION_ID_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), EFUSE_REVOCATION_ID_LEN); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_dec_only(uint32_t *buf, size_t len) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_DEC_EFUSE_ONLY, + }; + + if (len < EFUSE_DEC_ONLY_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), EFUSE_DEC_ONLY_LEN); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result +versal_efuse_read_puf_sec_ctrl(struct versal_efuse_puf_sec_ctrl_bits *buf) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_PUF_SEC_CTRL, + }; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), sizeof(*buf)); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_read_puf(struct versal_efuse_puf_header *buf) +{ + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_PUF, + }; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + memcpy(versal_get_read_buffer(&req), buf, sizeof(*buf)); + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(buf, versal_get_read_buffer(&req), sizeof(*buf)); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +/* + * This functionality requires building the PLM with XNVM_ACCESS_PUF_USER_DATA + * Calls will fail otherwise. + * When available, efuse_read_puf becomes unavailable. + */ +TEE_Result +versal_efuse_read_puf_as_user_fuse(struct versal_efuse_puf_user_fuse *p) +{ + uint32_t fuses[PUF_EFUSES_WORDS]__aligned_efuse = { 0 }; + struct versal_efuse_puf_fuse_addr lbuf __aligned_efuse = { + .env_monitor_dis = p->env_monitor_dis, + .prgm_puf_fuse = p->prgm_puf_fuse, + .start_row = p->start_row, + .num_rows = p->num_rows, + .data_addr = virt_to_phys(fuses), + }; + struct versal_nvm_read_req req = { + .efuse_id = EFUSE_READ_PUF_USER_FUSES, + }; + + req.ibuf[0].buf = &lbuf; + req.ibuf[0].len = sizeof(lbuf); + req.ibuf[1].buf = fuses; + req.ibuf[1].len = sizeof(fuses); + + if (versal_nvm_read(&req)) + return TEE_ERROR_GENERIC; + + memcpy(p->data_addr, fuses, sizeof(fuses)); + + return TEE_SUCCESS; +} + +/* + * This functionality requires building the PLM with XNVM_ACCESS_PUF_USER_DATA. + * Calls will fail otherwise. + * When available, efuse_write_puf becomes unavailable. + */ +TEE_Result +versal_efuse_write_puf_as_user_fuse(struct versal_efuse_puf_user_fuse *p) +{ + uint32_t fuses[PUF_EFUSES_WORDS]__aligned_efuse = { 0 }; + struct versal_efuse_puf_fuse_addr lbuf __aligned_efuse = { + .env_monitor_dis = p->env_monitor_dis, + .prgm_puf_fuse = p->prgm_puf_fuse, + .start_row = p->start_row, + .num_rows = p->num_rows, + .data_addr = virt_to_phys(fuses), + }; + struct versal_nvm_write_req req = { + .efuse_id = EFUSE_PUF_USER_FUSE_WRITE, + }; + + memcpy(fuses, p->data_addr, sizeof(p->data_addr)); + + req.ibuf[0].buf = &lbuf; + req.ibuf[0].len = sizeof(lbuf); + req.ibuf[1].buf = fuses; + req.ibuf[1].len = sizeof(fuses); + + if (versal_nvm_write(&req)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +TEE_Result versal_efuse_write_puf(struct versal_efuse_puf_header *buf) +{ + struct versal_efuse_puf_header cfg __aligned_efuse = { }; + struct versal_nvm_write_req req __aligned_efuse = { + .efuse_id = EFUSE_WRITE_PUF, + }; + + memcpy(&cfg, buf, sizeof(*buf)); + + req.ibuf[0].buf = &cfg; + req.ibuf[0].len = sizeof(cfg); + + if (versal_nvm_write(&req)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +TEE_Result versal_bbram_write_aes_key(uint8_t *key, size_t len) +{ + struct versal_nvm_write_req req __aligned_efuse = { + .efuse_id = BBRAM_WRITE_AES_KEY, + .bbram.aes_key_len = len, + }; + void *buf = NULL; + + if (len != 32) + return TEE_ERROR_BAD_PARAMETERS; + + buf = alloc_cache_aligned(1024); + if (!buf) + return TEE_ERROR_OUT_OF_MEMORY; + + memcpy(buf, key, len); + + req.ibuf[0].buf = buf; + req.ibuf[0].len = 1024; + + if (versal_nvm_write(&req)) { + free(buf); + return TEE_ERROR_GENERIC; + } + free(buf); + + return TEE_SUCCESS; +} + +TEE_Result versal_bbram_zeroize(void) +{ + struct versal_nvm_write_req req __aligned_efuse = { + .efuse_id = BBRAM_ZEROIZE, + }; + + if (versal_nvm_write(&req)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +TEE_Result versal_bbram_write_user_data(uint32_t data) +{ + struct versal_nvm_write_req req __aligned_efuse = { + .efuse_id = BBRAM_WRITE_USER_DATA, + .bbram.user_data = data, + }; + + if (versal_nvm_write(&req)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +TEE_Result versal_bbram_read_user_data(uint32_t *data) +{ + struct versal_nvm_read_req req = { + .efuse_id = BBRAM_READ_USER_DATA, + }; + + if (versal_alloc_read_buffer(&req)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (versal_nvm_read(&req)) { + versal_free_read_buffer(&req); + return TEE_ERROR_GENERIC; + } + + memcpy(data, versal_get_read_buffer(&req), sizeof(*data)); + versal_free_read_buffer(&req); + + return TEE_SUCCESS; +} + +TEE_Result versal_bbram_lock_write_user_data(void) +{ + struct versal_nvm_write_req req __aligned_efuse = { + .efuse_id = BBRAM_LOCK_WRITE_USER_DATA, + }; + + if (versal_nvm_write(&req)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/versal_pm.c b/optee_os/core/drivers/versal_pm.c new file mode 100644 index 0000000..84b1cbe --- /dev/null +++ b/optee_os/core/drivers/versal_pm.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2022 Foundries.io Ltd + * Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* VERSAL_SIP_UID: 2ab9e4ec-93b9-11e7-a019dfe0dbad0ae0 */ +#define VERSAL_SIP_UID_0 U(0xece4b92a) +#define VERSAL_SIP_UID_1 U(0xe711b993) +#define VERSAL_SIP_UID_2 U(0xe0df19a0) +#define VERSAL_SIP_UID_3 U(0xe00aaddb) +#define VERSAL_SIP_MAJOR 0 +#define VERSAL_SIP_MINOR 1 + +#define VERSAL_SIP_SVC_VERSION 0x8200ff03 +#define VERSAL_SIP_SVC_UID 0x8200ff01 +#define VERSAL_SIP_SVC 0xc2000000 + +#define PAYLOAD_ARG_CNT 8 + +/* MBOX IPI */ +#define PM_MODULE_SHIFT 8 +#define PM_MODULE 2 +#define PM_API_ID(x) ((PM_MODULE << PM_MODULE_SHIFT) | (x)) +#define VERSAL_PM_MAJOR 0 +#define VERSAL_PM_MINOR 1 + +/* PM API ids */ +#define PM_GET_API_VERSION 1 +#define PM_GET_DEVICE_STATUS 3 +#define PM_GET_OP_CHARACTERISTIC 4 +#define PM_REGISTER_NOTIFIER 5 +#define PM_REQ_SUSPEND 6 +#define PM_SELF_SUSPEND 7 +#define PM_FORCE_POWERDOWN 8 +#define PM_ABORT_SUSPEND 9 +#define PM_REQ_WAKEUP 10 +#define PM_SET_WAKEUP_SOURCE 11 +#define PM_SYSTEM_SHUTDOWN 12 +#define PM_REQUEST_DEVICE 13 +#define PM_RELEASE_DEVICE 14 +#define PM_SET_REQUIREMENT 15 +#define PM_SET_MAX_LATENCY 16 +#define PM_RESET_ASSERT 17 +#define PM_RESET_GET_STATUS 18 +#define PM_INIT_FINALIZE 21 +#define PM_GET_CHIPID 24 +#define PM_PINCTRL_REQUEST 28 +#define PM_PINCTRL_RELEASE 29 +#define PM_PINCTRL_GET_FUNCTION 30 +#define PM_PINCTRL_SET_FUNCTION 31 +#define PM_PINCTRL_CONFIG_PARAM_GET 32 +#define PM_PINCTRL_CONFIG_PARAM_SET 33 +#define PM_IOCTL 34 +#define PM_QUERY_DATA 35 +#define PM_CLOCK_ENABLE 36 +#define PM_CLOCK_DISABLE 37 +#define PM_CLOCK_GETSTATE 38 +#define PM_CLOCK_SETDIVIDER 39 +#define PM_CLOCK_GETDIVIDER 40 +#define PM_CLOCK_SETRATE 41 +#define PM_CLOCK_GETRATE 42 +#define PM_CLOCK_SETPARENT 43 +#define PM_CLOCK_GETPARENT 44 +#define PM_PLL_SET_PARAMETER 48 +#define PM_PLL_GET_PARAMETER 49 +#define PM_PLL_SET_MODE 50 +#define PM_PLL_GET_MODE 51 +#define PM_FEATURE_CHECK 63 + +/* Loader API id */ +#define PM_LOAD_PDI 0x701 + +/* PDI sources */ +#define PDI_SRC_JTAG 0x0 +#define PDI_SRC_QSPI24 0x1 +#define PDI_SRC_QSPI32 0x2 +#define PDI_SRC_SD0 0x3 +#define PDI_SRC_EMMC0 0x4 +#define PDI_SRC_SD1 0x5 +#define PDI_SRC_EMMC1 0x6 +#define PDI_SRC_USB 0x7 +#define PDI_SRC_OSPI 0x8 +#define PDI_SRC_SBI 0x9 +#define PDI_SRC_SMAP 0xA +#define PDI_SRC_PCIE 0xB +#define PDI_SRC_SD1_LS 0xE +#define PDI_SRC_DDR 0xF + +struct versal_sip_payload { + uint32_t data[PAYLOAD_ARG_CNT]; +}; + +static uint32_t versal_sip_call(uint32_t smc_fid, uint32_t arg0, uint32_t arg1, + uint32_t arg2, uint32_t arg3, + struct versal_sip_payload *payload) +{ + struct thread_smc_args args = { + .a0 = smc_fid, + .a1 = reg_pair_to_64(arg1, arg0), + .a2 = reg_pair_to_64(arg3, arg2), + }; + + thread_smccc(&args); + + if (payload) { + reg_pair_from_64(args.a0, &payload->data[1], &payload->data[0]); + reg_pair_from_64(args.a1, &payload->data[3], &payload->data[2]); + reg_pair_from_64(args.a2, &payload->data[5], &payload->data[4]); + reg_pair_from_64(args.a3, &payload->data[7], &payload->data[6]); + } + + /* allow the PLM to output its debug information */ + if (IS_ENABLED(CFG_VERSAL_TRACE_PLM)) + mdelay(500); + + return args.a0; +} + +/* SIP call to program the FPGA has been obsoleted, use the PLM */ +TEE_Result versal_write_fpga(paddr_t pa) +{ + struct versal_ipi_cmd cmd = { }; + + cmd.data[0] = PM_LOAD_PDI; + cmd.data[1] = PDI_SRC_DDR; + reg_pair_from_64(pa, &cmd.data[2], &cmd.data[3]); + + if (versal_mbox_notify(&cmd, NULL, NULL)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +TEE_Result versal_soc_version(uint8_t *version) +{ + struct versal_sip_payload p = { }; + const uint32_t version_shift = 12; + + if (!version) + return TEE_ERROR_BAD_PARAMETERS; + + if (versal_sip_call(VERSAL_SIP_SVC | PM_GET_CHIPID, 0, 0, 0, 0, &p)) + return TEE_ERROR_GENERIC; + + *version = p.data[2] >> version_shift; + + return TEE_SUCCESS; +} + +static bool uuid_is_versal_pm(void) +{ + struct versal_sip_payload p = { }; + + versal_sip_call(VERSAL_SIP_SVC_UID, 0, 0, 0, 0, &p); + + if (p.data[0] == VERSAL_SIP_UID_0 && p.data[2] == VERSAL_SIP_UID_1 && + p.data[4] == VERSAL_SIP_UID_2 && p.data[6] == VERSAL_SIP_UID_3) + return true; + + return false; +} + +static TEE_Result versal_check_pm_abi(void) +{ + struct versal_sip_payload p = { }; + struct versal_ipi_cmd cmd = { }; + struct versal_ipi_cmd rsp = { }; + unsigned int major = 0; + unsigned int minor = 0; + + if (!uuid_is_versal_pm()) { + EMSG("Invalid SiP Service"); + return TEE_ERROR_GENERIC; + } + + if (versal_sip_call(VERSAL_SIP_SVC_VERSION, 0, 0, 0, 0, &p)) + return TEE_ERROR_GENERIC; + + major = p.data[0]; + minor = p.data[2]; + if (major != VERSAL_SIP_MAJOR || minor < VERSAL_SIP_MINOR) { + EMSG("Invalid SiP version: Major %d, Minor %d", major, minor); + return TEE_ERROR_GENERIC; + } + + cmd.data[0] = PM_API_ID(PM_GET_API_VERSION); + if (versal_mbox_notify(&cmd, &rsp, NULL)) + return TEE_ERROR_GENERIC; + + major = rsp.data[1] & 0xFFFF; + minor = rsp.data[1] >> 16; + if (major != VERSAL_PM_MAJOR || minor < VERSAL_PM_MINOR) { + EMSG("Invalid PM version: Major %d, Minor %d", major, minor); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +early_init_late(versal_check_pm_abi); diff --git a/optee_os/core/drivers/versal_puf.c b/optee_os/core/drivers/versal_puf.c new file mode 100644 index 0000000..d65cbe1 --- /dev/null +++ b/optee_os/core/drivers/versal_puf.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2022 Foundries.io Ltd + * Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Protocol API with the remote processor */ +#define VERSAL_PUF_MODULE_SHIFT 8 +#define VERSAL_PUF_MODULE 12 +#define PUF_API_ID(_id) ((VERSAL_PUF_MODULE << VERSAL_PUF_MODULE_SHIFT) | (_id)) + +enum versal_puf_error { + /* Registration */ + ERROR_INVALID_PARAM = 0x02, + ERROR_INVALID_SYNDROME_MODE = 0x03, + ERROR_SYNDROME_WORD_WAIT_TIMEOUT = 0x04, + ERROR_PUF_DONE_WAIT_TIMEOUT = 0x07, + ERROR_REGISTRATION_INVALID = 0x08, + SHUTTER_GVF_MISMATCH = 0x09, + ERROR_SYN_DATA_ERROR = 0x0A, + IRO_FREQ_WRITE_MISMATCH = 0x0B, + /* Regeneration */ + ERROR_CHASH_NOT_PROGRAMMED = 0x10, + ERROR_PUF_STATUS_DONE_TIMEOUT = 0x11, + ERROR_INVALID_REGENERATION_TYPE = 0x12, + ERROR_INVALID_PUF_OPERATION = 0x13, + ERROR_REGENERATION_INVALID = 0x14, + ERROR_REGEN_PUF_HD_INVALID = 0x15, + ERROR_INVALID_READ_HD_INPUT = 0x16, + ERROR_PUF_DONE_KEY_NT_RDY = 0x17, + ERROR_PUF_DONE_ID_NT_RDY = 0x18, + ERROR_PUF_ID_ZERO_TIMEOUT = 0x19, +}; + +#define VERSAL_PUF_ERROR(m) { .error = (m), .name = TO_STR(m) } + +static const char *versal_puf_error(uint8_t err) +{ + struct { + enum versal_puf_error error; + const char *name; + } elist[] = { + /* Registration */ + VERSAL_PUF_ERROR(ERROR_INVALID_PARAM), + VERSAL_PUF_ERROR(ERROR_INVALID_SYNDROME_MODE), + VERSAL_PUF_ERROR(ERROR_SYNDROME_WORD_WAIT_TIMEOUT), + VERSAL_PUF_ERROR(ERROR_PUF_DONE_WAIT_TIMEOUT), + VERSAL_PUF_ERROR(ERROR_REGISTRATION_INVALID), + VERSAL_PUF_ERROR(SHUTTER_GVF_MISMATCH), + VERSAL_PUF_ERROR(ERROR_SYN_DATA_ERROR), + VERSAL_PUF_ERROR(IRO_FREQ_WRITE_MISMATCH), + /* Regeneration */ + VERSAL_PUF_ERROR(ERROR_CHASH_NOT_PROGRAMMED), + VERSAL_PUF_ERROR(ERROR_PUF_STATUS_DONE_TIMEOUT), + VERSAL_PUF_ERROR(ERROR_INVALID_REGENERATION_TYPE), + VERSAL_PUF_ERROR(ERROR_INVALID_PUF_OPERATION), + VERSAL_PUF_ERROR(ERROR_REGENERATION_INVALID), + VERSAL_PUF_ERROR(ERROR_REGEN_PUF_HD_INVALID), + VERSAL_PUF_ERROR(ERROR_INVALID_READ_HD_INPUT), + VERSAL_PUF_ERROR(ERROR_PUF_DONE_KEY_NT_RDY), + VERSAL_PUF_ERROR(ERROR_PUF_DONE_ID_NT_RDY), + VERSAL_PUF_ERROR(ERROR_PUF_ID_ZERO_TIMEOUT), + }; + size_t error = 0; + size_t index = 0; + + if (err <= ERROR_PUF_ID_ZERO_TIMEOUT && err >= ERROR_INVALID_PARAM) { + index = err - ERROR_INVALID_PARAM; + + /* Spectre gadget protection: array index is external event */ + error = confine_array_index(index, ARRAY_SIZE(elist)); + if (elist[error].name) + return elist[error].name; + + return "Invalid"; + } + + return "Unknown"; +} + +/* + * Register the Physical Unclonable Function (prior operating with it) + * + * This must happen during the device provisioning phase and can be done from + * the Secure World via this interface or from an earlier firmware. + */ +TEE_Result versal_puf_register(struct versal_puf_data *buf, + struct versal_puf_cfg *cfg) +{ + struct versal_puf_data_req req __aligned_puf = { }; + struct versal_mbox_mem request = { + .alloc_len = sizeof(req), + .len = sizeof(req), + .buf = &req, + }; + struct versal_mbox_mem efuse_syn_data_addr = { }; + struct versal_mbox_mem syndrome_data_addr = { }; + struct versal_mbox_mem puf_id_addr = { }; + struct versal_mbox_mem hash_addr = { }; + struct versal_mbox_mem aux_addr = { }; + struct versal_ipi_cmd arg = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + + versal_mbox_alloc(sizeof(buf->puf_id), buf->puf_id, &puf_id_addr); + versal_mbox_alloc(sizeof(buf->chash), &buf->chash, &hash_addr); + versal_mbox_alloc(sizeof(buf->aux), &buf->aux, &aux_addr); + versal_mbox_alloc(sizeof(buf->efuse_syn_data), buf->efuse_syn_data, + &efuse_syn_data_addr); + versal_mbox_alloc(sizeof(buf->syndrome_data), buf->syndrome_data, + &syndrome_data_addr); + + arg.ibuf[0].mem = request; + arg.ibuf[1].mem = syndrome_data_addr; + arg.ibuf[2].mem = hash_addr; + arg.ibuf[3].mem = aux_addr; + arg.ibuf[4].mem = puf_id_addr; + arg.ibuf[5].mem = efuse_syn_data_addr; + + req.efuse_syn_data_addr = virt_to_phys(efuse_syn_data_addr.buf); + req.syndrome_data_addr = virt_to_phys(syndrome_data_addr.buf); + req.puf_id_addr = virt_to_phys(puf_id_addr.buf); + req.hash_addr = virt_to_phys(hash_addr.buf); + req.aux_addr = virt_to_phys(aux_addr.buf); + + req.global_var_filter = cfg->global_var_filter; + req.shutter_value = cfg->shutter_value; + req.puf_operation = cfg->puf_operation; + req.read_option = cfg->read_option; + req.reg_mode = cfg->reg_mode; + + arg.data[0] = PUF_API_ID(VERSAL_PUF_REGISTER); + reg_pair_from_64(virt_to_phys(arg.ibuf[0].mem.buf), + &arg.data[2], &arg.data[1]); + + if (versal_mbox_notify(&arg, NULL, &err)) { + EMSG("Versal, failed to register the PUF [%s]", + versal_puf_error(err)); + + ret = TEE_ERROR_GENERIC; + } + + /* Return the generated data */ + memcpy(buf->puf_id, puf_id_addr.buf, sizeof(buf->puf_id)); + memcpy(&buf->chash, hash_addr.buf, sizeof(buf->chash)); + memcpy(&buf->aux, aux_addr.buf, sizeof(buf->aux)); + memcpy(buf->efuse_syn_data, efuse_syn_data_addr.buf, + sizeof(buf->efuse_syn_data)); + memcpy(buf->syndrome_data, syndrome_data_addr.buf, + sizeof(buf->syndrome_data)); + + free(syndrome_data_addr.buf); + free(hash_addr.buf); + free(aux_addr.buf); + free(puf_id_addr.buf); + free(efuse_syn_data_addr.buf); + + return ret; +} + +/* + * Re-seed the PUF circuitry so it can re-generate the Key Encryption Key. + * + * Depending on the configuration options it might use eFused data instead of + * the helper data provided via the interface. + */ +TEE_Result versal_puf_regenerate(struct versal_puf_data *buf, + struct versal_puf_cfg *cfg) +{ + struct versal_puf_data_req req __aligned_puf = { }; + struct versal_mbox_mem request = { + .alloc_len = sizeof(req), + .len = sizeof(req), + .buf = &req, + }; + struct versal_mbox_mem efuse_syn_data_addr = { }; + struct versal_mbox_mem syndrome_data_addr = { }; + struct versal_mbox_mem puf_id_addr = { }; + struct versal_mbox_mem hash_addr = { }; + struct versal_mbox_mem aux_addr = { }; + struct versal_ipi_cmd arg = { }; + TEE_Result ret = TEE_SUCCESS; + uint32_t err = 0; + + versal_mbox_alloc(sizeof(buf->puf_id), buf->puf_id, &puf_id_addr); + versal_mbox_alloc(sizeof(buf->chash), &buf->chash, &hash_addr); + versal_mbox_alloc(sizeof(buf->aux), &buf->aux, &aux_addr); + versal_mbox_alloc(sizeof(buf->efuse_syn_data), buf->efuse_syn_data, + &efuse_syn_data_addr); + versal_mbox_alloc(sizeof(buf->syndrome_data), buf->syndrome_data, + &syndrome_data_addr); + + arg.ibuf[0].mem = request; + arg.ibuf[1].mem = syndrome_data_addr; + arg.ibuf[2].mem = hash_addr; + arg.ibuf[3].mem = aux_addr; + arg.ibuf[4].mem = puf_id_addr; + arg.ibuf[5].mem = efuse_syn_data_addr; + + req.efuse_syn_data_addr = virt_to_phys(efuse_syn_data_addr.buf); + req.syndrome_addr = virt_to_phys(syndrome_data_addr.buf); + req.puf_id_addr = virt_to_phys(puf_id_addr.buf); + req.hash_addr = virt_to_phys(hash_addr.buf); + req.aux_addr = virt_to_phys(aux_addr.buf); + + req.global_var_filter = cfg->global_var_filter; + req.shutter_value = cfg->shutter_value; + req.puf_operation = cfg->puf_operation; + req.read_option = cfg->read_option; + req.reg_mode = cfg->reg_mode; + + arg.data[0] = PUF_API_ID(VERSAL_PUF_REGENERATE); + reg_pair_from_64(virt_to_phys(arg.ibuf[0].mem.buf), + &arg.data[2], &arg.data[1]); + + if (versal_mbox_notify(&arg, NULL, &err)) { + EMSG("Versal, failed to regenerate the PUF [%s]", + versal_puf_error(err)); + + ret = TEE_ERROR_GENERIC; + } + + /* Return the updated PUF_ID */ + memcpy(buf->puf_id, puf_id_addr.buf, sizeof(buf->puf_id)); + + free(syndrome_data_addr.buf); + free(hash_addr.buf); + free(aux_addr.buf); + free(puf_id_addr.buf); + free(efuse_syn_data_addr.buf); + + return ret; +} + +/* + * Clear/Hide the PUF Unique ID + * + * The fully accessible (non-secret) Unique ID is generated from the PUF + */ +TEE_Result versal_puf_clear_id(void) +{ + struct versal_ipi_cmd arg = { }; + + arg.data[0] = PUF_API_ID(VERSAL_PUF_CLEAR_ID); + + if (versal_mbox_notify(&arg, NULL, NULL)) { + EMSG("Versal, failed to clear the PUF_ID"); + + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +/* Check that the API id is available to the client */ +TEE_Result versal_puf_check_api(enum versal_puf_api id) +{ + struct versal_ipi_cmd arg = { }; + + arg.data[0] = PUF_API_ID(VERSAL_PUF_API_FEATURES); + arg.data[1] = id; + + if (versal_mbox_notify(&arg, NULL, NULL)) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/versal_sha3_384.c b/optee_os/core/drivers/versal_sha3_384.c new file mode 100644 index 0000000..3a46db0 --- /dev/null +++ b/optee_os/core/drivers/versal_sha3_384.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2022 + * Author: Jorge Ramirez + */ + +#include +#include +#include +#include +#include + +#define VERSAL_SHA3_384_FIRST_PACKET BIT(30) +#define VERSAL_SHA3_384_NEXT_PACKET BIT(31) + +static struct mutex lock = MUTEX_INITIALIZER; +static bool engine_ready; + +static TEE_Result input_plaintext(const uint8_t *src, size_t src_len) +{ + uint32_t first = VERSAL_SHA3_384_FIRST_PACKET; + struct versal_cmd_args arg = { .dlen = 1, }; + struct versal_mbox_mem p = { }; + TEE_Result ret = TEE_SUCCESS; + size_t len = 0; + size_t i = 0; + + while (src_len && !ret) { + len = MIN(src_len, SMALL_PAGE_SIZE); + src_len -= len; + versal_mbox_alloc(len, src + i * SMALL_PAGE_SIZE, &p); + + arg.data[0] = first | VERSAL_SHA3_384_NEXT_PACKET | len; + arg.ibuf[0].mem = p; + ret = versal_crypto_request(VERSAL_SHA3_UPDATE, &arg, NULL); + if (ret) + EMSG("VERSAL_SHA3_UPDATE [%ld, len = %zu]", i, len); + + free(p.buf); + first = 0; + i++; + } + + return ret; +} + +static TEE_Result get_ciphertext(uint8_t *dst, size_t dst_len) +{ + struct versal_cmd_args arg = { }; + struct versal_mbox_mem p = { }; + TEE_Result ret = TEE_SUCCESS; + + versal_mbox_alloc(TEE_SHA384_HASH_SIZE, NULL, &p); + + arg.ibuf[0].mem = p; + ret = versal_crypto_request(VERSAL_SHA3_UPDATE, &arg, NULL); + if (!ret) + memcpy(dst, p.buf, MIN(dst_len, (size_t)TEE_SHA384_HASH_SIZE)); + else + EMSG("VERSAL_SHA3_UPDATE final"); + + free(p.buf); + + return ret; +} + +TEE_Result versal_sha3_384(const uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len) +{ + TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; + + if (!src_len || !dst_len || !src || !dst) + return ret; + + if (!engine_ready) + return TEE_ERROR_BAD_STATE; + + mutex_lock(&lock); + + ret = input_plaintext(src, src_len); + if (!ret) + ret = get_ciphertext(dst, dst_len); + + mutex_unlock(&lock); + + return ret; +} + +static TEE_Result versal_sha3_384_init(void) +{ + struct versal_cmd_args arg = { }; + TEE_Result ret = TEE_SUCCESS; + + ret = versal_crypto_request(VERSAL_SHA3_KAT, &arg, NULL); + if (!ret) + engine_ready = true; + + return ret; +} + +/* Be available for the HUK */ +service_init(versal_sha3_384_init); diff --git a/optee_os/core/drivers/versal_trng.c b/optee_os/core/drivers/versal_trng.c new file mode 100644 index 0000000..792c46a --- /dev/null +++ b/optee_os/core/drivers/versal_trng.c @@ -0,0 +1,1149 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (C) 2022 Xilinx, Inc. All rights reserved. + * Copyright (C) 2022 Foundries Ltd. + * + * Driver port from Xilinx's FSBL implementation, jorge@foundries.io + * + * The Xilinx True Random Number Generator(TRNG) module in Versal - PMC TRNG + * consists of an entropy source, a deterministic random bit generator (DRBG) + * and health test logic, which tests the randomness of the generated data. + * The entropy source for the unit is an array of Ring Oscillators. + * + * The Versal PMC TRNG is envisaged to operate in three basic modes: DRNG, PTRNG + * and HRNG . Each of these can be operated with or without Derivative Function + * (DF), resulting in a total of 6 different modes of operation. + * + * NIST SP-800-90A practically requires the true random generators based on + * CTR_DRBG to include a derivation function (DF). This is expected to be + * implemented inside the Silicon (TRNG IP). However, the version of the IP used + * in Versal PMC does not have this capability. Hence, a software + * implementation of the DF is done in this driver. + * + * DRNG mode: Deterministic Random Number Generator mode. + * In this mode, the DRBG portion of the TRNG is used. User provides + * the (external) seed. + * PTRNG mode: Physical True Random Number Generator mode (aka Entropy mode). + * In this mode digitized Entropy source is output as random number. + * HRNG mode: Hybrid Random Number Generator mode. + * This is combination of above two modes in which the Entropy source + * is used to provide the seed, which is fed to the DRBG, which in + * turn generates the random number. + * + * DRNG mode with DF: It may not be common usecase to use the DF with DRNG as + * the general expectation would be that the seed would have sufficient entropy. + * However, the below guideline from section 10.2.1 of NIST SP-800-90A implies + * that need for DF for DRNG mode too: "..the DRBG mechanism is specified to + * allow an implementation tradeoff with respect to the use of this derivation + * function. The use of the derivation function is optional if either an + * approved RBG or an entropy source provides full entropy output when entropy + * input is requested by the DRBG mechanism. Otherwise, the derivation function + * shall be used". Sufficient large entropy data from user is fed to DF to + * generate the seed which will be loaded into the external seed registers. + * From here, it is similar to regular DRNG mode. + * + * PTRNG mode with DF: This mode is similar to PTRNG mode, however, the entropy + * data from the core output registers are accumulated and fed to the DF + * (instead of directly consuming it). The output of the DF would be final + * random data. In this mode, the output of DF is not seed but the random data. + * + * HRNG mode with DF: This mode is the combination of the above two modes. + * The entropy data is fed to the DF to produce seed. This seed is loaded to the + * external seed registers which provide seed to the DRBG. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TRNG_BASE 0xF1230000 +#define TRNG_SIZE 0x10000 + +#define TRNG_STATUS 0x04 +#define TRNG_STATUS_QCNT_SHIFT 9 +#define TRNG_STATUS_QCNT_MASK (BIT(9) | BIT(10) | BIT(11)) +#define TRNG_STATUS_CERTF_MASK BIT(3) +#define TRNG_STATUS_DTF_MASK BIT(1) +#define TRNG_STATUS_DONE_MASK BIT(0) +#define TRNG_CTRL 0x08 +#define TRNG_CTRL_EUMODE_MASK BIT(8) +#define TRNG_CTRL_PRNGMODE_MASK BIT(7) +#define TRNG_CTRL_PRNGSTART_MASK BIT(5) +#define TRNG_CTRL_PRNGXS_MASK BIT(3) +#define TRNG_CTRL_TRSSEN_MASK BIT(2) +#define TRNG_CTRL_PRNGSRST_MASK BIT(0) +#define TRNG_EXT_SEED_0 0x40 +/* + * Below registers are not directly referenced in driver but are accessed + * with offset from TRNG_EXT_SEED_0 + * Register: TRNG_EXT_SEED_1 0x00000044 + * Register: TRNG_EXT_SEED_2 0x00000048 + * Register: TRNG_EXT_SEED_3 0x0000004C + * Register: TRNG_EXT_SEED_4 0x00000050 + * Register: TRNG_EXT_SEED_5 0x00000054 + * Register: TRNG_EXT_SEED_6 0x00000058 + * Register: TRNG_EXT_SEED_7 0x0000005C + * Register: TRNG_EXT_SEED_8 0x00000060 + * Register: TRNG_EXT_SEED_9 0x00000064 + * Register: TRNG_EXT_SEED_10 0x00000068 + * Register: TRNG_EXT_SEED_11 0x0000006C + */ +#define TRNG_PER_STRING_0 0x80 +/* + * Below registers are not directly referenced in driver but are accessed + * with offset from TRNG_PER_STRING_0 + * Register: TRNG_PER_STRING_1 0x00000084 + * Register: TRNG_PER_STRING_2 0x00000088 + * Register: TRNG_PER_STRING_3 0x0000008C + * Register: TRNG_PER_STRING_4 0x00000090 + * Register: TRNG_PER_STRING_5 0x00000094 + * Register: TRNG_PER_STRING_6 0x00000098 + * Register: TRNG_PER_STRING_7 0x0000009C + * Register: TRNG_PER_STRING_8 0x000000A0 + * Register: TRNG_PER_STRING_9 0x000000A4 + * Register: TRNG_PER_STRING_10 0x000000A8 + * Register: TRNG_PER_STRING_11 0x000000AC + */ +#define TRNG_CORE_OUTPUT 0xC0 +#define TRNG_RESET 0xD0 +#define TRNG_RESET_VAL_MASK BIT(0) +#define TRNG_OSC_EN 0xD4 +#define TRNG_OSC_EN_VAL_MASK BIT(0) + +/* TRNG configuration */ +#define TRNG_BURST_SIZE 16 +#define TRNG_BURST_SIZE_BITS 128 +#define TRNG_NUM_INIT_REGS 12 +#define TRNG_REG_SIZE 32 +#define TRNG_BYTES_PER_REG 4 +#define TRNG_MAX_QCNT 4 +#define TRNG_RESEED_TIMEOUT 15000 +#define TRNG_GENERATE_TIMEOUT 8000 +#define TRNG_MIN_DFLENMULT 2 +#define TRNG_MAX_DFLENMULT 9 +#define PRNGMODE_RESEED 0 +#define PRNGMODE_GEN TRNG_CTRL_PRNGMODE_MASK +#define RESET_DELAY 10 +#define TRNG_SEC_STRENGTH_LEN 32 +#define TRNG_PERS_STR_REGS 12 +#define TRNG_PERS_STR_LEN 48 +#define TRNG_SEED_REGS 12 +#define TRNG_SEED_LEN 48 +#define TRNG_GEN_LEN 32 +#define RAND_BUF_LEN 4 +#define BYTES_PER_BLOCK 16 +#define ALL_A_PATTERN_32 0xAAAAAAAA +#define ALL_5_PATTERN_32 0x55555555 + +/* Derivative function helper macros */ +#define DF_SEED 0 +#define DF_RAND 1 +#define DF_IP_IV_LEN 4 +#define DF_PAD_DATA_LEN 8 +#define MAX_PRE_DF_LEN 160 +#define MAX_PRE_DF_LEN_WORDS 40 +#define DF_PERS_STR_LEN TRNG_PERS_STR_LEN +#define DF_PAD_VAL 0x80 +#define DF_KEY_LEN 32 +#define BLK_SIZE 16 +#define MAX_ROUNDS 14 + +enum trng_status { + TRNG_UNINITIALIZED = 0, + TRNG_HEALTHY, + TRNG_ERROR, + TRNG_CATASTROPHIC +}; + +enum trng_mode { + TRNG_HRNG = 0, + TRNG_DRNG, + TRNG_PTRNG +}; + +struct trng_cfg { + paddr_t base; + vaddr_t addr; + size_t len; +}; + +struct trng_usr_cfg { + enum trng_mode mode; + uint64_t seed_life; /* number of TRNG requests per seed */ + bool predict_en; /* enable prediction resistance */ + bool pstr_en; /* enable personalization string */ + uint32_t pstr[TRNG_PERS_STR_REGS]; + bool iseed_en; /* enable an initial seed */ + uint32_t init_seed[MAX_PRE_DF_LEN_WORDS]; + uint32_t df_disable; /* disable the derivative function */ + uint32_t dfmul; /* derivative function multiplier */ +}; + +struct trng_stats { + uint64_t bytes; + uint64_t bytes_reseed; + uint64_t elapsed_seed_life; +}; + +/* block cipher derivative function algorithm */ +struct trng_dfin { + uint32_t ivc[DF_IP_IV_LEN]; + uint32_t val1; + uint32_t val2; + uint8_t entropy[MAX_PRE_DF_LEN]; /* input entropy */ + uint8_t pstr[DF_PERS_STR_LEN]; /* personalization string */ + uint8_t pad_data[DF_PAD_DATA_LEN]; /* pad to multiples of 16 bytes*/ +}; + +struct versal_trng { + struct trng_cfg cfg; + struct trng_usr_cfg usr_cfg; + struct trng_stats stats; + enum trng_status status; + uint32_t buf[RAND_BUF_LEN]; /* buffer of random bits */ + size_t len; + struct trng_dfin dfin; + uint8_t dfout[TRNG_SEED_LEN]; /* output of the DF operation */ +}; + +/* Derivative function variables */ +static unsigned char sbx1[256]; +static unsigned char sbx2[256]; +static unsigned char sbx3[256]; +static unsigned char schedule[BLK_SIZE * (MAX_ROUNDS + 1)]; +static unsigned int rounds; + +static void rota4(uint8_t *a, uint8_t *b, uint8_t *c, uint8_t *d) +{ + uint8_t t = *a; + + *a = sbx1[*b]; + *b = sbx1[*c]; + *c = sbx1[*d]; + *d = sbx1[t]; +} + +static void rota2(uint8_t *a, uint8_t *b) +{ + uint8_t t = *a; + + *a = sbx1[*b]; + *b = sbx1[t]; +} + +static void sbox4(uint8_t *a, uint8_t *b, uint8_t *c, uint8_t *d) +{ + *a = sbx1[*a]; + *b = sbx1[*b]; + *c = sbx1[*c]; + *d = sbx1[*d]; +} + +static void xorb(uint8_t *res, const uint8_t *in) +{ + size_t i = 0; + + for (i = 0; i < BLK_SIZE; ++i) + res[i] ^= in[i]; +} + +static void set_key(uint8_t *res, const uint8_t *src, unsigned int roundval) +{ + memcpy(res, src, BLK_SIZE); + xorb(res, schedule + roundval * BLK_SIZE); +} + +static void mix_column_sbox(uint8_t *dst, const uint8_t *f) +{ + size_t i = 0; + size_t a = 0; + size_t b = 0; + size_t c = 0; + size_t d = 0; + + for (i = 0; i < 4; i++) { + a = 4 * i; + b = (0x5 + a) % 16; + c = (0xa + a) % 16; + d = (0xf + a) % 16; + dst[0 + a] = sbx2[f[a]] ^ sbx3[f[b]] ^ sbx1[f[c]] ^ sbx1[f[d]]; + dst[1 + a] = sbx1[f[a]] ^ sbx2[f[b]] ^ sbx3[f[c]] ^ sbx1[f[d]]; + dst[2 + a] = sbx1[f[a]] ^ sbx1[f[b]] ^ sbx2[f[c]] ^ sbx3[f[d]]; + dst[3 + a] = sbx3[f[a]] ^ sbx1[f[b]] ^ sbx1[f[c]] ^ sbx2[f[d]]; + } +} + +static void shift_row_sbox(uint8_t *f) +{ + sbox4(&f[0], &f[4], &f[8], &f[12]); + rota4(&f[1], &f[5], &f[9], &f[13]); + rota2(&f[2], &f[10]); + rota2(&f[6], &f[14]); + rota4(&f[15], &f[11], &f[7], &f[3]); +} + +static void encrypt(uint8_t *in, uint8_t *out) +{ + uint8_t fa[BLK_SIZE] = { 0 }; + uint8_t fb[BLK_SIZE] = { 0 }; + size_t roundval = 0; + + set_key(fa, in, 0); + for (roundval = 1; roundval < rounds; ++roundval) { + mix_column_sbox(fb, fa); + set_key(fa, fb, roundval); + } + + shift_row_sbox(fa); + set_key(out, fa, roundval); +} + +static void checksum(unsigned char *in, uint8_t *iv, int max_blk) +{ + while (max_blk > 0) { + xorb(iv, in); + encrypt(iv, iv); + in += BLK_SIZE; + max_blk -= 1; + } +} + +static void setup_key(const unsigned char *k, size_t klen) +{ + unsigned char rcon = 1; + size_t sch_size = 240; + size_t i = 0; + + rounds = MAX_ROUNDS; + memcpy(schedule, k, klen); + for (i = klen; i < sch_size; i += 4) { + unsigned char t0 = 0; + unsigned char t1 = 0; + unsigned char t2 = 0; + unsigned char t3 = 0; + int ik = 0; + + t0 = schedule[i - 4]; + t1 = schedule[i - 3]; + t2 = schedule[i - 2]; + t3 = schedule[i - 1]; + if (i % klen == 0) { + rota4(&t0, &t1, &t2, &t3); + t0 ^= rcon; + rcon = (rcon << 1) ^ (((rcon >> 7) & 1) * 0x1B); + } else if (i % klen == 16) { + sbox4(&t0, &t1, &t2, &t3); + } + ik = i - klen; + schedule[i + 0] = schedule[ik + 0] ^ t0; + schedule[i + 1] = schedule[ik + 1] ^ t1; + schedule[i + 2] = schedule[ik + 2] ^ t2; + schedule[i + 3] = schedule[ik + 3] ^ t3; + } +} + +static void trng_df_init(void) +{ + const uint8_t sb[] = { + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, + 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, + 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, + 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, + 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, + 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, + 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, + 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, + 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 0xd0, 0xef, 0xaa, 0xfb, + 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, + 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, + 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, + 0x64, 0x5d, 0x19, 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, + 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, + 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, + 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, + 0xae, 0x08, 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, + 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, + 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, + 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, + 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, + 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, + }; + size_t i = 0; + + memcpy(sbx1, sb, sizeof(sb)); + for (i = 0; i < sizeof(sb); i++) { + sbx2[i] = (sb[i] << 1) ^ (((sb[i] >> 7) & 1) * 0x1B); + sbx3[i] = sbx2[i] ^ sb[i]; + } +} + +/* + * This function implements the Derivative Function by distilling the entropy + * available in its input into a smaller number of bits on the output. + * - per NIST SP80090A. + * + * The Block Cipher algorithm follows sections 10.3.2 and 10.3.3 of the + * NIST.SP.800-90Ar1 document + */ +static void trng_df_algorithm(struct versal_trng *trng, uint8_t *dfout, + uint32_t flag, const uint8_t *pstr) +{ + static bool df_init; + const uint8_t df_key[DF_KEY_LEN] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + }; + size_t dfin_len = sizeof(struct trng_dfin) + trng->len; + uint8_t *inp_blk = NULL; + uint8_t *out_blk = NULL; + uintptr_t reminder = 0; + size_t xfer_len = 0; + uint32_t index = 0; + uintptr_t src = 0; + uintptr_t dst = 0; + size_t offset = 0; + + if (!df_init) { + trng_df_init(); + df_init = true; + } + + if (flag == DF_SEED) + trng->dfin.val2 = TEE_U32_TO_BIG_ENDIAN(TRNG_PERS_STR_LEN); + else + trng->dfin.val2 = TEE_U32_TO_BIG_ENDIAN(TRNG_GEN_LEN); + + trng->dfin.pad_data[0] = DF_PAD_VAL; + + if (!pstr) { + if (trng->len > (MAX_PRE_DF_LEN + TRNG_PERS_STR_LEN)) + panic(); + + dfin_len = dfin_len - TRNG_PERS_STR_LEN - MAX_PRE_DF_LEN; + trng->dfin.val1 = TEE_U32_TO_BIG_ENDIAN(trng->len); + + xfer_len = DF_PAD_DATA_LEN; + src = (uintptr_t)trng->dfin.pad_data; + offset = MAX_PRE_DF_LEN + TRNG_PERS_STR_LEN - trng->len; + } else { + if (trng->len > MAX_PRE_DF_LEN) + panic(); + + memcpy(trng->dfin.pstr, pstr, TRNG_PERS_STR_LEN); + dfin_len = dfin_len - MAX_PRE_DF_LEN; + trng->dfin.val1 = TEE_U32_TO_BIG_ENDIAN(trng->len + + TRNG_PERS_STR_LEN); + xfer_len = DF_PAD_DATA_LEN + TRNG_PERS_STR_LEN; + src = (uintptr_t)trng->dfin.pstr; + offset = MAX_PRE_DF_LEN - trng->len; + } + + /* Move back into the dfin structure */ + dst = src - offset; + reminder = (uintptr_t)&trng->dfin + sizeof(trng->dfin) - offset; + if (offset) { + if (xfer_len > offset) + panic("Overlapping data"); + + memcpy((void *)dst, (void *)src, xfer_len); + memset((void *)reminder, 0, offset); + } + + /* DF algorithm - step 1 */ + setup_key(df_key, DF_KEY_LEN); + for (index = 0; index < TRNG_SEED_LEN; index += BLK_SIZE) { + memset((void *)(trng->dfout + index), 0, BLK_SIZE); + trng->dfin.ivc[0] = TEE_U32_TO_BIG_ENDIAN(index / BLK_SIZE); + checksum((unsigned char *)&trng->dfin, + trng->dfout + index, dfin_len / BLK_SIZE); + } + + /* DF algorithm - step 2 */ + setup_key(trng->dfout, DF_KEY_LEN); + for (index = 0; index < TRNG_SEED_LEN; index += BLK_SIZE) { + if (!index) + inp_blk = &dfout[TRNG_SEC_STRENGTH_LEN]; + else + inp_blk = &dfout[index - BLK_SIZE]; + + out_blk = &dfout[index]; + encrypt(inp_blk, out_blk); + } +} + +static uint32_t trng_read32(vaddr_t addr, size_t off) +{ + return io_read32(addr + off); +} + +static void trng_write32(vaddr_t addr, size_t off, uint32_t val) +{ + io_write32(addr + off, val); +} + +static void trng_clrset32(vaddr_t addr, size_t off, uint32_t mask, uint32_t val) +{ + io_clrsetbits32(addr + off, mask, mask & val); +} + +static void trng_write32_range(const struct versal_trng *trng, uint32_t start, + uint32_t num_regs, const uint8_t *buf) +{ + size_t off = 0; + uint32_t val = 0; + size_t cnt = 0; + size_t i = 0; + + for (i = 0; i < num_regs; ++i) { + if (!buf) { + off = start + i * TRNG_BYTES_PER_REG; + trng_write32(trng->cfg.addr, off, 0); + continue; + } + + val = 0; + for (cnt = 0; cnt < TRNG_BYTES_PER_REG; ++cnt) + val = (val << 8) | buf[i * TRNG_BYTES_PER_REG + cnt]; + + off = start + (TRNG_NUM_INIT_REGS - 1 - i) * TRNG_BYTES_PER_REG; + trng_write32(trng->cfg.addr, off, val); + } +} + +static TEE_Result trng_wait_for_event(vaddr_t addr, size_t off, uint32_t mask, + uint32_t event, uint32_t time_out) +{ + uint64_t tref = timeout_init_us(time_out); + + do { + if (timeout_elapsed(tref)) + break; + } while ((io_read32(addr + off) & mask) != event); + + /* Normal world might have suspended the OP-TEE thread, check again */ + if ((io_read32(addr + off) & mask) != event) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static void trng_soft_reset(const struct versal_trng *trng) +{ + trng_clrset32(trng->cfg.addr, TRNG_CTRL, TRNG_CTRL_PRNGSRST_MASK, + TRNG_CTRL_PRNGSRST_MASK); + udelay(RESET_DELAY); + trng_clrset32(trng->cfg.addr, TRNG_CTRL, TRNG_CTRL_PRNGSRST_MASK, 0); +} + +static void trng_reset(const struct versal_trng *trng) +{ + trng_write32(trng->cfg.addr, TRNG_RESET, TRNG_RESET_VAL_MASK); + udelay(RESET_DELAY); + trng_write32(trng->cfg.addr, TRNG_RESET, 0); + trng_soft_reset(trng); +} + +static void trng_hold_reset(const struct versal_trng *trng) +{ + trng_clrset32(trng->cfg.addr, TRNG_CTRL, + TRNG_CTRL_PRNGSRST_MASK, TRNG_CTRL_PRNGSRST_MASK); + trng_write32(trng->cfg.addr, TRNG_RESET, TRNG_RESET_VAL_MASK); + udelay(RESET_DELAY); +} + +static TEE_Result trng_check_seed(uint8_t *entropy, uint32_t len) +{ + uint32_t *p = (void *)entropy; + size_t i = 0; + + for (i = 0; i < len / sizeof(*p); i++) { + if (p[i] == ALL_A_PATTERN_32) + return TEE_ERROR_GENERIC; + + if (p[i] == ALL_5_PATTERN_32) + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +static TEE_Result trng_collect_random(struct versal_trng *trng, uint8_t *dst, + size_t len) +{ + const size_t bursts = len / TRNG_BURST_SIZE; + const size_t words = TRNG_BURST_SIZE_BITS / TRNG_REG_SIZE; + uint32_t *p = (void *)dst; + size_t bcnt = 0; + size_t wcnt = 0; + size_t index = 0; + uint32_t val = 0; + bool match = false; + + trng_clrset32(trng->cfg.addr, TRNG_CTRL, + TRNG_CTRL_PRNGSTART_MASK, TRNG_CTRL_PRNGSTART_MASK); + + /* + * Loop as many times based on len requested. In each burst 128 bits + * are generated, which is reflected in QCNT value of 4 by hardware. + */ + for (bcnt = 0; bcnt < bursts; bcnt++) { + if (trng_wait_for_event(trng->cfg.addr, + TRNG_STATUS, TRNG_STATUS_QCNT_MASK, + TRNG_MAX_QCNT << TRNG_STATUS_QCNT_SHIFT, + TRNG_GENERATE_TIMEOUT)) { + EMSG("Timeout waiting for randomness"); + return TEE_ERROR_GENERIC; + } + + /* + * DTF flag set during generate indicates catastrophic + * condition, which needs to be checked for every time unless we + * are in PTRNG mode + */ + if (trng->usr_cfg.mode != TRNG_PTRNG) { + val = trng_read32(trng->cfg.addr, TRNG_STATUS); + if (val & TRNG_STATUS_DTF_MASK) { + EMSG("Catastrophic DFT error"); + trng->status = TRNG_CATASTROPHIC; + + return TEE_ERROR_GENERIC; + } + } + /* + * Read the core output register 4 times to consume the random + * data generated for every burst. + */ + match = true; + for (wcnt = 0; wcnt < words; wcnt++) { + val = trng_read32(trng->cfg.addr, TRNG_CORE_OUTPUT); + + if (bcnt > 0 && trng->buf[wcnt] != val) + match = false; + + trng->buf[wcnt] = val; + + if (dst) { + p[index] = TEE_U32_TO_BIG_ENDIAN(val); + index++; + } + } + + if (bursts > 1 && bcnt > 0 && match) { + EMSG("Catastrophic software error"); + trng->status = TRNG_CATASTROPHIC; + return TEE_ERROR_GENERIC; + } + } + + return TEE_SUCCESS; +} + +static TEE_Result trng_reseed_internal_nodf(struct versal_trng *trng, + uint8_t *eseed, uint8_t *str) +{ + uint8_t entropy[TRNG_SEED_LEN] = { 0 }; + uint8_t *seed = NULL; + + switch (trng->usr_cfg.mode) { + case TRNG_HRNG: + trng_write32(trng->cfg.addr, TRNG_OSC_EN, TRNG_OSC_EN_VAL_MASK); + trng_soft_reset(trng); + trng_write32(trng->cfg.addr, TRNG_CTRL, + TRNG_CTRL_EUMODE_MASK | TRNG_CTRL_TRSSEN_MASK); + + if (trng_collect_random(trng, entropy, TRNG_SEED_LEN)) + return TEE_ERROR_GENERIC; + + if (trng_check_seed(entropy, TRNG_SEED_LEN)) + return TEE_ERROR_GENERIC; + + seed = entropy; + break; + case TRNG_DRNG: + seed = eseed; + break; + default: + seed = NULL; + break; + } + + trng_write32_range(trng, TRNG_EXT_SEED_0, TRNG_SEED_REGS, seed); + if (str) + trng_write32_range(trng, TRNG_PER_STRING_0, TRNG_PERS_STR_REGS, + str); + + return TEE_SUCCESS; +} + +static TEE_Result trng_reseed_internal_df(struct versal_trng *trng, + uint8_t *eseed, uint8_t *str) +{ + memset(&trng->dfin, 0, sizeof(trng->dfin)); + + switch (trng->usr_cfg.mode) { + case TRNG_HRNG: + trng_write32(trng->cfg.addr, TRNG_OSC_EN, TRNG_OSC_EN_VAL_MASK); + trng_soft_reset(trng); + trng_write32(trng->cfg.addr, TRNG_CTRL, + TRNG_CTRL_EUMODE_MASK | TRNG_CTRL_TRSSEN_MASK); + + if (trng_collect_random(trng, trng->dfin.entropy, trng->len)) + return TEE_ERROR_GENERIC; + + if (trng_check_seed(trng->dfin.entropy, trng->len)) + return TEE_ERROR_GENERIC; + break; + case TRNG_DRNG: + memcpy(trng->dfin.entropy, eseed, trng->len); + break; + default: + break; + } + + trng_df_algorithm(trng, trng->dfout, DF_SEED, str); + trng_write32_range(trng, TRNG_EXT_SEED_0, TRNG_SEED_REGS, trng->dfout); + + return TEE_SUCCESS; +} + +static TEE_Result trng_reseed_internal(struct versal_trng *trng, + uint8_t *eseed, uint8_t *str, + uint32_t mul) +{ + uint32_t val = 0; + + trng->stats.bytes_reseed = 0; + trng->stats.elapsed_seed_life = 0; + + if (trng->usr_cfg.df_disable) + trng->len = TRNG_SEED_LEN; + else + trng->len = (mul + 1) * BYTES_PER_BLOCK; + + if (trng->usr_cfg.df_disable) { + if (trng_reseed_internal_nodf(trng, eseed, str)) + goto error; + } else { + if (trng_reseed_internal_df(trng, eseed, str)) + goto error; + } + + trng_write32(trng->cfg.addr, TRNG_CTRL, + PRNGMODE_RESEED | TRNG_CTRL_PRNGXS_MASK); + + /* Start the reseed operation */ + trng_clrset32(trng->cfg.addr, TRNG_CTRL, TRNG_CTRL_PRNGSTART_MASK, + TRNG_CTRL_PRNGSTART_MASK); + + if (trng_wait_for_event(trng->cfg.addr, TRNG_STATUS, + TRNG_STATUS_DONE_MASK, TRNG_STATUS_DONE_MASK, + TRNG_RESEED_TIMEOUT)) + goto error; + + /* Check SP800 - 90B (entropy health test error) */ + val = trng_read32(trng->cfg.addr, TRNG_STATUS) & TRNG_STATUS_CERTF_MASK; + if (val == TRNG_STATUS_CERTF_MASK) + goto error; + + trng_clrset32(trng->cfg.addr, TRNG_CTRL, TRNG_CTRL_PRNGSTART_MASK, 0); + return TEE_SUCCESS; +error: + trng->status = TRNG_ERROR; + return TEE_ERROR_GENERIC; +} + +static TEE_Result trng_instantiate(struct versal_trng *trng, + const struct trng_usr_cfg *usr_cfg) +{ + uint8_t *seed = NULL; + uint8_t *pers = NULL; + + if (!trng) + return TEE_ERROR_GENERIC; + + if (!usr_cfg) + goto error; + + if (trng->status != TRNG_UNINITIALIZED) + goto error; + + if (usr_cfg->mode != TRNG_HRNG && usr_cfg->mode != TRNG_DRNG && + usr_cfg->mode != TRNG_PTRNG) + goto error; + + if (usr_cfg->mode != TRNG_PTRNG && !usr_cfg->seed_life) + goto error; + + if (!usr_cfg->iseed_en && usr_cfg->mode == TRNG_DRNG) + goto error; + + if (usr_cfg->iseed_en && usr_cfg->mode == TRNG_HRNG) + goto error; + + if (!usr_cfg->df_disable && + (usr_cfg->dfmul < TRNG_MIN_DFLENMULT || + usr_cfg->dfmul > TRNG_MAX_DFLENMULT)) + goto error; + + if (usr_cfg->df_disable && usr_cfg->dfmul) + goto error; + + if (usr_cfg->mode == TRNG_PTRNG && + (usr_cfg->iseed_en || usr_cfg->pstr_en || + usr_cfg->predict_en || usr_cfg->seed_life)) + goto error; + + memcpy(&trng->usr_cfg, usr_cfg, sizeof(struct trng_usr_cfg)); + trng_reset(trng); + + if (trng->usr_cfg.iseed_en) + seed = (void *)trng->usr_cfg.init_seed; + + if (trng->usr_cfg.pstr_en) + pers = (void *)trng->usr_cfg.pstr; + + if (trng->usr_cfg.mode != TRNG_PTRNG) { + if (trng_reseed_internal(trng, seed, pers, trng->usr_cfg.dfmul)) + goto error; + } + + trng->status = TRNG_HEALTHY; + return TEE_SUCCESS; +error: + trng->status = TRNG_ERROR; + return TEE_ERROR_GENERIC; +} + +static TEE_Result trng_reseed(struct versal_trng *trng, uint8_t *eseed, + uint32_t mul) +{ + if (!trng) + return TEE_ERROR_GENERIC; + + if (trng->status != TRNG_HEALTHY) + goto error; + + if (trng->usr_cfg.mode != TRNG_DRNG && trng->usr_cfg.mode != TRNG_HRNG) + goto error; + + if (trng->usr_cfg.mode == TRNG_DRNG && !eseed) + goto error; + + if (trng->usr_cfg.mode != TRNG_DRNG && eseed) + goto error; + + if (!trng->usr_cfg.df_disable) { + if (mul < TRNG_MIN_DFLENMULT || mul > TRNG_MAX_DFLENMULT) + goto error; + } + + if (trng->usr_cfg.df_disable && mul) + goto error; + + if (eseed && !memcmp(eseed, trng->usr_cfg.init_seed, trng->len)) + goto error; + + if (trng_reseed_internal(trng, eseed, NULL, mul)) + goto error; + + return TEE_SUCCESS; +error: + trng->status = TRNG_ERROR; + return TEE_ERROR_GENERIC; +} + +static TEE_Result trng_generate(struct versal_trng *trng, uint8_t *buf, + size_t blen, bool predict) +{ + uint32_t len = TRNG_SEC_STRENGTH_LEN; + uint8_t *p = buf; + + if (!trng) + return TEE_ERROR_GENERIC; + + if (!p) + goto error; + + if (blen < TRNG_SEC_STRENGTH_LEN) + goto error; + + if (trng->status != TRNG_HEALTHY) + goto error; + + if (trng->usr_cfg.mode == TRNG_PTRNG && predict) + goto error; + + if (!trng->usr_cfg.predict_en && predict) + goto error; + + switch (trng->usr_cfg.mode) { + case TRNG_HRNG: + if (trng->stats.elapsed_seed_life >= trng->usr_cfg.seed_life) { + if (trng_reseed_internal(trng, NULL, NULL, 0)) + goto error; + } + + if (predict && trng->stats.elapsed_seed_life > 0) { + if (trng_reseed_internal(trng, NULL, NULL, 0)) + goto error; + } + + trng_write32(trng->cfg.addr, TRNG_CTRL, PRNGMODE_GEN); + break; + case TRNG_DRNG: + if (trng->stats.elapsed_seed_life > trng->usr_cfg.seed_life) + goto error; + + if (predict && trng->stats.elapsed_seed_life > 0) + goto error; + + trng_write32(trng->cfg.addr, TRNG_CTRL, PRNGMODE_GEN); + break; + default: + if (!trng->usr_cfg.df_disable) { + memset(&trng->dfin, 0, sizeof(trng->dfin)); + len = (trng->usr_cfg.dfmul + 1) * BYTES_PER_BLOCK; + trng->len = len; + p = trng->dfin.entropy; + } + /* Enable the 8 ring oscillators used for entropy source */ + trng_write32(trng->cfg.addr, TRNG_OSC_EN, TRNG_OSC_EN_VAL_MASK); + trng_soft_reset(trng); + trng_write32(trng->cfg.addr, TRNG_CTRL, + TRNG_CTRL_EUMODE_MASK | TRNG_CTRL_TRSSEN_MASK); + break; + } + + if (trng_collect_random(trng, p, len)) + goto error; + + trng->stats.bytes_reseed += len; + trng->stats.bytes += len; + trng->stats.elapsed_seed_life++; + + if (!trng->usr_cfg.df_disable && trng->usr_cfg.mode == TRNG_PTRNG) + trng_df_algorithm(trng, buf, DF_RAND, NULL); + + return TEE_SUCCESS; +error: + if (trng->status != TRNG_CATASTROPHIC) + trng->status = TRNG_ERROR; + + return TEE_ERROR_GENERIC; +} + +static TEE_Result trng_release(struct versal_trng *trng) +{ + if (!trng) + return TEE_ERROR_GENERIC; + + if (trng->status == TRNG_UNINITIALIZED) + goto error; + + trng_write32_range(trng, TRNG_EXT_SEED_0, TRNG_SEED_REGS, NULL); + trng_write32_range(trng, TRNG_PER_STRING_0, TRNG_PERS_STR_REGS, NULL); + trng_hold_reset(trng); + + /* Clear the instance */ + memset(&trng->usr_cfg, 0, sizeof(trng->usr_cfg)); + memset(trng->buf, 0, sizeof(trng->buf)); + memset(trng->dfout, 0, sizeof(trng->dfout)); + trng->status = TRNG_UNINITIALIZED; + + return TEE_SUCCESS; +error: + trng->status = TRNG_ERROR; + + return TEE_ERROR_GENERIC; +} + +/* Health tests should be run when the configured mode is of PTRNG or HRNG */ +static TEE_Result trng_health_test(struct versal_trng *trng) +{ + struct trng_usr_cfg tests = { + .mode = TRNG_HRNG, + .seed_life = 10, + .dfmul = 7, + .predict_en = false, + .iseed_en = false, + .pstr_en = false, + .df_disable = false, + }; + + if (trng_instantiate(trng, &tests)) + goto error; + + if (trng_release(trng)) + goto error; + + return TEE_SUCCESS; +error: + trng->status = TRNG_ERROR; + + return TEE_ERROR_GENERIC; +} + +/* + * The KAT test should be run when the TRNG is configured in DRNG or HRNG mode. + * If KAT fails, the driver has to be put in error state. + */ +static TEE_Result trng_kat_test(struct versal_trng *trng) +{ + struct trng_usr_cfg tests = { + .mode = TRNG_DRNG, + .seed_life = 5, + .dfmul = 2, + .predict_en = false, + .iseed_en = true, + .pstr_en = true, + .df_disable = false, + }; + const uint8_t ext_seed[TRNG_SEED_LEN] = { + 0x3BU, 0xC3U, 0xEDU, 0x64U, 0xF4U, 0x80U, 0x1CU, 0xC7U, + 0x14U, 0xCCU, 0x35U, 0xEDU, 0x57U, 0x01U, 0x2AU, 0xE4U, + 0xBCU, 0xEFU, 0xDEU, 0xF6U, 0x7CU, 0x46U, 0xA6U, 0x34U, + 0xC6U, 0x79U, 0xE8U, 0x91U, 0x5DU, 0xB1U, 0xDBU, 0xA7U, + 0x49U, 0xA5U, 0xBBU, 0x4FU, 0xEDU, 0x30U, 0xB3U, 0x7BU, + 0xA9U, 0x8BU, 0xF5U, 0x56U, 0x4DU, 0x40U, 0x18U, 0x9FU, + }; + const uint8_t pers_str[TRNG_PERS_STR_LEN] = { + 0xB2U, 0x80U, 0x7EU, 0x4CU, 0xD0U, 0xE4U, 0xE2U, 0xA9U, + 0x2FU, 0x1FU, 0x5DU, 0xC1U, 0xA2U, 0x1FU, 0x40U, 0xFCU, + 0x1FU, 0x24U, 0x5DU, 0x42U, 0x61U, 0x80U, 0xE6U, 0xE9U, + 0x71U, 0x05U, 0x17U, 0x5BU, 0xAFU, 0x70U, 0x30U, 0x18U, + 0xBCU, 0x23U, 0x18U, 0x15U, 0xCBU, 0xB8U, 0xA6U, 0x3EU, + 0x83U, 0xB8U, 0x4AU, 0xFEU, 0x38U, 0xFCU, 0x25U, 0x87U, + }; + const uint8_t expected_out[TRNG_GEN_LEN] = { + 0x91U, 0x9AU, 0x6BU, 0x99U, 0xD5U, 0xBCU, 0x2CU, 0x11U, + 0x5FU, 0x3AU, 0xFCU, 0x0BU, 0x0EU, 0x7BU, 0xC7U, 0x69U, + 0x4DU, 0xE1U, 0xE5U, 0xFEU, 0x59U, 0x9EU, 0xAAU, 0x41U, + 0xD3U, 0x48U, 0xFDU, 0x3DU, 0xD2U, 0xC4U, 0x50U, 0x1EU, + }; + uint8_t out[TRNG_GEN_LEN] = { 0 }; + + if (!trng) + return TEE_ERROR_GENERIC; + + memcpy(&tests.init_seed, ext_seed, sizeof(ext_seed)); + memcpy(tests.pstr, pers_str, sizeof(pers_str)); + + if (trng_instantiate(trng, &tests)) + goto error; + + if (trng_generate(trng, out, sizeof(out), false)) + goto error; + + if (memcmp(out, expected_out, TRNG_GEN_LEN)) { + EMSG("K.A.T mismatch"); + goto error; + } + + if (trng_release(trng)) + goto error; + + return TEE_SUCCESS; +error: + trng->status = TRNG_ERROR; + return TEE_ERROR_GENERIC; +} + +static struct versal_trng versal_trng = { + .cfg.base = TRNG_BASE, + .cfg.len = TRNG_SIZE, +}; + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + uint8_t random[TRNG_SEC_STRENGTH_LEN] = { 0 }; + uint8_t *p = buf; + size_t i = 0; + + for (i = 0; i < len / TRNG_SEC_STRENGTH_LEN; i++) { + if (trng_generate(&versal_trng, p + i * TRNG_SEC_STRENGTH_LEN, + TRNG_SEC_STRENGTH_LEN, false)) + panic(); + } + + if (len % TRNG_SEC_STRENGTH_LEN) { + if (trng_generate(&versal_trng, random, TRNG_SEC_STRENGTH_LEN, + false)) + panic(); + memcpy(p + i * TRNG_SEC_STRENGTH_LEN, random, + len % TRNG_SEC_STRENGTH_LEN); + } + + return TEE_SUCCESS; +} + +void plat_rng_init(void) +{ +} + +static TEE_Result trng_hrng_mode_init(void) +{ + const uint8_t pers_str[TRNG_PERS_STR_LEN] = { + 0xB2, 0x80, 0x7E, 0x4C, 0xD0, 0xE4, 0xE2, 0xA9, + 0x2F, 0x1F, 0x5D, 0xC1, 0xA2, 0x1F, 0x40, 0xFC, + 0x1F, 0x24, 0x5D, 0x42, 0x61, 0x80, 0xE6, 0xE9, + 0x71, 0x05, 0x17, 0x5B, 0xAF, 0x70, 0x30, 0x18, + 0xBC, 0x23, 0x18, 0x15, 0xCB, 0xB8, 0xA6, 0x3E, + 0x83, 0xB8, 0x4A, 0xFE, 0x38, 0xFC, 0x25, 0x87, + }; + /* configure in hybrid mode with derivative function enabled */ + struct trng_usr_cfg usr_cfg = { + .mode = TRNG_HRNG, + .seed_life = CFG_VERSAL_TRNG_SEED_LIFE, + .predict_en = false, + .df_disable = false, + .dfmul = CFG_VERSAL_TRNG_DF_MUL, + .iseed_en = false, + .pstr_en = true, + }; + + memcpy(usr_cfg.pstr, pers_str, TRNG_PERS_STR_LEN); + versal_trng.cfg.addr = (vaddr_t)core_mmu_add_mapping(MEM_AREA_IO_SEC, + versal_trng.cfg.base, + versal_trng.cfg.len); + if (!versal_trng.cfg.addr) { + EMSG("Failed to map TRNG"); + panic(); + } + + if (trng_kat_test(&versal_trng)) { + EMSG("KAT Failed"); + panic(); + } + + if (trng_health_test(&versal_trng)) { + EMSG("RunHealthTest Failed"); + panic(); + } + + if (trng_instantiate(&versal_trng, &usr_cfg)) { + EMSG("Driver instantiation Failed"); + panic(); + } + + if (trng_reseed(&versal_trng, NULL, usr_cfg.dfmul)) { + EMSG("Reseed Failed"); + panic(); + } + + return TEE_SUCCESS; +} + +driver_init(trng_hrng_mode_init); diff --git a/optee_os/core/drivers/wdt/sub.mk b/optee_os/core/drivers/wdt/sub.mk new file mode 100644 index 0000000..3e20ff8 --- /dev/null +++ b/optee_os/core/drivers/wdt/sub.mk @@ -0,0 +1,2 @@ +srcs-$(CFG_WDT) += watchdog.c +srcs-$(CFG_WDT_SM_HANDLER) += watchdog_sm.c diff --git a/optee_os/core/drivers/wdt/watchdog.c b/optee_os/core/drivers/wdt/watchdog.c new file mode 100644 index 0000000..274f4e2 --- /dev/null +++ b/optee_os/core/drivers/wdt/watchdog.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microchip + */ + +#include + +struct wdt_chip *wdt_chip; + +TEE_Result watchdog_register(struct wdt_chip *chip) +{ + if (!chip->ops->start || !chip->ops->ping || !chip->ops->set_timeout) + return TEE_ERROR_BAD_PARAMETERS; + + wdt_chip = chip; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/wdt/watchdog_sm.c b/optee_os/core/drivers/wdt/watchdog_sm.c new file mode 100644 index 0000000..a9fbc1b --- /dev/null +++ b/optee_os/core/drivers/wdt/watchdog_sm.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 Microchip + */ + +#include +#include +#include +#include +#include + +static unsigned long wdt_min_timeout; +static unsigned long wdt_max_timeout; +/* Lock for timeout variables */ +static unsigned int wdt_lock = SPINLOCK_UNLOCK; + +enum sm_handler_ret __wdt_sm_handler(struct thread_smc_args *args) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t exceptions = 0; + unsigned long min_timeout = 0; + unsigned long max_timeout = 0; + + switch (args->a1) { + case SMCWD_INIT: + exceptions = cpu_spin_lock_xsave(&wdt_lock); + res = watchdog_init(&wdt_min_timeout, &wdt_max_timeout); + cpu_spin_unlock_xrestore(&wdt_lock, exceptions); + + if (res) { + args->a0 = PSCI_RET_INTERNAL_FAILURE; + } else { + args->a0 = PSCI_RET_SUCCESS; + args->a1 = wdt_min_timeout; + args->a2 = wdt_max_timeout; + } + break; + case SMCWD_SET_TIMEOUT: + exceptions = cpu_spin_lock_xsave(&wdt_lock); + min_timeout = wdt_min_timeout; + max_timeout = wdt_max_timeout; + cpu_spin_unlock_xrestore(&wdt_lock, exceptions); + + if (args->a2 < min_timeout || args->a2 > max_timeout) { + args->a0 = PSCI_RET_INVALID_PARAMETERS; + break; + } + + watchdog_settimeout(args->a2); + args->a0 = PSCI_RET_SUCCESS; + break; + case SMCWD_ENABLE: + if (args->a2 == 0) { + watchdog_stop(); + args->a0 = PSCI_RET_SUCCESS; + } else if (args->a2 == 1) { + watchdog_start(); + args->a0 = PSCI_RET_SUCCESS; + } else { + args->a0 = PSCI_RET_INVALID_PARAMETERS; + } + break; + case SMCWD_PET: + watchdog_ping(); + args->a0 = PSCI_RET_SUCCESS; + break; + /* SMCWD_GET_TIMELEFT is optional */ + case SMCWD_GET_TIMELEFT: + default: + args->a0 = PSCI_RET_NOT_SUPPORTED; + } + + return SM_HANDLER_SMC_HANDLED; +} diff --git a/optee_os/core/drivers/xiphera_trng.c b/optee_os/core/drivers/xiphera_trng.c new file mode 100644 index 0000000..9786c19 --- /dev/null +++ b/optee_os/core/drivers/xiphera_trng.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Vaisala Oyj + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CONTROL_REG 0x00000000 +#define STATUS_REG 0x00000004 +#define RAND_REG 0x00000000 + +#define HOST_TO_TRNG_RESET 0x00000001 +#define HOST_TO_TRNG_RELEASE_RESET 0x00000002 +#define HOST_TO_TRNG_ENABLE 0x80000000 +#define HOST_TO_TRNG_ZEROIZE 0x80000004 +#define HOST_TO_TRNG_ACK_ZEROIZE 0x80000008 +#define HOST_TO_TRNG_READ 0x8000000F + +/* trng statuses */ +#define TRNG_ACK_RESET 0x000000AC +#define TRNG_SUCCESSFUL_STARTUP 0x00000057 +#define TRNG_FAILED_STARTUP 0x000000FA +#define TRNG_NEW_RAND_AVAILABLE 0x000000ED + +static unsigned int trng_lock = SPINLOCK_UNLOCK; + +static vaddr_t xiphera_trng_base; + +static bool xiphera_trng_random_available(void) +{ + uint32_t status = 0; + + status = io_read32(xiphera_trng_base + STATUS_REG); + + return status == TRNG_NEW_RAND_AVAILABLE; +} + +static uint32_t xiphera_trng_read32(void) +{ + uint32_t value = 0; + + value = io_read32(xiphera_trng_base + RAND_REG); + + /* + * Ack that RNG value has been consumed and trigger new one to be + * generated + */ + io_write32(xiphera_trng_base + CONTROL_REG, HOST_TO_TRNG_READ); + io_write32(xiphera_trng_base + CONTROL_REG, HOST_TO_TRNG_ENABLE); + + return value; +} + +/* This is a true RNG, no need for seeding */ +void plat_rng_init(void) +{ +} + +TEE_Result hw_get_random_bytes(void *buf, size_t len) +{ + uint8_t *rngbuf = buf; + uint32_t val = 0; + size_t len_to_copy = 0; + + assert(buf); + assert(xiphera_trng_base); + + while (len) { + uint32_t exceptions = cpu_spin_lock_xsave(&trng_lock); + + if (xiphera_trng_random_available()) { + val = xiphera_trng_read32(); + + len_to_copy = MIN(len, sizeof(uint32_t)); + memcpy(rngbuf, &val, len_to_copy); + rngbuf += len_to_copy; + len -= len_to_copy; + } + + cpu_spin_unlock_xrestore(&trng_lock, exceptions); + } + + return TEE_SUCCESS; +} + +static TEE_Result xiphera_trng_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + int dt_status = fdt_get_status(fdt, node); + uint32_t status = 0; + size_t size = 0; + + /* Skip non-secure instances */ + if (dt_status != DT_STATUS_OK_SEC) + return TEE_ERROR_NODE_DISABLED; + + if (xiphera_trng_base) { + EMSG("Only one secure instance is supported"); + return TEE_ERROR_GENERIC; + } + + if (dt_map_dev(fdt, node, &xiphera_trng_base, &size, DT_MAP_AUTO) < 0) + return TEE_ERROR_GENERIC; + + /* + * The TRNG needs to be first reset in order to provide stable + * operation. + * + * Reset of the chip should complete within 200 us but in some cases it + * could take up to 400 us. If it is not ready within 400 us assume + * there is problem. + */ + io_write32(xiphera_trng_base + CONTROL_REG, HOST_TO_TRNG_RESET); + udelay(200); + + status = io_read32(xiphera_trng_base + STATUS_REG); + if (status != TRNG_ACK_RESET) { + /* + * Give it additional 200 us to allow it to reset. + * + * If still not done -> error out. + */ + udelay(200); + status = io_read32(xiphera_trng_base + STATUS_REG); + if (status != TRNG_ACK_RESET) { + EMSG("Failed to reset TRNG\n"); + return TEE_ERROR_GENERIC; + } + } + + /* + * Now TRNG should be internally stable. + * + * Clear internal random number generation engine to start in stable + * state and give it 20 ms to enable good random number entropy and + * then check that random number engine is ready. + */ + io_write32(xiphera_trng_base + CONTROL_REG, + HOST_TO_TRNG_RELEASE_RESET); + io_write32(xiphera_trng_base + CONTROL_REG, HOST_TO_TRNG_ENABLE); + io_write32(xiphera_trng_base + CONTROL_REG, HOST_TO_TRNG_ZEROIZE); + mdelay(20); + + status = io_read32(xiphera_trng_base + STATUS_REG); + if (status != TRNG_SUCCESSFUL_STARTUP) { + /* + * Check specifically if there were startup test errors to aid + * in debugging TRNG implementation in FPGA + */ + if (status == TRNG_FAILED_STARTUP) { + EMSG("Startup tests have failed\n"); + return TEE_ERROR_GENERIC; + } + + EMSG("Startup tests yielded no response -> TRNG stuck\n"); + return TEE_ERROR_GENERIC; + } + + io_write32(xiphera_trng_base + CONTROL_REG, HOST_TO_TRNG_ACK_ZEROIZE); + + DMSG("TRNG initialized\n"); + + return TEE_SUCCESS; +} + +static const struct dt_device_match xiphera_trng_match_table[] = { + { .compatible = "xiphera,xip8001b-trng" }, + { } +}; + +DEFINE_DT_DRIVER(xiphera_trng_dt_driver) = { + .name = "xiphera_trng", + .type = DT_DRIVER_NOTYPE, + .match_table = xiphera_trng_match_table, + .probe = xiphera_trng_probe, +}; diff --git a/optee_os/core/drivers/zynqmp_csu_aes.c b/optee_os/core/drivers/zynqmp_csu_aes.c new file mode 100644 index 0000000..ae5e686 --- /dev/null +++ b/optee_os/core/drivers/zynqmp_csu_aes.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2021 + * Author: Jorge Ramirez + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* CSU AES registers */ +#define AES_STS_OFFSET 0x00 +#define AES_KEY_SRC_OFFSET 0x04 +#define AES_KEY_LOAD_OFFSET 0x08 +#define AES_START_MSG_OFFSET 0x0C +#define AES_RESET_OFFSET 0x10 +#define AES_KEY_CLR_OFFSET 0x14 +#define AES_CFG_OFFSET 0x18 + +#define AES_KEY_LOAD 1 +#define AES_STS_AES_BUSY BIT(0) +#define AES_STS_AES_KEY_ZEROED BIT(8) +#define AES_STS_KUP_ZEROED BIT(9) +#define AES_STS_KEY_INIT_DONE BIT(4) +#define AES_STS_GCM_TAG_OK BIT(3) +#define AES_START_MSG 1 +#define AES_CFG_ENC 1 +#define AES_CFG_DEC 0 +#define AES_RESET_SET 1 +#define AES_RESET_CLR 0 +#define AES_KEY_ZERO BIT(0) +#define AES_KUP_ZERO BIT(1) + +#define AES_TIMEOUT_USEC 2000000 + +enum aes_op { AES_DEC, AES_ENC }; + +static TEE_Result aes_wait(uint32_t event, bool set) +{ + vaddr_t aes = core_mmu_get_va(ZYNQMP_CSU_AES_BASE, MEM_AREA_IO_SEC, + ZYNQMP_CSU_AES_SIZE); + uint64_t tref = timeout_init_us(AES_TIMEOUT_USEC); + uint32_t status = 0; + + if (!aes) + return TEE_ERROR_GENERIC; + + while (!timeout_elapsed(tref)) { + status = io_read32(aes + AES_STS_OFFSET) & event; + if ((set && status == event) || (!set && status != event)) + return TEE_SUCCESS; + } + + return TEE_ERROR_GENERIC; +} + +static TEE_Result aes_transfer_enc(const void *src, void *dst, size_t dst_len, + void *tag, const void *iv) +{ + void *p = (uint8_t *)dst + dst_len - ZYNQMP_GCM_TAG_SIZE; + uint8_t iv_padded[ZYNQMP_CSUDMA_MIN_SIZE] __aligned_csudma = { 0 }; + TEE_Result ret = TEE_SUCCESS; + + if (dst_len < ZYNQMP_GCM_TAG_SIZE) { + EMSG("Invalid length"); + return TEE_ERROR_GENERIC; + } + + ret = zynqmp_csudma_prepare(); + if (ret) { + EMSG("DMA can't initialize"); + return ret; + } + + /* Prepare destination */ + ret = zynqmp_csudma_transfer(ZYNQMP_CSUDMA_DST_CHANNEL, dst, dst_len, + 0); + if (ret) { + EMSG("DMA transfer failed, invalid destination buffer"); + goto out; + } + + /* Inputs */ + memcpy(iv_padded, iv, ZYNQMP_GCM_IV_SIZE); + ret = zynqmp_csudma_transfer(ZYNQMP_CSUDMA_SRC_CHANNEL, + (void *)iv_padded, ZYNQMP_CSUDMA_MIN_SIZE, + 0); + if (ret) { + EMSG("DMA transfer failed, invalid IV buffer"); + goto out; + } + + ret = zynqmp_csudma_sync(ZYNQMP_CSUDMA_SRC_CHANNEL); + if (ret) { + EMSG("DMA IV transfer timeout"); + goto out; + } + + ret = zynqmp_csudma_transfer(ZYNQMP_CSUDMA_SRC_CHANNEL, + (void *)src, dst_len - ZYNQMP_GCM_TAG_SIZE, + ZYNQMP_CSUDMA_DONE); + if (ret) { + EMSG("DMA transfer failed, invalid source buffer"); + goto out; + } + + ret = zynqmp_csudma_sync(ZYNQMP_CSUDMA_SRC_CHANNEL); + if (ret) { + EMSG("DMA source transfer timeout"); + goto out; + } + + /* Wait for completion */ + ret = zynqmp_csudma_sync(ZYNQMP_CSUDMA_DST_CHANNEL); + if (ret) { + EMSG("DMA destination transfer timeout"); + goto out; + } + + /* Transfer the GCM tag */ + memcpy(tag, p, ZYNQMP_GCM_TAG_SIZE); +out: + zynqmp_csudma_unprepare(); + + return ret; +} + +static TEE_Result aes_transfer_dec(const void *src, void *dst, size_t len, + const void *tag, const void *iv) +{ + uint8_t iv_padded[ZYNQMP_CSUDMA_MIN_SIZE] __aligned_csudma = { 0 }; + TEE_Result ret = TEE_SUCCESS; + + ret = zynqmp_csudma_prepare(); + if (ret) { + EMSG("DMA can't initialize"); + return ret; + } + + /* Prepare destination */ + ret = zynqmp_csudma_transfer(ZYNQMP_CSUDMA_DST_CHANNEL, dst, len, 0); + if (ret) { + EMSG("DMA transfer failed, invalid destination buffer"); + goto out; + } + + /* Inputs */ + memcpy(iv_padded, iv, ZYNQMP_GCM_IV_SIZE); + ret = zynqmp_csudma_transfer(ZYNQMP_CSUDMA_SRC_CHANNEL, + (void *)iv_padded, ZYNQMP_CSUDMA_MIN_SIZE, + 0); + if (ret) { + EMSG("DMA transfer failed, invalid IV buffer"); + goto out; + } + + ret = zynqmp_csudma_sync(ZYNQMP_CSUDMA_SRC_CHANNEL); + if (ret) { + EMSG("DMA IV transfer timeout"); + goto out; + } + + ret = zynqmp_csudma_transfer(ZYNQMP_CSUDMA_SRC_CHANNEL, (void *)src, + len, ZYNQMP_CSUDMA_DONE); + if (ret) { + EMSG("DMA transfer failed, invalid source buffer"); + goto out; + } + + ret = zynqmp_csudma_sync(ZYNQMP_CSUDMA_SRC_CHANNEL); + if (ret) { + EMSG("DMA source transfer timeout"); + goto out; + } + + ret = zynqmp_csudma_transfer(ZYNQMP_CSUDMA_SRC_CHANNEL, (void *)tag, + ZYNQMP_GCM_TAG_SIZE, ZYNQMP_CSUDMA_DONE); + if (ret) { + EMSG("DMA transfer failed, invalid tag buffer"); + goto out; + } + + ret = zynqmp_csudma_sync(ZYNQMP_CSUDMA_SRC_CHANNEL); + if (ret) { + EMSG("DMA tag transfer timeout"); + goto out; + } + + /* Wait for completion*/ + ret = zynqmp_csudma_sync(ZYNQMP_CSUDMA_DST_CHANNEL); + if (ret) + EMSG("DMA destination transfer timeout"); +out: + zynqmp_csudma_unprepare(); + + return ret; +} + +static TEE_Result aes_prepare_op(enum aes_op op, enum zynqmp_csu_key key) +{ + vaddr_t aes = core_mmu_get_va(ZYNQMP_CSU_AES_BASE, MEM_AREA_IO_SEC, + ZYNQMP_CSU_AES_SIZE); + vaddr_t csu = core_mmu_get_va(CSU_BASE, MEM_AREA_IO_SEC, CSU_SIZE); + TEE_Result ret = TEE_SUCCESS; + + if (!aes || !csu) + return TEE_ERROR_GENERIC; + + /* Connect DMA0 in/out to AES */ + io_write32(csu + ZYNQMP_CSU_SSS_CFG_OFFSET, + ZYNQMP_CSU_SSS_DMA0_STREAM_TO_AES); + + /* Reset the AES */ + io_write32(aes + AES_RESET_OFFSET, AES_RESET_SET); + io_write32(aes + AES_RESET_OFFSET, AES_RESET_CLR); + + /* Load the key */ + io_write32(aes + AES_KEY_CLR_OFFSET, 0); + io_write32(aes + AES_KEY_SRC_OFFSET, key); + io_write32(aes + AES_KEY_LOAD_OFFSET, AES_KEY_LOAD); + ret = aes_wait(AES_STS_KEY_INIT_DONE, true); + if (ret) { + EMSG("Timeout loading the key"); + return TEE_ERROR_GENERIC; + } + + /* Configure operation */ + io_write32(aes + AES_CFG_OFFSET, + op == AES_DEC ? AES_CFG_DEC : AES_CFG_ENC); + + /* Prepare the CSU for the DMA */ + io_write32(csu + ZYNQMP_CSU_DMA_RESET_OFFSET, ZYNQMP_CSU_DMA_RESET_SET); + io_write32(csu + ZYNQMP_CSU_DMA_RESET_OFFSET, ZYNQMP_CSU_DMA_RESET_CLR); + + /* Start the message */ + io_write32(aes + AES_START_MSG_OFFSET, AES_START_MSG); + + return TEE_SUCCESS; +} + +static TEE_Result aes_done_op(enum aes_op op, TEE_Result ret) +{ + vaddr_t aes = core_mmu_get_va(ZYNQMP_CSU_AES_BASE, MEM_AREA_IO_SEC, + ZYNQMP_CSU_AES_SIZE); + uint32_t val = 0; + + if (!aes) + return TEE_ERROR_GENERIC; + + if (!ret && op == AES_DEC) { + /* on decompression we must validate the GCM tag */ + val = io_read32(aes + AES_STS_OFFSET) & AES_STS_GCM_TAG_OK; + if (!val) { + EMSG("AES-GCM tag mismatch"); + return TEE_ERROR_GENERIC; + } + } + + val = io_read32(aes + AES_KEY_CLR_OFFSET); + io_write32(aes + AES_KEY_CLR_OFFSET, val | AES_KEY_ZERO | AES_KUP_ZERO); + if (aes_wait(AES_STS_AES_KEY_ZEROED | AES_STS_KUP_ZEROED, true)) + EMSG("Failed to clear the AES key"); + io_write32(aes + AES_KEY_CLR_OFFSET, val); + + io_write32(aes + AES_RESET_OFFSET, AES_RESET_SET); + + return ret; +} + +TEE_Result zynqmp_csu_aes_decrypt_data(const void *src, size_t src_len, + void *dst, size_t dst_len, + const void *tag, size_t tag_len, + const void *iv, size_t iv_len, + enum zynqmp_csu_key key) +{ + TEE_Result ret = TEE_SUCCESS; + + if (key != ZYNQMP_CSU_AES_KEY_SRC_DEV) { + EMSG("Key type not supported"); + return TEE_ERROR_NOT_IMPLEMENTED; + } + + if (src_len % 4 || dst_len != src_len) { + EMSG("Invalid source size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (iv_len != ZYNQMP_GCM_IV_SIZE) { + EMSG("Invalid IV size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (tag_len != ZYNQMP_GCM_TAG_SIZE) { + EMSG("Invalid tag size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!src || !dst || !tag || !iv) { + EMSG("Invalid input value"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = aes_prepare_op(AES_DEC, key); + if (ret) { + EMSG("Decrypt init failed"); + goto out; + } + + ret = aes_transfer_dec(src, dst, src_len, tag, iv); + if (ret) { + EMSG("DMA transfer failed"); + goto out; + } + + ret = aes_wait(AES_STS_AES_BUSY, false); + if (ret) + EMSG("AES-GCM transfer failed"); +out: + return aes_done_op(AES_DEC, ret); +} + +TEE_Result zynqmp_csu_aes_encrypt_data(const void *src, size_t src_len, + void *dst, size_t dst_len, + void *tag, size_t tag_len, + const void *iv, size_t iv_len, + enum zynqmp_csu_key key) +{ + TEE_Result ret = TEE_SUCCESS; + + if (key != ZYNQMP_CSU_AES_KEY_SRC_DEV) { + EMSG("Key type not supported"); + return TEE_ERROR_NOT_IMPLEMENTED; + } + + if (src_len % 4 || dst_len != ZYNQMP_CSU_AES_DST_LEN(src_len)) { + EMSG("Invalid source size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (iv_len != ZYNQMP_GCM_IV_SIZE) { + EMSG("Invalid IV size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (tag_len != ZYNQMP_GCM_TAG_SIZE) { + EMSG("Invalid tag size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!src || !dst || !tag || !iv) { + EMSG("Invalid input value"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = aes_prepare_op(AES_ENC, key); + if (ret) { + EMSG("Encrypt init failed"); + goto out; + } + + ret = aes_transfer_enc(src, dst, dst_len, tag, iv); + if (ret) { + EMSG("DMA transfer failed"); + goto out; + } + + ret = aes_wait(AES_STS_AES_BUSY, false); + if (ret) + EMSG("AES transfer failed"); +out: + return aes_done_op(AES_ENC, ret); +} + +static const char *const dt_ctrl_match_table[] = { + "xlnx,zynqmp-aes", +}; + +TEE_Result zynqmp_csu_aes_dt_enable_secure_status(void) +{ + unsigned int i = 0; + void *fdt = NULL; + int node = -1; + + fdt = get_external_dt(); + if (!fdt) + return TEE_SUCCESS; + + for (i = 0; i < ARRAY_SIZE(dt_ctrl_match_table); i++) { + node = fdt_node_offset_by_compatible(fdt, 0, + dt_ctrl_match_table[i]); + if (node >= 0) + break; + } + + if (node < 0) + return TEE_SUCCESS; + + if (fdt_get_status(fdt, node) == DT_STATUS_DISABLED) + return TEE_SUCCESS; + + if (dt_enable_secure_status(fdt, node)) { + EMSG("Not able to set the AES-GCM DTB entry secure"); + return TEE_ERROR_NOT_SUPPORTED; + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/zynqmp_csu_puf.c b/optee_os/core/drivers/zynqmp_csu_puf.c new file mode 100644 index 0000000..1f8dc4f --- /dev/null +++ b/optee_os/core/drivers/zynqmp_csu_puf.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 Foundries.io Ltd. + * + * Jorge Ramirez-Ortiz + */ +#include +#include +#include +#include +#include +#include +#include + +#define PUF_CMD_OFFSET 0x00 +#define PUF_CFG0_OFFSET 0x04 +#define PUF_CFG1_OFFSET 0x08 +#define PUF_SHUT_OFFSET 0x0C +#define PUF_STATUS_OFFSET 0x10 +#define PUF_WORD_OFFSET 0x18 + +#define PUF_REGENERATION 4 +#define PUF_RESET 6 + +#define PUF_CFG0_DEFAULT 0x02 +#define PUF_SHUT_DEFAULT 0x01000100 +#define PUF_REGEN_TIME_MS 6 + +TEE_Result zynqmp_csu_puf_regenerate(void) +{ + vaddr_t puf = core_mmu_get_va(ZYNQMP_CSU_PUF_BASE, MEM_AREA_IO_SEC, + ZYNQMP_CSU_PUF_SIZE); + vaddr_t csu = core_mmu_get_va(CSU_BASE, MEM_AREA_IO_SEC, CSU_SIZE); + uint32_t status = 0; + + if (!puf || !csu) + return TEE_ERROR_GENERIC; + + io_write32(puf + PUF_CFG0_OFFSET, PUF_CFG0_DEFAULT); + io_write32(puf + PUF_SHUT_OFFSET, PUF_SHUT_DEFAULT); + io_write32(puf + PUF_CMD_OFFSET, PUF_REGENERATION); + mdelay(PUF_REGEN_TIME_MS); + + status = io_read32(csu + ZYNQMP_CSU_ISR_OFFSET); + if (status & ZYNQMP_CSU_ISR_PUF_ACC_ERROR_MASK) { + EMSG("regeneration failed"); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} + +void zynqmp_csu_puf_reset(void) +{ + vaddr_t puf = core_mmu_get_va(ZYNQMP_CSU_PUF_BASE, MEM_AREA_IO_SEC, + ZYNQMP_CSU_PUF_SIZE); + + io_write32(puf + PUF_CMD_OFFSET, PUF_RESET); +} + +static TEE_Result zynqmp_csu_puf_init(void) +{ + vaddr_t csu = core_mmu_get_va(CSU_BASE, MEM_AREA_IO_SEC, CSU_SIZE); + uint32_t status = 0; + + /* if the bootloader has been authenticated, reserve the PUF */ + status = io_read32(csu + ZYNQMP_CSU_STATUS_OFFSET); + if (status & ZYNQMP_CSU_STATUS_AUTH) + return zynqmp_csu_aes_dt_enable_secure_status(); + + return TEE_SUCCESS; +} + +driver_init(zynqmp_csu_puf_init); diff --git a/optee_os/core/drivers/zynqmp_csudma.c b/optee_os/core/drivers/zynqmp_csudma.c new file mode 100644 index 0000000..8809635 --- /dev/null +++ b/optee_os/core/drivers/zynqmp_csudma.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 Foundries.io Ltd. + * Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CSUDMA_ADDR_OFFSET 0x00 +#define CSUDMA_SIZE_OFFSET 0x04 +#define CSUDMA_STS_OFFSET 0x08 +#define CSUDMA_CTRL_OFFSET 0x0C +#define CSUDMA_CRC_OFFSET 0x10 +#define CSUDMA_I_STS_OFFSET 0x14 +#define CSUDMA_I_EN_OFFSET 0x18 +#define CSUDMA_I_DIS_OFFSET 0x1C +#define CSUDMA_I_MASK_OFFSET 0x20 +#define CSUDMA_CTRL2_OFFSET 0x24 +#define CSUDMA_ADDR_MSB_OFFSET 0x28 + +#define CSUDMA_OFFSET_DIFF 0x0800 + +#define CSUDMA_ADDR_MASK GENMASK_32(31, 2) +#define CSUDMA_ADDR_LSB_MASK (BIT(0) | BIT(1)) +#define CSUDMA_ADDR_MSB_MASK GENMASK_32(16, 0) +#define CSUDMA_ADDR_MSB_SHIFT 32 +#define CSUDMA_SIZE_SHIFT 2 +#define CSUDMA_STS_BUSY_MASK BIT(0) +#define CSUDMA_CTRL_ENDIAN_MASK BIT(23) +#define CSUDMA_LAST_WORD_MASK BIT(0) +#define CSUDMA_IXR_DONE_MASK BIT(1) +#define CSUDMA_IXR_SRC_MASK GENMASK_32(6, 0) +#define CSUDMA_IXR_DST_MASK GENMASK_32(7, 1) + +#define CSUDMA_DONE_TIMEOUT_USEC 3000000 + +register_phys_mem_pgdir(MEM_AREA_IO_SEC, CSUDMA_BASE, CSUDMA_SIZE); + +static void csudma_clear_intr(enum zynqmp_csudma_channel channel, uint32_t mask) +{ + vaddr_t dma = core_mmu_get_va(CSUDMA_BASE, MEM_AREA_IO_SEC, + CSUDMA_SIZE); + uint32_t val = CSUDMA_IXR_SRC_MASK; + + if (channel == ZYNQMP_CSUDMA_DST_CHANNEL) { + dma += CSUDMA_OFFSET_DIFF; + val = CSUDMA_IXR_DST_MASK; + } + + io_write32(dma + CSUDMA_I_STS_OFFSET, val & mask); +} + +TEE_Result zynqmp_csudma_sync(enum zynqmp_csudma_channel channel) +{ + vaddr_t dma = core_mmu_get_va(CSUDMA_BASE, MEM_AREA_IO_SEC, + CSUDMA_SIZE); + uint64_t tref = timeout_init_us(CSUDMA_DONE_TIMEOUT_USEC); + uint32_t status = 0; + + if (!dma) + return TEE_ERROR_GENERIC; + + if (channel == ZYNQMP_CSUDMA_DST_CHANNEL) + dma = dma + CSUDMA_OFFSET_DIFF; + + while (!timeout_elapsed(tref)) { + status = io_read32(dma + CSUDMA_I_STS_OFFSET); + if ((status & CSUDMA_IXR_DONE_MASK) == CSUDMA_IXR_DONE_MASK) { + csudma_clear_intr(channel, CSUDMA_IXR_DONE_MASK); + return TEE_SUCCESS; + } + } + + return TEE_ERROR_GENERIC; +} + +TEE_Result zynqmp_csudma_prepare(void) +{ + vaddr_t dma = core_mmu_get_va(CSUDMA_BASE, MEM_AREA_IO_SEC, + CSUDMA_SIZE); + + if (!dma) + return TEE_ERROR_GENERIC; + + io_setbits32(dma + CSUDMA_CTRL_OFFSET, CSUDMA_CTRL_ENDIAN_MASK); + dma = dma + CSUDMA_OFFSET_DIFF; + io_setbits32(dma + CSUDMA_CTRL_OFFSET, CSUDMA_CTRL_ENDIAN_MASK); + + return TEE_SUCCESS; +} + +void zynqmp_csudma_unprepare(void) +{ + vaddr_t dma = core_mmu_get_va(CSUDMA_BASE, MEM_AREA_IO_SEC, + CSUDMA_SIZE); + + io_clrbits32(dma + CSUDMA_CTRL_OFFSET, CSUDMA_CTRL_ENDIAN_MASK); + dma = dma + CSUDMA_OFFSET_DIFF; + io_clrbits32(dma + CSUDMA_CTRL_OFFSET, CSUDMA_CTRL_ENDIAN_MASK); +} + +TEE_Result zynqmp_csudma_transfer(enum zynqmp_csudma_channel channel, + void *addr, size_t len, uint8_t notify) +{ + vaddr_t dma = core_mmu_get_va(CSUDMA_BASE, MEM_AREA_IO_SEC, + CSUDMA_SIZE); + paddr_t phys = virt_to_phys(addr); + uint32_t addr_offset = 0; + + if (!dma) + return TEE_ERROR_GENERIC; + + if (len % sizeof(uint32_t)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!IS_ALIGNED(phys, ZYNQMP_CSUDMA_ALIGN)) { + EMSG("Invalid alignment"); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* convert to 32 bit word transfers */ + len = len / sizeof(uint32_t); + + if (channel == ZYNQMP_CSUDMA_DST_CHANNEL) { + dma = dma + CSUDMA_OFFSET_DIFF; + dcache_inv_range(addr, SHIFT_U64(len, CSUDMA_SIZE_SHIFT)); + } else { + dcache_clean_range(addr, SHIFT_U64(len, CSUDMA_SIZE_SHIFT)); + } + + addr_offset = phys & CSUDMA_ADDR_MASK; + io_write32(dma + CSUDMA_ADDR_OFFSET, addr_offset); + + addr_offset = phys >> CSUDMA_ADDR_MSB_SHIFT; + io_write32(dma + CSUDMA_ADDR_MSB_OFFSET, addr_offset); + io_write32(dma + CSUDMA_SIZE_OFFSET, + SHIFT_U32(len, CSUDMA_SIZE_SHIFT) | notify); + + return TEE_SUCCESS; +} diff --git a/optee_os/core/drivers/zynqmp_huk.c b/optee_os/core/drivers/zynqmp_huk.c new file mode 100644 index 0000000..bb6727c --- /dev/null +++ b/optee_os/core/drivers/zynqmp_huk.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 Foundries.io Ltd. + * Jorge Ramirez-Ortiz + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct { + uint8_t key[HW_UNIQUE_KEY_LENGTH]; + bool ready; +} huk; + +__weak TEE_Result tee_zynqmp_get_device_dna(uint8_t *device_dna, size_t size) +{ + if (size != ZYNQMP_EFUSE_LEN(DNA)) + return TEE_ERROR_BAD_PARAMETERS; + + /* Get Device DNA from the PS eFuses */ + return zynqmp_efuse_read(device_dna, size, DNA, false); +} + +/* + * Generate HUK source data + * + * Performs SHA256 over of data: + * - Device DNA (from PL preferably) + * - Selected user eFuses (HUK seed) + * + * HUK source data is later on AES encrypted with device key to shuffle source + * data even further with secret key. + * + * Note: Even though the device key is secret used details for HUK source data + * should not be exposed to REE environment. + * + * Note: You should not change HUK source data generation parameters after + * devices have been deployed. + * + * @device_dna: Value of Device DNA + * @device_dna_size: Size of Device DNA + * @huk_source: Output buffer for HUK source data + * @huk_source_size: Output buffer size for HUK source data + * Return a TEE_Result compliant status + */ +static TEE_Result tee_zynqmp_generate_huk_src(const uint8_t *device_dna, + size_t device_dna_size, + uint8_t *huk_source, + size_t huk_source_size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t user_efuse = 0; + void *ctx = NULL; + int i = 0; + + assert(device_dna_size == ZYNQMP_EFUSE_LEN(DNA)); + assert(huk_source_size == HW_UNIQUE_KEY_LENGTH); + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SHA256); + if (res) + return res; + + res = crypto_hash_init(ctx); + if (res) + goto out; + + res = crypto_hash_update(ctx, device_dna, device_dna_size); + if (res) + goto out; + + /* Hash selected user eFuses */ + for (i = 0; i < (USER7 - USER0 + 1); i++) { + if (CFG_ZYNQMP_HUK_USER_EFUSE_MASK & BIT(i)) { + DMSG("Use User eFuse %d for HUK source data", i); + + res = zynqmp_efuse_read((uint8_t *)&user_efuse, + sizeof(user_efuse), USER0 + i, + false); + if (res) + goto out; + + res = crypto_hash_update(ctx, (uint8_t *)&user_efuse, + sizeof(user_efuse)); + if (res) + goto out; + } + } + + res = crypto_hash_final(ctx, huk_source, huk_source_size); +out: + crypto_hash_free_ctx(ctx); + memzero_explicit(&user_efuse, sizeof(user_efuse)); + return res; +} + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + vaddr_t csu = core_mmu_get_va(CSU_BASE, MEM_AREA_IO_SEC, CSU_SIZE); + uint8_t device_dna[ZYNQMP_EFUSE_LEN(DNA)] = { 0 }; + uint8_t src[HW_UNIQUE_KEY_LENGTH] __aligned_csuaes = { 0 }; + uint8_t iv[ZYNQMP_GCM_IV_SIZE] = { 0 }; + uint8_t tag[ZYNQMP_GCM_TAG_SIZE] __aligned_csuaes = { 0 }; + uint8_t sha[HW_UNIQUE_KEY_LENGTH] = { 0 }; + uint8_t dst[ZYNQMP_CSU_AES_DST_LEN(sizeof(src))] + __aligned_csuaes = { 0 }; + TEE_Result ret = TEE_ERROR_GENERIC; + uint32_t status = 0; + + static_assert(sizeof(device_dna) == ZYNQMP_GCM_IV_SIZE); + + if (huk.ready) + goto out; + + ret = tee_zynqmp_get_device_dna(device_dna, sizeof(device_dna)); + if (ret) { + EMSG("Can't read the Device DNA"); + goto cleanup; + } + + status = io_read32(csu + ZYNQMP_CSU_STATUS_OFFSET); + if (!(status & ZYNQMP_CSU_STATUS_AUTH)) { + /* The DNA is a unique identifier valid but not secure */ + IMSG("CSU authentication disabled, using development HUK"); + + /* Use hash of device DNA for development HUK */ + ret = tee_hash_createdigest(TEE_ALG_SHA256, device_dna, + sizeof(device_dna), huk.key, + sizeof(huk.key)); + if (ret) { + EMSG("Can't generate the SHA256 for the DNA eFuse"); + goto cleanup; + } + + huk.ready = true; + goto out; + } + + /* Use device DNA for IV */ + memcpy(iv, device_dna, sizeof(device_dna)); + + /* Generate HUK source data */ + ret = tee_zynqmp_generate_huk_src(device_dna, sizeof(device_dna), src, + sizeof(src)); + if (ret) { + EMSG("Failed to generate HUK source data"); + goto cleanup; + } + +#ifdef CFG_ZYNQMP_CSU_PUF + /* + * Neither the PMUFW nor the PUF hardware provide an indication of the + * PUF KEK registration status. The verification algorithm that follows + * encrypts and then decrypts the resulting string regenerating the + * PUF KEK in between: if the outputs match, then the PUF KEK was + * registered properly and we can use it to generate the HUK. + */ + zynqmp_csu_puf_reset(); + + ret = zynqmp_csu_puf_regenerate(); + if (ret) { + EMSG("PUF regeneration error"); + goto cleanup; + } +#endif + + memcpy(sha, src, sizeof(sha)); + /* The dst buffer must be large enough to include the generated tag */ + ret = zynqmp_csu_aes_encrypt_data(src, sizeof(src), dst, sizeof(dst), + tag, sizeof(tag), iv, sizeof(iv), + ZYNQMP_CSU_AES_KEY_SRC_DEV); + if (ret) { + EMSG("Can't encrypt DNA, please make sure PUF was registered"); + goto cleanup; + } + +#ifdef CFG_ZYNQMP_CSU_PUF + /* regenerate the PUF KEK */ + ret = zynqmp_csu_puf_regenerate(); + if (ret) { + EMSG("PUF regeneration error"); + goto cleanup; + } +#endif + memset(src, 0, sizeof(src)); + /* Ignore the tag data from the dst buffer - pass a smaller size */ + ret = zynqmp_csu_aes_decrypt_data(dst, sizeof(src), src, sizeof(src), + tag, sizeof(tag), iv, + ZYNQMP_EFUSE_LEN(DNA), + ZYNQMP_CSU_AES_KEY_SRC_DEV); + if (ret) { + EMSG("Can't decrypt DNA, please make sure PUF was registered"); + goto cleanup; + } + + if (memcmp(src, sha, sizeof(sha))) { + EMSG("PUF not ready, can't create HUK"); + ret = TEE_ERROR_GENERIC; + goto cleanup; + } + + IMSG("HUK ready"); + + /* + * The HUK is the SHA-256 of Device DNA with optional User eFuses + * included and then AES-GCM encrypted with the selected Device Key + * using the Device DNA as the IV. + */ + memcpy(huk.key, dst, sizeof(huk.key)); + huk.ready = true; +out: + memcpy(hwkey->data, huk.key, HW_UNIQUE_KEY_LENGTH); + ret = TEE_SUCCESS; + +cleanup: + /* Cleanup stack memory so that there are no left overs */ + memzero_explicit(dst, sizeof(dst)); + memzero_explicit(sha, sizeof(sha)); + memzero_explicit(tag, sizeof(tag)); + memzero_explicit(iv, sizeof(iv)); + memzero_explicit(src, sizeof(src)); + memzero_explicit(device_dna, sizeof(device_dna)); + + return ret; +} diff --git a/optee_os/core/drivers/zynqmp_pm.c b/optee_os/core/drivers/zynqmp_pm.c new file mode 100644 index 0000000..46c8d51 --- /dev/null +++ b/optee_os/core/drivers/zynqmp_pm.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2021 Foundries.io Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * For additional details about ZynqMP specific SMC ID's and PM request + * handling in TF-A check + * https://xilinx-wiki.atlassian.net/wiki/spaces/A/pages/18842107/Arm+Trusted+Firmware + */ +#define EFUSE_ACCESS_SMC 0xC2000035 +#define VERSION_ACCESS_SMC 0xC2000018 + +#define EFUSE_NOT_ENABLED 29 +#define VERSION_MASK GENMASK_32(3, 0) + +static uint32_t zynqmp_sip_call(uint32_t pm_api_id, uint32_t arg0, + uint32_t arg1, uint32_t arg2, uint32_t arg3, + uint32_t *payload) +{ + struct thread_smc_args args = { + .a0 = pm_api_id, + .a1 = reg_pair_to_64(arg1, arg0), + .a2 = reg_pair_to_64(arg3, arg2), + }; + + thread_smccc(&args); + + if (payload) { + switch (pm_api_id) { + case EFUSE_ACCESS_SMC: + *payload = args.a0 >> 32; + break; + case VERSION_ACCESS_SMC: + *payload = args.a1 & VERSION_MASK; + break; + default: + break; + } + } + + return args.a0; +} + +/* + * Stores all required details to read/write eFuse memory. + * @src: Physical address of the buffer to store the data to be + * written/read (in LE) + * @size: number of 32-bit words to be read/written + * @offset: offset in bytes to be read from/written to + * @flag: EFUSE_READ - represents eFuse read operation + * EFUSE_WRITE - represents eFuse write operation + * @pufuserfuse:0 - represents non-PUF eFuses, offset is used for read/write + * 1 - represents PUF user eFuse row number. + */ +struct xilinx_efuse { + uint64_t src; + uint32_t size; + uint32_t offset; + uint32_t flag; + uint32_t pufuserfuse; +}; + +enum efuse_op { EFUSE_READ = 0, EFUSE_WRITE = 1 }; + +#define EFUSE_ELT(__x) \ + [__x] = { \ + .offset = ZYNQMP_EFUSE_##__x##_OFFSET, \ + .bytes = ZYNQMP_EFUSE_##__x##_LENGTH, \ + } + +static const struct { + uint32_t offset; + uint32_t bytes; +} efuse_tbl[] = { + EFUSE_ELT(DNA), + EFUSE_ELT(IP_DISABLE), + EFUSE_ELT(USER0), + EFUSE_ELT(USER1), + EFUSE_ELT(USER2), + EFUSE_ELT(USER3), + EFUSE_ELT(USER4), + EFUSE_ELT(USER5), + EFUSE_ELT(USER6), + EFUSE_ELT(USER7), + EFUSE_ELT(MISC_USER_CTRL), + EFUSE_ELT(SEC_CTRL), +}; + +static TEE_Result efuse_op(enum efuse_op op, uint8_t *buf, size_t buf_sz, + enum zynqmp_efuse_id id, bool puf) +{ + struct xilinx_efuse *efuse_op = NULL; + uint8_t *tmpbuf = NULL; + paddr_t addr = 0; + uint32_t efuse_ret = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + if (!buf) + return TEE_ERROR_BAD_PARAMETERS; + + if (id >= ARRAY_SIZE(efuse_tbl)) { + EMSG("Invalid efuse"); + return TEE_ERROR_BAD_PARAMETERS; + } + + efuse_op = alloc_cache_aligned(sizeof(*efuse_op)); + if (!efuse_op) { + EMSG("Failed to allocate cache aligned buffer for operation"); + return TEE_ERROR_OUT_OF_MEMORY; + } + + tmpbuf = alloc_cache_aligned(buf_sz); + if (!tmpbuf) { + EMSG("Failed to allocate cache aligned buffer for data"); + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + if (op == EFUSE_WRITE) + memcpy(tmpbuf, buf, buf_sz); + + efuse_op->size = efuse_tbl[id].bytes / sizeof(uint32_t); + efuse_op->offset = efuse_tbl[id].offset; + efuse_op->src = virt_to_phys(tmpbuf); + efuse_op->pufuserfuse = puf; + efuse_op->flag = op; + + cache_operation(TEE_CACHECLEAN, tmpbuf, buf_sz); + cache_operation(TEE_CACHECLEAN, efuse_op, sizeof(*efuse_op)); + + addr = virt_to_phys(efuse_op); + + efuse_ret = zynqmp_sip_call(EFUSE_ACCESS_SMC, addr >> 32, addr, 0, 0, + NULL); + if (efuse_ret) { + if (efuse_ret == EFUSE_NOT_ENABLED) + EMSG("eFuse access is not enabled"); + else + EMSG("Error in eFuse access %#"PRIx32, efuse_ret); + res = TEE_ERROR_GENERIC; + goto out; + } + + if (op == EFUSE_READ) { + res = cache_operation(TEE_CACHEINVALIDATE, tmpbuf, buf_sz); + if (res) + goto out; + memcpy(buf, tmpbuf, buf_sz); + } + + res = TEE_SUCCESS; + +out: + free(tmpbuf); + free(efuse_op); + return res; +} + +TEE_Result zynqmp_efuse_read(uint8_t *buf, size_t sz, enum zynqmp_efuse_id id, + bool puf) +{ + return efuse_op(EFUSE_READ, buf, sz, id, puf); +} + +TEE_Result zynqmp_efuse_write(uint8_t *buf, size_t sz, enum zynqmp_efuse_id id, + bool puf) +{ + return efuse_op(EFUSE_WRITE, buf, sz, id, puf); +} + +TEE_Result zynqmp_soc_version(uint32_t *version) +{ + uint32_t res = 0; + + if (!version) + return TEE_ERROR_BAD_PARAMETERS; + + res = zynqmp_sip_call(VERSION_ACCESS_SMC, 0, 0, 0, 0, version); + if (res) { + EMSG("Failed to retrieve version"); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/include/bench.h b/optee_os/core/include/bench.h new file mode 100644 index 0000000..cdff577 --- /dev/null +++ b/optee_os/core/include/bench.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef BENCH_H +#define BENCH_H + +#include +#include +#include +#include + +/* + * Cycle count divider is enabled (in PMCR), + * CCNT value is incremented every 64th clock cycle + */ +#define TEE_BENCH_DIVIDER U(64) + +/* Max amount of timestamps per buffer */ +#define TEE_BENCH_MAX_STAMPS U(32) +#define TEE_BENCH_MAX_MASK (TEE_BENCH_MAX_STAMPS - 1) + +#define OPTEE_MSG_RPC_CMD_BENCH_REG_NEW U(0) +#define OPTEE_MSG_RPC_CMD_BENCH_REG_DEL U(1) + +/* OP-TEE susbsystems ids */ +#define TEE_BENCH_CLIENT U(0x10000000) +#define TEE_BENCH_KMOD U(0x20000000) +#define TEE_BENCH_CORE U(0x30000000) +#define TEE_BENCH_UTEE U(0x40000000) +#define TEE_BENCH_DUMB_TA U(0xF0000001) + +/* storing timestamp */ +struct tee_time_st { + uint64_t cnt; /* stores value from CNTPCT register */ + uint64_t addr; /* stores value from program counter register */ + uint64_t src; /* OP-TEE subsystem id */ +}; + +/* per-cpu circular buffer for timestamps */ +struct tee_ts_cpu_buf { + uint64_t head; + uint64_t tail; + struct tee_time_st stamps[TEE_BENCH_MAX_STAMPS]; +}; + +/* memory layout for shared memory, where timestamps will be stored */ +struct tee_ts_global { + uint64_t cores; + struct tee_ts_cpu_buf cpu_buf[]; +}; + +#ifdef CFG_TEE_BENCHMARK +void bm_timestamp(void); +#else +static inline void bm_timestamp(void) {} +#endif /* CFG_TEE_BENCHMARK */ + +#endif /* BENCH_H */ diff --git a/optee_os/core/include/console.h b/optee_os/core/include/console.h new file mode 100644 index 0000000..f2e57fb --- /dev/null +++ b/optee_os/core/include/console.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef CONSOLE_H +#define CONSOLE_H + +#include +#include + + +void console_init(void); +void console_putc(int ch); +void console_flush(void); + +struct serial_chip; +void register_serial_console(struct serial_chip *chip); + +#ifdef CFG_DT +/* + * Get console info from a reacheable DTB. Check the embedded DTB and fall + * back to the external DTB. + * + * If the DTB does not specify a chosen (or secure-chosen) node, we assume + * DTB does not provide specific console directive. Early console may remain. + * If the DTB does not specify any console in the chosen (or secure-chosen) + * node, we assume there is no console. Early console would be disabled. + * + * @fdt_out: Output DTB address where console directive is found + * @offs_out: Output offset in the DTB where console directive is found + * @path_out: Output string configuration of the console from the DTB. + * (*path_out) shall be freed using nex_free(). + * @params_out: Output console parameters found from the DTB. + * (*params_out) shall be freed using nex_free(). + * + * Return a TEE_Result compliant return value + * + */ +TEE_Result get_console_node_from_dt(void *fdt, int *offs_out, + char **path_out, char **params_out); + +/* + * Check if the /secure-chosen or /chosen node in the DT contains an + * stdout-path value for which we have a compatible driver. If so, switch + * the console to this device. + */ +void configure_console_from_dt(void); +#else +static inline void configure_console_from_dt(void) +{} +#endif /* !CFG_DT */ + +#endif /* CONSOLE_H */ + diff --git a/optee_os/core/include/crypto/crypto.h b/optee_os/core/include/crypto/crypto.h new file mode 100644 index 0000000..d0ee5f3 --- /dev/null +++ b/optee_os/core/include/crypto/crypto.h @@ -0,0 +1,425 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2017, Linaro Limited + */ + +/* + * This is the Cryptographic Provider API (CP API). + * + * This defines how most crypto syscalls that implement the Cryptographic + * Operations API can invoke the actual providers of cryptographic algorithms + * (such as LibTomCrypt). + * + * To add a new provider, you need to provide an implementation of this + * interface. + * + * The following parameters are commonly used. + * + * @ctx: context allocated by the syscall, for later use by the algorithm + * @algo: algorithm identifier (TEE_ALG_*) + */ + +#ifndef __CRYPTO_CRYPTO_H +#define __CRYPTO_CRYPTO_H + +#include +#include + +TEE_Result crypto_init(void); + +/* Message digest functions */ +TEE_Result crypto_hash_alloc_ctx(void **ctx, uint32_t algo); +TEE_Result crypto_hash_init(void *ctx); +TEE_Result crypto_hash_update(void *ctx, const uint8_t *data, size_t len); +TEE_Result crypto_hash_final(void *ctx, uint8_t *digest, size_t len); +void crypto_hash_free_ctx(void *ctx); +void crypto_hash_copy_state(void *dst_ctx, void *src_ctx); + +/* Symmetric ciphers */ +TEE_Result crypto_cipher_alloc_ctx(void **ctx, uint32_t algo); +TEE_Result crypto_cipher_init(void *ctx, TEE_OperationMode mode, + const uint8_t *key1, size_t key1_len, + const uint8_t *key2, size_t key2_len, + const uint8_t *iv, size_t iv_len); +TEE_Result crypto_cipher_update(void *ctx, TEE_OperationMode mode, + bool last_block, const uint8_t *data, + size_t len, uint8_t *dst); +void crypto_cipher_final(void *ctx); +TEE_Result crypto_cipher_get_block_size(uint32_t algo, size_t *size); +void crypto_cipher_free_ctx(void *ctx); +void crypto_cipher_copy_state(void *dst_ctx, void *src_ctx); + +/* Message Authentication Code functions */ +TEE_Result crypto_mac_alloc_ctx(void **ctx, uint32_t algo); +TEE_Result crypto_mac_init(void *ctx, const uint8_t *key, size_t len); +TEE_Result crypto_mac_update(void *ctx, const uint8_t *data, size_t len); +TEE_Result crypto_mac_final(void *ctx, uint8_t *digest, size_t digest_len); +void crypto_mac_free_ctx(void *ctx); +void crypto_mac_copy_state(void *dst_ctx, void *src_ctx); + +/* Authenticated encryption */ +TEE_Result crypto_authenc_alloc_ctx(void **ctx, uint32_t algo); +TEE_Result crypto_authenc_init(void *ctx, TEE_OperationMode mode, + const uint8_t *key, size_t key_len, + const uint8_t *nonce, size_t nonce_len, + size_t tag_len, size_t aad_len, + size_t payload_len); +TEE_Result crypto_authenc_update_aad(void *ctx, TEE_OperationMode mode, + const uint8_t *data, size_t len); +TEE_Result crypto_authenc_update_payload(void *ctx, TEE_OperationMode mode, + const uint8_t *src_data, + size_t src_len, uint8_t *dst_data, + size_t *dst_len); +TEE_Result crypto_authenc_enc_final(void *ctx, const uint8_t *src_data, + size_t src_len, uint8_t *dst_data, + size_t *dst_len, uint8_t *dst_tag, + size_t *dst_tag_len); +TEE_Result crypto_authenc_dec_final(void *ctx, const uint8_t *src_data, + size_t src_len, uint8_t *dst_data, + size_t *dst_len, const uint8_t *tag, + size_t tag_len); +void crypto_authenc_final(void *ctx); +void crypto_authenc_free_ctx(void *ctx); +void crypto_authenc_copy_state(void *dst_ctx, void *src_ctx); + +/* Informs crypto that the data in the buffer will be removed from storage */ +TEE_Result crypto_storage_obj_del(struct tee_obj *obj); + +/* Implementation-defined big numbers */ + +/* + * Allocate a bignum capable of holding an unsigned integer value of + * up to bitsize bits + */ +struct bignum *crypto_bignum_allocate(size_t size_bits); +TEE_Result crypto_bignum_bin2bn(const uint8_t *from, size_t fromsize, + struct bignum *to); +size_t crypto_bignum_num_bytes(struct bignum *a); +size_t crypto_bignum_num_bits(struct bignum *a); +void crypto_bignum_bn2bin(const struct bignum *from, uint8_t *to); +void crypto_bignum_copy(struct bignum *to, const struct bignum *from); +void crypto_bignum_free(struct bignum **a); +void crypto_bignum_clear(struct bignum *a); + +/* return -1 if ab */ +int32_t crypto_bignum_compare(struct bignum *a, struct bignum *b); + +/* Asymmetric algorithms */ + +struct rsa_keypair { + struct bignum *e; /* Public exponent */ + struct bignum *d; /* Private exponent */ + struct bignum *n; /* Modulus */ + + /* Optional CRT parameters (all NULL if unused) */ + struct bignum *p; /* N = pq */ + struct bignum *q; + struct bignum *qp; /* 1/q mod p */ + struct bignum *dp; /* d mod (p-1) */ + struct bignum *dq; /* d mod (q-1) */ +}; + +struct rsa_public_key { + struct bignum *e; /* Public exponent */ + struct bignum *n; /* Modulus */ +}; + +struct dsa_keypair { + struct bignum *g; /* Generator of subgroup (public) */ + struct bignum *p; /* Prime number (public) */ + struct bignum *q; /* Order of subgroup (public) */ + struct bignum *y; /* Public key */ + struct bignum *x; /* Private key */ +}; + +struct dsa_public_key { + struct bignum *g; /* Generator of subgroup (public) */ + struct bignum *p; /* Prime number (public) */ + struct bignum *q; /* Order of subgroup (public) */ + struct bignum *y; /* Public key */ +}; + +struct dh_keypair { + struct bignum *g; /* Generator of Z_p (shared) */ + struct bignum *p; /* Prime modulus (shared) */ + struct bignum *x; /* Private key */ + struct bignum *y; /* Public key y = g^x */ + + /* + * Optional parameters used by key generation. + * When not used, q == NULL and xbits == 0 + */ + struct bignum *q; /* x must be in the range [2, q-2] */ + uint32_t xbits; /* Number of bits in the private key */ +}; + +struct ecc_public_key { + struct bignum *x; /* Public value x */ + struct bignum *y; /* Public value y */ + uint32_t curve; /* Curve type */ + const struct crypto_ecc_public_ops *ops; /* Key Operations */ +}; + +struct ecc_keypair { + struct bignum *d; /* Private value */ + struct bignum *x; /* Public value x */ + struct bignum *y; /* Public value y */ + uint32_t curve; /* Curve type */ + const struct crypto_ecc_keypair_ops *ops; /* Key Operations */ +}; + +struct montgomery_keypair { + uint8_t *priv; /* Private value */ + uint8_t *pub; /* Public value */ +}; + +struct ed25519_keypair { + uint8_t *priv; + uint8_t *pub; + uint32_t curve; +}; + +struct ed25519_public_key { + uint8_t *pub; + uint32_t curve; +}; + +/* + * Key allocation functions + * Allocate the bignum's inside a key structure. + * TEE core will later use crypto_bignum_free(). + */ +TEE_Result crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *s, + size_t key_size_bits); +TEE_Result crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *s, + size_t key_size_bits); +void crypto_acipher_free_rsa_public_key(struct rsa_public_key *s); +void crypto_acipher_free_rsa_keypair(struct rsa_keypair *s); +TEE_Result crypto_acipher_alloc_dsa_keypair(struct dsa_keypair *s, + size_t key_size_bits); +TEE_Result crypto_acipher_alloc_dsa_public_key(struct dsa_public_key *s, + size_t key_size_bits); +TEE_Result crypto_acipher_alloc_dh_keypair(struct dh_keypair *s, + size_t key_size_bits); +TEE_Result crypto_acipher_alloc_ecc_public_key(struct ecc_public_key *s, + uint32_t key_type, + size_t key_size_bits); +TEE_Result crypto_acipher_alloc_ecc_keypair(struct ecc_keypair *s, + uint32_t key_type, + size_t key_size_bits); +void crypto_acipher_free_ecc_public_key(struct ecc_public_key *s); +TEE_Result crypto_acipher_alloc_x25519_keypair(struct montgomery_keypair *s, + size_t key_size_bits); +TEE_Result crypto_acipher_alloc_x448_keypair(struct montgomery_keypair *s, + size_t key_size_bits); +TEE_Result crypto_acipher_alloc_ed25519_keypair(struct ed25519_keypair *s, + size_t key_size_bits); +TEE_Result +crypto_acipher_alloc_ed25519_public_key(struct ed25519_public_key *key, + size_t key_size); + +/* + * Key generation functions + */ +TEE_Result crypto_acipher_gen_rsa_key(struct rsa_keypair *key, size_t key_size); +TEE_Result crypto_acipher_gen_dsa_key(struct dsa_keypair *key, size_t key_size); +TEE_Result crypto_acipher_gen_dh_key(struct dh_keypair *key, struct bignum *q, + size_t xbits, size_t key_size); +TEE_Result crypto_acipher_gen_ecc_key(struct ecc_keypair *key, size_t key_size); +TEE_Result crypto_acipher_gen_x25519_key(struct montgomery_keypair *key, + size_t key_size); +TEE_Result crypto_acipher_gen_x448_key(struct montgomery_keypair *key, + size_t key_size); +TEE_Result crypto_acipher_gen_ed25519_key(struct ed25519_keypair *key, + size_t key_size); +TEE_Result crypto_acipher_ed25519_sign(struct ed25519_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len); +TEE_Result crypto_acipher_ed25519ctx_sign(struct ed25519_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len, + bool ph_flag, + const uint8_t *ctx, size_t ctxlen); +TEE_Result crypto_acipher_ed25519_verify(struct ed25519_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len); +TEE_Result crypto_acipher_ed25519ctx_verify(struct ed25519_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len, + bool ph_flag, + const uint8_t *ctx, size_t ctxlen); + +TEE_Result crypto_acipher_dh_shared_secret(struct dh_keypair *private_key, + struct bignum *public_key, + struct bignum *secret); + +TEE_Result crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len); +TEE_Result crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len); +TEE_Result crypto_acipher_rsaes_decrypt(uint32_t algo, struct rsa_keypair *key, + const uint8_t *label, size_t label_len, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len); +TEE_Result crypto_acipher_rsaes_encrypt(uint32_t algo, + struct rsa_public_key *key, + const uint8_t *label, size_t label_len, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len); +/* RSA SSA sign/verify: if salt_len == -1, use default value */ +TEE_Result crypto_acipher_rsassa_sign(uint32_t algo, struct rsa_keypair *key, + int salt_len, const uint8_t *msg, + size_t msg_len, uint8_t *sig, + size_t *sig_len); +TEE_Result crypto_acipher_rsassa_verify(uint32_t algo, + struct rsa_public_key *key, + int salt_len, const uint8_t *msg, + size_t msg_len, const uint8_t *sig, + size_t sig_len); +TEE_Result crypto_acipher_dsa_sign(uint32_t algo, struct dsa_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len); +TEE_Result crypto_acipher_dsa_verify(uint32_t algo, struct dsa_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len); +TEE_Result crypto_acipher_ecc_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len); +TEE_Result crypto_acipher_ecc_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len); +TEE_Result crypto_acipher_ecc_shared_secret(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, + unsigned long *secret_len); +TEE_Result crypto_acipher_sm2_pke_decrypt(struct ecc_keypair *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len); +TEE_Result crypto_acipher_sm2_pke_encrypt(struct ecc_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len); +TEE_Result crypto_acipher_x25519_shared_secret(struct montgomery_keypair + *private_key, + void *public_key, void *secret, + unsigned long *secret_len); +TEE_Result crypto_acipher_x448_shared_secret(struct montgomery_keypair + *private_key, + void *public_key, void *secret, + unsigned long *secret_len); + +struct sm2_kep_parms { + uint8_t *out; + size_t out_len; + bool is_initiator; + const uint8_t *initiator_id; + size_t initiator_id_len; + const uint8_t *responder_id; + size_t responder_id_len; + const uint8_t *conf_in; + size_t conf_in_len; + uint8_t *conf_out; + size_t conf_out_len; +}; + +TEE_Result crypto_acipher_sm2_kep_derive(struct ecc_keypair *my_key, + struct ecc_keypair *my_eph_key, + struct ecc_public_key *peer_key, + struct ecc_public_key *peer_eph_key, + struct sm2_kep_parms *p); + +/* + * Verifies a SHA-256 hash, doesn't require crypto_init() to be called in + * advance and has as few dependencies as possible. + * + * This function is primarily used by pager and early initialization code + * where the complete crypto library isn't available. + */ +TEE_Result hash_sha256_check(const uint8_t *hash, const uint8_t *data, + size_t data_size); + +/* + * Computes a SHA-512/256 hash, vetted conditioner as per NIST.SP.800-90B. + * It doesn't require crypto_init() to be called in advance and has as few + * dependencies as possible. + * + * This function could be used inside interrupt context where the crypto + * library can't be used due to mutex handling. + */ +TEE_Result hash_sha512_256_compute(uint8_t *digest, const uint8_t *data, + size_t data_size); + +#define CRYPTO_RNG_SRC_IS_QUICK(sid) (!!((sid) & 1)) + +/* + * enum crypto_rng_src - RNG entropy source + * + * Identifiers for different RNG entropy sources. The lowest bit indicates + * if the source is to be merely queued (bit is 1) or if it's delivered + * directly to the pool. The difference is that in the latter case RPC to + * normal world can be performed and in the former it must not. + */ +enum crypto_rng_src { + CRYPTO_RNG_SRC_JITTER_SESSION = (0 << 1 | 0), + CRYPTO_RNG_SRC_JITTER_RPC = (1 << 1 | 1), + CRYPTO_RNG_SRC_NONSECURE = (1 << 1 | 0), +}; + +/* + * crypto_rng_init() - initialize the RNG + * @data: buffer with initial seed + * @dlen: length of @data + */ +TEE_Result crypto_rng_init(const void *data, size_t dlen); + +/* + * crypto_rng_add_event() - supply entropy to RNG from a source + * @sid: Source identifier, should be unique for a specific source + * @pnum: Pool number, acquired using crypto_rng_get_next_pool_num() + * @data: Data associated with the event + * @dlen: Length of @data + * + * @sid controls whether the event is merly queued in a ring buffer or if + * it's added to one of the pools directly. If CRYPTO_RNG_SRC_IS_QUICK() is + * true (lowest bit set) events are queue otherwise added to corresponding + * pool. If CRYPTO_RNG_SRC_IS_QUICK() is false, eventual queued events are + * added to their queues too. + */ +void crypto_rng_add_event(enum crypto_rng_src sid, unsigned int *pnum, + const void *data, size_t dlen); + +/* + * crypto_rng_read() - read cryptograhically secure RNG + * @buf: Buffer to hold the data + * @len: Length of buffer. + * + * Eventual queued events are also added to their pools during this + * function call. + */ +TEE_Result crypto_rng_read(void *buf, size_t len); + +/* + * crypto_aes_expand_enc_key() - Expand an AES key + * @key: AES key buffer + * @key_len: Size of the @key buffer in bytes + * @enc_key: Expanded AES encryption key buffer + * @enc_keylen: Size of the @enc_key buffer in bytes + * @rounds: Number of rounds to be used during encryption + */ +TEE_Result crypto_aes_expand_enc_key(const void *key, size_t key_len, + void *enc_key, size_t enc_keylen, + unsigned int *rounds); + +/* + * crypto_aes_enc_block() - Encrypt an AES block + * @enc_key: Expanded AES encryption key + * @enc_keylen: Size of @enc_key in bytes + * @rounds: Number of rounds + * @src: Source buffer of one AES block (16 bytes) + * @dst: Destination buffer of one AES block (16 bytes) + */ +void crypto_aes_enc_block(const void *enc_key, size_t enc_keylen, + unsigned int rounds, const void *src, void *dst); + +#endif /* __CRYPTO_CRYPTO_H */ diff --git a/optee_os/core/include/crypto/crypto_accel.h b/optee_os/core/include/crypto/crypto_accel.h new file mode 100644 index 0000000..c6a043f --- /dev/null +++ b/optee_os/core/include/crypto/crypto_accel.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020-2023, Linaro Limited + */ + +#ifndef __CRYPTO_CRYPTO_ACCEL_H +#define __CRYPTO_CRYPTO_ACCEL_H + +#include + +TEE_Result crypto_accel_aes_expand_keys(const void *key, size_t key_len, + void *enc_key, void *dec_key, + size_t expanded_key_len, + unsigned int *round_count); + +void crypto_accel_aes_ecb_enc(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count); +void crypto_accel_aes_ecb_dec(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count); + +void crypto_accel_aes_cbc_enc(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count, void *iv); +void crypto_accel_aes_cbc_dec(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count, void *iv); + +void crypto_accel_aes_ctr_be_enc(void *out, const void *in, const void *key, + unsigned int round_count, + unsigned int block_count, void *iv); + +void crypto_accel_aes_xts_enc(void *out, const void *in, const void *key1, + unsigned int round_count, + unsigned int block_count, const void *key2, + void *tweak); +void crypto_accel_aes_xts_dec(void *out, const void *in, const void *key1, + unsigned int round_count, + unsigned int block_count, const void *key2, + void *tweak); + +void crypto_accel_sha1_compress(uint32_t state[5], const void *src, + unsigned int block_count); +void crypto_accel_sha256_compress(uint32_t state[8], const void *src, + unsigned int block_count); +void crypto_accel_sha512_compress(uint64_t state[8], const void *src, + unsigned int block_count); +void crypto_accel_sha3_compress(uint64_t state[25], const void *src, + unsigned int block_count, + unsigned int digest_size); +void crypto_accel_sm3_compress(uint32_t state[8], const void *src, + unsigned int block_count); + +void crypto_accel_sm4_setkey_enc(uint32_t sk[32], const uint8_t key[16]); +void crypto_accel_sm4_setkey_dec(uint32_t sk[32], const uint8_t key[16]); +void crypto_accel_sm4_ecb_enc(void *out, const void *in, const void *key, + unsigned int len); +void crypto_accel_sm4_cbc_enc(void *out, const void *in, const void *key, + unsigned int len, void *iv); +void crypto_accel_sm4_cbc_dec(void *out, const void *in, const void *key, + unsigned int len, void *iv); +void crypto_accel_sm4_ctr_enc(void *out, const void *in, const void *key, + unsigned int len, void *iv); +void crypto_accel_sm4_xts_enc(void *out, const void *in, const void *key1, + const void *key2, unsigned int len, void *iv); +void crypto_accel_sm4_xts_dec(void *out, const void *in, const void *key1, + const void *key2, unsigned int len, void *iv); + +#endif /*__CRYPTO_CRYPTO_ACCEL_H*/ diff --git a/optee_os/core/include/crypto/crypto_impl.h b/optee_os/core/include/crypto/crypto_impl.h new file mode 100644 index 0000000..ccff72c --- /dev/null +++ b/optee_os/core/include/crypto/crypto_impl.h @@ -0,0 +1,556 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2021, SumUp Services GmbH + */ + +#ifndef __CRYPTO_CRYPTO_IMPL_H +#define __CRYPTO_CRYPTO_IMPL_H + +#include +#include + +/* + * The crypto context used by the crypto_hash_*() functions is defined by + * struct crypto_hash_ctx. + */ +struct crypto_hash_ctx { + const struct crypto_hash_ops *ops; +}; + +struct crypto_hash_ops { + TEE_Result (*init)(struct crypto_hash_ctx *ctx); + TEE_Result (*update)(struct crypto_hash_ctx *ctx, const uint8_t *data, + size_t len); + TEE_Result (*final)(struct crypto_hash_ctx *ctx, uint8_t *digest, + size_t len); + void (*free_ctx)(struct crypto_hash_ctx *ctx); + void (*copy_state)(struct crypto_hash_ctx *dst_ctx, + struct crypto_hash_ctx *src_ctx); +}; + +#define CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(name, type) \ + static inline TEE_Result \ + crypto_##name##_alloc_ctx(struct crypto_##type##_ctx **ctx __unused) \ + { return TEE_ERROR_NOT_IMPLEMENTED; } + +#if defined(CFG_CRYPTO_MD5) +TEE_Result crypto_md5_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(md5, hash) +#endif + +#if defined(CFG_CRYPTO_SHA1) +TEE_Result crypto_sha1_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha1, hash) +#endif + +#if defined(CFG_CRYPTO_SHA224) +TEE_Result crypto_sha224_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha224, hash) +#endif + +#if defined(CFG_CRYPTO_SHA256) +TEE_Result crypto_sha256_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha256, hash) +#endif + +#if defined(CFG_CRYPTO_SHA384) +TEE_Result crypto_sha384_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha384, hash) +#endif + +#if defined(CFG_CRYPTO_SHA512) +TEE_Result crypto_sha512_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha512, hash) +#endif + +#if defined(CFG_CRYPTO_SM3) +TEE_Result crypto_sm3_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sm3, hash) +#endif + +#if defined(CFG_CRYPTO_SHA3_224) +TEE_Result crypto_sha3_224_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha3_224, hash) +#endif + +#if defined(CFG_CRYPTO_SHA3_256) +TEE_Result crypto_sha3_256_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha3_256, hash) +#endif + +#if defined(CFG_CRYPTO_SHA3_384) +TEE_Result crypto_sha3_384_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha3_384, hash) +#endif + +#if defined(CFG_CRYPTO_SHA3_512) +TEE_Result crypto_sha3_512_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sha3_512, hash) +#endif + +#if defined(CFG_CRYPTO_SHAKE128) +TEE_Result crypto_shake128_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(shake128, hash) +#endif + +#if defined(CFG_CRYPTO_SHAKE256) +TEE_Result crypto_shake256_alloc_ctx(struct crypto_hash_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(shake256, hash) +#endif + +/* + * The crypto context used by the crypto_mac_*() functions is defined by + * struct crypto_mac_ctx. + */ +struct crypto_mac_ctx { + const struct crypto_mac_ops *ops; +}; + +struct crypto_mac_ops { + TEE_Result (*init)(struct crypto_mac_ctx *ctx, const uint8_t *key, + size_t len); + TEE_Result (*update)(struct crypto_mac_ctx *ctx, const uint8_t *data, + size_t len); + TEE_Result (*final)(struct crypto_mac_ctx *ctx, uint8_t *digest, + size_t len); + void (*free_ctx)(struct crypto_mac_ctx *ctx); + void (*copy_state)(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx); +}; + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_MD5) +TEE_Result crypto_hmac_md5_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_md5, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA1) +TEE_Result crypto_hmac_sha1_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha1, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA224) +TEE_Result crypto_hmac_sha224_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha224, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA256) +TEE_Result crypto_hmac_sha256_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha256, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA384) +TEE_Result crypto_hmac_sha384_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha384, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA512) +TEE_Result crypto_hmac_sha512_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha512, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA3_224) +TEE_Result crypto_hmac_sha3_224_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha3_224, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA3_256) +TEE_Result crypto_hmac_sha3_256_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha3_256, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA3_384) +TEE_Result crypto_hmac_sha3_384_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha3_384, mac) +#endif + +#if defined(CFG_CRYPTO_HMAC) && defined(CFG_CRYPTO_SHA3_512) +TEE_Result crypto_hmac_sha3_512_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sha3_512, mac) +#endif + +#if defined(CFG_CRYPTO_SM3) && defined(CFG_CRYPTO_HMAC) +TEE_Result crypto_hmac_sm3_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(hmac_sm3, mac) +#endif + +#if defined(CFG_CRYPTO_CBC_MAC) +TEE_Result crypto_aes_cbc_mac_nopad_alloc_ctx(struct crypto_mac_ctx **ctx); +TEE_Result crypto_aes_cbc_mac_pkcs5_alloc_ctx(struct crypto_mac_ctx **ctx); +TEE_Result crypto_des_cbc_mac_nopad_alloc_ctx(struct crypto_mac_ctx **ctx); +TEE_Result crypto_des_cbc_mac_pkcs5_alloc_ctx(struct crypto_mac_ctx **ctx); +TEE_Result crypto_des3_cbc_mac_nopad_alloc_ctx(struct crypto_mac_ctx **ctx); +TEE_Result crypto_des3_cbc_mac_pkcs5_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_cbc_mac_nopad, mac) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_cbc_mac_pkcs5, mac) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des_cbc_mac_nopad, mac) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des_cbc_mac_pkcs5, mac) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des3_cbc_mac_nopad, mac) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des3_cbc_mac_pkcs5, mac) +#endif + +#if defined(CFG_CRYPTO_CMAC) +TEE_Result crypto_aes_cmac_alloc_ctx(struct crypto_mac_ctx **ctx); +TEE_Result crypto_des3_cmac_alloc_ctx(struct crypto_mac_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_cmac, mac) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des3_cmac, mac) +#endif + +/* + * The crypto context used by the crypto_cipher_*() functions is defined by + * struct crypto_cipher_ctx. + */ +struct crypto_cipher_ctx { + const struct crypto_cipher_ops *ops; +}; + +struct crypto_cipher_ops { + TEE_Result (*init)(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, + const uint8_t *key1, size_t key1_len, + const uint8_t *key2, size_t key2_len, + const uint8_t *iv, size_t iv_len); + TEE_Result (*update)(struct crypto_cipher_ctx *ctx, bool last_block, + const uint8_t *data, size_t len, uint8_t *dst); + void (*final)(struct crypto_cipher_ctx *ctx); + + void (*free_ctx)(struct crypto_cipher_ctx *ctx); + void (*copy_state)(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx); +}; + +#if defined(CFG_CRYPTO_AES) && defined(CFG_CRYPTO_ECB) +TEE_Result crypto_aes_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_ecb, cipher) +#endif + +#if defined(CFG_CRYPTO_AES) && defined(CFG_CRYPTO_CBC) +TEE_Result crypto_aes_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_cbc, cipher) +#endif + +#if defined(CFG_CRYPTO_AES) && defined(CFG_CRYPTO_CTR) +TEE_Result crypto_aes_ctr_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_ctr, cipher) +#endif + +#if defined(CFG_CRYPTO_AES) && defined(CFG_CRYPTO_CTS) +TEE_Result crypto_aes_cts_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_cts, cipher) +#endif + +#if defined(CFG_CRYPTO_AES) && defined(CFG_CRYPTO_XTS) +TEE_Result crypto_aes_xts_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(aes_xts, cipher) +#endif + +#if defined(CFG_CRYPTO_DES) && defined(CFG_CRYPTO_ECB) +TEE_Result crypto_des_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx); +TEE_Result crypto_des3_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des_ecb, cipher) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des3_ecb, cipher) +#endif + +#if defined(CFG_CRYPTO_DES) && defined(CFG_CRYPTO_CBC) +TEE_Result crypto_des_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx); +TEE_Result crypto_des3_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des_cbc, cipher) +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(des3_cbc, cipher) +#endif + +#if defined(CFG_CRYPTO_SM4) && defined(CFG_CRYPTO_ECB) +TEE_Result crypto_sm4_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sm4_ecb, cipher) +#endif + +#if defined(CFG_CRYPTO_SM4) && defined(CFG_CRYPTO_CBC) +TEE_Result crypto_sm4_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sm4_cbc, cipher) +#endif + +#if defined(CFG_CRYPTO_SM4) && defined(CFG_CRYPTO_CTR) +TEE_Result crypto_sm4_ctr_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sm4_ctr, cipher) +#endif + +#if defined(CFG_CRYPTO_SM4) && defined(CFG_CRYPTO_XTS) +TEE_Result crypto_sm4_xts_alloc_ctx(struct crypto_cipher_ctx **ctx); +#else +CRYPTO_ALLOC_CTX_NOT_IMPLEMENTED(sm4_xts, cipher) +#endif + +/* + * The crypto context used by the crypto_authen_*() functions below is + * defined by struct crypto_authenc_ctx. + */ +struct crypto_authenc_ctx { + const struct crypto_authenc_ops *ops; +}; + +struct crypto_authenc_ops { + TEE_Result (*init)(struct crypto_authenc_ctx *ctx, + TEE_OperationMode mode, + const uint8_t *key, size_t key_len, + const uint8_t *nonce, size_t nonce_len, + size_t tag_len, size_t aad_len, + size_t payload_len); + TEE_Result (*update_aad)(struct crypto_authenc_ctx *ctx, + const uint8_t *data, size_t len); + TEE_Result (*update_payload)(struct crypto_authenc_ctx *ctx, + TEE_OperationMode mode, + const uint8_t *src_data, size_t len, + uint8_t *dst_data); + TEE_Result (*enc_final)(struct crypto_authenc_ctx *ctx, + const uint8_t *src_data, size_t len, + uint8_t *dst_data, uint8_t *dst_tag, + size_t *dst_tag_len); + TEE_Result (*dec_final)(struct crypto_authenc_ctx *ctx, + const uint8_t *src_data, size_t len, + uint8_t *dst_data, const uint8_t *tag, + size_t tag_len); + void (*final)(struct crypto_authenc_ctx *ctx); + void (*free_ctx)(struct crypto_authenc_ctx *ctx); + void (*copy_state)(struct crypto_authenc_ctx *dst_ctx, + struct crypto_authenc_ctx *src_ctx); +}; + +TEE_Result crypto_aes_ccm_alloc_ctx(struct crypto_authenc_ctx **ctx); +TEE_Result crypto_aes_gcm_alloc_ctx(struct crypto_authenc_ctx **ctx); + +#ifdef CFG_CRYPTO_DRV_HASH +TEE_Result drvcrypt_hash_alloc_ctx(struct crypto_hash_ctx **ctx, uint32_t algo); +#else +static inline TEE_Result +drvcrypt_hash_alloc_ctx(struct crypto_hash_ctx **ctx __unused, + uint32_t algo __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /* CFG_CRYPTO_DRV_HASH */ + +#ifdef CFG_CRYPTO_DRV_CIPHER +TEE_Result drvcrypt_cipher_alloc_ctx(struct crypto_cipher_ctx **ctx, + uint32_t algo); +#else +static inline TEE_Result +drvcrypt_cipher_alloc_ctx(struct crypto_cipher_ctx **ctx __unused, + uint32_t algo __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /* CFG_CRYPTO_DRV_CIPHER */ + +#ifdef CFG_CRYPTO_DRV_MAC +/* Cryptographic MAC driver context allocation */ +TEE_Result drvcrypt_mac_alloc_ctx(struct crypto_mac_ctx **ctx, uint32_t algo); +#else +static inline TEE_Result +drvcrypt_mac_alloc_ctx(struct crypto_mac_ctx **ctx __unused, + uint32_t algo __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /* CFG_CRYPTO_DRV_MAC */ + +#ifdef CFG_CRYPTO_DRV_AUTHENC +/* Cryptographic Authenticated Encryption driver context allocation */ +TEE_Result drvcrypt_authenc_alloc_ctx(struct crypto_authenc_ctx **ctx, + uint32_t algo); +#else +static inline TEE_Result +drvcrypt_authenc_alloc_ctx(struct crypto_authenc_ctx **ctx __unused, + uint32_t algo __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /* CFG_CRYPTO_DRV_AUTHENC */ +/* + * The ECC public key operations used by the crypto_acipher_ecc_*() and + * crypto_acipher_free_ecc_*() functions. + * Reference set in ecc_public_key when key allocated. + * + * @free is mandatory + * @verify is optional + * @encrypt is optional + */ +struct crypto_ecc_public_ops { + void (*free)(struct ecc_public_key *key); + TEE_Result (*verify)(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len); + TEE_Result (*encrypt)(struct ecc_public_key *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len); +}; + +/* + * The ECC keypair operations used by the crypto_acipher_ecc_*() and + * crypto_acipher_gen_ecc_*() functions. + * Reference set in ecc_keypair when key allocated. + * + * @generate is mandatory + * @sign is optional + * @shared_secret is optional + * @decrypt is optional + */ +struct crypto_ecc_keypair_ops { + TEE_Result (*generate)(struct ecc_keypair *key, size_t key_size_bits); + TEE_Result (*sign)(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, uint8_t *sig, + size_t *sig_len); + TEE_Result (*shared_secret)(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, unsigned long *secret_len); + TEE_Result (*decrypt)(struct ecc_keypair *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len); +}; + +#ifdef CFG_CRYPTO_ECC +const struct crypto_ecc_keypair_ops * +crypto_asym_get_ecc_keypair_ops(uint32_t key_type); + +const struct crypto_ecc_public_ops * +crypto_asym_get_ecc_public_ops(uint32_t key_type); + +TEE_Result crypto_asym_alloc_ecc_public_key(struct ecc_public_key *key, + uint32_t key_type, + size_t key_size_bits); +TEE_Result crypto_asym_alloc_ecc_keypair(struct ecc_keypair *key, + uint32_t key_type, + size_t key_size_bits); +#else +static inline TEE_Result +crypto_asym_alloc_ecc_public_key(struct ecc_public_key *key __unused, + uint32_t key_type __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static inline const struct crypto_ecc_keypair_ops * +crypto_asym_get_keypair_ops(uint32_t key_type __unused) +{ + return NULL; +} + +static inline const struct crypto_ecc_public_ops * +crypto_asym_get_ecc_public_ops(uint32_t key_type __unused) +{ + return NULL; +} + +static inline TEE_Result +crypto_asym_alloc_ecc_keypair(struct ecc_keypair *key __unused, + uint32_t key_type __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /* CFG_CRYPTO_ECC */ + +#ifdef CFG_CRYPTO_DRV_ECC +TEE_Result drvcrypt_asym_alloc_ecc_public_key(struct ecc_public_key *key, + uint32_t key_type, + size_t key_size_bits); +TEE_Result drvcrypt_asym_alloc_ecc_keypair(struct ecc_keypair *key, + uint32_t key_type, + size_t key_size_bits); +#else +static inline TEE_Result +drvcrypt_asym_alloc_ecc_public_key(struct ecc_public_key *key __unused, + uint32_t key_type __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static inline TEE_Result +drvcrypt_asym_alloc_ecc_keypair(struct ecc_keypair *key __unused, + uint32_t key_type __unused, + size_t key_size_bits __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /* CFG_CRYPTO_DRV_ECC */ + +TEE_Result sw_crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *s, + size_t key_size_bits); + +TEE_Result sw_crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *s, + size_t key_size_bits); + +void sw_crypto_acipher_free_rsa_public_key(struct rsa_public_key *s); + +void sw_crypto_acipher_free_rsa_keypair(struct rsa_keypair *s); + +TEE_Result sw_crypto_acipher_gen_rsa_key(struct rsa_keypair *key, + size_t key_size); + +TEE_Result sw_crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len); +TEE_Result sw_crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len); +TEE_Result sw_crypto_acipher_rsaes_decrypt(uint32_t algo, + struct rsa_keypair *key, + const uint8_t *label, + size_t label_len, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len); + +TEE_Result sw_crypto_acipher_rsaes_encrypt(uint32_t algo, + struct rsa_public_key *key, + const uint8_t *label, + size_t label_len, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len); + +TEE_Result sw_crypto_acipher_rsassa_sign(uint32_t algo, struct rsa_keypair *key, + int salt_len, const uint8_t *msg, + size_t msg_len, uint8_t *sig, + size_t *sig_len); + +TEE_Result sw_crypto_acipher_rsassa_verify(uint32_t algo, + struct rsa_public_key *key, + int salt_len, const uint8_t *msg, + size_t msg_len, const uint8_t *sig, + size_t sig_len); +#endif /*__CRYPTO_CRYPTO_IMPL_H*/ diff --git a/optee_os/core/include/crypto/crypto_se.h b/optee_os/core/include/crypto/crypto_se.h new file mode 100644 index 0000000..eb7452f --- /dev/null +++ b/optee_os/core/include/crypto/crypto_se.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2021 - All Rights Reserved + * Author: Jorge Ramirez + */ + /* + * This is the Cryptographic Secure Element API, part of the Cryptographic + * Provider API. + * + * These requests shall be handled in the secure element normally placed on + * a serial communication bus (SPI, I2C). + */ +#ifndef __CRYPTO_SE_H +#define __CRYPTO_SE_H + +#include + +/* + * Type identifier for the APDU message as described by Smart Card Standard + * ISO7816-4 about ADPU message bodies decoding convention: + * + * https://cardwerk.com/smart-card-standard-iso7816-4-section-5-basic-organizations/#chap5_3_2 + */ +enum crypto_apdu_type { + CRYPTO_APDU_CASE_NO_HINT, + CRYPTO_APDU_CASE_1, + CRYPTO_APDU_CASE_2, + CRYPTO_APDU_CASE_2E, + CRYPTO_APDU_CASE_3, + CRYPTO_APDU_CASE_3E, + CRYPTO_APDU_CASE_4, + CRYPTO_APDU_CASE_4E, +}; + +TEE_Result crypto_se_do_apdu(enum crypto_apdu_type type, + uint8_t *header, size_t hdr_len, + uint8_t *src_data, size_t src_len, + uint8_t *dst_data, size_t *dst_len); + +/* + * Enable Secure Channel Protocol 03 to communicate with the Secure Element. + * + * Since SCP03 uses symmetric encryption, this interface also allows the user to + * attempt the rotation the keys stored in the Secure Element. + * + * https://globalplatform.org/wp-content/uploads/2014/07/GPC_2.3_D_SCP03_v1.1.2_PublicRelease.pdf + */ +TEE_Result crypto_se_enable_scp03(bool rotate_keys); +#endif diff --git a/optee_os/core/include/crypto/internal_aes-gcm.h b/optee_os/core/include/crypto/internal_aes-gcm.h new file mode 100644 index 0000000..d8e8ec9 --- /dev/null +++ b/optee_os/core/include/crypto/internal_aes-gcm.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + * + */ + +#ifndef __CRYPTO_INTERNAL_AES_GCM_H +#define __CRYPTO_INTERNAL_AES_GCM_H + +#include +#include +#include +#include + +#ifdef CFG_CRYPTO_WITH_CE +#include +#else +struct internal_ghash_key { +#ifdef CFG_AES_GCM_TABLE_BASED + uint64_t HL[16]; + uint64_t HH[16]; +#else + uint64_t hash_subkey[2]; +#endif +}; +#endif + +struct internal_aes_gcm_key { + /* AES (CTR) encryption key and number of rounds */ + uint64_t data[30]; + unsigned int rounds; +}; + +struct internal_aes_gcm_state { + uint64_t ctr[2]; + + struct internal_ghash_key ghash_key; + uint8_t hash_state[TEE_AES_BLOCK_SIZE]; + + uint8_t buf_tag[TEE_AES_BLOCK_SIZE]; + uint8_t buf_hash[TEE_AES_BLOCK_SIZE]; + uint8_t buf_cryp[TEE_AES_BLOCK_SIZE]; + + unsigned int tag_len; + unsigned int aad_bytes; + unsigned int payload_bytes; + unsigned int buf_pos; +}; + +struct internal_aes_gcm_ctx { + struct internal_aes_gcm_state state; + struct internal_aes_gcm_key key; +}; + +TEE_Result internal_aes_gcm_init(struct internal_aes_gcm_ctx *ctx, + TEE_OperationMode mode, const void *key, + size_t key_len, const void *nonce, + size_t nonce_len, size_t tag_len); +TEE_Result internal_aes_gcm_update_aad(struct internal_aes_gcm_ctx *ctx, + const void *data, size_t len); +TEE_Result internal_aes_gcm_update_payload(struct internal_aes_gcm_ctx *ctx, + TEE_OperationMode mode, + const void *src, size_t len, + void *dst); +TEE_Result internal_aes_gcm_enc_final(struct internal_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + void *tag, size_t *tag_len); +TEE_Result internal_aes_gcm_dec_final(struct internal_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + const void *tag, size_t tag_len); + +void internal_aes_gcm_inc_ctr(struct internal_aes_gcm_state *state); +void internal_aes_gcm_dec_ctr(struct internal_aes_gcm_state *state); + +TEE_Result internal_aes_gcm_enc(const struct internal_aes_gcm_key *enc_key, + const void *nonce, size_t nonce_len, + const void *aad, size_t aad_len, + const void *src, size_t len, void *dst, + void *tag, size_t *tag_len); + +TEE_Result internal_aes_gcm_dec(const struct internal_aes_gcm_key *enc_key, + const void *nonce, size_t nonce_len, + const void *aad, size_t aad_len, + const void *src, size_t len, void *dst, + const void *tag, size_t tag_len); + +void internal_aes_gcm_gfmul(const uint64_t X[2], const uint64_t Y[2], + uint64_t product[2]); + +static inline void internal_aes_gcm_xor_block(void *dst, const void *src) +{ + uint64_t *d = dst; + const uint64_t *s = src; + + assert(IS_ALIGNED_WITH_TYPE(dst, uint64_t)); + assert(IS_ALIGNED_WITH_TYPE(src, uint64_t)); + + d[0] ^= s[0]; + d[1] ^= s[1]; +} + +static inline bool internal_aes_gcm_ptr_is_block_aligned(const void *p) +{ + return !((vaddr_t)p & (TEE_AES_BLOCK_SIZE - 1)); +} + +#ifdef CFG_AES_GCM_TABLE_BASED +void internal_aes_gcm_ghash_gen_tbl(struct internal_ghash_key *ghash_key, + const struct internal_aes_gcm_key *enc_key); +void internal_aes_gcm_ghash_mult_tbl(struct internal_ghash_key *ghash_key, + const unsigned char x[16], + unsigned char output[16]); +#endif + +/* + * Must be implemented in core/arch/arm/crypto/ if CFG_CRYPTO_WITH_CE=y + */ +void internal_aes_gcm_set_key(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *enc_key); + +void internal_aes_gcm_ghash_update(struct internal_aes_gcm_state *state, + const void *head, const void *data, + size_t num_blocks); +/* + * Internal weak function that can be overridden with hardware specific + * implementation. + */ +void +internal_aes_gcm_update_payload_blocks(struct internal_aes_gcm_state *state, + const struct internal_aes_gcm_key *ek, + TEE_OperationMode mode, const void *src, + size_t num_blocks, void *dst); + +#endif /*__CRYPTO_INTERNAL_AES_GCM_H*/ diff --git a/optee_os/core/include/crypto/sm2-kdf.h b/optee_os/core/include/crypto/sm2-kdf.h new file mode 100644 index 0000000..5a012c0 --- /dev/null +++ b/optee_os/core/include/crypto/sm2-kdf.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020-2021, Huawei Technologies Co., Ltd + */ + +#ifndef CRYPTO_SM2_KDF_H +#define CRYPTO_SM2_KDF_H + +#include +#include + +TEE_Result sm2_kdf(const uint8_t *Z, size_t Z_len, uint8_t *t, size_t tlen); +#endif /* CRYPTO_SM2_KDF_H */ diff --git a/optee_os/core/include/drivers/amlogic_uart.h b/optee_os/core/include/drivers/amlogic_uart.h new file mode 100644 index 0000000..97b55f2 --- /dev/null +++ b/optee_os/core/include/drivers/amlogic_uart.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + +#ifndef AMLOGIC_UART_H +#define AMLOGIC_UART_H + +#include +#include + +struct amlogic_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void amlogic_uart_init(struct amlogic_uart_data *pd, paddr_t base); + +#endif /* AMLOGIC_UART_H */ diff --git a/optee_os/core/include/drivers/atmel_rstc.h b/optee_os/core/include/drivers/atmel_rstc.h new file mode 100644 index 0000000..6561d0b --- /dev/null +++ b/optee_os/core/include/drivers/atmel_rstc.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Microchip + */ +#ifndef __DRIVERS_ATMEL_RSTC_H +#define __DRIVERS_ATMEL_RSTC_H + +#include +#include + +#if defined(CFG_ATMEL_RSTC) +bool atmel_rstc_available(void); + +void __noreturn atmel_rstc_reset(void); +#else +static inline bool atmel_rstc_available(void) +{ + return false; +} + +static inline void atmel_rstc_reset(void) {} +#endif + +#endif /* __DRIVERS_ATMEL_RSTC_H */ diff --git a/optee_os/core/include/drivers/atmel_rtc.h b/optee_os/core/include/drivers/atmel_rtc.h new file mode 100644 index 0000000..ad52186 --- /dev/null +++ b/optee_os/core/include/drivers/atmel_rtc.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Microchip + */ + +#ifndef __DRIVERS_ATMEL_RTC_H +#define __DRIVERS_ATMEL_RTC_H + +#include +#include + +#ifdef CFG_ATMEL_RTC +TEE_Result atmel_rtc_get_tamper_timestamp(struct optee_rtc_time *tm); +#else +static inline +TEE_Result atmel_rtc_get_tamper_timestamp(struct optee_rtc_time *tm __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +#endif /* __DRIVERS_ATMEL_RTC_H */ diff --git a/optee_os/core/include/drivers/atmel_saic.h b/optee_os/core/include/drivers/atmel_saic.h new file mode 100644 index 0000000..c1b8b60 --- /dev/null +++ b/optee_os/core/include/drivers/atmel_saic.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Microchip + */ + +#ifndef __DRIVERS_ATMEL_SAIC_H +#define __DRIVERS_ATMEL_SAIC_H + +#include +#include + +#define AT91_AIC_WPKEY 0x414943 + +#define AT91_AIC_SSR 0x0 +#define AT91_AIC_SSR_ITSEL_MASK GENMASK_32(6, 0) + +#define AT91_AIC_SMR 0x4 +#define AT91_AIC_SMR_SRC_SHIFT 5 +#define AT91_AIC_SMR_LEVEL SHIFT_U32(0, AT91_AIC_SMR_SRC_SHIFT) +#define AT91_AIC_SMR_NEG_EDGE SHIFT_U32(1, AT91_AIC_SMR_SRC_SHIFT) +#define AT91_AIC_SMR_HIGH_LEVEL SHIFT_U32(2, AT91_AIC_SMR_SRC_SHIFT) +#define AT91_AIC_SMR_POS_EDGE SHIFT_U32(3, AT91_AIC_SMR_SRC_SHIFT) + +#define AT91_AIC_SVR 0x8 +#define AT91_AIC_IVR 0x10 +#define AT91_AIC_ISR 0x18 + +#define AT91_AIC_IPR0 0x20 +#define AT91_AIC_IPR1 0x24 +#define AT91_AIC_IPR2 0x28 +#define AT91_AIC_IPR3 0x2c +#define AT91_AIC_IMR 0x30 +#define AT91_AIC_CISR 0x34 +#define AT91_AIC_EOICR 0x38 +#define AT91_AIC_SPU 0x3c +#define AT91_AIC_IECR 0x40 +#define AT91_AIC_IDCR 0x44 +#define AT91_AIC_ICCR 0x48 +#define AT91_AIC_ISCR 0x4c +#define AT91_AIC_DCR 0x6c +#define AT91_AIC_WPMR 0xe4 +#define AT91_AIC_WPSR 0xe8 + +TEE_Result atmel_saic_setup(void); + +#endif /*__DRIVERS_ATMEL_SAIC_H*/ diff --git a/optee_os/core/include/drivers/atmel_shdwc.h b/optee_os/core/include/drivers/atmel_shdwc.h new file mode 100644 index 0000000..fbc7ead --- /dev/null +++ b/optee_os/core/include/drivers/atmel_shdwc.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015 Atmel Corporation, + * Nicolas Ferre + * Copyright (c) 2021, Microchip + */ + +#ifndef __DRIVERS_ATMEL_SHDWC_H +#define __DRIVERS_ATMEL_SHDWC_H + +#include +#include +#include +#include + +/* Shut Down Control Register */ +#define AT91_SHDW_CR 0x00 +/* Shut Down command */ +#define AT91_SHDW_SHDW BIT(0) +/* KEY Password */ +#define AT91_SHDW_KEY SHIFT_U32(0xa5UL, 24) + +/* Shut Down Mode Register */ +#define AT91_SHDW_MR 0x04 +#define AT91_SHDW_WKUPDBC_SHIFT 24 +#define AT91_SHDW_WKUPDBC_MASK GENMASK_32(26, 24) +#define AT91_SHDW_WKUPDBC(x) (SHIFT_U32((x), AT91_SHDW_WKUPDBC_SHIFT) & \ + AT91_SHDW_WKUPDBC_MASK) +#define AT91_SHDW_RTCWKEN BIT32(17) + +/* Shut Down Status Register */ +#define AT91_SHDW_SR 0x08 +#define AT91_SHDW_WKUPIS_SHIFT 16 +#define AT91_SHDW_WKUPIS_MASK GENMASK_32(31, 16) +#define AT91_SHDW_WKUPIS(x) (BIT32((x) + AT91_SHDW_WKUPIS_SHIFT)) + +/* Shutdown Wake-up Inputs Register */ +#define AT91_SHDW_WUIR 0x0c +#define AT91_SHDW_WKUPEN_MASK GENMASK_32(15, 0) +#define AT91_SHDW_WKUPEN(x) (BIT32(x) & AT91_SHDW_WKUPEN_MASK) +#define AT91_SHDW_WKUPT_SHIFT 16 +#define AT91_SHDW_WKUPT_MASK GENMASK_32(31, 16) +#define AT91_SHDW_WKUPT(x) (BIT32((x) + AT91_SHDW_WKUPT_SHIFT)) + +#ifndef __ASSEMBLER__ +#if defined(CFG_ATMEL_SHDWC) + +void __atmel_shdwc_shutdown(uint32_t mpddrc_base, uint32_t shdwc_base, + uint32_t pmc_base); + +bool atmel_shdwc_available(void); + +void __noreturn atmel_shdwc_shutdown(void); +#else +static inline bool atmel_shdwc_available(void) +{ + return false; +} + +static inline void atmel_shdwc_shutdown(void) {} +#endif /* defined(CFG_ATMEL_SHDWC) */ +#endif /* __ASSEMBLER__*/ + +#endif /* __DRIVERS_ATMEL_SHDWC_H */ diff --git a/optee_os/core/include/drivers/atmel_uart.h b/optee_os/core/include/drivers/atmel_uart.h new file mode 100644 index 0000000..aa5563f --- /dev/null +++ b/optee_os/core/include/drivers/atmel_uart.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Timesys Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef ATMEL_UART_H +#define ATMEL_UART_H + +#include +#include + +struct atmel_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void atmel_uart_init(struct atmel_uart_data *pd, paddr_t base); + +#endif /* ATMEL_UART_H */ diff --git a/optee_os/core/include/drivers/bcm/bnxt.h b/optee_os/core/include/drivers/bcm/bnxt.h new file mode 100644 index 0000000..7136c7e --- /dev/null +++ b/optee_os/core/include/drivers/bcm/bnxt.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef BNXT_H +#define BNXT_H + +#include +#include +#include +#include + +#define BNXT_SUCCESS 0 +#define BNXT_FAILURE (!BNXT_SUCCESS) + +uint32_t bnxt_write32_multiple(uintptr_t dst, + uintptr_t src, + uint32_t num_entries, + int src_4byte_increment); +void bnxt_handshake_clear(void); +void bnxt_chimp_halt(void); +void bnxt_kong_halt(void); +int bnxt_fastboot(uintptr_t addr); +uint32_t bnxt_wait_handshake(uint32_t timeout); +uint32_t bnxt_health_status(void); +TEE_Result bnxt_load_fw(int chip_type); +TEE_Result bnxt_copy_crash_dump(uint8_t *d, uint32_t offset, uint32_t len); + +struct bnxt_images_info { + vaddr_t bnxt_fw_vaddr; + uint32_t bnxt_fw_len; + vaddr_t bnxt_cfg_vaddr; + uint32_t bnxt_cfg_len; + vaddr_t bnxt_bspd_cfg_vaddr; + uint32_t bnxt_bspd_cfg_len; +}; + +/* Reserve 1K for BSPD data */ +#define BNXT_IMG_SECMEM_OFFSET 0x400 + +int get_bnxt_images_info(struct bnxt_images_info *bnxt_info, + int chip_type, vaddr_t ddr_dest); + +#endif diff --git a/optee_os/core/include/drivers/bcm_gpio.h b/optee_os/core/include/drivers/bcm_gpio.h new file mode 100644 index 0000000..6606656 --- /dev/null +++ b/optee_os/core/include/drivers/bcm_gpio.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef BCM_GPIO_H +#define BCM_GPIO_H + +#include +#include +#include + +/** + * struct bcm_gpio_chip describes GPIO controller chip instance + * @chip: generic GPIO chip handle. + * @gpio_base: starting GPIO number managed by this GPIO controller. + * @ngpios: number of GPIOs managed by this GPIO controller. + * @base: virtual base address of the GPIO controller registers. + */ +struct bcm_gpio_chip { + struct gpio_chip chip; + unsigned int gpio_base; + unsigned int ngpios; + vaddr_t base; + + SLIST_ENTRY(bcm_gpio_chip) link; +}; + +/* Returns bcm_gpio_chip handle for a GPIO pin */ +struct bcm_gpio_chip *bcm_gpio_pin_to_chip(unsigned int pin); +/* Set gpiopin as secure */ +void iproc_gpio_set_secure(int gpiopin); +#endif /* BCM_GPIO_H */ diff --git a/optee_os/core/include/drivers/bcm_hwrng.h b/optee_os/core/include/drivers/bcm_hwrng.h new file mode 100644 index 0000000..abf213d --- /dev/null +++ b/optee_os/core/include/drivers/bcm_hwrng.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef BCM_HWRNG_H +#define BCM_HWRNG_H + +#include + +uint32_t bcm_hwrng_read_rng(uint32_t *p_out, uint32_t words_to_read); + +#endif /* BCM_HWRNG_H */ diff --git a/optee_os/core/include/drivers/bcm_sotp.h b/optee_os/core/include/drivers/bcm_sotp.h new file mode 100644 index 0000000..2476004 --- /dev/null +++ b/optee_os/core/include/drivers/bcm_sotp.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef BCM_SOTP_H +#define BCM_SOTP_H + +#include +#include + +#define SOTP_ECC_ERR_DETECT BIT64(63) + +/** + * Reads from sotp fuse at given row address. + * @row_addr: row address + * @sotp_add_ecc: ecc memory support flag + * @rdata: pointer to sotp data value + * @returns TEE_Result value + */ +TEE_Result bcm_iproc_sotp_mem_read(uint32_t row_addr, bool sotp_add_ecc, + uint64_t *rdata); + +/** + * Writes to sotp fuse at given row address. + * @row_addr: row address + * @sotp_add_ecc: ecc memory support flag + * @wdata: data to be written to sotp fuse + * @returns TEE_Result value + */ +TEE_Result bcm_iproc_sotp_mem_write(uint32_t row_addr, bool sotp_add_ecc, + uint64_t wdata); + +#endif diff --git a/optee_os/core/include/drivers/caam_extension.h b/optee_os/core/include/drivers/caam_extension.h new file mode 100644 index 0000000..d023217 --- /dev/null +++ b/optee_os/core/include/drivers/caam_extension.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2023 NXP + */ +#ifndef __DRIVERS_CAAM_EXTENSION_H__ +#define __DRIVERS_CAAM_EXTENSION_H__ + +#include +#include +#include + +#ifdef CFG_NXP_CAAM_MP_DRV +/* + * Export the MPMR content. + * We assume that it is filled with message given in parameter. + * It contains 32 registers of 8 bits (32 bytes). + * + * @mpmr [out] MPMR buffer read + * @size [in/out] MPMR buffer size exported + */ +TEE_Result caam_mp_export_mpmr(uint8_t *mpmr, size_t *size); + +/* + * Export the Manufacturing Protection Public Key. + * + * @pubkey [out] Public key read + * @size [in/out] Public key size exported + */ +TEE_Result caam_mp_export_publickey(uint8_t *pubkey, size_t *size); + +/* + * MPSign function. + * This function takes the value in the MPMR if it exists + * and concatenates any additional data (certificate). + * The signature over the message is done with the private key. + * + * @data [in] Data to sign + * @data_size [in] Data size to sign + * @sig [out] Signature + * @sig_size [in/out] Signature size + */ +TEE_Result caam_mp_sign(uint8_t *data, size_t *data_size, uint8_t *sig, + size_t *sig_size); +#endif /* CFG_NXP_CAAM_MP_DRV */ + +#ifdef CFG_NXP_CAAM_DEK_DRV +/* + * Data encryption key generation using CAAM Secure Memory. + * + * @blob_data [in/out] Blob data + */ +TEE_Result caam_dek_generate(const uint8_t *payload, size_t payload_size, + uint8_t *dek, size_t dek_size); +#endif /* CFG_NXP_CAAM_DEK_DRV */ +#endif /* __CAAM_EXTENSION_H__ */ diff --git a/optee_os/core/include/drivers/cbmem_console.h b/optee_os/core/include/drivers/cbmem_console.h new file mode 100644 index 0000000..c8234d3 --- /dev/null +++ b/optee_os/core/include/drivers/cbmem_console.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023, Google Limited + */ +#ifndef DRIVERS_CBMEM_CONSOLE_H +#define DRIVERS_CBMEM_CONSOLE_H + +#include + +bool cbmem_console_init_from_dt(void *fdt); + +#endif /* DRIVERS_CBMEM_CONSOLE_H */ + diff --git a/optee_os/core/include/drivers/cdns_uart.h b/optee_os/core/include/drivers/cdns_uart.h new file mode 100644 index 0000000..19516ff --- /dev/null +++ b/optee_os/core/include/drivers/cdns_uart.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Xilinx Inc + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef CDNS_UART_H +#define CDNS_UART_H + +#include +#include + +struct cdns_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void cdns_uart_init(struct cdns_uart_data *pd, paddr_t base, uint32_t uart_clk, + uint32_t baud_rate); + +#endif /* CDNS_UART_H */ diff --git a/optee_os/core/include/drivers/clk.h b/optee_os/core/include/drivers/clk.h new file mode 100644 index 0000000..13cc9fa --- /dev/null +++ b/optee_os/core/include/drivers/clk.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Bootlin + */ + +#ifndef __DRIVERS_CLK_H +#define __DRIVERS_CLK_H + +#include +#include +#include + +/* Flags for clock */ +#define CLK_SET_RATE_GATE BIT(0) /* must be gated across rate change */ +#define CLK_SET_PARENT_GATE BIT(1) /* must be gated across re-parent */ + +/** + * struct clk - Clock structure + * + * @name: Clock name + * @priv: Private data for the clock provider + * @ops: Clock operations + * @parent: Current parent + * @rate: Current clock rate (cached after init or rate change) + * @flags: Specific clock flags + * @enabled_count: Enable/disable reference counter + * @num_parents: Number of parents + * @parents: Array of possible parents of the clock + */ +struct clk { + const char *name; + void *priv; + const struct clk_ops *ops; + struct clk *parent; + unsigned long rate; + unsigned int flags; + struct refcount enabled_count; + size_t num_parents; + struct clk *parents[]; +}; + +/** + * struct clk_ops + * + * @enable: Enable the clock + * @disable: Disable the clock + * @set_parent: Set the clock parent based on index + * @get_parent: Get the current parent index of the clock + * @set_rate: Set the clock rate + * @get_rate: Get the clock rate + * @get_rates_array: Get the supported clock rates as array + */ +struct clk_ops { + TEE_Result (*enable)(struct clk *clk); + void (*disable)(struct clk *clk); + TEE_Result (*set_parent)(struct clk *clk, size_t index); + size_t (*get_parent)(struct clk *clk); + TEE_Result (*set_rate)(struct clk *clk, unsigned long rate, + unsigned long parent_rate); + unsigned long (*get_rate)(struct clk *clk, + unsigned long parent_rate); + TEE_Result (*get_rates_array)(struct clk *clk, size_t start_index, + unsigned long *rates, size_t *nb_elts); +}; + +/** + * Return the clock name + * + * @clk: Clock for which the name is needed + * Return a const char * pointing to the clock name + */ +static inline const char *clk_get_name(struct clk *clk) +{ + return clk->name; +} + +/** + * clk_alloc - Allocate a clock structure + * + * @name: Clock name + * @ops: Clock operations + * @parent_clks: Parents of the clock + * @parent_count: Number of parents of the clock + * + * Return a clock struct properly initialized or NULL if allocation failed + */ +struct clk *clk_alloc(const char *name, const struct clk_ops *ops, + struct clk **parent_clks, size_t parent_count); + +/** + * clk_free - Free a clock structure + * + * @clk: Clock to be freed or NULL + */ +void clk_free(struct clk *clk); + +/** + * clk_register - Register a clock within the clock framework + * + * @clk: Clock struct to be registered + * Return a TEE_Result compliant value + */ +TEE_Result clk_register(struct clk *clk); + +/** + * clk_get_rate - Get clock rate + * + * @clk: Clock for which the rate is needed + * Return the clock rate in Hz + */ +unsigned long clk_get_rate(struct clk *clk); + +/** + * clk_set_rate - Set a clock rate + * + * @clk: Clock to be set with the rate + * @rate: Rate to set in Hz + * Return a TEE_Result compliant value + */ +TEE_Result clk_set_rate(struct clk *clk, unsigned long rate); + +/** + * clk_enable - Enable a clock and its ascendance + * + * @clk: Clock to be enabled + * Return a TEE_Result compliant value + */ +TEE_Result clk_enable(struct clk *clk); + +/** + * clk_disable - Disable a clock + * + * @clk: Clock to be disabled + */ +void clk_disable(struct clk *clk); + +/** + * clk_is_enabled - Informative state on the clock + * + * This function is useful during specific system sequences where core + * executes atomically (primary core boot, some low power sequences). + * + * @clk: Clock refernece + */ +bool clk_is_enabled(struct clk *clk); + +/** + * clk_get_parent - Get the current clock parent + * + * @clk: Clock for which the parent is needed + * Return the clock parent or NULL if there is no parent + */ +struct clk *clk_get_parent(struct clk *clk); + +/** + * clk_get_num_parents - Get the number of parents for a clock + * + * @clk: Clock for which the number of parents is needed + * Return the number of parents + */ +static inline size_t clk_get_num_parents(struct clk *clk) +{ + return clk->num_parents; +} + +/** + * Get a clock parent by its index + * + * @clk: Clock for which the parent is needed + * @pidx: Parent index for the clock + * Return the clock parent at index @pidx or NULL if out of bound + */ +struct clk *clk_get_parent_by_index(struct clk *clk, size_t pidx); + +/** + * clk_set_parent - Set the current clock parent + * + * @clk: Clock for which the parent should be set + * @parent: Parent clock to set + * Return a TEE_Result compliant value + */ +TEE_Result clk_set_parent(struct clk *clk, struct clk *parent); + +/** + * clk_get_rates_array - Get supported rates as an array + * + * @clk: Clock for which the rates are requested + * @start_index: start index of requested rates + * @rates: Array of rates allocated by caller or NULL to query count of rates + * @nb_elts: Max number of elements that the array can hold as input. Contains + * the number of elements that was added in the array as output. + * Returns a TEE_Result compliant value + */ +TEE_Result clk_get_rates_array(struct clk *clk, size_t start_index, + unsigned long *rates, size_t *nb_elts); + +#endif /* __DRIVERS_CLK_H */ diff --git a/optee_os/core/include/drivers/clk_dt.h b/optee_os/core/include/drivers/clk_dt.h new file mode 100644 index 0000000..eb164ac --- /dev/null +++ b/optee_os/core/include/drivers/clk_dt.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Bootlin + */ + +#ifndef __DRIVERS_CLK_DT_H +#define __DRIVERS_CLK_DT_H + +#include +#include +#include +#include + +/** + * CLK_DT_DECLARE - Declare a clock driver + * @__name: Clock driver name + * @__compat: Compatible string + * @__probe: Clock probe function + */ +#define CLK_DT_DECLARE(__name, __compat, __probe) \ + static const struct dt_device_match __name ## _match_table[] = { \ + { .compatible = __compat }, \ + { } \ + }; \ + DEFINE_DT_DRIVER(__name ## _dt_driver) = { \ + .name = # __name, \ + .type = DT_DRIVER_CLK, \ + .match_table = __name ## _match_table, \ + .probe = __probe, \ + } + +/** + * clk_dt_get_by_index - Get a clock at a specific index in "clocks" property + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the subnode containing a clock property + * @clk_idx: Clock index to get + * @clk: Output clock reference upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if clock is not initialized + * Return TEE_ERROR_ITEM_NOT_FOUND if the DT does not provide a clock reference + * Return any other TEE_Result compliant code in case of error + */ +TEE_Result clk_dt_get_by_index(const void *fdt, int nodeoffset, + unsigned int clk_idx, struct clk **clk); + +/** + * clk_dt_get_by_name - Get a clock matching a name in "clock-names" property + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the subnode containing a clock property + * @name: Clock name to get + * @clk: Output clock reference upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if clock is not initialized + * Return TEE_ERROR_ITEM_NOT_FOUND if the DT does not provide a clock reference + * Return any other TEE_Result compliant code in case of error + */ +TEE_Result clk_dt_get_by_name(const void *fdt, int nodeoffset, + const char *name, struct clk **clk); + +/** + * clk_dt_get_func - Typedef of function to get clock from devicetree properties + * + * @args: Pointer to devicetree description of the clock to parse + * @data: Pointer to data given at clk_dt_register_clk_provider() call + * @clk: Output clock reference upon success + */ +typedef TEE_Result (*clk_dt_get_func)(struct dt_pargs *args, void *data, + struct clk **out_clk); + +/** + * clk_dt_register_clk_provider - Register a clock provider + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the clock + * @func: Callback to match the devicetree clock with a clock struct + * @data: Data which will be passed to the get_dt_clk callback + * Returns TEE_Result value + */ +static inline TEE_Result clk_dt_register_clk_provider(const void *fdt, + int nodeoffset, + clk_dt_get_func func, + void *data) +{ + return dt_driver_register_provider(fdt, nodeoffset, + (get_of_device_func)func, data, + DT_DRIVER_CLK); +} + +/** + * clk_dt_get_simple_clk: simple clock matching function for single clock + * providers + */ +static inline TEE_Result clk_dt_get_simple_clk(struct dt_pargs *args __unused, + void *data, struct clk **out_clk) +{ + *out_clk = data; + + return TEE_SUCCESS; +} + +#endif /* __DRIVERS_CLK_DT_H */ diff --git a/optee_os/core/include/drivers/gic.h b/optee_os/core/include/drivers/gic.h new file mode 100644 index 0000000..cb580ec --- /dev/null +++ b/optee_os/core/include/drivers/gic.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef __DRIVERS_GIC_H +#define __DRIVERS_GIC_H +#include +#include + +#if defined(CFG_ARM_GICV3) +#define GIC_DIST_REG_SIZE 0x10000 +#define GIC_CPU_REG_SIZE 0x10000 +#else +#define GIC_DIST_REG_SIZE 0x1000 +#define GIC_CPU_REG_SIZE 0x1000 +#endif + +#define GIC_PPI_BASE U(16) +#define GIC_SPI_BASE U(32) + +#define GIC_SGI_TO_ITNUM(x) (x) +#define GIC_PPI_TO_ITNUM(x) ((x) + GIC_PPI_BASE) +#define GIC_SPI_TO_ITNUM(x) ((x) + GIC_SPI_BASE) + +/* + * The two gic_init_* functions initializes the struct gic_data which is + * then used by the other functions. + */ + +/* Initialize GIC */ +void gic_init(paddr_t gicc_base_pa, paddr_t gicd_base_pa); + +/* Only initialize CPU GIC interface, mainly use for secondary CPUs */ +void gic_cpu_init(void); + +/* Print GIC state to console */ +void gic_dump_state(void); +#endif /*__DRIVERS_GIC_H*/ diff --git a/optee_os/core/include/drivers/gpio.h b/optee_os/core/include/drivers/gpio.h new file mode 100644 index 0000000..7ce00e1 --- /dev/null +++ b/optee_os/core/include/drivers/gpio.h @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef DRIVERS_GPIO_H +#define DRIVERS_GPIO_H + +#include +#include +#include +#include +#include + +/** + * GPIO_DT_DECLARE - Declare a GPIO controller driver with a single + * device tree compatible string. + * + * @__name: GPIO controller driver name + * @__compat: Compatible string + * @__probe: GPIO controller probe function + */ +#define GPIO_DT_DECLARE(__name, __compat, __probe) \ + static const struct dt_device_match __name ## _match_table[] = { \ + { .compatible = __compat }, \ + { } \ + }; \ + DEFINE_DT_DRIVER(__name ## _dt_driver) = { \ + .name = # __name, \ + .type = DT_DRIVER_GPIO, \ + .match_table = __name ## _match_table, \ + .probe = __probe, \ + } + +enum gpio_dir { + GPIO_DIR_OUT, + GPIO_DIR_IN +}; + +enum gpio_level { + GPIO_LEVEL_LOW, + GPIO_LEVEL_HIGH +}; + +enum gpio_interrupt { + GPIO_INTERRUPT_DISABLE, + GPIO_INTERRUPT_ENABLE +}; + +struct gpio; +struct gpio_ops; + +struct gpio_chip { + const struct gpio_ops *ops; +}; + +struct gpio_ops { + /* Get GPIO direction current configuration */ + enum gpio_dir (*get_direction)(struct gpio_chip *chip, + unsigned int gpio_pin); + /* Set GPIO direction configuration */ + void (*set_direction)(struct gpio_chip *chip, unsigned int gpio_pin, + enum gpio_dir direction); + /* Get GPIO current level */ + enum gpio_level (*get_value)(struct gpio_chip *chip, + unsigned int gpio_pin); + /* Set GPIO level */ + void (*set_value)(struct gpio_chip *chip, unsigned int gpio_pin, + enum gpio_level value); + /* Get GPIO interrupt state */ + enum gpio_interrupt (*get_interrupt)(struct gpio_chip *chip, + unsigned int gpio_pin); + /* Enable or disable a GPIO interrupt */ + void (*set_interrupt)(struct gpio_chip *chip, unsigned int gpio_pin, + enum gpio_interrupt enable_disable); + /* Release GPIO resources */ + void (*put)(struct gpio_chip *chip, struct gpio *gpio); +}; + +/* + * struct gpio - GPIO pin description + * @chip: GPIO controller chip reference + * @dt_flags: Pin boolean properties set from DT node + * @pin: Pin number in GPIO controller + */ +struct gpio { + struct gpio_chip *chip; + uint32_t dt_flags; + unsigned int pin; +}; + +static inline bool gpio_ops_is_valid(const struct gpio_ops *ops) +{ + return ops->set_direction && ops->get_direction && ops->get_value && + ops->set_value; +} + +static inline void gpio_set_direction(struct gpio *gpio, enum gpio_dir dir) +{ + gpio->chip->ops->set_direction(gpio->chip, gpio->pin, dir); +} + +static inline enum gpio_dir gpio_get_direction(struct gpio *gpio) +{ + return gpio->chip->ops->get_direction(gpio->chip, gpio->pin); +} + +static inline void gpio_set_value(struct gpio *gpio, enum gpio_level value) +{ + if (gpio->dt_flags & GPIO_ACTIVE_LOW) + value = !value; + + gpio->chip->ops->set_value(gpio->chip, gpio->pin, value); +} + +static inline enum gpio_level gpio_get_value(struct gpio *gpio) +{ + enum gpio_level value = GPIO_LEVEL_LOW; + + value = gpio->chip->ops->get_value(gpio->chip, gpio->pin); + + if (gpio->dt_flags & GPIO_ACTIVE_LOW) + value = !value; + + return value; +} + +static inline void gpio_put(struct gpio *gpio) +{ + assert(!gpio || (gpio->chip && gpio->chip->ops)); + + if (gpio && gpio->chip->ops->put) + gpio->chip->ops->put(gpio->chip, gpio); +} + +#if defined(CFG_DT) && defined(CFG_DRIVERS_GPIO) +/** + * gpio_dt_alloc_pin() - Get an allocated GPIO instance from its DT phandle + * + * @pargs: Pointer to devicetree description of the GPIO controller to parse + * @res: Output result code of the operation: + * TEE_SUCCESS in case of success + * TEE_ERROR_DEFER_DRIVER_INIT if GPIO controller is not initialized + * Any TEE_Result compliant code in case of error. + * + * Returns a struct gpio pointer pointing to a GPIO instance matching + * the devicetree description or NULL if invalid description in which case + * @res provides the error code. + */ +TEE_Result gpio_dt_alloc_pin(struct dt_pargs *pargs, struct gpio **gpio); + +/** + * gpio_dt_get_by_index() - Get a GPIO controller at a specific index in + * 'gpios' property + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the subnode containing a 'gpios' property + * @index: GPIO pin index in '*-gpios' property + * @gpio_name: Name of the GPIO pin + * @gpio: Output GPIO pin reference upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if GPIO controller is not initialized + * Return a TEE_Result compliant code in case of error + */ +TEE_Result gpio_dt_get_by_index(const void *fdt, int nodeoffset, + unsigned int index, const char *gpio_name, + struct gpio **gpio); +#else +static inline TEE_Result gpio_dt_get_by_index(const void *fdt __unused, + int nodeoffset __unused, + unsigned int index __unused, + const char *gpio_name __unused, + struct gpio **gpio __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result gpio_dt_alloc_pin(struct dt_pargs *pargs __unused, + struct gpio **gpio __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif /*CFG_DT*/ + +/** + * gpio_dt_get_func - Typedef of function to get GPIO instance from + * devicetree properties + * + * @pargs: Pointer to GPIO phandle and its argument in the FDT + * @data: Pointer to the data given at gpio_dt_register_provider() call + * @res: Output result code of the operation: + * TEE_SUCCESS in case of success + * TEE_ERROR_DEFER_DRIVER_INIT if GPIO controller is not initialized + * Any TEE_Result compliant code in case of error. + * + * Returns a struct GPIO pointer pointing to a GPIO instance matching + * the devicetree description or NULL if invalid description in which case + * @res provides the error code. + */ +typedef TEE_Result (*gpio_dt_get_func)(struct dt_pargs *pargs, void *data, + struct gpio **out_gpio); + +/** + * gpio_dt_register_provider() - Register a GPIO controller provider + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the GPIO controller + * @get_dt_gpio: Callback to match the GPIO controller with a struct gpio + * @data: Opaque reference which will be passed to the get_dt_gpio callback + * Returns TEE_Result value + */ +static inline TEE_Result gpio_register_provider(const void *fdt, int nodeoffset, + gpio_dt_get_func get_dt_gpio, + void *data) +{ + return dt_driver_register_provider(fdt, nodeoffset, + (get_of_device_func)get_dt_gpio, + data, DT_DRIVER_GPIO); +} + +#endif /* DRIVERS_GPIO_H */ diff --git a/optee_os/core/include/drivers/hfic.h b/optee_os/core/include/drivers/hfic.h new file mode 100644 index 0000000..0c03c84 --- /dev/null +++ b/optee_os/core/include/drivers/hfic.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Linaro Limited + */ + +#ifndef __DRIVERS_HFIC_H +#define __DRIVERS_HFIC_H + +void hfic_init(void); + +#endif /*__DRIVERS_HFIC_H*/ diff --git a/optee_os/core/include/drivers/hi16xx_uart.h b/optee_os/core/include/drivers/hi16xx_uart.h new file mode 100644 index 0000000..83413fc --- /dev/null +++ b/optee_os/core/include/drivers/hi16xx_uart.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +/* + * UART driver for Hisilicon Hi16xx and Phosphor V660 (hip05) SoCs + */ + +#ifndef HI16XX_UART_H +#define HI16XX_UART_H + +#include +#include + +#define HI16XX_UART_REG_SIZE 0xF8 + +struct hi16xx_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void hi16xx_uart_init(struct hi16xx_uart_data *pd, paddr_t base, + uint32_t uart_clk, uint32_t baud_rate); + +#endif /* HI16XX_UART_H */ diff --git a/optee_os/core/include/drivers/i2c.h b/optee_os/core/include/drivers/i2c.h new file mode 100644 index 0000000..1cc9cda --- /dev/null +++ b/optee_os/core/include/drivers/i2c.h @@ -0,0 +1,344 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Microchip + */ + +#ifndef __DRIVERS_I2C_H +#define __DRIVERS_I2C_H + +#include +#include +#include + +/** + * DEFINE_I2C_DEV_DRIVER - Declare an I2C driver + * + * @__name: I2C device driver name + * @__match_table: match table associated to the driver + * @__i2c_probe: I2C probe function with the following prototype: + * TEE_Result (*probe)(struct i2c_dev *i2c_dev, const void *fdt, + * int node, const void *compat_data); + */ +#define DEFINE_I2C_DEV_DRIVER(__name, __match_table, __i2c_probe) \ + static TEE_Result __name ## _probe_i2c_dev(const void *fdt, int node, \ + const void *compat_data) \ + { \ + struct i2c_dev *i2c_dev = NULL; \ + TEE_Result res = TEE_ERROR_GENERIC; \ + \ + res = i2c_dt_get_dev(fdt, node, &i2c_dev); \ + if (res) \ + return res; \ + return __i2c_probe(i2c_dev, fdt, node, compat_data); \ + } \ + DEFINE_DT_DRIVER(__name ## _dt_driver) = { \ + .name = # __name, \ + .type = DT_DRIVER_I2C, \ + .match_table = __match_table, \ + .probe = __name ## _probe_i2c_dev, \ + } + +#define I2C_SMBUS_MAX_BUF_SIZE 32 + +enum i2c_smbus_dir { + I2C_SMBUS_READ, + I2C_SMBUS_WRITE, +}; + +enum i2c_smbus_protocol { + I2C_SMBUS_PROTO_BYTE, + /* + * Like block but does not insert "count" in sent data, useful for + * EEPROM read for instance which is not SMBus but requires such + * sequence. + */ + I2C_SMBUS_PROTO_BLOCK_RAW, +}; + +struct i2c_ctrl; + +/** + * struct i2c_dev - I2C device + * + * @ctrl: I2C controller associated to the device + * @addr: device address on the I2C bus + */ +struct i2c_dev { + struct i2c_ctrl *ctrl; + uint16_t addr; +}; + +/** + * struct i2c_ctrl_ops - Operations provided by I2C controller drivers + * + * @read: I2C read operation + * @write: I2C write operation + * @smbus: SMBus protocol operation + */ +struct i2c_ctrl_ops { + TEE_Result (*read)(struct i2c_dev *i2c_dev, uint8_t *buf, size_t len); + TEE_Result (*write)(struct i2c_dev *i2c_dev, const uint8_t *buf, + size_t len); + TEE_Result (*smbus)(struct i2c_dev *i2c_dev, enum i2c_smbus_dir dir, + enum i2c_smbus_protocol proto, uint8_t cmd_code, + uint8_t *buf, size_t len); +}; + +/** + * struct i2c_ctrl - I2C controller + * + * @ops: Operations associated to the I2C controller + */ +struct i2c_ctrl { + const struct i2c_ctrl_ops *ops; +}; + +#ifdef CFG_DRIVERS_I2C +/** + * i2c_create_dev - Create and i2c_dev struct from device-tree + * + * @i2c_ctrl: Controller to be used with this device + * @fdt: Device-tree to work on + * @node: Node to work on in @fdt provided device-tree + * + * Return an i2c_dev struct filled from device-tree description + */ +struct i2c_dev *i2c_create_dev(struct i2c_ctrl *i2c_ctrl, const void *fdt, + int node); + +/** + * i2c_write() - Execute an I2C write on the I2C bus + * + * @i2c_dev: I2C device used for writing + * @buf: Buffer of data to be written + * @len: Length of data to be written + * + * Return a TEE_Result compliant value + */ +static inline TEE_Result i2c_write(struct i2c_dev *i2c_dev, const uint8_t *buf, + size_t len) +{ + if (!i2c_dev->ctrl->ops->write) + return TEE_ERROR_NOT_SUPPORTED; + + return i2c_dev->ctrl->ops->write(i2c_dev, buf, len); +} + +/** + * i2c_read() - Execute an I2C read on the I2C bus + * + * @i2c_dev: I2C device used for reading + * @buf: Buffer containing the read data + * @len: Length of data to be read + * + * Return a TEE_Result compliant value + */ +static inline TEE_Result i2c_read(struct i2c_dev *i2c_dev, uint8_t *buf, + size_t len) +{ + if (!i2c_dev->ctrl->ops->read) + return TEE_ERROR_NOT_SUPPORTED; + + return i2c_dev->ctrl->ops->read(i2c_dev, buf, len); +} + +/** + * i2c_smbus_raw() - Execute a raw SMBUS request + * + * @i2c_dev: I2C device used for SMBus operation + * @dir: Direction for the SMBus transfer + * @proto: SMBus Protocol to be executed + * @cmd_code: Command code + * @buf: Buffer used for read/write operation + * @len: Length of buffer to be read/write + * + * Return a TEE_Result compliant value + */ +static inline TEE_Result i2c_smbus_raw(struct i2c_dev *i2c_dev, + enum i2c_smbus_dir dir, + enum i2c_smbus_protocol proto, + uint8_t cmd_code, uint8_t *buf, + size_t len) +{ + if (!i2c_dev->ctrl->ops->smbus) + return TEE_ERROR_NOT_SUPPORTED; + + if (len > I2C_SMBUS_MAX_BUF_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + return i2c_dev->ctrl->ops->smbus(i2c_dev, dir, proto, cmd_code, buf, + len); +} + +/** + * i2c_dt_get_dev - Get an I2C device from a DT node + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the I2C bus consumer + * @i2c_dev: Output I2C bus device upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if I2C controller is not initialized + * Return TEE_ERROR_ITEM_NOT_FOUND if the I2C controller node does not exist + * Return a TEE_Result compliant code in case of error + */ +static inline TEE_Result i2c_dt_get_dev(const void *fdt, int nodeoffset, + struct i2c_dev **out_i2c_dev) +{ + TEE_Result res = TEE_ERROR_GENERIC; + void *i2c_dev = NULL; + + res = dt_driver_device_from_parent(fdt, nodeoffset, DT_DRIVER_I2C, + &i2c_dev); + if (!res) + *out_i2c_dev = i2c_dev; + + return res; +} +#else +static inline TEE_Result i2c_write(struct i2c_dev *i2c_dev __unused, + const uint8_t *buf __unused, + size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result i2c_read(struct i2c_dev *i2c_dev __unused, + uint8_t *buf __unused, size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result i2c_smbus_raw(struct i2c_dev *i2c_dev __unused, + enum i2c_smbus_dir dir __unused, + enum i2c_smbus_protocol proto __unused, + uint8_t cmd_code __unused, + uint8_t *buf __unused, + size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result i2c_dt_get_dev(const void *fdt __unused, + int nodeoffset __unused, + struct i2c_dev **i2c_dev) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +/** + * i2c_dt_get_func - Typedef of function to get I2C bus device from + * devicetree properties + * + * @args: Pointer to devicetree description of the I2C bus device to parse + * @data: Pointer to data given at i2c_dt_register_provider() call + * @out_device: Output pointer to I2C device upon success + */ +typedef TEE_Result (*i2c_dt_get_func)(struct dt_pargs *args, void *data, + struct i2c_dev **out_device); + +/** + * i2c_dt_register_provider - Register a I2C controller provider and add all the + * child nodes of this controller in the DT probe list. + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the I2C controller + * @get_dt_i2c: Callback to match the I2C controller with a struct i2c + * @data: Data which will be passed to the get_dt_i2c callback + * Returns TEE_Result value + */ +static inline TEE_Result i2c_register_provider(const void *fdt, int nodeoffset, + i2c_dt_get_func get_dt_i2c, + void *data) +{ + int subnode = -1; + TEE_Result res = TEE_ERROR_GENERIC; + + res = dt_driver_register_provider(fdt, nodeoffset, + (get_of_device_func)get_dt_i2c, + data, DT_DRIVER_I2C); + if (res) + return res; + + fdt_for_each_subnode(subnode, fdt, nodeoffset) + dt_driver_maybe_add_probe_node(fdt, subnode); + + return TEE_SUCCESS; +} + +/** + * i2c_smbus_read_byte_data() - Execute a read byte SMBus protocol operation + * + * @i2c_dev: I2C device used for SMBus operation + * @cmd_code: Command code to read + * @byte: Returned byte value read from device + * + * Return a TEE_Result compliant value + */ +static inline TEE_Result i2c_smbus_read_byte_data(struct i2c_dev *i2c_dev, + uint8_t cmd_code, + uint8_t *byte) +{ + return i2c_smbus_raw(i2c_dev, I2C_SMBUS_READ, I2C_SMBUS_PROTO_BYTE, + cmd_code, byte, 1); +} + +/** + * i2c_smbus_write_byte_data() - Execute a write byte SMBus protocol operation + * + * @i2c_dev: I2C device used for SMBus operation + * @cmd_code: Command code for write operation + * @byte: Byte to be written to the device + * + * Return a TEE_Result compliant value + */ +static inline TEE_Result i2c_smbus_write_byte_data(struct i2c_dev *i2c_dev, + uint8_t cmd_code, + uint8_t byte) +{ + return i2c_smbus_raw(i2c_dev, I2C_SMBUS_WRITE, I2C_SMBUS_PROTO_BYTE, + cmd_code, &byte, 1); +} + +/** + * i2c_bus_read_block_raw() - Execute a non-standard SMBus raw block read. + * This does not insert the "count" of byte to be written unlike the SMBus block + * read operation. + * + * @i2c_dev: I2C device used for SMBus operation + * @cmd_code: Command code for read operation + * @buf: Buffer of data read from device + * @len: Length of data to be read from the device + * + * Return a TEE_Result compliant value + */ +static inline TEE_Result i2c_bus_read_block_raw(struct i2c_dev *i2c_dev, + uint8_t cmd_code, uint8_t *buf, + size_t len) +{ + return i2c_smbus_raw(i2c_dev, I2C_SMBUS_READ, I2C_SMBUS_PROTO_BLOCK_RAW, + cmd_code, buf, len); +} + +/** + * i2c_bus_write_block_raw() - Execute a non-standard SMBus raw block write. + * This does not insert the "count" of byte to be written unlike the SMBus block + * write operation. + * + * @i2c_dev: I2C device used for SMBus operation + * @cmd_code: Command code for write operation + * @buf: Buffer of data to be written to the device + * @len: Length of data to be written to the device + * + * Return a TEE_Result compliant value + */ +static inline TEE_Result i2c_bus_write_block_raw(struct i2c_dev *i2c_dev, + uint8_t cmd_code, + uint8_t *buf, size_t len) +{ + return i2c_smbus_raw(i2c_dev, I2C_SMBUS_WRITE, + I2C_SMBUS_PROTO_BLOCK_RAW, cmd_code, buf, len); +} + +#endif diff --git a/optee_os/core/include/drivers/imx/dcp.h b/optee_os/core/include/drivers/imx/dcp.h new file mode 100644 index 0000000..a01ae30 --- /dev/null +++ b/optee_os/core/include/drivers/imx/dcp.h @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020 NXP + */ +#ifndef __IMX_DCP_H__ +#define __IMX_DCP_H__ + +#include +#include +#include +#include + +#define DCP_SHA_BLOCK_SIZE U(64) +#define DCP_AES128_BLOCK_SIZE U(16) +#define DCP_AES128_KEY_SIZE DCP_AES128_BLOCK_SIZE +#define DCP_AES128_IV_SIZE DCP_AES128_BLOCK_SIZE + +enum dcp_key_mode { + DCP_SRAM0 = 0, + DCP_SRAM1, + DCP_SRAM2, + DCP_SRAM3, + DCP_PAYLOAD, + DCP_OTP, +}; + +enum dcp_aes_mode { + DCP_ECB = 0, + DCP_CBC, +}; + +enum dcp_aes_op { + DCP_ENCRYPT = 0, + DCP_DECRYPT, +}; + +enum dcp_hash_config { + DCP_SHA1 = 0, + DCP_SHA256, +}; + +enum dcp_channel { + DCP_CHANN0 = 0, + DCP_CHANN1, + DCP_CHANN2, + DCP_CHANN3, + DCP_NB_CHANNELS, +}; + +/* DCP work packet descriptor is a hardware data structure */ +struct dcp_descriptor { + uint32_t next; + uint32_t ctrl0; + uint32_t ctrl1; + /* Source buffer physical address */ + uint32_t src_buffer; + /* Destination buffer physical address */ + uint32_t dest_buffer; + uint32_t buff_size; + /* Payload buffer physical address */ + uint32_t payload; + uint32_t status; +}; + +struct dcp_align_buf { + uint8_t *data; + paddr_t paddr; + size_t size; +}; + +struct dcp_data { + struct dcp_descriptor desc; + enum dcp_channel channel; +}; + +struct dcp_hash_data { + struct dcp_data dcp_data; + bool initialized; + enum dcp_hash_config alg; + struct dcp_align_buf ctx; + size_t ctx_size; +}; + +struct dcp_hashalg { + unsigned int type; + unsigned int size; +}; + +struct dcp_cipher_data { + struct dcp_data dcp_data; + bool initialized; + uint8_t iv[DCP_AES128_IV_SIZE]; + uint8_t key[DCP_AES128_KEY_SIZE]; + /* payload buffer holds the key and the iv */ + struct dcp_align_buf payload; + size_t payload_size; +}; + +struct dcp_cipher_init { + enum dcp_aes_op op; + enum dcp_aes_mode mode; + enum dcp_key_mode key_mode; + uint8_t *key; + uint8_t *iv; +}; + +/* + * Perform AES-CMAC operation + * + * @init Cipher operation context + * @input Input message + * @input_size Input message size + * @output Output MAC + */ +TEE_Result dcp_cmac(struct dcp_cipher_init *init, uint8_t *input, + size_t input_size, uint8_t *output); + +/* + * Store key in the SRAM + * + * @key Buffer containing the key to store (128 bit) + * @index Index of the key (0, 1, 2 or 3) + */ +TEE_Result dcp_store_key(uint32_t *key, unsigned int index); + +/* + * Initialize AES-128 operation + * + * @data Cipher operation context + * @init Data for aesdata initialization + */ +TEE_Result dcp_cipher_do_init(struct dcp_cipher_data *data, + struct dcp_cipher_init *init); + +/* + * Update AES-128 operation + * + * @data Cipher operation context + * @src Source data to encrypt/decrypt + * @dst [out] Destination buffer + * @size Size of source data in bytes, must be 16 bytes multiple + */ +TEE_Result dcp_cipher_do_update(struct dcp_cipher_data *data, + const uint8_t *src, uint8_t *dst, size_t size); + +/* + * Finalize AES-128 operation + * + * @data Cipher operation context + */ +void dcp_cipher_do_final(struct dcp_cipher_data *data); + +/* + * Initialize hash operation + * + * @hashdata Hash operation context + */ +TEE_Result dcp_sha_do_init(struct dcp_hash_data *hashdata); + +/* + * Update hash operation + * + * @hashdata Hash operation context + * @data Buffer to hash + * @len Size of the input buffer in bytes + */ +TEE_Result dcp_sha_do_update(struct dcp_hash_data *hashdata, + const uint8_t *data, size_t len); + +/* + * Finalize the hash operation + * + * @hashdata Hash operation context + * @digest [out] Result of the hash operation + * @digest_size Digest buffer size in bytes + */ +TEE_Result dcp_sha_do_final(struct dcp_hash_data *hashdata, uint8_t *digest, + size_t digest_size); + +/* + * Disable the use of the DCP unique key (0xFE in the DCP key-select field). + */ +void dcp_disable_unique_key(void); + +/* Initialize DCP */ +TEE_Result dcp_init(void); + +#ifndef CFG_DT +static inline TEE_Result dcp_vbase(vaddr_t *base __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif /* CFG_DT */ + +#endif /* __IMX_DCP_H__ */ diff --git a/optee_os/core/include/drivers/imx_i2c.h b/optee_os/core/include/drivers/imx_i2c.h new file mode 100644 index 0000000..7012b71 --- /dev/null +++ b/optee_os/core/include/drivers/imx_i2c.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2020 Foundries.io + * Jorge Ramirez-Ortiz + */ +#ifndef __DRIVERS_IMX_I2C_H +#define __DRIVERS_IMX_I2C_H + +#include +#include + +TEE_Result imx_i2c_write(uint8_t bid, uint8_t chip, const uint8_t *p, int l); +TEE_Result imx_i2c_read(uint8_t bid, uint8_t chip, uint8_t *p, int l); +TEE_Result imx_i2c_probe(uint8_t bid, uint8_t chip); +TEE_Result imx_i2c_init(uint8_t bid, int bps); + +#endif /*__DRIVERS_IMX_I2C_H*/ diff --git a/optee_os/core/include/drivers/imx_mu.h b/optee_os/core/include/drivers/imx_mu.h new file mode 100644 index 0000000..716af95 --- /dev/null +++ b/optee_os/core/include/drivers/imx_mu.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2020-2022 NXP + */ +#ifndef __DRIVERS_IMX_MU_H +#define __DRIVERS_IMX_MU_H + +#include +#include +#include + +#define IMX_MU_DATA_U32(mesg, idx) ((mesg)->data.u32[(idx)]) +#define IMX_MU_DATA_U16(mesg, idx) ((mesg)->data.u16[(idx)]) +#define IMX_MU_DATA_U8(mesg, idx) ((mesg)->data.u8[(idx)]) + +#define IMX_MU_MSG_SIZE 17 +#define IMX_MU_NB_CHANNEL 4 + +#if defined(CFG_MX8ULP) || defined(CFG_MX93) +struct imx_mu_msg_header { + uint8_t version; + uint8_t size; + uint8_t command; + uint8_t tag; +}; +#elif defined(CFG_MX8QM) || defined(CFG_MX8QX) || defined(CFG_MX8DXL) +struct imx_mu_msg_header { + uint8_t version; + uint8_t size; + uint8_t tag; + uint8_t command; +}; +#else +#error "Platform not supported" +#endif + +/* + * i.MX MU message format + * Note: the header format differs depending of the platform. + */ +struct imx_mu_msg { + struct imx_mu_msg_header header; + union { + uint32_t u32[IMX_MU_MSG_SIZE]; + uint16_t u16[IMX_MU_MSG_SIZE * 2]; + uint8_t u8[IMX_MU_MSG_SIZE * 4]; + } data; +}; + +/* + * Initialize the MU interface + * + * @base: virtual base address of the MU controller + */ +void imx_mu_init(vaddr_t base); + +/* + * Initiate a communication with the external controller. It sends a message + * and return the answer of the controller. + * + * @base: virtual base address of the MU controller + * @[in/out]msg: message sent and received + * @wait_for_answer: true if an answer from the controller is expected, false + * otherwise + */ +TEE_Result imx_mu_call(vaddr_t base, struct imx_mu_msg *msg, + bool wait_for_answer); +#endif /* __DRIVERS_IMX_MU_H */ diff --git a/optee_os/core/include/drivers/imx_ocotp.h b/optee_os/core/include/drivers/imx_ocotp.h new file mode 100644 index 0000000..72d4b79 --- /dev/null +++ b/optee_os/core/include/drivers/imx_ocotp.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + */ + +#ifndef _IMX_OCOTP_H +#define _IMX_OCOTP_H + +#include + +/* The i.MX UID is 64 bits long */ +#define IMX_UID_SIZE sizeof(uint64_t) + +/* + * Read OCOTP shadow register + * + * @bank Fuse bank number + * @word Fuse word number + * @[out]val Shadow register value + */ +TEE_Result imx_ocotp_read(unsigned int bank, unsigned int word, uint32_t *val); +#endif /* _IMX_OCOTP_H */ diff --git a/optee_os/core/include/drivers/imx_sc_api.h b/optee_os/core/include/drivers/imx_sc_api.h new file mode 100644 index 0000000..d8190ee --- /dev/null +++ b/optee_os/core/include/drivers/imx_sc_api.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2016 Freescale Semiconductor, Inc. + * Copyright 2017-2021 NXP + */ +#ifndef __DRIVERS_IMX_SC_API_H +#define __DRIVERS_IMX_SC_API_H + +#include + +/* + * Initializes the Secure Controller + */ +TEE_Result imx_sc_driver_init(void); + +/* + * Enables a CAAM Job Ring for the secure world with the following steps: + * - Assign CAAM JR resources to secure world + * - Power ON CAAM JR resources + * + * @jr_index Index of the CAAM Job Ring to enable + */ +TEE_Result imx_sc_rm_enable_jr(unsigned int jr_index); + +/* + * Starts the random number generator and returns the RNG status. + * + * Note that the RNG is started automatically after all CPUs are booted. This + * function can then be used to start the RNG earlier or to check the RNG + * status. + */ +TEE_Result imx_sc_seco_start_rng(void); +#endif /* __DRIVERS_IMX_SC_API_H */ diff --git a/optee_os/core/include/drivers/imx_snvs.h b/optee_os/core/include/drivers/imx_snvs.h new file mode 100644 index 0000000..74f9750 --- /dev/null +++ b/optee_os/core/include/drivers/imx_snvs.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2020 Pengutronix + * Rouven Czerwinski + * Copyright 2022-2023 NXP + */ +#ifndef __DRIVERS_IMX_SNVS_H +#define __DRIVERS_IMX_SNVS_H + +#include + +/* Set the OTPMK Key as Master key */ +#ifdef CFG_IMX_SNVS +TEE_Result imx_snvs_set_master_otpmk(void); +bool snvs_is_device_closed(void); +void imx_snvs_shutdown(void); +#else +static inline bool snvs_is_device_closed(void) +{ + return false; +} + +static inline TEE_Result imx_snvs_set_master_otpmk(void) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static inline void imx_snvs_shutdown(void) {} +#endif + +#endif /* __DRIVERS_IMX_SNVS_H */ diff --git a/optee_os/core/include/drivers/imx_uart.h b/optee_os/core/include/drivers/imx_uart.h new file mode 100644 index 0000000..b9df517 --- /dev/null +++ b/optee_os/core/include/drivers/imx_uart.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef IMX_UART_H +#define IMX_UART_H + +#include +#include + +struct imx_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void imx_uart_init(struct imx_uart_data *pd, paddr_t base); + +#endif /* IMX_UART_H */ diff --git a/optee_os/core/include/drivers/imx_wdog.h b/optee_os/core/include/drivers/imx_wdog.h new file mode 100644 index 0000000..2c76073 --- /dev/null +++ b/optee_os/core/include/drivers/imx_wdog.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017-2019 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __IMX_WDOG_H +#define __IMX_WDOG_H + +#include + +/* i.MX6/7D */ +#define WDT_WCR 0x00 +#define WDT_WCR_WDA BIT(5) +#define WDT_WCR_SRS BIT(4) +#define WDT_WCR_WRE BIT(3) +#define WDT_WCR_WDE BIT(2) +#define WDT_WCR_WDZST BIT(0) + +#define WDT_WSR 0x02 +#define WDT_SEQ1 0x5555 +#define WDT_SEQ2 0xAAAA + +/* 7ULP */ +#define WDOG_CNT 0x4 +#define WDOG_TOVAL 0x8 + +#define REFRESH_SEQ0 0xA602 +#define REFRESH_SEQ1 0xB480 +#define REFRESH ((REFRESH_SEQ1 << 16) | REFRESH_SEQ0) + +#define UNLOCK_SEQ0 0xC520 +#define UNLOCK_SEQ1 0xD928 +#define UNLOCK ((UNLOCK_SEQ1 << 16) | UNLOCK_SEQ0) + +#define WDOG_CS 0x0 +#define WDOG_CS_CMD32EN BIT(13) +#define WDOG_CS_ULK BIT(11) +#define WDOG_CS_RCS BIT(10) +#define WDOG_CS_EN BIT(7) +#define WDOG_CS_UPDATE BIT(5) + +/* Exposed for psci reset */ +void __noreturn imx_wdog_restart(bool external_reset); +#endif diff --git a/optee_os/core/include/drivers/lpc_uart.h b/optee_os/core/include/drivers/lpc_uart.h new file mode 100644 index 0000000..3e235b4 --- /dev/null +++ b/optee_os/core/include/drivers/lpc_uart.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, HiSilicon Limited + */ + +#ifndef LPC_UART_H +#define LPC_UART_H + +#include +#include + +#define UART_SEND_LOOP_MAX 1000000 +#define UART_THR 0x00 +#define UART_LSR 0x05 + +#define UART_USR_BUS 0x01 + +#define LPC_BASE 0x201190000 +#define LPC_SIZE 0x1000000 + +#define LPC_START_REG_OFFSET (0x00) +#define LPC_OP_STATUS_REG_OFFSET (0x04) +#define LPC_IRQ_ST_REG_OFFSET (0x08) +#define LPC_OP_LEN_REG_OFFSET (0x10) +#define LPC_CMD_REG_OFFSET (0x14) +#define LPC_FWH_ID_MSIZE_REG_OFFSET (0x18) +#define LPC_ADDR_REG_OFFSET (0x20) +#define LPC_WDATA_REG_OFFSET (0x24) +#define LPC_RDATA_REG_OFFSET (0x28) +#define LPC_LONG_CNT_REG_OFFSET (0x30) +#define LPC_TX_FIFO_ST_REG_OFFSET (0x50) +#define LPC_RX_FIFO_ST_REG_OFFSET (0x54) +#define LPC_TIME_OUT_REG_OFFSET (0x58) +#define LPC_SIRQ_CTRL0_REG_OFFSET (0x80) +#define LPC_SIRQ_CTRL1_REG_OFFSET (0x84) +#define LPC_SIRQ_INT_REG_OFFSET (0x90) +#define LPC_SIRQ_INT_MASK_REG_OFFSET (0x94) +#define LPC_SIRQ_STAT_REG_OFFSET (0xa0) + +#define LPC_SINGLE_READ (0x8) +#define LPC_SINGLE_WRITE (0x9) +#define LPC_IRQ_ST_ON (0x2) +#define LPC_RADTA_LEN (0x40) + +struct lpc_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void lpc_uart_init(struct lpc_uart_data *pd, paddr_t base, + uint32_t uart_clk, uint32_t baud_rate); + +#endif /* LPC_UART_H */ diff --git a/optee_os/core/include/drivers/ls_dspi.h b/optee_os/core/include/drivers/ls_dspi.h new file mode 100644 index 0000000..0c83f2b --- /dev/null +++ b/optee_os/core/include/drivers/ls_dspi.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + * + * Helper Code for DSPI Controller driver + */ + +#ifndef __DRIVERS_LS_DSPI_H +#define __DRIVERS_LS_DSPI_H + +#include +#include + +/* Clock and transfer attributes */ +#define DSPI_CTAR_BRD 0x80000000 /* Double Baud Rate [0] */ +#define DSPI_CTAR_FMSZ(x) (((x) & 0x0F) << 27) /* Frame Size [1-4] */ +#define DSPI_CTAR_CPOL 0x04000000 /* Clock Polarity [5] */ +#define DSPI_CTAR_CPHA 0x02000000 /* Clock Phase [6] */ +#define DSPI_CTAR_LSBFE 0x01000000 /* LSB First [7] */ +#define DSPI_CTAR_PCS_SCK(x) (((x) & 0x03) << 22) /* PCSSCK [8-9] */ +#define DSPI_CTAR_PA_SCK(x) (((x) & 0x03) << 20) /* PASC [10-11] */ +#define DSPI_CTAR_P_DT(x) (((x) & 0x03) << 18) /* PDT [12-13] */ +#define DSPI_CTAR_BRP(x) \ + (((x) & 0x03) << 16) /* Baud Rate Prescaler [14-15] */ +#define DSPI_CTAR_CS_SCK(x) (((x) & 0x0F) << 12) /* CSSCK [16-19] */ +#define DSPI_CTAR_A_SCK(x) (((x) & 0x0F) << 8) /* ASC [20-23] */ +#define DSPI_CTAR_A_DT(x) (((x) & 0x0F) << 4) /* DT [24-27] */ +#define DSPI_CTAR_BR(x) ((x) & 0x0F) /* Baud Rate Scaler [28-31] */ + +/* SPI mode flags */ +#define SPI_CPHA BIT(0) /* clock phase */ +#define SPI_CPOL BIT(1) /* clock polarity */ +#define SPI_CS_HIGH BIT(2) /* CS active high */ +#define SPI_LSB_FIRST BIT(3) /* per-word bits-on-wire */ +#define SPI_CONT BIT(4) /* Continuous CS mode */ + +/* + * struct ls_dspi_data describes DSPI controller chip instance + * The structure contains below members: + * chip: generic spi_chip instance + * base: DSPI controller base address + * bus_clk_hz: DSPI input clk frequency + * speed_hz: Default SCK frequency=1mhz + * num_chipselect: chip/slave selection + * slave_bus: bus value of slave + * slave_cs: chip slect value of slave + * slave_speed_max_hz: max spped of slave + * slave_mode: mode of slave + * slave_data_size_bits:Data size to be transferred (8 or 16 bits) + * ctar_val: value of Clock and Transfer Attributes Register (CTAR) + * ctar_sel: CTAR0 or CTAR1 + */ +struct ls_dspi_data { + struct spi_chip chip; + vaddr_t base; + unsigned int bus_clk_hz; + unsigned int speed_hz; + unsigned int num_chipselect; + unsigned int slave_bus; + unsigned int slave_cs; + unsigned int slave_speed_max_hz; + unsigned int slave_mode; + unsigned int slave_data_size_bits; + unsigned int ctar_val; + unsigned int ctar_sel; +}; + +/* + * Initialize DSPI Controller + * dspi_data: DSPI controller chip instance + */ +TEE_Result ls_dspi_init(struct ls_dspi_data *dspi_data); + +#endif /* __DRIVERS_LS_DSPI_H */ diff --git a/optee_os/core/include/drivers/ls_gpio.h b/optee_os/core/include/drivers/ls_gpio.h new file mode 100644 index 0000000..594fb35 --- /dev/null +++ b/optee_os/core/include/drivers/ls_gpio.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + * + * Helper Code for GPIO controller driver + * + */ + +#ifndef __DRIVERS_LS_GPIO_H +#define __DRIVERS_LS_GPIO_H + +#include +#include +#include +#include + +/* supported ports for GPIO controller */ +#define MAX_GPIO_PINS U(31) + +/* map register values to LE by subtracting pin number from MAX GPIO PINS */ +#define PIN_SHIFT(x) BIT(MAX_GPIO_PINS - (x)) + +/* gpio register offsets */ +#define GPIODIR U(0x0) /* direction register */ +#define GPIOODR U(0x4) /* open drain register */ +#define GPIODAT U(0x8) /* data register */ +#define GPIOIER U(0xc) /* interrupt event register */ +#define GPIOIMR U(0x10) /* interrupt mask register */ +#define GPIOICR U(0x14) /* interrupt control register */ +#define GPIOIBE U(0x18) /* input buffer enable register */ + +/* + * struct ls_gpio_chip_data describes GPIO controller chip instance + * The structure contains below members: + * chip: generic GPIO chip handle. + * gpio_base: starting GPIO module base address managed by this GPIO + * controller. + * gpio_controller: GPIO controller to be used. + */ +struct ls_gpio_chip_data { + struct gpio_chip chip; + vaddr_t gpio_base; + uint8_t gpio_controller; +}; + +/* + * Initialize GPIO Controller + * gpio_data is a pointer of type 'struct ls_gpio_chip_data'. + */ +TEE_Result ls_gpio_init(struct ls_gpio_chip_data *gpio_data); + +#endif /* __DRIVERS_LS_GPIO_H */ diff --git a/optee_os/core/include/drivers/ls_i2c.h b/optee_os/core/include/drivers/ls_i2c.h new file mode 100644 index 0000000..88ef5fa --- /dev/null +++ b/optee_os/core/include/drivers/ls_i2c.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + * I2C driver for I2C Controller + */ + +#ifndef __DRIVERS_LS_I2C_H +#define __DRIVERS_LS_I2C_H + +#include +#include +#include + +/* + * Module Disable + * 0b - The module is enabled. You must clear this field before any other IBCR + * fields have any effect. + * 1b - The module is reset and disabled. This is the power-on reset situation. + * When high, the interface is held in reset, but registers can still be + * accessed. Status register fields (IBSR) are not valid when the module + * is disabled. + */ +#define I2C_IBCR_MDIS U(0x80) + +/* I2c Bus Interrupt Enable */ +#define I2C_IBCR_IBIE U(0x40) + +/* + * Master / Slave Mode 0b - Slave mode 1b - Master mode + * When you change this field from 0 to 1, the module generates a START signal + * on the bus and selects the master mode. When you change this field from 1 to + * 0, the module generates a STOP signal and changes the operation mode from + * master to slave. You should generate a STOP signal only if IBSR[IBIF]=1. + * The module clears this field without generating a STOP signal when the + * master loses arbitration. + */ +#define I2C_IBCR_MSSL U(0x20) + +/* 0b - Receive 1b - Transmit */ +#define I2C_IBCR_TXRX U(0x10) + +/* + * Data acknowledge disable + * Values written to this field are only used when the I2C module is a receiver, + * not a transmitter. + * 0b - The module sends an acknowledge signal to the bus at the 9th clock bit + * after receiving one byte of data. + * 1b - The module does not send an acknowledge-signal response (that is, + * acknowledge bit = 1). + */ +#define I2C_IBCR_NOACK U(0x08) + +/* + * Repeat START + * If the I2C module is the current bus master, and you program RSTA=1, the I2C + * module generates a repeated START condition. This field always reads as a 0. + * If you attempt a repeated START at the wrong time, if the bus is owned by + * another master the result is loss of arbitration. + */ +#define I2C_IBCR_RSTA U(0x04) + +/* DMA enable */ +#define I2C_IBCR_DMAEN U(0x02) + +/* Transfer Complete */ +#define I2C_IBSR_TCF U(0x80) + +/* I2C bus Busy. 0b - Bus is idle, 1b - Bus is busy */ +#define I2C_IBSR_IBB U(0x20) + +/* Arbitration Lost. software must clear this field by writing a one to it. */ +#define I2C_IBSR_IBAL U(0x10) + +/* I2C bus interrupt flag */ +#define I2C_IBSR_IBIF U(0x02) + +/* + * Received acknowledge + * 0b - Acknowledge received + * 1b - No acknowledge received + */ +#define I2C_IBSR_RXAK U(0x01) + +/* Bus idle interrupt enable */ +#define I2C_IBIC_BIIE U(0x80) + +/* Glitch filter enable */ +#define I2C_IBDBG_GLFLT_EN U(0x08) + +#define I2C_FLAG_WRITE U(0x00000000) +#define I2C_FLAG_READ U(0x00000001) + +#define I2C_BUS_TEST_BUSY true +#define I2C_BUS_TEST_IDLE !I2C_BUS_TEST_BUSY +#define I2C_BUS_TEST_RX_ACK true +#define I2C_BUS_NO_TEST_RX_ACK !I2C_BUS_TEST_RX_ACK + +#define I2C_NUM_RETRIES U(500) + +struct i2c_regs { + uint8_t ibad; /* I2c Bus Address Register */ + uint8_t ibfd; /* I2c Bus Frequency Divider Register */ + uint8_t ibcr; /* I2c Bus Control Register */ + uint8_t ibsr; /* I2c Bus Status Register */ + uint8_t ibdr; /* I2C Bus Data I/O Register */ + uint8_t ibic; /* I2C Bus Interrupt Config Register */ + uint8_t ibdbg; /* I2C Bus Debug Register */ +}; + +/* + * sorted list of clock divisor, ibfd register value pairs + */ +struct i2c_clock_divisor_pair { + uint16_t divisor; + uint8_t ibfd; /* I2c Bus Frequency Divider Register value */ +}; + +/* + * I2C device operation + * The i2c_operation describes a subset of an I2C transaction in which + * the I2C controller is either sending or receiving bytes from the bus. + * Some transactions will consist of a single operation while others will + * be two or more. + */ +struct i2c_operation { + /* Flags to qualify the I2C operation. */ + uint32_t flags; + + /* + * Number of bytes to send to or receive from the I2C device.A ping + * is indicated by setting the length_in_bytes to zero. + */ + unsigned int length_in_bytes; + + /* + * Pointer to a buffer containing the data to send or to receive from + * the I2C device. The Buffer must be at least length_in_bytes in size. + */ + uint8_t *buffer; +}; + +/* + * Structure to initialize I2C controller. + */ +struct ls_i2c_data { + /* I2C Controller to initialize */ + uint8_t i2c_controller; + /* + * base will be filled by i2c_init() which will be used in + * subsequent calls for reading/writing data. + */ + vaddr_t base; + /* I2C bus clock frequency */ + uint64_t i2c_bus_clock; + /* I2C speed */ + uint64_t speed; +}; + +/* + * Structure to fill for I2C read/write operation + */ +struct i2c_reg_request { + /* + * Number of operations to perform. + * This will depend on peripheral for which it is used. + */ + unsigned int operation_count; + /* Operation to perform */ + struct i2c_operation *operation; +}; + +/* + * Initialize I2C Controller, based on data passed in + * i2c_data. + */ +TEE_Result i2c_init(struct ls_i2c_data *i2c_data); + +/* + * Software reset of the entire I2C module. + * The module is reset and disabled. + * Status register fields (IBSR) are cleared. + * base Base Address of I2c controller's registers + */ +void i2c_reset(vaddr_t base); + +/* + * Transfer data to/from I2c slave device + * base Base Address of I2c controller's registers + * slave_address Slave Address from which data is to be read + * i2c_operation Pointer to an i2c_operation structure + * operation_count Number of operations. + */ +TEE_Result i2c_bus_xfer(vaddr_t base, uint32_t slave_address, + struct i2c_operation *i2c_operation, + unsigned int operation_count); + +#endif /* __DRIVERS_LS_I2C_H */ diff --git a/optee_os/core/include/drivers/ls_sec_mon.h b/optee_os/core/include/drivers/ls_sec_mon.h new file mode 100644 index 0000000..2906e7a --- /dev/null +++ b/optee_os/core/include/drivers/ls_sec_mon.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 Microsoft + * + * Definitions for the NXP LX2160A-series Security Monitor (SecMon) driver. + */ + +#ifndef __DRIVERS_LS_SEC_MON_H +#define __DRIVERS_LS_SEC_MON_H + +#include +#include + +/** + * struct ls_sec_mon_data - Compact data struct of all SecMon registers. + * @hplr: HP Lock Register. + * @hpcomr: HP Command Register. + * @hpsicr: HP Security Interrupt Control Register. + * @hpsvcr: HP Security Violation Control Register. + * @hpsr: HP Status Register. + * @hpsvsr: HP Security Violation Status Register. + * @hphacivr: HP High Assurance Counter IV Register. + * @hphacr: HP High Assurance Counter Register. + * @lplr: LP Lock Register. + * @lpcr: LP Control Register. + * @lpmkcr: LP Master Key Control Register. + * @lpsvcr: LP Security Violation Control Register. + * @lptdcr: LP Tamper Detectors Configuration Register. + * @lpsr: LP Status Register. + * @lpsmcmr: LP Secure Monotonic Counter MSB Register. + * @lpsmclr: LP Secure Monotonic Counter LSB Register. + * @lppgdr: LP Power Glitch Detector Register. + * @lpzmkr[8]: LP Zeroizable Master Key Registers. + * @lpgpr[4]: LP General Purpose Registers. + * @hpvidr1: HP Version ID Register 1. + * @hpvidr2: HP Version ID Register 2. + */ +struct ls_sec_mon_data { + uint32_t hplr; + uint32_t hpcomr; + uint32_t hpsicr; + uint32_t hpsvcr; + uint32_t hpsr; + uint32_t hpsvsr; + uint32_t hphacivr; + uint32_t hphacr; + uint32_t lplr; + uint32_t lpcr; + uint32_t lpmkcr; + uint32_t lpsvcr; + uint32_t lptdcr; + uint32_t lpsr; + uint32_t lpsmcmr; + uint32_t lpsmclr; + uint32_t lppgdr; + uint32_t lpzmkr[8]; + uint32_t lpgpr[4]; + uint32_t hpvidr1; + uint32_t hpvidr2; +}; + +/** + * ls_sec_mon_read() - Read a copy of the SecMon register data if the SecMon + * driver was successfully initialized. + * @data: Location to save SecMon data. + * + * Return: 0 if successful or > 0 on error. + */ +TEE_Result ls_sec_mon_read(struct ls_sec_mon_data *data); + +/** + * ls_sec_mon_status() - Check if the SecMon driver was initialized + * successfully. + * + * Return: 0 if init was successful or TEE_ERROR_GENERIC on init failed. + */ +TEE_Result ls_sec_mon_status(void); + +#endif /* __DRIVERS_LS_SEC_MON_H */ diff --git a/optee_os/core/include/drivers/ls_sfp.h b/optee_os/core/include/drivers/ls_sfp.h new file mode 100644 index 0000000..929a1fe --- /dev/null +++ b/optee_os/core/include/drivers/ls_sfp.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 Microsoft + * + * Definitions for the NXP LX2160A-series Security Fuse Processor (SFP) driver. + */ + +#ifndef __DRIVERS_LS_SFP_H +#define __DRIVERS_LS_SFP_H + +#include +#include +#include + +/* SFP instruction register */ +/* SFP is big endian */ +#define SFP_INGR_PROGFB_CMD 0x2 +#define SFP_INGR_ERROR_MASK 0x100 +#define SFP_INGR_FUSE_TIMEOUT_US 150000 + +/* SFP configuration register */ +#define SFP_SFPCR_SB_MASK 0x20000000 +#define SFP_SFPCR_SB_OFFSET 29 + +/* SFP OEM security policy register 0 */ +#define SFP_OSPR0_WP_MASK 0x1 +#define SFP_OSPR0_ITS_MASK 0x4 +#define SFP_OSPR0_ITS_OFFSET 0x2 + +/* SFP OEM security policy register 1 */ +#define SFP_OSPR1_DBLEV_MASK 0x7 +#define SFP_OSPR1_DBLEV_OPEN 0x0 +#define SFP_OSPR1_DBLEV_CLOSED_NOTIFY 0x1 +#define SFP_OSPR1_DBLEV_CLOSED_SILENT 0x2 +#define SFP_OSPR1_DBLEV_CLOSED 0x4 + +/** + * struct ls_sfp_data - Compact data struct of all SFP registers. + * @ingr: Instruction Register. + * @svhesr: Secret Value Hamming Error Status Register. + * @sfpcr: SFP Configuration Register. + * @version: SFP Version Register. + * @ospr0: OEM Security Policy Register 0. + * @ospr1: OEM Security Policy Register 1. + * @dcvr0: Debug Challenge Value Register 0. + * @dcvr1: Debug Challenge Value Register 1. + * @drvr0: Debug Response Value Register 0. + * @drvr1: Debug Response Value Register 1. + * @fswpr: Factory Section Write Protect Register. + * @fuidr0: Factory Unique ID Register 0. + * @fuidr1: Factory Unique ID Register 1. + * @isbccr: ISBC Configuration Register. + * @fspfr[0x3]: Factory Scratch Pad Fuse Registers. + * @otpmkr[0x8]: One Time Programmable Master Key. + * @srkhr[0x8]: Super Root Key Hash Register. + * @ouidr[0x5]: OEM Unique ID/Scratch Pad Fuse Registers. + */ +struct ls_sfp_data { + uint32_t ingr; + uint32_t svhesr; + uint32_t sfpcr; + uint32_t version; + uint32_t ospr0; + uint32_t ospr1; + uint32_t dcvr0; + uint32_t dcvr1; + uint32_t drvr0; + uint32_t drvr1; + uint32_t fswpr; + uint32_t fuidr0; + uint32_t fuidr1; + uint32_t isbccr; + uint32_t fspfr[0x3]; + uint32_t otpmkr[0x8]; + uint32_t srkhr[0x8]; + uint32_t ouidr[0x5]; +}; + +/** + * ls_sfp_read() - Read a copy of the SFP register data if the SFP driver was + * successfully initialized. + * @data: Location to save SFP data. + * + * Return: TEE_SUCCESS or > 0 on error + */ +TEE_Result ls_sfp_read(struct ls_sfp_data *data); + +/** + * ls_sfp_get_debug_level() - Read the last 3 bits of the SFP OSPR1 register + * which denotes the debug level. + * @dblev: Pointer location to store the read debug level. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_get_debug_level(uint32_t *dblev); + +/** + * ls_sfp_get_its() - Read bit 29 of the SFP OSPR0 register which denotes the + * ITS flag. + * @its: Pointer location to store the ITS flag. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_get_its(uint32_t *its); + +/** + * ls_sfp_get_ouid() - Read the SFP OUID register at the given index. + * @index: Index of the OUID register to read. + * @ouid: Pointer location to store the OIUD register value. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_get_ouid(uint32_t index, uint32_t *ouid); + +/** + * ls_sfp_get_sb() - Read bit 2 of the SFP SFPCR register which denotes the + * secure boot flag. + * @sb: Pointer location to store the secure boot flag. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_get_sb(uint32_t *sb); + +/** + * ls_sfp_get_srkh() - Read the SFP SRKH register at the given index. + * @index: Index of the SRKH register to read. + * @srkh: Pointer location to store the SRKH register value. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_get_srkh(uint32_t index, uint32_t *srkh); + +/** + * ls_sfp_set_debug_level() - Set the last 3 bits of the SFP OSPR1 register + * which denotes the debug level. + * @dblev: Value to write into the SFP OSPR1 register. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_set_debug_level(uint32_t dblev); + +/** + * ls_sfp_set_its_wp() - Set bits 29 and 31 of the SFP OSPR0 register which + * denote the ITS and write protect flags respectively. + * + * WARNING - Setting the ITS and write protect flags will lock the mirror + * registers and permanently prevent any further programming of the fuse block. + * The system will also be forced to always attempt to secure boot which + * requires signature validation and the absence of any hardware security + * violations when booting. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_set_its_wp(void); + +/** + * ls_sfp_set_ouid() - Write to the SFP OUID register at the given index. + * @index: Index of the OUID register to write. + * @ouid: Value to write into the SFP OUID register. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_set_ouid(uint32_t index, uint32_t ouid); + +/** + * ls_sfp_status() - Check if the SFP driver was initialized successfully. + * + * Return: TEE_SUCCESS or > 0 on error. + */ +TEE_Result ls_sfp_status(void); + +#endif /* __DRIVERS_LS_SFP_H */ diff --git a/optee_os/core/include/drivers/mvebu_uart.h b/optee_os/core/include/drivers/mvebu_uart.h new file mode 100644 index 0000000..fed9ac8 --- /dev/null +++ b/optee_os/core/include/drivers/mvebu_uart.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2017 Marvell International Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef MVEBU_UART_H +#define MVEBU_UART_H + +#include +#include + +struct mvebu_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void mvebu_uart_init(struct mvebu_uart_data *pd, paddr_t pbase, + uint32_t uart_clk, uint32_t baud_rate); + +#endif /* MVEBU_UART_H */ diff --git a/optee_os/core/include/drivers/ns16550.h b/optee_os/core/include/drivers/ns16550.h new file mode 100644 index 0000000..de0be9e --- /dev/null +++ b/optee_os/core/include/drivers/ns16550.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2015 Freescale Semiconductor, Inc. + * Copyright (c) 2020, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef NS16550_H +#define NS16550_H + +#include +#include +#include + +#define NS16550_UART_REG_SIZE 0x1000 + +#define IO_WIDTH_U8 0 +#define IO_WIDTH_U32 1 + +struct ns16550_data { + struct io_pa_va base; + struct serial_chip chip; + uint8_t io_width; + uint8_t reg_shift; +}; + +static inline unsigned int serial_in(vaddr_t addr, uint8_t io_width) +{ + if (io_width == IO_WIDTH_U32) + return io_read32(addr); + else + return io_read8(addr); +} + +static inline void serial_out(vaddr_t addr, uint8_t io_width, int ch) +{ + if (io_width == IO_WIDTH_U32) + io_write32(addr, ch); + else + io_write8(addr, ch); +} + +void ns16550_init(struct ns16550_data *pd, paddr_t base, uint8_t io_width, + uint8_t reg_shift); + +#endif /* NS16550_H */ diff --git a/optee_os/core/include/drivers/pinctrl.h b/optee_os/core/include/drivers/pinctrl.h new file mode 100644 index 0000000..bdf7166 --- /dev/null +++ b/optee_os/core/include/drivers/pinctrl.h @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022-2023, Microchip + */ + +#ifndef __DRIVERS_PINCTRL_H +#define __DRIVERS_PINCTRL_H + +#include +#include +#include +#include + +enum pinctrl_dt_prop { + /* Property "bias-disable" found in pinctrl node */ + PINCTRL_DT_PROP_BIAS_DISABLE, + /* Property "bias-pull-up" found in pinctrl node */ + PINCTRL_DT_PROP_BIAS_PULL_UP, + /* Property "bias-pull-down" found in pinctrl node */ + PINCTRL_DT_PROP_BIAS_PULL_DOWN, + /* Terminal ID */ + PINCTRL_DT_PROP_MAX +}; + +/* + * struct pinconf - Pinctrl device + * @ops: Operation handlers + * @priv: Pinctrl driver private data + */ +struct pinconf { + const struct pinctrl_ops *ops; + void *priv; +}; + +/* + * struct pinctrl_state - Pinctrl configuration state + * @conf_count: Number of cells in @confs + * @confs: Array of pin configurations related to the pinctrl config state + */ +struct pinctrl_state { + unsigned int conf_count; + struct pinconf *confs[]; +}; + +struct pinctrl_ops { + /* Apply a pinctrl configuration */ + TEE_Result (*conf_apply)(struct pinconf *conf); + /* Release resources allocated for a pinctrl configuration */ + void (*conf_free)(struct pinconf *conf); +}; + +/** + * pinctrl_dt_get_func - Typedef of function to get a pin configuration from + * a device tree property + * + * @args: Pointer to device tree phandle arguments of the pin control reference + * @data: Pointer to data given at pinctrl_register_provider() call + * @out_pinconf: Output pin configuration reference upon success + */ +typedef TEE_Result (*pinctrl_dt_get_func)(struct dt_pargs *pargs, void *data, + struct pinconf **out_pinconf); + +#ifdef CFG_DRIVERS_PINCTRL +/** + * pinctrl_dt_register_provider - Register a pinctrl controller provider + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the pin controller + * @get_pinctrl: Callback to match the pin controller with a struct pinconf + * @data: Data which will be passed to the get_pinctrl callback + * Return a TEE_Result compliant value + */ +static inline TEE_Result pinctrl_register_provider(const void *fdt, + int nodeoffset, + pinctrl_dt_get_func func, + void *data) +{ + return dt_driver_register_provider(fdt, nodeoffset, + (get_of_device_func)func, data, + DT_DRIVER_PINCTRL); +} + +/** + * pinctrl_get_state_by_name - Obtain a pinctrl state by name + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the pin controller + * @name: name of the pinctrl state to obtain from device-tree + * @state: Pointer filled with the retrieved state, must be freed after use + using pinctrl_free_state() + * Return a TEE_Result compliant value + */ +TEE_Result pinctrl_get_state_by_name(const void *fdt, int nodeoffset, + const char *name, + struct pinctrl_state **state); + +/** + * pinctrl_get_state_by_idx - Obtain a pinctrl state by index + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the pin controller + * @pinctrl_id: Index of the pinctrl state to obtain from device-tree + * @state: Pointer filled with the retrieved state, must be freed after use + using pinctrl_free_state() + * Return a TEE_Result compliant value + */ +TEE_Result pinctrl_get_state_by_idx(const void *fdt, int nodeoffset, + unsigned int pinctrl_id, + struct pinctrl_state **state); + +/** + * pinctrl_free_state - Free a pinctrl state that was previously obtained + * + * @state: State to be freed + */ +void pinctrl_free_state(struct pinctrl_state *state); + +/** + * pinctrl_apply_state - apply a pinctrl state + * + * @state: State to be applied + * Return a TEE_Result compliant value + */ +TEE_Result pinctrl_apply_state(struct pinctrl_state *state); + +/* + * pinctrl_parse_dt_pin_modes - Parse DT node properties + * @fdt: Device tree to work on + * @nodeoffset: Pinctrl node + * @modes: Output allocated regulator properties + * Return a TEE_Result compliant value + */ +TEE_Result pinctrl_parse_dt_pin_modes(const void *fdt, int nodeoffset, + bitstr_t **modes); +#else /* CFG_DRIVERS_PINCTRL */ +static inline TEE_Result +pinctrl_register_provider(const void *fdt __unused, int nodeoffset __unused, + get_of_device_func func __unused, void *data __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result +pinctrl_get_state_by_name(const void *fdt __unused, int nodeoffset __unused, + const char *name __unused, + struct pinctrl_state **state __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result +pinctrl_get_state_by_idx(const void *fdt __unused, int nodeoffset __unused, + unsigned int pinctrl_id __unused, + struct pinctrl_state **state __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline void pinctrl_free_state(struct pinctrl_state *state __unused) +{ +} + +static inline TEE_Result pinctrl_apply_state(struct pinctrl_state *s __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result pinctrl_parse_dt_pin_modes(const void *fdt __unused, + int nodeoffset __unused, + bitstr_t **modes __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +#endif /* CFG_DRIVERS_PINCTRL */ +#endif /* __DRIVERS_PINCTRL_H */ diff --git a/optee_os/core/include/drivers/pl011.h b/optee_os/core/include/drivers/pl011.h new file mode 100644 index 0000000..b46880e --- /dev/null +++ b/optee_os/core/include/drivers/pl011.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ +#ifndef PL011_H +#define PL011_H + +#include +#include + +#define PL011_REG_SIZE 0x1000 + +struct pl011_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void pl011_init(struct pl011_data *pd, paddr_t pbase, uint32_t uart_clk, + uint32_t baud_rate); + +#endif /* PL011_H */ diff --git a/optee_os/core/include/drivers/pl022_spi.h b/optee_os/core/include/drivers/pl022_spi.h new file mode 100644 index 0000000..c20b01b --- /dev/null +++ b/optee_os/core/include/drivers/pl022_spi.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * + */ + +#ifndef __PL022_SPI_H__ +#define __PL022_SPI_H__ + +#include +#include + +#define PL022_REG_SIZE 0x1000 + +enum pl022_cs_control { + PL022_CS_CTRL_AUTO_GPIO, + PL022_CS_CTRL_CB, + PL022_CS_CTRL_MANUAL +}; + +struct pl022_cs_gpio_data { + struct gpio_chip *chip; + unsigned int pin_num; +}; + +union pl022_cs_data { + struct pl022_cs_gpio_data gpio_data; + void (*cs_cb)(enum gpio_level value); +}; + +struct pl022_data { + union pl022_cs_data cs_data; + struct spi_chip chip; + vaddr_t base; + enum spi_mode mode; + enum pl022_cs_control cs_control; + unsigned int clk_hz; + unsigned int speed_hz; + unsigned int data_size_bits; + bool loopback; +}; + +void pl022_init(struct pl022_data *pd); + +#endif /* __PL022_SPI_H__ */ + diff --git a/optee_os/core/include/drivers/pl061_gpio.h b/optee_os/core/include/drivers/pl061_gpio.h new file mode 100644 index 0000000..1676b2f --- /dev/null +++ b/optee_os/core/include/drivers/pl061_gpio.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef __PL061_GPIO_H__ +#define __PL061_GPIO_H__ + +#include +#include + +#define PL061_REG_SIZE 0x1000 + +enum pl061_mode_control { + PL061_MC_SW, + PL061_MC_HW +}; + +struct pl061_data { + struct gpio_chip chip; +}; + +void pl061_register(vaddr_t base_addr, unsigned int gpio_dev); +void pl061_init(struct pl061_data *pd); +enum pl061_mode_control pl061_get_mode_control(unsigned int gpio_pin); +void pl061_set_mode_control(unsigned int gpio_pin, + enum pl061_mode_control hw_sw); + +#endif /* __PL061_GPIO_H__ */ diff --git a/optee_os/core/include/drivers/plic.h b/optee_os/core/include/drivers/plic.h new file mode 100644 index 0000000..148fc1e --- /dev/null +++ b/optee_os/core/include/drivers/plic.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#ifndef DRIVERS_PLIC_H +#define DRIVERS_PLIC_H + +#include +#include +#include + +void plic_init(paddr_t plic_base_pa); +void plic_hart_init(void); +void plic_it_handle(void); +void plic_dump_state(void); + +#endif /*DRIVERS_PLIC_H*/ diff --git a/optee_os/core/include/drivers/pm/sam/atmel_pm.h b/optee_os/core/include/drivers/pm/sam/atmel_pm.h new file mode 100644 index 0000000..a9801d6 --- /dev/null +++ b/optee_os/core/include/drivers/pm/sam/atmel_pm.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Microchip + */ +#ifndef __DRIVERS_PM_SAM_ATMEL_PM_H +#define __DRIVERS_PM_SAM_ATMEL_PM_H + +#include +#include +#include +#include +#include +#include + +struct sm_nsec_ctx; + +#if defined(CFG_ATMEL_PM) + +static inline bool atmel_pm_suspend_available(void) +{ + return true; +} + +void atmel_pm_cpu_idle(void); + +TEE_Result atmel_pm_suspend(uintptr_t entry, struct sm_nsec_ctx *nsec); + +TEE_Result sama5d2_pm_init(const void *fdt, vaddr_t shdwc); + +enum sm_handler_ret at91_pm_set_suspend_mode(struct thread_smc_args *args); + +enum sm_handler_ret at91_pm_get_suspend_mode(struct thread_smc_args *args); + +#else + +static inline void atmel_pm_cpu_idle(void) {}; + +static inline bool atmel_pm_suspend_available(void) +{ + return false; +} + +static inline TEE_Result atmel_pm_suspend(uintptr_t entry __unused, + struct sm_nsec_ctx *nsec __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result sama5d2_pm_init(const void *fdt __unused, + vaddr_t shdwc __unused) +{ + return TEE_SUCCESS; +} + +static inline enum sm_handler_ret +at91_pm_set_suspend_mode(struct thread_smc_args *args __unused) +{ + return SM_HANDLER_PENDING_SMC; +} + +static inline enum sm_handler_ret +at91_pm_get_suspend_mode(struct thread_smc_args *args __unused) +{ + return SM_HANDLER_PENDING_SMC; +} + +#endif + +#endif /* __DRIVERS_PM_SAM_ATMEL_PM_H */ diff --git a/optee_os/core/include/drivers/regulator.h b/optee_os/core/include/drivers/regulator.h new file mode 100644 index 0000000..e7c52a7 --- /dev/null +++ b/optee_os/core/include/drivers/regulator.h @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023, STMicroelectronics + */ +#ifndef DRIVERS_REGULATOR_H +#define DRIVERS_REGULATOR_H + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Regulator property flags: related to device tree binding properties */ + +/* Regulator should never be disabled. DT property: regulator-always-on */ +#define REGULATOR_ALWAYS_ON BIT(0) +/* Enables pull down mode. DT property: regulator-pull-down */ +#define REGULATOR_PULL_DOWN BIT(1) +/* + * It's expected that this regulator was left on by the bootloader. + * The core shouldn't prevent it from being turned off later. + * DT property: regulator-boot-on + */ +#define REGULATOR_BOOT_ON BIT(2) + +#define REGULATOR_FLAGS_MASK (REGULATOR_ALWAYS_ON | REGULATOR_PULL_DOWN | \ + REGULATOR_BOOT_ON) + +struct regulator_ops; + +/* + * struct regu_dt_desc - Regulator description passed to regulator_dt_register() + * @priv: Regulator driver private data + * @name: Regulator string name for debug purpose + * @supply_name: Regulator supply name for node property *-supply or NULL + * @ops: Operation handlers for the regulator + * @regulator: Pointer to preallocated regulator or NULL if none + */ +struct regu_dt_desc { + void *priv; + const char *name; + const char *supply_name; + const struct regulator_ops *ops; + struct regulator *regulator; +}; + +/* + * Defines the format of struct voltages::entries + * + * If regulator_voltages::type is VOLTAGE_TYPE_FULL_LIST, then + * regulator_voltages@entries stores regulator_voltages::num_levels cells, + * listing supported voltage levels in uV from lowest to highest value. + * + * If regulator_voltages::type is VOLTAGE_TYPE_INCREMENT, then + * regulator_voltages::entries stores 3 cells: min level, max level and + * level increment step, all in uV. When so, regulator_voltages::num_levels + * is meaningless. + */ +enum voltage_type { + VOLTAGE_TYPE_INVALID = 0, + VOLTAGE_TYPE_FULL_LIST, /* extensive list in uV */ + VOLTAGE_TYPE_INCREMENT /* min, max, increment (in uV) */ +}; + +/* + * struct regulator_voltages - Voltage levels description + * @type: Type of level description + * @num_levels: Number of cells of @entries when @type is VOLTAGE_TYPE_FULL_LIST + * @entries: Voltage level information in uV + * + */ +struct regulator_voltages { + enum voltage_type type; + size_t num_levels; + int entries[]; +}; + +/* + * struct regulator - A regulator instance + * @ops: Operation handlers for the regulator + * @supply: Regulator supply reference or NULL if none + * @priv: Regulator driver private data + * @name: Regulator string name for debug purpose or NULL + * @min_uv: Min possible voltage level in microvolt (uV) + * @max_uv: Max possible voltage level in microvolt (uV) + * @cur_uv: Current voltage level in microvolt (uV) + * @flags: REGULATOR_* property flags + * @refcount: Regulator enable request reference counter + * @lock: Mutex for concurrent access protection + * @voltages_fallback: Default supported voltage range description + * @link: Link in initialized regulator list + */ +struct regulator { + /* Fields initialized by caller of regulator_register() */ + const struct regulator_ops *ops; + struct regulator *supply; + void *priv; + char *name; + int min_uv; + int max_uv; + /* Fields internal to regulator framework */ + int cur_uv; + unsigned int flags; + unsigned int refcount; + struct mutex lock; /* Concurrent access protection */ + struct voltages_fallback { + struct regulator_voltages desc; + int levels[3]; + } voltages_fallback; + size_t levels_count_fallback; + SLIST_ENTRY(regulator) link; +}; + +/* + * struct regulator_ops - Regulator operation handlers + * + * @set_state: Enable or disable a regulator + * @get_state: Get regulator effective state + * @set_voltage: Set voltage level in microvolt (uV) + * @get_voltage: Get current voltage in microvolt (uV) + * @supported_voltages: Get supported levels description + * @supplied_init: Optional, finalize initialization once supply is ready + */ +struct regulator_ops { + TEE_Result (*set_state)(struct regulator *r, bool enabled); + TEE_Result (*get_state)(struct regulator *r, bool *enabled); + TEE_Result (*set_voltage)(struct regulator *r, int level_uv); + TEE_Result (*get_voltage)(struct regulator *r, int *level_uv); + TEE_Result (*supported_voltages)(struct regulator *r, + struct regulator_voltages **voltages); + TEE_Result (*supplied_init)(struct regulator *r, const void *fdt, + int node); +}; + +#ifdef CFG_DRIVERS_REGULATOR +/* + * regulator_enable() - Enable regulator + * @regulator: Regulator reference + */ +TEE_Result regulator_enable(struct regulator *regulator); + +/* + * regulator_disable() - Disable regulator + * @regulator: Regulator reference + */ +TEE_Result regulator_disable(struct regulator *regulator); + +/* + * regulator_is_enabled() - Return whether or not regulator is currently enabled + * despite its refcount value. + * @regulator: Regulator reference + */ +bool regulator_is_enabled(struct regulator *regulator); + +/* + * regulator_set_voltage() - Set regulator to target level in microvolt + * @regulator: Regulator reference + * @level_uv: Level in microvolt + */ +TEE_Result regulator_set_voltage(struct regulator *regulator, int level_uv); + +/* + * regulator_register() - Register and initialize a regulator + * @regulator: Regulator reference + */ +TEE_Result regulator_register(struct regulator *regulator); + +/* Print registered regulators and their state to the output console */ +void regulator_print_state(const char *message); +#else +static inline TEE_Result regulator_enable(struct regulator *regulator __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result regulator_disable(struct regulator *regulator __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline bool regulator_is_enabled(struct regulator *regulator __unused) +{ + return false; +} + +static inline TEE_Result regulator_set_voltage(struct regulator *regul __unused, + int level_mv __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result regulator_init(struct regulator *regulator __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline void regulator_print_state(const char *message __unused) +{ +} +#endif /*CFG_DRIVERS_REGULATOR*/ + +#if defined(CFG_DRIVERS_REGULATOR) && defined(CFG_DT) +/* + * regulator_dt_get_supply() - Get a regulator supply from name and DT node + * @fdt: FDT to work on + * @node: DT node of the regulator consumer + * @supply_name: Name of the supply in DT property xxx-supply + * @regulator: Output regulator upon success + * + * Upon success, this function provides the pointer to regulator + * defined by DT binding property @name-supply phandle reference. + * + * This function returns TEE_ERROR_DEFER_DRIVER_INIT if supply exists but is + * not yet initialized. + */ +TEE_Result regulator_dt_get_supply(const void *fdt, int node, + const char *supply_name, + struct regulator **regulator); + +/* + * regulator_dt_register() - Register a regulator to related to a DT node + * @fdt: FDT to work on + * @node: DT node of the regulator exposed by regulator driver + * @provider_node: Node where xxx-supply property is found or -1 if no supply. + * @desc: Description of the regulator to register + * + * This function registers and initializes a regulator instance once its supply + * if found, if any. Regulators registered with this function can be found by + * their consumer drivers using API function regulator_dt_get_supply() or like. + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_OUT_OF_MEMORY if failed on memory allocation + * Return any other TEE_Result compliant code in case of error + */ +TEE_Result regulator_dt_register(const void *fdt, int node, int provider_node, + const struct regu_dt_desc *desc); +#else +static inline TEE_Result regulator_dt_get_supply(const void *fdt __unused, + int node __unused, + const char *supply __unused, + struct regulator **r __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result +regulator_dt_register(const void *fdt __unused, int node __unused, + int provider_node __unused, + const struct regu_dt_desc *d __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif /* CFG_DRIVERS_REGULATOR && CFG_DT */ + +/* + * regulator_name() - Return regulator name or NULL + * @regulator: Regulator reference + */ +static inline const char *regulator_name(struct regulator *regulator) +{ + return regulator->name; +} + +/* + * regulator_is_always_on() - Return the state of REGULATOR_ALWAYS_ON flag + * @regulator: Regulator reference + */ +static inline bool regulator_is_always_on(struct regulator *regulator) +{ + return regulator->flags & REGULATOR_ALWAYS_ON; +} + +/* + * regulator_set_min_voltage() - Set regulator to its min level + * @regulator: Regulator reference + */ +static inline TEE_Result regulator_set_min_voltage(struct regulator *regulator) +{ + return regulator_set_voltage(regulator, regulator->min_uv); +} + +/* + * regulator_get_voltage() - Get regulator current level in microvolt + * @regulator: Regulator reference + */ +static inline int regulator_get_voltage(struct regulator *regulator) +{ + return regulator->cur_uv; +} + +/* + * regulator_get_range() - Get regulator min and/or max support levels + * @regulator: Regulator reference + * @min_mv: Output reference to min level in microvolt (uV) or NULL + * @max_mv: Output reference to max level in microvolt (uV) or NULL + */ +static inline void regulator_get_range(struct regulator *regulator, int *min_uv, + int *max_uv) +{ + assert(regulator); + if (min_uv) + *min_uv = regulator->min_uv; + if (max_uv) + *max_uv = regulator->max_uv; +} + +/* + * regulator_supported_voltages() - Get regulator supported levels in microvolt + * @regulator: Regulator reference + * @voltages: Output description supported voltage levels + */ +TEE_Result regulator_supported_voltages(struct regulator *regulator, + struct regulator_voltages **voltages); +#endif /* DRIVERS_REGULATOR_H */ diff --git a/optee_os/core/include/drivers/rstctrl.h b/optee_os/core/include/drivers/rstctrl.h new file mode 100644 index 0000000..d111240 --- /dev/null +++ b/optee_os/core/include/drivers/rstctrl.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Linaro Limited + */ + +#ifndef __DRIVERS_RSTCTRL_H +#define __DRIVERS_RSTCTRL_H + +#include +#include +#include + +struct rstctrl; + +struct rstctrl_ops { + /* + * Operators on reset control(s) exposed by a reset controller + * + * @assert_level: Assert reset level on control with a timeout hint + * @deassert_level: Deassert reset level on control with a timeout hint + * @get_name: Get a string name for the controller, or NULL is none + * + * Operator functions @assert_level and @deassert_level use arguments: + * @rstctrl: Reset controller + * @id: Identifier for the reset level control in the reset controller + * @to_ms: Timeout in microseconds or RSTCTRL_NO_TIMEOUT, may be ignored + * by reset controller. + * Return a TEE_Result compliant code. + */ + TEE_Result (*assert_level)(struct rstctrl *rstctrl, unsigned int to_us); + TEE_Result (*deassert_level)(struct rstctrl *rstctrl, + unsigned int to_us); + const char *(*get_name)(struct rstctrl *rstctrl); +}; + +/* + * struct rstctrl - Instance of a control exposed by a reset controller + * @ops: Operators of the reset controller + * @exclusive: Set when a consumer has exclusive control on the reset level + */ +struct rstctrl { + const struct rstctrl_ops *ops; + bool exclusive; +}; + +/** + * RSTCTRL_DECLARE - Declare a reset controller driver with a single + * device tree compatible string. + * + * @__name: Reset controller driver name + * @__compat: Compatible string + * @__probe: Reset controller probe function + */ +#define RSTCTRL_DT_DECLARE(__name, __compat, __probe) \ + static const struct dt_device_match __name ## _match_table[] = { \ + { .compatible = __compat }, \ + { } \ + }; \ + DEFINE_DT_DRIVER(__name ## _dt_driver) = { \ + .name = # __name, \ + .type = DT_DRIVER_RSTCTRL, \ + .match_table = __name ## _match_table, \ + .probe = __probe, \ + } + +/* + * Platform driver may ignore the timeout hint according to their + * capabilities. RSTCTRL_NO_TIMEOUT specifies no timeout hint. + */ +#define RSTCTRL_NO_TIMEOUT 0 + +/* + * rstctrl_assert_to - Assert reset control possibly with timeout + * rstctrl_assert - Assert reset control + * rstctrl_deassert_to - Deassert reset control possibly with timeout + * rstctrl_deassert - Deassert reset control + * + * @rstctrl: Reset controller + * @to_us: Timeout in microseconds + * Return a TEE_Result compliant code + */ +static inline TEE_Result rstctrl_assert_to(struct rstctrl *rstctrl, + unsigned int to_us) +{ + return rstctrl->ops->assert_level(rstctrl, to_us); +} + +static inline TEE_Result rstctrl_assert(struct rstctrl *rstctrl) +{ + return rstctrl_assert_to(rstctrl, RSTCTRL_NO_TIMEOUT); +} + +static inline TEE_Result rstctrl_deassert_to(struct rstctrl *rstctrl, + unsigned int to_us) +{ + return rstctrl->ops->deassert_level(rstctrl, to_us); +} + +static inline TEE_Result rstctrl_deassert(struct rstctrl *rstctrl) +{ + return rstctrl_deassert_to(rstctrl, RSTCTRL_NO_TIMEOUT); +} + +/* + * rstctrl_name - Get a name for the reset level control or NULL + * + * @rstctrl: Reset controller + * Return a pointer to controller name or NULL + */ +static inline const char *rstctrl_name(struct rstctrl *rstctrl) +{ + if (rstctrl->ops->get_name) + return rstctrl->ops->get_name(rstctrl); + + return NULL; +} + +/** + * rstctrl_dt_get_exclusive - Get exclusive access to reset controller + * + * @rstctrl: Reset controller + * Return a TEE_Result compliant value + */ +TEE_Result rstctrl_get_exclusive(struct rstctrl *rstctrl); + +/** + * rstctrl_put_exclusive - Release exclusive access to target + * + * @rstctrl: Reset controller + */ +void rstctrl_put_exclusive(struct rstctrl *rstctrl); + +/** + * rstctrl_ops_is_valid - Check reset controller ops is valid + * + * @ops: Reference to reset controller operator instance + */ +static inline bool rstctrl_ops_is_valid(const struct rstctrl_ops *ops) +{ + return ops && ops->assert_level && ops->deassert_level; +} + +#ifdef CFG_DT +/** + * rstctrl_dt_get_by_index - Get a reset controller at a specific index in + * 'resets' property + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the subnode containing a 'resets' property + * @index: Reset controller index in 'resets' property + * @rstctrl: Output reset controller reference upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if reset controller is not initialized + * Return TEE_ERROR_ITEM_NOT_FOUND if the resets property does not exist + * Return a TEE_Result compliant code in case of error + */ +static inline TEE_Result rstctrl_dt_get_by_index(const void *fdt, + int nodeoffset, + unsigned int index, + struct rstctrl **out_rstctrl) +{ + TEE_Result res = TEE_ERROR_GENERIC; + void *rstctrl = NULL; + + res = dt_driver_device_from_node_idx_prop("resets", fdt, nodeoffset, + index, DT_DRIVER_RSTCTRL, + &rstctrl); + if (!res) + *out_rstctrl = rstctrl; + + return res; +} +#else +static inline TEE_Result rstctrl_dt_get_by_index(const void *fdt __unused, + int nodeoffset __unused, + unsigned int index __unused, + struct rstctrl **ctrl __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif /*CFG_DT*/ + +/** + * rstctrl_dt_get_by_name - Get a reset controller matching a name in the + * 'reset-names' property + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the subnode containing a 'resets' property + * @name: Reset controller name to get + * @rstctrl: Output reset controller reference upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if reset controller is not initialized + * Return TEE_ERROR_ITEM_NOT_FOUND if the reset-names property does not exist + * Return a TEE_Result compliant code in case of error + */ +TEE_Result rstctrl_dt_get_by_name(const void *fdt, int nodeoffset, + const char *name, struct rstctrl **rstctrl); + +/** + * rstctrl_dt_get_func - Typedef of function to get reset controller from + * devicetree properties + * + * @args: Pointer to devicetree description of the reset controller to parse + * @data: Pointer to data given at rstctrl_dt_register_provider() call + * @rstctrl: Output reset controller reference upon success + */ +typedef TEE_Result (*rstctrl_dt_get_func)(struct dt_pargs *args, void *data, + struct rstctrl **out_rstctrl); + +/** + * rstctrl_dt_register_provider - Register a reset controller provider + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset of the reset controller + * @func: Callback to match the reset controller with a struct rstctrl + * @data: Data which will be passed to the get_dt_rstctrl callback + * Returns TEE_Result value + */ +static inline TEE_Result rstctrl_register_provider(const void *fdt, + int nodeoffset, + rstctrl_dt_get_func func, + void *data) +{ + return dt_driver_register_provider(fdt, nodeoffset, + (get_of_device_func)func, data, + DT_DRIVER_RSTCTRL); +} +#endif /* __DRIVERS_RSTCTRL_H */ + diff --git a/optee_os/core/include/drivers/rtc.h b/optee_os/core/include/drivers/rtc.h new file mode 100644 index 0000000..c7f05be --- /dev/null +++ b/optee_os/core/include/drivers/rtc.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 Microchip. + */ + +#ifndef DRIVERS_RTC_H +#define DRIVERS_RTC_H + +#include +#include + +/* The RTC allows to set/get offset for correction */ +#define RTC_CORRECTION_FEATURE BIT(0) + +struct optee_rtc_time { + uint32_t tm_year; + uint32_t tm_mon; + uint32_t tm_mday; + uint32_t tm_hour; + uint32_t tm_min; + uint32_t tm_sec; + uint32_t tm_wday; +}; + +struct rtc { + const struct rtc_ops *ops; + struct optee_rtc_time range_min; + struct optee_rtc_time range_max; +}; + +/* + * struct rtc_ops - The RTC device operations + * + * @get_time: Get the RTC time. + * @set_time: Set the RTC time. + * @get_offset: Get the RTC offset. + * @set_offset: Set the RTC offset + */ +struct rtc_ops { + TEE_Result (*get_time)(struct rtc *rtc, struct optee_rtc_time *tm); + TEE_Result (*set_time)(struct rtc *rtc, struct optee_rtc_time *tm); + TEE_Result (*get_offset)(struct rtc *rtc, long *offset); + TEE_Result (*set_offset)(struct rtc *rtc, long offset); +}; + +#ifdef CFG_DRIVERS_RTC +extern struct rtc *rtc_device; + +/* Register a RTC device as the system RTC */ +void rtc_register(struct rtc *rtc); + +static inline TEE_Result rtc_get_info(uint64_t *features, + struct optee_rtc_time *range_min, + struct optee_rtc_time *range_max) +{ + if (!rtc_device) + return TEE_ERROR_NOT_SUPPORTED; + + if (rtc_device->ops->set_offset) + *features = RTC_CORRECTION_FEATURE; + *range_min = rtc_device->range_min; + *range_max = rtc_device->range_max; + + return TEE_SUCCESS; +} + +static inline TEE_Result rtc_get_time(struct optee_rtc_time *tm) +{ + if (!rtc_device) + return TEE_ERROR_NOT_SUPPORTED; + + return rtc_device->ops->get_time(rtc_device, tm); +} + +static inline TEE_Result rtc_set_time(struct optee_rtc_time *tm) +{ + if (!rtc_device || !rtc_device->ops->set_time) + return TEE_ERROR_NOT_SUPPORTED; + + return rtc_device->ops->set_time(rtc_device, tm); +} + +static inline TEE_Result rtc_get_offset(long *offset) +{ + if (!rtc_device || !rtc_device->ops->get_offset) + return TEE_ERROR_NOT_SUPPORTED; + + return rtc_device->ops->get_offset(rtc_device, offset); +} + +static inline TEE_Result rtc_set_offset(long offset) +{ + if (!rtc_device || !rtc_device->ops->set_offset) + return TEE_ERROR_NOT_SUPPORTED; + + return rtc_device->ops->set_offset(rtc_device, offset); +} + +#else + +static inline void rtc_register(struct rtc *rtc __unused) {} + +static inline TEE_Result rtc_get_info(uint64_t *features __unused, + struct optee_rtc_time *range_min __unused, + struct optee_rtc_time *range_max __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result rtc_get_time(struct optee_rtc_time *tm __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result rtc_set_time(struct optee_rtc_time *tm __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result rtc_get_offset(long *offset __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result rtc_set_offset(long offset __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif +#endif /* DRIVERS_RTC_H */ diff --git a/optee_os/core/include/drivers/sam/at91_ddr.h b/optee_os/core/include/drivers/sam/at91_ddr.h new file mode 100644 index 0000000..84e13e6 --- /dev/null +++ b/optee_os/core/include/drivers/sam/at91_ddr.h @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Header file for the Atmel DDR/SDR SDRAM Controller + * + * Copyright (C) 2010 Atmel Corporation + * Nicolas Ferre + */ + +#ifndef AT91_DDRSDR_H +#define AT91_DDRSDR_H + +/* Mode Register */ +#define AT91_DDRSDRC_MR 0x00 +/* Command Mode */ +#define AT91_DDRSDRC_MODE SHIFT_U32(0x7, 0) +#define AT91_DDRSDRC_MODE_NORMAL 0 +#define AT91_DDRSDRC_MODE_NOP 1 +#define AT91_DDRSDRC_MODE_PRECHARGE 2 +#define AT91_DDRSDRC_MODE_LMR 3 +#define AT91_DDRSDRC_MODE_REFRESH 4 +#define AT91_DDRSDRC_MODE_EXT_LMR 5 +#define AT91_DDRSDRC_MODE_DEEP 6 + +/* Refresh Timer Register */ +#define AT91_DDRSDRC_RTR 0x04 +/* Refresh Timer Counter */ +#define AT91_DDRSDRC_COUNT SHIFT_U32(0xfff, 0) + +/* Configuration Register */ +#define AT91_DDRSDRC_CR 0x08 +/* Number of Column Bits */ +#define AT91_DDRSDRC_NC SHIFT_U32(3, 0) +#define AT91_DDRSDRC_NC_SDR8 SHIFT_U32(0, 0) +#define AT91_DDRSDRC_NC_SDR9 BIT(0) +#define AT91_DDRSDRC_NC_SDR10 SHIFT_U32(2, 0) +#define AT91_DDRSDRC_NC_SDR11 SHIFT_U32(3, 0) +#define AT91_DDRSDRC_NC_DDR9 SHIFT_U32(0, 0) +#define AT91_DDRSDRC_NC_DDR10 BIT(0) +#define AT91_DDRSDRC_NC_DDR11 SHIFT_U32(2, 0) +#define AT91_DDRSDRC_NC_DDR12 SHIFT_U32(3, 0) +/* Number of Row Bits */ +#define AT91_DDRSDRC_NR SHIFT_U32(3, 2) +#define AT91_DDRSDRC_NR_11 SHIFT_U32(0, 2) +#define AT91_DDRSDRC_NR_12 BIT(2) +#define AT91_DDRSDRC_NR_13 SHIFT_U32(2, 2) +#define AT91_DDRSDRC_NR_14 SHIFT_U32(3, 2) +/* CAS Latency */ +#define AT91_DDRSDRC_CAS SHIFT_U32(7, 4) +#define AT91_DDRSDRC_CAS_2 SHIFT_U32(2, 4) +#define AT91_DDRSDRC_CAS_3 SHIFT_U32(3, 4) +#define AT91_DDRSDRC_CAS_25 SHIFT_U32(6, 4) +/* Reset DLL */ +#define AT91_DDRSDRC_RST_DLL BIT(7) +/* Output impedance control */ +#define AT91_DDRSDRC_DICDS BIT(8) +/* Disable DLL [SAM9 Only] */ +#define AT91_DDRSDRC_DIS_DLL BIT(9) +/* Off-Chip Driver [SAM9 Only] */ +#define AT91_DDRSDRC_OCD BIT(12) +/* Mask Data is Shared [SAM9 Only] */ +#define AT91_DDRSDRC_DQMS BIT(16) +/* Active Bank X to Burst Stop Read Access Bank Y [SAM9 Only] */ +#define AT91_DDRSDRC_ACTBST BIT(18) + +/* Timing 0 Register */ +#define AT91_DDRSDRC_T0PR 0x0C +/* Active to Precharge delay */ +#define AT91_DDRSDRC_TRAS SHIFT_U32(0xf, 0) +/* Row to Column delay */ +#define AT91_DDRSDRC_TRCD SHIFT_U32(0xf, 4) +/* Write recovery delay */ +#define AT91_DDRSDRC_TWR SHIFT_U32(0xf, 8) +/* Row cycle delay */ +#define AT91_DDRSDRC_TRC SHIFT_U32(0xf, 12) +/* Row precharge delay */ +#define AT91_DDRSDRC_TRP SHIFT_U32(0xf, 16) +/* Active BankA to BankB */ +#define AT91_DDRSDRC_TRRD SHIFT_U32(0xf, 20) +/* Internal Write to Read delay */ +#define AT91_DDRSDRC_TWTR SHIFT_U32(0x7, 24) +/* Reduce Write to Read Delay [SAM9 Only] */ +#define AT91_DDRSDRC_RED_WRRD SHIFT_U32(0x1, 27) +/* Load mode to active/refresh delay */ +#define AT91_DDRSDRC_TMRD SHIFT_U32(0xf, 28) + +/* Timing 1 Register */ +#define AT91_DDRSDRC_T1PR 0x10 +/* Row Cycle Delay */ +#define AT91_DDRSDRC_TRFC SHIFT_U32(0x1f, 0) +/* Exit self-refresh to non-read */ +#define AT91_DDRSDRC_TXSNR SHIFT_U32(0xff, 8) +/* Exit self-refresh to read */ +#define AT91_DDRSDRC_TXSRD SHIFT_U32(0xff, 16) +/* Exit power-down delay */ +#define AT91_DDRSDRC_TXP SHIFT_U32(0xf, 24) + +/* Timing 2 Register [SAM9 Only] */ +#define AT91_DDRSDRC_T2PR 0x14 +/* Exit active power down delay to read command in mode "Fast Exit" */ +#define AT91_DDRSDRC_TXARD SHIFT_U32(0xf, 0) +/* Exit active power down delay to read command in mode "Slow Exit" */ +#define AT91_DDRSDRC_TXARDS SHIFT_U32(0xf, 4) +/* Row Precharge All delay */ +#define AT91_DDRSDRC_TRPA SHIFT_U32(0xf, 8) +/* Read to Precharge delay */ +#define AT91_DDRSDRC_TRTP SHIFT_U32(0x7, 12) + +/* Low Power Register */ +#define AT91_DDRSDRC_LPR 0x1C +/* Low-power Configurations */ +#define AT91_DDRSDRC_LPCB SHIFT_U32(3, 0) +#define AT91_DDRSDRC_LPCB_DISABLE 0 +#define AT91_DDRSDRC_LPCB_SELF_REFRESH 1 +#define AT91_DDRSDRC_LPCB_POWER_DOWN 2 +#define AT91_DDRSDRC_LPCB_DEEP_POWER_DOWN 3 +/* Clock Frozen */ +#define AT91_DDRSDRC_CLKFR BIT(2) +/* LPDDR Power Off */ +#define AT91_DDRSDRC_LPDDR2_PWOFF BIT(3) +/* Partial Array Self Refresh */ +#define AT91_DDRSDRC_PASR SHIFT_U32(7, 4) +/* Temperature Compensated Self Refresh */ +#define AT91_DDRSDRC_TCSR SHIFT_U32(3, 8) +/* Drive Strength */ +#define AT91_DDRSDRC_DS SHIFT_U32(3, 10) +/* Time to define when Low Power Mode is enabled */ +#define AT91_DDRSDRC_TIMEOUT SHIFT_U32(3, 12) +#define AT91_DDRSDRC_TIMEOUT_0_CLK_CYCLES SHIFT_U32(0, 12) +#define AT91_DDRSDRC_TIMEOUT_64_CLK_CYCLES BIT(12) +#define AT91_DDRSDRC_TIMEOUT_128_CLK_CYCLES SHIFT_U32(2, 12) +/* Active power down exit time */ +#define AT91_DDRSDRC_APDE BIT(16) +/* Update load mode register and extended mode register */ +#define AT91_DDRSDRC_UPD_MR SHIFT_U32(3, 20) + +/* Memory Device Register */ +#define AT91_DDRSDRC_MDR 0x20 +/* Memory Device Type */ +#define AT91_DDRSDRC_MD SHIFT_U32(7, 0) +#define AT91_DDRSDRC_MD_SDR 0 +#define AT91_DDRSDRC_MD_LOW_POWER_SDR 1 +#define AT91_DDRSDRC_MD_LOW_POWER_DDR 3 +#define AT91_DDRSDRC_MD_LPDDR3 5 +/* [SAM9 Only] */ +#define AT91_DDRSDRC_MD_DDR2 6 +#define AT91_DDRSDRC_MD_LPDDR2 7 +/* Data Bus Width */ +#define AT91_DDRSDRC_DBW BIT(4) +#define AT91_DDRSDRC_DBW_32BITS SHIFT_U32(0, 4) +#define AT91_DDRSDRC_DBW_16BITS BIT(4) + +/* DLL Information Register */ +#define AT91_DDRSDRC_DLL 0x24 +/* Master Delay increment */ +#define AT91_DDRSDRC_MDINC BIT(0) +/* Master Delay decrement */ +#define AT91_DDRSDRC_MDDEC BIT(1) +/* Master Delay Overflow */ +#define AT91_DDRSDRC_MDOVF BIT(2) +/* Master Delay value */ +#define AT91_DDRSDRC_MDVAL SHIFT_U32(0xff, 8) + +/* High Speed Register [SAM9 Only] */ +#define AT91_DDRSDRC_HS 0x2C +/* Anticip read access is disabled */ +#define AT91_DDRSDRC_DIS_ATCP_RD BIT(2) + +/* Delay I/O Register n */ +#define AT91_DDRSDRC_DELAY(n) (0x30 + (0x4 * (n))) + +/* Write Protect Mode Register [SAM9 Only] */ +#define AT91_DDRSDRC_WPMR 0xE4 +/* Write protect enable */ +#define AT91_DDRSDRC_WP BIT(0) +/* Write protect key */ +#define AT91_DDRSDRC_WPKEY SHIFT_U32(0xffffff, 8) +/* Write protect key = "DDR" */ +#define AT91_DDRSDRC_KEY SHIFT_U32(0x444452, 8) + +/* Write Protect Status Register [SAM9 Only] */ +#define AT91_DDRSDRC_WPSR 0xE8 +/* Write protect violation status */ +#define AT91_DDRSDRC_WPVS BIT(0) +/* Write protect violation source */ +#define AT91_DDRSDRC_WPVSRC SHIFT_U32(0xffff, 8) + +#endif diff --git a/optee_os/core/include/drivers/scif.h b/optee_os/core/include/drivers/scif.h new file mode 100644 index 0000000..fe81871 --- /dev/null +++ b/optee_os/core/include/drivers/scif.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, GlobalLogic + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SCIF_H +#define SCIF_H + +#include +#include + +#define SCIF_REG_SIZE 0x1000 + +struct scif_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void scif_uart_init(struct scif_uart_data *pd, paddr_t base); + +#endif /* SCIF */ diff --git a/optee_os/core/include/drivers/scmi-msg.h b/optee_os/core/include/drivers/scmi-msg.h new file mode 100644 index 0000000..08dd31c --- /dev/null +++ b/optee_os/core/include/drivers/scmi-msg.h @@ -0,0 +1,411 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2019-2021, Linaro Limited + */ + +#ifndef SCMI_MSG_H +#define SCMI_MSG_H + +#include +#include +#include +#include +#include +#include + +/* Minimum size expected for SMT based shared memory message buffers */ +#define SMT_BUF_SLOT_SIZE U(128) + +/* Standard values for SCMI voltage domain protocol configuration state */ +#define SCMI_VOLTAGE_DOMAIN_CONFIG_ARCH_ON U(0x7) +#define SCMI_VOLTAGE_DOMAIN_CONFIG_ARCH_OFF U(0) + +/* A channel abstract a communication path between agent and server */ +struct scmi_msg_channel; + +/* + * struct scmi_msg_channel - Shared memory buffer for a agent-to-server channel + * + * @shm_addr: Address of the shared memory for the SCMI channel + * @shm_size: Byte size of the shared memory for the SCMI channel + * @busy: True when channel is busy, flase when channel is free + * @threaded: True is executed in a threaded context, false otherwise + */ +struct scmi_msg_channel { + struct io_pa_va shm_addr; + size_t shm_size; + bool busy; + bool threaded; +}; + +#ifdef CFG_SCMI_MSG_SMT +/* + * Initialize SMT memory buffer, called by platform at init for each + * agent channel using the SMT header format. + * This function depends on CFG_SCMI_MSG_SMT. + * + * @channel: Pointer to the channel shared memory to be initialized + */ +void scmi_smt_init_agent_channel(struct scmi_msg_channel *channel); + +/* + * Set SMT shared buffer location + * + * @channel: SCMI channel reference + * @base: virtual address of the shared buffer or NULL to clear the reference + */ +void scmi_smt_set_shared_buffer(struct scmi_msg_channel *channel, void *base); +#else +static inline +void scmi_smt_init_agent_channel(struct scmi_msg_channel *channel __unused) +{ + panic(); +} + +static inline +void scmi_smt_set_shared_buffer(struct scmi_msg_channel *channel __unused, + void *base __unused) +{ +} +#endif /* CFG_SCMI_MSG_SMT */ + +#ifdef CFG_SCMI_MSG_SMT_FASTCALL_ENTRY +/* + * Process SMT formatted message in a fastcall SMC execution context. + * Called by platform on SMC entry. When returning, output message is + * available in shared memory for agent to read the response. + * This function depends on CFG_SCMI_MSG_SMT_FASTCALL_ENTRY. + * + * @channel_id: SCMI channel ID the SMT belongs to + */ +void scmi_smt_fastcall_smc_entry(unsigned int channel_id); +#else +static inline void scmi_smt_fastcall_smc_entry(unsigned int channel_id __unused) +{ +} +#endif + +#ifdef CFG_SCMI_MSG_SMT_INTERRUPT_ENTRY +/* + * Process SMT formatted message in a secure interrupt execution context. + * Called by platform interrupt handler. When returning, output message is + * available in shared memory for agent to read the response. + * This function depends on CFG_SCMI_MSG_SMT_INTERRUPT_ENTRY. + * + * @channel_id: SCMI channel ID the SMT belongs to + */ +void scmi_smt_interrupt_entry(unsigned int channel_id); +#else +static inline void scmi_smt_interrupt_entry(unsigned int channel_id __unused) +{ +} +#endif + +#ifdef CFG_SCMI_MSG_SMT_THREAD_ENTRY +/* + * Process SMT formatted message in a TEE thread execution context. + * When returning, output message is available in shared memory for + * agent to read the response. + * This function depends on CFG_SCMI_MSG_SMT_THREAD_ENTRY. + * + * @channel_id: SCMI channel ID the SMT belongs to + */ +void scmi_smt_threaded_entry(unsigned int channel_id); +#else +static inline void scmi_smt_threaded_entry(unsigned int channel_id __unused) +{ +} +#endif + +#ifdef CFG_SCMI_MSG_SHM_MSG +/* + * Process MSG formatted message in a TEE thread execution context. + * When returning, output message is available in shared memory for + * agent to read the response. + * This function depends on CFG_SCMI_MSG_MSG_THREAD_ENTRY. + * + * @channel_id: SCMI channel ID + * @in_buf: Shared buffer storing input SCMI message + * @in_size: Byte size of @in_buf, including MSG header and message payload + * @out_buf: Shared buffer storing input SCMI message + * @out_size: [in] @out_buf max byte size + * [out] @out_buf output byte size (MSG header and message payload) + */ +TEE_Result scmi_msg_threaded_entry(unsigned int channel_id, + void *in_buf, size_t in_size, + void *out_buf, size_t *out_size); +#else +static inline TEE_Result scmi_msg_threaded_entry(unsigned int chan_id __unused, + void *in_buf __unused, + size_t in_size __unused, + void *out_buf __unused, + size_t *out_size __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +struct clk; + +#ifdef CFG_SCMI_MSG_USE_CLK +/* + * Expose a clock through SCMI + * @clk: Clock to be exposed + * @channel_id: SCMI server channel exposing the clock + * @scmi_id: SCMI ID of the clock within the channel + */ +TEE_Result scmi_clk_add(struct clk *clk, unsigned int channel_id, + unsigned int scmi_id); +#else +static inline TEE_Result scmi_clk_add(struct clk *clk __unused, + unsigned int channel_id __unused, + unsigned int scmi_id __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +/* Platform callback functions */ + +/* + * Return the SCMI channel related to an agent + * @channel_id: SCMI channel ID + * Return a pointer to channel on success, NULL otherwise + */ +struct scmi_msg_channel *plat_scmi_get_channel(unsigned int channel_id); + +/* Scmi-msg uses the channel ID as handle. Must channel_id is valid */ +static inline unsigned int scmi_smt_channel_handle(unsigned int channel_id) +{ + assert(plat_scmi_get_channel(channel_id)); + + return channel_id; +} + +/* + * Return how many SCMI protocols supported by the platform + * According to the SCMI specification, this function does not target + * a specific channel ID and shall return all platform known capabilities. + */ +size_t plat_scmi_protocol_count(void); + +/* + * Get the count and list of SCMI protocols (but base) supported for an agent + * + * @channel_id: SCMI channel ID + * Return a pointer to a null terminated array supported protocol IDs. + */ +const uint8_t *plat_scmi_protocol_list(unsigned int channel_id); + +/* Get the name of the SCMI vendor for the platform */ +const char *plat_scmi_vendor_name(void); + +/* Get the name of the SCMI sub-vendor for the platform */ +const char *plat_scmi_sub_vendor_name(void); + +/* Handlers for SCMI Clock protocol services */ + +/* + * Return number of clock controllers for an agent + * @channel_id: SCMI channel ID + * Return number of clock controllers + */ +size_t plat_scmi_clock_count(unsigned int channel_id); + +/* + * Get clock controller string ID (aka name) + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * Return pointer to name or NULL + */ +const char *plat_scmi_clock_get_name(unsigned int channel_id, + unsigned int scmi_id); + +/* + * Get clock possible rate as an array of frequencies in Hertz. + * + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * @start_index: Requested start index for the exposed rates array + * @rates: Output rates array or NULL if only querying @nb_elts + * @nb_elts: [in] Array size of @rates, [out] Number of rates loaded in @rates + * Return an SCMI compliant error code + */ +int32_t plat_scmi_clock_rates_array(unsigned int channel_id, + unsigned int scmi_id, size_t start_index, + unsigned long *rates, size_t *nb_elts); + +/* + * Get clock possible rate as range with regular steps in Hertz + * + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * @min_max_step: 3 cell array for min, max and step rate data + * Return an SCMI compliant error code + */ +int32_t plat_scmi_clock_rates_by_step(unsigned int channel_id, + unsigned int scmi_id, + unsigned long *min_max_step); + +/* + * Get clock rate in Hertz + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * Return clock rate or 0 if not supported + */ +unsigned long plat_scmi_clock_get_rate(unsigned int channel_id, + unsigned int scmi_id); + +/* + * Set clock rate in Hertz + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * @rate: Target clock frequency in Hertz + * Return a compliant SCMI error code + */ +int32_t plat_scmi_clock_set_rate(unsigned int channel_id, unsigned int scmi_id, + unsigned long rate); + +/* + * Get clock state (enabled or disabled) + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * Return 1 if clock is enabled, 0 if disables, or a negative SCMI error code + */ +int32_t plat_scmi_clock_get_state(unsigned int channel_id, + unsigned int scmi_id); + +/* + * Get clock state (enabled or disabled) + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * @enable_not_disable: Enable clock if true, disable clock otherwise + * Return a compliant SCMI error code + */ +int32_t plat_scmi_clock_set_state(unsigned int channel_id, unsigned int scmi_id, + bool enable_not_disable); + +/* Handlers for SCMI Reset Domain protocol services */ + +/* + * Return number of reset domains for the agent + * @channel_id: SCMI channel ID + * Return number of reset domains + */ +size_t plat_scmi_rd_count(unsigned int channel_id); + +/* + * Get reset domain string ID (aka name) + * @channel_id: SCMI channel ID + * @scmi_id: SCMI reset domain ID + * Return pointer to name or NULL + */ +const char *plat_scmi_rd_get_name(unsigned int channel_id, + unsigned int scmi_id); + +/* + * Perform a reset cycle on a target reset domain + * @channel_id: SCMI channel ID + * @scmi_id: SCMI reset domain ID + * @state: Target reset state (see SCMI specification, 0 means context loss) + * Return a compliant SCMI error code + */ +int32_t plat_scmi_rd_autonomous(unsigned int channel_id, unsigned int scmi_id, + unsigned int state); + +/* + * Assert or deassert target reset domain + * @channel_id: SCMI channel ID + * @scmi_id: SCMI reset domain ID + * @assert_not_deassert: Assert domain if true, otherwise deassert domain + * Return a compliant SCMI error code + */ +int32_t plat_scmi_rd_set_state(unsigned int channel_id, unsigned int scmi_id, + bool assert_not_deassert); + +/* Handlers for SCMI Voltage Domain protocol services */ + +/* + * Return number of voltage domain for an agent + * @channel_id: SCMI channel ID + * Return number of voltage domains + */ +size_t plat_scmi_voltd_count(unsigned int channel_id); + +/* + * Get clock controller string ID (aka name) + * @channel_id: SCMI channel ID + * @scmi_id: SCMI voltage domain ID + * Return pointer to name or NULL + */ +const char *plat_scmi_voltd_get_name(unsigned int channel_id, + unsigned int scmi_id); + +/* + * Get voltage domain possible levels as an array of voltages in microvolt. + * + * @channel_id: SCMI channel ID + * @scmi_id: SCMI voltage domain ID + * @start_index: Level index to start from. + * @levels: If NULL, function returns, else output rates array + * @nb_elts: Array size of @levels. + * Return an SCMI compliant error code + */ +int32_t plat_scmi_voltd_levels_array(unsigned int channel_id, + unsigned int scmi_id, size_t start_index, + long *levels, size_t *nb_elts); + +/* + * Get voltage domain possible levels as range with regular steps in microvolt + * + * @channel_id: SCMI channel ID + * @scmi_id: SCMI voltage domain ID + * @min_max_step: 3 cell array for min, max and step voltage data + * Return an SCMI compliant error code + */ +int32_t plat_scmi_voltd_levels_by_step(unsigned int channel_id, + unsigned int scmi_id, + long *min_max_step); + +/* + * Get current voltage domain level in microvolt + * @channel_id: SCMI channel ID + * @scmi_id: SCMI voltage domain ID + * @level: Out parameter for the current voltage level + * Return an SCMI compliant error code + */ +int32_t plat_scmi_voltd_get_level(unsigned int channel_id, unsigned int scmi_id, + long *level); + +/* + * Set voltage domain level voltage domain + * @channel_id: SCMI channel ID + * @scmi_id: SCMI clock ID + * @level: Target voltage domain level in microvolt + * Return a compliant SCMI error code + */ +int32_t plat_scmi_voltd_set_level(unsigned int channel_id, unsigned int scmi_id, + long level); + +/* + * Get voltage domain state configuration (enabled or disabled) + * @channel_id: SCMI channel ID + * @scmi_id: SCMI voltage domain ID + * @config: output state configuration value SCMI_VOLTAGE_DOMAIN_CONFIG_* + * Return a compliant SCMI error code + */ +int32_t plat_scmi_voltd_get_config(unsigned int channel_id, + unsigned int scmi_id, uint32_t *config); + +/* + * Get voltage domain state configuration (enabled or disabled) + * @channel_id: SCMI channel ID + * @scmi_id: SCMI voltage domain ID + * @config: Target state configuration value SCMI_VOLTAGE_DOMAIN_CONFIG_* + * Return a compliant SCMI error code + */ +int32_t plat_scmi_voltd_set_config(unsigned int channel_id, + unsigned int scmi_id, uint32_t config); + +#endif /* SCMI_MSG_H */ diff --git a/optee_os/core/include/drivers/scmi.h b/optee_os/core/include/drivers/scmi.h new file mode 100644 index 0000000..9497206 --- /dev/null +++ b/optee_os/core/include/drivers/scmi.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2015-2019, Arm Limited and Contributors. All rights reserved. + */ +#ifndef SCMI_MSG_SCMI_H +#define SCMI_MSG_SCMI_H + +#define SCMI_PROTOCOL_ID_BASE 0x10 +#define SCMI_PROTOCOL_ID_POWER_DOMAIN 0x11 +#define SCMI_PROTOCOL_ID_SYS_POWER 0x12 +#define SCMI_PROTOCOL_ID_PERF 0x13 +#define SCMI_PROTOCOL_ID_CLOCK 0x14 +#define SCMI_PROTOCOL_ID_SENSOR 0x15 +#define SCMI_PROTOCOL_ID_RESET_DOMAIN 0x16 +#define SCMI_PROTOCOL_ID_VOLTAGE_DOMAIN 0x17 + +/* SCMI error codes reported to agent through server-to-agent messages */ +#define SCMI_SUCCESS 0 +#define SCMI_NOT_SUPPORTED (-1) +#define SCMI_INVALID_PARAMETERS (-2) +#define SCMI_DENIED (-3) +#define SCMI_NOT_FOUND (-4) +#define SCMI_OUT_OF_RANGE (-5) +#define SCMI_BUSY (-6) +#define SCMI_COMMS_ERROR (-7) +#define SCMI_GENERIC_ERROR (-8) +#define SCMI_HARDWARE_ERROR (-9) +#define SCMI_PROTOCOL_ERROR (-10) + +#endif /* SCMI_MSG_SCMI_H */ diff --git a/optee_os/core/include/drivers/serial.h b/optee_os/core/include/drivers/serial.h new file mode 100644 index 0000000..4c2f964 --- /dev/null +++ b/optee_os/core/include/drivers/serial.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ +#ifndef __DRIVERS_SERIAL_H +#define __DRIVERS_SERIAL_H + +#include +#include +#include +#include +#include + +struct serial_chip { + const struct serial_ops *ops; +}; + +struct serial_ops { + /* Mandatory handler */ + void (*putc)(struct serial_chip *chip, int ch); + /* Optional handlers */ + void (*flush)(struct serial_chip *chip); + bool (*have_rx_data)(struct serial_chip *chip); + int (*getchar)(struct serial_chip *chip); +}; + +struct serial_driver { + /* Allocate device data and return the inner serial_chip */ + struct serial_chip *(*dev_alloc)(void); + /* + * Initialize device from FDT node. @parms is device-specific, + * its meaning is as defined by the DT bindings for the characters + * following the ":" in /chosen/stdout-path. Typically for UART + * devices this is {{{}}} where: + * baud - baud rate in decimal + * parity - 'n' (none), 'o', (odd) or 'e' (even) + * bits - number of data bits + * flow - 'r' (rts) + * For example: 115200n8r + */ + int (*dev_init)(struct serial_chip *dev, const void *fdt, + int offset, const char *parms); + void (*dev_free)(struct serial_chip *dev); +}; + +#endif /*__DRIVERS_SERIASERIAL_H*/ diff --git a/optee_os/core/include/drivers/serial8250_uart.h b/optee_os/core/include/drivers/serial8250_uart.h new file mode 100644 index 0000000..40816c9 --- /dev/null +++ b/optee_os/core/include/drivers/serial8250_uart.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef SERIAL8250_UART_H +#define SERIAL8250_UART_H + +#include +#include + +#define SERIAL8250_UART_REG_SIZE 0x20 + +struct serial8250_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void serial8250_uart_init(struct serial8250_uart_data *pd, paddr_t base, + uint32_t uart_clk, uint32_t baud_rate); + +#endif /* SERIAL8250_UART_H */ + diff --git a/optee_os/core/include/drivers/sp805_wdt.h b/optee_os/core/include/drivers/sp805_wdt.h new file mode 100644 index 0000000..0d5d313 --- /dev/null +++ b/optee_os/core/include/drivers/sp805_wdt.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef SP805_WDT_H +#define SP805_WDT_H + +#include +#include +#include +#include + +/* SP805 register offset */ +#define WDT_LOAD_OFFSET 0x000 +#define WDT_CONTROL_OFFSET 0x008 +#define WDT_INTCLR_OFFSET 0x00c +#define WDT_LOCK_OFFSET 0xc00 +#define WDT_SIZE 0xc04 + +/* Magic word to unlock the wd registers */ +#define WDT_UNLOCK_KEY 0x1ACCE551 +#define WDT_LOCK_KEY 0x1 + +/* Register field definitions */ +#define WDT_INT_EN BIT(0) +#define WDT_RESET_EN BIT(1) +#define WDT_INT_CLR BIT(0) + +#define WDT_LOAD_MIN 0x1 + +typedef void (*sp805_itr_handler_func_t)(struct wdt_chip *chip); + +struct sp805_wdt_data { + struct io_pa_va base; + struct wdt_chip chip; + uint32_t clk_rate; + uint32_t load_val; + uint32_t itr_num; + sp805_itr_handler_func_t itr_handler; +}; + +/* + * Initialize sp805 watchdog timer + * + * @pd: allocated sp805 watchdog timer platform data + * @base: physical base address of sp805 watchdog timer + * @clk_rate: rate of the clock driving the watchdog timer hardware + * @timeout: watchdog timer timeout in seconds + * Return a TEE_Result compliant status + */ +TEE_Result sp805_wdt_init(struct sp805_wdt_data *pd, paddr_t base, + uint32_t clk_rate, uint32_t timeout); + +/* + * Optionally register sp805 watchdog timer interrupt handler + * + * @pd: platform data of sp805 watchdog timer for which interrupt handler + * is to be registered + * @itr_num: sp805 watchdog timer interrupt id + * @itr_flag: interrupt attributes + * @itr_handler: Optional interrupt handler callback + * Return a TEE_Result compliant status + */ +TEE_Result sp805_register_itr_handler(struct sp805_wdt_data *pd, + uint32_t itr_num, uint32_t itr_flag, + sp805_itr_handler_func_t itr_handler); + +#endif /* SP805_WDT_H */ diff --git a/optee_os/core/include/drivers/sprd_uart.h b/optee_os/core/include/drivers/sprd_uart.h new file mode 100644 index 0000000..6482622 --- /dev/null +++ b/optee_os/core/include/drivers/sprd_uart.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Spreadtrum Communications Inc. + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SPRD_UART_H +#define SPRD_UART_H + +#include +#include + +struct sprd_uart_data { + struct io_pa_va base; + struct serial_chip chip; +}; + +void sprd_uart_init(struct sprd_uart_data *pd, paddr_t base); + +#endif /* SPRD_UART_H */ + diff --git a/optee_os/core/include/drivers/stih_asc.h b/optee_os/core/include/drivers/stih_asc.h new file mode 100644 index 0000000..21e8297 --- /dev/null +++ b/optee_os/core/include/drivers/stih_asc.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ +#ifndef STIH_ASC_H +#define STIH_ASC_H + +#include +#include + +#define STIH_ASC_REG_SIZE 0x1000 + +struct stih_asc_pd { + struct io_pa_va base; + struct serial_chip chip; +}; + +void stih_asc_init(struct stih_asc_pd *pb, vaddr_t base); + +#endif /* STIH_ASC_H */ + diff --git a/optee_os/core/include/drivers/stm32_bsec.h b/optee_os/core/include/drivers/stm32_bsec.h new file mode 100644 index 0000000..a598425 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_bsec.h @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2017-2022, STMicroelectronics + */ + +#ifndef __STM32_BSEC_H +#define __STM32_BSEC_H + +#include +#include +#include + +/* BSEC_DEBUG */ +#define BSEC_HDPEN BIT(4) +#define BSEC_SPIDEN BIT(5) +#define BSEC_SPINDEN BIT(6) +#define BSEC_DBGSWGEN BIT(10) +#define BSEC_DEBUG_ALL (BSEC_HDPEN | \ + BSEC_SPIDEN | \ + BSEC_SPINDEN | \ + BSEC_DBGSWGEN) + +#define BSEC_BITS_PER_WORD (8U * sizeof(uint32_t)) +#define BSEC_BYTES_PER_WORD sizeof(uint32_t) + +/* BSEC different global states */ +enum stm32_bsec_sec_state { + BSEC_STATE_SEC_CLOSED, + BSEC_STATE_SEC_OPEN, + BSEC_STATE_INVALID +}; + +/* + * Load OTP from SAFMEM and provide its value + * @value: Output read value + * @otp_id: OTP number + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_shadow_read_otp(uint32_t *value, uint32_t otp_id); + +/* + * Copy SAFMEM OTP to BSEC data. + * @otp_id: OTP number. + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_shadow_register(uint32_t otp_id); + +/* + * Read an OTP data value + * @value: Output read value + * @otp_id: OTP number + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_read_otp(uint32_t *value, uint32_t otp_id); + +/* + * Write value in BSEC data register + * @value: Value to write + * @otp_id: OTP number + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_write_otp(uint32_t value, uint32_t otp_id); + +/* + * Program a bit in SAFMEM without BSEC data refresh + * @value: Value to program. + * @otp_id: OTP number. + * Return a TEE_Result compliant return value + */ +#ifdef CFG_STM32_BSEC_WRITE +TEE_Result stm32_bsec_program_otp(uint32_t value, uint32_t otp_id); +#else +static inline TEE_Result stm32_bsec_program_otp(uint32_t value __unused, + uint32_t otp_id __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +/* + * Permanent lock of OTP in SAFMEM + * @otp_id: OTP number + * Return a TEE_Result compliant return value + */ +#ifdef CFG_STM32_BSEC_WRITE +TEE_Result stm32_bsec_permanent_lock_otp(uint32_t otp_id); +#else +static inline TEE_Result stm32_bsec_permanent_lock_otp(uint32_t otp_id __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +/* + * Enable/disable debug service + * @value: Value to write + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_write_debug_conf(uint32_t value); + +/* Return debug configuration read from BSEC */ +uint32_t stm32_bsec_read_debug_conf(void); + +/* + * Write shadow-read lock + * @otp_id: OTP number + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_set_sr_lock(uint32_t otp_id); + +/* + * Read shadow-read lock + * @otp_id: OTP number + * @locked: (out) true if shadow-read is locked, false if not locked. + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_read_sr_lock(uint32_t otp_id, bool *locked); + +/* + * Write shadow-write lock + * @otp_id: OTP number + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_set_sw_lock(uint32_t otp_id); + +/* + * Read shadow-write lock + * @otp_id: OTP number + * @locked: (out) true if shadow-write is locked, false if not locked. + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_read_sw_lock(uint32_t otp_id, bool *locked); + +/* + * Write shadow-program lock + * @otp_id: OTP number + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_set_sp_lock(uint32_t otp_id); + +/* + * Read shadow-program lock + * @otp_id: OTP number + * @locked: (out) true if shadow-program is locked, false if not locked. + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_read_sp_lock(uint32_t otp_id, bool *locked); + +/* + * Read permanent lock status + * @otp_id: OTP number + * @locked: (out) true if permanent lock is locked, false if not locked. + * Return a TEE_Result compliant return value + */ +TEE_Result stm32_bsec_read_permanent_lock(uint32_t otp_id, bool *locked); + +/* + * Return true if OTP can be read, false otherwise + * @otp_id: OTP number + */ +bool stm32_bsec_can_access_otp(uint32_t otp_id); + +/* + * Return true if non-secure world is allowed to read the target OTP + * @otp_id: OTP number + */ +bool stm32_bsec_nsec_can_access_otp(uint32_t otp_id); + +/* + * Find and get OTP location from its name. + * @name: sub-node name to look up. + * @otp_id: pointer to output OTP number or NULL. + * @otp_bit_offset: pointer to output OTP bit offset in the NVMEM cell or NULL. + * @otp_bit_len: pointer to output OTP length in bits or NULL. + * Return a TEE_Result compliant status + */ +TEE_Result stm32_bsec_find_otp_in_nvmem_layout(const char *name, + uint32_t *otp_id, + uint8_t *otp_bit_offset, + size_t *otp_bit_len); + +/* + * Find and get OTP location from its phandle. + * @phandle: node phandle to look up. + * @otp_id: pointer to read OTP number or NULL. + * @otp_bit_offset: pointer to read offset in OTP in bits or NULL. + * @otp_bit_len: pointer to read OTP length in bits or NULL. + * Return a TEE_Result compliant status + */ +TEE_Result stm32_bsec_find_otp_by_phandle(const uint32_t phandle, + uint32_t *otp_id, + uint8_t *otp_bit_offset, + size_t *otp_bit_len); + +/* + * Get BSEC global sec state. + * @sec_state: Global BSEC current sec state + * Return a TEE_Result compliant status + */ +TEE_Result stm32_bsec_get_state(enum stm32_bsec_sec_state *sec_state); + +#endif /*__STM32_BSEC_H*/ diff --git a/optee_os/core/include/drivers/stm32_etzpc.h b/optee_os/core/include/drivers/stm32_etzpc.h new file mode 100644 index 0000000..3a87ff4 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_etzpc.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2014, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2018-2019, STMicroelectronics + */ + +#ifndef __STM32_ETZPC_H__ +#define __STM32_ETZPC_H__ + +#include +#include + +enum etzpc_decprot_attributes { + ETZPC_DECPROT_S_RW = 0, + ETZPC_DECPROT_NS_R_S_W = 1, + ETZPC_DECPROT_MCU_ISOLATION = 2, + ETZPC_DECPROT_NS_RW = 3, + ETZPC_DECPROT_MAX = 4, +}; + +#define ETZPC_TZMA_ALL_SECURE GENMASK_32(9, 0) +#define ETZPC_TZMA_ALL_NO_SECURE 0x0 + +/* + * Load a DECPROT configuration + * @decprot_id: ID that is the index of the DECPROT in the ETZPC interface + * @decprot_attr: Restriction access attributes + */ +void etzpc_configure_decprot(uint32_t decprot_id, + enum etzpc_decprot_attributes decprot_attr); + +/* + * Get the DECPROT attribute + * @decprot_id: ID that is the index of the DECPROT in the ETZPC interface + * Return attribute of this DECPROT + */ +enum etzpc_decprot_attributes etzpc_get_decprot(uint32_t decprot_id); + +/* + * Lock access to the DECPROT attributes + * @decprot_id: ID that is the index of the DECPROT in the ETZPC interface + */ +void etzpc_lock_decprot(uint32_t decprot_id); + +/* + * Return the lock status of the target DECPROT + * @decprot_id: ID that is the index of the DECPROT in the ETZPC interface + */ +bool etzpc_get_lock_decprot(uint32_t decprot_id); + +/* + * Configure the target TZMA read only size + * @tzma_id: ID that is the index of the TZMA in the ETZPC interface + * @tzma_value: Read-only size + */ +void etzpc_configure_tzma(uint32_t tzma_id, uint16_t tzma_value); + +/* + * Get the target TZMA read only size + * @tzma_id: ID that is the index of the TZMA in the ETZPC interface + * Return the size of read-only area + */ +uint16_t etzpc_get_tzma(uint32_t tzma_id); + +/* + * Lock the target TZMA + * @tzma_id: ID that is the index of the TZMA in the ETZPC interface + */ +void etzpc_lock_tzma(uint32_t tzma_id); + +/* + * Return the lock status of the target TZMA + * @tzma_id: ID that is the index of the TZMA in the ETZPC interface + * Return true if TZMA is locked, false otherwise + */ +bool etzpc_get_lock_tzma(uint32_t tzma_id); +#endif /*__STM32_ETZPC_H__*/ diff --git a/optee_os/core/include/drivers/stm32_gpio.h b/optee_os/core/include/drivers/stm32_gpio.h new file mode 100644 index 0000000..adc8535 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_gpio.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2017-2023, STMicroelectronics + */ + +#ifndef DRIVERS_STM32_GPIO_H +#define DRIVERS_STM32_GPIO_H + +#include +#include +#include +#include +#include + +struct pinctrl_state; +struct stm32_pinctrl; + +#ifdef CFG_STM32_GPIO +/* + * Configure pin muxing access permission: can be secure or not + * + * @bank: GPIO bank identifier as assigned by the platform + * @pin: Pin number in the GPIO bank + * @secure: True if pin is secure, false otherwise + */ +void stm32_gpio_set_secure_cfg(unsigned int bank, unsigned int pin, + bool secure); + +/* + * Get the number of GPIO pins supported by a target GPIO bank + * + * @fdt: device tree reference + * @pinctrl_node: pinctrl node which GPIO bank node belongs to + * @bank: target GPIO bank ID + * Return number of GPIO pins (>= 0) or a negative value on error + */ +int stm32_get_gpio_count(void *fdt, int pinctrl_node, unsigned int bank); + +/* + * Configure pin muxing access permission: can be secure or not + * + * @pinctrl: Pin control state where STM32_GPIO pin are to configure + * @secure: True if pin is secure, false otherwise + */ +void stm32_pinctrl_set_secure_cfg(struct pinctrl_state *pinctrl, bool secure); + +/* + * Get the bank and pin indices related to a pin control state + * @pinctrl: Pinctrl state + * @bank: Output bank indices array or NULL + * @pin: Output pin indices array or NULL + * @count: [in] Number of cells of @bank and @pin, [out] pin count in @pinctrl + */ +void stm32_gpio_pinctrl_bank_pin(struct pinctrl_state *pinctrl, + unsigned int *bank, unsigned int *pin, + unsigned int *count); +#else +static inline void +stm32_pinctrl_set_secure_cfg(struct pinctrl_state *pinctrl __unused, + bool secure __unused) +{ +} + +static inline void stm32_gpio_pinctrl_bank_pin(struct pinctrl_state *p __unused, + unsigned int *bank __unused, + unsigned int *pin __unused, + unsigned int *count __unused) +{ +} +#endif /*CFG_STM32_GPIO*/ +#endif /*DRIVERS_STM32_GPIO_H*/ diff --git a/optee_os/core/include/drivers/stm32_i2c.h b/optee_os/core/include/drivers/stm32_i2c.h new file mode 100644 index 0000000..a1c8348 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_i2c.h @@ -0,0 +1,281 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ +/* + * Copyright (c) 2017-2023, STMicroelectronics + */ + +#ifndef DRIVERS_STM32_I2C_H +#define DRIVERS_STM32_I2C_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * I2C specification values as per version 6.0, 4th of April 2014 [1], + * table 10 page 48: Characteristics of the SDA and SCL bus lines for + * Standard, Fast, and Fast-mode Plus I2C-bus devices. + * + * [1] https://www.nxp.com/docs/en/user-guide/UM10204.pdf + */ +#define I2C_STANDARD_RATE U(100000) +#define I2C_FAST_RATE U(400000) +#define I2C_FAST_PLUS_RATE U(1000000) + +/* + * struct stm32_i2c_init_s - STM32 I2C configuration data + * + * @dt_status: non-secure/secure status read from DT + * @pbase: I2C interface base address + * @reg_size: I2C interface register map size + * @clock: I2C bus/interface clock + * @addr_mode_10b_not_7b: True if 10bit addressing mode, otherwise 7bit mode + * @own_address1: 7-bit or 10-bit first device own address. + * @dual_address_mode: True if enabling Dual-Addressing mode + * @own_address2: 7-bit second device own address (Dual-Addressing mode) + * @own_address2_masks: Acknowledge mask address (Dual-Addressing mode) + * @general_call_mode: True if enbling General-Call mode + * @no_stretch_mode: If enabling the No-Stretch mode + * @rise_time: SCL clock pin rising time in nanoseconds + * @fall_time: SCL clock pin falling time in nanoseconds + * @bus_rate: Specifies the I2C clock frequency in Hertz + * @analog_filter: True if enabling analog filter + * @digital_filter_coef: filter coef (below STM32_I2C_DIGITAL_FILTER_MAX) + */ +struct stm32_i2c_init_s { + unsigned int dt_status; + paddr_t pbase; + size_t reg_size; + struct clk *clock; + bool addr_mode_10b_not_7b; + uint32_t own_address1; + bool dual_address_mode; + uint32_t own_address2; + uint32_t own_address2_masks; + bool general_call_mode; + bool no_stretch_mode; + uint32_t rise_time; + uint32_t fall_time; + uint32_t bus_rate; + bool analog_filter; + uint8_t digital_filter_coef; +}; + +enum i2c_state_e { + I2C_STATE_RESET, /* Not yet initialized */ + I2C_STATE_READY, /* Ready for use */ + I2C_STATE_BUSY, /* Internal process ongoing */ + I2C_STATE_BUSY_TX, /* Data Transmission ongoing */ + I2C_STATE_BUSY_RX, /* Data Reception ongoing */ + I2C_STATE_SUSPENDED, /* Bus is supended */ +}; + +enum i2c_mode_e { + I2C_MODE_NONE, /* No active communication */ + I2C_MODE_MASTER, /* Communication in Master Mode */ + I2C_MODE_SLAVE, /* Communication in Slave Mode */ + I2C_MODE_MEM, /* Communication in Memory Mode */ +}; + +#define I2C_ERROR_NONE U(0x0) +#define I2C_ERROR_BERR BIT(0) +#define I2C_ERROR_ARLO BIT(1) +#define I2C_ERROR_ACKF BIT(2) +#define I2C_ERROR_OVR BIT(3) +#define I2C_ERROR_DMA BIT(4) +#define I2C_ERROR_TIMEOUT BIT(5) +#define I2C_ERROR_SIZE BIT(6) + +/* I2C interface registers state */ +struct i2c_cfg { + uint32_t timingr; + uint32_t oar1; + uint32_t oar2; + uint32_t cr1; + uint32_t cr2; +}; + +/* + * I2C bus device + * @base: I2C SoC registers base address + * @reg_size: I2C SoC registers address map size + * @dt_status: non-secure/secure status read from DT + * @clock: clock ID + * @i2c_state: Driver state ID I2C_STATE_* + * @i2c_err: Last error code I2C_ERROR_* + * @saved_timing: Saved timing value if already computed + * @saved_frequency: Saved frequency value if already computed + * @sec_cfg: I2C registers configuration storage + * @pinctrl: Pin control configuration for the I2C bus in active state + * @pinctrl_sleep: Pin control configuration for the I2C bus in standby state + */ +struct i2c_handle_s { + struct io_pa_va base; + size_t reg_size; + unsigned int dt_status; + struct clk *clock; + enum i2c_state_e i2c_state; + uint32_t i2c_err; + uint32_t saved_timing; + unsigned long saved_frequency; + struct i2c_cfg sec_cfg; + struct pinctrl_state *pinctrl; + struct pinctrl_state *pinctrl_sleep; +}; + +/* + * struct stm32_i2c_dev - Bus consumer device over an STM32 I2C bus + * @i2c_dev: I2C consumer instance + * @i2c_ctrl: I2C bus control operation + * @handle: Handle on a single STM32 I2C bus interface + */ +struct stm32_i2c_dev { + struct i2c_dev i2c_dev; + struct i2c_ctrl i2c_ctrl; + struct i2c_handle_s *handle; +}; + +/* STM32 specific defines */ +#define STM32_I2C_RISE_TIME_DEFAULT U(25) /* ns */ +#define STM32_I2C_FALL_TIME_DEFAULT U(10) /* ns */ +#define STM32_I2C_ANALOG_FILTER_DELAY_MIN U(50) /* ns */ +#define STM32_I2C_ANALOG_FILTER_DELAY_MAX U(260) /* ns */ +#define STM32_I2C_DIGITAL_FILTER_MAX U(16) + +/* + * Fill struct stm32_i2c_init_s from DT content for a given I2C node + * + * @fdt: Reference to DT + * @node: Target I2C node in the DT + * @init: Output stm32_i2c_init_s structure + * @pinctrl_active: Output active I2C pinctrl state + * @pinctrl_sleep: Output suspended I2C pinctrl state + * Return a TEE_Result compliant value + */ +TEE_Result stm32_i2c_get_setup_from_fdt(void *fdt, int node, + struct stm32_i2c_init_s *init, + struct pinctrl_state **pinctrl_active, + struct pinctrl_state **pinctrl_sleep); + +/* + * Initialize I2C bus handle from input configuration directives + * + * @hi2c: Reference to I2C bus handle structure + * @init_data: Input stm32_i2c_init_s structure + * Return 0 on success else a negative value + */ +int stm32_i2c_init(struct i2c_handle_s *hi2c, + struct stm32_i2c_init_s *init_data); + +/* + * Send a memory write request in the I2C bus + * + * @hi2c: Reference to I2C bus handle structure + * @dev_addr: Target device I2C address + * @mem_addr: Target device memory address + * @mem_addr_size: Byte size of internal memory address + * @p_data: Data to be written + * @size: Byte size of the data to be written + * @timeout_ms: Timeout value in milliseconds + * Return 0 on success else a negative value + */ +int stm32_i2c_mem_write(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint32_t mem_addr, uint32_t mem_addr_size, + uint8_t *p_data, size_t size, unsigned int timeout_ms); + +/* + * Send a memory read request in the I2C bus + * + * @hi2c: Reference to I2C bus handle structure + * @dev_addr: Target device I2C address + * @mem_addr: Target device memory address + * @mem_addr_size: Byte size of internal memory address + * @p_data: Data to be read + * @size: Byte size of the data to be read + * @timeout_ms: Timeout value in milliseconds + * Return 0 on success else a negative value + */ +int stm32_i2c_mem_read(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint32_t mem_addr, uint32_t mem_addr_size, + uint8_t *p_data, size_t size, unsigned int timeout_ms); + +/* + * Send a data buffer in master mode on the I2C bus + * + * @hi2c: Reference to I2C bus handle structure + * @dev_addr: Target device I2C address + * @p_data: Data to be sent + * @size: Byte size of the data to be sent + * @timeout_ms: Timeout value in milliseconds + * Return 0 on success else a negative value + */ +int stm32_i2c_master_transmit(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint8_t *p_data, size_t size, + unsigned int timeout_ms); + +/* + * Receive a data buffer in master mode on the I2C bus + * + * @hi2c: Reference to I2C bus handle structure + * @dev_addr: Target device I2C address + * @p_data: Buffer for the received data + * @size: Byte size of the data to be received + * @timeout_ms: Timeout value in milliseconds + * Return 0 on success else a negative value + */ +int stm32_i2c_master_receive(struct i2c_handle_s *hi2c, uint32_t dev_addr, + uint8_t *p_data, size_t size, + unsigned int timeout_ms); + +/* + * Optimized 1 byte read/write function for unpaged sequences. + * 8-bit addressing mode / single byte transferred / use default I2C timeout. + * Return 0 on success else a negative value + */ +int stm32_i2c_read_write_membyte(struct i2c_handle_s *hi2c, uint16_t dev_addr, + unsigned int mem_addr, uint8_t *p_data, + bool write); + +/* + * Check link with the I2C device + * + * @hi2c: Reference to I2C bus handle structure + * @dev_addr: Target device I2C address + * @trials: Number of attempts of I2C request + * @timeout_ms: Timeout value in milliseconds for each I2C request + * Return 0 on success else a negative value + */ +bool stm32_i2c_is_device_ready(struct i2c_handle_s *hi2c, uint32_t dev_addr, + unsigned int trials, unsigned int timeout_ms); + +/* + * Suspend I2C bus. + * Bus owner is reponsible for calling stm32_i2c_suspend(). + * + * @hi2c: Reference to I2C bus handle structure + */ +void stm32_i2c_suspend(struct i2c_handle_s *hi2c); + +/* + * Resume I2C bus. + * Bus owner is reponsible for calling stm32_i2c_resume(). + * + * @hi2c: Reference to I2C bus handle structure + */ +void stm32_i2c_resume(struct i2c_handle_s *hi2c); + +/* + * Return true if I2C bus is enabled for secure world only, false otherwise + */ +static inline bool i2c_is_secure(struct i2c_handle_s *hi2c) +{ + return hi2c->dt_status == DT_STATUS_OK_SEC; +} + +#endif /* DRIVERS_STM32_I2C_H*/ diff --git a/optee_os/core/include/drivers/stm32_iwdg.h b/optee_os/core/include/drivers/stm32_iwdg.h new file mode 100644 index 0000000..f1ad734 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_iwdg.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2018-2022, STMicroelectronics - All Rights Reserved + */ + +#ifndef DRIVERS_STM32_IWDG_H +#define DRIVERS_STM32_IWDG_H + +#include + +/* + * struct stm32_iwdg_otp_data - Fuses configuration related to an IWDG + * @hw_enabled - IWDDG instance is enabled by early hardware boot stage + * @disable_on_stop - IWDG instance freezes when SoC is in STOP mode + * @disable_on_standby - IWDG instance freezes when SoC is in STANDBY mode + */ +struct stm32_iwdg_otp_data { + bool hw_enabled; + bool disable_on_stop; + bool disable_on_standby; +}; + +/* + * Platform shall implement this function for IWDG instance to retrieve its + * OTP/fuse configuration. + */ +TEE_Result stm32_get_iwdg_otp_config(paddr_t pbase, + struct stm32_iwdg_otp_data *otp_data); + +/* Refresh all registered IWDG watchdog instance */ +void stm32_iwdg_refresh(void); + +#endif /*DRIVERS_STM32_IWDG_H*/ diff --git a/optee_os/core/include/drivers/stm32_shared_io.h b/optee_os/core/include/drivers/stm32_shared_io.h new file mode 100644 index 0000000..8d39119 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_shared_io.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, STMicroelectronics + */ + +#ifndef __DRIVERS_STM32_SHARED_IO_H__ +#define __DRIVERS_STM32_SHARED_IO_H__ + +#include +#include + +/* + * Shared registers support: common lock for accessing SoC registers + * shared between several drivers. + */ +void io_clrsetbits32_stm32shregs(vaddr_t va, uint32_t clr, uint32_t set); +void io_mask32_stm32shregs(vaddr_t va, uint32_t value, uint32_t mask); + +static inline void io_setbits32_stm32shregs(vaddr_t va, uint32_t value) +{ + io_mask32_stm32shregs(va, value, value); +} + +static inline void io_clrbits32_stm32shregs(vaddr_t va, uint32_t value) +{ + io_mask32_stm32shregs(va, 0, value); +} + +#endif /* __DRIVERS_STM32_SHARED_IO_H__ */ diff --git a/optee_os/core/include/drivers/stm32_tamp.h b/optee_os/core/include/drivers/stm32_tamp.h new file mode 100644 index 0000000..88c8849 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_tamp.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2021-2022, STMicroelectronics + */ + +#ifndef __DRIVERS_STM32_TAMP_H__ +#define __DRIVERS_STM32_TAMP_H__ + +#include +#include +#include + +/* + * struct stm32_bkpregs_conf - Interface for stm32_tamp_set_secure_bkpregs() + * @nb_zone1_regs - Number of backup registers in zone 1 + * @nb_zone2_regs - Number of backup registers in zone 2 + * + * TAMP backup registers access permissions + * + * Zone 1: read/write in secure state, no access in non-secure state + * Zone 2: read/write in secure state, read-only in non-secure state + * Zone 3: read/write in secure state, read/write in non-secure state + * + * Protection zone 1 + * If nb_zone1_regs == 0 no backup register are in zone 1. + * Otherwise backup registers from TAMP_BKP0R to TAMP_BKPR are in zone 1, + * with = (@nb_zone1_regs - 1). + * + * Protection zone 2 + * If nb_zone2_regs == 0 no backup register are in zone 2. + * Otherwise backup registers from TAMP_BKPR ro TAMP_BKPR are in zone 2, + * with = @nb_zone1_regs and = (@nb_zone1_regs1 + @nb_zone2_regs - 1). + * + * Protection zone 3 + * Backup registers from TAMP_BKPR to last backup register are in zone 3, + * with = (@nb_zone1_regs1 + @nb_zone2_regs). + */ +struct stm32_bkpregs_conf { + uint32_t nb_zone1_regs; + uint32_t nb_zone2_regs; +}; + +#ifdef CFG_STM32_TAMP +/* + * stm32_tamp_set_secure_bkprwregs() - Configure backup registers zone. + * @conf - Configuration to be programmed + */ +TEE_Result stm32_tamp_set_secure_bkpregs(struct stm32_bkpregs_conf *conf); +#else +static inline +TEE_Result stm32_tamp_set_secure_bkpregs(struct stm32_bkpregs_conf *c __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif +#endif /* __DRIVERS_STM32_TAMP_H__ */ diff --git a/optee_os/core/include/drivers/stm32_uart.h b/optee_os/core/include/drivers/stm32_uart.h new file mode 100644 index 0000000..250737f --- /dev/null +++ b/optee_os/core/include/drivers/stm32_uart.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2023, STMicroelectronics + */ + +#ifndef DRIVERS_STM32_UART_H +#define DRIVERS_STM32_UART_H + +#include +#include +#include +#include +#include +#include + +struct stm32_uart_pdata { + struct io_pa_va base; + struct serial_chip chip; + bool secure; + struct clk *clock; + struct pinctrl_state *pinctrl; + struct pinctrl_state *pinctrl_sleep; +}; + +/* + * stm32_uart_init - Initialize a UART serial chip and base address + * @pd: Output initialized UART platform data + * @base: UART interface physical base address + */ +void stm32_uart_init(struct stm32_uart_pdata *pd, vaddr_t base); + +/* + * stm32_uart_init_from_dt_node - Initialize a UART instance from a DTB node + * @fdt: DTB base address + * @node: Target node offset in the DTB + * Returns an alloced (malloc) and inited UART platform data on success or NULL + * + * This function gets a STM32 UART configuration directives from a DTB node + * and initializes a UART driver instance. + * When the DTB specifies that the device is disabled, the function returns + * NULL. Other issues panic the sequence. + */ +struct stm32_uart_pdata *stm32_uart_init_from_dt_node(void *fdt, int node); + +#endif /*DRIVERS_STM32_UART_H*/ diff --git a/optee_os/core/include/drivers/stm32_vrefbuf.h b/optee_os/core/include/drivers/stm32_vrefbuf.h new file mode 100644 index 0000000..5aa5618 --- /dev/null +++ b/optee_os/core/include/drivers/stm32_vrefbuf.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2023, STMicroelectronics + */ +#ifndef DRIVERS_STM32_VREFBUF_H +#define DRIVERS_STM32_VREFBUF_H + +#include + +#ifdef CFG_STM32_VREFBUF +/* Return VREFBUF regulator handler if registered */ +struct regulator *stm32_vrefbuf_regulator(void); +#else +static inline struct regulator *stm32_vrefbuf_regulator(void) +{ + return NULL; +} +#endif + +#endif /*DRIVERS_STM32_VREFBUF_H*/ diff --git a/optee_os/core/include/drivers/stm32mp13_rcc.h b/optee_os/core/include/drivers/stm32mp13_rcc.h new file mode 100644 index 0000000..3f76ef7 --- /dev/null +++ b/optee_os/core/include/drivers/stm32mp13_rcc.h @@ -0,0 +1,1879 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2021, STMicroelectronics - All Rights Reserved + */ + +#ifndef __DRIVERS_STM32MP13_RCC_H__ +#define __DRIVERS_STM32MP13_RCC_H__ + +#include + +#define RCC_SECCFGR U(0x0) +#define RCC_MP_SREQSETR U(0x100) +#define RCC_MP_SREQCLRR U(0x104) +#define RCC_MP_APRSTCR U(0x108) +#define RCC_MP_APRSTSR U(0x10C) +#define RCC_PWRLPDLYCR U(0x110) +#define RCC_MP_GRSTCSETR U(0x114) +#define RCC_BR_RSTSCLRR U(0x118) +#define RCC_MP_RSTSSETR U(0x11C) +#define RCC_MP_RSTSCLRR U(0x120) +#define RCC_MP_IWDGFZSETR U(0x124) +#define RCC_MP_IWDGFZCLRR U(0x128) +#define RCC_MP_CIER U(0x200) +#define RCC_MP_CIFR U(0x204) +#define RCC_BDCR U(0x400) +#define RCC_RDLSICR U(0x404) +#define RCC_OCENSETR U(0x420) +#define RCC_OCENCLRR U(0x424) +#define RCC_OCRDYR U(0x428) +#define RCC_HSICFGR U(0x440) +#define RCC_CSICFGR U(0x444) +#define RCC_MCO1CFGR U(0x460) +#define RCC_MCO2CFGR U(0x464) +#define RCC_DBGCFGR U(0x468) +#define RCC_RCK12SELR U(0x480) +#define RCC_RCK3SELR U(0x484) +#define RCC_RCK4SELR U(0x488) +#define RCC_PLL1CR U(0x4A0) +#define RCC_PLL1CFGR1 U(0x4A4) +#define RCC_PLL1CFGR2 U(0x4A8) +#define RCC_PLL1FRACR U(0x4AC) +#define RCC_PLL1CSGR U(0x4B0) +#define RCC_PLL2CR U(0x4D0) +#define RCC_PLL2CFGR1 U(0x4D4) +#define RCC_PLL2CFGR2 U(0x4D8) +#define RCC_PLL2FRACR U(0x4DC) +#define RCC_PLL2CSGR U(0x4E0) +#define RCC_PLL3CR U(0x500) +#define RCC_PLL3CFGR1 U(0x504) +#define RCC_PLL3CFGR2 U(0x508) +#define RCC_PLL3FRACR U(0x50C) +#define RCC_PLL3CSGR U(0x510) +#define RCC_PLL4CR U(0x520) +#define RCC_PLL4CFGR1 U(0x524) +#define RCC_PLL4CFGR2 U(0x528) +#define RCC_PLL4FRACR U(0x52C) +#define RCC_PLL4CSGR U(0x530) +#define RCC_MPCKSELR U(0x540) +#define RCC_ASSCKSELR U(0x544) +#define RCC_MSSCKSELR U(0x548) +#define RCC_CPERCKSELR U(0x54C) +#define RCC_RTCDIVR U(0x560) +#define RCC_MPCKDIVR U(0x564) +#define RCC_AXIDIVR U(0x568) +#define RCC_MLAHBDIVR U(0x56C) +#define RCC_APB1DIVR U(0x570) +#define RCC_APB2DIVR U(0x574) +#define RCC_APB3DIVR U(0x578) +#define RCC_APB4DIVR U(0x57C) +#define RCC_APB5DIVR U(0x580) +#define RCC_APB6DIVR U(0x584) +#define RCC_TIMG1PRER U(0x5A0) +#define RCC_TIMG2PRER U(0x5A4) +#define RCC_TIMG3PRER U(0x5A8) +#define RCC_DDRITFCR U(0x5C0) +#define RCC_I2C12CKSELR U(0x600) +#define RCC_I2C345CKSELR U(0x604) +#define RCC_SPI2S1CKSELR U(0x608) +#define RCC_SPI2S23CKSELR U(0x60C) +#define RCC_SPI45CKSELR U(0x610) +#define RCC_UART12CKSELR U(0x614) +#define RCC_UART35CKSELR U(0x618) +#define RCC_UART4CKSELR U(0x61C) +#define RCC_UART6CKSELR U(0x620) +#define RCC_UART78CKSELR U(0x624) +#define RCC_LPTIM1CKSELR U(0x628) +#define RCC_LPTIM23CKSELR U(0x62C) +#define RCC_LPTIM45CKSELR U(0x630) +#define RCC_SAI1CKSELR U(0x634) +#define RCC_SAI2CKSELR U(0x638) +#define RCC_FDCANCKSELR U(0x63C) +#define RCC_SPDIFCKSELR U(0x640) +#define RCC_ADC12CKSELR U(0x644) +#define RCC_SDMMC12CKSELR U(0x648) +#define RCC_ETH12CKSELR U(0x64C) +#define RCC_USBCKSELR U(0x650) +#define RCC_QSPICKSELR U(0x654) +#define RCC_FMCCKSELR U(0x658) +#define RCC_RNG1CKSELR U(0x65C) +#define RCC_STGENCKSELR U(0x660) +#define RCC_DCMIPPCKSELR U(0x664) +#define RCC_SAESCKSELR U(0x668) +#define RCC_APB1RSTSETR U(0x6A0) +#define RCC_APB1RSTCLRR U(0x6A4) +#define RCC_APB2RSTSETR U(0x6A8) +#define RCC_APB2RSTCLRR U(0x6AC) +#define RCC_APB3RSTSETR U(0x6B0) +#define RCC_APB3RSTCLRR U(0x6B4) +#define RCC_APB4RSTSETR U(0x6B8) +#define RCC_APB4RSTCLRR U(0x6BC) +#define RCC_APB5RSTSETR U(0x6C0) +#define RCC_APB5RSTCLRR U(0x6C4) +#define RCC_APB6RSTSETR U(0x6C8) +#define RCC_APB6RSTCLRR U(0x6CC) +#define RCC_AHB2RSTSETR U(0x6D0) +#define RCC_AHB2RSTCLRR U(0x6D4) +#define RCC_AHB4RSTSETR U(0x6E0) +#define RCC_AHB4RSTCLRR U(0x6E4) +#define RCC_AHB5RSTSETR U(0x6E8) +#define RCC_AHB5RSTCLRR U(0x6EC) +#define RCC_AHB6RSTSETR U(0x6F0) +#define RCC_AHB6RSTCLRR U(0x6F4) +#define RCC_MP_APB1ENSETR U(0x700) +#define RCC_MP_APB1ENCLRR U(0x704) +#define RCC_MP_APB2ENSETR U(0x708) +#define RCC_MP_APB2ENCLRR U(0x70C) +#define RCC_MP_APB3ENSETR U(0x710) +#define RCC_MP_APB3ENCLRR U(0x714) +#define RCC_MP_S_APB3ENSETR U(0x718) +#define RCC_MP_S_APB3ENCLRR U(0x71C) +#define RCC_MP_NS_APB3ENSETR U(0x720) +#define RCC_MP_NS_APB3ENCLRR U(0x724) +#define RCC_MP_APB4ENSETR U(0x728) +#define RCC_MP_APB4ENCLRR U(0x72C) +#define RCC_MP_S_APB4ENSETR U(0x730) +#define RCC_MP_S_APB4ENCLRR U(0x734) +#define RCC_MP_NS_APB4ENSETR U(0x738) +#define RCC_MP_NS_APB4ENCLRR U(0x73C) +#define RCC_MP_APB5ENSETR U(0x740) +#define RCC_MP_APB5ENCLRR U(0x744) +#define RCC_MP_APB6ENSETR U(0x748) +#define RCC_MP_APB6ENCLRR U(0x74C) +#define RCC_MP_AHB2ENSETR U(0x750) +#define RCC_MP_AHB2ENCLRR U(0x754) +#define RCC_MP_AHB4ENSETR U(0x760) +#define RCC_MP_AHB4ENCLRR U(0x764) +#define RCC_MP_S_AHB4ENSETR U(0x768) +#define RCC_MP_S_AHB4ENCLRR U(0x76C) +#define RCC_MP_NS_AHB4ENSETR U(0x770) +#define RCC_MP_NS_AHB4ENCLRR U(0x774) +#define RCC_MP_AHB5ENSETR U(0x778) +#define RCC_MP_AHB5ENCLRR U(0x77C) +#define RCC_MP_AHB6ENSETR U(0x780) +#define RCC_MP_AHB6ENCLRR U(0x784) +#define RCC_MP_S_AHB6ENSETR U(0x788) +#define RCC_MP_S_AHB6ENCLRR U(0x78C) +#define RCC_MP_NS_AHB6ENSETR U(0x790) +#define RCC_MP_NS_AHB6ENCLRR U(0x794) +#define RCC_MP_APB1LPENSETR U(0x800) +#define RCC_MP_APB1LPENCLRR U(0x804) +#define RCC_MP_APB2LPENSETR U(0x808) +#define RCC_MP_APB2LPENCLRR U(0x80C) +#define RCC_MP_APB3LPENSETR U(0x810) +#define RCC_MP_APB3LPENCLRR U(0x814) +#define RCC_MP_S_APB3LPENSETR U(0x818) +#define RCC_MP_S_APB3LPENCLRR U(0x81C) +#define RCC_MP_NS_APB3LPENSETR U(0x820) +#define RCC_MP_NS_APB3LPENCLRR U(0x824) +#define RCC_MP_APB4LPENSETR U(0x828) +#define RCC_MP_APB4LPENCLRR U(0x82C) +#define RCC_MP_S_APB4LPENSETR U(0x830) +#define RCC_MP_S_APB4LPENCLRR U(0x834) +#define RCC_MP_NS_APB4LPENSETR U(0x838) +#define RCC_MP_NS_APB4LPENCLRR U(0x83C) +#define RCC_MP_APB5LPENSETR U(0x840) +#define RCC_MP_APB5LPENCLRR U(0x844) +#define RCC_MP_APB6LPENSETR U(0x848) +#define RCC_MP_APB6LPENCLRR U(0x84C) +#define RCC_MP_AHB2LPENSETR U(0x850) +#define RCC_MP_AHB2LPENCLRR U(0x854) +#define RCC_MP_AHB4LPENSETR U(0x858) +#define RCC_MP_AHB4LPENCLRR U(0x85C) +#define RCC_MP_S_AHB4LPENSETR U(0x868) +#define RCC_MP_S_AHB4LPENCLRR U(0x86C) +#define RCC_MP_NS_AHB4LPENSETR U(0x870) +#define RCC_MP_NS_AHB4LPENCLRR U(0x874) +#define RCC_MP_AHB5LPENSETR U(0x878) +#define RCC_MP_AHB5LPENCLRR U(0x87C) +#define RCC_MP_AHB6LPENSETR U(0x880) +#define RCC_MP_AHB6LPENCLRR U(0x884) +#define RCC_MP_S_AHB6LPENSETR U(0x888) +#define RCC_MP_S_AHB6LPENCLRR U(0x88C) +#define RCC_MP_NS_AHB6LPENSETR U(0x890) +#define RCC_MP_NS_AHB6LPENCLRR U(0x894) +#define RCC_MP_S_AXIMLPENSETR U(0x898) +#define RCC_MP_S_AXIMLPENCLRR U(0x89C) +#define RCC_MP_NS_AXIMLPENSETR U(0x8A0) +#define RCC_MP_NS_AXIMLPENCLRR U(0x8A4) +#define RCC_MP_MLAHBLPENSETR U(0x8A8) +#define RCC_MP_MLAHBLPENCLRR U(0x8AC) +#define RCC_APB3SECSR U(0x8C0) +#define RCC_APB4SECSR U(0x8C4) +#define RCC_APB5SECSR U(0x8C8) +#define RCC_APB6SECSR U(0x8CC) +#define RCC_AHB2SECSR U(0x8D0) +#define RCC_AHB4SECSR U(0x8D4) +#define RCC_AHB5SECSR U(0x8D8) +#define RCC_AHB6SECSR U(0x8DC) +#define RCC_VERR U(0xFF4) +#define RCC_IDR U(0xFF8) +#define RCC_SIDR U(0xFFC) + +/* RCC_SECCFGR register fields */ +#define RCC_SECCFGR_HSISEC BIT(0) +#define RCC_SECCFGR_CSISEC BIT(1) +#define RCC_SECCFGR_HSESEC BIT(2) +#define RCC_SECCFGR_LSISEC BIT(3) +#define RCC_SECCFGR_LSESEC BIT(4) +#define RCC_SECCFGR_PLL12SEC BIT(8) +#define RCC_SECCFGR_PLL3SEC BIT(9) +#define RCC_SECCFGR_PLL4SEC BIT(10) +#define RCC_SECCFGR_MPUSEC BIT(11) +#define RCC_SECCFGR_AXISEC BIT(12) +#define RCC_SECCFGR_MLAHBSEC BIT(13) +#define RCC_SECCFGR_APB3DIVSEC BIT(16) +#define RCC_SECCFGR_APB4DIVSEC BIT(17) +#define RCC_SECCFGR_APB5DIVSEC BIT(18) +#define RCC_SECCFGR_APB6DIVSEC BIT(19) +#define RCC_SECCFGR_TIMG3SEC BIT(20) +#define RCC_SECCFGR_CPERSEC BIT(21) +#define RCC_SECCFGR_MCO1SEC BIT(22) +#define RCC_SECCFGR_MCO2SEC BIT(23) +#define RCC_SECCFGR_STPSEC BIT(24) +#define RCC_SECCFGR_RSTSEC BIT(25) +#define RCC_SECCFGR_PWRSEC BIT(31) + +/* RCC_MP_SREQSETR register fields */ +#define RCC_MP_SREQSETR_STPREQ_P0 BIT(0) + +/* RCC_MP_SREQCLRR register fields */ +#define RCC_MP_SREQCLRR_STPREQ_P0 BIT(0) + +/* RCC_MP_APRSTCR register fields */ +#define RCC_MP_APRSTCR_RDCTLEN BIT(0) +#define RCC_MP_APRSTCR_RSTTO_MASK GENMASK_32(14, 8) +#define RCC_MP_APRSTCR_RSTTO_SHIFT 8 + +/* RCC_MP_APRSTSR register fields */ +#define RCC_MP_APRSTSR_RSTTOV_MASK GENMASK_32(14, 8) +#define RCC_MP_APRSTSR_RSTTOV_SHIFT 8 + +/* RCC_PWRLPDLYCR register fields */ +#define RCC_PWRLPDLYCR_PWRLP_DLY_MASK GENMASK_32(21, 0) +#define RCC_PWRLPDLYCR_PWRLP_DLY_SHIFT 0 + +/* RCC_MP_GRSTCSETR register fields */ +#define RCC_MP_GRSTCSETR_MPSYSRST BIT(0) +#define RCC_MP_GRSTCSETR_MPUP0RST BIT(4) + +/* RCC_BR_RSTSCLRR register fields */ +#define RCC_BR_RSTSCLRR_PORRSTF BIT(0) +#define RCC_BR_RSTSCLRR_BORRSTF BIT(1) +#define RCC_BR_RSTSCLRR_PADRSTF BIT(2) +#define RCC_BR_RSTSCLRR_HCSSRSTF BIT(3) +#define RCC_BR_RSTSCLRR_VCORERSTF BIT(4) +#define RCC_BR_RSTSCLRR_VCPURSTF BIT(5) +#define RCC_BR_RSTSCLRR_MPSYSRSTF BIT(6) +#define RCC_BR_RSTSCLRR_IWDG1RSTF BIT(8) +#define RCC_BR_RSTSCLRR_IWDG2RSTF BIT(9) +#define RCC_BR_RSTSCLRR_MPUP0RSTF BIT(13) + +/* RCC_MP_RSTSSETR register fields */ +#define RCC_MP_RSTSSETR_PORRSTF BIT(0) +#define RCC_MP_RSTSSETR_BORRSTF BIT(1) +#define RCC_MP_RSTSSETR_PADRSTF BIT(2) +#define RCC_MP_RSTSSETR_HCSSRSTF BIT(3) +#define RCC_MP_RSTSSETR_VCORERSTF BIT(4) +#define RCC_MP_RSTSSETR_VCPURSTF BIT(5) +#define RCC_MP_RSTSSETR_MPSYSRSTF BIT(6) +#define RCC_MP_RSTSSETR_IWDG1RSTF BIT(8) +#define RCC_MP_RSTSSETR_IWDG2RSTF BIT(9) +#define RCC_MP_RSTSSETR_STP2RSTF BIT(10) +#define RCC_MP_RSTSSETR_STDBYRSTF BIT(11) +#define RCC_MP_RSTSSETR_CSTDBYRSTF BIT(12) +#define RCC_MP_RSTSSETR_MPUP0RSTF BIT(13) +#define RCC_MP_RSTSSETR_SPARE BIT(15) + +/* RCC_MP_RSTSCLRR register fields */ +#define RCC_MP_RSTSCLRR_PORRSTF BIT(0) +#define RCC_MP_RSTSCLRR_BORRSTF BIT(1) +#define RCC_MP_RSTSCLRR_PADRSTF BIT(2) +#define RCC_MP_RSTSCLRR_HCSSRSTF BIT(3) +#define RCC_MP_RSTSCLRR_VCORERSTF BIT(4) +#define RCC_MP_RSTSCLRR_VCPURSTF BIT(5) +#define RCC_MP_RSTSCLRR_MPSYSRSTF BIT(6) +#define RCC_MP_RSTSCLRR_IWDG1RSTF BIT(8) +#define RCC_MP_RSTSCLRR_IWDG2RSTF BIT(9) +#define RCC_MP_RSTSCLRR_STP2RSTF BIT(10) +#define RCC_MP_RSTSCLRR_STDBYRSTF BIT(11) +#define RCC_MP_RSTSCLRR_CSTDBYRSTF BIT(12) +#define RCC_MP_RSTSCLRR_MPUP0RSTF BIT(13) +#define RCC_MP_RSTSCLRR_SPARE BIT(15) + +/* RCC_MP_IWDGFZSETR register fields */ +#define RCC_MP_IWDGFZSETR_FZ_IWDG1 BIT(0) +#define RCC_MP_IWDGFZSETR_FZ_IWDG2 BIT(1) + +/* RCC_MP_IWDGFZCLRR register fields */ +#define RCC_MP_IWDGFZCLRR_FZ_IWDG1 BIT(0) +#define RCC_MP_IWDGFZCLRR_FZ_IWDG2 BIT(1) + +/* RCC_MP_CIER register fields */ +#define RCC_MP_CIER_LSIRDYIE BIT(0) +#define RCC_MP_CIER_LSERDYIE BIT(1) +#define RCC_MP_CIER_HSIRDYIE BIT(2) +#define RCC_MP_CIER_HSERDYIE BIT(3) +#define RCC_MP_CIER_CSIRDYIE BIT(4) +#define RCC_MP_CIER_PLL1DYIE BIT(8) +#define RCC_MP_CIER_PLL2DYIE BIT(9) +#define RCC_MP_CIER_PLL3DYIE BIT(10) +#define RCC_MP_CIER_PLL4DYIE BIT(11) +#define RCC_MP_CIER_LSECSSIE BIT(16) +#define RCC_MP_CIER_WKUPIE BIT(20) + +/* RCC_MP_CIFR register fields */ +#define RCC_MP_CIFR_LSIRDYF BIT(0) +#define RCC_MP_CIFR_LSERDYF BIT(1) +#define RCC_MP_CIFR_HSIRDYF BIT(2) +#define RCC_MP_CIFR_HSERDYF BIT(3) +#define RCC_MP_CIFR_CSIRDYF BIT(4) +#define RCC_MP_CIFR_PLL1DYF BIT(8) +#define RCC_MP_CIFR_PLL2DYF BIT(9) +#define RCC_MP_CIFR_PLL3DYF BIT(10) +#define RCC_MP_CIFR_PLL4DYF BIT(11) +#define RCC_MP_CIFR_LSECSSF BIT(16) +#define RCC_MP_CIFR_WKUPF BIT(20) + +/* RCC_BDCR register fields */ +#define RCC_BDCR_LSEON BIT(0) +#define RCC_BDCR_LSEBYP BIT(1) +#define RCC_BDCR_LSERDY BIT(2) +#define RCC_BDCR_DIGBYP BIT(3) +#define RCC_BDCR_LSEDRV_MASK GENMASK_32(5, 4) +#define RCC_BDCR_LSEDRV_SHIFT 4 +#define RCC_BDCR_LSECSSON BIT(8) +#define RCC_BDCR_LSECSSD BIT(9) +#define RCC_BDCR_RTCSRC_MASK GENMASK_32(17, 16) +#define RCC_BDCR_RTCSRC_SHIFT 16 +#define RCC_BDCR_RTCCKEN BIT(20) +#define RCC_BDCR_VSWRST BIT(31) + +#define RCC_BDCR_LSEBYP_BIT 1 +#define RCC_BDCR_LSERDY_BIT 2 +#define RCC_BDCR_DIGBYP_BIT 3 +#define RCC_BDCR_LSECSSON_BIT 8 + +#define RCC_BDCR_LSEDRV_WIDTH 2 + +/* RCC_RDLSICR register fields */ +#define RCC_RDLSICR_LSION BIT(0) +#define RCC_RDLSICR_LSIRDY BIT(1) +#define RCC_RDLSICR_MRD_MASK GENMASK_32(20, 16) +#define RCC_RDLSICR_MRD_SHIFT 16 +#define RCC_RDLSICR_EADLY_MASK GENMASK_32(26, 24) +#define RCC_RDLSICR_EADLY_SHIFT 24 +#define RCC_RDLSICR_SPARE_MASK GENMASK_32(31, 27) +#define RCC_RDLSICR_SPARE_SHIFT 27 + +#define RCC_RDLSICR_LSIRDY_BIT 1 + +/* RCC_OCENSETR register fields */ +#define RCC_OCENSETR_HSION BIT(0) +#define RCC_OCENSETR_HSIKERON BIT(1) +#define RCC_OCENSETR_CSION BIT(4) +#define RCC_OCENSETR_CSIKERON BIT(5) +#define RCC_OCENSETR_DIGBYP BIT(7) +#define RCC_OCENSETR_HSEON BIT(8) +#define RCC_OCENSETR_HSEKERON BIT(9) +#define RCC_OCENSETR_HSEBYP BIT(10) +#define RCC_OCENSETR_HSECSSON BIT(11) + +#define RCC_OCENR_DIGBYP_BIT 7 +#define RCC_OCENR_HSEBYP_BIT 10 +#define RCC_OCENR_HSECSSON_BIT 11 + +/* RCC_OCENCLRR register fields */ +#define RCC_OCENCLRR_HSION BIT(0) +#define RCC_OCENCLRR_HSIKERON BIT(1) +#define RCC_OCENCLRR_CSION BIT(4) +#define RCC_OCENCLRR_CSIKERON BIT(5) +#define RCC_OCENCLRR_DIGBYP BIT(7) +#define RCC_OCENCLRR_HSEON BIT(8) +#define RCC_OCENCLRR_HSEKERON BIT(9) +#define RCC_OCENCLRR_HSEBYP BIT(10) + +/* RCC_OCRDYR register fields */ +#define RCC_OCRDYR_HSIRDY BIT(0) +#define RCC_OCRDYR_HSIDIVRDY BIT(2) +#define RCC_OCRDYR_CSIRDY BIT(4) +#define RCC_OCRDYR_HSERDY BIT(8) +#define RCC_OCRDYR_MPUCKRDY BIT(23) +#define RCC_OCRDYR_AXICKRDY BIT(24) + +#define RCC_OCRDYR_HSIRDY_BIT 0 +#define RCC_OCRDYR_HSIDIVRDY_BIT 2 +#define RCC_OCRDYR_CSIRDY_BIT 4 +#define RCC_OCRDYR_HSERDY_BIT 8 + +/* RCC_HSICFGR register fields */ +#define RCC_HSICFGR_HSIDIV_MASK GENMASK_32(1, 0) +#define RCC_HSICFGR_HSIDIV_SHIFT 0 +#define RCC_HSICFGR_HSITRIM_MASK GENMASK_32(14, 8) +#define RCC_HSICFGR_HSITRIM_SHIFT 8 +#define RCC_HSICFGR_HSICAL_MASK GENMASK_32(27, 16) +#define RCC_HSICFGR_HSICAL_SHIFT 16 + +/* RCC_CSICFGR register fields */ +#define RCC_CSICFGR_CSITRIM_MASK GENMASK_32(12, 8) +#define RCC_CSICFGR_CSITRIM_SHIFT 8 +#define RCC_CSICFGR_CSICAL_MASK GENMASK_32(23, 16) +#define RCC_CSICFGR_CSICAL_SHIFT 16 + +/* RCC_MCO1CFGR register fields */ +#define RCC_MCO1CFGR_MCO1SEL_MASK GENMASK_32(2, 0) +#define RCC_MCO1CFGR_MCO1SEL_SHIFT 0 +#define RCC_MCO1CFGR_MCO1DIV_MASK GENMASK_32(7, 4) +#define RCC_MCO1CFGR_MCO1DIV_SHIFT 4 +#define RCC_MCO1CFGR_MCO1ON BIT(12) + +/* RCC_MCO2CFGR register fields */ +#define RCC_MCO2CFGR_MCO2SEL_MASK GENMASK_32(2, 0) +#define RCC_MCO2CFGR_MCO2SEL_SHIFT 0 +#define RCC_MCO2CFGR_MCO2DIV_MASK GENMASK_32(7, 4) +#define RCC_MCO2CFGR_MCO2DIV_SHIFT 4 +#define RCC_MCO2CFGR_MCO2ON BIT(12) + +/* RCC_DBGCFGR register fields */ +#define RCC_DBGCFGR_TRACEDIV_MASK GENMASK_32(2, 0) +#define RCC_DBGCFGR_TRACEDIV_SHIFT 0 +#define RCC_DBGCFGR_DBGCKEN BIT(8) +#define RCC_DBGCFGR_TRACECKEN BIT(9) +#define RCC_DBGCFGR_DBGRST BIT(12) + +/* RCC_RCK12SELR register fields */ +#define RCC_RCK12SELR_PLL12SRC_MASK GENMASK_32(1, 0) +#define RCC_RCK12SELR_PLL12SRC_SHIFT 0 +#define RCC_RCK12SELR_PLL12SRCRDY BIT(31) + +/* RCC_RCK3SELR register fields */ +#define RCC_RCK3SELR_PLL3SRC_MASK GENMASK_32(1, 0) +#define RCC_RCK3SELR_PLL3SRC_SHIFT 0 +#define RCC_RCK3SELR_PLL3SRCRDY BIT(31) + +/* RCC_RCK4SELR register fields */ +#define RCC_RCK4SELR_PLL4SRC_MASK GENMASK_32(1, 0) +#define RCC_RCK4SELR_PLL4SRC_SHIFT 0 +#define RCC_RCK4SELR_PLL4SRCRDY BIT(31) + +/* RCC_PLL1CR register fields */ +#define RCC_PLL1CR_PLLON BIT(0) +#define RCC_PLL1CR_PLL1RDY BIT(1) +#define RCC_PLL1CR_SSCG_CTRL BIT(2) +#define RCC_PLL1CR_DIVPEN BIT(4) +#define RCC_PLL1CR_DIVQEN BIT(5) +#define RCC_PLL1CR_DIVREN BIT(6) + +/* RCC_PLL1CFGR1 register fields */ +#define RCC_PLL1CFGR1_DIVN_MASK GENMASK_32(8, 0) +#define RCC_PLL1CFGR1_DIVN_SHIFT 0 +#define RCC_PLL1CFGR1_DIVM1_MASK GENMASK_32(21, 16) +#define RCC_PLL1CFGR1_DIVM1_SHIFT 16 + +/* RCC_PLL1CFGR2 register fields */ +#define RCC_PLL1CFGR2_DIVP_MASK GENMASK_32(6, 0) +#define RCC_PLL1CFGR2_DIVP_SHIFT 0 +#define RCC_PLL1CFGR2_DIVQ_MASK GENMASK_32(14, 8) +#define RCC_PLL1CFGR2_DIVQ_SHIFT 8 +#define RCC_PLL1CFGR2_DIVR_MASK GENMASK_32(22, 16) +#define RCC_PLL1CFGR2_DIVR_SHIFT 16 + +/* RCC_PLL1FRACR register fields */ +#define RCC_PLL1FRACR_FRACV_MASK GENMASK_32(15, 3) +#define RCC_PLL1FRACR_FRACV_SHIFT 3 +#define RCC_PLL1FRACR_FRACLE BIT(16) + +/* RCC_PLL1CSGR register fields */ +#define RCC_PLL1CSGR_MOD_PER_MASK GENMASK_32(12, 0) +#define RCC_PLL1CSGR_MOD_PER_SHIFT 0 +#define RCC_PLL1CSGR_TPDFN_DIS BIT(13) +#define RCC_PLL1CSGR_RPDFN_DIS BIT(14) +#define RCC_PLL1CSGR_SSCG_MODE BIT(15) +#define RCC_PLL1CSGR_INC_STEP_MASK GENMASK_32(30, 16) +#define RCC_PLL1CSGR_INC_STEP_SHIFT 16 + +/* RCC_PLL2CR register fields */ +#define RCC_PLL2CR_PLLON BIT(0) +#define RCC_PLL2CR_PLL2RDY BIT(1) +#define RCC_PLL2CR_SSCG_CTRL BIT(2) +#define RCC_PLL2CR_DIVPEN BIT(4) +#define RCC_PLL2CR_DIVQEN BIT(5) +#define RCC_PLL2CR_DIVREN BIT(6) + +/* RCC_PLL2CFGR1 register fields */ +#define RCC_PLL2CFGR1_DIVN_MASK GENMASK_32(8, 0) +#define RCC_PLL2CFGR1_DIVN_SHIFT 0 +#define RCC_PLL2CFGR1_DIVM2_MASK GENMASK_32(21, 16) +#define RCC_PLL2CFGR1_DIVM2_SHIFT 16 + +/* RCC_PLL2CFGR2 register fields */ +#define RCC_PLL2CFGR2_DIVP_MASK GENMASK_32(6, 0) +#define RCC_PLL2CFGR2_DIVP_SHIFT 0 +#define RCC_PLL2CFGR2_DIVQ_MASK GENMASK_32(14, 8) +#define RCC_PLL2CFGR2_DIVQ_SHIFT 8 +#define RCC_PLL2CFGR2_DIVR_MASK GENMASK_32(22, 16) +#define RCC_PLL2CFGR2_DIVR_SHIFT 16 + +/* RCC_PLL2FRACR register fields */ +#define RCC_PLL2FRACR_FRACV_MASK GENMASK_32(15, 3) +#define RCC_PLL2FRACR_FRACV_SHIFT 3 +#define RCC_PLL2FRACR_FRACLE BIT(16) + +/* RCC_PLL2CSGR register fields */ +#define RCC_PLL2CSGR_MOD_PER_MASK GENMASK_32(12, 0) +#define RCC_PLL2CSGR_MOD_PER_SHIFT 0 +#define RCC_PLL2CSGR_TPDFN_DIS BIT(13) +#define RCC_PLL2CSGR_RPDFN_DIS BIT(14) +#define RCC_PLL2CSGR_SSCG_MODE BIT(15) +#define RCC_PLL2CSGR_INC_STEP_MASK GENMASK_32(30, 16) +#define RCC_PLL2CSGR_INC_STEP_SHIFT 16 + +/* RCC_PLL3CR register fields */ +#define RCC_PLL3CR_PLLON BIT(0) +#define RCC_PLL3CR_PLL3RDY BIT(1) +#define RCC_PLL3CR_SSCG_CTRL BIT(2) +#define RCC_PLL3CR_DIVPEN BIT(4) +#define RCC_PLL3CR_DIVQEN BIT(5) +#define RCC_PLL3CR_DIVREN BIT(6) + +/* RCC_PLL3CFGR1 register fields */ +#define RCC_PLL3CFGR1_DIVN_MASK GENMASK_32(8, 0) +#define RCC_PLL3CFGR1_DIVN_SHIFT 0 +#define RCC_PLL3CFGR1_DIVM3_MASK GENMASK_32(21, 16) +#define RCC_PLL3CFGR1_DIVM3_SHIFT 16 +#define RCC_PLL3CFGR1_IFRGE_MASK GENMASK_32(25, 24) +#define RCC_PLL3CFGR1_IFRGE_SHIFT 24 + +/* RCC_PLL3CFGR2 register fields */ +#define RCC_PLL3CFGR2_DIVP_MASK GENMASK_32(6, 0) +#define RCC_PLL3CFGR2_DIVP_SHIFT 0 +#define RCC_PLL3CFGR2_DIVQ_MASK GENMASK_32(14, 8) +#define RCC_PLL3CFGR2_DIVQ_SHIFT 8 +#define RCC_PLL3CFGR2_DIVR_MASK GENMASK_32(22, 16) +#define RCC_PLL3CFGR2_DIVR_SHIFT 16 + +/* RCC_PLL3FRACR register fields */ +#define RCC_PLL3FRACR_FRACV_MASK GENMASK_32(15, 3) +#define RCC_PLL3FRACR_FRACV_SHIFT 3 +#define RCC_PLL3FRACR_FRACLE BIT(16) + +/* RCC_PLL3CSGR register fields */ +#define RCC_PLL3CSGR_MOD_PER_MASK GENMASK_32(12, 0) +#define RCC_PLL3CSGR_MOD_PER_SHIFT 0 +#define RCC_PLL3CSGR_TPDFN_DIS BIT(13) +#define RCC_PLL3CSGR_RPDFN_DIS BIT(14) +#define RCC_PLL3CSGR_SSCG_MODE BIT(15) +#define RCC_PLL3CSGR_INC_STEP_MASK GENMASK_32(30, 16) +#define RCC_PLL3CSGR_INC_STEP_SHIFT 16 + +/* RCC_PLL4CR register fields */ +#define RCC_PLL4CR_PLLON BIT(0) +#define RCC_PLL4CR_PLL4RDY BIT(1) +#define RCC_PLL4CR_SSCG_CTRL BIT(2) +#define RCC_PLL4CR_DIVPEN BIT(4) +#define RCC_PLL4CR_DIVQEN BIT(5) +#define RCC_PLL4CR_DIVREN BIT(6) + +/* RCC_PLL4CFGR1 register fields */ +#define RCC_PLL4CFGR1_DIVN_MASK GENMASK_32(8, 0) +#define RCC_PLL4CFGR1_DIVN_SHIFT 0 +#define RCC_PLL4CFGR1_DIVM4_MASK GENMASK_32(21, 16) +#define RCC_PLL4CFGR1_DIVM4_SHIFT 16 +#define RCC_PLL4CFGR1_IFRGE_MASK GENMASK_32(25, 24) +#define RCC_PLL4CFGR1_IFRGE_SHIFT 24 + +/* RCC_PLL4CFGR2 register fields */ +#define RCC_PLL4CFGR2_DIVP_MASK GENMASK_32(6, 0) +#define RCC_PLL4CFGR2_DIVP_SHIFT 0 +#define RCC_PLL4CFGR2_DIVQ_MASK GENMASK_32(14, 8) +#define RCC_PLL4CFGR2_DIVQ_SHIFT 8 +#define RCC_PLL4CFGR2_DIVR_MASK GENMASK_32(22, 16) +#define RCC_PLL4CFGR2_DIVR_SHIFT 16 + +/* RCC_PLL4FRACR register fields */ +#define RCC_PLL4FRACR_FRACV_MASK GENMASK_32(15, 3) +#define RCC_PLL4FRACR_FRACV_SHIFT 3 +#define RCC_PLL4FRACR_FRACLE BIT(16) + +/* RCC_PLL4CSGR register fields */ +#define RCC_PLL4CSGR_MOD_PER_MASK GENMASK_32(12, 0) +#define RCC_PLL4CSGR_MOD_PER_SHIFT 0 +#define RCC_PLL4CSGR_TPDFN_DIS BIT(13) +#define RCC_PLL4CSGR_RPDFN_DIS BIT(14) +#define RCC_PLL4CSGR_SSCG_MODE BIT(15) +#define RCC_PLL4CSGR_INC_STEP_MASK GENMASK_32(30, 16) +#define RCC_PLL4CSGR_INC_STEP_SHIFT 16 + +/* RCC_MPCKSELR register fields */ +#define RCC_MPCKSELR_MPUSRC_MASK GENMASK_32(1, 0) +#define RCC_MPCKSELR_MPUSRC_SHIFT 0 +#define RCC_MPCKSELR_MPUSRCRDY BIT(31) + +/* RCC_ASSCKSELR register fields */ +#define RCC_ASSCKSELR_AXISSRC_MASK GENMASK_32(2, 0) +#define RCC_ASSCKSELR_AXISSRC_SHIFT 0 +#define RCC_ASSCKSELR_AXISSRCRDY BIT(31) + +/* RCC_MSSCKSELR register fields */ +#define RCC_MSSCKSELR_MLAHBSSRC_MASK GENMASK_32(1, 0) +#define RCC_MSSCKSELR_MLAHBSSRC_SHIFT 0 +#define RCC_MSSCKSELR_MLAHBSSRCRDY BIT(31) + +/* RCC_CPERCKSELR register fields */ +#define RCC_CPERCKSELR_CKPERSRC_MASK GENMASK_32(1, 0) +#define RCC_CPERCKSELR_CKPERSRC_SHIFT 0 + +/* RCC_RTCDIVR register fields */ +#define RCC_RTCDIVR_RTCDIV_MASK GENMASK_32(5, 0) +#define RCC_RTCDIVR_RTCDIV_SHIFT 0 + +/* RCC_MPCKDIVR register fields */ +#define RCC_MPCKDIVR_MPUDIV_MASK GENMASK_32(3, 0) +#define RCC_MPCKDIVR_MPUDIV_SHIFT 0 +#define RCC_MPCKDIVR_MPUDIVRDY BIT(31) + +/* RCC_AXIDIVR register fields */ +#define RCC_AXIDIVR_AXIDIV_MASK GENMASK_32(2, 0) +#define RCC_AXIDIVR_AXIDIV_SHIFT 0 +#define RCC_AXIDIVR_AXIDIVRDY BIT(31) + +/* RCC_MLAHBDIVR register fields */ +#define RCC_MLAHBDIVR_MLAHBDIV_MASK GENMASK_32(3, 0) +#define RCC_MLAHBDIVR_MLAHBDIV_SHIFT 0 +#define RCC_MLAHBDIVR_MLAHBDIVRDY BIT(31) + +/* RCC_APB1DIVR register fields */ +#define RCC_APB1DIVR_APB1DIV_MASK GENMASK_32(2, 0) +#define RCC_APB1DIVR_APB1DIV_SHIFT 0 +#define RCC_APB1DIVR_APB1DIVRDY BIT(31) + +/* RCC_APB2DIVR register fields */ +#define RCC_APB2DIVR_APB2DIV_MASK GENMASK_32(2, 0) +#define RCC_APB2DIVR_APB2DIV_SHIFT 0 +#define RCC_APB2DIVR_APB2DIVRDY BIT(31) + +/* RCC_APB3DIVR register fields */ +#define RCC_APB3DIVR_APB3DIV_MASK GENMASK_32(2, 0) +#define RCC_APB3DIVR_APB3DIV_SHIFT 0 +#define RCC_APB3DIVR_APB3DIVRDY BIT(31) + +/* RCC_APB4DIVR register fields */ +#define RCC_APB4DIVR_APB4DIV_MASK GENMASK_32(2, 0) +#define RCC_APB4DIVR_APB4DIV_SHIFT 0 +#define RCC_APB4DIVR_APB4DIVRDY BIT(31) + +/* RCC_APB5DIVR register fields */ +#define RCC_APB5DIVR_APB5DIV_MASK GENMASK_32(2, 0) +#define RCC_APB5DIVR_APB5DIV_SHIFT 0 +#define RCC_APB5DIVR_APB5DIVRDY BIT(31) + +/* RCC_APB6DIVR register fields */ +#define RCC_APB6DIVR_APB6DIV_MASK GENMASK_32(2, 0) +#define RCC_APB6DIVR_APB6DIV_SHIFT 0 +#define RCC_APB6DIVR_APB6DIVRDY BIT(31) + +/* RCC_TIMG1PRER register fields */ +#define RCC_TIMG1PRER_TIMG1PRE BIT(0) +#define RCC_TIMG1PRER_TIMG1PRERDY BIT(31) + +/* RCC_TIMG2PRER register fields */ +#define RCC_TIMG2PRER_TIMG2PRE BIT(0) +#define RCC_TIMG2PRER_TIMG2PRERDY BIT(31) + +/* RCC_TIMG3PRER register fields */ +#define RCC_TIMG3PRER_TIMG3PRE BIT(0) +#define RCC_TIMG3PRER_TIMG3PRERDY BIT(31) + +/* RCC_DDRITFCR register fields */ +#define RCC_DDRITFCR_DDRC1EN BIT(0) +#define RCC_DDRITFCR_DDRC1LPEN BIT(1) +#define RCC_DDRITFCR_DDRPHYCEN BIT(4) +#define RCC_DDRITFCR_DDRPHYCLPEN BIT(5) +#define RCC_DDRITFCR_DDRCAPBEN BIT(6) +#define RCC_DDRITFCR_DDRCAPBLPEN BIT(7) +#define RCC_DDRITFCR_AXIDCGEN BIT(8) +#define RCC_DDRITFCR_DDRPHYCAPBEN BIT(9) +#define RCC_DDRITFCR_DDRPHYCAPBLPEN BIT(10) +#define RCC_DDRITFCR_KERDCG_DLY_MASK GENMASK_32(13, 11) +#define RCC_DDRITFCR_KERDCG_DLY_SHIFT 11 +#define RCC_DDRITFCR_DDRCAPBRST BIT(14) +#define RCC_DDRITFCR_DDRCAXIRST BIT(15) +#define RCC_DDRITFCR_DDRCORERST BIT(16) +#define RCC_DDRITFCR_DPHYAPBRST BIT(17) +#define RCC_DDRITFCR_DPHYRST BIT(18) +#define RCC_DDRITFCR_DPHYCTLRST BIT(19) +#define RCC_DDRITFCR_DDRCKMOD_MASK GENMASK_32(22, 20) +#define RCC_DDRITFCR_DDRCKMOD_SHIFT 20 +#define RCC_DDRITFCR_GSKPMOD BIT(23) +#define RCC_DDRITFCR_GSKPCTRL BIT(24) +#define RCC_DDRITFCR_DFILP_WIDTH_MASK GENMASK_32(27, 25) +#define RCC_DDRITFCR_DFILP_WIDTH_SHIFT 25 +#define RCC_DDRITFCR_GSKP_DUR_MASK GENMASK_32(31, 28) +#define RCC_DDRITFCR_GSKP_DUR_SHIFT 28 + +/* RCC_I2C12CKSELR register fields */ +#define RCC_I2C12CKSELR_I2C12SRC_MASK GENMASK_32(2, 0) +#define RCC_I2C12CKSELR_I2C12SRC_SHIFT 0 + +/* RCC_I2C345CKSELR register fields */ +#define RCC_I2C345CKSELR_I2C3SRC_MASK GENMASK_32(2, 0) +#define RCC_I2C345CKSELR_I2C3SRC_SHIFT 0 +#define RCC_I2C345CKSELR_I2C4SRC_MASK GENMASK_32(5, 3) +#define RCC_I2C345CKSELR_I2C4SRC_SHIFT 3 +#define RCC_I2C345CKSELR_I2C5SRC_MASK GENMASK_32(8, 6) +#define RCC_I2C345CKSELR_I2C5SRC_SHIFT 6 + +/* RCC_SPI2S1CKSELR register fields */ +#define RCC_SPI2S1CKSELR_SPI1SRC_MASK GENMASK_32(2, 0) +#define RCC_SPI2S1CKSELR_SPI1SRC_SHIFT 0 + +/* RCC_SPI2S23CKSELR register fields */ +#define RCC_SPI2S23CKSELR_SPI23SRC_MASK GENMASK_32(2, 0) +#define RCC_SPI2S23CKSELR_SPI23SRC_SHIFT 0 + +/* RCC_SPI45CKSELR register fields */ +#define RCC_SPI45CKSELR_SPI4SRC_MASK GENMASK_32(2, 0) +#define RCC_SPI45CKSELR_SPI4SRC_SHIFT 0 +#define RCC_SPI45CKSELR_SPI5SRC_MASK GENMASK_32(5, 3) +#define RCC_SPI45CKSELR_SPI5SRC_SHIFT 3 + +/* RCC_UART12CKSELR register fields */ +#define RCC_UART12CKSELR_UART1SRC_MASK GENMASK_32(2, 0) +#define RCC_UART12CKSELR_UART1SRC_SHIFT 0 +#define RCC_UART12CKSELR_UART2SRC_MASK GENMASK_32(5, 3) +#define RCC_UART12CKSELR_UART2SRC_SHIFT 3 + +/* RCC_UART35CKSELR register fields */ +#define RCC_UART35CKSELR_UART35SRC_MASK GENMASK_32(2, 0) +#define RCC_UART35CKSELR_UART35SRC_SHIFT 0 + +/* RCC_UART4CKSELR register fields */ +#define RCC_UART4CKSELR_UART4SRC_MASK GENMASK_32(2, 0) +#define RCC_UART4CKSELR_UART4SRC_SHIFT 0 + +/* RCC_UART6CKSELR register fields */ +#define RCC_UART6CKSELR_UART6SRC_MASK GENMASK_32(2, 0) +#define RCC_UART6CKSELR_UART6SRC_SHIFT 0 + +/* RCC_UART78CKSELR register fields */ +#define RCC_UART78CKSELR_UART78SRC_MASK GENMASK_32(2, 0) +#define RCC_UART78CKSELR_UART78SRC_SHIFT 0 + +/* RCC_LPTIM1CKSELR register fields */ +#define RCC_LPTIM1CKSELR_LPTIM1SRC_MASK GENMASK_32(2, 0) +#define RCC_LPTIM1CKSELR_LPTIM1SRC_SHIFT 0 + +/* RCC_LPTIM23CKSELR register fields */ +#define RCC_LPTIM23CKSELR_LPTIM2SRC_MASK GENMASK_32(2, 0) +#define RCC_LPTIM23CKSELR_LPTIM2SRC_SHIFT 0 +#define RCC_LPTIM23CKSELR_LPTIM3SRC_MASK GENMASK_32(5, 3) +#define RCC_LPTIM23CKSELR_LPTIM3SRC_SHIFT 3 + +/* RCC_LPTIM45CKSELR register fields */ +#define RCC_LPTIM45CKSELR_LPTIM45SRC_MASK GENMASK_32(2, 0) +#define RCC_LPTIM45CKSELR_LPTIM45SRC_SHIFT 0 + +/* RCC_SAI1CKSELR register fields */ +#define RCC_SAI1CKSELR_SAI1SRC_MASK GENMASK_32(2, 0) +#define RCC_SAI1CKSELR_SAI1SRC_SHIFT 0 + +/* RCC_SAI2CKSELR register fields */ +#define RCC_SAI2CKSELR_SAI2SRC_MASK GENMASK_32(2, 0) +#define RCC_SAI2CKSELR_SAI2SRC_SHIFT 0 + +/* RCC_FDCANCKSELR register fields */ +#define RCC_FDCANCKSELR_FDCANSRC_MASK GENMASK_32(1, 0) +#define RCC_FDCANCKSELR_FDCANSRC_SHIFT 0 + +/* RCC_SPDIFCKSELR register fields */ +#define RCC_SPDIFCKSELR_SPDIFSRC_MASK GENMASK_32(1, 0) +#define RCC_SPDIFCKSELR_SPDIFSRC_SHIFT 0 + +/* RCC_ADC12CKSELR register fields */ +#define RCC_ADC12CKSELR_ADC1SRC_MASK GENMASK_32(1, 0) +#define RCC_ADC12CKSELR_ADC1SRC_SHIFT 0 +#define RCC_ADC12CKSELR_ADC2SRC_MASK GENMASK_32(3, 2) +#define RCC_ADC12CKSELR_ADC2SRC_SHIFT 2 + +/* RCC_SDMMC12CKSELR register fields */ +#define RCC_SDMMC12CKSELR_SDMMC1SRC_MASK GENMASK_32(2, 0) +#define RCC_SDMMC12CKSELR_SDMMC1SRC_SHIFT 0 +#define RCC_SDMMC12CKSELR_SDMMC2SRC_MASK GENMASK_32(5, 3) +#define RCC_SDMMC12CKSELR_SDMMC2SRC_SHIFT 3 + +/* RCC_ETH12CKSELR register fields */ +#define RCC_ETH12CKSELR_ETH1SRC_MASK GENMASK_32(1, 0) +#define RCC_ETH12CKSELR_ETH1SRC_SHIFT 0 +#define RCC_ETH12CKSELR_ETH1PTPDIV_MASK GENMASK_32(7, 4) +#define RCC_ETH12CKSELR_ETH1PTPDIV_SHIFT 4 +#define RCC_ETH12CKSELR_ETH2SRC_MASK GENMASK_32(9, 8) +#define RCC_ETH12CKSELR_ETH2SRC_SHIFT 8 +#define RCC_ETH12CKSELR_ETH2PTPDIV_MASK GENMASK_32(15, 12) +#define RCC_ETH12CKSELR_ETH2PTPDIV_SHIFT 12 + +/* RCC_USBCKSELR register fields */ +#define RCC_USBCKSELR_USBPHYSRC_MASK GENMASK_32(1, 0) +#define RCC_USBCKSELR_USBPHYSRC_SHIFT 0 +#define RCC_USBCKSELR_USBOSRC BIT(4) + +/* RCC_QSPICKSELR register fields */ +#define RCC_QSPICKSELR_QSPISRC_MASK GENMASK_32(1, 0) +#define RCC_QSPICKSELR_QSPISRC_SHIFT 0 + +/* RCC_FMCCKSELR register fields */ +#define RCC_FMCCKSELR_FMCSRC_MASK GENMASK_32(1, 0) +#define RCC_FMCCKSELR_FMCSRC_SHIFT 0 + +/* RCC_RNG1CKSELR register fields */ +#define RCC_RNG1CKSELR_RNG1SRC_MASK GENMASK_32(1, 0) +#define RCC_RNG1CKSELR_RNG1SRC_SHIFT 0 + +/* RCC_STGENCKSELR register fields */ +#define RCC_STGENCKSELR_STGENSRC_MASK GENMASK_32(1, 0) +#define RCC_STGENCKSELR_STGENSRC_SHIFT 0 + +/* RCC_DCMIPPCKSELR register fields */ +#define RCC_DCMIPPCKSELR_DCMIPPSRC_MASK GENMASK_32(1, 0) +#define RCC_DCMIPPCKSELR_DCMIPPSRC_SHIFT 0 + +/* RCC_SAESCKSELR register fields */ +#define RCC_SAESCKSELR_SAESSRC_MASK GENMASK_32(1, 0) +#define RCC_SAESCKSELR_SAESSRC_SHIFT 0 + +/* RCC_APB1RSTSETR register fields */ +#define RCC_APB1RSTSETR_TIM2RST BIT(0) +#define RCC_APB1RSTSETR_TIM3RST BIT(1) +#define RCC_APB1RSTSETR_TIM4RST BIT(2) +#define RCC_APB1RSTSETR_TIM5RST BIT(3) +#define RCC_APB1RSTSETR_TIM6RST BIT(4) +#define RCC_APB1RSTSETR_TIM7RST BIT(5) +#define RCC_APB1RSTSETR_LPTIM1RST BIT(9) +#define RCC_APB1RSTSETR_SPI2RST BIT(11) +#define RCC_APB1RSTSETR_SPI3RST BIT(12) +#define RCC_APB1RSTSETR_USART3RST BIT(15) +#define RCC_APB1RSTSETR_UART4RST BIT(16) +#define RCC_APB1RSTSETR_UART5RST BIT(17) +#define RCC_APB1RSTSETR_UART7RST BIT(18) +#define RCC_APB1RSTSETR_UART8RST BIT(19) +#define RCC_APB1RSTSETR_I2C1RST BIT(21) +#define RCC_APB1RSTSETR_I2C2RST BIT(22) +#define RCC_APB1RSTSETR_SPDIFRST BIT(26) + +/* RCC_APB1RSTCLRR register fields */ +#define RCC_APB1RSTCLRR_TIM2RST BIT(0) +#define RCC_APB1RSTCLRR_TIM3RST BIT(1) +#define RCC_APB1RSTCLRR_TIM4RST BIT(2) +#define RCC_APB1RSTCLRR_TIM5RST BIT(3) +#define RCC_APB1RSTCLRR_TIM6RST BIT(4) +#define RCC_APB1RSTCLRR_TIM7RST BIT(5) +#define RCC_APB1RSTCLRR_LPTIM1RST BIT(9) +#define RCC_APB1RSTCLRR_SPI2RST BIT(11) +#define RCC_APB1RSTCLRR_SPI3RST BIT(12) +#define RCC_APB1RSTCLRR_USART3RST BIT(15) +#define RCC_APB1RSTCLRR_UART4RST BIT(16) +#define RCC_APB1RSTCLRR_UART5RST BIT(17) +#define RCC_APB1RSTCLRR_UART7RST BIT(18) +#define RCC_APB1RSTCLRR_UART8RST BIT(19) +#define RCC_APB1RSTCLRR_I2C1RST BIT(21) +#define RCC_APB1RSTCLRR_I2C2RST BIT(22) +#define RCC_APB1RSTCLRR_SPDIFRST BIT(26) + +/* RCC_APB2RSTSETR register fields */ +#define RCC_APB2RSTSETR_TIM1RST BIT(0) +#define RCC_APB2RSTSETR_TIM8RST BIT(1) +#define RCC_APB2RSTSETR_SPI1RST BIT(8) +#define RCC_APB2RSTSETR_USART6RST BIT(13) +#define RCC_APB2RSTSETR_SAI1RST BIT(16) +#define RCC_APB2RSTSETR_SAI2RST BIT(17) +#define RCC_APB2RSTSETR_DFSDMRST BIT(20) +#define RCC_APB2RSTSETR_FDCANRST BIT(24) + +/* RCC_APB2RSTCLRR register fields */ +#define RCC_APB2RSTCLRR_TIM1RST BIT(0) +#define RCC_APB2RSTCLRR_TIM8RST BIT(1) +#define RCC_APB2RSTCLRR_SPI1RST BIT(8) +#define RCC_APB2RSTCLRR_USART6RST BIT(13) +#define RCC_APB2RSTCLRR_SAI1RST BIT(16) +#define RCC_APB2RSTCLRR_SAI2RST BIT(17) +#define RCC_APB2RSTCLRR_DFSDMRST BIT(20) +#define RCC_APB2RSTCLRR_FDCANRST BIT(24) + +/* RCC_APB3RSTSETR register fields */ +#define RCC_APB3RSTSETR_LPTIM2RST BIT(0) +#define RCC_APB3RSTSETR_LPTIM3RST BIT(1) +#define RCC_APB3RSTSETR_LPTIM4RST BIT(2) +#define RCC_APB3RSTSETR_LPTIM5RST BIT(3) +#define RCC_APB3RSTSETR_SYSCFGRST BIT(11) +#define RCC_APB3RSTSETR_VREFRST BIT(13) +#define RCC_APB3RSTSETR_DTSRST BIT(16) +#define RCC_APB3RSTSETR_PMBCTRLRST BIT(17) + +/* RCC_APB3RSTCLRR register fields */ +#define RCC_APB3RSTCLRR_LPTIM2RST BIT(0) +#define RCC_APB3RSTCLRR_LPTIM3RST BIT(1) +#define RCC_APB3RSTCLRR_LPTIM4RST BIT(2) +#define RCC_APB3RSTCLRR_LPTIM5RST BIT(3) +#define RCC_APB3RSTCLRR_SYSCFGRST BIT(11) +#define RCC_APB3RSTCLRR_VREFRST BIT(13) +#define RCC_APB3RSTCLRR_DTSRST BIT(16) +#define RCC_APB3RSTCLRR_PMBCTRLRST BIT(17) + +/* RCC_APB4RSTSETR register fields */ +#define RCC_APB4RSTSETR_LTDCRST BIT(0) +#define RCC_APB4RSTSETR_DCMIPPRST BIT(1) +#define RCC_APB4RSTSETR_DDRPERFMRST BIT(8) +#define RCC_APB4RSTSETR_USBPHYRST BIT(16) + +/* RCC_APB4RSTCLRR register fields */ +#define RCC_APB4RSTCLRR_LTDCRST BIT(0) +#define RCC_APB4RSTCLRR_DCMIPPRST BIT(1) +#define RCC_APB4RSTCLRR_DDRPERFMRST BIT(8) +#define RCC_APB4RSTCLRR_USBPHYRST BIT(16) + +/* RCC_APB5RSTSETR register fields */ +#define RCC_APB5RSTSETR_STGENRST BIT(20) + +/* RCC_APB5RSTCLRR register fields */ +#define RCC_APB5RSTCLRR_STGENRST BIT(20) + +/* RCC_APB6RSTSETR register fields */ +#define RCC_APB6RSTSETR_USART1RST BIT(0) +#define RCC_APB6RSTSETR_USART2RST BIT(1) +#define RCC_APB6RSTSETR_SPI4RST BIT(2) +#define RCC_APB6RSTSETR_SPI5RST BIT(3) +#define RCC_APB6RSTSETR_I2C3RST BIT(4) +#define RCC_APB6RSTSETR_I2C4RST BIT(5) +#define RCC_APB6RSTSETR_I2C5RST BIT(6) +#define RCC_APB6RSTSETR_TIM12RST BIT(7) +#define RCC_APB6RSTSETR_TIM13RST BIT(8) +#define RCC_APB6RSTSETR_TIM14RST BIT(9) +#define RCC_APB6RSTSETR_TIM15RST BIT(10) +#define RCC_APB6RSTSETR_TIM16RST BIT(11) +#define RCC_APB6RSTSETR_TIM17RST BIT(12) + +/* RCC_APB6RSTCLRR register fields */ +#define RCC_APB6RSTCLRR_USART1RST BIT(0) +#define RCC_APB6RSTCLRR_USART2RST BIT(1) +#define RCC_APB6RSTCLRR_SPI4RST BIT(2) +#define RCC_APB6RSTCLRR_SPI5RST BIT(3) +#define RCC_APB6RSTCLRR_I2C3RST BIT(4) +#define RCC_APB6RSTCLRR_I2C4RST BIT(5) +#define RCC_APB6RSTCLRR_I2C5RST BIT(6) +#define RCC_APB6RSTCLRR_TIM12RST BIT(7) +#define RCC_APB6RSTCLRR_TIM13RST BIT(8) +#define RCC_APB6RSTCLRR_TIM14RST BIT(9) +#define RCC_APB6RSTCLRR_TIM15RST BIT(10) +#define RCC_APB6RSTCLRR_TIM16RST BIT(11) +#define RCC_APB6RSTCLRR_TIM17RST BIT(12) + +/* RCC_AHB2RSTSETR register fields */ +#define RCC_AHB2RSTSETR_DMA1RST BIT(0) +#define RCC_AHB2RSTSETR_DMA2RST BIT(1) +#define RCC_AHB2RSTSETR_DMAMUX1RST BIT(2) +#define RCC_AHB2RSTSETR_DMA3RST BIT(3) +#define RCC_AHB2RSTSETR_DMAMUX2RST BIT(4) +#define RCC_AHB2RSTSETR_ADC1RST BIT(5) +#define RCC_AHB2RSTSETR_ADC2RST BIT(6) +#define RCC_AHB2RSTSETR_USBORST BIT(8) + +/* RCC_AHB2RSTCLRR register fields */ +#define RCC_AHB2RSTCLRR_DMA1RST BIT(0) +#define RCC_AHB2RSTCLRR_DMA2RST BIT(1) +#define RCC_AHB2RSTCLRR_DMAMUX1RST BIT(2) +#define RCC_AHB2RSTCLRR_DMA3RST BIT(3) +#define RCC_AHB2RSTCLRR_DMAMUX2RST BIT(4) +#define RCC_AHB2RSTCLRR_ADC1RST BIT(5) +#define RCC_AHB2RSTCLRR_ADC2RST BIT(6) +#define RCC_AHB2RSTCLRR_USBORST BIT(8) + +/* RCC_AHB4RSTSETR register fields */ +#define RCC_AHB4RSTSETR_GPIOARST BIT(0) +#define RCC_AHB4RSTSETR_GPIOBRST BIT(1) +#define RCC_AHB4RSTSETR_GPIOCRST BIT(2) +#define RCC_AHB4RSTSETR_GPIODRST BIT(3) +#define RCC_AHB4RSTSETR_GPIOERST BIT(4) +#define RCC_AHB4RSTSETR_GPIOFRST BIT(5) +#define RCC_AHB4RSTSETR_GPIOGRST BIT(6) +#define RCC_AHB4RSTSETR_GPIOHRST BIT(7) +#define RCC_AHB4RSTSETR_GPIOIRST BIT(8) +#define RCC_AHB4RSTSETR_TSCRST BIT(15) + +/* RCC_AHB4RSTCLRR register fields */ +#define RCC_AHB4RSTCLRR_GPIOARST BIT(0) +#define RCC_AHB4RSTCLRR_GPIOBRST BIT(1) +#define RCC_AHB4RSTCLRR_GPIOCRST BIT(2) +#define RCC_AHB4RSTCLRR_GPIODRST BIT(3) +#define RCC_AHB4RSTCLRR_GPIOERST BIT(4) +#define RCC_AHB4RSTCLRR_GPIOFRST BIT(5) +#define RCC_AHB4RSTCLRR_GPIOGRST BIT(6) +#define RCC_AHB4RSTCLRR_GPIOHRST BIT(7) +#define RCC_AHB4RSTCLRR_GPIOIRST BIT(8) +#define RCC_AHB4RSTCLRR_TSCRST BIT(15) + +/* RCC_AHB5RSTSETR register fields */ +#define RCC_AHB5RSTSETR_PKARST BIT(2) +#define RCC_AHB5RSTSETR_SAESRST BIT(3) +#define RCC_AHB5RSTSETR_CRYP1RST BIT(4) +#define RCC_AHB5RSTSETR_HASH1RST BIT(5) +#define RCC_AHB5RSTSETR_RNG1RST BIT(6) +#define RCC_AHB5RSTSETR_AXIMCRST BIT(16) + +/* RCC_AHB5RSTCLRR register fields */ +#define RCC_AHB5RSTCLRR_PKARST BIT(2) +#define RCC_AHB5RSTCLRR_SAESRST BIT(3) +#define RCC_AHB5RSTCLRR_CRYP1RST BIT(4) +#define RCC_AHB5RSTCLRR_HASH1RST BIT(5) +#define RCC_AHB5RSTCLRR_RNG1RST BIT(6) +#define RCC_AHB5RSTCLRR_AXIMCRST BIT(16) + +/* RCC_AHB6RSTSETR register fields */ +#define RCC_AHB6RSTSETR_MDMARST BIT(0) +#define RCC_AHB6RSTSETR_MCERST BIT(1) +#define RCC_AHB6RSTSETR_ETH1MACRST BIT(10) +#define RCC_AHB6RSTSETR_FMCRST BIT(12) +#define RCC_AHB6RSTSETR_QSPIRST BIT(14) +#define RCC_AHB6RSTSETR_SDMMC1RST BIT(16) +#define RCC_AHB6RSTSETR_SDMMC2RST BIT(17) +#define RCC_AHB6RSTSETR_CRC1RST BIT(20) +#define RCC_AHB6RSTSETR_USBHRST BIT(24) +#define RCC_AHB6RSTSETR_ETH2MACRST BIT(30) + +/* RCC_AHB6RSTCLRR register fields */ +#define RCC_AHB6RSTCLRR_MDMARST BIT(0) +#define RCC_AHB6RSTCLRR_MCERST BIT(1) +#define RCC_AHB6RSTCLRR_ETH1MACRST BIT(10) +#define RCC_AHB6RSTCLRR_FMCRST BIT(12) +#define RCC_AHB6RSTCLRR_QSPIRST BIT(14) +#define RCC_AHB6RSTCLRR_SDMMC1RST BIT(16) +#define RCC_AHB6RSTCLRR_SDMMC2RST BIT(17) +#define RCC_AHB6RSTCLRR_CRC1RST BIT(20) +#define RCC_AHB6RSTCLRR_USBHRST BIT(24) +#define RCC_AHB6RSTCLRR_ETH2MACRST BIT(30) + +/* RCC_MP_APB1ENSETR register fields */ +#define RCC_MP_APB1ENSETR_TIM2EN BIT(0) +#define RCC_MP_APB1ENSETR_TIM3EN BIT(1) +#define RCC_MP_APB1ENSETR_TIM4EN BIT(2) +#define RCC_MP_APB1ENSETR_TIM5EN BIT(3) +#define RCC_MP_APB1ENSETR_TIM6EN BIT(4) +#define RCC_MP_APB1ENSETR_TIM7EN BIT(5) +#define RCC_MP_APB1ENSETR_LPTIM1EN BIT(9) +#define RCC_MP_APB1ENSETR_SPI2EN BIT(11) +#define RCC_MP_APB1ENSETR_SPI3EN BIT(12) +#define RCC_MP_APB1ENSETR_USART3EN BIT(15) +#define RCC_MP_APB1ENSETR_UART4EN BIT(16) +#define RCC_MP_APB1ENSETR_UART5EN BIT(17) +#define RCC_MP_APB1ENSETR_UART7EN BIT(18) +#define RCC_MP_APB1ENSETR_UART8EN BIT(19) +#define RCC_MP_APB1ENSETR_I2C1EN BIT(21) +#define RCC_MP_APB1ENSETR_I2C2EN BIT(22) +#define RCC_MP_APB1ENSETR_SPDIFEN BIT(26) + +/* RCC_MP_APB1ENCLRR register fields */ +#define RCC_MP_APB1ENCLRR_TIM2EN BIT(0) +#define RCC_MP_APB1ENCLRR_TIM3EN BIT(1) +#define RCC_MP_APB1ENCLRR_TIM4EN BIT(2) +#define RCC_MP_APB1ENCLRR_TIM5EN BIT(3) +#define RCC_MP_APB1ENCLRR_TIM6EN BIT(4) +#define RCC_MP_APB1ENCLRR_TIM7EN BIT(5) +#define RCC_MP_APB1ENCLRR_LPTIM1EN BIT(9) +#define RCC_MP_APB1ENCLRR_SPI2EN BIT(11) +#define RCC_MP_APB1ENCLRR_SPI3EN BIT(12) +#define RCC_MP_APB1ENCLRR_USART3EN BIT(15) +#define RCC_MP_APB1ENCLRR_UART4EN BIT(16) +#define RCC_MP_APB1ENCLRR_UART5EN BIT(17) +#define RCC_MP_APB1ENCLRR_UART7EN BIT(18) +#define RCC_MP_APB1ENCLRR_UART8EN BIT(19) +#define RCC_MP_APB1ENCLRR_I2C1EN BIT(21) +#define RCC_MP_APB1ENCLRR_I2C2EN BIT(22) +#define RCC_MP_APB1ENCLRR_SPDIFEN BIT(26) + +/* RCC_MP_APB2ENSETR register fields */ +#define RCC_MP_APB2ENSETR_TIM1EN BIT(0) +#define RCC_MP_APB2ENSETR_TIM8EN BIT(1) +#define RCC_MP_APB2ENSETR_SPI1EN BIT(8) +#define RCC_MP_APB2ENSETR_USART6EN BIT(13) +#define RCC_MP_APB2ENSETR_SAI1EN BIT(16) +#define RCC_MP_APB2ENSETR_SAI2EN BIT(17) +#define RCC_MP_APB2ENSETR_DFSDMEN BIT(20) +#define RCC_MP_APB2ENSETR_ADFSDMEN BIT(21) +#define RCC_MP_APB2ENSETR_FDCANEN BIT(24) + +/* RCC_MP_APB2ENCLRR register fields */ +#define RCC_MP_APB2ENCLRR_TIM1EN BIT(0) +#define RCC_MP_APB2ENCLRR_TIM8EN BIT(1) +#define RCC_MP_APB2ENCLRR_SPI1EN BIT(8) +#define RCC_MP_APB2ENCLRR_USART6EN BIT(13) +#define RCC_MP_APB2ENCLRR_SAI1EN BIT(16) +#define RCC_MP_APB2ENCLRR_SAI2EN BIT(17) +#define RCC_MP_APB2ENCLRR_DFSDMEN BIT(20) +#define RCC_MP_APB2ENCLRR_ADFSDMEN BIT(21) +#define RCC_MP_APB2ENCLRR_FDCANEN BIT(24) + +/* RCC_MP_APB3ENSETR register fields */ +#define RCC_MP_APB3ENSETR_LPTIM2EN BIT(0) +#define RCC_MP_APB3ENSETR_LPTIM3EN BIT(1) +#define RCC_MP_APB3ENSETR_LPTIM4EN BIT(2) +#define RCC_MP_APB3ENSETR_LPTIM5EN BIT(3) +#define RCC_MP_APB3ENSETR_VREFEN BIT(13) +#define RCC_MP_APB3ENSETR_DTSEN BIT(16) +#define RCC_MP_APB3ENSETR_PMBCTRLEN BIT(17) +#define RCC_MP_APB3ENSETR_HDPEN BIT(20) + +/* RCC_MP_APB3ENCLRR register fields */ +#define RCC_MP_APB3ENCLRR_LPTIM2EN BIT(0) +#define RCC_MP_APB3ENCLRR_LPTIM3EN BIT(1) +#define RCC_MP_APB3ENCLRR_LPTIM4EN BIT(2) +#define RCC_MP_APB3ENCLRR_LPTIM5EN BIT(3) +#define RCC_MP_APB3ENCLRR_VREFEN BIT(13) +#define RCC_MP_APB3ENCLRR_DTSEN BIT(16) +#define RCC_MP_APB3ENCLRR_PMBCTRLEN BIT(17) +#define RCC_MP_APB3ENCLRR_HDPEN BIT(20) + +/* RCC_MP_S_APB3ENSETR register fields */ +#define RCC_MP_S_APB3ENSETR_SYSCFGEN BIT(0) + +/* RCC_MP_S_APB3ENCLRR register fields */ +#define RCC_MP_S_APB3ENCLRR_SYSCFGEN BIT(0) + +/* RCC_MP_NS_APB3ENSETR register fields */ +#define RCC_MP_NS_APB3ENSETR_SYSCFGEN BIT(0) + +/* RCC_MP_NS_APB3ENCLRR register fields */ +#define RCC_MP_NS_APB3ENCLRR_SYSCFGEN BIT(0) + +/* RCC_MP_APB4ENSETR register fields */ +#define RCC_MP_APB4ENSETR_DCMIPPEN BIT(1) +#define RCC_MP_APB4ENSETR_DDRPERFMEN BIT(8) +#define RCC_MP_APB4ENSETR_IWDG2APBEN BIT(15) +#define RCC_MP_APB4ENSETR_USBPHYEN BIT(16) +#define RCC_MP_APB4ENSETR_STGENROEN BIT(20) + +/* RCC_MP_APB4ENCLRR register fields */ +#define RCC_MP_APB4ENCLRR_DCMIPPEN BIT(1) +#define RCC_MP_APB4ENCLRR_DDRPERFMEN BIT(8) +#define RCC_MP_APB4ENCLRR_IWDG2APBEN BIT(15) +#define RCC_MP_APB4ENCLRR_USBPHYEN BIT(16) +#define RCC_MP_APB4ENCLRR_STGENROEN BIT(20) + +/* RCC_MP_S_APB4ENSETR register fields */ +#define RCC_MP_S_APB4ENSETR_LTDCEN BIT(0) + +/* RCC_MP_S_APB4ENCLRR register fields */ +#define RCC_MP_S_APB4ENCLRR_LTDCEN BIT(0) + +/* RCC_MP_NS_APB4ENSETR register fields */ +#define RCC_MP_NS_APB4ENSETR_LTDCEN BIT(0) + +/* RCC_MP_NS_APB4ENCLRR register fields */ +#define RCC_MP_NS_APB4ENCLRR_LTDCEN BIT(0) + +/* RCC_MP_APB5ENSETR register fields */ +#define RCC_MP_APB5ENSETR_RTCAPBEN BIT(8) +#define RCC_MP_APB5ENSETR_TZCEN BIT(11) +#define RCC_MP_APB5ENSETR_ETZPCEN BIT(13) +#define RCC_MP_APB5ENSETR_IWDG1APBEN BIT(15) +#define RCC_MP_APB5ENSETR_BSECEN BIT(16) +#define RCC_MP_APB5ENSETR_STGENCEN BIT(20) + +/* RCC_MP_APB5ENCLRR register fields */ +#define RCC_MP_APB5ENCLRR_RTCAPBEN BIT(8) +#define RCC_MP_APB5ENCLRR_TZCEN BIT(11) +#define RCC_MP_APB5ENCLRR_ETZPCEN BIT(13) +#define RCC_MP_APB5ENCLRR_IWDG1APBEN BIT(15) +#define RCC_MP_APB5ENCLRR_BSECEN BIT(16) +#define RCC_MP_APB5ENCLRR_STGENCEN BIT(20) + +/* RCC_MP_APB6ENSETR register fields */ +#define RCC_MP_APB6ENSETR_USART1EN BIT(0) +#define RCC_MP_APB6ENSETR_USART2EN BIT(1) +#define RCC_MP_APB6ENSETR_SPI4EN BIT(2) +#define RCC_MP_APB6ENSETR_SPI5EN BIT(3) +#define RCC_MP_APB6ENSETR_I2C3EN BIT(4) +#define RCC_MP_APB6ENSETR_I2C4EN BIT(5) +#define RCC_MP_APB6ENSETR_I2C5EN BIT(6) +#define RCC_MP_APB6ENSETR_TIM12EN BIT(7) +#define RCC_MP_APB6ENSETR_TIM13EN BIT(8) +#define RCC_MP_APB6ENSETR_TIM14EN BIT(9) +#define RCC_MP_APB6ENSETR_TIM15EN BIT(10) +#define RCC_MP_APB6ENSETR_TIM16EN BIT(11) +#define RCC_MP_APB6ENSETR_TIM17EN BIT(12) + +/* RCC_MP_APB6ENCLRR register fields */ +#define RCC_MP_APB6ENCLRR_USART1EN BIT(0) +#define RCC_MP_APB6ENCLRR_USART2EN BIT(1) +#define RCC_MP_APB6ENCLRR_SPI4EN BIT(2) +#define RCC_MP_APB6ENCLRR_SPI5EN BIT(3) +#define RCC_MP_APB6ENCLRR_I2C3EN BIT(4) +#define RCC_MP_APB6ENCLRR_I2C4EN BIT(5) +#define RCC_MP_APB6ENCLRR_I2C5EN BIT(6) +#define RCC_MP_APB6ENCLRR_TIM12EN BIT(7) +#define RCC_MP_APB6ENCLRR_TIM13EN BIT(8) +#define RCC_MP_APB6ENCLRR_TIM14EN BIT(9) +#define RCC_MP_APB6ENCLRR_TIM15EN BIT(10) +#define RCC_MP_APB6ENCLRR_TIM16EN BIT(11) +#define RCC_MP_APB6ENCLRR_TIM17EN BIT(12) + +/* RCC_MP_AHB2ENSETR register fields */ +#define RCC_MP_AHB2ENSETR_DMA1EN BIT(0) +#define RCC_MP_AHB2ENSETR_DMA2EN BIT(1) +#define RCC_MP_AHB2ENSETR_DMAMUX1EN BIT(2) +#define RCC_MP_AHB2ENSETR_DMA3EN BIT(3) +#define RCC_MP_AHB2ENSETR_DMAMUX2EN BIT(4) +#define RCC_MP_AHB2ENSETR_ADC1EN BIT(5) +#define RCC_MP_AHB2ENSETR_ADC2EN BIT(6) +#define RCC_MP_AHB2ENSETR_USBOEN BIT(8) + +/* RCC_MP_AHB2ENCLRR register fields */ +#define RCC_MP_AHB2ENCLRR_DMA1EN BIT(0) +#define RCC_MP_AHB2ENCLRR_DMA2EN BIT(1) +#define RCC_MP_AHB2ENCLRR_DMAMUX1EN BIT(2) +#define RCC_MP_AHB2ENCLRR_DMA3EN BIT(3) +#define RCC_MP_AHB2ENCLRR_DMAMUX2EN BIT(4) +#define RCC_MP_AHB2ENCLRR_ADC1EN BIT(5) +#define RCC_MP_AHB2ENCLRR_ADC2EN BIT(6) +#define RCC_MP_AHB2ENCLRR_USBOEN BIT(8) + +/* RCC_MP_AHB4ENSETR register fields */ +#define RCC_MP_AHB4ENSETR_TSCEN BIT(15) + +/* RCC_MP_AHB4ENCLRR register fields */ +#define RCC_MP_AHB4ENCLRR_TSCEN BIT(15) + +/* RCC_MP_S_AHB4ENSETR register fields */ +#define RCC_MP_S_AHB4ENSETR_GPIOAEN BIT(0) +#define RCC_MP_S_AHB4ENSETR_GPIOBEN BIT(1) +#define RCC_MP_S_AHB4ENSETR_GPIOCEN BIT(2) +#define RCC_MP_S_AHB4ENSETR_GPIODEN BIT(3) +#define RCC_MP_S_AHB4ENSETR_GPIOEEN BIT(4) +#define RCC_MP_S_AHB4ENSETR_GPIOFEN BIT(5) +#define RCC_MP_S_AHB4ENSETR_GPIOGEN BIT(6) +#define RCC_MP_S_AHB4ENSETR_GPIOHEN BIT(7) +#define RCC_MP_S_AHB4ENSETR_GPIOIEN BIT(8) + +/* RCC_MP_S_AHB4ENCLRR register fields */ +#define RCC_MP_S_AHB4ENCLRR_GPIOAEN BIT(0) +#define RCC_MP_S_AHB4ENCLRR_GPIOBEN BIT(1) +#define RCC_MP_S_AHB4ENCLRR_GPIOCEN BIT(2) +#define RCC_MP_S_AHB4ENCLRR_GPIODEN BIT(3) +#define RCC_MP_S_AHB4ENCLRR_GPIOEEN BIT(4) +#define RCC_MP_S_AHB4ENCLRR_GPIOFEN BIT(5) +#define RCC_MP_S_AHB4ENCLRR_GPIOGEN BIT(6) +#define RCC_MP_S_AHB4ENCLRR_GPIOHEN BIT(7) +#define RCC_MP_S_AHB4ENCLRR_GPIOIEN BIT(8) + +/* RCC_MP_NS_AHB4ENSETR register fields */ +#define RCC_MP_NS_AHB4ENSETR_GPIOAEN BIT(0) +#define RCC_MP_NS_AHB4ENSETR_GPIOBEN BIT(1) +#define RCC_MP_NS_AHB4ENSETR_GPIOCEN BIT(2) +#define RCC_MP_NS_AHB4ENSETR_GPIODEN BIT(3) +#define RCC_MP_NS_AHB4ENSETR_GPIOEEN BIT(4) +#define RCC_MP_NS_AHB4ENSETR_GPIOFEN BIT(5) +#define RCC_MP_NS_AHB4ENSETR_GPIOGEN BIT(6) +#define RCC_MP_NS_AHB4ENSETR_GPIOHEN BIT(7) +#define RCC_MP_NS_AHB4ENSETR_GPIOIEN BIT(8) + +/* RCC_MP_NS_AHB4ENCLRR register fields */ +#define RCC_MP_NS_AHB4ENCLRR_GPIOAEN BIT(0) +#define RCC_MP_NS_AHB4ENCLRR_GPIOBEN BIT(1) +#define RCC_MP_NS_AHB4ENCLRR_GPIOCEN BIT(2) +#define RCC_MP_NS_AHB4ENCLRR_GPIODEN BIT(3) +#define RCC_MP_NS_AHB4ENCLRR_GPIOEEN BIT(4) +#define RCC_MP_NS_AHB4ENCLRR_GPIOFEN BIT(5) +#define RCC_MP_NS_AHB4ENCLRR_GPIOGEN BIT(6) +#define RCC_MP_NS_AHB4ENCLRR_GPIOHEN BIT(7) +#define RCC_MP_NS_AHB4ENCLRR_GPIOIEN BIT(8) + +/* RCC_MP_AHB5ENSETR register fields */ +#define RCC_MP_AHB5ENSETR_PKAEN BIT(2) +#define RCC_MP_AHB5ENSETR_SAESEN BIT(3) +#define RCC_MP_AHB5ENSETR_CRYP1EN BIT(4) +#define RCC_MP_AHB5ENSETR_HASH1EN BIT(5) +#define RCC_MP_AHB5ENSETR_RNG1EN BIT(6) +#define RCC_MP_AHB5ENSETR_BKPSRAMEN BIT(8) +#define RCC_MP_AHB5ENSETR_AXIMCEN BIT(16) + +/* RCC_MP_AHB5ENCLRR register fields */ +#define RCC_MP_AHB5ENCLRR_PKAEN BIT(2) +#define RCC_MP_AHB5ENCLRR_SAESEN BIT(3) +#define RCC_MP_AHB5ENCLRR_CRYP1EN BIT(4) +#define RCC_MP_AHB5ENCLRR_HASH1EN BIT(5) +#define RCC_MP_AHB5ENCLRR_RNG1EN BIT(6) +#define RCC_MP_AHB5ENCLRR_BKPSRAMEN BIT(8) +#define RCC_MP_AHB5ENCLRR_AXIMCEN BIT(16) + +/* RCC_MP_AHB6ENSETR register fields */ +#define RCC_MP_AHB6ENSETR_MCEEN BIT(1) +#define RCC_MP_AHB6ENSETR_ETH1CKEN BIT(7) +#define RCC_MP_AHB6ENSETR_ETH1TXEN BIT(8) +#define RCC_MP_AHB6ENSETR_ETH1RXEN BIT(9) +#define RCC_MP_AHB6ENSETR_ETH1MACEN BIT(10) +#define RCC_MP_AHB6ENSETR_FMCEN BIT(12) +#define RCC_MP_AHB6ENSETR_QSPIEN BIT(14) +#define RCC_MP_AHB6ENSETR_SDMMC1EN BIT(16) +#define RCC_MP_AHB6ENSETR_SDMMC2EN BIT(17) +#define RCC_MP_AHB6ENSETR_CRC1EN BIT(20) +#define RCC_MP_AHB6ENSETR_USBHEN BIT(24) +#define RCC_MP_AHB6ENSETR_ETH2CKEN BIT(27) +#define RCC_MP_AHB6ENSETR_ETH2TXEN BIT(28) +#define RCC_MP_AHB6ENSETR_ETH2RXEN BIT(29) +#define RCC_MP_AHB6ENSETR_ETH2MACEN BIT(30) + +/* RCC_MP_AHB6ENCLRR register fields */ +#define RCC_MP_AHB6ENCLRR_MCEEN BIT(1) +#define RCC_MP_AHB6ENCLRR_ETH1CKEN BIT(7) +#define RCC_MP_AHB6ENCLRR_ETH1TXEN BIT(8) +#define RCC_MP_AHB6ENCLRR_ETH1RXEN BIT(9) +#define RCC_MP_AHB6ENCLRR_ETH1MACEN BIT(10) +#define RCC_MP_AHB6ENCLRR_FMCEN BIT(12) +#define RCC_MP_AHB6ENCLRR_QSPIEN BIT(14) +#define RCC_MP_AHB6ENCLRR_SDMMC1EN BIT(16) +#define RCC_MP_AHB6ENCLRR_SDMMC2EN BIT(17) +#define RCC_MP_AHB6ENCLRR_CRC1EN BIT(20) +#define RCC_MP_AHB6ENCLRR_USBHEN BIT(24) +#define RCC_MP_AHB6ENCLRR_ETH2CKEN BIT(27) +#define RCC_MP_AHB6ENCLRR_ETH2TXEN BIT(28) +#define RCC_MP_AHB6ENCLRR_ETH2RXEN BIT(29) +#define RCC_MP_AHB6ENCLRR_ETH2MACEN BIT(30) + +/* RCC_MP_S_AHB6ENSETR register fields */ +#define RCC_MP_S_AHB6ENSETR_MDMAEN BIT(0) + +/* RCC_MP_S_AHB6ENCLRR register fields */ +#define RCC_MP_S_AHB6ENCLRR_MDMAEN BIT(0) + +/* RCC_MP_NS_AHB6ENSETR register fields */ +#define RCC_MP_NS_AHB6ENSETR_MDMAEN BIT(0) + +/* RCC_MP_NS_AHB6ENCLRR register fields */ +#define RCC_MP_NS_AHB6ENCLRR_MDMAEN BIT(0) + +/* RCC_MP_APB1LPENSETR register fields */ +#define RCC_MP_APB1LPENSETR_TIM2LPEN BIT(0) +#define RCC_MP_APB1LPENSETR_TIM3LPEN BIT(1) +#define RCC_MP_APB1LPENSETR_TIM4LPEN BIT(2) +#define RCC_MP_APB1LPENSETR_TIM5LPEN BIT(3) +#define RCC_MP_APB1LPENSETR_TIM6LPEN BIT(4) +#define RCC_MP_APB1LPENSETR_TIM7LPEN BIT(5) +#define RCC_MP_APB1LPENSETR_LPTIM1LPEN BIT(9) +#define RCC_MP_APB1LPENSETR_SPI2LPEN BIT(11) +#define RCC_MP_APB1LPENSETR_SPI3LPEN BIT(12) +#define RCC_MP_APB1LPENSETR_USART3LPEN BIT(15) +#define RCC_MP_APB1LPENSETR_UART4LPEN BIT(16) +#define RCC_MP_APB1LPENSETR_UART5LPEN BIT(17) +#define RCC_MP_APB1LPENSETR_UART7LPEN BIT(18) +#define RCC_MP_APB1LPENSETR_UART8LPEN BIT(19) +#define RCC_MP_APB1LPENSETR_I2C1LPEN BIT(21) +#define RCC_MP_APB1LPENSETR_I2C2LPEN BIT(22) +#define RCC_MP_APB1LPENSETR_SPDIFLPEN BIT(26) + +/* RCC_MP_APB1LPENCLRR register fields */ +#define RCC_MP_APB1LPENCLRR_TIM2LPEN BIT(0) +#define RCC_MP_APB1LPENCLRR_TIM3LPEN BIT(1) +#define RCC_MP_APB1LPENCLRR_TIM4LPEN BIT(2) +#define RCC_MP_APB1LPENCLRR_TIM5LPEN BIT(3) +#define RCC_MP_APB1LPENCLRR_TIM6LPEN BIT(4) +#define RCC_MP_APB1LPENCLRR_TIM7LPEN BIT(5) +#define RCC_MP_APB1LPENCLRR_LPTIM1LPEN BIT(9) +#define RCC_MP_APB1LPENCLRR_SPI2LPEN BIT(11) +#define RCC_MP_APB1LPENCLRR_SPI3LPEN BIT(12) +#define RCC_MP_APB1LPENCLRR_USART3LPEN BIT(15) +#define RCC_MP_APB1LPENCLRR_UART4LPEN BIT(16) +#define RCC_MP_APB1LPENCLRR_UART5LPEN BIT(17) +#define RCC_MP_APB1LPENCLRR_UART7LPEN BIT(18) +#define RCC_MP_APB1LPENCLRR_UART8LPEN BIT(19) +#define RCC_MP_APB1LPENCLRR_I2C1LPEN BIT(21) +#define RCC_MP_APB1LPENCLRR_I2C2LPEN BIT(22) +#define RCC_MP_APB1LPENCLRR_SPDIFLPEN BIT(26) + +/* RCC_MP_APB2LPENSETR register fields */ +#define RCC_MP_APB2LPENSETR_TIM1LPEN BIT(0) +#define RCC_MP_APB2LPENSETR_TIM8LPEN BIT(1) +#define RCC_MP_APB2LPENSETR_SPI1LPEN BIT(8) +#define RCC_MP_APB2LPENSETR_USART6LPEN BIT(13) +#define RCC_MP_APB2LPENSETR_SAI1LPEN BIT(16) +#define RCC_MP_APB2LPENSETR_SAI2LPEN BIT(17) +#define RCC_MP_APB2LPENSETR_DFSDMLPEN BIT(20) +#define RCC_MP_APB2LPENSETR_ADFSDMLPEN BIT(21) +#define RCC_MP_APB2LPENSETR_FDCANLPEN BIT(24) + +/* RCC_MP_APB2LPENCLRR register fields */ +#define RCC_MP_APB2LPENCLRR_TIM1LPEN BIT(0) +#define RCC_MP_APB2LPENCLRR_TIM8LPEN BIT(1) +#define RCC_MP_APB2LPENCLRR_SPI1LPEN BIT(8) +#define RCC_MP_APB2LPENCLRR_USART6LPEN BIT(13) +#define RCC_MP_APB2LPENCLRR_SAI1LPEN BIT(16) +#define RCC_MP_APB2LPENCLRR_SAI2LPEN BIT(17) +#define RCC_MP_APB2LPENCLRR_DFSDMLPEN BIT(20) +#define RCC_MP_APB2LPENCLRR_ADFSDMLPEN BIT(21) +#define RCC_MP_APB2LPENCLRR_FDCANLPEN BIT(24) + +/* RCC_MP_APB3LPENSETR register fields */ +#define RCC_MP_APB3LPENSETR_LPTIM2LPEN BIT(0) +#define RCC_MP_APB3LPENSETR_LPTIM3LPEN BIT(1) +#define RCC_MP_APB3LPENSETR_LPTIM4LPEN BIT(2) +#define RCC_MP_APB3LPENSETR_LPTIM5LPEN BIT(3) +#define RCC_MP_APB3LPENSETR_VREFLPEN BIT(13) +#define RCC_MP_APB3LPENSETR_DTSLPEN BIT(16) +#define RCC_MP_APB3LPENSETR_PMBCTRLLPEN BIT(17) + +/* RCC_MP_APB3LPENCLRR register fields */ +#define RCC_MP_APB3LPENCLRR_LPTIM2LPEN BIT(0) +#define RCC_MP_APB3LPENCLRR_LPTIM3LPEN BIT(1) +#define RCC_MP_APB3LPENCLRR_LPTIM4LPEN BIT(2) +#define RCC_MP_APB3LPENCLRR_LPTIM5LPEN BIT(3) +#define RCC_MP_APB3LPENCLRR_VREFLPEN BIT(13) +#define RCC_MP_APB3LPENCLRR_DTSLPEN BIT(16) +#define RCC_MP_APB3LPENCLRR_PMBCTRLLPEN BIT(17) + +/* RCC_MP_S_APB3LPENSETR register fields */ +#define RCC_MP_S_APB3LPENSETR_SYSCFGLPEN BIT(0) + +/* RCC_MP_S_APB3LPENCLRR register fields */ +#define RCC_MP_S_APB3LPENCLRR_SYSCFGLPEN BIT(0) + +/* RCC_MP_NS_APB3LPENSETR register fields */ +#define RCC_MP_NS_APB3LPENSETR_SYSCFGLPEN BIT(0) + +/* RCC_MP_NS_APB3LPENCLRR register fields */ +#define RCC_MP_NS_APB3LPENCLRR_SYSCFGLPEN BIT(0) + +/* RCC_MP_APB4LPENSETR register fields */ +#define RCC_MP_APB4LPENSETR_DCMIPPLPEN BIT(1) +#define RCC_MP_APB4LPENSETR_DDRPERFMLPEN BIT(8) +#define RCC_MP_APB4LPENSETR_IWDG2APBLPEN BIT(15) +#define RCC_MP_APB4LPENSETR_USBPHYLPEN BIT(16) +#define RCC_MP_APB4LPENSETR_STGENROLPEN BIT(20) +#define RCC_MP_APB4LPENSETR_STGENROSTPEN BIT(21) + +/* RCC_MP_APB4LPENCLRR register fields */ +#define RCC_MP_APB4LPENCLRR_DCMIPPLPEN BIT(1) +#define RCC_MP_APB4LPENCLRR_DDRPERFMLPEN BIT(8) +#define RCC_MP_APB4LPENCLRR_IWDG2APBLPEN BIT(15) +#define RCC_MP_APB4LPENCLRR_USBPHYLPEN BIT(16) +#define RCC_MP_APB4LPENCLRR_STGENROLPEN BIT(20) +#define RCC_MP_APB4LPENCLRR_STGENROSTPEN BIT(21) + +/* RCC_MP_S_APB4LPENSETR register fields */ +#define RCC_MP_S_APB4LPENSETR_LTDCLPEN BIT(0) + +/* RCC_MP_S_APB4LPENCLRR register fields */ +#define RCC_MP_S_APB4LPENCLRR_LTDCLPEN BIT(0) + +/* RCC_MP_NS_APB4LPENSETR register fields */ +#define RCC_MP_NS_APB4LPENSETR_LTDCLPEN BIT(0) + +/* RCC_MP_NS_APB4LPENCLRR register fields */ +#define RCC_MP_NS_APB4LPENCLRR_LTDCLPEN BIT(0) + +/* RCC_MP_APB5LPENSETR register fields */ +#define RCC_MP_APB5LPENSETR_RTCAPBLPEN BIT(8) +#define RCC_MP_APB5LPENSETR_TZCLPEN BIT(11) +#define RCC_MP_APB5LPENSETR_ETZPCLPEN BIT(13) +#define RCC_MP_APB5LPENSETR_IWDG1APBLPEN BIT(15) +#define RCC_MP_APB5LPENSETR_BSECLPEN BIT(16) +#define RCC_MP_APB5LPENSETR_STGENCLPEN BIT(20) +#define RCC_MP_APB5LPENSETR_STGENCSTPEN BIT(21) + +/* RCC_MP_APB5LPENCLRR register fields */ +#define RCC_MP_APB5LPENCLRR_RTCAPBLPEN BIT(8) +#define RCC_MP_APB5LPENCLRR_TZCLPEN BIT(11) +#define RCC_MP_APB5LPENCLRR_ETZPCLPEN BIT(13) +#define RCC_MP_APB5LPENCLRR_IWDG1APBLPEN BIT(15) +#define RCC_MP_APB5LPENCLRR_BSECLPEN BIT(16) +#define RCC_MP_APB5LPENCLRR_STGENCLPEN BIT(20) +#define RCC_MP_APB5LPENCLRR_STGENCSTPEN BIT(21) + +/* RCC_MP_APB6LPENSETR register fields */ +#define RCC_MP_APB6LPENSETR_USART1LPEN BIT(0) +#define RCC_MP_APB6LPENSETR_USART2LPEN BIT(1) +#define RCC_MP_APB6LPENSETR_SPI4LPEN BIT(2) +#define RCC_MP_APB6LPENSETR_SPI5LPEN BIT(3) +#define RCC_MP_APB6LPENSETR_I2C3LPEN BIT(4) +#define RCC_MP_APB6LPENSETR_I2C4LPEN BIT(5) +#define RCC_MP_APB6LPENSETR_I2C5LPEN BIT(6) +#define RCC_MP_APB6LPENSETR_TIM12LPEN BIT(7) +#define RCC_MP_APB6LPENSETR_TIM13LPEN BIT(8) +#define RCC_MP_APB6LPENSETR_TIM14LPEN BIT(9) +#define RCC_MP_APB6LPENSETR_TIM15LPEN BIT(10) +#define RCC_MP_APB6LPENSETR_TIM16LPEN BIT(11) +#define RCC_MP_APB6LPENSETR_TIM17LPEN BIT(12) + +/* RCC_MP_APB6LPENCLRR register fields */ +#define RCC_MP_APB6LPENCLRR_USART1LPEN BIT(0) +#define RCC_MP_APB6LPENCLRR_USART2LPEN BIT(1) +#define RCC_MP_APB6LPENCLRR_SPI4LPEN BIT(2) +#define RCC_MP_APB6LPENCLRR_SPI5LPEN BIT(3) +#define RCC_MP_APB6LPENCLRR_I2C3LPEN BIT(4) +#define RCC_MP_APB6LPENCLRR_I2C4LPEN BIT(5) +#define RCC_MP_APB6LPENCLRR_I2C5LPEN BIT(6) +#define RCC_MP_APB6LPENCLRR_TIM12LPEN BIT(7) +#define RCC_MP_APB6LPENCLRR_TIM13LPEN BIT(8) +#define RCC_MP_APB6LPENCLRR_TIM14LPEN BIT(9) +#define RCC_MP_APB6LPENCLRR_TIM15LPEN BIT(10) +#define RCC_MP_APB6LPENCLRR_TIM16LPEN BIT(11) +#define RCC_MP_APB6LPENCLRR_TIM17LPEN BIT(12) + +/* RCC_MP_AHB2LPENSETR register fields */ +#define RCC_MP_AHB2LPENSETR_DMA1LPEN BIT(0) +#define RCC_MP_AHB2LPENSETR_DMA2LPEN BIT(1) +#define RCC_MP_AHB2LPENSETR_DMAMUX1LPEN BIT(2) +#define RCC_MP_AHB2LPENSETR_DMA3LPEN BIT(3) +#define RCC_MP_AHB2LPENSETR_DMAMUX2LPEN BIT(4) +#define RCC_MP_AHB2LPENSETR_ADC1LPEN BIT(5) +#define RCC_MP_AHB2LPENSETR_ADC2LPEN BIT(6) +#define RCC_MP_AHB2LPENSETR_USBOLPEN BIT(8) + +/* RCC_MP_AHB2LPENCLRR register fields */ +#define RCC_MP_AHB2LPENCLRR_DMA1LPEN BIT(0) +#define RCC_MP_AHB2LPENCLRR_DMA2LPEN BIT(1) +#define RCC_MP_AHB2LPENCLRR_DMAMUX1LPEN BIT(2) +#define RCC_MP_AHB2LPENCLRR_DMA3LPEN BIT(3) +#define RCC_MP_AHB2LPENCLRR_DMAMUX2LPEN BIT(4) +#define RCC_MP_AHB2LPENCLRR_ADC1LPEN BIT(5) +#define RCC_MP_AHB2LPENCLRR_ADC2LPEN BIT(6) +#define RCC_MP_AHB2LPENCLRR_USBOLPEN BIT(8) + +/* RCC_MP_AHB4LPENSETR register fields */ +#define RCC_MP_AHB4LPENSETR_TSCLPEN BIT(15) + +/* RCC_MP_AHB4LPENCLRR register fields */ +#define RCC_MP_AHB4LPENCLRR_TSCLPEN BIT(15) + +/* RCC_MP_S_AHB4LPENSETR register fields */ +#define RCC_MP_S_AHB4LPENSETR_GPIOALPEN BIT(0) +#define RCC_MP_S_AHB4LPENSETR_GPIOBLPEN BIT(1) +#define RCC_MP_S_AHB4LPENSETR_GPIOCLPEN BIT(2) +#define RCC_MP_S_AHB4LPENSETR_GPIODLPEN BIT(3) +#define RCC_MP_S_AHB4LPENSETR_GPIOELPEN BIT(4) +#define RCC_MP_S_AHB4LPENSETR_GPIOFLPEN BIT(5) +#define RCC_MP_S_AHB4LPENSETR_GPIOGLPEN BIT(6) +#define RCC_MP_S_AHB4LPENSETR_GPIOHLPEN BIT(7) +#define RCC_MP_S_AHB4LPENSETR_GPIOILPEN BIT(8) + +/* RCC_MP_S_AHB4LPENCLRR register fields */ +#define RCC_MP_S_AHB4LPENCLRR_GPIOALPEN BIT(0) +#define RCC_MP_S_AHB4LPENCLRR_GPIOBLPEN BIT(1) +#define RCC_MP_S_AHB4LPENCLRR_GPIOCLPEN BIT(2) +#define RCC_MP_S_AHB4LPENCLRR_GPIODLPEN BIT(3) +#define RCC_MP_S_AHB4LPENCLRR_GPIOELPEN BIT(4) +#define RCC_MP_S_AHB4LPENCLRR_GPIOFLPEN BIT(5) +#define RCC_MP_S_AHB4LPENCLRR_GPIOGLPEN BIT(6) +#define RCC_MP_S_AHB4LPENCLRR_GPIOHLPEN BIT(7) +#define RCC_MP_S_AHB4LPENCLRR_GPIOILPEN BIT(8) + +/* RCC_MP_NS_AHB4LPENSETR register fields */ +#define RCC_MP_NS_AHB4LPENSETR_GPIOALPEN BIT(0) +#define RCC_MP_NS_AHB4LPENSETR_GPIOBLPEN BIT(1) +#define RCC_MP_NS_AHB4LPENSETR_GPIOCLPEN BIT(2) +#define RCC_MP_NS_AHB4LPENSETR_GPIODLPEN BIT(3) +#define RCC_MP_NS_AHB4LPENSETR_GPIOELPEN BIT(4) +#define RCC_MP_NS_AHB4LPENSETR_GPIOFLPEN BIT(5) +#define RCC_MP_NS_AHB4LPENSETR_GPIOGLPEN BIT(6) +#define RCC_MP_NS_AHB4LPENSETR_GPIOHLPEN BIT(7) +#define RCC_MP_NS_AHB4LPENSETR_GPIOILPEN BIT(8) + +/* RCC_MP_NS_AHB4LPENCLRR register fields */ +#define RCC_MP_NS_AHB4LPENCLRR_GPIOALPEN BIT(0) +#define RCC_MP_NS_AHB4LPENCLRR_GPIOBLPEN BIT(1) +#define RCC_MP_NS_AHB4LPENCLRR_GPIOCLPEN BIT(2) +#define RCC_MP_NS_AHB4LPENCLRR_GPIODLPEN BIT(3) +#define RCC_MP_NS_AHB4LPENCLRR_GPIOELPEN BIT(4) +#define RCC_MP_NS_AHB4LPENCLRR_GPIOFLPEN BIT(5) +#define RCC_MP_NS_AHB4LPENCLRR_GPIOGLPEN BIT(6) +#define RCC_MP_NS_AHB4LPENCLRR_GPIOHLPEN BIT(7) +#define RCC_MP_NS_AHB4LPENCLRR_GPIOILPEN BIT(8) + +/* RCC_MP_AHB5LPENSETR register fields */ +#define RCC_MP_AHB5LPENSETR_PKALPEN BIT(2) +#define RCC_MP_AHB5LPENSETR_SAESLPEN BIT(3) +#define RCC_MP_AHB5LPENSETR_CRYP1LPEN BIT(4) +#define RCC_MP_AHB5LPENSETR_HASH1LPEN BIT(5) +#define RCC_MP_AHB5LPENSETR_RNG1LPEN BIT(6) +#define RCC_MP_AHB5LPENSETR_BKPSRAMLPEN BIT(8) + +/* RCC_MP_AHB5LPENCLRR register fields */ +#define RCC_MP_AHB5LPENCLRR_PKALPEN BIT(2) +#define RCC_MP_AHB5LPENCLRR_SAESLPEN BIT(3) +#define RCC_MP_AHB5LPENCLRR_CRYP1LPEN BIT(4) +#define RCC_MP_AHB5LPENCLRR_HASH1LPEN BIT(5) +#define RCC_MP_AHB5LPENCLRR_RNG1LPEN BIT(6) +#define RCC_MP_AHB5LPENCLRR_BKPSRAMLPEN BIT(8) + +/* RCC_MP_AHB6LPENSETR register fields */ +#define RCC_MP_AHB6LPENSETR_MCELPEN BIT(1) +#define RCC_MP_AHB6LPENSETR_ETH1CKLPEN BIT(7) +#define RCC_MP_AHB6LPENSETR_ETH1TXLPEN BIT(8) +#define RCC_MP_AHB6LPENSETR_ETH1RXLPEN BIT(9) +#define RCC_MP_AHB6LPENSETR_ETH1MACLPEN BIT(10) +#define RCC_MP_AHB6LPENSETR_ETH1STPEN BIT(11) +#define RCC_MP_AHB6LPENSETR_FMCLPEN BIT(12) +#define RCC_MP_AHB6LPENSETR_QSPILPEN BIT(14) +#define RCC_MP_AHB6LPENSETR_SDMMC1LPEN BIT(16) +#define RCC_MP_AHB6LPENSETR_SDMMC2LPEN BIT(17) +#define RCC_MP_AHB6LPENSETR_CRC1LPEN BIT(20) +#define RCC_MP_AHB6LPENSETR_USBHLPEN BIT(24) +#define RCC_MP_AHB6LPENSETR_ETH2CKLPEN BIT(27) +#define RCC_MP_AHB6LPENSETR_ETH2TXLPEN BIT(28) +#define RCC_MP_AHB6LPENSETR_ETH2RXLPEN BIT(29) +#define RCC_MP_AHB6LPENSETR_ETH2MACLPEN BIT(30) +#define RCC_MP_AHB6LPENSETR_ETH2STPEN BIT(31) + +/* RCC_MP_AHB6LPENCLRR register fields */ +#define RCC_MP_AHB6LPENCLRR_MCELPEN BIT(1) +#define RCC_MP_AHB6LPENCLRR_ETH1CKLPEN BIT(7) +#define RCC_MP_AHB6LPENCLRR_ETH1TXLPEN BIT(8) +#define RCC_MP_AHB6LPENCLRR_ETH1RXLPEN BIT(9) +#define RCC_MP_AHB6LPENCLRR_ETH1MACLPEN BIT(10) +#define RCC_MP_AHB6LPENCLRR_ETH1STPEN BIT(11) +#define RCC_MP_AHB6LPENCLRR_FMCLPEN BIT(12) +#define RCC_MP_AHB6LPENCLRR_QSPILPEN BIT(14) +#define RCC_MP_AHB6LPENCLRR_SDMMC1LPEN BIT(16) +#define RCC_MP_AHB6LPENCLRR_SDMMC2LPEN BIT(17) +#define RCC_MP_AHB6LPENCLRR_CRC1LPEN BIT(20) +#define RCC_MP_AHB6LPENCLRR_USBHLPEN BIT(24) +#define RCC_MP_AHB6LPENCLRR_ETH2CKLPEN BIT(27) +#define RCC_MP_AHB6LPENCLRR_ETH2TXLPEN BIT(28) +#define RCC_MP_AHB6LPENCLRR_ETH2RXLPEN BIT(29) +#define RCC_MP_AHB6LPENCLRR_ETH2MACLPEN BIT(30) +#define RCC_MP_AHB6LPENCLRR_ETH2STPEN BIT(31) + +/* RCC_MP_S_AHB6LPENSETR register fields */ +#define RCC_MP_S_AHB6LPENSETR_MDMALPEN BIT(0) + +/* RCC_MP_S_AHB6LPENCLRR register fields */ +#define RCC_MP_S_AHB6LPENCLRR_MDMALPEN BIT(0) + +/* RCC_MP_NS_AHB6LPENSETR register fields */ +#define RCC_MP_NS_AHB6LPENSETR_MDMALPEN BIT(0) + +/* RCC_MP_NS_AHB6LPENCLRR register fields */ +#define RCC_MP_NS_AHB6LPENCLRR_MDMALPEN BIT(0) + +/* RCC_MP_S_AXIMLPENSETR register fields */ +#define RCC_MP_S_AXIMLPENSETR_SYSRAMLPEN BIT(0) + +/* RCC_MP_S_AXIMLPENCLRR register fields */ +#define RCC_MP_S_AXIMLPENCLRR_SYSRAMLPEN BIT(0) + +/* RCC_MP_NS_AXIMLPENSETR register fields */ +#define RCC_MP_NS_AXIMLPENSETR_SYSRAMLPEN BIT(0) + +/* RCC_MP_NS_AXIMLPENCLRR register fields */ +#define RCC_MP_NS_AXIMLPENCLRR_SYSRAMLPEN BIT(0) + +/* RCC_MP_MLAHBLPENSETR register fields */ +#define RCC_MP_MLAHBLPENSETR_SRAM1LPEN BIT(0) +#define RCC_MP_MLAHBLPENSETR_SRAM2LPEN BIT(1) +#define RCC_MP_MLAHBLPENSETR_SRAM3LPEN BIT(2) + +/* RCC_MP_MLAHBLPENCLRR register fields */ +#define RCC_MP_MLAHBLPENCLRR_SRAM1LPEN BIT(0) +#define RCC_MP_MLAHBLPENCLRR_SRAM2LPEN BIT(1) +#define RCC_MP_MLAHBLPENCLRR_SRAM3LPEN BIT(2) + +/* RCC_APB3SECSR register fields */ +#define RCC_APB3SECSR_LPTIM2SECF BIT(0) +#define RCC_APB3SECSR_LPTIM3SECF BIT(1) +#define RCC_APB3SECSR_VREFSECF BIT(13) + +/* RCC_APB4SECSR register fields */ +#define RCC_APB4SECSR_DCMIPPSECF BIT(1) +#define RCC_APB4SECSR_USBPHYSECF BIT(16) + +/* RCC_APB5SECSR register fields */ +#define RCC_APB5SECSR_RTCSECF BIT(8) +#define RCC_APB5SECSR_TZCSECF BIT(11) +#define RCC_APB5SECSR_ETZPCSECF BIT(13) +#define RCC_APB5SECSR_IWDG1SECF BIT(15) +#define RCC_APB5SECSR_BSECSECF BIT(16) +#define RCC_APB5SECSR_STGENCSECF_MASK GENMASK_32(21, 20) +#define RCC_APB5SECSR_STGENCSECF_SHIFT 20 + +/* RCC_APB6SECSR register fields */ +#define RCC_APB6SECSR_USART1SECF BIT(0) +#define RCC_APB6SECSR_USART2SECF BIT(1) +#define RCC_APB6SECSR_SPI4SECF BIT(2) +#define RCC_APB6SECSR_SPI5SECF BIT(3) +#define RCC_APB6SECSR_I2C3SECF BIT(4) +#define RCC_APB6SECSR_I2C4SECF BIT(5) +#define RCC_APB6SECSR_I2C5SECF BIT(6) +#define RCC_APB6SECSR_TIM12SECF BIT(7) +#define RCC_APB6SECSR_TIM13SECF BIT(8) +#define RCC_APB6SECSR_TIM14SECF BIT(9) +#define RCC_APB6SECSR_TIM15SECF BIT(10) +#define RCC_APB6SECSR_TIM16SECF BIT(11) +#define RCC_APB6SECSR_TIM17SECF BIT(12) + +/* RCC_AHB2SECSR register fields */ +#define RCC_AHB2SECSR_DMA3SECF BIT(3) +#define RCC_AHB2SECSR_DMAMUX2SECF BIT(4) +#define RCC_AHB2SECSR_ADC1SECF BIT(5) +#define RCC_AHB2SECSR_ADC2SECF BIT(6) +#define RCC_AHB2SECSR_USBOSECF BIT(8) + +/* RCC_AHB4SECSR register fields */ +#define RCC_AHB4SECSR_TSCSECF BIT(15) + +/* RCC_AHB5SECSR register fields */ +#define RCC_AHB5SECSR_PKASECF BIT(2) +#define RCC_AHB5SECSR_SAESSECF BIT(3) +#define RCC_AHB5SECSR_CRYP1SECF BIT(4) +#define RCC_AHB5SECSR_HASH1SECF BIT(5) +#define RCC_AHB5SECSR_RNG1SECF BIT(6) +#define RCC_AHB5SECSR_BKPSRAMSECF BIT(8) + +/* RCC_AHB6SECSR register fields */ +#define RCC_AHB6SECSR_MCESECF BIT(1) +#define RCC_AHB6SECSR_ETH1SECF_MASK GENMASK_32(11, 7) +#define RCC_AHB6SECSR_ETH1SECF_SHIFT 7 +#define RCC_AHB6SECSR_FMCSECF BIT(12) +#define RCC_AHB6SECSR_QSPISECF BIT(14) +#define RCC_AHB6SECSR_SDMMC1SECF BIT(16) +#define RCC_AHB6SECSR_SDMMC2SECF BIT(17) +#define RCC_AHB6SECSR_ETH2SECF_MASK GENMASK_32(31, 27) +#define RCC_AHB6SECSR_ETH2SECF_SHIFT 27 + +/* RCC_VERR register fields */ +#define RCC_VERR_MINREV_MASK GENMASK_32(3, 0) +#define RCC_VERR_MINREV_SHIFT 0 +#define RCC_VERR_MAJREV_MASK GENMASK_32(7, 4) +#define RCC_VERR_MAJREV_SHIFT 4 + +/* RCC_IDR register fields */ +#define RCC_IDR_ID_MASK GENMASK_32(31, 0) +#define RCC_IDR_ID_SHIFT 0 + +/* RCC_SIDR register fields */ +#define RCC_SIDR_SID_MASK GENMASK_32(31, 0) +#define RCC_SIDR_SID_SHIFT 0 + +/* Used for all RCC_PLLCR registers */ +#define RCC_PLLNCR_PLLON BIT(0) +#define RCC_PLLNCR_PLLRDY BIT(1) +#define RCC_PLLNCR_SSCG_CTRL BIT(2) +#define RCC_PLLNCR_DIVPEN BIT(4) +#define RCC_PLLNCR_DIVQEN BIT(5) +#define RCC_PLLNCR_DIVREN BIT(6) +#define RCC_PLLNCR_DIVEN_SHIFT 4 + +/* Used for all RCC_PLLCFGR1 registers */ +#define RCC_PLLNCFGR1_DIVM_SHIFT 16 +#define RCC_PLLNCFGR1_DIVM_MASK GENMASK_32(21, 16) +#define RCC_PLLNCFGR1_DIVN_SHIFT 0 +#define RCC_PLLNCFGR1_DIVN_MASK GENMASK_32(8, 0) + +/* Only for PLL3 and PLL4 */ +#define RCC_PLLNCFGR1_IFRGE_SHIFT 24 +#define RCC_PLLNCFGR1_IFRGE_MASK GENMASK_32(25, 24) + +/* Used for all RCC_PLLCFGR2 registers */ +#define RCC_PLLNCFGR2_DIVX_MASK GENMASK_32(6, 0) +#define RCC_PLLNCFGR2_DIVP_SHIFT 0 +#define RCC_PLLNCFGR2_DIVP_MASK GENMASK_32(6, 0) +#define RCC_PLLNCFGR2_DIVQ_SHIFT 8 +#define RCC_PLLNCFGR2_DIVQ_MASK GENMASK_32(14, 8) +#define RCC_PLLNCFGR2_DIVR_SHIFT 16 +#define RCC_PLLNCFGR2_DIVR_MASK GENMASK_32(22, 16) + +/* Used for all RCC_PLLFRACR registers */ +#define RCC_PLLNFRACR_FRACV_SHIFT 3 +#define RCC_PLLNFRACR_FRACV_MASK GENMASK_32(15, 3) +#define RCC_PLLNFRACR_FRACLE BIT(16) + +/* Used for all RCC_PLLCSGR registers */ +#define RCC_PLLNCSGR_INC_STEP_SHIFT 16 +#define RCC_PLLNCSGR_INC_STEP_MASK GENMASK_32(30, 16) +#define RCC_PLLNCSGR_MOD_PER_SHIFT 0 +#define RCC_PLLNCSGR_MOD_PER_MASK GENMASK_32(12, 0) +#define RCC_PLLNCSGR_SSCG_MODE_SHIFT 15 +#define RCC_PLLNCSGR_SSCG_MODE_MASK BIT(15) + +/* Used for most of RCC_SELR registers */ +#define RCC_SELR_SRC_MASK GENMASK_32(2, 0) +#define RCC_SELR_REFCLK_SRC_MASK GENMASK_32(1, 0) +#define RCC_SELR_SRCRDY BIT(31) + +/* Values of RCC_MPCKSELR register */ +#define RCC_MPCKSELR_HSI 0x00000000 +#define RCC_MPCKSELR_HSE 0x00000001 +#define RCC_MPCKSELR_PLL 0x00000002 +#define RCC_MPCKSELR_PLL_MPUDIV 0x00000003 + +/* Values of RCC_ASSCKSELR register */ +#define RCC_ASSCKSELR_HSI 0x00000000 +#define RCC_ASSCKSELR_HSE 0x00000001 +#define RCC_ASSCKSELR_PLL 0x00000002 + +/* Values of RCC_MSSCKSELR register */ +#define RCC_MSSCKSELR_HSI 0x00000000 +#define RCC_MSSCKSELR_HSE 0x00000001 +#define RCC_MSSCKSELR_CSI 0x00000002 +#define RCC_MSSCKSELR_PLL 0x00000003 + +/* Values of RCC_CPERCKSELR register */ +#define RCC_CPERCKSELR_HSI 0x00000000 +#define RCC_CPERCKSELR_CSI 0x00000001 +#define RCC_CPERCKSELR_HSE 0x00000002 + +/* Used for most of DIVR register: max div for RTC */ +#define RCC_DIVR_DIV_MASK GENMASK_32(5, 0) +#define RCC_DIVR_DIVRDY BIT(31) + +/* Masks for specific DIVR registers */ +#define RCC_APBXDIV_MASK GENMASK_32(2, 0) +#define RCC_MPUDIV_MASK GENMASK_32(2, 0) +#define RCC_AXIDIV_MASK GENMASK_32(2, 0) +#define RCC_MLAHBDIV_MASK GENMASK_32(3, 0) + +/* Used for TIMER Prescaler */ +#define RCC_TIMGXPRER_TIMGXPRE BIT(0) + +/* Offset between RCC_MP_xxxENSETR and RCC_MP_xxxENCLRR registers */ +#define RCC_MP_ENCLRR_OFFSET U(4) + +/* Offset between RCC_xxxRSTSETR and RCC_xxxRSTCLRR registers */ +#define RCC_RSTCLRR_OFFSET U(4) + +/* RCC_OCENSETR register fields */ +#define RCC_OCENR_HSION BIT(0) +#define RCC_OCENR_HSIKERON BIT(1) +#define RCC_OCENR_CSION BIT(4) +#define RCC_OCENR_CSIKERON BIT(5) +#define RCC_OCENR_DIGBYP BIT(7) +#define RCC_OCENR_HSEON BIT(8) +#define RCC_OCENR_HSEKERON BIT(9) +#define RCC_OCENR_HSEBYP BIT(10) +#define RCC_OCENR_HSECSSON BIT(11) + +#define RCC_OCENR_DIGBYP_BIT 7 +#define RCC_OCENR_HSEBYP_BIT 10 +#define RCC_OCENR_HSECSSON_BIT 11 + +/* Used for RCC_MCO related operations */ +#define RCC_MCOCFG_MCOON BIT(12) +#define RCC_MCOCFG_MCODIV_MASK GENMASK_32(7, 4) +#define RCC_MCOCFG_MCODIV_SHIFT 4 +#define RCC_MCOCFG_MCOSRC_MASK GENMASK_32(2, 0) + +#define RCC_UART4CKSELR_HSI 0x00000002 + +#define RCC_CPERCKSELR_PERSRC_MASK GENMASK_32(1, 0) +#define RCC_CPERCKSELR_PERSRC_SHIFT 0 + +#define RCC_USBCKSELR_USBOSRC_MASK BIT(4) +#define RCC_USBCKSELR_USBOSRC_SHIFT 4 + +#define RCC_DDRITFCR_DDRCKMOD_SSR 0 +#define RCC_DDRITFCR_DDRCKMOD_ASR1 BIT(20) +#define RCC_DDRITFCR_DDRCKMOD_HSR1 BIT(21) + +#define RCC_DDRITFCR_DDRC2EN BIT(0) +#define RCC_DDRITFCR_DDRC2LPEN BIT(1) + +#define RCC_MP_CIFR_MASK U(0x110F1F) +#define RCC_OFFSET_MASK GENMASK_32(11, 0) + +vaddr_t stm32_rcc_base(void); + +#endif /*__DRIVERS_STM32MP13_RCC_H__*/ diff --git a/optee_os/core/include/drivers/stm32mp1_rcc.h b/optee_os/core/include/drivers/stm32mp1_rcc.h new file mode 100644 index 0000000..311cea8 --- /dev/null +++ b/optee_os/core/include/drivers/stm32mp1_rcc.h @@ -0,0 +1,566 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2017-2018, STMicroelectronics + */ + +#ifndef __DRIVERS_STM32MP1_RCC_H__ +#define __DRIVERS_STM32MP1_RCC_H__ + +#include +#include +#include + +#define RCC_TZCR 0x00 +#define RCC_OCENSETR 0x0C +#define RCC_OCENCLRR 0x10 +#define RCC_HSICFGR 0x18 +#define RCC_CSICFGR 0x1C +#define RCC_MPCKSELR 0x20 +#define RCC_ASSCKSELR 0x24 +#define RCC_RCK12SELR 0x28 +#define RCC_MPCKDIVR 0x2C +#define RCC_AXIDIVR 0x30 +#define RCC_APB4DIVR 0x3C +#define RCC_APB5DIVR 0x40 +#define RCC_RTCDIVR 0x44 +#define RCC_MSSCKSELR 0x48 +#define RCC_PLL1CR 0x80 +#define RCC_PLL1CFGR1 0x84 +#define RCC_PLL1CFGR2 0x88 +#define RCC_PLL1FRACR 0x8C +#define RCC_PLL1CSGR 0x90 +#define RCC_PLL2CR 0x94 +#define RCC_PLL2CFGR1 0x98 +#define RCC_PLL2CFGR2 0x9C +#define RCC_PLL2FRACR 0xA0 +#define RCC_PLL2CSGR 0xA4 +#define RCC_I2C46CKSELR 0xC0 +#define RCC_SPI6CKSELR 0xC4 +#define RCC_UART1CKSELR 0xC8 +#define RCC_RNG1CKSELR 0xCC +#define RCC_CPERCKSELR 0xD0 +#define RCC_STGENCKSELR 0xD4 +#define RCC_DDRITFCR 0xD8 +#define RCC_MP_BOOTCR 0x100 +#define RCC_MP_SREQSETR 0x104 +#define RCC_MP_SREQCLRR 0x108 +#define RCC_MP_GCR 0x10C +#define RCC_MP_APRSTCR 0x110 +#define RCC_MP_APRSTSR 0x114 +#define RCC_BDCR 0x140 +#define RCC_RDLSICR 0x144 +#define RCC_APB4RSTSETR 0x180 +#define RCC_APB4RSTCLRR 0x184 +#define RCC_APB5RSTSETR 0x188 +#define RCC_APB5RSTCLRR 0x18C +#define RCC_AHB5RSTSETR 0x190 +#define RCC_AHB5RSTCLRR 0x194 +#define RCC_AHB6RSTSETR 0x198 +#define RCC_AHB6RSTCLRR 0x19C +#define RCC_TZAHB6RSTSETR 0x1A0 +#define RCC_TZAHB6RSTCLRR 0x1A4 +#define RCC_MP_APB4ENSETR 0x200 +#define RCC_MP_APB4ENCLRR 0x204 +#define RCC_MP_APB5ENSETR 0x208 +#define RCC_MP_APB5ENCLRR 0x20C +#define RCC_MP_AHB5ENSETR 0x210 +#define RCC_MP_AHB5ENCLRR 0x214 +#define RCC_MP_AHB6ENSETR 0x218 +#define RCC_MP_AHB6ENCLRR 0x21C +#define RCC_MP_TZAHB6ENSETR 0x220 +#define RCC_MP_TZAHB6ENCLRR 0x224 +#define RCC_MC_APB4ENSETR 0x280 +#define RCC_MC_APB4ENCLRR 0x284 +#define RCC_MC_APB5ENSETR 0x288 +#define RCC_MC_APB5ENCLRR 0x28C +#define RCC_MC_AHB5ENSETR 0x290 +#define RCC_MC_AHB5ENCLRR 0x294 +#define RCC_MC_AHB6ENSETR 0x298 +#define RCC_MC_AHB6ENCLRR 0x29C +#define RCC_MP_APB4LPENSETR 0x300 +#define RCC_MP_APB4LPENCLRR 0x304 +#define RCC_MP_APB5LPENSETR 0x308 +#define RCC_MP_APB5LPENCLRR 0x30C +#define RCC_MP_AHB5LPENSETR 0x310 +#define RCC_MP_AHB5LPENCLRR 0x314 +#define RCC_MP_AHB6LPENSETR 0x318 +#define RCC_MP_AHB6LPENCLRR 0x31C +#define RCC_MP_TZAHB6LPENSETR 0x320 +#define RCC_MP_TZAHB6LPENCLRR 0x324 +#define RCC_MC_APB4LPENSETR 0x380 +#define RCC_MC_APB4LPENCLRR 0x384 +#define RCC_MC_APB5LPENSETR 0x388 +#define RCC_MC_APB5LPENCLRR 0x38C +#define RCC_MC_AHB5LPENSETR 0x390 +#define RCC_MC_AHB5LPENCLRR 0x394 +#define RCC_MC_AHB6LPENSETR 0x398 +#define RCC_MC_AHB6LPENCLRR 0x39C +#define RCC_BR_RSTSCLRR 0x400 +#define RCC_MP_GRSTCSETR 0x404 +#define RCC_MP_RSTSCLRR 0x408 +#define RCC_MP_IWDGFZSETR 0x40C +#define RCC_MP_IWDGFZCLRR 0x410 +#define RCC_MP_CIER 0x414 +#define RCC_MP_CIFR 0x418 +#define RCC_PWRLPDLYCR 0x41C +#define RCC_MP_RSTSSETR 0x420 +#define RCC_MCO1CFGR 0x800 +#define RCC_MCO2CFGR 0x804 +#define RCC_OCRDYR 0x808 +#define RCC_DBGCFGR 0x80C +#define RCC_RCK3SELR 0x820 +#define RCC_RCK4SELR 0x824 +#define RCC_TIMG1PRER 0x828 +#define RCC_TIMG2PRER 0x82C +#define RCC_MCUDIVR 0x830 +#define RCC_APB1DIVR 0x834 +#define RCC_APB2DIVR 0x838 +#define RCC_APB3DIVR 0x83C +#define RCC_PLL3CR 0x880 +#define RCC_PLL3CFGR1 0x884 +#define RCC_PLL3CFGR2 0x888 +#define RCC_PLL3FRACR 0x88C +#define RCC_PLL3CSGR 0x890 +#define RCC_PLL4CR 0x894 +#define RCC_PLL4CFGR1 0x898 +#define RCC_PLL4CFGR2 0x89C +#define RCC_PLL4FRACR 0x8A0 +#define RCC_PLL4CSGR 0x8A4 +#define RCC_I2C12CKSELR 0x8C0 +#define RCC_I2C35CKSELR 0x8C4 +#define RCC_SAI1CKSELR 0x8C8 +#define RCC_SAI2CKSELR 0x8CC +#define RCC_SAI3CKSELR 0x8D0 +#define RCC_SAI4CKSELR 0x8D4 +#define RCC_SPI2S1CKSELR 0x8D8 +#define RCC_SPI2S23CKSELR 0x8DC +#define RCC_SPI45CKSELR 0x8E0 +#define RCC_UART6CKSELR 0x8E4 +#define RCC_UART24CKSELR 0x8E8 +#define RCC_UART35CKSELR 0x8EC +#define RCC_UART78CKSELR 0x8F0 +#define RCC_SDMMC12CKSELR 0x8F4 +#define RCC_SDMMC3CKSELR 0x8F8 +#define RCC_ETHCKSELR 0x8FC +#define RCC_QSPICKSELR 0x900 +#define RCC_FMCCKSELR 0x904 +#define RCC_FDCANCKSELR 0x90C +#define RCC_SPDIFCKSELR 0x914 +#define RCC_CECCKSELR 0x918 +#define RCC_USBCKSELR 0x91C +#define RCC_RNG2CKSELR 0x920 +#define RCC_DSICKSELR 0x924 +#define RCC_ADCCKSELR 0x928 +#define RCC_LPTIM45CKSELR 0x92C +#define RCC_LPTIM23CKSELR 0x930 +#define RCC_LPTIM1CKSELR 0x934 +#define RCC_APB1RSTSETR 0x980 +#define RCC_APB1RSTCLRR 0x984 +#define RCC_APB2RSTSETR 0x988 +#define RCC_APB2RSTCLRR 0x98C +#define RCC_APB3RSTSETR 0x990 +#define RCC_APB3RSTCLRR 0x994 +#define RCC_AHB2RSTSETR 0x998 +#define RCC_AHB2RSTCLRR 0x99C +#define RCC_AHB3RSTSETR 0x9A0 +#define RCC_AHB3RSTCLRR 0x9A4 +#define RCC_AHB4RSTSETR 0x9A8 +#define RCC_AHB4RSTCLRR 0x9AC +#define RCC_MP_APB1ENSETR 0xA00 +#define RCC_MP_APB1ENCLRR 0xA04 +#define RCC_MP_APB2ENSETR 0xA08 +#define RCC_MP_APB2ENCLRR 0xA0C +#define RCC_MP_APB3ENSETR 0xA10 +#define RCC_MP_APB3ENCLRR 0xA14 +#define RCC_MP_AHB2ENSETR 0xA18 +#define RCC_MP_AHB2ENCLRR 0xA1C +#define RCC_MP_AHB3ENSETR 0xA20 +#define RCC_MP_AHB3ENCLRR 0xA24 +#define RCC_MP_AHB4ENSETR 0xA28 +#define RCC_MP_AHB4ENCLRR 0xA2C +#define RCC_MP_MLAHBENSETR 0xA38 +#define RCC_MP_MLAHBENCLRR 0xA3C +#define RCC_MC_APB1ENSETR 0xA80 +#define RCC_MC_APB1ENCLRR 0xA84 +#define RCC_MC_APB2ENSETR 0xA88 +#define RCC_MC_APB2ENCLRR 0xA8C +#define RCC_MC_APB3ENSETR 0xA90 +#define RCC_MC_APB3ENCLRR 0xA94 +#define RCC_MC_AHB2ENSETR 0xA98 +#define RCC_MC_AHB2ENCLRR 0xA9C +#define RCC_MC_AHB3ENSETR 0xAA0 +#define RCC_MC_AHB3ENCLRR 0xAA4 +#define RCC_MC_AHB4ENSETR 0xAA8 +#define RCC_MC_AHB4ENCLRR 0xAAC +#define RCC_MC_AXIMENSETR 0xAB0 +#define RCC_MC_AXIMENCLRR 0xAB4 +#define RCC_MC_MLAHBENSETR 0xAB8 +#define RCC_MC_MLAHBENCLRR 0xABC +#define RCC_MP_APB1LPENSETR 0xB00 +#define RCC_MP_APB1LPENCLRR 0xB04 +#define RCC_MP_APB2LPENSETR 0xB08 +#define RCC_MP_APB2LPENCLRR 0xB0C +#define RCC_MP_APB3LPENSETR 0xB10 +#define RCC_MP_APB3LPENCLRR 0xB14 +#define RCC_MP_AHB2LPENSETR 0xB18 +#define RCC_MP_AHB2LPENCLRR 0xB1C +#define RCC_MP_AHB3LPENSETR 0xB20 +#define RCC_MP_AHB3LPENCLRR 0xB24 +#define RCC_MP_AHB4LPENSETR 0xB28 +#define RCC_MP_AHB4LPENCLRR 0xB2C +#define RCC_MP_AXIMLPENSETR 0xB30 +#define RCC_MP_AXIMLPENCLRR 0xB34 +#define RCC_MP_MLAHBLPENSETR 0xB38 +#define RCC_MP_MLAHBLPENCLRR 0xB3C +#define RCC_MC_APB1LPENSETR 0xB80 +#define RCC_MC_APB1LPENCLRR 0xB84 +#define RCC_MC_APB2LPENSETR 0xB88 +#define RCC_MC_APB2LPENCLRR 0xB8C +#define RCC_MC_APB3LPENSETR 0xB90 +#define RCC_MC_APB3LPENCLRR 0xB94 +#define RCC_MC_AHB2LPENSETR 0xB98 +#define RCC_MC_AHB2LPENCLRR 0xB9C +#define RCC_MC_AHB3LPENSETR 0xBA0 +#define RCC_MC_AHB3LPENCLRR 0xBA4 +#define RCC_MC_AHB4LPENSETR 0xBA8 +#define RCC_MC_AHB4LPENCLRR 0xBAC +#define RCC_MC_AXIMLPENSETR 0xBB0 +#define RCC_MC_AXIMLPENCLRR 0xBB4 +#define RCC_MC_MLAHBLPENSETR 0xBB8 +#define RCC_MC_MLAHBLPENCLRR 0xBBC +#define RCC_MC_RSTSCLRR 0xC00 +#define RCC_MC_CIER 0xC14 +#define RCC_MC_CIFR 0xC18 +#define RCC_VERR 0xFF4 +#define RCC_IDR 0xFF8 +#define RCC_SIDR 0xFFC + +#define RCC_OFFSET_MASK GENMASK_32(11, 0) + +/* Values for RCC_TZCR register */ +#define RCC_TZCR_TZEN BIT(0) +#define RCC_TZCR_MCKPROT BIT(1) + +/* Used for most of RCC_SELR registers */ +#define RCC_SELR_SRC_MASK GENMASK_32(2, 0) +#define RCC_SELR_REFCLK_SRC_MASK GENMASK_32(1, 0) +#define RCC_SELR_SRCRDY BIT(31) + +/* Values of RCC_MPCKSELR register */ +#define RCC_MPCKSELR_HSI 0x00000000 +#define RCC_MPCKSELR_HSE 0x00000001 +#define RCC_MPCKSELR_PLL 0x00000002 +#define RCC_MPCKSELR_PLL_MPUDIV 0x00000003 + +/* Values of RCC_ASSCKSELR register */ +#define RCC_ASSCKSELR_HSI 0x00000000 +#define RCC_ASSCKSELR_HSE 0x00000001 +#define RCC_ASSCKSELR_PLL 0x00000002 + +/* Values of RCC_MSSCKSELR register */ +#define RCC_MSSCKSELR_HSI 0x00000000 +#define RCC_MSSCKSELR_HSE 0x00000001 +#define RCC_MSSCKSELR_CSI 0x00000002 +#define RCC_MSSCKSELR_PLL 0x00000003 + +/* Values of RCC_CPERCKSELR register */ +#define RCC_CPERCKSELR_HSI 0x00000000 +#define RCC_CPERCKSELR_CSI 0x00000001 +#define RCC_CPERCKSELR_HSE 0x00000002 + +/* used for most of RCC_DIVR registers: max div for RTC */ +#define RCC_DIVR_DIV_MASK GENMASK_32(5, 0) +#define RCC_DIVR_DIVRDY BIT(31) + +/* Masks for specific DIVR registers */ +#define RCC_APBXDIV_MASK GENMASK_32(2, 0) +#define RCC_MPUDIV_MASK GENMASK_32(2, 0) +#define RCC_AXIDIV_MASK GENMASK_32(2, 0) +#define RCC_MCUDIV_MASK GENMASK_32(3, 0) + +/* Used for TIMER Prescaler */ +#define RCC_TIMGXPRER_TIMGXPRE BIT(0) + +/* Offset between RCC_MP_xxxENSETR and RCC_MP_xxxENCLRR registers */ +#define RCC_MP_ENCLRR_OFFSET 4u + +/* Offset between RCC_MP_xxxRSTSETR and RCC_MP_xxxRSTCLRR registers */ +#define RCC_MP_RSTCLRR_OFFSET 4u + +/* Fields of RCC_BDCR register */ +#define RCC_BDCR_LSEON BIT(0) +#define RCC_BDCR_LSEBYP BIT(1) +#define RCC_BDCR_LSERDY BIT(2) +#define RCC_BDCR_DIGBYP BIT(3) +#define RCC_BDCR_LSEDRV_MASK GENMASK_32(5, 4) +#define RCC_BDCR_LSEDRV_SHIFT 4 +#define RCC_BDCR_LSECSSON BIT(8) +#define RCC_BDCR_RTCCKEN_POS 20 +#define RCC_BDCR_RTCCKEN BIT(RCC_BDCR_RTCCKEN_POS) +#define RCC_BDCR_RTCSRC_MASK GENMASK_32(17, 16) +#define RCC_BDCR_RTCSRC_SHIFT 16 +#define RCC_BDCR_VSWRST BIT(31) + +/* Fields of RCC_RDLSICR register */ +#define RCC_RDLSICR_LSION BIT(0) +#define RCC_RDLSICR_LSIRDY BIT(1) + +/* Used for all RCC_PLLCR registers */ +#define RCC_PLLNCR_PLLON BIT(0) +#define RCC_PLLNCR_PLLRDY BIT(1) +#define RCC_PLLNCR_SSCG_CTRL BIT(2) +#define RCC_PLLNCR_DIVPEN BIT(4) +#define RCC_PLLNCR_DIVQEN BIT(5) +#define RCC_PLLNCR_DIVREN BIT(6) +#define RCC_PLLNCR_DIVEN_SHIFT 4 + +/* Used for all RCC_PLLCFGR1 registers */ +#define RCC_PLLNCFGR1_DIVM_SHIFT 16 +#define RCC_PLLNCFGR1_DIVM_MASK GENMASK_32(21, 16) +#define RCC_PLLNCFGR1_DIVN_SHIFT 0 +#define RCC_PLLNCFGR1_DIVN_MASK GENMASK_32(8, 0) +/* Only for PLL3 and PLL4 */ +#define RCC_PLLNCFGR1_IFRGE_SHIFT 24 +#define RCC_PLLNCFGR1_IFRGE_MASK GENMASK_32(25, 24) + +/* Used for all RCC_PLLCFGR2 registers */ +#define RCC_PLLNCFGR2_DIVX_MASK GENMASK_32(6, 0) +#define RCC_PLLNCFGR2_DIVP_SHIFT 0 +#define RCC_PLLNCFGR2_DIVP_MASK GENMASK_32(6, 0) +#define RCC_PLLNCFGR2_DIVQ_SHIFT 8 +#define RCC_PLLNCFGR2_DIVQ_MASK GENMASK_32(14, 8) +#define RCC_PLLNCFGR2_DIVR_SHIFT 16 +#define RCC_PLLNCFGR2_DIVR_MASK GENMASK_32(22, 16) + +/* Used for all RCC_PLLFRACR registers */ +#define RCC_PLLNFRACR_FRACV_SHIFT 3 +#define RCC_PLLNFRACR_FRACV_MASK GENMASK_32(15, 3) +#define RCC_PLLNFRACR_FRACLE BIT(16) + +/* Used for all RCC_PLLCSGR registers */ +#define RCC_PLLNCSGR_INC_STEP_SHIFT 16 +#define RCC_PLLNCSGR_INC_STEP_MASK GENMASK_32(30, 16) +#define RCC_PLLNCSGR_MOD_PER_SHIFT 0 +#define RCC_PLLNCSGR_MOD_PER_MASK GENMASK_32(12, 0) +#define RCC_PLLNCSGR_SSCG_MODE_SHIFT 15 +#define RCC_PLLNCSGR_SSCG_MODE_MASK BIT(15) + +/* Used for RCC_OCENSETR and RCC_OCENCLRR registers */ +#define RCC_OCENR_HSION BIT(0) +#define RCC_OCENR_HSIKERON BIT(1) +#define RCC_OCENR_CSION BIT(4) +#define RCC_OCENR_CSIKERON BIT(5) +#define RCC_OCENR_DIGBYP BIT(7) +#define RCC_OCENR_HSEON BIT(8) +#define RCC_OCENR_HSEKERON BIT(9) +#define RCC_OCENR_HSEBYP BIT(10) +#define RCC_OCENR_HSECSSON BIT(11) + +/* Fields of RCC_OCRDYR register */ +#define RCC_OCRDYR_HSIRDY BIT(0) +#define RCC_OCRDYR_HSIDIVRDY BIT(2) +#define RCC_OCRDYR_CSIRDY BIT(4) +#define RCC_OCRDYR_HSERDY BIT(8) + +/* Fields of RCC_DDRITFCR register */ +#define RCC_DDRITFCR_DDRC1EN BIT(0) +#define RCC_DDRITFCR_DDRC1LPEN BIT(1) +#define RCC_DDRITFCR_DDRC2EN BIT(2) +#define RCC_DDRITFCR_DDRC2LPEN BIT(3) +#define RCC_DDRITFCR_DDRPHYCEN BIT(4) +#define RCC_DDRITFCR_DDRPHYCLPEN BIT(5) +#define RCC_DDRITFCR_DDRCAPBEN BIT(6) +#define RCC_DDRITFCR_DDRCAPBLPEN BIT(7) +#define RCC_DDRITFCR_AXIDCGEN BIT(8) +#define RCC_DDRITFCR_DDRPHYCAPBEN BIT(9) +#define RCC_DDRITFCR_DDRPHYCAPBLPEN BIT(10) +#define RCC_DDRITFCR_DDRCAPBRST BIT(14) +#define RCC_DDRITFCR_DDRCAXIRST BIT(15) +#define RCC_DDRITFCR_DDRCORERST BIT(16) +#define RCC_DDRITFCR_DPHYAPBRST BIT(17) +#define RCC_DDRITFCR_DPHYRST BIT(18) +#define RCC_DDRITFCR_DPHYCTLRST BIT(19) +#define RCC_DDRITFCR_DDRCKMOD_MASK GENMASK_32(22, 20) +#define RCC_DDRITFCR_DDRCKMOD_SHIFT 20 +#define RCC_DDRITFCR_DDRCKMOD_SSR 0 +#define RCC_DDRITFCR_DDRCKMOD_ASR1 BIT(20) +#define RCC_DDRITFCR_DDRCKMOD_HSR1 BIT(21) +#define RCC_DDRITFCR_GSKPCTRL BIT(24) + +/* Fields of RCC_HSICFGR register */ +#define RCC_HSICFGR_HSIDIV_MASK GENMASK_32(1, 0) +#define RCC_HSICFGR_HSITRIM_SHIFT 8 +#define RCC_HSICFGR_HSITRIM_MASK GENMASK_32(14, 8) +#define RCC_HSICFGR_HSICAL_SHIFT 16 +#define RCC_HSICFGR_HSICAL_MASK GENMASK_32(27, 16) + +/* Fields of RCC_CSICFGR register */ +#define RCC_CSICFGR_CSITRIM_SHIFT 8 +#define RCC_CSICFGR_CSITRIM_MASK GENMASK_32(12, 8) +#define RCC_CSICFGR_CSICAL_SHIFT 16 +#define RCC_CSICFGR_CSICAL_MASK GENMASK_32(23, 16) + +/* Used for RCC_MCO related operations */ +#define RCC_MCOCFG_MCOON BIT(12) +#define RCC_MCOCFG_MCODIV_MASK GENMASK_32(7, 4) +#define RCC_MCOCFG_MCODIV_SHIFT 4 +#define RCC_MCOCFG_MCOSRC_MASK GENMASK_32(2, 0) + +/* Fields of RCC_DBGCFGR register */ +#define RCC_DBGCFGR_DBGCKEN BIT(8) + +/* RCC register fields for reset reasons */ +#define RCC_MP_RSTSCLRR_PORRSTF BIT(0) +#define RCC_MP_RSTSCLRR_BORRSTF BIT(1) +#define RCC_MP_RSTSCLRR_PADRSTF BIT(2) +#define RCC_MP_RSTSCLRR_HCSSRSTF BIT(3) +#define RCC_MP_RSTSCLRR_VCORERSTF BIT(4) +#define RCC_MP_RSTSCLRR_MPSYSRSTF BIT(6) +#define RCC_MP_RSTSCLRR_MCSYSRSTF BIT(7) +#define RCC_MP_RSTSCLRR_IWDG1RSTF BIT(8) +#define RCC_MP_RSTSCLRR_IWDG2RSTF BIT(9) +#define RCC_MP_RSTSCLRR_STDBYRSTF BIT(11) +#define RCC_MP_RSTSCLRR_CSTDBYRSTF BIT(12) + +/* Global Reset Register */ +#define RCC_MP_GRSTCSETR_MPSYSRST BIT(0) +#define RCC_MP_GRSTCSETR_MCURST BIT(1) +#define RCC_MP_GRSTCSETR_MPUP0RST BIT(4) +#define RCC_MP_GRSTCSETR_MPUP1RST BIT(5) + +/* Clock Source Interrupt Flag Register */ +#define RCC_MP_CIFR_LSIRDYF BIT(0) +#define RCC_MP_CIFR_LSERDYF BIT(1) +#define RCC_MP_CIFR_HSIRDYF BIT(2) +#define RCC_MP_CIFR_HSERDYF BIT(3) +#define RCC_MP_CIFR_CSIRDYF BIT(4) +#define RCC_MP_CIFR_PLL1DYF BIT(8) +#define RCC_MP_CIFR_PLL2DYF BIT(9) +#define RCC_MP_CIFR_PLL3DYF BIT(10) +#define RCC_MP_CIFR_PLL4DYF BIT(11) +#define RCC_MP_CIFR_LSECSSF BIT(16) +#define RCC_MP_CIFR_WKUPF BIT(20) +#define RCC_MP_CIFR_MASK (RCC_MP_CIFR_LSIRDYF | RCC_MP_CIFR_LSERDYF | \ + RCC_MP_CIFR_HSIRDYF | RCC_MP_CIFR_HSERDYF | \ + RCC_MP_CIFR_CSIRDYF | RCC_MP_CIFR_PLL1DYF | \ + RCC_MP_CIFR_PLL2DYF | RCC_MP_CIFR_PLL3DYF | \ + RCC_MP_CIFR_PLL4DYF | RCC_MP_CIFR_LSECSSF | \ + RCC_MP_CIFR_WKUPF) + +/* Stop Request Set Register */ +#define RCC_MP_SREQSETR_STPREQ_P0 BIT(0) +#define RCC_MP_SREQSETR_STPREQ_P1 BIT(1) + +/* Stop Request Clear Register */ +#define RCC_MP_SREQCLRR_STPREQ_P0 BIT(0) +#define RCC_MP_SREQCLRR_STPREQ_P1 BIT(1) + +/* Global Control Register */ +#define RCC_MP_GCR_BOOT_MCU BIT(0) + +/* RCC_MP_APB5RST(SET|CLR)R bit fields */ +#define RCC_APB5RSTSETR_SPI6RST BIT(0) +#define RCC_APB5RSTSETR_I2C4RST BIT(2) +#define RCC_APB5RSTSETR_I2C6RST BIT(3) +#define RCC_APB5RSTSETR_USART1RST BIT(4) +#define RCC_APB5RSTSETR_STGENRST BIT(20) + +/* RCC_MP_APB1EN(SET|CLR)R bit fields */ +#define RCC_MP_APB1ENSETR_I2C5EN_POS 24 + +#define RCC_MP_APB1ENSETR_I2C5EN BIT(RCC_MP_APB1ENSETR_I2C5EN_POS) + +/* RCC_MP_APB5EN(SET|CLR)R bit fields */ +#define RCC_MP_APB5ENSETR_SPI6EN_POS 0 +#define RCC_MP_APB5ENSETR_I2C4EN_POS 2 +#define RCC_MP_APB5ENSETR_I2C6EN_POS 3 +#define RCC_MP_APB5ENSETR_USART1EN_POS 4 +#define RCC_MP_APB5ENSETR_RTCAPBEN_POS 8 +#define RCC_MP_APB5ENSETR_TZC1EN_POS 11 +#define RCC_MP_APB5ENSETR_TZC2EN_POS 12 +#define RCC_MP_APB5ENSETR_TZPCEN_POS 13 +#define RCC_MP_APB5ENSETR_IWDG1APBEN_POS 15 +#define RCC_MP_APB5ENSETR_BSECEN_POS 16 +#define RCC_MP_APB5ENSETR_STGENEN_POS 20 + +#define RCC_MP_APB5ENSETR_SPI6EN BIT(RCC_MP_APB5ENSETR_SPI6EN_POS) +#define RCC_MP_APB5ENSETR_I2C4EN BIT(RCC_MP_APB5ENSETR_I2C4EN_POS) +#define RCC_MP_APB5ENSETR_I2C6EN BIT(RCC_MP_APB5ENSETR_I2C6EN_POS) +#define RCC_MP_APB5ENSETR_USART1EN BIT(RCC_MP_APB5ENSETR_USART1EN_POS) +#define RCC_MP_APB5ENSETR_RTCAPBEN BIT(RCC_MP_APB5ENSETR_RTCAPBEN_POS) +#define RCC_MP_APB5ENSETR_TZC1EN BIT(RCC_MP_APB5ENSETR_TZC1EN_POS) +#define RCC_MP_APB5ENSETR_TZC2EN BIT(RCC_MP_APB5ENSETR_TZC2EN_POS) +#define RCC_MP_APB5ENSETR_TZPCEN BIT(RCC_MP_APB5ENSETR_TZPCEN_POS) +#define RCC_MP_APB5ENSETR_IWDG1APBEN BIT(RCC_MP_APB5ENSETR_IWDG1APBEN_POS) +#define RCC_MP_APB5ENSETR_BSECEN BIT(RCC_MP_APB5ENSETR_BSECEN_POS) +#define RCC_MP_APB5ENSETR_STGENEN BIT(RCC_MP_APB5ENSETR_STGENEN_POS) + +/* RCC_MP_APB5LPEN(SET|CLR)R bit fields */ +#define RCC_MP_APB5LPENSETR_SPI6LPEN BIT(0) +#define RCC_MP_APB5LPENSETR_I2C4LPEN BIT(2) +#define RCC_MP_APB5LPENSETR_I2C6LPEN BIT(3) +#define RCC_MP_APB5LPENSETR_USART1LPEN BIT(4) +#define RCC_MP_APB5LPENSETR_RTCAPBLPEN BIT(8) +#define RCC_MP_APB5LPENSETR_TZC1LPEN BIT(11) +#define RCC_MP_APB5LPENSETR_TZC2LPEN BIT(12) +#define RCC_MP_APB5LPENSETR_TZPCLPEN BIT(13) +#define RCC_MP_APB5LPENSETR_IWDG1APBLPEN BIT(15) +#define RCC_MP_APB5LPENSETR_BSECLPEN BIT(16) +#define RCC_MP_APB5LPENSETR_STGENLPEN BIT(20) +#define RCC_MP_APB5LPENSETR_STGENSTPEN BIT(21) + +/* RCC_MP_AHB5RST(SET|CLR)R bit fields */ +#define RCC_AHB5RSTSETR_GPIOZRST BIT(0) +#define RCC_AHB5RSTSETR_CRYP1RST BIT(4) +#define RCC_AHB5RSTSETR_HASH1RST BIT(5) +#define RCC_AHB5RSTSETR_RNG1RST BIT(6) +#define RCC_AHB5RSTSETR_AXIMCRST BIT(16) + +/* RCC_MP_AHB5EN(SET|CLR)R bit fields */ +#define RCC_MP_AHB5ENSETR_GPIOZEN_POS 0 +#define RCC_MP_AHB5ENSETR_CRYP1EN_POS 4 +#define RCC_MP_AHB5ENSETR_HASH1EN_POS 5 +#define RCC_MP_AHB5ENSETR_RNG1EN_POS 6 +#define RCC_MP_AHB5ENSETR_BKPSRAMEN_POS 8 +#define RCC_MP_AHB5ENSETR_AXIMCEN_POS 16 + +#define RCC_MP_AHB5ENSETR_GPIOZEN BIT(RCC_MP_AHB5ENSETR_GPIOZEN_POS) +#define RCC_MP_AHB5ENSETR_CRYP1EN BIT(RCC_MP_AHB5ENSETR_CRYP1EN_POS) +#define RCC_MP_AHB5ENSETR_HASH1EN BIT(RCC_MP_AHB5ENSETR_HASH1EN_POS) +#define RCC_MP_AHB5ENSETR_RNG1EN BIT(RCC_MP_AHB5ENSETR_RNG1EN_POS) +#define RCC_MP_AHB5ENSETR_BKPSRAMEN BIT(RCC_MP_AHB5ENSETR_BKPSRAMEN_POS) +#define RCC_MP_AHB5ENSETR_AXIMCEN BIT(RCC_MP_AHB5ENSETR_AXIMCEN_POS) + +/* RCC_MP_AHB5LPEN(SET|CLR)R bit fields */ +#define RCC_MP_AHB5LPENSETR_GPIOZLPEN BIT(0) +#define RCC_MP_AHB5LPENSETR_CRYP1LPEN BIT(4) +#define RCC_MP_AHB5LPENSETR_HASH1LPEN BIT(5) +#define RCC_MP_AHB5LPENSETR_RNG1LPEN BIT(6) +#define RCC_MP_AHB5LPENSETR_BKPSRAMLPEN BIT(8) + +/* RCC_MP_TZAHB6EN(SET|CLR)R bit fields */ +#define RCC_MP_TZAHB6ENSETR_MDMA_POS 0 +#define RCC_MP_TZAHB6ENSETR_MDMA BIT(RCC_MP_TZAHB6ENSETR_MDMA_POS) + +/* RCC_MP_IWDGFZ(SET|CLR)R bit fields */ +#define RCC_MP_IWDGFZSETR_IWDG1 BIT(0) +#define RCC_MP_IWDGFZSETR_IWDG2 BIT(1) + +#define DT_RCC_CLK_COMPAT "st,stm32mp1-rcc" +#define DT_RCC_SECURE_CLK_COMPAT "st,stm32mp1-rcc-secure" + +#ifndef __ASSEMBLER__ +#include + +static inline bool stm32_rcc_is_secure(void) +{ + return io_read32(stm32_rcc_base() + RCC_TZCR) & RCC_TZCR_TZEN; +} + +static inline bool stm32_rcc_is_mckprot(void) +{ + return io_read32(stm32_rcc_base() + RCC_TZCR) & RCC_TZCR_MCKPROT; +} +#endif /*__ASSEMBLER__*/ + +#endif /*__DRIVERS_STM32MP1_RCC_H__*/ diff --git a/optee_os/core/include/drivers/stm32mp_dt_bindings.h b/optee_os/core/include/drivers/stm32mp_dt_bindings.h new file mode 100644 index 0000000..2d7a27c --- /dev/null +++ b/optee_os/core/include/drivers/stm32mp_dt_bindings.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, STMicroelectronics + */ +#ifndef STM32MP_DT_BINDINGS_H +#define STM32MP_DT_BINDINGS_H + +#ifdef CFG_STM32MP13 +#include +#include +#include +#include +#endif + +#ifdef CFG_STM32MP15 +#include +#include +#include +#endif + +#endif /* STM32MP_DT_BINDINGS_H */ diff --git a/optee_os/core/include/drivers/stpmic1.h b/optee_os/core/include/drivers/stpmic1.h new file mode 100644 index 0000000..0d020ca --- /dev/null +++ b/optee_os/core/include/drivers/stpmic1.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2016-2018, STMicroelectronics - All Rights Reserved + */ + +#ifndef __STPMIC1_H__ +#define __STPMIC1_H__ + +#include +#include + +#define TURN_ON_REG 0x1U +#define TURN_OFF_REG 0x2U +#define ICC_LDO_TURN_OFF_REG 0x3U +#define ICC_BUCK_TURN_OFF_REG 0x4U +#define RESET_STATUS_REG 0x5U +#define VERSION_STATUS_REG 0x6U +#define MAIN_CONTROL_REG 0x10U +#define PADS_PULL_REG 0x11U +#define BUCK_PULL_DOWN_REG 0x12U +#define LDO14_PULL_DOWN_REG 0x13U +#define LDO56_PULL_DOWN_REG 0x14U +#define VIN_CONTROL_REG 0x15U +#define PONKEY_TIMER_REG 0x16U +#define MASK_RANK_BUCK_REG 0x17U +#define MASK_RESET_BUCK_REG 0x18U +#define MASK_RANK_LDO_REG 0x19U +#define MASK_RESET_LDO_REG 0x1AU +#define WATCHDOG_CONTROL_REG 0x1BU +#define WATCHDOG_TIMER_REG 0x1CU +#define BUCK_ICC_TURNOFF_REG 0x1DU +#define LDO_ICC_TURNOFF_REG 0x1EU +#define BUCK_APM_CONTROL_REG 0x1FU +#define BUCK1_CONTROL_REG 0x20U +#define BUCK2_CONTROL_REG 0x21U +#define BUCK3_CONTROL_REG 0x22U +#define BUCK4_CONTROL_REG 0x23U +#define VREF_DDR_CONTROL_REG 0x24U +#define LDO1_CONTROL_REG 0x25U +#define LDO2_CONTROL_REG 0x26U +#define LDO3_CONTROL_REG 0x27U +#define LDO4_CONTROL_REG 0x28U +#define LDO5_CONTROL_REG 0x29U +#define LDO6_CONTROL_REG 0x2AU +#define BUCK1_PWRCTRL_REG 0x30U +#define BUCK2_PWRCTRL_REG 0x31U +#define BUCK3_PWRCTRL_REG 0x32U +#define BUCK4_PWRCTRL_REG 0x33U +#define VREF_DDR_PWRCTRL_REG 0x34U +#define LDO1_PWRCTRL_REG 0x35U +#define LDO2_PWRCTRL_REG 0x36U +#define LDO3_PWRCTRL_REG 0x37U +#define LDO4_PWRCTRL_REG 0x38U +#define LDO5_PWRCTRL_REG 0x39U +#define LDO6_PWRCTRL_REG 0x3AU +#define FREQUENCY_SPREADING_REG 0x3BU +#define USB_CONTROL_REG 0x40U +#define ITLATCH1_REG 0x50U +#define ITLATCH2_REG 0x51U +#define ITLATCH3_REG 0x52U +#define ITLATCH4_REG 0x53U +#define ITSETLATCH1_REG 0x60U +#define ITSETLATCH2_REG 0x61U +#define ITSETLATCH3_REG 0x62U +#define ITSETLATCH4_REG 0x63U +#define ITCLEARLATCH1_REG 0x70U +#define ITCLEARLATCH2_REG 0x71U +#define ITCLEARLATCH3_REG 0x72U +#define ITCLEARLATCH4_REG 0x73U +#define ITMASK1_REG 0x80U +#define ITMASK2_REG 0x81U +#define ITMASK3_REG 0x82U +#define ITMASK4_REG 0x83U +#define ITSETMASK1_REG 0x90U +#define ITSETMASK2_REG 0x91U +#define ITSETMASK3_REG 0x92U +#define ITSETMASK4_REG 0x93U +#define ITCLEARMASK1_REG 0xA0U +#define ITCLEARMASK2_REG 0xA1U +#define ITCLEARMASK3_REG 0xA2U +#define ITCLEARMASK4_REG 0xA3U +#define ITSOURCE1_REG 0xB0U +#define ITSOURCE2_REG 0xB1U +#define ITSOURCE3_REG 0xB2U +#define ITSOURCE4_REG 0xB3U + +/* Registers masks */ +#define LDO_VOLTAGE_MASK GENMASK_32(6, 2) +#define BUCK_VOLTAGE_MASK GENMASK_32(7, 2) +#define LDO_BUCK_VOLTAGE_SHIFT 2 +#define LDO_BUCK_ENABLE_POS 0 +#define LDO_BUCK_ENABLE_MASK BIT(LDO_BUCK_ENABLE_POS) +#define LDO_BUCK_HPLP_POS 1 +#define LDO_BUCK_RANK_MASK BIT(0) +#define LDO_BUCK_RESET_MASK BIT(0) +#define LDO_BUCK_PULL_DOWN_MASK GENMASK_32(1, 0) + +/* Pull down register */ +#define BUCK1_PULL_DOWN_SHIFT 0 +#define BUCK2_PULL_DOWN_SHIFT 2 +#define BUCK3_PULL_DOWN_SHIFT 4 +#define BUCK4_PULL_DOWN_SHIFT 6 +#define VREF_DDR_PULL_DOWN_SHIFT 4 + +/* Buck Mask reset register */ +#define BUCK1_MASK_RESET_SHIFT 0 +#define BUCK2_MASK_RESET_SHIFT 1 +#define BUCK3_MASK_RESET_SHIFT 2 +#define BUCK4_MASK_RESET_SHIFT 3 + +/* LDO Mask reset register */ +#define LDO1_MASK_RESET_SHIFT 0 +#define LDO2_MASK_RESET_SHIFT 1 +#define LDO3_MASK_RESET_SHIFT 2 +#define LDO4_MASK_RESET_SHIFT 3 +#define LDO5_MASK_RESET_SHIFT 4 +#define LDO6_MASK_RESET_SHIFT 5 +#define VREF_DDR_MASK_RESET_SHIFT 6 + +/* Main PMIC Control Register (MAIN_CONTROL_REG) */ +#define ICC_EVENT_ENABLED BIT(4) +#define PWRCTRL_POLARITY_HIGH BIT(3) +#define PWRCTRL_PIN_VALID BIT(2) +#define RESTART_REQUEST_ENABLED BIT(1) +#define SOFTWARE_SWITCH_OFF_ENABLED BIT(0) + +/* Main PMIC PADS Control Register (PADS_PULL_REG) */ +#define WAKEUP_DETECTOR_DISABLED BIT(4) +#define PWRCTRL_PD_ACTIVE BIT(3) +#define PWRCTRL_PU_ACTIVE BIT(2) +#define WAKEUP_PD_ACTIVE BIT(1) +#define PONKEY_PU_ACTIVE BIT(0) + +/* Main PMIC VINLOW Control Register (VIN_CONTROL_REGC DMSC) */ +#define SWIN_DETECTOR_ENABLED BIT(7) +#define SWOUT_DETECTOR_ENABLED BIT(6) +#define VINLOW_HYST_MASK GENMASK_32(5, 4) +#define VINLOW_HYST_SHIFT 4 +#define VINLOW_THRESHOLD_MASK GENMASK_32(3, 1) +#define VINLOW_THRESHOLD_SHIFT 1 +#define VINLOW_ENABLED BIT(0) +#define VINLOW_CTRL_REG_MASK GENMASK_32(7, 0) + +/* USB Control Register */ +#define BOOST_OVP_DISABLED_POS 7 +#define VBUS_OTG_DETECTION_DISABLED_POS 6 +#define OCP_LIMIT_HIGH_POS 3 +#define SWIN_SWOUT_ENABLED_POS 2 +#define USBSW_OTG_SWITCH_ENABLED_POS 1 +#define BOOST_ENABLED_POS 0 + +/* + * Bind SPMIC1 device driver with a specific I2C bus instance + * @i2c_handle: target I2C instance to use + * @i2c_addr: I2C address of the STPMIC1 device + */ +void stpmic1_bind_i2c(struct i2c_handle_s *i2c_handle, uint16_t i2c_addr); + +/* Read STPMIC1 device version information */ +int stpmic1_get_version(unsigned long *version); + +/* Read STPMIC1 device internal registers content */ +void stpmic1_dump_regulators(void); + +/* Enable power control in STPMIC1 device */ +int stpmic1_powerctrl_on(void); + +/* Disable STPMIC1 device */ +int stpmic1_switch_off(void); + +/* Read/write/update STPMIC1 device internal register */ +int stpmic1_register_read(uint8_t register_id, uint8_t *value); +int stpmic1_register_write(uint8_t register_id, uint8_t value); +int stpmic1_register_update(uint8_t register_id, uint8_t value, uint8_t mask); + +int stpmic1_regulator_mask_reset_set(const char *name); + +/* API for low power configuration of regulators driven from STPMIC1 device */ +int stpmic1_lp_copy_reg(const char *name); +int stpmic1_lp_reg_on_off(const char *name, uint8_t enable); +int stpmic1_lp_set_mode(const char *name, uint8_t hplp); +int stpmic1_lp_set_voltage(const char *name, uint16_t millivolts); + +/* + * Specific API for controlling regulators driven from STPMIC1 device + * from unpaged execution context of the STPMIC1 driver. + */ + +/* + * The STPMIC1 is accessed during low power sequence in unpaged + * execution context. To prevent adding an unpaged constraint on + * STPMIC1 regulator definitions, conversion tables and device tree + * content, the regulators configurations are read from device tree + * at boot time and saved in memory for being applied at runtime + * without needing pager support. + * + * There are 2 types of regulator configuration loaded during such + * low power and unpaged sequences: boot-on (bo) configuration and + * low power (lp) configuration. + */ +struct stpmic1_bo_cfg { + uint8_t ctrl_reg; + uint8_t min_value; + uint8_t enable_pos; + uint8_t mask; + uint8_t pd_reg; + uint8_t pd_value; + uint8_t pd_mask; + uint8_t mrst_reg; + uint8_t mrst_value; + uint8_t mrst_mask; +}; + +struct stpmic1_lp_cfg { + uint8_t ctrl_reg; + uint8_t lp_reg; + uint8_t value; + uint8_t mask; +}; + +int stpmic1_bo_enable_cfg(const char *name, struct stpmic1_bo_cfg *cfg); +int stpmic1_bo_enable_unpg(struct stpmic1_bo_cfg *cfg); +int stpmic1_bo_voltage_cfg(const char *name, uint16_t min_millivolt, + struct stpmic1_bo_cfg *cfg); +int stpmic1_bo_voltage_unpg(struct stpmic1_bo_cfg *cfg); + +int stpmic1_bo_pull_down_cfg(const char *name, + struct stpmic1_bo_cfg *cfg); +int stpmic1_bo_pull_down_unpg(struct stpmic1_bo_cfg *cfg); + +int stpmic1_bo_mask_reset_cfg(const char *name, struct stpmic1_bo_cfg *cfg); +int stpmic1_bo_mask_reset_unpg(struct stpmic1_bo_cfg *cfg); + +bool stpmic1_regu_has_lp_cfg(const char *name); +int stpmic1_lp_cfg(const char *name, struct stpmic1_lp_cfg *cfg); +int stpmic1_lp_load_unpg(struct stpmic1_lp_cfg *cfg); +int stpmic1_lp_on_off_unpg(struct stpmic1_lp_cfg *cfg, int enable); +int stpmic1_lp_mode_unpg(struct stpmic1_lp_cfg *cfg, + unsigned int mode); +int stpmic1_lp_voltage_cfg(const char *name, uint16_t millivolts, + struct stpmic1_lp_cfg *cfg); +int stpmic1_lp_voltage_unpg(struct stpmic1_lp_cfg *cfg); + +#endif /*__STPMIC1_H__*/ diff --git a/optee_os/core/include/drivers/stpmic1_regulator.h b/optee_os/core/include/drivers/stpmic1_regulator.h new file mode 100644 index 0000000..5e0e655 --- /dev/null +++ b/optee_os/core/include/drivers/stpmic1_regulator.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021-2022, STMicroelectronics + */ + +#ifndef __DRIVERS_STPMIC1_REGULATOR_H +#define __DRIVERS_STPMIC1_REGULATOR_H + +#include + +/* + * Return true if @name refers to a knwon regulator, return false otherwise + */ +bool stpmic1_regulator_is_valid(const char *name); + +/* + * Enable STPMIC1 regulator identified by @name. + * Return 0 on success and a non-0 value if failing + */ +int stpmic1_regulator_enable(const char *name); + +/* + * Disable STPMIC1 regulator identified by @name. + * Return 0 on success and a non-0 value if failing + */ +int stpmic1_regulator_disable(const char *name); + +/* + * Return true if regulator identified by @name is enabled and false otherwise. + * Return 0 on success and a non-0 value if failing + */ +bool stpmic1_is_regulator_enabled(const char *name); + +/* + * Retrieve regulator levels array (in millivolts) and/or levels count + * @name: regulator identifier + * @levels: output reference for an arrays of the supported levels, or NULL + * @levels_count: output reference for number of supported levels, or NULL + */ +void stpmic1_regulator_levels_mv(const char *name, const uint16_t **levels, + size_t *levels_count); + +/* + * Set voltage level @millivolt for target regulator @name + * @name: regulator identifier + * @millivot: target voltage level, in mV + */ +int stpmic1_regulator_voltage_set(const char *name, uint16_t millivolts); + +/* + * Get current voltage level (in millivolt) for target regulator @name + * @name: regulator identifier + * Return a positive millivolt level on success or a negative value on error + */ +int stpmic1_regulator_voltage_get(const char *name); + +#endif /*__DRIVERS_STPMIC1_REGULATOR_H*/ diff --git a/optee_os/core/include/drivers/tzc380.h b/optee_os/core/include/drivers/tzc380.h new file mode 100644 index 0000000..fdcad27 --- /dev/null +++ b/optee_os/core/include/drivers/tzc380.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2017 NXP + * All rights reserved. + * + * Peng Fan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DRIVERS_TZC380_H +#define __DRIVERS_TZC380_H + +#include +#include +#include +#include +#include + +#define TZC400_REG_SIZE U(0x1000) + +#define BUILD_CONFIG_OFF U(0x000) +#define ACTION_OFF U(0x004) +#define LOCKDOWN_RANGE_OFF U(0x008) +#define LOCKDOWN_SELECT_OFF U(0x00C) +#define INT_STATUS U(0x010) +#define INT_CLEAR U(0x014) + +#define FAIL_ADDRESS_LOW_OFF U(0x020) +#define FAIL_ADDRESS_HIGH_OFF U(0x024) +#define FAIL_CONTROL_OFF U(0x028) +#define FAIL_ID U(0x02c) + +#define SPECULATION_CTRL_OFF U(0x030) +#define SECURITY_INV_EN_OFF U(0x034) + +#define REGION_SETUP_LOW_OFF(n) (U(0x100) + (n) * U(0x10)) +#define REGION_SETUP_HIGH_OFF(n) (U(0x104) + (n) * U(0x10)) +#define REGION_ATTRIBUTES_OFF(n) (U(0x108) + (n) * U(0x10)) + +/* ID Registers */ +#define PID0_OFF U(0xfe0) +#define PID1_OFF U(0xfe4) +#define PID2_OFF U(0xfe8) +#define PID3_OFF U(0xfec) +#define PID4_OFF U(0xfd0) +#define CID0_OFF U(0xff0) +#define CID1_OFF U(0xff4) +#define CID2_OFF U(0xff8) +#define CID3_OFF U(0xffc) + +#define BUILD_CONFIG_AW_SHIFT U(8) +#define BUILD_CONFIG_AW_MASK U(0x3f) +#define BUILD_CONFIG_NR_SHIFT U(0) +#define BUILD_CONFIG_NR_MASK U(0xf) + +#define ACTION_RV_SHIFT U(0) +#define ACTION_RV_MASK U(0x3) +#define ACTION_RV_LOWOK U(0x0) +#define ACTION_RV_LOWERR U(0x1) +#define ACTION_RV_HIGHOK U(0x2) +#define ACTION_RV_HIGHERR U(0x3) + +/* Speculation is enabled by default. */ +#define SPECULATION_CTRL_WRITE_DISABLE BIT(1) +#define SPECULATION_CTRL_READ_DISABLE BIT(0) + +#define INT_STATUS_OVERRUN_SHIFT U(1) +#define INT_STATUS_OVERRUN_MASK U(0x1) +#define INT_STATUS_STATUS_SHIFT U(0) +#define INT_STATUS_STATUS_MASK U(0x1) + +#define INT_CLEAR_CLEAR_SHIFT U(0) +#define INT_CLEAR_CLEAR_MASK U(0x1) + +#define TZC380_COMPONENT_ID U(0xb105f00d) +#define TZC380_PERIPH_ID_LOW U(0x001bb380) +#define TZC380_PERIPH_ID_HIGH U(0x00000004) + +/******************************************************************************* + * Function & variable prototypes + ******************************************************************************/ + +/* + * What type of action is expected when an access violation occurs. + * The memory requested is zeroed. But we can also raise and event to + * let the system know it happened. + * We can raise an interrupt(INT) and/or cause an exception(ERR). + * TZC_ACTION_NONE - No interrupt, no Exception + * TZC_ACTION_ERR - No interrupt, raise exception -> sync external + * data abort + * TZC_ACTION_INT - Raise interrupt, no exception + * TZC_ACTION_ERR_INT - Raise interrupt, raise exception -> sync + * external data abort + */ +enum tzc_action { + TZC_ACTION_NONE = 0, + TZC_ACTION_ERR = 1, + TZC_ACTION_INT = 2, + TZC_ACTION_ERR_INT = (TZC_ACTION_ERR | TZC_ACTION_INT) +}; + + +#define TZC_SP_NS_W BIT(0) +#define TZC_SP_NS_R BIT(1) +#define TZC_SP_S_W BIT(2) +#define TZC_SP_S_R BIT(3) + +#define TZC_ATTR_SP_SHIFT U(28) +#define TZC_ATTR_SP_MASK GENMASK_32(31, 28) +#define TZC_ATTR_SP_ALL SHIFT_U32(TZC_SP_S_W | TZC_SP_S_R | \ + TZC_SP_NS_W | TZC_SP_NS_R, \ + TZC_ATTR_SP_SHIFT) +#define TZC_ATTR_SP_S_RW SHIFT_U32(TZC_SP_S_W | TZC_SP_S_R, \ + TZC_ATTR_SP_SHIFT) +#define TZC_ATTR_SP_NS_RW SHIFT_U32(TZC_SP_NS_W | TZC_SP_NS_R, \ + TZC_ATTR_SP_SHIFT) + +#define TZC_REGION_SIZE_32K U(0xe) +#define TZC_REGION_SIZE_64K U(0xf) +#define TZC_REGION_SIZE_128K U(0x10) +#define TZC_REGION_SIZE_256K U(0x11) +#define TZC_REGION_SIZE_512K U(0x12) +#define TZC_REGION_SIZE_1M U(0x13) +#define TZC_REGION_SIZE_2M U(0x14) +#define TZC_REGION_SIZE_4M U(0x15) +#define TZC_REGION_SIZE_8M U(0x16) +#define TZC_REGION_SIZE_16M U(0x17) +#define TZC_REGION_SIZE_32M U(0x18) +#define TZC_REGION_SIZE_64M U(0x19) +#define TZC_REGION_SIZE_128M U(0x1a) +#define TZC_REGION_SIZE_256M U(0x1b) +#define TZC_REGION_SIZE_512M U(0x1c) +#define TZC_REGION_SIZE_1G U(0x1d) +#define TZC_REGION_SIZE_2G U(0x1e) +#define TZC_REGION_SIZE_4G U(0x1f) +#define TZC_REGION_SIZE_8G U(0x20) +#define TZC_REGION_SIZE_16G U(0x21) +#define TZC_REGION_SIZE_32G U(0x22) +#define TZC_REGION_SIZE_64G U(0x23) +#define TZC_REGION_SIZE_128G U(0x24) +#define TZC_REGION_SIZE_256G U(0x25) +#define TZC_REGION_SIZE_512G U(0x26) +#define TZC_REGION_SIZE_1T U(0x27) +#define TZC_REGION_SIZE_2T U(0x28) +#define TZC_REGION_SIZE_4T U(0x29) +#define TZC_REGION_SIZE_8T U(0x2a) +#define TZC_REGION_SIZE_16T U(0x2b) +#define TZC_REGION_SIZE_32T U(0x2c) +#define TZC_REGION_SIZE_64T U(0x2d) +#define TZC_REGION_SIZE_128T U(0x2e) +#define TZC_REGION_SIZE_256T U(0x2f) +#define TZC_REGION_SIZE_512T U(0x30) +#define TZC_REGION_SIZE_1P U(0x31) +#define TZC_REGION_SIZE_2P U(0x32) +#define TZC_REGION_SIZE_4P U(0x33) +#define TZC_REGION_SIZE_8P U(0x34) +#define TZC_REGION_SIZE_16P U(0x35) +#define TZC_REGION_SIZE_32P U(0x36) +#define TZC_REGION_SIZE_64P U(0x37) +#define TZC_REGION_SIZE_128P U(0x38) +#define TZC_REGION_SIZE_256P U(0x39) +#define TZC_REGION_SIZE_512P U(0x3a) +#define TZC_REGION_SIZE_1E U(0x3b) +#define TZC_REGION_SIZE_2E U(0x3c) +#define TZC_REGION_SIZE_4E U(0x3d) +#define TZC_REGION_SIZE_8E U(0x3e) +#define TZC_REGION_SIZE_16E U(0x3f) + +#define TZC_REGION_SIZE_SHIFT U(0x1) +#define TZC_REGION_SIZE_MASK GENMASK_32(6, 1) +#define TZC_ATTR_REGION_SIZE(s) SHIFT_U32(s, TZC_REGION_SIZE_SHIFT) + +#define TZC_SUBREGION_DIS_SHIFT U(8) +#define TZC_SUBREGION_DIS_MASK GENMASK_32(15, 8) +#define TZC_ATTR_SUBREGION_DIS(subreg) \ + (BIT((subreg) + TZC_SUBREGION_DIS_SHIFT) & \ + TZC_SUBREGION_DIS_MASK) + +#define TZC_ATTR_REGION_EN_SHIFT U(0x0) +#define TZC_ATTR_REGION_EN_MASK U(0x1) + +#define TZC_ATTR_REGION_EN +#define TZC_ATTR_REGION_ENABLE U(0x1) +#define TZC_ATTR_REGION_DISABLE U(0x0) + +#define LOCKDOWN_RANGE_ENABLE BIT(31) + +#define LOCKDOWN_SELECT_RANGE_ENABLE BIT(0) + +void tzc_init(vaddr_t base); +void tzc_configure_region(uint8_t region, vaddr_t region_base, uint32_t attr); +void tzc_region_enable(uint8_t region); +void tzc_security_inversion_en(vaddr_t base); +void tzc_set_action(enum tzc_action action); +uint32_t tzc_get_action(void); +void tzc_fail_dump(void); +void tzc_int_clear(void); +int tzc_auto_configure(vaddr_t addr, vaddr_t rsize, uint32_t attr, + uint8_t region); +TEE_Result tzc_regions_lockdown(void); + +#if TRACE_LEVEL >= TRACE_DEBUG +void tzc_dump_state(void); +#else +static inline void tzc_dump_state(void) +{ +} +#endif + +#endif /* __DRIVERS_TZC400_H */ diff --git a/optee_os/core/include/drivers/tzc400.h b/optee_os/core/include/drivers/tzc400.h new file mode 100644 index 0000000..74ba57e --- /dev/null +++ b/optee_os/core/include/drivers/tzc400.h @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: (BSD-2-Clause AND BSD-3-Clause) */ +/* + * Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +/* + * Copyright (c) 2014, ARM Limited and Contributors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of ARM nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DRIVERS_TZC400_H +#define __DRIVERS_TZC400_H + +#include +#include +#include +#include +#include + +#define TZC400_REG_SIZE U(0x1000) + +#define BUILD_CONFIG_OFF U(0x000) +#define ACTION_OFF U(0x004) +#define GATE_KEEPER_OFF U(0x008) +#define SPECULATION_CTRL_OFF U(0x00c) +#define INT_STATUS U(0x010) +#define INT_CLEAR U(0x014) + +#define FAIL_ADDRESS_LOW_OFF U(0x020) +#define FAIL_ADDRESS_HIGH_OFF U(0x024) +#define FAIL_CONTROL_OFF U(0x028) +#define FAIL_ID_OFF U(0x02c) +#define FAIL_FILTER_OFF(idx) (U(0x10) * (idx)) + +#define FAIL_ADDRESS_LOW(idx) (FAIL_ADDRESS_LOW_OFF + FAIL_FILTER_OFF(idx)) +#define FAIL_ADDRESS_HIGH(idx) (FAIL_ADDRESS_HIGH_OFF + FAIL_FILTER_OFF(idx)) +#define FAIL_CONTROL(idx) (FAIL_CONTROL_OFF + FAIL_FILTER_OFF(idx)) +#define FAIL_ID(idx) (FAIL_ID_OFF + FAIL_FILTER_OFF(idx)) + +#define REGION_BASE_LOW_OFF U(0x100) +#define REGION_BASE_HIGH_OFF U(0x104) +#define REGION_TOP_LOW_OFF U(0x108) +#define REGION_TOP_HIGH_OFF U(0x10c) +#define REGION_ATTRIBUTES_OFF U(0x110) +#define REGION_ID_ACCESS_OFF U(0x114) +#define REGION_NUM_OFF(region) (U(0x20) * (region)) + +/* ID Registers */ +#define PID0_OFF U(0xfe0) +#define PID1_OFF U(0xfe4) +#define PID2_OFF U(0xfe8) +#define PID3_OFF U(0xfec) +#define PID4_OFF U(0xfd0) +#define PID5_OFF U(0xfd4) +#define PID6_OFF U(0xfd8) +#define PID7_OFF U(0xfdc) +#define CID0_OFF U(0xff0) +#define CID1_OFF U(0xff4) +#define CID2_OFF U(0xff8) +#define CID3_OFF U(0xffc) + +#define BUILD_CONFIG_NF_SHIFT U(24) +#define BUILD_CONFIG_NF_MASK U(0x3) +#define BUILD_CONFIG_AW_SHIFT U(8) +#define BUILD_CONFIG_AW_MASK U(0x3f) +#define BUILD_CONFIG_NR_SHIFT U(0) +#define BUILD_CONFIG_NR_MASK U(0x1f) + +/* Not describing the case where regions 1 to 8 overlap */ +#define ACTION_RV_SHIFT U(0) +#define ACTION_RV_MASK U(0x3) +#define ACTION_RV_LOWOK U(0x0) +#define ACTION_RV_LOWERR U(0x1) +#define ACTION_RV_HIGHOK U(0x2) +#define ACTION_RV_HIGHERR U(0x3) + +/* + * Number of gate keepers is implementation defined. But we know the max for + * this device is 4. Get implementation details from BUILD_CONFIG. + */ +#define GATE_KEEPER_OS_SHIFT U(16) +#define GATE_KEEPER_OS_MASK U(0xf) +#define GATE_KEEPER_OR_SHIFT U(0) +#define GATE_KEEPER_OR_MASK U(0xf) +#define GATE_KEEPER_FILTER_MASK U(0x1) + +/* Speculation is enabled by default. */ +#define SPECULATION_CTRL_WRITE_DISABLE BIT(1) +#define SPECULATION_CTRL_READ_DISABLE BIT(0) + +/* Max number of filters allowed is 4. */ +#define INT_STATUS_OVERLAP_SHIFT U(16) +#define INT_STATUS_OVERLAP_MASK U(0xf) +#define INT_STATUS_OVERRUN_SHIFT U(8) +#define INT_STATUS_OVERRUN_MASK U(0xf) +#define INT_STATUS_STATUS_SHIFT U(0) +#define INT_STATUS_STATUS_MASK U(0xf) + +#define INT_CLEAR_CLEAR_SHIFT U(0) +#define INT_CLEAR_CLEAR_MASK U(0xf) + +/* If set write access, else read access */ +#define FAIL_CONTROL_DIRECTION_WRITE BIT(24) +/* If set non-secure access, else secure access */ +#define FAIL_CONTROL_NONSECURE BIT(21) +/* If set privileged access, else unprivileged access */ +#define FAIL_CONTROL_PRIVILEGED BIT(20) + +/* + * FAIL_ID_ID_MASK depends on AID_WIDTH which is platform specific. + * Platform should provide the value on initialisation. + */ +#define FAIL_ID_VNET_SHIFT U(24) +#define FAIL_ID_VNET_MASK U(0xf) +#define FAIL_ID_ID_SHIFT U(0) + +/* Used along with 'enum tzc_region_attributes' below */ +#define REG_ATTR_SEC_SHIFT U(30) +#define REG_ATTR_F_EN_SHIFT U(0) +#define REG_ATTR_F_EN_MASK U(0xf) +#define REG_ATTR_FILTER_BIT(x) SHIFT_U32(BIT(x), REG_ATTR_F_EN_SHIFT) +#define REG_ATTR_FILTER_BIT_ALL SHIFT_U32(REG_ATTR_F_EN_MASK, \ + REG_ATTR_F_EN_SHIFT) + +#define REGION_ID_ACCESS_NSAID_WR_EN_SHIFT U(16) +#define REGION_ID_ACCESS_NSAID_RD_EN_SHIFT U(0) +#define REGION_ID_ACCESS_NSAID_ID_MASK U(0xf) + + +/* Macros for setting Region ID access permissions based on NSAID */ +#define TZC_REGION_ACCESS_RD(id) \ + SHIFT_U32(BIT(id & REGION_ID_ACCESS_NSAID_ID_MASK), \ + REGION_ID_ACCESS_NSAID_RD_EN_SHIFT) +#define TZC_REGION_ACCESS_WR(id) \ + SHIFT_U32(BIT(id & REGION_ID_ACCESS_NSAID_ID_MASK), \ + REGION_ID_ACCESS_NSAID_WR_EN_SHIFT) +#define TZC_REGION_ACCESS_RDWR(id) \ + (TZC_REGION_ACCESS_RD(id) | TZC_REGION_ACCESS_WR(id)) + +/* Filters are bit mapped 0 to 3. */ +#define TZC400_COMPONENT_ID U(0xb105f00d) + +/******************************************************************************* + * Function & variable prototypes + ******************************************************************************/ + +/* + * What type of action is expected when an access violation occurs. + * The memory requested is zeroed. But we can also raise and event to + * let the system know it happened. + * We can raise an interrupt(INT) and/or cause an exception(ERR). + * TZC_ACTION_NONE - No interrupt, no Exception + * TZC_ACTION_ERR - No interrupt, raise exception -> sync external + * data abort + * TZC_ACTION_INT - Raise interrupt, no exception + * TZC_ACTION_ERR_INT - Raise interrupt, raise exception -> sync + * external data abort + */ +enum tzc_action { + TZC_ACTION_NONE = 0, + TZC_ACTION_ERR = 1, + TZC_ACTION_INT = 2, + TZC_ACTION_ERR_INT = (TZC_ACTION_ERR | TZC_ACTION_INT) +}; + +/* + * Controls secure access to a region. If not enabled secure access is not + * allowed to region. + */ +enum tzc_region_attributes { + TZC_REGION_S_NONE = 0, + TZC_REGION_S_RD = 1, + TZC_REGION_S_WR = 2, + TZC_REGION_S_RDWR = (TZC_REGION_S_RD | TZC_REGION_S_WR) +}; + +struct tzc_region_config { + uint32_t filters; + vaddr_t base; + vaddr_t top; + enum tzc_region_attributes sec_attr; + uint32_t ns_device_access; +}; + +void tzc_init(vaddr_t base); +void tzc_configure_region(uint8_t region, const struct tzc_region_config *cfg); +TEE_Result tzc_get_region_config(uint8_t region, struct tzc_region_config *cfg); +void tzc_enable_filters(void); +void tzc_disable_filters(void); +void tzc_set_action(enum tzc_action action); + +void tzc_fail_dump(void); +void tzc_int_clear(void); + +#if TRACE_LEVEL >= TRACE_DEBUG +void tzc_dump_state(void); +#else +static inline void tzc_dump_state(void) +{ +} +#endif + +#endif /* __DRIVERS_TZC400_H */ diff --git a/optee_os/core/include/drivers/versal_gpio.h b/optee_os/core/include/drivers/versal_gpio.h new file mode 100644 index 0000000..3070cc1 --- /dev/null +++ b/optee_os/core/include/drivers/versal_gpio.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022 Foundries.io Ltd + */ + +#ifndef __DRIVERS_VERSAL_GPIO_H +#define __DRIVERS_VERSAL_GPIO_H + +#include +#include +#include +#include + +#define VERSAL_GPIO_MAX_BANK 5 + +struct versal_gpio_platform_data { + const char *label; + uint16_t ngpio; + uint32_t max_bank; + uint32_t bank_min[VERSAL_GPIO_MAX_BANK]; + uint32_t bank_max[VERSAL_GPIO_MAX_BANK]; +}; + +struct versal_gpio_platdata { + paddr_t base; + const struct versal_gpio_platform_data *p_data; +}; + +struct versal_gpio_chip { + struct gpio_chip chip; + struct versal_gpio_platdata plat; + vaddr_t base; +}; + +TEE_Result versal_gpio_pmc_init(struct versal_gpio_chip *chip); +TEE_Result versal_gpio_ps_init(struct versal_gpio_chip *chip); + +#endif /* __DRIVERS_VERSAL_GPIO_H */ diff --git a/optee_os/core/include/drivers/versal_mbox.h b/optee_os/core/include/drivers/versal_mbox.h new file mode 100644 index 0000000..04e0c09 --- /dev/null +++ b/optee_os/core/include/drivers/versal_mbox.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Foundries.io Ltd + */ +#ifndef __DRIVERS_VERSAL_MBOX_H +#define __DRIVERS_VERSAL_MBOX_H + +#include +#include +#include + +#define VERSAL_MAX_IPI_BUF 7 + +struct versal_mbox_mem { + size_t alloc_len; + size_t len; + void *buf; +}; + +struct versal_ipi_buf { + struct versal_mbox_mem mem; + bool only_cache; +}; + +struct versal_ipi_cmd { + uint32_t data[8]; + struct versal_ipi_buf ibuf[VERSAL_MAX_IPI_BUF]; +}; + +TEE_Result versal_mbox_notify(struct versal_ipi_cmd *cmd, + struct versal_ipi_cmd *rsp, uint32_t *err); +TEE_Result versal_mbox_alloc(size_t len, const void *init, + struct versal_mbox_mem *mem); +#endif /* __DRIVERS_VERSAL_MBOX_H */ diff --git a/optee_os/core/include/drivers/versal_nvm.h b/optee_os/core/include/drivers/versal_nvm.h new file mode 100644 index 0000000..948ebb2 --- /dev/null +++ b/optee_os/core/include/drivers/versal_nvm.h @@ -0,0 +1,273 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022 Foundries.io Ltd + */ + +#ifndef __DRIVERS_VERSAL_NVM_H__ +#define __DRIVERS_VERSAL_NVM_H__ + +#include +#include +#include +#include +#include + +#define PUF_EFUSES_WORDS (128) +#define PUF_SYN_DATA_WORDS (127) +#define EFUSE_MAX_USER_FUSES (64) + +#define EFUSE_OFFCHIP_REVOCATION_ID_LEN (4) +#define EFUSE_REVOCATION_ID_LEN (4) +#define EFUSE_DEC_ONLY_LEN (4) +#define EFUSE_DNA_LEN (16) +#define EFUSE_PPK_LEN (32) +#define EFUSE_IV_LEN (12) + +enum versal_nvm_iv_type { + EFUSE_META_HEADER_IV_RANGE = 0, + EFUSE_BLACK_IV, + EFUSE_PLM_IV_RANGE, + EFUSE_DATA_PARTITION_IV_RANGE, +}; + +enum versal_nvm_ppk_type { + EFUSE_PPK0 = 0, + EFUSE_PPK1, + EFUSE_PPK2 +}; + +enum versal_nvm_revocation_id { + EFUSE_REVOCATION_ID_0 = 0, + EFUSE_REVOCATION_ID_1, + EFUSE_REVOCATION_ID_2, + EFUSE_REVOCATION_ID_3, + EFUSE_REVOCATION_ID_4, + EFUSE_REVOCATION_ID_5, + EFUSE_REVOCATION_ID_6, + EFUSE_REVOCATION_ID_7 +}; + +enum versal_nvm_offchip_id { + EFUSE_INVLD = -1, + EFUSE_OFFCHIP_REVOKE_ID_0 = 0, + EFUSE_OFFCHIP_REVOKE_ID_1, + EFUSE_OFFCHIP_REVOKE_ID_2, + EFUSE_OFFCHIP_REVOKE_ID_3, + EFUSE_OFFCHIP_REVOKE_ID_4, + EFUSE_OFFCHIP_REVOKE_ID_5, + EFUSE_OFFCHIP_REVOKE_ID_6, + EFUSE_OFFCHIP_REVOKE_ID_7 +}; + +/* + * All structures mapped to the PLM processor must be address_and_size aligned + * to the cacheline_len. + */ + +struct versal_efuse_glitch_cfg_bits { + uint8_t prgm_glitch; + uint8_t glitch_det_wr_lk; + uint32_t glitch_det_trim; + uint8_t gd_rom_monitor_en; + uint8_t gd_halt_boot_en; + uint8_t pad[53]; +}; + +struct versal_efuse_aes_keys { + uint8_t prgm_aes_key; + uint8_t prgm_user_key0; + uint8_t prgm_user_key1; + uint32_t aes_key[8]; + uint32_t user_key0[8]; + uint32_t user_key1[8]; + uint8_t pad[25]; +}; + +struct versal_efuse_ppk_hash { + uint8_t prgm_ppk0_hash; + uint8_t prgm_ppk1_hash; + uint8_t prgm_ppk2_hash; + uint32_t ppk0_hash[8]; + uint32_t ppk1_hash[8]; + uint32_t ppk2_hash[8]; + uint8_t pad[89]; +}; + +struct versal_efuse_dec_only { + uint8_t prgm_dec_only; + uint8_t pad[63]; +}; + +struct versal_efuse_revoke_ids { + uint8_t prgm_revoke_id; + uint32_t revoke_id[8]; + uint8_t pad[89]; +}; + +struct versal_efuse_offchip_ids { + uint8_t prgm_offchip_id; + uint32_t offchip_id[8]; + uint8_t pad[89]; +}; + +struct versal_efuse_user_data { + uint32_t start; + uint32_t num; + uint64_t addr; + uint8_t pad[48]; +}; + +struct versal_efuse_puf_fuse { + uint8_t env_monitor_dis; + uint8_t prgm_puf_fuse; + uint32_t start; + uint32_t num; + uint64_t addr; + uint8_t pad[104]; +}; + +struct versal_efuse_ivs { + uint8_t prgm_meta_header_iv; + uint8_t prgm_blk_obfus_iv; + uint8_t prgm_plm_iv; + uint8_t prgm_data_partition_iv; + uint32_t meta_header_iv[3]; + uint32_t blk_obfus_iv[3]; + uint32_t plm_iv[3]; + uint32_t data_partition_iv[3]; + uint8_t pad[12]; +}; + +struct versal_efuse_misc_ctrl_bits { + uint8_t glitch_det_halt_boot_en; + uint8_t glitch_det_rom_monitor_en; + uint8_t halt_boot_error; + uint8_t halt_boot_env; + uint8_t crypto_kat_en; + uint8_t lbist_en; + uint8_t safety_mission_en; + uint8_t ppk0_invalid; + uint8_t ppk1_invalid; + uint8_t ppk2_invalid; + uint8_t pad[54]; +}; + +struct versal_efuse_puf_sec_ctrl_bits { + uint8_t puf_regen_dis; + uint8_t puf_hd_invalid; + uint8_t puf_test2_dis; + uint8_t puf_dis; + uint8_t puf_syn_lk; + uint8_t pad[59]; +}; + +struct versal_efuse_sec_misc1_bits { + uint8_t lpd_mbist_en; + uint8_t pmc_mbist_en; + uint8_t lpd_noc_sc_en; + uint8_t sysmon_volt_mon_en; + uint8_t sysmon_temp_mon_en; + uint8_t pad[59]; +}; + +struct versal_efuse_boot_env_ctrl_bits { + uint8_t prgm_sysmon_temp_hot; + uint8_t prgm_sysmon_volt_pmc; + uint8_t prgm_sysmon_volt_pslp; + uint8_t prgm_sysmon_temp_cold; + uint8_t sysmon_temp_en; + uint8_t sysmon_volt_en; + uint8_t sysmon_volt_soc; + uint8_t sysmon_temp_hot; + uint8_t sysmon_volt_pmc; + uint8_t sysmon_volt_pslp; + uint8_t sysmon_temp_cold; + uint8_t pad[53]; +}; + +struct versal_efuse_sec_ctrl_bits { + uint8_t aes_dis; + uint8_t jtag_err_out_dis; + uint8_t jtag_dis; + uint8_t ppk0_wr_lk; + uint8_t ppk1_wr_lk; + uint8_t ppk2_wr_lk; + uint8_t aes_crc_lk; + uint8_t aes_wr_lk; + uint8_t user_key0_crc_lk; + uint8_t user_key0_wr_lk; + uint8_t user_key1_crc_lk; + uint8_t user_key1_wr_lk; + uint8_t sec_dbg_dis; + uint8_t sec_lock_dbg_dis; + uint8_t boot_env_wr_lk; + uint8_t reg_init_dis; + uint8_t pad[48]; +}; + +struct versal_efuse_puf_header { + struct versal_efuse_puf_sec_ctrl_bits sec_ctrl; + uint8_t prmg_puf_helper_data; + uint8_t env_monitor_dis; + uint32_t efuse_syn_data[PUF_SYN_DATA_WORDS]; + uint32_t chash; + uint32_t aux; + uint8_t pad[56]; +}; + +struct versal_efuse_puf_user_fuse { + uint32_t data_addr[PUF_EFUSES_WORDS]; + uint8_t env_monitor_dis; + uint8_t prgm_puf_fuse; + uint32_t start_row; + uint32_t num_rows; +}; + +TEE_Result versal_efuse_read_dna(uint32_t *buf, size_t len); +TEE_Result versal_efuse_read_user_data(uint32_t *buf, size_t len, + uint32_t first, size_t num); +TEE_Result versal_efuse_read_iv(uint32_t *buf, size_t len, + enum versal_nvm_iv_type type); +TEE_Result versal_efuse_read_ppk(uint32_t *buf, size_t len, + enum versal_nvm_ppk_type type); +TEE_Result versal_efuse_write_user_data(uint32_t *buf, size_t len, + uint32_t first, size_t num); +TEE_Result versal_efuse_write_aes_keys(struct versal_efuse_aes_keys *keys); +TEE_Result versal_efuse_write_ppk_hash(struct versal_efuse_ppk_hash *hash); +TEE_Result versal_efuse_write_iv(struct versal_efuse_ivs *p); +TEE_Result versal_efuse_write_dec_only(struct versal_efuse_dec_only *p); +TEE_Result versal_efuse_write_sec(struct versal_efuse_sec_ctrl_bits *p); +TEE_Result versal_efuse_write_misc(struct versal_efuse_misc_ctrl_bits *p); +TEE_Result versal_efuse_write_glitch_cfg(struct versal_efuse_glitch_cfg_bits + *p); +TEE_Result versal_efuse_write_boot_env(struct versal_efuse_boot_env_ctrl_bits + *p); +TEE_Result versal_efuse_write_sec_misc1(struct versal_efuse_sec_misc1_bits *p); +TEE_Result versal_efuse_write_offchip_ids(struct versal_efuse_offchip_ids *p); +TEE_Result versal_efuse_write_revoke_ppk(enum versal_nvm_ppk_type type); +TEE_Result versal_efuse_write_revoke_id(uint32_t id); +TEE_Result versal_efuse_read_revoke_id(uint32_t *buf, size_t len, + enum versal_nvm_revocation_id id); +TEE_Result versal_efuse_read_misc_ctrl(struct versal_efuse_misc_ctrl_bits *buf); +TEE_Result versal_efuse_read_sec_ctrl(struct versal_efuse_sec_ctrl_bits *buf); +TEE_Result versal_efuse_read_sec_misc1(struct versal_efuse_sec_misc1_bits *buf); +TEE_Result +versal_efuse_read_boot_env_ctrl(struct versal_efuse_boot_env_ctrl_bits *buf); +TEE_Result versal_efuse_read_offchip_revoke_id(uint32_t *buf, size_t len, + enum versal_nvm_offchip_id id); +TEE_Result versal_efuse_read_dec_only(uint32_t *buf, size_t len); +TEE_Result versal_efuse_read_puf_sec_ctrl(struct versal_efuse_puf_sec_ctrl_bits + *buf); +TEE_Result versal_efuse_read_puf(struct versal_efuse_puf_header *buf); +TEE_Result versal_efuse_read_puf_as_user_fuse(struct versal_efuse_puf_user_fuse + *p); +TEE_Result versal_efuse_write_puf_as_user_fuse(struct versal_efuse_puf_user_fuse + *p); +TEE_Result versal_efuse_write_puf(struct versal_efuse_puf_header *buf); +TEE_Result versal_bbram_write_aes_key(uint8_t *key, size_t len); +TEE_Result versal_bbram_zeroize(void); +TEE_Result versal_bbram_write_user_data(uint32_t data); +TEE_Result versal_bbram_read_user_data(uint32_t *data); +TEE_Result versal_bbram_lock_write_user_data(void); + +#endif /*__DRIVERS_VERSAL_NVM_H__*/ diff --git a/optee_os/core/include/drivers/versal_pm.h b/optee_os/core/include/drivers/versal_pm.h new file mode 100644 index 0000000..0a1bd5e --- /dev/null +++ b/optee_os/core/include/drivers/versal_pm.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022 Foundries.io Ltd + */ + +#ifndef __DRIVERS_VERSAL_PM_H__ +#define __DRIVERS_VERSAL_PM_H__ + +#include +#include + +TEE_Result versal_soc_version(uint8_t *version); +TEE_Result versal_write_fpga(paddr_t pa); + +#endif /*__DRIVERS_VERSAL_PM_H__*/ diff --git a/optee_os/core/include/drivers/versal_puf.h b/optee_os/core/include/drivers/versal_puf.h new file mode 100644 index 0000000..06811b3 --- /dev/null +++ b/optee_os/core/include/drivers/versal_puf.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022 Foundries.io Ltd + * Jorge Ramirez-Ortiz + */ + +#ifndef __DRIVERS_VERSAL_PUF_H +#define __DRIVERS_VERSAL_PUF_H + +#include +#include +#include +#include + +#define VERSAL_PUF_REGISTRATION 0x0 +#define VERSAL_PUF_REGEN_ON_DEMAND 0x1 +#define VERSAL_PUF_REGEN_ID_ONLY 0x2 +#define VERSAL_PUF_SHUTTER_VALUE 0x81000100 +#define VERSAL_PUF_SYNDROME_MODE_4K 0x0 +#define VERSAL_PUF_GLBL_VAR_FLTR_OPTION 1 +#define VERSAL_PUF_READ_FROM_RAM 0 +#define VERSAL_PUF_READ_FROM_EFUSE_CACHE 1 +#define VERSAL_PUF_4K_PUF_SYN_LEN_IN_WORDS 140 + +#define VERSAL_PUF_EFUSE_SYN_WORDS 127 +#define VERSAL_PUF_SYNDROME_WORDS 350 +#define VERSAL_PUF_ID_WORDS 8 +#define VERSAL_PUF_HASH_LEN 4 +#define VERSAL_PUF_AUX_LEN 4 + +struct versal_puf_data { + uint32_t syndrome_data[VERSAL_PUF_SYNDROME_WORDS]; + uint32_t chash; + uint32_t aux; + uint32_t puf_id[VERSAL_PUF_ID_WORDS]; + uint32_t efuse_syn_data[VERSAL_PUF_EFUSE_SYN_WORDS]; +}; + +struct versal_puf_cfg { + uint8_t reg_mode; + uint8_t puf_operation; + uint8_t global_var_filter; + uint8_t read_option; + uint32_t shutter_value; +}; + +struct versal_puf_data_req { + uint8_t reg_mode; + uint8_t puf_operation; + uint8_t global_var_filter; + uint8_t read_option; + uint32_t shutter_value; + uint64_t syndrome_data_addr; + uint64_t hash_addr; + uint64_t aux_addr; + uint64_t puf_id_addr; + uint64_t syndrome_addr; + uint64_t efuse_syn_data_addr; + uint8_t pad[8]; +}; + +enum versal_puf_api { + VERSAL_PUF_API_FEATURES = 0U, + VERSAL_PUF_REGISTER, + VERSAL_PUF_REGENERATE, + VERSAL_PUF_CLEAR_ID, +}; + +#define __aligned_puf __aligned(CACHELINE_LEN) + +TEE_Result versal_puf_regenerate(struct versal_puf_data *buf, + struct versal_puf_cfg *cfg); +TEE_Result versal_puf_register(struct versal_puf_data *buf, + struct versal_puf_cfg *cfg); +TEE_Result versal_puf_check_api(enum versal_puf_api id); +TEE_Result versal_puf_clear_id(void); + +#endif /* __DRIVERS_VERSAL_PUF_H */ diff --git a/optee_os/core/include/drivers/versal_sha3_384.h b/optee_os/core/include/drivers/versal_sha3_384.h new file mode 100644 index 0000000..8ce99e5 --- /dev/null +++ b/optee_os/core/include/drivers/versal_sha3_384.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022 Foundries.io Ltd + */ + +#ifndef __DRIVERS_VERSAL_SHA3_384_H__ +#define __DRIVERS_VERSAL_SHA3_384_H__ + +#include +#include +#include +#include + +TEE_Result versal_sha3_384(const uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len); + +#endif /*__DRIVERS_VERSAL_SHA3_384_H__*/ diff --git a/optee_os/core/include/drivers/wdt.h b/optee_os/core/include/drivers/wdt.h new file mode 100644 index 0000000..b07152e --- /dev/null +++ b/optee_os/core/include/drivers/wdt.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 Broadcom. + */ + +#ifndef DRIVERS_WDT_H +#define DRIVERS_WDT_H + +#include +#include +#include +#include +#include + +struct wdt_chip { + const struct wdt_ops *ops; + struct itr_handler *wdt_itr; +}; + +/* + * struct wdt_ops - The watchdog device operations + * + * @init: The routine to initialized the watchdog resources. + * @start: The routine for starting the watchdog device. + * @stop: The routine for stopping the watchdog device. + * @ping: The routine that sends a keepalive ping to the watchdog device. + * @set_timeout:The routine that finds the load value that will reset system in + * required timeout (in seconds). + * + * The wdt_ops structure contains a list of low-level operations + * that control a watchdog device. + */ +struct wdt_ops { + TEE_Result (*init)(struct wdt_chip *chip, unsigned long *min_timeout, + unsigned long *max_timeout); + void (*start)(struct wdt_chip *chip); + void (*stop)(struct wdt_chip *chip); + void (*ping)(struct wdt_chip *chip); + TEE_Result (*set_timeout)(struct wdt_chip *chip, unsigned long timeout); +}; + +#ifdef CFG_WDT +extern struct wdt_chip *wdt_chip; + +/* Register a watchdog as the system watchdog */ +TEE_Result watchdog_register(struct wdt_chip *chip); + +static inline +TEE_Result watchdog_init(unsigned long *min_timeout, unsigned long *max_timeout) +{ + if (!wdt_chip) + return TEE_ERROR_NOT_SUPPORTED; + + if (!wdt_chip->ops->init) + return TEE_SUCCESS; + + return wdt_chip->ops->init(wdt_chip, min_timeout, max_timeout); +} + +static inline void watchdog_start(void) +{ + if (wdt_chip) + wdt_chip->ops->start(wdt_chip); +} + +static inline void watchdog_stop(void) +{ + if (wdt_chip && wdt_chip->ops->stop) + wdt_chip->ops->stop(wdt_chip); +} + +static inline void watchdog_ping(void) +{ + if (wdt_chip) + wdt_chip->ops->ping(wdt_chip); +} + +static inline void watchdog_settimeout(unsigned long timeout) +{ + if (wdt_chip) + wdt_chip->ops->set_timeout(wdt_chip, timeout); +} +#else +static inline TEE_Result watchdog_register(struct wdt_chip *chip __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result watchdog_init(unsigned long *min_timeout __unused, + unsigned long *max_timeout __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline void watchdog_start(void) {} +static inline void watchdog_stop(void) {} +static inline void watchdog_ping(void) {} +static inline void watchdog_settimeout(unsigned long timeout __unused) {} +#endif + +#ifdef CFG_WDT_SM_HANDLER +enum sm_handler_ret __wdt_sm_handler(struct thread_smc_args *args); + +static inline +enum sm_handler_ret wdt_sm_handler(struct thread_smc_args *args) +{ + if (args->a0 != CFG_WDT_SM_HANDLER_ID) + return SM_HANDLER_PENDING_SMC; + + return __wdt_sm_handler(args); +} +#else +static inline +enum sm_handler_ret wdt_sm_handler(struct thread_smc_args *args __unused) +{ + return SM_HANDLER_PENDING_SMC; +} +#endif + +#endif /* DRIVERS_WDT_H */ diff --git a/optee_os/core/include/drivers/zynqmp_csu.h b/optee_os/core/include/drivers/zynqmp_csu.h new file mode 100644 index 0000000..985689b --- /dev/null +++ b/optee_os/core/include/drivers/zynqmp_csu.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2021 + * Author: Jorge Ramirez + */ + +#ifndef __DRIVERS_ZYNQMP_CSU_H_ +#define __DRIVERS_ZYNQMP_CSU_H_ + +/* CSU registers */ +#define ZYNQMP_CSU_STATUS_OFFSET 0x00 +#define ZYNQMP_CSU_CTRL_OFFSET 0x04 +#define ZYNQMP_CSU_SSS_CFG_OFFSET 0x08 +#define ZYNQMP_CSU_DMA_RESET_OFFSET 0x0c +#define ZYNQMP_CSU_MULTI_BOOT_OFFSET 0x10 +#define ZYNQMP_CSU_TAMPER_TRIG_OFFSET 0x14 +#define ZYNQMP_CSU_FT_STATUS_OFFSET 0x18 +#define ZYNQMP_CSU_ISR_OFFSET 0x20 + +#define ZYNQMP_CSU_STATUS_AUTH BIT(0) +#define ZYNQMP_CSU_SSS_DMA0_STREAM_TO_AES 0x5A0 +#define ZYNQMP_CSU_DMA_RESET_SET 1 +#define ZYNQMP_CSU_DMA_RESET_CLR 0 +#define ZYNQMP_CSU_ISR_PUF_ACC_ERROR_MASK BIT(12) + +/* AES-GCM */ +#define ZYNQMP_CSU_AES_BASE (CSU_BASE + 0x1000) +#define ZYNQMP_CSU_AES_SIZE 0x1000 + +/* SHA */ +#define ZYNQMP_CSU_SHA_BASE (CSU_BASE + 0x2000) +#define ZYNQMP_CSU_SHA_SIZE 0x1000 + +/* PCAP */ +#define ZYNQMP_CSU_PCAP_BASE (CSU_BASE + 0x3000) +#define ZYNQMP_CSU_PCAP_SIZE 0x1000 + +/* PUF */ +#define ZYNQMP_CSU_PUF_BASE (CSU_BASE + 0x4000) +#define ZYNQMP_CSU_PUF_SIZE 0x1000 + +/* TAMPER */ +#define ZYNQMP_CSU_TAMPER_BASE (CSU_BASE + 0x5000) +#define ZYNQMP_CSU_TAMPER_SIZE 0x38 + +#endif diff --git a/optee_os/core/include/drivers/zynqmp_csu_aes.h b/optee_os/core/include/drivers/zynqmp_csu_aes.h new file mode 100644 index 0000000..3a1a2e5 --- /dev/null +++ b/optee_os/core/include/drivers/zynqmp_csu_aes.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2021 + * Author: Jorge Ramirez + */ + +#ifndef __DRIVERS_ZYNQMP_CSU_AES_H_ +#define __DRIVERS_ZYNQMP_CSU_AES_H_ + +#include +#include +#include + +#define __aligned_csuaes __aligned_csudma + +#define ZYNQMP_GCM_TAG_SIZE 16 +#define ZYNQMP_GCM_IV_SIZE 12 + +enum zynqmp_csu_key { + ZYNQMP_CSU_AES_KEY_SRC_KUP = 0, + ZYNQMP_CSU_AES_KEY_SRC_DEV +}; + +#define ZYNQMP_CSU_AES_DST_LEN(x) ((x) + ZYNQMP_GCM_TAG_SIZE) + +TEE_Result zynqmp_csu_aes_encrypt_data(const void *src, size_t src_len, + void *dst, size_t dst_len, + void *tag, size_t tag_len, + const void *iv, size_t iv_len, + enum zynqmp_csu_key key); + +TEE_Result zynqmp_csu_aes_decrypt_data(const void *src, size_t src_len, + void *dst, size_t len, + const void *tag, size_t tag_len, + const void *iv, size_t iv_len, + enum zynqmp_csu_key key); + +TEE_Result zynqmp_csu_aes_dt_enable_secure_status(void); + +#endif diff --git a/optee_os/core/include/drivers/zynqmp_csu_puf.h b/optee_os/core/include/drivers/zynqmp_csu_puf.h new file mode 100644 index 0000000..0bdc0d3 --- /dev/null +++ b/optee_os/core/include/drivers/zynqmp_csu_puf.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2021 + * Author: Jorge Ramirez + */ + +#ifndef __DRIVERS_ZYNQMP_CSU_PUF_H_ +#define __DRIVERS_ZYNQMP_CSU_PUF_H_ + +#include + +TEE_Result zynqmp_csu_puf_regenerate(void); +void zynqmp_csu_puf_reset(void); + +#endif diff --git a/optee_os/core/include/drivers/zynqmp_csudma.h b/optee_os/core/include/drivers/zynqmp_csudma.h new file mode 100644 index 0000000..afbf51c --- /dev/null +++ b/optee_os/core/include/drivers/zynqmp_csudma.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) Foundries Ltd. 2021 + * Author: Jorge Ramirez + */ + +#ifndef __DRIVERS_ZYNQMP_CSUDMA_H_ +#define __DRIVERS_ZYNQMP_CSUDMA_H_ + +#include +#include +#include + +#define ZYNQMP_CSUDMA_ALIGN 64 +#define __aligned_csudma __aligned(ZYNQMP_CSUDMA_ALIGN) + +#define ZYNQMP_CSUDMA_MIN_SIZE 16 +#define ZYNQMP_CSUDMA_DONE BIT(0) + +enum zynqmp_csudma_channel { + ZYNQMP_CSUDMA_SRC_CHANNEL = 0, + ZYNQMP_CSUDMA_DST_CHANNEL +}; + +TEE_Result zynqmp_csudma_transfer(enum zynqmp_csudma_channel channel, + void *address, size_t len, uint8_t notify); +TEE_Result zynqmp_csudma_sync(enum zynqmp_csudma_channel channel); +TEE_Result zynqmp_csudma_prepare(void); +void zynqmp_csudma_unprepare(void); + +#endif diff --git a/optee_os/core/include/drivers/zynqmp_efuse.h b/optee_os/core/include/drivers/zynqmp_efuse.h new file mode 100644 index 0000000..4647e8b --- /dev/null +++ b/optee_os/core/include/drivers/zynqmp_efuse.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2021 Foundries.io Ltd + */ + +#ifndef __DRIVERS_ZYNQMP_EFUSE_H__ +#define __DRIVERS_ZYNQMP_EFUSE_H__ + +#include + +#define ZYNQMP_EFUSE_DNA_OFFSET 0xc +#define ZYNQMP_EFUSE_DNA_LENGTH 12 + +#define ZYNQMP_EFUSE_IP_DISABLE_OFFSET 0x18 +#define ZYNQMP_EFUSE_IP_DISABLE_LENGTH 4 + +#define ZYNQMP_EFUSE_USER0_OFFSET 0x20 +#define ZYNQMP_EFUSE_USER0_LENGTH 4 + +#define ZYNQMP_EFUSE_USER1_OFFSET 0x24 +#define ZYNQMP_EFUSE_USER1_LENGTH 4 + +#define ZYNQMP_EFUSE_USER2_OFFSET 0x28 +#define ZYNQMP_EFUSE_USER2_LENGTH 4 + +#define ZYNQMP_EFUSE_USER3_OFFSET 0x2c +#define ZYNQMP_EFUSE_USER3_LENGTH 4 + +#define ZYNQMP_EFUSE_USER4_OFFSET 0x30 +#define ZYNQMP_EFUSE_USER4_LENGTH 4 + +#define ZYNQMP_EFUSE_USER5_OFFSET 0x34 +#define ZYNQMP_EFUSE_USER5_LENGTH 4 + +#define ZYNQMP_EFUSE_USER6_OFFSET 0x38 +#define ZYNQMP_EFUSE_USER6_LENGTH 4 + +#define ZYNQMP_EFUSE_USER7_OFFSET 0x3c +#define ZYNQMP_EFUSE_USER7_LENGTH 4 + +#define ZYNQMP_EFUSE_MISC_USER_CTRL_OFFSET 0x40 +#define ZYNQMP_EFUSE_MISC_USER_CTRL_LENGTH 4 + +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_0 BIT(0) +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_1 BIT(1) +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_2 BIT(2) +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_3 BIT(3) +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_4 BIT(4) +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_5 BIT(5) +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_6 BIT(6) +#define ZYNQMP_EFUSE_MISC_USER_CTRL_USR_WRLK_7 BIT(7) + +#define ZYNQMP_EFUSE_SEC_CTRL_OFFSET 0x58 +#define ZYNQMP_EFUSE_SEC_CTRL_LENGTH 4 + +#endif /*__DRIVERS_ZYNQMP_EFUSE_H__*/ diff --git a/optee_os/core/include/drivers/zynqmp_huk.h b/optee_os/core/include/drivers/zynqmp_huk.h new file mode 100644 index 0000000..44396cd --- /dev/null +++ b/optee_os/core/include/drivers/zynqmp_huk.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022, Vaisala Oyj + */ + +#ifndef __DRIVERS_ZYNQMP_HUK_H__ +#define __DRIVERS_ZYNQMP_HUK_H__ + +/* + * Query Device DNA from the device + * + * Note: configured as weak so that it can be replaced with implementation to + * get read only PL Device DNA from FPGA array. This would need custom IP core + * for access. + * + * Both PS and PL Device DNA's have 96 bits. + * + * Xilinx recommends to use PL Device DNA for device identification. + * + * ref: + * + * 71342 - Zynq UltraScale+ Device - PS DNA is not write protected and is a + * different value than the PL DNA + * + * https://support.xilinx.com/s/article/71342 + * + * @device_dna: Memory buffer to receive value of Device DNA + * @size: length of device_dna buffer (requires buffer of 12 bytes) + * Return a TEE_Result compliant status + */ +TEE_Result tee_zynqmp_get_device_dna(uint8_t *device_dna, size_t size); + +#endif /* __DRIVERS_ZYNQMP_HUK_H__ */ diff --git a/optee_os/core/include/drivers/zynqmp_pm.h b/optee_os/core/include/drivers/zynqmp_pm.h new file mode 100644 index 0000000..62de2b1 --- /dev/null +++ b/optee_os/core/include/drivers/zynqmp_pm.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2021 Foundries.io Ltd + */ + +#ifndef __DRIVERS_ZYNQMP_PM_H__ +#define __DRIVERS_ZYNQMP_PM_H__ + +#include +#include +#include +#include + +/* + * Information about accessing eFuses and the Physically Uncloneable Function + * (PUF) Support can be found at + * https://www.xilinx.com/support/documentation/application_notes/xapp1319-zynq-usp-prog-nvm.pdf + */ +#define ZYNQMP_NONPUF_EFUSE 0 +#define ZYNQMP_PUF_EFUSE 1 + +/* List of eFuse identifiers */ +enum zynqmp_efuse_id { + DNA = 0, IP_DISABLE, USER0, USER1, USER2, USER3, USER4, USER5, USER6, + USER7, MISC_USER_CTRL, SEC_CTRL, +}; + +/* Valid bytes in the eFuse */ +#define ZYNQMP_EFUSE_LEN(_id) ZYNQMP_EFUSE_##_id##_LENGTH + +/* Memory required to access the eFuse */ +#define ZYNQMP_EFUSE_MEM(_id) (ROUNDUP(ZYNQMP_EFUSE_LEN(_id), CACHELINE_LEN)) + +/* Alignment required in the buffers used to read the eFuse */ +#define __aligned_efuse __aligned(CACHELINE_LEN) + +/* + * Read eFuse memory + * @buf: buffer, where eFuse date will be stored. The data is returned + * in LE byte order. + * @buf_sz: buffer size in bytes + * @id: eFuse identifier. + * @puf: is eFuse PUF, ZYNQMP_PUF_EFUSE/ZYNQMP_NONPUF_EFUSE + * Return a TEE_Result compliant status + */ +TEE_Result zynqmp_efuse_read(uint8_t *buf, size_t buf_sz, + enum zynqmp_efuse_id id, bool puf); + +/* + * Program eFuse memory + * @buf: buffer where eFuse data are stored in LE byte order. + * @buf_sz: buffer size in bytes + * @id: eFuse identifier. + * @puf: is eFuse PUF, ZYNQMP_PUF_EFUSE/ZYNQMP_NONPUF_EFUSE + * Return a TEE_Result compliant status + */ +TEE_Result zynqmp_efuse_write(uint8_t *buf, size_t buf_sz, + enum zynqmp_efuse_id id, bool puf); + +/* + * Read the SoC version number: + * Different eFuse bitfields carry different meaning depending on this version. + */ +TEE_Result zynqmp_soc_version(uint32_t *version); + +#endif /*__DRIVERS_ZYNQMP_PM_H__*/ diff --git a/optee_os/core/include/dt-bindings/clock/at91.h b/optee_os/core/include/dt-bindings/clock/at91.h new file mode 100644 index 0000000..79470e3 --- /dev/null +++ b/optee_os/core/include/dt-bindings/clock/at91.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later or BSD-3-Clause */ +/* + * Copyright (C) 2021 Microchip + * + * This header provides constants for AT91 pmc status. + * + * The constants defined in this header are being used in dts. + */ + +#ifndef _DT_BINDINGS_CLK_AT91_H +#define _DT_BINDINGS_CLK_AT91_H + +#define PMC_TYPE_CORE 0 +#define PMC_TYPE_SYSTEM 1 +#define PMC_TYPE_PERIPHERAL 2 +#define PMC_TYPE_GCK 3 +#define PMC_TYPE_PROGRAMMABLE 4 + +#define PMC_SLOW 0 +#define PMC_MCK 1 +#define PMC_UTMI 2 +#define PMC_MAIN 3 +#define PMC_MCK2 4 +#define PMC_I2S0_MUX 5 +#define PMC_I2S1_MUX 6 +#define PMC_PLLACK 7 +#define PMC_PLLBCK 8 +#define PMC_AUDIOPLLCK 9 +#define PMC_MCK_PRES 10 +#define PMC_AUDIOPLL_FRACCK 11 +#define PMC_USBCK 12 +#define PMC_SAMA5D2_CORE_CLK_COUNT 13 + +/* SAMA7G5 */ +#define PMC_CPUPLL (PMC_MAIN + 1) +#define PMC_SYSPLL (PMC_MAIN + 2) +#define PMC_DDRPLL (PMC_MAIN + 3) +#define PMC_IMGPLL (PMC_MAIN + 4) +#define PMC_BAUDPLL (PMC_MAIN + 5) +#define PMC_AUDIOPMCPLL (PMC_MAIN + 6) +#define PMC_AUDIOIOPLL (PMC_MAIN + 7) +#define PMC_ETHPLL (PMC_MAIN + 8) +#define PMC_CPU (PMC_MAIN + 9) + +#define AT91_SCMI_CLK_CORE_MCK 0 +#define AT91_SCMI_CLK_CORE_UTMI 1 +#define AT91_SCMI_CLK_CORE_MAIN 2 +#define AT91_SCMI_CLK_CORE_MCK2 3 +#define AT91_SCMI_CLK_CORE_I2S0_MUX 4 +#define AT91_SCMI_CLK_CORE_I2S1_MUX 5 +#define AT91_SCMI_CLK_CORE_PLLACK 6 +#define AT91_SCMI_CLK_CORE_PLLBCK 7 +#define AT91_SCMI_CLK_CORE_AUDIOPLLCK 8 +#define AT91_SCMI_CLK_CORE_MCK_PRES 9 + +#define AT91_SCMI_CLK_SYSTEM_DDRCK 10 +#define AT91_SCMI_CLK_SYSTEM_LCDCK 11 +#define AT91_SCMI_CLK_SYSTEM_UHPCK 12 +#define AT91_SCMI_CLK_SYSTEM_UDPCK 13 +#define AT91_SCMI_CLK_SYSTEM_PCK0 14 +#define AT91_SCMI_CLK_SYSTEM_PCK1 15 +#define AT91_SCMI_CLK_SYSTEM_PCK2 16 +#define AT91_SCMI_CLK_SYSTEM_ISCCK 17 + +#define AT91_SCMI_CLK_PERIPH_MACB0_CLK 18 +#define AT91_SCMI_CLK_PERIPH_TDES_CLK 19 +#define AT91_SCMI_CLK_PERIPH_MATRIX1_CLK 20 +#define AT91_SCMI_CLK_PERIPH_HSMC_CLK 21 +#define AT91_SCMI_CLK_PERIPH_PIOA_CLK 22 +#define AT91_SCMI_CLK_PERIPH_FLX0_CLK 23 +#define AT91_SCMI_CLK_PERIPH_FLX1_CLK 24 +#define AT91_SCMI_CLK_PERIPH_FLX2_CLK 25 +#define AT91_SCMI_CLK_PERIPH_FLX3_CLK 26 +#define AT91_SCMI_CLK_PERIPH_FLX4_CLK 27 +#define AT91_SCMI_CLK_PERIPH_UART0_CLK 28 +#define AT91_SCMI_CLK_PERIPH_UART1_CLK 29 +#define AT91_SCMI_CLK_PERIPH_UART2_CLK 30 +#define AT91_SCMI_CLK_PERIPH_UART3_CLK 31 +#define AT91_SCMI_CLK_PERIPH_UART4_CLK 32 +#define AT91_SCMI_CLK_PERIPH_TWI0_CLK 33 +#define AT91_SCMI_CLK_PERIPH_TWI1_CLK 34 +#define AT91_SCMI_CLK_PERIPH_SPI0_CLK 35 +#define AT91_SCMI_CLK_PERIPH_SPI1_CLK 36 +#define AT91_SCMI_CLK_PERIPH_TCB0_CLK 37 +#define AT91_SCMI_CLK_PERIPH_TCB1_CLK 38 +#define AT91_SCMI_CLK_PERIPH_PWM_CLK 39 +#define AT91_SCMI_CLK_PERIPH_ADC_CLK 40 +#define AT91_SCMI_CLK_PERIPH_UHPHS_CLK 41 +#define AT91_SCMI_CLK_PERIPH_UDPHS_CLK 42 +#define AT91_SCMI_CLK_PERIPH_SSC0_CLK 43 +#define AT91_SCMI_CLK_PERIPH_SSC1_CLK 44 +#define AT91_SCMI_CLK_PERIPH_TRNG_CLK 45 +#define AT91_SCMI_CLK_PERIPH_PDMIC_CLK 46 +#define AT91_SCMI_CLK_PERIPH_SECURAM_CLK 47 +#define AT91_SCMI_CLK_PERIPH_I2S0_CLK 48 +#define AT91_SCMI_CLK_PERIPH_I2S1_CLK 49 +#define AT91_SCMI_CLK_PERIPH_CAN0_CLK 50 +#define AT91_SCMI_CLK_PERIPH_CAN1_CLK 51 +#define AT91_SCMI_CLK_PERIPH_PTC_CLK 52 +#define AT91_SCMI_CLK_PERIPH_CLASSD_CLK 53 +#define AT91_SCMI_CLK_PERIPH_DMA0_CLK 54 +#define AT91_SCMI_CLK_PERIPH_DMA1_CLK 55 +#define AT91_SCMI_CLK_PERIPH_AES_CLK 56 +#define AT91_SCMI_CLK_PERIPH_AESB_CLK 57 +#define AT91_SCMI_CLK_PERIPH_SHA_CLK 58 +#define AT91_SCMI_CLK_PERIPH_MPDDR_CLK 59 +#define AT91_SCMI_CLK_PERIPH_MATRIX0_CLK 60 +#define AT91_SCMI_CLK_PERIPH_SDMMC0_HCLK 61 +#define AT91_SCMI_CLK_PERIPH_SDMMC1_HCLK 62 +#define AT91_SCMI_CLK_PERIPH_LCDC_CLK 63 +#define AT91_SCMI_CLK_PERIPH_ISC_CLK 64 +#define AT91_SCMI_CLK_PERIPH_QSPI0_CLK 65 +#define AT91_SCMI_CLK_PERIPH_QSPI1_CLK 66 + +#define AT91_SCMI_CLK_GCK_SDMMC0_GCLK 67 +#define AT91_SCMI_CLK_GCK_SDMMC1_GCLK 68 +#define AT91_SCMI_CLK_GCK_TCB0_GCLK 69 +#define AT91_SCMI_CLK_GCK_TCB1_GCLK 70 +#define AT91_SCMI_CLK_GCK_PWM_GCLK 71 +#define AT91_SCMI_CLK_GCK_ISC_GCLK 72 +#define AT91_SCMI_CLK_GCK_PDMIC_GCLK 73 +#define AT91_SCMI_CLK_GCK_I2S0_GCLK 74 +#define AT91_SCMI_CLK_GCK_I2S1_GCLK 75 +#define AT91_SCMI_CLK_GCK_CAN0_GCLK 76 +#define AT91_SCMI_CLK_GCK_CAN1_GCLK 77 +#define AT91_SCMI_CLK_GCK_CLASSD_GCLK 78 + +#define AT91_SCMI_CLK_PROG_PROG0 79 +#define AT91_SCMI_CLK_PROG_PROG1 80 +#define AT91_SCMI_CLK_PROG_PROG2 81 + +#define AT91_SCMI_CLK_SCKC_SLOWCK_32K 82 + +#endif diff --git a/optee_os/core/include/dt-bindings/clock/stm32mp1-clks.h b/optee_os/core/include/dt-bindings/clock/stm32mp1-clks.h new file mode 100644 index 0000000..615152e --- /dev/null +++ b/optee_os/core/include/dt-bindings/clock/stm32mp1-clks.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (C) STMicroelectronics 2018 - All Rights Reserved + * Author: Gabriel Fernandez for STMicroelectronics. + */ + +#ifndef _DT_BINDINGS_STM32MP1_CLKS_H_ +#define _DT_BINDINGS_STM32MP1_CLKS_H_ + +/* OSCILLATOR clocks */ +#define CK_HSE 0 +#define CK_CSI 1 +#define CK_LSI 2 +#define CK_LSE 3 +#define CK_HSI 4 +#define CK_HSE_DIV2 5 + +/* Bus clocks */ +#define TIM2 6 +#define TIM3 7 +#define TIM4 8 +#define TIM5 9 +#define TIM6 10 +#define TIM7 11 +#define TIM12 12 +#define TIM13 13 +#define TIM14 14 +#define LPTIM1 15 +#define SPI2 16 +#define SPI3 17 +#define USART2 18 +#define USART3 19 +#define UART4 20 +#define UART5 21 +#define UART7 22 +#define UART8 23 +#define I2C1 24 +#define I2C2 25 +#define I2C3 26 +#define I2C5 27 +#define SPDIF 28 +#define CEC 29 +#define DAC12 30 +#define MDIO 31 +#define TIM1 32 +#define TIM8 33 +#define TIM15 34 +#define TIM16 35 +#define TIM17 36 +#define SPI1 37 +#define SPI4 38 +#define SPI5 39 +#define USART6 40 +#define SAI1 41 +#define SAI2 42 +#define SAI3 43 +#define DFSDM 44 +#define FDCAN 45 +#define LPTIM2 46 +#define LPTIM3 47 +#define LPTIM4 48 +#define LPTIM5 49 +#define SAI4 50 +#define SYSCFG 51 +#define VREF 52 +#define TMPSENS 53 +#define PMBCTRL 54 +#define HDP 55 +#define LTDC 56 +#define DSI 57 +#define IWDG2 58 +#define USBPHY 59 +#define STGENRO 60 +#define SPI6 61 +#define I2C4 62 +#define I2C6 63 +#define USART1 64 +#define RTCAPB 65 +#define TZC1 66 +#define TZPC 67 +#define IWDG1 68 +#define BSEC 69 +#define STGEN 70 +#define DMA1 71 +#define DMA2 72 +#define DMAMUX 73 +#define ADC12 74 +#define USBO 75 +#define SDMMC3 76 +#define DCMI 77 +#define CRYP2 78 +#define HASH2 79 +#define RNG2 80 +#define CRC2 81 +#define HSEM 82 +#define IPCC 83 +#define GPIOA 84 +#define GPIOB 85 +#define GPIOC 86 +#define GPIOD 87 +#define GPIOE 88 +#define GPIOF 89 +#define GPIOG 90 +#define GPIOH 91 +#define GPIOI 92 +#define GPIOJ 93 +#define GPIOK 94 +#define GPIOZ 95 +#define CRYP1 96 +#define HASH1 97 +#define RNG1 98 +#define BKPSRAM 99 +#define MDMA 100 +#define GPU 101 +#define ETHCK 102 +#define ETHTX 103 +#define ETHRX 104 +#define ETHMAC 105 +#define FMC 106 +#define QSPI 107 +#define SDMMC1 108 +#define SDMMC2 109 +#define CRC1 110 +#define USBH 111 +#define ETHSTP 112 +#define TZC2 113 + +/* Kernel clocks */ +#define SDMMC1_K 118 +#define SDMMC2_K 119 +#define SDMMC3_K 120 +#define FMC_K 121 +#define QSPI_K 122 +#define ETHCK_K 123 +#define RNG1_K 124 +#define RNG2_K 125 +#define GPU_K 126 +#define USBPHY_K 127 +#define STGEN_K 128 +#define SPDIF_K 129 +#define SPI1_K 130 +#define SPI2_K 131 +#define SPI3_K 132 +#define SPI4_K 133 +#define SPI5_K 134 +#define SPI6_K 135 +#define CEC_K 136 +#define I2C1_K 137 +#define I2C2_K 138 +#define I2C3_K 139 +#define I2C4_K 140 +#define I2C5_K 141 +#define I2C6_K 142 +#define LPTIM1_K 143 +#define LPTIM2_K 144 +#define LPTIM3_K 145 +#define LPTIM4_K 146 +#define LPTIM5_K 147 +#define USART1_K 148 +#define USART2_K 149 +#define USART3_K 150 +#define UART4_K 151 +#define UART5_K 152 +#define USART6_K 153 +#define UART7_K 154 +#define UART8_K 155 +#define DFSDM_K 156 +#define FDCAN_K 157 +#define SAI1_K 158 +#define SAI2_K 159 +#define SAI3_K 160 +#define SAI4_K 161 +#define ADC12_K 162 +#define DSI_K 163 +#define DSI_PX 164 +#define ADFSDM_K 165 +#define USBO_K 166 +#define LTDC_PX 167 +#define DAC12_K 168 +#define ETHPTP_K 169 + +/* PLL */ +#define PLL1 176 +#define PLL2 177 +#define PLL3 178 +#define PLL4 179 + +/* ODF */ +#define PLL1_P 180 +#define PLL1_Q 181 +#define PLL1_R 182 +#define PLL2_P 183 +#define PLL2_Q 184 +#define PLL2_R 185 +#define PLL3_P 186 +#define PLL3_Q 187 +#define PLL3_R 188 +#define PLL4_P 189 +#define PLL4_Q 190 +#define PLL4_R 191 + +/* AUX */ +#define RTC 192 + +/* MCLK */ +#define CK_PER 193 +#define CK_MPU 194 +#define CK_AXI 195 +#define CK_MCU 196 + +/* Time base */ +#define TIM2_K 197 +#define TIM3_K 198 +#define TIM4_K 199 +#define TIM5_K 200 +#define TIM6_K 201 +#define TIM7_K 202 +#define TIM12_K 203 +#define TIM13_K 204 +#define TIM14_K 205 +#define TIM1_K 206 +#define TIM8_K 207 +#define TIM15_K 208 +#define TIM16_K 209 +#define TIM17_K 210 + +/* MCO clocks */ +#define CK_MCO1 211 +#define CK_MCO2 212 + +/* TRACE & DEBUG clocks */ +#define CK_DBG 214 +#define CK_TRACE 215 + +/* DDR */ +#define DDRC1 220 +#define DDRC1LP 221 +#define DDRC2 222 +#define DDRC2LP 223 +#define DDRPHYC 224 +#define DDRPHYCLP 225 +#define DDRCAPB 226 +#define DDRCAPBLP 227 +#define AXIDCG 228 +#define DDRPHYCAPB 229 +#define DDRPHYCAPBLP 230 +#define DDRPERFM 231 + +#define STM32MP1_LAST_CLK 232 + +#define LTDC_K LTDC_PX +#define ETHMAC_K ETHCK_K + +/* SCMI clock identifiers */ +#define CK_SCMI_HSE 0 +#define CK_SCMI_HSI 1 +#define CK_SCMI_CSI 2 +#define CK_SCMI_LSE 3 +#define CK_SCMI_LSI 4 +#define CK_SCMI_PLL2_Q 5 +#define CK_SCMI_PLL2_R 6 +#define CK_SCMI_MPU 7 +#define CK_SCMI_AXI 8 +#define CK_SCMI_BSEC 9 +#define CK_SCMI_CRYP1 10 +#define CK_SCMI_GPIOZ 11 +#define CK_SCMI_HASH1 12 +#define CK_SCMI_I2C4 13 +#define CK_SCMI_I2C6 14 +#define CK_SCMI_IWDG1 15 +#define CK_SCMI_RNG1 16 +#define CK_SCMI_RTC 17 +#define CK_SCMI_RTCAPB 18 +#define CK_SCMI_SPI6 19 +#define CK_SCMI_USART1 20 + +#endif /* _DT_BINDINGS_STM32MP1_CLKS_H_ */ diff --git a/optee_os/core/include/dt-bindings/clock/stm32mp13-clks.h b/optee_os/core/include/dt-bindings/clock/stm32mp13-clks.h new file mode 100644 index 0000000..ec00898 --- /dev/null +++ b/optee_os/core/include/dt-bindings/clock/stm32mp13-clks.h @@ -0,0 +1,236 @@ +/* SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause */ +/* + * Copyright (C) STMicroelectronics 2022 - All Rights Reserved + * Author: Gabriel Fernandez + */ + +#ifndef _DT_BINDINGS_STM32MP13_CLKS_H_ +#define _DT_BINDINGS_STM32MP13_CLKS_H_ + +/* OSCILLATOR clocks */ +#define CK_HSE 0 +#define CK_CSI 1 +#define CK_LSI 2 +#define CK_LSE 3 +#define CK_HSI 4 +#define CK_HSE_DIV2 5 + +/* PLL */ +#define PLL1 6 +#define PLL2 7 +#define PLL3 8 +#define PLL4 9 + +/* ODF */ +#define PLL1_P 10 +#define PLL1_Q 11 +#define PLL1_R 12 +#define PLL2_P 13 +#define PLL2_Q 14 +#define PLL2_R 15 +#define PLL3_P 16 +#define PLL3_Q 17 +#define PLL3_R 18 +#define PLL4_P 19 +#define PLL4_Q 20 +#define PLL4_R 21 + +/* SYSTEM CLOCK */ +#define CK_PER 22 +#define CK_MPU 23 +#define CK_AXI 24 +#define CK_MLAHB 25 + +#define PCLK1 26 +#define PCLK2 27 +#define PCLK3 28 +#define PCLK4 29 +#define PCLK5 30 +#define PCLK6 31 + +/* BASE TIMER */ +#define CK_TIMG1 32 +#define CK_TIMG2 33 +#define CK_TIMG3 34 + +/* AUX */ +#define RTC 35 + +/* TRACE & DEBUG clocks */ +#define CK_DBG 36 +#define CK_TRACE 37 + +/* MCO clocks */ +#define CK_MCO1 38 +#define CK_MCO2 39 + +/* IP clocks */ +#define SYSCFG 40 +#define VREF 41 +#define TMPSENS 42 +#define PMBCTRL 43 +#define HDP 44 +#define IWDG2 45 +#define STGENRO 46 +#define USART1 47 +#define RTCAPB 48 +#define TZC 49 +#define TZPC 50 +#define IWDG1 51 +#define BSEC 52 +#define DMA1 53 +#define DMA2 54 +#define DMAMUX1 55 +#define DMAMUX2 56 +#define GPIOA 57 +#define GPIOB 58 +#define GPIOC 59 +#define GPIOD 60 +#define GPIOE 61 +#define GPIOF 62 +#define GPIOG 63 +#define GPIOH 64 +#define GPIOI 65 +#define CRYP1 66 +#define HASH1 67 +#define BKPSRAM 68 +#define MDMA 69 +#define CRC1 70 +#define USBH 71 +#define DMA3 72 +#define TSC 73 +#define PKA 74 +#define AXIMC 75 +#define MCE 76 +#define ETH1TX 77 +#define ETH2TX 78 +#define ETH1RX 79 +#define ETH2RX 80 +#define ETH1MAC 81 +#define ETH2MAC 82 +#define ETH1STP 83 +#define ETH2STP 84 + +/* IP clocks with parents */ +#define SDMMC1_K 85 +#define SDMMC2_K 86 +#define ADC1_K 87 +#define ADC2_K 88 +#define FMC_K 89 +#define QSPI_K 90 +#define RNG1_K 91 +#define USBPHY_K 92 +#define STGEN_K 93 +#define SPDIF_K 94 +#define SPI1_K 95 +#define SPI2_K 96 +#define SPI3_K 97 +#define SPI4_K 98 +#define SPI5_K 99 +#define I2C1_K 100 +#define I2C2_K 101 +#define I2C3_K 102 +#define I2C4_K 103 +#define I2C5_K 104 +#define TIM2_K 105 +#define TIM3_K 106 +#define TIM4_K 107 +#define TIM5_K 108 +#define TIM6_K 109 +#define TIM7_K 110 +#define TIM12_K 111 +#define TIM13_K 112 +#define TIM14_K 113 +#define TIM1_K 114 +#define TIM8_K 115 +#define TIM15_K 116 +#define TIM16_K 117 +#define TIM17_K 118 +#define LPTIM1_K 119 +#define LPTIM2_K 120 +#define LPTIM3_K 121 +#define LPTIM4_K 122 +#define LPTIM5_K 123 +#define USART1_K 124 +#define USART2_K 125 +#define USART3_K 126 +#define UART4_K 127 +#define UART5_K 128 +#define USART6_K 129 +#define UART7_K 130 +#define UART8_K 131 +#define DFSDM_K 132 +#define FDCAN_K 133 +#define SAI1_K 134 +#define SAI2_K 135 +#define ADFSDM_K 136 +#define USBO_K 137 +#define LTDC_PX 138 +#define ETH1CK_K 139 +#define ETH1PTP_K 140 +#define ETH2CK_K 141 +#define ETH2PTP_K 142 +#define DCMIPP_K 143 +#define SAES_K 144 +#define DTS_K 145 + +/* DDR */ +#define DDRC1 146 +#define DDRC1LP 147 +#define DDRC2 148 +#define DDRC2LP 149 +#define DDRPHYC 150 +#define DDRPHYCLP 151 +#define DDRCAPB 152 +#define DDRCAPBLP 153 +#define AXIDCG 154 +#define DDRPHYCAPB 155 +#define DDRPHYCAPBLP 156 +#define DDRPERFM 157 + +#define ADC1 158 +#define ADC2 159 +#define SAI1 160 +#define SAI2 161 + +#define SPI1 162 +#define SPI2 163 +#define SPI3 164 +#define SPI4 165 +#define SPI5 166 + +#define STM32MP1_LAST_CLK 167 + +/* SCMI clock identifiers */ +#define CK_SCMI_HSE 0 +#define CK_SCMI_HSI 1 +#define CK_SCMI_CSI 2 +#define CK_SCMI_LSE 3 +#define CK_SCMI_LSI 4 +#define CK_SCMI_HSE_DIV2 5 +#define CK_SCMI_PLL2_Q 6 +#define CK_SCMI_PLL2_R 7 +#define CK_SCMI_PLL3_P 8 +#define CK_SCMI_PLL3_Q 9 +#define CK_SCMI_PLL3_R 10 +#define CK_SCMI_PLL4_P 11 +#define CK_SCMI_PLL4_Q 12 +#define CK_SCMI_PLL4_R 13 +#define CK_SCMI_MPU 14 +#define CK_SCMI_AXI 15 +#define CK_SCMI_MLAHB 16 +#define CK_SCMI_CKPER 17 +#define CK_SCMI_PCLK1 18 +#define CK_SCMI_PCLK2 19 +#define CK_SCMI_PCLK3 20 +#define CK_SCMI_PCLK4 21 +#define CK_SCMI_PCLK5 22 +#define CK_SCMI_PCLK6 23 +#define CK_SCMI_CKTIMG1 24 +#define CK_SCMI_CKTIMG2 25 +#define CK_SCMI_CKTIMG3 26 +#define CK_SCMI_RTC 27 +#define CK_SCMI_RTCAPB 28 +#define CK_SCMI_BSEC 29 + +#endif /* _DT_BINDINGS_STM32MP13_CLKS_H_ */ diff --git a/optee_os/core/include/dt-bindings/clock/stm32mp13-clksrc.h b/optee_os/core/include/dt-bindings/clock/stm32mp13-clksrc.h new file mode 100644 index 0000000..209ad5d --- /dev/null +++ b/optee_os/core/include/dt-bindings/clock/stm32mp13-clksrc.h @@ -0,0 +1,384 @@ +/* SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause */ +/* + * Copyright (C) STMicroelectronics 2022 - All Rights Reserved + * Author: Gabriel Fernandez + */ + +#ifndef _DT_BINDINGS_CLOCK_STM32MP13_CLKSRC_H_ +#define _DT_BINDINGS_CLOCK_STM32MP13_CLKSRC_H_ + +#define CMD_DIV 0 +#define CMD_MUX 1 +#define CMD_CLK 2 + +#define CMD_SHIFT 26 +#define CMD_MASK 0xFC000000 +#define CMD_DATA_MASK 0x03FFFFFF + +#define DIV_ID_SHIFT 8 +#define DIV_ID_MASK 0x0000FF00 + +#define DIV_DIVN_SHIFT 0 +#define DIV_DIVN_MASK 0x000000FF + +#define MUX_ID_SHIFT 4 +#define MUX_ID_MASK 0x00000FF0 + +#define MUX_SEL_SHIFT 0 +#define MUX_SEL_MASK 0x0000000F + +#define CLK_ID_MASK GENMASK_32(19, 11) +#define CLK_ID_SHIFT 11 +#define CLK_ON_MASK 0x00000400 +#define CLK_ON_SHIFT 10 +#define CLK_DIV_MASK GENMASK_32(9, 4) +#define CLK_DIV_SHIFT 4 +#define CLK_SEL_MASK GENMASK_32(3, 0) +#define CLK_SEL_SHIFT 0 + +#define DIV_PLL1DIVP 0 +#define DIV_PLL2DIVP 1 +#define DIV_PLL2DIVQ 2 +#define DIV_PLL2DIVR 3 +#define DIV_PLL3DIVP 4 +#define DIV_PLL3DIVQ 5 +#define DIV_PLL3DIVR 6 +#define DIV_PLL4DIVP 7 +#define DIV_PLL4DIVQ 8 +#define DIV_PLL4DIVR 9 +#define DIV_MPU 10 +#define DIV_AXI 11 +#define DIV_MLAHB 12 +#define DIV_APB1 13 +#define DIV_APB2 14 +#define DIV_APB3 15 +#define DIV_APB4 16 +#define DIV_APB5 17 +#define DIV_APB6 18 +#define DIV_RTC 19 +#define DIV_MCO1 20 +#define DIV_MCO2 21 +#define DIV_HSI 22 +#define DIV_TRACE 23 +#define DIV_ETH1PTP 24 +#define DIV_ETH2PTP 25 +#define DIV_NB 26 + +#define DIV(div_id, div) ((CMD_DIV << CMD_SHIFT) |\ + ((div_id) << DIV_ID_SHIFT) |\ + (div)) + +#define CLKSRC(mux_id, sel) ((CMD_MUX << CMD_SHIFT) |\ + ((mux_id) << MUX_ID_SHIFT) |\ + (sel)) + +/* CLK output is enable */ +#define CLK_SRC(clk_id, sel) ((CMD_CLK << CMD_SHIFT) |\ + ((clk_id) << CLK_ID_SHIFT) |\ + (sel) | CLK_ON_MASK) + +#define CLK_DISABLED(clk_id) ((CMD_CLK << CMD_SHIFT) |\ + ((clk_id) << CLK_ID_SHIFT)) + +#define MUX_MPU 0 +#define MUX_AXI 1 +#define MUX_MLAHB 2 +#define MUX_PLL12 3 +#define MUX_PLL3 4 +#define MUX_PLL4 5 +#define MUX_RTC 6 +#define MUX_MCO1 7 +#define MUX_MCO2 8 +#define MUX_CKPER 9 +#define MUX_ADC1 10 +#define MUX_ADC2 11 +#define MUX_DCMIPP 12 +#define MUX_ETH1 13 +#define MUX_ETH2 14 +#define MUX_FDCAN 15 +#define MUX_FMC 16 +#define MUX_I2C12 17 +#define MUX_I2C3 18 +#define MUX_I2C4 19 +#define MUX_I2C5 20 +#define MUX_LPTIM1 21 +#define MUX_LPTIM2 22 +#define MUX_LPTIM3 23 +#define MUX_LPTIM45 24 +#define MUX_QSPI 25 +#define MUX_RNG1 26 +#define MUX_SAES 27 +#define MUX_SAI1 28 +#define MUX_SAI2 29 +#define MUX_SDMMC1 30 +#define MUX_SDMMC2 31 +#define MUX_SPDIF 32 +#define MUX_SPI1 33 +#define MUX_SPI23 34 +#define MUX_SPI4 35 +#define MUX_SPI5 36 +#define MUX_STGEN 37 +#define MUX_UART1 38 +#define MUX_UART2 39 +#define MUX_UART35 40 +#define MUX_UART4 41 +#define MUX_UART6 42 +#define MUX_UART78 43 +#define MUX_USBO 44 +#define MUX_USBPHY 45 +#define MUX_NB 46 + +/* ADC MUX is the first Kernel MUX */ +#define MUX_KERNEL_BEGIN MUX_ADC1 + +#define CLK_MPU_HSI CLKSRC(MUX_MPU, 0) +#define CLK_MPU_HSE CLKSRC(MUX_MPU, 1) +#define CLK_MPU_PLL1P CLKSRC(MUX_MPU, 2) +#define CLK_MPU_PLL1P_DIV CLKSRC(MUX_MPU, 3) + +#define CLK_AXI_HSI CLKSRC(MUX_AXI, 0) +#define CLK_AXI_HSE CLKSRC(MUX_AXI, 1) +#define CLK_AXI_PLL2P CLKSRC(MUX_AXI, 2) + +#define CLK_MLAHBS_HSI CLKSRC(MUX_MLAHB, 0) +#define CLK_MLAHBS_HSE CLKSRC(MUX_MLAHB, 1) +#define CLK_MLAHBS_CSI CLKSRC(MUX_MLAHB, 2) +#define CLK_MLAHBS_PLL3 CLKSRC(MUX_MLAHB, 3) + +#define CLK_PLL12_HSI CLKSRC(MUX_PLL12, 0) +#define CLK_PLL12_HSE CLKSRC(MUX_PLL12, 1) + +#define CLK_PLL3_HSI CLKSRC(MUX_PLL3, 0) +#define CLK_PLL3_HSE CLKSRC(MUX_PLL3, 1) +#define CLK_PLL3_CSI CLKSRC(MUX_PLL3, 2) + +#define CLK_PLL4_HSI CLKSRC(MUX_PLL4, 0) +#define CLK_PLL4_HSE CLKSRC(MUX_PLL4, 1) +#define CLK_PLL4_CSI CLKSRC(MUX_PLL4, 2) + +#define CLK_RTC_DISABLED CLKSRC(RTC, 0) +#define CLK_RTC_LSE CLKSRC(RTC, 1) +#define CLK_RTC_LSI CLKSRC(RTC, 2) +#define CLK_RTC_HSE CLKSRC(RTC, 3) + +#define CLK_MCO1_HSI CLK_SRC(CK_MCO1, 0) +#define CLK_MCO1_HSE CLK_SRC(CK_MCO1, 1) +#define CLK_MCO1_CSI CLK_SRC(CK_MCO1, 2) +#define CLK_MCO1_LSI CLK_SRC(CK_MCO1, 3) +#define CLK_MCO1_LSE CLK_SRC(CK_MCO1, 4) +#define CLK_MCO1_DISABLED CLK_DISABLED(CK_MCO1) + +#define CLK_MCO2_MPU CLK_SRC(CK_MCO2, 0) +#define CLK_MCO2_AXI CLK_SRC(CK_MCO2, 1) +#define CLK_MCO2_MLAHB CLK_SRC(CK_MCO2, 2) +#define CLK_MCO2_PLL4 CLK_SRC(CK_MCO2, 3) +#define CLK_MCO2_HSE CLK_SRC(CK_MCO2, 4) +#define CLK_MCO2_HSI CLK_SRC(CK_MCO2, 5) +#define CLK_MCO2_DISABLED CLK_DISABLED(CK_MCO2) + +#define CLK_CKPER_HSI CLKSRC(MUX_CKPER, 0) +#define CLK_CKPER_CSI CLKSRC(MUX_CKPER, 1) +#define CLK_CKPER_HSE CLKSRC(MUX_CKPER, 2) +#define CLK_CKPER_DISABLED CLKSRC(MUX_CKPER, 3) + +#define CLK_I2C12_PCLK1 CLKSRC(MUX_I2C12, 0) +#define CLK_I2C12_PLL4R CLKSRC(MUX_I2C12, 1) +#define CLK_I2C12_HSI CLKSRC(MUX_I2C12, 2) +#define CLK_I2C12_CSI CLKSRC(MUX_I2C12, 3) + +#define CLK_I2C3_PCLK6 CLKSRC(MUX_I2C3, 0) +#define CLK_I2C3_PLL4R CLKSRC(MUX_I2C3, 1) +#define CLK_I2C3_HSI CLKSRC(MUX_I2C3, 2) +#define CLK_I2C3_CSI CLKSRC(MUX_I2C3, 3) + +#define CLK_I2C4_PCLK6 CLKSRC(MUX_I2C4, 0) +#define CLK_I2C4_PLL4R CLKSRC(MUX_I2C4, 1) +#define CLK_I2C4_HSI CLKSRC(MUX_I2C4, 2) +#define CLK_I2C4_CSI CLKSRC(MUX_I2C4, 3) + +#define CLK_I2C5_PCLK6 CLKSRC(MUX_I2C5, 0) +#define CLK_I2C5_PLL4R CLKSRC(MUX_I2C5, 1) +#define CLK_I2C5_HSI CLKSRC(MUX_I2C5, 2) +#define CLK_I2C5_CSI CLKSRC(MUX_I2C5, 3) + +#define CLK_SPI1_PLL4P CLKSRC(MUX_SPI1, 0) +#define CLK_SPI1_PLL3Q CLKSRC(MUX_SPI1, 1) +#define CLK_SPI1_I2SCKIN CLKSRC(MUX_SPI1, 2) +#define CLK_SPI1_CKPER CLKSRC(MUX_SPI1, 3) +#define CLK_SPI1_PLL3R CLKSRC(MUX_SPI1, 4) + +#define CLK_SPI23_PLL4P CLKSRC(MUX_SPI23, 0) +#define CLK_SPI23_PLL3Q CLKSRC(MUX_SPI23, 1) +#define CLK_SPI23_I2SCKIN CLKSRC(MUX_SPI23, 2) +#define CLK_SPI23_CKPER CLKSRC(MUX_SPI23, 3) +#define CLK_SPI23_PLL3R CLKSRC(MUX_SPI23, 4) + +#define CLK_SPI4_PCLK6 CLKSRC(MUX_SPI4, 0) +#define CLK_SPI4_PLL4Q CLKSRC(MUX_SPI4, 1) +#define CLK_SPI4_HSI CLKSRC(MUX_SPI4, 2) +#define CLK_SPI4_CSI CLKSRC(MUX_SPI4, 3) +#define CLK_SPI4_HSE CLKSRC(MUX_SPI4, 4) +#define CLK_SPI4_I2SCKIN CLKSRC(MUX_SPI4, 5) + +#define CLK_SPI5_PCLK6 CLKSRC(MUX_SPI5, 0) +#define CLK_SPI5_PLL4Q CLKSRC(MUX_SPI5, 1) +#define CLK_SPI5_HSI CLKSRC(MUX_SPI5, 2) +#define CLK_SPI5_CSI CLKSRC(MUX_SPI5, 3) +#define CLK_SPI5_HSE CLKSRC(MUX_SPI5, 4) + +#define CLK_UART1_PCLK6 CLKSRC(MUX_UART1, 0) +#define CLK_UART1_PLL3Q CLKSRC(MUX_UART1, 1) +#define CLK_UART1_HSI CLKSRC(MUX_UART1, 2) +#define CLK_UART1_CSI CLKSRC(MUX_UART1, 3) +#define CLK_UART1_PLL4Q CLKSRC(MUX_UART1, 4) +#define CLK_UART1_HSE CLKSRC(MUX_UART1, 5) + +#define CLK_UART2_PCLK6 CLKSRC(MUX_UART2, 0) +#define CLK_UART2_PLL3Q CLKSRC(MUX_UART2, 1) +#define CLK_UART2_HSI CLKSRC(MUX_UART2, 2) +#define CLK_UART2_CSI CLKSRC(MUX_UART2, 3) +#define CLK_UART2_PLL4Q CLKSRC(MUX_UART2, 4) +#define CLK_UART2_HSE CLKSRC(MUX_UART2, 5) + +#define CLK_UART35_PCLK1 CLKSRC(MUX_UART35, 0) +#define CLK_UART35_PLL4Q CLKSRC(MUX_UART35, 1) +#define CLK_UART35_HSI CLKSRC(MUX_UART35, 2) +#define CLK_UART35_CSI CLKSRC(MUX_UART35, 3) +#define CLK_UART35_HSE CLKSRC(MUX_UART35, 4) + +#define CLK_UART4_PCLK1 CLKSRC(MUX_UART4, 0) +#define CLK_UART4_PLL4Q CLKSRC(MUX_UART4, 1) +#define CLK_UART4_HSI CLKSRC(MUX_UART4, 2) +#define CLK_UART4_CSI CLKSRC(MUX_UART4, 3) +#define CLK_UART4_HSE CLKSRC(MUX_UART4, 4) + +#define CLK_UART6_PCLK2 CLKSRC(MUX_UART6, 0) +#define CLK_UART6_PLL4Q CLKSRC(MUX_UART6, 1) +#define CLK_UART6_HSI CLKSRC(MUX_UART6, 2) +#define CLK_UART6_CSI CLKSRC(MUX_UART6, 3) +#define CLK_UART6_HSE CLKSRC(MUX_UART6, 4) + +#define CLK_UART78_PCLK1 CLKSRC(MUX_UART78, 0) +#define CLK_UART78_PLL4Q CLKSRC(MUX_UART78, 1) +#define CLK_UART78_HSI CLKSRC(MUX_UART78, 2) +#define CLK_UART78_CSI CLKSRC(MUX_UART78, 3) +#define CLK_UART78_HSE CLKSRC(MUX_UART78, 4) + +#define CLK_LPTIM1_PCLK1 CLKSRC(MUX_LPTIM1, 0) +#define CLK_LPTIM1_PLL4P CLKSRC(MUX_LPTIM1, 1) +#define CLK_LPTIM1_PLL3Q CLKSRC(MUX_LPTIM1, 2) +#define CLK_LPTIM1_LSE CLKSRC(MUX_LPTIM1, 3) +#define CLK_LPTIM1_LSI CLKSRC(MUX_LPTIM1, 4) +#define CLK_LPTIM1_CKPER CLKSRC(MUX_LPTIM1, 5) + +#define CLK_LPTIM2_PCLK3 CLKSRC(MUX_LPTIM2, 0) +#define CLK_LPTIM2_PLL4Q CLKSRC(MUX_LPTIM2, 1) +#define CLK_LPTIM2_CKPER CLKSRC(MUX_LPTIM2, 2) +#define CLK_LPTIM2_LSE CLKSRC(MUX_LPTIM2, 3) +#define CLK_LPTIM2_LSI CLKSRC(MUX_LPTIM2, 4) + +#define CLK_LPTIM3_PCLK3 CLKSRC(MUX_LPTIM3, 0) +#define CLK_LPTIM3_PLL4Q CLKSRC(MUX_LPTIM3, 1) +#define CLK_LPTIM3_CKPER CLKSRC(MUX_LPTIM3, 2) +#define CLK_LPTIM3_LSE CLKSRC(MUX_LPTIM3, 3) +#define CLK_LPTIM3_LSI CLKSRC(MUX_LPTIM3, 4) + +#define CLK_LPTIM45_PCLK3 CLKSRC(MUX_LPTIM45, 0) +#define CLK_LPTIM45_PLL4P CLKSRC(MUX_LPTIM45, 1) +#define CLK_LPTIM45_PLL3Q CLKSRC(MUX_LPTIM45, 2) +#define CLK_LPTIM45_LSE CLKSRC(MUX_LPTIM45, 3) +#define CLK_LPTIM45_LSI CLKSRC(MUX_LPTIM45, 4) +#define CLK_LPTIM45_CKPER CLKSRC(MUX_LPTIM45, 5) + +#define CLK_SAI1_PLL4Q CLKSRC(MUX_SAI1, 0) +#define CLK_SAI1_PLL3Q CLKSRC(MUX_SAI1, 1) +#define CLK_SAI1_I2SCKIN CLKSRC(MUX_SAI1, 2) +#define CLK_SAI1_CKPER CLKSRC(MUX_SAI1, 3) +#define CLK_SAI1_PLL3R CLKSRC(MUX_SAI1, 4) + +#define CLK_SAI2_PLL4Q CLKSRC(MUX_SAI2, 0) +#define CLK_SAI2_PLL3Q CLKSRC(MUX_SAI2, 1) +#define CLK_SAI2_I2SCKIN CLKSRC(MUX_SAI2, 2) +#define CLK_SAI2_CKPER CLKSRC(MUX_SAI2, 3) +#define CLK_SAI2_SPDIF CLKSRC(MUX_SAI2, 4) +#define CLK_SAI2_PLL3R CLKSRC(MUX_SAI2, 5) + +#define CLK_FDCAN_HSE CLKSRC(MUX_FDCAN, 0) +#define CLK_FDCAN_PLL3Q CLKSRC(MUX_FDCAN, 1) +#define CLK_FDCAN_PLL4Q CLKSRC(MUX_FDCAN, 2) +#define CLK_FDCAN_PLL4R CLKSRC(MUX_FDCAN, 3) + +#define CLK_SPDIF_PLL4P CLKSRC(MUX_SPDIF, 0) +#define CLK_SPDIF_PLL3Q CLKSRC(MUX_SPDIF, 1) +#define CLK_SPDIF_HSI CLKSRC(MUX_SPDIF, 2) + +#define CLK_ADC1_PLL4R CLKSRC(MUX_ADC1, 0) +#define CLK_ADC1_CKPER CLKSRC(MUX_ADC1, 1) +#define CLK_ADC1_PLL3Q CLKSRC(MUX_ADC1, 2) + +#define CLK_ADC2_PLL4R CLKSRC(MUX_ADC2, 0) +#define CLK_ADC2_CKPER CLKSRC(MUX_ADC2, 1) +#define CLK_ADC2_PLL3Q CLKSRC(MUX_ADC2, 2) + +#define CLK_SDMMC1_HCLK6 CLKSRC(MUX_SDMMC1, 0) +#define CLK_SDMMC1_PLL3R CLKSRC(MUX_SDMMC1, 1) +#define CLK_SDMMC1_PLL4P CLKSRC(MUX_SDMMC1, 2) +#define CLK_SDMMC1_HSI CLKSRC(MUX_SDMMC1, 3) + +#define CLK_SDMMC2_HCLK6 CLKSRC(MUX_SDMMC2, 0) +#define CLK_SDMMC2_PLL3R CLKSRC(MUX_SDMMC2, 1) +#define CLK_SDMMC2_PLL4P CLKSRC(MUX_SDMMC2, 2) +#define CLK_SDMMC2_HSI CLKSRC(MUX_SDMMC2, 3) + +#define CLK_ETH1_PLL4P CLKSRC(MUX_ETH1, 0) +#define CLK_ETH1_PLL3Q CLKSRC(MUX_ETH1, 1) + +#define CLK_ETH2_PLL4P CLKSRC(MUX_ETH2, 0) +#define CLK_ETH2_PLL3Q CLKSRC(MUX_ETH2, 1) + +#define CLK_USBPHY_HSE CLKSRC(MUX_USBPHY, 0) +#define CLK_USBPHY_PLL4R CLKSRC(MUX_USBPHY, 1) +#define CLK_USBPHY_HSE_DIV2 CLKSRC(MUX_USBPHY, 2) + +#define CLK_USBO_PLL4R CLKSRC(MUX_USBO, 0) +#define CLK_USBO_USBPHY CLKSRC(MUX_USBO, 1) + +#define CLK_QSPI_ACLK CLKSRC(MUX_QSPI, 0) +#define CLK_QSPI_PLL3R CLKSRC(MUX_QSPI, 1) +#define CLK_QSPI_PLL4P CLKSRC(MUX_QSPI, 2) +#define CLK_QSPI_CKPER CLKSRC(MUX_QSPI, 3) + +#define CLK_FMC_ACLK CLKSRC(MUX_FMC, 0) +#define CLK_FMC_PLL3R CLKSRC(MUX_FMC, 1) +#define CLK_FMC_PLL4P CLKSRC(MUX_FMC, 2) +#define CLK_FMC_CKPER CLKSRC(MUX_FMC, 3) + +#define CLK_RNG1_CSI CLKSRC(MUX_RNG1, 0) +#define CLK_RNG1_PLL4R CLKSRC(MUX_RNG1, 1) +#define CLK_RNG1_LSE CLKSRC(MUX_RNG1, 2) +#define CLK_RNG1_LSI CLKSRC(MUX_RNG1, 3) + +#define CLK_STGEN_HSI CLKSRC(MUX_STGEN, 0) +#define CLK_STGEN_HSE CLKSRC(MUX_STGEN, 1) + +#define CLK_DCMIPP_ACLK CLKSRC(MUX_DCMIPP, 0) +#define CLK_DCMIPP_PLL2Q CLKSRC(MUX_DCMIPP, 1) +#define CLK_DCMIPP_PLL4P CLKSRC(MUX_DCMIPP, 2) +#define CLK_DCMIPP_CKPER CLKSRC(MUX_DCMIPP, 3) + +#define CLK_SAES_AXI CLKSRC(MUX_SAES, 0) +#define CLK_SAES_CKPER CLKSRC(MUX_SAES, 1) +#define CLK_SAES_PLL4R CLKSRC(MUX_SAES, 2) +#define CLK_SAES_LSI CLKSRC(MUX_SAES, 3) + +/* define for st,pll /csg */ +#define SSCG_MODE_CENTER_SPREAD 0 +#define SSCG_MODE_DOWN_SPREAD 1 + +/* define for st,drive */ +#define LSEDRV_LOWEST 0 +#define LSEDRV_MEDIUM_LOW 1 +#define LSEDRV_MEDIUM_HIGH 2 +#define LSEDRV_HIGHEST 3 + +#endif /* _DT_BINDINGS_CLOCK_STM32MP13_CLKSRC_H_ */ diff --git a/optee_os/core/include/dt-bindings/dma/at91.h b/optee_os/core/include/dt-bindings/dma/at91.h new file mode 100644 index 0000000..fe4bc60 --- /dev/null +++ b/optee_os/core/include/dt-bindings/dma/at91.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * This header provides macros for at91 dma bindings. + * + * Copyright (C) 2013 Ludovic Desroches + */ + +#ifndef __DT_BINDINGS_DMA_AT91_H__ +#define __DT_BINDINGS_DMA_AT91_H__ + +/* ---------- HDMAC ---------- */ + +/* + * Source and/or destination peripheral ID + */ +#define AT91_DMA_CFG_PER_ID_MASK (0xff) +#define AT91_DMA_CFG_PER_ID(id) ((id) & AT91_DMA_CFG_PER_ID_MASK) + +/* + * FIFO configuration: it defines when a request is serviced. + */ +#define AT91_DMA_CFG_FIFOCFG_OFFSET (8) +#define AT91_DMA_CFG_FIFOCFG_MASK (0xf << AT91_DMA_CFG_FIFOCFG_OFFSET) +/* half FIFO (default behavior) */ +#define AT91_DMA_CFG_FIFOCFG_HALF (0x0 << AT91_DMA_CFG_FIFOCFG_OFFSET) +/* largest defined AHB burst */ +#define AT91_DMA_CFG_FIFOCFG_ALAP (0x1 << AT91_DMA_CFG_FIFOCFG_OFFSET) +/* single AHB access */ +#define AT91_DMA_CFG_FIFOCFG_ASAP (0x2 << AT91_DMA_CFG_FIFOCFG_OFFSET) + +/* ---------- XDMAC ---------- */ +#define AT91_XDMAC_DT_MEM_IF_MASK (0x1) +#define AT91_XDMAC_DT_MEM_IF_OFFSET (13) +#define AT91_XDMAC_DT_MEM_IF(mem_if) \ + (((mem_if) & AT91_XDMAC_DT_MEM_IF_MASK) \ + << AT91_XDMAC_DT_MEM_IF_OFFSET) +#define AT91_XDMAC_DT_GET_MEM_IF(cfg) \ + (((cfg) >> AT91_XDMAC_DT_MEM_IF_OFFSET) \ + & AT91_XDMAC_DT_MEM_IF_MASK) + +#define AT91_XDMAC_DT_PER_IF_MASK (0x1) +#define AT91_XDMAC_DT_PER_IF_OFFSET (14) +#define AT91_XDMAC_DT_PER_IF(per_if) \ + (((per_if) & AT91_XDMAC_DT_PER_IF_MASK) \ + << AT91_XDMAC_DT_PER_IF_OFFSET) +#define AT91_XDMAC_DT_GET_PER_IF(cfg) \ + (((cfg) >> AT91_XDMAC_DT_PER_IF_OFFSET) \ + & AT91_XDMAC_DT_PER_IF_MASK) + +#define AT91_XDMAC_DT_PERID_MASK (0x7f) +#define AT91_XDMAC_DT_PERID_OFFSET (24) +#define AT91_XDMAC_DT_PERID(perid) (((perid) & AT91_XDMAC_DT_PERID_MASK) \ + << AT91_XDMAC_DT_PERID_OFFSET) +#define AT91_XDMAC_DT_GET_PERID(cfg) (((cfg) >> AT91_XDMAC_DT_PERID_OFFSET) \ + & AT91_XDMAC_DT_PERID_MASK) + +#endif /* __DT_BINDINGS_AT91_DMA_H__ */ diff --git a/optee_os/core/include/dt-bindings/gpio/atmel,piobu.h b/optee_os/core/include/dt-bindings/gpio/atmel,piobu.h new file mode 100644 index 0000000..d92af71 --- /dev/null +++ b/optee_os/core/include/dt-bindings/gpio/atmel,piobu.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * Copyright (C) 2022 Microchip + */ + +#ifndef _DT_BINDINGS_GPIO_ATMEL_PIOBU_H +#define _DT_BINDINGS_GPIO_ATMEL_PIOBU_H + +#define PIOBU_PIN_AFV_SHIFT 0 +#define PIOBU_PIN_AFV_MASK 0xF +#define PIOBU_PIN_AFV(val) (((val) & PIOBU_PIN_AFV_MASK) >> \ + PIOBU_PIN_AFV_SHIFT) + +#define PIOBU_PIN_RFV_SHIFT 4 +#define PIOBU_PIN_RFV_MASK 0xF0 +#define PIOBU_PIN_RFV(val) (((val) & PIOBU_PIN_RFV_MASK) >> \ + PIOBU_PIN_RFV_SHIFT) + +#define PIOBU_PIN_PULL_MODE_SHIFT 8 +#define PIOBU_PIN_PULL_MODE_MASK (0x3 << PIOBU_PIN_PULL_MODE_SHIFT) +#define PIOBU_PIN_PULL_MODE(val) (((val) & PIOBU_PIN_PULL_MODE_MASK) >> \ + PIOBU_PIN_PULL_MODE_SHIFT) +#define PIOBU_PIN_PULL_NONE 0 +#define PIOBU_PIN_PULL_UP 1 +#define PIOBU_PIN_PULL_DOWN 2 + +#define PIOBU_PIN_DEF_LEVEL_SHIFT 10 +#define PIOBU_PIN_DEF_LEVEL_MASK (BIT(PIOBU_PIN_DEF_LEVEL_SHIFT)) +#define PIOBU_PIN_DEF_LEVEL(val) (((val) & PIOBU_PIN_DEF_LEVEL_MASK) >> \ + PIOBU_PIN_DEF_LEVEL_SHIFT) +#define PIOBU_PIN_DEF_LEVEL_LOW 0 +#define PIOBU_PIN_DEF_LEVEL_HIGH 1 + +#define PIOBU_PIN_WAKEUP_SHIFT 11 +#define PIOBU_PIN_WAKEUP_MASK (BIT(PIOBU_PIN_WAKEUP_SHIFT)) +#define PIOBU_PIN_WAKEUP(val) (((val) & PIOBU_PIN_WAKEUP_MASK) >> \ + PIOBU_PIN_WAKEUP_SHIFT) +#define PIOBU_PIN_WAKEUP_DISABLE 0 +#define PIOBU_PIN_WAKEUP_ENABLE 1 + +#define PIOBU_PIN_INPUT(afv, rfv, pull_mode, def_level, wakeup) \ + ((afv) | \ + ((rfv) << PIOBU_PIN_RFV_SHIFT) & PIOBU_PIN_RFV_MASK | \ + ((pull_mode) << PIOBU_PIN_PULL_MODE_SHIFT) & \ + PIOBU_PIN_PULL_MODE_MASK | \ + ((def_level) << PIOBU_PIN_DEF_LEVEL_SHIFT) & \ + PIOBU_PIN_DEF_LEVEL_MASK | \ + ((wakeup) << PIOBU_PIN_WAKEUP_SHIFT) & PIOBU_PIN_WAKEUP_MASK) + +#endif /* _DT_BINDINGS_GPIO_ATMEL_PIOBU_H */ diff --git a/optee_os/core/include/dt-bindings/gpio/gpio.h b/optee_os/core/include/dt-bindings/gpio/gpio.h new file mode 100644 index 0000000..a635554 --- /dev/null +++ b/optee_os/core/include/dt-bindings/gpio/gpio.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * This header provides constants for most GPIO bindings. + * + * Most GPIO bindings include a flags cell as part of the GPIO specifier. + * In most cases, the format of the flags cell uses the standard values + * defined in this header. + */ + +#ifndef _DT_BINDINGS_GPIO_GPIO_H +#define _DT_BINDINGS_GPIO_GPIO_H + +/* Bit 0 express polarity */ +#define GPIO_ACTIVE_HIGH 0 +#define GPIO_ACTIVE_LOW 1 + +/* Bit 1 express single-endedness */ +#define GPIO_PUSH_PULL 0 +#define GPIO_SINGLE_ENDED 2 + +/* Bit 2 express Open drain or open source */ +#define GPIO_LINE_OPEN_SOURCE 0 +#define GPIO_LINE_OPEN_DRAIN 4 + +/* + * Open Drain/Collector is the combination of single-ended open drain interface. + * Open Source/Emitter is the combination of single-ended open source interface. + */ +#define GPIO_OPEN_DRAIN (GPIO_SINGLE_ENDED | GPIO_LINE_OPEN_DRAIN) +#define GPIO_OPEN_SOURCE (GPIO_SINGLE_ENDED | GPIO_LINE_OPEN_SOURCE) + +/* Bit 3 express GPIO suspend/resume and reset persistence */ +#define GPIO_PERSISTENT 0 +#define GPIO_TRANSITORY 8 + +/* Bit 4 express pull up */ +#define GPIO_PULL_UP 16 + +/* Bit 5 express pull down */ +#define GPIO_PULL_DOWN 32 + +#endif diff --git a/optee_os/core/include/dt-bindings/iio/adc/at91-sama5d2_adc.h b/optee_os/core/include/dt-bindings/iio/adc/at91-sama5d2_adc.h new file mode 100644 index 0000000..25060f6 --- /dev/null +++ b/optee_os/core/include/dt-bindings/iio/adc/at91-sama5d2_adc.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (C) 2021 Microchip + * + * This header provides constants for configuring the AT91 SAMA5D2 ADC + */ + +#ifndef _DT_BINDINGS_IIO_ADC_AT91_SAMA5D2_ADC_H +#define _DT_BINDINGS_IIO_ADC_AT91_SAMA5D2_ADC_H + +/* X relative position channel index */ +#define AT91_SAMA5D2_ADC_X_CHANNEL 24 +/* Y relative position channel index */ +#define AT91_SAMA5D2_ADC_Y_CHANNEL 25 +/* pressure channel index */ +#define AT91_SAMA5D2_ADC_P_CHANNEL 26 + +#endif diff --git a/optee_os/core/include/dt-bindings/interrupt-controller/arm-gic.h b/optee_os/core/include/dt-bindings/interrupt-controller/arm-gic.h new file mode 100644 index 0000000..35b6f69 --- /dev/null +++ b/optee_os/core/include/dt-bindings/interrupt-controller/arm-gic.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * This header provides constants for the ARM GIC. + */ + +#ifndef _DT_BINDINGS_INTERRUPT_CONTROLLER_ARM_GIC_H +#define _DT_BINDINGS_INTERRUPT_CONTROLLER_ARM_GIC_H + +#include + +/* interrupt specifier cell 0 */ + +#define GIC_SPI 0 +#define GIC_PPI 1 + +/* + * Interrupt specifier cell 2. + * The flags in irq.h are valid, plus those below. + */ +#define GIC_CPU_MASK_RAW(x) ((x) << 8) +#define GIC_CPU_MASK_SIMPLE(num) GIC_CPU_MASK_RAW((1 << (num)) - 1) + +#endif diff --git a/optee_os/core/include/dt-bindings/interrupt-controller/irq.h b/optee_os/core/include/dt-bindings/interrupt-controller/irq.h new file mode 100644 index 0000000..9e3d183 --- /dev/null +++ b/optee_os/core/include/dt-bindings/interrupt-controller/irq.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * This header provides constants for most IRQ bindings. + * + * Most IRQ bindings include a flags cell as part of the IRQ specifier. + * In most cases, the format of the flags cell uses the standard values + * defined in this header. + */ + +#ifndef _DT_BINDINGS_INTERRUPT_CONTROLLER_IRQ_H +#define _DT_BINDINGS_INTERRUPT_CONTROLLER_IRQ_H + +#define IRQ_TYPE_NONE 0 +#define IRQ_TYPE_EDGE_RISING 1 +#define IRQ_TYPE_EDGE_FALLING 2 +#define IRQ_TYPE_EDGE_BOTH (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING) +#define IRQ_TYPE_LEVEL_HIGH 4 +#define IRQ_TYPE_LEVEL_LOW 8 + +#endif diff --git a/optee_os/core/include/dt-bindings/mfd/atmel-flexcom.h b/optee_os/core/include/dt-bindings/mfd/atmel-flexcom.h new file mode 100644 index 0000000..6ece63d --- /dev/null +++ b/optee_os/core/include/dt-bindings/mfd/atmel-flexcom.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * This header provides macros for Atmel Flexcom DT bindings. + * + * Copyright (C) 2015 Cyrille Pitchen + */ + +#ifndef __DT_BINDINGS_MFD_ATMEL_FLEXCOM_H__ +#define __DT_BINDINGS_MFD_ATMEL_FLEXCOM_H__ + +#define ATMEL_FLEXCOM_MODE_USART 1 +#define ATMEL_FLEXCOM_MODE_SPI 2 +#define ATMEL_FLEXCOM_MODE_TWI 3 + +#endif /* __DT_BINDINGS_MFD_ATMEL_FLEXCOM_H__ */ diff --git a/optee_os/core/include/dt-bindings/mfd/st,stpmic1.h b/optee_os/core/include/dt-bindings/mfd/st,stpmic1.h new file mode 100644 index 0000000..321cd08 --- /dev/null +++ b/optee_os/core/include/dt-bindings/mfd/st,stpmic1.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) STMicroelectronics 2018 - All Rights Reserved + * Author: Philippe Peurichard , + * Pascal Paillet for STMicroelectronics. + */ + +#ifndef __DT_BINDINGS_STPMIC1_H__ +#define __DT_BINDINGS_STPMIC1_H__ + +/* IRQ definitions */ +#define IT_PONKEY_F 0 +#define IT_PONKEY_R 1 +#define IT_WAKEUP_F 2 +#define IT_WAKEUP_R 3 +#define IT_VBUS_OTG_F 4 +#define IT_VBUS_OTG_R 5 +#define IT_SWOUT_F 6 +#define IT_SWOUT_R 7 + +#define IT_CURLIM_BUCK1 8 +#define IT_CURLIM_BUCK2 9 +#define IT_CURLIM_BUCK3 10 +#define IT_CURLIM_BUCK4 11 +#define IT_OCP_OTG 12 +#define IT_OCP_SWOUT 13 +#define IT_OCP_BOOST 14 +#define IT_OVP_BOOST 15 + +#define IT_CURLIM_LDO1 16 +#define IT_CURLIM_LDO2 17 +#define IT_CURLIM_LDO3 18 +#define IT_CURLIM_LDO4 19 +#define IT_CURLIM_LDO5 20 +#define IT_CURLIM_LDO6 21 +#define IT_SHORT_SWOTG 22 +#define IT_SHORT_SWOUT 23 + +#define IT_TWARN_F 24 +#define IT_TWARN_R 25 +#define IT_VINLOW_F 26 +#define IT_VINLOW_R 27 +#define IT_SWIN_F 30 +#define IT_SWIN_R 31 + +/* BUCK MODES definitions */ +#define STPMIC1_BUCK_MODE_NORMAL 0 +#define STPMIC1_BUCK_MODE_LP 2 + +#endif /* __DT_BINDINGS_STPMIC1_H__ */ diff --git a/optee_os/core/include/dt-bindings/pinctrl/stm32-pinfunc.h b/optee_os/core/include/dt-bindings/pinctrl/stm32-pinfunc.h new file mode 100644 index 0000000..370a25a --- /dev/null +++ b/optee_os/core/include/dt-bindings/pinctrl/stm32-pinfunc.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright (C) STMicroelectronics 2017 - All Rights Reserved + * Author: Torgue Alexandre for STMicroelectronics. + */ + +#ifndef _DT_BINDINGS_STM32_PINFUNC_H +#define _DT_BINDINGS_STM32_PINFUNC_H + +/* define PIN modes */ +#define GPIO 0x0 +#define AF0 0x1 +#define AF1 0x2 +#define AF2 0x3 +#define AF3 0x4 +#define AF4 0x5 +#define AF5 0x6 +#define AF6 0x7 +#define AF7 0x8 +#define AF8 0x9 +#define AF9 0xa +#define AF10 0xb +#define AF11 0xc +#define AF12 0xd +#define AF13 0xe +#define AF14 0xf +#define AF15 0x10 +#define ANALOG 0x11 +#define RSVD 0x12 + +/* define Pins number*/ +#define PIN_NO(port, line) (((port) - 'A') * 0x10 + (line)) + +#define STM32_PINMUX(port, line, mode) (((PIN_NO(port, line)) << 8) | (mode)) + +/* package information */ +#define STM32MP_PKG_AA 0x1 +#define STM32MP_PKG_AB 0x2 +#define STM32MP_PKG_AC 0x4 +#define STM32MP_PKG_AD 0x8 + +#endif /* _DT_BINDINGS_STM32_PINFUNC_H */ + diff --git a/optee_os/core/include/dt-bindings/regulator/active-semi,8945a-regulator.h b/optee_os/core/include/dt-bindings/regulator/active-semi,8945a-regulator.h new file mode 100644 index 0000000..045e112 --- /dev/null +++ b/optee_os/core/include/dt-bindings/regulator/active-semi,8945a-regulator.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (c) 2018 Microchip Technology, Inc. All rights reserved. + * + * Device Tree binding constants for the ACT8945A PMIC regulators + */ + +#ifndef _DT_BINDINGS_REGULATOR_ACT8945A_H +#define _DT_BINDINGS_REGULATOR_ACT8945A_H + +/* + * These constants should be used to specify regulator modes in device tree for + * ACT8945A regulators as follows: + * ACT8945A_REGULATOR_MODE_FIXED: It is specific to DCDC regulators and it + * specifies the usage of fixed-frequency + * PWM. + * + * ACT8945A_REGULATOR_MODE_NORMAL: It is specific to LDO regulators and it + * specifies the usage of normal mode. + * + * ACT8945A_REGULATOR_MODE_LOWPOWER: For DCDC and LDO regulators; it specify + * the usage of proprietary power-saving + * mode. + */ + +#define ACT8945A_REGULATOR_MODE_FIXED 1 +#define ACT8945A_REGULATOR_MODE_NORMAL 2 +#define ACT8945A_REGULATOR_MODE_LOWPOWER 3 + +#endif diff --git a/optee_os/core/include/dt-bindings/regulator/st,stm32mp13-regulator.h b/optee_os/core/include/dt-bindings/regulator/st,stm32mp13-regulator.h new file mode 100644 index 0000000..a4ea712 --- /dev/null +++ b/optee_os/core/include/dt-bindings/regulator/st,stm32mp13-regulator.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * Copyright (C) 2020-2022, STMicroelectronics - All Rights Reserved + */ + +#ifndef __DT_BINDINGS_REGULATOR_ST_STM32MP13_REGULATOR_H +#define __DT_BINDINGS_REGULATOR_ST_STM32MP13_REGULATOR_H + +/* SCMI voltage domains identifiers */ + +/* SOC Internal regulators */ +#define VOLTD_SCMI_REG11 0 +#define VOLTD_SCMI_REG18 1 +#define VOLTD_SCMI_USB33 2 +#define VOLTD_SCMI_SDMMC1_IO 3 +#define VOLTD_SCMI_SDMMC2_IO 4 +#define VOLTD_SCMI_VREFBUF 5 + +/* STPMIC1 regulators */ +#define VOLTD_SCMI_STPMIC1_BUCK1 6 +#define VOLTD_SCMI_STPMIC1_BUCK2 7 +#define VOLTD_SCMI_STPMIC1_BUCK3 8 +#define VOLTD_SCMI_STPMIC1_BUCK4 9 +#define VOLTD_SCMI_STPMIC1_LDO1 10 +#define VOLTD_SCMI_STPMIC1_LDO2 11 +#define VOLTD_SCMI_STPMIC1_LDO3 12 +#define VOLTD_SCMI_STPMIC1_LDO4 13 +#define VOLTD_SCMI_STPMIC1_LDO5 14 +#define VOLTD_SCMI_STPMIC1_LDO6 15 +#define VOLTD_SCMI_STPMIC1_VREFDDR 16 +#define VOLTD_SCMI_STPMIC1_BOOST 17 +#define VOLTD_SCMI_STPMIC1_PWR_SW1 18 +#define VOLTD_SCMI_STPMIC1_PWR_SW2 19 + +/* External regulators */ +#define VOLTD_SCMI_REGU0 20 +#define VOLTD_SCMI_REGU1 21 +#define VOLTD_SCMI_REGU2 22 +#define VOLTD_SCMI_REGU3 23 +#define VOLTD_SCMI_REGU4 24 + +#endif /*__DT_BINDINGS_REGULATOR_ST_STM32MP13_REGULATOR_H */ diff --git a/optee_os/core/include/dt-bindings/regulator/st,stm32mp15-regulator.h b/optee_os/core/include/dt-bindings/regulator/st,stm32mp15-regulator.h new file mode 100644 index 0000000..b3bccff --- /dev/null +++ b/optee_os/core/include/dt-bindings/regulator/st,stm32mp15-regulator.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * Copyright (C) 2020, STMicroelectronics - All Rights Reserved + */ + +#ifndef __DT_BINDINGS_REGULATOR_ST_STM32MP15_REGULATOR_H +#define __DT_BINDINGS_REGULATOR_ST_STM32MP15_REGULATOR_H + +/* SCMI voltage domain identifiers */ + +#define VOLTD_SCMI_REG11 0 +#define VOLTD_SCMI_REG18 1 +#define VOLTD_SCMI_USB33 2 +#define VOLTD_SCMI_STPMIC1_BUCK1 3 +#define VOLTD_SCMI_STPMIC1_BUCK2 4 +#define VOLTD_SCMI_STPMIC1_BUCK3 5 +#define VOLTD_SCMI_STPMIC1_BUCK4 6 +#define VOLTD_SCMI_STPMIC1_LDO1 7 +#define VOLTD_SCMI_STPMIC1_LDO2 8 +#define VOLTD_SCMI_STPMIC1_LDO3 9 +#define VOLTD_SCMI_STPMIC1_LDO4 10 +#define VOLTD_SCMI_STPMIC1_LDO5 11 +#define VOLTD_SCMI_STPMIC1_LDO6 12 +#define VOLTD_SCMI_STPMIC1_VREFDDR 13 +#define VOLTD_SCMI_STPMIC1_BOOST 14 +#define VOLTD_SCMI_STPMIC1_PWR_SW1 15 +#define VOLTD_SCMI_STPMIC1_PWR_SW2 16 + +#endif /*__DT_BINDINGS_REGULATOR_ST_STM32MP15_REGULATOR_H */ diff --git a/optee_os/core/include/dt-bindings/reset/stm32mp1-resets.h b/optee_os/core/include/dt-bindings/reset/stm32mp1-resets.h new file mode 100644 index 0000000..84ab3c7 --- /dev/null +++ b/optee_os/core/include/dt-bindings/reset/stm32mp1-resets.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (C) STMicroelectronics 2018 - All Rights Reserved + * Author: Gabriel Fernandez for STMicroelectronics. + */ + +#ifndef _DT_BINDINGS_STM32MP1_RESET_H_ +#define _DT_BINDINGS_STM32MP1_RESET_H_ + +#define MCU_HOLD_BOOT_R 2144 +#define LTDC_R 3072 +#define DSI_R 3076 +#define DDRPERFM_R 3080 +#define USBPHY_R 3088 +#define SPI6_R 3136 +#define I2C4_R 3138 +#define I2C6_R 3139 +#define USART1_R 3140 +#define STGEN_R 3156 +#define GPIOZ_R 3200 +#define CRYP1_R 3204 +#define HASH1_R 3205 +#define RNG1_R 3206 +#define AXIM_R 3216 +#define GPU_R 3269 +#define ETHMAC_R 3274 +#define FMC_R 3276 +#define QSPI_R 3278 +#define SDMMC1_R 3280 +#define SDMMC2_R 3281 +#define CRC1_R 3284 +#define USBH_R 3288 +#define MDMA_R 3328 +#define MPSYST_R 8224 +#define MCU_R 8225 +#define TIM2_R 19456 +#define TIM3_R 19457 +#define TIM4_R 19458 +#define TIM5_R 19459 +#define TIM6_R 19460 +#define TIM7_R 19461 +#define TIM12_R 16462 +#define TIM13_R 16463 +#define TIM14_R 16464 +#define LPTIM1_R 19465 +#define SPI2_R 19467 +#define SPI3_R 19468 +#define USART2_R 19470 +#define USART3_R 19471 +#define UART4_R 19472 +#define UART5_R 19473 +#define UART7_R 19474 +#define UART8_R 19475 +#define I2C1_R 19477 +#define I2C2_R 19478 +#define I2C3_R 19479 +#define I2C5_R 19480 +#define SPDIF_R 19482 +#define CEC_R 19483 +#define DAC12_R 19485 +#define MDIO_R 19847 +#define TIM1_R 19520 +#define TIM8_R 19521 +#define TIM15_R 19522 +#define TIM16_R 19523 +#define TIM17_R 19524 +#define SPI1_R 19528 +#define SPI4_R 19529 +#define SPI5_R 19530 +#define USART6_R 19533 +#define SAI1_R 19536 +#define SAI2_R 19537 +#define SAI3_R 19538 +#define DFSDM_R 19540 +#define FDCAN_R 19544 +#define LPTIM2_R 19584 +#define LPTIM3_R 19585 +#define LPTIM4_R 19586 +#define LPTIM5_R 19587 +#define SAI4_R 19592 +#define SYSCFG_R 19595 +#define VREF_R 19597 +#define TMPSENS_R 19600 +#define PMBCTRL_R 19601 +#define DMA1_R 19648 +#define DMA2_R 19649 +#define DMAMUX_R 19650 +#define ADC12_R 19653 +#define USBO_R 19656 +#define SDMMC3_R 19664 +#define CAMITF_R 19712 +#define CRYP2_R 19716 +#define HASH2_R 19717 +#define RNG2_R 19718 +#define CRC2_R 19719 +#define HSEM_R 19723 +#define MBOX_R 19724 +#define GPIOA_R 19776 +#define GPIOB_R 19777 +#define GPIOC_R 19778 +#define GPIOD_R 19779 +#define GPIOE_R 19780 +#define GPIOF_R 19781 +#define GPIOG_R 19782 +#define GPIOH_R 19783 +#define GPIOI_R 19784 +#define GPIOJ_R 19785 +#define GPIOK_R 19786 + +/* SCMI reset domain identifiers */ +#define RST_SCMI_SPI6 0 +#define RST_SCMI_I2C4 1 +#define RST_SCMI_I2C6 2 +#define RST_SCMI_USART1 3 +#define RST_SCMI_STGEN 4 +#define RST_SCMI_GPIOZ 5 +#define RST_SCMI_CRYP1 6 +#define RST_SCMI_HASH1 7 +#define RST_SCMI_RNG1 8 +#define RST_SCMI_MDMA 9 +#define RST_SCMI_MCU 10 +#define RST_SCMI_MCU_HOLD_BOOT 11 + +#endif /* _DT_BINDINGS_STM32MP1_RESET_H_ */ diff --git a/optee_os/core/include/dt-bindings/reset/stm32mp13-resets.h b/optee_os/core/include/dt-bindings/reset/stm32mp13-resets.h new file mode 100644 index 0000000..14634cd --- /dev/null +++ b/optee_os/core/include/dt-bindings/reset/stm32mp13-resets.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (C) STMicroelectronics 2022 - All Rights Reserved + * Author: Gabriel Fernandez + */ + +#ifndef _DT_BINDINGS_STM32MP13_RESET_H_ +#define _DT_BINDINGS_STM32MP13_RESET_H_ + +#define MPSYST_R 2208 +#define TIM2_R 13568 +#define TIM3_R 13569 +#define TIM4_R 13570 +#define TIM5_R 13571 +#define TIM6_R 13572 +#define TIM7_R 13573 +#define LPTIM1_R 13577 +#define SPI2_R 13579 +#define SPI3_R 13580 +#define USART3_R 13583 +#define UART4_R 13584 +#define UART5_R 13585 +#define UART7_R 13586 +#define UART8_R 13587 +#define I2C1_R 13589 +#define I2C2_R 13590 +#define SPDIF_R 13594 +#define TIM1_R 13632 +#define TIM8_R 13633 +#define SPI1_R 13640 +#define USART6_R 13645 +#define SAI1_R 13648 +#define SAI2_R 13649 +#define DFSDM_R 13652 +#define FDCAN_R 13656 +#define LPTIM2_R 13696 +#define LPTIM3_R 13697 +#define LPTIM4_R 13698 +#define LPTIM5_R 13699 +#define SYSCFG_R 13707 +#define VREF_R 13709 +#define DTS_R 13712 +#define PMBCTRL_R 13713 +#define LTDC_R 13760 +#define DCMIPP_R 13761 +#define DDRPERFM_R 13768 +#define USBPHY_R 13776 +#define STGEN_R 13844 +#define USART1_R 13888 +#define USART2_R 13889 +#define SPI4_R 13890 +#define SPI5_R 13891 +#define I2C3_R 13892 +#define I2C4_R 13893 +#define I2C5_R 13894 +#define TIM12_R 13895 +#define TIM13_R 13896 +#define TIM14_R 13897 +#define TIM15_R 13898 +#define TIM16_R 13899 +#define TIM17_R 13900 +#define DMA1_R 13952 +#define DMA2_R 13953 +#define DMAMUX1_R 13954 +#define DMA3_R 13955 +#define DMAMUX2_R 13956 +#define ADC1_R 13957 +#define ADC2_R 13958 +#define USBO_R 13960 +#define GPIOA_R 14080 +#define GPIOB_R 14081 +#define GPIOC_R 14082 +#define GPIOD_R 14083 +#define GPIOE_R 14084 +#define GPIOF_R 14085 +#define GPIOG_R 14086 +#define GPIOH_R 14087 +#define GPIOI_R 14088 +#define TSC_R 14095 +#define PKA_R 14146 +#define SAES_R 14147 +#define CRYP1_R 14148 +#define HASH1_R 14149 +#define RNG1_R 14150 +#define AXIMC_R 14160 +#define MDMA_R 14208 +#define MCE_R 14209 +#define ETH1MAC_R 14218 +#define FMC_R 14220 +#define QSPI_R 14222 +#define SDMMC1_R 14224 +#define SDMMC2_R 14225 +#define CRC1_R 14228 +#define USBH_R 14232 +#define ETH2MAC_R 14238 + +/* SCMI reset domain identifiers */ +#define RST_SCMI_LTDC 0 +#define RST_SCMI_MDMA 1 + +#endif /* _DT_BINDINGS_STM32MP13_RESET_H_ */ diff --git a/optee_os/core/include/dt-bindings/rtc/rtc-stm32.h b/optee_os/core/include/dt-bindings/rtc/rtc-stm32.h new file mode 100644 index 0000000..8e77aae --- /dev/null +++ b/optee_os/core/include/dt-bindings/rtc/rtc-stm32.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ +/* + * Copyright (C) STMicroelectronics 2022-2023 - All Rights Reserved + * This header provides constants for STM32_RTC bindings. + */ + +#ifndef _DT_BINDINGS_RTC_STM32_H +#define _DT_BINDINGS_RTC_STM32_H + +#define RTC_OUT1 0 +#define RTC_OUT2 1 +#define RTC_OUT2_RMP 2 + +#endif /*_DT_BINDINGS_RTC_STM32_H*/ diff --git a/optee_os/core/include/gen-asm-defines.h b/optee_os/core/include/gen-asm-defines.h new file mode 100644 index 0000000..ebf330f --- /dev/null +++ b/optee_os/core/include/gen-asm-defines.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2018, Linaro Limited */ + +#ifndef GEN_ASM_DEFINES_H +#define GEN_ASM_DEFINES_H + +#define DEFINES void __defines(void); void __defines(void) + +#define DEFINE(def, val) \ + asm volatile("\n.ascii \"==>" #def " %c0 " #val "\"" : : "i" (val)); + +#endif /*GEN_ASM_DEFINES_H*/ diff --git a/optee_os/core/include/initcall.h b/optee_os/core/include/initcall.h new file mode 100644 index 0000000..28aa5ea --- /dev/null +++ b/optee_os/core/include/initcall.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef INITCALL_H +#define INITCALL_H + +#include +#include +#include + +struct initcall { + TEE_Result (*func)(void); +#if TRACE_LEVEL >= TRACE_DEBUG + int level; + const char *func_name; +#endif +}; + +#if TRACE_LEVEL >= TRACE_DEBUG +#define __define_initcall(type, lvl, fn) \ + SCATTERED_ARRAY_DEFINE_PG_ITEM_ORDERED(type ## call, lvl, \ + struct initcall) = \ + { .func = (fn), .level = (lvl), .func_name = #fn, } +#else +#define __define_initcall(type, lvl, fn) \ + SCATTERED_ARRAY_DEFINE_PG_ITEM_ORDERED(type ## call, lvl, \ + struct initcall) = \ + { .func = (fn), } +#endif + +#define preinitcall_begin \ + SCATTERED_ARRAY_BEGIN(preinitcall, struct initcall) +#define preinitcall_end SCATTERED_ARRAY_END(preinitcall, struct initcall) + +#define initcall_begin SCATTERED_ARRAY_BEGIN(initcall, struct initcall) +#define initcall_end SCATTERED_ARRAY_END(initcall, struct initcall) + +#define finalcall_begin SCATTERED_ARRAY_BEGIN(finalcall, struct initcall) +#define finalcall_end SCATTERED_ARRAY_END(finalcall, struct initcall) + +/* + * The preinit_*(), *_init() and boot_final() macros are used to register + * callback functions to be called at different stages during + * initialization. + * + * Functions registered with preinit_*() are always called before functions + * registered with *_init(). + * + * Functions registered with boot_final() are called before exiting to + * normal world the first time. + * + * Without virtualization this happens in the order of the defines below. + * + * However, with virtualization things are a bit different. boot_final() + * functions are called first before exiting to normal world the first + * time. Functions registered with boot_final() can only operate on the + * nexus. preinit_*() functions are called early before the first yielding + * call into the partition, in the newly created partition. *_init() + * functions are called at the first yielding call. + * + * +-------------------------------+-----------------------------------+ + * | Without virtualization | With virtualization | + * +-------------------------------+-----------------------------------+ + * | At the end of boot_init_primary_late() just before the print: | + * | "Primary CPU switching to normal world boot" | + * +-------------------------------+-----------------------------------+ + * | 1. call_preinitcalls() | In the nexus | + * | 2. call_initcalls() +-----------------------------------+ + * | 3. call_finalcalls() | 1. call_finalcalls() | + * +-------------------------------+-----------------------------------+ + * | "Primary CPU switching to normal world boot" is printed | + * +-------------------------------+-----------------------------------+ + * | A guest is created and | + * | virt_guest_created() is called. | + * | After the partition has been | + * | created and activated. | + * +-----------------------------------+ + * | 2. call_preinitcalls() | + * +-----------------------------------+ + * | When the partition is receiving | + * | the first yielding call | + * | virt_on_stdcall() is called. | + * +-----------------------------------+ + * | 3. call_initcalls() | + * +-----------------------------------+ + */ + +#define preinit_early(fn) __define_initcall(preinit, 1, fn) +#define preinit(fn) __define_initcall(preinit, 2, fn) +#define preinit_late(fn) __define_initcall(preinit, 3, fn) + +#define early_init(fn) __define_initcall(init, 1, fn) +#define early_init_late(fn) __define_initcall(init, 2, fn) +#define service_init(fn) __define_initcall(init, 3, fn) +#define service_init_late(fn) __define_initcall(init, 4, fn) +#define driver_init(fn) __define_initcall(init, 5, fn) +#define driver_init_late(fn) __define_initcall(init, 6, fn) +#define release_init_resource(fn) __define_initcall(init, 7, fn) + +#define boot_final(fn) __define_initcall(final, 1, fn) + +void call_preinitcalls(void); +void call_initcalls(void); +void call_finalcalls(void); + +#endif diff --git a/optee_os/core/include/io.h b/optee_os/core/include/io.h new file mode 100644 index 0000000..85365ce --- /dev/null +++ b/optee_os/core/include/io.h @@ -0,0 +1,303 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2019, Linaro Limited + */ +#ifndef IO_H +#define IO_H + +#include +#include +#include +#include + +/* + * Make sure that compiler reads/writes given variable only once. This is needed + * in cases when we have normal shared memory, and this memory can be changed + * at any moment. Compiler does not knows about this, so it can optimize memory + * access in any way, including repeated accesses from the same address. + * These macro enforce compiler to access memory only once. + */ +#define READ_ONCE(p) __compiler_atomic_load(&(p)) +#define WRITE_ONCE(p, v) __compiler_atomic_store(&(p), (v)) + +static inline void io_write8(vaddr_t addr, uint8_t val) +{ + *(volatile uint8_t *)addr = val; +} + +static inline void io_write16(vaddr_t addr, uint16_t val) +{ + *(volatile uint16_t *)addr = val; +} + +static inline void io_write32(vaddr_t addr, uint32_t val) +{ + *(volatile uint32_t *)addr = val; +} + +static inline void io_write64(vaddr_t addr, uint64_t val) +{ + *(volatile uint64_t *)addr = val; +} + +static inline uint8_t io_read8(vaddr_t addr) +{ + return *(volatile uint8_t *)addr; +} + +static inline uint16_t io_read16(vaddr_t addr) +{ + return *(volatile uint16_t *)addr; +} + +static inline uint32_t io_read32(vaddr_t addr) +{ + return *(volatile uint32_t *)addr; +} + +static inline uint64_t io_read64(vaddr_t addr) +{ + return *(volatile uint64_t *)addr; +} + +static inline void io_mask8(vaddr_t addr, uint8_t val, uint8_t mask) +{ + io_write8(addr, (io_read8(addr) & ~mask) | (val & mask)); +} + +static inline void io_mask16(vaddr_t addr, uint16_t val, uint16_t mask) +{ + io_write16(addr, (io_read16(addr) & ~mask) | (val & mask)); +} + +static inline void io_mask32(vaddr_t addr, uint32_t val, uint32_t mask) +{ + io_write32(addr, (io_read32(addr) & ~mask) | (val & mask)); +} + +static inline uint64_t get_be64(const void *p) +{ + return TEE_U64_FROM_BIG_ENDIAN(*(const uint64_t *)p); +} + +static inline void put_be64(void *p, uint64_t val) +{ + *(uint64_t *)p = TEE_U64_TO_BIG_ENDIAN(val); +} + +static inline uint32_t get_be32(const void *p) +{ + return TEE_U32_FROM_BIG_ENDIAN(*(const uint32_t *)p); +} + +static inline void put_be32(void *p, uint32_t val) +{ + *(uint32_t *)p = TEE_U32_TO_BIG_ENDIAN(val); +} + +static inline uint16_t get_be16(const void *p) +{ + return TEE_U16_FROM_BIG_ENDIAN(*(const uint16_t *)p); +} + +static inline void put_be16(void *p, uint16_t val) +{ + *(uint16_t *)p = TEE_U16_TO_BIG_ENDIAN(val); +} + +static inline void put_le32(const void *p, uint32_t val) +{ + *(uint32_t *)p = val; +} + +static inline uint32_t get_le32(const void *p) +{ + return *(const uint32_t *)p; +} + +static inline void put_le64(const void *p, uint64_t val) +{ + *(uint64_t *)p = val; +} + +static inline uint64_t get_le64(const void *p) +{ + return *(const uint64_t *)p; +} + +/* Unaligned accesses */ + +struct __unaligned_u16_t { uint16_t x; } __packed; +struct __unaligned_u32_t { uint32_t x; } __packed; +struct __unaligned_u64_t { uint64_t x; } __packed; + +static inline uint64_t get_unaligned_be64(const void *p) +{ + const struct __unaligned_u64_t *tmp = p; + + return TEE_U64_FROM_BIG_ENDIAN(tmp->x); +} + +static inline void put_unaligned_be64(void *p, uint64_t val) +{ + struct __unaligned_u64_t *tmp = p; + + tmp->x = TEE_U64_TO_BIG_ENDIAN(val); +} + +static inline uint32_t get_unaligned_be32(const void *p) +{ + const struct __unaligned_u32_t *tmp = p; + + return TEE_U32_FROM_BIG_ENDIAN(tmp->x); +} + +static inline void put_unaligned_be32(void *p, uint32_t val) +{ + struct __unaligned_u32_t *tmp = p; + + tmp->x = TEE_U32_TO_BIG_ENDIAN(val); +} + +static inline uint16_t get_unaligned_be16(const void *p) +{ + const struct __unaligned_u16_t *tmp = p; + + return TEE_U16_FROM_BIG_ENDIAN(tmp->x); +} + +static inline void put_unaligned_be16(void *p, uint16_t val) +{ + struct __unaligned_u16_t *tmp = p; + + tmp->x = TEE_U16_TO_BIG_ENDIAN(val); +} + +static inline void put_unaligned_le64(void *p, uint64_t val) +{ + struct __unaligned_u64_t *tmp = p; + + tmp->x = val; +} + +static inline uint64_t get_unaligned_le64(const void *p) +{ + const struct __unaligned_u64_t *tmp = p; + + return tmp->x; +} + +static inline void put_unaligned_le32(void *p, uint32_t val) +{ + struct __unaligned_u32_t *tmp = p; + + tmp->x = val; +} + +static inline uint32_t get_unaligned_le32(const void *p) +{ + const struct __unaligned_u32_t *tmp = p; + + return tmp->x; +} + +static inline void put_unaligned_le16(void *p, uint16_t val) +{ + struct __unaligned_u16_t *tmp = p; + + tmp->x = val; +} + +static inline uint16_t get_unaligned_le16(const void *p) +{ + const struct __unaligned_u16_t *tmp = p; + + return tmp->x; +} + +/* + * Set and clear bits helpers. + * + * @addr is the address of the memory cell accessed + * @set_mask represents the bit mask of the bit(s) to set, aka set to 1 + * @clear_mask represents the bit mask of the bit(s) to clear, aka reset to 0 + * + * io_clrsetbits32() clears then sets the target bits in this order. If a bit + * position is defined by both @set_mask and @clear_mask, the bit will be set. + */ +static inline void io_setbits32(vaddr_t addr, uint32_t set_mask) +{ + io_write32(addr, io_read32(addr) | set_mask); +} + +static inline void io_clrbits32(vaddr_t addr, uint32_t clear_mask) +{ + io_write32(addr, io_read32(addr) & ~clear_mask); +} + +static inline void io_clrsetbits32(vaddr_t addr, uint32_t clear_mask, + uint32_t set_mask) +{ + io_write32(addr, (io_read32(addr) & ~clear_mask) | set_mask); +} + +static inline void io_setbits16(vaddr_t addr, uint16_t set_mask) +{ + io_write16(addr, io_read16(addr) | set_mask); +} + +static inline void io_clrbits16(vaddr_t addr, uint16_t clear_mask) +{ + io_write16(addr, io_read16(addr) & ~clear_mask); +} + +static inline void io_clrsetbits16(vaddr_t addr, uint16_t clear_mask, + uint16_t set_mask) +{ + io_write16(addr, (io_read16(addr) & ~clear_mask) | set_mask); +} + +static inline void io_setbits8(vaddr_t addr, uint8_t set_mask) +{ + io_write8(addr, io_read8(addr) | set_mask); +} + +static inline void io_clrbits8(vaddr_t addr, uint8_t clear_mask) +{ + io_write8(addr, io_read8(addr) & ~clear_mask); +} + +static inline void io_clrsetbits8(vaddr_t addr, uint8_t clear_mask, + uint8_t set_mask) +{ + io_write8(addr, (io_read8(addr) & ~clear_mask) | set_mask); +} + +/* + * Poll on a IO memory content or timeout + * + * @_addr is the address of the memory cell accessed + * @_val represents the val of the memory cell accessed + * @_cond represents the condition to get the correct value + * @_delay_us represents the read interval in mircorseconds + * @_timeout_us represents the timeout period in microseconds + * + * @return nonzero value means timeout, 0 means got right value + */ +#define IO_READ32_POLL_TIMEOUT(_addr, _val, _cond, _delay_us, _timeout_us) \ + ({ \ + uint32_t __timeout = 0; \ + uint32_t __delay = (_delay_us); \ + \ + while (__timeout < (_timeout_us)) { \ + (_val) = io_read32(_addr); \ + if (_cond) \ + break; \ + __timeout += (__delay); \ + udelay(__delay); \ + } \ + (_val) = io_read32(_addr); \ + !(_cond); \ + }) + +#endif /*IO_H*/ diff --git a/optee_os/core/include/keep.h b/optee_os/core/include/keep.h new file mode 100644 index 0000000..0dbe910 --- /dev/null +++ b/optee_os/core/include/keep.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef KEEP_H +#define KEEP_H + +#ifdef __ASSEMBLER__ + + .macro DECLARE_KEEP_PAGER sym + .pushsection __keep_meta_vars_pager, "a" + .global ____keep_pager_\sym + ____keep_pager_\sym: + .long \sym + .popsection + .endm + + .macro DECLARE_KEEP_INIT sym + .pushsection __keep_meta_vars_init, "a" + .global ____keep_init_\sym + ____keep_init_\sym: + .long \sym + .popsection + .endm + +#else + +#include + +#define __DECLARE_KEEP_PAGER2(sym, file_id) \ + extern const unsigned long ____keep_pager_##sym; \ + const unsigned long ____keep_pager_##sym##_##file_id \ + __section("__keep_meta_vars_pager") = (unsigned long)&(sym) + +#define __DECLARE_KEEP_PAGER1(sym, file_id) __DECLARE_KEEP_PAGER2(sym, file_id) +#define DECLARE_KEEP_PAGER(sym) __DECLARE_KEEP_PAGER1(sym, __FILE_ID__) + +#define __DECLARE_KEEP_INIT2(sym, file_id) \ + extern const unsigned long ____keep_init_##sym##file_id; \ + const unsigned long ____keep_init_##sym##_##file_id \ + __section("__keep_meta_vars_init") = (unsigned long)&(sym) + +#define __DECLARE_KEEP_INIT1(sym, file_id) __DECLARE_KEEP_INIT2(sym, file_id) +#define DECLARE_KEEP_INIT(sym) __DECLARE_KEEP_INIT1(sym, __FILE_ID__) + +#endif /* __ASSEMBLER__ */ + +#endif /*KEEP_H*/ diff --git a/optee_os/core/include/kernel/abort.h b/optee_os/core/include/kernel/abort.h new file mode 100644 index 0000000..ed0e5c0 --- /dev/null +++ b/optee_os/core/include/kernel/abort.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2021, Linaro Limited + */ + +#ifndef KERNEL_ABORT_H +#define KERNEL_ABORT_H + +#define ABORT_TYPE_UNDEF 0 +#define ABORT_TYPE_PREFETCH 1 +#define ABORT_TYPE_DATA 2 +/* Dump stack on user mode panic (not an abort) */ +#define ABORT_TYPE_USER_MODE_PANIC 3 + +#ifndef __ASSEMBLER__ + +#include +#include + +struct abort_info { + uint32_t abort_type; + uint32_t fault_descr; /* only valid for data of prefetch abort */ + vaddr_t va; + uint32_t pc; + struct thread_abort_regs *regs; +}; + +/* Print abort info to the console */ +void abort_print(struct abort_info *ai); +/* Print abort info + stack dump to the console */ +void abort_print_error(struct abort_info *ai); + +void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs); + +bool abort_is_user_exception(struct abort_info *ai); + +bool abort_is_write_fault(struct abort_info *ai); + +/* Called from a normal thread */ +void abort_print_current_ts(void); + +#endif /*__ASSEMBLER__*/ +#endif /*KERNEL_ABORT_H*/ + diff --git a/optee_os/core/include/kernel/asan.h b/optee_os/core/include/kernel/asan.h new file mode 100644 index 0000000..567a168 --- /dev/null +++ b/optee_os/core/include/kernel/asan.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ +#ifndef __KERNEL_ASAN_H +#define __KERNEL_ASAN_H + +#include + +#define ASAN_DATA_RED_ZONE -1 +#define ASAN_HEAP_RED_ZONE -2 + +#define ASAN_BLOCK_SIZE U(8) +#define ASAN_BLOCK_SHIFT U(3) +#define ASAN_BLOCK_MASK (ASAN_BLOCK_SIZE - 1) + +#ifndef __ASSEMBLER__ +#include +#include +#include + +void asan_set_shadowed(const void *va_begin, const void *va_end); +void asan_start(void); + +#ifdef CFG_CORE_SANITIZE_KADDRESS +void asan_tag_no_access(const void *begin, const void *end); +void asan_tag_access(const void *begin, const void *end); +void asan_tag_heap_free(const void *begin, const void *end); +void *asan_memset_unchecked(void *s, int c, size_t n); +void *asan_memcpy_unchecked(void *__restrict s1, const void *__restrict s2, + size_t n); +#else +static inline void asan_tag_no_access(const void *begin __unused, + const void *end __unused) +{ +} +static inline void asan_tag_access(const void *begin __unused, + const void *end __unused) +{ +} +static inline void asan_tag_heap_free(const void *begin __unused, + const void *end __unused) +{ +} + +static inline void *asan_memset_unchecked(void *s, int c, size_t n) +{ + return memset(s, c, n); +} + +static inline void *asan_memcpy_unchecked(void *__restrict s1, + const void *__restrict s2, size_t n) +{ + return memcpy(s1, s2, n); +} + +#endif + +#endif /*__ASSEMBLER__*/ +#endif /*__KERNEL_ASAN_H*/ diff --git a/optee_os/core/include/kernel/boot.h b/optee_os/core/include/kernel/boot.h new file mode 100644 index 0000000..14025b3 --- /dev/null +++ b/optee_os/core/include/kernel/boot.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2020, Linaro Limited + * Copyright (c) 2021-2023, Arm Limited + */ +#ifndef __KERNEL_BOOT_H +#define __KERNEL_BOOT_H + +#include +#include + +/* + * struct boot_embdata - Embedded boot data + * @total_len: Total length of the embedded boot data + * @num_blobs: Number of blobs in the embedded boot data, always 2 even if + * one blob is empty + * @hashes_offset: Offset of hashes from start of this struct + * @hashes_len: Length of hashes + * @reloc_offset: Offset of reloc from start of this struct + * @reloc_len: Length of reloc + * + * This struct is initialized by scripts/gen_tee_bin.py and must be kept + * in sync with that script. The struct and the following data is loaded + * at different addresses at boot depending on CFG_WITH_PAGER. + * + * If configured with CFG_WITH_PAGER=y the struct with data is following + * init part, this is together with the init part moved by the primary CPU + * so it ends up at __init_end. Whatever need to be saved for later need to + * be copied to a safe location in init_runtime(). + * + * If configured with CFG_WITH_PAGER=n following the struct with data is + * __data_end, this is moved by the primary CPU so it ends up at __end. + */ +struct boot_embdata { + uint32_t total_len; + uint32_t num_blobs; + uint32_t hashes_offset; + uint32_t hashes_len; + uint32_t reloc_offset; + uint32_t reloc_len; +}; + +extern const struct core_mmu_config boot_mmu_config; + +/* @nsec_entry is unused if using CFG_WITH_ARM_TRUSTED_FW */ +void boot_init_primary_early(unsigned long pageable_part, + unsigned long nsec_entry); +void boot_init_primary_late(unsigned long fdt, unsigned long manifest); +void boot_init_memtag(void); +void boot_save_boot_info(void *boot_info); + +void __panic_at_smc_return(void) __noreturn; + +#if defined(CFG_WITH_ARM_TRUSTED_FW) +unsigned long cpu_on_handler(unsigned long a0, unsigned long a1); +unsigned long boot_cpu_on_handler(unsigned long a0, unsigned long a1); +#else +void boot_init_secondary(unsigned long nsec_entry); +#endif + +void boot_primary_init_intc(void); +void boot_secondary_init_intc(void); + +void init_sec_mon(unsigned long nsec_entry); +void init_tee_runtime(void); + +/* weak routines eventually overridden by platform */ +void plat_cpu_reset_early(void); +void plat_primary_init_early(void); +unsigned long plat_get_aslr_seed(void); +unsigned long plat_get_freq(void); +#if defined(_CFG_CORE_STACK_PROTECTOR) || defined(CFG_WITH_STACK_CANARIES) +/* + * plat_get_random_stack_canaries() - Get random values for stack canaries. + * @buf: Pointer to the buffer where to store canaries + * @ncan: The number of canaries to generate. + * @size: The size (in bytes) of each canary. + * + * This function has a __weak default implementation. + */ +void plat_get_random_stack_canaries(void *buf, size_t ncan, size_t size); +#endif +void arm_cl2_config(vaddr_t pl310); +void arm_cl2_enable(vaddr_t pl310); + +#if defined(CFG_BOOT_SECONDARY_REQUEST) +void boot_set_core_ns_entry(size_t core_idx, uintptr_t entry, + uintptr_t context_id); + +int boot_core_release(size_t core_idx, paddr_t entry); +struct ns_entry_context *boot_core_hpen(void); +#endif + +/* Returns TOS_FW_CONFIG DTB or SP manifest DTB if present, otherwise NULL */ +void *get_manifest_dt(void); + +/* + * get_aslr_seed() - return a random seed for core ASLR + * @fdt: Pointer to a device tree if CFG_DT_ADDR=y + * + * This function has a __weak default implementation. + */ +unsigned long get_aslr_seed(void *fdt); + +#endif /* __KERNEL_BOOT_H */ diff --git a/optee_os/core/include/kernel/cache_helpers.h b/optee_os/core/include/kernel/cache_helpers.h new file mode 100644 index 0000000..247b0f2 --- /dev/null +++ b/optee_os/core/include/kernel/cache_helpers.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __KERNEL_CACHE_HELPERS_H +#define __KERNEL_CACHE_HELPERS_H + +#ifndef __ASSEMBLER__ +#include +#include +#endif + +/* Data Cache set/way op type defines */ +#define DCACHE_OP_INV 0x0 +#define DCACHE_OP_CLEAN_INV 0x1 +#define DCACHE_OP_CLEAN 0x2 + +#ifndef __ASSEMBLER__ +void dcache_cleaninv_range(void *addr, size_t size); +void dcache_clean_range(void *addr, size_t size); +void dcache_inv_range(void *addr, size_t size); +void dcache_clean_range_pou(void *addr, size_t size); + +void icache_inv_all(void); +void icache_inv_range(void *addr, size_t size); +void icache_inv_user_range(void *addr, size_t size); + +void dcache_op_louis(unsigned long op_type); +void dcache_op_all(unsigned long op_type); + +void dcache_op_level1(unsigned long op_type); +void dcache_op_level2(unsigned long op_type); +void dcache_op_level3(unsigned long op_type); + +/* + * Get system maximum cache line size. + */ +static inline unsigned int cache_get_max_line_size(void) +{ + return 1 << CFG_MAX_CACHE_LINE_SHIFT; +} +#endif /*!__ASSEMBLER__*/ + +#endif /*__KERNEL_CACHE_HELPERS_H*/ diff --git a/optee_os/core/include/kernel/chip_services.h b/optee_os/core/include/kernel/chip_services.h new file mode 100644 index 0000000..a22237c --- /dev/null +++ b/optee_os/core/include/kernel/chip_services.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef CHIP_SERVICES_H +#define CHIP_SERVICES_H + +/* Die ID */ +#define FVR_DIE_ID_NUM_REGS 3 + +#endif diff --git a/optee_os/core/include/kernel/delay.h b/optee_os/core/include/kernel/delay.h new file mode 100644 index 0000000..6cb7268 --- /dev/null +++ b/optee_os/core/include/kernel/delay.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __KERNEL_DELAY_H +#define __KERNEL_DELAY_H + +#include +#include +#include + +void udelay(uint32_t us); +void mdelay(uint32_t ms); + +#endif diff --git a/optee_os/core/include/kernel/dt.h b/optee_os/core/include/kernel/dt.h new file mode 100644 index 0000000..7a7bd9a --- /dev/null +++ b/optee_os/core/include/kernel/dt.h @@ -0,0 +1,448 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2021, Linaro Limited + */ + +#ifndef KERNEL_DT_H +#define KERNEL_DT_H + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Bitfield to reflect status and secure-status values ("okay", "disabled" + * or not present) + */ +#define DT_STATUS_DISABLED U(0) +#define DT_STATUS_OK_NSEC BIT(0) +#define DT_STATUS_OK_SEC BIT(1) + +#define DT_INFO_INVALID_REG ((paddr_t)-1) +#define DT_INFO_INVALID_REG_SIZE ((size_t)-1) +#define DT_INFO_INVALID_CLOCK -1 +#define DT_INFO_INVALID_RESET -1 +#define DT_INFO_INVALID_INTERRUPT -1 + +/* + * @status: Bit mask for DT_STATUS_* + * @reg: Device register physical base address or DT_INFO_INVALID_REG + * @reg_size: Device register size or DT_INFO_INVALID_REG_SIZE + * @clock: Device identifier (positive value) or DT_INFO_INVALID_CLOCK + * @reset: Device reset identifier (positive value) or DT_INFO_INVALID_CLOCK + * @interrupt: Device interrupt identifier (positive value) or + * DT_INFO_INVALID_INTERRUPT + * @type: IRQ_TYPE_* value parsed from interrupts properties or IRQ_TYPE_NONE if + * not present + * @prio: interrupt priority parsed from interrupts properties or 0 if not + * present + */ +struct dt_node_info { + unsigned int status; + paddr_t reg; + size_t reg_size; + int clock; + int reset; + int interrupt; + uint32_t type; + uint32_t prio; +}; + +/* + * DT-aware drivers + */ + +struct dt_device_match { + const char *compatible; + const void *compat_data; +}; + +/* + * DT_MAP_AUTO: Uses status properties from device tree to determine mapping. + * DT_MAP_SECURE: Force mapping for device to be secure. + * DT_MAP_NON_SECURE: Force mapping for device to be non-secure. + */ +enum dt_map_dev_directive { + DT_MAP_AUTO, + DT_MAP_SECURE, + DT_MAP_NON_SECURE +}; + +/* + * struct dt_descriptor - Descriptor of the device tree + * @blob: Pointer to the device tree binary + * @frag_id: Used ID of fragments for device tree overlay + */ +struct dt_descriptor { + void *blob; +#ifdef _CFG_USE_DTB_OVERLAY + int frag_id; +#endif +}; + +extern uint8_t embedded_secure_dtb[]; + +/* + * dt_getprop_as_number() - get a DT property a unsigned number + * @fdt: DT base address + * @nodeoffset: node offset + * @name: property string name + * @num: output number read + * Return 0 on success and a negative FDT error value on error + * + * The size of the property determines if it is read as an unsigned 32-bit + * or 64-bit integer. + */ +int dt_getprop_as_number(const void *fdt, int nodeoffset, const char *name, + uint64_t *num); + +#ifdef CFG_DT +/* + * Find a driver that is suitable for the given DT node, that is, with + * a matching "compatible" property. + * + * @fdt: pointer to the device tree + * @offs: node offset + */ +const struct dt_driver *dt_find_compatible_driver(const void *fdt, int offs); + +/* + * Map a device into secure or non-secure memory and return the base VA and + * the mapping size. The mapping is done with type MEM_AREA_IO_SEC or + * MEM_AREA_IO_NSEC, depending on the device status. + * If the mapping already exists, the function simply returns the @vbase and + * @size information. + * + * @offs is the offset of the node that describes the device in @fdt. + * @base receives the base virtual address corresponding to the base physical + * address of the "reg" property + * @size receives the size of the mapping + * @mapping what kind of mapping is done for memory. + * + * Returns 0 on success or -1 in case of error. + */ +int dt_map_dev(const void *fdt, int offs, vaddr_t *base, size_t *size, + enum dt_map_dev_directive mapping); + +/* + * Check whether the node at @offs contains the property with propname or not. + * + * @offs is the offset of the node that describes the device in @fdt. + * @propname is the property that need to check + * + * Returns true on success or false if no propname. + */ +bool dt_have_prop(const void *fdt, int offs, const char *propname); + +/* + * Modify or add "status" property to "disabled" + * + * @fdt reference to the Device Tree + * @node is the node offset to modify + * + * Returns 0 on success or -1 on failure + */ +int dt_disable_status(void *fdt, int node); + +/* + * Force secure-status = "okay" and status="disabled" for the target node. + * + * @fdt reference to the Device Tree + * @node is the node offset to modify + * + * Returns 0 on success or -1 on failure + */ +int dt_enable_secure_status(void *fdt, int node); + +/* + * FDT manipulation functions, not provided by + */ + +/* + * Return the base address for the "reg" property of the specified node or + * (paddr_t)-1 in case of error + */ +paddr_t fdt_reg_base_address(const void *fdt, int offs); + +/* + * Return the reg size for the reg property of the specified node or -1 in case + * of error + */ +size_t fdt_reg_size(const void *fdt, int offs); + +/* + * Read the status and secure-status properties into a bitfield. + * Return -1 on failure, DT_STATUS_DISABLED if the node is disabled, + * otherwise return a combination of DT_STATUS_OK_NSEC and DT_STATUS_OK_SEC. + */ +int fdt_get_status(const void *fdt, int offs); + +/* + * fdt_fill_device_info - Get generic device info from a node + * + * This function fills the generic information from a given node. + * Currently supports a single base register, a single clock, + * a single reset ID line and a single interrupt ID. + * Default DT_INFO_* macros are used when the relate property is not found. + */ +void fdt_fill_device_info(const void *fdt, struct dt_node_info *info, + int node); +/* + * Read cells from a given property of the given node. Any number of 32-bit + * cells of the property can be read. Returns 0 on success, or a negative + * FDT error value otherwise. + */ +int fdt_read_uint32_array(const void *fdt, int node, const char *prop_name, + uint32_t *array, size_t count); + +/* + * Read one cell from a given multi-value property of the given node. + * Returns 0 on success, or a negative FDT error value otherwise. + */ +int fdt_read_uint32_index(const void *fdt, int node, const char *prop_name, + int index, uint32_t *value); + +/* + * Read one cell from a given property of the given node. + * Returns 0 on success, or a negative FDT error value otherwise. + */ +int fdt_read_uint32(const void *fdt, int node, const char *prop_name, + uint32_t *value); + +/* + * Read one cell from a property of a cell or default to a given value + * Returns the 32bit cell value or @dflt_value on failure. + */ +uint32_t fdt_read_uint32_default(const void *fdt, int node, + const char *prop_name, uint32_t dflt_value); + +/* + * This function fills reg node info (base & size) with an index. + * + * Returns 0 on success and a negative FDT error code on failure. + */ +int fdt_get_reg_props_by_index(const void *fdt, int node, int index, + paddr_t *base, size_t *size); + +/* + * This function fills reg node info (base & size) with an index found by + * checking the reg-names node. + * + * Returns 0 on success and a negative FDT error code on failure. + */ +int fdt_get_reg_props_by_name(const void *fdt, int node, const char *name, + paddr_t *base, size_t *size); + +/* Returns embedded DTB if present, then external DTB if found, then NULL */ +void *get_dt(void); + +/* + * get_secure_dt() - returns secure DTB for drivers + * + * Returns device tree that is considered secure for drivers to use. + * + * 1. Returns embedded DTB if available, + * 2. Secure external DTB if available, + * 3. If neither then NULL + */ +void *get_secure_dt(void); + +/* Returns embedded DTB location if present, otherwise NULL */ +void *get_embedded_dt(void); + +/* Returns true if passed DTB is same as Embedded DTB, otherwise false */ +static inline bool is_embedded_dt(void *fdt) +{ + return fdt && fdt == get_embedded_dt(); +} + +/* Returns DTB descriptor of the external DTB if present, otherwise NULL */ +struct dt_descriptor *get_external_dt_desc(void); + +/* + * init_external_dt() - Initialize the external DTB located at given address. + * @phys_dt: Physical address where the external DTB located. + * + * Initialize the external DTB. + * + * 1. Add MMU mapping of the external DTB, + * 2. Initialize device tree overlay + */ +void init_external_dt(unsigned long phys_dt); + +/* Returns external DTB if present, otherwise NULL */ +void *get_external_dt(void); + +/* + * add_dt_path_subnode() - Add new child node into a parent node. + * @dt: Pointer to a device tree descriptor which has DTB. + * @path: Path to the parent node. + * @subnode: Name of the child node. + * + * Returns the offset of the child node in DTB on success or a negative libfdt + * error number. + */ +int add_dt_path_subnode(struct dt_descriptor *dt, const char *path, + const char *subnode); + +/* + * add_res_mem_dt_node() - Create "reserved-memory" parent and child nodes. + * @dt: Pointer to a device tree descriptor which has DTB. + * @name: Name of the child node. + * @pa: Physical address of specific reserved memory region. + * @size: Size of specific reserved memory region. + * + * Returns 0 if succeeds, otherwise a negative libfdt error number. + */ +int add_res_mem_dt_node(struct dt_descriptor *dt, const char *name, + paddr_t pa, size_t size); + +#else /* !CFG_DT */ + +static inline const struct dt_driver *dt_find_compatible_driver( + const void *fdt __unused, + int offs __unused) +{ + return NULL; +} + +static inline int dt_map_dev(const void *fdt __unused, int offs __unused, + vaddr_t *vbase __unused, size_t *size __unused, + enum dt_map_dev_directive mapping __unused) +{ + return -1; +} + +static inline paddr_t fdt_reg_base_address(const void *fdt __unused, + int offs __unused) +{ + return (paddr_t)-1; +} + +static inline size_t fdt_reg_size(const void *fdt __unused, + int offs __unused) +{ + return (size_t)-1; +} + +static inline int fdt_get_status(const void *fdt __unused, int offs __unused) +{ + return -1; +} + +__noreturn +static inline void fdt_fill_device_info(const void *fdt __unused, + struct dt_node_info *info __unused, + int node __unused) +{ + panic(); +} + +static inline int fdt_read_uint32_array(const void *fdt __unused, + int node __unused, + const char *prop_name __unused, + uint32_t *array __unused, + size_t count __unused) +{ + return -1; +} + +static inline int fdt_read_uint32(const void *fdt __unused, + int node __unused, + const char *prop_name __unused, + uint32_t *value __unused) +{ + return -1; +} + +static inline uint32_t fdt_read_uint32_default(const void *fdt __unused, + int node __unused, + const char *prop_name __unused, + uint32_t dflt_value __unused) +{ + return dflt_value; +} + +static inline int fdt_read_uint32_index(const void *fdt __unused, + int node __unused, + const char *prop_name __unused, + int index __unused, + uint32_t *value __unused) +{ + return -1; +} + +static inline int fdt_get_reg_props_by_index(const void *fdt __unused, + int node __unused, + int index __unused, + paddr_t *base __unused, + size_t *size __unused) +{ + return -1; +} + +static inline int fdt_get_reg_props_by_name(const void *fdt __unused, + int node __unused, + const char *name __unused, + paddr_t *base __unused, + size_t *size __unused) +{ + return -1; +} + +static inline void *get_dt(void) +{ + return NULL; +} + +static inline void *get_secure_dt(void) +{ + return NULL; +} + +static inline void *get_embedded_dt(void) +{ + return NULL; +} + +static inline bool is_embedded_dt(void *fdt __unused) +{ + return false; +} + +static inline struct dt_descriptor *get_external_dt_desc(void) +{ + return NULL; +} + +static inline void init_external_dt(unsigned long phys_dt __unused) +{ +} + +static inline void *get_external_dt(void) +{ + return NULL; +} + +static inline int add_dt_path_subnode(struct dt_descriptor *dt __unused, + const char *path __unused, + const char *subnode __unused) +{ + return -1; +} + +static inline int add_res_mem_dt_node(struct dt_descriptor *dt __unused, + const char *name __unused, + paddr_t pa __unused, + size_t size __unused) +{ + return -1; +} + +#endif /* !CFG_DT */ +#endif /* KERNEL_DT_H */ diff --git a/optee_os/core/include/kernel/dt_driver.h b/optee_os/core/include/kernel/dt_driver.h new file mode 100644 index 0000000..38a934a --- /dev/null +++ b/optee_os/core/include/kernel/dt_driver.h @@ -0,0 +1,282 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Linaro Limited + * Copyright (c) 2021, Bootlin + * Copyright (c) 2021, STMicroelectronics + */ + +#ifndef __DT_DRIVER_H +#define __DT_DRIVER_H + +#include +#include +#include +#include + +/* + * Type indentifiers for registered device drivers consumer can query + * + * DT_DRIVER_NOTYPE Generic type for when no generic FDT parsing is supported + * DT_DRIVER_UART UART driver currently designed for console means + * DT_DRIVER_CLK Clock controller using generic clock DT bindings + * DT_DRIVER_RSTCTRL Reset controller using generic reset DT bindings + * DT_DRIVER_I2C I2C bus controller using generic I2C bus DT bindings + * DT_DRIVER_GPIO GPIO controller using generic GPIO DT bindings + * DT_DRIVER_PINCTRL Pin controller using generic reset DT bindings + * DT_DRIVER_INTERRUPT Interrupt controller using generic DT bindings + * DT_DRIVER_REGULATOR Voltage regulator controller using generic DT bindings + */ +enum dt_driver_type { + DT_DRIVER_NOTYPE, + DT_DRIVER_UART, + DT_DRIVER_CLK, + DT_DRIVER_RSTCTRL, + DT_DRIVER_I2C, + DT_DRIVER_GPIO, + DT_DRIVER_PINCTRL, + DT_DRIVER_INTERRUPT, + DT_DRIVER_REGULATOR, +}; + +/* + * dt_driver_probe_func - Callback probe function for a driver. + * + * @fdt: FDT base address + * @nodeoffset: Offset of the node in the FDT + * @compat_data: Data registered for the compatible that probed the device + * + * Return TEE_SUCCESS on successful probe, + * TEE_ERROR_DEFER_DRIVER_INIT if probe must be deferred + * TEE_ERROR_ITEM_NOT_FOUND when no driver matched node's compatible string + * Any other TEE_ERROR_* compliant code. + */ +typedef TEE_Result (*dt_driver_probe_func)(const void *fdt, int nodeoffset, + const void *compat_data); + +/* + * Driver instance registered to be probed on compatible node found in the DT. + * + * @name: Driver name + * @type: Drive type + * @match_table: Compatible matching identifiers, null terminated + * @driver: Driver private reference or NULL + * @probe: Probe callback (see dt_driver_probe_func) or NULL + */ +struct dt_driver { + const char *name; + enum dt_driver_type type; + const struct dt_device_match *match_table; /* null-terminated */ + const void *driver; + TEE_Result (*probe)(const void *fdt, int node, const void *compat_data); +}; + +#define DEFINE_DT_DRIVER(name) \ + SCATTERED_ARRAY_DEFINE_PG_ITEM(dt_drivers, struct dt_driver) + +#define for_each_dt_driver(drv) \ + for (drv = SCATTERED_ARRAY_BEGIN(dt_drivers, struct dt_driver); \ + drv < SCATTERED_ARRAY_END(dt_drivers, struct dt_driver); \ + drv++) + +/* Opaque reference to DT driver device provider instance */ +struct dt_driver_provider; + +/** + * struct dt_pargs - Devicetree phandle arguments + * @fdt: Device-tree to work on + * @phandle_node: Node pointed by the specifier phandle + * @args_count: Count of cells for the device + * @args: Device consumer specifiers + */ +struct dt_pargs { + const void *fdt; + int phandle_node; + int args_count; + uint32_t args[]; +}; + +/* + * get_of_device_func - Callback function for returning a driver private + * instance based on a FDT phandle with possible arguments and the + * registered dt_driver private data reference. + * + * @parg: phandle argument(s) referencing the device in the FDT. + * @data: driver private data registered in struct dt_driver. + * @res: Output result code of the operation: + * TEE_SUCCESS in case of success + * TEE_ERROR_DEFER_DRIVER_INIT if device driver is not yet initialized + * Any TEE_Result compliant code in case of error. + * + * Return a device opaque reference, e.g. a struct clk pointer for a clock + * driver, or NULL if not found in which case @res provides the error code. + */ +typedef TEE_Result (*get_of_device_func)(struct dt_pargs *parg, void *data, + void **out_device); + +/** + * dt_driver_register_provider - Register a driver provider + * + * @fdt: Device tree to work on + * @nodeoffset: Node offset in the FDT + * @get_of_device: Function to match the devicetree with a device instance + * @data: Data which will be passed to the @get_of_device callback + * @type: Driver type + * + * @get_of_device returns a void *. Driver provider is expected to + * include a shim helper to cast to device reference into provider driver + * target structure reference (e.g (struct clk *) for clock devices). + */ +TEE_Result dt_driver_register_provider(const void *fdt, int nodeoffset, + get_of_device_func get_of_device, + void *data, enum dt_driver_type type); + +/* + * dt_driver_device_from_node_idx_prop - Return a device instance based on a + * property name and FDT information + * + * @prop_name: DT property name, e.g. "clocks" for clock resources + * @fdt: FDT base address + * @nodeoffset: node offset in the FDT + * @prop_idx: index of the phandle data in the property + * @type: Driver type + * @out_device: output device opaque reference upon support, for example + * a struct clk pointer for a clock driver. + + * Return code: + * TEE_SUCCESS in case of success, + * TEE_ERROR_DEFER_DRIVER_INIT if device driver is not yet initialized + * TEE_ERROR_ITEM_NOT_FOUND if @prop_name does not match a property's name + * or @prop_idx does not match any index in @prop_name phandle list + * Any TEE_Result compliant code in case of error. + */ +TEE_Result dt_driver_device_from_node_idx_prop(const char *prop_name, + const void *fdt, int nodeoffset, + unsigned int prop_idx, + enum dt_driver_type type, + void **out_device); + +/* + * dt_driver_device_from_parent - Return a device instance based on the parent. + * This is mainly used for the devices that are children of a controller + * such as I2C, SPI and so on. + * + * @fdt: FDT base address + * @nodeoffset: node offset in the FDT + * @type: Driver type + * @dout_device: output device opaque reference upon success, for example + * a struct i2c_dev pointer for a I2C bus driver + * + * Return code: + * TEE_SUCCESS in case of success, + * TEE_ERROR_DEFER_DRIVER_INIT if device driver is not yet initialized + * Any TEE_Result compliant code in case of error. + */ +TEE_Result dt_driver_device_from_parent(const void *fdt, int nodeoffset, + enum dt_driver_type type, + void **out_device); + +/* + * dt_driver_device_from_node_idx_prop_phandle() - Same as + * dt_driver_device_from_node_idx_prop() but phandle is not the first + * cells in property @prop_name but is passed as an argument. + * + * This function is used for DT bindings as "interrupts" property where the + * property carries the interrupt information but not the interrupt controller + * phandle which is found in a specific property (here "interrupt-parent"). + */ +TEE_Result dt_driver_device_from_node_idx_prop_phandle(const char *prop_name, + const void *fdt, + int nodeoffs, + unsigned int prop_index, + enum dt_driver_type type, + uint32_t phandle, + void **out_device); + +/* + * dt_driver_get_crypto() - Request crypto support for driver initialization + * + * Return TEE_SUCCESS if cryptography services are initialized, otherwise return + * TEE_ERROR_DEFER_DRIVER_INIT. + */ +TEE_Result dt_driver_get_crypto(void); + +#ifdef CFG_DT +/* Inform DT driver probe sequence that core crypto support is initialized */ +void dt_driver_crypt_init_complete(void); +#else +static inline void dt_driver_crypt_init_complete(void) {} +#endif + +/* + * Return driver provider reference from its node offset value in the FDT + */ +struct dt_driver_provider * +dt_driver_get_provider_by_node(int nodeoffset, enum dt_driver_type type); + +/* + * Return driver provider reference from its phandle value in the FDT + */ +struct dt_driver_provider * +dt_driver_get_provider_by_phandle(uint32_t phandle, enum dt_driver_type type); + +/* + * Return number cells used for phandle arguments by a driver provider + */ +unsigned int dt_driver_provider_cells(struct dt_driver_provider *prv); + +/* + * Return provider private data registered by dt_driver_register_provider() + */ +void *dt_driver_provider_priv_data(struct dt_driver_provider *prv); + +/* + * dt_driver_probe_device_by_node - Probe matching driver to create a device + * from a FDT node + * + * @fdt: FDT base address + * @nodeoffset: Node byte offset from FDT base + * @type: Target driver to match or DT_DRIVER_ANY + * + * Read the dt_driver database. Compatible list is looked up in the order + * of the FDT "compatible" property list. @type can be used to probe only + * specific drivers. + * + */ +TEE_Result dt_driver_probe_device_by_node(const void *fdt, int nodeoffset, + enum dt_driver_type type); + +/* + * Get cells count of a device node given its dt_driver type + * + * @fdt: FDT base address + * @nodeoffset: Node offset on the FDT for the device + * @type: One of the supported DT_DRIVER_* value. + * + * Return a positive cell count value (>= 0) or a negative FDT_ error code + */ +int fdt_get_dt_driver_cells(const void *fdt, int nodeoffset, + enum dt_driver_type type); + +/* + * Called by bus like nodes to propose a node for dt_driver probing + * + * @fdt: FDT base address + * @nodeoffset: Node offset on the FDT for the device + */ +TEE_Result dt_driver_maybe_add_probe_node(const void *fdt, int nodeoffset); + +#ifdef CFG_DT_DRIVER_EMBEDDED_TEST +/* + * Return TEE_ERROR_NOT_IMPLEMENTED if test are not implemented + * otherwise return TEE_ERROR_GENERIC if some test has failed + * otherwise return TEE_SUCCESS (tests succeed or skipped) + */ +TEE_Result dt_driver_test_status(void); +#else +static inline TEE_Result dt_driver_test_status(void) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif + +#endif /* __DT_DRIVER_H */ diff --git a/optee_os/core/include/kernel/early_ta.h b/optee_os/core/include/kernel/early_ta.h new file mode 100644 index 0000000..b7f9f38 --- /dev/null +++ b/optee_os/core/include/kernel/early_ta.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, 2020, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ +#ifndef KERNEL_EARLY_TA_H +#define KERNEL_EARLY_TA_H + +#include +#include +#include +#include + +#define for_each_early_ta(_ta) \ + SCATTERED_ARRAY_FOREACH(_ta, early_tas, struct embedded_ts) + +#endif /* KERNEL_EARLY_TA_H */ + diff --git a/optee_os/core/include/kernel/embedded_ts.h b/optee_os/core/include/kernel/embedded_ts.h new file mode 100644 index 0000000..d2ae758 --- /dev/null +++ b/optee_os/core/include/kernel/embedded_ts.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ +#ifndef KERNEL_EMBEDDED_TS_H +#define KERNEL_EMBEDDED_TS_H + +#include +#include +#include +#include +#include + +struct embedded_ts { + uint32_t flags; + TEE_UUID uuid; + uint32_t size; + uint32_t uncompressed_size; /* 0: not compressed */ + const uint8_t *ts; /* @size bytes */ +}; + +struct ts_store_handle; + +TEE_Result emb_ts_read(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len); +void emb_ts_close(struct ts_store_handle *h); + +TEE_Result emb_ts_open(const TEE_UUID *uuid, + struct ts_store_handle **h, + const struct embedded_ts* (*find_ts) + (const TEE_UUID *uuid)); +TEE_Result emb_ts_get_size(const struct ts_store_handle *h, size_t *size); +TEE_Result emb_ts_get_tag(const struct ts_store_handle *h, + uint8_t *tag, unsigned int *tag_len); +#endif /* KERNEL_EMBEDDED_TS_H */ + diff --git a/optee_os/core/include/kernel/handle.h b/optee_os/core/include/kernel/handle.h new file mode 100644 index 0000000..eb4375d --- /dev/null +++ b/optee_os/core/include/kernel/handle.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ +#ifndef KERNEL_HANDLE_H +#define KERNEL_HANDLE_H + +#include +#include + +struct handle_db { + void **ptrs; + size_t max_ptrs; +}; + +#define HANDLE_DB_INITIALIZER { NULL, 0 } + +/* + * Frees all internal data structures of the database, but does not free + * the db pointer. The database is safe to reuse after it's destroyed, it + * will just be empty again. If ptr_destructor is non-null it will be + * called for each registered pointer before the database is cleared. + */ +void handle_db_destroy(struct handle_db *db, void (*ptr_destructor)(void *ptr)); + +/* Checks if the associated pointers of all handles in the database are NULL. */ +bool handle_db_is_empty(struct handle_db *db); + +/* + * Allocates a new handle and assigns the supplied pointer to it, + * ptr must not be NULL. + * The function returns + * >= 0 on success and + * -1 on failure + */ +int handle_get(struct handle_db *db, void *ptr); + +/* + * Deallocates a handle. Returns the assiciated pointer of the handle + * if the handle was valid or NULL if it's invalid. + */ +void *handle_put(struct handle_db *db, int handle); + +/* + * Returns the assiciated pointer of the handle if the handle is a valid + * handle. + * Returns NULL on failure. + */ +void *handle_lookup(struct handle_db *db, int handle); + +#endif /*KERNEL_HANDLE_H*/ diff --git a/optee_os/core/include/kernel/huk_subkey.h b/optee_os/core/include/kernel/huk_subkey.h new file mode 100644 index 0000000..f8e876b --- /dev/null +++ b/optee_os/core/include/kernel/huk_subkey.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef __KERNEL_HUK_SUBKEY_H +#define __KERNEL_HUK_SUBKEY_H + +#include +#include +#include + +/* + * enum huk_subkey_usage - subkey usage identifier + * @HUK_SUBKEY_RPMB: RPMB key + * @HUK_SUBKEY_SSK: Secure Storage key + * @HUK_SUBKEY_DIE_ID: Representing the die ID + * @HUK_SUBKEY_UNIQUE_TA: TA unique key + * @HUK_SUBKEY_TA_ENC: TA encryption key + * @HUK_SUBKEY_SE050: SCP03 set of encryption keys + * + * Add more identifiers as needed, be careful to not change the already + * assigned numbers as that will affect the derived subkey. + */ +enum huk_subkey_usage { + /* + * All IDs are explicitly assigned to make it easier to keep then + * constant. + */ + HUK_SUBKEY_RPMB = 0, + HUK_SUBKEY_SSK = 1, + HUK_SUBKEY_DIE_ID = 2, + HUK_SUBKEY_UNIQUE_TA = 3, + HUK_SUBKEY_TA_ENC = 4, + HUK_SUBKEY_SE050 = 5, +}; + +#define HUK_SUBKEY_MAX_LEN TEE_SHA256_HASH_SIZE + +/* + * huk_subkey_derive() - Derive a subkey from the hardware unique key + * @usage: Intended usage of the subkey + * @const_data: Constant data to generate different subkeys with + * the same usage + * @const_data_len: Length of constant data + * @subkey: Generated subkey + * @subkey_len: Required size of the subkey, sizes larger than + * HUK_SUBKEY_MAX_LEN are not accepted. + * + * Returns a subkey derived from the hardware unique key. Given the same + * input the same subkey is returned each time. + * Function huk_subkey_derive() is __weak to allow platform specific + * implementation. + * __huk_subkey_derive() implements the default behavior of HUK derivation. + * + * Return TEE_SUCCES on success or an error code on failure. + */ +TEE_Result huk_subkey_derive(enum huk_subkey_usage usage, + const void *const_data, size_t const_data_len, + uint8_t *subkey, size_t subkey_len); +TEE_Result __huk_subkey_derive(enum huk_subkey_usage usage, + const void *const_data, size_t const_data_len, + uint8_t *subkey, size_t subkey_len); + + +#endif /*__KERNEL_HUK_SUBKEY_H*/ diff --git a/optee_os/core/include/kernel/interrupt.h b/optee_os/core/include/kernel/interrupt.h new file mode 100644 index 0000000..c5b66a9 --- /dev/null +++ b/optee_os/core/include/kernel/interrupt.h @@ -0,0 +1,429 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2019, Linaro Limited + */ +#ifndef __KERNEL_INTERRUPT_H +#define __KERNEL_INTERRUPT_H + +#include +#include +#include +#include +#include +#include + +#define ITRF_TRIGGER_LEVEL BIT(0) +#define ITRF_SHARED BIT(1) + +struct itr_handler; + +/* + * struct itr_chip - Interrupt controller + * + * @ops Operation callback functions + * @name Controller name, for debug purpose + * @handlers Registered handlers list head + * @dt_get_irq Device tree node parsing function + */ +struct itr_chip { + const struct itr_ops *ops; + const char *name; + SLIST_HEAD(, itr_handler) handlers; + /* + * dt_get_irq - parse a device tree interrupt property + * + * @properties raw interrupt property from device tree + * @count number of elements in @properties + * @type If not NULL, output interrupt type (IRQ_TYPE_* defines) + * or IRQ_TYPE_NONE if unknown + * @prio If not NULL, output interrupt priority value or 0 if unknown + */ + int (*dt_get_irq)(const uint32_t *properties, int count, uint32_t *type, + uint32_t *prio); +}; + +/* + * struct itr_ops - Interrupt controller operations + * @add Register and configure an interrupt + * @enable Enable an interrupt + * @disable Disable an interrupt + * @mask Mask an interrupt, may be called from an interrupt context + * @unmask Unmask an interrupt, may be called from an interrupt context + * @raise_pi Raise per-cpu interrupt or NULL if not applicable + * @raise_sgi Raise a SGI or NULL if not applicable to that controller + * @set_affinity Set interrupt/cpu affinity or NULL if not applicable + * + * Handlers @enable, @disable, @mask, @unmask and @add are mandated. Handlers + * @mask and @unmask have unpaged memory contrainsts. See itr_chip_is_valid(). + */ +struct itr_ops { + void (*add)(struct itr_chip *chip, size_t it, uint32_t type, + uint32_t prio); + void (*enable)(struct itr_chip *chip, size_t it); + void (*disable)(struct itr_chip *chip, size_t it); + void (*mask)(struct itr_chip *chip, size_t it); + void (*unmask)(struct itr_chip *chip, size_t it); + void (*raise_pi)(struct itr_chip *chip, size_t it); + void (*raise_sgi)(struct itr_chip *chip, size_t it, + uint8_t cpu_mask); + void (*set_affinity)(struct itr_chip *chip, size_t it, + uint8_t cpu_mask); +}; + +/* Interrupt handler return value */ +enum itr_return { + ITRR_NONE, + ITRR_HANDLED, +}; + +/* Interrupt handler signature */ +typedef enum itr_return (*itr_handler_t)(struct itr_handler *h); + +/* + * struct itr_handler - Interrupt handler reference + * @it Interrupt number + * @flags Property bit flags (ITRF_*) or 0 + * @data Private data for that interrupt handler + * @chip Interrupt controller chip device + * @link Reference in controller handler list + */ +struct itr_handler { + size_t it; + uint32_t flags; + itr_handler_t handler; + void *data; + struct itr_chip *chip; + SLIST_ENTRY(itr_handler) link; +}; + +#define ITR_HANDLER(_chip, _itr_num, _flags, _fn, _priv) \ + ((struct itr_handler){ \ + .chip = (_chip), .it = (_itr_num), .flags = (_flags), \ + .handler = (_fn), .data = (_priv), \ + }) + +/* + * Return true only if interrupt chip provides required handlers + * @chip: Interrupt controller reference + */ +static inline bool itr_chip_is_valid(struct itr_chip *chip) +{ + return chip && is_unpaged(chip) && chip->ops && + is_unpaged((void *)chip->ops) && + chip->ops->mask && is_unpaged(chip->ops->mask) && + chip->ops->unmask && is_unpaged(chip->ops->unmask) && + chip->ops->enable && chip->ops->disable && + chip->ops->add; +} + +/* + * Initialise an interrupt controller handle + * @chip Interrupt controller + */ +TEE_Result itr_chip_init(struct itr_chip *chip); + +/* + * Initialise main interrupt controller driver + * @data Main controller main data reference to register + */ +void interrupt_main_init(struct itr_chip *data); + +/* Retrieve main interrupt controller reference */ +struct itr_chip *interrupt_get_main_chip(void); + +#ifdef CFG_DT +/* + * Get the DT interrupt property at @node. In the DT an interrupt property can + * specify additional information which can be retrieved with @type and @prio. + * + * @fdt reference to the Device Tree + * @node is the node offset to read the interrupt property from + * @type interrupt type (IRQ_TYPE_* defines) if specified by interrupt property + * or IRQ_TYPE_NONE if not. Can be NULL if not needed + * @prio interrupt priority if specified by interrupt property or 0 if not. Can + * be NULL if not needed + * + * Returns the interrupt number if value >= 0 + * otherwise DT_INFO_INVALID_INTERRUPT + */ +int dt_get_irq_type_prio(const void *fdt, int node, uint32_t *type, + uint32_t *prio); + +/* + * Get the DT interrupt property at @node + */ +static inline int dt_get_irq(const void *fdt, int node) +{ + return dt_get_irq_type_prio(fdt, node, NULL, NULL); +} +#endif + +struct itr_handler *itr_alloc_add_type_prio(size_t it, itr_handler_t handler, + uint32_t flags, void *data, + uint32_t type, uint32_t prio); +void itr_free(struct itr_handler *hdl); +void itr_add_type_prio(struct itr_handler *handler, uint32_t type, + uint32_t prio); +void itr_enable(size_t it); +void itr_disable(size_t it); +/* raise the Peripheral Interrupt corresponding to the interrupt ID */ +void itr_raise_pi(size_t it); +/* + * raise the Software Generated Interrupt corresponding to the interrupt ID, + * the cpu_mask represents which cpu interface to forward. + */ +void itr_raise_sgi(size_t it, uint8_t cpu_mask); +/* + * let corresponding interrupt forward to the cpu interface + * according to the cpu_mask. + */ +void itr_set_affinity(size_t it, uint8_t cpu_mask); + +/* + * __weak overridable function which is called when a secure interrupt is + * received. The default function calls panic() immediately, platforms which + * expects to receive secure interrupts should override this function. + */ +void interrupt_main_handler(void); + +static inline void itr_add(struct itr_handler *handler) +{ + itr_add_type_prio(handler, IRQ_TYPE_NONE, 0); +} + +static inline struct itr_handler *itr_alloc_add(size_t it, + itr_handler_t handler, + uint32_t flags, void *data) +{ + return itr_alloc_add_type_prio(it, handler, flags, data, IRQ_TYPE_NONE, + 0); +} + +/* + * Interrupt controller chip API functions + */ + +/* + * interrupt_call_handlers() - Call registered handlers for an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + * + * This function is called from an interrupt context by a primary interrupt + * handler. This function calls the handlers registered for that interrupt. + * If interrupt is not handled, it is masked. + */ +void interrupt_call_handlers(struct itr_chip *chip, size_t itr_num); + +/* + * interrupt_mask() - Mask an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + * + * This function may be called in interrupt context + */ +static inline void interrupt_mask(struct itr_chip *chip, size_t itr_num) +{ + chip->ops->mask(chip, itr_num); +} + +/* + * interrupt_unmask() - Unmask an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + * + * This function may be called in interrupt context + */ +static inline void interrupt_unmask(struct itr_chip *chip, size_t itr_num) +{ + chip->ops->unmask(chip, itr_num); +} + +/* + * interrupt_enable() - Enable an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + */ +static inline void interrupt_enable(struct itr_chip *chip, size_t itr_num) +{ + chip->ops->enable(chip, itr_num); +} + +/* + * interrupt_disable() - Disable an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + */ +static inline void interrupt_disable(struct itr_chip *chip, size_t itr_num) +{ + chip->ops->disable(chip, itr_num); +} + +/* + * interrupt_can_raise_pi() - Return whether controller embeds raise_pi + * @chip Interrupt controller + */ +static inline bool interrupt_can_raise_pi(struct itr_chip *chip) +{ + return chip->ops->raise_pi; +} + +/* + * interrupt_can_raise_sgi() - Return whether controller embeds raise_sgi + * @chip Interrupt controller + */ +static inline bool interrupt_can_raise_sgi(struct itr_chip *chip) +{ + return chip->ops->raise_sgi; +} + +/* + * interrupt_can_set_affinity() - Return whether controller embeds set_affinity + * @chip Interrupt controller + */ +static inline bool interrupt_can_set_affinity(struct itr_chip *chip) +{ + return chip->ops->set_affinity; +} + +/* + * interrupt_raise_pi() - Raise a peripheral interrupt of a controller + * @chip Interrupt controller + * @itr_num Interrupt number to raise + */ +static inline void interrupt_raise_pi(struct itr_chip *chip, size_t itr_num) +{ + assert(interrupt_can_raise_pi(chip)); + chip->ops->raise_pi(chip, itr_num); +} + +/* + * interrupt_raise_sgi() - Raise a software generiated interrupt of a controller + * @chip Interrupt controller + * @itr_num Interrupt number to raise + * @cpu_mask Mask of the CPUs targeted by the interrupt + */ +static inline void interrupt_raise_sgi(struct itr_chip *chip, size_t itr_num, + uint8_t cpu_mask) +{ + assert(interrupt_can_raise_sgi(chip)); + chip->ops->raise_sgi(chip, itr_num, cpu_mask); +} + +/* + * interrupt_set_affinity() - Set CPU affinity for a controller interrupt + * @chip Interrupt controller + * @itr_num Interrupt number to raise + * @cpu_mask Mask of the CPUs targeted by the interrupt + */ +static inline void interrupt_set_affinity(struct itr_chip *chip, size_t itr_num, + uint8_t cpu_mask) +{ + assert(interrupt_can_set_affinity(chip)); + chip->ops->set_affinity(chip, itr_num, cpu_mask); +} + +/* + * interrupt_configure() - Configure an interrupt in an interrupt controller + * @chip Interrupt controller + * @itr_num Interrupt number + * @type Interrupt trigger type (IRQ_TYPE_* defines) or IRQ_TYPE_NONE + * @prio Interrupt priority or 0 + * + * Interrupt consumers that get their interrupt from the DT do not need to + * call interrupt_configure() since the interrupt configuration has already + * been done by interrupt controller based on the DT bidings. + */ +TEE_Result interrupt_configure(struct itr_chip *chip, size_t itr_num, + uint32_t type, uint32_t prio); + +/* + * interrupt_add_and_configure_handler() - Register and configure a handler + * @hdl Interrupt handler to register + * @type Interrupt trigger type (IRQ_TYPE_* defines) or IRQ_TYPE_NONE + * @prio Interrupt priority or 0 + */ +TEE_Result interrupt_add_configure_handler(struct itr_handler *hdl, + uint32_t type, uint32_t prio); + +/* + * interrupt_add_handler() - Register an interrupt handler + * @hdl Interrupt handler to register + * + * This helper function assumes interrupt type is set to IRQ_TYPE_NONE + * and interrupt priority to 0. + */ +static inline TEE_Result interrupt_add_handler(struct itr_handler *hdl) +{ + return interrupt_add_configure_handler(hdl, IRQ_TYPE_NONE, 0); +} + +/* + * interrupt_add_handler_with_chip() - Register an interrupt handler providing + * the interrupt chip reference in specific argument @chip. + * @chip Interrupt controller + * @h Interrupt handler to register + */ +static inline TEE_Result interrupt_add_handler_with_chip(struct itr_chip *chip, + struct itr_handler *h) +{ + h->chip = chip; + return interrupt_add_handler(h); +} + +/* + * interrupt_remove_handler() - Remove a registered interrupt handler + * @hdl Interrupt handler to remove + * + * This function is the counterpart of interrupt_add_handler(). + * This function may panic on non-NULL invalid @hdl reference. + */ +void interrupt_remove_handler(struct itr_handler *hdl); + +/* + * interrupt_alloc_add_conf_handler() - Allocate, configure, register a handler + * @chip Interrupt controller + * @itr_num Interrupt number + * @handler Interrupt handler to register + * @flags Bitmask flag ITRF_* + * @data Private data reference passed to @handler + * @type Interrupt trigger type (IRQ_TYPE_* defines) or IRQ_TYPE_NONE + * @prio Interrupt priority or 0 + * @out_hdl NULL or output pointer to allocated struct itr_handler + */ +TEE_Result interrupt_alloc_add_conf_handler(struct itr_chip *chip, + size_t it_num, + itr_handler_t handler, + uint32_t flags, void *data, + uint32_t type, uint32_t prio, + struct itr_handler **out_hdl); + +/* + * interrupt_alloc_add_handler() - Allocate and register an interrupt handler + * @chip Interrupt controller + * @itr_num Interrupt number + * @handler Interrupt handler to register + * @flags Bitmask flag ITRF_* + * @data Private data reference passed to @handler + * @out_hdl NULL or output pointer to allocated struct itr_handler + */ +static inline TEE_Result interrupt_alloc_add_handler(struct itr_chip *chip, + size_t it_num, + itr_handler_t handler, + uint32_t flags, + void *data, + struct itr_handler **hdl) +{ + return interrupt_alloc_add_conf_handler(chip, it_num, handler, flags, + data, IRQ_TYPE_NONE, 0, hdl); +} + +/* + * interrupt_remove_free_handler() - Remove/free a registered interrupt handler + * @hdl Interrupt handler to remove and free + * + * This function is the counterpart of interrupt_alloc_add_handler() + * and interrupt_alloc_add_conf_handler(). + * This function may panic on non-NULL invalid @hdl reference. + */ +void interrupt_remove_free_handler(struct itr_handler *hdl); +#endif /*__KERNEL_INTERRUPT_H*/ diff --git a/optee_os/core/include/kernel/ldelf_loader.h b/optee_os/core/include/kernel/ldelf_loader.h new file mode 100644 index 0000000..4d5e92c --- /dev/null +++ b/optee_os/core/include/kernel/ldelf_loader.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Arm Limited + */ + +#include +#include +#include + +#ifndef KERNEL_LDELF_LOADER_H +#define KERNEL_LDELF_LOADER_H + +TEE_Result ldelf_load_ldelf(struct user_mode_ctx *uctx); +TEE_Result ldelf_init_with_ldelf(struct ts_session *sess, + struct user_mode_ctx *uctx); +TEE_Result ldelf_dump_state(struct user_mode_ctx *uctx); +TEE_Result ldelf_dump_ftrace(struct user_mode_ctx *uctx, + void *buf, size_t *blen); +TEE_Result ldelf_dlopen(struct user_mode_ctx *uctx, TEE_UUID *uuid, + uint32_t flags); +TEE_Result ldelf_dlsym(struct user_mode_ctx *uctx, TEE_UUID *uuid, + const char *sym, size_t symlen, vaddr_t *val); + +#endif /* KERNEL_LDELF_LOADER_H */ diff --git a/optee_os/core/include/kernel/ldelf_syscalls.h b/optee_os/core/include/kernel/ldelf_syscalls.h new file mode 100644 index 0000000..5dd89a7 --- /dev/null +++ b/optee_os/core/include/kernel/ldelf_syscalls.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2019, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include +#include +#include +#include +#include +#include + +#ifndef KERNEL_LDELF_SYSCALLS_H +#define KERNEL_LDELF_SYSCALLS_H + +struct system_ctx { + struct handle_db db; + const struct ts_store_ops *store_op; +}; + +TEE_Result ldelf_syscall_map_zi(vaddr_t *va, size_t num_bytes, size_t pad_begin, + size_t pad_end, unsigned long flags); +TEE_Result ldelf_syscall_unmap(vaddr_t va, size_t num_bytes); +TEE_Result ldelf_syscall_open_bin(const TEE_UUID *uuid, size_t uuid_size, + uint32_t *handle); +TEE_Result ldelf_syscall_close_bin(unsigned long handle); +TEE_Result ldelf_syscall_map_bin(vaddr_t *va, size_t num_bytes, + unsigned long handle, size_t offs_bytes, + size_t pad_begin, size_t pad_end, + unsigned long flags); +TEE_Result ldelf_syscall_copy_from_bin(void *dst, size_t offs, size_t num_bytes, + unsigned long handle); +TEE_Result ldelf_syscall_set_prot(unsigned long va, size_t num_bytes, + unsigned long flags); +TEE_Result ldelf_syscall_remap(unsigned long old_va, vaddr_t *new_va, + size_t num_bytes, size_t pad_begin, + size_t pad_end); +TEE_Result ldelf_syscall_gen_rnd_num(void *buf, size_t num_bytes); +void ldelf_sess_cleanup(struct ts_session *sess); + +#endif /* KERNEL_LDELF_SYSCALLS_H */ diff --git a/optee_os/core/include/kernel/linker.h b/optee_os/core/include/kernel/linker.h new file mode 100644 index 0000000..949ffd8 --- /dev/null +++ b/optee_os/core/include/kernel/linker.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ +#ifndef __KERNEL_LINKER_H +#define __KERNEL_LINKER_H + +#include +#include + +/* + * Symbols exported by the link script. + */ + +#ifdef ARM32 + +/* + * These addresses will be the start or end of the exception binary search + * index table (.ARM.exidx section) + */ +extern const uint8_t __exidx_start[]; +extern const uint8_t __exidx_end[]; +extern const uint8_t __extab_start[]; +extern const uint8_t __extab_end[]; + +#endif + +#define VCORE_UNPG_RX_PA ((unsigned long)__vcore_unpg_rx_start) +#define VCORE_UNPG_RX_SZ ((size_t)(__vcore_unpg_rx_end - \ + __vcore_unpg_rx_start)) +#define VCORE_UNPG_RO_PA ((unsigned long)__vcore_unpg_ro_start) +#define VCORE_UNPG_RO_SZ ((size_t)(__vcore_unpg_ro_end - \ + __vcore_unpg_ro_start)) +#define VCORE_UNPG_RW_PA ((unsigned long)__vcore_unpg_rw_start) +#define VCORE_UNPG_RW_SZ ((size_t)(__vcore_unpg_rw_end - \ + __vcore_unpg_rw_start)) +#define VCORE_NEX_RW_PA ((unsigned long)__vcore_nex_rw_start) +#define VCORE_NEX_RW_SZ ((size_t)(__vcore_nex_rw_end - \ + __vcore_nex_rw_start)) +#define VCORE_INIT_RX_PA ((unsigned long)__vcore_init_rx_start) +#define VCORE_INIT_RX_SZ ((size_t)(__vcore_init_rx_end - \ + __vcore_init_rx_start)) +#define VCORE_INIT_RO_PA ((unsigned long)__vcore_init_ro_start) +#define VCORE_INIT_RO_SZ ((size_t)(__vcore_init_ro_end - \ + __vcore_init_ro_start)) + +#define VCORE_START_VA ((vaddr_t)__text_start) + +#define EMIT_SECTION_INFO_SYMBOLS(section_name) \ + extern const uint8_t __vcore_ ## section_name ## _start[]; \ + extern const uint8_t __vcore_ ## section_name ## _end[]; \ + extern const uint8_t __vcore_ ## section_name ## _size[] + +EMIT_SECTION_INFO_SYMBOLS(unpg_rx); +EMIT_SECTION_INFO_SYMBOLS(unpg_ro); +EMIT_SECTION_INFO_SYMBOLS(unpg_rw); +EMIT_SECTION_INFO_SYMBOLS(nex_rw); +EMIT_SECTION_INFO_SYMBOLS(init_ro); +EMIT_SECTION_INFO_SYMBOLS(init_rx); + +#undef EMIT_SECTION_INFO_SYMBOLS + +extern const uint8_t __text_start[]; +extern const uint8_t __text_data_start[]; +extern const uint8_t __text_data_end[]; +extern const uint8_t __text_end[]; +extern const uint8_t __end[]; + +extern const uint8_t __identity_map_init_start[]; +extern const uint8_t __identity_map_init_end[]; + +extern uint8_t __data_start[]; +extern const uint8_t __data_end[]; +extern const uint8_t __rodata_start[]; +extern const uint8_t __rodata_end[]; +extern const uint8_t __bss_start[]; +extern const uint8_t __bss_end[]; +extern const uint8_t __nozi_start[]; +extern const uint8_t __nozi_end[]; +extern const uint8_t __nozi_stack_start[]; +extern const uint8_t __nozi_stack_end[]; +extern const uint8_t __init_start[]; +extern const uint8_t __init_end[]; + +extern uint8_t __heap1_start[]; +extern const uint8_t __heap1_end[]; +extern uint8_t __heap2_start[]; +extern const uint8_t __heap2_end[]; + +extern uint8_t __nex_heap_start[]; +extern const uint8_t __nex_heap_end[]; + +extern const uint8_t __pageable_part_start[]; +extern const uint8_t __pageable_part_end[]; +extern const uint8_t __pageable_start[]; +extern const uint8_t __pageable_end[]; + +extern const uint8_t __rodata_init_start[]; +extern const uint8_t __rodata_init_end[]; +extern const uint8_t __rodata_pageable_start[]; +extern const uint8_t __rodata_pageable_end[]; +extern const uint8_t __text_init_start[]; +extern const uint8_t __text_init_end[]; +extern const uint8_t __text_pageable_start[]; +extern const uint8_t __text_pageable_end[]; + +#define ASAN_SHADOW_PA ((paddr_t)(vaddr_t)__asan_shadow_start) +#define ASAN_SHADOW_SZ ((size_t)__asan_shadow_size) +extern const uint8_t __asan_shadow_start[]; +extern const uint8_t __asan_shadow_end[]; +extern const uint8_t __asan_shadow_size[]; + +#define ASAN_MAP_PA ((paddr_t)(vaddr_t)__asan_map_start) +#define ASAN_MAP_SZ ((size_t)__asan_map_size) +extern const uint8_t __asan_map_start[]; +extern const uint8_t __asan_map_end[]; +extern const uint8_t __asan_map_size[]; + +extern const vaddr_t __ctor_list; +extern const vaddr_t __ctor_end; + +/* Generated by core/arch/$(ARCH)/kernel/link.mk */ +extern const char core_v_str[]; + +#endif /*__KERNEL_LINKER_H*/ + diff --git a/optee_os/core/include/kernel/lockdep.h b/optee_os/core/include/kernel/lockdep.h new file mode 100644 index 0000000..e6d786d --- /dev/null +++ b/optee_os/core/include/kernel/lockdep.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ + +#ifndef __KERNEL_LOCKDEP_H +#define __KERNEL_LOCKDEP_H + +#include +#include +#include +#include +#include +#include + +/* + * Lock graph. If node A has an edge to node B, then A was locked before B in + * the same thread of execution. + */ + +struct lockdep_edge { + struct lockdep_node *to; + uintptr_t thread_id; + vaddr_t *call_stack_from; + vaddr_t *call_stack_to; + STAILQ_ENTRY(lockdep_edge) link; +}; + +STAILQ_HEAD(lockdep_edge_head, lockdep_edge); + +struct lockdep_node { + uintptr_t lock_id; /* For instance, address of actual lock object */ + struct lockdep_edge_head edges; + TAILQ_ENTRY(lockdep_node) link; + uint8_t flags; /* Used temporarily when walking the graph */ +}; + +TAILQ_HEAD(lockdep_node_head, lockdep_node); + +/* Per-thread queue of currently owned locks (point to nodes in the graph) */ + +struct lockdep_lock { + struct lockdep_node *node; + vaddr_t *call_stack; + TAILQ_ENTRY(lockdep_lock) link; +}; + +TAILQ_HEAD(lockdep_lock_head, lockdep_lock); + +#ifdef CFG_LOCKDEP + +/* + * Functions used internally and for testing the algorithm. Actual locking code + * should use the wrappers below (which panic in case of error). + */ +TEE_Result __lockdep_lock_acquire(struct lockdep_node_head *graph, + struct lockdep_lock_head *owned, + uintptr_t id); +TEE_Result __lockdep_lock_tryacquire(struct lockdep_node_head *graph, + struct lockdep_lock_head *owned, + uintptr_t id); +TEE_Result __lockdep_lock_release(struct lockdep_lock_head *owned, + uintptr_t id); + +/* Delete all elements in @graph */ +void lockdep_graph_delete(struct lockdep_node_head *graph); + +/* Delete all elements in @queue */ +void lockdep_queue_delete(struct lockdep_lock_head *queue); + +/* + * Acquire lock @id, while already holding the locks in @owned. + * @owned represent the caller; there should be one instance per thread of + * execution. @graph is the directed acyclic graph (DAG) to be used for + * potential deadlock detection; use the same @graph for all the locks of the + * same type as lock @id. + * + * This function will panic() if the acquire operation would result in a lock + * hierarchy violation (potential deadlock). + */ +static inline void lockdep_lock_acquire(struct lockdep_node_head *graph, + struct lockdep_lock_head *owned, + uintptr_t id) +{ + TEE_Result res = __lockdep_lock_acquire(graph, owned, id); + + if (res) { + EMSG("lockdep: error %#" PRIx32, res); + panic(); + } +} + +/* + * Non-blocking acquire lock @id, while already holding the locks in @owned. + * @owned represent the caller; there should be one instance per thread of + * execution. @graph is the directed acyclic graph (DAG) to be used for + * potential deadlock detection; use the same @graph for all the locks of the + * same type as lock @id. + */ +static inline void lockdep_lock_tryacquire(struct lockdep_node_head *graph, + struct lockdep_lock_head *owned, + uintptr_t id) +{ + TEE_Result res = __lockdep_lock_tryacquire(graph, owned, id); + + if (res) { + EMSG("lockdep: error %#" PRIx32, res); + panic(); + } +} + +/* + * Release lock @id. The lock is removed from @owned. + * + * This function will panic() if the lock is not held by the caller. + */ +static inline void lockdep_lock_release(struct lockdep_lock_head *owned, + uintptr_t id) +{ + TEE_Result res = __lockdep_lock_release(owned, id); + + if (res) { + EMSG("lockdep: error %#" PRIx32, res); + panic(); + } +} + +/* + * Destroy lock @id in @graph. The lock is freed. + */ +void lockdep_lock_destroy(struct lockdep_node_head *graph, uintptr_t id); + +/* Initialize lockdep for mutex objects (kernel/mutex.h) */ +void mutex_lockdep_init(void); + +#else /* CFG_LOCKDEP */ + +static inline void lockdep_lock_acquire(struct lockdep_node_head *g __unused, + struct lockdep_lock_head *o __unused, + uintptr_t id __unused) +{} + +static inline void lockdep_lock_release(struct lockdep_lock_head *o __unused, + uintptr_t id __unused) +{} + +static inline void +lockdep_lock_destroy(struct lockdep_node_head *graph __unused, + uintptr_t id __unused) +{} + +static inline void mutex_lockdep_init(void) +{} + +#endif /* !CFG_LOCKDEP */ + +#endif /* !__KERNEL_LOCKDEP_H */ diff --git a/optee_os/core/include/kernel/misc.h b/optee_os/core/include/kernel/misc.h new file mode 100644 index 0000000..35cfa5f --- /dev/null +++ b/optee_os/core/include/kernel/misc.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef KERNEL_MISC_H +#define KERNEL_MISC_H + +#include +#include +#include +#include + +size_t __get_core_pos(void); + +static inline size_t __noprof get_core_pos(void) +{ + /* + * Foreign interrupts must be disabled before playing with current + * core since we otherwise may be rescheduled to a different core. + */ + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + return __get_core_pos(); +} + +#endif /*KERNEL_MISC_H*/ diff --git a/optee_os/core/include/kernel/msg_param.h b/optee_os/core/include/kernel/msg_param.h new file mode 100644 index 0000000..841c3bf --- /dev/null +++ b/optee_os/core/include/kernel/msg_param.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, EPAM Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef KERNEL_MSG_PARAM_H +#define KERNEL_MSG_PARAM_H + +#include +#include +#include +#include +#include +#include + +/** + * msg_param_mobj_from_noncontig() - construct mobj from non-contiguous + * list of pages. + * + * @buf_ptr - optee_msg_param.u.tmem.buf_ptr value + * @size - optee_msg_param.u.tmem.size value + * @shm_ref - optee_msg_param.u.tmem.shm_ref value + * @map_buffer - true if buffer needs to be mapped into OP-TEE address space + * + * return: + * mobj or NULL on error + */ +#ifdef CFG_CORE_DYN_SHM +struct mobj *msg_param_mobj_from_noncontig(paddr_t buf_ptr, size_t size, + uint64_t shm_ref, bool map_buffer); +#else +static inline struct mobj * +msg_param_mobj_from_noncontig(paddr_t buf_ptr __unused, size_t size __unused, + uint64_t shm_ref __unused, + bool map_buffer __unused) +{ + return NULL; +} +#endif + +/** + * msg_param_attr_is_tmem - helper functions that cheks if attribute is tmem + * + * @attr - attribute to check + * + * return: + * corresponding size field + */ +static inline bool msg_param_attr_is_tmem(uint64_t attr) +{ + switch (attr & OPTEE_MSG_ATTR_TYPE_MASK) { + case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: + return true; + default: + return false; + } +} + +#endif /*KERNEL_MSG_PARAM_H*/ diff --git a/optee_os/core/include/kernel/mutex.h b/optee_os/core/include/kernel/mutex.h new file mode 100644 index 0000000..e4c470b --- /dev/null +++ b/optee_os/core/include/kernel/mutex.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2017, Linaro Limited + */ +#ifndef KERNEL_MUTEX_H +#define KERNEL_MUTEX_H + +#include +#include +#include +#include + +struct mutex { + unsigned spin_lock; /* used when operating on this struct */ + struct wait_queue wq; + short state; /* -1: write, 0: unlocked, > 0: readers */ +}; + +#define MUTEX_INITIALIZER { .wq = WAIT_QUEUE_INITIALIZER } + +struct recursive_mutex { + struct mutex m; /* used when lock_depth goes 0 -> 1 or 1 -> 0 */ + short int owner; + struct refcount lock_depth; +}; + +#define RECURSIVE_MUTEX_INITIALIZER { .m = MUTEX_INITIALIZER, \ + .owner = THREAD_ID_INVALID } + +TAILQ_HEAD(mutex_head, mutex); + +void mutex_init(struct mutex *m); +void mutex_destroy(struct mutex *m); + +void mutex_init_recursive(struct recursive_mutex *m); +void mutex_destroy_recursive(struct recursive_mutex *m); +unsigned int mutex_get_recursive_lock_depth(struct recursive_mutex *m); + +#ifdef CFG_MUTEX_DEBUG +void mutex_unlock_debug(struct mutex *m, const char *fname, int lineno); +#define mutex_unlock(m) mutex_unlock_debug((m), __FILE__, __LINE__) + +void mutex_lock_debug(struct mutex *m, const char *fname, int lineno); +#define mutex_lock(m) mutex_lock_debug((m), __FILE__, __LINE__) + +bool mutex_trylock_debug(struct mutex *m, const char *fname, int lineno); +#define mutex_trylock(m) mutex_trylock_debug((m), __FILE__, __LINE__) + +void mutex_read_unlock_debug(struct mutex *m, const char *fname, int lineno); +#define mutex_read_unlock(m) mutex_read_unlock_debug((m), __FILE__, __LINE__) + +void mutex_read_lock_debug(struct mutex *m, const char *fname, int lineno); +#define mutex_read_lock(m) mutex_read_lock_debug((m), __FILE__, __LINE__) + +bool mutex_read_trylock_debug(struct mutex *m, const char *fname, int lineno); +#define mutex_read_trylock(m) mutex_read_trylock_debug((m), __FILE__, __LINE__) + +void mutex_unlock_recursive_debug(struct recursive_mutex *m, const char *fname, + int lineno); +#define mutex_unlock_recursive(m) mutex_unlock_recursive_debug((m), __FILE__, \ + __LINE__) + +void mutex_lock_recursive_debug(struct recursive_mutex *m, const char *fname, + int lineno); +#define mutex_lock_recursive(m) mutex_lock_recursive_debug((m), __FILE__, \ + __LINE__) +#else +void mutex_unlock(struct mutex *m); +void mutex_lock(struct mutex *m); +bool mutex_trylock(struct mutex *m); +void mutex_read_unlock(struct mutex *m); +void mutex_read_lock(struct mutex *m); +bool mutex_read_trylock(struct mutex *m); + +void mutex_unlock_recursive(struct recursive_mutex *m); +void mutex_lock_recursive(struct recursive_mutex *m); +#endif + +struct condvar { + unsigned spin_lock; + struct mutex *m; +}; +#define CONDVAR_INITIALIZER { .m = NULL } + +void condvar_init(struct condvar *cv); +void condvar_destroy(struct condvar *cv); + +#ifdef CFG_MUTEX_DEBUG +void condvar_signal_debug(struct condvar *cv, const char *fname, int lineno); +#define condvar_signal(cv) condvar_signal_debug((cv), __FILE__, __LINE__) + +void condvar_broadcast_debug(struct condvar *cv, const char *fname, int lineno); +#define condvar_broadcast(cv) condvar_broadcast_debug((cv), __FILE__, __LINE__) + +void condvar_wait_debug(struct condvar *cv, struct mutex *m, + const char *fname, int lineno); +#define condvar_wait(cv, m) condvar_wait_debug((cv), (m), __FILE__, __LINE__) +#else +void condvar_signal(struct condvar *cv); +void condvar_broadcast(struct condvar *cv); +void condvar_wait(struct condvar *cv, struct mutex *m); +#endif + +#endif /*KERNEL_MUTEX_H*/ + diff --git a/optee_os/core/include/kernel/notif.h b/optee_os/core/include/kernel/notif.h new file mode 100644 index 0000000..71b53d8 --- /dev/null +++ b/optee_os/core/include/kernel/notif.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Linaro Limited + */ + +#ifndef __KERNEL_NOTIF_H +#define __KERNEL_NOTIF_H + +#include +#include +#include +#include +#include + +/* + * Notification values are divided into two kinds, asynchronous and + * synchronous, where the asynchronous has the lowest values. + * They are ordered as: + * 0 Do bottom half + * 1..NOTIF_ASYNC_MAX Free for signalling in PTAs and should be + * allocated with notif_alloc_async_value() + * NOTIF_SYNC_VALUE_BASE.. Used as NOTIF_SYNC_VALUE_BASE + thread_id + * NOTIF_VALUE_MAX for mutex and condvar wait/wakeup + * + * Any value can be signalled with notif_send_sync() while only the ones + * <= NOTIF_ASYNC_VALUE_MAX can be signalled with notif_send_async(). + */ + +#if defined(CFG_CORE_ASYNC_NOTIF) +#define NOTIF_ASYNC_VALUE_MAX U(63) +#define NOTIF_SYNC_VALUE_BASE (NOTIF_ASYNC_VALUE_MAX + U(1)) +#else +#define NOTIF_SYNC_VALUE_BASE 0 +#endif + +#define NOTIF_VALUE_MAX (NOTIF_SYNC_VALUE_BASE + \ + CFG_NUM_THREADS) + +#define NOTIF_VALUE_DO_BOTTOM_HALF 0 + +/* + * enum notif_event - Notification of an event + * @NOTIF_EVENT_STARTED: Delivered in an atomic context to inform + * drivers that normal world has enabled + * asynchronous notifications. + * @NOTIF_EVENT_DO_BOTTOM_HALF: Delivered in a yielding context to let a + * driver do bottom half processing. + * @NOTIF_EVENT_STOPPED: Delivered in a yielding contest to inform + * drivers that normal world is about to disable + * asynchronous notifications. + * + * Once a driver has received a @NOTIF_EVENT_STARTED asynchronous notifications + * driving the @NOTIF_EVENT_DO_BOTTOM_HALF deliveries is enabled. + * + * In case a @NOTIF_EVENT_STOPPED is received there will be no more + * @NOTIF_EVENT_DO_BOTTOM_HALF events delivered, until @NOTIF_EVENT_STARTED + * has been delivered again. + * + * Note that while a @NOTIF_EVENT_STOPPED is being delivered at the same + * time may a @NOTIF_EVENT_STARTED be delivered again so a driver is + * required to sychronize accesses to its internal state. + */ +enum notif_event { + NOTIF_EVENT_STARTED, + NOTIF_EVENT_DO_BOTTOM_HALF, + NOTIF_EVENT_STOPPED, +}; + +/* + * struct notif_driver - Registration of driver notification + * @atomic_cb: A callback called in an atomic context from + * notif_deliver_atomic_event(). Currently only used to + * signal @NOTIF_EVENT_STARTED. + * @yielding_cb: A callback called in a yielding context from + * notif_deliver_event(). Currently only used to signal + * @NOTIF_EVENT_DO_BOTTOM_HALF and @NOTIF_EVENT_STOPPED. + * + * A atomic context means that interrupts are masked and a common spinlock + * is held. Calls via @atomic_cb are only atomic with regards to each + * other, other CPUs may execute yielding calls or even receive interrupts. + * + * A yielding context means that the function is executing in a normal + * threaded context allowing RPC and synchronization with other thread + * using mutexes and condition variables. + */ +struct notif_driver { + void (*atomic_cb)(struct notif_driver *ndrv, enum notif_event ev); + void (*yielding_cb)(struct notif_driver *ndrv, enum notif_event ev); + SLIST_ENTRY(notif_driver) link; +}; + +#if defined(CFG_CORE_ASYNC_NOTIF) +bool notif_async_is_started(void); +#else +static inline bool notif_async_is_started(void) +{ + return false; +} +#endif + +TEE_Result notif_alloc_async_value(uint32_t *value); +void notif_free_async_value(uint32_t value); + +/* + * Wait in normal world for a value to be sent by notif_send() + */ +TEE_Result notif_wait(uint32_t value); + +/* + * Send an asynchronous value, note that it must be <= NOTIF_ASYNC_VALUE_MAX + */ +#if defined(CFG_CORE_ASYNC_NOTIF) +void notif_send_async(uint32_t value); +#else +static inline void notif_send_async(uint32_t value __unused) +{ +} +#endif + +/* + * Send a sychronous value, note that it must be <= NOTIF_VALUE_MAX. The + * notification is synchronous even if the value happens to belong in the + * asynchronous range. + */ +TEE_Result notif_send_sync(uint32_t value); + +/* + * Called by device drivers. + */ +#if defined(CFG_CORE_ASYNC_NOTIF) +void notif_register_driver(struct notif_driver *ndrv); +void notif_unregister_driver(struct notif_driver *ndrv); +#else +static inline void notif_register_driver(struct notif_driver *ndrv __unused) +{ +} + +static inline void notif_unregister_driver(struct notif_driver *ndrv __unused) +{ +} +#endif + +/* This is called from a fast call */ +#if defined(CFG_CORE_ASYNC_NOTIF) +uint32_t notif_get_value(bool *value_valid, bool *value_pending); +#else +static inline uint32_t notif_get_value(bool *value_valid, bool *value_pending) +{ + *value_valid = false; + *value_pending = false; + return UINT32_MAX; +} +#endif + +/* + * These are called from yielding calls + */ +#if defined(CFG_CORE_ASYNC_NOTIF) +void notif_deliver_atomic_event(enum notif_event ev); +void notif_deliver_event(enum notif_event ev); +#else +static inline void notif_deliver_atomic_event(enum notif_event ev __unused) +{ +} + +static inline void notif_deliver_event(enum notif_event ev __unused) +{ +} +#endif + +#endif /*__KERNEL_NOTIF_H*/ diff --git a/optee_os/core/include/kernel/nv_counter.h b/optee_os/core/include/kernel/nv_counter.h new file mode 100644 index 0000000..bdf071e --- /dev/null +++ b/optee_os/core/include/kernel/nv_counter.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023, Linaro Limited + */ + +#ifndef __KERNEL_NV_COUNTER_H +#define __KERNEL_NV_COUNTER_H + +#include +#include + +TEE_Result nv_counter_get_ree_fs(uint32_t *value); +TEE_Result nv_counter_incr_ree_fs_to(uint32_t value); + +#endif /*__KERNEL_NV_COUNTER_H*/ diff --git a/optee_os/core/include/kernel/panic.h b/optee_os/core/include/kernel/panic.h new file mode 100644 index 0000000..9dc38dc --- /dev/null +++ b/optee_os/core/include/kernel/panic.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef KERNEL_PANIC_H +#define KERNEL_PANIC_H + +#include + +/* debug disabled => __FILE__, ... and panic message are not built. */ +#if defined(CFG_TEE_CORE_DEBUG) +#define __panic(str) __do_panic(__FILE__, __LINE__, __func__, str) +#else +#define __panic(str) __do_panic((void *)0, 0, (void *)0, str) +#endif + +void __do_panic(const char *file, const int line, const char *func, + const char *msg) __noreturn; + +/* + * Suppress GCC warning on expansion of the panic() macro with no argument: + * 'ISO C99 requires at least one argument for the "..." in a variadic macro' + * Occurs when '-pedantic' is combined with '-std=gnu99'. + * Suppression applies only to this file and the expansion of macros defined in + * this file. + */ +#pragma GCC system_header + +/* panic() can get a string or no argument */ +#define _panic0() __panic((void *)0) +#define _panic1(s) __panic(s) +#define _panic_fn(a, b, name, ...) name +#define panic(...) _panic_fn("", ##__VA_ARGS__, _panic1, _panic0)(__VA_ARGS__) + +/* + * Weak function used in __do_panic() to put the current CPU on hold. + * If no arch-specific override is provided, defaults to a busy loop. + */ +void cpu_idle(void); + +#endif /*KERNEL_PANIC_H*/ diff --git a/optee_os/core/include/kernel/pm.h b/optee_os/core/include/kernel/pm.h new file mode 100644 index 0000000..2badce9 --- /dev/null +++ b/optee_os/core/include/kernel/pm.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ + +#ifndef __KERNEL_PM_H +#define __KERNEL_PM_H + +#include +#include +#include + +/* + * Platform hints on targeted power state. Hints are stored in a 32bit + * unsigned value. Lower bits defines generic resource bit flags. Higher + * bits stores a platform specific value specific platform driver may + * understand. Registered callbacks may choose to use or ignore these hints. + * + * PM_HINT_CLOCK_STATE - When set clock shall be suspended/restored + * PM_HINT_POWER_STATE - When set device power shall be suspended/restored + * PM_HINT_IO_STATE - When set IO pins shall be suspended/restored + * PM_HINT_CONTEXT_STATE - When set the full context be suspended/restored + * PM_HINT_PLATFORM_STATE_MASK - Bit mask reserved for platform specific hints + * PM_HINT_PLATFORM_STATE_SHIFT - LSBit position of platform specific hints mask + */ +#define PM_HINT_CLOCK_STATE BIT(0) +#define PM_HINT_POWER_STATE BIT(1) +#define PM_HINT_IO_STATE BIT(2) +#define PM_HINT_CONTEXT_STATE BIT(3) +#define PM_HINT_PLATFORM_STATE_MASK GENMASK_32(31, 16) +#define PM_HINT_PLATFORM_STATE_SHIFT U(16) + +/* + * PM_OP_SUSPEND: platform is suspending to a target low power state + * PM_OP_RESUME: platform is resuming from low power state + */ +enum pm_op { + PM_OP_SUSPEND = 0, + PM_OP_RESUME = 1, +}; + +/* + * Registered callbacks are called the ordering directives specified + * by the PM_CB_ORDER_* value. Driver ordered callbacks at suspended + * first/resumed last. Core service ordered callbacks are suspended + * last/resumed first. + */ +enum pm_callback_order { + PM_CB_ORDER_DRIVER = 0, + PM_CB_ORDER_CORE_SERVICE, + PM_CB_ORDER_MAX +}; + +#define PM_CALLBACK_HANDLE_INITIALIZER(_callback, _handle, _order, _name)\ + ((struct pm_callback_handle){ \ + .callback = (_callback), \ + .handle = (_handle), \ + .order = (_order), \ + .name = (_name), \ + }) + +#define PM_CALLBACK_GET_HANDLE(pm_handle) ((pm_handle)->handle) + +struct pm_callback_handle; +typedef TEE_Result (*pm_callback)(enum pm_op op, uint32_t pm_hint, + const struct pm_callback_handle *pm_handle); + +/* + * Drivers and services can register a callback function for the platform + * suspend and resume sequences. A private address handle can be registered + * with the callback and retrieved from the callback. Callback can be + * registered with a specific call order as defined per PM_CB_ORDER_*. + * + * Callback shall return an error if failing to complete target transition. + * This information may be used by the platform to resume a platform on + * non-fatal failure to suspend. + * + * Callback implementations should ensure their functions belong to unpaged + * memory sections (see DECLARE_KEEP_PAGER()) since the callback is likely to + * be called from an unpaged execution context. + * + * Power Mamagement callback functions API: + * + * TEE_Result (*callback)(enum pm_op op, + * unsigned int pm_hint, + * const struct pm_callback_handle *pm_handle); + * + * @op - Target operation: either PM_SUSPEND or PM_RESUME + * @pm_hint - Hints on power state platform suspends to /resumes from. + * PM_STATE_HINT_* defines the supported values. + * @pm_handle - Reference to the struct pm_callback_handle related to to + * registered callback. Callback can retrieve the registered + * private handle with PM_CALLBACK_GET_HANDLE(). + * + * Return a TEE_Result compliant return code + */ +/* + * struct pm_callback_handle store the callback registration directives. + * + * @callback - Registered callback function + * @handle - Registered private handler for the callback + * @order - Registered callback call order priority (PM_CB_ORDER_*) + * @flags - Flags set by pm core to keep track of execution + * @name - Registered callback name + */ +struct pm_callback_handle { + /* Set by the caller when registering a callback */ + pm_callback callback; + void *handle; + uint8_t order; + /* Set by the system according to execution context */ + uint8_t flags; + const char *name; +}; + +/* + * Register a callback for suspend/resume sequence + * Refer to struct pm_callback_handle for description of the callbacks + * API and the registration directives. + * + * @pm_handle: Reference callback registration directives + */ +void register_pm_cb(struct pm_callback_handle *pm_handle); + +/* + * Register a driver callback for generic suspend/resume. + * Refer to struct pm_callback_handle for description of the callbacks + * API. + * + * @callback: Registered callback function + * @handle: Registered private handle argument for the callback + * @name: Registered callback name + */ +static inline void register_pm_driver_cb(pm_callback callback, void *handle, + const char *name) +{ + register_pm_cb(&PM_CALLBACK_HANDLE_INITIALIZER(callback, handle, + PM_CB_ORDER_DRIVER, + name)); +} + +/* + * Register a core service callback for generic suspend/resume. + * Refer to struct pm_callback_handle for description of the callbacks + * API. + * + * @callback: Registered callback function + * @handle: Registered private handle argument for the callback + * @name: Registered callback name + */ +static inline void register_pm_core_service_cb(pm_callback callback, + void *handle, const char *name) +{ + register_pm_cb(&PM_CALLBACK_HANDLE_INITIALIZER(callback, handle, + PM_CB_ORDER_CORE_SERVICE, + name)); +} + +/* + * Request call to registered PM callbacks + * + * @op: Either PM_OP_SUSPEND or PM_OP_RESUME + * @pm_hint: Hint (PM_HINT_*) on state the platform suspends to/resumes from. + * + * Return a TEE_Result compliant status + */ +TEE_Result pm_change_state(enum pm_op op, uint32_t pm_hint); + +#endif /*__KERNEL_PM_H*/ diff --git a/optee_os/core/include/kernel/pseudo_ta.h b/optee_os/core/include/kernel/pseudo_ta.h new file mode 100644 index 0000000..132c59f --- /dev/null +++ b/optee_os/core/include/kernel/pseudo_ta.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef KERNEL_PSEUDO_TA_H +#define KERNEL_PSEUDO_TA_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PTA_MANDATORY_FLAGS (TA_FLAG_SINGLE_INSTANCE | \ + TA_FLAG_MULTI_SESSION | \ + TA_FLAG_INSTANCE_KEEP_ALIVE) + +#define PTA_ALLOWED_FLAGS (PTA_MANDATORY_FLAGS | \ + TA_FLAG_SECURE_DATA_PATH | \ + TA_FLAG_CONCURRENT | \ + TA_FLAG_DEVICE_ENUM) + +#define PTA_DEFAULT_FLAGS PTA_MANDATORY_FLAGS + +struct pseudo_ta_head { + TEE_UUID uuid; + const char *name; + uint32_t flags; + + TEE_Result (*create_entry_point)(void); + void (*destroy_entry_point)(void); + TEE_Result (*open_session_entry_point)(uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS], + void **ppSessionContext); + void (*close_session_entry_point)(void *pSessionContext); + TEE_Result (*invoke_command_entry_point)(void *pSessionContext, + uint32_t nCommandID, uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]); +}; + +#define pseudo_ta_register(...) \ + SCATTERED_ARRAY_DEFINE_PG_ITEM(pseudo_tas, struct pseudo_ta_head) = \ + { __VA_ARGS__ } + +struct pseudo_ta_ctx { + const struct pseudo_ta_head *pseudo_ta; + struct tee_ta_ctx ctx; +}; + +bool is_pseudo_ta_ctx(struct ts_ctx *ctx); + +static inline struct pseudo_ta_ctx *to_pseudo_ta_ctx(struct ts_ctx *ctx) +{ + assert(is_pseudo_ta_ctx(ctx)); + return container_of(ctx, struct pseudo_ta_ctx, ctx.ts_ctx); +} + +TEE_Result tee_ta_init_pseudo_ta_session(const TEE_UUID *uuid, + struct tee_ta_session *s); + +/* + * Helper functions for PTAs to support calls from a TA when CFG_PAN=y + */ + +static inline bool is_caller_ta_with_pan(void) +{ + struct ts_session *s = NULL; + + if (!IS_ENABLED(CFG_PAN)) + return false; + s = ts_get_calling_session(); + return s && is_user_ta_ctx(s->ctx); +} + +/* + * If caller is a TA and PAN is enabled, allocate bounce buffers for each + * memref in @params and build @bparams, then make *@oparams point to @bparams. + * Otherwise just make *@oparams point to @params. + */ +TEE_Result to_bounce_params(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS], + TEE_Param bparams[TEE_NUM_PARAMS], + TEE_Param **oparams); + +/* + * If @eparams == @bparams, copy data from @bparams to @params. Otherwise, do + * nothing. + */ +TEE_Result from_bounce_params(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS], + TEE_Param bparams[TEE_NUM_PARAMS], + TEE_Param *eparams); + +#endif /* KERNEL_PSEUDO_TA_H */ + diff --git a/optee_os/core/include/kernel/refcount.h b/optee_os/core/include/kernel/refcount.h new file mode 100644 index 0000000..6ae648e --- /dev/null +++ b/optee_os/core/include/kernel/refcount.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __KERNEL_REFCOUNT_H +#define __KERNEL_REFCOUNT_H + +#include + +/* + * Reference counter + * + * When val is 0, refcount_inc() does not change the value and returns false. + * Otherwise, it increments the value and returns true. + * + * refcount_dec() decrements the value and returns true when the call + * caused the value to become 0, false otherwise. + * + * Since each call to refcount_dec() is supposed to match a call to + * refcount_inc(), refcount_dec() called for val == 0 should never happen. + * + * This behaviour makes this pattern possible: + * if (!refcount_inc(r)) { + * mutex_lock(m); + * // Some other thread may have initialized o by now so check that + * // we still need to initialize o. + * if (!o) { + * o = initialize(); + * refcount_set(r, 1); + * } + * mutex_unlock(m); + * } + * + * or + * if (refcount_dec(r)) { + * mutex_lock(m); + * // Now that we have the mutex o can't be ininialized/uninitialized + * // by any other thread, check that the refcount value is still 0 + * // to guard against the thread above already having reinitialized o + * if (!refcount_val(r) && o) + * uninitialize(o) + * mutex_unlock(m); + * } + * + * where r if the reference counter, o is the object and m the mutex + * protecting the object. + */ + +struct refcount { + unsigned int val; +}; + +/* Increases refcount by 1, return true if val > 0 else false */ +bool refcount_inc(struct refcount *r); +/* Decreases refcount by 1, return true if val == 0 else false */ +bool refcount_dec(struct refcount *r); + +static inline void refcount_set(struct refcount *r, unsigned int val) +{ + atomic_store_uint(&r->val, val); +} + +static inline unsigned int refcount_val(struct refcount *r) +{ + return atomic_load_uint(&r->val); +} + +#endif /*!__KERNEL_REFCOUNT_H*/ diff --git a/optee_os/core/include/kernel/rpc_io_i2c.h b/optee_os/core/include/kernel/rpc_io_i2c.h new file mode 100644 index 0000000..50ddbcd --- /dev/null +++ b/optee_os/core/include/kernel/rpc_io_i2c.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020 Foundries Ltd + */ + +#ifndef __RPC_IO_I2C_H +#define __RPC_IO_I2C_H + +#include +#include + +/* I2C master transfer mode */ +enum rpc_i2c_mode { + RPC_I2C_MODE_WRITE = OPTEE_RPC_I2C_TRANSFER_WR, + RPC_I2C_MODE_READ = OPTEE_RPC_I2C_TRANSFER_RD, +}; + +/* I2C master transfer control flags */ +#define RPC_I2C_FLAGS_TEN_BIT OPTEE_RPC_I2C_FLAGS_TEN_BIT + +/* + * The bus identifier defines an implicit ABI with the REE. + * Using this service to access I2C slaves on REE dynamically assigned buses is + * not recommended unless there is a guarantee that the bus identifier will + * persist across reboots. + */ +struct rpc_i2c_request { + enum rpc_i2c_mode mode; + uint16_t bus; /* bus identifier used by the REE [0..n] */ + uint16_t chip; /* slave identifier from its data sheet */ + uint16_t flags; /* transfer flags (ie: ten bit chip address) */ + uint8_t *buffer; + size_t buffer_len; +}; + +TEE_Result rpc_io_i2c_transfer(struct rpc_i2c_request *p, size_t *bytes); + +#endif /* __RPC_IO_I2C_H */ diff --git a/optee_os/core/include/kernel/scall.h b/optee_os/core/include/kernel/scall.h new file mode 100644 index 0000000..1bdbffd --- /dev/null +++ b/optee_os/core/include/kernel/scall.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2022, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ +#ifndef __KERNEL_SCALL_H +#define __KERNEL_SCALL_H + +#include + +/* + * Generic "pointer to function" type. Actual syscalls take zero or more + * arguments and return TEE_Result. + */ +typedef void (*syscall_t)(void); + +struct thread_scall_regs; + +/* Helper function for scall_handle_user_ta() and scall_handle_ldelf() */ +uint32_t scall_do_call(struct thread_scall_regs *regs, syscall_t func); + +/* Registered as .handle_scall in struct tee_ta_ops for user TAs. */ +bool scall_handle_user_ta(struct thread_scall_regs *regs); + +/* Separate syscall handler for calls from ldelf */ +bool scall_handle_ldelf(struct thread_scall_regs *regs); + +/* + * Called from the assembly functions syscall_sys_return() and + * syscall_panic() to update the register values in the struct + * thread_scall_regs to return back to TEE Core from an earlier call to + * thread_enter_user_mode(). + */ +uint32_t scall_sys_return_helper(uint32_t ret, bool panic, uint32_t panic_code, + struct thread_scall_regs *regs); + +/* Saves TA panic stack, arch-specific implementation */ +void scall_save_panic_stack(struct thread_scall_regs *regs); + +#endif /*__KERNEL_SCALL_H*/ diff --git a/optee_os/core/include/kernel/spinlock.h b/optee_os/core/include/kernel/spinlock.h new file mode 100644 index 0000000..fc3e2e4 --- /dev/null +++ b/optee_os/core/include/kernel/spinlock.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef KERNEL_SPINLOCK_H +#define KERNEL_SPINLOCK_H + +#define SPINLOCK_LOCK 1 +#define SPINLOCK_UNLOCK 0 + +#ifndef __ASSEMBLER__ +#include +#include +#include +#include + +#ifdef CFG_TEE_CORE_DEBUG +void spinlock_count_incr(void); +void spinlock_count_decr(void); +bool have_spinlock(void); +static inline void __nostackcheck assert_have_no_spinlock(void) +{ + assert(!have_spinlock()); +} +#else +static inline void spinlock_count_incr(void) { } +static inline void spinlock_count_decr(void) { } +static inline void __nostackcheck assert_have_no_spinlock(void) { } +#endif + +void __cpu_spin_lock(unsigned int *lock); +void __cpu_spin_unlock(unsigned int *lock); +/* returns 0 on locking success, non zero on failure */ +unsigned int __cpu_spin_trylock(unsigned int *lock); + +static inline void cpu_spin_lock_no_dldetect(unsigned int *lock) +{ + assert(thread_foreign_intr_disabled()); + __cpu_spin_lock(lock); + spinlock_count_incr(); +} + +#ifdef CFG_TEE_CORE_DEBUG +#define cpu_spin_lock(lock) \ + cpu_spin_lock_dldetect(__func__, __LINE__, lock) + +static inline void cpu_spin_lock_dldetect(const char *func, const int line, + unsigned int *lock) +{ + unsigned int retries = 0; + unsigned int reminder = 0; + + assert(thread_foreign_intr_disabled()); + + while (__cpu_spin_trylock(lock)) { + retries++; + if (!retries) { + /* wrapped, time to report */ + trace_printf(func, line, TRACE_ERROR, true, + "possible spinlock deadlock reminder %u", + reminder); + if (reminder < UINT_MAX) + reminder++; + } + } + + spinlock_count_incr(); +} +#else +static inline void cpu_spin_lock(unsigned int *lock) +{ + cpu_spin_lock_no_dldetect(lock); +} +#endif + +static inline bool cpu_spin_trylock(unsigned int *lock) +{ + unsigned int rc; + + assert(thread_foreign_intr_disabled()); + rc = __cpu_spin_trylock(lock); + if (!rc) + spinlock_count_incr(); + return !rc; +} + +static inline void cpu_spin_unlock(unsigned int *lock) +{ + assert(thread_foreign_intr_disabled()); + __cpu_spin_unlock(lock); + spinlock_count_decr(); +} + +static inline uint32_t cpu_spin_lock_xsave_no_dldetect(unsigned int *lock) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + cpu_spin_lock(lock); + return exceptions; +} + +#ifdef CFG_TEE_CORE_DEBUG +#define cpu_spin_lock_xsave(lock) \ + cpu_spin_lock_xsave_dldetect(__func__, __LINE__, lock) + +static inline uint32_t cpu_spin_lock_xsave_dldetect(const char *func, + const int line, + unsigned int *lock) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + cpu_spin_lock_dldetect(func, line, lock); + return exceptions; +} +#else +static inline uint32_t cpu_spin_lock_xsave(unsigned int *lock) +{ + return cpu_spin_lock_xsave_no_dldetect(lock); +} +#endif + +static inline void cpu_spin_unlock_xrestore(unsigned int *lock, + uint32_t exceptions) +{ + cpu_spin_unlock(lock); + thread_unmask_exceptions(exceptions); +} +#endif /* __ASSEMBLER__ */ + +#endif /* KERNEL_SPINLOCK_H */ diff --git a/optee_os/core/include/kernel/tee_common.h b/optee_os/core/include/kernel/tee_common.h new file mode 100644 index 0000000..5c5da2d --- /dev/null +++ b/optee_os/core/include/kernel/tee_common.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TEE_COMMON_H +#define TEE_COMMON_H + +#include + +#ifdef MEASURE_TIME +/* + * Initializes mesaure time. Initializes RTT0 to highest possible + * resolution. + */ +void tee_mtime_init(void); + +/* + * Adds a time stamp together the description. Note that only the pointer + * is copied, not the contents to minimize impact. + */ +void tee_mtime_stamp(const char *descr); + +/* + * Prints a report of measured times and reinitializes clears the table of + * saved time stamps. + */ +void tee_mtime_report(void); + +void tee_mtime_perftest(void); +#else +/* Empty macros to not have any impact on code when not meassuring time */ +#define tee_mtime_init() do { } while (0) +#define tee_mtime_stamp(descr) do { } while (0) +#define tee_mtime_report() do { } while (0) +#define tee_mtime_perftest() do { } while (0) +#endif + +#endif /* TEE_COMMON_H */ diff --git a/optee_os/core/include/kernel/tee_common_otp.h b/optee_os/core/include/kernel/tee_common_otp.h new file mode 100644 index 0000000..2b0d3a3 --- /dev/null +++ b/optee_os/core/include/kernel/tee_common_otp.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TEE_COMMON_OTP_H +#define TEE_COMMON_OTP_H + +#include +#include +#include +#include +#include + +struct tee_hw_unique_key { + uint8_t data[HW_UNIQUE_KEY_LENGTH]; +}; + +TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey); +int tee_otp_get_die_id(uint8_t *buffer, size_t len); +TEE_Result tee_otp_get_ta_enc_key(uint32_t key_type, uint8_t *buffer, + size_t len); + +#endif /* TEE_COMMON_OTP_H */ diff --git a/optee_os/core/include/kernel/tee_misc.h b/optee_os/core/include/kernel/tee_misc.h new file mode 100644 index 0000000..035acd3 --- /dev/null +++ b/optee_os/core/include/kernel/tee_misc.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TEE_MISC_H +#define TEE_MISC_H + +#include + +/* + * Macro to derive hex string buffer size from binary buffer size & the + * reverse + */ +#define TEE_B2HS_HSBUF_SIZE(x) ((x) * 2 + 1) +#define TEE_HS2B_BBUF_SIZE(x) ((x + 1) >> 1) + +/* + * binary to hex string buffer + * Returns the number of data bytes written to the hex string + */ +uint32_t tee_b2hs(uint8_t *b, uint8_t *hs, uint32_t blen, uint32_t hslen); + +/* + * hex string to binary buffer + * Returns the number of data bytes written to the bin buffer + */ +uint32_t tee_hs2b(uint8_t *hs, uint8_t *b, uint32_t hslen, uint32_t blen); + +/* + * Is buffer 'b' inside/outside/overlapping area 'a'? + * + * core_is_buffer_inside() - return true if buffer is inside memory area + * core_is_buffer_outside() - return true if buffer is outside area + * core_is_buffer_intersect() - return true if buffer overlaps area + * + * Warning: core_is_buffer_inside(x,x,x,x)==false does NOT mean + * core_is_buffer_outside(x,x,x,x)==true. + * + * Arguments use by each of these routines: + * @b - buffer start address (handled has an unsigned offset) + * @bl - length (in bytes) of the target buffer + * @a - memory area start address (handled has an unsigned offset) + * @al - memory area length (in byte) + */ +bool core_is_buffer_inside(paddr_t b, paddr_size_t bl, + paddr_t a, paddr_size_t al); +bool core_is_buffer_outside(paddr_t b, paddr_size_t bl, + paddr_t a, paddr_size_t al); +bool core_is_buffer_intersect(paddr_t b, paddr_size_t bl, + paddr_t a, paddr_size_t al); + +/** + * Allocate maximum cache line aligned memory buffer. + * + * Both size and base address of the memory buffer will be maximum cache line + * aligned to make it safe to perform cache maintenance operations over the + * allocated area. + * + * This is needed when non-cache coherent peripherals are used and memory area + * is shared between CPU and peripheral. + * + * Allocated memory is zeroed. + * + * Release memory with free(). + * + * @size Size in bytes to allocate + * @return NULL on failure or a pointer to allocated memory on success. + */ +void *alloc_cache_aligned(size_t size); + +#endif /* TEE_MISC_H */ diff --git a/optee_os/core/include/kernel/tee_ta_manager.h b/optee_os/core/include/kernel/tee_ta_manager.h new file mode 100644 index 0000000..ef689b5 --- /dev/null +++ b/optee_os/core/include/kernel/tee_ta_manager.h @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2017, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#ifndef TEE_TA_MANAGER_H +#define TEE_TA_MANAGER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Magic TEE identity pointer: set when teecore requests a TA close */ +#define KERN_IDENTITY ((TEE_Identity *)-1) +/* Operation is initiated by a client (non-secure) app */ +#define NSAPP_IDENTITY (NULL) + +TAILQ_HEAD(tee_ta_session_head, tee_ta_session); +TAILQ_HEAD(tee_ta_ctx_head, tee_ta_ctx); + +struct mobj; + +struct param_val { + uint32_t a; + uint32_t b; +}; + +struct param_mem { + struct mobj *mobj; + size_t size; + size_t offs; +}; + +struct tee_ta_param { + uint32_t types; + union { + struct param_val val; + struct param_mem mem; + } u[TEE_NUM_PARAMS]; +}; + +struct user_ta_ctx; + +#if defined(CFG_TA_GPROF_SUPPORT) +struct sample_buf { + uint32_t nsamples; /* Size of @samples array in uint16_t */ + uint32_t offset; /* Passed from user mode */ + uint32_t scale; /* Passed from user mode */ + uint32_t count; /* Number of samples taken */ + bool enabled; /* Sampling enabled? */ + uint16_t *samples; + uint64_t usr; /* Total user CPU time for this session */ + uint64_t usr_entered; /* When this session last entered user mode */ + uint32_t freq; /* @usr divided by @freq is in seconds */ +}; +#endif + +/* Context of a loaded TA */ +struct tee_ta_ctx { + uint32_t flags; /* TA_FLAGS from TA header */ + TAILQ_ENTRY(tee_ta_ctx) link; + struct ts_ctx ts_ctx; + uint32_t panicked; /* True if TA has panicked, written from asm */ + uint32_t panic_code; /* Code supplied for panic */ + uint32_t ref_count; /* Reference counter for multi session TA */ + bool busy; /* Context is busy and cannot be entered */ + bool is_releasing; /* Context is about to be released */ + struct condvar busy_cv; /* CV used when context is busy */ +}; + +struct tee_ta_session { + TAILQ_ENTRY(tee_ta_session) link; + struct ts_session ts_sess; + uint32_t id; /* Session handle (0 is invalid) */ + TEE_Identity clnt_id; /* Identify of client */ + struct tee_ta_param *param; + TEE_ErrorOrigin err_origin; + bool cancel; /* True if TA invocation is cancelled */ + bool cancel_mask; /* True if cancel is masked */ + TEE_Time cancel_time; /* Time when to cancel the TA invocation */ + uint32_t ref_count; /* reference counter */ + struct condvar refc_cv; /* CV used to wait for ref_count to be 0 */ + struct condvar lock_cv; /* CV used to wait for lock */ + short int lock_thread; /* Id of thread holding the lock */ + bool unlink; /* True if session is to be unlinked */ +}; + +/* Registered contexts */ +extern struct tee_ta_ctx_head tee_ctxes; + +extern struct mutex tee_ta_mutex; +extern struct condvar tee_ta_init_cv; + +TEE_Result tee_ta_open_session(TEE_ErrorOrigin *err, + struct tee_ta_session **sess, + struct tee_ta_session_head *open_sessions, + const TEE_UUID *uuid, + const TEE_Identity *clnt_id, + uint32_t cancel_req_to, + struct tee_ta_param *param); + +TEE_Result tee_ta_invoke_command(TEE_ErrorOrigin *err, + struct tee_ta_session *sess, + const TEE_Identity *clnt_id, + uint32_t cancel_req_to, uint32_t cmd, + struct tee_ta_param *param); + +TEE_Result tee_ta_cancel_command(TEE_ErrorOrigin *err, + struct tee_ta_session *sess, + const TEE_Identity *clnt_id); + +bool tee_ta_session_is_cancelled(struct tee_ta_session *s, TEE_Time *curr_time); + +/*----------------------------------------------------------------------------- + * Function called to close a TA. + * Parameters: + * id - The session id (in) + * Returns: + * TEE_Result + *---------------------------------------------------------------------------*/ +TEE_Result tee_ta_close_session(struct tee_ta_session *sess, + struct tee_ta_session_head *open_sessions, + const TEE_Identity *clnt_id); + + + +struct tee_ta_session *tee_ta_find_session(uint32_t id, + struct tee_ta_session_head *open_sessions); + +struct tee_ta_session *tee_ta_get_session(uint32_t id, bool exclusive, + struct tee_ta_session_head *open_sessions); + +void tee_ta_put_session(struct tee_ta_session *sess); + +#if defined(CFG_TA_GPROF_SUPPORT) +void tee_ta_update_session_utime_suspend(void); +void tee_ta_update_session_utime_resume(void); +void tee_ta_gprof_sample_pc(vaddr_t pc); +#else +static inline void tee_ta_update_session_utime_suspend(void) {} +static inline void tee_ta_update_session_utime_resume(void) {} +static inline void tee_ta_gprof_sample_pc(vaddr_t pc __unused) {} +#endif +#if defined(CFG_FTRACE_SUPPORT) +void tee_ta_ftrace_update_times_suspend(void); +void tee_ta_ftrace_update_times_resume(void); +#else +static inline void tee_ta_ftrace_update_times_suspend(void) {} +static inline void tee_ta_ftrace_update_times_resume(void) {} +#endif + +bool is_ta_ctx(struct ts_ctx *ctx); + +struct tee_ta_session *to_ta_session(struct ts_session *sess); + +static inline struct tee_ta_ctx *__noprof to_ta_ctx(struct ts_ctx *ctx) +{ + assert(is_ta_ctx(ctx)); + return container_of(ctx, struct tee_ta_ctx, ts_ctx); +} + +#if defined(CFG_TA_STATS) +TEE_Result tee_ta_instance_stats(void *buff, size_t *buff_size); +#endif + +#endif diff --git a/optee_os/core/include/kernel/tee_time.h b/optee_os/core/include/kernel/tee_time.h new file mode 100644 index 0000000..3961551 --- /dev/null +++ b/optee_os/core/include/kernel/tee_time.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_TIME_H +#define TEE_TIME_H + +#include "tee_api_types.h" + +TEE_Result tee_time_get_sys_time(TEE_Time *time); +uint32_t tee_time_get_sys_time_protection_level(void); +TEE_Result tee_time_get_ta_time(const TEE_UUID *uuid, TEE_Time *time); +TEE_Result tee_time_get_ree_time(TEE_Time *time); +TEE_Result tee_time_set_ta_time(const TEE_UUID *uuid, const TEE_Time *time); +/* Releases CPU through OP-TEE RPC which switches to Normal World */ +void tee_time_wait(uint32_t milliseconds_delay); +/* Busy wait */ +void tee_time_busy_wait(uint32_t milliseconds_delay); + +#endif diff --git a/optee_os/core/include/kernel/thread.h b/optee_os/core/include/kernel/thread.h new file mode 100644 index 0000000..022aeec --- /dev/null +++ b/optee_os/core/include/kernel/thread.h @@ -0,0 +1,399 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2016-2017, Linaro Limited + * Copyright (c) 2020-2021, Arm Limited + */ + +#ifndef KERNEL_THREAD_H +#define KERNEL_THREAD_H + +#ifndef __ASSEMBLER__ +#include +#include +#include +#endif +#include +#include + +#define THREAD_FLAGS_COPY_ARGS_ON_RETURN BIT(0) +#define THREAD_FLAGS_FOREIGN_INTR_ENABLE BIT(1) +#define THREAD_FLAGS_EXIT_ON_FOREIGN_INTR BIT(2) +#define THREAD_FLAGS_FFA_ONLY BIT(3) + +#define THREAD_ID_0 0 +#define THREAD_ID_INVALID -1 + +#define THREAD_RPC_MAX_NUM_PARAMS U(4) + +#ifndef __ASSEMBLER__ + +struct thread_specific_data { + TAILQ_HEAD(, ts_session) sess_stack; + struct ts_ctx *ctx; +#ifdef CFG_CORE_FFA + uint32_t rpc_target_info; +#endif + uint32_t abort_type; + uint32_t abort_descr; + vaddr_t abort_va; + unsigned int abort_core; + struct thread_abort_regs abort_regs; +#ifdef CFG_CORE_DEBUG_CHECK_STACKS + bool stackcheck_recursion; +#endif + unsigned int syscall_recursion; +#ifdef CFG_FAULT_MITIGATION + struct ftmn_func_arg *ftmn_arg; +#endif +}; + +void thread_init_canaries(void); +void thread_init_primary(void); +void thread_init_per_cpu(void); + +#if defined(CFG_WITH_STACK_CANARIES) +void thread_update_canaries(void); +#else +static inline void thread_update_canaries(void) { } +#endif + +struct thread_core_local *thread_get_core_local(void); + +/* + * Sets the stacks to be used by the different threads. Use THREAD_ID_0 for + * first stack, THREAD_ID_0 + 1 for the next and so on. + * + * Returns true on success and false on errors. + */ +bool thread_init_stack(uint32_t stack_id, vaddr_t sp); + +/* + * Initializes thread contexts. Called in thread_init_boot_thread() if + * virtualization is disabled. Virtualization subsystem calls it for + * every new guest otherwise. + */ +void thread_init_threads(void); + +/* + * Called by the init CPU. Sets temporary stack mode for all CPUs + * (curr_thread = -1 and THREAD_CLF_TMP) and sets the temporary stack limit for + * the init CPU. + */ +void thread_init_thread_core_local(void); +void thread_init_core_local_stacks(void); + +#if defined(CFG_CORE_PAUTH) +void thread_init_thread_pauth_keys(void); +void thread_init_core_local_pauth_keys(void); +#else +static inline void thread_init_thread_pauth_keys(void) { } +static inline void thread_init_core_local_pauth_keys(void) { } +#endif + +/* + * Initializes a thread to be used during boot + */ +void thread_init_boot_thread(void); + +/* + * Clears the current thread id + * Only supposed to be used during initialization. + */ +void thread_clr_boot_thread(void); + +/* + * Returns current thread id. + */ +short int thread_get_id(void); + +/* + * Returns current thread id, return -1 on failure. + */ +short int thread_get_id_may_fail(void); + +/* Returns Thread Specific Data (TSD) pointer. */ +struct thread_specific_data *thread_get_tsd(void); + +/* + * Sets foreign interrupts status for current thread, must only be called + * from an active thread context. + * + * enable == true -> enable foreign interrupts + * enable == false -> disable foreign interrupts + */ +void thread_set_foreign_intr(bool enable); + +/* + * Restores the foreign interrupts status (in CPSR) for current thread, must + * only be called from an active thread context. + */ +void thread_restore_foreign_intr(void); + +/* + * thread_get_exceptions() - return current exception mask + */ +uint32_t thread_get_exceptions(void); + +/* + * thread_set_exceptions() - set exception mask + * @exceptions: exception mask to set + * + * Any previous exception mask is replaced by this exception mask, that is, + * old bits are cleared and replaced by these. + */ +void thread_set_exceptions(uint32_t exceptions); + +/* + * thread_mask_exceptions() - Masks (disables) specified asynchronous exceptions + * @exceptions exceptions to mask + * @returns old exception state + */ +uint32_t thread_mask_exceptions(uint32_t exceptions); + +/* + * thread_unmask_exceptions() - Unmasks asynchronous exceptions + * @state Old asynchronous exception state to restore (returned by + * thread_mask_exceptions()) + */ +void thread_unmask_exceptions(uint32_t state); + + +static inline bool __nostackcheck thread_foreign_intr_disabled(void) +{ + return !!(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); +} + +/* + * thread_enter_user_mode() - Enters user mode + * @a0: Passed in r/x0 for user_func + * @a1: Passed in r/x1 for user_func + * @a2: Passed in r/x2 for user_func + * @a3: Passed in r/x3 for user_func + * @user_sp: Assigned sp value in user mode + * @user_func: Function to execute in user mode + * @is_32bit: True if TA should execute in Aarch32, false if Aarch64 + * @exit_status0: Pointer to opaque exit staus 0 + * @exit_status1: Pointer to opaque exit staus 1 + * + * This functions enters user mode with the argument described above, + * @exit_status0 and @exit_status1 are filled in by thread_unwind_user_mode() + * when returning back to the caller of this function through an exception + * handler. + * + * @Returns what's passed in "ret" to thread_unwind_user_mode() + */ +uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, + unsigned long a2, unsigned long a3, unsigned long user_sp, + unsigned long entry_func, bool is_32bit, + uint32_t *exit_status0, uint32_t *exit_status1); + +/* + * thread_unwind_user_mode() - Unwinds kernel stack from user entry + * @ret: Value to return from thread_enter_user_mode() + * @exit_status0: Exit status 0 + * @exit_status1: Exit status 1 + * + * This is the function that exception handlers can return into + * to resume execution in kernel mode instead of user mode. + * + * This function is closely coupled with thread_enter_user_mode() since it + * need to restore registers saved by thread_enter_user_mode() and when it + * returns make it look like thread_enter_user_mode() just returned. It is + * expected that the stack pointer is where thread_enter_user_mode() left + * it. The stack will be unwound and the function will return to where + * thread_enter_user_mode() was called from. Exit_status0 and exit_status1 + * are filled in the corresponding pointers supplied to + * thread_enter_user_mode(). + */ +void thread_unwind_user_mode(uint32_t ret, uint32_t exit_status0, + uint32_t exit_status1); + +/* + * Returns the start address (bottom) of the stack for the current thread, + * zero if there is no current thread. + */ +vaddr_t thread_stack_start(void); + + +/* Returns the stack size for the current thread */ +size_t thread_stack_size(void); + +/* + * Returns the start (top, lowest address) and end (bottom, highest address) of + * the current stack (thread, temporary or abort stack). + * When CFG_CORE_DEBUG_CHECK_STACKS=y, the @hard parameter tells if the hard or + * soft limits are queried. The difference between soft and hard is that for the + * latter, the stack start includes some additional space to let any function + * overflow the soft limit and still be able to print a stack dump in this case. + */ +bool get_stack_limits(vaddr_t *start, vaddr_t *end, bool hard); + +static inline bool __nostackcheck get_stack_soft_limits(vaddr_t *start, + vaddr_t *end) +{ + return get_stack_limits(start, end, false); +} + +static inline bool __nostackcheck get_stack_hard_limits(vaddr_t *start, + vaddr_t *end) +{ + return get_stack_limits(start, end, true); +} + +bool thread_is_in_normal_mode(void); + +/* + * Returns true if previous exeception also was in abort mode. + * + * Note: it's only valid to call this function from an abort exception + * handler before interrupts has been re-enabled. + */ +bool thread_is_from_abort_mode(void); + +/** + * Allocates data for payload buffers. + * + * @size: size in bytes of payload buffer + * + * @returns mobj that describes allocated buffer or NULL on error + */ +struct mobj *thread_rpc_alloc_payload(size_t size); + +/** + * Free physical memory previously allocated with thread_rpc_alloc_payload() + * + * @mobj: mobj that describes the buffer + */ +void thread_rpc_free_payload(struct mobj *mobj); + +/** + * Allocate data for payload buffers only shared with the non-secure kernel + * + * @size: size in bytes of payload buffer + * + * @returns mobj that describes allocated buffer or NULL on error + */ +struct mobj *thread_rpc_alloc_kernel_payload(size_t size); + +/** + * Free physical memory previously allocated with + * thread_rpc_alloc_kernel_payload() + * + * @mobj: mobj that describes the buffer + */ +void thread_rpc_free_kernel_payload(struct mobj *mobj); + +struct thread_param_memref { + size_t offs; + size_t size; + struct mobj *mobj; +}; + +struct thread_param_value { + uint64_t a; + uint64_t b; + uint64_t c; +}; + +/* + * Note that there's some arithmetics done on the value so it's important + * to keep in IN, OUT, INOUT order. + */ +enum thread_param_attr { + THREAD_PARAM_ATTR_NONE = 0, + THREAD_PARAM_ATTR_VALUE_IN, + THREAD_PARAM_ATTR_VALUE_OUT, + THREAD_PARAM_ATTR_VALUE_INOUT, + THREAD_PARAM_ATTR_MEMREF_IN, + THREAD_PARAM_ATTR_MEMREF_OUT, + THREAD_PARAM_ATTR_MEMREF_INOUT, +}; + +struct thread_param { + enum thread_param_attr attr; + union { + struct thread_param_memref memref; + struct thread_param_value value; + } u; +}; + +#define THREAD_PARAM_MEMREF(_direction, _mobj, _offs, _size) \ + (struct thread_param){ \ + .attr = THREAD_PARAM_ATTR_MEMREF_ ## _direction, .u.memref = { \ + .mobj = (_mobj), .offs = (_offs), .size = (_size) } \ + } + +#define THREAD_PARAM_VALUE(_direction, _a, _b, _c) \ + (struct thread_param){ \ + .attr = THREAD_PARAM_ATTR_VALUE_ ## _direction, .u.value = { \ + .a = (_a), .b = (_b), .c = (_c) } \ + } + +/** + * Does an RPC using a preallocated argument buffer + * @cmd: RPC cmd + * @num_params: number of parameters + * @params: RPC parameters + * @returns RPC return value + */ +uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, + struct thread_param *params); + +/** + * Allocate data for payload buffers. + * Buffer is exported to user mode applications. + * + * @size: size in bytes of payload buffer + * + * @returns mobj that describes allocated buffer or NULL on error + */ +struct mobj *thread_rpc_alloc_global_payload(size_t size); + +/** + * Free physical memory previously allocated with + * thread_rpc_alloc_global_payload() + * + * @mobj: mobj that describes the buffer + */ +void thread_rpc_free_global_payload(struct mobj *mobj); + +/* + * enum thread_shm_type - type of non-secure shared memory + * @THREAD_SHM_TYPE_APPLICATION - user space application shared memory + * @THREAD_SHM_TYPE_KERNEL_PRIVATE - kernel private shared memory + * @THREAD_SHM_TYPE_GLOBAL - user space and kernel shared memory + */ +enum thread_shm_type { + THREAD_SHM_TYPE_APPLICATION, + THREAD_SHM_TYPE_KERNEL_PRIVATE, + THREAD_SHM_TYPE_GLOBAL, +}; + +/* + * enum thread_shm_cache_user - user of a cache allocation + * @THREAD_SHM_CACHE_USER_SOCKET - socket communication + * @THREAD_SHM_CACHE_USER_FS - filesystem access + * @THREAD_SHM_CACHE_USER_I2C - I2C communication + * + * To ensure that each user of the shared memory cache doesn't interfere + * with each other a unique ID per user is used. + */ +enum thread_shm_cache_user { + THREAD_SHM_CACHE_USER_SOCKET, + THREAD_SHM_CACHE_USER_FS, + THREAD_SHM_CACHE_USER_I2C, +}; + +/* + * Returns a pointer to the cached RPC memory. Each thread and @user tuple + * has a unique cache. The pointer is guaranteed to point to a large enough + * area or to be NULL. + */ +void *thread_rpc_shm_cache_alloc(enum thread_shm_cache_user user, + enum thread_shm_type shm_type, + size_t size, struct mobj **mobj); + +#endif /*__ASSEMBLER__*/ + +#endif /*KERNEL_THREAD_H*/ diff --git a/optee_os/core/include/kernel/thread_private.h b/optee_os/core/include/kernel/thread_private.h new file mode 100644 index 0000000..1d83dd5 --- /dev/null +++ b/optee_os/core/include/kernel/thread_private.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef __KERNEL_THREAD_PRIVATE_H +#define __KERNEL_THREAD_PRIVATE_H + +#include +#ifndef __ASSEMBLER__ + +#include +#include +#include +#include +#include + +enum thread_state { + THREAD_STATE_FREE, + THREAD_STATE_SUSPENDED, + THREAD_STATE_ACTIVE, +}; + +struct thread_shm_cache_entry { + struct mobj *mobj; + size_t size; + enum thread_shm_type type; + enum thread_shm_cache_user user; + SLIST_ENTRY(thread_shm_cache_entry) link; +}; + +SLIST_HEAD(thread_shm_cache, thread_shm_cache_entry); + +struct thread_ctx { + struct thread_ctx_regs regs; + enum thread_state state; + vaddr_t stack_va_end; + uint32_t flags; + struct core_mmu_user_map user_map; + bool have_user_map; +#if defined(ARM64) || defined(RV64) + vaddr_t kern_sp; /* Saved kernel SP during user TA execution */ +#endif +#ifdef CFG_CORE_PAUTH + struct thread_pauth_keys keys; +#endif +#ifdef CFG_WITH_VFP + struct thread_vfp_state vfp_state; +#endif + void *rpc_arg; + struct mobj *rpc_mobj; + struct thread_shm_cache shm_cache; + struct thread_specific_data tsd; +}; +#endif /*__ASSEMBLER__*/ + +/* Describes the flags field of struct thread_core_local */ +#define THREAD_CLF_SAVED_SHIFT 4 +#define THREAD_CLF_CURR_SHIFT 0 +#define THREAD_CLF_MASK 0xf +#define THREAD_CLF_TMP_SHIFT 0 +#define THREAD_CLF_ABORT_SHIFT 1 +#define THREAD_CLF_IRQ_SHIFT 2 +#define THREAD_CLF_FIQ_SHIFT 3 + +#define THREAD_CLF_TMP BIT(THREAD_CLF_TMP_SHIFT) +#define THREAD_CLF_ABORT BIT(THREAD_CLF_ABORT_SHIFT) +#define THREAD_CLF_IRQ BIT(THREAD_CLF_IRQ_SHIFT) +#define THREAD_CLF_FIQ BIT(THREAD_CLF_FIQ_SHIFT) + +#ifndef __ASSEMBLER__ +extern const void *stack_tmp_export; +extern const uint32_t stack_tmp_stride; +extern struct thread_ctx threads[]; +extern struct thread_core_local thread_core_local[]; + +#ifdef CFG_WITH_STACK_CANARIES +#define STACK_CANARY_SIZE (4 * sizeof(long)) +#else +#define STACK_CANARY_SIZE 0 +#endif + +/* Checks stack canaries */ +void thread_check_canaries(void); + +void thread_lock_global(void); +void thread_unlock_global(void); + +/* Frees the cache of allocated FS RPC memory */ +void thread_rpc_shm_cache_clear(struct thread_shm_cache *cache); +#endif /*__ASSEMBLER__*/ +#endif /*__KERNEL_THREAD_PRIVATE_H*/ diff --git a/optee_os/core/include/kernel/time_source.h b/optee_os/core/include/kernel/time_source.h new file mode 100644 index 0000000..fce5391 --- /dev/null +++ b/optee_os/core/include/kernel/time_source.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#include + +struct time_source { + const char *name; + uint32_t protection_level; + TEE_Result (*get_sys_time)(TEE_Time *time); +}; +void time_source_init(void); + +#define REGISTER_TIME_SOURCE(source) \ + void time_source_init(void) { \ + _time_source = source; \ + } + +extern struct time_source _time_source; diff --git a/optee_os/core/include/kernel/timer.h b/optee_os/core/include/kernel/timer.h new file mode 100644 index 0000000..8aaca44 --- /dev/null +++ b/optee_os/core/include/kernel/timer.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2018, Linaro Limited + */ + +#ifndef __TIMER_H +#define __TIMER_H + +void generic_timer_start(uint32_t time_ms); +void generic_timer_stop(void); + +/* Handler for timer expiry interrupt */ +void generic_timer_handler(uint32_t time_ms); + +#endif /* __TIMER_H */ diff --git a/optee_os/core/include/kernel/tpm.h b/optee_os/core/include/kernel/tpm.h new file mode 100644 index 0000000..6e053b3 --- /dev/null +++ b/optee_os/core/include/kernel/tpm.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020-2022, ARM Limited. All rights reserved. + */ + +#ifndef __KERNEL_TPM_H__ +#define __KERNEL_TPM_H__ + +#include + +#ifdef CFG_CORE_TPM_EVENT_LOG + +/* + * Returns the TPM Event Log information previously retrieved + * by @tpm_map_log_area. + * + * @buf Pointer to a buffer where to store a copy of the TPM Event log. + */ +TEE_Result tpm_get_event_log(void *buf, size_t *size); + +TEE_Result tpm_get_event_log_size(size_t *size); + +/* + * Reads the TPM Event log information and store it internally. + * If support for DTB is enabled, it will read the parameters there. + * Otherwise, it will use the constant parameters hardcoded in conf.mk. + * + * @fdt Pointer to the DTB blob where the TPM Event log information + * is expected to be. + */ +void tpm_map_log_area(void *fdt); + +#else + +static inline TEE_Result tpm_get_event_log(void *buf __unused, + size_t *size __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result tpm_get_event_log_size(size_t *size __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline void tpm_map_log_area(void *fdt __unused) +{} + +#endif /* CFG_CORE_TPM_EVENT_LOG */ + +#endif /* __KERNEL_TPM_H__ */ diff --git a/optee_os/core/include/kernel/trace_ta.h b/optee_os/core/include/kernel/trace_ta.h new file mode 100644 index 0000000..44644e6 --- /dev/null +++ b/optee_os/core/include/kernel/trace_ta.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ +#ifndef TRACE_TA_H +#define TRACE_TA_H + +#include + +/* Macros to trace TA related events, logs, TA crash info etc */ +#ifdef CFG_TEE_CORE_TA_TRACE +#define TAMSG(...) EMSG(__VA_ARGS__) +#define TAMSG_RAW(...) EMSG_RAW(__VA_ARGS__) +#else +#define TAMSG(...) +#define TAMSG_RAW(...) +#endif + +#endif /*TRACE_TA_H*/ + diff --git a/optee_os/core/include/kernel/ts_manager.h b/optee_os/core/include/kernel/ts_manager.h new file mode 100644 index 0000000..ab14654 --- /dev/null +++ b/optee_os/core/include/kernel/ts_manager.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2017-2020, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#ifndef __KERNEL_TS_MANAGER_H +#define __KERNEL_TS_MANAGER_H + +#include +#include + +struct ts_ctx { + TEE_UUID uuid; + const struct ts_ops *ops; +}; + +struct thread_scall_regs; +struct ts_session { + TAILQ_ENTRY(ts_session) link_tsd; + struct ts_ctx *ctx; /* Generic TS context */ +#if defined(CFG_TA_GPROF_SUPPORT) + struct sample_buf *sbuf; /* Profiling data (PC sampling) */ +#endif +#if defined(CFG_FTRACE_SUPPORT) + struct ftrace_buf *fbuf; /* ftrace buffer */ +#endif + /* + * Used by PTAs to store session specific information, or used by ldelf + * syscalls to store handlers of opened TA/SP binaries. + */ + void *user_ctx; + bool (*handle_scall)(struct thread_scall_regs *regs); +}; + +enum ts_gprof_status { + TS_GPROF_SUSPEND, + TS_GPROF_RESUME, +}; + +/* + * struct ts_ops - Trusted Service operations + * The operations below are called when: + * @enter_open_session(): opening a session to the service + * @enter_invoke_cmd(): invoking a command in the service + * @enter_close_session(): closing a session to the service + * @dump_mem_stats(): dumping heap state of the service + * @dump_state(): dumping active memory mappings + * @dump_ftrace(): dumping the ftrace data via RPC + * @release_state(): the service has panicked and as much state + * as possible need to be released + * @destroy(): freeing the struct ts_ctx removing all + * trace of the service + * @get_instance_id(): a unique ID of the service is needed + * @handle_scall(): handling a syscall from the service + * @gprof_set_status(): updating the gprof status of the service + */ +struct ts_ops { + TEE_Result (*enter_open_session)(struct ts_session *s); + TEE_Result (*enter_invoke_cmd)(struct ts_session *s, uint32_t cmd); + void (*enter_close_session)(struct ts_session *s); +#if defined(CFG_TA_STATS) + TEE_Result (*dump_mem_stats)(struct ts_session *s); +#endif + void (*dump_state)(struct ts_ctx *ctx); + void (*dump_ftrace)(struct ts_ctx *ctx); + void (*release_state)(struct ts_ctx *ctx); + void (*destroy)(struct ts_ctx *ctx); + uint32_t (*get_instance_id)(struct ts_ctx *ctx); + bool (*handle_scall)(struct thread_scall_regs *regs); +#ifdef CFG_TA_GPROF_SUPPORT + void (*gprof_set_status)(enum ts_gprof_status status); +#endif +}; + +struct ts_session *ts_get_current_session(void); +struct ts_session *ts_get_current_session_may_fail(void); + +void ts_push_current_session(struct ts_session *sess); +struct ts_session *ts_pop_current_session(void); +struct ts_session *ts_get_calling_session(void); + +#endif /*__KERNEL_TS_MANAGER_H*/ diff --git a/optee_os/core/include/kernel/ts_store.h b/optee_os/core/include/kernel/ts_store.h new file mode 100644 index 0000000..58c2173 --- /dev/null +++ b/optee_os/core/include/kernel/ts_store.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2019, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ +#ifndef __KERNEL_TS_STORE_H +#define __KERNEL_TS_STORE_H + +#include + +struct ts_store_handle; +struct ts_store_ops { + /* + * Human-readable string to describe where the TS comes from. + * For debug purposes only. + */ + const char *description; + /* + * Open a TS. Does not guarantee that the TS is valid or even exists. + */ + TEE_Result (*open)(const TEE_UUID *uuid, + struct ts_store_handle **h); + /* + * Return the size of the unencrypted TS binary, that is: the TS + * header (struct ta_head or sp_head) plus the ELF data. + */ + TEE_Result (*get_size)(const struct ts_store_handle *h, + size_t *size); + + /* + * Return the tag or hash of the TS binary. Used to uniquely + * identify the binary also if the binary happens to be updated. + */ + TEE_Result (*get_tag)(const struct ts_store_handle *h, + uint8_t *tag, unsigned int *tag_len); + /* + * Read the TS sequentially, from the start of the TS header (struct + * ta_head or sp_head) up to the end of the ELF. + * The TEE core is expected to read *exactly* get_size() bytes in total + * unless an error occurs. Therefore, an implementation may rely on the + * condition (current offset == total size) to detect the last call to + * this function. + * @data_core: pointer to secure memory where the TS bytes should be + * copied. + * @data_user: pointer to user memory where the TS bytes should be + * copied. + * At least one of @data_core and @data_user are normally NULL, but + * both are also permitted to be non-NULL. + * If @data_core == NULL and @data_user == NULL and @len != 0, the + * function should just skip @len bytes. + */ + TEE_Result (*read)(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len); + /* + * Close a TS handle. Do nothing if @h == NULL. + */ + void (*close)(struct ts_store_handle *h); +}; + +/* + * Registers a TA storage. + * + * A TA is loaded from the first TA storage in which the TA can be found. + * TA storage is searched in order of priority, where lower values are + * tried first. + * + * Note prio must be unique per storage in order to avoid dependency on + * registration order. This is enforced by a deliberate linker error in + * case of conflict. + * + * Also note that TA storage is sorted lexicographically instead of + * numerically. + */ +#define REGISTER_TA_STORE(prio) \ + int __tee_ta_store_##prio __unused; \ + SCATTERED_ARRAY_DEFINE_PG_ITEM_ORDERED(ta_stores, prio, \ + struct ts_store_ops) + +/* + * Registers a SP storage. + * + * The SP store is separate from the TA store. The user of the stores knows if + * it needs to access the TA store or if it needs to access the SP one. + */ +#define REGISTER_SP_STORE(prio) \ + int __tee_sp_store_##prio __unused; \ + SCATTERED_ARRAY_DEFINE_PG_ITEM_ORDERED(sp_stores, prio, \ + struct ts_store_ops) +#endif /*__KERNEL_TS_STORE_H*/ diff --git a/optee_os/core/include/kernel/unwind.h b/optee_os/core/include/kernel/unwind.h new file mode 100644 index 0000000..9a01b46 --- /dev/null +++ b/optee_os/core/include/kernel/unwind.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/*- + * Copyright (c) 2015-2019 Linaro Limited + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ + +#ifndef KERNEL_UNWIND +#define KERNEL_UNWIND + +#include + +#if defined(CFG_UNWIND) && (TRACE_LEVEL > 0) +void print_kernel_stack(void); +#else +static inline void print_kernel_stack(void) +{ +} +#endif + +#ifdef CFG_UNWIND +/* Get current call stack as an array allocated on the heap */ +vaddr_t *unw_get_kernel_stack(void); +#else +static inline void *unw_get_kernel_stack(void) +{ + return NULL; +} +#endif /* CFG_UNWIND */ + +#endif /*KERNEL_UNWIND*/ diff --git a/optee_os/core/include/kernel/user_access.h b/optee_os/core/include/kernel/user_access.h new file mode 100644 index 0000000..54b3bfd --- /dev/null +++ b/optee_os/core/include/kernel/user_access.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, Linaro Limited + */ +#ifndef __KERNEL_USER_ACCESS_H +#define __KERNEL_USER_ACCESS_H + +#include +#include +#include +#include + +#ifdef CFG_WITH_USER_TA +TEE_Result check_user_access(uint32_t flags, const void *uaddr, size_t len); +TEE_Result copy_from_user_private(void *kaddr, const void *uaddr, size_t len); +TEE_Result copy_from_user(void *kaddr, const void *uaddr, size_t len); +TEE_Result copy_to_user_private(void *uaddr, const void *kaddr, size_t len); +TEE_Result copy_to_user(void *uaddr, const void *kaddr, size_t len); +#else +static inline TEE_Result check_user_access(uint32_t flags __unused, + const void *uaddr __unused, + size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result copy_from_user_private(void *kaddr __unused, + const void *uaddr __unused, + size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result copy_from_user(void *kaddr __unused, + const void *uaddr __unused, + size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result copy_to_user_private(void *uaddr __unused, + const void *kaddr __unused, + size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +static inline TEE_Result copy_to_user(void *uaddr __unused, + const void *kaddr __unused, + size_t len __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +/* + * bb_alloc() - Allocate a bounce buffer + * @len: Length of bounce buffer + * + * The bounce buffer is allocated from a per user TA context region reserved + * for bounce buffers. Buffers are allocated in a stack like fashion so + * only the last buffer can be free. Buffers generally don't have to be + * freed, all bounce buffer allocations are reset on each syscall entry. + * + * Return NULL on failure or a valid pointer on success. + */ +void *bb_alloc(size_t len); + +/* + * bb_free() - Free a bounce buffer + * @bb: Buffer + * @len: Length of buffer + * + * The bounce buffer is only freed if it is last on the stack of allocated + * bounce buffers. This function does normally not need to be called, see + * description of bb_alloc(). + */ +void bb_free(void *bb, size_t len); + +/* + * bb_free_wipe() - Wipe and free a bounce buffer + * @bb: Buffer + * @len: Length of buffer + * + * The bounce buffer is always wiped if @bb is non-NULL, but only freed if + * it is last on the stack of allocated bounce buffers. + */ +void bb_free_wipe(void *bb, size_t len); + +/* + * bb_reset() - Reset bounce buffer allocation + * + * Resets the bounce buffer allocatation state, old pointers allocated + * with bb_alloc() should not be used any longer. + */ +void bb_reset(void); + +TEE_Result clear_user(void *uaddr, size_t n); + +size_t strnlen_user(const void *s, size_t n); + +#define __BB_MEMDUP(memdup_func, src, len, p) ({ \ + TEE_Result __res = TEE_SUCCESS; \ + void *__p = NULL; \ + \ + __res = memdup_func((src), (len), &__p); \ + if (!__res) \ + *(p) = __p; \ + __res; \ +}) + +/* + * bb_memdup_user() - Duplicate a user-space buffer into a bounce buffer + * @src: Pointer to the user buffer to be duplicated. + * @len: Length of the user buffer to be duplicated. + * @p: Holds duplicated bounce buffer on success, or unchanged on failure. + * Note that the returned buffer is allocated by bb_alloc() and + * normally doesn't have to be freed. + * Return TEE_SUCCESS on success. + * Return TEE_ERROR_OUT_OF_MEMORY or TEE_ERROR_ACCESS_DENIED on error. + */ +TEE_Result bb_memdup_user(const void *src, size_t len, void **p); +#define BB_MEMDUP_USER(src, len, p) \ + __BB_MEMDUP(bb_memdup_user, (src), (len), (p)) + +/* + * bb_memdup_user_private() - Duplicate a private user-space buffer + * @src: Pointer to the user buffer to be duplicated. The buffer should + * be private to current TA (i.e., !TEE_MEMORY_ACCESS_ANY_OWNER). + * @len: Length of the user buffer to be duplicated. + * @p: Holds duplicated kernel buffer on success, or unchanged on failure. + * Note that the returned buffer is allocated by bb_alloc() and + * normally doesn't have to be freed. + * Return TEE_SUCCESS on success. + * Return TEE_ERROR_OUT_OF_MEMORY or TEE_ERROR_ACCESS_DENIED on error. + */ +TEE_Result bb_memdup_user_private(const void *src, size_t len, void **p); +#define BB_MEMDUP_USER_PRIVATE(src, len, p) \ + __BB_MEMDUP(bb_memdup_user_private, (src), (len), (p)) + +/* + * bb_strndup_user() - Duplicate a user-space string into a bounce buffer + * @src: Pointer to the user string to be duplicated. + * @maxlen: Maximum length of the user string + * @dst: Holds duplicated string on success, or unchanged on failure. + * @dstlen: Length of string, excluding the terminating zero, returned in + * @dst. + * + * Note that the returned buffer is allocated by bb_alloc() and normally + * doesn't have to be freed. But if it is to be freed the supplied length + * to bb_free() should be dstlen + 1. + * + * Return TEE_SUCCESS on success. + * Return TEE_ERROR_OUT_OF_MEMORY or TEE_ERROR_ACCESS_DENIED on error. + */ +TEE_Result bb_strndup_user(const char *src, size_t maxlen, char **dst, + size_t *dstlen); + +TEE_Result copy_kaddr_to_uref(uint32_t *uref, void *kaddr); + +uint32_t kaddr_to_uref(void *kaddr); +vaddr_t uref_to_vaddr(uint32_t uref); +static inline void *uref_to_kaddr(uint32_t uref) +{ + return (void *)uref_to_vaddr(uref); +} + +#define GET_USER_SCALAR(_x, _p) ({ \ + TEE_Result __res = TEE_SUCCESS; \ + typeof(_p) __p = (_p); \ + \ + static_assert(sizeof(_x) == sizeof(*__p)); \ + \ + __res = copy_from_user(&(_x), (const void *)__p, sizeof(*__p)); \ + __res; \ +}) + +#define PUT_USER_SCALAR(_x, _p) ({ \ + TEE_Result __res = TEE_SUCCESS; \ + typeof(_p) __p = (_p); \ + \ + static_assert(sizeof(_x) == sizeof(*__p)); \ + \ + __res = copy_to_user((void *)__p, &(_x), sizeof(*__p)); \ + __res; \ +}) + +#endif /*__KERNEL_USER_ACCESS_H*/ diff --git a/optee_os/core/include/kernel/user_mode_ctx.h b/optee_os/core/include/kernel/user_mode_ctx.h new file mode 100644 index 0000000..6d65d14 --- /dev/null +++ b/optee_os/core/include/kernel/user_mode_ctx.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2021, Arm Limited + */ + +#ifndef __KERNEL_USER_MODE_CTX_H +#define __KERNEL_USER_MODE_CTX_H + +#include +#include +#include +#include +#include +#include + +static inline bool is_user_mode_ctx(struct ts_ctx *ctx) +{ + return is_user_ta_ctx(ctx) || is_stmm_ctx(ctx) || is_sp_ctx(ctx); +} + +static inline struct user_mode_ctx *to_user_mode_ctx(struct ts_ctx *ctx) +{ + if (is_user_ta_ctx(ctx)) + return &to_user_ta_ctx(ctx)->uctx; + else if (is_sp_ctx(ctx)) + return &to_sp_ctx(ctx)->uctx; + else + return &to_stmm_ctx(ctx)->uctx; +} + +void user_mode_ctx_print_mappings(struct user_mode_ctx *umctx); + +#endif /*__KERNEL_USER_MODE_CTX_H*/ diff --git a/optee_os/core/include/kernel/user_mode_ctx_struct.h b/optee_os/core/include/kernel/user_mode_ctx_struct.h new file mode 100644 index 0000000..6c91f4e --- /dev/null +++ b/optee_os/core/include/kernel/user_mode_ctx_struct.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019-2021, Linaro Limited + * Copyright (c) 2020-2023, Arm Limited + */ + +#ifndef __KERNEL_USER_MODE_CTX_STRUCT_H +#define __KERNEL_USER_MODE_CTX_STRUCT_H + +#include +#include +#include + +/* + * struct user_mode_ctx - user mode context + * @vm_info: Virtual memory map of this context + * @regions: Memory regions registered by pager + * @vfp: State of VFP registers + * @keys: Pointer authentication keys + * @ts_ctx: Generic TS context + * @entry_func: Entry address in TS + * @dump_entry_func: Entry address in TS for dumping address mappings + * and stack trace + * @ftrace_entry_func: Entry address in ldelf for dumping ftrace data + * @dl_entry_func: Entry address in ldelf for dynamic linking + * @ldelf_stack_ptr: Stack pointer used for dumping address mappings and + * stack trace + * @is_32bit: True if 32-bit TS, false if 64-bit TS + * @is_initializing: True if TS is not fully loaded + * @stack_ptr: Stack pointer + * @bbuf: Bounce buffer for user buffers + * @bbuf_size: Size of bounce buffer + * @bbuf_offs: Offset to unused part of bounce buffer + */ +struct user_mode_ctx { + struct vm_info vm_info; + struct vm_paged_region_head *regions; + struct pgt_cache pgt_cache; +#if defined(CFG_WITH_VFP) + struct thread_user_vfp_state vfp; +#endif +#if defined(CFG_TA_PAUTH) + struct thread_pauth_keys keys; +#endif + struct ts_ctx *ts_ctx; + uaddr_t entry_func; + uaddr_t load_addr; + uaddr_t dump_entry_func; +#ifdef CFG_FTRACE_SUPPORT + uaddr_t ftrace_entry_func; +#endif + uaddr_t dl_entry_func; + uaddr_t ldelf_stack_ptr; + bool is_32bit; + bool is_initializing; + vaddr_t stack_ptr; + uint8_t *bbuf; + size_t bbuf_size; + size_t bbuf_offs; +}; +#endif /*__KERNEL_USER_MODE_CTX_STRUCT_H*/ + diff --git a/optee_os/core/include/kernel/user_ta.h b/optee_os/core/include/kernel/user_ta.h new file mode 100644 index 0000000..942c948 --- /dev/null +++ b/optee_os/core/include/kernel/user_ta.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ +#ifndef KERNEL_USER_TA_H +#define KERNEL_USER_TA_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TAILQ_HEAD(tee_cryp_state_head, tee_cryp_state); +TAILQ_HEAD(tee_obj_head, tee_obj); +TAILQ_HEAD(tee_storage_enum_head, tee_storage_enum); +SLIST_HEAD(load_seg_head, load_seg); + +/* + * struct user_ta_ctx - user TA context + * @open_sessions: List of sessions opened by this TA + * @cryp_states: List of cryp states created by this TA + * @objects: List of storage objects opened by this TA + * @storage_enums: List of storage enumerators opened by this TA + * @ta_time_offs: Time reference used by the TA + * @uctx: Generic user mode context + * @ctx: Generic TA context + */ +struct user_ta_ctx { + struct tee_ta_session_head open_sessions; + struct tee_cryp_state_head cryp_states; + struct tee_obj_head objects; + struct tee_storage_enum_head storage_enums; + void *ta_time_offs; + struct user_mode_ctx uctx; + struct tee_ta_ctx ta_ctx; +}; + +#ifdef CFG_WITH_USER_TA +bool is_user_ta_ctx(struct ts_ctx *ctx); +#else +static inline bool __noprof is_user_ta_ctx(struct ts_ctx *ctx __unused) +{ + return false; +} +#endif + +static inline struct user_ta_ctx *to_user_ta_ctx(struct ts_ctx *ctx) +{ + assert(is_user_ta_ctx(ctx)); + return container_of(ctx, struct user_ta_ctx, ta_ctx.ts_ctx); +} + +#ifdef CFG_WITH_USER_TA +TEE_Result tee_ta_init_user_ta_session(const TEE_UUID *uuid, + struct tee_ta_session *s); +#else +static inline TEE_Result tee_ta_init_user_ta_session( + const TEE_UUID *uuid __unused, + struct tee_ta_session *s __unused) +{ + return TEE_ERROR_ITEM_NOT_FOUND; +} +#endif + + +#endif /*KERNEL_USER_TA_H*/ diff --git a/optee_os/core/include/kernel/virtualization.h b/optee_os/core/include/kernel/virtualization.h new file mode 100644 index 0000000..c4a7f69 --- /dev/null +++ b/optee_os/core/include/kernel/virtualization.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2018, EPAM Systems. All rights reserved. */ + +#ifndef KERNEL_VIRTUALIZATION_H +#define KERNEL_VIRTUALIZATION_H + +#include +#include +#include +#include + +#define HYP_CLNT_ID 0 + +#if defined(CFG_NS_VIRTUALIZATION) +/** + * virt_guest_created() - create new VM partition + * @guest_id: VM id provided by hypervisor + * + * This function is called by hypervisor (via fast SMC) + * when hypervisor creates new guest VM, so OP-TEE + * can prepare partition for that VM + */ +TEE_Result virt_guest_created(uint16_t guest_id); + +/** + * virt_guest_destroyed() - destroy existing VM partition + * @guest_id: VM id provided by hypervisor + * + * This function is called by hypervisor (via fast SMC) + * when hypervisor is ready to destroy guest VM. Hypervisor + * must ensure that there are no ongoing calls from this + * VM right now. + */ +TEE_Result virt_guest_destroyed(uint16_t guest_id); + +/** + * virt_set_guest() - set guest VM context for current core + * @guest_id: VM id provided by hypervisor + * + * This function switches memory partitions, so TEE part of + * OP-TEE will see memory associated with current guest. + * It should be called on entry to OP-TEE + */ +TEE_Result virt_set_guest(uint16_t guest_id); + +/** + * virt_unset_guest() - set default memory partition + * + * This function should be called upon leaving OP-TEE, + * to switch to default memory partition, so all TEE-specific + * memory will be unmapped. This is safety measure to ensure + * that TEE memory is untouched when there is no active VM. + */ +void virt_unset_guest(void); + +/** + * virt_on_stdcall() - std call hook + * + * This hook is called on every std call, but really is needed + * only once: to initialize TEE runtime for current guest VM + */ +void virt_on_stdcall(void); + +/* + * Next function are needed because virtualization subsystem manages + * memory in own way. There is no one static memory map, instead + * every guest gets own memory map. + */ + +/** + * virt_init_memory() - initialize memory for virtualization subsystem + * @memory_map: current OP-TEE memory map + * @secmem0_base: base of first secure memory range + * @secmem0_size: size of first secure memory range + * @secmem1_base: base of an eventual second secure memory range, 0 if unused + * @secmem1_size: size of an eventual second secure memory range, 0 if unused + */ +void virt_init_memory(struct tee_mmap_region *memory_map, paddr_t secmem0_base, + paddr_size_t secmem0_size, paddr_t secmem1_base, + paddr_size_t secmem1_size); + +/** + * virt_get_memory_map() - get current memory map + */ +struct tee_mmap_region *virt_get_memory_map(void); + +/** + * virt_get_ta_ram() - get TA RAM mapping for current VM + * @start: beginning of TA RAM returned here + * @end: end of TA RAM returned here + */ +void virt_get_ta_ram(vaddr_t *start, vaddr_t *end); + +/** + * virt_get_current_guest_id() - return current guest ID + * + * Returns current guest ID or 0 if none is set. + */ +uint16_t virt_get_current_guest_id(void); + +#else +static inline TEE_Result virt_guest_created(uint16_t guest_id __unused) +{ return TEE_ERROR_NOT_SUPPORTED; } + +static inline TEE_Result virt_guest_destroyed(uint16_t guest_id __unused) +{ return TEE_ERROR_NOT_SUPPORTED; } + +static inline TEE_Result virt_set_guest(uint16_t guest_id __unused) +{ return TEE_ERROR_NOT_SUPPORTED; } + +static inline void virt_unset_guest(void) { } +static inline void virt_on_stdcall(void) { } +static inline struct tee_mmap_region *virt_get_memory_map(void) { return NULL; } +static inline void +virt_get_ta_ram(vaddr_t *start __unused, vaddr_t *end __unused) { } +static inline void virt_init_memory(struct tee_mmap_region *memory_map __unused, + paddr_t secmem0_base __unused, + paddr_size_t secmem0_size __unused, + paddr_t secmem1_base __unused, + paddr_size_t secmem1_size __unused) { } +static inline uint16_t virt_get_current_guest_id(void) { return 0; } +#endif /*CFG_NS_VIRTUALIZATION*/ + +#if defined(CFG_CORE_SEL1_SPMC) && defined(CFG_NS_VIRTUALIZATION) +TEE_Result virt_add_cookie_to_current_guest(uint64_t cookie); +void virt_remove_cookie(uint64_t cookie); +uint16_t virt_find_guest_by_cookie(uint64_t cookie); + +#else +static inline TEE_Result +virt_add_cookie_to_current_guest(uint64_t cookie __unused) +{ return TEE_ERROR_NOT_SUPPORTED; } +static inline void virt_remove_cookie(uint64_t cookie __unused) { } +static inline uint16_t virt_find_guest_by_cookie(uint64_t cookie __unused) +{ return 0; } +#endif + +#endif /* KERNEL_VIRTUALIZATION_H */ diff --git a/optee_os/core/include/kernel/wait_queue.h b/optee_os/core/include/kernel/wait_queue.h new file mode 100644 index 0000000..a13c02b --- /dev/null +++ b/optee_os/core/include/kernel/wait_queue.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef KERNEL_WAIT_QUEUE_H +#define KERNEL_WAIT_QUEUE_H + +#include +#include + +struct wait_queue_elem; +SLIST_HEAD(wait_queue, wait_queue_elem); + +#define WAIT_QUEUE_INITIALIZER { .slh_first = NULL } + +struct condvar; +struct wait_queue_elem { + short handle; + bool done; + bool wait_read; + struct condvar *cv; + SLIST_ENTRY(wait_queue_elem) link; +}; + +/* + * Initializes a wait queue + */ +void wq_init(struct wait_queue *wq); + +/* + * Initializes a wait queue element and adds it to the wait queue. This + * function is supposed to be called before the lock that protects the + * resource we need to wait for is released. + * + * One call to this function must be followed by one call to wq_wait_final() + * on the same wait queue element. + */ +void wq_wait_init_condvar(struct wait_queue *wq, struct wait_queue_elem *wqe, + struct condvar *cv, bool wait_read); + +static inline void wq_wait_init(struct wait_queue *wq, + struct wait_queue_elem *wqe, bool wait_read) +{ + wq_wait_init_condvar(wq, wqe, NULL, wait_read); +} + +/* Waits for the wait queue element to the awakened. */ +void wq_wait_final(struct wait_queue *wq, struct wait_queue_elem *wqe, + const void *sync_obj, const char *fname, int lineno); + +/* Wakes up the first wait queue element in the wait queue, if there is one */ +void wq_wake_next(struct wait_queue *wq, const void *sync_obj, + const char *fname, int lineno); + +/* Returns true if the wait queue doesn't contain any elements */ +bool wq_is_empty(struct wait_queue *wq); + +void wq_promote_condvar(struct wait_queue *wq, struct condvar *cv, + bool only_one, const void *sync_obj, const char *fname, + int lineno); +bool wq_have_condvar(struct wait_queue *wq, struct condvar *cv); + +#endif /*KERNEL_WAIT_QUEUE_H*/ + diff --git a/optee_os/core/include/mm/core_memprot.h b/optee_os/core/include/mm/core_memprot.h new file mode 100644 index 0000000..d809ad3 --- /dev/null +++ b/optee_os/core/include/mm/core_memprot.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef CORE_MEMPROT_H +#define CORE_MEMPROT_H + +#include +#include + +/* + * "pbuf_is" support. + * + * core_vbuf_is()/core_pbuf_is() can be used to check if a teecore mapped + * virtual address or a physical address is "Secure", "Unsecure", "external + * RAM" and some other fancy attributes. + * + * DO NOT use 'buf_is(Secure, buffer)==false' as a assumption that buffer is + * UnSecured ! This is NOT a valid asumption ! A buffer is certified UnSecured + * only if 'buf_is(UnSecure, buffer)==true'. + */ + +/* memory atttributes */ +enum buf_is_attr { + CORE_MEM_CACHED, + CORE_MEM_NSEC_SHM, + CORE_MEM_NON_SEC, + CORE_MEM_SEC, + CORE_MEM_TEE_RAM, + CORE_MEM_TA_RAM, + CORE_MEM_SDP_MEM, + CORE_MEM_REG_SHM, +}; + +/* redirect legacy tee_vbuf_is() and tee_pbuf_is() to our routines */ +#define tee_pbuf_is core_pbuf_is +#define tee_vbuf_is core_vbuf_is + +/* Convenience macros */ +#define tee_pbuf_is_non_sec(buf, len) \ + core_pbuf_is(CORE_MEM_NON_SEC, (paddr_t)(buf), (len)) + +#define tee_pbuf_is_sec(buf, len) \ + core_pbuf_is(CORE_MEM_SEC, (paddr_t)(buf), (len)) + +#define tee_vbuf_is_non_sec(buf, len) \ + core_vbuf_is(CORE_MEM_NON_SEC, (void *)(buf), (len)) + +#define tee_vbuf_is_sec(buf, len) \ + core_vbuf_is(CORE_MEM_SEC, (void *)(buf), (len)) + +/* + * This function return true if the buf complies with supplied flags. + * If this function returns false buf doesn't comply with supplied flags + * or something went wrong. + * + * Note that returning false doesn't guarantee that buf complies with + * the complement of the supplied flags. + */ +bool core_pbuf_is(uint32_t flags, paddr_t pbuf, size_t len); + +/* + * Translates the supplied virtual address to a physical address and uses + * tee_phys_buf_is() to check the compliance of the buffer. + */ +bool core_vbuf_is(uint32_t flags, const void *vbuf, size_t len); + +/* + * Translate physical address to virtual address using specified mapping. + * Also tries to find proper mapping which have counterpart translation + * for specified length of data starting from given physical address. + * Len parameter can be set to 1 if caller knows that requested (pa + len) + * doesn`t cross mapping granule boundary. + * Returns NULL on failure or a valid virtual address on success. + */ +void *phys_to_virt(paddr_t pa, enum teecore_memtypes m, size_t len); + +/* + * Translate physical address to virtual address trying MEM_AREA_IO_SEC + * first then MEM_AREA_IO_NSEC if not found. Like phys_to_virt() tries + * to find proper mapping relying on length parameter. + * Returns NULL on failure or a valid virtual address on success. + */ +void *phys_to_virt_io(paddr_t pa, size_t len); + +/* + * Translate virtual address to physical address + * Returns 0 on failure or a valid physical address on success. + */ +paddr_t virt_to_phys(void *va); + +/* + * Return runtime usable address, irrespective of whether + * the MMU is enabled or not. In case of MMU enabled also will be performed + * check for requested amount of data is present in found mapping. + */ +vaddr_t core_mmu_get_va(paddr_t pa, enum teecore_memtypes type, size_t len); + +/* Return true if @va relates to a unpaged section else false */ +bool is_unpaged(void *va); + +struct io_pa_va { + paddr_t pa; + vaddr_t va; +}; + +/* + * Helper function to return a physical or virtual address for a device, + * depending on whether the MMU is enabled or not + * io_pa_or_va() uses secure mapped IO memory if found or fallback to + * non-secure mapped IO memory. + */ +vaddr_t io_pa_or_va_secure(struct io_pa_va *p, size_t len); +vaddr_t io_pa_or_va_nsec(struct io_pa_va *p, size_t len); +vaddr_t io_pa_or_va(struct io_pa_va *p, size_t len); + +#endif /* CORE_MEMPROT_H */ diff --git a/optee_os/core/include/mm/core_mmu.h b/optee_os/core/include/mm/core_mmu.h new file mode 100644 index 0000000..c6f2586 --- /dev/null +++ b/optee_os/core/include/mm/core_mmu.h @@ -0,0 +1,692 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef CORE_MMU_H +#define CORE_MMU_H + +#ifndef __ASSEMBLER__ +#include +#include +#include +#include +#include +#include +#endif + +#include +#include + +/* A small page is the smallest unit of memory that can be mapped */ +#define SMALL_PAGE_SIZE BIT(SMALL_PAGE_SHIFT) +#define SMALL_PAGE_MASK ((paddr_t)SMALL_PAGE_SIZE - 1) + +/* + * PGDIR is the translation table above the translation table that holds + * the pages. + */ +#define CORE_MMU_PGDIR_SIZE BIT(CORE_MMU_PGDIR_SHIFT) +#define CORE_MMU_PGDIR_MASK ((paddr_t)CORE_MMU_PGDIR_SIZE - 1) + +/* TA user space code, data, stack and heap are mapped using this granularity */ +#define CORE_MMU_USER_CODE_SIZE BIT(CORE_MMU_USER_CODE_SHIFT) +#define CORE_MMU_USER_CODE_MASK ((paddr_t)CORE_MMU_USER_CODE_SIZE - 1) + +/* TA user space parameters are mapped using this granularity */ +#define CORE_MMU_USER_PARAM_SIZE BIT(CORE_MMU_USER_PARAM_SHIFT) +#define CORE_MMU_USER_PARAM_MASK ((paddr_t)CORE_MMU_USER_PARAM_SIZE - 1) + +/* + * Identify mapping constraint: virtual base address is the physical start addr. + * If platform did not set some macros, some get default value. + */ +#ifndef TEE_RAM_VA_SIZE +#define TEE_RAM_VA_SIZE CORE_MMU_PGDIR_SIZE +#endif + +#ifndef TEE_LOAD_ADDR +#define TEE_LOAD_ADDR TEE_RAM_START +#endif + +#ifndef STACK_ALIGNMENT +#define STACK_ALIGNMENT (sizeof(long) * U(2)) +#endif + +#ifndef __ASSEMBLER__ +/* + * Memory area type: + * MEM_AREA_END: Reserved, marks the end of a table of mapping areas. + * MEM_AREA_TEE_RAM: core RAM (read/write/executable, secure, reserved to TEE) + * MEM_AREA_TEE_RAM_RX: core private read-only/executable memory (secure) + * MEM_AREA_TEE_RAM_RO: core private read-only/non-executable memory (secure) + * MEM_AREA_TEE_RAM_RW: core private read/write/non-executable memory (secure) + * MEM_AREA_INIT_RAM_RO: init private read-only/non-executable memory (secure) + * MEM_AREA_INIT_RAM_RX: init private read-only/executable memory (secure) + * MEM_AREA_NEX_RAM_RO: nexus private read-only/non-executable memory (secure) + * MEM_AREA_NEX_RAM_RW: nexus private r/w/non-executable memory (secure) + * MEM_AREA_TEE_COHERENT: teecore coherent RAM (secure, reserved to TEE) + * MEM_AREA_TEE_ASAN: core address sanitizer RAM (secure, reserved to TEE) + * MEM_AREA_IDENTITY_MAP_RX: core identity mapped r/o executable memory (secure) + * MEM_AREA_TA_RAM: Secure RAM where teecore loads/exec TA instances. + * MEM_AREA_NSEC_SHM: NonSecure shared RAM between NSec and TEE. + * MEM_AREA_NEX_NSEC_SHM: nexus non-secure shared RAM between NSec and TEE. + * MEM_AREA_RAM_NSEC: NonSecure RAM storing data + * MEM_AREA_RAM_SEC: Secure RAM storing some secrets + * MEM_AREA_IO_NSEC: NonSecure HW mapped registers + * MEM_AREA_IO_SEC: Secure HW mapped registers + * MEM_AREA_EXT_DT: Memory loads external device tree + * MEM_AREA_MANIFEST_DT: Memory loads manifest device tree + * MEM_AREA_RES_VASPACE: Reserved virtual memory space + * MEM_AREA_SHM_VASPACE: Virtual memory space for dynamic shared memory buffers + * MEM_AREA_TS_VASPACE: TS va space, only used with phys_to_virt() + * MEM_AREA_DDR_OVERALL: Overall DDR address range, candidate to dynamic shm. + * MEM_AREA_SEC_RAM_OVERALL: Whole secure RAM + * MEM_AREA_MAXTYPE: lower invalid 'type' value + */ +enum teecore_memtypes { + MEM_AREA_END = 0, + MEM_AREA_TEE_RAM, + MEM_AREA_TEE_RAM_RX, + MEM_AREA_TEE_RAM_RO, + MEM_AREA_TEE_RAM_RW, + MEM_AREA_INIT_RAM_RO, + MEM_AREA_INIT_RAM_RX, + MEM_AREA_NEX_RAM_RO, + MEM_AREA_NEX_RAM_RW, + MEM_AREA_TEE_COHERENT, + MEM_AREA_TEE_ASAN, + MEM_AREA_IDENTITY_MAP_RX, + MEM_AREA_TA_RAM, + MEM_AREA_NSEC_SHM, + MEM_AREA_NEX_NSEC_SHM, + MEM_AREA_RAM_NSEC, + MEM_AREA_RAM_SEC, + MEM_AREA_IO_NSEC, + MEM_AREA_IO_SEC, + MEM_AREA_EXT_DT, + MEM_AREA_MANIFEST_DT, + MEM_AREA_RES_VASPACE, + MEM_AREA_SHM_VASPACE, + MEM_AREA_TS_VASPACE, + MEM_AREA_PAGER_VASPACE, + MEM_AREA_SDP_MEM, + MEM_AREA_DDR_OVERALL, + MEM_AREA_SEC_RAM_OVERALL, + MEM_AREA_MAXTYPE +}; + +static inline const char *teecore_memtype_name(enum teecore_memtypes type) +{ + static const char * const names[] = { + [MEM_AREA_END] = "END", + [MEM_AREA_TEE_RAM] = "TEE_RAM_RWX", + [MEM_AREA_TEE_RAM_RX] = "TEE_RAM_RX", + [MEM_AREA_TEE_RAM_RO] = "TEE_RAM_RO", + [MEM_AREA_TEE_RAM_RW] = "TEE_RAM_RW", + [MEM_AREA_INIT_RAM_RO] = "INIT_RAM_RO", + [MEM_AREA_INIT_RAM_RX] = "INIT_RAM_RX", + [MEM_AREA_NEX_RAM_RO] = "NEX_RAM_RO", + [MEM_AREA_NEX_RAM_RW] = "NEX_RAM_RW", + [MEM_AREA_TEE_ASAN] = "TEE_ASAN", + [MEM_AREA_IDENTITY_MAP_RX] = "IDENTITY_MAP_RX", + [MEM_AREA_TEE_COHERENT] = "TEE_COHERENT", + [MEM_AREA_TA_RAM] = "TA_RAM", + [MEM_AREA_NSEC_SHM] = "NSEC_SHM", + [MEM_AREA_NEX_NSEC_SHM] = "NEX_NSEC_SHM", + [MEM_AREA_RAM_NSEC] = "RAM_NSEC", + [MEM_AREA_RAM_SEC] = "RAM_SEC", + [MEM_AREA_IO_NSEC] = "IO_NSEC", + [MEM_AREA_IO_SEC] = "IO_SEC", + [MEM_AREA_EXT_DT] = "EXT_DT", + [MEM_AREA_MANIFEST_DT] = "MANIFEST_DT", + [MEM_AREA_RES_VASPACE] = "RES_VASPACE", + [MEM_AREA_SHM_VASPACE] = "SHM_VASPACE", + [MEM_AREA_TS_VASPACE] = "TS_VASPACE", + [MEM_AREA_PAGER_VASPACE] = "PAGER_VASPACE", + [MEM_AREA_SDP_MEM] = "SDP_MEM", + [MEM_AREA_DDR_OVERALL] = "DDR_OVERALL", + [MEM_AREA_SEC_RAM_OVERALL] = "SEC_RAM_OVERALL", + }; + + COMPILE_TIME_ASSERT(ARRAY_SIZE(names) == MEM_AREA_MAXTYPE); + return names[type]; +} + +#ifdef CFG_CORE_RWDATA_NOEXEC +#define MEM_AREA_TEE_RAM_RW_DATA MEM_AREA_TEE_RAM_RW +#else +#define MEM_AREA_TEE_RAM_RW_DATA MEM_AREA_TEE_RAM +#endif + +struct core_mmu_phys_mem { + const char *name; + enum teecore_memtypes type; + __extension__ union { +#if __SIZEOF_LONG__ != __SIZEOF_PADDR__ + struct { + uint32_t lo_addr; + uint32_t hi_addr; + }; +#endif + paddr_t addr; + }; + __extension__ union { +#if __SIZEOF_LONG__ != __SIZEOF_PADDR__ + struct { + uint32_t lo_size; + uint32_t hi_size; + }; +#endif + paddr_size_t size; + }; +}; + +#define __register_memory(_name, _type, _addr, _size, _section) \ + SCATTERED_ARRAY_DEFINE_ITEM(_section, struct core_mmu_phys_mem) = \ + { .name = (_name), .type = (_type), .addr = (_addr), \ + .size = (_size) } + +#if __SIZEOF_LONG__ != __SIZEOF_PADDR__ +#define __register_memory_ul(_name, _type, _addr, _size, _section) \ + SCATTERED_ARRAY_DEFINE_ITEM(_section, struct core_mmu_phys_mem) = \ + { .name = (_name), .type = (_type), .lo_addr = (_addr), \ + .lo_size = (_size) } +#else +#define __register_memory_ul(_name, _type, _addr, _size, _section) \ + __register_memory(_name, _type, _addr, _size, _section) +#endif + +#define register_phys_mem(type, addr, size) \ + __register_memory(#addr, (type), (addr), (size), \ + phys_mem_map) + +#define register_phys_mem_ul(type, addr, size) \ + __register_memory_ul(#addr, (type), (addr), (size), \ + phys_mem_map) + +/* Same as register_phys_mem() but with PGDIR_SIZE granularity */ +#define register_phys_mem_pgdir(type, addr, size) \ + __register_memory(#addr, type, ROUNDDOWN(addr, CORE_MMU_PGDIR_SIZE), \ + ROUNDUP(size + addr - \ + ROUNDDOWN(addr, CORE_MMU_PGDIR_SIZE), \ + CORE_MMU_PGDIR_SIZE), phys_mem_map) + +#ifdef CFG_SECURE_DATA_PATH +#define register_sdp_mem(addr, size) \ + __register_memory(#addr, MEM_AREA_SDP_MEM, (addr), (size), \ + phys_sdp_mem) +#else +#define register_sdp_mem(addr, size) \ + static int CONCAT(__register_sdp_mem_unused, __COUNTER__) \ + __unused +#endif + +/* register_dynamic_shm() is deprecated, please use register_ddr() instead */ +#define register_dynamic_shm(addr, size) \ + __register_memory(#addr, MEM_AREA_DDR_OVERALL, (addr), (size), \ + phys_ddr_overall_compat) + +/* + * register_ddr() - Define a memory range + * @addr: Base address + * @size: Length + * + * This macro can be used multiple times to define disjoint ranges. While + * initializing holes are carved out of these ranges where it overlaps with + * special memory, for instance memory registered with register_sdp_mem(). + * + * The memory that remains is accepted as non-secure shared memory when + * communicating with normal world. + * + * This macro is an alternative to supply the memory description with a + * devicetree blob. + */ +#define register_ddr(addr, size) \ + __register_memory(#addr, MEM_AREA_DDR_OVERALL, (addr), \ + (size), phys_ddr_overall) + +#define phys_ddr_overall_begin \ + SCATTERED_ARRAY_BEGIN(phys_ddr_overall, struct core_mmu_phys_mem) + +#define phys_ddr_overall_end \ + SCATTERED_ARRAY_END(phys_ddr_overall, struct core_mmu_phys_mem) + +#define phys_ddr_overall_compat_begin \ + SCATTERED_ARRAY_BEGIN(phys_ddr_overall_compat, struct core_mmu_phys_mem) + +#define phys_ddr_overall_compat_end \ + SCATTERED_ARRAY_END(phys_ddr_overall_compat, struct core_mmu_phys_mem) + +#define phys_sdp_mem_begin \ + SCATTERED_ARRAY_BEGIN(phys_sdp_mem, struct core_mmu_phys_mem) + +#define phys_sdp_mem_end \ + SCATTERED_ARRAY_END(phys_sdp_mem, struct core_mmu_phys_mem) + +#define phys_mem_map_begin \ + SCATTERED_ARRAY_BEGIN(phys_mem_map, struct core_mmu_phys_mem) + +#define phys_mem_map_end \ + SCATTERED_ARRAY_END(phys_mem_map, struct core_mmu_phys_mem) + +#ifdef CFG_CORE_RESERVED_SHM +/* Default NSec shared memory allocated from NSec world */ +extern unsigned long default_nsec_shm_paddr; +extern unsigned long default_nsec_shm_size; +#endif + +/* + * Physical load address of OP-TEE updated during boot if needed to reflect + * the value used. + */ +#ifdef CFG_CORE_PHYS_RELOCATABLE +extern unsigned long core_mmu_tee_load_pa; +#else +extern const unsigned long core_mmu_tee_load_pa; +#endif + +void core_init_mmu_map(unsigned long seed, struct core_mmu_config *cfg); +void core_init_mmu_regs(struct core_mmu_config *cfg); + +/* Arch specific function to help optimizing 1 MMU xlat table */ +bool core_mmu_prefer_tee_ram_at_top(paddr_t paddr); + +/* + * struct mmu_partition - stores MMU partition. + * + * Basically it represent whole MMU mapping. It is possible + * to create multiple partitions, and change them in runtime, + * effectively changing how OP-TEE sees memory. + * This is opaque struct which is defined differently for + * v7 and LPAE MMUs + * + * This structure used mostly when virtualization is enabled. + * When CFG_NS_VIRTUALIZATION==n only default partition exists. + */ +struct mmu_partition; + +/* + * core_mmu_get_user_va_range() - Return range of user va space + * @base: Lowest user virtual address + * @size: Size in bytes of user address space + */ +void core_mmu_get_user_va_range(vaddr_t *base, size_t *size); + +/* + * enum core_mmu_fault - different kinds of faults + * @CORE_MMU_FAULT_ALIGNMENT: alignment fault + * @CORE_MMU_FAULT_DEBUG_EVENT: debug event + * @CORE_MMU_FAULT_TRANSLATION: translation fault + * @CORE_MMU_FAULT_WRITE_PERMISSION: Permission fault during write + * @CORE_MMU_FAULT_READ_PERMISSION: Permission fault during read + * @CORE_MMU_FAULT_ASYNC_EXTERNAL: asynchronous external abort + * @CORE_MMU_FAULT_ACCESS_BIT: access bit fault + * @CORE_MMU_FAULT_TAG_CHECK: tag check fault + * @CORE_MMU_FAULT_OTHER: Other/unknown fault + */ +enum core_mmu_fault { + CORE_MMU_FAULT_ALIGNMENT, + CORE_MMU_FAULT_DEBUG_EVENT, + CORE_MMU_FAULT_TRANSLATION, + CORE_MMU_FAULT_WRITE_PERMISSION, + CORE_MMU_FAULT_READ_PERMISSION, + CORE_MMU_FAULT_ASYNC_EXTERNAL, + CORE_MMU_FAULT_ACCESS_BIT, + CORE_MMU_FAULT_TAG_CHECK, + CORE_MMU_FAULT_OTHER, +}; + +/* + * core_mmu_get_fault_type() - get fault type + * @fault_descr: Content of fault status or exception syndrome register + * @returns an enum describing the content of fault status register. + */ +enum core_mmu_fault core_mmu_get_fault_type(uint32_t fault_descr); + +/* + * core_mm_type_to_attr() - convert memory type to attribute + * @t: memory type + * @returns an attribute that can be passed to core_mm_set_entry() and friends + */ +uint32_t core_mmu_type_to_attr(enum teecore_memtypes t); + +/* + * core_mmu_create_user_map() - Create user mode mapping + * @uctx: Pointer to user mode context + * @map: MMU configuration to use when activating this VA space + */ +void core_mmu_create_user_map(struct user_mode_ctx *uctx, + struct core_mmu_user_map *map); +/* + * core_mmu_get_user_map() - Reads current MMU configuration for user VA space + * @map: MMU configuration for current user VA space. + */ +void core_mmu_get_user_map(struct core_mmu_user_map *map); + +/* + * core_mmu_set_user_map() - Set new MMU configuration for user VA space + * @map: User context MMU configuration or NULL to set core VA space + * + * Activate user VA space mapping and set its ASID if @map is not NULL, + * otherwise activate core mapping and set ASID to 0. + */ +void core_mmu_set_user_map(struct core_mmu_user_map *map); + +/* + * struct core_mmu_table_info - Properties for a translation table + * @table: Pointer to translation table + * @va_base: VA base address of the transaltion table + * @level: Translation table level + * @next_level: Finer grained translation table level according to @level. + * @shift: The shift of each entry in the table + * @num_entries: Number of entries in this table. + */ +struct core_mmu_table_info { + void *table; + vaddr_t va_base; + unsigned num_entries; +#ifdef CFG_NS_VIRTUALIZATION + struct mmu_partition *prtn; +#endif + uint8_t level; + uint8_t shift; + uint8_t next_level; +}; + +/* + * core_mmu_find_table() - Locates a translation table + * @prtn: MMU partition where search should be performed + * @va: Virtual address for the table to cover + * @max_level: Don't traverse beyond this level + * @tbl_info: Pointer to where to store properties. + * @return true if a translation table was found, false on error + */ +bool core_mmu_find_table(struct mmu_partition *prtn, vaddr_t va, + unsigned max_level, + struct core_mmu_table_info *tbl_info); + +/* + * core_mmu_entry_to_finer_grained() - divide mapping at current level into + * smaller ones so memory can be mapped with finer granularity + * @tbl_info: table where target record located + * @idx: index of record for which a pdgir must be setup. + * @secure: true/false if pgdir maps secure/non-secure memory (32bit mmu) + * @return true on successful, false on error + */ +bool core_mmu_entry_to_finer_grained(struct core_mmu_table_info *tbl_info, + unsigned int idx, bool secure); + +void core_mmu_set_entry_primitive(void *table, size_t level, size_t idx, + paddr_t pa, uint32_t attr); + +void core_mmu_get_user_pgdir(struct core_mmu_table_info *pgd_info); + +/* + * core_mmu_set_entry() - Set entry in translation table + * @tbl_info: Translation table properties + * @idx: Index of entry to update + * @pa: Physical address to assign entry + * @attr: Attributes to assign entry + */ +void core_mmu_set_entry(struct core_mmu_table_info *tbl_info, unsigned idx, + paddr_t pa, uint32_t attr); + +void core_mmu_get_entry_primitive(const void *table, size_t level, size_t idx, + paddr_t *pa, uint32_t *attr); + +/* + * core_mmu_get_entry() - Get entry from translation table + * @tbl_info: Translation table properties + * @idx: Index of entry to read + * @pa: Physical address is returned here if pa is not NULL + * @attr: Attributues are returned here if attr is not NULL + */ +void core_mmu_get_entry(struct core_mmu_table_info *tbl_info, unsigned idx, + paddr_t *pa, uint32_t *attr); + +/* + * core_mmu_va2idx() - Translate from virtual address to table index + * @tbl_info: Translation table properties + * @va: Virtual address to translate + * @returns index in transaltion table + */ +static inline unsigned core_mmu_va2idx(struct core_mmu_table_info *tbl_info, + vaddr_t va) +{ + return (va - tbl_info->va_base) >> tbl_info->shift; +} + +/* + * core_mmu_idx2va() - Translate from table index to virtual address + * @tbl_info: Translation table properties + * @idx: Index to translate + * @returns Virtual address + */ +static inline vaddr_t core_mmu_idx2va(struct core_mmu_table_info *tbl_info, + unsigned idx) +{ + return (idx << tbl_info->shift) + tbl_info->va_base; +} + +/* + * core_mmu_get_block_offset() - Get offset inside a block/page + * @tbl_info: Translation table properties + * @pa: Physical address + * @returns offset within one block of the translation table + */ +static inline size_t core_mmu_get_block_offset( + struct core_mmu_table_info *tbl_info, paddr_t pa) +{ + return pa & ((1 << tbl_info->shift) - 1); +} + +/* + * core_mmu_is_dynamic_vaspace() - Check if memory region belongs to + * empty virtual address space that is used for dymanic mappings + * @mm: memory region to be checked + * @returns result of the check + */ +static inline bool core_mmu_is_dynamic_vaspace(struct tee_mmap_region *mm) +{ + return mm->type == MEM_AREA_RES_VASPACE || + mm->type == MEM_AREA_SHM_VASPACE; +} + +/* + * core_mmu_map_pages() - map list of pages at given virtual address + * @vstart: Virtual address where mapping begins + * @pages: Array of page addresses + * @num_pages: Number of pages + * @memtype: Type of memmory to be mapped + * + * Note: This function asserts that pages are not mapped executeable for + * kernel (privileged) mode. + * + * @returns: TEE_SUCCESS on success, TEE_ERROR_XXX on error + */ +TEE_Result core_mmu_map_pages(vaddr_t vstart, paddr_t *pages, size_t num_pages, + enum teecore_memtypes memtype); + +/* + * core_mmu_map_contiguous_pages() - map range of pages at given virtual address + * @vstart: Virtual address where mapping begins + * @pstart: Physical address of the first page + * @num_pages: Number of pages + * @memtype: Type of memmory to be mapped + * + * Note: This function asserts that pages are not mapped executeable for + * kernel (privileged) mode. + * + * @returns: TEE_SUCCESS on success, TEE_ERROR_XXX on error + */ +TEE_Result core_mmu_map_contiguous_pages(vaddr_t vstart, paddr_t pstart, + size_t num_pages, + enum teecore_memtypes memtype); + +/* + * core_mmu_unmap_pages() - remove mapping at given virtual address + * @vstart: Virtual address where mapping begins + * @num_pages: Number of pages to unmap + */ +void core_mmu_unmap_pages(vaddr_t vstart, size_t num_pages); + +/* + * core_mmu_user_mapping_is_active() - Report if user mapping is active + * @returns true if a user VA space is active, false if user VA space is + * inactive. + */ +bool core_mmu_user_mapping_is_active(void); + +/* + * core_mmu_mattr_is_ok() - Check that supplied mem attributes can be used + * @returns true if the attributes can be used, false if not. + */ +bool core_mmu_mattr_is_ok(uint32_t mattr); + +void core_mmu_get_mem_by_type(enum teecore_memtypes type, vaddr_t *s, + vaddr_t *e); + +enum teecore_memtypes core_mmu_get_type_by_pa(paddr_t pa); + +/* routines to retreive shared mem configuration */ +static inline bool core_mmu_is_shm_cached(void) +{ + return mattr_is_cached(core_mmu_type_to_attr(MEM_AREA_NSEC_SHM)); +} + +TEE_Result core_mmu_remove_mapping(enum teecore_memtypes type, void *addr, + size_t len); +void *core_mmu_add_mapping(enum teecore_memtypes type, paddr_t addr, + size_t len); + +/* + * core_mmu_find_mapping_exclusive() - Find mapping of specified type and + * length. If more than one mapping of + * specified type is present, NULL will be + * returned. + * @type: memory type + * @len: length in bytes + */ +struct tee_mmap_region * +core_mmu_find_mapping_exclusive(enum teecore_memtypes type, size_t len); + +/* + * tlbi_va_range() - Invalidate TLB for virtual address range + * @va: start virtual address, must be a multiple of @granule + * @len: length in bytes of range, must be a multiple of @granule + * @granule: granularity of mapping, supported values are + * CORE_MMU_PGDIR_SIZE or SMALL_PAGE_SIZE. This value must + * match the actual mappings. + */ +void tlbi_va_range(vaddr_t va, size_t len, size_t granule); + +/* + * tlbi_va_range_asid() - Invalidate TLB for virtual address range for + * a specific ASID + * @va: start virtual address, must be a multiple of @granule + * @len: length in bytes of range, must be a multiple of @granule + * @granule: granularity of mapping, supported values are + * CORE_MMU_PGDIR_SIZE or SMALL_PAGE_SIZE. This value must + * match the actual mappings. + * @asid: Address space identifier + */ +void tlbi_va_range_asid(vaddr_t va, size_t len, size_t granule, uint32_t asid); + +/* Check cpu mmu enabled or not */ +bool cpu_mmu_enabled(void); + +#ifdef CFG_CORE_DYN_SHM +/* + * Check if platform defines nsec DDR range(s). + * Static SHM (MEM_AREA_NSEC_SHM) is not covered by this API as it is + * always present. + */ +bool core_mmu_nsec_ddr_is_defined(void); + +void core_mmu_set_discovered_nsec_ddr(struct core_mmu_phys_mem *start, + size_t nelems); +#endif + +/* Initialize MMU partition */ +void core_init_mmu_prtn(struct mmu_partition *prtn, struct tee_mmap_region *mm); + +unsigned int asid_alloc(void); +void asid_free(unsigned int asid); + +#ifdef CFG_SECURE_DATA_PATH +/* Alloc and fill SDP memory objects table - table is NULL terminated */ +struct mobj **core_sdp_mem_create_mobjs(void); +#endif + +#ifdef CFG_NS_VIRTUALIZATION +size_t core_mmu_get_total_pages_size(void); +struct mmu_partition *core_alloc_mmu_prtn(void *tables); +void core_free_mmu_prtn(struct mmu_partition *prtn); +void core_mmu_set_prtn(struct mmu_partition *prtn); +void core_mmu_set_default_prtn(void); +void core_mmu_set_default_prtn_tbl(void); +#endif + +void core_mmu_init_virtualization(void); + +/* init some allocation pools */ +void core_mmu_init_ta_ram(void); + +void core_init_mmu(struct tee_mmap_region *mm); + +void core_mmu_set_info_table(struct core_mmu_table_info *tbl_info, + unsigned int level, vaddr_t va_base, void *table); +void core_mmu_populate_user_map(struct core_mmu_table_info *dir_info, + struct user_mode_ctx *uctx); +void core_mmu_map_region(struct mmu_partition *prtn, + struct tee_mmap_region *mm); + +bool arch_va2pa_helper(void *va, paddr_t *pa); + +static inline bool core_mmap_is_end_of_table(const struct tee_mmap_region *mm) +{ + return mm->type == MEM_AREA_END; +} + +static inline bool core_mmu_check_end_pa(paddr_t pa, size_t len) +{ + paddr_t end_pa = 0; + + if (ADD_OVERFLOW(pa, len - 1, &end_pa)) + return false; + return core_mmu_check_max_pa(end_pa); +} + +/* + * core_mmu_set_secure_memory() - set physical secure memory range + * @base: base address of secure memory + * @size: size of secure memory + * + * The physical secure memory range is not known in advance when OP-TEE is + * relocatable, this information must be supplied once during boot before + * the translation tables can be initialized and the MMU enabled. + */ +void core_mmu_set_secure_memory(paddr_t base, size_t size); + +/* + * core_mmu_get_secure_memory() - get physical secure memory range + * @base: base address of secure memory + * @size: size of secure memory + * + * The physical secure memory range returned covers at least the memory + * range used by OP-TEE Core, but may cover more memory depending on the + * configuration. + */ +void core_mmu_get_secure_memory(paddr_t *base, paddr_size_t *size); + +/* + * core_mmu_get_ta_range() - get physical memory range reserved for TAs + * @base: [out] range base address ref or NULL + * @size: [out] range size ref or NULL + */ +void core_mmu_get_ta_range(paddr_t *base, size_t *size); + +#endif /*__ASSEMBLER__*/ + +#endif /* CORE_MMU_H */ diff --git a/optee_os/core/include/mm/file.h b/optee_os/core/include/mm/file.h new file mode 100644 index 0000000..1879507 --- /dev/null +++ b/optee_os/core/include/mm/file.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef __MM_FILE_H +#define __MM_FILE_H + +#include +#include +#include + +/* This is supposed to be large enough to hold any hash or tag used */ +#define FILE_TAG_SIZE TEE_SHA256_HASH_SIZE + +/* + * struct file_slice - one slice of a file + * @fobj: Fobj holding the data of this slice + * @page_offset: Offset in pages into the file where the @fobj is + * located. + */ +struct file_slice { + struct fobj *fobj; + unsigned int page_offset; +}; + +struct file; + +/* + * file_lock() - Lock the file + * @f: File pointer + * + * Waits until the file can be locked and with the file put in locked state. + */ +void file_lock(struct file *f); + +/* + * file_lock() - Try to lock the file without blocking + * @f: File pointer + * + * Returns false if file cannot be locked without blocking. + * Returns true if the file has been put in locked state. + */ +bool file_trylock(struct file *f); + +/* + * file_unlock() - Unlock the file + * @f: File pointer + * + * File must be in locked state. Releases the previous lock and returns. + */ +void file_unlock(struct file *f); + +/* + * file_add_slice() - Add a slice to a file + * @f: File pointer + * @fobj: Fobj holding the data of this slice + * @page_offset: Offset in pages into the file (@f) where the @fobj is + * located. + * + * File must be in locked state. + * + * Returns TEE_SUCCESS on success or a TEE_ERROR_* code on failure. + */ +TEE_Result file_add_slice(struct file *f, struct fobj *fobj, + unsigned int page_offset); + +/* + * file_get() - Increase file reference counter + * @f: File pointer + * + * Returns @f, if @f isn't NULL its reference counter is first increased. + */ +struct file *file_get(struct file *f); + +/* + * file_get_by_tag() - Finds a file based on tag and increase reference counter + * @tag: Tag of the file + * @taglen: Length of @tag + * + * If a file doesn't exist it's created with the supplied tag. + * + * Returns a file with an increased reference counter, or NULL on failure. + */ +struct file *file_get_by_tag(const uint8_t *tag, unsigned int taglen); + +/* + * file_put() - Decrease reference counter of file + * @f: File pointer + * + * If reference counter reaches 0, matching the numbers of file_new() + + * file_get() + file_get_by_tag(), the file is removed with reference + * counters for all contained fobjs decreased. + */ +void file_put(struct file *f); + +/* + * file_find_slice() - Find a slice covering the @page_offset + * @f: File pointer + * @page_offset: Offset that must be covered + * + * File must be in locked state. + * + * If a matching file slice is found it is returned, else NULL is returned. + */ +struct file_slice *file_find_slice(struct file *f, unsigned int page_offset); + +#endif /*__MM_FILE_H*/ + diff --git a/optee_os/core/include/mm/fobj.h b/optee_os/core/include/mm/fobj.h new file mode 100644 index 0000000..1428660 --- /dev/null +++ b/optee_os/core/include/mm/fobj.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019-2021, Linaro Limited + */ + +#ifndef __MM_FOBJ_H +#define __MM_FOBJ_H + +#include +#include +#include +#include +#include +#include + +/* + * struct fobj - file object storage abstraction + * @ops: Operations pointer + * @num_pages: Number of pages covered + * @refc: Reference counter + */ +struct fobj { + const struct fobj_ops *ops; + unsigned int num_pages; + struct refcount refc; +#ifdef CFG_WITH_PAGER + struct vm_paged_region_head regions; +#endif +}; + +/* + * struct fobj_ops - operations struct for struct fobj + * @free: Frees the @fobj + * @load_page: Loads page with index @page_idx at address @va + * @save_page: Saves page with index @page_idx from address @va + * @get_iv_vaddr: Returns virtual address of tag and IV for the page at + * @page_idx if tag and IV are paged for this fobj + * @get_pa: Returns physical address of page at @page_idx if not paged + */ +struct fobj_ops { + void (*free)(struct fobj *fobj); +#ifdef CFG_WITH_PAGER + TEE_Result (*load_page)(struct fobj *fobj, unsigned int page_idx, + void *va); + TEE_Result (*save_page)(struct fobj *fobj, unsigned int page_idx, + const void *va); + vaddr_t (*get_iv_vaddr)(struct fobj *fobj, unsigned int page_idx); +#endif + paddr_t (*get_pa)(struct fobj *fobj, unsigned int page_idx); +}; + +#ifdef CFG_WITH_PAGER +/* + * fobj_locked_paged_alloc() - Allocate storage which is locked in memory + * @num_pages: Number of pages covered + * + * This object only supports loading pages zero initialized. Saving a page + * will result in an error. + * + * Returns a valid pointer on success or NULL on failure. + */ +struct fobj *fobj_locked_paged_alloc(unsigned int num_pages); + +/* + * fobj_rw_paged_alloc() - Allocate read/write storage + * @num_pages: Number of pages covered + * + * This object supports both load and saving of pages. Pages are zero + * initialized the first time they are loaded. + * + * Returns a valid pointer on success or NULL on failure. + */ +struct fobj *fobj_rw_paged_alloc(unsigned int num_pages); + +/* + * fobj_ro_paged_alloc() - Allocate initialized read-only storage + * @num_pages: Number of pages covered + * @hashes: Hashes to verify the pages + * @store: Clear text data for all pages + * + * This object only support loading pages with an already provided content + * in @store. When a page is loaded it will be verified against an hash in + * @hash. Saving a page will result in an error. + * + * Returns a valid pointer on success or NULL on failure. + */ +struct fobj *fobj_ro_paged_alloc(unsigned int num_pages, void *hashes, + void *store); + +/* + * fobj_ro_reloc_paged_alloc() - Allocate initialized read-only storage with + * relocation + * @num_pages: Number of pages covered + * @hashes: Hashes to verify the pages + * @reloc_offs: Offset from the base address in the relocations in @reloc + * @reloc: Relocation data + * @reloc_len: Length of relocation data + * @store: Clear text data for all pages + * + * This object is like fobj_ro_paged_alloc() above, but in addition the + * relocation information is applied to a populated page. This makes sure + * the offset to which all pages are relocated doesn't leak out to storage. + * + * Returns a valid pointer on success or NULL on failure. + */ +struct fobj *fobj_ro_reloc_paged_alloc(unsigned int num_pages, void *hashes, + unsigned int reloc_offs, + const void *reloc, + unsigned int reloc_len, void *store); + +/* + * fobj_load_page() - Load a page into memory + * @fobj: Fobj pointer + * @page_index: Index of page in @fobj + * @va: Address where content should be stored and verified + * + * Returns TEE_SUCCESS on success or TEE_ERROR_* on failure. + */ +static inline TEE_Result fobj_load_page(struct fobj *fobj, + unsigned int page_idx, void *va) +{ + if (fobj) + return fobj->ops->load_page(fobj, page_idx, va); + + return TEE_ERROR_GENERIC; +} + +/* + * fobj_save_page() - Save a page into storage + * @fobj: Fobj pointer + * @page_index: Index of page in @fobj + * @va: Address of the page to store. + * + * Returns TEE_SUCCESS on success or TEE_ERROR_* on failure. + */ +static inline TEE_Result fobj_save_page(struct fobj *fobj, + unsigned int page_idx, const void *va) +{ + if (fobj) + return fobj->ops->save_page(fobj, page_idx, va); + + return TEE_ERROR_GENERIC; +} + +static inline vaddr_t fobj_get_iv_vaddr(struct fobj *fobj, + unsigned int page_idx) +{ + if (fobj && fobj->ops->get_iv_vaddr) + return fobj->ops->get_iv_vaddr(fobj, page_idx); + + return 0; +} +#endif + +/* + * fobj_ta_mem_alloc() - Allocates TA memory + * @num_pages: Number of pages + * + * If paging of user TAs read/write paged fobj is allocated otherwise a + * fobj which uses unpaged secure memory directly. + * + * Returns a valid pointer on success or NULL on failure. + */ +#ifdef CFG_PAGED_USER_TA +#define fobj_ta_mem_alloc(num_pages) fobj_rw_paged_alloc(num_pages) +#else +/* + * fobj_sec_mem_alloc() - Allocates storage directly in secure memory + * @num_pages: Number of pages + * + * Returns a valid pointer on success or NULL on failure. + */ +struct fobj *fobj_sec_mem_alloc(unsigned int num_pages); + +#define fobj_ta_mem_alloc(num_pages) fobj_sec_mem_alloc(num_pages) +#endif + +/* + * fobj_get() - Increase fobj reference count + * @fobj: Fobj pointer + * + * Returns @fobj, if @fobj isn't NULL its reference counter is first + * increased. + */ +static inline struct fobj *fobj_get(struct fobj *fobj) +{ + if (fobj && !refcount_inc(&fobj->refc)) + panic(); + + return fobj; +} + +/* + * fobj_put() - Decrease reference counter of fobj + * @fobj: Fobj pointer + * + * If reference counter reaches 0, matching the numbers of fobj_alloc_*() + + * fobj_get(), the fobj is freed. + */ +static inline void fobj_put(struct fobj *fobj) +{ + if (fobj && refcount_dec(&fobj->refc)) + fobj->ops->free(fobj); +} + +#endif /*__MM_FOBJ_H*/ diff --git a/optee_os/core/include/mm/mobj.h b/optee_os/core/include/mm/mobj.h new file mode 100644 index 0000000..3b73b00 --- /dev/null +++ b/optee_os/core/include/mm/mobj.h @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, 2022 Linaro Limited + */ + +#ifndef __MM_MOBJ_H +#define __MM_MOBJ_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +struct mobj { + const struct mobj_ops *ops; + size_t size; + size_t phys_granule; + struct refcount refc; +}; + +struct mobj_ops { + void *(*get_va)(struct mobj *mobj, size_t offs, size_t len); + TEE_Result (*get_pa)(struct mobj *mobj, size_t offs, size_t granule, + paddr_t *pa); + size_t (*get_phys_offs)(struct mobj *mobj, size_t granule); + TEE_Result (*get_mem_type)(struct mobj *mobj, uint32_t *mt); + bool (*matches)(struct mobj *mobj, enum buf_is_attr attr); + void (*free)(struct mobj *mobj); + uint64_t (*get_cookie)(struct mobj *mobj); + struct fobj *(*get_fobj)(struct mobj *mobj); + TEE_Result (*inc_map)(struct mobj *mobj); + TEE_Result (*dec_map)(struct mobj *mobj); +}; + +extern struct mobj mobj_virt; +extern struct mobj *mobj_sec_ddr; +extern struct mobj *mobj_tee_ram_rx; +extern struct mobj *mobj_tee_ram_rw; + +/* + * mobj_get_va() - get virtual address of a mapped mobj + * @mobj: memory object + * @offset: find the va of this offset into @mobj + * @len: how many bytes after @offset that must be valid, can be 1 if + * the caller knows by other means that the expected buffer is + * available. + * + * return a virtual address on success or NULL on error + */ +static inline void *mobj_get_va(struct mobj *mobj, size_t offset, size_t len) +{ + if (mobj && mobj->ops && mobj->ops->get_va) + return mobj->ops->get_va(mobj, offset, len); + return NULL; +} + +static inline TEE_Result mobj_get_pa(struct mobj *mobj, size_t offs, + size_t granule, paddr_t *pa) +{ + if (mobj && mobj->ops && mobj->ops->get_pa) + return mobj->ops->get_pa(mobj, offs, granule, pa); + return TEE_ERROR_GENERIC; +} + +static inline size_t mobj_get_phys_offs(struct mobj *mobj, size_t granule) +{ + if (mobj && mobj->ops && mobj->ops->get_phys_offs) + return mobj->ops->get_phys_offs(mobj, granule); + return 0; +} + +static inline TEE_Result mobj_get_mem_type(struct mobj *mobj, uint32_t *mt) +{ + if (mobj && mobj->ops && mobj->ops->get_mem_type) + return mobj->ops->get_mem_type(mobj, mt); + return TEE_ERROR_GENERIC; +} + +static inline bool mobj_matches(struct mobj *mobj, enum buf_is_attr attr) +{ + if (mobj && mobj->ops && mobj->ops->matches) + return mobj->ops->matches(mobj, attr); + return false; +} + +/** + * mobj_inc_map() - increase map count + * @mobj: pointer to a MOBJ + * + * Maps the MOBJ if it isn't mapped already and increases the map count + * Each call to mobj_inc_map() is supposed to be matches by a call to + * mobj_dec_map(). + * + * Returns TEE_SUCCESS on success or an error code on failure + */ +static inline TEE_Result mobj_inc_map(struct mobj *mobj) +{ + if (mobj && mobj->ops) { + if (mobj->ops->inc_map) + return mobj->ops->inc_map(mobj); + return TEE_SUCCESS; + } + return TEE_ERROR_GENERIC; +} + +/** + * mobj_dec_map() - decrease map count + * @mobj: pointer to a MOBJ + * + * Decreases the map count and also unmaps the MOBJ if the map count + * reaches 0. Each call to mobj_inc_map() is supposed to be matched by a + * call to mobj_dec_map(). + * + * Returns TEE_SUCCESS on success or an error code on failure + */ +static inline TEE_Result mobj_dec_map(struct mobj *mobj) +{ + if (mobj && mobj->ops) { + if (mobj->ops->dec_map) + return mobj->ops->dec_map(mobj); + return TEE_SUCCESS; + } + return TEE_ERROR_GENERIC; +} + +/** + * mobj_get() - get a MOBJ + * @mobj: Pointer to a MOBJ or NULL + * + * Increases reference counter of the @mobj + * + * Returns @mobj with reference counter increased or NULL if @mobj was NULL + */ +static inline struct mobj *mobj_get(struct mobj *mobj) +{ + if (mobj && !refcount_inc(&mobj->refc)) + panic(); + + return mobj; +} + +/** + * mobj_put() - put a MOBJ + * @mobj: Pointer to a MOBJ or NULL + * + * Decreases reference counter of the @mobj and frees it if the counter + * reaches 0. + */ +static inline void mobj_put(struct mobj *mobj) +{ + if (mobj && refcount_dec(&mobj->refc)) + mobj->ops->free(mobj); +} + +/** + * mobj_put_wipe() - wipe and put a MOBJ + * @mobj: Pointer to a MOBJ or NULL + * + * Clears the memory represented by the mobj and then puts it. + */ +static inline void mobj_put_wipe(struct mobj *mobj) +{ + if (mobj) { + void *buf = mobj_get_va(mobj, 0, mobj->size); + + if (buf) + memzero_explicit(buf, mobj->size); + mobj_put(mobj); + } +} + +static inline uint64_t mobj_get_cookie(struct mobj *mobj) +{ + if (mobj && mobj->ops && mobj->ops->get_cookie) + return mobj->ops->get_cookie(mobj); + +#if defined(CFG_CORE_FFA) + return OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; +#else + return 0; +#endif +} + +static inline struct fobj *mobj_get_fobj(struct mobj *mobj) +{ + if (mobj && mobj->ops && mobj->ops->get_fobj) + return mobj->ops->get_fobj(mobj); + + return NULL; +} + +static inline bool mobj_is_nonsec(struct mobj *mobj) +{ + return mobj_matches(mobj, CORE_MEM_NON_SEC); +} + +static inline bool mobj_is_secure(struct mobj *mobj) +{ + return mobj_matches(mobj, CORE_MEM_SEC); +} + +static inline bool mobj_is_sdp_mem(struct mobj *mobj) +{ + return mobj_matches(mobj, CORE_MEM_SDP_MEM); +} + +static inline size_t mobj_get_phys_granule(struct mobj *mobj) +{ + if (mobj->phys_granule) + return mobj->phys_granule; + return mobj->size; +} + +static inline bool mobj_check_offset_and_len(struct mobj *mobj, size_t offset, + size_t len) +{ + size_t end_offs = 0; + + return len && !ADD_OVERFLOW(offset, len - 1, &end_offs) && + end_offs < mobj->size; +} + +struct mobj *mobj_phys_alloc(paddr_t pa, size_t size, uint32_t cattr, + enum buf_is_attr battr); + +#if defined(CFG_CORE_FFA) +struct mobj *mobj_ffa_get_by_cookie(uint64_t cookie, + unsigned int internal_offs); + +TEE_Result mobj_ffa_unregister_by_cookie(uint64_t cookie); + +/* Functions for SPMC */ +#ifdef CFG_CORE_SEL1_SPMC +struct mobj_ffa *mobj_ffa_sel1_spmc_new(uint64_t cookie, + unsigned int num_pages); +void mobj_ffa_sel1_spmc_delete(struct mobj_ffa *mobj); +TEE_Result mobj_ffa_sel1_spmc_reclaim(uint64_t cookie); +#else +struct mobj_ffa *mobj_ffa_spmc_new(uint64_t cookie, unsigned int num_pages); +void mobj_ffa_spmc_delete(struct mobj_ffa *mobj); +#endif + +uint64_t mobj_ffa_get_cookie(struct mobj_ffa *mobj); +TEE_Result mobj_ffa_add_pages_at(struct mobj_ffa *mobj, unsigned int *idx, + paddr_t pa, unsigned int num_pages); +uint64_t mobj_ffa_push_to_inactive(struct mobj_ffa *mobj); + +#elif defined(CFG_CORE_DYN_SHM) +/* reg_shm represents TEE shared memory */ +struct mobj *mobj_reg_shm_alloc(paddr_t *pages, size_t num_pages, + paddr_t page_offset, uint64_t cookie); + +/** + * mobj_reg_shm_get_by_cookie() - get a MOBJ based on cookie + * @cookie: Cookie used by normal world when suppling the shared memory + * + * Searches for a registered shared memory MOBJ and if one with a matching + * @cookie is found its reference counter is increased before returning + * the MOBJ. + * + * Returns a valid pointer on success or NULL on failure. + */ +struct mobj *mobj_reg_shm_get_by_cookie(uint64_t cookie); + +TEE_Result mobj_reg_shm_release_by_cookie(uint64_t cookie); + +/** + * mobj_reg_shm_unguard() - unguards a reg_shm + * @mobj: pointer to a registered shared memory mobj + * + * A registered shared memory mobj is normally guarded against being + * released with mobj_reg_shm_try_release_by_cookie(). After this function + * has returned the mobj can be released by a call to + * mobj_reg_shm_try_release_by_cookie() if the reference counter allows it. + */ +void mobj_reg_shm_unguard(struct mobj *mobj); + +/* + * mapped_shm represents registered shared buffer + * which is mapped into OPTEE va space + */ +struct mobj *mobj_mapped_shm_alloc(paddr_t *pages, size_t num_pages, + paddr_t page_offset, uint64_t cookie); +#endif /*CFG_CORE_DYN_SHM*/ + +#if !defined(CFG_CORE_DYN_SHM) +static inline struct mobj *mobj_mapped_shm_alloc(paddr_t *pages __unused, + size_t num_pages __unused, + paddr_t page_offset __unused, + uint64_t cookie __unused) +{ + return NULL; +} + +static inline struct mobj *mobj_reg_shm_get_by_cookie(uint64_t cookie __unused) +{ + return NULL; +} +#endif + +struct mobj *mobj_shm_alloc(paddr_t pa, size_t size, uint64_t cookie); + +#ifdef CFG_PAGED_USER_TA +bool mobj_is_paged(struct mobj *mobj); +#else +static inline bool mobj_is_paged(struct mobj *mobj __unused) +{ + return false; +} +#endif + +struct mobj *mobj_with_fobj_alloc(struct fobj *fobj, struct file *file, + uint32_t mem_type); + +#endif /*__MM_MOBJ_H*/ diff --git a/optee_os/core/include/mm/pgt_cache.h b/optee_os/core/include/mm/pgt_cache.h new file mode 100644 index 0000000..13cd9b2 --- /dev/null +++ b/optee_os/core/include/mm/pgt_cache.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ +#ifndef MM_PGT_CACHE_H +#define MM_PGT_CACHE_H + +#ifdef CFG_WITH_LPAE +#define PGT_SIZE (4 * 1024) +#define PGT_NUM_PGT_PER_PAGE 1 +#else +#define PGT_SIZE (1 * 1024) +#define PGT_NUM_PGT_PER_PAGE 4 +#endif + +#include +#include +#include +#include +#include + +struct ts_ctx; + +struct pgt { + void *tbl; + vaddr_t vabase; +#if !defined(CFG_CORE_PREALLOC_EL0_TBLS) + struct ts_ctx *ctx; +#endif + bool populated; +#if defined(CFG_PAGED_USER_TA) + uint16_t num_used_entries; +#endif +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) || \ + (defined(CFG_WITH_PAGER) && !defined(CFG_WITH_LPAE)) + struct pgt_parent *parent; +#endif + SLIST_ENTRY(pgt) link; +}; + +/* + * A proper value for PGT_CACHE_SIZE depends on many factors: CFG_WITH_LPAE, + * CFG_TA_ASLR, size of TA, size of memrefs passed to TA, CFG_ULIBS_SHARED and + * possibly others. The value is based on the number of threads as an indicator + * on how large the system might be. + */ +#if CFG_NUM_THREADS < 2 +#define PGT_CACHE_SIZE 4 +#elif (CFG_NUM_THREADS == 2 && !defined(CFG_WITH_LPAE)) +#define PGT_CACHE_SIZE 8 +#else +#define PGT_CACHE_SIZE ROUNDUP(CFG_NUM_THREADS * 2, PGT_NUM_PGT_PER_PAGE) +#endif + +SLIST_HEAD(pgt_cache, pgt); +struct user_mode_ctx; + +bool pgt_check_avail(struct user_mode_ctx *uctx); + +/* + * pgt_get_all() - makes all needed translation tables available + * @uctx: the context to own the tables + * + * Guaranteed to succeed, but may need to sleep for a while to get all the + * needed translation tables. + */ +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) +static inline void pgt_get_all(struct user_mode_ctx *uctx __unused) { } +#else +void pgt_get_all(struct user_mode_ctx *uctx); +#endif + +/* + * pgt_put_all() - informs the translation table manager that these tables + * will not be needed for a while + * @uctx: the context owning the tables to make inactive + */ +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) +static inline void pgt_put_all(struct user_mode_ctx *uctx __unused) { } +#else +void pgt_put_all(struct user_mode_ctx *uctx); +#endif + +void pgt_clear_range(struct user_mode_ctx *uctx, vaddr_t begin, vaddr_t end); +void pgt_flush_range(struct user_mode_ctx *uctx, vaddr_t begin, vaddr_t last); + +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) +static inline struct pgt *pgt_pop_from_cache_list(vaddr_t vabase __unused, + struct ts_ctx *ctx __unused) +{ return NULL; } +static inline void pgt_push_to_cache_list(struct pgt *pgt __unused) { } +#else +struct pgt *pgt_pop_from_cache_list(vaddr_t vabase, struct ts_ctx *ctx); +void pgt_push_to_cache_list(struct pgt *pgt); +#endif + +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) +static inline void pgt_init(void) { } +#else +void pgt_init(void); +#endif + +void pgt_flush(struct user_mode_ctx *uctx); + +#if defined(CFG_PAGED_USER_TA) +static inline void pgt_inc_used_entries(struct pgt *pgt) +{ + pgt->num_used_entries++; + assert(pgt->num_used_entries); +} + +static inline void pgt_dec_used_entries(struct pgt *pgt) +{ + assert(pgt->num_used_entries); + pgt->num_used_entries--; +} + +static inline void pgt_set_used_entries(struct pgt *pgt, size_t val) +{ + pgt->num_used_entries = val; +} + +#else +static inline void pgt_inc_used_entries(struct pgt *pgt __unused) +{ +} + +static inline void pgt_dec_used_entries(struct pgt *pgt __unused) +{ +} + +static inline void pgt_set_used_entries(struct pgt *pgt __unused, + size_t val __unused) +{ +} + +#endif + +#endif /*MM_PGT_CACHE_H*/ diff --git a/optee_os/core/include/mm/sp_mem.h b/optee_os/core/include/mm/sp_mem.h new file mode 100644 index 0000000..957db36 --- /dev/null +++ b/optee_os/core/include/mm/sp_mem.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021-2022, Arm Limited. + */ + +#ifndef __MM_SP_MEM_H +#define __MM_SP_MEM_H + +#include +#include +#include + +struct sp_mem; + +/* + * struct sp_mem_receiver - Memory object instance for a FF-A receiver endpoint + * + * @perm: Receiver permission on the shared memory + * @ref_count: Count retrieve requests from endpoint + * @smem: Shared memory reference + * @link: Link in related list + */ +struct sp_mem_receiver { + struct ffa_mem_access_perm perm; + uint8_t ref_count; + struct sp_mem *smem; + + SLIST_ENTRY(sp_mem_receiver) link; +}; + +/* + * sp_mem_map_region represents the memory address when using FF-A shares. + * Instead of storing the physical addresses and the size of the region, we use + * the mobj's which were already used by the SPs. The offset is used to point + * to the specific location inside the mobj memory range. + */ +struct sp_mem_map_region { + struct mobj *mobj; + /* + * Offset (in pages) inside the mobj which is used to retrieve the + * location. + */ + uint32_t page_offset; + uint32_t page_count; + + SLIST_ENTRY(sp_mem_map_region) link; +}; + +SLIST_HEAD(sp_mem_receiver_head, sp_mem_receiver); +SLIST_HEAD(sp_mem_regions_head, sp_mem_map_region); + +/* + * sp_mem is used as the main place to store the FF-A shares information. + * For each FFA_SHARE message a new sp_mem object is created. + * The receivers field is used to store all receiver specific information. + * The regions field is used to store all data needed for retrieving the + * shared addresses. + */ +struct sp_mem { + struct sp_mem_regions_head regions; + struct sp_mem_receiver_head receivers; + /* Data which was passed inside struct ffa_mem_transaction */ + uint16_t sender_id; + uint8_t mem_reg_attr; + uint32_t flags; + uint64_t global_handle; + uint64_t tag; + + SLIST_ENTRY(sp_mem) link; +}; + +struct sp_mem *sp_mem_new(void); +struct sp_mem_receiver *sp_mem_get_receiver(uint32_t s_id, struct sp_mem *smem); +struct sp_mem *sp_mem_get(uint64_t handle); + +bool sp_mem_is_shared(struct sp_mem_map_region *new_reg); +void *sp_mem_get_va(const struct user_mode_ctx *uctx, size_t offset, + struct mobj *mobj); +void sp_mem_remove(struct sp_mem *s_mem); +void sp_mem_add(struct sp_mem *smem); +struct mobj *sp_mem_new_mobj(uint64_t pages, uint32_t mem_type, bool is_secure); +int sp_mem_add_pages(struct mobj *mobj, unsigned int *idx, + paddr_t pa, unsigned int num_pages); +#endif /*__MM_SP_MEM_H*/ diff --git a/optee_os/core/include/mm/tee_mm.h b/optee_os/core/include/mm/tee_mm.h new file mode 100644 index 0000000..481f583 --- /dev/null +++ b/optee_os/core/include/mm/tee_mm.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_MM_H +#define TEE_MM_H + +#include +#include + +/* Define to indicate default pool initiation */ +#define TEE_MM_POOL_NO_FLAGS 0 +/* Flag to indicate that memory is allocated from hi address to low address */ +#define TEE_MM_POOL_HI_ALLOC (1u << 0) +/* Flag to indicate that pool should use nex_malloc instead of malloc */ +#define TEE_MM_POOL_NEX_MALLOC (1u << 1) + +struct _tee_mm_entry_t { + struct _tee_mm_pool_t *pool; + struct _tee_mm_entry_t *next; + uint32_t offset; /* offset in pages/sections */ + uint32_t size; /* size in pages/sections */ +}; +typedef struct _tee_mm_entry_t tee_mm_entry_t; + +struct _tee_mm_pool_t { + tee_mm_entry_t *entry; + paddr_t lo; /* low boundary of the pool */ + paddr_size_t size; /* pool size */ + uint32_t flags; /* Config flags for the pool */ + uint8_t shift; /* size shift */ + unsigned int lock; +#ifdef CFG_WITH_STATS + size_t max_allocated; +#endif +}; +typedef struct _tee_mm_pool_t tee_mm_pool_t; + +/* Physical Secure DDR pool */ +extern tee_mm_pool_t tee_mm_sec_ddr; + +/* Virtual eSRAM pool */ +extern tee_mm_pool_t tee_mm_vcore; + +/* Shared memory pool */ +extern tee_mm_pool_t tee_mm_shm; + +/* + * Returns a pointer to the mm covering the supplied address, + * if no mm is found NULL is returned. + */ +tee_mm_entry_t *tee_mm_find(const tee_mm_pool_t *pool, paddr_t addr); + +/* + * Validates that an address (addr) is part of the secure virtual memory + * Returns false if not valid, true if valid + * NOTE: This function is executed in abort mode. + * Please take care of stack usage + */ +static inline bool tee_mm_validate(const tee_mm_pool_t *pool, paddr_t addr) +{ + return tee_mm_find(pool, addr) != 0; +} + +/* + * Returns a virtual address to the start of the allocated memory + * for the mm entry. + */ +uintptr_t tee_mm_get_smem(const tee_mm_entry_t *mm); + +/* Init managed memory area */ +bool tee_mm_init(tee_mm_pool_t *pool, paddr_t lo, paddr_size_t size, + uint8_t shift, uint32_t flags); + +/* Kill managed memory area*/ +void tee_mm_final(tee_mm_pool_t *pool); + +/* + * Allocates size number of bytes in the paged virtual address space + * Returns a handle to the memory. The handle is used as an input to + * the tee_mm_free function. + */ +tee_mm_entry_t *tee_mm_alloc(tee_mm_pool_t *pool, size_t size); + +/* Allocate supplied memory range if it's free */ +tee_mm_entry_t *tee_mm_alloc2(tee_mm_pool_t *pool, paddr_t base, size_t size); + +/* + * Frees the entry in the paged virtual address space pointed to by the + * input parameter p + */ +void tee_mm_free(tee_mm_entry_t *p); + +/* Returns size in sections or pages */ +static inline size_t tee_mm_get_size(tee_mm_entry_t *p) +{ + return p->size; +} + +/* Returns offset from start of area in sections or pages */ +static inline uint32_t tee_mm_get_offset(tee_mm_entry_t *p) +{ + return p->offset; +} + +/* Return size of the mm entry in bytes */ +size_t tee_mm_get_bytes(const tee_mm_entry_t *mm); + +bool tee_mm_addr_is_within_range(const tee_mm_pool_t *pool, paddr_t addr); + +bool tee_mm_is_empty(tee_mm_pool_t *pool); + +#ifdef CFG_WITH_STATS +void tee_mm_get_pool_stats(tee_mm_pool_t *pool, struct malloc_stats *stats, + bool reset); +#endif + +#endif diff --git a/optee_os/core/include/mm/tee_mmu_types.h b/optee_os/core/include/mm/tee_mmu_types.h new file mode 100644 index 0000000..e643d27 --- /dev/null +++ b/optee_os/core/include/mm/tee_mmu_types.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2021, Linaro Limited + * Copyright (c) 2022, Arm Limited. + */ +#ifndef TEE_MMU_TYPES_H +#define TEE_MMU_TYPES_H + +#include +#include +#include + +#define TEE_MATTR_VALID_BLOCK BIT(0) +#define TEE_MATTR_TABLE BIT(3) +#define TEE_MATTR_PR BIT(4) +#define TEE_MATTR_PW BIT(5) +#define TEE_MATTR_PX BIT(6) +#define TEE_MATTR_PRW (TEE_MATTR_PR | TEE_MATTR_PW) +#define TEE_MATTR_PRX (TEE_MATTR_PR | TEE_MATTR_PX) +#define TEE_MATTR_PRWX (TEE_MATTR_PRW | TEE_MATTR_PX) +#define TEE_MATTR_UR BIT(7) +#define TEE_MATTR_UW BIT(8) +#define TEE_MATTR_UX BIT(9) +#define TEE_MATTR_URW (TEE_MATTR_UR | TEE_MATTR_UW) +#define TEE_MATTR_URX (TEE_MATTR_UR | TEE_MATTR_UX) +#define TEE_MATTR_URWX (TEE_MATTR_URW | TEE_MATTR_UX) +#define TEE_MATTR_PROT_MASK \ + (TEE_MATTR_PRWX | TEE_MATTR_URWX | TEE_MATTR_GUARDED) + +#define TEE_MATTR_GLOBAL BIT(10) +#define TEE_MATTR_SECURE BIT(11) + +#define TEE_MATTR_MEM_TYPE_MASK U(0x7) +#define TEE_MATTR_MEM_TYPE_SHIFT U(12) +/* These are shifted TEE_MATTR_MEM_TYPE_SHIFT */ + +/* + * Device-nGnRnE most restrictive (equivalent to Strongly Ordered memory + * in the ARMv7 architecture). + * https://developer.arm.com/documentation/den0024/a/Memory-Ordering/Memory-types/Device-memory + * + * If an ARMv7 architecture operating system runs on a Cortex-A53 processor, + * the Device memory type matches the nGnRE encoding and the Strongly-Ordered + * memory type matches the nGnRnE memory type. + * https://developer.arm.com/documentation/den0024/a/Memory-Ordering/Memory-types/Device-memory + */ +#define TEE_MATTR_MEM_TYPE_DEV U(0) /* Device-nGnRE */ +#define TEE_MATTR_MEM_TYPE_CACHED U(1) +#define TEE_MATTR_MEM_TYPE_STRONGLY_O U(2) /* Device-nGnRnE */ +#define TEE_MATTR_MEM_TYPE_TAGGED U(3) + +#define TEE_MATTR_GUARDED BIT(15) + +/* + * Tags TA mappings which are only used during a single call (open session + * or invoke command parameters). + */ +#define VM_FLAG_EPHEMERAL BIT(0) +/* + * Tags TA mappings that must not be removed (kernel mappings while in user + * mode). + */ +#define VM_FLAG_PERMANENT BIT(1) +/* Tags TA mappings that may be shared with other TAs. */ +#define VM_FLAG_SHAREABLE BIT(2) +/* Tags temporary mappings added to load the ldelf binary */ +#define VM_FLAG_LDELF BIT(3) +/* + * The mapping should only be mapped read-only, not enforced by the vm_* + * functions. + */ +#define VM_FLAG_READONLY BIT(4) + +/* + * Set of flags used by tee_mmu_is_vbuf_inside_ta_private() and + * tee_mmu_is_vbuf_intersect_ta_private() to tell if a certain region is + * mapping TA internal memory or not. + */ +#define VM_FLAGS_NONPRIV (VM_FLAG_EPHEMERAL | \ + VM_FLAG_PERMANENT | \ + VM_FLAG_SHAREABLE) + +struct tee_mmap_region { + unsigned int type; /* enum teecore_memtypes */ + unsigned int region_size; + paddr_t pa; + vaddr_t va; + size_t size; + uint32_t attr; /* TEE_MATTR_* above */ +}; + +struct vm_region { + struct mobj *mobj; + size_t offset; + vaddr_t va; + size_t size; + uint16_t attr; /* TEE_MATTR_* above */ + uint16_t flags; /* VM_FLAGS_* above */ + TAILQ_ENTRY(vm_region) link; +}; + +enum vm_paged_region_type { + PAGED_REGION_TYPE_RO, + PAGED_REGION_TYPE_RW, + PAGED_REGION_TYPE_LOCK, +}; + +struct vm_paged_region { + struct fobj *fobj; + size_t fobj_pgoffs; + enum vm_paged_region_type type; + uint32_t flags; + vaddr_t base; + size_t size; + struct pgt **pgt_array; + TAILQ_ENTRY(vm_paged_region) link; + TAILQ_ENTRY(vm_paged_region) fobj_link; +}; + +TAILQ_HEAD(vm_paged_region_head, vm_paged_region); +TAILQ_HEAD(vm_region_head, vm_region); + +struct vm_info { + struct vm_region_head regions; + unsigned int asid; +}; + +static inline void mattr_perm_to_str(char *str, size_t size, uint32_t attr) +{ + if (size < 7) + return; + + str[0] = (attr & TEE_MATTR_UR) ? 'r' : '-'; + str[1] = (attr & TEE_MATTR_UW) ? 'w' : '-'; + str[2] = (attr & TEE_MATTR_UX) ? 'x' : '-'; + str[3] = (attr & TEE_MATTR_PR) ? 'R' : '-'; + str[4] = (attr & TEE_MATTR_PW) ? 'W' : '-'; + str[5] = (attr & TEE_MATTR_PX) ? 'X' : '-'; + str[6] = '\0'; +} + +static inline bool mattr_is_cached(uint32_t mattr) +{ + uint32_t mem_type = (mattr >> TEE_MATTR_MEM_TYPE_SHIFT) & + TEE_MATTR_MEM_TYPE_MASK; + + return mem_type == TEE_MATTR_MEM_TYPE_CACHED || + mem_type == TEE_MATTR_MEM_TYPE_TAGGED; +} +#endif diff --git a/optee_os/core/include/mm/tee_pager.h b/optee_os/core/include/mm/tee_pager.h new file mode 100644 index 0000000..4e42e64 --- /dev/null +++ b/optee_os/core/include/mm/tee_pager.h @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2021, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef MM_TEE_PAGER_H +#define MM_TEE_PAGER_H + +#include +#include +#include +#include +#include +#include +#include + +/* + * tee_pager_early_init() - Perform early initialization of pager + * + * Panics if some error occurs + */ +void tee_pager_early_init(void); + +/* + * tee_pager_get_table_info() - Fills in table info for address mapped in + * translation table managed by the pager. + * @va: address to look up + * @ti: filled in table info + * + * Returns true if address is in the pager translation tables else false + */ +bool tee_pager_get_table_info(vaddr_t va, struct core_mmu_table_info *ti); + +/* + * tee_pager_phys_to_virt() - Translate physical address to virtual address + * looking in the pager page tables + * @pa: address to translate + * @len: check for length is mapped linearly in CORE_MMU_PGDIR_SIZE range + * + * Returns found virtual address or NULL on error + */ +void *tee_pager_phys_to_virt(paddr_t pa, size_t len); + +/* + * tee_pager_set_alias_area() - Initialize pager alias area + * @mm_alias: The alias area where all physical pages managed by the + * pager are aliased + * + * Panics if called twice or some other error occurs. + */ +void tee_pager_set_alias_area(tee_mm_entry_t *mm_alias); + +/* + * tee_pager_init_iv_region() - Initialized pager region for tags IVs used by RW + * paged fobjs + * @fobj: fobj backing the region + * + * Panics if called twice or some other error occurs. + * + * Returns virtual address of start of IV region. + */ +vaddr_t tee_pager_init_iv_region(struct fobj *fobj); + +/* + * tee_pager_generate_authenc_key() - Generates authenc key for r/w paging + * + * Needs to draw random from RNG, panics if some error occurs. + */ +#ifdef CFG_WITH_PAGER +void tee_pager_generate_authenc_key(void); +#else +static inline void tee_pager_generate_authenc_key(void) +{ +} +#endif + +/* + * tee_pager_add_core_region() - Adds a pageable core region + * @base: base of covered memory region + * @type: type of memory region + * @fobj: fobj backing the region + * + * Non-page aligned base or size will cause a panic. + */ +void tee_pager_add_core_region(vaddr_t base, enum vm_paged_region_type type, + struct fobj *fobj); + +/* + * tee_pager_add_um_region() - Adds a pageable user TA region + * @uctx: user mode context of the region + * @base: base of covered memory region + * @fobj: fobj of the store backing the memory region + * + * The mapping is created suitable to initialize the memory content while + * loading the TA. Once the TA is properly loaded the regions should be + * finalized with tee_pager_set_um_region_attr() to get more strict settings. + * + * Return TEE_SUCCESS on success, anything else if the region can't be added + */ +#ifdef CFG_PAGED_USER_TA +TEE_Result tee_pager_add_um_region(struct user_mode_ctx *uctx, vaddr_t base, + struct fobj *fobj, uint32_t prot); +#else +static inline TEE_Result +tee_pager_add_um_region(struct user_mode_ctx *uctx __unused, + vaddr_t base __unused, struct fobj *fobj __unused, + uint32_t prot __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +/* + * tee_pager_set_um_region_attr() - Set attributes of a initialized memory + * region + * @uctx: user mode context of the region + * @base: base of covered memory region + * @size: size of covered memory region + * @flags: TEE_MATTR_U* flags describing permissions of the region + * + * Return true on success of false if the region can't be updated + */ +#ifdef CFG_PAGED_USER_TA +bool tee_pager_set_um_region_attr(struct user_mode_ctx *uctx, vaddr_t base, + size_t size, uint32_t flags); +#else +static inline bool +tee_pager_set_um_region_attr(struct user_mode_ctx *uctx __unused, + vaddr_t base __unused, size_t size __unused, + uint32_t flags __unused) +{ + return false; +} +#endif + +#ifdef CFG_PAGED_USER_TA +void tee_pager_rem_um_region(struct user_mode_ctx *uctx, vaddr_t base, + size_t size); +#else +static inline void tee_pager_rem_um_region(struct user_mode_ctx *uctx __unused, + vaddr_t base __unused, + size_t size __unused) +{ +} +#endif + +#ifdef CFG_PAGED_USER_TA +TEE_Result tee_pager_split_um_region(struct user_mode_ctx *uctx, vaddr_t va); +#else +static inline TEE_Result +tee_pager_split_um_region(struct user_mode_ctx *uctx __unused, + vaddr_t va __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +#endif +#ifdef CFG_PAGED_USER_TA +void tee_pager_merge_um_region(struct user_mode_ctx *uctx, vaddr_t va, + size_t len); +#else +static inline void +tee_pager_merge_um_region(struct user_mode_ctx *uctx __unused, + vaddr_t va __unused, size_t len __unused) +{ +} +#endif + +/* + * tee_pager_rem_uma_regions() - Remove all user TA regions + * @uctx: user mode context + * + * This function is called when a user mode context is teared down. + */ +#ifdef CFG_PAGED_USER_TA +void tee_pager_rem_um_regions(struct user_mode_ctx *uctx); +#else +static inline void tee_pager_rem_um_regions(struct user_mode_ctx *uctx __unused) +{ +} +#endif + +/* + * tee_pager_assign_um_tables() - Assigns translation table to a user ta + * @uctx: user mode context + * + * This function is called to assign translation tables for the pageable + * regions of a user TA. + */ +#ifdef CFG_PAGED_USER_TA +void tee_pager_assign_um_tables(struct user_mode_ctx *uctx); +#else +static inline void +tee_pager_assign_um_tables(struct user_mode_ctx *uctx __unused) +{ +} +#endif + +/* + * Adds physical pages to the pager to use. The supplied virtual address range + * is searched for mapped physical pages and unmapped pages are ignored. + * + * vaddr is the first virtual address + * npages is the number of pages to add + */ +void tee_pager_add_pages(vaddr_t vaddr, size_t npages, bool unmap); + +/* + * tee_pager_alloc() - Allocate read-write virtual memory from pager. + * @size: size of memory in bytes + * + * @return NULL on failure or a pointer to the virtual memory on success. + */ +void *tee_pager_alloc(size_t size); + +#ifdef CFG_PAGED_USER_TA +/* + * tee_pager_pgt_save_and_release_entries() - Save dirty pages to backing store + * and remove physical page from translation table + * @pgt: page table descriptor + * + * This function is called when a translation table needs to be recycled + */ +void tee_pager_pgt_save_and_release_entries(struct pgt *pgt); +#else +static inline void +tee_pager_pgt_save_and_release_entries(struct pgt *pgt __unused) +{ +} +#endif + +/* + * tee_pager_release_phys() - Release physical pages used for mapping + * @addr: virtual address of first page to release + * @size: number of bytes to release + * + * Only pages completely covered by the supplied range are affected. This + * function only supplies a hint to the pager that the physical page can be + * reused. The caller can't expect a released memory range to hold a + * specific bit pattern when used next time. + * + * Note that the virtual memory allocation is still valid after this + * function has returned, it's just the content that may or may not have + * changed. + */ +#ifdef CFG_WITH_PAGER +void tee_pager_release_phys(void *addr, size_t size); +#else +static inline void tee_pager_release_phys(void *addr __unused, + size_t size __unused) +{ +} +#endif + +/* + * Statistics on the pager + */ +struct tee_pager_stats { + size_t hidden_hits; + size_t ro_hits; + size_t rw_hits; + size_t zi_released; + size_t npages; /* number of load pages */ + size_t npages_all; /* number of pages */ +}; + +#ifdef CFG_WITH_PAGER +void tee_pager_get_stats(struct tee_pager_stats *stats); +bool tee_pager_handle_fault(struct abort_info *ai); +#else /*CFG_WITH_PAGER*/ +static inline bool tee_pager_handle_fault(struct abort_info *ai __unused) +{ + return false; +} + +static inline void tee_pager_get_stats(struct tee_pager_stats *stats) +{ + memset(stats, 0, sizeof(struct tee_pager_stats)); +} +#endif /*CFG_WITH_PAGER*/ + +void tee_pager_invalidate_fobj(struct fobj *fobj); + +#endif /*MM_TEE_PAGER_H*/ diff --git a/optee_os/core/include/mm/vm.h b/optee_os/core/include/mm/vm.h new file mode 100644 index 0000000..f1509ba --- /dev/null +++ b/optee_os/core/include/mm/vm.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TEE_MMU_H +#define TEE_MMU_H + +#include +#include +#include + +/* Allocate context resources like ASID and MMU table information */ +TEE_Result vm_info_init(struct user_mode_ctx *uctx, struct ts_ctx *ts_ctx); + +/* Release context resources like ASID */ +void vm_info_final(struct user_mode_ctx *uctx); + +/* + * Creates a memory map of a mobj. + * Desired virtual address can be specified in @va otherwise @va must be + * initialized to 0 if the next available can be chosen. + * + * @pad_begin and @pad_end specify how much extra free space should be kept + * when establishing the map. This allows mapping the first part of for + * instance an ELF file while knowing that the next part which has to be of + * a certain offset from the first part also will succeed. + */ + +TEE_Result vm_map_pad(struct user_mode_ctx *uctx, vaddr_t *va, size_t len, + uint32_t prot, uint32_t flags, struct mobj *mobj, + size_t offs, size_t pad_begin, size_t pad_end, + size_t align); + +/* + * Creates a memory map of a mobj. + * Desired virtual address can be specified in @va otherwise @va must be + * initialized to 0 if the next available can be chosen. + */ +static inline TEE_Result vm_map(struct user_mode_ctx *uctx, vaddr_t *va, + size_t len, uint32_t prot, uint32_t flags, + struct mobj *mobj, size_t offs) +{ + return vm_map_pad(uctx, va, len, prot, flags, mobj, offs, 0, 0, 0); +} + +TEE_Result vm_remap(struct user_mode_ctx *uctx, vaddr_t *new_va, vaddr_t old_va, + size_t len, size_t pad_begin, size_t pad_end); + +TEE_Result vm_get_flags(struct user_mode_ctx *uctx, vaddr_t va, size_t len, + uint32_t *flags); + +TEE_Result vm_get_prot(struct user_mode_ctx *uctx, vaddr_t va, size_t len, + uint16_t *prot); + +TEE_Result vm_set_prot(struct user_mode_ctx *uctx, vaddr_t va, size_t len, + uint32_t prot); + +TEE_Result vm_unmap(struct user_mode_ctx *uctx, vaddr_t va, size_t len); + +/* Map parameters for a user TA */ +TEE_Result vm_map_param(struct user_mode_ctx *uctx, struct tee_ta_param *param, + void *param_va[TEE_NUM_PARAMS]); +void vm_clean_param(struct user_mode_ctx *uctx); + +/* + * User mode private memory is defined as user mode image static segment + * (code, ro/rw static data, heap, stack). The sole other virtual memory + * mapped to user mode are memref parameters. These later are considered + * outside user mode private memory as it might be accessed by the user + * mode context and its client(s). + */ +bool vm_buf_is_inside_um_private(const struct user_mode_ctx *uctx, + const void *va, size_t size); + +bool vm_buf_intersects_um_private(const struct user_mode_ctx *uctx, + const void *va, size_t size); + +TEE_Result vm_buf_to_mboj_offs(const struct user_mode_ctx *uctx, + const void *va, size_t size, + struct mobj **mobj, size_t *offs); + +/* Helper function for virt_to_phys(), shouldn't be used directly elsewhere */ +TEE_Result vm_va2pa(const struct user_mode_ctx *uctx, void *ua, paddr_t *pa); + +/* Helper function for phys_to_virt(), shouldn't be used directly elsewhere */ +void *vm_pa2va(const struct user_mode_ctx *uctx, paddr_t pa, size_t pa_size); + +/* + * Return TEE_SUCCESS or TEE_ERROR_ACCESS_DENIED when buffer exists or return + * another TEE_Result code. + */ +TEE_Result vm_check_access_rights(const struct user_mode_ctx *uctx, + uint32_t flags, uaddr_t uaddr, size_t len); + +/* Set user context @ctx or core privileged context if @ctx is NULL */ +void vm_set_ctx(struct ts_ctx *ctx); + +struct mobj *vm_get_mobj(struct user_mode_ctx *uctx, vaddr_t va, size_t *len, + uint16_t *prot, size_t *offs); +#endif /*TEE_MMU_H*/ diff --git a/optee_os/core/include/optee_msg.h b/optee_os/core/include/optee_msg.h new file mode 100644 index 0000000..7cc795d --- /dev/null +++ b/optee_os/core/include/optee_msg.h @@ -0,0 +1,345 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2020, Linaro Limited + */ +#ifndef _OPTEE_MSG_H +#define _OPTEE_MSG_H + +#include +#include +#include + +/* + * This file defines the OP-TEE message protocol used to communicate + * with an instance of OP-TEE running in secure world. + */ + +/***************************************************************************** + * Part 1 - formatting of messages + *****************************************************************************/ + +#define OPTEE_MSG_ATTR_TYPE_NONE U(0x0) +#define OPTEE_MSG_ATTR_TYPE_VALUE_INPUT U(0x1) +#define OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT U(0x2) +#define OPTEE_MSG_ATTR_TYPE_VALUE_INOUT U(0x3) +#define OPTEE_MSG_ATTR_TYPE_RMEM_INPUT U(0x5) +#define OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT U(0x6) +#define OPTEE_MSG_ATTR_TYPE_RMEM_INOUT U(0x7) +#define OPTEE_MSG_ATTR_TYPE_FMEM_INPUT OPTEE_MSG_ATTR_TYPE_RMEM_INPUT +#define OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT +#define OPTEE_MSG_ATTR_TYPE_FMEM_INOUT OPTEE_MSG_ATTR_TYPE_RMEM_INOUT +#define OPTEE_MSG_ATTR_TYPE_TMEM_INPUT U(0x9) +#define OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT U(0xa) +#define OPTEE_MSG_ATTR_TYPE_TMEM_INOUT U(0xb) + +#define OPTEE_MSG_ATTR_TYPE_MASK GENMASK_32(7, 0) + +/* + * Meta parameter to be absorbed by the Secure OS and not passed + * to the Trusted Application. + * + * Currently only used with OPTEE_MSG_CMD_OPEN_SESSION. + */ +#define OPTEE_MSG_ATTR_META BIT(8) + +/* + * Pointer to a list of pages used to register user-defined SHM buffer. + * Used with OPTEE_MSG_ATTR_TYPE_TMEM_*. + * buf_ptr should point to the beginning of the buffer. Buffer will contain + * list of page addresses. OP-TEE core can reconstruct contiguous buffer from + * that page addresses list. Page addresses are stored as 64 bit values. + * Last entry on a page should point to the next page of buffer. + * Every entry in buffer should point to a 4k page beginning (12 least + * significant bits must be equal to zero). + * + * 12 least significant bits of optee_msg_param.u.tmem.buf_ptr should hold + * page offset of user buffer. + * + * So, entries should be placed like members of this structure: + * + * struct page_data { + * uint64_t pages_array[OPTEE_MSG_NONCONTIG_PAGE_SIZE/sizeof(uint64_t) - 1]; + * uint64_t next_page_data; + * }; + * + * Structure is designed to exactly fit into the page size + * OPTEE_MSG_NONCONTIG_PAGE_SIZE which is a standard 4KB page. + * + * The size of 4KB is chosen because this is the smallest page size for ARM + * architectures. If REE uses larger pages, it should divide them to 4KB ones. + */ +#define OPTEE_MSG_ATTR_NONCONTIG BIT(9) + +/* + * Memory attributes for caching passed with temp memrefs. The actual value + * used is defined outside the message protocol with the exception of + * OPTEE_MSG_ATTR_CACHE_PREDEFINED which means the attributes already + * defined for the memory range should be used. If optee_smc.h is used as + * bearer of this protocol OPTEE_SMC_SHM_* is used for values. + */ +#define OPTEE_MSG_ATTR_CACHE_SHIFT U(16) +#define OPTEE_MSG_ATTR_CACHE_MASK GENMASK_32(2, 0) +#define OPTEE_MSG_ATTR_CACHE_PREDEFINED U(0) + +/* + * Same values as TEE_LOGIN_* from TEE Internal API + */ +#define OPTEE_MSG_LOGIN_PUBLIC U(0x00000000) +#define OPTEE_MSG_LOGIN_USER U(0x00000001) +#define OPTEE_MSG_LOGIN_GROUP U(0x00000002) +#define OPTEE_MSG_LOGIN_APPLICATION U(0x00000004) +#define OPTEE_MSG_LOGIN_APPLICATION_USER U(0x00000005) +#define OPTEE_MSG_LOGIN_APPLICATION_GROUP U(0x00000006) + +/* + * Page size used in non-contiguous buffer entries + */ +#define OPTEE_MSG_NONCONTIG_PAGE_SIZE U(4096) + +#define OPTEE_MSG_FMEM_INVALID_GLOBAL_ID 0xffffffffffffffff + +#ifndef __ASSEMBLER__ +/** + * struct optee_msg_param_tmem - temporary memory reference parameter + * @buf_ptr: Address of the buffer + * @size: Size of the buffer + * @shm_ref: Temporary shared memory reference, pointer to a struct tee_shm + * + * Secure and normal world communicates pointers as physical address + * instead of the virtual address. This is because secure and normal world + * have completely independent memory mapping. Normal world can even have a + * hypervisor which need to translate the guest physical address (AKA IPA + * in ARM documentation) to a real physical address before passing the + * structure to secure world. + */ +struct optee_msg_param_tmem { + uint64_t buf_ptr; + uint64_t size; + uint64_t shm_ref; +}; + +/** + * struct optee_msg_param_rmem - registered memory reference parameter + * @offs: Offset into shared memory reference + * @size: Size of the buffer + * @shm_ref: Shared memory reference, pointer to a struct tee_shm + */ +struct optee_msg_param_rmem { + uint64_t offs; + uint64_t size; + uint64_t shm_ref; +}; + +/** + * struct optee_msg_param_fmem - FF-A memory reference parameter + * @offs_lower: Lower bits of offset into shared memory reference + * @offs_upper: Upper bits of offset into shared memory reference + * @internal_offs: Internal offset into the first page of shared memory + * reference + * @size: Size of the buffer + * @global_id: Global identifier of the shared memory + */ +struct optee_msg_param_fmem { + uint32_t offs_low; + uint16_t offs_high; + uint16_t internal_offs; + uint64_t size; + uint64_t global_id; +}; + +/** + * struct optee_msg_param_value - opaque value parameter + * + * Value parameters are passed unchecked between normal and secure world. + */ +struct optee_msg_param_value { + uint64_t a; + uint64_t b; + uint64_t c; +}; + +/** + * struct optee_msg_param - parameter used together with struct optee_msg_arg + * @attr: attributes + * @tmem: parameter by temporary memory reference + * @rmem: parameter by registered memory reference + * @fmem: parameter by FF-A registered memory reference + * @value: parameter by opaque value + * + * @attr & OPTEE_MSG_ATTR_TYPE_MASK indicates if tmem, rmem or value is used in + * the union. OPTEE_MSG_ATTR_TYPE_VALUE_* indicates value, + * OPTEE_MSG_ATTR_TYPE_TMEM_* indicates @tmem and + * OPTEE_MSG_ATTR_TYPE_RMEM_* or the alias PTEE_MSG_ATTR_TYPE_FMEM_* indicates + * @rmem or @fmem depending on the conduit. + * OPTEE_MSG_ATTR_TYPE_NONE indicates that none of the members are used. + */ +struct optee_msg_param { + uint64_t attr; + union { + struct optee_msg_param_tmem tmem; + struct optee_msg_param_rmem rmem; + struct optee_msg_param_fmem fmem; + struct optee_msg_param_value value; + } u; +}; + +/** + * struct optee_msg_arg - call argument + * @cmd: Command, one of OPTEE_MSG_CMD_* or OPTEE_MSG_RPC_CMD_* + * @func: Trusted Application function, specific to the Trusted Application, + * used if cmd == OPTEE_MSG_CMD_INVOKE_COMMAND + * @session: In parameter for all OPTEE_MSG_CMD_* except + * OPTEE_MSG_CMD_OPEN_SESSION where it's an output parameter instead + * @cancel_id: Cancellation id, a unique value to identify this request + * @ret: return value + * @ret_origin: origin of the return value + * @num_params: number of parameters supplied to the OS Command + * @params: the parameters supplied to the OS Command + * + * All normal calls to Trusted OS uses this struct. If cmd requires further + * information than what these fields hold it can be passed as a parameter + * tagged as meta (setting the OPTEE_MSG_ATTR_META bit in corresponding + * attrs field). All parameters tagged as meta have to come first. + */ +struct optee_msg_arg { + uint32_t cmd; + uint32_t func; + uint32_t session; + uint32_t cancel_id; + uint32_t pad; + uint32_t ret; + uint32_t ret_origin; + uint32_t num_params; + + /* num_params tells the actual number of element in params */ + struct optee_msg_param params[]; +}; + +/** + * OPTEE_MSG_GET_ARG_SIZE - return size of struct optee_msg_arg + * + * @num_params: Number of parameters embedded in the struct optee_msg_arg + * + * Returns the size of the struct optee_msg_arg together with the number + * of embedded parameters. + */ +#define OPTEE_MSG_GET_ARG_SIZE(num_params) \ + (sizeof(struct optee_msg_arg) + \ + sizeof(struct optee_msg_param) * (num_params)) + +/* + * Defines the maximum value of @num_params that can be passed to + * OPTEE_MSG_GET_ARG_SIZE without a risk of crossing page boundary. + */ +#define OPTEE_MSG_MAX_NUM_PARAMS \ + ((OPTEE_MSG_NONCONTIG_PAGE_SIZE - sizeof(struct optee_msg_arg)) / \ + sizeof(struct optee_msg_param)) + +#endif /*__ASSEMBLER__*/ + +/***************************************************************************** + * Part 2 - requests from normal world + *****************************************************************************/ + +/* + * Return the following UID if using API specified in this file without + * further extensions: + * 384fb3e0-e7f8-11e3-af63-0002a5d5c51b. + * Represented in 4 32-bit words in OPTEE_MSG_UID_0, OPTEE_MSG_UID_1, + * OPTEE_MSG_UID_2, OPTEE_MSG_UID_3. + */ +#define OPTEE_MSG_UID_0 U(0x384fb3e0) +#define OPTEE_MSG_UID_1 U(0xe7f811e3) +#define OPTEE_MSG_UID_2 U(0xaf630002) +#define OPTEE_MSG_UID_3 U(0xa5d5c51b) +#define OPTEE_MSG_FUNCID_CALLS_UID U(0xFF01) + +/* + * Returns 2.0 if using API specified in this file without further + * extensions. Represented in 2 32-bit words in OPTEE_MSG_REVISION_MAJOR + * and OPTEE_MSG_REVISION_MINOR + */ +#define OPTEE_MSG_REVISION_MAJOR U(2) +#define OPTEE_MSG_REVISION_MINOR U(0) +#define OPTEE_MSG_FUNCID_CALLS_REVISION U(0xFF03) + +/* + * Get UUID of Trusted OS. + * + * Used by non-secure world to figure out which Trusted OS is installed. + * Note that returned UUID is the UUID of the Trusted OS, not of the API. + * + * Returns UUID in 4 32-bit words in the same way as + * OPTEE_MSG_FUNCID_CALLS_UID described above. + */ +#define OPTEE_MSG_OS_OPTEE_UUID_0 U(0x486178e0) +#define OPTEE_MSG_OS_OPTEE_UUID_1 U(0xe7f811e3) +#define OPTEE_MSG_OS_OPTEE_UUID_2 U(0xbc5e0002) +#define OPTEE_MSG_OS_OPTEE_UUID_3 U(0xa5d5c51b) +#define OPTEE_MSG_FUNCID_GET_OS_UUID U(0x0000) + +/* + * Get revision of Trusted OS. + * + * Used by non-secure world to figure out which version of the Trusted OS + * is installed. Note that the returned revision is the revision of the + * Trusted OS, not of the API. + * + * Returns revision in 2 32-bit words in the same way as + * OPTEE_MSG_CALLS_REVISION described above. + */ +#define OPTEE_MSG_FUNCID_GET_OS_REVISION U(0x0001) + +/* + * Do a secure call with struct optee_msg_arg as argument + * The OPTEE_MSG_CMD_* below defines what goes in struct optee_msg_arg::cmd + * + * OPTEE_MSG_CMD_OPEN_SESSION opens a session to a Trusted Application. + * The first two parameters are tagged as meta, holding two value + * parameters to pass the following information: + * param[0].u.value.a-b uuid of Trusted Application + * param[1].u.value.a-b uuid of Client + * param[1].u.value.c Login class of client OPTEE_MSG_LOGIN_* + * + * OPTEE_MSG_CMD_INVOKE_COMMAND invokes a command a previously opened + * session to a Trusted Application. struct optee_msg_arg::func is Trusted + * Application function, specific to the Trusted Application. + * + * OPTEE_MSG_CMD_CLOSE_SESSION closes a previously opened session to + * Trusted Application. + * + * OPTEE_MSG_CMD_CANCEL cancels a currently invoked command. + * + * OPTEE_MSG_CMD_REGISTER_SHM registers a shared memory reference. The + * information is passed as: + * [in] param[0].attr OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + * [| OPTEE_MSG_ATTR_NONCONTIG] + * [in] param[0].u.tmem.buf_ptr physical address (of first fragment) + * [in] param[0].u.tmem.size size (of first fragment) + * [in] param[0].u.tmem.shm_ref holds shared memory reference + * + * OPTEE_MSG_CMD_UNREGISTER_SHM unregisteres a previously registered shared + * memory reference. The information is passed as: + * [in] param[0].attr OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + * [in] param[0].u.rmem.shm_ref holds shared memory reference + * [in] param[0].u.rmem.offs 0 + * [in] param[0].u.rmem.size 0 + * + * OPTEE_MSG_CMD_DO_BOTTOM_HALF does the scheduled bottom half processing + * of a driver. + * + * OPTEE_MSG_CMD_STOP_ASYNC_NOTIF informs secure world that from now is + * normal world unable to process asynchronous notifications. Typically + * used when the driver is shut down. + */ +#define OPTEE_MSG_CMD_OPEN_SESSION U(0) +#define OPTEE_MSG_CMD_INVOKE_COMMAND U(1) +#define OPTEE_MSG_CMD_CLOSE_SESSION U(2) +#define OPTEE_MSG_CMD_CANCEL U(3) +#define OPTEE_MSG_CMD_REGISTER_SHM U(4) +#define OPTEE_MSG_CMD_UNREGISTER_SHM U(5) +#define OPTEE_MSG_CMD_DO_BOTTOM_HALF U(6) +#define OPTEE_MSG_CMD_STOP_ASYNC_NOTIF U(7) +#define OPTEE_MSG_FUNCID_CALL_WITH_ARG U(0x0004) + +#endif /* _OPTEE_MSG_H */ diff --git a/optee_os/core/include/optee_rpc_cmd.h b/optee_os/core/include/optee_rpc_cmd.h new file mode 100644 index 0000000..816f0a2 --- /dev/null +++ b/optee_os/core/include/optee_rpc_cmd.h @@ -0,0 +1,391 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2021, Linaro Limited + */ + +#ifndef __OPTEE_RPC_CMD_H +#define __OPTEE_RPC_CMD_H + +/* + * All RPC is done with a struct optee_msg_arg as bearer of information, + * struct optee_msg_arg::arg holds values defined by OPTEE_RPC_CMD_* below. + * Only the commands handled by the kernel driver are defined here. + * + * RPC communication with tee-supplicant is reversed compared to normal + * client communication described above. The supplicant receives requests + * and sends responses. + */ + +/* + * Load a TA into memory + * + * Since the size of the TA isn't known in advance the size of the TA is + * can be queried with a NULL buffer. + * + * [in] value[0].a-b UUID + * [out] memref[1] Buffer with TA + */ +#define OPTEE_RPC_CMD_LOAD_TA U(0) + +/* + * Replay Protected Memory Block access + * + * [in] memref[0] Frames to device + * [out] memref[1] Frames from device + */ +#define OPTEE_RPC_CMD_RPMB U(1) + +/* + * File system access, see definition of protocol below + */ +#define OPTEE_RPC_CMD_FS U(2) + +/* + * Get time + * + * Returns number of seconds and nano seconds since the Epoch, + * 1970-01-01 00:00:00 +0000 (UTC). + * + * [out] value[0].a Number of seconds + * [out] value[0].b Number of nano seconds. + */ +#define OPTEE_RPC_CMD_GET_TIME U(3) + +/* + * Notification from/to secure world. + * + * If secure world needs to wait for something, for instance a mutex, it + * does a notification wait request instead of spinning in secure world. + * Conversely a synchronous notification can be sent when a secure + * world mutex with a thread waiting thread is unlocked. + * + * This interface can also be used to wait for a asynchronous notification + * which instead is sent via a non-secure interrupt. + * + * Waiting on notification + * [in] value[0].a OPTEE_RPC_NOTIFICATION_WAIT + * [in] value[0].b notification value + * + * Sending a synchronous notification + * [in] value[0].a OPTEE_RPC_NOTIFICATION_SEND + * [in] value[0].b notification value + */ +#define OPTEE_RPC_CMD_NOTIFICATION U(4) +#define OPTEE_RPC_NOTIFICATION_WAIT U(0) +#define OPTEE_RPC_NOTIFICATION_SEND U(1) + +/* + * Suspend execution + * + * [in] value[0].a Number of milliseconds to suspend + */ +#define OPTEE_RPC_CMD_SUSPEND U(5) + +/* + * Allocate a piece of shared memory + * + * [in] value[0].a Type of memory one of + * OPTEE_RPC_SHM_TYPE_* below + * [in] value[0].b Requested size + * [in] value[0].c Required alignment + * [out] memref[0] Buffer + */ +#define OPTEE_RPC_CMD_SHM_ALLOC U(6) +/* Memory that can be shared with a non-secure user space application */ +#define OPTEE_RPC_SHM_TYPE_APPL U(0) +/* Memory only shared with non-secure kernel */ +#define OPTEE_RPC_SHM_TYPE_KERNEL U(1) +/* + * Memory shared with non-secure kernel and exported to a non-secure user + * space application + */ +#define OPTEE_RPC_SHM_TYPE_GLOBAL U(2) + +/* + * Free shared memory previously allocated with OPTEE_RPC_CMD_SHM_ALLOC + * + * [in] value[0].a Type of memory one of + * OPTEE_RPC_SHM_TYPE_* above + * [in] value[0].b Value of shared memory reference or cookie + */ +#define OPTEE_RPC_CMD_SHM_FREE U(7) + +/* Was OPTEE_RPC_CMD_SQL_FS, which isn't supported any longer */ +#define OPTEE_RPC_CMD_SQL_FS_RESERVED U(8) + +/* + * Send TA profiling information to normal world + * + * [in/out] value[0].a File identifier. Must be set to 0 on + * first call. A value >= 1 will be + * returned on success. Re-use this value + * to append data to the same file. + * [in] memref[1] TA UUID + * [in] memref[2] Profile data + */ +#define OPTEE_RPC_CMD_GPROF U(9) + +/* + * Socket command, see definition of protocol below + */ +#define OPTEE_RPC_CMD_SOCKET U(10) + +/* + * Send TA function graph data to normal world + * + * [in/out] value[0].a File identifier. Must be set to 0 on + * first call. A value >= 1 will be + * returned on success. Re-use this value + * to append data to the same file. + * [in] memref[1] TA UUID + * [in] memref[2] function graph data + */ +#define OPTEE_RPC_CMD_FTRACE U(11) + +/* + * tee-supplicant plugin command, see definition of protocol below + */ +#define OPTEE_RPC_CMD_SUPP_PLUGIN U(12) + +/* + * Register timestamp buffer in the linux kernel optee driver + * + * [in] value[0].a Subcommand (register buffer, unregister buffer) + * [in] value[0].b Physical address of timestamp buffer + * [in] value[0].c Size of buffer + */ +#define OPTEE_RPC_CMD_BENCH_REG U(20) + +/* + * Issue master requests (read and write operations) to an I2C chip. + * + * [in] value[0].a Transfer mode (OPTEE_RPC_I2C_TRANSFER_*) + * [in] value[0].b The I2C bus (a.k.a adapter). + * 16 bit field. + * [in] value[0].c The I2C chip (a.k.a address). + * 16 bit field (either 7 or 10 bit effective). + * [in] value[1].a The I2C master control flags (ie, 10 bit address). + * 16 bit field. + * [in/out] memref[2] Buffer used for data transfers. + * [out] value[3].a Number of bytes transferred by the REE. + */ +#define OPTEE_RPC_CMD_I2C_TRANSFER U(21) + +/* I2C master transfer modes */ +#define OPTEE_RPC_I2C_TRANSFER_RD U(0) +#define OPTEE_RPC_I2C_TRANSFER_WR U(1) + +/* I2C master control flags */ +#define OPTEE_RPC_I2C_FLAGS_TEN_BIT BIT(0) + +/* + * Definition of protocol for command OPTEE_RPC_CMD_FS + */ + +/* + * Open a file + * + * [in] value[0].a OPTEE_RPC_FS_OPEN + * [in] memref[1] A string holding the file name + * [out] value[2].a File descriptor of open file + */ +#define OPTEE_RPC_FS_OPEN U(0) + +/* + * Create a file + * + * [in] value[0].a OPTEE_RPC_FS_CREATE + * [in] memref[1] A string holding the file name + * [out] value[2].a File descriptor of open file + */ +#define OPTEE_RPC_FS_CREATE U(1) + +/* + * Close a file + * + * [in] value[0].a OPTEE_RPC_FS_CLOSE + * [in] value[0].b File descriptor of open file. + */ +#define OPTEE_RPC_FS_CLOSE U(2) + +/* + * Read from a file + * + * [in] value[0].a OPTEE_RPC_FS_READ + * [in] value[0].b File descriptor of open file + * [in] value[0].c Offset into file + * [out] memref[1] Buffer to hold returned data + */ +#define OPTEE_RPC_FS_READ U(3) + +/* + * Write to a file + * + * [in] value[0].a OPTEE_RPC_FS_WRITE + * [in] value[0].b File descriptor of open file + * [in] value[0].c Offset into file + * [in] memref[1] Buffer holding data to be written + */ +#define OPTEE_RPC_FS_WRITE U(4) + +/* + * Truncate a file + * + * [in] value[0].a OPTEE_RPC_FS_TRUNCATE + * [in] value[0].b File descriptor of open file + * [in] value[0].c Length of file. + */ +#define OPTEE_RPC_FS_TRUNCATE U(5) + +/* + * Remove a file + * + * [in] value[0].a OPTEE_RPC_FS_REMOVE + * [in] memref[1] A string holding the file name + */ +#define OPTEE_RPC_FS_REMOVE U(6) + +/* + * Rename a file + * + * [in] value[0].a OPTEE_RPC_FS_RENAME + * [in] value[0].b True if existing target should be removed + * [in] memref[1] A string holding the old file name + * [in] memref[2] A string holding the new file name + */ +#define OPTEE_RPC_FS_RENAME U(7) + +/* + * Opens a directory for file listing + * + * [in] value[0].a OPTEE_RPC_FS_OPENDIR + * [in] memref[1] A string holding the name of the directory + * [out] value[2].a Handle to open directory + */ +#define OPTEE_RPC_FS_OPENDIR U(8) + +/* + * Closes a directory handle + * + * [in] value[0].a OPTEE_RPC_FS_CLOSEDIR + * [in] value[0].b Handle to open directory + */ +#define OPTEE_RPC_FS_CLOSEDIR U(9) + +/* + * Read next file name of directory + * + * + * [in] value[0].a OPTEE_RPC_FS_READDIR + * [in] value[0].b Handle to open directory + * [out] memref[1] A string holding the file name + */ +#define OPTEE_RPC_FS_READDIR U(10) + +/* End of definition of protocol for command OPTEE_RPC_CMD_FS */ + +/* + * Definition of protocol for command OPTEE_RPC_CMD_SOCKET + */ + +#define OPTEE_RPC_SOCKET_TIMEOUT_NONBLOCKING U(0) +#define OPTEE_RPC_SOCKET_TIMEOUT_BLOCKING U(0xffffffff) + +/* + * Open socket + * + * [in] value[0].a OPTEE_RPC_SOCKET_OPEN + * [in] value[0].b TA instance id + * [in] value[1].a Server port number + * [in] value[1].b Protocol, TEE_ISOCKET_PROTOCOLID_* + * [in] value[1].c Ip version TEE_IP_VERSION_* from tee_ipsocket.h + * [in] memref[2] Server address + * [out] value[3].a Socket handle (32-bit) + */ +#define OPTEE_RPC_SOCKET_OPEN U(0) + +/* + * Close socket + * + * [in] value[0].a OPTEE_RPC_SOCKET_CLOSE + * [in] value[0].b TA instance id + * [in] value[0].c Socket handle + */ +#define OPTEE_RPC_SOCKET_CLOSE U(1) + +/* + * Close all sockets + * + * [in] value[0].a OPTEE_RPC_SOCKET_CLOSE_ALL + * [in] value[0].b TA instance id + */ +#define OPTEE_RPC_SOCKET_CLOSE_ALL U(2) + +/* + * Send data on socket + * + * [in] value[0].a OPTEE_RPC_SOCKET_SEND + * [in] value[0].b TA instance id + * [in] value[0].c Socket handle + * [in] memref[1] Buffer to transmit + * [in] value[2].a Timeout ms or OPTEE_RPC_SOCKET_TIMEOUT_* + * [out] value[2].b Number of transmitted bytes + */ +#define OPTEE_RPC_SOCKET_SEND U(3) + +/* + * Receive data on socket + * + * [in] value[0].a OPTEE_RPC_SOCKET_RECV + * [in] value[0].b TA instance id + * [in] value[0].c Socket handle + * [out] memref[1] Buffer to receive + * [in] value[2].a Timeout ms or OPTEE_RPC_SOCKET_TIMEOUT_* + */ +#define OPTEE_RPC_SOCKET_RECV U(4) + +/* + * Perform IOCTL on socket + * + * [in] value[0].a OPTEE_RPC_SOCKET_IOCTL + * [in] value[0].b TA instance id + * [in] value[0].c Socket handle + * [in/out] memref[1] Buffer + * [in] value[2].a Ioctl command + */ +#define OPTEE_RPC_SOCKET_IOCTL U(5) + +/* End of definition of protocol for command OPTEE_RPC_CMD_SOCKET */ + +/* + * Definition of protocol for command OPTEE_RPC_CMD_SUPP_PLUGIN + */ + +/* + * Invoke tee-supplicant's plugin. + * + * [in] value[0].a OPTEE_RPC_SUPP_PLUGIN_INVOKE + * [in] value[0].b uuid.d1 + * [in] value[0].c uuid.d2 + * [in] value[1].a uuid.d3 + * [in] value[1].b uuid.d4 + * [in] value[1].c cmd for plugin + * [in] value[2].a sub_cmd for plugin + * [out] value[2].b length of the outbuf (memref[3]), if out is needed. + * [in/out] memref[3] buffer holding data for plugin + * + * UUID serialized into octets: + * b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 b11 b12 b13 b14 b15 + * d1 | d2 | d3 | d4 + * + * The endianness of words d1, d2, d3 and d4 must be little-endian. + * d1 word contains [b3 b2 b1 b0] + * d2 word contains [b7 b6 b5 b4] + * d3 word contains [b11 b10 b9 b8] + * d4 word contains [b15 b14 b13 b12] + */ +#define OPTEE_RPC_SUPP_PLUGIN_INVOKE U(0) + +/* End of definition of protocol for command OPTEE_RPC_CMD_SUPP_PLUGIN */ + +#endif /*__OPTEE_RPC_CMD_H*/ diff --git a/optee_os/core/include/rng_support.h b/optee_os/core/include/rng_support.h new file mode 100644 index 0000000..1dc4d89 --- /dev/null +++ b/optee_os/core/include/rng_support.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __RNG_SUPPORT_H__ +#define __RNG_SUPPORT_H__ + +#include + +TEE_Result hw_get_random_bytes(void *buf, size_t blen); + +#endif /* __RNG_SUPPORT_H__ */ diff --git a/optee_os/core/include/scattered_array.h b/optee_os/core/include/scattered_array.h new file mode 100644 index 0000000..1b337de --- /dev/null +++ b/optee_os/core/include/scattered_array.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ +#ifndef __SCATTERED_ARRAY_H +#define __SCATTERED_ARRAY_H + +#include +#include + +/* + * A scattered array is assembled from items declared in different source + * files depending on something like "SORT(.scattered_array*)" in the link + * script to get everything assembled in the right order. + * + * Whenever a new scattered array is created with the macros below there's + * no need to update the link script. + */ + +#define __SCT_ARRAY_DEF_ITEM3(element_type, element_name, section_name) \ + static const element_type element_name; \ + DECLARE_KEEP_INIT(element_name); \ + static const element_type element_name __used \ + __section(section_name) + +#define __SCT_ARRAY_DEF_PG_ITEM3(element_type, element_name, section_name) \ + static const element_type element_name __used \ + __section(section_name) + +#define __SCT_ARRAY_DEF_ITEM2(array_name, order, id, element_type) \ + __SCT_ARRAY_DEF_ITEM3(element_type, \ + __scattered_array_ ## id ## array_name, \ + ".scattered_array_" #array_name "_1_" #order) + +#define __SCT_ARRAY_DEF_PG_ITEM2(array_name, order, id, element_type) \ + __SCT_ARRAY_DEF_PG_ITEM3(element_type, \ + __scattered_array_ ## id ## array_name, \ + ".scattered_array_" #array_name "_1_" #order) + +#define __SCT_ARRAY_DEF_ITEM1(array_name, order, id, element_type) \ + __SCT_ARRAY_DEF_ITEM2(array_name, order, id, element_type) + +#define __SCT_ARRAY_DEF_PG_ITEM1(array_name, order, id, element_type) \ + __SCT_ARRAY_DEF_PG_ITEM2(array_name, order, id, element_type) + +/* + * Defines an item in a scattered array, sorted based on @order. + * @array_name: Name of the scattered array + * @order: Tag on which this item is sorted in the array + * @element_type: The type of the elemenet + */ +#define SCATTERED_ARRAY_DEFINE_ITEM_ORDERED(array_name, order, element_type) \ + __SCT_ARRAY_DEF_ITEM1(array_name, order, __COUNTER__, element_type) + +/* + * Same as SCATTERED_ARRAY_DEFINE_ITEM_ORDERED except that references + * to other objects (for instance null terminated strings) are allowed + * to reside in the paged area without residing in the init area + */ +#define SCATTERED_ARRAY_DEFINE_PG_ITEM_ORDERED(array_name, order, \ + element_type) \ + __SCT_ARRAY_DEF_PG_ITEM1(array_name, order, __COUNTER__, element_type) + +/* + * Defines an item in a scattered array + * @array_name: Name of the scattered array + * @element_type: The type of the elemenet + */ +#define SCATTERED_ARRAY_DEFINE_ITEM(array_name, element_type) \ + __SCT_ARRAY_DEF_ITEM1(array_name, 0, __COUNTER__, element_type) + +/* + * Same as SCATTERED_ARRAY_DEFINE_ITEM except that references to other + * objects (for instance null terminated strings) are allowed to reside in + * the paged area without residing in the init area + */ +#define SCATTERED_ARRAY_DEFINE_PG_ITEM(array_name, element_type) \ + __SCT_ARRAY_DEF_PG_ITEM1(array_name, 0, __COUNTER__, element_type) + +/* + * Returns the first element in a scattered array + * @array_name: Name of the scattered array + * @element_type: The type of the elemenet + */ +#define SCATTERED_ARRAY_BEGIN(array_name, element_type) (__extension__({ \ + static const element_type __scattered_array_begin[0] __unused \ + __section(".scattered_array_" #array_name "_0"); \ + \ + (const element_type *)scattered_array_relax_ptr( \ + __scattered_array_begin); \ + })) + +/* + * Returns one entry past the last element in a scattered array + * @array_name: Name of the scattered array + * @element_type: The type of the elemenet + */ +#define SCATTERED_ARRAY_END(array_name, element_type) (__extension__({ \ + static const element_type __scattered_array_end[0] __unused \ + __section(".scattered_array_" #array_name "_2"); \ + \ + __scattered_array_end; \ + })) + +/* + * Loop over all elements in the scattered array + * @elem: Iterator + * @array_name: Name of the scattered array + * @element_type: The type of the elemenet + */ +#define SCATTERED_ARRAY_FOREACH(elem, array_name, element_type) \ + for ((elem) = SCATTERED_ARRAY_BEGIN(array_name, element_type); \ + (elem) < SCATTERED_ARRAY_END(array_name, element_type); (elem)++) + +/* + * scattered_array_relax_ptr() - relax pointer attributes + * @p pointer to return + * + * If the pointer returned from the array __scattered_array_begin[] in + * SCATTERED_ARRAY_BEGIN() is passed directly the compiler may notice that + * it's an empty array and emit warnings. With the address passed via this + * function the compiler will have no such knowledge about the pointer. + * + * Returns supplied pointer. + */ +const void *scattered_array_relax_ptr(const void *p) __attr_const; + +#endif /*__SCATTERED_ARRAY_H*/ + diff --git a/optee_os/core/include/signed_hdr.h b/optee_os/core/include/signed_hdr.h new file mode 100644 index 0000000..edeb446 --- /dev/null +++ b/optee_os/core/include/signed_hdr.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef SIGNED_HDR_H +#define SIGNED_HDR_H + +#include +#include +#include +#include + +enum shdr_img_type { + SHDR_TA = 0, + SHDR_BOOTSTRAP_TA = 1, + SHDR_ENCRYPTED_TA = 2, + SHDR_SUBKEY = 3, +}; + +#define SHDR_MAGIC 0x4f545348 + +/** + * struct shdr - signed header + * @magic: magic number must match SHDR_MAGIC + * @img_type: image type, values defined by enum shdr_img_type + * @img_size: image size in bytes + * @algo: algorithm, defined by public key algorithms TEE_ALG_* + * from TEE Internal API specification + * @hash_size: size of the signed hash + * @sig_size: size of the signature + * @hash: hash of an image + * @sig: signature of @hash + */ +struct shdr { + uint32_t magic; + uint32_t img_type; + uint32_t img_size; + uint32_t algo; + uint16_t hash_size; + uint16_t sig_size; + /* + * Commented out element used to visualize the layout dynamic part + * of the struct. + * + * hash is accessed through the macro SHDR_GET_HASH and + * signature is accessed through the macro SHDR_GET_SIG + * + * uint8_t hash[hash_size]; + * uint8_t sig[sig_size]; + */ +}; + +static inline size_t shdr_get_size(const struct shdr *shdr) +{ + size_t s = sizeof(*shdr); + + if (ADD_OVERFLOW(s, shdr->hash_size, &s) || + ADD_OVERFLOW(s, shdr->sig_size, &s)) + return 0; + + return s; +} + +#define SHDR_GET_SIZE(x) shdr_get_size((x)) +#define SHDR_GET_HASH(x) (uint8_t *)(((struct shdr *)(x)) + 1) +#define SHDR_GET_SIG(x) (SHDR_GET_HASH(x) + (x)->hash_size) + +/** + * struct shdr_subkey - subkey header + * @uuid: UUID of the subkey + * @name_size: The size of a name field that follows right + * after this header, before the next signed header. + * @subkey_version: Version of the subkey + * @max_depth: Maximum depth supported below this subkey + * @algo: Algorithm, defined by public key algorithms TEE_ALG_* + * from TEE Internal API specification + * @attr_count: Number of attributes for the public key matching + * @algo. + * @attrs: Attributes for the public key matching @algo. + * @attrs[].id: Attribute ID TEE_ATTR_* from GlobalPlatform + * @attrs[].offs: Offset of the attribute value from start of + * struct shdr_subkey + * @attrs[].size: Attribute size + * + * The @uuid defines UUID URN Namespace (RFC4122), the next UUID after this + * header (another subkey or a TA) must be in the namespace of this UUID. + * This means that further subkeys or TAs have their UUID fixed in the + * hierarchy and cannot be moved up or below another subkey. + * + * If @name_size is non-zero it indicates that a name field of this size + * exists and is used to generate the UUID of the following TA or subkey. + * If it's zero the following TA or subkey must have a matching UUID. + * + * The @subkey_version field is used as a rollback measure. The version is + * checked against earlier saved values of this subkey. If the latest known + * version is less than this the stored value is updated. If the latest + * known version is larger than this then the subkey is refused. + * + * The @max_depth defines how many levels are allowed below this subkey, + * the value 0 means only TAs are allowed below. The value 1 means that + * eventual subkeys below must have the value 0 in their @max_depth field. + * + * Each attribute of @attrs must be within range of the image size of this + * header defined in the preceding struct shdr. + * + * The next struct shdr is found right after the indicated end of the + * previous struct shdr. Signature verification starts over with the + * next struct shdr using this subkey instead of the root key. + */ +struct shdr_subkey { + uint8_t uuid[sizeof(TEE_UUID)]; + uint32_t name_size; + uint32_t subkey_version; + uint32_t max_depth; + uint32_t algo; + uint32_t attr_count; + struct shdr_subkey_attr { + uint32_t id; + uint32_t offs; + uint32_t size; + } attrs[]; +}; + +/** + * struct shdr_bootstrap_ta - bootstrap TA subheader + * @uuid: UUID of the TA + * @ta_version: Version of the TA + */ +struct shdr_bootstrap_ta { + uint8_t uuid[sizeof(TEE_UUID)]; + uint32_t ta_version; +}; + +/** + * struct shdr_encrypted_ta - encrypted TA header + * @enc_algo: authenticated encyption algorithm, defined by symmetric key + * algorithms TEE_ALG_* from TEE Internal API + * specification + * @flags: authenticated encyption flags + * @iv_size: size of the initialization vector + * @tag_size: size of the authentication tag + * @iv: initialization vector + * @tag: authentication tag + */ +struct shdr_encrypted_ta { + uint32_t enc_algo; + uint32_t flags; + uint16_t iv_size; + uint16_t tag_size; + /* + * Commented out element used to visualize the layout dynamic part + * of the struct. + * + * iv is accessed through the macro SHDR_ENC_GET_IV and + * tag is accessed through the macro SHDR_ENC_GET_TAG + * + * uint8_t iv[iv_size]; + * uint8_t tag[tag_size]; + */ +}; + +#define SHDR_ENC_KEY_TYPE_MASK 0x1 + +enum shdr_enc_key_type { + SHDR_ENC_KEY_DEV_SPECIFIC = 0, + SHDR_ENC_KEY_CLASS_WIDE = 1, +}; + +static inline size_t shdr_enc_get_size(const struct shdr_encrypted_ta *ehdr) +{ + size_t s = sizeof(*ehdr); + + if (ADD_OVERFLOW(s, ehdr->iv_size, &s) || + ADD_OVERFLOW(s, ehdr->tag_size, &s)) + return 0; + + return s; +} + +#define SHDR_ENC_GET_SIZE(x) shdr_enc_get_size((x)) +#define SHDR_ENC_GET_IV(x) ((uint8_t *) \ + (((struct shdr_encrypted_ta *)(x)) + 1)) +#define SHDR_ENC_GET_TAG(x) ({ typeof(x) _x = (x); \ + (SHDR_ENC_GET_IV(_x) + _x->iv_size); }) + +/* + * Allocates a struct shdr large enough to hold the entire header, + * excluding a subheader like struct shdr_bootstrap_ta. + */ +struct shdr *shdr_alloc_and_copy(size_t offs, const void *img, size_t img_size); + +/* Frees a previously allocated struct shdr */ +static inline void shdr_free(struct shdr *shdr) +{ + free(shdr); +} + +struct shdr_pub_key { + uint32_t main_algo; + uint8_t uuid[sizeof(TEE_UUID)]; + uint8_t next_uuid[sizeof(TEE_UUID)]; + uint32_t max_depth; + uint32_t name_size; + uint32_t version; + union { + struct rsa_public_key *rsa; + } pub_key; +}; + +TEE_Result shdr_load_pub_key(const struct shdr *shdr, size_t offs, + const uint8_t *ns_img, size_t ns_img_size, + const uint8_t next_uuid[sizeof(TEE_UUID)], + uint32_t max_depth, struct shdr_pub_key *key); +void shdr_free_pub_key(struct shdr_pub_key *key); +TEE_Result shdr_verify_signature2(struct shdr_pub_key *key, + const struct shdr *shdr); + +/* + * Verifies the signature in the @shdr. + * + * Note that the static part of struct shdr and payload still need to be + * checked against the hash contained in the header. + * + * Returns TEE_SUCCESS on success or TEE_ERROR_SECURITY on failure + */ +TEE_Result shdr_verify_signature(const struct shdr *shdr); + +#endif /*SIGNED_HDR_H*/ diff --git a/optee_os/core/include/spi.h b/optee_os/core/include/spi.h new file mode 100644 index 0000000..334b5ed --- /dev/null +++ b/optee_os/core/include/spi.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef __SPI_H__ +#define __SPI_H__ + +#include + +enum spi_mode { + SPI_MODE0, + SPI_MODE1, + SPI_MODE2, + SPI_MODE3 +}; + +enum spi_result { + SPI_OK, + SPI_ERR_CFG, + SPI_ERR_PKTCNT, + SPI_ERR_GENERIC +}; + +struct spi_chip { + const struct spi_ops *ops; +}; + +struct spi_ops { + void (*configure)(struct spi_chip *chip); + void (*start)(struct spi_chip *chip); + enum spi_result (*txrx8)(struct spi_chip *chip, uint8_t *wdat, + uint8_t *rdat, size_t num_pkts); + enum spi_result (*txrx16)(struct spi_chip *chip, uint16_t *wdat, + uint16_t *rdat, size_t num_pkts); + void (*end)(struct spi_chip *chip); + void (*flushfifo)(struct spi_chip *chip); +}; + +#endif /* __SPI_H__ */ + diff --git a/optee_os/core/include/ta_pub_key.h b/optee_os/core/include/ta_pub_key.h new file mode 100644 index 0000000..1259e01 --- /dev/null +++ b/optee_os/core/include/ta_pub_key.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef KERNEL_TA_PUB_KEY_H +#define KERNEL_TA_PUB_KEY_H + +#include + +extern const uint32_t ta_pub_key_exponent; +extern const uint8_t ta_pub_key_modulus[]; +extern const size_t ta_pub_key_modulus_size; + +#endif /*KERNEL_TA_PUB_KEY_H*/ + diff --git a/optee_os/core/include/tee/cache.h b/optee_os/core/include/tee/cache.h new file mode 100644 index 0000000..568b251 --- /dev/null +++ b/optee_os/core/include/tee/cache.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef TEE_CACHE_H +#define TEE_CACHE_H + +#include + +TEE_Result cache_operation(enum utee_cache_operation op, void *va, size_t len); + +#endif /* TEE_CACHE_H */ diff --git a/optee_os/core/include/tee/entry_std.h b/optee_os/core/include/tee/entry_std.h new file mode 100644 index 0000000..d228bad --- /dev/null +++ b/optee_os/core/include/tee/entry_std.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_ENTRY_STD_H +#define TEE_ENTRY_STD_H + +#include +#include +#include + +/* + * Standard call entry, __weak, overridable. If overridden should call + * __tee_entry_std() at the end in order to handle the standard functions. + * + * These functions are called in a normal thread context. + */ +TEE_Result tee_entry_std(struct optee_msg_arg *arg, uint32_t num_params); +TEE_Result __tee_entry_std(struct optee_msg_arg *arg, uint32_t num_params); + +/* Get list head for sessions opened from non-secure */ +void nsec_sessions_list_head(struct tee_ta_session_head **open_sessions); + +#endif /* TEE_ENTRY_STD_H */ diff --git a/optee_os/core/include/tee/fs_dirfile.h b/optee_os/core/include/tee/fs_dirfile.h new file mode 100644 index 0000000..2ed6791 --- /dev/null +++ b/optee_os/core/include/tee/fs_dirfile.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __TEE_FS_DIRFILE_H +#define __TEE_FS_DIRFILE_H + +#include +#include + +struct tee_fs_dirfile_dirh; + +/** + * struct tee_fs_dirfile_fileh - file handle + * @file_number: sequence number of a file + * @hash: hash of file, to be supplied to tee_fs_htree_open() + * @idx: index of the file handle in the dirfile + */ +struct tee_fs_dirfile_fileh { + uint32_t file_number; + uint8_t hash[TEE_FS_HTREE_HASH_SIZE]; + int idx; +}; + +/** + * struct tee_fs_dirfile_operations - file interface supplied by user of this + * interface + * @open: opens a file + * @close: closes a file, changes are discarded unless + * @commit_writes is called before + * @read: reads from an open file + * @write: writes to an open file + * @commit_writes: commits changes since the file was opened + */ +struct tee_fs_dirfile_operations { + TEE_Result (*open)(bool create, uint8_t *hash, uint32_t min_counter, + const TEE_UUID *uuid, + struct tee_fs_dirfile_fileh *dfh, + struct tee_file_handle **fh); + void (*close)(struct tee_file_handle *fh); + TEE_Result (*read)(struct tee_file_handle *fh, size_t pos, void *buf, + size_t *len); + TEE_Result (*write)(struct tee_file_handle *fh, size_t pos, + const void *buf, size_t len); + TEE_Result (*commit_writes)(struct tee_file_handle *fh, uint8_t *hash, + uint32_t *counter); +}; + +/** + * tee_fs_dirfile_open() - opens a dirfile handle + * @create: true if a new dirfile is to be created, else the dirfile + * is read opened and verified + * @hash: hash of underlying file + * @min_counter: the smallest accepted value in struct htree_image.counter + * @fops: file interface + * @dirh: returned dirfile handle + */ +TEE_Result tee_fs_dirfile_open(bool create, uint8_t *hash, uint32_t min_counter, + const struct tee_fs_dirfile_operations *fops, + struct tee_fs_dirfile_dirh **dirh); +/** + * tee_fs_dirfile_close() - closes a dirfile handle + * @dirh: dirfile handle + * + * All changes since last call to tee_fs_dirfile_commit_writes() are + * discarded. + */ +void tee_fs_dirfile_close(struct tee_fs_dirfile_dirh *dirh); + +/** + * tee_fs_dirfile_commit_writes() - commit updates of dirfile + * @dirh: dirfile handle + * @hash: hash of underlying file is copied here if not NULL + * @counter: version counter of underlying file is copied here if not NULL + */ +TEE_Result tee_fs_dirfile_commit_writes(struct tee_fs_dirfile_dirh *dirh, + uint8_t *hash, uint32_t *counter); + +/** + * tee_fs_dirfile_get_tmp() - get a temporary file handle + * @dirh: dirfile handle + * @dfh: returned temporary file handle + * + * Note, nothing is queued up as changes to the dirfile with this function. + */ +TEE_Result tee_fs_dirfile_get_tmp(struct tee_fs_dirfile_dirh *dirh, + struct tee_fs_dirfile_fileh *dfh); + +/** + * tee_fs_dirfile_find() - find a file handle + * @dirh: dirfile handle + * @uuid: uuid of requesting TA + * @oid: object id + * @oidlen: length of object id + * @dfh: returned file handle + */ +TEE_Result tee_fs_dirfile_find(struct tee_fs_dirfile_dirh *dirh, + const TEE_UUID *uuid, const void *oid, + size_t oidlen, struct tee_fs_dirfile_fileh *dfh); + +/** + * tee_fs_dirfile_fileh_to_fname() - get string representation of file handle + * @dfh: file handle + * @fname: buffer + * @fnlen: length of buffer, updated to used length + */ +TEE_Result tee_fs_dirfile_fileh_to_fname(const struct tee_fs_dirfile_fileh *dfh, + char *fname, size_t *fnlen); + +/** + * tee_fs_dirfile_rename() - changes/supplies file handle object id + * @dirh: dirfile handle + * @uuid: uuid of requesting TA + * @dfh: file handle + * @oid: object id + * @oidlen: length of object id + * + * If the supplied object id already is used by another file is that file + * removed from the dirfile. + */ +TEE_Result tee_fs_dirfile_rename(struct tee_fs_dirfile_dirh *dirh, + const TEE_UUID *uuid, + struct tee_fs_dirfile_fileh *dfh, + const void *oid, size_t oidlen); + +/** + * tee_fs_dirfile_remove() - remove file + * @dirh: dirfile handle + * @dfh: file handle + */ +TEE_Result tee_fs_dirfile_remove(struct tee_fs_dirfile_dirh *dirh, + const struct tee_fs_dirfile_fileh *dfh); + +/** + * tee_fs_dirfile_update_hash() - update hash of file handle + * @dirh: filefile handle + * @dfh: file handle + */ +TEE_Result tee_fs_dirfile_update_hash(struct tee_fs_dirfile_dirh *dirh, + const struct tee_fs_dirfile_fileh *dfh); + +/** + * tee_fs_dirfile_get_next() - get object id of next file + * @dirh: dirfile handle + * @uuid: uuid of requesting TA + * @idx: pointer to index + * @oid: object id + * @oidlen: length of object id + * + * If @idx contains -1 the first object id is returned, *@idx is updated + * with the index of the file. + */ +TEE_Result tee_fs_dirfile_get_next(struct tee_fs_dirfile_dirh *dirh, + const TEE_UUID *uuid, int *idx, void *oid, + size_t *oidlen); + +#endif /*__TEE_FS_DIRFILE_H*/ diff --git a/optee_os/core/include/tee/fs_htree.h b/optee_os/core/include/tee/fs_htree.h new file mode 100644 index 0000000..19f9426 --- /dev/null +++ b/optee_os/core/include/tee/fs_htree.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __TEE_FS_HTREE_H +#define __TEE_FS_HTREE_H + +/* + * The purpose of this API is to provide file integrity and confidentiality + * in order to implement secure storage. On-disk data structures are + * duplicated to make updates atomic, an update is finalized to disk with + * tee_fs_htree_sync_to_storage(). + * + * This implementation doesn't provide rollback protection, it only + * guarantees the integrity and confidentiality of the file. + */ + +#include +#include +#include + +#define TEE_FS_HTREE_HASH_SIZE TEE_SHA256_HASH_SIZE +#define TEE_FS_HTREE_IV_SIZE U(16) +#define TEE_FS_HTREE_FEK_SIZE U(16) +#define TEE_FS_HTREE_TAG_SIZE U(16) + +/* Internal struct provided to let the rpc callbacks know the size if needed */ +struct tee_fs_htree_node_image { + /* Note that calc_node_hash() depends on hash first in struct */ + uint8_t hash[TEE_FS_HTREE_HASH_SIZE]; + uint8_t iv[TEE_FS_HTREE_IV_SIZE]; + uint8_t tag[TEE_FS_HTREE_TAG_SIZE]; + uint16_t flags; +}; + +/* + * This struct is not interpreted by the hash tree, it's up to the user of + * the interface to update etc if needed. + */ +struct tee_fs_htree_meta { + uint64_t length; +}; + +/* Internal struct needed by struct tee_fs_htree_image */ +struct tee_fs_htree_imeta { + struct tee_fs_htree_meta meta; + uint32_t max_node_id; +}; + +/* Internal struct provided to let the rpc callbacks know the size if needed */ +struct tee_fs_htree_image { + uint8_t iv[TEE_FS_HTREE_IV_SIZE]; + uint8_t tag[TEE_FS_HTREE_TAG_SIZE]; + uint8_t enc_fek[TEE_FS_HTREE_FEK_SIZE]; + uint8_t imeta[sizeof(struct tee_fs_htree_imeta)]; + uint32_t counter; +}; + +/** + * enum tee_fs_htree_type - type of hash tree element + * @TEE_FS_HTREE_TYPE_HEAD: indicates a struct tee_fs_htree_image + * @TEE_FS_HTREE_TYPE_NODE: indicates a struct tee_fs_htree_node_image + * @TEE_FS_HTREE_TYPE_BLOCK: indicates a data block + */ +enum tee_fs_htree_type { + TEE_FS_HTREE_TYPE_HEAD, + TEE_FS_HTREE_TYPE_NODE, + TEE_FS_HTREE_TYPE_BLOCK, +}; + +struct tee_fs_rpc_operation; + +/** + * struct tee_fs_htree_storage - storage description supplied by user of + * this interface + * @block_size: size of data blocks + * @rpc_read_init: initialize a struct tee_fs_rpc_operation for an RPC read + * operation + * @rpc_write_init: initialize a struct tee_fs_rpc_operation for an RPC + * write operation + * + * The @idx arguments starts counting from 0. The @vers arguments are either + * 0 or 1. The @data arguments is a pointer to a buffer in non-secure shared + * memory where the encrypted data is stored. + */ +struct tee_fs_htree_storage { + size_t block_size; + TEE_Result (*rpc_read_init)(void *aux, struct tee_fs_rpc_operation *op, + enum tee_fs_htree_type type, size_t idx, + uint8_t vers, void **data); + TEE_Result (*rpc_read_final)(struct tee_fs_rpc_operation *op, + size_t *bytes); + TEE_Result (*rpc_write_init)(void *aux, struct tee_fs_rpc_operation *op, + enum tee_fs_htree_type type, size_t idx, + uint8_t vers, void **data); + TEE_Result (*rpc_write_final)(struct tee_fs_rpc_operation *op); +}; + +struct tee_fs_htree; + +/** + * tee_fs_htree_open() - opens/creates a hash tree + * @create: true if a new hash tree is to be created, else the hash tree + * is read in and verified + * @hash: hash of root node, ignored if NULL + * @min_counter: the smallest accepted value in struct htree_image.counter + * @uuid: uuid of requesting TA, may be NULL if not from a TA + * @stor: storage description + * @stor_aux: auxilary pointer supplied to callbacks in struct + * tee_fs_htree_storage + * @ht: returned hash tree on success + */ +TEE_Result tee_fs_htree_open(bool create, uint8_t *hash, uint32_t min_counter, + const TEE_UUID *uuid, + const struct tee_fs_htree_storage *stor, + void *stor_aux, struct tee_fs_htree **ht); +/** + * tee_fs_htree_close() - close a hash tree + * @ht: hash tree + */ +void tee_fs_htree_close(struct tee_fs_htree **ht); + +/** + * tee_fs_htree_get_meta() - get a pointer to associated struct + * tee_fs_htree_meta + * @ht: hash tree + */ +struct tee_fs_htree_meta *tee_fs_htree_get_meta(struct tee_fs_htree *ht); + +/** + * tee_fs_htree_meta_set_dirty() - tell hash tree that meta were modified + */ +void tee_fs_htree_meta_set_dirty(struct tee_fs_htree *ht); + +/** + * tee_fs_htree_sync_to_storage() - synchronize hash tree to storage + * @ht: hash tree + * @hash: hash of root node is copied to this if not NULL + * @counter: ever increasing version counter is copied to this if not NULL + * + * Frees the hash tree and sets *ht to NULL on failure and returns an error code + */ +TEE_Result tee_fs_htree_sync_to_storage(struct tee_fs_htree **ht, + uint8_t *hash, uint32_t *counter); + +/** + * tee_fs_htree_truncate() - truncate a hash tree + * @ht: hash tree + * @block_num: the number of nodes to truncate to + * + * Frees the hash tree and sets *ht to NULL on failure and returns an error code + */ +TEE_Result tee_fs_htree_truncate(struct tee_fs_htree **ht, size_t block_num); + +/** + * tee_fs_htree_write_block() - encrypt and write a data block to storage + * @ht: hash tree + * @block_num: block number + * @block: pointer to a block of stor->block_size size + * + * Frees the hash tree and sets *ht to NULL on failure and returns an error code + */ +TEE_Result tee_fs_htree_write_block(struct tee_fs_htree **ht, size_t block_num, + const void *block); +/** + * tee_fs_htree_write_block() - read and decrypt a data block from storage + * @ht: hash tree + * @block_num: block number + * @block: pointer to a block of stor->block_size size + * + * Frees the hash tree and sets *ht to NULL on failure and returns an error code + */ +TEE_Result tee_fs_htree_read_block(struct tee_fs_htree **ht, size_t block_num, + void *block); + +#endif /*__TEE_FS_HTREE_H*/ diff --git a/optee_os/core/include/tee/svc_cache.h b/optee_os/core/include/tee/svc_cache.h new file mode 100644 index 0000000..a2bccde --- /dev/null +++ b/optee_os/core/include/tee/svc_cache.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef SVC_CACHE_H +#define SVC_CACHE_H + +#include +#include + +#ifdef CFG_CACHE_API +TEE_Result syscall_cache_operation(void *va, size_t len, unsigned long op); +#else +#define syscall_cache_operation syscall_not_supported +#endif + +#endif /*SVC_CACHE_H*/ diff --git a/optee_os/core/include/tee/tadb.h b/optee_os/core/include/tee/tadb.h new file mode 100644 index 0000000..33dd700 --- /dev/null +++ b/optee_os/core/include/tee/tadb.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __TEE_TADB_H +#define __TEE_TADB_H + +#include + +struct tee_tadb_ta_write; +struct tee_tadb_ta_read; + +/* + * struct tee_tadb_property + * @uuid: UUID of Trusted Application (TA) or Security Domain (SD) + * @version: Version of TA or SD + * @custom_size:Size of customized properties, prepended to the encrypted + * TA binary + * @bin_size: Size of the binary TA + */ +struct tee_tadb_property { + TEE_UUID uuid; + uint32_t version; + uint32_t custom_size; + uint32_t bin_size; +}; + +struct tee_fs_rpc_operation; + +struct tee_tadb_file_operations { + TEE_Result (*open)(uint32_t file_number, int *fd); + TEE_Result (*create)(uint32_t file_number, int *fd); + void (*close)(int fd); + TEE_Result (*remove)(uint32_t file_number); + + TEE_Result (*read_init)(struct tee_fs_rpc_operation *op, int fd, + size_t pos, uint8_t **data, size_t bytes); + TEE_Result (*read_final)(struct tee_fs_rpc_operation *op, + size_t *bytes); + + TEE_Result (*write_init)(struct tee_fs_rpc_operation *op, int fd, + size_t pos, uint8_t **data, size_t len); + TEE_Result (*write_final)(struct tee_fs_rpc_operation *op); +}; + +TEE_Result tee_tadb_ta_create(const struct tee_tadb_property *property, + struct tee_tadb_ta_write **ta); +TEE_Result tee_tadb_ta_write(struct tee_tadb_ta_write *ta, const void *buf, + size_t len); +void tee_tadb_ta_close_and_delete(struct tee_tadb_ta_write *ta); +TEE_Result tee_tadb_ta_close_and_commit(struct tee_tadb_ta_write *ta); + +TEE_Result tee_tadb_ta_delete(const TEE_UUID *uuid); + +TEE_Result tee_tadb_ta_open(const TEE_UUID *uuid, struct tee_tadb_ta_read **ta); +const struct tee_tadb_property * +tee_tadb_ta_get_property(struct tee_tadb_ta_read *ta); +TEE_Result tee_tadb_get_tag(struct tee_tadb_ta_read *ta, uint8_t *tag, + unsigned int *tag_len); +TEE_Result tee_tadb_ta_read(struct tee_tadb_ta_read *ta, void *buf_core, + void *buf_user, size_t *len); +void tee_tadb_ta_close(struct tee_tadb_ta_read *ta); + + +#endif /*__TEE_TADB_H*/ diff --git a/optee_os/core/include/tee/tee_cryp_concat_kdf.h b/optee_os/core/include/tee/tee_cryp_concat_kdf.h new file mode 100644 index 0000000..49d3348 --- /dev/null +++ b/optee_os/core/include/tee/tee_cryp_concat_kdf.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEE_CRYP_CONCAT_KDF_H +#define TEE_CRYP_CONCAT_KDF_H + +#include + +TEE_Result tee_cryp_concat_kdf(uint32_t hash_id, const uint8_t *shared_secret, + size_t shared_secret_len, + const uint8_t *other_info, + size_t other_info_len, uint8_t *derived_key, + size_t derived_key_len); + +#endif /* TEE_CRYP_CONCAT_KDF_H */ diff --git a/optee_os/core/include/tee/tee_cryp_hkdf.h b/optee_os/core/include/tee/tee_cryp_hkdf.h new file mode 100644 index 0000000..be0703e --- /dev/null +++ b/optee_os/core/include/tee/tee_cryp_hkdf.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEE_CRYP_HKDF_H +#define TEE_CRYP_HKDF_H + +#include + +TEE_Result tee_cryp_hkdf(uint32_t hash_id, const uint8_t *ikm, size_t ikm_len, + const uint8_t *salt, size_t salt_len, + const uint8_t *info, size_t info_len, uint8_t *okm, + size_t okm_len); + +#endif /* TEE_CRYP_HKDF_H */ diff --git a/optee_os/core/include/tee/tee_cryp_pbkdf2.h b/optee_os/core/include/tee/tee_cryp_pbkdf2.h new file mode 100644 index 0000000..81b7beb --- /dev/null +++ b/optee_os/core/include/tee/tee_cryp_pbkdf2.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEE_CRYP_PBKDF2_H +#define TEE_CRYP_PBKDF2_H + +#include + +TEE_Result tee_cryp_pbkdf2(uint32_t hash_id, const uint8_t *password, + size_t password_len, const uint8_t *salt, + size_t salt_len, uint32_t iteration_count, + uint8_t *derived_key, size_t derived_key_len); + +#endif /* TEE_CRYP_PBKDF2_H */ diff --git a/optee_os/core/include/tee/tee_cryp_utl.h b/optee_os/core/include/tee/tee_cryp_utl.h new file mode 100644 index 0000000..98a694a --- /dev/null +++ b/optee_os/core/include/tee/tee_cryp_utl.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, Linaro Limited + */ + +#ifndef TEE_CRYP_UTL_H +#define TEE_CRYP_UTL_H + +#include +#include + +TEE_Result tee_alg_get_digest_size(uint32_t algo, size_t *size); +TEE_Result tee_hash_createdigest(uint32_t algo, const uint8_t *data, + size_t datalen, uint8_t *digest, + size_t digestlen); +TEE_Result tee_cipher_get_block_size(uint32_t algo, size_t *size); +TEE_Result tee_do_cipher_update(void *ctx, uint32_t algo, + TEE_OperationMode mode, bool last_block, + const uint8_t *data, size_t len, uint8_t *dst); + +/* + * plat_prng_add_jitter_entropy() - Adds jitter to RNG entropy pool + * @sid: source ID, normally unique per location of the call + * @pnum: pointer where the pool number for this @sid is stored + * + * Note that the supplied @sid controls (CRYPTO_RNG_SRC_IS_QUICK()) whether + * RPC is allowed to be performed or the event just will be queued for later + * consumption. + */ +void plat_prng_add_jitter_entropy(enum crypto_rng_src sid, unsigned int *pnum); + +void plat_rng_init(void); + +#endif diff --git a/optee_os/core/include/tee/tee_fs.h b/optee_os/core/include/tee/tee_fs.h new file mode 100644 index 0000000..1b632d7 --- /dev/null +++ b/optee_os/core/include/tee/tee_fs.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_FS_H +#define TEE_FS_H + +#include +#include +#include +#include + +#define TEE_FS_NAME_MAX U(350) + +typedef int64_t tee_fs_off_t; +typedef uint32_t tee_fs_mode_t; + +struct tee_fs_dirent { + uint8_t oid[TEE_OBJECT_ID_MAX_LEN]; + size_t oidlen; +}; + +struct tee_fs_dir; +struct tee_file_handle; +struct tee_pobj; + +/* + * tee_fs implements a POSIX like secure file system with GP extension + */ +struct tee_file_operations { + TEE_Result (*open)(struct tee_pobj *po, size_t *size, + struct tee_file_handle **fh); + TEE_Result (*create)(struct tee_pobj *po, bool overwrite, + const void *head, size_t head_size, + const void *attr, size_t attr_size, + const void *data_core, const void *data_user, + size_t data_size, struct tee_file_handle **fh); + void (*close)(struct tee_file_handle **fh); + TEE_Result (*read)(struct tee_file_handle *fh, size_t pos, + void *buf_core, void *buf_user, size_t *len); + TEE_Result (*write)(struct tee_file_handle *fh, size_t pos, + const void *buf_core, const void *buf_user, + size_t len); + TEE_Result (*rename)(struct tee_pobj *old_po, struct tee_pobj *new_po, + bool overwrite); + TEE_Result (*remove)(struct tee_pobj *po); + TEE_Result (*truncate)(struct tee_file_handle *fh, size_t size); + + TEE_Result (*opendir)(const TEE_UUID *uuid, struct tee_fs_dir **d); + TEE_Result (*readdir)(struct tee_fs_dir *d, struct tee_fs_dirent **ent); + void (*closedir)(struct tee_fs_dir *d); +}; + +#ifdef CFG_REE_FS +extern const struct tee_file_operations ree_fs_ops; +#endif +#ifdef CFG_RPMB_FS +extern const struct tee_file_operations rpmb_fs_ops; + +TEE_Result tee_rpmb_fs_raw_open(const char *fname, bool create, + struct tee_file_handle **fh); + +/** + * Weak function which can be overridden by platforms to indicate that the RPMB + * key is ready to be written. Defaults to true, platforms can return false to + * prevent a RPMB key write in the wrong state. + */ +bool plat_rpmb_key_is_ready(void); +#endif + +/* + * Returns the appropriate tee_file_operations for the specified storage ID. + * The value TEE_STORAGE_PRIVATE will select the REE FS if available, otherwise + * RPMB. + */ +static inline const struct tee_file_operations * +tee_svc_storage_file_ops(uint32_t storage_id) +{ + switch (storage_id) { + case TEE_STORAGE_PRIVATE: +#if defined(CFG_REE_FS) + return &ree_fs_ops; +#elif defined(CFG_RPMB_FS) + return &rpmb_fs_ops; +#else + return NULL; +#endif +#ifdef CFG_REE_FS + case TEE_STORAGE_PRIVATE_REE: + return &ree_fs_ops; +#endif +#ifdef CFG_RPMB_FS + case TEE_STORAGE_PRIVATE_RPMB: + return &rpmb_fs_ops; +#endif + default: + return NULL; + } +} + +#endif /*TEE_FS_H*/ diff --git a/optee_os/core/include/tee/tee_fs_key_manager.h b/optee_os/core/include/tee/tee_fs_key_manager.h new file mode 100644 index 0000000..ac57630 --- /dev/null +++ b/optee_os/core/include/tee/tee_fs_key_manager.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef TEE_FS_KEY_MANAGER_H +#define TEE_FS_KEY_MANAGER_H + +#include +#include + +#define TEE_FS_KM_CHIP_ID_LENGTH U(32) +#define TEE_FS_KM_HMAC_ALG TEE_ALG_HMAC_SHA256 +#define TEE_FS_KM_ENC_FEK_ALG TEE_ALG_AES_ECB_NOPAD +#define TEE_FS_KM_SSK_SIZE TEE_SHA256_HASH_SIZE +#define TEE_FS_KM_TSK_SIZE TEE_SHA256_HASH_SIZE +#define TEE_FS_KM_FEK_SIZE U(16) /* bytes */ + +TEE_Result tee_fs_generate_fek(const TEE_UUID *uuid, void *encrypted_fek, + size_t fek_size); +TEE_Result tee_fs_crypt_block(const TEE_UUID *uuid, uint8_t *out, + const uint8_t *in, size_t size, + uint16_t blk_idx, const uint8_t *encrypted_fek, + TEE_OperationMode mode); + +TEE_Result tee_fs_fek_crypt(const TEE_UUID *uuid, TEE_OperationMode mode, + const uint8_t *in_key, size_t size, + uint8_t *out_key); + +#endif diff --git a/optee_os/core/include/tee/tee_fs_rpc.h b/optee_os/core/include/tee/tee_fs_rpc.h new file mode 100644 index 0000000..ae49682 --- /dev/null +++ b/optee_os/core/include/tee/tee_fs_rpc.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +/* + * Interface with tee-supplicant for file operations + */ + +#ifndef TEE_FS_RPC_H +#define TEE_FS_RPC_H + +#include +#include +#include +#include +#include + +struct tee_fs_rpc_operation { + uint32_t id; + struct thread_param params[THREAD_RPC_MAX_NUM_PARAMS]; + size_t num_params; +}; + +struct tee_fs_dirfile_fileh; + +TEE_Result tee_fs_rpc_open_dfh(uint32_t id, + const struct tee_fs_dirfile_fileh *dfh, int *fd); +TEE_Result tee_fs_rpc_create_dfh(uint32_t id, + const struct tee_fs_dirfile_fileh *dfh, + int *fd); +TEE_Result tee_fs_rpc_close(uint32_t id, int fd); + +TEE_Result tee_fs_rpc_read_init(struct tee_fs_rpc_operation *op, + uint32_t id, int fd, tee_fs_off_t offset, + size_t data_len, void **out_data); +TEE_Result tee_fs_rpc_read_final(struct tee_fs_rpc_operation *op, + size_t *data_len); + +TEE_Result tee_fs_rpc_write_init(struct tee_fs_rpc_operation *op, + uint32_t id, int fd, tee_fs_off_t offset, + size_t data_len, void **data); +TEE_Result tee_fs_rpc_write_final(struct tee_fs_rpc_operation *op); + + +TEE_Result tee_fs_rpc_truncate(uint32_t id, int fd, size_t len); +TEE_Result tee_fs_rpc_remove_dfh(uint32_t id, + const struct tee_fs_dirfile_fileh *dfh); +#endif /* TEE_FS_RPC_H */ diff --git a/optee_os/core/include/tee/tee_obj.h b/optee_os/core/include/tee/tee_obj.h new file mode 100644 index 0000000..adbe543 --- /dev/null +++ b/optee_os/core/include/tee/tee_obj.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_OBJ_H +#define TEE_OBJ_H + +#include +#include +#include +#include + +#define TEE_USAGE_DEFAULT 0xffffffff + +struct tee_obj { + TAILQ_ENTRY(tee_obj) link; + TEE_ObjectInfo info; + bool busy; /* true if used by an operation */ + uint32_t have_attrs; /* bitfield identifying set properties */ + void *attr; + size_t ds_pos; + struct tee_pobj *pobj; /* ptr to persistant object */ + struct tee_file_handle *fh; +}; + +void tee_obj_add(struct user_ta_ctx *utc, struct tee_obj *o); + +TEE_Result tee_obj_get(struct user_ta_ctx *utc, vaddr_t obj_id, + struct tee_obj **obj); + +void tee_obj_close(struct user_ta_ctx *utc, struct tee_obj *o); + +void tee_obj_close_all(struct user_ta_ctx *utc); + +TEE_Result tee_obj_verify(struct tee_ta_session *sess, struct tee_obj *o); + +struct tee_obj *tee_obj_alloc(void); +void tee_obj_free(struct tee_obj *o); + +#endif diff --git a/optee_os/core/include/tee/tee_pobj.h b/optee_os/core/include/tee/tee_pobj.h new file mode 100644 index 0000000..bd05da4 --- /dev/null +++ b/optee_os/core/include/tee/tee_pobj.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_POBJ_H +#define TEE_POBJ_H + +#include +#include +#include +#include + +struct tee_pobj { + TAILQ_ENTRY(tee_pobj) link; + uint32_t refcnt; + TEE_UUID uuid; + void *obj_id; + uint32_t obj_id_len; + uint32_t flags; + bool temporary; /* can be changed while creating == true */ + bool creating; /* can only be changed with mutex held */ + /* Filesystem handling this object */ + const struct tee_file_operations *fops; +}; + +enum tee_pobj_usage { + TEE_POBJ_USAGE_OPEN, + TEE_POBJ_USAGE_RENAME, + TEE_POBJ_USAGE_CREATE, + TEE_POBJ_USAGE_ENUM, +}; + +TEE_Result tee_pobj_get(TEE_UUID *uuid, void *obj_id, uint32_t obj_id_len, + uint32_t flags, enum tee_pobj_usage usage, + const struct tee_file_operations *fops, + struct tee_pobj **obj); + +void tee_pobj_create_final(struct tee_pobj *obj); + +TEE_Result tee_pobj_release(struct tee_pobj *obj); + +TEE_Result tee_pobj_rename(struct tee_pobj *obj, void *obj_id, + uint32_t obj_id_len); + +#endif diff --git a/optee_os/core/include/tee/tee_supp_plugin_rpc.h b/optee_os/core/include/tee/tee_supp_plugin_rpc.h new file mode 100644 index 0000000..2d725c9 --- /dev/null +++ b/optee_os/core/include/tee/tee_supp_plugin_rpc.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#ifndef TEE_SUPP_PLUGIN_RPC_H +#define TEE_SUPP_PLUGIN_RPC_H + +#include +#include +#include + +TEE_Result tee_invoke_supp_plugin_rpc(const TEE_UUID *uuid, uint32_t cmd, + uint32_t sub_cmd, void *buf_core, + void *buf_user, size_t len, + size_t *outlen); + +#endif /* TEE_SUPP_PLUGIN_RPC_H */ diff --git a/optee_os/core/include/tee/tee_svc.h b/optee_os/core/include/tee/tee_svc.h new file mode 100644 index 0000000..27a04c0 --- /dev/null +++ b/optee_os/core/include/tee/tee_svc.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TEE_SVC_H +#define TEE_SVC_H + +#include +#include +#include +#include +#include + +struct tee_ta_session; + +/* TA Properties */ +struct tee_props { + const char *name; + + /* prop_type is of type enum user_ta_prop_type*/ + const uint32_t prop_type; + + /* either get_prop_func or both data and len */ + TEE_Result (*get_prop_func)(struct ts_session *sess, + void *buf, size_t *blen); + const void *data; + const size_t len; +}; + +struct tee_vendor_props { + const struct tee_props *props; + size_t len; +}; +void syscall_sys_return(unsigned long ret); + +void syscall_log(const void *buf, size_t len); + +void syscall_panic(unsigned long code); + +TEE_Result syscall_not_supported(void); + +/* prop_set defined by enum utee_property */ +TEE_Result syscall_get_property(unsigned long prop_set, + unsigned long index, + void *name, uint32_t *name_len, + void *buf, uint32_t *blen, + uint32_t *prop_type); +TEE_Result syscall_get_property_name_to_index(unsigned long prop_set, + void *name, + unsigned long name_len, + uint32_t *index); + +TEE_Result syscall_open_ta_session(const TEE_UUID *dest, + unsigned long cancel_req_to, struct utee_params *params, + uint32_t *sess, uint32_t *ret_orig); + +TEE_Result syscall_close_ta_session(unsigned long sess); + +TEE_Result syscall_invoke_ta_command(unsigned long sess, + unsigned long cancel_req_to, unsigned long cmd_id, + struct utee_params *params, uint32_t *ret_orig); + +TEE_Result syscall_check_access_rights(unsigned long flags, const void *buf, + size_t len); + +TEE_Result syscall_get_cancellation_flag(uint32_t *cancel); + +TEE_Result syscall_unmask_cancellation(uint32_t *old_mask); + +TEE_Result syscall_mask_cancellation(uint32_t *old_mask); + +TEE_Result syscall_wait(unsigned long timeout); + +TEE_Result syscall_get_time(unsigned long cat, TEE_Time *time); +TEE_Result syscall_set_ta_time(const TEE_Time *time); + +#endif /* TEE_SVC_H */ diff --git a/optee_os/core/include/tee/tee_svc_cryp.h b/optee_os/core/include/tee/tee_svc_cryp.h new file mode 100644 index 0000000..3932cdf --- /dev/null +++ b/optee_os/core/include/tee/tee_svc_cryp.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* +* Copyright (c) 2014, STMicroelectronics International N.V. +*/ +#ifndef TEE_SVC_CRYP_H +#define TEE_SVC_CRYP_H + +#include +#include +#include + +struct user_ta_ctx; + +TEE_Result syscall_cryp_obj_get_info(unsigned long obj, + struct utee_object_info *info); +TEE_Result syscall_cryp_obj_restrict_usage(unsigned long obj, + unsigned long usage); +TEE_Result syscall_cryp_obj_get_attr(unsigned long obj, unsigned long attr_id, + void *buffer, uint64_t *size); + +TEE_Result syscall_cryp_obj_alloc(unsigned long obj_type, + unsigned long max_key_size, uint32_t *obj); +TEE_Result syscall_cryp_obj_close(unsigned long obj); +TEE_Result syscall_cryp_obj_reset(unsigned long obj); +TEE_Result syscall_cryp_obj_populate(unsigned long obj, + struct utee_attribute *attrs, unsigned long attr_count); +TEE_Result syscall_cryp_obj_copy(unsigned long dst_obj, + unsigned long src_obj); +TEE_Result syscall_obj_generate_key(unsigned long obj, unsigned long key_size, + const struct utee_attribute *params, + unsigned long param_count); + +TEE_Result syscall_cryp_state_alloc(unsigned long algo, unsigned long op_mode, + unsigned long key1, unsigned long key2, + uint32_t *state); +TEE_Result syscall_cryp_state_copy(unsigned long dst, unsigned long src); +TEE_Result syscall_cryp_state_free(unsigned long state); +void tee_svc_cryp_free_states(struct user_ta_ctx *utc); + +/* iv and iv_len are ignored for hash algorithms */ +TEE_Result syscall_hash_init(unsigned long state, const void *iv, + size_t iv_len); +TEE_Result syscall_hash_update(unsigned long state, const void *chunk, + size_t chunk_size); +TEE_Result syscall_hash_final(unsigned long state, const void *chunk, + size_t chunk_size, void *hash, uint64_t *hash_len); + +TEE_Result syscall_cipher_init(unsigned long state, const void *iv, + size_t iv_len); +TEE_Result syscall_cipher_update(unsigned long state, const void *src, + size_t src_len, void *dest, uint64_t *dest_len); +TEE_Result syscall_cipher_final(unsigned long state, const void *src, + size_t src_len, void *dest, uint64_t *dest_len); + +TEE_Result syscall_cryp_derive_key(unsigned long state, + const struct utee_attribute *params, + unsigned long param_count, unsigned long derived_key); + +TEE_Result syscall_cryp_random_number_generate(void *buf, size_t blen); + +TEE_Result syscall_authenc_init(unsigned long state, const void *nonce, + size_t nonce_len, size_t tag_len, + size_t aad_len, size_t payload_len); +TEE_Result syscall_authenc_update_aad(unsigned long state, + const void *aad_data, size_t aad_data_len); +TEE_Result syscall_authenc_update_payload(unsigned long state, + const void *src_data, size_t src_len, void *dest_data, + uint64_t *dest_len); +TEE_Result syscall_authenc_enc_final(unsigned long state, + const void *src_data, size_t src_len, void *dest_data, + uint64_t *dest_len, void *tag, uint64_t *tag_len); +TEE_Result syscall_authenc_dec_final(unsigned long state, + const void *src_data, size_t src_len, void *dest_data, + uint64_t *dest_len, const void *tag, size_t tag_len); + +TEE_Result syscall_asymm_operate(unsigned long state, + const struct utee_attribute *usr_params, + size_t num_params, const void *src_data, + size_t src_len, void *dest_data, uint64_t *dest_len); +TEE_Result syscall_asymm_verify(unsigned long state, + const struct utee_attribute *usr_params, + size_t num_params, const void *data, size_t data_len, + const void *sig, size_t sig_len); + +TEE_Result tee_obj_set_type(struct tee_obj *o, uint32_t obj_type, + size_t max_key_size); + +void tee_obj_attr_free(struct tee_obj *o); +void tee_obj_attr_clear(struct tee_obj *o); +TEE_Result tee_obj_attr_to_binary(struct tee_obj *o, void *data, + size_t *data_len); +TEE_Result tee_obj_attr_from_binary(struct tee_obj *o, const void *data, + size_t data_len); +TEE_Result tee_obj_attr_copy_from(struct tee_obj *o, const struct tee_obj *src); + +#endif /* TEE_SVC_CRYP_H */ diff --git a/optee_os/core/include/tee/tee_svc_storage.h b/optee_os/core/include/tee/tee_svc_storage.h new file mode 100644 index 0000000..c5d763a --- /dev/null +++ b/optee_os/core/include/tee/tee_svc_storage.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_SVC_STORAGE_H +#define TEE_SVC_STORAGE_H + +#include +#include +#include + +/* + * Persistant Object Functions + */ +TEE_Result syscall_storage_obj_open(unsigned long storage_id, void *object_id, + size_t object_id_len, unsigned long flags, + uint32_t *obj); + +TEE_Result syscall_storage_obj_create(unsigned long storage_id, void *object_id, + size_t object_id_len, unsigned long flags, + unsigned long attr, void *data, size_t len, + uint32_t *obj); + +TEE_Result syscall_storage_obj_del(unsigned long obj); + +TEE_Result syscall_storage_obj_rename(unsigned long obj, void *object_id, + size_t object_id_len); + +/* + * Persistent Object Enumeration Functions + */ +TEE_Result syscall_storage_alloc_enum(uint32_t *obj_enum); + +TEE_Result syscall_storage_free_enum(unsigned long obj_enum); + +TEE_Result syscall_storage_reset_enum(unsigned long obj_enum); + +TEE_Result syscall_storage_start_enum(unsigned long obj_enum, + unsigned long storage_id); + +TEE_Result syscall_storage_next_enum(unsigned long obj_enum, + struct utee_object_info *info, + void *obj_id, uint64_t *len); + +/* + * Data Stream Access Functions + */ +TEE_Result syscall_storage_obj_read(unsigned long obj, void *data, size_t len, + uint64_t *count); + +TEE_Result syscall_storage_obj_write(unsigned long obj, void *data, + size_t len); + +TEE_Result syscall_storage_obj_trunc(unsigned long obj, size_t len); + +TEE_Result syscall_storage_obj_seek(unsigned long obj, int32_t offset, + unsigned long whence); + +void tee_svc_storage_close_all_enum(struct user_ta_ctx *utc); + +void tee_svc_storage_init(void); + +#endif /* TEE_SVC_STORAGE_H */ diff --git a/optee_os/core/include/tee/tee_ta_enc_manager.h b/optee_os/core/include/tee/tee_ta_enc_manager.h new file mode 100644 index 0000000..5e1b2e5 --- /dev/null +++ b/optee_os/core/include/tee/tee_ta_enc_manager.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef TEE_TA_ENC_MANAGER_H +#define TEE_TA_ENC_MANAGER_H + +#include +#include +#include + +#define TEE_TA_ENC_KEY_SIZE TEE_SHA256_HASH_SIZE + +TEE_Result tee_ta_decrypt_init(void **enc_ctx, struct shdr_encrypted_ta *ehdr, + size_t len); +TEE_Result tee_ta_decrypt_update(void *enc_ctx, uint8_t *dst, uint8_t *src, + size_t len); +TEE_Result tee_ta_decrypt_final(void *enc_ctx, struct shdr_encrypted_ta *ehdr, + uint8_t *dst, uint8_t *src, size_t len); + +#endif diff --git a/optee_os/core/include/tee/uuid.h b/optee_os/core/include/tee/uuid.h new file mode 100644 index 0000000..f203800 --- /dev/null +++ b/optee_os/core/include/tee/uuid.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef __TEE_UUID +#define __TEE_UUID + +#include +#include + +/** + * tee_uuid_to_octets() - serialize a TEE_UUID into an octet string + * @dst: pointer to octet string + * @src: pointer TEE_UUID + */ +void tee_uuid_to_octets(uint8_t *dst, const TEE_UUID *src); + +/** + * tee_uuid_from_octets() - de-serialize an octet string into a TEE_UUID + * @dst: pointer TEE_UUID + * @src: pointer to octet string + */ +void tee_uuid_from_octets(TEE_UUID *dst, const uint8_t *src); + +#endif /*__TEE_UUID*/ diff --git a/optee_os/core/kernel/asan.c b/optee_os/core/kernel/asan.c new file mode 100644 index 0000000..0f2d670 --- /dev/null +++ b/optee_os/core/kernel/asan.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __GCC_VERSION >= 70000 +#define ASAN_ABI_VERSION 7 +#else +#define ASAN_ABI_VERSION 6 +#endif + +struct asan_source_location { + const char *file_name; + int line_no; + int column_no; +}; + +struct asan_global { + uintptr_t beg; + uintptr_t size; + uintptr_t size_with_redzone; + const char *name; + const char *module_name; + uintptr_t has_dynamic_init; + struct asan_source_location *location; +#if ASAN_ABI_VERSION >= 7 + uintptr_t odr_indicator; +#endif +}; + +static vaddr_t asan_va_base; +static size_t asan_va_size; +static bool asan_active; + +static int8_t *va_to_shadow(const void *va) +{ + vaddr_t sa = ((vaddr_t)va / ASAN_BLOCK_SIZE) + CFG_ASAN_SHADOW_OFFSET; + + return (int8_t *)sa; +} + +static size_t va_range_to_shadow_size(const void *begin, const void *end) +{ + return ((vaddr_t)end - (vaddr_t)begin) / ASAN_BLOCK_SIZE; +} + +static bool va_range_inside_shadow(const void *begin, const void *end) +{ + vaddr_t b = (vaddr_t)begin; + vaddr_t e = (vaddr_t)end; + + if (b >= e) + return false; + return (b >= asan_va_base) && (e <= (asan_va_base + asan_va_size)); +} + +static bool va_range_outside_shadow(const void *begin, const void *end) +{ + vaddr_t b = (vaddr_t)begin; + vaddr_t e = (vaddr_t)end; + + if (b >= e) + return false; + return (e <= asan_va_base) || (b >= (asan_va_base + asan_va_size)); +} + +static size_t va_misalignment(const void *va) +{ + return (vaddr_t)va & ASAN_BLOCK_MASK; +} + +static bool va_is_well_aligned(const void *va) +{ + return !va_misalignment(va); +} + +void asan_set_shadowed(const void *begin, const void *end) +{ + vaddr_t b = (vaddr_t)begin; + vaddr_t e = (vaddr_t)end; + + assert(!asan_va_base); + assert(va_is_well_aligned(begin)); + assert(va_is_well_aligned(end)); + assert(b < e); + + asan_va_base = b; + asan_va_size = e - b; +} + +void asan_tag_no_access(const void *begin, const void *end) +{ + assert(va_is_well_aligned(begin)); + assert(va_is_well_aligned(end)); + assert(va_range_inside_shadow(begin, end)); + + asan_memset_unchecked(va_to_shadow(begin), ASAN_DATA_RED_ZONE, + va_range_to_shadow_size(begin, end)); +} + +void asan_tag_access(const void *begin, const void *end) +{ + if (!asan_va_base || (begin == end)) + return; + + assert(va_range_inside_shadow(begin, end)); + assert(va_is_well_aligned(begin)); + + asan_memset_unchecked(va_to_shadow(begin), 0, + va_range_to_shadow_size(begin, end)); + if (!va_is_well_aligned(end)) + *va_to_shadow(end) = ASAN_BLOCK_SIZE - va_misalignment(end); +} + +void asan_tag_heap_free(const void *begin, const void *end) +{ + if (!asan_va_base) + return; + + assert(va_range_inside_shadow(begin, end)); + assert(va_is_well_aligned(begin)); + assert(va_is_well_aligned(end)); + + asan_memset_unchecked(va_to_shadow(begin), ASAN_HEAP_RED_ZONE, + va_range_to_shadow_size(begin, end)); +} + +__inhibit_loop_to_libcall void *asan_memset_unchecked(void *s, int c, size_t n) +{ + uint8_t *b = s; + size_t m; + + for (m = 0; m < n; m++) + b[m] = c; + + return s; +} + +__inhibit_loop_to_libcall +void *asan_memcpy_unchecked(void *__restrict dst, const void *__restrict src, + size_t len) +{ + uint8_t *__restrict d = dst; + const uint8_t *__restrict s = src; + size_t n; + + for (n = 0; n < len; n++) + d[n] = s[n]; + + return dst; +} + +void asan_start(void) +{ + assert(asan_va_base && !asan_active); + asan_active = true; +} + +static void check_access(vaddr_t addr, size_t size) +{ + void *begin = (void *)addr; + void *end = (void *)(addr + size); + int8_t *a; + int8_t *e; + + if (!asan_active || !size) + return; + if (va_range_outside_shadow(begin, end)) + return; + /* + * If it isn't outside it has to be completely inside or there's a + * problem. + */ + if (!va_range_inside_shadow(begin, end)) + panic(); + + e = va_to_shadow((void *)(addr + size - 1)); + for (a = va_to_shadow(begin); a <= e; a++) + if (*a < 0) + panic(); + + if (!va_is_well_aligned(end) && + va_misalignment(end) > (size_t)(*e - ASAN_BLOCK_SIZE)) + panic(); +} + +static void check_load(vaddr_t addr, size_t size) +{ + check_access(addr, size); +} + +static void check_store(vaddr_t addr, size_t size) +{ + check_access(addr, size); +} + +static void __noreturn report_load(vaddr_t addr __unused, size_t size __unused) +{ + panic(); +} + +static void __noreturn report_store(vaddr_t addr __unused, size_t size __unused) +{ + panic(); +} + + + +#define DEFINE_ASAN_FUNC(type, size) \ + void __asan_##type##size(vaddr_t addr); \ + void __asan_##type##size(vaddr_t addr) \ + { check_##type(addr, size); } \ + void __asan_##type##size##_noabort(vaddr_t addr); \ + void __asan_##type##size##_noabort(vaddr_t addr) \ + { check_##type(addr, size); } \ + void __asan_report_##type##size##_noabort(vaddr_t addr);\ + void __noreturn __asan_report_##type##size##_noabort(vaddr_t addr) \ + { report_##type(addr, size); } + +DEFINE_ASAN_FUNC(load, 1) +DEFINE_ASAN_FUNC(load, 2) +DEFINE_ASAN_FUNC(load, 4) +DEFINE_ASAN_FUNC(load, 8) +DEFINE_ASAN_FUNC(load, 16) +DEFINE_ASAN_FUNC(store, 1) +DEFINE_ASAN_FUNC(store, 2) +DEFINE_ASAN_FUNC(store, 4) +DEFINE_ASAN_FUNC(store, 8) +DEFINE_ASAN_FUNC(store, 16) + +void __asan_loadN_noabort(vaddr_t addr, size_t size); +void __asan_loadN_noabort(vaddr_t addr, size_t size) +{ + check_load(addr, size); +} + +void __asan_storeN_noabort(vaddr_t addr, size_t size); +void __asan_storeN_noabort(vaddr_t addr, size_t size) +{ + check_store(addr, size); +} + +void __asan_report_load_n_noabort(vaddr_t addr, size_t size); +void __noreturn __asan_report_load_n_noabort(vaddr_t addr, size_t size) +{ + report_load(addr, size); +} + +void __asan_report_store_n_noabort(vaddr_t addr, size_t size); +void __noreturn __asan_report_store_n_noabort(vaddr_t addr, size_t size) +{ + report_store(addr, size); +} + +void __asan_handle_no_return(void); +void __asan_handle_no_return(void) +{ +} + +void __asan_register_globals(struct asan_global *globals, size_t size); +void __asan_register_globals(struct asan_global *globals, size_t size) +{ + size_t n; + + for (n = 0; n < size; n++) + asan_tag_access((void *)globals[n].beg, + (void *)(globals[n].beg + globals[n].size)); +} +DECLARE_KEEP_INIT(__asan_register_globals); + +void __asan_unregister_globals(struct asan_global *globals, size_t size); +void __asan_unregister_globals(struct asan_global *globals __unused, + size_t size __unused) +{ +} diff --git a/optee_os/core/kernel/assert.c b/optee_os/core/kernel/assert.c new file mode 100644 index 0000000..1d2e3c9 --- /dev/null +++ b/optee_os/core/kernel/assert.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include + +/* assert log and break for the optee kernel */ + +void __nostackcheck _assert_log(const char *expr __maybe_unused, + const char *file __maybe_unused, + const int line __maybe_unused, + const char *func __maybe_unused) +{ +#if defined(CFG_TEE_CORE_DEBUG) + EMSG_RAW("assertion '%s' failed at %s:%d <%s>", + expr, file, line, func); +#else + EMSG_RAW("assertion failed"); +#endif +} + +void __noreturn _assert_break(void) +{ + panic(); +} diff --git a/optee_os/core/kernel/console.c b/optee_os/core/kernel/console.c new file mode 100644 index 0000000..6291b2c --- /dev/null +++ b/optee_os/core/kernel/console.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct serial_chip *serial_console __nex_bss; + +void __weak console_putc(int ch) +{ + if (!serial_console) + return; + + if (ch == '\n') + serial_console->ops->putc(serial_console, '\r'); + serial_console->ops->putc(serial_console, ch); +} + +void __weak console_flush(void) +{ + if (!serial_console || !serial_console->ops->flush) + return; + + serial_console->ops->flush(serial_console); +} + +void register_serial_console(struct serial_chip *chip) +{ + serial_console = chip; +} + +#ifdef CFG_DT +static int find_chosen_node(void *fdt) +{ + int offset = 0; + + if (!fdt) + return -1; + + offset = fdt_path_offset(fdt, "/secure-chosen"); + + if (offset < 0) + offset = fdt_path_offset(fdt, "/chosen"); + + return offset; +} + +TEE_Result get_console_node_from_dt(void *fdt, int *offs_out, + char **path_out, char **params_out) +{ + const struct fdt_property *prop; + const char *uart; + const char *parms = NULL; + int offs; + char *stdout_data; + char *p; + TEE_Result rc = TEE_ERROR_GENERIC; + + /* Probe console from secure DT and fallback to non-secure DT */ + offs = find_chosen_node(fdt); + if (offs < 0) { + DMSG("No console directive from DTB"); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + prop = fdt_get_property(fdt, offs, "stdout-path", NULL); + if (!prop) { + /* + * A secure-chosen or chosen node is present but defined + * no stdout-path property: no console expected + */ + IMSG("Switching off console"); + register_serial_console(NULL); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + stdout_data = nex_strdup(prop->data); + if (!stdout_data) + panic(); + p = strchr(stdout_data, ':'); + if (p) { + *p = '\0'; + parms = p + 1; + } + + /* stdout-path may refer to an alias */ + uart = fdt_get_alias(fdt, stdout_data); + if (!uart) { + /* Not an alias, assume we have a node path */ + uart = stdout_data; + } + offs = fdt_path_offset(fdt, uart); + if (offs >= 0) { + if (offs_out) + *offs_out = offs; + if (params_out) + *params_out = parms ? nex_strdup(parms) : NULL; + if (path_out) + *path_out = uart ? nex_strdup(uart) : NULL; + + rc = TEE_SUCCESS; + } + + nex_free(stdout_data); + + return rc; +} + +void configure_console_from_dt(void) +{ + const struct dt_driver *dt_drv; + const struct serial_driver *sdrv; + struct serial_chip *dev; + char *uart = NULL; + char *parms = NULL; + void *fdt; + int offs; + + fdt = get_dt(); + + if (IS_ENABLED(CFG_CBMEM_CONSOLE) && cbmem_console_init_from_dt(fdt)) + return; + + if (get_console_node_from_dt(fdt, &offs, &uart, &parms)) + return; + + dt_drv = dt_find_compatible_driver(fdt, offs); + if (!dt_drv || dt_drv->type != DT_DRIVER_UART) + goto out; + + sdrv = (const struct serial_driver *)dt_drv->driver; + if (!sdrv) + goto out; + + dev = sdrv->dev_alloc(); + if (!dev) + goto out; + + /* + * If the console is the same as the early console, dev_init() might + * clear pending data. Flush to avoid that. + */ + console_flush(); + if (sdrv->dev_init(dev, fdt, offs, parms) < 0) { + sdrv->dev_free(dev); + goto out; + } + + IMSG("Switching console to device: %s", uart); + register_serial_console(dev); +out: + nex_free(uart); + nex_free(parms); +} + +#endif /* CFG_DT */ diff --git a/optee_os/core/kernel/delay.c b/optee_os/core/kernel/delay.c new file mode 100644 index 0000000..1c404f9 --- /dev/null +++ b/optee_os/core/kernel/delay.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + * Copyright (C) 2017, Fuzhou Rockchip Electronics Co., Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#ifdef CFG_CORE_HAS_GENERIC_TIMER +void udelay(uint32_t us) +{ + uint64_t target = timeout_init_us(us); + + while (!timeout_elapsed(target)) + ; +} +#else + +void udelay(uint32_t us) +{ + uint64_t cycles = 0; + uint32_t cycles_to_wait = 0; + + cycles = (uint64_t)us * ((uint64_t)plat_get_freq() / 1000000ULL); + + while (cycles) { + cycles_to_wait = MIN(cycles, UINT32_MAX); + wait_cycles(cycles_to_wait); + cycles -= cycles_to_wait; + } +} +#endif + +void mdelay(uint32_t ms) +{ + udelay(1000 * ms); +} diff --git a/optee_os/core/kernel/dt.c b/optee_os/core/kernel/dt.c new file mode 100644 index 0000000..299aba1 --- /dev/null +++ b/optee_os/core/kernel/dt.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct dt_descriptor external_dt __nex_bss; + +const struct dt_driver *dt_find_compatible_driver(const void *fdt, int offs) +{ + const struct dt_device_match *dm; + const struct dt_driver *drv; + + for_each_dt_driver(drv) { + for (dm = drv->match_table; dm; dm++) { + if (!dm->compatible) { + break; + } + if (!fdt_node_check_compatible(fdt, offs, + dm->compatible)) { + return drv; + } + } + } + + return NULL; +} + +bool dt_have_prop(const void *fdt, int offs, const char *propname) +{ + const void *prop; + + prop = fdt_getprop(fdt, offs, propname, NULL); + + return prop; +} + +int dt_disable_status(void *fdt, int node) +{ + const char *prop = NULL; + int len = 0; + + prop = fdt_getprop(fdt, node, "status", &len); + if (!prop) { + if (fdt_setprop_string(fdt, node, "status", "disabled")) + return -1; + } else { + /* + * Status is there, modify it. + * Ask to set "disabled" value to the property. The value + * will be automatically truncated with "len" size by the + * fdt_setprop_inplace function. + * Setting a value different from "ok" or "okay" will disable + * the property. + * Setting a truncated value of "disabled" with the original + * property "len" is preferred to not increase the DT size and + * losing time in recalculating the overall DT offsets. + * If original length of the status property is larger than + * "disabled", the property will start with "disabled" and be + * completed with the rest of the original property. + */ + if (fdt_setprop_inplace(fdt, node, "status", "disabled", len)) + return -1; + } + + return 0; +} + +int dt_enable_secure_status(void *fdt, int node) +{ + if (dt_disable_status(fdt, node)) { + EMSG("Unable to disable Normal Status"); + return -1; + } + + if (fdt_setprop_string(fdt, node, "secure-status", "okay")) + return -1; + + return 0; +} + +int dt_map_dev(const void *fdt, int offs, vaddr_t *base, size_t *size, + enum dt_map_dev_directive mapping) +{ + enum teecore_memtypes mtype; + paddr_t pbase; + vaddr_t vbase; + size_t sz; + int st; + + assert(cpu_mmu_enabled()); + + st = fdt_get_status(fdt, offs); + if (st == DT_STATUS_DISABLED) + return -1; + + pbase = fdt_reg_base_address(fdt, offs); + if (pbase == DT_INFO_INVALID_REG) + return -1; + sz = fdt_reg_size(fdt, offs); + if (sz == DT_INFO_INVALID_REG_SIZE) + return -1; + + switch (mapping) { + case DT_MAP_AUTO: + if ((st & DT_STATUS_OK_SEC) && !(st & DT_STATUS_OK_NSEC)) + mtype = MEM_AREA_IO_SEC; + else + mtype = MEM_AREA_IO_NSEC; + break; + case DT_MAP_SECURE: + mtype = MEM_AREA_IO_SEC; + break; + case DT_MAP_NON_SECURE: + mtype = MEM_AREA_IO_NSEC; + break; + default: + panic("Invalid mapping specified"); + break; + } + + /* Check if we have a mapping, create one if needed */ + vbase = (vaddr_t)core_mmu_add_mapping(mtype, pbase, sz); + if (!vbase) { + EMSG("Failed to map %zu bytes at PA 0x%"PRIxPA, + (size_t)sz, pbase); + return -1; + } + + *base = vbase; + *size = sz; + return 0; +} + +/* Read a physical address (n=1 or 2 cells) */ +static paddr_t fdt_read_paddr(const uint32_t *cell, int n) +{ + paddr_t addr; + + if (n < 1 || n > 2) + goto bad; + + addr = fdt32_to_cpu(*cell); + cell++; + if (n == 2) { +#ifdef ARM32 + if (addr) { + /* High order 32 bits can't be nonzero */ + goto bad; + } + addr = fdt32_to_cpu(*cell); +#else + addr = (addr << 32) | fdt32_to_cpu(*cell); +#endif + } + + return addr; +bad: + return DT_INFO_INVALID_REG; + +} + +paddr_t fdt_reg_base_address(const void *fdt, int offs) +{ + const void *reg; + int ncells; + int len; + int parent; + + parent = fdt_parent_offset(fdt, offs); + if (parent < 0) + return DT_INFO_INVALID_REG; + + reg = fdt_getprop(fdt, offs, "reg", &len); + if (!reg) + return DT_INFO_INVALID_REG; + + ncells = fdt_address_cells(fdt, parent); + if (ncells < 0) + return DT_INFO_INVALID_REG; + + return fdt_read_paddr(reg, ncells); +} + +static size_t fdt_read_size(const uint32_t *cell, int n) +{ + uint32_t sz = 0; + + sz = fdt32_to_cpu(*cell); + if (n == 2) { + if (sz) + return DT_INFO_INVALID_REG_SIZE; + + cell++; + sz = fdt32_to_cpu(*cell); + } + + return sz; +} + +size_t fdt_reg_size(const void *fdt, int offs) +{ + const uint32_t *reg; + int n; + int len; + int parent; + + parent = fdt_parent_offset(fdt, offs); + if (parent < 0) + return DT_INFO_INVALID_REG_SIZE; + + reg = (const uint32_t *)fdt_getprop(fdt, offs, "reg", &len); + if (!reg) + return DT_INFO_INVALID_REG_SIZE; + + n = fdt_address_cells(fdt, parent); + if (n < 1 || n > 2) + return DT_INFO_INVALID_REG_SIZE; + + reg += n; + + n = fdt_size_cells(fdt, parent); + if (n < 1 || n > 2) + return DT_INFO_INVALID_REG_SIZE; + + return fdt_read_size(reg, n); +} + +static bool is_okay(const char *st, int len) +{ + return !strncmp(st, "ok", len) || !strncmp(st, "okay", len); +} + +int fdt_get_status(const void *fdt, int offs) +{ + const char *prop; + int st = 0; + int len; + + prop = fdt_getprop(fdt, offs, "status", &len); + if (!prop || is_okay(prop, len)) { + /* If status is not specified, it defaults to "okay" */ + st |= DT_STATUS_OK_NSEC; + } + + prop = fdt_getprop(fdt, offs, "secure-status", &len); + if (!prop) { + /* + * When secure-status is not specified it defaults to the same + * value as status + */ + if (st & DT_STATUS_OK_NSEC) + st |= DT_STATUS_OK_SEC; + } else { + if (is_okay(prop, len)) + st |= DT_STATUS_OK_SEC; + } + + return st; +} + +void fdt_fill_device_info(const void *fdt, struct dt_node_info *info, int offs) +{ + struct dt_node_info dinfo = { + .reg = DT_INFO_INVALID_REG, + .reg_size = DT_INFO_INVALID_REG_SIZE, + .clock = DT_INFO_INVALID_CLOCK, + .reset = DT_INFO_INVALID_RESET, + .interrupt = DT_INFO_INVALID_INTERRUPT, + }; + const fdt32_t *cuint; + + dinfo.reg = fdt_reg_base_address(fdt, offs); + dinfo.reg_size = fdt_reg_size(fdt, offs); + + cuint = fdt_getprop(fdt, offs, "clocks", NULL); + if (cuint) { + cuint++; + dinfo.clock = (int)fdt32_to_cpu(*cuint); + } + + cuint = fdt_getprop(fdt, offs, "resets", NULL); + if (cuint) { + cuint++; + dinfo.reset = (int)fdt32_to_cpu(*cuint); + } + + dinfo.interrupt = dt_get_irq_type_prio(fdt, offs, &dinfo.type, + &dinfo.prio); + + dinfo.status = fdt_get_status(fdt, offs); + + *info = dinfo; +} + +int fdt_read_uint32_array(const void *fdt, int node, const char *prop_name, + uint32_t *array, size_t count) +{ + const fdt32_t *cuint = NULL; + int len = 0; + uint32_t i = 0; + + cuint = fdt_getprop(fdt, node, prop_name, &len); + if (!cuint) + return len; + + if ((uint32_t)len != (count * sizeof(uint32_t))) + return -FDT_ERR_BADLAYOUT; + + for (i = 0; i < ((uint32_t)len / sizeof(uint32_t)); i++) { + *array = fdt32_to_cpu(*cuint); + array++; + cuint++; + } + + return 0; +} + +int fdt_read_uint32_index(const void *fdt, int node, const char *prop_name, + int index, uint32_t *value) +{ + const fdt32_t *cuint = NULL; + int len = 0; + + cuint = fdt_getprop(fdt, node, prop_name, &len); + if (!cuint) + return len; + + if ((uint32_t)len < (sizeof(uint32_t) * (index + 1))) + return -FDT_ERR_BADLAYOUT; + + *value = fdt32_to_cpu(cuint[index]); + + return 0; +} + +int fdt_read_uint32(const void *fdt, int node, const char *prop_name, + uint32_t *value) +{ + return fdt_read_uint32_array(fdt, node, prop_name, value, 1); +} + +uint32_t fdt_read_uint32_default(const void *fdt, int node, + const char *prop_name, uint32_t dflt_value) +{ + uint32_t ret = dflt_value; + + fdt_read_uint32_index(fdt, node, prop_name, 0, &ret); + + return ret; +} + +int fdt_get_reg_props_by_index(const void *fdt, int node, int index, + paddr_t *base, size_t *size) +{ + const fdt32_t *prop = NULL; + int parent = 0; + int len = 0; + int address_cells = 0; + int size_cells = 0; + int cell = 0; + + parent = fdt_parent_offset(fdt, node); + if (parent < 0) + return parent; + + address_cells = fdt_address_cells(fdt, parent); + if (address_cells < 0) + return address_cells; + + size_cells = fdt_size_cells(fdt, parent); + if (size_cells < 0) + return size_cells; + + cell = index * (address_cells + size_cells); + + prop = fdt_getprop(fdt, node, "reg", &len); + if (!prop) + return len; + + if (((cell + address_cells + size_cells) * (int)sizeof(uint32_t)) > len) + return -FDT_ERR_BADVALUE; + + if (base) { + *base = fdt_read_paddr(&prop[cell], address_cells); + if (*base == DT_INFO_INVALID_REG) + return -FDT_ERR_BADVALUE; + } + + if (size) { + *size = fdt_read_size(&prop[cell + address_cells], size_cells); + if (*size == DT_INFO_INVALID_REG_SIZE) + return -FDT_ERR_BADVALUE; + } + + return 0; +} + +int fdt_get_reg_props_by_name(const void *fdt, int node, const char *name, + paddr_t *base, size_t *size) +{ + int index = 0; + + index = fdt_stringlist_search(fdt, node, "reg-names", name); + if (index < 0) + return index; + + return fdt_get_reg_props_by_index(fdt, node, index, base, size); +} + +int dt_getprop_as_number(const void *fdt, int nodeoffset, const char *name, + uint64_t *num) +{ + const void *prop = NULL; + int len = 0; + + prop = fdt_getprop(fdt, nodeoffset, name, &len); + if (!prop) + return len; + + switch (len) { + case sizeof(uint32_t): + *num = fdt32_ld(prop); + return 0; + case sizeof(uint64_t): + *num = fdt64_ld(prop); + return 0; + default: + return -FDT_ERR_BADVALUE; + } +} + +void *get_dt(void) +{ + void *fdt = get_embedded_dt(); + + if (!fdt) + fdt = get_external_dt(); + + return fdt; +} + +void *get_secure_dt(void) +{ + void *fdt = get_embedded_dt(); + + if (!fdt && IS_ENABLED(CFG_MAP_EXT_DT_SECURE)) + fdt = get_external_dt(); + + return fdt; +} + +#if defined(CFG_EMBED_DTB) +void *get_embedded_dt(void) +{ + static bool checked; + + assert(cpu_mmu_enabled()); + + if (!checked) { + IMSG("Embedded DTB found"); + + if (fdt_check_header(embedded_secure_dtb)) + panic("Invalid embedded DTB"); + + checked = true; + } + + return embedded_secure_dtb; +} +#else +void *get_embedded_dt(void) +{ + return NULL; +} +#endif /*CFG_EMBED_DTB*/ + +#ifdef _CFG_USE_DTB_OVERLAY +static int add_dt_overlay_fragment(struct dt_descriptor *dt, int ioffs) +{ + char frag[32] = { }; + int offs = 0; + int ret = 0; + + snprintf(frag, sizeof(frag), "fragment@%d", dt->frag_id); + offs = fdt_add_subnode(dt->blob, ioffs, frag); + if (offs < 0) + return offs; + + dt->frag_id += 1; + + ret = fdt_setprop_string(dt->blob, offs, "target-path", "/"); + if (ret < 0) + return ret; + + return fdt_add_subnode(dt->blob, offs, "__overlay__"); +} + +static int init_dt_overlay(struct dt_descriptor *dt, int __maybe_unused dt_size) +{ + int fragment = 0; + + if (IS_ENABLED(CFG_EXTERNAL_DTB_OVERLAY)) { + if (!fdt_check_header(dt->blob)) { + fdt_for_each_subnode(fragment, dt->blob, 0) + dt->frag_id += 1; + return 0; + } + } + + return fdt_create_empty_tree(dt->blob, dt_size); +} +#else +static int add_dt_overlay_fragment(struct dt_descriptor *dt __unused, int offs) +{ + return offs; +} + +static int init_dt_overlay(struct dt_descriptor *dt __unused, + int dt_size __unused) +{ + return 0; +} +#endif /* _CFG_USE_DTB_OVERLAY */ + +struct dt_descriptor *get_external_dt_desc(void) +{ + if (!IS_ENABLED(CFG_EXTERNAL_DT)) + return NULL; + + return &external_dt; +} + +void init_external_dt(unsigned long phys_dt) +{ + struct dt_descriptor *dt = &external_dt; + void *fdt = NULL; + int ret = 0; + + if (!IS_ENABLED(CFG_EXTERNAL_DT)) + return; + + if (!phys_dt) { + /* + * No need to panic as we're not using the DT in OP-TEE + * yet, we're only adding some nodes for normal world use. + * This makes the switch to using DT easier as we can boot + * a newer OP-TEE with older boot loaders. Once we start to + * initialize devices based on DT we'll likely panic + * instead of returning here. + */ + IMSG("No non-secure external DT"); + return; + } + + fdt = core_mmu_add_mapping(MEM_AREA_EXT_DT, phys_dt, CFG_DTB_MAX_SIZE); + if (!fdt) + panic("Failed to map external DTB"); + + dt->blob = fdt; + + ret = init_dt_overlay(dt, CFG_DTB_MAX_SIZE); + if (ret < 0) { + EMSG("Device Tree Overlay init fail @ %#lx: error %d", phys_dt, + ret); + panic(); + } + + ret = fdt_open_into(fdt, fdt, CFG_DTB_MAX_SIZE); + if (ret < 0) { + EMSG("Invalid Device Tree at %#lx: error %d", phys_dt, ret); + panic(); + } + + IMSG("Non-secure external DT found"); +} + +void *get_external_dt(void) +{ + if (!IS_ENABLED(CFG_EXTERNAL_DT)) + return NULL; + + assert(cpu_mmu_enabled()); + return external_dt.blob; +} + +static TEE_Result release_external_dt(void) +{ + int ret = 0; + + if (!IS_ENABLED(CFG_EXTERNAL_DT)) + return TEE_SUCCESS; + + if (!external_dt.blob) + return TEE_SUCCESS; + + ret = fdt_pack(external_dt.blob); + if (ret < 0) { + EMSG("Failed to pack Device Tree at 0x%" PRIxPA ": error %d", + virt_to_phys(external_dt.blob), ret); + panic(); + } + + if (core_mmu_remove_mapping(MEM_AREA_EXT_DT, external_dt.blob, + CFG_DTB_MAX_SIZE)) + panic("Failed to remove temporary Device Tree mapping"); + + /* External DTB no more reached, reset pointer to invalid */ + external_dt.blob = NULL; + + return TEE_SUCCESS; +} + +boot_final(release_external_dt); + +int add_dt_path_subnode(struct dt_descriptor *dt, const char *path, + const char *subnode) +{ + int offs = 0; + + offs = fdt_path_offset(dt->blob, path); + if (offs < 0) + return offs; + offs = add_dt_overlay_fragment(dt, offs); + if (offs < 0) + return offs; + return fdt_add_subnode(dt->blob, offs, subnode); +} + +static void set_dt_val(void *data, uint32_t cell_size, uint64_t val) +{ + if (cell_size == 1) { + fdt32_t v = cpu_to_fdt32((uint32_t)val); + + memcpy(data, &v, sizeof(v)); + } else { + fdt64_t v = cpu_to_fdt64(val); + + memcpy(data, &v, sizeof(v)); + } +} + +int add_res_mem_dt_node(struct dt_descriptor *dt, const char *name, + paddr_t pa, size_t size) +{ + int offs = 0; + int ret = 0; + int addr_size = -1; + int len_size = -1; + bool found = true; + char subnode_name[80] = { }; + + offs = fdt_path_offset(dt->blob, "/reserved-memory"); + + if (offs < 0) { + found = false; + offs = 0; + } + + if (IS_ENABLED2(_CFG_USE_DTB_OVERLAY)) { + len_size = sizeof(paddr_t) / sizeof(uint32_t); + addr_size = sizeof(paddr_t) / sizeof(uint32_t); + } else { + len_size = fdt_size_cells(dt->blob, offs); + if (len_size < 0) + return len_size; + addr_size = fdt_address_cells(dt->blob, offs); + if (addr_size < 0) + return addr_size; + } + + if (!found) { + offs = add_dt_path_subnode(dt, "/", "reserved-memory"); + if (offs < 0) + return offs; + ret = fdt_setprop_cell(dt->blob, offs, "#address-cells", + addr_size); + if (ret < 0) + return ret; + ret = fdt_setprop_cell(dt->blob, offs, "#size-cells", len_size); + if (ret < 0) + return ret; + ret = fdt_setprop(dt->blob, offs, "ranges", NULL, 0); + if (ret < 0) + return ret; + } + + ret = snprintf(subnode_name, sizeof(subnode_name), + "%s@%" PRIxPA, name, pa); + if (ret < 0 || ret >= (int)sizeof(subnode_name)) + DMSG("truncated node \"%s@%" PRIxPA"\"", name, pa); + offs = fdt_add_subnode(dt->blob, offs, subnode_name); + if (offs >= 0) { + uint32_t data[FDT_MAX_NCELLS * 2] = { }; + + set_dt_val(data, addr_size, pa); + set_dt_val(data + addr_size, len_size, size); + ret = fdt_setprop(dt->blob, offs, "reg", data, + sizeof(uint32_t) * (addr_size + len_size)); + if (ret < 0) + return ret; + ret = fdt_setprop(dt->blob, offs, "no-map", NULL, 0); + if (ret < 0) + return ret; + } else { + return offs; + } + return 0; +} diff --git a/optee_os/core/kernel/dt_driver.c b/optee_os/core/kernel/dt_driver.c new file mode 100644 index 0000000..256aefa --- /dev/null +++ b/optee_os/core/kernel/dt_driver.c @@ -0,0 +1,880 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Linaro Limited + * Copyright (c) 2021, Bootlin + * Copyright (c) 2021, Linaro Limited + * Copyright (c) 2021, STMicroelectronics + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * struct dt_driver_probe - Node instance in secure FDT to probe a driver for + * + * @link: List hook + * @nodeoffset: Node offset of device referenced in the FDT + * @type: One of DT_DRIVER_* or DT_DRIVER_NOTYPE. + * @deferrals: Driver probe deferrals count + * @dt_drv: Matching driver to probe if found or NULL + * @dm: Matching reference if applicable or NULL + */ +struct dt_driver_probe { + int nodeoffset; + enum dt_driver_type type; + unsigned int deferrals; + const struct dt_driver *dt_drv; + const struct dt_device_match *dm; + TAILQ_ENTRY(dt_driver_probe) link; +}; + +/* + * struct dt_driver_provider - DT related info on probed device + * + * Saves information on the probed device so that device + * drivers can get resources from DT phandle and related arguments. + * + * @nodeoffset: Node offset of device referenced in the FDT + * @type: One of DT_DRIVER_* or DT_DRIVER_NOTYPE. + * @provider_cells: Cells count in the FDT used by the driver's references + * @get_of_device: Function to get driver's device ref from phandle data + * @priv_data: Driver private data passed as @get_of_device argument + * @link: Reference in DT driver providers list + */ +struct dt_driver_provider { + int nodeoffset; + enum dt_driver_type type; + unsigned int provider_cells; + uint32_t phandle; + get_of_device_func get_of_device; + void *priv_data; + SLIST_ENTRY(dt_driver_provider) link; +}; + +/* + * Device driver providers are able to provide a driver specific instance + * related to device phandle arguments found in the secure embedded FDT. + */ +static SLIST_HEAD(, dt_driver_provider) dt_driver_provider_list = + SLIST_HEAD_INITIALIZER(dt_driver_provider_list); + +/* FDT nodes for which a matching driver is to be probed */ +static TAILQ_HEAD(dt_driver_probe_head, dt_driver_probe) dt_driver_probe_list = + TAILQ_HEAD_INITIALIZER(dt_driver_probe_list); + +/* FDT nodes for which a matching driver has been successfully probed */ +static TAILQ_HEAD(, dt_driver_probe) dt_driver_ready_list = + TAILQ_HEAD_INITIALIZER(dt_driver_ready_list); + +/* List of the nodes for which a compatible driver but reported a failure */ +static TAILQ_HEAD(, dt_driver_probe) dt_driver_failed_list = + TAILQ_HEAD_INITIALIZER(dt_driver_failed_list); + +/* Flag enabled when a new node (possibly typed) is added in the probe list */ +static bool added_node; + +/* Resolve drivers dependencies on core crypto layer */ +static bool tee_crypt_is_ready; + +void dt_driver_crypt_init_complete(void) +{ + assert(!tee_crypt_is_ready); + tee_crypt_is_ready = true; +} + +TEE_Result dt_driver_get_crypto(void) +{ + if (tee_crypt_is_ready) + return TEE_SUCCESS; + else + return TEE_ERROR_DEFER_DRIVER_INIT; +} + +static void assert_type_is_valid(enum dt_driver_type type) +{ + switch (type) { + case DT_DRIVER_NOTYPE: + case DT_DRIVER_CLK: + case DT_DRIVER_RSTCTRL: + case DT_DRIVER_UART: + case DT_DRIVER_GPIO: + case DT_DRIVER_I2C: + case DT_DRIVER_PINCTRL: + case DT_DRIVER_INTERRUPT: + case DT_DRIVER_REGULATOR: + return; + default: + assert(0); + } +} + +/* + * Driver provider registering API functions + */ + +TEE_Result dt_driver_register_provider(const void *fdt, int nodeoffset, + get_of_device_func get_of_device, + void *priv, enum dt_driver_type type) +{ + struct dt_driver_provider *prv = NULL; + int provider_cells = 0; + uint32_t phandle = 0; + + assert_type_is_valid(type); + + provider_cells = fdt_get_dt_driver_cells(fdt, nodeoffset, type); + if (provider_cells < 0) { + DMSG("Failed to find provider cells: %d", provider_cells); + return TEE_ERROR_GENERIC; + } + + phandle = fdt_get_phandle(fdt, nodeoffset); + if (phandle == (uint32_t)-1) { + DMSG("Failed to find provide phandle"); + return TEE_ERROR_GENERIC; + } + + prv = calloc(1, sizeof(*prv)); + if (!prv) + return TEE_ERROR_OUT_OF_MEMORY; + + prv->nodeoffset = nodeoffset; + prv->type = type; + prv->provider_cells = provider_cells; + prv->phandle = phandle; + prv->get_of_device = get_of_device; + prv->priv_data = priv; + + SLIST_INSERT_HEAD(&dt_driver_provider_list, prv, link); + + return TEE_SUCCESS; +} + +static bool dt_driver_use_parent_controller(enum dt_driver_type type) +{ + switch (type) { + case DT_DRIVER_PINCTRL: + return true; + default: + return false; + } +} + +/* + * Helper functions for dt_drivers querying driver provider information + */ + +int fdt_get_dt_driver_cells(const void *fdt, int nodeoffset, + enum dt_driver_type type) +{ + const char *cells_name = NULL; + const fdt32_t *c = NULL; + int len = 0; + + if (dt_driver_use_parent_controller(type)) + return 0; + + switch (type) { + case DT_DRIVER_CLK: + cells_name = "#clock-cells"; + break; + case DT_DRIVER_INTERRUPT: + cells_name = "#interrupt-cells"; + break; + case DT_DRIVER_RSTCTRL: + cells_name = "#reset-cells"; + break; + case DT_DRIVER_GPIO: + cells_name = "#gpio-cells"; + break; + case DT_DRIVER_I2C: + case DT_DRIVER_REGULATOR: + return 0; + default: + panic(); + } + + c = fdt_getprop(fdt, nodeoffset, cells_name, &len); + if (!c) + return len; + + if (len != sizeof(*c)) + return -FDT_ERR_BADNCELLS; + + return fdt32_to_cpu(*c); +} + +unsigned int dt_driver_provider_cells(struct dt_driver_provider *prv) +{ + return prv->provider_cells; +} + +void *dt_driver_provider_priv_data(struct dt_driver_provider *prv) +{ + return prv->priv_data; +} + +struct dt_driver_provider * +dt_driver_get_provider_by_node(int nodeoffset, enum dt_driver_type type) +{ + struct dt_driver_provider *prv = NULL; + + SLIST_FOREACH(prv, &dt_driver_provider_list, link) + if (prv->nodeoffset == nodeoffset && prv->type == type) + return prv; + + return NULL; +} + +struct dt_driver_provider * +dt_driver_get_provider_by_phandle(uint32_t phandle, enum dt_driver_type type) +{ + struct dt_driver_provider *prv = NULL; + + SLIST_FOREACH(prv, &dt_driver_provider_list, link) + if (prv->phandle == phandle && prv->type == type) + return prv; + + return NULL; +} + +static TEE_Result device_from_provider_prop(struct dt_driver_provider *prv, + const void *fdt, int phandle_node, + const uint32_t *prop, void **device) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct dt_pargs *pargs = NULL; + unsigned int n = 0; + + pargs = calloc(1, prv->provider_cells * sizeof(uint32_t *) + + sizeof(*pargs)); + if (!pargs) + return TEE_ERROR_OUT_OF_MEMORY; + + pargs->fdt = fdt; + pargs->phandle_node = phandle_node; + pargs->args_count = prv->provider_cells; + for (n = 0; n < prv->provider_cells; n++) + pargs->args[n] = fdt32_to_cpu(prop[n + 1]); + + res = prv->get_of_device(pargs, prv->priv_data, device); + + free(pargs); + + return res; +} + +TEE_Result dt_driver_device_from_parent(const void *fdt, int nodeoffset, + enum dt_driver_type type, void **device) +{ + int parent = -1; + struct dt_driver_provider *prv = NULL; + + assert(fdt == get_secure_dt()); + + parent = fdt_parent_offset(fdt, nodeoffset); + if (parent < 0) + return TEE_ERROR_GENERIC; + + prv = dt_driver_get_provider_by_node(parent, type); + if (!prv) { + /* No provider registered yet */ + return TEE_ERROR_DEFER_DRIVER_INIT; + } + + return device_from_provider_prop(prv, fdt, nodeoffset, NULL, device); +} + +TEE_Result dt_driver_device_from_node_idx_prop_phandle(const char *prop_name, + const void *fdt, + int nodeoffs, + unsigned int prop_index, + enum dt_driver_type type, + uint32_t phandle, + void **device) +{ + int len = 0; + const uint32_t *prop = NULL; + int phandle_node_unused = -1; + struct dt_driver_provider *prv = NULL; + + prop = fdt_getprop(fdt, nodeoffs, prop_name, &len); + if (!prop) { + if (len != -FDT_ERR_NOTFOUND) { + DMSG("Corrupted node %s", prop_name); + return TEE_ERROR_GENERIC; + } else { + DMSG("Property %s missing in node %s", prop_name, + fdt_get_name(fdt, nodeoffs, NULL)); + return TEE_ERROR_ITEM_NOT_FOUND; + } + } + + prv = dt_driver_get_provider_by_phandle(phandle, type); + if (!prv) + return TEE_ERROR_DEFER_DRIVER_INIT; + + prop_index *= dt_driver_provider_cells(prv); + if ((prop_index + 1) * sizeof(*prop) > (size_t)len) + return TEE_ERROR_ITEM_NOT_FOUND; + + return device_from_provider_prop(prv, fdt, phandle_node_unused, + prop + prop_index, device); +} + +TEE_Result dt_driver_device_from_node_idx_prop(const char *prop_name, + const void *fdt, int nodeoffset, + unsigned int prop_idx, + enum dt_driver_type type, + void **device) +{ + int len = 0; + int idx = 0; + int idx32 = 0; + int prv_cells = 0; + int phandle_node = -1; + uint32_t phandle = 0; + const uint32_t *prop = NULL; + struct dt_driver_provider *prv = NULL; + + prop = fdt_getprop(fdt, nodeoffset, prop_name, &len); + if (!prop) { + DMSG("Property %s missing in node %s", prop_name, + fdt_get_name(fdt, nodeoffset, NULL)); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + while (idx < len) { + idx32 = idx / sizeof(uint32_t); + phandle = fdt32_to_cpu(prop[idx32]); + if (!phandle) { + if (!prop_idx) + break; + idx += sizeof(phandle); + prop_idx--; + continue; + } + + /* + * In some cases, pinctrl, i2c, nvmem, etc, the consumer phandle + * points directly to a subnode of the parent. In such cases, + * the provider does not have any "-cells" property and + * potentially no "phandle" property. + */ + if (dt_driver_use_parent_controller(type)) { + phandle_node = fdt_node_offset_by_phandle(fdt, phandle); + if (phandle_node < 0) + return TEE_ERROR_GENERIC; + + nodeoffset = fdt_parent_offset(fdt, phandle_node); + if (nodeoffset < 0) + return TEE_ERROR_GENERIC; + + prv = dt_driver_get_provider_by_node(nodeoffset, type); + if (!prv) + return TEE_ERROR_DEFER_DRIVER_INIT; + } else { + prv = dt_driver_get_provider_by_phandle(phandle, type); + if (!prv) + return TEE_ERROR_DEFER_DRIVER_INIT; + } + + prv_cells = dt_driver_provider_cells(prv); + if (prop_idx) { + prop_idx--; + idx += sizeof(phandle) + prv_cells * sizeof(uint32_t); + continue; + } + + return device_from_provider_prop(prv, fdt, phandle_node, + prop + idx32, device); + } + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +static void __maybe_unused print_probe_list(const void *fdt __maybe_unused) +{ + struct dt_driver_probe *elt = NULL; + unsigned int count = 0; + + TAILQ_FOREACH(elt, &dt_driver_probe_list, link) + count++; + + DMSG("Probe list: %u elements", count); + TAILQ_FOREACH(elt, &dt_driver_probe_list, link) + DMSG("|- Driver %s probes on node %s", + elt->dt_drv->name, + fdt_get_name(fdt, elt->nodeoffset, NULL)); + + DMSG("`- Probe list end"); + + count = 0; + TAILQ_FOREACH(elt, &dt_driver_failed_list, link) + count++; + + DMSG("Failed list: %u elements", count); + TAILQ_FOREACH(elt, &dt_driver_failed_list, link) + EMSG("|- Driver %s on node %s failed", elt->dt_drv->name, + fdt_get_name(fdt, elt->nodeoffset, NULL)); + + DMSG("`- Failed list end"); +} + +/* + * Probe element: push to ready list if succeeds, push to probe list if probe + * if deferred, panic with an error trace otherwise. + */ +static TEE_Result probe_driver_node(const void *fdt, + struct dt_driver_probe *elt) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const char __maybe_unused *drv_name = NULL; + const char __maybe_unused *node_name = NULL; + + node_name = fdt_get_name(fdt, elt->nodeoffset, NULL); + drv_name = elt->dt_drv->name; + + if (!elt->dt_drv->probe) { + DMSG("No probe operator for driver %s, skipped", drv_name); + return TEE_SUCCESS; + } + + FMSG("Probing %s on node %s", drv_name, node_name); + + res = elt->dt_drv->probe(fdt, elt->nodeoffset, elt->dm->compat_data); + switch (res) { + case TEE_SUCCESS: + TAILQ_INSERT_HEAD(&dt_driver_ready_list, elt, link); + + DMSG("element: %s on node %s initialized", drv_name, node_name); + break; + case TEE_ERROR_DEFER_DRIVER_INIT: + elt->deferrals++; + TAILQ_INSERT_TAIL(&dt_driver_probe_list, elt, link); + + DMSG("element: %s on node %s deferred %u time(s)", drv_name, + node_name, elt->deferrals); + break; + case TEE_ERROR_NODE_DISABLED: + DMSG("element: %s on node %s is disabled", drv_name, node_name); + break; + default: + TAILQ_INSERT_HEAD(&dt_driver_failed_list, elt, link); + + EMSG("Failed to probe %s on node %s: %#"PRIx32, + drv_name, node_name, res); + break; + } + + return res; +} + +static TEE_Result alloc_elt_and_probe(const void *fdt, int node, + const struct dt_driver *dt_drv, + const struct dt_device_match *dm) +{ + struct dt_driver_probe *elt = NULL; + + /* Will be freed when lists are released */ + elt = calloc(1, sizeof(*elt)); + if (!elt) + return TEE_ERROR_OUT_OF_MEMORY; + + elt->nodeoffset = node; + elt->dt_drv = dt_drv; + elt->dm = dm; + elt->type = dt_drv->type; + + return probe_driver_node(fdt, elt); +} + +/* Lookup a compatible driver, possibly of a specific @type, for the FDT node */ +static TEE_Result probe_device_by_compat(const void *fdt, int node, + const char *compat, + enum dt_driver_type type) +{ + const struct dt_driver *drv = NULL; + const struct dt_device_match *dm = NULL; + + for_each_dt_driver(drv) { + if (drv->type != type) + continue; + + for (dm = drv->match_table; dm && dm->compatible; dm++) + if (strcmp(dm->compatible, compat) == 0) + return alloc_elt_and_probe(fdt, node, drv, dm); + } + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +/* + * Lookup the best matching compatible driver, possibly of a specific @type, + * for the FDT node. + */ +TEE_Result dt_driver_probe_device_by_node(const void *fdt, int nodeoffset, + enum dt_driver_type type) +{ + int idx = 0; + int len = 0; + int count = 0; + const char *compat = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + assert_type_is_valid(type); + + count = fdt_stringlist_count(fdt, nodeoffset, "compatible"); + if (count < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + for (idx = 0; idx < count; idx++) { + compat = fdt_stringlist_get(fdt, nodeoffset, "compatible", + idx, &len); + if (!compat) + return TEE_ERROR_GENERIC; + + res = probe_device_by_compat(fdt, nodeoffset, compat, type); + + if (res != TEE_ERROR_ITEM_NOT_FOUND) + return res; + } + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +static TEE_Result process_probe_list(const void *fdt) +{ + struct dt_driver_probe *elt = NULL; + struct dt_driver_probe *prev = NULL; + static unsigned int __maybe_unused loop_count; + static unsigned int __maybe_unused deferral_loop_count; + bool __maybe_unused one_deferred = false; + bool one_probed_ok = false; + + do { + loop_count++; + FMSG("Probe loop %u after %u for deferral(s)", loop_count, + deferral_loop_count); + + /* Hack here for TRACE_DEBUG messages on probe list elements */ + if (TRACE_LEVEL >= TRACE_FLOW) + print_probe_list(fdt); + + if (TAILQ_EMPTY(&dt_driver_probe_list)) + return TEE_SUCCESS; + + /* + * Probe from current end to top. Deferred probed node are + * pushed back after current tail for the next probe round. + * Reset probe result flags and see status after probe round. + */ + one_deferred = false; + one_probed_ok = false; + added_node = false; + + TAILQ_FOREACH_REVERSE_SAFE(elt, &dt_driver_probe_list, + dt_driver_probe_head, link, prev) { + TAILQ_REMOVE(&dt_driver_probe_list, elt, link); + + switch (probe_driver_node(fdt, elt)) { + case TEE_SUCCESS: + one_probed_ok = true; + break; + case TEE_ERROR_DEFER_DRIVER_INIT: + one_deferred = true; + break; + default: + break; + } + } + + if (one_deferred) + deferral_loop_count++; + + } while (added_node || one_probed_ok); + + DMSG("Unresolved dependencies after %u rounds, %u deferred", + loop_count, deferral_loop_count); + + if (one_deferred) + return TEE_ERROR_DEFER_DRIVER_INIT; + else + return TEE_ERROR_GENERIC; +} + +static int driver_probe_compare(struct dt_driver_probe *candidate, + struct dt_driver_probe *elt) +{ + if (candidate->nodeoffset != elt->nodeoffset || + candidate->type != elt->type) + return 1; + + assert(elt->dt_drv == candidate->dt_drv); + return 0; +} + +/* + * Return TEE_SUCCESS if compatible found + * TEE_ERROR_OUT_OF_MEMORY if heap is exhausted + */ +static TEE_Result add_node_to_probe(const void *fdt, int node, + const struct dt_driver *dt_drv, + const struct dt_device_match *dm) +{ + const char __maybe_unused *node_name = fdt_get_name(fdt, node, NULL); + const char __maybe_unused *drv_name = dt_drv->name; + struct dt_driver_probe *elt = NULL; + struct dt_driver_probe elt_new = { + .dm = dm, + .dt_drv = dt_drv, + .nodeoffset = node, + .type = dt_drv->type, + }; + + /* If node/type found in probe list or ready list, nothing to do */ + TAILQ_FOREACH(elt, &dt_driver_probe_list, link) + if (!driver_probe_compare(&elt_new, elt)) + return TEE_SUCCESS; + + TAILQ_FOREACH(elt, &dt_driver_ready_list, link) + if (!driver_probe_compare(&elt_new, elt)) + return TEE_SUCCESS; + + elt = malloc(sizeof(*elt)); + if (!elt) + return TEE_ERROR_OUT_OF_MEMORY; + + DMSG("element: %s on node %s", node_name, drv_name); + + memcpy(elt, &elt_new, sizeof(*elt)); + + added_node = true; + + TAILQ_INSERT_TAIL(&dt_driver_probe_list, elt, link); + + /* Hack here for TRACE_DEBUG messages on current probe list elements */ + if (TRACE_LEVEL >= TRACE_FLOW) + print_probe_list(fdt); + + return TEE_SUCCESS; +} + +/* + * Add a node to the probe list if a dt_driver matches target compatible. + * + * If @type is DT_DRIVER_ANY, probe list can hold only 1 driver to probe for + * the node. A node may probe several drivers if have a unique driver type. + * + * Return TEE_SUCCESS if compatible found + * TEE_ERROR_ITEM_NOT_FOUND if no matching driver + * TEE_ERROR_OUT_OF_MEMORY if heap is exhausted + */ +static TEE_Result add_probe_node_by_compat(const void *fdt, int node, + const char *compat) +{ + TEE_Result res = TEE_ERROR_ITEM_NOT_FOUND; + const struct dt_driver *dt_drv = NULL; + const struct dt_device_match *dm = NULL; + uint32_t found_types = 0; + + for_each_dt_driver(dt_drv) { + for (dm = dt_drv->match_table; dm && dm->compatible; dm++) { + if (strcmp(dm->compatible, compat) == 0) { + assert(dt_drv->type < 32); + + res = add_node_to_probe(fdt, node, dt_drv, dm); + if (res) + return res; + + if (found_types & BIT(dt_drv->type)) { + EMSG("Driver %s multi hit on type %u", + dt_drv->name, dt_drv->type); + panic(); + } + found_types |= BIT(dt_drv->type); + + /* Matching found for this driver, try next */ + break; + } + } + } + + return res; +} + +/* + * Add the node to the probe list if matching compatible drivers are found. + * Follow node's compatible property list ordering to find matching driver. + */ +TEE_Result dt_driver_maybe_add_probe_node(const void *fdt, int node) +{ + int idx = 0; + int len = 0; + int count = 0; + const char *compat = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + if (fdt_get_status(fdt, node) == DT_STATUS_DISABLED) + return TEE_SUCCESS; + + count = fdt_stringlist_count(fdt, node, "compatible"); + if (count < 0) + return TEE_SUCCESS; + + for (idx = 0; idx < count; idx++) { + compat = fdt_stringlist_get(fdt, node, "compatible", idx, &len); + assert(compat && len > 0); + + res = add_probe_node_by_compat(fdt, node, compat); + + /* Stop lookup if something was found */ + if (res != TEE_ERROR_ITEM_NOT_FOUND) + return res; + } + + return TEE_SUCCESS; +} + +static void parse_node(const void *fdt, int node) +{ + TEE_Result __maybe_unused res = TEE_ERROR_GENERIC; + int subnode = 0; + + fdt_for_each_subnode(subnode, fdt, node) { + res = dt_driver_maybe_add_probe_node(fdt, subnode); + if (res) { + EMSG("Failed on node %s with %#"PRIx32, + fdt_get_name(fdt, subnode, NULL), res); + panic(); + } + + /* + * Rescursively parse the FDT, skipping disabled nodes. + * FDT is expected reliable and core shall have sufficient + * stack depth to possibly parse all DT nodes. + */ + if (IS_ENABLED(CFG_DRIVERS_DT_RECURSIVE_PROBE)) { + if (fdt_get_status(fdt, subnode) == DT_STATUS_DISABLED) + continue; + + parse_node(fdt, subnode); + } + } +} + +/* + * Parse FDT for nodes and save in probe list the node for which a dt_driver + * matches node's compatible property. + */ +static TEE_Result probe_dt_drivers_early(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const void *fdt = NULL; + + fdt = get_secure_dt(); + if (!fdt) + return TEE_SUCCESS; + + parse_node(fdt, fdt_path_offset(fdt, "/")); + + res = process_probe_list(fdt); + if (res == TEE_ERROR_DEFER_DRIVER_INIT) { + DMSG("Deferred drivers probing"); + print_probe_list(fdt); + res = TEE_SUCCESS; + } + + return res; +} + +static TEE_Result probe_dt_drivers(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const void *fdt = NULL; + + fdt = get_secure_dt(); + if (!fdt) + return TEE_SUCCESS; + + res = process_probe_list(fdt); + if (res || !TAILQ_EMPTY(&dt_driver_failed_list)) { + EMSG("Probe sequence result: %#"PRIx32, res); + print_probe_list(fdt); + panic(); + } + + return TEE_SUCCESS; +} + +early_init_late(probe_dt_drivers_early); +driver_init(probe_dt_drivers); + +static TEE_Result release_probe_lists(void) +{ + struct dt_driver_probe *elt = NULL; + struct dt_driver_probe *next = NULL; + struct dt_driver_provider *prov = NULL; + struct dt_driver_provider *next_prov = NULL; + const void *fdt = NULL; + + fdt = get_secure_dt(); + if (!fdt) + return TEE_SUCCESS; + + assert(fdt && TAILQ_EMPTY(&dt_driver_probe_list)); + + TAILQ_FOREACH_SAFE(elt, &dt_driver_ready_list, link, next) + free(elt); + + TAILQ_FOREACH_SAFE(elt, &dt_driver_failed_list, link, next) + free(elt); + + SLIST_FOREACH_SAFE(prov, &dt_driver_provider_list, link, next_prov) + free(prov); + + return TEE_SUCCESS; +} + +release_init_resource(release_probe_lists); + +/* + * Simple bus support: handy to parse subnodes + */ +static TEE_Result simple_bus_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int subnode = 0; + + fdt_for_each_subnode(subnode, fdt, node) { + res = dt_driver_maybe_add_probe_node(fdt, subnode); + if (res) { + EMSG("Failed on node %s with %#"PRIx32, + fdt_get_name(fdt, subnode, NULL), res); + panic(); + } + } + + return TEE_SUCCESS; +} + +static const struct dt_device_match simple_bus_match_table[] = { + { .compatible = "simple-bus" }, + { } +}; + +DEFINE_DT_DRIVER(simple_bus_dt_driver) = { + .name = "simple-bus", + .match_table = simple_bus_match_table, + .probe = simple_bus_probe, +}; diff --git a/optee_os/core/kernel/early_ta.c b/optee_os/core/kernel/early_ta.c new file mode 100644 index 0000000..94f6c75 --- /dev/null +++ b/optee_os/core/kernel/early_ta.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const struct embedded_ts *find_early_ta(const TEE_UUID *uuid) +{ + const struct embedded_ts *ta = NULL; + + for_each_early_ta(ta) + if (!memcmp(&ta->uuid, uuid, sizeof(*uuid))) + return ta; + + return NULL; +} + +static TEE_Result early_ta_open(const TEE_UUID *uuid, + struct ts_store_handle **h) +{ + return emb_ts_open(uuid, h, find_early_ta); +} + +REGISTER_TA_STORE(2) = { + .description = "early TA", + .open = early_ta_open, + .get_size = emb_ts_get_size, + .get_tag = emb_ts_get_tag, + .read = emb_ts_read, + .close = emb_ts_close, +}; + +static TEE_Result early_ta_init(void) +{ + const struct embedded_ts *ta = NULL; + char __maybe_unused msg[60] = { '\0', }; + + for_each_early_ta(ta) { + if (ta->uncompressed_size) + snprintf(msg, sizeof(msg), + " (compressed, uncompressed %u)", + ta->uncompressed_size); + else + msg[0] = '\0'; + DMSG("Early TA %pUl size %u%s", (void *)&ta->uuid, ta->size, + msg); + } + + return TEE_SUCCESS; +} + +service_init(early_ta_init); diff --git a/optee_os/core/kernel/embedded_ts.c b/optee_os/core/kernel/embedded_ts.c new file mode 100644 index 0000000..9392260 --- /dev/null +++ b/optee_os/core/kernel/embedded_ts.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ts_store_handle { + const struct embedded_ts *ts; + size_t offs; + z_stream strm; +}; + +static void *zalloc(void *opaque __unused, unsigned int items, + unsigned int size) +{ + return mempool_alloc(mempool_default, items * size); +} + +static void zfree(void *opaque __unused, void *address) +{ + mempool_free(mempool_default, address); +} + +static bool decompression_init(z_stream *strm, + const struct embedded_ts *ts) +{ + int st = Z_OK; + + strm->next_in = ts->ts; + strm->avail_in = ts->size; + strm->zalloc = zalloc; + strm->zfree = zfree; + st = inflateInit(strm); + if (st != Z_OK) { + EMSG("Decompression initialization error (%d)", st); + return false; + } + + return true; +} + +TEE_Result emb_ts_open(const TEE_UUID *uuid, + struct ts_store_handle **h, + const struct embedded_ts* + (*find_ts) (const TEE_UUID *uuid)) +{ + struct ts_store_handle *handle = NULL; + const struct embedded_ts *ts = NULL; + + ts = find_ts(uuid); + if (!ts) + return TEE_ERROR_ITEM_NOT_FOUND; + + handle = calloc(1, sizeof(*handle)); + if (!handle) + return TEE_ERROR_OUT_OF_MEMORY; + + if (ts->uncompressed_size) { + if (!decompression_init(&handle->strm, ts)) { + free(handle); + return TEE_ERROR_BAD_FORMAT; + } + } + handle->ts = ts; + *h = handle; + + return TEE_SUCCESS; +} + +TEE_Result emb_ts_get_size(const struct ts_store_handle *h, size_t *size) +{ + const struct embedded_ts *ts = h->ts; + + if (ts->uncompressed_size) + *size = ts->uncompressed_size; + else + *size = ts->size; + + return TEE_SUCCESS; +} + +TEE_Result emb_ts_get_tag(const struct ts_store_handle *h, + uint8_t *tag, unsigned int *tag_len) +{ + TEE_Result res = TEE_SUCCESS; + void *ctx = NULL; + + if (!tag || *tag_len < TEE_SHA256_HASH_SIZE) { + *tag_len = TEE_SHA256_HASH_SIZE; + return TEE_ERROR_SHORT_BUFFER; + } + *tag_len = TEE_SHA256_HASH_SIZE; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SHA256); + if (res) + return res; + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, h->ts->ts, h->ts->size); + if (res) + goto out; + res = crypto_hash_final(ctx, tag, *tag_len); +out: + crypto_hash_free_ctx(ctx); + return res; +} + +static TEE_Result read_uncompressed(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *src = (uint8_t *)h->ts->ts + h->offs; + size_t next_offs = 0; + + if (ADD_OVERFLOW(h->offs, len, &next_offs) || + next_offs > h->ts->size) + return TEE_ERROR_BAD_PARAMETERS; + if (data_core) + memcpy(data_core, src, len); + if (data_user) + res = copy_to_user(data_user, src, len); + if (!res) + h->offs = next_offs; + + return res; +} + +static TEE_Result read_compressed(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len) +{ + TEE_Result res = TEE_SUCCESS; + z_stream *strm = &h->strm; + size_t total = 0; + uint8_t *bb = NULL; + size_t bb_len = 0; + size_t out = 0; + int st = Z_OK; + + /* Inflate into a 1kB bounce buffer */ + bb_len = MIN(len, 1024U); + bb = bb_alloc(bb_len); + if (!bb) { + EMSG("Out of memory"); + return TEE_ERROR_OUT_OF_MEMORY; + } + + strm->avail_out = bb_len; + strm->next_out = bb; + + /* + * Loop until we get as many bytes as requested, or an error occurs. + * inflate() returns: + * - Z_OK when progress was made, but neither the end of the input + * stream nor the end of the output buffer were met. + * - Z_STREAM_END when the end of the intput stream was reached. + * - Z_BUF_ERROR when there is still input to process but the output + * buffer is full (not a "hard" error, decompression can proceeed + * later). + */ + do { + out = strm->total_out; + st = inflate(strm, Z_SYNC_FLUSH); + out = strm->total_out - out; + FMSG("%zu bytes", out); + if (data_core) + memcpy((uint8_t *)data_core + total, bb, out); + if (data_user) { + res = copy_to_user((uint8_t *)data_user + total, bb, + out); + if (res) + goto out; + } + total += out; + /* + * Reset the pointer since we've just copied out the last + * data. + */ + strm->next_out = bb; + strm->avail_out = MIN(len - total, bb_len); + } while ((st == Z_OK || st == Z_BUF_ERROR) && (total != len)); + + if (st != Z_OK && st != Z_STREAM_END) { + EMSG("Decompression error (%d)", st); + res = TEE_ERROR_GENERIC; + goto out; + } + res = TEE_SUCCESS; +out: + bb_free(bb, bb_len); + + return res; +} + +TEE_Result emb_ts_read(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len) +{ + if (h->ts->uncompressed_size) + return read_compressed(h, data_core, data_user, len); + else + return read_uncompressed(h, data_core, data_user, len); +} + +void emb_ts_close(struct ts_store_handle *h) +{ + if (h->ts->uncompressed_size) + inflateEnd(&h->strm); + free(h); +} + diff --git a/optee_os/core/kernel/handle.c b/optee_os/core/kernel/handle.c new file mode 100644 index 0000000..96f01cb --- /dev/null +++ b/optee_os/core/kernel/handle.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ +#include +#include +#include + +/* + * Define the initial capacity of the database. It should be a low number + * multiple of 2 since some databases a likely to only use a few handles. + * Since the algorithm is to doubles up when growing it shouldn't cause a + * noticable overhead on large databases. + */ +#define HANDLE_DB_INITIAL_MAX_PTRS 4 + +void handle_db_destroy(struct handle_db *db, void (*ptr_destructor)(void *ptr)) +{ + if (db) { + if (ptr_destructor) { + size_t n = 0; + + for (n = 0; n < db->max_ptrs; n++) + if (db->ptrs[n]) + ptr_destructor(db->ptrs[n]); + } + free(db->ptrs); + db->ptrs = NULL; + db->max_ptrs = 0; + } +} + +bool handle_db_is_empty(struct handle_db *db) +{ + size_t n = 0; + + if (db) { + for (n = 0; n < db->max_ptrs; n++) { + if (db->ptrs[n]) + return false; + } + } + return true; +} + +int handle_get(struct handle_db *db, void *ptr) +{ + size_t n; + void *p; + size_t new_max_ptrs; + + if (!db || !ptr) + return -1; + + /* Try to find an empty location */ + for (n = 0; n < db->max_ptrs; n++) { + if (!db->ptrs[n]) { + db->ptrs[n] = ptr; + return n; + } + } + + /* No location available, grow the ptrs array */ + if (db->max_ptrs) + new_max_ptrs = db->max_ptrs * 2; + else + new_max_ptrs = HANDLE_DB_INITIAL_MAX_PTRS; + p = realloc(db->ptrs, new_max_ptrs * sizeof(void *)); + if (!p) + return -1; + db->ptrs = p; + memset(db->ptrs + db->max_ptrs, 0, + (new_max_ptrs - db->max_ptrs) * sizeof(void *)); + db->max_ptrs = new_max_ptrs; + + /* Since n stopped at db->max_ptrs there is an empty location there */ + db->ptrs[n] = ptr; + return n; +} + +void *handle_put(struct handle_db *db, int handle) +{ + void *p; + + if (!db || handle < 0 || (size_t)handle >= db->max_ptrs) + return NULL; + + p = db->ptrs[handle]; + db->ptrs[handle] = NULL; + return p; +} + +void *handle_lookup(struct handle_db *db, int handle) +{ + if (!db || handle < 0 || (size_t)handle >= db->max_ptrs) + return NULL; + + return db->ptrs[handle]; +} diff --git a/optee_os/core/kernel/huk_subkey.c b/optee_os/core/kernel/huk_subkey.c new file mode 100644 index 0000000..66b1447 --- /dev/null +++ b/optee_os/core/kernel/huk_subkey.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +static TEE_Result mac_usage(void *ctx, uint32_t usage) +{ + return crypto_mac_update(ctx, (const void *)&usage, sizeof(usage)); +} + +#ifdef CFG_CORE_HUK_SUBKEY_COMPAT +static TEE_Result get_otp_die_id(uint8_t *buffer, size_t len) +{ + static const char pattern[4] = { 'B', 'E', 'E', 'F' }; + size_t i; + + if (IS_ENABLED(CFG_CORE_HUK_SUBKEY_COMPAT_USE_OTP_DIE_ID)) + return tee_otp_get_die_id(buffer, len); + + for (i = 0; i < len; i++) + buffer[i] = pattern[i % 4]; + + return TEE_SUCCESS; +} + +/* + * This does special treatment for RPMB and SSK key derivations to give + * the same result as when huk_subkey_derive() wasn't used. + */ +static TEE_Result huk_compat(void *ctx, enum huk_subkey_usage usage) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t chip_id[TEE_FS_KM_CHIP_ID_LENGTH] = { 0 }; + static uint8_t ssk_str[] = "ONLY_FOR_tee_fs_ssk"; + + switch (usage) { + case HUK_SUBKEY_RPMB: + return TEE_SUCCESS; + case HUK_SUBKEY_SSK: + res = get_otp_die_id(chip_id, sizeof(chip_id)); + if (res) + return res; + res = crypto_mac_update(ctx, chip_id, sizeof(chip_id)); + if (res) + return res; + return crypto_mac_update(ctx, ssk_str, sizeof(ssk_str)); + default: + return mac_usage(ctx, usage); + } + +} +#endif /*CFG_CORE_HUK_SUBKEY_COMPAT*/ + +TEE_Result __huk_subkey_derive(enum huk_subkey_usage usage, + const void *const_data, size_t const_data_len, + uint8_t *subkey, size_t subkey_len) +{ + void *ctx = NULL; + struct tee_hw_unique_key huk = { }; + TEE_Result res = TEE_SUCCESS; + + if (subkey_len > HUK_SUBKEY_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + if (!const_data && const_data_len) + return TEE_ERROR_BAD_PARAMETERS; + + res = crypto_mac_alloc_ctx(&ctx, TEE_ALG_HMAC_SHA256); + if (res) + return res; + + res = tee_otp_get_hw_unique_key(&huk); + if (res) + goto out; + + res = crypto_mac_init(ctx, huk.data, sizeof(huk.data)); + if (res) + goto out; + +#ifdef CFG_CORE_HUK_SUBKEY_COMPAT + res = huk_compat(ctx, usage); +#else + res = mac_usage(ctx, usage); +#endif + if (res) + goto out; + + if (const_data) { + res = crypto_mac_update(ctx, const_data, const_data_len); + if (res) + goto out; + } + + res = crypto_mac_final(ctx, subkey, subkey_len); +out: + if (res) + memzero_explicit(subkey, subkey_len); + memzero_explicit(&huk, sizeof(huk)); + crypto_mac_free_ctx(ctx); + return res; +} + +TEE_Result huk_subkey_derive(enum huk_subkey_usage usage, + const void *const_data, size_t const_data_len, + uint8_t *subkey, size_t subkey_len) +__weak __alias("__huk_subkey_derive"); diff --git a/optee_os/core/kernel/initcall.c b/optee_os/core/kernel/initcall.c new file mode 100644 index 0000000..148a178 --- /dev/null +++ b/optee_os/core/kernel/initcall.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include + +/* + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +void __weak call_preinitcalls(void) +{ + const struct initcall *call = NULL; + TEE_Result ret = TEE_SUCCESS; + + for (call = preinitcall_begin; call < preinitcall_end; call++) { + DMSG("level %d %s()", call->level, call->func_name); + ret = call->func(); + if (ret != TEE_SUCCESS) { + EMSG("Preinitcall __text_start + 0x%08" PRIxVA + " failed", (vaddr_t)call - VCORE_START_VA); + } + } +} + +/* + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +void __weak call_initcalls(void) +{ + const struct initcall *call = NULL; + TEE_Result ret = TEE_SUCCESS; + + for (call = initcall_begin; call < initcall_end; call++) { + DMSG("level %d %s()", call->level, call->func_name); + ret = call->func(); + if (ret != TEE_SUCCESS) { + EMSG("Initcall __text_start + 0x%08" PRIxVA + " failed", (vaddr_t)call - VCORE_START_VA); + } + } +} + +/* + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +void __weak call_finalcalls(void) +{ + const struct initcall *call = NULL; + TEE_Result ret = TEE_SUCCESS; + + for (call = finalcall_begin; call < finalcall_end; call++) { + DMSG("level %d %s()", call->level, call->func_name); + ret = call->func(); + if (ret != TEE_SUCCESS) { + EMSG("Finalcall __text_start + 0x%08" PRIxVA + " failed", (vaddr_t)call - VCORE_START_VA); + } + } +} diff --git a/optee_os/core/kernel/interrupt.c b/optee_os/core/kernel/interrupt.c new file mode 100644 index 0000000..9fbc9b2 --- /dev/null +++ b/optee_os/core/kernel/interrupt.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * NOTE! + * + * We're assuming that there's no concurrent use of this interface, except + * delivery of interrupts in parallel. Synchronization will be needed when + * we begin to modify settings after boot initialization. + */ + +static struct itr_chip *itr_main_chip __nex_bss; + +TEE_Result itr_chip_init(struct itr_chip *chip) +{ + if (!itr_chip_is_valid(chip)) + return TEE_ERROR_BAD_PARAMETERS; + + SLIST_INIT(&chip->handlers); + + return TEE_SUCCESS; +} + +void interrupt_main_init(struct itr_chip *chip) +{ + if (itr_chip_init(chip)) + panic(); + + itr_main_chip = chip; +} + +struct itr_chip *interrupt_get_main_chip(void) +{ + assert(itr_main_chip); + return itr_main_chip; +} + +#ifdef CFG_DT +int dt_get_irq_type_prio(const void *fdt, int node, uint32_t *type, + uint32_t *prio) +{ + const uint32_t *prop = NULL; + int count = 0; + int it_num = DT_INFO_INVALID_INTERRUPT; + + if (!itr_main_chip || !itr_main_chip->dt_get_irq) + return it_num; + + prop = fdt_getprop(fdt, node, "interrupts", &count); + if (!prop) + return it_num; + + return itr_main_chip->dt_get_irq(prop, count, type, prio); +} +#endif + +struct itr_handler *itr_alloc_add_type_prio(size_t it, itr_handler_t handler, + uint32_t flags, void *data, + uint32_t type, uint32_t prio) +{ + struct itr_handler *hdl = calloc(1, sizeof(*hdl)); + + if (hdl) { + hdl->it = it; + hdl->handler = handler; + hdl->flags = flags; + hdl->data = data; + itr_add_type_prio(hdl, type, prio); + } + + return hdl; +} + +void itr_free(struct itr_handler *hdl) +{ + if (!hdl) + return; + + itr_main_chip->ops->disable(itr_main_chip, hdl->it); + + SLIST_REMOVE(&itr_main_chip->handlers, hdl, itr_handler, link); + free(hdl); +} + +void itr_add_type_prio(struct itr_handler *h, uint32_t type, uint32_t prio) +{ + struct itr_handler __maybe_unused *hdl = NULL; + + SLIST_FOREACH(hdl, &itr_main_chip->handlers, link) + if (hdl->it == h->it) + assert((hdl->flags & ITRF_SHARED) && + (h->flags & ITRF_SHARED)); + + itr_main_chip->ops->add(itr_main_chip, h->it, type, prio); + SLIST_INSERT_HEAD(&itr_main_chip->handlers, h, link); +} + +void itr_enable(size_t it) +{ + itr_main_chip->ops->enable(itr_main_chip, it); +} + +void itr_disable(size_t it) +{ + itr_main_chip->ops->disable(itr_main_chip, it); +} + +void itr_raise_pi(size_t it) +{ + itr_main_chip->ops->raise_pi(itr_main_chip, it); +} + +void itr_raise_sgi(size_t it, uint8_t cpu_mask) +{ + itr_main_chip->ops->raise_sgi(itr_main_chip, it, cpu_mask); +} + +void itr_set_affinity(size_t it, uint8_t cpu_mask) +{ + itr_main_chip->ops->set_affinity(itr_main_chip, it, cpu_mask); +} + +/* This function is supposed to be overridden in platform specific code */ +void __weak __noreturn interrupt_main_handler(void) +{ + panic("Secure interrupt handler not defined"); +} + +/* + * Interrupt controller chip support + */ +void interrupt_call_handlers(struct itr_chip *chip, size_t itr_num) +{ + struct itr_handler *h = NULL; + bool was_handled = false; + + assert(chip); + + SLIST_FOREACH(h, &chip->handlers, link) { + if (h->it == itr_num) { + if (h->handler(h) == ITRR_HANDLED) + was_handled = true; + else if (!(h->flags & ITRF_SHARED)) + break; + } + } + + if (!was_handled) { + EMSG("Mask unhandled interrupt %s:%zu", chip->name, itr_num); + interrupt_mask(chip, itr_num); + } +} + +TEE_Result interrupt_configure(struct itr_chip *chip, size_t itr_num, + uint32_t type, uint32_t prio) +{ + chip->ops->add(chip, itr_num, type, prio); + + return TEE_SUCCESS; +} + +TEE_Result interrupt_add_configure_handler(struct itr_handler *hdl, + uint32_t type, uint32_t prio) +{ + struct itr_handler *h = NULL; + + assert(hdl && hdl->chip->ops && is_unpaged(hdl) && + hdl->handler && is_unpaged(hdl->handler)); + + SLIST_FOREACH(h, &hdl->chip->handlers, link) { + if (h->it == hdl->it && + (!(hdl->flags & ITRF_SHARED) || + !(h->flags & ITRF_SHARED))) { + EMSG("Shared and non-shared flags on interrupt %s#%zu", + hdl->chip->name, hdl->it); + return TEE_ERROR_GENERIC; + } + } + + interrupt_configure(hdl->chip, hdl->it, type, prio); + + SLIST_INSERT_HEAD(&hdl->chip->handlers, hdl, link); + + return TEE_SUCCESS; +} + +void interrupt_remove_handler(struct itr_handler *hdl) +{ + struct itr_handler *h = NULL; + bool disable_itr = true; + + if (!hdl) + return; + + SLIST_FOREACH(h, &hdl->chip->handlers, link) + if (h == hdl) + break; + if (!h) { + DMSG("Invalid %s:%zu", hdl->chip->name, hdl->it); + assert(false); + return; + } + + if (hdl->flags & ITRF_SHARED) { + SLIST_FOREACH(h, &hdl->chip->handlers, link) { + if (h != hdl && h->it == hdl->it) { + disable_itr = false; + break; + } + } + } + + if (disable_itr) + interrupt_disable(hdl->chip, hdl->it); + + SLIST_REMOVE(&hdl->chip->handlers, hdl, itr_handler, link); +} + +TEE_Result interrupt_alloc_add_conf_handler(struct itr_chip *chip, + size_t itr_num, + itr_handler_t handler, + uint32_t flags, void *data, + uint32_t type, uint32_t prio, + struct itr_handler **out_hdl) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct itr_handler *hdl = NULL; + + hdl = calloc(1, sizeof(*hdl)); + if (!hdl) + return TEE_ERROR_OUT_OF_MEMORY; + + *hdl = ITR_HANDLER(chip, itr_num, flags, handler, data); + + res = interrupt_add_configure_handler(hdl, type, prio); + if (res) { + free(hdl); + return res; + } + + if (out_hdl) + *out_hdl = hdl; + + return TEE_SUCCESS; +} + +void interrupt_remove_free_handler(struct itr_handler *hdl) +{ + if (hdl) { + interrupt_remove_handler(hdl); + free(hdl); + } +} diff --git a/optee_os/core/kernel/ldelf_loader.c b/optee_os/core/kernel/ldelf_loader.c new file mode 100644 index 0000000..cf00f7d --- /dev/null +++ b/optee_os/core/kernel/ldelf_loader.c @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2015-2020, 2022 Linaro Limited + * Copyright (c) 2020-2023, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BOUNCE_BUFFER_SIZE 4096 + +extern uint8_t ldelf_data[]; +extern const unsigned int ldelf_code_size; +extern const unsigned int ldelf_data_size; +extern const unsigned int ldelf_entry; + +/* ldelf has the same architecture/register width as the kernel */ +#if defined(ARM32) || defined(RV32) +static const bool is_32bit = true; +#else +static const bool is_32bit; +#endif + +static TEE_Result alloc_and_map_fobj(struct user_mode_ctx *uctx, size_t sz, + uint32_t prot, uint32_t flags, vaddr_t *va) +{ + size_t num_pgs = ROUNDUP(sz, SMALL_PAGE_SIZE) / SMALL_PAGE_SIZE; + struct fobj *fobj = fobj_ta_mem_alloc(num_pgs); + struct mobj *mobj = mobj_with_fobj_alloc(fobj, NULL, + TEE_MATTR_MEM_TYPE_TAGGED); + TEE_Result res = TEE_SUCCESS; + + fobj_put(fobj); + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + res = vm_map(uctx, va, num_pgs * SMALL_PAGE_SIZE, prot, flags, mobj, 0); + mobj_put(mobj); + + return res; +} + +/* + * This function may leave a few mappings behind on error, but that's taken + * care of by tee_ta_init_user_ta_session() since the entire context is + * removed then. + */ +TEE_Result ldelf_load_ldelf(struct user_mode_ctx *uctx) +{ + TEE_Result res = TEE_SUCCESS; + vaddr_t stack_addr = 0; + vaddr_t code_addr = 0; + vaddr_t rw_addr = 0; + vaddr_t bb_addr = 0; + uint32_t prot = 0; + + uctx->is_32bit = is_32bit; + + res = alloc_and_map_fobj(uctx, BOUNCE_BUFFER_SIZE, TEE_MATTR_PRW, 0, + &bb_addr); + if (res) + return res; + uctx->bbuf = (void *)bb_addr; + uctx->bbuf_size = BOUNCE_BUFFER_SIZE; + + res = alloc_and_map_fobj(uctx, LDELF_STACK_SIZE, + TEE_MATTR_URW | TEE_MATTR_PRW, VM_FLAG_LDELF, + &stack_addr); + if (res) + return res; + uctx->ldelf_stack_ptr = stack_addr + LDELF_STACK_SIZE; + + res = alloc_and_map_fobj(uctx, ldelf_code_size, TEE_MATTR_PRW, + VM_FLAG_LDELF, &code_addr); + if (res) + return res; + uctx->entry_func = code_addr + ldelf_entry; + + rw_addr = ROUNDUP(code_addr + ldelf_code_size, SMALL_PAGE_SIZE); + res = alloc_and_map_fobj(uctx, ldelf_data_size, + TEE_MATTR_URW | TEE_MATTR_PRW, VM_FLAG_LDELF, + &rw_addr); + if (res) + return res; + + vm_set_ctx(uctx->ts_ctx); + + memcpy((void *)code_addr, ldelf_data, ldelf_code_size); + + res = copy_to_user((void *)rw_addr, ldelf_data + ldelf_code_size, + ldelf_data_size); + if (res) + return res; + + prot = TEE_MATTR_URX; + if (IS_ENABLED(CFG_CORE_BTI)) + prot |= TEE_MATTR_GUARDED; + + res = vm_set_prot(uctx, code_addr, + ROUNDUP(ldelf_code_size, SMALL_PAGE_SIZE), prot); + if (res) + return res; + + DMSG("ldelf load address %#"PRIxVA, code_addr); + + return TEE_SUCCESS; +} + +TEE_Result ldelf_init_with_ldelf(struct ts_session *sess, + struct user_mode_ctx *uctx) +{ + TEE_Result res = TEE_SUCCESS; + struct ldelf_arg *arg = NULL; + uint32_t panic_code = 0; + uint32_t panicked = 0; + uaddr_t usr_stack = 0; + struct ldelf_arg *arg_bbuf = NULL; + + usr_stack = uctx->ldelf_stack_ptr; + usr_stack -= ROUNDUP(sizeof(*arg), STACK_ALIGNMENT); + arg = (struct ldelf_arg *)usr_stack; + sess->handle_scall = scall_handle_ldelf; + + res = clear_user(arg, sizeof(*arg)); + if (res) + return res; + + res = PUT_USER_SCALAR(uctx->ts_ctx->uuid, &arg->uuid); + if (res) + return res; + + res = thread_enter_user_mode((vaddr_t)arg, 0, 0, 0, + usr_stack, uctx->entry_func, + is_32bit, &panicked, &panic_code); + + sess->handle_scall = sess->ctx->ops->handle_scall; + thread_user_clear_vfp(uctx); + ldelf_sess_cleanup(sess); + + if (panicked) { + abort_print_current_ts(); + EMSG("ldelf panicked"); + return TEE_ERROR_GENERIC; + } + if (res) { + EMSG("ldelf failed with res: %#"PRIx32, res); + return res; + } + + res = BB_MEMDUP_USER(arg, sizeof(*arg), &arg_bbuf); + if (res) + return res; + + if (is_user_ta_ctx(uctx->ts_ctx)) { + /* + * This is already checked by the elf loader, but since it runs + * in user mode we're not trusting it entirely. + */ + if (arg_bbuf->flags & ~TA_FLAGS_MASK) + return TEE_ERROR_BAD_FORMAT; + + to_user_ta_ctx(uctx->ts_ctx)->ta_ctx.flags = arg_bbuf->flags; + } + + uctx->is_32bit = arg_bbuf->is_32bit; + uctx->entry_func = arg_bbuf->entry_func; + uctx->load_addr = arg_bbuf->load_addr; + uctx->stack_ptr = arg_bbuf->stack_ptr; + uctx->dump_entry_func = arg_bbuf->dump_entry; +#ifdef CFG_FTRACE_SUPPORT + uctx->ftrace_entry_func = arg_bbuf->ftrace_entry; + sess->fbuf = arg_bbuf->fbuf; +#endif + uctx->dl_entry_func = arg_bbuf->dl_entry; + + bb_free(arg_bbuf, sizeof(*arg)); + + return TEE_SUCCESS; +} + +TEE_Result ldelf_dump_state(struct user_mode_ctx *uctx) +{ + TEE_Result res = TEE_SUCCESS; + uaddr_t usr_stack = uctx->ldelf_stack_ptr; + struct dump_entry_arg *arg = NULL; + uint32_t panic_code = 0; + uint32_t panicked = 0; + struct thread_specific_data *tsd = thread_get_tsd(); + struct ts_session *sess = NULL; + struct vm_region *r = NULL; + size_t arg_size = 0; + size_t n = 0; + + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) + if (r->attr & TEE_MATTR_URWX) + n++; + + arg_size = ROUNDUP(sizeof(*arg) + n * sizeof(struct dump_map), + STACK_ALIGNMENT); + + usr_stack = uctx->ldelf_stack_ptr; + usr_stack -= arg_size; + + arg = bb_alloc(arg_size); + if (!arg) + return TEE_ERROR_OUT_OF_MEMORY; + memset(arg, 0, arg_size); + + arg->num_maps = n; + n = 0; + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) { + if (r->attr & TEE_MATTR_URWX) { + if (r->mobj) + mobj_get_pa(r->mobj, r->offset, 0, + &arg->maps[n].pa); + arg->maps[n].va = r->va; + arg->maps[n].sz = r->size; + if (r->attr & TEE_MATTR_UR) + arg->maps[n].flags |= DUMP_MAP_READ; + if (r->attr & TEE_MATTR_UW) + arg->maps[n].flags |= DUMP_MAP_WRITE; + if (r->attr & TEE_MATTR_UX) + arg->maps[n].flags |= DUMP_MAP_EXEC; + if (r->attr & TEE_MATTR_SECURE) + arg->maps[n].flags |= DUMP_MAP_SECURE; + if (r->flags & VM_FLAG_EPHEMERAL) + arg->maps[n].flags |= DUMP_MAP_EPHEM; + if (r->flags & VM_FLAG_LDELF) + arg->maps[n].flags |= DUMP_MAP_LDELF; + n++; + } + } + + arg->is_32bit = uctx->is_32bit; +#ifdef ARM32 + arg->arm32.regs[0] = tsd->abort_regs.r0; + arg->arm32.regs[1] = tsd->abort_regs.r1; + arg->arm32.regs[2] = tsd->abort_regs.r2; + arg->arm32.regs[3] = tsd->abort_regs.r3; + arg->arm32.regs[4] = tsd->abort_regs.r4; + arg->arm32.regs[5] = tsd->abort_regs.r5; + arg->arm32.regs[6] = tsd->abort_regs.r6; + arg->arm32.regs[7] = tsd->abort_regs.r7; + arg->arm32.regs[8] = tsd->abort_regs.r8; + arg->arm32.regs[9] = tsd->abort_regs.r9; + arg->arm32.regs[10] = tsd->abort_regs.r10; + arg->arm32.regs[11] = tsd->abort_regs.r11; + arg->arm32.regs[12] = tsd->abort_regs.ip; + arg->arm32.regs[13] = tsd->abort_regs.usr_sp; /*SP*/ + arg->arm32.regs[14] = tsd->abort_regs.usr_lr; /*LR*/ + arg->arm32.regs[15] = tsd->abort_regs.elr; /*PC*/ +#endif /*ARM32*/ +#ifdef ARM64 + if (uctx->is_32bit) { + arg->arm32.regs[0] = tsd->abort_regs.x0; + arg->arm32.regs[1] = tsd->abort_regs.x1; + arg->arm32.regs[2] = tsd->abort_regs.x2; + arg->arm32.regs[3] = tsd->abort_regs.x3; + arg->arm32.regs[4] = tsd->abort_regs.x4; + arg->arm32.regs[5] = tsd->abort_regs.x5; + arg->arm32.regs[6] = tsd->abort_regs.x6; + arg->arm32.regs[7] = tsd->abort_regs.x7; + arg->arm32.regs[8] = tsd->abort_regs.x8; + arg->arm32.regs[9] = tsd->abort_regs.x9; + arg->arm32.regs[10] = tsd->abort_regs.x10; + arg->arm32.regs[11] = tsd->abort_regs.x11; + arg->arm32.regs[12] = tsd->abort_regs.x12; + arg->arm32.regs[13] = tsd->abort_regs.x13; /*SP*/ + arg->arm32.regs[14] = tsd->abort_regs.x14; /*LR*/ + arg->arm32.regs[15] = tsd->abort_regs.elr; /*PC*/ + } else { + arg->arm64.fp = tsd->abort_regs.x29; + arg->arm64.pc = tsd->abort_regs.elr; + arg->arm64.sp = tsd->abort_regs.sp_el0; + } +#endif /*ARM64*/ +#if defined(RV64) || defined(RV32) + arg->rv.fp = tsd->abort_regs.s0; + arg->rv.pc = tsd->abort_regs.epc; + arg->rv.sp = tsd->abort_regs.sp; +#endif /*RV64||RV32*/ + + res = copy_to_user((void *)usr_stack, arg, arg_size); + if (res) + return res; + + sess = ts_get_current_session(); + sess->handle_scall = scall_handle_ldelf; + + res = thread_enter_user_mode(usr_stack, 0, 0, 0, + usr_stack, uctx->dump_entry_func, + is_32bit, &panicked, &panic_code); + + sess->handle_scall = sess->ctx->ops->handle_scall; + thread_user_clear_vfp(uctx); + ldelf_sess_cleanup(sess); + + if (panicked) { + uctx->dump_entry_func = 0; + EMSG("ldelf dump function panicked"); + abort_print_current_ts(); + res = TEE_ERROR_TARGET_DEAD; + } + + return res; +} + +#ifdef CFG_FTRACE_SUPPORT +TEE_Result ldelf_dump_ftrace(struct user_mode_ctx *uctx, + void *buf, size_t *blen) +{ + uaddr_t usr_stack = uctx->ldelf_stack_ptr; + TEE_Result res = TEE_SUCCESS; + uint32_t panic_code = 0; + uint32_t panicked = 0; + size_t *arg = NULL; + struct ts_session *sess = NULL; + + if (!uctx->ftrace_entry_func) + return TEE_ERROR_NOT_SUPPORTED; + + usr_stack -= ROUNDUP(sizeof(*arg), STACK_ALIGNMENT); + arg = (size_t *)usr_stack; + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)arg, sizeof(*arg)); + if (res) { + EMSG("ldelf stack is inaccessible!"); + return res; + } + + *arg = *blen; + + sess = ts_get_current_session(); + sess->handle_scall = scall_handle_ldelf; + + res = thread_enter_user_mode((vaddr_t)buf, (vaddr_t)arg, 0, 0, + usr_stack, uctx->ftrace_entry_func, + is_32bit, &panicked, &panic_code); + + sess->handle_scall = sess->ctx->ops->handle_scall; + thread_user_clear_vfp(uctx); + ldelf_sess_cleanup(sess); + + if (panicked) { + uctx->ftrace_entry_func = 0; + EMSG("ldelf ftrace function panicked"); + abort_print_current_ts(); + res = TEE_ERROR_TARGET_DEAD; + } + + if (!res) { + if (*arg > *blen) + res = TEE_ERROR_SHORT_BUFFER; + *blen = *arg; + } + + return res; +} +#endif /*CFG_FTRACE_SUPPORT*/ + +TEE_Result ldelf_dlopen(struct user_mode_ctx *uctx, TEE_UUID *uuid, + uint32_t flags) +{ + uaddr_t usr_stack = uctx->ldelf_stack_ptr; + TEE_Result res = TEE_ERROR_GENERIC; + struct dl_entry_arg *usr_arg = NULL; + struct dl_entry_arg *arg = NULL; + uint32_t panic_code = 0; + uint32_t panicked = 0; + struct ts_session *sess = NULL; + + assert(uuid); + + arg = bb_alloc(sizeof(*arg)); + if (!arg) + return TEE_ERROR_OUT_OF_MEMORY; + + memset(arg, 0, sizeof(*arg)); + arg->cmd = LDELF_DL_ENTRY_DLOPEN; + arg->dlopen.uuid = *uuid; + arg->dlopen.flags = flags; + + usr_stack -= ROUNDUP(sizeof(*arg), STACK_ALIGNMENT); + usr_arg = (void *)usr_stack; + + res = copy_to_user(usr_arg, arg, sizeof(*arg)); + if (res) + return res; + + sess = ts_get_current_session(); + sess->handle_scall = scall_handle_ldelf; + + res = thread_enter_user_mode(usr_stack, 0, 0, 0, + usr_stack, uctx->dl_entry_func, + is_32bit, &panicked, &panic_code); + + sess->handle_scall = sess->ctx->ops->handle_scall; + ldelf_sess_cleanup(sess); + + if (panicked) { + EMSG("ldelf dl_entry function panicked"); + abort_print_current_ts(); + res = TEE_ERROR_TARGET_DEAD; + } + if (!res) { + TEE_Result res2 = TEE_SUCCESS; + + res2 = GET_USER_SCALAR(res, &usr_arg->ret); + if (res2) + res = res2; + } + + return res; +} + +TEE_Result ldelf_dlsym(struct user_mode_ctx *uctx, TEE_UUID *uuid, + const char *sym, size_t symlen, vaddr_t *val) +{ + uaddr_t usr_stack = uctx->ldelf_stack_ptr; + TEE_Result res = TEE_ERROR_GENERIC; + struct dl_entry_arg *usr_arg = NULL; + struct dl_entry_arg *arg = NULL; + uint32_t panic_code = 0; + uint32_t panicked = 0; + struct ts_session *sess = NULL; + + usr_stack -= ROUNDUP(sizeof(*arg) + symlen + 1, STACK_ALIGNMENT); + usr_arg = (void *)usr_stack; + arg = bb_alloc(sizeof(*arg)); + if (!arg) + return TEE_ERROR_OUT_OF_MEMORY; + memset(arg, 0, sizeof(*arg)); + arg->cmd = LDELF_DL_ENTRY_DLSYM; + arg->dlsym.uuid = *uuid; + res = copy_to_user(usr_arg, arg, sizeof(*arg)); + if (res) + return res; + res = copy_to_user(usr_arg->dlsym.symbol, sym, symlen + 1); + if (res) + return res; + + sess = ts_get_current_session(); + sess->handle_scall = scall_handle_ldelf; + + res = thread_enter_user_mode((vaddr_t)usr_arg, 0, 0, 0, + usr_stack, uctx->dl_entry_func, + is_32bit, &panicked, &panic_code); + + sess->handle_scall = sess->ctx->ops->handle_scall; + ldelf_sess_cleanup(sess); + + if (panicked) { + EMSG("ldelf dl_entry function panicked"); + abort_print_current_ts(); + res = TEE_ERROR_TARGET_DEAD; + } + if (!res) { + TEE_Result res2 = TEE_SUCCESS; + + res2 = GET_USER_SCALAR(res, &usr_arg->ret); + if (res2) + res = res2; + if (!res) + res = GET_USER_SCALAR(*val, &usr_arg->dlsym.val); + } + + return res; +} diff --git a/optee_os/core/kernel/ldelf_syscalls.c b/optee_os/core/kernel/ldelf_syscalls.c new file mode 100644 index 0000000..1c4fcef --- /dev/null +++ b/optee_os/core/kernel/ldelf_syscalls.c @@ -0,0 +1,612 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2019, 2022 Linaro Limited + * Copyright (c) 2020-2021, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct bin_handle { + const struct ts_store_ops *op; + struct ts_store_handle *h; + struct file *f; + size_t offs_bytes; + size_t size_bytes; +}; + +static void unmap_or_panic(struct user_mode_ctx *uctx, vaddr_t va, + size_t byte_count) +{ + TEE_Result res = vm_unmap(uctx, va, byte_count); + + if (res) { + EMSG("vm_unmap(%#"PRIxVA", %#zx) returned %#"PRIx32, + va, byte_count, res); + panic("Can't restore memory map"); + } +} + +TEE_Result ldelf_syscall_map_zi(vaddr_t *va, size_t num_bytes, size_t pad_begin, + size_t pad_end, unsigned long flags) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(sess->ctx); + struct fobj *f = NULL; + struct mobj *mobj = NULL; + uint32_t prot = TEE_MATTR_URW | TEE_MATTR_PRW; + uint32_t vm_flags = 0; + vaddr_t va_copy = 0; + + if (flags & ~LDELF_MAP_FLAG_SHAREABLE) + return TEE_ERROR_BAD_PARAMETERS; + + res = GET_USER_SCALAR(va_copy, va); + if (res) + return res; + + if (flags & LDELF_MAP_FLAG_SHAREABLE) + vm_flags |= VM_FLAG_SHAREABLE; + + f = fobj_ta_mem_alloc(ROUNDUP_DIV(num_bytes, SMALL_PAGE_SIZE)); + if (!f) + return TEE_ERROR_OUT_OF_MEMORY; + mobj = mobj_with_fobj_alloc(f, NULL, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(f); + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + res = vm_map_pad(uctx, &va_copy, num_bytes, prot, vm_flags, + mobj, 0, pad_begin, pad_end, 0); + mobj_put(mobj); + if (!res) { + res = PUT_USER_SCALAR(va_copy, va); + if (res) + unmap_or_panic(uctx, va_copy, num_bytes); + } + + return res; +} + +TEE_Result ldelf_syscall_unmap(vaddr_t va, size_t num_bytes) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(sess->ctx); + size_t sz = ROUNDUP(num_bytes, SMALL_PAGE_SIZE); + uint32_t vm_flags = 0; + vaddr_t end_va = 0; + + /* + * The vm_get_flags() and vm_unmap() are supposed to detect or handle + * overflow directly or indirectly. However, since this function is an + * API function it's worth having an extra guard here. If nothing else, + * to increase code clarity. + */ + if (ADD_OVERFLOW(va, sz, &end_va)) + return TEE_ERROR_BAD_PARAMETERS; + + res = vm_get_flags(uctx, va, sz, &vm_flags); + if (res) + return res; + if (vm_flags & VM_FLAG_PERMANENT) + return TEE_ERROR_ACCESS_DENIED; + + return vm_unmap(uctx, va, sz); +} + +static void bin_close(void *ptr) +{ + struct bin_handle *binh = ptr; + + if (binh) { + if (binh->op && binh->h) + binh->op->close(binh->h); + file_put(binh->f); + } + free(binh); +} + +TEE_Result ldelf_syscall_open_bin(const TEE_UUID *uuid, size_t uuid_size, + uint32_t *handle) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(sess->ctx); + struct system_ctx *sys_ctx = sess->user_ctx; + struct bin_handle *binh = NULL; + uint8_t tag[FILE_TAG_SIZE] = { 0 }; + unsigned int tag_len = sizeof(tag); + TEE_UUID *bb_uuid = NULL; + int h = 0; + + res = BB_MEMDUP_USER(uuid, sizeof(*uuid), &bb_uuid); + if (res) + return res; + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)handle, sizeof(uint32_t)); + if (res) + return res; + + if (uuid_size != sizeof(*uuid)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!sys_ctx) { + sys_ctx = calloc(1, sizeof(*sys_ctx)); + if (!sys_ctx) + return TEE_ERROR_OUT_OF_MEMORY; + sess->user_ctx = sys_ctx; + } + + binh = calloc(1, sizeof(*binh)); + if (!binh) + return TEE_ERROR_OUT_OF_MEMORY; + + if (is_user_ta_ctx(sess->ctx) || is_stmm_ctx(sess->ctx)) { + SCATTERED_ARRAY_FOREACH(binh->op, ta_stores, + struct ts_store_ops) { + DMSG("Lookup user TA ELF %pUl (%s)", + (void *)bb_uuid, binh->op->description); + + res = binh->op->open(bb_uuid, &binh->h); + DMSG("res=%#"PRIx32, res); + if (res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + break; + } + } else if (is_sp_ctx(sess->ctx)) { + SCATTERED_ARRAY_FOREACH(binh->op, sp_stores, + struct ts_store_ops) { + DMSG("Lookup user SP ELF %pUl (%s)", + (void *)bb_uuid, binh->op->description); + + res = binh->op->open(bb_uuid, &binh->h); + DMSG("res=%#"PRIx32, res); + if (res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + break; + } + } else { + res = TEE_ERROR_ITEM_NOT_FOUND; + } + + if (res) + goto err; + + res = binh->op->get_size(binh->h, &binh->size_bytes); + if (res) + goto err; + res = binh->op->get_tag(binh->h, tag, &tag_len); + if (res) + goto err; + binh->f = file_get_by_tag(tag, tag_len); + if (!binh->f) + goto err_oom; + + h = handle_get(&sys_ctx->db, binh); + if (h < 0) + goto err_oom; + res = PUT_USER_SCALAR(h, handle); + if (res) { + handle_put(&sys_ctx->db, h); + goto err; + } + + return TEE_SUCCESS; + +err_oom: + res = TEE_ERROR_OUT_OF_MEMORY; +err: + bin_close(binh); + return res; +} + +TEE_Result ldelf_syscall_close_bin(unsigned long handle) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct system_ctx *sys_ctx = sess->user_ctx; + struct bin_handle *binh = NULL; + + if (!sys_ctx) + return TEE_ERROR_BAD_PARAMETERS; + + binh = handle_put(&sys_ctx->db, handle); + if (!binh) + return TEE_ERROR_BAD_PARAMETERS; + + if (binh->offs_bytes < binh->size_bytes) + res = binh->op->read(binh->h, NULL, NULL, + binh->size_bytes - binh->offs_bytes); + + bin_close(binh); + if (handle_db_is_empty(&sys_ctx->db)) { + handle_db_destroy(&sys_ctx->db, bin_close); + free(sys_ctx); + sess->user_ctx = NULL; + } + + return res; +} + +static TEE_Result binh_copy_to(struct bin_handle *binh, vaddr_t va_core, + vaddr_t va_user, size_t offs_bytes, + size_t num_bytes) +{ + TEE_Result res = TEE_SUCCESS; + size_t next_offs = 0; + + if (offs_bytes < binh->offs_bytes) + return TEE_ERROR_BAD_STATE; + + if (ADD_OVERFLOW(offs_bytes, num_bytes, &next_offs)) + return TEE_ERROR_BAD_PARAMETERS; + + if (offs_bytes > binh->offs_bytes) { + res = binh->op->read(binh->h, NULL, NULL, + offs_bytes - binh->offs_bytes); + if (res) + return res; + binh->offs_bytes = offs_bytes; + } + + if (next_offs > binh->size_bytes) { + size_t rb = binh->size_bytes - binh->offs_bytes; + + res = binh->op->read(binh->h, (void *)va_core, + (void *)va_user, rb); + if (res) + return res; + if (va_core) + memset((uint8_t *)va_core + rb, 0, num_bytes - rb); + if (va_user) { + res = clear_user((uint8_t *)va_user + rb, + num_bytes - rb); + if (res) + return res; + } + binh->offs_bytes = binh->size_bytes; + } else { + res = binh->op->read(binh->h, (void *)va_core, + (void *)va_user, num_bytes); + if (res) + return res; + binh->offs_bytes = next_offs; + } + + return TEE_SUCCESS; +} + +TEE_Result ldelf_syscall_map_bin(vaddr_t *va, size_t num_bytes, + unsigned long handle, size_t offs_bytes, + size_t pad_begin, size_t pad_end, + unsigned long flags) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(sess->ctx); + struct system_ctx *sys_ctx = sess->user_ctx; + struct bin_handle *binh = NULL; + uint32_t num_rounded_bytes = 0; + struct file_slice *fs = NULL; + bool file_is_locked = false; + struct mobj *mobj = NULL; + uint32_t offs_pages = 0; + size_t num_pages = 0; + vaddr_t va_copy = 0; + uint32_t prot = 0; + const uint32_t accept_flags = LDELF_MAP_FLAG_SHAREABLE | + LDELF_MAP_FLAG_WRITEABLE | + LDELF_MAP_FLAG_BTI | + LDELF_MAP_FLAG_EXECUTABLE; + + res = GET_USER_SCALAR(va_copy, va); + if (res) + return res; + + if (!sys_ctx) + return TEE_ERROR_BAD_PARAMETERS; + + binh = handle_lookup(&sys_ctx->db, handle); + if (!binh) + return TEE_ERROR_BAD_PARAMETERS; + + if ((flags & accept_flags) != flags) + return TEE_ERROR_BAD_PARAMETERS; + + if ((flags & LDELF_MAP_FLAG_SHAREABLE) && + (flags & LDELF_MAP_FLAG_WRITEABLE)) + return TEE_ERROR_BAD_PARAMETERS; + + if ((flags & LDELF_MAP_FLAG_EXECUTABLE) && + (flags & LDELF_MAP_FLAG_WRITEABLE)) + return TEE_ERROR_BAD_PARAMETERS; + + if (offs_bytes & SMALL_PAGE_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + prot = TEE_MATTR_UR | TEE_MATTR_PR; + if (flags & LDELF_MAP_FLAG_WRITEABLE) + prot |= TEE_MATTR_UW | TEE_MATTR_PW; + if (flags & LDELF_MAP_FLAG_EXECUTABLE) + prot |= TEE_MATTR_UX; + if (flags & LDELF_MAP_FLAG_BTI) + prot |= TEE_MATTR_GUARDED; + + offs_pages = offs_bytes >> SMALL_PAGE_SHIFT; + if (ROUNDUP_OVERFLOW(num_bytes, SMALL_PAGE_SIZE, &num_rounded_bytes)) + return TEE_ERROR_BAD_PARAMETERS; + num_pages = num_rounded_bytes / SMALL_PAGE_SIZE; + + if (!file_trylock(binh->f)) { + /* + * Before we can block on the file lock we must make all + * our page tables available for reclaiming in order to + * avoid a dead-lock with the other thread (which already + * is holding the file lock) mapping lots of memory below. + */ + vm_set_ctx(NULL); + file_lock(binh->f); + vm_set_ctx(uctx->ts_ctx); + } + file_is_locked = true; + fs = file_find_slice(binh->f, offs_pages); + if (fs) { + /* If there's registered slice it has to match */ + if (fs->page_offset != offs_pages || + num_pages > fs->fobj->num_pages) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + /* If there's a slice we must be mapping shareable */ + if (!(flags & LDELF_MAP_FLAG_SHAREABLE)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + mobj = mobj_with_fobj_alloc(fs->fobj, binh->f, + TEE_MATTR_MEM_TYPE_TAGGED); + if (!mobj) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + res = vm_map_pad(uctx, &va_copy, num_rounded_bytes, + prot, VM_FLAG_READONLY, + mobj, 0, pad_begin, pad_end, 0); + mobj_put(mobj); + if (res) + goto err; + } else { + struct fobj *f = fobj_ta_mem_alloc(num_pages); + struct file *file = NULL; + uint32_t vm_flags = 0; + + if (!f) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + if (!(flags & LDELF_MAP_FLAG_WRITEABLE)) { + file = binh->f; + vm_flags |= VM_FLAG_READONLY; + } + + mobj = mobj_with_fobj_alloc(f, file, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(f); + if (!mobj) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + res = vm_map_pad(uctx, &va_copy, num_rounded_bytes, + TEE_MATTR_PRW, vm_flags, mobj, 0, + pad_begin, pad_end, 0); + mobj_put(mobj); + if (res) + goto err; + res = binh_copy_to(binh, va_copy, 0, offs_bytes, num_bytes); + if (res) + goto err_unmap_va; + res = vm_set_prot(uctx, va_copy, num_rounded_bytes, + prot); + if (res) + goto err_unmap_va; + + /* + * The context currently is active set it again to update + * the mapping. + */ + vm_set_ctx(uctx->ts_ctx); + + if (!(flags & LDELF_MAP_FLAG_WRITEABLE)) { + res = file_add_slice(binh->f, f, offs_pages); + if (res) + goto err_unmap_va; + } + } + + res = PUT_USER_SCALAR(va_copy, va); + if (res) + goto err_unmap_va; + + file_unlock(binh->f); + + return TEE_SUCCESS; + +err_unmap_va: + unmap_or_panic(uctx, va_copy, num_rounded_bytes); + + /* + * The context currently is active set it again to update + * the mapping. + */ + vm_set_ctx(uctx->ts_ctx); + +err: + if (file_is_locked) + file_unlock(binh->f); + + return res; +} + +TEE_Result ldelf_syscall_copy_from_bin(void *dst, size_t offs, size_t num_bytes, + unsigned long handle) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(sess->ctx); + struct system_ctx *sys_ctx = sess->user_ctx; + struct bin_handle *binh = NULL; + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)dst, num_bytes); + if (res) + return res; + + if (!sys_ctx) + return TEE_ERROR_BAD_PARAMETERS; + + binh = handle_lookup(&sys_ctx->db, handle); + if (!binh) + return TEE_ERROR_BAD_PARAMETERS; + + return binh_copy_to(binh, 0, (vaddr_t)dst, offs, num_bytes); +} + +TEE_Result ldelf_syscall_set_prot(unsigned long va, size_t num_bytes, + unsigned long flags) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(sess->ctx); + size_t sz = ROUNDUP(num_bytes, SMALL_PAGE_SIZE); + uint32_t prot = TEE_MATTR_UR | TEE_MATTR_PR; + uint32_t vm_flags = 0; + vaddr_t end_va = 0; + const uint32_t accept_flags = LDELF_MAP_FLAG_WRITEABLE | + LDELF_MAP_FLAG_BTI | + LDELF_MAP_FLAG_EXECUTABLE; + + if ((flags & accept_flags) != flags) + return TEE_ERROR_BAD_PARAMETERS; + if (flags & LDELF_MAP_FLAG_WRITEABLE) + prot |= TEE_MATTR_UW | TEE_MATTR_PW; + if (flags & LDELF_MAP_FLAG_EXECUTABLE) + prot |= TEE_MATTR_UX; + if (flags & LDELF_MAP_FLAG_BTI) + prot |= TEE_MATTR_GUARDED; + + /* + * The vm_get_flags() and vm_unmap() are supposed to detect or handle + * overflow directly or indirectly. However, since this function is an + * API function it's worth having an extra guard here. If nothing else, + * to increase code clarity. + */ + if (ADD_OVERFLOW(va, sz, &end_va)) + return TEE_ERROR_BAD_PARAMETERS; + + res = vm_get_flags(uctx, va, sz, &vm_flags); + if (res) + return res; + if (vm_flags & VM_FLAG_PERMANENT) + return TEE_ERROR_ACCESS_DENIED; + + /* + * If the segment is a mapping of a part of a file (vm_flags & + * VM_FLAG_READONLY) it cannot be made writeable as all mapped + * files are mapped read-only. + */ + if ((vm_flags & VM_FLAG_READONLY) && + (prot & (TEE_MATTR_UW | TEE_MATTR_PW))) + return TEE_ERROR_ACCESS_DENIED; + + return vm_set_prot(uctx, va, sz, prot); +} + +TEE_Result ldelf_syscall_remap(unsigned long old_va, vaddr_t *new_va, + size_t num_bytes, size_t pad_begin, + size_t pad_end) +{ + TEE_Result res = TEE_SUCCESS; + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(sess->ctx); + uint32_t vm_flags = 0; + vaddr_t va_copy = 0; + + res = GET_USER_SCALAR(va_copy, new_va); + if (res) + return res; + res = vm_get_flags(uctx, old_va, num_bytes, &vm_flags); + if (res) + return res; + if (vm_flags & VM_FLAG_PERMANENT) + return TEE_ERROR_ACCESS_DENIED; + + res = vm_remap(uctx, &va_copy, old_va, num_bytes, pad_begin, pad_end); + if (res) + return res; + + res = PUT_USER_SCALAR(va_copy, new_va); + if (res) { + TEE_Result res2 = TEE_SUCCESS; + vaddr_t va = old_va; + + res2 = vm_remap(uctx, &va, va_copy, num_bytes, 0, 0); + if (res2) { + EMSG("vm_remap(%#"PRIxVA", %#"PRIxVA", %#zx) returned %#"PRIx32, + va, va_copy, num_bytes, res2); + panic("Can't restore memory map"); + } + return res; + } + + return TEE_SUCCESS; +} + +TEE_Result ldelf_syscall_gen_rnd_num(void *buf, size_t num_bytes) +{ + TEE_Result res = TEE_SUCCESS; + void *bb = NULL; + + bb = bb_alloc(num_bytes); + if (!bb) + return TEE_ERROR_OUT_OF_MEMORY; + + res = crypto_rng_read(bb, num_bytes); + if (res) + return res; + + return copy_to_user(buf, bb, num_bytes); +} + +/* + * Should be called after returning from ldelf. If user_ctx is not NULL means + * that ldelf crashed or otherwise didn't complete properly. This function will + * close the remaining handles and free the context structs allocated by ldelf. + */ +void ldelf_sess_cleanup(struct ts_session *sess) +{ + struct system_ctx *sys_ctx = sess->user_ctx; + + if (sys_ctx) { + handle_db_destroy(&sys_ctx->db, bin_close); + free(sys_ctx); + sess->user_ctx = NULL; + } +} diff --git a/optee_os/core/kernel/lockdep.c b/optee_os/core/kernel/lockdep.c new file mode 100644 index 0000000..5952e5d --- /dev/null +++ b/optee_os/core/kernel/lockdep.c @@ -0,0 +1,513 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* lockdep_node::flags values */ +/* Flags used for depth-first topological sorting */ +#define LOCKDEP_NODE_TEMP_MARK BIT(0) +#define LOCKDEP_NODE_PERM_MARK BIT(1) +/* Flag used during breadth-first search (print shortest cycle) */ +#define LOCKDEP_NODE_BFS_VISITED BIT(2) + +/* Find node in graph or add it */ +static struct lockdep_node *lockdep_add_to_graph( + struct lockdep_node_head *graph, + uintptr_t lock_id) +{ + struct lockdep_node *node = NULL; + + assert(graph); + TAILQ_FOREACH(node, graph, link) + if (node->lock_id == lock_id) + return node; + + node = calloc(1, sizeof(*node)); + if (!node) + return NULL; + + node->lock_id = lock_id; + STAILQ_INIT(&node->edges); + TAILQ_INSERT_TAIL(graph, node, link); + + return node; +} + +static vaddr_t *dup_call_stack(vaddr_t *stack) +{ + vaddr_t *nstack = NULL; + int n = 0; + + if (!stack) + return NULL; + + while (stack[n]) + n++; + + nstack = malloc((n + 1) * sizeof(vaddr_t)); + if (!nstack) + return NULL; + + memcpy(nstack, stack, (n + 1) * sizeof(vaddr_t)); + + return nstack; +} + +static void lockdep_print_call_stack(vaddr_t *stack) +{ + vaddr_t *p = NULL; + + if (!IS_ENABLED(CFG_LOCKDEP_RECORD_STACK)) + return; + + EMSG_RAW("Call stack:"); + for (p = stack; p && *p; p++) + EMSG_RAW(" %#" PRIxPTR, *p); +} + +static TEE_Result lockdep_add_edge(struct lockdep_node *from, + struct lockdep_node *to, + vaddr_t *call_stack_from, + vaddr_t *call_stack_to, + uintptr_t thread_id) +{ + struct lockdep_edge *edge = NULL; + + STAILQ_FOREACH(edge, &from->edges, link) + if (edge->to == to) + return TEE_SUCCESS; + + edge = calloc(1, sizeof(*edge)); + if (!edge) + return TEE_ERROR_OUT_OF_MEMORY; + edge->to = to; + edge->call_stack_from = dup_call_stack(call_stack_from); + edge->call_stack_to = dup_call_stack(call_stack_to); + edge->thread_id = thread_id; + STAILQ_INSERT_TAIL(&from->edges, edge, link); + + return TEE_SUCCESS; +} + +struct lockdep_bfs { + struct lockdep_node *node; + uintptr_t *path; + int pathlen; + TAILQ_ENTRY(lockdep_bfs) link; +}; + +TAILQ_HEAD(lockdep_bfs_head, lockdep_bfs); + +static void lockdep_bfs_queue_delete(struct lockdep_bfs_head *queue) +{ + struct lockdep_bfs *cur = NULL; + struct lockdep_bfs *next = NULL; + + TAILQ_FOREACH_SAFE(cur, queue, link, next) { + TAILQ_REMOVE(queue, cur, link); + free(cur->path); + free(cur); + } +} + +/* + * Print shortest cycle in @graph that contains @node. + * This function performs an iterative breadth-first search starting from @node, + * and stops when it reaches @node again. In each node we're tracking the path + * from the start node. + */ +static uintptr_t *lockdep_graph_get_shortest_cycle(struct lockdep_node *node) +{ + struct lockdep_bfs_head queue; + struct lockdep_bfs *qe = NULL; + uintptr_t *ret = NULL; + + TAILQ_INIT(&queue); + node->flags |= LOCKDEP_NODE_BFS_VISITED; + + qe = calloc(1, sizeof(*qe)); + if (!qe) + goto out; + qe->node = node; + qe->path = malloc(sizeof(uintptr_t)); + if (!qe->path) + goto out; + qe->path[0] = node->lock_id; + qe->pathlen = 1; + TAILQ_INSERT_TAIL(&queue, qe, link); + + while (!TAILQ_EMPTY(&queue)) { + struct lockdep_node *n = NULL; + struct lockdep_edge *e = NULL; + + qe = TAILQ_FIRST(&queue); + n = qe->node; + TAILQ_REMOVE(&queue, qe, link); + + STAILQ_FOREACH(e, &n->edges, link) { + if (e->to->lock_id == node->lock_id) { + uintptr_t *tmp = NULL; + size_t nlen = qe->pathlen + 1; + + /* + * Cycle found. Terminate cycle path with NULL + * and return it. + */ + tmp = realloc(qe->path, + nlen * sizeof(uintptr_t)); + if (!tmp) { + EMSG("Out of memory"); + free(qe->path); + ret = NULL; + goto out; + } + qe->path = tmp; + qe->path[nlen - 1] = 0; + ret = qe->path; + goto out; + } + + if (!(e->to->flags & LOCKDEP_NODE_BFS_VISITED)) { + size_t nlen = 0; + struct lockdep_bfs *nqe = NULL; + + e->to->flags |= LOCKDEP_NODE_BFS_VISITED; + + nlen = qe->pathlen + 1; + nqe = calloc(1, sizeof(*nqe)); + if (!nqe) + goto out; + nqe->node = e->to; + nqe->path = malloc(nlen * sizeof(uintptr_t)); + if (!nqe->path) + goto out; + nqe->pathlen = nlen; + memcpy(nqe->path, qe->path, + qe->pathlen * sizeof(uintptr_t)); + nqe->path[nlen - 1] = e->to->lock_id; + TAILQ_INSERT_TAIL(&queue, nqe, link); + } + } + free(qe->path); + free(qe); + qe = NULL; + } + +out: + free(qe); + lockdep_bfs_queue_delete(&queue); + return ret; +} + +static TEE_Result lockdep_visit(struct lockdep_node *node) +{ + struct lockdep_edge *e = NULL; + + if (node->flags & LOCKDEP_NODE_PERM_MARK) + return TEE_SUCCESS; + + if (node->flags & LOCKDEP_NODE_TEMP_MARK) + return TEE_ERROR_BAD_STATE; /* Not a DAG! */ + + node->flags |= LOCKDEP_NODE_TEMP_MARK; + + STAILQ_FOREACH(e, &node->edges, link) { + TEE_Result res = lockdep_visit(e->to); + + if (res) + return res; + } + + node->flags |= LOCKDEP_NODE_PERM_MARK; + return TEE_SUCCESS; +} + +static TEE_Result lockdep_graph_sort(struct lockdep_node_head *graph) +{ + struct lockdep_node *node = NULL; + + TAILQ_FOREACH(node, graph, link) { + if (!node->flags) { + /* Unmarked node */ + TEE_Result res = lockdep_visit(node); + + if (res) + return res; + } + } + + TAILQ_FOREACH(node, graph, link) + node->flags = 0; + + return TEE_SUCCESS; +} + +static struct lockdep_edge *lockdep_find_edge(struct lockdep_node_head *graph, + uintptr_t from, uintptr_t to) +{ + struct lockdep_node *node = NULL; + struct lockdep_edge *edge = NULL; + + TAILQ_FOREACH(node, graph, link) + if (node->lock_id == from) + STAILQ_FOREACH(edge, &node->edges, link) + if (edge->to->lock_id == to) + return edge; + return NULL; +} + +static void lockdep_print_edge_info(uintptr_t from __maybe_unused, + struct lockdep_edge *edge) +{ + uintptr_t __maybe_unused to = edge->to->lock_id; + const char __maybe_unused *at_msg = ""; + const char __maybe_unused *acq_msg = ""; + + if (IS_ENABLED(CFG_LOCKDEP_RECORD_STACK)) { + at_msg = " at:"; + acq_msg = " acquired at:"; + } + + EMSG_RAW("-> Thread %#" PRIxPTR " acquired lock %#" PRIxPTR "%s", + edge->thread_id, to, at_msg); + lockdep_print_call_stack(edge->call_stack_to); + EMSG_RAW("...while holding lock %#" PRIxPTR "%s", + from, acq_msg); + lockdep_print_call_stack(edge->call_stack_from); +} + +/* + * Find cycle containing @node in the lock graph, then print full debug + * information about each edge (thread that acquired the locks and call stacks) + */ +static void lockdep_print_cycle_info(struct lockdep_node_head *graph, + struct lockdep_node *node) +{ + struct lockdep_edge *edge = NULL; + uintptr_t *cycle = NULL; + uintptr_t *p = NULL; + uintptr_t from = 0; + uintptr_t to = 0; + + cycle = lockdep_graph_get_shortest_cycle(node); + assert(cycle && cycle[0]); + EMSG_RAW("-> Shortest cycle:"); + for (p = cycle; *p; p++) + EMSG_RAW(" Lock %#" PRIxPTR, *p); + for (p = cycle; ; p++) { + if (!*p) { + assert(p != cycle); + from = to; + to = cycle[0]; + edge = lockdep_find_edge(graph, from, to); + lockdep_print_edge_info(from, edge); + break; + } + if (p != cycle) + from = to; + to = *p; + if (p != cycle) { + edge = lockdep_find_edge(graph, from, to); + lockdep_print_edge_info(from, edge); + } + } + free(cycle); +} + +static vaddr_t *lockdep_get_kernel_stack(void) +{ + if (IS_ENABLED(CFG_LOCKDEP_RECORD_STACK)) + return unw_get_kernel_stack(); + + return NULL; +} + +TEE_Result __lockdep_lock_acquire(struct lockdep_node_head *graph, + struct lockdep_lock_head *owned, + uintptr_t id) +{ + struct lockdep_node *node = lockdep_add_to_graph(graph, id); + struct lockdep_lock *lock = NULL; + TEE_Result res = TEE_SUCCESS; + vaddr_t *acq_stack = NULL; + + if (!node) + return TEE_ERROR_OUT_OF_MEMORY; + + acq_stack = lockdep_get_kernel_stack(); + + TAILQ_FOREACH(lock, owned, link) { + res = lockdep_add_edge(lock->node, node, lock->call_stack, + acq_stack, (uintptr_t)owned); + if (res) + return res; + } + + res = lockdep_graph_sort(graph); + if (res) { + EMSG_RAW("Potential deadlock detected!"); + EMSG_RAW("When trying to acquire lock %#" PRIxPTR, id); + lockdep_print_cycle_info(graph, node); + return res; + } + + lock = calloc(1, sizeof(*lock)); + if (!lock) + return TEE_ERROR_OUT_OF_MEMORY; + + lock->node = node; + lock->call_stack = acq_stack; + TAILQ_INSERT_TAIL(owned, lock, link); + + return TEE_SUCCESS; +} + +/* + * Call this when it is known that the thread has been able to acquire the lock. + * Similar to __lockdep_lock_acquire(), but since the operation is non-blocking, + * no dependency to currently owned locks are created. + */ +TEE_Result __lockdep_lock_tryacquire(struct lockdep_node_head *graph, + struct lockdep_lock_head *owned, + uintptr_t id) +{ + struct lockdep_node *node = lockdep_add_to_graph(graph, id); + struct lockdep_lock *lock = NULL; + vaddr_t *acq_stack = NULL; + + if (!node) + return TEE_ERROR_OUT_OF_MEMORY; + + acq_stack = lockdep_get_kernel_stack(); + + lock = calloc(1, sizeof(*lock)); + if (!lock) + return TEE_ERROR_OUT_OF_MEMORY; + + lock->node = node; + lock->call_stack = acq_stack; + TAILQ_INSERT_TAIL(owned, lock, link); + + return TEE_SUCCESS; +} + +TEE_Result __lockdep_lock_release(struct lockdep_lock_head *owned, uintptr_t id) +{ + struct lockdep_lock *lock = NULL; + + TAILQ_FOREACH_REVERSE(lock, owned, lockdep_lock_head, link) { + if (lock->node->lock_id == id) { + TAILQ_REMOVE(owned, lock, link); + free(lock->call_stack); + free(lock); + return TEE_SUCCESS; + } + } + + EMSG_RAW("Thread %p does not own lock %#" PRIxPTR, (void *)owned, id); + return TEE_ERROR_ITEM_NOT_FOUND; +} + +static void lockdep_free_edge(struct lockdep_edge *edge) +{ + free(edge->call_stack_from); + free(edge->call_stack_to); + free(edge); +} + +static void lockdep_node_delete(struct lockdep_node *node) +{ + struct lockdep_edge *edge = NULL; + struct lockdep_edge *next = NULL; + + STAILQ_FOREACH_SAFE(edge, &node->edges, link, next) + lockdep_free_edge(edge); + + free(node); +} + +void lockdep_graph_delete(struct lockdep_node_head *graph) +{ + struct lockdep_node *node = NULL; + struct lockdep_node *next = NULL; + + TAILQ_FOREACH_SAFE(node, graph, link, next) { + TAILQ_REMOVE(graph, node, link); + lockdep_node_delete(node); + } +} + +void lockdep_queue_delete(struct lockdep_lock_head *owned) +{ + struct lockdep_lock *lock = NULL; + struct lockdep_lock *next = NULL; + + TAILQ_FOREACH_SAFE(lock, owned, link, next) { + TAILQ_REMOVE(owned, lock, link); + free(lock); + } +} + +static void lockdep_node_destroy(struct lockdep_node_head *graph, + struct lockdep_node *node) +{ + struct lockdep_edge *edge = NULL; + struct lockdep_edge *next = NULL; + struct lockdep_node *from = NULL; + + TAILQ_REMOVE(graph, node, link); + + /* + * Loop over all nodes in the graph to remove all edges with the + * node to remove in the "to" field. + */ + TAILQ_FOREACH(from, graph, link) { + edge = STAILQ_FIRST(&from->edges); + while (edge && edge->to == node) { + STAILQ_REMOVE_HEAD(&from->edges, link); + lockdep_free_edge(edge); + edge = STAILQ_FIRST(&from->edges); + } + + if (!edge) + continue; + + next = STAILQ_NEXT(edge, link); + while (next) { + if (next->to == node) { + STAILQ_REMOVE_AFTER(&from->edges, edge, link); + lockdep_free_edge(next); + } else { + edge = next; + } + next = STAILQ_NEXT(edge, link); + } + } + + STAILQ_FOREACH_SAFE(edge, &node->edges, link, next) + lockdep_free_edge(edge); + + free(node); +} + +void lockdep_lock_destroy(struct lockdep_node_head *graph, uintptr_t lock_id) +{ + struct lockdep_node *node = NULL; + + assert(graph); + TAILQ_FOREACH(node, graph, link) { + if (node->lock_id == lock_id) { + lockdep_node_destroy(graph, node); + break; + } + } +} diff --git a/optee_os/core/kernel/msg_param.c b/optee_os/core/kernel/msg_param.c new file mode 100644 index 0000000..5b7fec8 --- /dev/null +++ b/optee_os/core/kernel/msg_param.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, EPAM Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +/** + * msg_param_extract_pages() - extract list of pages from + * OPTEE_MSG_ATTR_NONCONTIG buffer. + * + * @buffer: pointer to parameters array + * @pages: output array of page addresses + * @num_pages: number of pages in array + * + * return: + * true on success, false otherwise + * + * @buffer points to the physical address of a structure that can be viewed as + * + * struct page_data { + * uint64_t pages_array[OPTEE_MSG_NONCONTIG_PAGE_SIZE/sizeof(uint64_t) - 1]; + * uint64_t next_page_data; + * }; + * + * So, it is a linked list of arrays, where each element of linked list fits + * exactly into one 4K page. + * + * This function extracts data from arrays into one array pointed by @pages + * + * @buffer points to data shared with normal world, so some precautions + * should be taken. + */ +static bool msg_param_extract_pages(paddr_t buffer, paddr_t *pages, + size_t num_pages) +{ + size_t cnt; + struct mobj *mobj; + paddr_t page; + uint64_t *va; + bool ret = false; + + if (buffer & SMALL_PAGE_MASK) + return false; + + /* + * There we map first page of array. + * mobj_mapped_shm_alloc() will check if page resides in nonsec ddr + */ + mobj = mobj_mapped_shm_alloc(&buffer, 1, 0, 0); + if (!mobj) + return false; + + va = mobj_get_va(mobj, 0, SMALL_PAGE_SIZE); + assert(va); + + for (cnt = 0; cnt < num_pages; cnt++, va++) { + /* + * If we about to roll over page boundary, then last entry holds + * address of next page of array. Unmap current page and map + * next one + */ + if (!((vaddr_t)(va + 1) & SMALL_PAGE_MASK)) { + page = *va; + if (page & SMALL_PAGE_MASK) + goto out; + + mobj_put(mobj); + mobj = mobj_mapped_shm_alloc(&page, 1, 0, 0); + if (!mobj) + goto out; + + va = mobj_get_va(mobj, 0, SMALL_PAGE_SIZE); + assert(va); + } + pages[cnt] = *va; + if (pages[cnt] & SMALL_PAGE_MASK) + goto out; + } + + ret = true; +out: + mobj_put(mobj); + return ret; +} + +struct mobj *msg_param_mobj_from_noncontig(paddr_t buf_ptr, size_t size, + uint64_t shm_ref, bool map_buffer) +{ + struct mobj *mobj = NULL; + paddr_t *pages = NULL; + paddr_t page_offset = 0; + size_t num_pages = 0; + size_t size_plus_offs = 0; + size_t msize = 0; + + page_offset = buf_ptr & SMALL_PAGE_MASK; + if (ADD_OVERFLOW(size, page_offset, &size_plus_offs)) + return NULL; + num_pages = (size_plus_offs - 1) / SMALL_PAGE_SIZE + 1; + if (MUL_OVERFLOW(num_pages, sizeof(paddr_t), &msize)) + return NULL; + + pages = malloc(msize); + if (!pages) + return NULL; + + if (!msg_param_extract_pages(buf_ptr & ~SMALL_PAGE_MASK, + pages, num_pages)) + goto out; + + if (map_buffer) + mobj = mobj_mapped_shm_alloc(pages, num_pages, page_offset, + shm_ref); + else + mobj = mobj_reg_shm_alloc(pages, num_pages, page_offset, + shm_ref); +out: + free(pages); + return mobj; +} diff --git a/optee_os/core/kernel/mutex.c b/optee_os/core/kernel/mutex.c new file mode 100644 index 0000000..45028c4 --- /dev/null +++ b/optee_os/core/kernel/mutex.c @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "mutex_lockdep.h" + +void mutex_init(struct mutex *m) +{ + *m = (struct mutex)MUTEX_INITIALIZER; +} + +void mutex_init_recursive(struct recursive_mutex *m) +{ + *m = (struct recursive_mutex)RECURSIVE_MUTEX_INITIALIZER; +} + +static void __mutex_lock(struct mutex *m, const char *fname, int lineno) +{ + assert_have_no_spinlock(); + assert(thread_get_id_may_fail() != THREAD_ID_INVALID); + assert(thread_is_in_normal_mode()); + + mutex_lock_check(m); + + while (true) { + uint32_t old_itr_status; + bool can_lock; + struct wait_queue_elem wqe; + + /* + * If the mutex is locked we need to initialize the wqe + * before releasing the spinlock to guarantee that we don't + * miss the wakeup from mutex_unlock(). + * + * If the mutex is unlocked we don't need to use the wqe at + * all. + */ + + old_itr_status = cpu_spin_lock_xsave(&m->spin_lock); + + can_lock = !m->state; + if (!can_lock) { + wq_wait_init(&m->wq, &wqe, false /* wait_read */); + } else { + m->state = -1; /* write locked */ + } + + cpu_spin_unlock_xrestore(&m->spin_lock, old_itr_status); + + if (!can_lock) { + /* + * Someone else is holding the lock, wait in normal + * world for the lock to become available. + */ + wq_wait_final(&m->wq, &wqe, m, fname, lineno); + } else + return; + } +} + +static void __mutex_lock_recursive(struct recursive_mutex *m, const char *fname, + int lineno) +{ + short int ct = thread_get_id(); + + assert_have_no_spinlock(); + assert(thread_is_in_normal_mode()); + + if (atomic_load_short(&m->owner) == ct) { + if (!refcount_inc(&m->lock_depth)) + panic(); + return; + } + + __mutex_lock(&m->m, fname, lineno); + + assert(m->owner == THREAD_ID_INVALID); + atomic_store_short(&m->owner, ct); + refcount_set(&m->lock_depth, 1); +} + +static void __mutex_unlock(struct mutex *m, const char *fname, int lineno) +{ + uint32_t old_itr_status; + + assert_have_no_spinlock(); + assert(thread_get_id_may_fail() != THREAD_ID_INVALID); + + mutex_unlock_check(m); + + old_itr_status = cpu_spin_lock_xsave(&m->spin_lock); + + if (!m->state) + panic(); + + m->state = 0; + + cpu_spin_unlock_xrestore(&m->spin_lock, old_itr_status); + + wq_wake_next(&m->wq, m, fname, lineno); +} + +static void __mutex_unlock_recursive(struct recursive_mutex *m, + const char *fname, int lineno) +{ + assert_have_no_spinlock(); + assert(m->owner == thread_get_id()); + + if (refcount_dec(&m->lock_depth)) { + /* + * Do an atomic store to match the atomic load in + * __mutex_lock_recursive() + */ + atomic_store_short(&m->owner, THREAD_ID_INVALID); + __mutex_unlock(&m->m, fname, lineno); + } +} + +static bool __mutex_trylock(struct mutex *m, const char *fname __unused, + int lineno __unused) +{ + uint32_t old_itr_status; + bool can_lock_write; + + assert_have_no_spinlock(); + assert(thread_get_id_may_fail() != THREAD_ID_INVALID); + + old_itr_status = cpu_spin_lock_xsave(&m->spin_lock); + + can_lock_write = !m->state; + if (can_lock_write) + m->state = -1; + + cpu_spin_unlock_xrestore(&m->spin_lock, old_itr_status); + + if (can_lock_write) + mutex_trylock_check(m); + + return can_lock_write; +} + +static void __mutex_read_unlock(struct mutex *m, const char *fname, int lineno) +{ + uint32_t old_itr_status; + short new_state; + + assert_have_no_spinlock(); + assert(thread_get_id_may_fail() != THREAD_ID_INVALID); + + old_itr_status = cpu_spin_lock_xsave(&m->spin_lock); + + if (m->state <= 0) + panic(); + m->state--; + new_state = m->state; + + cpu_spin_unlock_xrestore(&m->spin_lock, old_itr_status); + + /* Wake eventual waiters if the mutex was unlocked */ + if (!new_state) + wq_wake_next(&m->wq, m, fname, lineno); +} + +static void __mutex_read_lock(struct mutex *m, const char *fname, int lineno) +{ + assert_have_no_spinlock(); + assert(thread_get_id_may_fail() != THREAD_ID_INVALID); + assert(thread_is_in_normal_mode()); + + while (true) { + uint32_t old_itr_status; + bool can_lock; + struct wait_queue_elem wqe; + + /* + * If the mutex is locked we need to initialize the wqe + * before releasing the spinlock to guarantee that we don't + * miss the wakeup from mutex_unlock(). + * + * If the mutex is unlocked we don't need to use the wqe at + * all. + */ + + old_itr_status = cpu_spin_lock_xsave(&m->spin_lock); + + can_lock = m->state != -1; + if (!can_lock) { + wq_wait_init(&m->wq, &wqe, true /* wait_read */); + } else { + m->state++; /* read_locked */ + } + + cpu_spin_unlock_xrestore(&m->spin_lock, old_itr_status); + + if (!can_lock) { + /* + * Someone else is holding the lock, wait in normal + * world for the lock to become available. + */ + wq_wait_final(&m->wq, &wqe, m, fname, lineno); + } else + return; + } +} + +static bool __mutex_read_trylock(struct mutex *m, const char *fname __unused, + int lineno __unused) +{ + uint32_t old_itr_status; + bool can_lock; + + assert_have_no_spinlock(); + assert(thread_get_id_may_fail() != THREAD_ID_INVALID); + assert(thread_is_in_normal_mode()); + + old_itr_status = cpu_spin_lock_xsave(&m->spin_lock); + + can_lock = m->state != -1; + if (can_lock) + m->state++; + + cpu_spin_unlock_xrestore(&m->spin_lock, old_itr_status); + + return can_lock; +} + +#ifdef CFG_MUTEX_DEBUG +void mutex_unlock_debug(struct mutex *m, const char *fname, int lineno) +{ + __mutex_unlock(m, fname, lineno); +} + +void mutex_lock_debug(struct mutex *m, const char *fname, int lineno) +{ + __mutex_lock(m, fname, lineno); +} + +bool mutex_trylock_debug(struct mutex *m, const char *fname, int lineno) +{ + return __mutex_trylock(m, fname, lineno); +} + +void mutex_read_unlock_debug(struct mutex *m, const char *fname, int lineno) +{ + __mutex_read_unlock(m, fname, lineno); +} + +void mutex_read_lock_debug(struct mutex *m, const char *fname, int lineno) +{ + __mutex_read_lock(m, fname, lineno); +} + +bool mutex_read_trylock_debug(struct mutex *m, const char *fname, int lineno) +{ + return __mutex_read_trylock(m, fname, lineno); +} + +void mutex_unlock_recursive_debug(struct recursive_mutex *m, const char *fname, + int lineno) +{ + __mutex_unlock_recursive(m, fname, lineno); +} + +void mutex_lock_recursive_debug(struct recursive_mutex *m, const char *fname, + int lineno) +{ + __mutex_lock_recursive(m, fname, lineno); +} +#else +void mutex_unlock(struct mutex *m) +{ + __mutex_unlock(m, NULL, -1); +} + +void mutex_unlock_recursive(struct recursive_mutex *m) +{ + __mutex_unlock_recursive(m, NULL, -1); +} + +void mutex_lock(struct mutex *m) +{ + __mutex_lock(m, NULL, -1); +} + +void mutex_lock_recursive(struct recursive_mutex *m) +{ + __mutex_lock_recursive(m, NULL, -1); +} + +bool mutex_trylock(struct mutex *m) +{ + return __mutex_trylock(m, NULL, -1); +} + +void mutex_read_unlock(struct mutex *m) +{ + __mutex_read_unlock(m, NULL, -1); +} + +void mutex_read_lock(struct mutex *m) +{ + __mutex_read_lock(m, NULL, -1); +} + +bool mutex_read_trylock(struct mutex *m) +{ + return __mutex_read_trylock(m, NULL, -1); +} +#endif + +void mutex_destroy(struct mutex *m) +{ + /* + * Caller guarantees that no one will try to take the mutex so + * there's no need to take the spinlock before accessing it. + */ + if (m->state) + panic(); + if (!wq_is_empty(&m->wq)) + panic("waitqueue not empty"); + mutex_destroy_check(m); +} + +void mutex_destroy_recursive(struct recursive_mutex *m) +{ + mutex_destroy(&m->m); +} + +unsigned int mutex_get_recursive_lock_depth(struct recursive_mutex *m) +{ + assert_have_no_spinlock(); + assert(m->owner == thread_get_id()); + + return refcount_val(&m->lock_depth); +} + +void condvar_init(struct condvar *cv) +{ + *cv = (struct condvar)CONDVAR_INITIALIZER; +} + +void condvar_destroy(struct condvar *cv) +{ + if (cv->m && wq_have_condvar(&cv->m->wq, cv)) + panic(); + + condvar_init(cv); +} + +static void cv_signal(struct condvar *cv, bool only_one, const char *fname, + int lineno) +{ + uint32_t old_itr_status; + struct mutex *m; + + old_itr_status = cpu_spin_lock_xsave(&cv->spin_lock); + m = cv->m; + cpu_spin_unlock_xrestore(&cv->spin_lock, old_itr_status); + + if (m) + wq_promote_condvar(&m->wq, cv, only_one, m, fname, lineno); + +} + +#ifdef CFG_MUTEX_DEBUG +void condvar_signal_debug(struct condvar *cv, const char *fname, int lineno) +{ + cv_signal(cv, true /* only one */, fname, lineno); +} + +void condvar_broadcast_debug(struct condvar *cv, const char *fname, int lineno) +{ + cv_signal(cv, false /* all */, fname, lineno); +} + +#else +void condvar_signal(struct condvar *cv) +{ + cv_signal(cv, true /* only one */, NULL, -1); +} + +void condvar_broadcast(struct condvar *cv) +{ + cv_signal(cv, false /* all */, NULL, -1); +} +#endif /*CFG_MUTEX_DEBUG*/ + +static void __condvar_wait(struct condvar *cv, struct mutex *m, + const char *fname, int lineno) +{ + uint32_t old_itr_status; + struct wait_queue_elem wqe; + short old_state; + short new_state; + + mutex_unlock_check(m); + + /* Link this condvar to this mutex until reinitialized */ + old_itr_status = cpu_spin_lock_xsave(&cv->spin_lock); + if (cv->m && cv->m != m) + panic("invalid mutex"); + + cv->m = m; + cpu_spin_unlock(&cv->spin_lock); + + cpu_spin_lock(&m->spin_lock); + + if (!m->state) + panic(); + old_state = m->state; + /* Add to mutex wait queue as a condvar waiter */ + wq_wait_init_condvar(&m->wq, &wqe, cv, m->state > 0); + + if (m->state > 1) { + /* Multiple read locks, remove one */ + m->state--; + } else { + /* Only one lock (read or write), unlock the mutex */ + m->state = 0; + } + new_state = m->state; + + cpu_spin_unlock_xrestore(&m->spin_lock, old_itr_status); + + /* Wake eventual waiters if the mutex was unlocked */ + if (!new_state) + wq_wake_next(&m->wq, m, fname, lineno); + + wq_wait_final(&m->wq, &wqe, m, fname, lineno); + + if (old_state > 0) + mutex_read_lock(m); + else + mutex_lock(m); +} + +#ifdef CFG_MUTEX_DEBUG +void condvar_wait_debug(struct condvar *cv, struct mutex *m, + const char *fname, int lineno) +{ + __condvar_wait(cv, m, fname, lineno); +} +#else +void condvar_wait(struct condvar *cv, struct mutex *m) +{ + __condvar_wait(cv, m, NULL, -1); +} +#endif diff --git a/optee_os/core/kernel/mutex_lockdep.c b/optee_os/core/kernel/mutex_lockdep.c new file mode 100644 index 0000000..2095c86 --- /dev/null +++ b/optee_os/core/kernel/mutex_lockdep.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "mutex_lockdep.h" + +/* Global graph of all mutexes used in the code */ +static struct lockdep_node_head graph = TAILQ_HEAD_INITIALIZER(graph); + +/* Protects @graph */ +static unsigned int graph_lock = SPINLOCK_UNLOCK; + +/* + * One queue per thread, contains the mutexes the thread owns at any point in + * time (in aquire order) + */ +static struct lockdep_lock_head owned[CFG_NUM_THREADS]; + +void mutex_lockdep_init(void) +{ + int n = 0; + + for (n = 0; n < CFG_NUM_THREADS; n++) + TAILQ_INIT(&owned[n]); + + DMSG("lockdep is enabled for mutexes"); +} + +void mutex_lock_check(struct mutex *m) +{ + short int thread = thread_get_id(); + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&graph_lock); + lockdep_lock_acquire(&graph, &owned[thread], (uintptr_t)m); + cpu_spin_unlock_xrestore(&graph_lock, exceptions); +} + +void mutex_trylock_check(struct mutex *m) +{ + short int thread = thread_get_id(); + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&graph_lock); + lockdep_lock_tryacquire(&graph, &owned[thread], (uintptr_t)m); + cpu_spin_unlock_xrestore(&graph_lock, exceptions); +} + +void mutex_unlock_check(struct mutex *m) +{ + short int thread = thread_get_id(); + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&graph_lock); + lockdep_lock_release(&owned[thread], (uintptr_t)m); + cpu_spin_unlock_xrestore(&graph_lock, exceptions); +} + +void mutex_destroy_check(struct mutex *m) +{ + uint32_t exceptions = cpu_spin_lock_xsave(&graph_lock); + + lockdep_lock_destroy(&graph, (uintptr_t)m); + cpu_spin_unlock_xrestore(&graph_lock, exceptions); +} diff --git a/optee_os/core/kernel/mutex_lockdep.h b/optee_os/core/kernel/mutex_lockdep.h new file mode 100644 index 0000000..119551d --- /dev/null +++ b/optee_os/core/kernel/mutex_lockdep.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ +#ifndef MUTEX_LOCKDEP_H +#define MUTEX_LOCKDEP_H + +#include +#include + +#ifdef CFG_LOCKDEP + +void mutex_lock_check(struct mutex *m); + +void mutex_trylock_check(struct mutex *m); + +void mutex_unlock_check(struct mutex *m); + +void mutex_destroy_check(struct mutex *m); + +#else + +static inline void mutex_lock_check(struct mutex *m __unused) +{} + +static inline void mutex_trylock_check(struct mutex *m __unused) +{} + +static inline void mutex_unlock_check(struct mutex *m __unused) +{} + +static inline void mutex_destroy_check(struct mutex *m __unused) +{} + +#endif /* !CFG_LOCKDEP */ + +#endif /* MUTEX_LOCKDEP_H */ diff --git a/optee_os/core/kernel/notif.c b/optee_os/core/kernel/notif.c new file mode 100644 index 0000000..7ecc246 --- /dev/null +++ b/optee_os/core/kernel/notif.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CFG_CORE_ASYNC_NOTIF) +static struct mutex notif_mutex = MUTEX_INITIALIZER; +static unsigned int notif_lock = SPINLOCK_UNLOCK; + +SLIST_HEAD(notif_driver_head, notif_driver); +static struct notif_driver_head notif_driver_head = + SLIST_HEAD_INITIALIZER(¬if_driver_head); + +static bitstr_t bit_decl(notif_values, NOTIF_ASYNC_VALUE_MAX + 1); +static bitstr_t bit_decl(notif_alloc_values, NOTIF_ASYNC_VALUE_MAX + 1); +static bool notif_started; + +TEE_Result notif_alloc_async_value(uint32_t *val) +{ + static bool alloc_values_inited; + uint32_t old_itr_status = 0; + int bit = 0; + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + if (!alloc_values_inited) { + bit_set(notif_alloc_values, NOTIF_VALUE_DO_BOTTOM_HALF); + alloc_values_inited = true; + } + + bit_ffc(notif_alloc_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); + if (bit >= 0) { + *val = bit; + bit_set(notif_alloc_values, bit); + } + + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); + + if (bit < 0) + return TEE_ERROR_OUT_OF_MEMORY; + + return TEE_SUCCESS; +} + +void notif_free_async_value(uint32_t val) +{ + uint32_t old_itr_status = 0; + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + assert(val < NOTIF_ASYNC_VALUE_MAX); + assert(bit_test(notif_alloc_values, val)); + bit_clear(notif_alloc_values, val); + + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); +} + +uint32_t notif_get_value(bool *value_valid, bool *value_pending) +{ + uint32_t old_itr_status = 0; + uint32_t res = 0; + int bit = 0; + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + bit_ffs(notif_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); + *value_valid = (bit >= 0); + if (!*value_valid) { + *value_pending = false; + goto out; + } + + res = bit; + bit_clear(notif_values, res); + bit_ffs(notif_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); + *value_pending = (bit >= 0); +out: + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); + + return res; +} + +void notif_send_async(uint32_t value) +{ + uint32_t old_itr_status = 0; + + static_assert(CFG_CORE_ASYNC_NOTIF_GIC_INTID >= GIC_PPI_BASE); + + assert(value <= NOTIF_ASYNC_VALUE_MAX); + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + DMSG("0x%"PRIx32, value); + bit_set(notif_values, value); + itr_raise_pi(CFG_CORE_ASYNC_NOTIF_GIC_INTID); + + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); +} + +bool notif_async_is_started(void) +{ + uint32_t old_itr_status = 0; + bool ret = false; + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + ret = notif_started; + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); + + return ret; +} + +void notif_register_driver(struct notif_driver *ndrv) +{ + uint32_t old_itr_status = 0; + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + SLIST_INSERT_HEAD(¬if_driver_head, ndrv, link); + + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); +} + +void notif_unregister_driver(struct notif_driver *ndrv) +{ + uint32_t old_itr_status = 0; + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + SLIST_REMOVE(¬if_driver_head, ndrv, notif_driver, link); + + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); +} + +void notif_deliver_atomic_event(enum notif_event ev) +{ + uint32_t old_itr_status = 0; + struct notif_driver *nd = NULL; + + assert(ev == NOTIF_EVENT_STARTED); + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + if (notif_started) { + DMSG("Already started"); + goto out; + } + notif_started = true; + + SLIST_FOREACH(nd, ¬if_driver_head, link) + if (nd->atomic_cb) + nd->atomic_cb(nd, ev); + +out: + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); +} + +void notif_deliver_event(enum notif_event ev) +{ + uint32_t old_itr_status = 0; + struct notif_driver *nd = NULL; + struct notif_driver *nd_tmp = NULL; + + assert(ev == NOTIF_EVENT_DO_BOTTOM_HALF || ev == NOTIF_EVENT_STOPPED); + + /* Serialize all yielding notifications */ + mutex_lock(¬if_mutex); + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + if (!notif_started) { + DMSG("Not started ev %d", (int)ev); + goto out; + } + + if (ev == NOTIF_EVENT_STOPPED) + notif_started = false; + + SLIST_FOREACH_SAFE(nd, ¬if_driver_head, link, nd_tmp) { + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); + + if (nd->yielding_cb) + nd->yielding_cb(nd, ev); + + old_itr_status = cpu_spin_lock_xsave(¬if_lock); + + if (ev == NOTIF_EVENT_STOPPED && notif_started) { + DMSG("Started again while stopping"); + goto out; + } + } + +out: + cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); + mutex_unlock(¬if_mutex); +} +#endif /*CFG_CORE_ASYNC_NOTIF*/ + +static TEE_Result notif_rpc(uint32_t func, uint32_t value) +{ + struct thread_param params = THREAD_PARAM_VALUE(IN, func, value, 0); + + return thread_rpc_cmd(OPTEE_RPC_CMD_NOTIFICATION, 1, ¶ms); +} + +TEE_Result notif_wait(uint32_t value) +{ + return notif_rpc(OPTEE_RPC_NOTIFICATION_WAIT, value); +} + +TEE_Result notif_send_sync(uint32_t value) +{ + return notif_rpc(OPTEE_RPC_NOTIFICATION_SEND, value); +} diff --git a/optee_os/core/kernel/nv_counter.c b/optee_os/core/kernel/nv_counter.c new file mode 100644 index 0000000..ce397f2 --- /dev/null +++ b/optee_os/core/kernel/nv_counter.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023, Linaro Limited + */ + +#include +#include + +TEE_Result __weak nv_counter_get_ree_fs(uint32_t *value __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +TEE_Result __weak nv_counter_incr_ree_fs_to(uint32_t value __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} diff --git a/optee_os/core/kernel/otp_stubs.c b/optee_os/core/kernel/otp_stubs.c new file mode 100644 index 0000000..e570b1a --- /dev/null +++ b/optee_os/core/kernel/otp_stubs.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +/* + * Override these in your platform code to really fetch device-unique + * bits from e-fuses or whatever. + * + * The default implementation just sets it to a constant. + */ + +__weak TEE_Result tee_otp_get_hw_unique_key(struct tee_hw_unique_key *hwkey) +{ + memset(&hwkey->data[0], 0, sizeof(hwkey->data)); + return TEE_SUCCESS; +} + +__weak int tee_otp_get_die_id(uint8_t *buffer, size_t len) +{ + if (huk_subkey_derive(HUK_SUBKEY_DIE_ID, NULL, 0, buffer, len)) + return -1; + + return 0; +} + +#ifdef CFG_WITH_USER_TA +/* + * Override this API on your platform to provide TA encryption key as + * per your security requirements. There can be two options for this key: + * + * 1) Unique per device encryption key. + * 2) Class wide encryption key. + * + * The default implementation chooses option (1). + */ +__weak TEE_Result tee_otp_get_ta_enc_key(uint32_t key_type __maybe_unused, + uint8_t *buffer, size_t len) +{ + assert(key_type == SHDR_ENC_KEY_DEV_SPECIFIC); + + if (huk_subkey_derive(HUK_SUBKEY_TA_ENC, ta_pub_key_modulus, + ta_pub_key_modulus_size, buffer, len)) + return TEE_ERROR_SECURITY; + + return TEE_SUCCESS; +} +#endif diff --git a/optee_os/core/kernel/panic.c b/optee_os/core/kernel/panic.c new file mode 100644 index 0000000..18028fb --- /dev/null +++ b/optee_os/core/kernel/panic.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include + +void __do_panic(const char *file __maybe_unused, + const int line __maybe_unused, + const char *func __maybe_unused, + const char *msg __maybe_unused) +{ + /* disable prehemption */ + (void)thread_mask_exceptions(THREAD_EXCP_ALL); + + /* TODO: notify other cores */ + + /* trace: Panic ['panic-string-message' ]at FILE:LINE []" */ + if (!file && !func && !msg) + EMSG_RAW("Panic"); + else + EMSG_RAW("Panic %s%s%sat %s:%d %s%s%s", + msg ? "'" : "", msg ? msg : "", msg ? "' " : "", + file ? file : "?", file ? line : 0, + func ? "<" : "", func ? func : "", func ? ">" : ""); + + print_kernel_stack(); + /* abort current execution */ + while (1) + cpu_idle(); +} + +void __weak cpu_idle(void) +{ +} diff --git a/optee_os/core/kernel/pm.c b/optee_os/core/kernel/pm.c new file mode 100644 index 0000000..4146464 --- /dev/null +++ b/optee_os/core/kernel/pm.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PM_FLAG_SUSPENDED BIT(0) + +static struct pm_callback_handle *pm_cb_ref; +static size_t pm_cb_count; + +static const char no_name[] = "no-name"; +DECLARE_KEEP_PAGER(no_name); + +static void verify_cb_args(struct pm_callback_handle *pm_hdl) +{ + if (is_unpaged((void *)(vaddr_t)pm_change_state) && + (!is_unpaged((void *)(vaddr_t)pm_hdl->callback) || + (pm_hdl->handle && !is_unpaged(pm_hdl->handle)))) { + EMSG("PM callbacks mandates unpaged arguments: %p %p", + (void *)(vaddr_t)pm_hdl->callback, pm_hdl->handle); + panic(); + } +} + +void register_pm_cb(struct pm_callback_handle *pm_hdl) +{ + struct pm_callback_handle *ref = NULL; + const char *name = pm_hdl->name; + size_t count = pm_cb_count; + + verify_cb_args(pm_hdl); + + if (!name) + name = no_name; + + if (!is_unpaged((void *)name)) { + name = strdup(name); + if (!name) + panic(); + } + + ref = realloc(pm_cb_ref, sizeof(*ref) * (count + 1)); + if (!ref) + panic(); + + ref[count] = *pm_hdl; + ref[count].flags = 0; + ref[count].name = name; + + pm_cb_count = count + 1; + pm_cb_ref = ref; +} + +static TEE_Result do_pm_callback(enum pm_op op, uint32_t pm_hint, + struct pm_callback_handle *hdl) +{ + TEE_Result res = TEE_ERROR_GENERIC; + bool suspending = op == PM_OP_SUSPEND; + + if (suspending == (bool)(hdl->flags & PM_FLAG_SUSPENDED)) + return TEE_SUCCESS; + + DMSG("%s %s", suspending ? "Suspend" : "Resume", hdl->name); + + res = hdl->callback(op, pm_hint, hdl); + if (res) { + EMSG("%s %s (%p) failed: %#"PRIx32, suspending ? "Suspend" : + "Resume", hdl->name, (void *)(vaddr_t)hdl->callback, res); + return res; + } + + if (suspending) + hdl->flags |= PM_FLAG_SUSPENDED; + else + hdl->flags &= ~PM_FLAG_SUSPENDED; + + return TEE_SUCCESS; +} + +static TEE_Result call_callbacks(enum pm_op op, uint32_t pm_hint, + enum pm_callback_order order) +{ + struct pm_callback_handle *hdl = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + size_t n = 0; + + /* + * Suspend first the last registered instances. + * Resume first the first registered instances. + */ + if (op == PM_OP_SUSPEND) + hdl = pm_cb_ref + pm_cb_count - 1; + else + hdl = pm_cb_ref; + + for (n = 0; n < pm_cb_count; n++) { + if (hdl->order == order) { + res = do_pm_callback(op, pm_hint, hdl); + if (res) + return res; + } + + if (op == PM_OP_SUSPEND) + hdl--; + else + hdl++; + } + + return TEE_SUCCESS; +} + +TEE_Result pm_change_state(enum pm_op op, uint32_t pm_hint) +{ + enum pm_callback_order cnt = PM_CB_ORDER_DRIVER; + TEE_Result res = TEE_ERROR_GENERIC; + + switch (op) { + case PM_OP_SUSPEND: + for (cnt = PM_CB_ORDER_DRIVER; cnt < PM_CB_ORDER_MAX; cnt++) { + res = call_callbacks(op, pm_hint, cnt); + if (res) + return res; + } + break; + case PM_OP_RESUME: + for (cnt = PM_CB_ORDER_MAX; cnt > PM_CB_ORDER_DRIVER; cnt--) { + res = call_callbacks(op, pm_hint, cnt - 1); + if (res) + return res; + } + break; + default: + panic(); + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/kernel/pseudo_ta.c b/optee_os/core/kernel/pseudo_ta.c new file mode 100644 index 0000000..828a1d8 --- /dev/null +++ b/optee_os/core/kernel/pseudo_ta.c @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_SECURE_DATA_PATH +static bool client_is_secure(struct ts_session *s) +{ + /* rely on core entry to have constrained client IDs */ + if (to_ta_session(s)->clnt_id.login == TEE_LOGIN_TRUSTED_APP) + return true; + + return false; +} + +static bool validate_in_param(struct ts_session *s, struct mobj *mobj) +{ + /* Supplying NULL to query buffer size is OK */ + if (!mobj) + return true; + + /* for secure clients, core entry always holds valid memref objects */ + if (client_is_secure(s)) + return true; + + /* all non-secure memory references are handled by PTAs */ + if (mobj_is_nonsec(mobj)) + return true; + + return false; +} +#else +static bool validate_in_param(struct ts_session *s __unused, + struct mobj *mobj __unused) +{ + /* At this point, core has filled only valid accessible memref mobj */ + return true; +} +#endif + +/* Maps pseudo TA params */ +static TEE_Result copy_in_param(struct ts_session *s __maybe_unused, + struct tee_ta_param *param, + TEE_Param tee_param[TEE_NUM_PARAMS], + bool did_map[TEE_NUM_PARAMS]) +{ + size_t n; + void *va; + struct param_mem *mem; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(param->types, n)) { + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + tee_param[n].value.a = param->u[n].val.a; + tee_param[n].value.b = param->u[n].val.b; + break; + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + mem = ¶m->u[n].mem; + if (!validate_in_param(s, mem->mobj)) + return TEE_ERROR_BAD_PARAMETERS; + if (mem->size) { + TEE_Result res = mobj_inc_map(mem->mobj); + + if (res) + return res; + did_map[n] = true; + va = mobj_get_va(mem->mobj, mem->offs, + mem->size); + if (!va) + return TEE_ERROR_BAD_PARAMETERS; + } else { + va = NULL; + } + + tee_param[n].memref.buffer = va; + tee_param[n].memref.size = mem->size; + break; + default: + memset(tee_param + n, 0, sizeof(TEE_Param)); + break; + } + } + + return TEE_SUCCESS; +} + +static void update_out_param(TEE_Param tee_param[TEE_NUM_PARAMS], + struct tee_ta_param *param) +{ + size_t n; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(param->types, n)) { + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + param->u[n].val.a = tee_param[n].value.a; + param->u[n].val.b = tee_param[n].value.b; + break; + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + param->u[n].mem.size = tee_param[n].memref.size; + break; + default: + break; + } + } +} + +static void unmap_mapped_param(struct tee_ta_param *param, + bool did_map[TEE_NUM_PARAMS]) +{ + size_t n; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + if (did_map[n]) { + TEE_Result res __maybe_unused; + + res = mobj_dec_map(param->u[n].mem.mobj); + assert(!res); + } + } +} + +static TEE_Result pseudo_ta_enter_open_session(struct ts_session *s) +{ + TEE_Result res = TEE_SUCCESS; + struct pseudo_ta_ctx *stc = to_pseudo_ta_ctx(s->ctx); + struct tee_ta_session *ta_sess = to_ta_session(s); + TEE_Param tee_param[TEE_NUM_PARAMS] = { }; + bool did_map[TEE_NUM_PARAMS] = { false }; + + ts_push_current_session(s); + ta_sess->err_origin = TEE_ORIGIN_TRUSTED_APP; + + if (stc->ctx.ref_count == 1 && stc->pseudo_ta->create_entry_point) { + res = stc->pseudo_ta->create_entry_point(); + if (res != TEE_SUCCESS) + goto out; + } + + if (stc->pseudo_ta->open_session_entry_point) { + void **user_ctx = &s->user_ctx; + uint32_t param_types = 0; + + if (ta_sess->param) { + res = copy_in_param(s, ta_sess->param, tee_param, + did_map); + if (res != TEE_SUCCESS) { + unmap_mapped_param(ta_sess->param, did_map); + ta_sess->err_origin = TEE_ORIGIN_TEE; + goto out; + } + param_types = ta_sess->param->types; + } + + res = stc->pseudo_ta->open_session_entry_point(param_types, + tee_param, + user_ctx); + if (ta_sess->param) { + update_out_param(tee_param, ta_sess->param); + unmap_mapped_param(ta_sess->param, did_map); + } + } + +out: + ts_pop_current_session(); + return res; +} + +static TEE_Result pseudo_ta_enter_invoke_cmd(struct ts_session *s, uint32_t cmd) +{ + TEE_Result res = TEE_SUCCESS; + struct pseudo_ta_ctx *stc = to_pseudo_ta_ctx(s->ctx); + struct tee_ta_session *ta_sess = to_ta_session(s); + uint32_t param_types = 0; + TEE_Param tee_param[TEE_NUM_PARAMS] = { }; + bool did_map[TEE_NUM_PARAMS] = { false }; + + ts_push_current_session(s); + if (ta_sess->param) { + res = copy_in_param(s, ta_sess->param, tee_param, did_map); + if (res != TEE_SUCCESS) { + unmap_mapped_param(ta_sess->param, did_map); + ta_sess->err_origin = TEE_ORIGIN_TEE; + goto out; + } + param_types = ta_sess->param->types; + } + + ta_sess->err_origin = TEE_ORIGIN_TRUSTED_APP; + res = stc->pseudo_ta->invoke_command_entry_point(s->user_ctx, cmd, + param_types, + tee_param); + if (ta_sess->param) { + update_out_param(tee_param, ta_sess->param); + unmap_mapped_param(ta_sess->param, did_map); + } +out: + ts_pop_current_session(); + return res; +} + +static void pseudo_ta_enter_close_session(struct ts_session *s) +{ + struct pseudo_ta_ctx *stc = to_pseudo_ta_ctx(s->ctx); + void *user_ctx = s->user_ctx; + + ts_push_current_session(s); + + if (stc->pseudo_ta->close_session_entry_point) + stc->pseudo_ta->close_session_entry_point(user_ctx); + + if (stc->ctx.ref_count == 1 && stc->pseudo_ta->destroy_entry_point) + stc->pseudo_ta->destroy_entry_point(); + + ts_pop_current_session(); +} + +static void pseudo_ta_destroy(struct ts_ctx *ctx) +{ + free(to_pseudo_ta_ctx(ctx)); +} + +static const struct ts_ops pseudo_ta_ops = { + .enter_open_session = pseudo_ta_enter_open_session, + .enter_invoke_cmd = pseudo_ta_enter_invoke_cmd, + .enter_close_session = pseudo_ta_enter_close_session, + .destroy = pseudo_ta_destroy, +}; + +bool __noprof is_pseudo_ta_ctx(struct ts_ctx *ctx) +{ + return ctx->ops == &pseudo_ta_ops; +} + +/* Insures declared pseudo TAs conforms with core expectations */ +static TEE_Result verify_pseudo_tas_conformance(void) +{ + const struct pseudo_ta_head *start = + SCATTERED_ARRAY_BEGIN(pseudo_tas, struct pseudo_ta_head); + const struct pseudo_ta_head *end = + SCATTERED_ARRAY_END(pseudo_tas, struct pseudo_ta_head); + const struct pseudo_ta_head *pta; + + for (pta = start; pta < end; pta++) { + const struct pseudo_ta_head *pta2; + + /* PTAs must all have a specific UUID */ + for (pta2 = pta + 1; pta2 < end; pta2++) { + if (!memcmp(&pta->uuid, &pta2->uuid, sizeof(TEE_UUID))) + goto err; + } + + if (!pta->name || + (pta->flags & PTA_MANDATORY_FLAGS) != PTA_MANDATORY_FLAGS || + pta->flags & ~PTA_ALLOWED_FLAGS || + !pta->invoke_command_entry_point) + goto err; + } + return TEE_SUCCESS; +err: + DMSG("pseudo TA error at %p", (void *)pta); + panic("PTA"); +} + +service_init(verify_pseudo_tas_conformance); + +/*----------------------------------------------------------------------------- + * Initialises a session based on the UUID or ptr to the ta + * Returns ptr to the session (ta_session) and a TEE_Result + *---------------------------------------------------------------------------*/ +TEE_Result tee_ta_init_pseudo_ta_session(const TEE_UUID *uuid, + struct tee_ta_session *s) +{ + struct pseudo_ta_ctx *stc = NULL; + struct tee_ta_ctx *ctx; + const struct pseudo_ta_head *ta; + + DMSG("Lookup pseudo TA %pUl", (void *)uuid); + + ta = SCATTERED_ARRAY_BEGIN(pseudo_tas, struct pseudo_ta_head); + while (true) { + if (ta >= SCATTERED_ARRAY_END(pseudo_tas, + struct pseudo_ta_head)) + return TEE_ERROR_ITEM_NOT_FOUND; + if (memcmp(&ta->uuid, uuid, sizeof(TEE_UUID)) == 0) + break; + ta++; + } + + /* Load a new TA and create a session */ + DMSG("Open %s", ta->name); + stc = calloc(1, sizeof(struct pseudo_ta_ctx)); + if (!stc) + return TEE_ERROR_OUT_OF_MEMORY; + ctx = &stc->ctx; + + ctx->ref_count = 1; + ctx->flags = ta->flags; + stc->pseudo_ta = ta; + ctx->ts_ctx.uuid = ta->uuid; + ctx->ts_ctx.ops = &pseudo_ta_ops; + + mutex_lock(&tee_ta_mutex); + s->ts_sess.ctx = &ctx->ts_ctx; + TAILQ_INSERT_TAIL(&tee_ctxes, ctx, link); + mutex_unlock(&tee_ta_mutex); + + DMSG("%s : %pUl", stc->pseudo_ta->name, (void *)&ctx->ts_ctx.uuid); + + return TEE_SUCCESS; +} + +TEE_Result to_bounce_params(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS], + TEE_Param bparams[TEE_NUM_PARAMS], + TEE_Param **oparams) +{ + TEE_Result res = TEE_ERROR_GENERIC; + void *kptr = NULL; + void *uptr = NULL; + size_t size = 0; + int i = 0; + + if (!is_caller_ta_with_pan()) { + *oparams = params; + return TEE_SUCCESS; + } + + for (i = 0; i < TEE_NUM_PARAMS; i++) { + switch (TEE_PARAM_TYPE_GET(param_types, i)) { + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + size = params[i].memref.size; + uptr = params[i].memref.buffer; + kptr = bb_alloc(size); + if (!kptr) + return TEE_ERROR_OUT_OF_MEMORY; + bparams[i].memref.buffer = kptr; + bparams[i].memref.size = size; + break; + default: + break; + } + switch (TEE_PARAM_TYPE_GET(param_types, i)) { + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + res = copy_from_user(kptr, uptr, size); + if (res) + return res; + break; + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + bparams[i].value.a = params[i].value.a; + bparams[i].value.b = params[i].value.b; + break; + default: + break; + } + } + *oparams = bparams; + + return TEE_SUCCESS; +} + +TEE_Result from_bounce_params(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS], + TEE_Param bparams[TEE_NUM_PARAMS], + TEE_Param *eparams) +{ + TEE_Result res = TEE_ERROR_GENERIC; + void *kptr = NULL; + void *uptr = NULL; + size_t size = 0; + int i = 0; + + if (eparams == params) + return TEE_SUCCESS; + + for (i = 0; i < TEE_NUM_PARAMS; i++) { + switch (TEE_PARAM_TYPE_GET(param_types, i)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + uptr = params[i].memref.buffer; + kptr = bparams[i].memref.buffer; + size = bparams[i].memref.size; + res = copy_to_user(uptr, kptr, size); + if (res) + return res; + params[i].memref.size = size; + break; + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + params[i].value.a = bparams[i].value.a; + params[i].value.b = bparams[i].value.b; + break; + default: + break; + } + } + + return res; +} diff --git a/optee_os/core/kernel/ree_fs_ta.c b/optee_os/core/kernel/ree_fs_ta.c new file mode 100644 index 0000000..9550364 --- /dev/null +++ b/optee_os/core/kernel/ree_fs_ta.c @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, 2019, Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ + +/* + * Security properties of REE-FS TAs + * ================================= + * + * Authentication only + * ------------------- + * + * Required security properties: + * 1. Authentication and non-repudiation of a TA to Service Provider (SP). + * 2. Integrity of a TA. + * + * To satisfy (1) and (2), SP needs to sign TA and OP-TEE core needs to verify + * the signature using SP public key with computed hash of the TA. + * + * Authentication along with Confidentiality + * ----------------------------------------- + * + * Required security properties: + * 1. Authentication and non-repudiation of a TA to Service Provider (SP). + * 2. Confidentiality of a TA. + * 3. Integrity of an encrypted TA blob. + * + * To satisfy (1), SP needs to sign plain TA and OP-TEE core needs to verify the + * signature using SP public key with computed hash of the TA. + * + * To satisfy (2) and (3), SP needs to do authenticated encryption of TA and + * OP-TEE core needs to do authenticated decryption of TA to retrieve its + * contents. Here encryption provides the confidentiality of TA and MAC tag + * provides the integrity of encrypted TA blob. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ree_fs_ta_handle { + struct shdr *nw_ta; /* Non-secure (shared memory) */ + size_t nw_ta_size; + struct mobj *mobj; + size_t offs; + struct shdr *shdr; /* Verified secure copy of @nw_ta's signed header */ + void *hash_ctx; + void *enc_ctx; + struct shdr_bootstrap_ta *bs_hdr; + struct shdr_encrypted_ta *ehdr; +}; + +struct ver_db_entry { + uint8_t uuid[sizeof(TEE_UUID)]; + uint32_t version; +}; + +struct ver_db_hdr { + uint32_t db_version; + uint32_t nb_entries; +}; + +static const char ta_ver_db[] = "ta_ver.db"; +static const char subkey_ver_db[] = "subkey_ver.db"; +static struct mutex ver_db_mutex = MUTEX_INITIALIZER; + +static TEE_Result check_update_version(const char *db_name, + const uint8_t uuid[sizeof(TEE_UUID)], + uint32_t version) +{ + struct ver_db_entry db_entry = { }; + const struct tee_file_operations *ops = NULL; + struct tee_file_handle *fh = NULL; + TEE_Result res = TEE_SUCCESS; + bool entry_found = false; + size_t len = 0; + unsigned int i = 0; + struct ver_db_hdr db_hdr = { }; + struct tee_pobj pobj = { + .obj_id = (void *)db_name, + .obj_id_len = strlen(db_name) + 1, + }; + + ops = tee_svc_storage_file_ops(TEE_STORAGE_PRIVATE); + if (!ops) + return TEE_SUCCESS; /* Compiled with no secure storage */ + + mutex_lock(&ver_db_mutex); + + res = ops->open(&pobj, NULL, &fh); + if (res != TEE_SUCCESS && res != TEE_ERROR_ITEM_NOT_FOUND) + goto out; + + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + res = ops->create(&pobj, false, NULL, 0, NULL, 0, NULL, NULL, + 0, &fh); + if (res != TEE_SUCCESS) + goto out; + + res = ops->write(fh, 0, &db_hdr, NULL, sizeof(db_hdr)); + if (res != TEE_SUCCESS) + goto out; + } else { + len = sizeof(db_hdr); + + res = ops->read(fh, 0, &db_hdr, NULL, &len); + if (res != TEE_SUCCESS) { + goto out; + } else if (len != sizeof(db_hdr)) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + } + + for (i = 0; i < db_hdr.nb_entries; i++) { + len = sizeof(db_entry); + + res = ops->read(fh, sizeof(db_hdr) + (i * len), &db_entry, + NULL, &len); + if (res != TEE_SUCCESS) { + goto out; + } else if (len != sizeof(db_entry)) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + if (!memcmp(uuid, db_entry.uuid, sizeof(TEE_UUID))) { + entry_found = true; + break; + } + } + + if (entry_found) { + if (db_entry.version > version) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto out; + } else if (db_entry.version < version) { + memcpy(db_entry.uuid, uuid, sizeof(TEE_UUID)); + db_entry.version = version; + len = sizeof(db_entry); + res = ops->write(fh, sizeof(db_hdr) + (i * len), + &db_entry, NULL, len); + if (res != TEE_SUCCESS) + goto out; + } + } else { + memcpy(db_entry.uuid, uuid, sizeof(TEE_UUID)); + db_entry.version = version; + len = sizeof(db_entry); + res = ops->write(fh, sizeof(db_hdr) + (db_hdr.nb_entries * len), + &db_entry, NULL, len); + if (res != TEE_SUCCESS) + goto out; + + db_hdr.nb_entries++; + res = ops->write(fh, 0, &db_hdr, NULL, sizeof(db_hdr)); + if (res != TEE_SUCCESS) + goto out; + } + +out: + ops->close(&fh); + mutex_unlock(&ver_db_mutex); + return res; +} + +/* + * Load a TA via RPC with UUID defined by input param @uuid. The virtual + * address of the raw TA binary is received in out parameter @ta. + */ +static TEE_Result rpc_load(const TEE_UUID *uuid, struct shdr **ta, + size_t *ta_size, struct mobj **mobj) +{ + TEE_Result res; + struct thread_param params[2]; + + if (!uuid || !ta || !mobj || !ta_size) + return TEE_ERROR_BAD_PARAMETERS; + + memset(params, 0, sizeof(params)); + params[0].attr = THREAD_PARAM_ATTR_VALUE_IN; + tee_uuid_to_octets((void *)¶ms[0].u.value, uuid); + params[1].attr = THREAD_PARAM_ATTR_MEMREF_OUT; + + res = thread_rpc_cmd(OPTEE_RPC_CMD_LOAD_TA, 2, params); + if (res != TEE_SUCCESS) + return res; + + *mobj = thread_rpc_alloc_payload(params[1].u.memref.size); + if (!*mobj) + return TEE_ERROR_OUT_OF_MEMORY; + + *ta = mobj_get_va(*mobj, 0, params[1].u.memref.size); + if (!*ta) { + res = TEE_ERROR_SHORT_BUFFER; + goto exit; + } + /* We don't expect NULL as thread_rpc_alloc_payload() was successful */ + assert(*ta); + *ta_size = params[1].u.memref.size; + + params[0].attr = THREAD_PARAM_ATTR_VALUE_IN; + tee_uuid_to_octets((void *)¶ms[0].u.value, uuid); + params[1].attr = THREAD_PARAM_ATTR_MEMREF_OUT; + params[1].u.memref.offs = 0; + params[1].u.memref.mobj = *mobj; + + res = thread_rpc_cmd(OPTEE_RPC_CMD_LOAD_TA, 2, params); +exit: + if (res != TEE_SUCCESS) + thread_rpc_free_payload(*mobj); + + return res; +} + +static TEE_Result ree_fs_ta_open(const TEE_UUID *uuid, + struct ts_store_handle **h) +{ + uint8_t next_uuid[sizeof(TEE_UUID)] = { }; + struct ree_fs_ta_handle *handle; + uint8_t *next_uuid_ptr = NULL; + struct shdr *shdr = NULL; + struct mobj *mobj = NULL; + void *hash_ctx = NULL; + struct shdr *ta = NULL; + size_t ta_size = 0; + TEE_Result res = TEE_SUCCESS; + size_t offs = 0; + struct shdr_bootstrap_ta *bs_hdr = NULL; + struct shdr_encrypted_ta *ehdr = NULL; + size_t shdr_sz = 0; + uint32_t max_depth = UINT32_MAX; + struct ftmn ftmn = { }; + unsigned int incr0_count = 0; + + handle = calloc(1, sizeof(*handle)); + if (!handle) + return TEE_ERROR_OUT_OF_MEMORY; + + /* Request TA from tee-supplicant */ + res = rpc_load(uuid, &ta, &ta_size, &mobj); + if (res != TEE_SUCCESS) + goto error; + + /* Make secure copy of signed header */ + shdr = shdr_alloc_and_copy(0, ta, ta_size); + if (!shdr) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + + /* Validate header signature */ + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, shdr_verify_signature, shdr); + incr0_count++; + if (res != TEE_SUCCESS) + goto error_free_payload; + + shdr_sz = SHDR_GET_SIZE(shdr); + if (!shdr_sz) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + offs = shdr_sz; + + while (shdr->img_type == SHDR_SUBKEY) { + struct shdr_pub_key pub_key = { }; + + if (offs > ta_size) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + + res = shdr_load_pub_key(shdr, offs, (const void *)ta, + ta_size, next_uuid_ptr, max_depth, + &pub_key); + if (res) + goto error_free_payload; + + if (ADD_OVERFLOW(offs, shdr->img_size, &offs) || + ADD_OVERFLOW(offs, pub_key.name_size, &offs) || + offs > ta_size) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + max_depth = pub_key.max_depth; + memcpy(next_uuid, pub_key.next_uuid, sizeof(TEE_UUID)); + next_uuid_ptr = next_uuid; + + res = check_update_version(subkey_ver_db, pub_key.uuid, + pub_key.version); + if (res) { + res = TEE_ERROR_SECURITY; + shdr_free_pub_key(&pub_key); + goto error_free_payload; + } + + shdr_free(shdr); + shdr = shdr_alloc_and_copy(offs, ta, ta_size); + res = TEE_ERROR_SECURITY; + if (shdr) { + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + shdr_verify_signature2, &pub_key, shdr); + incr0_count++; + } + shdr_free_pub_key(&pub_key); + if (res) + goto error_free_payload; + + shdr_sz = SHDR_GET_SIZE(shdr); + if (!shdr_sz) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + offs += shdr_sz; + if (offs > ta_size) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + } + + if (shdr->img_type != SHDR_TA && shdr->img_type != SHDR_BOOTSTRAP_TA && + shdr->img_type != SHDR_ENCRYPTED_TA) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + + /* + * If we're verifying this TA using a subkey, make sure that + * the UUID of the TA belongs to the namespace defined by the subkey. + * The namespace is defined as in RFC4122, that is, valid UUID + * is calculated as a V5 UUID SHA-512(subkey UUID, "name string"). + */ + if (next_uuid_ptr) { + TEE_UUID check_uuid = { }; + + tee_uuid_from_octets(&check_uuid, next_uuid_ptr); + if (memcmp(&check_uuid, uuid, sizeof(*uuid))) { + res = TEE_ERROR_SECURITY; + goto error_free_payload; + } + } + + /* + * Initialize a hash context and run the algorithm over the signed + * header (less the final file hash and its signature of course) + */ + res = crypto_hash_alloc_ctx(&hash_ctx, + TEE_DIGEST_HASH_TO_ALGO(shdr->algo)); + if (res != TEE_SUCCESS) + goto error_free_payload; + res = crypto_hash_init(hash_ctx); + if (res != TEE_SUCCESS) + goto error_free_hash; + res = crypto_hash_update(hash_ctx, (uint8_t *)shdr, sizeof(*shdr)); + if (res != TEE_SUCCESS) + goto error_free_hash; + + if (shdr->img_type == SHDR_BOOTSTRAP_TA || + shdr->img_type == SHDR_ENCRYPTED_TA) { + TEE_UUID bs_uuid = { }; + size_t sz = shdr_sz; + + if (ADD_OVERFLOW(sz, sizeof(*bs_hdr), &sz) || ta_size < sz) { + res = TEE_ERROR_SECURITY; + goto error_free_hash; + } + + bs_hdr = malloc(sizeof(*bs_hdr)); + if (!bs_hdr) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto error_free_hash; + } + + memcpy(bs_hdr, (uint8_t *)ta + offs, sizeof(*bs_hdr)); + + /* + * There's a check later that the UUID embedded inside the + * ELF is matching, but since we now have easy access to + * the expected uuid of the TA we check it a bit earlier + * here. + */ + tee_uuid_from_octets(&bs_uuid, bs_hdr->uuid); + if (memcmp(&bs_uuid, uuid, sizeof(TEE_UUID))) { + res = TEE_ERROR_SECURITY; + goto error_free_hash; + } + + res = crypto_hash_update(hash_ctx, (uint8_t *)bs_hdr, + sizeof(*bs_hdr)); + if (res != TEE_SUCCESS) + goto error_free_hash; + offs += sizeof(*bs_hdr); + handle->bs_hdr = bs_hdr; + } + + if (shdr->img_type == SHDR_ENCRYPTED_TA) { + struct shdr_encrypted_ta img_ehdr = { }; + size_t sz = shdr_sz; + size_t ehdr_sz = 0; + + if (ADD_OVERFLOW(sz, sizeof(struct shdr_bootstrap_ta), &sz) || + ADD_OVERFLOW(sz, sizeof(img_ehdr), &sz) || + ta_size < sz) { + res = TEE_ERROR_SECURITY; + goto error_free_hash; + } + + memcpy(&img_ehdr, ((uint8_t *)ta + offs), sizeof(img_ehdr)); + ehdr_sz = SHDR_ENC_GET_SIZE(&img_ehdr); + sz -= sizeof(img_ehdr); + if (!ehdr_sz || ADD_OVERFLOW(sz, ehdr_sz, &sz) || + ta_size < sz) { + res = TEE_ERROR_SECURITY; + goto error_free_hash; + } + + /* + * This is checked further down too, but we must sanity + * check shdr->img_size before it's used below. + */ + if (ta_size != offs + ehdr_sz + shdr->img_size) { + res = TEE_ERROR_SECURITY; + goto error_free_hash; + } + + ehdr = malloc(ehdr_sz); + if (!ehdr) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto error_free_hash; + } + + *ehdr = img_ehdr; + memcpy((uint8_t *)ehdr + sizeof(img_ehdr), + (uint8_t *)ta + offs + sizeof(img_ehdr), + ehdr_sz - sizeof(img_ehdr)); + + res = crypto_hash_update(hash_ctx, (uint8_t *)ehdr, ehdr_sz); + if (res != TEE_SUCCESS) + goto error_free_hash; + + res = tee_ta_decrypt_init(&handle->enc_ctx, ehdr, + shdr->img_size); + if (res != TEE_SUCCESS) + goto error_free_hash; + + offs += ehdr_sz; + handle->ehdr = ehdr; + } + + if (ta_size != offs + shdr->img_size) { + res = TEE_ERROR_SECURITY; + goto error_free_hash; + } + + handle->nw_ta = ta; + handle->nw_ta_size = ta_size; + handle->offs = offs; + handle->hash_ctx = hash_ctx; + handle->shdr = shdr; + handle->mobj = mobj; + *h = (struct ts_store_handle *)handle; + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR1, + FTMN_STEP_COUNT(incr0_count), TEE_SUCCESS); + return TEE_SUCCESS; + +error_free_hash: + crypto_hash_free_ctx(hash_ctx); +error_free_payload: + thread_rpc_free_payload(mobj); +error: + free(ehdr); + free(bs_hdr); + shdr_free(shdr); + free(handle); + FTMN_SET_CHECK_RES_NOT_ZERO(&ftmn, FTMN_INCR1, res); + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR1, + FTMN_STEP_COUNT(incr0_count, 1), res); + return res; +} + +static TEE_Result ree_fs_ta_get_size(const struct ts_store_handle *h, + size_t *size) +{ + struct ree_fs_ta_handle *handle = (struct ree_fs_ta_handle *)h; + + *size = handle->shdr->img_size; + return TEE_SUCCESS; +} + +static TEE_Result ree_fs_ta_get_tag(const struct ts_store_handle *h, + uint8_t *tag, unsigned int *tag_len) +{ + struct ree_fs_ta_handle *handle = (struct ree_fs_ta_handle *)h; + + if (!tag || *tag_len < handle->shdr->hash_size) { + *tag_len = handle->shdr->hash_size; + return TEE_ERROR_SHORT_BUFFER; + } + *tag_len = handle->shdr->hash_size; + + memcpy(tag, SHDR_GET_HASH(handle->shdr), handle->shdr->hash_size); + + return TEE_SUCCESS; +} + +static TEE_Result check_digest(struct ree_fs_ta_handle *h) +{ + void *digest = NULL; + TEE_Result res; + + digest = malloc(h->shdr->hash_size); + if (!digest) + return TEE_ERROR_OUT_OF_MEMORY; + res = crypto_hash_final(h->hash_ctx, digest, h->shdr->hash_size); + if (res != TEE_SUCCESS) { + res = TEE_ERROR_SECURITY; + goto out; + } + if (FTMN_CALLEE_DONE_MEMCMP(memcmp, digest, SHDR_GET_HASH(h->shdr), + h->shdr->hash_size)) + res = TEE_ERROR_SECURITY; +out: + free(digest); + return res; +} + +static TEE_Result ree_fs_ta_read(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len) +{ + struct ree_fs_ta_handle *handle = (struct ree_fs_ta_handle *)h; + uint8_t *src = (uint8_t *)handle->nw_ta + handle->offs; + size_t bb_len = MIN(1024U, len); + size_t next_offs = 0; + TEE_Result res = TEE_SUCCESS; + size_t num_bytes = 0; + size_t dst_len = 0; + void *dst = NULL; + void *bb = NULL; + + + if (ADD_OVERFLOW(handle->offs, len, &next_offs) || + next_offs > handle->nw_ta_size) + return TEE_ERROR_BAD_PARAMETERS; + + if (data_core) { + dst = data_core; + dst_len = len; + } else { + bb = bb_alloc(bb_len); + if (!bb) + return TEE_ERROR_OUT_OF_MEMORY; + dst = bb; + dst_len = bb_len; + } + + /* + * This loop will only run once if data_core is non-NULL, but as + * many times as needed if the bounce buffer bb is used. That's why + * dst doesn't need to be updated in the loop. + */ + while (num_bytes < len) { + size_t n = MIN(dst_len, len - num_bytes); + + if (handle->shdr->img_type == SHDR_ENCRYPTED_TA) { + res = tee_ta_decrypt_update(handle->enc_ctx, dst, + src + num_bytes, n); + if (res) { + res = TEE_ERROR_SECURITY; + goto out; + } + } else { + memcpy(dst, src + num_bytes, n); + } + + res = crypto_hash_update(handle->hash_ctx, dst, n); + if (res) { + res = TEE_ERROR_SECURITY; + goto out; + } + if (data_user) { + res = copy_to_user((uint8_t *)data_user + num_bytes, + dst, n); + if (res) { + res = TEE_ERROR_SECURITY; + goto out; + } + } + num_bytes += n; + } + + handle->offs = next_offs; + if (handle->offs == handle->nw_ta_size) { + if (handle->shdr->img_type == SHDR_ENCRYPTED_TA) { + /* + * Last read: time to finalize authenticated + * decryption. + */ + res = tee_ta_decrypt_final(handle->enc_ctx, + handle->ehdr, NULL, NULL, 0); + if (res != TEE_SUCCESS) { + res = TEE_ERROR_SECURITY; + goto out; + } + } + /* + * Last read: time to check if our digest matches the expected + * one (from the signed header) + */ + res = check_digest(handle); + if (res != TEE_SUCCESS) + goto out; + + if (handle->bs_hdr) + res = check_update_version(ta_ver_db, + handle->bs_hdr->uuid, + handle->bs_hdr->ta_version); + } +out: + bb_free(bb, bb_len); + return res; +} + +static void ree_fs_ta_close(struct ts_store_handle *h) +{ + struct ree_fs_ta_handle *handle = (struct ree_fs_ta_handle *)h; + + if (!handle) + return; + thread_rpc_free_payload(handle->mobj); + crypto_hash_free_ctx(handle->hash_ctx); + free(handle->shdr); + free(handle->ehdr); + free(handle->bs_hdr); + free(handle); +} + +#ifndef CFG_REE_FS_TA_BUFFERED +REGISTER_TA_STORE(9) = { + .description = "REE", + .open = ree_fs_ta_open, + .get_size = ree_fs_ta_get_size, + .get_tag = ree_fs_ta_get_tag, + .read = ree_fs_ta_read, + .close = ree_fs_ta_close, +}; +#endif + +#ifdef CFG_REE_FS_TA_BUFFERED + +/* + * This is a wrapper around the "REE FS" TA store. + * The whole TA/library is read into a temporary buffer during .open(). This + * allows the binary to be authenticated before any data is read and processed + * by the upper layer (ELF loader). + */ + +struct buf_ree_fs_ta_handle { + struct ts_store_handle *h; /* Note: a REE FS TA store handle */ + size_t ta_size; + tee_mm_entry_t *mm; + uint8_t *buf; + size_t offs; + uint8_t *tag; + unsigned int tag_len; +}; + +static TEE_Result buf_ta_open(const TEE_UUID *uuid, + struct ts_store_handle **h) +{ + struct buf_ree_fs_ta_handle *handle = NULL; + struct ftmn ftmn = { }; + TEE_Result res = TEE_SUCCESS; + + handle = calloc(1, sizeof(*handle)); + if (!handle) + return TEE_ERROR_OUT_OF_MEMORY; + FTMN_PUSH_LINKED_CALL(&ftmn, FTMN_FUNC_HASH("ree_fs_ta_open")); + res = ree_fs_ta_open(uuid, &handle->h); + if (!res) + FTMN_SET_CHECK_RES_FROM_CALL(&ftmn, FTMN_INCR0, res); + FTMN_POP_LINKED_CALL(&ftmn); + if (res) + goto err_free_handle; + ftmn_checkpoint(&ftmn, FTMN_INCR1); + + res = ree_fs_ta_get_size(handle->h, &handle->ta_size); + if (res) + goto err; + + res = ree_fs_ta_get_tag(handle->h, NULL, &handle->tag_len); + if (res != TEE_ERROR_SHORT_BUFFER) { + res = TEE_ERROR_GENERIC; + goto err; + } + handle->tag = malloc(handle->tag_len); + if (!handle->tag) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + res = ree_fs_ta_get_tag(handle->h, handle->tag, &handle->tag_len); + if (res) + goto err; + + handle->mm = tee_mm_alloc(&tee_mm_sec_ddr, handle->ta_size); + if (!handle->mm) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + handle->buf = phys_to_virt(tee_mm_get_smem(handle->mm), + MEM_AREA_TA_RAM, handle->ta_size); + if (!handle->buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + FTMN_PUSH_LINKED_CALL(&ftmn, FTMN_FUNC_HASH("check_digest")); + res = ree_fs_ta_read(handle->h, handle->buf, NULL, handle->ta_size); + if (!res) + FTMN_SET_CHECK_RES_FROM_CALL(&ftmn, FTMN_INCR0, res); + FTMN_POP_LINKED_CALL(&ftmn); + if (res) + goto err; + ftmn_checkpoint(&ftmn, FTMN_INCR1); + + *h = (struct ts_store_handle *)handle; + ree_fs_ta_close(handle->h); + return ftmn_return_res(&ftmn, FTMN_STEP_COUNT(2, 2), TEE_SUCCESS); + +err: + ree_fs_ta_close(handle->h); + tee_mm_free(handle->mm); + free(handle->tag); +err_free_handle: + free(handle); + return res; +} + +static TEE_Result buf_ta_get_size(const struct ts_store_handle *h, + size_t *size) +{ + struct buf_ree_fs_ta_handle *handle = (struct buf_ree_fs_ta_handle *)h; + + *size = handle->ta_size; + return TEE_SUCCESS; +} + +static TEE_Result buf_ta_read(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len) +{ + struct buf_ree_fs_ta_handle *handle = (struct buf_ree_fs_ta_handle *)h; + uint8_t *src = handle->buf + handle->offs; + TEE_Result res = TEE_SUCCESS; + size_t next_offs = 0; + + if (ADD_OVERFLOW(handle->offs, len, &next_offs) || + next_offs > handle->ta_size) + return TEE_ERROR_BAD_PARAMETERS; + + if (data_core) + memcpy(data_core, src, len); + if (data_user) { + res = copy_to_user(data_user, src, len); + if (res) + return res; + } + handle->offs = next_offs; + return TEE_SUCCESS; +} + +static TEE_Result buf_ta_get_tag(const struct ts_store_handle *h, + uint8_t *tag, unsigned int *tag_len) +{ + struct buf_ree_fs_ta_handle *handle = (struct buf_ree_fs_ta_handle *)h; + + *tag_len = handle->tag_len; + if (!tag || *tag_len < handle->tag_len) + return TEE_ERROR_SHORT_BUFFER; + + memcpy(tag, handle->tag, handle->tag_len); + + return TEE_SUCCESS; +} + +static void buf_ta_close(struct ts_store_handle *h) +{ + struct buf_ree_fs_ta_handle *handle = (struct buf_ree_fs_ta_handle *)h; + + if (!handle) + return; + tee_mm_free(handle->mm); + free(handle->tag); + free(handle); +} + +REGISTER_TA_STORE(9) = { + .description = "REE [buffered]", + .open = buf_ta_open, + .get_size = buf_ta_get_size, + .get_tag = buf_ta_get_tag, + .read = buf_ta_read, + .close = buf_ta_close, +}; + +#endif /* CFG_REE_FS_TA_BUFFERED */ diff --git a/optee_os/core/kernel/refcount.c b/optee_os/core/kernel/refcount.c new file mode 100644 index 0000000..64b08a6 --- /dev/null +++ b/optee_os/core/kernel/refcount.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include + +bool refcount_inc(struct refcount *r) +{ + unsigned int nval; + unsigned int oval = atomic_load_uint(&r->val); + + while (true) { + nval = oval + 1; + + /* r->val is 0, we can't do anything more. */ + if (!oval) + return false; + + if (atomic_cas_uint(&r->val, &oval, nval)) + return true; + /* + * At this point atomic_cas_uint() has updated oval to the + * current r->val. + */ + } +} + +bool refcount_dec(struct refcount *r) +{ + unsigned int nval; + unsigned int oval = atomic_load_uint(&r->val); + + while (true) { + assert(oval); + nval = oval - 1; + + if (atomic_cas_uint(&r->val, &oval, nval)) { + /* + * Value has been updated, if value was set to 0 + * return true to indicate that. + */ + return !nval; + } + /* + * At this point atomic_cas_uint() has updated oval to the + * current r->val. + */ + } +} diff --git a/optee_os/core/kernel/scall.c b/optee_os/core/kernel/scall.c new file mode 100644 index 0000000..2dc9222 --- /dev/null +++ b/optee_os/core/kernel/scall.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2022, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if (TRACE_LEVEL == TRACE_FLOW) && defined(CFG_TEE_CORE_TA_TRACE) +#define TRACE_SYSCALLS +#endif + +struct syscall_entry { + syscall_t fn; +#ifdef TRACE_SYSCALLS + const char *name; +#endif +}; + +#ifdef TRACE_SYSCALLS +#define SYSCALL_ENTRY(_fn) { .fn = (syscall_t)_fn, .name = #_fn } +#else +#define SYSCALL_ENTRY(_fn) { .fn = (syscall_t)_fn } +#endif + +/* + * This array is ordered according to the SYSCALL ids TEE_SCN_xxx + */ +static const struct syscall_entry tee_syscall_table[] = { + SYSCALL_ENTRY(syscall_sys_return), + SYSCALL_ENTRY(syscall_log), + SYSCALL_ENTRY(syscall_panic), + SYSCALL_ENTRY(syscall_get_property), + SYSCALL_ENTRY(syscall_get_property_name_to_index), + SYSCALL_ENTRY(syscall_open_ta_session), + SYSCALL_ENTRY(syscall_close_ta_session), + SYSCALL_ENTRY(syscall_invoke_ta_command), + SYSCALL_ENTRY(syscall_check_access_rights), + SYSCALL_ENTRY(syscall_get_cancellation_flag), + SYSCALL_ENTRY(syscall_unmask_cancellation), + SYSCALL_ENTRY(syscall_mask_cancellation), + SYSCALL_ENTRY(syscall_wait), + SYSCALL_ENTRY(syscall_get_time), + SYSCALL_ENTRY(syscall_set_ta_time), + SYSCALL_ENTRY(syscall_cryp_state_alloc), + SYSCALL_ENTRY(syscall_cryp_state_copy), + SYSCALL_ENTRY(syscall_cryp_state_free), + SYSCALL_ENTRY(syscall_hash_init), + SYSCALL_ENTRY(syscall_hash_update), + SYSCALL_ENTRY(syscall_hash_final), + SYSCALL_ENTRY(syscall_cipher_init), + SYSCALL_ENTRY(syscall_cipher_update), + SYSCALL_ENTRY(syscall_cipher_final), + SYSCALL_ENTRY(syscall_cryp_obj_get_info), + SYSCALL_ENTRY(syscall_cryp_obj_restrict_usage), + SYSCALL_ENTRY(syscall_cryp_obj_get_attr), + SYSCALL_ENTRY(syscall_cryp_obj_alloc), + SYSCALL_ENTRY(syscall_cryp_obj_close), + SYSCALL_ENTRY(syscall_cryp_obj_reset), + SYSCALL_ENTRY(syscall_cryp_obj_populate), + SYSCALL_ENTRY(syscall_cryp_obj_copy), + SYSCALL_ENTRY(syscall_cryp_derive_key), + SYSCALL_ENTRY(syscall_cryp_random_number_generate), + SYSCALL_ENTRY(syscall_authenc_init), + SYSCALL_ENTRY(syscall_authenc_update_aad), + SYSCALL_ENTRY(syscall_authenc_update_payload), + SYSCALL_ENTRY(syscall_authenc_enc_final), + SYSCALL_ENTRY(syscall_authenc_dec_final), + SYSCALL_ENTRY(syscall_asymm_operate), + SYSCALL_ENTRY(syscall_asymm_verify), + SYSCALL_ENTRY(syscall_storage_obj_open), + SYSCALL_ENTRY(syscall_storage_obj_create), + SYSCALL_ENTRY(syscall_storage_obj_del), + SYSCALL_ENTRY(syscall_storage_obj_rename), + SYSCALL_ENTRY(syscall_storage_alloc_enum), + SYSCALL_ENTRY(syscall_storage_free_enum), + SYSCALL_ENTRY(syscall_storage_reset_enum), + SYSCALL_ENTRY(syscall_storage_start_enum), + SYSCALL_ENTRY(syscall_storage_next_enum), + SYSCALL_ENTRY(syscall_storage_obj_read), + SYSCALL_ENTRY(syscall_storage_obj_write), + SYSCALL_ENTRY(syscall_storage_obj_trunc), + SYSCALL_ENTRY(syscall_storage_obj_seek), + SYSCALL_ENTRY(syscall_obj_generate_key), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_not_supported), + SYSCALL_ENTRY(syscall_cache_operation), +}; + +/* + * The ldelf return, log, panic syscalls have the same functionality and syscall + * number as the user TAs'. To avoid unnecessary code duplication, the ldelf SVC + * handler doesn't implement separate functions for these. + */ +static const struct syscall_entry ldelf_syscall_table[] = { + SYSCALL_ENTRY(syscall_sys_return), + SYSCALL_ENTRY(syscall_log), + SYSCALL_ENTRY(syscall_panic), + SYSCALL_ENTRY(ldelf_syscall_map_zi), + SYSCALL_ENTRY(ldelf_syscall_unmap), + SYSCALL_ENTRY(ldelf_syscall_open_bin), + SYSCALL_ENTRY(ldelf_syscall_close_bin), + SYSCALL_ENTRY(ldelf_syscall_map_bin), + SYSCALL_ENTRY(ldelf_syscall_copy_from_bin), + SYSCALL_ENTRY(ldelf_syscall_set_prot), + SYSCALL_ENTRY(ldelf_syscall_remap), + SYSCALL_ENTRY(ldelf_syscall_gen_rnd_num), +}; + +#ifdef TRACE_SYSCALLS +static void trace_syscall(size_t num) +{ + if (num == TEE_SCN_RETURN || num == TEE_SCN_LOG || num > TEE_SCN_MAX) + return; + FMSG("syscall #%zu (%s)", num, tee_syscall_table[num].name); +} +#else +static void trace_syscall(size_t num __unused) +{ +} +#endif + +#ifdef CFG_SYSCALL_FTRACE +static void __noprof ftrace_syscall_enter(size_t num) +{ + struct ts_session *s = NULL; + + /* + * Syscalls related to inter-TA communication can't be traced in the + * caller TA's ftrace buffer as it involves context switching to callee + * TA's context. Moreover, user can enable ftrace for callee TA to dump + * function trace in corresponding ftrace buffer. + */ + if (num == TEE_SCN_OPEN_TA_SESSION || num == TEE_SCN_CLOSE_TA_SESSION || + num == TEE_SCN_INVOKE_TA_COMMAND) + return; + + s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); + if (s && s->fbuf) + s->fbuf->syscall_trace_enabled = true; +} + +static void __noprof ftrace_syscall_leave(void) +{ + struct ts_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); + + if (s && s->fbuf) + s->fbuf->syscall_trace_enabled = false; +} +#else +static void __noprof ftrace_syscall_enter(size_t num __unused) +{ +} + +static void __noprof ftrace_syscall_leave(void) +{ +} +#endif + +static syscall_t get_tee_syscall_func(size_t num) +{ + /* Cast away const */ + struct syscall_entry *sc_table = (void *)tee_syscall_table; + + static_assert(ARRAY_SIZE(tee_syscall_table) == (TEE_SCN_MAX + 1)); + + if (num > TEE_SCN_MAX) + return (syscall_t)syscall_not_supported; + + return load_no_speculate(&sc_table[num].fn, &sc_table[0].fn, + &sc_table[TEE_SCN_MAX].fn + 1); +} + +bool scall_handle_user_ta(struct thread_scall_regs *regs) +{ + size_t scn = 0; + size_t max_args = 0; + syscall_t scf = NULL; + + bb_reset(); + scall_get_max_args(regs, &scn, &max_args); + + trace_syscall(scn); + + if (max_args > TEE_SVC_MAX_ARGS) { + DMSG("Too many arguments for SCN %zu (%zu)", scn, max_args); + scall_set_retval(regs, TEE_ERROR_GENERIC); + return true; /* return to user mode */ + } + + scf = get_tee_syscall_func(scn); + + ftrace_syscall_enter(scn); + + scall_set_retval(regs, scall_do_call(regs, scf)); + + ftrace_syscall_leave(); + + /* + * Return true if we're to return to user mode, + * thread_scall_handler() will take care of the rest. + */ + return scn != TEE_SCN_RETURN && scn != TEE_SCN_PANIC; +} + +static syscall_t get_ldelf_syscall_func(size_t num) +{ + /* Cast away const */ + struct syscall_entry *sc_table = (void *)ldelf_syscall_table; + + COMPILE_TIME_ASSERT(ARRAY_SIZE(ldelf_syscall_table) == + (LDELF_SCN_MAX + 1)); + + if (num > LDELF_SCN_MAX) + return (syscall_t)syscall_not_supported; + + return load_no_speculate(&sc_table[num].fn, &sc_table[0].fn, + &sc_table[LDELF_SCN_MAX].fn + 1); +} + +bool scall_handle_ldelf(struct thread_scall_regs *regs) +{ + size_t scn = 0; + size_t max_args = 0; + syscall_t scf = NULL; + + bb_reset(); + scall_get_max_args(regs, &scn, &max_args); + + trace_syscall(scn); + + if (max_args > TEE_SVC_MAX_ARGS) { + DMSG("Too many arguments for SCN %zu (%zu)", scn, max_args); + scall_set_retval(regs, TEE_ERROR_GENERIC); + return true; /* return to user mode */ + } + + scf = get_ldelf_syscall_func(scn); + + ftrace_syscall_enter(scn); + + scall_set_retval(regs, scall_do_call(regs, scf)); + + ftrace_syscall_leave(); + + /* + * Return true if we're to return to user mode, + * thread_scall_handler() will take care of the rest. + */ + return scn != LDELF_RETURN && scn != LDELF_PANIC; +} + +uint32_t scall_sys_return_helper(uint32_t ret, bool panic, uint32_t panic_code, + struct thread_scall_regs *regs) +{ + if (panic) { + TAMSG_RAW(""); + TAMSG_RAW("TA panicked with code 0x%" PRIx32, panic_code); + scall_save_panic_stack(regs); + } + + scall_set_sys_return_regs(regs, panic, panic_code); + + return ret; +} diff --git a/optee_os/core/kernel/scattered_array.c b/optee_os/core/kernel/scattered_array.c new file mode 100644 index 0000000..9cb1f6f --- /dev/null +++ b/optee_os/core/kernel/scattered_array.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +#include + +const void *scattered_array_relax_ptr(const void *p) +{ + return p; +} diff --git a/optee_os/core/kernel/secstor_ta.c b/optee_os/core/kernel/secstor_ta.c new file mode 100644 index 0000000..b520c25 --- /dev/null +++ b/optee_os/core/kernel/secstor_ta.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include + +static TEE_Result secstor_ta_open(const TEE_UUID *uuid, + struct ts_store_handle **handle) +{ + TEE_Result res; + struct tee_tadb_ta_read *ta; + size_t l; + const struct tee_tadb_property *prop; + + res = tee_tadb_ta_open(uuid, &ta); + if (res) + return res; + prop = tee_tadb_ta_get_property(ta); + + l = prop->custom_size; + res = tee_tadb_ta_read(ta, NULL, NULL, &l); + if (res) + goto err; + if (l != prop->custom_size) { + res = TEE_ERROR_CORRUPT_OBJECT; + goto err; + } + + *handle = (struct ts_store_handle *)ta; + + return TEE_SUCCESS; +err: + tee_tadb_ta_close(ta); + return res; +} + +static TEE_Result secstor_ta_get_size(const struct ts_store_handle *h, + size_t *size) +{ + struct tee_tadb_ta_read *ta = (struct tee_tadb_ta_read *)h; + const struct tee_tadb_property *prop = tee_tadb_ta_get_property(ta); + + *size = prop->bin_size; + + return TEE_SUCCESS; +} + +static TEE_Result secstor_ta_get_tag(const struct ts_store_handle *h, + uint8_t *tag, unsigned int *tag_len) +{ + return tee_tadb_get_tag((struct tee_tadb_ta_read *)h, tag, tag_len); +} + +static TEE_Result secstor_ta_read(struct ts_store_handle *h, void *data_core, + void *data_user, size_t len) +{ + struct tee_tadb_ta_read *ta = (struct tee_tadb_ta_read *)h; + size_t l = len; + TEE_Result res = tee_tadb_ta_read(ta, data_core, data_user, &l); + + if (res) + return res; + if (l != len) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; +} + +static void secstor_ta_close(struct ts_store_handle *h) +{ + struct tee_tadb_ta_read *ta = (struct tee_tadb_ta_read *)h; + + tee_tadb_ta_close(ta); +} + +REGISTER_TA_STORE(4) = { + .description = "Secure Storage TA", + .open = secstor_ta_open, + .get_size = secstor_ta_get_size, + .get_tag = secstor_ta_get_tag, + .read = secstor_ta_read, + .close = secstor_ta_close, +}; diff --git a/optee_os/core/kernel/show_conf.c b/optee_os/core/kernel/show_conf.c new file mode 100644 index 0000000..c6495ed --- /dev/null +++ b/optee_os/core/kernel/show_conf.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 1019 Huawei Technologies Co., Ltd + */ + +#include +#include + +extern const char conf_str[]; + +static TEE_Result show_conf(void) +{ +#if (TRACE_LEVEL >= TRACE_INFO) + IMSG("Contents of conf.mk (decode with 'base64 -d | xz -d'):"); + trace_ext_puts(conf_str); +#endif + return TEE_SUCCESS; +} +service_init(show_conf); diff --git a/optee_os/core/kernel/spin_lock_debug.c b/optee_os/core/kernel/spin_lock_debug.c new file mode 100644 index 0000000..b7b7c3d --- /dev/null +++ b/optee_os/core/kernel/spin_lock_debug.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include + +void spinlock_count_incr(void) +{ + struct thread_core_local *l = thread_get_core_local(); + + l->locked_count++; + assert(l->locked_count); +} + +void spinlock_count_decr(void) +{ + struct thread_core_local *l = thread_get_core_local(); + + assert(l->locked_count); + l->locked_count--; +} + +bool __nostackcheck have_spinlock(void) +{ + struct thread_core_local *l; + + if (!thread_foreign_intr_disabled()) { + /* + * Normally we can't be holding a spinlock since doing so would + * imply foreign interrupts are disabled (or the spinlock + * logic is flawed). + */ + return false; + } + + l = thread_get_core_local(); + + return !!l->locked_count; +} diff --git a/optee_os/core/kernel/sub.mk b/optee_os/core/kernel/sub.mk new file mode 100644 index 0000000..75d3368 --- /dev/null +++ b/optee_os/core/kernel/sub.mk @@ -0,0 +1,60 @@ +srcs-$(CFG_CORE_SANITIZE_KADDRESS) += asan.c +cflags-remove-asan.c-y += $(cflags_kasan) +srcs-$(CFG_TEE_CORE_DEBUG) += spin_lock_debug.c +srcs-y += assert.c +srcs-y += console.c +srcs-$(CFG_DT) += dt.c +srcs-$(CFG_DT) += dt_driver.c +srcs-y += pm.c +srcs-y += handle.c +srcs-y += interrupt.c +ifeq ($(CFG_WITH_USER_TA),y) +srcs-y += ldelf_loader.c +srcs-y += ldelf_syscalls.c +srcs-y += scall.c +endif +srcs-$(CFG_LOCKDEP) += lockdep.c +ifneq ($(CFG_CORE_FFA),y) +srcs-$(CFG_CORE_DYN_SHM) += msg_param.c +endif +srcs-y += panic.c +srcs-y += trace_ext.c +srcs-y += refcount.c +srcs-y += delay.c +srcs-y += tee_time.c +srcs-$(CFG_SECURE_TIME_SOURCE_REE) += tee_time_ree.c +srcs-y += otp_stubs.c +srcs-y += tee_misc.c +srcs-y += tee_ta_manager.c +srcs-y += ts_manager.c +srcs-$(CFG_CORE_SANITIZE_UNDEFINED) += ubsan.c +srcs-y += scattered_array.c +srcs-y += huk_subkey.c +srcs-$(CFG_SHOW_CONF_ON_BOOT) += show_conf.c +srcs-y += user_mode_ctx.c +srcs-$(CFG_CORE_TPM_EVENT_LOG) += tpm.c +srcs-y += initcall.c +srcs-$(CFG_WITH_USER_TA) += user_access.c +srcs-y += mutex.c +srcs-$(CFG_LOCKDEP) += mutex_lockdep.c +srcs-y += wait_queue.c +srcs-y += notif.c +srcs-y += thread.c + +ifeq ($(CFG_WITH_USER_TA),y) +srcs-y += user_ta.c +srcs-$(CFG_REE_FS_TA) += ree_fs_ta.c +srcs-$(CFG_EARLY_TA) += early_ta.c +srcs-$(CFG_SECSTOR_TA) += secstor_ta.c +endif + +srcs-$(CFG_EMBEDDED_TS) += embedded_ts.c +srcs-y += pseudo_ta.c + +ifeq ($(CFG_SYSCALL_FTRACE),y) +# We would not like to profile spin_lock_debug.c file as it provides +# common APIs that are needed for ftrace framework to trace syscalls. +# So profiling this file could create an incorrect cyclic behaviour. +cflags-remove-spin_lock_debug.c-$(CFG_TEE_CORE_DEBUG) += -pg +endif +srcs-y += nv_counter.c diff --git a/optee_os/core/kernel/tee_misc.c b/optee_os/core/kernel/tee_misc.c new file mode 100644 index 0000000..240e702 --- /dev/null +++ b/optee_os/core/kernel/tee_misc.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static uint8_t tee_b2hs_add_base(uint8_t in) +{ + if (in > 9) + return in + 55; + else + return in + 48; +} + +static int tee_hs2b_rem_base(uint8_t in, uint8_t *out) +{ + if (in < 48 || in > 70 || (in > 57 && in < 65)) + return -1; + + if (in < 58) + *out = in - 48; + else + *out = in - 55; + + return 0; +} + +uint32_t tee_b2hs(uint8_t *b, uint8_t *hs, uint32_t blen, uint32_t hslen) +{ + uint32_t i = 0; + + if (blen * 2 + 1 > hslen) + return 0; + + for (; i < blen; i++) { + hs[i * 2 + 1] = tee_b2hs_add_base(b[i] & 0xf); + hs[i * 2] = tee_b2hs_add_base(b[i] >> 4); + } + hs[blen * 2] = 0; + + return blen * 2; +} + +uint32_t tee_hs2b(uint8_t *hs, uint8_t *b, uint32_t hslen, uint32_t blen) +{ + uint32_t i = 0; + uint32_t len = TEE_HS2B_BBUF_SIZE(hslen); + uint8_t hi; + uint8_t lo; + + if (len > blen) + return 0; + + for (; i < len; i++) { + if (tee_hs2b_rem_base(hs[i * 2], &hi)) + return 0; + if (tee_hs2b_rem_base(hs[i * 2 + 1], &lo)) + return 0; + b[i] = (hi << 4) + lo; + } + + return len; +} + +static bool is_valid_conf_and_notnull_size(paddr_t b, paddr_size_t bl, + paddr_t a, paddr_size_t al) +{ + /* invalid config return false */ + if ((b - 1 + bl < b) || (a - 1 + al < a)) + return false; + /* null sized areas are never inside / outside / overlap */ + if (!bl || !al) + return false; + return true; +} + +/* Returns true when buffer 'b' is fully contained in area 'a' */ +bool core_is_buffer_inside(paddr_t b, paddr_size_t bl, + paddr_t a, paddr_size_t al) +{ + /* invalid config or "null size" return false */ + if (!is_valid_conf_and_notnull_size(b, bl, a, al)) + return false; + + if ((b >= a) && (b - 1 + bl <= a - 1 + al)) + return true; + return false; +} + +/* Returns true when buffer 'b' is fully outside area 'a' */ +bool core_is_buffer_outside(paddr_t b, paddr_size_t bl, + paddr_t a, paddr_size_t al) +{ + /* invalid config or "null size" return false */ + if (!is_valid_conf_and_notnull_size(b, bl, a, al)) + return false; + + if ((b + bl - 1 < a) || (b > a + al - 1)) + return true; + return false; +} + +/* Returns true when buffer 'b' intersects area 'a' */ +bool core_is_buffer_intersect(paddr_t b, paddr_size_t bl, + paddr_t a, paddr_size_t al) +{ + /* invalid config or "null size" return false */ + if (!is_valid_conf_and_notnull_size(b, bl, a, al)) + return false; + + if ((b + bl - 1 < a) || (b > a + al - 1)) + return false; + return true; +} + +void *alloc_cache_aligned(size_t size) +{ + void *ptr = NULL; + size_t alloc_size = 0; + uint32_t cacheline_size = 0; + + cacheline_size = cache_get_max_line_size(); + if (ROUNDUP_OVERFLOW(size, cacheline_size, &alloc_size)) + return NULL; + + ptr = memalign(cacheline_size, alloc_size); + if (!ptr) + return NULL; + + memset(ptr, 0, size); + + return ptr; +} diff --git a/optee_os/core/kernel/tee_ta_manager.c b/optee_os/core/kernel/tee_ta_manager.c new file mode 100644 index 0000000..97dee43 --- /dev/null +++ b/optee_os/core/kernel/tee_ta_manager.c @@ -0,0 +1,1141 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CFG_TA_STATS) +#define MAX_DUMP_SESS_NUM (16) +struct tee_ta_dump_stats { + TEE_UUID uuid; + uint32_t panicked; /* True if TA has panicked */ + uint32_t sess_num; /* Number of opened session */ + struct malloc_stats heap; +}; + +struct tee_ta_dump_ctx { + TEE_UUID uuid; + uint32_t panicked; + bool is_user_ta; + uint32_t sess_num; + uint32_t sess_id[MAX_DUMP_SESS_NUM]; +}; +#endif + +/* This mutex protects the critical section in tee_ta_init_session */ +struct mutex tee_ta_mutex = MUTEX_INITIALIZER; +/* This condvar is used when waiting for a TA context to become initialized */ +struct condvar tee_ta_init_cv = CONDVAR_INITIALIZER; +struct tee_ta_ctx_head tee_ctxes = TAILQ_HEAD_INITIALIZER(tee_ctxes); + +#ifndef CFG_CONCURRENT_SINGLE_INSTANCE_TA +static struct condvar tee_ta_cv = CONDVAR_INITIALIZER; +static short int tee_ta_single_instance_thread = THREAD_ID_INVALID; +static size_t tee_ta_single_instance_count; +#endif + +#ifdef CFG_CONCURRENT_SINGLE_INSTANCE_TA +static void lock_single_instance(void) +{ +} + +static void unlock_single_instance(void) +{ +} + +static bool has_single_instance_lock(void) +{ + return false; +} +#else +static void lock_single_instance(void) +{ + /* Requires tee_ta_mutex to be held */ + if (tee_ta_single_instance_thread != thread_get_id()) { + /* Wait until the single-instance lock is available. */ + while (tee_ta_single_instance_thread != THREAD_ID_INVALID) + condvar_wait(&tee_ta_cv, &tee_ta_mutex); + + tee_ta_single_instance_thread = thread_get_id(); + assert(tee_ta_single_instance_count == 0); + } + + tee_ta_single_instance_count++; +} + +static void unlock_single_instance(void) +{ + /* Requires tee_ta_mutex to be held */ + assert(tee_ta_single_instance_thread == thread_get_id()); + assert(tee_ta_single_instance_count > 0); + + tee_ta_single_instance_count--; + if (tee_ta_single_instance_count == 0) { + tee_ta_single_instance_thread = THREAD_ID_INVALID; + condvar_signal(&tee_ta_cv); + } +} + +static bool has_single_instance_lock(void) +{ + /* Requires tee_ta_mutex to be held */ + return tee_ta_single_instance_thread == thread_get_id(); +} +#endif + +struct tee_ta_session *__noprof to_ta_session(struct ts_session *sess) +{ + assert(is_ta_ctx(sess->ctx) || is_stmm_ctx(sess->ctx)); + return container_of(sess, struct tee_ta_session, ts_sess); +} + +static struct tee_ta_ctx *ts_to_ta_ctx(struct ts_ctx *ctx) +{ + if (is_ta_ctx(ctx)) + return to_ta_ctx(ctx); + + if (is_stmm_ctx(ctx)) + return &(to_stmm_ctx(ctx)->ta_ctx); + + panic("bad context"); +} + +static bool tee_ta_try_set_busy(struct tee_ta_ctx *ctx) +{ + bool rc = true; + + if (ctx->flags & TA_FLAG_CONCURRENT) + return true; + + mutex_lock(&tee_ta_mutex); + + if (ctx->flags & TA_FLAG_SINGLE_INSTANCE) + lock_single_instance(); + + if (has_single_instance_lock()) { + if (ctx->busy) { + /* + * We're holding the single-instance lock and the + * TA is busy, as waiting now would only cause a + * dead-lock, we release the lock and return false. + */ + rc = false; + if (ctx->flags & TA_FLAG_SINGLE_INSTANCE) + unlock_single_instance(); + } + } else { + /* + * We're not holding the single-instance lock, we're free to + * wait for the TA to become available. + */ + while (ctx->busy) + condvar_wait(&ctx->busy_cv, &tee_ta_mutex); + } + + /* Either it's already true or we should set it to true */ + ctx->busy = true; + + mutex_unlock(&tee_ta_mutex); + return rc; +} + +static void tee_ta_set_busy(struct tee_ta_ctx *ctx) +{ + if (!tee_ta_try_set_busy(ctx)) + panic(); +} + +static void tee_ta_clear_busy(struct tee_ta_ctx *ctx) +{ + if (ctx->flags & TA_FLAG_CONCURRENT) + return; + + mutex_lock(&tee_ta_mutex); + + assert(ctx->busy); + ctx->busy = false; + condvar_signal(&ctx->busy_cv); + + if (ctx->flags & TA_FLAG_SINGLE_INSTANCE) + unlock_single_instance(); + + mutex_unlock(&tee_ta_mutex); +} + +static void dec_session_ref_count(struct tee_ta_session *s) +{ + assert(s->ref_count > 0); + s->ref_count--; + if (s->ref_count == 1) + condvar_signal(&s->refc_cv); +} + +void tee_ta_put_session(struct tee_ta_session *s) +{ + mutex_lock(&tee_ta_mutex); + + if (s->lock_thread == thread_get_id()) { + s->lock_thread = THREAD_ID_INVALID; + condvar_signal(&s->lock_cv); + } + dec_session_ref_count(s); + + mutex_unlock(&tee_ta_mutex); +} + +static struct tee_ta_session *tee_ta_find_session_nolock(uint32_t id, + struct tee_ta_session_head *open_sessions) +{ + struct tee_ta_session *s = NULL; + struct tee_ta_session *found = NULL; + + TAILQ_FOREACH(s, open_sessions, link) { + if (s->id == id) { + found = s; + break; + } + } + + return found; +} + +struct tee_ta_session *tee_ta_find_session(uint32_t id, + struct tee_ta_session_head *open_sessions) +{ + struct tee_ta_session *s = NULL; + + mutex_lock(&tee_ta_mutex); + + s = tee_ta_find_session_nolock(id, open_sessions); + + mutex_unlock(&tee_ta_mutex); + + return s; +} + +struct tee_ta_session *tee_ta_get_session(uint32_t id, bool exclusive, + struct tee_ta_session_head *open_sessions) +{ + struct tee_ta_session *s; + + mutex_lock(&tee_ta_mutex); + + while (true) { + s = tee_ta_find_session_nolock(id, open_sessions); + if (!s) + break; + if (s->unlink) { + s = NULL; + break; + } + s->ref_count++; + if (!exclusive) + break; + + assert(s->lock_thread != thread_get_id()); + + while (s->lock_thread != THREAD_ID_INVALID && !s->unlink) + condvar_wait(&s->lock_cv, &tee_ta_mutex); + + if (s->unlink) { + dec_session_ref_count(s); + s = NULL; + break; + } + + s->lock_thread = thread_get_id(); + break; + } + + mutex_unlock(&tee_ta_mutex); + return s; +} + +static void tee_ta_unlink_session(struct tee_ta_session *s, + struct tee_ta_session_head *open_sessions) +{ + mutex_lock(&tee_ta_mutex); + + assert(s->ref_count >= 1); + assert(s->lock_thread == thread_get_id()); + assert(!s->unlink); + + s->unlink = true; + condvar_broadcast(&s->lock_cv); + + while (s->ref_count != 1) + condvar_wait(&s->refc_cv, &tee_ta_mutex); + + TAILQ_REMOVE(open_sessions, s, link); + + mutex_unlock(&tee_ta_mutex); +} + +static void destroy_session(struct tee_ta_session *s, + struct tee_ta_session_head *open_sessions) +{ +#if defined(CFG_FTRACE_SUPPORT) + if (s->ts_sess.ctx && s->ts_sess.ctx->ops->dump_ftrace) { + ts_push_current_session(&s->ts_sess); + s->ts_sess.fbuf = NULL; + s->ts_sess.ctx->ops->dump_ftrace(s->ts_sess.ctx); + ts_pop_current_session(); + } +#endif + + tee_ta_unlink_session(s, open_sessions); +#if defined(CFG_TA_GPROF_SUPPORT) + free(s->ts_sess.sbuf); +#endif + free(s); +} + +static void destroy_context(struct tee_ta_ctx *ctx) +{ + DMSG("Destroy TA ctx (0x%" PRIxVA ")", (vaddr_t)ctx); + + condvar_destroy(&ctx->busy_cv); + ctx->ts_ctx.ops->destroy(&ctx->ts_ctx); +} + +/* + * tee_ta_context_find - Find TA in session list based on a UUID (input) + * Returns a pointer to the session + */ +static struct tee_ta_ctx *tee_ta_context_find(const TEE_UUID *uuid) +{ + struct tee_ta_ctx *ctx; + + TAILQ_FOREACH(ctx, &tee_ctxes, link) { + if (memcmp(&ctx->ts_ctx.uuid, uuid, sizeof(TEE_UUID)) == 0) + return ctx; + } + + return NULL; +} + +/* check if requester (client ID) matches session initial client */ +static TEE_Result check_client(struct tee_ta_session *s, const TEE_Identity *id) +{ + if (id == KERN_IDENTITY) + return TEE_SUCCESS; + + if (id == NSAPP_IDENTITY) { + if (s->clnt_id.login == TEE_LOGIN_TRUSTED_APP) { + DMSG("nsec tries to hijack TA session"); + return TEE_ERROR_ACCESS_DENIED; + } + return TEE_SUCCESS; + } + + if (memcmp(&s->clnt_id, id, sizeof(TEE_Identity)) != 0) { + DMSG("client id mismatch"); + return TEE_ERROR_ACCESS_DENIED; + } + return TEE_SUCCESS; +} + +/* + * Check if invocation parameters matches TA properties + * + * @s - current session handle + * @param - already identified memory references hold a valid 'mobj'. + * + * Policy: + * - All TAs can access 'non-secure' shared memory. + * - All TAs can access TEE private memory (seccpy) + * - Only SDP flagged TAs can accept SDP memory references. + */ +#ifndef CFG_SECURE_DATA_PATH +static bool check_params(struct tee_ta_session *sess __unused, + struct tee_ta_param *param __unused) +{ + /* + * When CFG_SECURE_DATA_PATH is not enabled, SDP memory references + * are rejected at OP-TEE core entry. Hence here all TAs have same + * permissions regarding memory reference parameters. + */ + return true; +} +#else +static bool check_params(struct tee_ta_session *sess, + struct tee_ta_param *param) +{ + int n; + + /* + * When CFG_SECURE_DATA_PATH is enabled, OP-TEE entry allows SHM and + * SDP memory references. Only TAs flagged SDP can access SDP memory. + */ + if (sess->ts_sess.ctx && + ts_to_ta_ctx(sess->ts_sess.ctx)->flags & TA_FLAG_SECURE_DATA_PATH) + return true; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uint32_t param_type = TEE_PARAM_TYPE_GET(param->types, n); + struct param_mem *mem = ¶m->u[n].mem; + + if (param_type != TEE_PARAM_TYPE_MEMREF_INPUT && + param_type != TEE_PARAM_TYPE_MEMREF_OUTPUT && + param_type != TEE_PARAM_TYPE_MEMREF_INOUT) + continue; + if (!mem->size) + continue; + if (mobj_is_sdp_mem(mem->mobj)) + return false; + } + return true; +} +#endif + +static void set_invoke_timeout(struct tee_ta_session *sess, + uint32_t cancel_req_to) +{ + TEE_Time current_time; + TEE_Time cancel_time; + + if (cancel_req_to == TEE_TIMEOUT_INFINITE) + goto infinite; + + if (tee_time_get_sys_time(¤t_time) != TEE_SUCCESS) + goto infinite; + + if (ADD_OVERFLOW(current_time.seconds, cancel_req_to / 1000, + &cancel_time.seconds)) + goto infinite; + + cancel_time.millis = current_time.millis + cancel_req_to % 1000; + if (cancel_time.millis > 1000) { + if (ADD_OVERFLOW(current_time.seconds, 1, + &cancel_time.seconds)) + goto infinite; + + cancel_time.seconds++; + cancel_time.millis -= 1000; + } + + sess->cancel_time = cancel_time; + return; + +infinite: + sess->cancel_time.seconds = UINT32_MAX; + sess->cancel_time.millis = UINT32_MAX; +} + +/*----------------------------------------------------------------------------- + * Close a Trusted Application and free available resources + *---------------------------------------------------------------------------*/ +TEE_Result tee_ta_close_session(struct tee_ta_session *csess, + struct tee_ta_session_head *open_sessions, + const TEE_Identity *clnt_id) +{ + struct tee_ta_session *sess = NULL; + struct tee_ta_ctx *ctx = NULL; + struct ts_ctx *ts_ctx = NULL; + bool keep_alive = false; + + DMSG("csess 0x%" PRIxVA " id %u", + (vaddr_t)csess, csess ? csess->id : UINT_MAX); + + if (!csess) + return TEE_ERROR_ITEM_NOT_FOUND; + + sess = tee_ta_get_session(csess->id, true, open_sessions); + + if (!sess) { + EMSG("session 0x%" PRIxVA " to be removed is not found", + (vaddr_t)csess); + return TEE_ERROR_ITEM_NOT_FOUND; + } + + if (check_client(sess, clnt_id) != TEE_SUCCESS) { + tee_ta_put_session(sess); + return TEE_ERROR_BAD_PARAMETERS; /* intentional generic error */ + } + + DMSG("Destroy session"); + + ts_ctx = sess->ts_sess.ctx; + if (!ts_ctx) { + destroy_session(sess, open_sessions); + return TEE_SUCCESS; + } + + ctx = ts_to_ta_ctx(ts_ctx); + if (ctx->panicked) { + destroy_session(sess, open_sessions); + } else { + tee_ta_set_busy(ctx); + set_invoke_timeout(sess, TEE_TIMEOUT_INFINITE); + ts_ctx->ops->enter_close_session(&sess->ts_sess); + destroy_session(sess, open_sessions); + tee_ta_clear_busy(ctx); + } + + mutex_lock(&tee_ta_mutex); + + if (ctx->ref_count <= 0) + panic(); + + ctx->ref_count--; + keep_alive = (ctx->flags & TA_FLAG_INSTANCE_KEEP_ALIVE) && + (ctx->flags & TA_FLAG_SINGLE_INSTANCE); + if (!ctx->ref_count && (ctx->panicked || !keep_alive)) { + if (!ctx->is_releasing) { + TAILQ_REMOVE(&tee_ctxes, ctx, link); + ctx->is_releasing = true; + } + mutex_unlock(&tee_ta_mutex); + + destroy_context(ctx); + } else + mutex_unlock(&tee_ta_mutex); + + return TEE_SUCCESS; +} + +static TEE_Result tee_ta_init_session_with_context(struct tee_ta_session *s, + const TEE_UUID *uuid) +{ + struct tee_ta_ctx *ctx = NULL; + + while (true) { + ctx = tee_ta_context_find(uuid); + if (!ctx) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (!is_user_ta_ctx(&ctx->ts_ctx) || + !to_user_ta_ctx(&ctx->ts_ctx)->uctx.is_initializing) + break; + /* + * Context is still initializing, wait here until it's + * fully initialized. Note that we're searching for the + * context again since it may have been removed while we + * where sleeping. + */ + condvar_wait(&tee_ta_init_cv, &tee_ta_mutex); + } + + /* + * If TA isn't single instance it should be loaded as new + * instance instead of doing anything with this instance. + * So tell the caller that we didn't find the TA it the + * caller will load a new instance. + */ + if ((ctx->flags & TA_FLAG_SINGLE_INSTANCE) == 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* + * The TA is single instance, if it isn't multi session we + * can't create another session unless its reference is zero + */ + if (!(ctx->flags & TA_FLAG_MULTI_SESSION) && ctx->ref_count) + return TEE_ERROR_BUSY; + + DMSG("Re-open TA %pUl", (void *)&ctx->ts_ctx.uuid); + + ctx->ref_count++; + s->ts_sess.ctx = &ctx->ts_ctx; + s->ts_sess.handle_scall = s->ts_sess.ctx->ops->handle_scall; + return TEE_SUCCESS; +} + +static uint32_t new_session_id(struct tee_ta_session_head *open_sessions) +{ + struct tee_ta_session *last = NULL; + uint32_t saved = 0; + uint32_t id = 1; + + last = TAILQ_LAST(open_sessions, tee_ta_session_head); + if (last) { + /* This value is less likely to be already used */ + id = last->id + 1; + if (!id) + id++; /* 0 is not valid */ + } + + saved = id; + do { + if (!tee_ta_find_session_nolock(id, open_sessions)) + return id; + id++; + if (!id) + id++; + } while (id != saved); + + return 0; +} + +static TEE_Result tee_ta_init_session(TEE_ErrorOrigin *err, + struct tee_ta_session_head *open_sessions, + const TEE_UUID *uuid, + struct tee_ta_session **sess) +{ + TEE_Result res; + struct tee_ta_session *s = calloc(1, sizeof(struct tee_ta_session)); + + *err = TEE_ORIGIN_TEE; + if (!s) + return TEE_ERROR_OUT_OF_MEMORY; + + s->cancel_mask = true; + condvar_init(&s->refc_cv); + condvar_init(&s->lock_cv); + s->lock_thread = THREAD_ID_INVALID; + s->ref_count = 1; + + mutex_lock(&tee_ta_mutex); + s->id = new_session_id(open_sessions); + if (!s->id) { + res = TEE_ERROR_OVERFLOW; + goto err_mutex_unlock; + } + + TAILQ_INSERT_TAIL(open_sessions, s, link); + + /* Look for already loaded TA */ + res = tee_ta_init_session_with_context(s, uuid); + mutex_unlock(&tee_ta_mutex); + if (res == TEE_SUCCESS || res != TEE_ERROR_ITEM_NOT_FOUND) + goto out; + + /* Look for secure partition */ + res = stmm_init_session(uuid, s); + if (res == TEE_SUCCESS || res != TEE_ERROR_ITEM_NOT_FOUND) + goto out; + + /* Look for pseudo TA */ + res = tee_ta_init_pseudo_ta_session(uuid, s); + if (res == TEE_SUCCESS || res != TEE_ERROR_ITEM_NOT_FOUND) + goto out; + + /* Look for user TA */ + res = tee_ta_init_user_ta_session(uuid, s); + +out: + if (!res) { + *sess = s; + return TEE_SUCCESS; + } + + mutex_lock(&tee_ta_mutex); + TAILQ_REMOVE(open_sessions, s, link); +err_mutex_unlock: + mutex_unlock(&tee_ta_mutex); + free(s); + return res; +} + +static void release_ta_ctx(struct tee_ta_ctx *ctx) +{ + bool was_releasing = false; + + mutex_lock(&tee_ta_mutex); + was_releasing = ctx->is_releasing; + ctx->is_releasing = true; + if (!was_releasing) { + DMSG("Releasing panicked TA ctx"); + TAILQ_REMOVE(&tee_ctxes, ctx, link); + } + mutex_unlock(&tee_ta_mutex); + + if (!was_releasing) + ctx->ts_ctx.ops->release_state(&ctx->ts_ctx); +} + +TEE_Result tee_ta_open_session(TEE_ErrorOrigin *err, + struct tee_ta_session **sess, + struct tee_ta_session_head *open_sessions, + const TEE_UUID *uuid, + const TEE_Identity *clnt_id, + uint32_t cancel_req_to, + struct tee_ta_param *param) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_ta_session *s = NULL; + struct tee_ta_ctx *ctx = NULL; + struct ts_ctx *ts_ctx = NULL; + bool panicked = false; + bool was_busy = false; + + res = tee_ta_init_session(err, open_sessions, uuid, &s); + if (res != TEE_SUCCESS) { + DMSG("init session failed 0x%x", res); + return res; + } + + if (!check_params(s, param)) + return TEE_ERROR_BAD_PARAMETERS; + + ts_ctx = s->ts_sess.ctx; + ctx = ts_to_ta_ctx(ts_ctx); + + if (tee_ta_try_set_busy(ctx)) { + if (!ctx->panicked) { + /* Save identity of the owner of the session */ + s->clnt_id = *clnt_id; + s->param = param; + set_invoke_timeout(s, cancel_req_to); + res = ts_ctx->ops->enter_open_session(&s->ts_sess); + s->param = NULL; + } + + panicked = ctx->panicked; + if (panicked) { + release_ta_ctx(ctx); + res = TEE_ERROR_TARGET_DEAD; + } + + tee_ta_clear_busy(ctx); + } else { + /* Deadlock avoided */ + res = TEE_ERROR_BUSY; + was_busy = true; + } + + /* + * Origin error equal to TEE_ORIGIN_TRUSTED_APP for "regular" error, + * apart from panicking. + */ + if (panicked || was_busy) + *err = TEE_ORIGIN_TEE; + else + *err = s->err_origin; + + tee_ta_put_session(s); + if (panicked || res != TEE_SUCCESS) + tee_ta_close_session(s, open_sessions, KERN_IDENTITY); + + if (!res) + *sess = s; + else + EMSG("Failed. Return error 0x%x", res); + + return res; +} + +TEE_Result tee_ta_invoke_command(TEE_ErrorOrigin *err, + struct tee_ta_session *sess, + const TEE_Identity *clnt_id, + uint32_t cancel_req_to, uint32_t cmd, + struct tee_ta_param *param) +{ + struct tee_ta_ctx *ta_ctx = NULL; + struct ts_ctx *ts_ctx = NULL; + TEE_Result res = TEE_SUCCESS; + bool panicked = false; + + if (check_client(sess, clnt_id) != TEE_SUCCESS) + return TEE_ERROR_BAD_PARAMETERS; /* intentional generic error */ + + if (!check_params(sess, param)) + return TEE_ERROR_BAD_PARAMETERS; + + ts_ctx = sess->ts_sess.ctx; + ta_ctx = ts_to_ta_ctx(ts_ctx); + + tee_ta_set_busy(ta_ctx); + + if (!ta_ctx->panicked) { + sess->param = param; + set_invoke_timeout(sess, cancel_req_to); + res = ts_ctx->ops->enter_invoke_cmd(&sess->ts_sess, cmd); + sess->param = NULL; + } + + panicked = ta_ctx->panicked; + if (panicked) { + release_ta_ctx(ta_ctx); + res = TEE_ERROR_TARGET_DEAD; + } + + tee_ta_clear_busy(ta_ctx); + + /* + * Origin error equal to TEE_ORIGIN_TRUSTED_APP for "regular" error, + * apart from panicking. + */ + if (panicked) + *err = TEE_ORIGIN_TEE; + else + *err = sess->err_origin; + + /* Short buffer is not an effective error case */ + if (res != TEE_SUCCESS && res != TEE_ERROR_SHORT_BUFFER) + DMSG("Error: %x of %d", res, *err); + + return res; +} + +#if defined(CFG_TA_STATS) +static TEE_Result dump_ta_memstats(struct tee_ta_session *s, + struct tee_ta_param *param) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_ta_ctx *ctx = NULL; + struct ts_ctx *ts_ctx = NULL; + bool panicked = false; + + ts_ctx = s->ts_sess.ctx; + if (!ts_ctx) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (is_user_ta_ctx(ts_ctx) && + to_user_ta_ctx(ts_ctx)->uctx.is_initializing) + return TEE_ERROR_BAD_STATE; + + ctx = ts_to_ta_ctx(ts_ctx); + + if (tee_ta_try_set_busy(ctx)) { + if (!ctx->panicked) { + s->param = param; + set_invoke_timeout(s, TEE_TIMEOUT_INFINITE); + res = ts_ctx->ops->dump_mem_stats(&s->ts_sess); + s->param = NULL; + } + + panicked = ctx->panicked; + if (panicked) { + release_ta_ctx(ctx); + res = TEE_ERROR_TARGET_DEAD; + } + + tee_ta_clear_busy(ctx); + } else { + /* Deadlock avoided */ + res = TEE_ERROR_BUSY; + } + + return res; +} + +static void init_dump_ctx(struct tee_ta_dump_ctx *dump_ctx) +{ + struct tee_ta_session *sess = NULL; + struct tee_ta_session_head *open_sessions = NULL; + struct tee_ta_ctx *ctx = NULL; + unsigned int n = 0; + + nsec_sessions_list_head(&open_sessions); + /* + * Scan all sessions opened from secure side by searching through + * all available TA instances and for each context, scan all opened + * sessions. + */ + TAILQ_FOREACH(ctx, &tee_ctxes, link) { + unsigned int cnt = 0; + + if (!is_user_ta_ctx(&ctx->ts_ctx)) + continue; + + memcpy(&dump_ctx[n].uuid, &ctx->ts_ctx.uuid, + sizeof(ctx->ts_ctx.uuid)); + dump_ctx[n].panicked = ctx->panicked; + dump_ctx[n].is_user_ta = is_user_ta_ctx(&ctx->ts_ctx); + TAILQ_FOREACH(sess, open_sessions, link) { + if (sess->ts_sess.ctx == &ctx->ts_ctx) { + if (cnt == MAX_DUMP_SESS_NUM) + break; + + dump_ctx[n].sess_id[cnt] = sess->id; + cnt++; + } + } + + dump_ctx[n].sess_num = cnt; + n++; + } +} + +static TEE_Result dump_ta_stats(struct tee_ta_dump_ctx *dump_ctx, + struct tee_ta_dump_stats *dump_stats, + size_t ta_count) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_ta_session *sess = NULL; + struct tee_ta_session_head *open_sessions = NULL; + struct tee_ta_param param = { }; + unsigned int i = 0; + unsigned int j = 0; + + nsec_sessions_list_head(&open_sessions); + + for (i = 0; i < ta_count; i++) { + struct tee_ta_dump_stats *stats = &dump_stats[i]; + + memcpy(&stats->uuid, &dump_ctx[i].uuid, + sizeof(dump_ctx[i].uuid)); + stats->panicked = dump_ctx[i].panicked; + stats->sess_num = dump_ctx[i].sess_num; + + /* Find a session from dump context */ + for (j = 0, sess = NULL; j < dump_ctx[i].sess_num && !sess; j++) + sess = tee_ta_get_session(dump_ctx[i].sess_id[j], true, + open_sessions); + + if (!sess) + continue; + /* If session is existing, get its heap stats */ + memset(¶m, 0, sizeof(struct tee_ta_param)); + param.types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE); + res = dump_ta_memstats(sess, ¶m); + if (res == TEE_SUCCESS) { + stats->heap.allocated = param.u[0].val.a; + stats->heap.max_allocated = param.u[0].val.b; + stats->heap.size = param.u[1].val.a; + stats->heap.num_alloc_fail = param.u[1].val.b; + stats->heap.biggest_alloc_fail = param.u[2].val.a; + stats->heap.biggest_alloc_fail_used = param.u[2].val.b; + } else { + memset(&stats->heap, 0, sizeof(stats->heap)); + } + tee_ta_put_session(sess); + } + + return TEE_SUCCESS; +} + +TEE_Result tee_ta_instance_stats(void *buf, size_t *buf_size) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_ta_dump_stats *dump_stats = NULL; + struct tee_ta_dump_ctx *dump_ctx = NULL; + struct tee_ta_ctx *ctx = NULL; + size_t sz = 0; + size_t ta_count = 0; + + if (!buf_size) + return TEE_ERROR_BAD_PARAMETERS; + + mutex_lock(&tee_ta_mutex); + + /* Go through all available TA and calc out the actual buffer size. */ + TAILQ_FOREACH(ctx, &tee_ctxes, link) + if (is_user_ta_ctx(&ctx->ts_ctx)) + ta_count++; + + sz = sizeof(struct tee_ta_dump_stats) * ta_count; + if (!sz) { + /* sz = 0 means there is no UTA, return no item found. */ + res = TEE_ERROR_ITEM_NOT_FOUND; + } else if (!buf || *buf_size < sz) { + /* + * buf is null or pass size less than actual size + * means caller try to query the buffer size. + * update *buf_size. + */ + *buf_size = sz; + res = TEE_ERROR_SHORT_BUFFER; + } else if (!IS_ALIGNED_WITH_TYPE(buf, uint32_t)) { + DMSG("Data alignment"); + res = TEE_ERROR_BAD_PARAMETERS; + } else { + dump_stats = (struct tee_ta_dump_stats *)buf; + dump_ctx = malloc(sizeof(struct tee_ta_dump_ctx) * ta_count); + if (!dump_ctx) + res = TEE_ERROR_OUT_OF_MEMORY; + else + init_dump_ctx(dump_ctx); + } + mutex_unlock(&tee_ta_mutex); + + if (res != TEE_SUCCESS) + return res; + + /* Dump user ta stats by iterating dump_ctx[] */ + res = dump_ta_stats(dump_ctx, dump_stats, ta_count); + if (res == TEE_SUCCESS) + *buf_size = sz; + + free(dump_ctx); + return res; +} +#endif + +TEE_Result tee_ta_cancel_command(TEE_ErrorOrigin *err, + struct tee_ta_session *sess, + const TEE_Identity *clnt_id) +{ + *err = TEE_ORIGIN_TEE; + + if (check_client(sess, clnt_id) != TEE_SUCCESS) + return TEE_ERROR_BAD_PARAMETERS; /* intentional generic error */ + + sess->cancel = true; + return TEE_SUCCESS; +} + +bool tee_ta_session_is_cancelled(struct tee_ta_session *s, TEE_Time *curr_time) +{ + TEE_Time current_time; + + if (s->cancel_mask) + return false; + + if (s->cancel) + return true; + + if (s->cancel_time.seconds == UINT32_MAX) + return false; + + if (curr_time != NULL) + current_time = *curr_time; + else if (tee_time_get_sys_time(¤t_time) != TEE_SUCCESS) + return false; + + if (current_time.seconds > s->cancel_time.seconds || + (current_time.seconds == s->cancel_time.seconds && + current_time.millis >= s->cancel_time.millis)) { + return true; + } + + return false; +} + +#if defined(CFG_TA_GPROF_SUPPORT) +void tee_ta_gprof_sample_pc(vaddr_t pc) +{ + struct ts_session *s = ts_get_current_session(); + struct user_ta_ctx *utc = NULL; + struct sample_buf *sbuf = NULL; + TEE_Result res = 0; + size_t idx = 0; + + sbuf = s->sbuf; + if (!sbuf || !sbuf->enabled) + return; /* PC sampling is not enabled */ + + idx = (((uint64_t)pc - sbuf->offset)/2 * sbuf->scale)/65536; + if (idx < sbuf->nsamples) { + utc = to_user_ta_ctx(s->ctx); + res = vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)&sbuf->samples[idx], + sizeof(*sbuf->samples)); + if (res != TEE_SUCCESS) + return; + sbuf->samples[idx]++; + } + sbuf->count++; +} + +static void gprof_update_session_utime(bool suspend, struct ts_session *s, + uint64_t now) +{ + struct sample_buf *sbuf = s->sbuf; + + if (!sbuf) + return; + + if (suspend) { + assert(sbuf->usr_entered); + sbuf->usr += now - sbuf->usr_entered; + sbuf->usr_entered = 0; + } else { + assert(!sbuf->usr_entered); + if (!now) + now++; /* 0 is reserved */ + sbuf->usr_entered = now; + } +} + +/* + * Update user-mode CPU time for the current session + * @suspend: true if session is being suspended (leaving user mode), false if + * it is resumed (entering user mode) + */ +static void tee_ta_update_session_utime(bool suspend) +{ + struct ts_session *s = ts_get_current_session(); + uint64_t now = barrier_read_counter_timer(); + + gprof_update_session_utime(suspend, s, now); +} + +void tee_ta_update_session_utime_suspend(void) +{ + tee_ta_update_session_utime(true); +} + +void tee_ta_update_session_utime_resume(void) +{ + tee_ta_update_session_utime(false); +} +#endif + +#if defined(CFG_FTRACE_SUPPORT) +static void ftrace_update_times(bool suspend) +{ + struct ts_session *s = ts_get_current_session_may_fail(); + struct ftrace_buf *fbuf = NULL; + uint64_t now = 0; + uint32_t i = 0; + + if (!s) + return; + + now = barrier_read_counter_timer(); + + fbuf = s->fbuf; + if (!fbuf) + return; + + if (suspend) { + fbuf->suspend_time = now; + } else { + for (i = 0; i <= fbuf->ret_idx; i++) + fbuf->begin_time[i] += now - fbuf->suspend_time; + } +} + +void tee_ta_ftrace_update_times_suspend(void) +{ + ftrace_update_times(true); +} + +void tee_ta_ftrace_update_times_resume(void) +{ + ftrace_update_times(false); +} +#endif + +bool __noprof is_ta_ctx(struct ts_ctx *ctx) +{ + return is_user_ta_ctx(ctx) || is_pseudo_ta_ctx(ctx); +} diff --git a/optee_os/core/kernel/tee_time.c b/optee_os/core/kernel/tee_time.c new file mode 100644 index 0000000..e1a462b --- /dev/null +++ b/optee_os/core/kernel/tee_time.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2020, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct time_source _time_source; + +static TEE_Result register_time_source(void) +{ + time_source_init(); + + return TEE_SUCCESS; +} +early_init(register_time_source); + +TEE_Result tee_time_get_sys_time(TEE_Time *time) +{ + return _time_source.get_sys_time(time); +} + +uint32_t tee_time_get_sys_time_protection_level(void) +{ + return _time_source.protection_level; +} + +void tee_time_wait(uint32_t milliseconds_delay) +{ + struct thread_param params = + THREAD_PARAM_VALUE(IN, milliseconds_delay, 0, 0); + + thread_rpc_cmd(OPTEE_RPC_CMD_SUSPEND, 1, ¶ms); +} + +/* + * tee_time_get_ree_time(): this function implements the GP Internal API + * function TEE_GetREETime() + * Goal is to get the time of the Rich Execution Environment + * This is why this time is provided through the supplicant + */ +TEE_Result tee_time_get_ree_time(TEE_Time *time) +{ + struct thread_param params = THREAD_PARAM_VALUE(OUT, 0, 0, 0); + TEE_Result res = TEE_SUCCESS; + + if (!time) + return TEE_ERROR_BAD_PARAMETERS; + + res = thread_rpc_cmd(OPTEE_RPC_CMD_GET_TIME, 1, ¶ms); + if (res == TEE_SUCCESS) { + time->seconds = params.u.value.a; + time->millis = params.u.value.b / 1000000; + } + + return res; +} diff --git a/optee_os/core/kernel/tee_time_ree.c b/optee_os/core/kernel/tee_time_ree.c new file mode 100644 index 0000000..5e744ad --- /dev/null +++ b/optee_os/core/kernel/tee_time_ree.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Linaro Limited + */ + +#include +#include +#include + +static TEE_Time prev; + +static struct mutex time_mu = MUTEX_INITIALIZER; + +static TEE_Result get_monotonic_ree_time(TEE_Time *time) +{ + TEE_Result res; + + res = tee_time_get_ree_time(time); + if (res != TEE_SUCCESS) + return res; + + mutex_lock(&time_mu); + if (time->seconds < prev.seconds || + (time->seconds == prev.seconds && + time->millis < prev.millis)) + *time = prev; /* REE time was rolled back */ + else + prev = *time; + mutex_unlock(&time_mu); + + return res; +} + +static const struct time_source ree_time_source = { + .name = "ree", + .protection_level = 100, + .get_sys_time = get_monotonic_ree_time, +}; + +REGISTER_TIME_SOURCE(ree_time_source) diff --git a/optee_os/core/kernel/thread.c b/optee_os/core/kernel/thread.c new file mode 100644 index 0000000..2a1f22d --- /dev/null +++ b/optee_os/core/kernel/thread.c @@ -0,0 +1,695 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2022, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020-2021, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct thread_ctx threads[CFG_NUM_THREADS]; + +struct thread_core_local thread_core_local[CFG_TEE_CORE_NB_CORE] __nex_bss; + +/* + * Stacks + * + * [Lower addresses on the left] + * + * [ STACK_CANARY_SIZE/2 | STACK_CHECK_EXTRA | STACK_XXX_SIZE | STACK_CANARY_SIZE/2 ] + * ^ ^ ^ ^ + * stack_xxx[n] "hard" top "soft" top bottom + */ + +#ifdef CFG_WITH_STACK_CANARIES +static uint32_t start_canary_value = 0xdedede00; +static uint32_t end_canary_value = 0xababab00; +#define GET_START_CANARY(name, stack_num) name[stack_num][0] +#define GET_END_CANARY(name, stack_num) \ + name[stack_num][sizeof(name[stack_num]) / sizeof(uint32_t) - 1] +#endif + +#define DECLARE_STACK(name, num_stacks, stack_size, linkage) \ +linkage uint32_t name[num_stacks] \ + [ROUNDUP(stack_size + STACK_CANARY_SIZE + STACK_CHECK_EXTRA, \ + STACK_ALIGNMENT) / sizeof(uint32_t)] \ + __attribute__((section(".nozi_stack." # name), \ + aligned(STACK_ALIGNMENT))) + +#define GET_STACK(stack) ((vaddr_t)(stack) + STACK_SIZE(stack)) + +DECLARE_STACK(stack_tmp, CFG_TEE_CORE_NB_CORE, STACK_TMP_SIZE, + /* global linkage */); +DECLARE_STACK(stack_abt, CFG_TEE_CORE_NB_CORE, STACK_ABT_SIZE, static); +#ifndef CFG_WITH_PAGER +DECLARE_STACK(stack_thread, CFG_NUM_THREADS, STACK_THREAD_SIZE, static); +#endif + +#define GET_STACK_TOP_HARD(stack, n) \ + ((vaddr_t)&(stack)[n] + STACK_CANARY_SIZE / 2) +#define GET_STACK_TOP_SOFT(stack, n) \ + (GET_STACK_TOP_HARD(stack, n) + STACK_CHECK_EXTRA) +#define GET_STACK_BOTTOM(stack, n) ((vaddr_t)&(stack)[n] + sizeof(stack[n]) - \ + STACK_CANARY_SIZE / 2) + +const uint32_t stack_tmp_stride __section(".identity_map.stack_tmp_stride") = + sizeof(stack_tmp[0]); + +/* + * This stack setup info is required by secondary boot cores before they + * each locally enable the pager (the mmu). Hence kept in pager sections. + */ +DECLARE_KEEP_PAGER(stack_tmp_stride); + +static unsigned int thread_global_lock __nex_bss = SPINLOCK_UNLOCK; + +void thread_init_canaries(void) +{ +#ifdef CFG_WITH_STACK_CANARIES + size_t n; +#define INIT_CANARY(name) \ + for (n = 0; n < ARRAY_SIZE(name); n++) { \ + uint32_t *start_canary = &GET_START_CANARY(name, n); \ + uint32_t *end_canary = &GET_END_CANARY(name, n); \ + \ + *start_canary = start_canary_value; \ + *end_canary = end_canary_value; \ + } + + INIT_CANARY(stack_tmp); + INIT_CANARY(stack_abt); +#if !defined(CFG_WITH_PAGER) && !defined(CFG_NS_VIRTUALIZATION) + INIT_CANARY(stack_thread); +#endif +#endif/*CFG_WITH_STACK_CANARIES*/ +} + +#if defined(CFG_WITH_STACK_CANARIES) +void thread_update_canaries(void) +{ + uint32_t canary[2] = { }; + uint32_t exceptions = 0; + + plat_get_random_stack_canaries(canary, ARRAY_SIZE(canary), + sizeof(canary[0])); + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + + thread_check_canaries(); + + start_canary_value = canary[0]; + end_canary_value = canary[1]; + thread_init_canaries(); + + thread_unmask_exceptions(exceptions); +} +#endif + +#define CANARY_DIED(stack, loc, n, addr) \ + do { \ + EMSG_RAW("Dead canary at %s of '%s[%zu]' (%p)", #loc, #stack, \ + n, (void *)addr); \ + panic(); \ + } while (0) + +void thread_check_canaries(void) +{ +#ifdef CFG_WITH_STACK_CANARIES + uint32_t *canary = NULL; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(stack_tmp); n++) { + canary = &GET_START_CANARY(stack_tmp, n); + if (*canary != start_canary_value) + CANARY_DIED(stack_tmp, start, n, canary); + canary = &GET_END_CANARY(stack_tmp, n); + if (*canary != end_canary_value) + CANARY_DIED(stack_tmp, end, n, canary); + } + + for (n = 0; n < ARRAY_SIZE(stack_abt); n++) { + canary = &GET_START_CANARY(stack_abt, n); + if (*canary != start_canary_value) + CANARY_DIED(stack_abt, start, n, canary); + canary = &GET_END_CANARY(stack_abt, n); + if (*canary != end_canary_value) + CANARY_DIED(stack_abt, end, n, canary); + } +#if !defined(CFG_WITH_PAGER) && !defined(CFG_NS_VIRTUALIZATION) + for (n = 0; n < ARRAY_SIZE(stack_thread); n++) { + canary = &GET_START_CANARY(stack_thread, n); + if (*canary != start_canary_value) + CANARY_DIED(stack_thread, start, n, canary); + canary = &GET_END_CANARY(stack_thread, n); + if (*canary != end_canary_value) + CANARY_DIED(stack_thread, end, n, canary); + } +#endif +#endif/*CFG_WITH_STACK_CANARIES*/ +} + +void thread_lock_global(void) +{ + cpu_spin_lock(&thread_global_lock); +} + +void thread_unlock_global(void) +{ + cpu_spin_unlock(&thread_global_lock); +} + +static struct thread_core_local * __nostackcheck +get_core_local(unsigned int pos) +{ + /* + * Foreign interrupts must be disabled before playing with core_local + * since we otherwise may be rescheduled to a different core in the + * middle of this function. + */ + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + + assert(pos < CFG_TEE_CORE_NB_CORE); + return &thread_core_local[pos]; +} + +struct thread_core_local * __nostackcheck thread_get_core_local(void) +{ + unsigned int pos = get_core_pos(); + + return get_core_local(pos); +} + +#ifdef CFG_CORE_DEBUG_CHECK_STACKS +static void print_stack_limits(void) +{ + size_t n = 0; + vaddr_t __maybe_unused start = 0; + vaddr_t __maybe_unused end = 0; + + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { + start = GET_STACK_TOP_SOFT(stack_tmp, n); + end = GET_STACK_BOTTOM(stack_tmp, n); + DMSG("tmp [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); + } + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { + start = GET_STACK_TOP_SOFT(stack_abt, n); + end = GET_STACK_BOTTOM(stack_abt, n); + DMSG("abt [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); + } + for (n = 0; n < CFG_NUM_THREADS; n++) { + end = threads[n].stack_va_end; + start = end - STACK_THREAD_SIZE + STACK_CHECK_EXTRA; + DMSG("thr [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); + } +} + +static void check_stack_limits(void) +{ + vaddr_t stack_start = 0; + vaddr_t stack_end = 0; + /* Any value in the current stack frame will do */ + vaddr_t current_sp = (vaddr_t)&stack_start; + + if (!get_stack_soft_limits(&stack_start, &stack_end)) + panic("Unknown stack limits"); + if (current_sp < stack_start || current_sp > stack_end) { + EMSG("Stack pointer out of range: 0x%" PRIxVA " not in [0x%" + PRIxVA " .. 0x%" PRIxVA "]", current_sp, stack_start, + stack_end); + print_stack_limits(); + panic(); + } +} + +static bool * __nostackcheck get_stackcheck_recursion_flag(void) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + unsigned int pos = get_core_pos(); + struct thread_core_local *l = get_core_local(pos); + int ct = l->curr_thread; + bool *p = NULL; + + if (l->flags & (THREAD_CLF_ABORT | THREAD_CLF_TMP)) + p = &l->stackcheck_recursion; + else if (!l->flags) + p = &threads[ct].tsd.stackcheck_recursion; + + thread_unmask_exceptions(exceptions); + return p; +} + +void __cyg_profile_func_enter(void *this_fn, void *call_site); +void __nostackcheck __cyg_profile_func_enter(void *this_fn __unused, + void *call_site __unused) +{ + bool *p = get_stackcheck_recursion_flag(); + + assert(p); + if (*p) + return; + *p = true; + check_stack_limits(); + *p = false; +} + +void __cyg_profile_func_exit(void *this_fn, void *call_site); +void __nostackcheck __cyg_profile_func_exit(void *this_fn __unused, + void *call_site __unused) +{ +} +#else +static void print_stack_limits(void) +{ +} +#endif + +void thread_init_boot_thread(void) +{ + struct thread_core_local *l = thread_get_core_local(); + + thread_init_threads(); + + l->curr_thread = 0; + threads[0].state = THREAD_STATE_ACTIVE; +} + +void __nostackcheck thread_clr_boot_thread(void) +{ + struct thread_core_local *l = thread_get_core_local(); + + assert(l->curr_thread >= 0 && l->curr_thread < CFG_NUM_THREADS); + assert(threads[l->curr_thread].state == THREAD_STATE_ACTIVE); + threads[l->curr_thread].state = THREAD_STATE_FREE; + l->curr_thread = THREAD_ID_INVALID; +} + +void __nostackcheck *thread_get_tmp_sp(void) +{ + struct thread_core_local *l = thread_get_core_local(); + + /* + * Called from assembly when switching to the temporary stack, so flags + * need updating + */ + l->flags |= THREAD_CLF_TMP; + + return (void *)l->tmp_stack_va_end; +} + +vaddr_t thread_stack_start(void) +{ + struct thread_ctx *thr; + int ct = thread_get_id_may_fail(); + + if (ct == THREAD_ID_INVALID) + return 0; + + thr = threads + ct; + return thr->stack_va_end - STACK_THREAD_SIZE; +} + +size_t thread_stack_size(void) +{ + return STACK_THREAD_SIZE; +} + +bool get_stack_limits(vaddr_t *start, vaddr_t *end, bool hard) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + unsigned int pos = get_core_pos(); + struct thread_core_local *l = get_core_local(pos); + int ct = l->curr_thread; + bool ret = false; + + if (l->flags & THREAD_CLF_TMP) { + if (hard) + *start = GET_STACK_TOP_HARD(stack_tmp, pos); + else + *start = GET_STACK_TOP_SOFT(stack_tmp, pos); + *end = GET_STACK_BOTTOM(stack_tmp, pos); + ret = true; + } else if (l->flags & THREAD_CLF_ABORT) { + if (hard) + *start = GET_STACK_TOP_HARD(stack_abt, pos); + else + *start = GET_STACK_TOP_SOFT(stack_abt, pos); + *end = GET_STACK_BOTTOM(stack_abt, pos); + ret = true; + } else if (!l->flags) { + if (ct < 0 || ct >= CFG_NUM_THREADS) + goto out; + + *end = threads[ct].stack_va_end; + *start = *end - STACK_THREAD_SIZE; + if (!hard) + *start += STACK_CHECK_EXTRA; + ret = true; + } +out: + thread_unmask_exceptions(exceptions); + return ret; +} + +bool thread_is_from_abort_mode(void) +{ + struct thread_core_local *l = thread_get_core_local(); + + return (l->flags >> THREAD_CLF_SAVED_SHIFT) & THREAD_CLF_ABORT; +} + +/* + * This function should always be accurate, but it might be possible to + * implement a more efficient depending on cpu architecture. + */ +bool __weak thread_is_in_normal_mode(void) +{ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + struct thread_core_local *l = thread_get_core_local(); + bool ret; + + /* + * If any bit in l->flags is set aside from THREAD_CLF_TMP we're + * handling some exception. + */ + ret = (l->curr_thread != THREAD_ID_INVALID) && + !(l->flags & ~THREAD_CLF_TMP); + thread_unmask_exceptions(exceptions); + + return ret; +} + +short int __noprof thread_get_id_may_fail(void) +{ + /* + * thread_get_core_local() requires foreign interrupts to be disabled + */ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + struct thread_core_local *l = thread_get_core_local(); + short int ct = l->curr_thread; + + thread_unmask_exceptions(exceptions); + return ct; +} + +short int __noprof thread_get_id(void) +{ + short int ct = thread_get_id_may_fail(); + + /* Thread ID has to fit in a short int */ + COMPILE_TIME_ASSERT(CFG_NUM_THREADS <= SHRT_MAX); + assert(ct >= 0 && ct < CFG_NUM_THREADS); + return ct; +} + +#ifdef CFG_WITH_PAGER +static void init_thread_stacks(void) +{ + size_t n = 0; + + /* + * Allocate virtual memory for thread stacks. + */ + for (n = 0; n < CFG_NUM_THREADS; n++) { + tee_mm_entry_t *mm = NULL; + vaddr_t sp = 0; + size_t num_pages = 0; + struct fobj *fobj = NULL; + + /* Find vmem for thread stack and its protection gap */ + mm = tee_mm_alloc(&tee_mm_vcore, + SMALL_PAGE_SIZE + STACK_THREAD_SIZE); + assert(mm); + + /* Claim eventual physical page */ + tee_pager_add_pages(tee_mm_get_smem(mm), tee_mm_get_size(mm), + true); + + num_pages = tee_mm_get_bytes(mm) / SMALL_PAGE_SIZE - 1; + fobj = fobj_locked_paged_alloc(num_pages); + + /* Add the region to the pager */ + tee_pager_add_core_region(tee_mm_get_smem(mm) + SMALL_PAGE_SIZE, + PAGED_REGION_TYPE_LOCK, fobj); + fobj_put(fobj); + + /* init effective stack */ + sp = tee_mm_get_smem(mm) + tee_mm_get_bytes(mm); + asan_tag_access((void *)tee_mm_get_smem(mm), (void *)sp); + if (!thread_init_stack(n, sp)) + panic("init stack failed"); + } +} +#else +static void init_thread_stacks(void) +{ + size_t n; + + /* Assign the thread stacks */ + for (n = 0; n < CFG_NUM_THREADS; n++) { + if (!thread_init_stack(n, GET_STACK_BOTTOM(stack_thread, n))) + panic("thread_init_stack failed"); + } +} +#endif /*CFG_WITH_PAGER*/ + +void thread_init_threads(void) +{ + size_t n = 0; + + init_thread_stacks(); + print_stack_limits(); + pgt_init(); + + mutex_lockdep_init(); + + for (n = 0; n < CFG_NUM_THREADS; n++) + TAILQ_INIT(&threads[n].tsd.sess_stack); +} + +void __nostackcheck thread_init_thread_core_local(void) +{ + size_t n = 0; + struct thread_core_local *tcl = thread_core_local; + + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { + tcl[n].curr_thread = THREAD_ID_INVALID; + tcl[n].flags = THREAD_CLF_TMP; + } + tcl[0].tmp_stack_va_end = GET_STACK_BOTTOM(stack_tmp, 0); +} + +void __nostackcheck thread_init_core_local_stacks(void) +{ + size_t n = 0; + struct thread_core_local *tcl = thread_core_local; + + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { + tcl[n].tmp_stack_va_end = GET_STACK_BOTTOM(stack_tmp, n) - + STACK_TMP_OFFS; + tcl[n].abt_stack_va_end = GET_STACK_BOTTOM(stack_abt, n); + } +} + +#if defined(CFG_CORE_PAUTH) +void thread_init_thread_pauth_keys(void) +{ + size_t n = 0; + + for (n = 0; n < CFG_NUM_THREADS; n++) + if (crypto_rng_read(&threads[n].keys, sizeof(threads[n].keys))) + panic("Failed to init thread pauth keys"); +} + +void thread_init_core_local_pauth_keys(void) +{ + struct thread_core_local *tcl = thread_core_local; + size_t n = 0; + + for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) + if (crypto_rng_read(&tcl[n].keys, sizeof(tcl[n].keys))) + panic("Failed to init core local pauth keys"); +} +#endif + +struct thread_specific_data * __noprof thread_get_tsd(void) +{ + return &threads[thread_get_id()].tsd; +} + +struct thread_ctx_regs * __nostackcheck thread_get_ctx_regs(void) +{ + struct thread_core_local *l = thread_get_core_local(); + + assert(l->curr_thread != THREAD_ID_INVALID); + return &threads[l->curr_thread].regs; +} + +void thread_set_foreign_intr(bool enable) +{ + /* thread_get_core_local() requires foreign interrupts to be disabled */ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + struct thread_core_local *l; + + l = thread_get_core_local(); + + assert(l->curr_thread != THREAD_ID_INVALID); + + if (enable) { + threads[l->curr_thread].flags |= + THREAD_FLAGS_FOREIGN_INTR_ENABLE; + thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); + } else { + /* + * No need to disable foreign interrupts here since they're + * already disabled above. + */ + threads[l->curr_thread].flags &= + ~THREAD_FLAGS_FOREIGN_INTR_ENABLE; + } +} + +void thread_restore_foreign_intr(void) +{ + /* thread_get_core_local() requires foreign interrupts to be disabled */ + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + struct thread_core_local *l; + + l = thread_get_core_local(); + + assert(l->curr_thread != THREAD_ID_INVALID); + + if (threads[l->curr_thread].flags & THREAD_FLAGS_FOREIGN_INTR_ENABLE) + thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); +} + +static struct mobj *alloc_shm(enum thread_shm_type shm_type, size_t size) +{ + switch (shm_type) { + case THREAD_SHM_TYPE_APPLICATION: + return thread_rpc_alloc_payload(size); + case THREAD_SHM_TYPE_KERNEL_PRIVATE: + return thread_rpc_alloc_kernel_payload(size); + case THREAD_SHM_TYPE_GLOBAL: + return thread_rpc_alloc_global_payload(size); + default: + return NULL; + } +} + +static void clear_shm_cache_entry(struct thread_shm_cache_entry *ce) +{ + if (ce->mobj) { + switch (ce->type) { + case THREAD_SHM_TYPE_APPLICATION: + thread_rpc_free_payload(ce->mobj); + break; + case THREAD_SHM_TYPE_KERNEL_PRIVATE: + thread_rpc_free_kernel_payload(ce->mobj); + break; + case THREAD_SHM_TYPE_GLOBAL: + thread_rpc_free_global_payload(ce->mobj); + break; + default: + assert(0); /* "can't happen" */ + break; + } + } + ce->mobj = NULL; + ce->size = 0; +} + +static struct thread_shm_cache_entry * +get_shm_cache_entry(enum thread_shm_cache_user user) +{ + struct thread_shm_cache *cache = &threads[thread_get_id()].shm_cache; + struct thread_shm_cache_entry *ce = NULL; + + SLIST_FOREACH(ce, cache, link) + if (ce->user == user) + return ce; + + ce = calloc(1, sizeof(*ce)); + if (ce) { + ce->user = user; + SLIST_INSERT_HEAD(cache, ce, link); + } + + return ce; +} + +void *thread_rpc_shm_cache_alloc(enum thread_shm_cache_user user, + enum thread_shm_type shm_type, + size_t size, struct mobj **mobj) +{ + struct thread_shm_cache_entry *ce = NULL; + size_t sz = size; + paddr_t p = 0; + void *va = NULL; + + if (!size) + return NULL; + + ce = get_shm_cache_entry(user); + if (!ce) + return NULL; + + /* + * Always allocate in page chunks as normal world allocates payload + * memory as complete pages. + */ + sz = ROUNDUP(size, SMALL_PAGE_SIZE); + + if (ce->type != shm_type || sz > ce->size) { + clear_shm_cache_entry(ce); + + ce->mobj = alloc_shm(shm_type, sz); + if (!ce->mobj) + return NULL; + + if (mobj_get_pa(ce->mobj, 0, 0, &p)) + goto err; + + if (!IS_ALIGNED_WITH_TYPE(p, uint64_t)) + goto err; + + va = mobj_get_va(ce->mobj, 0, sz); + if (!va) + goto err; + + ce->size = sz; + ce->type = shm_type; + } else { + va = mobj_get_va(ce->mobj, 0, sz); + if (!va) + goto err; + } + *mobj = ce->mobj; + + return va; +err: + clear_shm_cache_entry(ce); + return NULL; +} + +void thread_rpc_shm_cache_clear(struct thread_shm_cache *cache) +{ + while (true) { + struct thread_shm_cache_entry *ce = SLIST_FIRST(cache); + + if (!ce) + break; + SLIST_REMOVE_HEAD(cache, link); + clear_shm_cache_entry(ce); + free(ce); + } +} diff --git a/optee_os/core/kernel/tpm.c b/optee_os/core/kernel/tpm.c new file mode 100644 index 0000000..4091a3e --- /dev/null +++ b/optee_os/core/kernel/tpm.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020-2023, ARM Limited. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +static void *tpm_log_addr; +static size_t tpm_log_size; + +/* + * Check whether the node at @offs contains TPM Event Log information or not. + * + * @offs is the offset of the node that describes the device in @fdt. + * @buf will contain the phy address of the TPM Event log. + * @size will contain the size of the mapped area. + * + * Returns the size of the mapped area or < 0 on failure. + */ +#ifdef CFG_DT +static int read_dt_tpm_log_info(void *fdt, int node, paddr_t *buf, + size_t *size) +{ + const uint32_t *property = NULL; + const uint64_t zero_addr = 0; + int len_prop = 0; + paddr_t log_addr = 0; + int err = 0; +#ifdef CFG_MAP_EXT_DT_SECURE + const char *dt_tpm_event_log_addr = "tpm_event_log_addr"; +#else + const char *dt_tpm_event_log_addr = "tpm_event_log_sm_addr"; +#endif + + /* + * Get the TPM Log address. + */ + property = fdt_getprop(fdt, node, dt_tpm_event_log_addr, &len_prop); + + if (!property || len_prop != sizeof(uint32_t) * 2) + return -1; + + log_addr = fdt32_to_cpu(property[1]); + + if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { + err = fdt_setprop(fdt, node, dt_tpm_event_log_addr, &zero_addr, + sizeof(uint32_t) * 2); + if (err < 0) { + EMSG("Error setting property DTB to zero\n"); + return err; + } + } + + /* + * Get the TPM Log size. + */ + property = fdt_getprop(fdt, node, "tpm_event_log_size", &len_prop); + + if (!property || len_prop != sizeof(uint32_t)) + return -1; + + *size = fdt32_to_cpu(property[0]); + *buf = log_addr; + + return *size; +} +#endif + +static void get_tpm_phys_params(void *fdt __maybe_unused, + paddr_t *addr, size_t *size) +{ +#ifdef CFG_DT + int node = 0; + const char *dt_tpm_match_table = { + "arm,tpm_event_log", + }; + + if (!fdt) { + EMSG("TPM: No DTB found"); + return; + } + + node = fdt_node_offset_by_compatible(fdt, -1, dt_tpm_match_table); + + if (node < 0) { + EMSG("TPM: Fail to find TPM node %i", node); + return; + } + + if (read_dt_tpm_log_info((void *)fdt, node, addr, size) < 0) { + EMSG("TPM: Fail to retrieve DTB properties from node %i", + node); + return; + } +#else + *size = CFG_TPM_MAX_LOG_SIZE; + *addr = CFG_TPM_LOG_BASE_ADDR; +#endif /* CFG_DT */ +} + +TEE_Result tpm_get_event_log(void *buf, size_t *size) +{ + const size_t buf_size = *size; + + *size = tpm_log_size; + if (!buf) { + EMSG("TPM: Invalid buffer"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (buf_size < tpm_log_size) { + EMSG("TPM: Not enough space for the log: %zu, %zu", + buf_size, tpm_log_size); + return TEE_ERROR_SHORT_BUFFER; + } + + memcpy(buf, tpm_log_addr, tpm_log_size); + + return TEE_SUCCESS; +} + +TEE_Result tpm_get_event_log_size(size_t *size) +{ + *size = tpm_log_size; + + return TEE_SUCCESS; +} + +void tpm_map_log_area(void *fdt) +{ + paddr_t log_addr = 0; + unsigned int rounded_size = 0; + + get_tpm_phys_params(fdt, &log_addr, &tpm_log_size); + + DMSG("TPM Event log PA: %#" PRIxPA, log_addr); + DMSG("TPM Event log size: %zu Bytes", tpm_log_size); + + rounded_size = ROUNDUP(tpm_log_size, SMALL_PAGE_SIZE); + + tpm_log_addr = core_mmu_add_mapping(MEM_AREA_RAM_SEC, log_addr, + rounded_size); + if (!tpm_log_addr) { + EMSG("TPM: Failed to map TPM log memory"); + return; + } +} diff --git a/optee_os/core/kernel/trace_ext.c b/optee_os/core/kernel/trace_ext.c new file mode 100644 index 0000000..6ed3bf9 --- /dev/null +++ b/optee_os/core/kernel/trace_ext.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include + +const char trace_ext_prefix[] = "TC"; +int trace_level __nex_data = TRACE_LEVEL; +static unsigned int puts_lock __nex_bss = SPINLOCK_UNLOCK; + +void __weak plat_trace_ext_puts(const char *str __unused) +{ +} + +void trace_ext_puts(const char *str) +{ + uint32_t itr_status = thread_mask_exceptions(THREAD_EXCP_ALL); + bool mmu_enabled = cpu_mmu_enabled(); + bool was_contended = false; + const char *p; + + if (mmu_enabled && !cpu_spin_trylock(&puts_lock)) { + was_contended = true; + cpu_spin_lock_no_dldetect(&puts_lock); + } + + plat_trace_ext_puts(str); + + console_flush(); + + if (was_contended) + console_putc('*'); + + for (p = str; *p; p++) + console_putc(*p); + + console_flush(); + + if (mmu_enabled) + cpu_spin_unlock(&puts_lock); + + thread_unmask_exceptions(itr_status); +} + +int trace_ext_get_thread_id(void) +{ + return thread_get_id_may_fail(); +} + +int trace_ext_get_core_id(void) +{ + /* If foreign interrupts aren't masked we report invalid core ID */ + if (thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR) + return get_core_pos(); + else + return -1; +} diff --git a/optee_os/core/kernel/ts_manager.c b/optee_os/core/kernel/ts_manager.c new file mode 100644 index 0000000..b279463 --- /dev/null +++ b/optee_os/core/kernel/ts_manager.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static void update_current_ctx(struct thread_specific_data *tsd) +{ + struct ts_ctx *ctx = NULL; + struct ts_session *s = TAILQ_FIRST(&tsd->sess_stack); + + if (s) { + if (is_pseudo_ta_ctx(s->ctx)) + s = TAILQ_NEXT(s, link_tsd); + + if (s) + ctx = s->ctx; + } + + if (tsd->ctx != ctx) + vm_set_ctx(ctx); + /* + * If current context is of user mode, then it has to be active too. + */ + if (is_user_mode_ctx(ctx) != core_mmu_user_mapping_is_active()) + panic("unexpected active mapping"); +} + +void ts_push_current_session(struct ts_session *s) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + + TAILQ_INSERT_HEAD(&tsd->sess_stack, s, link_tsd); + update_current_ctx(tsd); +} + +struct ts_session *ts_pop_current_session(void) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + struct ts_session *s = TAILQ_FIRST(&tsd->sess_stack); + + if (s) { + TAILQ_REMOVE(&tsd->sess_stack, s, link_tsd); + update_current_ctx(tsd); + } + return s; +} + +struct ts_session *ts_get_calling_session(void) +{ + return TAILQ_NEXT(ts_get_current_session(), link_tsd); +} + +struct ts_session *ts_get_current_session_may_fail(void) +{ + return TAILQ_FIRST(&thread_get_tsd()->sess_stack); +} + +struct ts_session *ts_get_current_session(void) +{ + struct ts_session *s = ts_get_current_session_may_fail(); + + if (!s) + panic(); + return s; +} diff --git a/optee_os/core/kernel/ubsan.c b/optee_os/core/kernel/ubsan.c new file mode 100644 index 0000000..04b0163 --- /dev/null +++ b/optee_os/core/kernel/ubsan.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include + +struct source_location { + const char *file_name; + uint32_t line; + uint32_t column; +}; + +struct type_descriptor { + uint16_t type_kind; + uint16_t type_info; + char type_name[1]; +}; + +struct type_mismatch_data { + struct source_location loc; + struct type_descriptor *type; + unsigned long alignment; + unsigned char type_check_kind; +}; + +struct overflow_data { + struct source_location loc; + struct type_descriptor *type; +}; + +struct shift_out_of_bounds_data { + struct source_location loc; + struct type_descriptor *lhs_type; + struct type_descriptor *rhs_type; +}; + +struct out_of_bounds_data { + struct source_location loc; + struct type_descriptor *array_type; + struct type_descriptor *index_type; +}; + +struct unreachable_data { + struct source_location loc; +}; + +struct vla_bound_data { + struct source_location loc; + struct type_descriptor *type; +}; + +struct invalid_value_data { + struct source_location loc; + struct type_descriptor *type; +}; + +struct nonnull_arg_data { + struct source_location loc; +}; + +/* + * When compiling with -fsanitize=undefined the compiler expects functions + * with the following signatures. The functions are never called directly, + * only when undefined behavior is detected in instrumented code. + */ +void __ubsan_handle_type_mismatch(struct type_mismatch_data *data, + unsigned long ptr); +void __ubsan_handle_add_overflow(struct overflow_data *data, + unsigned long lhs, unsigned long rhs); +void __ubsan_handle_sub_overflow(struct overflow_data *data, + unsigned long lhs, unsigned long rhs); +void __ubsan_handle_mul_overflow(struct overflow_data *data, + unsigned long lhs, unsigned long rhs); +void __ubsan_handle_negate_overflow(struct overflow_data *data, + unsigned long old_val); +void __ubsan_handle_divrem_overflow(struct overflow_data *data, + unsigned long lhs, unsigned long rhs); +void __ubsan_handle_shift_out_of_bounds(struct shift_out_of_bounds_data *data, + unsigned long lhs, unsigned long rhs); +void __ubsan_handle_out_of_bounds(struct out_of_bounds_data *data, + unsigned long idx); +void __ubsan_handle_unreachable(struct unreachable_data *data); +void __ubsan_handle_missing_return(struct unreachable_data *data); +void __ubsan_handle_vla_bound_not_positive(struct vla_bound_data *data, + unsigned long bound); +void __ubsan_handle_load_invalid_value(struct invalid_value_data *data, + unsigned long val); +void __ubsan_handle_nonnull_arg(struct nonnull_arg_data *data +#if __GCC_VERSION < 60000 + , size_t arg_no +#endif + ); + +static void print_loc(const char *func, struct source_location *loc) +{ + const char *f = func; + const char func_prefix[] = "__ubsan_handle"; + + if (!memcmp(f, func_prefix, sizeof(func_prefix) - 1)) + f += sizeof(func_prefix); + + EMSG_RAW("Undefined behavior %s at %s:%" PRIu32 " col %" PRIu32, + f, loc->file_name, loc->line, loc->column); +} + + +static volatile bool ubsan_panic = true; + +void __ubsan_handle_type_mismatch(struct type_mismatch_data *data, + unsigned long ptr __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_add_overflow(struct overflow_data *data, + unsigned long lhs __unused, + unsigned long rhs __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_sub_overflow(struct overflow_data *data, + unsigned long lhs __unused, + unsigned long rhs __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_mul_overflow(struct overflow_data *data, + unsigned long lhs __unused, + unsigned long rhs __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_negate_overflow(struct overflow_data *data, + unsigned long old_val __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_divrem_overflow(struct overflow_data *data, + unsigned long lhs __unused, + unsigned long rhs __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_shift_out_of_bounds(struct shift_out_of_bounds_data *data, + unsigned long lhs __unused, + unsigned long rhs __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_out_of_bounds(struct out_of_bounds_data *data, + unsigned long idx __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_unreachable(struct unreachable_data *data) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __noreturn __ubsan_handle_missing_return(struct unreachable_data *data) +{ + print_loc(__func__, &data->loc); + panic(); +} + +void __ubsan_handle_vla_bound_not_positive(struct vla_bound_data *data, + unsigned long bound __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_load_invalid_value(struct invalid_value_data *data, + unsigned long val __unused) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} + +void __ubsan_handle_nonnull_arg(struct nonnull_arg_data *data +#if __GCC_VERSION < 60000 + , size_t arg_no __unused +#endif + ) +{ + print_loc(__func__, &data->loc); + if (ubsan_panic) + panic(); +} diff --git a/optee_os/core/kernel/user_access.c b/optee_os/core/kernel/user_access.c new file mode 100644 index 0000000..22b48fb --- /dev/null +++ b/optee_os/core/kernel/user_access.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2015-2020, 2022 Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BB_ALIGNMENT (sizeof(long) * 2) + +static struct user_mode_ctx *get_current_uctx(void) +{ + struct ts_session *s = ts_get_current_session(); + + if (!is_user_mode_ctx(s->ctx)) { + /* + * We may be called within a PTA session, which doesn't + * have a user_mode_ctx. Here, try to retrieve the + * user_mode_ctx associated with the calling session. + */ + s = TAILQ_NEXT(s, link_tsd); + if (!s || !is_user_mode_ctx(s->ctx)) + return NULL; + } + + return to_user_mode_ctx(s->ctx); +} + +TEE_Result check_user_access(uint32_t flags, const void *uaddr, size_t len) +{ + struct user_mode_ctx *uctx = get_current_uctx(); + + if (!uctx) + return TEE_ERROR_GENERIC; + + return vm_check_access_rights(uctx, flags, (vaddr_t)uaddr, len); +} + +TEE_Result copy_from_user(void *kaddr, const void *uaddr, size_t len) +{ + uint32_t flags = TEE_MEMORY_ACCESS_READ | TEE_MEMORY_ACCESS_ANY_OWNER; + TEE_Result res = TEE_SUCCESS; + + uaddr = memtag_strip_tag_const(uaddr); + res = check_user_access(flags, uaddr, len); + if (!res) { + enter_user_access(); + memcpy(kaddr, uaddr, len); + exit_user_access(); + } + + return res; +} + +TEE_Result copy_to_user(void *uaddr, const void *kaddr, size_t len) +{ + uint32_t flags = TEE_MEMORY_ACCESS_WRITE | TEE_MEMORY_ACCESS_ANY_OWNER; + TEE_Result res = TEE_SUCCESS; + + uaddr = memtag_strip_tag(uaddr); + res = check_user_access(flags, uaddr, len); + if (!res) { + enter_user_access(); + memcpy(uaddr, kaddr, len); + exit_user_access(); + } + + return res; +} + +TEE_Result copy_from_user_private(void *kaddr, const void *uaddr, size_t len) +{ + uint32_t flags = TEE_MEMORY_ACCESS_READ; + TEE_Result res = TEE_SUCCESS; + + uaddr = memtag_strip_tag_const(uaddr); + res = check_user_access(flags, uaddr, len); + if (!res) { + enter_user_access(); + memcpy(kaddr, uaddr, len); + exit_user_access(); + } + + return res; +} + +TEE_Result copy_to_user_private(void *uaddr, const void *kaddr, size_t len) +{ + uint32_t flags = TEE_MEMORY_ACCESS_WRITE; + TEE_Result res = TEE_SUCCESS; + + uaddr = memtag_strip_tag(uaddr); + res = check_user_access(flags, uaddr, len); + if (!res) { + enter_user_access(); + memcpy(uaddr, kaddr, len); + exit_user_access(); + } + + return res; +} + +static void *maybe_tag_bb(void *buf, size_t sz) +{ + static_assert(MEMTAG_GRANULE_SIZE <= BB_ALIGNMENT); + + if (!MEMTAG_IS_ENABLED) + return buf; + + assert(!((vaddr_t)buf % MEMTAG_GRANULE_SIZE)); + return memtag_set_random_tags(buf, ROUNDUP(sz, MEMTAG_GRANULE_SIZE)); +} + +static void maybe_untag_bb(void *buf, size_t sz) +{ + if (MEMTAG_IS_ENABLED) { + assert(!((vaddr_t)buf % MEMTAG_GRANULE_SIZE)); + memtag_set_tags(buf, ROUNDUP(sz, MEMTAG_GRANULE_SIZE), 0); + } +} + +void *bb_alloc(size_t len) +{ + struct user_mode_ctx *uctx = get_current_uctx(); + size_t offs = 0; + void *bb = NULL; + + if (uctx && !ADD_OVERFLOW(uctx->bbuf_offs, len, &offs) && + offs <= uctx->bbuf_size) { + bb = maybe_tag_bb(uctx->bbuf + uctx->bbuf_offs, len); + uctx->bbuf_offs = ROUNDUP(offs, BB_ALIGNMENT); + } + return bb; +} + +static void bb_free_helper(struct user_mode_ctx *uctx, vaddr_t bb, size_t len) +{ + vaddr_t bbuf = (vaddr_t)uctx->bbuf; + + if (bb >= bbuf && IS_ALIGNED(bb, BB_ALIGNMENT)) { + size_t prev_offs = bb - bbuf; + + /* + * Even if we can't update offset we can still invalidate + * the memory allocation. + */ + maybe_untag_bb((void *)bb, len); + + if (prev_offs + ROUNDUP(len, BB_ALIGNMENT) == uctx->bbuf_offs) + uctx->bbuf_offs = prev_offs; + } +} + +void bb_free(void *bb, size_t len) +{ + struct user_mode_ctx *uctx = get_current_uctx(); + + if (uctx) + bb_free_helper(uctx, memtag_strip_tag_vaddr(bb), len); +} + +void bb_free_wipe(void *bb, size_t len) +{ + if (bb) + memset(bb, 0, len); + bb_free(bb, len); +} + +void bb_reset(void) +{ + struct user_mode_ctx *uctx = get_current_uctx(); + + if (uctx) { + /* + * Only the part up to the offset have been allocated, so + * no need to clear tags beyond that. + */ + maybe_untag_bb(uctx->bbuf, uctx->bbuf_offs); + + uctx->bbuf_offs = 0; + } +} + +TEE_Result clear_user(void *uaddr, size_t n) +{ + uint32_t flags = TEE_MEMORY_ACCESS_WRITE | TEE_MEMORY_ACCESS_ANY_OWNER; + TEE_Result res = TEE_SUCCESS; + + uaddr = memtag_strip_tag(uaddr); + res = check_user_access(flags, uaddr, n); + if (res) + return res; + + enter_user_access(); + memset(uaddr, 0, n); + exit_user_access(); + + return TEE_SUCCESS; +} + +size_t strnlen_user(const void *uaddr, size_t len) +{ + uint32_t flags = TEE_MEMORY_ACCESS_READ | TEE_MEMORY_ACCESS_ANY_OWNER; + TEE_Result res = TEE_SUCCESS; + size_t n = 0; + + if (!len) + return 0; + + uaddr = memtag_strip_tag_const(uaddr); + res = check_user_access(flags, uaddr, len); + if (!res) { + enter_user_access(); + n = strnlen(uaddr, len); + exit_user_access(); + } + + return n; +} + +static TEE_Result __bb_memdup_user(TEE_Result (*copy_func)(void *uaddr, + const void *kaddr, + size_t len), + const void *src, size_t len, void **p) +{ + TEE_Result res = TEE_SUCCESS; + void *buf = NULL; + + buf = bb_alloc(len); + if (!buf) + return TEE_ERROR_OUT_OF_MEMORY; + + if (len) + res = copy_func(buf, src, len); + + if (res) + bb_free(buf, len); + else + *p = buf; + + return res; +} + +TEE_Result bb_memdup_user(const void *src, size_t len, void **p) +{ + return __bb_memdup_user(copy_from_user, src, len, p); +} + +TEE_Result bb_memdup_user_private(const void *src, size_t len, void **p) +{ + return __bb_memdup_user(copy_from_user_private, src, len, p); +} + +TEE_Result bb_strndup_user(const char *src, size_t maxlen, char **dst, + size_t *dstlen) +{ + uint32_t flags = TEE_MEMORY_ACCESS_READ | TEE_MEMORY_ACCESS_ANY_OWNER; + TEE_Result res = TEE_SUCCESS; + size_t l = 0; + char *d = NULL; + + src = memtag_strip_tag_const(src); + if (maxlen) { + res = check_user_access(flags, src, maxlen); + if (res) + return res; + + enter_user_access(); + l = strnlen(src, maxlen); + exit_user_access(); + } + + d = bb_alloc(l + 1); + if (!d) + return TEE_ERROR_OUT_OF_MEMORY; + + if (l) { + enter_user_access(); + memcpy(d, src, l); + exit_user_access(); + } + + d[l] = 0; + + *dst = d; + *dstlen = l; + return TEE_SUCCESS; +} + +TEE_Result copy_kaddr_to_uref(uint32_t *uref, void *kaddr) +{ + uint32_t ref = kaddr_to_uref(kaddr); + + return copy_to_user_private(uref, &ref, sizeof(ref)); +} + +uint32_t kaddr_to_uref(void *kaddr) +{ + if (MEMTAG_IS_ENABLED) { + unsigned int uref_tag_shift = 32 - MEMTAG_TAG_WIDTH; + vaddr_t uref = memtag_strip_tag_vaddr(kaddr); + + uref -= VCORE_START_VA; + assert(uref < (UINT32_MAX >> MEMTAG_TAG_WIDTH)); + uref |= memtag_get_tag(kaddr) << uref_tag_shift; + return uref; + } + + assert(((vaddr_t)kaddr - VCORE_START_VA) < UINT32_MAX); + return (vaddr_t)kaddr - VCORE_START_VA; +} + +vaddr_t uref_to_vaddr(uint32_t uref) +{ + if (MEMTAG_IS_ENABLED) { + vaddr_t u = uref & (UINT32_MAX >> MEMTAG_TAG_WIDTH); + unsigned int uref_tag_shift = 32 - MEMTAG_TAG_WIDTH; + uint8_t tag = uref >> uref_tag_shift; + + return memtag_insert_tag_vaddr(VCORE_START_VA + u, tag); + } + + return VCORE_START_VA + uref; +} diff --git a/optee_os/core/kernel/user_mode_ctx.c b/optee_os/core/kernel/user_mode_ctx.c new file mode 100644 index 0000000..12906f3 --- /dev/null +++ b/optee_os/core/kernel/user_mode_ctx.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include + +void user_mode_ctx_print_mappings(struct user_mode_ctx *uctx) +{ + struct vm_region *r = NULL; + char flags[7] = { '\0', }; + size_t n = 0; + + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) { + paddr_t pa = 0; + + if (r->mobj) + mobj_get_pa(r->mobj, r->offset, 0, &pa); + + mattr_perm_to_str(flags, sizeof(flags), r->attr); + EMSG_RAW(" region %2zu: va 0x%0*" PRIxVA " pa 0x%0*" PRIxPA + " size 0x%06zx flags %s", + n, PRIxVA_WIDTH, r->va, PRIxPA_WIDTH, pa, r->size, + flags); + n++; + } +} diff --git a/optee_os/core/kernel/user_ta.c b/optee_os/core/kernel/user_ta.c new file mode 100644 index 0000000..315f5f1 --- /dev/null +++ b/optee_os/core/kernel/user_ta.c @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2015-2022 Linaro Limited + * Copyright (c) 2020, Arm Limited. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static TEE_Result init_utee_param(struct utee_params *up, + const struct tee_ta_param *p, + void *va[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + size_t n = 0; + struct utee_params *up_bbuf = NULL; + + up_bbuf = bb_alloc(sizeof(struct utee_params)); + if (!up_bbuf) + return TEE_ERROR_OUT_OF_MEMORY; + + up_bbuf->types = p->types; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uintptr_t a; + uintptr_t b; + + switch (TEE_PARAM_TYPE_GET(p->types, n)) { + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + a = (uintptr_t)va[n]; + b = p->u[n].mem.size; + break; + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + a = p->u[n].val.a; + b = p->u[n].val.b; + break; + default: + a = 0; + b = 0; + break; + } + /* See comment for struct utee_params in utee_types.h */ + up_bbuf->vals[n * 2] = a; + up_bbuf->vals[n * 2 + 1] = b; + } + + res = copy_to_user(up, up_bbuf, sizeof(struct utee_params)); + + bb_free(up_bbuf, sizeof(struct utee_params)); + + return res; +} + +static void update_from_utee_param(struct tee_ta_param *p, + const struct utee_params *up) +{ + TEE_Result res = TEE_SUCCESS; + size_t n = 0; + struct utee_params *up_bbuf = NULL; + + res = BB_MEMDUP_USER(up, sizeof(*up), &up_bbuf); + if (res) + return; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(p->types, n)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + /* See comment for struct utee_params in utee_types.h */ + p->u[n].mem.size = up_bbuf->vals[n * 2 + 1]; + break; + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + /* See comment for struct utee_params in utee_types.h */ + p->u[n].val.a = up_bbuf->vals[n * 2]; + p->u[n].val.b = up_bbuf->vals[n * 2 + 1]; + break; + default: + break; + } + } + + bb_free(up_bbuf, sizeof(*up)); +} + +static bool inc_recursion(void) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + + if (tsd->syscall_recursion >= CFG_CORE_MAX_SYSCALL_RECURSION) { + DMSG("Maximum allowed recursion depth reached (%u)", + CFG_CORE_MAX_SYSCALL_RECURSION); + return false; + } + + tsd->syscall_recursion++; + return true; +} + +static void dec_recursion(void) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + + assert(tsd->syscall_recursion); + tsd->syscall_recursion--; +} + +static TEE_Result user_ta_enter(struct ts_session *session, + enum utee_entry_func func, uint32_t cmd) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_params *usr_params = NULL; + uaddr_t usr_stack = 0; + struct user_ta_ctx *utc = to_user_ta_ctx(session->ctx); + struct tee_ta_session *ta_sess = to_ta_session(session); + struct ts_session *ts_sess __maybe_unused = NULL; + void *param_va[TEE_NUM_PARAMS] = { NULL }; + + if (!inc_recursion()) { + /* Using this error code since we've run out of resources. */ + res = TEE_ERROR_OUT_OF_MEMORY; + goto out_clr_cancel; + } + if (ta_sess->param) { + /* Map user space memory */ + res = vm_map_param(&utc->uctx, ta_sess->param, param_va); + if (res != TEE_SUCCESS) + goto out; + } + + /* Switch to user ctx */ + ts_push_current_session(session); + + /* Make room for usr_params at top of stack */ + usr_stack = utc->uctx.stack_ptr; + usr_stack -= ROUNDUP(sizeof(struct utee_params), STACK_ALIGNMENT); + usr_params = (struct utee_params *)usr_stack; + if (ta_sess->param) + res = init_utee_param(usr_params, ta_sess->param, param_va); + else + res = clear_user(usr_params, sizeof(*usr_params)); + + if (res) + goto out_pop_session; + + res = thread_enter_user_mode(func, kaddr_to_uref(session), + (vaddr_t)usr_params, cmd, usr_stack, + utc->uctx.entry_func, utc->uctx.is_32bit, + &utc->ta_ctx.panicked, + &utc->ta_ctx.panic_code); + + thread_user_clear_vfp(&utc->uctx); + + if (utc->ta_ctx.panicked) { + abort_print_current_ts(); + DMSG("tee_user_ta_enter: TA panicked with code 0x%x", + utc->ta_ctx.panic_code); + res = TEE_ERROR_TARGET_DEAD; + } else { + /* + * According to GP spec the origin should allways be set to + * the TA after TA execution + */ + ta_sess->err_origin = TEE_ORIGIN_TRUSTED_APP; + } + + if (ta_sess->param) { + /* Copy out value results */ + update_from_utee_param(ta_sess->param, usr_params); + } + +out_pop_session: + if (ta_sess->param) { + /* + * Clear out the parameter mappings added with + * vm_clean_param() above. + */ + vm_clean_param(&utc->uctx); + } + ts_sess = ts_pop_current_session(); + assert(ts_sess == session); + +out: + dec_recursion(); +out_clr_cancel: + /* + * Clear the cancel state now that the user TA has returned. The next + * time the TA will be invoked will be with a new operation and should + * not have an old cancellation pending. + */ + ta_sess->cancel = false; + + return res; +} + +static TEE_Result user_ta_enter_open_session(struct ts_session *s) +{ + return user_ta_enter(s, UTEE_ENTRY_FUNC_OPEN_SESSION, 0); +} + +static TEE_Result user_ta_enter_invoke_cmd(struct ts_session *s, uint32_t cmd) +{ + return user_ta_enter(s, UTEE_ENTRY_FUNC_INVOKE_COMMAND, cmd); +} + +static void user_ta_enter_close_session(struct ts_session *s) +{ + /* Only if the TA was fully initialized by ldelf */ + if (!to_user_ta_ctx(s->ctx)->uctx.is_initializing) + user_ta_enter(s, UTEE_ENTRY_FUNC_CLOSE_SESSION, 0); +} + +#if defined(CFG_TA_STATS) +static TEE_Result user_ta_enter_dump_memstats(struct ts_session *s) +{ + return user_ta_enter(s, UTEE_ENTRY_FUNC_DUMP_MEMSTATS, 0); +} +#endif + +static void dump_state_no_ldelf_dbg(struct user_ta_ctx *utc) +{ + user_mode_ctx_print_mappings(&utc->uctx); +} + +static void user_ta_dump_state(struct ts_ctx *ctx) +{ + struct user_ta_ctx *utc = to_user_ta_ctx(ctx); + + if (utc->uctx.dump_entry_func) { + TEE_Result res = ldelf_dump_state(&utc->uctx); + + if (!res || res == TEE_ERROR_TARGET_DEAD) + return; + /* + * Fall back to dump_state_no_ldelf_dbg() if + * ldelf_dump_state() fails for some reason. + * + * If ldelf_dump_state() failed with panic + * we are done since abort_print_current_ts() will be + * called which will dump the memory map. + */ + } + + dump_state_no_ldelf_dbg(utc); +} + +#ifdef CFG_FTRACE_SUPPORT +static void user_ta_dump_ftrace(struct ts_ctx *ctx) +{ + uint32_t prot = TEE_MATTR_URW; + struct user_ta_ctx *utc = to_user_ta_ctx(ctx); + struct thread_param params[3] = { }; + TEE_Result res = TEE_SUCCESS; + struct mobj *mobj = NULL; + uint8_t *ubuf = NULL; + void *buf = NULL; + size_t pl_sz = 0; + size_t blen = 0, ld_addr_len = 0; + vaddr_t va = 0; + + res = ldelf_dump_ftrace(&utc->uctx, NULL, &blen); + if (res != TEE_ERROR_SHORT_BUFFER) + return; + +#define LOAD_ADDR_DUMP_SIZE 64 + pl_sz = ROUNDUP(blen + sizeof(TEE_UUID) + LOAD_ADDR_DUMP_SIZE, + SMALL_PAGE_SIZE); + + mobj = thread_rpc_alloc_payload(pl_sz); + if (!mobj) { + EMSG("Ftrace thread_rpc_alloc_payload failed"); + return; + } + + buf = mobj_get_va(mobj, 0, pl_sz); + if (!buf) + goto out_free_pl; + + res = vm_map(&utc->uctx, &va, mobj->size, prot, VM_FLAG_EPHEMERAL, + mobj, 0); + if (res) + goto out_free_pl; + + ubuf = (uint8_t *)va + mobj_get_phys_offs(mobj, mobj->phys_granule); + memcpy(ubuf, &ctx->uuid, sizeof(TEE_UUID)); + ubuf += sizeof(TEE_UUID); + + ld_addr_len = snprintk((char *)ubuf, LOAD_ADDR_DUMP_SIZE, + "TEE load address @ %#"PRIxVA"\n", + VCORE_START_VA); + ubuf += ld_addr_len; + + res = ldelf_dump_ftrace(&utc->uctx, ubuf, &blen); + if (res) { + EMSG("Ftrace dump failed: %#"PRIx32, res); + goto out_unmap_pl; + } + + params[0] = THREAD_PARAM_VALUE(INOUT, 0, 0, 0); + params[1] = THREAD_PARAM_MEMREF(IN, mobj, 0, sizeof(TEE_UUID)); + params[2] = THREAD_PARAM_MEMREF(IN, mobj, sizeof(TEE_UUID), + blen + ld_addr_len); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_FTRACE, 3, params); + if (res) + EMSG("Ftrace thread_rpc_cmd res: %#"PRIx32, res); + +out_unmap_pl: + res = vm_unmap(&utc->uctx, va, mobj->size); + assert(!res); +out_free_pl: + thread_rpc_free_payload(mobj); +} +#endif /*CFG_FTRACE_SUPPORT*/ + +#ifdef CFG_TA_GPROF_SUPPORT +static void user_ta_gprof_set_status(enum ts_gprof_status status) +{ + if (status == TS_GPROF_SUSPEND) + tee_ta_update_session_utime_suspend(); + else + tee_ta_update_session_utime_resume(); +} +#endif /*CFG_TA_GPROF_SUPPORT*/ + + +static void release_utc_state(struct user_ta_ctx *utc) +{ + /* + * Close sessions opened by this TA + * Note that tee_ta_close_session() removes the item + * from the utc->open_sessions list. + */ + while (!TAILQ_EMPTY(&utc->open_sessions)) { + tee_ta_close_session(TAILQ_FIRST(&utc->open_sessions), + &utc->open_sessions, KERN_IDENTITY); + } + + vm_info_final(&utc->uctx); + + /* Free cryp states created by this TA */ + tee_svc_cryp_free_states(utc); + /* Close cryp objects opened by this TA */ + tee_obj_close_all(utc); + /* Free emums created by this TA */ + tee_svc_storage_close_all_enum(utc); +} + +static void free_utc(struct user_ta_ctx *utc) +{ + release_utc_state(utc); + free(utc); +} + +static void user_ta_release_state(struct ts_ctx *ctx) +{ + release_utc_state(to_user_ta_ctx(ctx)); +} + +static void user_ta_ctx_destroy(struct ts_ctx *ctx) +{ + free_utc(to_user_ta_ctx(ctx)); +} + +static uint32_t user_ta_get_instance_id(struct ts_ctx *ctx) +{ + return to_user_ta_ctx(ctx)->uctx.vm_info.asid; +} + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct ts_ops user_ta_ops __weak __relrodata_unpaged("user_ta_ops") = { + .enter_open_session = user_ta_enter_open_session, + .enter_invoke_cmd = user_ta_enter_invoke_cmd, + .enter_close_session = user_ta_enter_close_session, +#if defined(CFG_TA_STATS) + .dump_mem_stats = user_ta_enter_dump_memstats, +#endif + .dump_state = user_ta_dump_state, +#ifdef CFG_FTRACE_SUPPORT + .dump_ftrace = user_ta_dump_ftrace, +#endif + .release_state = user_ta_release_state, + .destroy = user_ta_ctx_destroy, + .get_instance_id = user_ta_get_instance_id, + .handle_scall = scall_handle_user_ta, +#ifdef CFG_TA_GPROF_SUPPORT + .gprof_set_status = user_ta_gprof_set_status, +#endif +}; + +static void set_ta_ctx_ops(struct tee_ta_ctx *ctx) +{ + ctx->ts_ctx.ops = &user_ta_ops; +} + +bool __noprof is_user_ta_ctx(struct ts_ctx *ctx) +{ + return ctx && ctx->ops == &user_ta_ops; +} + +static TEE_Result check_ta_store(void) +{ + const struct ts_store_ops *op = NULL; + + SCATTERED_ARRAY_FOREACH(op, ta_stores, struct ts_store_ops) + DMSG("TA store: \"%s\"", op->description); + + return TEE_SUCCESS; +} +service_init(check_ta_store); + +TEE_Result tee_ta_init_user_ta_session(const TEE_UUID *uuid, + struct tee_ta_session *s) +{ + TEE_Result res = TEE_SUCCESS; + struct user_ta_ctx *utc = NULL; + + utc = calloc(1, sizeof(struct user_ta_ctx)); + if (!utc) + return TEE_ERROR_OUT_OF_MEMORY; + + TAILQ_INIT(&utc->open_sessions); + TAILQ_INIT(&utc->cryp_states); + TAILQ_INIT(&utc->objects); + TAILQ_INIT(&utc->storage_enums); + condvar_init(&utc->ta_ctx.busy_cv); + utc->ta_ctx.ref_count = 1; + + /* + * Set context TA operation structure. It is required by generic + * implementation to identify userland TA versus pseudo TA contexts. + */ + set_ta_ctx_ops(&utc->ta_ctx); + + utc->ta_ctx.ts_ctx.uuid = *uuid; + res = vm_info_init(&utc->uctx, &utc->ta_ctx.ts_ctx); + if (res) + goto out; + utc->uctx.is_initializing = true; + +#ifdef CFG_TA_PAUTH + crypto_rng_read(&utc->uctx.keys, sizeof(utc->uctx.keys)); +#endif + + mutex_lock(&tee_ta_mutex); + s->ts_sess.ctx = &utc->ta_ctx.ts_ctx; + s->ts_sess.handle_scall = s->ts_sess.ctx->ops->handle_scall; + /* + * Another thread trying to load this same TA may need to wait + * until this context is fully initialized. This is needed to + * handle single instance TAs. + */ + TAILQ_INSERT_TAIL(&tee_ctxes, &utc->ta_ctx, link); + mutex_unlock(&tee_ta_mutex); + + /* + * We must not hold tee_ta_mutex while allocating page tables as + * that may otherwise lead to a deadlock. + */ + ts_push_current_session(&s->ts_sess); + + res = ldelf_load_ldelf(&utc->uctx); + if (!res) + res = ldelf_init_with_ldelf(&s->ts_sess, &utc->uctx); + + ts_pop_current_session(); + + mutex_lock(&tee_ta_mutex); + + if (!res) { + utc->uctx.is_initializing = false; + } else { + s->ts_sess.ctx = NULL; + TAILQ_REMOVE(&tee_ctxes, &utc->ta_ctx, link); + } + + /* The state has changed for the context, notify eventual waiters. */ + condvar_broadcast(&tee_ta_init_cv); + + mutex_unlock(&tee_ta_mutex); + +out: + if (res) { + condvar_destroy(&utc->ta_ctx.busy_cv); + free_utc(utc); + } + + return res; +} diff --git a/optee_os/core/kernel/wait_queue.c b/optee_os/core/kernel/wait_queue.c new file mode 100644 index 0000000..1326ef8 --- /dev/null +++ b/optee_os/core/kernel/wait_queue.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2021, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned wq_spin_lock; + + +void wq_init(struct wait_queue *wq) +{ + *wq = (struct wait_queue)WAIT_QUEUE_INITIALIZER; +} + +static void do_notif(TEE_Result (*fn)(uint32_t), int id, + const char *cmd_str __maybe_unused, + const void *sync_obj __maybe_unused, + const char *fname, int lineno __maybe_unused) +{ + TEE_Result res = TEE_SUCCESS; + + if (fname) + DMSG("%s thread %d %p %s:%d", cmd_str, id, + sync_obj, fname, lineno); + else + DMSG("%s thread %d %p", cmd_str, id, sync_obj); + + res = fn(id + NOTIF_SYNC_VALUE_BASE); + if (res) + DMSG("%s thread %d res %#"PRIx32, cmd_str, id, res); +} + +static void slist_add_tail(struct wait_queue *wq, struct wait_queue_elem *wqe) +{ + struct wait_queue_elem *wqe_iter; + + /* Add elem to end of wait queue */ + wqe_iter = SLIST_FIRST(wq); + if (wqe_iter) { + while (SLIST_NEXT(wqe_iter, link)) + wqe_iter = SLIST_NEXT(wqe_iter, link); + SLIST_INSERT_AFTER(wqe_iter, wqe, link); + } else + SLIST_INSERT_HEAD(wq, wqe, link); +} + +void wq_wait_init_condvar(struct wait_queue *wq, struct wait_queue_elem *wqe, + struct condvar *cv, bool wait_read) +{ + uint32_t old_itr_status; + + wqe->handle = thread_get_id(); + wqe->done = false; + wqe->wait_read = wait_read; + wqe->cv = cv; + + old_itr_status = cpu_spin_lock_xsave(&wq_spin_lock); + + slist_add_tail(wq, wqe); + + cpu_spin_unlock_xrestore(&wq_spin_lock, old_itr_status); +} + +void wq_wait_final(struct wait_queue *wq, struct wait_queue_elem *wqe, + const void *sync_obj, const char *fname, int lineno) +{ + uint32_t old_itr_status; + unsigned done; + + do { + do_notif(notif_wait, wqe->handle, + "sleep", sync_obj, fname, lineno); + + old_itr_status = cpu_spin_lock_xsave(&wq_spin_lock); + + done = wqe->done; + if (done) + SLIST_REMOVE(wq, wqe, wait_queue_elem, link); + + cpu_spin_unlock_xrestore(&wq_spin_lock, old_itr_status); + } while (!done); +} + +void wq_wake_next(struct wait_queue *wq, const void *sync_obj, + const char *fname, int lineno) +{ + uint32_t old_itr_status; + struct wait_queue_elem *wqe; + int handle = -1; + bool do_wakeup = false; + bool wake_type_assigned = false; + bool wake_read = false; /* avoid gcc warning */ + + /* + * If next type is wait_read wakeup all wqe with wait_read true. + * If next type isn't wait_read wakeup only the first wqe which isn't + * done. + */ + + while (true) { + old_itr_status = cpu_spin_lock_xsave(&wq_spin_lock); + + SLIST_FOREACH(wqe, wq, link) { + if (wqe->cv) + continue; + if (wqe->done) + continue; + if (!wake_type_assigned) { + wake_read = wqe->wait_read; + wake_type_assigned = true; + } + + if (wqe->wait_read != wake_read) + continue; + + wqe->done = true; + handle = wqe->handle; + do_wakeup = true; + break; + } + + cpu_spin_unlock_xrestore(&wq_spin_lock, old_itr_status); + + if (do_wakeup) + do_notif(notif_send_sync, handle, + "wake ", sync_obj, fname, lineno); + + if (!do_wakeup || !wake_read) + break; + do_wakeup = false; + } +} + +void wq_promote_condvar(struct wait_queue *wq, struct condvar *cv, + bool only_one, const void *sync_obj __unused, + const char *fname, int lineno __maybe_unused) +{ + uint32_t old_itr_status; + struct wait_queue_elem *wqe; + + if (!cv) + return; + + old_itr_status = cpu_spin_lock_xsave(&wq_spin_lock); + + /* + * Find condvar waiter(s) and promote each to an active waiter. + * This is a bit unfair to eventual other active waiters as a + * condvar waiter is added to the queue when waiting for the + * condvar. + */ + SLIST_FOREACH(wqe, wq, link) { + if (wqe->cv == cv) { + if (fname) + FMSG("promote thread %u %p %s:%d", + wqe->handle, (void *)cv->m, fname, lineno); + else + FMSG("promote thread %u %p", + wqe->handle, (void *)cv->m); + + wqe->cv = NULL; + if (only_one) + break; + } + } + + cpu_spin_unlock_xrestore(&wq_spin_lock, old_itr_status); +} + +bool wq_have_condvar(struct wait_queue *wq, struct condvar *cv) +{ + uint32_t old_itr_status; + struct wait_queue_elem *wqe; + bool rc = false; + + old_itr_status = cpu_spin_lock_xsave(&wq_spin_lock); + + SLIST_FOREACH(wqe, wq, link) { + if (wqe->cv == cv) { + rc = true; + break; + } + } + + cpu_spin_unlock_xrestore(&wq_spin_lock, old_itr_status); + + return rc; +} + +bool wq_is_empty(struct wait_queue *wq) +{ + uint32_t old_itr_status; + bool ret; + + old_itr_status = cpu_spin_lock_xsave(&wq_spin_lock); + + ret = SLIST_EMPTY(wq); + + cpu_spin_unlock_xrestore(&wq_spin_lock, old_itr_status); + + return ret; +} diff --git a/optee_os/core/lib/libfdt/README.license b/optee_os/core/lib/libfdt/README.license new file mode 100644 index 0000000..102b004 --- /dev/null +++ b/optee_os/core/lib/libfdt/README.license @@ -0,0 +1,56 @@ +Licensing and contribution policy of dtc and libfdt +=================================================== + +This dtc package contains two pieces of software: dtc itself, and +libfdt which comprises the files in the libfdt/ subdirectory. These +two pieces of software, although closely related, are quite distinct. +dtc does not incorporate or rely on libfdt for its operation, nor vice +versa. It is important that these two pieces of software have +different license conditions. + +As SPDX license tags in each source file attest, dtc is licensed +under the GNU GPL. The full text of the GPL can be found in the file +entitled 'GPL' which should be included in this package. dtc code, +therefore, may not be incorporated into works which do not have a GPL +compatible license. + +libfdt, however, is GPL/BSD dual-licensed. That is, it may be used +either under the terms of the GPL, or under the terms of the 2-clause +BSD license (aka the ISC license). The full terms of that license can +be found are in the file entitled 'BSD-2-Clause'. This is, in +practice, equivalent to being BSD licensed, since the terms of the BSD +license are strictly more permissive than the GPL. + +I made the decision to license libfdt in this way because I want to +encourage widespread and correct usage of flattened device trees, +including by proprietary or otherwise GPL-incompatible firmware or +tools. Allowing libfdt to be used under the terms of the BSD license +makes that it easier for vendors or authors of such software to do so. + +This does mean that libfdt code could be "stolen" - say, included in a +proprietary fimware and extended without contributing those extensions +back to the libfdt mainline. While I hope that doesn't happen, I +believe the goal of allowing libfdt to be widely used is more +important than avoiding that. libfdt is quite small, and hardly +rocket science; so the incentive for such impolite behaviour is small, +and the inconvenience caused thereby is not dire. + +Licenses such as the LGPL which would allow code to be used in non-GPL +software, but also require contributions to be returned were +considered. However, libfdt is designed to be used in firmwares and +other environments with unusual technical constraints. It's difficult +to anticipate all possible changes which might be needed to meld +libfdt into such environments and so difficult to suitably word a +license that puts the boundary between what is and isn't permitted in +the intended place. Again, I judged encouraging widespread use of +libfdt by keeping the license terms simple and familiar to be the more +important goal. + +**IMPORTANT** It's intended that all of libfdt as released remain +permissively licensed this way. Therefore only contributions which +are released under these terms can be merged into the libfdt mainline. + + +David Gibson +(principal original author of dtc and libfdt) +2 November 2007 diff --git a/optee_os/core/lib/libfdt/fdt.c b/optee_os/core/lib/libfdt/fdt.c new file mode 100644 index 0000000..d6ce7c0 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +/* + * Minimal sanity check for a read-only tree. fdt_ro_probe_() checks + * that the given buffer contains what appears to be a flattened + * device tree with sane information in its header. + */ +int32_t fdt_ro_probe_(const void *fdt) +{ + uint32_t totalsize = fdt_totalsize(fdt); + + if (fdt_magic(fdt) == FDT_MAGIC) { + /* Complete tree */ + if (fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION) + return -FDT_ERR_BADVERSION; + if (fdt_last_comp_version(fdt) > FDT_LAST_SUPPORTED_VERSION) + return -FDT_ERR_BADVERSION; + } else if (fdt_magic(fdt) == FDT_SW_MAGIC) { + /* Unfinished sequential-write blob */ + if (fdt_size_dt_struct(fdt) == 0) + return -FDT_ERR_BADSTATE; + } else { + return -FDT_ERR_BADMAGIC; + } + + if (totalsize < INT32_MAX) + return totalsize; + else + return -FDT_ERR_TRUNCATED; +} + +static int check_off_(uint32_t hdrsize, uint32_t totalsize, uint32_t off) +{ + return (off >= hdrsize) && (off <= totalsize); +} + +static int check_block_(uint32_t hdrsize, uint32_t totalsize, + uint32_t base, uint32_t size) +{ + if (!check_off_(hdrsize, totalsize, base)) + return 0; /* block start out of bounds */ + if ((base + size) < base) + return 0; /* overflow */ + if (!check_off_(hdrsize, totalsize, base + size)) + return 0; /* block end out of bounds */ + return 1; +} + +size_t fdt_header_size_(uint32_t version) +{ + if (version <= 1) + return FDT_V1_SIZE; + else if (version <= 2) + return FDT_V2_SIZE; + else if (version <= 3) + return FDT_V3_SIZE; + else if (version <= 16) + return FDT_V16_SIZE; + else + return FDT_V17_SIZE; +} + +int fdt_check_header(const void *fdt) +{ + size_t hdrsize; + + if (fdt_magic(fdt) != FDT_MAGIC) + return -FDT_ERR_BADMAGIC; + hdrsize = fdt_header_size(fdt); + if ((fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION) + || (fdt_last_comp_version(fdt) > FDT_LAST_SUPPORTED_VERSION)) + return -FDT_ERR_BADVERSION; + if (fdt_version(fdt) < fdt_last_comp_version(fdt)) + return -FDT_ERR_BADVERSION; + + if ((fdt_totalsize(fdt) < hdrsize) + || (fdt_totalsize(fdt) > INT_MAX)) + return -FDT_ERR_TRUNCATED; + + /* Bounds check memrsv block */ + if (!check_off_(hdrsize, fdt_totalsize(fdt), fdt_off_mem_rsvmap(fdt))) + return -FDT_ERR_TRUNCATED; + + /* Bounds check structure block */ + if (fdt_version(fdt) < 17) { + if (!check_off_(hdrsize, fdt_totalsize(fdt), + fdt_off_dt_struct(fdt))) + return -FDT_ERR_TRUNCATED; + } else { + if (!check_block_(hdrsize, fdt_totalsize(fdt), + fdt_off_dt_struct(fdt), + fdt_size_dt_struct(fdt))) + return -FDT_ERR_TRUNCATED; + } + + /* Bounds check strings block */ + if (!check_block_(hdrsize, fdt_totalsize(fdt), + fdt_off_dt_strings(fdt), fdt_size_dt_strings(fdt))) + return -FDT_ERR_TRUNCATED; + + return 0; +} + +const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int len) +{ + unsigned absoffset = offset + fdt_off_dt_struct(fdt); + + if ((absoffset < offset) + || ((absoffset + len) < absoffset) + || (absoffset + len) > fdt_totalsize(fdt)) + return NULL; + + if (fdt_version(fdt) >= 0x11) + if (((offset + len) < offset) + || ((offset + len) > fdt_size_dt_struct(fdt))) + return NULL; + + return fdt_offset_ptr_(fdt, offset); +} + +uint32_t fdt_next_tag(const void *fdt, int startoffset, int *nextoffset) +{ + const fdt32_t *tagp, *lenp; + uint32_t tag; + int offset = startoffset; + const char *p; + + *nextoffset = -FDT_ERR_TRUNCATED; + tagp = fdt_offset_ptr(fdt, offset, FDT_TAGSIZE); + if (!tagp) + return FDT_END; /* premature end */ + tag = fdt32_to_cpu(*tagp); + offset += FDT_TAGSIZE; + + *nextoffset = -FDT_ERR_BADSTRUCTURE; + switch (tag) { + case FDT_BEGIN_NODE: + /* skip name */ + do { + p = fdt_offset_ptr(fdt, offset++, 1); + } while (p && (*p != '\0')); + if (!p) + return FDT_END; /* premature end */ + break; + + case FDT_PROP: + lenp = fdt_offset_ptr(fdt, offset, sizeof(*lenp)); + if (!lenp) + return FDT_END; /* premature end */ + /* skip-name offset, length and value */ + offset += sizeof(struct fdt_property) - FDT_TAGSIZE + + fdt32_to_cpu(*lenp); + if (fdt_version(fdt) < 0x10 && fdt32_to_cpu(*lenp) >= 8 && + ((offset - fdt32_to_cpu(*lenp)) % 8) != 0) + offset += 4; + break; + + case FDT_END: + case FDT_END_NODE: + case FDT_NOP: + break; + + default: + return FDT_END; + } + + if (!fdt_offset_ptr(fdt, startoffset, offset - startoffset)) + return FDT_END; /* premature end */ + + *nextoffset = FDT_TAGALIGN(offset); + return tag; +} + +int fdt_check_node_offset_(const void *fdt, int offset) +{ + if ((offset < 0) || (offset % FDT_TAGSIZE) + || (fdt_next_tag(fdt, offset, &offset) != FDT_BEGIN_NODE)) + return -FDT_ERR_BADOFFSET; + + return offset; +} + +int fdt_check_prop_offset_(const void *fdt, int offset) +{ + if ((offset < 0) || (offset % FDT_TAGSIZE) + || (fdt_next_tag(fdt, offset, &offset) != FDT_PROP)) + return -FDT_ERR_BADOFFSET; + + return offset; +} + +int fdt_next_node(const void *fdt, int offset, int *depth) +{ + int nextoffset = 0; + uint32_t tag; + + if (offset >= 0) + if ((nextoffset = fdt_check_node_offset_(fdt, offset)) < 0) + return nextoffset; + + do { + offset = nextoffset; + tag = fdt_next_tag(fdt, offset, &nextoffset); + + switch (tag) { + case FDT_PROP: + case FDT_NOP: + break; + + case FDT_BEGIN_NODE: + if (depth) + (*depth)++; + break; + + case FDT_END_NODE: + if (depth && ((--(*depth)) < 0)) + return nextoffset; + break; + + case FDT_END: + if ((nextoffset >= 0) + || ((nextoffset == -FDT_ERR_TRUNCATED) && !depth)) + return -FDT_ERR_NOTFOUND; + else + return nextoffset; + } + } while (tag != FDT_BEGIN_NODE); + + return offset; +} + +int fdt_first_subnode(const void *fdt, int offset) +{ + int depth = 0; + + offset = fdt_next_node(fdt, offset, &depth); + if (offset < 0 || depth != 1) + return -FDT_ERR_NOTFOUND; + + return offset; +} + +int fdt_next_subnode(const void *fdt, int offset) +{ + int depth = 1; + + /* + * With respect to the parent, the depth of the next subnode will be + * the same as the last. + */ + do { + offset = fdt_next_node(fdt, offset, &depth); + if (offset < 0 || depth < 1) + return -FDT_ERR_NOTFOUND; + } while (depth > 1); + + return offset; +} + +const char *fdt_find_string_(const char *strtab, int tabsize, const char *s) +{ + int len = strlen(s) + 1; + const char *last = strtab + tabsize - len; + const char *p; + + for (p = strtab; p <= last; p++) + if (memcmp(p, s, len) == 0) + return p; + return NULL; +} + +int fdt_move(const void *fdt, void *buf, int bufsize) +{ + FDT_RO_PROBE(fdt); + + if (fdt_totalsize(fdt) > bufsize) + return -FDT_ERR_NOSPACE; + + memmove(buf, fdt, fdt_totalsize(fdt)); + return 0; +} diff --git a/optee_os/core/lib/libfdt/fdt_addresses.c b/optee_os/core/lib/libfdt/fdt_addresses.c new file mode 100644 index 0000000..9a82cd0 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_addresses.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2014 David Gibson + * Copyright (C) 2018 embedded brains GmbH + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +static int fdt_cells(const void *fdt, int nodeoffset, const char *name) +{ + const fdt32_t *c; + uint32_t val; + int len; + + c = fdt_getprop(fdt, nodeoffset, name, &len); + if (!c) + return len; + + if (len != sizeof(*c)) + return -FDT_ERR_BADNCELLS; + + val = fdt32_to_cpu(*c); + if (val > FDT_MAX_NCELLS) + return -FDT_ERR_BADNCELLS; + + return (int)val; +} + +int fdt_address_cells(const void *fdt, int nodeoffset) +{ + int val; + + val = fdt_cells(fdt, nodeoffset, "#address-cells"); + if (val == 0) + return -FDT_ERR_BADNCELLS; + if (val == -FDT_ERR_NOTFOUND) + return 2; + return val; +} + +int fdt_size_cells(const void *fdt, int nodeoffset) +{ + int val; + + val = fdt_cells(fdt, nodeoffset, "#size-cells"); + if (val == -FDT_ERR_NOTFOUND) + return 1; + return val; +} + +/* This function assumes that [address|size]_cells is 1 or 2 */ +int fdt_appendprop_addrrange(void *fdt, int parent, int nodeoffset, + const char *name, uint64_t addr, uint64_t size) +{ + int addr_cells, size_cells, ret; + uint8_t data[sizeof(fdt64_t) * 2], *prop; + + ret = fdt_address_cells(fdt, parent); + if (ret < 0) + return ret; + addr_cells = ret; + + ret = fdt_size_cells(fdt, parent); + if (ret < 0) + return ret; + size_cells = ret; + + /* check validity of address */ + prop = data; + if (addr_cells == 1) { + if ((addr > UINT32_MAX) || ((UINT32_MAX + 1 - addr) < size)) + return -FDT_ERR_BADVALUE; + + fdt32_st(prop, (uint32_t)addr); + } else if (addr_cells == 2) { + fdt64_st(prop, addr); + } else { + return -FDT_ERR_BADNCELLS; + } + + /* check validity of size */ + prop += addr_cells * sizeof(fdt32_t); + if (size_cells == 1) { + if (size > UINT32_MAX) + return -FDT_ERR_BADVALUE; + + fdt32_st(prop, (uint32_t)size); + } else if (size_cells == 2) { + fdt64_st(prop, size); + } else { + return -FDT_ERR_BADNCELLS; + } + + return fdt_appendprop(fdt, nodeoffset, name, data, + (addr_cells + size_cells) * sizeof(fdt32_t)); +} diff --git a/optee_os/core/lib/libfdt/fdt_empty_tree.c b/optee_os/core/lib/libfdt/fdt_empty_tree.c new file mode 100644 index 0000000..49d54d4 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_empty_tree.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2012 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +int fdt_create_empty_tree(void *buf, int bufsize) +{ + int err; + + err = fdt_create(buf, bufsize); + if (err) + return err; + + err = fdt_finish_reservemap(buf); + if (err) + return err; + + err = fdt_begin_node(buf, ""); + if (err) + return err; + + err = fdt_end_node(buf); + if (err) + return err; + + err = fdt_finish(buf); + if (err) + return err; + + return fdt_open_into(buf, buf, bufsize); +} diff --git a/optee_os/core/lib/libfdt/fdt_overlay.c b/optee_os/core/lib/libfdt/fdt_overlay.c new file mode 100644 index 0000000..e74fe92 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_overlay.c @@ -0,0 +1,881 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2016 Free Electrons + * Copyright (C) 2016 NextThing Co. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +/** + * overlay_get_target_phandle - retrieves the target phandle of a fragment + * @fdto: pointer to the device tree overlay blob + * @fragment: node offset of the fragment in the overlay + * + * overlay_get_target_phandle() retrieves the target phandle of an + * overlay fragment when that fragment uses a phandle (target + * property) instead of a path (target-path property). + * + * returns: + * the phandle pointed by the target property + * 0, if the phandle was not found + * -1, if the phandle was malformed + */ +static uint32_t overlay_get_target_phandle(const void *fdto, int fragment) +{ + const fdt32_t *val; + int len; + + val = fdt_getprop(fdto, fragment, "target", &len); + if (!val) + return 0; + + if ((len != sizeof(*val)) || (fdt32_to_cpu(*val) == (uint32_t)-1)) + return (uint32_t)-1; + + return fdt32_to_cpu(*val); +} + +/** + * overlay_get_target - retrieves the offset of a fragment's target + * @fdt: Base device tree blob + * @fdto: Device tree overlay blob + * @fragment: node offset of the fragment in the overlay + * @pathp: pointer which receives the path of the target (or NULL) + * + * overlay_get_target() retrieves the target offset in the base + * device tree of a fragment, no matter how the actual targeting is + * done (through a phandle or a path) + * + * returns: + * the targeted node offset in the base device tree + * Negative error code on error + */ +static int overlay_get_target(const void *fdt, const void *fdto, + int fragment, char const **pathp) +{ + uint32_t phandle; + const char *path = NULL; + int path_len = 0, ret; + + /* Try first to do a phandle based lookup */ + phandle = overlay_get_target_phandle(fdto, fragment); + if (phandle == (uint32_t)-1) + return -FDT_ERR_BADPHANDLE; + + /* no phandle, try path */ + if (!phandle) { + /* And then a path based lookup */ + path = fdt_getprop(fdto, fragment, "target-path", &path_len); + if (path) + ret = fdt_path_offset(fdt, path); + else + ret = path_len; + } else + ret = fdt_node_offset_by_phandle(fdt, phandle); + + /* + * If we haven't found either a target or a + * target-path property in a node that contains a + * __overlay__ subnode (we wouldn't be called + * otherwise), consider it a improperly written + * overlay + */ + if (ret < 0 && path_len == -FDT_ERR_NOTFOUND) + ret = -FDT_ERR_BADOVERLAY; + + /* return on error */ + if (ret < 0) + return ret; + + /* return pointer to path (if available) */ + if (pathp) + *pathp = path ? path : NULL; + + return ret; +} + +/** + * overlay_phandle_add_offset - Increases a phandle by an offset + * @fdt: Base device tree blob + * @node: Device tree overlay blob + * @name: Name of the property to modify (phandle or linux,phandle) + * @delta: offset to apply + * + * overlay_phandle_add_offset() increments a node phandle by a given + * offset. + * + * returns: + * 0 on success. + * Negative error code on error + */ +static int overlay_phandle_add_offset(void *fdt, int node, + const char *name, uint32_t delta) +{ + const fdt32_t *val; + uint32_t adj_val; + int len; + + val = fdt_getprop(fdt, node, name, &len); + if (!val) + return len; + + if (len != sizeof(*val)) + return -FDT_ERR_BADPHANDLE; + + adj_val = fdt32_to_cpu(*val); + if ((adj_val + delta) < adj_val) + return -FDT_ERR_NOPHANDLES; + + adj_val += delta; + if (adj_val == (uint32_t)-1) + return -FDT_ERR_NOPHANDLES; + + return fdt_setprop_inplace_u32(fdt, node, name, adj_val); +} + +/** + * overlay_adjust_node_phandles - Offsets the phandles of a node + * @fdto: Device tree overlay blob + * @node: Offset of the node we want to adjust + * @delta: Offset to shift the phandles of + * + * overlay_adjust_node_phandles() adds a constant to all the phandles + * of a given node. This is mainly use as part of the overlay + * application process, when we want to update all the overlay + * phandles to not conflict with the overlays of the base device tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_adjust_node_phandles(void *fdto, int node, + uint32_t delta) +{ + int child; + int ret; + + ret = overlay_phandle_add_offset(fdto, node, "phandle", delta); + if (ret && ret != -FDT_ERR_NOTFOUND) + return ret; + + ret = overlay_phandle_add_offset(fdto, node, "linux,phandle", delta); + if (ret && ret != -FDT_ERR_NOTFOUND) + return ret; + + fdt_for_each_subnode(child, fdto, node) { + ret = overlay_adjust_node_phandles(fdto, child, delta); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_adjust_local_phandles - Adjust the phandles of a whole overlay + * @fdto: Device tree overlay blob + * @delta: Offset to shift the phandles of + * + * overlay_adjust_local_phandles() adds a constant to all the + * phandles of an overlay. This is mainly use as part of the overlay + * application process, when we want to update all the overlay + * phandles to not conflict with the overlays of the base device tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_adjust_local_phandles(void *fdto, uint32_t delta) +{ + /* + * Start adjusting the phandles from the overlay root + */ + return overlay_adjust_node_phandles(fdto, 0, delta); +} + +/** + * overlay_update_local_node_references - Adjust the overlay references + * @fdto: Device tree overlay blob + * @tree_node: Node offset of the node to operate on + * @fixup_node: Node offset of the matching local fixups node + * @delta: Offset to shift the phandles of + * + * overlay_update_local_nodes_references() update the phandles + * pointing to a node within the device tree overlay by adding a + * constant delta. + * + * This is mainly used as part of a device tree application process, + * where you want the device tree overlays phandles to not conflict + * with the ones from the base device tree before merging them. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_update_local_node_references(void *fdto, + int tree_node, + int fixup_node, + uint32_t delta) +{ + int fixup_prop; + int fixup_child; + int ret; + + fdt_for_each_property_offset(fixup_prop, fdto, fixup_node) { + const fdt32_t *fixup_val; + const char *tree_val; + const char *name; + int fixup_len; + int tree_len; + int i; + + fixup_val = fdt_getprop_by_offset(fdto, fixup_prop, + &name, &fixup_len); + if (!fixup_val) + return fixup_len; + + if (fixup_len % sizeof(uint32_t)) + return -FDT_ERR_BADOVERLAY; + + tree_val = fdt_getprop(fdto, tree_node, name, &tree_len); + if (!tree_val) { + if (tree_len == -FDT_ERR_NOTFOUND) + return -FDT_ERR_BADOVERLAY; + + return tree_len; + } + + for (i = 0; i < (fixup_len / sizeof(uint32_t)); i++) { + fdt32_t adj_val; + uint32_t poffset; + + poffset = fdt32_to_cpu(fixup_val[i]); + + /* + * phandles to fixup can be unaligned. + * + * Use a memcpy for the architectures that do + * not support unaligned accesses. + */ + memcpy(&adj_val, tree_val + poffset, sizeof(adj_val)); + + adj_val = cpu_to_fdt32(fdt32_to_cpu(adj_val) + delta); + + ret = fdt_setprop_inplace_namelen_partial(fdto, + tree_node, + name, + strlen(name), + poffset, + &adj_val, + sizeof(adj_val)); + if (ret == -FDT_ERR_NOSPACE) + return -FDT_ERR_BADOVERLAY; + + if (ret) + return ret; + } + } + + fdt_for_each_subnode(fixup_child, fdto, fixup_node) { + const char *fixup_child_name = fdt_get_name(fdto, fixup_child, + NULL); + int tree_child; + + tree_child = fdt_subnode_offset(fdto, tree_node, + fixup_child_name); + if (tree_child == -FDT_ERR_NOTFOUND) + return -FDT_ERR_BADOVERLAY; + if (tree_child < 0) + return tree_child; + + ret = overlay_update_local_node_references(fdto, + tree_child, + fixup_child, + delta); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_update_local_references - Adjust the overlay references + * @fdto: Device tree overlay blob + * @delta: Offset to shift the phandles of + * + * overlay_update_local_references() update all the phandles pointing + * to a node within the device tree overlay by adding a constant + * delta to not conflict with the base overlay. + * + * This is mainly used as part of a device tree application process, + * where you want the device tree overlays phandles to not conflict + * with the ones from the base device tree before merging them. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_update_local_references(void *fdto, uint32_t delta) +{ + int fixups; + + fixups = fdt_path_offset(fdto, "/__local_fixups__"); + if (fixups < 0) { + /* There's no local phandles to adjust, bail out */ + if (fixups == -FDT_ERR_NOTFOUND) + return 0; + + return fixups; + } + + /* + * Update our local references from the root of the tree + */ + return overlay_update_local_node_references(fdto, 0, fixups, + delta); +} + +/** + * overlay_fixup_one_phandle - Set an overlay phandle to the base one + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * @symbols_off: Node offset of the symbols node in the base device tree + * @path: Path to a node holding a phandle in the overlay + * @path_len: number of path characters to consider + * @name: Name of the property holding the phandle reference in the overlay + * @name_len: number of name characters to consider + * @poffset: Offset within the overlay property where the phandle is stored + * @label: Label of the node referenced by the phandle + * + * overlay_fixup_one_phandle() resolves an overlay phandle pointing to + * a node in the base device tree. + * + * This is part of the device tree overlay application process, when + * you want all the phandles in the overlay to point to the actual + * base dt nodes. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_fixup_one_phandle(void *fdt, void *fdto, + int symbols_off, + const char *path, uint32_t path_len, + const char *name, uint32_t name_len, + int poffset, const char *label) +{ + const char *symbol_path; + uint32_t phandle; + fdt32_t phandle_prop; + int symbol_off, fixup_off; + int prop_len; + + if (symbols_off < 0) + return symbols_off; + + symbol_path = fdt_getprop(fdt, symbols_off, label, + &prop_len); + if (!symbol_path) + return prop_len; + + symbol_off = fdt_path_offset(fdt, symbol_path); + if (symbol_off < 0) + return symbol_off; + + phandle = fdt_get_phandle(fdt, symbol_off); + if (!phandle) + return -FDT_ERR_NOTFOUND; + + fixup_off = fdt_path_offset_namelen(fdto, path, path_len); + if (fixup_off == -FDT_ERR_NOTFOUND) + return -FDT_ERR_BADOVERLAY; + if (fixup_off < 0) + return fixup_off; + + phandle_prop = cpu_to_fdt32(phandle); + return fdt_setprop_inplace_namelen_partial(fdto, fixup_off, + name, name_len, poffset, + &phandle_prop, + sizeof(phandle_prop)); +} + +/** + * overlay_fixup_phandle - Set an overlay phandle to the base one + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * @symbols_off: Node offset of the symbols node in the base device tree + * @property: Property offset in the overlay holding the list of fixups + * + * overlay_fixup_phandle() resolves all the overlay phandles pointed + * to in a __fixups__ property, and updates them to match the phandles + * in use in the base device tree. + * + * This is part of the device tree overlay application process, when + * you want all the phandles in the overlay to point to the actual + * base dt nodes. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_fixup_phandle(void *fdt, void *fdto, int symbols_off, + int property) +{ + const char *value; + const char *label; + int len; + + value = fdt_getprop_by_offset(fdto, property, + &label, &len); + if (!value) { + if (len == -FDT_ERR_NOTFOUND) + return -FDT_ERR_INTERNAL; + + return len; + } + + do { + const char *path, *name, *fixup_end; + const char *fixup_str = value; + uint32_t path_len, name_len; + uint32_t fixup_len; + char *sep, *endptr; + int poffset, ret; + + fixup_end = memchr(value, '\0', len); + if (!fixup_end) + return -FDT_ERR_BADOVERLAY; + fixup_len = fixup_end - fixup_str; + + len -= fixup_len + 1; + value += fixup_len + 1; + + path = fixup_str; + sep = memchr(fixup_str, ':', fixup_len); + if (!sep || *sep != ':') + return -FDT_ERR_BADOVERLAY; + + path_len = sep - path; + if (path_len == (fixup_len - 1)) + return -FDT_ERR_BADOVERLAY; + + fixup_len -= path_len + 1; + name = sep + 1; + sep = memchr(name, ':', fixup_len); + if (!sep || *sep != ':') + return -FDT_ERR_BADOVERLAY; + + name_len = sep - name; + if (!name_len) + return -FDT_ERR_BADOVERLAY; + + poffset = strtoul(sep + 1, &endptr, 10); + if ((*endptr != '\0') || (endptr <= (sep + 1))) + return -FDT_ERR_BADOVERLAY; + + ret = overlay_fixup_one_phandle(fdt, fdto, symbols_off, + path, path_len, name, name_len, + poffset, label); + if (ret) + return ret; + } while (len > 0); + + return 0; +} + +/** + * overlay_fixup_phandles - Resolve the overlay phandles to the base + * device tree + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * + * overlay_fixup_phandles() resolves all the overlay phandles pointing + * to nodes in the base device tree. + * + * This is one of the steps of the device tree overlay application + * process, when you want all the phandles in the overlay to point to + * the actual base dt nodes. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_fixup_phandles(void *fdt, void *fdto) +{ + int fixups_off, symbols_off; + int property; + + /* We can have overlays without any fixups */ + fixups_off = fdt_path_offset(fdto, "/__fixups__"); + if (fixups_off == -FDT_ERR_NOTFOUND) + return 0; /* nothing to do */ + if (fixups_off < 0) + return fixups_off; + + /* And base DTs without symbols */ + symbols_off = fdt_path_offset(fdt, "/__symbols__"); + if ((symbols_off < 0 && (symbols_off != -FDT_ERR_NOTFOUND))) + return symbols_off; + + fdt_for_each_property_offset(property, fdto, fixups_off) { + int ret; + + ret = overlay_fixup_phandle(fdt, fdto, symbols_off, property); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_apply_node - Merges a node into the base device tree + * @fdt: Base Device Tree blob + * @target: Node offset in the base device tree to apply the fragment to + * @fdto: Device tree overlay blob + * @node: Node offset in the overlay holding the changes to merge + * + * overlay_apply_node() merges a node into a target base device tree + * node pointed. + * + * This is part of the final step in the device tree overlay + * application process, when all the phandles have been adjusted and + * resolved and you just have to merge overlay into the base device + * tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_apply_node(void *fdt, int target, + void *fdto, int node) +{ + int property; + int subnode; + + fdt_for_each_property_offset(property, fdto, node) { + const char *name; + const void *prop; + int prop_len; + int ret; + + prop = fdt_getprop_by_offset(fdto, property, &name, + &prop_len); + if (prop_len == -FDT_ERR_NOTFOUND) + return -FDT_ERR_INTERNAL; + if (prop_len < 0) + return prop_len; + + ret = fdt_setprop(fdt, target, name, prop, prop_len); + if (ret) + return ret; + } + + fdt_for_each_subnode(subnode, fdto, node) { + const char *name = fdt_get_name(fdto, subnode, NULL); + int nnode; + int ret; + + nnode = fdt_add_subnode(fdt, target, name); + if (nnode == -FDT_ERR_EXISTS) { + nnode = fdt_subnode_offset(fdt, target, name); + if (nnode == -FDT_ERR_NOTFOUND) + return -FDT_ERR_INTERNAL; + } + + if (nnode < 0) + return nnode; + + ret = overlay_apply_node(fdt, nnode, fdto, subnode); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_merge - Merge an overlay into its base device tree + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * + * overlay_merge() merges an overlay into its base device tree. + * + * This is the next to last step in the device tree overlay application + * process, when all the phandles have been adjusted and resolved and + * you just have to merge overlay into the base device tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_merge(void *fdt, void *fdto) +{ + int fragment; + + fdt_for_each_subnode(fragment, fdto, 0) { + int overlay; + int target; + int ret; + + /* + * Each fragments will have an __overlay__ node. If + * they don't, it's not supposed to be merged + */ + overlay = fdt_subnode_offset(fdto, fragment, "__overlay__"); + if (overlay == -FDT_ERR_NOTFOUND) + continue; + + if (overlay < 0) + return overlay; + + target = overlay_get_target(fdt, fdto, fragment, NULL); + if (target < 0) + return target; + + ret = overlay_apply_node(fdt, target, fdto, overlay); + if (ret) + return ret; + } + + return 0; +} + +static int get_path_len(const void *fdt, int nodeoffset) +{ + int len = 0, namelen; + const char *name; + + FDT_RO_PROBE(fdt); + + for (;;) { + name = fdt_get_name(fdt, nodeoffset, &namelen); + if (!name) + return namelen; + + /* root? we're done */ + if (namelen == 0) + break; + + nodeoffset = fdt_parent_offset(fdt, nodeoffset); + if (nodeoffset < 0) + return nodeoffset; + len += namelen + 1; + } + + /* in case of root pretend it's "/" */ + if (len == 0) + len++; + return len; +} + +/** + * overlay_symbol_update - Update the symbols of base tree after a merge + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * + * overlay_symbol_update() updates the symbols of the base tree with the + * symbols of the applied overlay + * + * This is the last step in the device tree overlay application + * process, allowing the reference of overlay symbols by subsequent + * overlay operations. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_symbol_update(void *fdt, void *fdto) +{ + int root_sym, ov_sym, prop, path_len, fragment, target; + int len, frag_name_len, ret, rel_path_len; + const char *s, *e; + const char *path; + const char *name; + const char *frag_name; + const char *rel_path; + const char *target_path; + char *buf; + void *p; + + ov_sym = fdt_subnode_offset(fdto, 0, "__symbols__"); + + /* if no overlay symbols exist no problem */ + if (ov_sym < 0) + return 0; + + root_sym = fdt_subnode_offset(fdt, 0, "__symbols__"); + + /* it no root symbols exist we should create them */ + if (root_sym == -FDT_ERR_NOTFOUND) + root_sym = fdt_add_subnode(fdt, 0, "__symbols__"); + + /* any error is fatal now */ + if (root_sym < 0) + return root_sym; + + /* iterate over each overlay symbol */ + fdt_for_each_property_offset(prop, fdto, ov_sym) { + path = fdt_getprop_by_offset(fdto, prop, &name, &path_len); + if (!path) + return path_len; + + /* verify it's a string property (terminated by a single \0) */ + if (path_len < 1 || memchr(path, '\0', path_len) != &path[path_len - 1]) + return -FDT_ERR_BADVALUE; + + /* keep end marker to avoid strlen() */ + e = path + path_len; + + if (*path != '/') + return -FDT_ERR_BADVALUE; + + /* get fragment name first */ + s = strchr(path + 1, '/'); + if (!s) { + /* Symbol refers to something that won't end + * up in the target tree */ + continue; + } + + frag_name = path + 1; + frag_name_len = s - path - 1; + + /* verify format; safe since "s" lies in \0 terminated prop */ + len = sizeof("/__overlay__/") - 1; + if ((e - s) > len && (memcmp(s, "/__overlay__/", len) == 0)) { + /* //__overlay__/ */ + rel_path = s + len; + rel_path_len = e - rel_path; + } else if ((e - s) == len + && (memcmp(s, "/__overlay__", len - 1) == 0)) { + /* //__overlay__ */ + rel_path = ""; + rel_path_len = 0; + } else { + /* Symbol refers to something that won't end + * up in the target tree */ + continue; + } + + /* find the fragment index in which the symbol lies */ + ret = fdt_subnode_offset_namelen(fdto, 0, frag_name, + frag_name_len); + /* not found? */ + if (ret < 0) + return -FDT_ERR_BADOVERLAY; + fragment = ret; + + /* an __overlay__ subnode must exist */ + ret = fdt_subnode_offset(fdto, fragment, "__overlay__"); + if (ret < 0) + return -FDT_ERR_BADOVERLAY; + + /* get the target of the fragment */ + ret = overlay_get_target(fdt, fdto, fragment, &target_path); + if (ret < 0) + return ret; + target = ret; + + /* if we have a target path use */ + if (!target_path) { + ret = get_path_len(fdt, target); + if (ret < 0) + return ret; + len = ret; + } else { + len = strlen(target_path); + } + + ret = fdt_setprop_placeholder(fdt, root_sym, name, + len + (len > 1) + rel_path_len + 1, &p); + if (ret < 0) + return ret; + + if (!target_path) { + /* again in case setprop_placeholder changed it */ + ret = overlay_get_target(fdt, fdto, fragment, &target_path); + if (ret < 0) + return ret; + target = ret; + } + + buf = p; + if (len > 1) { /* target is not root */ + if (!target_path) { + ret = fdt_get_path(fdt, target, buf, len + 1); + if (ret < 0) + return ret; + } else + memcpy(buf, target_path, len + 1); + + } else + len--; + + buf[len] = '/'; + memcpy(buf + len + 1, rel_path, rel_path_len); + buf[len + 1 + rel_path_len] = '\0'; + } + + return 0; +} + +int fdt_overlay_apply(void *fdt, void *fdto) +{ + uint32_t delta; + int ret; + + FDT_RO_PROBE(fdt); + FDT_RO_PROBE(fdto); + + ret = fdt_find_max_phandle(fdt, &delta); + if (ret) + goto err; + + ret = overlay_adjust_local_phandles(fdto, delta); + if (ret) + goto err; + + ret = overlay_update_local_references(fdto, delta); + if (ret) + goto err; + + ret = overlay_fixup_phandles(fdt, fdto); + if (ret) + goto err; + + ret = overlay_merge(fdt, fdto); + if (ret) + goto err; + + ret = overlay_symbol_update(fdt, fdto); + if (ret) + goto err; + + /* + * The overlay has been damaged, erase its magic. + */ + fdt_set_magic(fdto, ~0); + + return 0; + +err: + /* + * The overlay might have been damaged, erase its magic. + */ + fdt_set_magic(fdto, ~0); + + /* + * The base device tree might have been damaged, erase its + * magic. + */ + fdt_set_magic(fdt, ~0); + + return ret; +} diff --git a/optee_os/core/lib/libfdt/fdt_ro.c b/optee_os/core/lib/libfdt/fdt_ro.c new file mode 100644 index 0000000..a5c2797 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_ro.c @@ -0,0 +1,898 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +static int fdt_nodename_eq_(const void *fdt, int offset, + const char *s, int len) +{ + int olen; + const char *p = fdt_get_name(fdt, offset, &olen); + + if (!p || olen < len) + /* short match */ + return 0; + + if (memcmp(p, s, len) != 0) + return 0; + + if (p[len] == '\0') + return 1; + else if (!memchr(s, '@', len) && (p[len] == '@')) + return 1; + else + return 0; +} + +const char *fdt_get_string(const void *fdt, int stroffset, int *lenp) +{ + int32_t totalsize = fdt_ro_probe_(fdt); + uint32_t absoffset = stroffset + fdt_off_dt_strings(fdt); + size_t len; + int err; + const char *s, *n; + + err = totalsize; + if (totalsize < 0) + goto fail; + + err = -FDT_ERR_BADOFFSET; + if (absoffset >= totalsize) + goto fail; + len = totalsize - absoffset; + + if (fdt_magic(fdt) == FDT_MAGIC) { + if (stroffset < 0) + goto fail; + if (fdt_version(fdt) >= 17) { + if (stroffset >= fdt_size_dt_strings(fdt)) + goto fail; + if ((fdt_size_dt_strings(fdt) - stroffset) < len) + len = fdt_size_dt_strings(fdt) - stroffset; + } + } else if (fdt_magic(fdt) == FDT_SW_MAGIC) { + if ((stroffset >= 0) + || (stroffset < -fdt_size_dt_strings(fdt))) + goto fail; + if ((-stroffset) < len) + len = -stroffset; + } else { + err = -FDT_ERR_INTERNAL; + goto fail; + } + + s = (const char *)fdt + absoffset; + n = memchr(s, '\0', len); + if (!n) { + /* missing terminating NULL */ + err = -FDT_ERR_TRUNCATED; + goto fail; + } + + if (lenp) + *lenp = n - s; + return s; + +fail: + if (lenp) + *lenp = err; + return NULL; +} + +const char *fdt_string(const void *fdt, int stroffset) +{ + return fdt_get_string(fdt, stroffset, NULL); +} + +static int fdt_string_eq_(const void *fdt, int stroffset, + const char *s, int len) +{ + int slen; + const char *p = fdt_get_string(fdt, stroffset, &slen); + + return p && (slen == len) && (memcmp(p, s, len) == 0); +} + +int fdt_find_max_phandle(const void *fdt, uint32_t *phandle) +{ + uint32_t max = 0; + int offset = -1; + + while (true) { + uint32_t value; + + offset = fdt_next_node(fdt, offset, NULL); + if (offset < 0) { + if (offset == -FDT_ERR_NOTFOUND) + break; + + return offset; + } + + value = fdt_get_phandle(fdt, offset); + + if (value > max) + max = value; + } + + if (phandle) + *phandle = max; + + return 0; +} + +int fdt_generate_phandle(const void *fdt, uint32_t *phandle) +{ + uint32_t max; + int err; + + err = fdt_find_max_phandle(fdt, &max); + if (err < 0) + return err; + + if (max == FDT_MAX_PHANDLE) + return -FDT_ERR_NOPHANDLES; + + if (phandle) + *phandle = max + 1; + + return 0; +} + +static const struct fdt_reserve_entry *fdt_mem_rsv(const void *fdt, int n) +{ + int offset = n * sizeof(struct fdt_reserve_entry); + int absoffset = fdt_off_mem_rsvmap(fdt) + offset; + + if (absoffset < fdt_off_mem_rsvmap(fdt)) + return NULL; + if (absoffset > fdt_totalsize(fdt) - sizeof(struct fdt_reserve_entry)) + return NULL; + return fdt_mem_rsv_(fdt, n); +} + +int fdt_get_mem_rsv(const void *fdt, int n, uint64_t *address, uint64_t *size) +{ + const struct fdt_reserve_entry *re; + + FDT_RO_PROBE(fdt); + re = fdt_mem_rsv(fdt, n); + if (!re) + return -FDT_ERR_BADOFFSET; + + *address = fdt64_ld(&re->address); + *size = fdt64_ld(&re->size); + return 0; +} + +int fdt_num_mem_rsv(const void *fdt) +{ + int i; + const struct fdt_reserve_entry *re; + + for (i = 0; (re = fdt_mem_rsv(fdt, i)) != NULL; i++) { + if (fdt64_ld(&re->size) == 0) + return i; + } + return -FDT_ERR_TRUNCATED; +} + +static int nextprop_(const void *fdt, int offset) +{ + uint32_t tag; + int nextoffset; + + do { + tag = fdt_next_tag(fdt, offset, &nextoffset); + + switch (tag) { + case FDT_END: + if (nextoffset >= 0) + return -FDT_ERR_BADSTRUCTURE; + else + return nextoffset; + + case FDT_PROP: + return offset; + } + offset = nextoffset; + } while (tag == FDT_NOP); + + return -FDT_ERR_NOTFOUND; +} + +int fdt_subnode_offset_namelen(const void *fdt, int offset, + const char *name, int namelen) +{ + int depth; + + FDT_RO_PROBE(fdt); + + for (depth = 0; + (offset >= 0) && (depth >= 0); + offset = fdt_next_node(fdt, offset, &depth)) + if ((depth == 1) + && fdt_nodename_eq_(fdt, offset, name, namelen)) + return offset; + + if (depth < 0) + return -FDT_ERR_NOTFOUND; + return offset; /* error */ +} + +int fdt_subnode_offset(const void *fdt, int parentoffset, + const char *name) +{ + return fdt_subnode_offset_namelen(fdt, parentoffset, name, strlen(name)); +} + +int fdt_path_offset_namelen(const void *fdt, const char *path, int namelen) +{ + const char *end = path + namelen; + const char *p = path; + int offset = 0; + + FDT_RO_PROBE(fdt); + + /* see if we have an alias */ + if (*path != '/') { + const char *q = memchr(path, '/', end - p); + + if (!q) + q = end; + + p = fdt_get_alias_namelen(fdt, p, q - p); + if (!p) + return -FDT_ERR_BADPATH; + offset = fdt_path_offset(fdt, p); + + p = q; + } + + while (p < end) { + const char *q; + + while (*p == '/') { + p++; + if (p == end) + return offset; + } + q = memchr(p, '/', end - p); + if (! q) + q = end; + + offset = fdt_subnode_offset_namelen(fdt, offset, p, q-p); + if (offset < 0) + return offset; + + p = q; + } + + return offset; +} + +int fdt_path_offset(const void *fdt, const char *path) +{ + return fdt_path_offset_namelen(fdt, path, strlen(path)); +} + +const char *fdt_get_name(const void *fdt, int nodeoffset, int *len) +{ + const struct fdt_node_header *nh = fdt_offset_ptr_(fdt, nodeoffset); + const char *nameptr; + int err; + + if (((err = fdt_ro_probe_(fdt)) < 0) + || ((err = fdt_check_node_offset_(fdt, nodeoffset)) < 0)) + goto fail; + + nameptr = nh->name; + + if (fdt_version(fdt) < 0x10) { + /* + * For old FDT versions, match the naming conventions of V16: + * give only the leaf name (after all /). The actual tree + * contents are loosely checked. + */ + const char *leaf; + leaf = strrchr(nameptr, '/'); + if (leaf == NULL) { + err = -FDT_ERR_BADSTRUCTURE; + goto fail; + } + nameptr = leaf+1; + } + + if (len) + *len = strlen(nameptr); + + return nameptr; + + fail: + if (len) + *len = err; + return NULL; +} + +int fdt_first_property_offset(const void *fdt, int nodeoffset) +{ + int offset; + + if ((offset = fdt_check_node_offset_(fdt, nodeoffset)) < 0) + return offset; + + return nextprop_(fdt, offset); +} + +int fdt_next_property_offset(const void *fdt, int offset) +{ + if ((offset = fdt_check_prop_offset_(fdt, offset)) < 0) + return offset; + + return nextprop_(fdt, offset); +} + +static const struct fdt_property *fdt_get_property_by_offset_(const void *fdt, + int offset, + int *lenp) +{ + int err; + const struct fdt_property *prop; + + if ((err = fdt_check_prop_offset_(fdt, offset)) < 0) { + if (lenp) + *lenp = err; + return NULL; + } + + prop = fdt_offset_ptr_(fdt, offset); + + if (lenp) + *lenp = fdt32_ld(&prop->len); + + return prop; +} + +const struct fdt_property *fdt_get_property_by_offset(const void *fdt, + int offset, + int *lenp) +{ + /* Prior to version 16, properties may need realignment + * and this API does not work. fdt_getprop_*() will, however. */ + + if (fdt_version(fdt) < 0x10) { + if (lenp) + *lenp = -FDT_ERR_BADVERSION; + return NULL; + } + + return fdt_get_property_by_offset_(fdt, offset, lenp); +} + +static const struct fdt_property *fdt_get_property_namelen_(const void *fdt, + int offset, + const char *name, + int namelen, + int *lenp, + int *poffset) +{ + for (offset = fdt_first_property_offset(fdt, offset); + (offset >= 0); + (offset = fdt_next_property_offset(fdt, offset))) { + const struct fdt_property *prop; + + if (!(prop = fdt_get_property_by_offset_(fdt, offset, lenp))) { + offset = -FDT_ERR_INTERNAL; + break; + } + if (fdt_string_eq_(fdt, fdt32_ld(&prop->nameoff), + name, namelen)) { + if (poffset) + *poffset = offset; + return prop; + } + } + + if (lenp) + *lenp = offset; + return NULL; +} + + +const struct fdt_property *fdt_get_property_namelen(const void *fdt, + int offset, + const char *name, + int namelen, int *lenp) +{ + /* Prior to version 16, properties may need realignment + * and this API does not work. fdt_getprop_*() will, however. */ + if (fdt_version(fdt) < 0x10) { + if (lenp) + *lenp = -FDT_ERR_BADVERSION; + return NULL; + } + + return fdt_get_property_namelen_(fdt, offset, name, namelen, lenp, + NULL); +} + + +const struct fdt_property *fdt_get_property(const void *fdt, + int nodeoffset, + const char *name, int *lenp) +{ + return fdt_get_property_namelen(fdt, nodeoffset, name, + strlen(name), lenp); +} + +const void *fdt_getprop_namelen(const void *fdt, int nodeoffset, + const char *name, int namelen, int *lenp) +{ + int poffset; + const struct fdt_property *prop; + + prop = fdt_get_property_namelen_(fdt, nodeoffset, name, namelen, lenp, + &poffset); + if (!prop) + return NULL; + + /* Handle realignment */ + if (fdt_version(fdt) < 0x10 && (poffset + sizeof(*prop)) % 8 && + fdt32_ld(&prop->len) >= 8) + return prop->data + 4; + return prop->data; +} + +const void *fdt_getprop_by_offset(const void *fdt, int offset, + const char **namep, int *lenp) +{ + const struct fdt_property *prop; + + prop = fdt_get_property_by_offset_(fdt, offset, lenp); + if (!prop) + return NULL; + if (namep) { + const char *name; + int namelen; + name = fdt_get_string(fdt, fdt32_ld(&prop->nameoff), + &namelen); + if (!name) { + if (lenp) + *lenp = namelen; + return NULL; + } + *namep = name; + } + + /* Handle realignment */ + if (fdt_version(fdt) < 0x10 && (offset + sizeof(*prop)) % 8 && + fdt32_ld(&prop->len) >= 8) + return prop->data + 4; + return prop->data; +} + +const void *fdt_getprop(const void *fdt, int nodeoffset, + const char *name, int *lenp) +{ + return fdt_getprop_namelen(fdt, nodeoffset, name, strlen(name), lenp); +} + +uint32_t fdt_get_phandle(const void *fdt, int nodeoffset) +{ + const fdt32_t *php; + int len; + + /* FIXME: This is a bit sub-optimal, since we potentially scan + * over all the properties twice. */ + php = fdt_getprop(fdt, nodeoffset, "phandle", &len); + if (!php || (len != sizeof(*php))) { + php = fdt_getprop(fdt, nodeoffset, "linux,phandle", &len); + if (!php || (len != sizeof(*php))) + return 0; + } + + return fdt32_ld(php); +} + +const char *fdt_get_alias_namelen(const void *fdt, + const char *name, int namelen) +{ + int aliasoffset; + + aliasoffset = fdt_path_offset(fdt, "/aliases"); + if (aliasoffset < 0) + return NULL; + + return fdt_getprop_namelen(fdt, aliasoffset, name, namelen, NULL); +} + +const char *fdt_get_alias(const void *fdt, const char *name) +{ + return fdt_get_alias_namelen(fdt, name, strlen(name)); +} + +int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen) +{ + int pdepth = 0, p = 0; + int offset, depth, namelen; + const char *name; + + FDT_RO_PROBE(fdt); + + if (buflen < 2) + return -FDT_ERR_NOSPACE; + + for (offset = 0, depth = 0; + (offset >= 0) && (offset <= nodeoffset); + offset = fdt_next_node(fdt, offset, &depth)) { + while (pdepth > depth) { + do { + p--; + } while (buf[p-1] != '/'); + pdepth--; + } + + if (pdepth >= depth) { + name = fdt_get_name(fdt, offset, &namelen); + if (!name) + return namelen; + if ((p + namelen + 1) <= buflen) { + memcpy(buf + p, name, namelen); + p += namelen; + buf[p++] = '/'; + pdepth++; + } + } + + if (offset == nodeoffset) { + if (pdepth < (depth + 1)) + return -FDT_ERR_NOSPACE; + + if (p > 1) /* special case so that root path is "/", not "" */ + p--; + buf[p] = '\0'; + return 0; + } + } + + if ((offset == -FDT_ERR_NOTFOUND) || (offset >= 0)) + return -FDT_ERR_BADOFFSET; + else if (offset == -FDT_ERR_BADOFFSET) + return -FDT_ERR_BADSTRUCTURE; + + return offset; /* error from fdt_next_node() */ +} + +int fdt_supernode_atdepth_offset(const void *fdt, int nodeoffset, + int supernodedepth, int *nodedepth) +{ + int offset, depth; + int supernodeoffset = -FDT_ERR_INTERNAL; + + FDT_RO_PROBE(fdt); + + if (supernodedepth < 0) + return -FDT_ERR_NOTFOUND; + + for (offset = 0, depth = 0; + (offset >= 0) && (offset <= nodeoffset); + offset = fdt_next_node(fdt, offset, &depth)) { + if (depth == supernodedepth) + supernodeoffset = offset; + + if (offset == nodeoffset) { + if (nodedepth) + *nodedepth = depth; + + if (supernodedepth > depth) + return -FDT_ERR_NOTFOUND; + else + return supernodeoffset; + } + } + + if ((offset == -FDT_ERR_NOTFOUND) || (offset >= 0)) + return -FDT_ERR_BADOFFSET; + else if (offset == -FDT_ERR_BADOFFSET) + return -FDT_ERR_BADSTRUCTURE; + + return offset; /* error from fdt_next_node() */ +} + +int fdt_node_depth(const void *fdt, int nodeoffset) +{ + int nodedepth; + int err; + + err = fdt_supernode_atdepth_offset(fdt, nodeoffset, 0, &nodedepth); + if (err) + return (err < 0) ? err : -FDT_ERR_INTERNAL; + return nodedepth; +} + +int fdt_parent_offset(const void *fdt, int nodeoffset) +{ + int nodedepth = fdt_node_depth(fdt, nodeoffset); + + if (nodedepth < 0) + return nodedepth; + return fdt_supernode_atdepth_offset(fdt, nodeoffset, + nodedepth - 1, NULL); +} + +int fdt_node_offset_by_prop_value(const void *fdt, int startoffset, + const char *propname, + const void *propval, int proplen) +{ + int offset; + const void *val; + int len; + + FDT_RO_PROBE(fdt); + + /* FIXME: The algorithm here is pretty horrible: we scan each + * property of a node in fdt_getprop(), then if that didn't + * find what we want, we scan over them again making our way + * to the next node. Still it's the easiest to implement + * approach; performance can come later. */ + for (offset = fdt_next_node(fdt, startoffset, NULL); + offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + val = fdt_getprop(fdt, offset, propname, &len); + if (val && (len == proplen) + && (memcmp(val, propval, len) == 0)) + return offset; + } + + return offset; /* error from fdt_next_node() */ +} + +int fdt_node_offset_by_phandle(const void *fdt, uint32_t phandle) +{ + int offset; + + if ((phandle == 0) || (phandle == -1)) + return -FDT_ERR_BADPHANDLE; + + FDT_RO_PROBE(fdt); + + /* FIXME: The algorithm here is pretty horrible: we + * potentially scan each property of a node in + * fdt_get_phandle(), then if that didn't find what + * we want, we scan over them again making our way to the next + * node. Still it's the easiest to implement approach; + * performance can come later. */ + for (offset = fdt_next_node(fdt, -1, NULL); + offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + if (fdt_get_phandle(fdt, offset) == phandle) + return offset; + } + + return offset; /* error from fdt_next_node() */ +} + +int fdt_stringlist_contains(const char *strlist, int listlen, const char *str) +{ + int len = strlen(str); + const char *p; + + while (listlen >= len) { + if (memcmp(str, strlist, len+1) == 0) + return 1; + p = memchr(strlist, '\0', listlen); + if (!p) + return 0; /* malformed strlist.. */ + listlen -= (p-strlist) + 1; + strlist = p + 1; + } + return 0; +} + +int fdt_stringlist_count(const void *fdt, int nodeoffset, const char *property) +{ + const char *list, *end; + int length, count = 0; + + list = fdt_getprop(fdt, nodeoffset, property, &length); + if (!list) + return length; + + end = list + length; + + while (list < end) { + length = strnlen(list, end - list) + 1; + + /* Abort if the last string isn't properly NUL-terminated. */ + if (list + length > end) + return -FDT_ERR_BADVALUE; + + list += length; + count++; + } + + return count; +} + +int fdt_stringlist_search(const void *fdt, int nodeoffset, const char *property, + const char *string) +{ + int length, len, idx = 0; + const char *list, *end; + + list = fdt_getprop(fdt, nodeoffset, property, &length); + if (!list) + return length; + + len = strlen(string) + 1; + end = list + length; + + while (list < end) { + length = strnlen(list, end - list) + 1; + + /* Abort if the last string isn't properly NUL-terminated. */ + if (list + length > end) + return -FDT_ERR_BADVALUE; + + if (length == len && memcmp(list, string, length) == 0) + return idx; + + list += length; + idx++; + } + + return -FDT_ERR_NOTFOUND; +} + +const char *fdt_stringlist_get(const void *fdt, int nodeoffset, + const char *property, int idx, + int *lenp) +{ + const char *list, *end; + int length; + + list = fdt_getprop(fdt, nodeoffset, property, &length); + if (!list) { + if (lenp) + *lenp = length; + + return NULL; + } + + end = list + length; + + while (list < end) { + length = strnlen(list, end - list) + 1; + + /* Abort if the last string isn't properly NUL-terminated. */ + if (list + length > end) { + if (lenp) + *lenp = -FDT_ERR_BADVALUE; + + return NULL; + } + + if (idx == 0) { + if (lenp) + *lenp = length - 1; + + return list; + } + + list += length; + idx--; + } + + if (lenp) + *lenp = -FDT_ERR_NOTFOUND; + + return NULL; +} + +int fdt_node_check_compatible(const void *fdt, int nodeoffset, + const char *compatible) +{ + const void *prop; + int len; + + prop = fdt_getprop(fdt, nodeoffset, "compatible", &len); + if (!prop) + return len; + + return !fdt_stringlist_contains(prop, len, compatible); +} + +int fdt_node_offset_by_compatible(const void *fdt, int startoffset, + const char *compatible) +{ + int offset, err; + + FDT_RO_PROBE(fdt); + + /* FIXME: The algorithm here is pretty horrible: we scan each + * property of a node in fdt_node_check_compatible(), then if + * that didn't find what we want, we scan over them again + * making our way to the next node. Still it's the easiest to + * implement approach; performance can come later. */ + for (offset = fdt_next_node(fdt, startoffset, NULL); + offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + err = fdt_node_check_compatible(fdt, offset, compatible); + if ((err < 0) && (err != -FDT_ERR_NOTFOUND)) + return err; + else if (err == 0) + return offset; + } + + return offset; /* error from fdt_next_node() */ +} + +int fdt_check_full(const void *fdt, size_t bufsize) +{ + int err; + int num_memrsv; + int offset, nextoffset = 0; + uint32_t tag; + unsigned depth = 0; + const void *prop; + const char *propname; + + if (bufsize < FDT_V1_SIZE) + return -FDT_ERR_TRUNCATED; + err = fdt_check_header(fdt); + if (err != 0) + return err; + if (bufsize < fdt_totalsize(fdt)) + return -FDT_ERR_TRUNCATED; + + num_memrsv = fdt_num_mem_rsv(fdt); + if (num_memrsv < 0) + return num_memrsv; + + while (1) { + offset = nextoffset; + tag = fdt_next_tag(fdt, offset, &nextoffset); + + if (nextoffset < 0) + return nextoffset; + + switch (tag) { + case FDT_NOP: + break; + + case FDT_END: + if (depth != 0) + return -FDT_ERR_BADSTRUCTURE; + return 0; + + case FDT_BEGIN_NODE: + depth++; + if (depth > INT_MAX) + return -FDT_ERR_BADSTRUCTURE; + break; + + case FDT_END_NODE: + if (depth == 0) + return -FDT_ERR_BADSTRUCTURE; + depth--; + break; + + case FDT_PROP: + prop = fdt_getprop_by_offset(fdt, offset, &propname, + &err); + if (!prop) + return err; + break; + + default: + return -FDT_ERR_INTERNAL; + } + } +} diff --git a/optee_os/core/lib/libfdt/fdt_rw.c b/optee_os/core/lib/libfdt/fdt_rw.c new file mode 100644 index 0000000..8795947 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_rw.c @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +static int fdt_blocks_misordered_(const void *fdt, + int mem_rsv_size, int struct_size) +{ + return (fdt_off_mem_rsvmap(fdt) < FDT_ALIGN(sizeof(struct fdt_header), 8)) + || (fdt_off_dt_struct(fdt) < + (fdt_off_mem_rsvmap(fdt) + mem_rsv_size)) + || (fdt_off_dt_strings(fdt) < + (fdt_off_dt_struct(fdt) + struct_size)) + || (fdt_totalsize(fdt) < + (fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt))); +} + +static int fdt_rw_probe_(void *fdt) +{ + FDT_RO_PROBE(fdt); + + if (fdt_version(fdt) < 17) + return -FDT_ERR_BADVERSION; + if (fdt_blocks_misordered_(fdt, sizeof(struct fdt_reserve_entry), + fdt_size_dt_struct(fdt))) + return -FDT_ERR_BADLAYOUT; + if (fdt_version(fdt) > 17) + fdt_set_version(fdt, 17); + + return 0; +} + +#define FDT_RW_PROBE(fdt) \ + { \ + int err_; \ + if ((err_ = fdt_rw_probe_(fdt)) != 0) \ + return err_; \ + } + +static inline int fdt_data_size_(void *fdt) +{ + return fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt); +} + +static int fdt_splice_(void *fdt, void *splicepoint, int oldlen, int newlen) +{ + char *p = splicepoint; + char *end = (char *)fdt + fdt_data_size_(fdt); + + if (((p + oldlen) < p) || ((p + oldlen) > end)) + return -FDT_ERR_BADOFFSET; + if ((p < (char *)fdt) || ((end - oldlen + newlen) < (char *)fdt)) + return -FDT_ERR_BADOFFSET; + if ((end - oldlen + newlen) > ((char *)fdt + fdt_totalsize(fdt))) + return -FDT_ERR_NOSPACE; + memmove(p + newlen, p + oldlen, end - p - oldlen); + return 0; +} + +static int fdt_splice_mem_rsv_(void *fdt, struct fdt_reserve_entry *p, + int oldn, int newn) +{ + int delta = (newn - oldn) * sizeof(*p); + int err; + err = fdt_splice_(fdt, p, oldn * sizeof(*p), newn * sizeof(*p)); + if (err) + return err; + fdt_set_off_dt_struct(fdt, fdt_off_dt_struct(fdt) + delta); + fdt_set_off_dt_strings(fdt, fdt_off_dt_strings(fdt) + delta); + return 0; +} + +static int fdt_splice_struct_(void *fdt, void *p, + int oldlen, int newlen) +{ + int delta = newlen - oldlen; + int err; + + if ((err = fdt_splice_(fdt, p, oldlen, newlen))) + return err; + + fdt_set_size_dt_struct(fdt, fdt_size_dt_struct(fdt) + delta); + fdt_set_off_dt_strings(fdt, fdt_off_dt_strings(fdt) + delta); + return 0; +} + +/* Must only be used to roll back in case of error */ +static void fdt_del_last_string_(void *fdt, const char *s) +{ + int newlen = strlen(s) + 1; + + fdt_set_size_dt_strings(fdt, fdt_size_dt_strings(fdt) - newlen); +} + +static int fdt_splice_string_(void *fdt, int newlen) +{ + void *p = (char *)fdt + + fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt); + int err; + + if ((err = fdt_splice_(fdt, p, 0, newlen))) + return err; + + fdt_set_size_dt_strings(fdt, fdt_size_dt_strings(fdt) + newlen); + return 0; +} + +static int fdt_find_add_string_(void *fdt, const char *s, int *allocated) +{ + char *strtab = (char *)fdt + fdt_off_dt_strings(fdt); + const char *p; + char *new; + int len = strlen(s) + 1; + int err; + + *allocated = 0; + + p = fdt_find_string_(strtab, fdt_size_dt_strings(fdt), s); + if (p) + /* found it */ + return (p - strtab); + + new = strtab + fdt_size_dt_strings(fdt); + err = fdt_splice_string_(fdt, len); + if (err) + return err; + + *allocated = 1; + + memcpy(new, s, len); + return (new - strtab); +} + +int fdt_add_mem_rsv(void *fdt, uint64_t address, uint64_t size) +{ + struct fdt_reserve_entry *re; + int err; + + FDT_RW_PROBE(fdt); + + re = fdt_mem_rsv_w_(fdt, fdt_num_mem_rsv(fdt)); + err = fdt_splice_mem_rsv_(fdt, re, 0, 1); + if (err) + return err; + + re->address = cpu_to_fdt64(address); + re->size = cpu_to_fdt64(size); + return 0; +} + +int fdt_del_mem_rsv(void *fdt, int n) +{ + struct fdt_reserve_entry *re = fdt_mem_rsv_w_(fdt, n); + + FDT_RW_PROBE(fdt); + + if (n >= fdt_num_mem_rsv(fdt)) + return -FDT_ERR_NOTFOUND; + + return fdt_splice_mem_rsv_(fdt, re, 1, 0); +} + +static int fdt_resize_property_(void *fdt, int nodeoffset, const char *name, + int len, struct fdt_property **prop) +{ + int oldlen; + int err; + + *prop = fdt_get_property_w(fdt, nodeoffset, name, &oldlen); + if (!*prop) + return oldlen; + + if ((err = fdt_splice_struct_(fdt, (*prop)->data, FDT_TAGALIGN(oldlen), + FDT_TAGALIGN(len)))) + return err; + + (*prop)->len = cpu_to_fdt32(len); + return 0; +} + +static int fdt_add_property_(void *fdt, int nodeoffset, const char *name, + int len, struct fdt_property **prop) +{ + int proplen; + int nextoffset; + int namestroff; + int err; + int allocated; + + if ((nextoffset = fdt_check_node_offset_(fdt, nodeoffset)) < 0) + return nextoffset; + + namestroff = fdt_find_add_string_(fdt, name, &allocated); + if (namestroff < 0) + return namestroff; + + *prop = fdt_offset_ptr_w_(fdt, nextoffset); + proplen = sizeof(**prop) + FDT_TAGALIGN(len); + + err = fdt_splice_struct_(fdt, *prop, 0, proplen); + if (err) { + if (allocated) + fdt_del_last_string_(fdt, name); + return err; + } + + (*prop)->tag = cpu_to_fdt32(FDT_PROP); + (*prop)->nameoff = cpu_to_fdt32(namestroff); + (*prop)->len = cpu_to_fdt32(len); + return 0; +} + +int fdt_set_name(void *fdt, int nodeoffset, const char *name) +{ + char *namep; + int oldlen, newlen; + int err; + + FDT_RW_PROBE(fdt); + + namep = (char *)(uintptr_t)fdt_get_name(fdt, nodeoffset, &oldlen); + if (!namep) + return oldlen; + + newlen = strlen(name); + + err = fdt_splice_struct_(fdt, namep, FDT_TAGALIGN(oldlen+1), + FDT_TAGALIGN(newlen+1)); + if (err) + return err; + + memcpy(namep, name, newlen+1); + return 0; +} + +int fdt_setprop_placeholder(void *fdt, int nodeoffset, const char *name, + int len, void **prop_data) +{ + struct fdt_property *prop; + int err; + + FDT_RW_PROBE(fdt); + + err = fdt_resize_property_(fdt, nodeoffset, name, len, &prop); + if (err == -FDT_ERR_NOTFOUND) + err = fdt_add_property_(fdt, nodeoffset, name, len, &prop); + if (err) + return err; + + *prop_data = prop->data; + return 0; +} + +int fdt_setprop(void *fdt, int nodeoffset, const char *name, + const void *val, int len) +{ + void *prop_data; + int err; + + err = fdt_setprop_placeholder(fdt, nodeoffset, name, len, &prop_data); + if (err) + return err; + + if (len) + memcpy(prop_data, val, len); + return 0; +} + +int fdt_appendprop(void *fdt, int nodeoffset, const char *name, + const void *val, int len) +{ + struct fdt_property *prop; + int err, oldlen, newlen; + + FDT_RW_PROBE(fdt); + + prop = fdt_get_property_w(fdt, nodeoffset, name, &oldlen); + if (prop) { + newlen = len + oldlen; + err = fdt_splice_struct_(fdt, prop->data, + FDT_TAGALIGN(oldlen), + FDT_TAGALIGN(newlen)); + if (err) + return err; + prop->len = cpu_to_fdt32(newlen); + memcpy(prop->data + oldlen, val, len); + } else { + err = fdt_add_property_(fdt, nodeoffset, name, len, &prop); + if (err) + return err; + memcpy(prop->data, val, len); + } + return 0; +} + +int fdt_delprop(void *fdt, int nodeoffset, const char *name) +{ + struct fdt_property *prop; + int len, proplen; + + FDT_RW_PROBE(fdt); + + prop = fdt_get_property_w(fdt, nodeoffset, name, &len); + if (!prop) + return len; + + proplen = sizeof(*prop) + FDT_TAGALIGN(len); + return fdt_splice_struct_(fdt, prop, proplen, 0); +} + +int fdt_add_subnode_namelen(void *fdt, int parentoffset, + const char *name, int namelen) +{ + struct fdt_node_header *nh; + int offset, nextoffset; + int nodelen; + int err; + uint32_t tag; + fdt32_t *endtag; + + FDT_RW_PROBE(fdt); + + offset = fdt_subnode_offset_namelen(fdt, parentoffset, name, namelen); + if (offset >= 0) + return -FDT_ERR_EXISTS; + else if (offset != -FDT_ERR_NOTFOUND) + return offset; + + /* Try to place the new node after the parent's properties */ + fdt_next_tag(fdt, parentoffset, &nextoffset); /* skip the BEGIN_NODE */ + do { + offset = nextoffset; + tag = fdt_next_tag(fdt, offset, &nextoffset); + } while ((tag == FDT_PROP) || (tag == FDT_NOP)); + + nh = fdt_offset_ptr_w_(fdt, offset); + nodelen = sizeof(*nh) + FDT_TAGALIGN(namelen+1) + FDT_TAGSIZE; + + err = fdt_splice_struct_(fdt, nh, 0, nodelen); + if (err) + return err; + + nh->tag = cpu_to_fdt32(FDT_BEGIN_NODE); + memset(nh->name, 0, FDT_TAGALIGN(namelen+1)); + memcpy(nh->name, name, namelen); + endtag = (fdt32_t *)((char *)nh + nodelen - FDT_TAGSIZE); + *endtag = cpu_to_fdt32(FDT_END_NODE); + + return offset; +} + +int fdt_add_subnode(void *fdt, int parentoffset, const char *name) +{ + return fdt_add_subnode_namelen(fdt, parentoffset, name, strlen(name)); +} + +int fdt_del_node(void *fdt, int nodeoffset) +{ + int endoffset; + + FDT_RW_PROBE(fdt); + + endoffset = fdt_node_end_offset_(fdt, nodeoffset); + if (endoffset < 0) + return endoffset; + + return fdt_splice_struct_(fdt, fdt_offset_ptr_w_(fdt, nodeoffset), + endoffset - nodeoffset, 0); +} + +static void fdt_packblocks_(const char *old, char *new, + int mem_rsv_size, int struct_size) +{ + int mem_rsv_off, struct_off, strings_off; + + mem_rsv_off = FDT_ALIGN(sizeof(struct fdt_header), 8); + struct_off = mem_rsv_off + mem_rsv_size; + strings_off = struct_off + struct_size; + + memmove(new + mem_rsv_off, old + fdt_off_mem_rsvmap(old), mem_rsv_size); + fdt_set_off_mem_rsvmap(new, mem_rsv_off); + + memmove(new + struct_off, old + fdt_off_dt_struct(old), struct_size); + fdt_set_off_dt_struct(new, struct_off); + fdt_set_size_dt_struct(new, struct_size); + + memmove(new + strings_off, old + fdt_off_dt_strings(old), + fdt_size_dt_strings(old)); + fdt_set_off_dt_strings(new, strings_off); + fdt_set_size_dt_strings(new, fdt_size_dt_strings(old)); +} + +int fdt_open_into(const void *fdt, void *buf, int bufsize) +{ + int err; + int mem_rsv_size, struct_size; + int newsize; + const char *fdtstart = fdt; + const char *fdtend = fdtstart + fdt_totalsize(fdt); + char *tmp; + + FDT_RO_PROBE(fdt); + + mem_rsv_size = (fdt_num_mem_rsv(fdt)+1) + * sizeof(struct fdt_reserve_entry); + + if (fdt_version(fdt) >= 17) { + struct_size = fdt_size_dt_struct(fdt); + } else { + struct_size = 0; + while (fdt_next_tag(fdt, struct_size, &struct_size) != FDT_END) + ; + if (struct_size < 0) + return struct_size; + } + + if (!fdt_blocks_misordered_(fdt, mem_rsv_size, struct_size)) { + /* no further work necessary */ + err = fdt_move(fdt, buf, bufsize); + if (err) + return err; + fdt_set_version(buf, 17); + fdt_set_size_dt_struct(buf, struct_size); + fdt_set_totalsize(buf, bufsize); + return 0; + } + + /* Need to reorder */ + newsize = FDT_ALIGN(sizeof(struct fdt_header), 8) + mem_rsv_size + + struct_size + fdt_size_dt_strings(fdt); + + if (bufsize < newsize) + return -FDT_ERR_NOSPACE; + + /* First attempt to build converted tree at beginning of buffer */ + tmp = buf; + /* But if that overlaps with the old tree... */ + if (((tmp + newsize) > fdtstart) && (tmp < fdtend)) { + /* Try right after the old tree instead */ + tmp = (char *)(uintptr_t)fdtend; + if ((tmp + newsize) > ((char *)buf + bufsize)) + return -FDT_ERR_NOSPACE; + } + + fdt_packblocks_(fdt, tmp, mem_rsv_size, struct_size); + memmove(buf, tmp, newsize); + + fdt_set_magic(buf, FDT_MAGIC); + fdt_set_totalsize(buf, bufsize); + fdt_set_version(buf, 17); + fdt_set_last_comp_version(buf, 16); + fdt_set_boot_cpuid_phys(buf, fdt_boot_cpuid_phys(fdt)); + + return 0; +} + +int fdt_pack(void *fdt) +{ + int mem_rsv_size; + + FDT_RW_PROBE(fdt); + + mem_rsv_size = (fdt_num_mem_rsv(fdt)+1) + * sizeof(struct fdt_reserve_entry); + fdt_packblocks_(fdt, fdt, mem_rsv_size, fdt_size_dt_struct(fdt)); + fdt_set_totalsize(fdt, fdt_data_size_(fdt)); + + return 0; +} diff --git a/optee_os/core/lib/libfdt/fdt_strerror.c b/optee_os/core/lib/libfdt/fdt_strerror.c new file mode 100644 index 0000000..768db66 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_strerror.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +struct fdt_errtabent { + const char *str; +}; + +#define FDT_ERRTABENT(val) \ + [(val)] = { .str = #val, } + +static struct fdt_errtabent fdt_errtable[] = { + FDT_ERRTABENT(FDT_ERR_NOTFOUND), + FDT_ERRTABENT(FDT_ERR_EXISTS), + FDT_ERRTABENT(FDT_ERR_NOSPACE), + + FDT_ERRTABENT(FDT_ERR_BADOFFSET), + FDT_ERRTABENT(FDT_ERR_BADPATH), + FDT_ERRTABENT(FDT_ERR_BADPHANDLE), + FDT_ERRTABENT(FDT_ERR_BADSTATE), + + FDT_ERRTABENT(FDT_ERR_TRUNCATED), + FDT_ERRTABENT(FDT_ERR_BADMAGIC), + FDT_ERRTABENT(FDT_ERR_BADVERSION), + FDT_ERRTABENT(FDT_ERR_BADSTRUCTURE), + FDT_ERRTABENT(FDT_ERR_BADLAYOUT), + FDT_ERRTABENT(FDT_ERR_INTERNAL), + FDT_ERRTABENT(FDT_ERR_BADNCELLS), + FDT_ERRTABENT(FDT_ERR_BADVALUE), + FDT_ERRTABENT(FDT_ERR_BADOVERLAY), + FDT_ERRTABENT(FDT_ERR_NOPHANDLES), + FDT_ERRTABENT(FDT_ERR_BADFLAGS), +}; +#define FDT_ERRTABSIZE (sizeof(fdt_errtable) / sizeof(fdt_errtable[0])) + +const char *fdt_strerror(int errval) +{ + if (errval > 0) + return ""; + else if (errval == 0) + return ""; + else if (errval > -FDT_ERRTABSIZE) { + const char *s = fdt_errtable[-errval].str; + + if (s) + return s; + } + + return ""; +} diff --git a/optee_os/core/lib/libfdt/fdt_sw.c b/optee_os/core/lib/libfdt/fdt_sw.c new file mode 100644 index 0000000..76bea22 --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_sw.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +static int fdt_sw_probe_(void *fdt) +{ + if (fdt_magic(fdt) == FDT_MAGIC) + return -FDT_ERR_BADSTATE; + else if (fdt_magic(fdt) != FDT_SW_MAGIC) + return -FDT_ERR_BADMAGIC; + return 0; +} + +#define FDT_SW_PROBE(fdt) \ + { \ + int err; \ + if ((err = fdt_sw_probe_(fdt)) != 0) \ + return err; \ + } + +/* 'memrsv' state: Initial state after fdt_create() + * + * Allowed functions: + * fdt_add_reservmap_entry() + * fdt_finish_reservemap() [moves to 'struct' state] + */ +static int fdt_sw_probe_memrsv_(void *fdt) +{ + int err = fdt_sw_probe_(fdt); + if (err) + return err; + + if (fdt_off_dt_strings(fdt) != 0) + return -FDT_ERR_BADSTATE; + return 0; +} + +#define FDT_SW_PROBE_MEMRSV(fdt) \ + { \ + int err; \ + if ((err = fdt_sw_probe_memrsv_(fdt)) != 0) \ + return err; \ + } + +/* 'struct' state: Enter this state after fdt_finish_reservemap() + * + * Allowed functions: + * fdt_begin_node() + * fdt_end_node() + * fdt_property*() + * fdt_finish() [moves to 'complete' state] + */ +static int fdt_sw_probe_struct_(void *fdt) +{ + int err = fdt_sw_probe_(fdt); + if (err) + return err; + + if (fdt_off_dt_strings(fdt) != fdt_totalsize(fdt)) + return -FDT_ERR_BADSTATE; + return 0; +} + +#define FDT_SW_PROBE_STRUCT(fdt) \ + { \ + int err; \ + if ((err = fdt_sw_probe_struct_(fdt)) != 0) \ + return err; \ + } + +static inline uint32_t sw_flags(void *fdt) +{ + /* assert: (fdt_magic(fdt) == FDT_SW_MAGIC) */ + return fdt_last_comp_version(fdt); +} + +/* 'complete' state: Enter this state after fdt_finish() + * + * Allowed functions: none + */ + +static void *fdt_grab_space_(void *fdt, size_t len) +{ + int offset = fdt_size_dt_struct(fdt); + int spaceleft; + + spaceleft = fdt_totalsize(fdt) - fdt_off_dt_struct(fdt) + - fdt_size_dt_strings(fdt); + + if ((offset + len < offset) || (offset + len > spaceleft)) + return NULL; + + fdt_set_size_dt_struct(fdt, offset + len); + return fdt_offset_ptr_w_(fdt, offset); +} + +int fdt_create_with_flags(void *buf, int bufsize, uint32_t flags) +{ + const size_t hdrsize = FDT_ALIGN(sizeof(struct fdt_header), + sizeof(struct fdt_reserve_entry)); + void *fdt = buf; + + if (bufsize < hdrsize) + return -FDT_ERR_NOSPACE; + + if (flags & ~FDT_CREATE_FLAGS_ALL) + return -FDT_ERR_BADFLAGS; + + memset(buf, 0, bufsize); + + /* + * magic and last_comp_version keep intermediate state during the fdt + * creation process, which is replaced with the proper FDT format by + * fdt_finish(). + * + * flags should be accessed with sw_flags(). + */ + fdt_set_magic(fdt, FDT_SW_MAGIC); + fdt_set_version(fdt, FDT_LAST_SUPPORTED_VERSION); + fdt_set_last_comp_version(fdt, flags); + + fdt_set_totalsize(fdt, bufsize); + + fdt_set_off_mem_rsvmap(fdt, hdrsize); + fdt_set_off_dt_struct(fdt, fdt_off_mem_rsvmap(fdt)); + fdt_set_off_dt_strings(fdt, 0); + + return 0; +} + +int fdt_create(void *buf, int bufsize) +{ + return fdt_create_with_flags(buf, bufsize, 0); +} + +int fdt_resize(void *fdt, void *buf, int bufsize) +{ + size_t headsize, tailsize; + char *oldtail, *newtail; + + FDT_SW_PROBE(fdt); + + headsize = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt); + tailsize = fdt_size_dt_strings(fdt); + + if ((headsize + tailsize) > fdt_totalsize(fdt)) + return -FDT_ERR_INTERNAL; + + if ((headsize + tailsize) > bufsize) + return -FDT_ERR_NOSPACE; + + oldtail = (char *)fdt + fdt_totalsize(fdt) - tailsize; + newtail = (char *)buf + bufsize - tailsize; + + /* Two cases to avoid clobbering data if the old and new + * buffers partially overlap */ + if (buf <= fdt) { + memmove(buf, fdt, headsize); + memmove(newtail, oldtail, tailsize); + } else { + memmove(newtail, oldtail, tailsize); + memmove(buf, fdt, headsize); + } + + fdt_set_totalsize(buf, bufsize); + if (fdt_off_dt_strings(buf)) + fdt_set_off_dt_strings(buf, bufsize); + + return 0; +} + +int fdt_add_reservemap_entry(void *fdt, uint64_t addr, uint64_t size) +{ + struct fdt_reserve_entry *re; + int offset; + + FDT_SW_PROBE_MEMRSV(fdt); + + offset = fdt_off_dt_struct(fdt); + if ((offset + sizeof(*re)) > fdt_totalsize(fdt)) + return -FDT_ERR_NOSPACE; + + re = (struct fdt_reserve_entry *)((char *)fdt + offset); + re->address = cpu_to_fdt64(addr); + re->size = cpu_to_fdt64(size); + + fdt_set_off_dt_struct(fdt, offset + sizeof(*re)); + + return 0; +} + +int fdt_finish_reservemap(void *fdt) +{ + int err = fdt_add_reservemap_entry(fdt, 0, 0); + + if (err) + return err; + + fdt_set_off_dt_strings(fdt, fdt_totalsize(fdt)); + return 0; +} + +int fdt_begin_node(void *fdt, const char *name) +{ + struct fdt_node_header *nh; + int namelen; + + FDT_SW_PROBE_STRUCT(fdt); + + namelen = strlen(name) + 1; + nh = fdt_grab_space_(fdt, sizeof(*nh) + FDT_TAGALIGN(namelen)); + if (! nh) + return -FDT_ERR_NOSPACE; + + nh->tag = cpu_to_fdt32(FDT_BEGIN_NODE); + memcpy(nh->name, name, namelen); + return 0; +} + +int fdt_end_node(void *fdt) +{ + fdt32_t *en; + + FDT_SW_PROBE_STRUCT(fdt); + + en = fdt_grab_space_(fdt, FDT_TAGSIZE); + if (! en) + return -FDT_ERR_NOSPACE; + + *en = cpu_to_fdt32(FDT_END_NODE); + return 0; +} + +static int fdt_add_string_(void *fdt, const char *s) +{ + char *strtab = (char *)fdt + fdt_totalsize(fdt); + int strtabsize = fdt_size_dt_strings(fdt); + int len = strlen(s) + 1; + int struct_top, offset; + + offset = -strtabsize - len; + struct_top = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt); + if (fdt_totalsize(fdt) + offset < struct_top) + return 0; /* no more room :( */ + + memcpy(strtab + offset, s, len); + fdt_set_size_dt_strings(fdt, strtabsize + len); + return offset; +} + +/* Must only be used to roll back in case of error */ +static void fdt_del_last_string_(void *fdt, const char *s) +{ + int strtabsize = fdt_size_dt_strings(fdt); + int len = strlen(s) + 1; + + fdt_set_size_dt_strings(fdt, strtabsize - len); +} + +static int fdt_find_add_string_(void *fdt, const char *s, int *allocated) +{ + char *strtab = (char *)fdt + fdt_totalsize(fdt); + int strtabsize = fdt_size_dt_strings(fdt); + const char *p; + + *allocated = 0; + + p = fdt_find_string_(strtab - strtabsize, strtabsize, s); + if (p) + return p - strtab; + + *allocated = 1; + + return fdt_add_string_(fdt, s); +} + +int fdt_property_placeholder(void *fdt, const char *name, int len, void **valp) +{ + struct fdt_property *prop; + int nameoff; + int allocated; + + FDT_SW_PROBE_STRUCT(fdt); + + /* String de-duplication can be slow, _NO_NAME_DEDUP skips it */ + if (sw_flags(fdt) & FDT_CREATE_FLAG_NO_NAME_DEDUP) { + allocated = 1; + nameoff = fdt_add_string_(fdt, name); + } else { + nameoff = fdt_find_add_string_(fdt, name, &allocated); + } + if (nameoff == 0) + return -FDT_ERR_NOSPACE; + + prop = fdt_grab_space_(fdt, sizeof(*prop) + FDT_TAGALIGN(len)); + if (! prop) { + if (allocated) + fdt_del_last_string_(fdt, name); + return -FDT_ERR_NOSPACE; + } + + prop->tag = cpu_to_fdt32(FDT_PROP); + prop->nameoff = cpu_to_fdt32(nameoff); + prop->len = cpu_to_fdt32(len); + *valp = prop->data; + return 0; +} + +int fdt_property(void *fdt, const char *name, const void *val, int len) +{ + void *ptr; + int ret; + + ret = fdt_property_placeholder(fdt, name, len, &ptr); + if (ret) + return ret; + memcpy(ptr, val, len); + return 0; +} + +int fdt_finish(void *fdt) +{ + char *p = (char *)fdt; + fdt32_t *end; + int oldstroffset, newstroffset; + uint32_t tag; + int offset, nextoffset; + + FDT_SW_PROBE_STRUCT(fdt); + + /* Add terminator */ + end = fdt_grab_space_(fdt, sizeof(*end)); + if (! end) + return -FDT_ERR_NOSPACE; + *end = cpu_to_fdt32(FDT_END); + + /* Relocate the string table */ + oldstroffset = fdt_totalsize(fdt) - fdt_size_dt_strings(fdt); + newstroffset = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt); + memmove(p + newstroffset, p + oldstroffset, fdt_size_dt_strings(fdt)); + fdt_set_off_dt_strings(fdt, newstroffset); + + /* Walk the structure, correcting string offsets */ + offset = 0; + while ((tag = fdt_next_tag(fdt, offset, &nextoffset)) != FDT_END) { + if (tag == FDT_PROP) { + struct fdt_property *prop = + fdt_offset_ptr_w_(fdt, offset); + int nameoff; + + nameoff = fdt32_to_cpu(prop->nameoff); + nameoff += fdt_size_dt_strings(fdt); + prop->nameoff = cpu_to_fdt32(nameoff); + } + offset = nextoffset; + } + if (nextoffset < 0) + return nextoffset; + + /* Finally, adjust the header */ + fdt_set_totalsize(fdt, newstroffset + fdt_size_dt_strings(fdt)); + + /* And fix up fields that were keeping intermediate state. */ + fdt_set_last_comp_version(fdt, FDT_FIRST_SUPPORTED_VERSION); + fdt_set_magic(fdt, FDT_MAGIC); + + return 0; +} diff --git a/optee_os/core/lib/libfdt/fdt_wip.c b/optee_os/core/lib/libfdt/fdt_wip.c new file mode 100644 index 0000000..f64139e --- /dev/null +++ b/optee_os/core/lib/libfdt/fdt_wip.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include +#include + +#include "libfdt_internal.h" + +int fdt_setprop_inplace_namelen_partial(void *fdt, int nodeoffset, + const char *name, int namelen, + uint32_t idx, const void *val, + int len) +{ + void *propval; + int proplen; + + propval = fdt_getprop_namelen_w(fdt, nodeoffset, name, namelen, + &proplen); + if (!propval) + return proplen; + + if (proplen < (len + idx)) + return -FDT_ERR_NOSPACE; + + memcpy((char *)propval + idx, val, len); + return 0; +} + +int fdt_setprop_inplace(void *fdt, int nodeoffset, const char *name, + const void *val, int len) +{ + const void *propval; + int proplen; + + propval = fdt_getprop(fdt, nodeoffset, name, &proplen); + if (!propval) + return proplen; + + if (proplen != len) + return -FDT_ERR_NOSPACE; + + return fdt_setprop_inplace_namelen_partial(fdt, nodeoffset, name, + strlen(name), 0, + val, len); +} + +static void fdt_nop_region_(void *start, int len) +{ + fdt32_t *p; + + for (p = start; (char *)p < ((char *)start + len); p++) + *p = cpu_to_fdt32(FDT_NOP); +} + +int fdt_nop_property(void *fdt, int nodeoffset, const char *name) +{ + struct fdt_property *prop; + int len; + + prop = fdt_get_property_w(fdt, nodeoffset, name, &len); + if (!prop) + return len; + + fdt_nop_region_(prop, len + sizeof(*prop)); + + return 0; +} + +int fdt_node_end_offset_(void *fdt, int offset) +{ + int depth = 0; + + while ((offset >= 0) && (depth >= 0)) + offset = fdt_next_node(fdt, offset, &depth); + + return offset; +} + +int fdt_nop_node(void *fdt, int nodeoffset) +{ + int endoffset; + + endoffset = fdt_node_end_offset_(fdt, nodeoffset); + if (endoffset < 0) + return endoffset; + + fdt_nop_region_(fdt_offset_ptr_w(fdt, nodeoffset, 0), + endoffset - nodeoffset); + return 0; +} diff --git a/optee_os/core/lib/libfdt/include/fdt.h b/optee_os/core/lib/libfdt/include/fdt.h new file mode 100644 index 0000000..0c91aa7 --- /dev/null +++ b/optee_os/core/lib/libfdt/include/fdt.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +#ifndef FDT_H +#define FDT_H +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + * Copyright 2012 Kim Phillips, Freescale Semiconductor. + */ + +#ifndef __ASSEMBLY__ + +struct fdt_header { + fdt32_t magic; /* magic word FDT_MAGIC */ + fdt32_t totalsize; /* total size of DT block */ + fdt32_t off_dt_struct; /* offset to structure */ + fdt32_t off_dt_strings; /* offset to strings */ + fdt32_t off_mem_rsvmap; /* offset to memory reserve map */ + fdt32_t version; /* format version */ + fdt32_t last_comp_version; /* last compatible version */ + + /* version 2 fields below */ + fdt32_t boot_cpuid_phys; /* Which physical CPU id we're + booting on */ + /* version 3 fields below */ + fdt32_t size_dt_strings; /* size of the strings block */ + + /* version 17 fields below */ + fdt32_t size_dt_struct; /* size of the structure block */ +}; + +struct fdt_reserve_entry { + fdt64_t address; + fdt64_t size; +}; + +struct fdt_node_header { + fdt32_t tag; + char name[]; +}; + +struct fdt_property { + fdt32_t tag; + fdt32_t len; + fdt32_t nameoff; + char data[]; +}; + +#endif /* !__ASSEMBLY */ + +#define FDT_MAGIC 0xd00dfeed /* 4: version, 4: total size */ +#define FDT_TAGSIZE sizeof(fdt32_t) + +#define FDT_BEGIN_NODE 0x1 /* Start node: full name */ +#define FDT_END_NODE 0x2 /* End node */ +#define FDT_PROP 0x3 /* Property: name off, + size, content */ +#define FDT_NOP 0x4 /* nop */ +#define FDT_END 0x9 + +#define FDT_V1_SIZE (7*sizeof(fdt32_t)) +#define FDT_V2_SIZE (FDT_V1_SIZE + sizeof(fdt32_t)) +#define FDT_V3_SIZE (FDT_V2_SIZE + sizeof(fdt32_t)) +#define FDT_V16_SIZE FDT_V3_SIZE +#define FDT_V17_SIZE (FDT_V16_SIZE + sizeof(fdt32_t)) + +#endif /* FDT_H */ diff --git a/optee_os/core/lib/libfdt/include/libfdt.h b/optee_os/core/lib/libfdt/include/libfdt.h new file mode 100644 index 0000000..8037f39 --- /dev/null +++ b/optee_os/core/lib/libfdt/include/libfdt.h @@ -0,0 +1,2071 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +#ifndef LIBFDT_H +#define LIBFDT_H +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ + +#include +#include + +#define FDT_FIRST_SUPPORTED_VERSION 0x02 +#define FDT_LAST_SUPPORTED_VERSION 0x11 + +/* Error codes: informative error codes */ +#define FDT_ERR_NOTFOUND 1 + /* FDT_ERR_NOTFOUND: The requested node or property does not exist */ +#define FDT_ERR_EXISTS 2 + /* FDT_ERR_EXISTS: Attempted to create a node or property which + * already exists */ +#define FDT_ERR_NOSPACE 3 + /* FDT_ERR_NOSPACE: Operation needed to expand the device + * tree, but its buffer did not have sufficient space to + * contain the expanded tree. Use fdt_open_into() to move the + * device tree to a buffer with more space. */ + +/* Error codes: codes for bad parameters */ +#define FDT_ERR_BADOFFSET 4 + /* FDT_ERR_BADOFFSET: Function was passed a structure block + * offset which is out-of-bounds, or which points to an + * unsuitable part of the structure for the operation. */ +#define FDT_ERR_BADPATH 5 + /* FDT_ERR_BADPATH: Function was passed a badly formatted path + * (e.g. missing a leading / for a function which requires an + * absolute path) */ +#define FDT_ERR_BADPHANDLE 6 + /* FDT_ERR_BADPHANDLE: Function was passed an invalid phandle. + * This can be caused either by an invalid phandle property + * length, or the phandle value was either 0 or -1, which are + * not permitted. */ +#define FDT_ERR_BADSTATE 7 + /* FDT_ERR_BADSTATE: Function was passed an incomplete device + * tree created by the sequential-write functions, which is + * not sufficiently complete for the requested operation. */ + +/* Error codes: codes for bad device tree blobs */ +#define FDT_ERR_TRUNCATED 8 + /* FDT_ERR_TRUNCATED: FDT or a sub-block is improperly + * terminated (overflows, goes outside allowed bounds, or + * isn't properly terminated). */ +#define FDT_ERR_BADMAGIC 9 + /* FDT_ERR_BADMAGIC: Given "device tree" appears not to be a + * device tree at all - it is missing the flattened device + * tree magic number. */ +#define FDT_ERR_BADVERSION 10 + /* FDT_ERR_BADVERSION: Given device tree has a version which + * can't be handled by the requested operation. For + * read-write functions, this may mean that fdt_open_into() is + * required to convert the tree to the expected version. */ +#define FDT_ERR_BADSTRUCTURE 11 + /* FDT_ERR_BADSTRUCTURE: Given device tree has a corrupt + * structure block or other serious error (e.g. misnested + * nodes, or subnodes preceding properties). */ +#define FDT_ERR_BADLAYOUT 12 + /* FDT_ERR_BADLAYOUT: For read-write functions, the given + * device tree has it's sub-blocks in an order that the + * function can't handle (memory reserve map, then structure, + * then strings). Use fdt_open_into() to reorganize the tree + * into a form suitable for the read-write operations. */ + +/* "Can't happen" error indicating a bug in libfdt */ +#define FDT_ERR_INTERNAL 13 + /* FDT_ERR_INTERNAL: libfdt has failed an internal assertion. + * Should never be returned, if it is, it indicates a bug in + * libfdt itself. */ + +/* Errors in device tree content */ +#define FDT_ERR_BADNCELLS 14 + /* FDT_ERR_BADNCELLS: Device tree has a #address-cells, #size-cells + * or similar property with a bad format or value */ + +#define FDT_ERR_BADVALUE 15 + /* FDT_ERR_BADVALUE: Device tree has a property with an unexpected + * value. For example: a property expected to contain a string list + * is not NUL-terminated within the length of its value. */ + +#define FDT_ERR_BADOVERLAY 16 + /* FDT_ERR_BADOVERLAY: The device tree overlay, while + * correctly structured, cannot be applied due to some + * unexpected or missing value, property or node. */ + +#define FDT_ERR_NOPHANDLES 17 + /* FDT_ERR_NOPHANDLES: The device tree doesn't have any + * phandle available anymore without causing an overflow */ + +#define FDT_ERR_BADFLAGS 18 + /* FDT_ERR_BADFLAGS: The function was passed a flags field that + * contains invalid flags or an invalid combination of flags. */ + +#define FDT_ERR_MAX 18 + +/* constants */ +#define FDT_MAX_PHANDLE 0xfffffffe + /* Valid values for phandles range from 1 to 2^32-2. */ + +/**********************************************************************/ +/* Low-level functions (you probably don't need these) */ +/**********************************************************************/ + +#ifndef SWIG /* This function is not useful in Python */ +const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int checklen); +#endif +static inline void *fdt_offset_ptr_w(void *fdt, int offset, int checklen) +{ + return (void *)(uintptr_t)fdt_offset_ptr(fdt, offset, checklen); +} + +uint32_t fdt_next_tag(const void *fdt, int offset, int *nextoffset); + +/* + * Alignment helpers: + * These helpers access words from a device tree blob. They're + * built to work even with unaligned pointers on platforms (ike + * ARM) that don't like unaligned loads and stores + */ + +static inline uint32_t fdt32_ld(const fdt32_t *p) +{ + const uint8_t *bp = (const uint8_t *)p; + + return ((uint32_t)bp[0] << 24) + | ((uint32_t)bp[1] << 16) + | ((uint32_t)bp[2] << 8) + | bp[3]; +} + +static inline void fdt32_st(void *property, uint32_t value) +{ + uint8_t *bp = property; + + bp[0] = value >> 24; + bp[1] = (value >> 16) & 0xff; + bp[2] = (value >> 8) & 0xff; + bp[3] = value & 0xff; +} + +static inline uint64_t fdt64_ld(const fdt64_t *p) +{ + const uint8_t *bp = (const uint8_t *)p; + + return ((uint64_t)bp[0] << 56) + | ((uint64_t)bp[1] << 48) + | ((uint64_t)bp[2] << 40) + | ((uint64_t)bp[3] << 32) + | ((uint64_t)bp[4] << 24) + | ((uint64_t)bp[5] << 16) + | ((uint64_t)bp[6] << 8) + | bp[7]; +} + +static inline void fdt64_st(void *property, uint64_t value) +{ + uint8_t *bp = property; + + bp[0] = value >> 56; + bp[1] = (value >> 48) & 0xff; + bp[2] = (value >> 40) & 0xff; + bp[3] = (value >> 32) & 0xff; + bp[4] = (value >> 24) & 0xff; + bp[5] = (value >> 16) & 0xff; + bp[6] = (value >> 8) & 0xff; + bp[7] = value & 0xff; +} + +/**********************************************************************/ +/* Traversal functions */ +/**********************************************************************/ + +int fdt_next_node(const void *fdt, int offset, int *depth); + +/** + * fdt_first_subnode() - get offset of first direct subnode + * + * @fdt: FDT blob + * @offset: Offset of node to check + * @return offset of first subnode, or -FDT_ERR_NOTFOUND if there is none + */ +int fdt_first_subnode(const void *fdt, int offset); + +/** + * fdt_next_subnode() - get offset of next direct subnode + * + * After first calling fdt_first_subnode(), call this function repeatedly to + * get direct subnodes of a parent node. + * + * @fdt: FDT blob + * @offset: Offset of previous subnode + * @return offset of next subnode, or -FDT_ERR_NOTFOUND if there are no more + * subnodes + */ +int fdt_next_subnode(const void *fdt, int offset); + +/** + * fdt_for_each_subnode - iterate over all subnodes of a parent + * + * @node: child node (int, lvalue) + * @fdt: FDT blob (const void *) + * @parent: parent node (int) + * + * This is actually a wrapper around a for loop and would be used like so: + * + * fdt_for_each_subnode(node, fdt, parent) { + * Use node + * ... + * } + * + * if ((node < 0) && (node != -FDT_ERR_NOTFOUND)) { + * Error handling + * } + * + * Note that this is implemented as a macro and @node is used as + * iterator in the loop. The parent variable be constant or even a + * literal. + * + */ +#define fdt_for_each_subnode(node, fdt, parent) \ + for (node = fdt_first_subnode(fdt, parent); \ + node >= 0; \ + node = fdt_next_subnode(fdt, node)) + +/**********************************************************************/ +/* General functions */ +/**********************************************************************/ +#define fdt_get_header(fdt, field) \ + (fdt32_ld(&((const struct fdt_header *)(fdt))->field)) +#define fdt_magic(fdt) (fdt_get_header(fdt, magic)) +#define fdt_totalsize(fdt) (fdt_get_header(fdt, totalsize)) +#define fdt_off_dt_struct(fdt) (fdt_get_header(fdt, off_dt_struct)) +#define fdt_off_dt_strings(fdt) (fdt_get_header(fdt, off_dt_strings)) +#define fdt_off_mem_rsvmap(fdt) (fdt_get_header(fdt, off_mem_rsvmap)) +#define fdt_version(fdt) (fdt_get_header(fdt, version)) +#define fdt_last_comp_version(fdt) (fdt_get_header(fdt, last_comp_version)) +#define fdt_boot_cpuid_phys(fdt) (fdt_get_header(fdt, boot_cpuid_phys)) +#define fdt_size_dt_strings(fdt) (fdt_get_header(fdt, size_dt_strings)) +#define fdt_size_dt_struct(fdt) (fdt_get_header(fdt, size_dt_struct)) + +#define fdt_set_hdr_(name) \ + static inline void fdt_set_##name(void *fdt, uint32_t val) \ + { \ + struct fdt_header *fdth = (struct fdt_header *)fdt; \ + fdth->name = cpu_to_fdt32(val); \ + } +fdt_set_hdr_(magic); +fdt_set_hdr_(totalsize); +fdt_set_hdr_(off_dt_struct); +fdt_set_hdr_(off_dt_strings); +fdt_set_hdr_(off_mem_rsvmap); +fdt_set_hdr_(version); +fdt_set_hdr_(last_comp_version); +fdt_set_hdr_(boot_cpuid_phys); +fdt_set_hdr_(size_dt_strings); +fdt_set_hdr_(size_dt_struct); +#undef fdt_set_hdr_ + +/** + * fdt_header_size - return the size of the tree's header + * @fdt: pointer to a flattened device tree + */ +size_t fdt_header_size_(uint32_t version); +static inline size_t fdt_header_size(const void *fdt) +{ + return fdt_header_size_(fdt_version(fdt)); +} + +/** + * fdt_check_header - sanity check a device tree header + + * @fdt: pointer to data which might be a flattened device tree + * + * fdt_check_header() checks that the given buffer contains what + * appears to be a flattened device tree, and that the header contains + * valid information (to the extent that can be determined from the + * header alone). + * + * returns: + * 0, if the buffer appears to contain a valid device tree + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_TRUNCATED, standard meanings, as above + */ +int fdt_check_header(const void *fdt); + +/** + * fdt_move - move a device tree around in memory + * @fdt: pointer to the device tree to move + * @buf: pointer to memory where the device is to be moved + * @bufsize: size of the memory space at buf + * + * fdt_move() relocates, if possible, the device tree blob located at + * fdt to the buffer at buf of size bufsize. The buffer may overlap + * with the existing device tree blob at fdt. Therefore, + * fdt_move(fdt, fdt, fdt_totalsize(fdt)) + * should always succeed. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, bufsize is insufficient to contain the device tree + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, standard meanings + */ +int fdt_move(const void *fdt, void *buf, int bufsize); + +/**********************************************************************/ +/* Read-only functions */ +/**********************************************************************/ + +int fdt_check_full(const void *fdt, size_t bufsize); + +/** + * fdt_get_string - retrieve a string from the strings block of a device tree + * @fdt: pointer to the device tree blob + * @stroffset: offset of the string within the strings block (native endian) + * @lenp: optional pointer to return the string's length + * + * fdt_get_string() retrieves a pointer to a single string from the + * strings block of the device tree blob at fdt, and optionally also + * returns the string's length in *lenp. + * + * returns: + * a pointer to the string, on success + * NULL, if stroffset is out of bounds, or doesn't point to a valid string + */ +const char *fdt_get_string(const void *fdt, int stroffset, int *lenp); + +/** + * fdt_string - retrieve a string from the strings block of a device tree + * @fdt: pointer to the device tree blob + * @stroffset: offset of the string within the strings block (native endian) + * + * fdt_string() retrieves a pointer to a single string from the + * strings block of the device tree blob at fdt. + * + * returns: + * a pointer to the string, on success + * NULL, if stroffset is out of bounds, or doesn't point to a valid string + */ +const char *fdt_string(const void *fdt, int stroffset); + +/** + * fdt_find_max_phandle - find and return the highest phandle in a tree + * @fdt: pointer to the device tree blob + * @phandle: return location for the highest phandle value found in the tree + * + * fdt_find_max_phandle() finds the highest phandle value in the given device + * tree. The value returned in @phandle is only valid if the function returns + * success. + * + * returns: + * 0 on success or a negative error code on failure + */ +int fdt_find_max_phandle(const void *fdt, uint32_t *phandle); + +/** + * fdt_get_max_phandle - retrieves the highest phandle in a tree + * @fdt: pointer to the device tree blob + * + * fdt_get_max_phandle retrieves the highest phandle in the given + * device tree. This will ignore badly formatted phandles, or phandles + * with a value of 0 or -1. + * + * This function is deprecated in favour of fdt_find_max_phandle(). + * + * returns: + * the highest phandle on success + * 0, if no phandle was found in the device tree + * -1, if an error occurred + */ +static inline uint32_t fdt_get_max_phandle(const void *fdt) +{ + uint32_t phandle; + int err; + + err = fdt_find_max_phandle(fdt, &phandle); + if (err < 0) + return (uint32_t)-1; + + return phandle; +} + +/** + * fdt_generate_phandle - return a new, unused phandle for a device tree blob + * @fdt: pointer to the device tree blob + * @phandle: return location for the new phandle + * + * Walks the device tree blob and looks for the highest phandle value. On + * success, the new, unused phandle value (one higher than the previously + * highest phandle value in the device tree blob) will be returned in the + * @phandle parameter. + * + * Returns: + * 0 on success or a negative error-code on failure + */ +int fdt_generate_phandle(const void *fdt, uint32_t *phandle); + +/** + * fdt_num_mem_rsv - retrieve the number of memory reserve map entries + * @fdt: pointer to the device tree blob + * + * Returns the number of entries in the device tree blob's memory + * reservation map. This does not include the terminating 0,0 entry + * or any other (0,0) entries reserved for expansion. + * + * returns: + * the number of entries + */ +int fdt_num_mem_rsv(const void *fdt); + +/** + * fdt_get_mem_rsv - retrieve one memory reserve map entry + * @fdt: pointer to the device tree blob + * @address, @size: pointers to 64-bit variables + * + * On success, *address and *size will contain the address and size of + * the n-th reserve map entry from the device tree blob, in + * native-endian format. + * + * returns: + * 0, on success + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, standard meanings + */ +int fdt_get_mem_rsv(const void *fdt, int n, uint64_t *address, uint64_t *size); + +/** + * fdt_subnode_offset_namelen - find a subnode based on substring + * @fdt: pointer to the device tree blob + * @parentoffset: structure block offset of a node + * @name: name of the subnode to locate + * @namelen: number of characters of name to consider + * + * Identical to fdt_subnode_offset(), but only examine the first + * namelen characters of name for matching the subnode name. This is + * useful for finding subnodes based on a portion of a larger string, + * such as a full path. + */ +#ifndef SWIG /* Not available in Python */ +int fdt_subnode_offset_namelen(const void *fdt, int parentoffset, + const char *name, int namelen); +#endif +/** + * fdt_subnode_offset - find a subnode of a given node + * @fdt: pointer to the device tree blob + * @parentoffset: structure block offset of a node + * @name: name of the subnode to locate + * + * fdt_subnode_offset() finds a subnode of the node at structure block + * offset parentoffset with the given name. name may include a unit + * address, in which case fdt_subnode_offset() will find the subnode + * with that unit address, or the unit address may be omitted, in + * which case fdt_subnode_offset() will find an arbitrary subnode + * whose name excluding unit address matches the given name. + * + * returns: + * structure block offset of the requested subnode (>=0), on success + * -FDT_ERR_NOTFOUND, if the requested subnode does not exist + * -FDT_ERR_BADOFFSET, if parentoffset did not point to an FDT_BEGIN_NODE + * tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings. + */ +int fdt_subnode_offset(const void *fdt, int parentoffset, const char *name); + +/** + * fdt_path_offset_namelen - find a tree node by its full path + * @fdt: pointer to the device tree blob + * @path: full path of the node to locate + * @namelen: number of characters of path to consider + * + * Identical to fdt_path_offset(), but only consider the first namelen + * characters of path as the path name. + */ +#ifndef SWIG /* Not available in Python */ +int fdt_path_offset_namelen(const void *fdt, const char *path, int namelen); +#endif + +/** + * fdt_path_offset - find a tree node by its full path + * @fdt: pointer to the device tree blob + * @path: full path of the node to locate + * + * fdt_path_offset() finds a node of a given path in the device tree. + * Each path component may omit the unit address portion, but the + * results of this are undefined if any such path component is + * ambiguous (that is if there are multiple nodes at the relevant + * level matching the given component, differentiated only by unit + * address). + * + * returns: + * structure block offset of the node with the requested path (>=0), on + * success + * -FDT_ERR_BADPATH, given path does not begin with '/' or is invalid + * -FDT_ERR_NOTFOUND, if the requested node does not exist + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings. + */ +int fdt_path_offset(const void *fdt, const char *path); + +/** + * fdt_get_name - retrieve the name of a given node + * @fdt: pointer to the device tree blob + * @nodeoffset: structure block offset of the starting node + * @lenp: pointer to an integer variable (will be overwritten) or NULL + * + * fdt_get_name() retrieves the name (including unit address) of the + * device tree node at structure block offset nodeoffset. If lenp is + * non-NULL, the length of this name is also returned, in the integer + * pointed to by lenp. + * + * returns: + * pointer to the node's name, on success + * If lenp is non-NULL, *lenp contains the length of that name + * (>=0) + * NULL, on error + * if lenp is non-NULL *lenp contains an error code (<0): + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE + * tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, standard meanings + */ +const char *fdt_get_name(const void *fdt, int nodeoffset, int *lenp); + +/** + * fdt_first_property_offset - find the offset of a node's first property + * @fdt: pointer to the device tree blob + * @nodeoffset: structure block offset of a node + * + * fdt_first_property_offset() finds the first property of the node at + * the given structure block offset. + * + * returns: + * structure block offset of the property (>=0), on success + * -FDT_ERR_NOTFOUND, if the requested node has no properties + * -FDT_ERR_BADOFFSET, if nodeoffset did not point to an FDT_BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings. + */ +int fdt_first_property_offset(const void *fdt, int nodeoffset); + +/** + * fdt_next_property_offset - step through a node's properties + * @fdt: pointer to the device tree blob + * @offset: structure block offset of a property + * + * fdt_next_property_offset() finds the property immediately after the + * one at the given structure block offset. This will be a property + * of the same node as the given property. + * + * returns: + * structure block offset of the next property (>=0), on success + * -FDT_ERR_NOTFOUND, if the given property is the last in its node + * -FDT_ERR_BADOFFSET, if nodeoffset did not point to an FDT_PROP tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings. + */ +int fdt_next_property_offset(const void *fdt, int offset); + +/** + * fdt_for_each_property_offset - iterate over all properties of a node + * + * @property_offset: property offset (int, lvalue) + * @fdt: FDT blob (const void *) + * @node: node offset (int) + * + * This is actually a wrapper around a for loop and would be used like so: + * + * fdt_for_each_property_offset(property, fdt, node) { + * Use property + * ... + * } + * + * if ((property < 0) && (property != -FDT_ERR_NOTFOUND)) { + * Error handling + * } + * + * Note that this is implemented as a macro and property is used as + * iterator in the loop. The node variable can be constant or even a + * literal. + */ +#define fdt_for_each_property_offset(property, fdt, node) \ + for (property = fdt_first_property_offset(fdt, node); \ + property >= 0; \ + property = fdt_next_property_offset(fdt, property)) + +/** + * fdt_get_property_by_offset - retrieve the property at a given offset + * @fdt: pointer to the device tree blob + * @offset: offset of the property to retrieve + * @lenp: pointer to an integer variable (will be overwritten) or NULL + * + * fdt_get_property_by_offset() retrieves a pointer to the + * fdt_property structure within the device tree blob at the given + * offset. If lenp is non-NULL, the length of the property value is + * also returned, in the integer pointed to by lenp. + * + * Note that this code only works on device tree versions >= 16. fdt_getprop() + * works on all versions. + * + * returns: + * pointer to the structure representing the property + * if lenp is non-NULL, *lenp contains the length of the property + * value (>=0) + * NULL, on error + * if lenp is non-NULL, *lenp contains an error code (<0): + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_PROP tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +const struct fdt_property *fdt_get_property_by_offset(const void *fdt, + int offset, + int *lenp); + +/** + * fdt_get_property_namelen - find a property based on substring + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to find + * @name: name of the property to find + * @namelen: number of characters of name to consider + * @lenp: pointer to an integer variable (will be overwritten) or NULL + * + * Identical to fdt_get_property(), but only examine the first namelen + * characters of name for matching the property name. + */ +#ifndef SWIG /* Not available in Python */ +const struct fdt_property *fdt_get_property_namelen(const void *fdt, + int nodeoffset, + const char *name, + int namelen, int *lenp); +#endif + +/** + * fdt_get_property - find a given property in a given node + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to find + * @name: name of the property to find + * @lenp: pointer to an integer variable (will be overwritten) or NULL + * + * fdt_get_property() retrieves a pointer to the fdt_property + * structure within the device tree blob corresponding to the property + * named 'name' of the node at offset nodeoffset. If lenp is + * non-NULL, the length of the property value is also returned, in the + * integer pointed to by lenp. + * + * returns: + * pointer to the structure representing the property + * if lenp is non-NULL, *lenp contains the length of the property + * value (>=0) + * NULL, on error + * if lenp is non-NULL, *lenp contains an error code (<0): + * -FDT_ERR_NOTFOUND, node does not have named property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE + * tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +const struct fdt_property *fdt_get_property(const void *fdt, int nodeoffset, + const char *name, int *lenp); +static inline struct fdt_property *fdt_get_property_w(void *fdt, int nodeoffset, + const char *name, + int *lenp) +{ + return (struct fdt_property *)(uintptr_t) + fdt_get_property(fdt, nodeoffset, name, lenp); +} + +/** + * fdt_getprop_by_offset - retrieve the value of a property at a given offset + * @fdt: pointer to the device tree blob + * @offset: offset of the property to read + * @namep: pointer to a string variable (will be overwritten) or NULL + * @lenp: pointer to an integer variable (will be overwritten) or NULL + * + * fdt_getprop_by_offset() retrieves a pointer to the value of the + * property at structure block offset 'offset' (this will be a pointer + * to within the device blob itself, not a copy of the value). If + * lenp is non-NULL, the length of the property value is also + * returned, in the integer pointed to by lenp. If namep is non-NULL, + * the property's namne will also be returned in the char * pointed to + * by namep (this will be a pointer to within the device tree's string + * block, not a new copy of the name). + * + * returns: + * pointer to the property's value + * if lenp is non-NULL, *lenp contains the length of the property + * value (>=0) + * if namep is non-NULL *namep contiains a pointer to the property + * name. + * NULL, on error + * if lenp is non-NULL, *lenp contains an error code (<0): + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_PROP tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +#ifndef SWIG /* This function is not useful in Python */ +const void *fdt_getprop_by_offset(const void *fdt, int offset, + const char **namep, int *lenp); +#endif + +/** + * fdt_getprop_namelen - get property value based on substring + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to find + * @name: name of the property to find + * @namelen: number of characters of name to consider + * @lenp: pointer to an integer variable (will be overwritten) or NULL + * + * Identical to fdt_getprop(), but only examine the first namelen + * characters of name for matching the property name. + */ +#ifndef SWIG /* Not available in Python */ +const void *fdt_getprop_namelen(const void *fdt, int nodeoffset, + const char *name, int namelen, int *lenp); +static inline void *fdt_getprop_namelen_w(void *fdt, int nodeoffset, + const char *name, int namelen, + int *lenp) +{ + return (void *)(uintptr_t)fdt_getprop_namelen(fdt, nodeoffset, name, + namelen, lenp); +} +#endif + +/** + * fdt_getprop - retrieve the value of a given property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to find + * @name: name of the property to find + * @lenp: pointer to an integer variable (will be overwritten) or NULL + * + * fdt_getprop() retrieves a pointer to the value of the property + * named 'name' of the node at offset nodeoffset (this will be a + * pointer to within the device blob itself, not a copy of the value). + * If lenp is non-NULL, the length of the property value is also + * returned, in the integer pointed to by lenp. + * + * returns: + * pointer to the property's value + * if lenp is non-NULL, *lenp contains the length of the property + * value (>=0) + * NULL, on error + * if lenp is non-NULL, *lenp contains an error code (<0): + * -FDT_ERR_NOTFOUND, node does not have named property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE + * tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +const void *fdt_getprop(const void *fdt, int nodeoffset, + const char *name, int *lenp); +static inline void *fdt_getprop_w(void *fdt, int nodeoffset, + const char *name, int *lenp) +{ + return (void *)(uintptr_t)fdt_getprop(fdt, nodeoffset, name, lenp); +} + +/** + * fdt_get_phandle - retrieve the phandle of a given node + * @fdt: pointer to the device tree blob + * @nodeoffset: structure block offset of the node + * + * fdt_get_phandle() retrieves the phandle of the device tree node at + * structure block offset nodeoffset. + * + * returns: + * the phandle of the node at nodeoffset, on success (!= 0, != -1) + * 0, if the node has no phandle, or another error occurs + */ +uint32_t fdt_get_phandle(const void *fdt, int nodeoffset); + +/** + * fdt_get_alias_namelen - get alias based on substring + * @fdt: pointer to the device tree blob + * @name: name of the alias th look up + * @namelen: number of characters of name to consider + * + * Identical to fdt_get_alias(), but only examine the first namelen + * characters of name for matching the alias name. + */ +#ifndef SWIG /* Not available in Python */ +const char *fdt_get_alias_namelen(const void *fdt, + const char *name, int namelen); +#endif + +/** + * fdt_get_alias - retrieve the path referenced by a given alias + * @fdt: pointer to the device tree blob + * @name: name of the alias th look up + * + * fdt_get_alias() retrieves the value of a given alias. That is, the + * value of the property named 'name' in the node /aliases. + * + * returns: + * a pointer to the expansion of the alias named 'name', if it exists + * NULL, if the given alias or the /aliases node does not exist + */ +const char *fdt_get_alias(const void *fdt, const char *name); + +/** + * fdt_get_path - determine the full path of a node + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose path to find + * @buf: character buffer to contain the returned path (will be overwritten) + * @buflen: size of the character buffer at buf + * + * fdt_get_path() computes the full path of the node at offset + * nodeoffset, and records that path in the buffer at buf. + * + * NOTE: This function is expensive, as it must scan the device tree + * structure from the start to nodeoffset. + * + * returns: + * 0, on success + * buf contains the absolute path of the node at + * nodeoffset, as a NUL-terminated string. + * -FDT_ERR_BADOFFSET, nodeoffset does not refer to a BEGIN_NODE tag + * -FDT_ERR_NOSPACE, the path of the given node is longer than (bufsize-1) + * characters and will not fit in the given buffer. + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen); + +/** + * fdt_supernode_atdepth_offset - find a specific ancestor of a node + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose parent to find + * @supernodedepth: depth of the ancestor to find + * @nodedepth: pointer to an integer variable (will be overwritten) or NULL + * + * fdt_supernode_atdepth_offset() finds an ancestor of the given node + * at a specific depth from the root (where the root itself has depth + * 0, its immediate subnodes depth 1 and so forth). So + * fdt_supernode_atdepth_offset(fdt, nodeoffset, 0, NULL); + * will always return 0, the offset of the root node. If the node at + * nodeoffset has depth D, then: + * fdt_supernode_atdepth_offset(fdt, nodeoffset, D, NULL); + * will return nodeoffset itself. + * + * NOTE: This function is expensive, as it must scan the device tree + * structure from the start to nodeoffset. + * + * returns: + * structure block offset of the node at node offset's ancestor + * of depth supernodedepth (>=0), on success + * -FDT_ERR_BADOFFSET, nodeoffset does not refer to a BEGIN_NODE tag + * -FDT_ERR_NOTFOUND, supernodedepth was greater than the depth of + * nodeoffset + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_supernode_atdepth_offset(const void *fdt, int nodeoffset, + int supernodedepth, int *nodedepth); + +/** + * fdt_node_depth - find the depth of a given node + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose parent to find + * + * fdt_node_depth() finds the depth of a given node. The root node + * has depth 0, its immediate subnodes depth 1 and so forth. + * + * NOTE: This function is expensive, as it must scan the device tree + * structure from the start to nodeoffset. + * + * returns: + * depth of the node at nodeoffset (>=0), on success + * -FDT_ERR_BADOFFSET, nodeoffset does not refer to a BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_node_depth(const void *fdt, int nodeoffset); + +/** + * fdt_parent_offset - find the parent of a given node + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose parent to find + * + * fdt_parent_offset() locates the parent node of a given node (that + * is, it finds the offset of the node which contains the node at + * nodeoffset as a subnode). + * + * NOTE: This function is expensive, as it must scan the device tree + * structure from the start to nodeoffset, *twice*. + * + * returns: + * structure block offset of the parent of the node at nodeoffset + * (>=0), on success + * -FDT_ERR_BADOFFSET, nodeoffset does not refer to a BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_parent_offset(const void *fdt, int nodeoffset); + +/** + * fdt_node_offset_by_prop_value - find nodes with a given property value + * @fdt: pointer to the device tree blob + * @startoffset: only find nodes after this offset + * @propname: property name to check + * @propval: property value to search for + * @proplen: length of the value in propval + * + * fdt_node_offset_by_prop_value() returns the offset of the first + * node after startoffset, which has a property named propname whose + * value is of length proplen and has value equal to propval; or if + * startoffset is -1, the very first such node in the tree. + * + * To iterate through all nodes matching the criterion, the following + * idiom can be used: + * offset = fdt_node_offset_by_prop_value(fdt, -1, propname, + * propval, proplen); + * while (offset != -FDT_ERR_NOTFOUND) { + * // other code here + * offset = fdt_node_offset_by_prop_value(fdt, offset, propname, + * propval, proplen); + * } + * + * Note the -1 in the first call to the function, if 0 is used here + * instead, the function will never locate the root node, even if it + * matches the criterion. + * + * returns: + * structure block offset of the located node (>= 0, >startoffset), + * on success + * -FDT_ERR_NOTFOUND, no node matching the criterion exists in the + * tree after startoffset + * -FDT_ERR_BADOFFSET, nodeoffset does not refer to a BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_node_offset_by_prop_value(const void *fdt, int startoffset, + const char *propname, + const void *propval, int proplen); + +/** + * fdt_node_offset_by_phandle - find the node with a given phandle + * @fdt: pointer to the device tree blob + * @phandle: phandle value + * + * fdt_node_offset_by_phandle() returns the offset of the node + * which has the given phandle value. If there is more than one node + * in the tree with the given phandle (an invalid tree), results are + * undefined. + * + * returns: + * structure block offset of the located node (>= 0), on success + * -FDT_ERR_NOTFOUND, no node with that phandle exists + * -FDT_ERR_BADPHANDLE, given phandle value was invalid (0 or -1) + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_node_offset_by_phandle(const void *fdt, uint32_t phandle); + +/** + * fdt_node_check_compatible: check a node's compatible property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of a tree node + * @compatible: string to match against + * + * + * fdt_node_check_compatible() returns 0 if the given node contains a + * 'compatible' property with the given string as one of its elements, + * it returns non-zero otherwise, or on error. + * + * returns: + * 0, if the node has a 'compatible' property listing the given string + * 1, if the node has a 'compatible' property, but it does not list + * the given string + * -FDT_ERR_NOTFOUND, if the given node has no 'compatible' property + * -FDT_ERR_BADOFFSET, if nodeoffset does not refer to a BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_node_check_compatible(const void *fdt, int nodeoffset, + const char *compatible); + +/** + * fdt_node_offset_by_compatible - find nodes with a given 'compatible' value + * @fdt: pointer to the device tree blob + * @startoffset: only find nodes after this offset + * @compatible: 'compatible' string to match against + * + * fdt_node_offset_by_compatible() returns the offset of the first + * node after startoffset, which has a 'compatible' property which + * lists the given compatible string; or if startoffset is -1, the + * very first such node in the tree. + * + * To iterate through all nodes matching the criterion, the following + * idiom can be used: + * offset = fdt_node_offset_by_compatible(fdt, -1, compatible); + * while (offset != -FDT_ERR_NOTFOUND) { + * // other code here + * offset = fdt_node_offset_by_compatible(fdt, offset, compatible); + * } + * + * Note the -1 in the first call to the function, if 0 is used here + * instead, the function will never locate the root node, even if it + * matches the criterion. + * + * returns: + * structure block offset of the located node (>= 0, >startoffset), + * on success + * -FDT_ERR_NOTFOUND, no node matching the criterion exists in the + * tree after startoffset + * -FDT_ERR_BADOFFSET, nodeoffset does not refer to a BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, standard meanings + */ +int fdt_node_offset_by_compatible(const void *fdt, int startoffset, + const char *compatible); + +/** + * fdt_stringlist_contains - check a string list property for a string + * @strlist: Property containing a list of strings to check + * @listlen: Length of property + * @str: String to search for + * + * This is a utility function provided for convenience. The list contains + * one or more strings, each terminated by \0, as is found in a device tree + * "compatible" property. + * + * @return: 1 if the string is found in the list, 0 not found, or invalid list + */ +int fdt_stringlist_contains(const char *strlist, int listlen, const char *str); + +/** + * fdt_stringlist_count - count the number of strings in a string list + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of a tree node + * @property: name of the property containing the string list + * @return: + * the number of strings in the given property + * -FDT_ERR_BADVALUE if the property value is not NUL-terminated + * -FDT_ERR_NOTFOUND if the property does not exist + */ +int fdt_stringlist_count(const void *fdt, int nodeoffset, const char *property); + +/** + * fdt_stringlist_search - find a string in a string list and return its index + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of a tree node + * @property: name of the property containing the string list + * @string: string to look up in the string list + * + * Note that it is possible for this function to succeed on property values + * that are not NUL-terminated. That's because the function will stop after + * finding the first occurrence of @string. This can for example happen with + * small-valued cell properties, such as #address-cells, when searching for + * the empty string. + * + * @return: + * the index of the string in the list of strings + * -FDT_ERR_BADVALUE if the property value is not NUL-terminated + * -FDT_ERR_NOTFOUND if the property does not exist or does not contain + * the given string + */ +int fdt_stringlist_search(const void *fdt, int nodeoffset, const char *property, + const char *string); + +/** + * fdt_stringlist_get() - obtain the string at a given index in a string list + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of a tree node + * @property: name of the property containing the string list + * @index: index of the string to return + * @lenp: return location for the string length or an error code on failure + * + * Note that this will successfully extract strings from properties with + * non-NUL-terminated values. For example on small-valued cell properties + * this function will return the empty string. + * + * If non-NULL, the length of the string (on success) or a negative error-code + * (on failure) will be stored in the integer pointer to by lenp. + * + * @return: + * A pointer to the string at the given index in the string list or NULL on + * failure. On success the length of the string will be stored in the memory + * location pointed to by the lenp parameter, if non-NULL. On failure one of + * the following negative error codes will be returned in the lenp parameter + * (if non-NULL): + * -FDT_ERR_BADVALUE if the property value is not NUL-terminated + * -FDT_ERR_NOTFOUND if the property does not exist + */ +const char *fdt_stringlist_get(const void *fdt, int nodeoffset, + const char *property, int index, + int *lenp); + +/**********************************************************************/ +/* Read-only functions (addressing related) */ +/**********************************************************************/ + +/** + * FDT_MAX_NCELLS - maximum value for #address-cells and #size-cells + * + * This is the maximum value for #address-cells, #size-cells and + * similar properties that will be processed by libfdt. IEE1275 + * requires that OF implementations handle values up to 4. + * Implementations may support larger values, but in practice higher + * values aren't used. + */ +#define FDT_MAX_NCELLS 4 + +/** + * fdt_address_cells - retrieve address size for a bus represented in the tree + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node to find the address size for + * + * When the node has a valid #address-cells property, returns its value. + * + * returns: + * 0 <= n < FDT_MAX_NCELLS, on success + * 2, if the node has no #address-cells property + * -FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid + * #address-cells property + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_address_cells(const void *fdt, int nodeoffset); + +/** + * fdt_size_cells - retrieve address range size for a bus represented in the + * tree + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node to find the address range size for + * + * When the node has a valid #size-cells property, returns its value. + * + * returns: + * 0 <= n < FDT_MAX_NCELLS, on success + * 1, if the node has no #size-cells property + * -FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid + * #size-cells property + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_size_cells(const void *fdt, int nodeoffset); + + +/**********************************************************************/ +/* Write-in-place functions */ +/**********************************************************************/ + +/** + * fdt_setprop_inplace_namelen_partial - change a property's value, + * but not its size + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @namelen: number of characters of name to consider + * @idx: index of the property to change in the array + * @val: pointer to data to replace the property value with + * @len: length of the property value + * + * Identical to fdt_setprop_inplace(), but modifies the given property + * starting from the given index, and using only the first characters + * of the name. It is useful when you want to manipulate only one value of + * an array and you have a string that doesn't end with \0. + */ +#ifndef SWIG /* Not available in Python */ +int fdt_setprop_inplace_namelen_partial(void *fdt, int nodeoffset, + const char *name, int namelen, + uint32_t idx, const void *val, + int len); +#endif + +/** + * fdt_setprop_inplace - change a property's value, but not its size + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: pointer to data to replace the property value with + * @len: length of the property value + * + * fdt_setprop_inplace() replaces the value of a given property with + * the data in val, of length len. This function cannot change the + * size of a property, and so will only work if len is equal to the + * current length of the property. + * + * This function will alter only the bytes in the blob which contain + * the given property value, and will not alter or move any other part + * of the tree. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, if len is not equal to the property's current length + * -FDT_ERR_NOTFOUND, node does not have the named property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +#ifndef SWIG /* Not available in Python */ +int fdt_setprop_inplace(void *fdt, int nodeoffset, const char *name, + const void *val, int len); +#endif + +/** + * fdt_setprop_inplace_u32 - change the value of a 32-bit integer property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: 32-bit integer value to replace the property with + * + * fdt_setprop_inplace_u32() replaces the value of a given property + * with the 32-bit integer value in val, converting val to big-endian + * if necessary. This function cannot change the size of a property, + * and so will only work if the property already exists and has length + * 4. + * + * This function will alter only the bytes in the blob which contain + * the given property value, and will not alter or move any other part + * of the tree. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, if the property's length is not equal to 4 + * -FDT_ERR_NOTFOUND, node does not have the named property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +static inline int fdt_setprop_inplace_u32(void *fdt, int nodeoffset, + const char *name, uint32_t val) +{ + fdt32_t tmp = cpu_to_fdt32(val); + return fdt_setprop_inplace(fdt, nodeoffset, name, &tmp, sizeof(tmp)); +} + +/** + * fdt_setprop_inplace_u64 - change the value of a 64-bit integer property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: 64-bit integer value to replace the property with + * + * fdt_setprop_inplace_u64() replaces the value of a given property + * with the 64-bit integer value in val, converting val to big-endian + * if necessary. This function cannot change the size of a property, + * and so will only work if the property already exists and has length + * 8. + * + * This function will alter only the bytes in the blob which contain + * the given property value, and will not alter or move any other part + * of the tree. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, if the property's length is not equal to 8 + * -FDT_ERR_NOTFOUND, node does not have the named property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +static inline int fdt_setprop_inplace_u64(void *fdt, int nodeoffset, + const char *name, uint64_t val) +{ + fdt64_t tmp = cpu_to_fdt64(val); + return fdt_setprop_inplace(fdt, nodeoffset, name, &tmp, sizeof(tmp)); +} + +/** + * fdt_setprop_inplace_cell - change the value of a single-cell property + * + * This is an alternative name for fdt_setprop_inplace_u32() + */ +static inline int fdt_setprop_inplace_cell(void *fdt, int nodeoffset, + const char *name, uint32_t val) +{ + return fdt_setprop_inplace_u32(fdt, nodeoffset, name, val); +} + +/** + * fdt_nop_property - replace a property with nop tags + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to nop + * @name: name of the property to nop + * + * fdt_nop_property() will replace a given property's representation + * in the blob with FDT_NOP tags, effectively removing it from the + * tree. + * + * This function will alter only the bytes in the blob which contain + * the property, and will not alter or move any other part of the + * tree. + * + * returns: + * 0, on success + * -FDT_ERR_NOTFOUND, node does not have the named property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_nop_property(void *fdt, int nodeoffset, const char *name); + +/** + * fdt_nop_node - replace a node (subtree) with nop tags + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node to nop + * + * fdt_nop_node() will replace a given node's representation in the + * blob, including all its subnodes, if any, with FDT_NOP tags, + * effectively removing it from the tree. + * + * This function will alter only the bytes in the blob which contain + * the node and its properties and subnodes, and will not alter or + * move any other part of the tree. + * + * returns: + * 0, on success + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_nop_node(void *fdt, int nodeoffset); + +/**********************************************************************/ +/* Sequential write functions */ +/**********************************************************************/ + +/* fdt_create_with_flags flags */ +#define FDT_CREATE_FLAG_NO_NAME_DEDUP 0x1 + /* FDT_CREATE_FLAG_NO_NAME_DEDUP: Do not try to de-duplicate property + * names in the fdt. This can result in faster creation times, but + * a larger fdt. */ + +#define FDT_CREATE_FLAGS_ALL (FDT_CREATE_FLAG_NO_NAME_DEDUP) + +/** + * fdt_create_with_flags - begin creation of a new fdt + * @fdt: pointer to memory allocated where fdt will be created + * @bufsize: size of the memory space at fdt + * @flags: a valid combination of FDT_CREATE_FLAG_ flags, or 0. + * + * fdt_create_with_flags() begins the process of creating a new fdt with + * the sequential write interface. + * + * fdt creation process must end with fdt_finished() to produce a valid fdt. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, bufsize is insufficient for a minimal fdt + * -FDT_ERR_BADFLAGS, flags is not valid + */ +int fdt_create_with_flags(void *buf, int bufsize, uint32_t flags); + +/** + * fdt_create - begin creation of a new fdt + * @fdt: pointer to memory allocated where fdt will be created + * @bufsize: size of the memory space at fdt + * + * fdt_create() is equivalent to fdt_create_with_flags() with flags=0. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, bufsize is insufficient for a minimal fdt + */ +int fdt_create(void *buf, int bufsize); + +int fdt_resize(void *fdt, void *buf, int bufsize); +int fdt_add_reservemap_entry(void *fdt, uint64_t addr, uint64_t size); +int fdt_finish_reservemap(void *fdt); +int fdt_begin_node(void *fdt, const char *name); +int fdt_property(void *fdt, const char *name, const void *val, int len); +static inline int fdt_property_u32(void *fdt, const char *name, uint32_t val) +{ + fdt32_t tmp = cpu_to_fdt32(val); + return fdt_property(fdt, name, &tmp, sizeof(tmp)); +} +static inline int fdt_property_u64(void *fdt, const char *name, uint64_t val) +{ + fdt64_t tmp = cpu_to_fdt64(val); + return fdt_property(fdt, name, &tmp, sizeof(tmp)); +} + +#ifndef SWIG /* Not available in Python */ +static inline int fdt_property_cell(void *fdt, const char *name, uint32_t val) +{ + return fdt_property_u32(fdt, name, val); +} +#endif + +/** + * fdt_property_placeholder - add a new property and return a ptr to its value + * + * @fdt: pointer to the device tree blob + * @name: name of property to add + * @len: length of property value in bytes + * @valp: returns a pointer to where where the value should be placed + * + * returns: + * 0, on success + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_NOSPACE, standard meanings + */ +int fdt_property_placeholder(void *fdt, const char *name, int len, void **valp); + +#define fdt_property_string(fdt, name, str) \ + fdt_property(fdt, name, str, strlen(str)+1) +int fdt_end_node(void *fdt); +int fdt_finish(void *fdt); + +/**********************************************************************/ +/* Read-write functions */ +/**********************************************************************/ + +int fdt_create_empty_tree(void *buf, int bufsize); +int fdt_open_into(const void *fdt, void *buf, int bufsize); +int fdt_pack(void *fdt); + +/** + * fdt_add_mem_rsv - add one memory reserve map entry + * @fdt: pointer to the device tree blob + * @address, @size: 64-bit values (native endian) + * + * Adds a reserve map entry to the given blob reserving a region at + * address address of length size. + * + * This function will insert data into the reserve map and will + * therefore change the indexes of some entries in the table. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new reservation entry + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_add_mem_rsv(void *fdt, uint64_t address, uint64_t size); + +/** + * fdt_del_mem_rsv - remove a memory reserve map entry + * @fdt: pointer to the device tree blob + * @n: entry to remove + * + * fdt_del_mem_rsv() removes the n-th memory reserve map entry from + * the blob. + * + * This function will delete data from the reservation table and will + * therefore change the indexes of some entries in the table. + * + * returns: + * 0, on success + * -FDT_ERR_NOTFOUND, there is no entry of the given index (i.e. there + * are less than n+1 reserve map entries) + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_del_mem_rsv(void *fdt, int n); + +/** + * fdt_set_name - change the name of a given node + * @fdt: pointer to the device tree blob + * @nodeoffset: structure block offset of a node + * @name: name to give the node + * + * fdt_set_name() replaces the name (including unit address, if any) + * of the given node with the given string. NOTE: this function can't + * efficiently check if the new name is unique amongst the given + * node's siblings; results are undefined if this function is invoked + * with a name equal to one of the given node's siblings. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob + * to contain the new name + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, standard meanings + */ +int fdt_set_name(void *fdt, int nodeoffset, const char *name); + +/** + * fdt_setprop - create or change a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: pointer to data to set the property value to + * @len: length of the property value + * + * fdt_setprop() sets the value of the named property in the given + * node to the given value and length, creating the property if it + * does not already exist. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_setprop(void *fdt, int nodeoffset, const char *name, + const void *val, int len); + +/** + * fdt_setprop_placeholder - allocate space for a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @len: length of the property value + * @prop_data: return pointer to property data + * + * fdt_setprop_placeholer() allocates the named property in the given node. + * If the property exists it is resized. In either case a pointer to the + * property data is returned. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_setprop_placeholder(void *fdt, int nodeoffset, const char *name, + int len, void **prop_data); + +/** + * fdt_setprop_u32 - set a property to a 32-bit integer + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: 32-bit integer value for the property (native endian) + * + * fdt_setprop_u32() sets the value of the named property in the given + * node to the given 32-bit integer value (converting to big-endian if + * necessary), or creates a new property with that value if it does + * not already exist. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +static inline int fdt_setprop_u32(void *fdt, int nodeoffset, const char *name, + uint32_t val) +{ + fdt32_t tmp = cpu_to_fdt32(val); + return fdt_setprop(fdt, nodeoffset, name, &tmp, sizeof(tmp)); +} + +/** + * fdt_setprop_u64 - set a property to a 64-bit integer + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: 64-bit integer value for the property (native endian) + * + * fdt_setprop_u64() sets the value of the named property in the given + * node to the given 64-bit integer value (converting to big-endian if + * necessary), or creates a new property with that value if it does + * not already exist. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +static inline int fdt_setprop_u64(void *fdt, int nodeoffset, const char *name, + uint64_t val) +{ + fdt64_t tmp = cpu_to_fdt64(val); + return fdt_setprop(fdt, nodeoffset, name, &tmp, sizeof(tmp)); +} + +/** + * fdt_setprop_cell - set a property to a single cell value + * + * This is an alternative name for fdt_setprop_u32() + */ +static inline int fdt_setprop_cell(void *fdt, int nodeoffset, const char *name, + uint32_t val) +{ + return fdt_setprop_u32(fdt, nodeoffset, name, val); +} + +/** + * fdt_setprop_string - set a property to a string value + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @str: string value for the property + * + * fdt_setprop_string() sets the value of the named property in the + * given node to the given string value (using the length of the + * string to determine the new length of the property), or creates a + * new property with that value if it does not already exist. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +#define fdt_setprop_string(fdt, nodeoffset, name, str) \ + fdt_setprop((fdt), (nodeoffset), (name), (str), strlen(str)+1) + + +/** + * fdt_setprop_empty - set a property to an empty value + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * + * fdt_setprop_empty() sets the value of the named property in the + * given node to an empty (zero length) value, or creates a new empty + * property if it does not already exist. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +#define fdt_setprop_empty(fdt, nodeoffset, name) \ + fdt_setprop((fdt), (nodeoffset), (name), NULL, 0) + +/** + * fdt_appendprop - append to or create a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to append to + * @val: pointer to data to append to the property value + * @len: length of the data to append to the property value + * + * fdt_appendprop() appends the value to the named property in the + * given node, creating the property if it does not already exist. + * + * This function may insert data into the blob, and will therefore + * change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_appendprop(void *fdt, int nodeoffset, const char *name, + const void *val, int len); + +/** + * fdt_appendprop_u32 - append a 32-bit integer value to a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: 32-bit integer value to append to the property (native endian) + * + * fdt_appendprop_u32() appends the given 32-bit integer value + * (converting to big-endian if necessary) to the value of the named + * property in the given node, or creates a new property with that + * value if it does not already exist. + * + * This function may insert data into the blob, and will therefore + * change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +static inline int fdt_appendprop_u32(void *fdt, int nodeoffset, + const char *name, uint32_t val) +{ + fdt32_t tmp = cpu_to_fdt32(val); + return fdt_appendprop(fdt, nodeoffset, name, &tmp, sizeof(tmp)); +} + +/** + * fdt_appendprop_u64 - append a 64-bit integer value to a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @val: 64-bit integer value to append to the property (native endian) + * + * fdt_appendprop_u64() appends the given 64-bit integer value + * (converting to big-endian if necessary) to the value of the named + * property in the given node, or creates a new property with that + * value if it does not already exist. + * + * This function may insert data into the blob, and will therefore + * change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +static inline int fdt_appendprop_u64(void *fdt, int nodeoffset, + const char *name, uint64_t val) +{ + fdt64_t tmp = cpu_to_fdt64(val); + return fdt_appendprop(fdt, nodeoffset, name, &tmp, sizeof(tmp)); +} + +/** + * fdt_appendprop_cell - append a single cell value to a property + * + * This is an alternative name for fdt_appendprop_u32() + */ +static inline int fdt_appendprop_cell(void *fdt, int nodeoffset, + const char *name, uint32_t val) +{ + return fdt_appendprop_u32(fdt, nodeoffset, name, val); +} + +/** + * fdt_appendprop_string - append a string to a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @str: string value to append to the property + * + * fdt_appendprop_string() appends the given string to the value of + * the named property in the given node, or creates a new property + * with that value if it does not already exist. + * + * This function may insert data into the blob, and will therefore + * change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +#define fdt_appendprop_string(fdt, nodeoffset, name, str) \ + fdt_appendprop((fdt), (nodeoffset), (name), (str), strlen(str)+1) + +/** + * fdt_appendprop_addrrange - append a address range property + * @fdt: pointer to the device tree blob + * @parent: offset of the parent node + * @nodeoffset: offset of the node to add a property at + * @name: name of property + * @addr: start address of a given range + * @size: size of a given range + * + * fdt_appendprop_addrrange() appends an address range value (start + * address and size) to the value of the named property in the given + * node, or creates a new property with that value if it does not + * already exist. + * If "name" is not specified, a default "reg" is used. + * Cell sizes are determined by parent's #address-cells and #size-cells. + * + * This function may insert data into the blob, and will therefore + * change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid + * #address-cells property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADVALUE, addr or size doesn't fit to respective cells size + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain a new property + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_appendprop_addrrange(void *fdt, int parent, int nodeoffset, + const char *name, uint64_t addr, uint64_t size); + +/** + * fdt_delprop - delete a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to nop + * @name: name of the property to nop + * + * fdt_del_property() will delete the given property. + * + * This function will delete data from the blob, and will therefore + * change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOTFOUND, node does not have the named property + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_delprop(void *fdt, int nodeoffset, const char *name); + +/** + * fdt_add_subnode_namelen - creates a new node based on substring + * @fdt: pointer to the device tree blob + * @parentoffset: structure block offset of a node + * @name: name of the subnode to locate + * @namelen: number of characters of name to consider + * + * Identical to fdt_add_subnode(), but use only the first namelen + * characters of name as the name of the new node. This is useful for + * creating subnodes based on a portion of a larger string, such as a + * full path. + */ +#ifndef SWIG /* Not available in Python */ +int fdt_add_subnode_namelen(void *fdt, int parentoffset, + const char *name, int namelen); +#endif + +/** + * fdt_add_subnode - creates a new node + * @fdt: pointer to the device tree blob + * @parentoffset: structure block offset of a node + * @name: name of the subnode to locate + * + * fdt_add_subnode() creates a new node as a subnode of the node at + * structure block offset parentoffset, with the given name (which + * should include the unit address, if any). + * + * This function will insert data into the blob, and will therefore + * change the offsets of some existing nodes. + + * returns: + * structure block offset of the created nodeequested subnode (>=0), on + * success + * -FDT_ERR_NOTFOUND, if the requested subnode does not exist + * -FDT_ERR_BADOFFSET, if parentoffset did not point to an FDT_BEGIN_NODE + * tag + * -FDT_ERR_EXISTS, if the node at parentoffset already has a subnode of + * the given name + * -FDT_ERR_NOSPACE, if there is insufficient free space in the + * blob to contain the new node + * -FDT_ERR_NOSPACE + * -FDT_ERR_BADLAYOUT + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings. + */ +int fdt_add_subnode(void *fdt, int parentoffset, const char *name); + +/** + * fdt_del_node - delete a node (subtree) + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node to nop + * + * fdt_del_node() will remove the given node, including all its + * subnodes if any, from the blob. + * + * This function will delete data from the blob, and will therefore + * change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_del_node(void *fdt, int nodeoffset); + +/** + * fdt_overlay_apply - Applies a DT overlay on a base DT + * @fdt: pointer to the base device tree blob + * @fdto: pointer to the device tree overlay blob + * + * fdt_overlay_apply() will apply the given device tree overlay on the + * given base device tree. + * + * Expect the base device tree to be modified, even if the function + * returns an error. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there's not enough space in the base device tree + * -FDT_ERR_NOTFOUND, the overlay points to some inexistant nodes or + * properties in the base DT + * -FDT_ERR_BADPHANDLE, + * -FDT_ERR_BADOVERLAY, + * -FDT_ERR_NOPHANDLES, + * -FDT_ERR_INTERNAL, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADOFFSET, + * -FDT_ERR_BADPATH, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_overlay_apply(void *fdt, void *fdto); + +/**********************************************************************/ +/* Debugging / informational functions */ +/**********************************************************************/ + +const char *fdt_strerror(int errval); + +#endif /* LIBFDT_H */ diff --git a/optee_os/core/lib/libfdt/include/libfdt_env.h b/optee_os/core/lib/libfdt/include/libfdt_env.h new file mode 100644 index 0000000..73b6d40 --- /dev/null +++ b/optee_os/core/lib/libfdt/include/libfdt_env.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +#ifndef LIBFDT_ENV_H +#define LIBFDT_ENV_H +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + * Copyright 2012 Kim Phillips, Freescale Semiconductor. + */ + +#include +#include +#include +#include +#include +#include + +#ifdef __CHECKER__ +#define FDT_FORCE __attribute__((force)) +#define FDT_BITWISE __attribute__((bitwise)) +#else +#define FDT_FORCE +#define FDT_BITWISE +#endif + +typedef uint16_t FDT_BITWISE fdt16_t; +typedef uint32_t FDT_BITWISE fdt32_t; +typedef uint64_t FDT_BITWISE fdt64_t; + +#define EXTRACT_BYTE(x, n) ((unsigned long long)((uint8_t *)&x)[n]) +#define CPU_TO_FDT16(x) ((EXTRACT_BYTE(x, 0) << 8) | EXTRACT_BYTE(x, 1)) +#define CPU_TO_FDT32(x) ((EXTRACT_BYTE(x, 0) << 24) | (EXTRACT_BYTE(x, 1) << 16) | \ + (EXTRACT_BYTE(x, 2) << 8) | EXTRACT_BYTE(x, 3)) +#define CPU_TO_FDT64(x) ((EXTRACT_BYTE(x, 0) << 56) | (EXTRACT_BYTE(x, 1) << 48) | \ + (EXTRACT_BYTE(x, 2) << 40) | (EXTRACT_BYTE(x, 3) << 32) | \ + (EXTRACT_BYTE(x, 4) << 24) | (EXTRACT_BYTE(x, 5) << 16) | \ + (EXTRACT_BYTE(x, 6) << 8) | EXTRACT_BYTE(x, 7)) + +static inline uint16_t fdt16_to_cpu(fdt16_t x) +{ + return (FDT_FORCE uint16_t)CPU_TO_FDT16(x); +} +static inline fdt16_t cpu_to_fdt16(uint16_t x) +{ + return (FDT_FORCE fdt16_t)CPU_TO_FDT16(x); +} + +static inline uint32_t fdt32_to_cpu(fdt32_t x) +{ + return (FDT_FORCE uint32_t)CPU_TO_FDT32(x); +} +static inline fdt32_t cpu_to_fdt32(uint32_t x) +{ + return (FDT_FORCE fdt32_t)CPU_TO_FDT32(x); +} + +static inline uint64_t fdt64_to_cpu(fdt64_t x) +{ + return (FDT_FORCE uint64_t)CPU_TO_FDT64(x); +} +static inline fdt64_t cpu_to_fdt64(uint64_t x) +{ + return (FDT_FORCE fdt64_t)CPU_TO_FDT64(x); +} +#undef CPU_TO_FDT64 +#undef CPU_TO_FDT32 +#undef CPU_TO_FDT16 +#undef EXTRACT_BYTE + +#ifdef __APPLE__ +#include + +/* strnlen() is not available on Mac OS < 10.7 */ +# if !defined(MAC_OS_X_VERSION_10_7) || (MAC_OS_X_VERSION_MAX_ALLOWED < \ + MAC_OS_X_VERSION_10_7) + +#define strnlen fdt_strnlen + +/* + * fdt_strnlen: returns the length of a string or max_count - which ever is + * smallest. + * Input 1 string: the string whose size is to be determined + * Input 2 max_count: the maximum value returned by this function + * Output: length of the string or max_count (the smallest of the two) + */ +static inline size_t fdt_strnlen(const char *string, size_t max_count) +{ + const char *p = memchr(string, 0, max_count); + return p ? p - string : max_count; +} + +#endif /* !defined(MAC_OS_X_VERSION_10_7) || (MAC_OS_X_VERSION_MAX_ALLOWED < + MAC_OS_X_VERSION_10_7) */ + +#endif /* __APPLE__ */ + +#endif /* LIBFDT_ENV_H */ diff --git a/optee_os/core/lib/libfdt/libfdt_internal.h b/optee_os/core/lib/libfdt/libfdt_internal.h new file mode 100644 index 0000000..741eeb3 --- /dev/null +++ b/optee_os/core/lib/libfdt/libfdt_internal.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +#ifndef LIBFDT_INTERNAL_H +#define LIBFDT_INTERNAL_H +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include + +#define FDT_ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) +#define FDT_TAGALIGN(x) (FDT_ALIGN((x), FDT_TAGSIZE)) + +int fdt_ro_probe_(const void *fdt); +#define FDT_RO_PROBE(fdt) \ + { \ + int totalsize_; \ + if ((totalsize_ = fdt_ro_probe_(fdt)) < 0) \ + return totalsize_; \ + } + +int fdt_check_node_offset_(const void *fdt, int offset); +int fdt_check_prop_offset_(const void *fdt, int offset); +const char *fdt_find_string_(const char *strtab, int tabsize, const char *s); +int fdt_node_end_offset_(void *fdt, int nodeoffset); + +static inline const void *fdt_offset_ptr_(const void *fdt, int offset) +{ + return (const char *)fdt + fdt_off_dt_struct(fdt) + offset; +} + +static inline void *fdt_offset_ptr_w_(void *fdt, int offset) +{ + return (void *)(uintptr_t)fdt_offset_ptr_(fdt, offset); +} + +static inline const struct fdt_reserve_entry *fdt_mem_rsv_(const void *fdt, int n) +{ + const struct fdt_reserve_entry *rsv_table = + (const struct fdt_reserve_entry *) + ((const char *)fdt + fdt_off_mem_rsvmap(fdt)); + + return rsv_table + n; +} +static inline struct fdt_reserve_entry *fdt_mem_rsv_w_(void *fdt, int n) +{ + return (void *)(uintptr_t)fdt_mem_rsv_(fdt, n); +} + +#define FDT_SW_MAGIC (~FDT_MAGIC) + +#endif /* LIBFDT_INTERNAL_H */ diff --git a/optee_os/core/lib/libfdt/sub.mk b/optee_os/core/lib/libfdt/sub.mk new file mode 100644 index 0000000..3ea4b35 --- /dev/null +++ b/optee_os/core/lib/libfdt/sub.mk @@ -0,0 +1,6 @@ +global-incdirs-y += include +cflags-y += -Wno-cast-align +cflags-y += -Wno-sign-compare +cflags-y += -Wno-switch-default +srcs-y += fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c +srcs-y += fdt_empty_tree.c fdt_addresses.c fdt_overlay.c diff --git a/optee_os/core/lib/libtomcrypt/acipher_helpers.h b/optee_os/core/lib/libtomcrypt/acipher_helpers.h new file mode 100644 index 0000000..de3210f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/acipher_helpers.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#ifndef ACIPHER_HELPERS_H +#define ACIPHER_HELPERS_H + +#include +#include +#include +#include +#include + +static inline bool bn_alloc_max(struct bignum **s) +{ + *s = crypto_bignum_allocate(_CFG_CORE_LTC_BIGNUM_MAX_BITS); + + return *s; +} + +static inline TEE_Result convert_ltc_verify_status(int ltc_res, int ltc_stat) +{ + switch (ltc_res) { + case CRYPT_OK: + if (ltc_stat == 1) + return TEE_SUCCESS; + else + return TEE_ERROR_SIGNATURE_INVALID; + case CRYPT_INVALID_PACKET: + case CRYPT_PK_INVALID_SIZE: + return TEE_ERROR_SIGNATURE_INVALID; + default: + return TEE_ERROR_GENERIC; + } +} + +#ifdef _CFG_CORE_LTC_ECC +TEE_Result ecc_populate_ltc_private_key(ecc_key *ltc_key, + struct ecc_keypair *key, + uint32_t algo, size_t *key_size_bytes); +TEE_Result ecc_populate_ltc_public_key(ecc_key *ltc_key, + struct ecc_public_key *key, + uint32_t algo, size_t *key_size_bytes); +#endif + +/* Write bignum to fixed size buffer in big endian order */ +#define mp_to_unsigned_bin2(a, b, c) \ + do { \ + void *_a = (a); \ + mp_to_unsigned_bin(_a, (b) + (c) - mp_unsigned_bin_size(_a)); \ + } while(0) + +#ifdef _CFG_CORE_LTC_SM2_DSA +TEE_Result sm2_ltc_dsa_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, uint8_t *sig, + size_t *sig_len); + +TEE_Result sm2_ltc_dsa_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len); +#else +static inline TEE_Result +sm2_ltc_dsa_sign(uint32_t algo __unused, struct ecc_keypair *key __unused, + const uint8_t *msg __unused, size_t msg_len __unused, + uint8_t *sig __unused, size_t *sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static inline TEE_Result +sm2_ltc_dsa_verify(uint32_t algo __unused, struct ecc_public_key *key __unused, + const uint8_t *msg __unused, size_t msg_len __unused, + const uint8_t *sig __unused, size_t sig_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif + +#ifdef _CFG_CORE_LTC_SM2_PKE +TEE_Result sm2_ltc_pke_decrypt(struct ecc_keypair *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len); + +TEE_Result sm2_ltc_pke_encrypt(struct ecc_public_key *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len); + +#else +static inline TEE_Result sm2_ltc_pke_decrypt(struct ecc_keypair *key __unused, + const uint8_t *src __unused, + size_t src_len __unused, + uint8_t *dst __unused, + size_t *dst_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static inline TEE_Result +sm2_ltc_pke_encrypt(struct ecc_public_key *key __unused, + const uint8_t *src __unused, size_t src_len __unused, + uint8_t *dst __unused, size_t *dst_len __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif +#endif /* ACIPHER_HELPERS_H */ diff --git a/optee_os/core/lib/libtomcrypt/aes.c b/optee_os/core/lib/libtomcrypt/aes.c new file mode 100644 index 0000000..3989140 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/aes.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +TEE_Result crypto_aes_expand_enc_key(const void *key, size_t key_len, + void *enc_key, size_t enc_keylen, + unsigned int *rounds) +{ +#ifdef _CFG_CORE_LTC_AES_ACCEL + return crypto_accel_aes_expand_keys(key, key_len, enc_key, NULL, + enc_keylen, rounds); +#else + symmetric_key skey; + + if (enc_keylen < sizeof(skey.rijndael.eK)) + return TEE_ERROR_BAD_PARAMETERS; + + if (aes_setup(key, key_len, 0, &skey)) + return TEE_ERROR_BAD_PARAMETERS; + + memcpy(enc_key, skey.rijndael.eK, sizeof(skey.rijndael.eK)); + *rounds = skey.rijndael.Nr; +#endif + return TEE_SUCCESS; +} + +void crypto_aes_enc_block(const void *enc_key, size_t enc_keylen __maybe_unused, + unsigned int rounds, const void *src, void *dst) +{ +#ifdef _CFG_CORE_LTC_AES_ACCEL + crypto_accel_aes_ecb_enc(dst, src, enc_key, rounds, 1); +#else + symmetric_key skey; + + assert(enc_keylen >= sizeof(skey.rijndael.eK)); + memcpy(skey.rijndael.eK, enc_key, sizeof(skey.rijndael.eK)); + skey.rijndael.Nr = rounds; + if (aes_ecb_encrypt(src, dst, &skey)) + panic(); +#endif +} diff --git a/optee_os/core/lib/libtomcrypt/aes_accel.c b/optee_os/core/lib/libtomcrypt/aes_accel.c new file mode 100644 index 0000000..0fb4bfc --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/aes_accel.c @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + * All rights reserved. + * Copyright (c) 2001-2007, Tom St Denis + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtom.org + */ + +/* + * AES cipher for ARMv8 with Crypto Extensions + * + * Copyright (C) 2013 Linaro Ltd + */ + +#include +#include +#include + +int rijndael_setup(const unsigned char *key, int keylen, int num_rounds, + symmetric_key *skey) +{ + unsigned int round_count = 0; + + LTC_ARGCHK(key); + LTC_ARGCHK(skey); + + if (keylen != 16 && keylen != 24 && keylen != 32) + return CRYPT_INVALID_KEYSIZE; + + if (crypto_accel_aes_expand_keys(key, keylen, skey->rijndael.eK, + skey->rijndael.dK, + sizeof(skey->rijndael.eK), + &round_count)) + return CRYPT_INVALID_ARG; + + if (num_rounds && (unsigned int)num_rounds != round_count) + return CRYPT_INVALID_ROUNDS; + + skey->rijndael.Nr = round_count; + + return CRYPT_OK; +} + +void rijndael_done(symmetric_key *skey __unused) +{ +} + +int rijndael_keysize(int *keysize) +{ + LTC_ARGCHK(keysize); + + if (*keysize < 16) + return CRYPT_INVALID_KEYSIZE; + else if (*keysize < 24) + *keysize = 16; + else if (*keysize < 32) + *keysize = 24; + else + *keysize = 32; + + return CRYPT_OK; +} + +static int aes_ecb_encrypt_nblocks(const unsigned char *pt, unsigned char *ct, + unsigned long blocks, + const symmetric_key *skey) +{ + LTC_ARGCHK(pt); + LTC_ARGCHK(ct); + LTC_ARGCHK(skey); + + crypto_accel_aes_ecb_enc(ct, pt, skey->rijndael.eK, skey->rijndael.Nr, + blocks); + return CRYPT_OK; +} + +static int aes_ecb_decrypt_nblocks(const unsigned char *ct, unsigned char *pt, + unsigned long blocks, + const symmetric_key *skey) +{ + LTC_ARGCHK(pt); + LTC_ARGCHK(ct); + LTC_ARGCHK(skey); + + crypto_accel_aes_ecb_dec(pt, ct, skey->rijndael.dK, skey->rijndael.Nr, + blocks); + + return CRYPT_OK; +} + +int rijndael_ecb_encrypt(const unsigned char *pt, unsigned char *ct, + const symmetric_key *skey) +{ + return aes_ecb_encrypt_nblocks(pt, ct, 1, skey); +} + +int rijndael_ecb_decrypt(const unsigned char *ct, unsigned char *pt, + const symmetric_key *skey) +{ + return aes_ecb_decrypt_nblocks(ct, pt, 1, skey); +} + +static int aes_cbc_encrypt_nblocks(const unsigned char *pt, unsigned char *ct, + unsigned long blocks, unsigned char *IV, + symmetric_key *skey) +{ + LTC_ARGCHK(pt); + LTC_ARGCHK(ct); + LTC_ARGCHK(IV); + LTC_ARGCHK(skey); + + crypto_accel_aes_cbc_enc(ct, pt, skey->rijndael.eK, skey->rijndael.Nr, + blocks, IV); + + return CRYPT_OK; +} + +static int aes_cbc_decrypt_nblocks(const unsigned char *ct, unsigned char *pt, + unsigned long blocks, unsigned char *IV, + symmetric_key *skey) +{ + LTC_ARGCHK(pt); + LTC_ARGCHK(ct); + LTC_ARGCHK(IV); + LTC_ARGCHK(skey); + + crypto_accel_aes_cbc_dec(pt, ct, skey->rijndael.dK, skey->rijndael.Nr, + blocks, IV); + + return CRYPT_OK; +} + +static int aes_ctr_encrypt_nblocks(const unsigned char *pt, unsigned char *ct, + unsigned long blocks, unsigned char *IV, + int mode, symmetric_key *skey) +{ + LTC_ARGCHK(pt); + LTC_ARGCHK(ct); + LTC_ARGCHK(IV); + LTC_ARGCHK(skey); + + if (mode == CTR_COUNTER_LITTLE_ENDIAN) { + /* Accelerated algorithm supports big endian only */ + return CRYPT_ERROR; + } + + crypto_accel_aes_ctr_be_enc(ct, pt, skey->rijndael.eK, + skey->rijndael.Nr, blocks, IV); + + return CRYPT_OK; +} + +static int aes_xts_encrypt_nblocks(const unsigned char *pt, unsigned char *ct, + unsigned long blocks, unsigned char *tweak, + const symmetric_key *skey1, + const symmetric_key *skey2) +{ + LTC_ARGCHK(pt); + LTC_ARGCHK(ct); + LTC_ARGCHK(tweak); + LTC_ARGCHK(skey1); + LTC_ARGCHK(skey2); + LTC_ARGCHK(skey1->rijndael.Nr == skey2->rijndael.Nr); + + + crypto_accel_aes_xts_enc(ct, pt, skey1->rijndael.eK, + skey1->rijndael.Nr, blocks, + skey2->rijndael.eK, tweak); + + return CRYPT_OK; +} + +static int aes_xts_decrypt_nblocks(const unsigned char *ct, unsigned char *pt, + unsigned long blocks, unsigned char *tweak, + const symmetric_key *skey1, + const symmetric_key *skey2) +{ + LTC_ARGCHK(pt); + LTC_ARGCHK(ct); + LTC_ARGCHK(tweak); + LTC_ARGCHK(skey1); + LTC_ARGCHK(skey2); + LTC_ARGCHK(skey1->rijndael.Nr == skey2->rijndael.Nr); + + crypto_accel_aes_xts_dec(pt, ct, skey1->rijndael.dK, + skey1->rijndael.Nr, blocks, + skey2->rijndael.eK, tweak); + + return CRYPT_OK; +} + +const struct ltc_cipher_descriptor aes_desc = { + .name = "aes", + .ID = 6, + .min_key_length = 16, + .max_key_length = 32, + .block_length = 16, + .default_rounds = 10, + .setup = rijndael_setup, + .ecb_encrypt = rijndael_ecb_encrypt, + .ecb_decrypt = rijndael_ecb_decrypt, + .done = rijndael_done, + .keysize = rijndael_keysize, + .accel_ecb_encrypt = aes_ecb_encrypt_nblocks, + .accel_ecb_decrypt = aes_ecb_decrypt_nblocks, + .accel_cbc_encrypt = aes_cbc_encrypt_nblocks, + .accel_cbc_decrypt = aes_cbc_decrypt_nblocks, + .accel_ctr_encrypt = aes_ctr_encrypt_nblocks, + .accel_xts_encrypt = aes_xts_encrypt_nblocks, + .accel_xts_decrypt = aes_xts_decrypt_nblocks, +}; diff --git a/optee_os/core/lib/libtomcrypt/cbc.c b/optee_os/core/lib/libtomcrypt/cbc.c new file mode 100644 index 0000000..3a3b458 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/cbc.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "des2_key.h" + +struct ltc_cbc_ctx { + struct crypto_cipher_ctx ctx; + int cipher_idx; + bool des3; + int (*update)(const unsigned char *src, unsigned char *dst, + unsigned long len, symmetric_CBC *cbc); + symmetric_CBC state; +}; + +static const struct crypto_cipher_ops ltc_cbc_ops; + +static struct ltc_cbc_ctx *to_cbc_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == <c_cbc_ops); + + return container_of(ctx, struct ltc_cbc_ctx, ctx); +} + +static TEE_Result ltc_cbc_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv, size_t iv_len) +{ + struct ltc_cbc_ctx *c = to_cbc_ctx(ctx); + uint8_t tmp[24] = { 0 }; + const uint8_t *k = key1; + size_t kl = key1_len; + + if ((int)iv_len != cipher_descriptor[c->cipher_idx]->block_length) + return TEE_ERROR_BAD_PARAMETERS; + + if (mode == TEE_MODE_ENCRYPT) + c->update = cbc_encrypt; + else + c->update = cbc_decrypt; + + if (c->des3) + get_des2_key(&k, &kl, tmp); + + if (cbc_start(c->cipher_idx, iv, k, kl, 0, &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_cbc_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct ltc_cbc_ctx *c = to_cbc_ctx(ctx); + + if (c->update && c->update(data, dst, len, &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static void ltc_cbc_final(struct crypto_cipher_ctx *ctx) +{ + cbc_done(&to_cbc_ctx(ctx)->state); +} + +static void ltc_cbc_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_cbc_ctx(ctx)); +} + +static void ltc_cbc_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct ltc_cbc_ctx *src = to_cbc_ctx(src_ctx); + struct ltc_cbc_ctx *dst = to_cbc_ctx(dst_ctx); + + assert(src->cipher_idx == dst->cipher_idx); + dst->update = src->update; + dst->state = src->state; +} + +static const struct crypto_cipher_ops ltc_cbc_ops = { + .init = ltc_cbc_init, + .update = ltc_cbc_update, + .final = ltc_cbc_final, + .free_ctx = ltc_cbc_free_ctx, + .copy_state = ltc_cbc_copy_state, +}; + +static TEE_Result ltc_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx_ret, + int cipher_idx, bool des3) +{ + struct ltc_cbc_ctx *c = NULL; + + if (cipher_idx < 0) + return TEE_ERROR_NOT_SUPPORTED; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = <c_cbc_ops; + c->cipher_idx = cipher_idx; + c->des3 = des3; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} + +#if defined(_CFG_CORE_LTC_AES) +TEE_Result crypto_aes_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx) +{ + return ltc_cbc_alloc_ctx(ctx, find_cipher("aes"), false); +} +#endif + +#if defined(_CFG_CORE_LTC_DES) +TEE_Result crypto_des_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx) +{ + return ltc_cbc_alloc_ctx(ctx, find_cipher("des"), false); +} + +TEE_Result crypto_des3_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx) +{ + return ltc_cbc_alloc_ctx(ctx, find_cipher("3des"), true); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/ccm.c b/optee_os/core/lib/libtomcrypt/ccm.c new file mode 100644 index 0000000..17f3898 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/ccm.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TEE_CCM_KEY_MAX_LENGTH 32 +#define TEE_CCM_NONCE_MAX_LENGTH 13 +#define TEE_CCM_TAG_MAX_LENGTH 16 + +struct tee_ccm_state { + struct crypto_authenc_ctx aectx; + ccm_state ctx; /* the ccm state as defined by LTC */ + size_t tag_len; /* tag length */ +}; + +static const struct crypto_authenc_ops aes_ccm_ops; + +TEE_Result crypto_aes_ccm_alloc_ctx(struct crypto_authenc_ctx **ctx_ret) +{ + struct tee_ccm_state *ctx = calloc(1, sizeof(*ctx)); + + if (!ctx) + return TEE_ERROR_OUT_OF_MEMORY; + ctx->aectx.ops = &aes_ccm_ops; + + *ctx_ret = &ctx->aectx; + return TEE_SUCCESS; +} + +static struct tee_ccm_state *to_tee_ccm_state(struct crypto_authenc_ctx *aectx) +{ + assert(aectx && aectx->ops == &aes_ccm_ops); + + return container_of(aectx, struct tee_ccm_state, aectx); +} + +static void crypto_aes_ccm_free_ctx(struct crypto_authenc_ctx *aectx) +{ + free(to_tee_ccm_state(aectx)); +} + +static void crypto_aes_ccm_copy_state(struct crypto_authenc_ctx *dst_aectx, + struct crypto_authenc_ctx *src_aectx) +{ + struct tee_ccm_state *dst_ctx = to_tee_ccm_state(dst_aectx); + struct tee_ccm_state *src_ctx = to_tee_ccm_state(src_aectx); + + dst_ctx->ctx = src_ctx->ctx; + dst_ctx->tag_len = src_ctx->tag_len; +} + +static TEE_Result crypto_aes_ccm_init(struct crypto_authenc_ctx *aectx, + TEE_OperationMode mode __unused, + const uint8_t *key, size_t key_len, + const uint8_t *nonce, size_t nonce_len, + size_t tag_len, size_t aad_len, + size_t payload_len) +{ + int ltc_res = 0; + int ltc_cipherindex = find_cipher("aes"); + struct tee_ccm_state *ccm = to_tee_ccm_state(aectx); + + if (ltc_cipherindex < 0) + return TEE_ERROR_NOT_SUPPORTED; + + /* reset the state */ + memset(&ccm->ctx, 0, sizeof(ccm->ctx)); + ccm->tag_len = tag_len; + + /* Check the key length */ + if ((!key) || (key_len > TEE_CCM_KEY_MAX_LENGTH)) + return TEE_ERROR_BAD_PARAMETERS; + + /* check the nonce */ + if (nonce_len > TEE_CCM_NONCE_MAX_LENGTH) + return TEE_ERROR_BAD_PARAMETERS; + + /* Check that payload_len and aad_len will fit into ints */ + if (payload_len > INT_MAX || aad_len > INT_MAX) + return TEE_ERROR_BAD_PARAMETERS; + + /* check the tag len */ + if ((tag_len < 4) || (tag_len > TEE_CCM_TAG_MAX_LENGTH) || + (tag_len % 2 != 0)) + return TEE_ERROR_NOT_SUPPORTED; + + ltc_res = ccm_init(&ccm->ctx, ltc_cipherindex, key, key_len, + payload_len, tag_len, aad_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + /* Add the IV */ + ltc_res = ccm_add_nonce(&ccm->ctx, nonce, nonce_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result crypto_aes_ccm_update_aad(struct crypto_authenc_ctx *aectx, + const uint8_t *data, size_t len) +{ + struct tee_ccm_state *ccm = to_tee_ccm_state(aectx); + int ltc_res = 0; + + /* Add the AAD (note: aad can be NULL if aadlen == 0) */ + ltc_res = ccm_add_aad(&ccm->ctx, data, len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result +crypto_aes_ccm_update_payload(struct crypto_authenc_ctx *aectx, + TEE_OperationMode mode, const uint8_t *src_data, + size_t len, uint8_t *dst_data) +{ + int ltc_res = 0; + int dir = 0; + struct tee_ccm_state *ccm = to_tee_ccm_state(aectx); + unsigned char *pt = NULL; + unsigned char *ct = NULL; + + if (mode == TEE_MODE_ENCRYPT) { + pt = (unsigned char *)src_data; + ct = dst_data; + dir = CCM_ENCRYPT; + } else { + pt = dst_data; + ct = (unsigned char *)src_data; + dir = CCM_DECRYPT; + } + ltc_res = ccm_process(&ccm->ctx, pt, len, ct, dir); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result crypto_aes_ccm_enc_final(struct crypto_authenc_ctx *aectx, + const uint8_t *src_data, + size_t len, uint8_t *dst_data, + uint8_t *dst_tag, + size_t *dst_tag_len) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_ccm_state *ccm = to_tee_ccm_state(aectx); + int ltc_res = 0; + + /* Finalize the remaining buffer */ + res = crypto_aes_ccm_update_payload(aectx, TEE_MODE_ENCRYPT, src_data, + len, dst_data); + if (res != TEE_SUCCESS) + return res; + + /* Check the tag length */ + if (*dst_tag_len < ccm->tag_len) { + *dst_tag_len = ccm->tag_len; + return TEE_ERROR_SHORT_BUFFER; + } + *dst_tag_len = ccm->tag_len; + + /* Compute the tag */ + ltc_res = ccm_done(&ccm->ctx, dst_tag, + (unsigned long *)dst_tag_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result crypto_aes_ccm_dec_final(struct crypto_authenc_ctx *aectx, + const uint8_t *src_data, size_t len, + uint8_t *dst_data, + const uint8_t *tag, size_t tag_len) +{ + TEE_Result res = TEE_ERROR_BAD_STATE; + struct tee_ccm_state *ccm = to_tee_ccm_state(aectx); + int ltc_res = 0; + uint8_t dst_tag[TEE_CCM_TAG_MAX_LENGTH] = { 0 }; + unsigned long ltc_tag_len = tag_len; + + if (tag_len == 0) + return TEE_ERROR_SHORT_BUFFER; + if (tag_len > TEE_CCM_TAG_MAX_LENGTH) + return TEE_ERROR_BAD_STATE; + + /* Process the last buffer, if any */ + res = crypto_aes_ccm_update_payload(aectx, TEE_MODE_DECRYPT, src_data, + len, dst_data); + if (res != TEE_SUCCESS) + return res; + + /* Finalize the authentication */ + ltc_res = ccm_done(&ccm->ctx, dst_tag, <c_tag_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + if (consttime_memcmp(dst_tag, tag, tag_len) != 0) + res = TEE_ERROR_MAC_INVALID; + else + res = TEE_SUCCESS; + return res; +} + +static void crypto_aes_ccm_final(struct crypto_authenc_ctx *aectx) +{ + ccm_reset(&to_tee_ccm_state(aectx)->ctx); +} + +static const struct crypto_authenc_ops aes_ccm_ops = { + .init = crypto_aes_ccm_init, + .update_aad = crypto_aes_ccm_update_aad, + .update_payload = crypto_aes_ccm_update_payload, + .enc_final = crypto_aes_ccm_enc_final, + .dec_final = crypto_aes_ccm_dec_final, + .final = crypto_aes_ccm_final, + .free_ctx = crypto_aes_ccm_free_ctx, + .copy_state = crypto_aes_ccm_copy_state, +}; diff --git a/optee_os/core/lib/libtomcrypt/cmac.c b/optee_os/core/lib/libtomcrypt/cmac.c new file mode 100644 index 0000000..e8cad3c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/cmac.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + * Copyright (c) 2021, SumUp Services GmbH + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ltc_omac_ctx { + struct crypto_mac_ctx ctx; + int cipher_idx; + omac_state state; +}; + +static const struct crypto_mac_ops ltc_omac_ops; + +static struct ltc_omac_ctx *to_omac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx && ctx->ops == <c_omac_ops); + + return container_of(ctx, struct ltc_omac_ctx, ctx); +} + +static TEE_Result ltc_omac_init(struct crypto_mac_ctx *ctx, const uint8_t *key, + size_t len) +{ + struct ltc_omac_ctx *hc = to_omac_ctx(ctx); + + if (omac_init(&hc->state, hc->cipher_idx, key, len) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_omac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + if (omac_process(&to_omac_ctx(ctx)->state, data, len) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_omac_final(struct crypto_mac_ctx *ctx, uint8_t *digest, + size_t len) +{ + unsigned long l = len; + + if (omac_done(&to_omac_ctx(ctx)->state, digest, &l) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static void ltc_omac_free_ctx(struct crypto_mac_ctx *ctx) +{ + free(to_omac_ctx(ctx)); +} + +static void ltc_omac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct ltc_omac_ctx *src = to_omac_ctx(src_ctx); + struct ltc_omac_ctx *dst = to_omac_ctx(dst_ctx); + + assert(src->cipher_idx == dst->cipher_idx); + dst->state = src->state; +} + +static const struct crypto_mac_ops ltc_omac_ops = { + .init = ltc_omac_init, + .update = ltc_omac_update, + .final = ltc_omac_final, + .free_ctx = ltc_omac_free_ctx, + .copy_state = ltc_omac_copy_state, +}; + +static TEE_Result crypto_common_cmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret, + const char *cipher) +{ + struct ltc_omac_ctx *ctx = NULL; + int cipher_idx = find_cipher(cipher); + + if (!ctx_ret) + return TEE_ERROR_BAD_PARAMETERS; + + if (cipher_idx < 0) + return TEE_ERROR_NOT_SUPPORTED; + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) + return TEE_ERROR_OUT_OF_MEMORY; + + ctx->ctx.ops = <c_omac_ops; + ctx->cipher_idx = cipher_idx; + *ctx_ret = &ctx->ctx; + + return TEE_SUCCESS; +} + +TEE_Result crypto_aes_cmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret) +{ + return crypto_common_cmac_alloc_ctx(ctx_ret, "aes"); +} + +TEE_Result crypto_des3_cmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret) +{ + return crypto_common_cmac_alloc_ctx(ctx_ret, "3des"); +} diff --git a/optee_os/core/lib/libtomcrypt/ctr.c b/optee_os/core/lib/libtomcrypt/ctr.c new file mode 100644 index 0000000..1462f23 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/ctr.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +struct ltc_ctr_ctx { + struct crypto_cipher_ctx ctx; + int cipher_idx; + int (*update)(const unsigned char *src, unsigned char *dst, + unsigned long len, symmetric_CTR *ctr); + symmetric_CTR state; +}; + +static const struct crypto_cipher_ops ltc_ctr_ops; + +static struct ltc_ctr_ctx *to_ctr_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == <c_ctr_ops); + + return container_of(ctx, struct ltc_ctr_ctx, ctx); +} + +static TEE_Result ltc_ctr_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct ltc_ctr_ctx *c = to_ctr_ctx(ctx); + + if ((int)iv_len != cipher_descriptor[c->cipher_idx]->block_length) + return TEE_ERROR_BAD_PARAMETERS; + + if (mode == TEE_MODE_ENCRYPT) + c->update = ctr_encrypt; + else + c->update = ctr_decrypt; + + if (ctr_start(c->cipher_idx, iv, key1, key1_len, 0, + CTR_COUNTER_BIG_ENDIAN, &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_ctr_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct ltc_ctr_ctx *c = to_ctr_ctx(ctx); + + if (c->update && c->update(data, dst, len, &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static void ltc_ctr_final(struct crypto_cipher_ctx *ctx) +{ + ctr_done(&to_ctr_ctx(ctx)->state); +} + +static void ltc_ctr_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_ctr_ctx(ctx)); +} + +static void ltc_ctr_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct ltc_ctr_ctx *src = to_ctr_ctx(src_ctx); + struct ltc_ctr_ctx *dst = to_ctr_ctx(dst_ctx); + + assert(src->cipher_idx == dst->cipher_idx); + dst->update = src->update; + dst->state = src->state; +} + +static const struct crypto_cipher_ops ltc_ctr_ops = { + .init = ltc_ctr_init, + .update = ltc_ctr_update, + .final = ltc_ctr_final, + .free_ctx = ltc_ctr_free_ctx, + .copy_state = ltc_ctr_copy_state, +}; + +TEE_Result crypto_aes_ctr_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct ltc_ctr_ctx *c = NULL; + int cipher_idx = find_cipher("aes"); + + if (cipher_idx < 0) + return TEE_ERROR_NOT_SUPPORTED; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = <c_ctr_ops; + c->cipher_idx = cipher_idx; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/lib/libtomcrypt/des2_key.h b/optee_os/core/lib/libtomcrypt/des2_key.h new file mode 100644 index 0000000..1835a27 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/des2_key.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +static inline void get_des2_key(const uint8_t **key, size_t *key_len, + uint8_t *tmp) +{ + if (*key_len == 16) { + /* + * This corresponds to a 2DES key. The 2DES encryption + * algorithm is similar to 3DES. Both perform and + * encryption step, then a decryption step, followed + * by another encryption step (EDE). However 2DES uses + * the same key for both of the encryption (E) steps. + */ + memcpy(tmp, *key, 16); + memcpy(tmp + 16, *key, 8); + *key = tmp; + *key_len = 24; + } +} diff --git a/optee_os/core/lib/libtomcrypt/dh.c b/optee_os/core/lib/libtomcrypt/dh.c new file mode 100644 index 0000000..b1d0a4d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/dh.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +TEE_Result crypto_acipher_alloc_dh_keypair(struct dh_keypair *s, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->g)) + return TEE_ERROR_OUT_OF_MEMORY; + if (!bn_alloc_max(&s->p)) + goto err; + if (!bn_alloc_max(&s->y)) + goto err; + if (!bn_alloc_max(&s->x)) + goto err; + if (!bn_alloc_max(&s->q)) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->g); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->y); + crypto_bignum_free(&s->x); + return TEE_ERROR_OUT_OF_MEMORY; +} + +TEE_Result crypto_acipher_gen_dh_key(struct dh_keypair *key, struct bignum *q, + size_t xbits, size_t key_size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + dh_key ltc_tmp_key = { }; + int ltc_res = 0; + + if (key_size != 8 * mp_unsigned_bin_size(key->p)) + return TEE_ERROR_BAD_PARAMETERS; + + ltc_res = mp_init_multi(<c_tmp_key.base, <c_tmp_key.prime, NULL); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_OUT_OF_MEMORY; + + /* Generate the DH key */ + mp_copy(key->g, ltc_tmp_key.base); + mp_copy(key->p, ltc_tmp_key.prime); + ltc_res = dh_make_key(NULL, find_prng("prng_crypto"), q, xbits, + <c_tmp_key); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_PARAMETERS; + } else { + ltc_mp.copy(ltc_tmp_key.y, key->y); + ltc_mp.copy(ltc_tmp_key.x, key->x); + res = TEE_SUCCESS; + } + + dh_free(<c_tmp_key); + return res; +} + +TEE_Result crypto_acipher_dh_shared_secret(struct dh_keypair *private_key, + struct bignum *public_key, + struct bignum *secret) +{ + int err; + + if (!private_key || !public_key || !secret) + return TEE_ERROR_BAD_PARAMETERS; + + err = mp_exptmod(public_key, private_key->x, private_key->p, secret); + return ((err == CRYPT_OK) ? TEE_SUCCESS : TEE_ERROR_BAD_PARAMETERS); + +} diff --git a/optee_os/core/lib/libtomcrypt/dsa.c b/optee_os/core/lib/libtomcrypt/dsa.c new file mode 100644 index 0000000..d6243c4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/dsa.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +TEE_Result crypto_acipher_alloc_dsa_keypair(struct dsa_keypair *s, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->g)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (!bn_alloc_max(&s->p)) + goto err; + if (!bn_alloc_max(&s->q)) + goto err; + if (!bn_alloc_max(&s->y)) + goto err; + if (!bn_alloc_max(&s->x)) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->g); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->q); + crypto_bignum_free(&s->y); + return TEE_ERROR_OUT_OF_MEMORY; +} + +TEE_Result crypto_acipher_alloc_dsa_public_key(struct dsa_public_key *s, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->g)) + return TEE_ERROR_OUT_OF_MEMORY; + + if (!bn_alloc_max(&s->p)) + goto err; + if (!bn_alloc_max(&s->q)) + goto err; + if (!bn_alloc_max(&s->y)) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->g); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->q); + return TEE_ERROR_OUT_OF_MEMORY; +} + +TEE_Result crypto_acipher_gen_dsa_key(struct dsa_keypair *key, size_t key_size) +{ + dsa_key ltc_tmp_key = { }; + int ltc_res = 0; + + if (key_size != 8 * mp_unsigned_bin_size(key->p)) + return TEE_ERROR_BAD_PARAMETERS; + + ltc_res = mp_init_multi(<c_tmp_key.g, <c_tmp_key.p, <c_tmp_key.q, + <c_tmp_key.x, <c_tmp_key.y, NULL); + if (ltc_res) + return TEE_ERROR_OUT_OF_MEMORY; + + /* Copy the key parameters */ + mp_copy(key->g, ltc_tmp_key.g); + mp_copy(key->p, ltc_tmp_key.p); + mp_copy(key->q, ltc_tmp_key.q); + + /* Generate the DSA key */ + ltc_res = dsa_generate_key(NULL, find_prng("prng_crypto"), + <c_tmp_key); + if (ltc_res) + return TEE_ERROR_BAD_PARAMETERS; + + /* Copy the key */ + mp_copy(ltc_tmp_key.y, key->y); + mp_copy(ltc_tmp_key.x, key->x); + + /* Free the temporary key */ + dsa_free(<c_tmp_key); + + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_dsa_sign(uint32_t algo, struct dsa_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + TEE_Result res; + size_t hash_size; + int ltc_res; + void *r, *s; + dsa_key ltc_key = { + .type = PK_PRIVATE, + .qord = mp_unsigned_bin_size(key->q), + .g = key->g, + .p = key->p, + .q = key->q, + .y = key->y, + .x = key->x, + }; + + if (algo != TEE_ALG_DSA_SHA1 && + algo != TEE_ALG_DSA_SHA224 && + algo != TEE_ALG_DSA_SHA256) { + res = TEE_ERROR_NOT_IMPLEMENTED; + goto err; + } + + res = tee_alg_get_digest_size(TEE_DIGEST_HASH_TO_ALGO(algo), + &hash_size); + if (res != TEE_SUCCESS) + goto err; + if (mp_unsigned_bin_size(ltc_key.q) < hash_size) + hash_size = mp_unsigned_bin_size(ltc_key.q); + if (msg_len != hash_size) { + res = TEE_ERROR_SECURITY; + goto err; + } + + if (*sig_len < 2 * mp_unsigned_bin_size(ltc_key.q)) { + *sig_len = 2 * mp_unsigned_bin_size(ltc_key.q); + res = TEE_ERROR_SHORT_BUFFER; + goto err; + } + + ltc_res = mp_init_multi(&r, &s, NULL); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + ltc_res = dsa_sign_hash_raw(msg, msg_len, r, s, NULL, + find_prng("prng_crypto"), <c_key); + + if (ltc_res == CRYPT_OK) { + *sig_len = 2 * mp_unsigned_bin_size(ltc_key.q); + memset(sig, 0, *sig_len); + mp_to_unsigned_bin(r, (uint8_t *)sig + *sig_len/2 - + mp_unsigned_bin_size(r)); + mp_to_unsigned_bin(s, (uint8_t *)sig + *sig_len - + mp_unsigned_bin_size(s)); + res = TEE_SUCCESS; + } else { + res = TEE_ERROR_GENERIC; + } + + mp_clear_multi(r, s, NULL); + +err: + return res; +} + +TEE_Result crypto_acipher_dsa_verify(uint32_t algo, struct dsa_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result res; + int ltc_stat, ltc_res; + void *r, *s; + dsa_key ltc_key = { + .type = PK_PUBLIC, + .qord = mp_unsigned_bin_size(key->q), + .g = key->g, + .p = key->p, + .q = key->q, + .y = key->y + }; + + if (algo != TEE_ALG_DSA_SHA1 && + algo != TEE_ALG_DSA_SHA224 && + algo != TEE_ALG_DSA_SHA256) { + res = TEE_ERROR_NOT_IMPLEMENTED; + goto err; + } + + ltc_res = mp_init_multi(&r, &s, NULL); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + mp_read_unsigned_bin(r, (uint8_t *)sig, sig_len/2); + mp_read_unsigned_bin(s, (uint8_t *)sig + sig_len/2, sig_len/2); + ltc_res = dsa_verify_hash_raw(r, s, msg, msg_len, <c_stat, <c_key); + mp_clear_multi(r, s, NULL); + res = convert_ltc_verify_status(ltc_res, ltc_stat); +err: + return res; +} diff --git a/optee_os/core/lib/libtomcrypt/ecb.c b/optee_os/core/lib/libtomcrypt/ecb.c new file mode 100644 index 0000000..c57f502 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/ecb.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "des2_key.h" + +struct ltc_ecb_ctx { + struct crypto_cipher_ctx ctx; + int cipher_idx; + bool des3; + int (*update)(const unsigned char *src, unsigned char *dst, + unsigned long len, symmetric_ECB *ecb); + symmetric_ECB state; +}; + +static const struct crypto_cipher_ops ltc_ecb_ops; + +static struct ltc_ecb_ctx *to_ecb_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == <c_ecb_ops); + + return container_of(ctx, struct ltc_ecb_ctx, ctx); +} + +static TEE_Result ltc_ecb_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct ltc_ecb_ctx *c = to_ecb_ctx(ctx); + uint8_t tmp[24] = { 0 }; + const uint8_t *k = key1; + size_t kl = key1_len; + + if (mode == TEE_MODE_ENCRYPT) + c->update = ecb_encrypt; + else + c->update = ecb_decrypt; + + if (c->des3) + get_des2_key(&k, &kl, tmp); + + if (ecb_start(c->cipher_idx, k, kl, 0, &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_ecb_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct ltc_ecb_ctx *c = to_ecb_ctx(ctx); + + if (c->update && c->update(data, dst, len, &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static void ltc_ecb_final(struct crypto_cipher_ctx *ctx) +{ + ecb_done(&to_ecb_ctx(ctx)->state); +} + +static void ltc_ecb_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_ecb_ctx(ctx)); +} + +static void ltc_ecb_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct ltc_ecb_ctx *src = to_ecb_ctx(src_ctx); + struct ltc_ecb_ctx *dst = to_ecb_ctx(dst_ctx); + + assert(src->cipher_idx == dst->cipher_idx); + dst->update = src->update; + dst->state = src->state; +} + +static const struct crypto_cipher_ops ltc_ecb_ops = { + .init = ltc_ecb_init, + .update = ltc_ecb_update, + .final = ltc_ecb_final, + .free_ctx = ltc_ecb_free_ctx, + .copy_state = ltc_ecb_copy_state, +}; + +static TEE_Result ltc_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx_ret, + int cipher_idx, bool des3) +{ + struct ltc_ecb_ctx *c = NULL; + + if (cipher_idx < 0) + return TEE_ERROR_NOT_SUPPORTED; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = <c_ecb_ops; + c->cipher_idx = cipher_idx; + c->des3 = des3; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} + +#if defined(_CFG_CORE_LTC_AES) +TEE_Result crypto_aes_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx) +{ + return ltc_ecb_alloc_ctx(ctx, find_cipher("aes"), false); +} +#endif + +#if defined(_CFG_CORE_LTC_DES) +TEE_Result crypto_des_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx) +{ + return ltc_ecb_alloc_ctx(ctx, find_cipher("des"), false); +} + +TEE_Result crypto_des3_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx) +{ + return ltc_ecb_alloc_ctx(ctx, find_cipher("3des"), true); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/ecc.c b/optee_os/core/lib/libtomcrypt/ecc.c new file mode 100644 index 0000000..ac05212 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/ecc.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +static void _ltc_ecc_free_public_key(struct ecc_public_key *s) +{ + if (!s) + return; + + crypto_bignum_free(&s->x); + crypto_bignum_free(&s->y); +} + +/* + * For a given TEE @curve, return key size and LTC curve name. Also check that + * @algo is compatible with this curve. + * @curve: TEE_ECC_CURVE_NIST_P192, ... + * @algo: TEE_ALG_ECDSA_SHA1, ... + */ +static TEE_Result ecc_get_curve_info(uint32_t curve, uint32_t algo, + size_t *key_size_bytes, + size_t *key_size_bits, + const char **curve_name) +{ + size_t size_bytes = 0; + size_t size_bits = 0; + const char *name = NULL; + + /* + * Excerpt of libtomcrypt documentation: + * ecc_make_key(... key_size ...): The keysize is the size of the + * modulus in bytes desired. Currently directly supported values + * are 12, 16, 20, 24, 28, 32, 48, and 65 bytes which correspond + * to key sizes of 112, 128, 160, 192, 224, 256, 384, and 521 bits + * respectively. + */ + + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + size_bits = 192; + size_bytes = 24; + name = "NISTP192"; + if ((algo != 0) && (algo != TEE_ALG_ECDSA_SHA1) && + (algo != TEE_ALG_ECDH_DERIVE_SHARED_SECRET)) + return TEE_ERROR_BAD_PARAMETERS; + break; + case TEE_ECC_CURVE_NIST_P224: + size_bits = 224; + size_bytes = 28; + name = "NISTP224"; + if ((algo != 0) && (algo != TEE_ALG_ECDSA_SHA224) && + (algo != TEE_ALG_ECDH_DERIVE_SHARED_SECRET)) + return TEE_ERROR_BAD_PARAMETERS; + break; + case TEE_ECC_CURVE_NIST_P256: + size_bits = 256; + size_bytes = 32; + name = "NISTP256"; + if ((algo != 0) && (algo != TEE_ALG_ECDSA_SHA256) && + (algo != TEE_ALG_ECDH_DERIVE_SHARED_SECRET)) + return TEE_ERROR_BAD_PARAMETERS; + break; + case TEE_ECC_CURVE_NIST_P384: + size_bits = 384; + size_bytes = 48; + name = "NISTP384"; + if ((algo != 0) && (algo != TEE_ALG_ECDSA_SHA384) && + (algo != TEE_ALG_ECDH_DERIVE_SHARED_SECRET)) + return TEE_ERROR_BAD_PARAMETERS; + break; + case TEE_ECC_CURVE_NIST_P521: + size_bits = 521; + size_bytes = 66; + name = "NISTP521"; + if ((algo != 0) && (algo != TEE_ALG_ECDSA_SHA512) && + (algo != TEE_ALG_ECDH_DERIVE_SHARED_SECRET)) + return TEE_ERROR_BAD_PARAMETERS; + break; + case TEE_ECC_CURVE_SM2: + size_bits = 256; + size_bytes = 32; + name = "SM2"; + if ((algo != 0) && (algo != TEE_ALG_SM2_PKE) && + (algo != TEE_ALG_SM2_DSA_SM3) && + (algo != TEE_ALG_SM2_KEP)) + return TEE_ERROR_BAD_PARAMETERS; + break; + default: + return TEE_ERROR_NOT_SUPPORTED; + } + + if (key_size_bytes) + *key_size_bytes = size_bytes; + if (key_size_bits) + *key_size_bits = size_bits; + if (curve_name) + *curve_name = name; + return TEE_SUCCESS; +} + +/* Note: this function clears the key before setting the curve */ +static TEE_Result ecc_set_curve_from_name(ecc_key *ltc_key, + const char *curve_name) +{ + const ltc_ecc_curve *curve = NULL; + int ltc_res = 0; + + ltc_res = ecc_find_curve(curve_name, &curve); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_NOT_SUPPORTED; + + ltc_res = ecc_set_curve(curve, ltc_key); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result _ltc_ecc_generate_keypair(struct ecc_keypair *key, + size_t key_size) +{ + TEE_Result res; + ecc_key ltc_tmp_key; + int ltc_res; + size_t key_size_bytes = 0; + size_t key_size_bits = 0; + const char *name = NULL; + + res = ecc_get_curve_info(key->curve, 0, &key_size_bytes, &key_size_bits, + &name); + if (res != TEE_SUCCESS) + return res; + + if (key_size != key_size_bits) + return TEE_ERROR_BAD_PARAMETERS; + + res = ecc_set_curve_from_name(<c_tmp_key, name); + if (res) + return res; + + /* Generate the ECC key */ + ltc_res = ecc_generate_key(NULL, find_prng("prng_crypto"), + <c_tmp_key); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + /* check the size of the keys */ + if (((size_t)mp_count_bits(ltc_tmp_key.pubkey.x) > key_size_bits) || + ((size_t)mp_count_bits(ltc_tmp_key.pubkey.y) > key_size_bits) || + ((size_t)mp_count_bits(ltc_tmp_key.k) > key_size_bits)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + /* check LTC is returning z==1 */ + if (mp_count_bits(ltc_tmp_key.pubkey.z) != 1) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + /* Copy the key */ + ltc_mp.copy(ltc_tmp_key.k, key->d); + ltc_mp.copy(ltc_tmp_key.pubkey.x, key->x); + ltc_mp.copy(ltc_tmp_key.pubkey.y, key->y); + + res = TEE_SUCCESS; + +exit: + ecc_free(<c_tmp_key); /* Free the temporary key */ + return res; +} + +/* + * Given a keypair "key", populate the Libtomcrypt private key "ltc_key" + * It also returns the key size, in bytes + */ +TEE_Result ecc_populate_ltc_private_key(ecc_key *ltc_key, + struct ecc_keypair *key, + uint32_t algo, size_t *key_size_bytes) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const char *name = NULL; + + res = ecc_get_curve_info(key->curve, algo, key_size_bytes, NULL, &name); + if (res) + return res; + + memset(ltc_key, 0, sizeof(*ltc_key)); + + res = ecc_set_curve_from_name(ltc_key, name); + if (res) + return res; + + ltc_key->type = PK_PRIVATE; + mp_copy(key->d, ltc_key->k); + mp_copy(key->x, ltc_key->pubkey.x); + mp_copy(key->y, ltc_key->pubkey.y); + mp_set_int(ltc_key->pubkey.z, 1); + + return TEE_SUCCESS; +} + +/* + * Given a public "key", populate the Libtomcrypt public key "ltc_key" + * It also returns the key size, in bytes + */ +TEE_Result ecc_populate_ltc_public_key(ecc_key *ltc_key, + struct ecc_public_key *key, + uint32_t algo, size_t *key_size_bytes) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const char *name = NULL; + uint8_t one[1] = { 1 }; + + res = ecc_get_curve_info(key->curve, algo, key_size_bytes, NULL, &name); + if (res) + return res; + + memset(ltc_key, 0, sizeof(*ltc_key)); + + res = ecc_set_curve_from_name(ltc_key, name); + if (res) + return res; + + ltc_key->type = PK_PUBLIC; + + mp_copy(key->x, ltc_key->pubkey.x); + mp_copy(key->y, ltc_key->pubkey.y); + mp_read_unsigned_bin(ltc_key->pubkey.z, one, sizeof(one)); + + return TEE_SUCCESS; +} + +static TEE_Result _ltc_ecc_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int ltc_res = 0; + size_t key_size_bytes = 0; + ecc_key ltc_key = { }; + unsigned long ltc_sig_len = 0; + + if (algo == 0) + return TEE_ERROR_BAD_PARAMETERS; + + res = ecc_populate_ltc_private_key(<c_key, key, algo, + &key_size_bytes); + if (res != TEE_SUCCESS) + return res; + + if (*sig_len < 2 * key_size_bytes) { + *sig_len = 2 * key_size_bytes; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + ltc_sig_len = *sig_len; + ltc_res = ecc_sign_hash_rfc7518(msg, msg_len, sig, <c_sig_len, + NULL, find_prng("prng_crypto"), <c_key); + if (ltc_res == CRYPT_OK) { + res = TEE_SUCCESS; + } else { + res = TEE_ERROR_GENERIC; + } + *sig_len = ltc_sig_len; + +out: + ecc_free(<c_key); + return res; +} + +static TEE_Result _ltc_ecc_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int ltc_stat = 0; + int ltc_res = 0; + size_t key_size_bytes = 0; + ecc_key ltc_key = { }; + + if (algo == 0) + return TEE_ERROR_BAD_PARAMETERS; + + res = ecc_populate_ltc_public_key(<c_key, key, algo, &key_size_bytes); + if (res != TEE_SUCCESS) + goto out; + + /* check keysize vs sig_len */ + if ((key_size_bytes * 2) != sig_len) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + ltc_res = ecc_verify_hash_rfc7518(sig, sig_len, msg, msg_len, <c_stat, + <c_key); + res = convert_ltc_verify_status(ltc_res, ltc_stat); +out: + ecc_free(<c_key); + return res; +} + +static TEE_Result _ltc_ecc_shared_secret(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, + unsigned long *secret_len) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int ltc_res = 0; + ecc_key ltc_private_key = { }; + ecc_key ltc_public_key = { }; + size_t key_size_bytes = 0; + + /* Check the curves are the same */ + if (private_key->curve != public_key->curve) + return TEE_ERROR_BAD_PARAMETERS; + + res = ecc_populate_ltc_private_key(<c_private_key, private_key, + 0, &key_size_bytes); + if (res != TEE_SUCCESS) + goto out; + res = ecc_populate_ltc_public_key(<c_public_key, public_key, + 0, &key_size_bytes); + if (res != TEE_SUCCESS) + goto out; + + ltc_res = ecc_shared_secret(<c_private_key, <c_public_key, + secret, secret_len); + if (ltc_res == CRYPT_OK) + res = TEE_SUCCESS; + else + res = TEE_ERROR_BAD_PARAMETERS; + +out: + ecc_free(<c_private_key); + ecc_free(<c_public_key); + return res; +} + +static const struct crypto_ecc_keypair_ops ecc_keypair_ops = { + .generate = _ltc_ecc_generate_keypair, + .sign = _ltc_ecc_sign, + .shared_secret = _ltc_ecc_shared_secret, +}; + +static const struct crypto_ecc_public_ops ecc_public_key_ops = { + .free = _ltc_ecc_free_public_key, + .verify = _ltc_ecc_verify, +}; + +static const struct crypto_ecc_keypair_ops sm2_dsa_keypair_ops = { + .generate = _ltc_ecc_generate_keypair, + .sign = sm2_ltc_dsa_sign, +}; + +static const struct crypto_ecc_public_ops sm2_dsa_public_key_ops = { + .free = _ltc_ecc_free_public_key, + .verify = sm2_ltc_dsa_verify, +}; + +static const struct crypto_ecc_keypair_ops sm2_pke_keypair_ops = { + .generate = _ltc_ecc_generate_keypair, + .decrypt = sm2_ltc_pke_decrypt, +}; + +static const struct crypto_ecc_public_ops sm2_pke_public_key_ops = { + .free = _ltc_ecc_free_public_key, + .encrypt = sm2_ltc_pke_encrypt, +}; + +static const struct crypto_ecc_keypair_ops sm2_kep_keypair_ops = { + .generate = _ltc_ecc_generate_keypair, +}; + +static const struct crypto_ecc_public_ops sm2_kep_public_key_ops = { + .free = _ltc_ecc_free_public_key, +}; + +const struct crypto_ecc_keypair_ops * +crypto_asym_get_ecc_keypair_ops( uint32_t key_type) +{ + switch (key_type) { + case TEE_TYPE_ECDSA_KEYPAIR: + case TEE_TYPE_ECDH_KEYPAIR: + return &ecc_keypair_ops; + case TEE_TYPE_SM2_DSA_KEYPAIR: + if (!IS_ENABLED(_CFG_CORE_LTC_SM2_DSA)) + return NULL; + return &sm2_dsa_keypair_ops; + case TEE_TYPE_SM2_PKE_KEYPAIR: + if (!IS_ENABLED(_CFG_CORE_LTC_SM2_PKE)) + return NULL; + return &sm2_pke_keypair_ops; + case TEE_TYPE_SM2_KEP_KEYPAIR: + if (!IS_ENABLED(_CFG_CORE_LTC_SM2_KEP)) + return NULL; + return &sm2_kep_keypair_ops; + default: + return NULL; + } +} + +TEE_Result crypto_asym_alloc_ecc_keypair(struct ecc_keypair *s, + uint32_t key_type, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + + switch (key_type) { + case TEE_TYPE_ECDSA_KEYPAIR: + case TEE_TYPE_ECDH_KEYPAIR: + s->ops = &ecc_keypair_ops; + break; + case TEE_TYPE_SM2_DSA_KEYPAIR: + if (!IS_ENABLED2(_CFG_CORE_LTC_SM2_DSA)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_dsa_keypair_ops; + break; + case TEE_TYPE_SM2_PKE_KEYPAIR: + if (!IS_ENABLED2(_CFG_CORE_LTC_SM2_PKE)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_pke_keypair_ops; + break; + case TEE_TYPE_SM2_KEP_KEYPAIR: + if (!IS_ENABLED2(_CFG_CORE_LTC_SM2_KEP)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_kep_keypair_ops; + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + + if (!bn_alloc_max(&s->d)) + goto err; + if (!bn_alloc_max(&s->x)) + goto err; + if (!bn_alloc_max(&s->y)) + goto err; + + return TEE_SUCCESS; + +err: + s->ops = NULL; + + crypto_bignum_free(&s->d); + crypto_bignum_free(&s->x); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +const struct crypto_ecc_public_ops* +crypto_asym_get_ecc_public_ops(uint32_t key_type) +{ + switch (key_type) { + case TEE_TYPE_ECDSA_PUBLIC_KEY: + case TEE_TYPE_ECDH_PUBLIC_KEY: + return &ecc_public_key_ops; + case TEE_TYPE_SM2_DSA_PUBLIC_KEY: + if (!IS_ENABLED(_CFG_CORE_LTC_SM2_DSA)) + return NULL; + return &sm2_dsa_public_key_ops; + case TEE_TYPE_SM2_PKE_PUBLIC_KEY: + if (!IS_ENABLED(_CFG_CORE_LTC_SM2_PKE)) + return NULL; + return &sm2_pke_public_key_ops; + case TEE_TYPE_SM2_KEP_PUBLIC_KEY: + if (!IS_ENABLED(_CFG_CORE_LTC_SM2_KEP)) + return NULL; + return &sm2_kep_public_key_ops; + default: + return NULL; + } +} + +TEE_Result crypto_asym_alloc_ecc_public_key(struct ecc_public_key *s, + uint32_t key_type, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + + switch (key_type) { + case TEE_TYPE_ECDSA_PUBLIC_KEY: + case TEE_TYPE_ECDH_PUBLIC_KEY: + s->ops = &ecc_public_key_ops; + break; + case TEE_TYPE_SM2_DSA_PUBLIC_KEY: + if (!IS_ENABLED2(_CFG_CORE_LTC_SM2_DSA)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_dsa_public_key_ops; + break; + case TEE_TYPE_SM2_PKE_PUBLIC_KEY: + if (!IS_ENABLED2(_CFG_CORE_LTC_SM2_PKE)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_pke_public_key_ops; + break; + case TEE_TYPE_SM2_KEP_PUBLIC_KEY: + if (!IS_ENABLED2(_CFG_CORE_LTC_SM2_KEP)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_kep_public_key_ops; + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + + if (!bn_alloc_max(&s->x)) + goto err; + if (!bn_alloc_max(&s->y)) + goto err; + + return TEE_SUCCESS; + +err: + s->ops = NULL; + + crypto_bignum_free(&s->x); + + return TEE_ERROR_OUT_OF_MEMORY; +} diff --git a/optee_os/core/lib/libtomcrypt/ed25519.c b/optee_os/core/lib/libtomcrypt/ed25519.c new file mode 100644 index 0000000..763c097 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/ed25519.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Technology Innovation Institute (TII) + * Copyright (c) 2022, EPAM Systems + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +#define ED25519_KEY_SIZE UL(256) + +TEE_Result crypto_acipher_alloc_ed25519_keypair(struct ed25519_keypair *key, + size_t key_size) +{ + if (!key || key_size != ED25519_KEY_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + memset(key, 0, sizeof(*key)); + + key->priv = calloc(1, key_size >> 3); + key->pub = calloc(1, key_size >> 3); + + if (!key->priv || !key->pub) { + free(key->priv); + free(key->pub); + return TEE_ERROR_OUT_OF_MEMORY; + } + + return TEE_SUCCESS; +} + +TEE_Result +crypto_acipher_alloc_ed25519_public_key(struct ed25519_public_key *key, + size_t key_size) +{ + if (!key || key_size != ED25519_KEY_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + memset(key, 0, sizeof(*key)); + + key->pub = calloc(1, key_size >> 3); + + if (!key->pub) + return TEE_ERROR_OUT_OF_MEMORY; + + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_gen_ed25519_key(struct ed25519_keypair *key, + size_t key_size) +{ + curve25519_key ltc_tmp_key = { }; + + if (key_size != ED25519_KEY_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + if (ed25519_make_key(NULL, find_prng("prng_crypto"), + <c_tmp_key) != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + assert(key_size >= sizeof(ltc_tmp_key.pub) && + key_size >= sizeof(ltc_tmp_key.priv)); + + memcpy(key->pub, ltc_tmp_key.pub, sizeof(ltc_tmp_key.pub)); + memcpy(key->priv, ltc_tmp_key.priv, sizeof(ltc_tmp_key.priv)); + memzero_explicit(<c_tmp_key, sizeof(ltc_tmp_key)); + + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_ed25519_sign(struct ed25519_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + int err; + unsigned long siglen = 0; + curve25519_key private_key = { + .type = PK_PRIVATE, + .algo = LTC_OID_ED25519, + }; + + if (!key || !sig_len) + return TEE_ERROR_BAD_PARAMETERS; + + siglen = *sig_len; + + memcpy(private_key.priv, key->priv, sizeof(private_key.priv)); + memcpy(private_key.pub, key->pub, sizeof(private_key.pub)); + + err = ed25519_sign(msg, msg_len, sig, &siglen, &private_key); + + memzero_explicit(&private_key, sizeof(private_key)); + + if (err != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + *sig_len = siglen; + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_ed25519ctx_sign(struct ed25519_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len, + bool ph_flag, + const uint8_t *ctx, size_t ctxlen) +{ + int err = CRYPT_ERROR; + unsigned long siglen = 0; + curve25519_key private_key = { + .type = PK_PRIVATE, + .algo = LTC_OID_ED25519, + }; + + if (!key || !sig_len) + return TEE_ERROR_BAD_PARAMETERS; + + siglen = *sig_len; + + memcpy(private_key.priv, key->priv, sizeof(private_key.priv)); + memcpy(private_key.pub, key->pub, sizeof(private_key.pub)); + + if (ph_flag) { + err = ed25519ph_sign(msg, msg_len, sig, &siglen, + ctx, ctxlen, &private_key); + } else { + err = ed25519ctx_sign(msg, msg_len, sig, &siglen, + ctx, ctxlen, &private_key); + } + + memzero_explicit(&private_key, sizeof(private_key)); + + if (err != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + *sig_len = siglen; + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_ed25519_verify(struct ed25519_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + int stat = 0; + curve25519_key public_key = { + .type = PK_PUBLIC, + .algo = LTC_OID_ED25519, + }; + + if (!key) + return TEE_ERROR_BAD_PARAMETERS; + + memcpy(public_key.pub, key->pub, sizeof(public_key.pub)); + + if (ed25519_verify(msg, msg_len, sig, sig_len, &stat, + &public_key) != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + if (stat != 1) + return TEE_ERROR_SIGNATURE_INVALID; + + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_ed25519ctx_verify(struct ed25519_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len, + bool ph_flag, + const uint8_t *ctx, size_t ctxlen) +{ + int stat = 0; + curve25519_key public_key = { + .type = PK_PUBLIC, + .algo = LTC_OID_ED25519, + }; + + if (!key) + return TEE_ERROR_BAD_PARAMETERS; + + memcpy(public_key.pub, key->pub, sizeof(public_key.pub)); + + if (ph_flag) { + if (ed25519ph_verify(msg, msg_len, sig, sig_len, ctx, ctxlen, + &stat, &public_key) != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + } else { + if (ed25519ctx_verify(msg, msg_len, sig, sig_len, ctx, ctxlen, + &stat, &public_key) != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + } + + if (stat != 1) + return TEE_ERROR_SIGNATURE_INVALID; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/lib/libtomcrypt/gcm.c b/optee_os/core/lib/libtomcrypt/gcm.c new file mode 100644 index 0000000..b6848c7 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/gcm.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define TEE_GCM_TAG_MAX_LENGTH 16 + +struct tee_gcm_state { + struct crypto_authenc_ctx aectx; + gcm_state ctx; /* the gcm state as defined by LTC */ + size_t tag_len; /* tag length */ +}; + +static const struct crypto_authenc_ops aes_gcm_ops; + +static struct tee_gcm_state *to_tee_gcm_state(struct crypto_authenc_ctx *aectx) +{ + assert(aectx && aectx->ops == &aes_gcm_ops); + + return container_of(aectx, struct tee_gcm_state, aectx); +} + +TEE_Result crypto_aes_gcm_alloc_ctx(struct crypto_authenc_ctx **ctx_ret) +{ + struct tee_gcm_state *ctx = calloc(1, sizeof(*ctx)); + + if (!ctx) + return TEE_ERROR_OUT_OF_MEMORY; + ctx->aectx.ops = &aes_gcm_ops; + + *ctx_ret = &ctx->aectx; + + return TEE_SUCCESS; +} + +static void crypto_aes_gcm_free_ctx(struct crypto_authenc_ctx *aectx) +{ + free(to_tee_gcm_state(aectx)); +} + +static void crypto_aes_gcm_copy_state(struct crypto_authenc_ctx *dst_aectx, + struct crypto_authenc_ctx *src_aectx) +{ + struct tee_gcm_state *dst_ctx = to_tee_gcm_state(dst_aectx); + struct tee_gcm_state *src_ctx = to_tee_gcm_state(src_aectx); + + dst_ctx->ctx = src_ctx->ctx; + dst_ctx->tag_len = src_ctx->tag_len; +} + +static TEE_Result crypto_aes_gcm_init(struct crypto_authenc_ctx *aectx, + TEE_OperationMode mode __unused, + const uint8_t *key, size_t key_len, + const uint8_t *nonce, size_t nonce_len, + size_t tag_len, size_t aad_len __unused, + size_t payload_len __unused) +{ + int ltc_res = 0; + int ltc_cipherindex = find_cipher("aes"); + struct tee_gcm_state *gcm = to_tee_gcm_state(aectx); + + if (ltc_cipherindex < 0) + return TEE_ERROR_NOT_SUPPORTED; + + /* reset the state */ + memset(&gcm->ctx, 0, sizeof(gcm->ctx)); + gcm->tag_len = tag_len; + + ltc_res = gcm_init(&gcm->ctx, ltc_cipherindex, key, key_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + /* Add the IV */ + ltc_res = gcm_add_iv(&gcm->ctx, nonce, nonce_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result crypto_aes_gcm_update_aad(struct crypto_authenc_ctx *aectx, + const uint8_t *data, size_t len) +{ + struct tee_gcm_state *gcm = to_tee_gcm_state(aectx); + int ltc_res = 0; + + /* Add the AAD (note: aad can be NULL if aadlen == 0) */ + ltc_res = gcm_add_aad(&gcm->ctx, data, len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result +crypto_aes_gcm_update_payload(struct crypto_authenc_ctx *aectx, + TEE_OperationMode mode, const uint8_t *src_data, + size_t len, uint8_t *dst_data) +{ + TEE_Result res = TEE_SUCCESS; + int ltc_res = 0; + int dir = 0; + struct tee_gcm_state *gcm = to_tee_gcm_state(aectx); + unsigned char *pt = NULL; + unsigned char *ct = NULL; + + if (mode == TEE_MODE_ENCRYPT) { + pt = (unsigned char *)src_data; + ct = dst_data; + dir = GCM_ENCRYPT; + } else { + pt = dst_data; + ct = (unsigned char *)src_data; + dir = GCM_DECRYPT; + } + + /* aad is optional ==> add one without length */ + if (gcm->ctx.mode == LTC_GCM_MODE_IV) { + res = crypto_aes_gcm_update_aad(aectx, NULL, 0); + if (res != TEE_SUCCESS) + return res; + } + + /* process the data */ + ltc_res = gcm_process(&gcm->ctx, pt, len, ct, dir); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result crypto_aes_gcm_enc_final(struct crypto_authenc_ctx *aectx, + const uint8_t *src_data, size_t len, + uint8_t *dst_data, uint8_t *dst_tag, + size_t *dst_tag_len) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_gcm_state *gcm = to_tee_gcm_state(aectx); + int ltc_res = 0; + + /* Finalize the remaining buffer */ + res = crypto_aes_gcm_update_payload(aectx, TEE_MODE_ENCRYPT, src_data, + len, dst_data); + if (res != TEE_SUCCESS) + return res; + + /* Check the tag length */ + if (*dst_tag_len < gcm->tag_len) { + *dst_tag_len = gcm->tag_len; + return TEE_ERROR_SHORT_BUFFER; + } + *dst_tag_len = gcm->tag_len; + + /* Compute the tag */ + ltc_res = gcm_done(&gcm->ctx, dst_tag, (unsigned long *)dst_tag_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result crypto_aes_gcm_dec_final(struct crypto_authenc_ctx *aectx, + const uint8_t *src_data, size_t len, + uint8_t *dst_data, + const uint8_t *tag, size_t tag_len) +{ + TEE_Result res = TEE_ERROR_BAD_STATE; + struct tee_gcm_state *gcm = to_tee_gcm_state(aectx); + int ltc_res = 0; + uint8_t dst_tag[TEE_GCM_TAG_MAX_LENGTH] = { 0 }; + unsigned long ltc_tag_len = tag_len; + + if (tag_len == 0) + return TEE_ERROR_SHORT_BUFFER; + if (tag_len > TEE_GCM_TAG_MAX_LENGTH) + return TEE_ERROR_BAD_STATE; + + /* Process the last buffer, if any */ + res = crypto_aes_gcm_update_payload(aectx, TEE_MODE_DECRYPT, src_data, + len, dst_data); + if (res != TEE_SUCCESS) + return res; + + /* Finalize the authentication */ + ltc_res = gcm_done(&gcm->ctx, dst_tag, <c_tag_len); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_STATE; + + if (consttime_memcmp(dst_tag, tag, tag_len) != 0) + res = TEE_ERROR_MAC_INVALID; + else + res = TEE_SUCCESS; + return res; +} + +static void crypto_aes_gcm_final(struct crypto_authenc_ctx *aectx) +{ + gcm_reset(&to_tee_gcm_state(aectx)->ctx); +} + +static const struct crypto_authenc_ops aes_gcm_ops = { + .init = crypto_aes_gcm_init, + .update_aad = crypto_aes_gcm_update_aad, + .update_payload = crypto_aes_gcm_update_payload, + .enc_final = crypto_aes_gcm_enc_final, + .dec_final = crypto_aes_gcm_dec_final, + .final = crypto_aes_gcm_final, + .free_ctx = crypto_aes_gcm_free_ctx, + .copy_state = crypto_aes_gcm_copy_state, +}; diff --git a/optee_os/core/lib/libtomcrypt/hash.c b/optee_os/core/lib/libtomcrypt/hash.c new file mode 100644 index 0000000..d948eb6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/hash.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/****************************************************************************** + * Message digest functions + ******************************************************************************/ + +struct ltc_hash_ctx { + struct crypto_hash_ctx ctx; + const struct ltc_hash_descriptor *descr; + hash_state state; +}; + +static const struct crypto_hash_ops ltc_hash_ops; + +static struct ltc_hash_ctx *to_hash_ctx(struct crypto_hash_ctx *ctx) +{ + assert(ctx && ctx->ops == <c_hash_ops); + + return container_of(ctx, struct ltc_hash_ctx, ctx); +} + +static TEE_Result ltc_hash_init(struct crypto_hash_ctx *ctx) +{ + struct ltc_hash_ctx *hc = to_hash_ctx(ctx); + + if (hc->descr->init(&hc->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_hash_update(struct crypto_hash_ctx *ctx, + const uint8_t *data, size_t len) +{ + struct ltc_hash_ctx *hc = to_hash_ctx(ctx); + + if (hc->descr->process(&hc->state, data, len) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_hash_final(struct crypto_hash_ctx *ctx, uint8_t *digest, + size_t len) +{ + struct ltc_hash_ctx *hc = to_hash_ctx(ctx); + size_t hash_size = hc->descr->hashsize; + uint8_t block_digest[TEE_MAX_HASH_SIZE] = { 0 }; + uint8_t *tmp_digest = NULL; + + if (len == 0) + return TEE_ERROR_BAD_PARAMETERS; + + if (hash_size > len) { + if (hash_size > sizeof(block_digest)) + return TEE_ERROR_BAD_STATE; + tmp_digest = block_digest; /* use a tempory buffer */ + } else { + tmp_digest = digest; + } + + if (hc->descr->done(&hc->state, tmp_digest) == CRYPT_OK) { + if (hash_size > len) + memcpy(digest, tmp_digest, len); + } else { + return TEE_ERROR_BAD_STATE; + } + + return TEE_SUCCESS; +} + +static void ltc_hash_free_ctx(struct crypto_hash_ctx *ctx) +{ + free(to_hash_ctx(ctx)); +} + +static void ltc_hash_copy_state(struct crypto_hash_ctx *dst_ctx, + struct crypto_hash_ctx *src_ctx) +{ + struct ltc_hash_ctx *src = to_hash_ctx(src_ctx); + struct ltc_hash_ctx *dst = to_hash_ctx(dst_ctx); + + assert(src->descr == dst->descr); + dst->state = src->state; +} + +static const struct crypto_hash_ops ltc_hash_ops = { + .init = ltc_hash_init, + .update = ltc_hash_update, + .final = ltc_hash_final, + .free_ctx = ltc_hash_free_ctx, + .copy_state = ltc_hash_copy_state, +}; + +static TEE_Result ltc_hash_alloc_ctx(struct crypto_hash_ctx **ctx_ret, + int ltc_hash_idx) +{ + struct ltc_hash_ctx *ctx = NULL; + + if (ltc_hash_idx < 0) + return TEE_ERROR_NOT_SUPPORTED; + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) + return TEE_ERROR_OUT_OF_MEMORY; + + ctx->ctx.ops = <c_hash_ops; + ctx->descr = hash_descriptor[ltc_hash_idx]; + + *ctx_ret = &ctx->ctx; + + return TEE_SUCCESS; +} + +#if defined(_CFG_CORE_LTC_MD5) +TEE_Result crypto_md5_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("md5")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA1) +TEE_Result crypto_sha1_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha1")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA224) +TEE_Result crypto_sha224_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha224")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA256) +TEE_Result crypto_sha256_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha256")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA384) +TEE_Result crypto_sha384_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha384")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA512) +TEE_Result crypto_sha512_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha512")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA256) +TEE_Result hash_sha256_check(const uint8_t *hash, const uint8_t *data, + size_t data_size) +{ + hash_state hs; + uint8_t digest[TEE_SHA256_HASH_SIZE]; + + if (sha256_init(&hs) != CRYPT_OK) + return TEE_ERROR_GENERIC; + if (sha256_process(&hs, data, data_size) != CRYPT_OK) + return TEE_ERROR_GENERIC; + if (sha256_done(&hs, digest) != CRYPT_OK) + return TEE_ERROR_GENERIC; + if (consttime_memcmp(digest, hash, sizeof(digest)) != 0) + return TEE_ERROR_SECURITY; + return TEE_SUCCESS; +} +#endif + +#if defined(_CFG_CORE_LTC_SHA512_256) +TEE_Result hash_sha512_256_compute(uint8_t *digest, const uint8_t *data, + size_t data_size) +{ + hash_state hs; + + if (sha512_256_init(&hs) != CRYPT_OK) + return TEE_ERROR_GENERIC; + if (sha512_256_process(&hs, data, data_size) != CRYPT_OK) + return TEE_ERROR_GENERIC; + if (sha512_256_done(&hs, digest) != CRYPT_OK) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_224) +TEE_Result crypto_sha3_224_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha3-224")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_256) +TEE_Result crypto_sha3_256_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha3-256")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_384) +TEE_Result crypto_sha3_384_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha3-384")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_512) +TEE_Result crypto_sha3_512_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return ltc_hash_alloc_ctx(ctx, find_hash("sha3-512")); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/hmac.c b/optee_os/core/lib/libtomcrypt/hmac.c new file mode 100644 index 0000000..3aafb0c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/hmac.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ltc_hmac_ctx { + struct crypto_mac_ctx ctx; + int hash_idx; + hmac_state state; +}; + +static const struct crypto_mac_ops ltc_hmac_ops; + +static struct ltc_hmac_ctx *to_hmac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx && ctx->ops == <c_hmac_ops); + + return container_of(ctx, struct ltc_hmac_ctx, ctx); +} + +static TEE_Result ltc_hmac_init(struct crypto_mac_ctx *ctx, const uint8_t *key, + size_t len) +{ + struct ltc_hmac_ctx *hc = to_hmac_ctx(ctx); + + if (hmac_init(&hc->state, hc->hash_idx, key, len) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_hmac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + if (hmac_process(&to_hmac_ctx(ctx)->state, data, len) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_hmac_final(struct crypto_mac_ctx *ctx, uint8_t *digest, + size_t len) +{ + unsigned long l = len; + + if (hmac_done(&to_hmac_ctx(ctx)->state, digest, &l) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static void ltc_hmac_free_ctx(struct crypto_mac_ctx *ctx) +{ + free(to_hmac_ctx(ctx)); +} + +static void ltc_hmac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct ltc_hmac_ctx *src = to_hmac_ctx(src_ctx); + struct ltc_hmac_ctx *dst = to_hmac_ctx(dst_ctx); + + assert(src->hash_idx == dst->hash_idx); + dst->state = src->state; +} + +static const struct crypto_mac_ops ltc_hmac_ops = { + .init = ltc_hmac_init, + .update = ltc_hmac_update, + .final = ltc_hmac_final, + .free_ctx = ltc_hmac_free_ctx, + .copy_state = ltc_hmac_copy_state, +}; + +static TEE_Result ltc_hmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret, + int hash_idx) +{ + struct ltc_hmac_ctx *ctx = NULL; + + if (hash_idx < 0) + return TEE_ERROR_NOT_SUPPORTED; + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) + return TEE_ERROR_OUT_OF_MEMORY; + + ctx->ctx.ops = <c_hmac_ops; + ctx->hash_idx = hash_idx; + *ctx_ret = &ctx->ctx; + + return TEE_SUCCESS; +} + +#if defined(_CFG_CORE_LTC_MD5) +TEE_Result crypto_hmac_md5_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("md5")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA1) +TEE_Result crypto_hmac_sha1_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha1")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA224) +TEE_Result crypto_hmac_sha224_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha224")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA256) +TEE_Result crypto_hmac_sha256_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha256")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA384) +TEE_Result crypto_hmac_sha384_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha384")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA512) +TEE_Result crypto_hmac_sha512_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha512")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_224) +TEE_Result crypto_hmac_sha3_224_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha3-224")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_256) +TEE_Result crypto_hmac_sha3_256_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha3-256")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_384) +TEE_Result crypto_hmac_sha3_384_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha3-384")); +} +#endif + +#if defined(_CFG_CORE_LTC_SHA3_512) +TEE_Result crypto_hmac_sha3_512_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return ltc_hmac_alloc_ctx(ctx, find_hash("sha3-512")); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/include/tomcrypt_init.h b/optee_os/core/lib/libtomcrypt/include/tomcrypt_init.h new file mode 100644 index 0000000..ca12322 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/include/tomcrypt_init.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#ifndef __TOMCRYPT_INIT_H +#define __TOMCRYPT_INIT_H + +void tomcrypt_init(void); + +#endif /*__TOMCRYPT_INIT_H*/ diff --git a/optee_os/core/lib/libtomcrypt/include/tomcrypt_mp.h b/optee_os/core/lib/libtomcrypt/include/tomcrypt_mp.h new file mode 100644 index 0000000..fd4df63 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/include/tomcrypt_mp.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TOMCRYPT_MP_H_ +#define TOMCRYPT_MP_H_ + +#if defined(_CFG_CORE_LTC_ACIPHER) +void init_mp_tomcrypt(void); +#else +static inline void init_mp_tomcrypt(void) { } +#endif + +#endif /* TOMCRYPT_MP_H_ */ diff --git a/optee_os/core/lib/libtomcrypt/mpi_desc.c b/optee_os/core/lib/libtomcrypt/mpi_desc.c new file mode 100644 index 0000000..791db1e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/mpi_desc.c @@ -0,0 +1,782 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_CFG_CORE_LTC_PAGER) +#include +#include +#endif + +/* Size needed for xtest to pass reliably on both ARM32 and ARM64 */ +#define MPI_MEMPOOL_SIZE (46 * 1024) + +/* From mbedtls/library/bignum.c */ +#define ciL (sizeof(mbedtls_mpi_uint)) /* chars in limb */ +#define biL (ciL << 3) /* bits in limb */ +#define BITS_TO_LIMBS(i) ((i) / biL + ((i) % biL != 0)) + +#if defined(_CFG_CORE_LTC_PAGER) +/* allocate pageable_zi vmem for mp scratch memory pool */ +static struct mempool *get_mp_scratch_memory_pool(void) +{ + size_t size; + void *data; + + size = ROUNDUP(MPI_MEMPOOL_SIZE, SMALL_PAGE_SIZE); + data = tee_pager_alloc(size); + if (!data) + panic(); + + return mempool_alloc_pool(data, size, tee_pager_release_phys); +} +#else /* _CFG_CORE_LTC_PAGER */ +static struct mempool *get_mp_scratch_memory_pool(void) +{ + static uint8_t data[MPI_MEMPOOL_SIZE] __aligned(MEMPOOL_ALIGN); + + return mempool_alloc_pool(data, sizeof(data), NULL); +} +#endif + +void init_mp_tomcrypt(void) +{ + struct mempool *p = get_mp_scratch_memory_pool(); + + if (!p) + panic(); + mbedtls_mpi_mempool = p; + assert(!mempool_default); + mempool_default = p; +} + +static int init(void **a) +{ + mbedtls_mpi *bn = mempool_alloc(mbedtls_mpi_mempool, sizeof(*bn)); + + if (!bn) + return CRYPT_MEM; + + mbedtls_mpi_init_mempool(bn); + *a = bn; + return CRYPT_OK; +} + +static int init_size(int size_bits __unused, void **a) +{ + return init(a); +} + +static void deinit(void *a) +{ + mbedtls_mpi_free((mbedtls_mpi *)a); + mempool_free(mbedtls_mpi_mempool, a); +} + +static int neg(void *a, void *b) +{ + if (mbedtls_mpi_copy(b, a)) + return CRYPT_MEM; + ((mbedtls_mpi *)b)->s *= -1; + return CRYPT_OK; +} + +static int copy(void *a, void *b) +{ + if (mbedtls_mpi_copy(b, a)) + return CRYPT_MEM; + return CRYPT_OK; +} + +static int init_copy(void **a, void *b) +{ + if (init(a) != CRYPT_OK) { + return CRYPT_MEM; + } + return copy(b, *a); +} + +/* ---- trivial ---- */ +static int set_int(void *a, ltc_mp_digit b) +{ + uint32_t b32 = b; + + if (b32 != b) + return CRYPT_INVALID_ARG; + + mbedtls_mpi_uint p = b32; + mbedtls_mpi bn = { .s = 1, .n = 1, .p = &p }; + + if (mbedtls_mpi_copy(a, &bn)) + return CRYPT_MEM; + return CRYPT_OK; +} + +static unsigned long get_int(void *a) +{ + mbedtls_mpi *bn = a; + + if (!bn->n) + return 0; + + return bn->p[bn->n - 1]; +} + +static ltc_mp_digit get_digit(void *a, int n) +{ + mbedtls_mpi *bn = a; + + COMPILE_TIME_ASSERT(sizeof(ltc_mp_digit) >= sizeof(mbedtls_mpi_uint)); + + if (n < 0 || (size_t)n >= bn->n) + return 0; + + return bn->p[n]; +} + +static int get_digit_count(void *a) +{ + return ROUNDUP(mbedtls_mpi_size(a), sizeof(mbedtls_mpi_uint)) / + sizeof(mbedtls_mpi_uint); +} + +static int compare(void *a, void *b) +{ + int ret = mbedtls_mpi_cmp_mpi(a, b); + + if (ret < 0) + return LTC_MP_LT; + + if (ret > 0) + return LTC_MP_GT; + + return LTC_MP_EQ; +} + +static int compare_d(void *a, ltc_mp_digit b) +{ + unsigned long v = b; + unsigned int shift = 31; + uint32_t mask = BIT(shift) - 1; + mbedtls_mpi bn; + + mbedtls_mpi_init_mempool(&bn); + while (true) { + mbedtls_mpi_add_int(&bn, &bn, v & mask); + v >>= shift; + if (!v) + break; + mbedtls_mpi_shift_l(&bn, shift); + } + + int ret = compare(a, &bn); + + mbedtls_mpi_free(&bn); + + return ret; +} + +static int count_bits(void *a) +{ + return mbedtls_mpi_bitlen(a); +} + +static int count_lsb_bits(void *a) +{ + return mbedtls_mpi_lsb(a); +} + + +static int twoexpt(void *a, int n) +{ + if (mbedtls_mpi_set_bit(a, n, 1)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +/* ---- conversions ---- */ + +/* read ascii string */ +static int read_radix(void *a, const char *b, int radix) +{ + int res = mbedtls_mpi_read_string(a, radix, b); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + if (res) + return CRYPT_ERROR; + + return CRYPT_OK; +} + +/* write one */ +static int write_radix(void *a, char *b, int radix) +{ + size_t ol = SIZE_MAX; + int res = mbedtls_mpi_write_string(a, radix, b, ol, &ol); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + if (res) + return CRYPT_ERROR; + + return CRYPT_OK; +} + +/* get size as unsigned char string */ +static unsigned long unsigned_size(void *a) +{ + return mbedtls_mpi_size(a); +} + +/* store */ +static int unsigned_write(void *a, unsigned char *b) +{ + int res = mbedtls_mpi_write_binary(a, b, unsigned_size(a)); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + if (res) + return CRYPT_ERROR; + + return CRYPT_OK; +} + +/* read */ +static int unsigned_read(void *a, unsigned char *b, unsigned long len) +{ + int res = mbedtls_mpi_read_binary(a, b, len); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + if (res) + return CRYPT_ERROR; + + return CRYPT_OK; +} + +/* add */ +static int add(void *a, void *b, void *c) +{ + if (mbedtls_mpi_add_mpi(c, a, b)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +static int addi(void *a, ltc_mp_digit b, void *c) +{ + uint32_t b32 = b; + + if (b32 != b) + return CRYPT_INVALID_ARG; + + mbedtls_mpi_uint p = b32; + mbedtls_mpi bn = { .s = 1, .n = 1, .p = &p }; + + return add(a, &bn, c); +} + +/* sub */ +static int sub(void *a, void *b, void *c) +{ + if (mbedtls_mpi_sub_mpi(c, a, b)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +static int subi(void *a, ltc_mp_digit b, void *c) +{ + uint32_t b32 = b; + + if (b32 != b) + return CRYPT_INVALID_ARG; + + mbedtls_mpi_uint p = b32; + mbedtls_mpi bn = { .s = 1, .n = 1, .p = &p }; + + return sub(a, &bn, c); +} + +/* mul */ +static int mul(void *a, void *b, void *c) +{ + if (mbedtls_mpi_mul_mpi(c, a, b)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +static int muli(void *a, ltc_mp_digit b, void *c) +{ + if (b > (unsigned long) UINT32_MAX) + return CRYPT_INVALID_ARG; + + if (mbedtls_mpi_mul_int(c, a, b)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +/* sqr */ +static int sqr(void *a, void *b) +{ + return mul(a, a, b); +} + +/* div */ +static int divide(void *a, void *b, void *c, void *d) +{ + int res = mbedtls_mpi_div_mpi(c, d, a, b); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + if (res) + return CRYPT_ERROR; + + return CRYPT_OK; +} + +static int div_2(void *a, void *b) +{ + if (mbedtls_mpi_copy(b, a)) + return CRYPT_MEM; + + if (mbedtls_mpi_shift_r(b, 1)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +/* modi */ +static int modi(void *a, ltc_mp_digit b, ltc_mp_digit *c) +{ + mbedtls_mpi bn_b; + mbedtls_mpi bn_c; + int res = 0; + + mbedtls_mpi_init_mempool(&bn_b); + mbedtls_mpi_init_mempool(&bn_c); + + res = set_int(&bn_b, b); + if (res) + return res; + + res = mbedtls_mpi_mod_mpi(&bn_c, &bn_b, a); + if (!res) + *c = get_int(&bn_c); + + mbedtls_mpi_free(&bn_b); + mbedtls_mpi_free(&bn_c); + + if (res) + return CRYPT_MEM; + + return CRYPT_OK; +} + +/* gcd */ +static int gcd(void *a, void *b, void *c) +{ + if (mbedtls_mpi_gcd(c, a, b)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +/* lcm */ +static int lcm(void *a, void *b, void *c) +{ + int res = CRYPT_MEM; + mbedtls_mpi tmp; + + mbedtls_mpi_init_mempool(&tmp); + if (mbedtls_mpi_mul_mpi(&tmp, a, b)) + goto out; + + if (mbedtls_mpi_gcd(c, a, b)) + goto out; + + /* We use the following equality: gcd(a, b) * lcm(a, b) = a * b */ + res = divide(&tmp, c, c, NULL); +out: + mbedtls_mpi_free(&tmp); + return res; +} + +static int mod(void *a, void *b, void *c) +{ + int res = mbedtls_mpi_mod_mpi(c, a, b); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + if (res) + return CRYPT_ERROR; + + return CRYPT_OK; +} + +static int addmod(void *a, void *b, void *c, void *d) +{ + int res = add(a, b, d); + + if (res) + return res; + + return mod(d, c, d); +} + +static int submod(void *a, void *b, void *c, void *d) +{ + int res = sub(a, b, d); + + if (res) + return res; + + return mod(d, c, d); +} + +static int mulmod(void *a, void *b, void *c, void *d) +{ + int res; + mbedtls_mpi ta; + mbedtls_mpi tb; + + mbedtls_mpi_init_mempool(&ta); + mbedtls_mpi_init_mempool(&tb); + + res = mod(a, c, &ta); + if (res) + goto out; + res = mod(b, c, &tb); + if (res) + goto out; + res = mul(&ta, &tb, d); + if (res) + goto out; + res = mod(d, c, d); +out: + mbedtls_mpi_free(&ta); + mbedtls_mpi_free(&tb); + return res; +} + +static int sqrmod(void *a, void *b, void *c) +{ + return mulmod(a, a, b, c); +} + +/* invmod */ +static int invmod(void *a, void *b, void *c) +{ + int res = mbedtls_mpi_inv_mod(c, a, b); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + if (res) + return CRYPT_ERROR; + + return CRYPT_OK; +} + + +/* setup */ +static int montgomery_setup(void *a, void **b) +{ + *b = mempool_alloc(mbedtls_mpi_mempool, sizeof(mbedtls_mpi_uint)); + if (!*b) + return CRYPT_MEM; + + mbedtls_mpi_montg_init(*b, a); + + return CRYPT_OK; +} + +/* get normalization value */ +static int montgomery_normalization(void *a, void *b) +{ + size_t c = ROUNDUP(mbedtls_mpi_size(b), sizeof(mbedtls_mpi_uint)) * 8; + + if (mbedtls_mpi_lset(a, 1)) + return CRYPT_MEM; + if (mbedtls_mpi_shift_l(a, c)) + return CRYPT_MEM; + if (mbedtls_mpi_mod_mpi(a, a, b)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +/* reduce */ +static int montgomery_reduce(void *a, void *b, void *c) +{ + mbedtls_mpi A; + mbedtls_mpi *N = b; + mbedtls_mpi_uint *mm = c; + mbedtls_mpi T; + int ret = CRYPT_MEM; + + mbedtls_mpi_init_mempool(&T); + mbedtls_mpi_init_mempool(&A); + + if (mbedtls_mpi_grow(&T, (N->n + 1) * 2)) + goto out; + + if (mbedtls_mpi_cmp_mpi(a, N) > 0) { + if (mbedtls_mpi_mod_mpi(&A, a, N)) + goto out; + } else { + if (mbedtls_mpi_copy(&A, a)) + goto out; + } + + if (mbedtls_mpi_grow(&A, N->n + 1)) + goto out; + + mbedtls_mpi_montred(&A, N, *mm, &T); + + if (mbedtls_mpi_copy(a, &A)) + goto out; + + ret = CRYPT_OK; +out: + mbedtls_mpi_free(&A); + mbedtls_mpi_free(&T); + + return ret; +} + +/* clean up */ +static void montgomery_deinit(void *a) +{ + mempool_free(mbedtls_mpi_mempool, a); +} + +/* + * This function calculates: + * d = a^b mod c + * + * @a: base + * @b: exponent + * @c: modulus + * @d: destination + */ +static int exptmod(void *a, void *b, void *c, void *d) +{ + int res; + + if (d == a || d == b || d == c) { + mbedtls_mpi dest; + + mbedtls_mpi_init_mempool(&dest); + res = mbedtls_mpi_exp_mod(&dest, a, b, c, NULL); + if (!res) + res = mbedtls_mpi_copy(d, &dest); + mbedtls_mpi_free(&dest); + } else { + res = mbedtls_mpi_exp_mod(d, a, b, c, NULL); + } + + if (res) + return CRYPT_MEM; + else + return CRYPT_OK; +} + +static int rng_read(void *ignored __unused, unsigned char *buf, size_t blen) +{ + if (crypto_rng_read(buf, blen)) + return MBEDTLS_ERR_MPI_FILE_IO_ERROR; + return 0; +} + +static int isprime(void *a, int b, int *c) +{ + int res = mbedtls_mpi_is_prime_ext(a, b, rng_read, NULL); + + if (res == MBEDTLS_ERR_MPI_ALLOC_FAILED) + return CRYPT_MEM; + + if (res) + *c = LTC_MP_NO; + else + *c = LTC_MP_YES; + + return CRYPT_OK; +} + +static int mpi_rand(void *a, int size) +{ + if (mbedtls_mpi_fill_random(a, size, rng_read, NULL)) + return CRYPT_MEM; + + return CRYPT_OK; +} + +ltc_math_descriptor ltc_mp = { + .name = "MPI", + .bits_per_digit = sizeof(mbedtls_mpi_uint) * 8, + + .init = init, + .init_size = init_size, + .init_copy = init_copy, + .deinit = deinit, + + .neg = neg, + .copy = copy, + + .set_int = set_int, + .get_int = get_int, + .get_digit = get_digit, + .get_digit_count = get_digit_count, + .compare = compare, + .compare_d = compare_d, + .count_bits = count_bits, + .count_lsb_bits = count_lsb_bits, + .twoexpt = twoexpt, + + .read_radix = read_radix, + .write_radix = write_radix, + .unsigned_size = unsigned_size, + .unsigned_write = unsigned_write, + .unsigned_read = unsigned_read, + + .add = add, + .addi = addi, + .sub = sub, + .subi = subi, + .mul = mul, + .muli = muli, + .sqr = sqr, + .mpdiv = divide, + .div_2 = div_2, + .modi = modi, + .gcd = gcd, + .lcm = lcm, + + .mulmod = mulmod, + .sqrmod = sqrmod, + .invmod = invmod, + + .montgomery_setup = montgomery_setup, + .montgomery_normalization = montgomery_normalization, + .montgomery_reduce = montgomery_reduce, + .montgomery_deinit = montgomery_deinit, + + .exptmod = exptmod, + .isprime = isprime, + +#ifdef LTC_MECC +#ifdef LTC_MECC_FP + .ecc_ptmul = ltc_ecc_fp_mulmod, +#else + .ecc_ptmul = ltc_ecc_mulmod, +#endif /* LTC_MECC_FP */ + .ecc_ptadd = ltc_ecc_projective_add_point, + .ecc_ptdbl = ltc_ecc_projective_dbl_point, + .ecc_map = ltc_ecc_map, +#ifdef LTC_ECC_SHAMIR +#ifdef LTC_MECC_FP + .ecc_mul2add = ltc_ecc_fp_mul2add, +#else + .ecc_mul2add = ltc_ecc_mul2add, +#endif /* LTC_MECC_FP */ +#endif /* LTC_ECC_SHAMIR */ +#endif /* LTC_MECC */ + +#ifdef LTC_MRSA + .rsa_keygen = rsa_make_key, + .rsa_me = rsa_exptmod, +#endif + .addmod = addmod, + .submod = submod, + .rand = mpi_rand, + +}; + +size_t crypto_bignum_num_bytes(struct bignum *a) +{ + return mbedtls_mpi_size((mbedtls_mpi *)a); +} + +size_t crypto_bignum_num_bits(struct bignum *a) +{ + return mbedtls_mpi_bitlen((mbedtls_mpi *)a); +} + +int32_t crypto_bignum_compare(struct bignum *a, struct bignum *b) +{ + return mbedtls_mpi_cmp_mpi((mbedtls_mpi *)a, (mbedtls_mpi *)b); +} + +void crypto_bignum_bn2bin(const struct bignum *from, uint8_t *to) +{ + const mbedtls_mpi *f = (const mbedtls_mpi *)from; + int rc __maybe_unused = 0; + + rc = mbedtls_mpi_write_binary(f, (void *)to, mbedtls_mpi_size(f)); + assert(!rc); +} + +TEE_Result crypto_bignum_bin2bn(const uint8_t *from, size_t fromsize, + struct bignum *to) +{ + if (mbedtls_mpi_read_binary((mbedtls_mpi *)to, (const void *)from, + fromsize)) + return TEE_ERROR_BAD_PARAMETERS; + return TEE_SUCCESS; +} + +void crypto_bignum_copy(struct bignum *to, const struct bignum *from) +{ + int rc __maybe_unused = 0; + + rc = mbedtls_mpi_copy((mbedtls_mpi *)to, (const mbedtls_mpi *)from); + assert(!rc); +} + +struct bignum *crypto_bignum_allocate(size_t size_bits) +{ + mbedtls_mpi *bn = malloc(sizeof(*bn)); + + if (!bn) + return NULL; + + mbedtls_mpi_init(bn); + if (mbedtls_mpi_grow(bn, BITS_TO_LIMBS(size_bits))) { + free(bn); + return NULL; + } + + return (struct bignum *)bn; +} + +void crypto_bignum_free(struct bignum **s) +{ + assert(s); + + mbedtls_mpi_free((mbedtls_mpi *)*s); + free(*s); + *s = NULL; +} + +void crypto_bignum_clear(struct bignum *s) +{ + mbedtls_mpi *bn = (mbedtls_mpi *)s; + + bn->s = 1; + if (bn->p) + memset(bn->p, 0, sizeof(*bn->p) * bn->n); +} diff --git a/optee_os/core/lib/libtomcrypt/rsa.c b/optee_os/core/lib/libtomcrypt/rsa.c new file mode 100644 index 0000000..f63bfe9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/rsa.c @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, 2022 Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + + +/* + * Compute the LibTomCrypt "hashindex" given a TEE Algorithm "algo" + * Return + * - TEE_SUCCESS in case of success, + * - TEE_ERROR_BAD_PARAMETERS in case algo is not a valid algo + * - TEE_ERROR_NOT_SUPPORTED in case algo is not supported by LTC + * Return -1 in case of error + */ +static TEE_Result tee_algo_to_ltc_hashindex(uint32_t algo, int *ltc_hashindex) +{ + switch (algo) { +#if defined(_CFG_CORE_LTC_SHA1_DESC) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1: + *ltc_hashindex = find_hash("sha1"); + break; +#endif +#if defined(_CFG_CORE_LTC_MD5_DESC) + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_MD5: + *ltc_hashindex = find_hash("md5"); + break; +#endif +#if defined(_CFG_CORE_LTC_SHA224_DESC) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224: + *ltc_hashindex = find_hash("sha224"); + break; +#endif +#if defined(_CFG_CORE_LTC_SHA256_DESC) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256: + *ltc_hashindex = find_hash("sha256"); + break; +#endif +#if defined(_CFG_CORE_LTC_SHA384_DESC) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384: + *ltc_hashindex = find_hash("sha384"); + break; +#endif +#if defined(_CFG_CORE_LTC_SHA512_DESC) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512: + *ltc_hashindex = find_hash("sha512"); + break; +#endif + case TEE_ALG_RSASSA_PKCS1_V1_5: + case TEE_ALG_RSAES_PKCS1_V1_5: + /* invalid one. but it should not be used anyway */ + *ltc_hashindex = -1; + return TEE_SUCCESS; + + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + if (*ltc_hashindex < 0) + return TEE_ERROR_NOT_SUPPORTED; + else + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *s, + size_t key_size_bits __unused) +__weak __alias("sw_crypto_acipher_alloc_rsa_keypair"); + +TEE_Result sw_crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *s, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->e)) + return TEE_ERROR_OUT_OF_MEMORY; + if (!bn_alloc_max(&s->d)) + goto err; + if (!bn_alloc_max(&s->n)) + goto err; + if (!bn_alloc_max(&s->p)) + goto err; + if (!bn_alloc_max(&s->q)) + goto err; + if (!bn_alloc_max(&s->qp)) + goto err; + if (!bn_alloc_max(&s->dp)) + goto err; + if (!bn_alloc_max(&s->dq)) + goto err; + + return TEE_SUCCESS; +err: + crypto_acipher_free_rsa_keypair(s); + return TEE_ERROR_OUT_OF_MEMORY; +} + + +TEE_Result crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *s, + size_t key_size_bits __unused) +__weak __alias("sw_crypto_acipher_alloc_rsa_public_key"); + +TEE_Result sw_crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *s, + size_t key_size_bits __unused) +{ + memset(s, 0, sizeof(*s)); + if (!bn_alloc_max(&s->e)) + return TEE_ERROR_OUT_OF_MEMORY; + if (!bn_alloc_max(&s->n)) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->e); + return TEE_ERROR_OUT_OF_MEMORY; +} + + +void crypto_acipher_free_rsa_public_key(struct rsa_public_key *s) +__weak __alias("sw_crypto_acipher_free_rsa_public_key"); + +void sw_crypto_acipher_free_rsa_public_key(struct rsa_public_key *s) +{ + if (!s) + return; + crypto_bignum_free(&s->n); + crypto_bignum_free(&s->e); +} + + +void crypto_acipher_free_rsa_keypair(struct rsa_keypair *s) +__weak __alias("sw_crypto_acipher_free_rsa_keypair"); + +void sw_crypto_acipher_free_rsa_keypair(struct rsa_keypair *s) +{ + if (!s) + return; + crypto_bignum_free(&s->e); + crypto_bignum_free(&s->d); + crypto_bignum_free(&s->n); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->q); + crypto_bignum_free(&s->qp); + crypto_bignum_free(&s->dp); + crypto_bignum_free(&s->dq); +} + +TEE_Result crypto_acipher_gen_rsa_key(struct rsa_keypair *key, + size_t key_size) +__weak __alias("sw_crypto_acipher_gen_rsa_key"); + +TEE_Result sw_crypto_acipher_gen_rsa_key(struct rsa_keypair *key, + size_t key_size) +{ + TEE_Result res; + rsa_key ltc_tmp_key; + int ltc_res; + + /* Generate a temporary RSA key */ + ltc_res = rsa_make_key_bn_e(NULL, find_prng("prng_crypto"), + key_size / 8, key->e, <c_tmp_key); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_PARAMETERS; + } else if ((size_t)mp_count_bits(ltc_tmp_key.N) != key_size) { + rsa_free(<c_tmp_key); + res = TEE_ERROR_BAD_PARAMETERS; + } else { + /* Copy the key */ + ltc_mp.copy(ltc_tmp_key.d, key->d); + ltc_mp.copy(ltc_tmp_key.N, key->n); + ltc_mp.copy(ltc_tmp_key.p, key->p); + ltc_mp.copy(ltc_tmp_key.q, key->q); + ltc_mp.copy(ltc_tmp_key.qP, key->qp); + ltc_mp.copy(ltc_tmp_key.dP, key->dp); + ltc_mp.copy(ltc_tmp_key.dQ, key->dq); + + /* Free the temporary key */ + rsa_free(<c_tmp_key); + res = TEE_SUCCESS; + } + + return res; +} + +static TEE_Result rsadorep(rsa_key *ltc_key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *buf = NULL; + unsigned long blen, offset; + int ltc_res; + + /* + * Use a temporary buffer since we don't know exactly how large the + * required size of the out buffer without doing a partial decrypt. + * We know the upper bound though. + */ + blen = _CFG_CORE_LTC_BIGNUM_MAX_BITS / sizeof(uint8_t); + buf = mempool_alloc(mempool_default, blen); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = rsa_exptmod(src, src_len, buf, &blen, ltc_key->type, + ltc_key); + switch (ltc_res) { + case CRYPT_PK_NOT_PRIVATE: + case CRYPT_PK_INVALID_TYPE: + case CRYPT_PK_INVALID_SIZE: + case CRYPT_INVALID_PACKET: + EMSG("rsa_exptmod() returned %d", ltc_res); + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + case CRYPT_OK: + break; + default: + /* This will result in a panic */ + EMSG("rsa_exptmod() returned %d", ltc_res); + res = TEE_ERROR_GENERIC; + goto out; + } + + /* Remove the zero-padding (leave one zero if buff is all zeroes) */ + offset = 0; + while ((offset < blen - 1) && (buf[offset] == 0)) + offset++; + + if (*dst_len < blen - offset) { + *dst_len = blen - offset; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + res = TEE_SUCCESS; + *dst_len = blen - offset; + memcpy(dst, (char *)buf + offset, *dst_len); + +out: + mempool_free(mempool_default, buf); + + return res; +} + +TEE_Result crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsanopad_encrypt"); + +TEE_Result sw_crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +{ + TEE_Result res; + rsa_key ltc_key = { 0, }; + + ltc_key.type = PK_PUBLIC; + ltc_key.e = key->e; + ltc_key.N = key->n; + + res = rsadorep(<c_key, src, src_len, dst, dst_len); + return res; +} + +TEE_Result crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsanopad_decrypt"); + +TEE_Result sw_crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +{ + TEE_Result res; + rsa_key ltc_key = { 0, }; + + ltc_key.type = PK_PRIVATE; + ltc_key.e = key->e; + ltc_key.N = key->n; + ltc_key.d = key->d; + if (key->p && crypto_bignum_num_bytes(key->p)) { + ltc_key.p = key->p; + ltc_key.q = key->q; + ltc_key.qP = key->qp; + ltc_key.dP = key->dp; + ltc_key.dQ = key->dq; + } + + res = rsadorep(<c_key, src, src_len, dst, dst_len); + return res; +} + +TEE_Result crypto_acipher_rsaes_decrypt(uint32_t algo, + struct rsa_keypair *key, + const uint8_t *label, + size_t label_len, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsaes_decrypt"); + +TEE_Result sw_crypto_acipher_rsaes_decrypt(uint32_t algo, + struct rsa_keypair *key, + const uint8_t *label, + size_t label_len, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + void *buf = NULL; + unsigned long blen; + int ltc_hashindex, ltc_res, ltc_stat, ltc_rsa_algo; + size_t mod_size; + rsa_key ltc_key = { 0, }; + + ltc_key.type = PK_PRIVATE; + ltc_key.e = key->e; + ltc_key.d = key->d; + ltc_key.N = key->n; + if (key->p && crypto_bignum_num_bytes(key->p)) { + ltc_key.p = key->p; + ltc_key.q = key->q; + ltc_key.qP = key->qp; + ltc_key.dP = key->dp; + ltc_key.dQ = key->dq; + } + + /* Get the algorithm */ + res = tee_algo_to_ltc_hashindex(algo, <c_hashindex); + if (res != TEE_SUCCESS) { + EMSG("tee_algo_to_ltc_hashindex() returned %d", (int)res); + goto out; + } + + /* + * Use a temporary buffer since we don't know exactly how large + * the required size of the out buffer without doing a partial + * decrypt. We know the upper bound though. + */ + if (algo == TEE_ALG_RSAES_PKCS1_V1_5) { + mod_size = ltc_mp.unsigned_size((void *)(ltc_key.N)); + blen = mod_size - 11; + ltc_rsa_algo = LTC_PKCS_1_V1_5; + } else { + /* Decoded message is always shorter than encrypted message */ + blen = src_len; + ltc_rsa_algo = LTC_PKCS_1_OAEP; + } + + buf = mempool_alloc(mempool_default, blen); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = rsa_decrypt_key_ex(src, src_len, buf, &blen, + ((label_len == 0) ? 0 : label), label_len, + ltc_hashindex, ltc_rsa_algo, <c_stat, + <c_key); + switch (ltc_res) { + case CRYPT_PK_INVALID_PADDING: + case CRYPT_INVALID_PACKET: + case CRYPT_PK_INVALID_SIZE: + EMSG("rsa_decrypt_key_ex() returned %d", ltc_res); + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + case CRYPT_OK: + break; + default: + /* This will result in a panic */ + EMSG("rsa_decrypt_key_ex() returned %d", ltc_res); + res = TEE_ERROR_GENERIC; + goto out; + } + if (ltc_stat != 1) { + /* This will result in a panic */ + EMSG("rsa_decrypt_key_ex() returned %d and %d", + ltc_res, ltc_stat); + res = TEE_ERROR_GENERIC; + goto out; + } + + if (*dst_len < blen) { + *dst_len = blen; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + res = TEE_SUCCESS; + *dst_len = blen; + memcpy(dst, buf, blen); + +out: + mempool_free(mempool_default, buf); + + return res; +} + +TEE_Result crypto_acipher_rsaes_encrypt(uint32_t algo, + struct rsa_public_key *key, + const uint8_t *label, + size_t label_len, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsaes_encrypt"); + +TEE_Result sw_crypto_acipher_rsaes_encrypt(uint32_t algo, + struct rsa_public_key *key, + const uint8_t *label, + size_t label_len, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +{ + TEE_Result res; + uint32_t mod_size; + int ltc_hashindex, ltc_res, ltc_rsa_algo; + rsa_key ltc_key = { + .type = PK_PUBLIC, + .e = key->e, + .N = key->n + }; + + mod_size = ltc_mp.unsigned_size((void *)(ltc_key.N)); + if (*dst_len < mod_size) { + *dst_len = mod_size; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + *dst_len = mod_size; + + /* Get the algorithm */ + res = tee_algo_to_ltc_hashindex(algo, <c_hashindex); + if (res != TEE_SUCCESS) + goto out; + + if (algo == TEE_ALG_RSAES_PKCS1_V1_5) + ltc_rsa_algo = LTC_PKCS_1_V1_5; + else + ltc_rsa_algo = LTC_PKCS_1_OAEP; + + ltc_res = rsa_encrypt_key_ex(src, src_len, dst, + (unsigned long *)(dst_len), label, + label_len, NULL, find_prng("prng_crypto"), + ltc_hashindex, ltc_rsa_algo, <c_key); + switch (ltc_res) { + case CRYPT_PK_INVALID_PADDING: + case CRYPT_INVALID_PACKET: + case CRYPT_PK_INVALID_SIZE: + EMSG("rsa_encrypt_key_ex() returned %d", ltc_res); + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + case CRYPT_OK: + break; + default: + /* This will result in a panic */ + res = TEE_ERROR_GENERIC; + goto out; + } + res = TEE_SUCCESS; + +out: + return res; +} + +TEE_Result crypto_acipher_rsassa_sign(uint32_t algo, struct rsa_keypair *key, + int salt_len, const uint8_t *msg, + size_t msg_len, uint8_t *sig, + size_t *sig_len) +__weak __alias("sw_crypto_acipher_rsassa_sign"); + +TEE_Result sw_crypto_acipher_rsassa_sign(uint32_t algo, struct rsa_keypair *key, + int salt_len, const uint8_t *msg, + size_t msg_len, uint8_t *sig, + size_t *sig_len) +{ + TEE_Result res; + size_t hash_size, mod_size; + int ltc_res, ltc_rsa_algo, ltc_hashindex; + unsigned long ltc_sig_len; + rsa_key ltc_key = { 0, }; + + ltc_key.type = PK_PRIVATE; + ltc_key.e = key->e; + ltc_key.N = key->n; + ltc_key.d = key->d; + if (key->p && crypto_bignum_num_bytes(key->p)) { + ltc_key.p = key->p; + ltc_key.q = key->q; + ltc_key.qP = key->qp; + ltc_key.dP = key->dp; + ltc_key.dQ = key->dq; + } + + switch (algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5: + ltc_rsa_algo = LTC_PKCS_1_V1_5_NA1; + break; + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + ltc_rsa_algo = LTC_PKCS_1_V1_5; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + ltc_rsa_algo = LTC_PKCS_1_PSS; + break; + default: + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + if (ltc_rsa_algo != LTC_PKCS_1_V1_5_NA1) { + ltc_res = tee_algo_to_ltc_hashindex(algo, <c_hashindex); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + res = tee_alg_get_digest_size(TEE_DIGEST_HASH_TO_ALGO(algo), + &hash_size); + if (res != TEE_SUCCESS) + goto err; + + if (msg_len != hash_size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + } + + mod_size = ltc_mp.unsigned_size((void *)(ltc_key.N)); + + if (*sig_len < mod_size) { + *sig_len = mod_size; + res = TEE_ERROR_SHORT_BUFFER; + goto err; + } + + ltc_sig_len = mod_size; + + ltc_res = rsa_sign_hash_ex(msg, msg_len, sig, <c_sig_len, + ltc_rsa_algo, NULL, find_prng("prng_crypto"), + ltc_hashindex, salt_len, <c_key); + + *sig_len = ltc_sig_len; + + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + res = TEE_SUCCESS; + +err: + return res; +} + +TEE_Result crypto_acipher_rsassa_verify(uint32_t algo, + struct rsa_public_key *key, + int salt_len, const uint8_t *msg, + size_t msg_len, const uint8_t *sig, + size_t sig_len) +__weak __alias("sw_crypto_acipher_rsassa_verify"); + +TEE_Result sw_crypto_acipher_rsassa_verify(uint32_t algo, + struct rsa_public_key *key, + int salt_len, const uint8_t *msg, + size_t msg_len, const uint8_t *sig, + size_t sig_len) +{ + TEE_Result res; + uint32_t bigint_size; + size_t hash_size; + int stat, ltc_hashindex, ltc_res, ltc_rsa_algo; + rsa_key ltc_key = { + .type = PK_PUBLIC, + .e = key->e, + .N = key->n + }; + struct ftmn ftmn = { }; + + /* + * The caller expects to call crypto_acipher_rsassa_verify(), + * update the hash as needed. + */ + FTMN_CALLEE_SWAP_HASH(FTMN_FUNC_HASH("crypto_acipher_rsassa_verify")); + + if (algo != TEE_ALG_RSASSA_PKCS1_V1_5) { + res = tee_alg_get_digest_size(TEE_DIGEST_HASH_TO_ALGO(algo), + &hash_size); + if (res != TEE_SUCCESS) + goto err; + + if (msg_len != hash_size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + } + + bigint_size = ltc_mp.unsigned_size(ltc_key.N); + if (sig_len < bigint_size) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto err; + } + + /* Get the algorithm */ + if (algo != TEE_ALG_RSASSA_PKCS1_V1_5) { + res = tee_algo_to_ltc_hashindex(algo, <c_hashindex); + if (res != TEE_SUCCESS) + goto err; + } + + switch (algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5: + ltc_rsa_algo = LTC_PKCS_1_V1_5_NA1; + break; + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + ltc_rsa_algo = LTC_PKCS_1_V1_5; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + ltc_rsa_algo = LTC_PKCS_1_PSS; + break; + default: + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + FTMN_PUSH_LINKED_CALL(&ftmn, FTMN_FUNC_HASH("rsa_verify_hash_ex")); + ltc_res = rsa_verify_hash_ex(sig, sig_len, msg, msg_len, ltc_rsa_algo, + ltc_hashindex, salt_len, &stat, <c_key); + res = convert_ltc_verify_status(ltc_res, stat); + if (res) + FTMN_SET_CHECK_RES_NOT_ZERO(&ftmn, FTMN_INCR0, res); + else + FTMN_SET_CHECK_RES_FROM_CALL(&ftmn, FTMN_INCR0, 0); + FTMN_POP_LINKED_CALL(&ftmn); + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR0, FTMN_STEP_COUNT(1), res); + return res; +err: + FTMN_CALLEE_DONE_NOT_ZERO(res); + return res; +} diff --git a/optee_os/core/lib/libtomcrypt/sha1_accel.c b/optee_os/core/lib/libtomcrypt/sha1_accel.c new file mode 100644 index 0000000..b4e3cec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sha1_accel.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, 2020, Linaro Limited + * Copyright (c) 2001-2007, Tom St Denis + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtom.org + */ +#include +#include +/** + @file sha1.c + LTC_SHA1 code by Tom St Denis +*/ + + +const struct ltc_hash_descriptor sha1_desc = +{ + "sha1", + 2, + 20, + 64, + + /* OID */ + { 1, 3, 14, 3, 2, 26, }, + 6, + + &sha1_init, + &sha1_process, + &sha1_done, + &sha1_test, + NULL +}; + +static int sha1_compress_nblocks(hash_state *md, const unsigned char *buf, + int blocks) +{ + void *state = md->sha1.state; + + COMPILE_TIME_ASSERT(sizeof(md->sha1.state[0]) == sizeof(uint32_t)); + + crypto_accel_sha1_compress(state, buf, blocks); + return CRYPT_OK; +} + +static int sha1_compress(hash_state *md, const unsigned char *buf) +{ + return sha1_compress_nblocks(md, buf, 1); +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS_NBLOCKS(sha1_process, sha1_compress_nblocks, sha1, 64) + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha1_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->sha1.state[0] = 0x67452301UL; + md->sha1.state[1] = 0xefcdab89UL; + md->sha1.state[2] = 0x98badcfeUL; + md->sha1.state[3] = 0x10325476UL; + md->sha1.state[4] = 0xc3d2e1f0UL; + md->sha1.curlen = 0; + md->sha1.length = 0; + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (20 bytes) + @return CRYPT_OK if successful +*/ +int sha1_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha1.curlen >= sizeof(md->sha1.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->sha1.length += md->sha1.curlen * 8; + + /* append the '1' bit */ + md->sha1.buf[md->sha1.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->sha1.curlen > 56) { + while (md->sha1.curlen < 64) { + md->sha1.buf[md->sha1.curlen++] = (unsigned char)0; + } + sha1_compress(md, md->sha1.buf); + md->sha1.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->sha1.curlen < 56) { + md->sha1.buf[md->sha1.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->sha1.length, md->sha1.buf+56); + sha1_compress(md, md->sha1.buf); + + /* copy output */ + for (i = 0; i < 5; i++) { + STORE32H(md->sha1.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha1_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[20]; + } tests[] = { + { "abc", + { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, + 0xba, 0x3e, 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, + 0x9c, 0xd0, 0xd8, 0x9d } + }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + { 0x84, 0x98, 0x3E, 0x44, 0x1C, 0x3B, 0xD2, 0x6E, + 0xBA, 0xAE, 0x4A, 0xA1, 0xF9, 0x51, 0x29, 0xE5, + 0xE5, 0x46, 0x70, 0xF1 } + } + }; + + int i; + unsigned char tmp[20]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha1_init(&md); + sha1_process(&md, (unsigned char*)tests[i].msg, (unsigned long)strlen(tests[i].msg)); + sha1_done(&md, tmp); + if (XMEMCMP(tmp, tests[i].hash, 20) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} diff --git a/optee_os/core/lib/libtomcrypt/sha256_accel.c b/optee_os/core/lib/libtomcrypt/sha256_accel.c new file mode 100644 index 0000000..7e491b1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sha256_accel.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + * All rights reserved. + * Copyright (c) 2001-2007, Tom St Denis + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtom.org + */ +#include +#include + +#ifdef LTC_SHA256 + +const struct ltc_hash_descriptor sha256_desc = +{ + "sha256", + 0, + 32, + 64, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 1, }, + 9, + + &sha256_init, + &sha256_process, + &sha256_done, + &sha256_test, + NULL +}; + + +/* Implemented in assembly */ +int sha256_ce_transform(ulong32 *state, const unsigned char *buf, int blocks); + +static int sha256_compress_nblocks(hash_state *md, const unsigned char *buf, + int blocks) +{ + void *state = md->sha256.state; + + COMPILE_TIME_ASSERT(sizeof(md->sha256.state[0]) == sizeof(uint32_t)); + + crypto_accel_sha256_compress(state, buf, blocks); + return CRYPT_OK; +} + +static int sha256_compress(hash_state *md, const unsigned char *buf) +{ + return sha256_compress_nblocks(md, buf, 1); +} + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha256_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->sha256.curlen = 0; + md->sha256.length = 0; + md->sha256.state[0] = 0x6A09E667UL; + md->sha256.state[1] = 0xBB67AE85UL; + md->sha256.state[2] = 0x3C6EF372UL; + md->sha256.state[3] = 0xA54FF53AUL; + md->sha256.state[4] = 0x510E527FUL; + md->sha256.state[5] = 0x9B05688CUL; + md->sha256.state[6] = 0x1F83D9ABUL; + md->sha256.state[7] = 0x5BE0CD19UL; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS_NBLOCKS(sha256_process, sha256_compress_nblocks, sha256, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +int sha256_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha256.curlen >= sizeof(md->sha256.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->sha256.length += md->sha256.curlen * 8; + + /* append the '1' bit */ + md->sha256.buf[md->sha256.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->sha256.curlen > 56) { + while (md->sha256.curlen < 64) { + md->sha256.buf[md->sha256.curlen++] = (unsigned char)0; + } + sha256_compress(md, md->sha256.buf); + md->sha256.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->sha256.curlen < 56) { + md->sha256.buf[md->sha256.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->sha256.length, md->sha256.buf+56); + sha256_compress(md, md->sha256.buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->sha256.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha256_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[32]; + } tests[] = { + { "abc", + { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, + 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, + 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad } + }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + { 0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, + 0xe5, 0xc0, 0x26, 0x93, 0x0c, 0x3e, 0x60, 0x39, + 0xa3, 0x3c, 0xe4, 0x59, 0x64, 0xff, 0x21, 0x67, + 0xf6, 0xec, 0xed, 0xd4, 0x19, 0xdb, 0x06, 0xc1 } + }, + }; + + int i; + unsigned char tmp[32]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha256_init(&md); + sha256_process(&md, (unsigned char*)tests[i].msg, (unsigned long)strlen(tests[i].msg)); + sha256_done(&md, tmp); + if (XMEMCMP(tmp, tests[i].hash, 32) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif /*LTC_SHA256*/ diff --git a/optee_os/core/lib/libtomcrypt/sha3_accel.c b/optee_os/core/lib/libtomcrypt/sha3_accel.c new file mode 100644 index 0000000..10dbe34 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sha3_accel.c @@ -0,0 +1,245 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/* + * Copyright (c) 2023, Linaro Limited + */ + +/* based on https://github.com/brainhub/SHA3IUF (public domain) */ + +#include +#include +#include + +#ifdef LTC_SHA3 + +const struct ltc_hash_descriptor sha3_224_desc = +{ + "sha3-224", /* name of hash */ + 17, /* internal ID */ + 28, /* Size of digest in octets */ + 144, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,7 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_224_init, + &sha3_process, + &sha3_done, + &sha3_224_test, + NULL +}; + +const struct ltc_hash_descriptor sha3_256_desc = +{ + "sha3-256", /* name of hash */ + 18, /* internal ID */ + 32, /* Size of digest in octets */ + 136, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,8 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_256_init, + &sha3_process, + &sha3_done, + &sha3_256_test, + NULL +}; + +const struct ltc_hash_descriptor sha3_384_desc = +{ + "sha3-384", /* name of hash */ + 19, /* internal ID */ + 48, /* Size of digest in octets */ + 104, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,9 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_384_init, + &sha3_process, + &sha3_done, + &sha3_384_test, + NULL +}; + +const struct ltc_hash_descriptor sha3_512_desc = +{ + "sha3-512", /* name of hash */ + 20, /* internal ID */ + 64, /* Size of digest in octets */ + 72, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,10 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_512_init, + &sha3_process, + &sha3_done, + &sha3_512_test, + NULL +}; + +/* Public Inteface */ + +int sha3_224_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 224 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +int sha3_256_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 256 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +int sha3_384_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 384 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +int sha3_512_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 512 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +int sha3_shake_init(hash_state *md, int num) +{ + LTC_ARGCHK(md != NULL); + if (num != 128 && num != 256) return CRYPT_INVALID_ARG; + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = (unsigned short)(2 * num / (8 * sizeof(ulong64))); + return CRYPT_OK; +} + +int sha3_process(hash_state *md, const unsigned char *in, unsigned long inlen) +{ + unsigned int digest_size = 0; + unsigned int block_count = 0; + unsigned int block_size = 0; + void *state = NULL; + unsigned int l = 0; + + if (!inlen) + return CRYPT_OK; + LTC_ARGCHK(md); + LTC_ARGCHK(in); + + block_size = 200 - md->sha3.capacity_words * 8; + digest_size = md->sha3.capacity_words * 8 / 2; + state = md->sha3.s; + + if (md->sha3.byte_index) { + l = MIN(block_size - md->sha3.byte_index, inlen); + memcpy(md->sha3.sb + md->sha3.byte_index, in, l); + in += l; + inlen -= l; + md->sha3.byte_index += l; + if (md->sha3.byte_index == block_size) { + crypto_accel_sha3_compress(state, md->sha3.sb, 1, + digest_size); + md->sha3.byte_index = 0; + } + + if (!inlen) + return CRYPT_OK; + } + + if (inlen > block_size) { + block_count = inlen / block_size; + crypto_accel_sha3_compress(state, in, block_count, + digest_size); + in += block_count * block_size; + inlen -= block_count * block_size; + } + + memcpy(md->sha3.sb + md->sha3.byte_index, in, inlen); + md->sha3.byte_index += inlen; + + return CRYPT_OK; +} + +static void copy_out_digest(ulong64 *s, unsigned int digest_size, + unsigned char *out) +{ + unsigned int n = 0; + + for (n = 0; n < digest_size / sizeof(uint64_t); n++) { + put_unaligned_le64(out, s[n]); + out += sizeof(uint64_t); + } + + if (digest_size % sizeof(uint64_t)) + put_unaligned_le32(out, s[n]); +} + +int sha3_done(hash_state *md, unsigned char *out) +{ + unsigned int digest_size = 0; + unsigned int block_size = 0; + void *state = NULL; + uint8_t *buf = NULL; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + block_size = 200 - md->sha3.capacity_words * 8; + digest_size = md->sha3.capacity_words * 8 / 2; + state = md->sha3.s; + buf = md->sha3.sb; + + buf[md->sha3.byte_index++] = 0x06; + memset(buf + md->sha3.byte_index, 0, block_size - md->sha3.byte_index); + buf[block_size - 1] |= 0x80; + crypto_accel_sha3_compress(state, buf, 1, digest_size); + + copy_out_digest(state, digest_size, out); + + return CRYPT_OK; +} + + +int sha3_shake_done(hash_state *md, unsigned char *out, unsigned long outlen) +{ + unsigned int digest_size = 0; + unsigned int block_size = 0; + void *state = NULL; + uint8_t *buf = NULL; + unsigned int n = 0; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + block_size = 200 - md->sha3.capacity_words * 8; + digest_size = md->sha3.capacity_words * 8 / 2; + state = md->sha3.s; + buf = md->sha3.sb; + + if (!md->sha3.xof_flag) { + buf[md->sha3.byte_index++] = 0x1f; + memset(buf + md->sha3.byte_index, 0, + block_size - md->sha3.byte_index); + buf[block_size - 1] |= 0x80; + crypto_accel_sha3_compress(state, buf, 1, digest_size); + md->sha3.byte_index = 0; + copy_out_digest(state, block_size, buf); + md->sha3.xof_flag = 1; + } + + for (n = 0; n < outlen; n++) { + if (md->sha3.byte_index >= block_size) { + memset(buf, 0, block_size); + crypto_accel_sha3_compress(state, buf, 1, digest_size); + md->sha3.byte_index = 0; + copy_out_digest(state, block_size, buf); + } + out[n] = buf[md->sha3.byte_index]; + md->sha3.byte_index++; + } + + return CRYPT_OK; +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/sha512_accel.c b/optee_os/core/lib/libtomcrypt/sha512_accel.c new file mode 100644 index 0000000..948bbec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sha512_accel.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: Unlicense AND BSD-2-Clause +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* Copyright (c) 2023 Linaro Limited */ +#include +#include + +/** + @param sha512.c + LTC_SHA512 by Tom St Denis +*/ + +#ifdef LTC_SHA512 + +const struct ltc_hash_descriptor sha512_desc = +{ + "sha512", + 5, + 64, + 128, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 3, }, + 9, + + &sha512_init, + &sha512_process, + &sha512_done, + &sha512_test, + NULL +}; + + +/* Implemented in assembly */ +int sha512_ce_transform(ulong64 *state, const unsigned char *buf, int blocks); + +static int sha512_compress_nblocks(hash_state *md, const unsigned char *buf, + int blocks) +{ + void *state = md->sha512.state; + + COMPILE_TIME_ASSERT(sizeof(md->sha512.state[0]) == sizeof(uint64_t)); + + crypto_accel_sha512_compress(state, buf, blocks); + + return CRYPT_OK; +} + +static int sha512_compress(hash_state *md, const unsigned char *buf) +{ + return sha512_compress_nblocks(md, buf, 1); +} + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha512_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->sha512.curlen = 0; + md->sha512.length = 0; + md->sha512.state[0] = CONST64(0x6a09e667f3bcc908); + md->sha512.state[1] = CONST64(0xbb67ae8584caa73b); + md->sha512.state[2] = CONST64(0x3c6ef372fe94f82b); + md->sha512.state[3] = CONST64(0xa54ff53a5f1d36f1); + md->sha512.state[4] = CONST64(0x510e527fade682d1); + md->sha512.state[5] = CONST64(0x9b05688c2b3e6c1f); + md->sha512.state[6] = CONST64(0x1f83d9abfb41bd6b); + md->sha512.state[7] = CONST64(0x5be0cd19137e2179); + return CRYPT_OK; +} + +HASH_PROCESS_NBLOCKS(sha512_process, sha512_compress_nblocks, sha512, 128) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (64 bytes) + @return CRYPT_OK if successful +*/ +int sha512_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha512.curlen >= sizeof(md->sha512.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->sha512.length += md->sha512.curlen * CONST64(8); + + /* append the '1' bit */ + md->sha512.buf[md->sha512.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 112 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->sha512.curlen > 112) { + while (md->sha512.curlen < 128) { + md->sha512.buf[md->sha512.curlen++] = (unsigned char)0; + } + sha512_compress(md, md->sha512.buf); + md->sha512.curlen = 0; + } + + /* pad upto 120 bytes of zeroes + * note: that from 112 to 120 is the 64 MSB of the length. We assume that you won't hash + * > 2^64 bits of data... :-) + */ + while (md->sha512.curlen < 120) { + md->sha512.buf[md->sha512.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->sha512.length, md->sha512.buf+120); + sha512_compress(md, md->sha512.buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE64H(md->sha512.state[i], out+(8*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha512_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[64]; + } tests[] = { + { "abc", + { 0xdd, 0xaf, 0x35, 0xa1, 0x93, 0x61, 0x7a, 0xba, + 0xcc, 0x41, 0x73, 0x49, 0xae, 0x20, 0x41, 0x31, + 0x12, 0xe6, 0xfa, 0x4e, 0x89, 0xa9, 0x7e, 0xa2, + 0x0a, 0x9e, 0xee, 0xe6, 0x4b, 0x55, 0xd3, 0x9a, + 0x21, 0x92, 0x99, 0x2a, 0x27, 0x4f, 0xc1, 0xa8, + 0x36, 0xba, 0x3c, 0x23, 0xa3, 0xfe, 0xeb, 0xbd, + 0x45, 0x4d, 0x44, 0x23, 0x64, 0x3c, 0xe8, 0x0e, + 0x2a, 0x9a, 0xc9, 0x4f, 0xa5, 0x4c, 0xa4, 0x9f } + }, + { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu", + { 0x8e, 0x95, 0x9b, 0x75, 0xda, 0xe3, 0x13, 0xda, + 0x8c, 0xf4, 0xf7, 0x28, 0x14, 0xfc, 0x14, 0x3f, + 0x8f, 0x77, 0x79, 0xc6, 0xeb, 0x9f, 0x7f, 0xa1, + 0x72, 0x99, 0xae, 0xad, 0xb6, 0x88, 0x90, 0x18, + 0x50, 0x1d, 0x28, 0x9e, 0x49, 0x00, 0xf7, 0xe4, + 0x33, 0x1b, 0x99, 0xde, 0xc4, 0xb5, 0x43, 0x3a, + 0xc7, 0xd3, 0x29, 0xee, 0xb6, 0xdd, 0x26, 0x54, + 0x5e, 0x96, 0xe5, 0x5b, 0x87, 0x4b, 0xe9, 0x09 } + }, + }; + + int i; + unsigned char tmp[64]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha512_init(&md); + sha512_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha512_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA512", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif /*LTC_SHA512*/ diff --git a/optee_os/core/lib/libtomcrypt/shake.c b/optee_os/core/lib/libtomcrypt/shake.c new file mode 100644 index 0000000..50ce545 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/shake.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +struct shake_ctx { + struct crypto_hash_ctx ctx; + struct sha3_state sha3; +}; + +static struct shake_ctx *to_shake_ctx(struct crypto_hash_ctx *ctx) +{ + return container_of(ctx, struct shake_ctx, ctx); +} + +static TEE_Result do_shake_init(struct crypto_hash_ctx *ctx, unsigned int num) +{ + struct shake_ctx *c = to_shake_ctx(ctx); + + if (sha3_shake_init((void *)&c->sha3, num) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result do_sha3_update(struct crypto_hash_ctx *ctx, + const uint8_t *data, size_t len) +{ + struct shake_ctx *c = to_shake_ctx(ctx); + + if (sha3_process((void *)&c->sha3, data, len) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result do_shake_final(struct crypto_hash_ctx *ctx, + uint8_t *digest, size_t len) +{ + struct shake_ctx *c = to_shake_ctx(ctx); + + if (sha3_shake_done((void *)&c->sha3, digest, len) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result do_shake_alloc_ctx(struct crypto_hash_ctx **ctx_ret, + const struct crypto_hash_ops *ops) +{ + struct shake_ctx *ctx = calloc(1, sizeof(*ctx)); + + if (!ctx) + return TEE_ERROR_OUT_OF_MEMORY; + + ctx->ctx.ops = ops; + *ctx_ret = &ctx->ctx; + + return TEE_SUCCESS; +} + +static void do_sha3_free_ctx(struct crypto_hash_ctx *ctx) +{ + struct shake_ctx *c = to_shake_ctx(ctx); + + free(c); +} +static void do_sha3_copy_state(struct crypto_hash_ctx *dst_ctx, + struct crypto_hash_ctx *src_ctx) +{ + struct shake_ctx *dc = to_shake_ctx(dst_ctx); + struct shake_ctx *sc = to_shake_ctx(src_ctx); + + assert(sc->ctx.ops == dc->ctx.ops); + dc->sha3 = sc->sha3; +} + +#if defined(_CFG_CORE_LTC_SHAKE128) +static TEE_Result do_shake128_init(struct crypto_hash_ctx *ctx) +{ + return do_shake_init(ctx, 128); +} + +static const struct crypto_hash_ops shake128_ops = { + .init = do_shake128_init, + .update = do_sha3_update, + .final = do_shake_final, + .free_ctx = do_sha3_free_ctx, + .copy_state = do_sha3_copy_state, +}; + +TEE_Result crypto_shake128_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return do_shake_alloc_ctx(ctx, &shake128_ops); +} +#endif + +#if defined(_CFG_CORE_LTC_SHAKE256) +static TEE_Result do_shake256_init(struct crypto_hash_ctx *ctx) +{ + return do_shake_init(ctx, 256); +} + +static const struct crypto_hash_ops shake256_ops = { + .init = do_shake256_init, + .update = do_sha3_update, + .final = do_shake_final, + .free_ctx = do_sha3_free_ctx, + .copy_state = do_sha3_copy_state, +}; + +TEE_Result crypto_shake256_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return do_shake_alloc_ctx(ctx, &shake256_ops); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/sm2-dsa.c b/optee_os/core/lib/libtomcrypt/sm2-dsa.c new file mode 100644 index 0000000..3652569 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sm2-dsa.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +/* SM2 uses 256 bit unsigned integers in big endian format */ +#define SM2_INT_SIZE_BYTES 32 + +/* + * GM/T 0003.1‒2012 Part1 2 Section 6.1 + */ +TEE_Result sm2_ltc_dsa_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, uint8_t *sig, + size_t *sig_len) +{ + TEE_Result res = TEE_SUCCESS; + ecc_point *x1y1p = NULL; + ecc_key ltc_key = { }; + int ltc_res = 0; + void *k = NULL; + void *e = NULL; + void *r = NULL; + void *s = NULL; + void *tmp = NULL; + + if (*sig_len < 2 * SM2_INT_SIZE_BYTES) { + *sig_len = 64; + return TEE_ERROR_SHORT_BUFFER; + } + + ltc_res = mp_init_multi(&k, &e, &r, &s, &tmp, NULL); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_OUT_OF_MEMORY; + + x1y1p = ltc_ecc_new_point(); + if (!x1y1p) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = ecc_populate_ltc_private_key(<c_key, key, algo, NULL); + if (res) + goto out; + + /* + * Steps A1 and A2 are the generation of the hash value e from user + * information (ZA) and the message to be signed (M). There are not done + * here since @msg is expected to be the hash value e already. + */ + + /* Step A3: generate random number 1 <= k < n */ +A3: + ltc_res = rand_bn_upto(k, ltc_key.dp.order, NULL, + find_prng("prng_crypto")); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step A4: compute (x1, y1) = [k]G */ + + ltc_res = ltc_ecc_mulmod(k, <c_key.dp.base, x1y1p, ltc_key.dp.A, + ltc_key.dp.prime, 1); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step A5: compute r = (e + x1) mod n */ + + mp_read_unsigned_bin(e, (unsigned char *)msg, msg_len); + ltc_res = mp_addmod(e, x1y1p->x, ltc_key.dp.order, r); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = mp_add(r, k, tmp); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + if (mp_cmp_d(r, 0) == LTC_MP_EQ || + mp_cmp(tmp, ltc_key.dp.order) == LTC_MP_EQ) + goto A3; + + /* Step A6: compute s = ((1 + dA)^-1 * (k - r*dA)) mod n */ + + ltc_res = mp_add_d(ltc_key.k, 1, s); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = mp_invmod(s, ltc_key.dp.order, s); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = mp_mul(r, ltc_key.k, tmp); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = mp_sub(k, tmp, tmp); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = mp_mulmod(s, tmp, ltc_key.dp.order, s); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step A7: convert (r, s) to binary for output */ + + *sig_len = 2 * SM2_INT_SIZE_BYTES; + memset(sig, 0, *sig_len); + mp_to_unsigned_bin2(r, sig, SM2_INT_SIZE_BYTES); + mp_to_unsigned_bin2(s, sig + SM2_INT_SIZE_BYTES, SM2_INT_SIZE_BYTES); +out: + ecc_free(<c_key); + ltc_ecc_del_point(x1y1p); + mp_clear_multi(k, e, r, s, tmp, NULL); + return res; +} + +/* + * GM/T 0003.1‒2012 Part1 2 Section 7.1 + */ +TEE_Result sm2_ltc_dsa_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result res = TEE_SUCCESS; + ecc_key ltc_key = { }; + int ltc_res = 0; + void *rprime = NULL; + void *sprime = NULL; + void *t = NULL; + void *mp = NULL; + void *mu = NULL; + void *ma = NULL; + void *eprime = NULL; + void *R = NULL; + ecc_point *x1y1p = NULL; + + if (sig_len != 64) + return TEE_ERROR_BAD_PARAMETERS; + + ltc_res = mp_init_multi(&rprime, &sprime, &t, &mu, &ma, &eprime, &R, + NULL); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_OUT_OF_MEMORY; + + mp_read_unsigned_bin(rprime, (unsigned char *)sig, 32); + mp_read_unsigned_bin(sprime, (unsigned char *)sig + 32, 32); + + res = ecc_populate_ltc_public_key(<c_key, key, algo, NULL); + if (res) + goto out; + + /* Step B1: verify r' in [1, n - 1] */ + + if (mp_cmp_d(rprime, 1) == LTC_MP_LT || + mp_cmp(rprime, ltc_key.dp.order) != LTC_MP_LT) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + /* Step B2: verify s' in [1, n - 1] */ + + if (mp_cmp_d(sprime, 1) == LTC_MP_LT || + mp_cmp(sprime, ltc_key.dp.order) != LTC_MP_LT) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + /* + * Steps B3: M'bar = (ZA || M') and B4: e' = Hv(M'bar) are not done here + * because @msg is supposed to contain the hash value e' already. + */ + + /* Step B5: t = (r' + s') mod n and check t != 0 */ + + ltc_res = mp_addmod(rprime, sprime, ltc_key.dp.order, t); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + if (mp_cmp_d(t, 0) == LTC_MP_EQ) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + /* Step B6: (x1', y1') = [s']G + [t]PA */ + + x1y1p = ltc_ecc_new_point(); + if (!x1y1p) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + ltc_res = mp_montgomery_setup(ltc_key.dp.prime, &mp); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = mp_montgomery_normalization(mu, ltc_key.dp.prime); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = mp_mulmod(ltc_key.dp.A, mu, ltc_key.dp.prime, ma); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + ltc_res = ltc_ecc_mul2add(<c_key.dp.base, sprime, <c_key.pubkey, t, + x1y1p, ma, ltc_key.dp.prime); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step B7: compute R = (e' + x1') mod n and verify R == r' */ + + mp_read_unsigned_bin(eprime, (unsigned char *)msg, msg_len); + ltc_res = mp_addmod(eprime, x1y1p->x, ltc_key.dp.order, R); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + if (mp_cmp(R, rprime) != LTC_MP_EQ) + res = TEE_ERROR_SIGNATURE_INVALID; +out: + mp_montgomery_free(mp); + ltc_ecc_del_point(x1y1p); + ecc_free(<c_key); + mp_clear_multi(rprime, sprime, t, mu, ma, eprime, R, NULL); + return res; +} diff --git a/optee_os/core/lib/libtomcrypt/sm2-kep.c b/optee_os/core/lib/libtomcrypt/sm2-kep.c new file mode 100644 index 0000000..7bddb5f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sm2-kep.c @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +/* SM2 uses 256 bit unsigned integers in big endian format */ +#define SM2_INT_SIZE_BYTES 32 + +/* + * Compute a hash of a user's identity and public key + * For user A: ZA = SM3(ENTLA || IDA || a || b || xG || yG || xA || yA) + */ +static TEE_Result sm2_kep_compute_Z(uint8_t *Z, size_t Zlen, const uint8_t *id, + size_t idlen, const ecc_key *key) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t ENTLEN[2] = { }; + uint8_t buf[SM2_INT_SIZE_BYTES]; + void *ctx = NULL; + + if (Zlen < TEE_SM3_HASH_SIZE) + return TEE_ERROR_SHORT_BUFFER; + + /* + * ENTLEN is the length in bits if the user's distinguished identifier + * encoded over 16 bits in big endian format. + */ + ENTLEN[0] = (idlen * 8) >> 8; + ENTLEN[1] = idlen * 8; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + + res = crypto_hash_init(ctx); + if (res) + goto out; + + res = crypto_hash_update(ctx, ENTLEN, sizeof(ENTLEN)); + if (res) + goto out; + + res = crypto_hash_update(ctx, id, idlen); + if (res) + goto out; + + mp_to_unsigned_bin2(key->dp.A, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mp_to_unsigned_bin2(key->dp.B, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mp_to_unsigned_bin2(key->dp.base.x, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mp_to_unsigned_bin2(key->dp.base.y, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mp_to_unsigned_bin2(key->pubkey.x, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mp_to_unsigned_bin2(key->pubkey.y, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + res = crypto_hash_final(ctx, Z, TEE_SM3_HASH_SIZE); +out: + crypto_hash_free_ctx(ctx); + return res; +} + +/* + * Compute a verification value, to be checked against the value sent by the + * peer. + * On the initiator's side: + * S1 = SM3(0x02 || yU || SM3(xU || ZA || ZB || x1 || y1 || x2 || y2)) + * On the responder's side: + * S2 = SM3(0x03 || yV || SM3(xV || ZA || ZB || x1 || y1 || x2 || y2)) + */ +static TEE_Result sm2_kep_compute_S(uint8_t *S, size_t S_len, uint8_t flag, + ecc_point *UV, const uint8_t *ZAZB, + size_t ZAZB_len, ecc_key *initiator_eph_key, + ecc_key *responder_eph_key) +{ + uint8_t hash[TEE_SM3_HASH_SIZE] = { }; + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t buf[SM2_INT_SIZE_BYTES]; + void *ctx = NULL; + + if (S_len < TEE_SM3_HASH_SIZE) + return TEE_ERROR_SHORT_BUFFER; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + + /* Compute the inner hash */ + + res = crypto_hash_init(ctx); + if (res) + goto out; + + /* xU or xV */ + mp_to_unsigned_bin2(UV->x, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* ZA || ZB */ + res = crypto_hash_update(ctx, ZAZB, ZAZB_len); + if (res) + goto out; + + /* x1 */ + mp_to_unsigned_bin2(initiator_eph_key->pubkey.x, buf, + SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* y1 */ + mp_to_unsigned_bin2(initiator_eph_key->pubkey.y, buf, + SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* x2 */ + mp_to_unsigned_bin2(responder_eph_key->pubkey.x, buf, + SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* y2 */ + mp_to_unsigned_bin2(responder_eph_key->pubkey.y, buf, + SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + res = crypto_hash_final(ctx, hash, sizeof(hash)); + if (res) + goto out; + + /* Now compute S */ + + res = crypto_hash_init(ctx); + if (res) + goto out; + + /* 0x02 or 0x03 */ + res = crypto_hash_update(ctx, &flag, sizeof(flag)); + if (res) + goto out; + + /* yU or yV */ + mp_to_unsigned_bin2(UV->y, buf, SM2_INT_SIZE_BYTES); + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* Inner SM3(...) */ + res = crypto_hash_update(ctx, hash, sizeof(hash)); + if (res) + goto out; + + res = crypto_hash_final(ctx, S, TEE_SM3_HASH_SIZE); + +out: + crypto_hash_free_ctx(ctx); + return res; + +} + +/* + * GM/T 0003.1‒2012 Part 3 Section 6.1 + * Key exchange protocol + */ +static TEE_Result sm2_kep_derive(ecc_key *my_key, ecc_key *my_eph_key, + ecc_key *peer_key, ecc_key *peer_eph_key, + struct sm2_kep_parms *p) +{ + /* + * Variable names and documented steps reflect the initator side (user A + * in the spec), but the other side is quite similar hence only one + * function. + */ + uint8_t xUyUZAZB[2 * SM2_INT_SIZE_BYTES + 2 * TEE_SM3_HASH_SIZE] = { }; + ecc_key *initiator_eph_key = p->is_initiator ? my_eph_key : + peer_eph_key; + ecc_key *responder_eph_key = p->is_initiator ? peer_eph_key : + my_eph_key; + ecc_key *initiator_key = p->is_initiator ? my_key : peer_key; + ecc_key *responder_key = p->is_initiator ? peer_key : my_key; + TEE_Result res = TEE_ERROR_BAD_STATE; + uint8_t tmp[SM2_INT_SIZE_BYTES]; + void *n = my_key->dp.order; + ecc_point *U = NULL; + void *x1bar = NULL; + void *x2bar = NULL; + void *tA = NULL; + void *h = NULL; + void *htA = NULL; + void *mp = NULL; + void *mu = NULL; + void *ma = NULL; + void *one = NULL; + int ltc_res = 0; + int inf = 0; + + ltc_res = mp_init_multi(&x1bar, &x2bar, &tA, &h, &htA, &mu, &ma, &one, + NULL); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + U = ltc_ecc_new_point(); + if (!U) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + /* + * Steps A1-A3 are supposedly done already (generate ephemeral key, send + * it to peer). + * Step A4: (x1, y1) = RA; x1bar = 2^w + (x1 & (2^w - 1)) + */ + + mp_to_unsigned_bin2(my_eph_key->pubkey.x, tmp, SM2_INT_SIZE_BYTES); + tmp[SM2_INT_SIZE_BYTES / 2] |= 0x80; + mp_read_unsigned_bin(x1bar, tmp + SM2_INT_SIZE_BYTES / 2, + SM2_INT_SIZE_BYTES / 2); + + /* Step A5: tA = (dA + x1bar * rA) mod n */ + + ltc_res = mp_mulmod(x1bar, my_eph_key->k, n, tA); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = mp_addmod(tA, my_key->k, n, tA); + if (ltc_res != CRYPT_OK) + goto out; + + /* Step A6: verify whether RB verifies the curve equation */ + + ltc_res = ltc_ecc_is_point(&peer_eph_key->dp, peer_eph_key->pubkey.x, + peer_eph_key->pubkey.y); + if (ltc_res != CRYPT_OK) + goto out; + + /* Step A6 (continued): (x2, y2) = RB; x2bar = 2^w + (x2 & (2^w - 1)) */ + + mp_to_unsigned_bin2(peer_eph_key->pubkey.x, tmp, SM2_INT_SIZE_BYTES); + tmp[SM2_INT_SIZE_BYTES / 2] |= 0x80; + mp_read_unsigned_bin(x2bar, tmp + SM2_INT_SIZE_BYTES / 2, + SM2_INT_SIZE_BYTES / 2); + + + /* Step A7: compute U = [h.tA](PB + [x2bar]RB) and check for infinity */ + + ltc_res = mp_montgomery_setup(peer_key->dp.prime, &mp); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = mp_montgomery_normalization(mu, peer_key->dp.prime); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = mp_mulmod(peer_key->dp.A, mu, peer_key->dp.prime, ma); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = mp_set_int(one, 1); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = ltc_ecc_mul2add(&peer_key->pubkey, one, &peer_eph_key->pubkey, + x2bar, U, ma, peer_key->dp.prime); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = mp_set_int(h, peer_key->dp.cofactor); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = mp_mul(h, tA, htA); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = ltc_ecc_mulmod(htA, U, U, peer_key->dp.A, peer_key->dp.prime, + 1); + if (ltc_res != CRYPT_OK) + goto out; + + ltc_res = ltc_ecc_is_point_at_infinity(U, peer_key->dp.prime, &inf); + if (ltc_res != CRYPT_OK) + goto out; + + if (inf) + goto out; + + /* Step A8: compute KA = KDF(xU || yU || ZA || ZB, klen) */ + + /* xU */ + mp_to_unsigned_bin2(U->x, xUyUZAZB, SM2_INT_SIZE_BYTES); + + /* yU */ + mp_to_unsigned_bin2(U->y, xUyUZAZB + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + + /* ZA */ + res = sm2_kep_compute_Z(xUyUZAZB + 2 * SM2_INT_SIZE_BYTES, + TEE_SM3_HASH_SIZE, p->initiator_id, + p->initiator_id_len, initiator_key); + if (res) + goto out; + + /* ZB */ + res = sm2_kep_compute_Z(xUyUZAZB + 2 * SM2_INT_SIZE_BYTES + + TEE_SM3_HASH_SIZE, + TEE_SM3_HASH_SIZE, p->responder_id, + p->responder_id_len, responder_key); + if (res) + goto out; + + res = sm2_kdf(xUyUZAZB, sizeof(xUyUZAZB), p->out, p->out_len); + if (res) + goto out; + + /* Step A9: compute S1 and check S1 == SB */ + + if (p->conf_in) { + uint8_t S1[TEE_SM3_HASH_SIZE]; + uint8_t flag = p->is_initiator ? 0x02 : 0x03; + + if (p->conf_in_len < TEE_SM3_HASH_SIZE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + res = sm2_kep_compute_S(S1, sizeof(S1), flag, U, + xUyUZAZB + 2 * SM2_INT_SIZE_BYTES, + 2 * SM2_INT_SIZE_BYTES, + initiator_eph_key, responder_eph_key); + if (res) + goto out; + + if (consttime_memcmp(S1, p->conf_in, sizeof(S1))) { + /* Verification failed */ + res = TEE_ERROR_BAD_STATE; + goto out; + } + } + + /* Step A10: compute SA */ + + if (p->conf_out) { + uint8_t flag = p->is_initiator ? 0x03 : 0x02; + + if (p->conf_out_len < TEE_SM3_HASH_SIZE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = sm2_kep_compute_S(p->conf_out, TEE_SM3_HASH_SIZE, flag, U, + xUyUZAZB + 2 * SM2_INT_SIZE_BYTES, + 2 * SM2_INT_SIZE_BYTES, + initiator_eph_key, responder_eph_key); + if (res) + goto out; + } +out: + mp_montgomery_free(mp); + ltc_ecc_del_point(U); + mp_clear_multi(x1bar, x2bar, tA, h, htA, mu, ma, one, NULL); + return res; +} + +TEE_Result crypto_acipher_sm2_kep_derive(struct ecc_keypair *my_key, + struct ecc_keypair *my_eph_key, + struct ecc_public_key *peer_key, + struct ecc_public_key *peer_eph_key, + struct sm2_kep_parms *p) +{ + TEE_Result res = TEE_SUCCESS; + ecc_key ltc_my_key = { }; + ecc_key ltc_my_eph_key = { }; + ecc_key ltc_peer_key = { }; + ecc_key ltc_peer_eph_key = { }; + + res = ecc_populate_ltc_private_key(<c_my_key, my_key, + TEE_ALG_SM2_KEP, NULL); + if (res) + goto out; + + res = ecc_populate_ltc_private_key(<c_my_eph_key, my_eph_key, + TEE_ALG_SM2_KEP, NULL); + if (res) + goto out; + + res = ecc_populate_ltc_public_key(<c_peer_key, peer_key, + TEE_ALG_SM2_KEP, NULL); + if (res) + goto out; + + res = ecc_populate_ltc_public_key(<c_peer_eph_key, peer_eph_key, + TEE_ALG_SM2_KEP, NULL); + if (res) + goto out; + + res = sm2_kep_derive(<c_my_key, <c_my_eph_key, <c_peer_key, + <c_peer_eph_key, p); +out: + ecc_free(<c_peer_eph_key); + ecc_free(<c_peer_key); + ecc_free(<c_my_eph_key); + ecc_free(<c_my_key); + return res; +} diff --git a/optee_os/core/lib/libtomcrypt/sm2-pke.c b/optee_os/core/lib/libtomcrypt/sm2-pke.c new file mode 100644 index 0000000..56a3cc5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sm2-pke.c @@ -0,0 +1,515 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +/* SM2 uses 256 bit unsigned integers in big endian format */ +#define SM2_INT_SIZE_BYTES 32 + +static TEE_Result +sm2_uncompressed_bytes_to_point(ecc_point *p, const ltc_ecc_dp *dp, + const uint8_t *x1y1, size_t max_size, + size_t *consumed) +{ + uint8_t *ptr = (uint8_t *)x1y1; + uint8_t one[] = { 1 }; + int ltc_res = 0; + + if (max_size < (size_t)(2 * SM2_INT_SIZE_BYTES)) + return TEE_ERROR_BAD_PARAMETERS; + + ltc_res = mp_read_unsigned_bin(p->x, ptr, SM2_INT_SIZE_BYTES); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + ptr += SM2_INT_SIZE_BYTES; + + ltc_res = mp_read_unsigned_bin(p->y, ptr, SM2_INT_SIZE_BYTES); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + ltc_res = ltc_ecc_is_point(dp, p->x, p->y); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + mp_read_unsigned_bin(p->z, one, sizeof(one)); + + *consumed = 2 * SM2_INT_SIZE_BYTES + 1; /* PC */ + + return TEE_SUCCESS; +} + +/* + * GM/T 0003.1‒2012 Part 1 Section 4.2.9 + * Conversion of a byte string @buf to a point @p. Makes sure @p is on the curve + * defined by domain parameters @dp. + * Note: only the uncompressed form is supported. Uncompressed and hybrid forms + * are TBD. + */ +static TEE_Result sm2_bytes_to_point(ecc_point *p, const ltc_ecc_dp *dp, + const uint8_t *buf, size_t max_size, + size_t *consumed) +{ + uint8_t PC = 0; + + if (!max_size) + return TEE_ERROR_BAD_PARAMETERS; + + PC = buf[0]; + + switch (PC) { + case 0x02: + case 0x03: + /* Compressed form */ + return TEE_ERROR_NOT_SUPPORTED; + case 0x04: + /* UNcompressed form */ + return sm2_uncompressed_bytes_to_point(p, dp, buf + 1, + max_size - 1, consumed); + case 0x06: + case 0x07: + /* Hybrid form */ + return TEE_ERROR_NOT_SUPPORTED; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_ERROR_GENERIC; +} + +static bool is_zero(const uint8_t *buf, size_t size) +{ + uint8_t v = 0; + size_t i = 0; + + for (i = 0; i < size; i++) + v |= buf[i]; + + return !v; +} + +/* + * GM/T 0003.1‒2012 Part 4 Section 7.1 + * Decryption algorithm + */ +TEE_Result sm2_ltc_pke_decrypt(struct ecc_keypair *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t x2y2[64] = { }; + ecc_key ltc_key = { }; + ecc_point *C1 = NULL; + size_t C1_len = 0; + ecc_point *S = NULL; + ecc_point *x2y2p = NULL; + void *ctx = NULL; + int ltc_res = 0; + void *h = NULL; + int inf = 0; + uint8_t *t = NULL; + size_t C2_len = 0; + size_t i = 0; + size_t out_len = 0; + uint8_t *eom = NULL; + uint8_t u[TEE_SM3_HASH_SIZE] = { }; + + /* + * Input buffer src is (C1 || C2 || C3) + * - C1 represents a point (should be on the curve) + * - C2 is the encrypted message + * - C3 is a SM3 hash + */ + + res = ecc_populate_ltc_private_key(<c_key, key, TEE_ALG_SM2_PKE, + NULL); + if (res) + goto out; + + /* Step B1: read and validate point C1 from encrypted message */ + + C1 = ltc_ecc_new_point(); + if (!C1) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = sm2_bytes_to_point(C1, <c_key.dp, src, src_len, &C1_len); + if (res) + goto out; + + /* Step B2: S = [h]C1 */ + + if (ltc_key.dp.cofactor != 1) { + S = ltc_ecc_new_point(); + if (!S) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = mp_init_multi(&h, NULL); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = mp_set_int(h, ltc_key.dp.cofactor); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + ltc_res = ltc_ecc_mulmod(h, C1, S, ltc_key.dp.A, + ltc_key.dp.prime, 1); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + ltc_res = ltc_ecc_is_point_at_infinity(S, ltc_key.dp.prime, + &inf); + } else { + ltc_res = ltc_ecc_is_point_at_infinity(C1, ltc_key.dp.prime, + &inf); + } + if (ltc_res != CRYPT_OK || inf) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step B3: (x2, y2) = [dB]C1 */ + + x2y2p = ltc_ecc_new_point(); + if (!x2y2p) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = ltc_ecc_mulmod(ltc_key.k, C1, x2y2p, ltc_key.dp.A, + ltc_key.dp.prime, 1); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + if (mp_unsigned_bin_size(x2y2p->x) > SM2_INT_SIZE_BYTES || + mp_unsigned_bin_size(x2y2p->y) > SM2_INT_SIZE_BYTES) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + mp_to_unsigned_bin2(x2y2p->x, x2y2, SM2_INT_SIZE_BYTES); + mp_to_unsigned_bin2(x2y2p->y, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + + /* Step B4: t = KDF(x2 || y2, klen) */ + + /* C = C1 || C2 || C3 */ + if (src_len <= C1_len + TEE_SM3_HASH_SIZE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + C2_len = src_len - C1_len - TEE_SM3_HASH_SIZE; + + t = calloc(1, C2_len); + if (!t) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = sm2_kdf(x2y2, sizeof(x2y2), t, C2_len); + if (res) + goto out; + + if (is_zero(t, C2_len)) { + res = TEE_ERROR_CIPHERTEXT_INVALID; + goto out; + } + + /* Step B5: get C2 from C and compute Mprime = C2 (+) t */ + + out_len = MIN(*dst_len, C2_len); + for (i = 0; i < out_len; i++) + dst[i] = src[C1_len + i] ^ t[i]; + *dst_len = out_len; + if (out_len < C2_len) { + eom = calloc(1, C2_len - out_len); + if (!eom) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + for (i = out_len; i < C2_len; i++) + eom[i - out_len] = src[C1_len + i] ^ t[i]; + } + + /* Step B6: compute u = Hash(x2 || M' || y2) and compare with C3 */ + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, x2y2, SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_update(ctx, dst, out_len); + if (res) + goto out; + if (out_len < C2_len) { + res = crypto_hash_update(ctx, eom, C2_len - out_len); + if (res) + goto out; + } + res = crypto_hash_update(ctx, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_final(ctx, u, sizeof(u)); + if (res) + goto out; + + if (consttime_memcmp(u, src + C1_len + C2_len, TEE_SM3_HASH_SIZE)) { + res = TEE_ERROR_CIPHERTEXT_INVALID; + goto out; + } +out: + free(eom); + free(t); + crypto_hash_free_ctx(ctx); + ltc_ecc_del_point(x2y2p); + ltc_ecc_del_point(S); + ltc_ecc_del_point(C1); + mp_clear_multi(h, NULL); + ecc_free(<c_key); + return res; +} + +/* + * GM/T 0003.1‒2012 Part 1 Section 4.2.8 + * Conversion of point @p to a byte string @buf (uncompressed form). + */ +static TEE_Result sm2_point_to_bytes(uint8_t *buf, size_t *size, + const ecc_point *p) +{ + size_t xsize = mp_unsigned_bin_size(p->x); + size_t ysize = mp_unsigned_bin_size(p->y); + size_t sz = 2 * SM2_INT_SIZE_BYTES + 1; + + if (xsize > SM2_INT_SIZE_BYTES || ysize > SM2_INT_SIZE_BYTES || + *size < sz) + return TEE_ERROR_BAD_STATE; + + memset(buf, 0, sz); + buf[0] = 0x04; /* Uncompressed form indicator */ + mp_to_unsigned_bin2(p->x, buf + 1, SM2_INT_SIZE_BYTES); + mp_to_unsigned_bin2(p->y, buf + 1 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + + *size = sz; + + return TEE_SUCCESS; +} + +/* + * GM/T 0003.1‒2012 Part 4 Section 6.1 + * Encryption algorithm + */ +TEE_Result sm2_ltc_pke_encrypt(struct ecc_public_key *key, const uint8_t *src, + size_t src_len, uint8_t *dst, size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + ecc_key ltc_key = { }; + ecc_point *x2y2p = NULL; + ecc_point *C1 = NULL; + ecc_point *S = NULL; + uint8_t x2y2[64] = { }; + uint8_t *t = NULL; + int ltc_res = 0; + void *k = NULL; + void *h = NULL; + int inf = 0; + size_t C1_len = 0; + void *ctx = NULL; + size_t i = 0; + + ltc_res = mp_init_multi(&k, &h, NULL); + if (ltc_res != CRYPT_OK) + return TEE_ERROR_OUT_OF_MEMORY; + + res = ecc_populate_ltc_public_key(<c_key, key, TEE_ALG_SM2_PKE, NULL); + if (res) + goto out; + + /* Step A1: generate random number 1 <= k < n */ + + ltc_res = rand_bn_upto(k, ltc_key.dp.order, NULL, + find_prng("prng_crypto")); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step A2: compute C1 = [k]G */ + + C1 = ltc_ecc_new_point(); + if (!C1) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = ltc_ecc_mulmod(k, <c_key.dp.base, C1, ltc_key.dp.A, + ltc_key.dp.prime, 1); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step A3: compute S = [h]PB and check for infinity */ + + if (ltc_key.dp.cofactor != 1) { + S = ltc_ecc_new_point(); + if (!S) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = mp_set_int(h, ltc_key.dp.cofactor); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + ltc_res = ltc_ecc_mulmod(h, <c_key.pubkey, S, ltc_key.dp.A, + ltc_key.dp.prime, 1); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + ltc_res = ltc_ecc_is_point_at_infinity(S, ltc_key.dp.prime, + &inf); + } else { + ltc_res = ltc_ecc_is_point_at_infinity(<c_key.pubkey, + ltc_key.dp.prime, &inf); + } + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + if (inf) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step A4: compute (x2, y2) = [k]PB */ + + x2y2p = ltc_ecc_new_point(); + if (!x2y2p) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + ltc_res = ltc_ecc_mulmod(k, <c_key.pubkey, x2y2p, ltc_key.dp.A, + ltc_key.dp.prime, 1); + if (ltc_res != CRYPT_OK) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + if (mp_unsigned_bin_size(x2y2p->x) > SM2_INT_SIZE_BYTES || + mp_unsigned_bin_size(x2y2p->y) > SM2_INT_SIZE_BYTES) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + mp_to_unsigned_bin2(x2y2p->x, x2y2, SM2_INT_SIZE_BYTES); + mp_to_unsigned_bin2(x2y2p->y, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + + /* Step A5: compute t = KDF(x2 || y2, klen) */ + + t = calloc(1, src_len); + if (!t) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = sm2_kdf(x2y2, sizeof(x2y2), t, src_len); + if (res) + goto out; + + if (is_zero(t, src_len)) { + res = TEE_ERROR_CIPHERTEXT_INVALID; + goto out; + } + + /* + * Steps A6, A7, A8: + * Compute C2 = M (+) t + * Compute C3 = Hash(x2 || M || y2) + * Output C = C1 || C2 || C3 + */ + + /* C1 */ + C1_len = *dst_len; + res = sm2_point_to_bytes(dst, &C1_len, C1); + if (res) + goto out; + + if (*dst_len < C1_len + src_len + TEE_SM3_HASH_SIZE) { + *dst_len = C1_len + src_len + TEE_SM3_HASH_SIZE; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + /* C2 */ + for (i = 0; i < src_len; i++) + dst[i + C1_len] = src[i] ^ t[i]; + + /* C3 */ + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, x2y2, SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_update(ctx, src, src_len); + if (res) + goto out; + res = crypto_hash_update(ctx, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_final(ctx, dst + C1_len + src_len, TEE_SM3_HASH_SIZE); + if (res) + goto out; + + *dst_len = C1_len + src_len + TEE_SM3_HASH_SIZE; +out: + crypto_hash_free_ctx(ctx); + free(t); + ltc_ecc_del_point(x2y2p); + ltc_ecc_del_point(S); + ltc_ecc_del_point(C1); + ecc_free(<c_key); + mp_clear_multi(k, h, NULL); + return res; +} diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/aes/aes.c b/optee_os/core/lib/libtomcrypt/src/ciphers/aes/aes.c new file mode 100644 index 0000000..dd6f14e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/aes/aes.c @@ -0,0 +1,744 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* AES implementation by Tom St Denis + * + * Derived from the Public Domain source code by + +--- + * rijndael-alg-fst.c + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto +--- + */ +/** + @file aes.c + Implementation of AES +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_RIJNDAEL + +#ifndef ENCRYPT_ONLY + +#define SETUP rijndael_setup +#define ECB_ENC rijndael_ecb_encrypt +#define ECB_DEC rijndael_ecb_decrypt +#define ECB_DONE rijndael_done +#define ECB_TEST rijndael_test +#define ECB_KS rijndael_keysize + +const struct ltc_cipher_descriptor rijndael_desc = +{ + "rijndael", + 6, + 16, 32, 16, 10, + SETUP, ECB_ENC, ECB_DEC, ECB_TEST, ECB_DONE, ECB_KS, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +const struct ltc_cipher_descriptor aes_desc = +{ + "aes", + 6, + 16, 32, 16, 10, + SETUP, ECB_ENC, ECB_DEC, ECB_TEST, ECB_DONE, ECB_KS, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +#else + +#define SETUP rijndael_enc_setup +#define ECB_ENC rijndael_enc_ecb_encrypt +#define ECB_KS rijndael_enc_keysize +#define ECB_DONE rijndael_enc_done + +const struct ltc_cipher_descriptor rijndael_enc_desc = +{ + "rijndael", + 6, + 16, 32, 16, 10, + SETUP, ECB_ENC, NULL, NULL, ECB_DONE, ECB_KS, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +const struct ltc_cipher_descriptor aes_enc_desc = +{ + "aes", + 6, + 16, 32, 16, 10, + SETUP, ECB_ENC, NULL, NULL, ECB_DONE, ECB_KS, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +#endif + +#define LTC_AES_TAB_C +#include "aes_tab.c" + +static ulong32 setup_mix(ulong32 temp) +{ + return (Te4_3[LTC_BYTE(temp, 2)]) ^ + (Te4_2[LTC_BYTE(temp, 1)]) ^ + (Te4_1[LTC_BYTE(temp, 0)]) ^ + (Te4_0[LTC_BYTE(temp, 3)]); +} + +#ifndef ENCRYPT_ONLY +#ifdef LTC_SMALL_CODE +static ulong32 setup_mix2(ulong32 temp) +{ + return Td0(255 & Te4[LTC_BYTE(temp, 3)]) ^ + Td1(255 & Te4[LTC_BYTE(temp, 2)]) ^ + Td2(255 & Te4[LTC_BYTE(temp, 1)]) ^ + Td3(255 & Te4[LTC_BYTE(temp, 0)]); +} +#endif +#endif + + /** + Initialize the AES (Rijndael) block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int SETUP(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int i; + ulong32 temp, *rk; +#ifndef ENCRYPT_ONLY + ulong32 *rrk; +#endif + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (keylen != 16 && keylen != 24 && keylen != 32) { + return CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 0 && num_rounds != (10 + ((keylen/8)-2)*2)) { + return CRYPT_INVALID_ROUNDS; + } + + skey->rijndael.Nr = 10 + ((keylen/8)-2)*2; + + /* setup the forward key */ + i = 0; + rk = skey->rijndael.eK; + LOAD32H(rk[0], key ); + LOAD32H(rk[1], key + 4); + LOAD32H(rk[2], key + 8); + LOAD32H(rk[3], key + 12); + if (keylen == 16) { + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ setup_mix(temp) ^ rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + if (++i == 10) { + break; + } + rk += 4; + } + } else if (keylen == 24) { + LOAD32H(rk[4], key + 16); + LOAD32H(rk[5], key + 20); + for (;;) { + #ifdef _MSC_VER + temp = skey->rijndael.eK[rk - skey->rijndael.eK + 5]; + #else + temp = rk[5]; + #endif + rk[ 6] = rk[ 0] ^ setup_mix(temp) ^ rcon[i]; + rk[ 7] = rk[ 1] ^ rk[ 6]; + rk[ 8] = rk[ 2] ^ rk[ 7]; + rk[ 9] = rk[ 3] ^ rk[ 8]; + if (++i == 8) { + break; + } + rk[10] = rk[ 4] ^ rk[ 9]; + rk[11] = rk[ 5] ^ rk[10]; + rk += 6; + } + } else if (keylen == 32) { + LOAD32H(rk[4], key + 16); + LOAD32H(rk[5], key + 20); + LOAD32H(rk[6], key + 24); + LOAD32H(rk[7], key + 28); + for (;;) { + #ifdef _MSC_VER + temp = skey->rijndael.eK[rk - skey->rijndael.eK + 7]; + #else + temp = rk[7]; + #endif + rk[ 8] = rk[ 0] ^ setup_mix(temp) ^ rcon[i]; + rk[ 9] = rk[ 1] ^ rk[ 8]; + rk[10] = rk[ 2] ^ rk[ 9]; + rk[11] = rk[ 3] ^ rk[10]; + if (++i == 7) { + break; + } + temp = rk[11]; + rk[12] = rk[ 4] ^ setup_mix(RORc(temp, 8)); + rk[13] = rk[ 5] ^ rk[12]; + rk[14] = rk[ 6] ^ rk[13]; + rk[15] = rk[ 7] ^ rk[14]; + rk += 8; + } + } else { + /* this can't happen */ + /* coverity[dead_error_line] */ + return CRYPT_ERROR; + } + +#ifndef ENCRYPT_ONLY + /* setup the inverse key now */ + rk = skey->rijndael.dK; + rrk = skey->rijndael.eK + (28 + keylen) - 4; + + /* apply the inverse MixColumn transform to all round keys but the first and the last: */ + /* copy first */ + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk = *rrk; + rk -= 3; rrk -= 3; + + for (i = 1; i < skey->rijndael.Nr; i++) { + rrk -= 4; + rk += 4; + #ifdef LTC_SMALL_CODE + temp = rrk[0]; + rk[0] = setup_mix2(temp); + temp = rrk[1]; + rk[1] = setup_mix2(temp); + temp = rrk[2]; + rk[2] = setup_mix2(temp); + temp = rrk[3]; + rk[3] = setup_mix2(temp); + #else + temp = rrk[0]; + rk[0] = + Tks0[LTC_BYTE(temp, 3)] ^ + Tks1[LTC_BYTE(temp, 2)] ^ + Tks2[LTC_BYTE(temp, 1)] ^ + Tks3[LTC_BYTE(temp, 0)]; + temp = rrk[1]; + rk[1] = + Tks0[LTC_BYTE(temp, 3)] ^ + Tks1[LTC_BYTE(temp, 2)] ^ + Tks2[LTC_BYTE(temp, 1)] ^ + Tks3[LTC_BYTE(temp, 0)]; + temp = rrk[2]; + rk[2] = + Tks0[LTC_BYTE(temp, 3)] ^ + Tks1[LTC_BYTE(temp, 2)] ^ + Tks2[LTC_BYTE(temp, 1)] ^ + Tks3[LTC_BYTE(temp, 0)]; + temp = rrk[3]; + rk[3] = + Tks0[LTC_BYTE(temp, 3)] ^ + Tks1[LTC_BYTE(temp, 2)] ^ + Tks2[LTC_BYTE(temp, 1)] ^ + Tks3[LTC_BYTE(temp, 0)]; + #endif + + } + + /* copy last */ + rrk -= 4; + rk += 4; + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk = *rrk; +#endif /* ENCRYPT_ONLY */ + + return CRYPT_OK; +} + +/** + Encrypts a block of text with AES + @param pt The input plaintext (16 bytes) + @param ct The output ciphertext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_rijndael_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int ECB_ENC(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + ulong32 s0, s1, s2, s3, t0, t1, t2, t3; + const ulong32 *rk; + int Nr, r; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + Nr = skey->rijndael.Nr; + + if (Nr < 2 || Nr > 16) + return CRYPT_INVALID_ROUNDS; + + rk = skey->rijndael.eK; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + LOAD32H(s0, pt ); s0 ^= rk[0]; + LOAD32H(s1, pt + 4); s1 ^= rk[1]; + LOAD32H(s2, pt + 8); s2 ^= rk[2]; + LOAD32H(s3, pt + 12); s3 ^= rk[3]; + +#ifdef LTC_SMALL_CODE + + for (r = 0; ; r++) { + rk += 4; + t0 = + Te0(LTC_BYTE(s0, 3)) ^ + Te1(LTC_BYTE(s1, 2)) ^ + Te2(LTC_BYTE(s2, 1)) ^ + Te3(LTC_BYTE(s3, 0)) ^ + rk[0]; + t1 = + Te0(LTC_BYTE(s1, 3)) ^ + Te1(LTC_BYTE(s2, 2)) ^ + Te2(LTC_BYTE(s3, 1)) ^ + Te3(LTC_BYTE(s0, 0)) ^ + rk[1]; + t2 = + Te0(LTC_BYTE(s2, 3)) ^ + Te1(LTC_BYTE(s3, 2)) ^ + Te2(LTC_BYTE(s0, 1)) ^ + Te3(LTC_BYTE(s1, 0)) ^ + rk[2]; + t3 = + Te0(LTC_BYTE(s3, 3)) ^ + Te1(LTC_BYTE(s0, 2)) ^ + Te2(LTC_BYTE(s1, 1)) ^ + Te3(LTC_BYTE(s2, 0)) ^ + rk[3]; + if (r == Nr-2) { + break; + } + s0 = t0; s1 = t1; s2 = t2; s3 = t3; + } + rk += 4; + +#else + + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = + Te0(LTC_BYTE(s0, 3)) ^ + Te1(LTC_BYTE(s1, 2)) ^ + Te2(LTC_BYTE(s2, 1)) ^ + Te3(LTC_BYTE(s3, 0)) ^ + rk[4]; + t1 = + Te0(LTC_BYTE(s1, 3)) ^ + Te1(LTC_BYTE(s2, 2)) ^ + Te2(LTC_BYTE(s3, 1)) ^ + Te3(LTC_BYTE(s0, 0)) ^ + rk[5]; + t2 = + Te0(LTC_BYTE(s2, 3)) ^ + Te1(LTC_BYTE(s3, 2)) ^ + Te2(LTC_BYTE(s0, 1)) ^ + Te3(LTC_BYTE(s1, 0)) ^ + rk[6]; + t3 = + Te0(LTC_BYTE(s3, 3)) ^ + Te1(LTC_BYTE(s0, 2)) ^ + Te2(LTC_BYTE(s1, 1)) ^ + Te3(LTC_BYTE(s2, 0)) ^ + rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = + Te0(LTC_BYTE(t0, 3)) ^ + Te1(LTC_BYTE(t1, 2)) ^ + Te2(LTC_BYTE(t2, 1)) ^ + Te3(LTC_BYTE(t3, 0)) ^ + rk[0]; + s1 = + Te0(LTC_BYTE(t1, 3)) ^ + Te1(LTC_BYTE(t2, 2)) ^ + Te2(LTC_BYTE(t3, 1)) ^ + Te3(LTC_BYTE(t0, 0)) ^ + rk[1]; + s2 = + Te0(LTC_BYTE(t2, 3)) ^ + Te1(LTC_BYTE(t3, 2)) ^ + Te2(LTC_BYTE(t0, 1)) ^ + Te3(LTC_BYTE(t1, 0)) ^ + rk[2]; + s3 = + Te0(LTC_BYTE(t3, 3)) ^ + Te1(LTC_BYTE(t0, 2)) ^ + Te2(LTC_BYTE(t1, 1)) ^ + Te3(LTC_BYTE(t2, 0)) ^ + rk[3]; + } + +#endif + + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = + (Te4_3[LTC_BYTE(t0, 3)]) ^ + (Te4_2[LTC_BYTE(t1, 2)]) ^ + (Te4_1[LTC_BYTE(t2, 1)]) ^ + (Te4_0[LTC_BYTE(t3, 0)]) ^ + rk[0]; + STORE32H(s0, ct); + s1 = + (Te4_3[LTC_BYTE(t1, 3)]) ^ + (Te4_2[LTC_BYTE(t2, 2)]) ^ + (Te4_1[LTC_BYTE(t3, 1)]) ^ + (Te4_0[LTC_BYTE(t0, 0)]) ^ + rk[1]; + STORE32H(s1, ct+4); + s2 = + (Te4_3[LTC_BYTE(t2, 3)]) ^ + (Te4_2[LTC_BYTE(t3, 2)]) ^ + (Te4_1[LTC_BYTE(t0, 1)]) ^ + (Te4_0[LTC_BYTE(t1, 0)]) ^ + rk[2]; + STORE32H(s2, ct+8); + s3 = + (Te4_3[LTC_BYTE(t3, 3)]) ^ + (Te4_2[LTC_BYTE(t0, 2)]) ^ + (Te4_1[LTC_BYTE(t1, 1)]) ^ + (Te4_0[LTC_BYTE(t2, 0)]) ^ + rk[3]; + STORE32H(s3, ct+12); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int ECB_ENC(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_rijndael_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(unsigned long)*8 + sizeof(unsigned long*) + sizeof(int)*2); + return err; +} +#endif + +#ifndef ENCRYPT_ONLY + +/** + Decrypts a block of text with AES + @param ct The input ciphertext (16 bytes) + @param pt The output plaintext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_rijndael_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int ECB_DEC(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + ulong32 s0, s1, s2, s3, t0, t1, t2, t3; + const ulong32 *rk; + int Nr, r; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + Nr = skey->rijndael.Nr; + + if (Nr < 2 || Nr > 16) + return CRYPT_INVALID_ROUNDS; + + rk = skey->rijndael.dK; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + LOAD32H(s0, ct ); s0 ^= rk[0]; + LOAD32H(s1, ct + 4); s1 ^= rk[1]; + LOAD32H(s2, ct + 8); s2 ^= rk[2]; + LOAD32H(s3, ct + 12); s3 ^= rk[3]; + +#ifdef LTC_SMALL_CODE + for (r = 0; ; r++) { + rk += 4; + t0 = + Td0(LTC_BYTE(s0, 3)) ^ + Td1(LTC_BYTE(s3, 2)) ^ + Td2(LTC_BYTE(s2, 1)) ^ + Td3(LTC_BYTE(s1, 0)) ^ + rk[0]; + t1 = + Td0(LTC_BYTE(s1, 3)) ^ + Td1(LTC_BYTE(s0, 2)) ^ + Td2(LTC_BYTE(s3, 1)) ^ + Td3(LTC_BYTE(s2, 0)) ^ + rk[1]; + t2 = + Td0(LTC_BYTE(s2, 3)) ^ + Td1(LTC_BYTE(s1, 2)) ^ + Td2(LTC_BYTE(s0, 1)) ^ + Td3(LTC_BYTE(s3, 0)) ^ + rk[2]; + t3 = + Td0(LTC_BYTE(s3, 3)) ^ + Td1(LTC_BYTE(s2, 2)) ^ + Td2(LTC_BYTE(s1, 1)) ^ + Td3(LTC_BYTE(s0, 0)) ^ + rk[3]; + if (r == Nr-2) { + break; + } + s0 = t0; s1 = t1; s2 = t2; s3 = t3; + } + rk += 4; + +#else + + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + + t0 = + Td0(LTC_BYTE(s0, 3)) ^ + Td1(LTC_BYTE(s3, 2)) ^ + Td2(LTC_BYTE(s2, 1)) ^ + Td3(LTC_BYTE(s1, 0)) ^ + rk[4]; + t1 = + Td0(LTC_BYTE(s1, 3)) ^ + Td1(LTC_BYTE(s0, 2)) ^ + Td2(LTC_BYTE(s3, 1)) ^ + Td3(LTC_BYTE(s2, 0)) ^ + rk[5]; + t2 = + Td0(LTC_BYTE(s2, 3)) ^ + Td1(LTC_BYTE(s1, 2)) ^ + Td2(LTC_BYTE(s0, 1)) ^ + Td3(LTC_BYTE(s3, 0)) ^ + rk[6]; + t3 = + Td0(LTC_BYTE(s3, 3)) ^ + Td1(LTC_BYTE(s2, 2)) ^ + Td2(LTC_BYTE(s1, 1)) ^ + Td3(LTC_BYTE(s0, 0)) ^ + rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + + s0 = + Td0(LTC_BYTE(t0, 3)) ^ + Td1(LTC_BYTE(t3, 2)) ^ + Td2(LTC_BYTE(t2, 1)) ^ + Td3(LTC_BYTE(t1, 0)) ^ + rk[0]; + s1 = + Td0(LTC_BYTE(t1, 3)) ^ + Td1(LTC_BYTE(t0, 2)) ^ + Td2(LTC_BYTE(t3, 1)) ^ + Td3(LTC_BYTE(t2, 0)) ^ + rk[1]; + s2 = + Td0(LTC_BYTE(t2, 3)) ^ + Td1(LTC_BYTE(t1, 2)) ^ + Td2(LTC_BYTE(t0, 1)) ^ + Td3(LTC_BYTE(t3, 0)) ^ + rk[2]; + s3 = + Td0(LTC_BYTE(t3, 3)) ^ + Td1(LTC_BYTE(t2, 2)) ^ + Td2(LTC_BYTE(t1, 1)) ^ + Td3(LTC_BYTE(t0, 0)) ^ + rk[3]; + } +#endif + + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = + (Td4[LTC_BYTE(t0, 3)] & 0xff000000) ^ + (Td4[LTC_BYTE(t3, 2)] & 0x00ff0000) ^ + (Td4[LTC_BYTE(t2, 1)] & 0x0000ff00) ^ + (Td4[LTC_BYTE(t1, 0)] & 0x000000ff) ^ + rk[0]; + STORE32H(s0, pt); + s1 = + (Td4[LTC_BYTE(t1, 3)] & 0xff000000) ^ + (Td4[LTC_BYTE(t0, 2)] & 0x00ff0000) ^ + (Td4[LTC_BYTE(t3, 1)] & 0x0000ff00) ^ + (Td4[LTC_BYTE(t2, 0)] & 0x000000ff) ^ + rk[1]; + STORE32H(s1, pt+4); + s2 = + (Td4[LTC_BYTE(t2, 3)] & 0xff000000) ^ + (Td4[LTC_BYTE(t1, 2)] & 0x00ff0000) ^ + (Td4[LTC_BYTE(t0, 1)] & 0x0000ff00) ^ + (Td4[LTC_BYTE(t3, 0)] & 0x000000ff) ^ + rk[2]; + STORE32H(s2, pt+8); + s3 = + (Td4[LTC_BYTE(t3, 3)] & 0xff000000) ^ + (Td4[LTC_BYTE(t2, 2)] & 0x00ff0000) ^ + (Td4[LTC_BYTE(t1, 1)] & 0x0000ff00) ^ + (Td4[LTC_BYTE(t0, 0)] & 0x000000ff) ^ + rk[3]; + STORE32H(s3, pt+12); + + return CRYPT_OK; +} + + +#ifdef LTC_CLEAN_STACK +int ECB_DEC(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_rijndael_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(unsigned long)*8 + sizeof(unsigned long*) + sizeof(int)*2); + return err; +} +#endif + +/** + Performs a self-test of the AES block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int ECB_TEST(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + int err; + static const struct { + int keylen; + unsigned char key[32], pt[16], ct[16]; + } tests[] = { + { 16, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }, + { 0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30, + 0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a } + }, { + 24, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }, + { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }, + { 0xdd, 0xa9, 0x7c, 0xa4, 0x86, 0x4c, 0xdf, 0xe0, + 0x6e, 0xaf, 0x70, 0xa0, 0xec, 0x0d, 0x71, 0x91 } + }, { + 32, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }, + { 0x8e, 0xa2, 0xb7, 0xca, 0x51, 0x67, 0x45, 0xbf, + 0xea, 0xfc, 0x49, 0x90, 0x4b, 0x49, 0x60, 0x89 } + } + }; + + symmetric_key key; + unsigned char tmp[2][16]; + int i, y; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + zeromem(&key, sizeof(key)); + if ((err = rijndael_setup(tests[i].key, tests[i].keylen, 0, &key)) != CRYPT_OK) { + return err; + } + + rijndael_ecb_encrypt(tests[i].pt, tmp[0], &key); + rijndael_ecb_decrypt(tmp[0], tmp[1], &key); + if (compare_testvector(tmp[0], 16, tests[i].ct, 16, "AES Encrypt", i) || + compare_testvector(tmp[1], 16, tests[i].pt, 16, "AES Decrypt", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 16; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) rijndael_ecb_encrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 1000; y++) rijndael_ecb_decrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 16; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; + #endif +} + +#endif /* ENCRYPT_ONLY */ + + +/** Terminate the context + @param skey The scheduled key +*/ +void ECB_DONE(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int ECB_KS(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize < 24) { + *keysize = 16; + return CRYPT_OK; + } + if (*keysize < 32) { + *keysize = 24; + return CRYPT_OK; + } + *keysize = 32; + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/aes/aes_tab.c b/optee_os/core/lib/libtomcrypt/src/ciphers/aes/aes_tab.c new file mode 100644 index 0000000..5e59004 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/aes/aes_tab.c @@ -0,0 +1,1022 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/* The precomputed tables for AES */ +/* +Te0[x] = S [x].[02, 01, 01, 03]; +Te1[x] = S [x].[03, 02, 01, 01]; +Te2[x] = S [x].[01, 03, 02, 01]; +Te3[x] = S [x].[01, 01, 03, 02]; +Te4[x] = S [x].[01, 01, 01, 01]; + +Td0[x] = Si[x].[0e, 09, 0d, 0b]; +Td1[x] = Si[x].[0b, 0e, 09, 0d]; +Td2[x] = Si[x].[0d, 0b, 0e, 09]; +Td3[x] = Si[x].[09, 0d, 0b, 0e]; +Td4[x] = Si[x].[01, 01, 01, 01]; +*/ + +#ifdef LTC_AES_TAB_C + +/** + @file aes_tab.c + AES tables +*/ +static const ulong32 TE0[256] = { + 0xc66363a5UL, 0xf87c7c84UL, 0xee777799UL, 0xf67b7b8dUL, + 0xfff2f20dUL, 0xd66b6bbdUL, 0xde6f6fb1UL, 0x91c5c554UL, + 0x60303050UL, 0x02010103UL, 0xce6767a9UL, 0x562b2b7dUL, + 0xe7fefe19UL, 0xb5d7d762UL, 0x4dababe6UL, 0xec76769aUL, + 0x8fcaca45UL, 0x1f82829dUL, 0x89c9c940UL, 0xfa7d7d87UL, + 0xeffafa15UL, 0xb25959ebUL, 0x8e4747c9UL, 0xfbf0f00bUL, + 0x41adadecUL, 0xb3d4d467UL, 0x5fa2a2fdUL, 0x45afafeaUL, + 0x239c9cbfUL, 0x53a4a4f7UL, 0xe4727296UL, 0x9bc0c05bUL, + 0x75b7b7c2UL, 0xe1fdfd1cUL, 0x3d9393aeUL, 0x4c26266aUL, + 0x6c36365aUL, 0x7e3f3f41UL, 0xf5f7f702UL, 0x83cccc4fUL, + 0x6834345cUL, 0x51a5a5f4UL, 0xd1e5e534UL, 0xf9f1f108UL, + 0xe2717193UL, 0xabd8d873UL, 0x62313153UL, 0x2a15153fUL, + 0x0804040cUL, 0x95c7c752UL, 0x46232365UL, 0x9dc3c35eUL, + 0x30181828UL, 0x379696a1UL, 0x0a05050fUL, 0x2f9a9ab5UL, + 0x0e070709UL, 0x24121236UL, 0x1b80809bUL, 0xdfe2e23dUL, + 0xcdebeb26UL, 0x4e272769UL, 0x7fb2b2cdUL, 0xea75759fUL, + 0x1209091bUL, 0x1d83839eUL, 0x582c2c74UL, 0x341a1a2eUL, + 0x361b1b2dUL, 0xdc6e6eb2UL, 0xb45a5aeeUL, 0x5ba0a0fbUL, + 0xa45252f6UL, 0x763b3b4dUL, 0xb7d6d661UL, 0x7db3b3ceUL, + 0x5229297bUL, 0xdde3e33eUL, 0x5e2f2f71UL, 0x13848497UL, + 0xa65353f5UL, 0xb9d1d168UL, 0x00000000UL, 0xc1eded2cUL, + 0x40202060UL, 0xe3fcfc1fUL, 0x79b1b1c8UL, 0xb65b5bedUL, + 0xd46a6abeUL, 0x8dcbcb46UL, 0x67bebed9UL, 0x7239394bUL, + 0x944a4adeUL, 0x984c4cd4UL, 0xb05858e8UL, 0x85cfcf4aUL, + 0xbbd0d06bUL, 0xc5efef2aUL, 0x4faaaae5UL, 0xedfbfb16UL, + 0x864343c5UL, 0x9a4d4dd7UL, 0x66333355UL, 0x11858594UL, + 0x8a4545cfUL, 0xe9f9f910UL, 0x04020206UL, 0xfe7f7f81UL, + 0xa05050f0UL, 0x783c3c44UL, 0x259f9fbaUL, 0x4ba8a8e3UL, + 0xa25151f3UL, 0x5da3a3feUL, 0x804040c0UL, 0x058f8f8aUL, + 0x3f9292adUL, 0x219d9dbcUL, 0x70383848UL, 0xf1f5f504UL, + 0x63bcbcdfUL, 0x77b6b6c1UL, 0xafdada75UL, 0x42212163UL, + 0x20101030UL, 0xe5ffff1aUL, 0xfdf3f30eUL, 0xbfd2d26dUL, + 0x81cdcd4cUL, 0x180c0c14UL, 0x26131335UL, 0xc3ecec2fUL, + 0xbe5f5fe1UL, 0x359797a2UL, 0x884444ccUL, 0x2e171739UL, + 0x93c4c457UL, 0x55a7a7f2UL, 0xfc7e7e82UL, 0x7a3d3d47UL, + 0xc86464acUL, 0xba5d5de7UL, 0x3219192bUL, 0xe6737395UL, + 0xc06060a0UL, 0x19818198UL, 0x9e4f4fd1UL, 0xa3dcdc7fUL, + 0x44222266UL, 0x542a2a7eUL, 0x3b9090abUL, 0x0b888883UL, + 0x8c4646caUL, 0xc7eeee29UL, 0x6bb8b8d3UL, 0x2814143cUL, + 0xa7dede79UL, 0xbc5e5ee2UL, 0x160b0b1dUL, 0xaddbdb76UL, + 0xdbe0e03bUL, 0x64323256UL, 0x743a3a4eUL, 0x140a0a1eUL, + 0x924949dbUL, 0x0c06060aUL, 0x4824246cUL, 0xb85c5ce4UL, + 0x9fc2c25dUL, 0xbdd3d36eUL, 0x43acacefUL, 0xc46262a6UL, + 0x399191a8UL, 0x319595a4UL, 0xd3e4e437UL, 0xf279798bUL, + 0xd5e7e732UL, 0x8bc8c843UL, 0x6e373759UL, 0xda6d6db7UL, + 0x018d8d8cUL, 0xb1d5d564UL, 0x9c4e4ed2UL, 0x49a9a9e0UL, + 0xd86c6cb4UL, 0xac5656faUL, 0xf3f4f407UL, 0xcfeaea25UL, + 0xca6565afUL, 0xf47a7a8eUL, 0x47aeaee9UL, 0x10080818UL, + 0x6fbabad5UL, 0xf0787888UL, 0x4a25256fUL, 0x5c2e2e72UL, + 0x381c1c24UL, 0x57a6a6f1UL, 0x73b4b4c7UL, 0x97c6c651UL, + 0xcbe8e823UL, 0xa1dddd7cUL, 0xe874749cUL, 0x3e1f1f21UL, + 0x964b4bddUL, 0x61bdbddcUL, 0x0d8b8b86UL, 0x0f8a8a85UL, + 0xe0707090UL, 0x7c3e3e42UL, 0x71b5b5c4UL, 0xcc6666aaUL, + 0x904848d8UL, 0x06030305UL, 0xf7f6f601UL, 0x1c0e0e12UL, + 0xc26161a3UL, 0x6a35355fUL, 0xae5757f9UL, 0x69b9b9d0UL, + 0x17868691UL, 0x99c1c158UL, 0x3a1d1d27UL, 0x279e9eb9UL, + 0xd9e1e138UL, 0xebf8f813UL, 0x2b9898b3UL, 0x22111133UL, + 0xd26969bbUL, 0xa9d9d970UL, 0x078e8e89UL, 0x339494a7UL, + 0x2d9b9bb6UL, 0x3c1e1e22UL, 0x15878792UL, 0xc9e9e920UL, + 0x87cece49UL, 0xaa5555ffUL, 0x50282878UL, 0xa5dfdf7aUL, + 0x038c8c8fUL, 0x59a1a1f8UL, 0x09898980UL, 0x1a0d0d17UL, + 0x65bfbfdaUL, 0xd7e6e631UL, 0x844242c6UL, 0xd06868b8UL, + 0x824141c3UL, 0x299999b0UL, 0x5a2d2d77UL, 0x1e0f0f11UL, + 0x7bb0b0cbUL, 0xa85454fcUL, 0x6dbbbbd6UL, 0x2c16163aUL, +}; + +#if !defined(PELI_TAB) && defined(LTC_SMALL_CODE) +static const ulong32 Te4[256] = { + 0x63636363UL, 0x7c7c7c7cUL, 0x77777777UL, 0x7b7b7b7bUL, + 0xf2f2f2f2UL, 0x6b6b6b6bUL, 0x6f6f6f6fUL, 0xc5c5c5c5UL, + 0x30303030UL, 0x01010101UL, 0x67676767UL, 0x2b2b2b2bUL, + 0xfefefefeUL, 0xd7d7d7d7UL, 0xababababUL, 0x76767676UL, + 0xcacacacaUL, 0x82828282UL, 0xc9c9c9c9UL, 0x7d7d7d7dUL, + 0xfafafafaUL, 0x59595959UL, 0x47474747UL, 0xf0f0f0f0UL, + 0xadadadadUL, 0xd4d4d4d4UL, 0xa2a2a2a2UL, 0xafafafafUL, + 0x9c9c9c9cUL, 0xa4a4a4a4UL, 0x72727272UL, 0xc0c0c0c0UL, + 0xb7b7b7b7UL, 0xfdfdfdfdUL, 0x93939393UL, 0x26262626UL, + 0x36363636UL, 0x3f3f3f3fUL, 0xf7f7f7f7UL, 0xccccccccUL, + 0x34343434UL, 0xa5a5a5a5UL, 0xe5e5e5e5UL, 0xf1f1f1f1UL, + 0x71717171UL, 0xd8d8d8d8UL, 0x31313131UL, 0x15151515UL, + 0x04040404UL, 0xc7c7c7c7UL, 0x23232323UL, 0xc3c3c3c3UL, + 0x18181818UL, 0x96969696UL, 0x05050505UL, 0x9a9a9a9aUL, + 0x07070707UL, 0x12121212UL, 0x80808080UL, 0xe2e2e2e2UL, + 0xebebebebUL, 0x27272727UL, 0xb2b2b2b2UL, 0x75757575UL, + 0x09090909UL, 0x83838383UL, 0x2c2c2c2cUL, 0x1a1a1a1aUL, + 0x1b1b1b1bUL, 0x6e6e6e6eUL, 0x5a5a5a5aUL, 0xa0a0a0a0UL, + 0x52525252UL, 0x3b3b3b3bUL, 0xd6d6d6d6UL, 0xb3b3b3b3UL, + 0x29292929UL, 0xe3e3e3e3UL, 0x2f2f2f2fUL, 0x84848484UL, + 0x53535353UL, 0xd1d1d1d1UL, 0x00000000UL, 0xededededUL, + 0x20202020UL, 0xfcfcfcfcUL, 0xb1b1b1b1UL, 0x5b5b5b5bUL, + 0x6a6a6a6aUL, 0xcbcbcbcbUL, 0xbebebebeUL, 0x39393939UL, + 0x4a4a4a4aUL, 0x4c4c4c4cUL, 0x58585858UL, 0xcfcfcfcfUL, + 0xd0d0d0d0UL, 0xefefefefUL, 0xaaaaaaaaUL, 0xfbfbfbfbUL, + 0x43434343UL, 0x4d4d4d4dUL, 0x33333333UL, 0x85858585UL, + 0x45454545UL, 0xf9f9f9f9UL, 0x02020202UL, 0x7f7f7f7fUL, + 0x50505050UL, 0x3c3c3c3cUL, 0x9f9f9f9fUL, 0xa8a8a8a8UL, + 0x51515151UL, 0xa3a3a3a3UL, 0x40404040UL, 0x8f8f8f8fUL, + 0x92929292UL, 0x9d9d9d9dUL, 0x38383838UL, 0xf5f5f5f5UL, + 0xbcbcbcbcUL, 0xb6b6b6b6UL, 0xdadadadaUL, 0x21212121UL, + 0x10101010UL, 0xffffffffUL, 0xf3f3f3f3UL, 0xd2d2d2d2UL, + 0xcdcdcdcdUL, 0x0c0c0c0cUL, 0x13131313UL, 0xececececUL, + 0x5f5f5f5fUL, 0x97979797UL, 0x44444444UL, 0x17171717UL, + 0xc4c4c4c4UL, 0xa7a7a7a7UL, 0x7e7e7e7eUL, 0x3d3d3d3dUL, + 0x64646464UL, 0x5d5d5d5dUL, 0x19191919UL, 0x73737373UL, + 0x60606060UL, 0x81818181UL, 0x4f4f4f4fUL, 0xdcdcdcdcUL, + 0x22222222UL, 0x2a2a2a2aUL, 0x90909090UL, 0x88888888UL, + 0x46464646UL, 0xeeeeeeeeUL, 0xb8b8b8b8UL, 0x14141414UL, + 0xdedededeUL, 0x5e5e5e5eUL, 0x0b0b0b0bUL, 0xdbdbdbdbUL, + 0xe0e0e0e0UL, 0x32323232UL, 0x3a3a3a3aUL, 0x0a0a0a0aUL, + 0x49494949UL, 0x06060606UL, 0x24242424UL, 0x5c5c5c5cUL, + 0xc2c2c2c2UL, 0xd3d3d3d3UL, 0xacacacacUL, 0x62626262UL, + 0x91919191UL, 0x95959595UL, 0xe4e4e4e4UL, 0x79797979UL, + 0xe7e7e7e7UL, 0xc8c8c8c8UL, 0x37373737UL, 0x6d6d6d6dUL, + 0x8d8d8d8dUL, 0xd5d5d5d5UL, 0x4e4e4e4eUL, 0xa9a9a9a9UL, + 0x6c6c6c6cUL, 0x56565656UL, 0xf4f4f4f4UL, 0xeaeaeaeaUL, + 0x65656565UL, 0x7a7a7a7aUL, 0xaeaeaeaeUL, 0x08080808UL, + 0xbabababaUL, 0x78787878UL, 0x25252525UL, 0x2e2e2e2eUL, + 0x1c1c1c1cUL, 0xa6a6a6a6UL, 0xb4b4b4b4UL, 0xc6c6c6c6UL, + 0xe8e8e8e8UL, 0xddddddddUL, 0x74747474UL, 0x1f1f1f1fUL, + 0x4b4b4b4bUL, 0xbdbdbdbdUL, 0x8b8b8b8bUL, 0x8a8a8a8aUL, + 0x70707070UL, 0x3e3e3e3eUL, 0xb5b5b5b5UL, 0x66666666UL, + 0x48484848UL, 0x03030303UL, 0xf6f6f6f6UL, 0x0e0e0e0eUL, + 0x61616161UL, 0x35353535UL, 0x57575757UL, 0xb9b9b9b9UL, + 0x86868686UL, 0xc1c1c1c1UL, 0x1d1d1d1dUL, 0x9e9e9e9eUL, + 0xe1e1e1e1UL, 0xf8f8f8f8UL, 0x98989898UL, 0x11111111UL, + 0x69696969UL, 0xd9d9d9d9UL, 0x8e8e8e8eUL, 0x94949494UL, + 0x9b9b9b9bUL, 0x1e1e1e1eUL, 0x87878787UL, 0xe9e9e9e9UL, + 0xcecececeUL, 0x55555555UL, 0x28282828UL, 0xdfdfdfdfUL, + 0x8c8c8c8cUL, 0xa1a1a1a1UL, 0x89898989UL, 0x0d0d0d0dUL, + 0xbfbfbfbfUL, 0xe6e6e6e6UL, 0x42424242UL, 0x68686868UL, + 0x41414141UL, 0x99999999UL, 0x2d2d2d2dUL, 0x0f0f0f0fUL, + 0xb0b0b0b0UL, 0x54545454UL, 0xbbbbbbbbUL, 0x16161616UL, +}; +#endif + +#ifndef ENCRYPT_ONLY + +static const ulong32 TD0[256] = { + 0x51f4a750UL, 0x7e416553UL, 0x1a17a4c3UL, 0x3a275e96UL, + 0x3bab6bcbUL, 0x1f9d45f1UL, 0xacfa58abUL, 0x4be30393UL, + 0x2030fa55UL, 0xad766df6UL, 0x88cc7691UL, 0xf5024c25UL, + 0x4fe5d7fcUL, 0xc52acbd7UL, 0x26354480UL, 0xb562a38fUL, + 0xdeb15a49UL, 0x25ba1b67UL, 0x45ea0e98UL, 0x5dfec0e1UL, + 0xc32f7502UL, 0x814cf012UL, 0x8d4697a3UL, 0x6bd3f9c6UL, + 0x038f5fe7UL, 0x15929c95UL, 0xbf6d7aebUL, 0x955259daUL, + 0xd4be832dUL, 0x587421d3UL, 0x49e06929UL, 0x8ec9c844UL, + 0x75c2896aUL, 0xf48e7978UL, 0x99583e6bUL, 0x27b971ddUL, + 0xbee14fb6UL, 0xf088ad17UL, 0xc920ac66UL, 0x7dce3ab4UL, + 0x63df4a18UL, 0xe51a3182UL, 0x97513360UL, 0x62537f45UL, + 0xb16477e0UL, 0xbb6bae84UL, 0xfe81a01cUL, 0xf9082b94UL, + 0x70486858UL, 0x8f45fd19UL, 0x94de6c87UL, 0x527bf8b7UL, + 0xab73d323UL, 0x724b02e2UL, 0xe31f8f57UL, 0x6655ab2aUL, + 0xb2eb2807UL, 0x2fb5c203UL, 0x86c57b9aUL, 0xd33708a5UL, + 0x302887f2UL, 0x23bfa5b2UL, 0x02036abaUL, 0xed16825cUL, + 0x8acf1c2bUL, 0xa779b492UL, 0xf307f2f0UL, 0x4e69e2a1UL, + 0x65daf4cdUL, 0x0605bed5UL, 0xd134621fUL, 0xc4a6fe8aUL, + 0x342e539dUL, 0xa2f355a0UL, 0x058ae132UL, 0xa4f6eb75UL, + 0x0b83ec39UL, 0x4060efaaUL, 0x5e719f06UL, 0xbd6e1051UL, + 0x3e218af9UL, 0x96dd063dUL, 0xdd3e05aeUL, 0x4de6bd46UL, + 0x91548db5UL, 0x71c45d05UL, 0x0406d46fUL, 0x605015ffUL, + 0x1998fb24UL, 0xd6bde997UL, 0x894043ccUL, 0x67d99e77UL, + 0xb0e842bdUL, 0x07898b88UL, 0xe7195b38UL, 0x79c8eedbUL, + 0xa17c0a47UL, 0x7c420fe9UL, 0xf8841ec9UL, 0x00000000UL, + 0x09808683UL, 0x322bed48UL, 0x1e1170acUL, 0x6c5a724eUL, + 0xfd0efffbUL, 0x0f853856UL, 0x3daed51eUL, 0x362d3927UL, + 0x0a0fd964UL, 0x685ca621UL, 0x9b5b54d1UL, 0x24362e3aUL, + 0x0c0a67b1UL, 0x9357e70fUL, 0xb4ee96d2UL, 0x1b9b919eUL, + 0x80c0c54fUL, 0x61dc20a2UL, 0x5a774b69UL, 0x1c121a16UL, + 0xe293ba0aUL, 0xc0a02ae5UL, 0x3c22e043UL, 0x121b171dUL, + 0x0e090d0bUL, 0xf28bc7adUL, 0x2db6a8b9UL, 0x141ea9c8UL, + 0x57f11985UL, 0xaf75074cUL, 0xee99ddbbUL, 0xa37f60fdUL, + 0xf701269fUL, 0x5c72f5bcUL, 0x44663bc5UL, 0x5bfb7e34UL, + 0x8b432976UL, 0xcb23c6dcUL, 0xb6edfc68UL, 0xb8e4f163UL, + 0xd731dccaUL, 0x42638510UL, 0x13972240UL, 0x84c61120UL, + 0x854a247dUL, 0xd2bb3df8UL, 0xaef93211UL, 0xc729a16dUL, + 0x1d9e2f4bUL, 0xdcb230f3UL, 0x0d8652ecUL, 0x77c1e3d0UL, + 0x2bb3166cUL, 0xa970b999UL, 0x119448faUL, 0x47e96422UL, + 0xa8fc8cc4UL, 0xa0f03f1aUL, 0x567d2cd8UL, 0x223390efUL, + 0x87494ec7UL, 0xd938d1c1UL, 0x8ccaa2feUL, 0x98d40b36UL, + 0xa6f581cfUL, 0xa57ade28UL, 0xdab78e26UL, 0x3fadbfa4UL, + 0x2c3a9de4UL, 0x5078920dUL, 0x6a5fcc9bUL, 0x547e4662UL, + 0xf68d13c2UL, 0x90d8b8e8UL, 0x2e39f75eUL, 0x82c3aff5UL, + 0x9f5d80beUL, 0x69d0937cUL, 0x6fd52da9UL, 0xcf2512b3UL, + 0xc8ac993bUL, 0x10187da7UL, 0xe89c636eUL, 0xdb3bbb7bUL, + 0xcd267809UL, 0x6e5918f4UL, 0xec9ab701UL, 0x834f9aa8UL, + 0xe6956e65UL, 0xaaffe67eUL, 0x21bccf08UL, 0xef15e8e6UL, + 0xbae79bd9UL, 0x4a6f36ceUL, 0xea9f09d4UL, 0x29b07cd6UL, + 0x31a4b2afUL, 0x2a3f2331UL, 0xc6a59430UL, 0x35a266c0UL, + 0x744ebc37UL, 0xfc82caa6UL, 0xe090d0b0UL, 0x33a7d815UL, + 0xf104984aUL, 0x41ecdaf7UL, 0x7fcd500eUL, 0x1791f62fUL, + 0x764dd68dUL, 0x43efb04dUL, 0xccaa4d54UL, 0xe49604dfUL, + 0x9ed1b5e3UL, 0x4c6a881bUL, 0xc12c1fb8UL, 0x4665517fUL, + 0x9d5eea04UL, 0x018c355dUL, 0xfa877473UL, 0xfb0b412eUL, + 0xb3671d5aUL, 0x92dbd252UL, 0xe9105633UL, 0x6dd64713UL, + 0x9ad7618cUL, 0x37a10c7aUL, 0x59f8148eUL, 0xeb133c89UL, + 0xcea927eeUL, 0xb761c935UL, 0xe11ce5edUL, 0x7a47b13cUL, + 0x9cd2df59UL, 0x55f2733fUL, 0x1814ce79UL, 0x73c737bfUL, + 0x53f7cdeaUL, 0x5ffdaa5bUL, 0xdf3d6f14UL, 0x7844db86UL, + 0xcaaff381UL, 0xb968c43eUL, 0x3824342cUL, 0xc2a3405fUL, + 0x161dc372UL, 0xbce2250cUL, 0x283c498bUL, 0xff0d9541UL, + 0x39a80171UL, 0x080cb3deUL, 0xd8b4e49cUL, 0x6456c190UL, + 0x7bcb8461UL, 0xd532b670UL, 0x486c5c74UL, 0xd0b85742UL, +}; + +static const ulong32 Td4[256] = { + 0x52525252UL, 0x09090909UL, 0x6a6a6a6aUL, 0xd5d5d5d5UL, + 0x30303030UL, 0x36363636UL, 0xa5a5a5a5UL, 0x38383838UL, + 0xbfbfbfbfUL, 0x40404040UL, 0xa3a3a3a3UL, 0x9e9e9e9eUL, + 0x81818181UL, 0xf3f3f3f3UL, 0xd7d7d7d7UL, 0xfbfbfbfbUL, + 0x7c7c7c7cUL, 0xe3e3e3e3UL, 0x39393939UL, 0x82828282UL, + 0x9b9b9b9bUL, 0x2f2f2f2fUL, 0xffffffffUL, 0x87878787UL, + 0x34343434UL, 0x8e8e8e8eUL, 0x43434343UL, 0x44444444UL, + 0xc4c4c4c4UL, 0xdedededeUL, 0xe9e9e9e9UL, 0xcbcbcbcbUL, + 0x54545454UL, 0x7b7b7b7bUL, 0x94949494UL, 0x32323232UL, + 0xa6a6a6a6UL, 0xc2c2c2c2UL, 0x23232323UL, 0x3d3d3d3dUL, + 0xeeeeeeeeUL, 0x4c4c4c4cUL, 0x95959595UL, 0x0b0b0b0bUL, + 0x42424242UL, 0xfafafafaUL, 0xc3c3c3c3UL, 0x4e4e4e4eUL, + 0x08080808UL, 0x2e2e2e2eUL, 0xa1a1a1a1UL, 0x66666666UL, + 0x28282828UL, 0xd9d9d9d9UL, 0x24242424UL, 0xb2b2b2b2UL, + 0x76767676UL, 0x5b5b5b5bUL, 0xa2a2a2a2UL, 0x49494949UL, + 0x6d6d6d6dUL, 0x8b8b8b8bUL, 0xd1d1d1d1UL, 0x25252525UL, + 0x72727272UL, 0xf8f8f8f8UL, 0xf6f6f6f6UL, 0x64646464UL, + 0x86868686UL, 0x68686868UL, 0x98989898UL, 0x16161616UL, + 0xd4d4d4d4UL, 0xa4a4a4a4UL, 0x5c5c5c5cUL, 0xccccccccUL, + 0x5d5d5d5dUL, 0x65656565UL, 0xb6b6b6b6UL, 0x92929292UL, + 0x6c6c6c6cUL, 0x70707070UL, 0x48484848UL, 0x50505050UL, + 0xfdfdfdfdUL, 0xededededUL, 0xb9b9b9b9UL, 0xdadadadaUL, + 0x5e5e5e5eUL, 0x15151515UL, 0x46464646UL, 0x57575757UL, + 0xa7a7a7a7UL, 0x8d8d8d8dUL, 0x9d9d9d9dUL, 0x84848484UL, + 0x90909090UL, 0xd8d8d8d8UL, 0xababababUL, 0x00000000UL, + 0x8c8c8c8cUL, 0xbcbcbcbcUL, 0xd3d3d3d3UL, 0x0a0a0a0aUL, + 0xf7f7f7f7UL, 0xe4e4e4e4UL, 0x58585858UL, 0x05050505UL, + 0xb8b8b8b8UL, 0xb3b3b3b3UL, 0x45454545UL, 0x06060606UL, + 0xd0d0d0d0UL, 0x2c2c2c2cUL, 0x1e1e1e1eUL, 0x8f8f8f8fUL, + 0xcacacacaUL, 0x3f3f3f3fUL, 0x0f0f0f0fUL, 0x02020202UL, + 0xc1c1c1c1UL, 0xafafafafUL, 0xbdbdbdbdUL, 0x03030303UL, + 0x01010101UL, 0x13131313UL, 0x8a8a8a8aUL, 0x6b6b6b6bUL, + 0x3a3a3a3aUL, 0x91919191UL, 0x11111111UL, 0x41414141UL, + 0x4f4f4f4fUL, 0x67676767UL, 0xdcdcdcdcUL, 0xeaeaeaeaUL, + 0x97979797UL, 0xf2f2f2f2UL, 0xcfcfcfcfUL, 0xcecececeUL, + 0xf0f0f0f0UL, 0xb4b4b4b4UL, 0xe6e6e6e6UL, 0x73737373UL, + 0x96969696UL, 0xacacacacUL, 0x74747474UL, 0x22222222UL, + 0xe7e7e7e7UL, 0xadadadadUL, 0x35353535UL, 0x85858585UL, + 0xe2e2e2e2UL, 0xf9f9f9f9UL, 0x37373737UL, 0xe8e8e8e8UL, + 0x1c1c1c1cUL, 0x75757575UL, 0xdfdfdfdfUL, 0x6e6e6e6eUL, + 0x47474747UL, 0xf1f1f1f1UL, 0x1a1a1a1aUL, 0x71717171UL, + 0x1d1d1d1dUL, 0x29292929UL, 0xc5c5c5c5UL, 0x89898989UL, + 0x6f6f6f6fUL, 0xb7b7b7b7UL, 0x62626262UL, 0x0e0e0e0eUL, + 0xaaaaaaaaUL, 0x18181818UL, 0xbebebebeUL, 0x1b1b1b1bUL, + 0xfcfcfcfcUL, 0x56565656UL, 0x3e3e3e3eUL, 0x4b4b4b4bUL, + 0xc6c6c6c6UL, 0xd2d2d2d2UL, 0x79797979UL, 0x20202020UL, + 0x9a9a9a9aUL, 0xdbdbdbdbUL, 0xc0c0c0c0UL, 0xfefefefeUL, + 0x78787878UL, 0xcdcdcdcdUL, 0x5a5a5a5aUL, 0xf4f4f4f4UL, + 0x1f1f1f1fUL, 0xddddddddUL, 0xa8a8a8a8UL, 0x33333333UL, + 0x88888888UL, 0x07070707UL, 0xc7c7c7c7UL, 0x31313131UL, + 0xb1b1b1b1UL, 0x12121212UL, 0x10101010UL, 0x59595959UL, + 0x27272727UL, 0x80808080UL, 0xececececUL, 0x5f5f5f5fUL, + 0x60606060UL, 0x51515151UL, 0x7f7f7f7fUL, 0xa9a9a9a9UL, + 0x19191919UL, 0xb5b5b5b5UL, 0x4a4a4a4aUL, 0x0d0d0d0dUL, + 0x2d2d2d2dUL, 0xe5e5e5e5UL, 0x7a7a7a7aUL, 0x9f9f9f9fUL, + 0x93939393UL, 0xc9c9c9c9UL, 0x9c9c9c9cUL, 0xefefefefUL, + 0xa0a0a0a0UL, 0xe0e0e0e0UL, 0x3b3b3b3bUL, 0x4d4d4d4dUL, + 0xaeaeaeaeUL, 0x2a2a2a2aUL, 0xf5f5f5f5UL, 0xb0b0b0b0UL, + 0xc8c8c8c8UL, 0xebebebebUL, 0xbbbbbbbbUL, 0x3c3c3c3cUL, + 0x83838383UL, 0x53535353UL, 0x99999999UL, 0x61616161UL, + 0x17171717UL, 0x2b2b2b2bUL, 0x04040404UL, 0x7e7e7e7eUL, + 0xbabababaUL, 0x77777777UL, 0xd6d6d6d6UL, 0x26262626UL, + 0xe1e1e1e1UL, 0x69696969UL, 0x14141414UL, 0x63636363UL, + 0x55555555UL, 0x21212121UL, 0x0c0c0c0cUL, 0x7d7d7d7dUL, +}; + +#endif /* ENCRYPT_ONLY */ + +#ifdef LTC_SMALL_CODE + +#define Te0(x) TE0[x] +#define Te1(x) RORc(TE0[x], 8) +#define Te2(x) RORc(TE0[x], 16) +#define Te3(x) RORc(TE0[x], 24) + +#define Td0(x) TD0[x] +#define Td1(x) RORc(TD0[x], 8) +#define Td2(x) RORc(TD0[x], 16) +#define Td3(x) RORc(TD0[x], 24) + +#define Te4_0 0x000000FF & Te4 +#define Te4_1 0x0000FF00 & Te4 +#define Te4_2 0x00FF0000 & Te4 +#define Te4_3 0xFF000000 & Te4 + +#else + +#define Te0(x) TE0[x] +#define Te1(x) TE1[x] +#define Te2(x) TE2[x] +#define Te3(x) TE3[x] + +#define Td0(x) TD0[x] +#define Td1(x) TD1[x] +#define Td2(x) TD2[x] +#define Td3(x) TD3[x] + +static const ulong32 TE1[256] = { + 0xa5c66363UL, 0x84f87c7cUL, 0x99ee7777UL, 0x8df67b7bUL, + 0x0dfff2f2UL, 0xbdd66b6bUL, 0xb1de6f6fUL, 0x5491c5c5UL, + 0x50603030UL, 0x03020101UL, 0xa9ce6767UL, 0x7d562b2bUL, + 0x19e7fefeUL, 0x62b5d7d7UL, 0xe64dababUL, 0x9aec7676UL, + 0x458fcacaUL, 0x9d1f8282UL, 0x4089c9c9UL, 0x87fa7d7dUL, + 0x15effafaUL, 0xebb25959UL, 0xc98e4747UL, 0x0bfbf0f0UL, + 0xec41adadUL, 0x67b3d4d4UL, 0xfd5fa2a2UL, 0xea45afafUL, + 0xbf239c9cUL, 0xf753a4a4UL, 0x96e47272UL, 0x5b9bc0c0UL, + 0xc275b7b7UL, 0x1ce1fdfdUL, 0xae3d9393UL, 0x6a4c2626UL, + 0x5a6c3636UL, 0x417e3f3fUL, 0x02f5f7f7UL, 0x4f83ccccUL, + 0x5c683434UL, 0xf451a5a5UL, 0x34d1e5e5UL, 0x08f9f1f1UL, + 0x93e27171UL, 0x73abd8d8UL, 0x53623131UL, 0x3f2a1515UL, + 0x0c080404UL, 0x5295c7c7UL, 0x65462323UL, 0x5e9dc3c3UL, + 0x28301818UL, 0xa1379696UL, 0x0f0a0505UL, 0xb52f9a9aUL, + 0x090e0707UL, 0x36241212UL, 0x9b1b8080UL, 0x3ddfe2e2UL, + 0x26cdebebUL, 0x694e2727UL, 0xcd7fb2b2UL, 0x9fea7575UL, + 0x1b120909UL, 0x9e1d8383UL, 0x74582c2cUL, 0x2e341a1aUL, + 0x2d361b1bUL, 0xb2dc6e6eUL, 0xeeb45a5aUL, 0xfb5ba0a0UL, + 0xf6a45252UL, 0x4d763b3bUL, 0x61b7d6d6UL, 0xce7db3b3UL, + 0x7b522929UL, 0x3edde3e3UL, 0x715e2f2fUL, 0x97138484UL, + 0xf5a65353UL, 0x68b9d1d1UL, 0x00000000UL, 0x2cc1ededUL, + 0x60402020UL, 0x1fe3fcfcUL, 0xc879b1b1UL, 0xedb65b5bUL, + 0xbed46a6aUL, 0x468dcbcbUL, 0xd967bebeUL, 0x4b723939UL, + 0xde944a4aUL, 0xd4984c4cUL, 0xe8b05858UL, 0x4a85cfcfUL, + 0x6bbbd0d0UL, 0x2ac5efefUL, 0xe54faaaaUL, 0x16edfbfbUL, + 0xc5864343UL, 0xd79a4d4dUL, 0x55663333UL, 0x94118585UL, + 0xcf8a4545UL, 0x10e9f9f9UL, 0x06040202UL, 0x81fe7f7fUL, + 0xf0a05050UL, 0x44783c3cUL, 0xba259f9fUL, 0xe34ba8a8UL, + 0xf3a25151UL, 0xfe5da3a3UL, 0xc0804040UL, 0x8a058f8fUL, + 0xad3f9292UL, 0xbc219d9dUL, 0x48703838UL, 0x04f1f5f5UL, + 0xdf63bcbcUL, 0xc177b6b6UL, 0x75afdadaUL, 0x63422121UL, + 0x30201010UL, 0x1ae5ffffUL, 0x0efdf3f3UL, 0x6dbfd2d2UL, + 0x4c81cdcdUL, 0x14180c0cUL, 0x35261313UL, 0x2fc3ececUL, + 0xe1be5f5fUL, 0xa2359797UL, 0xcc884444UL, 0x392e1717UL, + 0x5793c4c4UL, 0xf255a7a7UL, 0x82fc7e7eUL, 0x477a3d3dUL, + 0xacc86464UL, 0xe7ba5d5dUL, 0x2b321919UL, 0x95e67373UL, + 0xa0c06060UL, 0x98198181UL, 0xd19e4f4fUL, 0x7fa3dcdcUL, + 0x66442222UL, 0x7e542a2aUL, 0xab3b9090UL, 0x830b8888UL, + 0xca8c4646UL, 0x29c7eeeeUL, 0xd36bb8b8UL, 0x3c281414UL, + 0x79a7dedeUL, 0xe2bc5e5eUL, 0x1d160b0bUL, 0x76addbdbUL, + 0x3bdbe0e0UL, 0x56643232UL, 0x4e743a3aUL, 0x1e140a0aUL, + 0xdb924949UL, 0x0a0c0606UL, 0x6c482424UL, 0xe4b85c5cUL, + 0x5d9fc2c2UL, 0x6ebdd3d3UL, 0xef43acacUL, 0xa6c46262UL, + 0xa8399191UL, 0xa4319595UL, 0x37d3e4e4UL, 0x8bf27979UL, + 0x32d5e7e7UL, 0x438bc8c8UL, 0x596e3737UL, 0xb7da6d6dUL, + 0x8c018d8dUL, 0x64b1d5d5UL, 0xd29c4e4eUL, 0xe049a9a9UL, + 0xb4d86c6cUL, 0xfaac5656UL, 0x07f3f4f4UL, 0x25cfeaeaUL, + 0xafca6565UL, 0x8ef47a7aUL, 0xe947aeaeUL, 0x18100808UL, + 0xd56fbabaUL, 0x88f07878UL, 0x6f4a2525UL, 0x725c2e2eUL, + 0x24381c1cUL, 0xf157a6a6UL, 0xc773b4b4UL, 0x5197c6c6UL, + 0x23cbe8e8UL, 0x7ca1ddddUL, 0x9ce87474UL, 0x213e1f1fUL, + 0xdd964b4bUL, 0xdc61bdbdUL, 0x860d8b8bUL, 0x850f8a8aUL, + 0x90e07070UL, 0x427c3e3eUL, 0xc471b5b5UL, 0xaacc6666UL, + 0xd8904848UL, 0x05060303UL, 0x01f7f6f6UL, 0x121c0e0eUL, + 0xa3c26161UL, 0x5f6a3535UL, 0xf9ae5757UL, 0xd069b9b9UL, + 0x91178686UL, 0x5899c1c1UL, 0x273a1d1dUL, 0xb9279e9eUL, + 0x38d9e1e1UL, 0x13ebf8f8UL, 0xb32b9898UL, 0x33221111UL, + 0xbbd26969UL, 0x70a9d9d9UL, 0x89078e8eUL, 0xa7339494UL, + 0xb62d9b9bUL, 0x223c1e1eUL, 0x92158787UL, 0x20c9e9e9UL, + 0x4987ceceUL, 0xffaa5555UL, 0x78502828UL, 0x7aa5dfdfUL, + 0x8f038c8cUL, 0xf859a1a1UL, 0x80098989UL, 0x171a0d0dUL, + 0xda65bfbfUL, 0x31d7e6e6UL, 0xc6844242UL, 0xb8d06868UL, + 0xc3824141UL, 0xb0299999UL, 0x775a2d2dUL, 0x111e0f0fUL, + 0xcb7bb0b0UL, 0xfca85454UL, 0xd66dbbbbUL, 0x3a2c1616UL, +}; +static const ulong32 TE2[256] = { + 0x63a5c663UL, 0x7c84f87cUL, 0x7799ee77UL, 0x7b8df67bUL, + 0xf20dfff2UL, 0x6bbdd66bUL, 0x6fb1de6fUL, 0xc55491c5UL, + 0x30506030UL, 0x01030201UL, 0x67a9ce67UL, 0x2b7d562bUL, + 0xfe19e7feUL, 0xd762b5d7UL, 0xabe64dabUL, 0x769aec76UL, + 0xca458fcaUL, 0x829d1f82UL, 0xc94089c9UL, 0x7d87fa7dUL, + 0xfa15effaUL, 0x59ebb259UL, 0x47c98e47UL, 0xf00bfbf0UL, + 0xadec41adUL, 0xd467b3d4UL, 0xa2fd5fa2UL, 0xafea45afUL, + 0x9cbf239cUL, 0xa4f753a4UL, 0x7296e472UL, 0xc05b9bc0UL, + 0xb7c275b7UL, 0xfd1ce1fdUL, 0x93ae3d93UL, 0x266a4c26UL, + 0x365a6c36UL, 0x3f417e3fUL, 0xf702f5f7UL, 0xcc4f83ccUL, + 0x345c6834UL, 0xa5f451a5UL, 0xe534d1e5UL, 0xf108f9f1UL, + 0x7193e271UL, 0xd873abd8UL, 0x31536231UL, 0x153f2a15UL, + 0x040c0804UL, 0xc75295c7UL, 0x23654623UL, 0xc35e9dc3UL, + 0x18283018UL, 0x96a13796UL, 0x050f0a05UL, 0x9ab52f9aUL, + 0x07090e07UL, 0x12362412UL, 0x809b1b80UL, 0xe23ddfe2UL, + 0xeb26cdebUL, 0x27694e27UL, 0xb2cd7fb2UL, 0x759fea75UL, + 0x091b1209UL, 0x839e1d83UL, 0x2c74582cUL, 0x1a2e341aUL, + 0x1b2d361bUL, 0x6eb2dc6eUL, 0x5aeeb45aUL, 0xa0fb5ba0UL, + 0x52f6a452UL, 0x3b4d763bUL, 0xd661b7d6UL, 0xb3ce7db3UL, + 0x297b5229UL, 0xe33edde3UL, 0x2f715e2fUL, 0x84971384UL, + 0x53f5a653UL, 0xd168b9d1UL, 0x00000000UL, 0xed2cc1edUL, + 0x20604020UL, 0xfc1fe3fcUL, 0xb1c879b1UL, 0x5bedb65bUL, + 0x6abed46aUL, 0xcb468dcbUL, 0xbed967beUL, 0x394b7239UL, + 0x4ade944aUL, 0x4cd4984cUL, 0x58e8b058UL, 0xcf4a85cfUL, + 0xd06bbbd0UL, 0xef2ac5efUL, 0xaae54faaUL, 0xfb16edfbUL, + 0x43c58643UL, 0x4dd79a4dUL, 0x33556633UL, 0x85941185UL, + 0x45cf8a45UL, 0xf910e9f9UL, 0x02060402UL, 0x7f81fe7fUL, + 0x50f0a050UL, 0x3c44783cUL, 0x9fba259fUL, 0xa8e34ba8UL, + 0x51f3a251UL, 0xa3fe5da3UL, 0x40c08040UL, 0x8f8a058fUL, + 0x92ad3f92UL, 0x9dbc219dUL, 0x38487038UL, 0xf504f1f5UL, + 0xbcdf63bcUL, 0xb6c177b6UL, 0xda75afdaUL, 0x21634221UL, + 0x10302010UL, 0xff1ae5ffUL, 0xf30efdf3UL, 0xd26dbfd2UL, + 0xcd4c81cdUL, 0x0c14180cUL, 0x13352613UL, 0xec2fc3ecUL, + 0x5fe1be5fUL, 0x97a23597UL, 0x44cc8844UL, 0x17392e17UL, + 0xc45793c4UL, 0xa7f255a7UL, 0x7e82fc7eUL, 0x3d477a3dUL, + 0x64acc864UL, 0x5de7ba5dUL, 0x192b3219UL, 0x7395e673UL, + 0x60a0c060UL, 0x81981981UL, 0x4fd19e4fUL, 0xdc7fa3dcUL, + 0x22664422UL, 0x2a7e542aUL, 0x90ab3b90UL, 0x88830b88UL, + 0x46ca8c46UL, 0xee29c7eeUL, 0xb8d36bb8UL, 0x143c2814UL, + 0xde79a7deUL, 0x5ee2bc5eUL, 0x0b1d160bUL, 0xdb76addbUL, + 0xe03bdbe0UL, 0x32566432UL, 0x3a4e743aUL, 0x0a1e140aUL, + 0x49db9249UL, 0x060a0c06UL, 0x246c4824UL, 0x5ce4b85cUL, + 0xc25d9fc2UL, 0xd36ebdd3UL, 0xacef43acUL, 0x62a6c462UL, + 0x91a83991UL, 0x95a43195UL, 0xe437d3e4UL, 0x798bf279UL, + 0xe732d5e7UL, 0xc8438bc8UL, 0x37596e37UL, 0x6db7da6dUL, + 0x8d8c018dUL, 0xd564b1d5UL, 0x4ed29c4eUL, 0xa9e049a9UL, + 0x6cb4d86cUL, 0x56faac56UL, 0xf407f3f4UL, 0xea25cfeaUL, + 0x65afca65UL, 0x7a8ef47aUL, 0xaee947aeUL, 0x08181008UL, + 0xbad56fbaUL, 0x7888f078UL, 0x256f4a25UL, 0x2e725c2eUL, + 0x1c24381cUL, 0xa6f157a6UL, 0xb4c773b4UL, 0xc65197c6UL, + 0xe823cbe8UL, 0xdd7ca1ddUL, 0x749ce874UL, 0x1f213e1fUL, + 0x4bdd964bUL, 0xbddc61bdUL, 0x8b860d8bUL, 0x8a850f8aUL, + 0x7090e070UL, 0x3e427c3eUL, 0xb5c471b5UL, 0x66aacc66UL, + 0x48d89048UL, 0x03050603UL, 0xf601f7f6UL, 0x0e121c0eUL, + 0x61a3c261UL, 0x355f6a35UL, 0x57f9ae57UL, 0xb9d069b9UL, + 0x86911786UL, 0xc15899c1UL, 0x1d273a1dUL, 0x9eb9279eUL, + 0xe138d9e1UL, 0xf813ebf8UL, 0x98b32b98UL, 0x11332211UL, + 0x69bbd269UL, 0xd970a9d9UL, 0x8e89078eUL, 0x94a73394UL, + 0x9bb62d9bUL, 0x1e223c1eUL, 0x87921587UL, 0xe920c9e9UL, + 0xce4987ceUL, 0x55ffaa55UL, 0x28785028UL, 0xdf7aa5dfUL, + 0x8c8f038cUL, 0xa1f859a1UL, 0x89800989UL, 0x0d171a0dUL, + 0xbfda65bfUL, 0xe631d7e6UL, 0x42c68442UL, 0x68b8d068UL, + 0x41c38241UL, 0x99b02999UL, 0x2d775a2dUL, 0x0f111e0fUL, + 0xb0cb7bb0UL, 0x54fca854UL, 0xbbd66dbbUL, 0x163a2c16UL, +}; +static const ulong32 TE3[256] = { + + 0x6363a5c6UL, 0x7c7c84f8UL, 0x777799eeUL, 0x7b7b8df6UL, + 0xf2f20dffUL, 0x6b6bbdd6UL, 0x6f6fb1deUL, 0xc5c55491UL, + 0x30305060UL, 0x01010302UL, 0x6767a9ceUL, 0x2b2b7d56UL, + 0xfefe19e7UL, 0xd7d762b5UL, 0xababe64dUL, 0x76769aecUL, + 0xcaca458fUL, 0x82829d1fUL, 0xc9c94089UL, 0x7d7d87faUL, + 0xfafa15efUL, 0x5959ebb2UL, 0x4747c98eUL, 0xf0f00bfbUL, + 0xadadec41UL, 0xd4d467b3UL, 0xa2a2fd5fUL, 0xafafea45UL, + 0x9c9cbf23UL, 0xa4a4f753UL, 0x727296e4UL, 0xc0c05b9bUL, + 0xb7b7c275UL, 0xfdfd1ce1UL, 0x9393ae3dUL, 0x26266a4cUL, + 0x36365a6cUL, 0x3f3f417eUL, 0xf7f702f5UL, 0xcccc4f83UL, + 0x34345c68UL, 0xa5a5f451UL, 0xe5e534d1UL, 0xf1f108f9UL, + 0x717193e2UL, 0xd8d873abUL, 0x31315362UL, 0x15153f2aUL, + 0x04040c08UL, 0xc7c75295UL, 0x23236546UL, 0xc3c35e9dUL, + 0x18182830UL, 0x9696a137UL, 0x05050f0aUL, 0x9a9ab52fUL, + 0x0707090eUL, 0x12123624UL, 0x80809b1bUL, 0xe2e23ddfUL, + 0xebeb26cdUL, 0x2727694eUL, 0xb2b2cd7fUL, 0x75759feaUL, + 0x09091b12UL, 0x83839e1dUL, 0x2c2c7458UL, 0x1a1a2e34UL, + 0x1b1b2d36UL, 0x6e6eb2dcUL, 0x5a5aeeb4UL, 0xa0a0fb5bUL, + 0x5252f6a4UL, 0x3b3b4d76UL, 0xd6d661b7UL, 0xb3b3ce7dUL, + 0x29297b52UL, 0xe3e33eddUL, 0x2f2f715eUL, 0x84849713UL, + 0x5353f5a6UL, 0xd1d168b9UL, 0x00000000UL, 0xeded2cc1UL, + 0x20206040UL, 0xfcfc1fe3UL, 0xb1b1c879UL, 0x5b5bedb6UL, + 0x6a6abed4UL, 0xcbcb468dUL, 0xbebed967UL, 0x39394b72UL, + 0x4a4ade94UL, 0x4c4cd498UL, 0x5858e8b0UL, 0xcfcf4a85UL, + 0xd0d06bbbUL, 0xefef2ac5UL, 0xaaaae54fUL, 0xfbfb16edUL, + 0x4343c586UL, 0x4d4dd79aUL, 0x33335566UL, 0x85859411UL, + 0x4545cf8aUL, 0xf9f910e9UL, 0x02020604UL, 0x7f7f81feUL, + 0x5050f0a0UL, 0x3c3c4478UL, 0x9f9fba25UL, 0xa8a8e34bUL, + 0x5151f3a2UL, 0xa3a3fe5dUL, 0x4040c080UL, 0x8f8f8a05UL, + 0x9292ad3fUL, 0x9d9dbc21UL, 0x38384870UL, 0xf5f504f1UL, + 0xbcbcdf63UL, 0xb6b6c177UL, 0xdada75afUL, 0x21216342UL, + 0x10103020UL, 0xffff1ae5UL, 0xf3f30efdUL, 0xd2d26dbfUL, + 0xcdcd4c81UL, 0x0c0c1418UL, 0x13133526UL, 0xecec2fc3UL, + 0x5f5fe1beUL, 0x9797a235UL, 0x4444cc88UL, 0x1717392eUL, + 0xc4c45793UL, 0xa7a7f255UL, 0x7e7e82fcUL, 0x3d3d477aUL, + 0x6464acc8UL, 0x5d5de7baUL, 0x19192b32UL, 0x737395e6UL, + 0x6060a0c0UL, 0x81819819UL, 0x4f4fd19eUL, 0xdcdc7fa3UL, + 0x22226644UL, 0x2a2a7e54UL, 0x9090ab3bUL, 0x8888830bUL, + 0x4646ca8cUL, 0xeeee29c7UL, 0xb8b8d36bUL, 0x14143c28UL, + 0xdede79a7UL, 0x5e5ee2bcUL, 0x0b0b1d16UL, 0xdbdb76adUL, + 0xe0e03bdbUL, 0x32325664UL, 0x3a3a4e74UL, 0x0a0a1e14UL, + 0x4949db92UL, 0x06060a0cUL, 0x24246c48UL, 0x5c5ce4b8UL, + 0xc2c25d9fUL, 0xd3d36ebdUL, 0xacacef43UL, 0x6262a6c4UL, + 0x9191a839UL, 0x9595a431UL, 0xe4e437d3UL, 0x79798bf2UL, + 0xe7e732d5UL, 0xc8c8438bUL, 0x3737596eUL, 0x6d6db7daUL, + 0x8d8d8c01UL, 0xd5d564b1UL, 0x4e4ed29cUL, 0xa9a9e049UL, + 0x6c6cb4d8UL, 0x5656faacUL, 0xf4f407f3UL, 0xeaea25cfUL, + 0x6565afcaUL, 0x7a7a8ef4UL, 0xaeaee947UL, 0x08081810UL, + 0xbabad56fUL, 0x787888f0UL, 0x25256f4aUL, 0x2e2e725cUL, + 0x1c1c2438UL, 0xa6a6f157UL, 0xb4b4c773UL, 0xc6c65197UL, + 0xe8e823cbUL, 0xdddd7ca1UL, 0x74749ce8UL, 0x1f1f213eUL, + 0x4b4bdd96UL, 0xbdbddc61UL, 0x8b8b860dUL, 0x8a8a850fUL, + 0x707090e0UL, 0x3e3e427cUL, 0xb5b5c471UL, 0x6666aaccUL, + 0x4848d890UL, 0x03030506UL, 0xf6f601f7UL, 0x0e0e121cUL, + 0x6161a3c2UL, 0x35355f6aUL, 0x5757f9aeUL, 0xb9b9d069UL, + 0x86869117UL, 0xc1c15899UL, 0x1d1d273aUL, 0x9e9eb927UL, + 0xe1e138d9UL, 0xf8f813ebUL, 0x9898b32bUL, 0x11113322UL, + 0x6969bbd2UL, 0xd9d970a9UL, 0x8e8e8907UL, 0x9494a733UL, + 0x9b9bb62dUL, 0x1e1e223cUL, 0x87879215UL, 0xe9e920c9UL, + 0xcece4987UL, 0x5555ffaaUL, 0x28287850UL, 0xdfdf7aa5UL, + 0x8c8c8f03UL, 0xa1a1f859UL, 0x89898009UL, 0x0d0d171aUL, + 0xbfbfda65UL, 0xe6e631d7UL, 0x4242c684UL, 0x6868b8d0UL, + 0x4141c382UL, 0x9999b029UL, 0x2d2d775aUL, 0x0f0f111eUL, + 0xb0b0cb7bUL, 0x5454fca8UL, 0xbbbbd66dUL, 0x16163a2cUL, +}; + +#ifndef PELI_TAB +static const ulong32 Te4_0[] = { +0x00000063UL, 0x0000007cUL, 0x00000077UL, 0x0000007bUL, 0x000000f2UL, 0x0000006bUL, 0x0000006fUL, 0x000000c5UL, +0x00000030UL, 0x00000001UL, 0x00000067UL, 0x0000002bUL, 0x000000feUL, 0x000000d7UL, 0x000000abUL, 0x00000076UL, +0x000000caUL, 0x00000082UL, 0x000000c9UL, 0x0000007dUL, 0x000000faUL, 0x00000059UL, 0x00000047UL, 0x000000f0UL, +0x000000adUL, 0x000000d4UL, 0x000000a2UL, 0x000000afUL, 0x0000009cUL, 0x000000a4UL, 0x00000072UL, 0x000000c0UL, +0x000000b7UL, 0x000000fdUL, 0x00000093UL, 0x00000026UL, 0x00000036UL, 0x0000003fUL, 0x000000f7UL, 0x000000ccUL, +0x00000034UL, 0x000000a5UL, 0x000000e5UL, 0x000000f1UL, 0x00000071UL, 0x000000d8UL, 0x00000031UL, 0x00000015UL, +0x00000004UL, 0x000000c7UL, 0x00000023UL, 0x000000c3UL, 0x00000018UL, 0x00000096UL, 0x00000005UL, 0x0000009aUL, +0x00000007UL, 0x00000012UL, 0x00000080UL, 0x000000e2UL, 0x000000ebUL, 0x00000027UL, 0x000000b2UL, 0x00000075UL, +0x00000009UL, 0x00000083UL, 0x0000002cUL, 0x0000001aUL, 0x0000001bUL, 0x0000006eUL, 0x0000005aUL, 0x000000a0UL, +0x00000052UL, 0x0000003bUL, 0x000000d6UL, 0x000000b3UL, 0x00000029UL, 0x000000e3UL, 0x0000002fUL, 0x00000084UL, +0x00000053UL, 0x000000d1UL, 0x00000000UL, 0x000000edUL, 0x00000020UL, 0x000000fcUL, 0x000000b1UL, 0x0000005bUL, +0x0000006aUL, 0x000000cbUL, 0x000000beUL, 0x00000039UL, 0x0000004aUL, 0x0000004cUL, 0x00000058UL, 0x000000cfUL, +0x000000d0UL, 0x000000efUL, 0x000000aaUL, 0x000000fbUL, 0x00000043UL, 0x0000004dUL, 0x00000033UL, 0x00000085UL, +0x00000045UL, 0x000000f9UL, 0x00000002UL, 0x0000007fUL, 0x00000050UL, 0x0000003cUL, 0x0000009fUL, 0x000000a8UL, +0x00000051UL, 0x000000a3UL, 0x00000040UL, 0x0000008fUL, 0x00000092UL, 0x0000009dUL, 0x00000038UL, 0x000000f5UL, +0x000000bcUL, 0x000000b6UL, 0x000000daUL, 0x00000021UL, 0x00000010UL, 0x000000ffUL, 0x000000f3UL, 0x000000d2UL, +0x000000cdUL, 0x0000000cUL, 0x00000013UL, 0x000000ecUL, 0x0000005fUL, 0x00000097UL, 0x00000044UL, 0x00000017UL, +0x000000c4UL, 0x000000a7UL, 0x0000007eUL, 0x0000003dUL, 0x00000064UL, 0x0000005dUL, 0x00000019UL, 0x00000073UL, +0x00000060UL, 0x00000081UL, 0x0000004fUL, 0x000000dcUL, 0x00000022UL, 0x0000002aUL, 0x00000090UL, 0x00000088UL, +0x00000046UL, 0x000000eeUL, 0x000000b8UL, 0x00000014UL, 0x000000deUL, 0x0000005eUL, 0x0000000bUL, 0x000000dbUL, +0x000000e0UL, 0x00000032UL, 0x0000003aUL, 0x0000000aUL, 0x00000049UL, 0x00000006UL, 0x00000024UL, 0x0000005cUL, +0x000000c2UL, 0x000000d3UL, 0x000000acUL, 0x00000062UL, 0x00000091UL, 0x00000095UL, 0x000000e4UL, 0x00000079UL, +0x000000e7UL, 0x000000c8UL, 0x00000037UL, 0x0000006dUL, 0x0000008dUL, 0x000000d5UL, 0x0000004eUL, 0x000000a9UL, +0x0000006cUL, 0x00000056UL, 0x000000f4UL, 0x000000eaUL, 0x00000065UL, 0x0000007aUL, 0x000000aeUL, 0x00000008UL, +0x000000baUL, 0x00000078UL, 0x00000025UL, 0x0000002eUL, 0x0000001cUL, 0x000000a6UL, 0x000000b4UL, 0x000000c6UL, +0x000000e8UL, 0x000000ddUL, 0x00000074UL, 0x0000001fUL, 0x0000004bUL, 0x000000bdUL, 0x0000008bUL, 0x0000008aUL, +0x00000070UL, 0x0000003eUL, 0x000000b5UL, 0x00000066UL, 0x00000048UL, 0x00000003UL, 0x000000f6UL, 0x0000000eUL, +0x00000061UL, 0x00000035UL, 0x00000057UL, 0x000000b9UL, 0x00000086UL, 0x000000c1UL, 0x0000001dUL, 0x0000009eUL, +0x000000e1UL, 0x000000f8UL, 0x00000098UL, 0x00000011UL, 0x00000069UL, 0x000000d9UL, 0x0000008eUL, 0x00000094UL, +0x0000009bUL, 0x0000001eUL, 0x00000087UL, 0x000000e9UL, 0x000000ceUL, 0x00000055UL, 0x00000028UL, 0x000000dfUL, +0x0000008cUL, 0x000000a1UL, 0x00000089UL, 0x0000000dUL, 0x000000bfUL, 0x000000e6UL, 0x00000042UL, 0x00000068UL, +0x00000041UL, 0x00000099UL, 0x0000002dUL, 0x0000000fUL, 0x000000b0UL, 0x00000054UL, 0x000000bbUL, 0x00000016UL +}; + +static const ulong32 Te4_1[] = { +0x00006300UL, 0x00007c00UL, 0x00007700UL, 0x00007b00UL, 0x0000f200UL, 0x00006b00UL, 0x00006f00UL, 0x0000c500UL, +0x00003000UL, 0x00000100UL, 0x00006700UL, 0x00002b00UL, 0x0000fe00UL, 0x0000d700UL, 0x0000ab00UL, 0x00007600UL, +0x0000ca00UL, 0x00008200UL, 0x0000c900UL, 0x00007d00UL, 0x0000fa00UL, 0x00005900UL, 0x00004700UL, 0x0000f000UL, +0x0000ad00UL, 0x0000d400UL, 0x0000a200UL, 0x0000af00UL, 0x00009c00UL, 0x0000a400UL, 0x00007200UL, 0x0000c000UL, +0x0000b700UL, 0x0000fd00UL, 0x00009300UL, 0x00002600UL, 0x00003600UL, 0x00003f00UL, 0x0000f700UL, 0x0000cc00UL, +0x00003400UL, 0x0000a500UL, 0x0000e500UL, 0x0000f100UL, 0x00007100UL, 0x0000d800UL, 0x00003100UL, 0x00001500UL, +0x00000400UL, 0x0000c700UL, 0x00002300UL, 0x0000c300UL, 0x00001800UL, 0x00009600UL, 0x00000500UL, 0x00009a00UL, +0x00000700UL, 0x00001200UL, 0x00008000UL, 0x0000e200UL, 0x0000eb00UL, 0x00002700UL, 0x0000b200UL, 0x00007500UL, +0x00000900UL, 0x00008300UL, 0x00002c00UL, 0x00001a00UL, 0x00001b00UL, 0x00006e00UL, 0x00005a00UL, 0x0000a000UL, +0x00005200UL, 0x00003b00UL, 0x0000d600UL, 0x0000b300UL, 0x00002900UL, 0x0000e300UL, 0x00002f00UL, 0x00008400UL, +0x00005300UL, 0x0000d100UL, 0x00000000UL, 0x0000ed00UL, 0x00002000UL, 0x0000fc00UL, 0x0000b100UL, 0x00005b00UL, +0x00006a00UL, 0x0000cb00UL, 0x0000be00UL, 0x00003900UL, 0x00004a00UL, 0x00004c00UL, 0x00005800UL, 0x0000cf00UL, +0x0000d000UL, 0x0000ef00UL, 0x0000aa00UL, 0x0000fb00UL, 0x00004300UL, 0x00004d00UL, 0x00003300UL, 0x00008500UL, +0x00004500UL, 0x0000f900UL, 0x00000200UL, 0x00007f00UL, 0x00005000UL, 0x00003c00UL, 0x00009f00UL, 0x0000a800UL, +0x00005100UL, 0x0000a300UL, 0x00004000UL, 0x00008f00UL, 0x00009200UL, 0x00009d00UL, 0x00003800UL, 0x0000f500UL, +0x0000bc00UL, 0x0000b600UL, 0x0000da00UL, 0x00002100UL, 0x00001000UL, 0x0000ff00UL, 0x0000f300UL, 0x0000d200UL, +0x0000cd00UL, 0x00000c00UL, 0x00001300UL, 0x0000ec00UL, 0x00005f00UL, 0x00009700UL, 0x00004400UL, 0x00001700UL, +0x0000c400UL, 0x0000a700UL, 0x00007e00UL, 0x00003d00UL, 0x00006400UL, 0x00005d00UL, 0x00001900UL, 0x00007300UL, +0x00006000UL, 0x00008100UL, 0x00004f00UL, 0x0000dc00UL, 0x00002200UL, 0x00002a00UL, 0x00009000UL, 0x00008800UL, +0x00004600UL, 0x0000ee00UL, 0x0000b800UL, 0x00001400UL, 0x0000de00UL, 0x00005e00UL, 0x00000b00UL, 0x0000db00UL, +0x0000e000UL, 0x00003200UL, 0x00003a00UL, 0x00000a00UL, 0x00004900UL, 0x00000600UL, 0x00002400UL, 0x00005c00UL, +0x0000c200UL, 0x0000d300UL, 0x0000ac00UL, 0x00006200UL, 0x00009100UL, 0x00009500UL, 0x0000e400UL, 0x00007900UL, +0x0000e700UL, 0x0000c800UL, 0x00003700UL, 0x00006d00UL, 0x00008d00UL, 0x0000d500UL, 0x00004e00UL, 0x0000a900UL, +0x00006c00UL, 0x00005600UL, 0x0000f400UL, 0x0000ea00UL, 0x00006500UL, 0x00007a00UL, 0x0000ae00UL, 0x00000800UL, +0x0000ba00UL, 0x00007800UL, 0x00002500UL, 0x00002e00UL, 0x00001c00UL, 0x0000a600UL, 0x0000b400UL, 0x0000c600UL, +0x0000e800UL, 0x0000dd00UL, 0x00007400UL, 0x00001f00UL, 0x00004b00UL, 0x0000bd00UL, 0x00008b00UL, 0x00008a00UL, +0x00007000UL, 0x00003e00UL, 0x0000b500UL, 0x00006600UL, 0x00004800UL, 0x00000300UL, 0x0000f600UL, 0x00000e00UL, +0x00006100UL, 0x00003500UL, 0x00005700UL, 0x0000b900UL, 0x00008600UL, 0x0000c100UL, 0x00001d00UL, 0x00009e00UL, +0x0000e100UL, 0x0000f800UL, 0x00009800UL, 0x00001100UL, 0x00006900UL, 0x0000d900UL, 0x00008e00UL, 0x00009400UL, +0x00009b00UL, 0x00001e00UL, 0x00008700UL, 0x0000e900UL, 0x0000ce00UL, 0x00005500UL, 0x00002800UL, 0x0000df00UL, +0x00008c00UL, 0x0000a100UL, 0x00008900UL, 0x00000d00UL, 0x0000bf00UL, 0x0000e600UL, 0x00004200UL, 0x00006800UL, +0x00004100UL, 0x00009900UL, 0x00002d00UL, 0x00000f00UL, 0x0000b000UL, 0x00005400UL, 0x0000bb00UL, 0x00001600UL +}; + +static const ulong32 Te4_2[] = { +0x00630000UL, 0x007c0000UL, 0x00770000UL, 0x007b0000UL, 0x00f20000UL, 0x006b0000UL, 0x006f0000UL, 0x00c50000UL, +0x00300000UL, 0x00010000UL, 0x00670000UL, 0x002b0000UL, 0x00fe0000UL, 0x00d70000UL, 0x00ab0000UL, 0x00760000UL, +0x00ca0000UL, 0x00820000UL, 0x00c90000UL, 0x007d0000UL, 0x00fa0000UL, 0x00590000UL, 0x00470000UL, 0x00f00000UL, +0x00ad0000UL, 0x00d40000UL, 0x00a20000UL, 0x00af0000UL, 0x009c0000UL, 0x00a40000UL, 0x00720000UL, 0x00c00000UL, +0x00b70000UL, 0x00fd0000UL, 0x00930000UL, 0x00260000UL, 0x00360000UL, 0x003f0000UL, 0x00f70000UL, 0x00cc0000UL, +0x00340000UL, 0x00a50000UL, 0x00e50000UL, 0x00f10000UL, 0x00710000UL, 0x00d80000UL, 0x00310000UL, 0x00150000UL, +0x00040000UL, 0x00c70000UL, 0x00230000UL, 0x00c30000UL, 0x00180000UL, 0x00960000UL, 0x00050000UL, 0x009a0000UL, +0x00070000UL, 0x00120000UL, 0x00800000UL, 0x00e20000UL, 0x00eb0000UL, 0x00270000UL, 0x00b20000UL, 0x00750000UL, +0x00090000UL, 0x00830000UL, 0x002c0000UL, 0x001a0000UL, 0x001b0000UL, 0x006e0000UL, 0x005a0000UL, 0x00a00000UL, +0x00520000UL, 0x003b0000UL, 0x00d60000UL, 0x00b30000UL, 0x00290000UL, 0x00e30000UL, 0x002f0000UL, 0x00840000UL, +0x00530000UL, 0x00d10000UL, 0x00000000UL, 0x00ed0000UL, 0x00200000UL, 0x00fc0000UL, 0x00b10000UL, 0x005b0000UL, +0x006a0000UL, 0x00cb0000UL, 0x00be0000UL, 0x00390000UL, 0x004a0000UL, 0x004c0000UL, 0x00580000UL, 0x00cf0000UL, +0x00d00000UL, 0x00ef0000UL, 0x00aa0000UL, 0x00fb0000UL, 0x00430000UL, 0x004d0000UL, 0x00330000UL, 0x00850000UL, +0x00450000UL, 0x00f90000UL, 0x00020000UL, 0x007f0000UL, 0x00500000UL, 0x003c0000UL, 0x009f0000UL, 0x00a80000UL, +0x00510000UL, 0x00a30000UL, 0x00400000UL, 0x008f0000UL, 0x00920000UL, 0x009d0000UL, 0x00380000UL, 0x00f50000UL, +0x00bc0000UL, 0x00b60000UL, 0x00da0000UL, 0x00210000UL, 0x00100000UL, 0x00ff0000UL, 0x00f30000UL, 0x00d20000UL, +0x00cd0000UL, 0x000c0000UL, 0x00130000UL, 0x00ec0000UL, 0x005f0000UL, 0x00970000UL, 0x00440000UL, 0x00170000UL, +0x00c40000UL, 0x00a70000UL, 0x007e0000UL, 0x003d0000UL, 0x00640000UL, 0x005d0000UL, 0x00190000UL, 0x00730000UL, +0x00600000UL, 0x00810000UL, 0x004f0000UL, 0x00dc0000UL, 0x00220000UL, 0x002a0000UL, 0x00900000UL, 0x00880000UL, +0x00460000UL, 0x00ee0000UL, 0x00b80000UL, 0x00140000UL, 0x00de0000UL, 0x005e0000UL, 0x000b0000UL, 0x00db0000UL, +0x00e00000UL, 0x00320000UL, 0x003a0000UL, 0x000a0000UL, 0x00490000UL, 0x00060000UL, 0x00240000UL, 0x005c0000UL, +0x00c20000UL, 0x00d30000UL, 0x00ac0000UL, 0x00620000UL, 0x00910000UL, 0x00950000UL, 0x00e40000UL, 0x00790000UL, +0x00e70000UL, 0x00c80000UL, 0x00370000UL, 0x006d0000UL, 0x008d0000UL, 0x00d50000UL, 0x004e0000UL, 0x00a90000UL, +0x006c0000UL, 0x00560000UL, 0x00f40000UL, 0x00ea0000UL, 0x00650000UL, 0x007a0000UL, 0x00ae0000UL, 0x00080000UL, +0x00ba0000UL, 0x00780000UL, 0x00250000UL, 0x002e0000UL, 0x001c0000UL, 0x00a60000UL, 0x00b40000UL, 0x00c60000UL, +0x00e80000UL, 0x00dd0000UL, 0x00740000UL, 0x001f0000UL, 0x004b0000UL, 0x00bd0000UL, 0x008b0000UL, 0x008a0000UL, +0x00700000UL, 0x003e0000UL, 0x00b50000UL, 0x00660000UL, 0x00480000UL, 0x00030000UL, 0x00f60000UL, 0x000e0000UL, +0x00610000UL, 0x00350000UL, 0x00570000UL, 0x00b90000UL, 0x00860000UL, 0x00c10000UL, 0x001d0000UL, 0x009e0000UL, +0x00e10000UL, 0x00f80000UL, 0x00980000UL, 0x00110000UL, 0x00690000UL, 0x00d90000UL, 0x008e0000UL, 0x00940000UL, +0x009b0000UL, 0x001e0000UL, 0x00870000UL, 0x00e90000UL, 0x00ce0000UL, 0x00550000UL, 0x00280000UL, 0x00df0000UL, +0x008c0000UL, 0x00a10000UL, 0x00890000UL, 0x000d0000UL, 0x00bf0000UL, 0x00e60000UL, 0x00420000UL, 0x00680000UL, +0x00410000UL, 0x00990000UL, 0x002d0000UL, 0x000f0000UL, 0x00b00000UL, 0x00540000UL, 0x00bb0000UL, 0x00160000UL +}; + +static const ulong32 Te4_3[] = { +0x63000000UL, 0x7c000000UL, 0x77000000UL, 0x7b000000UL, 0xf2000000UL, 0x6b000000UL, 0x6f000000UL, 0xc5000000UL, +0x30000000UL, 0x01000000UL, 0x67000000UL, 0x2b000000UL, 0xfe000000UL, 0xd7000000UL, 0xab000000UL, 0x76000000UL, +0xca000000UL, 0x82000000UL, 0xc9000000UL, 0x7d000000UL, 0xfa000000UL, 0x59000000UL, 0x47000000UL, 0xf0000000UL, +0xad000000UL, 0xd4000000UL, 0xa2000000UL, 0xaf000000UL, 0x9c000000UL, 0xa4000000UL, 0x72000000UL, 0xc0000000UL, +0xb7000000UL, 0xfd000000UL, 0x93000000UL, 0x26000000UL, 0x36000000UL, 0x3f000000UL, 0xf7000000UL, 0xcc000000UL, +0x34000000UL, 0xa5000000UL, 0xe5000000UL, 0xf1000000UL, 0x71000000UL, 0xd8000000UL, 0x31000000UL, 0x15000000UL, +0x04000000UL, 0xc7000000UL, 0x23000000UL, 0xc3000000UL, 0x18000000UL, 0x96000000UL, 0x05000000UL, 0x9a000000UL, +0x07000000UL, 0x12000000UL, 0x80000000UL, 0xe2000000UL, 0xeb000000UL, 0x27000000UL, 0xb2000000UL, 0x75000000UL, +0x09000000UL, 0x83000000UL, 0x2c000000UL, 0x1a000000UL, 0x1b000000UL, 0x6e000000UL, 0x5a000000UL, 0xa0000000UL, +0x52000000UL, 0x3b000000UL, 0xd6000000UL, 0xb3000000UL, 0x29000000UL, 0xe3000000UL, 0x2f000000UL, 0x84000000UL, +0x53000000UL, 0xd1000000UL, 0x00000000UL, 0xed000000UL, 0x20000000UL, 0xfc000000UL, 0xb1000000UL, 0x5b000000UL, +0x6a000000UL, 0xcb000000UL, 0xbe000000UL, 0x39000000UL, 0x4a000000UL, 0x4c000000UL, 0x58000000UL, 0xcf000000UL, +0xd0000000UL, 0xef000000UL, 0xaa000000UL, 0xfb000000UL, 0x43000000UL, 0x4d000000UL, 0x33000000UL, 0x85000000UL, +0x45000000UL, 0xf9000000UL, 0x02000000UL, 0x7f000000UL, 0x50000000UL, 0x3c000000UL, 0x9f000000UL, 0xa8000000UL, +0x51000000UL, 0xa3000000UL, 0x40000000UL, 0x8f000000UL, 0x92000000UL, 0x9d000000UL, 0x38000000UL, 0xf5000000UL, +0xbc000000UL, 0xb6000000UL, 0xda000000UL, 0x21000000UL, 0x10000000UL, 0xff000000UL, 0xf3000000UL, 0xd2000000UL, +0xcd000000UL, 0x0c000000UL, 0x13000000UL, 0xec000000UL, 0x5f000000UL, 0x97000000UL, 0x44000000UL, 0x17000000UL, +0xc4000000UL, 0xa7000000UL, 0x7e000000UL, 0x3d000000UL, 0x64000000UL, 0x5d000000UL, 0x19000000UL, 0x73000000UL, +0x60000000UL, 0x81000000UL, 0x4f000000UL, 0xdc000000UL, 0x22000000UL, 0x2a000000UL, 0x90000000UL, 0x88000000UL, +0x46000000UL, 0xee000000UL, 0xb8000000UL, 0x14000000UL, 0xde000000UL, 0x5e000000UL, 0x0b000000UL, 0xdb000000UL, +0xe0000000UL, 0x32000000UL, 0x3a000000UL, 0x0a000000UL, 0x49000000UL, 0x06000000UL, 0x24000000UL, 0x5c000000UL, +0xc2000000UL, 0xd3000000UL, 0xac000000UL, 0x62000000UL, 0x91000000UL, 0x95000000UL, 0xe4000000UL, 0x79000000UL, +0xe7000000UL, 0xc8000000UL, 0x37000000UL, 0x6d000000UL, 0x8d000000UL, 0xd5000000UL, 0x4e000000UL, 0xa9000000UL, +0x6c000000UL, 0x56000000UL, 0xf4000000UL, 0xea000000UL, 0x65000000UL, 0x7a000000UL, 0xae000000UL, 0x08000000UL, +0xba000000UL, 0x78000000UL, 0x25000000UL, 0x2e000000UL, 0x1c000000UL, 0xa6000000UL, 0xb4000000UL, 0xc6000000UL, +0xe8000000UL, 0xdd000000UL, 0x74000000UL, 0x1f000000UL, 0x4b000000UL, 0xbd000000UL, 0x8b000000UL, 0x8a000000UL, +0x70000000UL, 0x3e000000UL, 0xb5000000UL, 0x66000000UL, 0x48000000UL, 0x03000000UL, 0xf6000000UL, 0x0e000000UL, +0x61000000UL, 0x35000000UL, 0x57000000UL, 0xb9000000UL, 0x86000000UL, 0xc1000000UL, 0x1d000000UL, 0x9e000000UL, +0xe1000000UL, 0xf8000000UL, 0x98000000UL, 0x11000000UL, 0x69000000UL, 0xd9000000UL, 0x8e000000UL, 0x94000000UL, +0x9b000000UL, 0x1e000000UL, 0x87000000UL, 0xe9000000UL, 0xce000000UL, 0x55000000UL, 0x28000000UL, 0xdf000000UL, +0x8c000000UL, 0xa1000000UL, 0x89000000UL, 0x0d000000UL, 0xbf000000UL, 0xe6000000UL, 0x42000000UL, 0x68000000UL, +0x41000000UL, 0x99000000UL, 0x2d000000UL, 0x0f000000UL, 0xb0000000UL, 0x54000000UL, 0xbb000000UL, 0x16000000UL +}; +#endif /* pelimac */ + +#ifndef ENCRYPT_ONLY + +static const ulong32 TD1[256] = { + 0x5051f4a7UL, 0x537e4165UL, 0xc31a17a4UL, 0x963a275eUL, + 0xcb3bab6bUL, 0xf11f9d45UL, 0xabacfa58UL, 0x934be303UL, + 0x552030faUL, 0xf6ad766dUL, 0x9188cc76UL, 0x25f5024cUL, + 0xfc4fe5d7UL, 0xd7c52acbUL, 0x80263544UL, 0x8fb562a3UL, + 0x49deb15aUL, 0x6725ba1bUL, 0x9845ea0eUL, 0xe15dfec0UL, + 0x02c32f75UL, 0x12814cf0UL, 0xa38d4697UL, 0xc66bd3f9UL, + 0xe7038f5fUL, 0x9515929cUL, 0xebbf6d7aUL, 0xda955259UL, + 0x2dd4be83UL, 0xd3587421UL, 0x2949e069UL, 0x448ec9c8UL, + 0x6a75c289UL, 0x78f48e79UL, 0x6b99583eUL, 0xdd27b971UL, + 0xb6bee14fUL, 0x17f088adUL, 0x66c920acUL, 0xb47dce3aUL, + 0x1863df4aUL, 0x82e51a31UL, 0x60975133UL, 0x4562537fUL, + 0xe0b16477UL, 0x84bb6baeUL, 0x1cfe81a0UL, 0x94f9082bUL, + 0x58704868UL, 0x198f45fdUL, 0x8794de6cUL, 0xb7527bf8UL, + 0x23ab73d3UL, 0xe2724b02UL, 0x57e31f8fUL, 0x2a6655abUL, + 0x07b2eb28UL, 0x032fb5c2UL, 0x9a86c57bUL, 0xa5d33708UL, + 0xf2302887UL, 0xb223bfa5UL, 0xba02036aUL, 0x5ced1682UL, + 0x2b8acf1cUL, 0x92a779b4UL, 0xf0f307f2UL, 0xa14e69e2UL, + 0xcd65daf4UL, 0xd50605beUL, 0x1fd13462UL, 0x8ac4a6feUL, + 0x9d342e53UL, 0xa0a2f355UL, 0x32058ae1UL, 0x75a4f6ebUL, + 0x390b83ecUL, 0xaa4060efUL, 0x065e719fUL, 0x51bd6e10UL, + 0xf93e218aUL, 0x3d96dd06UL, 0xaedd3e05UL, 0x464de6bdUL, + 0xb591548dUL, 0x0571c45dUL, 0x6f0406d4UL, 0xff605015UL, + 0x241998fbUL, 0x97d6bde9UL, 0xcc894043UL, 0x7767d99eUL, + 0xbdb0e842UL, 0x8807898bUL, 0x38e7195bUL, 0xdb79c8eeUL, + 0x47a17c0aUL, 0xe97c420fUL, 0xc9f8841eUL, 0x00000000UL, + 0x83098086UL, 0x48322bedUL, 0xac1e1170UL, 0x4e6c5a72UL, + 0xfbfd0effUL, 0x560f8538UL, 0x1e3daed5UL, 0x27362d39UL, + 0x640a0fd9UL, 0x21685ca6UL, 0xd19b5b54UL, 0x3a24362eUL, + 0xb10c0a67UL, 0x0f9357e7UL, 0xd2b4ee96UL, 0x9e1b9b91UL, + 0x4f80c0c5UL, 0xa261dc20UL, 0x695a774bUL, 0x161c121aUL, + 0x0ae293baUL, 0xe5c0a02aUL, 0x433c22e0UL, 0x1d121b17UL, + 0x0b0e090dUL, 0xadf28bc7UL, 0xb92db6a8UL, 0xc8141ea9UL, + 0x8557f119UL, 0x4caf7507UL, 0xbbee99ddUL, 0xfda37f60UL, + 0x9ff70126UL, 0xbc5c72f5UL, 0xc544663bUL, 0x345bfb7eUL, + 0x768b4329UL, 0xdccb23c6UL, 0x68b6edfcUL, 0x63b8e4f1UL, + 0xcad731dcUL, 0x10426385UL, 0x40139722UL, 0x2084c611UL, + 0x7d854a24UL, 0xf8d2bb3dUL, 0x11aef932UL, 0x6dc729a1UL, + 0x4b1d9e2fUL, 0xf3dcb230UL, 0xec0d8652UL, 0xd077c1e3UL, + 0x6c2bb316UL, 0x99a970b9UL, 0xfa119448UL, 0x2247e964UL, + 0xc4a8fc8cUL, 0x1aa0f03fUL, 0xd8567d2cUL, 0xef223390UL, + 0xc787494eUL, 0xc1d938d1UL, 0xfe8ccaa2UL, 0x3698d40bUL, + 0xcfa6f581UL, 0x28a57adeUL, 0x26dab78eUL, 0xa43fadbfUL, + 0xe42c3a9dUL, 0x0d507892UL, 0x9b6a5fccUL, 0x62547e46UL, + 0xc2f68d13UL, 0xe890d8b8UL, 0x5e2e39f7UL, 0xf582c3afUL, + 0xbe9f5d80UL, 0x7c69d093UL, 0xa96fd52dUL, 0xb3cf2512UL, + 0x3bc8ac99UL, 0xa710187dUL, 0x6ee89c63UL, 0x7bdb3bbbUL, + 0x09cd2678UL, 0xf46e5918UL, 0x01ec9ab7UL, 0xa8834f9aUL, + 0x65e6956eUL, 0x7eaaffe6UL, 0x0821bccfUL, 0xe6ef15e8UL, + 0xd9bae79bUL, 0xce4a6f36UL, 0xd4ea9f09UL, 0xd629b07cUL, + 0xaf31a4b2UL, 0x312a3f23UL, 0x30c6a594UL, 0xc035a266UL, + 0x37744ebcUL, 0xa6fc82caUL, 0xb0e090d0UL, 0x1533a7d8UL, + 0x4af10498UL, 0xf741ecdaUL, 0x0e7fcd50UL, 0x2f1791f6UL, + 0x8d764dd6UL, 0x4d43efb0UL, 0x54ccaa4dUL, 0xdfe49604UL, + 0xe39ed1b5UL, 0x1b4c6a88UL, 0xb8c12c1fUL, 0x7f466551UL, + 0x049d5eeaUL, 0x5d018c35UL, 0x73fa8774UL, 0x2efb0b41UL, + 0x5ab3671dUL, 0x5292dbd2UL, 0x33e91056UL, 0x136dd647UL, + 0x8c9ad761UL, 0x7a37a10cUL, 0x8e59f814UL, 0x89eb133cUL, + 0xeecea927UL, 0x35b761c9UL, 0xede11ce5UL, 0x3c7a47b1UL, + 0x599cd2dfUL, 0x3f55f273UL, 0x791814ceUL, 0xbf73c737UL, + 0xea53f7cdUL, 0x5b5ffdaaUL, 0x14df3d6fUL, 0x867844dbUL, + 0x81caaff3UL, 0x3eb968c4UL, 0x2c382434UL, 0x5fc2a340UL, + 0x72161dc3UL, 0x0cbce225UL, 0x8b283c49UL, 0x41ff0d95UL, + 0x7139a801UL, 0xde080cb3UL, 0x9cd8b4e4UL, 0x906456c1UL, + 0x617bcb84UL, 0x70d532b6UL, 0x74486c5cUL, 0x42d0b857UL, +}; +static const ulong32 TD2[256] = { + 0xa75051f4UL, 0x65537e41UL, 0xa4c31a17UL, 0x5e963a27UL, + 0x6bcb3babUL, 0x45f11f9dUL, 0x58abacfaUL, 0x03934be3UL, + 0xfa552030UL, 0x6df6ad76UL, 0x769188ccUL, 0x4c25f502UL, + 0xd7fc4fe5UL, 0xcbd7c52aUL, 0x44802635UL, 0xa38fb562UL, + 0x5a49deb1UL, 0x1b6725baUL, 0x0e9845eaUL, 0xc0e15dfeUL, + 0x7502c32fUL, 0xf012814cUL, 0x97a38d46UL, 0xf9c66bd3UL, + 0x5fe7038fUL, 0x9c951592UL, 0x7aebbf6dUL, 0x59da9552UL, + 0x832dd4beUL, 0x21d35874UL, 0x692949e0UL, 0xc8448ec9UL, + 0x896a75c2UL, 0x7978f48eUL, 0x3e6b9958UL, 0x71dd27b9UL, + 0x4fb6bee1UL, 0xad17f088UL, 0xac66c920UL, 0x3ab47dceUL, + 0x4a1863dfUL, 0x3182e51aUL, 0x33609751UL, 0x7f456253UL, + 0x77e0b164UL, 0xae84bb6bUL, 0xa01cfe81UL, 0x2b94f908UL, + 0x68587048UL, 0xfd198f45UL, 0x6c8794deUL, 0xf8b7527bUL, + 0xd323ab73UL, 0x02e2724bUL, 0x8f57e31fUL, 0xab2a6655UL, + 0x2807b2ebUL, 0xc2032fb5UL, 0x7b9a86c5UL, 0x08a5d337UL, + 0x87f23028UL, 0xa5b223bfUL, 0x6aba0203UL, 0x825ced16UL, + 0x1c2b8acfUL, 0xb492a779UL, 0xf2f0f307UL, 0xe2a14e69UL, + 0xf4cd65daUL, 0xbed50605UL, 0x621fd134UL, 0xfe8ac4a6UL, + 0x539d342eUL, 0x55a0a2f3UL, 0xe132058aUL, 0xeb75a4f6UL, + 0xec390b83UL, 0xefaa4060UL, 0x9f065e71UL, 0x1051bd6eUL, + 0x8af93e21UL, 0x063d96ddUL, 0x05aedd3eUL, 0xbd464de6UL, + 0x8db59154UL, 0x5d0571c4UL, 0xd46f0406UL, 0x15ff6050UL, + 0xfb241998UL, 0xe997d6bdUL, 0x43cc8940UL, 0x9e7767d9UL, + 0x42bdb0e8UL, 0x8b880789UL, 0x5b38e719UL, 0xeedb79c8UL, + 0x0a47a17cUL, 0x0fe97c42UL, 0x1ec9f884UL, 0x00000000UL, + 0x86830980UL, 0xed48322bUL, 0x70ac1e11UL, 0x724e6c5aUL, + 0xfffbfd0eUL, 0x38560f85UL, 0xd51e3daeUL, 0x3927362dUL, + 0xd9640a0fUL, 0xa621685cUL, 0x54d19b5bUL, 0x2e3a2436UL, + 0x67b10c0aUL, 0xe70f9357UL, 0x96d2b4eeUL, 0x919e1b9bUL, + 0xc54f80c0UL, 0x20a261dcUL, 0x4b695a77UL, 0x1a161c12UL, + 0xba0ae293UL, 0x2ae5c0a0UL, 0xe0433c22UL, 0x171d121bUL, + 0x0d0b0e09UL, 0xc7adf28bUL, 0xa8b92db6UL, 0xa9c8141eUL, + 0x198557f1UL, 0x074caf75UL, 0xddbbee99UL, 0x60fda37fUL, + 0x269ff701UL, 0xf5bc5c72UL, 0x3bc54466UL, 0x7e345bfbUL, + 0x29768b43UL, 0xc6dccb23UL, 0xfc68b6edUL, 0xf163b8e4UL, + 0xdccad731UL, 0x85104263UL, 0x22401397UL, 0x112084c6UL, + 0x247d854aUL, 0x3df8d2bbUL, 0x3211aef9UL, 0xa16dc729UL, + 0x2f4b1d9eUL, 0x30f3dcb2UL, 0x52ec0d86UL, 0xe3d077c1UL, + 0x166c2bb3UL, 0xb999a970UL, 0x48fa1194UL, 0x642247e9UL, + 0x8cc4a8fcUL, 0x3f1aa0f0UL, 0x2cd8567dUL, 0x90ef2233UL, + 0x4ec78749UL, 0xd1c1d938UL, 0xa2fe8ccaUL, 0x0b3698d4UL, + 0x81cfa6f5UL, 0xde28a57aUL, 0x8e26dab7UL, 0xbfa43fadUL, + 0x9de42c3aUL, 0x920d5078UL, 0xcc9b6a5fUL, 0x4662547eUL, + 0x13c2f68dUL, 0xb8e890d8UL, 0xf75e2e39UL, 0xaff582c3UL, + 0x80be9f5dUL, 0x937c69d0UL, 0x2da96fd5UL, 0x12b3cf25UL, + 0x993bc8acUL, 0x7da71018UL, 0x636ee89cUL, 0xbb7bdb3bUL, + 0x7809cd26UL, 0x18f46e59UL, 0xb701ec9aUL, 0x9aa8834fUL, + 0x6e65e695UL, 0xe67eaaffUL, 0xcf0821bcUL, 0xe8e6ef15UL, + 0x9bd9bae7UL, 0x36ce4a6fUL, 0x09d4ea9fUL, 0x7cd629b0UL, + 0xb2af31a4UL, 0x23312a3fUL, 0x9430c6a5UL, 0x66c035a2UL, + 0xbc37744eUL, 0xcaa6fc82UL, 0xd0b0e090UL, 0xd81533a7UL, + 0x984af104UL, 0xdaf741ecUL, 0x500e7fcdUL, 0xf62f1791UL, + 0xd68d764dUL, 0xb04d43efUL, 0x4d54ccaaUL, 0x04dfe496UL, + 0xb5e39ed1UL, 0x881b4c6aUL, 0x1fb8c12cUL, 0x517f4665UL, + 0xea049d5eUL, 0x355d018cUL, 0x7473fa87UL, 0x412efb0bUL, + 0x1d5ab367UL, 0xd25292dbUL, 0x5633e910UL, 0x47136dd6UL, + 0x618c9ad7UL, 0x0c7a37a1UL, 0x148e59f8UL, 0x3c89eb13UL, + 0x27eecea9UL, 0xc935b761UL, 0xe5ede11cUL, 0xb13c7a47UL, + 0xdf599cd2UL, 0x733f55f2UL, 0xce791814UL, 0x37bf73c7UL, + 0xcdea53f7UL, 0xaa5b5ffdUL, 0x6f14df3dUL, 0xdb867844UL, + 0xf381caafUL, 0xc43eb968UL, 0x342c3824UL, 0x405fc2a3UL, + 0xc372161dUL, 0x250cbce2UL, 0x498b283cUL, 0x9541ff0dUL, + 0x017139a8UL, 0xb3de080cUL, 0xe49cd8b4UL, 0xc1906456UL, + 0x84617bcbUL, 0xb670d532UL, 0x5c74486cUL, 0x5742d0b8UL, +}; +static const ulong32 TD3[256] = { + 0xf4a75051UL, 0x4165537eUL, 0x17a4c31aUL, 0x275e963aUL, + 0xab6bcb3bUL, 0x9d45f11fUL, 0xfa58abacUL, 0xe303934bUL, + 0x30fa5520UL, 0x766df6adUL, 0xcc769188UL, 0x024c25f5UL, + 0xe5d7fc4fUL, 0x2acbd7c5UL, 0x35448026UL, 0x62a38fb5UL, + 0xb15a49deUL, 0xba1b6725UL, 0xea0e9845UL, 0xfec0e15dUL, + 0x2f7502c3UL, 0x4cf01281UL, 0x4697a38dUL, 0xd3f9c66bUL, + 0x8f5fe703UL, 0x929c9515UL, 0x6d7aebbfUL, 0x5259da95UL, + 0xbe832dd4UL, 0x7421d358UL, 0xe0692949UL, 0xc9c8448eUL, + 0xc2896a75UL, 0x8e7978f4UL, 0x583e6b99UL, 0xb971dd27UL, + 0xe14fb6beUL, 0x88ad17f0UL, 0x20ac66c9UL, 0xce3ab47dUL, + 0xdf4a1863UL, 0x1a3182e5UL, 0x51336097UL, 0x537f4562UL, + 0x6477e0b1UL, 0x6bae84bbUL, 0x81a01cfeUL, 0x082b94f9UL, + 0x48685870UL, 0x45fd198fUL, 0xde6c8794UL, 0x7bf8b752UL, + 0x73d323abUL, 0x4b02e272UL, 0x1f8f57e3UL, 0x55ab2a66UL, + 0xeb2807b2UL, 0xb5c2032fUL, 0xc57b9a86UL, 0x3708a5d3UL, + 0x2887f230UL, 0xbfa5b223UL, 0x036aba02UL, 0x16825cedUL, + 0xcf1c2b8aUL, 0x79b492a7UL, 0x07f2f0f3UL, 0x69e2a14eUL, + 0xdaf4cd65UL, 0x05bed506UL, 0x34621fd1UL, 0xa6fe8ac4UL, + 0x2e539d34UL, 0xf355a0a2UL, 0x8ae13205UL, 0xf6eb75a4UL, + 0x83ec390bUL, 0x60efaa40UL, 0x719f065eUL, 0x6e1051bdUL, + 0x218af93eUL, 0xdd063d96UL, 0x3e05aeddUL, 0xe6bd464dUL, + 0x548db591UL, 0xc45d0571UL, 0x06d46f04UL, 0x5015ff60UL, + 0x98fb2419UL, 0xbde997d6UL, 0x4043cc89UL, 0xd99e7767UL, + 0xe842bdb0UL, 0x898b8807UL, 0x195b38e7UL, 0xc8eedb79UL, + 0x7c0a47a1UL, 0x420fe97cUL, 0x841ec9f8UL, 0x00000000UL, + 0x80868309UL, 0x2bed4832UL, 0x1170ac1eUL, 0x5a724e6cUL, + 0x0efffbfdUL, 0x8538560fUL, 0xaed51e3dUL, 0x2d392736UL, + 0x0fd9640aUL, 0x5ca62168UL, 0x5b54d19bUL, 0x362e3a24UL, + 0x0a67b10cUL, 0x57e70f93UL, 0xee96d2b4UL, 0x9b919e1bUL, + 0xc0c54f80UL, 0xdc20a261UL, 0x774b695aUL, 0x121a161cUL, + 0x93ba0ae2UL, 0xa02ae5c0UL, 0x22e0433cUL, 0x1b171d12UL, + 0x090d0b0eUL, 0x8bc7adf2UL, 0xb6a8b92dUL, 0x1ea9c814UL, + 0xf1198557UL, 0x75074cafUL, 0x99ddbbeeUL, 0x7f60fda3UL, + 0x01269ff7UL, 0x72f5bc5cUL, 0x663bc544UL, 0xfb7e345bUL, + 0x4329768bUL, 0x23c6dccbUL, 0xedfc68b6UL, 0xe4f163b8UL, + 0x31dccad7UL, 0x63851042UL, 0x97224013UL, 0xc6112084UL, + 0x4a247d85UL, 0xbb3df8d2UL, 0xf93211aeUL, 0x29a16dc7UL, + 0x9e2f4b1dUL, 0xb230f3dcUL, 0x8652ec0dUL, 0xc1e3d077UL, + 0xb3166c2bUL, 0x70b999a9UL, 0x9448fa11UL, 0xe9642247UL, + 0xfc8cc4a8UL, 0xf03f1aa0UL, 0x7d2cd856UL, 0x3390ef22UL, + 0x494ec787UL, 0x38d1c1d9UL, 0xcaa2fe8cUL, 0xd40b3698UL, + 0xf581cfa6UL, 0x7ade28a5UL, 0xb78e26daUL, 0xadbfa43fUL, + 0x3a9de42cUL, 0x78920d50UL, 0x5fcc9b6aUL, 0x7e466254UL, + 0x8d13c2f6UL, 0xd8b8e890UL, 0x39f75e2eUL, 0xc3aff582UL, + 0x5d80be9fUL, 0xd0937c69UL, 0xd52da96fUL, 0x2512b3cfUL, + 0xac993bc8UL, 0x187da710UL, 0x9c636ee8UL, 0x3bbb7bdbUL, + 0x267809cdUL, 0x5918f46eUL, 0x9ab701ecUL, 0x4f9aa883UL, + 0x956e65e6UL, 0xffe67eaaUL, 0xbccf0821UL, 0x15e8e6efUL, + 0xe79bd9baUL, 0x6f36ce4aUL, 0x9f09d4eaUL, 0xb07cd629UL, + 0xa4b2af31UL, 0x3f23312aUL, 0xa59430c6UL, 0xa266c035UL, + 0x4ebc3774UL, 0x82caa6fcUL, 0x90d0b0e0UL, 0xa7d81533UL, + 0x04984af1UL, 0xecdaf741UL, 0xcd500e7fUL, 0x91f62f17UL, + 0x4dd68d76UL, 0xefb04d43UL, 0xaa4d54ccUL, 0x9604dfe4UL, + 0xd1b5e39eUL, 0x6a881b4cUL, 0x2c1fb8c1UL, 0x65517f46UL, + 0x5eea049dUL, 0x8c355d01UL, 0x877473faUL, 0x0b412efbUL, + 0x671d5ab3UL, 0xdbd25292UL, 0x105633e9UL, 0xd647136dUL, + 0xd7618c9aUL, 0xa10c7a37UL, 0xf8148e59UL, 0x133c89ebUL, + 0xa927eeceUL, 0x61c935b7UL, 0x1ce5ede1UL, 0x47b13c7aUL, + 0xd2df599cUL, 0xf2733f55UL, 0x14ce7918UL, 0xc737bf73UL, + 0xf7cdea53UL, 0xfdaa5b5fUL, 0x3d6f14dfUL, 0x44db8678UL, + 0xaff381caUL, 0x68c43eb9UL, 0x24342c38UL, 0xa3405fc2UL, + 0x1dc37216UL, 0xe2250cbcUL, 0x3c498b28UL, 0x0d9541ffUL, + 0xa8017139UL, 0x0cb3de08UL, 0xb4e49cd8UL, 0x56c19064UL, + 0xcb84617bUL, 0x32b670d5UL, 0x6c5c7448UL, 0xb85742d0UL, +}; + +static const ulong32 Tks0[] = { +0x00000000UL, 0x0e090d0bUL, 0x1c121a16UL, 0x121b171dUL, 0x3824342cUL, 0x362d3927UL, 0x24362e3aUL, 0x2a3f2331UL, +0x70486858UL, 0x7e416553UL, 0x6c5a724eUL, 0x62537f45UL, 0x486c5c74UL, 0x4665517fUL, 0x547e4662UL, 0x5a774b69UL, +0xe090d0b0UL, 0xee99ddbbUL, 0xfc82caa6UL, 0xf28bc7adUL, 0xd8b4e49cUL, 0xd6bde997UL, 0xc4a6fe8aUL, 0xcaaff381UL, +0x90d8b8e8UL, 0x9ed1b5e3UL, 0x8ccaa2feUL, 0x82c3aff5UL, 0xa8fc8cc4UL, 0xa6f581cfUL, 0xb4ee96d2UL, 0xbae79bd9UL, +0xdb3bbb7bUL, 0xd532b670UL, 0xc729a16dUL, 0xc920ac66UL, 0xe31f8f57UL, 0xed16825cUL, 0xff0d9541UL, 0xf104984aUL, +0xab73d323UL, 0xa57ade28UL, 0xb761c935UL, 0xb968c43eUL, 0x9357e70fUL, 0x9d5eea04UL, 0x8f45fd19UL, 0x814cf012UL, +0x3bab6bcbUL, 0x35a266c0UL, 0x27b971ddUL, 0x29b07cd6UL, 0x038f5fe7UL, 0x0d8652ecUL, 0x1f9d45f1UL, 0x119448faUL, +0x4be30393UL, 0x45ea0e98UL, 0x57f11985UL, 0x59f8148eUL, 0x73c737bfUL, 0x7dce3ab4UL, 0x6fd52da9UL, 0x61dc20a2UL, +0xad766df6UL, 0xa37f60fdUL, 0xb16477e0UL, 0xbf6d7aebUL, 0x955259daUL, 0x9b5b54d1UL, 0x894043ccUL, 0x87494ec7UL, +0xdd3e05aeUL, 0xd33708a5UL, 0xc12c1fb8UL, 0xcf2512b3UL, 0xe51a3182UL, 0xeb133c89UL, 0xf9082b94UL, 0xf701269fUL, +0x4de6bd46UL, 0x43efb04dUL, 0x51f4a750UL, 0x5ffdaa5bUL, 0x75c2896aUL, 0x7bcb8461UL, 0x69d0937cUL, 0x67d99e77UL, +0x3daed51eUL, 0x33a7d815UL, 0x21bccf08UL, 0x2fb5c203UL, 0x058ae132UL, 0x0b83ec39UL, 0x1998fb24UL, 0x1791f62fUL, +0x764dd68dUL, 0x7844db86UL, 0x6a5fcc9bUL, 0x6456c190UL, 0x4e69e2a1UL, 0x4060efaaUL, 0x527bf8b7UL, 0x5c72f5bcUL, +0x0605bed5UL, 0x080cb3deUL, 0x1a17a4c3UL, 0x141ea9c8UL, 0x3e218af9UL, 0x302887f2UL, 0x223390efUL, 0x2c3a9de4UL, +0x96dd063dUL, 0x98d40b36UL, 0x8acf1c2bUL, 0x84c61120UL, 0xaef93211UL, 0xa0f03f1aUL, 0xb2eb2807UL, 0xbce2250cUL, +0xe6956e65UL, 0xe89c636eUL, 0xfa877473UL, 0xf48e7978UL, 0xdeb15a49UL, 0xd0b85742UL, 0xc2a3405fUL, 0xccaa4d54UL, +0x41ecdaf7UL, 0x4fe5d7fcUL, 0x5dfec0e1UL, 0x53f7cdeaUL, 0x79c8eedbUL, 0x77c1e3d0UL, 0x65daf4cdUL, 0x6bd3f9c6UL, +0x31a4b2afUL, 0x3fadbfa4UL, 0x2db6a8b9UL, 0x23bfa5b2UL, 0x09808683UL, 0x07898b88UL, 0x15929c95UL, 0x1b9b919eUL, +0xa17c0a47UL, 0xaf75074cUL, 0xbd6e1051UL, 0xb3671d5aUL, 0x99583e6bUL, 0x97513360UL, 0x854a247dUL, 0x8b432976UL, +0xd134621fUL, 0xdf3d6f14UL, 0xcd267809UL, 0xc32f7502UL, 0xe9105633UL, 0xe7195b38UL, 0xf5024c25UL, 0xfb0b412eUL, +0x9ad7618cUL, 0x94de6c87UL, 0x86c57b9aUL, 0x88cc7691UL, 0xa2f355a0UL, 0xacfa58abUL, 0xbee14fb6UL, 0xb0e842bdUL, +0xea9f09d4UL, 0xe49604dfUL, 0xf68d13c2UL, 0xf8841ec9UL, 0xd2bb3df8UL, 0xdcb230f3UL, 0xcea927eeUL, 0xc0a02ae5UL, +0x7a47b13cUL, 0x744ebc37UL, 0x6655ab2aUL, 0x685ca621UL, 0x42638510UL, 0x4c6a881bUL, 0x5e719f06UL, 0x5078920dUL, +0x0a0fd964UL, 0x0406d46fUL, 0x161dc372UL, 0x1814ce79UL, 0x322bed48UL, 0x3c22e043UL, 0x2e39f75eUL, 0x2030fa55UL, +0xec9ab701UL, 0xe293ba0aUL, 0xf088ad17UL, 0xfe81a01cUL, 0xd4be832dUL, 0xdab78e26UL, 0xc8ac993bUL, 0xc6a59430UL, +0x9cd2df59UL, 0x92dbd252UL, 0x80c0c54fUL, 0x8ec9c844UL, 0xa4f6eb75UL, 0xaaffe67eUL, 0xb8e4f163UL, 0xb6edfc68UL, +0x0c0a67b1UL, 0x02036abaUL, 0x10187da7UL, 0x1e1170acUL, 0x342e539dUL, 0x3a275e96UL, 0x283c498bUL, 0x26354480UL, +0x7c420fe9UL, 0x724b02e2UL, 0x605015ffUL, 0x6e5918f4UL, 0x44663bc5UL, 0x4a6f36ceUL, 0x587421d3UL, 0x567d2cd8UL, +0x37a10c7aUL, 0x39a80171UL, 0x2bb3166cUL, 0x25ba1b67UL, 0x0f853856UL, 0x018c355dUL, 0x13972240UL, 0x1d9e2f4bUL, +0x47e96422UL, 0x49e06929UL, 0x5bfb7e34UL, 0x55f2733fUL, 0x7fcd500eUL, 0x71c45d05UL, 0x63df4a18UL, 0x6dd64713UL, +0xd731dccaUL, 0xd938d1c1UL, 0xcb23c6dcUL, 0xc52acbd7UL, 0xef15e8e6UL, 0xe11ce5edUL, 0xf307f2f0UL, 0xfd0efffbUL, +0xa779b492UL, 0xa970b999UL, 0xbb6bae84UL, 0xb562a38fUL, 0x9f5d80beUL, 0x91548db5UL, 0x834f9aa8UL, 0x8d4697a3UL +}; + +static const ulong32 Tks1[] = { +0x00000000UL, 0x0b0e090dUL, 0x161c121aUL, 0x1d121b17UL, 0x2c382434UL, 0x27362d39UL, 0x3a24362eUL, 0x312a3f23UL, +0x58704868UL, 0x537e4165UL, 0x4e6c5a72UL, 0x4562537fUL, 0x74486c5cUL, 0x7f466551UL, 0x62547e46UL, 0x695a774bUL, +0xb0e090d0UL, 0xbbee99ddUL, 0xa6fc82caUL, 0xadf28bc7UL, 0x9cd8b4e4UL, 0x97d6bde9UL, 0x8ac4a6feUL, 0x81caaff3UL, +0xe890d8b8UL, 0xe39ed1b5UL, 0xfe8ccaa2UL, 0xf582c3afUL, 0xc4a8fc8cUL, 0xcfa6f581UL, 0xd2b4ee96UL, 0xd9bae79bUL, +0x7bdb3bbbUL, 0x70d532b6UL, 0x6dc729a1UL, 0x66c920acUL, 0x57e31f8fUL, 0x5ced1682UL, 0x41ff0d95UL, 0x4af10498UL, +0x23ab73d3UL, 0x28a57adeUL, 0x35b761c9UL, 0x3eb968c4UL, 0x0f9357e7UL, 0x049d5eeaUL, 0x198f45fdUL, 0x12814cf0UL, +0xcb3bab6bUL, 0xc035a266UL, 0xdd27b971UL, 0xd629b07cUL, 0xe7038f5fUL, 0xec0d8652UL, 0xf11f9d45UL, 0xfa119448UL, +0x934be303UL, 0x9845ea0eUL, 0x8557f119UL, 0x8e59f814UL, 0xbf73c737UL, 0xb47dce3aUL, 0xa96fd52dUL, 0xa261dc20UL, +0xf6ad766dUL, 0xfda37f60UL, 0xe0b16477UL, 0xebbf6d7aUL, 0xda955259UL, 0xd19b5b54UL, 0xcc894043UL, 0xc787494eUL, +0xaedd3e05UL, 0xa5d33708UL, 0xb8c12c1fUL, 0xb3cf2512UL, 0x82e51a31UL, 0x89eb133cUL, 0x94f9082bUL, 0x9ff70126UL, +0x464de6bdUL, 0x4d43efb0UL, 0x5051f4a7UL, 0x5b5ffdaaUL, 0x6a75c289UL, 0x617bcb84UL, 0x7c69d093UL, 0x7767d99eUL, +0x1e3daed5UL, 0x1533a7d8UL, 0x0821bccfUL, 0x032fb5c2UL, 0x32058ae1UL, 0x390b83ecUL, 0x241998fbUL, 0x2f1791f6UL, +0x8d764dd6UL, 0x867844dbUL, 0x9b6a5fccUL, 0x906456c1UL, 0xa14e69e2UL, 0xaa4060efUL, 0xb7527bf8UL, 0xbc5c72f5UL, +0xd50605beUL, 0xde080cb3UL, 0xc31a17a4UL, 0xc8141ea9UL, 0xf93e218aUL, 0xf2302887UL, 0xef223390UL, 0xe42c3a9dUL, +0x3d96dd06UL, 0x3698d40bUL, 0x2b8acf1cUL, 0x2084c611UL, 0x11aef932UL, 0x1aa0f03fUL, 0x07b2eb28UL, 0x0cbce225UL, +0x65e6956eUL, 0x6ee89c63UL, 0x73fa8774UL, 0x78f48e79UL, 0x49deb15aUL, 0x42d0b857UL, 0x5fc2a340UL, 0x54ccaa4dUL, +0xf741ecdaUL, 0xfc4fe5d7UL, 0xe15dfec0UL, 0xea53f7cdUL, 0xdb79c8eeUL, 0xd077c1e3UL, 0xcd65daf4UL, 0xc66bd3f9UL, +0xaf31a4b2UL, 0xa43fadbfUL, 0xb92db6a8UL, 0xb223bfa5UL, 0x83098086UL, 0x8807898bUL, 0x9515929cUL, 0x9e1b9b91UL, +0x47a17c0aUL, 0x4caf7507UL, 0x51bd6e10UL, 0x5ab3671dUL, 0x6b99583eUL, 0x60975133UL, 0x7d854a24UL, 0x768b4329UL, +0x1fd13462UL, 0x14df3d6fUL, 0x09cd2678UL, 0x02c32f75UL, 0x33e91056UL, 0x38e7195bUL, 0x25f5024cUL, 0x2efb0b41UL, +0x8c9ad761UL, 0x8794de6cUL, 0x9a86c57bUL, 0x9188cc76UL, 0xa0a2f355UL, 0xabacfa58UL, 0xb6bee14fUL, 0xbdb0e842UL, +0xd4ea9f09UL, 0xdfe49604UL, 0xc2f68d13UL, 0xc9f8841eUL, 0xf8d2bb3dUL, 0xf3dcb230UL, 0xeecea927UL, 0xe5c0a02aUL, +0x3c7a47b1UL, 0x37744ebcUL, 0x2a6655abUL, 0x21685ca6UL, 0x10426385UL, 0x1b4c6a88UL, 0x065e719fUL, 0x0d507892UL, +0x640a0fd9UL, 0x6f0406d4UL, 0x72161dc3UL, 0x791814ceUL, 0x48322bedUL, 0x433c22e0UL, 0x5e2e39f7UL, 0x552030faUL, +0x01ec9ab7UL, 0x0ae293baUL, 0x17f088adUL, 0x1cfe81a0UL, 0x2dd4be83UL, 0x26dab78eUL, 0x3bc8ac99UL, 0x30c6a594UL, +0x599cd2dfUL, 0x5292dbd2UL, 0x4f80c0c5UL, 0x448ec9c8UL, 0x75a4f6ebUL, 0x7eaaffe6UL, 0x63b8e4f1UL, 0x68b6edfcUL, +0xb10c0a67UL, 0xba02036aUL, 0xa710187dUL, 0xac1e1170UL, 0x9d342e53UL, 0x963a275eUL, 0x8b283c49UL, 0x80263544UL, +0xe97c420fUL, 0xe2724b02UL, 0xff605015UL, 0xf46e5918UL, 0xc544663bUL, 0xce4a6f36UL, 0xd3587421UL, 0xd8567d2cUL, +0x7a37a10cUL, 0x7139a801UL, 0x6c2bb316UL, 0x6725ba1bUL, 0x560f8538UL, 0x5d018c35UL, 0x40139722UL, 0x4b1d9e2fUL, +0x2247e964UL, 0x2949e069UL, 0x345bfb7eUL, 0x3f55f273UL, 0x0e7fcd50UL, 0x0571c45dUL, 0x1863df4aUL, 0x136dd647UL, +0xcad731dcUL, 0xc1d938d1UL, 0xdccb23c6UL, 0xd7c52acbUL, 0xe6ef15e8UL, 0xede11ce5UL, 0xf0f307f2UL, 0xfbfd0effUL, +0x92a779b4UL, 0x99a970b9UL, 0x84bb6baeUL, 0x8fb562a3UL, 0xbe9f5d80UL, 0xb591548dUL, 0xa8834f9aUL, 0xa38d4697UL +}; + +static const ulong32 Tks2[] = { +0x00000000UL, 0x0d0b0e09UL, 0x1a161c12UL, 0x171d121bUL, 0x342c3824UL, 0x3927362dUL, 0x2e3a2436UL, 0x23312a3fUL, +0x68587048UL, 0x65537e41UL, 0x724e6c5aUL, 0x7f456253UL, 0x5c74486cUL, 0x517f4665UL, 0x4662547eUL, 0x4b695a77UL, +0xd0b0e090UL, 0xddbbee99UL, 0xcaa6fc82UL, 0xc7adf28bUL, 0xe49cd8b4UL, 0xe997d6bdUL, 0xfe8ac4a6UL, 0xf381caafUL, +0xb8e890d8UL, 0xb5e39ed1UL, 0xa2fe8ccaUL, 0xaff582c3UL, 0x8cc4a8fcUL, 0x81cfa6f5UL, 0x96d2b4eeUL, 0x9bd9bae7UL, +0xbb7bdb3bUL, 0xb670d532UL, 0xa16dc729UL, 0xac66c920UL, 0x8f57e31fUL, 0x825ced16UL, 0x9541ff0dUL, 0x984af104UL, +0xd323ab73UL, 0xde28a57aUL, 0xc935b761UL, 0xc43eb968UL, 0xe70f9357UL, 0xea049d5eUL, 0xfd198f45UL, 0xf012814cUL, +0x6bcb3babUL, 0x66c035a2UL, 0x71dd27b9UL, 0x7cd629b0UL, 0x5fe7038fUL, 0x52ec0d86UL, 0x45f11f9dUL, 0x48fa1194UL, +0x03934be3UL, 0x0e9845eaUL, 0x198557f1UL, 0x148e59f8UL, 0x37bf73c7UL, 0x3ab47dceUL, 0x2da96fd5UL, 0x20a261dcUL, +0x6df6ad76UL, 0x60fda37fUL, 0x77e0b164UL, 0x7aebbf6dUL, 0x59da9552UL, 0x54d19b5bUL, 0x43cc8940UL, 0x4ec78749UL, +0x05aedd3eUL, 0x08a5d337UL, 0x1fb8c12cUL, 0x12b3cf25UL, 0x3182e51aUL, 0x3c89eb13UL, 0x2b94f908UL, 0x269ff701UL, +0xbd464de6UL, 0xb04d43efUL, 0xa75051f4UL, 0xaa5b5ffdUL, 0x896a75c2UL, 0x84617bcbUL, 0x937c69d0UL, 0x9e7767d9UL, +0xd51e3daeUL, 0xd81533a7UL, 0xcf0821bcUL, 0xc2032fb5UL, 0xe132058aUL, 0xec390b83UL, 0xfb241998UL, 0xf62f1791UL, +0xd68d764dUL, 0xdb867844UL, 0xcc9b6a5fUL, 0xc1906456UL, 0xe2a14e69UL, 0xefaa4060UL, 0xf8b7527bUL, 0xf5bc5c72UL, +0xbed50605UL, 0xb3de080cUL, 0xa4c31a17UL, 0xa9c8141eUL, 0x8af93e21UL, 0x87f23028UL, 0x90ef2233UL, 0x9de42c3aUL, +0x063d96ddUL, 0x0b3698d4UL, 0x1c2b8acfUL, 0x112084c6UL, 0x3211aef9UL, 0x3f1aa0f0UL, 0x2807b2ebUL, 0x250cbce2UL, +0x6e65e695UL, 0x636ee89cUL, 0x7473fa87UL, 0x7978f48eUL, 0x5a49deb1UL, 0x5742d0b8UL, 0x405fc2a3UL, 0x4d54ccaaUL, +0xdaf741ecUL, 0xd7fc4fe5UL, 0xc0e15dfeUL, 0xcdea53f7UL, 0xeedb79c8UL, 0xe3d077c1UL, 0xf4cd65daUL, 0xf9c66bd3UL, +0xb2af31a4UL, 0xbfa43fadUL, 0xa8b92db6UL, 0xa5b223bfUL, 0x86830980UL, 0x8b880789UL, 0x9c951592UL, 0x919e1b9bUL, +0x0a47a17cUL, 0x074caf75UL, 0x1051bd6eUL, 0x1d5ab367UL, 0x3e6b9958UL, 0x33609751UL, 0x247d854aUL, 0x29768b43UL, +0x621fd134UL, 0x6f14df3dUL, 0x7809cd26UL, 0x7502c32fUL, 0x5633e910UL, 0x5b38e719UL, 0x4c25f502UL, 0x412efb0bUL, +0x618c9ad7UL, 0x6c8794deUL, 0x7b9a86c5UL, 0x769188ccUL, 0x55a0a2f3UL, 0x58abacfaUL, 0x4fb6bee1UL, 0x42bdb0e8UL, +0x09d4ea9fUL, 0x04dfe496UL, 0x13c2f68dUL, 0x1ec9f884UL, 0x3df8d2bbUL, 0x30f3dcb2UL, 0x27eecea9UL, 0x2ae5c0a0UL, +0xb13c7a47UL, 0xbc37744eUL, 0xab2a6655UL, 0xa621685cUL, 0x85104263UL, 0x881b4c6aUL, 0x9f065e71UL, 0x920d5078UL, +0xd9640a0fUL, 0xd46f0406UL, 0xc372161dUL, 0xce791814UL, 0xed48322bUL, 0xe0433c22UL, 0xf75e2e39UL, 0xfa552030UL, +0xb701ec9aUL, 0xba0ae293UL, 0xad17f088UL, 0xa01cfe81UL, 0x832dd4beUL, 0x8e26dab7UL, 0x993bc8acUL, 0x9430c6a5UL, +0xdf599cd2UL, 0xd25292dbUL, 0xc54f80c0UL, 0xc8448ec9UL, 0xeb75a4f6UL, 0xe67eaaffUL, 0xf163b8e4UL, 0xfc68b6edUL, +0x67b10c0aUL, 0x6aba0203UL, 0x7da71018UL, 0x70ac1e11UL, 0x539d342eUL, 0x5e963a27UL, 0x498b283cUL, 0x44802635UL, +0x0fe97c42UL, 0x02e2724bUL, 0x15ff6050UL, 0x18f46e59UL, 0x3bc54466UL, 0x36ce4a6fUL, 0x21d35874UL, 0x2cd8567dUL, +0x0c7a37a1UL, 0x017139a8UL, 0x166c2bb3UL, 0x1b6725baUL, 0x38560f85UL, 0x355d018cUL, 0x22401397UL, 0x2f4b1d9eUL, +0x642247e9UL, 0x692949e0UL, 0x7e345bfbUL, 0x733f55f2UL, 0x500e7fcdUL, 0x5d0571c4UL, 0x4a1863dfUL, 0x47136dd6UL, +0xdccad731UL, 0xd1c1d938UL, 0xc6dccb23UL, 0xcbd7c52aUL, 0xe8e6ef15UL, 0xe5ede11cUL, 0xf2f0f307UL, 0xfffbfd0eUL, +0xb492a779UL, 0xb999a970UL, 0xae84bb6bUL, 0xa38fb562UL, 0x80be9f5dUL, 0x8db59154UL, 0x9aa8834fUL, 0x97a38d46UL +}; + +static const ulong32 Tks3[] = { +0x00000000UL, 0x090d0b0eUL, 0x121a161cUL, 0x1b171d12UL, 0x24342c38UL, 0x2d392736UL, 0x362e3a24UL, 0x3f23312aUL, +0x48685870UL, 0x4165537eUL, 0x5a724e6cUL, 0x537f4562UL, 0x6c5c7448UL, 0x65517f46UL, 0x7e466254UL, 0x774b695aUL, +0x90d0b0e0UL, 0x99ddbbeeUL, 0x82caa6fcUL, 0x8bc7adf2UL, 0xb4e49cd8UL, 0xbde997d6UL, 0xa6fe8ac4UL, 0xaff381caUL, +0xd8b8e890UL, 0xd1b5e39eUL, 0xcaa2fe8cUL, 0xc3aff582UL, 0xfc8cc4a8UL, 0xf581cfa6UL, 0xee96d2b4UL, 0xe79bd9baUL, +0x3bbb7bdbUL, 0x32b670d5UL, 0x29a16dc7UL, 0x20ac66c9UL, 0x1f8f57e3UL, 0x16825cedUL, 0x0d9541ffUL, 0x04984af1UL, +0x73d323abUL, 0x7ade28a5UL, 0x61c935b7UL, 0x68c43eb9UL, 0x57e70f93UL, 0x5eea049dUL, 0x45fd198fUL, 0x4cf01281UL, +0xab6bcb3bUL, 0xa266c035UL, 0xb971dd27UL, 0xb07cd629UL, 0x8f5fe703UL, 0x8652ec0dUL, 0x9d45f11fUL, 0x9448fa11UL, +0xe303934bUL, 0xea0e9845UL, 0xf1198557UL, 0xf8148e59UL, 0xc737bf73UL, 0xce3ab47dUL, 0xd52da96fUL, 0xdc20a261UL, +0x766df6adUL, 0x7f60fda3UL, 0x6477e0b1UL, 0x6d7aebbfUL, 0x5259da95UL, 0x5b54d19bUL, 0x4043cc89UL, 0x494ec787UL, +0x3e05aeddUL, 0x3708a5d3UL, 0x2c1fb8c1UL, 0x2512b3cfUL, 0x1a3182e5UL, 0x133c89ebUL, 0x082b94f9UL, 0x01269ff7UL, +0xe6bd464dUL, 0xefb04d43UL, 0xf4a75051UL, 0xfdaa5b5fUL, 0xc2896a75UL, 0xcb84617bUL, 0xd0937c69UL, 0xd99e7767UL, +0xaed51e3dUL, 0xa7d81533UL, 0xbccf0821UL, 0xb5c2032fUL, 0x8ae13205UL, 0x83ec390bUL, 0x98fb2419UL, 0x91f62f17UL, +0x4dd68d76UL, 0x44db8678UL, 0x5fcc9b6aUL, 0x56c19064UL, 0x69e2a14eUL, 0x60efaa40UL, 0x7bf8b752UL, 0x72f5bc5cUL, +0x05bed506UL, 0x0cb3de08UL, 0x17a4c31aUL, 0x1ea9c814UL, 0x218af93eUL, 0x2887f230UL, 0x3390ef22UL, 0x3a9de42cUL, +0xdd063d96UL, 0xd40b3698UL, 0xcf1c2b8aUL, 0xc6112084UL, 0xf93211aeUL, 0xf03f1aa0UL, 0xeb2807b2UL, 0xe2250cbcUL, +0x956e65e6UL, 0x9c636ee8UL, 0x877473faUL, 0x8e7978f4UL, 0xb15a49deUL, 0xb85742d0UL, 0xa3405fc2UL, 0xaa4d54ccUL, +0xecdaf741UL, 0xe5d7fc4fUL, 0xfec0e15dUL, 0xf7cdea53UL, 0xc8eedb79UL, 0xc1e3d077UL, 0xdaf4cd65UL, 0xd3f9c66bUL, +0xa4b2af31UL, 0xadbfa43fUL, 0xb6a8b92dUL, 0xbfa5b223UL, 0x80868309UL, 0x898b8807UL, 0x929c9515UL, 0x9b919e1bUL, +0x7c0a47a1UL, 0x75074cafUL, 0x6e1051bdUL, 0x671d5ab3UL, 0x583e6b99UL, 0x51336097UL, 0x4a247d85UL, 0x4329768bUL, +0x34621fd1UL, 0x3d6f14dfUL, 0x267809cdUL, 0x2f7502c3UL, 0x105633e9UL, 0x195b38e7UL, 0x024c25f5UL, 0x0b412efbUL, +0xd7618c9aUL, 0xde6c8794UL, 0xc57b9a86UL, 0xcc769188UL, 0xf355a0a2UL, 0xfa58abacUL, 0xe14fb6beUL, 0xe842bdb0UL, +0x9f09d4eaUL, 0x9604dfe4UL, 0x8d13c2f6UL, 0x841ec9f8UL, 0xbb3df8d2UL, 0xb230f3dcUL, 0xa927eeceUL, 0xa02ae5c0UL, +0x47b13c7aUL, 0x4ebc3774UL, 0x55ab2a66UL, 0x5ca62168UL, 0x63851042UL, 0x6a881b4cUL, 0x719f065eUL, 0x78920d50UL, +0x0fd9640aUL, 0x06d46f04UL, 0x1dc37216UL, 0x14ce7918UL, 0x2bed4832UL, 0x22e0433cUL, 0x39f75e2eUL, 0x30fa5520UL, +0x9ab701ecUL, 0x93ba0ae2UL, 0x88ad17f0UL, 0x81a01cfeUL, 0xbe832dd4UL, 0xb78e26daUL, 0xac993bc8UL, 0xa59430c6UL, +0xd2df599cUL, 0xdbd25292UL, 0xc0c54f80UL, 0xc9c8448eUL, 0xf6eb75a4UL, 0xffe67eaaUL, 0xe4f163b8UL, 0xedfc68b6UL, +0x0a67b10cUL, 0x036aba02UL, 0x187da710UL, 0x1170ac1eUL, 0x2e539d34UL, 0x275e963aUL, 0x3c498b28UL, 0x35448026UL, +0x420fe97cUL, 0x4b02e272UL, 0x5015ff60UL, 0x5918f46eUL, 0x663bc544UL, 0x6f36ce4aUL, 0x7421d358UL, 0x7d2cd856UL, +0xa10c7a37UL, 0xa8017139UL, 0xb3166c2bUL, 0xba1b6725UL, 0x8538560fUL, 0x8c355d01UL, 0x97224013UL, 0x9e2f4b1dUL, +0xe9642247UL, 0xe0692949UL, 0xfb7e345bUL, 0xf2733f55UL, 0xcd500e7fUL, 0xc45d0571UL, 0xdf4a1863UL, 0xd647136dUL, +0x31dccad7UL, 0x38d1c1d9UL, 0x23c6dccbUL, 0x2acbd7c5UL, 0x15e8e6efUL, 0x1ce5ede1UL, 0x07f2f0f3UL, 0x0efffbfdUL, +0x79b492a7UL, 0x70b999a9UL, 0x6bae84bbUL, 0x62a38fb5UL, 0x5d80be9fUL, 0x548db591UL, 0x4f9aa883UL, 0x4697a38dUL +}; + +#endif /* ENCRYPT_ONLY */ + +#endif /* SMALL CODE */ + +#ifndef PELI_TAB +static const ulong32 rcon[] = { + 0x01000000UL, 0x02000000UL, 0x04000000UL, 0x08000000UL, + 0x10000000UL, 0x20000000UL, 0x40000000UL, 0x80000000UL, + 0x1B000000UL, 0x36000000UL, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ +}; +#endif + +#endif /* LTC_AES_TAB_C */ diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/aes/sub.mk b/optee_os/core/lib/libtomcrypt/src/ciphers/aes/sub.mk new file mode 100644 index 0000000..ac7b507 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/aes/sub.mk @@ -0,0 +1,5 @@ +cflags-y += -Wno-unused-parameter + +ifneq ($(_CFG_CORE_LTC_AES_ACCEL),y) +srcs-$(_CFG_CORE_LTC_AES_DESC) += aes.c +endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/anubis.c b/optee_os/core/lib/libtomcrypt/src/ciphers/anubis.c new file mode 100644 index 0000000..f6a1c58 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/anubis.c @@ -0,0 +1,1550 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file anubis.c + Anubis implementation derived from public domain source + Authors: Paulo S.L.M. Barreto and Vincent Rijmen. +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_ANUBIS + +const struct ltc_cipher_descriptor anubis_desc = { + "anubis", + 19, + 16, 40, 16, 12, + &anubis_setup, + &anubis_ecb_encrypt, + &anubis_ecb_decrypt, + &anubis_test, + &anubis_done, + &anubis_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +#define MAX_N 10 + +/* + * Though Anubis is endianness-neutral, the encryption tables are listed + * in BIG-ENDIAN format, which is adopted throughout this implementation + * (but little-endian notation would be equally suitable if consistently + * employed). + */ +#if defined(LTC_ANUBIS_TWEAK) + +static const ulong32 T0[256] = { + 0xba69d2bbU, 0x54a84de5U, 0x2f5ebce2U, 0x74e8cd25U, + 0x53a651f7U, 0xd3bb6bd0U, 0xd2b96fd6U, 0x4d9a29b3U, + 0x50a05dfdU, 0xac458acfU, 0x8d070e09U, 0xbf63c6a5U, + 0x70e0dd3dU, 0x52a455f1U, 0x9a29527bU, 0x4c982db5U, + 0xeac98f46U, 0xd5b773c4U, 0x97336655U, 0xd1bf63dcU, + 0x3366ccaaU, 0x51a259fbU, 0x5bb671c7U, 0xa651a2f3U, + 0xdea15ffeU, 0x48903dadU, 0xa84d9ad7U, 0x992f5e71U, + 0xdbab4be0U, 0x3264c8acU, 0xb773e695U, 0xfce5d732U, + 0xe3dbab70U, 0x9e214263U, 0x913f7e41U, 0x9b2b567dU, + 0xe2d9af76U, 0xbb6bd6bdU, 0x4182199bU, 0x6edca579U, + 0xa557aef9U, 0xcb8b0b80U, 0x6bd6b167U, 0x95376e59U, + 0xa15fbee1U, 0xf3fbeb10U, 0xb17ffe81U, 0x0204080cU, + 0xcc851792U, 0xc49537a2U, 0x1d3a744eU, 0x14285078U, + 0xc39b2bb0U, 0x63c69157U, 0xdaa94fe6U, 0x5dba69d3U, + 0x5fbe61dfU, 0xdca557f2U, 0x7dfae913U, 0xcd871394U, + 0x7ffee11fU, 0x5ab475c1U, 0x6cd8ad75U, 0x5cb86dd5U, + 0xf7f3fb08U, 0x264c98d4U, 0xffe3db38U, 0xedc79354U, + 0xe8cd874aU, 0x9d274e69U, 0x6fdea17fU, 0x8e010203U, + 0x19326456U, 0xa05dbae7U, 0xf0fde71aU, 0x890f1e11U, + 0x0f1e3c22U, 0x070e1c12U, 0xaf4386c5U, 0xfbebcb20U, + 0x08102030U, 0x152a547eU, 0x0d1a342eU, 0x04081018U, + 0x01020406U, 0x64c88d45U, 0xdfa35bf8U, 0x76ecc529U, + 0x79f2f90bU, 0xdda753f4U, 0x3d7af48eU, 0x162c5874U, + 0x3f7efc82U, 0x376edcb2U, 0x6ddaa973U, 0x3870e090U, + 0xb96fdeb1U, 0x73e6d137U, 0xe9cf834cU, 0x356ad4beU, + 0x55aa49e3U, 0x71e2d93bU, 0x7bf6f107U, 0x8c050a0fU, + 0x72e4d531U, 0x880d1a17U, 0xf6f1ff0eU, 0x2a54a8fcU, + 0x3e7cf884U, 0x5ebc65d9U, 0x274e9cd2U, 0x468c0589U, + 0x0c183028U, 0x65ca8943U, 0x68d0bd6dU, 0x61c2995bU, + 0x03060c0aU, 0xc19f23bcU, 0x57ae41efU, 0xd6b17fceU, + 0xd9af43ecU, 0x58b07dcdU, 0xd8ad47eaU, 0x66cc8549U, + 0xd7b37bc8U, 0x3a74e89cU, 0xc88d078aU, 0x3c78f088U, + 0xfae9cf26U, 0x96316253U, 0xa753a6f5U, 0x982d5a77U, + 0xecc59752U, 0xb86ddab7U, 0xc7933ba8U, 0xae4182c3U, + 0x69d2b96bU, 0x4b9631a7U, 0xab4b96ddU, 0xa94f9ed1U, + 0x67ce814fU, 0x0a14283cU, 0x478e018fU, 0xf2f9ef16U, + 0xb577ee99U, 0x224488ccU, 0xe5d7b364U, 0xeec19f5eU, + 0xbe61c2a3U, 0x2b56acfaU, 0x811f3e21U, 0x1224486cU, + 0x831b362dU, 0x1b366c5aU, 0x0e1c3824U, 0x23468ccaU, + 0xf5f7f304U, 0x458a0983U, 0x214284c6U, 0xce811f9eU, + 0x499239abU, 0x2c58b0e8U, 0xf9efc32cU, 0xe6d1bf6eU, + 0xb671e293U, 0x2850a0f0U, 0x172e5c72U, 0x8219322bU, + 0x1a34685cU, 0x8b0b161dU, 0xfee1df3eU, 0x8a09121bU, + 0x09122436U, 0xc98f038cU, 0x87132635U, 0x4e9c25b9U, + 0xe1dfa37cU, 0x2e5cb8e4U, 0xe4d5b762U, 0xe0dda77aU, + 0xebcb8b40U, 0x903d7a47U, 0xa455aaffU, 0x1e3c7844U, + 0x85172e39U, 0x60c09d5dU, 0x00000000U, 0x254a94deU, + 0xf4f5f702U, 0xf1ffe31cU, 0x94356a5fU, 0x0b162c3aU, + 0xe7d3bb68U, 0x75eac923U, 0xefc39b58U, 0x3468d0b8U, + 0x3162c4a6U, 0xd4b577c2U, 0xd0bd67daU, 0x86112233U, + 0x7efce519U, 0xad478ec9U, 0xfde7d334U, 0x2952a4f6U, + 0x3060c0a0U, 0x3b76ec9aU, 0x9f234665U, 0xf8edc72aU, + 0xc6913faeU, 0x13264c6aU, 0x060c1814U, 0x050a141eU, + 0xc59733a4U, 0x11224466U, 0x77eec12fU, 0x7cf8ed15U, + 0x7af4f501U, 0x78f0fd0dU, 0x366cd8b4U, 0x1c387048U, + 0x3972e496U, 0x59b279cbU, 0x18306050U, 0x56ac45e9U, + 0xb37bf68dU, 0xb07dfa87U, 0x244890d8U, 0x204080c0U, + 0xb279f28bU, 0x9239724bU, 0xa35bb6edU, 0xc09d27baU, + 0x44880d85U, 0x62c49551U, 0x10204060U, 0xb475ea9fU, + 0x84152a3fU, 0x43861197U, 0x933b764dU, 0xc2992fb6U, + 0x4a9435a1U, 0xbd67cea9U, 0x8f030605U, 0x2d5ab4eeU, + 0xbc65caafU, 0x9c254a6fU, 0x6ad4b561U, 0x40801d9dU, + 0xcf831b98U, 0xa259b2ebU, 0x801d3a27U, 0x4f9e21bfU, + 0x1f3e7c42U, 0xca890f86U, 0xaa4992dbU, 0x42841591U, +}; + +static const ulong32 T1[256] = { + 0x69babbd2U, 0xa854e54dU, 0x5e2fe2bcU, 0xe87425cdU, + 0xa653f751U, 0xbbd3d06bU, 0xb9d2d66fU, 0x9a4db329U, + 0xa050fd5dU, 0x45accf8aU, 0x078d090eU, 0x63bfa5c6U, + 0xe0703dddU, 0xa452f155U, 0x299a7b52U, 0x984cb52dU, + 0xc9ea468fU, 0xb7d5c473U, 0x33975566U, 0xbfd1dc63U, + 0x6633aaccU, 0xa251fb59U, 0xb65bc771U, 0x51a6f3a2U, + 0xa1defe5fU, 0x9048ad3dU, 0x4da8d79aU, 0x2f99715eU, + 0xabdbe04bU, 0x6432acc8U, 0x73b795e6U, 0xe5fc32d7U, + 0xdbe370abU, 0x219e6342U, 0x3f91417eU, 0x2b9b7d56U, + 0xd9e276afU, 0x6bbbbdd6U, 0x82419b19U, 0xdc6e79a5U, + 0x57a5f9aeU, 0x8bcb800bU, 0xd66b67b1U, 0x3795596eU, + 0x5fa1e1beU, 0xfbf310ebU, 0x7fb181feU, 0x04020c08U, + 0x85cc9217U, 0x95c4a237U, 0x3a1d4e74U, 0x28147850U, + 0x9bc3b02bU, 0xc6635791U, 0xa9dae64fU, 0xba5dd369U, + 0xbe5fdf61U, 0xa5dcf257U, 0xfa7d13e9U, 0x87cd9413U, + 0xfe7f1fe1U, 0xb45ac175U, 0xd86c75adU, 0xb85cd56dU, + 0xf3f708fbU, 0x4c26d498U, 0xe3ff38dbU, 0xc7ed5493U, + 0xcde84a87U, 0x279d694eU, 0xde6f7fa1U, 0x018e0302U, + 0x32195664U, 0x5da0e7baU, 0xfdf01ae7U, 0x0f89111eU, + 0x1e0f223cU, 0x0e07121cU, 0x43afc586U, 0xebfb20cbU, + 0x10083020U, 0x2a157e54U, 0x1a0d2e34U, 0x08041810U, + 0x02010604U, 0xc864458dU, 0xa3dff85bU, 0xec7629c5U, + 0xf2790bf9U, 0xa7ddf453U, 0x7a3d8ef4U, 0x2c167458U, + 0x7e3f82fcU, 0x6e37b2dcU, 0xda6d73a9U, 0x703890e0U, + 0x6fb9b1deU, 0xe67337d1U, 0xcfe94c83U, 0x6a35bed4U, + 0xaa55e349U, 0xe2713bd9U, 0xf67b07f1U, 0x058c0f0aU, + 0xe47231d5U, 0x0d88171aU, 0xf1f60effU, 0x542afca8U, + 0x7c3e84f8U, 0xbc5ed965U, 0x4e27d29cU, 0x8c468905U, + 0x180c2830U, 0xca654389U, 0xd0686dbdU, 0xc2615b99U, + 0x06030a0cU, 0x9fc1bc23U, 0xae57ef41U, 0xb1d6ce7fU, + 0xafd9ec43U, 0xb058cd7dU, 0xadd8ea47U, 0xcc664985U, + 0xb3d7c87bU, 0x743a9ce8U, 0x8dc88a07U, 0x783c88f0U, + 0xe9fa26cfU, 0x31965362U, 0x53a7f5a6U, 0x2d98775aU, + 0xc5ec5297U, 0x6db8b7daU, 0x93c7a83bU, 0x41aec382U, + 0xd2696bb9U, 0x964ba731U, 0x4babdd96U, 0x4fa9d19eU, + 0xce674f81U, 0x140a3c28U, 0x8e478f01U, 0xf9f216efU, + 0x77b599eeU, 0x4422cc88U, 0xd7e564b3U, 0xc1ee5e9fU, + 0x61bea3c2U, 0x562bfaacU, 0x1f81213eU, 0x24126c48U, + 0x1b832d36U, 0x361b5a6cU, 0x1c0e2438U, 0x4623ca8cU, + 0xf7f504f3U, 0x8a458309U, 0x4221c684U, 0x81ce9e1fU, + 0x9249ab39U, 0x582ce8b0U, 0xeff92cc3U, 0xd1e66ebfU, + 0x71b693e2U, 0x5028f0a0U, 0x2e17725cU, 0x19822b32U, + 0x341a5c68U, 0x0b8b1d16U, 0xe1fe3edfU, 0x098a1b12U, + 0x12093624U, 0x8fc98c03U, 0x13873526U, 0x9c4eb925U, + 0xdfe17ca3U, 0x5c2ee4b8U, 0xd5e462b7U, 0xdde07aa7U, + 0xcbeb408bU, 0x3d90477aU, 0x55a4ffaaU, 0x3c1e4478U, + 0x1785392eU, 0xc0605d9dU, 0x00000000U, 0x4a25de94U, + 0xf5f402f7U, 0xfff11ce3U, 0x35945f6aU, 0x160b3a2cU, + 0xd3e768bbU, 0xea7523c9U, 0xc3ef589bU, 0x6834b8d0U, + 0x6231a6c4U, 0xb5d4c277U, 0xbdd0da67U, 0x11863322U, + 0xfc7e19e5U, 0x47adc98eU, 0xe7fd34d3U, 0x5229f6a4U, + 0x6030a0c0U, 0x763b9aecU, 0x239f6546U, 0xedf82ac7U, + 0x91c6ae3fU, 0x26136a4cU, 0x0c061418U, 0x0a051e14U, + 0x97c5a433U, 0x22116644U, 0xee772fc1U, 0xf87c15edU, + 0xf47a01f5U, 0xf0780dfdU, 0x6c36b4d8U, 0x381c4870U, + 0x723996e4U, 0xb259cb79U, 0x30185060U, 0xac56e945U, + 0x7bb38df6U, 0x7db087faU, 0x4824d890U, 0x4020c080U, + 0x79b28bf2U, 0x39924b72U, 0x5ba3edb6U, 0x9dc0ba27U, + 0x8844850dU, 0xc4625195U, 0x20106040U, 0x75b49feaU, + 0x15843f2aU, 0x86439711U, 0x3b934d76U, 0x99c2b62fU, + 0x944aa135U, 0x67bda9ceU, 0x038f0506U, 0x5a2deeb4U, + 0x65bcafcaU, 0x259c6f4aU, 0xd46a61b5U, 0x80409d1dU, + 0x83cf981bU, 0x59a2ebb2U, 0x1d80273aU, 0x9e4fbf21U, + 0x3e1f427cU, 0x89ca860fU, 0x49aadb92U, 0x84429115U, +}; + +static const ulong32 T2[256] = { + 0xd2bbba69U, 0x4de554a8U, 0xbce22f5eU, 0xcd2574e8U, + 0x51f753a6U, 0x6bd0d3bbU, 0x6fd6d2b9U, 0x29b34d9aU, + 0x5dfd50a0U, 0x8acfac45U, 0x0e098d07U, 0xc6a5bf63U, + 0xdd3d70e0U, 0x55f152a4U, 0x527b9a29U, 0x2db54c98U, + 0x8f46eac9U, 0x73c4d5b7U, 0x66559733U, 0x63dcd1bfU, + 0xccaa3366U, 0x59fb51a2U, 0x71c75bb6U, 0xa2f3a651U, + 0x5ffedea1U, 0x3dad4890U, 0x9ad7a84dU, 0x5e71992fU, + 0x4be0dbabU, 0xc8ac3264U, 0xe695b773U, 0xd732fce5U, + 0xab70e3dbU, 0x42639e21U, 0x7e41913fU, 0x567d9b2bU, + 0xaf76e2d9U, 0xd6bdbb6bU, 0x199b4182U, 0xa5796edcU, + 0xaef9a557U, 0x0b80cb8bU, 0xb1676bd6U, 0x6e599537U, + 0xbee1a15fU, 0xeb10f3fbU, 0xfe81b17fU, 0x080c0204U, + 0x1792cc85U, 0x37a2c495U, 0x744e1d3aU, 0x50781428U, + 0x2bb0c39bU, 0x915763c6U, 0x4fe6daa9U, 0x69d35dbaU, + 0x61df5fbeU, 0x57f2dca5U, 0xe9137dfaU, 0x1394cd87U, + 0xe11f7ffeU, 0x75c15ab4U, 0xad756cd8U, 0x6dd55cb8U, + 0xfb08f7f3U, 0x98d4264cU, 0xdb38ffe3U, 0x9354edc7U, + 0x874ae8cdU, 0x4e699d27U, 0xa17f6fdeU, 0x02038e01U, + 0x64561932U, 0xbae7a05dU, 0xe71af0fdU, 0x1e11890fU, + 0x3c220f1eU, 0x1c12070eU, 0x86c5af43U, 0xcb20fbebU, + 0x20300810U, 0x547e152aU, 0x342e0d1aU, 0x10180408U, + 0x04060102U, 0x8d4564c8U, 0x5bf8dfa3U, 0xc52976ecU, + 0xf90b79f2U, 0x53f4dda7U, 0xf48e3d7aU, 0x5874162cU, + 0xfc823f7eU, 0xdcb2376eU, 0xa9736ddaU, 0xe0903870U, + 0xdeb1b96fU, 0xd13773e6U, 0x834ce9cfU, 0xd4be356aU, + 0x49e355aaU, 0xd93b71e2U, 0xf1077bf6U, 0x0a0f8c05U, + 0xd53172e4U, 0x1a17880dU, 0xff0ef6f1U, 0xa8fc2a54U, + 0xf8843e7cU, 0x65d95ebcU, 0x9cd2274eU, 0x0589468cU, + 0x30280c18U, 0x894365caU, 0xbd6d68d0U, 0x995b61c2U, + 0x0c0a0306U, 0x23bcc19fU, 0x41ef57aeU, 0x7fced6b1U, + 0x43ecd9afU, 0x7dcd58b0U, 0x47ead8adU, 0x854966ccU, + 0x7bc8d7b3U, 0xe89c3a74U, 0x078ac88dU, 0xf0883c78U, + 0xcf26fae9U, 0x62539631U, 0xa6f5a753U, 0x5a77982dU, + 0x9752ecc5U, 0xdab7b86dU, 0x3ba8c793U, 0x82c3ae41U, + 0xb96b69d2U, 0x31a74b96U, 0x96ddab4bU, 0x9ed1a94fU, + 0x814f67ceU, 0x283c0a14U, 0x018f478eU, 0xef16f2f9U, + 0xee99b577U, 0x88cc2244U, 0xb364e5d7U, 0x9f5eeec1U, + 0xc2a3be61U, 0xacfa2b56U, 0x3e21811fU, 0x486c1224U, + 0x362d831bU, 0x6c5a1b36U, 0x38240e1cU, 0x8cca2346U, + 0xf304f5f7U, 0x0983458aU, 0x84c62142U, 0x1f9ece81U, + 0x39ab4992U, 0xb0e82c58U, 0xc32cf9efU, 0xbf6ee6d1U, + 0xe293b671U, 0xa0f02850U, 0x5c72172eU, 0x322b8219U, + 0x685c1a34U, 0x161d8b0bU, 0xdf3efee1U, 0x121b8a09U, + 0x24360912U, 0x038cc98fU, 0x26358713U, 0x25b94e9cU, + 0xa37ce1dfU, 0xb8e42e5cU, 0xb762e4d5U, 0xa77ae0ddU, + 0x8b40ebcbU, 0x7a47903dU, 0xaaffa455U, 0x78441e3cU, + 0x2e398517U, 0x9d5d60c0U, 0x00000000U, 0x94de254aU, + 0xf702f4f5U, 0xe31cf1ffU, 0x6a5f9435U, 0x2c3a0b16U, + 0xbb68e7d3U, 0xc92375eaU, 0x9b58efc3U, 0xd0b83468U, + 0xc4a63162U, 0x77c2d4b5U, 0x67dad0bdU, 0x22338611U, + 0xe5197efcU, 0x8ec9ad47U, 0xd334fde7U, 0xa4f62952U, + 0xc0a03060U, 0xec9a3b76U, 0x46659f23U, 0xc72af8edU, + 0x3faec691U, 0x4c6a1326U, 0x1814060cU, 0x141e050aU, + 0x33a4c597U, 0x44661122U, 0xc12f77eeU, 0xed157cf8U, + 0xf5017af4U, 0xfd0d78f0U, 0xd8b4366cU, 0x70481c38U, + 0xe4963972U, 0x79cb59b2U, 0x60501830U, 0x45e956acU, + 0xf68db37bU, 0xfa87b07dU, 0x90d82448U, 0x80c02040U, + 0xf28bb279U, 0x724b9239U, 0xb6eda35bU, 0x27bac09dU, + 0x0d854488U, 0x955162c4U, 0x40601020U, 0xea9fb475U, + 0x2a3f8415U, 0x11974386U, 0x764d933bU, 0x2fb6c299U, + 0x35a14a94U, 0xcea9bd67U, 0x06058f03U, 0xb4ee2d5aU, + 0xcaafbc65U, 0x4a6f9c25U, 0xb5616ad4U, 0x1d9d4080U, + 0x1b98cf83U, 0xb2eba259U, 0x3a27801dU, 0x21bf4f9eU, + 0x7c421f3eU, 0x0f86ca89U, 0x92dbaa49U, 0x15914284U, +}; + +static const ulong32 T3[256] = { + 0xbbd269baU, 0xe54da854U, 0xe2bc5e2fU, 0x25cde874U, + 0xf751a653U, 0xd06bbbd3U, 0xd66fb9d2U, 0xb3299a4dU, + 0xfd5da050U, 0xcf8a45acU, 0x090e078dU, 0xa5c663bfU, + 0x3ddde070U, 0xf155a452U, 0x7b52299aU, 0xb52d984cU, + 0x468fc9eaU, 0xc473b7d5U, 0x55663397U, 0xdc63bfd1U, + 0xaacc6633U, 0xfb59a251U, 0xc771b65bU, 0xf3a251a6U, + 0xfe5fa1deU, 0xad3d9048U, 0xd79a4da8U, 0x715e2f99U, + 0xe04babdbU, 0xacc86432U, 0x95e673b7U, 0x32d7e5fcU, + 0x70abdbe3U, 0x6342219eU, 0x417e3f91U, 0x7d562b9bU, + 0x76afd9e2U, 0xbdd66bbbU, 0x9b198241U, 0x79a5dc6eU, + 0xf9ae57a5U, 0x800b8bcbU, 0x67b1d66bU, 0x596e3795U, + 0xe1be5fa1U, 0x10ebfbf3U, 0x81fe7fb1U, 0x0c080402U, + 0x921785ccU, 0xa23795c4U, 0x4e743a1dU, 0x78502814U, + 0xb02b9bc3U, 0x5791c663U, 0xe64fa9daU, 0xd369ba5dU, + 0xdf61be5fU, 0xf257a5dcU, 0x13e9fa7dU, 0x941387cdU, + 0x1fe1fe7fU, 0xc175b45aU, 0x75add86cU, 0xd56db85cU, + 0x08fbf3f7U, 0xd4984c26U, 0x38dbe3ffU, 0x5493c7edU, + 0x4a87cde8U, 0x694e279dU, 0x7fa1de6fU, 0x0302018eU, + 0x56643219U, 0xe7ba5da0U, 0x1ae7fdf0U, 0x111e0f89U, + 0x223c1e0fU, 0x121c0e07U, 0xc58643afU, 0x20cbebfbU, + 0x30201008U, 0x7e542a15U, 0x2e341a0dU, 0x18100804U, + 0x06040201U, 0x458dc864U, 0xf85ba3dfU, 0x29c5ec76U, + 0x0bf9f279U, 0xf453a7ddU, 0x8ef47a3dU, 0x74582c16U, + 0x82fc7e3fU, 0xb2dc6e37U, 0x73a9da6dU, 0x90e07038U, + 0xb1de6fb9U, 0x37d1e673U, 0x4c83cfe9U, 0xbed46a35U, + 0xe349aa55U, 0x3bd9e271U, 0x07f1f67bU, 0x0f0a058cU, + 0x31d5e472U, 0x171a0d88U, 0x0efff1f6U, 0xfca8542aU, + 0x84f87c3eU, 0xd965bc5eU, 0xd29c4e27U, 0x89058c46U, + 0x2830180cU, 0x4389ca65U, 0x6dbdd068U, 0x5b99c261U, + 0x0a0c0603U, 0xbc239fc1U, 0xef41ae57U, 0xce7fb1d6U, + 0xec43afd9U, 0xcd7db058U, 0xea47add8U, 0x4985cc66U, + 0xc87bb3d7U, 0x9ce8743aU, 0x8a078dc8U, 0x88f0783cU, + 0x26cfe9faU, 0x53623196U, 0xf5a653a7U, 0x775a2d98U, + 0x5297c5ecU, 0xb7da6db8U, 0xa83b93c7U, 0xc38241aeU, + 0x6bb9d269U, 0xa731964bU, 0xdd964babU, 0xd19e4fa9U, + 0x4f81ce67U, 0x3c28140aU, 0x8f018e47U, 0x16eff9f2U, + 0x99ee77b5U, 0xcc884422U, 0x64b3d7e5U, 0x5e9fc1eeU, + 0xa3c261beU, 0xfaac562bU, 0x213e1f81U, 0x6c482412U, + 0x2d361b83U, 0x5a6c361bU, 0x24381c0eU, 0xca8c4623U, + 0x04f3f7f5U, 0x83098a45U, 0xc6844221U, 0x9e1f81ceU, + 0xab399249U, 0xe8b0582cU, 0x2cc3eff9U, 0x6ebfd1e6U, + 0x93e271b6U, 0xf0a05028U, 0x725c2e17U, 0x2b321982U, + 0x5c68341aU, 0x1d160b8bU, 0x3edfe1feU, 0x1b12098aU, + 0x36241209U, 0x8c038fc9U, 0x35261387U, 0xb9259c4eU, + 0x7ca3dfe1U, 0xe4b85c2eU, 0x62b7d5e4U, 0x7aa7dde0U, + 0x408bcbebU, 0x477a3d90U, 0xffaa55a4U, 0x44783c1eU, + 0x392e1785U, 0x5d9dc060U, 0x00000000U, 0xde944a25U, + 0x02f7f5f4U, 0x1ce3fff1U, 0x5f6a3594U, 0x3a2c160bU, + 0x68bbd3e7U, 0x23c9ea75U, 0x589bc3efU, 0xb8d06834U, + 0xa6c46231U, 0xc277b5d4U, 0xda67bdd0U, 0x33221186U, + 0x19e5fc7eU, 0xc98e47adU, 0x34d3e7fdU, 0xf6a45229U, + 0xa0c06030U, 0x9aec763bU, 0x6546239fU, 0x2ac7edf8U, + 0xae3f91c6U, 0x6a4c2613U, 0x14180c06U, 0x1e140a05U, + 0xa43397c5U, 0x66442211U, 0x2fc1ee77U, 0x15edf87cU, + 0x01f5f47aU, 0x0dfdf078U, 0xb4d86c36U, 0x4870381cU, + 0x96e47239U, 0xcb79b259U, 0x50603018U, 0xe945ac56U, + 0x8df67bb3U, 0x87fa7db0U, 0xd8904824U, 0xc0804020U, + 0x8bf279b2U, 0x4b723992U, 0xedb65ba3U, 0xba279dc0U, + 0x850d8844U, 0x5195c462U, 0x60402010U, 0x9fea75b4U, + 0x3f2a1584U, 0x97118643U, 0x4d763b93U, 0xb62f99c2U, + 0xa135944aU, 0xa9ce67bdU, 0x0506038fU, 0xeeb45a2dU, + 0xafca65bcU, 0x6f4a259cU, 0x61b5d46aU, 0x9d1d8040U, + 0x981b83cfU, 0xebb259a2U, 0x273a1d80U, 0xbf219e4fU, + 0x427c3e1fU, 0x860f89caU, 0xdb9249aaU, 0x91158442U, +}; + +static const ulong32 T4[256] = { + 0xbabababaU, 0x54545454U, 0x2f2f2f2fU, 0x74747474U, + 0x53535353U, 0xd3d3d3d3U, 0xd2d2d2d2U, 0x4d4d4d4dU, + 0x50505050U, 0xacacacacU, 0x8d8d8d8dU, 0xbfbfbfbfU, + 0x70707070U, 0x52525252U, 0x9a9a9a9aU, 0x4c4c4c4cU, + 0xeaeaeaeaU, 0xd5d5d5d5U, 0x97979797U, 0xd1d1d1d1U, + 0x33333333U, 0x51515151U, 0x5b5b5b5bU, 0xa6a6a6a6U, + 0xdedededeU, 0x48484848U, 0xa8a8a8a8U, 0x99999999U, + 0xdbdbdbdbU, 0x32323232U, 0xb7b7b7b7U, 0xfcfcfcfcU, + 0xe3e3e3e3U, 0x9e9e9e9eU, 0x91919191U, 0x9b9b9b9bU, + 0xe2e2e2e2U, 0xbbbbbbbbU, 0x41414141U, 0x6e6e6e6eU, + 0xa5a5a5a5U, 0xcbcbcbcbU, 0x6b6b6b6bU, 0x95959595U, + 0xa1a1a1a1U, 0xf3f3f3f3U, 0xb1b1b1b1U, 0x02020202U, + 0xccccccccU, 0xc4c4c4c4U, 0x1d1d1d1dU, 0x14141414U, + 0xc3c3c3c3U, 0x63636363U, 0xdadadadaU, 0x5d5d5d5dU, + 0x5f5f5f5fU, 0xdcdcdcdcU, 0x7d7d7d7dU, 0xcdcdcdcdU, + 0x7f7f7f7fU, 0x5a5a5a5aU, 0x6c6c6c6cU, 0x5c5c5c5cU, + 0xf7f7f7f7U, 0x26262626U, 0xffffffffU, 0xededededU, + 0xe8e8e8e8U, 0x9d9d9d9dU, 0x6f6f6f6fU, 0x8e8e8e8eU, + 0x19191919U, 0xa0a0a0a0U, 0xf0f0f0f0U, 0x89898989U, + 0x0f0f0f0fU, 0x07070707U, 0xafafafafU, 0xfbfbfbfbU, + 0x08080808U, 0x15151515U, 0x0d0d0d0dU, 0x04040404U, + 0x01010101U, 0x64646464U, 0xdfdfdfdfU, 0x76767676U, + 0x79797979U, 0xddddddddU, 0x3d3d3d3dU, 0x16161616U, + 0x3f3f3f3fU, 0x37373737U, 0x6d6d6d6dU, 0x38383838U, + 0xb9b9b9b9U, 0x73737373U, 0xe9e9e9e9U, 0x35353535U, + 0x55555555U, 0x71717171U, 0x7b7b7b7bU, 0x8c8c8c8cU, + 0x72727272U, 0x88888888U, 0xf6f6f6f6U, 0x2a2a2a2aU, + 0x3e3e3e3eU, 0x5e5e5e5eU, 0x27272727U, 0x46464646U, + 0x0c0c0c0cU, 0x65656565U, 0x68686868U, 0x61616161U, + 0x03030303U, 0xc1c1c1c1U, 0x57575757U, 0xd6d6d6d6U, + 0xd9d9d9d9U, 0x58585858U, 0xd8d8d8d8U, 0x66666666U, + 0xd7d7d7d7U, 0x3a3a3a3aU, 0xc8c8c8c8U, 0x3c3c3c3cU, + 0xfafafafaU, 0x96969696U, 0xa7a7a7a7U, 0x98989898U, + 0xececececU, 0xb8b8b8b8U, 0xc7c7c7c7U, 0xaeaeaeaeU, + 0x69696969U, 0x4b4b4b4bU, 0xababababU, 0xa9a9a9a9U, + 0x67676767U, 0x0a0a0a0aU, 0x47474747U, 0xf2f2f2f2U, + 0xb5b5b5b5U, 0x22222222U, 0xe5e5e5e5U, 0xeeeeeeeeU, + 0xbebebebeU, 0x2b2b2b2bU, 0x81818181U, 0x12121212U, + 0x83838383U, 0x1b1b1b1bU, 0x0e0e0e0eU, 0x23232323U, + 0xf5f5f5f5U, 0x45454545U, 0x21212121U, 0xcecececeU, + 0x49494949U, 0x2c2c2c2cU, 0xf9f9f9f9U, 0xe6e6e6e6U, + 0xb6b6b6b6U, 0x28282828U, 0x17171717U, 0x82828282U, + 0x1a1a1a1aU, 0x8b8b8b8bU, 0xfefefefeU, 0x8a8a8a8aU, + 0x09090909U, 0xc9c9c9c9U, 0x87878787U, 0x4e4e4e4eU, + 0xe1e1e1e1U, 0x2e2e2e2eU, 0xe4e4e4e4U, 0xe0e0e0e0U, + 0xebebebebU, 0x90909090U, 0xa4a4a4a4U, 0x1e1e1e1eU, + 0x85858585U, 0x60606060U, 0x00000000U, 0x25252525U, + 0xf4f4f4f4U, 0xf1f1f1f1U, 0x94949494U, 0x0b0b0b0bU, + 0xe7e7e7e7U, 0x75757575U, 0xefefefefU, 0x34343434U, + 0x31313131U, 0xd4d4d4d4U, 0xd0d0d0d0U, 0x86868686U, + 0x7e7e7e7eU, 0xadadadadU, 0xfdfdfdfdU, 0x29292929U, + 0x30303030U, 0x3b3b3b3bU, 0x9f9f9f9fU, 0xf8f8f8f8U, + 0xc6c6c6c6U, 0x13131313U, 0x06060606U, 0x05050505U, + 0xc5c5c5c5U, 0x11111111U, 0x77777777U, 0x7c7c7c7cU, + 0x7a7a7a7aU, 0x78787878U, 0x36363636U, 0x1c1c1c1cU, + 0x39393939U, 0x59595959U, 0x18181818U, 0x56565656U, + 0xb3b3b3b3U, 0xb0b0b0b0U, 0x24242424U, 0x20202020U, + 0xb2b2b2b2U, 0x92929292U, 0xa3a3a3a3U, 0xc0c0c0c0U, + 0x44444444U, 0x62626262U, 0x10101010U, 0xb4b4b4b4U, + 0x84848484U, 0x43434343U, 0x93939393U, 0xc2c2c2c2U, + 0x4a4a4a4aU, 0xbdbdbdbdU, 0x8f8f8f8fU, 0x2d2d2d2dU, + 0xbcbcbcbcU, 0x9c9c9c9cU, 0x6a6a6a6aU, 0x40404040U, + 0xcfcfcfcfU, 0xa2a2a2a2U, 0x80808080U, 0x4f4f4f4fU, + 0x1f1f1f1fU, 0xcacacacaU, 0xaaaaaaaaU, 0x42424242U, +}; + +static const ulong32 T5[256] = { + 0x00000000U, 0x01020608U, 0x02040c10U, 0x03060a18U, + 0x04081820U, 0x050a1e28U, 0x060c1430U, 0x070e1238U, + 0x08103040U, 0x09123648U, 0x0a143c50U, 0x0b163a58U, + 0x0c182860U, 0x0d1a2e68U, 0x0e1c2470U, 0x0f1e2278U, + 0x10206080U, 0x11226688U, 0x12246c90U, 0x13266a98U, + 0x142878a0U, 0x152a7ea8U, 0x162c74b0U, 0x172e72b8U, + 0x183050c0U, 0x193256c8U, 0x1a345cd0U, 0x1b365ad8U, + 0x1c3848e0U, 0x1d3a4ee8U, 0x1e3c44f0U, 0x1f3e42f8U, + 0x2040c01dU, 0x2142c615U, 0x2244cc0dU, 0x2346ca05U, + 0x2448d83dU, 0x254ade35U, 0x264cd42dU, 0x274ed225U, + 0x2850f05dU, 0x2952f655U, 0x2a54fc4dU, 0x2b56fa45U, + 0x2c58e87dU, 0x2d5aee75U, 0x2e5ce46dU, 0x2f5ee265U, + 0x3060a09dU, 0x3162a695U, 0x3264ac8dU, 0x3366aa85U, + 0x3468b8bdU, 0x356abeb5U, 0x366cb4adU, 0x376eb2a5U, + 0x387090ddU, 0x397296d5U, 0x3a749ccdU, 0x3b769ac5U, + 0x3c7888fdU, 0x3d7a8ef5U, 0x3e7c84edU, 0x3f7e82e5U, + 0x40809d3aU, 0x41829b32U, 0x4284912aU, 0x43869722U, + 0x4488851aU, 0x458a8312U, 0x468c890aU, 0x478e8f02U, + 0x4890ad7aU, 0x4992ab72U, 0x4a94a16aU, 0x4b96a762U, + 0x4c98b55aU, 0x4d9ab352U, 0x4e9cb94aU, 0x4f9ebf42U, + 0x50a0fdbaU, 0x51a2fbb2U, 0x52a4f1aaU, 0x53a6f7a2U, + 0x54a8e59aU, 0x55aae392U, 0x56ace98aU, 0x57aeef82U, + 0x58b0cdfaU, 0x59b2cbf2U, 0x5ab4c1eaU, 0x5bb6c7e2U, + 0x5cb8d5daU, 0x5dbad3d2U, 0x5ebcd9caU, 0x5fbedfc2U, + 0x60c05d27U, 0x61c25b2fU, 0x62c45137U, 0x63c6573fU, + 0x64c84507U, 0x65ca430fU, 0x66cc4917U, 0x67ce4f1fU, + 0x68d06d67U, 0x69d26b6fU, 0x6ad46177U, 0x6bd6677fU, + 0x6cd87547U, 0x6dda734fU, 0x6edc7957U, 0x6fde7f5fU, + 0x70e03da7U, 0x71e23bafU, 0x72e431b7U, 0x73e637bfU, + 0x74e82587U, 0x75ea238fU, 0x76ec2997U, 0x77ee2f9fU, + 0x78f00de7U, 0x79f20befU, 0x7af401f7U, 0x7bf607ffU, + 0x7cf815c7U, 0x7dfa13cfU, 0x7efc19d7U, 0x7ffe1fdfU, + 0x801d2774U, 0x811f217cU, 0x82192b64U, 0x831b2d6cU, + 0x84153f54U, 0x8517395cU, 0x86113344U, 0x8713354cU, + 0x880d1734U, 0x890f113cU, 0x8a091b24U, 0x8b0b1d2cU, + 0x8c050f14U, 0x8d07091cU, 0x8e010304U, 0x8f03050cU, + 0x903d47f4U, 0x913f41fcU, 0x92394be4U, 0x933b4decU, + 0x94355fd4U, 0x953759dcU, 0x963153c4U, 0x973355ccU, + 0x982d77b4U, 0x992f71bcU, 0x9a297ba4U, 0x9b2b7dacU, + 0x9c256f94U, 0x9d27699cU, 0x9e216384U, 0x9f23658cU, + 0xa05de769U, 0xa15fe161U, 0xa259eb79U, 0xa35bed71U, + 0xa455ff49U, 0xa557f941U, 0xa651f359U, 0xa753f551U, + 0xa84dd729U, 0xa94fd121U, 0xaa49db39U, 0xab4bdd31U, + 0xac45cf09U, 0xad47c901U, 0xae41c319U, 0xaf43c511U, + 0xb07d87e9U, 0xb17f81e1U, 0xb2798bf9U, 0xb37b8df1U, + 0xb4759fc9U, 0xb57799c1U, 0xb67193d9U, 0xb77395d1U, + 0xb86db7a9U, 0xb96fb1a1U, 0xba69bbb9U, 0xbb6bbdb1U, + 0xbc65af89U, 0xbd67a981U, 0xbe61a399U, 0xbf63a591U, + 0xc09dba4eU, 0xc19fbc46U, 0xc299b65eU, 0xc39bb056U, + 0xc495a26eU, 0xc597a466U, 0xc691ae7eU, 0xc793a876U, + 0xc88d8a0eU, 0xc98f8c06U, 0xca89861eU, 0xcb8b8016U, + 0xcc85922eU, 0xcd879426U, 0xce819e3eU, 0xcf839836U, + 0xd0bddaceU, 0xd1bfdcc6U, 0xd2b9d6deU, 0xd3bbd0d6U, + 0xd4b5c2eeU, 0xd5b7c4e6U, 0xd6b1cefeU, 0xd7b3c8f6U, + 0xd8adea8eU, 0xd9afec86U, 0xdaa9e69eU, 0xdbabe096U, + 0xdca5f2aeU, 0xdda7f4a6U, 0xdea1febeU, 0xdfa3f8b6U, + 0xe0dd7a53U, 0xe1df7c5bU, 0xe2d97643U, 0xe3db704bU, + 0xe4d56273U, 0xe5d7647bU, 0xe6d16e63U, 0xe7d3686bU, + 0xe8cd4a13U, 0xe9cf4c1bU, 0xeac94603U, 0xebcb400bU, + 0xecc55233U, 0xedc7543bU, 0xeec15e23U, 0xefc3582bU, + 0xf0fd1ad3U, 0xf1ff1cdbU, 0xf2f916c3U, 0xf3fb10cbU, + 0xf4f502f3U, 0xf5f704fbU, 0xf6f10ee3U, 0xf7f308ebU, + 0xf8ed2a93U, 0xf9ef2c9bU, 0xfae92683U, 0xfbeb208bU, + 0xfce532b3U, 0xfde734bbU, 0xfee13ea3U, 0xffe338abU, +}; + +/** + * The round constants. + */ +static const ulong32 rc[] = { + 0xba542f74U, 0x53d3d24dU, 0x50ac8dbfU, 0x70529a4cU, + 0xead597d1U, 0x33515ba6U, 0xde48a899U, 0xdb32b7fcU, + 0xe39e919bU, 0xe2bb416eU, 0xa5cb6b95U, 0xa1f3b102U, + 0xccc41d14U, 0xc363da5dU, 0x5fdc7dcdU, 0x7f5a6c5cU, + 0xf726ffedU, 0xe89d6f8eU, 0x19a0f089U, +}; + + + +#else + + +static const ulong32 T0[256] = { + 0xa753a6f5U, 0xd3bb6bd0U, 0xe6d1bf6eU, 0x71e2d93bU, + 0xd0bd67daU, 0xac458acfU, 0x4d9a29b3U, 0x79f2f90bU, + 0x3a74e89cU, 0xc98f038cU, 0x913f7e41U, 0xfce5d732U, + 0x1e3c7844U, 0x478e018fU, 0x54a84de5U, 0xbd67cea9U, + 0x8c050a0fU, 0xa557aef9U, 0x7af4f501U, 0xfbebcb20U, + 0x63c69157U, 0xb86ddab7U, 0xdda753f4U, 0xd4b577c2U, + 0xe5d7b364U, 0xb37bf68dU, 0xc59733a4U, 0xbe61c2a3U, + 0xa94f9ed1U, 0x880d1a17U, 0x0c183028U, 0xa259b2ebU, + 0x3972e496U, 0xdfa35bf8U, 0x2952a4f6U, 0xdaa94fe6U, + 0x2b56acfaU, 0xa84d9ad7U, 0xcb8b0b80U, 0x4c982db5U, + 0x4b9631a7U, 0x224488ccU, 0xaa4992dbU, 0x244890d8U, + 0x4182199bU, 0x70e0dd3dU, 0xa651a2f3U, 0xf9efc32cU, + 0x5ab475c1U, 0xe2d9af76U, 0xb07dfa87U, 0x366cd8b4U, + 0x7dfae913U, 0xe4d5b762U, 0x3366ccaaU, 0xffe3db38U, + 0x60c09d5dU, 0x204080c0U, 0x08102030U, 0x8b0b161dU, + 0x5ebc65d9U, 0xab4b96ddU, 0x7ffee11fU, 0x78f0fd0dU, + 0x7cf8ed15U, 0x2c58b0e8U, 0x57ae41efU, 0xd2b96fd6U, + 0xdca557f2U, 0x6ddaa973U, 0x7efce519U, 0x0d1a342eU, + 0x53a651f7U, 0x94356a5fU, 0xc39b2bb0U, 0x2850a0f0U, + 0x274e9cd2U, 0x060c1814U, 0x5fbe61dfU, 0xad478ec9U, + 0x67ce814fU, 0x5cb86dd5U, 0x55aa49e3U, 0x48903dadU, + 0x0e1c3824U, 0x52a455f1U, 0xeac98f46U, 0x42841591U, + 0x5bb671c7U, 0x5dba69d3U, 0x3060c0a0U, 0x58b07dcdU, + 0x51a259fbU, 0x59b279cbU, 0x3c78f088U, 0x4e9c25b9U, + 0x3870e090U, 0x8a09121bU, 0x72e4d531U, 0x14285078U, + 0xe7d3bb68U, 0xc6913faeU, 0xdea15ffeU, 0x50a05dfdU, + 0x8e010203U, 0x9239724bU, 0xd1bf63dcU, 0x77eec12fU, + 0x933b764dU, 0x458a0983U, 0x9a29527bU, 0xce811f9eU, + 0x2d5ab4eeU, 0x03060c0aU, 0x62c49551U, 0xb671e293U, + 0xb96fdeb1U, 0xbf63c6a5U, 0x96316253U, 0x6bd6b167U, + 0x3f7efc82U, 0x070e1c12U, 0x1224486cU, 0xae4182c3U, + 0x40801d9dU, 0x3468d0b8U, 0x468c0589U, 0x3e7cf884U, + 0xdbab4be0U, 0xcf831b98U, 0xecc59752U, 0xcc851792U, + 0xc19f23bcU, 0xa15fbee1U, 0xc09d27baU, 0xd6b17fceU, + 0x1d3a744eU, 0xf4f5f702U, 0x61c2995bU, 0x3b76ec9aU, + 0x10204060U, 0xd8ad47eaU, 0x68d0bd6dU, 0xa05dbae7U, + 0xb17ffe81U, 0x0a14283cU, 0x69d2b96bU, 0x6cd8ad75U, + 0x499239abU, 0xfae9cf26U, 0x76ecc529U, 0xc49537a2U, + 0x9e214263U, 0x9b2b567dU, 0x6edca579U, 0x992f5e71U, + 0xc2992fb6U, 0xb773e695U, 0x982d5a77U, 0xbc65caafU, + 0x8f030605U, 0x85172e39U, 0x1f3e7c42U, 0xb475ea9fU, + 0xf8edc72aU, 0x11224466U, 0x2e5cb8e4U, 0x00000000U, + 0x254a94deU, 0x1c387048U, 0x2a54a8fcU, 0x3d7af48eU, + 0x050a141eU, 0x4f9e21bfU, 0x7bf6f107U, 0xb279f28bU, + 0x3264c8acU, 0x903d7a47U, 0xaf4386c5U, 0x19326456U, + 0xa35bb6edU, 0xf7f3fb08U, 0x73e6d137U, 0x9d274e69U, + 0x152a547eU, 0x74e8cd25U, 0xeec19f5eU, 0xca890f86U, + 0x9f234665U, 0x0f1e3c22U, 0x1b366c5aU, 0x75eac923U, + 0x86112233U, 0x84152a3fU, 0x9c254a6fU, 0x4a9435a1U, + 0x97336655U, 0x1a34685cU, 0x65ca8943U, 0xf6f1ff0eU, + 0xedc79354U, 0x09122436U, 0xbb6bd6bdU, 0x264c98d4U, + 0x831b362dU, 0xebcb8b40U, 0x6fdea17fU, 0x811f3e21U, + 0x04081018U, 0x6ad4b561U, 0x43861197U, 0x01020406U, + 0x172e5c72U, 0xe1dfa37cU, 0x87132635U, 0xf5f7f304U, + 0x8d070e09U, 0xe3dbab70U, 0x23468ccaU, 0x801d3a27U, + 0x44880d85U, 0x162c5874U, 0x66cc8549U, 0x214284c6U, + 0xfee1df3eU, 0xd5b773c4U, 0x3162c4a6U, 0xd9af43ecU, + 0x356ad4beU, 0x18306050U, 0x0204080cU, 0x64c88d45U, + 0xf2f9ef16U, 0xf1ffe31cU, 0x56ac45e9U, 0xcd871394U, + 0x8219322bU, 0xc88d078aU, 0xba69d2bbU, 0xf0fde71aU, + 0xefc39b58U, 0xe9cf834cU, 0xe8cd874aU, 0xfde7d334U, + 0x890f1e11U, 0xd7b37bc8U, 0xc7933ba8U, 0xb577ee99U, + 0xa455aaffU, 0x2f5ebce2U, 0x95376e59U, 0x13264c6aU, + 0x0b162c3aU, 0xf3fbeb10U, 0xe0dda77aU, 0x376edcb2U, +}; + +static const ulong32 T1[256] = { + 0x53a7f5a6U, 0xbbd3d06bU, 0xd1e66ebfU, 0xe2713bd9U, + 0xbdd0da67U, 0x45accf8aU, 0x9a4db329U, 0xf2790bf9U, + 0x743a9ce8U, 0x8fc98c03U, 0x3f91417eU, 0xe5fc32d7U, + 0x3c1e4478U, 0x8e478f01U, 0xa854e54dU, 0x67bda9ceU, + 0x058c0f0aU, 0x57a5f9aeU, 0xf47a01f5U, 0xebfb20cbU, + 0xc6635791U, 0x6db8b7daU, 0xa7ddf453U, 0xb5d4c277U, + 0xd7e564b3U, 0x7bb38df6U, 0x97c5a433U, 0x61bea3c2U, + 0x4fa9d19eU, 0x0d88171aU, 0x180c2830U, 0x59a2ebb2U, + 0x723996e4U, 0xa3dff85bU, 0x5229f6a4U, 0xa9dae64fU, + 0x562bfaacU, 0x4da8d79aU, 0x8bcb800bU, 0x984cb52dU, + 0x964ba731U, 0x4422cc88U, 0x49aadb92U, 0x4824d890U, + 0x82419b19U, 0xe0703dddU, 0x51a6f3a2U, 0xeff92cc3U, + 0xb45ac175U, 0xd9e276afU, 0x7db087faU, 0x6c36b4d8U, + 0xfa7d13e9U, 0xd5e462b7U, 0x6633aaccU, 0xe3ff38dbU, + 0xc0605d9dU, 0x4020c080U, 0x10083020U, 0x0b8b1d16U, + 0xbc5ed965U, 0x4babdd96U, 0xfe7f1fe1U, 0xf0780dfdU, + 0xf87c15edU, 0x582ce8b0U, 0xae57ef41U, 0xb9d2d66fU, + 0xa5dcf257U, 0xda6d73a9U, 0xfc7e19e5U, 0x1a0d2e34U, + 0xa653f751U, 0x35945f6aU, 0x9bc3b02bU, 0x5028f0a0U, + 0x4e27d29cU, 0x0c061418U, 0xbe5fdf61U, 0x47adc98eU, + 0xce674f81U, 0xb85cd56dU, 0xaa55e349U, 0x9048ad3dU, + 0x1c0e2438U, 0xa452f155U, 0xc9ea468fU, 0x84429115U, + 0xb65bc771U, 0xba5dd369U, 0x6030a0c0U, 0xb058cd7dU, + 0xa251fb59U, 0xb259cb79U, 0x783c88f0U, 0x9c4eb925U, + 0x703890e0U, 0x098a1b12U, 0xe47231d5U, 0x28147850U, + 0xd3e768bbU, 0x91c6ae3fU, 0xa1defe5fU, 0xa050fd5dU, + 0x018e0302U, 0x39924b72U, 0xbfd1dc63U, 0xee772fc1U, + 0x3b934d76U, 0x8a458309U, 0x299a7b52U, 0x81ce9e1fU, + 0x5a2deeb4U, 0x06030a0cU, 0xc4625195U, 0x71b693e2U, + 0x6fb9b1deU, 0x63bfa5c6U, 0x31965362U, 0xd66b67b1U, + 0x7e3f82fcU, 0x0e07121cU, 0x24126c48U, 0x41aec382U, + 0x80409d1dU, 0x6834b8d0U, 0x8c468905U, 0x7c3e84f8U, + 0xabdbe04bU, 0x83cf981bU, 0xc5ec5297U, 0x85cc9217U, + 0x9fc1bc23U, 0x5fa1e1beU, 0x9dc0ba27U, 0xb1d6ce7fU, + 0x3a1d4e74U, 0xf5f402f7U, 0xc2615b99U, 0x763b9aecU, + 0x20106040U, 0xadd8ea47U, 0xd0686dbdU, 0x5da0e7baU, + 0x7fb181feU, 0x140a3c28U, 0xd2696bb9U, 0xd86c75adU, + 0x9249ab39U, 0xe9fa26cfU, 0xec7629c5U, 0x95c4a237U, + 0x219e6342U, 0x2b9b7d56U, 0xdc6e79a5U, 0x2f99715eU, + 0x99c2b62fU, 0x73b795e6U, 0x2d98775aU, 0x65bcafcaU, + 0x038f0506U, 0x1785392eU, 0x3e1f427cU, 0x75b49feaU, + 0xedf82ac7U, 0x22116644U, 0x5c2ee4b8U, 0x00000000U, + 0x4a25de94U, 0x381c4870U, 0x542afca8U, 0x7a3d8ef4U, + 0x0a051e14U, 0x9e4fbf21U, 0xf67b07f1U, 0x79b28bf2U, + 0x6432acc8U, 0x3d90477aU, 0x43afc586U, 0x32195664U, + 0x5ba3edb6U, 0xf3f708fbU, 0xe67337d1U, 0x279d694eU, + 0x2a157e54U, 0xe87425cdU, 0xc1ee5e9fU, 0x89ca860fU, + 0x239f6546U, 0x1e0f223cU, 0x361b5a6cU, 0xea7523c9U, + 0x11863322U, 0x15843f2aU, 0x259c6f4aU, 0x944aa135U, + 0x33975566U, 0x341a5c68U, 0xca654389U, 0xf1f60effU, + 0xc7ed5493U, 0x12093624U, 0x6bbbbdd6U, 0x4c26d498U, + 0x1b832d36U, 0xcbeb408bU, 0xde6f7fa1U, 0x1f81213eU, + 0x08041810U, 0xd46a61b5U, 0x86439711U, 0x02010604U, + 0x2e17725cU, 0xdfe17ca3U, 0x13873526U, 0xf7f504f3U, + 0x078d090eU, 0xdbe370abU, 0x4623ca8cU, 0x1d80273aU, + 0x8844850dU, 0x2c167458U, 0xcc664985U, 0x4221c684U, + 0xe1fe3edfU, 0xb7d5c473U, 0x6231a6c4U, 0xafd9ec43U, + 0x6a35bed4U, 0x30185060U, 0x04020c08U, 0xc864458dU, + 0xf9f216efU, 0xfff11ce3U, 0xac56e945U, 0x87cd9413U, + 0x19822b32U, 0x8dc88a07U, 0x69babbd2U, 0xfdf01ae7U, + 0xc3ef589bU, 0xcfe94c83U, 0xcde84a87U, 0xe7fd34d3U, + 0x0f89111eU, 0xb3d7c87bU, 0x93c7a83bU, 0x77b599eeU, + 0x55a4ffaaU, 0x5e2fe2bcU, 0x3795596eU, 0x26136a4cU, + 0x160b3a2cU, 0xfbf310ebU, 0xdde07aa7U, 0x6e37b2dcU, +}; + +static const ulong32 T2[256] = { + 0xa6f5a753U, 0x6bd0d3bbU, 0xbf6ee6d1U, 0xd93b71e2U, + 0x67dad0bdU, 0x8acfac45U, 0x29b34d9aU, 0xf90b79f2U, + 0xe89c3a74U, 0x038cc98fU, 0x7e41913fU, 0xd732fce5U, + 0x78441e3cU, 0x018f478eU, 0x4de554a8U, 0xcea9bd67U, + 0x0a0f8c05U, 0xaef9a557U, 0xf5017af4U, 0xcb20fbebU, + 0x915763c6U, 0xdab7b86dU, 0x53f4dda7U, 0x77c2d4b5U, + 0xb364e5d7U, 0xf68db37bU, 0x33a4c597U, 0xc2a3be61U, + 0x9ed1a94fU, 0x1a17880dU, 0x30280c18U, 0xb2eba259U, + 0xe4963972U, 0x5bf8dfa3U, 0xa4f62952U, 0x4fe6daa9U, + 0xacfa2b56U, 0x9ad7a84dU, 0x0b80cb8bU, 0x2db54c98U, + 0x31a74b96U, 0x88cc2244U, 0x92dbaa49U, 0x90d82448U, + 0x199b4182U, 0xdd3d70e0U, 0xa2f3a651U, 0xc32cf9efU, + 0x75c15ab4U, 0xaf76e2d9U, 0xfa87b07dU, 0xd8b4366cU, + 0xe9137dfaU, 0xb762e4d5U, 0xccaa3366U, 0xdb38ffe3U, + 0x9d5d60c0U, 0x80c02040U, 0x20300810U, 0x161d8b0bU, + 0x65d95ebcU, 0x96ddab4bU, 0xe11f7ffeU, 0xfd0d78f0U, + 0xed157cf8U, 0xb0e82c58U, 0x41ef57aeU, 0x6fd6d2b9U, + 0x57f2dca5U, 0xa9736ddaU, 0xe5197efcU, 0x342e0d1aU, + 0x51f753a6U, 0x6a5f9435U, 0x2bb0c39bU, 0xa0f02850U, + 0x9cd2274eU, 0x1814060cU, 0x61df5fbeU, 0x8ec9ad47U, + 0x814f67ceU, 0x6dd55cb8U, 0x49e355aaU, 0x3dad4890U, + 0x38240e1cU, 0x55f152a4U, 0x8f46eac9U, 0x15914284U, + 0x71c75bb6U, 0x69d35dbaU, 0xc0a03060U, 0x7dcd58b0U, + 0x59fb51a2U, 0x79cb59b2U, 0xf0883c78U, 0x25b94e9cU, + 0xe0903870U, 0x121b8a09U, 0xd53172e4U, 0x50781428U, + 0xbb68e7d3U, 0x3faec691U, 0x5ffedea1U, 0x5dfd50a0U, + 0x02038e01U, 0x724b9239U, 0x63dcd1bfU, 0xc12f77eeU, + 0x764d933bU, 0x0983458aU, 0x527b9a29U, 0x1f9ece81U, + 0xb4ee2d5aU, 0x0c0a0306U, 0x955162c4U, 0xe293b671U, + 0xdeb1b96fU, 0xc6a5bf63U, 0x62539631U, 0xb1676bd6U, + 0xfc823f7eU, 0x1c12070eU, 0x486c1224U, 0x82c3ae41U, + 0x1d9d4080U, 0xd0b83468U, 0x0589468cU, 0xf8843e7cU, + 0x4be0dbabU, 0x1b98cf83U, 0x9752ecc5U, 0x1792cc85U, + 0x23bcc19fU, 0xbee1a15fU, 0x27bac09dU, 0x7fced6b1U, + 0x744e1d3aU, 0xf702f4f5U, 0x995b61c2U, 0xec9a3b76U, + 0x40601020U, 0x47ead8adU, 0xbd6d68d0U, 0xbae7a05dU, + 0xfe81b17fU, 0x283c0a14U, 0xb96b69d2U, 0xad756cd8U, + 0x39ab4992U, 0xcf26fae9U, 0xc52976ecU, 0x37a2c495U, + 0x42639e21U, 0x567d9b2bU, 0xa5796edcU, 0x5e71992fU, + 0x2fb6c299U, 0xe695b773U, 0x5a77982dU, 0xcaafbc65U, + 0x06058f03U, 0x2e398517U, 0x7c421f3eU, 0xea9fb475U, + 0xc72af8edU, 0x44661122U, 0xb8e42e5cU, 0x00000000U, + 0x94de254aU, 0x70481c38U, 0xa8fc2a54U, 0xf48e3d7aU, + 0x141e050aU, 0x21bf4f9eU, 0xf1077bf6U, 0xf28bb279U, + 0xc8ac3264U, 0x7a47903dU, 0x86c5af43U, 0x64561932U, + 0xb6eda35bU, 0xfb08f7f3U, 0xd13773e6U, 0x4e699d27U, + 0x547e152aU, 0xcd2574e8U, 0x9f5eeec1U, 0x0f86ca89U, + 0x46659f23U, 0x3c220f1eU, 0x6c5a1b36U, 0xc92375eaU, + 0x22338611U, 0x2a3f8415U, 0x4a6f9c25U, 0x35a14a94U, + 0x66559733U, 0x685c1a34U, 0x894365caU, 0xff0ef6f1U, + 0x9354edc7U, 0x24360912U, 0xd6bdbb6bU, 0x98d4264cU, + 0x362d831bU, 0x8b40ebcbU, 0xa17f6fdeU, 0x3e21811fU, + 0x10180408U, 0xb5616ad4U, 0x11974386U, 0x04060102U, + 0x5c72172eU, 0xa37ce1dfU, 0x26358713U, 0xf304f5f7U, + 0x0e098d07U, 0xab70e3dbU, 0x8cca2346U, 0x3a27801dU, + 0x0d854488U, 0x5874162cU, 0x854966ccU, 0x84c62142U, + 0xdf3efee1U, 0x73c4d5b7U, 0xc4a63162U, 0x43ecd9afU, + 0xd4be356aU, 0x60501830U, 0x080c0204U, 0x8d4564c8U, + 0xef16f2f9U, 0xe31cf1ffU, 0x45e956acU, 0x1394cd87U, + 0x322b8219U, 0x078ac88dU, 0xd2bbba69U, 0xe71af0fdU, + 0x9b58efc3U, 0x834ce9cfU, 0x874ae8cdU, 0xd334fde7U, + 0x1e11890fU, 0x7bc8d7b3U, 0x3ba8c793U, 0xee99b577U, + 0xaaffa455U, 0xbce22f5eU, 0x6e599537U, 0x4c6a1326U, + 0x2c3a0b16U, 0xeb10f3fbU, 0xa77ae0ddU, 0xdcb2376eU, +}; + +static const ulong32 T3[256] = { + 0xf5a653a7U, 0xd06bbbd3U, 0x6ebfd1e6U, 0x3bd9e271U, + 0xda67bdd0U, 0xcf8a45acU, 0xb3299a4dU, 0x0bf9f279U, + 0x9ce8743aU, 0x8c038fc9U, 0x417e3f91U, 0x32d7e5fcU, + 0x44783c1eU, 0x8f018e47U, 0xe54da854U, 0xa9ce67bdU, + 0x0f0a058cU, 0xf9ae57a5U, 0x01f5f47aU, 0x20cbebfbU, + 0x5791c663U, 0xb7da6db8U, 0xf453a7ddU, 0xc277b5d4U, + 0x64b3d7e5U, 0x8df67bb3U, 0xa43397c5U, 0xa3c261beU, + 0xd19e4fa9U, 0x171a0d88U, 0x2830180cU, 0xebb259a2U, + 0x96e47239U, 0xf85ba3dfU, 0xf6a45229U, 0xe64fa9daU, + 0xfaac562bU, 0xd79a4da8U, 0x800b8bcbU, 0xb52d984cU, + 0xa731964bU, 0xcc884422U, 0xdb9249aaU, 0xd8904824U, + 0x9b198241U, 0x3ddde070U, 0xf3a251a6U, 0x2cc3eff9U, + 0xc175b45aU, 0x76afd9e2U, 0x87fa7db0U, 0xb4d86c36U, + 0x13e9fa7dU, 0x62b7d5e4U, 0xaacc6633U, 0x38dbe3ffU, + 0x5d9dc060U, 0xc0804020U, 0x30201008U, 0x1d160b8bU, + 0xd965bc5eU, 0xdd964babU, 0x1fe1fe7fU, 0x0dfdf078U, + 0x15edf87cU, 0xe8b0582cU, 0xef41ae57U, 0xd66fb9d2U, + 0xf257a5dcU, 0x73a9da6dU, 0x19e5fc7eU, 0x2e341a0dU, + 0xf751a653U, 0x5f6a3594U, 0xb02b9bc3U, 0xf0a05028U, + 0xd29c4e27U, 0x14180c06U, 0xdf61be5fU, 0xc98e47adU, + 0x4f81ce67U, 0xd56db85cU, 0xe349aa55U, 0xad3d9048U, + 0x24381c0eU, 0xf155a452U, 0x468fc9eaU, 0x91158442U, + 0xc771b65bU, 0xd369ba5dU, 0xa0c06030U, 0xcd7db058U, + 0xfb59a251U, 0xcb79b259U, 0x88f0783cU, 0xb9259c4eU, + 0x90e07038U, 0x1b12098aU, 0x31d5e472U, 0x78502814U, + 0x68bbd3e7U, 0xae3f91c6U, 0xfe5fa1deU, 0xfd5da050U, + 0x0302018eU, 0x4b723992U, 0xdc63bfd1U, 0x2fc1ee77U, + 0x4d763b93U, 0x83098a45U, 0x7b52299aU, 0x9e1f81ceU, + 0xeeb45a2dU, 0x0a0c0603U, 0x5195c462U, 0x93e271b6U, + 0xb1de6fb9U, 0xa5c663bfU, 0x53623196U, 0x67b1d66bU, + 0x82fc7e3fU, 0x121c0e07U, 0x6c482412U, 0xc38241aeU, + 0x9d1d8040U, 0xb8d06834U, 0x89058c46U, 0x84f87c3eU, + 0xe04babdbU, 0x981b83cfU, 0x5297c5ecU, 0x921785ccU, + 0xbc239fc1U, 0xe1be5fa1U, 0xba279dc0U, 0xce7fb1d6U, + 0x4e743a1dU, 0x02f7f5f4U, 0x5b99c261U, 0x9aec763bU, + 0x60402010U, 0xea47add8U, 0x6dbdd068U, 0xe7ba5da0U, + 0x81fe7fb1U, 0x3c28140aU, 0x6bb9d269U, 0x75add86cU, + 0xab399249U, 0x26cfe9faU, 0x29c5ec76U, 0xa23795c4U, + 0x6342219eU, 0x7d562b9bU, 0x79a5dc6eU, 0x715e2f99U, + 0xb62f99c2U, 0x95e673b7U, 0x775a2d98U, 0xafca65bcU, + 0x0506038fU, 0x392e1785U, 0x427c3e1fU, 0x9fea75b4U, + 0x2ac7edf8U, 0x66442211U, 0xe4b85c2eU, 0x00000000U, + 0xde944a25U, 0x4870381cU, 0xfca8542aU, 0x8ef47a3dU, + 0x1e140a05U, 0xbf219e4fU, 0x07f1f67bU, 0x8bf279b2U, + 0xacc86432U, 0x477a3d90U, 0xc58643afU, 0x56643219U, + 0xedb65ba3U, 0x08fbf3f7U, 0x37d1e673U, 0x694e279dU, + 0x7e542a15U, 0x25cde874U, 0x5e9fc1eeU, 0x860f89caU, + 0x6546239fU, 0x223c1e0fU, 0x5a6c361bU, 0x23c9ea75U, + 0x33221186U, 0x3f2a1584U, 0x6f4a259cU, 0xa135944aU, + 0x55663397U, 0x5c68341aU, 0x4389ca65U, 0x0efff1f6U, + 0x5493c7edU, 0x36241209U, 0xbdd66bbbU, 0xd4984c26U, + 0x2d361b83U, 0x408bcbebU, 0x7fa1de6fU, 0x213e1f81U, + 0x18100804U, 0x61b5d46aU, 0x97118643U, 0x06040201U, + 0x725c2e17U, 0x7ca3dfe1U, 0x35261387U, 0x04f3f7f5U, + 0x090e078dU, 0x70abdbe3U, 0xca8c4623U, 0x273a1d80U, + 0x850d8844U, 0x74582c16U, 0x4985cc66U, 0xc6844221U, + 0x3edfe1feU, 0xc473b7d5U, 0xa6c46231U, 0xec43afd9U, + 0xbed46a35U, 0x50603018U, 0x0c080402U, 0x458dc864U, + 0x16eff9f2U, 0x1ce3fff1U, 0xe945ac56U, 0x941387cdU, + 0x2b321982U, 0x8a078dc8U, 0xbbd269baU, 0x1ae7fdf0U, + 0x589bc3efU, 0x4c83cfe9U, 0x4a87cde8U, 0x34d3e7fdU, + 0x111e0f89U, 0xc87bb3d7U, 0xa83b93c7U, 0x99ee77b5U, + 0xffaa55a4U, 0xe2bc5e2fU, 0x596e3795U, 0x6a4c2613U, + 0x3a2c160bU, 0x10ebfbf3U, 0x7aa7dde0U, 0xb2dc6e37U, +}; + +static const ulong32 T4[256] = { + 0xa7a7a7a7U, 0xd3d3d3d3U, 0xe6e6e6e6U, 0x71717171U, + 0xd0d0d0d0U, 0xacacacacU, 0x4d4d4d4dU, 0x79797979U, + 0x3a3a3a3aU, 0xc9c9c9c9U, 0x91919191U, 0xfcfcfcfcU, + 0x1e1e1e1eU, 0x47474747U, 0x54545454U, 0xbdbdbdbdU, + 0x8c8c8c8cU, 0xa5a5a5a5U, 0x7a7a7a7aU, 0xfbfbfbfbU, + 0x63636363U, 0xb8b8b8b8U, 0xddddddddU, 0xd4d4d4d4U, + 0xe5e5e5e5U, 0xb3b3b3b3U, 0xc5c5c5c5U, 0xbebebebeU, + 0xa9a9a9a9U, 0x88888888U, 0x0c0c0c0cU, 0xa2a2a2a2U, + 0x39393939U, 0xdfdfdfdfU, 0x29292929U, 0xdadadadaU, + 0x2b2b2b2bU, 0xa8a8a8a8U, 0xcbcbcbcbU, 0x4c4c4c4cU, + 0x4b4b4b4bU, 0x22222222U, 0xaaaaaaaaU, 0x24242424U, + 0x41414141U, 0x70707070U, 0xa6a6a6a6U, 0xf9f9f9f9U, + 0x5a5a5a5aU, 0xe2e2e2e2U, 0xb0b0b0b0U, 0x36363636U, + 0x7d7d7d7dU, 0xe4e4e4e4U, 0x33333333U, 0xffffffffU, + 0x60606060U, 0x20202020U, 0x08080808U, 0x8b8b8b8bU, + 0x5e5e5e5eU, 0xababababU, 0x7f7f7f7fU, 0x78787878U, + 0x7c7c7c7cU, 0x2c2c2c2cU, 0x57575757U, 0xd2d2d2d2U, + 0xdcdcdcdcU, 0x6d6d6d6dU, 0x7e7e7e7eU, 0x0d0d0d0dU, + 0x53535353U, 0x94949494U, 0xc3c3c3c3U, 0x28282828U, + 0x27272727U, 0x06060606U, 0x5f5f5f5fU, 0xadadadadU, + 0x67676767U, 0x5c5c5c5cU, 0x55555555U, 0x48484848U, + 0x0e0e0e0eU, 0x52525252U, 0xeaeaeaeaU, 0x42424242U, + 0x5b5b5b5bU, 0x5d5d5d5dU, 0x30303030U, 0x58585858U, + 0x51515151U, 0x59595959U, 0x3c3c3c3cU, 0x4e4e4e4eU, + 0x38383838U, 0x8a8a8a8aU, 0x72727272U, 0x14141414U, + 0xe7e7e7e7U, 0xc6c6c6c6U, 0xdedededeU, 0x50505050U, + 0x8e8e8e8eU, 0x92929292U, 0xd1d1d1d1U, 0x77777777U, + 0x93939393U, 0x45454545U, 0x9a9a9a9aU, 0xcecececeU, + 0x2d2d2d2dU, 0x03030303U, 0x62626262U, 0xb6b6b6b6U, + 0xb9b9b9b9U, 0xbfbfbfbfU, 0x96969696U, 0x6b6b6b6bU, + 0x3f3f3f3fU, 0x07070707U, 0x12121212U, 0xaeaeaeaeU, + 0x40404040U, 0x34343434U, 0x46464646U, 0x3e3e3e3eU, + 0xdbdbdbdbU, 0xcfcfcfcfU, 0xececececU, 0xccccccccU, + 0xc1c1c1c1U, 0xa1a1a1a1U, 0xc0c0c0c0U, 0xd6d6d6d6U, + 0x1d1d1d1dU, 0xf4f4f4f4U, 0x61616161U, 0x3b3b3b3bU, + 0x10101010U, 0xd8d8d8d8U, 0x68686868U, 0xa0a0a0a0U, + 0xb1b1b1b1U, 0x0a0a0a0aU, 0x69696969U, 0x6c6c6c6cU, + 0x49494949U, 0xfafafafaU, 0x76767676U, 0xc4c4c4c4U, + 0x9e9e9e9eU, 0x9b9b9b9bU, 0x6e6e6e6eU, 0x99999999U, + 0xc2c2c2c2U, 0xb7b7b7b7U, 0x98989898U, 0xbcbcbcbcU, + 0x8f8f8f8fU, 0x85858585U, 0x1f1f1f1fU, 0xb4b4b4b4U, + 0xf8f8f8f8U, 0x11111111U, 0x2e2e2e2eU, 0x00000000U, + 0x25252525U, 0x1c1c1c1cU, 0x2a2a2a2aU, 0x3d3d3d3dU, + 0x05050505U, 0x4f4f4f4fU, 0x7b7b7b7bU, 0xb2b2b2b2U, + 0x32323232U, 0x90909090U, 0xafafafafU, 0x19191919U, + 0xa3a3a3a3U, 0xf7f7f7f7U, 0x73737373U, 0x9d9d9d9dU, + 0x15151515U, 0x74747474U, 0xeeeeeeeeU, 0xcacacacaU, + 0x9f9f9f9fU, 0x0f0f0f0fU, 0x1b1b1b1bU, 0x75757575U, + 0x86868686U, 0x84848484U, 0x9c9c9c9cU, 0x4a4a4a4aU, + 0x97979797U, 0x1a1a1a1aU, 0x65656565U, 0xf6f6f6f6U, + 0xededededU, 0x09090909U, 0xbbbbbbbbU, 0x26262626U, + 0x83838383U, 0xebebebebU, 0x6f6f6f6fU, 0x81818181U, + 0x04040404U, 0x6a6a6a6aU, 0x43434343U, 0x01010101U, + 0x17171717U, 0xe1e1e1e1U, 0x87878787U, 0xf5f5f5f5U, + 0x8d8d8d8dU, 0xe3e3e3e3U, 0x23232323U, 0x80808080U, + 0x44444444U, 0x16161616U, 0x66666666U, 0x21212121U, + 0xfefefefeU, 0xd5d5d5d5U, 0x31313131U, 0xd9d9d9d9U, + 0x35353535U, 0x18181818U, 0x02020202U, 0x64646464U, + 0xf2f2f2f2U, 0xf1f1f1f1U, 0x56565656U, 0xcdcdcdcdU, + 0x82828282U, 0xc8c8c8c8U, 0xbabababaU, 0xf0f0f0f0U, + 0xefefefefU, 0xe9e9e9e9U, 0xe8e8e8e8U, 0xfdfdfdfdU, + 0x89898989U, 0xd7d7d7d7U, 0xc7c7c7c7U, 0xb5b5b5b5U, + 0xa4a4a4a4U, 0x2f2f2f2fU, 0x95959595U, 0x13131313U, + 0x0b0b0b0bU, 0xf3f3f3f3U, 0xe0e0e0e0U, 0x37373737U, +}; + +static const ulong32 T5[256] = { + 0x00000000U, 0x01020608U, 0x02040c10U, 0x03060a18U, + 0x04081820U, 0x050a1e28U, 0x060c1430U, 0x070e1238U, + 0x08103040U, 0x09123648U, 0x0a143c50U, 0x0b163a58U, + 0x0c182860U, 0x0d1a2e68U, 0x0e1c2470U, 0x0f1e2278U, + 0x10206080U, 0x11226688U, 0x12246c90U, 0x13266a98U, + 0x142878a0U, 0x152a7ea8U, 0x162c74b0U, 0x172e72b8U, + 0x183050c0U, 0x193256c8U, 0x1a345cd0U, 0x1b365ad8U, + 0x1c3848e0U, 0x1d3a4ee8U, 0x1e3c44f0U, 0x1f3e42f8U, + 0x2040c01dU, 0x2142c615U, 0x2244cc0dU, 0x2346ca05U, + 0x2448d83dU, 0x254ade35U, 0x264cd42dU, 0x274ed225U, + 0x2850f05dU, 0x2952f655U, 0x2a54fc4dU, 0x2b56fa45U, + 0x2c58e87dU, 0x2d5aee75U, 0x2e5ce46dU, 0x2f5ee265U, + 0x3060a09dU, 0x3162a695U, 0x3264ac8dU, 0x3366aa85U, + 0x3468b8bdU, 0x356abeb5U, 0x366cb4adU, 0x376eb2a5U, + 0x387090ddU, 0x397296d5U, 0x3a749ccdU, 0x3b769ac5U, + 0x3c7888fdU, 0x3d7a8ef5U, 0x3e7c84edU, 0x3f7e82e5U, + 0x40809d3aU, 0x41829b32U, 0x4284912aU, 0x43869722U, + 0x4488851aU, 0x458a8312U, 0x468c890aU, 0x478e8f02U, + 0x4890ad7aU, 0x4992ab72U, 0x4a94a16aU, 0x4b96a762U, + 0x4c98b55aU, 0x4d9ab352U, 0x4e9cb94aU, 0x4f9ebf42U, + 0x50a0fdbaU, 0x51a2fbb2U, 0x52a4f1aaU, 0x53a6f7a2U, + 0x54a8e59aU, 0x55aae392U, 0x56ace98aU, 0x57aeef82U, + 0x58b0cdfaU, 0x59b2cbf2U, 0x5ab4c1eaU, 0x5bb6c7e2U, + 0x5cb8d5daU, 0x5dbad3d2U, 0x5ebcd9caU, 0x5fbedfc2U, + 0x60c05d27U, 0x61c25b2fU, 0x62c45137U, 0x63c6573fU, + 0x64c84507U, 0x65ca430fU, 0x66cc4917U, 0x67ce4f1fU, + 0x68d06d67U, 0x69d26b6fU, 0x6ad46177U, 0x6bd6677fU, + 0x6cd87547U, 0x6dda734fU, 0x6edc7957U, 0x6fde7f5fU, + 0x70e03da7U, 0x71e23bafU, 0x72e431b7U, 0x73e637bfU, + 0x74e82587U, 0x75ea238fU, 0x76ec2997U, 0x77ee2f9fU, + 0x78f00de7U, 0x79f20befU, 0x7af401f7U, 0x7bf607ffU, + 0x7cf815c7U, 0x7dfa13cfU, 0x7efc19d7U, 0x7ffe1fdfU, + 0x801d2774U, 0x811f217cU, 0x82192b64U, 0x831b2d6cU, + 0x84153f54U, 0x8517395cU, 0x86113344U, 0x8713354cU, + 0x880d1734U, 0x890f113cU, 0x8a091b24U, 0x8b0b1d2cU, + 0x8c050f14U, 0x8d07091cU, 0x8e010304U, 0x8f03050cU, + 0x903d47f4U, 0x913f41fcU, 0x92394be4U, 0x933b4decU, + 0x94355fd4U, 0x953759dcU, 0x963153c4U, 0x973355ccU, + 0x982d77b4U, 0x992f71bcU, 0x9a297ba4U, 0x9b2b7dacU, + 0x9c256f94U, 0x9d27699cU, 0x9e216384U, 0x9f23658cU, + 0xa05de769U, 0xa15fe161U, 0xa259eb79U, 0xa35bed71U, + 0xa455ff49U, 0xa557f941U, 0xa651f359U, 0xa753f551U, + 0xa84dd729U, 0xa94fd121U, 0xaa49db39U, 0xab4bdd31U, + 0xac45cf09U, 0xad47c901U, 0xae41c319U, 0xaf43c511U, + 0xb07d87e9U, 0xb17f81e1U, 0xb2798bf9U, 0xb37b8df1U, + 0xb4759fc9U, 0xb57799c1U, 0xb67193d9U, 0xb77395d1U, + 0xb86db7a9U, 0xb96fb1a1U, 0xba69bbb9U, 0xbb6bbdb1U, + 0xbc65af89U, 0xbd67a981U, 0xbe61a399U, 0xbf63a591U, + 0xc09dba4eU, 0xc19fbc46U, 0xc299b65eU, 0xc39bb056U, + 0xc495a26eU, 0xc597a466U, 0xc691ae7eU, 0xc793a876U, + 0xc88d8a0eU, 0xc98f8c06U, 0xca89861eU, 0xcb8b8016U, + 0xcc85922eU, 0xcd879426U, 0xce819e3eU, 0xcf839836U, + 0xd0bddaceU, 0xd1bfdcc6U, 0xd2b9d6deU, 0xd3bbd0d6U, + 0xd4b5c2eeU, 0xd5b7c4e6U, 0xd6b1cefeU, 0xd7b3c8f6U, + 0xd8adea8eU, 0xd9afec86U, 0xdaa9e69eU, 0xdbabe096U, + 0xdca5f2aeU, 0xdda7f4a6U, 0xdea1febeU, 0xdfa3f8b6U, + 0xe0dd7a53U, 0xe1df7c5bU, 0xe2d97643U, 0xe3db704bU, + 0xe4d56273U, 0xe5d7647bU, 0xe6d16e63U, 0xe7d3686bU, + 0xe8cd4a13U, 0xe9cf4c1bU, 0xeac94603U, 0xebcb400bU, + 0xecc55233U, 0xedc7543bU, 0xeec15e23U, 0xefc3582bU, + 0xf0fd1ad3U, 0xf1ff1cdbU, 0xf2f916c3U, 0xf3fb10cbU, + 0xf4f502f3U, 0xf5f704fbU, 0xf6f10ee3U, 0xf7f308ebU, + 0xf8ed2a93U, 0xf9ef2c9bU, 0xfae92683U, 0xfbeb208bU, + 0xfce532b3U, 0xfde734bbU, 0xfee13ea3U, 0xffe338abU, +}; + +/** + * The round constants. + */ +static const ulong32 rc[] = { + 0xa7d3e671U, 0xd0ac4d79U, 0x3ac991fcU, 0x1e4754bdU, + 0x8ca57afbU, 0x63b8ddd4U, 0xe5b3c5beU, 0xa9880ca2U, + 0x39df29daU, 0x2ba8cb4cU, 0x4b22aa24U, 0x4170a6f9U, + 0x5ae2b036U, 0x7de433ffU, 0x6020088bU, 0x5eab7f78U, + 0x7c2c57d2U, 0xdc6d7e0dU, 0x5394c328U, +}; + +#endif + + /** + Initialize the Anubis block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +#ifdef LTC_CLEAN_STACK +static int s_anubis_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#else +int anubis_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#endif +{ + int N, R, i, pos, r; + ulong32 kappa[MAX_N]; + ulong32 inter[MAX_N] = { 0 }; /* initialize as all zeroes */ + ulong32 v, K0, K1, K2, K3; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* Valid sizes (in bytes) are 16, 20, 24, 28, 32, 36, and 40. */ + if ((keylen & 3) || (keylen < 16) || (keylen > 40)) { + return CRYPT_INVALID_KEYSIZE; + } + skey->anubis.keyBits = keylen*8; + + /* + * determine the N length parameter: + * (N.B. it is assumed that the key length is valid!) + */ + N = skey->anubis.keyBits >> 5; + + /* + * determine number of rounds from key size: + */ + skey->anubis.R = R = 8 + N; + + if (num_rounds != 0 && num_rounds != skey->anubis.R) { + return CRYPT_INVALID_ROUNDS; + } + + /* + * map cipher key to initial key state (mu): + */ + for (i = 0, pos = 0; i < N; i++, pos += 4) { + kappa[i] = + (((ulong32)key[pos ]) << 24) ^ + (((ulong32)key[pos + 1]) << 16) ^ + (((ulong32)key[pos + 2]) << 8) ^ + (((ulong32)key[pos + 3]) ); + } + + /* + * generate R + 1 round keys: + */ + for (r = 0; r <= R; r++) { + /* + * generate r-th round key K^r: + */ + K0 = T4[(kappa[N - 1] >> 24) & 0xff]; + K1 = T4[(kappa[N - 1] >> 16) & 0xff]; + K2 = T4[(kappa[N - 1] >> 8) & 0xff]; + K3 = T4[(kappa[N - 1] ) & 0xff]; + for (i = N - 2; i >= 0; i--) { + K0 = T4[(kappa[i] >> 24) & 0xff] ^ + (T5[(K0 >> 24) & 0xff] & 0xff000000U) ^ + (T5[(K0 >> 16) & 0xff] & 0x00ff0000U) ^ + (T5[(K0 >> 8) & 0xff] & 0x0000ff00U) ^ + (T5[(K0 ) & 0xff] & 0x000000ffU); + K1 = T4[(kappa[i] >> 16) & 0xff] ^ + (T5[(K1 >> 24) & 0xff] & 0xff000000U) ^ + (T5[(K1 >> 16) & 0xff] & 0x00ff0000U) ^ + (T5[(K1 >> 8) & 0xff] & 0x0000ff00U) ^ + (T5[(K1 ) & 0xff] & 0x000000ffU); + K2 = T4[(kappa[i] >> 8) & 0xff] ^ + (T5[(K2 >> 24) & 0xff] & 0xff000000U) ^ + (T5[(K2 >> 16) & 0xff] & 0x00ff0000U) ^ + (T5[(K2 >> 8) & 0xff] & 0x0000ff00U) ^ + (T5[(K2 ) & 0xff] & 0x000000ffU); + K3 = T4[(kappa[i] ) & 0xff] ^ + (T5[(K3 >> 24) & 0xff] & 0xff000000U) ^ + (T5[(K3 >> 16) & 0xff] & 0x00ff0000U) ^ + (T5[(K3 >> 8) & 0xff] & 0x0000ff00U) ^ + (T5[(K3 ) & 0xff] & 0x000000ffU); + } + /* + -- this is the code to use with the large U tables: + K0 = K1 = K2 = K3 = 0; + for (i = 0; i < N; i++) { + K0 ^= U[i][(kappa[i] >> 24) & 0xff]; + K1 ^= U[i][(kappa[i] >> 16) & 0xff]; + K2 ^= U[i][(kappa[i] >> 8) & 0xff]; + K3 ^= U[i][(kappa[i] ) & 0xff]; + } + */ + skey->anubis.roundKeyEnc[r][0] = K0; + skey->anubis.roundKeyEnc[r][1] = K1; + skey->anubis.roundKeyEnc[r][2] = K2; + skey->anubis.roundKeyEnc[r][3] = K3; + + /* + * compute kappa^{r+1} from kappa^r: + */ + if (r == R) { + break; + } + for (i = 0; i < N; i++) { + int j = i; + inter[i] = T0[(kappa[j--] >> 24) & 0xff]; if (j < 0) j = N - 1; + inter[i] ^= T1[(kappa[j--] >> 16) & 0xff]; if (j < 0) j = N - 1; + inter[i] ^= T2[(kappa[j--] >> 8) & 0xff]; if (j < 0) j = N - 1; + inter[i] ^= T3[(kappa[j ] ) & 0xff]; + } + kappa[0] = inter[0] ^ rc[r]; + for (i = 1; i < N; i++) { + kappa[i] = inter[i]; + } + } + + /* + * generate inverse key schedule: K'^0 = K^R, K'^R = K^0, K'^r = theta(K^{R-r}): + */ + for (i = 0; i < 4; i++) { + skey->anubis.roundKeyDec[0][i] = skey->anubis.roundKeyEnc[R][i]; + skey->anubis.roundKeyDec[R][i] = skey->anubis.roundKeyEnc[0][i]; + } + for (r = 1; r < R; r++) { + for (i = 0; i < 4; i++) { + v = skey->anubis.roundKeyEnc[R - r][i]; + skey->anubis.roundKeyDec[r][i] = + T0[T4[(v >> 24) & 0xff] & 0xff] ^ + T1[T4[(v >> 16) & 0xff] & 0xff] ^ + T2[T4[(v >> 8) & 0xff] & 0xff] ^ + T3[T4[(v ) & 0xff] & 0xff]; + } + } + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int anubis_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int err; + err = s_anubis_setup(key, keylen, num_rounds, skey); + burn_stack(sizeof(int) * 5 + sizeof(ulong32) * (MAX_N + MAX_N + 5)); + return err; +} +#endif + + +static void anubis_crypt(const unsigned char *plaintext, unsigned char *ciphertext, + const ulong32 roundKey[18 + 1][4], int R) { + int i, pos, r; + ulong32 state[4]; + ulong32 inter[4]; + + /* + * map plaintext block to cipher state (mu) + * and add initial round key (sigma[K^0]): + */ + for (i = 0, pos = 0; i < 4; i++, pos += 4) { + state[i] = + (((ulong32)plaintext[pos ]) << 24) ^ + (((ulong32)plaintext[pos + 1]) << 16) ^ + (((ulong32)plaintext[pos + 2]) << 8) ^ + (((ulong32)plaintext[pos + 3]) ) ^ + roundKey[0][i]; + } + + /* + * R - 1 full rounds: + */ + for (r = 1; r < R; r++) { + inter[0] = + T0[(state[0] >> 24) & 0xff] ^ + T1[(state[1] >> 24) & 0xff] ^ + T2[(state[2] >> 24) & 0xff] ^ + T3[(state[3] >> 24) & 0xff] ^ + roundKey[r][0]; + inter[1] = + T0[(state[0] >> 16) & 0xff] ^ + T1[(state[1] >> 16) & 0xff] ^ + T2[(state[2] >> 16) & 0xff] ^ + T3[(state[3] >> 16) & 0xff] ^ + roundKey[r][1]; + inter[2] = + T0[(state[0] >> 8) & 0xff] ^ + T1[(state[1] >> 8) & 0xff] ^ + T2[(state[2] >> 8) & 0xff] ^ + T3[(state[3] >> 8) & 0xff] ^ + roundKey[r][2]; + inter[3] = + T0[(state[0] ) & 0xff] ^ + T1[(state[1] ) & 0xff] ^ + T2[(state[2] ) & 0xff] ^ + T3[(state[3] ) & 0xff] ^ + roundKey[r][3]; + state[0] = inter[0]; + state[1] = inter[1]; + state[2] = inter[2]; + state[3] = inter[3]; + } + + /* + * last round: + */ + inter[0] = + (T0[(state[0] >> 24) & 0xff] & 0xff000000U) ^ + (T1[(state[1] >> 24) & 0xff] & 0x00ff0000U) ^ + (T2[(state[2] >> 24) & 0xff] & 0x0000ff00U) ^ + (T3[(state[3] >> 24) & 0xff] & 0x000000ffU) ^ + roundKey[R][0]; + inter[1] = + (T0[(state[0] >> 16) & 0xff] & 0xff000000U) ^ + (T1[(state[1] >> 16) & 0xff] & 0x00ff0000U) ^ + (T2[(state[2] >> 16) & 0xff] & 0x0000ff00U) ^ + (T3[(state[3] >> 16) & 0xff] & 0x000000ffU) ^ + roundKey[R][1]; + inter[2] = + (T0[(state[0] >> 8) & 0xff] & 0xff000000U) ^ + (T1[(state[1] >> 8) & 0xff] & 0x00ff0000U) ^ + (T2[(state[2] >> 8) & 0xff] & 0x0000ff00U) ^ + (T3[(state[3] >> 8) & 0xff] & 0x000000ffU) ^ + roundKey[R][2]; + inter[3] = + (T0[(state[0] ) & 0xff] & 0xff000000U) ^ + (T1[(state[1] ) & 0xff] & 0x00ff0000U) ^ + (T2[(state[2] ) & 0xff] & 0x0000ff00U) ^ + (T3[(state[3] ) & 0xff] & 0x000000ffU) ^ + roundKey[R][3]; + + /* + * map cipher state to ciphertext block (mu^{-1}): + */ + for (i = 0, pos = 0; i < 4; i++, pos += 4) { + ulong32 w = inter[i]; + ciphertext[pos ] = (unsigned char)(w >> 24); + ciphertext[pos + 1] = (unsigned char)(w >> 16); + ciphertext[pos + 2] = (unsigned char)(w >> 8); + ciphertext[pos + 3] = (unsigned char)(w ); + } +} + +/** + Encrypts a block of text with Anubis + @param pt The input plaintext (16 bytes) + @param ct The output ciphertext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int anubis_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + if (skey->anubis.R < 12 || skey->anubis.R > 18) { + return CRYPT_INVALID_ROUNDS; + } + + anubis_crypt(pt, ct, skey->anubis.roundKeyEnc, skey->anubis.R); + return CRYPT_OK; +} + +/** + Decrypts a block of text with Anubis + @param ct The input ciphertext (16 bytes) + @param pt The output plaintext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int anubis_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + if (skey->anubis.R < 12 || skey->anubis.R > 18) { + return CRYPT_INVALID_ROUNDS; + } + + anubis_crypt(ct, pt, skey->anubis.roundKeyDec, skey->anubis.R); + return CRYPT_OK; +} + +/** + Performs a self-test of the Anubis block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int anubis_test(void) +{ +#if !defined(LTC_TEST) + return CRYPT_NOP; +#else + static const struct test { + int keylen; + unsigned char pt[16], ct[16], key[40]; + } tests[] = { +#ifndef LTC_ANUBIS_TWEAK + /**** ORIGINAL LTC_ANUBIS ****/ + /* 128 bit keys */ +{ + 16, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xF0, 0x68, 0x60, 0xFC, 0x67, 0x30, 0xE8, 0x18, + 0xF1, 0x32, 0xC7, 0x8A, 0xF4, 0x13, 0x2A, 0xFE }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 16, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xA8, 0x66, 0x84, 0x80, 0x07, 0x74, 0x5C, 0x89, + 0xFC, 0x5E, 0xB5, 0xBA, 0xD4, 0xFE, 0x32, 0x6D }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +}, + + /* 160-bit keys */ +{ + 20, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xBD, 0x5E, 0x32, 0xBE, 0x51, 0x67, 0xA8, 0xE2, + 0x72, 0xD7, 0x95, 0x0F, 0x83, 0xC6, 0x8C, 0x31 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 } +}, { + 20, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x4C, 0x1F, 0x86, 0x2E, 0x11, 0xEB, 0xCE, 0xEB, + 0xFE, 0xB9, 0x73, 0xC9, 0xDF, 0xEF, 0x7A, 0xDB }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01 } +}, + + /* 192-bit keys */ +{ + 24, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x17, 0xAC, 0x57, 0x44, 0x9D, 0x59, 0x61, 0x66, + 0xD0, 0xC7, 0x9E, 0x04, 0x7C, 0xC7, 0x58, 0xF0 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 24, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x71, 0x52, 0xB4, 0xEB, 0x1D, 0xAA, 0x36, 0xFD, + 0x57, 0x14, 0x5F, 0x57, 0x04, 0x9F, 0x70, 0x74 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +}, + + /* 224-bit keys */ +{ + 28, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xA2, 0xF0, 0xA6, 0xB9, 0x17, 0x93, 0x2A, 0x3B, + 0xEF, 0x08, 0xE8, 0x7A, 0x58, 0xD6, 0xF8, 0x53 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 } +}, { + 28, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xF0, 0xCA, 0xFC, 0x78, 0x8B, 0x4B, 0x4E, 0x53, + 0x8B, 0xC4, 0x32, 0x6A, 0xF5, 0xB9, 0x1B, 0x5F }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01 } +}, + + /* 256-bit keys */ +{ + 32, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xE0, 0x86, 0xAC, 0x45, 0x6B, 0x3C, 0xE5, 0x13, + 0xED, 0xF5, 0xDF, 0xDD, 0xD6, 0x3B, 0x71, 0x93 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 32, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x50, 0x01, 0xB9, 0xF5, 0x21, 0xC1, 0xC1, 0x29, + 0x00, 0xD5, 0xEC, 0x98, 0x2B, 0x9E, 0xE8, 0x21 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +}, + + /* 288-bit keys */ +{ + 36, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xE8, 0xF4, 0xAF, 0x2B, 0x21, 0xA0, 0x87, 0x9B, + 0x41, 0x95, 0xB9, 0x71, 0x75, 0x79, 0x04, 0x7C }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 } +}, { + 36, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xE6, 0xA6, 0xA5, 0xBC, 0x8B, 0x63, 0x6F, 0xE2, + 0xBD, 0xA7, 0xA7, 0x53, 0xAB, 0x40, 0x22, 0xE0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01 } +}, + + /* 320-bit keys */ +{ + 40, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x17, 0x04, 0xD7, 0x2C, 0xC6, 0x85, 0x76, 0x02, + 0x4B, 0xCC, 0x39, 0x80, 0xD8, 0x22, 0xEA, 0xA4 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 40, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x7A, 0x41, 0xE6, 0x7D, 0x4F, 0xD8, 0x64, 0xF0, + 0x44, 0xA8, 0x3C, 0x73, 0x81, 0x7E, 0x53, 0xD8 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +} +#else + /**** Tweaked LTC_ANUBIS ****/ + /* 128 bit keys */ +{ + 16, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xB8, 0x35, 0xBD, 0xC3, 0x34, 0x82, 0x9D, 0x83, + 0x71, 0xBF, 0xA3, 0x71, 0xE4, 0xB3, 0xC4, 0xFD }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 16, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xE6, 0x14, 0x1E, 0xAF, 0xEB, 0xE0, 0x59, 0x3C, + 0x48, 0xE1, 0xCD, 0xF2, 0x1B, 0xBA, 0xA1, 0x89 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +}, + + /* 160-bit keys */ +{ + 20, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x97, 0x59, 0x79, 0x4B, 0x5C, 0xA0, 0x70, 0x73, + 0x24, 0xEF, 0xB3, 0x58, 0x67, 0xCA, 0xD4, 0xB3 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 } +}, { + 20, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xB8, 0x0D, 0xFB, 0x9B, 0xE4, 0xA1, 0x58, 0x87, + 0xB3, 0x76, 0xD5, 0x02, 0x18, 0x95, 0xC1, 0x2E }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01 } +}, + + /* 192-bit keys */ +{ + 24, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x7D, 0x62, 0x3B, 0x52, 0xC7, 0x4C, 0x64, 0xD8, + 0xEB, 0xC7, 0x2D, 0x57, 0x97, 0x85, 0x43, 0x8F }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 24, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xB1, 0x0A, 0x59, 0xDD, 0x5D, 0x5D, 0x8D, 0x67, + 0xEC, 0xEE, 0x4A, 0xC4, 0xBE, 0x4F, 0xA8, 0x4F }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +}, + + /* 224-bit keys */ +{ + 28, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x68, 0x9E, 0x05, 0x94, 0x6A, 0x94, 0x43, 0x8F, + 0xE7, 0x8E, 0x37, 0x3D, 0x24, 0x97, 0x92, 0xF5 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 } +}, { + 28, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xDD, 0xB7, 0xB0, 0xB4, 0xE9, 0xB4, 0x9B, 0x9C, + 0x38, 0x20, 0x25, 0x0B, 0x47, 0xC2, 0x1F, 0x89 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01 } +}, + + /* 256-bit keys */ +{ + 32, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x96, 0x00, 0xF0, 0x76, 0x91, 0x69, 0x29, 0x87, + 0xF5, 0xE5, 0x97, 0xDB, 0xDB, 0xAF, 0x1B, 0x0A }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 32, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x69, 0x9C, 0xAF, 0xDD, 0x94, 0xC7, 0xBC, 0x60, + 0x44, 0xFE, 0x02, 0x05, 0x8A, 0x6E, 0xEF, 0xBD }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +}, + + /* 288-bit keys */ +{ + 36, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x0F, 0xC7, 0xA2, 0xC0, 0x11, 0x17, 0xAC, 0x43, + 0x52, 0x5E, 0xDF, 0x6C, 0xF3, 0x96, 0x33, 0x6C }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 } +}, { + 36, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xAD, 0x08, 0x4F, 0xED, 0x55, 0xA6, 0x94, 0x3E, + 0x7E, 0x5E, 0xED, 0x05, 0xA1, 0x9D, 0x41, 0xB4 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01 } +}, + + /* 320-bit keys */ +{ + 40, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xFE, 0xE2, 0x0E, 0x2A, 0x9D, 0xC5, 0x83, 0xBA, + 0xA3, 0xA6, 0xD6, 0xA6, 0xF2, 0xE8, 0x06, 0xA5 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + 40, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x86, 0x3D, 0xCC, 0x4A, 0x60, 0x34, 0x9C, 0x28, + 0xA7, 0xDA, 0xA4, 0x3B, 0x0A, 0xD7, 0xFD, 0xC7 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +} +#endif +}; + int x, y; + unsigned char buf[2][16]; + symmetric_key skey; + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + anubis_setup(tests[x].key, tests[x].keylen, 0, &skey); + anubis_ecb_encrypt(tests[x].pt, buf[0], &skey); + anubis_ecb_decrypt(buf[0], buf[1], &skey); + if (compare_testvector(buf[0], 16, tests[x].ct, 16, "Anubis Encrypt", x) || + compare_testvector(buf[1], 16, tests[x].pt, 16, "Anubis Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + for (y = 0; y < 1000; y++) anubis_ecb_encrypt(buf[0], buf[0], &skey); + for (y = 0; y < 1000; y++) anubis_ecb_decrypt(buf[0], buf[0], &skey); + if (compare_testvector(buf[0], 16, tests[x].ct, 16, "Anubis 1000", 1000)) { + return CRYPT_FAIL_TESTVECTOR; + } + + } + return CRYPT_OK; +#endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void anubis_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int anubis_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize >= 40) { + *keysize = 40; + } else if (*keysize >= 36) { + *keysize = 36; + } else if (*keysize >= 32) { + *keysize = 32; + } else if (*keysize >= 28) { + *keysize = 28; + } else if (*keysize >= 24) { + *keysize = 24; + } else if (*keysize >= 20) { + *keysize = 20; + } else if (*keysize >= 16) { + *keysize = 16; + } else { + return CRYPT_INVALID_KEYSIZE; + } + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/blowfish.c b/optee_os/core/lib/libtomcrypt/src/ciphers/blowfish.c new file mode 100644 index 0000000..35efc13 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/blowfish.c @@ -0,0 +1,658 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/** + @file blowfish.c + Implementation of the Blowfish block cipher, Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_BLOWFISH + +const struct ltc_cipher_descriptor blowfish_desc = +{ + "blowfish", + 0, + 8, 56, 8, 16, + &blowfish_setup, + &blowfish_ecb_encrypt, + &blowfish_ecb_decrypt, + &blowfish_test, + &blowfish_done, + &blowfish_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 ORIG_P[16 + 2] = { + 0x243F6A88UL, 0x85A308D3UL, 0x13198A2EUL, 0x03707344UL, + 0xA4093822UL, 0x299F31D0UL, 0x082EFA98UL, 0xEC4E6C89UL, + 0x452821E6UL, 0x38D01377UL, 0xBE5466CFUL, 0x34E90C6CUL, + 0xC0AC29B7UL, 0xC97C50DDUL, 0x3F84D5B5UL, 0xB5470917UL, + 0x9216D5D9UL, 0x8979FB1BUL +}; + +static const ulong32 ORIG_S[4][256] = { + { 0xD1310BA6UL, 0x98DFB5ACUL, 0x2FFD72DBUL, 0xD01ADFB7UL, + 0xB8E1AFEDUL, 0x6A267E96UL, 0xBA7C9045UL, 0xF12C7F99UL, + 0x24A19947UL, 0xB3916CF7UL, 0x0801F2E2UL, 0x858EFC16UL, + 0x636920D8UL, 0x71574E69UL, 0xA458FEA3UL, 0xF4933D7EUL, + 0x0D95748FUL, 0x728EB658UL, 0x718BCD58UL, 0x82154AEEUL, + 0x7B54A41DUL, 0xC25A59B5UL, 0x9C30D539UL, 0x2AF26013UL, + 0xC5D1B023UL, 0x286085F0UL, 0xCA417918UL, 0xB8DB38EFUL, + 0x8E79DCB0UL, 0x603A180EUL, 0x6C9E0E8BUL, 0xB01E8A3EUL, + 0xD71577C1UL, 0xBD314B27UL, 0x78AF2FDAUL, 0x55605C60UL, + 0xE65525F3UL, 0xAA55AB94UL, 0x57489862UL, 0x63E81440UL, + 0x55CA396AUL, 0x2AAB10B6UL, 0xB4CC5C34UL, 0x1141E8CEUL, + 0xA15486AFUL, 0x7C72E993UL, 0xB3EE1411UL, 0x636FBC2AUL, + 0x2BA9C55DUL, 0x741831F6UL, 0xCE5C3E16UL, 0x9B87931EUL, + 0xAFD6BA33UL, 0x6C24CF5CUL, 0x7A325381UL, 0x28958677UL, + 0x3B8F4898UL, 0x6B4BB9AFUL, 0xC4BFE81BUL, 0x66282193UL, + 0x61D809CCUL, 0xFB21A991UL, 0x487CAC60UL, 0x5DEC8032UL, + 0xEF845D5DUL, 0xE98575B1UL, 0xDC262302UL, 0xEB651B88UL, + 0x23893E81UL, 0xD396ACC5UL, 0x0F6D6FF3UL, 0x83F44239UL, + 0x2E0B4482UL, 0xA4842004UL, 0x69C8F04AUL, 0x9E1F9B5EUL, + 0x21C66842UL, 0xF6E96C9AUL, 0x670C9C61UL, 0xABD388F0UL, + 0x6A51A0D2UL, 0xD8542F68UL, 0x960FA728UL, 0xAB5133A3UL, + 0x6EEF0B6CUL, 0x137A3BE4UL, 0xBA3BF050UL, 0x7EFB2A98UL, + 0xA1F1651DUL, 0x39AF0176UL, 0x66CA593EUL, 0x82430E88UL, + 0x8CEE8619UL, 0x456F9FB4UL, 0x7D84A5C3UL, 0x3B8B5EBEUL, + 0xE06F75D8UL, 0x85C12073UL, 0x401A449FUL, 0x56C16AA6UL, + 0x4ED3AA62UL, 0x363F7706UL, 0x1BFEDF72UL, 0x429B023DUL, + 0x37D0D724UL, 0xD00A1248UL, 0xDB0FEAD3UL, 0x49F1C09BUL, + 0x075372C9UL, 0x80991B7BUL, 0x25D479D8UL, 0xF6E8DEF7UL, + 0xE3FE501AUL, 0xB6794C3BUL, 0x976CE0BDUL, 0x04C006BAUL, + 0xC1A94FB6UL, 0x409F60C4UL, 0x5E5C9EC2UL, 0x196A2463UL, + 0x68FB6FAFUL, 0x3E6C53B5UL, 0x1339B2EBUL, 0x3B52EC6FUL, + 0x6DFC511FUL, 0x9B30952CUL, 0xCC814544UL, 0xAF5EBD09UL, + 0xBEE3D004UL, 0xDE334AFDUL, 0x660F2807UL, 0x192E4BB3UL, + 0xC0CBA857UL, 0x45C8740FUL, 0xD20B5F39UL, 0xB9D3FBDBUL, + 0x5579C0BDUL, 0x1A60320AUL, 0xD6A100C6UL, 0x402C7279UL, + 0x679F25FEUL, 0xFB1FA3CCUL, 0x8EA5E9F8UL, 0xDB3222F8UL, + 0x3C7516DFUL, 0xFD616B15UL, 0x2F501EC8UL, 0xAD0552ABUL, + 0x323DB5FAUL, 0xFD238760UL, 0x53317B48UL, 0x3E00DF82UL, + 0x9E5C57BBUL, 0xCA6F8CA0UL, 0x1A87562EUL, 0xDF1769DBUL, + 0xD542A8F6UL, 0x287EFFC3UL, 0xAC6732C6UL, 0x8C4F5573UL, + 0x695B27B0UL, 0xBBCA58C8UL, 0xE1FFA35DUL, 0xB8F011A0UL, + 0x10FA3D98UL, 0xFD2183B8UL, 0x4AFCB56CUL, 0x2DD1D35BUL, + 0x9A53E479UL, 0xB6F84565UL, 0xD28E49BCUL, 0x4BFB9790UL, + 0xE1DDF2DAUL, 0xA4CB7E33UL, 0x62FB1341UL, 0xCEE4C6E8UL, + 0xEF20CADAUL, 0x36774C01UL, 0xD07E9EFEUL, 0x2BF11FB4UL, + 0x95DBDA4DUL, 0xAE909198UL, 0xEAAD8E71UL, 0x6B93D5A0UL, + 0xD08ED1D0UL, 0xAFC725E0UL, 0x8E3C5B2FUL, 0x8E7594B7UL, + 0x8FF6E2FBUL, 0xF2122B64UL, 0x8888B812UL, 0x900DF01CUL, + 0x4FAD5EA0UL, 0x688FC31CUL, 0xD1CFF191UL, 0xB3A8C1ADUL, + 0x2F2F2218UL, 0xBE0E1777UL, 0xEA752DFEUL, 0x8B021FA1UL, + 0xE5A0CC0FUL, 0xB56F74E8UL, 0x18ACF3D6UL, 0xCE89E299UL, + 0xB4A84FE0UL, 0xFD13E0B7UL, 0x7CC43B81UL, 0xD2ADA8D9UL, + 0x165FA266UL, 0x80957705UL, 0x93CC7314UL, 0x211A1477UL, + 0xE6AD2065UL, 0x77B5FA86UL, 0xC75442F5UL, 0xFB9D35CFUL, + 0xEBCDAF0CUL, 0x7B3E89A0UL, 0xD6411BD3UL, 0xAE1E7E49UL, + 0x00250E2DUL, 0x2071B35EUL, 0x226800BBUL, 0x57B8E0AFUL, + 0x2464369BUL, 0xF009B91EUL, 0x5563911DUL, 0x59DFA6AAUL, + 0x78C14389UL, 0xD95A537FUL, 0x207D5BA2UL, 0x02E5B9C5UL, + 0x83260376UL, 0x6295CFA9UL, 0x11C81968UL, 0x4E734A41UL, + 0xB3472DCAUL, 0x7B14A94AUL, 0x1B510052UL, 0x9A532915UL, + 0xD60F573FUL, 0xBC9BC6E4UL, 0x2B60A476UL, 0x81E67400UL, + 0x08BA6FB5UL, 0x571BE91FUL, 0xF296EC6BUL, 0x2A0DD915UL, + 0xB6636521UL, 0xE7B9F9B6UL, 0xFF34052EUL, 0xC5855664UL, + 0x53B02D5DUL, 0xA99F8FA1UL, 0x08BA4799UL, 0x6E85076AUL }, + { 0x4B7A70E9UL, 0xB5B32944UL, 0xDB75092EUL, 0xC4192623UL, + 0xAD6EA6B0UL, 0x49A7DF7DUL, 0x9CEE60B8UL, 0x8FEDB266UL, + 0xECAA8C71UL, 0x699A17FFUL, 0x5664526CUL, 0xC2B19EE1UL, + 0x193602A5UL, 0x75094C29UL, 0xA0591340UL, 0xE4183A3EUL, + 0x3F54989AUL, 0x5B429D65UL, 0x6B8FE4D6UL, 0x99F73FD6UL, + 0xA1D29C07UL, 0xEFE830F5UL, 0x4D2D38E6UL, 0xF0255DC1UL, + 0x4CDD2086UL, 0x8470EB26UL, 0x6382E9C6UL, 0x021ECC5EUL, + 0x09686B3FUL, 0x3EBAEFC9UL, 0x3C971814UL, 0x6B6A70A1UL, + 0x687F3584UL, 0x52A0E286UL, 0xB79C5305UL, 0xAA500737UL, + 0x3E07841CUL, 0x7FDEAE5CUL, 0x8E7D44ECUL, 0x5716F2B8UL, + 0xB03ADA37UL, 0xF0500C0DUL, 0xF01C1F04UL, 0x0200B3FFUL, + 0xAE0CF51AUL, 0x3CB574B2UL, 0x25837A58UL, 0xDC0921BDUL, + 0xD19113F9UL, 0x7CA92FF6UL, 0x94324773UL, 0x22F54701UL, + 0x3AE5E581UL, 0x37C2DADCUL, 0xC8B57634UL, 0x9AF3DDA7UL, + 0xA9446146UL, 0x0FD0030EUL, 0xECC8C73EUL, 0xA4751E41UL, + 0xE238CD99UL, 0x3BEA0E2FUL, 0x3280BBA1UL, 0x183EB331UL, + 0x4E548B38UL, 0x4F6DB908UL, 0x6F420D03UL, 0xF60A04BFUL, + 0x2CB81290UL, 0x24977C79UL, 0x5679B072UL, 0xBCAF89AFUL, + 0xDE9A771FUL, 0xD9930810UL, 0xB38BAE12UL, 0xDCCF3F2EUL, + 0x5512721FUL, 0x2E6B7124UL, 0x501ADDE6UL, 0x9F84CD87UL, + 0x7A584718UL, 0x7408DA17UL, 0xBC9F9ABCUL, 0xE94B7D8CUL, + 0xEC7AEC3AUL, 0xDB851DFAUL, 0x63094366UL, 0xC464C3D2UL, + 0xEF1C1847UL, 0x3215D908UL, 0xDD433B37UL, 0x24C2BA16UL, + 0x12A14D43UL, 0x2A65C451UL, 0x50940002UL, 0x133AE4DDUL, + 0x71DFF89EUL, 0x10314E55UL, 0x81AC77D6UL, 0x5F11199BUL, + 0x043556F1UL, 0xD7A3C76BUL, 0x3C11183BUL, 0x5924A509UL, + 0xF28FE6EDUL, 0x97F1FBFAUL, 0x9EBABF2CUL, 0x1E153C6EUL, + 0x86E34570UL, 0xEAE96FB1UL, 0x860E5E0AUL, 0x5A3E2AB3UL, + 0x771FE71CUL, 0x4E3D06FAUL, 0x2965DCB9UL, 0x99E71D0FUL, + 0x803E89D6UL, 0x5266C825UL, 0x2E4CC978UL, 0x9C10B36AUL, + 0xC6150EBAUL, 0x94E2EA78UL, 0xA5FC3C53UL, 0x1E0A2DF4UL, + 0xF2F74EA7UL, 0x361D2B3DUL, 0x1939260FUL, 0x19C27960UL, + 0x5223A708UL, 0xF71312B6UL, 0xEBADFE6EUL, 0xEAC31F66UL, + 0xE3BC4595UL, 0xA67BC883UL, 0xB17F37D1UL, 0x018CFF28UL, + 0xC332DDEFUL, 0xBE6C5AA5UL, 0x65582185UL, 0x68AB9802UL, + 0xEECEA50FUL, 0xDB2F953BUL, 0x2AEF7DADUL, 0x5B6E2F84UL, + 0x1521B628UL, 0x29076170UL, 0xECDD4775UL, 0x619F1510UL, + 0x13CCA830UL, 0xEB61BD96UL, 0x0334FE1EUL, 0xAA0363CFUL, + 0xB5735C90UL, 0x4C70A239UL, 0xD59E9E0BUL, 0xCBAADE14UL, + 0xEECC86BCUL, 0x60622CA7UL, 0x9CAB5CABUL, 0xB2F3846EUL, + 0x648B1EAFUL, 0x19BDF0CAUL, 0xA02369B9UL, 0x655ABB50UL, + 0x40685A32UL, 0x3C2AB4B3UL, 0x319EE9D5UL, 0xC021B8F7UL, + 0x9B540B19UL, 0x875FA099UL, 0x95F7997EUL, 0x623D7DA8UL, + 0xF837889AUL, 0x97E32D77UL, 0x11ED935FUL, 0x16681281UL, + 0x0E358829UL, 0xC7E61FD6UL, 0x96DEDFA1UL, 0x7858BA99UL, + 0x57F584A5UL, 0x1B227263UL, 0x9B83C3FFUL, 0x1AC24696UL, + 0xCDB30AEBUL, 0x532E3054UL, 0x8FD948E4UL, 0x6DBC3128UL, + 0x58EBF2EFUL, 0x34C6FFEAUL, 0xFE28ED61UL, 0xEE7C3C73UL, + 0x5D4A14D9UL, 0xE864B7E3UL, 0x42105D14UL, 0x203E13E0UL, + 0x45EEE2B6UL, 0xA3AAABEAUL, 0xDB6C4F15UL, 0xFACB4FD0UL, + 0xC742F442UL, 0xEF6ABBB5UL, 0x654F3B1DUL, 0x41CD2105UL, + 0xD81E799EUL, 0x86854DC7UL, 0xE44B476AUL, 0x3D816250UL, + 0xCF62A1F2UL, 0x5B8D2646UL, 0xFC8883A0UL, 0xC1C7B6A3UL, + 0x7F1524C3UL, 0x69CB7492UL, 0x47848A0BUL, 0x5692B285UL, + 0x095BBF00UL, 0xAD19489DUL, 0x1462B174UL, 0x23820E00UL, + 0x58428D2AUL, 0x0C55F5EAUL, 0x1DADF43EUL, 0x233F7061UL, + 0x3372F092UL, 0x8D937E41UL, 0xD65FECF1UL, 0x6C223BDBUL, + 0x7CDE3759UL, 0xCBEE7460UL, 0x4085F2A7UL, 0xCE77326EUL, + 0xA6078084UL, 0x19F8509EUL, 0xE8EFD855UL, 0x61D99735UL, + 0xA969A7AAUL, 0xC50C06C2UL, 0x5A04ABFCUL, 0x800BCADCUL, + 0x9E447A2EUL, 0xC3453484UL, 0xFDD56705UL, 0x0E1E9EC9UL, + 0xDB73DBD3UL, 0x105588CDUL, 0x675FDA79UL, 0xE3674340UL, + 0xC5C43465UL, 0x713E38D8UL, 0x3D28F89EUL, 0xF16DFF20UL, + 0x153E21E7UL, 0x8FB03D4AUL, 0xE6E39F2BUL, 0xDB83ADF7UL }, + { 0xE93D5A68UL, 0x948140F7UL, 0xF64C261CUL, 0x94692934UL, + 0x411520F7UL, 0x7602D4F7UL, 0xBCF46B2EUL, 0xD4A20068UL, + 0xD4082471UL, 0x3320F46AUL, 0x43B7D4B7UL, 0x500061AFUL, + 0x1E39F62EUL, 0x97244546UL, 0x14214F74UL, 0xBF8B8840UL, + 0x4D95FC1DUL, 0x96B591AFUL, 0x70F4DDD3UL, 0x66A02F45UL, + 0xBFBC09ECUL, 0x03BD9785UL, 0x7FAC6DD0UL, 0x31CB8504UL, + 0x96EB27B3UL, 0x55FD3941UL, 0xDA2547E6UL, 0xABCA0A9AUL, + 0x28507825UL, 0x530429F4UL, 0x0A2C86DAUL, 0xE9B66DFBUL, + 0x68DC1462UL, 0xD7486900UL, 0x680EC0A4UL, 0x27A18DEEUL, + 0x4F3FFEA2UL, 0xE887AD8CUL, 0xB58CE006UL, 0x7AF4D6B6UL, + 0xAACE1E7CUL, 0xD3375FECUL, 0xCE78A399UL, 0x406B2A42UL, + 0x20FE9E35UL, 0xD9F385B9UL, 0xEE39D7ABUL, 0x3B124E8BUL, + 0x1DC9FAF7UL, 0x4B6D1856UL, 0x26A36631UL, 0xEAE397B2UL, + 0x3A6EFA74UL, 0xDD5B4332UL, 0x6841E7F7UL, 0xCA7820FBUL, + 0xFB0AF54EUL, 0xD8FEB397UL, 0x454056ACUL, 0xBA489527UL, + 0x55533A3AUL, 0x20838D87UL, 0xFE6BA9B7UL, 0xD096954BUL, + 0x55A867BCUL, 0xA1159A58UL, 0xCCA92963UL, 0x99E1DB33UL, + 0xA62A4A56UL, 0x3F3125F9UL, 0x5EF47E1CUL, 0x9029317CUL, + 0xFDF8E802UL, 0x04272F70UL, 0x80BB155CUL, 0x05282CE3UL, + 0x95C11548UL, 0xE4C66D22UL, 0x48C1133FUL, 0xC70F86DCUL, + 0x07F9C9EEUL, 0x41041F0FUL, 0x404779A4UL, 0x5D886E17UL, + 0x325F51EBUL, 0xD59BC0D1UL, 0xF2BCC18FUL, 0x41113564UL, + 0x257B7834UL, 0x602A9C60UL, 0xDFF8E8A3UL, 0x1F636C1BUL, + 0x0E12B4C2UL, 0x02E1329EUL, 0xAF664FD1UL, 0xCAD18115UL, + 0x6B2395E0UL, 0x333E92E1UL, 0x3B240B62UL, 0xEEBEB922UL, + 0x85B2A20EUL, 0xE6BA0D99UL, 0xDE720C8CUL, 0x2DA2F728UL, + 0xD0127845UL, 0x95B794FDUL, 0x647D0862UL, 0xE7CCF5F0UL, + 0x5449A36FUL, 0x877D48FAUL, 0xC39DFD27UL, 0xF33E8D1EUL, + 0x0A476341UL, 0x992EFF74UL, 0x3A6F6EABUL, 0xF4F8FD37UL, + 0xA812DC60UL, 0xA1EBDDF8UL, 0x991BE14CUL, 0xDB6E6B0DUL, + 0xC67B5510UL, 0x6D672C37UL, 0x2765D43BUL, 0xDCD0E804UL, + 0xF1290DC7UL, 0xCC00FFA3UL, 0xB5390F92UL, 0x690FED0BUL, + 0x667B9FFBUL, 0xCEDB7D9CUL, 0xA091CF0BUL, 0xD9155EA3UL, + 0xBB132F88UL, 0x515BAD24UL, 0x7B9479BFUL, 0x763BD6EBUL, + 0x37392EB3UL, 0xCC115979UL, 0x8026E297UL, 0xF42E312DUL, + 0x6842ADA7UL, 0xC66A2B3BUL, 0x12754CCCUL, 0x782EF11CUL, + 0x6A124237UL, 0xB79251E7UL, 0x06A1BBE6UL, 0x4BFB6350UL, + 0x1A6B1018UL, 0x11CAEDFAUL, 0x3D25BDD8UL, 0xE2E1C3C9UL, + 0x44421659UL, 0x0A121386UL, 0xD90CEC6EUL, 0xD5ABEA2AUL, + 0x64AF674EUL, 0xDA86A85FUL, 0xBEBFE988UL, 0x64E4C3FEUL, + 0x9DBC8057UL, 0xF0F7C086UL, 0x60787BF8UL, 0x6003604DUL, + 0xD1FD8346UL, 0xF6381FB0UL, 0x7745AE04UL, 0xD736FCCCUL, + 0x83426B33UL, 0xF01EAB71UL, 0xB0804187UL, 0x3C005E5FUL, + 0x77A057BEUL, 0xBDE8AE24UL, 0x55464299UL, 0xBF582E61UL, + 0x4E58F48FUL, 0xF2DDFDA2UL, 0xF474EF38UL, 0x8789BDC2UL, + 0x5366F9C3UL, 0xC8B38E74UL, 0xB475F255UL, 0x46FCD9B9UL, + 0x7AEB2661UL, 0x8B1DDF84UL, 0x846A0E79UL, 0x915F95E2UL, + 0x466E598EUL, 0x20B45770UL, 0x8CD55591UL, 0xC902DE4CUL, + 0xB90BACE1UL, 0xBB8205D0UL, 0x11A86248UL, 0x7574A99EUL, + 0xB77F19B6UL, 0xE0A9DC09UL, 0x662D09A1UL, 0xC4324633UL, + 0xE85A1F02UL, 0x09F0BE8CUL, 0x4A99A025UL, 0x1D6EFE10UL, + 0x1AB93D1DUL, 0x0BA5A4DFUL, 0xA186F20FUL, 0x2868F169UL, + 0xDCB7DA83UL, 0x573906FEUL, 0xA1E2CE9BUL, 0x4FCD7F52UL, + 0x50115E01UL, 0xA70683FAUL, 0xA002B5C4UL, 0x0DE6D027UL, + 0x9AF88C27UL, 0x773F8641UL, 0xC3604C06UL, 0x61A806B5UL, + 0xF0177A28UL, 0xC0F586E0UL, 0x006058AAUL, 0x30DC7D62UL, + 0x11E69ED7UL, 0x2338EA63UL, 0x53C2DD94UL, 0xC2C21634UL, + 0xBBCBEE56UL, 0x90BCB6DEUL, 0xEBFC7DA1UL, 0xCE591D76UL, + 0x6F05E409UL, 0x4B7C0188UL, 0x39720A3DUL, 0x7C927C24UL, + 0x86E3725FUL, 0x724D9DB9UL, 0x1AC15BB4UL, 0xD39EB8FCUL, + 0xED545578UL, 0x08FCA5B5UL, 0xD83D7CD3UL, 0x4DAD0FC4UL, + 0x1E50EF5EUL, 0xB161E6F8UL, 0xA28514D9UL, 0x6C51133CUL, + 0x6FD5C7E7UL, 0x56E14EC4UL, 0x362ABFCEUL, 0xDDC6C837UL, + 0xD79A3234UL, 0x92638212UL, 0x670EFA8EUL, 0x406000E0UL }, + { 0x3A39CE37UL, 0xD3FAF5CFUL, 0xABC27737UL, 0x5AC52D1BUL, + 0x5CB0679EUL, 0x4FA33742UL, 0xD3822740UL, 0x99BC9BBEUL, + 0xD5118E9DUL, 0xBF0F7315UL, 0xD62D1C7EUL, 0xC700C47BUL, + 0xB78C1B6BUL, 0x21A19045UL, 0xB26EB1BEUL, 0x6A366EB4UL, + 0x5748AB2FUL, 0xBC946E79UL, 0xC6A376D2UL, 0x6549C2C8UL, + 0x530FF8EEUL, 0x468DDE7DUL, 0xD5730A1DUL, 0x4CD04DC6UL, + 0x2939BBDBUL, 0xA9BA4650UL, 0xAC9526E8UL, 0xBE5EE304UL, + 0xA1FAD5F0UL, 0x6A2D519AUL, 0x63EF8CE2UL, 0x9A86EE22UL, + 0xC089C2B8UL, 0x43242EF6UL, 0xA51E03AAUL, 0x9CF2D0A4UL, + 0x83C061BAUL, 0x9BE96A4DUL, 0x8FE51550UL, 0xBA645BD6UL, + 0x2826A2F9UL, 0xA73A3AE1UL, 0x4BA99586UL, 0xEF5562E9UL, + 0xC72FEFD3UL, 0xF752F7DAUL, 0x3F046F69UL, 0x77FA0A59UL, + 0x80E4A915UL, 0x87B08601UL, 0x9B09E6ADUL, 0x3B3EE593UL, + 0xE990FD5AUL, 0x9E34D797UL, 0x2CF0B7D9UL, 0x022B8B51UL, + 0x96D5AC3AUL, 0x017DA67DUL, 0xD1CF3ED6UL, 0x7C7D2D28UL, + 0x1F9F25CFUL, 0xADF2B89BUL, 0x5AD6B472UL, 0x5A88F54CUL, + 0xE029AC71UL, 0xE019A5E6UL, 0x47B0ACFDUL, 0xED93FA9BUL, + 0xE8D3C48DUL, 0x283B57CCUL, 0xF8D56629UL, 0x79132E28UL, + 0x785F0191UL, 0xED756055UL, 0xF7960E44UL, 0xE3D35E8CUL, + 0x15056DD4UL, 0x88F46DBAUL, 0x03A16125UL, 0x0564F0BDUL, + 0xC3EB9E15UL, 0x3C9057A2UL, 0x97271AECUL, 0xA93A072AUL, + 0x1B3F6D9BUL, 0x1E6321F5UL, 0xF59C66FBUL, 0x26DCF319UL, + 0x7533D928UL, 0xB155FDF5UL, 0x03563482UL, 0x8ABA3CBBUL, + 0x28517711UL, 0xC20AD9F8UL, 0xABCC5167UL, 0xCCAD925FUL, + 0x4DE81751UL, 0x3830DC8EUL, 0x379D5862UL, 0x9320F991UL, + 0xEA7A90C2UL, 0xFB3E7BCEUL, 0x5121CE64UL, 0x774FBE32UL, + 0xA8B6E37EUL, 0xC3293D46UL, 0x48DE5369UL, 0x6413E680UL, + 0xA2AE0810UL, 0xDD6DB224UL, 0x69852DFDUL, 0x09072166UL, + 0xB39A460AUL, 0x6445C0DDUL, 0x586CDECFUL, 0x1C20C8AEUL, + 0x5BBEF7DDUL, 0x1B588D40UL, 0xCCD2017FUL, 0x6BB4E3BBUL, + 0xDDA26A7EUL, 0x3A59FF45UL, 0x3E350A44UL, 0xBCB4CDD5UL, + 0x72EACEA8UL, 0xFA6484BBUL, 0x8D6612AEUL, 0xBF3C6F47UL, + 0xD29BE463UL, 0x542F5D9EUL, 0xAEC2771BUL, 0xF64E6370UL, + 0x740E0D8DUL, 0xE75B1357UL, 0xF8721671UL, 0xAF537D5DUL, + 0x4040CB08UL, 0x4EB4E2CCUL, 0x34D2466AUL, 0x0115AF84UL, + 0xE1B00428UL, 0x95983A1DUL, 0x06B89FB4UL, 0xCE6EA048UL, + 0x6F3F3B82UL, 0x3520AB82UL, 0x011A1D4BUL, 0x277227F8UL, + 0x611560B1UL, 0xE7933FDCUL, 0xBB3A792BUL, 0x344525BDUL, + 0xA08839E1UL, 0x51CE794BUL, 0x2F32C9B7UL, 0xA01FBAC9UL, + 0xE01CC87EUL, 0xBCC7D1F6UL, 0xCF0111C3UL, 0xA1E8AAC7UL, + 0x1A908749UL, 0xD44FBD9AUL, 0xD0DADECBUL, 0xD50ADA38UL, + 0x0339C32AUL, 0xC6913667UL, 0x8DF9317CUL, 0xE0B12B4FUL, + 0xF79E59B7UL, 0x43F5BB3AUL, 0xF2D519FFUL, 0x27D9459CUL, + 0xBF97222CUL, 0x15E6FC2AUL, 0x0F91FC71UL, 0x9B941525UL, + 0xFAE59361UL, 0xCEB69CEBUL, 0xC2A86459UL, 0x12BAA8D1UL, + 0xB6C1075EUL, 0xE3056A0CUL, 0x10D25065UL, 0xCB03A442UL, + 0xE0EC6E0EUL, 0x1698DB3BUL, 0x4C98A0BEUL, 0x3278E964UL, + 0x9F1F9532UL, 0xE0D392DFUL, 0xD3A0342BUL, 0x8971F21EUL, + 0x1B0A7441UL, 0x4BA3348CUL, 0xC5BE7120UL, 0xC37632D8UL, + 0xDF359F8DUL, 0x9B992F2EUL, 0xE60B6F47UL, 0x0FE3F11DUL, + 0xE54CDA54UL, 0x1EDAD891UL, 0xCE6279CFUL, 0xCD3E7E6FUL, + 0x1618B166UL, 0xFD2C1D05UL, 0x848FD2C5UL, 0xF6FB2299UL, + 0xF523F357UL, 0xA6327623UL, 0x93A83531UL, 0x56CCCD02UL, + 0xACF08162UL, 0x5A75EBB5UL, 0x6E163697UL, 0x88D273CCUL, + 0xDE966292UL, 0x81B949D0UL, 0x4C50901BUL, 0x71C65614UL, + 0xE6C6C7BDUL, 0x327A140AUL, 0x45E1D006UL, 0xC3F27B9AUL, + 0xC9AA53FDUL, 0x62A80F00UL, 0xBB25BFE2UL, 0x35BDD2F6UL, + 0x71126905UL, 0xB2040222UL, 0xB6CBCF7CUL, 0xCD769C2BUL, + 0x53113EC0UL, 0x1640E3D3UL, 0x38ABBD60UL, 0x2547ADF0UL, + 0xBA38209CUL, 0xF746CE76UL, 0x77AFA1C5UL, 0x20756060UL, + 0x85CBFE4EUL, 0x8AE88DD8UL, 0x7AAAF9B0UL, 0x4CF9AA7EUL, + 0x1948C25CUL, 0x02FB8A8CUL, 0x01C36AE4UL, 0xD6EBE1F9UL, + 0x90D4F869UL, 0xA65CDEA0UL, 0x3F09252DUL, 0xC208E69FUL, + 0xB74E6132UL, 0xCE77E25BUL, 0x578FDFE3UL, 0x3AC372E6UL } +}; + +#ifndef __GNUC__ +#define F(x) ((S1[LTC_BYTE(x,3)] + S2[LTC_BYTE(x,2)]) ^ S3[LTC_BYTE(x,1)]) + S4[LTC_BYTE(x,0)] +#else +#define F(x) ((skey->blowfish.S[0][LTC_BYTE(x,3)] + skey->blowfish.S[1][LTC_BYTE(x,2)]) ^ skey->blowfish.S[2][LTC_BYTE(x,1)]) + skey->blowfish.S[3][LTC_BYTE(x,0)] +#endif + +static void s_blowfish_encipher(ulong32 *L, ulong32 *R, const symmetric_key *skey) +{ + int rounds; + + ulong32 l, r; +#ifndef __GNUC__ + const ulong32 *S1, *S2, *S3, *S4; + + S1 = skey->blowfish.S[0]; + S2 = skey->blowfish.S[1]; + S3 = skey->blowfish.S[2]; + S4 = skey->blowfish.S[3]; +#endif + + l = *L; + r = *R; + + /* do 16 rounds */ + for (rounds = 0; rounds < 16; ) { + l ^= skey->blowfish.K[rounds++]; r ^= F(l); + r ^= skey->blowfish.K[rounds++]; l ^= F(r); + l ^= skey->blowfish.K[rounds++]; r ^= F(l); + r ^= skey->blowfish.K[rounds++]; l ^= F(r); + } + + /* last keying */ + l ^= skey->blowfish.K[16]; + r ^= skey->blowfish.K[17]; + + *L = r; + *R = l; +} + +void blowfish_enc(ulong32 *data, unsigned long blocks, const symmetric_key *skey) +{ + unsigned long i; + ulong32 *d = data; + + for (i = 0; i < blocks; ++i) { + s_blowfish_encipher(d, d + 1, skey); + d += 2; + } +} + +static ulong32 s_blowfish_stream2word(const unsigned char *d, int dlen, int *cur) +{ + unsigned int z; + int y = *cur; + ulong32 ret = 0; + + for (z = 0; z < 4; z++) { + ret = (ret << 8) | ((ulong32)d[y++] & 255); + if (y == dlen) { + y = 0; + } + } + + *cur = y; + return ret; +} + + /** + Expand the Blowfish internal state + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param data The additional data you wish to pass (can be NULL) + @param datalen The additional data length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int blowfish_expand(const unsigned char *key, int keylen, + const unsigned char *data, int datalen, + symmetric_key *skey) +{ + ulong32 x, y, A, B[2]; + int i; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* load in key bytes (Supplied by David Hopwood) */ + i = 0; + for (x = 0; x < 18; x++) { + A = s_blowfish_stream2word(key, keylen, &i); + skey->blowfish.K[x] ^= A; + } + + + i = 0; + B[0] = 0; + B[1] = 0; + for (x = 0; x < 18; x += 2) { + if (data != NULL) { + B[0] ^= s_blowfish_stream2word(data, datalen, &i); + B[1] ^= s_blowfish_stream2word(data, datalen, &i); + } + /* encrypt it */ + s_blowfish_encipher(&B[0], &B[1], skey); + /* copy it */ + skey->blowfish.K[x] = B[0]; + skey->blowfish.K[x+1] = B[1]; + } + + /* encrypt S array */ + for (x = 0; x < 4; x++) { + for (y = 0; y < 256; y += 2) { + if (data != NULL) { + B[0] ^= s_blowfish_stream2word(data, datalen, &i); + B[1] ^= s_blowfish_stream2word(data, datalen, &i); + } + /* encrypt it */ + s_blowfish_encipher(&B[0], &B[1], skey); + /* copy it */ + skey->blowfish.S[x][y] = B[0]; + skey->blowfish.S[x][y+1] = B[1]; + } + } + +#ifdef LTC_CLEAN_STACK + zeromem(B, sizeof(B)); +#endif + + return CRYPT_OK; +} + +/** + Initialize the Blowfish block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful +*/ +int blowfish_setup(const unsigned char *key, int keylen, int num_rounds, + symmetric_key *skey) +{ + /* check key length */ + if (keylen < 8 || keylen > 56) { + return CRYPT_INVALID_KEYSIZE; + } + /* check rounds */ + if (num_rounds != 0 && num_rounds != 16) { + return CRYPT_INVALID_ROUNDS; + } + + return blowfish_setup_with_data(key, keylen, NULL, 0, skey); +} + +/** + Alternative initialize of the Blowfish block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param data The additional data you wish to pass (can be NULL) + @param datalen The additional data length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful +*/ + +int blowfish_setup_with_data(const unsigned char *key, int keylen, + const unsigned char *data, int datalen, + symmetric_key *skey) +{ + XMEMCPY(skey->blowfish.K, ORIG_P, sizeof(ORIG_P)); + XMEMCPY(skey->blowfish.S, ORIG_S, sizeof(ORIG_S)); + return blowfish_expand(key, keylen, data, datalen, skey); +} + +/** + Encrypts a block of text with Blowfish + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_blowfish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int blowfish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + ulong32 L, R; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + /* load it */ + LOAD32H(L, &pt[0]); + LOAD32H(R, &pt[4]); + + s_blowfish_encipher(&L, &R, skey); + + /* store */ + STORE32H(L, &ct[0]); + STORE32H(R, &ct[4]); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int blowfish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_blowfish_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(ulong32) * 2 + sizeof(int)); + return err; +} +#endif + +/** + Decrypts a block of text with Blowfish + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_blowfish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int blowfish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + ulong32 L, R; + int r; +#ifndef __GNUC__ + const ulong32 *S1, *S2, *S3, *S4; +#endif + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + +#ifndef __GNUC__ + S1 = skey->blowfish.S[0]; + S2 = skey->blowfish.S[1]; + S3 = skey->blowfish.S[2]; + S4 = skey->blowfish.S[3]; +#endif + + /* load it */ + LOAD32H(R, &ct[0]); + LOAD32H(L, &ct[4]); + + /* undo last keying */ + R ^= skey->blowfish.K[17]; + L ^= skey->blowfish.K[16]; + + /* do 16 rounds */ + for (r = 15; r > 0; ) { + L ^= F(R); R ^= skey->blowfish.K[r--]; + R ^= F(L); L ^= skey->blowfish.K[r--]; + L ^= F(R); R ^= skey->blowfish.K[r--]; + R ^= F(L); L ^= skey->blowfish.K[r--]; + } + + /* store */ + STORE32H(L, &pt[0]); + STORE32H(R, &pt[4]); + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int blowfish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_blowfish_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(ulong32) * 2 + sizeof(int)); + return err; +} +#endif + + +/** + Performs a self-test of the Blowfish block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int blowfish_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + int err; + symmetric_key key; + static const struct { + unsigned char key[8], pt[8], ct[8]; + } tests[] = { + { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + { 0x4E, 0xF9, 0x97, 0x45, 0x61, 0x98, 0xDD, 0x78} + }, + { + { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + { 0x51, 0x86, 0x6F, 0xD5, 0xB8, 0x5E, 0xCB, 0x8A} + }, + { + { 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + { 0x7D, 0x85, 0x6F, 0x9A, 0x61, 0x30, 0x63, 0xF2} + } + }; + unsigned char tmp[2][8]; + int x, y; + + for (x = 0; x < (int)(sizeof(tests) / sizeof(tests[0])); x++) { + /* setup key */ + if ((err = blowfish_setup(tests[x].key, 8, 16, &key)) != CRYPT_OK) { + return err; + } + + /* encrypt and decrypt */ + blowfish_ecb_encrypt(tests[x].pt, tmp[0], &key); + blowfish_ecb_decrypt(tmp[0], tmp[1], &key); + + /* compare */ + if ((compare_testvector(tmp[0], 8, tests[x].ct, 8, "Blowfish Encrypt", x) != 0) || + (compare_testvector(tmp[1], 8, tests[x].pt, 8, "Blowfish Decrypt", x) != 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) blowfish_ecb_encrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 1000; y++) blowfish_ecb_decrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 8; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + + + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void blowfish_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int blowfish_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + + if (*keysize < 8) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize > 56) { + *keysize = 56; + } + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/camellia.c b/optee_os/core/lib/libtomcrypt/src/ciphers/camellia.c new file mode 100644 index 0000000..733e963 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/camellia.c @@ -0,0 +1,720 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file camellia.c + Implementation by Tom St Denis of Elliptic Semiconductor +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_CAMELLIA + +const struct ltc_cipher_descriptor camellia_desc = { + "camellia", + 23, + 16, 32, 16, 18, + &camellia_setup, + &camellia_ecb_encrypt, + &camellia_ecb_decrypt, + &camellia_test, + &camellia_done, + &camellia_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 SP1110[] = { +0x70707000, 0x82828200, 0x2c2c2c00, 0xececec00, 0xb3b3b300, 0x27272700, 0xc0c0c000, 0xe5e5e500, +0xe4e4e400, 0x85858500, 0x57575700, 0x35353500, 0xeaeaea00, 0x0c0c0c00, 0xaeaeae00, 0x41414100, +0x23232300, 0xefefef00, 0x6b6b6b00, 0x93939300, 0x45454500, 0x19191900, 0xa5a5a500, 0x21212100, +0xededed00, 0x0e0e0e00, 0x4f4f4f00, 0x4e4e4e00, 0x1d1d1d00, 0x65656500, 0x92929200, 0xbdbdbd00, +0x86868600, 0xb8b8b800, 0xafafaf00, 0x8f8f8f00, 0x7c7c7c00, 0xebebeb00, 0x1f1f1f00, 0xcecece00, +0x3e3e3e00, 0x30303000, 0xdcdcdc00, 0x5f5f5f00, 0x5e5e5e00, 0xc5c5c500, 0x0b0b0b00, 0x1a1a1a00, +0xa6a6a600, 0xe1e1e100, 0x39393900, 0xcacaca00, 0xd5d5d500, 0x47474700, 0x5d5d5d00, 0x3d3d3d00, +0xd9d9d900, 0x01010100, 0x5a5a5a00, 0xd6d6d600, 0x51515100, 0x56565600, 0x6c6c6c00, 0x4d4d4d00, +0x8b8b8b00, 0x0d0d0d00, 0x9a9a9a00, 0x66666600, 0xfbfbfb00, 0xcccccc00, 0xb0b0b000, 0x2d2d2d00, +0x74747400, 0x12121200, 0x2b2b2b00, 0x20202000, 0xf0f0f000, 0xb1b1b100, 0x84848400, 0x99999900, +0xdfdfdf00, 0x4c4c4c00, 0xcbcbcb00, 0xc2c2c200, 0x34343400, 0x7e7e7e00, 0x76767600, 0x05050500, +0x6d6d6d00, 0xb7b7b700, 0xa9a9a900, 0x31313100, 0xd1d1d100, 0x17171700, 0x04040400, 0xd7d7d700, +0x14141400, 0x58585800, 0x3a3a3a00, 0x61616100, 0xdedede00, 0x1b1b1b00, 0x11111100, 0x1c1c1c00, +0x32323200, 0x0f0f0f00, 0x9c9c9c00, 0x16161600, 0x53535300, 0x18181800, 0xf2f2f200, 0x22222200, +0xfefefe00, 0x44444400, 0xcfcfcf00, 0xb2b2b200, 0xc3c3c300, 0xb5b5b500, 0x7a7a7a00, 0x91919100, +0x24242400, 0x08080800, 0xe8e8e800, 0xa8a8a800, 0x60606000, 0xfcfcfc00, 0x69696900, 0x50505000, +0xaaaaaa00, 0xd0d0d000, 0xa0a0a000, 0x7d7d7d00, 0xa1a1a100, 0x89898900, 0x62626200, 0x97979700, +0x54545400, 0x5b5b5b00, 0x1e1e1e00, 0x95959500, 0xe0e0e000, 0xffffff00, 0x64646400, 0xd2d2d200, +0x10101000, 0xc4c4c400, 0x00000000, 0x48484800, 0xa3a3a300, 0xf7f7f700, 0x75757500, 0xdbdbdb00, +0x8a8a8a00, 0x03030300, 0xe6e6e600, 0xdadada00, 0x09090900, 0x3f3f3f00, 0xdddddd00, 0x94949400, +0x87878700, 0x5c5c5c00, 0x83838300, 0x02020200, 0xcdcdcd00, 0x4a4a4a00, 0x90909000, 0x33333300, +0x73737300, 0x67676700, 0xf6f6f600, 0xf3f3f300, 0x9d9d9d00, 0x7f7f7f00, 0xbfbfbf00, 0xe2e2e200, +0x52525200, 0x9b9b9b00, 0xd8d8d800, 0x26262600, 0xc8c8c800, 0x37373700, 0xc6c6c600, 0x3b3b3b00, +0x81818100, 0x96969600, 0x6f6f6f00, 0x4b4b4b00, 0x13131300, 0xbebebe00, 0x63636300, 0x2e2e2e00, +0xe9e9e900, 0x79797900, 0xa7a7a700, 0x8c8c8c00, 0x9f9f9f00, 0x6e6e6e00, 0xbcbcbc00, 0x8e8e8e00, +0x29292900, 0xf5f5f500, 0xf9f9f900, 0xb6b6b600, 0x2f2f2f00, 0xfdfdfd00, 0xb4b4b400, 0x59595900, +0x78787800, 0x98989800, 0x06060600, 0x6a6a6a00, 0xe7e7e700, 0x46464600, 0x71717100, 0xbababa00, +0xd4d4d400, 0x25252500, 0xababab00, 0x42424200, 0x88888800, 0xa2a2a200, 0x8d8d8d00, 0xfafafa00, +0x72727200, 0x07070700, 0xb9b9b900, 0x55555500, 0xf8f8f800, 0xeeeeee00, 0xacacac00, 0x0a0a0a00, +0x36363600, 0x49494900, 0x2a2a2a00, 0x68686800, 0x3c3c3c00, 0x38383800, 0xf1f1f100, 0xa4a4a400, +0x40404000, 0x28282800, 0xd3d3d300, 0x7b7b7b00, 0xbbbbbb00, 0xc9c9c900, 0x43434300, 0xc1c1c100, +0x15151500, 0xe3e3e300, 0xadadad00, 0xf4f4f400, 0x77777700, 0xc7c7c700, 0x80808000, 0x9e9e9e00, +}; + +static const ulong32 SP0222[] = { +0x00e0e0e0, 0x00050505, 0x00585858, 0x00d9d9d9, 0x00676767, 0x004e4e4e, 0x00818181, 0x00cbcbcb, +0x00c9c9c9, 0x000b0b0b, 0x00aeaeae, 0x006a6a6a, 0x00d5d5d5, 0x00181818, 0x005d5d5d, 0x00828282, +0x00464646, 0x00dfdfdf, 0x00d6d6d6, 0x00272727, 0x008a8a8a, 0x00323232, 0x004b4b4b, 0x00424242, +0x00dbdbdb, 0x001c1c1c, 0x009e9e9e, 0x009c9c9c, 0x003a3a3a, 0x00cacaca, 0x00252525, 0x007b7b7b, +0x000d0d0d, 0x00717171, 0x005f5f5f, 0x001f1f1f, 0x00f8f8f8, 0x00d7d7d7, 0x003e3e3e, 0x009d9d9d, +0x007c7c7c, 0x00606060, 0x00b9b9b9, 0x00bebebe, 0x00bcbcbc, 0x008b8b8b, 0x00161616, 0x00343434, +0x004d4d4d, 0x00c3c3c3, 0x00727272, 0x00959595, 0x00ababab, 0x008e8e8e, 0x00bababa, 0x007a7a7a, +0x00b3b3b3, 0x00020202, 0x00b4b4b4, 0x00adadad, 0x00a2a2a2, 0x00acacac, 0x00d8d8d8, 0x009a9a9a, +0x00171717, 0x001a1a1a, 0x00353535, 0x00cccccc, 0x00f7f7f7, 0x00999999, 0x00616161, 0x005a5a5a, +0x00e8e8e8, 0x00242424, 0x00565656, 0x00404040, 0x00e1e1e1, 0x00636363, 0x00090909, 0x00333333, +0x00bfbfbf, 0x00989898, 0x00979797, 0x00858585, 0x00686868, 0x00fcfcfc, 0x00ececec, 0x000a0a0a, +0x00dadada, 0x006f6f6f, 0x00535353, 0x00626262, 0x00a3a3a3, 0x002e2e2e, 0x00080808, 0x00afafaf, +0x00282828, 0x00b0b0b0, 0x00747474, 0x00c2c2c2, 0x00bdbdbd, 0x00363636, 0x00222222, 0x00383838, +0x00646464, 0x001e1e1e, 0x00393939, 0x002c2c2c, 0x00a6a6a6, 0x00303030, 0x00e5e5e5, 0x00444444, +0x00fdfdfd, 0x00888888, 0x009f9f9f, 0x00656565, 0x00878787, 0x006b6b6b, 0x00f4f4f4, 0x00232323, +0x00484848, 0x00101010, 0x00d1d1d1, 0x00515151, 0x00c0c0c0, 0x00f9f9f9, 0x00d2d2d2, 0x00a0a0a0, +0x00555555, 0x00a1a1a1, 0x00414141, 0x00fafafa, 0x00434343, 0x00131313, 0x00c4c4c4, 0x002f2f2f, +0x00a8a8a8, 0x00b6b6b6, 0x003c3c3c, 0x002b2b2b, 0x00c1c1c1, 0x00ffffff, 0x00c8c8c8, 0x00a5a5a5, +0x00202020, 0x00898989, 0x00000000, 0x00909090, 0x00474747, 0x00efefef, 0x00eaeaea, 0x00b7b7b7, +0x00151515, 0x00060606, 0x00cdcdcd, 0x00b5b5b5, 0x00121212, 0x007e7e7e, 0x00bbbbbb, 0x00292929, +0x000f0f0f, 0x00b8b8b8, 0x00070707, 0x00040404, 0x009b9b9b, 0x00949494, 0x00212121, 0x00666666, +0x00e6e6e6, 0x00cecece, 0x00ededed, 0x00e7e7e7, 0x003b3b3b, 0x00fefefe, 0x007f7f7f, 0x00c5c5c5, +0x00a4a4a4, 0x00373737, 0x00b1b1b1, 0x004c4c4c, 0x00919191, 0x006e6e6e, 0x008d8d8d, 0x00767676, +0x00030303, 0x002d2d2d, 0x00dedede, 0x00969696, 0x00262626, 0x007d7d7d, 0x00c6c6c6, 0x005c5c5c, +0x00d3d3d3, 0x00f2f2f2, 0x004f4f4f, 0x00191919, 0x003f3f3f, 0x00dcdcdc, 0x00797979, 0x001d1d1d, +0x00525252, 0x00ebebeb, 0x00f3f3f3, 0x006d6d6d, 0x005e5e5e, 0x00fbfbfb, 0x00696969, 0x00b2b2b2, +0x00f0f0f0, 0x00313131, 0x000c0c0c, 0x00d4d4d4, 0x00cfcfcf, 0x008c8c8c, 0x00e2e2e2, 0x00757575, +0x00a9a9a9, 0x004a4a4a, 0x00575757, 0x00848484, 0x00111111, 0x00454545, 0x001b1b1b, 0x00f5f5f5, +0x00e4e4e4, 0x000e0e0e, 0x00737373, 0x00aaaaaa, 0x00f1f1f1, 0x00dddddd, 0x00595959, 0x00141414, +0x006c6c6c, 0x00929292, 0x00545454, 0x00d0d0d0, 0x00787878, 0x00707070, 0x00e3e3e3, 0x00494949, +0x00808080, 0x00505050, 0x00a7a7a7, 0x00f6f6f6, 0x00777777, 0x00939393, 0x00868686, 0x00838383, +0x002a2a2a, 0x00c7c7c7, 0x005b5b5b, 0x00e9e9e9, 0x00eeeeee, 0x008f8f8f, 0x00010101, 0x003d3d3d, +}; + +static const ulong32 SP3033[] = { +0x38003838, 0x41004141, 0x16001616, 0x76007676, 0xd900d9d9, 0x93009393, 0x60006060, 0xf200f2f2, +0x72007272, 0xc200c2c2, 0xab00abab, 0x9a009a9a, 0x75007575, 0x06000606, 0x57005757, 0xa000a0a0, +0x91009191, 0xf700f7f7, 0xb500b5b5, 0xc900c9c9, 0xa200a2a2, 0x8c008c8c, 0xd200d2d2, 0x90009090, +0xf600f6f6, 0x07000707, 0xa700a7a7, 0x27002727, 0x8e008e8e, 0xb200b2b2, 0x49004949, 0xde00dede, +0x43004343, 0x5c005c5c, 0xd700d7d7, 0xc700c7c7, 0x3e003e3e, 0xf500f5f5, 0x8f008f8f, 0x67006767, +0x1f001f1f, 0x18001818, 0x6e006e6e, 0xaf00afaf, 0x2f002f2f, 0xe200e2e2, 0x85008585, 0x0d000d0d, +0x53005353, 0xf000f0f0, 0x9c009c9c, 0x65006565, 0xea00eaea, 0xa300a3a3, 0xae00aeae, 0x9e009e9e, +0xec00ecec, 0x80008080, 0x2d002d2d, 0x6b006b6b, 0xa800a8a8, 0x2b002b2b, 0x36003636, 0xa600a6a6, +0xc500c5c5, 0x86008686, 0x4d004d4d, 0x33003333, 0xfd00fdfd, 0x66006666, 0x58005858, 0x96009696, +0x3a003a3a, 0x09000909, 0x95009595, 0x10001010, 0x78007878, 0xd800d8d8, 0x42004242, 0xcc00cccc, +0xef00efef, 0x26002626, 0xe500e5e5, 0x61006161, 0x1a001a1a, 0x3f003f3f, 0x3b003b3b, 0x82008282, +0xb600b6b6, 0xdb00dbdb, 0xd400d4d4, 0x98009898, 0xe800e8e8, 0x8b008b8b, 0x02000202, 0xeb00ebeb, +0x0a000a0a, 0x2c002c2c, 0x1d001d1d, 0xb000b0b0, 0x6f006f6f, 0x8d008d8d, 0x88008888, 0x0e000e0e, +0x19001919, 0x87008787, 0x4e004e4e, 0x0b000b0b, 0xa900a9a9, 0x0c000c0c, 0x79007979, 0x11001111, +0x7f007f7f, 0x22002222, 0xe700e7e7, 0x59005959, 0xe100e1e1, 0xda00dada, 0x3d003d3d, 0xc800c8c8, +0x12001212, 0x04000404, 0x74007474, 0x54005454, 0x30003030, 0x7e007e7e, 0xb400b4b4, 0x28002828, +0x55005555, 0x68006868, 0x50005050, 0xbe00bebe, 0xd000d0d0, 0xc400c4c4, 0x31003131, 0xcb00cbcb, +0x2a002a2a, 0xad00adad, 0x0f000f0f, 0xca00caca, 0x70007070, 0xff00ffff, 0x32003232, 0x69006969, +0x08000808, 0x62006262, 0x00000000, 0x24002424, 0xd100d1d1, 0xfb00fbfb, 0xba00baba, 0xed00eded, +0x45004545, 0x81008181, 0x73007373, 0x6d006d6d, 0x84008484, 0x9f009f9f, 0xee00eeee, 0x4a004a4a, +0xc300c3c3, 0x2e002e2e, 0xc100c1c1, 0x01000101, 0xe600e6e6, 0x25002525, 0x48004848, 0x99009999, +0xb900b9b9, 0xb300b3b3, 0x7b007b7b, 0xf900f9f9, 0xce00cece, 0xbf00bfbf, 0xdf00dfdf, 0x71007171, +0x29002929, 0xcd00cdcd, 0x6c006c6c, 0x13001313, 0x64006464, 0x9b009b9b, 0x63006363, 0x9d009d9d, +0xc000c0c0, 0x4b004b4b, 0xb700b7b7, 0xa500a5a5, 0x89008989, 0x5f005f5f, 0xb100b1b1, 0x17001717, +0xf400f4f4, 0xbc00bcbc, 0xd300d3d3, 0x46004646, 0xcf00cfcf, 0x37003737, 0x5e005e5e, 0x47004747, +0x94009494, 0xfa00fafa, 0xfc00fcfc, 0x5b005b5b, 0x97009797, 0xfe00fefe, 0x5a005a5a, 0xac00acac, +0x3c003c3c, 0x4c004c4c, 0x03000303, 0x35003535, 0xf300f3f3, 0x23002323, 0xb800b8b8, 0x5d005d5d, +0x6a006a6a, 0x92009292, 0xd500d5d5, 0x21002121, 0x44004444, 0x51005151, 0xc600c6c6, 0x7d007d7d, +0x39003939, 0x83008383, 0xdc00dcdc, 0xaa00aaaa, 0x7c007c7c, 0x77007777, 0x56005656, 0x05000505, +0x1b001b1b, 0xa400a4a4, 0x15001515, 0x34003434, 0x1e001e1e, 0x1c001c1c, 0xf800f8f8, 0x52005252, +0x20002020, 0x14001414, 0xe900e9e9, 0xbd00bdbd, 0xdd00dddd, 0xe400e4e4, 0xa100a1a1, 0xe000e0e0, +0x8a008a8a, 0xf100f1f1, 0xd600d6d6, 0x7a007a7a, 0xbb00bbbb, 0xe300e3e3, 0x40004040, 0x4f004f4f, +}; + +static const ulong32 SP4404[] = { +0x70700070, 0x2c2c002c, 0xb3b300b3, 0xc0c000c0, 0xe4e400e4, 0x57570057, 0xeaea00ea, 0xaeae00ae, +0x23230023, 0x6b6b006b, 0x45450045, 0xa5a500a5, 0xeded00ed, 0x4f4f004f, 0x1d1d001d, 0x92920092, +0x86860086, 0xafaf00af, 0x7c7c007c, 0x1f1f001f, 0x3e3e003e, 0xdcdc00dc, 0x5e5e005e, 0x0b0b000b, +0xa6a600a6, 0x39390039, 0xd5d500d5, 0x5d5d005d, 0xd9d900d9, 0x5a5a005a, 0x51510051, 0x6c6c006c, +0x8b8b008b, 0x9a9a009a, 0xfbfb00fb, 0xb0b000b0, 0x74740074, 0x2b2b002b, 0xf0f000f0, 0x84840084, +0xdfdf00df, 0xcbcb00cb, 0x34340034, 0x76760076, 0x6d6d006d, 0xa9a900a9, 0xd1d100d1, 0x04040004, +0x14140014, 0x3a3a003a, 0xdede00de, 0x11110011, 0x32320032, 0x9c9c009c, 0x53530053, 0xf2f200f2, +0xfefe00fe, 0xcfcf00cf, 0xc3c300c3, 0x7a7a007a, 0x24240024, 0xe8e800e8, 0x60600060, 0x69690069, +0xaaaa00aa, 0xa0a000a0, 0xa1a100a1, 0x62620062, 0x54540054, 0x1e1e001e, 0xe0e000e0, 0x64640064, +0x10100010, 0x00000000, 0xa3a300a3, 0x75750075, 0x8a8a008a, 0xe6e600e6, 0x09090009, 0xdddd00dd, +0x87870087, 0x83830083, 0xcdcd00cd, 0x90900090, 0x73730073, 0xf6f600f6, 0x9d9d009d, 0xbfbf00bf, +0x52520052, 0xd8d800d8, 0xc8c800c8, 0xc6c600c6, 0x81810081, 0x6f6f006f, 0x13130013, 0x63630063, +0xe9e900e9, 0xa7a700a7, 0x9f9f009f, 0xbcbc00bc, 0x29290029, 0xf9f900f9, 0x2f2f002f, 0xb4b400b4, +0x78780078, 0x06060006, 0xe7e700e7, 0x71710071, 0xd4d400d4, 0xabab00ab, 0x88880088, 0x8d8d008d, +0x72720072, 0xb9b900b9, 0xf8f800f8, 0xacac00ac, 0x36360036, 0x2a2a002a, 0x3c3c003c, 0xf1f100f1, +0x40400040, 0xd3d300d3, 0xbbbb00bb, 0x43430043, 0x15150015, 0xadad00ad, 0x77770077, 0x80800080, +0x82820082, 0xecec00ec, 0x27270027, 0xe5e500e5, 0x85850085, 0x35350035, 0x0c0c000c, 0x41410041, +0xefef00ef, 0x93930093, 0x19190019, 0x21210021, 0x0e0e000e, 0x4e4e004e, 0x65650065, 0xbdbd00bd, +0xb8b800b8, 0x8f8f008f, 0xebeb00eb, 0xcece00ce, 0x30300030, 0x5f5f005f, 0xc5c500c5, 0x1a1a001a, +0xe1e100e1, 0xcaca00ca, 0x47470047, 0x3d3d003d, 0x01010001, 0xd6d600d6, 0x56560056, 0x4d4d004d, +0x0d0d000d, 0x66660066, 0xcccc00cc, 0x2d2d002d, 0x12120012, 0x20200020, 0xb1b100b1, 0x99990099, +0x4c4c004c, 0xc2c200c2, 0x7e7e007e, 0x05050005, 0xb7b700b7, 0x31310031, 0x17170017, 0xd7d700d7, +0x58580058, 0x61610061, 0x1b1b001b, 0x1c1c001c, 0x0f0f000f, 0x16160016, 0x18180018, 0x22220022, +0x44440044, 0xb2b200b2, 0xb5b500b5, 0x91910091, 0x08080008, 0xa8a800a8, 0xfcfc00fc, 0x50500050, +0xd0d000d0, 0x7d7d007d, 0x89890089, 0x97970097, 0x5b5b005b, 0x95950095, 0xffff00ff, 0xd2d200d2, +0xc4c400c4, 0x48480048, 0xf7f700f7, 0xdbdb00db, 0x03030003, 0xdada00da, 0x3f3f003f, 0x94940094, +0x5c5c005c, 0x02020002, 0x4a4a004a, 0x33330033, 0x67670067, 0xf3f300f3, 0x7f7f007f, 0xe2e200e2, +0x9b9b009b, 0x26260026, 0x37370037, 0x3b3b003b, 0x96960096, 0x4b4b004b, 0xbebe00be, 0x2e2e002e, +0x79790079, 0x8c8c008c, 0x6e6e006e, 0x8e8e008e, 0xf5f500f5, 0xb6b600b6, 0xfdfd00fd, 0x59590059, +0x98980098, 0x6a6a006a, 0x46460046, 0xbaba00ba, 0x25250025, 0x42420042, 0xa2a200a2, 0xfafa00fa, +0x07070007, 0x55550055, 0xeeee00ee, 0x0a0a000a, 0x49490049, 0x68680068, 0x38380038, 0xa4a400a4, +0x28280028, 0x7b7b007b, 0xc9c900c9, 0xc1c100c1, 0xe3e300e3, 0xf4f400f4, 0xc7c700c7, 0x9e9e009e, +}; + +static const ulong64 key_sigma[] = { + CONST64(0xA09E667F3BCC908B), + CONST64(0xB67AE8584CAA73B2), + CONST64(0xC6EF372FE94F82BE), + CONST64(0x54FF53A5F1D36F1C), + CONST64(0x10E527FADE682D1D), + CONST64(0xB05688C2B3E6C1FD) +}; + +static ulong64 F(ulong64 x) +{ + ulong32 D, U; + +#define loc(i) ((8-i)*8) + + D = SP1110[(x >> loc(8)) & 0xFF] ^ SP0222[(x >> loc(5)) & 0xFF] ^ SP3033[(x >> loc(6)) & 0xFF] ^ SP4404[(x >> loc(7)) & 0xFF]; + U = SP1110[(x >> loc(1)) & 0xFF] ^ SP0222[(x >> loc(2)) & 0xFF] ^ SP3033[(x >> loc(3)) & 0xFF] ^ SP4404[(x >> loc(4)) & 0xFF]; + + D ^= U; + U = D ^ RORc(U, 8); + + return ((ulong64)U) | (((ulong64)D) << CONST64(32)); +} + +static void rot_128(const unsigned char *in, unsigned count, unsigned char *out) +{ + unsigned x, w, b; + + w = count >> 3; + b = count & 7; + + for (x = 0; x < 16; x++) { + out[x] = (in[(x+w)&15] << b) | (in[(x+w+1)&15] >> (8 - b)); + } +} + +int camellia_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + unsigned char T[48], kA[16], kB[16], kR[16], kL[16]; + int x; + ulong64 A, B; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* Valid sizes (in bytes) are 16, 24, 32 */ + if (keylen != 16 && keylen != 24 && keylen != 32) { + return CRYPT_INVALID_KEYSIZE; + } + + /* number of rounds */ + skey->camellia.R = (keylen == 16) ? 18 : 24; + + if (num_rounds != 0 && num_rounds != skey->camellia.R) { + return CRYPT_INVALID_ROUNDS; + } + + /* expand key */ + if (keylen == 16) { + for (x = 0; x < 16; x++) { + T[x] = key[x]; + T[x + 16] = 0; + } + } else if (keylen == 24) { + for (x = 0; x < 24; x++) { + T[x] = key[x]; + } + for (x = 24; x < 32; x++) { + T[x] = key[x-8] ^ 0xFF; + } + } else { + for (x = 0; x < 32; x++) { + T[x] = key[x]; + } + } + + for (x = 0; x < 16; x++) { + kL[x] = T[x]; + kR[x] = T[x + 16]; + } + + for (x = 32; x < 48; x++) { + T[x] = T[x - 32] ^ T[x - 16]; + } + + /* first two rounds */ + LOAD64H(A, T+32); LOAD64H(B, T+40); + B ^= F(A ^ key_sigma[0]); + A ^= F(B ^ key_sigma[1]); + STORE64H(A, T+32); STORE64H(B, T+40); + + /* xor kL in */ + for (x = 0; x < 16; x++) { T[x+32] ^= kL[x]; } + + /* next two rounds */ + LOAD64H(A, T+32); LOAD64H(B, T+40); + B ^= F(A ^ key_sigma[2]); + A ^= F(B ^ key_sigma[3]); + STORE64H(A, T+32); STORE64H(B, T+40); + + /* grab KA */ + for (x = 0; x < 16; x++) { kA[x] = T[x+32]; } + + /* xor kR in */ + for (x = 0; x < 16; x++) { T[x+32] ^= kR[x]; } + + if (keylen == 16) { + /* grab whitening keys kw1 and kw2 */ + LOAD64H(skey->camellia.kw[0], kL); + LOAD64H(skey->camellia.kw[1], kL+8); + + /* k1-k2 */ + LOAD64H(skey->camellia.k[0], kA); + LOAD64H(skey->camellia.k[1], kA+8); + + /* rotate kL by 15, k3/k4 */ + rot_128(kL, 15, T+32); + LOAD64H(skey->camellia.k[2], T+32); + LOAD64H(skey->camellia.k[3], T+40); + + /* rotate kA by 15, k5/k6 */ + rot_128(kA, 15, T+32); + LOAD64H(skey->camellia.k[4], T+32); + LOAD64H(skey->camellia.k[5], T+40); + + /* rotate kA by 30, kl1, kl2 */ + rot_128(kA, 30, T+32); + LOAD64H(skey->camellia.kl[0], T+32); + LOAD64H(skey->camellia.kl[1], T+40); + + /* rotate kL by 45, k7/k8 */ + rot_128(kL, 45, T+32); + LOAD64H(skey->camellia.k[6], T+32); + LOAD64H(skey->camellia.k[7], T+40); + + /* rotate kA by 45, k9/k10 */ + rot_128(kA, 45, T+32); + LOAD64H(skey->camellia.k[8], T+32); + rot_128(kL, 60, T+32); + LOAD64H(skey->camellia.k[9], T+40); + + /* rotate kA by 60, k11/k12 */ + rot_128(kA, 60, T+32); + LOAD64H(skey->camellia.k[10], T+32); + LOAD64H(skey->camellia.k[11], T+40); + + /* rotate kL by 77, kl3, kl4 */ + rot_128(kL, 77, T+32); + LOAD64H(skey->camellia.kl[2], T+32); + LOAD64H(skey->camellia.kl[3], T+40); + + /* rotate kL by 94, k13/k14 */ + rot_128(kL, 94, T+32); + LOAD64H(skey->camellia.k[12], T+32); + LOAD64H(skey->camellia.k[13], T+40); + + /* rotate kA by 94, k15/k16 */ + rot_128(kA, 94, T+32); + LOAD64H(skey->camellia.k[14], T+32); + LOAD64H(skey->camellia.k[15], T+40); + + /* rotate kL by 111, k17/k18 */ + rot_128(kL, 111, T+32); + LOAD64H(skey->camellia.k[16], T+32); + LOAD64H(skey->camellia.k[17], T+40); + + /* rotate kA by 111, kw3/kw4 */ + rot_128(kA, 111, T+32); + LOAD64H(skey->camellia.kw[2], T+32); + LOAD64H(skey->camellia.kw[3], T+40); + } else { + /* last two rounds */ + LOAD64H(A, T+32); LOAD64H(B, T+40); + B ^= F(A ^ key_sigma[4]); + A ^= F(B ^ key_sigma[5]); + STORE64H(A, T+32); STORE64H(B, T+40); + + /* grab kB */ + for (x = 0; x < 16; x++) { kB[x] = T[x+32]; } + + /* kw1/2 from kL*/ + LOAD64H(skey->camellia.kw[0], kL); + LOAD64H(skey->camellia.kw[1], kL+8); + + /* k1/k2 = kB */ + LOAD64H(skey->camellia.k[0], kB); + LOAD64H(skey->camellia.k[1], kB+8); + + /* k3/k4 = kR by 15 */ + rot_128(kR, 15, T+32); + LOAD64H(skey->camellia.k[2], T+32); + LOAD64H(skey->camellia.k[3], T+40); + + /* k5/k7 = kA by 15 */ + rot_128(kA, 15, T+32); + LOAD64H(skey->camellia.k[4], T+32); + LOAD64H(skey->camellia.k[5], T+40); + + /* kl1/2 = kR by 30 */ + rot_128(kR, 30, T+32); + LOAD64H(skey->camellia.kl[0], T+32); + LOAD64H(skey->camellia.kl[1], T+40); + + /* k7/k8 = kB by 30 */ + rot_128(kB, 30, T+32); + LOAD64H(skey->camellia.k[6], T+32); + LOAD64H(skey->camellia.k[7], T+40); + + /* k9/k10 = kL by 45 */ + rot_128(kL, 45, T+32); + LOAD64H(skey->camellia.k[8], T+32); + LOAD64H(skey->camellia.k[9], T+40); + + /* k11/k12 = kA by 45 */ + rot_128(kA, 45, T+32); + LOAD64H(skey->camellia.k[10], T+32); + LOAD64H(skey->camellia.k[11], T+40); + + /* kl3/4 = kL by 60 */ + rot_128(kL, 60, T+32); + LOAD64H(skey->camellia.kl[2], T+32); + LOAD64H(skey->camellia.kl[3], T+40); + + /* k13/k14 = kR by 60 */ + rot_128(kR, 60, T+32); + LOAD64H(skey->camellia.k[12], T+32); + LOAD64H(skey->camellia.k[13], T+40); + + /* k15/k16 = kB by 15 */ + rot_128(kB, 60, T+32); + LOAD64H(skey->camellia.k[14], T+32); + LOAD64H(skey->camellia.k[15], T+40); + + /* k17/k18 = kL by 77 */ + rot_128(kL, 77, T+32); + LOAD64H(skey->camellia.k[16], T+32); + LOAD64H(skey->camellia.k[17], T+40); + + /* kl5/6 = kA by 77 */ + rot_128(kA, 77, T+32); + LOAD64H(skey->camellia.kl[4], T+32); + LOAD64H(skey->camellia.kl[5], T+40); + + /* k19/k20 = kR by 94 */ + rot_128(kR, 94, T+32); + LOAD64H(skey->camellia.k[18], T+32); + LOAD64H(skey->camellia.k[19], T+40); + + /* k21/k22 = kA by 94 */ + rot_128(kA, 94, T+32); + LOAD64H(skey->camellia.k[20], T+32); + LOAD64H(skey->camellia.k[21], T+40); + + /* k23/k24 = kL by 111 */ + rot_128(kL, 111, T+32); + LOAD64H(skey->camellia.k[22], T+32); + LOAD64H(skey->camellia.k[23], T+40); + + /* kw2/kw3 = kB by 111 */ + rot_128(kB, 111, T+32); + LOAD64H(skey->camellia.kw[2], T+32); + LOAD64H(skey->camellia.kw[3], T+40); + } + + return CRYPT_OK; +} + +int camellia_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong64 L, R; + ulong32 a, b; + + LOAD64H(L, pt+0); LOAD64H(R, pt+8); + L ^= skey->camellia.kw[0]; + R ^= skey->camellia.kw[1]; + + /* first 6 rounds */ + R ^= F(L ^ skey->camellia.k[0]); + L ^= F(R ^ skey->camellia.k[1]); + R ^= F(L ^ skey->camellia.k[2]); + L ^= F(R ^ skey->camellia.k[3]); + R ^= F(L ^ skey->camellia.k[4]); + L ^= F(R ^ skey->camellia.k[5]); + + /* FL */ + a = (ulong32)(L >> 32); + b = (ulong32)(L & 0xFFFFFFFFUL); + b ^= ROL((a & (ulong32)(skey->camellia.kl[0] >> 32)), 1); + a ^= b | (skey->camellia.kl[0] & 0xFFFFFFFFU); + L = (((ulong64)a) << 32) | b; + + /* FL^-1 */ + a = (ulong32)(R >> 32); + b = (ulong32)(R & 0xFFFFFFFFUL); + a ^= b | (skey->camellia.kl[1] & 0xFFFFFFFFU); + b ^= ROL((a & (ulong32)(skey->camellia.kl[1] >> 32)), 1); + R = (((ulong64)a) << 32) | b; + + /* second 6 rounds */ + R ^= F(L ^ skey->camellia.k[6]); + L ^= F(R ^ skey->camellia.k[7]); + R ^= F(L ^ skey->camellia.k[8]); + L ^= F(R ^ skey->camellia.k[9]); + R ^= F(L ^ skey->camellia.k[10]); + L ^= F(R ^ skey->camellia.k[11]); + + /* FL */ + a = (ulong32)(L >> 32); + b = (ulong32)(L & 0xFFFFFFFFUL); + b ^= ROL((a & (ulong32)(skey->camellia.kl[2] >> 32)), 1); + a ^= b | (skey->camellia.kl[2] & 0xFFFFFFFFU); + L = (((ulong64)a) << 32) | b; + + /* FL^-1 */ + a = (ulong32)(R >> 32); + b = (ulong32)(R & 0xFFFFFFFFUL); + a ^= b | (skey->camellia.kl[3] & 0xFFFFFFFFU); + b ^= ROL((a & (ulong32)(skey->camellia.kl[3] >> 32)), 1); + R = (((ulong64)a) << 32) | b; + + /* third 6 rounds */ + R ^= F(L ^ skey->camellia.k[12]); + L ^= F(R ^ skey->camellia.k[13]); + R ^= F(L ^ skey->camellia.k[14]); + L ^= F(R ^ skey->camellia.k[15]); + R ^= F(L ^ skey->camellia.k[16]); + L ^= F(R ^ skey->camellia.k[17]); + + /* next FL */ + if (skey->camellia.R == 24) { + /* FL */ + a = (ulong32)(L >> 32); + b = (ulong32)(L & 0xFFFFFFFFUL); + b ^= ROL((a & (ulong32)(skey->camellia.kl[4] >> 32)), 1); + a ^= b | (skey->camellia.kl[4] & 0xFFFFFFFFU); + L = (((ulong64)a) << 32) | b; + + /* FL^-1 */ + a = (ulong32)(R >> 32); + b = (ulong32)(R & 0xFFFFFFFFUL); + a ^= b | (skey->camellia.kl[5] & 0xFFFFFFFFU); + b ^= ROL((a & (ulong32)(skey->camellia.kl[5] >> 32)), 1); + R = (((ulong64)a) << 32) | b; + + /* fourth 6 rounds */ + R ^= F(L ^ skey->camellia.k[18]); + L ^= F(R ^ skey->camellia.k[19]); + R ^= F(L ^ skey->camellia.k[20]); + L ^= F(R ^ skey->camellia.k[21]); + R ^= F(L ^ skey->camellia.k[22]); + L ^= F(R ^ skey->camellia.k[23]); + } + + L ^= skey->camellia.kw[3]; + R ^= skey->camellia.kw[2]; + + STORE64H(R, ct+0); STORE64H(L, ct+8); + + return CRYPT_OK; +} + +int camellia_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong64 L, R; + ulong32 a, b; + + LOAD64H(R, ct+0); LOAD64H(L, ct+8); + L ^= skey->camellia.kw[3]; + R ^= skey->camellia.kw[2]; + + /* next FL */ + if (skey->camellia.R == 24) { + /* fourth 6 rounds */ + L ^= F(R ^ skey->camellia.k[23]); + R ^= F(L ^ skey->camellia.k[22]); + L ^= F(R ^ skey->camellia.k[21]); + R ^= F(L ^ skey->camellia.k[20]); + L ^= F(R ^ skey->camellia.k[19]); + R ^= F(L ^ skey->camellia.k[18]); + + /* FL */ + a = (ulong32)(L >> 32); + b = (ulong32)(L & 0xFFFFFFFFUL); + a ^= b | (skey->camellia.kl[4] & 0xFFFFFFFFU); + b ^= ROL((a & (ulong32)(skey->camellia.kl[4] >> 32)), 1); + L = (((ulong64)a) << 32) | b; + + /* FL^-1 */ + a = (ulong32)(R >> 32); + b = (ulong32)(R & 0xFFFFFFFFUL); + b ^= ROL((a & (ulong32)(skey->camellia.kl[5] >> 32)), 1); + a ^= b | (skey->camellia.kl[5] & 0xFFFFFFFFU); + R = (((ulong64)a) << 32) | b; + + } + + /* third 6 rounds */ + L ^= F(R ^ skey->camellia.k[17]); + R ^= F(L ^ skey->camellia.k[16]); + L ^= F(R ^ skey->camellia.k[15]); + R ^= F(L ^ skey->camellia.k[14]); + L ^= F(R ^ skey->camellia.k[13]); + R ^= F(L ^ skey->camellia.k[12]); + + /* FL */ + a = (ulong32)(L >> 32); + b = (ulong32)(L & 0xFFFFFFFFUL); + a ^= b | (skey->camellia.kl[2] & 0xFFFFFFFFU); + b ^= ROL((a & (ulong32)(skey->camellia.kl[2] >> 32)), 1); + L = (((ulong64)a) << 32) | b; + + /* FL^-1 */ + a = (ulong32)(R >> 32); + b = (ulong32)(R & 0xFFFFFFFFUL); + b ^= ROL((a & (ulong32)(skey->camellia.kl[3] >> 32)), 1); + a ^= b | (skey->camellia.kl[3] & 0xFFFFFFFFU); + R = (((ulong64)a) << 32) | b; + + /* second 6 rounds */ + L ^= F(R ^ skey->camellia.k[11]); + R ^= F(L ^ skey->camellia.k[10]); + L ^= F(R ^ skey->camellia.k[9]); + R ^= F(L ^ skey->camellia.k[8]); + L ^= F(R ^ skey->camellia.k[7]); + R ^= F(L ^ skey->camellia.k[6]); + + /* FL */ + a = (ulong32)(L >> 32); + b = (ulong32)(L & 0xFFFFFFFFUL); + a ^= b | (skey->camellia.kl[0] & 0xFFFFFFFFU); + b ^= ROL((a & (ulong32)(skey->camellia.kl[0] >> 32)), 1); + L = (((ulong64)a) << 32) | b; + + /* FL^-1 */ + a = (ulong32)(R >> 32); + b = (ulong32)(R & 0xFFFFFFFFUL); + b ^= ROL((a & (ulong32)(skey->camellia.kl[1] >> 32)), 1); + a ^= b | (skey->camellia.kl[1] & 0xFFFFFFFFU); + R = (((ulong64)a) << 32) | b; + + /* first 6 rounds */ + L ^= F(R ^ skey->camellia.k[5]); + R ^= F(L ^ skey->camellia.k[4]); + L ^= F(R ^ skey->camellia.k[3]); + R ^= F(L ^ skey->camellia.k[2]); + L ^= F(R ^ skey->camellia.k[1]); + R ^= F(L ^ skey->camellia.k[0]); + + R ^= skey->camellia.kw[1]; + L ^= skey->camellia.kw[0]; + + STORE64H(R, pt+8); STORE64H(L, pt+0); + + return CRYPT_OK; +} + +int camellia_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + int keylen; + unsigned char key[32], pt[16], ct[16]; + } tests[] = { + +{ + 16, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10 }, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10 }, + { 0x67, 0x67, 0x31, 0x38, 0x54, 0x96, 0x69, 0x73, + 0x08, 0x57, 0x06, 0x56, 0x48, 0xea, 0xbe, 0x43 } +}, + +{ + 24, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77 }, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10 }, + { 0xb4, 0x99, 0x34, 0x01, 0xb3, 0xe9, 0x96, 0xf8, + 0x4e, 0xe5, 0xce, 0xe7, 0xd7, 0x9b, 0x09, 0xb9 } +}, + + +{ + 32, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10 }, + { 0x9a, 0xcc, 0x23, 0x7d, 0xff, 0x16, 0xd7, 0x6c, + 0x20, 0xef, 0x7c, 0x91, 0x9e, 0x3a, 0x75, 0x09 } +}, + +{ + 32, + { 0x60, 0x3D, 0xEB, 0x10, 0x15, 0xCA, 0x71, 0xBE, + 0x2B, 0x73, 0xAE, 0xF0, 0x85, 0x7D, 0x77, 0x81, + 0x1F, 0x35, 0x2C, 0x07, 0x3B, 0x61, 0x08, 0xD7, + 0x2D, 0x98, 0x10, 0xA3, 0x09, 0x14, 0xDF, 0xF4 }, + { 0xF6, 0x9F, 0x24, 0x45, 0xDF, 0x4F, 0x9B, 0x17, + 0xAD, 0x2B, 0x41, 0x7B, 0xE6, 0x6C, 0x37, 0x10 }, + { 0x79, 0x60, 0x10, 0x9F, 0xB6, 0xDC, 0x42, 0x94, + 0x7F, 0xCF, 0xE5, 0x9E, 0xA3, 0xC5, 0xEB, 0x6B } +} +}; + unsigned char buf[2][16]; + symmetric_key skey; + int err; + unsigned int x; + + for (x = 0; x < sizeof(tests)/sizeof(tests[0]); x++) { + zeromem(&skey, sizeof(skey)); + if ((err = camellia_setup(tests[x].key, tests[x].keylen, 0, &skey)) != CRYPT_OK) { + return err; + } + if ((err = camellia_ecb_encrypt(tests[x].pt, buf[0], &skey)) != CRYPT_OK) { + camellia_done(&skey); + return err; + } + if ((err = camellia_ecb_decrypt(tests[x].ct, buf[1], &skey)) != CRYPT_OK) { + camellia_done(&skey); + return err; + } + camellia_done(&skey); + if (compare_testvector(tests[x].ct, 16, buf[0], 16, "Camellia Encrypt", x) || + compare_testvector(tests[x].pt, 16, buf[1], 16, "Camellia Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +void camellia_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +int camellia_keysize(int *keysize) +{ + if (*keysize >= 32) { *keysize = 32; } + else if (*keysize >= 24) { *keysize = 24; } + else if (*keysize >= 16) { *keysize = 16; } + else return CRYPT_INVALID_KEYSIZE; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/cast5.c b/optee_os/core/lib/libtomcrypt/src/ciphers/cast5.c new file mode 100644 index 0000000..3b401a5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/cast5.c @@ -0,0 +1,705 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + + /** + @file cast5.c + Implementation of LTC_CAST5 (RFC 2144) by Tom St Denis + */ +#include "tomcrypt_private.h" + +#ifdef LTC_CAST5 + +const struct ltc_cipher_descriptor cast5_desc = { + "cast5", + 15, + 5, 16, 8, 16, + &cast5_setup, + &cast5_ecb_encrypt, + &cast5_ecb_decrypt, + &cast5_test, + &cast5_done, + &cast5_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 S1[256] = { +0x30fb40d4UL, 0x9fa0ff0bUL, 0x6beccd2fUL, 0x3f258c7aUL, 0x1e213f2fUL, 0x9c004dd3UL, +0x6003e540UL, 0xcf9fc949UL, 0xbfd4af27UL, 0x88bbbdb5UL, 0xe2034090UL, 0x98d09675UL, +0x6e63a0e0UL, 0x15c361d2UL, 0xc2e7661dUL, 0x22d4ff8eUL, 0x28683b6fUL, 0xc07fd059UL, +0xff2379c8UL, 0x775f50e2UL, 0x43c340d3UL, 0xdf2f8656UL, 0x887ca41aUL, 0xa2d2bd2dUL, +0xa1c9e0d6UL, 0x346c4819UL, 0x61b76d87UL, 0x22540f2fUL, 0x2abe32e1UL, 0xaa54166bUL, +0x22568e3aUL, 0xa2d341d0UL, 0x66db40c8UL, 0xa784392fUL, 0x004dff2fUL, 0x2db9d2deUL, +0x97943facUL, 0x4a97c1d8UL, 0x527644b7UL, 0xb5f437a7UL, 0xb82cbaefUL, 0xd751d159UL, +0x6ff7f0edUL, 0x5a097a1fUL, 0x827b68d0UL, 0x90ecf52eUL, 0x22b0c054UL, 0xbc8e5935UL, +0x4b6d2f7fUL, 0x50bb64a2UL, 0xd2664910UL, 0xbee5812dUL, 0xb7332290UL, 0xe93b159fUL, +0xb48ee411UL, 0x4bff345dUL, 0xfd45c240UL, 0xad31973fUL, 0xc4f6d02eUL, 0x55fc8165UL, +0xd5b1caadUL, 0xa1ac2daeUL, 0xa2d4b76dUL, 0xc19b0c50UL, 0x882240f2UL, 0x0c6e4f38UL, +0xa4e4bfd7UL, 0x4f5ba272UL, 0x564c1d2fUL, 0xc59c5319UL, 0xb949e354UL, 0xb04669feUL, +0xb1b6ab8aUL, 0xc71358ddUL, 0x6385c545UL, 0x110f935dUL, 0x57538ad5UL, 0x6a390493UL, +0xe63d37e0UL, 0x2a54f6b3UL, 0x3a787d5fUL, 0x6276a0b5UL, 0x19a6fcdfUL, 0x7a42206aUL, +0x29f9d4d5UL, 0xf61b1891UL, 0xbb72275eUL, 0xaa508167UL, 0x38901091UL, 0xc6b505ebUL, +0x84c7cb8cUL, 0x2ad75a0fUL, 0x874a1427UL, 0xa2d1936bUL, 0x2ad286afUL, 0xaa56d291UL, +0xd7894360UL, 0x425c750dUL, 0x93b39e26UL, 0x187184c9UL, 0x6c00b32dUL, 0x73e2bb14UL, +0xa0bebc3cUL, 0x54623779UL, 0x64459eabUL, 0x3f328b82UL, 0x7718cf82UL, 0x59a2cea6UL, +0x04ee002eUL, 0x89fe78e6UL, 0x3fab0950UL, 0x325ff6c2UL, 0x81383f05UL, 0x6963c5c8UL, +0x76cb5ad6UL, 0xd49974c9UL, 0xca180dcfUL, 0x380782d5UL, 0xc7fa5cf6UL, 0x8ac31511UL, +0x35e79e13UL, 0x47da91d0UL, 0xf40f9086UL, 0xa7e2419eUL, 0x31366241UL, 0x051ef495UL, +0xaa573b04UL, 0x4a805d8dUL, 0x548300d0UL, 0x00322a3cUL, 0xbf64cddfUL, 0xba57a68eUL, +0x75c6372bUL, 0x50afd341UL, 0xa7c13275UL, 0x915a0bf5UL, 0x6b54bfabUL, 0x2b0b1426UL, +0xab4cc9d7UL, 0x449ccd82UL, 0xf7fbf265UL, 0xab85c5f3UL, 0x1b55db94UL, 0xaad4e324UL, +0xcfa4bd3fUL, 0x2deaa3e2UL, 0x9e204d02UL, 0xc8bd25acUL, 0xeadf55b3UL, 0xd5bd9e98UL, +0xe31231b2UL, 0x2ad5ad6cUL, 0x954329deUL, 0xadbe4528UL, 0xd8710f69UL, 0xaa51c90fUL, +0xaa786bf6UL, 0x22513f1eUL, 0xaa51a79bUL, 0x2ad344ccUL, 0x7b5a41f0UL, 0xd37cfbadUL, +0x1b069505UL, 0x41ece491UL, 0xb4c332e6UL, 0x032268d4UL, 0xc9600accUL, 0xce387e6dUL, +0xbf6bb16cUL, 0x6a70fb78UL, 0x0d03d9c9UL, 0xd4df39deUL, 0xe01063daUL, 0x4736f464UL, +0x5ad328d8UL, 0xb347cc96UL, 0x75bb0fc3UL, 0x98511bfbUL, 0x4ffbcc35UL, 0xb58bcf6aUL, +0xe11f0abcUL, 0xbfc5fe4aUL, 0xa70aec10UL, 0xac39570aUL, 0x3f04442fUL, 0x6188b153UL, +0xe0397a2eUL, 0x5727cb79UL, 0x9ceb418fUL, 0x1cacd68dUL, 0x2ad37c96UL, 0x0175cb9dUL, +0xc69dff09UL, 0xc75b65f0UL, 0xd9db40d8UL, 0xec0e7779UL, 0x4744ead4UL, 0xb11c3274UL, +0xdd24cb9eUL, 0x7e1c54bdUL, 0xf01144f9UL, 0xd2240eb1UL, 0x9675b3fdUL, 0xa3ac3755UL, +0xd47c27afUL, 0x51c85f4dUL, 0x56907596UL, 0xa5bb15e6UL, 0x580304f0UL, 0xca042cf1UL, +0x011a37eaUL, 0x8dbfaadbUL, 0x35ba3e4aUL, 0x3526ffa0UL, 0xc37b4d09UL, 0xbc306ed9UL, +0x98a52666UL, 0x5648f725UL, 0xff5e569dUL, 0x0ced63d0UL, 0x7c63b2cfUL, 0x700b45e1UL, +0xd5ea50f1UL, 0x85a92872UL, 0xaf1fbda7UL, 0xd4234870UL, 0xa7870bf3UL, 0x2d3b4d79UL, +0x42e04198UL, 0x0cd0ede7UL, 0x26470db8UL, 0xf881814cUL, 0x474d6ad7UL, 0x7c0c5e5cUL, +0xd1231959UL, 0x381b7298UL, 0xf5d2f4dbUL, 0xab838653UL, 0x6e2f1e23UL, 0x83719c9eUL, +0xbd91e046UL, 0x9a56456eUL, 0xdc39200cUL, 0x20c8c571UL, 0x962bda1cUL, 0xe1e696ffUL, +0xb141ab08UL, 0x7cca89b9UL, 0x1a69e783UL, 0x02cc4843UL, 0xa2f7c579UL, 0x429ef47dUL, +0x427b169cUL, 0x5ac9f049UL, 0xdd8f0f00UL, 0x5c8165bfUL}; + +static const ulong32 S2[256] = { +0x1f201094UL, 0xef0ba75bUL, 0x69e3cf7eUL, 0x393f4380UL, 0xfe61cf7aUL, 0xeec5207aUL, +0x55889c94UL, 0x72fc0651UL, 0xada7ef79UL, 0x4e1d7235UL, 0xd55a63ceUL, 0xde0436baUL, +0x99c430efUL, 0x5f0c0794UL, 0x18dcdb7dUL, 0xa1d6eff3UL, 0xa0b52f7bUL, 0x59e83605UL, +0xee15b094UL, 0xe9ffd909UL, 0xdc440086UL, 0xef944459UL, 0xba83ccb3UL, 0xe0c3cdfbUL, +0xd1da4181UL, 0x3b092ab1UL, 0xf997f1c1UL, 0xa5e6cf7bUL, 0x01420ddbUL, 0xe4e7ef5bUL, +0x25a1ff41UL, 0xe180f806UL, 0x1fc41080UL, 0x179bee7aUL, 0xd37ac6a9UL, 0xfe5830a4UL, +0x98de8b7fUL, 0x77e83f4eUL, 0x79929269UL, 0x24fa9f7bUL, 0xe113c85bUL, 0xacc40083UL, +0xd7503525UL, 0xf7ea615fUL, 0x62143154UL, 0x0d554b63UL, 0x5d681121UL, 0xc866c359UL, +0x3d63cf73UL, 0xcee234c0UL, 0xd4d87e87UL, 0x5c672b21UL, 0x071f6181UL, 0x39f7627fUL, +0x361e3084UL, 0xe4eb573bUL, 0x602f64a4UL, 0xd63acd9cUL, 0x1bbc4635UL, 0x9e81032dUL, +0x2701f50cUL, 0x99847ab4UL, 0xa0e3df79UL, 0xba6cf38cUL, 0x10843094UL, 0x2537a95eUL, +0xf46f6ffeUL, 0xa1ff3b1fUL, 0x208cfb6aUL, 0x8f458c74UL, 0xd9e0a227UL, 0x4ec73a34UL, +0xfc884f69UL, 0x3e4de8dfUL, 0xef0e0088UL, 0x3559648dUL, 0x8a45388cUL, 0x1d804366UL, +0x721d9bfdUL, 0xa58684bbUL, 0xe8256333UL, 0x844e8212UL, 0x128d8098UL, 0xfed33fb4UL, +0xce280ae1UL, 0x27e19ba5UL, 0xd5a6c252UL, 0xe49754bdUL, 0xc5d655ddUL, 0xeb667064UL, +0x77840b4dUL, 0xa1b6a801UL, 0x84db26a9UL, 0xe0b56714UL, 0x21f043b7UL, 0xe5d05860UL, +0x54f03084UL, 0x066ff472UL, 0xa31aa153UL, 0xdadc4755UL, 0xb5625dbfUL, 0x68561be6UL, +0x83ca6b94UL, 0x2d6ed23bUL, 0xeccf01dbUL, 0xa6d3d0baUL, 0xb6803d5cUL, 0xaf77a709UL, +0x33b4a34cUL, 0x397bc8d6UL, 0x5ee22b95UL, 0x5f0e5304UL, 0x81ed6f61UL, 0x20e74364UL, +0xb45e1378UL, 0xde18639bUL, 0x881ca122UL, 0xb96726d1UL, 0x8049a7e8UL, 0x22b7da7bUL, +0x5e552d25UL, 0x5272d237UL, 0x79d2951cUL, 0xc60d894cUL, 0x488cb402UL, 0x1ba4fe5bUL, +0xa4b09f6bUL, 0x1ca815cfUL, 0xa20c3005UL, 0x8871df63UL, 0xb9de2fcbUL, 0x0cc6c9e9UL, +0x0beeff53UL, 0xe3214517UL, 0xb4542835UL, 0x9f63293cUL, 0xee41e729UL, 0x6e1d2d7cUL, +0x50045286UL, 0x1e6685f3UL, 0xf33401c6UL, 0x30a22c95UL, 0x31a70850UL, 0x60930f13UL, +0x73f98417UL, 0xa1269859UL, 0xec645c44UL, 0x52c877a9UL, 0xcdff33a6UL, 0xa02b1741UL, +0x7cbad9a2UL, 0x2180036fUL, 0x50d99c08UL, 0xcb3f4861UL, 0xc26bd765UL, 0x64a3f6abUL, +0x80342676UL, 0x25a75e7bUL, 0xe4e6d1fcUL, 0x20c710e6UL, 0xcdf0b680UL, 0x17844d3bUL, +0x31eef84dUL, 0x7e0824e4UL, 0x2ccb49ebUL, 0x846a3baeUL, 0x8ff77888UL, 0xee5d60f6UL, +0x7af75673UL, 0x2fdd5cdbUL, 0xa11631c1UL, 0x30f66f43UL, 0xb3faec54UL, 0x157fd7faUL, +0xef8579ccUL, 0xd152de58UL, 0xdb2ffd5eUL, 0x8f32ce19UL, 0x306af97aUL, 0x02f03ef8UL, +0x99319ad5UL, 0xc242fa0fUL, 0xa7e3ebb0UL, 0xc68e4906UL, 0xb8da230cUL, 0x80823028UL, +0xdcdef3c8UL, 0xd35fb171UL, 0x088a1bc8UL, 0xbec0c560UL, 0x61a3c9e8UL, 0xbca8f54dUL, +0xc72feffaUL, 0x22822e99UL, 0x82c570b4UL, 0xd8d94e89UL, 0x8b1c34bcUL, 0x301e16e6UL, +0x273be979UL, 0xb0ffeaa6UL, 0x61d9b8c6UL, 0x00b24869UL, 0xb7ffce3fUL, 0x08dc283bUL, +0x43daf65aUL, 0xf7e19798UL, 0x7619b72fUL, 0x8f1c9ba4UL, 0xdc8637a0UL, 0x16a7d3b1UL, +0x9fc393b7UL, 0xa7136eebUL, 0xc6bcc63eUL, 0x1a513742UL, 0xef6828bcUL, 0x520365d6UL, +0x2d6a77abUL, 0x3527ed4bUL, 0x821fd216UL, 0x095c6e2eUL, 0xdb92f2fbUL, 0x5eea29cbUL, +0x145892f5UL, 0x91584f7fUL, 0x5483697bUL, 0x2667a8ccUL, 0x85196048UL, 0x8c4baceaUL, +0x833860d4UL, 0x0d23e0f9UL, 0x6c387e8aUL, 0x0ae6d249UL, 0xb284600cUL, 0xd835731dUL, +0xdcb1c647UL, 0xac4c56eaUL, 0x3ebd81b3UL, 0x230eabb0UL, 0x6438bc87UL, 0xf0b5b1faUL, +0x8f5ea2b3UL, 0xfc184642UL, 0x0a036b7aUL, 0x4fb089bdUL, 0x649da589UL, 0xa345415eUL, +0x5c038323UL, 0x3e5d3bb9UL, 0x43d79572UL, 0x7e6dd07cUL, 0x06dfdf1eUL, 0x6c6cc4efUL, +0x7160a539UL, 0x73bfbe70UL, 0x83877605UL, 0x4523ecf1UL}; + +static const ulong32 S3[256] = { +0x8defc240UL, 0x25fa5d9fUL, 0xeb903dbfUL, 0xe810c907UL, 0x47607fffUL, 0x369fe44bUL, +0x8c1fc644UL, 0xaececa90UL, 0xbeb1f9bfUL, 0xeefbcaeaUL, 0xe8cf1950UL, 0x51df07aeUL, +0x920e8806UL, 0xf0ad0548UL, 0xe13c8d83UL, 0x927010d5UL, 0x11107d9fUL, 0x07647db9UL, +0xb2e3e4d4UL, 0x3d4f285eUL, 0xb9afa820UL, 0xfade82e0UL, 0xa067268bUL, 0x8272792eUL, +0x553fb2c0UL, 0x489ae22bUL, 0xd4ef9794UL, 0x125e3fbcUL, 0x21fffceeUL, 0x825b1bfdUL, +0x9255c5edUL, 0x1257a240UL, 0x4e1a8302UL, 0xbae07fffUL, 0x528246e7UL, 0x8e57140eUL, +0x3373f7bfUL, 0x8c9f8188UL, 0xa6fc4ee8UL, 0xc982b5a5UL, 0xa8c01db7UL, 0x579fc264UL, +0x67094f31UL, 0xf2bd3f5fUL, 0x40fff7c1UL, 0x1fb78dfcUL, 0x8e6bd2c1UL, 0x437be59bUL, +0x99b03dbfUL, 0xb5dbc64bUL, 0x638dc0e6UL, 0x55819d99UL, 0xa197c81cUL, 0x4a012d6eUL, +0xc5884a28UL, 0xccc36f71UL, 0xb843c213UL, 0x6c0743f1UL, 0x8309893cUL, 0x0feddd5fUL, +0x2f7fe850UL, 0xd7c07f7eUL, 0x02507fbfUL, 0x5afb9a04UL, 0xa747d2d0UL, 0x1651192eUL, +0xaf70bf3eUL, 0x58c31380UL, 0x5f98302eUL, 0x727cc3c4UL, 0x0a0fb402UL, 0x0f7fef82UL, +0x8c96fdadUL, 0x5d2c2aaeUL, 0x8ee99a49UL, 0x50da88b8UL, 0x8427f4a0UL, 0x1eac5790UL, +0x796fb449UL, 0x8252dc15UL, 0xefbd7d9bUL, 0xa672597dUL, 0xada840d8UL, 0x45f54504UL, +0xfa5d7403UL, 0xe83ec305UL, 0x4f91751aUL, 0x925669c2UL, 0x23efe941UL, 0xa903f12eUL, +0x60270df2UL, 0x0276e4b6UL, 0x94fd6574UL, 0x927985b2UL, 0x8276dbcbUL, 0x02778176UL, +0xf8af918dUL, 0x4e48f79eUL, 0x8f616ddfUL, 0xe29d840eUL, 0x842f7d83UL, 0x340ce5c8UL, +0x96bbb682UL, 0x93b4b148UL, 0xef303cabUL, 0x984faf28UL, 0x779faf9bUL, 0x92dc560dUL, +0x224d1e20UL, 0x8437aa88UL, 0x7d29dc96UL, 0x2756d3dcUL, 0x8b907ceeUL, 0xb51fd240UL, +0xe7c07ce3UL, 0xe566b4a1UL, 0xc3e9615eUL, 0x3cf8209dUL, 0x6094d1e3UL, 0xcd9ca341UL, +0x5c76460eUL, 0x00ea983bUL, 0xd4d67881UL, 0xfd47572cUL, 0xf76cedd9UL, 0xbda8229cUL, +0x127dadaaUL, 0x438a074eUL, 0x1f97c090UL, 0x081bdb8aUL, 0x93a07ebeUL, 0xb938ca15UL, +0x97b03cffUL, 0x3dc2c0f8UL, 0x8d1ab2ecUL, 0x64380e51UL, 0x68cc7bfbUL, 0xd90f2788UL, +0x12490181UL, 0x5de5ffd4UL, 0xdd7ef86aUL, 0x76a2e214UL, 0xb9a40368UL, 0x925d958fUL, +0x4b39fffaUL, 0xba39aee9UL, 0xa4ffd30bUL, 0xfaf7933bUL, 0x6d498623UL, 0x193cbcfaUL, +0x27627545UL, 0x825cf47aUL, 0x61bd8ba0UL, 0xd11e42d1UL, 0xcead04f4UL, 0x127ea392UL, +0x10428db7UL, 0x8272a972UL, 0x9270c4a8UL, 0x127de50bUL, 0x285ba1c8UL, 0x3c62f44fUL, +0x35c0eaa5UL, 0xe805d231UL, 0x428929fbUL, 0xb4fcdf82UL, 0x4fb66a53UL, 0x0e7dc15bUL, +0x1f081fabUL, 0x108618aeUL, 0xfcfd086dUL, 0xf9ff2889UL, 0x694bcc11UL, 0x236a5caeUL, +0x12deca4dUL, 0x2c3f8cc5UL, 0xd2d02dfeUL, 0xf8ef5896UL, 0xe4cf52daUL, 0x95155b67UL, +0x494a488cUL, 0xb9b6a80cUL, 0x5c8f82bcUL, 0x89d36b45UL, 0x3a609437UL, 0xec00c9a9UL, +0x44715253UL, 0x0a874b49UL, 0xd773bc40UL, 0x7c34671cUL, 0x02717ef6UL, 0x4feb5536UL, +0xa2d02fffUL, 0xd2bf60c4UL, 0xd43f03c0UL, 0x50b4ef6dUL, 0x07478cd1UL, 0x006e1888UL, +0xa2e53f55UL, 0xb9e6d4bcUL, 0xa2048016UL, 0x97573833UL, 0xd7207d67UL, 0xde0f8f3dUL, +0x72f87b33UL, 0xabcc4f33UL, 0x7688c55dUL, 0x7b00a6b0UL, 0x947b0001UL, 0x570075d2UL, +0xf9bb88f8UL, 0x8942019eUL, 0x4264a5ffUL, 0x856302e0UL, 0x72dbd92bUL, 0xee971b69UL, +0x6ea22fdeUL, 0x5f08ae2bUL, 0xaf7a616dUL, 0xe5c98767UL, 0xcf1febd2UL, 0x61efc8c2UL, +0xf1ac2571UL, 0xcc8239c2UL, 0x67214cb8UL, 0xb1e583d1UL, 0xb7dc3e62UL, 0x7f10bdceUL, +0xf90a5c38UL, 0x0ff0443dUL, 0x606e6dc6UL, 0x60543a49UL, 0x5727c148UL, 0x2be98a1dUL, +0x8ab41738UL, 0x20e1be24UL, 0xaf96da0fUL, 0x68458425UL, 0x99833be5UL, 0x600d457dUL, +0x282f9350UL, 0x8334b362UL, 0xd91d1120UL, 0x2b6d8da0UL, 0x642b1e31UL, 0x9c305a00UL, +0x52bce688UL, 0x1b03588aUL, 0xf7baefd5UL, 0x4142ed9cUL, 0xa4315c11UL, 0x83323ec5UL, +0xdfef4636UL, 0xa133c501UL, 0xe9d3531cUL, 0xee353783UL}; + +static const ulong32 S4[256] = { +0x9db30420UL, 0x1fb6e9deUL, 0xa7be7befUL, 0xd273a298UL, 0x4a4f7bdbUL, 0x64ad8c57UL, +0x85510443UL, 0xfa020ed1UL, 0x7e287affUL, 0xe60fb663UL, 0x095f35a1UL, 0x79ebf120UL, +0xfd059d43UL, 0x6497b7b1UL, 0xf3641f63UL, 0x241e4adfUL, 0x28147f5fUL, 0x4fa2b8cdUL, +0xc9430040UL, 0x0cc32220UL, 0xfdd30b30UL, 0xc0a5374fUL, 0x1d2d00d9UL, 0x24147b15UL, +0xee4d111aUL, 0x0fca5167UL, 0x71ff904cUL, 0x2d195ffeUL, 0x1a05645fUL, 0x0c13fefeUL, +0x081b08caUL, 0x05170121UL, 0x80530100UL, 0xe83e5efeUL, 0xac9af4f8UL, 0x7fe72701UL, +0xd2b8ee5fUL, 0x06df4261UL, 0xbb9e9b8aUL, 0x7293ea25UL, 0xce84ffdfUL, 0xf5718801UL, +0x3dd64b04UL, 0xa26f263bUL, 0x7ed48400UL, 0x547eebe6UL, 0x446d4ca0UL, 0x6cf3d6f5UL, +0x2649abdfUL, 0xaea0c7f5UL, 0x36338cc1UL, 0x503f7e93UL, 0xd3772061UL, 0x11b638e1UL, +0x72500e03UL, 0xf80eb2bbUL, 0xabe0502eUL, 0xec8d77deUL, 0x57971e81UL, 0xe14f6746UL, +0xc9335400UL, 0x6920318fUL, 0x081dbb99UL, 0xffc304a5UL, 0x4d351805UL, 0x7f3d5ce3UL, +0xa6c866c6UL, 0x5d5bcca9UL, 0xdaec6feaUL, 0x9f926f91UL, 0x9f46222fUL, 0x3991467dUL, +0xa5bf6d8eUL, 0x1143c44fUL, 0x43958302UL, 0xd0214eebUL, 0x022083b8UL, 0x3fb6180cUL, +0x18f8931eUL, 0x281658e6UL, 0x26486e3eUL, 0x8bd78a70UL, 0x7477e4c1UL, 0xb506e07cUL, +0xf32d0a25UL, 0x79098b02UL, 0xe4eabb81UL, 0x28123b23UL, 0x69dead38UL, 0x1574ca16UL, +0xdf871b62UL, 0x211c40b7UL, 0xa51a9ef9UL, 0x0014377bUL, 0x041e8ac8UL, 0x09114003UL, +0xbd59e4d2UL, 0xe3d156d5UL, 0x4fe876d5UL, 0x2f91a340UL, 0x557be8deUL, 0x00eae4a7UL, +0x0ce5c2ecUL, 0x4db4bba6UL, 0xe756bdffUL, 0xdd3369acUL, 0xec17b035UL, 0x06572327UL, +0x99afc8b0UL, 0x56c8c391UL, 0x6b65811cUL, 0x5e146119UL, 0x6e85cb75UL, 0xbe07c002UL, +0xc2325577UL, 0x893ff4ecUL, 0x5bbfc92dUL, 0xd0ec3b25UL, 0xb7801ab7UL, 0x8d6d3b24UL, +0x20c763efUL, 0xc366a5fcUL, 0x9c382880UL, 0x0ace3205UL, 0xaac9548aUL, 0xeca1d7c7UL, +0x041afa32UL, 0x1d16625aUL, 0x6701902cUL, 0x9b757a54UL, 0x31d477f7UL, 0x9126b031UL, +0x36cc6fdbUL, 0xc70b8b46UL, 0xd9e66a48UL, 0x56e55a79UL, 0x026a4cebUL, 0x52437effUL, +0x2f8f76b4UL, 0x0df980a5UL, 0x8674cde3UL, 0xedda04ebUL, 0x17a9be04UL, 0x2c18f4dfUL, +0xb7747f9dUL, 0xab2af7b4UL, 0xefc34d20UL, 0x2e096b7cUL, 0x1741a254UL, 0xe5b6a035UL, +0x213d42f6UL, 0x2c1c7c26UL, 0x61c2f50fUL, 0x6552daf9UL, 0xd2c231f8UL, 0x25130f69UL, +0xd8167fa2UL, 0x0418f2c8UL, 0x001a96a6UL, 0x0d1526abUL, 0x63315c21UL, 0x5e0a72ecUL, +0x49bafefdUL, 0x187908d9UL, 0x8d0dbd86UL, 0x311170a7UL, 0x3e9b640cUL, 0xcc3e10d7UL, +0xd5cad3b6UL, 0x0caec388UL, 0xf73001e1UL, 0x6c728affUL, 0x71eae2a1UL, 0x1f9af36eUL, +0xcfcbd12fUL, 0xc1de8417UL, 0xac07be6bUL, 0xcb44a1d8UL, 0x8b9b0f56UL, 0x013988c3UL, +0xb1c52fcaUL, 0xb4be31cdUL, 0xd8782806UL, 0x12a3a4e2UL, 0x6f7de532UL, 0x58fd7eb6UL, +0xd01ee900UL, 0x24adffc2UL, 0xf4990fc5UL, 0x9711aac5UL, 0x001d7b95UL, 0x82e5e7d2UL, +0x109873f6UL, 0x00613096UL, 0xc32d9521UL, 0xada121ffUL, 0x29908415UL, 0x7fbb977fUL, +0xaf9eb3dbUL, 0x29c9ed2aUL, 0x5ce2a465UL, 0xa730f32cUL, 0xd0aa3fe8UL, 0x8a5cc091UL, +0xd49e2ce7UL, 0x0ce454a9UL, 0xd60acd86UL, 0x015f1919UL, 0x77079103UL, 0xdea03af6UL, +0x78a8565eUL, 0xdee356dfUL, 0x21f05cbeUL, 0x8b75e387UL, 0xb3c50651UL, 0xb8a5c3efUL, +0xd8eeb6d2UL, 0xe523be77UL, 0xc2154529UL, 0x2f69efdfUL, 0xafe67afbUL, 0xf470c4b2UL, +0xf3e0eb5bUL, 0xd6cc9876UL, 0x39e4460cUL, 0x1fda8538UL, 0x1987832fUL, 0xca007367UL, +0xa99144f8UL, 0x296b299eUL, 0x492fc295UL, 0x9266beabUL, 0xb5676e69UL, 0x9bd3dddaUL, +0xdf7e052fUL, 0xdb25701cUL, 0x1b5e51eeUL, 0xf65324e6UL, 0x6afce36cUL, 0x0316cc04UL, +0x8644213eUL, 0xb7dc59d0UL, 0x7965291fUL, 0xccd6fd43UL, 0x41823979UL, 0x932bcdf6UL, +0xb657c34dUL, 0x4edfd282UL, 0x7ae5290cUL, 0x3cb9536bUL, 0x851e20feUL, 0x9833557eUL, +0x13ecf0b0UL, 0xd3ffb372UL, 0x3f85c5c1UL, 0x0aef7ed2UL}; + +static const ulong32 S5[256] = { +0x7ec90c04UL, 0x2c6e74b9UL, 0x9b0e66dfUL, 0xa6337911UL, 0xb86a7fffUL, 0x1dd358f5UL, +0x44dd9d44UL, 0x1731167fUL, 0x08fbf1faUL, 0xe7f511ccUL, 0xd2051b00UL, 0x735aba00UL, +0x2ab722d8UL, 0x386381cbUL, 0xacf6243aUL, 0x69befd7aUL, 0xe6a2e77fUL, 0xf0c720cdUL, +0xc4494816UL, 0xccf5c180UL, 0x38851640UL, 0x15b0a848UL, 0xe68b18cbUL, 0x4caadeffUL, +0x5f480a01UL, 0x0412b2aaUL, 0x259814fcUL, 0x41d0efe2UL, 0x4e40b48dUL, 0x248eb6fbUL, +0x8dba1cfeUL, 0x41a99b02UL, 0x1a550a04UL, 0xba8f65cbUL, 0x7251f4e7UL, 0x95a51725UL, +0xc106ecd7UL, 0x97a5980aUL, 0xc539b9aaUL, 0x4d79fe6aUL, 0xf2f3f763UL, 0x68af8040UL, +0xed0c9e56UL, 0x11b4958bUL, 0xe1eb5a88UL, 0x8709e6b0UL, 0xd7e07156UL, 0x4e29fea7UL, +0x6366e52dUL, 0x02d1c000UL, 0xc4ac8e05UL, 0x9377f571UL, 0x0c05372aUL, 0x578535f2UL, +0x2261be02UL, 0xd642a0c9UL, 0xdf13a280UL, 0x74b55bd2UL, 0x682199c0UL, 0xd421e5ecUL, +0x53fb3ce8UL, 0xc8adedb3UL, 0x28a87fc9UL, 0x3d959981UL, 0x5c1ff900UL, 0xfe38d399UL, +0x0c4eff0bUL, 0x062407eaUL, 0xaa2f4fb1UL, 0x4fb96976UL, 0x90c79505UL, 0xb0a8a774UL, +0xef55a1ffUL, 0xe59ca2c2UL, 0xa6b62d27UL, 0xe66a4263UL, 0xdf65001fUL, 0x0ec50966UL, +0xdfdd55bcUL, 0x29de0655UL, 0x911e739aUL, 0x17af8975UL, 0x32c7911cUL, 0x89f89468UL, +0x0d01e980UL, 0x524755f4UL, 0x03b63cc9UL, 0x0cc844b2UL, 0xbcf3f0aaUL, 0x87ac36e9UL, +0xe53a7426UL, 0x01b3d82bUL, 0x1a9e7449UL, 0x64ee2d7eUL, 0xcddbb1daUL, 0x01c94910UL, +0xb868bf80UL, 0x0d26f3fdUL, 0x9342ede7UL, 0x04a5c284UL, 0x636737b6UL, 0x50f5b616UL, +0xf24766e3UL, 0x8eca36c1UL, 0x136e05dbUL, 0xfef18391UL, 0xfb887a37UL, 0xd6e7f7d4UL, +0xc7fb7dc9UL, 0x3063fcdfUL, 0xb6f589deUL, 0xec2941daUL, 0x26e46695UL, 0xb7566419UL, +0xf654efc5UL, 0xd08d58b7UL, 0x48925401UL, 0xc1bacb7fUL, 0xe5ff550fUL, 0xb6083049UL, +0x5bb5d0e8UL, 0x87d72e5aUL, 0xab6a6ee1UL, 0x223a66ceUL, 0xc62bf3cdUL, 0x9e0885f9UL, +0x68cb3e47UL, 0x086c010fUL, 0xa21de820UL, 0xd18b69deUL, 0xf3f65777UL, 0xfa02c3f6UL, +0x407edac3UL, 0xcbb3d550UL, 0x1793084dUL, 0xb0d70ebaUL, 0x0ab378d5UL, 0xd951fb0cUL, +0xded7da56UL, 0x4124bbe4UL, 0x94ca0b56UL, 0x0f5755d1UL, 0xe0e1e56eUL, 0x6184b5beUL, +0x580a249fUL, 0x94f74bc0UL, 0xe327888eUL, 0x9f7b5561UL, 0xc3dc0280UL, 0x05687715UL, +0x646c6bd7UL, 0x44904db3UL, 0x66b4f0a3UL, 0xc0f1648aUL, 0x697ed5afUL, 0x49e92ff6UL, +0x309e374fUL, 0x2cb6356aUL, 0x85808573UL, 0x4991f840UL, 0x76f0ae02UL, 0x083be84dUL, +0x28421c9aUL, 0x44489406UL, 0x736e4cb8UL, 0xc1092910UL, 0x8bc95fc6UL, 0x7d869cf4UL, +0x134f616fUL, 0x2e77118dUL, 0xb31b2be1UL, 0xaa90b472UL, 0x3ca5d717UL, 0x7d161bbaUL, +0x9cad9010UL, 0xaf462ba2UL, 0x9fe459d2UL, 0x45d34559UL, 0xd9f2da13UL, 0xdbc65487UL, +0xf3e4f94eUL, 0x176d486fUL, 0x097c13eaUL, 0x631da5c7UL, 0x445f7382UL, 0x175683f4UL, +0xcdc66a97UL, 0x70be0288UL, 0xb3cdcf72UL, 0x6e5dd2f3UL, 0x20936079UL, 0x459b80a5UL, +0xbe60e2dbUL, 0xa9c23101UL, 0xeba5315cUL, 0x224e42f2UL, 0x1c5c1572UL, 0xf6721b2cUL, +0x1ad2fff3UL, 0x8c25404eUL, 0x324ed72fUL, 0x4067b7fdUL, 0x0523138eUL, 0x5ca3bc78UL, +0xdc0fd66eUL, 0x75922283UL, 0x784d6b17UL, 0x58ebb16eUL, 0x44094f85UL, 0x3f481d87UL, +0xfcfeae7bUL, 0x77b5ff76UL, 0x8c2302bfUL, 0xaaf47556UL, 0x5f46b02aUL, 0x2b092801UL, +0x3d38f5f7UL, 0x0ca81f36UL, 0x52af4a8aUL, 0x66d5e7c0UL, 0xdf3b0874UL, 0x95055110UL, +0x1b5ad7a8UL, 0xf61ed5adUL, 0x6cf6e479UL, 0x20758184UL, 0xd0cefa65UL, 0x88f7be58UL, +0x4a046826UL, 0x0ff6f8f3UL, 0xa09c7f70UL, 0x5346aba0UL, 0x5ce96c28UL, 0xe176eda3UL, +0x6bac307fUL, 0x376829d2UL, 0x85360fa9UL, 0x17e3fe2aUL, 0x24b79767UL, 0xf5a96b20UL, +0xd6cd2595UL, 0x68ff1ebfUL, 0x7555442cUL, 0xf19f06beUL, 0xf9e0659aUL, 0xeeb9491dUL, +0x34010718UL, 0xbb30cab8UL, 0xe822fe15UL, 0x88570983UL, 0x750e6249UL, 0xda627e55UL, +0x5e76ffa8UL, 0xb1534546UL, 0x6d47de08UL, 0xefe9e7d4UL}; + +static const ulong32 S6[256] = { +0xf6fa8f9dUL, 0x2cac6ce1UL, 0x4ca34867UL, 0xe2337f7cUL, 0x95db08e7UL, 0x016843b4UL, +0xeced5cbcUL, 0x325553acUL, 0xbf9f0960UL, 0xdfa1e2edUL, 0x83f0579dUL, 0x63ed86b9UL, +0x1ab6a6b8UL, 0xde5ebe39UL, 0xf38ff732UL, 0x8989b138UL, 0x33f14961UL, 0xc01937bdUL, +0xf506c6daUL, 0xe4625e7eUL, 0xa308ea99UL, 0x4e23e33cUL, 0x79cbd7ccUL, 0x48a14367UL, +0xa3149619UL, 0xfec94bd5UL, 0xa114174aUL, 0xeaa01866UL, 0xa084db2dUL, 0x09a8486fUL, +0xa888614aUL, 0x2900af98UL, 0x01665991UL, 0xe1992863UL, 0xc8f30c60UL, 0x2e78ef3cUL, +0xd0d51932UL, 0xcf0fec14UL, 0xf7ca07d2UL, 0xd0a82072UL, 0xfd41197eUL, 0x9305a6b0UL, +0xe86be3daUL, 0x74bed3cdUL, 0x372da53cUL, 0x4c7f4448UL, 0xdab5d440UL, 0x6dba0ec3UL, +0x083919a7UL, 0x9fbaeed9UL, 0x49dbcfb0UL, 0x4e670c53UL, 0x5c3d9c01UL, 0x64bdb941UL, +0x2c0e636aUL, 0xba7dd9cdUL, 0xea6f7388UL, 0xe70bc762UL, 0x35f29adbUL, 0x5c4cdd8dUL, +0xf0d48d8cUL, 0xb88153e2UL, 0x08a19866UL, 0x1ae2eac8UL, 0x284caf89UL, 0xaa928223UL, +0x9334be53UL, 0x3b3a21bfUL, 0x16434be3UL, 0x9aea3906UL, 0xefe8c36eUL, 0xf890cdd9UL, +0x80226daeUL, 0xc340a4a3UL, 0xdf7e9c09UL, 0xa694a807UL, 0x5b7c5eccUL, 0x221db3a6UL, +0x9a69a02fUL, 0x68818a54UL, 0xceb2296fUL, 0x53c0843aUL, 0xfe893655UL, 0x25bfe68aUL, +0xb4628abcUL, 0xcf222ebfUL, 0x25ac6f48UL, 0xa9a99387UL, 0x53bddb65UL, 0xe76ffbe7UL, +0xe967fd78UL, 0x0ba93563UL, 0x8e342bc1UL, 0xe8a11be9UL, 0x4980740dUL, 0xc8087dfcUL, +0x8de4bf99UL, 0xa11101a0UL, 0x7fd37975UL, 0xda5a26c0UL, 0xe81f994fUL, 0x9528cd89UL, +0xfd339fedUL, 0xb87834bfUL, 0x5f04456dUL, 0x22258698UL, 0xc9c4c83bUL, 0x2dc156beUL, +0x4f628daaUL, 0x57f55ec5UL, 0xe2220abeUL, 0xd2916ebfUL, 0x4ec75b95UL, 0x24f2c3c0UL, +0x42d15d99UL, 0xcd0d7fa0UL, 0x7b6e27ffUL, 0xa8dc8af0UL, 0x7345c106UL, 0xf41e232fUL, +0x35162386UL, 0xe6ea8926UL, 0x3333b094UL, 0x157ec6f2UL, 0x372b74afUL, 0x692573e4UL, +0xe9a9d848UL, 0xf3160289UL, 0x3a62ef1dUL, 0xa787e238UL, 0xf3a5f676UL, 0x74364853UL, +0x20951063UL, 0x4576698dUL, 0xb6fad407UL, 0x592af950UL, 0x36f73523UL, 0x4cfb6e87UL, +0x7da4cec0UL, 0x6c152daaUL, 0xcb0396a8UL, 0xc50dfe5dUL, 0xfcd707abUL, 0x0921c42fUL, +0x89dff0bbUL, 0x5fe2be78UL, 0x448f4f33UL, 0x754613c9UL, 0x2b05d08dUL, 0x48b9d585UL, +0xdc049441UL, 0xc8098f9bUL, 0x7dede786UL, 0xc39a3373UL, 0x42410005UL, 0x6a091751UL, +0x0ef3c8a6UL, 0x890072d6UL, 0x28207682UL, 0xa9a9f7beUL, 0xbf32679dUL, 0xd45b5b75UL, +0xb353fd00UL, 0xcbb0e358UL, 0x830f220aUL, 0x1f8fb214UL, 0xd372cf08UL, 0xcc3c4a13UL, +0x8cf63166UL, 0x061c87beUL, 0x88c98f88UL, 0x6062e397UL, 0x47cf8e7aUL, 0xb6c85283UL, +0x3cc2acfbUL, 0x3fc06976UL, 0x4e8f0252UL, 0x64d8314dUL, 0xda3870e3UL, 0x1e665459UL, +0xc10908f0UL, 0x513021a5UL, 0x6c5b68b7UL, 0x822f8aa0UL, 0x3007cd3eUL, 0x74719eefUL, +0xdc872681UL, 0x073340d4UL, 0x7e432fd9UL, 0x0c5ec241UL, 0x8809286cUL, 0xf592d891UL, +0x08a930f6UL, 0x957ef305UL, 0xb7fbffbdUL, 0xc266e96fUL, 0x6fe4ac98UL, 0xb173ecc0UL, +0xbc60b42aUL, 0x953498daUL, 0xfba1ae12UL, 0x2d4bd736UL, 0x0f25faabUL, 0xa4f3fcebUL, +0xe2969123UL, 0x257f0c3dUL, 0x9348af49UL, 0x361400bcUL, 0xe8816f4aUL, 0x3814f200UL, +0xa3f94043UL, 0x9c7a54c2UL, 0xbc704f57UL, 0xda41e7f9UL, 0xc25ad33aUL, 0x54f4a084UL, +0xb17f5505UL, 0x59357cbeUL, 0xedbd15c8UL, 0x7f97c5abUL, 0xba5ac7b5UL, 0xb6f6deafUL, +0x3a479c3aUL, 0x5302da25UL, 0x653d7e6aUL, 0x54268d49UL, 0x51a477eaUL, 0x5017d55bUL, +0xd7d25d88UL, 0x44136c76UL, 0x0404a8c8UL, 0xb8e5a121UL, 0xb81a928aUL, 0x60ed5869UL, +0x97c55b96UL, 0xeaec991bUL, 0x29935913UL, 0x01fdb7f1UL, 0x088e8dfaUL, 0x9ab6f6f5UL, +0x3b4cbf9fUL, 0x4a5de3abUL, 0xe6051d35UL, 0xa0e1d855UL, 0xd36b4cf1UL, 0xf544edebUL, +0xb0e93524UL, 0xbebb8fbdUL, 0xa2d762cfUL, 0x49c92f54UL, 0x38b5f331UL, 0x7128a454UL, +0x48392905UL, 0xa65b1db8UL, 0x851c97bdUL, 0xd675cf2fUL}; + +static const ulong32 S7[256] = { +0x85e04019UL, 0x332bf567UL, 0x662dbfffUL, 0xcfc65693UL, 0x2a8d7f6fUL, 0xab9bc912UL, +0xde6008a1UL, 0x2028da1fUL, 0x0227bce7UL, 0x4d642916UL, 0x18fac300UL, 0x50f18b82UL, +0x2cb2cb11UL, 0xb232e75cUL, 0x4b3695f2UL, 0xb28707deUL, 0xa05fbcf6UL, 0xcd4181e9UL, +0xe150210cUL, 0xe24ef1bdUL, 0xb168c381UL, 0xfde4e789UL, 0x5c79b0d8UL, 0x1e8bfd43UL, +0x4d495001UL, 0x38be4341UL, 0x913cee1dUL, 0x92a79c3fUL, 0x089766beUL, 0xbaeeadf4UL, +0x1286becfUL, 0xb6eacb19UL, 0x2660c200UL, 0x7565bde4UL, 0x64241f7aUL, 0x8248dca9UL, +0xc3b3ad66UL, 0x28136086UL, 0x0bd8dfa8UL, 0x356d1cf2UL, 0x107789beUL, 0xb3b2e9ceUL, +0x0502aa8fUL, 0x0bc0351eUL, 0x166bf52aUL, 0xeb12ff82UL, 0xe3486911UL, 0xd34d7516UL, +0x4e7b3affUL, 0x5f43671bUL, 0x9cf6e037UL, 0x4981ac83UL, 0x334266ceUL, 0x8c9341b7UL, +0xd0d854c0UL, 0xcb3a6c88UL, 0x47bc2829UL, 0x4725ba37UL, 0xa66ad22bUL, 0x7ad61f1eUL, +0x0c5cbafaUL, 0x4437f107UL, 0xb6e79962UL, 0x42d2d816UL, 0x0a961288UL, 0xe1a5c06eUL, +0x13749e67UL, 0x72fc081aUL, 0xb1d139f7UL, 0xf9583745UL, 0xcf19df58UL, 0xbec3f756UL, +0xc06eba30UL, 0x07211b24UL, 0x45c28829UL, 0xc95e317fUL, 0xbc8ec511UL, 0x38bc46e9UL, +0xc6e6fa14UL, 0xbae8584aUL, 0xad4ebc46UL, 0x468f508bUL, 0x7829435fUL, 0xf124183bUL, +0x821dba9fUL, 0xaff60ff4UL, 0xea2c4e6dUL, 0x16e39264UL, 0x92544a8bUL, 0x009b4fc3UL, +0xaba68cedUL, 0x9ac96f78UL, 0x06a5b79aUL, 0xb2856e6eUL, 0x1aec3ca9UL, 0xbe838688UL, +0x0e0804e9UL, 0x55f1be56UL, 0xe7e5363bUL, 0xb3a1f25dUL, 0xf7debb85UL, 0x61fe033cUL, +0x16746233UL, 0x3c034c28UL, 0xda6d0c74UL, 0x79aac56cUL, 0x3ce4e1adUL, 0x51f0c802UL, +0x98f8f35aUL, 0x1626a49fUL, 0xeed82b29UL, 0x1d382fe3UL, 0x0c4fb99aUL, 0xbb325778UL, +0x3ec6d97bUL, 0x6e77a6a9UL, 0xcb658b5cUL, 0xd45230c7UL, 0x2bd1408bUL, 0x60c03eb7UL, +0xb9068d78UL, 0xa33754f4UL, 0xf430c87dUL, 0xc8a71302UL, 0xb96d8c32UL, 0xebd4e7beUL, +0xbe8b9d2dUL, 0x7979fb06UL, 0xe7225308UL, 0x8b75cf77UL, 0x11ef8da4UL, 0xe083c858UL, +0x8d6b786fUL, 0x5a6317a6UL, 0xfa5cf7a0UL, 0x5dda0033UL, 0xf28ebfb0UL, 0xf5b9c310UL, +0xa0eac280UL, 0x08b9767aUL, 0xa3d9d2b0UL, 0x79d34217UL, 0x021a718dUL, 0x9ac6336aUL, +0x2711fd60UL, 0x438050e3UL, 0x069908a8UL, 0x3d7fedc4UL, 0x826d2befUL, 0x4eeb8476UL, +0x488dcf25UL, 0x36c9d566UL, 0x28e74e41UL, 0xc2610acaUL, 0x3d49a9cfUL, 0xbae3b9dfUL, +0xb65f8de6UL, 0x92aeaf64UL, 0x3ac7d5e6UL, 0x9ea80509UL, 0xf22b017dUL, 0xa4173f70UL, +0xdd1e16c3UL, 0x15e0d7f9UL, 0x50b1b887UL, 0x2b9f4fd5UL, 0x625aba82UL, 0x6a017962UL, +0x2ec01b9cUL, 0x15488aa9UL, 0xd716e740UL, 0x40055a2cUL, 0x93d29a22UL, 0xe32dbf9aUL, +0x058745b9UL, 0x3453dc1eUL, 0xd699296eUL, 0x496cff6fUL, 0x1c9f4986UL, 0xdfe2ed07UL, +0xb87242d1UL, 0x19de7eaeUL, 0x053e561aUL, 0x15ad6f8cUL, 0x66626c1cUL, 0x7154c24cUL, +0xea082b2aUL, 0x93eb2939UL, 0x17dcb0f0UL, 0x58d4f2aeUL, 0x9ea294fbUL, 0x52cf564cUL, +0x9883fe66UL, 0x2ec40581UL, 0x763953c3UL, 0x01d6692eUL, 0xd3a0c108UL, 0xa1e7160eUL, +0xe4f2dfa6UL, 0x693ed285UL, 0x74904698UL, 0x4c2b0eddUL, 0x4f757656UL, 0x5d393378UL, +0xa132234fUL, 0x3d321c5dUL, 0xc3f5e194UL, 0x4b269301UL, 0xc79f022fUL, 0x3c997e7eUL, +0x5e4f9504UL, 0x3ffafbbdUL, 0x76f7ad0eUL, 0x296693f4UL, 0x3d1fce6fUL, 0xc61e45beUL, +0xd3b5ab34UL, 0xf72bf9b7UL, 0x1b0434c0UL, 0x4e72b567UL, 0x5592a33dUL, 0xb5229301UL, +0xcfd2a87fUL, 0x60aeb767UL, 0x1814386bUL, 0x30bcc33dUL, 0x38a0c07dUL, 0xfd1606f2UL, +0xc363519bUL, 0x589dd390UL, 0x5479f8e6UL, 0x1cb8d647UL, 0x97fd61a9UL, 0xea7759f4UL, +0x2d57539dUL, 0x569a58cfUL, 0xe84e63adUL, 0x462e1b78UL, 0x6580f87eUL, 0xf3817914UL, +0x91da55f4UL, 0x40a230f3UL, 0xd1988f35UL, 0xb6e318d2UL, 0x3ffa50bcUL, 0x3d40f021UL, +0xc3c0bdaeUL, 0x4958c24cUL, 0x518f36b2UL, 0x84b1d370UL, 0x0fedce83UL, 0x878ddadaUL, +0xf2a279c7UL, 0x94e01be8UL, 0x90716f4bUL, 0x954b8aa3UL}; + +static const ulong32 S8[256] = { +0xe216300dUL, 0xbbddfffcUL, 0xa7ebdabdUL, 0x35648095UL, 0x7789f8b7UL, 0xe6c1121bUL, +0x0e241600UL, 0x052ce8b5UL, 0x11a9cfb0UL, 0xe5952f11UL, 0xece7990aUL, 0x9386d174UL, +0x2a42931cUL, 0x76e38111UL, 0xb12def3aUL, 0x37ddddfcUL, 0xde9adeb1UL, 0x0a0cc32cUL, +0xbe197029UL, 0x84a00940UL, 0xbb243a0fUL, 0xb4d137cfUL, 0xb44e79f0UL, 0x049eedfdUL, +0x0b15a15dUL, 0x480d3168UL, 0x8bbbde5aUL, 0x669ded42UL, 0xc7ece831UL, 0x3f8f95e7UL, +0x72df191bUL, 0x7580330dUL, 0x94074251UL, 0x5c7dcdfaUL, 0xabbe6d63UL, 0xaa402164UL, +0xb301d40aUL, 0x02e7d1caUL, 0x53571daeUL, 0x7a3182a2UL, 0x12a8ddecUL, 0xfdaa335dUL, +0x176f43e8UL, 0x71fb46d4UL, 0x38129022UL, 0xce949ad4UL, 0xb84769adUL, 0x965bd862UL, +0x82f3d055UL, 0x66fb9767UL, 0x15b80b4eUL, 0x1d5b47a0UL, 0x4cfde06fUL, 0xc28ec4b8UL, +0x57e8726eUL, 0x647a78fcUL, 0x99865d44UL, 0x608bd593UL, 0x6c200e03UL, 0x39dc5ff6UL, +0x5d0b00a3UL, 0xae63aff2UL, 0x7e8bd632UL, 0x70108c0cUL, 0xbbd35049UL, 0x2998df04UL, +0x980cf42aUL, 0x9b6df491UL, 0x9e7edd53UL, 0x06918548UL, 0x58cb7e07UL, 0x3b74ef2eUL, +0x522fffb1UL, 0xd24708ccUL, 0x1c7e27cdUL, 0xa4eb215bUL, 0x3cf1d2e2UL, 0x19b47a38UL, +0x424f7618UL, 0x35856039UL, 0x9d17dee7UL, 0x27eb35e6UL, 0xc9aff67bUL, 0x36baf5b8UL, +0x09c467cdUL, 0xc18910b1UL, 0xe11dbf7bUL, 0x06cd1af8UL, 0x7170c608UL, 0x2d5e3354UL, +0xd4de495aUL, 0x64c6d006UL, 0xbcc0c62cUL, 0x3dd00db3UL, 0x708f8f34UL, 0x77d51b42UL, +0x264f620fUL, 0x24b8d2bfUL, 0x15c1b79eUL, 0x46a52564UL, 0xf8d7e54eUL, 0x3e378160UL, +0x7895cda5UL, 0x859c15a5UL, 0xe6459788UL, 0xc37bc75fUL, 0xdb07ba0cUL, 0x0676a3abUL, +0x7f229b1eUL, 0x31842e7bUL, 0x24259fd7UL, 0xf8bef472UL, 0x835ffcb8UL, 0x6df4c1f2UL, +0x96f5b195UL, 0xfd0af0fcUL, 0xb0fe134cUL, 0xe2506d3dUL, 0x4f9b12eaUL, 0xf215f225UL, +0xa223736fUL, 0x9fb4c428UL, 0x25d04979UL, 0x34c713f8UL, 0xc4618187UL, 0xea7a6e98UL, +0x7cd16efcUL, 0x1436876cUL, 0xf1544107UL, 0xbedeee14UL, 0x56e9af27UL, 0xa04aa441UL, +0x3cf7c899UL, 0x92ecbae6UL, 0xdd67016dUL, 0x151682ebUL, 0xa842eedfUL, 0xfdba60b4UL, +0xf1907b75UL, 0x20e3030fUL, 0x24d8c29eUL, 0xe139673bUL, 0xefa63fb8UL, 0x71873054UL, +0xb6f2cf3bUL, 0x9f326442UL, 0xcb15a4ccUL, 0xb01a4504UL, 0xf1e47d8dUL, 0x844a1be5UL, +0xbae7dfdcUL, 0x42cbda70UL, 0xcd7dae0aUL, 0x57e85b7aUL, 0xd53f5af6UL, 0x20cf4d8cUL, +0xcea4d428UL, 0x79d130a4UL, 0x3486ebfbUL, 0x33d3cddcUL, 0x77853b53UL, 0x37effcb5UL, +0xc5068778UL, 0xe580b3e6UL, 0x4e68b8f4UL, 0xc5c8b37eUL, 0x0d809ea2UL, 0x398feb7cUL, +0x132a4f94UL, 0x43b7950eUL, 0x2fee7d1cUL, 0x223613bdUL, 0xdd06caa2UL, 0x37df932bUL, +0xc4248289UL, 0xacf3ebc3UL, 0x5715f6b7UL, 0xef3478ddUL, 0xf267616fUL, 0xc148cbe4UL, +0x9052815eUL, 0x5e410fabUL, 0xb48a2465UL, 0x2eda7fa4UL, 0xe87b40e4UL, 0xe98ea084UL, +0x5889e9e1UL, 0xefd390fcUL, 0xdd07d35bUL, 0xdb485694UL, 0x38d7e5b2UL, 0x57720101UL, +0x730edebcUL, 0x5b643113UL, 0x94917e4fUL, 0x503c2fbaUL, 0x646f1282UL, 0x7523d24aUL, +0xe0779695UL, 0xf9c17a8fUL, 0x7a5b2121UL, 0xd187b896UL, 0x29263a4dUL, 0xba510cdfUL, +0x81f47c9fUL, 0xad1163edUL, 0xea7b5965UL, 0x1a00726eUL, 0x11403092UL, 0x00da6d77UL, +0x4a0cdd61UL, 0xad1f4603UL, 0x605bdfb0UL, 0x9eedc364UL, 0x22ebe6a8UL, 0xcee7d28aUL, +0xa0e736a0UL, 0x5564a6b9UL, 0x10853209UL, 0xc7eb8f37UL, 0x2de705caUL, 0x8951570fUL, +0xdf09822bUL, 0xbd691a6cUL, 0xaa12e4f2UL, 0x87451c0fUL, 0xe0f6a27aUL, 0x3ada4819UL, +0x4cf1764fUL, 0x0d771c2bUL, 0x67cdb156UL, 0x350d8384UL, 0x5938fa0fUL, 0x42399ef3UL, +0x36997b07UL, 0x0e84093dUL, 0x4aa93e61UL, 0x8360d87bUL, 0x1fa98b0cUL, 0x1149382cUL, +0xe97625a5UL, 0x0614d1b7UL, 0x0e25244bUL, 0x0c768347UL, 0x589e8d82UL, 0x0d2059d1UL, +0xa466bb1eUL, 0xf8da0a82UL, 0x04f19130UL, 0xba6e4ec0UL, 0x99265164UL, 0x1ee7230dUL, +0x50b2ad80UL, 0xeaee6801UL, 0x8db2a283UL, 0xea8bf59eUL}; + +/* returns the i'th byte of a variable */ +#ifdef _MSC_VER + #define GB(x, i) ((unsigned char)((x[(15-i)>>2])>>(unsigned)(8*((15-i)&3)))) +#else + #define GB(x, i) (((x[(15-i)>>2])>>(unsigned)(8*((15-i)&3)))&255) +#endif + + /** + Initialize the LTC_CAST5 block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +#ifdef LTC_CLEAN_STACK +static int s_cast5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#else +int cast5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#endif +{ + ulong32 x[4], z[4]; + unsigned char buf[16]; + int y, i; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 12 && num_rounds != 16 && num_rounds != 0) { + return CRYPT_INVALID_ROUNDS; + } + + if (num_rounds == 12 && keylen > 10) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen < 5 || keylen > 16) { + return CRYPT_INVALID_KEYSIZE; + } + + /* extend the key as required */ + zeromem(buf, sizeof(buf)); + XMEMCPY(buf, key, (size_t)keylen); + + /* load and start the awful looking network */ + for (y = 0; y < 4; y++) { + LOAD32H(x[3-y],buf+4*y); + } + + for (i = y = 0; y < 2; y++) { + z[3] = x[3] ^ S5[GB(x, 0xD)] ^ S6[GB(x, 0xF)] ^ S7[GB(x, 0xC)] ^ S8[GB(x, 0xE)] ^ S7[GB(x, 0x8)]; + z[2] = x[1] ^ S5[GB(z, 0x0)] ^ S6[GB(z, 0x2)] ^ S7[GB(z, 0x1)] ^ S8[GB(z, 0x3)] ^ S8[GB(x, 0xA)]; + z[1] = x[0] ^ S5[GB(z, 0x7)] ^ S6[GB(z, 0x6)] ^ S7[GB(z, 0x5)] ^ S8[GB(z, 0x4)] ^ S5[GB(x, 0x9)]; + z[0] = x[2] ^ S5[GB(z, 0xA)] ^ S6[GB(z, 0x9)] ^ S7[GB(z, 0xb)] ^ S8[GB(z, 0x8)] ^ S6[GB(x, 0xB)]; + skey->cast5.K[i++] = S5[GB(z, 0x8)] ^ S6[GB(z, 0x9)] ^ S7[GB(z, 0x7)] ^ S8[GB(z, 0x6)] ^ S5[GB(z, 0x2)]; + skey->cast5.K[i++] = S5[GB(z, 0xA)] ^ S6[GB(z, 0xB)] ^ S7[GB(z, 0x5)] ^ S8[GB(z, 0x4)] ^ S6[GB(z, 0x6)]; + skey->cast5.K[i++] = S5[GB(z, 0xC)] ^ S6[GB(z, 0xd)] ^ S7[GB(z, 0x3)] ^ S8[GB(z, 0x2)] ^ S7[GB(z, 0x9)]; + skey->cast5.K[i++] = S5[GB(z, 0xE)] ^ S6[GB(z, 0xF)] ^ S7[GB(z, 0x1)] ^ S8[GB(z, 0x0)] ^ S8[GB(z, 0xc)]; + + x[3] = z[1] ^ S5[GB(z, 0x5)] ^ S6[GB(z, 0x7)] ^ S7[GB(z, 0x4)] ^ S8[GB(z, 0x6)] ^ S7[GB(z, 0x0)]; + x[2] = z[3] ^ S5[GB(x, 0x0)] ^ S6[GB(x, 0x2)] ^ S7[GB(x, 0x1)] ^ S8[GB(x, 0x3)] ^ S8[GB(z, 0x2)]; + x[1] = z[2] ^ S5[GB(x, 0x7)] ^ S6[GB(x, 0x6)] ^ S7[GB(x, 0x5)] ^ S8[GB(x, 0x4)] ^ S5[GB(z, 0x1)]; + x[0] = z[0] ^ S5[GB(x, 0xA)] ^ S6[GB(x, 0x9)] ^ S7[GB(x, 0xb)] ^ S8[GB(x, 0x8)] ^ S6[GB(z, 0x3)]; + skey->cast5.K[i++] = S5[GB(x, 0x3)] ^ S6[GB(x, 0x2)] ^ S7[GB(x, 0xc)] ^ S8[GB(x, 0xd)] ^ S5[GB(x, 0x8)]; + skey->cast5.K[i++] = S5[GB(x, 0x1)] ^ S6[GB(x, 0x0)] ^ S7[GB(x, 0xe)] ^ S8[GB(x, 0xf)] ^ S6[GB(x, 0xd)]; + skey->cast5.K[i++] = S5[GB(x, 0x7)] ^ S6[GB(x, 0x6)] ^ S7[GB(x, 0x8)] ^ S8[GB(x, 0x9)] ^ S7[GB(x, 0x3)]; + skey->cast5.K[i++] = S5[GB(x, 0x5)] ^ S6[GB(x, 0x4)] ^ S7[GB(x, 0xa)] ^ S8[GB(x, 0xb)] ^ S8[GB(x, 0x7)]; + + /* second half */ + z[3] = x[3] ^ S5[GB(x, 0xD)] ^ S6[GB(x, 0xF)] ^ S7[GB(x, 0xC)] ^ S8[GB(x, 0xE)] ^ S7[GB(x, 0x8)]; + z[2] = x[1] ^ S5[GB(z, 0x0)] ^ S6[GB(z, 0x2)] ^ S7[GB(z, 0x1)] ^ S8[GB(z, 0x3)] ^ S8[GB(x, 0xA)]; + z[1] = x[0] ^ S5[GB(z, 0x7)] ^ S6[GB(z, 0x6)] ^ S7[GB(z, 0x5)] ^ S8[GB(z, 0x4)] ^ S5[GB(x, 0x9)]; + z[0] = x[2] ^ S5[GB(z, 0xA)] ^ S6[GB(z, 0x9)] ^ S7[GB(z, 0xb)] ^ S8[GB(z, 0x8)] ^ S6[GB(x, 0xB)]; + skey->cast5.K[i++] = S5[GB(z, 0x3)] ^ S6[GB(z, 0x2)] ^ S7[GB(z, 0xc)] ^ S8[GB(z, 0xd)] ^ S5[GB(z, 0x9)]; + skey->cast5.K[i++] = S5[GB(z, 0x1)] ^ S6[GB(z, 0x0)] ^ S7[GB(z, 0xe)] ^ S8[GB(z, 0xf)] ^ S6[GB(z, 0xc)]; + skey->cast5.K[i++] = S5[GB(z, 0x7)] ^ S6[GB(z, 0x6)] ^ S7[GB(z, 0x8)] ^ S8[GB(z, 0x9)] ^ S7[GB(z, 0x2)]; + skey->cast5.K[i++] = S5[GB(z, 0x5)] ^ S6[GB(z, 0x4)] ^ S7[GB(z, 0xa)] ^ S8[GB(z, 0xb)] ^ S8[GB(z, 0x6)]; + + x[3] = z[1] ^ S5[GB(z, 0x5)] ^ S6[GB(z, 0x7)] ^ S7[GB(z, 0x4)] ^ S8[GB(z, 0x6)] ^ S7[GB(z, 0x0)]; + x[2] = z[3] ^ S5[GB(x, 0x0)] ^ S6[GB(x, 0x2)] ^ S7[GB(x, 0x1)] ^ S8[GB(x, 0x3)] ^ S8[GB(z, 0x2)]; + x[1] = z[2] ^ S5[GB(x, 0x7)] ^ S6[GB(x, 0x6)] ^ S7[GB(x, 0x5)] ^ S8[GB(x, 0x4)] ^ S5[GB(z, 0x1)]; + x[0] = z[0] ^ S5[GB(x, 0xA)] ^ S6[GB(x, 0x9)] ^ S7[GB(x, 0xb)] ^ S8[GB(x, 0x8)] ^ S6[GB(z, 0x3)]; + skey->cast5.K[i++] = S5[GB(x, 0x8)] ^ S6[GB(x, 0x9)] ^ S7[GB(x, 0x7)] ^ S8[GB(x, 0x6)] ^ S5[GB(x, 0x3)]; + skey->cast5.K[i++] = S5[GB(x, 0xa)] ^ S6[GB(x, 0xb)] ^ S7[GB(x, 0x5)] ^ S8[GB(x, 0x4)] ^ S6[GB(x, 0x7)]; + skey->cast5.K[i++] = S5[GB(x, 0xc)] ^ S6[GB(x, 0xd)] ^ S7[GB(x, 0x3)] ^ S8[GB(x, 0x2)] ^ S7[GB(x, 0x8)]; + skey->cast5.K[i++] = S5[GB(x, 0xe)] ^ S6[GB(x, 0xf)] ^ S7[GB(x, 0x1)] ^ S8[GB(x, 0x0)] ^ S8[GB(x, 0xd)]; + } + + skey->cast5.keylen = keylen; + +#ifdef LTC_CLEAN_STACK + zeromem(buf, sizeof(buf)); + zeromem(x, sizeof(x)); + zeromem(z, sizeof(z)); +#endif + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int cast5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int z; + z = s_cast5_setup(key, keylen, num_rounds, skey); + burn_stack(sizeof(ulong32)*8 + 16 + sizeof(int)*2); + return z; +} +#endif + +LTC_INLINE static ulong32 FI(ulong32 R, ulong32 Km, ulong32 Kr) +{ + ulong32 I; + I = (Km + R); + I = ROL(I, Kr); + return ((S1[LTC_BYTE(I, 3)] ^ S2[LTC_BYTE(I,2)]) - S3[LTC_BYTE(I,1)]) + S4[LTC_BYTE(I,0)]; +} + +LTC_INLINE static ulong32 FII(ulong32 R, ulong32 Km, ulong32 Kr) +{ + ulong32 I; + I = (Km ^ R); + I = ROL(I, Kr); + return ((S1[LTC_BYTE(I, 3)] - S2[LTC_BYTE(I,2)]) + S3[LTC_BYTE(I,1)]) ^ S4[LTC_BYTE(I,0)]; +} + +LTC_INLINE static ulong32 FIII(ulong32 R, ulong32 Km, ulong32 Kr) +{ + ulong32 I; + I = (Km - R); + I = ROL(I, Kr); + return ((S1[LTC_BYTE(I, 3)] + S2[LTC_BYTE(I,2)]) ^ S3[LTC_BYTE(I,1)]) - S4[LTC_BYTE(I,0)]; +} + +/** + Encrypts a block of text with LTC_CAST5 + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled +*/ +#ifdef LTC_CLEAN_STACK +static int s_cast5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int cast5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + ulong32 R, L; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(L,&pt[0]); + LOAD32H(R,&pt[4]); + L ^= FI(R, skey->cast5.K[0], skey->cast5.K[16]); + R ^= FII(L, skey->cast5.K[1], skey->cast5.K[17]); + L ^= FIII(R, skey->cast5.K[2], skey->cast5.K[18]); + R ^= FI(L, skey->cast5.K[3], skey->cast5.K[19]); + L ^= FII(R, skey->cast5.K[4], skey->cast5.K[20]); + R ^= FIII(L, skey->cast5.K[5], skey->cast5.K[21]); + L ^= FI(R, skey->cast5.K[6], skey->cast5.K[22]); + R ^= FII(L, skey->cast5.K[7], skey->cast5.K[23]); + L ^= FIII(R, skey->cast5.K[8], skey->cast5.K[24]); + R ^= FI(L, skey->cast5.K[9], skey->cast5.K[25]); + L ^= FII(R, skey->cast5.K[10], skey->cast5.K[26]); + R ^= FIII(L, skey->cast5.K[11], skey->cast5.K[27]); + if (skey->cast5.keylen > 10) { + L ^= FI(R, skey->cast5.K[12], skey->cast5.K[28]); + R ^= FII(L, skey->cast5.K[13], skey->cast5.K[29]); + L ^= FIII(R, skey->cast5.K[14], skey->cast5.K[30]); + R ^= FI(L, skey->cast5.K[15], skey->cast5.K[31]); + } + STORE32H(R,&ct[0]); + STORE32H(L,&ct[4]); + return CRYPT_OK; +} + + +#ifdef LTC_CLEAN_STACK +int cast5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_cast5_ecb_encrypt(pt,ct,skey); + burn_stack(sizeof(ulong32)*3); + return err; +} +#endif + +/** + Decrypts a block of text with LTC_CAST5 + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled +*/ +#ifdef LTC_CLEAN_STACK +static int s_cast5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int cast5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + ulong32 R, L; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(R,&ct[0]); + LOAD32H(L,&ct[4]); + if (skey->cast5.keylen > 10) { + R ^= FI(L, skey->cast5.K[15], skey->cast5.K[31]); + L ^= FIII(R, skey->cast5.K[14], skey->cast5.K[30]); + R ^= FII(L, skey->cast5.K[13], skey->cast5.K[29]); + L ^= FI(R, skey->cast5.K[12], skey->cast5.K[28]); + } + R ^= FIII(L, skey->cast5.K[11], skey->cast5.K[27]); + L ^= FII(R, skey->cast5.K[10], skey->cast5.K[26]); + R ^= FI(L, skey->cast5.K[9], skey->cast5.K[25]); + L ^= FIII(R, skey->cast5.K[8], skey->cast5.K[24]); + R ^= FII(L, skey->cast5.K[7], skey->cast5.K[23]); + L ^= FI(R, skey->cast5.K[6], skey->cast5.K[22]); + R ^= FIII(L, skey->cast5.K[5], skey->cast5.K[21]); + L ^= FII(R, skey->cast5.K[4], skey->cast5.K[20]); + R ^= FI(L, skey->cast5.K[3], skey->cast5.K[19]); + L ^= FIII(R, skey->cast5.K[2], skey->cast5.K[18]); + R ^= FII(L, skey->cast5.K[1], skey->cast5.K[17]); + L ^= FI(R, skey->cast5.K[0], skey->cast5.K[16]); + STORE32H(L,&pt[0]); + STORE32H(R,&pt[4]); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int cast5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_cast5_ecb_decrypt(ct,pt,skey); + burn_stack(sizeof(ulong32)*3); + return err; +} +#endif + +/** + Performs a self-test of the LTC_CAST5 block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int cast5_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + int keylen; + unsigned char key[16]; + unsigned char pt[8]; + unsigned char ct[8]; + } tests[] = { + { 16, + {0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x67, 0x89, 0x34, 0x56, 0x78, 0x9A}, + {0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + {0x23, 0x8B, 0x4F, 0xE5, 0x84, 0x7E, 0x44, 0xB2} + }, + { 10, + {0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + {0xEB, 0x6A, 0x71, 0x1A, 0x2C, 0x02, 0x27, 0x1B}, + }, + { 5, + {0x01, 0x23, 0x45, 0x67, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + {0x7A, 0xC8, 0x16, 0xD1, 0x6E, 0x9B, 0x30, 0x2E} + } + }; + int i, y, err; + symmetric_key key; + unsigned char tmp[2][8]; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + if ((err = cast5_setup(tests[i].key, tests[i].keylen, 0, &key)) != CRYPT_OK) { + return err; + } + cast5_ecb_encrypt(tests[i].pt, tmp[0], &key); + cast5_ecb_decrypt(tmp[0], tmp[1], &key); + if ((compare_testvector(tmp[0], 8, tests[i].ct, 8, "CAST5 Encrypt", i) != 0) || + (compare_testvector(tmp[1], 8, tests[i].pt, 8, "CAST5 Decrypt", i) != 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) cast5_ecb_encrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 1000; y++) cast5_ecb_decrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 8; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + + } + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void cast5_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int cast5_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 5) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize > 16) { + *keysize = 16; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/des.c b/optee_os/core/lib/libtomcrypt/src/ciphers/des.c new file mode 100644 index 0000000..5d00720 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/des.c @@ -0,0 +1,2093 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file des.c + DES code submitted by Dobes Vandermeer +*/ + +#ifdef LTC_DES + +#define EN0 0 +#define DE1 1 + +const struct ltc_cipher_descriptor des_desc = +{ + "des", + 13, + 8, 8, 8, 16, + &des_setup, + &des_ecb_encrypt, + &des_ecb_decrypt, + &des_test, + &des_done, + &des_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +const struct ltc_cipher_descriptor des3_desc = +{ + "3des", + 14, + 16, 24, 8, 16, + &des3_setup, + &des3_ecb_encrypt, + &des3_ecb_decrypt, + &des3_test, + &des3_done, + &des3_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 bytebit[8] = +{ + 0200, 0100, 040, 020, 010, 04, 02, 01 +}; + +static const ulong32 bigbyte[24] = +{ + 0x800000UL, 0x400000UL, 0x200000UL, 0x100000UL, + 0x80000UL, 0x40000UL, 0x20000UL, 0x10000UL, + 0x8000UL, 0x4000UL, 0x2000UL, 0x1000UL, + 0x800UL, 0x400UL, 0x200UL, 0x100UL, + 0x80UL, 0x40UL, 0x20UL, 0x10UL, + 0x8UL, 0x4UL, 0x2UL, 0x1L +}; + +/* Use the key schedule specific in the standard (ANSI X3.92-1981) */ + +static const unsigned char pc1[56] = { + 56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17, + 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35, + 62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, + 13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3 +}; + +static const unsigned char totrot[16] = { + 1, 2, 4, 6, + 8, 10, 12, 14, + 15, 17, 19, 21, + 23, 25, 27, 28 +}; + +static const unsigned char pc2[48] = { + 13, 16, 10, 23, 0, 4, 2, 27, 14, 5, 20, 9, + 22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1, + 40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47, + 43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 +}; + + +static const ulong32 SP1[64] = +{ + 0x01010400UL, 0x00000000UL, 0x00010000UL, 0x01010404UL, + 0x01010004UL, 0x00010404UL, 0x00000004UL, 0x00010000UL, + 0x00000400UL, 0x01010400UL, 0x01010404UL, 0x00000400UL, + 0x01000404UL, 0x01010004UL, 0x01000000UL, 0x00000004UL, + 0x00000404UL, 0x01000400UL, 0x01000400UL, 0x00010400UL, + 0x00010400UL, 0x01010000UL, 0x01010000UL, 0x01000404UL, + 0x00010004UL, 0x01000004UL, 0x01000004UL, 0x00010004UL, + 0x00000000UL, 0x00000404UL, 0x00010404UL, 0x01000000UL, + 0x00010000UL, 0x01010404UL, 0x00000004UL, 0x01010000UL, + 0x01010400UL, 0x01000000UL, 0x01000000UL, 0x00000400UL, + 0x01010004UL, 0x00010000UL, 0x00010400UL, 0x01000004UL, + 0x00000400UL, 0x00000004UL, 0x01000404UL, 0x00010404UL, + 0x01010404UL, 0x00010004UL, 0x01010000UL, 0x01000404UL, + 0x01000004UL, 0x00000404UL, 0x00010404UL, 0x01010400UL, + 0x00000404UL, 0x01000400UL, 0x01000400UL, 0x00000000UL, + 0x00010004UL, 0x00010400UL, 0x00000000UL, 0x01010004UL +}; + +static const ulong32 SP2[64] = +{ + 0x80108020UL, 0x80008000UL, 0x00008000UL, 0x00108020UL, + 0x00100000UL, 0x00000020UL, 0x80100020UL, 0x80008020UL, + 0x80000020UL, 0x80108020UL, 0x80108000UL, 0x80000000UL, + 0x80008000UL, 0x00100000UL, 0x00000020UL, 0x80100020UL, + 0x00108000UL, 0x00100020UL, 0x80008020UL, 0x00000000UL, + 0x80000000UL, 0x00008000UL, 0x00108020UL, 0x80100000UL, + 0x00100020UL, 0x80000020UL, 0x00000000UL, 0x00108000UL, + 0x00008020UL, 0x80108000UL, 0x80100000UL, 0x00008020UL, + 0x00000000UL, 0x00108020UL, 0x80100020UL, 0x00100000UL, + 0x80008020UL, 0x80100000UL, 0x80108000UL, 0x00008000UL, + 0x80100000UL, 0x80008000UL, 0x00000020UL, 0x80108020UL, + 0x00108020UL, 0x00000020UL, 0x00008000UL, 0x80000000UL, + 0x00008020UL, 0x80108000UL, 0x00100000UL, 0x80000020UL, + 0x00100020UL, 0x80008020UL, 0x80000020UL, 0x00100020UL, + 0x00108000UL, 0x00000000UL, 0x80008000UL, 0x00008020UL, + 0x80000000UL, 0x80100020UL, 0x80108020UL, 0x00108000UL +}; + +static const ulong32 SP3[64] = +{ + 0x00000208UL, 0x08020200UL, 0x00000000UL, 0x08020008UL, + 0x08000200UL, 0x00000000UL, 0x00020208UL, 0x08000200UL, + 0x00020008UL, 0x08000008UL, 0x08000008UL, 0x00020000UL, + 0x08020208UL, 0x00020008UL, 0x08020000UL, 0x00000208UL, + 0x08000000UL, 0x00000008UL, 0x08020200UL, 0x00000200UL, + 0x00020200UL, 0x08020000UL, 0x08020008UL, 0x00020208UL, + 0x08000208UL, 0x00020200UL, 0x00020000UL, 0x08000208UL, + 0x00000008UL, 0x08020208UL, 0x00000200UL, 0x08000000UL, + 0x08020200UL, 0x08000000UL, 0x00020008UL, 0x00000208UL, + 0x00020000UL, 0x08020200UL, 0x08000200UL, 0x00000000UL, + 0x00000200UL, 0x00020008UL, 0x08020208UL, 0x08000200UL, + 0x08000008UL, 0x00000200UL, 0x00000000UL, 0x08020008UL, + 0x08000208UL, 0x00020000UL, 0x08000000UL, 0x08020208UL, + 0x00000008UL, 0x00020208UL, 0x00020200UL, 0x08000008UL, + 0x08020000UL, 0x08000208UL, 0x00000208UL, 0x08020000UL, + 0x00020208UL, 0x00000008UL, 0x08020008UL, 0x00020200UL +}; + +static const ulong32 SP4[64] = +{ + 0x00802001UL, 0x00002081UL, 0x00002081UL, 0x00000080UL, + 0x00802080UL, 0x00800081UL, 0x00800001UL, 0x00002001UL, + 0x00000000UL, 0x00802000UL, 0x00802000UL, 0x00802081UL, + 0x00000081UL, 0x00000000UL, 0x00800080UL, 0x00800001UL, + 0x00000001UL, 0x00002000UL, 0x00800000UL, 0x00802001UL, + 0x00000080UL, 0x00800000UL, 0x00002001UL, 0x00002080UL, + 0x00800081UL, 0x00000001UL, 0x00002080UL, 0x00800080UL, + 0x00002000UL, 0x00802080UL, 0x00802081UL, 0x00000081UL, + 0x00800080UL, 0x00800001UL, 0x00802000UL, 0x00802081UL, + 0x00000081UL, 0x00000000UL, 0x00000000UL, 0x00802000UL, + 0x00002080UL, 0x00800080UL, 0x00800081UL, 0x00000001UL, + 0x00802001UL, 0x00002081UL, 0x00002081UL, 0x00000080UL, + 0x00802081UL, 0x00000081UL, 0x00000001UL, 0x00002000UL, + 0x00800001UL, 0x00002001UL, 0x00802080UL, 0x00800081UL, + 0x00002001UL, 0x00002080UL, 0x00800000UL, 0x00802001UL, + 0x00000080UL, 0x00800000UL, 0x00002000UL, 0x00802080UL +}; + +static const ulong32 SP5[64] = +{ + 0x00000100UL, 0x02080100UL, 0x02080000UL, 0x42000100UL, + 0x00080000UL, 0x00000100UL, 0x40000000UL, 0x02080000UL, + 0x40080100UL, 0x00080000UL, 0x02000100UL, 0x40080100UL, + 0x42000100UL, 0x42080000UL, 0x00080100UL, 0x40000000UL, + 0x02000000UL, 0x40080000UL, 0x40080000UL, 0x00000000UL, + 0x40000100UL, 0x42080100UL, 0x42080100UL, 0x02000100UL, + 0x42080000UL, 0x40000100UL, 0x00000000UL, 0x42000000UL, + 0x02080100UL, 0x02000000UL, 0x42000000UL, 0x00080100UL, + 0x00080000UL, 0x42000100UL, 0x00000100UL, 0x02000000UL, + 0x40000000UL, 0x02080000UL, 0x42000100UL, 0x40080100UL, + 0x02000100UL, 0x40000000UL, 0x42080000UL, 0x02080100UL, + 0x40080100UL, 0x00000100UL, 0x02000000UL, 0x42080000UL, + 0x42080100UL, 0x00080100UL, 0x42000000UL, 0x42080100UL, + 0x02080000UL, 0x00000000UL, 0x40080000UL, 0x42000000UL, + 0x00080100UL, 0x02000100UL, 0x40000100UL, 0x00080000UL, + 0x00000000UL, 0x40080000UL, 0x02080100UL, 0x40000100UL +}; + +static const ulong32 SP6[64] = +{ + 0x20000010UL, 0x20400000UL, 0x00004000UL, 0x20404010UL, + 0x20400000UL, 0x00000010UL, 0x20404010UL, 0x00400000UL, + 0x20004000UL, 0x00404010UL, 0x00400000UL, 0x20000010UL, + 0x00400010UL, 0x20004000UL, 0x20000000UL, 0x00004010UL, + 0x00000000UL, 0x00400010UL, 0x20004010UL, 0x00004000UL, + 0x00404000UL, 0x20004010UL, 0x00000010UL, 0x20400010UL, + 0x20400010UL, 0x00000000UL, 0x00404010UL, 0x20404000UL, + 0x00004010UL, 0x00404000UL, 0x20404000UL, 0x20000000UL, + 0x20004000UL, 0x00000010UL, 0x20400010UL, 0x00404000UL, + 0x20404010UL, 0x00400000UL, 0x00004010UL, 0x20000010UL, + 0x00400000UL, 0x20004000UL, 0x20000000UL, 0x00004010UL, + 0x20000010UL, 0x20404010UL, 0x00404000UL, 0x20400000UL, + 0x00404010UL, 0x20404000UL, 0x00000000UL, 0x20400010UL, + 0x00000010UL, 0x00004000UL, 0x20400000UL, 0x00404010UL, + 0x00004000UL, 0x00400010UL, 0x20004010UL, 0x00000000UL, + 0x20404000UL, 0x20000000UL, 0x00400010UL, 0x20004010UL +}; + +static const ulong32 SP7[64] = +{ + 0x00200000UL, 0x04200002UL, 0x04000802UL, 0x00000000UL, + 0x00000800UL, 0x04000802UL, 0x00200802UL, 0x04200800UL, + 0x04200802UL, 0x00200000UL, 0x00000000UL, 0x04000002UL, + 0x00000002UL, 0x04000000UL, 0x04200002UL, 0x00000802UL, + 0x04000800UL, 0x00200802UL, 0x00200002UL, 0x04000800UL, + 0x04000002UL, 0x04200000UL, 0x04200800UL, 0x00200002UL, + 0x04200000UL, 0x00000800UL, 0x00000802UL, 0x04200802UL, + 0x00200800UL, 0x00000002UL, 0x04000000UL, 0x00200800UL, + 0x04000000UL, 0x00200800UL, 0x00200000UL, 0x04000802UL, + 0x04000802UL, 0x04200002UL, 0x04200002UL, 0x00000002UL, + 0x00200002UL, 0x04000000UL, 0x04000800UL, 0x00200000UL, + 0x04200800UL, 0x00000802UL, 0x00200802UL, 0x04200800UL, + 0x00000802UL, 0x04000002UL, 0x04200802UL, 0x04200000UL, + 0x00200800UL, 0x00000000UL, 0x00000002UL, 0x04200802UL, + 0x00000000UL, 0x00200802UL, 0x04200000UL, 0x00000800UL, + 0x04000002UL, 0x04000800UL, 0x00000800UL, 0x00200002UL +}; + +static const ulong32 SP8[64] = +{ + 0x10001040UL, 0x00001000UL, 0x00040000UL, 0x10041040UL, + 0x10000000UL, 0x10001040UL, 0x00000040UL, 0x10000000UL, + 0x00040040UL, 0x10040000UL, 0x10041040UL, 0x00041000UL, + 0x10041000UL, 0x00041040UL, 0x00001000UL, 0x00000040UL, + 0x10040000UL, 0x10000040UL, 0x10001000UL, 0x00001040UL, + 0x00041000UL, 0x00040040UL, 0x10040040UL, 0x10041000UL, + 0x00001040UL, 0x00000000UL, 0x00000000UL, 0x10040040UL, + 0x10000040UL, 0x10001000UL, 0x00041040UL, 0x00040000UL, + 0x00041040UL, 0x00040000UL, 0x10041000UL, 0x00001000UL, + 0x00000040UL, 0x10040040UL, 0x00001000UL, 0x00041040UL, + 0x10001000UL, 0x00000040UL, 0x10000040UL, 0x10040000UL, + 0x10040040UL, 0x10000000UL, 0x00040000UL, 0x10001040UL, + 0x00000000UL, 0x10041040UL, 0x00040040UL, 0x10000040UL, + 0x10040000UL, 0x10001000UL, 0x10001040UL, 0x00000000UL, + 0x10041040UL, 0x00041000UL, 0x00041000UL, 0x00001040UL, + 0x00001040UL, 0x00040040UL, 0x10000000UL, 0x10041000UL +}; + +#ifndef LTC_SMALL_CODE + +static const ulong64 des_ip[8][256] = { + +{ CONST64(0x0000000000000000), CONST64(0x0000001000000000), CONST64(0x0000000000000010), CONST64(0x0000001000000010), + CONST64(0x0000100000000000), CONST64(0x0000101000000000), CONST64(0x0000100000000010), CONST64(0x0000101000000010), + CONST64(0x0000000000001000), CONST64(0x0000001000001000), CONST64(0x0000000000001010), CONST64(0x0000001000001010), + CONST64(0x0000100000001000), CONST64(0x0000101000001000), CONST64(0x0000100000001010), CONST64(0x0000101000001010), + CONST64(0x0010000000000000), CONST64(0x0010001000000000), CONST64(0x0010000000000010), CONST64(0x0010001000000010), + CONST64(0x0010100000000000), CONST64(0x0010101000000000), CONST64(0x0010100000000010), CONST64(0x0010101000000010), + CONST64(0x0010000000001000), CONST64(0x0010001000001000), CONST64(0x0010000000001010), CONST64(0x0010001000001010), + CONST64(0x0010100000001000), CONST64(0x0010101000001000), CONST64(0x0010100000001010), CONST64(0x0010101000001010), + CONST64(0x0000000000100000), CONST64(0x0000001000100000), CONST64(0x0000000000100010), CONST64(0x0000001000100010), + CONST64(0x0000100000100000), CONST64(0x0000101000100000), CONST64(0x0000100000100010), CONST64(0x0000101000100010), + CONST64(0x0000000000101000), CONST64(0x0000001000101000), CONST64(0x0000000000101010), CONST64(0x0000001000101010), + CONST64(0x0000100000101000), CONST64(0x0000101000101000), CONST64(0x0000100000101010), CONST64(0x0000101000101010), + CONST64(0x0010000000100000), CONST64(0x0010001000100000), CONST64(0x0010000000100010), CONST64(0x0010001000100010), + CONST64(0x0010100000100000), CONST64(0x0010101000100000), CONST64(0x0010100000100010), CONST64(0x0010101000100010), + CONST64(0x0010000000101000), CONST64(0x0010001000101000), CONST64(0x0010000000101010), CONST64(0x0010001000101010), + CONST64(0x0010100000101000), CONST64(0x0010101000101000), CONST64(0x0010100000101010), CONST64(0x0010101000101010), + CONST64(0x1000000000000000), CONST64(0x1000001000000000), CONST64(0x1000000000000010), CONST64(0x1000001000000010), + CONST64(0x1000100000000000), CONST64(0x1000101000000000), CONST64(0x1000100000000010), CONST64(0x1000101000000010), + CONST64(0x1000000000001000), CONST64(0x1000001000001000), CONST64(0x1000000000001010), CONST64(0x1000001000001010), + CONST64(0x1000100000001000), CONST64(0x1000101000001000), CONST64(0x1000100000001010), CONST64(0x1000101000001010), + CONST64(0x1010000000000000), CONST64(0x1010001000000000), CONST64(0x1010000000000010), CONST64(0x1010001000000010), + CONST64(0x1010100000000000), CONST64(0x1010101000000000), CONST64(0x1010100000000010), CONST64(0x1010101000000010), + CONST64(0x1010000000001000), CONST64(0x1010001000001000), CONST64(0x1010000000001010), CONST64(0x1010001000001010), + CONST64(0x1010100000001000), CONST64(0x1010101000001000), CONST64(0x1010100000001010), CONST64(0x1010101000001010), + CONST64(0x1000000000100000), CONST64(0x1000001000100000), CONST64(0x1000000000100010), CONST64(0x1000001000100010), + CONST64(0x1000100000100000), CONST64(0x1000101000100000), CONST64(0x1000100000100010), CONST64(0x1000101000100010), + CONST64(0x1000000000101000), CONST64(0x1000001000101000), CONST64(0x1000000000101010), CONST64(0x1000001000101010), + CONST64(0x1000100000101000), CONST64(0x1000101000101000), CONST64(0x1000100000101010), CONST64(0x1000101000101010), + CONST64(0x1010000000100000), CONST64(0x1010001000100000), CONST64(0x1010000000100010), CONST64(0x1010001000100010), + CONST64(0x1010100000100000), CONST64(0x1010101000100000), CONST64(0x1010100000100010), CONST64(0x1010101000100010), + CONST64(0x1010000000101000), CONST64(0x1010001000101000), CONST64(0x1010000000101010), CONST64(0x1010001000101010), + CONST64(0x1010100000101000), CONST64(0x1010101000101000), CONST64(0x1010100000101010), CONST64(0x1010101000101010), + CONST64(0x0000000010000000), CONST64(0x0000001010000000), CONST64(0x0000000010000010), CONST64(0x0000001010000010), + CONST64(0x0000100010000000), CONST64(0x0000101010000000), CONST64(0x0000100010000010), CONST64(0x0000101010000010), + CONST64(0x0000000010001000), CONST64(0x0000001010001000), CONST64(0x0000000010001010), CONST64(0x0000001010001010), + CONST64(0x0000100010001000), CONST64(0x0000101010001000), CONST64(0x0000100010001010), CONST64(0x0000101010001010), + CONST64(0x0010000010000000), CONST64(0x0010001010000000), CONST64(0x0010000010000010), CONST64(0x0010001010000010), + CONST64(0x0010100010000000), CONST64(0x0010101010000000), CONST64(0x0010100010000010), CONST64(0x0010101010000010), + CONST64(0x0010000010001000), CONST64(0x0010001010001000), CONST64(0x0010000010001010), CONST64(0x0010001010001010), + CONST64(0x0010100010001000), CONST64(0x0010101010001000), CONST64(0x0010100010001010), CONST64(0x0010101010001010), + CONST64(0x0000000010100000), CONST64(0x0000001010100000), CONST64(0x0000000010100010), CONST64(0x0000001010100010), + CONST64(0x0000100010100000), CONST64(0x0000101010100000), CONST64(0x0000100010100010), CONST64(0x0000101010100010), + CONST64(0x0000000010101000), CONST64(0x0000001010101000), CONST64(0x0000000010101010), CONST64(0x0000001010101010), + CONST64(0x0000100010101000), CONST64(0x0000101010101000), CONST64(0x0000100010101010), CONST64(0x0000101010101010), + CONST64(0x0010000010100000), CONST64(0x0010001010100000), CONST64(0x0010000010100010), CONST64(0x0010001010100010), + CONST64(0x0010100010100000), CONST64(0x0010101010100000), CONST64(0x0010100010100010), CONST64(0x0010101010100010), + CONST64(0x0010000010101000), CONST64(0x0010001010101000), CONST64(0x0010000010101010), CONST64(0x0010001010101010), + CONST64(0x0010100010101000), CONST64(0x0010101010101000), CONST64(0x0010100010101010), CONST64(0x0010101010101010), + CONST64(0x1000000010000000), CONST64(0x1000001010000000), CONST64(0x1000000010000010), CONST64(0x1000001010000010), + CONST64(0x1000100010000000), CONST64(0x1000101010000000), CONST64(0x1000100010000010), CONST64(0x1000101010000010), + CONST64(0x1000000010001000), CONST64(0x1000001010001000), CONST64(0x1000000010001010), CONST64(0x1000001010001010), + CONST64(0x1000100010001000), CONST64(0x1000101010001000), CONST64(0x1000100010001010), CONST64(0x1000101010001010), + CONST64(0x1010000010000000), CONST64(0x1010001010000000), CONST64(0x1010000010000010), CONST64(0x1010001010000010), + CONST64(0x1010100010000000), CONST64(0x1010101010000000), CONST64(0x1010100010000010), CONST64(0x1010101010000010), + CONST64(0x1010000010001000), CONST64(0x1010001010001000), CONST64(0x1010000010001010), CONST64(0x1010001010001010), + CONST64(0x1010100010001000), CONST64(0x1010101010001000), CONST64(0x1010100010001010), CONST64(0x1010101010001010), + CONST64(0x1000000010100000), CONST64(0x1000001010100000), CONST64(0x1000000010100010), CONST64(0x1000001010100010), + CONST64(0x1000100010100000), CONST64(0x1000101010100000), CONST64(0x1000100010100010), CONST64(0x1000101010100010), + CONST64(0x1000000010101000), CONST64(0x1000001010101000), CONST64(0x1000000010101010), CONST64(0x1000001010101010), + CONST64(0x1000100010101000), CONST64(0x1000101010101000), CONST64(0x1000100010101010), CONST64(0x1000101010101010), + CONST64(0x1010000010100000), CONST64(0x1010001010100000), CONST64(0x1010000010100010), CONST64(0x1010001010100010), + CONST64(0x1010100010100000), CONST64(0x1010101010100000), CONST64(0x1010100010100010), CONST64(0x1010101010100010), + CONST64(0x1010000010101000), CONST64(0x1010001010101000), CONST64(0x1010000010101010), CONST64(0x1010001010101010), + CONST64(0x1010100010101000), CONST64(0x1010101010101000), CONST64(0x1010100010101010), CONST64(0x1010101010101010) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000000800000000), CONST64(0x0000000000000008), CONST64(0x0000000800000008), + CONST64(0x0000080000000000), CONST64(0x0000080800000000), CONST64(0x0000080000000008), CONST64(0x0000080800000008), + CONST64(0x0000000000000800), CONST64(0x0000000800000800), CONST64(0x0000000000000808), CONST64(0x0000000800000808), + CONST64(0x0000080000000800), CONST64(0x0000080800000800), CONST64(0x0000080000000808), CONST64(0x0000080800000808), + CONST64(0x0008000000000000), CONST64(0x0008000800000000), CONST64(0x0008000000000008), CONST64(0x0008000800000008), + CONST64(0x0008080000000000), CONST64(0x0008080800000000), CONST64(0x0008080000000008), CONST64(0x0008080800000008), + CONST64(0x0008000000000800), CONST64(0x0008000800000800), CONST64(0x0008000000000808), CONST64(0x0008000800000808), + CONST64(0x0008080000000800), CONST64(0x0008080800000800), CONST64(0x0008080000000808), CONST64(0x0008080800000808), + CONST64(0x0000000000080000), CONST64(0x0000000800080000), CONST64(0x0000000000080008), CONST64(0x0000000800080008), + CONST64(0x0000080000080000), CONST64(0x0000080800080000), CONST64(0x0000080000080008), CONST64(0x0000080800080008), + CONST64(0x0000000000080800), CONST64(0x0000000800080800), CONST64(0x0000000000080808), CONST64(0x0000000800080808), + CONST64(0x0000080000080800), CONST64(0x0000080800080800), CONST64(0x0000080000080808), CONST64(0x0000080800080808), + CONST64(0x0008000000080000), CONST64(0x0008000800080000), CONST64(0x0008000000080008), CONST64(0x0008000800080008), + CONST64(0x0008080000080000), CONST64(0x0008080800080000), CONST64(0x0008080000080008), CONST64(0x0008080800080008), + CONST64(0x0008000000080800), CONST64(0x0008000800080800), CONST64(0x0008000000080808), CONST64(0x0008000800080808), + CONST64(0x0008080000080800), CONST64(0x0008080800080800), CONST64(0x0008080000080808), CONST64(0x0008080800080808), + CONST64(0x0800000000000000), CONST64(0x0800000800000000), CONST64(0x0800000000000008), CONST64(0x0800000800000008), + CONST64(0x0800080000000000), CONST64(0x0800080800000000), CONST64(0x0800080000000008), CONST64(0x0800080800000008), + CONST64(0x0800000000000800), CONST64(0x0800000800000800), CONST64(0x0800000000000808), CONST64(0x0800000800000808), + CONST64(0x0800080000000800), CONST64(0x0800080800000800), CONST64(0x0800080000000808), CONST64(0x0800080800000808), + CONST64(0x0808000000000000), CONST64(0x0808000800000000), CONST64(0x0808000000000008), CONST64(0x0808000800000008), + CONST64(0x0808080000000000), CONST64(0x0808080800000000), CONST64(0x0808080000000008), CONST64(0x0808080800000008), + CONST64(0x0808000000000800), CONST64(0x0808000800000800), CONST64(0x0808000000000808), CONST64(0x0808000800000808), + CONST64(0x0808080000000800), CONST64(0x0808080800000800), CONST64(0x0808080000000808), CONST64(0x0808080800000808), + CONST64(0x0800000000080000), CONST64(0x0800000800080000), CONST64(0x0800000000080008), CONST64(0x0800000800080008), + CONST64(0x0800080000080000), CONST64(0x0800080800080000), CONST64(0x0800080000080008), CONST64(0x0800080800080008), + CONST64(0x0800000000080800), CONST64(0x0800000800080800), CONST64(0x0800000000080808), CONST64(0x0800000800080808), + CONST64(0x0800080000080800), CONST64(0x0800080800080800), CONST64(0x0800080000080808), CONST64(0x0800080800080808), + CONST64(0x0808000000080000), CONST64(0x0808000800080000), CONST64(0x0808000000080008), CONST64(0x0808000800080008), + CONST64(0x0808080000080000), CONST64(0x0808080800080000), CONST64(0x0808080000080008), CONST64(0x0808080800080008), + CONST64(0x0808000000080800), CONST64(0x0808000800080800), CONST64(0x0808000000080808), CONST64(0x0808000800080808), + CONST64(0x0808080000080800), CONST64(0x0808080800080800), CONST64(0x0808080000080808), CONST64(0x0808080800080808), + CONST64(0x0000000008000000), CONST64(0x0000000808000000), CONST64(0x0000000008000008), CONST64(0x0000000808000008), + CONST64(0x0000080008000000), CONST64(0x0000080808000000), CONST64(0x0000080008000008), CONST64(0x0000080808000008), + CONST64(0x0000000008000800), CONST64(0x0000000808000800), CONST64(0x0000000008000808), CONST64(0x0000000808000808), + CONST64(0x0000080008000800), CONST64(0x0000080808000800), CONST64(0x0000080008000808), CONST64(0x0000080808000808), + CONST64(0x0008000008000000), CONST64(0x0008000808000000), CONST64(0x0008000008000008), CONST64(0x0008000808000008), + CONST64(0x0008080008000000), CONST64(0x0008080808000000), CONST64(0x0008080008000008), CONST64(0x0008080808000008), + CONST64(0x0008000008000800), CONST64(0x0008000808000800), CONST64(0x0008000008000808), CONST64(0x0008000808000808), + CONST64(0x0008080008000800), CONST64(0x0008080808000800), CONST64(0x0008080008000808), CONST64(0x0008080808000808), + CONST64(0x0000000008080000), CONST64(0x0000000808080000), CONST64(0x0000000008080008), CONST64(0x0000000808080008), + CONST64(0x0000080008080000), CONST64(0x0000080808080000), CONST64(0x0000080008080008), CONST64(0x0000080808080008), + CONST64(0x0000000008080800), CONST64(0x0000000808080800), CONST64(0x0000000008080808), CONST64(0x0000000808080808), + CONST64(0x0000080008080800), CONST64(0x0000080808080800), CONST64(0x0000080008080808), CONST64(0x0000080808080808), + CONST64(0x0008000008080000), CONST64(0x0008000808080000), CONST64(0x0008000008080008), CONST64(0x0008000808080008), + CONST64(0x0008080008080000), CONST64(0x0008080808080000), CONST64(0x0008080008080008), CONST64(0x0008080808080008), + CONST64(0x0008000008080800), CONST64(0x0008000808080800), CONST64(0x0008000008080808), CONST64(0x0008000808080808), + CONST64(0x0008080008080800), CONST64(0x0008080808080800), CONST64(0x0008080008080808), CONST64(0x0008080808080808), + CONST64(0x0800000008000000), CONST64(0x0800000808000000), CONST64(0x0800000008000008), CONST64(0x0800000808000008), + CONST64(0x0800080008000000), CONST64(0x0800080808000000), CONST64(0x0800080008000008), CONST64(0x0800080808000008), + CONST64(0x0800000008000800), CONST64(0x0800000808000800), CONST64(0x0800000008000808), CONST64(0x0800000808000808), + CONST64(0x0800080008000800), CONST64(0x0800080808000800), CONST64(0x0800080008000808), CONST64(0x0800080808000808), + CONST64(0x0808000008000000), CONST64(0x0808000808000000), CONST64(0x0808000008000008), CONST64(0x0808000808000008), + CONST64(0x0808080008000000), CONST64(0x0808080808000000), CONST64(0x0808080008000008), CONST64(0x0808080808000008), + CONST64(0x0808000008000800), CONST64(0x0808000808000800), CONST64(0x0808000008000808), CONST64(0x0808000808000808), + CONST64(0x0808080008000800), CONST64(0x0808080808000800), CONST64(0x0808080008000808), CONST64(0x0808080808000808), + CONST64(0x0800000008080000), CONST64(0x0800000808080000), CONST64(0x0800000008080008), CONST64(0x0800000808080008), + CONST64(0x0800080008080000), CONST64(0x0800080808080000), CONST64(0x0800080008080008), CONST64(0x0800080808080008), + CONST64(0x0800000008080800), CONST64(0x0800000808080800), CONST64(0x0800000008080808), CONST64(0x0800000808080808), + CONST64(0x0800080008080800), CONST64(0x0800080808080800), CONST64(0x0800080008080808), CONST64(0x0800080808080808), + CONST64(0x0808000008080000), CONST64(0x0808000808080000), CONST64(0x0808000008080008), CONST64(0x0808000808080008), + CONST64(0x0808080008080000), CONST64(0x0808080808080000), CONST64(0x0808080008080008), CONST64(0x0808080808080008), + CONST64(0x0808000008080800), CONST64(0x0808000808080800), CONST64(0x0808000008080808), CONST64(0x0808000808080808), + CONST64(0x0808080008080800), CONST64(0x0808080808080800), CONST64(0x0808080008080808), CONST64(0x0808080808080808) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000000400000000), CONST64(0x0000000000000004), CONST64(0x0000000400000004), + CONST64(0x0000040000000000), CONST64(0x0000040400000000), CONST64(0x0000040000000004), CONST64(0x0000040400000004), + CONST64(0x0000000000000400), CONST64(0x0000000400000400), CONST64(0x0000000000000404), CONST64(0x0000000400000404), + CONST64(0x0000040000000400), CONST64(0x0000040400000400), CONST64(0x0000040000000404), CONST64(0x0000040400000404), + CONST64(0x0004000000000000), CONST64(0x0004000400000000), CONST64(0x0004000000000004), CONST64(0x0004000400000004), + CONST64(0x0004040000000000), CONST64(0x0004040400000000), CONST64(0x0004040000000004), CONST64(0x0004040400000004), + CONST64(0x0004000000000400), CONST64(0x0004000400000400), CONST64(0x0004000000000404), CONST64(0x0004000400000404), + CONST64(0x0004040000000400), CONST64(0x0004040400000400), CONST64(0x0004040000000404), CONST64(0x0004040400000404), + CONST64(0x0000000000040000), CONST64(0x0000000400040000), CONST64(0x0000000000040004), CONST64(0x0000000400040004), + CONST64(0x0000040000040000), CONST64(0x0000040400040000), CONST64(0x0000040000040004), CONST64(0x0000040400040004), + CONST64(0x0000000000040400), CONST64(0x0000000400040400), CONST64(0x0000000000040404), CONST64(0x0000000400040404), + CONST64(0x0000040000040400), CONST64(0x0000040400040400), CONST64(0x0000040000040404), CONST64(0x0000040400040404), + CONST64(0x0004000000040000), CONST64(0x0004000400040000), CONST64(0x0004000000040004), CONST64(0x0004000400040004), + CONST64(0x0004040000040000), CONST64(0x0004040400040000), CONST64(0x0004040000040004), CONST64(0x0004040400040004), + CONST64(0x0004000000040400), CONST64(0x0004000400040400), CONST64(0x0004000000040404), CONST64(0x0004000400040404), + CONST64(0x0004040000040400), CONST64(0x0004040400040400), CONST64(0x0004040000040404), CONST64(0x0004040400040404), + CONST64(0x0400000000000000), CONST64(0x0400000400000000), CONST64(0x0400000000000004), CONST64(0x0400000400000004), + CONST64(0x0400040000000000), CONST64(0x0400040400000000), CONST64(0x0400040000000004), CONST64(0x0400040400000004), + CONST64(0x0400000000000400), CONST64(0x0400000400000400), CONST64(0x0400000000000404), CONST64(0x0400000400000404), + CONST64(0x0400040000000400), CONST64(0x0400040400000400), CONST64(0x0400040000000404), CONST64(0x0400040400000404), + CONST64(0x0404000000000000), CONST64(0x0404000400000000), CONST64(0x0404000000000004), CONST64(0x0404000400000004), + CONST64(0x0404040000000000), CONST64(0x0404040400000000), CONST64(0x0404040000000004), CONST64(0x0404040400000004), + CONST64(0x0404000000000400), CONST64(0x0404000400000400), CONST64(0x0404000000000404), CONST64(0x0404000400000404), + CONST64(0x0404040000000400), CONST64(0x0404040400000400), CONST64(0x0404040000000404), CONST64(0x0404040400000404), + CONST64(0x0400000000040000), CONST64(0x0400000400040000), CONST64(0x0400000000040004), CONST64(0x0400000400040004), + CONST64(0x0400040000040000), CONST64(0x0400040400040000), CONST64(0x0400040000040004), CONST64(0x0400040400040004), + CONST64(0x0400000000040400), CONST64(0x0400000400040400), CONST64(0x0400000000040404), CONST64(0x0400000400040404), + CONST64(0x0400040000040400), CONST64(0x0400040400040400), CONST64(0x0400040000040404), CONST64(0x0400040400040404), + CONST64(0x0404000000040000), CONST64(0x0404000400040000), CONST64(0x0404000000040004), CONST64(0x0404000400040004), + CONST64(0x0404040000040000), CONST64(0x0404040400040000), CONST64(0x0404040000040004), CONST64(0x0404040400040004), + CONST64(0x0404000000040400), CONST64(0x0404000400040400), CONST64(0x0404000000040404), CONST64(0x0404000400040404), + CONST64(0x0404040000040400), CONST64(0x0404040400040400), CONST64(0x0404040000040404), CONST64(0x0404040400040404), + CONST64(0x0000000004000000), CONST64(0x0000000404000000), CONST64(0x0000000004000004), CONST64(0x0000000404000004), + CONST64(0x0000040004000000), CONST64(0x0000040404000000), CONST64(0x0000040004000004), CONST64(0x0000040404000004), + CONST64(0x0000000004000400), CONST64(0x0000000404000400), CONST64(0x0000000004000404), CONST64(0x0000000404000404), + CONST64(0x0000040004000400), CONST64(0x0000040404000400), CONST64(0x0000040004000404), CONST64(0x0000040404000404), + CONST64(0x0004000004000000), CONST64(0x0004000404000000), CONST64(0x0004000004000004), CONST64(0x0004000404000004), + CONST64(0x0004040004000000), CONST64(0x0004040404000000), CONST64(0x0004040004000004), CONST64(0x0004040404000004), + CONST64(0x0004000004000400), CONST64(0x0004000404000400), CONST64(0x0004000004000404), CONST64(0x0004000404000404), + CONST64(0x0004040004000400), CONST64(0x0004040404000400), CONST64(0x0004040004000404), CONST64(0x0004040404000404), + CONST64(0x0000000004040000), CONST64(0x0000000404040000), CONST64(0x0000000004040004), CONST64(0x0000000404040004), + CONST64(0x0000040004040000), CONST64(0x0000040404040000), CONST64(0x0000040004040004), CONST64(0x0000040404040004), + CONST64(0x0000000004040400), CONST64(0x0000000404040400), CONST64(0x0000000004040404), CONST64(0x0000000404040404), + CONST64(0x0000040004040400), CONST64(0x0000040404040400), CONST64(0x0000040004040404), CONST64(0x0000040404040404), + CONST64(0x0004000004040000), CONST64(0x0004000404040000), CONST64(0x0004000004040004), CONST64(0x0004000404040004), + CONST64(0x0004040004040000), CONST64(0x0004040404040000), CONST64(0x0004040004040004), CONST64(0x0004040404040004), + CONST64(0x0004000004040400), CONST64(0x0004000404040400), CONST64(0x0004000004040404), CONST64(0x0004000404040404), + CONST64(0x0004040004040400), CONST64(0x0004040404040400), CONST64(0x0004040004040404), CONST64(0x0004040404040404), + CONST64(0x0400000004000000), CONST64(0x0400000404000000), CONST64(0x0400000004000004), CONST64(0x0400000404000004), + CONST64(0x0400040004000000), CONST64(0x0400040404000000), CONST64(0x0400040004000004), CONST64(0x0400040404000004), + CONST64(0x0400000004000400), CONST64(0x0400000404000400), CONST64(0x0400000004000404), CONST64(0x0400000404000404), + CONST64(0x0400040004000400), CONST64(0x0400040404000400), CONST64(0x0400040004000404), CONST64(0x0400040404000404), + CONST64(0x0404000004000000), CONST64(0x0404000404000000), CONST64(0x0404000004000004), CONST64(0x0404000404000004), + CONST64(0x0404040004000000), CONST64(0x0404040404000000), CONST64(0x0404040004000004), CONST64(0x0404040404000004), + CONST64(0x0404000004000400), CONST64(0x0404000404000400), CONST64(0x0404000004000404), CONST64(0x0404000404000404), + CONST64(0x0404040004000400), CONST64(0x0404040404000400), CONST64(0x0404040004000404), CONST64(0x0404040404000404), + CONST64(0x0400000004040000), CONST64(0x0400000404040000), CONST64(0x0400000004040004), CONST64(0x0400000404040004), + CONST64(0x0400040004040000), CONST64(0x0400040404040000), CONST64(0x0400040004040004), CONST64(0x0400040404040004), + CONST64(0x0400000004040400), CONST64(0x0400000404040400), CONST64(0x0400000004040404), CONST64(0x0400000404040404), + CONST64(0x0400040004040400), CONST64(0x0400040404040400), CONST64(0x0400040004040404), CONST64(0x0400040404040404), + CONST64(0x0404000004040000), CONST64(0x0404000404040000), CONST64(0x0404000004040004), CONST64(0x0404000404040004), + CONST64(0x0404040004040000), CONST64(0x0404040404040000), CONST64(0x0404040004040004), CONST64(0x0404040404040004), + CONST64(0x0404000004040400), CONST64(0x0404000404040400), CONST64(0x0404000004040404), CONST64(0x0404000404040404), + CONST64(0x0404040004040400), CONST64(0x0404040404040400), CONST64(0x0404040004040404), CONST64(0x0404040404040404) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000000200000000), CONST64(0x0000000000000002), CONST64(0x0000000200000002), + CONST64(0x0000020000000000), CONST64(0x0000020200000000), CONST64(0x0000020000000002), CONST64(0x0000020200000002), + CONST64(0x0000000000000200), CONST64(0x0000000200000200), CONST64(0x0000000000000202), CONST64(0x0000000200000202), + CONST64(0x0000020000000200), CONST64(0x0000020200000200), CONST64(0x0000020000000202), CONST64(0x0000020200000202), + CONST64(0x0002000000000000), CONST64(0x0002000200000000), CONST64(0x0002000000000002), CONST64(0x0002000200000002), + CONST64(0x0002020000000000), CONST64(0x0002020200000000), CONST64(0x0002020000000002), CONST64(0x0002020200000002), + CONST64(0x0002000000000200), CONST64(0x0002000200000200), CONST64(0x0002000000000202), CONST64(0x0002000200000202), + CONST64(0x0002020000000200), CONST64(0x0002020200000200), CONST64(0x0002020000000202), CONST64(0x0002020200000202), + CONST64(0x0000000000020000), CONST64(0x0000000200020000), CONST64(0x0000000000020002), CONST64(0x0000000200020002), + CONST64(0x0000020000020000), CONST64(0x0000020200020000), CONST64(0x0000020000020002), CONST64(0x0000020200020002), + CONST64(0x0000000000020200), CONST64(0x0000000200020200), CONST64(0x0000000000020202), CONST64(0x0000000200020202), + CONST64(0x0000020000020200), CONST64(0x0000020200020200), CONST64(0x0000020000020202), CONST64(0x0000020200020202), + CONST64(0x0002000000020000), CONST64(0x0002000200020000), CONST64(0x0002000000020002), CONST64(0x0002000200020002), + CONST64(0x0002020000020000), CONST64(0x0002020200020000), CONST64(0x0002020000020002), CONST64(0x0002020200020002), + CONST64(0x0002000000020200), CONST64(0x0002000200020200), CONST64(0x0002000000020202), CONST64(0x0002000200020202), + CONST64(0x0002020000020200), CONST64(0x0002020200020200), CONST64(0x0002020000020202), CONST64(0x0002020200020202), + CONST64(0x0200000000000000), CONST64(0x0200000200000000), CONST64(0x0200000000000002), CONST64(0x0200000200000002), + CONST64(0x0200020000000000), CONST64(0x0200020200000000), CONST64(0x0200020000000002), CONST64(0x0200020200000002), + CONST64(0x0200000000000200), CONST64(0x0200000200000200), CONST64(0x0200000000000202), CONST64(0x0200000200000202), + CONST64(0x0200020000000200), CONST64(0x0200020200000200), CONST64(0x0200020000000202), CONST64(0x0200020200000202), + CONST64(0x0202000000000000), CONST64(0x0202000200000000), CONST64(0x0202000000000002), CONST64(0x0202000200000002), + CONST64(0x0202020000000000), CONST64(0x0202020200000000), CONST64(0x0202020000000002), CONST64(0x0202020200000002), + CONST64(0x0202000000000200), CONST64(0x0202000200000200), CONST64(0x0202000000000202), CONST64(0x0202000200000202), + CONST64(0x0202020000000200), CONST64(0x0202020200000200), CONST64(0x0202020000000202), CONST64(0x0202020200000202), + CONST64(0x0200000000020000), CONST64(0x0200000200020000), CONST64(0x0200000000020002), CONST64(0x0200000200020002), + CONST64(0x0200020000020000), CONST64(0x0200020200020000), CONST64(0x0200020000020002), CONST64(0x0200020200020002), + CONST64(0x0200000000020200), CONST64(0x0200000200020200), CONST64(0x0200000000020202), CONST64(0x0200000200020202), + CONST64(0x0200020000020200), CONST64(0x0200020200020200), CONST64(0x0200020000020202), CONST64(0x0200020200020202), + CONST64(0x0202000000020000), CONST64(0x0202000200020000), CONST64(0x0202000000020002), CONST64(0x0202000200020002), + CONST64(0x0202020000020000), CONST64(0x0202020200020000), CONST64(0x0202020000020002), CONST64(0x0202020200020002), + CONST64(0x0202000000020200), CONST64(0x0202000200020200), CONST64(0x0202000000020202), CONST64(0x0202000200020202), + CONST64(0x0202020000020200), CONST64(0x0202020200020200), CONST64(0x0202020000020202), CONST64(0x0202020200020202), + CONST64(0x0000000002000000), CONST64(0x0000000202000000), CONST64(0x0000000002000002), CONST64(0x0000000202000002), + CONST64(0x0000020002000000), CONST64(0x0000020202000000), CONST64(0x0000020002000002), CONST64(0x0000020202000002), + CONST64(0x0000000002000200), CONST64(0x0000000202000200), CONST64(0x0000000002000202), CONST64(0x0000000202000202), + CONST64(0x0000020002000200), CONST64(0x0000020202000200), CONST64(0x0000020002000202), CONST64(0x0000020202000202), + CONST64(0x0002000002000000), CONST64(0x0002000202000000), CONST64(0x0002000002000002), CONST64(0x0002000202000002), + CONST64(0x0002020002000000), CONST64(0x0002020202000000), CONST64(0x0002020002000002), CONST64(0x0002020202000002), + CONST64(0x0002000002000200), CONST64(0x0002000202000200), CONST64(0x0002000002000202), CONST64(0x0002000202000202), + CONST64(0x0002020002000200), CONST64(0x0002020202000200), CONST64(0x0002020002000202), CONST64(0x0002020202000202), + CONST64(0x0000000002020000), CONST64(0x0000000202020000), CONST64(0x0000000002020002), CONST64(0x0000000202020002), + CONST64(0x0000020002020000), CONST64(0x0000020202020000), CONST64(0x0000020002020002), CONST64(0x0000020202020002), + CONST64(0x0000000002020200), CONST64(0x0000000202020200), CONST64(0x0000000002020202), CONST64(0x0000000202020202), + CONST64(0x0000020002020200), CONST64(0x0000020202020200), CONST64(0x0000020002020202), CONST64(0x0000020202020202), + CONST64(0x0002000002020000), CONST64(0x0002000202020000), CONST64(0x0002000002020002), CONST64(0x0002000202020002), + CONST64(0x0002020002020000), CONST64(0x0002020202020000), CONST64(0x0002020002020002), CONST64(0x0002020202020002), + CONST64(0x0002000002020200), CONST64(0x0002000202020200), CONST64(0x0002000002020202), CONST64(0x0002000202020202), + CONST64(0x0002020002020200), CONST64(0x0002020202020200), CONST64(0x0002020002020202), CONST64(0x0002020202020202), + CONST64(0x0200000002000000), CONST64(0x0200000202000000), CONST64(0x0200000002000002), CONST64(0x0200000202000002), + CONST64(0x0200020002000000), CONST64(0x0200020202000000), CONST64(0x0200020002000002), CONST64(0x0200020202000002), + CONST64(0x0200000002000200), CONST64(0x0200000202000200), CONST64(0x0200000002000202), CONST64(0x0200000202000202), + CONST64(0x0200020002000200), CONST64(0x0200020202000200), CONST64(0x0200020002000202), CONST64(0x0200020202000202), + CONST64(0x0202000002000000), CONST64(0x0202000202000000), CONST64(0x0202000002000002), CONST64(0x0202000202000002), + CONST64(0x0202020002000000), CONST64(0x0202020202000000), CONST64(0x0202020002000002), CONST64(0x0202020202000002), + CONST64(0x0202000002000200), CONST64(0x0202000202000200), CONST64(0x0202000002000202), CONST64(0x0202000202000202), + CONST64(0x0202020002000200), CONST64(0x0202020202000200), CONST64(0x0202020002000202), CONST64(0x0202020202000202), + CONST64(0x0200000002020000), CONST64(0x0200000202020000), CONST64(0x0200000002020002), CONST64(0x0200000202020002), + CONST64(0x0200020002020000), CONST64(0x0200020202020000), CONST64(0x0200020002020002), CONST64(0x0200020202020002), + CONST64(0x0200000002020200), CONST64(0x0200000202020200), CONST64(0x0200000002020202), CONST64(0x0200000202020202), + CONST64(0x0200020002020200), CONST64(0x0200020202020200), CONST64(0x0200020002020202), CONST64(0x0200020202020202), + CONST64(0x0202000002020000), CONST64(0x0202000202020000), CONST64(0x0202000002020002), CONST64(0x0202000202020002), + CONST64(0x0202020002020000), CONST64(0x0202020202020000), CONST64(0x0202020002020002), CONST64(0x0202020202020002), + CONST64(0x0202000002020200), CONST64(0x0202000202020200), CONST64(0x0202000002020202), CONST64(0x0202000202020202), + CONST64(0x0202020002020200), CONST64(0x0202020202020200), CONST64(0x0202020002020202), CONST64(0x0202020202020202) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000010000000000), CONST64(0x0000000000000100), CONST64(0x0000010000000100), + CONST64(0x0001000000000000), CONST64(0x0001010000000000), CONST64(0x0001000000000100), CONST64(0x0001010000000100), + CONST64(0x0000000000010000), CONST64(0x0000010000010000), CONST64(0x0000000000010100), CONST64(0x0000010000010100), + CONST64(0x0001000000010000), CONST64(0x0001010000010000), CONST64(0x0001000000010100), CONST64(0x0001010000010100), + CONST64(0x0100000000000000), CONST64(0x0100010000000000), CONST64(0x0100000000000100), CONST64(0x0100010000000100), + CONST64(0x0101000000000000), CONST64(0x0101010000000000), CONST64(0x0101000000000100), CONST64(0x0101010000000100), + CONST64(0x0100000000010000), CONST64(0x0100010000010000), CONST64(0x0100000000010100), CONST64(0x0100010000010100), + CONST64(0x0101000000010000), CONST64(0x0101010000010000), CONST64(0x0101000000010100), CONST64(0x0101010000010100), + CONST64(0x0000000001000000), CONST64(0x0000010001000000), CONST64(0x0000000001000100), CONST64(0x0000010001000100), + CONST64(0x0001000001000000), CONST64(0x0001010001000000), CONST64(0x0001000001000100), CONST64(0x0001010001000100), + CONST64(0x0000000001010000), CONST64(0x0000010001010000), CONST64(0x0000000001010100), CONST64(0x0000010001010100), + CONST64(0x0001000001010000), CONST64(0x0001010001010000), CONST64(0x0001000001010100), CONST64(0x0001010001010100), + CONST64(0x0100000001000000), CONST64(0x0100010001000000), CONST64(0x0100000001000100), CONST64(0x0100010001000100), + CONST64(0x0101000001000000), CONST64(0x0101010001000000), CONST64(0x0101000001000100), CONST64(0x0101010001000100), + CONST64(0x0100000001010000), CONST64(0x0100010001010000), CONST64(0x0100000001010100), CONST64(0x0100010001010100), + CONST64(0x0101000001010000), CONST64(0x0101010001010000), CONST64(0x0101000001010100), CONST64(0x0101010001010100), + CONST64(0x0000000100000000), CONST64(0x0000010100000000), CONST64(0x0000000100000100), CONST64(0x0000010100000100), + CONST64(0x0001000100000000), CONST64(0x0001010100000000), CONST64(0x0001000100000100), CONST64(0x0001010100000100), + CONST64(0x0000000100010000), CONST64(0x0000010100010000), CONST64(0x0000000100010100), CONST64(0x0000010100010100), + CONST64(0x0001000100010000), CONST64(0x0001010100010000), CONST64(0x0001000100010100), CONST64(0x0001010100010100), + CONST64(0x0100000100000000), CONST64(0x0100010100000000), CONST64(0x0100000100000100), CONST64(0x0100010100000100), + CONST64(0x0101000100000000), CONST64(0x0101010100000000), CONST64(0x0101000100000100), CONST64(0x0101010100000100), + CONST64(0x0100000100010000), CONST64(0x0100010100010000), CONST64(0x0100000100010100), CONST64(0x0100010100010100), + CONST64(0x0101000100010000), CONST64(0x0101010100010000), CONST64(0x0101000100010100), CONST64(0x0101010100010100), + CONST64(0x0000000101000000), CONST64(0x0000010101000000), CONST64(0x0000000101000100), CONST64(0x0000010101000100), + CONST64(0x0001000101000000), CONST64(0x0001010101000000), CONST64(0x0001000101000100), CONST64(0x0001010101000100), + CONST64(0x0000000101010000), CONST64(0x0000010101010000), CONST64(0x0000000101010100), CONST64(0x0000010101010100), + CONST64(0x0001000101010000), CONST64(0x0001010101010000), CONST64(0x0001000101010100), CONST64(0x0001010101010100), + CONST64(0x0100000101000000), CONST64(0x0100010101000000), CONST64(0x0100000101000100), CONST64(0x0100010101000100), + CONST64(0x0101000101000000), CONST64(0x0101010101000000), CONST64(0x0101000101000100), CONST64(0x0101010101000100), + CONST64(0x0100000101010000), CONST64(0x0100010101010000), CONST64(0x0100000101010100), CONST64(0x0100010101010100), + CONST64(0x0101000101010000), CONST64(0x0101010101010000), CONST64(0x0101000101010100), CONST64(0x0101010101010100), + CONST64(0x0000000000000001), CONST64(0x0000010000000001), CONST64(0x0000000000000101), CONST64(0x0000010000000101), + CONST64(0x0001000000000001), CONST64(0x0001010000000001), CONST64(0x0001000000000101), CONST64(0x0001010000000101), + CONST64(0x0000000000010001), CONST64(0x0000010000010001), CONST64(0x0000000000010101), CONST64(0x0000010000010101), + CONST64(0x0001000000010001), CONST64(0x0001010000010001), CONST64(0x0001000000010101), CONST64(0x0001010000010101), + CONST64(0x0100000000000001), CONST64(0x0100010000000001), CONST64(0x0100000000000101), CONST64(0x0100010000000101), + CONST64(0x0101000000000001), CONST64(0x0101010000000001), CONST64(0x0101000000000101), CONST64(0x0101010000000101), + CONST64(0x0100000000010001), CONST64(0x0100010000010001), CONST64(0x0100000000010101), CONST64(0x0100010000010101), + CONST64(0x0101000000010001), CONST64(0x0101010000010001), CONST64(0x0101000000010101), CONST64(0x0101010000010101), + CONST64(0x0000000001000001), CONST64(0x0000010001000001), CONST64(0x0000000001000101), CONST64(0x0000010001000101), + CONST64(0x0001000001000001), CONST64(0x0001010001000001), CONST64(0x0001000001000101), CONST64(0x0001010001000101), + CONST64(0x0000000001010001), CONST64(0x0000010001010001), CONST64(0x0000000001010101), CONST64(0x0000010001010101), + CONST64(0x0001000001010001), CONST64(0x0001010001010001), CONST64(0x0001000001010101), CONST64(0x0001010001010101), + CONST64(0x0100000001000001), CONST64(0x0100010001000001), CONST64(0x0100000001000101), CONST64(0x0100010001000101), + CONST64(0x0101000001000001), CONST64(0x0101010001000001), CONST64(0x0101000001000101), CONST64(0x0101010001000101), + CONST64(0x0100000001010001), CONST64(0x0100010001010001), CONST64(0x0100000001010101), CONST64(0x0100010001010101), + CONST64(0x0101000001010001), CONST64(0x0101010001010001), CONST64(0x0101000001010101), CONST64(0x0101010001010101), + CONST64(0x0000000100000001), CONST64(0x0000010100000001), CONST64(0x0000000100000101), CONST64(0x0000010100000101), + CONST64(0x0001000100000001), CONST64(0x0001010100000001), CONST64(0x0001000100000101), CONST64(0x0001010100000101), + CONST64(0x0000000100010001), CONST64(0x0000010100010001), CONST64(0x0000000100010101), CONST64(0x0000010100010101), + CONST64(0x0001000100010001), CONST64(0x0001010100010001), CONST64(0x0001000100010101), CONST64(0x0001010100010101), + CONST64(0x0100000100000001), CONST64(0x0100010100000001), CONST64(0x0100000100000101), CONST64(0x0100010100000101), + CONST64(0x0101000100000001), CONST64(0x0101010100000001), CONST64(0x0101000100000101), CONST64(0x0101010100000101), + CONST64(0x0100000100010001), CONST64(0x0100010100010001), CONST64(0x0100000100010101), CONST64(0x0100010100010101), + CONST64(0x0101000100010001), CONST64(0x0101010100010001), CONST64(0x0101000100010101), CONST64(0x0101010100010101), + CONST64(0x0000000101000001), CONST64(0x0000010101000001), CONST64(0x0000000101000101), CONST64(0x0000010101000101), + CONST64(0x0001000101000001), CONST64(0x0001010101000001), CONST64(0x0001000101000101), CONST64(0x0001010101000101), + CONST64(0x0000000101010001), CONST64(0x0000010101010001), CONST64(0x0000000101010101), CONST64(0x0000010101010101), + CONST64(0x0001000101010001), CONST64(0x0001010101010001), CONST64(0x0001000101010101), CONST64(0x0001010101010101), + CONST64(0x0100000101000001), CONST64(0x0100010101000001), CONST64(0x0100000101000101), CONST64(0x0100010101000101), + CONST64(0x0101000101000001), CONST64(0x0101010101000001), CONST64(0x0101000101000101), CONST64(0x0101010101000101), + CONST64(0x0100000101010001), CONST64(0x0100010101010001), CONST64(0x0100000101010101), CONST64(0x0100010101010101), + CONST64(0x0101000101010001), CONST64(0x0101010101010001), CONST64(0x0101000101010101), CONST64(0x0101010101010101) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000008000000000), CONST64(0x0000000000000080), CONST64(0x0000008000000080), + CONST64(0x0000800000000000), CONST64(0x0000808000000000), CONST64(0x0000800000000080), CONST64(0x0000808000000080), + CONST64(0x0000000000008000), CONST64(0x0000008000008000), CONST64(0x0000000000008080), CONST64(0x0000008000008080), + CONST64(0x0000800000008000), CONST64(0x0000808000008000), CONST64(0x0000800000008080), CONST64(0x0000808000008080), + CONST64(0x0080000000000000), CONST64(0x0080008000000000), CONST64(0x0080000000000080), CONST64(0x0080008000000080), + CONST64(0x0080800000000000), CONST64(0x0080808000000000), CONST64(0x0080800000000080), CONST64(0x0080808000000080), + CONST64(0x0080000000008000), CONST64(0x0080008000008000), CONST64(0x0080000000008080), CONST64(0x0080008000008080), + CONST64(0x0080800000008000), CONST64(0x0080808000008000), CONST64(0x0080800000008080), CONST64(0x0080808000008080), + CONST64(0x0000000000800000), CONST64(0x0000008000800000), CONST64(0x0000000000800080), CONST64(0x0000008000800080), + CONST64(0x0000800000800000), CONST64(0x0000808000800000), CONST64(0x0000800000800080), CONST64(0x0000808000800080), + CONST64(0x0000000000808000), CONST64(0x0000008000808000), CONST64(0x0000000000808080), CONST64(0x0000008000808080), + CONST64(0x0000800000808000), CONST64(0x0000808000808000), CONST64(0x0000800000808080), CONST64(0x0000808000808080), + CONST64(0x0080000000800000), CONST64(0x0080008000800000), CONST64(0x0080000000800080), CONST64(0x0080008000800080), + CONST64(0x0080800000800000), CONST64(0x0080808000800000), CONST64(0x0080800000800080), CONST64(0x0080808000800080), + CONST64(0x0080000000808000), CONST64(0x0080008000808000), CONST64(0x0080000000808080), CONST64(0x0080008000808080), + CONST64(0x0080800000808000), CONST64(0x0080808000808000), CONST64(0x0080800000808080), CONST64(0x0080808000808080), + CONST64(0x8000000000000000), CONST64(0x8000008000000000), CONST64(0x8000000000000080), CONST64(0x8000008000000080), + CONST64(0x8000800000000000), CONST64(0x8000808000000000), CONST64(0x8000800000000080), CONST64(0x8000808000000080), + CONST64(0x8000000000008000), CONST64(0x8000008000008000), CONST64(0x8000000000008080), CONST64(0x8000008000008080), + CONST64(0x8000800000008000), CONST64(0x8000808000008000), CONST64(0x8000800000008080), CONST64(0x8000808000008080), + CONST64(0x8080000000000000), CONST64(0x8080008000000000), CONST64(0x8080000000000080), CONST64(0x8080008000000080), + CONST64(0x8080800000000000), CONST64(0x8080808000000000), CONST64(0x8080800000000080), CONST64(0x8080808000000080), + CONST64(0x8080000000008000), CONST64(0x8080008000008000), CONST64(0x8080000000008080), CONST64(0x8080008000008080), + CONST64(0x8080800000008000), CONST64(0x8080808000008000), CONST64(0x8080800000008080), CONST64(0x8080808000008080), + CONST64(0x8000000000800000), CONST64(0x8000008000800000), CONST64(0x8000000000800080), CONST64(0x8000008000800080), + CONST64(0x8000800000800000), CONST64(0x8000808000800000), CONST64(0x8000800000800080), CONST64(0x8000808000800080), + CONST64(0x8000000000808000), CONST64(0x8000008000808000), CONST64(0x8000000000808080), CONST64(0x8000008000808080), + CONST64(0x8000800000808000), CONST64(0x8000808000808000), CONST64(0x8000800000808080), CONST64(0x8000808000808080), + CONST64(0x8080000000800000), CONST64(0x8080008000800000), CONST64(0x8080000000800080), CONST64(0x8080008000800080), + CONST64(0x8080800000800000), CONST64(0x8080808000800000), CONST64(0x8080800000800080), CONST64(0x8080808000800080), + CONST64(0x8080000000808000), CONST64(0x8080008000808000), CONST64(0x8080000000808080), CONST64(0x8080008000808080), + CONST64(0x8080800000808000), CONST64(0x8080808000808000), CONST64(0x8080800000808080), CONST64(0x8080808000808080), + CONST64(0x0000000080000000), CONST64(0x0000008080000000), CONST64(0x0000000080000080), CONST64(0x0000008080000080), + CONST64(0x0000800080000000), CONST64(0x0000808080000000), CONST64(0x0000800080000080), CONST64(0x0000808080000080), + CONST64(0x0000000080008000), CONST64(0x0000008080008000), CONST64(0x0000000080008080), CONST64(0x0000008080008080), + CONST64(0x0000800080008000), CONST64(0x0000808080008000), CONST64(0x0000800080008080), CONST64(0x0000808080008080), + CONST64(0x0080000080000000), CONST64(0x0080008080000000), CONST64(0x0080000080000080), CONST64(0x0080008080000080), + CONST64(0x0080800080000000), CONST64(0x0080808080000000), CONST64(0x0080800080000080), CONST64(0x0080808080000080), + CONST64(0x0080000080008000), CONST64(0x0080008080008000), CONST64(0x0080000080008080), CONST64(0x0080008080008080), + CONST64(0x0080800080008000), CONST64(0x0080808080008000), CONST64(0x0080800080008080), CONST64(0x0080808080008080), + CONST64(0x0000000080800000), CONST64(0x0000008080800000), CONST64(0x0000000080800080), CONST64(0x0000008080800080), + CONST64(0x0000800080800000), CONST64(0x0000808080800000), CONST64(0x0000800080800080), CONST64(0x0000808080800080), + CONST64(0x0000000080808000), CONST64(0x0000008080808000), CONST64(0x0000000080808080), CONST64(0x0000008080808080), + CONST64(0x0000800080808000), CONST64(0x0000808080808000), CONST64(0x0000800080808080), CONST64(0x0000808080808080), + CONST64(0x0080000080800000), CONST64(0x0080008080800000), CONST64(0x0080000080800080), CONST64(0x0080008080800080), + CONST64(0x0080800080800000), CONST64(0x0080808080800000), CONST64(0x0080800080800080), CONST64(0x0080808080800080), + CONST64(0x0080000080808000), CONST64(0x0080008080808000), CONST64(0x0080000080808080), CONST64(0x0080008080808080), + CONST64(0x0080800080808000), CONST64(0x0080808080808000), CONST64(0x0080800080808080), CONST64(0x0080808080808080), + CONST64(0x8000000080000000), CONST64(0x8000008080000000), CONST64(0x8000000080000080), CONST64(0x8000008080000080), + CONST64(0x8000800080000000), CONST64(0x8000808080000000), CONST64(0x8000800080000080), CONST64(0x8000808080000080), + CONST64(0x8000000080008000), CONST64(0x8000008080008000), CONST64(0x8000000080008080), CONST64(0x8000008080008080), + CONST64(0x8000800080008000), CONST64(0x8000808080008000), CONST64(0x8000800080008080), CONST64(0x8000808080008080), + CONST64(0x8080000080000000), CONST64(0x8080008080000000), CONST64(0x8080000080000080), CONST64(0x8080008080000080), + CONST64(0x8080800080000000), CONST64(0x8080808080000000), CONST64(0x8080800080000080), CONST64(0x8080808080000080), + CONST64(0x8080000080008000), CONST64(0x8080008080008000), CONST64(0x8080000080008080), CONST64(0x8080008080008080), + CONST64(0x8080800080008000), CONST64(0x8080808080008000), CONST64(0x8080800080008080), CONST64(0x8080808080008080), + CONST64(0x8000000080800000), CONST64(0x8000008080800000), CONST64(0x8000000080800080), CONST64(0x8000008080800080), + CONST64(0x8000800080800000), CONST64(0x8000808080800000), CONST64(0x8000800080800080), CONST64(0x8000808080800080), + CONST64(0x8000000080808000), CONST64(0x8000008080808000), CONST64(0x8000000080808080), CONST64(0x8000008080808080), + CONST64(0x8000800080808000), CONST64(0x8000808080808000), CONST64(0x8000800080808080), CONST64(0x8000808080808080), + CONST64(0x8080000080800000), CONST64(0x8080008080800000), CONST64(0x8080000080800080), CONST64(0x8080008080800080), + CONST64(0x8080800080800000), CONST64(0x8080808080800000), CONST64(0x8080800080800080), CONST64(0x8080808080800080), + CONST64(0x8080000080808000), CONST64(0x8080008080808000), CONST64(0x8080000080808080), CONST64(0x8080008080808080), + CONST64(0x8080800080808000), CONST64(0x8080808080808000), CONST64(0x8080800080808080), CONST64(0x8080808080808080) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000004000000000), CONST64(0x0000000000000040), CONST64(0x0000004000000040), + CONST64(0x0000400000000000), CONST64(0x0000404000000000), CONST64(0x0000400000000040), CONST64(0x0000404000000040), + CONST64(0x0000000000004000), CONST64(0x0000004000004000), CONST64(0x0000000000004040), CONST64(0x0000004000004040), + CONST64(0x0000400000004000), CONST64(0x0000404000004000), CONST64(0x0000400000004040), CONST64(0x0000404000004040), + CONST64(0x0040000000000000), CONST64(0x0040004000000000), CONST64(0x0040000000000040), CONST64(0x0040004000000040), + CONST64(0x0040400000000000), CONST64(0x0040404000000000), CONST64(0x0040400000000040), CONST64(0x0040404000000040), + CONST64(0x0040000000004000), CONST64(0x0040004000004000), CONST64(0x0040000000004040), CONST64(0x0040004000004040), + CONST64(0x0040400000004000), CONST64(0x0040404000004000), CONST64(0x0040400000004040), CONST64(0x0040404000004040), + CONST64(0x0000000000400000), CONST64(0x0000004000400000), CONST64(0x0000000000400040), CONST64(0x0000004000400040), + CONST64(0x0000400000400000), CONST64(0x0000404000400000), CONST64(0x0000400000400040), CONST64(0x0000404000400040), + CONST64(0x0000000000404000), CONST64(0x0000004000404000), CONST64(0x0000000000404040), CONST64(0x0000004000404040), + CONST64(0x0000400000404000), CONST64(0x0000404000404000), CONST64(0x0000400000404040), CONST64(0x0000404000404040), + CONST64(0x0040000000400000), CONST64(0x0040004000400000), CONST64(0x0040000000400040), CONST64(0x0040004000400040), + CONST64(0x0040400000400000), CONST64(0x0040404000400000), CONST64(0x0040400000400040), CONST64(0x0040404000400040), + CONST64(0x0040000000404000), CONST64(0x0040004000404000), CONST64(0x0040000000404040), CONST64(0x0040004000404040), + CONST64(0x0040400000404000), CONST64(0x0040404000404000), CONST64(0x0040400000404040), CONST64(0x0040404000404040), + CONST64(0x4000000000000000), CONST64(0x4000004000000000), CONST64(0x4000000000000040), CONST64(0x4000004000000040), + CONST64(0x4000400000000000), CONST64(0x4000404000000000), CONST64(0x4000400000000040), CONST64(0x4000404000000040), + CONST64(0x4000000000004000), CONST64(0x4000004000004000), CONST64(0x4000000000004040), CONST64(0x4000004000004040), + CONST64(0x4000400000004000), CONST64(0x4000404000004000), CONST64(0x4000400000004040), CONST64(0x4000404000004040), + CONST64(0x4040000000000000), CONST64(0x4040004000000000), CONST64(0x4040000000000040), CONST64(0x4040004000000040), + CONST64(0x4040400000000000), CONST64(0x4040404000000000), CONST64(0x4040400000000040), CONST64(0x4040404000000040), + CONST64(0x4040000000004000), CONST64(0x4040004000004000), CONST64(0x4040000000004040), CONST64(0x4040004000004040), + CONST64(0x4040400000004000), CONST64(0x4040404000004000), CONST64(0x4040400000004040), CONST64(0x4040404000004040), + CONST64(0x4000000000400000), CONST64(0x4000004000400000), CONST64(0x4000000000400040), CONST64(0x4000004000400040), + CONST64(0x4000400000400000), CONST64(0x4000404000400000), CONST64(0x4000400000400040), CONST64(0x4000404000400040), + CONST64(0x4000000000404000), CONST64(0x4000004000404000), CONST64(0x4000000000404040), CONST64(0x4000004000404040), + CONST64(0x4000400000404000), CONST64(0x4000404000404000), CONST64(0x4000400000404040), CONST64(0x4000404000404040), + CONST64(0x4040000000400000), CONST64(0x4040004000400000), CONST64(0x4040000000400040), CONST64(0x4040004000400040), + CONST64(0x4040400000400000), CONST64(0x4040404000400000), CONST64(0x4040400000400040), CONST64(0x4040404000400040), + CONST64(0x4040000000404000), CONST64(0x4040004000404000), CONST64(0x4040000000404040), CONST64(0x4040004000404040), + CONST64(0x4040400000404000), CONST64(0x4040404000404000), CONST64(0x4040400000404040), CONST64(0x4040404000404040), + CONST64(0x0000000040000000), CONST64(0x0000004040000000), CONST64(0x0000000040000040), CONST64(0x0000004040000040), + CONST64(0x0000400040000000), CONST64(0x0000404040000000), CONST64(0x0000400040000040), CONST64(0x0000404040000040), + CONST64(0x0000000040004000), CONST64(0x0000004040004000), CONST64(0x0000000040004040), CONST64(0x0000004040004040), + CONST64(0x0000400040004000), CONST64(0x0000404040004000), CONST64(0x0000400040004040), CONST64(0x0000404040004040), + CONST64(0x0040000040000000), CONST64(0x0040004040000000), CONST64(0x0040000040000040), CONST64(0x0040004040000040), + CONST64(0x0040400040000000), CONST64(0x0040404040000000), CONST64(0x0040400040000040), CONST64(0x0040404040000040), + CONST64(0x0040000040004000), CONST64(0x0040004040004000), CONST64(0x0040000040004040), CONST64(0x0040004040004040), + CONST64(0x0040400040004000), CONST64(0x0040404040004000), CONST64(0x0040400040004040), CONST64(0x0040404040004040), + CONST64(0x0000000040400000), CONST64(0x0000004040400000), CONST64(0x0000000040400040), CONST64(0x0000004040400040), + CONST64(0x0000400040400000), CONST64(0x0000404040400000), CONST64(0x0000400040400040), CONST64(0x0000404040400040), + CONST64(0x0000000040404000), CONST64(0x0000004040404000), CONST64(0x0000000040404040), CONST64(0x0000004040404040), + CONST64(0x0000400040404000), CONST64(0x0000404040404000), CONST64(0x0000400040404040), CONST64(0x0000404040404040), + CONST64(0x0040000040400000), CONST64(0x0040004040400000), CONST64(0x0040000040400040), CONST64(0x0040004040400040), + CONST64(0x0040400040400000), CONST64(0x0040404040400000), CONST64(0x0040400040400040), CONST64(0x0040404040400040), + CONST64(0x0040000040404000), CONST64(0x0040004040404000), CONST64(0x0040000040404040), CONST64(0x0040004040404040), + CONST64(0x0040400040404000), CONST64(0x0040404040404000), CONST64(0x0040400040404040), CONST64(0x0040404040404040), + CONST64(0x4000000040000000), CONST64(0x4000004040000000), CONST64(0x4000000040000040), CONST64(0x4000004040000040), + CONST64(0x4000400040000000), CONST64(0x4000404040000000), CONST64(0x4000400040000040), CONST64(0x4000404040000040), + CONST64(0x4000000040004000), CONST64(0x4000004040004000), CONST64(0x4000000040004040), CONST64(0x4000004040004040), + CONST64(0x4000400040004000), CONST64(0x4000404040004000), CONST64(0x4000400040004040), CONST64(0x4000404040004040), + CONST64(0x4040000040000000), CONST64(0x4040004040000000), CONST64(0x4040000040000040), CONST64(0x4040004040000040), + CONST64(0x4040400040000000), CONST64(0x4040404040000000), CONST64(0x4040400040000040), CONST64(0x4040404040000040), + CONST64(0x4040000040004000), CONST64(0x4040004040004000), CONST64(0x4040000040004040), CONST64(0x4040004040004040), + CONST64(0x4040400040004000), CONST64(0x4040404040004000), CONST64(0x4040400040004040), CONST64(0x4040404040004040), + CONST64(0x4000000040400000), CONST64(0x4000004040400000), CONST64(0x4000000040400040), CONST64(0x4000004040400040), + CONST64(0x4000400040400000), CONST64(0x4000404040400000), CONST64(0x4000400040400040), CONST64(0x4000404040400040), + CONST64(0x4000000040404000), CONST64(0x4000004040404000), CONST64(0x4000000040404040), CONST64(0x4000004040404040), + CONST64(0x4000400040404000), CONST64(0x4000404040404000), CONST64(0x4000400040404040), CONST64(0x4000404040404040), + CONST64(0x4040000040400000), CONST64(0x4040004040400000), CONST64(0x4040000040400040), CONST64(0x4040004040400040), + CONST64(0x4040400040400000), CONST64(0x4040404040400000), CONST64(0x4040400040400040), CONST64(0x4040404040400040), + CONST64(0x4040000040404000), CONST64(0x4040004040404000), CONST64(0x4040000040404040), CONST64(0x4040004040404040), + CONST64(0x4040400040404000), CONST64(0x4040404040404000), CONST64(0x4040400040404040), CONST64(0x4040404040404040) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000002000000000), CONST64(0x0000000000000020), CONST64(0x0000002000000020), + CONST64(0x0000200000000000), CONST64(0x0000202000000000), CONST64(0x0000200000000020), CONST64(0x0000202000000020), + CONST64(0x0000000000002000), CONST64(0x0000002000002000), CONST64(0x0000000000002020), CONST64(0x0000002000002020), + CONST64(0x0000200000002000), CONST64(0x0000202000002000), CONST64(0x0000200000002020), CONST64(0x0000202000002020), + CONST64(0x0020000000000000), CONST64(0x0020002000000000), CONST64(0x0020000000000020), CONST64(0x0020002000000020), + CONST64(0x0020200000000000), CONST64(0x0020202000000000), CONST64(0x0020200000000020), CONST64(0x0020202000000020), + CONST64(0x0020000000002000), CONST64(0x0020002000002000), CONST64(0x0020000000002020), CONST64(0x0020002000002020), + CONST64(0x0020200000002000), CONST64(0x0020202000002000), CONST64(0x0020200000002020), CONST64(0x0020202000002020), + CONST64(0x0000000000200000), CONST64(0x0000002000200000), CONST64(0x0000000000200020), CONST64(0x0000002000200020), + CONST64(0x0000200000200000), CONST64(0x0000202000200000), CONST64(0x0000200000200020), CONST64(0x0000202000200020), + CONST64(0x0000000000202000), CONST64(0x0000002000202000), CONST64(0x0000000000202020), CONST64(0x0000002000202020), + CONST64(0x0000200000202000), CONST64(0x0000202000202000), CONST64(0x0000200000202020), CONST64(0x0000202000202020), + CONST64(0x0020000000200000), CONST64(0x0020002000200000), CONST64(0x0020000000200020), CONST64(0x0020002000200020), + CONST64(0x0020200000200000), CONST64(0x0020202000200000), CONST64(0x0020200000200020), CONST64(0x0020202000200020), + CONST64(0x0020000000202000), CONST64(0x0020002000202000), CONST64(0x0020000000202020), CONST64(0x0020002000202020), + CONST64(0x0020200000202000), CONST64(0x0020202000202000), CONST64(0x0020200000202020), CONST64(0x0020202000202020), + CONST64(0x2000000000000000), CONST64(0x2000002000000000), CONST64(0x2000000000000020), CONST64(0x2000002000000020), + CONST64(0x2000200000000000), CONST64(0x2000202000000000), CONST64(0x2000200000000020), CONST64(0x2000202000000020), + CONST64(0x2000000000002000), CONST64(0x2000002000002000), CONST64(0x2000000000002020), CONST64(0x2000002000002020), + CONST64(0x2000200000002000), CONST64(0x2000202000002000), CONST64(0x2000200000002020), CONST64(0x2000202000002020), + CONST64(0x2020000000000000), CONST64(0x2020002000000000), CONST64(0x2020000000000020), CONST64(0x2020002000000020), + CONST64(0x2020200000000000), CONST64(0x2020202000000000), CONST64(0x2020200000000020), CONST64(0x2020202000000020), + CONST64(0x2020000000002000), CONST64(0x2020002000002000), CONST64(0x2020000000002020), CONST64(0x2020002000002020), + CONST64(0x2020200000002000), CONST64(0x2020202000002000), CONST64(0x2020200000002020), CONST64(0x2020202000002020), + CONST64(0x2000000000200000), CONST64(0x2000002000200000), CONST64(0x2000000000200020), CONST64(0x2000002000200020), + CONST64(0x2000200000200000), CONST64(0x2000202000200000), CONST64(0x2000200000200020), CONST64(0x2000202000200020), + CONST64(0x2000000000202000), CONST64(0x2000002000202000), CONST64(0x2000000000202020), CONST64(0x2000002000202020), + CONST64(0x2000200000202000), CONST64(0x2000202000202000), CONST64(0x2000200000202020), CONST64(0x2000202000202020), + CONST64(0x2020000000200000), CONST64(0x2020002000200000), CONST64(0x2020000000200020), CONST64(0x2020002000200020), + CONST64(0x2020200000200000), CONST64(0x2020202000200000), CONST64(0x2020200000200020), CONST64(0x2020202000200020), + CONST64(0x2020000000202000), CONST64(0x2020002000202000), CONST64(0x2020000000202020), CONST64(0x2020002000202020), + CONST64(0x2020200000202000), CONST64(0x2020202000202000), CONST64(0x2020200000202020), CONST64(0x2020202000202020), + CONST64(0x0000000020000000), CONST64(0x0000002020000000), CONST64(0x0000000020000020), CONST64(0x0000002020000020), + CONST64(0x0000200020000000), CONST64(0x0000202020000000), CONST64(0x0000200020000020), CONST64(0x0000202020000020), + CONST64(0x0000000020002000), CONST64(0x0000002020002000), CONST64(0x0000000020002020), CONST64(0x0000002020002020), + CONST64(0x0000200020002000), CONST64(0x0000202020002000), CONST64(0x0000200020002020), CONST64(0x0000202020002020), + CONST64(0x0020000020000000), CONST64(0x0020002020000000), CONST64(0x0020000020000020), CONST64(0x0020002020000020), + CONST64(0x0020200020000000), CONST64(0x0020202020000000), CONST64(0x0020200020000020), CONST64(0x0020202020000020), + CONST64(0x0020000020002000), CONST64(0x0020002020002000), CONST64(0x0020000020002020), CONST64(0x0020002020002020), + CONST64(0x0020200020002000), CONST64(0x0020202020002000), CONST64(0x0020200020002020), CONST64(0x0020202020002020), + CONST64(0x0000000020200000), CONST64(0x0000002020200000), CONST64(0x0000000020200020), CONST64(0x0000002020200020), + CONST64(0x0000200020200000), CONST64(0x0000202020200000), CONST64(0x0000200020200020), CONST64(0x0000202020200020), + CONST64(0x0000000020202000), CONST64(0x0000002020202000), CONST64(0x0000000020202020), CONST64(0x0000002020202020), + CONST64(0x0000200020202000), CONST64(0x0000202020202000), CONST64(0x0000200020202020), CONST64(0x0000202020202020), + CONST64(0x0020000020200000), CONST64(0x0020002020200000), CONST64(0x0020000020200020), CONST64(0x0020002020200020), + CONST64(0x0020200020200000), CONST64(0x0020202020200000), CONST64(0x0020200020200020), CONST64(0x0020202020200020), + CONST64(0x0020000020202000), CONST64(0x0020002020202000), CONST64(0x0020000020202020), CONST64(0x0020002020202020), + CONST64(0x0020200020202000), CONST64(0x0020202020202000), CONST64(0x0020200020202020), CONST64(0x0020202020202020), + CONST64(0x2000000020000000), CONST64(0x2000002020000000), CONST64(0x2000000020000020), CONST64(0x2000002020000020), + CONST64(0x2000200020000000), CONST64(0x2000202020000000), CONST64(0x2000200020000020), CONST64(0x2000202020000020), + CONST64(0x2000000020002000), CONST64(0x2000002020002000), CONST64(0x2000000020002020), CONST64(0x2000002020002020), + CONST64(0x2000200020002000), CONST64(0x2000202020002000), CONST64(0x2000200020002020), CONST64(0x2000202020002020), + CONST64(0x2020000020000000), CONST64(0x2020002020000000), CONST64(0x2020000020000020), CONST64(0x2020002020000020), + CONST64(0x2020200020000000), CONST64(0x2020202020000000), CONST64(0x2020200020000020), CONST64(0x2020202020000020), + CONST64(0x2020000020002000), CONST64(0x2020002020002000), CONST64(0x2020000020002020), CONST64(0x2020002020002020), + CONST64(0x2020200020002000), CONST64(0x2020202020002000), CONST64(0x2020200020002020), CONST64(0x2020202020002020), + CONST64(0x2000000020200000), CONST64(0x2000002020200000), CONST64(0x2000000020200020), CONST64(0x2000002020200020), + CONST64(0x2000200020200000), CONST64(0x2000202020200000), CONST64(0x2000200020200020), CONST64(0x2000202020200020), + CONST64(0x2000000020202000), CONST64(0x2000002020202000), CONST64(0x2000000020202020), CONST64(0x2000002020202020), + CONST64(0x2000200020202000), CONST64(0x2000202020202000), CONST64(0x2000200020202020), CONST64(0x2000202020202020), + CONST64(0x2020000020200000), CONST64(0x2020002020200000), CONST64(0x2020000020200020), CONST64(0x2020002020200020), + CONST64(0x2020200020200000), CONST64(0x2020202020200000), CONST64(0x2020200020200020), CONST64(0x2020202020200020), + CONST64(0x2020000020202000), CONST64(0x2020002020202000), CONST64(0x2020000020202020), CONST64(0x2020002020202020), + CONST64(0x2020200020202000), CONST64(0x2020202020202000), CONST64(0x2020200020202020), CONST64(0x2020202020202020) + }}; + +static const ulong64 des_fp[8][256] = { + +{ CONST64(0x0000000000000000), CONST64(0x0000008000000000), CONST64(0x0000000002000000), CONST64(0x0000008002000000), + CONST64(0x0000000000020000), CONST64(0x0000008000020000), CONST64(0x0000000002020000), CONST64(0x0000008002020000), + CONST64(0x0000000000000200), CONST64(0x0000008000000200), CONST64(0x0000000002000200), CONST64(0x0000008002000200), + CONST64(0x0000000000020200), CONST64(0x0000008000020200), CONST64(0x0000000002020200), CONST64(0x0000008002020200), + CONST64(0x0000000000000002), CONST64(0x0000008000000002), CONST64(0x0000000002000002), CONST64(0x0000008002000002), + CONST64(0x0000000000020002), CONST64(0x0000008000020002), CONST64(0x0000000002020002), CONST64(0x0000008002020002), + CONST64(0x0000000000000202), CONST64(0x0000008000000202), CONST64(0x0000000002000202), CONST64(0x0000008002000202), + CONST64(0x0000000000020202), CONST64(0x0000008000020202), CONST64(0x0000000002020202), CONST64(0x0000008002020202), + CONST64(0x0200000000000000), CONST64(0x0200008000000000), CONST64(0x0200000002000000), CONST64(0x0200008002000000), + CONST64(0x0200000000020000), CONST64(0x0200008000020000), CONST64(0x0200000002020000), CONST64(0x0200008002020000), + CONST64(0x0200000000000200), CONST64(0x0200008000000200), CONST64(0x0200000002000200), CONST64(0x0200008002000200), + CONST64(0x0200000000020200), CONST64(0x0200008000020200), CONST64(0x0200000002020200), CONST64(0x0200008002020200), + CONST64(0x0200000000000002), CONST64(0x0200008000000002), CONST64(0x0200000002000002), CONST64(0x0200008002000002), + CONST64(0x0200000000020002), CONST64(0x0200008000020002), CONST64(0x0200000002020002), CONST64(0x0200008002020002), + CONST64(0x0200000000000202), CONST64(0x0200008000000202), CONST64(0x0200000002000202), CONST64(0x0200008002000202), + CONST64(0x0200000000020202), CONST64(0x0200008000020202), CONST64(0x0200000002020202), CONST64(0x0200008002020202), + CONST64(0x0002000000000000), CONST64(0x0002008000000000), CONST64(0x0002000002000000), CONST64(0x0002008002000000), + CONST64(0x0002000000020000), CONST64(0x0002008000020000), CONST64(0x0002000002020000), CONST64(0x0002008002020000), + CONST64(0x0002000000000200), CONST64(0x0002008000000200), CONST64(0x0002000002000200), CONST64(0x0002008002000200), + CONST64(0x0002000000020200), CONST64(0x0002008000020200), CONST64(0x0002000002020200), CONST64(0x0002008002020200), + CONST64(0x0002000000000002), CONST64(0x0002008000000002), CONST64(0x0002000002000002), CONST64(0x0002008002000002), + CONST64(0x0002000000020002), CONST64(0x0002008000020002), CONST64(0x0002000002020002), CONST64(0x0002008002020002), + CONST64(0x0002000000000202), CONST64(0x0002008000000202), CONST64(0x0002000002000202), CONST64(0x0002008002000202), + CONST64(0x0002000000020202), CONST64(0x0002008000020202), CONST64(0x0002000002020202), CONST64(0x0002008002020202), + CONST64(0x0202000000000000), CONST64(0x0202008000000000), CONST64(0x0202000002000000), CONST64(0x0202008002000000), + CONST64(0x0202000000020000), CONST64(0x0202008000020000), CONST64(0x0202000002020000), CONST64(0x0202008002020000), + CONST64(0x0202000000000200), CONST64(0x0202008000000200), CONST64(0x0202000002000200), CONST64(0x0202008002000200), + CONST64(0x0202000000020200), CONST64(0x0202008000020200), CONST64(0x0202000002020200), CONST64(0x0202008002020200), + CONST64(0x0202000000000002), CONST64(0x0202008000000002), CONST64(0x0202000002000002), CONST64(0x0202008002000002), + CONST64(0x0202000000020002), CONST64(0x0202008000020002), CONST64(0x0202000002020002), CONST64(0x0202008002020002), + CONST64(0x0202000000000202), CONST64(0x0202008000000202), CONST64(0x0202000002000202), CONST64(0x0202008002000202), + CONST64(0x0202000000020202), CONST64(0x0202008000020202), CONST64(0x0202000002020202), CONST64(0x0202008002020202), + CONST64(0x0000020000000000), CONST64(0x0000028000000000), CONST64(0x0000020002000000), CONST64(0x0000028002000000), + CONST64(0x0000020000020000), CONST64(0x0000028000020000), CONST64(0x0000020002020000), CONST64(0x0000028002020000), + CONST64(0x0000020000000200), CONST64(0x0000028000000200), CONST64(0x0000020002000200), CONST64(0x0000028002000200), + CONST64(0x0000020000020200), CONST64(0x0000028000020200), CONST64(0x0000020002020200), CONST64(0x0000028002020200), + CONST64(0x0000020000000002), CONST64(0x0000028000000002), CONST64(0x0000020002000002), CONST64(0x0000028002000002), + CONST64(0x0000020000020002), CONST64(0x0000028000020002), CONST64(0x0000020002020002), CONST64(0x0000028002020002), + CONST64(0x0000020000000202), CONST64(0x0000028000000202), CONST64(0x0000020002000202), CONST64(0x0000028002000202), + CONST64(0x0000020000020202), CONST64(0x0000028000020202), CONST64(0x0000020002020202), CONST64(0x0000028002020202), + CONST64(0x0200020000000000), CONST64(0x0200028000000000), CONST64(0x0200020002000000), CONST64(0x0200028002000000), + CONST64(0x0200020000020000), CONST64(0x0200028000020000), CONST64(0x0200020002020000), CONST64(0x0200028002020000), + CONST64(0x0200020000000200), CONST64(0x0200028000000200), CONST64(0x0200020002000200), CONST64(0x0200028002000200), + CONST64(0x0200020000020200), CONST64(0x0200028000020200), CONST64(0x0200020002020200), CONST64(0x0200028002020200), + CONST64(0x0200020000000002), CONST64(0x0200028000000002), CONST64(0x0200020002000002), CONST64(0x0200028002000002), + CONST64(0x0200020000020002), CONST64(0x0200028000020002), CONST64(0x0200020002020002), CONST64(0x0200028002020002), + CONST64(0x0200020000000202), CONST64(0x0200028000000202), CONST64(0x0200020002000202), CONST64(0x0200028002000202), + CONST64(0x0200020000020202), CONST64(0x0200028000020202), CONST64(0x0200020002020202), CONST64(0x0200028002020202), + CONST64(0x0002020000000000), CONST64(0x0002028000000000), CONST64(0x0002020002000000), CONST64(0x0002028002000000), + CONST64(0x0002020000020000), CONST64(0x0002028000020000), CONST64(0x0002020002020000), CONST64(0x0002028002020000), + CONST64(0x0002020000000200), CONST64(0x0002028000000200), CONST64(0x0002020002000200), CONST64(0x0002028002000200), + CONST64(0x0002020000020200), CONST64(0x0002028000020200), CONST64(0x0002020002020200), CONST64(0x0002028002020200), + CONST64(0x0002020000000002), CONST64(0x0002028000000002), CONST64(0x0002020002000002), CONST64(0x0002028002000002), + CONST64(0x0002020000020002), CONST64(0x0002028000020002), CONST64(0x0002020002020002), CONST64(0x0002028002020002), + CONST64(0x0002020000000202), CONST64(0x0002028000000202), CONST64(0x0002020002000202), CONST64(0x0002028002000202), + CONST64(0x0002020000020202), CONST64(0x0002028000020202), CONST64(0x0002020002020202), CONST64(0x0002028002020202), + CONST64(0x0202020000000000), CONST64(0x0202028000000000), CONST64(0x0202020002000000), CONST64(0x0202028002000000), + CONST64(0x0202020000020000), CONST64(0x0202028000020000), CONST64(0x0202020002020000), CONST64(0x0202028002020000), + CONST64(0x0202020000000200), CONST64(0x0202028000000200), CONST64(0x0202020002000200), CONST64(0x0202028002000200), + CONST64(0x0202020000020200), CONST64(0x0202028000020200), CONST64(0x0202020002020200), CONST64(0x0202028002020200), + CONST64(0x0202020000000002), CONST64(0x0202028000000002), CONST64(0x0202020002000002), CONST64(0x0202028002000002), + CONST64(0x0202020000020002), CONST64(0x0202028000020002), CONST64(0x0202020002020002), CONST64(0x0202028002020002), + CONST64(0x0202020000000202), CONST64(0x0202028000000202), CONST64(0x0202020002000202), CONST64(0x0202028002000202), + CONST64(0x0202020000020202), CONST64(0x0202028000020202), CONST64(0x0202020002020202), CONST64(0x0202028002020202) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000000200000000), CONST64(0x0000000008000000), CONST64(0x0000000208000000), + CONST64(0x0000000000080000), CONST64(0x0000000200080000), CONST64(0x0000000008080000), CONST64(0x0000000208080000), + CONST64(0x0000000000000800), CONST64(0x0000000200000800), CONST64(0x0000000008000800), CONST64(0x0000000208000800), + CONST64(0x0000000000080800), CONST64(0x0000000200080800), CONST64(0x0000000008080800), CONST64(0x0000000208080800), + CONST64(0x0000000000000008), CONST64(0x0000000200000008), CONST64(0x0000000008000008), CONST64(0x0000000208000008), + CONST64(0x0000000000080008), CONST64(0x0000000200080008), CONST64(0x0000000008080008), CONST64(0x0000000208080008), + CONST64(0x0000000000000808), CONST64(0x0000000200000808), CONST64(0x0000000008000808), CONST64(0x0000000208000808), + CONST64(0x0000000000080808), CONST64(0x0000000200080808), CONST64(0x0000000008080808), CONST64(0x0000000208080808), + CONST64(0x0800000000000000), CONST64(0x0800000200000000), CONST64(0x0800000008000000), CONST64(0x0800000208000000), + CONST64(0x0800000000080000), CONST64(0x0800000200080000), CONST64(0x0800000008080000), CONST64(0x0800000208080000), + CONST64(0x0800000000000800), CONST64(0x0800000200000800), CONST64(0x0800000008000800), CONST64(0x0800000208000800), + CONST64(0x0800000000080800), CONST64(0x0800000200080800), CONST64(0x0800000008080800), CONST64(0x0800000208080800), + CONST64(0x0800000000000008), CONST64(0x0800000200000008), CONST64(0x0800000008000008), CONST64(0x0800000208000008), + CONST64(0x0800000000080008), CONST64(0x0800000200080008), CONST64(0x0800000008080008), CONST64(0x0800000208080008), + CONST64(0x0800000000000808), CONST64(0x0800000200000808), CONST64(0x0800000008000808), CONST64(0x0800000208000808), + CONST64(0x0800000000080808), CONST64(0x0800000200080808), CONST64(0x0800000008080808), CONST64(0x0800000208080808), + CONST64(0x0008000000000000), CONST64(0x0008000200000000), CONST64(0x0008000008000000), CONST64(0x0008000208000000), + CONST64(0x0008000000080000), CONST64(0x0008000200080000), CONST64(0x0008000008080000), CONST64(0x0008000208080000), + CONST64(0x0008000000000800), CONST64(0x0008000200000800), CONST64(0x0008000008000800), CONST64(0x0008000208000800), + CONST64(0x0008000000080800), CONST64(0x0008000200080800), CONST64(0x0008000008080800), CONST64(0x0008000208080800), + CONST64(0x0008000000000008), CONST64(0x0008000200000008), CONST64(0x0008000008000008), CONST64(0x0008000208000008), + CONST64(0x0008000000080008), CONST64(0x0008000200080008), CONST64(0x0008000008080008), CONST64(0x0008000208080008), + CONST64(0x0008000000000808), CONST64(0x0008000200000808), CONST64(0x0008000008000808), CONST64(0x0008000208000808), + CONST64(0x0008000000080808), CONST64(0x0008000200080808), CONST64(0x0008000008080808), CONST64(0x0008000208080808), + CONST64(0x0808000000000000), CONST64(0x0808000200000000), CONST64(0x0808000008000000), CONST64(0x0808000208000000), + CONST64(0x0808000000080000), CONST64(0x0808000200080000), CONST64(0x0808000008080000), CONST64(0x0808000208080000), + CONST64(0x0808000000000800), CONST64(0x0808000200000800), CONST64(0x0808000008000800), CONST64(0x0808000208000800), + CONST64(0x0808000000080800), CONST64(0x0808000200080800), CONST64(0x0808000008080800), CONST64(0x0808000208080800), + CONST64(0x0808000000000008), CONST64(0x0808000200000008), CONST64(0x0808000008000008), CONST64(0x0808000208000008), + CONST64(0x0808000000080008), CONST64(0x0808000200080008), CONST64(0x0808000008080008), CONST64(0x0808000208080008), + CONST64(0x0808000000000808), CONST64(0x0808000200000808), CONST64(0x0808000008000808), CONST64(0x0808000208000808), + CONST64(0x0808000000080808), CONST64(0x0808000200080808), CONST64(0x0808000008080808), CONST64(0x0808000208080808), + CONST64(0x0000080000000000), CONST64(0x0000080200000000), CONST64(0x0000080008000000), CONST64(0x0000080208000000), + CONST64(0x0000080000080000), CONST64(0x0000080200080000), CONST64(0x0000080008080000), CONST64(0x0000080208080000), + CONST64(0x0000080000000800), CONST64(0x0000080200000800), CONST64(0x0000080008000800), CONST64(0x0000080208000800), + CONST64(0x0000080000080800), CONST64(0x0000080200080800), CONST64(0x0000080008080800), CONST64(0x0000080208080800), + CONST64(0x0000080000000008), CONST64(0x0000080200000008), CONST64(0x0000080008000008), CONST64(0x0000080208000008), + CONST64(0x0000080000080008), CONST64(0x0000080200080008), CONST64(0x0000080008080008), CONST64(0x0000080208080008), + CONST64(0x0000080000000808), CONST64(0x0000080200000808), CONST64(0x0000080008000808), CONST64(0x0000080208000808), + CONST64(0x0000080000080808), CONST64(0x0000080200080808), CONST64(0x0000080008080808), CONST64(0x0000080208080808), + CONST64(0x0800080000000000), CONST64(0x0800080200000000), CONST64(0x0800080008000000), CONST64(0x0800080208000000), + CONST64(0x0800080000080000), CONST64(0x0800080200080000), CONST64(0x0800080008080000), CONST64(0x0800080208080000), + CONST64(0x0800080000000800), CONST64(0x0800080200000800), CONST64(0x0800080008000800), CONST64(0x0800080208000800), + CONST64(0x0800080000080800), CONST64(0x0800080200080800), CONST64(0x0800080008080800), CONST64(0x0800080208080800), + CONST64(0x0800080000000008), CONST64(0x0800080200000008), CONST64(0x0800080008000008), CONST64(0x0800080208000008), + CONST64(0x0800080000080008), CONST64(0x0800080200080008), CONST64(0x0800080008080008), CONST64(0x0800080208080008), + CONST64(0x0800080000000808), CONST64(0x0800080200000808), CONST64(0x0800080008000808), CONST64(0x0800080208000808), + CONST64(0x0800080000080808), CONST64(0x0800080200080808), CONST64(0x0800080008080808), CONST64(0x0800080208080808), + CONST64(0x0008080000000000), CONST64(0x0008080200000000), CONST64(0x0008080008000000), CONST64(0x0008080208000000), + CONST64(0x0008080000080000), CONST64(0x0008080200080000), CONST64(0x0008080008080000), CONST64(0x0008080208080000), + CONST64(0x0008080000000800), CONST64(0x0008080200000800), CONST64(0x0008080008000800), CONST64(0x0008080208000800), + CONST64(0x0008080000080800), CONST64(0x0008080200080800), CONST64(0x0008080008080800), CONST64(0x0008080208080800), + CONST64(0x0008080000000008), CONST64(0x0008080200000008), CONST64(0x0008080008000008), CONST64(0x0008080208000008), + CONST64(0x0008080000080008), CONST64(0x0008080200080008), CONST64(0x0008080008080008), CONST64(0x0008080208080008), + CONST64(0x0008080000000808), CONST64(0x0008080200000808), CONST64(0x0008080008000808), CONST64(0x0008080208000808), + CONST64(0x0008080000080808), CONST64(0x0008080200080808), CONST64(0x0008080008080808), CONST64(0x0008080208080808), + CONST64(0x0808080000000000), CONST64(0x0808080200000000), CONST64(0x0808080008000000), CONST64(0x0808080208000000), + CONST64(0x0808080000080000), CONST64(0x0808080200080000), CONST64(0x0808080008080000), CONST64(0x0808080208080000), + CONST64(0x0808080000000800), CONST64(0x0808080200000800), CONST64(0x0808080008000800), CONST64(0x0808080208000800), + CONST64(0x0808080000080800), CONST64(0x0808080200080800), CONST64(0x0808080008080800), CONST64(0x0808080208080800), + CONST64(0x0808080000000008), CONST64(0x0808080200000008), CONST64(0x0808080008000008), CONST64(0x0808080208000008), + CONST64(0x0808080000080008), CONST64(0x0808080200080008), CONST64(0x0808080008080008), CONST64(0x0808080208080008), + CONST64(0x0808080000000808), CONST64(0x0808080200000808), CONST64(0x0808080008000808), CONST64(0x0808080208000808), + CONST64(0x0808080000080808), CONST64(0x0808080200080808), CONST64(0x0808080008080808), CONST64(0x0808080208080808) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000000800000000), CONST64(0x0000000020000000), CONST64(0x0000000820000000), + CONST64(0x0000000000200000), CONST64(0x0000000800200000), CONST64(0x0000000020200000), CONST64(0x0000000820200000), + CONST64(0x0000000000002000), CONST64(0x0000000800002000), CONST64(0x0000000020002000), CONST64(0x0000000820002000), + CONST64(0x0000000000202000), CONST64(0x0000000800202000), CONST64(0x0000000020202000), CONST64(0x0000000820202000), + CONST64(0x0000000000000020), CONST64(0x0000000800000020), CONST64(0x0000000020000020), CONST64(0x0000000820000020), + CONST64(0x0000000000200020), CONST64(0x0000000800200020), CONST64(0x0000000020200020), CONST64(0x0000000820200020), + CONST64(0x0000000000002020), CONST64(0x0000000800002020), CONST64(0x0000000020002020), CONST64(0x0000000820002020), + CONST64(0x0000000000202020), CONST64(0x0000000800202020), CONST64(0x0000000020202020), CONST64(0x0000000820202020), + CONST64(0x2000000000000000), CONST64(0x2000000800000000), CONST64(0x2000000020000000), CONST64(0x2000000820000000), + CONST64(0x2000000000200000), CONST64(0x2000000800200000), CONST64(0x2000000020200000), CONST64(0x2000000820200000), + CONST64(0x2000000000002000), CONST64(0x2000000800002000), CONST64(0x2000000020002000), CONST64(0x2000000820002000), + CONST64(0x2000000000202000), CONST64(0x2000000800202000), CONST64(0x2000000020202000), CONST64(0x2000000820202000), + CONST64(0x2000000000000020), CONST64(0x2000000800000020), CONST64(0x2000000020000020), CONST64(0x2000000820000020), + CONST64(0x2000000000200020), CONST64(0x2000000800200020), CONST64(0x2000000020200020), CONST64(0x2000000820200020), + CONST64(0x2000000000002020), CONST64(0x2000000800002020), CONST64(0x2000000020002020), CONST64(0x2000000820002020), + CONST64(0x2000000000202020), CONST64(0x2000000800202020), CONST64(0x2000000020202020), CONST64(0x2000000820202020), + CONST64(0x0020000000000000), CONST64(0x0020000800000000), CONST64(0x0020000020000000), CONST64(0x0020000820000000), + CONST64(0x0020000000200000), CONST64(0x0020000800200000), CONST64(0x0020000020200000), CONST64(0x0020000820200000), + CONST64(0x0020000000002000), CONST64(0x0020000800002000), CONST64(0x0020000020002000), CONST64(0x0020000820002000), + CONST64(0x0020000000202000), CONST64(0x0020000800202000), CONST64(0x0020000020202000), CONST64(0x0020000820202000), + CONST64(0x0020000000000020), CONST64(0x0020000800000020), CONST64(0x0020000020000020), CONST64(0x0020000820000020), + CONST64(0x0020000000200020), CONST64(0x0020000800200020), CONST64(0x0020000020200020), CONST64(0x0020000820200020), + CONST64(0x0020000000002020), CONST64(0x0020000800002020), CONST64(0x0020000020002020), CONST64(0x0020000820002020), + CONST64(0x0020000000202020), CONST64(0x0020000800202020), CONST64(0x0020000020202020), CONST64(0x0020000820202020), + CONST64(0x2020000000000000), CONST64(0x2020000800000000), CONST64(0x2020000020000000), CONST64(0x2020000820000000), + CONST64(0x2020000000200000), CONST64(0x2020000800200000), CONST64(0x2020000020200000), CONST64(0x2020000820200000), + CONST64(0x2020000000002000), CONST64(0x2020000800002000), CONST64(0x2020000020002000), CONST64(0x2020000820002000), + CONST64(0x2020000000202000), CONST64(0x2020000800202000), CONST64(0x2020000020202000), CONST64(0x2020000820202000), + CONST64(0x2020000000000020), CONST64(0x2020000800000020), CONST64(0x2020000020000020), CONST64(0x2020000820000020), + CONST64(0x2020000000200020), CONST64(0x2020000800200020), CONST64(0x2020000020200020), CONST64(0x2020000820200020), + CONST64(0x2020000000002020), CONST64(0x2020000800002020), CONST64(0x2020000020002020), CONST64(0x2020000820002020), + CONST64(0x2020000000202020), CONST64(0x2020000800202020), CONST64(0x2020000020202020), CONST64(0x2020000820202020), + CONST64(0x0000200000000000), CONST64(0x0000200800000000), CONST64(0x0000200020000000), CONST64(0x0000200820000000), + CONST64(0x0000200000200000), CONST64(0x0000200800200000), CONST64(0x0000200020200000), CONST64(0x0000200820200000), + CONST64(0x0000200000002000), CONST64(0x0000200800002000), CONST64(0x0000200020002000), CONST64(0x0000200820002000), + CONST64(0x0000200000202000), CONST64(0x0000200800202000), CONST64(0x0000200020202000), CONST64(0x0000200820202000), + CONST64(0x0000200000000020), CONST64(0x0000200800000020), CONST64(0x0000200020000020), CONST64(0x0000200820000020), + CONST64(0x0000200000200020), CONST64(0x0000200800200020), CONST64(0x0000200020200020), CONST64(0x0000200820200020), + CONST64(0x0000200000002020), CONST64(0x0000200800002020), CONST64(0x0000200020002020), CONST64(0x0000200820002020), + CONST64(0x0000200000202020), CONST64(0x0000200800202020), CONST64(0x0000200020202020), CONST64(0x0000200820202020), + CONST64(0x2000200000000000), CONST64(0x2000200800000000), CONST64(0x2000200020000000), CONST64(0x2000200820000000), + CONST64(0x2000200000200000), CONST64(0x2000200800200000), CONST64(0x2000200020200000), CONST64(0x2000200820200000), + CONST64(0x2000200000002000), CONST64(0x2000200800002000), CONST64(0x2000200020002000), CONST64(0x2000200820002000), + CONST64(0x2000200000202000), CONST64(0x2000200800202000), CONST64(0x2000200020202000), CONST64(0x2000200820202000), + CONST64(0x2000200000000020), CONST64(0x2000200800000020), CONST64(0x2000200020000020), CONST64(0x2000200820000020), + CONST64(0x2000200000200020), CONST64(0x2000200800200020), CONST64(0x2000200020200020), CONST64(0x2000200820200020), + CONST64(0x2000200000002020), CONST64(0x2000200800002020), CONST64(0x2000200020002020), CONST64(0x2000200820002020), + CONST64(0x2000200000202020), CONST64(0x2000200800202020), CONST64(0x2000200020202020), CONST64(0x2000200820202020), + CONST64(0x0020200000000000), CONST64(0x0020200800000000), CONST64(0x0020200020000000), CONST64(0x0020200820000000), + CONST64(0x0020200000200000), CONST64(0x0020200800200000), CONST64(0x0020200020200000), CONST64(0x0020200820200000), + CONST64(0x0020200000002000), CONST64(0x0020200800002000), CONST64(0x0020200020002000), CONST64(0x0020200820002000), + CONST64(0x0020200000202000), CONST64(0x0020200800202000), CONST64(0x0020200020202000), CONST64(0x0020200820202000), + CONST64(0x0020200000000020), CONST64(0x0020200800000020), CONST64(0x0020200020000020), CONST64(0x0020200820000020), + CONST64(0x0020200000200020), CONST64(0x0020200800200020), CONST64(0x0020200020200020), CONST64(0x0020200820200020), + CONST64(0x0020200000002020), CONST64(0x0020200800002020), CONST64(0x0020200020002020), CONST64(0x0020200820002020), + CONST64(0x0020200000202020), CONST64(0x0020200800202020), CONST64(0x0020200020202020), CONST64(0x0020200820202020), + CONST64(0x2020200000000000), CONST64(0x2020200800000000), CONST64(0x2020200020000000), CONST64(0x2020200820000000), + CONST64(0x2020200000200000), CONST64(0x2020200800200000), CONST64(0x2020200020200000), CONST64(0x2020200820200000), + CONST64(0x2020200000002000), CONST64(0x2020200800002000), CONST64(0x2020200020002000), CONST64(0x2020200820002000), + CONST64(0x2020200000202000), CONST64(0x2020200800202000), CONST64(0x2020200020202000), CONST64(0x2020200820202000), + CONST64(0x2020200000000020), CONST64(0x2020200800000020), CONST64(0x2020200020000020), CONST64(0x2020200820000020), + CONST64(0x2020200000200020), CONST64(0x2020200800200020), CONST64(0x2020200020200020), CONST64(0x2020200820200020), + CONST64(0x2020200000002020), CONST64(0x2020200800002020), CONST64(0x2020200020002020), CONST64(0x2020200820002020), + CONST64(0x2020200000202020), CONST64(0x2020200800202020), CONST64(0x2020200020202020), CONST64(0x2020200820202020) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000002000000000), CONST64(0x0000000080000000), CONST64(0x0000002080000000), + CONST64(0x0000000000800000), CONST64(0x0000002000800000), CONST64(0x0000000080800000), CONST64(0x0000002080800000), + CONST64(0x0000000000008000), CONST64(0x0000002000008000), CONST64(0x0000000080008000), CONST64(0x0000002080008000), + CONST64(0x0000000000808000), CONST64(0x0000002000808000), CONST64(0x0000000080808000), CONST64(0x0000002080808000), + CONST64(0x0000000000000080), CONST64(0x0000002000000080), CONST64(0x0000000080000080), CONST64(0x0000002080000080), + CONST64(0x0000000000800080), CONST64(0x0000002000800080), CONST64(0x0000000080800080), CONST64(0x0000002080800080), + CONST64(0x0000000000008080), CONST64(0x0000002000008080), CONST64(0x0000000080008080), CONST64(0x0000002080008080), + CONST64(0x0000000000808080), CONST64(0x0000002000808080), CONST64(0x0000000080808080), CONST64(0x0000002080808080), + CONST64(0x8000000000000000), CONST64(0x8000002000000000), CONST64(0x8000000080000000), CONST64(0x8000002080000000), + CONST64(0x8000000000800000), CONST64(0x8000002000800000), CONST64(0x8000000080800000), CONST64(0x8000002080800000), + CONST64(0x8000000000008000), CONST64(0x8000002000008000), CONST64(0x8000000080008000), CONST64(0x8000002080008000), + CONST64(0x8000000000808000), CONST64(0x8000002000808000), CONST64(0x8000000080808000), CONST64(0x8000002080808000), + CONST64(0x8000000000000080), CONST64(0x8000002000000080), CONST64(0x8000000080000080), CONST64(0x8000002080000080), + CONST64(0x8000000000800080), CONST64(0x8000002000800080), CONST64(0x8000000080800080), CONST64(0x8000002080800080), + CONST64(0x8000000000008080), CONST64(0x8000002000008080), CONST64(0x8000000080008080), CONST64(0x8000002080008080), + CONST64(0x8000000000808080), CONST64(0x8000002000808080), CONST64(0x8000000080808080), CONST64(0x8000002080808080), + CONST64(0x0080000000000000), CONST64(0x0080002000000000), CONST64(0x0080000080000000), CONST64(0x0080002080000000), + CONST64(0x0080000000800000), CONST64(0x0080002000800000), CONST64(0x0080000080800000), CONST64(0x0080002080800000), + CONST64(0x0080000000008000), CONST64(0x0080002000008000), CONST64(0x0080000080008000), CONST64(0x0080002080008000), + CONST64(0x0080000000808000), CONST64(0x0080002000808000), CONST64(0x0080000080808000), CONST64(0x0080002080808000), + CONST64(0x0080000000000080), CONST64(0x0080002000000080), CONST64(0x0080000080000080), CONST64(0x0080002080000080), + CONST64(0x0080000000800080), CONST64(0x0080002000800080), CONST64(0x0080000080800080), CONST64(0x0080002080800080), + CONST64(0x0080000000008080), CONST64(0x0080002000008080), CONST64(0x0080000080008080), CONST64(0x0080002080008080), + CONST64(0x0080000000808080), CONST64(0x0080002000808080), CONST64(0x0080000080808080), CONST64(0x0080002080808080), + CONST64(0x8080000000000000), CONST64(0x8080002000000000), CONST64(0x8080000080000000), CONST64(0x8080002080000000), + CONST64(0x8080000000800000), CONST64(0x8080002000800000), CONST64(0x8080000080800000), CONST64(0x8080002080800000), + CONST64(0x8080000000008000), CONST64(0x8080002000008000), CONST64(0x8080000080008000), CONST64(0x8080002080008000), + CONST64(0x8080000000808000), CONST64(0x8080002000808000), CONST64(0x8080000080808000), CONST64(0x8080002080808000), + CONST64(0x8080000000000080), CONST64(0x8080002000000080), CONST64(0x8080000080000080), CONST64(0x8080002080000080), + CONST64(0x8080000000800080), CONST64(0x8080002000800080), CONST64(0x8080000080800080), CONST64(0x8080002080800080), + CONST64(0x8080000000008080), CONST64(0x8080002000008080), CONST64(0x8080000080008080), CONST64(0x8080002080008080), + CONST64(0x8080000000808080), CONST64(0x8080002000808080), CONST64(0x8080000080808080), CONST64(0x8080002080808080), + CONST64(0x0000800000000000), CONST64(0x0000802000000000), CONST64(0x0000800080000000), CONST64(0x0000802080000000), + CONST64(0x0000800000800000), CONST64(0x0000802000800000), CONST64(0x0000800080800000), CONST64(0x0000802080800000), + CONST64(0x0000800000008000), CONST64(0x0000802000008000), CONST64(0x0000800080008000), CONST64(0x0000802080008000), + CONST64(0x0000800000808000), CONST64(0x0000802000808000), CONST64(0x0000800080808000), CONST64(0x0000802080808000), + CONST64(0x0000800000000080), CONST64(0x0000802000000080), CONST64(0x0000800080000080), CONST64(0x0000802080000080), + CONST64(0x0000800000800080), CONST64(0x0000802000800080), CONST64(0x0000800080800080), CONST64(0x0000802080800080), + CONST64(0x0000800000008080), CONST64(0x0000802000008080), CONST64(0x0000800080008080), CONST64(0x0000802080008080), + CONST64(0x0000800000808080), CONST64(0x0000802000808080), CONST64(0x0000800080808080), CONST64(0x0000802080808080), + CONST64(0x8000800000000000), CONST64(0x8000802000000000), CONST64(0x8000800080000000), CONST64(0x8000802080000000), + CONST64(0x8000800000800000), CONST64(0x8000802000800000), CONST64(0x8000800080800000), CONST64(0x8000802080800000), + CONST64(0x8000800000008000), CONST64(0x8000802000008000), CONST64(0x8000800080008000), CONST64(0x8000802080008000), + CONST64(0x8000800000808000), CONST64(0x8000802000808000), CONST64(0x8000800080808000), CONST64(0x8000802080808000), + CONST64(0x8000800000000080), CONST64(0x8000802000000080), CONST64(0x8000800080000080), CONST64(0x8000802080000080), + CONST64(0x8000800000800080), CONST64(0x8000802000800080), CONST64(0x8000800080800080), CONST64(0x8000802080800080), + CONST64(0x8000800000008080), CONST64(0x8000802000008080), CONST64(0x8000800080008080), CONST64(0x8000802080008080), + CONST64(0x8000800000808080), CONST64(0x8000802000808080), CONST64(0x8000800080808080), CONST64(0x8000802080808080), + CONST64(0x0080800000000000), CONST64(0x0080802000000000), CONST64(0x0080800080000000), CONST64(0x0080802080000000), + CONST64(0x0080800000800000), CONST64(0x0080802000800000), CONST64(0x0080800080800000), CONST64(0x0080802080800000), + CONST64(0x0080800000008000), CONST64(0x0080802000008000), CONST64(0x0080800080008000), CONST64(0x0080802080008000), + CONST64(0x0080800000808000), CONST64(0x0080802000808000), CONST64(0x0080800080808000), CONST64(0x0080802080808000), + CONST64(0x0080800000000080), CONST64(0x0080802000000080), CONST64(0x0080800080000080), CONST64(0x0080802080000080), + CONST64(0x0080800000800080), CONST64(0x0080802000800080), CONST64(0x0080800080800080), CONST64(0x0080802080800080), + CONST64(0x0080800000008080), CONST64(0x0080802000008080), CONST64(0x0080800080008080), CONST64(0x0080802080008080), + CONST64(0x0080800000808080), CONST64(0x0080802000808080), CONST64(0x0080800080808080), CONST64(0x0080802080808080), + CONST64(0x8080800000000000), CONST64(0x8080802000000000), CONST64(0x8080800080000000), CONST64(0x8080802080000000), + CONST64(0x8080800000800000), CONST64(0x8080802000800000), CONST64(0x8080800080800000), CONST64(0x8080802080800000), + CONST64(0x8080800000008000), CONST64(0x8080802000008000), CONST64(0x8080800080008000), CONST64(0x8080802080008000), + CONST64(0x8080800000808000), CONST64(0x8080802000808000), CONST64(0x8080800080808000), CONST64(0x8080802080808000), + CONST64(0x8080800000000080), CONST64(0x8080802000000080), CONST64(0x8080800080000080), CONST64(0x8080802080000080), + CONST64(0x8080800000800080), CONST64(0x8080802000800080), CONST64(0x8080800080800080), CONST64(0x8080802080800080), + CONST64(0x8080800000008080), CONST64(0x8080802000008080), CONST64(0x8080800080008080), CONST64(0x8080802080008080), + CONST64(0x8080800000808080), CONST64(0x8080802000808080), CONST64(0x8080800080808080), CONST64(0x8080802080808080) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000004000000000), CONST64(0x0000000001000000), CONST64(0x0000004001000000), + CONST64(0x0000000000010000), CONST64(0x0000004000010000), CONST64(0x0000000001010000), CONST64(0x0000004001010000), + CONST64(0x0000000000000100), CONST64(0x0000004000000100), CONST64(0x0000000001000100), CONST64(0x0000004001000100), + CONST64(0x0000000000010100), CONST64(0x0000004000010100), CONST64(0x0000000001010100), CONST64(0x0000004001010100), + CONST64(0x0000000000000001), CONST64(0x0000004000000001), CONST64(0x0000000001000001), CONST64(0x0000004001000001), + CONST64(0x0000000000010001), CONST64(0x0000004000010001), CONST64(0x0000000001010001), CONST64(0x0000004001010001), + CONST64(0x0000000000000101), CONST64(0x0000004000000101), CONST64(0x0000000001000101), CONST64(0x0000004001000101), + CONST64(0x0000000000010101), CONST64(0x0000004000010101), CONST64(0x0000000001010101), CONST64(0x0000004001010101), + CONST64(0x0100000000000000), CONST64(0x0100004000000000), CONST64(0x0100000001000000), CONST64(0x0100004001000000), + CONST64(0x0100000000010000), CONST64(0x0100004000010000), CONST64(0x0100000001010000), CONST64(0x0100004001010000), + CONST64(0x0100000000000100), CONST64(0x0100004000000100), CONST64(0x0100000001000100), CONST64(0x0100004001000100), + CONST64(0x0100000000010100), CONST64(0x0100004000010100), CONST64(0x0100000001010100), CONST64(0x0100004001010100), + CONST64(0x0100000000000001), CONST64(0x0100004000000001), CONST64(0x0100000001000001), CONST64(0x0100004001000001), + CONST64(0x0100000000010001), CONST64(0x0100004000010001), CONST64(0x0100000001010001), CONST64(0x0100004001010001), + CONST64(0x0100000000000101), CONST64(0x0100004000000101), CONST64(0x0100000001000101), CONST64(0x0100004001000101), + CONST64(0x0100000000010101), CONST64(0x0100004000010101), CONST64(0x0100000001010101), CONST64(0x0100004001010101), + CONST64(0x0001000000000000), CONST64(0x0001004000000000), CONST64(0x0001000001000000), CONST64(0x0001004001000000), + CONST64(0x0001000000010000), CONST64(0x0001004000010000), CONST64(0x0001000001010000), CONST64(0x0001004001010000), + CONST64(0x0001000000000100), CONST64(0x0001004000000100), CONST64(0x0001000001000100), CONST64(0x0001004001000100), + CONST64(0x0001000000010100), CONST64(0x0001004000010100), CONST64(0x0001000001010100), CONST64(0x0001004001010100), + CONST64(0x0001000000000001), CONST64(0x0001004000000001), CONST64(0x0001000001000001), CONST64(0x0001004001000001), + CONST64(0x0001000000010001), CONST64(0x0001004000010001), CONST64(0x0001000001010001), CONST64(0x0001004001010001), + CONST64(0x0001000000000101), CONST64(0x0001004000000101), CONST64(0x0001000001000101), CONST64(0x0001004001000101), + CONST64(0x0001000000010101), CONST64(0x0001004000010101), CONST64(0x0001000001010101), CONST64(0x0001004001010101), + CONST64(0x0101000000000000), CONST64(0x0101004000000000), CONST64(0x0101000001000000), CONST64(0x0101004001000000), + CONST64(0x0101000000010000), CONST64(0x0101004000010000), CONST64(0x0101000001010000), CONST64(0x0101004001010000), + CONST64(0x0101000000000100), CONST64(0x0101004000000100), CONST64(0x0101000001000100), CONST64(0x0101004001000100), + CONST64(0x0101000000010100), CONST64(0x0101004000010100), CONST64(0x0101000001010100), CONST64(0x0101004001010100), + CONST64(0x0101000000000001), CONST64(0x0101004000000001), CONST64(0x0101000001000001), CONST64(0x0101004001000001), + CONST64(0x0101000000010001), CONST64(0x0101004000010001), CONST64(0x0101000001010001), CONST64(0x0101004001010001), + CONST64(0x0101000000000101), CONST64(0x0101004000000101), CONST64(0x0101000001000101), CONST64(0x0101004001000101), + CONST64(0x0101000000010101), CONST64(0x0101004000010101), CONST64(0x0101000001010101), CONST64(0x0101004001010101), + CONST64(0x0000010000000000), CONST64(0x0000014000000000), CONST64(0x0000010001000000), CONST64(0x0000014001000000), + CONST64(0x0000010000010000), CONST64(0x0000014000010000), CONST64(0x0000010001010000), CONST64(0x0000014001010000), + CONST64(0x0000010000000100), CONST64(0x0000014000000100), CONST64(0x0000010001000100), CONST64(0x0000014001000100), + CONST64(0x0000010000010100), CONST64(0x0000014000010100), CONST64(0x0000010001010100), CONST64(0x0000014001010100), + CONST64(0x0000010000000001), CONST64(0x0000014000000001), CONST64(0x0000010001000001), CONST64(0x0000014001000001), + CONST64(0x0000010000010001), CONST64(0x0000014000010001), CONST64(0x0000010001010001), CONST64(0x0000014001010001), + CONST64(0x0000010000000101), CONST64(0x0000014000000101), CONST64(0x0000010001000101), CONST64(0x0000014001000101), + CONST64(0x0000010000010101), CONST64(0x0000014000010101), CONST64(0x0000010001010101), CONST64(0x0000014001010101), + CONST64(0x0100010000000000), CONST64(0x0100014000000000), CONST64(0x0100010001000000), CONST64(0x0100014001000000), + CONST64(0x0100010000010000), CONST64(0x0100014000010000), CONST64(0x0100010001010000), CONST64(0x0100014001010000), + CONST64(0x0100010000000100), CONST64(0x0100014000000100), CONST64(0x0100010001000100), CONST64(0x0100014001000100), + CONST64(0x0100010000010100), CONST64(0x0100014000010100), CONST64(0x0100010001010100), CONST64(0x0100014001010100), + CONST64(0x0100010000000001), CONST64(0x0100014000000001), CONST64(0x0100010001000001), CONST64(0x0100014001000001), + CONST64(0x0100010000010001), CONST64(0x0100014000010001), CONST64(0x0100010001010001), CONST64(0x0100014001010001), + CONST64(0x0100010000000101), CONST64(0x0100014000000101), CONST64(0x0100010001000101), CONST64(0x0100014001000101), + CONST64(0x0100010000010101), CONST64(0x0100014000010101), CONST64(0x0100010001010101), CONST64(0x0100014001010101), + CONST64(0x0001010000000000), CONST64(0x0001014000000000), CONST64(0x0001010001000000), CONST64(0x0001014001000000), + CONST64(0x0001010000010000), CONST64(0x0001014000010000), CONST64(0x0001010001010000), CONST64(0x0001014001010000), + CONST64(0x0001010000000100), CONST64(0x0001014000000100), CONST64(0x0001010001000100), CONST64(0x0001014001000100), + CONST64(0x0001010000010100), CONST64(0x0001014000010100), CONST64(0x0001010001010100), CONST64(0x0001014001010100), + CONST64(0x0001010000000001), CONST64(0x0001014000000001), CONST64(0x0001010001000001), CONST64(0x0001014001000001), + CONST64(0x0001010000010001), CONST64(0x0001014000010001), CONST64(0x0001010001010001), CONST64(0x0001014001010001), + CONST64(0x0001010000000101), CONST64(0x0001014000000101), CONST64(0x0001010001000101), CONST64(0x0001014001000101), + CONST64(0x0001010000010101), CONST64(0x0001014000010101), CONST64(0x0001010001010101), CONST64(0x0001014001010101), + CONST64(0x0101010000000000), CONST64(0x0101014000000000), CONST64(0x0101010001000000), CONST64(0x0101014001000000), + CONST64(0x0101010000010000), CONST64(0x0101014000010000), CONST64(0x0101010001010000), CONST64(0x0101014001010000), + CONST64(0x0101010000000100), CONST64(0x0101014000000100), CONST64(0x0101010001000100), CONST64(0x0101014001000100), + CONST64(0x0101010000010100), CONST64(0x0101014000010100), CONST64(0x0101010001010100), CONST64(0x0101014001010100), + CONST64(0x0101010000000001), CONST64(0x0101014000000001), CONST64(0x0101010001000001), CONST64(0x0101014001000001), + CONST64(0x0101010000010001), CONST64(0x0101014000010001), CONST64(0x0101010001010001), CONST64(0x0101014001010001), + CONST64(0x0101010000000101), CONST64(0x0101014000000101), CONST64(0x0101010001000101), CONST64(0x0101014001000101), + CONST64(0x0101010000010101), CONST64(0x0101014000010101), CONST64(0x0101010001010101), CONST64(0x0101014001010101) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000000100000000), CONST64(0x0000000004000000), CONST64(0x0000000104000000), + CONST64(0x0000000000040000), CONST64(0x0000000100040000), CONST64(0x0000000004040000), CONST64(0x0000000104040000), + CONST64(0x0000000000000400), CONST64(0x0000000100000400), CONST64(0x0000000004000400), CONST64(0x0000000104000400), + CONST64(0x0000000000040400), CONST64(0x0000000100040400), CONST64(0x0000000004040400), CONST64(0x0000000104040400), + CONST64(0x0000000000000004), CONST64(0x0000000100000004), CONST64(0x0000000004000004), CONST64(0x0000000104000004), + CONST64(0x0000000000040004), CONST64(0x0000000100040004), CONST64(0x0000000004040004), CONST64(0x0000000104040004), + CONST64(0x0000000000000404), CONST64(0x0000000100000404), CONST64(0x0000000004000404), CONST64(0x0000000104000404), + CONST64(0x0000000000040404), CONST64(0x0000000100040404), CONST64(0x0000000004040404), CONST64(0x0000000104040404), + CONST64(0x0400000000000000), CONST64(0x0400000100000000), CONST64(0x0400000004000000), CONST64(0x0400000104000000), + CONST64(0x0400000000040000), CONST64(0x0400000100040000), CONST64(0x0400000004040000), CONST64(0x0400000104040000), + CONST64(0x0400000000000400), CONST64(0x0400000100000400), CONST64(0x0400000004000400), CONST64(0x0400000104000400), + CONST64(0x0400000000040400), CONST64(0x0400000100040400), CONST64(0x0400000004040400), CONST64(0x0400000104040400), + CONST64(0x0400000000000004), CONST64(0x0400000100000004), CONST64(0x0400000004000004), CONST64(0x0400000104000004), + CONST64(0x0400000000040004), CONST64(0x0400000100040004), CONST64(0x0400000004040004), CONST64(0x0400000104040004), + CONST64(0x0400000000000404), CONST64(0x0400000100000404), CONST64(0x0400000004000404), CONST64(0x0400000104000404), + CONST64(0x0400000000040404), CONST64(0x0400000100040404), CONST64(0x0400000004040404), CONST64(0x0400000104040404), + CONST64(0x0004000000000000), CONST64(0x0004000100000000), CONST64(0x0004000004000000), CONST64(0x0004000104000000), + CONST64(0x0004000000040000), CONST64(0x0004000100040000), CONST64(0x0004000004040000), CONST64(0x0004000104040000), + CONST64(0x0004000000000400), CONST64(0x0004000100000400), CONST64(0x0004000004000400), CONST64(0x0004000104000400), + CONST64(0x0004000000040400), CONST64(0x0004000100040400), CONST64(0x0004000004040400), CONST64(0x0004000104040400), + CONST64(0x0004000000000004), CONST64(0x0004000100000004), CONST64(0x0004000004000004), CONST64(0x0004000104000004), + CONST64(0x0004000000040004), CONST64(0x0004000100040004), CONST64(0x0004000004040004), CONST64(0x0004000104040004), + CONST64(0x0004000000000404), CONST64(0x0004000100000404), CONST64(0x0004000004000404), CONST64(0x0004000104000404), + CONST64(0x0004000000040404), CONST64(0x0004000100040404), CONST64(0x0004000004040404), CONST64(0x0004000104040404), + CONST64(0x0404000000000000), CONST64(0x0404000100000000), CONST64(0x0404000004000000), CONST64(0x0404000104000000), + CONST64(0x0404000000040000), CONST64(0x0404000100040000), CONST64(0x0404000004040000), CONST64(0x0404000104040000), + CONST64(0x0404000000000400), CONST64(0x0404000100000400), CONST64(0x0404000004000400), CONST64(0x0404000104000400), + CONST64(0x0404000000040400), CONST64(0x0404000100040400), CONST64(0x0404000004040400), CONST64(0x0404000104040400), + CONST64(0x0404000000000004), CONST64(0x0404000100000004), CONST64(0x0404000004000004), CONST64(0x0404000104000004), + CONST64(0x0404000000040004), CONST64(0x0404000100040004), CONST64(0x0404000004040004), CONST64(0x0404000104040004), + CONST64(0x0404000000000404), CONST64(0x0404000100000404), CONST64(0x0404000004000404), CONST64(0x0404000104000404), + CONST64(0x0404000000040404), CONST64(0x0404000100040404), CONST64(0x0404000004040404), CONST64(0x0404000104040404), + CONST64(0x0000040000000000), CONST64(0x0000040100000000), CONST64(0x0000040004000000), CONST64(0x0000040104000000), + CONST64(0x0000040000040000), CONST64(0x0000040100040000), CONST64(0x0000040004040000), CONST64(0x0000040104040000), + CONST64(0x0000040000000400), CONST64(0x0000040100000400), CONST64(0x0000040004000400), CONST64(0x0000040104000400), + CONST64(0x0000040000040400), CONST64(0x0000040100040400), CONST64(0x0000040004040400), CONST64(0x0000040104040400), + CONST64(0x0000040000000004), CONST64(0x0000040100000004), CONST64(0x0000040004000004), CONST64(0x0000040104000004), + CONST64(0x0000040000040004), CONST64(0x0000040100040004), CONST64(0x0000040004040004), CONST64(0x0000040104040004), + CONST64(0x0000040000000404), CONST64(0x0000040100000404), CONST64(0x0000040004000404), CONST64(0x0000040104000404), + CONST64(0x0000040000040404), CONST64(0x0000040100040404), CONST64(0x0000040004040404), CONST64(0x0000040104040404), + CONST64(0x0400040000000000), CONST64(0x0400040100000000), CONST64(0x0400040004000000), CONST64(0x0400040104000000), + CONST64(0x0400040000040000), CONST64(0x0400040100040000), CONST64(0x0400040004040000), CONST64(0x0400040104040000), + CONST64(0x0400040000000400), CONST64(0x0400040100000400), CONST64(0x0400040004000400), CONST64(0x0400040104000400), + CONST64(0x0400040000040400), CONST64(0x0400040100040400), CONST64(0x0400040004040400), CONST64(0x0400040104040400), + CONST64(0x0400040000000004), CONST64(0x0400040100000004), CONST64(0x0400040004000004), CONST64(0x0400040104000004), + CONST64(0x0400040000040004), CONST64(0x0400040100040004), CONST64(0x0400040004040004), CONST64(0x0400040104040004), + CONST64(0x0400040000000404), CONST64(0x0400040100000404), CONST64(0x0400040004000404), CONST64(0x0400040104000404), + CONST64(0x0400040000040404), CONST64(0x0400040100040404), CONST64(0x0400040004040404), CONST64(0x0400040104040404), + CONST64(0x0004040000000000), CONST64(0x0004040100000000), CONST64(0x0004040004000000), CONST64(0x0004040104000000), + CONST64(0x0004040000040000), CONST64(0x0004040100040000), CONST64(0x0004040004040000), CONST64(0x0004040104040000), + CONST64(0x0004040000000400), CONST64(0x0004040100000400), CONST64(0x0004040004000400), CONST64(0x0004040104000400), + CONST64(0x0004040000040400), CONST64(0x0004040100040400), CONST64(0x0004040004040400), CONST64(0x0004040104040400), + CONST64(0x0004040000000004), CONST64(0x0004040100000004), CONST64(0x0004040004000004), CONST64(0x0004040104000004), + CONST64(0x0004040000040004), CONST64(0x0004040100040004), CONST64(0x0004040004040004), CONST64(0x0004040104040004), + CONST64(0x0004040000000404), CONST64(0x0004040100000404), CONST64(0x0004040004000404), CONST64(0x0004040104000404), + CONST64(0x0004040000040404), CONST64(0x0004040100040404), CONST64(0x0004040004040404), CONST64(0x0004040104040404), + CONST64(0x0404040000000000), CONST64(0x0404040100000000), CONST64(0x0404040004000000), CONST64(0x0404040104000000), + CONST64(0x0404040000040000), CONST64(0x0404040100040000), CONST64(0x0404040004040000), CONST64(0x0404040104040000), + CONST64(0x0404040000000400), CONST64(0x0404040100000400), CONST64(0x0404040004000400), CONST64(0x0404040104000400), + CONST64(0x0404040000040400), CONST64(0x0404040100040400), CONST64(0x0404040004040400), CONST64(0x0404040104040400), + CONST64(0x0404040000000004), CONST64(0x0404040100000004), CONST64(0x0404040004000004), CONST64(0x0404040104000004), + CONST64(0x0404040000040004), CONST64(0x0404040100040004), CONST64(0x0404040004040004), CONST64(0x0404040104040004), + CONST64(0x0404040000000404), CONST64(0x0404040100000404), CONST64(0x0404040004000404), CONST64(0x0404040104000404), + CONST64(0x0404040000040404), CONST64(0x0404040100040404), CONST64(0x0404040004040404), CONST64(0x0404040104040404) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000000400000000), CONST64(0x0000000010000000), CONST64(0x0000000410000000), + CONST64(0x0000000000100000), CONST64(0x0000000400100000), CONST64(0x0000000010100000), CONST64(0x0000000410100000), + CONST64(0x0000000000001000), CONST64(0x0000000400001000), CONST64(0x0000000010001000), CONST64(0x0000000410001000), + CONST64(0x0000000000101000), CONST64(0x0000000400101000), CONST64(0x0000000010101000), CONST64(0x0000000410101000), + CONST64(0x0000000000000010), CONST64(0x0000000400000010), CONST64(0x0000000010000010), CONST64(0x0000000410000010), + CONST64(0x0000000000100010), CONST64(0x0000000400100010), CONST64(0x0000000010100010), CONST64(0x0000000410100010), + CONST64(0x0000000000001010), CONST64(0x0000000400001010), CONST64(0x0000000010001010), CONST64(0x0000000410001010), + CONST64(0x0000000000101010), CONST64(0x0000000400101010), CONST64(0x0000000010101010), CONST64(0x0000000410101010), + CONST64(0x1000000000000000), CONST64(0x1000000400000000), CONST64(0x1000000010000000), CONST64(0x1000000410000000), + CONST64(0x1000000000100000), CONST64(0x1000000400100000), CONST64(0x1000000010100000), CONST64(0x1000000410100000), + CONST64(0x1000000000001000), CONST64(0x1000000400001000), CONST64(0x1000000010001000), CONST64(0x1000000410001000), + CONST64(0x1000000000101000), CONST64(0x1000000400101000), CONST64(0x1000000010101000), CONST64(0x1000000410101000), + CONST64(0x1000000000000010), CONST64(0x1000000400000010), CONST64(0x1000000010000010), CONST64(0x1000000410000010), + CONST64(0x1000000000100010), CONST64(0x1000000400100010), CONST64(0x1000000010100010), CONST64(0x1000000410100010), + CONST64(0x1000000000001010), CONST64(0x1000000400001010), CONST64(0x1000000010001010), CONST64(0x1000000410001010), + CONST64(0x1000000000101010), CONST64(0x1000000400101010), CONST64(0x1000000010101010), CONST64(0x1000000410101010), + CONST64(0x0010000000000000), CONST64(0x0010000400000000), CONST64(0x0010000010000000), CONST64(0x0010000410000000), + CONST64(0x0010000000100000), CONST64(0x0010000400100000), CONST64(0x0010000010100000), CONST64(0x0010000410100000), + CONST64(0x0010000000001000), CONST64(0x0010000400001000), CONST64(0x0010000010001000), CONST64(0x0010000410001000), + CONST64(0x0010000000101000), CONST64(0x0010000400101000), CONST64(0x0010000010101000), CONST64(0x0010000410101000), + CONST64(0x0010000000000010), CONST64(0x0010000400000010), CONST64(0x0010000010000010), CONST64(0x0010000410000010), + CONST64(0x0010000000100010), CONST64(0x0010000400100010), CONST64(0x0010000010100010), CONST64(0x0010000410100010), + CONST64(0x0010000000001010), CONST64(0x0010000400001010), CONST64(0x0010000010001010), CONST64(0x0010000410001010), + CONST64(0x0010000000101010), CONST64(0x0010000400101010), CONST64(0x0010000010101010), CONST64(0x0010000410101010), + CONST64(0x1010000000000000), CONST64(0x1010000400000000), CONST64(0x1010000010000000), CONST64(0x1010000410000000), + CONST64(0x1010000000100000), CONST64(0x1010000400100000), CONST64(0x1010000010100000), CONST64(0x1010000410100000), + CONST64(0x1010000000001000), CONST64(0x1010000400001000), CONST64(0x1010000010001000), CONST64(0x1010000410001000), + CONST64(0x1010000000101000), CONST64(0x1010000400101000), CONST64(0x1010000010101000), CONST64(0x1010000410101000), + CONST64(0x1010000000000010), CONST64(0x1010000400000010), CONST64(0x1010000010000010), CONST64(0x1010000410000010), + CONST64(0x1010000000100010), CONST64(0x1010000400100010), CONST64(0x1010000010100010), CONST64(0x1010000410100010), + CONST64(0x1010000000001010), CONST64(0x1010000400001010), CONST64(0x1010000010001010), CONST64(0x1010000410001010), + CONST64(0x1010000000101010), CONST64(0x1010000400101010), CONST64(0x1010000010101010), CONST64(0x1010000410101010), + CONST64(0x0000100000000000), CONST64(0x0000100400000000), CONST64(0x0000100010000000), CONST64(0x0000100410000000), + CONST64(0x0000100000100000), CONST64(0x0000100400100000), CONST64(0x0000100010100000), CONST64(0x0000100410100000), + CONST64(0x0000100000001000), CONST64(0x0000100400001000), CONST64(0x0000100010001000), CONST64(0x0000100410001000), + CONST64(0x0000100000101000), CONST64(0x0000100400101000), CONST64(0x0000100010101000), CONST64(0x0000100410101000), + CONST64(0x0000100000000010), CONST64(0x0000100400000010), CONST64(0x0000100010000010), CONST64(0x0000100410000010), + CONST64(0x0000100000100010), CONST64(0x0000100400100010), CONST64(0x0000100010100010), CONST64(0x0000100410100010), + CONST64(0x0000100000001010), CONST64(0x0000100400001010), CONST64(0x0000100010001010), CONST64(0x0000100410001010), + CONST64(0x0000100000101010), CONST64(0x0000100400101010), CONST64(0x0000100010101010), CONST64(0x0000100410101010), + CONST64(0x1000100000000000), CONST64(0x1000100400000000), CONST64(0x1000100010000000), CONST64(0x1000100410000000), + CONST64(0x1000100000100000), CONST64(0x1000100400100000), CONST64(0x1000100010100000), CONST64(0x1000100410100000), + CONST64(0x1000100000001000), CONST64(0x1000100400001000), CONST64(0x1000100010001000), CONST64(0x1000100410001000), + CONST64(0x1000100000101000), CONST64(0x1000100400101000), CONST64(0x1000100010101000), CONST64(0x1000100410101000), + CONST64(0x1000100000000010), CONST64(0x1000100400000010), CONST64(0x1000100010000010), CONST64(0x1000100410000010), + CONST64(0x1000100000100010), CONST64(0x1000100400100010), CONST64(0x1000100010100010), CONST64(0x1000100410100010), + CONST64(0x1000100000001010), CONST64(0x1000100400001010), CONST64(0x1000100010001010), CONST64(0x1000100410001010), + CONST64(0x1000100000101010), CONST64(0x1000100400101010), CONST64(0x1000100010101010), CONST64(0x1000100410101010), + CONST64(0x0010100000000000), CONST64(0x0010100400000000), CONST64(0x0010100010000000), CONST64(0x0010100410000000), + CONST64(0x0010100000100000), CONST64(0x0010100400100000), CONST64(0x0010100010100000), CONST64(0x0010100410100000), + CONST64(0x0010100000001000), CONST64(0x0010100400001000), CONST64(0x0010100010001000), CONST64(0x0010100410001000), + CONST64(0x0010100000101000), CONST64(0x0010100400101000), CONST64(0x0010100010101000), CONST64(0x0010100410101000), + CONST64(0x0010100000000010), CONST64(0x0010100400000010), CONST64(0x0010100010000010), CONST64(0x0010100410000010), + CONST64(0x0010100000100010), CONST64(0x0010100400100010), CONST64(0x0010100010100010), CONST64(0x0010100410100010), + CONST64(0x0010100000001010), CONST64(0x0010100400001010), CONST64(0x0010100010001010), CONST64(0x0010100410001010), + CONST64(0x0010100000101010), CONST64(0x0010100400101010), CONST64(0x0010100010101010), CONST64(0x0010100410101010), + CONST64(0x1010100000000000), CONST64(0x1010100400000000), CONST64(0x1010100010000000), CONST64(0x1010100410000000), + CONST64(0x1010100000100000), CONST64(0x1010100400100000), CONST64(0x1010100010100000), CONST64(0x1010100410100000), + CONST64(0x1010100000001000), CONST64(0x1010100400001000), CONST64(0x1010100010001000), CONST64(0x1010100410001000), + CONST64(0x1010100000101000), CONST64(0x1010100400101000), CONST64(0x1010100010101000), CONST64(0x1010100410101000), + CONST64(0x1010100000000010), CONST64(0x1010100400000010), CONST64(0x1010100010000010), CONST64(0x1010100410000010), + CONST64(0x1010100000100010), CONST64(0x1010100400100010), CONST64(0x1010100010100010), CONST64(0x1010100410100010), + CONST64(0x1010100000001010), CONST64(0x1010100400001010), CONST64(0x1010100010001010), CONST64(0x1010100410001010), + CONST64(0x1010100000101010), CONST64(0x1010100400101010), CONST64(0x1010100010101010), CONST64(0x1010100410101010) + }, +{ CONST64(0x0000000000000000), CONST64(0x0000001000000000), CONST64(0x0000000040000000), CONST64(0x0000001040000000), + CONST64(0x0000000000400000), CONST64(0x0000001000400000), CONST64(0x0000000040400000), CONST64(0x0000001040400000), + CONST64(0x0000000000004000), CONST64(0x0000001000004000), CONST64(0x0000000040004000), CONST64(0x0000001040004000), + CONST64(0x0000000000404000), CONST64(0x0000001000404000), CONST64(0x0000000040404000), CONST64(0x0000001040404000), + CONST64(0x0000000000000040), CONST64(0x0000001000000040), CONST64(0x0000000040000040), CONST64(0x0000001040000040), + CONST64(0x0000000000400040), CONST64(0x0000001000400040), CONST64(0x0000000040400040), CONST64(0x0000001040400040), + CONST64(0x0000000000004040), CONST64(0x0000001000004040), CONST64(0x0000000040004040), CONST64(0x0000001040004040), + CONST64(0x0000000000404040), CONST64(0x0000001000404040), CONST64(0x0000000040404040), CONST64(0x0000001040404040), + CONST64(0x4000000000000000), CONST64(0x4000001000000000), CONST64(0x4000000040000000), CONST64(0x4000001040000000), + CONST64(0x4000000000400000), CONST64(0x4000001000400000), CONST64(0x4000000040400000), CONST64(0x4000001040400000), + CONST64(0x4000000000004000), CONST64(0x4000001000004000), CONST64(0x4000000040004000), CONST64(0x4000001040004000), + CONST64(0x4000000000404000), CONST64(0x4000001000404000), CONST64(0x4000000040404000), CONST64(0x4000001040404000), + CONST64(0x4000000000000040), CONST64(0x4000001000000040), CONST64(0x4000000040000040), CONST64(0x4000001040000040), + CONST64(0x4000000000400040), CONST64(0x4000001000400040), CONST64(0x4000000040400040), CONST64(0x4000001040400040), + CONST64(0x4000000000004040), CONST64(0x4000001000004040), CONST64(0x4000000040004040), CONST64(0x4000001040004040), + CONST64(0x4000000000404040), CONST64(0x4000001000404040), CONST64(0x4000000040404040), CONST64(0x4000001040404040), + CONST64(0x0040000000000000), CONST64(0x0040001000000000), CONST64(0x0040000040000000), CONST64(0x0040001040000000), + CONST64(0x0040000000400000), CONST64(0x0040001000400000), CONST64(0x0040000040400000), CONST64(0x0040001040400000), + CONST64(0x0040000000004000), CONST64(0x0040001000004000), CONST64(0x0040000040004000), CONST64(0x0040001040004000), + CONST64(0x0040000000404000), CONST64(0x0040001000404000), CONST64(0x0040000040404000), CONST64(0x0040001040404000), + CONST64(0x0040000000000040), CONST64(0x0040001000000040), CONST64(0x0040000040000040), CONST64(0x0040001040000040), + CONST64(0x0040000000400040), CONST64(0x0040001000400040), CONST64(0x0040000040400040), CONST64(0x0040001040400040), + CONST64(0x0040000000004040), CONST64(0x0040001000004040), CONST64(0x0040000040004040), CONST64(0x0040001040004040), + CONST64(0x0040000000404040), CONST64(0x0040001000404040), CONST64(0x0040000040404040), CONST64(0x0040001040404040), + CONST64(0x4040000000000000), CONST64(0x4040001000000000), CONST64(0x4040000040000000), CONST64(0x4040001040000000), + CONST64(0x4040000000400000), CONST64(0x4040001000400000), CONST64(0x4040000040400000), CONST64(0x4040001040400000), + CONST64(0x4040000000004000), CONST64(0x4040001000004000), CONST64(0x4040000040004000), CONST64(0x4040001040004000), + CONST64(0x4040000000404000), CONST64(0x4040001000404000), CONST64(0x4040000040404000), CONST64(0x4040001040404000), + CONST64(0x4040000000000040), CONST64(0x4040001000000040), CONST64(0x4040000040000040), CONST64(0x4040001040000040), + CONST64(0x4040000000400040), CONST64(0x4040001000400040), CONST64(0x4040000040400040), CONST64(0x4040001040400040), + CONST64(0x4040000000004040), CONST64(0x4040001000004040), CONST64(0x4040000040004040), CONST64(0x4040001040004040), + CONST64(0x4040000000404040), CONST64(0x4040001000404040), CONST64(0x4040000040404040), CONST64(0x4040001040404040), + CONST64(0x0000400000000000), CONST64(0x0000401000000000), CONST64(0x0000400040000000), CONST64(0x0000401040000000), + CONST64(0x0000400000400000), CONST64(0x0000401000400000), CONST64(0x0000400040400000), CONST64(0x0000401040400000), + CONST64(0x0000400000004000), CONST64(0x0000401000004000), CONST64(0x0000400040004000), CONST64(0x0000401040004000), + CONST64(0x0000400000404000), CONST64(0x0000401000404000), CONST64(0x0000400040404000), CONST64(0x0000401040404000), + CONST64(0x0000400000000040), CONST64(0x0000401000000040), CONST64(0x0000400040000040), CONST64(0x0000401040000040), + CONST64(0x0000400000400040), CONST64(0x0000401000400040), CONST64(0x0000400040400040), CONST64(0x0000401040400040), + CONST64(0x0000400000004040), CONST64(0x0000401000004040), CONST64(0x0000400040004040), CONST64(0x0000401040004040), + CONST64(0x0000400000404040), CONST64(0x0000401000404040), CONST64(0x0000400040404040), CONST64(0x0000401040404040), + CONST64(0x4000400000000000), CONST64(0x4000401000000000), CONST64(0x4000400040000000), CONST64(0x4000401040000000), + CONST64(0x4000400000400000), CONST64(0x4000401000400000), CONST64(0x4000400040400000), CONST64(0x4000401040400000), + CONST64(0x4000400000004000), CONST64(0x4000401000004000), CONST64(0x4000400040004000), CONST64(0x4000401040004000), + CONST64(0x4000400000404000), CONST64(0x4000401000404000), CONST64(0x4000400040404000), CONST64(0x4000401040404000), + CONST64(0x4000400000000040), CONST64(0x4000401000000040), CONST64(0x4000400040000040), CONST64(0x4000401040000040), + CONST64(0x4000400000400040), CONST64(0x4000401000400040), CONST64(0x4000400040400040), CONST64(0x4000401040400040), + CONST64(0x4000400000004040), CONST64(0x4000401000004040), CONST64(0x4000400040004040), CONST64(0x4000401040004040), + CONST64(0x4000400000404040), CONST64(0x4000401000404040), CONST64(0x4000400040404040), CONST64(0x4000401040404040), + CONST64(0x0040400000000000), CONST64(0x0040401000000000), CONST64(0x0040400040000000), CONST64(0x0040401040000000), + CONST64(0x0040400000400000), CONST64(0x0040401000400000), CONST64(0x0040400040400000), CONST64(0x0040401040400000), + CONST64(0x0040400000004000), CONST64(0x0040401000004000), CONST64(0x0040400040004000), CONST64(0x0040401040004000), + CONST64(0x0040400000404000), CONST64(0x0040401000404000), CONST64(0x0040400040404000), CONST64(0x0040401040404000), + CONST64(0x0040400000000040), CONST64(0x0040401000000040), CONST64(0x0040400040000040), CONST64(0x0040401040000040), + CONST64(0x0040400000400040), CONST64(0x0040401000400040), CONST64(0x0040400040400040), CONST64(0x0040401040400040), + CONST64(0x0040400000004040), CONST64(0x0040401000004040), CONST64(0x0040400040004040), CONST64(0x0040401040004040), + CONST64(0x0040400000404040), CONST64(0x0040401000404040), CONST64(0x0040400040404040), CONST64(0x0040401040404040), + CONST64(0x4040400000000000), CONST64(0x4040401000000000), CONST64(0x4040400040000000), CONST64(0x4040401040000000), + CONST64(0x4040400000400000), CONST64(0x4040401000400000), CONST64(0x4040400040400000), CONST64(0x4040401040400000), + CONST64(0x4040400000004000), CONST64(0x4040401000004000), CONST64(0x4040400040004000), CONST64(0x4040401040004000), + CONST64(0x4040400000404000), CONST64(0x4040401000404000), CONST64(0x4040400040404000), CONST64(0x4040401040404000), + CONST64(0x4040400000000040), CONST64(0x4040401000000040), CONST64(0x4040400040000040), CONST64(0x4040401040000040), + CONST64(0x4040400000400040), CONST64(0x4040401000400040), CONST64(0x4040400040400040), CONST64(0x4040401040400040), + CONST64(0x4040400000004040), CONST64(0x4040401000004040), CONST64(0x4040400040004040), CONST64(0x4040401040004040), + CONST64(0x4040400000404040), CONST64(0x4040401000404040), CONST64(0x4040400040404040), CONST64(0x4040401040404040) + }}; + +#endif + + +static void cookey(const ulong32 *raw1, ulong32 *keyout); + +#ifdef LTC_CLEAN_STACK +static void s_deskey(const unsigned char *key, short edf, ulong32 *keyout) +#else +static void deskey(const unsigned char *key, short edf, ulong32 *keyout) +#endif +{ + ulong32 i, j, l, m, n, kn[32]; + unsigned char pc1m[56], pcr[56]; + + for (j=0; j < 56; j++) { + l = (ulong32)pc1[j]; + m = l & 7; + pc1m[j] = (unsigned char)((key[l >> 3U] & bytebit[m]) == bytebit[m] ? 1 : 0); + } + + for (i=0; i < 16; i++) { + if (edf == DE1) { + m = (15 - i) << 1; + } else { + m = i << 1; + } + n = m + 1; + kn[m] = kn[n] = 0L; + for (j=0; j < 28; j++) { + l = j + (ulong32)totrot[i]; + if (l < 28) { + pcr[j] = pc1m[l]; + } else { + pcr[j] = pc1m[l - 28]; + } + } + for (/*j = 28*/; j < 56; j++) { + l = j + (ulong32)totrot[i]; + if (l < 56) { + pcr[j] = pc1m[l]; + } else { + pcr[j] = pc1m[l - 28]; + } + } + for (j=0; j < 24; j++) { + if ((int)pcr[(int)pc2[j]] != 0) { + kn[m] |= bigbyte[j]; + } + if ((int)pcr[(int)pc2[j+24]] != 0) { + kn[n] |= bigbyte[j]; + } + } + } + + cookey(kn, keyout); +} + +#ifdef LTC_CLEAN_STACK +static void deskey(const unsigned char *key, short edf, ulong32 *keyout) +{ + s_deskey(key, edf, keyout); + burn_stack(sizeof(int)*5 + sizeof(ulong32)*32 + sizeof(unsigned char)*112); +} +#endif + +#ifdef LTC_CLEAN_STACK +static void s_cookey(const ulong32 *raw1, ulong32 *keyout) +#else +static void cookey(const ulong32 *raw1, ulong32 *keyout) +#endif +{ + ulong32 *cook; + const ulong32 *raw0; + ulong32 dough[32]; + int i; + + cook = dough; + for(i=0; i < 16; i++, raw1++) + { + raw0 = raw1++; + *cook = (*raw0 & 0x00fc0000L) << 6; + *cook |= (*raw0 & 0x00000fc0L) << 10; + *cook |= (*raw1 & 0x00fc0000L) >> 10; + *cook++ |= (*raw1 & 0x00000fc0L) >> 6; + *cook = (*raw0 & 0x0003f000L) << 12; + *cook |= (*raw0 & 0x0000003fL) << 16; + *cook |= (*raw1 & 0x0003f000L) >> 4; + *cook++ |= (*raw1 & 0x0000003fL); + } + + XMEMCPY(keyout, dough, sizeof(dough)); +} + +#ifdef LTC_CLEAN_STACK +static void cookey(const ulong32 *raw1, ulong32 *keyout) +{ + s_cookey(raw1, keyout); + burn_stack(sizeof(ulong32 *) * 2 + sizeof(ulong32)*32 + sizeof(int)); +} +#endif + +#ifndef LTC_CLEAN_STACK +static void desfunc(ulong32 *block, const ulong32 *keys) +#else +static void s_desfunc(ulong32 *block, const ulong32 *keys) +#endif +{ + ulong32 work, right, leftt; + int cur_round; + + leftt = block[0]; + right = block[1]; + +#ifdef LTC_SMALL_CODE + work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL; + right ^= work; + leftt ^= (work << 4); + + work = ((leftt >> 16) ^ right) & 0x0000ffffL; + right ^= work; + leftt ^= (work << 16); + + work = ((right >> 2) ^ leftt) & 0x33333333L; + leftt ^= work; + right ^= (work << 2); + + work = ((right >> 8) ^ leftt) & 0x00ff00ffL; + leftt ^= work; + right ^= (work << 8); + + right = ROLc(right, 1); + work = (leftt ^ right) & 0xaaaaaaaaL; + + leftt ^= work; + right ^= work; + leftt = ROLc(leftt, 1); +#else + { + ulong64 tmp; + tmp = des_ip[0][LTC_BYTE(leftt, 0)] ^ + des_ip[1][LTC_BYTE(leftt, 1)] ^ + des_ip[2][LTC_BYTE(leftt, 2)] ^ + des_ip[3][LTC_BYTE(leftt, 3)] ^ + des_ip[4][LTC_BYTE(right, 0)] ^ + des_ip[5][LTC_BYTE(right, 1)] ^ + des_ip[6][LTC_BYTE(right, 2)] ^ + des_ip[7][LTC_BYTE(right, 3)]; + leftt = (ulong32)(tmp >> 32); + right = (ulong32)(tmp & 0xFFFFFFFFUL); + } +#endif + + for (cur_round = 0; cur_round < 8; cur_round++) { + work = RORc(right, 4) ^ *keys++; + leftt ^= SP7[work & 0x3fL] + ^ SP5[(work >> 8) & 0x3fL] + ^ SP3[(work >> 16) & 0x3fL] + ^ SP1[(work >> 24) & 0x3fL]; + work = right ^ *keys++; + leftt ^= SP8[ work & 0x3fL] + ^ SP6[(work >> 8) & 0x3fL] + ^ SP4[(work >> 16) & 0x3fL] + ^ SP2[(work >> 24) & 0x3fL]; + + work = RORc(leftt, 4) ^ *keys++; + right ^= SP7[ work & 0x3fL] + ^ SP5[(work >> 8) & 0x3fL] + ^ SP3[(work >> 16) & 0x3fL] + ^ SP1[(work >> 24) & 0x3fL]; + work = leftt ^ *keys++; + right ^= SP8[ work & 0x3fL] + ^ SP6[(work >> 8) & 0x3fL] + ^ SP4[(work >> 16) & 0x3fL] + ^ SP2[(work >> 24) & 0x3fL]; + } + +#ifdef LTC_SMALL_CODE + right = RORc(right, 1); + work = (leftt ^ right) & 0xaaaaaaaaL; + leftt ^= work; + right ^= work; + leftt = RORc(leftt, 1); + work = ((leftt >> 8) ^ right) & 0x00ff00ffL; + right ^= work; + leftt ^= (work << 8); + /* -- */ + work = ((leftt >> 2) ^ right) & 0x33333333L; + right ^= work; + leftt ^= (work << 2); + work = ((right >> 16) ^ leftt) & 0x0000ffffL; + leftt ^= work; + right ^= (work << 16); + work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL; + leftt ^= work; + right ^= (work << 4); +#else + { + ulong64 tmp; + tmp = des_fp[0][LTC_BYTE(leftt, 0)] ^ + des_fp[1][LTC_BYTE(leftt, 1)] ^ + des_fp[2][LTC_BYTE(leftt, 2)] ^ + des_fp[3][LTC_BYTE(leftt, 3)] ^ + des_fp[4][LTC_BYTE(right, 0)] ^ + des_fp[5][LTC_BYTE(right, 1)] ^ + des_fp[6][LTC_BYTE(right, 2)] ^ + des_fp[7][LTC_BYTE(right, 3)]; + leftt = (ulong32)(tmp >> 32); + right = (ulong32)(tmp & 0xFFFFFFFFUL); + } +#endif + + block[0] = right; + block[1] = leftt; +} + +#ifdef LTC_CLEAN_STACK +static void desfunc(ulong32 *block, const ulong32 *keys) +{ + s_desfunc(block, keys); + burn_stack(sizeof(ulong32) * 4 + sizeof(int)); +} +#endif + + /** + Initialize the LTC_DES block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int des_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 0 && num_rounds != 16) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen != 8) { + return CRYPT_INVALID_KEYSIZE; + } + + deskey(key, EN0, skey->des.ek); + deskey(key, DE1, skey->des.dk); + + return CRYPT_OK; +} + + /** + Initialize the 3LTC_DES-EDE block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int des3_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if(num_rounds != 0 && num_rounds != 16) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen != 24 && keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + deskey(key, EN0, skey->des3.ek[0]); + deskey(key+8, DE1, skey->des3.ek[1]); + if (keylen == 24) { + deskey(key+16, EN0, skey->des3.ek[2]); + } else { + /* two-key 3DES: K3=K1 */ + deskey(key, EN0, skey->des3.ek[2]); + } + + deskey(key, DE1, skey->des3.dk[2]); + deskey(key+8, EN0, skey->des3.dk[1]); + if (keylen == 24) { + deskey(key+16, DE1, skey->des3.dk[0]); + } else { + /* two-key 3DES: K3=K1 */ + deskey(key, DE1, skey->des3.dk[0]); + } + + return CRYPT_OK; +} + +/** + Encrypts a block of text with LTC_DES + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int des_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong32 work[2]; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + LOAD32H(work[0], pt+0); + LOAD32H(work[1], pt+4); + desfunc(work, skey->des.ek); + STORE32H(work[0],ct+0); + STORE32H(work[1],ct+4); + return CRYPT_OK; +} + +/** + Decrypts a block of text with LTC_DES + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int des_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong32 work[2]; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + LOAD32H(work[0], ct+0); + LOAD32H(work[1], ct+4); + desfunc(work, skey->des.dk); + STORE32H(work[0],pt+0); + STORE32H(work[1],pt+4); + return CRYPT_OK; +} + +/** + Encrypts a block of text with 3LTC_DES-EDE + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int des3_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong32 work[2]; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + LOAD32H(work[0], pt+0); + LOAD32H(work[1], pt+4); + desfunc(work, skey->des3.ek[0]); + desfunc(work, skey->des3.ek[1]); + desfunc(work, skey->des3.ek[2]); + STORE32H(work[0],ct+0); + STORE32H(work[1],ct+4); + return CRYPT_OK; +} + +/** + Decrypts a block of text with 3LTC_DES-EDE + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int des3_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong32 work[2]; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + LOAD32H(work[0], ct+0); + LOAD32H(work[1], ct+4); + desfunc(work, skey->des3.dk[0]); + desfunc(work, skey->des3.dk[1]); + desfunc(work, skey->des3.dk[2]); + STORE32H(work[0],pt+0); + STORE32H(work[1],pt+4); + return CRYPT_OK; +} + +/** + Performs a self-test of the LTC_DES block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int des_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct des_test_case { + unsigned char key[8], txt[8], out[8]; + } cases[] = { + { { 0x10, 0x31, 0x6E, 0x02, 0x8C, 0x8F, 0x3B, 0x4A }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x82, 0xDC, 0xBA, 0xFB, 0xDE, 0xAB, 0x66, 0x02 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x95, 0xF8, 0xA5, 0xE5, 0xDD, 0x31, 0xD9, 0x00 }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0xDD, 0x7F, 0x12, 0x1C, 0xA5, 0x01, 0x56, 0x19 }, + { 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x2E, 0x86, 0x53, 0x10, 0x4F, 0x38, 0x34, 0xEA }, + { 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x4B, 0xD3, 0x88, 0xFF, 0x6C, 0xD8, 0x1D, 0x4F }, + { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x20, 0xB9, 0xE7, 0x67, 0xB2, 0xFB, 0x14, 0x56 }, + { 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x55, 0x57, 0x93, 0x80, 0xD7, 0x71, 0x38, 0xEF }, + { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x6C, 0xC5, 0xDE, 0xFA, 0xAF, 0x04, 0x51, 0x2F }, + { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x0D, 0x9F, 0x27, 0x9B, 0xA5, 0xD8, 0x72, 0x60 }, + { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0xD9, 0x03, 0x1B, 0x02, 0x71, 0xBD, 0x5A, 0x0A }, + { 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + + { { 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x95, 0xA8, 0xD7, 0x28, 0x13, 0xDA, 0xA9, 0x4D } }, + { { 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x0E, 0xEC, 0x14, 0x87, 0xDD, 0x8C, 0x26, 0xD5 } }, + { { 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x7A, 0xD1, 0x6F, 0xFB, 0x79, 0xC4, 0x59, 0x26 } }, + { { 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xD3, 0x74, 0x62, 0x94, 0xCA, 0x6A, 0x6C, 0xF3 } }, + { { 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x80, 0x9F, 0x5F, 0x87, 0x3C, 0x1F, 0xD7, 0x61 } }, + { { 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xC0, 0x2F, 0xAF, 0xFE, 0xC9, 0x89, 0xD1, 0xFC } }, + { { 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x46, 0x15, 0xAA, 0x1D, 0x33, 0xE7, 0x2F, 0x10 } }, + { { 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x20, 0x55, 0x12, 0x33, 0x50, 0xC0, 0x08, 0x58 } }, + { { 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xDF, 0x3B, 0x99, 0xD6, 0x57, 0x73, 0x97, 0xC8 } }, + { { 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x31, 0xFE, 0x17, 0x36, 0x9B, 0x52, 0x88, 0xC9 } }, + { { 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xDF, 0xDD, 0x3C, 0xC6, 0x4D, 0xAE, 0x16, 0x42 } }, + { { 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x17, 0x8C, 0x83, 0xCE, 0x2B, 0x39, 0x9D, 0x94 } }, + { { 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x50, 0xF6, 0x36, 0x32, 0x4A, 0x9B, 0x7F, 0x80 } }, + { { 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xA8, 0x46, 0x8E, 0xE3, 0xBC, 0x18, 0xF0, 0x6D } }, + { { 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xA2, 0xDC, 0x9E, 0x92, 0xFD, 0x3C, 0xDE, 0x92 } }, + { { 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xCA, 0xC0, 0x9F, 0x79, 0x7D, 0x03, 0x12, 0x87 } }, + { { 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x90, 0xBA, 0x68, 0x0B, 0x22, 0xAE, 0xB5, 0x25 } }, + { { 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xCE, 0x7A, 0x24, 0xF3, 0x50, 0xE2, 0x80, 0xB6 } }, + { { 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x88, 0x2B, 0xFF, 0x0A, 0xA0, 0x1A, 0x0B, 0x87 } }, + { { 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x25, 0x61, 0x02, 0x88, 0x92, 0x45, 0x11, 0xC2 } }, + { { 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xC7, 0x15, 0x16, 0xC2, 0x9C, 0x75, 0xD1, 0x70 } }, + { { 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x51, 0x99, 0xC2, 0x9A, 0x52, 0xC9, 0xF0, 0x59 } }, + { { 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xC2, 0x2F, 0x0A, 0x29, 0x4A, 0x71, 0xF2, 0x9F } }, + { { 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xEE, 0x37, 0x14, 0x83, 0x71, 0x4C, 0x02, 0xEA } }, + { { 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xA8, 0x1F, 0xBD, 0x44, 0x8F, 0x9E, 0x52, 0x2F } }, + { { 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x4F, 0x64, 0x4C, 0x92, 0xE1, 0x92, 0xDF, 0xED } }, + { { 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x1A, 0xFA, 0x9A, 0x66, 0xA6, 0xDF, 0x92, 0xAE } }, + { { 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xB3, 0xC1, 0xCC, 0x71, 0x5C, 0xB8, 0x79, 0xD8 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x19, 0xD0, 0x32, 0xE6, 0x4A, 0xB0, 0xBD, 0x8B } }, + { { 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x3C, 0xFA, 0xA7, 0xA7, 0xDC, 0x87, 0x20, 0xDC } }, + { { 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xB7, 0x26, 0x5F, 0x7F, 0x44, 0x7A, 0xC6, 0xF3 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x9D, 0xB7, 0x3B, 0x3C, 0x0D, 0x16, 0x3F, 0x54 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x81, 0x81, 0xB6, 0x5B, 0xAB, 0xF4, 0xA9, 0x75 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x93, 0xC9, 0xB6, 0x40, 0x42, 0xEA, 0xA2, 0x40 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x55, 0x70, 0x53, 0x08, 0x29, 0x70, 0x55, 0x92 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x86, 0x38, 0x80, 0x9E, 0x87, 0x87, 0x87, 0xA0 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x41, 0xB9, 0xA7, 0x9A, 0xF7, 0x9A, 0xC2, 0x08 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x7A, 0x9B, 0xE4, 0x2F, 0x20, 0x09, 0xA8, 0x92 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x29, 0x03, 0x8D, 0x56, 0xBA, 0x6D, 0x27, 0x45 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x54, 0x95, 0xC6, 0xAB, 0xF1, 0xE5, 0xDF, 0x51 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xAE, 0x13, 0xDB, 0xD5, 0x61, 0x48, 0x89, 0x33 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x02, 0x4D, 0x1F, 0xFA, 0x89, 0x04, 0xE3, 0x89 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xD1, 0x39, 0x97, 0x12, 0xF9, 0x9B, 0xF0, 0x2E } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x14, 0xC1, 0xD7, 0xC1, 0xCF, 0xFE, 0xC7, 0x9E } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x1D, 0xE5, 0x27, 0x9D, 0xAE, 0x3B, 0xED, 0x6F } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xE9, 0x41, 0xA3, 0x3F, 0x85, 0x50, 0x13, 0x03 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xDA, 0x99, 0xDB, 0xBC, 0x9A, 0x03, 0xF3, 0x79 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xB7, 0xFC, 0x92, 0xF9, 0x1D, 0x8E, 0x92, 0xE9 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xAE, 0x8E, 0x5C, 0xAA, 0x3C, 0xA0, 0x4E, 0x85 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x9C, 0xC6, 0x2D, 0xF4, 0x3B, 0x6E, 0xED, 0x74 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xD8, 0x63, 0xDB, 0xB5, 0xC5, 0x9A, 0x91, 0xA0 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xA1, 0xAB, 0x21, 0x90, 0x54, 0x5B, 0x91, 0xD7 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x08, 0x75, 0x04, 0x1E, 0x64, 0xC5, 0x70, 0xF7 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x5A, 0x59, 0x45, 0x28, 0xBE, 0xBE, 0xF1, 0xCC } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xFC, 0xDB, 0x32, 0x91, 0xDE, 0x21, 0xF0, 0xC0 } }, + { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x86, 0x9E, 0xFD, 0x7F, 0x9F, 0x26, 0x5A, 0x09 } }, + + /*** more test cases you could add if you are not convinced (the above test cases aren't really too good): + + key plaintext ciphertext + 0000000000000000 0000000000000000 8CA64DE9C1B123A7 + FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF 7359B2163E4EDC58 + 3000000000000000 1000000000000001 958E6E627A05557B + 1111111111111111 1111111111111111 F40379AB9E0EC533 + 0123456789ABCDEF 1111111111111111 17668DFC7292532D + 1111111111111111 0123456789ABCDEF 8A5AE1F81AB8F2DD + 0000000000000000 0000000000000000 8CA64DE9C1B123A7 + FEDCBA9876543210 0123456789ABCDEF ED39D950FA74BCC4 + 7CA110454A1A6E57 01A1D6D039776742 690F5B0D9A26939B + 0131D9619DC1376E 5CD54CA83DEF57DA 7A389D10354BD271 + 07A1133E4A0B2686 0248D43806F67172 868EBB51CAB4599A + 3849674C2602319E 51454B582DDF440A 7178876E01F19B2A + 04B915BA43FEB5B6 42FD443059577FA2 AF37FB421F8C4095 + 0113B970FD34F2CE 059B5E0851CF143A 86A560F10EC6D85B + 0170F175468FB5E6 0756D8E0774761D2 0CD3DA020021DC09 + 43297FAD38E373FE 762514B829BF486A EA676B2CB7DB2B7A + 07A7137045DA2A16 3BDD119049372802 DFD64A815CAF1A0F + 04689104C2FD3B2F 26955F6835AF609A 5C513C9C4886C088 + 37D06BB516CB7546 164D5E404F275232 0A2AEEAE3FF4AB77 + 1F08260D1AC2465E 6B056E18759F5CCA EF1BF03E5DFA575A + 584023641ABA6176 004BD6EF09176062 88BF0DB6D70DEE56 + 025816164629B007 480D39006EE762F2 A1F9915541020B56 + 49793EBC79B3258F 437540C8698F3CFA 6FBF1CAFCFFD0556 + 4FB05E1515AB73A7 072D43A077075292 2F22E49BAB7CA1AC + 49E95D6D4CA229BF 02FE55778117F12A 5A6B612CC26CCE4A + 018310DC409B26D6 1D9D5C5018F728C2 5F4C038ED12B2E41 + 1C587F1C13924FEF 305532286D6F295A 63FAC0D034D9F793 + 0101010101010101 0123456789ABCDEF 617B3A0CE8F07100 + 1F1F1F1F0E0E0E0E 0123456789ABCDEF DB958605F8C8C606 + E0FEE0FEF1FEF1FE 0123456789ABCDEF EDBFD1C66C29CCC7 + 0000000000000000 FFFFFFFFFFFFFFFF 355550B2150E2451 + FFFFFFFFFFFFFFFF 0000000000000000 CAAAAF4DEAF1DBAE + 0123456789ABCDEF 0000000000000000 D5D44FF720683D0D + FEDCBA9876543210 FFFFFFFFFFFFFFFF 2A2BB008DF97C2F2 + + http://www.ecs.soton.ac.uk/~prw99r/ez438/vectors.txt + ***/ + }; + unsigned char key[8], pt[8], ct[8], tmp[8]; + symmetric_key skey; + int i, err; + + for (i = 0; i < (int)(sizeof(cases)/sizeof(cases[0])); i++) + { + if ((err = des_setup(cases[i].key, 8, 0, &skey)) != CRYPT_OK) { + return err; + } + + des_ecb_encrypt(cases[i].txt, ct, &skey); + + if (compare_testvector(ct, sizeof(ct), cases[i].out, 8, "DES Encrypt", i) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + des_ecb_decrypt(ct, pt, &skey); + + if (compare_testvector(pt, sizeof(pt), cases[i].txt, 8, "DES Decrypt", i) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + /* See if we can encrypt all zero bytes 1000 times, decrypt and come back to where we started */ + + for (i = 0; i < 8; i++) key[i] = i; + + if ((err = des_setup(key, 8, 0, &skey)) != CRYPT_OK) { + return err; + } + + for (i = 0; i < 8; i++) pt[i] = tmp[i] = 0; + for (i = 0; i < 1000; i++) des_ecb_encrypt(tmp, tmp, &skey); + for (i = 0; i < 1000; i++) des_ecb_decrypt(tmp, tmp, &skey); + + if (compare_testvector(tmp, 8, pt, 8, "DES", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; + #endif +} + +int des3_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct des3_test_case { + unsigned char key[16], txt[8], out[8]; + } cases[] = { + /* + https://www.cosic.esat.kuleuven.be/nessie/testvectors/bc/des/Triple-Des-2-Key-128-64.unverified.test-vectors + */ + { { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xFA, 0xFD, 0x50, 0x84, 0x37, 0x4F, 0xCE, 0x34 } }, + { { 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x60, 0xCC, 0x37, 0xB7, 0xB5, 0x37, 0xA1, 0xDC } }, + { { 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xBE, 0x3E, 0x73, 0x04, 0xFE, 0x92, 0xC2, 0xBC } }, + { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00 }, + { 0xE5, 0xA9, 0xE3, 0x80, 0x03, 0xA5, 0xA0, 0xFD }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }, + { 0xE4, 0xFC, 0x19, 0xD6, 0x94, 0x63, 0xB7, 0x83 }, + { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77 } }, + }; + unsigned char key[24], pt[8], ct[8], tmp[8]; + symmetric_key skey; + int i, err; + + if ((err = des_test()) != CRYPT_OK) { + return err; + } + + for (i = 0; i < (int)(sizeof(cases)/sizeof(cases[0])); i++) + { + if ((err = des3_setup(cases[i].key, 16, 0, &skey)) != CRYPT_OK) { + return err; + } + + des3_ecb_encrypt(cases[i].txt, ct, &skey); + + if (compare_testvector(ct, sizeof(ct), cases[i].out, 8, "3DES Encrypt", i) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + des3_ecb_decrypt(ct, pt, &skey); + + if (compare_testvector(pt, sizeof(pt), cases[i].txt, 8, "3DES Decrypt", i) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + /* See if we can encrypt all zero bytes 1000 times, decrypt and come back to where we started */ + + for (i = 0; i < 24; i++) key[i] = i; + + if ((err = des3_setup(key, 24, 0, &skey)) != CRYPT_OK) { + return err; + } + + for (i = 0; i < 8; i++) pt[i] = tmp[i] = 0; + for (i = 0; i < 1000; i++) des3_ecb_encrypt(tmp, tmp, &skey); + for (i = 0; i < 1000; i++) des3_ecb_decrypt(tmp, tmp, &skey); + + if (compare_testvector(tmp, 8, pt, 8, "3DES", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void des_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** Terminate the context + @param skey The scheduled key +*/ +void des3_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int des_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if(*keysize < 8) { + return CRYPT_INVALID_KEYSIZE; + } + *keysize = 8; + return CRYPT_OK; +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int des3_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize < 24) { + *keysize = 16; + return CRYPT_OK; + } + *keysize = 24; + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/idea.c b/optee_os/core/lib/libtomcrypt/src/ciphers/idea.c new file mode 100644 index 0000000..b195a4c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/idea.c @@ -0,0 +1,250 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* Based on idea.cpp - originally written and placed in the public domain by Wei Dai + https://github.com/weidai11/cryptopp/blob/master/idea.cpp + + Patents should be expired. On 2017-10-16 wikipedia says: + https://en.wikipedia.org/wiki/International_Data_Encryption_Algorithm + + A patent application for IDEA was first filed in Switzerland (CH A 1690/90) on May 18, 1990, + then an international patent application was filed under the Patent Cooperation Treaty on + May 16, 1991. Patents were eventually granted in Austria, France, Germany, Italy, the Netherlands, + Spain, Sweden, Switzerland, the United Kingdom, (European Patent Register entry for European + patent no. 0482154, filed May 16, 1991, issued June 22, 1994 and expired May 16, 2011), + the United States (U.S. Patent 5,214,703, issued May 25, 1993 and expired January 7, 2012) + and Japan (JP 3225440) (expired May 16, 2011). + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_IDEA + +const struct ltc_cipher_descriptor idea_desc = { + "idea", + 24, /* cipher_ID */ + 16, 16, 8, 8, /* min_key_len, max_key_len, block_len, default_rounds */ + &idea_setup, + &idea_ecb_encrypt, + &idea_ecb_decrypt, + &idea_test, + &idea_done, + &idea_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +typedef unsigned short int ushort16; + +#define LOW16(x) ((x)&0xffff) /* compiler should be able to optimize this away if x is 16 bits */ +#define HIGH16(x) ((x)>>16) +#define MUL(a,b) { \ + ulong32 p = (ulong32)LOW16(a) * b; \ + if (p) { \ + p = LOW16(p) - HIGH16(p); \ + a = (ushort16)p - (ushort16)HIGH16(p); \ + } \ + else \ + a = 1 - a - b; \ + } +#define STORE16(x,y) { (y)[0] = (unsigned char)(((x)>>8)&255); (y)[1] = (unsigned char)((x)&255); } +#define LOAD16(x,y) { x = ((ushort16)((y)[0] & 255)<<8) | ((ushort16)((y)[1] & 255)); } + +static ushort16 s_mul_inv(ushort16 x) +{ + ushort16 y = x; + unsigned i; + + for (i = 0; i < 15; i++) { + MUL(y, LOW16(y)); + MUL(y, x); + } + return LOW16(y); +} + +static ushort16 s_add_inv(ushort16 x) +{ + return LOW16(0 - x); +} + +static int s_setup_key(const unsigned char *key, symmetric_key *skey) +{ + int i, j; + ushort16 *e_key = skey->idea.ek; + ushort16 *d_key = skey->idea.dk; + + /* prepare enc key */ + for (i = 0; i < 8; i++) { + LOAD16(e_key[i], key + 2 * i); + } + for (; i < LTC_IDEA_KEYLEN; i++) { + j = (i - i % 8) - 8; + e_key[i] = LOW16((e_key[j+(i+1)%8] << 9) | (e_key[j+(i+2)%8] >> 7)); + } + + /* prepare dec key */ + for (i = 0; i < LTC_IDEA_ROUNDS; i++) { + d_key[i*6+0] = s_mul_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+0]); + d_key[i*6+1] = s_add_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+1+(i>0 ? 1 : 0)]); + d_key[i*6+2] = s_add_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+2-(i>0 ? 1 : 0)]); + d_key[i*6+3] = s_mul_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+3]); + d_key[i*6+4] = e_key[(LTC_IDEA_ROUNDS-1-i)*6+4]; + d_key[i*6+5] = e_key[(LTC_IDEA_ROUNDS-1-i)*6+5]; + } + d_key[i*6+0] = s_mul_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+0]); + d_key[i*6+1] = s_add_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+1]); + d_key[i*6+2] = s_add_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+2]); + d_key[i*6+3] = s_mul_inv(e_key[(LTC_IDEA_ROUNDS-i)*6+3]); + + return CRYPT_OK; +} + +static int s_process_block(const unsigned char *in, unsigned char *out, const ushort16 *m_key) +{ + int i; + ushort16 x0, x1, x2, x3, t0, t1; + + LOAD16(x0, in + 0); + LOAD16(x1, in + 2); + LOAD16(x2, in + 4); + LOAD16(x3, in + 6); + + for (i = 0; i < LTC_IDEA_ROUNDS; i++) { + MUL(x0, m_key[i*6+0]); + x1 += m_key[i*6+1]; + x2 += m_key[i*6+2]; + MUL(x3, m_key[i*6+3]); + t0 = x0^x2; + MUL(t0, m_key[i*6+4]); + t1 = t0 + (x1^x3); + MUL(t1, m_key[i*6+5]); + t0 += t1; + x0 ^= t1; + x3 ^= t0; + t0 ^= x1; + x1 = x2^t1; + x2 = t0; + } + + MUL(x0, m_key[LTC_IDEA_ROUNDS*6+0]); + x2 += m_key[LTC_IDEA_ROUNDS*6+1]; + x1 += m_key[LTC_IDEA_ROUNDS*6+2]; + MUL(x3, m_key[LTC_IDEA_ROUNDS*6+3]); + + STORE16(x0, out + 0); + STORE16(x2, out + 2); + STORE16(x1, out + 4); + STORE16(x3, out + 6); + + return CRYPT_OK; +} + +int idea_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 0 && num_rounds != 8) return CRYPT_INVALID_ROUNDS; + if (keylen != 16) return CRYPT_INVALID_KEYSIZE; + + return s_setup_key(key, skey); +} + +int idea_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_process_block(pt, ct, skey->idea.ek); +#ifdef LTC_CLEAN_STACK + burn_stack(sizeof(ushort16) * 6 + sizeof(int)); +#endif + return err; +} + +int idea_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_process_block(ct, pt, skey->idea.dk); +#ifdef LTC_CLEAN_STACK + burn_stack(sizeof(ushort16) * 6 + sizeof(int)); +#endif + return err; +} + +void idea_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +int idea_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + *keysize = 16; + return CRYPT_OK; +} + +int idea_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char key[16], pt[8], ct[8]; + } tests[] = { + { + /* key */ { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* pt */ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* ct */ { 0xB1, 0xF5, 0xF7, 0xF8, 0x79, 0x01, 0x37, 0x0F } + }, + { + /* key */ { 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* pt */ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* ct */ { 0xB3, 0x92, 0x7D, 0xFF, 0xB6, 0x35, 0x86, 0x26 } + }, + { + /* key */ { 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* pt */ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* ct */ { 0xE9, 0x87, 0xE0, 0x02, 0x9F, 0xB9, 0x97, 0x85 } + }, + { + /* key */ { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* pt */ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* ct */ { 0x75, 0x4A, 0x03, 0xCE, 0x08, 0xDB, 0x7D, 0xAA } + }, + { + /* key */ { 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* pt */ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + /* ct */ { 0xF0, 0x15, 0xF9, 0xFB, 0x0C, 0xFC, 0x7E, 0x1C } + }, + }; + + unsigned char buf[2][8]; + symmetric_key key; + int err, x; + + if (sizeof(ushort16) != 2) { + return CRYPT_FAIL_TESTVECTOR; + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + if ((err = idea_setup(tests[x].key, 16, 8, &key)) != CRYPT_OK) { + return err; + } + if ((err = idea_ecb_encrypt(tests[x].pt, buf[0], &key)) != CRYPT_OK) { + return err; + } + if (compare_testvector(buf[0], 8, tests[x].ct, 8, "IDEA Encrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + if ((err = idea_ecb_decrypt(tests[x].ct, buf[1], &key)) != CRYPT_OK) { + return err; + } + if (compare_testvector(buf[1], 8, tests[x].pt, 8, "IDEA Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/kasumi.c b/optee_os/core/lib/libtomcrypt/src/ciphers/kasumi.c new file mode 100644 index 0000000..1ade9d6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/kasumi.c @@ -0,0 +1,307 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file kasumi.c + Implementation of the 3GPP Kasumi block cipher + Derived from the 3GPP standard source code +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_KASUMI + +typedef unsigned u16; + +#define ROL16(x, y) ((((x)<<(y)) | ((x)>>(16-(y)))) & 0xFFFF) + +const struct ltc_cipher_descriptor kasumi_desc = { + "kasumi", + 21, + 16, 16, 8, 8, + &kasumi_setup, + &kasumi_ecb_encrypt, + &kasumi_ecb_decrypt, + &kasumi_test, + &kasumi_done, + &kasumi_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static u16 FI( u16 in, u16 subkey ) +{ + u16 nine, seven; + static const u16 S7[128] = { + 54, 50, 62, 56, 22, 34, 94, 96, 38, 6, 63, 93, 2, 18,123, 33, + 55,113, 39,114, 21, 67, 65, 12, 47, 73, 46, 27, 25,111,124, 81, + 53, 9,121, 79, 52, 60, 58, 48,101,127, 40,120,104, 70, 71, 43, + 20,122, 72, 61, 23,109, 13,100, 77, 1, 16, 7, 82, 10,105, 98, + 117,116, 76, 11, 89,106, 0,125,118, 99, 86, 69, 30, 57,126, 87, + 112, 51, 17, 5, 95, 14, 90, 84, 91, 8, 35,103, 32, 97, 28, 66, + 102, 31, 26, 45, 75, 4, 85, 92, 37, 74, 80, 49, 68, 29,115, 44, + 64,107,108, 24,110, 83, 36, 78, 42, 19, 15, 41, 88,119, 59, 3 }; + static const u16 S9[512] = { + 167,239,161,379,391,334, 9,338, 38,226, 48,358,452,385, 90,397, + 183,253,147,331,415,340, 51,362,306,500,262, 82,216,159,356,177, + 175,241,489, 37,206, 17, 0,333, 44,254,378, 58,143,220, 81,400, + 95, 3,315,245, 54,235,218,405,472,264,172,494,371,290,399, 76, + 165,197,395,121,257,480,423,212,240, 28,462,176,406,507,288,223, + 501,407,249,265, 89,186,221,428,164, 74,440,196,458,421,350,163, + 232,158,134,354, 13,250,491,142,191, 69,193,425,152,227,366,135, + 344,300,276,242,437,320,113,278, 11,243, 87,317, 36, 93,496, 27, + 487,446,482, 41, 68,156,457,131,326,403,339, 20, 39,115,442,124, + 475,384,508, 53,112,170,479,151,126,169, 73,268,279,321,168,364, + 363,292, 46,499,393,327,324, 24,456,267,157,460,488,426,309,229, + 439,506,208,271,349,401,434,236, 16,209,359, 52, 56,120,199,277, + 465,416,252,287,246, 6, 83,305,420,345,153,502, 65, 61,244,282, + 173,222,418, 67,386,368,261,101,476,291,195,430, 49, 79,166,330, + 280,383,373,128,382,408,155,495,367,388,274,107,459,417, 62,454, + 132,225,203,316,234, 14,301, 91,503,286,424,211,347,307,140,374, + 35,103,125,427, 19,214,453,146,498,314,444,230,256,329,198,285, + 50,116, 78,410, 10,205,510,171,231, 45,139,467, 29, 86,505, 32, + 72, 26,342,150,313,490,431,238,411,325,149,473, 40,119,174,355, + 185,233,389, 71,448,273,372, 55,110,178,322, 12,469,392,369,190, + 1,109,375,137,181, 88, 75,308,260,484, 98,272,370,275,412,111, + 336,318, 4,504,492,259,304, 77,337,435, 21,357,303,332,483, 18, + 47, 85, 25,497,474,289,100,269,296,478,270,106, 31,104,433, 84, + 414,486,394, 96, 99,154,511,148,413,361,409,255,162,215,302,201, + 266,351,343,144,441,365,108,298,251, 34,182,509,138,210,335,133, + 311,352,328,141,396,346,123,319,450,281,429,228,443,481, 92,404, + 485,422,248,297, 23,213,130,466, 22,217,283, 70,294,360,419,127, + 312,377, 7,468,194, 2,117,295,463,258,224,447,247,187, 80,398, + 284,353,105,390,299,471,470,184, 57,200,348, 63,204,188, 33,451, + 97, 30,310,219, 94,160,129,493, 64,179,263,102,189,207,114,402, + 438,477,387,122,192, 42,381, 5,145,118,180,449,293,323,136,380, + 43, 66, 60,455,341,445,202,432, 8,237, 15,376,436,464, 59,461}; + + /* The sixteen bit input is split into two unequal halves, * + * nine bits and seven bits - as is the subkey */ + + nine = (u16)(in>>7)&0x1FF; + seven = (u16)(in&0x7F); + + /* Now run the various operations */ + nine = (u16)(S9[nine] ^ seven); + seven = (u16)(S7[seven] ^ (nine & 0x7F)); + seven ^= (subkey>>9); + nine ^= (subkey&0x1FF); + nine = (u16)(S9[nine] ^ seven); + seven = (u16)(S7[seven] ^ (nine & 0x7F)); + return (u16)(seven<<9) + nine; +} + +static ulong32 FO( ulong32 in, int round_no, const symmetric_key *key) +{ + u16 left, right; + + /* Split the input into two 16-bit words */ + left = (u16)(in>>16); + right = (u16) in&0xFFFF; + + /* Now apply the same basic transformation three times */ + left ^= key->kasumi.KOi1[round_no]; + left = FI( left, key->kasumi.KIi1[round_no] ); + left ^= right; + + right ^= key->kasumi.KOi2[round_no]; + right = FI( right, key->kasumi.KIi2[round_no] ); + right ^= left; + + left ^= key->kasumi.KOi3[round_no]; + left = FI( left, key->kasumi.KIi3[round_no] ); + left ^= right; + + return (((ulong32)right)<<16)+left; +} + +static ulong32 FL( ulong32 in, int round_no, const symmetric_key *key ) +{ + u16 l, r, a, b; + /* split out the left and right halves */ + l = (u16)(in>>16); + r = (u16)(in)&0xFFFF; + /* do the FL() operations */ + a = (u16) (l & key->kasumi.KLi1[round_no]); + r ^= ROL16(a,1); + b = (u16)(r | key->kasumi.KLi2[round_no]); + l ^= ROL16(b,1); + /* put the two halves back together */ + + return (((ulong32)l)<<16) + r; +} + +int kasumi_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong32 left, right, temp; + int n; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(left, pt); + LOAD32H(right, pt+4); + + for (n = 0; n <= 7; ) { + temp = FL(left, n, skey); + temp = FO(temp, n++, skey); + right ^= temp; + temp = FO(right, n, skey); + temp = FL(temp, n++, skey); + left ^= temp; + } + + STORE32H(left, ct); + STORE32H(right, ct+4); + + return CRYPT_OK; +} + +int kasumi_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong32 left, right, temp; + int n; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(left, ct); + LOAD32H(right, ct+4); + + for (n = 7; n >= 0; ) { + temp = FO(right, n, skey); + temp = FL(temp, n--, skey); + left ^= temp; + temp = FL(left, n, skey); + temp = FO(temp, n--, skey); + right ^= temp; + } + + STORE32H(left, pt); + STORE32H(right, pt+4); + + return CRYPT_OK; +} + +int kasumi_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + static const u16 C[8] = { 0x0123,0x4567,0x89AB,0xCDEF, 0xFEDC,0xBA98,0x7654,0x3210 }; + u16 ukey[8], Kprime[8]; + int n; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 0 && num_rounds != 8) { + return CRYPT_INVALID_ROUNDS; + } + + /* Start by ensuring the subkeys are endian correct on a 16-bit basis */ + for (n = 0; n < 8; n++ ) { + ukey[n] = (((u16)key[2*n]) << 8) | key[2*n+1]; + } + + /* Now build the K'[] keys */ + for (n = 0; n < 8; n++) { + Kprime[n] = ukey[n] ^ C[n]; + } + + /* Finally construct the various sub keys */ + for(n = 0; n < 8; n++) { + skey->kasumi.KLi1[n] = ROL16(ukey[n],1); + skey->kasumi.KLi2[n] = Kprime[(n+2)&0x7]; + skey->kasumi.KOi1[n] = ROL16(ukey[(n+1)&0x7],5); + skey->kasumi.KOi2[n] = ROL16(ukey[(n+5)&0x7],8); + skey->kasumi.KOi3[n] = ROL16(ukey[(n+6)&0x7],13); + skey->kasumi.KIi1[n] = Kprime[(n+4)&0x7]; + skey->kasumi.KIi2[n] = Kprime[(n+3)&0x7]; + skey->kasumi.KIi3[n] = Kprime[(n+7)&0x7]; + } + + return CRYPT_OK; +} + +void kasumi_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +int kasumi_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize >= 16) { + *keysize = 16; + return CRYPT_OK; + } + return CRYPT_INVALID_KEYSIZE; +} + +int kasumi_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char key[16], pt[8], ct[8]; + } tests[] = { + +{ + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x4B, 0x58, 0xA7, 0x71, 0xAF, 0xC7, 0xE5, 0xE8 } +}, + +{ + { 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x7E, 0xEF, 0x11, 0x3C, 0x95, 0xBB, 0x5A, 0x77 } +}, + +{ + { 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x5F, 0x14, 0x06, 0x86, 0xD7, 0xAD, 0x5A, 0x39 }, +}, + +{ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x2E, 0x14, 0x91, 0xCF, 0x70, 0xAA, 0x46, 0x5D } +}, + +{ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xB5, 0x45, 0x86, 0xF4, 0xAB, 0x9A, 0xE5, 0x46 } +}, + +}; + unsigned char buf[2][8]; + symmetric_key key; + int err, x; + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + if ((err = kasumi_setup(tests[x].key, 16, 0, &key)) != CRYPT_OK) { + return err; + } + if ((err = kasumi_ecb_encrypt(tests[x].pt, buf[0], &key)) != CRYPT_OK) { + return err; + } + if ((err = kasumi_ecb_decrypt(tests[x].ct, buf[1], &key)) != CRYPT_OK) { + return err; + } + if (compare_testvector(buf[1], 8, tests[x].pt, 8, "Kasumi Decrypt", x) || + compare_testvector(buf[0], 8, tests[x].ct, 8, "Kasumi Encrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/khazad.c b/optee_os/core/lib/libtomcrypt/src/ciphers/khazad.c new file mode 100644 index 0000000..72cbda8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/khazad.c @@ -0,0 +1,840 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file khazad.c + Khazad implementation derived from public domain source + Authors: Paulo S.L.M. Barreto and Vincent Rijmen. +*/ + +#ifdef LTC_KHAZAD + +const struct ltc_cipher_descriptor khazad_desc = { + "khazad", + 18, + 16, 16, 8, 8, + &khazad_setup, + &khazad_ecb_encrypt, + &khazad_ecb_decrypt, + &khazad_test, + &khazad_done, + &khazad_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +#define R 8 + +static const ulong64 T0[256] = { + CONST64(0xbad3d268bbb96a01), CONST64(0x54fc4d19e59a66b1), CONST64(0x2f71bc93e26514cd), CONST64(0x749ccdb925871b51), + CONST64(0x53f55102f7a257a4), CONST64(0xd3686bb8d0d6be03), CONST64(0xd26b6fbdd6deb504), CONST64(0x4dd72964b35285fe), + CONST64(0x50f05d0dfdba4aad), CONST64(0xace98a26cf09e063), CONST64(0x8d8a0e83091c9684), CONST64(0xbfdcc679a5914d1a), + CONST64(0x7090ddad3da7374d), CONST64(0x52f65507f1aa5ca3), CONST64(0x9ab352c87ba417e1), CONST64(0x4cd42d61b55a8ef9), + CONST64(0xea238f65460320ac), CONST64(0xd56273a6c4e68411), CONST64(0x97a466f155cc68c2), CONST64(0xd16e63b2dcc6a80d), + CONST64(0x3355ccffaa85d099), CONST64(0x51f35908fbb241aa), CONST64(0x5bed712ac7e20f9c), CONST64(0xa6f7a204f359ae55), + CONST64(0xde7f5f81febec120), CONST64(0x48d83d75ad7aa2e5), CONST64(0xa8e59a32d729cc7f), CONST64(0x99b65ec771bc0ae8), + CONST64(0xdb704b90e096e63b), CONST64(0x3256c8faac8ddb9e), CONST64(0xb7c4e65195d11522), CONST64(0xfc19d72b32b3aace), + CONST64(0xe338ab48704b7393), CONST64(0x9ebf42dc63843bfd), CONST64(0x91ae7eef41fc52d0), CONST64(0x9bb056cd7dac1ce6), + CONST64(0xe23baf4d76437894), CONST64(0xbbd0d66dbdb16106), CONST64(0x41c319589b32f1da), CONST64(0x6eb2a5cb7957e517), + CONST64(0xa5f2ae0bf941b35c), CONST64(0xcb400bc08016564b), CONST64(0x6bbdb1da677fc20c), CONST64(0x95a26efb59dc7ecc), + CONST64(0xa1febe1fe1619f40), CONST64(0xf308eb1810cbc3e3), CONST64(0xb1cefe4f81e12f30), CONST64(0x0206080a0c10160e), + CONST64(0xcc4917db922e675e), CONST64(0xc45137f3a26e3f66), CONST64(0x1d2774694ee8cf53), CONST64(0x143c504478a09c6c), + CONST64(0xc3582be8b0560e73), CONST64(0x63a591f2573f9a34), CONST64(0xda734f95e69eed3c), CONST64(0x5de76934d3d2358e), + CONST64(0x5fe1613edfc22380), CONST64(0xdc79578bf2aed72e), CONST64(0x7d87e99413cf486e), CONST64(0xcd4a13de94266c59), + CONST64(0x7f81e19e1fdf5e60), CONST64(0x5aee752fc1ea049b), CONST64(0x6cb4adc17547f319), CONST64(0x5ce46d31d5da3e89), + CONST64(0xf704fb0c08ebefff), CONST64(0x266a98bed42d47f2), CONST64(0xff1cdb2438abb7c7), CONST64(0xed2a937e543b11b9), + CONST64(0xe825876f4a1336a2), CONST64(0x9dba4ed3699c26f4), CONST64(0x6fb1a1ce7f5fee10), CONST64(0x8e8f028c03048b8d), + CONST64(0x192b647d56c8e34f), CONST64(0xa0fdba1ae7699447), CONST64(0xf00de7171ad3deea), CONST64(0x89861e97113cba98), + CONST64(0x0f113c332278692d), CONST64(0x07091c1b12383115), CONST64(0xafec8629c511fd6a), CONST64(0xfb10cb30208b9bdb), + CONST64(0x0818202830405838), CONST64(0x153f54417ea8976b), CONST64(0x0d1734392e687f23), CONST64(0x040c101418202c1c), + CONST64(0x0103040506080b07), CONST64(0x64ac8de94507ab21), CONST64(0xdf7c5b84f8b6ca27), CONST64(0x769ac5b329970d5f), + CONST64(0x798bf9800bef6472), CONST64(0xdd7a538ef4a6dc29), CONST64(0x3d47f4c98ef5b2b3), CONST64(0x163a584e74b08a62), + CONST64(0x3f41fcc382e5a4bd), CONST64(0x3759dcebb2a5fc85), CONST64(0x6db7a9c4734ff81e), CONST64(0x3848e0d890dd95a8), + CONST64(0xb9d6de67b1a17708), CONST64(0x7395d1a237bf2a44), CONST64(0xe926836a4c1b3da5), CONST64(0x355fd4e1beb5ea8b), + CONST64(0x55ff491ce3926db6), CONST64(0x7193d9a83baf3c4a), CONST64(0x7b8df18a07ff727c), CONST64(0x8c890a860f149d83), + CONST64(0x7296d5a731b72143), CONST64(0x88851a921734b19f), CONST64(0xf607ff090ee3e4f8), CONST64(0x2a7ea882fc4d33d6), + CONST64(0x3e42f8c684edafba), CONST64(0x5ee2653bd9ca2887), CONST64(0x27699cbbd2254cf5), CONST64(0x46ca0543890ac0cf), + CONST64(0x0c14303c28607424), CONST64(0x65af89ec430fa026), CONST64(0x68b8bdd56d67df05), CONST64(0x61a399f85b2f8c3a), + CONST64(0x03050c0f0a181d09), CONST64(0xc15e23e2bc46187d), CONST64(0x57f94116ef827bb8), CONST64(0xd6677fa9cefe9918), + CONST64(0xd976439aec86f035), CONST64(0x58e87d25cdfa1295), CONST64(0xd875479fea8efb32), CONST64(0x66aa85e34917bd2f), + CONST64(0xd7647bacc8f6921f), CONST64(0x3a4ee8d29ccd83a6), CONST64(0xc84507cf8a0e4b42), CONST64(0x3c44f0cc88fdb9b4), + CONST64(0xfa13cf35268390dc), CONST64(0x96a762f453c463c5), CONST64(0xa7f4a601f551a552), CONST64(0x98b55ac277b401ef), + CONST64(0xec29977b52331abe), CONST64(0xb8d5da62b7a97c0f), CONST64(0xc7543bfca876226f), CONST64(0xaeef822cc319f66d), + CONST64(0x69bbb9d06b6fd402), CONST64(0x4bdd317aa762bfec), CONST64(0xabe0963ddd31d176), CONST64(0xa9e69e37d121c778), + CONST64(0x67a981e64f1fb628), CONST64(0x0a1e28223c504e36), CONST64(0x47c901468f02cbc8), CONST64(0xf20bef1d16c3c8e4), + CONST64(0xb5c2ee5b99c1032c), CONST64(0x226688aacc0d6bee), CONST64(0xe532b356647b4981), CONST64(0xee2f9f715e230cb0), + CONST64(0xbedfc27ca399461d), CONST64(0x2b7dac87fa4538d1), CONST64(0x819e3ebf217ce2a0), CONST64(0x1236485a6c90a67e), + CONST64(0x839836b52d6cf4ae), CONST64(0x1b2d6c775ad8f541), CONST64(0x0e1238362470622a), CONST64(0x23658cafca0560e9), + CONST64(0xf502f30604fbf9f1), CONST64(0x45cf094c8312ddc6), CONST64(0x216384a5c61576e7), CONST64(0xce4f1fd19e3e7150), + CONST64(0x49db3970ab72a9e2), CONST64(0x2c74b09ce87d09c4), CONST64(0xf916c33a2c9b8dd5), CONST64(0xe637bf596e635488), + CONST64(0xb6c7e25493d91e25), CONST64(0x2878a088f05d25d8), CONST64(0x17395c4b72b88165), CONST64(0x829b32b02b64ffa9), + CONST64(0x1a2e68725cd0fe46), CONST64(0x8b80169d1d2cac96), CONST64(0xfe1fdf213ea3bcc0), CONST64(0x8a8312981b24a791), + CONST64(0x091b242d3648533f), CONST64(0xc94603ca8c064045), CONST64(0x879426a1354cd8b2), CONST64(0x4ed2256bb94a98f7), + CONST64(0xe13ea3427c5b659d), CONST64(0x2e72b896e46d1fca), CONST64(0xe431b75362734286), CONST64(0xe03da7477a536e9a), + CONST64(0xeb208b60400b2bab), CONST64(0x90ad7aea47f459d7), CONST64(0xa4f1aa0eff49b85b), CONST64(0x1e22786644f0d25a), + CONST64(0x85922eab395ccebc), CONST64(0x60a09dfd5d27873d), CONST64(0x0000000000000000), CONST64(0x256f94b1de355afb), + CONST64(0xf401f70302f3f2f6), CONST64(0xf10ee3121cdbd5ed), CONST64(0x94a16afe5fd475cb), CONST64(0x0b1d2c273a584531), + CONST64(0xe734bb5c686b5f8f), CONST64(0x759fc9bc238f1056), CONST64(0xef2c9b74582b07b7), CONST64(0x345cd0e4b8bde18c), + CONST64(0x3153c4f5a695c697), CONST64(0xd46177a3c2ee8f16), CONST64(0xd06d67b7dacea30a), CONST64(0x869722a43344d3b5), + CONST64(0x7e82e59b19d75567), CONST64(0xadea8e23c901eb64), CONST64(0xfd1ad32e34bba1c9), CONST64(0x297ba48df6552edf), + CONST64(0x3050c0f0a09dcd90), CONST64(0x3b4decd79ac588a1), CONST64(0x9fbc46d9658c30fa), CONST64(0xf815c73f2a9386d2), + CONST64(0xc6573ff9ae7e2968), CONST64(0x13354c5f6a98ad79), CONST64(0x060a181e14303a12), CONST64(0x050f14111e28271b), + CONST64(0xc55233f6a4663461), CONST64(0x113344556688bb77), CONST64(0x7799c1b62f9f0658), CONST64(0x7c84ed9115c74369), + CONST64(0x7a8ef58f01f7797b), CONST64(0x7888fd850de76f75), CONST64(0x365ad8eeb4adf782), CONST64(0x1c24706c48e0c454), + CONST64(0x394be4dd96d59eaf), CONST64(0x59eb7920cbf21992), CONST64(0x1828607850c0e848), CONST64(0x56fa4513e98a70bf), + CONST64(0xb3c8f6458df1393e), CONST64(0xb0cdfa4a87e92437), CONST64(0x246c90b4d83d51fc), CONST64(0x206080a0c01d7de0), + CONST64(0xb2cbf2408bf93239), CONST64(0x92ab72e04be44fd9), CONST64(0xa3f8b615ed71894e), CONST64(0xc05d27e7ba4e137a), + CONST64(0x44cc0d49851ad6c1), CONST64(0x62a695f751379133), CONST64(0x103040506080b070), CONST64(0xb4c1ea5e9fc9082b), + CONST64(0x84912aae3f54c5bb), CONST64(0x43c511529722e7d4), CONST64(0x93a876e54dec44de), CONST64(0xc25b2fedb65e0574), + CONST64(0x4ade357fa16ab4eb), CONST64(0xbddace73a9815b14), CONST64(0x8f8c0689050c808a), CONST64(0x2d77b499ee7502c3), + CONST64(0xbcd9ca76af895013), CONST64(0x9cb94ad66f942df3), CONST64(0x6abeb5df6177c90b), CONST64(0x40c01d5d9d3afadd), + CONST64(0xcf4c1bd498367a57), CONST64(0xa2fbb210eb798249), CONST64(0x809d3aba2774e9a7), CONST64(0x4fd1216ebf4293f0), + CONST64(0x1f217c6342f8d95d), CONST64(0xca430fc5861e5d4c), CONST64(0xaae39238db39da71), CONST64(0x42c61557912aecd3), +}; + +static const ulong64 T1[256] = { + CONST64(0xd3ba68d2b9bb016a), CONST64(0xfc54194d9ae5b166), CONST64(0x712f93bc65e2cd14), CONST64(0x9c74b9cd8725511b), + CONST64(0xf5530251a2f7a457), CONST64(0x68d3b86bd6d003be), CONST64(0x6bd2bd6fded604b5), CONST64(0xd74d642952b3fe85), + CONST64(0xf0500d5dbafdad4a), CONST64(0xe9ac268a09cf63e0), CONST64(0x8a8d830e1c098496), CONST64(0xdcbf79c691a51a4d), + CONST64(0x9070addda73d4d37), CONST64(0xf6520755aaf1a35c), CONST64(0xb39ac852a47be117), CONST64(0xd44c612d5ab5f98e), + CONST64(0x23ea658f0346ac20), CONST64(0x62d5a673e6c41184), CONST64(0xa497f166cc55c268), CONST64(0x6ed1b263c6dc0da8), + CONST64(0x5533ffcc85aa99d0), CONST64(0xf3510859b2fbaa41), CONST64(0xed5b2a71e2c79c0f), CONST64(0xf7a604a259f355ae), + CONST64(0x7fde815fbefe20c1), CONST64(0xd848753d7aade5a2), CONST64(0xe5a8329a29d77fcc), CONST64(0xb699c75ebc71e80a), + CONST64(0x70db904b96e03be6), CONST64(0x5632fac88dac9edb), CONST64(0xc4b751e6d1952215), CONST64(0x19fc2bd7b332ceaa), + CONST64(0x38e348ab4b709373), CONST64(0xbf9edc428463fd3b), CONST64(0xae91ef7efc41d052), CONST64(0xb09bcd56ac7de61c), + CONST64(0x3be24daf43769478), CONST64(0xd0bb6dd6b1bd0661), CONST64(0xc3415819329bdaf1), CONST64(0xb26ecba5577917e5), + CONST64(0xf2a50bae41f95cb3), CONST64(0x40cbc00b16804b56), CONST64(0xbd6bdab17f670cc2), CONST64(0xa295fb6edc59cc7e), + CONST64(0xfea11fbe61e1409f), CONST64(0x08f318ebcb10e3c3), CONST64(0xceb14ffee181302f), CONST64(0x06020a08100c0e16), + CONST64(0x49ccdb172e925e67), CONST64(0x51c4f3376ea2663f), CONST64(0x271d6974e84e53cf), CONST64(0x3c144450a0786c9c), + CONST64(0x58c3e82b56b0730e), CONST64(0xa563f2913f57349a), CONST64(0x73da954f9ee63ced), CONST64(0xe75d3469d2d38e35), + CONST64(0xe15f3e61c2df8023), CONST64(0x79dc8b57aef22ed7), CONST64(0x877d94e9cf136e48), CONST64(0x4acdde132694596c), + CONST64(0x817f9ee1df1f605e), CONST64(0xee5a2f75eac19b04), CONST64(0xb46cc1ad477519f3), CONST64(0xe45c316ddad5893e), + CONST64(0x04f70cfbeb08ffef), CONST64(0x6a26be982dd4f247), CONST64(0x1cff24dbab38c7b7), CONST64(0x2aed7e933b54b911), + CONST64(0x25e86f87134aa236), CONST64(0xba9dd34e9c69f426), CONST64(0xb16fcea15f7f10ee), CONST64(0x8f8e8c0204038d8b), + CONST64(0x2b197d64c8564fe3), CONST64(0xfda01aba69e74794), CONST64(0x0df017e7d31aeade), CONST64(0x8689971e3c1198ba), + CONST64(0x110f333c78222d69), CONST64(0x09071b1c38121531), CONST64(0xecaf298611c56afd), CONST64(0x10fb30cb8b20db9b), + CONST64(0x1808282040303858), CONST64(0x3f154154a87e6b97), CONST64(0x170d3934682e237f), CONST64(0x0c04141020181c2c), + CONST64(0x030105040806070b), CONST64(0xac64e98d074521ab), CONST64(0x7cdf845bb6f827ca), CONST64(0x9a76b3c597295f0d), + CONST64(0x8b7980f9ef0b7264), CONST64(0x7add8e53a6f429dc), CONST64(0x473dc9f4f58eb3b2), CONST64(0x3a164e58b074628a), + CONST64(0x413fc3fce582bda4), CONST64(0x5937ebdca5b285fc), CONST64(0xb76dc4a94f731ef8), CONST64(0x4838d8e0dd90a895), + CONST64(0xd6b967dea1b10877), CONST64(0x9573a2d1bf37442a), CONST64(0x26e96a831b4ca53d), CONST64(0x5f35e1d4b5be8bea), + CONST64(0xff551c4992e3b66d), CONST64(0x9371a8d9af3b4a3c), CONST64(0x8d7b8af1ff077c72), CONST64(0x898c860a140f839d), + CONST64(0x9672a7d5b7314321), CONST64(0x8588921a34179fb1), CONST64(0x07f609ffe30ef8e4), CONST64(0x7e2a82a84dfcd633), + CONST64(0x423ec6f8ed84baaf), CONST64(0xe25e3b65cad98728), CONST64(0x6927bb9c25d2f54c), CONST64(0xca4643050a89cfc0), + CONST64(0x140c3c3060282474), CONST64(0xaf65ec890f4326a0), CONST64(0xb868d5bd676d05df), CONST64(0xa361f8992f5b3a8c), + CONST64(0x05030f0c180a091d), CONST64(0x5ec1e22346bc7d18), CONST64(0xf957164182efb87b), CONST64(0x67d6a97ffece1899), + CONST64(0x76d99a4386ec35f0), CONST64(0xe858257dfacd9512), CONST64(0x75d89f478eea32fb), CONST64(0xaa66e38517492fbd), + CONST64(0x64d7ac7bf6c81f92), CONST64(0x4e3ad2e8cd9ca683), CONST64(0x45c8cf070e8a424b), CONST64(0x443cccf0fd88b4b9), + CONST64(0x13fa35cf8326dc90), CONST64(0xa796f462c453c563), CONST64(0xf4a701a651f552a5), CONST64(0xb598c25ab477ef01), + CONST64(0x29ec7b973352be1a), CONST64(0xd5b862daa9b70f7c), CONST64(0x54c7fc3b76a86f22), CONST64(0xefae2c8219c36df6), + CONST64(0xbb69d0b96f6b02d4), CONST64(0xdd4b7a3162a7ecbf), CONST64(0xe0ab3d9631dd76d1), CONST64(0xe6a9379e21d178c7), + CONST64(0xa967e6811f4f28b6), CONST64(0x1e0a2228503c364e), CONST64(0xc9474601028fc8cb), CONST64(0x0bf21defc316e4c8), + CONST64(0xc2b55beec1992c03), CONST64(0x6622aa880dccee6b), CONST64(0x32e556b37b648149), CONST64(0x2fee719f235eb00c), + CONST64(0xdfbe7cc299a31d46), CONST64(0x7d2b87ac45fad138), CONST64(0x9e81bf3e7c21a0e2), CONST64(0x36125a48906c7ea6), + CONST64(0x9883b5366c2daef4), CONST64(0x2d1b776cd85a41f5), CONST64(0x120e363870242a62), CONST64(0x6523af8c05cae960), + CONST64(0x02f506f3fb04f1f9), CONST64(0xcf454c091283c6dd), CONST64(0x6321a58415c6e776), CONST64(0x4fced11f3e9e5071), + CONST64(0xdb49703972abe2a9), CONST64(0x742c9cb07de8c409), CONST64(0x16f93ac39b2cd58d), CONST64(0x37e659bf636e8854), + CONST64(0xc7b654e2d993251e), CONST64(0x782888a05df0d825), CONST64(0x39174b5cb8726581), CONST64(0x9b82b032642ba9ff), + CONST64(0x2e1a7268d05c46fe), CONST64(0x808b9d162c1d96ac), CONST64(0x1ffe21dfa33ec0bc), CONST64(0x838a9812241b91a7), + CONST64(0x1b092d2448363f53), CONST64(0x46c9ca03068c4540), CONST64(0x9487a1264c35b2d8), CONST64(0xd24e6b254ab9f798), + CONST64(0x3ee142a35b7c9d65), CONST64(0x722e96b86de4ca1f), CONST64(0x31e453b773628642), CONST64(0x3de047a7537a9a6e), + CONST64(0x20eb608b0b40ab2b), CONST64(0xad90ea7af447d759), CONST64(0xf1a40eaa49ff5bb8), CONST64(0x221e6678f0445ad2), + CONST64(0x9285ab2e5c39bcce), CONST64(0xa060fd9d275d3d87), CONST64(0x0000000000000000), CONST64(0x6f25b19435defb5a), + CONST64(0x01f403f7f302f6f2), CONST64(0x0ef112e3db1cedd5), CONST64(0xa194fe6ad45fcb75), CONST64(0x1d0b272c583a3145), + CONST64(0x34e75cbb6b688f5f), CONST64(0x9f75bcc98f235610), CONST64(0x2cef749b2b58b707), CONST64(0x5c34e4d0bdb88ce1), + CONST64(0x5331f5c495a697c6), CONST64(0x61d4a377eec2168f), CONST64(0x6dd0b767ceda0aa3), CONST64(0x9786a4224433b5d3), + CONST64(0x827e9be5d7196755), CONST64(0xeaad238e01c964eb), CONST64(0x1afd2ed3bb34c9a1), CONST64(0x7b298da455f6df2e), + CONST64(0x5030f0c09da090cd), CONST64(0x4d3bd7ecc59aa188), CONST64(0xbc9fd9468c65fa30), CONST64(0x15f83fc7932ad286), + CONST64(0x57c6f93f7eae6829), CONST64(0x35135f4c986a79ad), CONST64(0x0a061e183014123a), CONST64(0x0f051114281e1b27), + CONST64(0x52c5f63366a46134), CONST64(0x33115544886677bb), CONST64(0x9977b6c19f2f5806), CONST64(0x847c91edc7156943), + CONST64(0x8e7a8ff5f7017b79), CONST64(0x887885fde70d756f), CONST64(0x5a36eed8adb482f7), CONST64(0x241c6c70e04854c4), + CONST64(0x4b39dde4d596af9e), CONST64(0xeb592079f2cb9219), CONST64(0x28187860c05048e8), CONST64(0xfa5613458ae9bf70), + CONST64(0xc8b345f6f18d3e39), CONST64(0xcdb04afae9873724), CONST64(0x6c24b4903dd8fc51), CONST64(0x6020a0801dc0e07d), + CONST64(0xcbb240f2f98b3932), CONST64(0xab92e072e44bd94f), CONST64(0xf8a315b671ed4e89), CONST64(0x5dc0e7274eba7a13), + CONST64(0xcc44490d1a85c1d6), CONST64(0xa662f79537513391), CONST64(0x30105040806070b0), CONST64(0xc1b45eeac99f2b08), + CONST64(0x9184ae2a543fbbc5), CONST64(0xc54352112297d4e7), CONST64(0xa893e576ec4dde44), CONST64(0x5bc2ed2f5eb67405), + CONST64(0xde4a7f356aa1ebb4), CONST64(0xdabd73ce81a9145b), CONST64(0x8c8f89060c058a80), CONST64(0x772d99b475eec302), + CONST64(0xd9bc76ca89af1350), CONST64(0xb99cd64a946ff32d), CONST64(0xbe6adfb577610bc9), CONST64(0xc0405d1d3a9dddfa), + CONST64(0x4ccfd41b3698577a), CONST64(0xfba210b279eb4982), CONST64(0x9d80ba3a7427a7e9), CONST64(0xd14f6e2142bff093), + CONST64(0x211f637cf8425dd9), CONST64(0x43cac50f1e864c5d), CONST64(0xe3aa389239db71da), CONST64(0xc64257152a91d3ec), +}; + +static const ulong64 T2[256] = { + CONST64(0xd268bad36a01bbb9), CONST64(0x4d1954fc66b1e59a), CONST64(0xbc932f7114cde265), CONST64(0xcdb9749c1b512587), + CONST64(0x510253f557a4f7a2), CONST64(0x6bb8d368be03d0d6), CONST64(0x6fbdd26bb504d6de), CONST64(0x29644dd785feb352), + CONST64(0x5d0d50f04aadfdba), CONST64(0x8a26ace9e063cf09), CONST64(0x0e838d8a9684091c), CONST64(0xc679bfdc4d1aa591), + CONST64(0xddad7090374d3da7), CONST64(0x550752f65ca3f1aa), CONST64(0x52c89ab317e17ba4), CONST64(0x2d614cd48ef9b55a), + CONST64(0x8f65ea2320ac4603), CONST64(0x73a6d5628411c4e6), CONST64(0x66f197a468c255cc), CONST64(0x63b2d16ea80ddcc6), + CONST64(0xccff3355d099aa85), CONST64(0x590851f341aafbb2), CONST64(0x712a5bed0f9cc7e2), CONST64(0xa204a6f7ae55f359), + CONST64(0x5f81de7fc120febe), CONST64(0x3d7548d8a2e5ad7a), CONST64(0x9a32a8e5cc7fd729), CONST64(0x5ec799b60ae871bc), + CONST64(0x4b90db70e63be096), CONST64(0xc8fa3256db9eac8d), CONST64(0xe651b7c4152295d1), CONST64(0xd72bfc19aace32b3), + CONST64(0xab48e3387393704b), CONST64(0x42dc9ebf3bfd6384), CONST64(0x7eef91ae52d041fc), CONST64(0x56cd9bb01ce67dac), + CONST64(0xaf4de23b78947643), CONST64(0xd66dbbd06106bdb1), CONST64(0x195841c3f1da9b32), CONST64(0xa5cb6eb2e5177957), + CONST64(0xae0ba5f2b35cf941), CONST64(0x0bc0cb40564b8016), CONST64(0xb1da6bbdc20c677f), CONST64(0x6efb95a27ecc59dc), + CONST64(0xbe1fa1fe9f40e161), CONST64(0xeb18f308c3e310cb), CONST64(0xfe4fb1ce2f3081e1), CONST64(0x080a0206160e0c10), + CONST64(0x17dbcc49675e922e), CONST64(0x37f3c4513f66a26e), CONST64(0x74691d27cf534ee8), CONST64(0x5044143c9c6c78a0), + CONST64(0x2be8c3580e73b056), CONST64(0x91f263a59a34573f), CONST64(0x4f95da73ed3ce69e), CONST64(0x69345de7358ed3d2), + CONST64(0x613e5fe12380dfc2), CONST64(0x578bdc79d72ef2ae), CONST64(0xe9947d87486e13cf), CONST64(0x13decd4a6c599426), + CONST64(0xe19e7f815e601fdf), CONST64(0x752f5aee049bc1ea), CONST64(0xadc16cb4f3197547), CONST64(0x6d315ce43e89d5da), + CONST64(0xfb0cf704efff08eb), CONST64(0x98be266a47f2d42d), CONST64(0xdb24ff1cb7c738ab), CONST64(0x937eed2a11b9543b), + CONST64(0x876fe82536a24a13), CONST64(0x4ed39dba26f4699c), CONST64(0xa1ce6fb1ee107f5f), CONST64(0x028c8e8f8b8d0304), + CONST64(0x647d192be34f56c8), CONST64(0xba1aa0fd9447e769), CONST64(0xe717f00ddeea1ad3), CONST64(0x1e978986ba98113c), + CONST64(0x3c330f11692d2278), CONST64(0x1c1b070931151238), CONST64(0x8629afecfd6ac511), CONST64(0xcb30fb109bdb208b), + CONST64(0x2028081858383040), CONST64(0x5441153f976b7ea8), CONST64(0x34390d177f232e68), CONST64(0x1014040c2c1c1820), + CONST64(0x040501030b070608), CONST64(0x8de964acab214507), CONST64(0x5b84df7cca27f8b6), CONST64(0xc5b3769a0d5f2997), + CONST64(0xf980798b64720bef), CONST64(0x538edd7adc29f4a6), CONST64(0xf4c93d47b2b38ef5), CONST64(0x584e163a8a6274b0), + CONST64(0xfcc33f41a4bd82e5), CONST64(0xdceb3759fc85b2a5), CONST64(0xa9c46db7f81e734f), CONST64(0xe0d8384895a890dd), + CONST64(0xde67b9d67708b1a1), CONST64(0xd1a273952a4437bf), CONST64(0x836ae9263da54c1b), CONST64(0xd4e1355fea8bbeb5), + CONST64(0x491c55ff6db6e392), CONST64(0xd9a871933c4a3baf), CONST64(0xf18a7b8d727c07ff), CONST64(0x0a868c899d830f14), + CONST64(0xd5a77296214331b7), CONST64(0x1a928885b19f1734), CONST64(0xff09f607e4f80ee3), CONST64(0xa8822a7e33d6fc4d), + CONST64(0xf8c63e42afba84ed), CONST64(0x653b5ee22887d9ca), CONST64(0x9cbb27694cf5d225), CONST64(0x054346cac0cf890a), + CONST64(0x303c0c1474242860), CONST64(0x89ec65afa026430f), CONST64(0xbdd568b8df056d67), CONST64(0x99f861a38c3a5b2f), + CONST64(0x0c0f03051d090a18), CONST64(0x23e2c15e187dbc46), CONST64(0x411657f97bb8ef82), CONST64(0x7fa9d6679918cefe), + CONST64(0x439ad976f035ec86), CONST64(0x7d2558e81295cdfa), CONST64(0x479fd875fb32ea8e), CONST64(0x85e366aabd2f4917), + CONST64(0x7bacd764921fc8f6), CONST64(0xe8d23a4e83a69ccd), CONST64(0x07cfc8454b428a0e), CONST64(0xf0cc3c44b9b488fd), + CONST64(0xcf35fa1390dc2683), CONST64(0x62f496a763c553c4), CONST64(0xa601a7f4a552f551), CONST64(0x5ac298b501ef77b4), + CONST64(0x977bec291abe5233), CONST64(0xda62b8d57c0fb7a9), CONST64(0x3bfcc754226fa876), CONST64(0x822caeeff66dc319), + CONST64(0xb9d069bbd4026b6f), CONST64(0x317a4bddbfeca762), CONST64(0x963dabe0d176dd31), CONST64(0x9e37a9e6c778d121), + CONST64(0x81e667a9b6284f1f), CONST64(0x28220a1e4e363c50), CONST64(0x014647c9cbc88f02), CONST64(0xef1df20bc8e416c3), + CONST64(0xee5bb5c2032c99c1), CONST64(0x88aa22666beecc0d), CONST64(0xb356e5324981647b), CONST64(0x9f71ee2f0cb05e23), + CONST64(0xc27cbedf461da399), CONST64(0xac872b7d38d1fa45), CONST64(0x3ebf819ee2a0217c), CONST64(0x485a1236a67e6c90), + CONST64(0x36b58398f4ae2d6c), CONST64(0x6c771b2df5415ad8), CONST64(0x38360e12622a2470), CONST64(0x8caf236560e9ca05), + CONST64(0xf306f502f9f104fb), CONST64(0x094c45cfddc68312), CONST64(0x84a5216376e7c615), CONST64(0x1fd1ce4f71509e3e), + CONST64(0x397049dba9e2ab72), CONST64(0xb09c2c7409c4e87d), CONST64(0xc33af9168dd52c9b), CONST64(0xbf59e63754886e63), + CONST64(0xe254b6c71e2593d9), CONST64(0xa088287825d8f05d), CONST64(0x5c4b1739816572b8), CONST64(0x32b0829bffa92b64), + CONST64(0x68721a2efe465cd0), CONST64(0x169d8b80ac961d2c), CONST64(0xdf21fe1fbcc03ea3), CONST64(0x12988a83a7911b24), + CONST64(0x242d091b533f3648), CONST64(0x03cac94640458c06), CONST64(0x26a18794d8b2354c), CONST64(0x256b4ed298f7b94a), + CONST64(0xa342e13e659d7c5b), CONST64(0xb8962e721fcae46d), CONST64(0xb753e43142866273), CONST64(0xa747e03d6e9a7a53), + CONST64(0x8b60eb202bab400b), CONST64(0x7aea90ad59d747f4), CONST64(0xaa0ea4f1b85bff49), CONST64(0x78661e22d25a44f0), + CONST64(0x2eab8592cebc395c), CONST64(0x9dfd60a0873d5d27), CONST64(0x0000000000000000), CONST64(0x94b1256f5afbde35), + CONST64(0xf703f401f2f602f3), CONST64(0xe312f10ed5ed1cdb), CONST64(0x6afe94a175cb5fd4), CONST64(0x2c270b1d45313a58), + CONST64(0xbb5ce7345f8f686b), CONST64(0xc9bc759f1056238f), CONST64(0x9b74ef2c07b7582b), CONST64(0xd0e4345ce18cb8bd), + CONST64(0xc4f53153c697a695), CONST64(0x77a3d4618f16c2ee), CONST64(0x67b7d06da30adace), CONST64(0x22a48697d3b53344), + CONST64(0xe59b7e82556719d7), CONST64(0x8e23adeaeb64c901), CONST64(0xd32efd1aa1c934bb), CONST64(0xa48d297b2edff655), + CONST64(0xc0f03050cd90a09d), CONST64(0xecd73b4d88a19ac5), CONST64(0x46d99fbc30fa658c), CONST64(0xc73ff81586d22a93), + CONST64(0x3ff9c6572968ae7e), CONST64(0x4c5f1335ad796a98), CONST64(0x181e060a3a121430), CONST64(0x1411050f271b1e28), + CONST64(0x33f6c5523461a466), CONST64(0x44551133bb776688), CONST64(0xc1b6779906582f9f), CONST64(0xed917c84436915c7), + CONST64(0xf58f7a8e797b01f7), CONST64(0xfd8578886f750de7), CONST64(0xd8ee365af782b4ad), CONST64(0x706c1c24c45448e0), + CONST64(0xe4dd394b9eaf96d5), CONST64(0x792059eb1992cbf2), CONST64(0x60781828e84850c0), CONST64(0x451356fa70bfe98a), + CONST64(0xf645b3c8393e8df1), CONST64(0xfa4ab0cd243787e9), CONST64(0x90b4246c51fcd83d), CONST64(0x80a020607de0c01d), + CONST64(0xf240b2cb32398bf9), CONST64(0x72e092ab4fd94be4), CONST64(0xb615a3f8894eed71), CONST64(0x27e7c05d137aba4e), + CONST64(0x0d4944ccd6c1851a), CONST64(0x95f762a691335137), CONST64(0x40501030b0706080), CONST64(0xea5eb4c1082b9fc9), + CONST64(0x2aae8491c5bb3f54), CONST64(0x115243c5e7d49722), CONST64(0x76e593a844de4dec), CONST64(0x2fedc25b0574b65e), + CONST64(0x357f4adeb4eba16a), CONST64(0xce73bdda5b14a981), CONST64(0x06898f8c808a050c), CONST64(0xb4992d7702c3ee75), + CONST64(0xca76bcd95013af89), CONST64(0x4ad69cb92df36f94), CONST64(0xb5df6abec90b6177), CONST64(0x1d5d40c0fadd9d3a), + CONST64(0x1bd4cf4c7a579836), CONST64(0xb210a2fb8249eb79), CONST64(0x3aba809de9a72774), CONST64(0x216e4fd193f0bf42), + CONST64(0x7c631f21d95d42f8), CONST64(0x0fc5ca435d4c861e), CONST64(0x9238aae3da71db39), CONST64(0x155742c6ecd3912a), +}; + +static const ulong64 T3[256] = { + CONST64(0x68d2d3ba016ab9bb), CONST64(0x194dfc54b1669ae5), CONST64(0x93bc712fcd1465e2), CONST64(0xb9cd9c74511b8725), + CONST64(0x0251f553a457a2f7), CONST64(0xb86b68d303bed6d0), CONST64(0xbd6f6bd204b5ded6), CONST64(0x6429d74dfe8552b3), + CONST64(0x0d5df050ad4abafd), CONST64(0x268ae9ac63e009cf), CONST64(0x830e8a8d84961c09), CONST64(0x79c6dcbf1a4d91a5), + CONST64(0xaddd90704d37a73d), CONST64(0x0755f652a35caaf1), CONST64(0xc852b39ae117a47b), CONST64(0x612dd44cf98e5ab5), + CONST64(0x658f23eaac200346), CONST64(0xa67362d51184e6c4), CONST64(0xf166a497c268cc55), CONST64(0xb2636ed10da8c6dc), + CONST64(0xffcc553399d085aa), CONST64(0x0859f351aa41b2fb), CONST64(0x2a71ed5b9c0fe2c7), CONST64(0x04a2f7a655ae59f3), + CONST64(0x815f7fde20c1befe), CONST64(0x753dd848e5a27aad), CONST64(0x329ae5a87fcc29d7), CONST64(0xc75eb699e80abc71), + CONST64(0x904b70db3be696e0), CONST64(0xfac856329edb8dac), CONST64(0x51e6c4b72215d195), CONST64(0x2bd719fcceaab332), + CONST64(0x48ab38e393734b70), CONST64(0xdc42bf9efd3b8463), CONST64(0xef7eae91d052fc41), CONST64(0xcd56b09be61cac7d), + CONST64(0x4daf3be294784376), CONST64(0x6dd6d0bb0661b1bd), CONST64(0x5819c341daf1329b), CONST64(0xcba5b26e17e55779), + CONST64(0x0baef2a55cb341f9), CONST64(0xc00b40cb4b561680), CONST64(0xdab1bd6b0cc27f67), CONST64(0xfb6ea295cc7edc59), + CONST64(0x1fbefea1409f61e1), CONST64(0x18eb08f3e3c3cb10), CONST64(0x4ffeceb1302fe181), CONST64(0x0a0806020e16100c), + CONST64(0xdb1749cc5e672e92), CONST64(0xf33751c4663f6ea2), CONST64(0x6974271d53cfe84e), CONST64(0x44503c146c9ca078), + CONST64(0xe82b58c3730e56b0), CONST64(0xf291a563349a3f57), CONST64(0x954f73da3ced9ee6), CONST64(0x3469e75d8e35d2d3), + CONST64(0x3e61e15f8023c2df), CONST64(0x8b5779dc2ed7aef2), CONST64(0x94e9877d6e48cf13), CONST64(0xde134acd596c2694), + CONST64(0x9ee1817f605edf1f), CONST64(0x2f75ee5a9b04eac1), CONST64(0xc1adb46c19f34775), CONST64(0x316de45c893edad5), + CONST64(0x0cfb04f7ffefeb08), CONST64(0xbe986a26f2472dd4), CONST64(0x24db1cffc7b7ab38), CONST64(0x7e932aedb9113b54), + CONST64(0x6f8725e8a236134a), CONST64(0xd34eba9df4269c69), CONST64(0xcea1b16f10ee5f7f), CONST64(0x8c028f8e8d8b0403), + CONST64(0x7d642b194fe3c856), CONST64(0x1abafda0479469e7), CONST64(0x17e70df0eaded31a), CONST64(0x971e868998ba3c11), + CONST64(0x333c110f2d697822), CONST64(0x1b1c090715313812), CONST64(0x2986ecaf6afd11c5), CONST64(0x30cb10fbdb9b8b20), + CONST64(0x2820180838584030), CONST64(0x41543f156b97a87e), CONST64(0x3934170d237f682e), CONST64(0x14100c041c2c2018), + CONST64(0x05040301070b0806), CONST64(0xe98dac6421ab0745), CONST64(0x845b7cdf27cab6f8), CONST64(0xb3c59a765f0d9729), + CONST64(0x80f98b797264ef0b), CONST64(0x8e537add29dca6f4), CONST64(0xc9f4473db3b2f58e), CONST64(0x4e583a16628ab074), + CONST64(0xc3fc413fbda4e582), CONST64(0xebdc593785fca5b2), CONST64(0xc4a9b76d1ef84f73), CONST64(0xd8e04838a895dd90), + CONST64(0x67ded6b90877a1b1), CONST64(0xa2d19573442abf37), CONST64(0x6a8326e9a53d1b4c), CONST64(0xe1d45f358beab5be), + CONST64(0x1c49ff55b66d92e3), CONST64(0xa8d993714a3caf3b), CONST64(0x8af18d7b7c72ff07), CONST64(0x860a898c839d140f), + CONST64(0xa7d596724321b731), CONST64(0x921a85889fb13417), CONST64(0x09ff07f6f8e4e30e), CONST64(0x82a87e2ad6334dfc), + CONST64(0xc6f8423ebaafed84), CONST64(0x3b65e25e8728cad9), CONST64(0xbb9c6927f54c25d2), CONST64(0x4305ca46cfc00a89), + CONST64(0x3c30140c24746028), CONST64(0xec89af6526a00f43), CONST64(0xd5bdb86805df676d), CONST64(0xf899a3613a8c2f5b), + CONST64(0x0f0c0503091d180a), CONST64(0xe2235ec17d1846bc), CONST64(0x1641f957b87b82ef), CONST64(0xa97f67d61899fece), + CONST64(0x9a4376d935f086ec), CONST64(0x257de8589512facd), CONST64(0x9f4775d832fb8eea), CONST64(0xe385aa662fbd1749), + CONST64(0xac7b64d71f92f6c8), CONST64(0xd2e84e3aa683cd9c), CONST64(0xcf0745c8424b0e8a), CONST64(0xccf0443cb4b9fd88), + CONST64(0x35cf13fadc908326), CONST64(0xf462a796c563c453), CONST64(0x01a6f4a752a551f5), CONST64(0xc25ab598ef01b477), + CONST64(0x7b9729ecbe1a3352), CONST64(0x62dad5b80f7ca9b7), CONST64(0xfc3b54c76f2276a8), CONST64(0x2c82efae6df619c3), + CONST64(0xd0b9bb6902d46f6b), CONST64(0x7a31dd4becbf62a7), CONST64(0x3d96e0ab76d131dd), CONST64(0x379ee6a978c721d1), + CONST64(0xe681a96728b61f4f), CONST64(0x22281e0a364e503c), CONST64(0x4601c947c8cb028f), CONST64(0x1def0bf2e4c8c316), + CONST64(0x5beec2b52c03c199), CONST64(0xaa886622ee6b0dcc), CONST64(0x56b332e581497b64), CONST64(0x719f2feeb00c235e), + CONST64(0x7cc2dfbe1d4699a3), CONST64(0x87ac7d2bd13845fa), CONST64(0xbf3e9e81a0e27c21), CONST64(0x5a4836127ea6906c), + CONST64(0xb5369883aef46c2d), CONST64(0x776c2d1b41f5d85a), CONST64(0x3638120e2a627024), CONST64(0xaf8c6523e96005ca), + CONST64(0x06f302f5f1f9fb04), CONST64(0x4c09cf45c6dd1283), CONST64(0xa5846321e77615c6), CONST64(0xd11f4fce50713e9e), + CONST64(0x7039db49e2a972ab), CONST64(0x9cb0742cc4097de8), CONST64(0x3ac316f9d58d9b2c), CONST64(0x59bf37e68854636e), + CONST64(0x54e2c7b6251ed993), CONST64(0x88a07828d8255df0), CONST64(0x4b5c39176581b872), CONST64(0xb0329b82a9ff642b), + CONST64(0x72682e1a46fed05c), CONST64(0x9d16808b96ac2c1d), CONST64(0x21df1ffec0bca33e), CONST64(0x9812838a91a7241b), + CONST64(0x2d241b093f534836), CONST64(0xca0346c94540068c), CONST64(0xa1269487b2d84c35), CONST64(0x6b25d24ef7984ab9), + CONST64(0x42a33ee19d655b7c), CONST64(0x96b8722eca1f6de4), CONST64(0x53b731e486427362), CONST64(0x47a73de09a6e537a), + CONST64(0x608b20ebab2b0b40), CONST64(0xea7aad90d759f447), CONST64(0x0eaaf1a45bb849ff), CONST64(0x6678221e5ad2f044), + CONST64(0xab2e9285bcce5c39), CONST64(0xfd9da0603d87275d), CONST64(0x0000000000000000), CONST64(0xb1946f25fb5a35de), + CONST64(0x03f701f4f6f2f302), CONST64(0x12e30ef1edd5db1c), CONST64(0xfe6aa194cb75d45f), CONST64(0x272c1d0b3145583a), + CONST64(0x5cbb34e78f5f6b68), CONST64(0xbcc99f7556108f23), CONST64(0x749b2cefb7072b58), CONST64(0xe4d05c348ce1bdb8), + CONST64(0xf5c4533197c695a6), CONST64(0xa37761d4168feec2), CONST64(0xb7676dd00aa3ceda), CONST64(0xa4229786b5d34433), + CONST64(0x9be5827e6755d719), CONST64(0x238eeaad64eb01c9), CONST64(0x2ed31afdc9a1bb34), CONST64(0x8da47b29df2e55f6), + CONST64(0xf0c0503090cd9da0), CONST64(0xd7ec4d3ba188c59a), CONST64(0xd946bc9ffa308c65), CONST64(0x3fc715f8d286932a), + CONST64(0xf93f57c668297eae), CONST64(0x5f4c351379ad986a), CONST64(0x1e180a06123a3014), CONST64(0x11140f051b27281e), + CONST64(0xf63352c5613466a4), CONST64(0x5544331177bb8866), CONST64(0xb6c1997758069f2f), CONST64(0x91ed847c6943c715), + CONST64(0x8ff58e7a7b79f701), CONST64(0x85fd8878756fe70d), CONST64(0xeed85a3682f7adb4), CONST64(0x6c70241c54c4e048), + CONST64(0xdde44b39af9ed596), CONST64(0x2079eb599219f2cb), CONST64(0x7860281848e8c050), CONST64(0x1345fa56bf708ae9), + CONST64(0x45f6c8b33e39f18d), CONST64(0x4afacdb03724e987), CONST64(0xb4906c24fc513dd8), CONST64(0xa0806020e07d1dc0), + CONST64(0x40f2cbb23932f98b), CONST64(0xe072ab92d94fe44b), CONST64(0x15b6f8a34e8971ed), CONST64(0xe7275dc07a134eba), + CONST64(0x490dcc44c1d61a85), CONST64(0xf795a66233913751), CONST64(0x5040301070b08060), CONST64(0x5eeac1b42b08c99f), + CONST64(0xae2a9184bbc5543f), CONST64(0x5211c543d4e72297), CONST64(0xe576a893de44ec4d), CONST64(0xed2f5bc274055eb6), + CONST64(0x7f35de4aebb46aa1), CONST64(0x73cedabd145b81a9), CONST64(0x89068c8f8a800c05), CONST64(0x99b4772dc30275ee), + CONST64(0x76cad9bc135089af), CONST64(0xd64ab99cf32d946f), CONST64(0xdfb5be6a0bc97761), CONST64(0x5d1dc040ddfa3a9d), + CONST64(0xd41b4ccf577a3698), CONST64(0x10b2fba2498279eb), CONST64(0xba3a9d80a7e97427), CONST64(0x6e21d14ff09342bf), + CONST64(0x637c211f5dd9f842), CONST64(0xc50f43ca4c5d1e86), CONST64(0x3892e3aa71da39db), CONST64(0x5715c642d3ec2a91), +}; + +static const ulong64 T4[256] = { + CONST64(0xbbb96a01bad3d268), CONST64(0xe59a66b154fc4d19), CONST64(0xe26514cd2f71bc93), CONST64(0x25871b51749ccdb9), + CONST64(0xf7a257a453f55102), CONST64(0xd0d6be03d3686bb8), CONST64(0xd6deb504d26b6fbd), CONST64(0xb35285fe4dd72964), + CONST64(0xfdba4aad50f05d0d), CONST64(0xcf09e063ace98a26), CONST64(0x091c96848d8a0e83), CONST64(0xa5914d1abfdcc679), + CONST64(0x3da7374d7090ddad), CONST64(0xf1aa5ca352f65507), CONST64(0x7ba417e19ab352c8), CONST64(0xb55a8ef94cd42d61), + CONST64(0x460320acea238f65), CONST64(0xc4e68411d56273a6), CONST64(0x55cc68c297a466f1), CONST64(0xdcc6a80dd16e63b2), + CONST64(0xaa85d0993355ccff), CONST64(0xfbb241aa51f35908), CONST64(0xc7e20f9c5bed712a), CONST64(0xf359ae55a6f7a204), + CONST64(0xfebec120de7f5f81), CONST64(0xad7aa2e548d83d75), CONST64(0xd729cc7fa8e59a32), CONST64(0x71bc0ae899b65ec7), + CONST64(0xe096e63bdb704b90), CONST64(0xac8ddb9e3256c8fa), CONST64(0x95d11522b7c4e651), CONST64(0x32b3aacefc19d72b), + CONST64(0x704b7393e338ab48), CONST64(0x63843bfd9ebf42dc), CONST64(0x41fc52d091ae7eef), CONST64(0x7dac1ce69bb056cd), + CONST64(0x76437894e23baf4d), CONST64(0xbdb16106bbd0d66d), CONST64(0x9b32f1da41c31958), CONST64(0x7957e5176eb2a5cb), + CONST64(0xf941b35ca5f2ae0b), CONST64(0x8016564bcb400bc0), CONST64(0x677fc20c6bbdb1da), CONST64(0x59dc7ecc95a26efb), + CONST64(0xe1619f40a1febe1f), CONST64(0x10cbc3e3f308eb18), CONST64(0x81e12f30b1cefe4f), CONST64(0x0c10160e0206080a), + CONST64(0x922e675ecc4917db), CONST64(0xa26e3f66c45137f3), CONST64(0x4ee8cf531d277469), CONST64(0x78a09c6c143c5044), + CONST64(0xb0560e73c3582be8), CONST64(0x573f9a3463a591f2), CONST64(0xe69eed3cda734f95), CONST64(0xd3d2358e5de76934), + CONST64(0xdfc223805fe1613e), CONST64(0xf2aed72edc79578b), CONST64(0x13cf486e7d87e994), CONST64(0x94266c59cd4a13de), + CONST64(0x1fdf5e607f81e19e), CONST64(0xc1ea049b5aee752f), CONST64(0x7547f3196cb4adc1), CONST64(0xd5da3e895ce46d31), + CONST64(0x08ebeffff704fb0c), CONST64(0xd42d47f2266a98be), CONST64(0x38abb7c7ff1cdb24), CONST64(0x543b11b9ed2a937e), + CONST64(0x4a1336a2e825876f), CONST64(0x699c26f49dba4ed3), CONST64(0x7f5fee106fb1a1ce), CONST64(0x03048b8d8e8f028c), + CONST64(0x56c8e34f192b647d), CONST64(0xe7699447a0fdba1a), CONST64(0x1ad3deeaf00de717), CONST64(0x113cba9889861e97), + CONST64(0x2278692d0f113c33), CONST64(0x1238311507091c1b), CONST64(0xc511fd6aafec8629), CONST64(0x208b9bdbfb10cb30), + CONST64(0x3040583808182028), CONST64(0x7ea8976b153f5441), CONST64(0x2e687f230d173439), CONST64(0x18202c1c040c1014), + CONST64(0x06080b0701030405), CONST64(0x4507ab2164ac8de9), CONST64(0xf8b6ca27df7c5b84), CONST64(0x29970d5f769ac5b3), + CONST64(0x0bef6472798bf980), CONST64(0xf4a6dc29dd7a538e), CONST64(0x8ef5b2b33d47f4c9), CONST64(0x74b08a62163a584e), + CONST64(0x82e5a4bd3f41fcc3), CONST64(0xb2a5fc853759dceb), CONST64(0x734ff81e6db7a9c4), CONST64(0x90dd95a83848e0d8), + CONST64(0xb1a17708b9d6de67), CONST64(0x37bf2a447395d1a2), CONST64(0x4c1b3da5e926836a), CONST64(0xbeb5ea8b355fd4e1), + CONST64(0xe3926db655ff491c), CONST64(0x3baf3c4a7193d9a8), CONST64(0x07ff727c7b8df18a), CONST64(0x0f149d838c890a86), + CONST64(0x31b721437296d5a7), CONST64(0x1734b19f88851a92), CONST64(0x0ee3e4f8f607ff09), CONST64(0xfc4d33d62a7ea882), + CONST64(0x84edafba3e42f8c6), CONST64(0xd9ca28875ee2653b), CONST64(0xd2254cf527699cbb), CONST64(0x890ac0cf46ca0543), + CONST64(0x286074240c14303c), CONST64(0x430fa02665af89ec), CONST64(0x6d67df0568b8bdd5), CONST64(0x5b2f8c3a61a399f8), + CONST64(0x0a181d0903050c0f), CONST64(0xbc46187dc15e23e2), CONST64(0xef827bb857f94116), CONST64(0xcefe9918d6677fa9), + CONST64(0xec86f035d976439a), CONST64(0xcdfa129558e87d25), CONST64(0xea8efb32d875479f), CONST64(0x4917bd2f66aa85e3), + CONST64(0xc8f6921fd7647bac), CONST64(0x9ccd83a63a4ee8d2), CONST64(0x8a0e4b42c84507cf), CONST64(0x88fdb9b43c44f0cc), + CONST64(0x268390dcfa13cf35), CONST64(0x53c463c596a762f4), CONST64(0xf551a552a7f4a601), CONST64(0x77b401ef98b55ac2), + CONST64(0x52331abeec29977b), CONST64(0xb7a97c0fb8d5da62), CONST64(0xa876226fc7543bfc), CONST64(0xc319f66daeef822c), + CONST64(0x6b6fd40269bbb9d0), CONST64(0xa762bfec4bdd317a), CONST64(0xdd31d176abe0963d), CONST64(0xd121c778a9e69e37), + CONST64(0x4f1fb62867a981e6), CONST64(0x3c504e360a1e2822), CONST64(0x8f02cbc847c90146), CONST64(0x16c3c8e4f20bef1d), + CONST64(0x99c1032cb5c2ee5b), CONST64(0xcc0d6bee226688aa), CONST64(0x647b4981e532b356), CONST64(0x5e230cb0ee2f9f71), + CONST64(0xa399461dbedfc27c), CONST64(0xfa4538d12b7dac87), CONST64(0x217ce2a0819e3ebf), CONST64(0x6c90a67e1236485a), + CONST64(0x2d6cf4ae839836b5), CONST64(0x5ad8f5411b2d6c77), CONST64(0x2470622a0e123836), CONST64(0xca0560e923658caf), + CONST64(0x04fbf9f1f502f306), CONST64(0x8312ddc645cf094c), CONST64(0xc61576e7216384a5), CONST64(0x9e3e7150ce4f1fd1), + CONST64(0xab72a9e249db3970), CONST64(0xe87d09c42c74b09c), CONST64(0x2c9b8dd5f916c33a), CONST64(0x6e635488e637bf59), + CONST64(0x93d91e25b6c7e254), CONST64(0xf05d25d82878a088), CONST64(0x72b8816517395c4b), CONST64(0x2b64ffa9829b32b0), + CONST64(0x5cd0fe461a2e6872), CONST64(0x1d2cac968b80169d), CONST64(0x3ea3bcc0fe1fdf21), CONST64(0x1b24a7918a831298), + CONST64(0x3648533f091b242d), CONST64(0x8c064045c94603ca), CONST64(0x354cd8b2879426a1), CONST64(0xb94a98f74ed2256b), + CONST64(0x7c5b659de13ea342), CONST64(0xe46d1fca2e72b896), CONST64(0x62734286e431b753), CONST64(0x7a536e9ae03da747), + CONST64(0x400b2babeb208b60), CONST64(0x47f459d790ad7aea), CONST64(0xff49b85ba4f1aa0e), CONST64(0x44f0d25a1e227866), + CONST64(0x395ccebc85922eab), CONST64(0x5d27873d60a09dfd), CONST64(0x0000000000000000), CONST64(0xde355afb256f94b1), + CONST64(0x02f3f2f6f401f703), CONST64(0x1cdbd5edf10ee312), CONST64(0x5fd475cb94a16afe), CONST64(0x3a5845310b1d2c27), + CONST64(0x686b5f8fe734bb5c), CONST64(0x238f1056759fc9bc), CONST64(0x582b07b7ef2c9b74), CONST64(0xb8bde18c345cd0e4), + CONST64(0xa695c6973153c4f5), CONST64(0xc2ee8f16d46177a3), CONST64(0xdacea30ad06d67b7), CONST64(0x3344d3b5869722a4), + CONST64(0x19d755677e82e59b), CONST64(0xc901eb64adea8e23), CONST64(0x34bba1c9fd1ad32e), CONST64(0xf6552edf297ba48d), + CONST64(0xa09dcd903050c0f0), CONST64(0x9ac588a13b4decd7), CONST64(0x658c30fa9fbc46d9), CONST64(0x2a9386d2f815c73f), + CONST64(0xae7e2968c6573ff9), CONST64(0x6a98ad7913354c5f), CONST64(0x14303a12060a181e), CONST64(0x1e28271b050f1411), + CONST64(0xa4663461c55233f6), CONST64(0x6688bb7711334455), CONST64(0x2f9f06587799c1b6), CONST64(0x15c743697c84ed91), + CONST64(0x01f7797b7a8ef58f), CONST64(0x0de76f757888fd85), CONST64(0xb4adf782365ad8ee), CONST64(0x48e0c4541c24706c), + CONST64(0x96d59eaf394be4dd), CONST64(0xcbf2199259eb7920), CONST64(0x50c0e84818286078), CONST64(0xe98a70bf56fa4513), + CONST64(0x8df1393eb3c8f645), CONST64(0x87e92437b0cdfa4a), CONST64(0xd83d51fc246c90b4), CONST64(0xc01d7de0206080a0), + CONST64(0x8bf93239b2cbf240), CONST64(0x4be44fd992ab72e0), CONST64(0xed71894ea3f8b615), CONST64(0xba4e137ac05d27e7), + CONST64(0x851ad6c144cc0d49), CONST64(0x5137913362a695f7), CONST64(0x6080b07010304050), CONST64(0x9fc9082bb4c1ea5e), + CONST64(0x3f54c5bb84912aae), CONST64(0x9722e7d443c51152), CONST64(0x4dec44de93a876e5), CONST64(0xb65e0574c25b2fed), + CONST64(0xa16ab4eb4ade357f), CONST64(0xa9815b14bddace73), CONST64(0x050c808a8f8c0689), CONST64(0xee7502c32d77b499), + CONST64(0xaf895013bcd9ca76), CONST64(0x6f942df39cb94ad6), CONST64(0x6177c90b6abeb5df), CONST64(0x9d3afadd40c01d5d), + CONST64(0x98367a57cf4c1bd4), CONST64(0xeb798249a2fbb210), CONST64(0x2774e9a7809d3aba), CONST64(0xbf4293f04fd1216e), + CONST64(0x42f8d95d1f217c63), CONST64(0x861e5d4cca430fc5), CONST64(0xdb39da71aae39238), CONST64(0x912aecd342c61557), +}; + +static const ulong64 T5[256] = { + CONST64(0xb9bb016ad3ba68d2), CONST64(0x9ae5b166fc54194d), CONST64(0x65e2cd14712f93bc), CONST64(0x8725511b9c74b9cd), + CONST64(0xa2f7a457f5530251), CONST64(0xd6d003be68d3b86b), CONST64(0xded604b56bd2bd6f), CONST64(0x52b3fe85d74d6429), + CONST64(0xbafdad4af0500d5d), CONST64(0x09cf63e0e9ac268a), CONST64(0x1c0984968a8d830e), CONST64(0x91a51a4ddcbf79c6), + CONST64(0xa73d4d379070addd), CONST64(0xaaf1a35cf6520755), CONST64(0xa47be117b39ac852), CONST64(0x5ab5f98ed44c612d), + CONST64(0x0346ac2023ea658f), CONST64(0xe6c4118462d5a673), CONST64(0xcc55c268a497f166), CONST64(0xc6dc0da86ed1b263), + CONST64(0x85aa99d05533ffcc), CONST64(0xb2fbaa41f3510859), CONST64(0xe2c79c0fed5b2a71), CONST64(0x59f355aef7a604a2), + CONST64(0xbefe20c17fde815f), CONST64(0x7aade5a2d848753d), CONST64(0x29d77fcce5a8329a), CONST64(0xbc71e80ab699c75e), + CONST64(0x96e03be670db904b), CONST64(0x8dac9edb5632fac8), CONST64(0xd1952215c4b751e6), CONST64(0xb332ceaa19fc2bd7), + CONST64(0x4b70937338e348ab), CONST64(0x8463fd3bbf9edc42), CONST64(0xfc41d052ae91ef7e), CONST64(0xac7de61cb09bcd56), + CONST64(0x437694783be24daf), CONST64(0xb1bd0661d0bb6dd6), CONST64(0x329bdaf1c3415819), CONST64(0x577917e5b26ecba5), + CONST64(0x41f95cb3f2a50bae), CONST64(0x16804b5640cbc00b), CONST64(0x7f670cc2bd6bdab1), CONST64(0xdc59cc7ea295fb6e), + CONST64(0x61e1409ffea11fbe), CONST64(0xcb10e3c308f318eb), CONST64(0xe181302fceb14ffe), CONST64(0x100c0e1606020a08), + CONST64(0x2e925e6749ccdb17), CONST64(0x6ea2663f51c4f337), CONST64(0xe84e53cf271d6974), CONST64(0xa0786c9c3c144450), + CONST64(0x56b0730e58c3e82b), CONST64(0x3f57349aa563f291), CONST64(0x9ee63ced73da954f), CONST64(0xd2d38e35e75d3469), + CONST64(0xc2df8023e15f3e61), CONST64(0xaef22ed779dc8b57), CONST64(0xcf136e48877d94e9), CONST64(0x2694596c4acdde13), + CONST64(0xdf1f605e817f9ee1), CONST64(0xeac19b04ee5a2f75), CONST64(0x477519f3b46cc1ad), CONST64(0xdad5893ee45c316d), + CONST64(0xeb08ffef04f70cfb), CONST64(0x2dd4f2476a26be98), CONST64(0xab38c7b71cff24db), CONST64(0x3b54b9112aed7e93), + CONST64(0x134aa23625e86f87), CONST64(0x9c69f426ba9dd34e), CONST64(0x5f7f10eeb16fcea1), CONST64(0x04038d8b8f8e8c02), + CONST64(0xc8564fe32b197d64), CONST64(0x69e74794fda01aba), CONST64(0xd31aeade0df017e7), CONST64(0x3c1198ba8689971e), + CONST64(0x78222d69110f333c), CONST64(0x3812153109071b1c), CONST64(0x11c56afdecaf2986), CONST64(0x8b20db9b10fb30cb), + CONST64(0x4030385818082820), CONST64(0xa87e6b973f154154), CONST64(0x682e237f170d3934), CONST64(0x20181c2c0c041410), + CONST64(0x0806070b03010504), CONST64(0x074521abac64e98d), CONST64(0xb6f827ca7cdf845b), CONST64(0x97295f0d9a76b3c5), + CONST64(0xef0b72648b7980f9), CONST64(0xa6f429dc7add8e53), CONST64(0xf58eb3b2473dc9f4), CONST64(0xb074628a3a164e58), + CONST64(0xe582bda4413fc3fc), CONST64(0xa5b285fc5937ebdc), CONST64(0x4f731ef8b76dc4a9), CONST64(0xdd90a8954838d8e0), + CONST64(0xa1b10877d6b967de), CONST64(0xbf37442a9573a2d1), CONST64(0x1b4ca53d26e96a83), CONST64(0xb5be8bea5f35e1d4), + CONST64(0x92e3b66dff551c49), CONST64(0xaf3b4a3c9371a8d9), CONST64(0xff077c728d7b8af1), CONST64(0x140f839d898c860a), + CONST64(0xb73143219672a7d5), CONST64(0x34179fb18588921a), CONST64(0xe30ef8e407f609ff), CONST64(0x4dfcd6337e2a82a8), + CONST64(0xed84baaf423ec6f8), CONST64(0xcad98728e25e3b65), CONST64(0x25d2f54c6927bb9c), CONST64(0x0a89cfc0ca464305), + CONST64(0x60282474140c3c30), CONST64(0x0f4326a0af65ec89), CONST64(0x676d05dfb868d5bd), CONST64(0x2f5b3a8ca361f899), + CONST64(0x180a091d05030f0c), CONST64(0x46bc7d185ec1e223), CONST64(0x82efb87bf9571641), CONST64(0xfece189967d6a97f), + CONST64(0x86ec35f076d99a43), CONST64(0xfacd9512e858257d), CONST64(0x8eea32fb75d89f47), CONST64(0x17492fbdaa66e385), + CONST64(0xf6c81f9264d7ac7b), CONST64(0xcd9ca6834e3ad2e8), CONST64(0x0e8a424b45c8cf07), CONST64(0xfd88b4b9443cccf0), + CONST64(0x8326dc9013fa35cf), CONST64(0xc453c563a796f462), CONST64(0x51f552a5f4a701a6), CONST64(0xb477ef01b598c25a), + CONST64(0x3352be1a29ec7b97), CONST64(0xa9b70f7cd5b862da), CONST64(0x76a86f2254c7fc3b), CONST64(0x19c36df6efae2c82), + CONST64(0x6f6b02d4bb69d0b9), CONST64(0x62a7ecbfdd4b7a31), CONST64(0x31dd76d1e0ab3d96), CONST64(0x21d178c7e6a9379e), + CONST64(0x1f4f28b6a967e681), CONST64(0x503c364e1e0a2228), CONST64(0x028fc8cbc9474601), CONST64(0xc316e4c80bf21def), + CONST64(0xc1992c03c2b55bee), CONST64(0x0dccee6b6622aa88), CONST64(0x7b64814932e556b3), CONST64(0x235eb00c2fee719f), + CONST64(0x99a31d46dfbe7cc2), CONST64(0x45fad1387d2b87ac), CONST64(0x7c21a0e29e81bf3e), CONST64(0x906c7ea636125a48), + CONST64(0x6c2daef49883b536), CONST64(0xd85a41f52d1b776c), CONST64(0x70242a62120e3638), CONST64(0x05cae9606523af8c), + CONST64(0xfb04f1f902f506f3), CONST64(0x1283c6ddcf454c09), CONST64(0x15c6e7766321a584), CONST64(0x3e9e50714fced11f), + CONST64(0x72abe2a9db497039), CONST64(0x7de8c409742c9cb0), CONST64(0x9b2cd58d16f93ac3), CONST64(0x636e885437e659bf), + CONST64(0xd993251ec7b654e2), CONST64(0x5df0d825782888a0), CONST64(0xb872658139174b5c), CONST64(0x642ba9ff9b82b032), + CONST64(0xd05c46fe2e1a7268), CONST64(0x2c1d96ac808b9d16), CONST64(0xa33ec0bc1ffe21df), CONST64(0x241b91a7838a9812), + CONST64(0x48363f531b092d24), CONST64(0x068c454046c9ca03), CONST64(0x4c35b2d89487a126), CONST64(0x4ab9f798d24e6b25), + CONST64(0x5b7c9d653ee142a3), CONST64(0x6de4ca1f722e96b8), CONST64(0x7362864231e453b7), CONST64(0x537a9a6e3de047a7), + CONST64(0x0b40ab2b20eb608b), CONST64(0xf447d759ad90ea7a), CONST64(0x49ff5bb8f1a40eaa), CONST64(0xf0445ad2221e6678), + CONST64(0x5c39bcce9285ab2e), CONST64(0x275d3d87a060fd9d), CONST64(0x0000000000000000), CONST64(0x35defb5a6f25b194), + CONST64(0xf302f6f201f403f7), CONST64(0xdb1cedd50ef112e3), CONST64(0xd45fcb75a194fe6a), CONST64(0x583a31451d0b272c), + CONST64(0x6b688f5f34e75cbb), CONST64(0x8f2356109f75bcc9), CONST64(0x2b58b7072cef749b), CONST64(0xbdb88ce15c34e4d0), + CONST64(0x95a697c65331f5c4), CONST64(0xeec2168f61d4a377), CONST64(0xceda0aa36dd0b767), CONST64(0x4433b5d39786a422), + CONST64(0xd7196755827e9be5), CONST64(0x01c964ebeaad238e), CONST64(0xbb34c9a11afd2ed3), CONST64(0x55f6df2e7b298da4), + CONST64(0x9da090cd5030f0c0), CONST64(0xc59aa1884d3bd7ec), CONST64(0x8c65fa30bc9fd946), CONST64(0x932ad28615f83fc7), + CONST64(0x7eae682957c6f93f), CONST64(0x986a79ad35135f4c), CONST64(0x3014123a0a061e18), CONST64(0x281e1b270f051114), + CONST64(0x66a4613452c5f633), CONST64(0x886677bb33115544), CONST64(0x9f2f58069977b6c1), CONST64(0xc7156943847c91ed), + CONST64(0xf7017b798e7a8ff5), CONST64(0xe70d756f887885fd), CONST64(0xadb482f75a36eed8), CONST64(0xe04854c4241c6c70), + CONST64(0xd596af9e4b39dde4), CONST64(0xf2cb9219eb592079), CONST64(0xc05048e828187860), CONST64(0x8ae9bf70fa561345), + CONST64(0xf18d3e39c8b345f6), CONST64(0xe9873724cdb04afa), CONST64(0x3dd8fc516c24b490), CONST64(0x1dc0e07d6020a080), + CONST64(0xf98b3932cbb240f2), CONST64(0xe44bd94fab92e072), CONST64(0x71ed4e89f8a315b6), CONST64(0x4eba7a135dc0e727), + CONST64(0x1a85c1d6cc44490d), CONST64(0x37513391a662f795), CONST64(0x806070b030105040), CONST64(0xc99f2b08c1b45eea), + CONST64(0x543fbbc59184ae2a), CONST64(0x2297d4e7c5435211), CONST64(0xec4dde44a893e576), CONST64(0x5eb674055bc2ed2f), + CONST64(0x6aa1ebb4de4a7f35), CONST64(0x81a9145bdabd73ce), CONST64(0x0c058a808c8f8906), CONST64(0x75eec302772d99b4), + CONST64(0x89af1350d9bc76ca), CONST64(0x946ff32db99cd64a), CONST64(0x77610bc9be6adfb5), CONST64(0x3a9dddfac0405d1d), + CONST64(0x3698577a4ccfd41b), CONST64(0x79eb4982fba210b2), CONST64(0x7427a7e99d80ba3a), CONST64(0x42bff093d14f6e21), + CONST64(0xf8425dd9211f637c), CONST64(0x1e864c5d43cac50f), CONST64(0x39db71dae3aa3892), CONST64(0x2a91d3ecc6425715), +}; + +static const ulong64 T6[256] = { + CONST64(0x6a01bbb9d268bad3), CONST64(0x66b1e59a4d1954fc), CONST64(0x14cde265bc932f71), CONST64(0x1b512587cdb9749c), + CONST64(0x57a4f7a2510253f5), CONST64(0xbe03d0d66bb8d368), CONST64(0xb504d6de6fbdd26b), CONST64(0x85feb35229644dd7), + CONST64(0x4aadfdba5d0d50f0), CONST64(0xe063cf098a26ace9), CONST64(0x9684091c0e838d8a), CONST64(0x4d1aa591c679bfdc), + CONST64(0x374d3da7ddad7090), CONST64(0x5ca3f1aa550752f6), CONST64(0x17e17ba452c89ab3), CONST64(0x8ef9b55a2d614cd4), + CONST64(0x20ac46038f65ea23), CONST64(0x8411c4e673a6d562), CONST64(0x68c255cc66f197a4), CONST64(0xa80ddcc663b2d16e), + CONST64(0xd099aa85ccff3355), CONST64(0x41aafbb2590851f3), CONST64(0x0f9cc7e2712a5bed), CONST64(0xae55f359a204a6f7), + CONST64(0xc120febe5f81de7f), CONST64(0xa2e5ad7a3d7548d8), CONST64(0xcc7fd7299a32a8e5), CONST64(0x0ae871bc5ec799b6), + CONST64(0xe63be0964b90db70), CONST64(0xdb9eac8dc8fa3256), CONST64(0x152295d1e651b7c4), CONST64(0xaace32b3d72bfc19), + CONST64(0x7393704bab48e338), CONST64(0x3bfd638442dc9ebf), CONST64(0x52d041fc7eef91ae), CONST64(0x1ce67dac56cd9bb0), + CONST64(0x78947643af4de23b), CONST64(0x6106bdb1d66dbbd0), CONST64(0xf1da9b32195841c3), CONST64(0xe5177957a5cb6eb2), + CONST64(0xb35cf941ae0ba5f2), CONST64(0x564b80160bc0cb40), CONST64(0xc20c677fb1da6bbd), CONST64(0x7ecc59dc6efb95a2), + CONST64(0x9f40e161be1fa1fe), CONST64(0xc3e310cbeb18f308), CONST64(0x2f3081e1fe4fb1ce), CONST64(0x160e0c10080a0206), + CONST64(0x675e922e17dbcc49), CONST64(0x3f66a26e37f3c451), CONST64(0xcf534ee874691d27), CONST64(0x9c6c78a05044143c), + CONST64(0x0e73b0562be8c358), CONST64(0x9a34573f91f263a5), CONST64(0xed3ce69e4f95da73), CONST64(0x358ed3d269345de7), + CONST64(0x2380dfc2613e5fe1), CONST64(0xd72ef2ae578bdc79), CONST64(0x486e13cfe9947d87), CONST64(0x6c59942613decd4a), + CONST64(0x5e601fdfe19e7f81), CONST64(0x049bc1ea752f5aee), CONST64(0xf3197547adc16cb4), CONST64(0x3e89d5da6d315ce4), + CONST64(0xefff08ebfb0cf704), CONST64(0x47f2d42d98be266a), CONST64(0xb7c738abdb24ff1c), CONST64(0x11b9543b937eed2a), + CONST64(0x36a24a13876fe825), CONST64(0x26f4699c4ed39dba), CONST64(0xee107f5fa1ce6fb1), CONST64(0x8b8d0304028c8e8f), + CONST64(0xe34f56c8647d192b), CONST64(0x9447e769ba1aa0fd), CONST64(0xdeea1ad3e717f00d), CONST64(0xba98113c1e978986), + CONST64(0x692d22783c330f11), CONST64(0x311512381c1b0709), CONST64(0xfd6ac5118629afec), CONST64(0x9bdb208bcb30fb10), + CONST64(0x5838304020280818), CONST64(0x976b7ea85441153f), CONST64(0x7f232e6834390d17), CONST64(0x2c1c18201014040c), + CONST64(0x0b07060804050103), CONST64(0xab2145078de964ac), CONST64(0xca27f8b65b84df7c), CONST64(0x0d5f2997c5b3769a), + CONST64(0x64720beff980798b), CONST64(0xdc29f4a6538edd7a), CONST64(0xb2b38ef5f4c93d47), CONST64(0x8a6274b0584e163a), + CONST64(0xa4bd82e5fcc33f41), CONST64(0xfc85b2a5dceb3759), CONST64(0xf81e734fa9c46db7), CONST64(0x95a890dde0d83848), + CONST64(0x7708b1a1de67b9d6), CONST64(0x2a4437bfd1a27395), CONST64(0x3da54c1b836ae926), CONST64(0xea8bbeb5d4e1355f), + CONST64(0x6db6e392491c55ff), CONST64(0x3c4a3bafd9a87193), CONST64(0x727c07fff18a7b8d), CONST64(0x9d830f140a868c89), + CONST64(0x214331b7d5a77296), CONST64(0xb19f17341a928885), CONST64(0xe4f80ee3ff09f607), CONST64(0x33d6fc4da8822a7e), + CONST64(0xafba84edf8c63e42), CONST64(0x2887d9ca653b5ee2), CONST64(0x4cf5d2259cbb2769), CONST64(0xc0cf890a054346ca), + CONST64(0x74242860303c0c14), CONST64(0xa026430f89ec65af), CONST64(0xdf056d67bdd568b8), CONST64(0x8c3a5b2f99f861a3), + CONST64(0x1d090a180c0f0305), CONST64(0x187dbc4623e2c15e), CONST64(0x7bb8ef82411657f9), CONST64(0x9918cefe7fa9d667), + CONST64(0xf035ec86439ad976), CONST64(0x1295cdfa7d2558e8), CONST64(0xfb32ea8e479fd875), CONST64(0xbd2f491785e366aa), + CONST64(0x921fc8f67bacd764), CONST64(0x83a69ccde8d23a4e), CONST64(0x4b428a0e07cfc845), CONST64(0xb9b488fdf0cc3c44), + CONST64(0x90dc2683cf35fa13), CONST64(0x63c553c462f496a7), CONST64(0xa552f551a601a7f4), CONST64(0x01ef77b45ac298b5), + CONST64(0x1abe5233977bec29), CONST64(0x7c0fb7a9da62b8d5), CONST64(0x226fa8763bfcc754), CONST64(0xf66dc319822caeef), + CONST64(0xd4026b6fb9d069bb), CONST64(0xbfeca762317a4bdd), CONST64(0xd176dd31963dabe0), CONST64(0xc778d1219e37a9e6), + CONST64(0xb6284f1f81e667a9), CONST64(0x4e363c5028220a1e), CONST64(0xcbc88f02014647c9), CONST64(0xc8e416c3ef1df20b), + CONST64(0x032c99c1ee5bb5c2), CONST64(0x6beecc0d88aa2266), CONST64(0x4981647bb356e532), CONST64(0x0cb05e239f71ee2f), + CONST64(0x461da399c27cbedf), CONST64(0x38d1fa45ac872b7d), CONST64(0xe2a0217c3ebf819e), CONST64(0xa67e6c90485a1236), + CONST64(0xf4ae2d6c36b58398), CONST64(0xf5415ad86c771b2d), CONST64(0x622a247038360e12), CONST64(0x60e9ca058caf2365), + CONST64(0xf9f104fbf306f502), CONST64(0xddc68312094c45cf), CONST64(0x76e7c61584a52163), CONST64(0x71509e3e1fd1ce4f), + CONST64(0xa9e2ab72397049db), CONST64(0x09c4e87db09c2c74), CONST64(0x8dd52c9bc33af916), CONST64(0x54886e63bf59e637), + CONST64(0x1e2593d9e254b6c7), CONST64(0x25d8f05da0882878), CONST64(0x816572b85c4b1739), CONST64(0xffa92b6432b0829b), + CONST64(0xfe465cd068721a2e), CONST64(0xac961d2c169d8b80), CONST64(0xbcc03ea3df21fe1f), CONST64(0xa7911b2412988a83), + CONST64(0x533f3648242d091b), CONST64(0x40458c0603cac946), CONST64(0xd8b2354c26a18794), CONST64(0x98f7b94a256b4ed2), + CONST64(0x659d7c5ba342e13e), CONST64(0x1fcae46db8962e72), CONST64(0x42866273b753e431), CONST64(0x6e9a7a53a747e03d), + CONST64(0x2bab400b8b60eb20), CONST64(0x59d747f47aea90ad), CONST64(0xb85bff49aa0ea4f1), CONST64(0xd25a44f078661e22), + CONST64(0xcebc395c2eab8592), CONST64(0x873d5d279dfd60a0), CONST64(0x0000000000000000), CONST64(0x5afbde3594b1256f), + CONST64(0xf2f602f3f703f401), CONST64(0xd5ed1cdbe312f10e), CONST64(0x75cb5fd46afe94a1), CONST64(0x45313a582c270b1d), + CONST64(0x5f8f686bbb5ce734), CONST64(0x1056238fc9bc759f), CONST64(0x07b7582b9b74ef2c), CONST64(0xe18cb8bdd0e4345c), + CONST64(0xc697a695c4f53153), CONST64(0x8f16c2ee77a3d461), CONST64(0xa30adace67b7d06d), CONST64(0xd3b5334422a48697), + CONST64(0x556719d7e59b7e82), CONST64(0xeb64c9018e23adea), CONST64(0xa1c934bbd32efd1a), CONST64(0x2edff655a48d297b), + CONST64(0xcd90a09dc0f03050), CONST64(0x88a19ac5ecd73b4d), CONST64(0x30fa658c46d99fbc), CONST64(0x86d22a93c73ff815), + CONST64(0x2968ae7e3ff9c657), CONST64(0xad796a984c5f1335), CONST64(0x3a121430181e060a), CONST64(0x271b1e281411050f), + CONST64(0x3461a46633f6c552), CONST64(0xbb77668844551133), CONST64(0x06582f9fc1b67799), CONST64(0x436915c7ed917c84), + CONST64(0x797b01f7f58f7a8e), CONST64(0x6f750de7fd857888), CONST64(0xf782b4add8ee365a), CONST64(0xc45448e0706c1c24), + CONST64(0x9eaf96d5e4dd394b), CONST64(0x1992cbf2792059eb), CONST64(0xe84850c060781828), CONST64(0x70bfe98a451356fa), + CONST64(0x393e8df1f645b3c8), CONST64(0x243787e9fa4ab0cd), CONST64(0x51fcd83d90b4246c), CONST64(0x7de0c01d80a02060), + CONST64(0x32398bf9f240b2cb), CONST64(0x4fd94be472e092ab), CONST64(0x894eed71b615a3f8), CONST64(0x137aba4e27e7c05d), + CONST64(0xd6c1851a0d4944cc), CONST64(0x9133513795f762a6), CONST64(0xb070608040501030), CONST64(0x082b9fc9ea5eb4c1), + CONST64(0xc5bb3f542aae8491), CONST64(0xe7d49722115243c5), CONST64(0x44de4dec76e593a8), CONST64(0x0574b65e2fedc25b), + CONST64(0xb4eba16a357f4ade), CONST64(0x5b14a981ce73bdda), CONST64(0x808a050c06898f8c), CONST64(0x02c3ee75b4992d77), + CONST64(0x5013af89ca76bcd9), CONST64(0x2df36f944ad69cb9), CONST64(0xc90b6177b5df6abe), CONST64(0xfadd9d3a1d5d40c0), + CONST64(0x7a5798361bd4cf4c), CONST64(0x8249eb79b210a2fb), CONST64(0xe9a727743aba809d), CONST64(0x93f0bf42216e4fd1), + CONST64(0xd95d42f87c631f21), CONST64(0x5d4c861e0fc5ca43), CONST64(0xda71db399238aae3), CONST64(0xecd3912a155742c6), +}; + +static const ulong64 T7[256] = { + CONST64(0x016ab9bb68d2d3ba), CONST64(0xb1669ae5194dfc54), CONST64(0xcd1465e293bc712f), CONST64(0x511b8725b9cd9c74), + CONST64(0xa457a2f70251f553), CONST64(0x03bed6d0b86b68d3), CONST64(0x04b5ded6bd6f6bd2), CONST64(0xfe8552b36429d74d), + CONST64(0xad4abafd0d5df050), CONST64(0x63e009cf268ae9ac), CONST64(0x84961c09830e8a8d), CONST64(0x1a4d91a579c6dcbf), + CONST64(0x4d37a73daddd9070), CONST64(0xa35caaf10755f652), CONST64(0xe117a47bc852b39a), CONST64(0xf98e5ab5612dd44c), + CONST64(0xac200346658f23ea), CONST64(0x1184e6c4a67362d5), CONST64(0xc268cc55f166a497), CONST64(0x0da8c6dcb2636ed1), + CONST64(0x99d085aaffcc5533), CONST64(0xaa41b2fb0859f351), CONST64(0x9c0fe2c72a71ed5b), CONST64(0x55ae59f304a2f7a6), + CONST64(0x20c1befe815f7fde), CONST64(0xe5a27aad753dd848), CONST64(0x7fcc29d7329ae5a8), CONST64(0xe80abc71c75eb699), + CONST64(0x3be696e0904b70db), CONST64(0x9edb8dacfac85632), CONST64(0x2215d19551e6c4b7), CONST64(0xceaab3322bd719fc), + CONST64(0x93734b7048ab38e3), CONST64(0xfd3b8463dc42bf9e), CONST64(0xd052fc41ef7eae91), CONST64(0xe61cac7dcd56b09b), + CONST64(0x947843764daf3be2), CONST64(0x0661b1bd6dd6d0bb), CONST64(0xdaf1329b5819c341), CONST64(0x17e55779cba5b26e), + CONST64(0x5cb341f90baef2a5), CONST64(0x4b561680c00b40cb), CONST64(0x0cc27f67dab1bd6b), CONST64(0xcc7edc59fb6ea295), + CONST64(0x409f61e11fbefea1), CONST64(0xe3c3cb1018eb08f3), CONST64(0x302fe1814ffeceb1), CONST64(0x0e16100c0a080602), + CONST64(0x5e672e92db1749cc), CONST64(0x663f6ea2f33751c4), CONST64(0x53cfe84e6974271d), CONST64(0x6c9ca07844503c14), + CONST64(0x730e56b0e82b58c3), CONST64(0x349a3f57f291a563), CONST64(0x3ced9ee6954f73da), CONST64(0x8e35d2d33469e75d), + CONST64(0x8023c2df3e61e15f), CONST64(0x2ed7aef28b5779dc), CONST64(0x6e48cf1394e9877d), CONST64(0x596c2694de134acd), + CONST64(0x605edf1f9ee1817f), CONST64(0x9b04eac12f75ee5a), CONST64(0x19f34775c1adb46c), CONST64(0x893edad5316de45c), + CONST64(0xffefeb080cfb04f7), CONST64(0xf2472dd4be986a26), CONST64(0xc7b7ab3824db1cff), CONST64(0xb9113b547e932aed), + CONST64(0xa236134a6f8725e8), CONST64(0xf4269c69d34eba9d), CONST64(0x10ee5f7fcea1b16f), CONST64(0x8d8b04038c028f8e), + CONST64(0x4fe3c8567d642b19), CONST64(0x479469e71abafda0), CONST64(0xeaded31a17e70df0), CONST64(0x98ba3c11971e8689), + CONST64(0x2d697822333c110f), CONST64(0x153138121b1c0907), CONST64(0x6afd11c52986ecaf), CONST64(0xdb9b8b2030cb10fb), + CONST64(0x3858403028201808), CONST64(0x6b97a87e41543f15), CONST64(0x237f682e3934170d), CONST64(0x1c2c201814100c04), + CONST64(0x070b080605040301), CONST64(0x21ab0745e98dac64), CONST64(0x27cab6f8845b7cdf), CONST64(0x5f0d9729b3c59a76), + CONST64(0x7264ef0b80f98b79), CONST64(0x29dca6f48e537add), CONST64(0xb3b2f58ec9f4473d), CONST64(0x628ab0744e583a16), + CONST64(0xbda4e582c3fc413f), CONST64(0x85fca5b2ebdc5937), CONST64(0x1ef84f73c4a9b76d), CONST64(0xa895dd90d8e04838), + CONST64(0x0877a1b167ded6b9), CONST64(0x442abf37a2d19573), CONST64(0xa53d1b4c6a8326e9), CONST64(0x8beab5bee1d45f35), + CONST64(0xb66d92e31c49ff55), CONST64(0x4a3caf3ba8d99371), CONST64(0x7c72ff078af18d7b), CONST64(0x839d140f860a898c), + CONST64(0x4321b731a7d59672), CONST64(0x9fb13417921a8588), CONST64(0xf8e4e30e09ff07f6), CONST64(0xd6334dfc82a87e2a), + CONST64(0xbaafed84c6f8423e), CONST64(0x8728cad93b65e25e), CONST64(0xf54c25d2bb9c6927), CONST64(0xcfc00a894305ca46), + CONST64(0x247460283c30140c), CONST64(0x26a00f43ec89af65), CONST64(0x05df676dd5bdb868), CONST64(0x3a8c2f5bf899a361), + CONST64(0x091d180a0f0c0503), CONST64(0x7d1846bce2235ec1), CONST64(0xb87b82ef1641f957), CONST64(0x1899fecea97f67d6), + CONST64(0x35f086ec9a4376d9), CONST64(0x9512facd257de858), CONST64(0x32fb8eea9f4775d8), CONST64(0x2fbd1749e385aa66), + CONST64(0x1f92f6c8ac7b64d7), CONST64(0xa683cd9cd2e84e3a), CONST64(0x424b0e8acf0745c8), CONST64(0xb4b9fd88ccf0443c), + CONST64(0xdc90832635cf13fa), CONST64(0xc563c453f462a796), CONST64(0x52a551f501a6f4a7), CONST64(0xef01b477c25ab598), + CONST64(0xbe1a33527b9729ec), CONST64(0x0f7ca9b762dad5b8), CONST64(0x6f2276a8fc3b54c7), CONST64(0x6df619c32c82efae), + CONST64(0x02d46f6bd0b9bb69), CONST64(0xecbf62a77a31dd4b), CONST64(0x76d131dd3d96e0ab), CONST64(0x78c721d1379ee6a9), + CONST64(0x28b61f4fe681a967), CONST64(0x364e503c22281e0a), CONST64(0xc8cb028f4601c947), CONST64(0xe4c8c3161def0bf2), + CONST64(0x2c03c1995beec2b5), CONST64(0xee6b0dccaa886622), CONST64(0x81497b6456b332e5), CONST64(0xb00c235e719f2fee), + CONST64(0x1d4699a37cc2dfbe), CONST64(0xd13845fa87ac7d2b), CONST64(0xa0e27c21bf3e9e81), CONST64(0x7ea6906c5a483612), + CONST64(0xaef46c2db5369883), CONST64(0x41f5d85a776c2d1b), CONST64(0x2a6270243638120e), CONST64(0xe96005caaf8c6523), + CONST64(0xf1f9fb0406f302f5), CONST64(0xc6dd12834c09cf45), CONST64(0xe77615c6a5846321), CONST64(0x50713e9ed11f4fce), + CONST64(0xe2a972ab7039db49), CONST64(0xc4097de89cb0742c), CONST64(0xd58d9b2c3ac316f9), CONST64(0x8854636e59bf37e6), + CONST64(0x251ed99354e2c7b6), CONST64(0xd8255df088a07828), CONST64(0x6581b8724b5c3917), CONST64(0xa9ff642bb0329b82), + CONST64(0x46fed05c72682e1a), CONST64(0x96ac2c1d9d16808b), CONST64(0xc0bca33e21df1ffe), CONST64(0x91a7241b9812838a), + CONST64(0x3f5348362d241b09), CONST64(0x4540068cca0346c9), CONST64(0xb2d84c35a1269487), CONST64(0xf7984ab96b25d24e), + CONST64(0x9d655b7c42a33ee1), CONST64(0xca1f6de496b8722e), CONST64(0x8642736253b731e4), CONST64(0x9a6e537a47a73de0), + CONST64(0xab2b0b40608b20eb), CONST64(0xd759f447ea7aad90), CONST64(0x5bb849ff0eaaf1a4), CONST64(0x5ad2f0446678221e), + CONST64(0xbcce5c39ab2e9285), CONST64(0x3d87275dfd9da060), CONST64(0x0000000000000000), CONST64(0xfb5a35deb1946f25), + CONST64(0xf6f2f30203f701f4), CONST64(0xedd5db1c12e30ef1), CONST64(0xcb75d45ffe6aa194), CONST64(0x3145583a272c1d0b), + CONST64(0x8f5f6b685cbb34e7), CONST64(0x56108f23bcc99f75), CONST64(0xb7072b58749b2cef), CONST64(0x8ce1bdb8e4d05c34), + CONST64(0x97c695a6f5c45331), CONST64(0x168feec2a37761d4), CONST64(0x0aa3cedab7676dd0), CONST64(0xb5d34433a4229786), + CONST64(0x6755d7199be5827e), CONST64(0x64eb01c9238eeaad), CONST64(0xc9a1bb342ed31afd), CONST64(0xdf2e55f68da47b29), + CONST64(0x90cd9da0f0c05030), CONST64(0xa188c59ad7ec4d3b), CONST64(0xfa308c65d946bc9f), CONST64(0xd286932a3fc715f8), + CONST64(0x68297eaef93f57c6), CONST64(0x79ad986a5f4c3513), CONST64(0x123a30141e180a06), CONST64(0x1b27281e11140f05), + CONST64(0x613466a4f63352c5), CONST64(0x77bb886655443311), CONST64(0x58069f2fb6c19977), CONST64(0x6943c71591ed847c), + CONST64(0x7b79f7018ff58e7a), CONST64(0x756fe70d85fd8878), CONST64(0x82f7adb4eed85a36), CONST64(0x54c4e0486c70241c), + CONST64(0xaf9ed596dde44b39), CONST64(0x9219f2cb2079eb59), CONST64(0x48e8c05078602818), CONST64(0xbf708ae91345fa56), + CONST64(0x3e39f18d45f6c8b3), CONST64(0x3724e9874afacdb0), CONST64(0xfc513dd8b4906c24), CONST64(0xe07d1dc0a0806020), + CONST64(0x3932f98b40f2cbb2), CONST64(0xd94fe44be072ab92), CONST64(0x4e8971ed15b6f8a3), CONST64(0x7a134ebae7275dc0), + CONST64(0xc1d61a85490dcc44), CONST64(0x33913751f795a662), CONST64(0x70b0806050403010), CONST64(0x2b08c99f5eeac1b4), + CONST64(0xbbc5543fae2a9184), CONST64(0xd4e722975211c543), CONST64(0xde44ec4de576a893), CONST64(0x74055eb6ed2f5bc2), + CONST64(0xebb46aa17f35de4a), CONST64(0x145b81a973cedabd), CONST64(0x8a800c0589068c8f), CONST64(0xc30275ee99b4772d), + CONST64(0x135089af76cad9bc), CONST64(0xf32d946fd64ab99c), CONST64(0x0bc97761dfb5be6a), CONST64(0xddfa3a9d5d1dc040), + CONST64(0x577a3698d41b4ccf), CONST64(0x498279eb10b2fba2), CONST64(0xa7e97427ba3a9d80), CONST64(0xf09342bf6e21d14f), + CONST64(0x5dd9f842637c211f), CONST64(0x4c5d1e86c50f43ca), CONST64(0x71da39db3892e3aa), CONST64(0xd3ec2a915715c642), +}; + +static const ulong64 c[R + 1] = { + CONST64(0xba542f7453d3d24d), + CONST64(0x50ac8dbf70529a4c), + CONST64(0xead597d133515ba6), + CONST64(0xde48a899db32b7fc), + CONST64(0xe39e919be2bb416e), + CONST64(0xa5cb6b95a1f3b102), + CONST64(0xccc41d14c363da5d), + CONST64(0x5fdc7dcd7f5a6c5c), + CONST64(0xf726ffede89d6f8e), +}; + + /** + Initialize the Khazad block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int khazad_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int r; + const ulong64 *S; + ulong64 K2, K1; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + if (num_rounds != 8 && num_rounds != 0) { + return CRYPT_INVALID_ROUNDS; + } + + /* use 7th table */ + S = T7; + + /* + * map unsigned char array cipher key to initial key state (mu): + */ + K2 = + ((ulong64)key[ 0] << 56) ^ + ((ulong64)key[ 1] << 48) ^ + ((ulong64)key[ 2] << 40) ^ + ((ulong64)key[ 3] << 32) ^ + ((ulong64)key[ 4] << 24) ^ + ((ulong64)key[ 5] << 16) ^ + ((ulong64)key[ 6] << 8) ^ + ((ulong64)key[ 7] ); + K1 = + ((ulong64)key[ 8] << 56) ^ + ((ulong64)key[ 9] << 48) ^ + ((ulong64)key[10] << 40) ^ + ((ulong64)key[11] << 32) ^ + ((ulong64)key[12] << 24) ^ + ((ulong64)key[13] << 16) ^ + ((ulong64)key[14] << 8) ^ + ((ulong64)key[15] ); + + /* + * compute the round keys: + */ + for (r = 0; r <= R; r++) { + /* + * K[r] = rho(c[r], K1) ^ K2; + */ + skey->khazad.roundKeyEnc[r] = + T0[(int)(K1 >> 56) ] ^ + T1[(int)(K1 >> 48) & 0xff] ^ + T2[(int)(K1 >> 40) & 0xff] ^ + T3[(int)(K1 >> 32) & 0xff] ^ + T4[(int)(K1 >> 24) & 0xff] ^ + T5[(int)(K1 >> 16) & 0xff] ^ + T6[(int)(K1 >> 8) & 0xff] ^ + T7[(int)(K1 ) & 0xff] ^ + c[r] ^ K2; + K2 = K1; K1 = skey->khazad.roundKeyEnc[r]; + } + /* + * compute the inverse key schedule: + * K'^0 = K^R, K'^R = K^0, K'^r = theta(K^{R-r}) + */ + skey->khazad.roundKeyDec[0] = skey->khazad.roundKeyEnc[R]; + for (r = 1; r < R; r++) { + K1 = skey->khazad.roundKeyEnc[R - r]; + skey->khazad.roundKeyDec[r] = + T0[(int)S[(int)(K1 >> 56) ] & 0xff] ^ + T1[(int)S[(int)(K1 >> 48) & 0xff] & 0xff] ^ + T2[(int)S[(int)(K1 >> 40) & 0xff] & 0xff] ^ + T3[(int)S[(int)(K1 >> 32) & 0xff] & 0xff] ^ + T4[(int)S[(int)(K1 >> 24) & 0xff] & 0xff] ^ + T5[(int)S[(int)(K1 >> 16) & 0xff] & 0xff] ^ + T6[(int)S[(int)(K1 >> 8) & 0xff] & 0xff] ^ + T7[(int)S[(int)(K1 ) & 0xff] & 0xff]; + } + skey->khazad.roundKeyDec[R] = skey->khazad.roundKeyEnc[0]; + + return CRYPT_OK; +} + +static void khazad_crypt(const unsigned char *plaintext, unsigned char *ciphertext, + const ulong64 *roundKey) { + int r; + ulong64 state; + /* + * map plaintext block to cipher state (mu) + * and add initial round key (sigma[K^0]): + */ + state = + ((ulong64)plaintext[0] << 56) ^ + ((ulong64)plaintext[1] << 48) ^ + ((ulong64)plaintext[2] << 40) ^ + ((ulong64)plaintext[3] << 32) ^ + ((ulong64)plaintext[4] << 24) ^ + ((ulong64)plaintext[5] << 16) ^ + ((ulong64)plaintext[6] << 8) ^ + ((ulong64)plaintext[7] ) ^ + roundKey[0]; + + /* + * R - 1 full rounds: + */ + for (r = 1; r < R; r++) { + state = + T0[(int)(state >> 56) ] ^ + T1[(int)(state >> 48) & 0xff] ^ + T2[(int)(state >> 40) & 0xff] ^ + T3[(int)(state >> 32) & 0xff] ^ + T4[(int)(state >> 24) & 0xff] ^ + T5[(int)(state >> 16) & 0xff] ^ + T6[(int)(state >> 8) & 0xff] ^ + T7[(int)(state ) & 0xff] ^ + roundKey[r]; + } + + /* + * last round: + */ + state = + (T0[(int)(state >> 56) ] & CONST64(0xff00000000000000)) ^ + (T1[(int)(state >> 48) & 0xff] & CONST64(0x00ff000000000000)) ^ + (T2[(int)(state >> 40) & 0xff] & CONST64(0x0000ff0000000000)) ^ + (T3[(int)(state >> 32) & 0xff] & CONST64(0x000000ff00000000)) ^ + (T4[(int)(state >> 24) & 0xff] & CONST64(0x00000000ff000000)) ^ + (T5[(int)(state >> 16) & 0xff] & CONST64(0x0000000000ff0000)) ^ + (T6[(int)(state >> 8) & 0xff] & CONST64(0x000000000000ff00)) ^ + (T7[(int)(state ) & 0xff] & CONST64(0x00000000000000ff)) ^ + roundKey[R]; + + /* + * map cipher state to ciphertext block (mu^{-1}): + */ + ciphertext[0] = (unsigned char)(state >> 56); + ciphertext[1] = (unsigned char)(state >> 48); + ciphertext[2] = (unsigned char)(state >> 40); + ciphertext[3] = (unsigned char)(state >> 32); + ciphertext[4] = (unsigned char)(state >> 24); + ciphertext[5] = (unsigned char)(state >> 16); + ciphertext[6] = (unsigned char)(state >> 8); + ciphertext[7] = (unsigned char)(state ); +} + +/** + Encrypts a block of text with Khazad + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int khazad_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + khazad_crypt(pt, ct, skey->khazad.roundKeyEnc); + return CRYPT_OK; +} + +/** + Decrypts a block of text with Khazad + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int khazad_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + khazad_crypt(ct, pt, skey->khazad.roundKeyDec); + return CRYPT_OK; +} + +/** + Performs a self-test of the Khazad block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int khazad_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct test { + unsigned char pt[8], ct[8], key[16]; + } tests[] = { +{ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x49, 0xA4, 0xCE, 0x32, 0xAC, 0x19, 0x0E, 0x3F }, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x64, 0x5D, 0x77, 0x3E, 0x40, 0xAB, 0xDD, 0x53 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } +}, { + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x9E, 0x39, 0x98, 0x64, 0xF7, 0x8E, 0xCA, 0x02 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +}, { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + { 0xA9, 0xDF, 0x3D, 0x2C, 0x64, 0xD3, 0xEA, 0x28 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } +} +}; + int x, y; + unsigned char buf[2][8]; + symmetric_key skey; + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + khazad_setup(tests[x].key, 16, 0, &skey); + khazad_ecb_encrypt(tests[x].pt, buf[0], &skey); + khazad_ecb_decrypt(buf[0], buf[1], &skey); + if (compare_testvector(buf[0], 8, tests[x].ct, 8, "Khazad Encrypt", x) || + compare_testvector(buf[1], 8, tests[x].pt, 8, "Khazad Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + for (y = 0; y < 1000; y++) khazad_ecb_encrypt(buf[0], buf[0], &skey); + for (y = 0; y < 1000; y++) khazad_ecb_decrypt(buf[0], buf[0], &skey); + if (compare_testvector(buf[0], 8, tests[x].ct, 8, "Khazad 1000", 1000)) { + return CRYPT_FAIL_TESTVECTOR; + } + + } + return CRYPT_OK; +#endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void khazad_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int khazad_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize >= 16) { + *keysize = 16; + return CRYPT_OK; + } + return CRYPT_INVALID_KEYSIZE; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/kseed.c b/optee_os/core/lib/libtomcrypt/src/ciphers/kseed.c new file mode 100644 index 0000000..1e63aec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/kseed.c @@ -0,0 +1,366 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file kseed.c + seed implementation of SEED derived from RFC4269 + Tom St Denis +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_KSEED + +const struct ltc_cipher_descriptor kseed_desc = { + "seed", + 20, + 16, 16, 16, 16, + &kseed_setup, + &kseed_ecb_encrypt, + &kseed_ecb_decrypt, + &kseed_test, + &kseed_done, + &kseed_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 SS0[256] = { +0x2989A1A8UL,0x05858184UL,0x16C6D2D4UL,0x13C3D3D0UL,0x14445054UL,0x1D0D111CUL,0x2C8CA0ACUL,0x25052124UL, +0x1D4D515CUL,0x03434340UL,0x18081018UL,0x1E0E121CUL,0x11415150UL,0x3CCCF0FCUL,0x0ACAC2C8UL,0x23436360UL, +0x28082028UL,0x04444044UL,0x20002020UL,0x1D8D919CUL,0x20C0E0E0UL,0x22C2E2E0UL,0x08C8C0C8UL,0x17071314UL, +0x2585A1A4UL,0x0F8F838CUL,0x03030300UL,0x3B4B7378UL,0x3B8BB3B8UL,0x13031310UL,0x12C2D2D0UL,0x2ECEE2ECUL, +0x30407070UL,0x0C8C808CUL,0x3F0F333CUL,0x2888A0A8UL,0x32023230UL,0x1DCDD1DCUL,0x36C6F2F4UL,0x34447074UL, +0x2CCCE0ECUL,0x15859194UL,0x0B0B0308UL,0x17475354UL,0x1C4C505CUL,0x1B4B5358UL,0x3D8DB1BCUL,0x01010100UL, +0x24042024UL,0x1C0C101CUL,0x33437370UL,0x18889098UL,0x10001010UL,0x0CCCC0CCUL,0x32C2F2F0UL,0x19C9D1D8UL, +0x2C0C202CUL,0x27C7E3E4UL,0x32427270UL,0x03838380UL,0x1B8B9398UL,0x11C1D1D0UL,0x06868284UL,0x09C9C1C8UL, +0x20406060UL,0x10405050UL,0x2383A3A0UL,0x2BCBE3E8UL,0x0D0D010CUL,0x3686B2B4UL,0x1E8E929CUL,0x0F4F434CUL, +0x3787B3B4UL,0x1A4A5258UL,0x06C6C2C4UL,0x38487078UL,0x2686A2A4UL,0x12021210UL,0x2F8FA3ACUL,0x15C5D1D4UL, +0x21416160UL,0x03C3C3C0UL,0x3484B0B4UL,0x01414140UL,0x12425250UL,0x3D4D717CUL,0x0D8D818CUL,0x08080008UL, +0x1F0F131CUL,0x19899198UL,0x00000000UL,0x19091118UL,0x04040004UL,0x13435350UL,0x37C7F3F4UL,0x21C1E1E0UL, +0x3DCDF1FCUL,0x36467274UL,0x2F0F232CUL,0x27072324UL,0x3080B0B0UL,0x0B8B8388UL,0x0E0E020CUL,0x2B8BA3A8UL, +0x2282A2A0UL,0x2E4E626CUL,0x13839390UL,0x0D4D414CUL,0x29496168UL,0x3C4C707CUL,0x09090108UL,0x0A0A0208UL, +0x3F8FB3BCUL,0x2FCFE3ECUL,0x33C3F3F0UL,0x05C5C1C4UL,0x07878384UL,0x14041014UL,0x3ECEF2FCUL,0x24446064UL, +0x1ECED2DCUL,0x2E0E222CUL,0x0B4B4348UL,0x1A0A1218UL,0x06060204UL,0x21012120UL,0x2B4B6368UL,0x26466264UL, +0x02020200UL,0x35C5F1F4UL,0x12829290UL,0x0A8A8288UL,0x0C0C000CUL,0x3383B3B0UL,0x3E4E727CUL,0x10C0D0D0UL, +0x3A4A7278UL,0x07474344UL,0x16869294UL,0x25C5E1E4UL,0x26062224UL,0x00808080UL,0x2D8DA1ACUL,0x1FCFD3DCUL, +0x2181A1A0UL,0x30003030UL,0x37073334UL,0x2E8EA2ACUL,0x36063234UL,0x15051114UL,0x22022220UL,0x38083038UL, +0x34C4F0F4UL,0x2787A3A4UL,0x05454144UL,0x0C4C404CUL,0x01818180UL,0x29C9E1E8UL,0x04848084UL,0x17879394UL, +0x35053134UL,0x0BCBC3C8UL,0x0ECEC2CCUL,0x3C0C303CUL,0x31417170UL,0x11011110UL,0x07C7C3C4UL,0x09898188UL, +0x35457174UL,0x3BCBF3F8UL,0x1ACAD2D8UL,0x38C8F0F8UL,0x14849094UL,0x19495158UL,0x02828280UL,0x04C4C0C4UL, +0x3FCFF3FCUL,0x09494148UL,0x39093138UL,0x27476364UL,0x00C0C0C0UL,0x0FCFC3CCUL,0x17C7D3D4UL,0x3888B0B8UL, +0x0F0F030CUL,0x0E8E828CUL,0x02424240UL,0x23032320UL,0x11819190UL,0x2C4C606CUL,0x1BCBD3D8UL,0x2484A0A4UL, +0x34043034UL,0x31C1F1F0UL,0x08484048UL,0x02C2C2C0UL,0x2F4F636CUL,0x3D0D313CUL,0x2D0D212CUL,0x00404040UL, +0x3E8EB2BCUL,0x3E0E323CUL,0x3C8CB0BCUL,0x01C1C1C0UL,0x2A8AA2A8UL,0x3A8AB2B8UL,0x0E4E424CUL,0x15455154UL, +0x3B0B3338UL,0x1CCCD0DCUL,0x28486068UL,0x3F4F737CUL,0x1C8C909CUL,0x18C8D0D8UL,0x0A4A4248UL,0x16465254UL, +0x37477374UL,0x2080A0A0UL,0x2DCDE1ECUL,0x06464244UL,0x3585B1B4UL,0x2B0B2328UL,0x25456164UL,0x3ACAF2F8UL, +0x23C3E3E0UL,0x3989B1B8UL,0x3181B1B0UL,0x1F8F939CUL,0x1E4E525CUL,0x39C9F1F8UL,0x26C6E2E4UL,0x3282B2B0UL, +0x31013130UL,0x2ACAE2E8UL,0x2D4D616CUL,0x1F4F535CUL,0x24C4E0E4UL,0x30C0F0F0UL,0x0DCDC1CCUL,0x08888088UL, +0x16061214UL,0x3A0A3238UL,0x18485058UL,0x14C4D0D4UL,0x22426260UL,0x29092128UL,0x07070304UL,0x33033330UL, +0x28C8E0E8UL,0x1B0B1318UL,0x05050104UL,0x39497178UL,0x10809090UL,0x2A4A6268UL,0x2A0A2228UL,0x1A8A9298UL +}; + +static const ulong32 SS1[256] = { +0x38380830UL,0xE828C8E0UL,0x2C2D0D21UL,0xA42686A2UL,0xCC0FCFC3UL,0xDC1ECED2UL,0xB03383B3UL,0xB83888B0UL, +0xAC2F8FA3UL,0x60204060UL,0x54154551UL,0xC407C7C3UL,0x44044440UL,0x6C2F4F63UL,0x682B4B63UL,0x581B4B53UL, +0xC003C3C3UL,0x60224262UL,0x30330333UL,0xB43585B1UL,0x28290921UL,0xA02080A0UL,0xE022C2E2UL,0xA42787A3UL, +0xD013C3D3UL,0x90118191UL,0x10110111UL,0x04060602UL,0x1C1C0C10UL,0xBC3C8CB0UL,0x34360632UL,0x480B4B43UL, +0xEC2FCFE3UL,0x88088880UL,0x6C2C4C60UL,0xA82888A0UL,0x14170713UL,0xC404C4C0UL,0x14160612UL,0xF434C4F0UL, +0xC002C2C2UL,0x44054541UL,0xE021C1E1UL,0xD416C6D2UL,0x3C3F0F33UL,0x3C3D0D31UL,0x8C0E8E82UL,0x98188890UL, +0x28280820UL,0x4C0E4E42UL,0xF436C6F2UL,0x3C3E0E32UL,0xA42585A1UL,0xF839C9F1UL,0x0C0D0D01UL,0xDC1FCFD3UL, +0xD818C8D0UL,0x282B0B23UL,0x64264662UL,0x783A4A72UL,0x24270723UL,0x2C2F0F23UL,0xF031C1F1UL,0x70324272UL, +0x40024242UL,0xD414C4D0UL,0x40014141UL,0xC000C0C0UL,0x70334373UL,0x64274763UL,0xAC2C8CA0UL,0x880B8B83UL, +0xF437C7F3UL,0xAC2D8DA1UL,0x80008080UL,0x1C1F0F13UL,0xC80ACAC2UL,0x2C2C0C20UL,0xA82A8AA2UL,0x34340430UL, +0xD012C2D2UL,0x080B0B03UL,0xEC2ECEE2UL,0xE829C9E1UL,0x5C1D4D51UL,0x94148490UL,0x18180810UL,0xF838C8F0UL, +0x54174753UL,0xAC2E8EA2UL,0x08080800UL,0xC405C5C1UL,0x10130313UL,0xCC0DCDC1UL,0x84068682UL,0xB83989B1UL, +0xFC3FCFF3UL,0x7C3D4D71UL,0xC001C1C1UL,0x30310131UL,0xF435C5F1UL,0x880A8A82UL,0x682A4A62UL,0xB03181B1UL, +0xD011C1D1UL,0x20200020UL,0xD417C7D3UL,0x00020202UL,0x20220222UL,0x04040400UL,0x68284860UL,0x70314171UL, +0x04070703UL,0xD81BCBD3UL,0x9C1D8D91UL,0x98198991UL,0x60214161UL,0xBC3E8EB2UL,0xE426C6E2UL,0x58194951UL, +0xDC1DCDD1UL,0x50114151UL,0x90108090UL,0xDC1CCCD0UL,0x981A8A92UL,0xA02383A3UL,0xA82B8BA3UL,0xD010C0D0UL, +0x80018181UL,0x0C0F0F03UL,0x44074743UL,0x181A0A12UL,0xE023C3E3UL,0xEC2CCCE0UL,0x8C0D8D81UL,0xBC3F8FB3UL, +0x94168692UL,0x783B4B73UL,0x5C1C4C50UL,0xA02282A2UL,0xA02181A1UL,0x60234363UL,0x20230323UL,0x4C0D4D41UL, +0xC808C8C0UL,0x9C1E8E92UL,0x9C1C8C90UL,0x383A0A32UL,0x0C0C0C00UL,0x2C2E0E22UL,0xB83A8AB2UL,0x6C2E4E62UL, +0x9C1F8F93UL,0x581A4A52UL,0xF032C2F2UL,0x90128292UL,0xF033C3F3UL,0x48094941UL,0x78384870UL,0xCC0CCCC0UL, +0x14150511UL,0xF83BCBF3UL,0x70304070UL,0x74354571UL,0x7C3F4F73UL,0x34350531UL,0x10100010UL,0x00030303UL, +0x64244460UL,0x6C2D4D61UL,0xC406C6C2UL,0x74344470UL,0xD415C5D1UL,0xB43484B0UL,0xE82ACAE2UL,0x08090901UL, +0x74364672UL,0x18190911UL,0xFC3ECEF2UL,0x40004040UL,0x10120212UL,0xE020C0E0UL,0xBC3D8DB1UL,0x04050501UL, +0xF83ACAF2UL,0x00010101UL,0xF030C0F0UL,0x282A0A22UL,0x5C1E4E52UL,0xA82989A1UL,0x54164652UL,0x40034343UL, +0x84058581UL,0x14140410UL,0x88098981UL,0x981B8B93UL,0xB03080B0UL,0xE425C5E1UL,0x48084840UL,0x78394971UL, +0x94178793UL,0xFC3CCCF0UL,0x1C1E0E12UL,0x80028282UL,0x20210121UL,0x8C0C8C80UL,0x181B0B13UL,0x5C1F4F53UL, +0x74374773UL,0x54144450UL,0xB03282B2UL,0x1C1D0D11UL,0x24250521UL,0x4C0F4F43UL,0x00000000UL,0x44064642UL, +0xEC2DCDE1UL,0x58184850UL,0x50124252UL,0xE82BCBE3UL,0x7C3E4E72UL,0xD81ACAD2UL,0xC809C9C1UL,0xFC3DCDF1UL, +0x30300030UL,0x94158591UL,0x64254561UL,0x3C3C0C30UL,0xB43686B2UL,0xE424C4E0UL,0xB83B8BB3UL,0x7C3C4C70UL, +0x0C0E0E02UL,0x50104050UL,0x38390931UL,0x24260622UL,0x30320232UL,0x84048480UL,0x68294961UL,0x90138393UL, +0x34370733UL,0xE427C7E3UL,0x24240420UL,0xA42484A0UL,0xC80BCBC3UL,0x50134353UL,0x080A0A02UL,0x84078783UL, +0xD819C9D1UL,0x4C0C4C40UL,0x80038383UL,0x8C0F8F83UL,0xCC0ECEC2UL,0x383B0B33UL,0x480A4A42UL,0xB43787B3UL +}; + +static const ulong32 SS2[256] = { +0xA1A82989UL,0x81840585UL,0xD2D416C6UL,0xD3D013C3UL,0x50541444UL,0x111C1D0DUL,0xA0AC2C8CUL,0x21242505UL, +0x515C1D4DUL,0x43400343UL,0x10181808UL,0x121C1E0EUL,0x51501141UL,0xF0FC3CCCUL,0xC2C80ACAUL,0x63602343UL, +0x20282808UL,0x40440444UL,0x20202000UL,0x919C1D8DUL,0xE0E020C0UL,0xE2E022C2UL,0xC0C808C8UL,0x13141707UL, +0xA1A42585UL,0x838C0F8FUL,0x03000303UL,0x73783B4BUL,0xB3B83B8BUL,0x13101303UL,0xD2D012C2UL,0xE2EC2ECEUL, +0x70703040UL,0x808C0C8CUL,0x333C3F0FUL,0xA0A82888UL,0x32303202UL,0xD1DC1DCDUL,0xF2F436C6UL,0x70743444UL, +0xE0EC2CCCUL,0x91941585UL,0x03080B0BUL,0x53541747UL,0x505C1C4CUL,0x53581B4BUL,0xB1BC3D8DUL,0x01000101UL, +0x20242404UL,0x101C1C0CUL,0x73703343UL,0x90981888UL,0x10101000UL,0xC0CC0CCCUL,0xF2F032C2UL,0xD1D819C9UL, +0x202C2C0CUL,0xE3E427C7UL,0x72703242UL,0x83800383UL,0x93981B8BUL,0xD1D011C1UL,0x82840686UL,0xC1C809C9UL, +0x60602040UL,0x50501040UL,0xA3A02383UL,0xE3E82BCBUL,0x010C0D0DUL,0xB2B43686UL,0x929C1E8EUL,0x434C0F4FUL, +0xB3B43787UL,0x52581A4AUL,0xC2C406C6UL,0x70783848UL,0xA2A42686UL,0x12101202UL,0xA3AC2F8FUL,0xD1D415C5UL, +0x61602141UL,0xC3C003C3UL,0xB0B43484UL,0x41400141UL,0x52501242UL,0x717C3D4DUL,0x818C0D8DUL,0x00080808UL, +0x131C1F0FUL,0x91981989UL,0x00000000UL,0x11181909UL,0x00040404UL,0x53501343UL,0xF3F437C7UL,0xE1E021C1UL, +0xF1FC3DCDUL,0x72743646UL,0x232C2F0FUL,0x23242707UL,0xB0B03080UL,0x83880B8BUL,0x020C0E0EUL,0xA3A82B8BUL, +0xA2A02282UL,0x626C2E4EUL,0x93901383UL,0x414C0D4DUL,0x61682949UL,0x707C3C4CUL,0x01080909UL,0x02080A0AUL, +0xB3BC3F8FUL,0xE3EC2FCFUL,0xF3F033C3UL,0xC1C405C5UL,0x83840787UL,0x10141404UL,0xF2FC3ECEUL,0x60642444UL, +0xD2DC1ECEUL,0x222C2E0EUL,0x43480B4BUL,0x12181A0AUL,0x02040606UL,0x21202101UL,0x63682B4BUL,0x62642646UL, +0x02000202UL,0xF1F435C5UL,0x92901282UL,0x82880A8AUL,0x000C0C0CUL,0xB3B03383UL,0x727C3E4EUL,0xD0D010C0UL, +0x72783A4AUL,0x43440747UL,0x92941686UL,0xE1E425C5UL,0x22242606UL,0x80800080UL,0xA1AC2D8DUL,0xD3DC1FCFUL, +0xA1A02181UL,0x30303000UL,0x33343707UL,0xA2AC2E8EUL,0x32343606UL,0x11141505UL,0x22202202UL,0x30383808UL, +0xF0F434C4UL,0xA3A42787UL,0x41440545UL,0x404C0C4CUL,0x81800181UL,0xE1E829C9UL,0x80840484UL,0x93941787UL, +0x31343505UL,0xC3C80BCBUL,0xC2CC0ECEUL,0x303C3C0CUL,0x71703141UL,0x11101101UL,0xC3C407C7UL,0x81880989UL, +0x71743545UL,0xF3F83BCBUL,0xD2D81ACAUL,0xF0F838C8UL,0x90941484UL,0x51581949UL,0x82800282UL,0xC0C404C4UL, +0xF3FC3FCFUL,0x41480949UL,0x31383909UL,0x63642747UL,0xC0C000C0UL,0xC3CC0FCFUL,0xD3D417C7UL,0xB0B83888UL, +0x030C0F0FUL,0x828C0E8EUL,0x42400242UL,0x23202303UL,0x91901181UL,0x606C2C4CUL,0xD3D81BCBUL,0xA0A42484UL, +0x30343404UL,0xF1F031C1UL,0x40480848UL,0xC2C002C2UL,0x636C2F4FUL,0x313C3D0DUL,0x212C2D0DUL,0x40400040UL, +0xB2BC3E8EUL,0x323C3E0EUL,0xB0BC3C8CUL,0xC1C001C1UL,0xA2A82A8AUL,0xB2B83A8AUL,0x424C0E4EUL,0x51541545UL, +0x33383B0BUL,0xD0DC1CCCUL,0x60682848UL,0x737C3F4FUL,0x909C1C8CUL,0xD0D818C8UL,0x42480A4AUL,0x52541646UL, +0x73743747UL,0xA0A02080UL,0xE1EC2DCDUL,0x42440646UL,0xB1B43585UL,0x23282B0BUL,0x61642545UL,0xF2F83ACAUL, +0xE3E023C3UL,0xB1B83989UL,0xB1B03181UL,0x939C1F8FUL,0x525C1E4EUL,0xF1F839C9UL,0xE2E426C6UL,0xB2B03282UL, +0x31303101UL,0xE2E82ACAUL,0x616C2D4DUL,0x535C1F4FUL,0xE0E424C4UL,0xF0F030C0UL,0xC1CC0DCDUL,0x80880888UL, +0x12141606UL,0x32383A0AUL,0x50581848UL,0xD0D414C4UL,0x62602242UL,0x21282909UL,0x03040707UL,0x33303303UL, +0xE0E828C8UL,0x13181B0BUL,0x01040505UL,0x71783949UL,0x90901080UL,0x62682A4AUL,0x22282A0AUL,0x92981A8AUL +}; + +static const ulong32 SS3[256] = { +0x08303838UL,0xC8E0E828UL,0x0D212C2DUL,0x86A2A426UL,0xCFC3CC0FUL,0xCED2DC1EUL,0x83B3B033UL,0x88B0B838UL, +0x8FA3AC2FUL,0x40606020UL,0x45515415UL,0xC7C3C407UL,0x44404404UL,0x4F636C2FUL,0x4B63682BUL,0x4B53581BUL, +0xC3C3C003UL,0x42626022UL,0x03333033UL,0x85B1B435UL,0x09212829UL,0x80A0A020UL,0xC2E2E022UL,0x87A3A427UL, +0xC3D3D013UL,0x81919011UL,0x01111011UL,0x06020406UL,0x0C101C1CUL,0x8CB0BC3CUL,0x06323436UL,0x4B43480BUL, +0xCFE3EC2FUL,0x88808808UL,0x4C606C2CUL,0x88A0A828UL,0x07131417UL,0xC4C0C404UL,0x06121416UL,0xC4F0F434UL, +0xC2C2C002UL,0x45414405UL,0xC1E1E021UL,0xC6D2D416UL,0x0F333C3FUL,0x0D313C3DUL,0x8E828C0EUL,0x88909818UL, +0x08202828UL,0x4E424C0EUL,0xC6F2F436UL,0x0E323C3EUL,0x85A1A425UL,0xC9F1F839UL,0x0D010C0DUL,0xCFD3DC1FUL, +0xC8D0D818UL,0x0B23282BUL,0x46626426UL,0x4A72783AUL,0x07232427UL,0x0F232C2FUL,0xC1F1F031UL,0x42727032UL, +0x42424002UL,0xC4D0D414UL,0x41414001UL,0xC0C0C000UL,0x43737033UL,0x47636427UL,0x8CA0AC2CUL,0x8B83880BUL, +0xC7F3F437UL,0x8DA1AC2DUL,0x80808000UL,0x0F131C1FUL,0xCAC2C80AUL,0x0C202C2CUL,0x8AA2A82AUL,0x04303434UL, +0xC2D2D012UL,0x0B03080BUL,0xCEE2EC2EUL,0xC9E1E829UL,0x4D515C1DUL,0x84909414UL,0x08101818UL,0xC8F0F838UL, +0x47535417UL,0x8EA2AC2EUL,0x08000808UL,0xC5C1C405UL,0x03131013UL,0xCDC1CC0DUL,0x86828406UL,0x89B1B839UL, +0xCFF3FC3FUL,0x4D717C3DUL,0xC1C1C001UL,0x01313031UL,0xC5F1F435UL,0x8A82880AUL,0x4A62682AUL,0x81B1B031UL, +0xC1D1D011UL,0x00202020UL,0xC7D3D417UL,0x02020002UL,0x02222022UL,0x04000404UL,0x48606828UL,0x41717031UL, +0x07030407UL,0xCBD3D81BUL,0x8D919C1DUL,0x89919819UL,0x41616021UL,0x8EB2BC3EUL,0xC6E2E426UL,0x49515819UL, +0xCDD1DC1DUL,0x41515011UL,0x80909010UL,0xCCD0DC1CUL,0x8A92981AUL,0x83A3A023UL,0x8BA3A82BUL,0xC0D0D010UL, +0x81818001UL,0x0F030C0FUL,0x47434407UL,0x0A12181AUL,0xC3E3E023UL,0xCCE0EC2CUL,0x8D818C0DUL,0x8FB3BC3FUL, +0x86929416UL,0x4B73783BUL,0x4C505C1CUL,0x82A2A022UL,0x81A1A021UL,0x43636023UL,0x03232023UL,0x4D414C0DUL, +0xC8C0C808UL,0x8E929C1EUL,0x8C909C1CUL,0x0A32383AUL,0x0C000C0CUL,0x0E222C2EUL,0x8AB2B83AUL,0x4E626C2EUL, +0x8F939C1FUL,0x4A52581AUL,0xC2F2F032UL,0x82929012UL,0xC3F3F033UL,0x49414809UL,0x48707838UL,0xCCC0CC0CUL, +0x05111415UL,0xCBF3F83BUL,0x40707030UL,0x45717435UL,0x4F737C3FUL,0x05313435UL,0x00101010UL,0x03030003UL, +0x44606424UL,0x4D616C2DUL,0xC6C2C406UL,0x44707434UL,0xC5D1D415UL,0x84B0B434UL,0xCAE2E82AUL,0x09010809UL, +0x46727436UL,0x09111819UL,0xCEF2FC3EUL,0x40404000UL,0x02121012UL,0xC0E0E020UL,0x8DB1BC3DUL,0x05010405UL, +0xCAF2F83AUL,0x01010001UL,0xC0F0F030UL,0x0A22282AUL,0x4E525C1EUL,0x89A1A829UL,0x46525416UL,0x43434003UL, +0x85818405UL,0x04101414UL,0x89818809UL,0x8B93981BUL,0x80B0B030UL,0xC5E1E425UL,0x48404808UL,0x49717839UL, +0x87939417UL,0xCCF0FC3CUL,0x0E121C1EUL,0x82828002UL,0x01212021UL,0x8C808C0CUL,0x0B13181BUL,0x4F535C1FUL, +0x47737437UL,0x44505414UL,0x82B2B032UL,0x0D111C1DUL,0x05212425UL,0x4F434C0FUL,0x00000000UL,0x46424406UL, +0xCDE1EC2DUL,0x48505818UL,0x42525012UL,0xCBE3E82BUL,0x4E727C3EUL,0xCAD2D81AUL,0xC9C1C809UL,0xCDF1FC3DUL, +0x00303030UL,0x85919415UL,0x45616425UL,0x0C303C3CUL,0x86B2B436UL,0xC4E0E424UL,0x8BB3B83BUL,0x4C707C3CUL, +0x0E020C0EUL,0x40505010UL,0x09313839UL,0x06222426UL,0x02323032UL,0x84808404UL,0x49616829UL,0x83939013UL, +0x07333437UL,0xC7E3E427UL,0x04202424UL,0x84A0A424UL,0xCBC3C80BUL,0x43535013UL,0x0A02080AUL,0x87838407UL, +0xC9D1D819UL,0x4C404C0CUL,0x83838003UL,0x8F838C0FUL,0xCEC2CC0EUL,0x0B33383BUL,0x4A42480AUL,0x87B3B437UL +}; + +static const ulong32 KCi[16] = { +0x9E3779B9,0x3C6EF373, +0x78DDE6E6,0xF1BBCDCC, +0xE3779B99,0xC6EF3733, +0x8DDE6E67,0x1BBCDCCF, +0x3779B99E,0x6EF3733C, +0xDDE6E678,0xBBCDCCF1, +0x779B99E3,0xEF3733C6, +0xDE6E678D,0xBCDCCF1B +}; + +#define G(x) (SS3[((x)>>24)&255] ^ SS2[((x)>>16)&255] ^ SS1[((x)>>8)&255] ^ SS0[(x)&255]) + +#define F(L1, L2, R1, R2, K1, K2) \ + T2 = G((R1 ^ K1) ^ (R2 ^ K2)); \ + T = G( G(T2 + (R1 ^ K1)) + T2); \ + L2 ^= T; \ + L1 ^= (T + G(T2 + (R1 ^ K1))); \ + + /** + Initialize the SEED block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int kseed_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int i; + ulong32 tmp, k1, k2, k3, k4; + + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 16 && num_rounds != 0) { + return CRYPT_INVALID_ROUNDS; + } + + /* load key */ + LOAD32H(k1, key); + LOAD32H(k2, key+4); + LOAD32H(k3, key+8); + LOAD32H(k4, key+12); + + for (i = 0; i < 16; i++) { + skey->kseed.K[2*i+0] = G(k1 + k3 - KCi[i]); + skey->kseed.K[2*i+1] = G(k2 - k4 + KCi[i]); + if (i&1) { + tmp = k3; + k3 = ((k3 << 8) | (k4 >> 24)) & 0xFFFFFFFF; + k4 = ((k4 << 8) | (tmp >> 24)) & 0xFFFFFFFF; + } else { + tmp = k1; + k1 = ((k1 >> 8) | (k2 << 24)) & 0xFFFFFFFF; + k2 = ((k2 >> 8) | (tmp << 24)) & 0xFFFFFFFF; + } + /* reverse keys for decrypt */ + skey->kseed.dK[2*(15-i)+0] = skey->kseed.K[2*i+0]; + skey->kseed.dK[2*(15-i)+1] = skey->kseed.K[2*i+1]; + } + + return CRYPT_OK; +} + +static void rounds(ulong32 *P, const ulong32 *K) +{ + ulong32 T, T2; + int i; + for (i = 0; i < 16; i += 2) { + F(P[0], P[1], P[2], P[3], K[0], K[1]); + F(P[2], P[3], P[0], P[1], K[2], K[3]); + K += 4; + } +} + +/** + Encrypts a block of text with SEED + @param pt The input plaintext (16 bytes) + @param ct The output ciphertext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int kseed_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong32 P[4]; + LOAD32H(P[0], pt); + LOAD32H(P[1], pt+4); + LOAD32H(P[2], pt+8); + LOAD32H(P[3], pt+12); + rounds(P, skey->kseed.K); + STORE32H(P[2], ct); + STORE32H(P[3], ct+4); + STORE32H(P[0], ct+8); + STORE32H(P[1], ct+12); + return CRYPT_OK; +} + +/** + Decrypts a block of text with SEED + @param ct The input ciphertext (16 bytes) + @param pt The output plaintext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int kseed_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong32 P[4]; + LOAD32H(P[0], ct); + LOAD32H(P[1], ct+4); + LOAD32H(P[2], ct+8); + LOAD32H(P[3], ct+12); + rounds(P, skey->kseed.dK); + STORE32H(P[2], pt); + STORE32H(P[3], pt+4); + STORE32H(P[0], pt+8); + STORE32H(P[1], pt+12); + return CRYPT_OK; +} + +/** Terminate the context + @param skey The scheduled key +*/ +void kseed_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Performs a self-test of the SEED block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int kseed_test(void) +{ +#if !defined(LTC_TEST) + return CRYPT_NOP; +#else + static const struct test { + unsigned char pt[16], ct[16], key[16]; + } tests[] = { + +{ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F }, + { 0x5E,0xBA,0xC6,0xE0,0x05,0x4E,0x16,0x68,0x19,0xAF,0xF1,0xCC,0x6D,0x34,0x6C,0xDB }, + { 0 }, +}, + +{ + { 0 }, + { 0xC1,0x1F,0x22,0xF2,0x01,0x40,0x50,0x50,0x84,0x48,0x35,0x97,0xE4,0x37,0x0F,0x43 }, + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F }, +}, + +{ + { 0x83,0xA2,0xF8,0xA2,0x88,0x64,0x1F,0xB9,0xA4,0xE9,0xA5,0xCC,0x2F,0x13,0x1C,0x7D }, + { 0xEE,0x54,0xD1,0x3E,0xBC,0xAE,0x70,0x6D,0x22,0x6B,0xC3,0x14,0x2C,0xD4,0x0D,0x4A }, + { 0x47,0x06,0x48,0x08,0x51,0xE6,0x1B,0xE8,0x5D,0x74,0xBF,0xB3,0xFD,0x95,0x61,0x85 }, +}, + +{ + { 0xB4,0x1E,0x6B,0xE2,0xEB,0xA8,0x4A,0x14,0x8E,0x2E,0xED,0x84,0x59,0x3C,0x5E,0xC7 }, + { 0x9B,0x9B,0x7B,0xFC,0xD1,0x81,0x3C,0xB9,0x5D,0x0B,0x36,0x18,0xF4,0x0F,0x51,0x22 }, + { 0x28,0xDB,0xC3,0xBC,0x49,0xFF,0xD8,0x7D,0xCF,0xA5,0x09,0xB1,0x1D,0x42,0x2B,0xE7 }, +} +}; + int x; + unsigned char buf[2][16]; + symmetric_key skey; + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + kseed_setup(tests[x].key, 16, 0, &skey); + kseed_ecb_encrypt(tests[x].pt, buf[0], &skey); + kseed_ecb_decrypt(buf[0], buf[1], &skey); + if (compare_testvector(buf[0], 16, tests[x].ct, 16, "KSEED Encrypt", x) || + compare_testvector(buf[1], 16, tests[x].pt, 16, "KSEED Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int kseed_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize >= 16) { + *keysize = 16; + } else { + return CRYPT_INVALID_KEYSIZE; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/multi2.c b/optee_os/core/lib/libtomcrypt/src/ciphers/multi2.c new file mode 100644 index 0000000..e1a84ac --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/multi2.c @@ -0,0 +1,309 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file multi2.c + Multi-2 implementation (not public domain, hence the default disable) +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_MULTI2 + +static void s_pi1(ulong32 *p) +{ + p[1] ^= p[0]; +} + +static void s_pi2(ulong32 *p, const ulong32 *k) +{ + ulong32 t; + t = (p[1] + k[0]) & 0xFFFFFFFFUL; + t = (ROL(t, 1) + t - 1) & 0xFFFFFFFFUL; + t = (ROL(t, 4) ^ t) & 0xFFFFFFFFUL; + p[0] ^= t; +} + +static void s_pi3(ulong32 *p, const ulong32 *k) +{ + ulong32 t; + t = p[0] + k[1]; + t = (ROL(t, 2) + t + 1) & 0xFFFFFFFFUL; + t = (ROL(t, 8) ^ t) & 0xFFFFFFFFUL; + t = (t + k[2]) & 0xFFFFFFFFUL; + t = (ROL(t, 1) - t) & 0xFFFFFFFFUL; + t = ROL(t, 16) ^ (p[0] | t); + p[1] ^= t; +} + +static void s_pi4(ulong32 *p, const ulong32 *k) +{ + ulong32 t; + t = (p[1] + k[3]) & 0xFFFFFFFFUL; + t = (ROL(t, 2) + t + 1) & 0xFFFFFFFFUL; + p[0] ^= t; +} + +static void s_setup(const ulong32 *dk, const ulong32 *k, ulong32 *uk) +{ + int n, t; + ulong32 p[2]; + + p[0] = dk[0]; p[1] = dk[1]; + + t = 4; + n = 0; + s_pi1(p); + s_pi2(p, k); + uk[n++] = p[0]; + s_pi3(p, k); + uk[n++] = p[1]; + s_pi4(p, k); + uk[n++] = p[0]; + s_pi1(p); + uk[n++] = p[1]; + s_pi2(p, k+t); + uk[n++] = p[0]; + s_pi3(p, k+t); + uk[n++] = p[1]; + s_pi4(p, k+t); + uk[n++] = p[0]; + s_pi1(p); + uk[n++] = p[1]; +} + +static void s_encrypt(ulong32 *p, int N, const ulong32 *uk) +{ + int n, t; + for (t = n = 0; ; ) { + s_pi1(p); if (++n == N) break; + s_pi2(p, uk+t); if (++n == N) break; + s_pi3(p, uk+t); if (++n == N) break; + s_pi4(p, uk+t); if (++n == N) break; + t ^= 4; + } +} + +static void s_decrypt(ulong32 *p, int N, const ulong32 *uk) +{ + int n, t; + for (t = 4*(((N-1)>>2)&1), n = N; ; ) { + switch (n<=4 ? n : ((n-1)%4)+1) { + case 4: s_pi4(p, uk+t); --n; /* FALLTHROUGH */ + case 3: s_pi3(p, uk+t); --n; /* FALLTHROUGH */ + case 2: s_pi2(p, uk+t); --n; /* FALLTHROUGH */ + case 1: s_pi1(p); --n; break; + case 0: return; + } + t ^= 4; + } +} + +const struct ltc_cipher_descriptor multi2_desc = { + "multi2", + 22, + 40, 40, 8, 128, + &multi2_setup, + &multi2_ecb_encrypt, + &multi2_ecb_decrypt, + &multi2_test, + &multi2_done, + &multi2_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +int multi2_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + ulong32 sk[8], dk[2]; + int x; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (keylen != 40) return CRYPT_INVALID_KEYSIZE; + if (num_rounds == 0) num_rounds = 128; + + skey->multi2.N = num_rounds; + for (x = 0; x < 8; x++) { + LOAD32H(sk[x], key + x*4); + } + LOAD32H(dk[0], key + 32); + LOAD32H(dk[1], key + 36); + s_setup(dk, sk, skey->multi2.uk); + + zeromem(sk, sizeof(sk)); + zeromem(dk, sizeof(dk)); + return CRYPT_OK; +} + +/** + Encrypts a block of text with multi2 + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int multi2_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong32 p[2]; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + LOAD32H(p[0], pt); + LOAD32H(p[1], pt+4); + s_encrypt(p, skey->multi2.N, skey->multi2.uk); + STORE32H(p[0], ct); + STORE32H(p[1], ct+4); + return CRYPT_OK; +} + +/** + Decrypts a block of text with multi2 + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int multi2_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong32 p[2]; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + LOAD32H(p[0], ct); + LOAD32H(p[1], ct+4); + s_decrypt(p, skey->multi2.N, skey->multi2.uk); + STORE32H(p[0], pt); + STORE32H(p[1], pt+4); + return CRYPT_OK; +} + +/** + Performs a self-test of the multi2 block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int multi2_test(void) +{ + static const struct { + unsigned char key[40]; + unsigned char pt[8], ct[8]; + int rounds; + } tests[] = { +{ + { + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + 0x01, 0x23, 0x45, 0x67, + 0x89, 0xAB, 0xCD, 0xEF + }, + { + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }, + { + 0xf8, 0x94, 0x40, 0x84, + 0x5e, 0x11, 0xcf, 0x89 + }, + 128, +}, +{ + { + 0x35, 0x91, 0x9d, 0x96, + 0x07, 0x02, 0xe2, 0xce, + 0x8d, 0x0b, 0x58, 0x3c, + 0xc9, 0xc8, 0x9d, 0x59, + 0xa2, 0xae, 0x96, 0x4e, + 0x87, 0x82, 0x45, 0xed, + 0x3f, 0x2e, 0x62, 0xd6, + 0x36, 0x35, 0xd0, 0x67, + + 0xb1, 0x27, 0xb9, 0x06, + 0xe7, 0x56, 0x22, 0x38, + }, + { + 0x1f, 0xb4, 0x60, 0x60, + 0xd0, 0xb3, 0x4f, 0xa5 + }, + { + 0xca, 0x84, 0xa9, 0x34, + 0x75, 0xc8, 0x60, 0xe5 + }, + 216, +} +}; + unsigned char buf[8]; + symmetric_key skey; + int err, x; + + for (x = 1; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + if ((err = multi2_setup(tests[x].key, 40, tests[x].rounds, &skey)) != CRYPT_OK) { + return err; + } + if ((err = multi2_ecb_encrypt(tests[x].pt, buf, &skey)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(buf, 8, tests[x].ct, 8, "Multi2 Encrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + if ((err = multi2_ecb_decrypt(buf, buf, &skey)) != CRYPT_OK) { + return err; + } + if (compare_testvector(buf, 8, tests[x].pt, 8, "Multi2 Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + for (x = 128; x < 256; ++x) { + unsigned char ct[8]; + + if ((err = multi2_setup(tests[0].key, 40, x, &skey)) != CRYPT_OK) { + return err; + } + if ((err = multi2_ecb_encrypt(tests[0].pt, ct, &skey)) != CRYPT_OK) { + return err; + } + if ((err = multi2_ecb_decrypt(ct, buf, &skey)) != CRYPT_OK) { + return err; + } + if (compare_testvector(buf, 8, tests[0].pt, 8, "Multi2 Rounds", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + return CRYPT_OK; +} + +/** Terminate the context + @param skey The scheduled key +*/ +void multi2_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int multi2_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize >= 40) { + *keysize = 40; + } else { + return CRYPT_INVALID_KEYSIZE; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/noekeon.c b/optee_os/core/lib/libtomcrypt/src/ciphers/noekeon.c new file mode 100644 index 0000000..58d73ae --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/noekeon.c @@ -0,0 +1,317 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/** + @file noekeon.c + Implementation of the Noekeon block cipher by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_NOEKEON + +const struct ltc_cipher_descriptor noekeon_desc = +{ + "noekeon", + 16, + 16, 16, 16, 16, + &noekeon_setup, + &noekeon_ecb_encrypt, + &noekeon_ecb_decrypt, + &noekeon_test, + &noekeon_done, + &noekeon_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 RC[] = { + 0x00000080UL, 0x0000001bUL, 0x00000036UL, 0x0000006cUL, + 0x000000d8UL, 0x000000abUL, 0x0000004dUL, 0x0000009aUL, + 0x0000002fUL, 0x0000005eUL, 0x000000bcUL, 0x00000063UL, + 0x000000c6UL, 0x00000097UL, 0x00000035UL, 0x0000006aUL, + 0x000000d4UL +}; + +#define kTHETA(a, b, c, d) \ + temp = a^c; temp = temp ^ ROLc(temp, 8) ^ RORc(temp, 8); \ + b ^= temp; d ^= temp; \ + temp = b^d; temp = temp ^ ROLc(temp, 8) ^ RORc(temp, 8); \ + a ^= temp; c ^= temp; + +#define THETA(k, a, b, c, d) \ + temp = a^c; temp = temp ^ ROLc(temp, 8) ^ RORc(temp, 8); \ + b ^= temp ^ k[1]; d ^= temp ^ k[3]; \ + temp = b^d; temp = temp ^ ROLc(temp, 8) ^ RORc(temp, 8); \ + a ^= temp ^ k[0]; c ^= temp ^ k[2]; + +#define GAMMA(a, b, c, d) \ + b ^= ~(d|c); \ + a ^= c&b; \ + temp = d; d = a; a = temp;\ + c ^= a ^ b ^ d; \ + b ^= ~(d|c); \ + a ^= c&b; + +#define PI1(a, b, c, d) \ + b = ROLc(b, 1); c = ROLc(c, 5); d = ROLc(d, 2); + +#define PI2(a, b, c, d) \ + b = RORc(b, 1); c = RORc(c, 5); d = RORc(d, 2); + + /** + Initialize the Noekeon block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int noekeon_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + ulong32 temp; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 16 && num_rounds != 0) { + return CRYPT_INVALID_ROUNDS; + } + + LOAD32H(skey->noekeon.K[0],&key[0]); + LOAD32H(skey->noekeon.K[1],&key[4]); + LOAD32H(skey->noekeon.K[2],&key[8]); + LOAD32H(skey->noekeon.K[3],&key[12]); + + LOAD32H(skey->noekeon.dK[0],&key[0]); + LOAD32H(skey->noekeon.dK[1],&key[4]); + LOAD32H(skey->noekeon.dK[2],&key[8]); + LOAD32H(skey->noekeon.dK[3],&key[12]); + + kTHETA(skey->noekeon.dK[0], skey->noekeon.dK[1], skey->noekeon.dK[2], skey->noekeon.dK[3]); + + return CRYPT_OK; +} + +/** + Encrypts a block of text with Noekeon + @param pt The input plaintext (16 bytes) + @param ct The output ciphertext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_noekeon_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int noekeon_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + ulong32 a,b,c,d,temp; + int r; + + LTC_ARGCHK(skey != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + LOAD32H(a,&pt[0]); LOAD32H(b,&pt[4]); + LOAD32H(c,&pt[8]); LOAD32H(d,&pt[12]); + +#define ROUND(i) \ + a ^= RC[i]; \ + THETA(skey->noekeon.K, a,b,c,d); \ + PI1(a,b,c,d); \ + GAMMA(a,b,c,d); \ + PI2(a,b,c,d); + + for (r = 0; r < 16; ++r) { + ROUND(r); + } + +#undef ROUND + + a ^= RC[16]; + THETA(skey->noekeon.K, a, b, c, d); + + STORE32H(a,&ct[0]); STORE32H(b,&ct[4]); + STORE32H(c,&ct[8]); STORE32H(d,&ct[12]); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int noekeon_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_noekeon_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(ulong32) * 5 + sizeof(int)); + return err; +} +#endif + +/** + Decrypts a block of text with Noekeon + @param ct The input ciphertext (16 bytes) + @param pt The output plaintext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_noekeon_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int noekeon_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + ulong32 a,b,c,d, temp; + int r; + + LTC_ARGCHK(skey != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + LOAD32H(a,&ct[0]); LOAD32H(b,&ct[4]); + LOAD32H(c,&ct[8]); LOAD32H(d,&ct[12]); + + +#define ROUND(i) \ + THETA(skey->noekeon.dK, a,b,c,d); \ + a ^= RC[i]; \ + PI1(a,b,c,d); \ + GAMMA(a,b,c,d); \ + PI2(a,b,c,d); + + for (r = 16; r > 0; --r) { + ROUND(r); + } + +#undef ROUND + + THETA(skey->noekeon.dK, a,b,c,d); + a ^= RC[0]; + STORE32H(a,&pt[0]); STORE32H(b, &pt[4]); + STORE32H(c,&pt[8]); STORE32H(d, &pt[12]); + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int noekeon_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_noekeon_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(ulong32) * 5 + sizeof(int)); + return err; +} +#endif + +/** + Performs a self-test of the Noekeon block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int noekeon_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + int keylen; + unsigned char key[16], pt[16], ct[16]; + } tests[] = { + { + 16, + { 0xAA, 0x3C, 0x8C, 0x86, 0xD9, 0x8B, 0xF8, 0xBE, 0x21, 0xE0, 0x36, 0x09, 0x78, 0xFB, 0xE4, 0x90 }, + { 0xE4, 0x96, 0x6C, 0xD3, 0x13, 0xA0, 0x6C, 0xAF, 0xD0, 0x23, 0xC9, 0xFD, 0x45, 0x32, 0x23, 0x16 }, + { 0xA6, 0xEC, 0xB8, 0xA8, 0x61, 0xFD, 0x62, 0xD9, 0x13, 0x02, 0xFE, 0x9E, 0x47, 0x01, 0x3F, 0xC3 } + }, + { + 16, + { 0xED, 0x43, 0xD1, 0x87, 0x21, 0x7E, 0xE0, 0x97, 0x3D, 0x76, 0xC3, 0x37, 0x2E, 0x7D, 0xAE, 0xD3 }, + { 0xE3, 0x38, 0x32, 0xCC, 0xF2, 0x2F, 0x2F, 0x0A, 0x4A, 0x8B, 0x8F, 0x18, 0x12, 0x20, 0x17, 0xD3 }, + { 0x94, 0xA5, 0xDF, 0xF5, 0xAE, 0x1C, 0xBB, 0x22, 0xAD, 0xEB, 0xA7, 0x0D, 0xB7, 0x82, 0x90, 0xA0 } + }, + { + 16, + { 0x6F, 0xDC, 0x23, 0x38, 0xF2, 0x10, 0xFB, 0xD3, 0xC1, 0x8C, 0x02, 0xF6, 0xB4, 0x6A, 0xD5, 0xA8 }, + { 0xDB, 0x29, 0xED, 0xB5, 0x5F, 0xB3, 0x60, 0x3A, 0x92, 0xA8, 0xEB, 0x9C, 0x6D, 0x9D, 0x3E, 0x8F }, + { 0x78, 0xF3, 0x6F, 0xF8, 0x9E, 0xBB, 0x8C, 0x6A, 0xE8, 0x10, 0xF7, 0x00, 0x22, 0x15, 0x30, 0x3D } + }, + { + 16, + { 0x2C, 0x0C, 0x02, 0xEF, 0x6B, 0xC4, 0xF2, 0x0B, 0x2E, 0xB9, 0xE0, 0xBF, 0xD9, 0x36, 0xC2, 0x4E }, + { 0x84, 0xE2, 0xFE, 0x64, 0xB1, 0xB9, 0xFE, 0x76, 0xA8, 0x3F, 0x45, 0xC7, 0x40, 0x7A, 0xAF, 0xEE }, + { 0x2A, 0x08, 0xD6, 0xA2, 0x1C, 0x63, 0x08, 0xB0, 0xF8, 0xBC, 0xB3, 0xA1, 0x66, 0xF7, 0xAE, 0xCF } + }, + { + 16, + { 0x6F, 0x30, 0xF8, 0x9F, 0xDA, 0x6E, 0xA0, 0x91, 0x04, 0x0F, 0x6C, 0x8B, 0x7D, 0xF7, 0x2A, 0x4B }, + { 0x65, 0xB6, 0xA6, 0xD0, 0x42, 0x14, 0x08, 0x60, 0x34, 0x8D, 0x37, 0x2F, 0x01, 0xF0, 0x46, 0xBE }, + { 0x66, 0xAC, 0x0B, 0x62, 0x1D, 0x68, 0x11, 0xF5, 0x27, 0xB1, 0x13, 0x5D, 0xF3, 0x2A, 0xE9, 0x18 } + }, + { + 16, + { 0xCA, 0xA4, 0x16, 0xB7, 0x1C, 0x92, 0x2E, 0xAD, 0xEB, 0xA7, 0xDB, 0x69, 0x92, 0xCB, 0x35, 0xEF }, + { 0x81, 0x6F, 0x8E, 0x4D, 0x96, 0xC6, 0xB3, 0x67, 0x83, 0xF5, 0x63, 0xC7, 0x20, 0x6D, 0x40, 0x23 }, + { 0x44, 0xF7, 0x63, 0x62, 0xF0, 0x43, 0xBB, 0x67, 0x4A, 0x75, 0x12, 0x42, 0x46, 0x29, 0x28, 0x19 } + }, + { + 16, + { 0x6B, 0xCF, 0x22, 0x2F, 0xE0, 0x1B, 0xB0, 0xAA, 0xD8, 0x3C, 0x91, 0x99, 0x18, 0xB2, 0x28, 0xE8 }, + { 0x7C, 0x37, 0xC7, 0xD0, 0xAC, 0x92, 0x29, 0xF1, 0x60, 0x82, 0x93, 0x89, 0xAA, 0x61, 0xAA, 0xA9 }, + { 0xE5, 0x89, 0x1B, 0xB3, 0xFE, 0x8B, 0x0C, 0xA1, 0xA6, 0xC7, 0xBE, 0x12, 0x73, 0x0F, 0xC1, 0x19 } + }, + { + 16, + { 0xE6, 0xD0, 0xF1, 0x03, 0x2E, 0xDE, 0x70, 0x8D, 0xD8, 0x9E, 0x36, 0x5C, 0x05, 0x52, 0xE7, 0x0D }, + { 0xE2, 0x42, 0xE7, 0x92, 0x0E, 0xF7, 0x82, 0xA2, 0xB8, 0x21, 0x8D, 0x26, 0xBA, 0x2D, 0xE6, 0x32 }, + { 0x1E, 0xDD, 0x75, 0x22, 0xB9, 0x36, 0x8A, 0x0F, 0x32, 0xFD, 0xD4, 0x48, 0x65, 0x12, 0x5A, 0x2F } + } + }; + symmetric_key key; + unsigned char tmp[2][16]; + int err, i, y; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + zeromem(&key, sizeof(key)); + if ((err = noekeon_setup(tests[i].key, tests[i].keylen, 0, &key)) != CRYPT_OK) { + return err; + } + + noekeon_ecb_encrypt(tests[i].pt, tmp[0], &key); + noekeon_ecb_decrypt(tmp[0], tmp[1], &key); + if (compare_testvector(tmp[0], 16, tests[i].ct, 16, "Noekeon Encrypt", i) || + compare_testvector(tmp[1], 16, tests[i].pt, 16, "Noekeon Decrypt", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 16; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) noekeon_ecb_encrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 1000; y++) noekeon_ecb_decrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 16; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void noekeon_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int noekeon_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + *keysize = 16; + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/rc2.c b/optee_os/core/lib/libtomcrypt/src/ciphers/rc2.c new file mode 100644 index 0000000..4832424 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/rc2.c @@ -0,0 +1,408 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/**********************************************************************\ +* To commemorate the 1996 RSA Data Security Conference, the following * +* code is released into the public domain by its author. Prost! * +* * +* This cipher uses 16-bit words and little-endian byte ordering. * +* I wonder which processor it was optimized for? * +* * +* Thanks to CodeView, SoftIce, and D86 for helping bring this code to * +* the public. * +\**********************************************************************/ +#include "tomcrypt_private.h" + +/** + @file rc2.c + Implementation of RC2 with fixed effective key length of 64bits +*/ + +#ifdef LTC_RC2 + +const struct ltc_cipher_descriptor rc2_desc = { + "rc2", + 12, 8, 128, 8, 16, + &rc2_setup, + &rc2_ecb_encrypt, + &rc2_ecb_decrypt, + &rc2_test, + &rc2_done, + &rc2_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +/* 256-entry permutation table, probably derived somehow from pi */ +static const unsigned char permute[256] = { + 217,120,249,196, 25,221,181,237, 40,233,253,121, 74,160,216,157, + 198,126, 55,131, 43,118, 83,142, 98, 76,100,136, 68,139,251,162, + 23,154, 89,245,135,179, 79, 19, 97, 69,109,141, 9,129,125, 50, + 189,143, 64,235,134,183,123, 11,240,149, 33, 34, 92,107, 78,130, + 84,214,101,147,206, 96,178, 28,115, 86,192, 20,167,140,241,220, + 18,117,202, 31, 59,190,228,209, 66, 61,212, 48,163, 60,182, 38, + 111,191, 14,218, 70,105, 7, 87, 39,242, 29,155,188,148, 67, 3, + 248, 17,199,246,144,239, 62,231, 6,195,213, 47,200,102, 30,215, + 8,232,234,222,128, 82,238,247,132,170,114,172, 53, 77,106, 42, + 150, 26,210,113, 90, 21, 73,116, 75,159,208, 94, 4, 24,164,236, + 194,224, 65,110, 15, 81,203,204, 36,145,175, 80,161,244,112, 57, + 153,124, 58,133, 35,184,180,122,252, 2, 54, 91, 37, 85,151, 49, + 45, 93,250,152,227,138,146,174, 5,223, 41, 16,103,108,186,201, + 211, 0,230,207,225,158,168, 44, 99, 22, 1, 63, 88,226,137,169, + 13, 56, 52, 27,171, 51,255,176,187, 72, 12, 95,185,177,205, 46, + 197,243,219, 71,229,165,156,119, 10,166, 32,104,254,127,193,173 +}; + + /** + Initialize the RC2 block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param bits The effective key length in bits + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int rc2_setup_ex(const unsigned char *key, int keylen, int bits, int num_rounds, symmetric_key *skey) +{ + unsigned *xkey = skey->rc2.xkey; + unsigned char tmp[128]; + unsigned T8, TM; + int i; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (keylen == 0 || keylen > 128 || bits > 1024) { + return CRYPT_INVALID_KEYSIZE; + } + if (bits == 0) { + bits = 1024; + } + + if (num_rounds != 0 && num_rounds != 16) { + return CRYPT_INVALID_ROUNDS; + } + + for (i = 0; i < keylen; i++) { + tmp[i] = key[i] & 255; + } + + /* Phase 1: Expand input key to 128 bytes */ + if (keylen < 128) { + for (i = keylen; i < 128; i++) { + tmp[i] = permute[(tmp[i - 1] + tmp[i - keylen]) & 255]; + } + } + + /* Phase 2 - reduce effective key size to "bits" */ + T8 = (unsigned)(bits+7)>>3; + TM = (255 >> (unsigned)(7 & -bits)); + tmp[128 - T8] = permute[tmp[128 - T8] & TM]; + for (i = 127 - T8; i >= 0; i--) { + tmp[i] = permute[tmp[i + 1] ^ tmp[i + T8]]; + } + + /* Phase 3 - copy to xkey in little-endian order */ + for (i = 0; i < 64; i++) { + xkey[i] = (unsigned)tmp[2*i] + ((unsigned)tmp[2*i+1] << 8); + } + +#ifdef LTC_CLEAN_STACK + zeromem(tmp, sizeof(tmp)); +#endif + + return CRYPT_OK; +} + +/** + Initialize the RC2 block cipher + + The effective key length is here always keylen * 8 + + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful +*/ +int rc2_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + return rc2_setup_ex(key, keylen, keylen * 8, num_rounds, skey); +} + +/**********************************************************************\ +* Encrypt an 8-byte block of plaintext using the given key. * +\**********************************************************************/ +/** + Encrypts a block of text with RC2 + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_rc2_ecb_encrypt( const unsigned char *pt, + unsigned char *ct, + const symmetric_key *skey) +#else +int rc2_ecb_encrypt( const unsigned char *pt, + unsigned char *ct, + const symmetric_key *skey) +#endif +{ + const unsigned *xkey; + unsigned x76, x54, x32, x10, i; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + xkey = skey->rc2.xkey; + + x76 = ((unsigned)pt[7] << 8) + (unsigned)pt[6]; + x54 = ((unsigned)pt[5] << 8) + (unsigned)pt[4]; + x32 = ((unsigned)pt[3] << 8) + (unsigned)pt[2]; + x10 = ((unsigned)pt[1] << 8) + (unsigned)pt[0]; + + for (i = 0; i < 16; i++) { + x10 = (x10 + (x32 & ~x76) + (x54 & x76) + xkey[4*i+0]) & 0xFFFF; + x10 = ((x10 << 1) | (x10 >> 15)); + + x32 = (x32 + (x54 & ~x10) + (x76 & x10) + xkey[4*i+1]) & 0xFFFF; + x32 = ((x32 << 2) | (x32 >> 14)); + + x54 = (x54 + (x76 & ~x32) + (x10 & x32) + xkey[4*i+2]) & 0xFFFF; + x54 = ((x54 << 3) | (x54 >> 13)); + + x76 = (x76 + (x10 & ~x54) + (x32 & x54) + xkey[4*i+3]) & 0xFFFF; + x76 = ((x76 << 5) | (x76 >> 11)); + + if (i == 4 || i == 10) { + x10 = (x10 + xkey[x76 & 63]) & 0xFFFF; + x32 = (x32 + xkey[x10 & 63]) & 0xFFFF; + x54 = (x54 + xkey[x32 & 63]) & 0xFFFF; + x76 = (x76 + xkey[x54 & 63]) & 0xFFFF; + } + } + + ct[0] = (unsigned char)x10; + ct[1] = (unsigned char)(x10 >> 8); + ct[2] = (unsigned char)x32; + ct[3] = (unsigned char)(x32 >> 8); + ct[4] = (unsigned char)x54; + ct[5] = (unsigned char)(x54 >> 8); + ct[6] = (unsigned char)x76; + ct[7] = (unsigned char)(x76 >> 8); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc2_ecb_encrypt( const unsigned char *pt, + unsigned char *ct, + const symmetric_key *skey) +{ + int err = s_rc2_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(unsigned *) + sizeof(unsigned) * 5); + return err; +} +#endif + +/**********************************************************************\ +* Decrypt an 8-byte block of ciphertext using the given key. * +\**********************************************************************/ +/** + Decrypts a block of text with RC2 + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_rc2_ecb_decrypt( const unsigned char *ct, + unsigned char *pt, + const symmetric_key *skey) +#else +int rc2_ecb_decrypt( const unsigned char *ct, + unsigned char *pt, + const symmetric_key *skey) +#endif +{ + unsigned x76, x54, x32, x10; + const unsigned *xkey; + int i; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + xkey = skey->rc2.xkey; + + x76 = ((unsigned)ct[7] << 8) + (unsigned)ct[6]; + x54 = ((unsigned)ct[5] << 8) + (unsigned)ct[4]; + x32 = ((unsigned)ct[3] << 8) + (unsigned)ct[2]; + x10 = ((unsigned)ct[1] << 8) + (unsigned)ct[0]; + + for (i = 15; i >= 0; i--) { + if (i == 4 || i == 10) { + x76 = (x76 - xkey[x54 & 63]) & 0xFFFF; + x54 = (x54 - xkey[x32 & 63]) & 0xFFFF; + x32 = (x32 - xkey[x10 & 63]) & 0xFFFF; + x10 = (x10 - xkey[x76 & 63]) & 0xFFFF; + } + + x76 = ((x76 << 11) | (x76 >> 5)); + x76 = (x76 - ((x10 & ~x54) + (x32 & x54) + xkey[4*i+3])) & 0xFFFF; + + x54 = ((x54 << 13) | (x54 >> 3)); + x54 = (x54 - ((x76 & ~x32) + (x10 & x32) + xkey[4*i+2])) & 0xFFFF; + + x32 = ((x32 << 14) | (x32 >> 2)); + x32 = (x32 - ((x54 & ~x10) + (x76 & x10) + xkey[4*i+1])) & 0xFFFF; + + x10 = ((x10 << 15) | (x10 >> 1)); + x10 = (x10 - ((x32 & ~x76) + (x54 & x76) + xkey[4*i+0])) & 0xFFFF; + } + + pt[0] = (unsigned char)x10; + pt[1] = (unsigned char)(x10 >> 8); + pt[2] = (unsigned char)x32; + pt[3] = (unsigned char)(x32 >> 8); + pt[4] = (unsigned char)x54; + pt[5] = (unsigned char)(x54 >> 8); + pt[6] = (unsigned char)x76; + pt[7] = (unsigned char)(x76 >> 8); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc2_ecb_decrypt( const unsigned char *ct, + unsigned char *pt, + const symmetric_key *skey) +{ + int err = s_rc2_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(unsigned *) + sizeof(unsigned) * 4 + sizeof(int)); + return err; +} +#endif + +/** + Performs a self-test of the RC2 block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int rc2_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + int keylen, bits; + unsigned char key[16], pt[8], ct[8]; + } tests[] = { + + { 8, 63, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xeb, 0xb7, 0x73, 0xf9, 0x93, 0x27, 0x8e, 0xff } + }, + { 8, 64, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + { 0x27, 0x8b, 0x27, 0xe4, 0x2e, 0x2f, 0x0d, 0x49 } + }, + { 8, 64, + { 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + { 0x30, 0x64, 0x9e, 0xdf, 0x9b, 0xe7, 0xd2, 0xc2 } + }, + { 1, 64, + { 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x61, 0xa8, 0xa2, 0x44, 0xad, 0xac, 0xcc, 0xf0 } + }, + { 7, 64, + { 0x88, 0xbc, 0xa9, 0x0e, 0x90, 0x87, 0x5a, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x6c, 0xcf, 0x43, 0x08, 0x97, 0x4c, 0x26, 0x7f } + }, + { 16, 64, + { 0x88, 0xbc, 0xa9, 0x0e, 0x90, 0x87, 0x5a, 0x7f, + 0x0f, 0x79, 0xc3, 0x84, 0x62, 0x7b, 0xaf, 0xb2 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x1a, 0x80, 0x7d, 0x27, 0x2b, 0xbe, 0x5d, 0xb1 } + }, + { 16, 128, + { 0x88, 0xbc, 0xa9, 0x0e, 0x90, 0x87, 0x5a, 0x7f, + 0x0f, 0x79, 0xc3, 0x84, 0x62, 0x7b, 0xaf, 0xb2 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x22, 0x69, 0x55, 0x2a, 0xb0, 0xf8, 0x5c, 0xa6 } + } + }; + int x, y, err; + symmetric_key skey; + unsigned char tmp[2][8]; + + for (x = 0; x < (int)(sizeof(tests) / sizeof(tests[0])); x++) { + zeromem(tmp, sizeof(tmp)); + if (tests[x].bits == (tests[x].keylen * 8)) { + if ((err = rc2_setup(tests[x].key, tests[x].keylen, 0, &skey)) != CRYPT_OK) { + return err; + } + } + else { + if ((err = rc2_setup_ex(tests[x].key, tests[x].keylen, tests[x].bits, 0, &skey)) != CRYPT_OK) { + return err; + } + } + + rc2_ecb_encrypt(tests[x].pt, tmp[0], &skey); + rc2_ecb_decrypt(tmp[0], tmp[1], &skey); + + if (compare_testvector(tmp[0], 8, tests[x].ct, 8, "RC2 CT", x) || + compare_testvector(tmp[1], 8, tests[x].pt, 8, "RC2 PT", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) rc2_ecb_encrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 1000; y++) rc2_ecb_decrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 8; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void rc2_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int rc2_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 1) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize > 128) { + *keysize = 128; + } + return CRYPT_OK; +} + +#endif + + + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/rc5.c b/optee_os/core/lib/libtomcrypt/src/ciphers/rc5.c new file mode 100644 index 0000000..f21ba4f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/rc5.c @@ -0,0 +1,323 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file rc5.c + LTC_RC5 code by Tom St Denis +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_RC5 + +const struct ltc_cipher_descriptor rc5_desc = +{ + "rc5", + 2, + 8, 128, 8, 12, + &rc5_setup, + &rc5_ecb_encrypt, + &rc5_ecb_decrypt, + &rc5_test, + &rc5_done, + &rc5_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 stab[50] = { +0xb7e15163UL, 0x5618cb1cUL, 0xf45044d5UL, 0x9287be8eUL, 0x30bf3847UL, 0xcef6b200UL, 0x6d2e2bb9UL, 0x0b65a572UL, +0xa99d1f2bUL, 0x47d498e4UL, 0xe60c129dUL, 0x84438c56UL, 0x227b060fUL, 0xc0b27fc8UL, 0x5ee9f981UL, 0xfd21733aUL, +0x9b58ecf3UL, 0x399066acUL, 0xd7c7e065UL, 0x75ff5a1eUL, 0x1436d3d7UL, 0xb26e4d90UL, 0x50a5c749UL, 0xeedd4102UL, +0x8d14babbUL, 0x2b4c3474UL, 0xc983ae2dUL, 0x67bb27e6UL, 0x05f2a19fUL, 0xa42a1b58UL, 0x42619511UL, 0xe0990ecaUL, +0x7ed08883UL, 0x1d08023cUL, 0xbb3f7bf5UL, 0x5976f5aeUL, 0xf7ae6f67UL, 0x95e5e920UL, 0x341d62d9UL, 0xd254dc92UL, +0x708c564bUL, 0x0ec3d004UL, 0xacfb49bdUL, 0x4b32c376UL, 0xe96a3d2fUL, 0x87a1b6e8UL, 0x25d930a1UL, 0xc410aa5aUL, +0x62482413UL, 0x007f9dccUL +}; + + /** + Initialize the LTC_RC5 block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +#ifdef LTC_CLEAN_STACK +static int s_rc5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#else +int rc5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#endif +{ + ulong32 L[64], *S, A, B, i, j, v, s, t, l; + + LTC_ARGCHK(skey != NULL); + LTC_ARGCHK(key != NULL); + + /* test parameters */ + if (num_rounds == 0) { + num_rounds = rc5_desc.default_rounds; + } + + if (num_rounds < 12 || num_rounds > 24) { + return CRYPT_INVALID_ROUNDS; + } + + /* key must be between 64 and 1024 bits */ + if (keylen < 8 || keylen > 128) { + return CRYPT_INVALID_KEYSIZE; + } + + skey->rc5.rounds = num_rounds; + S = skey->rc5.K; + + /* copy the key into the L array */ + for (A = i = j = 0; i < (ulong32)keylen; ) { + A = (A << 8) | ((ulong32)(key[i++] & 255)); + if ((i & 3) == 0) { + L[j++] = BSWAP(A); + A = 0; + } + } + + if ((keylen & 3) != 0) { + A <<= (ulong32)((8 * (4 - (keylen&3)))); + L[j++] = BSWAP(A); + } + + /* setup the S array */ + t = (ulong32)(2 * (num_rounds + 1)); + XMEMCPY(S, stab, t * sizeof(*S)); + + /* mix buffer */ + s = 3 * MAX(t, j); + l = j; + for (A = B = i = j = v = 0; v < s; v++) { + A = S[i] = ROLc(S[i] + A + B, 3); + B = L[j] = ROL(L[j] + A + B, (A+B)); + if (++i == t) { i = 0; } + if (++j == l) { j = 0; } + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int x; + x = s_rc5_setup(key, keylen, num_rounds, skey); + burn_stack(sizeof(ulong32) * 122 + sizeof(int)); + return x; +} +#endif + +/** + Encrypts a block of text with LTC_RC5 + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_rc5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int rc5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + ulong32 A, B; + const ulong32 *K; + int r; + LTC_ARGCHK(skey != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + if (skey->rc5.rounds < 12 || skey->rc5.rounds > 24) { + return CRYPT_INVALID_ROUNDS; + } + + LOAD32L(A, &pt[0]); + LOAD32L(B, &pt[4]); + A += skey->rc5.K[0]; + B += skey->rc5.K[1]; + K = skey->rc5.K + 2; + + if ((skey->rc5.rounds & 1) == 0) { + for (r = 0; r < skey->rc5.rounds; r += 2) { + A = ROL(A ^ B, B) + K[0]; + B = ROL(B ^ A, A) + K[1]; + A = ROL(A ^ B, B) + K[2]; + B = ROL(B ^ A, A) + K[3]; + K += 4; + } + } else { + for (r = 0; r < skey->rc5.rounds; r++) { + A = ROL(A ^ B, B) + K[0]; + B = ROL(B ^ A, A) + K[1]; + K += 2; + } + } + STORE32L(A, &ct[0]); + STORE32L(B, &ct[4]); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_rc5_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(ulong32) * 2 + sizeof(int)); + return err; +} +#endif + +/** + Decrypts a block of text with LTC_RC5 + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_rc5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int rc5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + ulong32 A, B; + const ulong32 *K; + int r; + LTC_ARGCHK(skey != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + if (skey->rc5.rounds < 12 || skey->rc5.rounds > 24) { + return CRYPT_INVALID_ROUNDS; + } + + LOAD32L(A, &ct[0]); + LOAD32L(B, &ct[4]); + K = skey->rc5.K + (skey->rc5.rounds << 1); + + if ((skey->rc5.rounds & 1) == 0) { + K -= 2; + for (r = skey->rc5.rounds - 1; r >= 0; r -= 2) { + B = ROR(B - K[3], A) ^ A; + A = ROR(A - K[2], B) ^ B; + B = ROR(B - K[1], A) ^ A; + A = ROR(A - K[0], B) ^ B; + K -= 4; + } + } else { + for (r = skey->rc5.rounds - 1; r >= 0; r--) { + B = ROR(B - K[1], A) ^ A; + A = ROR(A - K[0], B) ^ B; + K -= 2; + } + } + A -= skey->rc5.K[0]; + B -= skey->rc5.K[1]; + STORE32L(A, &pt[0]); + STORE32L(B, &pt[4]); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_rc5_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(ulong32) * 2 + sizeof(int)); + return err; +} +#endif + +/** + Performs a self-test of the LTC_RC5 block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int rc5_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + unsigned char key[16], pt[8], ct[8]; + } tests[] = { + { + { 0x91, 0x5f, 0x46, 0x19, 0xbe, 0x41, 0xb2, 0x51, + 0x63, 0x55, 0xa5, 0x01, 0x10, 0xa9, 0xce, 0x91 }, + { 0x21, 0xa5, 0xdb, 0xee, 0x15, 0x4b, 0x8f, 0x6d }, + { 0xf7, 0xc0, 0x13, 0xac, 0x5b, 0x2b, 0x89, 0x52 } + }, + { + { 0x78, 0x33, 0x48, 0xe7, 0x5a, 0xeb, 0x0f, 0x2f, + 0xd7, 0xb1, 0x69, 0xbb, 0x8d, 0xc1, 0x67, 0x87 }, + { 0xF7, 0xC0, 0x13, 0xAC, 0x5B, 0x2B, 0x89, 0x52 }, + { 0x2F, 0x42, 0xB3, 0xB7, 0x03, 0x69, 0xFC, 0x92 } + }, + { + { 0xDC, 0x49, 0xdb, 0x13, 0x75, 0xa5, 0x58, 0x4f, + 0x64, 0x85, 0xb4, 0x13, 0xb5, 0xf1, 0x2b, 0xaf }, + { 0x2F, 0x42, 0xB3, 0xB7, 0x03, 0x69, 0xFC, 0x92 }, + { 0x65, 0xc1, 0x78, 0xb2, 0x84, 0xd1, 0x97, 0xcc } + } + }; + unsigned char tmp[2][8]; + int x, y, err; + symmetric_key key; + + for (x = 0; x < (int)(sizeof(tests) / sizeof(tests[0])); x++) { + /* setup key */ + if ((err = rc5_setup(tests[x].key, 16, 12, &key)) != CRYPT_OK) { + return err; + } + + /* encrypt and decrypt */ + rc5_ecb_encrypt(tests[x].pt, tmp[0], &key); + rc5_ecb_decrypt(tmp[0], tmp[1], &key); + + /* compare */ + if (compare_testvector(tmp[0], 8, tests[x].ct, 8, "RC5 Encrypt", x) != 0 || + compare_testvector(tmp[1], 8, tests[x].pt, 8, "RC5 Decrypt", x) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) rc5_ecb_encrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 1000; y++) rc5_ecb_decrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 8; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void rc5_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int rc5_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 8) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize > 128) { + *keysize = 128; + } + return CRYPT_OK; +} + +#endif + + + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/rc6.c b/optee_os/core/lib/libtomcrypt/src/ciphers/rc6.c new file mode 100644 index 0000000..d1341d3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/rc6.c @@ -0,0 +1,324 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file rc6.c + LTC_RC6 code by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_RC6 + +const struct ltc_cipher_descriptor rc6_desc = +{ + "rc6", + 3, + 8, 128, 16, 20, + &rc6_setup, + &rc6_ecb_encrypt, + &rc6_ecb_decrypt, + &rc6_test, + &rc6_done, + &rc6_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const ulong32 stab[44] = { +0xb7e15163UL, 0x5618cb1cUL, 0xf45044d5UL, 0x9287be8eUL, 0x30bf3847UL, 0xcef6b200UL, 0x6d2e2bb9UL, 0x0b65a572UL, +0xa99d1f2bUL, 0x47d498e4UL, 0xe60c129dUL, 0x84438c56UL, 0x227b060fUL, 0xc0b27fc8UL, 0x5ee9f981UL, 0xfd21733aUL, +0x9b58ecf3UL, 0x399066acUL, 0xd7c7e065UL, 0x75ff5a1eUL, 0x1436d3d7UL, 0xb26e4d90UL, 0x50a5c749UL, 0xeedd4102UL, +0x8d14babbUL, 0x2b4c3474UL, 0xc983ae2dUL, 0x67bb27e6UL, 0x05f2a19fUL, 0xa42a1b58UL, 0x42619511UL, 0xe0990ecaUL, +0x7ed08883UL, 0x1d08023cUL, 0xbb3f7bf5UL, 0x5976f5aeUL, 0xf7ae6f67UL, 0x95e5e920UL, 0x341d62d9UL, 0xd254dc92UL, +0x708c564bUL, 0x0ec3d004UL, 0xacfb49bdUL, 0x4b32c376UL }; + + /** + Initialize the LTC_RC6 block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +#ifdef LTC_CLEAN_STACK +static int s_rc6_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#else +int rc6_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#endif +{ + ulong32 L[64], S[50], A, B, i, j, v, s, l; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* test parameters */ + if (num_rounds != 0 && num_rounds != 20) { + return CRYPT_INVALID_ROUNDS; + } + + /* key must be between 64 and 1024 bits */ + if (keylen < 8 || keylen > 128) { + return CRYPT_INVALID_KEYSIZE; + } + + /* copy the key into the L array */ + for (A = i = j = 0; i < (ulong32)keylen; ) { + A = (A << 8) | ((ulong32)(key[i++] & 255)); + if (!(i & 3)) { + L[j++] = BSWAP(A); + A = 0; + } + } + + /* handle odd sized keys */ + if (keylen & 3) { + A <<= (8 * (4 - (keylen&3))); + L[j++] = BSWAP(A); + } + + /* setup the S array */ + XMEMCPY(S, stab, 44 * sizeof(stab[0])); + + /* mix buffer */ + s = 3 * MAX(44, j); + l = j; + for (A = B = i = j = v = 0; v < s; v++) { + A = S[i] = ROLc(S[i] + A + B, 3); + B = L[j] = ROL(L[j] + A + B, (A+B)); + if (++i == 44) { i = 0; } + if (++j == l) { j = 0; } + } + + /* copy to key */ + for (i = 0; i < 44; i++) { + skey->rc6.K[i] = S[i]; + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc6_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int x; + x = s_rc6_setup(key, keylen, num_rounds, skey); + burn_stack(sizeof(ulong32) * 122); + return x; +} +#endif + +/** + Encrypts a block of text with LTC_RC6 + @param pt The input plaintext (16 bytes) + @param ct The output ciphertext (16 bytes) + @param skey The key as scheduled +*/ +#ifdef LTC_CLEAN_STACK +static int s_rc6_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int rc6_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + ulong32 a,b,c,d,t,u; + const ulong32 *K; + int r; + + LTC_ARGCHK(skey != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LOAD32L(a,&pt[0]);LOAD32L(b,&pt[4]);LOAD32L(c,&pt[8]);LOAD32L(d,&pt[12]); + + b += skey->rc6.K[0]; + d += skey->rc6.K[1]; + +#define RND(a,b,c,d) \ + t = (b * (b + b + 1)); t = ROLc(t, 5); \ + u = (d * (d + d + 1)); u = ROLc(u, 5); \ + a = ROL(a^t,u) + K[0]; \ + c = ROL(c^u,t) + K[1]; K += 2; + + K = skey->rc6.K + 2; + for (r = 0; r < 20; r += 4) { + RND(a,b,c,d); + RND(b,c,d,a); + RND(c,d,a,b); + RND(d,a,b,c); + } + +#undef RND + + a += skey->rc6.K[42]; + c += skey->rc6.K[43]; + STORE32L(a,&ct[0]);STORE32L(b,&ct[4]);STORE32L(c,&ct[8]);STORE32L(d,&ct[12]); + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc6_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_rc6_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(ulong32) * 6 + sizeof(int)); + return err; +} +#endif + +/** + Decrypts a block of text with LTC_RC6 + @param ct The input ciphertext (16 bytes) + @param pt The output plaintext (16 bytes) + @param skey The key as scheduled +*/ +#ifdef LTC_CLEAN_STACK +static int s_rc6_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int rc6_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + ulong32 a,b,c,d,t,u; + const ulong32 *K; + int r; + + LTC_ARGCHK(skey != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + LOAD32L(a,&ct[0]);LOAD32L(b,&ct[4]);LOAD32L(c,&ct[8]);LOAD32L(d,&ct[12]); + a -= skey->rc6.K[42]; + c -= skey->rc6.K[43]; + +#define RND(a,b,c,d) \ + t = (b * (b + b + 1)); t = ROLc(t, 5); \ + u = (d * (d + d + 1)); u = ROLc(u, 5); \ + c = ROR(c - K[1], t) ^ u; \ + a = ROR(a - K[0], u) ^ t; K -= 2; + + K = skey->rc6.K + 40; + + for (r = 0; r < 20; r += 4) { + RND(d,a,b,c); + RND(c,d,a,b); + RND(b,c,d,a); + RND(a,b,c,d); + } + +#undef RND + + b -= skey->rc6.K[0]; + d -= skey->rc6.K[1]; + STORE32L(a,&pt[0]);STORE32L(b,&pt[4]);STORE32L(c,&pt[8]);STORE32L(d,&pt[12]); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int rc6_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_rc6_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(ulong32) * 6 + sizeof(int)); + return err; +} +#endif + +/** + Performs a self-test of the LTC_RC6 block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int rc6_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + int keylen; + unsigned char key[32], pt[16], ct[16]; + } tests[] = { + { + 16, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x78, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x02, 0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, + 0x8a, 0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1 }, + { 0x52, 0x4e, 0x19, 0x2f, 0x47, 0x15, 0xc6, 0x23, + 0x1f, 0x51, 0xf6, 0x36, 0x7e, 0xa4, 0x3f, 0x18 } + }, + { + 24, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x78, + 0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x02, 0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, + 0x8a, 0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1 }, + { 0x68, 0x83, 0x29, 0xd0, 0x19, 0xe5, 0x05, 0x04, + 0x1e, 0x52, 0xe9, 0x2a, 0xf9, 0x52, 0x91, 0xd4 } + }, + { + 32, + { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x78, + 0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf0, + 0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe }, + { 0x02, 0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, + 0x8a, 0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1 }, + { 0xc8, 0x24, 0x18, 0x16, 0xf0, 0xd7, 0xe4, 0x89, + 0x20, 0xad, 0x16, 0xa1, 0x67, 0x4e, 0x5d, 0x48 } + } + }; + unsigned char tmp[2][16]; + int x, y, err; + symmetric_key key; + + for (x = 0; x < (int)(sizeof(tests) / sizeof(tests[0])); x++) { + /* setup key */ + if ((err = rc6_setup(tests[x].key, tests[x].keylen, 0, &key)) != CRYPT_OK) { + return err; + } + + /* encrypt and decrypt */ + rc6_ecb_encrypt(tests[x].pt, tmp[0], &key); + rc6_ecb_decrypt(tmp[0], tmp[1], &key); + + /* compare */ + if (compare_testvector(tmp[0], 16, tests[x].ct, 16, "RC6 Encrypt", x) || + compare_testvector(tmp[1], 16, tests[x].pt, 16, "RC6 Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 16; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) rc6_ecb_encrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 1000; y++) rc6_ecb_decrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 16; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void rc6_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int rc6_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 8) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize > 128) { + *keysize = 128; + } + return CRYPT_OK; +} + +#endif /*LTC_RC6*/ + + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/safer/safer.c b/optee_os/core/lib/libtomcrypt/src/ciphers/safer/safer.c new file mode 100644 index 0000000..84cc2d0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/safer/safer.c @@ -0,0 +1,484 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/******************************************************************************* +* +* FILE: safer.c +* +* LTC_DESCRIPTION: block-cipher algorithm LTC_SAFER (Secure And Fast Encryption +* Routine) in its four versions: LTC_SAFER K-64, LTC_SAFER K-128, +* LTC_SAFER SK-64 and LTC_SAFER SK-128. +* +* AUTHOR: Richard De Moliner (demoliner@isi.ee.ethz.ch) +* Signal and Information Processing Laboratory +* Swiss Federal Institute of Technology +* CH-8092 Zuerich, Switzerland +* +* DATE: September 9, 1995 +* +* CHANGE HISTORY: +* +*******************************************************************************/ + +#include "tomcrypt_private.h" + +#ifdef LTC_SAFER + +#define LTC_SAFER_TAB_C +#include "safer_tab.c" + +const struct ltc_cipher_descriptor safer_k64_desc = { + "safer-k64", + 8, 8, 8, 8, LTC_SAFER_K64_DEFAULT_NOF_ROUNDS, + &safer_k64_setup, + &safer_ecb_encrypt, + &safer_ecb_decrypt, + &safer_k64_test, + &safer_done, + &safer_64_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + }, + + safer_sk64_desc = { + "safer-sk64", + 9, 8, 8, 8, LTC_SAFER_SK64_DEFAULT_NOF_ROUNDS, + &safer_sk64_setup, + &safer_ecb_encrypt, + &safer_ecb_decrypt, + &safer_sk64_test, + &safer_done, + &safer_64_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + }, + + safer_k128_desc = { + "safer-k128", + 10, 16, 16, 8, LTC_SAFER_K128_DEFAULT_NOF_ROUNDS, + &safer_k128_setup, + &safer_ecb_encrypt, + &safer_ecb_decrypt, + &safer_sk128_test, + &safer_done, + &safer_128_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + }, + + safer_sk128_desc = { + "safer-sk128", + 11, 16, 16, 8, LTC_SAFER_SK128_DEFAULT_NOF_ROUNDS, + &safer_sk128_setup, + &safer_ecb_encrypt, + &safer_ecb_decrypt, + &safer_sk128_test, + &safer_done, + &safer_128_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + }; + +/******************* Constants ************************************************/ +/* #define TAB_LEN 256 */ + +/******************* Assertions ***********************************************/ + +/******************* Macros ***************************************************/ +#define ROL8(x, n) ((unsigned char)((unsigned int)(x) << (n)\ + |(unsigned int)((x) & 0xFF) >> (8 - (n)))) +#define EXP(x) safer_ebox[(x) & 0xFF] +#define LOG(x) safer_lbox[(x) & 0xFF] +#define PHT(x, y) { y += x; x += y; } +#define IPHT(x, y) { x -= y; y -= x; } + +/******************* Types ****************************************************/ + +#ifdef LTC_CLEAN_STACK +static void s_safer_expand_userkey(const unsigned char *userkey_1, + const unsigned char *userkey_2, + unsigned int nof_rounds, + int strengthened, + safer_key_t key) +#else +static void safer_expand_userkey(const unsigned char *userkey_1, + const unsigned char *userkey_2, + unsigned int nof_rounds, + int strengthened, + safer_key_t key) +#endif +{ unsigned int i, j, k; + unsigned char ka[LTC_SAFER_BLOCK_LEN + 1]; + unsigned char kb[LTC_SAFER_BLOCK_LEN + 1]; + + if (LTC_SAFER_MAX_NOF_ROUNDS < nof_rounds) { + nof_rounds = LTC_SAFER_MAX_NOF_ROUNDS; + } + *key++ = (unsigned char)nof_rounds; + ka[LTC_SAFER_BLOCK_LEN] = (unsigned char)0; + kb[LTC_SAFER_BLOCK_LEN] = (unsigned char)0; + k = 0; + for (j = 0; j < LTC_SAFER_BLOCK_LEN; j++) { + ka[j] = ROL8(userkey_1[j], 5); + ka[LTC_SAFER_BLOCK_LEN] ^= ka[j]; + kb[j] = *key++ = userkey_2[j]; + kb[LTC_SAFER_BLOCK_LEN] ^= kb[j]; + } + for (i = 1; i <= nof_rounds; i++) { + for (j = 0; j < LTC_SAFER_BLOCK_LEN + 1; j++) { + ka[j] = ROL8(ka[j], 6); + kb[j] = ROL8(kb[j], 6); + } + if (strengthened) { + k = 2 * i - 1; + while (k >= (LTC_SAFER_BLOCK_LEN + 1)) { k -= LTC_SAFER_BLOCK_LEN + 1; } + } + for (j = 0; j < LTC_SAFER_BLOCK_LEN; j++) { + if (strengthened) { + *key++ = (ka[k] + + safer_ebox[(int)safer_ebox[(int)((18 * i + j + 1)&0xFF)]]) & 0xFF; + if (++k == (LTC_SAFER_BLOCK_LEN + 1)) { k = 0; } + } else { + *key++ = (ka[j] + safer_ebox[(int)safer_ebox[(int)((18 * i + j + 1)&0xFF)]]) & 0xFF; + } + } + if (strengthened) { + k = 2 * i; + while (k >= (LTC_SAFER_BLOCK_LEN + 1)) { k -= LTC_SAFER_BLOCK_LEN + 1; } + } + for (j = 0; j < LTC_SAFER_BLOCK_LEN; j++) { + if (strengthened) { + *key++ = (kb[k] + + safer_ebox[(int)safer_ebox[(int)((18 * i + j + 10)&0xFF)]]) & 0xFF; + if (++k == (LTC_SAFER_BLOCK_LEN + 1)) { k = 0; } + } else { + *key++ = (kb[j] + safer_ebox[(int)safer_ebox[(int)((18 * i + j + 10)&0xFF)]]) & 0xFF; + } + } + } + +#ifdef LTC_CLEAN_STACK + zeromem(ka, sizeof(ka)); + zeromem(kb, sizeof(kb)); +#endif +} + +#ifdef LTC_CLEAN_STACK +static void safer_expand_userkey(const unsigned char *userkey_1, + const unsigned char *userkey_2, + unsigned int nof_rounds, + int strengthened, + safer_key_t key) +{ + s_safer_expand_userkey(userkey_1, userkey_2, nof_rounds, strengthened, key); + burn_stack(sizeof(unsigned char) * (2 * (LTC_SAFER_BLOCK_LEN + 1)) + sizeof(unsigned int)*2); +} +#endif + +int safer_k64_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 0 && (num_rounds < 6 || num_rounds > LTC_SAFER_MAX_NOF_ROUNDS)) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen != 8) { + return CRYPT_INVALID_KEYSIZE; + } + + safer_expand_userkey(key, key, (unsigned int)(num_rounds != 0 ?num_rounds:LTC_SAFER_K64_DEFAULT_NOF_ROUNDS), 0, skey->safer.key); + return CRYPT_OK; +} + +int safer_sk64_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 0 && (num_rounds < 6 || num_rounds > LTC_SAFER_MAX_NOF_ROUNDS)) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen != 8) { + return CRYPT_INVALID_KEYSIZE; + } + + safer_expand_userkey(key, key, (unsigned int)(num_rounds != 0 ?num_rounds:LTC_SAFER_SK64_DEFAULT_NOF_ROUNDS), 1, skey->safer.key); + return CRYPT_OK; +} + +int safer_k128_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 0 && (num_rounds < 6 || num_rounds > LTC_SAFER_MAX_NOF_ROUNDS)) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + safer_expand_userkey(key, key+8, (unsigned int)(num_rounds != 0 ?num_rounds:LTC_SAFER_K128_DEFAULT_NOF_ROUNDS), 0, skey->safer.key); + return CRYPT_OK; +} + +int safer_sk128_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 0 && (num_rounds < 6 || num_rounds > LTC_SAFER_MAX_NOF_ROUNDS)) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + safer_expand_userkey(key, key+8, (unsigned int)(num_rounds != 0?num_rounds:LTC_SAFER_SK128_DEFAULT_NOF_ROUNDS), 1, skey->safer.key); + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_safer_ecb_encrypt(const unsigned char *pt, + unsigned char *ct, + const symmetric_key *skey) +#else +int safer_ecb_encrypt(const unsigned char *pt, + unsigned char *ct, + const symmetric_key *skey) +#endif +{ unsigned char a, b, c, d, e, f, g, h, t; + unsigned int round; + const unsigned char *key; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + key = skey->safer.key; + a = pt[0]; b = pt[1]; c = pt[2]; d = pt[3]; + e = pt[4]; f = pt[5]; g = pt[6]; h = pt[7]; + if (LTC_SAFER_MAX_NOF_ROUNDS < (round = *key)) round = LTC_SAFER_MAX_NOF_ROUNDS; + while(round-- > 0) + { + a ^= *++key; b += *++key; c += *++key; d ^= *++key; + e ^= *++key; f += *++key; g += *++key; h ^= *++key; + a = EXP(a) + *++key; b = LOG(b) ^ *++key; + c = LOG(c) ^ *++key; d = EXP(d) + *++key; + e = EXP(e) + *++key; f = LOG(f) ^ *++key; + g = LOG(g) ^ *++key; h = EXP(h) + *++key; + PHT(a, b); PHT(c, d); PHT(e, f); PHT(g, h); + PHT(a, c); PHT(e, g); PHT(b, d); PHT(f, h); + PHT(a, e); PHT(b, f); PHT(c, g); PHT(d, h); + t = b; b = e; e = c; c = t; t = d; d = f; f = g; g = t; + } + a ^= *++key; b += *++key; c += *++key; d ^= *++key; + e ^= *++key; f += *++key; g += *++key; h ^= *++key; + ct[0] = a & 0xFF; ct[1] = b & 0xFF; + ct[2] = c & 0xFF; ct[3] = d & 0xFF; + ct[4] = e & 0xFF; ct[5] = f & 0xFF; + ct[6] = g & 0xFF; ct[7] = h & 0xFF; + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int safer_ecb_encrypt(const unsigned char *pt, + unsigned char *ct, + const symmetric_key *skey) +{ + int err = s_safer_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(unsigned char) * 9 + sizeof(unsigned int) + sizeof(unsigned char *)); + return err; +} +#endif + +#ifdef LTC_CLEAN_STACK +static int s_safer_ecb_decrypt(const unsigned char *ct, + unsigned char *pt, + const symmetric_key *skey) +#else +int safer_ecb_decrypt(const unsigned char *ct, + unsigned char *pt, + const symmetric_key *skey) +#endif +{ unsigned char a, b, c, d, e, f, g, h, t; + unsigned int round; + const unsigned char *key; + + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(skey != NULL); + + key = skey->safer.key; + a = ct[0]; b = ct[1]; c = ct[2]; d = ct[3]; + e = ct[4]; f = ct[5]; g = ct[6]; h = ct[7]; + if (LTC_SAFER_MAX_NOF_ROUNDS < (round = *key)) round = LTC_SAFER_MAX_NOF_ROUNDS; + key += LTC_SAFER_BLOCK_LEN * (1 + 2 * round); + h ^= *key; g -= *--key; f -= *--key; e ^= *--key; + d ^= *--key; c -= *--key; b -= *--key; a ^= *--key; + while (round--) + { + t = e; e = b; b = c; c = t; t = f; f = d; d = g; g = t; + IPHT(a, e); IPHT(b, f); IPHT(c, g); IPHT(d, h); + IPHT(a, c); IPHT(e, g); IPHT(b, d); IPHT(f, h); + IPHT(a, b); IPHT(c, d); IPHT(e, f); IPHT(g, h); + h -= *--key; g ^= *--key; f ^= *--key; e -= *--key; + d -= *--key; c ^= *--key; b ^= *--key; a -= *--key; + h = LOG(h) ^ *--key; g = EXP(g) - *--key; + f = EXP(f) - *--key; e = LOG(e) ^ *--key; + d = LOG(d) ^ *--key; c = EXP(c) - *--key; + b = EXP(b) - *--key; a = LOG(a) ^ *--key; + } + pt[0] = a & 0xFF; pt[1] = b & 0xFF; + pt[2] = c & 0xFF; pt[3] = d & 0xFF; + pt[4] = e & 0xFF; pt[5] = f & 0xFF; + pt[6] = g & 0xFF; pt[7] = h & 0xFF; + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int safer_ecb_decrypt(const unsigned char *ct, + unsigned char *pt, + const symmetric_key *skey) +{ + int err = s_safer_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(unsigned char) * 9 + sizeof(unsigned int) + sizeof(unsigned char *)); + return err; +} +#endif + +int safer_64_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 8) { + return CRYPT_INVALID_KEYSIZE; + } + *keysize = 8; + return CRYPT_OK; +} + +int safer_128_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + *keysize = 16; + return CRYPT_OK; +} + +int safer_k64_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const unsigned char k64_pt[] = { 1, 2, 3, 4, 5, 6, 7, 8 }, + k64_key[] = { 8, 7, 6, 5, 4, 3, 2, 1 }, + k64_ct[] = { 200, 242, 156, 221, 135, 120, 62, 217 }; + + symmetric_key skey; + unsigned char buf[2][8]; + int err; + + /* test K64 */ + if ((err = safer_k64_setup(k64_key, 8, 6, &skey)) != CRYPT_OK) { + return err; + } + safer_ecb_encrypt(k64_pt, buf[0], &skey); + safer_ecb_decrypt(buf[0], buf[1], &skey); + + if (compare_testvector(buf[0], 8, k64_ct, 8, "Safer K64 Encrypt", 0) != 0 || + compare_testvector(buf[1], 8, k64_pt, 8, "Safer K64 Decrypt", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; + #endif +} + + +int safer_sk64_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const unsigned char sk64_pt[] = { 1, 2, 3, 4, 5, 6, 7, 8 }, + sk64_key[] = { 1, 2, 3, 4, 5, 6, 7, 8 }, + sk64_ct[] = { 95, 206, 155, 162, 5, 132, 56, 199 }; + + symmetric_key skey; + unsigned char buf[2][8]; + int err, y; + + /* test SK64 */ + if ((err = safer_sk64_setup(sk64_key, 8, 6, &skey)) != CRYPT_OK) { + return err; + } + + safer_ecb_encrypt(sk64_pt, buf[0], &skey); + safer_ecb_decrypt(buf[0], buf[1], &skey); + + if (compare_testvector(buf[0], 8, sk64_ct, 8, "Safer SK64 Encrypt", 0) != 0 || + compare_testvector(buf[1], 8, sk64_pt, 8, "Safer SK64 Decrypt", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) buf[0][y] = 0; + for (y = 0; y < 1000; y++) safer_ecb_encrypt(buf[0], buf[0], &skey); + for (y = 0; y < 1000; y++) safer_ecb_decrypt(buf[0], buf[0], &skey); + for (y = 0; y < 8; y++) if (buf[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void safer_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +int safer_sk128_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const unsigned char sk128_pt[] = { 1, 2, 3, 4, 5, 6, 7, 8 }, + sk128_key[] = { 1, 2, 3, 4, 5, 6, 7, 8, + 0, 0, 0, 0, 0, 0, 0, 0 }, + sk128_ct[] = { 255, 120, 17, 228, 179, 167, 46, 113 }; + + symmetric_key skey; + unsigned char buf[2][8]; + int err, y; + + /* test SK128 */ + if ((err = safer_sk128_setup(sk128_key, 16, 0, &skey)) != CRYPT_OK) { + return err; + } + safer_ecb_encrypt(sk128_pt, buf[0], &skey); + safer_ecb_decrypt(buf[0], buf[1], &skey); + + if (compare_testvector(buf[0], 8, sk128_ct, 8, "Safer SK128 Encrypt", 0) != 0 || + compare_testvector(buf[1], 8, sk128_pt, 8, "Safer SK128 Decrypt", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) buf[0][y] = 0; + for (y = 0; y < 1000; y++) safer_ecb_encrypt(buf[0], buf[0], &skey); + for (y = 0; y < 1000; y++) safer_ecb_decrypt(buf[0], buf[0], &skey); + for (y = 0; y < 8; y++) if (buf[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + + return CRYPT_OK; + #endif +} + +#endif + + + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/safer/safer_tab.c b/optee_os/core/lib/libtomcrypt/src/ciphers/safer/safer_tab.c new file mode 100644 index 0000000..4cc0e47 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/safer/safer_tab.c @@ -0,0 +1,54 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file safer_tab.c + Tables for LTC_SAFER block ciphers +*/ + +#ifdef LTC_SAFER_TAB_C + +/* This is the box defined by ebox[x] = 45^x mod 257. + * Its assumed that the value "256" corresponds to zero. */ +static const unsigned char safer_ebox[256] = { + 1, 45, 226, 147, 190, 69, 21, 174, 120, 3, 135, 164, 184, 56, 207, 63, + 8, 103, 9, 148, 235, 38, 168, 107, 189, 24, 52, 27, 187, 191, 114, 247, + 64, 53, 72, 156, 81, 47, 59, 85, 227, 192, 159, 216, 211, 243, 141, 177, +255, 167, 62, 220, 134, 119, 215, 166, 17, 251, 244, 186, 146, 145, 100, 131, +241, 51, 239, 218, 44, 181, 178, 43, 136, 209, 153, 203, 140, 132, 29, 20, +129, 151, 113, 202, 95, 163, 139, 87, 60, 130, 196, 82, 92, 28, 232, 160, + 4, 180, 133, 74, 246, 19, 84, 182, 223, 12, 26, 142, 222, 224, 57, 252, + 32, 155, 36, 78, 169, 152, 158, 171, 242, 96, 208, 108, 234, 250, 199, 217, + 0, 212, 31, 110, 67, 188, 236, 83, 137, 254, 122, 93, 73, 201, 50, 194, +249, 154, 248, 109, 22, 219, 89, 150, 68, 233, 205, 230, 70, 66, 143, 10, +193, 204, 185, 101, 176, 210, 198, 172, 30, 65, 98, 41, 46, 14, 116, 80, + 2, 90, 195, 37, 123, 138, 42, 91, 240, 6, 13, 71, 111, 112, 157, 126, + 16, 206, 18, 39, 213, 76, 79, 214, 121, 48, 104, 54, 117, 125, 228, 237, +128, 106, 144, 55, 162, 94, 118, 170, 197, 127, 61, 175, 165, 229, 25, 97, +253, 77, 124, 183, 11, 238, 173, 75, 34, 245, 231, 115, 35, 33, 200, 5, +225, 102, 221, 179, 88, 105, 99, 86, 15, 161, 49, 149, 23, 7, 58, 40 +}; + +/* This is the inverse of ebox or the base 45 logarithm */ +static const unsigned char safer_lbox[256] = { +128, 0, 176, 9, 96, 239, 185, 253, 16, 18, 159, 228, 105, 186, 173, 248, +192, 56, 194, 101, 79, 6, 148, 252, 25, 222, 106, 27, 93, 78, 168, 130, +112, 237, 232, 236, 114, 179, 21, 195, 255, 171, 182, 71, 68, 1, 172, 37, +201, 250, 142, 65, 26, 33, 203, 211, 13, 110, 254, 38, 88, 218, 50, 15, + 32, 169, 157, 132, 152, 5, 156, 187, 34, 140, 99, 231, 197, 225, 115, 198, +175, 36, 91, 135, 102, 39, 247, 87, 244, 150, 177, 183, 92, 139, 213, 84, +121, 223, 170, 246, 62, 163, 241, 17, 202, 245, 209, 23, 123, 147, 131, 188, +189, 82, 30, 235, 174, 204, 214, 53, 8, 200, 138, 180, 226, 205, 191, 217, +208, 80, 89, 63, 77, 98, 52, 10, 72, 136, 181, 86, 76, 46, 107, 158, +210, 61, 60, 3, 19, 251, 151, 81, 117, 74, 145, 113, 35, 190, 118, 42, + 95, 249, 212, 85, 11, 220, 55, 49, 22, 116, 215, 119, 167, 230, 7, 219, +164, 47, 70, 243, 97, 69, 103, 227, 12, 162, 59, 28, 133, 24, 4, 29, + 41, 160, 143, 178, 90, 216, 166, 126, 238, 141, 83, 75, 161, 154, 193, 14, +122, 73, 165, 44, 129, 196, 199, 54, 43, 127, 67, 149, 51, 242, 108, 104, +109, 240, 2, 40, 206, 221, 155, 234, 94, 153, 124, 20, 134, 207, 229, 66, +184, 64, 120, 45, 58, 233, 100, 31, 146, 144, 125, 57, 111, 224, 137, 48 +}; + +#endif /* LTC_SAFER_TAB_C */ + + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/safer/saferp.c b/optee_os/core/lib/libtomcrypt/src/ciphers/safer/saferp.c new file mode 100644 index 0000000..86cc708 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/safer/saferp.c @@ -0,0 +1,567 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file saferp.c + LTC_SAFER+ Implementation by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_SAFERP + +#define LTC_SAFER_TAB_C +#include "safer_tab.c" + +const struct ltc_cipher_descriptor saferp_desc = +{ + "safer+", + 4, + 16, 32, 16, 8, + &saferp_setup, + &saferp_ecb_encrypt, + &saferp_ecb_decrypt, + &saferp_test, + &saferp_done, + &saferp_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +/* ROUND(b,i) + * + * This is one forward key application. Note the basic form is + * key addition, substitution, key addition. The safer_ebox and safer_lbox + * are the exponentiation box and logarithm boxes respectively. + * The value of 'i' is the current round number which allows this + * function to be unrolled massively. Most of LTC_SAFER+'s speed + * comes from not having to compute indirect accesses into the + * array of 16 bytes b[0..15] which is the block of data +*/ + +#define ROUND(b, i) do { \ + b[0] = (safer_ebox[(b[0] ^ skey->saferp.K[i][0]) & 255] + skey->saferp.K[i+1][0]) & 255; \ + b[1] = safer_lbox[(b[1] + skey->saferp.K[i][1]) & 255] ^ skey->saferp.K[i+1][1]; \ + b[2] = safer_lbox[(b[2] + skey->saferp.K[i][2]) & 255] ^ skey->saferp.K[i+1][2]; \ + b[3] = (safer_ebox[(b[3] ^ skey->saferp.K[i][3]) & 255] + skey->saferp.K[i+1][3]) & 255; \ + b[4] = (safer_ebox[(b[4] ^ skey->saferp.K[i][4]) & 255] + skey->saferp.K[i+1][4]) & 255; \ + b[5] = safer_lbox[(b[5] + skey->saferp.K[i][5]) & 255] ^ skey->saferp.K[i+1][5]; \ + b[6] = safer_lbox[(b[6] + skey->saferp.K[i][6]) & 255] ^ skey->saferp.K[i+1][6]; \ + b[7] = (safer_ebox[(b[7] ^ skey->saferp.K[i][7]) & 255] + skey->saferp.K[i+1][7]) & 255; \ + b[8] = (safer_ebox[(b[8] ^ skey->saferp.K[i][8]) & 255] + skey->saferp.K[i+1][8]) & 255; \ + b[9] = safer_lbox[(b[9] + skey->saferp.K[i][9]) & 255] ^ skey->saferp.K[i+1][9]; \ + b[10] = safer_lbox[(b[10] + skey->saferp.K[i][10]) & 255] ^ skey->saferp.K[i+1][10]; \ + b[11] = (safer_ebox[(b[11] ^ skey->saferp.K[i][11]) & 255] + skey->saferp.K[i+1][11]) & 255; \ + b[12] = (safer_ebox[(b[12] ^ skey->saferp.K[i][12]) & 255] + skey->saferp.K[i+1][12]) & 255; \ + b[13] = safer_lbox[(b[13] + skey->saferp.K[i][13]) & 255] ^ skey->saferp.K[i+1][13]; \ + b[14] = safer_lbox[(b[14] + skey->saferp.K[i][14]) & 255] ^ skey->saferp.K[i+1][14]; \ + b[15] = (safer_ebox[(b[15] ^ skey->saferp.K[i][15]) & 255] + skey->saferp.K[i+1][15]) & 255; \ +} while (0) + +/* This is one inverse key application */ +#define iROUND(b, i) do { \ + b[0] = safer_lbox[(b[0] - skey->saferp.K[i+1][0]) & 255] ^ skey->saferp.K[i][0]; \ + b[1] = (safer_ebox[(b[1] ^ skey->saferp.K[i+1][1]) & 255] - skey->saferp.K[i][1]) & 255; \ + b[2] = (safer_ebox[(b[2] ^ skey->saferp.K[i+1][2]) & 255] - skey->saferp.K[i][2]) & 255; \ + b[3] = safer_lbox[(b[3] - skey->saferp.K[i+1][3]) & 255] ^ skey->saferp.K[i][3]; \ + b[4] = safer_lbox[(b[4] - skey->saferp.K[i+1][4]) & 255] ^ skey->saferp.K[i][4]; \ + b[5] = (safer_ebox[(b[5] ^ skey->saferp.K[i+1][5]) & 255] - skey->saferp.K[i][5]) & 255; \ + b[6] = (safer_ebox[(b[6] ^ skey->saferp.K[i+1][6]) & 255] - skey->saferp.K[i][6]) & 255; \ + b[7] = safer_lbox[(b[7] - skey->saferp.K[i+1][7]) & 255] ^ skey->saferp.K[i][7]; \ + b[8] = safer_lbox[(b[8] - skey->saferp.K[i+1][8]) & 255] ^ skey->saferp.K[i][8]; \ + b[9] = (safer_ebox[(b[9] ^ skey->saferp.K[i+1][9]) & 255] - skey->saferp.K[i][9]) & 255; \ + b[10] = (safer_ebox[(b[10] ^ skey->saferp.K[i+1][10]) & 255] - skey->saferp.K[i][10]) & 255; \ + b[11] = safer_lbox[(b[11] - skey->saferp.K[i+1][11]) & 255] ^ skey->saferp.K[i][11]; \ + b[12] = safer_lbox[(b[12] - skey->saferp.K[i+1][12]) & 255] ^ skey->saferp.K[i][12]; \ + b[13] = (safer_ebox[(b[13] ^ skey->saferp.K[i+1][13]) & 255] - skey->saferp.K[i][13]) & 255; \ + b[14] = (safer_ebox[(b[14] ^ skey->saferp.K[i+1][14]) & 255] - skey->saferp.K[i][14]) & 255; \ + b[15] = safer_lbox[(b[15] - skey->saferp.K[i+1][15]) & 255] ^ skey->saferp.K[i][15]; \ +} while (0) + +/* This is a forward single layer PHT transform. */ +#define PHT(b) do { \ + b[0] = (b[0] + (b[1] = (b[0] + b[1]) & 255)) & 255; \ + b[2] = (b[2] + (b[3] = (b[3] + b[2]) & 255)) & 255; \ + b[4] = (b[4] + (b[5] = (b[5] + b[4]) & 255)) & 255; \ + b[6] = (b[6] + (b[7] = (b[7] + b[6]) & 255)) & 255; \ + b[8] = (b[8] + (b[9] = (b[9] + b[8]) & 255)) & 255; \ + b[10] = (b[10] + (b[11] = (b[11] + b[10]) & 255)) & 255; \ + b[12] = (b[12] + (b[13] = (b[13] + b[12]) & 255)) & 255; \ + b[14] = (b[14] + (b[15] = (b[15] + b[14]) & 255)) & 255; \ +} while (0) + +/* This is an inverse single layer PHT transform */ +#define iPHT(b) do { \ + b[15] = (b[15] - (b[14] = (b[14] - b[15]) & 255)) & 255; \ + b[13] = (b[13] - (b[12] = (b[12] - b[13]) & 255)) & 255; \ + b[11] = (b[11] - (b[10] = (b[10] - b[11]) & 255)) & 255; \ + b[9] = (b[9] - (b[8] = (b[8] - b[9]) & 255)) & 255; \ + b[7] = (b[7] - (b[6] = (b[6] - b[7]) & 255)) & 255; \ + b[5] = (b[5] - (b[4] = (b[4] - b[5]) & 255)) & 255; \ + b[3] = (b[3] - (b[2] = (b[2] - b[3]) & 255)) & 255; \ + b[1] = (b[1] - (b[0] = (b[0] - b[1]) & 255)) & 255; \ + } while (0) + +/* This is the "Armenian" Shuffle. It takes the input from b and stores it in b2 */ +#define SHUF(b, b2) do { \ + b2[0] = b[8]; b2[1] = b[11]; b2[2] = b[12]; b2[3] = b[15]; \ + b2[4] = b[2]; b2[5] = b[1]; b2[6] = b[6]; b2[7] = b[5]; \ + b2[8] = b[10]; b2[9] = b[9]; b2[10] = b[14]; b2[11] = b[13]; \ + b2[12] = b[0]; b2[13] = b[7]; b2[14] = b[4]; b2[15] = b[3]; \ +} while (0) + +/* This is the inverse shuffle. It takes from b and gives to b2 */ +#define iSHUF(b, b2) do { \ + b2[0] = b[12]; b2[1] = b[5]; b2[2] = b[4]; b2[3] = b[15]; \ + b2[4] = b[14]; b2[5] = b[7]; b2[6] = b[6]; b2[7] = b[13]; \ + b2[8] = b[0]; b2[9] = b[9]; b2[10] = b[8]; b2[11] = b[1]; \ + b2[12] = b[2]; b2[13] = b[11]; b2[14] = b[10]; b2[15] = b[3]; \ +} while (0) + +/* The complete forward Linear Transform layer. + * Note that alternating usage of b and b2. + * Each round of LT starts in 'b' and ends in 'b2'. + */ +#define LT(b, b2) do { \ + PHT(b); SHUF(b, b2); \ + PHT(b2); SHUF(b2, b); \ + PHT(b); SHUF(b, b2); \ + PHT(b2); \ +} while (0) + +/* This is the inverse linear transform layer. */ +#define iLT(b, b2) do { \ + iPHT(b); \ + iSHUF(b, b2); iPHT(b2); \ + iSHUF(b2, b); iPHT(b); \ + iSHUF(b, b2); iPHT(b2); \ +} while (0) + +#ifdef LTC_SMALL_CODE + +static void s_round(unsigned char *b, int i, const symmetric_key *skey) +{ + ROUND(b, i); +} + +static void s_iround(unsigned char *b, int i, const symmetric_key *skey) +{ + iROUND(b, i); +} + +static void s_lt(unsigned char *b, unsigned char *b2) +{ + LT(b, b2); +} + +static void s_ilt(unsigned char *b, unsigned char *b2) +{ + iLT(b, b2); +} + +#undef ROUND +#define ROUND(b, i) s_round(b, i, skey) + +#undef iROUND +#define iROUND(b, i) s_iround(b, i, skey) + +#undef LT +#define LT(b, b2) s_lt(b, b2) + +#undef iLT +#define iLT(b, b2) s_ilt(b, b2) + +#endif + +/* These are the 33, 128-bit bias words for the key schedule */ +static const unsigned char safer_bias[33][16] = { +{ 70, 151, 177, 186, 163, 183, 16, 10, 197, 55, 179, 201, 90, 40, 172, 100}, +{ 236, 171, 170, 198, 103, 149, 88, 13, 248, 154, 246, 110, 102, 220, 5, 61}, +{ 138, 195, 216, 137, 106, 233, 54, 73, 67, 191, 235, 212, 150, 155, 104, 160}, +{ 93, 87, 146, 31, 213, 113, 92, 187, 34, 193, 190, 123, 188, 153, 99, 148}, +{ 42, 97, 184, 52, 50, 25, 253, 251, 23, 64, 230, 81, 29, 65, 68, 143}, +{ 221, 4, 128, 222, 231, 49, 214, 127, 1, 162, 247, 57, 218, 111, 35, 202}, +{ 58, 208, 28, 209, 48, 62, 18, 161, 205, 15, 224, 168, 175, 130, 89, 44}, +{ 125, 173, 178, 239, 194, 135, 206, 117, 6, 19, 2, 144, 79, 46, 114, 51}, +{ 192, 141, 207, 169, 129, 226, 196, 39, 47, 108, 122, 159, 82, 225, 21, 56}, +{ 252, 32, 66, 199, 8, 228, 9, 85, 94, 140, 20, 118, 96, 255, 223, 215}, +{ 250, 11, 33, 0, 26, 249, 166, 185, 232, 158, 98, 76, 217, 145, 80, 210}, +{ 24, 180, 7, 132, 234, 91, 164, 200, 14, 203, 72, 105, 75, 78, 156, 53}, +{ 69, 77, 84, 229, 37, 60, 12, 74, 139, 63, 204, 167, 219, 107, 174, 244}, +{ 45, 243, 124, 109, 157, 181, 38, 116, 242, 147, 83, 176, 240, 17, 237, 131}, +{ 182, 3, 22, 115, 59, 30, 142, 112, 189, 134, 27, 71, 126, 36, 86, 241}, +{ 136, 70, 151, 177, 186, 163, 183, 16, 10, 197, 55, 179, 201, 90, 40, 172}, +{ 220, 134, 119, 215, 166, 17, 251, 244, 186, 146, 145, 100, 131, 241, 51, 239}, +{ 44, 181, 178, 43, 136, 209, 153, 203, 140, 132, 29, 20, 129, 151, 113, 202}, +{ 163, 139, 87, 60, 130, 196, 82, 92, 28, 232, 160, 4, 180, 133, 74, 246}, +{ 84, 182, 223, 12, 26, 142, 222, 224, 57, 252, 32, 155, 36, 78, 169, 152}, +{ 171, 242, 96, 208, 108, 234, 250, 199, 217, 0, 212, 31, 110, 67, 188, 236}, +{ 137, 254, 122, 93, 73, 201, 50, 194, 249, 154, 248, 109, 22, 219, 89, 150}, +{ 233, 205, 230, 70, 66, 143, 10, 193, 204, 185, 101, 176, 210, 198, 172, 30}, +{ 98, 41, 46, 14, 116, 80, 2, 90, 195, 37, 123, 138, 42, 91, 240, 6}, +{ 71, 111, 112, 157, 126, 16, 206, 18, 39, 213, 76, 79, 214, 121, 48, 104}, +{ 117, 125, 228, 237, 128, 106, 144, 55, 162, 94, 118, 170, 197, 127, 61, 175}, +{ 229, 25, 97, 253, 77, 124, 183, 11, 238, 173, 75, 34, 245, 231, 115, 35}, +{ 200, 5, 225, 102, 221, 179, 88, 105, 99, 86, 15, 161, 49, 149, 23, 7}, +{ 40, 1, 45, 226, 147, 190, 69, 21, 174, 120, 3, 135, 164, 184, 56, 207}, +{ 8, 103, 9, 148, 235, 38, 168, 107, 189, 24, 52, 27, 187, 191, 114, 247}, +{ 53, 72, 156, 81, 47, 59, 85, 227, 192, 159, 216, 211, 243, 141, 177, 255}, +{ 62, 220, 134, 119, 215, 166, 17, 251, 244, 186, 146, 145, 100, 131, 241, 51}}; + + /** + Initialize the LTC_SAFER+ block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int saferp_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + unsigned x, y, z; + unsigned char t[33]; + static const int rounds[3] = { 8, 12, 16 }; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* check arguments */ + if (keylen != 16 && keylen != 24 && keylen != 32) { + return CRYPT_INVALID_KEYSIZE; + } + + /* Is the number of rounds valid? Either use zero for default or + * 8,12,16 rounds for 16,24,32 byte keys + */ + if (num_rounds != 0 && num_rounds != rounds[(keylen/8)-2]) { + return CRYPT_INVALID_ROUNDS; + } + + /* 128 bit key version */ + if (keylen == 16) { + /* copy key into t */ + for (x = y = 0; x < 16; x++) { + t[x] = key[x]; + y ^= key[x]; + } + t[16] = y; + + /* make round keys */ + for (x = 0; x < 16; x++) { + skey->saferp.K[0][x] = t[x]; + } + + /* make the 16 other keys as a transformation of the first key */ + for (x = 1; x < 17; x++) { + /* rotate 3 bits each */ + for (y = 0; y < 17; y++) { + t[y] = ((t[y]<<3)|(t[y]>>5)) & 255; + } + + /* select and add */ + z = x; + for (y = 0; y < 16; y++) { + skey->saferp.K[x][y] = (t[z] + safer_bias[x-1][y]) & 255; + if (++z == 17) { z = 0; } + } + } + skey->saferp.rounds = 8; + } else if (keylen == 24) { + /* copy key into t */ + for (x = y = 0; x < 24; x++) { + t[x] = key[x]; + y ^= key[x]; + } + t[24] = y; + + /* make round keys */ + for (x = 0; x < 16; x++) { + skey->saferp.K[0][x] = t[x]; + } + + for (x = 1; x < 25; x++) { + /* rotate 3 bits each */ + for (y = 0; y < 25; y++) { + t[y] = ((t[y]<<3)|(t[y]>>5)) & 255; + } + + /* select and add */ + z = x; + for (y = 0; y < 16; y++) { + skey->saferp.K[x][y] = (t[z] + safer_bias[x-1][y]) & 255; + if (++z == 25) { z = 0; } + } + } + skey->saferp.rounds = 12; + } else { + /* copy key into t */ + for (x = y = 0; x < 32; x++) { + t[x] = key[x]; + y ^= key[x]; + } + t[32] = y; + + /* make round keys */ + for (x = 0; x < 16; x++) { + skey->saferp.K[0][x] = t[x]; + } + + for (x = 1; x < 33; x++) { + /* rotate 3 bits each */ + for (y = 0; y < 33; y++) { + t[y] = ((t[y]<<3)|(t[y]>>5)) & 255; + } + + /* select and add */ + z = x; + for (y = 0; y < 16; y++) { + skey->saferp.K[x][y] = (t[z] + safer_bias[x-1][y]) & 255; + if (++z == 33) { z = 0; } + } + } + skey->saferp.rounds = 16; + } +#ifdef LTC_CLEAN_STACK + zeromem(t, sizeof(t)); +#endif + return CRYPT_OK; +} + +/** + Encrypts a block of text with LTC_SAFER+ + @param pt The input plaintext (16 bytes) + @param ct The output ciphertext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int saferp_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + unsigned char b[16]; + int x; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + if (skey->saferp.rounds < 8 || skey->saferp.rounds > 16) { + return CRYPT_INVALID_ROUNDS; + } + + /* do eight rounds */ + for (x = 0; x < 16; x++) { + b[x] = pt[x]; + } + ROUND(b, 0); LT(b, ct); + ROUND(ct, 2); LT(ct, b); + ROUND(b, 4); LT(b, ct); + ROUND(ct, 6); LT(ct, b); + ROUND(b, 8); LT(b, ct); + ROUND(ct, 10); LT(ct, b); + ROUND(b, 12); LT(b, ct); + ROUND(ct, 14); LT(ct, b); + /* 192-bit key? */ + if (skey->saferp.rounds > 8) { + ROUND(b, 16); LT(b, ct); + ROUND(ct, 18); LT(ct, b); + ROUND(b, 20); LT(b, ct); + ROUND(ct, 22); LT(ct, b); + } + /* 256-bit key? */ + if (skey->saferp.rounds > 12) { + ROUND(b, 24); LT(b, ct); + ROUND(ct, 26); LT(ct, b); + ROUND(b, 28); LT(b, ct); + ROUND(ct, 30); LT(ct, b); + } + ct[0] = b[0] ^ skey->saferp.K[skey->saferp.rounds*2][0]; + ct[1] = (b[1] + skey->saferp.K[skey->saferp.rounds*2][1]) & 255; + ct[2] = (b[2] + skey->saferp.K[skey->saferp.rounds*2][2]) & 255; + ct[3] = b[3] ^ skey->saferp.K[skey->saferp.rounds*2][3]; + ct[4] = b[4] ^ skey->saferp.K[skey->saferp.rounds*2][4]; + ct[5] = (b[5] + skey->saferp.K[skey->saferp.rounds*2][5]) & 255; + ct[6] = (b[6] + skey->saferp.K[skey->saferp.rounds*2][6]) & 255; + ct[7] = b[7] ^ skey->saferp.K[skey->saferp.rounds*2][7]; + ct[8] = b[8] ^ skey->saferp.K[skey->saferp.rounds*2][8]; + ct[9] = (b[9] + skey->saferp.K[skey->saferp.rounds*2][9]) & 255; + ct[10] = (b[10] + skey->saferp.K[skey->saferp.rounds*2][10]) & 255; + ct[11] = b[11] ^ skey->saferp.K[skey->saferp.rounds*2][11]; + ct[12] = b[12] ^ skey->saferp.K[skey->saferp.rounds*2][12]; + ct[13] = (b[13] + skey->saferp.K[skey->saferp.rounds*2][13]) & 255; + ct[14] = (b[14] + skey->saferp.K[skey->saferp.rounds*2][14]) & 255; + ct[15] = b[15] ^ skey->saferp.K[skey->saferp.rounds*2][15]; +#ifdef LTC_CLEAN_STACK + zeromem(b, sizeof(b)); +#endif + return CRYPT_OK; +} + +/** + Decrypts a block of text with LTC_SAFER+ + @param ct The input ciphertext (16 bytes) + @param pt The output plaintext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int saferp_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + unsigned char b[16]; + int x; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + if (skey->saferp.rounds < 8 || skey->saferp.rounds > 16) { + return CRYPT_INVALID_ROUNDS; + } + + /* do eight rounds */ + b[0] = ct[0] ^ skey->saferp.K[skey->saferp.rounds*2][0]; + b[1] = (ct[1] - skey->saferp.K[skey->saferp.rounds*2][1]) & 255; + b[2] = (ct[2] - skey->saferp.K[skey->saferp.rounds*2][2]) & 255; + b[3] = ct[3] ^ skey->saferp.K[skey->saferp.rounds*2][3]; + b[4] = ct[4] ^ skey->saferp.K[skey->saferp.rounds*2][4]; + b[5] = (ct[5] - skey->saferp.K[skey->saferp.rounds*2][5]) & 255; + b[6] = (ct[6] - skey->saferp.K[skey->saferp.rounds*2][6]) & 255; + b[7] = ct[7] ^ skey->saferp.K[skey->saferp.rounds*2][7]; + b[8] = ct[8] ^ skey->saferp.K[skey->saferp.rounds*2][8]; + b[9] = (ct[9] - skey->saferp.K[skey->saferp.rounds*2][9]) & 255; + b[10] = (ct[10] - skey->saferp.K[skey->saferp.rounds*2][10]) & 255; + b[11] = ct[11] ^ skey->saferp.K[skey->saferp.rounds*2][11]; + b[12] = ct[12] ^ skey->saferp.K[skey->saferp.rounds*2][12]; + b[13] = (ct[13] - skey->saferp.K[skey->saferp.rounds*2][13]) & 255; + b[14] = (ct[14] - skey->saferp.K[skey->saferp.rounds*2][14]) & 255; + b[15] = ct[15] ^ skey->saferp.K[skey->saferp.rounds*2][15]; + /* 256-bit key? */ + if (skey->saferp.rounds > 12) { + iLT(b, pt); iROUND(pt, 30); + iLT(pt, b); iROUND(b, 28); + iLT(b, pt); iROUND(pt, 26); + iLT(pt, b); iROUND(b, 24); + } + /* 192-bit key? */ + if (skey->saferp.rounds > 8) { + iLT(b, pt); iROUND(pt, 22); + iLT(pt, b); iROUND(b, 20); + iLT(b, pt); iROUND(pt, 18); + iLT(pt, b); iROUND(b, 16); + } + iLT(b, pt); iROUND(pt, 14); + iLT(pt, b); iROUND(b, 12); + iLT(b, pt); iROUND(pt,10); + iLT(pt, b); iROUND(b, 8); + iLT(b, pt); iROUND(pt,6); + iLT(pt, b); iROUND(b, 4); + iLT(b, pt); iROUND(pt,2); + iLT(pt, b); iROUND(b, 0); + for (x = 0; x < 16; x++) { + pt[x] = b[x]; + } +#ifdef LTC_CLEAN_STACK + zeromem(b, sizeof(b)); +#endif + return CRYPT_OK; +} + +/** + Performs a self-test of the LTC_SAFER+ block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int saferp_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + int keylen; + unsigned char key[32], pt[16], ct[16]; + } tests[] = { + { + 16, + { 41, 35, 190, 132, 225, 108, 214, 174, + 82, 144, 73, 241, 241, 187, 233, 235 }, + { 179, 166, 219, 60, 135, 12, 62, 153, + 36, 94, 13, 28, 6, 183, 71, 222 }, + { 224, 31, 182, 10, 12, 255, 84, 70, + 127, 13, 89, 249, 9, 57, 165, 220 } + }, { + 24, + { 72, 211, 143, 117, 230, 217, 29, 42, + 229, 192, 247, 43, 120, 129, 135, 68, + 14, 95, 80, 0, 212, 97, 141, 190 }, + { 123, 5, 21, 7, 59, 51, 130, 31, + 24, 112, 146, 218, 100, 84, 206, 177 }, + { 92, 136, 4, 63, 57, 95, 100, 0, + 150, 130, 130, 16, 193, 111, 219, 133 } + }, { + 32, + { 243, 168, 141, 254, 190, 242, 235, 113, + 255, 160, 208, 59, 117, 6, 140, 126, + 135, 120, 115, 77, 208, 190, 130, 190, + 219, 194, 70, 65, 43, 140, 250, 48 }, + { 127, 112, 240, 167, 84, 134, 50, 149, + 170, 91, 104, 19, 11, 230, 252, 245 }, + { 88, 11, 25, 36, 172, 229, 202, 213, + 170, 65, 105, 153, 220, 104, 153, 138 } + } + }; + + unsigned char tmp[2][16]; + symmetric_key skey; + int err, i, y; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + if ((err = saferp_setup(tests[i].key, tests[i].keylen, 0, &skey)) != CRYPT_OK) { + return err; + } + saferp_ecb_encrypt(tests[i].pt, tmp[0], &skey); + saferp_ecb_decrypt(tmp[0], tmp[1], &skey); + + /* compare */ + if (compare_testvector(tmp[0], 16, tests[i].ct, 16, "Safer+ Encrypt", i) || + compare_testvector(tmp[1], 16, tests[i].pt, 16, "Safer+ Decrypt", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 16; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) saferp_ecb_encrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 1000; y++) saferp_ecb_decrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 16; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void saferp_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int saferp_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize < 24) { + *keysize = 16; + } else if (*keysize < 32) { + *keysize = 24; + } else { + *keysize = 32; + } + return CRYPT_OK; +} + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/serpent.c b/optee_os/core/lib/libtomcrypt/src/ciphers/serpent.c new file mode 100644 index 0000000..dbb999e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/serpent.c @@ -0,0 +1,717 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* Based on serpent.cpp - originally written and placed in the public domain by Wei Dai + https://github.com/weidai11/cryptopp/blob/master/serpent.cpp + + On 2017-10-16 wikipedia says: + "The Serpent cipher algorithm is in the public domain and has not been patented." + https://en.wikipedia.org/wiki/Serpent_(cipher) + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SERPENT + +const struct ltc_cipher_descriptor serpent_desc = { + "serpent", + 25, /* cipher_ID */ + 16, 32, 16, 32, /* min_key_len, max_key_len, block_len, default_rounds */ + &serpent_setup, + &serpent_ecb_encrypt, + &serpent_ecb_decrypt, + &serpent_test, + &serpent_done, + &serpent_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +/* linear transformation */ +#define s_lt(i,a,b,c,d,e) { \ + a = ROLc(a, 13); \ + c = ROLc(c, 3); \ + d = ROLc(d ^ c ^ (a << 3), 7); \ + b = ROLc(b ^ a ^ c, 1); \ + a = ROLc(a ^ b ^ d, 5); \ + c = ROLc(c ^ d ^ (b << 7), 22); \ + } + +/* inverse linear transformation */ +#define s_ilt(i,a,b,c,d,e) { \ + c = RORc(c, 22); \ + a = RORc(a, 5); \ + c ^= d ^ (b << 7); \ + a ^= b ^ d; \ + b = RORc(b, 1); \ + d = RORc(d, 7) ^ c ^ (a << 3); \ + b ^= a ^ c; \ + c = RORc(c, 3); \ + a = RORc(a, 13); \ + } + +/* order of output from S-box functions */ +#define s_beforeS0(f) f(0,a,b,c,d,e) +#define s_afterS0(f) f(1,b,e,c,a,d) +#define s_afterS1(f) f(2,c,b,a,e,d) +#define s_afterS2(f) f(3,a,e,b,d,c) +#define s_afterS3(f) f(4,e,b,d,c,a) +#define s_afterS4(f) f(5,b,a,e,c,d) +#define s_afterS5(f) f(6,a,c,b,e,d) +#define s_afterS6(f) f(7,a,c,d,b,e) +#define s_afterS7(f) f(8,d,e,b,a,c) + +/* order of output from inverse S-box functions */ +#define s_beforeI7(f) f(8,a,b,c,d,e) +#define s_afterI7(f) f(7,d,a,b,e,c) +#define s_afterI6(f) f(6,a,b,c,e,d) +#define s_afterI5(f) f(5,b,d,e,c,a) +#define s_afterI4(f) f(4,b,c,e,a,d) +#define s_afterI3(f) f(3,a,b,e,c,d) +#define s_afterI2(f) f(2,b,d,e,c,a) +#define s_afterI1(f) f(1,a,b,c,e,d) +#define s_afterI0(f) f(0,a,d,b,e,c) + +/* The instruction sequences for the S-box functions + * come from Dag Arne Osvik's paper "Speeding up Serpent". + */ + +#define s_s0(i, r0, r1, r2, r3, r4) { \ + r3 ^= r0; \ + r4 = r1; \ + r1 &= r3; \ + r4 ^= r2; \ + r1 ^= r0; \ + r0 |= r3; \ + r0 ^= r4; \ + r4 ^= r3; \ + r3 ^= r2; \ + r2 |= r1; \ + r2 ^= r4; \ + r4 = ~r4; \ + r4 |= r1; \ + r1 ^= r3; \ + r1 ^= r4; \ + r3 |= r0; \ + r1 ^= r3; \ + r4 ^= r3; \ +} + +#define s_i0(i, r0, r1, r2, r3, r4) { \ + r2 = ~r2; \ + r4 = r1; \ + r1 |= r0; \ + r4 = ~r4; \ + r1 ^= r2; \ + r2 |= r4; \ + r1 ^= r3; \ + r0 ^= r4; \ + r2 ^= r0; \ + r0 &= r3; \ + r4 ^= r0; \ + r0 |= r1; \ + r0 ^= r2; \ + r3 ^= r4; \ + r2 ^= r1; \ + r3 ^= r0; \ + r3 ^= r1; \ + r2 &= r3; \ + r4 ^= r2; \ +} + +#define s_s1(i, r0, r1, r2, r3, r4) { \ + r0 = ~r0; \ + r2 = ~r2; \ + r4 = r0; \ + r0 &= r1; \ + r2 ^= r0; \ + r0 |= r3; \ + r3 ^= r2; \ + r1 ^= r0; \ + r0 ^= r4; \ + r4 |= r1; \ + r1 ^= r3; \ + r2 |= r0; \ + r2 &= r4; \ + r0 ^= r1; \ + r1 &= r2; \ + r1 ^= r0; \ + r0 &= r2; \ + r0 ^= r4; \ +} + +#define s_i1(i, r0, r1, r2, r3, r4) { \ + r4 = r1; \ + r1 ^= r3; \ + r3 &= r1; \ + r4 ^= r2; \ + r3 ^= r0; \ + r0 |= r1; \ + r2 ^= r3; \ + r0 ^= r4; \ + r0 |= r2; \ + r1 ^= r3; \ + r0 ^= r1; \ + r1 |= r3; \ + r1 ^= r0; \ + r4 = ~r4; \ + r4 ^= r1; \ + r1 |= r0; \ + r1 ^= r0; \ + r1 |= r4; \ + r3 ^= r1; \ +} + +#define s_s2(i, r0, r1, r2, r3, r4) { \ + r4 = r0; \ + r0 &= r2; \ + r0 ^= r3; \ + r2 ^= r1; \ + r2 ^= r0; \ + r3 |= r4; \ + r3 ^= r1; \ + r4 ^= r2; \ + r1 = r3; \ + r3 |= r4; \ + r3 ^= r0; \ + r0 &= r1; \ + r4 ^= r0; \ + r1 ^= r3; \ + r1 ^= r4; \ + r4 = ~r4; \ +} + +#define s_i2(i, r0, r1, r2, r3, r4) { \ + r2 ^= r3; \ + r3 ^= r0; \ + r4 = r3; \ + r3 &= r2; \ + r3 ^= r1; \ + r1 |= r2; \ + r1 ^= r4; \ + r4 &= r3; \ + r2 ^= r3; \ + r4 &= r0; \ + r4 ^= r2; \ + r2 &= r1; \ + r2 |= r0; \ + r3 = ~r3; \ + r2 ^= r3; \ + r0 ^= r3; \ + r0 &= r1; \ + r3 ^= r4; \ + r3 ^= r0; \ +} + +#define s_s3(i, r0, r1, r2, r3, r4) { \ + r4 = r0; \ + r0 |= r3; \ + r3 ^= r1; \ + r1 &= r4; \ + r4 ^= r2; \ + r2 ^= r3; \ + r3 &= r0; \ + r4 |= r1; \ + r3 ^= r4; \ + r0 ^= r1; \ + r4 &= r0; \ + r1 ^= r3; \ + r4 ^= r2; \ + r1 |= r0; \ + r1 ^= r2; \ + r0 ^= r3; \ + r2 = r1; \ + r1 |= r3; \ + r1 ^= r0; \ +} + +#define s_i3(i, r0, r1, r2, r3, r4) { \ + r4 = r2; \ + r2 ^= r1; \ + r1 &= r2; \ + r1 ^= r0; \ + r0 &= r4; \ + r4 ^= r3; \ + r3 |= r1; \ + r3 ^= r2; \ + r0 ^= r4; \ + r2 ^= r0; \ + r0 |= r3; \ + r0 ^= r1; \ + r4 ^= r2; \ + r2 &= r3; \ + r1 |= r3; \ + r1 ^= r2; \ + r4 ^= r0; \ + r2 ^= r4; \ +} + +#define s_s4(i, r0, r1, r2, r3, r4) { \ + r1 ^= r3; \ + r3 = ~r3; \ + r2 ^= r3; \ + r3 ^= r0; \ + r4 = r1; \ + r1 &= r3; \ + r1 ^= r2; \ + r4 ^= r3; \ + r0 ^= r4; \ + r2 &= r4; \ + r2 ^= r0; \ + r0 &= r1; \ + r3 ^= r0; \ + r4 |= r1; \ + r4 ^= r0; \ + r0 |= r3; \ + r0 ^= r2; \ + r2 &= r3; \ + r0 = ~r0; \ + r4 ^= r2; \ +} + +#define s_i4(i, r0, r1, r2, r3, r4) { \ + r4 = r2; \ + r2 &= r3; \ + r2 ^= r1; \ + r1 |= r3; \ + r1 &= r0; \ + r4 ^= r2; \ + r4 ^= r1; \ + r1 &= r2; \ + r0 = ~r0; \ + r3 ^= r4; \ + r1 ^= r3; \ + r3 &= r0; \ + r3 ^= r2; \ + r0 ^= r1; \ + r2 &= r0; \ + r3 ^= r0; \ + r2 ^= r4; \ + r2 |= r3; \ + r3 ^= r0; \ + r2 ^= r1; \ +} + +#define s_s5(i, r0, r1, r2, r3, r4) { \ + r0 ^= r1; \ + r1 ^= r3; \ + r3 = ~r3; \ + r4 = r1; \ + r1 &= r0; \ + r2 ^= r3; \ + r1 ^= r2; \ + r2 |= r4; \ + r4 ^= r3; \ + r3 &= r1; \ + r3 ^= r0; \ + r4 ^= r1; \ + r4 ^= r2; \ + r2 ^= r0; \ + r0 &= r3; \ + r2 = ~r2; \ + r0 ^= r4; \ + r4 |= r3; \ + r2 ^= r4; \ +} + +#define s_i5(i, r0, r1, r2, r3, r4) { \ + r1 = ~r1; \ + r4 = r3; \ + r2 ^= r1; \ + r3 |= r0; \ + r3 ^= r2; \ + r2 |= r1; \ + r2 &= r0; \ + r4 ^= r3; \ + r2 ^= r4; \ + r4 |= r0; \ + r4 ^= r1; \ + r1 &= r2; \ + r1 ^= r3; \ + r4 ^= r2; \ + r3 &= r4; \ + r4 ^= r1; \ + r3 ^= r0; \ + r3 ^= r4; \ + r4 = ~r4; \ +} + +#define s_s6(i, r0, r1, r2, r3, r4) { \ + r2 = ~r2; \ + r4 = r3; \ + r3 &= r0; \ + r0 ^= r4; \ + r3 ^= r2; \ + r2 |= r4; \ + r1 ^= r3; \ + r2 ^= r0; \ + r0 |= r1; \ + r2 ^= r1; \ + r4 ^= r0; \ + r0 |= r3; \ + r0 ^= r2; \ + r4 ^= r3; \ + r4 ^= r0; \ + r3 = ~r3; \ + r2 &= r4; \ + r2 ^= r3; \ +} + +#define s_i6(i, r0, r1, r2, r3, r4) { \ + r0 ^= r2; \ + r4 = r2; \ + r2 &= r0; \ + r4 ^= r3; \ + r2 = ~r2; \ + r3 ^= r1; \ + r2 ^= r3; \ + r4 |= r0; \ + r0 ^= r2; \ + r3 ^= r4; \ + r4 ^= r1; \ + r1 &= r3; \ + r1 ^= r0; \ + r0 ^= r3; \ + r0 |= r2; \ + r3 ^= r1; \ + r4 ^= r0; \ +} + +#define s_s7(i, r0, r1, r2, r3, r4) { \ + r4 = r2; \ + r2 &= r1; \ + r2 ^= r3; \ + r3 &= r1; \ + r4 ^= r2; \ + r2 ^= r1; \ + r1 ^= r0; \ + r0 |= r4; \ + r0 ^= r2; \ + r3 ^= r1; \ + r2 ^= r3; \ + r3 &= r0; \ + r3 ^= r4; \ + r4 ^= r2; \ + r2 &= r0; \ + r4 = ~r4; \ + r2 ^= r4; \ + r4 &= r0; \ + r1 ^= r3; \ + r4 ^= r1; \ +} + +#define s_i7(i, r0, r1, r2, r3, r4) { \ + r4 = r2; \ + r2 ^= r0; \ + r0 &= r3; \ + r2 = ~r2; \ + r4 |= r3; \ + r3 ^= r1; \ + r1 |= r0; \ + r0 ^= r2; \ + r2 &= r4; \ + r1 ^= r2; \ + r2 ^= r0; \ + r0 |= r2; \ + r3 &= r4; \ + r0 ^= r3; \ + r4 ^= r1; \ + r3 ^= r4; \ + r4 |= r0; \ + r3 ^= r2; \ + r4 ^= r2; \ +} + +/* key xor */ +#define s_kx(r, a, b, c, d, e) { \ + a ^= k[4 * r + 0]; \ + b ^= k[4 * r + 1]; \ + c ^= k[4 * r + 2]; \ + d ^= k[4 * r + 3]; \ +} + +#define s_lk(r, a, b, c, d, e) { \ + a = k[(8-r)*4 + 0]; \ + b = k[(8-r)*4 + 1]; \ + c = k[(8-r)*4 + 2]; \ + d = k[(8-r)*4 + 3]; \ +} + +#define s_sk(r, a, b, c, d, e) { \ + k[(8-r)*4 + 4] = a; \ + k[(8-r)*4 + 5] = b; \ + k[(8-r)*4 + 6] = c; \ + k[(8-r)*4 + 7] = d; \ +} + +static int s_setup_key(const unsigned char *key, int keylen, int rounds, ulong32 *k) +{ + int i; + ulong32 t; + ulong32 k0[8] = { 0 }; /* zero-initialize */ + ulong32 a, b, c, d, e; + + for (i = 0; i < 8 && i < keylen/4; ++i) { + LOAD32L(k0[i], key + i * 4); + } + if (keylen < 32) { + k0[keylen/4] |= (ulong32)1 << ((keylen%4)*8); + } + + t = k0[7]; + for (i = 0; i < 8; ++i) { + k[i] = k0[i] = t = ROLc(k0[i] ^ k0[(i+3)%8] ^ k0[(i+5)%8] ^ t ^ 0x9e3779b9 ^ i, 11); + } + for (i = 8; i < 4*(rounds+1); ++i) { + k[i] = t = ROLc(k[i-8] ^ k[i-5] ^ k[i-3] ^ t ^ 0x9e3779b9 ^ i, 11); + } + k -= 20; + + for (i = 0; i < rounds/8; i++) { + s_afterS2(s_lk); s_afterS2(s_s3); s_afterS3(s_sk); + s_afterS1(s_lk); s_afterS1(s_s2); s_afterS2(s_sk); + s_afterS0(s_lk); s_afterS0(s_s1); s_afterS1(s_sk); + s_beforeS0(s_lk); s_beforeS0(s_s0); s_afterS0(s_sk); + k += 8*4; + s_afterS6(s_lk); s_afterS6(s_s7); s_afterS7(s_sk); + s_afterS5(s_lk); s_afterS5(s_s6); s_afterS6(s_sk); + s_afterS4(s_lk); s_afterS4(s_s5); s_afterS5(s_sk); + s_afterS3(s_lk); s_afterS3(s_s4); s_afterS4(s_sk); + } + s_afterS2(s_lk); s_afterS2(s_s3); s_afterS3(s_sk); + + return CRYPT_OK; +} + +static int s_enc_block(const unsigned char *in, unsigned char *out, const ulong32 *k) +{ + ulong32 a, b, c, d, e; + unsigned int i = 1; + + LOAD32L(a, in + 0); + LOAD32L(b, in + 4); + LOAD32L(c, in + 8); + LOAD32L(d, in + 12); + + do { + s_beforeS0(s_kx); s_beforeS0(s_s0); s_afterS0(s_lt); + s_afterS0(s_kx); s_afterS0(s_s1); s_afterS1(s_lt); + s_afterS1(s_kx); s_afterS1(s_s2); s_afterS2(s_lt); + s_afterS2(s_kx); s_afterS2(s_s3); s_afterS3(s_lt); + s_afterS3(s_kx); s_afterS3(s_s4); s_afterS4(s_lt); + s_afterS4(s_kx); s_afterS4(s_s5); s_afterS5(s_lt); + s_afterS5(s_kx); s_afterS5(s_s6); s_afterS6(s_lt); + s_afterS6(s_kx); s_afterS6(s_s7); + + if (i == 4) break; + + ++i; + c = b; + b = e; + e = d; + d = a; + a = e; + k += 32; + s_beforeS0(s_lt); + } while (1); + + s_afterS7(s_kx); + + STORE32L(d, out + 0); + STORE32L(e, out + 4); + STORE32L(b, out + 8); + STORE32L(a, out + 12); + + return CRYPT_OK; +} + +static int s_dec_block(const unsigned char *in, unsigned char *out, const ulong32 *k) +{ + ulong32 a, b, c, d, e; + unsigned int i; + + LOAD32L(a, in + 0); + LOAD32L(b, in + 4); + LOAD32L(c, in + 8); + LOAD32L(d, in + 12); + e = 0; LTC_UNUSED_PARAM(e); /* avoid scan-build warning */ + i = 4; + k += 96; + + s_beforeI7(s_kx); + goto start; + + do { + c = b; + b = d; + d = e; + k -= 32; + s_beforeI7(s_ilt); +start: + s_beforeI7(s_i7); s_afterI7(s_kx); + s_afterI7(s_ilt); s_afterI7(s_i6); s_afterI6(s_kx); + s_afterI6(s_ilt); s_afterI6(s_i5); s_afterI5(s_kx); + s_afterI5(s_ilt); s_afterI5(s_i4); s_afterI4(s_kx); + s_afterI4(s_ilt); s_afterI4(s_i3); s_afterI3(s_kx); + s_afterI3(s_ilt); s_afterI3(s_i2); s_afterI2(s_kx); + s_afterI2(s_ilt); s_afterI2(s_i1); s_afterI1(s_kx); + s_afterI1(s_ilt); s_afterI1(s_i0); s_afterI0(s_kx); + } while (--i != 0); + + STORE32L(a, out + 0); + STORE32L(d, out + 4); + STORE32L(b, out + 8); + STORE32L(e, out + 12); + + return CRYPT_OK; +} + +int serpent_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (num_rounds != 0 && num_rounds != 32) return CRYPT_INVALID_ROUNDS; + if (keylen != 16 && keylen != 24 && keylen != 32) return CRYPT_INVALID_KEYSIZE; + + err = s_setup_key(key, keylen, 32, skey->serpent.k); +#ifdef LTC_CLEAN_STACK + burn_stack(sizeof(ulong32) * 14 + sizeof(int)); +#endif + return err; +} + +int serpent_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_enc_block(pt, ct, skey->serpent.k); +#ifdef LTC_CLEAN_STACK + burn_stack(sizeof(ulong32) * 5 + sizeof(int)); +#endif + return err; +} + +int serpent_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_dec_block(ct, pt, skey->serpent.k); +#ifdef LTC_CLEAN_STACK + burn_stack(sizeof(ulong32) * 5 + sizeof(int)); +#endif + return err; +} + +void serpent_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +int serpent_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + + if (*keysize >= 32) { *keysize = 32; } + else if (*keysize >= 24) { *keysize = 24; } + else if (*keysize >= 16) { *keysize = 16; } + else return CRYPT_INVALID_KEYSIZE; + return CRYPT_OK; +} + +int serpent_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char key[32]; + int keylen; + unsigned char pt[16], ct[16]; + } tests[] = { + { + /* key */ {0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 32, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0xA2,0x23,0xAA,0x12,0x88,0x46,0x3C,0x0E,0x2B,0xE3,0x8E,0xBD,0x82,0x56,0x16,0xC0} + }, + { + /* key */ {0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 32, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0xEA,0xE1,0xD4,0x05,0x57,0x01,0x74,0xDF,0x7D,0xF2,0xF9,0x96,0x6D,0x50,0x91,0x59} + }, + { + /* key */ {0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 32, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0x65,0xF3,0x76,0x84,0x47,0x1E,0x92,0x1D,0xC8,0xA3,0x0F,0x45,0xB4,0x3C,0x44,0x99} + }, + { + /* key */ {0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 24, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0x9E,0x27,0x4E,0xAD,0x9B,0x73,0x7B,0xB2,0x1E,0xFC,0xFC,0xA5,0x48,0x60,0x26,0x89} + }, + { + /* key */ {0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 24, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0x92,0xFC,0x8E,0x51,0x03,0x99,0xE4,0x6A,0x04,0x1B,0xF3,0x65,0xE7,0xB3,0xAE,0x82} + }, + { + /* key */ {0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 24, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0x5E,0x0D,0xA3,0x86,0xC4,0x6A,0xD4,0x93,0xDE,0xA2,0x03,0xFD,0xC6,0xF5,0x7D,0x70} + }, + { + /* key */ {0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 16, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0x26,0x4E,0x54,0x81,0xEF,0xF4,0x2A,0x46,0x06,0xAB,0xDA,0x06,0xC0,0xBF,0xDA,0x3D} + }, + { + /* key */ {0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 16, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0x4A,0x23,0x1B,0x3B,0xC7,0x27,0x99,0x34,0x07,0xAC,0x6E,0xC8,0x35,0x0E,0x85,0x24} + }, + { + /* key */ {0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* keylen */ 16, + /* pt */ {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, + /* ct */ {0xE0,0x32,0x69,0xF9,0xE9,0xFD,0x85,0x3C,0x7D,0x81,0x56,0xDF,0x14,0xB9,0x8D,0x56} + } + }; + + unsigned char buf[2][16]; + symmetric_key key; + int err, x; + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + if ((err = serpent_setup(tests[x].key, tests[x].keylen, 0, &key)) != CRYPT_OK) { + return err; + } + if ((err = serpent_ecb_encrypt(tests[x].pt, buf[0], &key)) != CRYPT_OK) { + return err; + } + if (compare_testvector(buf[0], 16, tests[x].ct, 16, "SERPENT Encrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + if ((err = serpent_ecb_decrypt(tests[x].ct, buf[1], &key)) != CRYPT_OK) { + return err; + } + if (compare_testvector(buf[1], 16, tests[x].pt, 16, "SERPENT Decrypt", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/skipjack.c b/optee_os/core/lib/libtomcrypt/src/ciphers/skipjack.c new file mode 100644 index 0000000..0251946 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/skipjack.c @@ -0,0 +1,334 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file skipjack.c + Skipjack Implementation by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_SKIPJACK + +const struct ltc_cipher_descriptor skipjack_desc = +{ + "skipjack", + 17, + 10, 10, 8, 32, + &skipjack_setup, + &skipjack_ecb_encrypt, + &skipjack_ecb_decrypt, + &skipjack_test, + &skipjack_done, + &skipjack_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +static const unsigned char sbox[256] = { + 0xa3,0xd7,0x09,0x83,0xf8,0x48,0xf6,0xf4,0xb3,0x21,0x15,0x78,0x99,0xb1,0xaf,0xf9, + 0xe7,0x2d,0x4d,0x8a,0xce,0x4c,0xca,0x2e,0x52,0x95,0xd9,0x1e,0x4e,0x38,0x44,0x28, + 0x0a,0xdf,0x02,0xa0,0x17,0xf1,0x60,0x68,0x12,0xb7,0x7a,0xc3,0xe9,0xfa,0x3d,0x53, + 0x96,0x84,0x6b,0xba,0xf2,0x63,0x9a,0x19,0x7c,0xae,0xe5,0xf5,0xf7,0x16,0x6a,0xa2, + 0x39,0xb6,0x7b,0x0f,0xc1,0x93,0x81,0x1b,0xee,0xb4,0x1a,0xea,0xd0,0x91,0x2f,0xb8, + 0x55,0xb9,0xda,0x85,0x3f,0x41,0xbf,0xe0,0x5a,0x58,0x80,0x5f,0x66,0x0b,0xd8,0x90, + 0x35,0xd5,0xc0,0xa7,0x33,0x06,0x65,0x69,0x45,0x00,0x94,0x56,0x6d,0x98,0x9b,0x76, + 0x97,0xfc,0xb2,0xc2,0xb0,0xfe,0xdb,0x20,0xe1,0xeb,0xd6,0xe4,0xdd,0x47,0x4a,0x1d, + 0x42,0xed,0x9e,0x6e,0x49,0x3c,0xcd,0x43,0x27,0xd2,0x07,0xd4,0xde,0xc7,0x67,0x18, + 0x89,0xcb,0x30,0x1f,0x8d,0xc6,0x8f,0xaa,0xc8,0x74,0xdc,0xc9,0x5d,0x5c,0x31,0xa4, + 0x70,0x88,0x61,0x2c,0x9f,0x0d,0x2b,0x87,0x50,0x82,0x54,0x64,0x26,0x7d,0x03,0x40, + 0x34,0x4b,0x1c,0x73,0xd1,0xc4,0xfd,0x3b,0xcc,0xfb,0x7f,0xab,0xe6,0x3e,0x5b,0xa5, + 0xad,0x04,0x23,0x9c,0x14,0x51,0x22,0xf0,0x29,0x79,0x71,0x7e,0xff,0x8c,0x0e,0xe2, + 0x0c,0xef,0xbc,0x72,0x75,0x6f,0x37,0xa1,0xec,0xd3,0x8e,0x62,0x8b,0x86,0x10,0xe8, + 0x08,0x77,0x11,0xbe,0x92,0x4f,0x24,0xc5,0x32,0x36,0x9d,0xcf,0xf3,0xa6,0xbb,0xac, + 0x5e,0x6c,0xa9,0x13,0x57,0x25,0xb5,0xe3,0xbd,0xa8,0x3a,0x01,0x05,0x59,0x2a,0x46 +}; + +/* simple x + 1 (mod 10) in one step. */ +static const int keystep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; + +/* simple x - 1 (mod 10) in one step */ +static const int ikeystep[] = { 9, 0, 1, 2, 3, 4, 5, 6, 7, 8 }; + + /** + Initialize the Skipjack block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +int skipjack_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int x; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + if (keylen != 10) { + return CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 32 && num_rounds != 0) { + return CRYPT_INVALID_ROUNDS; + } + + /* make sure the key is in range for platforms where CHAR_BIT != 8 */ + for (x = 0; x < 10; x++) { + skey->skipjack.key[x] = key[x] & 255; + } + + return CRYPT_OK; +} + +#define RULE_A \ + tmp = g_func(w1, &kp, skey->skipjack.key); \ + w1 = tmp ^ w4 ^ x; \ + w4 = w3; w3 = w2; \ + w2 = tmp; + +#define RULE_B \ + tmp = g_func(w1, &kp, skey->skipjack.key); \ + tmp1 = w4; w4 = w3; \ + w3 = w1 ^ w2 ^ x; \ + w1 = tmp1; w2 = tmp; + +#define RULE_A1 \ + tmp = w1 ^ w2 ^ x; \ + w1 = ig_func(w2, &kp, skey->skipjack.key); \ + w2 = w3; w3 = w4; w4 = tmp; + +#define RULE_B1 \ + tmp = ig_func(w2, &kp, skey->skipjack.key); \ + w2 = tmp ^ w3 ^ x; \ + w3 = w4; w4 = w1; w1 = tmp; + +static unsigned g_func(unsigned w, int *kp, const unsigned char *key) +{ + unsigned char g1,g2; + + g1 = (w >> 8) & 255; g2 = w & 255; + g1 ^= sbox[g2^key[*kp]]; *kp = keystep[*kp]; + g2 ^= sbox[g1^key[*kp]]; *kp = keystep[*kp]; + g1 ^= sbox[g2^key[*kp]]; *kp = keystep[*kp]; + g2 ^= sbox[g1^key[*kp]]; *kp = keystep[*kp]; + return ((unsigned)g1<<8)|(unsigned)g2; +} + +static unsigned ig_func(unsigned w, int *kp, const unsigned char *key) +{ + unsigned char g1,g2; + + g1 = (w >> 8) & 255; g2 = w & 255; + *kp = ikeystep[*kp]; g2 ^= sbox[g1^key[*kp]]; + *kp = ikeystep[*kp]; g1 ^= sbox[g2^key[*kp]]; + *kp = ikeystep[*kp]; g2 ^= sbox[g1^key[*kp]]; + *kp = ikeystep[*kp]; g1 ^= sbox[g2^key[*kp]]; + return ((unsigned)g1<<8)|(unsigned)g2; +} + +/** + Encrypts a block of text with Skipjack + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_skipjack_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int skipjack_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + unsigned w1,w2,w3,w4,tmp,tmp1; + int x, kp; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + /* load block */ + w1 = ((unsigned)pt[0]<<8)|pt[1]; + w2 = ((unsigned)pt[2]<<8)|pt[3]; + w3 = ((unsigned)pt[4]<<8)|pt[5]; + w4 = ((unsigned)pt[6]<<8)|pt[7]; + + /* 8 rounds of RULE A */ + for (x = 1, kp = 0; x < 9; x++) { + RULE_A; + } + + /* 8 rounds of RULE B */ + for (; x < 17; x++) { + RULE_B; + } + + /* 8 rounds of RULE A */ + for (; x < 25; x++) { + RULE_A; + } + + /* 8 rounds of RULE B */ + for (; x < 33; x++) { + RULE_B; + } + + /* store block */ + ct[0] = (w1>>8)&255; ct[1] = w1&255; + ct[2] = (w2>>8)&255; ct[3] = w2&255; + ct[4] = (w3>>8)&255; ct[5] = w3&255; + ct[6] = (w4>>8)&255; ct[7] = w4&255; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int skipjack_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_skipjack_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(unsigned) * 8 + sizeof(int) * 2); + return err; +} +#endif + +/** + Decrypts a block of text with Skipjack + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_skipjack_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int skipjack_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + unsigned w1,w2,w3,w4,tmp; + int x, kp; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + /* load block */ + w1 = ((unsigned)ct[0]<<8)|ct[1]; + w2 = ((unsigned)ct[2]<<8)|ct[3]; + w3 = ((unsigned)ct[4]<<8)|ct[5]; + w4 = ((unsigned)ct[6]<<8)|ct[7]; + + /* 8 rounds of RULE B^-1 + + Note the value "kp = 8" comes from "kp = (32 * 4) mod 10" where 32*4 is 128 which mod 10 is 8 + */ + for (x = 32, kp = 8; x > 24; x--) { + RULE_B1; + } + + /* 8 rounds of RULE A^-1 */ + for (; x > 16; x--) { + RULE_A1; + } + + + /* 8 rounds of RULE B^-1 */ + for (; x > 8; x--) { + RULE_B1; + } + + /* 8 rounds of RULE A^-1 */ + for (; x > 0; x--) { + RULE_A1; + } + + /* store block */ + pt[0] = (w1>>8)&255; pt[1] = w1&255; + pt[2] = (w2>>8)&255; pt[3] = w2&255; + pt[4] = (w3>>8)&255; pt[5] = w3&255; + pt[6] = (w4>>8)&255; pt[7] = w4&255; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int skipjack_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_skipjack_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(unsigned) * 7 + sizeof(int) * 2); + return err; +} +#endif + +/** + Performs a self-test of the Skipjack block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int skipjack_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + unsigned char key[10], pt[8], ct[8]; + } tests[] = { + { + { 0x00, 0x99, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, + { 0x33, 0x22, 0x11, 0x00, 0xdd, 0xcc, 0xbb, 0xaa }, + { 0x25, 0x87, 0xca, 0xe2, 0x7a, 0x12, 0xd3, 0x00 } + } + }; + unsigned char buf[2][8]; + int x, y, err; + symmetric_key key; + + for (x = 0; x < (int)(sizeof(tests) / sizeof(tests[0])); x++) { + /* setup key */ + if ((err = skipjack_setup(tests[x].key, 10, 0, &key)) != CRYPT_OK) { + return err; + } + + /* encrypt and decrypt */ + skipjack_ecb_encrypt(tests[x].pt, buf[0], &key); + skipjack_ecb_decrypt(buf[0], buf[1], &key); + + /* compare */ + if (compare_testvector(buf[0], 8, tests[x].ct, 8, "Skipjack Encrypt", x) != 0 || + compare_testvector(buf[1], 8, tests[x].pt, 8, "Skipjack Decrypt", x) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) buf[0][y] = 0; + for (y = 0; y < 1000; y++) skipjack_ecb_encrypt(buf[0], buf[0], &key); + for (y = 0; y < 1000; y++) skipjack_ecb_decrypt(buf[0], buf[0], &key); + for (y = 0; y < 8; y++) if (buf[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void skipjack_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int skipjack_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 10) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize > 10) { + *keysize = 10; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/sub.mk b/optee_os/core/lib/libtomcrypt/src/ciphers/sub.mk new file mode 100644 index 0000000..78cf037 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/sub.mk @@ -0,0 +1,2 @@ +subdirs-$(call cfg-one-enabled, _CFG_CORE_LTC_AES _CFG_CORE_LTC_AES_DESC) += aes +srcs-$(_CFG_CORE_LTC_DES) += des.c diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/tea.c b/optee_os/core/lib/libtomcrypt/src/ciphers/tea.c new file mode 100644 index 0000000..fc413d4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/tea.c @@ -0,0 +1,209 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file tea.c + Implementation of TEA, Steffen Jaeckel +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_TEA + +const struct ltc_cipher_descriptor tea_desc = +{ + "tea", + 26, + 16, 16, 8, 32, + &tea_setup, + &tea_ecb_encrypt, + &tea_ecb_decrypt, + &tea_test, + &tea_done, + &tea_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +#define DELTA 0x9E3779B9uL +#define SUM 0xC6EF3720uL + +int tea_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* check arguments */ + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 0 && num_rounds != 32) { + return CRYPT_INVALID_ROUNDS; + } + + /* load key */ + LOAD32H(skey->tea.k[0], key+0); + LOAD32H(skey->tea.k[1], key+4); + LOAD32H(skey->tea.k[2], key+8); + LOAD32H(skey->tea.k[3], key+12); + + return CRYPT_OK; +} + +/** + Encrypts a block of text with TEA + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int tea_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong32 y, z, sum = 0; + const ulong32 delta = DELTA; + int r; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(y, &pt[0]); + LOAD32H(z, &pt[4]); + for (r = 0; r < 32; r++) { + sum += delta; + y += ((z<<4) + skey->tea.k[0]) ^ (z + sum) ^ ((z>>5) + skey->tea.k[1]); + z += ((y<<4) + skey->tea.k[2]) ^ (y + sum) ^ ((y>>5) + skey->tea.k[3]); + } + STORE32H(y, &ct[0]); + STORE32H(z, &ct[4]); + return CRYPT_OK; +} + +/** + Decrypts a block of text with TEA + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int tea_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong32 v0, v1, sum = SUM; + const ulong32 delta = DELTA; + int r; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(v0, &ct[0]); + LOAD32H(v1, &ct[4]); + + for (r = 0; r < 32; r++) { + v1 -= ((v0 << 4) + skey->tea.k[2]) ^ (v0 + sum) ^ ((v0 >> 5) + skey->tea.k[3]); + v0 -= ((v1 << 4) + skey->tea.k[0]) ^ (v1 + sum) ^ ((v1 >> 5) + skey->tea.k[1]); + sum -= delta; + } + + STORE32H(v0, &pt[0]); + STORE32H(v1, &pt[4]); + return CRYPT_OK; +} + +/** + Performs a self-test of the TEA block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int tea_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *key, *pt, *ct; + } tests[] = { + { + "00000000000000000000000000000000", + "0000000000000000", + "41ea3a0a94baa940" + }, { + "32a1e65408b63bb9214105744ec5d2e2", + "5ada1d89a9c3801a", + "dd46249e28aa0b4b" + }, { + "60388adadf70a1f5d9cb4e097d2c6c57", + "7a6adb4d69c53e0f", + "44b71215cf25368a" + }, { + "4368d2249bd0321eb7c56d5b63a1bfac", + "5a5d7ca2e186c41a", + "91f56dff7281794f" + }, { + "5c60bff27072d01c4513c5eb8f3a38ab", + "80d9c4adcf899635", + "2bb0f1b3c023ed11" + } + }; + unsigned char ptct[2][8]; + unsigned char tmp[2][8]; + unsigned char key[16]; + unsigned long l; + symmetric_key skey; + size_t i; + int err, y; + for (i = 0; i < sizeof(tests)/sizeof(tests[0]); i++) { + zeromem(&skey, sizeof(skey)); + + l = sizeof(key); + if ((err = base16_decode(tests[i].key, XSTRLEN(tests[i].key), key, &l)) != CRYPT_OK) return err; + l = sizeof(ptct[0]); + if ((err = base16_decode(tests[i].pt, XSTRLEN(tests[i].pt), ptct[0], &l)) != CRYPT_OK) return err; + l = sizeof(ptct[1]); + if ((err = base16_decode(tests[i].ct, XSTRLEN(tests[i].ct), ptct[1], &l)) != CRYPT_OK) return err; + + if ((err = tea_setup(key, 16, 0, &skey)) != CRYPT_OK) { + return err; + } + tea_ecb_encrypt(ptct[0], tmp[0], &skey); + tea_ecb_decrypt(tmp[0], tmp[1], &skey); + + if (compare_testvector(tmp[0], 8, ptct[1], 8, "TEA Encrypt", i) != 0 || + compare_testvector(tmp[1], 8, ptct[0], 8, "TEA Decrypt", i) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) tea_ecb_encrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 1000; y++) tea_ecb_decrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 8; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } /* for */ + + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void tea_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int tea_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + *keysize = 16; + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/twofish/twofish.c b/optee_os/core/lib/libtomcrypt/src/ciphers/twofish/twofish.c new file mode 100644 index 0000000..c006361 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/twofish/twofish.c @@ -0,0 +1,706 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + + /** + @file twofish.c + Implementation of Twofish by Tom St Denis + */ +#include "tomcrypt_private.h" + +#ifdef LTC_TWOFISH + +/* first LTC_TWOFISH_ALL_TABLES must ensure LTC_TWOFISH_TABLES is defined */ +#ifdef LTC_TWOFISH_ALL_TABLES +#ifndef LTC_TWOFISH_TABLES +#define LTC_TWOFISH_TABLES +#endif +#endif + +const struct ltc_cipher_descriptor twofish_desc = +{ + "twofish", + 7, + 16, 32, 16, 16, + &twofish_setup, + &twofish_ecb_encrypt, + &twofish_ecb_decrypt, + &twofish_test, + &twofish_done, + &twofish_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +/* the two polynomials */ +#ifndef LTC_TWOFISH_TABLES +#define MDS_POLY 0x169 +#endif +#ifndef LTC_TWOFISH_ALL_TABLES +#define RS_POLY 0x14D +#endif + +/* The 4x8 RS Linear Transform */ +static const unsigned char RS[4][8] = { + { 0x01, 0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E }, + { 0xA4, 0x56, 0x82, 0xF3, 0X1E, 0XC6, 0X68, 0XE5 }, + { 0X02, 0XA1, 0XFC, 0XC1, 0X47, 0XAE, 0X3D, 0X19 }, + { 0XA4, 0X55, 0X87, 0X5A, 0X58, 0XDB, 0X9E, 0X03 } +}; + +#ifdef LTC_TWOFISH_SMALL +/* sbox usage orderings */ +static const unsigned char qord[4][5] = { + { 1, 1, 0, 0, 1 }, + { 0, 1, 1, 0, 0 }, + { 0, 0, 0, 1, 1 }, + { 1, 0, 1, 1, 0 } +}; +#endif /* LTC_TWOFISH_SMALL */ + +#ifdef LTC_TWOFISH_TABLES + +#define LTC_TWOFISH_TAB_C +#include "twofish_tab.c" + +#define sbox(i, x) ((ulong32)SBOX[i][(x)&255]) + +#else + +/* The Q-box tables */ +static const unsigned char qbox[2][4][16] = { +{ + { 0x8, 0x1, 0x7, 0xD, 0x6, 0xF, 0x3, 0x2, 0x0, 0xB, 0x5, 0x9, 0xE, 0xC, 0xA, 0x4 }, + { 0xE, 0XC, 0XB, 0X8, 0X1, 0X2, 0X3, 0X5, 0XF, 0X4, 0XA, 0X6, 0X7, 0X0, 0X9, 0XD }, + { 0XB, 0XA, 0X5, 0XE, 0X6, 0XD, 0X9, 0X0, 0XC, 0X8, 0XF, 0X3, 0X2, 0X4, 0X7, 0X1 }, + { 0XD, 0X7, 0XF, 0X4, 0X1, 0X2, 0X6, 0XE, 0X9, 0XB, 0X3, 0X0, 0X8, 0X5, 0XC, 0XA } +}, +{ + { 0X2, 0X8, 0XB, 0XD, 0XF, 0X7, 0X6, 0XE, 0X3, 0X1, 0X9, 0X4, 0X0, 0XA, 0XC, 0X5 }, + { 0X1, 0XE, 0X2, 0XB, 0X4, 0XC, 0X3, 0X7, 0X6, 0XD, 0XA, 0X5, 0XF, 0X9, 0X0, 0X8 }, + { 0X4, 0XC, 0X7, 0X5, 0X1, 0X6, 0X9, 0XA, 0X0, 0XE, 0XD, 0X8, 0X2, 0XB, 0X3, 0XF }, + { 0xB, 0X9, 0X5, 0X1, 0XC, 0X3, 0XD, 0XE, 0X6, 0X4, 0X7, 0XF, 0X2, 0X0, 0X8, 0XA } +} +}; + +/* computes S_i[x] */ +#ifdef LTC_CLEAN_STACK +static ulong32 s_sbox(int i, ulong32 x) +#else +static ulong32 sbox(int i, ulong32 x) +#endif +{ + unsigned char a0,b0,a1,b1,a2,b2,a3,b3,a4,b4,y; + + /* a0,b0 = [x/16], x mod 16 */ + a0 = (unsigned char)((x>>4)&15); + b0 = (unsigned char)((x)&15); + + /* a1 = a0 ^ b0 */ + a1 = a0 ^ b0; + + /* b1 = a0 ^ ROR(b0, 1) ^ 8a0 */ + b1 = (a0 ^ ((b0<<3)|(b0>>1)) ^ (a0<<3)) & 15; + + /* a2,b2 = t0[a1], t1[b1] */ + a2 = qbox[i][0][(int)a1]; + b2 = qbox[i][1][(int)b1]; + + /* a3 = a2 ^ b2 */ + a3 = a2 ^ b2; + + /* b3 = a2 ^ ROR(b2, 1) ^ 8a2 */ + b3 = (a2 ^ ((b2<<3)|(b2>>1)) ^ (a2<<3)) & 15; + + /* a4,b4 = t2[a3], t3[b3] */ + a4 = qbox[i][2][(int)a3]; + b4 = qbox[i][3][(int)b3]; + + /* y = 16b4 + a4 */ + y = (b4 << 4) + a4; + + /* return result */ + return (ulong32)y; +} + +#ifdef LTC_CLEAN_STACK +static ulong32 sbox(int i, ulong32 x) +{ + ulong32 y; + y = s_sbox(i, x); + burn_stack(sizeof(unsigned char) * 11); + return y; +} +#endif /* LTC_CLEAN_STACK */ + +#endif /* LTC_TWOFISH_TABLES */ + +/* computes ab mod p */ +static ulong32 gf_mult(ulong32 a, ulong32 b, ulong32 p) +{ + ulong32 result, B[2], P[2]; + + P[1] = p; + B[1] = b; + result = P[0] = B[0] = 0; + + /* unrolled branchless GF multiplier */ + result ^= B[a&1]; a >>= 1; B[1] = P[B[1]>>7] ^ (B[1] << 1); + result ^= B[a&1]; a >>= 1; B[1] = P[B[1]>>7] ^ (B[1] << 1); + result ^= B[a&1]; a >>= 1; B[1] = P[B[1]>>7] ^ (B[1] << 1); + result ^= B[a&1]; a >>= 1; B[1] = P[B[1]>>7] ^ (B[1] << 1); + result ^= B[a&1]; a >>= 1; B[1] = P[B[1]>>7] ^ (B[1] << 1); + result ^= B[a&1]; a >>= 1; B[1] = P[B[1]>>7] ^ (B[1] << 1); + result ^= B[a&1]; a >>= 1; B[1] = P[B[1]>>7] ^ (B[1] << 1); + result ^= B[a&1]; + + return result; +} + +/* computes [y0 y1 y2 y3] = MDS . [x0] */ +#ifndef LTC_TWOFISH_TABLES +static ulong32 mds_column_mult(unsigned char in, int col) +{ + ulong32 x01, x5B, xEF; + + x01 = in; + x5B = gf_mult(in, 0x5B, MDS_POLY); + xEF = gf_mult(in, 0xEF, MDS_POLY); + + switch (col) { + case 0: + return (x01 << 0 ) | + (x5B << 8 ) | + (xEF << 16) | + (xEF << 24); + case 1: + return (xEF << 0 ) | + (xEF << 8 ) | + (x5B << 16) | + (x01 << 24); + case 2: + return (x5B << 0 ) | + (xEF << 8 ) | + (x01 << 16) | + (xEF << 24); + case 3: + return (x5B << 0 ) | + (x01 << 8 ) | + (xEF << 16) | + (x5B << 24); + } + /* avoid warnings, we'd never get here normally but just to calm compiler warnings... */ + return 0; +} + +#else /* !LTC_TWOFISH_TABLES */ + +#define mds_column_mult(x, i) mds_tab[i][x] + +#endif /* LTC_TWOFISH_TABLES */ + +/* Computes [y0 y1 y2 y3] = MDS . [x0 x1 x2 x3] */ +static void mds_mult(const unsigned char *in, unsigned char *out) +{ + int x; + ulong32 tmp; + for (tmp = x = 0; x < 4; x++) { + tmp ^= mds_column_mult(in[x], x); + } + STORE32L(tmp, out); +} + +#ifdef LTC_TWOFISH_ALL_TABLES +/* computes [y0 y1 y2 y3] = RS . [x0 x1 x2 x3 x4 x5 x6 x7] */ +static void rs_mult(const unsigned char *in, unsigned char *out) +{ + ulong32 tmp; + tmp = rs_tab0[in[0]] ^ rs_tab1[in[1]] ^ rs_tab2[in[2]] ^ rs_tab3[in[3]] ^ + rs_tab4[in[4]] ^ rs_tab5[in[5]] ^ rs_tab6[in[6]] ^ rs_tab7[in[7]]; + STORE32L(tmp, out); +} + +#else /* !LTC_TWOFISH_ALL_TABLES */ + +/* computes [y0 y1 y2 y3] = RS . [x0 x1 x2 x3 x4 x5 x6 x7] */ +static void rs_mult(const unsigned char *in, unsigned char *out) +{ + int x, y; + for (x = 0; x < 4; x++) { + out[x] = 0; + for (y = 0; y < 8; y++) { + out[x] ^= gf_mult(in[y], RS[x][y], RS_POLY); + } + } +} + +#endif + +/* computes h(x) */ +static void h_func(const unsigned char *in, unsigned char *out, const unsigned char *M, int k, int offset) +{ + int x; + unsigned char y[4]; + for (x = 0; x < 4; x++) { + y[x] = in[x]; + } + switch (k) { + case 4: + y[0] = (unsigned char)(sbox(1, (ulong32)y[0]) ^ M[4 * (6 + offset) + 0]); + y[1] = (unsigned char)(sbox(0, (ulong32)y[1]) ^ M[4 * (6 + offset) + 1]); + y[2] = (unsigned char)(sbox(0, (ulong32)y[2]) ^ M[4 * (6 + offset) + 2]); + y[3] = (unsigned char)(sbox(1, (ulong32)y[3]) ^ M[4 * (6 + offset) + 3]); + /* FALLTHROUGH */ + case 3: + y[0] = (unsigned char)(sbox(1, (ulong32)y[0]) ^ M[4 * (4 + offset) + 0]); + y[1] = (unsigned char)(sbox(1, (ulong32)y[1]) ^ M[4 * (4 + offset) + 1]); + y[2] = (unsigned char)(sbox(0, (ulong32)y[2]) ^ M[4 * (4 + offset) + 2]); + y[3] = (unsigned char)(sbox(0, (ulong32)y[3]) ^ M[4 * (4 + offset) + 3]); + /* FALLTHROUGH */ + case 2: + y[0] = (unsigned char)(sbox(1, sbox(0, sbox(0, (ulong32)y[0]) ^ M[4 * (2 + offset) + 0]) ^ M[4 * (0 + offset) + 0])); + y[1] = (unsigned char)(sbox(0, sbox(0, sbox(1, (ulong32)y[1]) ^ M[4 * (2 + offset) + 1]) ^ M[4 * (0 + offset) + 1])); + y[2] = (unsigned char)(sbox(1, sbox(1, sbox(0, (ulong32)y[2]) ^ M[4 * (2 + offset) + 2]) ^ M[4 * (0 + offset) + 2])); + y[3] = (unsigned char)(sbox(0, sbox(1, sbox(1, (ulong32)y[3]) ^ M[4 * (2 + offset) + 3]) ^ M[4 * (0 + offset) + 3])); + /* FALLTHROUGH */ + } + mds_mult(y, out); +} + +#ifndef LTC_TWOFISH_SMALL + +/* for GCC we don't use pointer aliases */ +#if defined(__GNUC__) + #define S1 skey->twofish.S[0] + #define S2 skey->twofish.S[1] + #define S3 skey->twofish.S[2] + #define S4 skey->twofish.S[3] +#endif + +/* the G function */ +#define g_func(x, dum) (S1[LTC_BYTE(x,0)] ^ S2[LTC_BYTE(x,1)] ^ S3[LTC_BYTE(x,2)] ^ S4[LTC_BYTE(x,3)]) +#define g1_func(x, dum) (S2[LTC_BYTE(x,0)] ^ S3[LTC_BYTE(x,1)] ^ S4[LTC_BYTE(x,2)] ^ S1[LTC_BYTE(x,3)]) + +#else + +#ifdef LTC_CLEAN_STACK +static ulong32 s_g_func(ulong32 x, const symmetric_key *key) +#else +static ulong32 g_func(ulong32 x, const symmetric_key *key) +#endif +{ + unsigned char g, i, y, z; + ulong32 res; + + res = 0; + for (y = 0; y < 4; y++) { + z = key->twofish.start; + + /* do unkeyed substitution */ + g = sbox(qord[y][z++], (x >> (8*y)) & 255); + + /* first subkey */ + i = 0; + + /* do key mixing+sbox until z==5 */ + while (z != 5) { + g = g ^ key->twofish.S[4*i++ + y]; + g = sbox(qord[y][z++], g); + } + + /* multiply g by a column of the MDS */ + res ^= mds_column_mult(g, y); + } + return res; +} + +#define g1_func(x, key) g_func(ROLc(x, 8), key) + +#ifdef LTC_CLEAN_STACK +static ulong32 g_func(ulong32 x, const symmetric_key *key) +{ + ulong32 y; + y = s_g_func(x, key); + burn_stack(sizeof(unsigned char) * 4 + sizeof(ulong32)); + return y; +} +#endif /* LTC_CLEAN_STACK */ + +#endif /* LTC_TWOFISH_SMALL */ + + /** + Initialize the Twofish block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +#ifdef LTC_CLEAN_STACK +static int s_twofish_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#else +int twofish_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +#endif +{ +#ifndef LTC_TWOFISH_SMALL + unsigned char S[4*4], tmpx0, tmpx1; +#endif + int k, x, y; + unsigned char tmp[4], tmp2[4], M[8*4]; + ulong32 A, B; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* invalid arguments? */ + if (num_rounds != 16 && num_rounds != 0) { + return CRYPT_INVALID_ROUNDS; + } + + if (keylen != 16 && keylen != 24 && keylen != 32) { + return CRYPT_INVALID_KEYSIZE; + } + + /* k = keysize/64 [but since our keysize is in bytes...] */ + k = keylen / 8; + + /* copy the key into M */ + for (x = 0; x < keylen; x++) { + M[x] = key[x] & 255; + } + + /* create the S[..] words */ +#ifndef LTC_TWOFISH_SMALL + for (x = 0; x < k; x++) { + rs_mult(M+(x*8), S+(x*4)); + } +#else + for (x = 0; x < k; x++) { + rs_mult(M+(x*8), skey->twofish.S+(x*4)); + } +#endif + + /* make subkeys */ + for (x = 0; x < 20; x++) { + /* A = h(p * 2x, Me) */ + for (y = 0; y < 4; y++) { + tmp[y] = x+x; + } + h_func(tmp, tmp2, M, k, 0); + LOAD32L(A, tmp2); + + /* B = ROL(h(p * (2x + 1), Mo), 8) */ + for (y = 0; y < 4; y++) { + tmp[y] = (unsigned char)(x+x+1); + } + h_func(tmp, tmp2, M, k, 1); + LOAD32L(B, tmp2); + B = ROLc(B, 8); + + /* K[2i] = A + B */ + skey->twofish.K[x+x] = (A + B) & 0xFFFFFFFFUL; + + /* K[2i+1] = (A + 2B) <<< 9 */ + skey->twofish.K[x+x+1] = ROLc(B + B + A, 9); + } + +#ifndef LTC_TWOFISH_SMALL + /* make the sboxes (large ram variant) */ + if (k == 2) { + for (x = 0; x < 256; x++) { + tmpx0 = (unsigned char)sbox(0, x); + tmpx1 = (unsigned char)sbox(1, x); + skey->twofish.S[0][x] = mds_column_mult(sbox(1, (sbox(0, tmpx0 ^ S[0]) ^ S[4])),0); + skey->twofish.S[1][x] = mds_column_mult(sbox(0, (sbox(0, tmpx1 ^ S[1]) ^ S[5])),1); + skey->twofish.S[2][x] = mds_column_mult(sbox(1, (sbox(1, tmpx0 ^ S[2]) ^ S[6])),2); + skey->twofish.S[3][x] = mds_column_mult(sbox(0, (sbox(1, tmpx1 ^ S[3]) ^ S[7])),3); + } + } else if (k == 3) { + for (x = 0; x < 256; x++) { + tmpx0 = (unsigned char)sbox(0, x); + tmpx1 = (unsigned char)sbox(1, x); + skey->twofish.S[0][x] = mds_column_mult(sbox(1, (sbox(0, sbox(0, tmpx1 ^ S[0]) ^ S[4]) ^ S[8])),0); + skey->twofish.S[1][x] = mds_column_mult(sbox(0, (sbox(0, sbox(1, tmpx1 ^ S[1]) ^ S[5]) ^ S[9])),1); + skey->twofish.S[2][x] = mds_column_mult(sbox(1, (sbox(1, sbox(0, tmpx0 ^ S[2]) ^ S[6]) ^ S[10])),2); + skey->twofish.S[3][x] = mds_column_mult(sbox(0, (sbox(1, sbox(1, tmpx0 ^ S[3]) ^ S[7]) ^ S[11])),3); + } + } else { + for (x = 0; x < 256; x++) { + tmpx0 = (unsigned char)sbox(0, x); + tmpx1 = (unsigned char)sbox(1, x); + skey->twofish.S[0][x] = mds_column_mult(sbox(1, (sbox(0, sbox(0, sbox(1, tmpx1 ^ S[0]) ^ S[4]) ^ S[8]) ^ S[12])),0); + skey->twofish.S[1][x] = mds_column_mult(sbox(0, (sbox(0, sbox(1, sbox(1, tmpx0 ^ S[1]) ^ S[5]) ^ S[9]) ^ S[13])),1); + skey->twofish.S[2][x] = mds_column_mult(sbox(1, (sbox(1, sbox(0, sbox(0, tmpx0 ^ S[2]) ^ S[6]) ^ S[10]) ^ S[14])),2); + skey->twofish.S[3][x] = mds_column_mult(sbox(0, (sbox(1, sbox(1, sbox(0, tmpx1 ^ S[3]) ^ S[7]) ^ S[11]) ^ S[15])),3); + } + } +#else + /* where to start in the sbox layers */ + /* small ram variant */ + switch (k) { + case 4 : skey->twofish.start = 0; break; + case 3 : skey->twofish.start = 1; break; + default: skey->twofish.start = 2; break; + } +#endif + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int twofish_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + int x; + x = s_twofish_setup(key, keylen, num_rounds, skey); + burn_stack(sizeof(int) * 7 + sizeof(unsigned char) * 56 + sizeof(ulong32) * 2); + return x; +} +#endif + +/** + Encrypts a block of text with Twofish + @param pt The input plaintext (16 bytes) + @param ct The output ciphertext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_twofish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#else +int twofish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +#endif +{ + ulong32 a,b,c,d,ta,tb,tc,td,t1,t2; + const ulong32 *k; + int r; +#if !defined(LTC_TWOFISH_SMALL) && !defined(__GNUC__) + const ulong32 *S1, *S2, *S3, *S4; +#endif + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + +#if !defined(LTC_TWOFISH_SMALL) && !defined(__GNUC__) + S1 = skey->twofish.S[0]; + S2 = skey->twofish.S[1]; + S3 = skey->twofish.S[2]; + S4 = skey->twofish.S[3]; +#endif + + LOAD32L(a,&pt[0]); LOAD32L(b,&pt[4]); + LOAD32L(c,&pt[8]); LOAD32L(d,&pt[12]); + a ^= skey->twofish.K[0]; + b ^= skey->twofish.K[1]; + c ^= skey->twofish.K[2]; + d ^= skey->twofish.K[3]; + + k = skey->twofish.K + 8; + for (r = 8; r != 0; --r) { + t2 = g1_func(b, skey); + t1 = g_func(a, skey) + t2; + c = RORc(c ^ (t1 + k[0]), 1); + d = ROLc(d, 1) ^ (t2 + t1 + k[1]); + + t2 = g1_func(d, skey); + t1 = g_func(c, skey) + t2; + a = RORc(a ^ (t1 + k[2]), 1); + b = ROLc(b, 1) ^ (t2 + t1 + k[3]); + k += 4; + } + + /* output with "undo last swap" */ + ta = c ^ skey->twofish.K[4]; + tb = d ^ skey->twofish.K[5]; + tc = a ^ skey->twofish.K[6]; + td = b ^ skey->twofish.K[7]; + + /* store output */ + STORE32L(ta,&ct[0]); STORE32L(tb,&ct[4]); + STORE32L(tc,&ct[8]); STORE32L(td,&ct[12]); + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int twofish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + int err = s_twofish_ecb_encrypt(pt, ct, skey); + burn_stack(sizeof(ulong32) * 10 + sizeof(int)); + return err; +} +#endif + +/** + Decrypts a block of text with Twofish + @param ct The input ciphertext (16 bytes) + @param pt The output plaintext (16 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +#ifdef LTC_CLEAN_STACK +static int s_twofish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#else +int twofish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +#endif +{ + ulong32 a,b,c,d,ta,tb,tc,td,t1,t2; + const ulong32 *k; + int r; +#if !defined(LTC_TWOFISH_SMALL) && !defined(__GNUC__) + const ulong32 *S1, *S2, *S3, *S4; +#endif + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + +#if !defined(LTC_TWOFISH_SMALL) && !defined(__GNUC__) + S1 = skey->twofish.S[0]; + S2 = skey->twofish.S[1]; + S3 = skey->twofish.S[2]; + S4 = skey->twofish.S[3]; +#endif + + /* load input */ + LOAD32L(ta,&ct[0]); LOAD32L(tb,&ct[4]); + LOAD32L(tc,&ct[8]); LOAD32L(td,&ct[12]); + + /* undo undo final swap */ + a = tc ^ skey->twofish.K[6]; + b = td ^ skey->twofish.K[7]; + c = ta ^ skey->twofish.K[4]; + d = tb ^ skey->twofish.K[5]; + + k = skey->twofish.K + 36; + for (r = 8; r != 0; --r) { + t2 = g1_func(d, skey); + t1 = g_func(c, skey) + t2; + a = ROLc(a, 1) ^ (t1 + k[2]); + b = RORc(b ^ (t2 + t1 + k[3]), 1); + + t2 = g1_func(b, skey); + t1 = g_func(a, skey) + t2; + c = ROLc(c, 1) ^ (t1 + k[0]); + d = RORc(d ^ (t2 + t1 + k[1]), 1); + k -= 4; + } + + /* pre-white */ + a ^= skey->twofish.K[0]; + b ^= skey->twofish.K[1]; + c ^= skey->twofish.K[2]; + d ^= skey->twofish.K[3]; + + /* store */ + STORE32L(a, &pt[0]); STORE32L(b, &pt[4]); + STORE32L(c, &pt[8]); STORE32L(d, &pt[12]); + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +int twofish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + int err = s_twofish_ecb_decrypt(ct, pt, skey); + burn_stack(sizeof(ulong32) * 10 + sizeof(int)); + return err; +} +#endif + +/** + Performs a self-test of the Twofish block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int twofish_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + int keylen; + unsigned char key[32], pt[16], ct[16]; + } tests[] = { + { 16, + { 0x9F, 0x58, 0x9F, 0x5C, 0xF6, 0x12, 0x2C, 0x32, + 0xB6, 0xBF, 0xEC, 0x2F, 0x2A, 0xE8, 0xC3, 0x5A }, + { 0xD4, 0x91, 0xDB, 0x16, 0xE7, 0xB1, 0xC3, 0x9E, + 0x86, 0xCB, 0x08, 0x6B, 0x78, 0x9F, 0x54, 0x19 }, + { 0x01, 0x9F, 0x98, 0x09, 0xDE, 0x17, 0x11, 0x85, + 0x8F, 0xAA, 0xC3, 0xA3, 0xBA, 0x20, 0xFB, 0xC3 } + }, { + 24, + { 0x88, 0xB2, 0xB2, 0x70, 0x6B, 0x10, 0x5E, 0x36, + 0xB4, 0x46, 0xBB, 0x6D, 0x73, 0x1A, 0x1E, 0x88, + 0xEF, 0xA7, 0x1F, 0x78, 0x89, 0x65, 0xBD, 0x44 }, + { 0x39, 0xDA, 0x69, 0xD6, 0xBA, 0x49, 0x97, 0xD5, + 0x85, 0xB6, 0xDC, 0x07, 0x3C, 0xA3, 0x41, 0xB2 }, + { 0x18, 0x2B, 0x02, 0xD8, 0x14, 0x97, 0xEA, 0x45, + 0xF9, 0xDA, 0xAC, 0xDC, 0x29, 0x19, 0x3A, 0x65 } + }, { + 32, + { 0xD4, 0x3B, 0xB7, 0x55, 0x6E, 0xA3, 0x2E, 0x46, + 0xF2, 0xA2, 0x82, 0xB7, 0xD4, 0x5B, 0x4E, 0x0D, + 0x57, 0xFF, 0x73, 0x9D, 0x4D, 0xC9, 0x2C, 0x1B, + 0xD7, 0xFC, 0x01, 0x70, 0x0C, 0xC8, 0x21, 0x6F }, + { 0x90, 0xAF, 0xE9, 0x1B, 0xB2, 0x88, 0x54, 0x4F, + 0x2C, 0x32, 0xDC, 0x23, 0x9B, 0x26, 0x35, 0xE6 }, + { 0x6C, 0xB4, 0x56, 0x1C, 0x40, 0xBF, 0x0A, 0x97, + 0x05, 0x93, 0x1C, 0xB6, 0xD4, 0x08, 0xE7, 0xFA } + } +}; + + + symmetric_key key; + unsigned char tmp[2][16]; + int err, i, y; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + if ((err = twofish_setup(tests[i].key, tests[i].keylen, 0, &key)) != CRYPT_OK) { + return err; + } + twofish_ecb_encrypt(tests[i].pt, tmp[0], &key); + twofish_ecb_decrypt(tmp[0], tmp[1], &key); + if (compare_testvector(tmp[0], 16, tests[i].ct, 16, "Twofish Encrypt", i) != 0 || + compare_testvector(tmp[1], 16, tests[i].pt, 16, "Twofish Decrypt", i) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 16; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) twofish_ecb_encrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 1000; y++) twofish_ecb_decrypt(tmp[0], tmp[0], &key); + for (y = 0; y < 16; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; +#endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void twofish_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int twofish_keysize(int *keysize) +{ + LTC_ARGCHK(keysize); + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + if (*keysize < 24) { + *keysize = 16; + return CRYPT_OK; + } + if (*keysize < 32) { + *keysize = 24; + return CRYPT_OK; + } + *keysize = 32; + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/twofish/twofish_tab.c b/optee_os/core/lib/libtomcrypt/src/ciphers/twofish/twofish_tab.c new file mode 100644 index 0000000..e52d284 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/twofish/twofish_tab.c @@ -0,0 +1,486 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + + /** + @file twofish_tab.c + Twofish tables, Tom St Denis + */ +#ifdef LTC_TWOFISH_TABLES +#ifdef LTC_TWOFISH_TAB_C + +/* pre generated 8x8 tables from the four 4x4s */ +static const unsigned char SBOX[2][256] = { +{ + 0xa9, 0x67, 0xb3, 0xe8, 0x04, 0xfd, 0xa3, 0x76, 0x9a, 0x92, + 0x80, 0x78, 0xe4, 0xdd, 0xd1, 0x38, 0x0d, 0xc6, 0x35, 0x98, + 0x18, 0xf7, 0xec, 0x6c, 0x43, 0x75, 0x37, 0x26, 0xfa, 0x13, + 0x94, 0x48, 0xf2, 0xd0, 0x8b, 0x30, 0x84, 0x54, 0xdf, 0x23, + 0x19, 0x5b, 0x3d, 0x59, 0xf3, 0xae, 0xa2, 0x82, 0x63, 0x01, + 0x83, 0x2e, 0xd9, 0x51, 0x9b, 0x7c, 0xa6, 0xeb, 0xa5, 0xbe, + 0x16, 0x0c, 0xe3, 0x61, 0xc0, 0x8c, 0x3a, 0xf5, 0x73, 0x2c, + 0x25, 0x0b, 0xbb, 0x4e, 0x89, 0x6b, 0x53, 0x6a, 0xb4, 0xf1, + 0xe1, 0xe6, 0xbd, 0x45, 0xe2, 0xf4, 0xb6, 0x66, 0xcc, 0x95, + 0x03, 0x56, 0xd4, 0x1c, 0x1e, 0xd7, 0xfb, 0xc3, 0x8e, 0xb5, + 0xe9, 0xcf, 0xbf, 0xba, 0xea, 0x77, 0x39, 0xaf, 0x33, 0xc9, + 0x62, 0x71, 0x81, 0x79, 0x09, 0xad, 0x24, 0xcd, 0xf9, 0xd8, + 0xe5, 0xc5, 0xb9, 0x4d, 0x44, 0x08, 0x86, 0xe7, 0xa1, 0x1d, + 0xaa, 0xed, 0x06, 0x70, 0xb2, 0xd2, 0x41, 0x7b, 0xa0, 0x11, + 0x31, 0xc2, 0x27, 0x90, 0x20, 0xf6, 0x60, 0xff, 0x96, 0x5c, + 0xb1, 0xab, 0x9e, 0x9c, 0x52, 0x1b, 0x5f, 0x93, 0x0a, 0xef, + 0x91, 0x85, 0x49, 0xee, 0x2d, 0x4f, 0x8f, 0x3b, 0x47, 0x87, + 0x6d, 0x46, 0xd6, 0x3e, 0x69, 0x64, 0x2a, 0xce, 0xcb, 0x2f, + 0xfc, 0x97, 0x05, 0x7a, 0xac, 0x7f, 0xd5, 0x1a, 0x4b, 0x0e, + 0xa7, 0x5a, 0x28, 0x14, 0x3f, 0x29, 0x88, 0x3c, 0x4c, 0x02, + 0xb8, 0xda, 0xb0, 0x17, 0x55, 0x1f, 0x8a, 0x7d, 0x57, 0xc7, + 0x8d, 0x74, 0xb7, 0xc4, 0x9f, 0x72, 0x7e, 0x15, 0x22, 0x12, + 0x58, 0x07, 0x99, 0x34, 0x6e, 0x50, 0xde, 0x68, 0x65, 0xbc, + 0xdb, 0xf8, 0xc8, 0xa8, 0x2b, 0x40, 0xdc, 0xfe, 0x32, 0xa4, + 0xca, 0x10, 0x21, 0xf0, 0xd3, 0x5d, 0x0f, 0x00, 0x6f, 0x9d, + 0x36, 0x42, 0x4a, 0x5e, 0xc1, 0xe0}, +{ + 0x75, 0xf3, 0xc6, 0xf4, 0xdb, 0x7b, 0xfb, 0xc8, 0x4a, 0xd3, + 0xe6, 0x6b, 0x45, 0x7d, 0xe8, 0x4b, 0xd6, 0x32, 0xd8, 0xfd, + 0x37, 0x71, 0xf1, 0xe1, 0x30, 0x0f, 0xf8, 0x1b, 0x87, 0xfa, + 0x06, 0x3f, 0x5e, 0xba, 0xae, 0x5b, 0x8a, 0x00, 0xbc, 0x9d, + 0x6d, 0xc1, 0xb1, 0x0e, 0x80, 0x5d, 0xd2, 0xd5, 0xa0, 0x84, + 0x07, 0x14, 0xb5, 0x90, 0x2c, 0xa3, 0xb2, 0x73, 0x4c, 0x54, + 0x92, 0x74, 0x36, 0x51, 0x38, 0xb0, 0xbd, 0x5a, 0xfc, 0x60, + 0x62, 0x96, 0x6c, 0x42, 0xf7, 0x10, 0x7c, 0x28, 0x27, 0x8c, + 0x13, 0x95, 0x9c, 0xc7, 0x24, 0x46, 0x3b, 0x70, 0xca, 0xe3, + 0x85, 0xcb, 0x11, 0xd0, 0x93, 0xb8, 0xa6, 0x83, 0x20, 0xff, + 0x9f, 0x77, 0xc3, 0xcc, 0x03, 0x6f, 0x08, 0xbf, 0x40, 0xe7, + 0x2b, 0xe2, 0x79, 0x0c, 0xaa, 0x82, 0x41, 0x3a, 0xea, 0xb9, + 0xe4, 0x9a, 0xa4, 0x97, 0x7e, 0xda, 0x7a, 0x17, 0x66, 0x94, + 0xa1, 0x1d, 0x3d, 0xf0, 0xde, 0xb3, 0x0b, 0x72, 0xa7, 0x1c, + 0xef, 0xd1, 0x53, 0x3e, 0x8f, 0x33, 0x26, 0x5f, 0xec, 0x76, + 0x2a, 0x49, 0x81, 0x88, 0xee, 0x21, 0xc4, 0x1a, 0xeb, 0xd9, + 0xc5, 0x39, 0x99, 0xcd, 0xad, 0x31, 0x8b, 0x01, 0x18, 0x23, + 0xdd, 0x1f, 0x4e, 0x2d, 0xf9, 0x48, 0x4f, 0xf2, 0x65, 0x8e, + 0x78, 0x5c, 0x58, 0x19, 0x8d, 0xe5, 0x98, 0x57, 0x67, 0x7f, + 0x05, 0x64, 0xaf, 0x63, 0xb6, 0xfe, 0xf5, 0xb7, 0x3c, 0xa5, + 0xce, 0xe9, 0x68, 0x44, 0xe0, 0x4d, 0x43, 0x69, 0x29, 0x2e, + 0xac, 0x15, 0x59, 0xa8, 0x0a, 0x9e, 0x6e, 0x47, 0xdf, 0x34, + 0x35, 0x6a, 0xcf, 0xdc, 0x22, 0xc9, 0xc0, 0x9b, 0x89, 0xd4, + 0xed, 0xab, 0x12, 0xa2, 0x0d, 0x52, 0xbb, 0x02, 0x2f, 0xa9, + 0xd7, 0x61, 0x1e, 0xb4, 0x50, 0x04, 0xf6, 0xc2, 0x16, 0x25, + 0x86, 0x56, 0x55, 0x09, 0xbe, 0x91} +}; + +/* the 4x4 MDS in a nicer format */ +static const ulong32 mds_tab[4][256] = { +{ +0x00000000UL, 0xefef5b01UL, 0xb7b7b602UL, 0x5858ed03UL, 0x07070504UL, 0xe8e85e05UL, 0xb0b0b306UL, 0x5f5fe807UL, +0x0e0e0a08UL, 0xe1e15109UL, 0xb9b9bc0aUL, 0x5656e70bUL, 0x09090f0cUL, 0xe6e6540dUL, 0xbebeb90eUL, 0x5151e20fUL, +0x1c1c1410UL, 0xf3f34f11UL, 0xababa212UL, 0x4444f913UL, 0x1b1b1114UL, 0xf4f44a15UL, 0xacaca716UL, 0x4343fc17UL, +0x12121e18UL, 0xfdfd4519UL, 0xa5a5a81aUL, 0x4a4af31bUL, 0x15151b1cUL, 0xfafa401dUL, 0xa2a2ad1eUL, 0x4d4df61fUL, +0x38382820UL, 0xd7d77321UL, 0x8f8f9e22UL, 0x6060c523UL, 0x3f3f2d24UL, 0xd0d07625UL, 0x88889b26UL, 0x6767c027UL, +0x36362228UL, 0xd9d97929UL, 0x8181942aUL, 0x6e6ecf2bUL, 0x3131272cUL, 0xdede7c2dUL, 0x8686912eUL, 0x6969ca2fUL, +0x24243c30UL, 0xcbcb6731UL, 0x93938a32UL, 0x7c7cd133UL, 0x23233934UL, 0xcccc6235UL, 0x94948f36UL, 0x7b7bd437UL, +0x2a2a3638UL, 0xc5c56d39UL, 0x9d9d803aUL, 0x7272db3bUL, 0x2d2d333cUL, 0xc2c2683dUL, 0x9a9a853eUL, 0x7575de3fUL, +0x70705040UL, 0x9f9f0b41UL, 0xc7c7e642UL, 0x2828bd43UL, 0x77775544UL, 0x98980e45UL, 0xc0c0e346UL, 0x2f2fb847UL, +0x7e7e5a48UL, 0x91910149UL, 0xc9c9ec4aUL, 0x2626b74bUL, 0x79795f4cUL, 0x9696044dUL, 0xcecee94eUL, 0x2121b24fUL, +0x6c6c4450UL, 0x83831f51UL, 0xdbdbf252UL, 0x3434a953UL, 0x6b6b4154UL, 0x84841a55UL, 0xdcdcf756UL, 0x3333ac57UL, +0x62624e58UL, 0x8d8d1559UL, 0xd5d5f85aUL, 0x3a3aa35bUL, 0x65654b5cUL, 0x8a8a105dUL, 0xd2d2fd5eUL, 0x3d3da65fUL, +0x48487860UL, 0xa7a72361UL, 0xffffce62UL, 0x10109563UL, 0x4f4f7d64UL, 0xa0a02665UL, 0xf8f8cb66UL, 0x17179067UL, +0x46467268UL, 0xa9a92969UL, 0xf1f1c46aUL, 0x1e1e9f6bUL, 0x4141776cUL, 0xaeae2c6dUL, 0xf6f6c16eUL, 0x19199a6fUL, +0x54546c70UL, 0xbbbb3771UL, 0xe3e3da72UL, 0x0c0c8173UL, 0x53536974UL, 0xbcbc3275UL, 0xe4e4df76UL, 0x0b0b8477UL, +0x5a5a6678UL, 0xb5b53d79UL, 0xededd07aUL, 0x02028b7bUL, 0x5d5d637cUL, 0xb2b2387dUL, 0xeaead57eUL, 0x05058e7fUL, +0xe0e0a080UL, 0x0f0ffb81UL, 0x57571682UL, 0xb8b84d83UL, 0xe7e7a584UL, 0x0808fe85UL, 0x50501386UL, 0xbfbf4887UL, +0xeeeeaa88UL, 0x0101f189UL, 0x59591c8aUL, 0xb6b6478bUL, 0xe9e9af8cUL, 0x0606f48dUL, 0x5e5e198eUL, 0xb1b1428fUL, +0xfcfcb490UL, 0x1313ef91UL, 0x4b4b0292UL, 0xa4a45993UL, 0xfbfbb194UL, 0x1414ea95UL, 0x4c4c0796UL, 0xa3a35c97UL, +0xf2f2be98UL, 0x1d1de599UL, 0x4545089aUL, 0xaaaa539bUL, 0xf5f5bb9cUL, 0x1a1ae09dUL, 0x42420d9eUL, 0xadad569fUL, +0xd8d888a0UL, 0x3737d3a1UL, 0x6f6f3ea2UL, 0x808065a3UL, 0xdfdf8da4UL, 0x3030d6a5UL, 0x68683ba6UL, 0x878760a7UL, +0xd6d682a8UL, 0x3939d9a9UL, 0x616134aaUL, 0x8e8e6fabUL, 0xd1d187acUL, 0x3e3edcadUL, 0x666631aeUL, 0x89896aafUL, +0xc4c49cb0UL, 0x2b2bc7b1UL, 0x73732ab2UL, 0x9c9c71b3UL, 0xc3c399b4UL, 0x2c2cc2b5UL, 0x74742fb6UL, 0x9b9b74b7UL, +0xcaca96b8UL, 0x2525cdb9UL, 0x7d7d20baUL, 0x92927bbbUL, 0xcdcd93bcUL, 0x2222c8bdUL, 0x7a7a25beUL, 0x95957ebfUL, +0x9090f0c0UL, 0x7f7fabc1UL, 0x272746c2UL, 0xc8c81dc3UL, 0x9797f5c4UL, 0x7878aec5UL, 0x202043c6UL, 0xcfcf18c7UL, +0x9e9efac8UL, 0x7171a1c9UL, 0x29294ccaUL, 0xc6c617cbUL, 0x9999ffccUL, 0x7676a4cdUL, 0x2e2e49ceUL, 0xc1c112cfUL, +0x8c8ce4d0UL, 0x6363bfd1UL, 0x3b3b52d2UL, 0xd4d409d3UL, 0x8b8be1d4UL, 0x6464bad5UL, 0x3c3c57d6UL, 0xd3d30cd7UL, +0x8282eed8UL, 0x6d6db5d9UL, 0x353558daUL, 0xdada03dbUL, 0x8585ebdcUL, 0x6a6ab0ddUL, 0x32325ddeUL, 0xdddd06dfUL, +0xa8a8d8e0UL, 0x474783e1UL, 0x1f1f6ee2UL, 0xf0f035e3UL, 0xafafdde4UL, 0x404086e5UL, 0x18186be6UL, 0xf7f730e7UL, +0xa6a6d2e8UL, 0x494989e9UL, 0x111164eaUL, 0xfefe3febUL, 0xa1a1d7ecUL, 0x4e4e8cedUL, 0x161661eeUL, 0xf9f93aefUL, +0xb4b4ccf0UL, 0x5b5b97f1UL, 0x03037af2UL, 0xecec21f3UL, 0xb3b3c9f4UL, 0x5c5c92f5UL, 0x04047ff6UL, 0xebeb24f7UL, +0xbabac6f8UL, 0x55559df9UL, 0x0d0d70faUL, 0xe2e22bfbUL, 0xbdbdc3fcUL, 0x525298fdUL, 0x0a0a75feUL, 0xe5e52effUL +}, +{ +0x00000000UL, 0x015befefUL, 0x02b6b7b7UL, 0x03ed5858UL, 0x04050707UL, 0x055ee8e8UL, 0x06b3b0b0UL, 0x07e85f5fUL, +0x080a0e0eUL, 0x0951e1e1UL, 0x0abcb9b9UL, 0x0be75656UL, 0x0c0f0909UL, 0x0d54e6e6UL, 0x0eb9bebeUL, 0x0fe25151UL, +0x10141c1cUL, 0x114ff3f3UL, 0x12a2ababUL, 0x13f94444UL, 0x14111b1bUL, 0x154af4f4UL, 0x16a7acacUL, 0x17fc4343UL, +0x181e1212UL, 0x1945fdfdUL, 0x1aa8a5a5UL, 0x1bf34a4aUL, 0x1c1b1515UL, 0x1d40fafaUL, 0x1eada2a2UL, 0x1ff64d4dUL, +0x20283838UL, 0x2173d7d7UL, 0x229e8f8fUL, 0x23c56060UL, 0x242d3f3fUL, 0x2576d0d0UL, 0x269b8888UL, 0x27c06767UL, +0x28223636UL, 0x2979d9d9UL, 0x2a948181UL, 0x2bcf6e6eUL, 0x2c273131UL, 0x2d7cdedeUL, 0x2e918686UL, 0x2fca6969UL, +0x303c2424UL, 0x3167cbcbUL, 0x328a9393UL, 0x33d17c7cUL, 0x34392323UL, 0x3562ccccUL, 0x368f9494UL, 0x37d47b7bUL, +0x38362a2aUL, 0x396dc5c5UL, 0x3a809d9dUL, 0x3bdb7272UL, 0x3c332d2dUL, 0x3d68c2c2UL, 0x3e859a9aUL, 0x3fde7575UL, +0x40507070UL, 0x410b9f9fUL, 0x42e6c7c7UL, 0x43bd2828UL, 0x44557777UL, 0x450e9898UL, 0x46e3c0c0UL, 0x47b82f2fUL, +0x485a7e7eUL, 0x49019191UL, 0x4aecc9c9UL, 0x4bb72626UL, 0x4c5f7979UL, 0x4d049696UL, 0x4ee9ceceUL, 0x4fb22121UL, +0x50446c6cUL, 0x511f8383UL, 0x52f2dbdbUL, 0x53a93434UL, 0x54416b6bUL, 0x551a8484UL, 0x56f7dcdcUL, 0x57ac3333UL, +0x584e6262UL, 0x59158d8dUL, 0x5af8d5d5UL, 0x5ba33a3aUL, 0x5c4b6565UL, 0x5d108a8aUL, 0x5efdd2d2UL, 0x5fa63d3dUL, +0x60784848UL, 0x6123a7a7UL, 0x62ceffffUL, 0x63951010UL, 0x647d4f4fUL, 0x6526a0a0UL, 0x66cbf8f8UL, 0x67901717UL, +0x68724646UL, 0x6929a9a9UL, 0x6ac4f1f1UL, 0x6b9f1e1eUL, 0x6c774141UL, 0x6d2caeaeUL, 0x6ec1f6f6UL, 0x6f9a1919UL, +0x706c5454UL, 0x7137bbbbUL, 0x72dae3e3UL, 0x73810c0cUL, 0x74695353UL, 0x7532bcbcUL, 0x76dfe4e4UL, 0x77840b0bUL, +0x78665a5aUL, 0x793db5b5UL, 0x7ad0ededUL, 0x7b8b0202UL, 0x7c635d5dUL, 0x7d38b2b2UL, 0x7ed5eaeaUL, 0x7f8e0505UL, +0x80a0e0e0UL, 0x81fb0f0fUL, 0x82165757UL, 0x834db8b8UL, 0x84a5e7e7UL, 0x85fe0808UL, 0x86135050UL, 0x8748bfbfUL, +0x88aaeeeeUL, 0x89f10101UL, 0x8a1c5959UL, 0x8b47b6b6UL, 0x8cafe9e9UL, 0x8df40606UL, 0x8e195e5eUL, 0x8f42b1b1UL, +0x90b4fcfcUL, 0x91ef1313UL, 0x92024b4bUL, 0x9359a4a4UL, 0x94b1fbfbUL, 0x95ea1414UL, 0x96074c4cUL, 0x975ca3a3UL, +0x98bef2f2UL, 0x99e51d1dUL, 0x9a084545UL, 0x9b53aaaaUL, 0x9cbbf5f5UL, 0x9de01a1aUL, 0x9e0d4242UL, 0x9f56adadUL, +0xa088d8d8UL, 0xa1d33737UL, 0xa23e6f6fUL, 0xa3658080UL, 0xa48ddfdfUL, 0xa5d63030UL, 0xa63b6868UL, 0xa7608787UL, +0xa882d6d6UL, 0xa9d93939UL, 0xaa346161UL, 0xab6f8e8eUL, 0xac87d1d1UL, 0xaddc3e3eUL, 0xae316666UL, 0xaf6a8989UL, +0xb09cc4c4UL, 0xb1c72b2bUL, 0xb22a7373UL, 0xb3719c9cUL, 0xb499c3c3UL, 0xb5c22c2cUL, 0xb62f7474UL, 0xb7749b9bUL, +0xb896cacaUL, 0xb9cd2525UL, 0xba207d7dUL, 0xbb7b9292UL, 0xbc93cdcdUL, 0xbdc82222UL, 0xbe257a7aUL, 0xbf7e9595UL, +0xc0f09090UL, 0xc1ab7f7fUL, 0xc2462727UL, 0xc31dc8c8UL, 0xc4f59797UL, 0xc5ae7878UL, 0xc6432020UL, 0xc718cfcfUL, +0xc8fa9e9eUL, 0xc9a17171UL, 0xca4c2929UL, 0xcb17c6c6UL, 0xccff9999UL, 0xcda47676UL, 0xce492e2eUL, 0xcf12c1c1UL, +0xd0e48c8cUL, 0xd1bf6363UL, 0xd2523b3bUL, 0xd309d4d4UL, 0xd4e18b8bUL, 0xd5ba6464UL, 0xd6573c3cUL, 0xd70cd3d3UL, +0xd8ee8282UL, 0xd9b56d6dUL, 0xda583535UL, 0xdb03dadaUL, 0xdceb8585UL, 0xddb06a6aUL, 0xde5d3232UL, 0xdf06ddddUL, +0xe0d8a8a8UL, 0xe1834747UL, 0xe26e1f1fUL, 0xe335f0f0UL, 0xe4ddafafUL, 0xe5864040UL, 0xe66b1818UL, 0xe730f7f7UL, +0xe8d2a6a6UL, 0xe9894949UL, 0xea641111UL, 0xeb3ffefeUL, 0xecd7a1a1UL, 0xed8c4e4eUL, 0xee611616UL, 0xef3af9f9UL, +0xf0ccb4b4UL, 0xf1975b5bUL, 0xf27a0303UL, 0xf321ececUL, 0xf4c9b3b3UL, 0xf5925c5cUL, 0xf67f0404UL, 0xf724ebebUL, +0xf8c6babaUL, 0xf99d5555UL, 0xfa700d0dUL, 0xfb2be2e2UL, 0xfcc3bdbdUL, 0xfd985252UL, 0xfe750a0aUL, 0xff2ee5e5UL +}, +{ +0x00000000UL, 0xef01ef5bUL, 0xb702b7b6UL, 0x580358edUL, 0x07040705UL, 0xe805e85eUL, 0xb006b0b3UL, 0x5f075fe8UL, +0x0e080e0aUL, 0xe109e151UL, 0xb90ab9bcUL, 0x560b56e7UL, 0x090c090fUL, 0xe60de654UL, 0xbe0ebeb9UL, 0x510f51e2UL, +0x1c101c14UL, 0xf311f34fUL, 0xab12aba2UL, 0x441344f9UL, 0x1b141b11UL, 0xf415f44aUL, 0xac16aca7UL, 0x431743fcUL, +0x1218121eUL, 0xfd19fd45UL, 0xa51aa5a8UL, 0x4a1b4af3UL, 0x151c151bUL, 0xfa1dfa40UL, 0xa21ea2adUL, 0x4d1f4df6UL, +0x38203828UL, 0xd721d773UL, 0x8f228f9eUL, 0x602360c5UL, 0x3f243f2dUL, 0xd025d076UL, 0x8826889bUL, 0x672767c0UL, +0x36283622UL, 0xd929d979UL, 0x812a8194UL, 0x6e2b6ecfUL, 0x312c3127UL, 0xde2dde7cUL, 0x862e8691UL, 0x692f69caUL, +0x2430243cUL, 0xcb31cb67UL, 0x9332938aUL, 0x7c337cd1UL, 0x23342339UL, 0xcc35cc62UL, 0x9436948fUL, 0x7b377bd4UL, +0x2a382a36UL, 0xc539c56dUL, 0x9d3a9d80UL, 0x723b72dbUL, 0x2d3c2d33UL, 0xc23dc268UL, 0x9a3e9a85UL, 0x753f75deUL, +0x70407050UL, 0x9f419f0bUL, 0xc742c7e6UL, 0x284328bdUL, 0x77447755UL, 0x9845980eUL, 0xc046c0e3UL, 0x2f472fb8UL, +0x7e487e5aUL, 0x91499101UL, 0xc94ac9ecUL, 0x264b26b7UL, 0x794c795fUL, 0x964d9604UL, 0xce4ecee9UL, 0x214f21b2UL, +0x6c506c44UL, 0x8351831fUL, 0xdb52dbf2UL, 0x345334a9UL, 0x6b546b41UL, 0x8455841aUL, 0xdc56dcf7UL, 0x335733acUL, +0x6258624eUL, 0x8d598d15UL, 0xd55ad5f8UL, 0x3a5b3aa3UL, 0x655c654bUL, 0x8a5d8a10UL, 0xd25ed2fdUL, 0x3d5f3da6UL, +0x48604878UL, 0xa761a723UL, 0xff62ffceUL, 0x10631095UL, 0x4f644f7dUL, 0xa065a026UL, 0xf866f8cbUL, 0x17671790UL, +0x46684672UL, 0xa969a929UL, 0xf16af1c4UL, 0x1e6b1e9fUL, 0x416c4177UL, 0xae6dae2cUL, 0xf66ef6c1UL, 0x196f199aUL, +0x5470546cUL, 0xbb71bb37UL, 0xe372e3daUL, 0x0c730c81UL, 0x53745369UL, 0xbc75bc32UL, 0xe476e4dfUL, 0x0b770b84UL, +0x5a785a66UL, 0xb579b53dUL, 0xed7aedd0UL, 0x027b028bUL, 0x5d7c5d63UL, 0xb27db238UL, 0xea7eead5UL, 0x057f058eUL, +0xe080e0a0UL, 0x0f810ffbUL, 0x57825716UL, 0xb883b84dUL, 0xe784e7a5UL, 0x088508feUL, 0x50865013UL, 0xbf87bf48UL, +0xee88eeaaUL, 0x018901f1UL, 0x598a591cUL, 0xb68bb647UL, 0xe98ce9afUL, 0x068d06f4UL, 0x5e8e5e19UL, 0xb18fb142UL, +0xfc90fcb4UL, 0x139113efUL, 0x4b924b02UL, 0xa493a459UL, 0xfb94fbb1UL, 0x149514eaUL, 0x4c964c07UL, 0xa397a35cUL, +0xf298f2beUL, 0x1d991de5UL, 0x459a4508UL, 0xaa9baa53UL, 0xf59cf5bbUL, 0x1a9d1ae0UL, 0x429e420dUL, 0xad9fad56UL, +0xd8a0d888UL, 0x37a137d3UL, 0x6fa26f3eUL, 0x80a38065UL, 0xdfa4df8dUL, 0x30a530d6UL, 0x68a6683bUL, 0x87a78760UL, +0xd6a8d682UL, 0x39a939d9UL, 0x61aa6134UL, 0x8eab8e6fUL, 0xd1acd187UL, 0x3ead3edcUL, 0x66ae6631UL, 0x89af896aUL, +0xc4b0c49cUL, 0x2bb12bc7UL, 0x73b2732aUL, 0x9cb39c71UL, 0xc3b4c399UL, 0x2cb52cc2UL, 0x74b6742fUL, 0x9bb79b74UL, +0xcab8ca96UL, 0x25b925cdUL, 0x7dba7d20UL, 0x92bb927bUL, 0xcdbccd93UL, 0x22bd22c8UL, 0x7abe7a25UL, 0x95bf957eUL, +0x90c090f0UL, 0x7fc17fabUL, 0x27c22746UL, 0xc8c3c81dUL, 0x97c497f5UL, 0x78c578aeUL, 0x20c62043UL, 0xcfc7cf18UL, +0x9ec89efaUL, 0x71c971a1UL, 0x29ca294cUL, 0xc6cbc617UL, 0x99cc99ffUL, 0x76cd76a4UL, 0x2ece2e49UL, 0xc1cfc112UL, +0x8cd08ce4UL, 0x63d163bfUL, 0x3bd23b52UL, 0xd4d3d409UL, 0x8bd48be1UL, 0x64d564baUL, 0x3cd63c57UL, 0xd3d7d30cUL, +0x82d882eeUL, 0x6dd96db5UL, 0x35da3558UL, 0xdadbda03UL, 0x85dc85ebUL, 0x6add6ab0UL, 0x32de325dUL, 0xdddfdd06UL, +0xa8e0a8d8UL, 0x47e14783UL, 0x1fe21f6eUL, 0xf0e3f035UL, 0xafe4afddUL, 0x40e54086UL, 0x18e6186bUL, 0xf7e7f730UL, +0xa6e8a6d2UL, 0x49e94989UL, 0x11ea1164UL, 0xfeebfe3fUL, 0xa1eca1d7UL, 0x4eed4e8cUL, 0x16ee1661UL, 0xf9eff93aUL, +0xb4f0b4ccUL, 0x5bf15b97UL, 0x03f2037aUL, 0xecf3ec21UL, 0xb3f4b3c9UL, 0x5cf55c92UL, 0x04f6047fUL, 0xebf7eb24UL, +0xbaf8bac6UL, 0x55f9559dUL, 0x0dfa0d70UL, 0xe2fbe22bUL, 0xbdfcbdc3UL, 0x52fd5298UL, 0x0afe0a75UL, 0xe5ffe52eUL +}, +{ +0x00000000UL, 0x5bef015bUL, 0xb6b702b6UL, 0xed5803edUL, 0x05070405UL, 0x5ee8055eUL, 0xb3b006b3UL, 0xe85f07e8UL, +0x0a0e080aUL, 0x51e10951UL, 0xbcb90abcUL, 0xe7560be7UL, 0x0f090c0fUL, 0x54e60d54UL, 0xb9be0eb9UL, 0xe2510fe2UL, +0x141c1014UL, 0x4ff3114fUL, 0xa2ab12a2UL, 0xf94413f9UL, 0x111b1411UL, 0x4af4154aUL, 0xa7ac16a7UL, 0xfc4317fcUL, +0x1e12181eUL, 0x45fd1945UL, 0xa8a51aa8UL, 0xf34a1bf3UL, 0x1b151c1bUL, 0x40fa1d40UL, 0xada21eadUL, 0xf64d1ff6UL, +0x28382028UL, 0x73d72173UL, 0x9e8f229eUL, 0xc56023c5UL, 0x2d3f242dUL, 0x76d02576UL, 0x9b88269bUL, 0xc06727c0UL, +0x22362822UL, 0x79d92979UL, 0x94812a94UL, 0xcf6e2bcfUL, 0x27312c27UL, 0x7cde2d7cUL, 0x91862e91UL, 0xca692fcaUL, +0x3c24303cUL, 0x67cb3167UL, 0x8a93328aUL, 0xd17c33d1UL, 0x39233439UL, 0x62cc3562UL, 0x8f94368fUL, 0xd47b37d4UL, +0x362a3836UL, 0x6dc5396dUL, 0x809d3a80UL, 0xdb723bdbUL, 0x332d3c33UL, 0x68c23d68UL, 0x859a3e85UL, 0xde753fdeUL, +0x50704050UL, 0x0b9f410bUL, 0xe6c742e6UL, 0xbd2843bdUL, 0x55774455UL, 0x0e98450eUL, 0xe3c046e3UL, 0xb82f47b8UL, +0x5a7e485aUL, 0x01914901UL, 0xecc94aecUL, 0xb7264bb7UL, 0x5f794c5fUL, 0x04964d04UL, 0xe9ce4ee9UL, 0xb2214fb2UL, +0x446c5044UL, 0x1f83511fUL, 0xf2db52f2UL, 0xa93453a9UL, 0x416b5441UL, 0x1a84551aUL, 0xf7dc56f7UL, 0xac3357acUL, +0x4e62584eUL, 0x158d5915UL, 0xf8d55af8UL, 0xa33a5ba3UL, 0x4b655c4bUL, 0x108a5d10UL, 0xfdd25efdUL, 0xa63d5fa6UL, +0x78486078UL, 0x23a76123UL, 0xceff62ceUL, 0x95106395UL, 0x7d4f647dUL, 0x26a06526UL, 0xcbf866cbUL, 0x90176790UL, +0x72466872UL, 0x29a96929UL, 0xc4f16ac4UL, 0x9f1e6b9fUL, 0x77416c77UL, 0x2cae6d2cUL, 0xc1f66ec1UL, 0x9a196f9aUL, +0x6c54706cUL, 0x37bb7137UL, 0xdae372daUL, 0x810c7381UL, 0x69537469UL, 0x32bc7532UL, 0xdfe476dfUL, 0x840b7784UL, +0x665a7866UL, 0x3db5793dUL, 0xd0ed7ad0UL, 0x8b027b8bUL, 0x635d7c63UL, 0x38b27d38UL, 0xd5ea7ed5UL, 0x8e057f8eUL, +0xa0e080a0UL, 0xfb0f81fbUL, 0x16578216UL, 0x4db8834dUL, 0xa5e784a5UL, 0xfe0885feUL, 0x13508613UL, 0x48bf8748UL, +0xaaee88aaUL, 0xf10189f1UL, 0x1c598a1cUL, 0x47b68b47UL, 0xafe98cafUL, 0xf4068df4UL, 0x195e8e19UL, 0x42b18f42UL, +0xb4fc90b4UL, 0xef1391efUL, 0x024b9202UL, 0x59a49359UL, 0xb1fb94b1UL, 0xea1495eaUL, 0x074c9607UL, 0x5ca3975cUL, +0xbef298beUL, 0xe51d99e5UL, 0x08459a08UL, 0x53aa9b53UL, 0xbbf59cbbUL, 0xe01a9de0UL, 0x0d429e0dUL, 0x56ad9f56UL, +0x88d8a088UL, 0xd337a1d3UL, 0x3e6fa23eUL, 0x6580a365UL, 0x8ddfa48dUL, 0xd630a5d6UL, 0x3b68a63bUL, 0x6087a760UL, +0x82d6a882UL, 0xd939a9d9UL, 0x3461aa34UL, 0x6f8eab6fUL, 0x87d1ac87UL, 0xdc3eaddcUL, 0x3166ae31UL, 0x6a89af6aUL, +0x9cc4b09cUL, 0xc72bb1c7UL, 0x2a73b22aUL, 0x719cb371UL, 0x99c3b499UL, 0xc22cb5c2UL, 0x2f74b62fUL, 0x749bb774UL, +0x96cab896UL, 0xcd25b9cdUL, 0x207dba20UL, 0x7b92bb7bUL, 0x93cdbc93UL, 0xc822bdc8UL, 0x257abe25UL, 0x7e95bf7eUL, +0xf090c0f0UL, 0xab7fc1abUL, 0x4627c246UL, 0x1dc8c31dUL, 0xf597c4f5UL, 0xae78c5aeUL, 0x4320c643UL, 0x18cfc718UL, +0xfa9ec8faUL, 0xa171c9a1UL, 0x4c29ca4cUL, 0x17c6cb17UL, 0xff99ccffUL, 0xa476cda4UL, 0x492ece49UL, 0x12c1cf12UL, +0xe48cd0e4UL, 0xbf63d1bfUL, 0x523bd252UL, 0x09d4d309UL, 0xe18bd4e1UL, 0xba64d5baUL, 0x573cd657UL, 0x0cd3d70cUL, +0xee82d8eeUL, 0xb56dd9b5UL, 0x5835da58UL, 0x03dadb03UL, 0xeb85dcebUL, 0xb06addb0UL, 0x5d32de5dUL, 0x06dddf06UL, +0xd8a8e0d8UL, 0x8347e183UL, 0x6e1fe26eUL, 0x35f0e335UL, 0xddafe4ddUL, 0x8640e586UL, 0x6b18e66bUL, 0x30f7e730UL, +0xd2a6e8d2UL, 0x8949e989UL, 0x6411ea64UL, 0x3ffeeb3fUL, 0xd7a1ecd7UL, 0x8c4eed8cUL, 0x6116ee61UL, 0x3af9ef3aUL, +0xccb4f0ccUL, 0x975bf197UL, 0x7a03f27aUL, 0x21ecf321UL, 0xc9b3f4c9UL, 0x925cf592UL, 0x7f04f67fUL, 0x24ebf724UL, +0xc6baf8c6UL, 0x9d55f99dUL, 0x700dfa70UL, 0x2be2fb2bUL, 0xc3bdfcc3UL, 0x9852fd98UL, 0x750afe75UL, 0x2ee5ff2eUL +}}; + +#ifdef LTC_TWOFISH_ALL_TABLES + +/* the 4x8 RS transform */ +static const ulong32 rs_tab0[256] = { +0x00000000LU, 0xa402a401LU, 0x05040502LU, 0xa106a103LU, 0x0a080a04LU, 0xae0aae05LU, 0x0f0c0f06LU, 0xab0eab07LU, +0x14101408LU, 0xb012b009LU, 0x1114110aLU, 0xb516b50bLU, 0x1e181e0cLU, 0xba1aba0dLU, 0x1b1c1b0eLU, 0xbf1ebf0fLU, +0x28202810LU, 0x8c228c11LU, 0x2d242d12LU, 0x89268913LU, 0x22282214LU, 0x862a8615LU, 0x272c2716LU, 0x832e8317LU, +0x3c303c18LU, 0x98329819LU, 0x3934391aLU, 0x9d369d1bLU, 0x3638361cLU, 0x923a921dLU, 0x333c331eLU, 0x973e971fLU, +0x50405020LU, 0xf442f421LU, 0x55445522LU, 0xf146f123LU, 0x5a485a24LU, 0xfe4afe25LU, 0x5f4c5f26LU, 0xfb4efb27LU, +0x44504428LU, 0xe052e029LU, 0x4154412aLU, 0xe556e52bLU, 0x4e584e2cLU, 0xea5aea2dLU, 0x4b5c4b2eLU, 0xef5eef2fLU, +0x78607830LU, 0xdc62dc31LU, 0x7d647d32LU, 0xd966d933LU, 0x72687234LU, 0xd66ad635LU, 0x776c7736LU, 0xd36ed337LU, +0x6c706c38LU, 0xc872c839LU, 0x6974693aLU, 0xcd76cd3bLU, 0x6678663cLU, 0xc27ac23dLU, 0x637c633eLU, 0xc77ec73fLU, +0xa080a040LU, 0x04820441LU, 0xa584a542LU, 0x01860143LU, 0xaa88aa44LU, 0x0e8a0e45LU, 0xaf8caf46LU, 0x0b8e0b47LU, +0xb490b448LU, 0x10921049LU, 0xb194b14aLU, 0x1596154bLU, 0xbe98be4cLU, 0x1a9a1a4dLU, 0xbb9cbb4eLU, 0x1f9e1f4fLU, +0x88a08850LU, 0x2ca22c51LU, 0x8da48d52LU, 0x29a62953LU, 0x82a88254LU, 0x26aa2655LU, 0x87ac8756LU, 0x23ae2357LU, +0x9cb09c58LU, 0x38b23859LU, 0x99b4995aLU, 0x3db63d5bLU, 0x96b8965cLU, 0x32ba325dLU, 0x93bc935eLU, 0x37be375fLU, +0xf0c0f060LU, 0x54c25461LU, 0xf5c4f562LU, 0x51c65163LU, 0xfac8fa64LU, 0x5eca5e65LU, 0xffccff66LU, 0x5bce5b67LU, +0xe4d0e468LU, 0x40d24069LU, 0xe1d4e16aLU, 0x45d6456bLU, 0xeed8ee6cLU, 0x4ada4a6dLU, 0xebdceb6eLU, 0x4fde4f6fLU, +0xd8e0d870LU, 0x7ce27c71LU, 0xdde4dd72LU, 0x79e67973LU, 0xd2e8d274LU, 0x76ea7675LU, 0xd7ecd776LU, 0x73ee7377LU, +0xccf0cc78LU, 0x68f26879LU, 0xc9f4c97aLU, 0x6df66d7bLU, 0xc6f8c67cLU, 0x62fa627dLU, 0xc3fcc37eLU, 0x67fe677fLU, +0x0d4d0d80LU, 0xa94fa981LU, 0x08490882LU, 0xac4bac83LU, 0x07450784LU, 0xa347a385LU, 0x02410286LU, 0xa643a687LU, +0x195d1988LU, 0xbd5fbd89LU, 0x1c591c8aLU, 0xb85bb88bLU, 0x1355138cLU, 0xb757b78dLU, 0x1651168eLU, 0xb253b28fLU, +0x256d2590LU, 0x816f8191LU, 0x20692092LU, 0x846b8493LU, 0x2f652f94LU, 0x8b678b95LU, 0x2a612a96LU, 0x8e638e97LU, +0x317d3198LU, 0x957f9599LU, 0x3479349aLU, 0x907b909bLU, 0x3b753b9cLU, 0x9f779f9dLU, 0x3e713e9eLU, 0x9a739a9fLU, +0x5d0d5da0LU, 0xf90ff9a1LU, 0x580958a2LU, 0xfc0bfca3LU, 0x570557a4LU, 0xf307f3a5LU, 0x520152a6LU, 0xf603f6a7LU, +0x491d49a8LU, 0xed1feda9LU, 0x4c194caaLU, 0xe81be8abLU, 0x431543acLU, 0xe717e7adLU, 0x461146aeLU, 0xe213e2afLU, +0x752d75b0LU, 0xd12fd1b1LU, 0x702970b2LU, 0xd42bd4b3LU, 0x7f257fb4LU, 0xdb27dbb5LU, 0x7a217ab6LU, 0xde23deb7LU, +0x613d61b8LU, 0xc53fc5b9LU, 0x643964baLU, 0xc03bc0bbLU, 0x6b356bbcLU, 0xcf37cfbdLU, 0x6e316ebeLU, 0xca33cabfLU, +0xadcdadc0LU, 0x09cf09c1LU, 0xa8c9a8c2LU, 0x0ccb0cc3LU, 0xa7c5a7c4LU, 0x03c703c5LU, 0xa2c1a2c6LU, 0x06c306c7LU, +0xb9ddb9c8LU, 0x1ddf1dc9LU, 0xbcd9bccaLU, 0x18db18cbLU, 0xb3d5b3ccLU, 0x17d717cdLU, 0xb6d1b6ceLU, 0x12d312cfLU, +0x85ed85d0LU, 0x21ef21d1LU, 0x80e980d2LU, 0x24eb24d3LU, 0x8fe58fd4LU, 0x2be72bd5LU, 0x8ae18ad6LU, 0x2ee32ed7LU, +0x91fd91d8LU, 0x35ff35d9LU, 0x94f994daLU, 0x30fb30dbLU, 0x9bf59bdcLU, 0x3ff73fddLU, 0x9ef19edeLU, 0x3af33adfLU, +0xfd8dfde0LU, 0x598f59e1LU, 0xf889f8e2LU, 0x5c8b5ce3LU, 0xf785f7e4LU, 0x538753e5LU, 0xf281f2e6LU, 0x568356e7LU, +0xe99de9e8LU, 0x4d9f4de9LU, 0xec99eceaLU, 0x489b48ebLU, 0xe395e3ecLU, 0x479747edLU, 0xe691e6eeLU, 0x429342efLU, +0xd5add5f0LU, 0x71af71f1LU, 0xd0a9d0f2LU, 0x74ab74f3LU, 0xdfa5dff4LU, 0x7ba77bf5LU, 0xdaa1daf6LU, 0x7ea37ef7LU, +0xc1bdc1f8LU, 0x65bf65f9LU, 0xc4b9c4faLU, 0x60bb60fbLU, 0xcbb5cbfcLU, 0x6fb76ffdLU, 0xceb1cefeLU, 0x6ab36affLU }; + +static const ulong32 rs_tab1[256] = { +0x00000000LU, 0x55a156a4LU, 0xaa0fac05LU, 0xffaefaa1LU, 0x191e150aLU, 0x4cbf43aeLU, 0xb311b90fLU, 0xe6b0efabLU, +0x323c2a14LU, 0x679d7cb0LU, 0x98338611LU, 0xcd92d0b5LU, 0x2b223f1eLU, 0x7e8369baLU, 0x812d931bLU, 0xd48cc5bfLU, +0x64785428LU, 0x31d9028cLU, 0xce77f82dLU, 0x9bd6ae89LU, 0x7d664122LU, 0x28c71786LU, 0xd769ed27LU, 0x82c8bb83LU, +0x56447e3cLU, 0x03e52898LU, 0xfc4bd239LU, 0xa9ea849dLU, 0x4f5a6b36LU, 0x1afb3d92LU, 0xe555c733LU, 0xb0f49197LU, +0xc8f0a850LU, 0x9d51fef4LU, 0x62ff0455LU, 0x375e52f1LU, 0xd1eebd5aLU, 0x844febfeLU, 0x7be1115fLU, 0x2e4047fbLU, +0xfacc8244LU, 0xaf6dd4e0LU, 0x50c32e41LU, 0x056278e5LU, 0xe3d2974eLU, 0xb673c1eaLU, 0x49dd3b4bLU, 0x1c7c6defLU, +0xac88fc78LU, 0xf929aadcLU, 0x0687507dLU, 0x532606d9LU, 0xb596e972LU, 0xe037bfd6LU, 0x1f994577LU, 0x4a3813d3LU, +0x9eb4d66cLU, 0xcb1580c8LU, 0x34bb7a69LU, 0x611a2ccdLU, 0x87aac366LU, 0xd20b95c2LU, 0x2da56f63LU, 0x780439c7LU, +0xddad1da0LU, 0x880c4b04LU, 0x77a2b1a5LU, 0x2203e701LU, 0xc4b308aaLU, 0x91125e0eLU, 0x6ebca4afLU, 0x3b1df20bLU, +0xef9137b4LU, 0xba306110LU, 0x459e9bb1LU, 0x103fcd15LU, 0xf68f22beLU, 0xa32e741aLU, 0x5c808ebbLU, 0x0921d81fLU, +0xb9d54988LU, 0xec741f2cLU, 0x13dae58dLU, 0x467bb329LU, 0xa0cb5c82LU, 0xf56a0a26LU, 0x0ac4f087LU, 0x5f65a623LU, +0x8be9639cLU, 0xde483538LU, 0x21e6cf99LU, 0x7447993dLU, 0x92f77696LU, 0xc7562032LU, 0x38f8da93LU, 0x6d598c37LU, +0x155db5f0LU, 0x40fce354LU, 0xbf5219f5LU, 0xeaf34f51LU, 0x0c43a0faLU, 0x59e2f65eLU, 0xa64c0cffLU, 0xf3ed5a5bLU, +0x27619fe4LU, 0x72c0c940LU, 0x8d6e33e1LU, 0xd8cf6545LU, 0x3e7f8aeeLU, 0x6bdedc4aLU, 0x947026ebLU, 0xc1d1704fLU, +0x7125e1d8LU, 0x2484b77cLU, 0xdb2a4dddLU, 0x8e8b1b79LU, 0x683bf4d2LU, 0x3d9aa276LU, 0xc23458d7LU, 0x97950e73LU, +0x4319cbccLU, 0x16b89d68LU, 0xe91667c9LU, 0xbcb7316dLU, 0x5a07dec6LU, 0x0fa68862LU, 0xf00872c3LU, 0xa5a92467LU, +0xf7173a0dLU, 0xa2b66ca9LU, 0x5d189608LU, 0x08b9c0acLU, 0xee092f07LU, 0xbba879a3LU, 0x44068302LU, 0x11a7d5a6LU, +0xc52b1019LU, 0x908a46bdLU, 0x6f24bc1cLU, 0x3a85eab8LU, 0xdc350513LU, 0x899453b7LU, 0x763aa916LU, 0x239bffb2LU, +0x936f6e25LU, 0xc6ce3881LU, 0x3960c220LU, 0x6cc19484LU, 0x8a717b2fLU, 0xdfd02d8bLU, 0x207ed72aLU, 0x75df818eLU, +0xa1534431LU, 0xf4f21295LU, 0x0b5ce834LU, 0x5efdbe90LU, 0xb84d513bLU, 0xedec079fLU, 0x1242fd3eLU, 0x47e3ab9aLU, +0x3fe7925dLU, 0x6a46c4f9LU, 0x95e83e58LU, 0xc04968fcLU, 0x26f98757LU, 0x7358d1f3LU, 0x8cf62b52LU, 0xd9577df6LU, +0x0ddbb849LU, 0x587aeeedLU, 0xa7d4144cLU, 0xf27542e8LU, 0x14c5ad43LU, 0x4164fbe7LU, 0xbeca0146LU, 0xeb6b57e2LU, +0x5b9fc675LU, 0x0e3e90d1LU, 0xf1906a70LU, 0xa4313cd4LU, 0x4281d37fLU, 0x172085dbLU, 0xe88e7f7aLU, 0xbd2f29deLU, +0x69a3ec61LU, 0x3c02bac5LU, 0xc3ac4064LU, 0x960d16c0LU, 0x70bdf96bLU, 0x251cafcfLU, 0xdab2556eLU, 0x8f1303caLU, +0x2aba27adLU, 0x7f1b7109LU, 0x80b58ba8LU, 0xd514dd0cLU, 0x33a432a7LU, 0x66056403LU, 0x99ab9ea2LU, 0xcc0ac806LU, +0x18860db9LU, 0x4d275b1dLU, 0xb289a1bcLU, 0xe728f718LU, 0x019818b3LU, 0x54394e17LU, 0xab97b4b6LU, 0xfe36e212LU, +0x4ec27385LU, 0x1b632521LU, 0xe4cddf80LU, 0xb16c8924LU, 0x57dc668fLU, 0x027d302bLU, 0xfdd3ca8aLU, 0xa8729c2eLU, +0x7cfe5991LU, 0x295f0f35LU, 0xd6f1f594LU, 0x8350a330LU, 0x65e04c9bLU, 0x30411a3fLU, 0xcfefe09eLU, 0x9a4eb63aLU, +0xe24a8ffdLU, 0xb7ebd959LU, 0x484523f8LU, 0x1de4755cLU, 0xfb549af7LU, 0xaef5cc53LU, 0x515b36f2LU, 0x04fa6056LU, +0xd076a5e9LU, 0x85d7f34dLU, 0x7a7909ecLU, 0x2fd85f48LU, 0xc968b0e3LU, 0x9cc9e647LU, 0x63671ce6LU, 0x36c64a42LU, +0x8632dbd5LU, 0xd3938d71LU, 0x2c3d77d0LU, 0x799c2174LU, 0x9f2ccedfLU, 0xca8d987bLU, 0x352362daLU, 0x6082347eLU, +0xb40ef1c1LU, 0xe1afa765LU, 0x1e015dc4LU, 0x4ba00b60LU, 0xad10e4cbLU, 0xf8b1b26fLU, 0x071f48ceLU, 0x52be1e6aLU }; + +static const ulong32 rs_tab2[256] = { +0x00000000LU, 0x87fc8255LU, 0x43b549aaLU, 0xc449cbffLU, 0x86279219LU, 0x01db104cLU, 0xc592dbb3LU, 0x426e59e6LU, +0x414e6932LU, 0xc6b2eb67LU, 0x02fb2098LU, 0x8507a2cdLU, 0xc769fb2bLU, 0x4095797eLU, 0x84dcb281LU, 0x032030d4LU, +0x829cd264LU, 0x05605031LU, 0xc1299bceLU, 0x46d5199bLU, 0x04bb407dLU, 0x8347c228LU, 0x470e09d7LU, 0xc0f28b82LU, +0xc3d2bb56LU, 0x442e3903LU, 0x8067f2fcLU, 0x079b70a9LU, 0x45f5294fLU, 0xc209ab1aLU, 0x064060e5LU, 0x81bce2b0LU, +0x4975e9c8LU, 0xce896b9dLU, 0x0ac0a062LU, 0x8d3c2237LU, 0xcf527bd1LU, 0x48aef984LU, 0x8ce7327bLU, 0x0b1bb02eLU, +0x083b80faLU, 0x8fc702afLU, 0x4b8ec950LU, 0xcc724b05LU, 0x8e1c12e3LU, 0x09e090b6LU, 0xcda95b49LU, 0x4a55d91cLU, +0xcbe93bacLU, 0x4c15b9f9LU, 0x885c7206LU, 0x0fa0f053LU, 0x4dcea9b5LU, 0xca322be0LU, 0x0e7be01fLU, 0x8987624aLU, +0x8aa7529eLU, 0x0d5bd0cbLU, 0xc9121b34LU, 0x4eee9961LU, 0x0c80c087LU, 0x8b7c42d2LU, 0x4f35892dLU, 0xc8c90b78LU, +0x92ea9fddLU, 0x15161d88LU, 0xd15fd677LU, 0x56a35422LU, 0x14cd0dc4LU, 0x93318f91LU, 0x5778446eLU, 0xd084c63bLU, +0xd3a4f6efLU, 0x545874baLU, 0x9011bf45LU, 0x17ed3d10LU, 0x558364f6LU, 0xd27fe6a3LU, 0x16362d5cLU, 0x91caaf09LU, +0x10764db9LU, 0x978acfecLU, 0x53c30413LU, 0xd43f8646LU, 0x9651dfa0LU, 0x11ad5df5LU, 0xd5e4960aLU, 0x5218145fLU, +0x5138248bLU, 0xd6c4a6deLU, 0x128d6d21LU, 0x9571ef74LU, 0xd71fb692LU, 0x50e334c7LU, 0x94aaff38LU, 0x13567d6dLU, +0xdb9f7615LU, 0x5c63f440LU, 0x982a3fbfLU, 0x1fd6bdeaLU, 0x5db8e40cLU, 0xda446659LU, 0x1e0dada6LU, 0x99f12ff3LU, +0x9ad11f27LU, 0x1d2d9d72LU, 0xd964568dLU, 0x5e98d4d8LU, 0x1cf68d3eLU, 0x9b0a0f6bLU, 0x5f43c494LU, 0xd8bf46c1LU, +0x5903a471LU, 0xdeff2624LU, 0x1ab6eddbLU, 0x9d4a6f8eLU, 0xdf243668LU, 0x58d8b43dLU, 0x9c917fc2LU, 0x1b6dfd97LU, +0x184dcd43LU, 0x9fb14f16LU, 0x5bf884e9LU, 0xdc0406bcLU, 0x9e6a5f5aLU, 0x1996dd0fLU, 0xdddf16f0LU, 0x5a2394a5LU, +0x699973f7LU, 0xee65f1a2LU, 0x2a2c3a5dLU, 0xadd0b808LU, 0xefbee1eeLU, 0x684263bbLU, 0xac0ba844LU, 0x2bf72a11LU, +0x28d71ac5LU, 0xaf2b9890LU, 0x6b62536fLU, 0xec9ed13aLU, 0xaef088dcLU, 0x290c0a89LU, 0xed45c176LU, 0x6ab94323LU, +0xeb05a193LU, 0x6cf923c6LU, 0xa8b0e839LU, 0x2f4c6a6cLU, 0x6d22338aLU, 0xeadeb1dfLU, 0x2e977a20LU, 0xa96bf875LU, +0xaa4bc8a1LU, 0x2db74af4LU, 0xe9fe810bLU, 0x6e02035eLU, 0x2c6c5ab8LU, 0xab90d8edLU, 0x6fd91312LU, 0xe8259147LU, +0x20ec9a3fLU, 0xa710186aLU, 0x6359d395LU, 0xe4a551c0LU, 0xa6cb0826LU, 0x21378a73LU, 0xe57e418cLU, 0x6282c3d9LU, +0x61a2f30dLU, 0xe65e7158LU, 0x2217baa7LU, 0xa5eb38f2LU, 0xe7856114LU, 0x6079e341LU, 0xa43028beLU, 0x23ccaaebLU, +0xa270485bLU, 0x258cca0eLU, 0xe1c501f1LU, 0x663983a4LU, 0x2457da42LU, 0xa3ab5817LU, 0x67e293e8LU, 0xe01e11bdLU, +0xe33e2169LU, 0x64c2a33cLU, 0xa08b68c3LU, 0x2777ea96LU, 0x6519b370LU, 0xe2e53125LU, 0x26acfadaLU, 0xa150788fLU, +0xfb73ec2aLU, 0x7c8f6e7fLU, 0xb8c6a580LU, 0x3f3a27d5LU, 0x7d547e33LU, 0xfaa8fc66LU, 0x3ee13799LU, 0xb91db5ccLU, +0xba3d8518LU, 0x3dc1074dLU, 0xf988ccb2LU, 0x7e744ee7LU, 0x3c1a1701LU, 0xbbe69554LU, 0x7faf5eabLU, 0xf853dcfeLU, +0x79ef3e4eLU, 0xfe13bc1bLU, 0x3a5a77e4LU, 0xbda6f5b1LU, 0xffc8ac57LU, 0x78342e02LU, 0xbc7de5fdLU, 0x3b8167a8LU, +0x38a1577cLU, 0xbf5dd529LU, 0x7b141ed6LU, 0xfce89c83LU, 0xbe86c565LU, 0x397a4730LU, 0xfd338ccfLU, 0x7acf0e9aLU, +0xb20605e2LU, 0x35fa87b7LU, 0xf1b34c48LU, 0x764fce1dLU, 0x342197fbLU, 0xb3dd15aeLU, 0x7794de51LU, 0xf0685c04LU, +0xf3486cd0LU, 0x74b4ee85LU, 0xb0fd257aLU, 0x3701a72fLU, 0x756ffec9LU, 0xf2937c9cLU, 0x36dab763LU, 0xb1263536LU, +0x309ad786LU, 0xb76655d3LU, 0x732f9e2cLU, 0xf4d31c79LU, 0xb6bd459fLU, 0x3141c7caLU, 0xf5080c35LU, 0x72f48e60LU, +0x71d4beb4LU, 0xf6283ce1LU, 0x3261f71eLU, 0xb59d754bLU, 0xf7f32cadLU, 0x700faef8LU, 0xb4466507LU, 0x33bae752LU }; + +static const ulong32 rs_tab3[256] = { +0x00000000LU, 0x5ac1f387LU, 0xb4cfab43LU, 0xee0e58c4LU, 0x25d31b86LU, 0x7f12e801LU, 0x911cb0c5LU, 0xcbdd4342LU, +0x4aeb3641LU, 0x102ac5c6LU, 0xfe249d02LU, 0xa4e56e85LU, 0x6f382dc7LU, 0x35f9de40LU, 0xdbf78684LU, 0x81367503LU, +0x949b6c82LU, 0xce5a9f05LU, 0x2054c7c1LU, 0x7a953446LU, 0xb1487704LU, 0xeb898483LU, 0x0587dc47LU, 0x5f462fc0LU, +0xde705ac3LU, 0x84b1a944LU, 0x6abff180LU, 0x307e0207LU, 0xfba34145LU, 0xa162b2c2LU, 0x4f6cea06LU, 0x15ad1981LU, +0x657bd849LU, 0x3fba2bceLU, 0xd1b4730aLU, 0x8b75808dLU, 0x40a8c3cfLU, 0x1a693048LU, 0xf467688cLU, 0xaea69b0bLU, +0x2f90ee08LU, 0x75511d8fLU, 0x9b5f454bLU, 0xc19eb6ccLU, 0x0a43f58eLU, 0x50820609LU, 0xbe8c5ecdLU, 0xe44dad4aLU, +0xf1e0b4cbLU, 0xab21474cLU, 0x452f1f88LU, 0x1feeec0fLU, 0xd433af4dLU, 0x8ef25ccaLU, 0x60fc040eLU, 0x3a3df789LU, +0xbb0b828aLU, 0xe1ca710dLU, 0x0fc429c9LU, 0x5505da4eLU, 0x9ed8990cLU, 0xc4196a8bLU, 0x2a17324fLU, 0x70d6c1c8LU, +0xcaf6fd92LU, 0x90370e15LU, 0x7e3956d1LU, 0x24f8a556LU, 0xef25e614LU, 0xb5e41593LU, 0x5bea4d57LU, 0x012bbed0LU, +0x801dcbd3LU, 0xdadc3854LU, 0x34d26090LU, 0x6e139317LU, 0xa5ced055LU, 0xff0f23d2LU, 0x11017b16LU, 0x4bc08891LU, +0x5e6d9110LU, 0x04ac6297LU, 0xeaa23a53LU, 0xb063c9d4LU, 0x7bbe8a96LU, 0x217f7911LU, 0xcf7121d5LU, 0x95b0d252LU, +0x1486a751LU, 0x4e4754d6LU, 0xa0490c12LU, 0xfa88ff95LU, 0x3155bcd7LU, 0x6b944f50LU, 0x859a1794LU, 0xdf5be413LU, +0xaf8d25dbLU, 0xf54cd65cLU, 0x1b428e98LU, 0x41837d1fLU, 0x8a5e3e5dLU, 0xd09fcddaLU, 0x3e91951eLU, 0x64506699LU, +0xe566139aLU, 0xbfa7e01dLU, 0x51a9b8d9LU, 0x0b684b5eLU, 0xc0b5081cLU, 0x9a74fb9bLU, 0x747aa35fLU, 0x2ebb50d8LU, +0x3b164959LU, 0x61d7badeLU, 0x8fd9e21aLU, 0xd518119dLU, 0x1ec552dfLU, 0x4404a158LU, 0xaa0af99cLU, 0xf0cb0a1bLU, +0x71fd7f18LU, 0x2b3c8c9fLU, 0xc532d45bLU, 0x9ff327dcLU, 0x542e649eLU, 0x0eef9719LU, 0xe0e1cfddLU, 0xba203c5aLU, +0xd9a1b769LU, 0x836044eeLU, 0x6d6e1c2aLU, 0x37afefadLU, 0xfc72acefLU, 0xa6b35f68LU, 0x48bd07acLU, 0x127cf42bLU, +0x934a8128LU, 0xc98b72afLU, 0x27852a6bLU, 0x7d44d9ecLU, 0xb6999aaeLU, 0xec586929LU, 0x025631edLU, 0x5897c26aLU, +0x4d3adbebLU, 0x17fb286cLU, 0xf9f570a8LU, 0xa334832fLU, 0x68e9c06dLU, 0x322833eaLU, 0xdc266b2eLU, 0x86e798a9LU, +0x07d1edaaLU, 0x5d101e2dLU, 0xb31e46e9LU, 0xe9dfb56eLU, 0x2202f62cLU, 0x78c305abLU, 0x96cd5d6fLU, 0xcc0caee8LU, +0xbcda6f20LU, 0xe61b9ca7LU, 0x0815c463LU, 0x52d437e4LU, 0x990974a6LU, 0xc3c88721LU, 0x2dc6dfe5LU, 0x77072c62LU, +0xf6315961LU, 0xacf0aae6LU, 0x42fef222LU, 0x183f01a5LU, 0xd3e242e7LU, 0x8923b160LU, 0x672de9a4LU, 0x3dec1a23LU, +0x284103a2LU, 0x7280f025LU, 0x9c8ea8e1LU, 0xc64f5b66LU, 0x0d921824LU, 0x5753eba3LU, 0xb95db367LU, 0xe39c40e0LU, +0x62aa35e3LU, 0x386bc664LU, 0xd6659ea0LU, 0x8ca46d27LU, 0x47792e65LU, 0x1db8dde2LU, 0xf3b68526LU, 0xa97776a1LU, +0x13574afbLU, 0x4996b97cLU, 0xa798e1b8LU, 0xfd59123fLU, 0x3684517dLU, 0x6c45a2faLU, 0x824bfa3eLU, 0xd88a09b9LU, +0x59bc7cbaLU, 0x037d8f3dLU, 0xed73d7f9LU, 0xb7b2247eLU, 0x7c6f673cLU, 0x26ae94bbLU, 0xc8a0cc7fLU, 0x92613ff8LU, +0x87cc2679LU, 0xdd0dd5feLU, 0x33038d3aLU, 0x69c27ebdLU, 0xa21f3dffLU, 0xf8dece78LU, 0x16d096bcLU, 0x4c11653bLU, +0xcd271038LU, 0x97e6e3bfLU, 0x79e8bb7bLU, 0x232948fcLU, 0xe8f40bbeLU, 0xb235f839LU, 0x5c3ba0fdLU, 0x06fa537aLU, +0x762c92b2LU, 0x2ced6135LU, 0xc2e339f1LU, 0x9822ca76LU, 0x53ff8934LU, 0x093e7ab3LU, 0xe7302277LU, 0xbdf1d1f0LU, +0x3cc7a4f3LU, 0x66065774LU, 0x88080fb0LU, 0xd2c9fc37LU, 0x1914bf75LU, 0x43d54cf2LU, 0xaddb1436LU, 0xf71ae7b1LU, +0xe2b7fe30LU, 0xb8760db7LU, 0x56785573LU, 0x0cb9a6f4LU, 0xc764e5b6LU, 0x9da51631LU, 0x73ab4ef5LU, 0x296abd72LU, +0xa85cc871LU, 0xf29d3bf6LU, 0x1c936332LU, 0x465290b5LU, 0x8d8fd3f7LU, 0xd74e2070LU, 0x394078b4LU, 0x63818b33LU }; + +static const ulong32 rs_tab4[256] = { +0x00000000LU, 0x58471e5aLU, 0xb08e3cb4LU, 0xe8c922eeLU, 0x2d517825LU, 0x7516667fLU, 0x9ddf4491LU, 0xc5985acbLU, +0x5aa2f04aLU, 0x02e5ee10LU, 0xea2cccfeLU, 0xb26bd2a4LU, 0x77f3886fLU, 0x2fb49635LU, 0xc77db4dbLU, 0x9f3aaa81LU, +0xb409ad94LU, 0xec4eb3ceLU, 0x04879120LU, 0x5cc08f7aLU, 0x9958d5b1LU, 0xc11fcbebLU, 0x29d6e905LU, 0x7191f75fLU, +0xeeab5ddeLU, 0xb6ec4384LU, 0x5e25616aLU, 0x06627f30LU, 0xc3fa25fbLU, 0x9bbd3ba1LU, 0x7374194fLU, 0x2b330715LU, +0x25121765LU, 0x7d55093fLU, 0x959c2bd1LU, 0xcddb358bLU, 0x08436f40LU, 0x5004711aLU, 0xb8cd53f4LU, 0xe08a4daeLU, +0x7fb0e72fLU, 0x27f7f975LU, 0xcf3edb9bLU, 0x9779c5c1LU, 0x52e19f0aLU, 0x0aa68150LU, 0xe26fa3beLU, 0xba28bde4LU, +0x911bbaf1LU, 0xc95ca4abLU, 0x21958645LU, 0x79d2981fLU, 0xbc4ac2d4LU, 0xe40ddc8eLU, 0x0cc4fe60LU, 0x5483e03aLU, +0xcbb94abbLU, 0x93fe54e1LU, 0x7b37760fLU, 0x23706855LU, 0xe6e8329eLU, 0xbeaf2cc4LU, 0x56660e2aLU, 0x0e211070LU, +0x4a242ecaLU, 0x12633090LU, 0xfaaa127eLU, 0xa2ed0c24LU, 0x677556efLU, 0x3f3248b5LU, 0xd7fb6a5bLU, 0x8fbc7401LU, +0x1086de80LU, 0x48c1c0daLU, 0xa008e234LU, 0xf84ffc6eLU, 0x3dd7a6a5LU, 0x6590b8ffLU, 0x8d599a11LU, 0xd51e844bLU, +0xfe2d835eLU, 0xa66a9d04LU, 0x4ea3bfeaLU, 0x16e4a1b0LU, 0xd37cfb7bLU, 0x8b3be521LU, 0x63f2c7cfLU, 0x3bb5d995LU, +0xa48f7314LU, 0xfcc86d4eLU, 0x14014fa0LU, 0x4c4651faLU, 0x89de0b31LU, 0xd199156bLU, 0x39503785LU, 0x611729dfLU, +0x6f3639afLU, 0x377127f5LU, 0xdfb8051bLU, 0x87ff1b41LU, 0x4267418aLU, 0x1a205fd0LU, 0xf2e97d3eLU, 0xaaae6364LU, +0x3594c9e5LU, 0x6dd3d7bfLU, 0x851af551LU, 0xdd5deb0bLU, 0x18c5b1c0LU, 0x4082af9aLU, 0xa84b8d74LU, 0xf00c932eLU, +0xdb3f943bLU, 0x83788a61LU, 0x6bb1a88fLU, 0x33f6b6d5LU, 0xf66eec1eLU, 0xae29f244LU, 0x46e0d0aaLU, 0x1ea7cef0LU, +0x819d6471LU, 0xd9da7a2bLU, 0x311358c5LU, 0x6954469fLU, 0xaccc1c54LU, 0xf48b020eLU, 0x1c4220e0LU, 0x44053ebaLU, +0x94485cd9LU, 0xcc0f4283LU, 0x24c6606dLU, 0x7c817e37LU, 0xb91924fcLU, 0xe15e3aa6LU, 0x09971848LU, 0x51d00612LU, +0xceeaac93LU, 0x96adb2c9LU, 0x7e649027LU, 0x26238e7dLU, 0xe3bbd4b6LU, 0xbbfccaecLU, 0x5335e802LU, 0x0b72f658LU, +0x2041f14dLU, 0x7806ef17LU, 0x90cfcdf9LU, 0xc888d3a3LU, 0x0d108968LU, 0x55579732LU, 0xbd9eb5dcLU, 0xe5d9ab86LU, +0x7ae30107LU, 0x22a41f5dLU, 0xca6d3db3LU, 0x922a23e9LU, 0x57b27922LU, 0x0ff56778LU, 0xe73c4596LU, 0xbf7b5bccLU, +0xb15a4bbcLU, 0xe91d55e6LU, 0x01d47708LU, 0x59936952LU, 0x9c0b3399LU, 0xc44c2dc3LU, 0x2c850f2dLU, 0x74c21177LU, +0xebf8bbf6LU, 0xb3bfa5acLU, 0x5b768742LU, 0x03319918LU, 0xc6a9c3d3LU, 0x9eeedd89LU, 0x7627ff67LU, 0x2e60e13dLU, +0x0553e628LU, 0x5d14f872LU, 0xb5ddda9cLU, 0xed9ac4c6LU, 0x28029e0dLU, 0x70458057LU, 0x988ca2b9LU, 0xc0cbbce3LU, +0x5ff11662LU, 0x07b60838LU, 0xef7f2ad6LU, 0xb738348cLU, 0x72a06e47LU, 0x2ae7701dLU, 0xc22e52f3LU, 0x9a694ca9LU, +0xde6c7213LU, 0x862b6c49LU, 0x6ee24ea7LU, 0x36a550fdLU, 0xf33d0a36LU, 0xab7a146cLU, 0x43b33682LU, 0x1bf428d8LU, +0x84ce8259LU, 0xdc899c03LU, 0x3440beedLU, 0x6c07a0b7LU, 0xa99ffa7cLU, 0xf1d8e426LU, 0x1911c6c8LU, 0x4156d892LU, +0x6a65df87LU, 0x3222c1ddLU, 0xdaebe333LU, 0x82acfd69LU, 0x4734a7a2LU, 0x1f73b9f8LU, 0xf7ba9b16LU, 0xaffd854cLU, +0x30c72fcdLU, 0x68803197LU, 0x80491379LU, 0xd80e0d23LU, 0x1d9657e8LU, 0x45d149b2LU, 0xad186b5cLU, 0xf55f7506LU, +0xfb7e6576LU, 0xa3397b2cLU, 0x4bf059c2LU, 0x13b74798LU, 0xd62f1d53LU, 0x8e680309LU, 0x66a121e7LU, 0x3ee63fbdLU, +0xa1dc953cLU, 0xf99b8b66LU, 0x1152a988LU, 0x4915b7d2LU, 0x8c8ded19LU, 0xd4caf343LU, 0x3c03d1adLU, 0x6444cff7LU, +0x4f77c8e2LU, 0x1730d6b8LU, 0xfff9f456LU, 0xa7beea0cLU, 0x6226b0c7LU, 0x3a61ae9dLU, 0xd2a88c73LU, 0x8aef9229LU, +0x15d538a8LU, 0x4d9226f2LU, 0xa55b041cLU, 0xfd1c1a46LU, 0x3884408dLU, 0x60c35ed7LU, 0x880a7c39LU, 0xd04d6263LU }; + +static const ulong32 rs_tab5[256] = { +0x00000000LU, 0xdbaec658LU, 0xfb11c1b0LU, 0x20bf07e8LU, 0xbb22cf2dLU, 0x608c0975LU, 0x40330e9dLU, 0x9b9dc8c5LU, +0x3b44d35aLU, 0xe0ea1502LU, 0xc05512eaLU, 0x1bfbd4b2LU, 0x80661c77LU, 0x5bc8da2fLU, 0x7b77ddc7LU, 0xa0d91b9fLU, +0x7688ebb4LU, 0xad262decLU, 0x8d992a04LU, 0x5637ec5cLU, 0xcdaa2499LU, 0x1604e2c1LU, 0x36bbe529LU, 0xed152371LU, +0x4dcc38eeLU, 0x9662feb6LU, 0xb6ddf95eLU, 0x6d733f06LU, 0xf6eef7c3LU, 0x2d40319bLU, 0x0dff3673LU, 0xd651f02bLU, +0xec5d9b25LU, 0x37f35d7dLU, 0x174c5a95LU, 0xcce29ccdLU, 0x577f5408LU, 0x8cd19250LU, 0xac6e95b8LU, 0x77c053e0LU, +0xd719487fLU, 0x0cb78e27LU, 0x2c0889cfLU, 0xf7a64f97LU, 0x6c3b8752LU, 0xb795410aLU, 0x972a46e2LU, 0x4c8480baLU, +0x9ad57091LU, 0x417bb6c9LU, 0x61c4b121LU, 0xba6a7779LU, 0x21f7bfbcLU, 0xfa5979e4LU, 0xdae67e0cLU, 0x0148b854LU, +0xa191a3cbLU, 0x7a3f6593LU, 0x5a80627bLU, 0x812ea423LU, 0x1ab36ce6LU, 0xc11daabeLU, 0xe1a2ad56LU, 0x3a0c6b0eLU, +0x95ba7b4aLU, 0x4e14bd12LU, 0x6eabbafaLU, 0xb5057ca2LU, 0x2e98b467LU, 0xf536723fLU, 0xd58975d7LU, 0x0e27b38fLU, +0xaefea810LU, 0x75506e48LU, 0x55ef69a0LU, 0x8e41aff8LU, 0x15dc673dLU, 0xce72a165LU, 0xeecda68dLU, 0x356360d5LU, +0xe33290feLU, 0x389c56a6LU, 0x1823514eLU, 0xc38d9716LU, 0x58105fd3LU, 0x83be998bLU, 0xa3019e63LU, 0x78af583bLU, +0xd87643a4LU, 0x03d885fcLU, 0x23678214LU, 0xf8c9444cLU, 0x63548c89LU, 0xb8fa4ad1LU, 0x98454d39LU, 0x43eb8b61LU, +0x79e7e06fLU, 0xa2492637LU, 0x82f621dfLU, 0x5958e787LU, 0xc2c52f42LU, 0x196be91aLU, 0x39d4eef2LU, 0xe27a28aaLU, +0x42a33335LU, 0x990df56dLU, 0xb9b2f285LU, 0x621c34ddLU, 0xf981fc18LU, 0x222f3a40LU, 0x02903da8LU, 0xd93efbf0LU, +0x0f6f0bdbLU, 0xd4c1cd83LU, 0xf47eca6bLU, 0x2fd00c33LU, 0xb44dc4f6LU, 0x6fe302aeLU, 0x4f5c0546LU, 0x94f2c31eLU, +0x342bd881LU, 0xef851ed9LU, 0xcf3a1931LU, 0x1494df69LU, 0x8f0917acLU, 0x54a7d1f4LU, 0x7418d61cLU, 0xafb61044LU, +0x6739f694LU, 0xbc9730ccLU, 0x9c283724LU, 0x4786f17cLU, 0xdc1b39b9LU, 0x07b5ffe1LU, 0x270af809LU, 0xfca43e51LU, +0x5c7d25ceLU, 0x87d3e396LU, 0xa76ce47eLU, 0x7cc22226LU, 0xe75feae3LU, 0x3cf12cbbLU, 0x1c4e2b53LU, 0xc7e0ed0bLU, +0x11b11d20LU, 0xca1fdb78LU, 0xeaa0dc90LU, 0x310e1ac8LU, 0xaa93d20dLU, 0x713d1455LU, 0x518213bdLU, 0x8a2cd5e5LU, +0x2af5ce7aLU, 0xf15b0822LU, 0xd1e40fcaLU, 0x0a4ac992LU, 0x91d70157LU, 0x4a79c70fLU, 0x6ac6c0e7LU, 0xb16806bfLU, +0x8b646db1LU, 0x50caabe9LU, 0x7075ac01LU, 0xabdb6a59LU, 0x3046a29cLU, 0xebe864c4LU, 0xcb57632cLU, 0x10f9a574LU, +0xb020beebLU, 0x6b8e78b3LU, 0x4b317f5bLU, 0x909fb903LU, 0x0b0271c6LU, 0xd0acb79eLU, 0xf013b076LU, 0x2bbd762eLU, +0xfdec8605LU, 0x2642405dLU, 0x06fd47b5LU, 0xdd5381edLU, 0x46ce4928LU, 0x9d608f70LU, 0xbddf8898LU, 0x66714ec0LU, +0xc6a8555fLU, 0x1d069307LU, 0x3db994efLU, 0xe61752b7LU, 0x7d8a9a72LU, 0xa6245c2aLU, 0x869b5bc2LU, 0x5d359d9aLU, +0xf2838ddeLU, 0x292d4b86LU, 0x09924c6eLU, 0xd23c8a36LU, 0x49a142f3LU, 0x920f84abLU, 0xb2b08343LU, 0x691e451bLU, +0xc9c75e84LU, 0x126998dcLU, 0x32d69f34LU, 0xe978596cLU, 0x72e591a9LU, 0xa94b57f1LU, 0x89f45019LU, 0x525a9641LU, +0x840b666aLU, 0x5fa5a032LU, 0x7f1aa7daLU, 0xa4b46182LU, 0x3f29a947LU, 0xe4876f1fLU, 0xc43868f7LU, 0x1f96aeafLU, +0xbf4fb530LU, 0x64e17368LU, 0x445e7480LU, 0x9ff0b2d8LU, 0x046d7a1dLU, 0xdfc3bc45LU, 0xff7cbbadLU, 0x24d27df5LU, +0x1ede16fbLU, 0xc570d0a3LU, 0xe5cfd74bLU, 0x3e611113LU, 0xa5fcd9d6LU, 0x7e521f8eLU, 0x5eed1866LU, 0x8543de3eLU, +0x259ac5a1LU, 0xfe3403f9LU, 0xde8b0411LU, 0x0525c249LU, 0x9eb80a8cLU, 0x4516ccd4LU, 0x65a9cb3cLU, 0xbe070d64LU, +0x6856fd4fLU, 0xb3f83b17LU, 0x93473cffLU, 0x48e9faa7LU, 0xd3743262LU, 0x08daf43aLU, 0x2865f3d2LU, 0xf3cb358aLU, +0x53122e15LU, 0x88bce84dLU, 0xa803efa5LU, 0x73ad29fdLU, 0xe830e138LU, 0x339e2760LU, 0x13212088LU, 0xc88fe6d0LU }; + +static const ulong32 rs_tab6[256] = { +0x00000000LU, 0x9e3d68dbLU, 0x717ad0fbLU, 0xef47b820LU, 0xe2f4edbbLU, 0x7cc98560LU, 0x938e3d40LU, 0x0db3559bLU, +0x89a5973bLU, 0x1798ffe0LU, 0xf8df47c0LU, 0x66e22f1bLU, 0x6b517a80LU, 0xf56c125bLU, 0x1a2baa7bLU, 0x8416c2a0LU, +0x5f076376LU, 0xc13a0badLU, 0x2e7db38dLU, 0xb040db56LU, 0xbdf38ecdLU, 0x23cee616LU, 0xcc895e36LU, 0x52b436edLU, +0xd6a2f44dLU, 0x489f9c96LU, 0xa7d824b6LU, 0x39e54c6dLU, 0x345619f6LU, 0xaa6b712dLU, 0x452cc90dLU, 0xdb11a1d6LU, +0xbe0ec6ecLU, 0x2033ae37LU, 0xcf741617LU, 0x51497eccLU, 0x5cfa2b57LU, 0xc2c7438cLU, 0x2d80fbacLU, 0xb3bd9377LU, +0x37ab51d7LU, 0xa996390cLU, 0x46d1812cLU, 0xd8ece9f7LU, 0xd55fbc6cLU, 0x4b62d4b7LU, 0xa4256c97LU, 0x3a18044cLU, +0xe109a59aLU, 0x7f34cd41LU, 0x90737561LU, 0x0e4e1dbaLU, 0x03fd4821LU, 0x9dc020faLU, 0x728798daLU, 0xecbaf001LU, +0x68ac32a1LU, 0xf6915a7aLU, 0x19d6e25aLU, 0x87eb8a81LU, 0x8a58df1aLU, 0x1465b7c1LU, 0xfb220fe1LU, 0x651f673aLU, +0x311cc195LU, 0xaf21a94eLU, 0x4066116eLU, 0xde5b79b5LU, 0xd3e82c2eLU, 0x4dd544f5LU, 0xa292fcd5LU, 0x3caf940eLU, +0xb8b956aeLU, 0x26843e75LU, 0xc9c38655LU, 0x57feee8eLU, 0x5a4dbb15LU, 0xc470d3ceLU, 0x2b376beeLU, 0xb50a0335LU, +0x6e1ba2e3LU, 0xf026ca38LU, 0x1f617218LU, 0x815c1ac3LU, 0x8cef4f58LU, 0x12d22783LU, 0xfd959fa3LU, 0x63a8f778LU, +0xe7be35d8LU, 0x79835d03LU, 0x96c4e523LU, 0x08f98df8LU, 0x054ad863LU, 0x9b77b0b8LU, 0x74300898LU, 0xea0d6043LU, +0x8f120779LU, 0x112f6fa2LU, 0xfe68d782LU, 0x6055bf59LU, 0x6de6eac2LU, 0xf3db8219LU, 0x1c9c3a39LU, 0x82a152e2LU, +0x06b79042LU, 0x988af899LU, 0x77cd40b9LU, 0xe9f02862LU, 0xe4437df9LU, 0x7a7e1522LU, 0x9539ad02LU, 0x0b04c5d9LU, +0xd015640fLU, 0x4e280cd4LU, 0xa16fb4f4LU, 0x3f52dc2fLU, 0x32e189b4LU, 0xacdce16fLU, 0x439b594fLU, 0xdda63194LU, +0x59b0f334LU, 0xc78d9befLU, 0x28ca23cfLU, 0xb6f74b14LU, 0xbb441e8fLU, 0x25797654LU, 0xca3ece74LU, 0x5403a6afLU, +0x6238cf67LU, 0xfc05a7bcLU, 0x13421f9cLU, 0x8d7f7747LU, 0x80cc22dcLU, 0x1ef14a07LU, 0xf1b6f227LU, 0x6f8b9afcLU, +0xeb9d585cLU, 0x75a03087LU, 0x9ae788a7LU, 0x04dae07cLU, 0x0969b5e7LU, 0x9754dd3cLU, 0x7813651cLU, 0xe62e0dc7LU, +0x3d3fac11LU, 0xa302c4caLU, 0x4c457ceaLU, 0xd2781431LU, 0xdfcb41aaLU, 0x41f62971LU, 0xaeb19151LU, 0x308cf98aLU, +0xb49a3b2aLU, 0x2aa753f1LU, 0xc5e0ebd1LU, 0x5bdd830aLU, 0x566ed691LU, 0xc853be4aLU, 0x2714066aLU, 0xb9296eb1LU, +0xdc36098bLU, 0x420b6150LU, 0xad4cd970LU, 0x3371b1abLU, 0x3ec2e430LU, 0xa0ff8cebLU, 0x4fb834cbLU, 0xd1855c10LU, +0x55939eb0LU, 0xcbaef66bLU, 0x24e94e4bLU, 0xbad42690LU, 0xb767730bLU, 0x295a1bd0LU, 0xc61da3f0LU, 0x5820cb2bLU, +0x83316afdLU, 0x1d0c0226LU, 0xf24bba06LU, 0x6c76d2ddLU, 0x61c58746LU, 0xfff8ef9dLU, 0x10bf57bdLU, 0x8e823f66LU, +0x0a94fdc6LU, 0x94a9951dLU, 0x7bee2d3dLU, 0xe5d345e6LU, 0xe860107dLU, 0x765d78a6LU, 0x991ac086LU, 0x0727a85dLU, +0x53240ef2LU, 0xcd196629LU, 0x225ede09LU, 0xbc63b6d2LU, 0xb1d0e349LU, 0x2fed8b92LU, 0xc0aa33b2LU, 0x5e975b69LU, +0xda8199c9LU, 0x44bcf112LU, 0xabfb4932LU, 0x35c621e9LU, 0x38757472LU, 0xa6481ca9LU, 0x490fa489LU, 0xd732cc52LU, +0x0c236d84LU, 0x921e055fLU, 0x7d59bd7fLU, 0xe364d5a4LU, 0xeed7803fLU, 0x70eae8e4LU, 0x9fad50c4LU, 0x0190381fLU, +0x8586fabfLU, 0x1bbb9264LU, 0xf4fc2a44LU, 0x6ac1429fLU, 0x67721704LU, 0xf94f7fdfLU, 0x1608c7ffLU, 0x8835af24LU, +0xed2ac81eLU, 0x7317a0c5LU, 0x9c5018e5LU, 0x026d703eLU, 0x0fde25a5LU, 0x91e34d7eLU, 0x7ea4f55eLU, 0xe0999d85LU, +0x648f5f25LU, 0xfab237feLU, 0x15f58fdeLU, 0x8bc8e705LU, 0x867bb29eLU, 0x1846da45LU, 0xf7016265LU, 0x693c0abeLU, +0xb22dab68LU, 0x2c10c3b3LU, 0xc3577b93LU, 0x5d6a1348LU, 0x50d946d3LU, 0xcee42e08LU, 0x21a39628LU, 0xbf9efef3LU, +0x3b883c53LU, 0xa5b55488LU, 0x4af2eca8LU, 0xd4cf8473LU, 0xd97cd1e8LU, 0x4741b933LU, 0xa8060113LU, 0x363b69c8LU }; + +static const ulong32 rs_tab7[256] = { +0x00000000LU, 0x0319e59eLU, 0x06328771LU, 0x052b62efLU, 0x0c6443e2LU, 0x0f7da67cLU, 0x0a56c493LU, 0x094f210dLU, +0x18c88689LU, 0x1bd16317LU, 0x1efa01f8LU, 0x1de3e466LU, 0x14acc56bLU, 0x17b520f5LU, 0x129e421aLU, 0x1187a784LU, +0x30dd415fLU, 0x33c4a4c1LU, 0x36efc62eLU, 0x35f623b0LU, 0x3cb902bdLU, 0x3fa0e723LU, 0x3a8b85ccLU, 0x39926052LU, +0x2815c7d6LU, 0x2b0c2248LU, 0x2e2740a7LU, 0x2d3ea539LU, 0x24718434LU, 0x276861aaLU, 0x22430345LU, 0x215ae6dbLU, +0x60f782beLU, 0x63ee6720LU, 0x66c505cfLU, 0x65dce051LU, 0x6c93c15cLU, 0x6f8a24c2LU, 0x6aa1462dLU, 0x69b8a3b3LU, +0x783f0437LU, 0x7b26e1a9LU, 0x7e0d8346LU, 0x7d1466d8LU, 0x745b47d5LU, 0x7742a24bLU, 0x7269c0a4LU, 0x7170253aLU, +0x502ac3e1LU, 0x5333267fLU, 0x56184490LU, 0x5501a10eLU, 0x5c4e8003LU, 0x5f57659dLU, 0x5a7c0772LU, 0x5965e2ecLU, +0x48e24568LU, 0x4bfba0f6LU, 0x4ed0c219LU, 0x4dc92787LU, 0x4486068aLU, 0x479fe314LU, 0x42b481fbLU, 0x41ad6465LU, +0xc0a34931LU, 0xc3baacafLU, 0xc691ce40LU, 0xc5882bdeLU, 0xccc70ad3LU, 0xcfdeef4dLU, 0xcaf58da2LU, 0xc9ec683cLU, +0xd86bcfb8LU, 0xdb722a26LU, 0xde5948c9LU, 0xdd40ad57LU, 0xd40f8c5aLU, 0xd71669c4LU, 0xd23d0b2bLU, 0xd124eeb5LU, +0xf07e086eLU, 0xf367edf0LU, 0xf64c8f1fLU, 0xf5556a81LU, 0xfc1a4b8cLU, 0xff03ae12LU, 0xfa28ccfdLU, 0xf9312963LU, +0xe8b68ee7LU, 0xebaf6b79LU, 0xee840996LU, 0xed9dec08LU, 0xe4d2cd05LU, 0xe7cb289bLU, 0xe2e04a74LU, 0xe1f9afeaLU, +0xa054cb8fLU, 0xa34d2e11LU, 0xa6664cfeLU, 0xa57fa960LU, 0xac30886dLU, 0xaf296df3LU, 0xaa020f1cLU, 0xa91bea82LU, +0xb89c4d06LU, 0xbb85a898LU, 0xbeaeca77LU, 0xbdb72fe9LU, 0xb4f80ee4LU, 0xb7e1eb7aLU, 0xb2ca8995LU, 0xb1d36c0bLU, +0x90898ad0LU, 0x93906f4eLU, 0x96bb0da1LU, 0x95a2e83fLU, 0x9cedc932LU, 0x9ff42cacLU, 0x9adf4e43LU, 0x99c6abddLU, +0x88410c59LU, 0x8b58e9c7LU, 0x8e738b28LU, 0x8d6a6eb6LU, 0x84254fbbLU, 0x873caa25LU, 0x8217c8caLU, 0x810e2d54LU, +0xcd0b9262LU, 0xce1277fcLU, 0xcb391513LU, 0xc820f08dLU, 0xc16fd180LU, 0xc276341eLU, 0xc75d56f1LU, 0xc444b36fLU, +0xd5c314ebLU, 0xd6daf175LU, 0xd3f1939aLU, 0xd0e87604LU, 0xd9a75709LU, 0xdabeb297LU, 0xdf95d078LU, 0xdc8c35e6LU, +0xfdd6d33dLU, 0xfecf36a3LU, 0xfbe4544cLU, 0xf8fdb1d2LU, 0xf1b290dfLU, 0xf2ab7541LU, 0xf78017aeLU, 0xf499f230LU, +0xe51e55b4LU, 0xe607b02aLU, 0xe32cd2c5LU, 0xe035375bLU, 0xe97a1656LU, 0xea63f3c8LU, 0xef489127LU, 0xec5174b9LU, +0xadfc10dcLU, 0xaee5f542LU, 0xabce97adLU, 0xa8d77233LU, 0xa198533eLU, 0xa281b6a0LU, 0xa7aad44fLU, 0xa4b331d1LU, +0xb5349655LU, 0xb62d73cbLU, 0xb3061124LU, 0xb01ff4baLU, 0xb950d5b7LU, 0xba493029LU, 0xbf6252c6LU, 0xbc7bb758LU, +0x9d215183LU, 0x9e38b41dLU, 0x9b13d6f2LU, 0x980a336cLU, 0x91451261LU, 0x925cf7ffLU, 0x97779510LU, 0x946e708eLU, +0x85e9d70aLU, 0x86f03294LU, 0x83db507bLU, 0x80c2b5e5LU, 0x898d94e8LU, 0x8a947176LU, 0x8fbf1399LU, 0x8ca6f607LU, +0x0da8db53LU, 0x0eb13ecdLU, 0x0b9a5c22LU, 0x0883b9bcLU, 0x01cc98b1LU, 0x02d57d2fLU, 0x07fe1fc0LU, 0x04e7fa5eLU, +0x15605ddaLU, 0x1679b844LU, 0x1352daabLU, 0x104b3f35LU, 0x19041e38LU, 0x1a1dfba6LU, 0x1f369949LU, 0x1c2f7cd7LU, +0x3d759a0cLU, 0x3e6c7f92LU, 0x3b471d7dLU, 0x385ef8e3LU, 0x3111d9eeLU, 0x32083c70LU, 0x37235e9fLU, 0x343abb01LU, +0x25bd1c85LU, 0x26a4f91bLU, 0x238f9bf4LU, 0x20967e6aLU, 0x29d95f67LU, 0x2ac0baf9LU, 0x2febd816LU, 0x2cf23d88LU, +0x6d5f59edLU, 0x6e46bc73LU, 0x6b6dde9cLU, 0x68743b02LU, 0x613b1a0fLU, 0x6222ff91LU, 0x67099d7eLU, 0x641078e0LU, +0x7597df64LU, 0x768e3afaLU, 0x73a55815LU, 0x70bcbd8bLU, 0x79f39c86LU, 0x7aea7918LU, 0x7fc11bf7LU, 0x7cd8fe69LU, +0x5d8218b2LU, 0x5e9bfd2cLU, 0x5bb09fc3LU, 0x58a97a5dLU, 0x51e65b50LU, 0x52ffbeceLU, 0x57d4dc21LU, 0x54cd39bfLU, +0x454a9e3bLU, 0x46537ba5LU, 0x4378194aLU, 0x4061fcd4LU, 0x492eddd9LU, 0x4a373847LU, 0x4f1c5aa8LU, 0x4c05bf36LU }; + +#endif /* LTC_TWOFISH_ALL_TABLES */ + +#endif /* LTC_TWOFISH_TAB_C */ +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/ciphers/xtea.c b/optee_os/core/lib/libtomcrypt/src/ciphers/xtea.c new file mode 100644 index 0000000..95aaa1a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/ciphers/xtea.c @@ -0,0 +1,251 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file xtea.c + Implementation of eXtended TEA, Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_XTEA + +const struct ltc_cipher_descriptor xtea_desc = +{ + "xtea", + 1, + 16, 16, 8, 32, + &xtea_setup, + &xtea_ecb_encrypt, + &xtea_ecb_decrypt, + &xtea_test, + &xtea_done, + &xtea_keysize, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +}; + +int xtea_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey) +{ + ulong32 x, sum, K[4]; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(skey != NULL); + + /* check arguments */ + if (keylen != 16) { + return CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 0 && num_rounds != 32) { + return CRYPT_INVALID_ROUNDS; + } + + /* load key */ + LOAD32H(K[0], key+0); + LOAD32H(K[1], key+4); + LOAD32H(K[2], key+8); + LOAD32H(K[3], key+12); + + for (x = sum = 0; x < 32; x++) { + skey->xtea.A[x] = (sum + K[sum&3]) & 0xFFFFFFFFUL; + sum = (sum + 0x9E3779B9UL) & 0xFFFFFFFFUL; + skey->xtea.B[x] = (sum + K[(sum>>11)&3]) & 0xFFFFFFFFUL; + } + +#ifdef LTC_CLEAN_STACK + zeromem(&K, sizeof(K)); +#endif + + return CRYPT_OK; +} + +/** + Encrypts a block of text with LTC_XTEA + @param pt The input plaintext (8 bytes) + @param ct The output ciphertext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int xtea_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey) +{ + ulong32 y, z; + int r; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(y, &pt[0]); + LOAD32H(z, &pt[4]); + for (r = 0; r < 32; r += 4) { + y = (y + ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r])) & 0xFFFFFFFFUL; + z = (z + ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r])) & 0xFFFFFFFFUL; + + y = (y + ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r+1])) & 0xFFFFFFFFUL; + z = (z + ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r+1])) & 0xFFFFFFFFUL; + + y = (y + ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r+2])) & 0xFFFFFFFFUL; + z = (z + ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r+2])) & 0xFFFFFFFFUL; + + y = (y + ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r+3])) & 0xFFFFFFFFUL; + z = (z + ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r+3])) & 0xFFFFFFFFUL; + } + STORE32H(y, &ct[0]); + STORE32H(z, &ct[4]); + return CRYPT_OK; +} + +/** + Decrypts a block of text with LTC_XTEA + @param ct The input ciphertext (8 bytes) + @param pt The output plaintext (8 bytes) + @param skey The key as scheduled + @return CRYPT_OK if successful +*/ +int xtea_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey) +{ + ulong32 y, z; + int r; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(skey != NULL); + + LOAD32H(y, &ct[0]); + LOAD32H(z, &ct[4]); + for (r = 31; r >= 0; r -= 4) { + z = (z - ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r])) & 0xFFFFFFFFUL; + y = (y - ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r])) & 0xFFFFFFFFUL; + + z = (z - ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r-1])) & 0xFFFFFFFFUL; + y = (y - ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r-1])) & 0xFFFFFFFFUL; + + z = (z - ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r-2])) & 0xFFFFFFFFUL; + y = (y - ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r-2])) & 0xFFFFFFFFUL; + + z = (z - ((((y<<4)^(y>>5)) + y) ^ skey->xtea.B[r-3])) & 0xFFFFFFFFUL; + y = (y - ((((z<<4)^(z>>5)) + z) ^ skey->xtea.A[r-3])) & 0xFFFFFFFFUL; + } + STORE32H(y, &pt[0]); + STORE32H(z, &pt[4]); + return CRYPT_OK; +} + +/** + Performs a self-test of the LTC_XTEA block cipher + @return CRYPT_OK if functional, CRYPT_NOP if self-test has been disabled +*/ +int xtea_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + unsigned char key[16], pt[8], ct[8]; + } tests[] = { + { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xde, 0xe9, 0xd4, 0xd8, 0xf7, 0x13, 0x1e, 0xd9 } + }, { + { 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0xa5, 0x97, 0xab, 0x41, 0x76, 0x01, 0x4d, 0x72 } + }, { + { 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06 }, + { 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02 }, + { 0xb1, 0xfd, 0x5d, 0xa9, 0xcc, 0x6d, 0xc9, 0xdc } + }, { + { 0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f, + 0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87 }, + { 0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87 }, + { 0x70, 0x4b, 0x31, 0x34, 0x47, 0x44, 0xdf, 0xab } + }, { + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + { 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48 }, + { 0x49, 0x7d, 0xf3, 0xd0, 0x72, 0x61, 0x2c, 0xb5 } + }, { + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + { 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41 }, + { 0xe7, 0x8f, 0x2d, 0x13, 0x74, 0x43, 0x41, 0xd8 } + }, { + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + { 0x5a, 0x5b, 0x6e, 0x27, 0x89, 0x48, 0xd7, 0x7f }, + { 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41 } + }, { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48 }, + { 0xa0, 0x39, 0x05, 0x89, 0xf8, 0xb8, 0xef, 0xa5 } + }, { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41 }, + { 0xed, 0x23, 0x37, 0x5a, 0x82, 0x1a, 0x8c, 0x2d } + }, { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x70, 0xe1, 0x22, 0x5d, 0x6e, 0x4e, 0x76, 0x55 }, + { 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41 } + } + }; + unsigned char tmp[2][8]; + symmetric_key skey; + int i, err, y; + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + zeromem(&skey, sizeof(skey)); + if ((err = xtea_setup(tests[i].key, 16, 0, &skey)) != CRYPT_OK) { + return err; + } + xtea_ecb_encrypt(tests[i].pt, tmp[0], &skey); + xtea_ecb_decrypt(tmp[0], tmp[1], &skey); + + if (compare_testvector(tmp[0], 8, tests[i].ct, 8, "XTEA Encrypt", i) != 0 || + compare_testvector(tmp[1], 8, tests[i].pt, 8, "XTEA Decrypt", i) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* now see if we can encrypt all zero bytes 1000 times, decrypt and come back where we started */ + for (y = 0; y < 8; y++) tmp[0][y] = 0; + for (y = 0; y < 1000; y++) xtea_ecb_encrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 1000; y++) xtea_ecb_decrypt(tmp[0], tmp[0], &skey); + for (y = 0; y < 8; y++) if (tmp[0][y] != 0) return CRYPT_FAIL_TESTVECTOR; + } /* for */ + + return CRYPT_OK; + #endif +} + +/** Terminate the context + @param skey The scheduled key +*/ +void xtea_done(symmetric_key *skey) +{ + LTC_UNUSED_PARAM(skey); +} + +/** + Gets suitable key size + @param keysize [in/out] The length of the recommended key (in bytes). This function will store the suitable size back in this variable. + @return CRYPT_OK if the input key size is acceptable. +*/ +int xtea_keysize(int *keysize) +{ + LTC_ARGCHK(keysize != NULL); + if (*keysize < 16) { + return CRYPT_INVALID_KEYSIZE; + } + *keysize = 16; + return CRYPT_OK; +} + + +#endif + + + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_add_aad.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_add_aad.c new file mode 100644 index 0000000..2bb9e0b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_add_aad.c @@ -0,0 +1,53 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_CCM_MODE + +/** + Add AAD to the CCM state + @param ccm The CCM state + @param adata The additional authentication data to add to the CCM state + @param adatalen The length of the AAD data. + @return CRYPT_OK on success + */ +int ccm_add_aad(ccm_state *ccm, + const unsigned char *adata, unsigned long adatalen) +{ + unsigned long y; + int err; + + LTC_ARGCHK(ccm != NULL); + LTC_ARGCHK(adata != NULL); + + if (ccm->aadlen < ccm->current_aadlen + adatalen) { + return CRYPT_INVALID_ARG; + } + ccm->current_aadlen += adatalen; + + /* now add the data */ + for (y = 0; y < adatalen; y++) { + if (ccm->x == 16) { + /* full block so let's encrypt it */ + if ((err = cipher_descriptor[ccm->cipher]->ecb_encrypt(ccm->PAD, ccm->PAD, &ccm->K)) != CRYPT_OK) { + return err; + } + ccm->x = 0; + } + ccm->PAD[ccm->x++] ^= adata[y]; + } + + /* remainder? */ + if (ccm->aadlen == ccm->current_aadlen) { + if (ccm->x != 0) { + if ((err = cipher_descriptor[ccm->cipher]->ecb_encrypt(ccm->PAD, ccm->PAD, &ccm->K)) != CRYPT_OK) { + return err; + } + } + ccm->x = 0; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_add_nonce.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_add_nonce.c new file mode 100644 index 0000000..b2c9219 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_add_nonce.c @@ -0,0 +1,106 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_CCM_MODE + +/** + Add nonce data to the CCM state + @param ccm The CCM state + @param nonce The nonce data to add + @param noncelen The length of the nonce + @return CRYPT_OK on success + */ +int ccm_add_nonce(ccm_state *ccm, + const unsigned char *nonce, unsigned long noncelen) +{ + unsigned long x, y, len; + int err; + + LTC_ARGCHK(ccm != NULL); + LTC_ARGCHK(nonce != NULL); + + /* increase L to match the nonce len */ + ccm->noncelen = (noncelen > 13) ? 13 : noncelen; + if ((15 - ccm->noncelen) > ccm->L) { + ccm->L = 15 - ccm->noncelen; + } + if (ccm->L > 8) { + return CRYPT_INVALID_ARG; + } + + /* decrease noncelen to match L */ + if ((ccm->noncelen + ccm->L) > 15) { + ccm->noncelen = 15 - ccm->L; + } + + /* form B_0 == flags | Nonce N | l(m) */ + x = 0; + ccm->PAD[x++] = (unsigned char)(((ccm->aadlen > 0) ? (1<<6) : 0) | + (((ccm->taglen - 2)>>1)<<3) | + (ccm->L-1)); + + /* nonce */ + for (y = 0; y < 15 - ccm->L; y++) { + ccm->PAD[x++] = nonce[y]; + } + + /* store len */ + len = ccm->ptlen; + + /* shift len so the upper bytes of len are the contents of the length */ + for (y = ccm->L; y < 4; y++) { + len <<= 8; + } + + /* store l(m) (only store 32-bits) */ + for (y = 0; ccm->L > 4 && (ccm->L-y)>4; y++) { + ccm->PAD[x++] = 0; + } + for (; y < ccm->L; y++) { + ccm->PAD[x++] = (unsigned char)((len >> 24) & 255); + len <<= 8; + } + + /* encrypt PAD */ + if ((err = cipher_descriptor[ccm->cipher]->ecb_encrypt(ccm->PAD, ccm->PAD, &ccm->K)) != CRYPT_OK) { + return err; + } + + /* handle header */ + ccm->x = 0; + if (ccm->aadlen > 0) { + /* store length */ + if (ccm->aadlen < ((1UL<<16) - (1UL<<8))) { + ccm->PAD[ccm->x++] ^= (ccm->aadlen>>8) & 255; + ccm->PAD[ccm->x++] ^= ccm->aadlen & 255; + } else { + ccm->PAD[ccm->x++] ^= 0xFF; + ccm->PAD[ccm->x++] ^= 0xFE; + ccm->PAD[ccm->x++] ^= (ccm->aadlen>>24) & 255; + ccm->PAD[ccm->x++] ^= (ccm->aadlen>>16) & 255; + ccm->PAD[ccm->x++] ^= (ccm->aadlen>>8) & 255; + ccm->PAD[ccm->x++] ^= ccm->aadlen & 255; + } + } + + /* setup the ctr counter */ + x = 0; + + /* flags */ + ccm->ctr[x++] = (unsigned char)ccm->L-1; + + /* nonce */ + for (y = 0; y < (16 - (ccm->L+1)); ++y) { + ccm->ctr[x++] = nonce[y]; + } + /* offset */ + while (x < 16) { + ccm->ctr[x++] = 0; + } + + ccm->CTRlen = 16; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_done.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_done.c new file mode 100644 index 0000000..8eeffe8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_done.c @@ -0,0 +1,55 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_CCM_MODE + +/** + Terminate a CCM stream + @param ccm The CCM state + @param tag [out] The destination for the MAC tag + @param taglen [in/out] The length of the MAC tag + @return CRYPT_OK on success + */ +int ccm_done(ccm_state *ccm, + unsigned char *tag, unsigned long *taglen) +{ + unsigned long x, y; + int err; + + LTC_ARGCHK(ccm != NULL); + + /* Check all data have been processed */ + if (ccm->ptlen != ccm->current_ptlen) { + return CRYPT_ERROR; + } + + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + + if (ccm->x != 0) { + if ((err = cipher_descriptor[ccm->cipher]->ecb_encrypt(ccm->PAD, ccm->PAD, &ccm->K)) != CRYPT_OK) { + return err; + } + } + + /* setup CTR for the TAG (zero the count) */ + for (y = 15; y > 15 - ccm->L; y--) { + ccm->ctr[y] = 0x00; + } + if ((err = cipher_descriptor[ccm->cipher]->ecb_encrypt(ccm->ctr, ccm->CTRPAD, &ccm->K)) != CRYPT_OK) { + return err; + } + + cipher_descriptor[ccm->cipher]->done(&ccm->K); + + /* store the TAG */ + for (x = 0; x < 16 && x < *taglen; x++) { + tag[x] = ccm->PAD[x] ^ ccm->CTRPAD[x]; + } + *taglen = x; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_init.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_init.c new file mode 100644 index 0000000..a913ed4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_init.c @@ -0,0 +1,64 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_CCM_MODE + +/** + Initialize a CCM state + @param ccm The CCM state to initialize + @param cipher The index of the cipher to use + @param key The secret key + @param keylen The length of the secret key + @param ptlen The length of the plain/cipher text that will be processed + @param taglen The max length of the MAC tag + @param aadlen The length of the AAD + + @return CRYPT_OK on success + */ +int ccm_init(ccm_state *ccm, int cipher, + const unsigned char *key, int keylen, int ptlen, int taglen, int aadlen) +{ + int err; + + LTC_ARGCHK(ccm != NULL); + LTC_ARGCHK(key != NULL); + + XMEMSET(ccm, 0, sizeof(ccm_state)); + + /* check cipher input */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + if (cipher_descriptor[cipher]->block_length != 16) { + return CRYPT_INVALID_CIPHER; + } + + /* make sure the taglen is valid */ + if (taglen < 4 || taglen > 16 || (taglen % 2) == 1 || aadlen < 0 || ptlen < 0) { + return CRYPT_INVALID_ARG; + } + ccm->taglen = taglen; + + /* schedule key */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, &ccm->K)) != CRYPT_OK) { + return err; + } + ccm->cipher = cipher; + + /* let's get the L value */ + ccm->ptlen = ptlen; + ccm->L = 0; + while (ptlen) { + ++ccm->L; + ptlen >>= 8; + } + if (ccm->L <= 1) { + ccm->L = 2; + } + + ccm->aadlen = aadlen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_memory.c new file mode 100644 index 0000000..be8133d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_memory.c @@ -0,0 +1,372 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ccm_memory.c + CCM support, process a block of memory, Tom St Denis +*/ + +#ifdef LTC_CCM_MODE + +/** + CCM encrypt/decrypt and produce an authentication tag + + *1 'pt', 'ct' and 'tag' can both be 'in' or 'out', depending on 'direction' + + @param cipher The index of the cipher desired + @param key The secret key to use + @param keylen The length of the secret key (octets) + @param uskey A previously scheduled key [optional can be NULL] + @param nonce The session nonce [use once] + @param noncelen The length of the nonce + @param header The header for the session + @param headerlen The length of the header (octets) + @param pt [*1] The plaintext + @param ptlen The length of the plaintext (octets) + @param ct [*1] The ciphertext + @param tag [*1] The destination tag + @param taglen The max size and resulting size of the authentication tag + @param direction Encrypt or Decrypt direction (0 or 1) + @return CRYPT_OK if successful +*/ +int ccm_memory(int cipher, + const unsigned char *key, unsigned long keylen, + symmetric_key *uskey, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen, + int direction) +{ + unsigned char PAD[16], ctr[16], CTRPAD[16], ptTag[16], b, *pt_real; + unsigned char *pt_work = NULL; + symmetric_key *skey; + int err; + unsigned long len, L, x, y, z, CTRlen; + + if (uskey == NULL) { + LTC_ARGCHK(key != NULL); + } + LTC_ARGCHK(nonce != NULL); + if (headerlen > 0) { + LTC_ARGCHK(header != NULL); + } + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + + pt_real = pt; + +#ifdef LTC_FAST + if (16 % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + /* check cipher input */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + if (cipher_descriptor[cipher]->block_length != 16) { + return CRYPT_INVALID_CIPHER; + } + + /* make sure the taglen is valid */ + if (*taglen < 4 || *taglen > 16 || (*taglen % 2) == 1 || headerlen > 0x7fffffffu) { + return CRYPT_INVALID_ARG; + } + + /* is there an accelerator? */ + if (cipher_descriptor[cipher]->accel_ccm_memory != NULL) { + return cipher_descriptor[cipher]->accel_ccm_memory( + key, keylen, + uskey, + nonce, noncelen, + header, headerlen, + pt, ptlen, + ct, + tag, taglen, + direction); + } + + /* let's get the L value */ + len = ptlen; + L = 0; + while (len) { + ++L; + len >>= 8; + } + if (L <= 1) { + L = 2; + } + + /* increase L to match the nonce len */ + noncelen = (noncelen > 13) ? 13 : noncelen; + if ((15 - noncelen) > L) { + L = 15 - noncelen; + } + if (L > 8) { + return CRYPT_INVALID_ARG; + } + + /* allocate mem for the symmetric key */ + if (uskey == NULL) { + skey = XMALLOC(sizeof(*skey)); + if (skey == NULL) { + return CRYPT_MEM; + } + + /* initialize the cipher */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, skey)) != CRYPT_OK) { + XFREE(skey); + return err; + } + } else { + skey = uskey; + } + + /* initialize buffer for pt */ + if (direction == CCM_DECRYPT && ptlen > 0) { + pt_work = XMALLOC(ptlen); + if (pt_work == NULL) { + goto error; + } + pt = pt_work; + } + + /* form B_0 == flags | Nonce N | l(m) */ + x = 0; + PAD[x++] = (unsigned char)(((headerlen > 0) ? (1<<6) : 0) | + (((*taglen - 2)>>1)<<3) | + (L-1)); + + /* nonce */ + for (y = 0; y < 15 - L; y++) { + PAD[x++] = nonce[y]; + } + + /* store len */ + len = ptlen; + + /* shift len so the upper bytes of len are the contents of the length */ + for (y = L; y < 4; y++) { + len <<= 8; + } + + /* store l(m) (only store 32-bits) */ + for (y = 0; L > 4 && (L-y)>4; y++) { + PAD[x++] = 0; + } + for (; y < L; y++) { + PAD[x++] = (unsigned char)((len >> 24) & 255); + len <<= 8; + } + + /* encrypt PAD */ + if ((err = cipher_descriptor[cipher]->ecb_encrypt(PAD, PAD, skey)) != CRYPT_OK) { + goto error; + } + + /* handle header */ + if (headerlen > 0) { + x = 0; + + /* store length */ + if (headerlen < ((1UL<<16) - (1UL<<8))) { + PAD[x++] ^= (headerlen>>8) & 255; + PAD[x++] ^= headerlen & 255; + } else { + PAD[x++] ^= 0xFF; + PAD[x++] ^= 0xFE; + PAD[x++] ^= (headerlen>>24) & 255; + PAD[x++] ^= (headerlen>>16) & 255; + PAD[x++] ^= (headerlen>>8) & 255; + PAD[x++] ^= headerlen & 255; + } + + /* now add the data */ + for (y = 0; y < headerlen; y++) { + if (x == 16) { + /* full block so let's encrypt it */ + if ((err = cipher_descriptor[cipher]->ecb_encrypt(PAD, PAD, skey)) != CRYPT_OK) { + goto error; + } + x = 0; + } + PAD[x++] ^= header[y]; + } + + /* remainder */ + if ((err = cipher_descriptor[cipher]->ecb_encrypt(PAD, PAD, skey)) != CRYPT_OK) { + goto error; + } + } + + /* setup the ctr counter */ + x = 0; + + /* flags */ + ctr[x++] = (unsigned char)L-1; + + /* nonce */ + for (y = 0; y < (16 - (L+1)); ++y) { + ctr[x++] = nonce[y]; + } + /* offset */ + while (x < 16) { + ctr[x++] = 0; + } + + x = 0; + CTRlen = 16; + + /* now handle the PT */ + if (ptlen > 0) { + y = 0; +#ifdef LTC_FAST + if (ptlen & ~15) { + if (direction == CCM_ENCRYPT) { + for (; y < (ptlen & ~15); y += 16) { + /* increment the ctr? */ + for (z = 15; z > 15-L; z--) { + ctr[z] = (ctr[z] + 1) & 255; + if (ctr[z]) break; + } + if ((err = cipher_descriptor[cipher]->ecb_encrypt(ctr, CTRPAD, skey)) != CRYPT_OK) { + goto error; + } + + /* xor the PT against the pad first */ + for (z = 0; z < 16; z += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&PAD[z])) ^= *(LTC_FAST_TYPE_PTR_CAST(&pt[y+z])); + *(LTC_FAST_TYPE_PTR_CAST(&ct[y+z])) = *(LTC_FAST_TYPE_PTR_CAST(&pt[y+z])) ^ *(LTC_FAST_TYPE_PTR_CAST(&CTRPAD[z])); + } + if ((err = cipher_descriptor[cipher]->ecb_encrypt(PAD, PAD, skey)) != CRYPT_OK) { + goto error; + } + } + } else { /* direction == CCM_DECRYPT */ + for (; y < (ptlen & ~15); y += 16) { + /* increment the ctr? */ + for (z = 15; z > 15-L; z--) { + ctr[z] = (ctr[z] + 1) & 255; + if (ctr[z]) break; + } + if ((err = cipher_descriptor[cipher]->ecb_encrypt(ctr, CTRPAD, skey)) != CRYPT_OK) { + goto error; + } + + /* xor the PT against the pad last */ + for (z = 0; z < 16; z += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&pt[y+z])) = *(LTC_FAST_TYPE_PTR_CAST(&ct[y+z])) ^ *(LTC_FAST_TYPE_PTR_CAST(&CTRPAD[z])); + *(LTC_FAST_TYPE_PTR_CAST(&PAD[z])) ^= *(LTC_FAST_TYPE_PTR_CAST(&pt[y+z])); + } + if ((err = cipher_descriptor[cipher]->ecb_encrypt(PAD, PAD, skey)) != CRYPT_OK) { + goto error; + } + } + } + } +#endif + + for (; y < ptlen; y++) { + /* increment the ctr? */ + if (CTRlen == 16) { + for (z = 15; z > 15-L; z--) { + ctr[z] = (ctr[z] + 1) & 255; + if (ctr[z]) break; + } + if ((err = cipher_descriptor[cipher]->ecb_encrypt(ctr, CTRPAD, skey)) != CRYPT_OK) { + goto error; + } + CTRlen = 0; + } + + /* if we encrypt we add the bytes to the MAC first */ + if (direction == CCM_ENCRYPT) { + b = pt[y]; + ct[y] = b ^ CTRPAD[CTRlen++]; + } else { + b = ct[y] ^ CTRPAD[CTRlen++]; + pt[y] = b; + } + + if (x == 16) { + if ((err = cipher_descriptor[cipher]->ecb_encrypt(PAD, PAD, skey)) != CRYPT_OK) { + goto error; + } + x = 0; + } + PAD[x++] ^= b; + } + + if (x != 0) { + if ((err = cipher_descriptor[cipher]->ecb_encrypt(PAD, PAD, skey)) != CRYPT_OK) { + goto error; + } + } + } + + /* setup CTR for the TAG (zero the count) */ + for (y = 15; y > 15 - L; y--) { + ctr[y] = 0x00; + } + if ((err = cipher_descriptor[cipher]->ecb_encrypt(ctr, CTRPAD, skey)) != CRYPT_OK) { + goto error; + } + + if (skey != uskey) { + cipher_descriptor[cipher]->done(skey); +#ifdef LTC_CLEAN_STACK + zeromem(skey, sizeof(*skey)); +#endif + } + + if (direction == CCM_ENCRYPT) { + /* store the TAG */ + for (x = 0; x < 16 && x < *taglen; x++) { + tag[x] = PAD[x] ^ CTRPAD[x]; + } + *taglen = x; + } else { /* direction == CCM_DECRYPT */ + /* decrypt the tag */ + for (x = 0; x < 16 && x < *taglen; x++) { + ptTag[x] = tag[x] ^ CTRPAD[x]; + } + *taglen = x; + + /* check validity of the decrypted tag against the computed PAD (in constant time) */ + /* HACK: the boolean value of XMEM_NEQ becomes either 0 (CRYPT_OK) or 1 (CRYPT_ERR). + * there should be a better way of setting the correct error code in constant + * time. + */ + err = XMEM_NEQ(ptTag, PAD, *taglen); + + /* Zero the plaintext if the tag was invalid (in constant time) */ + if (ptlen > 0) { + copy_or_zeromem(pt, pt_real, ptlen, err); + } + } + +#ifdef LTC_CLEAN_STACK + zeromem(PAD, sizeof(PAD)); + zeromem(CTRPAD, sizeof(CTRPAD)); + if (pt_work != NULL) { + zeromem(pt_work, ptlen); + } +#endif +error: + if (pt_work) { + XFREE(pt_work); + } + if (skey != uskey) { + XFREE(skey); + } + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_process.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_process.c new file mode 100644 index 0000000..41bfc2c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_process.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_CCM_MODE + +/** + Process plaintext/ciphertext through CCM + @param ccm The CCM state + @param pt The plaintext + @param ptlen The plaintext length (ciphertext length is the same) + @param ct The ciphertext + @param direction Encrypt or Decrypt mode (CCM_ENCRYPT or CCM_DECRYPT) + @return CRYPT_OK on success + */ +int ccm_process(ccm_state *ccm, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + int direction) +{ + unsigned char z, b; + unsigned long y; + int err; + + LTC_ARGCHK(ccm != NULL); + + /* Check aad has been correctly added */ + if (ccm->aadlen != ccm->current_aadlen) { + return CRYPT_ERROR; + } + + /* Check we do not process too much data */ + if (ccm->ptlen < ccm->current_ptlen + ptlen) { + return CRYPT_ERROR; + } + ccm->current_ptlen += ptlen; + + /* now handle the PT */ + if (ptlen > 0) { + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + for (y = 0; y < ptlen; y++) { + /* increment the ctr? */ + if (ccm->CTRlen == 16) { + for (z = 15; z > 15-ccm->L; z--) { + ccm->ctr[z] = (ccm->ctr[z] + 1) & 255; + if (ccm->ctr[z]) break; + } + if ((err = cipher_descriptor[ccm->cipher]->ecb_encrypt(ccm->ctr, ccm->CTRPAD, &ccm->K)) != CRYPT_OK) { + return err; + } + ccm->CTRlen = 0; + } + + /* if we encrypt we add the bytes to the MAC first */ + if (direction == CCM_ENCRYPT) { + b = pt[y]; + ct[y] = b ^ ccm->CTRPAD[ccm->CTRlen++]; + } else { + b = ct[y] ^ ccm->CTRPAD[ccm->CTRlen++]; + pt[y] = b; + } + + if (ccm->x == 16) { + if ((err = cipher_descriptor[ccm->cipher]->ecb_encrypt(ccm->PAD, ccm->PAD, &ccm->K)) != CRYPT_OK) { + return err; + } + ccm->x = 0; + } + ccm->PAD[ccm->x++] ^= b; + } + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_reset.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_reset.c new file mode 100644 index 0000000..93b8fec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_reset.c @@ -0,0 +1,25 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_CCM_MODE + +/** + Reset a CCM state to as if you just called ccm_init(). This saves the initialization time. + @param ccm The CCM state to reset + @return CRYPT_OK on success +*/ +int ccm_reset(ccm_state *ccm) +{ + LTC_ARGCHK(ccm != NULL); + zeromem(ccm->PAD, sizeof(ccm->PAD)); + zeromem(ccm->ctr, sizeof(ccm->ctr)); + zeromem(ccm->CTRPAD, sizeof(ccm->CTRPAD)); + ccm->CTRlen = 0; + ccm->current_ptlen = 0; + ccm->current_aadlen = 0; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_test.c b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_test.c new file mode 100644 index 0000000..fee2c6b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/ccm_test.c @@ -0,0 +1,274 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ccm_test.c + CCM support, process a block of memory, Tom St Denis +*/ + +#ifdef LTC_CCM_MODE + +int ccm_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char key[16]; + unsigned char nonce[16]; + int noncelen; + unsigned char header[64]; + int headerlen; + unsigned char pt[64]; + int ptlen; + unsigned char ct[64]; + unsigned char tag[16]; + unsigned long taglen; + } tests[] = { + +/* 13 byte nonce, 8 byte auth, 23 byte pt */ +{ + { 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF }, + { 0x00, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00, 0xA0, + 0xA1, 0xA2, 0xA3, 0xA4, 0xA5 }, + 13, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 }, + 8, + { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E }, + 23, + { 0x58, 0x8C, 0x97, 0x9A, 0x61, 0xC6, 0x63, 0xD2, + 0xF0, 0x66, 0xD0, 0xC2, 0xC0, 0xF9, 0x89, 0x80, + 0x6D, 0x5F, 0x6B, 0x61, 0xDA, 0xC3, 0x84 }, + { 0x17, 0xe8, 0xd1, 0x2c, 0xfd, 0xf9, 0x26, 0xe0 }, + 8 +}, + +/* 13 byte nonce, 12 byte header, 19 byte pt */ +{ + { 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF }, + { 0x00, 0x00, 0x00, 0x06, 0x05, 0x04, 0x03, 0xA0, + 0xA1, 0xA2, 0xA3, 0xA4, 0xA5 }, + 13, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B }, + 12, + { 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, + 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, + 0x1C, 0x1D, 0x1E }, + 19, + { 0xA2, 0x8C, 0x68, 0x65, 0x93, 0x9A, 0x9A, 0x79, + 0xFA, 0xAA, 0x5C, 0x4C, 0x2A, 0x9D, 0x4A, 0x91, + 0xCD, 0xAC, 0x8C }, + { 0x96, 0xC8, 0x61, 0xB9, 0xC9, 0xE6, 0x1E, 0xF1 }, + 8 +}, + +/* supplied by Brian Gladman */ +{ + { 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f }, + { 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16 }, + 7, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 }, + 8, + { 0x20, 0x21, 0x22, 0x23 }, + 4, + { 0x71, 0x62, 0x01, 0x5b }, + { 0x4d, 0xac, 0x25, 0x5d }, + 4 +}, + +{ + { 0xc9, 0x7c, 0x1f, 0x67, 0xce, 0x37, 0x11, 0x85, + 0x51, 0x4a, 0x8a, 0x19, 0xf2, 0xbd, 0xd5, 0x2f }, + { 0x00, 0x50, 0x30, 0xf1, 0x84, 0x44, 0x08, 0xb5, + 0x03, 0x97, 0x76, 0xe7, 0x0c }, + 13, + { 0x08, 0x40, 0x0f, 0xd2, 0xe1, 0x28, 0xa5, 0x7c, + 0x50, 0x30, 0xf1, 0x84, 0x44, 0x08, 0xab, 0xae, + 0xa5, 0xb8, 0xfc, 0xba, 0x00, 0x00 }, + 22, + { 0xf8, 0xba, 0x1a, 0x55, 0xd0, 0x2f, 0x85, 0xae, + 0x96, 0x7b, 0xb6, 0x2f, 0xb6, 0xcd, 0xa8, 0xeb, + 0x7e, 0x78, 0xa0, 0x50 }, + 20, + { 0xf3, 0xd0, 0xa2, 0xfe, 0x9a, 0x3d, 0xbf, 0x23, + 0x42, 0xa6, 0x43, 0xe4, 0x32, 0x46, 0xe8, 0x0c, + 0x3c, 0x04, 0xd0, 0x19 }, + { 0x78, 0x45, 0xce, 0x0b, 0x16, 0xf9, 0x76, 0x23 }, + 8 +}, + +}; + unsigned long taglen, x, y; + unsigned char buf[64], buf2[64], tag[16], tag2[16], tag3[16], zero[64]; + int err, idx; + symmetric_key skey; + ccm_state ccm; + + zeromem(zero, 64); + + idx = find_cipher("aes"); + if (idx == -1) { + idx = find_cipher("rijndael"); + if (idx == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (sizeof(tests)/sizeof(tests[0])); x++) { + for (y = 0; y < 2; y++) { + taglen = tests[x].taglen; + if (y == 0) { + if ((err = cipher_descriptor[idx]->setup(tests[x].key, 16, 0, &skey)) != CRYPT_OK) { + return err; + } + + if ((err = ccm_memory(idx, + tests[x].key, 16, + &skey, + tests[x].nonce, tests[x].noncelen, + tests[x].header, tests[x].headerlen, + (unsigned char*)tests[x].pt, tests[x].ptlen, + buf, + tag, &taglen, 0)) != CRYPT_OK) { + return err; + } + /* run a second time to make sure skey is not touched */ + if ((err = ccm_memory(idx, + tests[x].key, 16, + &skey, + tests[x].nonce, tests[x].noncelen, + tests[x].header, tests[x].headerlen, + (unsigned char*)tests[x].pt, tests[x].ptlen, + buf, + tag, &taglen, 0)) != CRYPT_OK) { + return err; + } + } else { + if ((err = ccm_init(&ccm, idx, tests[x].key, 16, tests[x].ptlen, tests[x].taglen, tests[x].headerlen)) != CRYPT_OK) { + return err; + } + if ((err = ccm_add_nonce(&ccm, tests[x].nonce, tests[x].noncelen)) != CRYPT_OK) { + return err; + } + if ((err = ccm_add_aad(&ccm, tests[x].header, tests[x].headerlen)) != CRYPT_OK) { + return err; + } + if ((err = ccm_process(&ccm, (unsigned char*)tests[x].pt, tests[x].ptlen, buf, CCM_ENCRYPT)) != CRYPT_OK) { + return err; + } + if ((err = ccm_done(&ccm, tag, &taglen)) != CRYPT_OK) { + return err; + } + } + + if (compare_testvector(buf, tests[x].ptlen, tests[x].ct, tests[x].ptlen, "CCM encrypt data", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + if (compare_testvector(tag, taglen, tests[x].tag, tests[x].taglen, "CCM encrypt tag", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + if (y == 0) { + XMEMCPY(tag3, tests[x].tag, tests[x].taglen); + taglen = tests[x].taglen; + if ((err = ccm_memory(idx, + tests[x].key, 16, + NULL, + tests[x].nonce, tests[x].noncelen, + tests[x].header, tests[x].headerlen, + buf2, tests[x].ptlen, + buf, + tag3, &taglen, 1 )) != CRYPT_OK) { + return err; + } + } else { + if ((err = ccm_init(&ccm, idx, tests[x].key, 16, tests[x].ptlen, tests[x].taglen, tests[x].headerlen)) != CRYPT_OK) { + return err; + } + if ((err = ccm_add_nonce(&ccm, tests[x].nonce, tests[x].noncelen)) != CRYPT_OK) { + return err; + } + if ((err = ccm_add_aad(&ccm, tests[x].header, tests[x].headerlen)) != CRYPT_OK) { + return err; + } + if ((err = ccm_process(&ccm, buf2, tests[x].ptlen, buf, CCM_DECRYPT)) != CRYPT_OK) { + return err; + } + if ((err = ccm_done(&ccm, tag2, &taglen)) != CRYPT_OK) { + return err; + } + } + + + if (compare_testvector(buf2, tests[x].ptlen, tests[x].pt, tests[x].ptlen, "CCM decrypt data", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + if (y == 0) { + /* check if decryption with the wrong tag does not reveal the plaintext */ + XMEMCPY(tag3, tests[x].tag, tests[x].taglen); + tag3[0] ^= 0xff; /* set the tag to the wrong value */ + taglen = tests[x].taglen; + if ((err = ccm_memory(idx, + tests[x].key, 16, + NULL, + tests[x].nonce, tests[x].noncelen, + tests[x].header, tests[x].headerlen, + buf2, tests[x].ptlen, + buf, + tag3, &taglen, 1 )) != CRYPT_ERROR) { + return CRYPT_FAIL_TESTVECTOR; + } + if (compare_testvector(buf2, tests[x].ptlen, zero, tests[x].ptlen, "CCM decrypt wrong tag", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } else { + if (compare_testvector(tag2, taglen, tests[x].tag, tests[x].taglen, "CCM decrypt tag", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + if (y == 0) { + cipher_descriptor[idx]->done(&skey); + } + } + } + + /* wycheproof failing test - https://github.com/libtom/libtomcrypt/pull/452 */ + { + unsigned char key[] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f }; + unsigned char iv[] = { 0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51 }; + unsigned char valid_tag[] = { 0x23,0x1a,0x2d,0x8f }; + unsigned char invalid_tag[] = { 0x23,0x1a,0x2d,0x8f,0x6a }; + unsigned char msg[] = { 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f }; + unsigned char ct[] = { 0xd3,0xda,0xb1,0xee,0x49,0x4c,0xc2,0x29,0x09,0x9d,0x6c,0xac,0x7d,0xf1,0x4a,0xdd }; + unsigned char pt[20] = { 0 }; + + /* VALID tag */ + taglen = sizeof(valid_tag); + err = ccm_memory(idx, key, sizeof(key), NULL, iv, sizeof(iv), NULL, 0, + pt, sizeof(ct), ct, valid_tag, &taglen, CCM_DECRYPT); + if ((err != CRYPT_OK) || (XMEMCMP(msg, pt, sizeof(msg)) != 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* INVALID tag */ + taglen = sizeof(invalid_tag); + err = ccm_memory(idx, key, sizeof(key), NULL, iv, sizeof(iv), NULL, 0, + pt, sizeof(ct), ct, invalid_tag, &taglen, CCM_DECRYPT); + if (err == CRYPT_OK) { + return CRYPT_FAIL_TESTVECTOR; /* should fail */ + } + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ccm/sub.mk b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/sub.mk new file mode 100644 index 0000000..8f43060 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ccm/sub.mk @@ -0,0 +1,6 @@ +srcs-y += ccm_init.c +srcs-y += ccm_add_nonce.c +srcs-y += ccm_add_aad.c +srcs-y += ccm_process.c +srcs-y += ccm_done.c +srcs-y += ccm_reset.c diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_add_aad.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_add_aad.c new file mode 100644 index 0000000..dc6b5bb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_add_aad.c @@ -0,0 +1,28 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Add AAD to the ChaCha20Poly1305 state + @param st The ChaCha20Poly1305 state + @param in The additional authentication data to add to the ChaCha20Poly1305 state + @param inlen The length of the ChaCha20Poly1305 data. + @return CRYPT_OK on success + */ +int chacha20poly1305_add_aad(chacha20poly1305_state *st, const unsigned char *in, unsigned long inlen) +{ + int err; + + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(st != NULL); + + if (st->aadflg == 0) return CRYPT_ERROR; + if ((err = poly1305_process(&st->poly, in, inlen)) != CRYPT_OK) return err; + st->aadlen += (ulong64)inlen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_decrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_decrypt.c new file mode 100644 index 0000000..e51322a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_decrypt.c @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Decrypt bytes of ciphertext with ChaCha20Poly1305 + @param st The ChaCha20Poly1305 state + @param in The ciphertext + @param inlen The length of the input (octets) + @param out [out] The plaintext (length inlen) + @return CRYPT_OK if successful +*/ +int chacha20poly1305_decrypt(chacha20poly1305_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + unsigned char padzero[16] = { 0 }; + unsigned long padlen; + int err; + + LTC_ARGCHK(st != NULL); + + if (st->aadflg) { + padlen = 16 - (unsigned long)(st->aadlen % 16); + if (padlen < 16) { + if ((err = poly1305_process(&st->poly, padzero, padlen)) != CRYPT_OK) return err; + } + st->aadflg = 0; /* no more AAD */ + } + if (st->aadflg) st->aadflg = 0; /* no more AAD */ + if ((err = poly1305_process(&st->poly, in, inlen)) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st->chacha, in, inlen, out)) != CRYPT_OK) return err; + st->ctlen += (ulong64)inlen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_done.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_done.c new file mode 100644 index 0000000..19d1640 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_done.c @@ -0,0 +1,36 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Terminate a ChaCha20Poly1305 stream + @param st The ChaCha20Poly1305 state + @param tag [out] The destination for the MAC tag + @param taglen [in/out] The length of the MAC tag + @return CRYPT_OK on success + */ +int chacha20poly1305_done(chacha20poly1305_state *st, unsigned char *tag, unsigned long *taglen) +{ + unsigned char padzero[16] = { 0 }; + unsigned long padlen; + unsigned char buf[16]; + int err; + + LTC_ARGCHK(st != NULL); + + padlen = 16 - (unsigned long)(st->ctlen % 16); + if (padlen < 16) { + if ((err = poly1305_process(&st->poly, padzero, padlen)) != CRYPT_OK) return err; + } + STORE64L(st->aadlen, buf); + STORE64L(st->ctlen, buf + 8); + if ((err = poly1305_process(&st->poly, buf, 16)) != CRYPT_OK) return err; + if ((err = poly1305_done(&st->poly, tag, taglen)) != CRYPT_OK) return err; + if ((err = chacha_done(&st->chacha)) != CRYPT_OK) return err; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_encrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_encrypt.c new file mode 100644 index 0000000..2a4208e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_encrypt.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Encrypt bytes of ciphertext with ChaCha20Poly1305 + @param st The ChaCha20Poly1305 state + @param in The plaintext + @param inlen The length of the input (octets) + @param out [out] The ciphertext (length inlen) + @return CRYPT_OK if successful +*/ +int chacha20poly1305_encrypt(chacha20poly1305_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + unsigned char padzero[16] = { 0 }; + unsigned long padlen; + int err; + + LTC_ARGCHK(st != NULL); + + if ((err = chacha_crypt(&st->chacha, in, inlen, out)) != CRYPT_OK) return err; + if (st->aadflg) { + padlen = 16 - (unsigned long)(st->aadlen % 16); + if (padlen < 16) { + if ((err = poly1305_process(&st->poly, padzero, padlen)) != CRYPT_OK) return err; + } + st->aadflg = 0; /* no more AAD */ + } + if ((err = poly1305_process(&st->poly, out, inlen)) != CRYPT_OK) return err; + st->ctlen += (ulong64)inlen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_init.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_init.c new file mode 100644 index 0000000..2654f4d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_init.c @@ -0,0 +1,20 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Initialize an ChaCha20Poly1305 context (only the key) + @param st [out] The destination of the ChaCha20Poly1305 state + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int chacha20poly1305_init(chacha20poly1305_state *st, const unsigned char *key, unsigned long keylen) +{ + return chacha_setup(&st->chacha, key, keylen, 20); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_memory.c new file mode 100644 index 0000000..6e3bf24 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_memory.c @@ -0,0 +1,72 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Process an entire GCM packet in one call. + @param key The secret key + @param keylen The length of the secret key + @param iv The initialization vector + @param ivlen The length of the initialization vector + @param aad The additional authentication data (header) + @param aadlen The length of the aad + @param in The plaintext + @param inlen The length of the plaintext (ciphertext length is the same) + @param out The ciphertext + @param tag [out] The MAC tag + @param taglen [in/out] The MAC tag length + @param direction Encrypt or Decrypt mode (CHACHA20POLY1305_ENCRYPT or CHACHA20POLY1305_DECRYPT) + @return CRYPT_OK on success + */ +int chacha20poly1305_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *aad, unsigned long aadlen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, + unsigned char *tag, unsigned long *taglen, + int direction) +{ + chacha20poly1305_state st; + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(iv != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + + if ((err = chacha20poly1305_init(&st, key, keylen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = chacha20poly1305_setiv(&st, iv, ivlen)) != CRYPT_OK) { goto LBL_ERR; } + if (aad && aadlen > 0) { + if ((err = chacha20poly1305_add_aad(&st, aad, aadlen)) != CRYPT_OK) { goto LBL_ERR; } + } + if (direction == CHACHA20POLY1305_ENCRYPT) { + if ((err = chacha20poly1305_encrypt(&st, in, inlen, out)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = chacha20poly1305_done(&st, tag, taglen)) != CRYPT_OK) { goto LBL_ERR; } + } + else if (direction == CHACHA20POLY1305_DECRYPT) { + unsigned char buf[MAXBLOCKSIZE]; + unsigned long buflen = sizeof(buf); + if ((err = chacha20poly1305_decrypt(&st, in, inlen, out)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = chacha20poly1305_done(&st, buf, &buflen)) != CRYPT_OK) { goto LBL_ERR; } + if (buflen != *taglen || XMEM_NEQ(buf, tag, buflen) != 0) { + err = CRYPT_ERROR; + goto LBL_ERR; + } + } + else { + err = CRYPT_INVALID_ARG; + goto LBL_ERR; + } +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(chacha20poly1305_state)); +#endif + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_setiv.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_setiv.c new file mode 100644 index 0000000..d5ec3e6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_setiv.c @@ -0,0 +1,58 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Set IV + counter data to the ChaCha20Poly1305 state and reset the context + @param st The ChaCha20Poly1305 state + @param iv The IV data to add + @param ivlen The length of the IV (must be 12 or 8) + @return CRYPT_OK on success + */ +int chacha20poly1305_setiv(chacha20poly1305_state *st, const unsigned char *iv, unsigned long ivlen) +{ + chacha_state tmp_st; + int i, err; + unsigned char polykey[32]; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(iv != NULL); + LTC_ARGCHK(ivlen == 12 || ivlen == 8); + + /* set IV for chacha20 */ + if (ivlen == 12) { + /* IV 96bit */ + if ((err = chacha_ivctr32(&st->chacha, iv, ivlen, 1)) != CRYPT_OK) return err; + } + else { + /* IV 64bit */ + if ((err = chacha_ivctr64(&st->chacha, iv, ivlen, 1)) != CRYPT_OK) return err; + } + + /* copy chacha20 key to temporary state */ + for(i = 0; i < 12; i++) tmp_st.input[i] = st->chacha.input[i]; + tmp_st.rounds = 20; + /* set IV */ + if (ivlen == 12) { + /* IV 32bit */ + if ((err = chacha_ivctr32(&tmp_st, iv, ivlen, 0)) != CRYPT_OK) return err; + } + else { + /* IV 64bit */ + if ((err = chacha_ivctr64(&tmp_st, iv, ivlen, 0)) != CRYPT_OK) return err; + } + /* (re)generate new poly1305 key */ + if ((err = chacha_keystream(&tmp_st, polykey, 32)) != CRYPT_OK) return err; + /* (re)initialise poly1305 */ + if ((err = poly1305_init(&st->poly, polykey, 32)) != CRYPT_OK) return err; + st->ctlen = 0; + st->aadlen = 0; + st->aadflg = 1; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_setiv_rfc7905.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_setiv_rfc7905.c new file mode 100644 index 0000000..75da634 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_setiv_rfc7905.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +/** + Set IV + counter data (with RFC7905-magic) to the ChaCha20Poly1305 state and reset the context + @param st The ChaCha20Poly1305 state + @param iv The IV data to add + @param ivlen The length of the IV (must be 12 or 8) + @param sequence_number 64bit sequence number which is incorporated into IV as described in RFC7905 + @return CRYPT_OK on success + */ +int chacha20poly1305_setiv_rfc7905(chacha20poly1305_state *st, const unsigned char *iv, unsigned long ivlen, ulong64 sequence_number) +{ + int i; + unsigned char combined_iv[12] = { 0 }; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(iv != NULL); + LTC_ARGCHK(ivlen == 12); + + STORE64L(sequence_number, combined_iv + 4); + for (i = 0; i < 12; i++) combined_iv[i] = iv[i] ^ combined_iv[i]; + return chacha20poly1305_setiv(st, combined_iv, 12); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_test.c b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_test.c new file mode 100644 index 0000000..30d4973 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/chachapoly/chacha20poly1305_test.c @@ -0,0 +1,159 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20POLY1305_MODE + +int chacha20poly1305_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + chacha20poly1305_state st1, st2; + unsigned char k[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + unsigned char i12[] = { 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47 }; + unsigned char i8[] = { 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43 }; + unsigned char aad[] = { 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7 }; + unsigned char enc[] = { 0xD3, 0x1A, 0x8D, 0x34, 0x64, 0x8E, 0x60, 0xDB, 0x7B, 0x86, 0xAF, 0xBC, 0x53, 0xEF, 0x7E, 0xC2, + 0xA4, 0xAD, 0xED, 0x51, 0x29, 0x6E, 0x08, 0xFE, 0xA9, 0xE2, 0xB5, 0xA7, 0x36, 0xEE, 0x62, 0xD6, + 0x3D, 0xBE, 0xA4, 0x5E, 0x8C, 0xA9, 0x67, 0x12, 0x82, 0xFA, 0xFB, 0x69, 0xDA, 0x92, 0x72, 0x8B, + 0x1A, 0x71, 0xDE, 0x0A, 0x9E, 0x06, 0x0B, 0x29, 0x05, 0xD6, 0xA5, 0xB6, 0x7E, 0xCD, 0x3B, 0x36, + 0x92, 0xDD, 0xBD, 0x7F, 0x2D, 0x77, 0x8B, 0x8C, 0x98, 0x03, 0xAE, 0xE3, 0x28, 0x09, 0x1B, 0x58, + 0xFA, 0xB3, 0x24, 0xE4, 0xFA, 0xD6, 0x75, 0x94, 0x55, 0x85, 0x80, 0x8B, 0x48, 0x31, 0xD7, 0xBC, + 0x3F, 0xF4, 0xDE, 0xF0, 0x8E, 0x4B, 0x7A, 0x9D, 0xE5, 0x76, 0xD2, 0x65, 0x86, 0xCE, 0xC6, 0x4B, + 0x61, 0x16 }; + unsigned char tag[] = { 0x1A, 0xE1, 0x0B, 0x59, 0x4F, 0x09, 0xE2, 0x6A, 0x7E, 0x90, 0x2E, 0xCB, 0xD0, 0x60, 0x06, 0x91 }; + char m[] = "Ladies and Gentlemen of the class of '99: If I could offer you only one tip for the future, sunscreen would be it."; + unsigned long mlen = XSTRLEN(m); + unsigned long len; + unsigned char rfc7905_pt[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }; + unsigned char rfc7905_enc[] = { 0xE4, 0x62, 0x85, 0xB4, 0x29, 0x95, 0x34, 0x96, 0xAB, 0xFB, 0x67, 0xCD, 0xAE, 0xAC, 0x94, 0x1E }; + unsigned char rfc7905_tag[] = { 0x16, 0x2C, 0x92, 0x48, 0x2A, 0xDB, 0xD3, 0x5D, 0x48, 0xBE, 0xC6, 0xFF, 0x10, 0x9C, 0xBA, 0xE4 }; + unsigned char ct[1000], pt[1000], emac[16], dmac[16]; + int err; + + /* encrypt IV 96bit */ + if ((err = chacha20poly1305_init(&st1, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_setiv(&st1, i12, sizeof(i12))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_add_aad(&st1, aad, sizeof(aad))) != CRYPT_OK) return err; + /* encrypt piece by piece */ + if ((err = chacha20poly1305_encrypt(&st1, (unsigned char *)m, 25, ct)) != CRYPT_OK) return err; + if ((err = chacha20poly1305_encrypt(&st1, (unsigned char *)m + 25, 10, ct + 25)) != CRYPT_OK) return err; + if ((err = chacha20poly1305_encrypt(&st1, (unsigned char *)m + 35, 35, ct + 35)) != CRYPT_OK) return err; + if ((err = chacha20poly1305_encrypt(&st1, (unsigned char *)m + 70, 5, ct + 70)) != CRYPT_OK) return err; + if ((err = chacha20poly1305_encrypt(&st1, (unsigned char *)m + 75, 5, ct + 75)) != CRYPT_OK) return err; + if ((err = chacha20poly1305_encrypt(&st1, (unsigned char *)m + 80, mlen - 80, ct + 80)) != CRYPT_OK) return err; + len = sizeof(emac); + if ((err = chacha20poly1305_done(&st1, emac, &len)) != CRYPT_OK) return err; + + if (compare_testvector(ct, mlen, enc, sizeof(enc), "ENC-CT", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(emac, len, tag, sizeof(tag), "ENC-TAG", 2) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* decrypt IV 96bit */ + if ((err = chacha20poly1305_init(&st2, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_setiv(&st2, i12, sizeof(i12))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_add_aad(&st2, aad, sizeof(aad))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_decrypt(&st2, ct, 21, pt)) != CRYPT_OK) return err; + if ((err = chacha20poly1305_decrypt(&st2, ct + 21, mlen - 21, pt + 21)) != CRYPT_OK) return err; + len = sizeof(dmac); + if ((err = chacha20poly1305_done(&st2, dmac, &len)) != CRYPT_OK) return err; + + if (compare_testvector(pt, mlen, m, mlen, "DEC-PT", 3) != 0) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(dmac, len, tag, sizeof(tag), "DEC-TAG", 4) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* chacha20poly1305_memory - encrypt */ + len = sizeof(emac); + if ((err = chacha20poly1305_memory(k, sizeof(k), i12, sizeof(i12), aad, sizeof(aad), (unsigned char *)m, + mlen, ct, emac, &len, CHACHA20POLY1305_ENCRYPT)) != CRYPT_OK) return err; + if (compare_testvector(ct, mlen, enc, sizeof(enc), "ENC-CT2", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(emac, len, tag, sizeof(tag), "ENC-TAG2", 2) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* chacha20poly1305_memory - decrypt */ + len = sizeof(dmac); + XMEMCPY(dmac, tag, sizeof(tag)); + if ((err = chacha20poly1305_memory(k, sizeof(k), i12, sizeof(i12), aad, sizeof(aad), + ct, mlen, pt, dmac, &len, CHACHA20POLY1305_DECRYPT)) != CRYPT_OK) return err; + if (compare_testvector(pt, mlen, m, mlen, "DEC-PT2", 3) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* encrypt - rfc7905 */ + if ((err = chacha20poly1305_init(&st1, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_setiv_rfc7905(&st1, i12, sizeof(i12), CONST64(0x1122334455667788))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_add_aad(&st1, aad, sizeof(aad))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_encrypt(&st1, rfc7905_pt, 16, ct)) != CRYPT_OK) return err; + len = sizeof(emac); + if ((err = chacha20poly1305_done(&st1, emac, &len)) != CRYPT_OK) return err; + + if (compare_testvector(ct, 16, rfc7905_enc, 16, "ENC-CT3", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(emac, len, rfc7905_tag, 16, "ENC-TAG3", 2) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* decrypt - rfc7905 */ + if ((err = chacha20poly1305_init(&st1, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_setiv_rfc7905(&st1, i12, sizeof(i12), CONST64(0x1122334455667788))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_add_aad(&st1, aad, sizeof(aad))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_decrypt(&st1, ct, 16, pt)) != CRYPT_OK) return err; + len = sizeof(dmac); + if ((err = chacha20poly1305_done(&st1, dmac, &len)) != CRYPT_OK) return err; + + if (compare_testvector(pt, 16, rfc7905_pt, 16, "DEC-CT3", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(dmac, len, rfc7905_tag, 16, "DEC-TAG3", 2) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* encrypt IV 64bit */ + if ((err = chacha20poly1305_init(&st1, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_setiv(&st1, i8, sizeof(i8))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_add_aad(&st1, aad, sizeof(aad))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_encrypt(&st1, (unsigned char *)m, mlen, ct)) != CRYPT_OK) return err; + len = sizeof(emac); + if ((err = chacha20poly1305_done(&st1, emac, &len)) != CRYPT_OK) return err; + + /* decrypt IV 64bit */ + if ((err = chacha20poly1305_init(&st2, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_setiv(&st2, i8, sizeof(i8))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_add_aad(&st2, aad, sizeof(aad))) != CRYPT_OK) return err; + if ((err = chacha20poly1305_decrypt(&st2, ct, mlen, pt)) != CRYPT_OK) return err; + len = sizeof(dmac); + if ((err = chacha20poly1305_done(&st2, dmac, &len)) != CRYPT_OK) return err; + + if (compare_testvector(pt, mlen, m, mlen, "DEC-PT4", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(dmac, len, emac, len, "DEC-TAG4", 2) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* wycheproof failing test - https://github.com/libtom/libtomcrypt/pull/451 */ + { + unsigned char key[] = { 0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0xaa,0xbb,0xcc,0xdd,0xee,0xff, + 0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0xaa,0xbb,0xcc,0xdd,0xee,0xff }; + unsigned char iv[] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b }; + unsigned char valid_tag[] = { 0xa3,0xe3,0xfd,0xf9,0xfb,0xa6,0x86,0x1b,0x5a,0xd2,0x60,0x7f,0x40,0xb7,0xf4,0x47 }; + unsigned char invalid_tag[] = { 0xa2,0xe3,0xfd,0xf9,0xfb,0xa6,0x86,0x1b,0x5a,0xd2,0x60,0x7f,0x40,0xb7,0xf4,0x47 }; + unsigned char waad[] = { 0x61,0x61,0x64 }; + unsigned char wct[] = { 0x00 }; + unsigned char wpt[20] = { 0 }; + unsigned char wtag[20] = { 0 }; + unsigned long taglen; + + /* encrypt */ + taglen = sizeof(wtag); + err = chacha20poly1305_memory(key, sizeof(key), iv, sizeof(iv), waad, sizeof(waad), + wpt, 0, wct, wtag, &taglen, CHACHA20POLY1305_ENCRYPT); + if (err != CRYPT_OK) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(wtag, taglen, valid_tag, sizeof(valid_tag), "WYCH", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + + /* VALID tag */ + taglen = sizeof(valid_tag); + err = chacha20poly1305_memory(key, sizeof(key), iv, sizeof(iv), waad, sizeof(waad), + wpt, 0, wct, valid_tag, &taglen, CHACHA20POLY1305_DECRYPT); + if (err != CRYPT_OK) return CRYPT_FAIL_TESTVECTOR; + + /* INVALID tag */ + taglen = sizeof(invalid_tag); + err = chacha20poly1305_memory(key, sizeof(key), iv, sizeof(iv), waad, sizeof(waad), + wpt, 0, wct, invalid_tag, &taglen, CHACHA20POLY1305_DECRYPT); + if (err == CRYPT_OK) { + return CRYPT_FAIL_TESTVECTOR; /* should fail */ + } + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_addheader.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_addheader.c new file mode 100644 index 0000000..a6b8f76 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_addheader.c @@ -0,0 +1,26 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/** + @file eax_addheader.c + EAX implementation, add meta-data, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + add header (metadata) to the stream + @param eax The current EAX state + @param header The header (meta-data) data you wish to add to the state + @param length The length of the header data + @return CRYPT_OK if successful +*/ +int eax_addheader(eax_state *eax, const unsigned char *header, + unsigned long length) +{ + LTC_ARGCHK(eax != NULL); + LTC_ARGCHK(header != NULL); + return omac_process(&eax->headeromac, header, length); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_decrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_decrypt.c new file mode 100644 index 0000000..cc03c5c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_decrypt.c @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file eax_decrypt.c + EAX implementation, decrypt block, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + Decrypt data with the EAX protocol + @param eax The EAX state + @param ct The ciphertext + @param pt [out] The plaintext + @param length The length (octets) of the ciphertext + @return CRYPT_OK if successful +*/ +int eax_decrypt(eax_state *eax, const unsigned char *ct, unsigned char *pt, + unsigned long length) +{ + int err; + + LTC_ARGCHK(eax != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + /* omac ciphertext */ + if ((err = omac_process(&eax->ctomac, ct, length)) != CRYPT_OK) { + return err; + } + + /* decrypt */ + return ctr_decrypt(ct, pt, length, &eax->ctr); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_decrypt_verify_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_decrypt_verify_memory.c new file mode 100644 index 0000000..5429d4a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_decrypt_verify_memory.c @@ -0,0 +1,99 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file eax_decrypt_verify_memory.c + EAX implementation, decrypt block of memory, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + Decrypt a block of memory and verify the provided MAC tag with EAX + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the key (octets) + @param nonce The nonce data (use once) for the session + @param noncelen The length of the nonce data. + @param header The session header data + @param headerlen The length of the header (octets) + @param ct The ciphertext + @param ctlen The length of the ciphertext (octets) + @param pt [out] The plaintext + @param tag The authentication tag provided by the encoder + @param taglen [in/out] The length of the tag (octets) + @param stat [out] The result of the decryption (1==valid tag, 0==invalid) + @return CRYPT_OK if successful regardless of the resulting tag comparison +*/ +int eax_decrypt_verify_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, + int *stat) +{ + int err; + eax_state *eax; + unsigned char *buf; + unsigned long buflen; + + LTC_ARGCHK(stat != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + + /* default to zero */ + *stat = 0; + + /* limit taglen */ + taglen = MIN(taglen, MAXBLOCKSIZE); + + /* allocate ram */ + buf = XMALLOC(taglen); + eax = XMALLOC(sizeof(*eax)); + if (eax == NULL || buf == NULL) { + if (eax != NULL) { + XFREE(eax); + } + if (buf != NULL) { + XFREE(buf); + } + return CRYPT_MEM; + } + + if ((err = eax_init(eax, cipher, key, keylen, nonce, noncelen, header, headerlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = eax_decrypt(eax, ct, pt, ctlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + buflen = taglen; + if ((err = eax_done(eax, buf, &buflen)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* compare tags */ + if (buflen >= taglen && XMEM_NEQ(buf, tag, taglen) == 0) { + *stat = 1; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(buf, taglen); + zeromem(eax, sizeof(*eax)); +#endif + + XFREE(eax); + XFREE(buf); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_done.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_done.c new file mode 100644 index 0000000..f2362cf --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_done.c @@ -0,0 +1,82 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file eax_done.c + EAX implementation, terminate session, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + Terminate an EAX session and get the tag. + @param eax The EAX state + @param tag [out] The destination of the authentication tag + @param taglen [in/out] The max length and resulting length of the authentication tag + @return CRYPT_OK if successful +*/ +int eax_done(eax_state *eax, unsigned char *tag, unsigned long *taglen) +{ + int err; + unsigned char *headermac, *ctmac; + unsigned long x, len; + + LTC_ARGCHK(eax != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + + /* allocate ram */ + headermac = XMALLOC(MAXBLOCKSIZE); + ctmac = XMALLOC(MAXBLOCKSIZE); + + if (headermac == NULL || ctmac == NULL) { + if (headermac != NULL) { + XFREE(headermac); + } + if (ctmac != NULL) { + XFREE(ctmac); + } + return CRYPT_MEM; + } + + /* finish ctomac */ + len = MAXBLOCKSIZE; + if ((err = omac_done(&eax->ctomac, ctmac, &len)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* finish headeromac */ + + /* note we specifically don't reset len so the two lens are minimal */ + + if ((err = omac_done(&eax->headeromac, headermac, &len)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* terminate the CTR chain */ + if ((err = ctr_done(&eax->ctr)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* compute N xor H xor C */ + for (x = 0; x < len && x < *taglen; x++) { + tag[x] = eax->N[x] ^ headermac[x] ^ ctmac[x]; + } + *taglen = x; + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(ctmac, MAXBLOCKSIZE); + zeromem(headermac, MAXBLOCKSIZE); + zeromem(eax, sizeof(*eax)); +#endif + + XFREE(ctmac); + XFREE(headermac); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_encrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_encrypt.c new file mode 100644 index 0000000..dba3239 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_encrypt.c @@ -0,0 +1,39 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file eax_encrypt.c + EAX implementation, encrypt block by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + Encrypt with EAX a block of data. + @param eax The EAX state + @param pt The plaintext to encrypt + @param ct [out] The ciphertext as encrypted + @param length The length of the plaintext (octets) + @return CRYPT_OK if successful +*/ +int eax_encrypt(eax_state *eax, const unsigned char *pt, unsigned char *ct, + unsigned long length) +{ + int err; + + LTC_ARGCHK(eax != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + /* encrypt */ + if ((err = ctr_encrypt(pt, ct, length, &eax->ctr)) != CRYPT_OK) { + return err; + } + + /* omac ciphertext */ + return omac_process(&eax->ctomac, ct, length); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_encrypt_authenticate_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_encrypt_authenticate_memory.c new file mode 100644 index 0000000..76aeec2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_encrypt_authenticate_memory.c @@ -0,0 +1,70 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file eax_encrypt_authenticate_memory.c + EAX implementation, encrypt a block of memory, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + EAX encrypt and produce an authentication tag + @param cipher The index of the cipher desired + @param key The secret key to use + @param keylen The length of the secret key (octets) + @param nonce The session nonce [use once] + @param noncelen The length of the nonce + @param header The header for the session + @param headerlen The length of the header (octets) + @param pt The plaintext + @param ptlen The length of the plaintext (octets) + @param ct [out] The ciphertext + @param tag [out] The destination tag + @param taglen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful +*/ +int eax_encrypt_authenticate_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen, + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen) +{ + int err; + eax_state *eax; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + + eax = XMALLOC(sizeof(*eax)); + + if ((err = eax_init(eax, cipher, key, keylen, nonce, noncelen, header, headerlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = eax_encrypt(eax, pt, ct, ptlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = eax_done(eax, tag, taglen)) != CRYPT_OK) { + goto LBL_ERR; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(eax, sizeof(*eax)); +#endif + + XFREE(eax); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_init.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_init.c new file mode 100644 index 0000000..6a74c3c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_init.c @@ -0,0 +1,132 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file eax_init.c + EAX implementation, initialized EAX state, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + Initialized an EAX state + @param eax [out] The EAX state to initialize + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @param nonce The use-once nonce for the session + @param noncelen The length of the nonce (octets) + @param header The header for the EAX state + @param headerlen The header length (octets) + @return CRYPT_OK if successful +*/ +int eax_init(eax_state *eax, int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen) +{ + unsigned char *buf; + int err, blklen; + omac_state *omac; + unsigned long len; + + + LTC_ARGCHK(eax != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(nonce != NULL); + if (headerlen > 0) { + LTC_ARGCHK(header != NULL); + } + + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + blklen = cipher_descriptor[cipher]->block_length; + + /* allocate ram */ + buf = XMALLOC(MAXBLOCKSIZE); + omac = XMALLOC(sizeof(*omac)); + + if (buf == NULL || omac == NULL) { + if (buf != NULL) { + XFREE(buf); + } + if (omac != NULL) { + XFREE(omac); + } + return CRYPT_MEM; + } + + /* N = LTC_OMAC_0K(nonce) */ + zeromem(buf, MAXBLOCKSIZE); + if ((err = omac_init(omac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* omac the [0]_n */ + if ((err = omac_process(omac, buf, blklen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* omac the nonce */ + if ((err = omac_process(omac, nonce, noncelen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* store result */ + len = sizeof(eax->N); + if ((err = omac_done(omac, eax->N, &len)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* H = LTC_OMAC_1K(header) */ + zeromem(buf, MAXBLOCKSIZE); + buf[blklen - 1] = 1; + + if ((err = omac_init(&eax->headeromac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* omac the [1]_n */ + if ((err = omac_process(&eax->headeromac, buf, blklen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* omac the header */ + if (headerlen != 0) { + if ((err = omac_process(&eax->headeromac, header, headerlen)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + /* note we don't finish the headeromac, this allows us to add more header later */ + + /* setup the CTR mode */ + if ((err = ctr_start(cipher, eax->N, key, keylen, 0, CTR_COUNTER_BIG_ENDIAN, &eax->ctr)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* setup the LTC_OMAC for the ciphertext */ + if ((err = omac_init(&eax->ctomac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* omac [2]_n */ + zeromem(buf, MAXBLOCKSIZE); + buf[blklen-1] = 2; + if ((err = omac_process(&eax->ctomac, buf, blklen)) != CRYPT_OK) { + goto LBL_ERR; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(buf, MAXBLOCKSIZE); + zeromem(omac, sizeof(*omac)); +#endif + + XFREE(omac); + XFREE(buf); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_test.c b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_test.c new file mode 100644 index 0000000..c613e0d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/eax/eax_test.c @@ -0,0 +1,249 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file eax_test.c + EAX implementation, self-test, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_EAX_MODE + +/** + Test the EAX implementation + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled +*/ +int eax_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + int keylen, + noncelen, + headerlen, + msglen; + + unsigned char key[MAXBLOCKSIZE], + nonce[MAXBLOCKSIZE], + header[MAXBLOCKSIZE], + plaintext[MAXBLOCKSIZE], + ciphertext[MAXBLOCKSIZE], + tag[MAXBLOCKSIZE]; + } tests[] = { + +/* NULL message */ +{ + 16, 0, 0, 0, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0 }, + /* header */ + { 0 }, + /* plaintext */ + { 0 }, + /* ciphertext */ + { 0 }, + /* tag */ + { 0x9a, 0xd0, 0x7e, 0x7d, 0xbf, 0xf3, 0x01, 0xf5, + 0x05, 0xde, 0x59, 0x6b, 0x96, 0x15, 0xdf, 0xff } +}, + +/* test with nonce */ +{ + 16, 16, 0, 0, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* header */ + { 0 }, + /* plaintext */ + { 0 }, + /* ciphertext */ + { 0 }, + /* tag */ + { 0x1c, 0xe1, 0x0d, 0x3e, 0xff, 0xd4, 0xca, 0xdb, + 0xe2, 0xe4, 0x4b, 0x58, 0xd6, 0x0a, 0xb9, 0xec } +}, + +/* test with header [no nonce] */ +{ + 16, 0, 16, 0, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0 }, + /* header */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* plaintext */ + { 0 }, + /* ciphertext */ + { 0 }, + /* tag */ + { 0x3a, 0x69, 0x8f, 0x7a, 0x27, 0x0e, 0x51, 0xb0, + 0xf6, 0x5b, 0x3d, 0x3e, 0x47, 0x19, 0x3c, 0xff } +}, + +/* test with header + nonce + plaintext */ +{ + 16, 16, 16, 32, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* header */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* plaintext */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + /* ciphertext */ + { 0x29, 0xd8, 0x78, 0xd1, 0xa3, 0xbe, 0x85, 0x7b, + 0x6f, 0xb8, 0xc8, 0xea, 0x59, 0x50, 0xa7, 0x78, + 0x33, 0x1f, 0xbf, 0x2c, 0xcf, 0x33, 0x98, 0x6f, + 0x35, 0xe8, 0xcf, 0x12, 0x1d, 0xcb, 0x30, 0xbc }, + /* tag */ + { 0x4f, 0xbe, 0x03, 0x38, 0xbe, 0x1c, 0x8c, 0x7e, + 0x1d, 0x7a, 0xe7, 0xe4, 0x5b, 0x92, 0xc5, 0x87 } +}, + +/* test with header + nonce + plaintext [not even sizes!] */ +{ + 16, 15, 14, 29, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e }, + /* header */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d }, + /* plaintext */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c }, + /* ciphertext */ + { 0xdd, 0x25, 0xc7, 0x54, 0xc5, 0xb1, 0x7c, 0x59, + 0x28, 0xb6, 0x9b, 0x73, 0x15, 0x5f, 0x7b, 0xb8, + 0x88, 0x8f, 0xaf, 0x37, 0x09, 0x1a, 0xd9, 0x2c, + 0x8a, 0x24, 0xdb, 0x86, 0x8b }, + /* tag */ + { 0x0d, 0x1a, 0x14, 0xe5, 0x22, 0x24, 0xff, 0xd2, + 0x3a, 0x05, 0xfa, 0x02, 0xcd, 0xef, 0x52, 0xda } +}, + +/* Vectors from Brian Gladman */ + +{ + 16, 16, 8, 0, + /* key */ + { 0x23, 0x39, 0x52, 0xde, 0xe4, 0xd5, 0xed, 0x5f, + 0x9b, 0x9c, 0x6d, 0x6f, 0xf8, 0x0f, 0xf4, 0x78 }, + /* nonce */ + { 0x62, 0xec, 0x67, 0xf9, 0xc3, 0xa4, 0xa4, 0x07, + 0xfc, 0xb2, 0xa8, 0xc4, 0x90, 0x31, 0xa8, 0xb3 }, + /* header */ + { 0x6b, 0xfb, 0x91, 0x4f, 0xd0, 0x7e, 0xae, 0x6b }, + /* PT */ + { 0x00 }, + /* CT */ + { 0x00 }, + /* tag */ + { 0xe0, 0x37, 0x83, 0x0e, 0x83, 0x89, 0xf2, 0x7b, + 0x02, 0x5a, 0x2d, 0x65, 0x27, 0xe7, 0x9d, 0x01 } +}, + +{ + 16, 16, 8, 2, + /* key */ + { 0x91, 0x94, 0x5d, 0x3f, 0x4d, 0xcb, 0xee, 0x0b, + 0xf4, 0x5e, 0xf5, 0x22, 0x55, 0xf0, 0x95, 0xa4 }, + /* nonce */ + { 0xbe, 0xca, 0xf0, 0x43, 0xb0, 0xa2, 0x3d, 0x84, + 0x31, 0x94, 0xba, 0x97, 0x2c, 0x66, 0xde, 0xbd }, + /* header */ + { 0xfa, 0x3b, 0xfd, 0x48, 0x06, 0xeb, 0x53, 0xfa }, + /* PT */ + { 0xf7, 0xfb }, + /* CT */ + { 0x19, 0xdd }, + /* tag */ + { 0x5c, 0x4c, 0x93, 0x31, 0x04, 0x9d, 0x0b, 0xda, + 0xb0, 0x27, 0x74, 0x08, 0xf6, 0x79, 0x67, 0xe5 } +}, + +{ + 16, 16, 8, 5, + /* key */ + { 0x01, 0xf7, 0x4a, 0xd6, 0x40, 0x77, 0xf2, 0xe7, + 0x04, 0xc0, 0xf6, 0x0a, 0xda, 0x3d, 0xd5, 0x23 }, + /* nonce */ + { 0x70, 0xc3, 0xdb, 0x4f, 0x0d, 0x26, 0x36, 0x84, + 0x00, 0xa1, 0x0e, 0xd0, 0x5d, 0x2b, 0xff, 0x5e }, + /* header */ + { 0x23, 0x4a, 0x34, 0x63, 0xc1, 0x26, 0x4a, 0xc6 }, + /* PT */ + { 0x1a, 0x47, 0xcb, 0x49, 0x33 }, + /* CT */ + { 0xd8, 0x51, 0xd5, 0xba, 0xe0 }, + /* Tag */ + { 0x3a, 0x59, 0xf2, 0x38, 0xa2, 0x3e, 0x39, 0x19, + 0x9d, 0xc9, 0x26, 0x66, 0x26, 0xc4, 0x0f, 0x80 } +} + +}; + int err, x, idx, res; + unsigned long len; + unsigned char outct[MAXBLOCKSIZE], outtag[MAXBLOCKSIZE]; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + len = sizeof(outtag); + if ((err = eax_encrypt_authenticate_memory(idx, tests[x].key, tests[x].keylen, + tests[x].nonce, tests[x].noncelen, tests[x].header, tests[x].headerlen, + tests[x].plaintext, tests[x].msglen, outct, outtag, &len)) != CRYPT_OK) { + return err; + } + if (compare_testvector(outtag, len, tests[x].tag, len, "EAX Tag", x) || + compare_testvector(outct, tests[x].msglen, tests[x].ciphertext, tests[x].msglen, "EAX CT", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* test decrypt */ + if ((err = eax_decrypt_verify_memory(idx, tests[x].key, tests[x].keylen, + tests[x].nonce, tests[x].noncelen, tests[x].header, tests[x].headerlen, + outct, tests[x].msglen, outct, outtag, len, &res)) != CRYPT_OK) { + return err; + } + if ((res != 1) || compare_testvector(outct, tests[x].msglen, tests[x].plaintext, tests[x].msglen, "EAX", x)) { +#ifdef LTC_TEST_DBG + printf("\n\nEAX: Failure-decrypt - res = %d\n", res); +#endif + return CRYPT_FAIL_TESTVECTOR; + } + + } + return CRYPT_OK; +#endif /* LTC_TEST */ +} + +#endif /* LTC_EAX_MODE */ diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_add_aad.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_add_aad.c new file mode 100644 index 0000000..5c3793e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_add_aad.c @@ -0,0 +1,114 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_add_aad.c + GCM implementation, Add AAD data to the stream, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Add AAD to the GCM state + @param gcm The GCM state + @param adata The additional authentication data to add to the GCM state + @param adatalen The length of the AAD data. + @return CRYPT_OK on success + */ +int gcm_add_aad(gcm_state *gcm, + const unsigned char *adata, unsigned long adatalen) +{ + unsigned long x; + int err; +#ifdef LTC_FAST + unsigned long y; +#endif + + LTC_ARGCHK(gcm != NULL); + if (adatalen > 0) { + LTC_ARGCHK(adata != NULL); + } + + if (gcm->buflen > 16 || gcm->buflen < 0) { + return CRYPT_INVALID_ARG; + } + + if ((err = cipher_is_valid(gcm->cipher)) != CRYPT_OK) { + return err; + } + + /* in IV mode? */ + if (gcm->mode == LTC_GCM_MODE_IV) { + /* IV length must be > 0 */ + if (gcm->buflen == 0 && gcm->totlen == 0) return CRYPT_ERROR; + /* let's process the IV */ + if (gcm->ivmode || gcm->buflen != 12) { + for (x = 0; x < (unsigned long)gcm->buflen; x++) { + gcm->X[x] ^= gcm->buf[x]; + } + if (gcm->buflen) { + gcm->totlen += gcm->buflen * CONST64(8); + gcm_mult_h(gcm, gcm->X); + } + + /* mix in the length */ + zeromem(gcm->buf, 8); + STORE64H(gcm->totlen, gcm->buf+8); + for (x = 0; x < 16; x++) { + gcm->X[x] ^= gcm->buf[x]; + } + gcm_mult_h(gcm, gcm->X); + + /* copy counter out */ + XMEMCPY(gcm->Y, gcm->X, 16); + zeromem(gcm->X, 16); + } else { + XMEMCPY(gcm->Y, gcm->buf, 12); + gcm->Y[12] = 0; + gcm->Y[13] = 0; + gcm->Y[14] = 0; + gcm->Y[15] = 1; + } + XMEMCPY(gcm->Y_0, gcm->Y, 16); + zeromem(gcm->buf, 16); + gcm->buflen = 0; + gcm->totlen = 0; + gcm->mode = LTC_GCM_MODE_AAD; + } + + if (gcm->mode != LTC_GCM_MODE_AAD || gcm->buflen >= 16) { + return CRYPT_INVALID_ARG; + } + + x = 0; +#ifdef LTC_FAST + if (gcm->buflen == 0 && adatalen > 15) { + for (x = 0; x < (adatalen & ~15); x += 16) { + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&gcm->X[y])) ^= *(LTC_FAST_TYPE_PTR_CAST(&adata[x + y])); + } + gcm_mult_h(gcm, gcm->X); + gcm->totlen += 128; + } + adata += x; + } +#endif + + + /* start adding AAD data to the state */ + for (; x < adatalen; x++) { + gcm->X[gcm->buflen++] ^= *adata++; + + if (gcm->buflen == 16) { + /* GF mult it */ + gcm_mult_h(gcm, gcm->X); + gcm->buflen = 0; + gcm->totlen += 128; + } + } + + return CRYPT_OK; +} +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_add_iv.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_add_iv.c new file mode 100644 index 0000000..33a2444 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_add_iv.c @@ -0,0 +1,82 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_add_iv.c + GCM implementation, add IV data to the state, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Add IV data to the GCM state + @param gcm The GCM state + @param IV The initial value data to add + @param IVlen The length of the IV + @return CRYPT_OK on success + */ +int gcm_add_iv(gcm_state *gcm, + const unsigned char *IV, unsigned long IVlen) +{ + unsigned long x, y; + int err; + + LTC_ARGCHK(gcm != NULL); + if (IVlen > 0) { + LTC_ARGCHK(IV != NULL); + } + + /* must be in IV mode */ + if (gcm->mode != LTC_GCM_MODE_IV) { + return CRYPT_INVALID_ARG; + } + + if (gcm->buflen >= 16 || gcm->buflen < 0) { + return CRYPT_INVALID_ARG; + } + + if ((err = cipher_is_valid(gcm->cipher)) != CRYPT_OK) { + return err; + } + + + /* trip the ivmode flag */ + if (IVlen + gcm->buflen > 12) { + gcm->ivmode |= 1; + } + + x = 0; +#ifdef LTC_FAST + if (gcm->buflen == 0) { + for (x = 0; x < (IVlen & ~15); x += 16) { + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&gcm->X[y])) ^= *(LTC_FAST_TYPE_PTR_CAST(&IV[x + y])); + } + gcm_mult_h(gcm, gcm->X); + gcm->totlen += 128; + } + IV += x; + } +#endif + + /* start adding IV data to the state */ + for (; x < IVlen; x++) { + gcm->buf[gcm->buflen++] = *IV++; + + if (gcm->buflen == 16) { + /* GF mult it */ + for (y = 0; y < 16; y++) { + gcm->X[y] ^= gcm->buf[y]; + } + gcm_mult_h(gcm, gcm->X); + gcm->buflen = 0; + gcm->totlen += 128; + } + } + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_done.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_done.c new file mode 100644 index 0000000..ee16d38 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_done.c @@ -0,0 +1,80 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_done.c + GCM implementation, Terminate the stream, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Terminate a GCM stream + @param gcm The GCM state + @param tag [out] The destination for the MAC tag + @param taglen [in/out] The length of the MAC tag + @return CRYPT_OK on success + */ +int gcm_done(gcm_state *gcm, + unsigned char *tag, unsigned long *taglen) +{ + unsigned long x; + int err; + + LTC_ARGCHK(gcm != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + + if (gcm->buflen > 16 || gcm->buflen < 0) { + return CRYPT_INVALID_ARG; + } + + if ((err = cipher_is_valid(gcm->cipher)) != CRYPT_OK) { + return err; + } + + if (gcm->mode == LTC_GCM_MODE_IV) { + /* let's process the IV */ + if ((err = gcm_add_aad(gcm, NULL, 0)) != CRYPT_OK) return err; + } + + if (gcm->mode == LTC_GCM_MODE_AAD) { + /* let's process the AAD */ + if ((err = gcm_process(gcm, NULL, 0, NULL, 0)) != CRYPT_OK) return err; + } + + if (gcm->mode != LTC_GCM_MODE_TEXT) { + return CRYPT_INVALID_ARG; + } + + /* handle remaining ciphertext */ + if (gcm->buflen) { + gcm->pttotlen += gcm->buflen * CONST64(8); + gcm_mult_h(gcm, gcm->X); + } + + /* length */ + STORE64H(gcm->totlen, gcm->buf); + STORE64H(gcm->pttotlen, gcm->buf+8); + for (x = 0; x < 16; x++) { + gcm->X[x] ^= gcm->buf[x]; + } + gcm_mult_h(gcm, gcm->X); + + /* encrypt original counter */ + if ((err = cipher_descriptor[gcm->cipher]->ecb_encrypt(gcm->Y_0, gcm->buf, &gcm->K)) != CRYPT_OK) { + return err; + } + for (x = 0; x < 16 && x < *taglen; x++) { + tag[x] = gcm->buf[x] ^ gcm->X[x]; + } + *taglen = x; + + cipher_descriptor[gcm->cipher]->done(&gcm->K); + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_gf_mult.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_gf_mult.c new file mode 100644 index 0000000..10cd3c9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_gf_mult.c @@ -0,0 +1,209 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_gf_mult.c + GCM implementation, do the GF mult, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#if defined(LTC_GCM_TABLES) || defined(LTC_LRW_TABLES) || (defined(LTC_GCM_MODE) && defined(LTC_FAST)) + +/* this is x*2^128 mod p(x) ... the results are 16 bytes each stored in a packed format. Since only the + * lower 16 bits are not zero'ed I removed the upper 14 bytes */ +const unsigned char gcm_shift_table[256*2] = { +0x00, 0x00, 0x01, 0xc2, 0x03, 0x84, 0x02, 0x46, 0x07, 0x08, 0x06, 0xca, 0x04, 0x8c, 0x05, 0x4e, +0x0e, 0x10, 0x0f, 0xd2, 0x0d, 0x94, 0x0c, 0x56, 0x09, 0x18, 0x08, 0xda, 0x0a, 0x9c, 0x0b, 0x5e, +0x1c, 0x20, 0x1d, 0xe2, 0x1f, 0xa4, 0x1e, 0x66, 0x1b, 0x28, 0x1a, 0xea, 0x18, 0xac, 0x19, 0x6e, +0x12, 0x30, 0x13, 0xf2, 0x11, 0xb4, 0x10, 0x76, 0x15, 0x38, 0x14, 0xfa, 0x16, 0xbc, 0x17, 0x7e, +0x38, 0x40, 0x39, 0x82, 0x3b, 0xc4, 0x3a, 0x06, 0x3f, 0x48, 0x3e, 0x8a, 0x3c, 0xcc, 0x3d, 0x0e, +0x36, 0x50, 0x37, 0x92, 0x35, 0xd4, 0x34, 0x16, 0x31, 0x58, 0x30, 0x9a, 0x32, 0xdc, 0x33, 0x1e, +0x24, 0x60, 0x25, 0xa2, 0x27, 0xe4, 0x26, 0x26, 0x23, 0x68, 0x22, 0xaa, 0x20, 0xec, 0x21, 0x2e, +0x2a, 0x70, 0x2b, 0xb2, 0x29, 0xf4, 0x28, 0x36, 0x2d, 0x78, 0x2c, 0xba, 0x2e, 0xfc, 0x2f, 0x3e, +0x70, 0x80, 0x71, 0x42, 0x73, 0x04, 0x72, 0xc6, 0x77, 0x88, 0x76, 0x4a, 0x74, 0x0c, 0x75, 0xce, +0x7e, 0x90, 0x7f, 0x52, 0x7d, 0x14, 0x7c, 0xd6, 0x79, 0x98, 0x78, 0x5a, 0x7a, 0x1c, 0x7b, 0xde, +0x6c, 0xa0, 0x6d, 0x62, 0x6f, 0x24, 0x6e, 0xe6, 0x6b, 0xa8, 0x6a, 0x6a, 0x68, 0x2c, 0x69, 0xee, +0x62, 0xb0, 0x63, 0x72, 0x61, 0x34, 0x60, 0xf6, 0x65, 0xb8, 0x64, 0x7a, 0x66, 0x3c, 0x67, 0xfe, +0x48, 0xc0, 0x49, 0x02, 0x4b, 0x44, 0x4a, 0x86, 0x4f, 0xc8, 0x4e, 0x0a, 0x4c, 0x4c, 0x4d, 0x8e, +0x46, 0xd0, 0x47, 0x12, 0x45, 0x54, 0x44, 0x96, 0x41, 0xd8, 0x40, 0x1a, 0x42, 0x5c, 0x43, 0x9e, +0x54, 0xe0, 0x55, 0x22, 0x57, 0x64, 0x56, 0xa6, 0x53, 0xe8, 0x52, 0x2a, 0x50, 0x6c, 0x51, 0xae, +0x5a, 0xf0, 0x5b, 0x32, 0x59, 0x74, 0x58, 0xb6, 0x5d, 0xf8, 0x5c, 0x3a, 0x5e, 0x7c, 0x5f, 0xbe, +0xe1, 0x00, 0xe0, 0xc2, 0xe2, 0x84, 0xe3, 0x46, 0xe6, 0x08, 0xe7, 0xca, 0xe5, 0x8c, 0xe4, 0x4e, +0xef, 0x10, 0xee, 0xd2, 0xec, 0x94, 0xed, 0x56, 0xe8, 0x18, 0xe9, 0xda, 0xeb, 0x9c, 0xea, 0x5e, +0xfd, 0x20, 0xfc, 0xe2, 0xfe, 0xa4, 0xff, 0x66, 0xfa, 0x28, 0xfb, 0xea, 0xf9, 0xac, 0xf8, 0x6e, +0xf3, 0x30, 0xf2, 0xf2, 0xf0, 0xb4, 0xf1, 0x76, 0xf4, 0x38, 0xf5, 0xfa, 0xf7, 0xbc, 0xf6, 0x7e, +0xd9, 0x40, 0xd8, 0x82, 0xda, 0xc4, 0xdb, 0x06, 0xde, 0x48, 0xdf, 0x8a, 0xdd, 0xcc, 0xdc, 0x0e, +0xd7, 0x50, 0xd6, 0x92, 0xd4, 0xd4, 0xd5, 0x16, 0xd0, 0x58, 0xd1, 0x9a, 0xd3, 0xdc, 0xd2, 0x1e, +0xc5, 0x60, 0xc4, 0xa2, 0xc6, 0xe4, 0xc7, 0x26, 0xc2, 0x68, 0xc3, 0xaa, 0xc1, 0xec, 0xc0, 0x2e, +0xcb, 0x70, 0xca, 0xb2, 0xc8, 0xf4, 0xc9, 0x36, 0xcc, 0x78, 0xcd, 0xba, 0xcf, 0xfc, 0xce, 0x3e, +0x91, 0x80, 0x90, 0x42, 0x92, 0x04, 0x93, 0xc6, 0x96, 0x88, 0x97, 0x4a, 0x95, 0x0c, 0x94, 0xce, +0x9f, 0x90, 0x9e, 0x52, 0x9c, 0x14, 0x9d, 0xd6, 0x98, 0x98, 0x99, 0x5a, 0x9b, 0x1c, 0x9a, 0xde, +0x8d, 0xa0, 0x8c, 0x62, 0x8e, 0x24, 0x8f, 0xe6, 0x8a, 0xa8, 0x8b, 0x6a, 0x89, 0x2c, 0x88, 0xee, +0x83, 0xb0, 0x82, 0x72, 0x80, 0x34, 0x81, 0xf6, 0x84, 0xb8, 0x85, 0x7a, 0x87, 0x3c, 0x86, 0xfe, +0xa9, 0xc0, 0xa8, 0x02, 0xaa, 0x44, 0xab, 0x86, 0xae, 0xc8, 0xaf, 0x0a, 0xad, 0x4c, 0xac, 0x8e, +0xa7, 0xd0, 0xa6, 0x12, 0xa4, 0x54, 0xa5, 0x96, 0xa0, 0xd8, 0xa1, 0x1a, 0xa3, 0x5c, 0xa2, 0x9e, +0xb5, 0xe0, 0xb4, 0x22, 0xb6, 0x64, 0xb7, 0xa6, 0xb2, 0xe8, 0xb3, 0x2a, 0xb1, 0x6c, 0xb0, 0xae, +0xbb, 0xf0, 0xba, 0x32, 0xb8, 0x74, 0xb9, 0xb6, 0xbc, 0xf8, 0xbd, 0x3a, 0xbf, 0x7c, 0xbe, 0xbe }; + +#endif + + +#if defined(LTC_GCM_MODE) || defined(LRW_MODE) + +#ifndef LTC_FAST +/* right shift */ +static void s_gcm_rightshift(unsigned char *a) +{ + int x; + for (x = 15; x > 0; x--) { + a[x] = (a[x]>>1) | ((a[x-1]<<7)&0x80); + } + a[0] >>= 1; +} + +/* c = b*a */ +static const unsigned char mask[] = { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 }; +static const unsigned char poly[] = { 0x00, 0xE1 }; + + +/** + GCM GF multiplier (internal use only) bitserial + @param a First value + @param b Second value + @param c Destination for a * b + */ +void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c) +{ + unsigned char Z[16], V[16]; + unsigned char x, y, z; + + zeromem(Z, 16); + XMEMCPY(V, a, 16); + for (x = 0; x < 128; x++) { + if (b[x>>3] & mask[x&7]) { + for (y = 0; y < 16; y++) { + Z[y] ^= V[y]; + } + } + z = V[15] & 0x01; + s_gcm_rightshift(V); + V[0] ^= poly[z]; + } + XMEMCPY(c, Z, 16); +} + +#else + +/* map normal numbers to "ieee" way ... e.g. bit reversed */ +#define M(x) ( ((x&8)>>3) | ((x&4)>>1) | ((x&2)<<1) | ((x&1)<<3) ) + +#define BPD (sizeof(LTC_FAST_TYPE) * 8) +#define WPV (1 + (16 / sizeof(LTC_FAST_TYPE))) + +/** + GCM GF multiplier (internal use only) word oriented + @param a First value + @param b Second value + @param c Destination for a * b + */ +void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c) +{ + int i, j, k, u; + LTC_FAST_TYPE B[16][WPV], tmp[32 / sizeof(LTC_FAST_TYPE)], pB[16 / sizeof(LTC_FAST_TYPE)], zz, z; + unsigned char pTmp[32]; + + /* create simple tables */ + zeromem(B[0], sizeof(B[0])); + zeromem(B[M(1)], sizeof(B[M(1)])); + +#ifdef ENDIAN_32BITWORD + for (i = 0; i < 4; i++) { + LOAD32H(B[M(1)][i], a + (i<<2)); + LOAD32L(pB[i], b + (i<<2)); + } +#else + for (i = 0; i < 2; i++) { + LOAD64H(B[M(1)][i], a + (i<<3)); + LOAD64L(pB[i], b + (i<<3)); + } +#endif + + /* now create 2, 4 and 8 */ + B[M(2)][0] = B[M(1)][0] >> 1; + B[M(4)][0] = B[M(1)][0] >> 2; + B[M(8)][0] = B[M(1)][0] >> 3; + for (i = 1; i < (int)WPV; i++) { + B[M(2)][i] = (B[M(1)][i-1] << (BPD-1)) | (B[M(1)][i] >> 1); + B[M(4)][i] = (B[M(1)][i-1] << (BPD-2)) | (B[M(1)][i] >> 2); + B[M(8)][i] = (B[M(1)][i-1] << (BPD-3)) | (B[M(1)][i] >> 3); + } + + /* now all values with two bits which are 3, 5, 6, 9, 10, 12 */ + for (i = 0; i < (int)WPV; i++) { + B[M(3)][i] = B[M(1)][i] ^ B[M(2)][i]; + B[M(5)][i] = B[M(1)][i] ^ B[M(4)][i]; + B[M(6)][i] = B[M(2)][i] ^ B[M(4)][i]; + B[M(9)][i] = B[M(1)][i] ^ B[M(8)][i]; + B[M(10)][i] = B[M(2)][i] ^ B[M(8)][i]; + B[M(12)][i] = B[M(8)][i] ^ B[M(4)][i]; + + /* now all 3 bit values and the only 4 bit value: 7, 11, 13, 14, 15 */ + B[M(7)][i] = B[M(3)][i] ^ B[M(4)][i]; + B[M(11)][i] = B[M(3)][i] ^ B[M(8)][i]; + B[M(13)][i] = B[M(1)][i] ^ B[M(12)][i]; + B[M(14)][i] = B[M(6)][i] ^ B[M(8)][i]; + B[M(15)][i] = B[M(7)][i] ^ B[M(8)][i]; + } + + zeromem(tmp, sizeof(tmp)); + + /* compute product four bits of each word at a time */ + /* for each nibble */ + for (i = (BPD/4)-1; i >= 0; i--) { + /* for each word */ + for (j = 0; j < (int)(WPV-1); j++) { + /* grab the 4 bits recall the nibbles are backwards so it's a shift by (i^1)*4 */ + u = (pB[j] >> ((i^1)<<2)) & 15; + + /* add offset by the word count the table looked up value to the result */ + for (k = 0; k < (int)WPV; k++) { + tmp[k+j] ^= B[u][k]; + } + } + /* shift result up by 4 bits */ + if (i != 0) { + for (z = j = 0; j < (int)(32 / sizeof(LTC_FAST_TYPE)); j++) { + zz = tmp[j] << (BPD-4); + tmp[j] = (tmp[j] >> 4) | z; + z = zz; + } + } + } + + /* store product */ +#ifdef ENDIAN_32BITWORD + for (i = 0; i < 8; i++) { + STORE32H(tmp[i], pTmp + (i<<2)); + } +#else + for (i = 0; i < 4; i++) { + STORE64H(tmp[i], pTmp + (i<<3)); + } +#endif + + /* reduce by taking most significant byte and adding the appropriate two byte sequence 16 bytes down */ + for (i = 31; i >= 16; i--) { + pTmp[i-16] ^= gcm_shift_table[((unsigned)pTmp[i]<<1)]; + pTmp[i-15] ^= gcm_shift_table[((unsigned)pTmp[i]<<1)+1]; + } + + for (i = 0; i < 16; i++) { + c[i] = pTmp[i]; + } + +} + +#endif + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_init.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_init.c new file mode 100644 index 0000000..5a32d07 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_init.c @@ -0,0 +1,95 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_init.c + GCM implementation, initialize state, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Initialize a GCM state + @param gcm The GCM state to initialize + @param cipher The index of the cipher to use + @param key The secret key + @param keylen The length of the secret key + @return CRYPT_OK on success + */ +int gcm_init(gcm_state *gcm, int cipher, + const unsigned char *key, int keylen) +{ + int err; + unsigned char B[16]; +#ifdef LTC_GCM_TABLES + int x, y, z, t; +#endif + + LTC_ARGCHK(gcm != NULL); + LTC_ARGCHK(key != NULL); + +#ifdef LTC_FAST + if (16 % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + /* is cipher valid? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + if (cipher_descriptor[cipher]->block_length != 16) { + return CRYPT_INVALID_CIPHER; + } + + /* schedule key */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, &gcm->K)) != CRYPT_OK) { + return err; + } + + /* H = E(0) */ + zeromem(B, 16); + if ((err = cipher_descriptor[cipher]->ecb_encrypt(B, gcm->H, &gcm->K)) != CRYPT_OK) { + return err; + } + + /* setup state */ + zeromem(gcm->buf, sizeof(gcm->buf)); + zeromem(gcm->X, sizeof(gcm->X)); + gcm->cipher = cipher; + gcm->mode = LTC_GCM_MODE_IV; + gcm->ivmode = 0; + gcm->buflen = 0; + gcm->totlen = 0; + gcm->pttotlen = 0; + +#ifdef LTC_GCM_TABLES + /* setup tables */ + + /* generate the first table as it has no shifting (from which we make the other tables) */ + zeromem(B, 16); + for (y = 0; y < 256; y++) { + B[0] = y; + gcm_gf_mult(gcm->H, B, &gcm->PC[0][y][0]); + } + + /* now generate the rest of the tables based the previous table */ + for (x = 1; x < 16; x++) { + for (y = 0; y < 256; y++) { + /* now shift it right by 8 bits */ + t = gcm->PC[x-1][y][15]; + for (z = 15; z > 0; z--) { + gcm->PC[x][y][z] = gcm->PC[x-1][y][z-1]; + } + gcm->PC[x][y][0] = gcm_shift_table[t<<1]; + gcm->PC[x][y][1] ^= gcm_shift_table[(t<<1)+1]; + } + } + +#endif + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_memory.c new file mode 100644 index 0000000..4ae0960 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_memory.c @@ -0,0 +1,113 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_memory.c + GCM implementation, process a packet, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Process an entire GCM packet in one call. + @param cipher Index of cipher to use + @param key The secret key + @param keylen The length of the secret key + @param IV The initialization vector + @param IVlen The length of the initialization vector + @param adata The additional authentication data (header) + @param adatalen The length of the adata + @param pt The plaintext + @param ptlen The length of the plaintext (ciphertext length is the same) + @param ct The ciphertext + @param tag [out] The MAC tag + @param taglen [in/out] The MAC tag length + @param direction Encrypt or Decrypt mode (GCM_ENCRYPT or GCM_DECRYPT) + @return CRYPT_OK on success + */ +int gcm_memory( int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *IV, unsigned long IVlen, + const unsigned char *adata, unsigned long adatalen, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen, + int direction) +{ + void *orig; + gcm_state *gcm; + int err; + + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + if (cipher_descriptor[cipher]->accel_gcm_memory != NULL) { + return cipher_descriptor[cipher]->accel_gcm_memory + (key, keylen, + IV, IVlen, + adata, adatalen, + pt, ptlen, + ct, + tag, taglen, + direction); + } + + + +#ifndef LTC_GCM_TABLES_SSE2 + orig = gcm = XMALLOC(sizeof(*gcm)); +#else + orig = gcm = XMALLOC(sizeof(*gcm) + 16); +#endif + if (gcm == NULL) { + return CRYPT_MEM; + } + + /* Force GCM to be on a multiple of 16 so we can use 128-bit aligned operations + * note that we only modify gcm and keep orig intact. This code is not portable + * but again it's only for SSE2 anyways, so who cares? + */ +#ifdef LTC_GCM_TABLES_SSE2 + if ((unsigned long)gcm & 15) { + gcm = (gcm_state *)((unsigned long)gcm + (16 - ((unsigned long)gcm & 15))); + } +#endif + + if ((err = gcm_init(gcm, cipher, key, keylen)) != CRYPT_OK) { + goto LTC_ERR; + } + if ((err = gcm_add_iv(gcm, IV, IVlen)) != CRYPT_OK) { + goto LTC_ERR; + } + if ((err = gcm_add_aad(gcm, adata, adatalen)) != CRYPT_OK) { + goto LTC_ERR; + } + if ((err = gcm_process(gcm, pt, ptlen, ct, direction)) != CRYPT_OK) { + goto LTC_ERR; + } + if (direction == GCM_ENCRYPT) { + if ((err = gcm_done(gcm, tag, taglen)) != CRYPT_OK) { + goto LTC_ERR; + } + } + else if (direction == GCM_DECRYPT) { + unsigned char buf[MAXBLOCKSIZE]; + unsigned long buflen = sizeof(buf); + if ((err = gcm_done(gcm, buf, &buflen)) != CRYPT_OK) { + goto LTC_ERR; + } + if (buflen != *taglen || XMEM_NEQ(buf, tag, buflen) != 0) { + err = CRYPT_ERROR; + } + } + else { + err = CRYPT_INVALID_ARG; + } +LTC_ERR: + XFREE(orig); + return err; +} +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h.c new file mode 100644 index 0000000..fcf3226 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h.c @@ -0,0 +1,47 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_mult_h.c + GCM implementation, do the GF mult, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#if defined(LTC_GCM_MODE) +/** + GCM multiply by H + @param gcm The GCM state which holds the H value + @param I The value to multiply H by + */ +void gcm_mult_h(const gcm_state *gcm, unsigned char *I) +{ + unsigned char T[16]; +#ifdef LTC_GCM_TABLES + int x; +#ifdef LTC_GCM_TABLES_SSE2 + asm("movdqa (%0),%%xmm0"::"r"(&gcm->PC[0][I[0]][0])); + for (x = 1; x < 16; x++) { + asm("pxor (%0),%%xmm0"::"r"(&gcm->PC[x][I[x]][0])); + } + asm("movdqa %%xmm0,(%0)"::"r"(&T)); +#else + int y; + XMEMCPY(T, &gcm->PC[0][I[0]][0], 16); + for (x = 1; x < 16; x++) { +#ifdef LTC_FAST + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(T + y)) ^= *(LTC_FAST_TYPE_PTR_CAST(&gcm->PC[x][I[x]][y])); + } +#else + for (y = 0; y < 16; y++) { + T[y] ^= gcm->PC[x][I[x]][y]; + } +#endif /* LTC_FAST */ + } +#endif /* LTC_GCM_TABLES_SSE2 */ +#else + gcm_gf_mult(gcm->H, I, T); +#endif + XMEMCPY(I, T, 16); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h_arm_ce.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h_arm_ce.c new file mode 100644 index 0000000..895a924 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h_arm_ce.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include +#include + +/** + GCM multiply by H + @param gcm The GCM state which holds the H value + @param I The value to multiply H by + */ +void gcm_mult_h(gcm_state *gcm, unsigned char *I) +{ + struct tomcrypt_arm_neon_state state; + const uint8_t zeroes[TEE_AES_BLOCK_SIZE] = { 0 }; + uint64_t k[2]; + uint64_t a; + uint64_t b; + uint64_t dg[2]; + + b = get_be64(gcm->H); + a = get_be64(gcm->H + 8); + + k[0] = (a << 1) | (b >> 63); + k[1] = (b << 1) | (a >> 63); + if (b >> 63) + k[1] ^= 0xc200000000000000UL; + + dg[1] = get_be64(I); + dg[0] = get_be64(I + 8); + + tomcrypt_arm_neon_enable(&state); +#ifdef _CFG_CORE_LTC_HWSUPP_PMULL + pmull_ghash_update_p64(1, dg, zeroes, k, NULL); +#else + pmull_ghash_update_p8(1, dg, zeroes, k, NULL); +#endif + tomcrypt_arm_neon_disable(&state); + + put_be64(I, dg[1]); + put_be64(I + 8, dg[0]); +} + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_process.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_process.c new file mode 100644 index 0000000..2031c8f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_process.c @@ -0,0 +1,150 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_process.c + GCM implementation, process message data, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Process plaintext/ciphertext through GCM + @param gcm The GCM state + @param pt The plaintext + @param ptlen The plaintext length (ciphertext length is the same) + @param ct The ciphertext + @param direction Encrypt or Decrypt mode (GCM_ENCRYPT or GCM_DECRYPT) + @return CRYPT_OK on success + */ +int gcm_process(gcm_state *gcm, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + int direction) +{ + unsigned long x; + int y, err; + unsigned char b; + + LTC_ARGCHK(gcm != NULL); + if (ptlen > 0) { + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + } + + if (gcm->buflen > 16 || gcm->buflen < 0) { + return CRYPT_INVALID_ARG; + } + + if ((err = cipher_is_valid(gcm->cipher)) != CRYPT_OK) { + return err; + } + + /* 0xFFFFFFFE0 = ((2^39)-256)/8 */ + if (gcm->pttotlen / 8 + (ulong64)gcm->buflen + (ulong64)ptlen >= CONST64(0xFFFFFFFE0)) { + return CRYPT_INVALID_ARG; + } + + if (gcm->mode == LTC_GCM_MODE_IV) { + /* let's process the IV */ + if ((err = gcm_add_aad(gcm, NULL, 0)) != CRYPT_OK) return err; + } + + /* in AAD mode? */ + if (gcm->mode == LTC_GCM_MODE_AAD) { + /* let's process the AAD */ + if (gcm->buflen) { + gcm->totlen += gcm->buflen * CONST64(8); + gcm_mult_h(gcm, gcm->X); + } + + /* increment counter */ + for (y = 15; y >= 12; y--) { + if (++gcm->Y[y] & 255) { break; } + } + /* encrypt the counter */ + if ((err = cipher_descriptor[gcm->cipher]->ecb_encrypt(gcm->Y, gcm->buf, &gcm->K)) != CRYPT_OK) { + return err; + } + + gcm->buflen = 0; + gcm->mode = LTC_GCM_MODE_TEXT; + } + + if (gcm->mode != LTC_GCM_MODE_TEXT) { + return CRYPT_INVALID_ARG; + } + + x = 0; +#ifdef LTC_FAST + if (gcm->buflen == 0) { + if (direction == GCM_ENCRYPT) { + for (x = 0; x < (ptlen & ~15); x += 16) { + /* ctr encrypt */ + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&ct[x + y])) = *(LTC_FAST_TYPE_PTR_CAST(&pt[x+y])) ^ *(LTC_FAST_TYPE_PTR_CAST(&gcm->buf[y])); + *(LTC_FAST_TYPE_PTR_CAST(&gcm->X[y])) ^= *(LTC_FAST_TYPE_PTR_CAST(&ct[x+y])); + } + /* GMAC it */ + gcm->pttotlen += 128; + gcm_mult_h(gcm, gcm->X); + /* increment counter */ + for (y = 15; y >= 12; y--) { + if (++gcm->Y[y] & 255) { break; } + } + if ((err = cipher_descriptor[gcm->cipher]->ecb_encrypt(gcm->Y, gcm->buf, &gcm->K)) != CRYPT_OK) { + return err; + } + } + } else { + for (x = 0; x < (ptlen & ~15); x += 16) { + /* ctr encrypt */ + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&gcm->X[y])) ^= *(LTC_FAST_TYPE_PTR_CAST(&ct[x+y])); + *(LTC_FAST_TYPE_PTR_CAST(&pt[x + y])) = *(LTC_FAST_TYPE_PTR_CAST(&ct[x+y])) ^ *(LTC_FAST_TYPE_PTR_CAST(&gcm->buf[y])); + } + /* GMAC it */ + gcm->pttotlen += 128; + gcm_mult_h(gcm, gcm->X); + /* increment counter */ + for (y = 15; y >= 12; y--) { + if (++gcm->Y[y] & 255) { break; } + } + if ((err = cipher_descriptor[gcm->cipher]->ecb_encrypt(gcm->Y, gcm->buf, &gcm->K)) != CRYPT_OK) { + return err; + } + } + } + } +#endif + + /* process text */ + for (; x < ptlen; x++) { + if (gcm->buflen == 16) { + gcm->pttotlen += 128; + gcm_mult_h(gcm, gcm->X); + + /* increment counter */ + for (y = 15; y >= 12; y--) { + if (++gcm->Y[y] & 255) { break; } + } + if ((err = cipher_descriptor[gcm->cipher]->ecb_encrypt(gcm->Y, gcm->buf, &gcm->K)) != CRYPT_OK) { + return err; + } + gcm->buflen = 0; + } + + if (direction == GCM_ENCRYPT) { + b = ct[x] = pt[x] ^ gcm->buf[gcm->buflen]; + } else { + b = ct[x]; + pt[x] = ct[x] ^ gcm->buf[gcm->buflen]; + } + gcm->X[gcm->buflen++] ^= b; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_reset.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_reset.c new file mode 100644 index 0000000..a071dcd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_reset.c @@ -0,0 +1,32 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_reset.c + GCM implementation, reset a used state so it can accept IV data, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Reset a GCM state to as if you just called gcm_init(). This saves the initialization time. + @param gcm The GCM state to reset + @return CRYPT_OK on success +*/ +int gcm_reset(gcm_state *gcm) +{ + LTC_ARGCHK(gcm != NULL); + + zeromem(gcm->buf, sizeof(gcm->buf)); + zeromem(gcm->X, sizeof(gcm->X)); + gcm->mode = LTC_GCM_MODE_IV; + gcm->ivmode = 0; + gcm->buflen = 0; + gcm->totlen = 0; + gcm->pttotlen = 0; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_test.c b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_test.c new file mode 100644 index 0000000..228db57 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/gcm_test.c @@ -0,0 +1,407 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file gcm_test.c + GCM implementation, testing, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_GCM_MODE + +/** + Test the GCM code + @return CRYPT_OK on success + */ +int gcm_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char K[32]; + int keylen; + unsigned char P[128]; + unsigned long ptlen; + unsigned char A[128]; + unsigned long alen; + unsigned char IV[128]; + unsigned long IVlen; + unsigned char C[128]; + unsigned char T[16]; + } tests[] = { + +/* test case #1 */ +{ + /* key */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + 16, + + /* plaintext */ + { 0 }, + 0, + + /* AAD data */ + { 0 }, + 0, + + /* IV */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 }, + 12, + + /* ciphertext */ + { 0 }, + + /* tag */ + { 0x58, 0xe2, 0xfc, 0xce, 0xfa, 0x7e, 0x30, 0x61, + 0x36, 0x7f, 0x1d, 0x57, 0xa4, 0xe7, 0x45, 0x5a } +}, + +/* test case #2 */ +{ + /* key */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + 16, + + /* PT */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + 16, + + /* ADATA */ + { 0 }, + 0, + + /* IV */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 }, + 12, + + /* CT */ + { 0x03, 0x88, 0xda, 0xce, 0x60, 0xb6, 0xa3, 0x92, + 0xf3, 0x28, 0xc2, 0xb9, 0x71, 0xb2, 0xfe, 0x78 }, + + /* TAG */ + { 0xab, 0x6e, 0x47, 0xd4, 0x2c, 0xec, 0x13, 0xbd, + 0xf5, 0x3a, 0x67, 0xb2, 0x12, 0x57, 0xbd, 0xdf } +}, + +/* test case #3 */ +{ + /* key */ + { 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, }, + 16, + + /* PT */ + { 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, }, + 64, + + /* ADATA */ + { 0 }, + 0, + + /* IV */ + { 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88, }, + 12, + + /* CT */ + { 0x42, 0x83, 0x1e, 0xc2, 0x21, 0x77, 0x74, 0x24, + 0x4b, 0x72, 0x21, 0xb7, 0x84, 0xd0, 0xd4, 0x9c, + 0xe3, 0xaa, 0x21, 0x2f, 0x2c, 0x02, 0xa4, 0xe0, + 0x35, 0xc1, 0x7e, 0x23, 0x29, 0xac, 0xa1, 0x2e, + 0x21, 0xd5, 0x14, 0xb2, 0x54, 0x66, 0x93, 0x1c, + 0x7d, 0x8f, 0x6a, 0x5a, 0xac, 0x84, 0xaa, 0x05, + 0x1b, 0xa3, 0x0b, 0x39, 0x6a, 0x0a, 0xac, 0x97, + 0x3d, 0x58, 0xe0, 0x91, 0x47, 0x3f, 0x59, 0x85, }, + + /* TAG */ + { 0x4d, 0x5c, 0x2a, 0xf3, 0x27, 0xcd, 0x64, 0xa6, + 0x2c, 0xf3, 0x5a, 0xbd, 0x2b, 0xa6, 0xfa, 0xb4, } +}, + +/* test case #4 */ +{ + /* key */ + { 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, }, + 16, + + /* PT */ + { 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, }, + 60, + + /* ADATA */ + { 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xab, 0xad, 0xda, 0xd2, }, + 20, + + /* IV */ + { 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88, }, + 12, + + /* CT */ + { 0x42, 0x83, 0x1e, 0xc2, 0x21, 0x77, 0x74, 0x24, + 0x4b, 0x72, 0x21, 0xb7, 0x84, 0xd0, 0xd4, 0x9c, + 0xe3, 0xaa, 0x21, 0x2f, 0x2c, 0x02, 0xa4, 0xe0, + 0x35, 0xc1, 0x7e, 0x23, 0x29, 0xac, 0xa1, 0x2e, + 0x21, 0xd5, 0x14, 0xb2, 0x54, 0x66, 0x93, 0x1c, + 0x7d, 0x8f, 0x6a, 0x5a, 0xac, 0x84, 0xaa, 0x05, + 0x1b, 0xa3, 0x0b, 0x39, 0x6a, 0x0a, 0xac, 0x97, + 0x3d, 0x58, 0xe0, 0x91, }, + + /* TAG */ + { 0x5b, 0xc9, 0x4f, 0xbc, 0x32, 0x21, 0xa5, 0xdb, + 0x94, 0xfa, 0xe9, 0x5a, 0xe7, 0x12, 0x1a, 0x47, } + +}, + +/* test case #5 */ +{ + /* key */ + { 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, }, + 16, + + /* PT */ + { 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, }, + 60, + + /* ADATA */ + { 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xab, 0xad, 0xda, 0xd2, }, + 20, + + /* IV */ + { 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, }, + 8, + + /* CT */ + { 0x61, 0x35, 0x3b, 0x4c, 0x28, 0x06, 0x93, 0x4a, + 0x77, 0x7f, 0xf5, 0x1f, 0xa2, 0x2a, 0x47, 0x55, + 0x69, 0x9b, 0x2a, 0x71, 0x4f, 0xcd, 0xc6, 0xf8, + 0x37, 0x66, 0xe5, 0xf9, 0x7b, 0x6c, 0x74, 0x23, + 0x73, 0x80, 0x69, 0x00, 0xe4, 0x9f, 0x24, 0xb2, + 0x2b, 0x09, 0x75, 0x44, 0xd4, 0x89, 0x6b, 0x42, + 0x49, 0x89, 0xb5, 0xe1, 0xeb, 0xac, 0x0f, 0x07, + 0xc2, 0x3f, 0x45, 0x98, }, + + /* TAG */ + { 0x36, 0x12, 0xd2, 0xe7, 0x9e, 0x3b, 0x07, 0x85, + 0x56, 0x1b, 0xe1, 0x4a, 0xac, 0xa2, 0xfc, 0xcb, } +}, + +/* test case #6 */ +{ + /* key */ + { 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, }, + 16, + + /* PT */ + { 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, }, + 60, + + /* ADATA */ + { 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xab, 0xad, 0xda, 0xd2, }, + 20, + + /* IV */ + { 0x93, 0x13, 0x22, 0x5d, 0xf8, 0x84, 0x06, 0xe5, + 0x55, 0x90, 0x9c, 0x5a, 0xff, 0x52, 0x69, 0xaa, + 0x6a, 0x7a, 0x95, 0x38, 0x53, 0x4f, 0x7d, 0xa1, + 0xe4, 0xc3, 0x03, 0xd2, 0xa3, 0x18, 0xa7, 0x28, + 0xc3, 0xc0, 0xc9, 0x51, 0x56, 0x80, 0x95, 0x39, + 0xfc, 0xf0, 0xe2, 0x42, 0x9a, 0x6b, 0x52, 0x54, + 0x16, 0xae, 0xdb, 0xf5, 0xa0, 0xde, 0x6a, 0x57, + 0xa6, 0x37, 0xb3, 0x9b, }, + 60, + + /* CT */ + { 0x8c, 0xe2, 0x49, 0x98, 0x62, 0x56, 0x15, 0xb6, + 0x03, 0xa0, 0x33, 0xac, 0xa1, 0x3f, 0xb8, 0x94, + 0xbe, 0x91, 0x12, 0xa5, 0xc3, 0xa2, 0x11, 0xa8, + 0xba, 0x26, 0x2a, 0x3c, 0xca, 0x7e, 0x2c, 0xa7, + 0x01, 0xe4, 0xa9, 0xa4, 0xfb, 0xa4, 0x3c, 0x90, + 0xcc, 0xdc, 0xb2, 0x81, 0xd4, 0x8c, 0x7c, 0x6f, + 0xd6, 0x28, 0x75, 0xd2, 0xac, 0xa4, 0x17, 0x03, + 0x4c, 0x34, 0xae, 0xe5, }, + + /* TAG */ + { 0x61, 0x9c, 0xc5, 0xae, 0xff, 0xfe, 0x0b, 0xfa, + 0x46, 0x2a, 0xf4, 0x3c, 0x16, 0x99, 0xd0, 0x50, } +}, + +/* test case #46 from BG (catches the LTC bug of v1.15) */ +{ + /* key */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + 16, + + /* PT */ + { 0xa2, 0xaa, 0xb3, 0xad, 0x8b, 0x17, 0xac, 0xdd, + 0xa2, 0x88, 0x42, 0x6c, 0xd7, 0xc4, 0x29, 0xb7, + 0xca, 0x86, 0xb7, 0xac, 0xa0, 0x58, 0x09, 0xc7, + 0x0c, 0xe8, 0x2d, 0xb2, 0x57, 0x11, 0xcb, 0x53, + 0x02, 0xeb, 0x27, 0x43, 0xb0, 0x36, 0xf3, 0xd7, + 0x50, 0xd6, 0xcf, 0x0d, 0xc0, 0xac, 0xb9, 0x29, + 0x50, 0xd5, 0x46, 0xdb, 0x30, 0x8f, 0x93, 0xb4, + 0xff, 0x24, 0x4a, 0xfa, 0x9d, 0xc7, 0x2b, 0xcd, + 0x75, 0x8d, 0x2c }, + 67, + + /* ADATA */ + { 0x68, 0x8e, 0x1a, 0xa9, 0x84, 0xde, 0x92, 0x6d, + 0xc7, 0xb4, 0xc4, 0x7f, 0x44 }, + 13, + + /* IV */ + { 0xb7, 0x21, 0x38, 0xb5, 0xa0, 0x5f, 0xf5, 0x07, + 0x0e, 0x8c, 0xd9, 0x41, 0x83, 0xf7, 0x61, 0xd8 }, + 16, + + /* CT */ + { 0xcb, 0xc8, 0xd2, 0xf1, 0x54, 0x81, 0xa4, 0xcc, + 0x7d, 0xd1, 0xe1, 0x9a, 0xaa, 0x83, 0xde, 0x56, + 0x78, 0x48, 0x3e, 0xc3, 0x59, 0xae, 0x7d, 0xec, + 0x2a, 0xb8, 0xd5, 0x34, 0xe0, 0x90, 0x6f, 0x4b, + 0x46, 0x63, 0xfa, 0xff, 0x58, 0xa8, 0xb2, 0xd7, + 0x33, 0xb8, 0x45, 0xee, 0xf7, 0xc9, 0xb3, 0x31, + 0xe9, 0xe1, 0x0e, 0xb2, 0x61, 0x2c, 0x99, 0x5f, + 0xeb, 0x1a, 0xc1, 0x5a, 0x62, 0x86, 0xcc, 0xe8, + 0xb2, 0x97, 0xa8 }, + + /* TAG */ + { 0x8d, 0x2d, 0x2a, 0x93, 0x72, 0x62, 0x6f, 0x6b, + 0xee, 0x85, 0x80, 0x27, 0x6a, 0x63, 0x66, 0xbf } +} + +/* rest of test cases are the same except AES key size changes... ignored... */ +}; + int idx, err; + unsigned long x, y; + unsigned char out[2][128], T[2][16]; + gcm_state gcm; + + /* find aes */ + idx = find_cipher("aes"); + if (idx == -1) { + idx = find_cipher("rijndael"); + if (idx == -1) { + return CRYPT_NOP; + } + } + + /* Special test case for empty AAD + empty PT */ + y = sizeof(T[0]); + if ((err = gcm_init(&gcm, idx, tests[0].K, tests[0].keylen)) != CRYPT_OK) return err; + if ((err = gcm_add_iv(&gcm, tests[0].IV, tests[0].IVlen)) != CRYPT_OK) return err; + /* intentionally skip gcm_add_aad + gcm_process */ + if ((err = gcm_done(&gcm, T[0], &y)) != CRYPT_OK) return err; + if (compare_testvector(T[0], y, tests[0].T, 16, "GCM Encrypt Tag-special", 0)) return CRYPT_FAIL_TESTVECTOR; + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + y = sizeof(T[0]); + if ((err = gcm_memory(idx, tests[x].K, tests[x].keylen, + tests[x].IV, tests[x].IVlen, + tests[x].A, tests[x].alen, + (unsigned char*)tests[x].P, tests[x].ptlen, + out[0], T[0], &y, GCM_ENCRYPT)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(out[0], tests[x].ptlen, tests[x].C, tests[x].ptlen, "GCM CT", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + if (compare_testvector(T[0], y, tests[x].T, 16, "GCM Encrypt Tag", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + y = sizeof(T[1]); + XMEMCPY(T[1], tests[x].T, 16); + if ((err = gcm_memory(idx, tests[x].K, tests[x].keylen, + tests[x].IV, tests[x].IVlen, + tests[x].A, tests[x].alen, + out[1], tests[x].ptlen, + out[0], T[1], &y, GCM_DECRYPT)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(out[1], tests[x].ptlen, tests[x].P, tests[x].ptlen, "GCM PT", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + /* wycheproof failing test - https://github.com/libtom/libtomcrypt/pull/451 */ + { + unsigned char key[] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f }; + unsigned char iv[] = { 0x50,0x51,0x52,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5a,0x5b }; + unsigned char valid_tag[] = { 0xd8,0x84,0x7d,0xbc,0x32,0x6a,0x06,0xe9,0x88,0xc7,0x7a,0xd3,0x86,0x3e,0x60,0x83 }; + unsigned char invalid_tag[] = { 0xd9,0x84,0x7d,0xbc,0x32,0x6a,0x06,0xe9,0x88,0xc7,0x7a,0xd3,0x86,0x3e,0x60,0x83 }; + unsigned char msg[] = { 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f }; + unsigned char ct[] = { 0xeb,0x15,0x6d,0x08,0x1e,0xd6,0xb6,0xb5,0x5f,0x46,0x12,0xf0,0x21,0xd8,0x7b,0x39 }; + unsigned char pt[20] = { 0 }; + unsigned long taglen; + + /* VALID tag */ + taglen = sizeof(valid_tag); + err = gcm_memory(idx, key, sizeof(key), iv, sizeof(iv), NULL, 0, + pt, sizeof(ct), ct, valid_tag, &taglen, GCM_DECRYPT); + if ((err != CRYPT_OK) || (XMEMCMP(msg, pt, sizeof(msg)) != 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* INVALID tag */ + taglen = sizeof(invalid_tag); + err = gcm_memory(idx, key, sizeof(key), iv, sizeof(iv), NULL, 0, + pt, sizeof(ct), ct, invalid_tag, &taglen, GCM_DECRYPT); + if (err == CRYPT_OK) { + return CRYPT_FAIL_TESTVECTOR; /* should fail */ + } + } + + return CRYPT_OK; +#endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/gcm/sub.mk b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/sub.mk new file mode 100644 index 0000000..6720c3f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/gcm/sub.mk @@ -0,0 +1,13 @@ +srcs-y += gcm_add_aad.c +srcs-y += gcm_add_iv.c +srcs-y += gcm_done.c +srcs-y += gcm_gf_mult.c +srcs-y += gcm_init.c +srcs-y += gcm_memory.c +ifeq ($(_CFG_CORE_LTC_CE),y) +srcs-y += gcm_mult_h_arm_ce.c +else +srcs-y += gcm_mult_h.c +endif +srcs-y += gcm_process.c +srcs-y += gcm_reset.c diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_decrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_decrypt.c new file mode 100644 index 0000000..6837427 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_decrypt.c @@ -0,0 +1,67 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_decrypt.c + OCB implementation, decrypt data, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Decrypt a block with OCB. + @param ocb The OCB state + @param ct The ciphertext (length of the block size of the block cipher) + @param pt [out] The plaintext (length of ct) + @return CRYPT_OK if successful +*/ +int ocb_decrypt(ocb_state *ocb, const unsigned char *ct, unsigned char *pt) +{ + unsigned char Z[MAXBLOCKSIZE], tmp[MAXBLOCKSIZE]; + int err, x; + + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + /* check if valid cipher */ + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + return err; + } + LTC_ARGCHK(cipher_descriptor[ocb->cipher]->ecb_decrypt != NULL); + + /* check length */ + if (ocb->block_len != cipher_descriptor[ocb->cipher]->block_length) { + return CRYPT_INVALID_ARG; + } + + /* Get Z[i] value */ + ocb_shift_xor(ocb, Z); + + /* xor ct in, encrypt, xor Z out */ + for (x = 0; x < ocb->block_len; x++) { + tmp[x] = ct[x] ^ Z[x]; + } + if ((err = cipher_descriptor[ocb->cipher]->ecb_decrypt(tmp, pt, &ocb->key)) != CRYPT_OK) { + return err; + } + for (x = 0; x < ocb->block_len; x++) { + pt[x] ^= Z[x]; + } + + /* compute checksum */ + for (x = 0; x < ocb->block_len; x++) { + ocb->checksum[x] ^= pt[x]; + } + + +#ifdef LTC_CLEAN_STACK + zeromem(Z, sizeof(Z)); + zeromem(tmp, sizeof(tmp)); +#endif + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_decrypt_verify_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_decrypt_verify_memory.c new file mode 100644 index 0000000..0f4dcb7 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_decrypt_verify_memory.c @@ -0,0 +1,74 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_decrypt_verify_memory.c + OCB implementation, helper to decrypt block of memory, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Decrypt and compare the tag with OCB. + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param nonce The session nonce (length of the block size of the block cipher) + @param ct The ciphertext + @param ctlen The length of the ciphertext (octets) + @param pt [out] The plaintext + @param tag The tag to compare against + @param taglen The length of the tag (octets) + @param stat [out] The result of the tag comparison (1==valid, 0==invalid) + @return CRYPT_OK if successful regardless of the tag comparison +*/ +int ocb_decrypt_verify_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, + int *stat) +{ + int err; + ocb_state *ocb; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(nonce != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(stat != NULL); + + /* allocate memory */ + ocb = XMALLOC(sizeof(ocb_state)); + if (ocb == NULL) { + return CRYPT_MEM; + } + + if ((err = ocb_init(ocb, cipher, key, keylen, nonce)) != CRYPT_OK) { + goto LBL_ERR; + } + + while (ctlen > (unsigned long)ocb->block_len) { + if ((err = ocb_decrypt(ocb, ct, pt)) != CRYPT_OK) { + goto LBL_ERR; + } + ctlen -= ocb->block_len; + pt += ocb->block_len; + ct += ocb->block_len; + } + + err = ocb_done_decrypt(ocb, ct, ctlen, pt, tag, taglen, stat); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(ocb, sizeof(ocb_state)); +#endif + + XFREE(ocb); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_done_decrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_done_decrypt.c new file mode 100644 index 0000000..3d516c9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_done_decrypt.c @@ -0,0 +1,68 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_done_decrypt.c + OCB implementation, terminate decryption, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Terminate a decrypting OCB state + @param ocb The OCB state + @param ct The ciphertext (if any) + @param ctlen The length of the ciphertext (octets) + @param pt [out] The plaintext + @param tag The authentication tag (to compare against) + @param taglen The length of the authentication tag provided + @param stat [out] The result of the tag comparison + @return CRYPT_OK if the process was successful regardless if the tag is valid +*/ +int ocb_done_decrypt(ocb_state *ocb, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, int *stat) +{ + int err; + unsigned char *tagbuf; + unsigned long tagbuflen; + + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(stat != NULL); + + /* default to failed */ + *stat = 0; + + /* allocate memory */ + tagbuf = XMALLOC(MAXBLOCKSIZE); + if (tagbuf == NULL) { + return CRYPT_MEM; + } + + tagbuflen = MAXBLOCKSIZE; + if ((err = s_ocb_done(ocb, ct, ctlen, pt, tagbuf, &tagbuflen, 1)) != CRYPT_OK) { + goto LBL_ERR; + } + + if (taglen <= tagbuflen && XMEM_NEQ(tagbuf, tag, taglen) == 0) { + *stat = 1; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(tagbuf, MAXBLOCKSIZE); +#endif + + XFREE(tagbuf); + + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_done_encrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_done_encrypt.c new file mode 100644 index 0000000..5cd39ad --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_done_encrypt.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_done_encrypt.c + OCB implementation, terminate encryption, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Terminate an encryption OCB state + @param ocb The OCB state + @param pt Remaining plaintext (if any) + @param ptlen The length of the plaintext (octets) + @param ct [out] The ciphertext (if any) + @param tag [out] The tag for the OCB stream + @param taglen [in/out] The max size and resulting size of the tag + @return CRYPT_OK if successful +*/ +int ocb_done_encrypt(ocb_state *ocb, const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, unsigned char *tag, unsigned long *taglen) +{ + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + return s_ocb_done(ocb, pt, ptlen, ct, tag, taglen, 0); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_encrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_encrypt.c new file mode 100644 index 0000000..ad6260f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_encrypt.c @@ -0,0 +1,60 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_encrypt.c + OCB implementation, encrypt data, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Encrypt a block of data with OCB. + @param ocb The OCB state + @param pt The plaintext (length of the block size of the block cipher) + @param ct [out] The ciphertext (same size as the pt) + @return CRYPT_OK if successful +*/ +int ocb_encrypt(ocb_state *ocb, const unsigned char *pt, unsigned char *ct) +{ + unsigned char Z[MAXBLOCKSIZE], tmp[MAXBLOCKSIZE]; + int err, x; + + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + return err; + } + if (ocb->block_len != cipher_descriptor[ocb->cipher].block_length) { + return CRYPT_INVALID_ARG; + } + + /* compute checksum */ + for (x = 0; x < ocb->block_len; x++) { + ocb->checksum[x] ^= pt[x]; + } + + /* Get Z[i] value */ + ocb_shift_xor(ocb, Z); + + /* xor pt in, encrypt, xor Z out */ + for (x = 0; x < ocb->block_len; x++) { + tmp[x] = pt[x] ^ Z[x]; + } + if ((err = cipher_descriptor[ocb->cipher].ecb_encrypt(tmp, ct, &ocb->key)) != CRYPT_OK) { + return err; + } + for (x = 0; x < ocb->block_len; x++) { + ct[x] ^= Z[x]; + } + +#ifdef LTC_CLEAN_STACK + zeromem(Z, sizeof(Z)); + zeromem(tmp, sizeof(tmp)); +#endif + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_encrypt_authenticate_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_encrypt_authenticate_memory.c new file mode 100644 index 0000000..7560a6e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_encrypt_authenticate_memory.c @@ -0,0 +1,72 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_encrypt_authenticate_memory.c + OCB implementation, encrypt block of memory, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Encrypt and generate an authentication code for a buffer of memory + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param nonce The session nonce (length of the block ciphers block size) + @param pt The plaintext + @param ptlen The length of the plaintext (octets) + @param ct [out] The ciphertext + @param tag [out] The authentication tag + @param taglen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful +*/ +int ocb_encrypt_authenticate_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen) +{ + int err; + ocb_state *ocb; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(nonce != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + + /* allocate ram */ + ocb = XMALLOC(sizeof(ocb_state)); + if (ocb == NULL) { + return CRYPT_MEM; + } + + if ((err = ocb_init(ocb, cipher, key, keylen, nonce)) != CRYPT_OK) { + goto LBL_ERR; + } + + while (ptlen > (unsigned long)ocb->block_len) { + if ((err = ocb_encrypt(ocb, pt, ct)) != CRYPT_OK) { + goto LBL_ERR; + } + ptlen -= ocb->block_len; + pt += ocb->block_len; + ct += ocb->block_len; + } + + err = ocb_done_encrypt(ocb, pt, ptlen, ct, tag, taglen); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(ocb, sizeof(ocb_state)); +#endif + + XFREE(ocb); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_init.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_init.c new file mode 100644 index 0000000..80321de --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_init.c @@ -0,0 +1,129 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_init.c + OCB implementation, initialize state, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +static const struct { + int len; + unsigned char poly_div[MAXBLOCKSIZE], + poly_mul[MAXBLOCKSIZE]; +} polys[] = { +{ + 8, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0D }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1B } +}, { + 16, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87 } +} +}; + +/** + Initialize an OCB context. + @param ocb [out] The destination of the OCB state + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @param nonce The session nonce (length of the block size of the cipher) + @return CRYPT_OK if successful +*/ +int ocb_init(ocb_state *ocb, int cipher, + const unsigned char *key, unsigned long keylen, const unsigned char *nonce) +{ + int poly, x, y, m, err; + + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(nonce != NULL); + + /* valid cipher? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* determine which polys to use */ + ocb->block_len = cipher_descriptor[cipher]->block_length; + x = (int)(sizeof(polys)/sizeof(polys[0])); + for (poly = 0; poly < x; poly++) { + if (polys[poly].len == ocb->block_len) { + break; + } + } + if (poly == x) { + return CRYPT_INVALID_ARG; /* block_len not found in polys */ + } + if (polys[poly].len != ocb->block_len) { + return CRYPT_INVALID_ARG; + } + + /* schedule the key */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, &ocb->key)) != CRYPT_OK) { + return err; + } + + /* find L = E[0] */ + zeromem(ocb->L, ocb->block_len); + if ((err = cipher_descriptor[cipher]->ecb_encrypt(ocb->L, ocb->L, &ocb->key)) != CRYPT_OK) { + return err; + } + + /* find R = E[N xor L] */ + for (x = 0; x < ocb->block_len; x++) { + ocb->R[x] = ocb->L[x] ^ nonce[x]; + } + if ((err = cipher_descriptor[cipher]->ecb_encrypt(ocb->R, ocb->R, &ocb->key)) != CRYPT_OK) { + return err; + } + + /* find Ls[i] = L << i for i == 0..31 */ + XMEMCPY(ocb->Ls[0], ocb->L, ocb->block_len); + for (x = 1; x < 32; x++) { + m = ocb->Ls[x-1][0] >> 7; + for (y = 0; y < ocb->block_len-1; y++) { + ocb->Ls[x][y] = ((ocb->Ls[x-1][y] << 1) | (ocb->Ls[x-1][y+1] >> 7)) & 255; + } + ocb->Ls[x][ocb->block_len-1] = (ocb->Ls[x-1][ocb->block_len-1] << 1) & 255; + + if (m == 1) { + for (y = 0; y < ocb->block_len; y++) { + ocb->Ls[x][y] ^= polys[poly].poly_mul[y]; + } + } + } + + /* find Lr = L / x */ + m = ocb->L[ocb->block_len-1] & 1; + + /* shift right */ + for (x = ocb->block_len - 1; x > 0; x--) { + ocb->Lr[x] = ((ocb->L[x] >> 1) | (ocb->L[x-1] << 7)) & 255; + } + ocb->Lr[0] = ocb->L[0] >> 1; + + if (m == 1) { + for (x = 0; x < ocb->block_len; x++) { + ocb->Lr[x] ^= polys[poly].poly_div[x]; + } + } + + /* set Li, checksum */ + zeromem(ocb->Li, ocb->block_len); + zeromem(ocb->checksum, ocb->block_len); + + /* set other params */ + ocb->block_index = 1; + ocb->cipher = cipher; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_ntz.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_ntz.c new file mode 100644 index 0000000..b0f5570 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_ntz.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_ntz.c + OCB implementation, internal function, by Tom St Denis +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Returns the number of leading zero bits [from lsb up] + @param x The 32-bit value to observe + @return The number of bits [from the lsb up] that are zero +*/ +int ocb_ntz(unsigned long x) +{ + int c; + x &= 0xFFFFFFFFUL; + c = 0; + while ((x & 1) == 0) { + ++c; + x >>= 1; + } + return c; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_shift_xor.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_shift_xor.c new file mode 100644 index 0000000..2f7bb3b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_shift_xor.c @@ -0,0 +1,27 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_shift_xor.c + OCB implementation, internal function, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Compute the shift/xor for OCB (internal function) + @param ocb The OCB state + @param Z The destination of the shift +*/ +void ocb_shift_xor(ocb_state *ocb, unsigned char *Z) +{ + int x, y; + y = ocb_ntz(ocb->block_index++); + for (x = 0; x < ocb->block_len; x++) { + ocb->Li[x] ^= ocb->Ls[y][x]; + Z[x] = ocb->Li[x] ^ ocb->R[x]; + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_test.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_test.c new file mode 100644 index 0000000..b03c2fd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/ocb_test.c @@ -0,0 +1,205 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb_test.c + OCB implementation, self-test by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/** + Test the OCB protocol + @return CRYPT_OK if successful +*/ +int ocb_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + int ptlen; + unsigned char key[16], nonce[16], pt[34], ct[34], tag[16]; + } tests[] = { + + /* OCB-AES-128-0B */ +{ + 0, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + /* pt */ + { 0 }, + /* ct */ + { 0 }, + /* tag */ + { 0x15, 0xd3, 0x7d, 0xd7, 0xc8, 0x90, 0xd5, 0xd6, + 0xac, 0xab, 0x92, 0x7b, 0xc0, 0xdc, 0x60, 0xee }, +}, + + + /* OCB-AES-128-3B */ +{ + 3, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + /* pt */ + { 0x00, 0x01, 0x02 }, + /* ct */ + { 0xfc, 0xd3, 0x7d }, + /* tag */ + { 0x02, 0x25, 0x47, 0x39, 0xa5, 0xe3, 0x56, 0x5a, + 0xe2, 0xdc, 0xd6, 0x2c, 0x65, 0x97, 0x46, 0xba }, +}, + + /* OCB-AES-128-16B */ +{ + 16, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + /* pt */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* ct */ + { 0x37, 0xdf, 0x8c, 0xe1, 0x5b, 0x48, 0x9b, 0xf3, + 0x1d, 0x0f, 0xc4, 0x4d, 0xa1, 0xfa, 0xf6, 0xd6 }, + /* tag */ + { 0xdf, 0xb7, 0x63, 0xeb, 0xdb, 0x5f, 0x0e, 0x71, + 0x9c, 0x7b, 0x41, 0x61, 0x80, 0x80, 0x04, 0xdf }, +}, + + /* OCB-AES-128-20B */ +{ + 20, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + /* pt */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13 }, + /* ct */ + { 0x01, 0xa0, 0x75, 0xf0, 0xd8, 0x15, 0xb1, 0xa4, + 0xe9, 0xc8, 0x81, 0xa1, 0xbc, 0xff, 0xc3, 0xeb, + 0x70, 0x03, 0xeb, 0x55}, + /* tag */ + { 0x75, 0x30, 0x84, 0x14, 0x4e, 0xb6, 0x3b, 0x77, + 0x0b, 0x06, 0x3c, 0x2e, 0x23, 0xcd, 0xa0, 0xbb }, +}, + + /* OCB-AES-128-32B */ +{ + 32, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + /* pt */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + /* ct */ + { 0x01, 0xa0, 0x75, 0xf0, 0xd8, 0x15, 0xb1, 0xa4, + 0xe9, 0xc8, 0x81, 0xa1, 0xbc, 0xff, 0xc3, 0xeb, + 0x4a, 0xfc, 0xbb, 0x7f, 0xed, 0xc0, 0x8c, 0xa8, + 0x65, 0x4c, 0x6d, 0x30, 0x4d, 0x16, 0x12, 0xfa }, + + /* tag */ + { 0xc1, 0x4c, 0xbf, 0x2c, 0x1a, 0x1f, 0x1c, 0x3c, + 0x13, 0x7e, 0xad, 0xea, 0x1f, 0x2f, 0x2f, 0xcf }, +}, + + /* OCB-AES-128-34B */ +{ + 34, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* nonce */ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }, + /* pt */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21 }, + /* ct */ + { 0x01, 0xa0, 0x75, 0xf0, 0xd8, 0x15, 0xb1, 0xa4, + 0xe9, 0xc8, 0x81, 0xa1, 0xbc, 0xff, 0xc3, 0xeb, + 0xd4, 0x90, 0x3d, 0xd0, 0x02, 0x5b, 0xa4, 0xaa, + 0x83, 0x7c, 0x74, 0xf1, 0x21, 0xb0, 0x26, 0x0f, + 0xa9, 0x5d }, + + /* tag */ + { 0xcf, 0x83, 0x41, 0xbb, 0x10, 0x82, 0x0c, 0xcf, + 0x14, 0xbd, 0xec, 0x56, 0xb8, 0xd7, 0xd6, 0xab }, +}, + +}; + + int err, x, idx, res; + unsigned long len; + unsigned char outct[MAXBLOCKSIZE], outtag[MAXBLOCKSIZE]; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + len = sizeof(outtag); + if ((err = ocb_encrypt_authenticate_memory(idx, tests[x].key, 16, + tests[x].nonce, tests[x].pt, tests[x].ptlen, outct, outtag, &len)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(outtag, len, tests[x].tag, sizeof(tests[x].tag), "OCB Tag", x) || + compare_testvector(outct, tests[x].ptlen, tests[x].ct, tests[x].ptlen, "OCB CT", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + if ((err = ocb_decrypt_verify_memory(idx, tests[x].key, 16, tests[x].nonce, outct, tests[x].ptlen, + outct, tests[x].tag, len, &res)) != CRYPT_OK) { + return err; + } + if ((res != 1) || compare_testvector(outct, tests[x].ptlen, tests[x].pt, tests[x].ptlen, "OCB", x)) { +#ifdef LTC_TEST_DBG + printf("\n\nOCB: Failure-decrypt - res = %d\n", res); +#endif + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif /* LTC_TEST */ +} + +#endif /* LTC_OCB_MODE */ + + +/* some comments + + -- it's hard to seek + -- hard to stream [you can't emit ciphertext until full block] + -- The setup is somewhat complicated... +*/ diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb/s_ocb_done.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/s_ocb_done.c new file mode 100644 index 0000000..b708dfd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb/s_ocb_done.c @@ -0,0 +1,136 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file s_ocb_done.c + OCB implementation, internal helper, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB_MODE + +/* Since the last block is encrypted in CTR mode the same code can + * be used to finish a decrypt or encrypt stream. The only difference + * is we XOR the final ciphertext into the checksum so we have to xor it + * before we CTR [decrypt] or after [encrypt] + * + * the names pt/ptlen/ct really just mean in/inlen/out but this is the way I wrote it... + */ + +/** + Shared code to finish an OCB stream + @param ocb The OCB state + @param pt The remaining plaintext [or input] + @param ptlen The length of the input (octets) + @param ct [out] The output buffer + @param tag [out] The destination for the authentication tag + @param taglen [in/out] The max size and resulting size of the authentication tag + @param mode The mode we are terminating, 0==encrypt, 1==decrypt + @return CRYPT_OK if successful +*/ +int s_ocb_done(ocb_state *ocb, const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, unsigned char *tag, unsigned long *taglen, int mode) + +{ + unsigned char *Z, *Y, *X; + int err, x; + + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + return err; + } + if (ocb->block_len != cipher_descriptor[ocb->cipher]->block_length || + (int)ptlen > ocb->block_len || (int)ptlen < 0) { + return CRYPT_INVALID_ARG; + } + + /* allocate ram */ + Z = XMALLOC(MAXBLOCKSIZE); + Y = XMALLOC(MAXBLOCKSIZE); + X = XMALLOC(MAXBLOCKSIZE); + if (X == NULL || Y == NULL || Z == NULL) { + if (X != NULL) { + XFREE(X); + } + if (Y != NULL) { + XFREE(Y); + } + if (Z != NULL) { + XFREE(Z); + } + return CRYPT_MEM; + } + + /* compute X[m] = len(pt[m]) XOR Lr XOR Z[m] */ + ocb_shift_xor(ocb, X); + XMEMCPY(Z, X, ocb->block_len); + + X[ocb->block_len-1] ^= (ptlen*8)&255; + X[ocb->block_len-2] ^= ((ptlen*8)>>8)&255; + for (x = 0; x < ocb->block_len; x++) { + X[x] ^= ocb->Lr[x]; + } + + /* Y[m] = E(X[m])) */ + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(X, Y, &ocb->key)) != CRYPT_OK) { + goto error; + } + + if (mode == 1) { + /* decrypt mode, so let's xor it first */ + /* xor C[m] into checksum */ + for (x = 0; x < (int)ptlen; x++) { + ocb->checksum[x] ^= ct[x]; + } + } + + /* C[m] = P[m] xor Y[m] */ + for (x = 0; x < (int)ptlen; x++) { + ct[x] = pt[x] ^ Y[x]; + } + + if (mode == 0) { + /* encrypt mode */ + /* xor C[m] into checksum */ + for (x = 0; x < (int)ptlen; x++) { + ocb->checksum[x] ^= ct[x]; + } + } + + /* xor Y[m] and Z[m] into checksum */ + for (x = 0; x < ocb->block_len; x++) { + ocb->checksum[x] ^= Y[x] ^ Z[x]; + } + + /* encrypt checksum, er... tag!! */ + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(ocb->checksum, X, &ocb->key)) != CRYPT_OK) { + goto error; + } + cipher_descriptor[ocb->cipher]->done(&ocb->key); + + /* now store it */ + for (x = 0; x < ocb->block_len && x < (int)*taglen; x++) { + tag[x] = X[x]; + } + *taglen = x; + +#ifdef LTC_CLEAN_STACK + zeromem(X, MAXBLOCKSIZE); + zeromem(Y, MAXBLOCKSIZE); + zeromem(Z, MAXBLOCKSIZE); + zeromem(ocb, sizeof(*ocb)); +#endif +error: + XFREE(X); + XFREE(Y); + XFREE(Z); + + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_add_aad.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_add_aad.c new file mode 100644 index 0000000..9a3811a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_add_aad.c @@ -0,0 +1,96 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_add_aad.c + OCB implementation, add AAD data, by Karel Miko +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Add one block of AAD data (internal function) + @param ocb The OCB state + @param aad_block [in] AAD data (block_len size) + @return CRYPT_OK if successful +*/ +static int s_ocb3_int_aad_add_block(ocb3_state *ocb, const unsigned char *aad_block) +{ + unsigned char tmp[MAXBLOCKSIZE]; + int err; + + /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ + ocb3_int_xor_blocks(ocb->aOffset_current, ocb->aOffset_current, ocb->L_[ocb3_int_ntz(ocb->ablock_index)], ocb->block_len); + + /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ + ocb3_int_xor_blocks(tmp, aad_block, ocb->aOffset_current, ocb->block_len); + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(tmp, tmp, &ocb->key)) != CRYPT_OK) { + return err; + } + ocb3_int_xor_blocks(ocb->aSum_current, ocb->aSum_current, tmp, ocb->block_len); + + ocb->ablock_index++; + + return CRYPT_OK; +} + +/** + Add AAD - additional associated data + @param ocb The OCB state + @param aad The AAD data + @param aadlen The size of AAD data (octets) + @return CRYPT_OK if successful +*/ +int ocb3_add_aad(ocb3_state *ocb, const unsigned char *aad, unsigned long aadlen) +{ + int err, x, full_blocks, full_blocks_len, last_block_len; + unsigned char *data; + unsigned long datalen, l; + + LTC_ARGCHK(ocb != NULL); + if (aadlen == 0) return CRYPT_OK; + LTC_ARGCHK(aad != NULL); + + if (ocb->adata_buffer_bytes > 0) { + l = ocb->block_len - ocb->adata_buffer_bytes; + if (l > aadlen) l = aadlen; + XMEMCPY(ocb->adata_buffer+ocb->adata_buffer_bytes, aad, l); + ocb->adata_buffer_bytes += l; + + if (ocb->adata_buffer_bytes == ocb->block_len) { + if ((err = s_ocb3_int_aad_add_block(ocb, ocb->adata_buffer)) != CRYPT_OK) { + return err; + } + ocb->adata_buffer_bytes = 0; + } + + data = (unsigned char *)aad + l; + datalen = aadlen - l; + } + else { + data = (unsigned char *)aad; + datalen = aadlen; + } + + if (datalen == 0) return CRYPT_OK; + + full_blocks = datalen/ocb->block_len; + full_blocks_len = full_blocks * ocb->block_len; + last_block_len = datalen - full_blocks_len; + + for (x=0; xblock_len)) != CRYPT_OK) { + return err; + } + } + + if (last_block_len>0) { + XMEMCPY(ocb->adata_buffer, data+full_blocks_len, last_block_len); + ocb->adata_buffer_bytes = last_block_len; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt.c new file mode 100644 index 0000000..068a53e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt.c @@ -0,0 +1,76 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_decrypt.c + OCB implementation, decrypt data, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Decrypt blocks of ciphertext with OCB + @param ocb The OCB state + @param ct The ciphertext (length multiple of the block size of the block cipher) + @param ctlen The length of the input (octets) + @param pt [out] The plaintext (length of ct) + @return CRYPT_OK if successful +*/ +int ocb3_decrypt(ocb3_state *ocb, const unsigned char *ct, unsigned long ctlen, unsigned char *pt) +{ + unsigned char tmp[MAXBLOCKSIZE]; + int err, i, full_blocks; + unsigned char *pt_b, *ct_b; + + LTC_ARGCHK(ocb != NULL); + if (ctlen == 0) return CRYPT_OK; /* no data, nothing to do */ + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(pt != NULL); + + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + return err; + } + if (ocb->block_len != cipher_descriptor[ocb->cipher]->block_length) { + return CRYPT_INVALID_ARG; + } + + if (ctlen % ocb->block_len) { /* ctlen has to bu multiple of block_len */ + return CRYPT_INVALID_ARG; + } + + full_blocks = ctlen/ocb->block_len; + for(i=0; iblock_len; + ct_b = (unsigned char *)ct+i*ocb->block_len; + + /* ocb->Offset_current[] = ocb->Offset_current[] ^ Offset_{ntz(block_index)} */ + ocb3_int_xor_blocks(ocb->Offset_current, ocb->Offset_current, ocb->L_[ocb3_int_ntz(ocb->block_index)], ocb->block_len); + + /* tmp[] = ct[] XOR ocb->Offset_current[] */ + ocb3_int_xor_blocks(tmp, ct_b, ocb->Offset_current, ocb->block_len); + + /* decrypt */ + if ((err = cipher_descriptor[ocb->cipher]->ecb_decrypt(tmp, tmp, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* pt[] = tmp[] XOR ocb->Offset_current[] */ + ocb3_int_xor_blocks(pt_b, tmp, ocb->Offset_current, ocb->block_len); + + /* ocb->checksum[] = ocb->checksum[] XOR pt[] */ + ocb3_int_xor_blocks(ocb->checksum, ocb->checksum, pt_b, ocb->block_len); + + ocb->block_index++; + } + + err = CRYPT_OK; + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(tmp, sizeof(tmp)); +#endif + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt_last.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt_last.c new file mode 100644 index 0000000..49bc5e8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt_last.c @@ -0,0 +1,101 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_decrypt_last.c + OCB implementation, internal helper, by Karel Miko +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Finish an OCB (decryption) stream + @param ocb The OCB state + @param ct The remaining ciphertext + @param ctlen The length of the ciphertext (octets) + @param pt [out] The output buffer + @return CRYPT_OK if successful +*/ +int ocb3_decrypt_last(ocb3_state *ocb, const unsigned char *ct, unsigned long ctlen, unsigned char *pt) +{ + unsigned char iOffset_star[MAXBLOCKSIZE]; + unsigned char iPad[MAXBLOCKSIZE]; + int err, x, full_blocks, full_blocks_len, last_block_len; + + LTC_ARGCHK(ocb != NULL); + if (ct == NULL) LTC_ARGCHK(ctlen == 0); + if (ctlen != 0) { + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(pt != NULL); + } + + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + goto LBL_ERR; + } + + full_blocks = ctlen/ocb->block_len; + full_blocks_len = full_blocks * ocb->block_len; + last_block_len = ctlen - full_blocks_len; + + /* process full blocks first */ + if (full_blocks>0) { + if ((err = ocb3_decrypt(ocb, ct, full_blocks_len, pt)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + if (last_block_len>0) { + /* Offset_* = Offset_m xor L_* */ + ocb3_int_xor_blocks(iOffset_star, ocb->Offset_current, ocb->L_star, ocb->block_len); + + /* Pad = ENCIPHER(K, Offset_*) */ + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(iOffset_star, iPad, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* P_* = C_* xor Pad[1..bitlen(C_*)] */ + ocb3_int_xor_blocks(pt+full_blocks_len, (unsigned char *)ct+full_blocks_len, iPad, last_block_len); + + /* Checksum_* = Checksum_m xor (P_* || 1 || zeros(127-bitlen(P_*))) */ + ocb3_int_xor_blocks(ocb->checksum, ocb->checksum, pt+full_blocks_len, last_block_len); + for(x=last_block_len; xblock_len; x++) { + if (x == last_block_len) { + ocb->checksum[x] ^= 0x80; + } else { + ocb->checksum[x] ^= 0x00; + } + } + + /* Tag = ENCIPHER(K, Checksum_* xor Offset_* xor L_$) xor HASH(K,A) */ + /* at this point we calculate only: Tag_part = ENCIPHER(K, Checksum_* xor Offset_* xor L_$) */ + for(x=0; xblock_len; x++) { + ocb->tag_part[x] = (ocb->checksum[x] ^ iOffset_star[x]) ^ ocb->L_dollar[x]; + } + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(ocb->tag_part, ocb->tag_part, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + } + else { + /* Tag = ENCIPHER(K, Checksum_m xor Offset_m xor L_$) xor HASH(K,A) */ + /* at this point we calculate only: Tag_part = ENCIPHER(K, Checksum_m xor Offset_m xor L_$) */ + for(x=0; xblock_len; x++) { + ocb->tag_part[x] = (ocb->checksum[x] ^ ocb->Offset_current[x]) ^ ocb->L_dollar[x]; + } + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(ocb->tag_part, ocb->tag_part, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + err = CRYPT_OK; + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(iOffset_star, MAXBLOCKSIZE); + zeromem(iPad, MAXBLOCKSIZE); +#endif + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt_verify_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt_verify_memory.c new file mode 100644 index 0000000..03d33b2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_decrypt_verify_memory.c @@ -0,0 +1,100 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_decrypt_verify_memory.c + OCB implementation, helper to decrypt block of memory, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Decrypt and compare the tag with OCB + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param nonce The session nonce (length of the block size of the block cipher) + @param noncelen The length of the nonce (octets) + @param adata The AAD - additional associated data + @param adatalen The length of AAD (octets) + @param ct The ciphertext + @param ctlen The length of the ciphertext (octets) + @param pt [out] The plaintext + @param tag The tag to compare against + @param taglen The length of the tag (octets) + @param stat [out] The result of the tag comparison (1==valid, 0==invalid) + @return CRYPT_OK if successful regardless of the tag comparison +*/ +int ocb3_decrypt_verify_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *adata, unsigned long adatalen, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, + int *stat) +{ + int err; + ocb3_state *ocb; + unsigned char *buf; + unsigned long buflen; + + LTC_ARGCHK(stat != NULL); + + /* default to zero */ + *stat = 0; + + /* limit taglen */ + taglen = MIN(taglen, MAXBLOCKSIZE); + + /* allocate memory */ + buf = XMALLOC(taglen); + ocb = XMALLOC(sizeof(ocb3_state)); + if (ocb == NULL || buf == NULL) { + if (ocb != NULL) { + XFREE(ocb); + } + if (buf != NULL) { + XFREE(buf); + } + return CRYPT_MEM; + } + + if ((err = ocb3_init(ocb, cipher, key, keylen, nonce, noncelen, taglen)) != CRYPT_OK) { + goto LBL_ERR; + } + + if (adata != NULL || adatalen != 0) { + if ((err = ocb3_add_aad(ocb, adata, adatalen)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + if ((err = ocb3_decrypt_last(ocb, ct, ctlen, pt)) != CRYPT_OK) { + goto LBL_ERR; + } + + buflen = taglen; + if ((err = ocb3_done(ocb, buf, &buflen)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* compare tags */ + if (buflen >= taglen && XMEM_NEQ(buf, tag, taglen) == 0) { + *stat = 1; + } + + err = CRYPT_OK; + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(ocb, sizeof(ocb3_state)); +#endif + + XFREE(ocb); + XFREE(buf); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_done.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_done.c new file mode 100644 index 0000000..074d4c1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_done.c @@ -0,0 +1,82 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_done.c + OCB implementation, INTERNAL ONLY helper, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Finish OCB processing and compute the tag + @param ocb The OCB state + @param tag [out] The destination for the authentication tag + @param taglen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful +*/ +int ocb3_done(ocb3_state *ocb, unsigned char *tag, unsigned long *taglen) +{ + unsigned char tmp[MAXBLOCKSIZE]; + int err, x; + + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(tag != NULL); + LTC_ARGCHK(taglen != NULL); + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* check taglen */ + if ((int)*taglen < ocb->tag_len) { + *taglen = (unsigned long)ocb->tag_len; + return CRYPT_BUFFER_OVERFLOW; + } + + /* finalize AAD processing */ + + if (ocb->adata_buffer_bytes>0) { + /* Offset_* = Offset_m xor L_* */ + ocb3_int_xor_blocks(ocb->aOffset_current, ocb->aOffset_current, ocb->L_star, ocb->block_len); + + /* CipherInput = (A_* || 1 || zeros(127-bitlen(A_*))) xor Offset_* */ + ocb3_int_xor_blocks(tmp, ocb->adata_buffer, ocb->aOffset_current, ocb->adata_buffer_bytes); + for(x=ocb->adata_buffer_bytes; xblock_len; x++) { + if (x == ocb->adata_buffer_bytes) { + tmp[x] = 0x80 ^ ocb->aOffset_current[x]; + } + else { + tmp[x] = 0x00 ^ ocb->aOffset_current[x]; + } + } + + /* Sum = Sum_m xor ENCIPHER(K, CipherInput) */ + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(tmp, tmp, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + ocb3_int_xor_blocks(ocb->aSum_current, ocb->aSum_current, tmp, ocb->block_len); + } + + /* finalize TAG computing */ + + /* at this point ocb->aSum_current = HASH(K, A) */ + /* tag = tag ^ HASH(K, A) */ + ocb3_int_xor_blocks(tmp, ocb->tag_part, ocb->aSum_current, ocb->block_len); + + /* copy tag bytes */ + for(x = 0; x < ocb->tag_len; x++) tag[x] = tmp[x]; + *taglen = (unsigned long)ocb->tag_len; + + err = CRYPT_OK; + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(tmp, MAXBLOCKSIZE); + zeromem(ocb, sizeof(*ocb)); +#endif + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt.c new file mode 100644 index 0000000..317f6f3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt.c @@ -0,0 +1,76 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_encrypt.c + OCB implementation, encrypt data, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Encrypt blocks of data with OCB + @param ocb The OCB state + @param pt The plaintext (length multiple of the block size of the block cipher) + @param ptlen The length of the input (octets) + @param ct [out] The ciphertext (same size as the pt) + @return CRYPT_OK if successful +*/ +int ocb3_encrypt(ocb3_state *ocb, const unsigned char *pt, unsigned long ptlen, unsigned char *ct) +{ + unsigned char tmp[MAXBLOCKSIZE]; + int err, i, full_blocks; + unsigned char *pt_b, *ct_b; + + LTC_ARGCHK(ocb != NULL); + if (ptlen == 0) return CRYPT_OK; /* no data, nothing to do */ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + return err; + } + if (ocb->block_len != cipher_descriptor[ocb->cipher]->block_length) { + return CRYPT_INVALID_ARG; + } + + if (ptlen % ocb->block_len) { /* ptlen has to bu multiple of block_len */ + return CRYPT_INVALID_ARG; + } + + full_blocks = ptlen/ocb->block_len; + for(i=0; iblock_len; + ct_b = (unsigned char *)ct+i*ocb->block_len; + + /* ocb->Offset_current[] = ocb->Offset_current[] ^ Offset_{ntz(block_index)} */ + ocb3_int_xor_blocks(ocb->Offset_current, ocb->Offset_current, ocb->L_[ocb3_int_ntz(ocb->block_index)], ocb->block_len); + + /* tmp[] = pt[] XOR ocb->Offset_current[] */ + ocb3_int_xor_blocks(tmp, pt_b, ocb->Offset_current, ocb->block_len); + + /* encrypt */ + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(tmp, tmp, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* ct[] = tmp[] XOR ocb->Offset_current[] */ + ocb3_int_xor_blocks(ct_b, tmp, ocb->Offset_current, ocb->block_len); + + /* ocb->checksum[] = ocb->checksum[] XOR pt[] */ + ocb3_int_xor_blocks(ocb->checksum, ocb->checksum, pt_b, ocb->block_len); + + ocb->block_index++; + } + + err = CRYPT_OK; + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(tmp, sizeof(tmp)); +#endif + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt_authenticate_memory.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt_authenticate_memory.c new file mode 100644 index 0000000..b118925 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt_authenticate_memory.c @@ -0,0 +1,72 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_encrypt_authenticate_memory.c + OCB implementation, encrypt block of memory, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Encrypt and generate an authentication code for a buffer of memory + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param nonce The session nonce (length of the block ciphers block size) + @param noncelen The length of the nonce (octets) + @param adata The AAD - additional associated data + @param adatalen The length of AAD (octets) + @param pt The plaintext + @param ptlen The length of the plaintext (octets) + @param ct [out] The ciphertext + @param tag [out] The authentication tag + @param taglen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful +*/ +int ocb3_encrypt_authenticate_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *adata, unsigned long adatalen, + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen) +{ + int err; + ocb3_state *ocb; + + LTC_ARGCHK(taglen != NULL); + + /* allocate memory */ + ocb = XMALLOC(sizeof(ocb3_state)); + if (ocb == NULL) { + return CRYPT_MEM; + } + + if ((err = ocb3_init(ocb, cipher, key, keylen, nonce, noncelen, *taglen)) != CRYPT_OK) { + goto LBL_ERR; + } + + if (adata != NULL || adatalen != 0) { + if ((err = ocb3_add_aad(ocb, adata, adatalen)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + if ((err = ocb3_encrypt_last(ocb, pt, ptlen, ct)) != CRYPT_OK) { + goto LBL_ERR; + } + + err = ocb3_done(ocb, tag, taglen); + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(ocb, sizeof(ocb3_state)); +#endif + + XFREE(ocb); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt_last.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt_last.c new file mode 100644 index 0000000..5aab93b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_encrypt_last.c @@ -0,0 +1,102 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_encrypt_last.c + OCB implementation, internal helper, by Karel Miko +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Finish an OCB (encryption) stream + @param ocb The OCB state + @param pt The remaining plaintext + @param ptlen The length of the plaintext (octets) + @param ct [out] The output buffer + @return CRYPT_OK if successful +*/ +int ocb3_encrypt_last(ocb3_state *ocb, const unsigned char *pt, unsigned long ptlen, unsigned char *ct) +{ + unsigned char iOffset_star[MAXBLOCKSIZE]; + unsigned char iPad[MAXBLOCKSIZE]; + int err, x, full_blocks, full_blocks_len, last_block_len; + + LTC_ARGCHK(ocb != NULL); + if (pt == NULL) LTC_ARGCHK(ptlen == 0); + if (ptlen != 0) { + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + } + + if ((err = cipher_is_valid(ocb->cipher)) != CRYPT_OK) { + goto LBL_ERR; + } + + full_blocks = ptlen/ocb->block_len; + full_blocks_len = full_blocks * ocb->block_len; + last_block_len = ptlen - full_blocks_len; + + /* process full blocks first */ + if (full_blocks>0) { + if ((err = ocb3_encrypt(ocb, pt, full_blocks_len, ct)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + /* at this point: m = ocb->block_index (last block index), Offset_m = ocb->Offset_current */ + + if (last_block_len>0) { + /* Offset_* = Offset_m xor L_* */ + ocb3_int_xor_blocks(iOffset_star, ocb->Offset_current, ocb->L_star, ocb->block_len); + + /* Pad = ENCIPHER(K, Offset_*) */ + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(iOffset_star, iPad, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* C_* = P_* xor Pad[1..bitlen(P_*)] */ + ocb3_int_xor_blocks(ct+full_blocks_len, pt+full_blocks_len, iPad, last_block_len); + + /* Checksum_* = Checksum_m xor (P_* || 1 || zeros(127-bitlen(P_*))) */ + ocb3_int_xor_blocks(ocb->checksum, ocb->checksum, pt+full_blocks_len, last_block_len); + for(x=last_block_len; xblock_len; x++) { + if (x == last_block_len) { + ocb->checksum[x] ^= 0x80; + } else { + ocb->checksum[x] ^= 0x00; + } + } + + /* Tag = ENCIPHER(K, Checksum_* xor Offset_* xor L_$) xor HASH(K,A) */ + /* at this point we calculate only: Tag_part = ENCIPHER(K, Checksum_* xor Offset_* xor L_$) */ + for(x=0; xblock_len; x++) { + ocb->tag_part[x] = (ocb->checksum[x] ^ iOffset_star[x]) ^ ocb->L_dollar[x]; + } + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(ocb->tag_part, ocb->tag_part, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + } else { + /* Tag = ENCIPHER(K, Checksum_m xor Offset_m xor L_$) xor HASH(K,A) */ + /* at this point we calculate only: Tag_part = ENCIPHER(K, Checksum_m xor Offset_m xor L_$) */ + for(x=0; xblock_len; x++) { + ocb->tag_part[x] = (ocb->checksum[x] ^ ocb->Offset_current[x]) ^ ocb->L_dollar[x]; + } + if ((err = cipher_descriptor[ocb->cipher]->ecb_encrypt(ocb->tag_part, ocb->tag_part, &ocb->key)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + err = CRYPT_OK; + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(iOffset_star, MAXBLOCKSIZE); + zeromem(iPad, MAXBLOCKSIZE); +#endif + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_init.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_init.c new file mode 100644 index 0000000..4882d64 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_init.c @@ -0,0 +1,186 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_init.c + OCB implementation, initialize state, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +static void s_ocb3_int_calc_offset_zero(ocb3_state *ocb, const unsigned char *nonce, unsigned long noncelen, unsigned long taglen) +{ + int x, y, bottom; + int idx, shift; + unsigned char iNonce[MAXBLOCKSIZE]; + unsigned char iKtop[MAXBLOCKSIZE]; + unsigned char iStretch[MAXBLOCKSIZE+8]; + + /* Nonce = zeros(127-bitlen(N)) || 1 || N */ + zeromem(iNonce, sizeof(iNonce)); + for (x = ocb->block_len-1, y=0; y<(int)noncelen; x--, y++) { + iNonce[x] = nonce[noncelen-y-1]; + } + iNonce[x] = 0x01; + iNonce[0] |= ((taglen*8) % 128) << 1; + + /* bottom = str2num(Nonce[123..128]) */ + bottom = iNonce[ocb->block_len-1] & 0x3F; + + /* Ktop = ENCIPHER(K, Nonce[1..122] || zeros(6)) */ + iNonce[ocb->block_len-1] = iNonce[ocb->block_len-1] & 0xC0; + if ((cipher_descriptor[ocb->cipher]->ecb_encrypt(iNonce, iKtop, &ocb->key)) != CRYPT_OK) { + zeromem(ocb->Offset_current, ocb->block_len); + return; + } + + /* Stretch = Ktop || (Ktop[1..64] xor Ktop[9..72]) */ + for (x = 0; x < ocb->block_len; x++) { + iStretch[x] = iKtop[x]; + } + for (y = 0; y < 8; y++) { + iStretch[x+y] = iKtop[y] ^ iKtop[y+1]; + } + + /* Offset_0 = Stretch[1+bottom..128+bottom] */ + idx = bottom / 8; + shift = (bottom % 8); + for (x = 0; x < ocb->block_len; x++) { + ocb->Offset_current[x] = iStretch[idx+x] << shift; + if (shift > 0) { + ocb->Offset_current[x] |= iStretch[idx+x+1] >> (8-shift); + } + } +} + +static const struct { + int len; + unsigned char poly_mul[MAXBLOCKSIZE]; +} polys[] = { +{ + 8, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1B } +}, { + 16, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87 } +} +}; + +/** + Initialize an OCB context + @param ocb [out] The destination of the OCB state + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @param nonce The session nonce + @param noncelen The length of the session nonce (octets, up to 15) + @param taglen The length of the tag (octets, up to 16) + @return CRYPT_OK if successful +*/ +int ocb3_init(ocb3_state *ocb, int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + unsigned long taglen) +{ + int poly, x, y, m, err; + unsigned char *previous, *current; + + LTC_ARGCHK(ocb != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(nonce != NULL); + + /* valid cipher? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + ocb->cipher = cipher; + + /* Valid Nonce? + * As of RFC7253: "string of no more than 120 bits" */ + if (noncelen > (120/8)) { + return CRYPT_INVALID_ARG; + } + + /* The blockcipher must have a 128-bit blocksize */ + if (cipher_descriptor[cipher]->block_length != 16) { + return CRYPT_INVALID_ARG; + } + + /* The TAGLEN may be any value up to 128 (bits) */ + if (taglen > 16) { + return CRYPT_INVALID_ARG; + } + ocb->tag_len = taglen; + + /* determine which polys to use */ + ocb->block_len = cipher_descriptor[cipher]->block_length; + x = (int)(sizeof(polys)/sizeof(polys[0])); + for (poly = 0; poly < x; poly++) { + if (polys[poly].len == ocb->block_len) { + break; + } + } + if (poly == x) { + return CRYPT_INVALID_ARG; /* block_len not found in polys */ + } + if (polys[poly].len != ocb->block_len) { + return CRYPT_INVALID_ARG; + } + + /* schedule the key */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, &ocb->key)) != CRYPT_OK) { + return err; + } + + /* L_* = ENCIPHER(K, zeros(128)) */ + zeromem(ocb->L_star, ocb->block_len); + if ((err = cipher_descriptor[cipher]->ecb_encrypt(ocb->L_star, ocb->L_star, &ocb->key)) != CRYPT_OK) { + return err; + } + + /* compute L_$, L_0, L_1, ... */ + for (x = -1; x < 32; x++) { + if (x == -1) { /* gonna compute: L_$ = double(L_*) */ + current = ocb->L_dollar; + previous = ocb->L_star; + } + else if (x == 0) { /* gonna compute: L_0 = double(L_$) */ + current = ocb->L_[0]; + previous = ocb->L_dollar; + } + else { /* gonna compute: L_i = double(L_{i-1}) for every integer i > 0 */ + current = ocb->L_[x]; + previous = ocb->L_[x-1]; + } + m = previous[0] >> 7; + for (y = 0; y < ocb->block_len-1; y++) { + current[y] = ((previous[y] << 1) | (previous[y+1] >> 7)) & 255; + } + current[ocb->block_len-1] = (previous[ocb->block_len-1] << 1) & 255; + if (m == 1) { + /* current[] = current[] XOR polys[poly].poly_mul[]*/ + ocb3_int_xor_blocks(current, current, polys[poly].poly_mul, ocb->block_len); + } + } + + /* initialize ocb->Offset_current = Offset_0 */ + s_ocb3_int_calc_offset_zero(ocb, nonce, noncelen, taglen); + + /* initialize checksum to all zeros */ + zeromem(ocb->checksum, ocb->block_len); + + /* set block index */ + ocb->block_index = 1; + + /* initialize AAD related stuff */ + ocb->ablock_index = 1; + ocb->adata_buffer_bytes = 0; + zeromem(ocb->aOffset_current, ocb->block_len); + zeromem(ocb->aSum_current, ocb->block_len); + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_int_ntz.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_int_ntz.c new file mode 100644 index 0000000..86942ce --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_int_ntz.c @@ -0,0 +1,29 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_int_ntz.c + OCB implementation, INTERNAL ONLY helper, by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Returns the number of leading zero bits [from lsb up] (internal function) + @param x The 32-bit value to observe + @return The number of bits [from the lsb up] that are zero +*/ +int ocb3_int_ntz(unsigned long x) +{ + int c; + x &= 0xFFFFFFFFUL; + c = 0; + while ((x & 1) == 0) { + ++c; + x >>= 1; + } + return c; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_int_xor_blocks.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_int_xor_blocks.c new file mode 100644 index 0000000..f9f342a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_int_xor_blocks.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_int_xor_blocks.c + OCB implementation, INTERNAL ONLY helper, by Karel Miko +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Compute xor for two blocks of bytes 'out = block_a XOR block_b' (internal function) + @param out The block of bytes (output) + @param block_a The block of bytes (input) + @param block_b The block of bytes (input) + @param block_len The size of block_a, block_b, out +*/ +void ocb3_int_xor_blocks(unsigned char *out, const unsigned char *block_a, const unsigned char *block_b, unsigned long block_len) +{ + int x; + if (out == block_a) { + for (x = 0; x < (int)block_len; x++) out[x] ^= block_b[x]; + } + else { + for (x = 0; x < (int)block_len; x++) out[x] = block_a[x] ^ block_b[x]; + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_test.c b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_test.c new file mode 100644 index 0000000..3a9816e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/ocb3/ocb3_test.c @@ -0,0 +1,299 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file ocb3_test.c + OCB implementation, self-test by Tom St Denis +*/ +#include "tomcrypt_private.h" + +#ifdef LTC_OCB3_MODE + +/** + Test the OCB protocol + @return CRYPT_OK if successful +*/ +int ocb3_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + /* test vectors from: http://tools.ietf.org/html/draft-krovetz-ocb-03 */ + unsigned char key[16] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F }; + unsigned char nonce[12] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B }; + const struct { + int ptlen; + int aadlen; + unsigned char pt[64], aad[64], ct[64], tag[16]; + } tests[] = { + + { /* index:0 */ + 0, /* PLAINTEXT length */ + 0, /* AAD length */ + { 0 }, /* PLAINTEXT */ + { 0 }, /* AAD */ + { 0 }, /* CIPHERTEXT */ + { 0x19,0x7b,0x9c,0x3c,0x44,0x1d,0x3c,0x83,0xea,0xfb,0x2b,0xef,0x63,0x3b,0x91,0x82 }, /* TAG */ + }, + { /* index:1 */ + 8, /* PLAINTEXT length */ + 8, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 }, /* AAD */ + { 0x92,0xb6,0x57,0x13,0x0a,0x74,0xb8,0x5a }, /* CIPHERTEXT */ + { 0x16,0xdc,0x76,0xa4,0x6d,0x47,0xe1,0xea,0xd5,0x37,0x20,0x9e,0x8a,0x96,0xd1,0x4e }, /* TAG */ + }, + { /* index:2 */ + 0, /* PLAINTEXT length */ + 8, /* AAD length */ + { 0 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 }, /* AAD */ + { 0 }, /* CIPHERTEXT */ + { 0x98,0xb9,0x15,0x52,0xc8,0xc0,0x09,0x18,0x50,0x44,0xe3,0x0a,0x6e,0xb2,0xfe,0x21 }, /* TAG */ + }, + { /* index:3 */ + 8, /* PLAINTEXT length */ + 0, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 }, /* PLAINTEXT */ + { 0 }, /* AAD */ + { 0x92,0xb6,0x57,0x13,0x0a,0x74,0xb8,0x5a }, /* CIPHERTEXT */ + { 0x97,0x1e,0xff,0xca,0xe1,0x9a,0xd4,0x71,0x6f,0x88,0xe8,0x7b,0x87,0x1f,0xbe,0xed }, /* TAG */ + }, + { /* index:4 */ + 16, /* PLAINTEXT length */ + 16, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22 }, /* CIPHERTEXT */ + { 0x77,0x6c,0x99,0x24,0xd6,0x72,0x3a,0x1f,0xc4,0x52,0x45,0x32,0xac,0x3e,0x5b,0xeb }, /* TAG */ + }, + { /* index:5 */ + 0, /* PLAINTEXT length */ + 16, /* AAD length */ + { 0 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f }, /* AAD */ + { 0 }, /* CIPHERTEXT */ + { 0x7d,0xdb,0x8e,0x6c,0xea,0x68,0x14,0x86,0x62,0x12,0x50,0x96,0x19,0xb1,0x9c,0xc6 }, /* TAG */ + }, + { /* index:6 */ + 16, /* PLAINTEXT length */ + 0, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f }, /* PLAINTEXT */ + { 0 }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22 }, /* CIPHERTEXT */ + { 0x13,0xcc,0x8b,0x74,0x78,0x07,0x12,0x1a,0x4c,0xbb,0x3e,0x4b,0xd6,0xb4,0x56,0xaf }, /* TAG */ + }, + { /* index:7 */ + 24, /* PLAINTEXT length */ + 24, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17 }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22,0xfc,0xfc,0xee,0x7a,0x2a,0x8d,0x4d,0x48 }, /* CIPHERTEXT */ + { 0x5f,0xa9,0x4f,0xc3,0xf3,0x88,0x20,0xf1,0xdc,0x3f,0x3d,0x1f,0xd4,0xe5,0x5e,0x1c }, /* TAG */ + }, + { /* index:8 */ + 0, /* PLAINTEXT length */ + 24, /* AAD length */ + { 0 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17 }, /* AAD */ + { 0 }, /* CIPHERTEXT */ + { 0x28,0x20,0x26,0xda,0x30,0x68,0xbc,0x9f,0xa1,0x18,0x68,0x1d,0x55,0x9f,0x10,0xf6 }, /* TAG */ + }, + { /* index:9 */ + 24, /* PLAINTEXT length */ + 0, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17 }, /* PLAINTEXT */ + { 0 }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22,0xfc,0xfc,0xee,0x7a,0x2a,0x8d,0x4d,0x48 }, /* CIPHERTEXT */ + { 0x6e,0xf2,0xf5,0x25,0x87,0xfd,0xa0,0xed,0x97,0xdc,0x7e,0xed,0xe2,0x41,0xdf,0x68 }, /* TAG */ + }, + { /* index:10 */ + 32, /* PLAINTEXT length */ + 32, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22,0xce,0xaa,0xb9,0xb0,0x5d,0xf7,0x71,0xa6,0x57,0x14,0x9d,0x53,0x77,0x34,0x63,0xcb }, /* CIPHERTEXT */ + { 0xb2,0xa0,0x40,0xdd,0x3b,0xd5,0x16,0x43,0x72,0xd7,0x6d,0x7b,0xb6,0x82,0x42,0x40 }, /* TAG */ + }, + { /* index:11 */ + 0, /* PLAINTEXT length */ + 32, /* AAD length */ + { 0 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f }, /* AAD */ + { 0 }, /* CIPHERTEXT */ + { 0xe1,0xe0,0x72,0x63,0x3b,0xad,0xe5,0x1a,0x60,0xe8,0x59,0x51,0xd9,0xc4,0x2a,0x1b }, /* TAG */ + }, + { /* index:12 */ + 32, /* PLAINTEXT length */ + 0, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f }, /* PLAINTEXT */ + { 0 }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22,0xce,0xaa,0xb9,0xb0,0x5d,0xf7,0x71,0xa6,0x57,0x14,0x9d,0x53,0x77,0x34,0x63,0xcb }, /* CIPHERTEXT */ + { 0x4a,0x3b,0xae,0x82,0x44,0x65,0xcf,0xda,0xf8,0xc4,0x1f,0xc5,0x0c,0x7d,0xf9,0xd9 }, /* TAG */ + }, + { /* index:13 */ + 40, /* PLAINTEXT length */ + 40, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27 }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22,0xce,0xaa,0xb9,0xb0,0x5d,0xf7,0x71,0xa6,0x57,0x14,0x9d,0x53,0x77,0x34,0x63,0xcb,0x68,0xc6,0x57,0x78,0xb0,0x58,0xa6,0x35 }, /* CIPHERTEXT */ + { 0x65,0x9c,0x62,0x32,0x11,0xde,0xea,0x0d,0xe3,0x0d,0x2c,0x38,0x18,0x79,0xf4,0xc8 }, /* TAG */ + }, + { /* index:14 */ + 0, /* PLAINTEXT length */ + 40, /* AAD length */ + { 0 }, /* PLAINTEXT */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27 }, /* AAD */ + { 0 }, /* CIPHERTEXT */ + { 0x7a,0xeb,0x7a,0x69,0xa1,0x68,0x7d,0xd0,0x82,0xca,0x27,0xb0,0xd9,0xa3,0x70,0x96 }, /* TAG */ + }, + { /* index:15 */ + 40, /* PLAINTEXT length */ + 0, /* AAD length */ + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27 }, /* PLAINTEXT */ + { 0 }, /* AAD */ + { 0xbe,0xa5,0xe8,0x79,0x8d,0xbe,0x71,0x10,0x03,0x1c,0x14,0x4d,0xa0,0xb2,0x61,0x22,0xce,0xaa,0xb9,0xb0,0x5d,0xf7,0x71,0xa6,0x57,0x14,0x9d,0x53,0x77,0x34,0x63,0xcb,0x68,0xc6,0x57,0x78,0xb0,0x58,0xa6,0x35 }, /* CIPHERTEXT */ + { 0x06,0x0c,0x84,0x67,0xf4,0xab,0xab,0x5e,0x8b,0x3c,0x20,0x67,0xa2,0xe1,0x15,0xdc }, /* TAG */ + }, + +}; + /* As of RFC 7253 - 'Appendix A. Sample Results' + * The next tuple shows a result with a tag length of 96 bits and a + different key. + + K: 0F0E0D0C0B0A09080706050403020100 + + N: BBAA9988776655443322110D + A: 000102030405060708090A0B0C0D0E0F1011121314151617 + 18191A1B1C1D1E1F2021222324252627 + P: 000102030405060708090A0B0C0D0E0F1011121314151617 + 18191A1B1C1D1E1F2021222324252627 + C: 1792A4E31E0755FB03E31B22116E6C2DDF9EFD6E33D536F1 + A0124B0A55BAE884ED93481529C76B6AD0C515F4D1CDD4FD + AC4F02AA + + The C has been split up in C and T (tag) + */ + const unsigned char K[] = { 0x0F,0x0E,0x0D,0x0C,0x0B,0x0A,0x09,0x08, + 0x07,0x06,0x05,0x04,0x03,0x02,0x01,0x00 }; + const unsigned char N[] = { 0xBB,0xAA,0x99,0x88,0x77,0x66,0x55,0x44, + 0x33,0x22,0x11,0x0D }; + const unsigned char A[] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07, + 0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17, + 0x18,0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F, + 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27 }; + const unsigned char P[] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07, + 0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17, + 0x18,0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F, + 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27 }; + const unsigned char C[] = { 0x17,0x92,0xA4,0xE3,0x1E,0x07,0x55,0xFB, + 0x03,0xE3,0x1B,0x22,0x11,0x6E,0x6C,0x2D, + 0xDF,0x9E,0xFD,0x6E,0x33,0xD5,0x36,0xF1, + 0xA0,0x12,0x4B,0x0A,0x55,0xBA,0xE8,0x84, + 0xED,0x93,0x48,0x15,0x29,0xC7,0x6B,0x6A }; + const unsigned char T[] = { 0xD0,0xC5,0x15,0xF4,0xD1,0xCD,0xD4,0xFD, + 0xAC,0x4F,0x02,0xAA }; + + int err, x, idx, res; + unsigned long len; + unsigned char outct[MAXBLOCKSIZE] = { 0 }; + unsigned char outtag[MAXBLOCKSIZE] = { 0 }; + ocb3_state ocb; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + len = 16; /* must be the same as the required taglen */ + if ((err = ocb3_encrypt_authenticate_memory(idx, + key, sizeof(key), + nonce, sizeof(nonce), + tests[x].aadlen != 0 ? tests[x].aad : NULL, tests[x].aadlen, + tests[x].ptlen != 0 ? tests[x].pt : NULL, tests[x].ptlen, + tests[x].ptlen != 0 ? outct : NULL, outtag, &len)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(outtag, len, tests[x].tag, sizeof(tests[x].tag), "OCB3 Tag", x) || + compare_testvector(outct, tests[x].ptlen, tests[x].ct, tests[x].ptlen, "OCB3 CT", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + if ((err = ocb3_decrypt_verify_memory(idx, + key, sizeof(key), + nonce, sizeof(nonce), + tests[x].aadlen != 0 ? tests[x].aad : NULL, tests[x].aadlen, + tests[x].ptlen != 0 ? outct : NULL, tests[x].ptlen, + tests[x].ptlen != 0 ? outct : NULL, tests[x].tag, len, &res)) != CRYPT_OK) { + return err; + } + if ((res != 1) || compare_testvector(outct, tests[x].ptlen, tests[x].pt, tests[x].ptlen, "OCB3", x)) { +#ifdef LTC_TEST_DBG + printf("\n\nOCB3: Failure-decrypt - res = %d\n", res); +#endif + return CRYPT_FAIL_TESTVECTOR; + } + } + + /* RFC 7253 - test vector with a tag length of 96 bits - part 1 */ + x = 99; + len = 12; + if ((err = ocb3_encrypt_authenticate_memory(idx, + K, sizeof(K), + N, sizeof(N), + A, sizeof(A), + P, sizeof(P), + outct, outtag, &len)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(outtag, len, T, sizeof(T), "OCB3 Tag", x) || + compare_testvector(outct, sizeof(P), C, sizeof(C), "OCB3 CT", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + + if ((err = ocb3_decrypt_verify_memory(idx, + K, sizeof(K), + N, sizeof(N), + A, sizeof(A), + C, sizeof(C), + outct, T, sizeof(T), &res)) != CRYPT_OK) { + return err; + } + if ((res != 1) || compare_testvector(outct, sizeof(C), P, sizeof(P), "OCB3", x)) { +#ifdef LTC_TEST_DBG + printf("\n\nOCB3: Failure-decrypt - res = %d\n", res); +#endif + return CRYPT_FAIL_TESTVECTOR; + } + + /* RFC 7253 - test vector with a tag length of 96 bits - part 2 */ + x = 100; + if ((err = ocb3_init(&ocb, idx, K, sizeof(K), N, sizeof(N), 12)) != CRYPT_OK) return err; + if ((err = ocb3_add_aad(&ocb, A, sizeof(A))) != CRYPT_OK) return err; + if ((err = ocb3_encrypt(&ocb, P, 32, outct)) != CRYPT_OK) return err; + if ((err = ocb3_encrypt_last(&ocb, P+32, sizeof(P)-32, outct+32)) != CRYPT_OK) return err; + len = sizeof(outtag); /* intentionally more than 12 */ + if ((err = ocb3_done(&ocb, outtag, &len)) != CRYPT_OK) return err; + if (compare_testvector(outct, sizeof(P), C, sizeof(C), "OCB3 CT", x)) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(outtag, len, T, sizeof(T), "OCB3 Tag.enc", x)) return CRYPT_FAIL_TESTVECTOR; + if ((err = ocb3_init(&ocb, idx, K, sizeof(K), N, sizeof(N), 12)) != CRYPT_OK) return err; + if ((err = ocb3_add_aad(&ocb, A, sizeof(A))) != CRYPT_OK) return err; + if ((err = ocb3_decrypt(&ocb, C, 32, outct)) != CRYPT_OK) return err; + if ((err = ocb3_decrypt_last(&ocb, C+32, sizeof(C)-32, outct+32)) != CRYPT_OK) return err; + len = sizeof(outtag); /* intentionally more than 12 */ + if ((err = ocb3_done(&ocb, outtag, &len)) != CRYPT_OK) return err; + if (compare_testvector(outct, sizeof(C), P, sizeof(P), "OCB3 PT", x)) return CRYPT_FAIL_TESTVECTOR; + if (compare_testvector(outtag, len, T, sizeof(T), "OCB3 Tag.dec", x)) return CRYPT_FAIL_TESTVECTOR; + + return CRYPT_OK; +#endif /* LTC_TEST */ +} + +#endif /* LTC_OCB3_MODE */ diff --git a/optee_os/core/lib/libtomcrypt/src/encauth/sub.mk b/optee_os/core/lib/libtomcrypt/src/encauth/sub.mk new file mode 100644 index 0000000..6ef9405 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/encauth/sub.mk @@ -0,0 +1,2 @@ +subdirs-$(_CFG_CORE_LTC_CCM) += ccm +subdirs-$(_CFG_CORE_LTC_GCM) += gcm diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/blake2b.c b/optee_os/core/lib/libtomcrypt/src/hashes/blake2b.c new file mode 100644 index 0000000..8cbeac2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/blake2b.c @@ -0,0 +1,628 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +/* see also https://www.ietf.org/rfc/rfc7693.txt */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2B + +enum blake2b_constant { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16, + BLAKE2B_PARAM_SIZE = 64 +}; + +/* param offsets */ +enum { + O_DIGEST_LENGTH = 0, + O_KEY_LENGTH = 1, + O_FANOUT = 2, + O_DEPTH = 3, + O_LEAF_LENGTH = 4, + O_NODE_OFFSET = 8, + O_XOF_LENGTH = 12, + O_NODE_DEPTH = 16, + O_INNER_LENGTH = 17, + O_RESERVED = 18, + O_SALT = 32, + O_PERSONAL = 48 +}; + +/* +struct blake2b_param { + unsigned char digest_length; + unsigned char key_length; + unsigned char fanout; + unsigned char depth; + ulong32 leaf_length; + ulong32 node_offset; + ulong32 xof_length; + unsigned char node_depth; + unsigned char inner_length; + unsigned char reserved[14]; + unsigned char salt[BLAKE2B_SALTBYTES]; + unsigned char personal[BLAKE2B_PERSONALBYTES]; +}; +*/ + +const struct ltc_hash_descriptor blake2b_160_desc = +{ + "blake2b-160", + 25, + 20, + 128, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 1, 5 }, + 11, + &blake2b_160_init, + &blake2b_process, + &blake2b_done, + &blake2b_160_test, + NULL +}; + +const struct ltc_hash_descriptor blake2b_256_desc = +{ + "blake2b-256", + 26, + 32, + 128, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 1, 8 }, + 11, + &blake2b_256_init, + &blake2b_process, + &blake2b_done, + &blake2b_256_test, + NULL +}; + +const struct ltc_hash_descriptor blake2b_384_desc = +{ + "blake2b-384", + 27, + 48, + 128, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 1, 12 }, + 11, + &blake2b_384_init, + &blake2b_process, + &blake2b_done, + &blake2b_384_test, + NULL +}; + +const struct ltc_hash_descriptor blake2b_512_desc = +{ + "blake2b-512", + 28, + 64, + 128, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 1, 16 }, + 11, + &blake2b_512_init, + &blake2b_process, + &blake2b_done, + &blake2b_512_test, + NULL +}; + +static const ulong64 blake2b_IV[8] = +{ + CONST64(0x6a09e667f3bcc908), CONST64(0xbb67ae8584caa73b), + CONST64(0x3c6ef372fe94f82b), CONST64(0xa54ff53a5f1d36f1), + CONST64(0x510e527fade682d1), CONST64(0x9b05688c2b3e6c1f), + CONST64(0x1f83d9abfb41bd6b), CONST64(0x5be0cd19137e2179) +}; + +static const unsigned char blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + +static void s_blake2b_set_lastnode(hash_state *md) { md->blake2b.f[1] = CONST64(0xffffffffffffffff); } + +/* Some helper functions, not necessarily useful */ +static int s_blake2b_is_lastblock(const hash_state *md) { return md->blake2b.f[0] != 0; } + +static void s_blake2b_set_lastblock(hash_state *md) +{ + if (md->blake2b.last_node) { + s_blake2b_set_lastnode(md); + } + md->blake2b.f[0] = CONST64(0xffffffffffffffff); +} + +static void s_blake2b_increment_counter(hash_state *md, ulong64 inc) +{ + md->blake2b.t[0] += inc; + if (md->blake2b.t[0] < inc) md->blake2b.t[1]++; +} + +static void s_blake2b_init0(hash_state *md) +{ + unsigned long i; + XMEMSET(&md->blake2b, 0, sizeof(md->blake2b)); + + for (i = 0; i < 8; ++i) { + md->blake2b.h[i] = blake2b_IV[i]; + } +} + +/* init xors IV with input parameter block */ +static int s_blake2b_init_param(hash_state *md, const unsigned char *P) +{ + unsigned long i; + + s_blake2b_init0(md); + + /* IV XOR ParamBlock */ + for (i = 0; i < 8; ++i) { + ulong64 tmp; + LOAD64L(tmp, P + i * 8); + md->blake2b.h[i] ^= tmp; + } + + md->blake2b.outlen = P[O_DIGEST_LENGTH]; + return CRYPT_OK; +} + +/** + Initialize the hash/MAC state + + Use this function to init for arbitrary sizes. + + Give a key and keylen to init for MAC mode. + + @param md The hash state you wish to initialize + @param outlen The desired output-length + @param key The key of the MAC + @param keylen The length of the key + @return CRYPT_OK if successful +*/ +int blake2b_init(hash_state *md, unsigned long outlen, const unsigned char *key, unsigned long keylen) +{ + unsigned char P[BLAKE2B_PARAM_SIZE]; + int err; + + LTC_ARGCHK(md != NULL); + + if ((!outlen) || (outlen > BLAKE2B_OUTBYTES)) { + return CRYPT_INVALID_ARG; + } + if ((key && !keylen) || (keylen && !key) || (keylen > BLAKE2B_KEYBYTES)) { + return CRYPT_INVALID_ARG; + } + + XMEMSET(P, 0, sizeof(P)); + + P[O_DIGEST_LENGTH] = (unsigned char)outlen; + P[O_KEY_LENGTH] = (unsigned char)keylen; + P[O_FANOUT] = 1; + P[O_DEPTH] = 1; + + err = s_blake2b_init_param(md, P); + if (err != CRYPT_OK) return err; + + if (key) { + unsigned char block[BLAKE2B_BLOCKBYTES]; + + XMEMSET(block, 0, BLAKE2B_BLOCKBYTES); + XMEMCPY(block, key, keylen); + blake2b_process(md, block, BLAKE2B_BLOCKBYTES); + +#ifdef LTC_CLEAN_STACK + zeromem(block, sizeof(block)); +#endif + } + + return CRYPT_OK; +} + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2b_160_init(hash_state *md) { return blake2b_init(md, 20, NULL, 0); } + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2b_256_init(hash_state *md) { return blake2b_init(md, 32, NULL, 0); } + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2b_384_init(hash_state *md) { return blake2b_init(md, 48, NULL, 0); } + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2b_512_init(hash_state *md) { return blake2b_init(md, 64, NULL, 0); } + +#define G(r, i, a, b, c, d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2 * i + 0]]; \ + d = ROR64(d ^ a, 32); \ + c = c + d; \ + b = ROR64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2 * i + 1]]; \ + d = ROR64(d ^ a, 16); \ + c = c + d; \ + b = ROR64(b ^ c, 63); \ + } while (0) + +#define ROUND(r) \ + do { \ + G(r, 0, v[0], v[4], v[8], v[12]); \ + G(r, 1, v[1], v[5], v[9], v[13]); \ + G(r, 2, v[2], v[6], v[10], v[14]); \ + G(r, 3, v[3], v[7], v[11], v[15]); \ + G(r, 4, v[0], v[5], v[10], v[15]); \ + G(r, 5, v[1], v[6], v[11], v[12]); \ + G(r, 6, v[2], v[7], v[8], v[13]); \ + G(r, 7, v[3], v[4], v[9], v[14]); \ + } while (0) + +#ifdef LTC_CLEAN_STACK +static int ss_blake2b_compress(hash_state *md, const unsigned char *buf) +#else +static int s_blake2b_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong64 m[16]; + ulong64 v[16]; + unsigned long i; + + for (i = 0; i < 16; ++i) { + LOAD64L(m[i], buf + i * sizeof(m[i])); + } + + for (i = 0; i < 8; ++i) { + v[i] = md->blake2b.h[i]; + } + + v[8] = blake2b_IV[0]; + v[9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ md->blake2b.t[0]; + v[13] = blake2b_IV[5] ^ md->blake2b.t[1]; + v[14] = blake2b_IV[6] ^ md->blake2b.f[0]; + v[15] = blake2b_IV[7] ^ md->blake2b.f[1]; + + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + ROUND(10); + ROUND(11); + + for (i = 0; i < 8; ++i) { + md->blake2b.h[i] = md->blake2b.h[i] ^ v[i] ^ v[i + 8]; + } + return CRYPT_OK; +} + +#undef G +#undef ROUND + +#ifdef LTC_CLEAN_STACK +static int s_blake2b_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_blake2b_compress(md, buf); + burn_stack(sizeof(ulong64) * 32 + sizeof(unsigned long)); + return err; +} +#endif + +/** + Process a block of memory through the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +int blake2b_process(hash_state *md, const unsigned char *in, unsigned long inlen) +{ + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + + if (md->blake2b.curlen > sizeof(md->blake2b.buf)) { + return CRYPT_INVALID_ARG; + } + + if (inlen > 0) { + unsigned long left = md->blake2b.curlen; + unsigned long fill = BLAKE2B_BLOCKBYTES - left; + if (inlen > fill) { + md->blake2b.curlen = 0; + XMEMCPY(md->blake2b.buf + (left % sizeof(md->blake2b.buf)), in, fill); /* Fill buffer */ + s_blake2b_increment_counter(md, BLAKE2B_BLOCKBYTES); + s_blake2b_compress(md, md->blake2b.buf); /* Compress */ + in += fill; + inlen -= fill; + while (inlen > BLAKE2B_BLOCKBYTES) { + s_blake2b_increment_counter(md, BLAKE2B_BLOCKBYTES); + s_blake2b_compress(md, in); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + XMEMCPY(md->blake2b.buf + md->blake2b.curlen, in, inlen); + md->blake2b.curlen += inlen; + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (size depending on the length used on init) + @return CRYPT_OK if successful +*/ +int blake2b_done(hash_state *md, unsigned char *out) +{ + unsigned char buffer[BLAKE2B_OUTBYTES] = { 0 }; + unsigned long i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + /* if(md->blakebs.outlen != outlen) return CRYPT_INVALID_ARG; */ + + if (s_blake2b_is_lastblock(md)) { + return CRYPT_ERROR; + } + + s_blake2b_increment_counter(md, md->blake2b.curlen); + s_blake2b_set_lastblock(md); + XMEMSET(md->blake2b.buf + md->blake2b.curlen, 0, BLAKE2B_BLOCKBYTES - md->blake2b.curlen); /* Padding */ + s_blake2b_compress(md, md->blake2b.buf); + + for (i = 0; i < 8; ++i) { /* Output full hash to temp buffer */ + STORE64L(md->blake2b.h[i], buffer + i * 8); + } + + XMEMCPY(out, buffer, md->blake2b.outlen); + zeromem(md, sizeof(hash_state)); +#ifdef LTC_CLEAN_STACK + zeromem(buffer, sizeof(buffer)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2b_512_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[64]; + } tests[] = { + { "", + { 0x78, 0x6a, 0x02, 0xf7, 0x42, 0x01, 0x59, 0x03, + 0xc6, 0xc6, 0xfd, 0x85, 0x25, 0x52, 0xd2, 0x72, + 0x91, 0x2f, 0x47, 0x40, 0xe1, 0x58, 0x47, 0x61, + 0x8a, 0x86, 0xe2, 0x17, 0xf7, 0x1f, 0x54, 0x19, + 0xd2, 0x5e, 0x10, 0x31, 0xaf, 0xee, 0x58, 0x53, + 0x13, 0x89, 0x64, 0x44, 0x93, 0x4e, 0xb0, 0x4b, + 0x90, 0x3a, 0x68, 0x5b, 0x14, 0x48, 0xb7, 0x55, + 0xd5, 0x6f, 0x70, 0x1a, 0xfe, 0x9b, 0xe2, 0xce } }, + { "abc", + { 0xba, 0x80, 0xa5, 0x3f, 0x98, 0x1c, 0x4d, 0x0d, + 0x6a, 0x27, 0x97, 0xb6, 0x9f, 0x12, 0xf6, 0xe9, + 0x4c, 0x21, 0x2f, 0x14, 0x68, 0x5a, 0xc4, 0xb7, + 0x4b, 0x12, 0xbb, 0x6f, 0xdb, 0xff, 0xa2, 0xd1, + 0x7d, 0x87, 0xc5, 0x39, 0x2a, 0xab, 0x79, 0x2d, + 0xc2, 0x52, 0xd5, 0xde, 0x45, 0x33, 0xcc, 0x95, + 0x18, 0xd3, 0x8a, 0xa8, 0xdb, 0xf1, 0x92, 0x5a, + 0xb9, 0x23, 0x86, 0xed, 0xd4, 0x00, 0x99, 0x23 } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[64]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2b_512_init(&md); + blake2b_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2b_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2B_512", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2b_384_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[48]; + } tests[] = { + { "", + { 0xb3, 0x28, 0x11, 0x42, 0x33, 0x77, 0xf5, 0x2d, + 0x78, 0x62, 0x28, 0x6e, 0xe1, 0xa7, 0x2e, 0xe5, + 0x40, 0x52, 0x43, 0x80, 0xfd, 0xa1, 0x72, 0x4a, + 0x6f, 0x25, 0xd7, 0x97, 0x8c, 0x6f, 0xd3, 0x24, + 0x4a, 0x6c, 0xaf, 0x04, 0x98, 0x81, 0x26, 0x73, + 0xc5, 0xe0, 0x5e, 0xf5, 0x83, 0x82, 0x51, 0x00 } }, + { "abc", + { 0x6f, 0x56, 0xa8, 0x2c, 0x8e, 0x7e, 0xf5, 0x26, + 0xdf, 0xe1, 0x82, 0xeb, 0x52, 0x12, 0xf7, 0xdb, + 0x9d, 0xf1, 0x31, 0x7e, 0x57, 0x81, 0x5d, 0xbd, + 0xa4, 0x60, 0x83, 0xfc, 0x30, 0xf5, 0x4e, 0xe6, + 0xc6, 0x6b, 0xa8, 0x3b, 0xe6, 0x4b, 0x30, 0x2d, + 0x7c, 0xba, 0x6c, 0xe1, 0x5b, 0xb5, 0x56, 0xf4 } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[48]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2b_384_init(&md); + blake2b_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2b_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2B_384", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2b_256_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[32]; + } tests[] = { + { "", + { 0x0e, 0x57, 0x51, 0xc0, 0x26, 0xe5, 0x43, 0xb2, + 0xe8, 0xab, 0x2e, 0xb0, 0x60, 0x99, 0xda, 0xa1, + 0xd1, 0xe5, 0xdf, 0x47, 0x77, 0x8f, 0x77, 0x87, + 0xfa, 0xab, 0x45, 0xcd, 0xf1, 0x2f, 0xe3, 0xa8 } }, + { "abc", + { 0xbd, 0xdd, 0x81, 0x3c, 0x63, 0x42, 0x39, 0x72, + 0x31, 0x71, 0xef, 0x3f, 0xee, 0x98, 0x57, 0x9b, + 0x94, 0x96, 0x4e, 0x3b, 0xb1, 0xcb, 0x3e, 0x42, + 0x72, 0x62, 0xc8, 0xc0, 0x68, 0xd5, 0x23, 0x19 } }, + { "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890", + { 0x0f, 0x6e, 0x01, 0x8d, 0x38, 0xd6, 0x3f, 0x08, + 0x4d, 0x58, 0xe3, 0x0c, 0x90, 0xfb, 0xa2, 0x41, + 0x5f, 0xca, 0x17, 0xfa, 0x66, 0x26, 0x49, 0xf3, + 0x8a, 0x30, 0x41, 0x7c, 0x57, 0xcd, 0xa8, 0x14 } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[32]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2b_256_init(&md); + blake2b_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2b_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2B_256", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2b_160_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[20]; + } tests[] = { + { "", + { 0x33, 0x45, 0x52, 0x4a, 0xbf, 0x6b, 0xbe, 0x18, + 0x09, 0x44, 0x92, 0x24, 0xb5, 0x97, 0x2c, 0x41, + 0x79, 0x0b, 0x6c, 0xf2 } }, + { "abc", + { 0x38, 0x42, 0x64, 0xf6, 0x76, 0xf3, 0x95, 0x36, + 0x84, 0x05, 0x23, 0xf2, 0x84, 0x92, 0x1c, 0xdc, + 0x68, 0xb6, 0x84, 0x6b } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[20]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2b_160_init(&md); + blake2b_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2b_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2B_160", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/blake2s.c b/optee_os/core/lib/libtomcrypt/src/hashes/blake2s.c new file mode 100644 index 0000000..e8cd6eb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/blake2s.c @@ -0,0 +1,603 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +/* see also https://www.ietf.org/rfc/rfc7693.txt */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2S + +enum blake2s_constant { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8, + BLAKE2S_PARAM_SIZE = 32 +}; + +/* param offsets */ +enum { + O_DIGEST_LENGTH = 0, + O_KEY_LENGTH = 1, + O_FANOUT = 2, + O_DEPTH = 3, + O_LEAF_LENGTH = 4, + O_NODE_OFFSET = 8, + O_XOF_LENGTH = 12, + O_NODE_DEPTH = 14, + O_INNER_LENGTH = 15, + O_SALT = 16, + O_PERSONAL = 24 +}; + +/* +struct blake2s_param { + unsigned char digest_length; + unsigned char key_length; + unsigned char fanout; + unsigned char depth; + ulong32 leaf_length; + ulong32 node_offset; + ushort16 xof_length; + unsigned char node_depth; + unsigned char inner_length; + unsigned char salt[BLAKE2S_SALTBYTES]; + unsigned char personal[BLAKE2S_PERSONALBYTES]; +}; +*/ + +const struct ltc_hash_descriptor blake2s_128_desc = +{ + "blake2s-128", + 21, + 16, + 64, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 2, 4 }, + 11, + &blake2s_128_init, + &blake2s_process, + &blake2s_done, + &blake2s_128_test, + NULL +}; + +const struct ltc_hash_descriptor blake2s_160_desc = +{ + "blake2s-160", + 22, + 20, + 64, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 2, 5 }, + 11, + &blake2s_160_init, + &blake2s_process, + &blake2s_done, + &blake2s_160_test, + NULL +}; + +const struct ltc_hash_descriptor blake2s_224_desc = +{ + "blake2s-224", + 23, + 28, + 64, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 2, 7 }, + 11, + &blake2s_224_init, + &blake2s_process, + &blake2s_done, + &blake2s_224_test, + NULL +}; + +const struct ltc_hash_descriptor blake2s_256_desc = +{ + "blake2s-256", + 24, + 32, + 64, + { 1, 3, 6, 1, 4, 1, 1722, 12, 2, 2, 8 }, + 11, + &blake2s_256_init, + &blake2s_process, + &blake2s_done, + &blake2s_256_test, + NULL +}; + +static const ulong32 blake2s_IV[8] = { + 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, + 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL +}; + +static const unsigned char blake2s_sigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static void s_blake2s_set_lastnode(hash_state *md) { md->blake2s.f[1] = 0xffffffffUL; } + +/* Some helper functions, not necessarily useful */ +static int s_blake2s_is_lastblock(const hash_state *md) { return md->blake2s.f[0] != 0; } + +static void s_blake2s_set_lastblock(hash_state *md) +{ + if (md->blake2s.last_node) { + s_blake2s_set_lastnode(md); + } + md->blake2s.f[0] = 0xffffffffUL; +} + +static void s_blake2s_increment_counter(hash_state *md, const ulong32 inc) +{ + md->blake2s.t[0] += inc; + if (md->blake2s.t[0] < inc) md->blake2s.t[1]++; +} + +static int s_blake2s_init0(hash_state *md) +{ + int i; + XMEMSET(&md->blake2s, 0, sizeof(struct blake2s_state)); + + for (i = 0; i < 8; ++i) { + md->blake2s.h[i] = blake2s_IV[i]; + } + + return CRYPT_OK; +} + +/* init2 xors IV with input parameter block */ +static int s_blake2s_init_param(hash_state *md, const unsigned char *P) +{ + unsigned long i; + + s_blake2s_init0(md); + + /* IV XOR ParamBlock */ + for (i = 0; i < 8; ++i) { + ulong32 tmp; + LOAD32L(tmp, P + i * 4); + md->blake2s.h[i] ^= tmp; + } + + md->blake2s.outlen = P[O_DIGEST_LENGTH]; + return CRYPT_OK; +} + +/** + Initialize the hash/MAC state + + Use this function to init for arbitrary sizes. + + Give a key and keylen to init for MAC mode. + + @param md The hash state you wish to initialize + @param outlen The desired output-length + @param key The key of the MAC + @param keylen The length of the key + @return CRYPT_OK if successful +*/ +int blake2s_init(hash_state *md, unsigned long outlen, const unsigned char *key, unsigned long keylen) +{ + unsigned char P[BLAKE2S_PARAM_SIZE]; + int err; + + LTC_ARGCHK(md != NULL); + + if ((!outlen) || (outlen > BLAKE2S_OUTBYTES)) { + return CRYPT_INVALID_ARG; + } + if ((key && !keylen) || (keylen && !key) || (keylen > BLAKE2S_KEYBYTES)) { + return CRYPT_INVALID_ARG; + } + + XMEMSET(P, 0, sizeof(P)); + + P[O_DIGEST_LENGTH] = (unsigned char)outlen; + P[O_KEY_LENGTH] = (unsigned char)keylen; + P[O_FANOUT] = 1; + P[O_DEPTH] = 1; + + err = s_blake2s_init_param(md, P); + if (err != CRYPT_OK) return err; + + if (key) { + unsigned char block[BLAKE2S_BLOCKBYTES]; + + XMEMSET(block, 0, BLAKE2S_BLOCKBYTES); + XMEMCPY(block, key, keylen); + blake2s_process(md, block, BLAKE2S_BLOCKBYTES); + +#ifdef LTC_CLEAN_STACK + zeromem(block, sizeof(block)); +#endif + } + return CRYPT_OK; +} + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2s_128_init(hash_state *md) { return blake2s_init(md, 16, NULL, 0); } + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2s_160_init(hash_state *md) { return blake2s_init(md, 20, NULL, 0); } + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2s_224_init(hash_state *md) { return blake2s_init(md, 28, NULL, 0); } + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int blake2s_256_init(hash_state *md) { return blake2s_init(md, 32, NULL, 0); } + +#define G(r, i, a, b, c, d) \ + do { \ + a = a + b + m[blake2s_sigma[r][2 * i + 0]]; \ + d = ROR(d ^ a, 16); \ + c = c + d; \ + b = ROR(b ^ c, 12); \ + a = a + b + m[blake2s_sigma[r][2 * i + 1]]; \ + d = ROR(d ^ a, 8); \ + c = c + d; \ + b = ROR(b ^ c, 7); \ + } while (0) +#define ROUND(r) \ + do { \ + G(r, 0, v[0], v[4], v[8], v[12]); \ + G(r, 1, v[1], v[5], v[9], v[13]); \ + G(r, 2, v[2], v[6], v[10], v[14]); \ + G(r, 3, v[3], v[7], v[11], v[15]); \ + G(r, 4, v[0], v[5], v[10], v[15]); \ + G(r, 5, v[1], v[6], v[11], v[12]); \ + G(r, 6, v[2], v[7], v[8], v[13]); \ + G(r, 7, v[3], v[4], v[9], v[14]); \ + } while (0) + +#ifdef LTC_CLEAN_STACK +static int ss_blake2s_compress(hash_state *md, const unsigned char *buf) +#else +static int s_blake2s_compress(hash_state *md, const unsigned char *buf) +#endif +{ + unsigned long i; + ulong32 m[16]; + ulong32 v[16]; + + for (i = 0; i < 16; ++i) { + LOAD32L(m[i], buf + i * sizeof(m[i])); + } + + for (i = 0; i < 8; ++i) { + v[i] = md->blake2s.h[i]; + } + + v[8] = blake2s_IV[0]; + v[9] = blake2s_IV[1]; + v[10] = blake2s_IV[2]; + v[11] = blake2s_IV[3]; + v[12] = md->blake2s.t[0] ^ blake2s_IV[4]; + v[13] = md->blake2s.t[1] ^ blake2s_IV[5]; + v[14] = md->blake2s.f[0] ^ blake2s_IV[6]; + v[15] = md->blake2s.f[1] ^ blake2s_IV[7]; + + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + + for (i = 0; i < 8; ++i) { + md->blake2s.h[i] = md->blake2s.h[i] ^ v[i] ^ v[i + 8]; + } + return CRYPT_OK; +} +#undef G +#undef ROUND + +#ifdef LTC_CLEAN_STACK +static int s_blake2s_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_blake2s_compress(md, buf); + burn_stack(sizeof(ulong32) * (32) + sizeof(unsigned long)); + return err; +} +#endif + +/** + Process a block of memory through the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +int blake2s_process(hash_state *md, const unsigned char *in, unsigned long inlen) +{ + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + + if (md->blake2s.curlen > sizeof(md->blake2s.buf)) { + return CRYPT_INVALID_ARG; + } + + if (inlen > 0) { + unsigned long left = md->blake2s.curlen; + unsigned long fill = BLAKE2S_BLOCKBYTES - left; + if (inlen > fill) { + md->blake2s.curlen = 0; + XMEMCPY(md->blake2s.buf + (left % sizeof(md->blake2s.buf)), in, fill); /* Fill buffer */ + s_blake2s_increment_counter(md, BLAKE2S_BLOCKBYTES); + s_blake2s_compress(md, md->blake2s.buf); /* Compress */ + in += fill; + inlen -= fill; + while (inlen > BLAKE2S_BLOCKBYTES) { + s_blake2s_increment_counter(md, BLAKE2S_BLOCKBYTES); + s_blake2s_compress(md, in); + in += BLAKE2S_BLOCKBYTES; + inlen -= BLAKE2S_BLOCKBYTES; + } + } + XMEMCPY(md->blake2s.buf + md->blake2s.curlen, in, inlen); + md->blake2s.curlen += inlen; + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (size depending on the length used on init) + @return CRYPT_OK if successful +*/ +int blake2s_done(hash_state *md, unsigned char *out) +{ + unsigned char buffer[BLAKE2S_OUTBYTES] = { 0 }; + unsigned long i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + /* if(md->blake2s.outlen != outlen) return CRYPT_INVALID_ARG; */ + + if (s_blake2s_is_lastblock(md)) { + return CRYPT_ERROR; + } + s_blake2s_increment_counter(md, md->blake2s.curlen); + s_blake2s_set_lastblock(md); + XMEMSET(md->blake2s.buf + md->blake2s.curlen, 0, BLAKE2S_BLOCKBYTES - md->blake2s.curlen); /* Padding */ + s_blake2s_compress(md, md->blake2s.buf); + + for (i = 0; i < 8; ++i) { /* Output full hash to temp buffer */ + STORE32L(md->blake2s.h[i], buffer + i * 4); + } + + XMEMCPY(out, buffer, md->blake2s.outlen); + zeromem(md, sizeof(hash_state)); +#ifdef LTC_CLEAN_STACK + zeromem(buffer, sizeof(buffer)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2s_256_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[32]; + } tests[] = { + { "", + { 0x69, 0x21, 0x7a, 0x30, 0x79, 0x90, 0x80, 0x94, + 0xe1, 0x11, 0x21, 0xd0, 0x42, 0x35, 0x4a, 0x7c, + 0x1f, 0x55, 0xb6, 0x48, 0x2c, 0xa1, 0xa5, 0x1e, + 0x1b, 0x25, 0x0d, 0xfd, 0x1e, 0xd0, 0xee, 0xf9 } }, + { "abc", + { 0x50, 0x8c, 0x5e, 0x8c, 0x32, 0x7c, 0x14, 0xe2, + 0xe1, 0xa7, 0x2b, 0xa3, 0x4e, 0xeb, 0x45, 0x2f, + 0x37, 0x45, 0x8b, 0x20, 0x9e, 0xd6, 0x3a, 0x29, + 0x4d, 0x99, 0x9b, 0x4c, 0x86, 0x67, 0x59, 0x82 } }, + { "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890" + "12345678901234567890123456789012345678901234567890", + { 0xa3, 0x78, 0x8b, 0x5b, 0x59, 0xee, 0xe4, 0x41, + 0x95, 0x23, 0x58, 0x00, 0xa4, 0xf9, 0xfa, 0x41, + 0x86, 0x0c, 0x7b, 0x1c, 0x35, 0xa2, 0x42, 0x70, + 0x50, 0x80, 0x79, 0x56, 0xe3, 0xbe, 0x31, 0x74 } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[32]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2s_256_init(&md); + blake2s_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2s_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2S_256", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + + } + return CRYPT_OK; +#endif +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2s_224_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[28]; + } tests[] = { + { "", + { 0x1f, 0xa1, 0x29, 0x1e, 0x65, 0x24, 0x8b, 0x37, + 0xb3, 0x43, 0x34, 0x75, 0xb2, 0xa0, 0xdd, 0x63, + 0xd5, 0x4a, 0x11, 0xec, 0xc4, 0xe3, 0xe0, 0x34, + 0xe7, 0xbc, 0x1e, 0xf4 } }, + { "abc", + { 0x0b, 0x03, 0x3f, 0xc2, 0x26, 0xdf, 0x7a, 0xbd, + 0xe2, 0x9f, 0x67, 0xa0, 0x5d, 0x3d, 0xc6, 0x2c, + 0xf2, 0x71, 0xef, 0x3d, 0xfe, 0xa4, 0xd3, 0x87, + 0x40, 0x7f, 0xbd, 0x55 } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[28]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2s_224_init(&md); + blake2s_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2s_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2S_224", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + + } + return CRYPT_OK; +#endif +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2s_160_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[20]; + } tests[] = { + { "", + { 0x35, 0x4c, 0x9c, 0x33, 0xf7, 0x35, 0x96, 0x24, + 0x18, 0xbd, 0xac, 0xb9, 0x47, 0x98, 0x73, 0x42, + 0x9c, 0x34, 0x91, 0x6f} }, + { "abc", + { 0x5a, 0xe3, 0xb9, 0x9b, 0xe2, 0x9b, 0x01, 0x83, + 0x4c, 0x3b, 0x50, 0x85, 0x21, 0xed, 0xe6, 0x04, + 0x38, 0xf8, 0xde, 0x17 } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[20]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2s_160_init(&md); + blake2s_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2s_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2S_160", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + + } + return CRYPT_OK; +#endif +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int blake2s_128_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[16]; + } tests[] = { + { "", + { 0x64, 0x55, 0x0d, 0x6f, 0xfe, 0x2c, 0x0a, 0x01, + 0xa1, 0x4a, 0xba, 0x1e, 0xad, 0xe0, 0x20, 0x0c } }, + { "abc", + { 0xaa, 0x49, 0x38, 0x11, 0x9b, 0x1d, 0xc7, 0xb8, + 0x7c, 0xba, 0xd0, 0xff, 0xd2, 0x00, 0xd0, 0xae } }, + + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[16]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + blake2s_128_init(&md); + blake2s_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + blake2s_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "BLAKE2S_128", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/chc/chc.c b/optee_os/core/lib/libtomcrypt/src/hashes/chc/chc.c new file mode 100644 index 0000000..7de59a2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/chc/chc.c @@ -0,0 +1,302 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file chc.c + CHC support. (Tom St Denis) +*/ + +#ifdef LTC_CHC_HASH + +#define UNDEFED_HASH -17 + +/* chc settings */ +static int cipher_idx=UNDEFED_HASH, /* which cipher */ + cipher_blocksize; /* blocksize of cipher */ + + +const struct ltc_hash_descriptor chc_desc = { + "chc_hash", 12, 0, 0, { 0 }, 0, + &chc_init, + &chc_process, + &chc_done, + &chc_test, + NULL +}; + +/** + Initialize the CHC state with a given cipher + @param cipher The index of the cipher you wish to bind + @return CRYPT_OK if successful +*/ +int chc_register(int cipher) +{ + int err, kl, idx; + + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* will it be valid? */ + kl = cipher_descriptor[cipher]->block_length; + + /* must be >64 bit block */ + if (kl <= 8) { + return CRYPT_INVALID_CIPHER; + } + + /* can we use the ideal keysize? */ + if ((err = cipher_descriptor[cipher]->keysize(&kl)) != CRYPT_OK) { + return err; + } + /* we require that key size == block size be a valid choice */ + if (kl != cipher_descriptor[cipher]->block_length) { + return CRYPT_INVALID_CIPHER; + } + + /* determine if chc_hash has been register_hash'ed already */ + if ((err = hash_is_valid(idx = find_hash("chc_hash"))) != CRYPT_OK) { + return err; + } + + /* store into descriptor */ + hash_descriptor[idx]->hashsize = + hash_descriptor[idx]->blocksize = cipher_descriptor[cipher]->block_length; + + /* store the idx and block size */ + cipher_idx = cipher; + cipher_blocksize = cipher_descriptor[cipher]->block_length; + return CRYPT_OK; +} + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int chc_init(hash_state *md) +{ + symmetric_key *key; + unsigned char buf[MAXBLOCKSIZE]; + int err; + + LTC_ARGCHK(md != NULL); + + /* is the cipher valid? */ + if ((err = cipher_is_valid(cipher_idx)) != CRYPT_OK) { + return err; + } + + if (cipher_blocksize != cipher_descriptor[cipher_idx]->block_length) { + return CRYPT_INVALID_CIPHER; + } + + if ((key = XMALLOC(sizeof(*key))) == NULL) { + return CRYPT_MEM; + } + + /* zero key and what not */ + zeromem(buf, cipher_blocksize); + if ((err = cipher_descriptor[cipher_idx]->setup(buf, cipher_blocksize, 0, key)) != CRYPT_OK) { + XFREE(key); + return err; + } + + /* encrypt zero block */ + cipher_descriptor[cipher_idx]->ecb_encrypt(buf, md->chc.state, key); + + /* zero other members */ + md->chc.length = 0; + md->chc.curlen = 0; + zeromem(md->chc.buf, sizeof(md->chc.buf)); + XFREE(key); + return CRYPT_OK; +} + +/* + key <= state + T0,T1 <= block + T0 <= encrypt T0 + state <= state xor T0 xor T1 +*/ +static int s_chc_compress(hash_state *md, const unsigned char *buf) +{ + unsigned char T[2][MAXBLOCKSIZE]; + symmetric_key *key; + int err, x; + + if ((key = XMALLOC(sizeof(*key))) == NULL) { + return CRYPT_MEM; + } + if ((err = cipher_descriptor[cipher_idx]->setup(md->chc.state, cipher_blocksize, 0, key)) != CRYPT_OK) { + XFREE(key); + return err; + } + XMEMCPY(T[1], buf, cipher_blocksize); + cipher_descriptor[cipher_idx]->ecb_encrypt(buf, T[0], key); + for (x = 0; x < cipher_blocksize; x++) { + md->chc.state[x] ^= T[0][x] ^ T[1][x]; + } +#ifdef LTC_CLEAN_STACK + zeromem(T, sizeof(T)); + zeromem(key, sizeof(*key)); +#endif + XFREE(key); + return CRYPT_OK; +} + +/** + Function for processing blocks + @param md The hash state + @param buf The data to hash + @param len The length of the data (octets) + @return CRYPT_OK if successful +*/ +static int ss_chc_process(hash_state * md, const unsigned char *in, unsigned long inlen); +static HASH_PROCESS(ss_chc_process, s_chc_compress, chc, (unsigned long)cipher_blocksize) + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +int chc_process(hash_state * md, const unsigned char *in, unsigned long inlen) +{ + int err; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + + /* is the cipher valid? */ + if ((err = cipher_is_valid(cipher_idx)) != CRYPT_OK) { + return err; + } + if (cipher_blocksize != cipher_descriptor[cipher_idx]->block_length) { + return CRYPT_INVALID_CIPHER; + } + + return ss_chc_process(md, in, inlen); +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (length of the block size of the block cipher) + @return CRYPT_OK if successful +*/ +int chc_done(hash_state *md, unsigned char *out) +{ + int err; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + /* is the cipher valid? */ + if ((err = cipher_is_valid(cipher_idx)) != CRYPT_OK) { + return err; + } + if (cipher_blocksize != cipher_descriptor[cipher_idx]->block_length) { + return CRYPT_INVALID_CIPHER; + } + + if (md->chc.curlen >= sizeof(md->chc.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->chc.length += md->chc.curlen * 8; + + /* append the '1' bit */ + md->chc.buf[md->chc.curlen++] = (unsigned char)0x80; + + /* if the length is currently above l-8 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->chc.curlen > (unsigned long)(cipher_blocksize - 8)) { + while (md->chc.curlen < (unsigned long)cipher_blocksize) { + md->chc.buf[md->chc.curlen++] = (unsigned char)0; + } + s_chc_compress(md, md->chc.buf); + md->chc.curlen = 0; + } + + /* pad upto l-8 bytes of zeroes */ + while (md->chc.curlen < (unsigned long)(cipher_blocksize - 8)) { + md->chc.buf[md->chc.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->chc.length, md->chc.buf+(cipher_blocksize-8)); + s_chc_compress(md, md->chc.buf); + + /* copy output */ + XMEMCPY(out, md->chc.state, cipher_blocksize); + +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int chc_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char *msg, + hash[MAXBLOCKSIZE]; + int len; + } tests[] = { +{ + (unsigned char *)"hello world", + { 0xcf, 0x57, 0x9d, 0xc3, 0x0a, 0x0e, 0xea, 0x61, + 0x0d, 0x54, 0x47, 0xc4, 0x3c, 0x06, 0xf5, 0x4e }, + 16 +} +}; + int i, oldhashidx, idx, err; + unsigned char tmp[MAXBLOCKSIZE]; + hash_state md; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + oldhashidx = cipher_idx; + chc_register(idx); + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + if ((err = chc_init(&md)) != CRYPT_OK) { + return err; + } + if ((err = chc_process(&md, tests[i].msg, XSTRLEN((char *)tests[i].msg))) != CRYPT_OK) { + return err; + } + if ((err = chc_done(&md, tmp)) != CRYPT_OK) { + return err; + } + if (compare_testvector(tmp, tests[i].len, tests[i].hash, tests[i].len, "CHC", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + if (oldhashidx != UNDEFED_HASH) { + chc_register(oldhashidx); + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_file.c b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_file.c new file mode 100644 index 0000000..d2711db --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_file.c @@ -0,0 +1,43 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifndef LTC_NO_FILE +/** + @file hash_file.c + Hash a file, Tom St Denis +*/ + +/** + @param hash The index of the hash desired + @param fname The name of the file you wish to hash + @param out [out] The destination of the digest + @param outlen [in/out] The max size and resulting size of the message digest + @result CRYPT_OK if successful +*/ +int hash_file(int hash, const char *fname, unsigned char *out, unsigned long *outlen) +{ + FILE *in; + int err; + LTC_ARGCHK(fname != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + + in = fopen(fname, "rb"); + if (in == NULL) { + return CRYPT_FILE_NOTFOUND; + } + + err = hash_filehandle(hash, in, out, outlen); + if (fclose(in) != 0) { + return CRYPT_ERROR; + } + + return err; +} +#endif /* #ifndef LTC_NO_FILE */ + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_filehandle.c b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_filehandle.c new file mode 100644 index 0000000..8ebf52a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_filehandle.c @@ -0,0 +1,64 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifndef LTC_NO_FILE +/** + @file hash_filehandle.c + Hash open files, Tom St Denis +*/ + +/** + Hash data from an open file handle. + @param hash The index of the hash you want to use + @param in The FILE* handle of the file you want to hash + @param out [out] The destination of the digest + @param outlen [in/out] The max size and resulting size of the digest + @result CRYPT_OK if successful +*/ +int hash_filehandle(int hash, FILE *in, unsigned char *out, unsigned long *outlen) +{ + hash_state md; + unsigned char *buf; + size_t x; + int err; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(in != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + goto LBL_ERR; + } + + if (*outlen < hash_descriptor[hash]->hashsize) { + *outlen = hash_descriptor[hash]->hashsize; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + if ((err = hash_descriptor[hash]->init(&md)) != CRYPT_OK) { + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = hash_descriptor[hash]->process(&md, buf, (unsigned long)x)) != CRYPT_OK) { + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + if ((err = hash_descriptor[hash]->done(&md, out)) == CRYPT_OK) { + *outlen = hash_descriptor[hash]->hashsize; + } + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: + XFREE(buf); + return err; +} +#endif /* #ifndef LTC_NO_FILE */ + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_memory.c b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_memory.c new file mode 100644 index 0000000..1dfd6f6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_memory.c @@ -0,0 +1,59 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_HASH_HELPERS +/** + @file hash_memory.c + Hash memory helper, Tom St Denis +*/ + +/** + Hash a block of memory and store the digest. + @param hash The index of the hash you wish to use + @param in The data you wish to hash + @param inlen The length of the data to hash (octets) + @param out [out] Where to store the digest + @param outlen [in/out] Max size and resulting size of the digest + @return CRYPT_OK if successful +*/ +int hash_memory(int hash, const unsigned char *in, unsigned long inlen, unsigned char *out, unsigned long *outlen) +{ + hash_state *md; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + + if (*outlen < hash_descriptor[hash]->hashsize) { + *outlen = hash_descriptor[hash]->hashsize; + return CRYPT_BUFFER_OVERFLOW; + } + + md = XMALLOC(sizeof(hash_state)); + if (md == NULL) { + return CRYPT_MEM; + } + + if ((err = hash_descriptor[hash]->init(md)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash]->process(md, in, inlen)) != CRYPT_OK) { + goto LBL_ERR; + } + err = hash_descriptor[hash]->done(md, out); + *outlen = hash_descriptor[hash]->hashsize; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + XFREE(md); + + return err; +} +#endif /* #ifdef LTC_HASH_HELPERS */ diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_memory_multi.c new file mode 100644 index 0000000..e58b3c5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/helper/hash_memory_multi.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +#ifdef LTC_HASH_HELPERS +/** + @file hash_memory_multi.c + Hash (multiple buffers) memory helper, Tom St Denis +*/ + +/** + Hash multiple (non-adjacent) blocks of memory at once. + @param hash The index of the hash you wish to use + @param out [out] Where to store the digest + @param outlen [in/out] Max size and resulting size of the digest + @param in The data you wish to hash + @param inlen The length of the data to hash (octets) + @param ... tuples of (data,len) pairs to hash, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int hash_memory_multi(int hash, unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) +{ + hash_state *md; + int err; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + + if (*outlen < hash_descriptor[hash]->hashsize) { + *outlen = hash_descriptor[hash]->hashsize; + return CRYPT_BUFFER_OVERFLOW; + } + + md = XMALLOC(sizeof(hash_state)); + if (md == NULL) { + return CRYPT_MEM; + } + + if ((err = hash_descriptor[hash]->init(md)) != CRYPT_OK) { + goto LBL_ERR; + } + + va_start(args, inlen); + curptr = in; + curlen = inlen; + for (;;) { + /* process buf */ + if ((err = hash_descriptor[hash]->process(md, curptr, curlen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* step to next */ + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) { + break; + } + curlen = va_arg(args, unsigned long); + } + err = hash_descriptor[hash]->done(md, out); + *outlen = hash_descriptor[hash]->hashsize; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + XFREE(md); + va_end(args); + return err; +} +#endif /* #ifdef LTC_HASH_HELPERS */ diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/helper/sub.mk b/optee_os/core/lib/libtomcrypt/src/hashes/helper/sub.mk new file mode 100644 index 0000000..c01be6d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/helper/sub.mk @@ -0,0 +1,4 @@ +cflags-y += -Wno-unused-parameter + +srcs-y += hash_memory.c +srcs-y += hash_memory_multi.c diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/md2.c b/optee_os/core/lib/libtomcrypt/src/hashes/md2.c new file mode 100644 index 0000000..6cf0a65 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/md2.c @@ -0,0 +1,240 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @param md2.c + LTC_MD2 (RFC 1319) hash function implementation by Tom St Denis +*/ + +#ifdef LTC_MD2 + +const struct ltc_hash_descriptor md2_desc = +{ + "md2", + 7, + 16, + 16, + + /* OID */ + { 1, 2, 840, 113549, 2, 2, }, + 6, + + &md2_init, + &md2_process, + &md2_done, + &md2_test, + NULL +}; + +static const unsigned char PI_SUBST[256] = { + 41, 46, 67, 201, 162, 216, 124, 1, 61, 54, 84, 161, 236, 240, 6, + 19, 98, 167, 5, 243, 192, 199, 115, 140, 152, 147, 43, 217, 188, + 76, 130, 202, 30, 155, 87, 60, 253, 212, 224, 22, 103, 66, 111, 24, + 138, 23, 229, 18, 190, 78, 196, 214, 218, 158, 222, 73, 160, 251, + 245, 142, 187, 47, 238, 122, 169, 104, 121, 145, 21, 178, 7, 63, + 148, 194, 16, 137, 11, 34, 95, 33, 128, 127, 93, 154, 90, 144, 50, + 39, 53, 62, 204, 231, 191, 247, 151, 3, 255, 25, 48, 179, 72, 165, + 181, 209, 215, 94, 146, 42, 172, 86, 170, 198, 79, 184, 56, 210, + 150, 164, 125, 182, 118, 252, 107, 226, 156, 116, 4, 241, 69, 157, + 112, 89, 100, 113, 135, 32, 134, 91, 207, 101, 230, 45, 168, 2, 27, + 96, 37, 173, 174, 176, 185, 246, 28, 70, 97, 105, 52, 64, 126, 15, + 85, 71, 163, 35, 221, 81, 175, 58, 195, 92, 249, 206, 186, 197, + 234, 38, 44, 83, 13, 110, 133, 40, 132, 9, 211, 223, 205, 244, 65, + 129, 77, 82, 106, 220, 55, 200, 108, 193, 171, 250, 36, 225, 123, + 8, 12, 189, 177, 74, 120, 136, 149, 139, 227, 99, 232, 109, 233, + 203, 213, 254, 59, 0, 29, 57, 242, 239, 183, 14, 102, 88, 208, 228, + 166, 119, 114, 248, 235, 117, 75, 10, 49, 68, 80, 180, 143, 237, + 31, 26, 219, 153, 141, 51, 159, 17, 131, 20 +}; + +/* adds 16 bytes to the checksum */ +static void s_md2_update_chksum(hash_state *md) +{ + int j; + unsigned char L; + L = md->md2.chksum[15]; + for (j = 0; j < 16; j++) { + +/* caution, the RFC says its "C[j] = S[M[i*16+j] xor L]" but the reference source code [and test vectors] say + otherwise. +*/ + L = (md->md2.chksum[j] ^= PI_SUBST[(int)(md->md2.buf[j] ^ L)] & 255); + } +} + +static void s_md2_compress(hash_state *md) +{ + int j, k; + unsigned char t; + + /* copy block */ + for (j = 0; j < 16; j++) { + md->md2.X[16+j] = md->md2.buf[j]; + md->md2.X[32+j] = md->md2.X[j] ^ md->md2.X[16+j]; + } + + t = (unsigned char)0; + + /* do 18 rounds */ + for (j = 0; j < 18; j++) { + for (k = 0; k < 48; k++) { + t = (md->md2.X[k] ^= PI_SUBST[(int)(t & 255)]); + } + t = (t + (unsigned char)j) & 255; + } +} + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int md2_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + + /* LTC_MD2 uses a zero'ed state... */ + zeromem(md->md2.X, sizeof(md->md2.X)); + zeromem(md->md2.chksum, sizeof(md->md2.chksum)); + zeromem(md->md2.buf, sizeof(md->md2.buf)); + md->md2.curlen = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +int md2_process(hash_state *md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + if (md-> md2 .curlen > sizeof(md-> md2 .buf)) { + return CRYPT_INVALID_ARG; + } + while (inlen > 0) { + n = MIN(inlen, (16 - md->md2.curlen)); + XMEMCPY(md->md2.buf + md->md2.curlen, in, (size_t)n); + md->md2.curlen += n; + in += n; + inlen -= n; + + /* is 16 bytes full? */ + if (md->md2.curlen == 16) { + s_md2_compress(md); + s_md2_update_chksum(md); + md->md2.curlen = 0; + } + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (16 bytes) + @return CRYPT_OK if successful +*/ +int md2_done(hash_state * md, unsigned char *out) +{ + unsigned long i, k; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->md2.curlen >= sizeof(md->md2.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* pad the message */ + k = 16 - md->md2.curlen; + for (i = md->md2.curlen; i < 16; i++) { + md->md2.buf[i] = (unsigned char)k; + } + + /* hash and update */ + s_md2_compress(md); + s_md2_update_chksum(md); + + /* hash checksum */ + XMEMCPY(md->md2.buf, md->md2.chksum, 16); + s_md2_compress(md); + + /* output is lower 16 bytes of X */ + XMEMCPY(out, md->md2.X, 16); + +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int md2_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[16]; + } tests[] = { + { "", + {0x83,0x50,0xe5,0xa3,0xe2,0x4c,0x15,0x3d, + 0xf2,0x27,0x5c,0x9f,0x80,0x69,0x27,0x73 + } + }, + { "a", + {0x32,0xec,0x01,0xec,0x4a,0x6d,0xac,0x72, + 0xc0,0xab,0x96,0xfb,0x34,0xc0,0xb5,0xd1 + } + }, + { "message digest", + {0xab,0x4f,0x49,0x6b,0xfb,0x2a,0x53,0x0b, + 0x21,0x9f,0xf3,0x30,0x31,0xfe,0x06,0xb0 + } + }, + { "abcdefghijklmnopqrstuvwxyz", + {0x4e,0x8d,0xdf,0xf3,0x65,0x02,0x92,0xab, + 0x5a,0x41,0x08,0xc3,0xaa,0x47,0x94,0x0b + } + }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + {0xda,0x33,0xde,0xf2,0xa4,0x2d,0xf1,0x39, + 0x75,0x35,0x28,0x46,0xc3,0x03,0x38,0xcd + } + }, + { "12345678901234567890123456789012345678901234567890123456789012345678901234567890", + {0xd5,0x97,0x6f,0x79,0xd8,0x3d,0x3a,0x0d, + 0xc9,0x80,0x6c,0x3c,0x66,0xf3,0xef,0xd8 + } + } + }; + + int i; + unsigned char tmp[16]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + md2_init(&md); + md2_process(&md, (unsigned char*)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + md2_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "MD2", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/md4.c b/optee_os/core/lib/libtomcrypt/src/hashes/md4.c new file mode 100644 index 0000000..aadad5f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/md4.c @@ -0,0 +1,296 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @param md4.c + Submitted by Dobes Vandermeer (dobes@smartt.com) +*/ + +#ifdef LTC_MD4 + +const struct ltc_hash_descriptor md4_desc = +{ + "md4", + 6, + 16, + 64, + + /* OID */ + { 1, 2, 840, 113549, 2, 4, }, + 6, + + &md4_init, + &md4_process, + &md4_done, + &md4_test, + NULL +}; + +#define S11 3 +#define S12 7 +#define S13 11 +#define S14 19 +#define S21 3 +#define S22 5 +#define S23 9 +#define S24 13 +#define S31 3 +#define S32 9 +#define S33 11 +#define S34 15 + +/* F, G and H are basic LTC_MD4 functions. */ +#define F(x, y, z) (z ^ (x & (y ^ z))) +#define G(x, y, z) ((x & y) | (z & (x | y))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) + +/* ROTATE_LEFT rotates x left n bits. */ +#define ROTATE_LEFT(x, n) ROLc(x, n) + +/* FF, GG and HH are transformations for rounds 1, 2 and 3 */ +/* Rotation is separate from addition to prevent recomputation */ + +#define FF(a, b, c, d, x, s) { \ + (a) += F ((b), (c), (d)) + (x); \ + (a) = ROTATE_LEFT ((a), (s)); \ + } +#define GG(a, b, c, d, x, s) { \ + (a) += G ((b), (c), (d)) + (x) + 0x5a827999UL; \ + (a) = ROTATE_LEFT ((a), (s)); \ + } +#define HH(a, b, c, d, x, s) { \ + (a) += H ((b), (c), (d)) + (x) + 0x6ed9eba1UL; \ + (a) = ROTATE_LEFT ((a), (s)); \ + } + +#ifdef LTC_CLEAN_STACK +static int ss_md4_compress(hash_state *md, const unsigned char *buf) +#else +static int s_md4_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong32 x[16], a, b, c, d; + int i; + + /* copy state */ + a = md->md4.state[0]; + b = md->md4.state[1]; + c = md->md4.state[2]; + d = md->md4.state[3]; + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32L(x[i], buf + (4*i)); + } + + /* Round 1 */ + FF (a, b, c, d, x[ 0], S11); /* 1 */ + FF (d, a, b, c, x[ 1], S12); /* 2 */ + FF (c, d, a, b, x[ 2], S13); /* 3 */ + FF (b, c, d, a, x[ 3], S14); /* 4 */ + FF (a, b, c, d, x[ 4], S11); /* 5 */ + FF (d, a, b, c, x[ 5], S12); /* 6 */ + FF (c, d, a, b, x[ 6], S13); /* 7 */ + FF (b, c, d, a, x[ 7], S14); /* 8 */ + FF (a, b, c, d, x[ 8], S11); /* 9 */ + FF (d, a, b, c, x[ 9], S12); /* 10 */ + FF (c, d, a, b, x[10], S13); /* 11 */ + FF (b, c, d, a, x[11], S14); /* 12 */ + FF (a, b, c, d, x[12], S11); /* 13 */ + FF (d, a, b, c, x[13], S12); /* 14 */ + FF (c, d, a, b, x[14], S13); /* 15 */ + FF (b, c, d, a, x[15], S14); /* 16 */ + + /* Round 2 */ + GG (a, b, c, d, x[ 0], S21); /* 17 */ + GG (d, a, b, c, x[ 4], S22); /* 18 */ + GG (c, d, a, b, x[ 8], S23); /* 19 */ + GG (b, c, d, a, x[12], S24); /* 20 */ + GG (a, b, c, d, x[ 1], S21); /* 21 */ + GG (d, a, b, c, x[ 5], S22); /* 22 */ + GG (c, d, a, b, x[ 9], S23); /* 23 */ + GG (b, c, d, a, x[13], S24); /* 24 */ + GG (a, b, c, d, x[ 2], S21); /* 25 */ + GG (d, a, b, c, x[ 6], S22); /* 26 */ + GG (c, d, a, b, x[10], S23); /* 27 */ + GG (b, c, d, a, x[14], S24); /* 28 */ + GG (a, b, c, d, x[ 3], S21); /* 29 */ + GG (d, a, b, c, x[ 7], S22); /* 30 */ + GG (c, d, a, b, x[11], S23); /* 31 */ + GG (b, c, d, a, x[15], S24); /* 32 */ + + /* Round 3 */ + HH (a, b, c, d, x[ 0], S31); /* 33 */ + HH (d, a, b, c, x[ 8], S32); /* 34 */ + HH (c, d, a, b, x[ 4], S33); /* 35 */ + HH (b, c, d, a, x[12], S34); /* 36 */ + HH (a, b, c, d, x[ 2], S31); /* 37 */ + HH (d, a, b, c, x[10], S32); /* 38 */ + HH (c, d, a, b, x[ 6], S33); /* 39 */ + HH (b, c, d, a, x[14], S34); /* 40 */ + HH (a, b, c, d, x[ 1], S31); /* 41 */ + HH (d, a, b, c, x[ 9], S32); /* 42 */ + HH (c, d, a, b, x[ 5], S33); /* 43 */ + HH (b, c, d, a, x[13], S34); /* 44 */ + HH (a, b, c, d, x[ 3], S31); /* 45 */ + HH (d, a, b, c, x[11], S32); /* 46 */ + HH (c, d, a, b, x[ 7], S33); /* 47 */ + HH (b, c, d, a, x[15], S34); /* 48 */ + + + /* Update our state */ + md->md4.state[0] = md->md4.state[0] + a; + md->md4.state[1] = md->md4.state[1] + b; + md->md4.state[2] = md->md4.state[2] + c; + md->md4.state[3] = md->md4.state[3] + d; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_md4_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_md4_compress(md, buf); + burn_stack(sizeof(ulong32) * 20 + sizeof(int)); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int md4_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->md4.state[0] = 0x67452301UL; + md->md4.state[1] = 0xefcdab89UL; + md->md4.state[2] = 0x98badcfeUL; + md->md4.state[3] = 0x10325476UL; + md->md4.length = 0; + md->md4.curlen = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(md4_process, s_md4_compress, md4, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (16 bytes) + @return CRYPT_OK if successful +*/ +int md4_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->md4.curlen >= sizeof(md->md4.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->md4.length += md->md4.curlen * 8; + + /* append the '1' bit */ + md->md4.buf[md->md4.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->md4.curlen > 56) { + while (md->md4.curlen < 64) { + md->md4.buf[md->md4.curlen++] = (unsigned char)0; + } + s_md4_compress(md, md->md4.buf); + md->md4.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->md4.curlen < 56) { + md->md4.buf[md->md4.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->md4.length, md->md4.buf+56); + s_md4_compress(md, md->md4.buf); + + /* copy output */ + for (i = 0; i < 4; i++) { + STORE32L(md->md4.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int md4_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct md4_test_case { + const char *input; + unsigned char hash[16]; + } tests[] = { + { "", + {0x31, 0xd6, 0xcf, 0xe0, 0xd1, 0x6a, 0xe9, 0x31, + 0xb7, 0x3c, 0x59, 0xd7, 0xe0, 0xc0, 0x89, 0xc0} }, + { "a", + {0xbd, 0xe5, 0x2c, 0xb3, 0x1d, 0xe3, 0x3e, 0x46, + 0x24, 0x5e, 0x05, 0xfb, 0xdb, 0xd6, 0xfb, 0x24} }, + { "abc", + {0xa4, 0x48, 0x01, 0x7a, 0xaf, 0x21, 0xd8, 0x52, + 0x5f, 0xc1, 0x0a, 0xe8, 0x7a, 0xa6, 0x72, 0x9d} }, + { "message digest", + {0xd9, 0x13, 0x0a, 0x81, 0x64, 0x54, 0x9f, 0xe8, + 0x18, 0x87, 0x48, 0x06, 0xe1, 0xc7, 0x01, 0x4b} }, + { "abcdefghijklmnopqrstuvwxyz", + {0xd7, 0x9e, 0x1c, 0x30, 0x8a, 0xa5, 0xbb, 0xcd, + 0xee, 0xa8, 0xed, 0x63, 0xdf, 0x41, 0x2d, 0xa9} }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + {0x04, 0x3f, 0x85, 0x82, 0xf2, 0x41, 0xdb, 0x35, + 0x1c, 0xe6, 0x27, 0xe1, 0x53, 0xe7, 0xf0, 0xe4} }, + { "12345678901234567890123456789012345678901234567890123456789012345678901234567890", + {0xe3, 0x3b, 0x4d, 0xdc, 0x9c, 0x38, 0xf2, 0x19, + 0x9c, 0x3e, 0x7b, 0x16, 0x4f, 0xcc, 0x05, 0x36} }, + }; + + int i; + unsigned char tmp[16]; + hash_state md; + + for(i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + md4_init(&md); + md4_process(&md, (unsigned char *)tests[i].input, (unsigned long)XSTRLEN(tests[i].input)); + md4_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "MD4", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + + } + return CRYPT_OK; + #endif +} + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/md5.c b/optee_os/core/lib/libtomcrypt/src/hashes/md5.c new file mode 100644 index 0000000..ad404e1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/md5.c @@ -0,0 +1,356 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +/** + @file md5.c + LTC_MD5 hash function by Tom St Denis +*/ + +#ifdef LTC_MD5 + +const struct ltc_hash_descriptor md5_desc = +{ + "md5", + 3, + 16, + 64, + + /* OID */ + { 1, 2, 840, 113549, 2, 5, }, + 6, + + &md5_init, + &md5_process, + &md5_done, + &md5_test, + NULL +}; + +#define F(x,y,z) (z ^ (x & (y ^ z))) +#define G(x,y,z) (y ^ (z & (y ^ x))) +#define H(x,y,z) (x^y^z) +#define I(x,y,z) (y^(x|(~z))) + +#ifdef LTC_SMALL_CODE + +#define FF(a,b,c,d,M,s,t) \ + a = (a + F(b,c,d) + M + t); a = ROL(a, s) + b; + +#define GG(a,b,c,d,M,s,t) \ + a = (a + G(b,c,d) + M + t); a = ROL(a, s) + b; + +#define HH(a,b,c,d,M,s,t) \ + a = (a + H(b,c,d) + M + t); a = ROL(a, s) + b; + +#define II(a,b,c,d,M,s,t) \ + a = (a + I(b,c,d) + M + t); a = ROL(a, s) + b; + +static const unsigned char Worder[64] = { + 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, + 1,6,11,0,5,10,15,4,9,14,3,8,13,2,7,12, + 5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2, + 0,7,14,5,12,3,10,1,8,15,6,13,4,11,2,9 +}; + +static const unsigned char Rorder[64] = { + 7,12,17,22,7,12,17,22,7,12,17,22,7,12,17,22, + 5,9,14,20,5,9,14,20,5,9,14,20,5,9,14,20, + 4,11,16,23,4,11,16,23,4,11,16,23,4,11,16,23, + 6,10,15,21,6,10,15,21,6,10,15,21,6,10,15,21 +}; + +static const ulong32 Korder[64] = { +0xd76aa478UL, 0xe8c7b756UL, 0x242070dbUL, 0xc1bdceeeUL, 0xf57c0fafUL, 0x4787c62aUL, 0xa8304613UL, 0xfd469501UL, +0x698098d8UL, 0x8b44f7afUL, 0xffff5bb1UL, 0x895cd7beUL, 0x6b901122UL, 0xfd987193UL, 0xa679438eUL, 0x49b40821UL, +0xf61e2562UL, 0xc040b340UL, 0x265e5a51UL, 0xe9b6c7aaUL, 0xd62f105dUL, 0x02441453UL, 0xd8a1e681UL, 0xe7d3fbc8UL, +0x21e1cde6UL, 0xc33707d6UL, 0xf4d50d87UL, 0x455a14edUL, 0xa9e3e905UL, 0xfcefa3f8UL, 0x676f02d9UL, 0x8d2a4c8aUL, +0xfffa3942UL, 0x8771f681UL, 0x6d9d6122UL, 0xfde5380cUL, 0xa4beea44UL, 0x4bdecfa9UL, 0xf6bb4b60UL, 0xbebfbc70UL, +0x289b7ec6UL, 0xeaa127faUL, 0xd4ef3085UL, 0x04881d05UL, 0xd9d4d039UL, 0xe6db99e5UL, 0x1fa27cf8UL, 0xc4ac5665UL, +0xf4292244UL, 0x432aff97UL, 0xab9423a7UL, 0xfc93a039UL, 0x655b59c3UL, 0x8f0ccc92UL, 0xffeff47dUL, 0x85845dd1UL, +0x6fa87e4fUL, 0xfe2ce6e0UL, 0xa3014314UL, 0x4e0811a1UL, 0xf7537e82UL, 0xbd3af235UL, 0x2ad7d2bbUL, 0xeb86d391UL +}; + +#else + +#define FF(a,b,c,d,M,s,t) \ + a = (a + F(b,c,d) + M + t); a = ROLc(a, s) + b; + +#define GG(a,b,c,d,M,s,t) \ + a = (a + G(b,c,d) + M + t); a = ROLc(a, s) + b; + +#define HH(a,b,c,d,M,s,t) \ + a = (a + H(b,c,d) + M + t); a = ROLc(a, s) + b; + +#define II(a,b,c,d,M,s,t) \ + a = (a + I(b,c,d) + M + t); a = ROLc(a, s) + b; + + +#endif + +#ifdef LTC_CLEAN_STACK +static int ss_md5_compress(hash_state *md, const unsigned char *buf) +#else +static int s_md5_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong32 i, W[16], a, b, c, d; +#ifdef LTC_SMALL_CODE + ulong32 t; +#endif + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32L(W[i], buf + (4*i)); + } + + /* copy state */ + a = md->md5.state[0]; + b = md->md5.state[1]; + c = md->md5.state[2]; + d = md->md5.state[3]; + +#ifdef LTC_SMALL_CODE + for (i = 0; i < 16; ++i) { + FF(a,b,c,d,W[Worder[i]],Rorder[i],Korder[i]); + t = d; d = c; c = b; b = a; a = t; + } + + for (; i < 32; ++i) { + GG(a,b,c,d,W[Worder[i]],Rorder[i],Korder[i]); + t = d; d = c; c = b; b = a; a = t; + } + + for (; i < 48; ++i) { + HH(a,b,c,d,W[Worder[i]],Rorder[i],Korder[i]); + t = d; d = c; c = b; b = a; a = t; + } + + for (; i < 64; ++i) { + II(a,b,c,d,W[Worder[i]],Rorder[i],Korder[i]); + t = d; d = c; c = b; b = a; a = t; + } + +#else + FF(a,b,c,d,W[0],7,0xd76aa478UL) + FF(d,a,b,c,W[1],12,0xe8c7b756UL) + FF(c,d,a,b,W[2],17,0x242070dbUL) + FF(b,c,d,a,W[3],22,0xc1bdceeeUL) + FF(a,b,c,d,W[4],7,0xf57c0fafUL) + FF(d,a,b,c,W[5],12,0x4787c62aUL) + FF(c,d,a,b,W[6],17,0xa8304613UL) + FF(b,c,d,a,W[7],22,0xfd469501UL) + FF(a,b,c,d,W[8],7,0x698098d8UL) + FF(d,a,b,c,W[9],12,0x8b44f7afUL) + FF(c,d,a,b,W[10],17,0xffff5bb1UL) + FF(b,c,d,a,W[11],22,0x895cd7beUL) + FF(a,b,c,d,W[12],7,0x6b901122UL) + FF(d,a,b,c,W[13],12,0xfd987193UL) + FF(c,d,a,b,W[14],17,0xa679438eUL) + FF(b,c,d,a,W[15],22,0x49b40821UL) + GG(a,b,c,d,W[1],5,0xf61e2562UL) + GG(d,a,b,c,W[6],9,0xc040b340UL) + GG(c,d,a,b,W[11],14,0x265e5a51UL) + GG(b,c,d,a,W[0],20,0xe9b6c7aaUL) + GG(a,b,c,d,W[5],5,0xd62f105dUL) + GG(d,a,b,c,W[10],9,0x02441453UL) + GG(c,d,a,b,W[15],14,0xd8a1e681UL) + GG(b,c,d,a,W[4],20,0xe7d3fbc8UL) + GG(a,b,c,d,W[9],5,0x21e1cde6UL) + GG(d,a,b,c,W[14],9,0xc33707d6UL) + GG(c,d,a,b,W[3],14,0xf4d50d87UL) + GG(b,c,d,a,W[8],20,0x455a14edUL) + GG(a,b,c,d,W[13],5,0xa9e3e905UL) + GG(d,a,b,c,W[2],9,0xfcefa3f8UL) + GG(c,d,a,b,W[7],14,0x676f02d9UL) + GG(b,c,d,a,W[12],20,0x8d2a4c8aUL) + HH(a,b,c,d,W[5],4,0xfffa3942UL) + HH(d,a,b,c,W[8],11,0x8771f681UL) + HH(c,d,a,b,W[11],16,0x6d9d6122UL) + HH(b,c,d,a,W[14],23,0xfde5380cUL) + HH(a,b,c,d,W[1],4,0xa4beea44UL) + HH(d,a,b,c,W[4],11,0x4bdecfa9UL) + HH(c,d,a,b,W[7],16,0xf6bb4b60UL) + HH(b,c,d,a,W[10],23,0xbebfbc70UL) + HH(a,b,c,d,W[13],4,0x289b7ec6UL) + HH(d,a,b,c,W[0],11,0xeaa127faUL) + HH(c,d,a,b,W[3],16,0xd4ef3085UL) + HH(b,c,d,a,W[6],23,0x04881d05UL) + HH(a,b,c,d,W[9],4,0xd9d4d039UL) + HH(d,a,b,c,W[12],11,0xe6db99e5UL) + HH(c,d,a,b,W[15],16,0x1fa27cf8UL) + HH(b,c,d,a,W[2],23,0xc4ac5665UL) + II(a,b,c,d,W[0],6,0xf4292244UL) + II(d,a,b,c,W[7],10,0x432aff97UL) + II(c,d,a,b,W[14],15,0xab9423a7UL) + II(b,c,d,a,W[5],21,0xfc93a039UL) + II(a,b,c,d,W[12],6,0x655b59c3UL) + II(d,a,b,c,W[3],10,0x8f0ccc92UL) + II(c,d,a,b,W[10],15,0xffeff47dUL) + II(b,c,d,a,W[1],21,0x85845dd1UL) + II(a,b,c,d,W[8],6,0x6fa87e4fUL) + II(d,a,b,c,W[15],10,0xfe2ce6e0UL) + II(c,d,a,b,W[6],15,0xa3014314UL) + II(b,c,d,a,W[13],21,0x4e0811a1UL) + II(a,b,c,d,W[4],6,0xf7537e82UL) + II(d,a,b,c,W[11],10,0xbd3af235UL) + II(c,d,a,b,W[2],15,0x2ad7d2bbUL) + II(b,c,d,a,W[9],21,0xeb86d391UL) +#endif + + md->md5.state[0] = md->md5.state[0] + a; + md->md5.state[1] = md->md5.state[1] + b; + md->md5.state[2] = md->md5.state[2] + c; + md->md5.state[3] = md->md5.state[3] + d; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_md5_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_md5_compress(md, buf); + burn_stack(sizeof(ulong32) * 21); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int md5_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->md5.state[0] = 0x67452301UL; + md->md5.state[1] = 0xefcdab89UL; + md->md5.state[2] = 0x98badcfeUL; + md->md5.state[3] = 0x10325476UL; + md->md5.curlen = 0; + md->md5.length = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(md5_process, s_md5_compress, md5, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (16 bytes) + @return CRYPT_OK if successful +*/ +int md5_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->md5.curlen >= sizeof(md->md5.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->md5.length += md->md5.curlen * 8; + + /* append the '1' bit */ + md->md5.buf[md->md5.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->md5.curlen > 56) { + while (md->md5.curlen < 64) { + md->md5.buf[md->md5.curlen++] = (unsigned char)0; + } + s_md5_compress(md, md->md5.buf); + md->md5.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->md5.curlen < 56) { + md->md5.buf[md->md5.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->md5.length, md->md5.buf+56); + s_md5_compress(md, md->md5.buf); + + /* copy output */ + for (i = 0; i < 4; i++) { + STORE32L(md->md5.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int md5_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[16]; + } tests[] = { + { "", + { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, + 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } }, + { "a", + {0x0c, 0xc1, 0x75, 0xb9, 0xc0, 0xf1, 0xb6, 0xa8, + 0x31, 0xc3, 0x99, 0xe2, 0x69, 0x77, 0x26, 0x61 } }, + { "abc", + { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, + 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } }, + { "message digest", + { 0xf9, 0x6b, 0x69, 0x7d, 0x7c, 0xb7, 0x93, 0x8d, + 0x52, 0x5a, 0x2f, 0x31, 0xaa, 0xf1, 0x61, 0xd0 } }, + { "abcdefghijklmnopqrstuvwxyz", + { 0xc3, 0xfc, 0xd3, 0xd7, 0x61, 0x92, 0xe4, 0x00, + 0x7d, 0xfb, 0x49, 0x6c, 0xca, 0x67, 0xe1, 0x3b } }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + { 0xd1, 0x74, 0xab, 0x98, 0xd2, 0x77, 0xd9, 0xf5, + 0xa5, 0x61, 0x1c, 0x2c, 0x9f, 0x41, 0x9d, 0x9f } }, + { "12345678901234567890123456789012345678901234567890123456789012345678901234567890", + { 0x57, 0xed, 0xf4, 0xa2, 0x2b, 0xe3, 0xc9, 0x55, + 0xac, 0x49, 0xda, 0x2e, 0x21, 0x07, 0xb6, 0x7a } }, + { NULL, { 0 } } + }; + + int i; + unsigned char tmp[16]; + hash_state md; + + for (i = 0; tests[i].msg != NULL; i++) { + md5_init(&md); + md5_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + md5_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "MD5", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/rmd128.c b/optee_os/core/lib/libtomcrypt/src/hashes/rmd128.c new file mode 100644 index 0000000..e8f63e1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/rmd128.c @@ -0,0 +1,396 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @param rmd128.c + RMD128 Hash function +*/ + +/* Implementation of LTC_RIPEMD-128 based on the source by Antoon Bosselaers, ESAT-COSIC + * + * This source has been radically overhauled to be portable and work within + * the LibTomCrypt API by Tom St Denis + */ + +#ifdef LTC_RIPEMD128 + +const struct ltc_hash_descriptor rmd128_desc = +{ + "rmd128", + 8, + 16, + 64, + + /* OID */ + { 1, 0, 10118, 3, 0, 50 }, + 6, + + &rmd128_init, + &rmd128_process, + &rmd128_done, + &rmd128_test, + NULL +}; + +/* the four basic functions F(), G() and H() */ +#define F(x, y, z) ((x) ^ (y) ^ (z)) +#define G(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define H(x, y, z) (((x) | ~(y)) ^ (z)) +#define I(x, y, z) (((x) & (z)) | ((y) & ~(z))) + +/* the eight basic operations FF() through III() */ +#define FF(a, b, c, d, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)); + +#define GG(a, b, c, d, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x5a827999UL;\ + (a) = ROLc((a), (s)); + +#define HH(a, b, c, d, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x6ed9eba1UL;\ + (a) = ROLc((a), (s)); + +#define II(a, b, c, d, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x8f1bbcdcUL;\ + (a) = ROLc((a), (s)); + +#define FFF(a, b, c, d, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)); + +#define GGG(a, b, c, d, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x6d703ef3UL;\ + (a) = ROLc((a), (s)); + +#define HHH(a, b, c, d, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x5c4dd124UL;\ + (a) = ROLc((a), (s)); + +#define III(a, b, c, d, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x50a28be6UL;\ + (a) = ROLc((a), (s)); + +#ifdef LTC_CLEAN_STACK +static int ss_rmd128_compress(hash_state *md, const unsigned char *buf) +#else +static int s_rmd128_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong32 aa,bb,cc,dd,aaa,bbb,ccc,ddd,X[16]; + int i; + + /* load words X */ + for (i = 0; i < 16; i++){ + LOAD32L(X[i], buf + (4 * i)); + } + + /* load state */ + aa = aaa = md->rmd128.state[0]; + bb = bbb = md->rmd128.state[1]; + cc = ccc = md->rmd128.state[2]; + dd = ddd = md->rmd128.state[3]; + + /* round 1 */ + FF(aa, bb, cc, dd, X[ 0], 11); + FF(dd, aa, bb, cc, X[ 1], 14); + FF(cc, dd, aa, bb, X[ 2], 15); + FF(bb, cc, dd, aa, X[ 3], 12); + FF(aa, bb, cc, dd, X[ 4], 5); + FF(dd, aa, bb, cc, X[ 5], 8); + FF(cc, dd, aa, bb, X[ 6], 7); + FF(bb, cc, dd, aa, X[ 7], 9); + FF(aa, bb, cc, dd, X[ 8], 11); + FF(dd, aa, bb, cc, X[ 9], 13); + FF(cc, dd, aa, bb, X[10], 14); + FF(bb, cc, dd, aa, X[11], 15); + FF(aa, bb, cc, dd, X[12], 6); + FF(dd, aa, bb, cc, X[13], 7); + FF(cc, dd, aa, bb, X[14], 9); + FF(bb, cc, dd, aa, X[15], 8); + + /* round 2 */ + GG(aa, bb, cc, dd, X[ 7], 7); + GG(dd, aa, bb, cc, X[ 4], 6); + GG(cc, dd, aa, bb, X[13], 8); + GG(bb, cc, dd, aa, X[ 1], 13); + GG(aa, bb, cc, dd, X[10], 11); + GG(dd, aa, bb, cc, X[ 6], 9); + GG(cc, dd, aa, bb, X[15], 7); + GG(bb, cc, dd, aa, X[ 3], 15); + GG(aa, bb, cc, dd, X[12], 7); + GG(dd, aa, bb, cc, X[ 0], 12); + GG(cc, dd, aa, bb, X[ 9], 15); + GG(bb, cc, dd, aa, X[ 5], 9); + GG(aa, bb, cc, dd, X[ 2], 11); + GG(dd, aa, bb, cc, X[14], 7); + GG(cc, dd, aa, bb, X[11], 13); + GG(bb, cc, dd, aa, X[ 8], 12); + + /* round 3 */ + HH(aa, bb, cc, dd, X[ 3], 11); + HH(dd, aa, bb, cc, X[10], 13); + HH(cc, dd, aa, bb, X[14], 6); + HH(bb, cc, dd, aa, X[ 4], 7); + HH(aa, bb, cc, dd, X[ 9], 14); + HH(dd, aa, bb, cc, X[15], 9); + HH(cc, dd, aa, bb, X[ 8], 13); + HH(bb, cc, dd, aa, X[ 1], 15); + HH(aa, bb, cc, dd, X[ 2], 14); + HH(dd, aa, bb, cc, X[ 7], 8); + HH(cc, dd, aa, bb, X[ 0], 13); + HH(bb, cc, dd, aa, X[ 6], 6); + HH(aa, bb, cc, dd, X[13], 5); + HH(dd, aa, bb, cc, X[11], 12); + HH(cc, dd, aa, bb, X[ 5], 7); + HH(bb, cc, dd, aa, X[12], 5); + + /* round 4 */ + II(aa, bb, cc, dd, X[ 1], 11); + II(dd, aa, bb, cc, X[ 9], 12); + II(cc, dd, aa, bb, X[11], 14); + II(bb, cc, dd, aa, X[10], 15); + II(aa, bb, cc, dd, X[ 0], 14); + II(dd, aa, bb, cc, X[ 8], 15); + II(cc, dd, aa, bb, X[12], 9); + II(bb, cc, dd, aa, X[ 4], 8); + II(aa, bb, cc, dd, X[13], 9); + II(dd, aa, bb, cc, X[ 3], 14); + II(cc, dd, aa, bb, X[ 7], 5); + II(bb, cc, dd, aa, X[15], 6); + II(aa, bb, cc, dd, X[14], 8); + II(dd, aa, bb, cc, X[ 5], 6); + II(cc, dd, aa, bb, X[ 6], 5); + II(bb, cc, dd, aa, X[ 2], 12); + + /* parallel round 1 */ + III(aaa, bbb, ccc, ddd, X[ 5], 8); + III(ddd, aaa, bbb, ccc, X[14], 9); + III(ccc, ddd, aaa, bbb, X[ 7], 9); + III(bbb, ccc, ddd, aaa, X[ 0], 11); + III(aaa, bbb, ccc, ddd, X[ 9], 13); + III(ddd, aaa, bbb, ccc, X[ 2], 15); + III(ccc, ddd, aaa, bbb, X[11], 15); + III(bbb, ccc, ddd, aaa, X[ 4], 5); + III(aaa, bbb, ccc, ddd, X[13], 7); + III(ddd, aaa, bbb, ccc, X[ 6], 7); + III(ccc, ddd, aaa, bbb, X[15], 8); + III(bbb, ccc, ddd, aaa, X[ 8], 11); + III(aaa, bbb, ccc, ddd, X[ 1], 14); + III(ddd, aaa, bbb, ccc, X[10], 14); + III(ccc, ddd, aaa, bbb, X[ 3], 12); + III(bbb, ccc, ddd, aaa, X[12], 6); + + /* parallel round 2 */ + HHH(aaa, bbb, ccc, ddd, X[ 6], 9); + HHH(ddd, aaa, bbb, ccc, X[11], 13); + HHH(ccc, ddd, aaa, bbb, X[ 3], 15); + HHH(bbb, ccc, ddd, aaa, X[ 7], 7); + HHH(aaa, bbb, ccc, ddd, X[ 0], 12); + HHH(ddd, aaa, bbb, ccc, X[13], 8); + HHH(ccc, ddd, aaa, bbb, X[ 5], 9); + HHH(bbb, ccc, ddd, aaa, X[10], 11); + HHH(aaa, bbb, ccc, ddd, X[14], 7); + HHH(ddd, aaa, bbb, ccc, X[15], 7); + HHH(ccc, ddd, aaa, bbb, X[ 8], 12); + HHH(bbb, ccc, ddd, aaa, X[12], 7); + HHH(aaa, bbb, ccc, ddd, X[ 4], 6); + HHH(ddd, aaa, bbb, ccc, X[ 9], 15); + HHH(ccc, ddd, aaa, bbb, X[ 1], 13); + HHH(bbb, ccc, ddd, aaa, X[ 2], 11); + + /* parallel round 3 */ + GGG(aaa, bbb, ccc, ddd, X[15], 9); + GGG(ddd, aaa, bbb, ccc, X[ 5], 7); + GGG(ccc, ddd, aaa, bbb, X[ 1], 15); + GGG(bbb, ccc, ddd, aaa, X[ 3], 11); + GGG(aaa, bbb, ccc, ddd, X[ 7], 8); + GGG(ddd, aaa, bbb, ccc, X[14], 6); + GGG(ccc, ddd, aaa, bbb, X[ 6], 6); + GGG(bbb, ccc, ddd, aaa, X[ 9], 14); + GGG(aaa, bbb, ccc, ddd, X[11], 12); + GGG(ddd, aaa, bbb, ccc, X[ 8], 13); + GGG(ccc, ddd, aaa, bbb, X[12], 5); + GGG(bbb, ccc, ddd, aaa, X[ 2], 14); + GGG(aaa, bbb, ccc, ddd, X[10], 13); + GGG(ddd, aaa, bbb, ccc, X[ 0], 13); + GGG(ccc, ddd, aaa, bbb, X[ 4], 7); + GGG(bbb, ccc, ddd, aaa, X[13], 5); + + /* parallel round 4 */ + FFF(aaa, bbb, ccc, ddd, X[ 8], 15); + FFF(ddd, aaa, bbb, ccc, X[ 6], 5); + FFF(ccc, ddd, aaa, bbb, X[ 4], 8); + FFF(bbb, ccc, ddd, aaa, X[ 1], 11); + FFF(aaa, bbb, ccc, ddd, X[ 3], 14); + FFF(ddd, aaa, bbb, ccc, X[11], 14); + FFF(ccc, ddd, aaa, bbb, X[15], 6); + FFF(bbb, ccc, ddd, aaa, X[ 0], 14); + FFF(aaa, bbb, ccc, ddd, X[ 5], 6); + FFF(ddd, aaa, bbb, ccc, X[12], 9); + FFF(ccc, ddd, aaa, bbb, X[ 2], 12); + FFF(bbb, ccc, ddd, aaa, X[13], 9); + FFF(aaa, bbb, ccc, ddd, X[ 9], 12); + FFF(ddd, aaa, bbb, ccc, X[ 7], 5); + FFF(ccc, ddd, aaa, bbb, X[10], 15); + FFF(bbb, ccc, ddd, aaa, X[14], 8); + + /* combine results */ + ddd += cc + md->rmd128.state[1]; /* final result for MDbuf[0] */ + md->rmd128.state[1] = md->rmd128.state[2] + dd + aaa; + md->rmd128.state[2] = md->rmd128.state[3] + aa + bbb; + md->rmd128.state[3] = md->rmd128.state[0] + bb + ccc; + md->rmd128.state[0] = ddd; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_rmd128_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_rmd128_compress(md, buf); + burn_stack(sizeof(ulong32) * 24 + sizeof(int)); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int rmd128_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->rmd128.state[0] = 0x67452301UL; + md->rmd128.state[1] = 0xefcdab89UL; + md->rmd128.state[2] = 0x98badcfeUL; + md->rmd128.state[3] = 0x10325476UL; + md->rmd128.curlen = 0; + md->rmd128.length = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(rmd128_process, s_rmd128_compress, rmd128, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (16 bytes) + @return CRYPT_OK if successful +*/ +int rmd128_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->rmd128.curlen >= sizeof(md->rmd128.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->rmd128.length += md->rmd128.curlen * 8; + + /* append the '1' bit */ + md->rmd128.buf[md->rmd128.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->rmd128.curlen > 56) { + while (md->rmd128.curlen < 64) { + md->rmd128.buf[md->rmd128.curlen++] = (unsigned char)0; + } + s_rmd128_compress(md, md->rmd128.buf); + md->rmd128.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->rmd128.curlen < 56) { + md->rmd128.buf[md->rmd128.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->rmd128.length, md->rmd128.buf+56); + s_rmd128_compress(md, md->rmd128.buf); + + /* copy output */ + for (i = 0; i < 4; i++) { + STORE32L(md->rmd128.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int rmd128_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[16]; + } tests[] = { + { "", + { 0xcd, 0xf2, 0x62, 0x13, 0xa1, 0x50, 0xdc, 0x3e, + 0xcb, 0x61, 0x0f, 0x18, 0xf6, 0xb3, 0x8b, 0x46 } + }, + { "a", + { 0x86, 0xbe, 0x7a, 0xfa, 0x33, 0x9d, 0x0f, 0xc7, + 0xcf, 0xc7, 0x85, 0xe7, 0x2f, 0x57, 0x8d, 0x33 } + }, + { "abc", + { 0xc1, 0x4a, 0x12, 0x19, 0x9c, 0x66, 0xe4, 0xba, + 0x84, 0x63, 0x6b, 0x0f, 0x69, 0x14, 0x4c, 0x77 } + }, + { "message digest", + { 0x9e, 0x32, 0x7b, 0x3d, 0x6e, 0x52, 0x30, 0x62, + 0xaf, 0xc1, 0x13, 0x2d, 0x7d, 0xf9, 0xd1, 0xb8 } + }, + { "abcdefghijklmnopqrstuvwxyz", + { 0xfd, 0x2a, 0xa6, 0x07, 0xf7, 0x1d, 0xc8, 0xf5, + 0x10, 0x71, 0x49, 0x22, 0xb3, 0x71, 0x83, 0x4e } + }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + { 0xd1, 0xe9, 0x59, 0xeb, 0x17, 0x9c, 0x91, 0x1f, + 0xae, 0xa4, 0x62, 0x4c, 0x60, 0xc5, 0xc7, 0x02 } + } + }; + + int i; + unsigned char tmp[16]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + rmd128_init(&md); + rmd128_process(&md, (unsigned char *)tests[i].msg, XSTRLEN(tests[i].msg)); + rmd128_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "RIPEMD128", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/rmd160.c b/optee_os/core/lib/libtomcrypt/src/hashes/rmd160.c new file mode 100644 index 0000000..1eba20a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/rmd160.c @@ -0,0 +1,455 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rmd160.c + RMD160 hash function +*/ + +/* Implementation of LTC_RIPEMD-160 based on the source by Antoon Bosselaers, ESAT-COSIC + * + * This source has been radically overhauled to be portable and work within + * the LibTomCrypt API by Tom St Denis + */ + +#ifdef LTC_RIPEMD160 + +const struct ltc_hash_descriptor rmd160_desc = +{ + "rmd160", + 9, + 20, + 64, + + /* OID */ + { 1, 3, 36, 3, 2, 1, }, + 6, + + &rmd160_init, + &rmd160_process, + &rmd160_done, + &rmd160_test, + NULL +}; + +/* the five basic functions F(), G() and H() */ +#define F(x, y, z) ((x) ^ (y) ^ (z)) +#define G(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define H(x, y, z) (((x) | ~(y)) ^ (z)) +#define I(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define J(x, y, z) ((x) ^ ((y) | ~(z))) + +/* the ten basic operations FF() through III() */ +#define FF(a, b, c, d, e, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define GG(a, b, c, d, e, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x5a827999UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define HH(a, b, c, d, e, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x6ed9eba1UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define II(a, b, c, d, e, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x8f1bbcdcUL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define JJ(a, b, c, d, e, x, s) \ + (a) += J((b), (c), (d)) + (x) + 0xa953fd4eUL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define FFF(a, b, c, d, e, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define GGG(a, b, c, d, e, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x7a6d76e9UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define HHH(a, b, c, d, e, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x6d703ef3UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define III(a, b, c, d, e, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x5c4dd124UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define JJJ(a, b, c, d, e, x, s) \ + (a) += J((b), (c), (d)) + (x) + 0x50a28be6UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + + +#ifdef LTC_CLEAN_STACK +static int ss_rmd160_compress(hash_state *md, const unsigned char *buf) +#else +static int s_rmd160_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong32 aa,bb,cc,dd,ee,aaa,bbb,ccc,ddd,eee,X[16]; + int i; + + /* load words X */ + for (i = 0; i < 16; i++){ + LOAD32L(X[i], buf + (4 * i)); + } + + /* load state */ + aa = aaa = md->rmd160.state[0]; + bb = bbb = md->rmd160.state[1]; + cc = ccc = md->rmd160.state[2]; + dd = ddd = md->rmd160.state[3]; + ee = eee = md->rmd160.state[4]; + + /* round 1 */ + FF(aa, bb, cc, dd, ee, X[ 0], 11); + FF(ee, aa, bb, cc, dd, X[ 1], 14); + FF(dd, ee, aa, bb, cc, X[ 2], 15); + FF(cc, dd, ee, aa, bb, X[ 3], 12); + FF(bb, cc, dd, ee, aa, X[ 4], 5); + FF(aa, bb, cc, dd, ee, X[ 5], 8); + FF(ee, aa, bb, cc, dd, X[ 6], 7); + FF(dd, ee, aa, bb, cc, X[ 7], 9); + FF(cc, dd, ee, aa, bb, X[ 8], 11); + FF(bb, cc, dd, ee, aa, X[ 9], 13); + FF(aa, bb, cc, dd, ee, X[10], 14); + FF(ee, aa, bb, cc, dd, X[11], 15); + FF(dd, ee, aa, bb, cc, X[12], 6); + FF(cc, dd, ee, aa, bb, X[13], 7); + FF(bb, cc, dd, ee, aa, X[14], 9); + FF(aa, bb, cc, dd, ee, X[15], 8); + + /* round 2 */ + GG(ee, aa, bb, cc, dd, X[ 7], 7); + GG(dd, ee, aa, bb, cc, X[ 4], 6); + GG(cc, dd, ee, aa, bb, X[13], 8); + GG(bb, cc, dd, ee, aa, X[ 1], 13); + GG(aa, bb, cc, dd, ee, X[10], 11); + GG(ee, aa, bb, cc, dd, X[ 6], 9); + GG(dd, ee, aa, bb, cc, X[15], 7); + GG(cc, dd, ee, aa, bb, X[ 3], 15); + GG(bb, cc, dd, ee, aa, X[12], 7); + GG(aa, bb, cc, dd, ee, X[ 0], 12); + GG(ee, aa, bb, cc, dd, X[ 9], 15); + GG(dd, ee, aa, bb, cc, X[ 5], 9); + GG(cc, dd, ee, aa, bb, X[ 2], 11); + GG(bb, cc, dd, ee, aa, X[14], 7); + GG(aa, bb, cc, dd, ee, X[11], 13); + GG(ee, aa, bb, cc, dd, X[ 8], 12); + + /* round 3 */ + HH(dd, ee, aa, bb, cc, X[ 3], 11); + HH(cc, dd, ee, aa, bb, X[10], 13); + HH(bb, cc, dd, ee, aa, X[14], 6); + HH(aa, bb, cc, dd, ee, X[ 4], 7); + HH(ee, aa, bb, cc, dd, X[ 9], 14); + HH(dd, ee, aa, bb, cc, X[15], 9); + HH(cc, dd, ee, aa, bb, X[ 8], 13); + HH(bb, cc, dd, ee, aa, X[ 1], 15); + HH(aa, bb, cc, dd, ee, X[ 2], 14); + HH(ee, aa, bb, cc, dd, X[ 7], 8); + HH(dd, ee, aa, bb, cc, X[ 0], 13); + HH(cc, dd, ee, aa, bb, X[ 6], 6); + HH(bb, cc, dd, ee, aa, X[13], 5); + HH(aa, bb, cc, dd, ee, X[11], 12); + HH(ee, aa, bb, cc, dd, X[ 5], 7); + HH(dd, ee, aa, bb, cc, X[12], 5); + + /* round 4 */ + II(cc, dd, ee, aa, bb, X[ 1], 11); + II(bb, cc, dd, ee, aa, X[ 9], 12); + II(aa, bb, cc, dd, ee, X[11], 14); + II(ee, aa, bb, cc, dd, X[10], 15); + II(dd, ee, aa, bb, cc, X[ 0], 14); + II(cc, dd, ee, aa, bb, X[ 8], 15); + II(bb, cc, dd, ee, aa, X[12], 9); + II(aa, bb, cc, dd, ee, X[ 4], 8); + II(ee, aa, bb, cc, dd, X[13], 9); + II(dd, ee, aa, bb, cc, X[ 3], 14); + II(cc, dd, ee, aa, bb, X[ 7], 5); + II(bb, cc, dd, ee, aa, X[15], 6); + II(aa, bb, cc, dd, ee, X[14], 8); + II(ee, aa, bb, cc, dd, X[ 5], 6); + II(dd, ee, aa, bb, cc, X[ 6], 5); + II(cc, dd, ee, aa, bb, X[ 2], 12); + + /* round 5 */ + JJ(bb, cc, dd, ee, aa, X[ 4], 9); + JJ(aa, bb, cc, dd, ee, X[ 0], 15); + JJ(ee, aa, bb, cc, dd, X[ 5], 5); + JJ(dd, ee, aa, bb, cc, X[ 9], 11); + JJ(cc, dd, ee, aa, bb, X[ 7], 6); + JJ(bb, cc, dd, ee, aa, X[12], 8); + JJ(aa, bb, cc, dd, ee, X[ 2], 13); + JJ(ee, aa, bb, cc, dd, X[10], 12); + JJ(dd, ee, aa, bb, cc, X[14], 5); + JJ(cc, dd, ee, aa, bb, X[ 1], 12); + JJ(bb, cc, dd, ee, aa, X[ 3], 13); + JJ(aa, bb, cc, dd, ee, X[ 8], 14); + JJ(ee, aa, bb, cc, dd, X[11], 11); + JJ(dd, ee, aa, bb, cc, X[ 6], 8); + JJ(cc, dd, ee, aa, bb, X[15], 5); + JJ(bb, cc, dd, ee, aa, X[13], 6); + + /* parallel round 1 */ + JJJ(aaa, bbb, ccc, ddd, eee, X[ 5], 8); + JJJ(eee, aaa, bbb, ccc, ddd, X[14], 9); + JJJ(ddd, eee, aaa, bbb, ccc, X[ 7], 9); + JJJ(ccc, ddd, eee, aaa, bbb, X[ 0], 11); + JJJ(bbb, ccc, ddd, eee, aaa, X[ 9], 13); + JJJ(aaa, bbb, ccc, ddd, eee, X[ 2], 15); + JJJ(eee, aaa, bbb, ccc, ddd, X[11], 15); + JJJ(ddd, eee, aaa, bbb, ccc, X[ 4], 5); + JJJ(ccc, ddd, eee, aaa, bbb, X[13], 7); + JJJ(bbb, ccc, ddd, eee, aaa, X[ 6], 7); + JJJ(aaa, bbb, ccc, ddd, eee, X[15], 8); + JJJ(eee, aaa, bbb, ccc, ddd, X[ 8], 11); + JJJ(ddd, eee, aaa, bbb, ccc, X[ 1], 14); + JJJ(ccc, ddd, eee, aaa, bbb, X[10], 14); + JJJ(bbb, ccc, ddd, eee, aaa, X[ 3], 12); + JJJ(aaa, bbb, ccc, ddd, eee, X[12], 6); + + /* parallel round 2 */ + III(eee, aaa, bbb, ccc, ddd, X[ 6], 9); + III(ddd, eee, aaa, bbb, ccc, X[11], 13); + III(ccc, ddd, eee, aaa, bbb, X[ 3], 15); + III(bbb, ccc, ddd, eee, aaa, X[ 7], 7); + III(aaa, bbb, ccc, ddd, eee, X[ 0], 12); + III(eee, aaa, bbb, ccc, ddd, X[13], 8); + III(ddd, eee, aaa, bbb, ccc, X[ 5], 9); + III(ccc, ddd, eee, aaa, bbb, X[10], 11); + III(bbb, ccc, ddd, eee, aaa, X[14], 7); + III(aaa, bbb, ccc, ddd, eee, X[15], 7); + III(eee, aaa, bbb, ccc, ddd, X[ 8], 12); + III(ddd, eee, aaa, bbb, ccc, X[12], 7); + III(ccc, ddd, eee, aaa, bbb, X[ 4], 6); + III(bbb, ccc, ddd, eee, aaa, X[ 9], 15); + III(aaa, bbb, ccc, ddd, eee, X[ 1], 13); + III(eee, aaa, bbb, ccc, ddd, X[ 2], 11); + + /* parallel round 3 */ + HHH(ddd, eee, aaa, bbb, ccc, X[15], 9); + HHH(ccc, ddd, eee, aaa, bbb, X[ 5], 7); + HHH(bbb, ccc, ddd, eee, aaa, X[ 1], 15); + HHH(aaa, bbb, ccc, ddd, eee, X[ 3], 11); + HHH(eee, aaa, bbb, ccc, ddd, X[ 7], 8); + HHH(ddd, eee, aaa, bbb, ccc, X[14], 6); + HHH(ccc, ddd, eee, aaa, bbb, X[ 6], 6); + HHH(bbb, ccc, ddd, eee, aaa, X[ 9], 14); + HHH(aaa, bbb, ccc, ddd, eee, X[11], 12); + HHH(eee, aaa, bbb, ccc, ddd, X[ 8], 13); + HHH(ddd, eee, aaa, bbb, ccc, X[12], 5); + HHH(ccc, ddd, eee, aaa, bbb, X[ 2], 14); + HHH(bbb, ccc, ddd, eee, aaa, X[10], 13); + HHH(aaa, bbb, ccc, ddd, eee, X[ 0], 13); + HHH(eee, aaa, bbb, ccc, ddd, X[ 4], 7); + HHH(ddd, eee, aaa, bbb, ccc, X[13], 5); + + /* parallel round 4 */ + GGG(ccc, ddd, eee, aaa, bbb, X[ 8], 15); + GGG(bbb, ccc, ddd, eee, aaa, X[ 6], 5); + GGG(aaa, bbb, ccc, ddd, eee, X[ 4], 8); + GGG(eee, aaa, bbb, ccc, ddd, X[ 1], 11); + GGG(ddd, eee, aaa, bbb, ccc, X[ 3], 14); + GGG(ccc, ddd, eee, aaa, bbb, X[11], 14); + GGG(bbb, ccc, ddd, eee, aaa, X[15], 6); + GGG(aaa, bbb, ccc, ddd, eee, X[ 0], 14); + GGG(eee, aaa, bbb, ccc, ddd, X[ 5], 6); + GGG(ddd, eee, aaa, bbb, ccc, X[12], 9); + GGG(ccc, ddd, eee, aaa, bbb, X[ 2], 12); + GGG(bbb, ccc, ddd, eee, aaa, X[13], 9); + GGG(aaa, bbb, ccc, ddd, eee, X[ 9], 12); + GGG(eee, aaa, bbb, ccc, ddd, X[ 7], 5); + GGG(ddd, eee, aaa, bbb, ccc, X[10], 15); + GGG(ccc, ddd, eee, aaa, bbb, X[14], 8); + + /* parallel round 5 */ + FFF(bbb, ccc, ddd, eee, aaa, X[12] , 8); + FFF(aaa, bbb, ccc, ddd, eee, X[15] , 5); + FFF(eee, aaa, bbb, ccc, ddd, X[10] , 12); + FFF(ddd, eee, aaa, bbb, ccc, X[ 4] , 9); + FFF(ccc, ddd, eee, aaa, bbb, X[ 1] , 12); + FFF(bbb, ccc, ddd, eee, aaa, X[ 5] , 5); + FFF(aaa, bbb, ccc, ddd, eee, X[ 8] , 14); + FFF(eee, aaa, bbb, ccc, ddd, X[ 7] , 6); + FFF(ddd, eee, aaa, bbb, ccc, X[ 6] , 8); + FFF(ccc, ddd, eee, aaa, bbb, X[ 2] , 13); + FFF(bbb, ccc, ddd, eee, aaa, X[13] , 6); + FFF(aaa, bbb, ccc, ddd, eee, X[14] , 5); + FFF(eee, aaa, bbb, ccc, ddd, X[ 0] , 15); + FFF(ddd, eee, aaa, bbb, ccc, X[ 3] , 13); + FFF(ccc, ddd, eee, aaa, bbb, X[ 9] , 11); + FFF(bbb, ccc, ddd, eee, aaa, X[11] , 11); + + /* combine results */ + ddd += cc + md->rmd160.state[1]; /* final result for md->rmd160.state[0] */ + md->rmd160.state[1] = md->rmd160.state[2] + dd + eee; + md->rmd160.state[2] = md->rmd160.state[3] + ee + aaa; + md->rmd160.state[3] = md->rmd160.state[4] + aa + bbb; + md->rmd160.state[4] = md->rmd160.state[0] + bb + ccc; + md->rmd160.state[0] = ddd; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_rmd160_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_rmd160_compress(md, buf); + burn_stack(sizeof(ulong32) * 26 + sizeof(int)); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int rmd160_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->rmd160.state[0] = 0x67452301UL; + md->rmd160.state[1] = 0xefcdab89UL; + md->rmd160.state[2] = 0x98badcfeUL; + md->rmd160.state[3] = 0x10325476UL; + md->rmd160.state[4] = 0xc3d2e1f0UL; + md->rmd160.curlen = 0; + md->rmd160.length = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(rmd160_process, s_rmd160_compress, rmd160, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (20 bytes) + @return CRYPT_OK if successful +*/ +int rmd160_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->rmd160.curlen >= sizeof(md->rmd160.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->rmd160.length += md->rmd160.curlen * 8; + + /* append the '1' bit */ + md->rmd160.buf[md->rmd160.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->rmd160.curlen > 56) { + while (md->rmd160.curlen < 64) { + md->rmd160.buf[md->rmd160.curlen++] = (unsigned char)0; + } + s_rmd160_compress(md, md->rmd160.buf); + md->rmd160.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->rmd160.curlen < 56) { + md->rmd160.buf[md->rmd160.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->rmd160.length, md->rmd160.buf+56); + s_rmd160_compress(md, md->rmd160.buf); + + /* copy output */ + for (i = 0; i < 5; i++) { + STORE32L(md->rmd160.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int rmd160_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[20]; + } tests[] = { + { "", + { 0x9c, 0x11, 0x85, 0xa5, 0xc5, 0xe9, 0xfc, 0x54, 0x61, 0x28, + 0x08, 0x97, 0x7e, 0xe8, 0xf5, 0x48, 0xb2, 0x25, 0x8d, 0x31 } + }, + { "a", + { 0x0b, 0xdc, 0x9d, 0x2d, 0x25, 0x6b, 0x3e, 0xe9, 0xda, 0xae, + 0x34, 0x7b, 0xe6, 0xf4, 0xdc, 0x83, 0x5a, 0x46, 0x7f, 0xfe } + }, + { "abc", + { 0x8e, 0xb2, 0x08, 0xf7, 0xe0, 0x5d, 0x98, 0x7a, 0x9b, 0x04, + 0x4a, 0x8e, 0x98, 0xc6, 0xb0, 0x87, 0xf1, 0x5a, 0x0b, 0xfc } + }, + { "message digest", + { 0x5d, 0x06, 0x89, 0xef, 0x49, 0xd2, 0xfa, 0xe5, 0x72, 0xb8, + 0x81, 0xb1, 0x23, 0xa8, 0x5f, 0xfa, 0x21, 0x59, 0x5f, 0x36 } + }, + { "abcdefghijklmnopqrstuvwxyz", + { 0xf7, 0x1c, 0x27, 0x10, 0x9c, 0x69, 0x2c, 0x1b, 0x56, 0xbb, + 0xdc, 0xeb, 0x5b, 0x9d, 0x28, 0x65, 0xb3, 0x70, 0x8d, 0xbc } + }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + { 0x12, 0xa0, 0x53, 0x38, 0x4a, 0x9c, 0x0c, 0x88, 0xe4, 0x05, + 0xa0, 0x6c, 0x27, 0xdc, 0xf4, 0x9a, 0xda, 0x62, 0xeb, 0x2b } + } + }; + + int i; + unsigned char tmp[20]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + rmd160_init(&md); + rmd160_process(&md, (unsigned char *)tests[i].msg, XSTRLEN(tests[i].msg)); + rmd160_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "RIPEMD160", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/rmd256.c b/optee_os/core/lib/libtomcrypt/src/hashes/rmd256.c new file mode 100644 index 0000000..0097198 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/rmd256.c @@ -0,0 +1,420 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @param rmd256.c + RLTC_MD256 Hash function +*/ + +#ifdef LTC_RIPEMD256 + +const struct ltc_hash_descriptor rmd256_desc = +{ + "rmd256", + 13, + 32, + 64, + + /* OID */ + { 1, 3, 36, 3, 2, 3 }, + 6, + + &rmd256_init, + &rmd256_process, + &rmd256_done, + &rmd256_test, + NULL +}; + +/* the four basic functions F(), G() and H() */ +#define F(x, y, z) ((x) ^ (y) ^ (z)) +#define G(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define H(x, y, z) (((x) | ~(y)) ^ (z)) +#define I(x, y, z) (((x) & (z)) | ((y) & ~(z))) + +/* the eight basic operations FF() through III() */ +#define FF(a, b, c, d, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)); + +#define GG(a, b, c, d, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x5a827999UL;\ + (a) = ROLc((a), (s)); + +#define HH(a, b, c, d, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x6ed9eba1UL;\ + (a) = ROLc((a), (s)); + +#define II(a, b, c, d, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x8f1bbcdcUL;\ + (a) = ROLc((a), (s)); + +#define FFF(a, b, c, d, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)); + +#define GGG(a, b, c, d, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x6d703ef3UL;\ + (a) = ROLc((a), (s)); + +#define HHH(a, b, c, d, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x5c4dd124UL;\ + (a) = ROLc((a), (s)); + +#define III(a, b, c, d, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x50a28be6UL;\ + (a) = ROLc((a), (s)); + +#ifdef LTC_CLEAN_STACK +static int ss_rmd256_compress(hash_state *md, const unsigned char *buf) +#else +static int s_rmd256_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong32 aa,bb,cc,dd,aaa,bbb,ccc,ddd,tmp,X[16]; + int i; + + /* load words X */ + for (i = 0; i < 16; i++){ + LOAD32L(X[i], buf + (4 * i)); + } + + /* load state */ + aa = md->rmd256.state[0]; + bb = md->rmd256.state[1]; + cc = md->rmd256.state[2]; + dd = md->rmd256.state[3]; + aaa = md->rmd256.state[4]; + bbb = md->rmd256.state[5]; + ccc = md->rmd256.state[6]; + ddd = md->rmd256.state[7]; + + /* round 1 */ + FF(aa, bb, cc, dd, X[ 0], 11); + FF(dd, aa, bb, cc, X[ 1], 14); + FF(cc, dd, aa, bb, X[ 2], 15); + FF(bb, cc, dd, aa, X[ 3], 12); + FF(aa, bb, cc, dd, X[ 4], 5); + FF(dd, aa, bb, cc, X[ 5], 8); + FF(cc, dd, aa, bb, X[ 6], 7); + FF(bb, cc, dd, aa, X[ 7], 9); + FF(aa, bb, cc, dd, X[ 8], 11); + FF(dd, aa, bb, cc, X[ 9], 13); + FF(cc, dd, aa, bb, X[10], 14); + FF(bb, cc, dd, aa, X[11], 15); + FF(aa, bb, cc, dd, X[12], 6); + FF(dd, aa, bb, cc, X[13], 7); + FF(cc, dd, aa, bb, X[14], 9); + FF(bb, cc, dd, aa, X[15], 8); + + /* parallel round 1 */ + III(aaa, bbb, ccc, ddd, X[ 5], 8); + III(ddd, aaa, bbb, ccc, X[14], 9); + III(ccc, ddd, aaa, bbb, X[ 7], 9); + III(bbb, ccc, ddd, aaa, X[ 0], 11); + III(aaa, bbb, ccc, ddd, X[ 9], 13); + III(ddd, aaa, bbb, ccc, X[ 2], 15); + III(ccc, ddd, aaa, bbb, X[11], 15); + III(bbb, ccc, ddd, aaa, X[ 4], 5); + III(aaa, bbb, ccc, ddd, X[13], 7); + III(ddd, aaa, bbb, ccc, X[ 6], 7); + III(ccc, ddd, aaa, bbb, X[15], 8); + III(bbb, ccc, ddd, aaa, X[ 8], 11); + III(aaa, bbb, ccc, ddd, X[ 1], 14); + III(ddd, aaa, bbb, ccc, X[10], 14); + III(ccc, ddd, aaa, bbb, X[ 3], 12); + III(bbb, ccc, ddd, aaa, X[12], 6); + + tmp = aa; aa = aaa; aaa = tmp; + + /* round 2 */ + GG(aa, bb, cc, dd, X[ 7], 7); + GG(dd, aa, bb, cc, X[ 4], 6); + GG(cc, dd, aa, bb, X[13], 8); + GG(bb, cc, dd, aa, X[ 1], 13); + GG(aa, bb, cc, dd, X[10], 11); + GG(dd, aa, bb, cc, X[ 6], 9); + GG(cc, dd, aa, bb, X[15], 7); + GG(bb, cc, dd, aa, X[ 3], 15); + GG(aa, bb, cc, dd, X[12], 7); + GG(dd, aa, bb, cc, X[ 0], 12); + GG(cc, dd, aa, bb, X[ 9], 15); + GG(bb, cc, dd, aa, X[ 5], 9); + GG(aa, bb, cc, dd, X[ 2], 11); + GG(dd, aa, bb, cc, X[14], 7); + GG(cc, dd, aa, bb, X[11], 13); + GG(bb, cc, dd, aa, X[ 8], 12); + + /* parallel round 2 */ + HHH(aaa, bbb, ccc, ddd, X[ 6], 9); + HHH(ddd, aaa, bbb, ccc, X[11], 13); + HHH(ccc, ddd, aaa, bbb, X[ 3], 15); + HHH(bbb, ccc, ddd, aaa, X[ 7], 7); + HHH(aaa, bbb, ccc, ddd, X[ 0], 12); + HHH(ddd, aaa, bbb, ccc, X[13], 8); + HHH(ccc, ddd, aaa, bbb, X[ 5], 9); + HHH(bbb, ccc, ddd, aaa, X[10], 11); + HHH(aaa, bbb, ccc, ddd, X[14], 7); + HHH(ddd, aaa, bbb, ccc, X[15], 7); + HHH(ccc, ddd, aaa, bbb, X[ 8], 12); + HHH(bbb, ccc, ddd, aaa, X[12], 7); + HHH(aaa, bbb, ccc, ddd, X[ 4], 6); + HHH(ddd, aaa, bbb, ccc, X[ 9], 15); + HHH(ccc, ddd, aaa, bbb, X[ 1], 13); + HHH(bbb, ccc, ddd, aaa, X[ 2], 11); + + tmp = bb; bb = bbb; bbb = tmp; + + /* round 3 */ + HH(aa, bb, cc, dd, X[ 3], 11); + HH(dd, aa, bb, cc, X[10], 13); + HH(cc, dd, aa, bb, X[14], 6); + HH(bb, cc, dd, aa, X[ 4], 7); + HH(aa, bb, cc, dd, X[ 9], 14); + HH(dd, aa, bb, cc, X[15], 9); + HH(cc, dd, aa, bb, X[ 8], 13); + HH(bb, cc, dd, aa, X[ 1], 15); + HH(aa, bb, cc, dd, X[ 2], 14); + HH(dd, aa, bb, cc, X[ 7], 8); + HH(cc, dd, aa, bb, X[ 0], 13); + HH(bb, cc, dd, aa, X[ 6], 6); + HH(aa, bb, cc, dd, X[13], 5); + HH(dd, aa, bb, cc, X[11], 12); + HH(cc, dd, aa, bb, X[ 5], 7); + HH(bb, cc, dd, aa, X[12], 5); + + /* parallel round 3 */ + GGG(aaa, bbb, ccc, ddd, X[15], 9); + GGG(ddd, aaa, bbb, ccc, X[ 5], 7); + GGG(ccc, ddd, aaa, bbb, X[ 1], 15); + GGG(bbb, ccc, ddd, aaa, X[ 3], 11); + GGG(aaa, bbb, ccc, ddd, X[ 7], 8); + GGG(ddd, aaa, bbb, ccc, X[14], 6); + GGG(ccc, ddd, aaa, bbb, X[ 6], 6); + GGG(bbb, ccc, ddd, aaa, X[ 9], 14); + GGG(aaa, bbb, ccc, ddd, X[11], 12); + GGG(ddd, aaa, bbb, ccc, X[ 8], 13); + GGG(ccc, ddd, aaa, bbb, X[12], 5); + GGG(bbb, ccc, ddd, aaa, X[ 2], 14); + GGG(aaa, bbb, ccc, ddd, X[10], 13); + GGG(ddd, aaa, bbb, ccc, X[ 0], 13); + GGG(ccc, ddd, aaa, bbb, X[ 4], 7); + GGG(bbb, ccc, ddd, aaa, X[13], 5); + + tmp = cc; cc = ccc; ccc = tmp; + + /* round 4 */ + II(aa, bb, cc, dd, X[ 1], 11); + II(dd, aa, bb, cc, X[ 9], 12); + II(cc, dd, aa, bb, X[11], 14); + II(bb, cc, dd, aa, X[10], 15); + II(aa, bb, cc, dd, X[ 0], 14); + II(dd, aa, bb, cc, X[ 8], 15); + II(cc, dd, aa, bb, X[12], 9); + II(bb, cc, dd, aa, X[ 4], 8); + II(aa, bb, cc, dd, X[13], 9); + II(dd, aa, bb, cc, X[ 3], 14); + II(cc, dd, aa, bb, X[ 7], 5); + II(bb, cc, dd, aa, X[15], 6); + II(aa, bb, cc, dd, X[14], 8); + II(dd, aa, bb, cc, X[ 5], 6); + II(cc, dd, aa, bb, X[ 6], 5); + II(bb, cc, dd, aa, X[ 2], 12); + + /* parallel round 4 */ + FFF(aaa, bbb, ccc, ddd, X[ 8], 15); + FFF(ddd, aaa, bbb, ccc, X[ 6], 5); + FFF(ccc, ddd, aaa, bbb, X[ 4], 8); + FFF(bbb, ccc, ddd, aaa, X[ 1], 11); + FFF(aaa, bbb, ccc, ddd, X[ 3], 14); + FFF(ddd, aaa, bbb, ccc, X[11], 14); + FFF(ccc, ddd, aaa, bbb, X[15], 6); + FFF(bbb, ccc, ddd, aaa, X[ 0], 14); + FFF(aaa, bbb, ccc, ddd, X[ 5], 6); + FFF(ddd, aaa, bbb, ccc, X[12], 9); + FFF(ccc, ddd, aaa, bbb, X[ 2], 12); + FFF(bbb, ccc, ddd, aaa, X[13], 9); + FFF(aaa, bbb, ccc, ddd, X[ 9], 12); + FFF(ddd, aaa, bbb, ccc, X[ 7], 5); + FFF(ccc, ddd, aaa, bbb, X[10], 15); + FFF(bbb, ccc, ddd, aaa, X[14], 8); + + tmp = dd; dd = ddd; ddd = tmp; + + /* combine results */ + md->rmd256.state[0] += aa; + md->rmd256.state[1] += bb; + md->rmd256.state[2] += cc; + md->rmd256.state[3] += dd; + md->rmd256.state[4] += aaa; + md->rmd256.state[5] += bbb; + md->rmd256.state[6] += ccc; + md->rmd256.state[7] += ddd; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_rmd256_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_rmd256_compress(md, buf); + burn_stack(sizeof(ulong32) * 25 + sizeof(int)); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int rmd256_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->rmd256.state[0] = 0x67452301UL; + md->rmd256.state[1] = 0xefcdab89UL; + md->rmd256.state[2] = 0x98badcfeUL; + md->rmd256.state[3] = 0x10325476UL; + md->rmd256.state[4] = 0x76543210UL; + md->rmd256.state[5] = 0xfedcba98UL; + md->rmd256.state[6] = 0x89abcdefUL; + md->rmd256.state[7] = 0x01234567UL; + md->rmd256.curlen = 0; + md->rmd256.length = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(rmd256_process, s_rmd256_compress, rmd256, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (16 bytes) + @return CRYPT_OK if successful +*/ +int rmd256_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->rmd256.curlen >= sizeof(md->rmd256.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->rmd256.length += md->rmd256.curlen * 8; + + /* append the '1' bit */ + md->rmd256.buf[md->rmd256.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->rmd256.curlen > 56) { + while (md->rmd256.curlen < 64) { + md->rmd256.buf[md->rmd256.curlen++] = (unsigned char)0; + } + s_rmd256_compress(md, md->rmd256.buf); + md->rmd256.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->rmd256.curlen < 56) { + md->rmd256.buf[md->rmd256.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->rmd256.length, md->rmd256.buf+56); + s_rmd256_compress(md, md->rmd256.buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32L(md->rmd256.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int rmd256_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[32]; + } tests[] = { + { "", + { 0x02, 0xba, 0x4c, 0x4e, 0x5f, 0x8e, 0xcd, 0x18, + 0x77, 0xfc, 0x52, 0xd6, 0x4d, 0x30, 0xe3, 0x7a, + 0x2d, 0x97, 0x74, 0xfb, 0x1e, 0x5d, 0x02, 0x63, + 0x80, 0xae, 0x01, 0x68, 0xe3, 0xc5, 0x52, 0x2d } + }, + { "a", + { 0xf9, 0x33, 0x3e, 0x45, 0xd8, 0x57, 0xf5, 0xd9, + 0x0a, 0x91, 0xba, 0xb7, 0x0a, 0x1e, 0xba, 0x0c, + 0xfb, 0x1b, 0xe4, 0xb0, 0x78, 0x3c, 0x9a, 0xcf, + 0xcd, 0x88, 0x3a, 0x91, 0x34, 0x69, 0x29, 0x25 } + }, + { "abc", + { 0xaf, 0xbd, 0x6e, 0x22, 0x8b, 0x9d, 0x8c, 0xbb, + 0xce, 0xf5, 0xca, 0x2d, 0x03, 0xe6, 0xdb, 0xa1, + 0x0a, 0xc0, 0xbc, 0x7d, 0xcb, 0xe4, 0x68, 0x0e, + 0x1e, 0x42, 0xd2, 0xe9, 0x75, 0x45, 0x9b, 0x65 } + }, + { "message digest", + { 0x87, 0xe9, 0x71, 0x75, 0x9a, 0x1c, 0xe4, 0x7a, + 0x51, 0x4d, 0x5c, 0x91, 0x4c, 0x39, 0x2c, 0x90, + 0x18, 0xc7, 0xc4, 0x6b, 0xc1, 0x44, 0x65, 0x55, + 0x4a, 0xfc, 0xdf, 0x54, 0xa5, 0x07, 0x0c, 0x0e } + }, + { "abcdefghijklmnopqrstuvwxyz", + { 0x64, 0x9d, 0x30, 0x34, 0x75, 0x1e, 0xa2, 0x16, + 0x77, 0x6b, 0xf9, 0xa1, 0x8a, 0xcc, 0x81, 0xbc, + 0x78, 0x96, 0x11, 0x8a, 0x51, 0x97, 0x96, 0x87, + 0x82, 0xdd, 0x1f, 0xd9, 0x7d, 0x8d, 0x51, 0x33 } + }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + { 0x57, 0x40, 0xa4, 0x08, 0xac, 0x16, 0xb7, 0x20, + 0xb8, 0x44, 0x24, 0xae, 0x93, 0x1c, 0xbb, 0x1f, + 0xe3, 0x63, 0xd1, 0xd0, 0xbf, 0x40, 0x17, 0xf1, + 0xa8, 0x9f, 0x7e, 0xa6, 0xde, 0x77, 0xa0, 0xb8 } + } + }; + + int i; + unsigned char tmp[32]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + rmd256_init(&md); + rmd256_process(&md, (unsigned char *)tests[i].msg, XSTRLEN(tests[i].msg)); + rmd256_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "RIPEMD256", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/rmd320.c b/optee_os/core/lib/libtomcrypt/src/hashes/rmd320.c new file mode 100644 index 0000000..0021d67 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/rmd320.c @@ -0,0 +1,485 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rmd320.c + RMD320 hash function +*/ + +#ifdef LTC_RIPEMD320 + +const struct ltc_hash_descriptor rmd320_desc = +{ + "rmd320", + 14, + 40, + 64, + + /* OID ... does not exist + * http://oid-info.com/get/1.3.36.3.2 */ + { 0 }, + 0, + + &rmd320_init, + &rmd320_process, + &rmd320_done, + &rmd320_test, + NULL +}; + +/* the five basic functions F(), G() and H() */ +#define F(x, y, z) ((x) ^ (y) ^ (z)) +#define G(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define H(x, y, z) (((x) | ~(y)) ^ (z)) +#define I(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define J(x, y, z) ((x) ^ ((y) | ~(z))) + +/* the ten basic operations FF() through III() */ +#define FF(a, b, c, d, e, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define GG(a, b, c, d, e, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x5a827999UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define HH(a, b, c, d, e, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x6ed9eba1UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define II(a, b, c, d, e, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x8f1bbcdcUL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define JJ(a, b, c, d, e, x, s) \ + (a) += J((b), (c), (d)) + (x) + 0xa953fd4eUL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define FFF(a, b, c, d, e, x, s) \ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define GGG(a, b, c, d, e, x, s) \ + (a) += G((b), (c), (d)) + (x) + 0x7a6d76e9UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define HHH(a, b, c, d, e, x, s) \ + (a) += H((b), (c), (d)) + (x) + 0x6d703ef3UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define III(a, b, c, d, e, x, s) \ + (a) += I((b), (c), (d)) + (x) + 0x5c4dd124UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + +#define JJJ(a, b, c, d, e, x, s) \ + (a) += J((b), (c), (d)) + (x) + 0x50a28be6UL;\ + (a) = ROLc((a), (s)) + (e);\ + (c) = ROLc((c), 10); + + +#ifdef LTC_CLEAN_STACK +static int ss_rmd320_compress(hash_state *md, const unsigned char *buf) +#else +static int s_rmd320_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong32 aa,bb,cc,dd,ee,aaa,bbb,ccc,ddd,eee,tmp,X[16]; + int i; + + /* load words X */ + for (i = 0; i < 16; i++){ + LOAD32L(X[i], buf + (4 * i)); + } + + /* load state */ + aa = md->rmd320.state[0]; + bb = md->rmd320.state[1]; + cc = md->rmd320.state[2]; + dd = md->rmd320.state[3]; + ee = md->rmd320.state[4]; + aaa = md->rmd320.state[5]; + bbb = md->rmd320.state[6]; + ccc = md->rmd320.state[7]; + ddd = md->rmd320.state[8]; + eee = md->rmd320.state[9]; + + /* round 1 */ + FF(aa, bb, cc, dd, ee, X[ 0], 11); + FF(ee, aa, bb, cc, dd, X[ 1], 14); + FF(dd, ee, aa, bb, cc, X[ 2], 15); + FF(cc, dd, ee, aa, bb, X[ 3], 12); + FF(bb, cc, dd, ee, aa, X[ 4], 5); + FF(aa, bb, cc, dd, ee, X[ 5], 8); + FF(ee, aa, bb, cc, dd, X[ 6], 7); + FF(dd, ee, aa, bb, cc, X[ 7], 9); + FF(cc, dd, ee, aa, bb, X[ 8], 11); + FF(bb, cc, dd, ee, aa, X[ 9], 13); + FF(aa, bb, cc, dd, ee, X[10], 14); + FF(ee, aa, bb, cc, dd, X[11], 15); + FF(dd, ee, aa, bb, cc, X[12], 6); + FF(cc, dd, ee, aa, bb, X[13], 7); + FF(bb, cc, dd, ee, aa, X[14], 9); + FF(aa, bb, cc, dd, ee, X[15], 8); + + /* parallel round 1 */ + JJJ(aaa, bbb, ccc, ddd, eee, X[ 5], 8); + JJJ(eee, aaa, bbb, ccc, ddd, X[14], 9); + JJJ(ddd, eee, aaa, bbb, ccc, X[ 7], 9); + JJJ(ccc, ddd, eee, aaa, bbb, X[ 0], 11); + JJJ(bbb, ccc, ddd, eee, aaa, X[ 9], 13); + JJJ(aaa, bbb, ccc, ddd, eee, X[ 2], 15); + JJJ(eee, aaa, bbb, ccc, ddd, X[11], 15); + JJJ(ddd, eee, aaa, bbb, ccc, X[ 4], 5); + JJJ(ccc, ddd, eee, aaa, bbb, X[13], 7); + JJJ(bbb, ccc, ddd, eee, aaa, X[ 6], 7); + JJJ(aaa, bbb, ccc, ddd, eee, X[15], 8); + JJJ(eee, aaa, bbb, ccc, ddd, X[ 8], 11); + JJJ(ddd, eee, aaa, bbb, ccc, X[ 1], 14); + JJJ(ccc, ddd, eee, aaa, bbb, X[10], 14); + JJJ(bbb, ccc, ddd, eee, aaa, X[ 3], 12); + JJJ(aaa, bbb, ccc, ddd, eee, X[12], 6); + + tmp = aa; aa = aaa; aaa = tmp; + + /* round 2 */ + GG(ee, aa, bb, cc, dd, X[ 7], 7); + GG(dd, ee, aa, bb, cc, X[ 4], 6); + GG(cc, dd, ee, aa, bb, X[13], 8); + GG(bb, cc, dd, ee, aa, X[ 1], 13); + GG(aa, bb, cc, dd, ee, X[10], 11); + GG(ee, aa, bb, cc, dd, X[ 6], 9); + GG(dd, ee, aa, bb, cc, X[15], 7); + GG(cc, dd, ee, aa, bb, X[ 3], 15); + GG(bb, cc, dd, ee, aa, X[12], 7); + GG(aa, bb, cc, dd, ee, X[ 0], 12); + GG(ee, aa, bb, cc, dd, X[ 9], 15); + GG(dd, ee, aa, bb, cc, X[ 5], 9); + GG(cc, dd, ee, aa, bb, X[ 2], 11); + GG(bb, cc, dd, ee, aa, X[14], 7); + GG(aa, bb, cc, dd, ee, X[11], 13); + GG(ee, aa, bb, cc, dd, X[ 8], 12); + + /* parallel round 2 */ + III(eee, aaa, bbb, ccc, ddd, X[ 6], 9); + III(ddd, eee, aaa, bbb, ccc, X[11], 13); + III(ccc, ddd, eee, aaa, bbb, X[ 3], 15); + III(bbb, ccc, ddd, eee, aaa, X[ 7], 7); + III(aaa, bbb, ccc, ddd, eee, X[ 0], 12); + III(eee, aaa, bbb, ccc, ddd, X[13], 8); + III(ddd, eee, aaa, bbb, ccc, X[ 5], 9); + III(ccc, ddd, eee, aaa, bbb, X[10], 11); + III(bbb, ccc, ddd, eee, aaa, X[14], 7); + III(aaa, bbb, ccc, ddd, eee, X[15], 7); + III(eee, aaa, bbb, ccc, ddd, X[ 8], 12); + III(ddd, eee, aaa, bbb, ccc, X[12], 7); + III(ccc, ddd, eee, aaa, bbb, X[ 4], 6); + III(bbb, ccc, ddd, eee, aaa, X[ 9], 15); + III(aaa, bbb, ccc, ddd, eee, X[ 1], 13); + III(eee, aaa, bbb, ccc, ddd, X[ 2], 11); + + tmp = bb; bb = bbb; bbb = tmp; + + /* round 3 */ + HH(dd, ee, aa, bb, cc, X[ 3], 11); + HH(cc, dd, ee, aa, bb, X[10], 13); + HH(bb, cc, dd, ee, aa, X[14], 6); + HH(aa, bb, cc, dd, ee, X[ 4], 7); + HH(ee, aa, bb, cc, dd, X[ 9], 14); + HH(dd, ee, aa, bb, cc, X[15], 9); + HH(cc, dd, ee, aa, bb, X[ 8], 13); + HH(bb, cc, dd, ee, aa, X[ 1], 15); + HH(aa, bb, cc, dd, ee, X[ 2], 14); + HH(ee, aa, bb, cc, dd, X[ 7], 8); + HH(dd, ee, aa, bb, cc, X[ 0], 13); + HH(cc, dd, ee, aa, bb, X[ 6], 6); + HH(bb, cc, dd, ee, aa, X[13], 5); + HH(aa, bb, cc, dd, ee, X[11], 12); + HH(ee, aa, bb, cc, dd, X[ 5], 7); + HH(dd, ee, aa, bb, cc, X[12], 5); + + /* parallel round 3 */ + HHH(ddd, eee, aaa, bbb, ccc, X[15], 9); + HHH(ccc, ddd, eee, aaa, bbb, X[ 5], 7); + HHH(bbb, ccc, ddd, eee, aaa, X[ 1], 15); + HHH(aaa, bbb, ccc, ddd, eee, X[ 3], 11); + HHH(eee, aaa, bbb, ccc, ddd, X[ 7], 8); + HHH(ddd, eee, aaa, bbb, ccc, X[14], 6); + HHH(ccc, ddd, eee, aaa, bbb, X[ 6], 6); + HHH(bbb, ccc, ddd, eee, aaa, X[ 9], 14); + HHH(aaa, bbb, ccc, ddd, eee, X[11], 12); + HHH(eee, aaa, bbb, ccc, ddd, X[ 8], 13); + HHH(ddd, eee, aaa, bbb, ccc, X[12], 5); + HHH(ccc, ddd, eee, aaa, bbb, X[ 2], 14); + HHH(bbb, ccc, ddd, eee, aaa, X[10], 13); + HHH(aaa, bbb, ccc, ddd, eee, X[ 0], 13); + HHH(eee, aaa, bbb, ccc, ddd, X[ 4], 7); + HHH(ddd, eee, aaa, bbb, ccc, X[13], 5); + + tmp = cc; cc = ccc; ccc = tmp; + + /* round 4 */ + II(cc, dd, ee, aa, bb, X[ 1], 11); + II(bb, cc, dd, ee, aa, X[ 9], 12); + II(aa, bb, cc, dd, ee, X[11], 14); + II(ee, aa, bb, cc, dd, X[10], 15); + II(dd, ee, aa, bb, cc, X[ 0], 14); + II(cc, dd, ee, aa, bb, X[ 8], 15); + II(bb, cc, dd, ee, aa, X[12], 9); + II(aa, bb, cc, dd, ee, X[ 4], 8); + II(ee, aa, bb, cc, dd, X[13], 9); + II(dd, ee, aa, bb, cc, X[ 3], 14); + II(cc, dd, ee, aa, bb, X[ 7], 5); + II(bb, cc, dd, ee, aa, X[15], 6); + II(aa, bb, cc, dd, ee, X[14], 8); + II(ee, aa, bb, cc, dd, X[ 5], 6); + II(dd, ee, aa, bb, cc, X[ 6], 5); + II(cc, dd, ee, aa, bb, X[ 2], 12); + + /* parallel round 4 */ + GGG(ccc, ddd, eee, aaa, bbb, X[ 8], 15); + GGG(bbb, ccc, ddd, eee, aaa, X[ 6], 5); + GGG(aaa, bbb, ccc, ddd, eee, X[ 4], 8); + GGG(eee, aaa, bbb, ccc, ddd, X[ 1], 11); + GGG(ddd, eee, aaa, bbb, ccc, X[ 3], 14); + GGG(ccc, ddd, eee, aaa, bbb, X[11], 14); + GGG(bbb, ccc, ddd, eee, aaa, X[15], 6); + GGG(aaa, bbb, ccc, ddd, eee, X[ 0], 14); + GGG(eee, aaa, bbb, ccc, ddd, X[ 5], 6); + GGG(ddd, eee, aaa, bbb, ccc, X[12], 9); + GGG(ccc, ddd, eee, aaa, bbb, X[ 2], 12); + GGG(bbb, ccc, ddd, eee, aaa, X[13], 9); + GGG(aaa, bbb, ccc, ddd, eee, X[ 9], 12); + GGG(eee, aaa, bbb, ccc, ddd, X[ 7], 5); + GGG(ddd, eee, aaa, bbb, ccc, X[10], 15); + GGG(ccc, ddd, eee, aaa, bbb, X[14], 8); + + tmp = dd; dd = ddd; ddd = tmp; + + /* round 5 */ + JJ(bb, cc, dd, ee, aa, X[ 4], 9); + JJ(aa, bb, cc, dd, ee, X[ 0], 15); + JJ(ee, aa, bb, cc, dd, X[ 5], 5); + JJ(dd, ee, aa, bb, cc, X[ 9], 11); + JJ(cc, dd, ee, aa, bb, X[ 7], 6); + JJ(bb, cc, dd, ee, aa, X[12], 8); + JJ(aa, bb, cc, dd, ee, X[ 2], 13); + JJ(ee, aa, bb, cc, dd, X[10], 12); + JJ(dd, ee, aa, bb, cc, X[14], 5); + JJ(cc, dd, ee, aa, bb, X[ 1], 12); + JJ(bb, cc, dd, ee, aa, X[ 3], 13); + JJ(aa, bb, cc, dd, ee, X[ 8], 14); + JJ(ee, aa, bb, cc, dd, X[11], 11); + JJ(dd, ee, aa, bb, cc, X[ 6], 8); + JJ(cc, dd, ee, aa, bb, X[15], 5); + JJ(bb, cc, dd, ee, aa, X[13], 6); + + /* parallel round 5 */ + FFF(bbb, ccc, ddd, eee, aaa, X[12] , 8); + FFF(aaa, bbb, ccc, ddd, eee, X[15] , 5); + FFF(eee, aaa, bbb, ccc, ddd, X[10] , 12); + FFF(ddd, eee, aaa, bbb, ccc, X[ 4] , 9); + FFF(ccc, ddd, eee, aaa, bbb, X[ 1] , 12); + FFF(bbb, ccc, ddd, eee, aaa, X[ 5] , 5); + FFF(aaa, bbb, ccc, ddd, eee, X[ 8] , 14); + FFF(eee, aaa, bbb, ccc, ddd, X[ 7] , 6); + FFF(ddd, eee, aaa, bbb, ccc, X[ 6] , 8); + FFF(ccc, ddd, eee, aaa, bbb, X[ 2] , 13); + FFF(bbb, ccc, ddd, eee, aaa, X[13] , 6); + FFF(aaa, bbb, ccc, ddd, eee, X[14] , 5); + FFF(eee, aaa, bbb, ccc, ddd, X[ 0] , 15); + FFF(ddd, eee, aaa, bbb, ccc, X[ 3] , 13); + FFF(ccc, ddd, eee, aaa, bbb, X[ 9] , 11); + FFF(bbb, ccc, ddd, eee, aaa, X[11] , 11); + + tmp = ee; ee = eee; eee = tmp; + + /* combine results */ + md->rmd320.state[0] += aa; + md->rmd320.state[1] += bb; + md->rmd320.state[2] += cc; + md->rmd320.state[3] += dd; + md->rmd320.state[4] += ee; + md->rmd320.state[5] += aaa; + md->rmd320.state[6] += bbb; + md->rmd320.state[7] += ccc; + md->rmd320.state[8] += ddd; + md->rmd320.state[9] += eee; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_rmd320_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_rmd320_compress(md, buf); + burn_stack(sizeof(ulong32) * 27 + sizeof(int)); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int rmd320_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->rmd320.state[0] = 0x67452301UL; + md->rmd320.state[1] = 0xefcdab89UL; + md->rmd320.state[2] = 0x98badcfeUL; + md->rmd320.state[3] = 0x10325476UL; + md->rmd320.state[4] = 0xc3d2e1f0UL; + md->rmd320.state[5] = 0x76543210UL; + md->rmd320.state[6] = 0xfedcba98UL; + md->rmd320.state[7] = 0x89abcdefUL; + md->rmd320.state[8] = 0x01234567UL; + md->rmd320.state[9] = 0x3c2d1e0fUL; + md->rmd320.curlen = 0; + md->rmd320.length = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(rmd320_process, s_rmd320_compress, rmd320, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (20 bytes) + @return CRYPT_OK if successful +*/ +int rmd320_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->rmd320.curlen >= sizeof(md->rmd320.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->rmd320.length += md->rmd320.curlen * 8; + + /* append the '1' bit */ + md->rmd320.buf[md->rmd320.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->rmd320.curlen > 56) { + while (md->rmd320.curlen < 64) { + md->rmd320.buf[md->rmd320.curlen++] = (unsigned char)0; + } + s_rmd320_compress(md, md->rmd320.buf); + md->rmd320.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->rmd320.curlen < 56) { + md->rmd320.buf[md->rmd320.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->rmd320.length, md->rmd320.buf+56); + s_rmd320_compress(md, md->rmd320.buf); + + /* copy output */ + for (i = 0; i < 10; i++) { + STORE32L(md->rmd320.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int rmd320_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + const char *msg; + unsigned char hash[40]; + } tests[] = { + { "", + { 0x22, 0xd6, 0x5d, 0x56, 0x61, 0x53, 0x6c, 0xdc, 0x75, 0xc1, + 0xfd, 0xf5, 0xc6, 0xde, 0x7b, 0x41, 0xb9, 0xf2, 0x73, 0x25, + 0xeb, 0xc6, 0x1e, 0x85, 0x57, 0x17, 0x7d, 0x70, 0x5a, 0x0e, + 0xc8, 0x80, 0x15, 0x1c, 0x3a, 0x32, 0xa0, 0x08, 0x99, 0xb8 } + }, + { "a", + { 0xce, 0x78, 0x85, 0x06, 0x38, 0xf9, 0x26, 0x58, 0xa5, 0xa5, + 0x85, 0x09, 0x75, 0x79, 0x92, 0x6d, 0xda, 0x66, 0x7a, 0x57, + 0x16, 0x56, 0x2c, 0xfc, 0xf6, 0xfb, 0xe7, 0x7f, 0x63, 0x54, + 0x2f, 0x99, 0xb0, 0x47, 0x05, 0xd6, 0x97, 0x0d, 0xff, 0x5d } + }, + { "abc", + { 0xde, 0x4c, 0x01, 0xb3, 0x05, 0x4f, 0x89, 0x30, 0xa7, 0x9d, + 0x09, 0xae, 0x73, 0x8e, 0x92, 0x30, 0x1e, 0x5a, 0x17, 0x08, + 0x5b, 0xef, 0xfd, 0xc1, 0xb8, 0xd1, 0x16, 0x71, 0x3e, 0x74, + 0xf8, 0x2f, 0xa9, 0x42, 0xd6, 0x4c, 0xdb, 0xc4, 0x68, 0x2d } + }, + { "message digest", + { 0x3a, 0x8e, 0x28, 0x50, 0x2e, 0xd4, 0x5d, 0x42, 0x2f, 0x68, + 0x84, 0x4f, 0x9d, 0xd3, 0x16, 0xe7, 0xb9, 0x85, 0x33, 0xfa, + 0x3f, 0x2a, 0x91, 0xd2, 0x9f, 0x84, 0xd4, 0x25, 0xc8, 0x8d, + 0x6b, 0x4e, 0xff, 0x72, 0x7d, 0xf6, 0x6a, 0x7c, 0x01, 0x97 } + }, + { "abcdefghijklmnopqrstuvwxyz", + { 0xca, 0xbd, 0xb1, 0x81, 0x0b, 0x92, 0x47, 0x0a, 0x20, 0x93, + 0xaa, 0x6b, 0xce, 0x05, 0x95, 0x2c, 0x28, 0x34, 0x8c, 0xf4, + 0x3f, 0xf6, 0x08, 0x41, 0x97, 0x51, 0x66, 0xbb, 0x40, 0xed, + 0x23, 0x40, 0x04, 0xb8, 0x82, 0x44, 0x63, 0xe6, 0xb0, 0x09 } + }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + { 0xd0, 0x34, 0xa7, 0x95, 0x0c, 0xf7, 0x22, 0x02, 0x1b, 0xa4, + 0xb8, 0x4d, 0xf7, 0x69, 0xa5, 0xde, 0x20, 0x60, 0xe2, 0x59, + 0xdf, 0x4c, 0x9b, 0xb4, 0xa4, 0x26, 0x8c, 0x0e, 0x93, 0x5b, + 0xbc, 0x74, 0x70, 0xa9, 0x69, 0xc9, 0xd0, 0x72, 0xa1, 0xac } + } + }; + + int i; + unsigned char tmp[40]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + rmd320_init(&md); + rmd320_process(&md, (unsigned char *)tests[i].msg, XSTRLEN(tests[i].msg)); + rmd320_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "RIPEMD320", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha1.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha1.c new file mode 100644 index 0000000..40709db --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha1.c @@ -0,0 +1,276 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file sha1.c + LTC_SHA1 code by Tom St Denis +*/ + + +#ifdef LTC_SHA1 + +const struct ltc_hash_descriptor sha1_desc = +{ + "sha1", + 2, + 20, + 64, + + /* OID */ + { 1, 3, 14, 3, 2, 26, }, + 6, + + &sha1_init, + &sha1_process, + &sha1_done, + &sha1_test, + NULL +}; + +#define F0(x,y,z) (z ^ (x & (y ^ z))) +#define F1(x,y,z) (x ^ y ^ z) +#define F2(x,y,z) ((x & y) | (z & (x | y))) +#define F3(x,y,z) (x ^ y ^ z) + +#ifdef LTC_CLEAN_STACK +static int ss_sha1_compress(hash_state *md, const unsigned char *buf) +#else +static int s_sha1_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong32 a,b,c,d,e,W[80],i; +#ifdef LTC_SMALL_CODE + ulong32 t; +#endif + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* copy state */ + a = md->sha1.state[0]; + b = md->sha1.state[1]; + c = md->sha1.state[2]; + d = md->sha1.state[3]; + e = md->sha1.state[4]; + + /* expand it */ + for (i = 16; i < 80; i++) { + W[i] = ROL(W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1); + } + + /* compress */ + /* round one */ + #define FF0(a,b,c,d,e,i) e = (ROLc(a, 5) + F0(b,c,d) + e + W[i] + 0x5a827999UL); b = ROLc(b, 30); + #define FF1(a,b,c,d,e,i) e = (ROLc(a, 5) + F1(b,c,d) + e + W[i] + 0x6ed9eba1UL); b = ROLc(b, 30); + #define FF2(a,b,c,d,e,i) e = (ROLc(a, 5) + F2(b,c,d) + e + W[i] + 0x8f1bbcdcUL); b = ROLc(b, 30); + #define FF3(a,b,c,d,e,i) e = (ROLc(a, 5) + F3(b,c,d) + e + W[i] + 0xca62c1d6UL); b = ROLc(b, 30); + +#ifdef LTC_SMALL_CODE + + for (i = 0; i < 20; ) { + FF0(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + + for (; i < 40; ) { + FF1(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + + for (; i < 60; ) { + FF2(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + + for (; i < 80; ) { + FF3(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + +#else + + for (i = 0; i < 20; ) { + FF0(a,b,c,d,e,i++); + FF0(e,a,b,c,d,i++); + FF0(d,e,a,b,c,i++); + FF0(c,d,e,a,b,i++); + FF0(b,c,d,e,a,i++); + } + + /* round two */ + for (; i < 40; ) { + FF1(a,b,c,d,e,i++); + FF1(e,a,b,c,d,i++); + FF1(d,e,a,b,c,i++); + FF1(c,d,e,a,b,i++); + FF1(b,c,d,e,a,i++); + } + + /* round three */ + for (; i < 60; ) { + FF2(a,b,c,d,e,i++); + FF2(e,a,b,c,d,i++); + FF2(d,e,a,b,c,i++); + FF2(c,d,e,a,b,i++); + FF2(b,c,d,e,a,i++); + } + + /* round four */ + for (; i < 80; ) { + FF3(a,b,c,d,e,i++); + FF3(e,a,b,c,d,i++); + FF3(d,e,a,b,c,i++); + FF3(c,d,e,a,b,i++); + FF3(b,c,d,e,a,i++); + } +#endif + + #undef FF0 + #undef FF1 + #undef FF2 + #undef FF3 + + /* store */ + md->sha1.state[0] = md->sha1.state[0] + a; + md->sha1.state[1] = md->sha1.state[1] + b; + md->sha1.state[2] = md->sha1.state[2] + c; + md->sha1.state[3] = md->sha1.state[3] + d; + md->sha1.state[4] = md->sha1.state[4] + e; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_sha1_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_sha1_compress(md, buf); + burn_stack(sizeof(ulong32) * 87); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha1_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->sha1.state[0] = 0x67452301UL; + md->sha1.state[1] = 0xefcdab89UL; + md->sha1.state[2] = 0x98badcfeUL; + md->sha1.state[3] = 0x10325476UL; + md->sha1.state[4] = 0xc3d2e1f0UL; + md->sha1.curlen = 0; + md->sha1.length = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(sha1_process, s_sha1_compress, sha1, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (20 bytes) + @return CRYPT_OK if successful +*/ +int sha1_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha1.curlen >= sizeof(md->sha1.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->sha1.length += md->sha1.curlen * 8; + + /* append the '1' bit */ + md->sha1.buf[md->sha1.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->sha1.curlen > 56) { + while (md->sha1.curlen < 64) { + md->sha1.buf[md->sha1.curlen++] = (unsigned char)0; + } + s_sha1_compress(md, md->sha1.buf); + md->sha1.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->sha1.curlen < 56) { + md->sha1.buf[md->sha1.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->sha1.length, md->sha1.buf+56); + s_sha1_compress(md, md->sha1.buf); + + /* copy output */ + for (i = 0; i < 5; i++) { + STORE32H(md->sha1.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha1_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[20]; + } tests[] = { + { "abc", + { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, + 0xba, 0x3e, 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, + 0x9c, 0xd0, 0xd8, 0x9d } + }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + { 0x84, 0x98, 0x3E, 0x44, 0x1C, 0x3B, 0xD2, 0x6E, + 0xBA, 0xAE, 0x4A, 0xA1, 0xF9, 0x51, 0x29, 0xE5, + 0xE5, 0x46, 0x70, 0xF1 } + } + }; + + int i; + unsigned char tmp[20]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha1_init(&md); + sha1_process(&md, (unsigned char*)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha1_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA1", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha224.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha224.c new file mode 100644 index 0000000..1e2f295 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha224.c @@ -0,0 +1,119 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/** + @param sha224.c + LTC_SHA-224 new NIST standard based off of LTC_SHA-256 truncated to 224 bits (Tom St Denis) +*/ + +#include "tomcrypt_private.h" + +#if defined(LTC_SHA224) && defined(LTC_SHA256) + +const struct ltc_hash_descriptor sha224_desc = +{ + "sha224", + 10, + 28, + 64, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 4, }, + 9, + + &sha224_init, + &sha256_process, + &sha224_done, + &sha224_test, + NULL +}; + +/* init the sha256 er... sha224 state ;-) */ +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha224_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->sha256.curlen = 0; + md->sha256.length = 0; + md->sha256.state[0] = 0xc1059ed8UL; + md->sha256.state[1] = 0x367cd507UL; + md->sha256.state[2] = 0x3070dd17UL; + md->sha256.state[3] = 0xf70e5939UL; + md->sha256.state[4] = 0xffc00b31UL; + md->sha256.state[5] = 0x68581511UL; + md->sha256.state[6] = 0x64f98fa7UL; + md->sha256.state[7] = 0xbefa4fa4UL; + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (28 bytes) + @return CRYPT_OK if successful +*/ +int sha224_done(hash_state * md, unsigned char *out) +{ + unsigned char buf[32]; + int err; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + err = sha256_done(md, buf); + XMEMCPY(out, buf, 28); +#ifdef LTC_CLEAN_STACK + zeromem(buf, sizeof(buf)); +#endif + return err; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha224_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[28]; + } tests[] = { + { "abc", + { 0x23, 0x09, 0x7d, 0x22, 0x34, 0x05, 0xd8, + 0x22, 0x86, 0x42, 0xa4, 0x77, 0xbd, 0xa2, + 0x55, 0xb3, 0x2a, 0xad, 0xbc, 0xe4, 0xbd, + 0xa0, 0xb3, 0xf7, 0xe3, 0x6c, 0x9d, 0xa7 } + }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + { 0x75, 0x38, 0x8b, 0x16, 0x51, 0x27, 0x76, + 0xcc, 0x5d, 0xba, 0x5d, 0xa1, 0xfd, 0x89, + 0x01, 0x50, 0xb0, 0xc6, 0x45, 0x5c, 0xb4, + 0xf5, 0x8b, 0x19, 0x52, 0x52, 0x25, 0x25 } + }, + }; + + int i; + unsigned char tmp[28]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha224_init(&md); + sha224_process(&md, (unsigned char*)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha224_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA224", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif /* defined(LTC_SHA224) && defined(LTC_SHA256) */ + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha256.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha256.c new file mode 100644 index 0000000..ff20068 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha256.c @@ -0,0 +1,322 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file sha256.c + LTC_SHA256 by Tom St Denis +*/ + +#ifdef LTC_SHA256 + +const struct ltc_hash_descriptor sha256_desc = +{ + "sha256", + 0, + 32, + 64, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 1, }, + 9, + + &sha256_init, + &sha256_process, + &sha256_done, + &sha256_test, + NULL +}; + +#ifdef LTC_SMALL_CODE +/* the K array */ +static const ulong32 K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; +#endif + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x),(n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +/* compress 512-bits */ +#ifdef LTC_CLEAN_STACK +static int ss_sha256_compress(hash_state * md, const unsigned char *buf) +#else +static int s_sha256_compress(hash_state * md, const unsigned char *buf) +#endif +{ + ulong32 S[8], W[64], t0, t1; +#ifdef LTC_SMALL_CODE + ulong32 t; +#endif + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->sha256.state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#ifdef LTC_SMALL_CODE +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 64; ++i) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i); + t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; + S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; + } +#else +#define RND(a,b,c,d,e,f,g,h,i,ki) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x71374491); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcf); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25b); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b01); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc6); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dc); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c8); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x14292967); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a85); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b2138); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d13); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a7354); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c85); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664b); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd6990624); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e3585); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa070); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c08); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774c); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4a); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc70208); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506ceb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2); +#endif +#undef RND + + /* feedback */ + for (i = 0; i < 8; i++) { + md->sha256.state[i] = md->sha256.state[i] + S[i]; + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_sha256_compress(hash_state * md, const unsigned char *buf) +{ + int err; + err = ss_sha256_compress(md, buf); + burn_stack(sizeof(ulong32) * 74); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha256_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->sha256.curlen = 0; + md->sha256.length = 0; + md->sha256.state[0] = 0x6A09E667UL; + md->sha256.state[1] = 0xBB67AE85UL; + md->sha256.state[2] = 0x3C6EF372UL; + md->sha256.state[3] = 0xA54FF53AUL; + md->sha256.state[4] = 0x510E527FUL; + md->sha256.state[5] = 0x9B05688CUL; + md->sha256.state[6] = 0x1F83D9ABUL; + md->sha256.state[7] = 0x5BE0CD19UL; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(sha256_process,s_sha256_compress, sha256, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +int sha256_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha256.curlen >= sizeof(md->sha256.buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->sha256.length += md->sha256.curlen * 8; + + /* append the '1' bit */ + md->sha256.buf[md->sha256.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->sha256.curlen > 56) { + while (md->sha256.curlen < 64) { + md->sha256.buf[md->sha256.curlen++] = (unsigned char)0; + } + s_sha256_compress(md, md->sha256.buf); + md->sha256.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->sha256.curlen < 56) { + md->sha256.buf[md->sha256.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->sha256.length, md->sha256.buf+56); + s_sha256_compress(md, md->sha256.buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->sha256.state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha256_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[32]; + } tests[] = { + { "abc", + { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, + 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, + 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad } + }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + { 0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, + 0xe5, 0xc0, 0x26, 0x93, 0x0c, 0x3e, 0x60, 0x39, + 0xa3, 0x3c, 0xe4, 0x59, 0x64, 0xff, 0x21, 0x67, + 0xf6, 0xec, 0xed, 0xd4, 0x19, 0xdb, 0x06, 0xc1 } + }, + }; + + int i; + unsigned char tmp[32]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha256_init(&md); + sha256_process(&md, (unsigned char*)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha256_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA256", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha384.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha384.c new file mode 100644 index 0000000..cb3391a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha384.c @@ -0,0 +1,124 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/** + @param sha384.c + LTC_SHA384 hash included in sha512.c, Tom St Denis +*/ + +#include "tomcrypt_private.h" + +#if defined(LTC_SHA384) && defined(LTC_SHA512) + +const struct ltc_hash_descriptor sha384_desc = +{ + "sha384", + 4, + 48, + 128, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 2, }, + 9, + + &sha384_init, + &sha512_process, + &sha384_done, + &sha384_test, + NULL +}; + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha384_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->sha512.curlen = 0; + md->sha512.length = 0; + md->sha512.state[0] = CONST64(0xcbbb9d5dc1059ed8); + md->sha512.state[1] = CONST64(0x629a292a367cd507); + md->sha512.state[2] = CONST64(0x9159015a3070dd17); + md->sha512.state[3] = CONST64(0x152fecd8f70e5939); + md->sha512.state[4] = CONST64(0x67332667ffc00b31); + md->sha512.state[5] = CONST64(0x8eb44a8768581511); + md->sha512.state[6] = CONST64(0xdb0c2e0d64f98fa7); + md->sha512.state[7] = CONST64(0x47b5481dbefa4fa4); + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (48 bytes) + @return CRYPT_OK if successful +*/ +int sha384_done(hash_state * md, unsigned char *out) +{ + unsigned char buf[64]; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha512.curlen >= sizeof(md->sha512.buf)) { + return CRYPT_INVALID_ARG; + } + + sha512_done(md, buf); + XMEMCPY(out, buf, 48); +#ifdef LTC_CLEAN_STACK + zeromem(buf, sizeof(buf)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha384_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[48]; + } tests[] = { + { "abc", + { 0xcb, 0x00, 0x75, 0x3f, 0x45, 0xa3, 0x5e, 0x8b, + 0xb5, 0xa0, 0x3d, 0x69, 0x9a, 0xc6, 0x50, 0x07, + 0x27, 0x2c, 0x32, 0xab, 0x0e, 0xde, 0xd1, 0x63, + 0x1a, 0x8b, 0x60, 0x5a, 0x43, 0xff, 0x5b, 0xed, + 0x80, 0x86, 0x07, 0x2b, 0xa1, 0xe7, 0xcc, 0x23, + 0x58, 0xba, 0xec, 0xa1, 0x34, 0xc8, 0x25, 0xa7 } + }, + { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu", + { 0x09, 0x33, 0x0c, 0x33, 0xf7, 0x11, 0x47, 0xe8, + 0x3d, 0x19, 0x2f, 0xc7, 0x82, 0xcd, 0x1b, 0x47, + 0x53, 0x11, 0x1b, 0x17, 0x3b, 0x3b, 0x05, 0xd2, + 0x2f, 0xa0, 0x80, 0x86, 0xe3, 0xb0, 0xf7, 0x12, + 0xfc, 0xc7, 0xc7, 0x1a, 0x55, 0x7e, 0x2d, 0xb9, + 0x66, 0xc3, 0xe9, 0xfa, 0x91, 0x74, 0x60, 0x39 } + }, + }; + + int i; + unsigned char tmp[48]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha384_init(&md); + sha384_process(&md, (unsigned char*)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha384_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA384", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif /* defined(LTC_SHA384) && defined(LTC_SHA512) */ diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512.c new file mode 100644 index 0000000..ef3a8c8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512.c @@ -0,0 +1,303 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @param sha512.c + LTC_SHA512 by Tom St Denis +*/ + +#ifdef LTC_SHA512 + +const struct ltc_hash_descriptor sha512_desc = +{ + "sha512", + 5, + 64, + 128, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 3, }, + 9, + + &sha512_init, + &sha512_process, + &sha512_done, + &sha512_test, + NULL +}; + +/* the K array */ +static const ulong64 K[80] = { +CONST64(0x428a2f98d728ae22), CONST64(0x7137449123ef65cd), +CONST64(0xb5c0fbcfec4d3b2f), CONST64(0xe9b5dba58189dbbc), +CONST64(0x3956c25bf348b538), CONST64(0x59f111f1b605d019), +CONST64(0x923f82a4af194f9b), CONST64(0xab1c5ed5da6d8118), +CONST64(0xd807aa98a3030242), CONST64(0x12835b0145706fbe), +CONST64(0x243185be4ee4b28c), CONST64(0x550c7dc3d5ffb4e2), +CONST64(0x72be5d74f27b896f), CONST64(0x80deb1fe3b1696b1), +CONST64(0x9bdc06a725c71235), CONST64(0xc19bf174cf692694), +CONST64(0xe49b69c19ef14ad2), CONST64(0xefbe4786384f25e3), +CONST64(0x0fc19dc68b8cd5b5), CONST64(0x240ca1cc77ac9c65), +CONST64(0x2de92c6f592b0275), CONST64(0x4a7484aa6ea6e483), +CONST64(0x5cb0a9dcbd41fbd4), CONST64(0x76f988da831153b5), +CONST64(0x983e5152ee66dfab), CONST64(0xa831c66d2db43210), +CONST64(0xb00327c898fb213f), CONST64(0xbf597fc7beef0ee4), +CONST64(0xc6e00bf33da88fc2), CONST64(0xd5a79147930aa725), +CONST64(0x06ca6351e003826f), CONST64(0x142929670a0e6e70), +CONST64(0x27b70a8546d22ffc), CONST64(0x2e1b21385c26c926), +CONST64(0x4d2c6dfc5ac42aed), CONST64(0x53380d139d95b3df), +CONST64(0x650a73548baf63de), CONST64(0x766a0abb3c77b2a8), +CONST64(0x81c2c92e47edaee6), CONST64(0x92722c851482353b), +CONST64(0xa2bfe8a14cf10364), CONST64(0xa81a664bbc423001), +CONST64(0xc24b8b70d0f89791), CONST64(0xc76c51a30654be30), +CONST64(0xd192e819d6ef5218), CONST64(0xd69906245565a910), +CONST64(0xf40e35855771202a), CONST64(0x106aa07032bbd1b8), +CONST64(0x19a4c116b8d2d0c8), CONST64(0x1e376c085141ab53), +CONST64(0x2748774cdf8eeb99), CONST64(0x34b0bcb5e19b48a8), +CONST64(0x391c0cb3c5c95a63), CONST64(0x4ed8aa4ae3418acb), +CONST64(0x5b9cca4f7763e373), CONST64(0x682e6ff3d6b2b8a3), +CONST64(0x748f82ee5defb2fc), CONST64(0x78a5636f43172f60), +CONST64(0x84c87814a1f0ab72), CONST64(0x8cc702081a6439ec), +CONST64(0x90befffa23631e28), CONST64(0xa4506cebde82bde9), +CONST64(0xbef9a3f7b2c67915), CONST64(0xc67178f2e372532b), +CONST64(0xca273eceea26619c), CONST64(0xd186b8c721c0c207), +CONST64(0xeada7dd6cde0eb1e), CONST64(0xf57d4f7fee6ed178), +CONST64(0x06f067aa72176fba), CONST64(0x0a637dc5a2c898a6), +CONST64(0x113f9804bef90dae), CONST64(0x1b710b35131c471b), +CONST64(0x28db77f523047d84), CONST64(0x32caab7b40c72493), +CONST64(0x3c9ebe0a15c9bebc), CONST64(0x431d67c49c100d4c), +CONST64(0x4cc5d4becb3e42b6), CONST64(0x597f299cfc657e2a), +CONST64(0x5fcb6fab3ad6faec), CONST64(0x6c44198c4a475817) +}; + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) ROR64c(x, n) +#define R(x, n) (((x)&CONST64(0xFFFFFFFFFFFFFFFF))>>((ulong64)n)) +#define Sigma0(x) (S(x, 28) ^ S(x, 34) ^ S(x, 39)) +#define Sigma1(x) (S(x, 14) ^ S(x, 18) ^ S(x, 41)) +#define Gamma0(x) (S(x, 1) ^ S(x, 8) ^ R(x, 7)) +#define Gamma1(x) (S(x, 19) ^ S(x, 61) ^ R(x, 6)) + +/* compress 1024-bits */ +#ifdef LTC_CLEAN_STACK +static int ss_sha512_compress(hash_state * md, const unsigned char *buf) +#else +static int s_sha512_compress(hash_state * md, const unsigned char *buf) +#endif +{ + ulong64 S[8], W[80], t0, t1; + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->sha512.state[i]; + } + + /* copy the state into 1024-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD64H(W[i], buf + (8*i)); + } + + /* fill W[16..79] */ + for (i = 16; i < 80; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#ifdef LTC_SMALL_CODE + for (i = 0; i < 80; i++) { + t0 = S[7] + Sigma1(S[4]) + Ch(S[4], S[5], S[6]) + K[i] + W[i]; + t1 = Sigma0(S[0]) + Maj(S[0], S[1], S[2]); + S[7] = S[6]; + S[6] = S[5]; + S[5] = S[4]; + S[4] = S[3] + t0; + S[3] = S[2]; + S[2] = S[1]; + S[1] = S[0]; + S[0] = t0 + t1; + } +#else +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 80; i += 8) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i+0); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],i+1); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],i+2); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],i+3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],i+4); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],i+5); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],i+6); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],i+7); + } +#endif + + + /* feedback */ + for (i = 0; i < 8; i++) { + md->sha512.state[i] = md->sha512.state[i] + S[i]; + } + + return CRYPT_OK; +} + +/* compress 1024-bits */ +#ifdef LTC_CLEAN_STACK +static int s_sha512_compress(hash_state * md, const unsigned char *buf) +{ + int err; + err = ss_sha512_compress(md, buf); + burn_stack(sizeof(ulong64) * 90 + sizeof(int)); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha512_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + md->sha512.curlen = 0; + md->sha512.length = 0; + md->sha512.state[0] = CONST64(0x6a09e667f3bcc908); + md->sha512.state[1] = CONST64(0xbb67ae8584caa73b); + md->sha512.state[2] = CONST64(0x3c6ef372fe94f82b); + md->sha512.state[3] = CONST64(0xa54ff53a5f1d36f1); + md->sha512.state[4] = CONST64(0x510e527fade682d1); + md->sha512.state[5] = CONST64(0x9b05688c2b3e6c1f); + md->sha512.state[6] = CONST64(0x1f83d9abfb41bd6b); + md->sha512.state[7] = CONST64(0x5be0cd19137e2179); + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(sha512_process, s_sha512_compress, sha512, 128) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (64 bytes) + @return CRYPT_OK if successful +*/ +int sha512_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha512.curlen >= sizeof(md->sha512.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->sha512.length += md->sha512.curlen * CONST64(8); + + /* append the '1' bit */ + md->sha512.buf[md->sha512.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 112 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->sha512.curlen > 112) { + while (md->sha512.curlen < 128) { + md->sha512.buf[md->sha512.curlen++] = (unsigned char)0; + } + s_sha512_compress(md, md->sha512.buf); + md->sha512.curlen = 0; + } + + /* pad upto 120 bytes of zeroes + * note: that from 112 to 120 is the 64 MSB of the length. We assume that you won't hash + * > 2^64 bits of data... :-) + */ + while (md->sha512.curlen < 120) { + md->sha512.buf[md->sha512.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->sha512.length, md->sha512.buf+120); + s_sha512_compress(md, md->sha512.buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE64H(md->sha512.state[i], out+(8*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha512_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[64]; + } tests[] = { + { "abc", + { 0xdd, 0xaf, 0x35, 0xa1, 0x93, 0x61, 0x7a, 0xba, + 0xcc, 0x41, 0x73, 0x49, 0xae, 0x20, 0x41, 0x31, + 0x12, 0xe6, 0xfa, 0x4e, 0x89, 0xa9, 0x7e, 0xa2, + 0x0a, 0x9e, 0xee, 0xe6, 0x4b, 0x55, 0xd3, 0x9a, + 0x21, 0x92, 0x99, 0x2a, 0x27, 0x4f, 0xc1, 0xa8, + 0x36, 0xba, 0x3c, 0x23, 0xa3, 0xfe, 0xeb, 0xbd, + 0x45, 0x4d, 0x44, 0x23, 0x64, 0x3c, 0xe8, 0x0e, + 0x2a, 0x9a, 0xc9, 0x4f, 0xa5, 0x4c, 0xa4, 0x9f } + }, + { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu", + { 0x8e, 0x95, 0x9b, 0x75, 0xda, 0xe3, 0x13, 0xda, + 0x8c, 0xf4, 0xf7, 0x28, 0x14, 0xfc, 0x14, 0x3f, + 0x8f, 0x77, 0x79, 0xc6, 0xeb, 0x9f, 0x7f, 0xa1, + 0x72, 0x99, 0xae, 0xad, 0xb6, 0x88, 0x90, 0x18, + 0x50, 0x1d, 0x28, 0x9e, 0x49, 0x00, 0xf7, 0xe4, + 0x33, 0x1b, 0x99, 0xde, 0xc4, 0xb5, 0x43, 0x3a, + 0xc7, 0xd3, 0x29, 0xee, 0xb6, 0xdd, 0x26, 0x54, + 0x5e, 0x96, 0xe5, 0x5b, 0x87, 0x4b, 0xe9, 0x09 } + }, + }; + + int i; + unsigned char tmp[64]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha512_init(&md); + sha512_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha512_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA512", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif + + + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512_224.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512_224.c new file mode 100644 index 0000000..861852e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512_224.c @@ -0,0 +1,120 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/** + @param sha512_224.c + SHA512/224 hash included in sha512.c +*/ + +#include "tomcrypt_private.h" + +#if defined(LTC_SHA512_224) && defined(LTC_SHA512) + +const struct ltc_hash_descriptor sha512_224_desc = +{ + "sha512-224", + 15, + 28, + 128, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 5, }, + 9, + + &sha512_224_init, + &sha512_process, + &sha512_224_done, + &sha512_224_test, + NULL +}; + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha512_224_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->sha512.curlen = 0; + md->sha512.length = 0; + md->sha512.state[0] = CONST64(0x8C3D37C819544DA2); + md->sha512.state[1] = CONST64(0x73E1996689DCD4D6); + md->sha512.state[2] = CONST64(0x1DFAB7AE32FF9C82); + md->sha512.state[3] = CONST64(0x679DD514582F9FCF); + md->sha512.state[4] = CONST64(0x0F6D2B697BD44DA8); + md->sha512.state[5] = CONST64(0x77E36F7304C48942); + md->sha512.state[6] = CONST64(0x3F9D85A86A1D36C8); + md->sha512.state[7] = CONST64(0x1112E6AD91D692A1); + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (48 bytes) + @return CRYPT_OK if successful +*/ +int sha512_224_done(hash_state * md, unsigned char *out) +{ + unsigned char buf[64]; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha512.curlen >= sizeof(md->sha512.buf)) { + return CRYPT_INVALID_ARG; + } + + sha512_done(md, buf); + XMEMCPY(out, buf, 28); +#ifdef LTC_CLEAN_STACK + zeromem(buf, sizeof(buf)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha512_224_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[28]; + } tests[] = { + { "abc", + { 0x46, 0x34, 0x27, 0x0F, 0x70, 0x7B, 0x6A, 0x54, + 0xDA, 0xAE, 0x75, 0x30, 0x46, 0x08, 0x42, 0xE2, + 0x0E, 0x37, 0xED, 0x26, 0x5C, 0xEE, 0xE9, 0xA4, + 0x3E, 0x89, 0x24, 0xAA } + }, + { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu", + { 0x23, 0xFE, 0xC5, 0xBB, 0x94, 0xD6, 0x0B, 0x23, + 0x30, 0x81, 0x92, 0x64, 0x0B, 0x0C, 0x45, 0x33, + 0x35, 0xD6, 0x64, 0x73, 0x4F, 0xE4, 0x0E, 0x72, + 0x68, 0x67, 0x4A, 0xF9 } + }, + }; + + int i; + unsigned char tmp[28]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha512_224_init(&md); + sha512_224_process(&md, (unsigned char*)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha512_224_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA512-224", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif /* defined(LTC_SHA384) && defined(LTC_SHA512) */ diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512_256.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512_256.c new file mode 100644 index 0000000..1f04154 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sha512_256.c @@ -0,0 +1,120 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +/** + @param sha512_256.c + SHA512/256 hash included in sha512.c +*/ + +#include "tomcrypt_private.h" + +#if defined(LTC_SHA512_256) && defined(LTC_SHA512) + +const struct ltc_hash_descriptor sha512_256_desc = +{ + "sha512-256", + 16, + 32, + 128, + + /* OID */ + { 2, 16, 840, 1, 101, 3, 4, 2, 6, }, + 9, + + &sha512_256_init, + &sha512_process, + &sha512_256_done, + &sha512_256_test, + NULL +}; + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int sha512_256_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->sha512.curlen = 0; + md->sha512.length = 0; + md->sha512.state[0] = CONST64(0x22312194FC2BF72C); + md->sha512.state[1] = CONST64(0x9F555FA3C84C64C2); + md->sha512.state[2] = CONST64(0x2393B86B6F53B151); + md->sha512.state[3] = CONST64(0x963877195940EABD); + md->sha512.state[4] = CONST64(0x96283EE2A88EFFE3); + md->sha512.state[5] = CONST64(0xBE5E1E2553863992); + md->sha512.state[6] = CONST64(0x2B0199FC2C85B8AA); + md->sha512.state[7] = CONST64(0x0EB72DDC81C52CA2); + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (48 bytes) + @return CRYPT_OK if successful +*/ +int sha512_256_done(hash_state * md, unsigned char *out) +{ + unsigned char buf[64]; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->sha512.curlen >= sizeof(md->sha512.buf)) { + return CRYPT_INVALID_ARG; + } + + sha512_done(md, buf); + XMEMCPY(out, buf, 32); +#ifdef LTC_CLEAN_STACK + zeromem(buf, sizeof(buf)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int sha512_256_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[32]; + } tests[] = { + { "abc", + { 0x53, 0x04, 0x8E, 0x26, 0x81, 0x94, 0x1E, 0xF9, + 0x9B, 0x2E, 0x29, 0xB7, 0x6B, 0x4C, 0x7D, 0xAB, + 0xE4, 0xC2, 0xD0, 0xC6, 0x34, 0xFC, 0x6D, 0x46, + 0xE0, 0xE2, 0xF1, 0x31, 0x07, 0xE7, 0xAF, 0x23 } + }, + { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu", + { 0x39, 0x28, 0xE1, 0x84, 0xFB, 0x86, 0x90, 0xF8, + 0x40, 0xDA, 0x39, 0x88, 0x12, 0x1D, 0x31, 0xBE, + 0x65, 0xCB, 0x9D, 0x3E, 0xF8, 0x3E, 0xE6, 0x14, + 0x6F, 0xEA, 0xC8, 0x61, 0xE1, 0x9B, 0x56, 0x3A } + }, + }; + + int i; + unsigned char tmp[32]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + sha512_256_init(&md); + sha512_256_process(&md, (unsigned char*)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + sha512_256_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA512-265", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif /* defined(LTC_SHA384) && defined(LTC_SHA512) */ diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sub.mk b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sub.mk new file mode 100644 index 0000000..cd701ec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha2/sub.mk @@ -0,0 +1,13 @@ +srcs-$(_CFG_CORE_LTC_SHA224_DESC) += sha224.c + +ifneq ($(_CFG_CORE_LTC_SHA256_ACCEL),y) +srcs-$(_CFG_CORE_LTC_SHA256_DESC) += sha256.c +endif + +srcs-$(_CFG_CORE_LTC_SHA384_DESC) += sha384.c + +ifneq ($(_CFG_CORE_LTC_SHA512_ACCEL),y) +srcs-$(_CFG_CORE_LTC_SHA512_DESC) += sha512.c +endif + +srcs-$(_CFG_CORE_LTC_SHA512_256) += sha512_256.c diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha3.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha3.c new file mode 100644 index 0000000..4758e34 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha3.c @@ -0,0 +1,378 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* based on https://github.com/brainhub/SHA3IUF (public domain) */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SHA3 + +const struct ltc_hash_descriptor sha3_224_desc = +{ + "sha3-224", /* name of hash */ + 17, /* internal ID */ + 28, /* Size of digest in octets */ + 144, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,7 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_224_init, + &sha3_process, + &sha3_done, + &sha3_224_test, + NULL +}; + +const struct ltc_hash_descriptor sha3_256_desc = +{ + "sha3-256", /* name of hash */ + 18, /* internal ID */ + 32, /* Size of digest in octets */ + 136, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,8 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_256_init, + &sha3_process, + &sha3_done, + &sha3_256_test, + NULL +}; + +const struct ltc_hash_descriptor sha3_384_desc = +{ + "sha3-384", /* name of hash */ + 19, /* internal ID */ + 48, /* Size of digest in octets */ + 104, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,9 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_384_init, + &sha3_process, + &sha3_done, + &sha3_384_test, + NULL +}; + +const struct ltc_hash_descriptor sha3_512_desc = +{ + "sha3-512", /* name of hash */ + 20, /* internal ID */ + 64, /* Size of digest in octets */ + 72, /* Input block size in octets */ + { 2,16,840,1,101,3,4,2,10 }, /* ASN.1 OID */ + 9, /* Length OID */ + &sha3_512_init, + &sha3_process, + &sha3_done, + &sha3_512_test, + NULL +}; +#endif + +#ifdef LTC_KECCAK +const struct ltc_hash_descriptor keccak_224_desc = +{ + "keccak224", /* name of hash */ + 29, /* internal ID */ + 28, /* Size of digest in octets */ + 144, /* Input block size in octets */ + { 0 }, 0, /* no ASN.1 OID */ + &sha3_224_init, + &sha3_process, + &keccak_done, + &keccak_224_test, + NULL +}; + +const struct ltc_hash_descriptor keccak_256_desc = +{ + "keccak256", /* name of hash */ + 30, /* internal ID */ + 32, /* Size of digest in octets */ + 136, /* Input block size in octets */ + { 0 }, 0, /* no ASN.1 OID */ + &sha3_256_init, + &sha3_process, + &keccak_done, + &keccak_256_test, + NULL +}; + +const struct ltc_hash_descriptor keccak_384_desc = +{ + "keccak384", /* name of hash */ + 31, /* internal ID */ + 48, /* Size of digest in octets */ + 104, /* Input block size in octets */ + { 0 }, 0, /* no ASN.1 OID */ + &sha3_384_init, + &sha3_process, + &keccak_done, + &keccak_384_test, + NULL +}; + +const struct ltc_hash_descriptor keccak_512_desc = +{ + "keccak512", /* name of hash */ + 32, /* internal ID */ + 64, /* Size of digest in octets */ + 72, /* Input block size in octets */ + { 0 }, 0, /* no ASN.1 OID */ + &sha3_512_init, + &sha3_process, + &keccak_done, + &keccak_512_test, + NULL +}; +#endif + +#if defined(LTC_SHA3) || defined(LTC_KECCAK) + +#define SHA3_KECCAK_SPONGE_WORDS 25 /* 1600 bits > 200 bytes > 25 x ulong64 */ +#define SHA3_KECCAK_ROUNDS 24 + +static const ulong64 s_keccakf_rndc[24] = { + CONST64(0x0000000000000001), CONST64(0x0000000000008082), + CONST64(0x800000000000808a), CONST64(0x8000000080008000), + CONST64(0x000000000000808b), CONST64(0x0000000080000001), + CONST64(0x8000000080008081), CONST64(0x8000000000008009), + CONST64(0x000000000000008a), CONST64(0x0000000000000088), + CONST64(0x0000000080008009), CONST64(0x000000008000000a), + CONST64(0x000000008000808b), CONST64(0x800000000000008b), + CONST64(0x8000000000008089), CONST64(0x8000000000008003), + CONST64(0x8000000000008002), CONST64(0x8000000000000080), + CONST64(0x000000000000800a), CONST64(0x800000008000000a), + CONST64(0x8000000080008081), CONST64(0x8000000000008080), + CONST64(0x0000000080000001), CONST64(0x8000000080008008) +}; + +static const unsigned s_keccakf_rotc[24] = { + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 +}; + +static const unsigned s_keccakf_piln[24] = { + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 +}; + +static void s_keccakf(ulong64 s[25]) +{ + int i, j, round; + ulong64 t, bc[5]; + + for(round = 0; round < SHA3_KECCAK_ROUNDS; round++) { + /* Theta */ + for(i = 0; i < 5; i++) { + bc[i] = s[i] ^ s[i + 5] ^ s[i + 10] ^ s[i + 15] ^ s[i + 20]; + } + for(i = 0; i < 5; i++) { + t = bc[(i + 4) % 5] ^ ROL64(bc[(i + 1) % 5], 1); + for(j = 0; j < 25; j += 5) { + s[j + i] ^= t; + } + } + /* Rho Pi */ + t = s[1]; + for(i = 0; i < 24; i++) { + j = s_keccakf_piln[i]; + bc[0] = s[j]; + s[j] = ROL64(t, s_keccakf_rotc[i]); + t = bc[0]; + } + /* Chi */ + for(j = 0; j < 25; j += 5) { + for(i = 0; i < 5; i++) { + bc[i] = s[j + i]; + } + for(i = 0; i < 5; i++) { + s[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; + } + } + /* Iota */ + s[0] ^= s_keccakf_rndc[round]; + } +} + +static LTC_INLINE int ss_done(hash_state *md, unsigned char *hash, ulong64 pad) +{ + unsigned i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(hash != NULL); + + md->sha3.s[md->sha3.word_index] ^= (md->sha3.saved ^ (pad << (md->sha3.byte_index * 8))); + md->sha3.s[SHA3_KECCAK_SPONGE_WORDS - md->sha3.capacity_words - 1] ^= CONST64(0x8000000000000000); + s_keccakf(md->sha3.s); + + /* store sha3.s[] as little-endian bytes into sha3.sb */ + for(i = 0; i < SHA3_KECCAK_SPONGE_WORDS; i++) { + STORE64L(md->sha3.s[i], md->sha3.sb + i * 8); + } + + XMEMCPY(hash, md->sha3.sb, md->sha3.capacity_words * 4); + return CRYPT_OK; +} + +/* Public Inteface */ + +int sha3_224_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 224 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +int sha3_256_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 256 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +int sha3_384_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 384 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +int sha3_512_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = 2 * 512 / (8 * sizeof(ulong64)); + return CRYPT_OK; +} + +#ifdef LTC_SHA3 +int sha3_shake_init(hash_state *md, int num) +{ + LTC_ARGCHK(md != NULL); + if (num != 128 && num != 256) return CRYPT_INVALID_ARG; + XMEMSET(&md->sha3, 0, sizeof(md->sha3)); + md->sha3.capacity_words = (unsigned short)(2 * num / (8 * sizeof(ulong64))); + return CRYPT_OK; +} +#endif + +int sha3_process(hash_state *md, const unsigned char *in, unsigned long inlen) +{ + /* 0...7 -- how much is needed to have a word */ + unsigned old_tail = (8 - md->sha3.byte_index) & 7; + + unsigned long words; + unsigned tail; + unsigned long i; + + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + + if(inlen < old_tail) { /* have no complete word or haven't started the word yet */ + while (inlen--) md->sha3.saved |= (ulong64) (*(in++)) << ((md->sha3.byte_index++) * 8); + return CRYPT_OK; + } + + if(old_tail) { /* will have one word to process */ + inlen -= old_tail; + while (old_tail--) md->sha3.saved |= (ulong64) (*(in++)) << ((md->sha3.byte_index++) * 8); + /* now ready to add saved to the sponge */ + md->sha3.s[md->sha3.word_index] ^= md->sha3.saved; + md->sha3.byte_index = 0; + md->sha3.saved = 0; + if(++md->sha3.word_index == (SHA3_KECCAK_SPONGE_WORDS - md->sha3.capacity_words)) { + s_keccakf(md->sha3.s); + md->sha3.word_index = 0; + } + } + + /* now work in full words directly from input */ + words = inlen / sizeof(ulong64); + tail = inlen - words * sizeof(ulong64); + + for(i = 0; i < words; i++, in += sizeof(ulong64)) { + ulong64 t; + LOAD64L(t, in); + md->sha3.s[md->sha3.word_index] ^= t; + if(++md->sha3.word_index == (SHA3_KECCAK_SPONGE_WORDS - md->sha3.capacity_words)) { + s_keccakf(md->sha3.s); + md->sha3.word_index = 0; + } + } + + /* finally, save the partial word */ + while (tail--) { + md->sha3.saved |= (ulong64) (*(in++)) << ((md->sha3.byte_index++) * 8); + } + return CRYPT_OK; +} + +#ifdef LTC_SHA3 +int sha3_done(hash_state *md, unsigned char *out) +{ + return ss_done(md, out, CONST64(0x06)); +} +#endif + +#ifdef LTC_KECCAK +int keccak_done(hash_state *md, unsigned char *out) +{ + return ss_done(md, out, CONST64(0x01)); +} +#endif + +#ifdef LTC_SHA3 +int sha3_shake_done(hash_state *md, unsigned char *out, unsigned long outlen) +{ + /* IMPORTANT NOTE: sha3_shake_done can be called many times */ + unsigned long idx; + unsigned i; + + if (outlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (!md->sha3.xof_flag) { + /* shake_xof operation must be done only once */ + md->sha3.s[md->sha3.word_index] ^= (md->sha3.saved ^ (CONST64(0x1F) << (md->sha3.byte_index * 8))); + md->sha3.s[SHA3_KECCAK_SPONGE_WORDS - md->sha3.capacity_words - 1] ^= CONST64(0x8000000000000000); + s_keccakf(md->sha3.s); + /* store sha3.s[] as little-endian bytes into sha3.sb */ + for(i = 0; i < SHA3_KECCAK_SPONGE_WORDS; i++) { + STORE64L(md->sha3.s[i], md->sha3.sb + i * 8); + } + md->sha3.byte_index = 0; + md->sha3.xof_flag = 1; + } + + for (idx = 0; idx < outlen; idx++) { + if(md->sha3.byte_index >= (SHA3_KECCAK_SPONGE_WORDS - md->sha3.capacity_words) * 8) { + s_keccakf(md->sha3.s); + /* store sha3.s[] as little-endian bytes into sha3.sb */ + for(i = 0; i < SHA3_KECCAK_SPONGE_WORDS; i++) { + STORE64L(md->sha3.s[i], md->sha3.sb + i * 8); + } + md->sha3.byte_index = 0; + } + out[idx] = md->sha3.sb[md->sha3.byte_index++]; + } + return CRYPT_OK; +} + +int sha3_shake_memory(int num, const unsigned char *in, unsigned long inlen, unsigned char *out, const unsigned long *outlen) +{ + hash_state md; + int err; + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + if ((err = sha3_shake_init(&md, num)) != CRYPT_OK) return err; + if ((err = sha3_shake_process(&md, in, inlen)) != CRYPT_OK) return err; + if ((err = sha3_shake_done(&md, out, *outlen)) != CRYPT_OK) return err; + return CRYPT_OK; +} +#endif + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sha3_test.c b/optee_os/core/lib/libtomcrypt/src/hashes/sha3_test.c new file mode 100644 index 0000000..95fcfcb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sha3_test.c @@ -0,0 +1,719 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* based on https://github.com/brainhub/SHA3IUF (public domain) */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SHA3 + +int sha3_224_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + unsigned char buf[200], hash[224 / 8]; + int i; + hash_state c; + const unsigned char c1 = 0xa3; + + const unsigned char sha3_224_empty[224 / 8] = { + 0x6b, 0x4e, 0x03, 0x42, 0x36, 0x67, 0xdb, 0xb7, + 0x3b, 0x6e, 0x15, 0x45, 0x4f, 0x0e, 0xb1, 0xab, + 0xd4, 0x59, 0x7f, 0x9a, 0x1b, 0x07, 0x8e, 0x3f, + 0x5b, 0x5a, 0x6b, 0xc7 + }; + + const unsigned char sha3_224_0xa3_200_times[224 / 8] = { + 0x93, 0x76, 0x81, 0x6a, 0xba, 0x50, 0x3f, 0x72, + 0xf9, 0x6c, 0xe7, 0xeb, 0x65, 0xac, 0x09, 0x5d, + 0xee, 0xe3, 0xbe, 0x4b, 0xf9, 0xbb, 0xc2, 0xa1, + 0xcb, 0x7e, 0x11, 0xe0 + }; + + XMEMSET(buf, c1, sizeof(buf)); + + /* SHA3-224 on an empty buffer */ + sha3_224_init(&c); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_224_empty, sizeof(sha3_224_empty), "SHA3-224", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-224 in two steps. [FIPS 202] */ + sha3_224_init(&c); + sha3_process(&c, buf, sizeof(buf) / 2); + sha3_process(&c, buf + sizeof(buf) / 2, sizeof(buf) / 2); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_224_0xa3_200_times, sizeof(sha3_224_0xa3_200_times), "SHA3-224", 1)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-224 byte-by-byte: 200 steps. [FIPS 202] */ + i = 200; + sha3_224_init(&c); + while (i--) { + sha3_process(&c, &c1, 1); + } + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_224_0xa3_200_times, sizeof(sha3_224_0xa3_200_times), "SHA3-224", 2)) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +int sha3_256_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + unsigned char buf[200], hash[256 / 8]; + int i; + hash_state c; + const unsigned char c1 = 0xa3; + + const unsigned char sha3_256_empty[256 / 8] = { + 0xa7, 0xff, 0xc6, 0xf8, 0xbf, 0x1e, 0xd7, 0x66, + 0x51, 0xc1, 0x47, 0x56, 0xa0, 0x61, 0xd6, 0x62, + 0xf5, 0x80, 0xff, 0x4d, 0xe4, 0x3b, 0x49, 0xfa, + 0x82, 0xd8, 0x0a, 0x4b, 0x80, 0xf8, 0x43, 0x4a + }; + const unsigned char sha3_256_0xa3_200_times[256 / 8] = { + 0x79, 0xf3, 0x8a, 0xde, 0xc5, 0xc2, 0x03, 0x07, + 0xa9, 0x8e, 0xf7, 0x6e, 0x83, 0x24, 0xaf, 0xbf, + 0xd4, 0x6c, 0xfd, 0x81, 0xb2, 0x2e, 0x39, 0x73, + 0xc6, 0x5f, 0xa1, 0xbd, 0x9d, 0xe3, 0x17, 0x87 + }; + + XMEMSET(buf, c1, sizeof(buf)); + + /* SHA3-256 on an empty buffer */ + sha3_256_init(&c); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_256_empty, sizeof(sha3_256_empty), "SHA3-256", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-256 as a single buffer. [FIPS 202] */ + sha3_256_init(&c); + sha3_process(&c, buf, sizeof(buf)); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_256_0xa3_200_times, sizeof(sha3_256_0xa3_200_times), "SHA3-256", 1)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-256 in two steps. [FIPS 202] */ + sha3_256_init(&c); + sha3_process(&c, buf, sizeof(buf) / 2); + sha3_process(&c, buf + sizeof(buf) / 2, sizeof(buf) / 2); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_256_0xa3_200_times, sizeof(sha3_256_0xa3_200_times), "SHA3-256", 2)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-256 byte-by-byte: 200 steps. [FIPS 202] */ + i = 200; + sha3_256_init(&c); + while (i--) { + sha3_process(&c, &c1, 1); + } + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_256_0xa3_200_times, sizeof(sha3_256_0xa3_200_times), "SHA3-256", 3)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-256 byte-by-byte: 135 bytes. Input from [Keccak]. Output + * matched with sha3sum. */ + sha3_256_init(&c); + sha3_process(&c, (unsigned char*) + "\xb7\x71\xd5\xce\xf5\xd1\xa4\x1a" + "\x93\xd1\x56\x43\xd7\x18\x1d\x2a" + "\x2e\xf0\xa8\xe8\x4d\x91\x81\x2f" + "\x20\xed\x21\xf1\x47\xbe\xf7\x32" + "\xbf\x3a\x60\xef\x40\x67\xc3\x73" + "\x4b\x85\xbc\x8c\xd4\x71\x78\x0f" + "\x10\xdc\x9e\x82\x91\xb5\x83\x39" + "\xa6\x77\xb9\x60\x21\x8f\x71\xe7" + "\x93\xf2\x79\x7a\xea\x34\x94\x06" + "\x51\x28\x29\x06\x5d\x37\xbb\x55" + "\xea\x79\x6f\xa4\xf5\x6f\xd8\x89" + "\x6b\x49\xb2\xcd\x19\xb4\x32\x15" + "\xad\x96\x7c\x71\x2b\x24\xe5\x03" + "\x2d\x06\x52\x32\xe0\x2c\x12\x74" + "\x09\xd2\xed\x41\x46\xb9\xd7\x5d" + "\x76\x3d\x52\xdb\x98\xd9\x49\xd3" + "\xb0\xfe\xd6\xa8\x05\x2f\xbb", 1080 / 8); + sha3_done(&c, hash); + if(compare_testvector(hash, sizeof(hash), + "\xa1\x9e\xee\x92\xbb\x20\x97\xb6" + "\x4e\x82\x3d\x59\x77\x98\xaa\x18" + "\xbe\x9b\x7c\x73\x6b\x80\x59\xab" + "\xfd\x67\x79\xac\x35\xac\x81\xb5", 256 / 8, "SHA3-256", 4)) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +int sha3_384_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + unsigned char buf[200], hash[384 / 8]; + int i; + hash_state c; + const unsigned char c1 = 0xa3; + + const unsigned char sha3_384_0xa3_200_times[384 / 8] = { + 0x18, 0x81, 0xde, 0x2c, 0xa7, 0xe4, 0x1e, 0xf9, + 0x5d, 0xc4, 0x73, 0x2b, 0x8f, 0x5f, 0x00, 0x2b, + 0x18, 0x9c, 0xc1, 0xe4, 0x2b, 0x74, 0x16, 0x8e, + 0xd1, 0x73, 0x26, 0x49, 0xce, 0x1d, 0xbc, 0xdd, + 0x76, 0x19, 0x7a, 0x31, 0xfd, 0x55, 0xee, 0x98, + 0x9f, 0x2d, 0x70, 0x50, 0xdd, 0x47, 0x3e, 0x8f + }; + + XMEMSET(buf, c1, sizeof(buf)); + + /* SHA3-384 as a single buffer. [FIPS 202] */ + sha3_384_init(&c); + sha3_process(&c, buf, sizeof(buf)); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_384_0xa3_200_times, sizeof(sha3_384_0xa3_200_times), "SHA3-384", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-384 in two steps. [FIPS 202] */ + sha3_384_init(&c); + sha3_process(&c, buf, sizeof(buf) / 2); + sha3_process(&c, buf + sizeof(buf) / 2, sizeof(buf) / 2); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_384_0xa3_200_times, sizeof(sha3_384_0xa3_200_times), "SHA3-384", 1)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-384 byte-by-byte: 200 steps. [FIPS 202] */ + i = 200; + sha3_384_init(&c); + while (i--) { + sha3_process(&c, &c1, 1); + } + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_384_0xa3_200_times, sizeof(sha3_384_0xa3_200_times), "SHA3-384", 2)) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +int sha3_512_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + unsigned char buf[200], hash[512 / 8]; + int i; + hash_state c; + const unsigned char c1 = 0xa3; + + const unsigned char sha3_512_0xa3_200_times[512 / 8] = { + 0xe7, 0x6d, 0xfa, 0xd2, 0x20, 0x84, 0xa8, 0xb1, + 0x46, 0x7f, 0xcf, 0x2f, 0xfa, 0x58, 0x36, 0x1b, + 0xec, 0x76, 0x28, 0xed, 0xf5, 0xf3, 0xfd, 0xc0, + 0xe4, 0x80, 0x5d, 0xc4, 0x8c, 0xae, 0xec, 0xa8, + 0x1b, 0x7c, 0x13, 0xc3, 0x0a, 0xdf, 0x52, 0xa3, + 0x65, 0x95, 0x84, 0x73, 0x9a, 0x2d, 0xf4, 0x6b, + 0xe5, 0x89, 0xc5, 0x1c, 0xa1, 0xa4, 0xa8, 0x41, + 0x6d, 0xf6, 0x54, 0x5a, 0x1c, 0xe8, 0xba, 0x00 + }; + + XMEMSET(buf, c1, sizeof(buf)); + + /* SHA3-512 as a single buffer. [FIPS 202] */ + sha3_512_init(&c); + sha3_process(&c, buf, sizeof(buf)); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_512_0xa3_200_times, sizeof(sha3_512_0xa3_200_times), "SHA3-512", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-512 in two steps. [FIPS 202] */ + sha3_512_init(&c); + sha3_process(&c, buf, sizeof(buf) / 2); + sha3_process(&c, buf + sizeof(buf) / 2, sizeof(buf) / 2); + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_512_0xa3_200_times, sizeof(sha3_512_0xa3_200_times), "SHA3-512", 1)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHA3-512 byte-by-byte: 200 steps. [FIPS 202] */ + i = 200; + sha3_512_init(&c); + while (i--) { + sha3_process(&c, &c1, 1); + } + sha3_done(&c, hash); + if (compare_testvector(hash, sizeof(hash), sha3_512_0xa3_200_times, sizeof(sha3_512_0xa3_200_times), "SHA3-512", 2)) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +int sha3_shake_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + unsigned char buf[200], hash[512]; + int i; + hash_state c; + const unsigned char c1 = 0xa3; + unsigned long len; + + const unsigned char shake256_empty[32] = { + 0xab, 0x0b, 0xae, 0x31, 0x63, 0x39, 0x89, 0x43, + 0x04, 0xe3, 0x58, 0x77, 0xb0, 0xc2, 0x8a, 0x9b, + 0x1f, 0xd1, 0x66, 0xc7, 0x96, 0xb9, 0xcc, 0x25, + 0x8a, 0x06, 0x4a, 0x8f, 0x57, 0xe2, 0x7f, 0x2a + }; + const unsigned char shake256_0xa3_200_times[32] = { + 0x6a, 0x1a, 0x9d, 0x78, 0x46, 0x43, 0x6e, 0x4d, + 0xca, 0x57, 0x28, 0xb6, 0xf7, 0x60, 0xee, 0xf0, + 0xca, 0x92, 0xbf, 0x0b, 0xe5, 0x61, 0x5e, 0x96, + 0x95, 0x9d, 0x76, 0x71, 0x97, 0xa0, 0xbe, 0xeb + }; + const unsigned char shake128_empty[32] = { + 0x43, 0xe4, 0x1b, 0x45, 0xa6, 0x53, 0xf2, 0xa5, + 0xc4, 0x49, 0x2c, 0x1a, 0xdd, 0x54, 0x45, 0x12, + 0xdd, 0xa2, 0x52, 0x98, 0x33, 0x46, 0x2b, 0x71, + 0xa4, 0x1a, 0x45, 0xbe, 0x97, 0x29, 0x0b, 0x6f + }; + const unsigned char shake128_0xa3_200_times[32] = { + 0x44, 0xc9, 0xfb, 0x35, 0x9f, 0xd5, 0x6a, 0xc0, + 0xa9, 0xa7, 0x5a, 0x74, 0x3c, 0xff, 0x68, 0x62, + 0xf1, 0x7d, 0x72, 0x59, 0xab, 0x07, 0x52, 0x16, + 0xc0, 0x69, 0x95, 0x11, 0x64, 0x3b, 0x64, 0x39 + }; + + XMEMSET(buf, c1, sizeof(buf)); + + /* SHAKE256 on an empty buffer */ + sha3_shake_init(&c, 256); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake256_empty), shake256_empty, sizeof(shake256_empty), "SHAKE256", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE256 via sha3_shake_memory [FIPS 202] */ + len = 512; + sha3_shake_memory(256, buf, sizeof(buf), hash, &len); + if (compare_testvector(hash + 480, sizeof(shake256_0xa3_200_times), shake256_0xa3_200_times, sizeof(shake256_0xa3_200_times), "SHAKE256", 1)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE256 as a single buffer. [FIPS 202] */ + sha3_shake_init(&c, 256); + sha3_shake_process(&c, buf, sizeof(buf)); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake256_0xa3_200_times), shake256_0xa3_200_times, sizeof(shake256_0xa3_200_times), "SHAKE256", 2)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE256 in two steps. [FIPS 202] */ + sha3_shake_init(&c, 256); + sha3_shake_process(&c, buf, sizeof(buf) / 2); + sha3_shake_process(&c, buf + sizeof(buf) / 2, sizeof(buf) / 2); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake256_0xa3_200_times), shake256_0xa3_200_times, sizeof(shake256_0xa3_200_times), "SHAKE256", 3)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE256 byte-by-byte: 200 steps. [FIPS 202] */ + i = 200; + sha3_shake_init(&c, 256); + while (i--) sha3_shake_process(&c, &c1, 1); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake256_0xa3_200_times), shake256_0xa3_200_times, sizeof(shake256_0xa3_200_times), "SHAKE256", 4)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE128 on an empty buffer */ + sha3_shake_init(&c, 128); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake128_empty), shake128_empty, sizeof(shake128_empty), "SHAKE128", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE128 via sha3_shake_memory [FIPS 202] */ + len = 512; + sha3_shake_memory(128, buf, sizeof(buf), hash, &len); + if (compare_testvector(hash + 480, sizeof(shake128_0xa3_200_times), shake128_0xa3_200_times, sizeof(shake128_0xa3_200_times), "SHAKE128", 1)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE128 as a single buffer. [FIPS 202] */ + sha3_shake_init(&c, 128); + sha3_shake_process(&c, buf, sizeof(buf)); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake128_0xa3_200_times), shake128_0xa3_200_times, sizeof(shake128_0xa3_200_times), "SHAKE128", 2)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE128 in two steps. [FIPS 202] */ + sha3_shake_init(&c, 128); + sha3_shake_process(&c, buf, sizeof(buf) / 2); + sha3_shake_process(&c, buf + sizeof(buf) / 2, sizeof(buf) / 2); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake128_0xa3_200_times), shake128_0xa3_200_times, sizeof(shake128_0xa3_200_times), "SHAKE128", 3)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* SHAKE128 byte-by-byte: 200 steps. [FIPS 202] */ + i = 200; + sha3_shake_init(&c, 128); + while (i--) sha3_shake_process(&c, &c1, 1); + for (i = 0; i < 16; i++) sha3_shake_done(&c, hash, 32); /* get 512 bytes, keep in hash the last 32 */ + if (compare_testvector(hash, sizeof(shake128_0xa3_200_times), shake128_0xa3_200_times, sizeof(shake128_0xa3_200_times), "SHAKE128", 4)) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +#endif + +#ifdef LTC_KECCAK + +int keccak_224_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + hash_state c; + unsigned char hash[MAXBLOCKSIZE]; + + keccak_224_init(&c); + keccak_process(&c, (unsigned char*) "\xcc", 1); + keccak_done(&c, hash); + if(compare_testvector(hash, 28, + "\xa9\xca\xb5\x9e\xb4\x0a\x10\xb2" + "\x46\x29\x0f\x2d\x60\x86\xe3\x2e" + "\x36\x89\xfa\xf1\xd2\x6b\x47\x0c" + "\x89\x9f\x28\x02", 28, + "KECCAK-224", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_224_init(&c); + keccak_process(&c, (unsigned char*)"\x41\xfb", 2); + keccak_done(&c, hash); + if(compare_testvector(hash, 28, + "\x61\x5b\xa3\x67\xaf\xdc\x35\xaa" + "\xc3\x97\xbc\x7e\xb5\xd5\x8d\x10" + "\x6a\x73\x4b\x24\x98\x6d\x5d\x97" + "\x8f\xef\xd6\x2c", 28, + "KECCAK-224", 1) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_224_init(&c); + keccak_process(&c, (unsigned char*) + "\x52\xa6\x08\xab\x21\xcc\xdd\x8a" + "\x44\x57\xa5\x7e\xde\x78\x21\x76", 16); + keccak_done(&c, hash); + if(compare_testvector(hash, 28, + "\x56\x79\xcd\x50\x9c\x51\x20\xaf" + "\x54\x79\x5c\xf4\x77\x14\x96\x41" + "\xcf\x27\xb2\xeb\xb6\xa5\xf9\x03" + "\x40\x70\x4e\x57", 28, + "KECCAK-224", 2) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_224_init(&c); + keccak_process(&c, (unsigned char*) + "\x43\x3c\x53\x03\x13\x16\x24\xc0" + "\x02\x1d\x86\x8a\x30\x82\x54\x75" + "\xe8\xd0\xbd\x30\x52\xa0\x22\x18" + "\x03\x98\xf4\xca\x44\x23\xb9\x82" + "\x14\xb6\xbe\xaa\xc2\x1c\x88\x07" + "\xa2\xc3\x3f\x8c\x93\xbd\x42\xb0" + "\x92\xcc\x1b\x06\xce\xdf\x32\x24" + "\xd5\xed\x1e\xc2\x97\x84\x44\x4f" + "\x22\xe0\x8a\x55\xaa\x58\x54\x2b" + "\x52\x4b\x02\xcd\x3d\x5d\x5f\x69" + "\x07\xaf\xe7\x1c\x5d\x74\x62\x22" + "\x4a\x3f\x9d\x9e\x53\xe7\xe0\x84" + "\x6d\xcb\xb4\xce", 100); + keccak_done(&c, hash); + if(compare_testvector(hash, 28, + "\x62\xb1\x0f\x1b\x62\x36\xeb\xc2" + "\xda\x72\x95\x77\x42\xa8\xd4\xe4" + "\x8e\x21\x3b\x5f\x89\x34\x60\x4b" + "\xfd\x4d\x2c\x3a", 28, + "KECCAK-224", 3) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +int keccak_256_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + hash_state c; + unsigned char hash[MAXBLOCKSIZE]; + + keccak_256_init(&c); + keccak_process(&c, (unsigned char*) "\xcc", 1); + keccak_done(&c, hash); + if(compare_testvector(hash, 32, + "\xee\xad\x6d\xbf\xc7\x34\x0a\x56" + "\xca\xed\xc0\x44\x69\x6a\x16\x88" + "\x70\x54\x9a\x6a\x7f\x6f\x56\x96" + "\x1e\x84\xa5\x4b\xd9\x97\x0b\x8a", 32, + "KECCAK-256", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_256_init(&c); + keccak_process(&c, (unsigned char*)"\x41\xfb", 2); + keccak_done(&c, hash); + if(compare_testvector(hash, 32, + "\xa8\xea\xce\xda\x4d\x47\xb3\x28" + "\x1a\x79\x5a\xd9\xe1\xea\x21\x22" + "\xb4\x07\xba\xf9\xaa\xbc\xb9\xe1" + "\x8b\x57\x17\xb7\x87\x35\x37\xd2", 32, + "KECCAK-256", 1) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_256_init(&c); + keccak_process(&c, (unsigned char*) + "\x52\xa6\x08\xab\x21\xcc\xdd\x8a" + "\x44\x57\xa5\x7e\xde\x78\x21\x76", 16); + keccak_done(&c, hash); + if(compare_testvector(hash, 32, + "\x0e\x32\xde\xfa\x20\x71\xf0\xb5" + "\xac\x0e\x6a\x10\x8b\x84\x2e\xd0" + "\xf1\xd3\x24\x97\x12\xf5\x8e\xe0" + "\xdd\xf9\x56\xfe\x33\x2a\x5f\x95", 32, + "KECCAK-256", 2) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_256_init(&c); + keccak_process(&c, (unsigned char*) + "\x43\x3c\x53\x03\x13\x16\x24\xc0" + "\x02\x1d\x86\x8a\x30\x82\x54\x75" + "\xe8\xd0\xbd\x30\x52\xa0\x22\x18" + "\x03\x98\xf4\xca\x44\x23\xb9\x82" + "\x14\xb6\xbe\xaa\xc2\x1c\x88\x07" + "\xa2\xc3\x3f\x8c\x93\xbd\x42\xb0" + "\x92\xcc\x1b\x06\xce\xdf\x32\x24" + "\xd5\xed\x1e\xc2\x97\x84\x44\x4f" + "\x22\xe0\x8a\x55\xaa\x58\x54\x2b" + "\x52\x4b\x02\xcd\x3d\x5d\x5f\x69" + "\x07\xaf\xe7\x1c\x5d\x74\x62\x22" + "\x4a\x3f\x9d\x9e\x53\xe7\xe0\x84" + "\x6d\xcb\xb4\xce", 100); + keccak_done(&c, hash); + if(compare_testvector(hash, 32, + "\xce\x87\xa5\x17\x3b\xff\xd9\x23" + "\x99\x22\x16\x58\xf8\x01\xd4\x5c" + "\x29\x4d\x90\x06\xee\x9f\x3f\x9d" + "\x41\x9c\x8d\x42\x77\x48\xdc\x41", 32, + "KECCAK-256", 3) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +int keccak_384_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + hash_state c; + unsigned char hash[MAXBLOCKSIZE]; + + keccak_384_init(&c); + keccak_process(&c, (unsigned char*) "\xcc", 1); + keccak_done(&c, hash); + if(compare_testvector(hash, 48, + "\x1b\x84\xe6\x2a\x46\xe5\xa2\x01" + "\x86\x17\x54\xaf\x5d\xc9\x5c\x4a" + "\x1a\x69\xca\xf4\xa7\x96\xae\x40" + "\x56\x80\x16\x1e\x29\x57\x26\x41" + "\xf5\xfa\x1e\x86\x41\xd7\x95\x83" + "\x36\xee\x7b\x11\xc5\x8f\x73\xe9", 48, + "KECCAK-384", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_384_init(&c); + keccak_process(&c, (unsigned char*)"\x41\xfb", 2); + keccak_done(&c, hash); + if(compare_testvector(hash, 48, + "\x49\x5c\xce\x27\x14\xcd\x72\xc8" + "\xc5\x3c\x33\x63\xd2\x2c\x58\xb5" + "\x59\x60\xfe\x26\xbe\x0b\xf3\xbb" + "\xc7\xa3\x31\x6d\xd5\x63\xad\x1d" + "\xb8\x41\x0e\x75\xee\xfe\xa6\x55" + "\xe3\x9d\x46\x70\xec\x0b\x17\x92", 48, + "KECCAK-384", 1) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_384_init(&c); + keccak_process(&c, (unsigned char*) + "\x52\xa6\x08\xab\x21\xcc\xdd\x8a" + "\x44\x57\xa5\x7e\xde\x78\x21\x76", 16); + keccak_done(&c, hash); + if(compare_testvector(hash, 48, + "\x18\x42\x2a\xc1\xd3\xa1\xe5\x4b" + "\xad\x87\x68\x83\xd2\xd6\xdd\x65" + "\xf6\x5c\x1d\x5f\x33\xa7\x12\x5c" + "\xc4\xc1\x86\x40\x5a\x12\xed\x64" + "\xba\x96\x67\x2e\xed\xda\x8c\x5a" + "\x63\x31\xd2\x86\x83\xf4\x88\xeb", 48, + "KECCAK-384", 2) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_384_init(&c); + keccak_process(&c, (unsigned char*) + "\x43\x3c\x53\x03\x13\x16\x24\xc0" + "\x02\x1d\x86\x8a\x30\x82\x54\x75" + "\xe8\xd0\xbd\x30\x52\xa0\x22\x18" + "\x03\x98\xf4\xca\x44\x23\xb9\x82" + "\x14\xb6\xbe\xaa\xc2\x1c\x88\x07" + "\xa2\xc3\x3f\x8c\x93\xbd\x42\xb0" + "\x92\xcc\x1b\x06\xce\xdf\x32\x24" + "\xd5\xed\x1e\xc2\x97\x84\x44\x4f" + "\x22\xe0\x8a\x55\xaa\x58\x54\x2b" + "\x52\x4b\x02\xcd\x3d\x5d\x5f\x69" + "\x07\xaf\xe7\x1c\x5d\x74\x62\x22" + "\x4a\x3f\x9d\x9e\x53\xe7\xe0\x84" + "\x6d\xcb\xb4\xce", 100); + keccak_done(&c, hash); + if(compare_testvector(hash, 48, + "\x13\x51\x14\x50\x8d\xd6\x3e\x27" + "\x9e\x70\x9c\x26\xf7\x81\x7c\x04" + "\x82\x76\x6c\xde\x49\x13\x2e\x3e" + "\xdf\x2e\xed\xd8\x99\x6f\x4e\x35" + "\x96\xd1\x84\x10\x0b\x38\x48\x68" + "\x24\x9f\x1d\x8b\x8f\xda\xa2\xc9", 48, + "KECCAK-384", 3) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +int keccak_512_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + hash_state c; + unsigned char hash[MAXBLOCKSIZE]; + + keccak_512_init(&c); + keccak_process(&c, (unsigned char*) "\xcc", 1); + keccak_done(&c, hash); + if(compare_testvector(hash, 64, + "\x86\x30\xc1\x3c\xbd\x06\x6e\xa7" + "\x4b\xbe\x7f\xe4\x68\xfe\xc1\xde" + "\xe1\x0e\xdc\x12\x54\xfb\x4c\x1b" + "\x7c\x5f\xd6\x9b\x64\x6e\x44\x16" + "\x0b\x8c\xe0\x1d\x05\xa0\x90\x8c" + "\xa7\x90\xdf\xb0\x80\xf4\xb5\x13" + "\xbc\x3b\x62\x25\xec\xe7\xa8\x10" + "\x37\x14\x41\xa5\xac\x66\x6e\xb9", 64, + "KECCAK-512", 0) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_512_init(&c); + keccak_process(&c, (unsigned char*)"\x41\xfb", 2); + keccak_done(&c, hash); + if(compare_testvector(hash, 64, + "\x55\x1d\xa6\x23\x6f\x8b\x96\xfc" + "\xe9\xf9\x7f\x11\x90\xe9\x01\x32" + "\x4f\x0b\x45\xe0\x6d\xbb\xb5\xcd" + "\xb8\x35\x5d\x6e\xd1\xdc\x34\xb3" + "\xf0\xea\xe7\xdc\xb6\x86\x22\xff" + "\x23\x2f\xa3\xce\xce\x0d\x46\x16" + "\xcd\xeb\x39\x31\xf9\x38\x03\x66" + "\x2a\x28\xdf\x1c\xd5\x35\xb7\x31", 64, + "KECCAK-512", 1) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_512_init(&c); + keccak_process(&c, (unsigned char*) + "\x52\xa6\x08\xab\x21\xcc\xdd\x8a" + "\x44\x57\xa5\x7e\xde\x78\x21\x76", 16); + keccak_done(&c, hash); + if(compare_testvector(hash, 64, + "\x4b\x39\xd3\xda\x5b\xcd\xf4\xd9" + "\xb7\x69\x01\x59\x95\x64\x43\x11" + "\xc1\x4c\x43\x5b\xf7\x2b\x10\x09" + "\xd6\xdd\x71\xb0\x1a\x63\xb9\x7c" + "\xfb\x59\x64\x18\xe8\xe4\x23\x42" + "\xd1\x17\xe0\x74\x71\xa8\x91\x43" + "\x14\xba\x7b\x0e\x26\x4d\xad\xf0" + "\xce\xa3\x81\x86\x8c\xbd\x43\xd1", 64, + "KECCAK-512", 2) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + keccak_512_init(&c); + keccak_process(&c, (unsigned char*) + "\x43\x3c\x53\x03\x13\x16\x24\xc0" + "\x02\x1d\x86\x8a\x30\x82\x54\x75" + "\xe8\xd0\xbd\x30\x52\xa0\x22\x18" + "\x03\x98\xf4\xca\x44\x23\xb9\x82" + "\x14\xb6\xbe\xaa\xc2\x1c\x88\x07" + "\xa2\xc3\x3f\x8c\x93\xbd\x42\xb0" + "\x92\xcc\x1b\x06\xce\xdf\x32\x24" + "\xd5\xed\x1e\xc2\x97\x84\x44\x4f" + "\x22\xe0\x8a\x55\xaa\x58\x54\x2b" + "\x52\x4b\x02\xcd\x3d\x5d\x5f\x69" + "\x07\xaf\xe7\x1c\x5d\x74\x62\x22" + "\x4a\x3f\x9d\x9e\x53\xe7\xe0\x84" + "\x6d\xcb\xb4\xce", 100); + keccak_done(&c, hash); + if(compare_testvector(hash, 64, + "\x52\x7d\x28\xe3\x41\xe6\xb1\x4f" + "\x46\x84\xad\xb4\xb8\x24\xc4\x96" + "\xc6\x48\x2e\x51\x14\x95\x65\xd3" + "\xd1\x72\x26\x82\x88\x84\x30\x6b" + "\x51\xd6\x14\x8a\x72\x62\x2c\x2b" + "\x75\xf5\xd3\x51\x0b\x79\x9d\x8b" + "\xdc\x03\xea\xed\xe4\x53\x67\x6a" + "\x6e\xc8\xfe\x03\xa1\xad\x0e\xab", 64, + "KECCAK-512", 3) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/sub.mk b/optee_os/core/lib/libtomcrypt/src/hashes/sub.mk new file mode 100644 index 0000000..6a35fe2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/sub.mk @@ -0,0 +1,14 @@ +srcs-$(_CFG_CORE_LTC_MD5_DESC) += md5.c + +ifeq ($(_CFG_CORE_LTC_SHA1_DESC),y) +ifneq ($(_CFG_CORE_LTC_SHA1_ACCEL),y) +srcs-y += sha1.c +endif +endif + +ifneq ($(_CFG_CORE_LTC_SHA3_ACCEL),y) +srcs-$(_CFG_CORE_LTC_SHA3_DESC) += sha3.c +endif +srcs-$(_CFG_CORE_LTC_SHA3_DESC) += sha3_test.c +subdirs-y += helper +subdirs-y += sha2 diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/tiger.c b/optee_os/core/lib/libtomcrypt/src/hashes/tiger.c new file mode 100644 index 0000000..8ca2d54 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/tiger.c @@ -0,0 +1,796 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file tiger.c + Tiger hash function, Tom St Denis +*/ + +#ifdef LTC_TIGER + +const struct ltc_hash_descriptor tiger_desc = +{ + "tiger", + 1, + 24, + 64, + + /* OID */ + { 1, 3, 6, 1, 4, 1, 11591, 12, 2, }, + 9, + + &tiger_init, + &tiger_process, + &tiger_done, + &tiger_test, + NULL +}; + +#define t1 (table) +#define t2 (table+256) +#define t3 (table+256*2) +#define t4 (table+256*3) + +static const ulong64 table[4*256] = { + CONST64(0x02AAB17CF7E90C5E) /* 0 */, CONST64(0xAC424B03E243A8EC) /* 1 */, + CONST64(0x72CD5BE30DD5FCD3) /* 2 */, CONST64(0x6D019B93F6F97F3A) /* 3 */, + CONST64(0xCD9978FFD21F9193) /* 4 */, CONST64(0x7573A1C9708029E2) /* 5 */, + CONST64(0xB164326B922A83C3) /* 6 */, CONST64(0x46883EEE04915870) /* 7 */, + CONST64(0xEAACE3057103ECE6) /* 8 */, CONST64(0xC54169B808A3535C) /* 9 */, + CONST64(0x4CE754918DDEC47C) /* 10 */, CONST64(0x0AA2F4DFDC0DF40C) /* 11 */, + CONST64(0x10B76F18A74DBEFA) /* 12 */, CONST64(0xC6CCB6235AD1AB6A) /* 13 */, + CONST64(0x13726121572FE2FF) /* 14 */, CONST64(0x1A488C6F199D921E) /* 15 */, + CONST64(0x4BC9F9F4DA0007CA) /* 16 */, CONST64(0x26F5E6F6E85241C7) /* 17 */, + CONST64(0x859079DBEA5947B6) /* 18 */, CONST64(0x4F1885C5C99E8C92) /* 19 */, + CONST64(0xD78E761EA96F864B) /* 20 */, CONST64(0x8E36428C52B5C17D) /* 21 */, + CONST64(0x69CF6827373063C1) /* 22 */, CONST64(0xB607C93D9BB4C56E) /* 23 */, + CONST64(0x7D820E760E76B5EA) /* 24 */, CONST64(0x645C9CC6F07FDC42) /* 25 */, + CONST64(0xBF38A078243342E0) /* 26 */, CONST64(0x5F6B343C9D2E7D04) /* 27 */, + CONST64(0xF2C28AEB600B0EC6) /* 28 */, CONST64(0x6C0ED85F7254BCAC) /* 29 */, + CONST64(0x71592281A4DB4FE5) /* 30 */, CONST64(0x1967FA69CE0FED9F) /* 31 */, + CONST64(0xFD5293F8B96545DB) /* 32 */, CONST64(0xC879E9D7F2A7600B) /* 33 */, + CONST64(0x860248920193194E) /* 34 */, CONST64(0xA4F9533B2D9CC0B3) /* 35 */, + CONST64(0x9053836C15957613) /* 36 */, CONST64(0xDB6DCF8AFC357BF1) /* 37 */, + CONST64(0x18BEEA7A7A370F57) /* 38 */, CONST64(0x037117CA50B99066) /* 39 */, + CONST64(0x6AB30A9774424A35) /* 40 */, CONST64(0xF4E92F02E325249B) /* 41 */, + CONST64(0x7739DB07061CCAE1) /* 42 */, CONST64(0xD8F3B49CECA42A05) /* 43 */, + CONST64(0xBD56BE3F51382F73) /* 44 */, CONST64(0x45FAED5843B0BB28) /* 45 */, + CONST64(0x1C813D5C11BF1F83) /* 46 */, CONST64(0x8AF0E4B6D75FA169) /* 47 */, + CONST64(0x33EE18A487AD9999) /* 48 */, CONST64(0x3C26E8EAB1C94410) /* 49 */, + CONST64(0xB510102BC0A822F9) /* 50 */, CONST64(0x141EEF310CE6123B) /* 51 */, + CONST64(0xFC65B90059DDB154) /* 52 */, CONST64(0xE0158640C5E0E607) /* 53 */, + CONST64(0x884E079826C3A3CF) /* 54 */, CONST64(0x930D0D9523C535FD) /* 55 */, + CONST64(0x35638D754E9A2B00) /* 56 */, CONST64(0x4085FCCF40469DD5) /* 57 */, + CONST64(0xC4B17AD28BE23A4C) /* 58 */, CONST64(0xCAB2F0FC6A3E6A2E) /* 59 */, + CONST64(0x2860971A6B943FCD) /* 60 */, CONST64(0x3DDE6EE212E30446) /* 61 */, + CONST64(0x6222F32AE01765AE) /* 62 */, CONST64(0x5D550BB5478308FE) /* 63 */, + CONST64(0xA9EFA98DA0EDA22A) /* 64 */, CONST64(0xC351A71686C40DA7) /* 65 */, + CONST64(0x1105586D9C867C84) /* 66 */, CONST64(0xDCFFEE85FDA22853) /* 67 */, + CONST64(0xCCFBD0262C5EEF76) /* 68 */, CONST64(0xBAF294CB8990D201) /* 69 */, + CONST64(0xE69464F52AFAD975) /* 70 */, CONST64(0x94B013AFDF133E14) /* 71 */, + CONST64(0x06A7D1A32823C958) /* 72 */, CONST64(0x6F95FE5130F61119) /* 73 */, + CONST64(0xD92AB34E462C06C0) /* 74 */, CONST64(0xED7BDE33887C71D2) /* 75 */, + CONST64(0x79746D6E6518393E) /* 76 */, CONST64(0x5BA419385D713329) /* 77 */, + CONST64(0x7C1BA6B948A97564) /* 78 */, CONST64(0x31987C197BFDAC67) /* 79 */, + CONST64(0xDE6C23C44B053D02) /* 80 */, CONST64(0x581C49FED002D64D) /* 81 */, + CONST64(0xDD474D6338261571) /* 82 */, CONST64(0xAA4546C3E473D062) /* 83 */, + CONST64(0x928FCE349455F860) /* 84 */, CONST64(0x48161BBACAAB94D9) /* 85 */, + CONST64(0x63912430770E6F68) /* 86 */, CONST64(0x6EC8A5E602C6641C) /* 87 */, + CONST64(0x87282515337DDD2B) /* 88 */, CONST64(0x2CDA6B42034B701B) /* 89 */, + CONST64(0xB03D37C181CB096D) /* 90 */, CONST64(0xE108438266C71C6F) /* 91 */, + CONST64(0x2B3180C7EB51B255) /* 92 */, CONST64(0xDF92B82F96C08BBC) /* 93 */, + CONST64(0x5C68C8C0A632F3BA) /* 94 */, CONST64(0x5504CC861C3D0556) /* 95 */, + CONST64(0xABBFA4E55FB26B8F) /* 96 */, CONST64(0x41848B0AB3BACEB4) /* 97 */, + CONST64(0xB334A273AA445D32) /* 98 */, CONST64(0xBCA696F0A85AD881) /* 99 */, + CONST64(0x24F6EC65B528D56C) /* 100 */, CONST64(0x0CE1512E90F4524A) /* 101 */, + CONST64(0x4E9DD79D5506D35A) /* 102 */, CONST64(0x258905FAC6CE9779) /* 103 */, + CONST64(0x2019295B3E109B33) /* 104 */, CONST64(0xF8A9478B73A054CC) /* 105 */, + CONST64(0x2924F2F934417EB0) /* 106 */, CONST64(0x3993357D536D1BC4) /* 107 */, + CONST64(0x38A81AC21DB6FF8B) /* 108 */, CONST64(0x47C4FBF17D6016BF) /* 109 */, + CONST64(0x1E0FAADD7667E3F5) /* 110 */, CONST64(0x7ABCFF62938BEB96) /* 111 */, + CONST64(0xA78DAD948FC179C9) /* 112 */, CONST64(0x8F1F98B72911E50D) /* 113 */, + CONST64(0x61E48EAE27121A91) /* 114 */, CONST64(0x4D62F7AD31859808) /* 115 */, + CONST64(0xECEBA345EF5CEAEB) /* 116 */, CONST64(0xF5CEB25EBC9684CE) /* 117 */, + CONST64(0xF633E20CB7F76221) /* 118 */, CONST64(0xA32CDF06AB8293E4) /* 119 */, + CONST64(0x985A202CA5EE2CA4) /* 120 */, CONST64(0xCF0B8447CC8A8FB1) /* 121 */, + CONST64(0x9F765244979859A3) /* 122 */, CONST64(0xA8D516B1A1240017) /* 123 */, + CONST64(0x0BD7BA3EBB5DC726) /* 124 */, CONST64(0xE54BCA55B86ADB39) /* 125 */, + CONST64(0x1D7A3AFD6C478063) /* 126 */, CONST64(0x519EC608E7669EDD) /* 127 */, + CONST64(0x0E5715A2D149AA23) /* 128 */, CONST64(0x177D4571848FF194) /* 129 */, + CONST64(0xEEB55F3241014C22) /* 130 */, CONST64(0x0F5E5CA13A6E2EC2) /* 131 */, + CONST64(0x8029927B75F5C361) /* 132 */, CONST64(0xAD139FABC3D6E436) /* 133 */, + CONST64(0x0D5DF1A94CCF402F) /* 134 */, CONST64(0x3E8BD948BEA5DFC8) /* 135 */, + CONST64(0xA5A0D357BD3FF77E) /* 136 */, CONST64(0xA2D12E251F74F645) /* 137 */, + CONST64(0x66FD9E525E81A082) /* 138 */, CONST64(0x2E0C90CE7F687A49) /* 139 */, + CONST64(0xC2E8BCBEBA973BC5) /* 140 */, CONST64(0x000001BCE509745F) /* 141 */, + CONST64(0x423777BBE6DAB3D6) /* 142 */, CONST64(0xD1661C7EAEF06EB5) /* 143 */, + CONST64(0xA1781F354DAACFD8) /* 144 */, CONST64(0x2D11284A2B16AFFC) /* 145 */, + CONST64(0xF1FC4F67FA891D1F) /* 146 */, CONST64(0x73ECC25DCB920ADA) /* 147 */, + CONST64(0xAE610C22C2A12651) /* 148 */, CONST64(0x96E0A810D356B78A) /* 149 */, + CONST64(0x5A9A381F2FE7870F) /* 150 */, CONST64(0xD5AD62EDE94E5530) /* 151 */, + CONST64(0xD225E5E8368D1427) /* 152 */, CONST64(0x65977B70C7AF4631) /* 153 */, + CONST64(0x99F889B2DE39D74F) /* 154 */, CONST64(0x233F30BF54E1D143) /* 155 */, + CONST64(0x9A9675D3D9A63C97) /* 156 */, CONST64(0x5470554FF334F9A8) /* 157 */, + CONST64(0x166ACB744A4F5688) /* 158 */, CONST64(0x70C74CAAB2E4AEAD) /* 159 */, + CONST64(0xF0D091646F294D12) /* 160 */, CONST64(0x57B82A89684031D1) /* 161 */, + CONST64(0xEFD95A5A61BE0B6B) /* 162 */, CONST64(0x2FBD12E969F2F29A) /* 163 */, + CONST64(0x9BD37013FEFF9FE8) /* 164 */, CONST64(0x3F9B0404D6085A06) /* 165 */, + CONST64(0x4940C1F3166CFE15) /* 166 */, CONST64(0x09542C4DCDF3DEFB) /* 167 */, + CONST64(0xB4C5218385CD5CE3) /* 168 */, CONST64(0xC935B7DC4462A641) /* 169 */, + CONST64(0x3417F8A68ED3B63F) /* 170 */, CONST64(0xB80959295B215B40) /* 171 */, + CONST64(0xF99CDAEF3B8C8572) /* 172 */, CONST64(0x018C0614F8FCB95D) /* 173 */, + CONST64(0x1B14ACCD1A3ACDF3) /* 174 */, CONST64(0x84D471F200BB732D) /* 175 */, + CONST64(0xC1A3110E95E8DA16) /* 176 */, CONST64(0x430A7220BF1A82B8) /* 177 */, + CONST64(0xB77E090D39DF210E) /* 178 */, CONST64(0x5EF4BD9F3CD05E9D) /* 179 */, + CONST64(0x9D4FF6DA7E57A444) /* 180 */, CONST64(0xDA1D60E183D4A5F8) /* 181 */, + CONST64(0xB287C38417998E47) /* 182 */, CONST64(0xFE3EDC121BB31886) /* 183 */, + CONST64(0xC7FE3CCC980CCBEF) /* 184 */, CONST64(0xE46FB590189BFD03) /* 185 */, + CONST64(0x3732FD469A4C57DC) /* 186 */, CONST64(0x7EF700A07CF1AD65) /* 187 */, + CONST64(0x59C64468A31D8859) /* 188 */, CONST64(0x762FB0B4D45B61F6) /* 189 */, + CONST64(0x155BAED099047718) /* 190 */, CONST64(0x68755E4C3D50BAA6) /* 191 */, + CONST64(0xE9214E7F22D8B4DF) /* 192 */, CONST64(0x2ADDBF532EAC95F4) /* 193 */, + CONST64(0x32AE3909B4BD0109) /* 194 */, CONST64(0x834DF537B08E3450) /* 195 */, + CONST64(0xFA209DA84220728D) /* 196 */, CONST64(0x9E691D9B9EFE23F7) /* 197 */, + CONST64(0x0446D288C4AE8D7F) /* 198 */, CONST64(0x7B4CC524E169785B) /* 199 */, + CONST64(0x21D87F0135CA1385) /* 200 */, CONST64(0xCEBB400F137B8AA5) /* 201 */, + CONST64(0x272E2B66580796BE) /* 202 */, CONST64(0x3612264125C2B0DE) /* 203 */, + CONST64(0x057702BDAD1EFBB2) /* 204 */, CONST64(0xD4BABB8EACF84BE9) /* 205 */, + CONST64(0x91583139641BC67B) /* 206 */, CONST64(0x8BDC2DE08036E024) /* 207 */, + CONST64(0x603C8156F49F68ED) /* 208 */, CONST64(0xF7D236F7DBEF5111) /* 209 */, + CONST64(0x9727C4598AD21E80) /* 210 */, CONST64(0xA08A0896670A5FD7) /* 211 */, + CONST64(0xCB4A8F4309EBA9CB) /* 212 */, CONST64(0x81AF564B0F7036A1) /* 213 */, + CONST64(0xC0B99AA778199ABD) /* 214 */, CONST64(0x959F1EC83FC8E952) /* 215 */, + CONST64(0x8C505077794A81B9) /* 216 */, CONST64(0x3ACAAF8F056338F0) /* 217 */, + CONST64(0x07B43F50627A6778) /* 218 */, CONST64(0x4A44AB49F5ECCC77) /* 219 */, + CONST64(0x3BC3D6E4B679EE98) /* 220 */, CONST64(0x9CC0D4D1CF14108C) /* 221 */, + CONST64(0x4406C00B206BC8A0) /* 222 */, CONST64(0x82A18854C8D72D89) /* 223 */, + CONST64(0x67E366B35C3C432C) /* 224 */, CONST64(0xB923DD61102B37F2) /* 225 */, + CONST64(0x56AB2779D884271D) /* 226 */, CONST64(0xBE83E1B0FF1525AF) /* 227 */, + CONST64(0xFB7C65D4217E49A9) /* 228 */, CONST64(0x6BDBE0E76D48E7D4) /* 229 */, + CONST64(0x08DF828745D9179E) /* 230 */, CONST64(0x22EA6A9ADD53BD34) /* 231 */, + CONST64(0xE36E141C5622200A) /* 232 */, CONST64(0x7F805D1B8CB750EE) /* 233 */, + CONST64(0xAFE5C7A59F58E837) /* 234 */, CONST64(0xE27F996A4FB1C23C) /* 235 */, + CONST64(0xD3867DFB0775F0D0) /* 236 */, CONST64(0xD0E673DE6E88891A) /* 237 */, + CONST64(0x123AEB9EAFB86C25) /* 238 */, CONST64(0x30F1D5D5C145B895) /* 239 */, + CONST64(0xBB434A2DEE7269E7) /* 240 */, CONST64(0x78CB67ECF931FA38) /* 241 */, + CONST64(0xF33B0372323BBF9C) /* 242 */, CONST64(0x52D66336FB279C74) /* 243 */, + CONST64(0x505F33AC0AFB4EAA) /* 244 */, CONST64(0xE8A5CD99A2CCE187) /* 245 */, + CONST64(0x534974801E2D30BB) /* 246 */, CONST64(0x8D2D5711D5876D90) /* 247 */, + CONST64(0x1F1A412891BC038E) /* 248 */, CONST64(0xD6E2E71D82E56648) /* 249 */, + CONST64(0x74036C3A497732B7) /* 250 */, CONST64(0x89B67ED96361F5AB) /* 251 */, + CONST64(0xFFED95D8F1EA02A2) /* 252 */, CONST64(0xE72B3BD61464D43D) /* 253 */, + CONST64(0xA6300F170BDC4820) /* 254 */, CONST64(0xEBC18760ED78A77A) /* 255 */, + CONST64(0xE6A6BE5A05A12138) /* 256 */, CONST64(0xB5A122A5B4F87C98) /* 257 */, + CONST64(0x563C6089140B6990) /* 258 */, CONST64(0x4C46CB2E391F5DD5) /* 259 */, + CONST64(0xD932ADDBC9B79434) /* 260 */, CONST64(0x08EA70E42015AFF5) /* 261 */, + CONST64(0xD765A6673E478CF1) /* 262 */, CONST64(0xC4FB757EAB278D99) /* 263 */, + CONST64(0xDF11C6862D6E0692) /* 264 */, CONST64(0xDDEB84F10D7F3B16) /* 265 */, + CONST64(0x6F2EF604A665EA04) /* 266 */, CONST64(0x4A8E0F0FF0E0DFB3) /* 267 */, + CONST64(0xA5EDEEF83DBCBA51) /* 268 */, CONST64(0xFC4F0A2A0EA4371E) /* 269 */, + CONST64(0xE83E1DA85CB38429) /* 270 */, CONST64(0xDC8FF882BA1B1CE2) /* 271 */, + CONST64(0xCD45505E8353E80D) /* 272 */, CONST64(0x18D19A00D4DB0717) /* 273 */, + CONST64(0x34A0CFEDA5F38101) /* 274 */, CONST64(0x0BE77E518887CAF2) /* 275 */, + CONST64(0x1E341438B3C45136) /* 276 */, CONST64(0xE05797F49089CCF9) /* 277 */, + CONST64(0xFFD23F9DF2591D14) /* 278 */, CONST64(0x543DDA228595C5CD) /* 279 */, + CONST64(0x661F81FD99052A33) /* 280 */, CONST64(0x8736E641DB0F7B76) /* 281 */, + CONST64(0x15227725418E5307) /* 282 */, CONST64(0xE25F7F46162EB2FA) /* 283 */, + CONST64(0x48A8B2126C13D9FE) /* 284 */, CONST64(0xAFDC541792E76EEA) /* 285 */, + CONST64(0x03D912BFC6D1898F) /* 286 */, CONST64(0x31B1AAFA1B83F51B) /* 287 */, + CONST64(0xF1AC2796E42AB7D9) /* 288 */, CONST64(0x40A3A7D7FCD2EBAC) /* 289 */, + CONST64(0x1056136D0AFBBCC5) /* 290 */, CONST64(0x7889E1DD9A6D0C85) /* 291 */, + CONST64(0xD33525782A7974AA) /* 292 */, CONST64(0xA7E25D09078AC09B) /* 293 */, + CONST64(0xBD4138B3EAC6EDD0) /* 294 */, CONST64(0x920ABFBE71EB9E70) /* 295 */, + CONST64(0xA2A5D0F54FC2625C) /* 296 */, CONST64(0xC054E36B0B1290A3) /* 297 */, + CONST64(0xF6DD59FF62FE932B) /* 298 */, CONST64(0x3537354511A8AC7D) /* 299 */, + CONST64(0xCA845E9172FADCD4) /* 300 */, CONST64(0x84F82B60329D20DC) /* 301 */, + CONST64(0x79C62CE1CD672F18) /* 302 */, CONST64(0x8B09A2ADD124642C) /* 303 */, + CONST64(0xD0C1E96A19D9E726) /* 304 */, CONST64(0x5A786A9B4BA9500C) /* 305 */, + CONST64(0x0E020336634C43F3) /* 306 */, CONST64(0xC17B474AEB66D822) /* 307 */, + CONST64(0x6A731AE3EC9BAAC2) /* 308 */, CONST64(0x8226667AE0840258) /* 309 */, + CONST64(0x67D4567691CAECA5) /* 310 */, CONST64(0x1D94155C4875ADB5) /* 311 */, + CONST64(0x6D00FD985B813FDF) /* 312 */, CONST64(0x51286EFCB774CD06) /* 313 */, + CONST64(0x5E8834471FA744AF) /* 314 */, CONST64(0xF72CA0AEE761AE2E) /* 315 */, + CONST64(0xBE40E4CDAEE8E09A) /* 316 */, CONST64(0xE9970BBB5118F665) /* 317 */, + CONST64(0x726E4BEB33DF1964) /* 318 */, CONST64(0x703B000729199762) /* 319 */, + CONST64(0x4631D816F5EF30A7) /* 320 */, CONST64(0xB880B5B51504A6BE) /* 321 */, + CONST64(0x641793C37ED84B6C) /* 322 */, CONST64(0x7B21ED77F6E97D96) /* 323 */, + CONST64(0x776306312EF96B73) /* 324 */, CONST64(0xAE528948E86FF3F4) /* 325 */, + CONST64(0x53DBD7F286A3F8F8) /* 326 */, CONST64(0x16CADCE74CFC1063) /* 327 */, + CONST64(0x005C19BDFA52C6DD) /* 328 */, CONST64(0x68868F5D64D46AD3) /* 329 */, + CONST64(0x3A9D512CCF1E186A) /* 330 */, CONST64(0x367E62C2385660AE) /* 331 */, + CONST64(0xE359E7EA77DCB1D7) /* 332 */, CONST64(0x526C0773749ABE6E) /* 333 */, + CONST64(0x735AE5F9D09F734B) /* 334 */, CONST64(0x493FC7CC8A558BA8) /* 335 */, + CONST64(0xB0B9C1533041AB45) /* 336 */, CONST64(0x321958BA470A59BD) /* 337 */, + CONST64(0x852DB00B5F46C393) /* 338 */, CONST64(0x91209B2BD336B0E5) /* 339 */, + CONST64(0x6E604F7D659EF19F) /* 340 */, CONST64(0xB99A8AE2782CCB24) /* 341 */, + CONST64(0xCCF52AB6C814C4C7) /* 342 */, CONST64(0x4727D9AFBE11727B) /* 343 */, + CONST64(0x7E950D0C0121B34D) /* 344 */, CONST64(0x756F435670AD471F) /* 345 */, + CONST64(0xF5ADD442615A6849) /* 346 */, CONST64(0x4E87E09980B9957A) /* 347 */, + CONST64(0x2ACFA1DF50AEE355) /* 348 */, CONST64(0xD898263AFD2FD556) /* 349 */, + CONST64(0xC8F4924DD80C8FD6) /* 350 */, CONST64(0xCF99CA3D754A173A) /* 351 */, + CONST64(0xFE477BACAF91BF3C) /* 352 */, CONST64(0xED5371F6D690C12D) /* 353 */, + CONST64(0x831A5C285E687094) /* 354 */, CONST64(0xC5D3C90A3708A0A4) /* 355 */, + CONST64(0x0F7F903717D06580) /* 356 */, CONST64(0x19F9BB13B8FDF27F) /* 357 */, + CONST64(0xB1BD6F1B4D502843) /* 358 */, CONST64(0x1C761BA38FFF4012) /* 359 */, + CONST64(0x0D1530C4E2E21F3B) /* 360 */, CONST64(0x8943CE69A7372C8A) /* 361 */, + CONST64(0xE5184E11FEB5CE66) /* 362 */, CONST64(0x618BDB80BD736621) /* 363 */, + CONST64(0x7D29BAD68B574D0B) /* 364 */, CONST64(0x81BB613E25E6FE5B) /* 365 */, + CONST64(0x071C9C10BC07913F) /* 366 */, CONST64(0xC7BEEB7909AC2D97) /* 367 */, + CONST64(0xC3E58D353BC5D757) /* 368 */, CONST64(0xEB017892F38F61E8) /* 369 */, + CONST64(0xD4EFFB9C9B1CC21A) /* 370 */, CONST64(0x99727D26F494F7AB) /* 371 */, + CONST64(0xA3E063A2956B3E03) /* 372 */, CONST64(0x9D4A8B9A4AA09C30) /* 373 */, + CONST64(0x3F6AB7D500090FB4) /* 374 */, CONST64(0x9CC0F2A057268AC0) /* 375 */, + CONST64(0x3DEE9D2DEDBF42D1) /* 376 */, CONST64(0x330F49C87960A972) /* 377 */, + CONST64(0xC6B2720287421B41) /* 378 */, CONST64(0x0AC59EC07C00369C) /* 379 */, + CONST64(0xEF4EAC49CB353425) /* 380 */, CONST64(0xF450244EEF0129D8) /* 381 */, + CONST64(0x8ACC46E5CAF4DEB6) /* 382 */, CONST64(0x2FFEAB63989263F7) /* 383 */, + CONST64(0x8F7CB9FE5D7A4578) /* 384 */, CONST64(0x5BD8F7644E634635) /* 385 */, + CONST64(0x427A7315BF2DC900) /* 386 */, CONST64(0x17D0C4AA2125261C) /* 387 */, + CONST64(0x3992486C93518E50) /* 388 */, CONST64(0xB4CBFEE0A2D7D4C3) /* 389 */, + CONST64(0x7C75D6202C5DDD8D) /* 390 */, CONST64(0xDBC295D8E35B6C61) /* 391 */, + CONST64(0x60B369D302032B19) /* 392 */, CONST64(0xCE42685FDCE44132) /* 393 */, + CONST64(0x06F3DDB9DDF65610) /* 394 */, CONST64(0x8EA4D21DB5E148F0) /* 395 */, + CONST64(0x20B0FCE62FCD496F) /* 396 */, CONST64(0x2C1B912358B0EE31) /* 397 */, + CONST64(0xB28317B818F5A308) /* 398 */, CONST64(0xA89C1E189CA6D2CF) /* 399 */, + CONST64(0x0C6B18576AAADBC8) /* 400 */, CONST64(0xB65DEAA91299FAE3) /* 401 */, + CONST64(0xFB2B794B7F1027E7) /* 402 */, CONST64(0x04E4317F443B5BEB) /* 403 */, + CONST64(0x4B852D325939D0A6) /* 404 */, CONST64(0xD5AE6BEEFB207FFC) /* 405 */, + CONST64(0x309682B281C7D374) /* 406 */, CONST64(0xBAE309A194C3B475) /* 407 */, + CONST64(0x8CC3F97B13B49F05) /* 408 */, CONST64(0x98A9422FF8293967) /* 409 */, + CONST64(0x244B16B01076FF7C) /* 410 */, CONST64(0xF8BF571C663D67EE) /* 411 */, + CONST64(0x1F0D6758EEE30DA1) /* 412 */, CONST64(0xC9B611D97ADEB9B7) /* 413 */, + CONST64(0xB7AFD5887B6C57A2) /* 414 */, CONST64(0x6290AE846B984FE1) /* 415 */, + CONST64(0x94DF4CDEACC1A5FD) /* 416 */, CONST64(0x058A5BD1C5483AFF) /* 417 */, + CONST64(0x63166CC142BA3C37) /* 418 */, CONST64(0x8DB8526EB2F76F40) /* 419 */, + CONST64(0xE10880036F0D6D4E) /* 420 */, CONST64(0x9E0523C9971D311D) /* 421 */, + CONST64(0x45EC2824CC7CD691) /* 422 */, CONST64(0x575B8359E62382C9) /* 423 */, + CONST64(0xFA9E400DC4889995) /* 424 */, CONST64(0xD1823ECB45721568) /* 425 */, + CONST64(0xDAFD983B8206082F) /* 426 */, CONST64(0xAA7D29082386A8CB) /* 427 */, + CONST64(0x269FCD4403B87588) /* 428 */, CONST64(0x1B91F5F728BDD1E0) /* 429 */, + CONST64(0xE4669F39040201F6) /* 430 */, CONST64(0x7A1D7C218CF04ADE) /* 431 */, + CONST64(0x65623C29D79CE5CE) /* 432 */, CONST64(0x2368449096C00BB1) /* 433 */, + CONST64(0xAB9BF1879DA503BA) /* 434 */, CONST64(0xBC23ECB1A458058E) /* 435 */, + CONST64(0x9A58DF01BB401ECC) /* 436 */, CONST64(0xA070E868A85F143D) /* 437 */, + CONST64(0x4FF188307DF2239E) /* 438 */, CONST64(0x14D565B41A641183) /* 439 */, + CONST64(0xEE13337452701602) /* 440 */, CONST64(0x950E3DCF3F285E09) /* 441 */, + CONST64(0x59930254B9C80953) /* 442 */, CONST64(0x3BF299408930DA6D) /* 443 */, + CONST64(0xA955943F53691387) /* 444 */, CONST64(0xA15EDECAA9CB8784) /* 445 */, + CONST64(0x29142127352BE9A0) /* 446 */, CONST64(0x76F0371FFF4E7AFB) /* 447 */, + CONST64(0x0239F450274F2228) /* 448 */, CONST64(0xBB073AF01D5E868B) /* 449 */, + CONST64(0xBFC80571C10E96C1) /* 450 */, CONST64(0xD267088568222E23) /* 451 */, + CONST64(0x9671A3D48E80B5B0) /* 452 */, CONST64(0x55B5D38AE193BB81) /* 453 */, + CONST64(0x693AE2D0A18B04B8) /* 454 */, CONST64(0x5C48B4ECADD5335F) /* 455 */, + CONST64(0xFD743B194916A1CA) /* 456 */, CONST64(0x2577018134BE98C4) /* 457 */, + CONST64(0xE77987E83C54A4AD) /* 458 */, CONST64(0x28E11014DA33E1B9) /* 459 */, + CONST64(0x270CC59E226AA213) /* 460 */, CONST64(0x71495F756D1A5F60) /* 461 */, + CONST64(0x9BE853FB60AFEF77) /* 462 */, CONST64(0xADC786A7F7443DBF) /* 463 */, + CONST64(0x0904456173B29A82) /* 464 */, CONST64(0x58BC7A66C232BD5E) /* 465 */, + CONST64(0xF306558C673AC8B2) /* 466 */, CONST64(0x41F639C6B6C9772A) /* 467 */, + CONST64(0x216DEFE99FDA35DA) /* 468 */, CONST64(0x11640CC71C7BE615) /* 469 */, + CONST64(0x93C43694565C5527) /* 470 */, CONST64(0xEA038E6246777839) /* 471 */, + CONST64(0xF9ABF3CE5A3E2469) /* 472 */, CONST64(0x741E768D0FD312D2) /* 473 */, + CONST64(0x0144B883CED652C6) /* 474 */, CONST64(0xC20B5A5BA33F8552) /* 475 */, + CONST64(0x1AE69633C3435A9D) /* 476 */, CONST64(0x97A28CA4088CFDEC) /* 477 */, + CONST64(0x8824A43C1E96F420) /* 478 */, CONST64(0x37612FA66EEEA746) /* 479 */, + CONST64(0x6B4CB165F9CF0E5A) /* 480 */, CONST64(0x43AA1C06A0ABFB4A) /* 481 */, + CONST64(0x7F4DC26FF162796B) /* 482 */, CONST64(0x6CBACC8E54ED9B0F) /* 483 */, + CONST64(0xA6B7FFEFD2BB253E) /* 484 */, CONST64(0x2E25BC95B0A29D4F) /* 485 */, + CONST64(0x86D6A58BDEF1388C) /* 486 */, CONST64(0xDED74AC576B6F054) /* 487 */, + CONST64(0x8030BDBC2B45805D) /* 488 */, CONST64(0x3C81AF70E94D9289) /* 489 */, + CONST64(0x3EFF6DDA9E3100DB) /* 490 */, CONST64(0xB38DC39FDFCC8847) /* 491 */, + CONST64(0x123885528D17B87E) /* 492 */, CONST64(0xF2DA0ED240B1B642) /* 493 */, + CONST64(0x44CEFADCD54BF9A9) /* 494 */, CONST64(0x1312200E433C7EE6) /* 495 */, + CONST64(0x9FFCC84F3A78C748) /* 496 */, CONST64(0xF0CD1F72248576BB) /* 497 */, + CONST64(0xEC6974053638CFE4) /* 498 */, CONST64(0x2BA7B67C0CEC4E4C) /* 499 */, + CONST64(0xAC2F4DF3E5CE32ED) /* 500 */, CONST64(0xCB33D14326EA4C11) /* 501 */, + CONST64(0xA4E9044CC77E58BC) /* 502 */, CONST64(0x5F513293D934FCEF) /* 503 */, + CONST64(0x5DC9645506E55444) /* 504 */, CONST64(0x50DE418F317DE40A) /* 505 */, + CONST64(0x388CB31A69DDE259) /* 506 */, CONST64(0x2DB4A83455820A86) /* 507 */, + CONST64(0x9010A91E84711AE9) /* 508 */, CONST64(0x4DF7F0B7B1498371) /* 509 */, + CONST64(0xD62A2EABC0977179) /* 510 */, CONST64(0x22FAC097AA8D5C0E) /* 511 */, + CONST64(0xF49FCC2FF1DAF39B) /* 512 */, CONST64(0x487FD5C66FF29281) /* 513 */, + CONST64(0xE8A30667FCDCA83F) /* 514 */, CONST64(0x2C9B4BE3D2FCCE63) /* 515 */, + CONST64(0xDA3FF74B93FBBBC2) /* 516 */, CONST64(0x2FA165D2FE70BA66) /* 517 */, + CONST64(0xA103E279970E93D4) /* 518 */, CONST64(0xBECDEC77B0E45E71) /* 519 */, + CONST64(0xCFB41E723985E497) /* 520 */, CONST64(0xB70AAA025EF75017) /* 521 */, + CONST64(0xD42309F03840B8E0) /* 522 */, CONST64(0x8EFC1AD035898579) /* 523 */, + CONST64(0x96C6920BE2B2ABC5) /* 524 */, CONST64(0x66AF4163375A9172) /* 525 */, + CONST64(0x2174ABDCCA7127FB) /* 526 */, CONST64(0xB33CCEA64A72FF41) /* 527 */, + CONST64(0xF04A4933083066A5) /* 528 */, CONST64(0x8D970ACDD7289AF5) /* 529 */, + CONST64(0x8F96E8E031C8C25E) /* 530 */, CONST64(0xF3FEC02276875D47) /* 531 */, + CONST64(0xEC7BF310056190DD) /* 532 */, CONST64(0xF5ADB0AEBB0F1491) /* 533 */, + CONST64(0x9B50F8850FD58892) /* 534 */, CONST64(0x4975488358B74DE8) /* 535 */, + CONST64(0xA3354FF691531C61) /* 536 */, CONST64(0x0702BBE481D2C6EE) /* 537 */, + CONST64(0x89FB24057DEDED98) /* 538 */, CONST64(0xAC3075138596E902) /* 539 */, + CONST64(0x1D2D3580172772ED) /* 540 */, CONST64(0xEB738FC28E6BC30D) /* 541 */, + CONST64(0x5854EF8F63044326) /* 542 */, CONST64(0x9E5C52325ADD3BBE) /* 543 */, + CONST64(0x90AA53CF325C4623) /* 544 */, CONST64(0xC1D24D51349DD067) /* 545 */, + CONST64(0x2051CFEEA69EA624) /* 546 */, CONST64(0x13220F0A862E7E4F) /* 547 */, + CONST64(0xCE39399404E04864) /* 548 */, CONST64(0xD9C42CA47086FCB7) /* 549 */, + CONST64(0x685AD2238A03E7CC) /* 550 */, CONST64(0x066484B2AB2FF1DB) /* 551 */, + CONST64(0xFE9D5D70EFBF79EC) /* 552 */, CONST64(0x5B13B9DD9C481854) /* 553 */, + CONST64(0x15F0D475ED1509AD) /* 554 */, CONST64(0x0BEBCD060EC79851) /* 555 */, + CONST64(0xD58C6791183AB7F8) /* 556 */, CONST64(0xD1187C5052F3EEE4) /* 557 */, + CONST64(0xC95D1192E54E82FF) /* 558 */, CONST64(0x86EEA14CB9AC6CA2) /* 559 */, + CONST64(0x3485BEB153677D5D) /* 560 */, CONST64(0xDD191D781F8C492A) /* 561 */, + CONST64(0xF60866BAA784EBF9) /* 562 */, CONST64(0x518F643BA2D08C74) /* 563 */, + CONST64(0x8852E956E1087C22) /* 564 */, CONST64(0xA768CB8DC410AE8D) /* 565 */, + CONST64(0x38047726BFEC8E1A) /* 566 */, CONST64(0xA67738B4CD3B45AA) /* 567 */, + CONST64(0xAD16691CEC0DDE19) /* 568 */, CONST64(0xC6D4319380462E07) /* 569 */, + CONST64(0xC5A5876D0BA61938) /* 570 */, CONST64(0x16B9FA1FA58FD840) /* 571 */, + CONST64(0x188AB1173CA74F18) /* 572 */, CONST64(0xABDA2F98C99C021F) /* 573 */, + CONST64(0x3E0580AB134AE816) /* 574 */, CONST64(0x5F3B05B773645ABB) /* 575 */, + CONST64(0x2501A2BE5575F2F6) /* 576 */, CONST64(0x1B2F74004E7E8BA9) /* 577 */, + CONST64(0x1CD7580371E8D953) /* 578 */, CONST64(0x7F6ED89562764E30) /* 579 */, + CONST64(0xB15926FF596F003D) /* 580 */, CONST64(0x9F65293DA8C5D6B9) /* 581 */, + CONST64(0x6ECEF04DD690F84C) /* 582 */, CONST64(0x4782275FFF33AF88) /* 583 */, + CONST64(0xE41433083F820801) /* 584 */, CONST64(0xFD0DFE409A1AF9B5) /* 585 */, + CONST64(0x4325A3342CDB396B) /* 586 */, CONST64(0x8AE77E62B301B252) /* 587 */, + CONST64(0xC36F9E9F6655615A) /* 588 */, CONST64(0x85455A2D92D32C09) /* 589 */, + CONST64(0xF2C7DEA949477485) /* 590 */, CONST64(0x63CFB4C133A39EBA) /* 591 */, + CONST64(0x83B040CC6EBC5462) /* 592 */, CONST64(0x3B9454C8FDB326B0) /* 593 */, + CONST64(0x56F56A9E87FFD78C) /* 594 */, CONST64(0x2DC2940D99F42BC6) /* 595 */, + CONST64(0x98F7DF096B096E2D) /* 596 */, CONST64(0x19A6E01E3AD852BF) /* 597 */, + CONST64(0x42A99CCBDBD4B40B) /* 598 */, CONST64(0xA59998AF45E9C559) /* 599 */, + CONST64(0x366295E807D93186) /* 600 */, CONST64(0x6B48181BFAA1F773) /* 601 */, + CONST64(0x1FEC57E2157A0A1D) /* 602 */, CONST64(0x4667446AF6201AD5) /* 603 */, + CONST64(0xE615EBCACFB0F075) /* 604 */, CONST64(0xB8F31F4F68290778) /* 605 */, + CONST64(0x22713ED6CE22D11E) /* 606 */, CONST64(0x3057C1A72EC3C93B) /* 607 */, + CONST64(0xCB46ACC37C3F1F2F) /* 608 */, CONST64(0xDBB893FD02AAF50E) /* 609 */, + CONST64(0x331FD92E600B9FCF) /* 610 */, CONST64(0xA498F96148EA3AD6) /* 611 */, + CONST64(0xA8D8426E8B6A83EA) /* 612 */, CONST64(0xA089B274B7735CDC) /* 613 */, + CONST64(0x87F6B3731E524A11) /* 614 */, CONST64(0x118808E5CBC96749) /* 615 */, + CONST64(0x9906E4C7B19BD394) /* 616 */, CONST64(0xAFED7F7E9B24A20C) /* 617 */, + CONST64(0x6509EADEEB3644A7) /* 618 */, CONST64(0x6C1EF1D3E8EF0EDE) /* 619 */, + CONST64(0xB9C97D43E9798FB4) /* 620 */, CONST64(0xA2F2D784740C28A3) /* 621 */, + CONST64(0x7B8496476197566F) /* 622 */, CONST64(0x7A5BE3E6B65F069D) /* 623 */, + CONST64(0xF96330ED78BE6F10) /* 624 */, CONST64(0xEEE60DE77A076A15) /* 625 */, + CONST64(0x2B4BEE4AA08B9BD0) /* 626 */, CONST64(0x6A56A63EC7B8894E) /* 627 */, + CONST64(0x02121359BA34FEF4) /* 628 */, CONST64(0x4CBF99F8283703FC) /* 629 */, + CONST64(0x398071350CAF30C8) /* 630 */, CONST64(0xD0A77A89F017687A) /* 631 */, + CONST64(0xF1C1A9EB9E423569) /* 632 */, CONST64(0x8C7976282DEE8199) /* 633 */, + CONST64(0x5D1737A5DD1F7ABD) /* 634 */, CONST64(0x4F53433C09A9FA80) /* 635 */, + CONST64(0xFA8B0C53DF7CA1D9) /* 636 */, CONST64(0x3FD9DCBC886CCB77) /* 637 */, + CONST64(0xC040917CA91B4720) /* 638 */, CONST64(0x7DD00142F9D1DCDF) /* 639 */, + CONST64(0x8476FC1D4F387B58) /* 640 */, CONST64(0x23F8E7C5F3316503) /* 641 */, + CONST64(0x032A2244E7E37339) /* 642 */, CONST64(0x5C87A5D750F5A74B) /* 643 */, + CONST64(0x082B4CC43698992E) /* 644 */, CONST64(0xDF917BECB858F63C) /* 645 */, + CONST64(0x3270B8FC5BF86DDA) /* 646 */, CONST64(0x10AE72BB29B5DD76) /* 647 */, + CONST64(0x576AC94E7700362B) /* 648 */, CONST64(0x1AD112DAC61EFB8F) /* 649 */, + CONST64(0x691BC30EC5FAA427) /* 650 */, CONST64(0xFF246311CC327143) /* 651 */, + CONST64(0x3142368E30E53206) /* 652 */, CONST64(0x71380E31E02CA396) /* 653 */, + CONST64(0x958D5C960AAD76F1) /* 654 */, CONST64(0xF8D6F430C16DA536) /* 655 */, + CONST64(0xC8FFD13F1BE7E1D2) /* 656 */, CONST64(0x7578AE66004DDBE1) /* 657 */, + CONST64(0x05833F01067BE646) /* 658 */, CONST64(0xBB34B5AD3BFE586D) /* 659 */, + CONST64(0x095F34C9A12B97F0) /* 660 */, CONST64(0x247AB64525D60CA8) /* 661 */, + CONST64(0xDCDBC6F3017477D1) /* 662 */, CONST64(0x4A2E14D4DECAD24D) /* 663 */, + CONST64(0xBDB5E6D9BE0A1EEB) /* 664 */, CONST64(0x2A7E70F7794301AB) /* 665 */, + CONST64(0xDEF42D8A270540FD) /* 666 */, CONST64(0x01078EC0A34C22C1) /* 667 */, + CONST64(0xE5DE511AF4C16387) /* 668 */, CONST64(0x7EBB3A52BD9A330A) /* 669 */, + CONST64(0x77697857AA7D6435) /* 670 */, CONST64(0x004E831603AE4C32) /* 671 */, + CONST64(0xE7A21020AD78E312) /* 672 */, CONST64(0x9D41A70C6AB420F2) /* 673 */, + CONST64(0x28E06C18EA1141E6) /* 674 */, CONST64(0xD2B28CBD984F6B28) /* 675 */, + CONST64(0x26B75F6C446E9D83) /* 676 */, CONST64(0xBA47568C4D418D7F) /* 677 */, + CONST64(0xD80BADBFE6183D8E) /* 678 */, CONST64(0x0E206D7F5F166044) /* 679 */, + CONST64(0xE258A43911CBCA3E) /* 680 */, CONST64(0x723A1746B21DC0BC) /* 681 */, + CONST64(0xC7CAA854F5D7CDD3) /* 682 */, CONST64(0x7CAC32883D261D9C) /* 683 */, + CONST64(0x7690C26423BA942C) /* 684 */, CONST64(0x17E55524478042B8) /* 685 */, + CONST64(0xE0BE477656A2389F) /* 686 */, CONST64(0x4D289B5E67AB2DA0) /* 687 */, + CONST64(0x44862B9C8FBBFD31) /* 688 */, CONST64(0xB47CC8049D141365) /* 689 */, + CONST64(0x822C1B362B91C793) /* 690 */, CONST64(0x4EB14655FB13DFD8) /* 691 */, + CONST64(0x1ECBBA0714E2A97B) /* 692 */, CONST64(0x6143459D5CDE5F14) /* 693 */, + CONST64(0x53A8FBF1D5F0AC89) /* 694 */, CONST64(0x97EA04D81C5E5B00) /* 695 */, + CONST64(0x622181A8D4FDB3F3) /* 696 */, CONST64(0xE9BCD341572A1208) /* 697 */, + CONST64(0x1411258643CCE58A) /* 698 */, CONST64(0x9144C5FEA4C6E0A4) /* 699 */, + CONST64(0x0D33D06565CF620F) /* 700 */, CONST64(0x54A48D489F219CA1) /* 701 */, + CONST64(0xC43E5EAC6D63C821) /* 702 */, CONST64(0xA9728B3A72770DAF) /* 703 */, + CONST64(0xD7934E7B20DF87EF) /* 704 */, CONST64(0xE35503B61A3E86E5) /* 705 */, + CONST64(0xCAE321FBC819D504) /* 706 */, CONST64(0x129A50B3AC60BFA6) /* 707 */, + CONST64(0xCD5E68EA7E9FB6C3) /* 708 */, CONST64(0xB01C90199483B1C7) /* 709 */, + CONST64(0x3DE93CD5C295376C) /* 710 */, CONST64(0xAED52EDF2AB9AD13) /* 711 */, + CONST64(0x2E60F512C0A07884) /* 712 */, CONST64(0xBC3D86A3E36210C9) /* 713 */, + CONST64(0x35269D9B163951CE) /* 714 */, CONST64(0x0C7D6E2AD0CDB5FA) /* 715 */, + CONST64(0x59E86297D87F5733) /* 716 */, CONST64(0x298EF221898DB0E7) /* 717 */, + CONST64(0x55000029D1A5AA7E) /* 718 */, CONST64(0x8BC08AE1B5061B45) /* 719 */, + CONST64(0xC2C31C2B6C92703A) /* 720 */, CONST64(0x94CC596BAF25EF42) /* 721 */, + CONST64(0x0A1D73DB22540456) /* 722 */, CONST64(0x04B6A0F9D9C4179A) /* 723 */, + CONST64(0xEFFDAFA2AE3D3C60) /* 724 */, CONST64(0xF7C8075BB49496C4) /* 725 */, + CONST64(0x9CC5C7141D1CD4E3) /* 726 */, CONST64(0x78BD1638218E5534) /* 727 */, + CONST64(0xB2F11568F850246A) /* 728 */, CONST64(0xEDFABCFA9502BC29) /* 729 */, + CONST64(0x796CE5F2DA23051B) /* 730 */, CONST64(0xAAE128B0DC93537C) /* 731 */, + CONST64(0x3A493DA0EE4B29AE) /* 732 */, CONST64(0xB5DF6B2C416895D7) /* 733 */, + CONST64(0xFCABBD25122D7F37) /* 734 */, CONST64(0x70810B58105DC4B1) /* 735 */, + CONST64(0xE10FDD37F7882A90) /* 736 */, CONST64(0x524DCAB5518A3F5C) /* 737 */, + CONST64(0x3C9E85878451255B) /* 738 */, CONST64(0x4029828119BD34E2) /* 739 */, + CONST64(0x74A05B6F5D3CECCB) /* 740 */, CONST64(0xB610021542E13ECA) /* 741 */, + CONST64(0x0FF979D12F59E2AC) /* 742 */, CONST64(0x6037DA27E4F9CC50) /* 743 */, + CONST64(0x5E92975A0DF1847D) /* 744 */, CONST64(0xD66DE190D3E623FE) /* 745 */, + CONST64(0x5032D6B87B568048) /* 746 */, CONST64(0x9A36B7CE8235216E) /* 747 */, + CONST64(0x80272A7A24F64B4A) /* 748 */, CONST64(0x93EFED8B8C6916F7) /* 749 */, + CONST64(0x37DDBFF44CCE1555) /* 750 */, CONST64(0x4B95DB5D4B99BD25) /* 751 */, + CONST64(0x92D3FDA169812FC0) /* 752 */, CONST64(0xFB1A4A9A90660BB6) /* 753 */, + CONST64(0x730C196946A4B9B2) /* 754 */, CONST64(0x81E289AA7F49DA68) /* 755 */, + CONST64(0x64669A0F83B1A05F) /* 756 */, CONST64(0x27B3FF7D9644F48B) /* 757 */, + CONST64(0xCC6B615C8DB675B3) /* 758 */, CONST64(0x674F20B9BCEBBE95) /* 759 */, + CONST64(0x6F31238275655982) /* 760 */, CONST64(0x5AE488713E45CF05) /* 761 */, + CONST64(0xBF619F9954C21157) /* 762 */, CONST64(0xEABAC46040A8EAE9) /* 763 */, + CONST64(0x454C6FE9F2C0C1CD) /* 764 */, CONST64(0x419CF6496412691C) /* 765 */, + CONST64(0xD3DC3BEF265B0F70) /* 766 */, CONST64(0x6D0E60F5C3578A9E) /* 767 */, + CONST64(0x5B0E608526323C55) /* 768 */, CONST64(0x1A46C1A9FA1B59F5) /* 769 */, + CONST64(0xA9E245A17C4C8FFA) /* 770 */, CONST64(0x65CA5159DB2955D7) /* 771 */, + CONST64(0x05DB0A76CE35AFC2) /* 772 */, CONST64(0x81EAC77EA9113D45) /* 773 */, + CONST64(0x528EF88AB6AC0A0D) /* 774 */, CONST64(0xA09EA253597BE3FF) /* 775 */, + CONST64(0x430DDFB3AC48CD56) /* 776 */, CONST64(0xC4B3A67AF45CE46F) /* 777 */, + CONST64(0x4ECECFD8FBE2D05E) /* 778 */, CONST64(0x3EF56F10B39935F0) /* 779 */, + CONST64(0x0B22D6829CD619C6) /* 780 */, CONST64(0x17FD460A74DF2069) /* 781 */, + CONST64(0x6CF8CC8E8510ED40) /* 782 */, CONST64(0xD6C824BF3A6ECAA7) /* 783 */, + CONST64(0x61243D581A817049) /* 784 */, CONST64(0x048BACB6BBC163A2) /* 785 */, + CONST64(0xD9A38AC27D44CC32) /* 786 */, CONST64(0x7FDDFF5BAAF410AB) /* 787 */, + CONST64(0xAD6D495AA804824B) /* 788 */, CONST64(0xE1A6A74F2D8C9F94) /* 789 */, + CONST64(0xD4F7851235DEE8E3) /* 790 */, CONST64(0xFD4B7F886540D893) /* 791 */, + CONST64(0x247C20042AA4BFDA) /* 792 */, CONST64(0x096EA1C517D1327C) /* 793 */, + CONST64(0xD56966B4361A6685) /* 794 */, CONST64(0x277DA5C31221057D) /* 795 */, + CONST64(0x94D59893A43ACFF7) /* 796 */, CONST64(0x64F0C51CCDC02281) /* 797 */, + CONST64(0x3D33BCC4FF6189DB) /* 798 */, CONST64(0xE005CB184CE66AF1) /* 799 */, + CONST64(0xFF5CCD1D1DB99BEA) /* 800 */, CONST64(0xB0B854A7FE42980F) /* 801 */, + CONST64(0x7BD46A6A718D4B9F) /* 802 */, CONST64(0xD10FA8CC22A5FD8C) /* 803 */, + CONST64(0xD31484952BE4BD31) /* 804 */, CONST64(0xC7FA975FCB243847) /* 805 */, + CONST64(0x4886ED1E5846C407) /* 806 */, CONST64(0x28CDDB791EB70B04) /* 807 */, + CONST64(0xC2B00BE2F573417F) /* 808 */, CONST64(0x5C9590452180F877) /* 809 */, + CONST64(0x7A6BDDFFF370EB00) /* 810 */, CONST64(0xCE509E38D6D9D6A4) /* 811 */, + CONST64(0xEBEB0F00647FA702) /* 812 */, CONST64(0x1DCC06CF76606F06) /* 813 */, + CONST64(0xE4D9F28BA286FF0A) /* 814 */, CONST64(0xD85A305DC918C262) /* 815 */, + CONST64(0x475B1D8732225F54) /* 816 */, CONST64(0x2D4FB51668CCB5FE) /* 817 */, + CONST64(0xA679B9D9D72BBA20) /* 818 */, CONST64(0x53841C0D912D43A5) /* 819 */, + CONST64(0x3B7EAA48BF12A4E8) /* 820 */, CONST64(0x781E0E47F22F1DDF) /* 821 */, + CONST64(0xEFF20CE60AB50973) /* 822 */, CONST64(0x20D261D19DFFB742) /* 823 */, + CONST64(0x16A12B03062A2E39) /* 824 */, CONST64(0x1960EB2239650495) /* 825 */, + CONST64(0x251C16FED50EB8B8) /* 826 */, CONST64(0x9AC0C330F826016E) /* 827 */, + CONST64(0xED152665953E7671) /* 828 */, CONST64(0x02D63194A6369570) /* 829 */, + CONST64(0x5074F08394B1C987) /* 830 */, CONST64(0x70BA598C90B25CE1) /* 831 */, + CONST64(0x794A15810B9742F6) /* 832 */, CONST64(0x0D5925E9FCAF8C6C) /* 833 */, + CONST64(0x3067716CD868744E) /* 834 */, CONST64(0x910AB077E8D7731B) /* 835 */, + CONST64(0x6A61BBDB5AC42F61) /* 836 */, CONST64(0x93513EFBF0851567) /* 837 */, + CONST64(0xF494724B9E83E9D5) /* 838 */, CONST64(0xE887E1985C09648D) /* 839 */, + CONST64(0x34B1D3C675370CFD) /* 840 */, CONST64(0xDC35E433BC0D255D) /* 841 */, + CONST64(0xD0AAB84234131BE0) /* 842 */, CONST64(0x08042A50B48B7EAF) /* 843 */, + CONST64(0x9997C4EE44A3AB35) /* 844 */, CONST64(0x829A7B49201799D0) /* 845 */, + CONST64(0x263B8307B7C54441) /* 846 */, CONST64(0x752F95F4FD6A6CA6) /* 847 */, + CONST64(0x927217402C08C6E5) /* 848 */, CONST64(0x2A8AB754A795D9EE) /* 849 */, + CONST64(0xA442F7552F72943D) /* 850 */, CONST64(0x2C31334E19781208) /* 851 */, + CONST64(0x4FA98D7CEAEE6291) /* 852 */, CONST64(0x55C3862F665DB309) /* 853 */, + CONST64(0xBD0610175D53B1F3) /* 854 */, CONST64(0x46FE6CB840413F27) /* 855 */, + CONST64(0x3FE03792DF0CFA59) /* 856 */, CONST64(0xCFE700372EB85E8F) /* 857 */, + CONST64(0xA7BE29E7ADBCE118) /* 858 */, CONST64(0xE544EE5CDE8431DD) /* 859 */, + CONST64(0x8A781B1B41F1873E) /* 860 */, CONST64(0xA5C94C78A0D2F0E7) /* 861 */, + CONST64(0x39412E2877B60728) /* 862 */, CONST64(0xA1265EF3AFC9A62C) /* 863 */, + CONST64(0xBCC2770C6A2506C5) /* 864 */, CONST64(0x3AB66DD5DCE1CE12) /* 865 */, + CONST64(0xE65499D04A675B37) /* 866 */, CONST64(0x7D8F523481BFD216) /* 867 */, + CONST64(0x0F6F64FCEC15F389) /* 868 */, CONST64(0x74EFBE618B5B13C8) /* 869 */, + CONST64(0xACDC82B714273E1D) /* 870 */, CONST64(0xDD40BFE003199D17) /* 871 */, + CONST64(0x37E99257E7E061F8) /* 872 */, CONST64(0xFA52626904775AAA) /* 873 */, + CONST64(0x8BBBF63A463D56F9) /* 874 */, CONST64(0xF0013F1543A26E64) /* 875 */, + CONST64(0xA8307E9F879EC898) /* 876 */, CONST64(0xCC4C27A4150177CC) /* 877 */, + CONST64(0x1B432F2CCA1D3348) /* 878 */, CONST64(0xDE1D1F8F9F6FA013) /* 879 */, + CONST64(0x606602A047A7DDD6) /* 880 */, CONST64(0xD237AB64CC1CB2C7) /* 881 */, + CONST64(0x9B938E7225FCD1D3) /* 882 */, CONST64(0xEC4E03708E0FF476) /* 883 */, + CONST64(0xFEB2FBDA3D03C12D) /* 884 */, CONST64(0xAE0BCED2EE43889A) /* 885 */, + CONST64(0x22CB8923EBFB4F43) /* 886 */, CONST64(0x69360D013CF7396D) /* 887 */, + CONST64(0x855E3602D2D4E022) /* 888 */, CONST64(0x073805BAD01F784C) /* 889 */, + CONST64(0x33E17A133852F546) /* 890 */, CONST64(0xDF4874058AC7B638) /* 891 */, + CONST64(0xBA92B29C678AA14A) /* 892 */, CONST64(0x0CE89FC76CFAADCD) /* 893 */, + CONST64(0x5F9D4E0908339E34) /* 894 */, CONST64(0xF1AFE9291F5923B9) /* 895 */, + CONST64(0x6E3480F60F4A265F) /* 896 */, CONST64(0xEEBF3A2AB29B841C) /* 897 */, + CONST64(0xE21938A88F91B4AD) /* 898 */, CONST64(0x57DFEFF845C6D3C3) /* 899 */, + CONST64(0x2F006B0BF62CAAF2) /* 900 */, CONST64(0x62F479EF6F75EE78) /* 901 */, + CONST64(0x11A55AD41C8916A9) /* 902 */, CONST64(0xF229D29084FED453) /* 903 */, + CONST64(0x42F1C27B16B000E6) /* 904 */, CONST64(0x2B1F76749823C074) /* 905 */, + CONST64(0x4B76ECA3C2745360) /* 906 */, CONST64(0x8C98F463B91691BD) /* 907 */, + CONST64(0x14BCC93CF1ADE66A) /* 908 */, CONST64(0x8885213E6D458397) /* 909 */, + CONST64(0x8E177DF0274D4711) /* 910 */, CONST64(0xB49B73B5503F2951) /* 911 */, + CONST64(0x10168168C3F96B6B) /* 912 */, CONST64(0x0E3D963B63CAB0AE) /* 913 */, + CONST64(0x8DFC4B5655A1DB14) /* 914 */, CONST64(0xF789F1356E14DE5C) /* 915 */, + CONST64(0x683E68AF4E51DAC1) /* 916 */, CONST64(0xC9A84F9D8D4B0FD9) /* 917 */, + CONST64(0x3691E03F52A0F9D1) /* 918 */, CONST64(0x5ED86E46E1878E80) /* 919 */, + CONST64(0x3C711A0E99D07150) /* 920 */, CONST64(0x5A0865B20C4E9310) /* 921 */, + CONST64(0x56FBFC1FE4F0682E) /* 922 */, CONST64(0xEA8D5DE3105EDF9B) /* 923 */, + CONST64(0x71ABFDB12379187A) /* 924 */, CONST64(0x2EB99DE1BEE77B9C) /* 925 */, + CONST64(0x21ECC0EA33CF4523) /* 926 */, CONST64(0x59A4D7521805C7A1) /* 927 */, + CONST64(0x3896F5EB56AE7C72) /* 928 */, CONST64(0xAA638F3DB18F75DC) /* 929 */, + CONST64(0x9F39358DABE9808E) /* 930 */, CONST64(0xB7DEFA91C00B72AC) /* 931 */, + CONST64(0x6B5541FD62492D92) /* 932 */, CONST64(0x6DC6DEE8F92E4D5B) /* 933 */, + CONST64(0x353F57ABC4BEEA7E) /* 934 */, CONST64(0x735769D6DA5690CE) /* 935 */, + CONST64(0x0A234AA642391484) /* 936 */, CONST64(0xF6F9508028F80D9D) /* 937 */, + CONST64(0xB8E319A27AB3F215) /* 938 */, CONST64(0x31AD9C1151341A4D) /* 939 */, + CONST64(0x773C22A57BEF5805) /* 940 */, CONST64(0x45C7561A07968633) /* 941 */, + CONST64(0xF913DA9E249DBE36) /* 942 */, CONST64(0xDA652D9B78A64C68) /* 943 */, + CONST64(0x4C27A97F3BC334EF) /* 944 */, CONST64(0x76621220E66B17F4) /* 945 */, + CONST64(0x967743899ACD7D0B) /* 946 */, CONST64(0xF3EE5BCAE0ED6782) /* 947 */, + CONST64(0x409F753600C879FC) /* 948 */, CONST64(0x06D09A39B5926DB6) /* 949 */, + CONST64(0x6F83AEB0317AC588) /* 950 */, CONST64(0x01E6CA4A86381F21) /* 951 */, + CONST64(0x66FF3462D19F3025) /* 952 */, CONST64(0x72207C24DDFD3BFB) /* 953 */, + CONST64(0x4AF6B6D3E2ECE2EB) /* 954 */, CONST64(0x9C994DBEC7EA08DE) /* 955 */, + CONST64(0x49ACE597B09A8BC4) /* 956 */, CONST64(0xB38C4766CF0797BA) /* 957 */, + CONST64(0x131B9373C57C2A75) /* 958 */, CONST64(0xB1822CCE61931E58) /* 959 */, + CONST64(0x9D7555B909BA1C0C) /* 960 */, CONST64(0x127FAFDD937D11D2) /* 961 */, + CONST64(0x29DA3BADC66D92E4) /* 962 */, CONST64(0xA2C1D57154C2ECBC) /* 963 */, + CONST64(0x58C5134D82F6FE24) /* 964 */, CONST64(0x1C3AE3515B62274F) /* 965 */, + CONST64(0xE907C82E01CB8126) /* 966 */, CONST64(0xF8ED091913E37FCB) /* 967 */, + CONST64(0x3249D8F9C80046C9) /* 968 */, CONST64(0x80CF9BEDE388FB63) /* 969 */, + CONST64(0x1881539A116CF19E) /* 970 */, CONST64(0x5103F3F76BD52457) /* 971 */, + CONST64(0x15B7E6F5AE47F7A8) /* 972 */, CONST64(0xDBD7C6DED47E9CCF) /* 973 */, + CONST64(0x44E55C410228BB1A) /* 974 */, CONST64(0xB647D4255EDB4E99) /* 975 */, + CONST64(0x5D11882BB8AAFC30) /* 976 */, CONST64(0xF5098BBB29D3212A) /* 977 */, + CONST64(0x8FB5EA14E90296B3) /* 978 */, CONST64(0x677B942157DD025A) /* 979 */, + CONST64(0xFB58E7C0A390ACB5) /* 980 */, CONST64(0x89D3674C83BD4A01) /* 981 */, + CONST64(0x9E2DA4DF4BF3B93B) /* 982 */, CONST64(0xFCC41E328CAB4829) /* 983 */, + CONST64(0x03F38C96BA582C52) /* 984 */, CONST64(0xCAD1BDBD7FD85DB2) /* 985 */, + CONST64(0xBBB442C16082AE83) /* 986 */, CONST64(0xB95FE86BA5DA9AB0) /* 987 */, + CONST64(0xB22E04673771A93F) /* 988 */, CONST64(0x845358C9493152D8) /* 989 */, + CONST64(0xBE2A488697B4541E) /* 990 */, CONST64(0x95A2DC2DD38E6966) /* 991 */, + CONST64(0xC02C11AC923C852B) /* 992 */, CONST64(0x2388B1990DF2A87B) /* 993 */, + CONST64(0x7C8008FA1B4F37BE) /* 994 */, CONST64(0x1F70D0C84D54E503) /* 995 */, + CONST64(0x5490ADEC7ECE57D4) /* 996 */, CONST64(0x002B3C27D9063A3A) /* 997 */, + CONST64(0x7EAEA3848030A2BF) /* 998 */, CONST64(0xC602326DED2003C0) /* 999 */, + CONST64(0x83A7287D69A94086) /* 1000 */, CONST64(0xC57A5FCB30F57A8A) /* 1001 */, + CONST64(0xB56844E479EBE779) /* 1002 */, CONST64(0xA373B40F05DCBCE9) /* 1003 */, + CONST64(0xD71A786E88570EE2) /* 1004 */, CONST64(0x879CBACDBDE8F6A0) /* 1005 */, + CONST64(0x976AD1BCC164A32F) /* 1006 */, CONST64(0xAB21E25E9666D78B) /* 1007 */, + CONST64(0x901063AAE5E5C33C) /* 1008 */, CONST64(0x9818B34448698D90) /* 1009 */, + CONST64(0xE36487AE3E1E8ABB) /* 1010 */, CONST64(0xAFBDF931893BDCB4) /* 1011 */, + CONST64(0x6345A0DC5FBBD519) /* 1012 */, CONST64(0x8628FE269B9465CA) /* 1013 */, + CONST64(0x1E5D01603F9C51EC) /* 1014 */, CONST64(0x4DE44006A15049B7) /* 1015 */, + CONST64(0xBF6C70E5F776CBB1) /* 1016 */, CONST64(0x411218F2EF552BED) /* 1017 */, + CONST64(0xCB0C0708705A36A3) /* 1018 */, CONST64(0xE74D14754F986044) /* 1019 */, + CONST64(0xCD56D9430EA8280E) /* 1020 */, CONST64(0xC12591D7535F5065) /* 1021 */, + CONST64(0xC83223F1720AEF96) /* 1022 */, CONST64(0xC3A0396F7363A51F) /* 1023 */}; + +/* one round of the hash function */ +LTC_INLINE static void tiger_round(ulong64 *a, ulong64 *b, ulong64 *c, ulong64 x, int mul) +{ + ulong64 tmp; + tmp = (*c ^= x); + *a -= t1[LTC_BYTE(tmp, 0)] ^ t2[LTC_BYTE(tmp, 2)] ^ t3[LTC_BYTE(tmp, 4)] ^ t4[LTC_BYTE(tmp, 6)]; + tmp = (*b += t4[LTC_BYTE(tmp, 1)] ^ t3[LTC_BYTE(tmp, 3)] ^ t2[LTC_BYTE(tmp,5)] ^ t1[LTC_BYTE(tmp,7)]); + switch (mul) { + case 5: *b = (tmp << 2) + tmp; break; + case 7: *b = (tmp << 3) - tmp; break; + case 9: *b = (tmp << 3) + tmp; break; + } +} + +/* one complete pass */ +static void s_pass(ulong64 *a, ulong64 *b, ulong64 *c, const ulong64 *x, int mul) +{ + tiger_round(a,b,c,x[0],mul); + tiger_round(b,c,a,x[1],mul); + tiger_round(c,a,b,x[2],mul); + tiger_round(a,b,c,x[3],mul); + tiger_round(b,c,a,x[4],mul); + tiger_round(c,a,b,x[5],mul); + tiger_round(a,b,c,x[6],mul); + tiger_round(b,c,a,x[7],mul); +} + +/* The key mixing schedule */ +static void s_key_schedule(ulong64 *x) +{ + x[0] -= x[7] ^ CONST64(0xA5A5A5A5A5A5A5A5); + x[1] ^= x[0]; + x[2] += x[1]; + x[3] -= x[2] ^ ((~x[1])<<19); + x[4] ^= x[3]; + x[5] += x[4]; + x[6] -= x[5] ^ ((~x[4])>>23); + x[7] ^= x[6]; + x[0] += x[7]; + x[1] -= x[0] ^ ((~x[7])<<19); + x[2] ^= x[1]; + x[3] += x[2]; + x[4] -= x[3] ^ ((~x[2])>>23); + x[5] ^= x[4]; + x[6] += x[5]; + x[7] -= x[6] ^ CONST64(0x0123456789ABCDEF); +} + +#ifdef LTC_CLEAN_STACK +static int ss_tiger_compress(hash_state *md, const unsigned char *buf) +#else +static int s_tiger_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong64 a, b, c, x[8]; + unsigned long i; + + /* load words */ + for (i = 0; i < 8; i++) { + LOAD64L(x[i],&buf[8*i]); + } + a = md->tiger.state[0]; + b = md->tiger.state[1]; + c = md->tiger.state[2]; + + s_pass(&a,&b,&c,x,5); + s_key_schedule(x); + s_pass(&c,&a,&b,x,7); + s_key_schedule(x); + s_pass(&b,&c,&a,x,9); + + /* store state */ + md->tiger.state[0] = a ^ md->tiger.state[0]; + md->tiger.state[1] = b - md->tiger.state[1]; + md->tiger.state[2] = c + md->tiger.state[2]; + + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int s_tiger_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_tiger_compress(md, buf); + burn_stack(sizeof(ulong64) * 11 + sizeof(unsigned long)); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int tiger_init(hash_state *md) +{ + LTC_ARGCHK(md != NULL); + md->tiger.state[0] = CONST64(0x0123456789ABCDEF); + md->tiger.state[1] = CONST64(0xFEDCBA9876543210); + md->tiger.state[2] = CONST64(0xF096A5B4C3B2E187); + md->tiger.curlen = 0; + md->tiger.length = 0; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(tiger_process, s_tiger_compress, tiger, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (24 bytes) + @return CRYPT_OK if successful +*/ +int tiger_done(hash_state * md, unsigned char *out) +{ + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->tiger.curlen >= sizeof(md->tiger.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->tiger.length += md->tiger.curlen * 8; + + /* append the '1' bit */ + md->tiger.buf[md->tiger.curlen++] = (unsigned char)0x01; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. */ + if (md->tiger.curlen > 56) { + while (md->tiger.curlen < 64) { + md->tiger.buf[md->tiger.curlen++] = (unsigned char)0; + } + s_tiger_compress(md, md->tiger.buf); + md->tiger.curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->tiger.curlen < 56) { + md->tiger.buf[md->tiger.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64L(md->tiger.length, md->tiger.buf+56); + s_tiger_compress(md, md->tiger.buf); + + /* copy output */ + STORE64L(md->tiger.state[0], &out[0]); + STORE64L(md->tiger.state[1], &out[8]); + STORE64L(md->tiger.state[2], &out[16]); +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(hash_state)); +#endif + + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int tiger_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + const char *msg; + unsigned char hash[24]; + } tests[] = { + { "", + { 0x32, 0x93, 0xac, 0x63, 0x0c, 0x13, 0xf0, 0x24, + 0x5f, 0x92, 0xbb, 0xb1, 0x76, 0x6e, 0x16, 0x16, + 0x7a, 0x4e, 0x58, 0x49, 0x2d, 0xde, 0x73, 0xf3 } + }, + { "abc", + { 0x2a, 0xab, 0x14, 0x84, 0xe8, 0xc1, 0x58, 0xf2, + 0xbf, 0xb8, 0xc5, 0xff, 0x41, 0xb5, 0x7a, 0x52, + 0x51, 0x29, 0x13, 0x1c, 0x95, 0x7b, 0x5f, 0x93 } + }, + { "Tiger", + { 0xdd, 0x00, 0x23, 0x07, 0x99, 0xf5, 0x00, 0x9f, + 0xec, 0x6d, 0xeb, 0xc8, 0x38, 0xbb, 0x6a, 0x27, + 0xdf, 0x2b, 0x9d, 0x6f, 0x11, 0x0c, 0x79, 0x37 } + }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", + { 0xf7, 0x1c, 0x85, 0x83, 0x90, 0x2a, 0xfb, 0x87, + 0x9e, 0xdf, 0xe6, 0x10, 0xf8, 0x2c, 0x0d, 0x47, + 0x86, 0xa3, 0xa5, 0x34, 0x50, 0x44, 0x86, 0xb5 } + }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", + { 0xc5, 0x40, 0x34, 0xe5, 0xb4, 0x3e, 0xb8, 0x00, + 0x58, 0x48, 0xa7, 0xe0, 0xae, 0x6a, 0xac, 0x76, + 0xe4, 0xff, 0x59, 0x0a, 0xe7, 0x15, 0xfd, 0x25 } + }, + }; + + int i; + unsigned char tmp[24]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + tiger_init(&md); + tiger_process(&md, (unsigned char *)tests[i].msg, (unsigned long)XSTRLEN(tests[i].msg)); + tiger_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "TIGER", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + +#endif + +/* +Hash of "": + 24F0130C63AC9332 16166E76B1BB925F F373DE2D49584E7A +Hash of "abc": + F258C1E88414AB2A 527AB541FFC5B8BF 935F7B951C132951 +Hash of "Tiger": + 9F00F599072300DD 276ABB38C8EB6DEC 37790C116F9D2BDF +Hash of "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-": + 87FB2A9083851CF7 470D2CF810E6DF9E B586445034A5A386 +Hash of "ABCDEFGHIJKLMNOPQRSTUVWXYZ=abcdefghijklmnopqrstuvwxyz+0123456789": + 467DB80863EBCE48 8DF1CD1261655DE9 57896565975F9197 +Hash of "Tiger - A Fast New Hash Function, by Ross Anderson and Eli Biham": + 0C410A042968868A 1671DA5A3FD29A72 5EC1E457D3CDB303 +Hash of "Tiger - A Fast New Hash Function, by Ross Anderson and Eli Biham, proceedings of Fast Software Encryption 3, Cambridge.": + EBF591D5AFA655CE 7F22894FF87F54AC 89C811B6B0DA3193 +Hash of "Tiger - A Fast New Hash Function, by Ross Anderson and Eli Biham, proceedings of Fast Software Encryption 3, Cambridge, 1996.": + 3D9AEB03D1BD1A63 57B2774DFD6D5B24 DD68151D503974FC +Hash of "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-": + 00B83EB4E53440C5 76AC6AAEE0A74858 25FD15E70A59FFE4 +*/ + + + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/whirl/whirl.c b/optee_os/core/lib/libtomcrypt/src/hashes/whirl/whirl.c new file mode 100644 index 0000000..da7db9c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/whirl/whirl.c @@ -0,0 +1,296 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file whirl.c + LTC_WHIRLPOOL (using their new sbox) hash function by Tom St Denis +*/ + +#include "tomcrypt_private.h" + +#ifdef LTC_WHIRLPOOL + +const struct ltc_hash_descriptor whirlpool_desc = +{ + "whirlpool", + 11, + 64, + 64, + + /* OID */ + { 1, 0, 10118, 3, 0, 55 }, + 6, + + &whirlpool_init, + &whirlpool_process, + &whirlpool_done, + &whirlpool_test, + NULL +}; + +/* the sboxes */ +#define LTC_WHIRLTAB_C +#include "whirltab.c" + +/* get a_{i,j} */ +#define GB(a,i,j) ((a[(i) & 7] >> (8 * (j))) & 255) + +/* shortcut macro to perform three functions at once */ +#define theta_pi_gamma(a, i) \ + (SB0(GB(a, i-0, 7)) ^ \ + SB1(GB(a, i-1, 6)) ^ \ + SB2(GB(a, i-2, 5)) ^ \ + SB3(GB(a, i-3, 4)) ^ \ + SB4(GB(a, i-4, 3)) ^ \ + SB5(GB(a, i-5, 2)) ^ \ + SB6(GB(a, i-6, 1)) ^ \ + SB7(GB(a, i-7, 0))) + +#ifdef LTC_CLEAN_STACK +static int ss_whirlpool_compress(hash_state *md, const unsigned char *buf) +#else +static int s_whirlpool_compress(hash_state *md, const unsigned char *buf) +#endif +{ + ulong64 K[2][8], T[3][8]; + int x, y; + + /* load the block/state */ + for (x = 0; x < 8; x++) { + K[0][x] = md->whirlpool.state[x]; + + LOAD64H(T[0][x], buf + (8 * x)); + T[2][x] = T[0][x]; + T[0][x] ^= K[0][x]; + } + + /* do rounds 1..10 */ + for (x = 0; x < 10; x += 2) { + /* odd round */ + /* apply main transform to K[0] into K[1] */ + for (y = 0; y < 8; y++) { + K[1][y] = theta_pi_gamma(K[0], y); + } + /* xor the constant */ + K[1][0] ^= cont[x]; + + /* apply main transform to T[0] into T[1] */ + for (y = 0; y < 8; y++) { + T[1][y] = theta_pi_gamma(T[0], y) ^ K[1][y]; + } + + /* even round */ + /* apply main transform to K[1] into K[0] */ + for (y = 0; y < 8; y++) { + K[0][y] = theta_pi_gamma(K[1], y); + } + /* xor the constant */ + K[0][0] ^= cont[x+1]; + + /* apply main transform to T[1] into T[0] */ + for (y = 0; y < 8; y++) { + T[0][y] = theta_pi_gamma(T[1], y) ^ K[0][y]; + } + } + + /* store state */ + for (x = 0; x < 8; x++) { + md->whirlpool.state[x] ^= T[0][x] ^ T[2][x]; + } + + return CRYPT_OK; +} + + +#ifdef LTC_CLEAN_STACK +static int s_whirlpool_compress(hash_state *md, const unsigned char *buf) +{ + int err; + err = ss_whirlpool_compress(md, buf); + burn_stack((5 * 8 * sizeof(ulong64)) + (2 * sizeof(int))); + return err; +} +#endif + + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +int whirlpool_init(hash_state * md) +{ + LTC_ARGCHK(md != NULL); + zeromem(&md->whirlpool, sizeof(md->whirlpool)); + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +HASH_PROCESS(whirlpool_process, s_whirlpool_compress, whirlpool, 64) + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (64 bytes) + @return CRYPT_OK if successful +*/ +int whirlpool_done(hash_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->whirlpool.curlen >= sizeof(md->whirlpool.buf)) { + return CRYPT_INVALID_ARG; + } + + /* increase the length of the message */ + md->whirlpool.length += md->whirlpool.curlen * 8; + + /* append the '1' bit */ + md->whirlpool.buf[md->whirlpool.curlen++] = (unsigned char)0x80; + + /* if the length is currently above 32 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->whirlpool.curlen > 32) { + while (md->whirlpool.curlen < 64) { + md->whirlpool.buf[md->whirlpool.curlen++] = (unsigned char)0; + } + s_whirlpool_compress(md, md->whirlpool.buf); + md->whirlpool.curlen = 0; + } + + /* pad upto 56 bytes of zeroes (should be 32 but we only support 64-bit lengths) */ + while (md->whirlpool.curlen < 56) { + md->whirlpool.buf[md->whirlpool.curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->whirlpool.length, md->whirlpool.buf+56); + s_whirlpool_compress(md, md->whirlpool.buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE64H(md->whirlpool.state[i], out+(8*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(*md)); +#endif + return CRYPT_OK; +} + +/** + Self-test the hash + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled +*/ +int whirlpool_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + static const struct { + int len; + unsigned char msg[128], hash[64]; + } tests[] = { + + /* NULL Message */ +{ + 0, + { 0x00 }, + { 0x19, 0xFA, 0x61, 0xD7, 0x55, 0x22, 0xA4, 0x66, 0x9B, 0x44, 0xE3, 0x9C, 0x1D, 0x2E, 0x17, 0x26, + 0xC5, 0x30, 0x23, 0x21, 0x30, 0xD4, 0x07, 0xF8, 0x9A, 0xFE, 0xE0, 0x96, 0x49, 0x97, 0xF7, 0xA7, + 0x3E, 0x83, 0xBE, 0x69, 0x8B, 0x28, 0x8F, 0xEB, 0xCF, 0x88, 0xE3, 0xE0, 0x3C, 0x4F, 0x07, 0x57, + 0xEA, 0x89, 0x64, 0xE5, 0x9B, 0x63, 0xD9, 0x37, 0x08, 0xB1, 0x38, 0xCC, 0x42, 0xA6, 0x6E, 0xB3 } +}, + + + /* 448-bits of 0 bits */ +{ + + 56, + { 0x00 }, + { 0x0B, 0x3F, 0x53, 0x78, 0xEB, 0xED, 0x2B, 0xF4, 0xD7, 0xBE, 0x3C, 0xFD, 0x81, 0x8C, 0x1B, 0x03, + 0xB6, 0xBB, 0x03, 0xD3, 0x46, 0x94, 0x8B, 0x04, 0xF4, 0xF4, 0x0C, 0x72, 0x6F, 0x07, 0x58, 0x70, + 0x2A, 0x0F, 0x1E, 0x22, 0x58, 0x80, 0xE3, 0x8D, 0xD5, 0xF6, 0xED, 0x6D, 0xE9, 0xB1, 0xE9, 0x61, + 0xE4, 0x9F, 0xC1, 0x31, 0x8D, 0x7C, 0xB7, 0x48, 0x22, 0xF3, 0xD0, 0xE2, 0xE9, 0xA7, 0xE7, 0xB0 } +}, + + /* 520-bits of 0 bits */ +{ + 65, + { 0x00 }, + { 0x85, 0xE1, 0x24, 0xC4, 0x41, 0x5B, 0xCF, 0x43, 0x19, 0x54, 0x3E, 0x3A, 0x63, 0xFF, 0x57, 0x1D, + 0x09, 0x35, 0x4C, 0xEE, 0xBE, 0xE1, 0xE3, 0x25, 0x30, 0x8C, 0x90, 0x69, 0xF4, 0x3E, 0x2A, 0xE4, + 0xD0, 0xE5, 0x1D, 0x4E, 0xB1, 0xE8, 0x64, 0x28, 0x70, 0x19, 0x4E, 0x95, 0x30, 0xD8, 0xD8, 0xAF, + 0x65, 0x89, 0xD1, 0xBF, 0x69, 0x49, 0xDD, 0xF9, 0x0A, 0x7F, 0x12, 0x08, 0x62, 0x37, 0x95, 0xB9 } +}, + + /* 512-bits, leading set */ +{ + 64, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x10, 0x3E, 0x00, 0x55, 0xA9, 0xB0, 0x90, 0xE1, 0x1C, 0x8F, 0xDD, 0xEB, 0xBA, 0x06, 0xC0, 0x5A, + 0xCE, 0x8B, 0x64, 0xB8, 0x96, 0x12, 0x8F, 0x6E, 0xED, 0x30, 0x71, 0xFC, 0xF3, 0xDC, 0x16, 0x94, + 0x67, 0x78, 0xE0, 0x72, 0x23, 0x23, 0x3F, 0xD1, 0x80, 0xFC, 0x40, 0xCC, 0xDB, 0x84, 0x30, 0xA6, + 0x40, 0xE3, 0x76, 0x34, 0x27, 0x1E, 0x65, 0x5C, 0xA1, 0x67, 0x4E, 0xBF, 0xF5, 0x07, 0xF8, 0xCB } +}, + + /* 512-bits, leading set of second byte */ +{ + 64, + { 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x35, 0x7B, 0x42, 0xEA, 0x79, 0xBC, 0x97, 0x86, 0x97, 0x5A, 0x3C, 0x44, 0x70, 0xAA, 0xB2, 0x3E, + 0x62, 0x29, 0x79, 0x7B, 0xAD, 0xBD, 0x54, 0x36, 0x5B, 0x54, 0x96, 0xE5, 0x5D, 0x9D, 0xD7, 0x9F, + 0xE9, 0x62, 0x4F, 0xB4, 0x22, 0x66, 0x93, 0x0A, 0x62, 0x8E, 0xD4, 0xDB, 0x08, 0xF9, 0xDD, 0x35, + 0xEF, 0x1B, 0xE1, 0x04, 0x53, 0xFC, 0x18, 0xF4, 0x2C, 0x7F, 0x5E, 0x1F, 0x9B, 0xAE, 0x55, 0xE0 } +}, + + /* 512-bits, leading set of last byte */ +{ + 64, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + { 0x8B, 0x39, 0x04, 0xDD, 0x19, 0x81, 0x41, 0x26, 0xFD, 0x02, 0x74, 0xAB, 0x49, 0xC5, 0x97, 0xF6, + 0xD7, 0x75, 0x33, 0x52, 0xA2, 0xDD, 0x91, 0xFD, 0x8F, 0x9F, 0x54, 0x05, 0x4C, 0x54, 0xBF, 0x0F, + 0x06, 0xDB, 0x4F, 0xF7, 0x08, 0xA3, 0xA2, 0x8B, 0xC3, 0x7A, 0x92, 0x1E, 0xEE, 0x11, 0xED, 0x7B, + 0x6A, 0x53, 0x79, 0x32, 0xCC, 0x5E, 0x94, 0xEE, 0x1E, 0xA6, 0x57, 0x60, 0x7E, 0x36, 0xC9, 0xF7 } +}, + +}; + + int i; + unsigned char tmp[64]; + hash_state md; + + for (i = 0; i < (int)(sizeof(tests)/sizeof(tests[0])); i++) { + whirlpool_init(&md); + whirlpool_process(&md, (unsigned char *)tests[i].msg, tests[i].len); + whirlpool_done(&md, tmp); + if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "WHIRLPOOL", i)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; + #endif +} + + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/hashes/whirl/whirltab.c b/optee_os/core/lib/libtomcrypt/src/hashes/whirl/whirltab.c new file mode 100644 index 0000000..42ecae4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/hashes/whirl/whirltab.c @@ -0,0 +1,586 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file whirltab.c + LTC_WHIRLPOOL tables, Tom St Denis +*/ + +#ifdef LTC_WHIRLTAB_C + +static const ulong64 sbox0[] = { +CONST64(0x18186018c07830d8), CONST64(0x23238c2305af4626), CONST64(0xc6c63fc67ef991b8), CONST64(0xe8e887e8136fcdfb), +CONST64(0x878726874ca113cb), CONST64(0xb8b8dab8a9626d11), CONST64(0x0101040108050209), CONST64(0x4f4f214f426e9e0d), +CONST64(0x3636d836adee6c9b), CONST64(0xa6a6a2a6590451ff), CONST64(0xd2d26fd2debdb90c), CONST64(0xf5f5f3f5fb06f70e), +CONST64(0x7979f979ef80f296), CONST64(0x6f6fa16f5fcede30), CONST64(0x91917e91fcef3f6d), CONST64(0x52525552aa07a4f8), +CONST64(0x60609d6027fdc047), CONST64(0xbcbccabc89766535), CONST64(0x9b9b569baccd2b37), CONST64(0x8e8e028e048c018a), +CONST64(0xa3a3b6a371155bd2), CONST64(0x0c0c300c603c186c), CONST64(0x7b7bf17bff8af684), CONST64(0x3535d435b5e16a80), +CONST64(0x1d1d741de8693af5), CONST64(0xe0e0a7e05347ddb3), CONST64(0xd7d77bd7f6acb321), CONST64(0xc2c22fc25eed999c), +CONST64(0x2e2eb82e6d965c43), CONST64(0x4b4b314b627a9629), CONST64(0xfefedffea321e15d), CONST64(0x575741578216aed5), +CONST64(0x15155415a8412abd), CONST64(0x7777c1779fb6eee8), CONST64(0x3737dc37a5eb6e92), CONST64(0xe5e5b3e57b56d79e), +CONST64(0x9f9f469f8cd92313), CONST64(0xf0f0e7f0d317fd23), CONST64(0x4a4a354a6a7f9420), CONST64(0xdada4fda9e95a944), +CONST64(0x58587d58fa25b0a2), CONST64(0xc9c903c906ca8fcf), CONST64(0x2929a429558d527c), CONST64(0x0a0a280a5022145a), +CONST64(0xb1b1feb1e14f7f50), CONST64(0xa0a0baa0691a5dc9), CONST64(0x6b6bb16b7fdad614), CONST64(0x85852e855cab17d9), +CONST64(0xbdbdcebd8173673c), CONST64(0x5d5d695dd234ba8f), CONST64(0x1010401080502090), CONST64(0xf4f4f7f4f303f507), +CONST64(0xcbcb0bcb16c08bdd), CONST64(0x3e3ef83eedc67cd3), CONST64(0x0505140528110a2d), CONST64(0x676781671fe6ce78), +CONST64(0xe4e4b7e47353d597), CONST64(0x27279c2725bb4e02), CONST64(0x4141194132588273), CONST64(0x8b8b168b2c9d0ba7), +CONST64(0xa7a7a6a7510153f6), CONST64(0x7d7de97dcf94fab2), CONST64(0x95956e95dcfb3749), CONST64(0xd8d847d88e9fad56), +CONST64(0xfbfbcbfb8b30eb70), CONST64(0xeeee9fee2371c1cd), CONST64(0x7c7ced7cc791f8bb), CONST64(0x6666856617e3cc71), +CONST64(0xdddd53dda68ea77b), CONST64(0x17175c17b84b2eaf), CONST64(0x4747014702468e45), CONST64(0x9e9e429e84dc211a), +CONST64(0xcaca0fca1ec589d4), CONST64(0x2d2db42d75995a58), CONST64(0xbfbfc6bf9179632e), CONST64(0x07071c07381b0e3f), +CONST64(0xadad8ead012347ac), CONST64(0x5a5a755aea2fb4b0), CONST64(0x838336836cb51bef), CONST64(0x3333cc3385ff66b6), +CONST64(0x636391633ff2c65c), CONST64(0x02020802100a0412), CONST64(0xaaaa92aa39384993), CONST64(0x7171d971afa8e2de), +CONST64(0xc8c807c80ecf8dc6), CONST64(0x19196419c87d32d1), CONST64(0x494939497270923b), CONST64(0xd9d943d9869aaf5f), +CONST64(0xf2f2eff2c31df931), CONST64(0xe3e3abe34b48dba8), CONST64(0x5b5b715be22ab6b9), CONST64(0x88881a8834920dbc), +CONST64(0x9a9a529aa4c8293e), CONST64(0x262698262dbe4c0b), CONST64(0x3232c8328dfa64bf), CONST64(0xb0b0fab0e94a7d59), +CONST64(0xe9e983e91b6acff2), CONST64(0x0f0f3c0f78331e77), CONST64(0xd5d573d5e6a6b733), CONST64(0x80803a8074ba1df4), +CONST64(0xbebec2be997c6127), CONST64(0xcdcd13cd26de87eb), CONST64(0x3434d034bde46889), CONST64(0x48483d487a759032), +CONST64(0xffffdbffab24e354), CONST64(0x7a7af57af78ff48d), CONST64(0x90907a90f4ea3d64), CONST64(0x5f5f615fc23ebe9d), +CONST64(0x202080201da0403d), CONST64(0x6868bd6867d5d00f), CONST64(0x1a1a681ad07234ca), CONST64(0xaeae82ae192c41b7), +CONST64(0xb4b4eab4c95e757d), CONST64(0x54544d549a19a8ce), CONST64(0x93937693ece53b7f), CONST64(0x222288220daa442f), +CONST64(0x64648d6407e9c863), CONST64(0xf1f1e3f1db12ff2a), CONST64(0x7373d173bfa2e6cc), CONST64(0x12124812905a2482), +CONST64(0x40401d403a5d807a), CONST64(0x0808200840281048), CONST64(0xc3c32bc356e89b95), CONST64(0xecec97ec337bc5df), +CONST64(0xdbdb4bdb9690ab4d), CONST64(0xa1a1bea1611f5fc0), CONST64(0x8d8d0e8d1c830791), CONST64(0x3d3df43df5c97ac8), +CONST64(0x97976697ccf1335b), CONST64(0x0000000000000000), CONST64(0xcfcf1bcf36d483f9), CONST64(0x2b2bac2b4587566e), +CONST64(0x7676c57697b3ece1), CONST64(0x8282328264b019e6), CONST64(0xd6d67fd6fea9b128), CONST64(0x1b1b6c1bd87736c3), +CONST64(0xb5b5eeb5c15b7774), CONST64(0xafaf86af112943be), CONST64(0x6a6ab56a77dfd41d), CONST64(0x50505d50ba0da0ea), +CONST64(0x45450945124c8a57), CONST64(0xf3f3ebf3cb18fb38), CONST64(0x3030c0309df060ad), CONST64(0xefef9bef2b74c3c4), +CONST64(0x3f3ffc3fe5c37eda), CONST64(0x55554955921caac7), CONST64(0xa2a2b2a2791059db), CONST64(0xeaea8fea0365c9e9), +CONST64(0x656589650fecca6a), CONST64(0xbabad2bab9686903), CONST64(0x2f2fbc2f65935e4a), CONST64(0xc0c027c04ee79d8e), +CONST64(0xdede5fdebe81a160), CONST64(0x1c1c701ce06c38fc), CONST64(0xfdfdd3fdbb2ee746), CONST64(0x4d4d294d52649a1f), +CONST64(0x92927292e4e03976), CONST64(0x7575c9758fbceafa), CONST64(0x06061806301e0c36), CONST64(0x8a8a128a249809ae), +CONST64(0xb2b2f2b2f940794b), CONST64(0xe6e6bfe66359d185), CONST64(0x0e0e380e70361c7e), CONST64(0x1f1f7c1ff8633ee7), +CONST64(0x6262956237f7c455), CONST64(0xd4d477d4eea3b53a), CONST64(0xa8a89aa829324d81), CONST64(0x96966296c4f43152), +CONST64(0xf9f9c3f99b3aef62), CONST64(0xc5c533c566f697a3), CONST64(0x2525942535b14a10), CONST64(0x59597959f220b2ab), +CONST64(0x84842a8454ae15d0), CONST64(0x7272d572b7a7e4c5), CONST64(0x3939e439d5dd72ec), CONST64(0x4c4c2d4c5a619816), +CONST64(0x5e5e655eca3bbc94), CONST64(0x7878fd78e785f09f), CONST64(0x3838e038ddd870e5), CONST64(0x8c8c0a8c14860598), +CONST64(0xd1d163d1c6b2bf17), CONST64(0xa5a5aea5410b57e4), CONST64(0xe2e2afe2434dd9a1), CONST64(0x616199612ff8c24e), +CONST64(0xb3b3f6b3f1457b42), CONST64(0x2121842115a54234), CONST64(0x9c9c4a9c94d62508), CONST64(0x1e1e781ef0663cee), +CONST64(0x4343114322528661), CONST64(0xc7c73bc776fc93b1), CONST64(0xfcfcd7fcb32be54f), CONST64(0x0404100420140824), +CONST64(0x51515951b208a2e3), CONST64(0x99995e99bcc72f25), CONST64(0x6d6da96d4fc4da22), CONST64(0x0d0d340d68391a65), +CONST64(0xfafacffa8335e979), CONST64(0xdfdf5bdfb684a369), CONST64(0x7e7ee57ed79bfca9), CONST64(0x242490243db44819), +CONST64(0x3b3bec3bc5d776fe), CONST64(0xabab96ab313d4b9a), CONST64(0xcece1fce3ed181f0), CONST64(0x1111441188552299), +CONST64(0x8f8f068f0c890383), CONST64(0x4e4e254e4a6b9c04), CONST64(0xb7b7e6b7d1517366), CONST64(0xebeb8beb0b60cbe0), +CONST64(0x3c3cf03cfdcc78c1), CONST64(0x81813e817cbf1ffd), CONST64(0x94946a94d4fe3540), CONST64(0xf7f7fbf7eb0cf31c), +CONST64(0xb9b9deb9a1676f18), CONST64(0x13134c13985f268b), CONST64(0x2c2cb02c7d9c5851), CONST64(0xd3d36bd3d6b8bb05), +CONST64(0xe7e7bbe76b5cd38c), CONST64(0x6e6ea56e57cbdc39), CONST64(0xc4c437c46ef395aa), CONST64(0x03030c03180f061b), +CONST64(0x565645568a13acdc), CONST64(0x44440d441a49885e), CONST64(0x7f7fe17fdf9efea0), CONST64(0xa9a99ea921374f88), +CONST64(0x2a2aa82a4d825467), CONST64(0xbbbbd6bbb16d6b0a), CONST64(0xc1c123c146e29f87), CONST64(0x53535153a202a6f1), +CONST64(0xdcdc57dcae8ba572), CONST64(0x0b0b2c0b58271653), CONST64(0x9d9d4e9d9cd32701), CONST64(0x6c6cad6c47c1d82b), +CONST64(0x3131c43195f562a4), CONST64(0x7474cd7487b9e8f3), CONST64(0xf6f6fff6e309f115), CONST64(0x464605460a438c4c), +CONST64(0xacac8aac092645a5), CONST64(0x89891e893c970fb5), CONST64(0x14145014a04428b4), CONST64(0xe1e1a3e15b42dfba), +CONST64(0x16165816b04e2ca6), CONST64(0x3a3ae83acdd274f7), CONST64(0x6969b9696fd0d206), CONST64(0x09092409482d1241), +CONST64(0x7070dd70a7ade0d7), CONST64(0xb6b6e2b6d954716f), CONST64(0xd0d067d0ceb7bd1e), CONST64(0xeded93ed3b7ec7d6), +CONST64(0xcccc17cc2edb85e2), CONST64(0x424215422a578468), CONST64(0x98985a98b4c22d2c), CONST64(0xa4a4aaa4490e55ed), +CONST64(0x2828a0285d885075), CONST64(0x5c5c6d5cda31b886), CONST64(0xf8f8c7f8933fed6b), CONST64(0x8686228644a411c2) +}; + +#ifdef LTC_SMALL_CODE + +#define SB0(x) sbox0[x] +#define SB1(x) ROR64c(sbox0[x], 8) +#define SB2(x) ROR64c(sbox0[x], 16) +#define SB3(x) ROR64c(sbox0[x], 24) +#define SB4(x) ROR64c(sbox0[x], 32) +#define SB5(x) ROR64c(sbox0[x], 40) +#define SB6(x) ROR64c(sbox0[x], 48) +#define SB7(x) ROR64c(sbox0[x], 56) + +#else + +#define SB0(x) sbox0[x] +#define SB1(x) sbox1[x] +#define SB2(x) sbox2[x] +#define SB3(x) sbox3[x] +#define SB4(x) sbox4[x] +#define SB5(x) sbox5[x] +#define SB6(x) sbox6[x] +#define SB7(x) sbox7[x] + + +static const ulong64 sbox1[] = { +CONST64(0xd818186018c07830), CONST64(0x2623238c2305af46), CONST64(0xb8c6c63fc67ef991), CONST64(0xfbe8e887e8136fcd), +CONST64(0xcb878726874ca113), CONST64(0x11b8b8dab8a9626d), CONST64(0x0901010401080502), CONST64(0x0d4f4f214f426e9e), +CONST64(0x9b3636d836adee6c), CONST64(0xffa6a6a2a6590451), CONST64(0x0cd2d26fd2debdb9), CONST64(0x0ef5f5f3f5fb06f7), +CONST64(0x967979f979ef80f2), CONST64(0x306f6fa16f5fcede), CONST64(0x6d91917e91fcef3f), CONST64(0xf852525552aa07a4), +CONST64(0x4760609d6027fdc0), CONST64(0x35bcbccabc897665), CONST64(0x379b9b569baccd2b), CONST64(0x8a8e8e028e048c01), +CONST64(0xd2a3a3b6a371155b), CONST64(0x6c0c0c300c603c18), CONST64(0x847b7bf17bff8af6), CONST64(0x803535d435b5e16a), +CONST64(0xf51d1d741de8693a), CONST64(0xb3e0e0a7e05347dd), CONST64(0x21d7d77bd7f6acb3), CONST64(0x9cc2c22fc25eed99), +CONST64(0x432e2eb82e6d965c), CONST64(0x294b4b314b627a96), CONST64(0x5dfefedffea321e1), CONST64(0xd5575741578216ae), +CONST64(0xbd15155415a8412a), CONST64(0xe87777c1779fb6ee), CONST64(0x923737dc37a5eb6e), CONST64(0x9ee5e5b3e57b56d7), +CONST64(0x139f9f469f8cd923), CONST64(0x23f0f0e7f0d317fd), CONST64(0x204a4a354a6a7f94), CONST64(0x44dada4fda9e95a9), +CONST64(0xa258587d58fa25b0), CONST64(0xcfc9c903c906ca8f), CONST64(0x7c2929a429558d52), CONST64(0x5a0a0a280a502214), +CONST64(0x50b1b1feb1e14f7f), CONST64(0xc9a0a0baa0691a5d), CONST64(0x146b6bb16b7fdad6), CONST64(0xd985852e855cab17), +CONST64(0x3cbdbdcebd817367), CONST64(0x8f5d5d695dd234ba), CONST64(0x9010104010805020), CONST64(0x07f4f4f7f4f303f5), +CONST64(0xddcbcb0bcb16c08b), CONST64(0xd33e3ef83eedc67c), CONST64(0x2d0505140528110a), CONST64(0x78676781671fe6ce), +CONST64(0x97e4e4b7e47353d5), CONST64(0x0227279c2725bb4e), CONST64(0x7341411941325882), CONST64(0xa78b8b168b2c9d0b), +CONST64(0xf6a7a7a6a7510153), CONST64(0xb27d7de97dcf94fa), CONST64(0x4995956e95dcfb37), CONST64(0x56d8d847d88e9fad), +CONST64(0x70fbfbcbfb8b30eb), CONST64(0xcdeeee9fee2371c1), CONST64(0xbb7c7ced7cc791f8), CONST64(0x716666856617e3cc), +CONST64(0x7bdddd53dda68ea7), CONST64(0xaf17175c17b84b2e), CONST64(0x454747014702468e), CONST64(0x1a9e9e429e84dc21), +CONST64(0xd4caca0fca1ec589), CONST64(0x582d2db42d75995a), CONST64(0x2ebfbfc6bf917963), CONST64(0x3f07071c07381b0e), +CONST64(0xacadad8ead012347), CONST64(0xb05a5a755aea2fb4), CONST64(0xef838336836cb51b), CONST64(0xb63333cc3385ff66), +CONST64(0x5c636391633ff2c6), CONST64(0x1202020802100a04), CONST64(0x93aaaa92aa393849), CONST64(0xde7171d971afa8e2), +CONST64(0xc6c8c807c80ecf8d), CONST64(0xd119196419c87d32), CONST64(0x3b49493949727092), CONST64(0x5fd9d943d9869aaf), +CONST64(0x31f2f2eff2c31df9), CONST64(0xa8e3e3abe34b48db), CONST64(0xb95b5b715be22ab6), CONST64(0xbc88881a8834920d), +CONST64(0x3e9a9a529aa4c829), CONST64(0x0b262698262dbe4c), CONST64(0xbf3232c8328dfa64), CONST64(0x59b0b0fab0e94a7d), +CONST64(0xf2e9e983e91b6acf), CONST64(0x770f0f3c0f78331e), CONST64(0x33d5d573d5e6a6b7), CONST64(0xf480803a8074ba1d), +CONST64(0x27bebec2be997c61), CONST64(0xebcdcd13cd26de87), CONST64(0x893434d034bde468), CONST64(0x3248483d487a7590), +CONST64(0x54ffffdbffab24e3), CONST64(0x8d7a7af57af78ff4), CONST64(0x6490907a90f4ea3d), CONST64(0x9d5f5f615fc23ebe), +CONST64(0x3d202080201da040), CONST64(0x0f6868bd6867d5d0), CONST64(0xca1a1a681ad07234), CONST64(0xb7aeae82ae192c41), +CONST64(0x7db4b4eab4c95e75), CONST64(0xce54544d549a19a8), CONST64(0x7f93937693ece53b), CONST64(0x2f222288220daa44), +CONST64(0x6364648d6407e9c8), CONST64(0x2af1f1e3f1db12ff), CONST64(0xcc7373d173bfa2e6), CONST64(0x8212124812905a24), +CONST64(0x7a40401d403a5d80), CONST64(0x4808082008402810), CONST64(0x95c3c32bc356e89b), CONST64(0xdfecec97ec337bc5), +CONST64(0x4ddbdb4bdb9690ab), CONST64(0xc0a1a1bea1611f5f), CONST64(0x918d8d0e8d1c8307), CONST64(0xc83d3df43df5c97a), +CONST64(0x5b97976697ccf133), CONST64(0x0000000000000000), CONST64(0xf9cfcf1bcf36d483), CONST64(0x6e2b2bac2b458756), +CONST64(0xe17676c57697b3ec), CONST64(0xe68282328264b019), CONST64(0x28d6d67fd6fea9b1), CONST64(0xc31b1b6c1bd87736), +CONST64(0x74b5b5eeb5c15b77), CONST64(0xbeafaf86af112943), CONST64(0x1d6a6ab56a77dfd4), CONST64(0xea50505d50ba0da0), +CONST64(0x5745450945124c8a), CONST64(0x38f3f3ebf3cb18fb), CONST64(0xad3030c0309df060), CONST64(0xc4efef9bef2b74c3), +CONST64(0xda3f3ffc3fe5c37e), CONST64(0xc755554955921caa), CONST64(0xdba2a2b2a2791059), CONST64(0xe9eaea8fea0365c9), +CONST64(0x6a656589650fecca), CONST64(0x03babad2bab96869), CONST64(0x4a2f2fbc2f65935e), CONST64(0x8ec0c027c04ee79d), +CONST64(0x60dede5fdebe81a1), CONST64(0xfc1c1c701ce06c38), CONST64(0x46fdfdd3fdbb2ee7), CONST64(0x1f4d4d294d52649a), +CONST64(0x7692927292e4e039), CONST64(0xfa7575c9758fbcea), CONST64(0x3606061806301e0c), CONST64(0xae8a8a128a249809), +CONST64(0x4bb2b2f2b2f94079), CONST64(0x85e6e6bfe66359d1), CONST64(0x7e0e0e380e70361c), CONST64(0xe71f1f7c1ff8633e), +CONST64(0x556262956237f7c4), CONST64(0x3ad4d477d4eea3b5), CONST64(0x81a8a89aa829324d), CONST64(0x5296966296c4f431), +CONST64(0x62f9f9c3f99b3aef), CONST64(0xa3c5c533c566f697), CONST64(0x102525942535b14a), CONST64(0xab59597959f220b2), +CONST64(0xd084842a8454ae15), CONST64(0xc57272d572b7a7e4), CONST64(0xec3939e439d5dd72), CONST64(0x164c4c2d4c5a6198), +CONST64(0x945e5e655eca3bbc), CONST64(0x9f7878fd78e785f0), CONST64(0xe53838e038ddd870), CONST64(0x988c8c0a8c148605), +CONST64(0x17d1d163d1c6b2bf), CONST64(0xe4a5a5aea5410b57), CONST64(0xa1e2e2afe2434dd9), CONST64(0x4e616199612ff8c2), +CONST64(0x42b3b3f6b3f1457b), CONST64(0x342121842115a542), CONST64(0x089c9c4a9c94d625), CONST64(0xee1e1e781ef0663c), +CONST64(0x6143431143225286), CONST64(0xb1c7c73bc776fc93), CONST64(0x4ffcfcd7fcb32be5), CONST64(0x2404041004201408), +CONST64(0xe351515951b208a2), CONST64(0x2599995e99bcc72f), CONST64(0x226d6da96d4fc4da), CONST64(0x650d0d340d68391a), +CONST64(0x79fafacffa8335e9), CONST64(0x69dfdf5bdfb684a3), CONST64(0xa97e7ee57ed79bfc), CONST64(0x19242490243db448), +CONST64(0xfe3b3bec3bc5d776), CONST64(0x9aabab96ab313d4b), CONST64(0xf0cece1fce3ed181), CONST64(0x9911114411885522), +CONST64(0x838f8f068f0c8903), CONST64(0x044e4e254e4a6b9c), CONST64(0x66b7b7e6b7d15173), CONST64(0xe0ebeb8beb0b60cb), +CONST64(0xc13c3cf03cfdcc78), CONST64(0xfd81813e817cbf1f), CONST64(0x4094946a94d4fe35), CONST64(0x1cf7f7fbf7eb0cf3), +CONST64(0x18b9b9deb9a1676f), CONST64(0x8b13134c13985f26), CONST64(0x512c2cb02c7d9c58), CONST64(0x05d3d36bd3d6b8bb), +CONST64(0x8ce7e7bbe76b5cd3), CONST64(0x396e6ea56e57cbdc), CONST64(0xaac4c437c46ef395), CONST64(0x1b03030c03180f06), +CONST64(0xdc565645568a13ac), CONST64(0x5e44440d441a4988), CONST64(0xa07f7fe17fdf9efe), CONST64(0x88a9a99ea921374f), +CONST64(0x672a2aa82a4d8254), CONST64(0x0abbbbd6bbb16d6b), CONST64(0x87c1c123c146e29f), CONST64(0xf153535153a202a6), +CONST64(0x72dcdc57dcae8ba5), CONST64(0x530b0b2c0b582716), CONST64(0x019d9d4e9d9cd327), CONST64(0x2b6c6cad6c47c1d8), +CONST64(0xa43131c43195f562), CONST64(0xf37474cd7487b9e8), CONST64(0x15f6f6fff6e309f1), CONST64(0x4c464605460a438c), +CONST64(0xa5acac8aac092645), CONST64(0xb589891e893c970f), CONST64(0xb414145014a04428), CONST64(0xbae1e1a3e15b42df), +CONST64(0xa616165816b04e2c), CONST64(0xf73a3ae83acdd274), CONST64(0x066969b9696fd0d2), CONST64(0x4109092409482d12), +CONST64(0xd77070dd70a7ade0), CONST64(0x6fb6b6e2b6d95471), CONST64(0x1ed0d067d0ceb7bd), CONST64(0xd6eded93ed3b7ec7), +CONST64(0xe2cccc17cc2edb85), CONST64(0x68424215422a5784), CONST64(0x2c98985a98b4c22d), CONST64(0xeda4a4aaa4490e55), +CONST64(0x752828a0285d8850), CONST64(0x865c5c6d5cda31b8), CONST64(0x6bf8f8c7f8933fed), CONST64(0xc28686228644a411) +}; + +static const ulong64 sbox2[] = { +CONST64(0x30d818186018c078), CONST64(0x462623238c2305af), CONST64(0x91b8c6c63fc67ef9), CONST64(0xcdfbe8e887e8136f), +CONST64(0x13cb878726874ca1), CONST64(0x6d11b8b8dab8a962), CONST64(0x0209010104010805), CONST64(0x9e0d4f4f214f426e), +CONST64(0x6c9b3636d836adee), CONST64(0x51ffa6a6a2a65904), CONST64(0xb90cd2d26fd2debd), CONST64(0xf70ef5f5f3f5fb06), +CONST64(0xf2967979f979ef80), CONST64(0xde306f6fa16f5fce), CONST64(0x3f6d91917e91fcef), CONST64(0xa4f852525552aa07), +CONST64(0xc04760609d6027fd), CONST64(0x6535bcbccabc8976), CONST64(0x2b379b9b569baccd), CONST64(0x018a8e8e028e048c), +CONST64(0x5bd2a3a3b6a37115), CONST64(0x186c0c0c300c603c), CONST64(0xf6847b7bf17bff8a), CONST64(0x6a803535d435b5e1), +CONST64(0x3af51d1d741de869), CONST64(0xddb3e0e0a7e05347), CONST64(0xb321d7d77bd7f6ac), CONST64(0x999cc2c22fc25eed), +CONST64(0x5c432e2eb82e6d96), CONST64(0x96294b4b314b627a), CONST64(0xe15dfefedffea321), CONST64(0xaed5575741578216), +CONST64(0x2abd15155415a841), CONST64(0xeee87777c1779fb6), CONST64(0x6e923737dc37a5eb), CONST64(0xd79ee5e5b3e57b56), +CONST64(0x23139f9f469f8cd9), CONST64(0xfd23f0f0e7f0d317), CONST64(0x94204a4a354a6a7f), CONST64(0xa944dada4fda9e95), +CONST64(0xb0a258587d58fa25), CONST64(0x8fcfc9c903c906ca), CONST64(0x527c2929a429558d), CONST64(0x145a0a0a280a5022), +CONST64(0x7f50b1b1feb1e14f), CONST64(0x5dc9a0a0baa0691a), CONST64(0xd6146b6bb16b7fda), CONST64(0x17d985852e855cab), +CONST64(0x673cbdbdcebd8173), CONST64(0xba8f5d5d695dd234), CONST64(0x2090101040108050), CONST64(0xf507f4f4f7f4f303), +CONST64(0x8bddcbcb0bcb16c0), CONST64(0x7cd33e3ef83eedc6), CONST64(0x0a2d050514052811), CONST64(0xce78676781671fe6), +CONST64(0xd597e4e4b7e47353), CONST64(0x4e0227279c2725bb), CONST64(0x8273414119413258), CONST64(0x0ba78b8b168b2c9d), +CONST64(0x53f6a7a7a6a75101), CONST64(0xfab27d7de97dcf94), CONST64(0x374995956e95dcfb), CONST64(0xad56d8d847d88e9f), +CONST64(0xeb70fbfbcbfb8b30), CONST64(0xc1cdeeee9fee2371), CONST64(0xf8bb7c7ced7cc791), CONST64(0xcc716666856617e3), +CONST64(0xa77bdddd53dda68e), CONST64(0x2eaf17175c17b84b), CONST64(0x8e45474701470246), CONST64(0x211a9e9e429e84dc), +CONST64(0x89d4caca0fca1ec5), CONST64(0x5a582d2db42d7599), CONST64(0x632ebfbfc6bf9179), CONST64(0x0e3f07071c07381b), +CONST64(0x47acadad8ead0123), CONST64(0xb4b05a5a755aea2f), CONST64(0x1bef838336836cb5), CONST64(0x66b63333cc3385ff), +CONST64(0xc65c636391633ff2), CONST64(0x041202020802100a), CONST64(0x4993aaaa92aa3938), CONST64(0xe2de7171d971afa8), +CONST64(0x8dc6c8c807c80ecf), CONST64(0x32d119196419c87d), CONST64(0x923b494939497270), CONST64(0xaf5fd9d943d9869a), +CONST64(0xf931f2f2eff2c31d), CONST64(0xdba8e3e3abe34b48), CONST64(0xb6b95b5b715be22a), CONST64(0x0dbc88881a883492), +CONST64(0x293e9a9a529aa4c8), CONST64(0x4c0b262698262dbe), CONST64(0x64bf3232c8328dfa), CONST64(0x7d59b0b0fab0e94a), +CONST64(0xcff2e9e983e91b6a), CONST64(0x1e770f0f3c0f7833), CONST64(0xb733d5d573d5e6a6), CONST64(0x1df480803a8074ba), +CONST64(0x6127bebec2be997c), CONST64(0x87ebcdcd13cd26de), CONST64(0x68893434d034bde4), CONST64(0x903248483d487a75), +CONST64(0xe354ffffdbffab24), CONST64(0xf48d7a7af57af78f), CONST64(0x3d6490907a90f4ea), CONST64(0xbe9d5f5f615fc23e), +CONST64(0x403d202080201da0), CONST64(0xd00f6868bd6867d5), CONST64(0x34ca1a1a681ad072), CONST64(0x41b7aeae82ae192c), +CONST64(0x757db4b4eab4c95e), CONST64(0xa8ce54544d549a19), CONST64(0x3b7f93937693ece5), CONST64(0x442f222288220daa), +CONST64(0xc86364648d6407e9), CONST64(0xff2af1f1e3f1db12), CONST64(0xe6cc7373d173bfa2), CONST64(0x248212124812905a), +CONST64(0x807a40401d403a5d), CONST64(0x1048080820084028), CONST64(0x9b95c3c32bc356e8), CONST64(0xc5dfecec97ec337b), +CONST64(0xab4ddbdb4bdb9690), CONST64(0x5fc0a1a1bea1611f), CONST64(0x07918d8d0e8d1c83), CONST64(0x7ac83d3df43df5c9), +CONST64(0x335b97976697ccf1), CONST64(0x0000000000000000), CONST64(0x83f9cfcf1bcf36d4), CONST64(0x566e2b2bac2b4587), +CONST64(0xece17676c57697b3), CONST64(0x19e68282328264b0), CONST64(0xb128d6d67fd6fea9), CONST64(0x36c31b1b6c1bd877), +CONST64(0x7774b5b5eeb5c15b), CONST64(0x43beafaf86af1129), CONST64(0xd41d6a6ab56a77df), CONST64(0xa0ea50505d50ba0d), +CONST64(0x8a5745450945124c), CONST64(0xfb38f3f3ebf3cb18), CONST64(0x60ad3030c0309df0), CONST64(0xc3c4efef9bef2b74), +CONST64(0x7eda3f3ffc3fe5c3), CONST64(0xaac755554955921c), CONST64(0x59dba2a2b2a27910), CONST64(0xc9e9eaea8fea0365), +CONST64(0xca6a656589650fec), CONST64(0x6903babad2bab968), CONST64(0x5e4a2f2fbc2f6593), CONST64(0x9d8ec0c027c04ee7), +CONST64(0xa160dede5fdebe81), CONST64(0x38fc1c1c701ce06c), CONST64(0xe746fdfdd3fdbb2e), CONST64(0x9a1f4d4d294d5264), +CONST64(0x397692927292e4e0), CONST64(0xeafa7575c9758fbc), CONST64(0x0c3606061806301e), CONST64(0x09ae8a8a128a2498), +CONST64(0x794bb2b2f2b2f940), CONST64(0xd185e6e6bfe66359), CONST64(0x1c7e0e0e380e7036), CONST64(0x3ee71f1f7c1ff863), +CONST64(0xc4556262956237f7), CONST64(0xb53ad4d477d4eea3), CONST64(0x4d81a8a89aa82932), CONST64(0x315296966296c4f4), +CONST64(0xef62f9f9c3f99b3a), CONST64(0x97a3c5c533c566f6), CONST64(0x4a102525942535b1), CONST64(0xb2ab59597959f220), +CONST64(0x15d084842a8454ae), CONST64(0xe4c57272d572b7a7), CONST64(0x72ec3939e439d5dd), CONST64(0x98164c4c2d4c5a61), +CONST64(0xbc945e5e655eca3b), CONST64(0xf09f7878fd78e785), CONST64(0x70e53838e038ddd8), CONST64(0x05988c8c0a8c1486), +CONST64(0xbf17d1d163d1c6b2), CONST64(0x57e4a5a5aea5410b), CONST64(0xd9a1e2e2afe2434d), CONST64(0xc24e616199612ff8), +CONST64(0x7b42b3b3f6b3f145), CONST64(0x42342121842115a5), CONST64(0x25089c9c4a9c94d6), CONST64(0x3cee1e1e781ef066), +CONST64(0x8661434311432252), CONST64(0x93b1c7c73bc776fc), CONST64(0xe54ffcfcd7fcb32b), CONST64(0x0824040410042014), +CONST64(0xa2e351515951b208), CONST64(0x2f2599995e99bcc7), CONST64(0xda226d6da96d4fc4), CONST64(0x1a650d0d340d6839), +CONST64(0xe979fafacffa8335), CONST64(0xa369dfdf5bdfb684), CONST64(0xfca97e7ee57ed79b), CONST64(0x4819242490243db4), +CONST64(0x76fe3b3bec3bc5d7), CONST64(0x4b9aabab96ab313d), CONST64(0x81f0cece1fce3ed1), CONST64(0x2299111144118855), +CONST64(0x03838f8f068f0c89), CONST64(0x9c044e4e254e4a6b), CONST64(0x7366b7b7e6b7d151), CONST64(0xcbe0ebeb8beb0b60), +CONST64(0x78c13c3cf03cfdcc), CONST64(0x1ffd81813e817cbf), CONST64(0x354094946a94d4fe), CONST64(0xf31cf7f7fbf7eb0c), +CONST64(0x6f18b9b9deb9a167), CONST64(0x268b13134c13985f), CONST64(0x58512c2cb02c7d9c), CONST64(0xbb05d3d36bd3d6b8), +CONST64(0xd38ce7e7bbe76b5c), CONST64(0xdc396e6ea56e57cb), CONST64(0x95aac4c437c46ef3), CONST64(0x061b03030c03180f), +CONST64(0xacdc565645568a13), CONST64(0x885e44440d441a49), CONST64(0xfea07f7fe17fdf9e), CONST64(0x4f88a9a99ea92137), +CONST64(0x54672a2aa82a4d82), CONST64(0x6b0abbbbd6bbb16d), CONST64(0x9f87c1c123c146e2), CONST64(0xa6f153535153a202), +CONST64(0xa572dcdc57dcae8b), CONST64(0x16530b0b2c0b5827), CONST64(0x27019d9d4e9d9cd3), CONST64(0xd82b6c6cad6c47c1), +CONST64(0x62a43131c43195f5), CONST64(0xe8f37474cd7487b9), CONST64(0xf115f6f6fff6e309), CONST64(0x8c4c464605460a43), +CONST64(0x45a5acac8aac0926), CONST64(0x0fb589891e893c97), CONST64(0x28b414145014a044), CONST64(0xdfbae1e1a3e15b42), +CONST64(0x2ca616165816b04e), CONST64(0x74f73a3ae83acdd2), CONST64(0xd2066969b9696fd0), CONST64(0x124109092409482d), +CONST64(0xe0d77070dd70a7ad), CONST64(0x716fb6b6e2b6d954), CONST64(0xbd1ed0d067d0ceb7), CONST64(0xc7d6eded93ed3b7e), +CONST64(0x85e2cccc17cc2edb), CONST64(0x8468424215422a57), CONST64(0x2d2c98985a98b4c2), CONST64(0x55eda4a4aaa4490e), +CONST64(0x50752828a0285d88), CONST64(0xb8865c5c6d5cda31), CONST64(0xed6bf8f8c7f8933f), CONST64(0x11c28686228644a4) +}; + +static const ulong64 sbox3[] = { +CONST64(0x7830d818186018c0), CONST64(0xaf462623238c2305), CONST64(0xf991b8c6c63fc67e), CONST64(0x6fcdfbe8e887e813), +CONST64(0xa113cb878726874c), CONST64(0x626d11b8b8dab8a9), CONST64(0x0502090101040108), CONST64(0x6e9e0d4f4f214f42), +CONST64(0xee6c9b3636d836ad), CONST64(0x0451ffa6a6a2a659), CONST64(0xbdb90cd2d26fd2de), CONST64(0x06f70ef5f5f3f5fb), +CONST64(0x80f2967979f979ef), CONST64(0xcede306f6fa16f5f), CONST64(0xef3f6d91917e91fc), CONST64(0x07a4f852525552aa), +CONST64(0xfdc04760609d6027), CONST64(0x766535bcbccabc89), CONST64(0xcd2b379b9b569bac), CONST64(0x8c018a8e8e028e04), +CONST64(0x155bd2a3a3b6a371), CONST64(0x3c186c0c0c300c60), CONST64(0x8af6847b7bf17bff), CONST64(0xe16a803535d435b5), +CONST64(0x693af51d1d741de8), CONST64(0x47ddb3e0e0a7e053), CONST64(0xacb321d7d77bd7f6), CONST64(0xed999cc2c22fc25e), +CONST64(0x965c432e2eb82e6d), CONST64(0x7a96294b4b314b62), CONST64(0x21e15dfefedffea3), CONST64(0x16aed55757415782), +CONST64(0x412abd15155415a8), CONST64(0xb6eee87777c1779f), CONST64(0xeb6e923737dc37a5), CONST64(0x56d79ee5e5b3e57b), +CONST64(0xd923139f9f469f8c), CONST64(0x17fd23f0f0e7f0d3), CONST64(0x7f94204a4a354a6a), CONST64(0x95a944dada4fda9e), +CONST64(0x25b0a258587d58fa), CONST64(0xca8fcfc9c903c906), CONST64(0x8d527c2929a42955), CONST64(0x22145a0a0a280a50), +CONST64(0x4f7f50b1b1feb1e1), CONST64(0x1a5dc9a0a0baa069), CONST64(0xdad6146b6bb16b7f), CONST64(0xab17d985852e855c), +CONST64(0x73673cbdbdcebd81), CONST64(0x34ba8f5d5d695dd2), CONST64(0x5020901010401080), CONST64(0x03f507f4f4f7f4f3), +CONST64(0xc08bddcbcb0bcb16), CONST64(0xc67cd33e3ef83eed), CONST64(0x110a2d0505140528), CONST64(0xe6ce78676781671f), +CONST64(0x53d597e4e4b7e473), CONST64(0xbb4e0227279c2725), CONST64(0x5882734141194132), CONST64(0x9d0ba78b8b168b2c), +CONST64(0x0153f6a7a7a6a751), CONST64(0x94fab27d7de97dcf), CONST64(0xfb374995956e95dc), CONST64(0x9fad56d8d847d88e), +CONST64(0x30eb70fbfbcbfb8b), CONST64(0x71c1cdeeee9fee23), CONST64(0x91f8bb7c7ced7cc7), CONST64(0xe3cc716666856617), +CONST64(0x8ea77bdddd53dda6), CONST64(0x4b2eaf17175c17b8), CONST64(0x468e454747014702), CONST64(0xdc211a9e9e429e84), +CONST64(0xc589d4caca0fca1e), CONST64(0x995a582d2db42d75), CONST64(0x79632ebfbfc6bf91), CONST64(0x1b0e3f07071c0738), +CONST64(0x2347acadad8ead01), CONST64(0x2fb4b05a5a755aea), CONST64(0xb51bef838336836c), CONST64(0xff66b63333cc3385), +CONST64(0xf2c65c636391633f), CONST64(0x0a04120202080210), CONST64(0x384993aaaa92aa39), CONST64(0xa8e2de7171d971af), +CONST64(0xcf8dc6c8c807c80e), CONST64(0x7d32d119196419c8), CONST64(0x70923b4949394972), CONST64(0x9aaf5fd9d943d986), +CONST64(0x1df931f2f2eff2c3), CONST64(0x48dba8e3e3abe34b), CONST64(0x2ab6b95b5b715be2), CONST64(0x920dbc88881a8834), +CONST64(0xc8293e9a9a529aa4), CONST64(0xbe4c0b262698262d), CONST64(0xfa64bf3232c8328d), CONST64(0x4a7d59b0b0fab0e9), +CONST64(0x6acff2e9e983e91b), CONST64(0x331e770f0f3c0f78), CONST64(0xa6b733d5d573d5e6), CONST64(0xba1df480803a8074), +CONST64(0x7c6127bebec2be99), CONST64(0xde87ebcdcd13cd26), CONST64(0xe468893434d034bd), CONST64(0x75903248483d487a), +CONST64(0x24e354ffffdbffab), CONST64(0x8ff48d7a7af57af7), CONST64(0xea3d6490907a90f4), CONST64(0x3ebe9d5f5f615fc2), +CONST64(0xa0403d202080201d), CONST64(0xd5d00f6868bd6867), CONST64(0x7234ca1a1a681ad0), CONST64(0x2c41b7aeae82ae19), +CONST64(0x5e757db4b4eab4c9), CONST64(0x19a8ce54544d549a), CONST64(0xe53b7f93937693ec), CONST64(0xaa442f222288220d), +CONST64(0xe9c86364648d6407), CONST64(0x12ff2af1f1e3f1db), CONST64(0xa2e6cc7373d173bf), CONST64(0x5a24821212481290), +CONST64(0x5d807a40401d403a), CONST64(0x2810480808200840), CONST64(0xe89b95c3c32bc356), CONST64(0x7bc5dfecec97ec33), +CONST64(0x90ab4ddbdb4bdb96), CONST64(0x1f5fc0a1a1bea161), CONST64(0x8307918d8d0e8d1c), CONST64(0xc97ac83d3df43df5), +CONST64(0xf1335b97976697cc), CONST64(0x0000000000000000), CONST64(0xd483f9cfcf1bcf36), CONST64(0x87566e2b2bac2b45), +CONST64(0xb3ece17676c57697), CONST64(0xb019e68282328264), CONST64(0xa9b128d6d67fd6fe), CONST64(0x7736c31b1b6c1bd8), +CONST64(0x5b7774b5b5eeb5c1), CONST64(0x2943beafaf86af11), CONST64(0xdfd41d6a6ab56a77), CONST64(0x0da0ea50505d50ba), +CONST64(0x4c8a574545094512), CONST64(0x18fb38f3f3ebf3cb), CONST64(0xf060ad3030c0309d), CONST64(0x74c3c4efef9bef2b), +CONST64(0xc37eda3f3ffc3fe5), CONST64(0x1caac75555495592), CONST64(0x1059dba2a2b2a279), CONST64(0x65c9e9eaea8fea03), +CONST64(0xecca6a656589650f), CONST64(0x686903babad2bab9), CONST64(0x935e4a2f2fbc2f65), CONST64(0xe79d8ec0c027c04e), +CONST64(0x81a160dede5fdebe), CONST64(0x6c38fc1c1c701ce0), CONST64(0x2ee746fdfdd3fdbb), CONST64(0x649a1f4d4d294d52), +CONST64(0xe0397692927292e4), CONST64(0xbceafa7575c9758f), CONST64(0x1e0c360606180630), CONST64(0x9809ae8a8a128a24), +CONST64(0x40794bb2b2f2b2f9), CONST64(0x59d185e6e6bfe663), CONST64(0x361c7e0e0e380e70), CONST64(0x633ee71f1f7c1ff8), +CONST64(0xf7c4556262956237), CONST64(0xa3b53ad4d477d4ee), CONST64(0x324d81a8a89aa829), CONST64(0xf4315296966296c4), +CONST64(0x3aef62f9f9c3f99b), CONST64(0xf697a3c5c533c566), CONST64(0xb14a102525942535), CONST64(0x20b2ab59597959f2), +CONST64(0xae15d084842a8454), CONST64(0xa7e4c57272d572b7), CONST64(0xdd72ec3939e439d5), CONST64(0x6198164c4c2d4c5a), +CONST64(0x3bbc945e5e655eca), CONST64(0x85f09f7878fd78e7), CONST64(0xd870e53838e038dd), CONST64(0x8605988c8c0a8c14), +CONST64(0xb2bf17d1d163d1c6), CONST64(0x0b57e4a5a5aea541), CONST64(0x4dd9a1e2e2afe243), CONST64(0xf8c24e616199612f), +CONST64(0x457b42b3b3f6b3f1), CONST64(0xa542342121842115), CONST64(0xd625089c9c4a9c94), CONST64(0x663cee1e1e781ef0), +CONST64(0x5286614343114322), CONST64(0xfc93b1c7c73bc776), CONST64(0x2be54ffcfcd7fcb3), CONST64(0x1408240404100420), +CONST64(0x08a2e351515951b2), CONST64(0xc72f2599995e99bc), CONST64(0xc4da226d6da96d4f), CONST64(0x391a650d0d340d68), +CONST64(0x35e979fafacffa83), CONST64(0x84a369dfdf5bdfb6), CONST64(0x9bfca97e7ee57ed7), CONST64(0xb44819242490243d), +CONST64(0xd776fe3b3bec3bc5), CONST64(0x3d4b9aabab96ab31), CONST64(0xd181f0cece1fce3e), CONST64(0x5522991111441188), +CONST64(0x8903838f8f068f0c), CONST64(0x6b9c044e4e254e4a), CONST64(0x517366b7b7e6b7d1), CONST64(0x60cbe0ebeb8beb0b), +CONST64(0xcc78c13c3cf03cfd), CONST64(0xbf1ffd81813e817c), CONST64(0xfe354094946a94d4), CONST64(0x0cf31cf7f7fbf7eb), +CONST64(0x676f18b9b9deb9a1), CONST64(0x5f268b13134c1398), CONST64(0x9c58512c2cb02c7d), CONST64(0xb8bb05d3d36bd3d6), +CONST64(0x5cd38ce7e7bbe76b), CONST64(0xcbdc396e6ea56e57), CONST64(0xf395aac4c437c46e), CONST64(0x0f061b03030c0318), +CONST64(0x13acdc565645568a), CONST64(0x49885e44440d441a), CONST64(0x9efea07f7fe17fdf), CONST64(0x374f88a9a99ea921), +CONST64(0x8254672a2aa82a4d), CONST64(0x6d6b0abbbbd6bbb1), CONST64(0xe29f87c1c123c146), CONST64(0x02a6f153535153a2), +CONST64(0x8ba572dcdc57dcae), CONST64(0x2716530b0b2c0b58), CONST64(0xd327019d9d4e9d9c), CONST64(0xc1d82b6c6cad6c47), +CONST64(0xf562a43131c43195), CONST64(0xb9e8f37474cd7487), CONST64(0x09f115f6f6fff6e3), CONST64(0x438c4c464605460a), +CONST64(0x2645a5acac8aac09), CONST64(0x970fb589891e893c), CONST64(0x4428b414145014a0), CONST64(0x42dfbae1e1a3e15b), +CONST64(0x4e2ca616165816b0), CONST64(0xd274f73a3ae83acd), CONST64(0xd0d2066969b9696f), CONST64(0x2d12410909240948), +CONST64(0xade0d77070dd70a7), CONST64(0x54716fb6b6e2b6d9), CONST64(0xb7bd1ed0d067d0ce), CONST64(0x7ec7d6eded93ed3b), +CONST64(0xdb85e2cccc17cc2e), CONST64(0x578468424215422a), CONST64(0xc22d2c98985a98b4), CONST64(0x0e55eda4a4aaa449), +CONST64(0x8850752828a0285d), CONST64(0x31b8865c5c6d5cda), CONST64(0x3fed6bf8f8c7f893), CONST64(0xa411c28686228644) +}; + +static const ulong64 sbox4[] = { +CONST64(0xc07830d818186018), CONST64(0x05af462623238c23), CONST64(0x7ef991b8c6c63fc6), CONST64(0x136fcdfbe8e887e8), +CONST64(0x4ca113cb87872687), CONST64(0xa9626d11b8b8dab8), CONST64(0x0805020901010401), CONST64(0x426e9e0d4f4f214f), +CONST64(0xadee6c9b3636d836), CONST64(0x590451ffa6a6a2a6), CONST64(0xdebdb90cd2d26fd2), CONST64(0xfb06f70ef5f5f3f5), +CONST64(0xef80f2967979f979), CONST64(0x5fcede306f6fa16f), CONST64(0xfcef3f6d91917e91), CONST64(0xaa07a4f852525552), +CONST64(0x27fdc04760609d60), CONST64(0x89766535bcbccabc), CONST64(0xaccd2b379b9b569b), CONST64(0x048c018a8e8e028e), +CONST64(0x71155bd2a3a3b6a3), CONST64(0x603c186c0c0c300c), CONST64(0xff8af6847b7bf17b), CONST64(0xb5e16a803535d435), +CONST64(0xe8693af51d1d741d), CONST64(0x5347ddb3e0e0a7e0), CONST64(0xf6acb321d7d77bd7), CONST64(0x5eed999cc2c22fc2), +CONST64(0x6d965c432e2eb82e), CONST64(0x627a96294b4b314b), CONST64(0xa321e15dfefedffe), CONST64(0x8216aed557574157), +CONST64(0xa8412abd15155415), CONST64(0x9fb6eee87777c177), CONST64(0xa5eb6e923737dc37), CONST64(0x7b56d79ee5e5b3e5), +CONST64(0x8cd923139f9f469f), CONST64(0xd317fd23f0f0e7f0), CONST64(0x6a7f94204a4a354a), CONST64(0x9e95a944dada4fda), +CONST64(0xfa25b0a258587d58), CONST64(0x06ca8fcfc9c903c9), CONST64(0x558d527c2929a429), CONST64(0x5022145a0a0a280a), +CONST64(0xe14f7f50b1b1feb1), CONST64(0x691a5dc9a0a0baa0), CONST64(0x7fdad6146b6bb16b), CONST64(0x5cab17d985852e85), +CONST64(0x8173673cbdbdcebd), CONST64(0xd234ba8f5d5d695d), CONST64(0x8050209010104010), CONST64(0xf303f507f4f4f7f4), +CONST64(0x16c08bddcbcb0bcb), CONST64(0xedc67cd33e3ef83e), CONST64(0x28110a2d05051405), CONST64(0x1fe6ce7867678167), +CONST64(0x7353d597e4e4b7e4), CONST64(0x25bb4e0227279c27), CONST64(0x3258827341411941), CONST64(0x2c9d0ba78b8b168b), +CONST64(0x510153f6a7a7a6a7), CONST64(0xcf94fab27d7de97d), CONST64(0xdcfb374995956e95), CONST64(0x8e9fad56d8d847d8), +CONST64(0x8b30eb70fbfbcbfb), CONST64(0x2371c1cdeeee9fee), CONST64(0xc791f8bb7c7ced7c), CONST64(0x17e3cc7166668566), +CONST64(0xa68ea77bdddd53dd), CONST64(0xb84b2eaf17175c17), CONST64(0x02468e4547470147), CONST64(0x84dc211a9e9e429e), +CONST64(0x1ec589d4caca0fca), CONST64(0x75995a582d2db42d), CONST64(0x9179632ebfbfc6bf), CONST64(0x381b0e3f07071c07), +CONST64(0x012347acadad8ead), CONST64(0xea2fb4b05a5a755a), CONST64(0x6cb51bef83833683), CONST64(0x85ff66b63333cc33), +CONST64(0x3ff2c65c63639163), CONST64(0x100a041202020802), CONST64(0x39384993aaaa92aa), CONST64(0xafa8e2de7171d971), +CONST64(0x0ecf8dc6c8c807c8), CONST64(0xc87d32d119196419), CONST64(0x7270923b49493949), CONST64(0x869aaf5fd9d943d9), +CONST64(0xc31df931f2f2eff2), CONST64(0x4b48dba8e3e3abe3), CONST64(0xe22ab6b95b5b715b), CONST64(0x34920dbc88881a88), +CONST64(0xa4c8293e9a9a529a), CONST64(0x2dbe4c0b26269826), CONST64(0x8dfa64bf3232c832), CONST64(0xe94a7d59b0b0fab0), +CONST64(0x1b6acff2e9e983e9), CONST64(0x78331e770f0f3c0f), CONST64(0xe6a6b733d5d573d5), CONST64(0x74ba1df480803a80), +CONST64(0x997c6127bebec2be), CONST64(0x26de87ebcdcd13cd), CONST64(0xbde468893434d034), CONST64(0x7a75903248483d48), +CONST64(0xab24e354ffffdbff), CONST64(0xf78ff48d7a7af57a), CONST64(0xf4ea3d6490907a90), CONST64(0xc23ebe9d5f5f615f), +CONST64(0x1da0403d20208020), CONST64(0x67d5d00f6868bd68), CONST64(0xd07234ca1a1a681a), CONST64(0x192c41b7aeae82ae), +CONST64(0xc95e757db4b4eab4), CONST64(0x9a19a8ce54544d54), CONST64(0xece53b7f93937693), CONST64(0x0daa442f22228822), +CONST64(0x07e9c86364648d64), CONST64(0xdb12ff2af1f1e3f1), CONST64(0xbfa2e6cc7373d173), CONST64(0x905a248212124812), +CONST64(0x3a5d807a40401d40), CONST64(0x4028104808082008), CONST64(0x56e89b95c3c32bc3), CONST64(0x337bc5dfecec97ec), +CONST64(0x9690ab4ddbdb4bdb), CONST64(0x611f5fc0a1a1bea1), CONST64(0x1c8307918d8d0e8d), CONST64(0xf5c97ac83d3df43d), +CONST64(0xccf1335b97976697), CONST64(0x0000000000000000), CONST64(0x36d483f9cfcf1bcf), CONST64(0x4587566e2b2bac2b), +CONST64(0x97b3ece17676c576), CONST64(0x64b019e682823282), CONST64(0xfea9b128d6d67fd6), CONST64(0xd87736c31b1b6c1b), +CONST64(0xc15b7774b5b5eeb5), CONST64(0x112943beafaf86af), CONST64(0x77dfd41d6a6ab56a), CONST64(0xba0da0ea50505d50), +CONST64(0x124c8a5745450945), CONST64(0xcb18fb38f3f3ebf3), CONST64(0x9df060ad3030c030), CONST64(0x2b74c3c4efef9bef), +CONST64(0xe5c37eda3f3ffc3f), CONST64(0x921caac755554955), CONST64(0x791059dba2a2b2a2), CONST64(0x0365c9e9eaea8fea), +CONST64(0x0fecca6a65658965), CONST64(0xb9686903babad2ba), CONST64(0x65935e4a2f2fbc2f), CONST64(0x4ee79d8ec0c027c0), +CONST64(0xbe81a160dede5fde), CONST64(0xe06c38fc1c1c701c), CONST64(0xbb2ee746fdfdd3fd), CONST64(0x52649a1f4d4d294d), +CONST64(0xe4e0397692927292), CONST64(0x8fbceafa7575c975), CONST64(0x301e0c3606061806), CONST64(0x249809ae8a8a128a), +CONST64(0xf940794bb2b2f2b2), CONST64(0x6359d185e6e6bfe6), CONST64(0x70361c7e0e0e380e), CONST64(0xf8633ee71f1f7c1f), +CONST64(0x37f7c45562629562), CONST64(0xeea3b53ad4d477d4), CONST64(0x29324d81a8a89aa8), CONST64(0xc4f4315296966296), +CONST64(0x9b3aef62f9f9c3f9), CONST64(0x66f697a3c5c533c5), CONST64(0x35b14a1025259425), CONST64(0xf220b2ab59597959), +CONST64(0x54ae15d084842a84), CONST64(0xb7a7e4c57272d572), CONST64(0xd5dd72ec3939e439), CONST64(0x5a6198164c4c2d4c), +CONST64(0xca3bbc945e5e655e), CONST64(0xe785f09f7878fd78), CONST64(0xddd870e53838e038), CONST64(0x148605988c8c0a8c), +CONST64(0xc6b2bf17d1d163d1), CONST64(0x410b57e4a5a5aea5), CONST64(0x434dd9a1e2e2afe2), CONST64(0x2ff8c24e61619961), +CONST64(0xf1457b42b3b3f6b3), CONST64(0x15a5423421218421), CONST64(0x94d625089c9c4a9c), CONST64(0xf0663cee1e1e781e), +CONST64(0x2252866143431143), CONST64(0x76fc93b1c7c73bc7), CONST64(0xb32be54ffcfcd7fc), CONST64(0x2014082404041004), +CONST64(0xb208a2e351515951), CONST64(0xbcc72f2599995e99), CONST64(0x4fc4da226d6da96d), CONST64(0x68391a650d0d340d), +CONST64(0x8335e979fafacffa), CONST64(0xb684a369dfdf5bdf), CONST64(0xd79bfca97e7ee57e), CONST64(0x3db4481924249024), +CONST64(0xc5d776fe3b3bec3b), CONST64(0x313d4b9aabab96ab), CONST64(0x3ed181f0cece1fce), CONST64(0x8855229911114411), +CONST64(0x0c8903838f8f068f), CONST64(0x4a6b9c044e4e254e), CONST64(0xd1517366b7b7e6b7), CONST64(0x0b60cbe0ebeb8beb), +CONST64(0xfdcc78c13c3cf03c), CONST64(0x7cbf1ffd81813e81), CONST64(0xd4fe354094946a94), CONST64(0xeb0cf31cf7f7fbf7), +CONST64(0xa1676f18b9b9deb9), CONST64(0x985f268b13134c13), CONST64(0x7d9c58512c2cb02c), CONST64(0xd6b8bb05d3d36bd3), +CONST64(0x6b5cd38ce7e7bbe7), CONST64(0x57cbdc396e6ea56e), CONST64(0x6ef395aac4c437c4), CONST64(0x180f061b03030c03), +CONST64(0x8a13acdc56564556), CONST64(0x1a49885e44440d44), CONST64(0xdf9efea07f7fe17f), CONST64(0x21374f88a9a99ea9), +CONST64(0x4d8254672a2aa82a), CONST64(0xb16d6b0abbbbd6bb), CONST64(0x46e29f87c1c123c1), CONST64(0xa202a6f153535153), +CONST64(0xae8ba572dcdc57dc), CONST64(0x582716530b0b2c0b), CONST64(0x9cd327019d9d4e9d), CONST64(0x47c1d82b6c6cad6c), +CONST64(0x95f562a43131c431), CONST64(0x87b9e8f37474cd74), CONST64(0xe309f115f6f6fff6), CONST64(0x0a438c4c46460546), +CONST64(0x092645a5acac8aac), CONST64(0x3c970fb589891e89), CONST64(0xa04428b414145014), CONST64(0x5b42dfbae1e1a3e1), +CONST64(0xb04e2ca616165816), CONST64(0xcdd274f73a3ae83a), CONST64(0x6fd0d2066969b969), CONST64(0x482d124109092409), +CONST64(0xa7ade0d77070dd70), CONST64(0xd954716fb6b6e2b6), CONST64(0xceb7bd1ed0d067d0), CONST64(0x3b7ec7d6eded93ed), +CONST64(0x2edb85e2cccc17cc), CONST64(0x2a57846842421542), CONST64(0xb4c22d2c98985a98), CONST64(0x490e55eda4a4aaa4), +CONST64(0x5d8850752828a028), CONST64(0xda31b8865c5c6d5c), CONST64(0x933fed6bf8f8c7f8), CONST64(0x44a411c286862286) +}; + +static const ulong64 sbox5[] = { +CONST64(0x18c07830d8181860), CONST64(0x2305af462623238c), CONST64(0xc67ef991b8c6c63f), CONST64(0xe8136fcdfbe8e887), +CONST64(0x874ca113cb878726), CONST64(0xb8a9626d11b8b8da), CONST64(0x0108050209010104), CONST64(0x4f426e9e0d4f4f21), +CONST64(0x36adee6c9b3636d8), CONST64(0xa6590451ffa6a6a2), CONST64(0xd2debdb90cd2d26f), CONST64(0xf5fb06f70ef5f5f3), +CONST64(0x79ef80f2967979f9), CONST64(0x6f5fcede306f6fa1), CONST64(0x91fcef3f6d91917e), CONST64(0x52aa07a4f8525255), +CONST64(0x6027fdc04760609d), CONST64(0xbc89766535bcbcca), CONST64(0x9baccd2b379b9b56), CONST64(0x8e048c018a8e8e02), +CONST64(0xa371155bd2a3a3b6), CONST64(0x0c603c186c0c0c30), CONST64(0x7bff8af6847b7bf1), CONST64(0x35b5e16a803535d4), +CONST64(0x1de8693af51d1d74), CONST64(0xe05347ddb3e0e0a7), CONST64(0xd7f6acb321d7d77b), CONST64(0xc25eed999cc2c22f), +CONST64(0x2e6d965c432e2eb8), CONST64(0x4b627a96294b4b31), CONST64(0xfea321e15dfefedf), CONST64(0x578216aed5575741), +CONST64(0x15a8412abd151554), CONST64(0x779fb6eee87777c1), CONST64(0x37a5eb6e923737dc), CONST64(0xe57b56d79ee5e5b3), +CONST64(0x9f8cd923139f9f46), CONST64(0xf0d317fd23f0f0e7), CONST64(0x4a6a7f94204a4a35), CONST64(0xda9e95a944dada4f), +CONST64(0x58fa25b0a258587d), CONST64(0xc906ca8fcfc9c903), CONST64(0x29558d527c2929a4), CONST64(0x0a5022145a0a0a28), +CONST64(0xb1e14f7f50b1b1fe), CONST64(0xa0691a5dc9a0a0ba), CONST64(0x6b7fdad6146b6bb1), CONST64(0x855cab17d985852e), +CONST64(0xbd8173673cbdbdce), CONST64(0x5dd234ba8f5d5d69), CONST64(0x1080502090101040), CONST64(0xf4f303f507f4f4f7), +CONST64(0xcb16c08bddcbcb0b), CONST64(0x3eedc67cd33e3ef8), CONST64(0x0528110a2d050514), CONST64(0x671fe6ce78676781), +CONST64(0xe47353d597e4e4b7), CONST64(0x2725bb4e0227279c), CONST64(0x4132588273414119), CONST64(0x8b2c9d0ba78b8b16), +CONST64(0xa7510153f6a7a7a6), CONST64(0x7dcf94fab27d7de9), CONST64(0x95dcfb374995956e), CONST64(0xd88e9fad56d8d847), +CONST64(0xfb8b30eb70fbfbcb), CONST64(0xee2371c1cdeeee9f), CONST64(0x7cc791f8bb7c7ced), CONST64(0x6617e3cc71666685), +CONST64(0xdda68ea77bdddd53), CONST64(0x17b84b2eaf17175c), CONST64(0x4702468e45474701), CONST64(0x9e84dc211a9e9e42), +CONST64(0xca1ec589d4caca0f), CONST64(0x2d75995a582d2db4), CONST64(0xbf9179632ebfbfc6), CONST64(0x07381b0e3f07071c), +CONST64(0xad012347acadad8e), CONST64(0x5aea2fb4b05a5a75), CONST64(0x836cb51bef838336), CONST64(0x3385ff66b63333cc), +CONST64(0x633ff2c65c636391), CONST64(0x02100a0412020208), CONST64(0xaa39384993aaaa92), CONST64(0x71afa8e2de7171d9), +CONST64(0xc80ecf8dc6c8c807), CONST64(0x19c87d32d1191964), CONST64(0x497270923b494939), CONST64(0xd9869aaf5fd9d943), +CONST64(0xf2c31df931f2f2ef), CONST64(0xe34b48dba8e3e3ab), CONST64(0x5be22ab6b95b5b71), CONST64(0x8834920dbc88881a), +CONST64(0x9aa4c8293e9a9a52), CONST64(0x262dbe4c0b262698), CONST64(0x328dfa64bf3232c8), CONST64(0xb0e94a7d59b0b0fa), +CONST64(0xe91b6acff2e9e983), CONST64(0x0f78331e770f0f3c), CONST64(0xd5e6a6b733d5d573), CONST64(0x8074ba1df480803a), +CONST64(0xbe997c6127bebec2), CONST64(0xcd26de87ebcdcd13), CONST64(0x34bde468893434d0), CONST64(0x487a75903248483d), +CONST64(0xffab24e354ffffdb), CONST64(0x7af78ff48d7a7af5), CONST64(0x90f4ea3d6490907a), CONST64(0x5fc23ebe9d5f5f61), +CONST64(0x201da0403d202080), CONST64(0x6867d5d00f6868bd), CONST64(0x1ad07234ca1a1a68), CONST64(0xae192c41b7aeae82), +CONST64(0xb4c95e757db4b4ea), CONST64(0x549a19a8ce54544d), CONST64(0x93ece53b7f939376), CONST64(0x220daa442f222288), +CONST64(0x6407e9c86364648d), CONST64(0xf1db12ff2af1f1e3), CONST64(0x73bfa2e6cc7373d1), CONST64(0x12905a2482121248), +CONST64(0x403a5d807a40401d), CONST64(0x0840281048080820), CONST64(0xc356e89b95c3c32b), CONST64(0xec337bc5dfecec97), +CONST64(0xdb9690ab4ddbdb4b), CONST64(0xa1611f5fc0a1a1be), CONST64(0x8d1c8307918d8d0e), CONST64(0x3df5c97ac83d3df4), +CONST64(0x97ccf1335b979766), CONST64(0x0000000000000000), CONST64(0xcf36d483f9cfcf1b), CONST64(0x2b4587566e2b2bac), +CONST64(0x7697b3ece17676c5), CONST64(0x8264b019e6828232), CONST64(0xd6fea9b128d6d67f), CONST64(0x1bd87736c31b1b6c), +CONST64(0xb5c15b7774b5b5ee), CONST64(0xaf112943beafaf86), CONST64(0x6a77dfd41d6a6ab5), CONST64(0x50ba0da0ea50505d), +CONST64(0x45124c8a57454509), CONST64(0xf3cb18fb38f3f3eb), CONST64(0x309df060ad3030c0), CONST64(0xef2b74c3c4efef9b), +CONST64(0x3fe5c37eda3f3ffc), CONST64(0x55921caac7555549), CONST64(0xa2791059dba2a2b2), CONST64(0xea0365c9e9eaea8f), +CONST64(0x650fecca6a656589), CONST64(0xbab9686903babad2), CONST64(0x2f65935e4a2f2fbc), CONST64(0xc04ee79d8ec0c027), +CONST64(0xdebe81a160dede5f), CONST64(0x1ce06c38fc1c1c70), CONST64(0xfdbb2ee746fdfdd3), CONST64(0x4d52649a1f4d4d29), +CONST64(0x92e4e03976929272), CONST64(0x758fbceafa7575c9), CONST64(0x06301e0c36060618), CONST64(0x8a249809ae8a8a12), +CONST64(0xb2f940794bb2b2f2), CONST64(0xe66359d185e6e6bf), CONST64(0x0e70361c7e0e0e38), CONST64(0x1ff8633ee71f1f7c), +CONST64(0x6237f7c455626295), CONST64(0xd4eea3b53ad4d477), CONST64(0xa829324d81a8a89a), CONST64(0x96c4f43152969662), +CONST64(0xf99b3aef62f9f9c3), CONST64(0xc566f697a3c5c533), CONST64(0x2535b14a10252594), CONST64(0x59f220b2ab595979), +CONST64(0x8454ae15d084842a), CONST64(0x72b7a7e4c57272d5), CONST64(0x39d5dd72ec3939e4), CONST64(0x4c5a6198164c4c2d), +CONST64(0x5eca3bbc945e5e65), CONST64(0x78e785f09f7878fd), CONST64(0x38ddd870e53838e0), CONST64(0x8c148605988c8c0a), +CONST64(0xd1c6b2bf17d1d163), CONST64(0xa5410b57e4a5a5ae), CONST64(0xe2434dd9a1e2e2af), CONST64(0x612ff8c24e616199), +CONST64(0xb3f1457b42b3b3f6), CONST64(0x2115a54234212184), CONST64(0x9c94d625089c9c4a), CONST64(0x1ef0663cee1e1e78), +CONST64(0x4322528661434311), CONST64(0xc776fc93b1c7c73b), CONST64(0xfcb32be54ffcfcd7), CONST64(0x0420140824040410), +CONST64(0x51b208a2e3515159), CONST64(0x99bcc72f2599995e), CONST64(0x6d4fc4da226d6da9), CONST64(0x0d68391a650d0d34), +CONST64(0xfa8335e979fafacf), CONST64(0xdfb684a369dfdf5b), CONST64(0x7ed79bfca97e7ee5), CONST64(0x243db44819242490), +CONST64(0x3bc5d776fe3b3bec), CONST64(0xab313d4b9aabab96), CONST64(0xce3ed181f0cece1f), CONST64(0x1188552299111144), +CONST64(0x8f0c8903838f8f06), CONST64(0x4e4a6b9c044e4e25), CONST64(0xb7d1517366b7b7e6), CONST64(0xeb0b60cbe0ebeb8b), +CONST64(0x3cfdcc78c13c3cf0), CONST64(0x817cbf1ffd81813e), CONST64(0x94d4fe354094946a), CONST64(0xf7eb0cf31cf7f7fb), +CONST64(0xb9a1676f18b9b9de), CONST64(0x13985f268b13134c), CONST64(0x2c7d9c58512c2cb0), CONST64(0xd3d6b8bb05d3d36b), +CONST64(0xe76b5cd38ce7e7bb), CONST64(0x6e57cbdc396e6ea5), CONST64(0xc46ef395aac4c437), CONST64(0x03180f061b03030c), +CONST64(0x568a13acdc565645), CONST64(0x441a49885e44440d), CONST64(0x7fdf9efea07f7fe1), CONST64(0xa921374f88a9a99e), +CONST64(0x2a4d8254672a2aa8), CONST64(0xbbb16d6b0abbbbd6), CONST64(0xc146e29f87c1c123), CONST64(0x53a202a6f1535351), +CONST64(0xdcae8ba572dcdc57), CONST64(0x0b582716530b0b2c), CONST64(0x9d9cd327019d9d4e), CONST64(0x6c47c1d82b6c6cad), +CONST64(0x3195f562a43131c4), CONST64(0x7487b9e8f37474cd), CONST64(0xf6e309f115f6f6ff), CONST64(0x460a438c4c464605), +CONST64(0xac092645a5acac8a), CONST64(0x893c970fb589891e), CONST64(0x14a04428b4141450), CONST64(0xe15b42dfbae1e1a3), +CONST64(0x16b04e2ca6161658), CONST64(0x3acdd274f73a3ae8), CONST64(0x696fd0d2066969b9), CONST64(0x09482d1241090924), +CONST64(0x70a7ade0d77070dd), CONST64(0xb6d954716fb6b6e2), CONST64(0xd0ceb7bd1ed0d067), CONST64(0xed3b7ec7d6eded93), +CONST64(0xcc2edb85e2cccc17), CONST64(0x422a578468424215), CONST64(0x98b4c22d2c98985a), CONST64(0xa4490e55eda4a4aa), +CONST64(0x285d8850752828a0), CONST64(0x5cda31b8865c5c6d), CONST64(0xf8933fed6bf8f8c7), CONST64(0x8644a411c2868622) +}; + +static const ulong64 sbox6[] = { +CONST64(0x6018c07830d81818), CONST64(0x8c2305af46262323), CONST64(0x3fc67ef991b8c6c6), CONST64(0x87e8136fcdfbe8e8), +CONST64(0x26874ca113cb8787), CONST64(0xdab8a9626d11b8b8), CONST64(0x0401080502090101), CONST64(0x214f426e9e0d4f4f), +CONST64(0xd836adee6c9b3636), CONST64(0xa2a6590451ffa6a6), CONST64(0x6fd2debdb90cd2d2), CONST64(0xf3f5fb06f70ef5f5), +CONST64(0xf979ef80f2967979), CONST64(0xa16f5fcede306f6f), CONST64(0x7e91fcef3f6d9191), CONST64(0x5552aa07a4f85252), +CONST64(0x9d6027fdc0476060), CONST64(0xcabc89766535bcbc), CONST64(0x569baccd2b379b9b), CONST64(0x028e048c018a8e8e), +CONST64(0xb6a371155bd2a3a3), CONST64(0x300c603c186c0c0c), CONST64(0xf17bff8af6847b7b), CONST64(0xd435b5e16a803535), +CONST64(0x741de8693af51d1d), CONST64(0xa7e05347ddb3e0e0), CONST64(0x7bd7f6acb321d7d7), CONST64(0x2fc25eed999cc2c2), +CONST64(0xb82e6d965c432e2e), CONST64(0x314b627a96294b4b), CONST64(0xdffea321e15dfefe), CONST64(0x41578216aed55757), +CONST64(0x5415a8412abd1515), CONST64(0xc1779fb6eee87777), CONST64(0xdc37a5eb6e923737), CONST64(0xb3e57b56d79ee5e5), +CONST64(0x469f8cd923139f9f), CONST64(0xe7f0d317fd23f0f0), CONST64(0x354a6a7f94204a4a), CONST64(0x4fda9e95a944dada), +CONST64(0x7d58fa25b0a25858), CONST64(0x03c906ca8fcfc9c9), CONST64(0xa429558d527c2929), CONST64(0x280a5022145a0a0a), +CONST64(0xfeb1e14f7f50b1b1), CONST64(0xbaa0691a5dc9a0a0), CONST64(0xb16b7fdad6146b6b), CONST64(0x2e855cab17d98585), +CONST64(0xcebd8173673cbdbd), CONST64(0x695dd234ba8f5d5d), CONST64(0x4010805020901010), CONST64(0xf7f4f303f507f4f4), +CONST64(0x0bcb16c08bddcbcb), CONST64(0xf83eedc67cd33e3e), CONST64(0x140528110a2d0505), CONST64(0x81671fe6ce786767), +CONST64(0xb7e47353d597e4e4), CONST64(0x9c2725bb4e022727), CONST64(0x1941325882734141), CONST64(0x168b2c9d0ba78b8b), +CONST64(0xa6a7510153f6a7a7), CONST64(0xe97dcf94fab27d7d), CONST64(0x6e95dcfb37499595), CONST64(0x47d88e9fad56d8d8), +CONST64(0xcbfb8b30eb70fbfb), CONST64(0x9fee2371c1cdeeee), CONST64(0xed7cc791f8bb7c7c), CONST64(0x856617e3cc716666), +CONST64(0x53dda68ea77bdddd), CONST64(0x5c17b84b2eaf1717), CONST64(0x014702468e454747), CONST64(0x429e84dc211a9e9e), +CONST64(0x0fca1ec589d4caca), CONST64(0xb42d75995a582d2d), CONST64(0xc6bf9179632ebfbf), CONST64(0x1c07381b0e3f0707), +CONST64(0x8ead012347acadad), CONST64(0x755aea2fb4b05a5a), CONST64(0x36836cb51bef8383), CONST64(0xcc3385ff66b63333), +CONST64(0x91633ff2c65c6363), CONST64(0x0802100a04120202), CONST64(0x92aa39384993aaaa), CONST64(0xd971afa8e2de7171), +CONST64(0x07c80ecf8dc6c8c8), CONST64(0x6419c87d32d11919), CONST64(0x39497270923b4949), CONST64(0x43d9869aaf5fd9d9), +CONST64(0xeff2c31df931f2f2), CONST64(0xabe34b48dba8e3e3), CONST64(0x715be22ab6b95b5b), CONST64(0x1a8834920dbc8888), +CONST64(0x529aa4c8293e9a9a), CONST64(0x98262dbe4c0b2626), CONST64(0xc8328dfa64bf3232), CONST64(0xfab0e94a7d59b0b0), +CONST64(0x83e91b6acff2e9e9), CONST64(0x3c0f78331e770f0f), CONST64(0x73d5e6a6b733d5d5), CONST64(0x3a8074ba1df48080), +CONST64(0xc2be997c6127bebe), CONST64(0x13cd26de87ebcdcd), CONST64(0xd034bde468893434), CONST64(0x3d487a7590324848), +CONST64(0xdbffab24e354ffff), CONST64(0xf57af78ff48d7a7a), CONST64(0x7a90f4ea3d649090), CONST64(0x615fc23ebe9d5f5f), +CONST64(0x80201da0403d2020), CONST64(0xbd6867d5d00f6868), CONST64(0x681ad07234ca1a1a), CONST64(0x82ae192c41b7aeae), +CONST64(0xeab4c95e757db4b4), CONST64(0x4d549a19a8ce5454), CONST64(0x7693ece53b7f9393), CONST64(0x88220daa442f2222), +CONST64(0x8d6407e9c8636464), CONST64(0xe3f1db12ff2af1f1), CONST64(0xd173bfa2e6cc7373), CONST64(0x4812905a24821212), +CONST64(0x1d403a5d807a4040), CONST64(0x2008402810480808), CONST64(0x2bc356e89b95c3c3), CONST64(0x97ec337bc5dfecec), +CONST64(0x4bdb9690ab4ddbdb), CONST64(0xbea1611f5fc0a1a1), CONST64(0x0e8d1c8307918d8d), CONST64(0xf43df5c97ac83d3d), +CONST64(0x6697ccf1335b9797), CONST64(0x0000000000000000), CONST64(0x1bcf36d483f9cfcf), CONST64(0xac2b4587566e2b2b), +CONST64(0xc57697b3ece17676), CONST64(0x328264b019e68282), CONST64(0x7fd6fea9b128d6d6), CONST64(0x6c1bd87736c31b1b), +CONST64(0xeeb5c15b7774b5b5), CONST64(0x86af112943beafaf), CONST64(0xb56a77dfd41d6a6a), CONST64(0x5d50ba0da0ea5050), +CONST64(0x0945124c8a574545), CONST64(0xebf3cb18fb38f3f3), CONST64(0xc0309df060ad3030), CONST64(0x9bef2b74c3c4efef), +CONST64(0xfc3fe5c37eda3f3f), CONST64(0x4955921caac75555), CONST64(0xb2a2791059dba2a2), CONST64(0x8fea0365c9e9eaea), +CONST64(0x89650fecca6a6565), CONST64(0xd2bab9686903baba), CONST64(0xbc2f65935e4a2f2f), CONST64(0x27c04ee79d8ec0c0), +CONST64(0x5fdebe81a160dede), CONST64(0x701ce06c38fc1c1c), CONST64(0xd3fdbb2ee746fdfd), CONST64(0x294d52649a1f4d4d), +CONST64(0x7292e4e039769292), CONST64(0xc9758fbceafa7575), CONST64(0x1806301e0c360606), CONST64(0x128a249809ae8a8a), +CONST64(0xf2b2f940794bb2b2), CONST64(0xbfe66359d185e6e6), CONST64(0x380e70361c7e0e0e), CONST64(0x7c1ff8633ee71f1f), +CONST64(0x956237f7c4556262), CONST64(0x77d4eea3b53ad4d4), CONST64(0x9aa829324d81a8a8), CONST64(0x6296c4f431529696), +CONST64(0xc3f99b3aef62f9f9), CONST64(0x33c566f697a3c5c5), CONST64(0x942535b14a102525), CONST64(0x7959f220b2ab5959), +CONST64(0x2a8454ae15d08484), CONST64(0xd572b7a7e4c57272), CONST64(0xe439d5dd72ec3939), CONST64(0x2d4c5a6198164c4c), +CONST64(0x655eca3bbc945e5e), CONST64(0xfd78e785f09f7878), CONST64(0xe038ddd870e53838), CONST64(0x0a8c148605988c8c), +CONST64(0x63d1c6b2bf17d1d1), CONST64(0xaea5410b57e4a5a5), CONST64(0xafe2434dd9a1e2e2), CONST64(0x99612ff8c24e6161), +CONST64(0xf6b3f1457b42b3b3), CONST64(0x842115a542342121), CONST64(0x4a9c94d625089c9c), CONST64(0x781ef0663cee1e1e), +CONST64(0x1143225286614343), CONST64(0x3bc776fc93b1c7c7), CONST64(0xd7fcb32be54ffcfc), CONST64(0x1004201408240404), +CONST64(0x5951b208a2e35151), CONST64(0x5e99bcc72f259999), CONST64(0xa96d4fc4da226d6d), CONST64(0x340d68391a650d0d), +CONST64(0xcffa8335e979fafa), CONST64(0x5bdfb684a369dfdf), CONST64(0xe57ed79bfca97e7e), CONST64(0x90243db448192424), +CONST64(0xec3bc5d776fe3b3b), CONST64(0x96ab313d4b9aabab), CONST64(0x1fce3ed181f0cece), CONST64(0x4411885522991111), +CONST64(0x068f0c8903838f8f), CONST64(0x254e4a6b9c044e4e), CONST64(0xe6b7d1517366b7b7), CONST64(0x8beb0b60cbe0ebeb), +CONST64(0xf03cfdcc78c13c3c), CONST64(0x3e817cbf1ffd8181), CONST64(0x6a94d4fe35409494), CONST64(0xfbf7eb0cf31cf7f7), +CONST64(0xdeb9a1676f18b9b9), CONST64(0x4c13985f268b1313), CONST64(0xb02c7d9c58512c2c), CONST64(0x6bd3d6b8bb05d3d3), +CONST64(0xbbe76b5cd38ce7e7), CONST64(0xa56e57cbdc396e6e), CONST64(0x37c46ef395aac4c4), CONST64(0x0c03180f061b0303), +CONST64(0x45568a13acdc5656), CONST64(0x0d441a49885e4444), CONST64(0xe17fdf9efea07f7f), CONST64(0x9ea921374f88a9a9), +CONST64(0xa82a4d8254672a2a), CONST64(0xd6bbb16d6b0abbbb), CONST64(0x23c146e29f87c1c1), CONST64(0x5153a202a6f15353), +CONST64(0x57dcae8ba572dcdc), CONST64(0x2c0b582716530b0b), CONST64(0x4e9d9cd327019d9d), CONST64(0xad6c47c1d82b6c6c), +CONST64(0xc43195f562a43131), CONST64(0xcd7487b9e8f37474), CONST64(0xfff6e309f115f6f6), CONST64(0x05460a438c4c4646), +CONST64(0x8aac092645a5acac), CONST64(0x1e893c970fb58989), CONST64(0x5014a04428b41414), CONST64(0xa3e15b42dfbae1e1), +CONST64(0x5816b04e2ca61616), CONST64(0xe83acdd274f73a3a), CONST64(0xb9696fd0d2066969), CONST64(0x2409482d12410909), +CONST64(0xdd70a7ade0d77070), CONST64(0xe2b6d954716fb6b6), CONST64(0x67d0ceb7bd1ed0d0), CONST64(0x93ed3b7ec7d6eded), +CONST64(0x17cc2edb85e2cccc), CONST64(0x15422a5784684242), CONST64(0x5a98b4c22d2c9898), CONST64(0xaaa4490e55eda4a4), +CONST64(0xa0285d8850752828), CONST64(0x6d5cda31b8865c5c), CONST64(0xc7f8933fed6bf8f8), CONST64(0x228644a411c28686) +}; + +static const ulong64 sbox7[] = { +CONST64(0x186018c07830d818), CONST64(0x238c2305af462623), CONST64(0xc63fc67ef991b8c6), CONST64(0xe887e8136fcdfbe8), +CONST64(0x8726874ca113cb87), CONST64(0xb8dab8a9626d11b8), CONST64(0x0104010805020901), CONST64(0x4f214f426e9e0d4f), +CONST64(0x36d836adee6c9b36), CONST64(0xa6a2a6590451ffa6), CONST64(0xd26fd2debdb90cd2), CONST64(0xf5f3f5fb06f70ef5), +CONST64(0x79f979ef80f29679), CONST64(0x6fa16f5fcede306f), CONST64(0x917e91fcef3f6d91), CONST64(0x525552aa07a4f852), +CONST64(0x609d6027fdc04760), CONST64(0xbccabc89766535bc), CONST64(0x9b569baccd2b379b), CONST64(0x8e028e048c018a8e), +CONST64(0xa3b6a371155bd2a3), CONST64(0x0c300c603c186c0c), CONST64(0x7bf17bff8af6847b), CONST64(0x35d435b5e16a8035), +CONST64(0x1d741de8693af51d), CONST64(0xe0a7e05347ddb3e0), CONST64(0xd77bd7f6acb321d7), CONST64(0xc22fc25eed999cc2), +CONST64(0x2eb82e6d965c432e), CONST64(0x4b314b627a96294b), CONST64(0xfedffea321e15dfe), CONST64(0x5741578216aed557), +CONST64(0x155415a8412abd15), CONST64(0x77c1779fb6eee877), CONST64(0x37dc37a5eb6e9237), CONST64(0xe5b3e57b56d79ee5), +CONST64(0x9f469f8cd923139f), CONST64(0xf0e7f0d317fd23f0), CONST64(0x4a354a6a7f94204a), CONST64(0xda4fda9e95a944da), +CONST64(0x587d58fa25b0a258), CONST64(0xc903c906ca8fcfc9), CONST64(0x29a429558d527c29), CONST64(0x0a280a5022145a0a), +CONST64(0xb1feb1e14f7f50b1), CONST64(0xa0baa0691a5dc9a0), CONST64(0x6bb16b7fdad6146b), CONST64(0x852e855cab17d985), +CONST64(0xbdcebd8173673cbd), CONST64(0x5d695dd234ba8f5d), CONST64(0x1040108050209010), CONST64(0xf4f7f4f303f507f4), +CONST64(0xcb0bcb16c08bddcb), CONST64(0x3ef83eedc67cd33e), CONST64(0x05140528110a2d05), CONST64(0x6781671fe6ce7867), +CONST64(0xe4b7e47353d597e4), CONST64(0x279c2725bb4e0227), CONST64(0x4119413258827341), CONST64(0x8b168b2c9d0ba78b), +CONST64(0xa7a6a7510153f6a7), CONST64(0x7de97dcf94fab27d), CONST64(0x956e95dcfb374995), CONST64(0xd847d88e9fad56d8), +CONST64(0xfbcbfb8b30eb70fb), CONST64(0xee9fee2371c1cdee), CONST64(0x7ced7cc791f8bb7c), CONST64(0x66856617e3cc7166), +CONST64(0xdd53dda68ea77bdd), CONST64(0x175c17b84b2eaf17), CONST64(0x47014702468e4547), CONST64(0x9e429e84dc211a9e), +CONST64(0xca0fca1ec589d4ca), CONST64(0x2db42d75995a582d), CONST64(0xbfc6bf9179632ebf), CONST64(0x071c07381b0e3f07), +CONST64(0xad8ead012347acad), CONST64(0x5a755aea2fb4b05a), CONST64(0x8336836cb51bef83), CONST64(0x33cc3385ff66b633), +CONST64(0x6391633ff2c65c63), CONST64(0x020802100a041202), CONST64(0xaa92aa39384993aa), CONST64(0x71d971afa8e2de71), +CONST64(0xc807c80ecf8dc6c8), CONST64(0x196419c87d32d119), CONST64(0x4939497270923b49), CONST64(0xd943d9869aaf5fd9), +CONST64(0xf2eff2c31df931f2), CONST64(0xe3abe34b48dba8e3), CONST64(0x5b715be22ab6b95b), CONST64(0x881a8834920dbc88), +CONST64(0x9a529aa4c8293e9a), CONST64(0x2698262dbe4c0b26), CONST64(0x32c8328dfa64bf32), CONST64(0xb0fab0e94a7d59b0), +CONST64(0xe983e91b6acff2e9), CONST64(0x0f3c0f78331e770f), CONST64(0xd573d5e6a6b733d5), CONST64(0x803a8074ba1df480), +CONST64(0xbec2be997c6127be), CONST64(0xcd13cd26de87ebcd), CONST64(0x34d034bde4688934), CONST64(0x483d487a75903248), +CONST64(0xffdbffab24e354ff), CONST64(0x7af57af78ff48d7a), CONST64(0x907a90f4ea3d6490), CONST64(0x5f615fc23ebe9d5f), +CONST64(0x2080201da0403d20), CONST64(0x68bd6867d5d00f68), CONST64(0x1a681ad07234ca1a), CONST64(0xae82ae192c41b7ae), +CONST64(0xb4eab4c95e757db4), CONST64(0x544d549a19a8ce54), CONST64(0x937693ece53b7f93), CONST64(0x2288220daa442f22), +CONST64(0x648d6407e9c86364), CONST64(0xf1e3f1db12ff2af1), CONST64(0x73d173bfa2e6cc73), CONST64(0x124812905a248212), +CONST64(0x401d403a5d807a40), CONST64(0x0820084028104808), CONST64(0xc32bc356e89b95c3), CONST64(0xec97ec337bc5dfec), +CONST64(0xdb4bdb9690ab4ddb), CONST64(0xa1bea1611f5fc0a1), CONST64(0x8d0e8d1c8307918d), CONST64(0x3df43df5c97ac83d), +CONST64(0x976697ccf1335b97), CONST64(0x0000000000000000), CONST64(0xcf1bcf36d483f9cf), CONST64(0x2bac2b4587566e2b), +CONST64(0x76c57697b3ece176), CONST64(0x82328264b019e682), CONST64(0xd67fd6fea9b128d6), CONST64(0x1b6c1bd87736c31b), +CONST64(0xb5eeb5c15b7774b5), CONST64(0xaf86af112943beaf), CONST64(0x6ab56a77dfd41d6a), CONST64(0x505d50ba0da0ea50), +CONST64(0x450945124c8a5745), CONST64(0xf3ebf3cb18fb38f3), CONST64(0x30c0309df060ad30), CONST64(0xef9bef2b74c3c4ef), +CONST64(0x3ffc3fe5c37eda3f), CONST64(0x554955921caac755), CONST64(0xa2b2a2791059dba2), CONST64(0xea8fea0365c9e9ea), +CONST64(0x6589650fecca6a65), CONST64(0xbad2bab9686903ba), CONST64(0x2fbc2f65935e4a2f), CONST64(0xc027c04ee79d8ec0), +CONST64(0xde5fdebe81a160de), CONST64(0x1c701ce06c38fc1c), CONST64(0xfdd3fdbb2ee746fd), CONST64(0x4d294d52649a1f4d), +CONST64(0x927292e4e0397692), CONST64(0x75c9758fbceafa75), CONST64(0x061806301e0c3606), CONST64(0x8a128a249809ae8a), +CONST64(0xb2f2b2f940794bb2), CONST64(0xe6bfe66359d185e6), CONST64(0x0e380e70361c7e0e), CONST64(0x1f7c1ff8633ee71f), +CONST64(0x62956237f7c45562), CONST64(0xd477d4eea3b53ad4), CONST64(0xa89aa829324d81a8), CONST64(0x966296c4f4315296), +CONST64(0xf9c3f99b3aef62f9), CONST64(0xc533c566f697a3c5), CONST64(0x25942535b14a1025), CONST64(0x597959f220b2ab59), +CONST64(0x842a8454ae15d084), CONST64(0x72d572b7a7e4c572), CONST64(0x39e439d5dd72ec39), CONST64(0x4c2d4c5a6198164c), +CONST64(0x5e655eca3bbc945e), CONST64(0x78fd78e785f09f78), CONST64(0x38e038ddd870e538), CONST64(0x8c0a8c148605988c), +CONST64(0xd163d1c6b2bf17d1), CONST64(0xa5aea5410b57e4a5), CONST64(0xe2afe2434dd9a1e2), CONST64(0x6199612ff8c24e61), +CONST64(0xb3f6b3f1457b42b3), CONST64(0x21842115a5423421), CONST64(0x9c4a9c94d625089c), CONST64(0x1e781ef0663cee1e), +CONST64(0x4311432252866143), CONST64(0xc73bc776fc93b1c7), CONST64(0xfcd7fcb32be54ffc), CONST64(0x0410042014082404), +CONST64(0x515951b208a2e351), CONST64(0x995e99bcc72f2599), CONST64(0x6da96d4fc4da226d), CONST64(0x0d340d68391a650d), +CONST64(0xfacffa8335e979fa), CONST64(0xdf5bdfb684a369df), CONST64(0x7ee57ed79bfca97e), CONST64(0x2490243db4481924), +CONST64(0x3bec3bc5d776fe3b), CONST64(0xab96ab313d4b9aab), CONST64(0xce1fce3ed181f0ce), CONST64(0x1144118855229911), +CONST64(0x8f068f0c8903838f), CONST64(0x4e254e4a6b9c044e), CONST64(0xb7e6b7d1517366b7), CONST64(0xeb8beb0b60cbe0eb), +CONST64(0x3cf03cfdcc78c13c), CONST64(0x813e817cbf1ffd81), CONST64(0x946a94d4fe354094), CONST64(0xf7fbf7eb0cf31cf7), +CONST64(0xb9deb9a1676f18b9), CONST64(0x134c13985f268b13), CONST64(0x2cb02c7d9c58512c), CONST64(0xd36bd3d6b8bb05d3), +CONST64(0xe7bbe76b5cd38ce7), CONST64(0x6ea56e57cbdc396e), CONST64(0xc437c46ef395aac4), CONST64(0x030c03180f061b03), +CONST64(0x5645568a13acdc56), CONST64(0x440d441a49885e44), CONST64(0x7fe17fdf9efea07f), CONST64(0xa99ea921374f88a9), +CONST64(0x2aa82a4d8254672a), CONST64(0xbbd6bbb16d6b0abb), CONST64(0xc123c146e29f87c1), CONST64(0x535153a202a6f153), +CONST64(0xdc57dcae8ba572dc), CONST64(0x0b2c0b582716530b), CONST64(0x9d4e9d9cd327019d), CONST64(0x6cad6c47c1d82b6c), +CONST64(0x31c43195f562a431), CONST64(0x74cd7487b9e8f374), CONST64(0xf6fff6e309f115f6), CONST64(0x4605460a438c4c46), +CONST64(0xac8aac092645a5ac), CONST64(0x891e893c970fb589), CONST64(0x145014a04428b414), CONST64(0xe1a3e15b42dfbae1), +CONST64(0x165816b04e2ca616), CONST64(0x3ae83acdd274f73a), CONST64(0x69b9696fd0d20669), CONST64(0x092409482d124109), +CONST64(0x70dd70a7ade0d770), CONST64(0xb6e2b6d954716fb6), CONST64(0xd067d0ceb7bd1ed0), CONST64(0xed93ed3b7ec7d6ed), +CONST64(0xcc17cc2edb85e2cc), CONST64(0x4215422a57846842), CONST64(0x985a98b4c22d2c98), CONST64(0xa4aaa4490e55eda4), +CONST64(0x28a0285d88507528), CONST64(0x5c6d5cda31b8865c), CONST64(0xf8c7f8933fed6bf8), CONST64(0x86228644a411c286) +}; + +#endif + +static const ulong64 cont[] = { +CONST64(0x1823c6e887b8014f), +CONST64(0x36a6d2f5796f9152), +CONST64(0x60bc9b8ea30c7b35), +CONST64(0x1de0d7c22e4bfe57), +CONST64(0x157737e59ff04ada), +CONST64(0x58c9290ab1a06b85), +CONST64(0xbd5d10f4cb3e0567), +CONST64(0xe427418ba77d95d8), +CONST64(0xfbee7c66dd17479e), +CONST64(0xca2dbf07ad5a8333), +CONST64(0x6302aa71c81949d9), +}; + +#endif /* LTC_WHIRLTAB_C */ diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt.h new file mode 100644 index 0000000..c310a8c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt.h @@ -0,0 +1,95 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#ifndef TOMCRYPT_H_ +#define TOMCRYPT_H_ +#include +#include +#include +#include +#include +#include +#include +#include + +/* use configuration data */ +#include "tomcrypt_custom.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* version */ +#define CRYPT 0x0118 +#define SCRYPT "1.18.2-develop" + +/* max size of either a cipher/hash block or symmetric key [largest of the two] */ +#define MAXBLOCKSIZE 144 + +#ifndef TAB_SIZE +/* descriptor table size */ +#define TAB_SIZE 34 +#endif + +/* error codes [will be expanded in future releases] */ +enum { + CRYPT_OK=0, /* Result OK */ + CRYPT_ERROR, /* Generic Error */ + CRYPT_NOP, /* Not a failure but no operation was performed */ + + CRYPT_INVALID_KEYSIZE, /* Invalid key size given */ + CRYPT_INVALID_ROUNDS, /* Invalid number of rounds */ + CRYPT_FAIL_TESTVECTOR, /* Algorithm failed test vectors */ + + CRYPT_BUFFER_OVERFLOW, /* Not enough space for output */ + CRYPT_INVALID_PACKET, /* Invalid input packet given */ + + CRYPT_INVALID_PRNGSIZE, /* Invalid number of bits for a PRNG */ + CRYPT_ERROR_READPRNG, /* Could not read enough from PRNG */ + + CRYPT_INVALID_CIPHER, /* Invalid cipher specified */ + CRYPT_INVALID_HASH, /* Invalid hash specified */ + CRYPT_INVALID_PRNG, /* Invalid PRNG specified */ + + CRYPT_MEM, /* Out of memory */ + + CRYPT_PK_TYPE_MISMATCH, /* Not equivalent types of PK keys */ + CRYPT_PK_NOT_PRIVATE, /* Requires a private PK key */ + + CRYPT_INVALID_ARG, /* Generic invalid argument */ + CRYPT_FILE_NOTFOUND, /* File Not Found */ + + CRYPT_PK_INVALID_TYPE, /* Invalid type of PK key */ + + CRYPT_OVERFLOW, /* An overflow of a value was detected/prevented */ + + CRYPT_PK_ASN1_ERROR, /* An error occurred while en- or decoding ASN.1 data */ + + CRYPT_INPUT_TOO_LONG, /* The input was longer than expected. */ + + CRYPT_PK_INVALID_SIZE, /* Invalid size input for PK parameters */ + + CRYPT_INVALID_PRIME_SIZE,/* Invalid size of prime requested */ + CRYPT_PK_INVALID_PADDING, /* Invalid padding on input */ + + CRYPT_HASH_OVERFLOW /* Hash applied to too many bits */ +}; + +#include "tomcrypt_cfg.h" +#include "tomcrypt_macros.h" +#include "tomcrypt_cipher.h" +#include "tomcrypt_hash.h" +#include "tomcrypt_mac.h" +#include "tomcrypt_prng.h" +#include "tomcrypt_pk.h" +#include "tomcrypt_math.h" +#include "tomcrypt_misc.h" +#include "tomcrypt_argchk.h" +#include "tomcrypt_pkcs.h" + +#ifdef __cplusplus + } +#endif + +#endif /* TOMCRYPT_H_ */ + diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_argchk.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_argchk.h new file mode 100644 index 0000000..f3884e9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_argchk.h @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* Defines the LTC_ARGCHK macro used within the library */ +/* ARGTYPE is defined in tomcrypt_cfg.h */ + +/* ARGTYPE is per default defined to 0 */ +#if ARGTYPE == 0 + +#include + +LTC_NORETURN void crypt_argchk(const char *v, const char *s, int d); +#define LTC_ARGCHK(x) do { if (!(x)) { crypt_argchk(#x, __FILE__, __LINE__); } }while(0) +#define LTC_ARGCHKVD(x) do { if (!(x)) { crypt_argchk(#x, __FILE__, __LINE__); } }while(0) + +#elif ARGTYPE == 1 + +/* fatal type of error */ +#define LTC_ARGCHK(x) assert((x)) +#define LTC_ARGCHKVD(x) LTC_ARGCHK(x) + +#elif ARGTYPE == 2 + +#define LTC_ARGCHK(x) if (!(x)) { fprintf(stderr, "\nwarning: ARGCHK failed at %s:%d\n", __FILE__, __LINE__); } +#define LTC_ARGCHKVD(x) LTC_ARGCHK(x) + +#elif ARGTYPE == 3 + +#define LTC_ARGCHK(x) LTC_UNUSED_PARAM(x) +#define LTC_ARGCHKVD(x) LTC_ARGCHK(x) + +#elif ARGTYPE == 4 + +#define LTC_ARGCHK(x) if (!(x)) return CRYPT_INVALID_ARG; +#define LTC_ARGCHKVD(x) if (!(x)) return; + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_arm_neon.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_arm_neon.h new file mode 100644 index 0000000..1b50266 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_arm_neon.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef TOMCRYPT_ARM_NEON_H +#define TOMCRYPT_ARM_NEON_H + +#include + +struct tomcrypt_arm_neon_state { + ulong32 state; +}; + +/* Temporarily enables neon instructions */ +void tomcrypt_arm_neon_enable(struct tomcrypt_arm_neon_state *state); +/* Disables neon instructions after a call to tomcrypt_arm_neon_enable() */ +void tomcrypt_arm_neon_disable(struct tomcrypt_arm_neon_state *state); + +#endif /*TOMCRYPT_ARM_NEON_H*/ diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_cfg.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_cfg.h new file mode 100644 index 0000000..3dad466 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_cfg.h @@ -0,0 +1,323 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* This is the build config file. + * + * With this you can setup what to include/exclude automatically during any build. Just comment + * out the line that #define's the word for the thing you want to remove. phew! + */ + +#ifndef TOMCRYPT_CFG_H +#define TOMCRYPT_CFG_H + +#if defined(_WIN32) || defined(_MSC_VER) + #define LTC_CALL __cdecl +#elif !defined(LTC_CALL) + #define LTC_CALL +#endif + +#ifndef LTC_EXPORT + #define LTC_EXPORT +#endif + +/* certain platforms use macros for these, making the prototypes broken */ +#ifndef LTC_NO_PROTOTYPES + +/* you can change how memory allocation works ... */ +LTC_EXPORT void * LTC_CALL XMALLOC(size_t n); +LTC_EXPORT void * LTC_CALL XREALLOC(void *p, size_t n); +LTC_EXPORT void * LTC_CALL XCALLOC(size_t n, size_t s); +LTC_EXPORT void LTC_CALL XFREE(void *p); + +LTC_EXPORT void LTC_CALL XQSORT(void *base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)); + + +/* change the clock function too */ +LTC_EXPORT clock_t LTC_CALL XCLOCK(void); + +/* various other functions */ +LTC_EXPORT void * LTC_CALL XMEMCPY(void *dest, const void *src, size_t n); +LTC_EXPORT int LTC_CALL XMEMCMP(const void *s1, const void *s2, size_t n); +LTC_EXPORT void * LTC_CALL XMEMSET(void *s, int c, size_t n); + +LTC_EXPORT int LTC_CALL XSTRCMP(const char *s1, const char *s2); + +#endif + +/* some compilers do not like "inline" (or maybe "static inline"), namely: HP cc, IBM xlc */ +#if defined(__GNUC__) || defined(__xlc__) + #define LTC_INLINE __inline__ +#elif defined(_MSC_VER) || defined(__HP_cc) + #define LTC_INLINE __inline +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define LTC_INLINE inline +#else + #define LTC_INLINE +#endif + +#if defined(__clang__) || defined(__GNUC_MINOR__) +#define LTC_NORETURN __attribute__ ((noreturn)) +#elif defined(_MSC_VER) +#define LTC_NORETURN __declspec(noreturn) +#else +#define LTC_NORETURN +#endif + +/* type of argument checking, 0=default, 1=fatal and 2=error+continue, 3=nothing */ +#ifndef ARGTYPE + #define ARGTYPE 0 +#endif + +#undef LTC_ENCRYPT +#define LTC_ENCRYPT 0 +#undef LTC_DECRYPT +#define LTC_DECRYPT 1 + +/* Controls endianess and size of registers. Leave uncommented to get platform neutral [slower] code + * + * Note: in order to use the optimized macros your platform must support unaligned 32 and 64 bit read/writes. + * The x86 platforms allow this but some others [ARM for instance] do not. On those platforms you **MUST** + * use the portable [slower] macros. + */ +/* detect x86/i386 32bit */ +#if defined(__i386__) || defined(__i386) || defined(_M_IX86) + #define ENDIAN_LITTLE + #define ENDIAN_32BITWORD + #define LTC_FAST +#endif + +/* detect amd64/x64 */ +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) + #define ENDIAN_LITTLE + #define ENDIAN_64BITWORD + #define LTC_FAST +#endif + +/* detect PPC32 */ +#if defined(LTC_PPC32) + #define ENDIAN_BIG + #define ENDIAN_32BITWORD + #define LTC_FAST +#endif + +/* detects MIPS R5900 processors (PS2) */ +#if (defined(__R5900) || defined(R5900) || defined(__R5900__)) && (defined(_mips) || defined(__mips__) || defined(mips)) + #define ENDIAN_64BITWORD + #if defined(_MIPSEB) || defined(__MIPSEB) || defined(__MIPSEB__) + #define ENDIAN_BIG + #else + #define ENDIAN_LITTLE + #endif +#endif + +/* detect AIX */ +#if defined(_AIX) && defined(_BIG_ENDIAN) + #define ENDIAN_BIG + #if defined(__LP64__) || defined(_ARCH_PPC64) + #define ENDIAN_64BITWORD + #else + #define ENDIAN_32BITWORD + #endif +#endif + +/* detect HP-UX */ +#if defined(__hpux) || defined(__hpux__) + #define ENDIAN_BIG + #if defined(__ia64) || defined(__ia64__) || defined(__LP64__) + #define ENDIAN_64BITWORD + #else + #define ENDIAN_32BITWORD + #endif +#endif + +/* detect Apple OS X */ +#if defined(__APPLE__) && defined(__MACH__) + #if defined(__LITTLE_ENDIAN__) || defined(__x86_64__) + #define ENDIAN_LITTLE + #else + #define ENDIAN_BIG + #endif + #if defined(__LP64__) || defined(__x86_64__) + #define ENDIAN_64BITWORD + #else + #define ENDIAN_32BITWORD + #endif +#endif + +/* detect SPARC and SPARC64 */ +#if defined(__sparc__) || defined(__sparc) + #define ENDIAN_BIG + #if defined(__arch64__) || defined(__sparcv9) || defined(__sparc_v9__) + #define ENDIAN_64BITWORD + #else + #define ENDIAN_32BITWORD + #endif +#endif + +/* detect IBM S390(x) */ +#if defined(__s390x__) || defined(__s390__) + #define ENDIAN_BIG + #if defined(__s390x__) + #define ENDIAN_64BITWORD + #else + #define ENDIAN_32BITWORD + #endif +#endif + +/* detect PPC64 */ +#if defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) + #define ENDIAN_64BITWORD + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #define ENDIAN_BIG + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define ENDIAN_LITTLE + #endif + #define LTC_FAST +#endif + +/* endianness fallback */ +#if !defined(ENDIAN_BIG) && !defined(ENDIAN_LITTLE) + #if defined(_BYTE_ORDER) && _BYTE_ORDER == _BIG_ENDIAN || \ + defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN || \ + defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ || \ + defined(__BIG_ENDIAN__) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(_MIPSEB) || defined(__MIPSEB) || defined(__MIPSEB__) || \ + defined(__m68k__) + #define ENDIAN_BIG + #elif defined(_BYTE_ORDER) && _BYTE_ORDER == _LITTLE_ENDIAN || \ + defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN || \ + defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ || \ + defined(__LITTLE_ENDIAN__) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(_MIPSEL) || defined(__MIPSEL) || defined(__MIPSEL__) + #define ENDIAN_LITTLE + #else + #error Cannot detect endianness + #endif +#endif + +/* ulong64: 64-bit data type */ +#ifdef _MSC_VER + #define CONST64(n) n ## ui64 + typedef unsigned __int64 ulong64; + typedef __int64 long64; +#else + #define CONST64(n) n ## ULL + typedef unsigned long long ulong64; + typedef long long long64; +#endif + +/* ulong32: "32-bit at least" data type */ +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || \ + defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) || \ + defined(__s390x__) || defined(__arch64__) || defined(__aarch64__) || \ + defined(__sparcv9) || defined(__sparc_v9__) || defined(__sparc64__) || \ + defined(__ia64) || defined(__ia64__) || defined(__itanium__) || defined(_M_IA64) || \ + defined(__LP64__) || defined(_LP64) || defined(__64BIT__) + typedef unsigned ulong32; + #if !defined(ENDIAN_64BITWORD) && !defined(ENDIAN_32BITWORD) + #define ENDIAN_64BITWORD + #endif +#else + typedef unsigned long ulong32; + #if !defined(ENDIAN_64BITWORD) && !defined(ENDIAN_32BITWORD) + #define ENDIAN_32BITWORD + #endif +#endif + +#if defined(ENDIAN_64BITWORD) && !defined(_MSC_VER) +typedef unsigned long long ltc_mp_digit; +#else +typedef unsigned long ltc_mp_digit; +#endif + +/* No asm is a quick way to disable anything "not portable" */ +#ifdef LTC_NO_ASM + #define ENDIAN_NEUTRAL + #undef ENDIAN_32BITWORD + #undef ENDIAN_64BITWORD + #undef LTC_FAST + #define LTC_NO_BSWAP + #define LTC_NO_ROLC + #define LTC_NO_ROTATE +#endif + +/* No LTC_FAST if: explicitly disabled OR non-gcc/non-clang compiler OR old gcc OR using -ansi -std=c99 */ +#if defined(LTC_NO_FAST) || (__GNUC__ < 4) || defined(__STRICT_ANSI__) + #undef LTC_FAST +#endif + +#ifdef LTC_FAST + #define LTC_FAST_TYPE_PTR_CAST(x) ((LTC_FAST_TYPE*)(void*)(x)) + #ifdef ENDIAN_64BITWORD + typedef ulong64 __attribute__((__may_alias__)) LTC_FAST_TYPE; + #else + typedef ulong32 __attribute__((__may_alias__)) LTC_FAST_TYPE; + #endif +#endif + +#if !defined(ENDIAN_NEUTRAL) && (defined(ENDIAN_BIG) || defined(ENDIAN_LITTLE)) && !(defined(ENDIAN_32BITWORD) || defined(ENDIAN_64BITWORD)) + #error You must specify a word size as well as endianess in tomcrypt_cfg.h +#endif + +#if !(defined(ENDIAN_BIG) || defined(ENDIAN_LITTLE)) + #define ENDIAN_NEUTRAL +#endif + +#if (defined(ENDIAN_32BITWORD) && defined(ENDIAN_64BITWORD)) + #error Cannot be 32 and 64 bit words... +#endif + +/* gcc 4.3 and up has a bswap builtin; detect it by gcc version. + * clang also supports the bswap builtin, and although clang pretends + * to be gcc (macro-wise, anyway), clang pretends to be a version + * prior to gcc 4.3, so we can't detect bswap that way. Instead, + * clang has a __has_builtin mechanism that can be used to check + * for builtins: + * http://clang.llvm.org/docs/LanguageExtensions.html#feature_check */ +#ifndef __has_builtin + #define __has_builtin(x) 0 +#endif +#if !defined(LTC_NO_BSWAP) && defined(__GNUC__) && \ + ((__GNUC__ * 100 + __GNUC_MINOR__ >= 403) || \ + (__has_builtin(__builtin_bswap32) && __has_builtin(__builtin_bswap64))) + #define LTC_HAVE_BSWAP_BUILTIN +#endif + +#if !defined(LTC_NO_ROTATE) && (__has_builtin(__builtin_rotateleft32) && __has_builtin(__builtin_rotateright32)) + #define LTC_HAVE_ROTATE_BUILTIN +#endif + +#if defined(__GNUC__) + #define LTC_ALIGN(n) __attribute__((aligned(n))) +#else + #define LTC_ALIGN(n) +#endif + +/* Define `LTC_NO_NULL_TERMINATION_CHECK` in the user code + * before including `tomcrypt.h` to disable this functionality. + */ +#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(LTC_NO_NULL_TERMINATION_CHECK) +# define LTC_NULL_TERMINATED __attribute__((sentinel)) +#else +# define LTC_NULL_TERMINATED +#endif + +#if defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 405) +# define LTC_DEPRECATED(s) __attribute__((deprecated("replaced by " #s))) +# define PRIVATE_LTC_DEPRECATED_PRAGMA(s) _Pragma(#s) +# define LTC_DEPRECATED_PRAGMA(s) PRIVATE_LTC_DEPRECATED_PRAGMA(GCC warning s) +#elif defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 301) +# define LTC_DEPRECATED(s) __attribute__((deprecated)) +# define LTC_DEPRECATED_PRAGMA(s) +#elif defined(_MSC_VER) && _MSC_VER >= 1500 + /* supported since Visual Studio 2008 */ +# define LTC_DEPRECATED(s) __declspec(deprecated("replaced by " #s)) +# define LTC_DEPRECATED_PRAGMA(s) __pragma(message(s)) +#else +# define LTC_DEPRECATED(s) +# define LTC_DEPRECATED_PRAGMA(s) +#endif + +#endif /* TOMCRYPT_CFG_H */ diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_cipher.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_cipher.h new file mode 100644 index 0000000..17a0994 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_cipher.h @@ -0,0 +1,1160 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* ---- SYMMETRIC KEY STUFF ----- + * + * We put each of the ciphers scheduled keys in their own structs then we put all of + * the key formats in one union. This makes the function prototypes easier to use. + */ +#ifdef LTC_BLOWFISH +struct blowfish_key { + ulong32 S[4][256]; + ulong32 K[18]; +}; +#endif + +#ifdef LTC_RC5 +struct rc5_key { + int rounds; + ulong32 K[50]; +}; +#endif + +#ifdef LTC_RC6 +struct rc6_key { + ulong32 K[44]; +}; +#endif + +#ifdef LTC_SAFERP +struct saferp_key { + unsigned char K[33][16]; + long rounds; +}; +#endif + +#ifdef LTC_RIJNDAEL +struct rijndael_key { + ulong32 eK[60], dK[60]; + int Nr; +}; +#endif + +#ifdef LTC_KSEED +struct kseed_key { + ulong32 K[32], dK[32]; +}; +#endif + +#ifdef LTC_KASUMI +struct kasumi_key { + ulong32 KLi1[8], KLi2[8], + KOi1[8], KOi2[8], KOi3[8], + KIi1[8], KIi2[8], KIi3[8]; +}; +#endif + +#ifdef LTC_XTEA +struct xtea_key { + unsigned long A[32], B[32]; +}; +#endif + +#ifdef LTC_TWOFISH +#ifndef LTC_TWOFISH_SMALL + struct twofish_key { + ulong32 S[4][256], K[40]; + }; +#else + struct twofish_key { + ulong32 K[40]; + unsigned char S[32], start; + }; +#endif +#endif + +#ifdef LTC_SAFER +#define LTC_SAFER_K64_DEFAULT_NOF_ROUNDS 6 +#define LTC_SAFER_K128_DEFAULT_NOF_ROUNDS 10 +#define LTC_SAFER_SK64_DEFAULT_NOF_ROUNDS 8 +#define LTC_SAFER_SK128_DEFAULT_NOF_ROUNDS 10 +#define LTC_SAFER_MAX_NOF_ROUNDS 13 +#define LTC_SAFER_BLOCK_LEN 8 +#define LTC_SAFER_KEY_LEN (1 + LTC_SAFER_BLOCK_LEN * (1 + 2 * LTC_SAFER_MAX_NOF_ROUNDS)) +typedef unsigned char safer_block_t[LTC_SAFER_BLOCK_LEN]; +typedef unsigned char safer_key_t[LTC_SAFER_KEY_LEN]; +struct safer_key { safer_key_t key; }; +#endif + +#ifdef LTC_RC2 +struct rc2_key { unsigned xkey[64]; }; +#endif + +#ifdef LTC_DES +struct des_key { + ulong32 ek[32], dk[32]; +}; + +struct des3_key { + ulong32 ek[3][32], dk[3][32]; +}; +#endif + +#ifdef LTC_CAST5 +struct cast5_key { + ulong32 K[32], keylen; +}; +#endif + +#ifdef LTC_NOEKEON +struct noekeon_key { + ulong32 K[4], dK[4]; +}; +#endif + +#ifdef LTC_SKIPJACK +struct skipjack_key { + unsigned char key[10]; +}; +#endif + +#ifdef LTC_KHAZAD +struct khazad_key { + ulong64 roundKeyEnc[8 + 1]; + ulong64 roundKeyDec[8 + 1]; +}; +#endif + +#ifdef LTC_ANUBIS +struct anubis_key { + int keyBits; + int R; + ulong32 roundKeyEnc[18 + 1][4]; + ulong32 roundKeyDec[18 + 1][4]; +}; +#endif + +#ifdef LTC_MULTI2 +struct multi2_key { + int N; + ulong32 uk[8]; +}; +#endif + +#ifdef LTC_CAMELLIA +struct camellia_key { + int R; + ulong64 kw[4], k[24], kl[6]; +}; +#endif + +#ifdef LTC_IDEA +/* rounds */ +#define LTC_IDEA_ROUNDS 8 +/* key schedule length in # of unsigned shorts */ +#define LTC_IDEA_KEYLEN 6*LTC_IDEA_ROUNDS+4 +struct idea_key { + unsigned short int ek[LTC_IDEA_KEYLEN]; /* enc key */ + unsigned short int dk[LTC_IDEA_KEYLEN]; /* dec key */ +}; +#endif + +#ifdef LTC_SERPENT +struct serpent_key { + ulong32 k[33*4]; +}; +#endif + +#ifdef LTC_TEA +struct tea_key { + ulong32 k[4]; +}; +#endif + +typedef union Symmetric_key { +#ifdef LTC_DES + struct des_key des; + struct des3_key des3; +#endif +#ifdef LTC_RC2 + struct rc2_key rc2; +#endif +#ifdef LTC_SAFER + struct safer_key safer; +#endif +#ifdef LTC_TWOFISH + struct twofish_key twofish; +#endif +#ifdef LTC_BLOWFISH + struct blowfish_key blowfish; +#endif +#ifdef LTC_RC5 + struct rc5_key rc5; +#endif +#ifdef LTC_RC6 + struct rc6_key rc6; +#endif +#ifdef LTC_SAFERP + struct saferp_key saferp; +#endif +#ifdef LTC_RIJNDAEL + struct rijndael_key rijndael; +#endif +#ifdef LTC_XTEA + struct xtea_key xtea; +#endif +#ifdef LTC_CAST5 + struct cast5_key cast5; +#endif +#ifdef LTC_NOEKEON + struct noekeon_key noekeon; +#endif +#ifdef LTC_SKIPJACK + struct skipjack_key skipjack; +#endif +#ifdef LTC_KHAZAD + struct khazad_key khazad; +#endif +#ifdef LTC_ANUBIS + struct anubis_key anubis; +#endif +#ifdef LTC_KSEED + struct kseed_key kseed; +#endif +#ifdef LTC_KASUMI + struct kasumi_key kasumi; +#endif +#ifdef LTC_MULTI2 + struct multi2_key multi2; +#endif +#ifdef LTC_CAMELLIA + struct camellia_key camellia; +#endif +#ifdef LTC_IDEA + struct idea_key idea; +#endif +#ifdef LTC_SERPENT + struct serpent_key serpent; +#endif +#ifdef LTC_TEA + struct tea_key tea; +#endif + void *data; +} symmetric_key; + +#ifdef LTC_ECB_MODE +/** A block cipher ECB structure */ +typedef struct { + /** The index of the cipher chosen */ + int cipher, + /** The block size of the given cipher */ + blocklen; + /** The scheduled key */ + symmetric_key key; +} symmetric_ECB; +#endif + +#ifdef LTC_CFB_MODE +/** A block cipher CFB structure */ +typedef struct { + /** The index of the cipher chosen */ + int cipher, + /** The block size of the given cipher */ + blocklen, + /** The padding offset */ + padlen; + /** The current IV */ + unsigned char IV[MAXBLOCKSIZE], + /** The pad used to encrypt/decrypt */ + pad[MAXBLOCKSIZE]; + /** The scheduled key */ + symmetric_key key; +} symmetric_CFB; +#endif + +#ifdef LTC_OFB_MODE +/** A block cipher OFB structure */ +typedef struct { + /** The index of the cipher chosen */ + int cipher, + /** The block size of the given cipher */ + blocklen, + /** The padding offset */ + padlen; + /** The current IV */ + unsigned char IV[MAXBLOCKSIZE]; + /** The scheduled key */ + symmetric_key key; +} symmetric_OFB; +#endif + +#ifdef LTC_CBC_MODE +/** A block cipher CBC structure */ +typedef struct { + /** The index of the cipher chosen */ + int cipher, + /** The block size of the given cipher */ + blocklen; + /** The current IV */ + unsigned char IV[MAXBLOCKSIZE]; + /** The scheduled key */ + symmetric_key key; +} symmetric_CBC; +#endif + + +#ifdef LTC_CTR_MODE +/** A block cipher CTR structure */ +typedef struct { + /** The index of the cipher chosen */ + int cipher, + /** The block size of the given cipher */ + blocklen, + /** The padding offset */ + padlen, + /** The mode (endianess) of the CTR, 0==little, 1==big */ + mode, + /** counter width */ + ctrlen; + + /** The counter */ + unsigned char ctr[MAXBLOCKSIZE]; + /** The pad used to encrypt/decrypt */ + unsigned char pad[MAXBLOCKSIZE] LTC_ALIGN(16); + /** The scheduled key */ + symmetric_key key; +} symmetric_CTR; +#endif + + +#ifdef LTC_LRW_MODE +/** A LRW structure */ +typedef struct { + /** The index of the cipher chosen (must be a 128-bit block cipher) */ + int cipher; + + /** The current IV */ + unsigned char IV[16], + + /** the tweak key */ + tweak[16], + + /** The current pad, it's the product of the first 15 bytes against the tweak key */ + pad[16]; + + /** The scheduled symmetric key */ + symmetric_key key; + +#ifdef LTC_LRW_TABLES + /** The pre-computed multiplication table */ + unsigned char PC[16][256][16]; +#endif +} symmetric_LRW; +#endif + +#ifdef LTC_F8_MODE +/** A block cipher F8 structure */ +typedef struct { + /** The index of the cipher chosen */ + int cipher, + /** The block size of the given cipher */ + blocklen, + /** The padding offset */ + padlen; + /** The current IV */ + unsigned char IV[MAXBLOCKSIZE], + MIV[MAXBLOCKSIZE]; + /** Current block count */ + ulong32 blockcnt; + /** The scheduled key */ + symmetric_key key; +} symmetric_F8; +#endif + + +/** cipher descriptor table, last entry has "name == NULL" to mark the end of table */ +extern const struct ltc_cipher_descriptor { + /** name of cipher */ + const char *name; + /** internal ID */ + unsigned char ID; + /** min keysize (octets) */ + int min_key_length, + /** max keysize (octets) */ + max_key_length, + /** block size (octets) */ + block_length, + /** default number of rounds */ + default_rounds; + /** Setup the cipher + @param key The input symmetric key + @param keylen The length of the input key (octets) + @param num_rounds The requested number of rounds (0==default) + @param skey [out] The destination of the scheduled key + @return CRYPT_OK if successful + */ + int (*setup)(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); + /** Encrypt a block + @param pt The plaintext + @param ct [out] The ciphertext + @param skey The scheduled key + @return CRYPT_OK if successful + */ + int (*ecb_encrypt)(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); + /** Decrypt a block + @param ct The ciphertext + @param pt [out] The plaintext + @param skey The scheduled key + @return CRYPT_OK if successful + */ + int (*ecb_decrypt)(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); + /** Test the block cipher + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled + */ + int (*test)(void); + + /** Terminate the context + @param skey The scheduled key + */ + void (*done)(symmetric_key *skey); + + /** Determine a key size + @param keysize [in/out] The size of the key desired and the suggested size + @return CRYPT_OK if successful + */ + int (*keysize)(int *keysize); + +/** Accelerators **/ + /** Accelerated ECB encryption + @param pt Plaintext + @param ct Ciphertext + @param blocks The number of complete blocks to process + @param skey The scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_ecb_encrypt)(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey); + + /** Accelerated ECB decryption + @param pt Plaintext + @param ct Ciphertext + @param blocks The number of complete blocks to process + @param skey The scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_ecb_decrypt)(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey); + + /** Accelerated CBC encryption + @param pt Plaintext + @param ct Ciphertext + @param blocks The number of complete blocks to process + @param IV The initial value (input/output) + @param skey The scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_cbc_encrypt)(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, symmetric_key *skey); + + /** Accelerated CBC decryption + @param pt Plaintext + @param ct Ciphertext + @param blocks The number of complete blocks to process + @param IV The initial value (input/output) + @param skey The scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_cbc_decrypt)(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, symmetric_key *skey); + + /** Accelerated CTR encryption + @param pt Plaintext + @param ct Ciphertext + @param blocks The number of complete blocks to process + @param IV The initial value (input/output) + @param mode little or big endian counter (mode=0 or mode=1) + @param skey The scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_ctr_encrypt)(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, symmetric_key *skey); + + /** Accelerated LRW + @param pt Plaintext + @param ct Ciphertext + @param blocks The number of complete blocks to process + @param IV The initial value (input/output) + @param tweak The LRW tweak + @param skey The scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_lrw_encrypt)(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, const unsigned char *tweak, symmetric_key *skey); + + /** Accelerated LRW + @param ct Ciphertext + @param pt Plaintext + @param blocks The number of complete blocks to process + @param IV The initial value (input/output) + @param tweak The LRW tweak + @param skey The scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_lrw_decrypt)(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const unsigned char *tweak, symmetric_key *skey); + + /** Accelerated CCM packet (one-shot) + @param key The secret key to use + @param keylen The length of the secret key (octets) + @param uskey A previously scheduled key [optional can be NULL] + @param nonce The session nonce [use once] + @param noncelen The length of the nonce + @param header The header for the session + @param headerlen The length of the header (octets) + @param pt [out] The plaintext + @param ptlen The length of the plaintext (octets) + @param ct [out] The ciphertext + @param tag [out] The destination tag + @param taglen [in/out] The max size and resulting size of the authentication tag + @param direction Encrypt or Decrypt direction (0 or 1) + @return CRYPT_OK if successful + */ + int (*accel_ccm_memory)( + const unsigned char *key, unsigned long keylen, + symmetric_key *uskey, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen, + int direction); + + /** Accelerated GCM packet (one shot) + @param key The secret key + @param keylen The length of the secret key + @param IV The initialization vector + @param IVlen The length of the initialization vector + @param adata The additional authentication data (header) + @param adatalen The length of the adata + @param pt The plaintext + @param ptlen The length of the plaintext (ciphertext length is the same) + @param ct The ciphertext + @param tag [out] The MAC tag + @param taglen [in/out] The MAC tag length + @param direction Encrypt or Decrypt mode (GCM_ENCRYPT or GCM_DECRYPT) + @return CRYPT_OK on success + */ + int (*accel_gcm_memory)( + const unsigned char *key, unsigned long keylen, + const unsigned char *IV, unsigned long IVlen, + const unsigned char *adata, unsigned long adatalen, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen, + int direction); + + /** Accelerated one shot LTC_OMAC + @param key The secret key + @param keylen The key length (octets) + @param in The message + @param inlen Length of message (octets) + @param out [out] Destination for tag + @param outlen [in/out] Initial and final size of out + @return CRYPT_OK on success + */ + int (*omac_memory)( + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + + /** Accelerated one shot XCBC + @param key The secret key + @param keylen The key length (octets) + @param in The message + @param inlen Length of message (octets) + @param out [out] Destination for tag + @param outlen [in/out] Initial and final size of out + @return CRYPT_OK on success + */ + int (*xcbc_memory)( + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + + /** Accelerated one shot F9 + @param key The secret key + @param keylen The key length (octets) + @param in The message + @param inlen Length of message (octets) + @param out [out] Destination for tag + @param outlen [in/out] Initial and final size of out + @return CRYPT_OK on success + @remark Requires manual padding + */ + int (*f9_memory)( + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + + /** Accelerated XTS encryption + @param pt Plaintext + @param ct Ciphertext + @param blocks The number of complete blocks to process + @param tweak The 128-bit encryption tweak (input/output). + The tweak should not be encrypted on input, but + next tweak will be copied encrypted on output. + @param skey1 The first scheduled key context + @param skey2 The second scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_xts_encrypt)(const unsigned char *pt, unsigned char *ct, + unsigned long blocks, unsigned char *tweak, + const symmetric_key *skey1, const symmetric_key *skey2); + + /** Accelerated XTS decryption + @param ct Ciphertext + @param pt Plaintext + @param blocks The number of complete blocks to process + @param tweak The 128-bit encryption tweak (input/output). + The tweak should not be encrypted on input, but + next tweak will be copied encrypted on output. + @param skey1 The first scheduled key context + @param skey2 The second scheduled key context + @return CRYPT_OK if successful + */ + int (*accel_xts_decrypt)(const unsigned char *ct, unsigned char *pt, + unsigned long blocks, unsigned char *tweak, + const symmetric_key *skey1, const symmetric_key *skey2); +} *cipher_descriptor[]; + +#ifdef LTC_BLOWFISH +int blowfish_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int blowfish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int blowfish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int blowfish_test(void); +void blowfish_done(symmetric_key *skey); +int blowfish_keysize(int *keysize); +extern const struct ltc_cipher_descriptor blowfish_desc; +#endif + +#ifdef LTC_RC5 +int rc5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int rc5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int rc5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int rc5_test(void); +void rc5_done(symmetric_key *skey); +int rc5_keysize(int *keysize); +extern const struct ltc_cipher_descriptor rc5_desc; +#endif + +#ifdef LTC_RC6 +int rc6_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int rc6_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int rc6_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int rc6_test(void); +void rc6_done(symmetric_key *skey); +int rc6_keysize(int *keysize); +extern const struct ltc_cipher_descriptor rc6_desc; +#endif + +#ifdef LTC_RC2 +int rc2_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int rc2_setup_ex(const unsigned char *key, int keylen, int bits, int num_rounds, symmetric_key *skey); +int rc2_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int rc2_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int rc2_test(void); +void rc2_done(symmetric_key *skey); +int rc2_keysize(int *keysize); +extern const struct ltc_cipher_descriptor rc2_desc; +#endif + +#ifdef LTC_SAFERP +int saferp_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int saferp_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int saferp_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int saferp_test(void); +void saferp_done(symmetric_key *skey); +int saferp_keysize(int *keysize); +extern const struct ltc_cipher_descriptor saferp_desc; +#endif + +#ifdef LTC_SAFER +int safer_k64_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int safer_sk64_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int safer_k128_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int safer_sk128_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int safer_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int safer_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int safer_k64_test(void); +int safer_sk64_test(void); +int safer_sk128_test(void); +void safer_done(symmetric_key *skey); +int safer_64_keysize(int *keysize); +int safer_128_keysize(int *keysize); +extern const struct ltc_cipher_descriptor safer_k64_desc, safer_k128_desc, safer_sk64_desc, safer_sk128_desc; +#endif + +#ifdef LTC_RIJNDAEL + +/* make aes an alias */ +#define aes_setup rijndael_setup +#define aes_ecb_encrypt rijndael_ecb_encrypt +#define aes_ecb_decrypt rijndael_ecb_decrypt +#define aes_test rijndael_test +#define aes_done rijndael_done +#define aes_keysize rijndael_keysize + +#define aes_enc_setup rijndael_enc_setup +#define aes_enc_ecb_encrypt rijndael_enc_ecb_encrypt +#define aes_enc_keysize rijndael_enc_keysize + +int rijndael_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int rijndael_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int rijndael_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int rijndael_test(void); +void rijndael_done(symmetric_key *skey); +int rijndael_keysize(int *keysize); +int rijndael_enc_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int rijndael_enc_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +void rijndael_enc_done(symmetric_key *skey); +int rijndael_enc_keysize(int *keysize); +extern const struct ltc_cipher_descriptor rijndael_desc, aes_desc; +extern const struct ltc_cipher_descriptor rijndael_enc_desc, aes_enc_desc; +#endif + +#ifdef LTC_XTEA +int xtea_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int xtea_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int xtea_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int xtea_test(void); +void xtea_done(symmetric_key *skey); +int xtea_keysize(int *keysize); +extern const struct ltc_cipher_descriptor xtea_desc; +#endif + +#ifdef LTC_TWOFISH +int twofish_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int twofish_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int twofish_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int twofish_test(void); +void twofish_done(symmetric_key *skey); +int twofish_keysize(int *keysize); +extern const struct ltc_cipher_descriptor twofish_desc; +#endif + +#ifdef LTC_DES +int des_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int des_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int des_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int des_test(void); +void des_done(symmetric_key *skey); +int des_keysize(int *keysize); +int des3_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int des3_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int des3_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int des3_test(void); +void des3_done(symmetric_key *skey); +int des3_keysize(int *keysize); +extern const struct ltc_cipher_descriptor des_desc, des3_desc; +#endif + +#ifdef LTC_CAST5 +int cast5_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int cast5_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int cast5_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int cast5_test(void); +void cast5_done(symmetric_key *skey); +int cast5_keysize(int *keysize); +extern const struct ltc_cipher_descriptor cast5_desc; +#endif + +#ifdef LTC_NOEKEON +int noekeon_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int noekeon_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int noekeon_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int noekeon_test(void); +void noekeon_done(symmetric_key *skey); +int noekeon_keysize(int *keysize); +extern const struct ltc_cipher_descriptor noekeon_desc; +#endif + +#ifdef LTC_SKIPJACK +int skipjack_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int skipjack_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int skipjack_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int skipjack_test(void); +void skipjack_done(symmetric_key *skey); +int skipjack_keysize(int *keysize); +extern const struct ltc_cipher_descriptor skipjack_desc; +#endif + +#ifdef LTC_KHAZAD +int khazad_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int khazad_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int khazad_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int khazad_test(void); +void khazad_done(symmetric_key *skey); +int khazad_keysize(int *keysize); +extern const struct ltc_cipher_descriptor khazad_desc; +#endif + +#ifdef LTC_ANUBIS +int anubis_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int anubis_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int anubis_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int anubis_test(void); +void anubis_done(symmetric_key *skey); +int anubis_keysize(int *keysize); +extern const struct ltc_cipher_descriptor anubis_desc; +#endif + +#ifdef LTC_KSEED +int kseed_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int kseed_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int kseed_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int kseed_test(void); +void kseed_done(symmetric_key *skey); +int kseed_keysize(int *keysize); +extern const struct ltc_cipher_descriptor kseed_desc; +#endif + +#ifdef LTC_KASUMI +int kasumi_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int kasumi_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int kasumi_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int kasumi_test(void); +void kasumi_done(symmetric_key *skey); +int kasumi_keysize(int *keysize); +extern const struct ltc_cipher_descriptor kasumi_desc; +#endif + + +#ifdef LTC_MULTI2 +int multi2_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int multi2_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int multi2_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int multi2_test(void); +void multi2_done(symmetric_key *skey); +int multi2_keysize(int *keysize); +extern const struct ltc_cipher_descriptor multi2_desc; +#endif + +#ifdef LTC_CAMELLIA +int camellia_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int camellia_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int camellia_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int camellia_test(void); +void camellia_done(symmetric_key *skey); +int camellia_keysize(int *keysize); +extern const struct ltc_cipher_descriptor camellia_desc; +#endif + +#ifdef LTC_IDEA +int idea_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int idea_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int idea_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int idea_test(void); +void idea_done(symmetric_key *skey); +int idea_keysize(int *keysize); +extern const struct ltc_cipher_descriptor idea_desc; +#endif + +#ifdef LTC_SERPENT +int serpent_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int serpent_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int serpent_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int serpent_test(void); +void serpent_done(symmetric_key *skey); +int serpent_keysize(int *keysize); +extern const struct ltc_cipher_descriptor serpent_desc; +#endif + +#ifdef LTC_TEA +int tea_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey); +int tea_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey); +int tea_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey); +int tea_test(void); +void tea_done(symmetric_key *skey); +int tea_keysize(int *keysize); +extern const struct ltc_cipher_descriptor tea_desc; +#endif + +#ifdef LTC_ECB_MODE +int ecb_start(int cipher, const unsigned char *key, + int keylen, int num_rounds, symmetric_ECB *ecb); +int ecb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_ECB *ecb); +int ecb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_ECB *ecb); +int ecb_done(symmetric_ECB *ecb); +#endif + +#ifdef LTC_CFB_MODE +int cfb_start(int cipher, const unsigned char *IV, const unsigned char *key, + int keylen, int num_rounds, symmetric_CFB *cfb); +int cfb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CFB *cfb); +int cfb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_CFB *cfb); +int cfb_getiv(unsigned char *IV, unsigned long *len, const symmetric_CFB *cfb); +int cfb_setiv(const unsigned char *IV, unsigned long len, symmetric_CFB *cfb); +int cfb_done(symmetric_CFB *cfb); +#endif + +#ifdef LTC_OFB_MODE +int ofb_start(int cipher, const unsigned char *IV, const unsigned char *key, + int keylen, int num_rounds, symmetric_OFB *ofb); +int ofb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_OFB *ofb); +int ofb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_OFB *ofb); +int ofb_getiv(unsigned char *IV, unsigned long *len, const symmetric_OFB *ofb); +int ofb_setiv(const unsigned char *IV, unsigned long len, symmetric_OFB *ofb); +int ofb_done(symmetric_OFB *ofb); +#endif + +#ifdef LTC_CBC_MODE +int cbc_start(int cipher, const unsigned char *IV, const unsigned char *key, + int keylen, int num_rounds, symmetric_CBC *cbc); +int cbc_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CBC *cbc); +int cbc_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_CBC *cbc); +int cbc_getiv(unsigned char *IV, unsigned long *len, const symmetric_CBC *cbc); +int cbc_setiv(const unsigned char *IV, unsigned long len, symmetric_CBC *cbc); +int cbc_done(symmetric_CBC *cbc); +#endif + +#ifdef LTC_CTR_MODE + +#define CTR_COUNTER_LITTLE_ENDIAN 0x0000 +#define CTR_COUNTER_BIG_ENDIAN 0x1000 +#define LTC_CTR_RFC3686 0x2000 + +int ctr_start( int cipher, + const unsigned char *IV, + const unsigned char *key, int keylen, + int num_rounds, int ctr_mode, + symmetric_CTR *ctr); +int ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CTR *ctr); +int ctr_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_CTR *ctr); +int ctr_getiv(unsigned char *IV, unsigned long *len, const symmetric_CTR *ctr); +int ctr_setiv(const unsigned char *IV, unsigned long len, symmetric_CTR *ctr); +int ctr_done(symmetric_CTR *ctr); +int ctr_test(void); +#endif + +#ifdef LTC_LRW_MODE + +#define LRW_ENCRYPT LTC_ENCRYPT +#define LRW_DECRYPT LTC_DECRYPT + +int lrw_start( int cipher, + const unsigned char *IV, + const unsigned char *key, int keylen, + const unsigned char *tweak, + int num_rounds, + symmetric_LRW *lrw); +int lrw_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_LRW *lrw); +int lrw_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_LRW *lrw); +int lrw_getiv(unsigned char *IV, unsigned long *len, const symmetric_LRW *lrw); +int lrw_setiv(const unsigned char *IV, unsigned long len, symmetric_LRW *lrw); +int lrw_done(symmetric_LRW *lrw); +int lrw_test(void); + +/* don't call */ +int lrw_process(const unsigned char *pt, unsigned char *ct, unsigned long len, int mode, symmetric_LRW *lrw); +#endif + +#ifdef LTC_F8_MODE +int f8_start( int cipher, const unsigned char *IV, + const unsigned char *key, int keylen, + const unsigned char *salt_key, int skeylen, + int num_rounds, symmetric_F8 *f8); +int f8_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_F8 *f8); +int f8_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_F8 *f8); +int f8_getiv(unsigned char *IV, unsigned long *len, const symmetric_F8 *f8); +int f8_setiv(const unsigned char *IV, unsigned long len, symmetric_F8 *f8); +int f8_done(symmetric_F8 *f8); +int f8_test_mode(void); +#endif + +#ifdef LTC_XTS_MODE +typedef struct { + symmetric_key key1, key2; + int cipher; +} symmetric_xts; + +int xts_start( int cipher, + const unsigned char *key1, + const unsigned char *key2, + unsigned long keylen, + int num_rounds, + symmetric_xts *xts); + +int xts_encrypt( + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tweak, + const symmetric_xts *xts); +int xts_decrypt( + const unsigned char *ct, unsigned long ptlen, + unsigned char *pt, + unsigned char *tweak, + const symmetric_xts *xts); + +void xts_done(symmetric_xts *xts); +int xts_test(void); +void xts_mult_x(unsigned char *I); +#endif + +int find_cipher(const char *name); +int find_cipher_any(const char *name, int blocklen, int keylen); +int find_cipher_id(unsigned char ID); +int register_cipher(const struct ltc_cipher_descriptor *cipher); +int unregister_cipher(const struct ltc_cipher_descriptor *cipher); +int register_all_ciphers(void); +int cipher_is_valid(int idx); + +LTC_MUTEX_PROTO(ltc_cipher_mutex) + +/* ---- stream ciphers ---- */ + +#ifdef LTC_CHACHA + +typedef struct { + ulong32 input[16]; + unsigned char kstream[64]; + unsigned long ksleft; + unsigned long ivlen; + int rounds; +} chacha_state; + +int chacha_setup(chacha_state *st, const unsigned char *key, unsigned long keylen, int rounds); +int chacha_ivctr32(chacha_state *st, const unsigned char *iv, unsigned long ivlen, ulong32 counter); +int chacha_ivctr64(chacha_state *st, const unsigned char *iv, unsigned long ivlen, ulong64 counter); +int chacha_crypt(chacha_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int chacha_keystream(chacha_state *st, unsigned char *out, unsigned long outlen); +int chacha_done(chacha_state *st); +int chacha_test(void); +int chacha_memory(const unsigned char *key, unsigned long keylen, unsigned long rounds, + const unsigned char *iv, unsigned long ivlen, ulong64 counter, + const unsigned char *datain, unsigned long datalen, unsigned char *dataout); + +#endif /* LTC_CHACHA */ + +#ifdef LTC_SALSA20 + +typedef struct { + ulong32 input[16]; + unsigned char kstream[64]; + unsigned long ksleft; + unsigned long ivlen; + int rounds; +} salsa20_state; + +int salsa20_setup(salsa20_state *st, const unsigned char *key, unsigned long keylen, int rounds); +int salsa20_ivctr64(salsa20_state *st, const unsigned char *iv, unsigned long ivlen, ulong64 counter); +int salsa20_crypt(salsa20_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int salsa20_keystream(salsa20_state *st, unsigned char *out, unsigned long outlen); +int salsa20_done(salsa20_state *st); +int salsa20_test(void); +int salsa20_memory(const unsigned char *key, unsigned long keylen, unsigned long rounds, + const unsigned char *iv, unsigned long ivlen, ulong64 counter, + const unsigned char *datain, unsigned long datalen, unsigned char *dataout); + +#endif /* LTC_SALSA20 */ + +#ifdef LTC_XSALSA20 + +int xsalsa20_setup(salsa20_state *st, const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + int rounds); +int xsalsa20_test(void); +int xsalsa20_memory(const unsigned char *key, unsigned long keylen, unsigned long rounds, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *datain, unsigned long datalen, unsigned char *dataout); + +#endif /* LTC_XSALSA20 */ + +#ifdef LTC_SOSEMANUK + +typedef struct { + ulong32 kc[100]; /* key_context */ + ulong32 s00, s01, s02, s03, s04, s05, s06, s07, s08, s09; + ulong32 r1, r2; + /* + * Buffering: the stream cipher produces output data by + * blocks of 640 bits. buf[] contains such a block, and + * "ptr" is the index of the next output byte. + */ + unsigned char buf[80]; + unsigned ptr; +} sosemanuk_state; + +int sosemanuk_setup(sosemanuk_state *st, const unsigned char *key, unsigned long keylen); +int sosemanuk_setiv(sosemanuk_state *st, const unsigned char *iv, unsigned long ivlen); +int sosemanuk_crypt(sosemanuk_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int sosemanuk_keystream(sosemanuk_state *st, unsigned char *out, unsigned long outlen); +int sosemanuk_done(sosemanuk_state *st); +int sosemanuk_test(void); +int sosemanuk_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout); + +#endif /* LTC_SOSEMANUK */ + +#ifdef LTC_RABBIT + +typedef struct { + ulong32 x[8]; + ulong32 c[8]; + ulong32 carry; +} rabbit_ctx; + +typedef struct { + rabbit_ctx master_ctx; + rabbit_ctx work_ctx; + unsigned char block[16]; /* last keystream block containing unused bytes */ + ulong32 unused; /* count fm right */ +} rabbit_state; + +int rabbit_setup(rabbit_state* st, const unsigned char *key, unsigned long keylen); +int rabbit_setiv(rabbit_state* st, const unsigned char *iv, unsigned long ivlen); +int rabbit_crypt(rabbit_state* st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int rabbit_keystream(rabbit_state* st, unsigned char *out, unsigned long outlen); +int rabbit_done(rabbit_state *st); +int rabbit_test(void); +int rabbit_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout); + +#endif /* LTC_RABBIT */ + +#ifdef LTC_RC4_STREAM + +typedef struct { + unsigned int x, y; + unsigned char buf[256]; +} rc4_state; + +int rc4_stream_setup(rc4_state *st, const unsigned char *key, unsigned long keylen); +int rc4_stream_crypt(rc4_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int rc4_stream_keystream(rc4_state *st, unsigned char *out, unsigned long outlen); +int rc4_stream_done(rc4_state *st); +int rc4_stream_test(void); +int rc4_stream_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout); + +#endif /* LTC_RC4_STREAM */ + +#ifdef LTC_SOBER128_STREAM + +typedef struct { + ulong32 R[17], /* Working storage for the shift register */ + initR[17], /* saved register contents */ + konst, /* key dependent constant */ + sbuf; /* partial word encryption buffer */ + int nbuf; /* number of part-word stream bits buffered */ +} sober128_state; + +int sober128_stream_setup(sober128_state *st, const unsigned char *key, unsigned long keylen); +int sober128_stream_setiv(sober128_state *st, const unsigned char *iv, unsigned long ivlen); +int sober128_stream_crypt(sober128_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int sober128_stream_keystream(sober128_state *st, unsigned char *out, unsigned long outlen); +int sober128_stream_done(sober128_state *st); +int sober128_stream_test(void); +int sober128_stream_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout); + +#endif /* LTC_SOBER128_STREAM */ diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_custom.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_custom.h new file mode 100644 index 0000000..62eab00 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_custom.h @@ -0,0 +1,751 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#ifndef TOMCRYPT_CUSTOM_H_ +#define TOMCRYPT_CUSTOM_H_ + +/* macros for various libc functions you can change for embedded targets */ +#ifndef XMALLOC +#define XMALLOC malloc +#endif +#ifndef XREALLOC +#define XREALLOC realloc +#endif +#ifndef XCALLOC +#define XCALLOC calloc +#endif +#ifndef XFREE +#define XFREE free +#endif + +#ifndef XMEMSET +#define XMEMSET memset +#endif +#ifndef XMEMCPY +#define XMEMCPY memcpy +#endif +#ifndef XMEMMOVE +#define XMEMMOVE memmove +#endif +#ifndef XMEMCMP +#define XMEMCMP memcmp +#endif +/* A memory compare function that has to run in constant time, + * c.f. mem_neq() API summary. + */ +#ifndef XMEM_NEQ +#define XMEM_NEQ mem_neq +#endif +#ifndef XSTRCMP +#define XSTRCMP strcmp +#endif +#ifndef XSTRLEN +#define XSTRLEN strlen +#endif +#ifndef XSTRNCPY +#define XSTRNCPY strncpy +#endif + +#ifndef XCLOCK +#define XCLOCK clock +#endif + +#ifndef XQSORT +#define XQSORT qsort +#endif + +#if ( defined(malloc) || defined(realloc) || defined(calloc) || defined(free) || \ + defined(memset) || defined(memcpy) || defined(memcmp) || defined(strcmp) || \ + defined(strlen) || defined(strncpy) || defined(clock) || defined(qsort) ) \ + && !defined(LTC_NO_PROTOTYPES) +#define LTC_NO_PROTOTYPES +#endif + +/* shortcut to disable automatic inclusion */ +#if defined LTC_NOTHING && !defined LTC_EASY + #define LTC_NO_CIPHERS + #define LTC_NO_MODES + #define LTC_NO_HASHES + #define LTC_NO_MACS + #define LTC_NO_PRNGS + #define LTC_NO_PK + #define LTC_NO_PKCS + #define LTC_NO_MISC +#endif /* LTC_NOTHING */ + +/* Easy button? */ +#ifdef LTC_EASY + #define LTC_NO_CIPHERS + #define LTC_RIJNDAEL + #define LTC_BLOWFISH + #define LTC_DES + #define LTC_CAST5 + + #define LTC_NO_MODES + #define LTC_ECB_MODE + #define LTC_CBC_MODE + #define LTC_CTR_MODE + + #define LTC_NO_HASHES + #define LTC_SHA1 + #define LTC_SHA3 + #define LTC_SHA512 + #define LTC_SHA384 + #define LTC_SHA256 + #define LTC_SHA224 + #define LTC_HASH_HELPERS + + #define LTC_NO_MACS + #define LTC_HMAC + #define LTC_OMAC + #define LTC_CCM_MODE + + #define LTC_NO_PRNGS + #define LTC_SPRNG + #define LTC_YARROW + #define LTC_DEVRANDOM + #define LTC_TRY_URANDOM_FIRST + #define LTC_RNG_GET_BYTES + #define LTC_RNG_MAKE_PRNG + + #define LTC_NO_PK + #define LTC_MRSA + #define LTC_MECC + + #define LTC_NO_MISC + #define LTC_BASE64 +#endif /* LTC_EASY */ + +/* The minimal set of functionality to run the tests */ +#ifdef LTC_MINIMAL + #define LTC_RIJNDAEL + #define LTC_SHA256 + #define LTC_YARROW + #define LTC_CTR_MODE + + #define LTC_RNG_MAKE_PRNG + #define LTC_RNG_GET_BYTES + #define LTC_DEVRANDOM + #define LTC_TRY_URANDOM_FIRST + + #undef LTC_NO_FILE +#endif /* LTC_MINIMAL */ + +/* Enable self-test test vector checking */ +#ifndef LTC_NO_TEST + #define LTC_TEST +#endif +/* Enable extended self-tests */ +/* #define LTC_TEST_EXT */ + +/* Use small code where possible */ +/* #define LTC_SMALL_CODE */ + +/* clean the stack of functions which put private information on stack */ +/* #define LTC_CLEAN_STACK */ + +/* disable all file related functions */ +/* #define LTC_NO_FILE */ + +/* disable all forms of ASM */ +/* #define LTC_NO_ASM */ + +/* disable FAST mode */ +/* #define LTC_NO_FAST */ + +/* disable BSWAP on x86 */ +/* #define LTC_NO_BSWAP */ + +/* ---> math provider? <--- */ +#ifndef LTC_NO_MATH + +/* LibTomMath */ +/* #define LTM_DESC */ + +/* TomsFastMath */ +/* #define TFM_DESC */ + +/* GNU Multiple Precision Arithmetic Library */ +/* #define GMP_DESC */ + +#endif /* LTC_NO_MATH */ + +/* ---> Symmetric Block Ciphers <--- */ +#ifndef LTC_NO_CIPHERS + +#define LTC_BLOWFISH +#define LTC_RC2 +#define LTC_RC5 +#define LTC_RC6 +#define LTC_SAFERP +#define LTC_RIJNDAEL +#define LTC_XTEA +/* _TABLES tells it to use tables during setup, _SMALL means to use the smaller scheduled key format + * (saves 4KB of ram), _ALL_TABLES enables all tables during setup */ +#define LTC_TWOFISH +#ifndef LTC_NO_TABLES + #define LTC_TWOFISH_TABLES + /* #define LTC_TWOFISH_ALL_TABLES */ +#else + #define LTC_TWOFISH_SMALL +#endif +/* #define LTC_TWOFISH_SMALL */ +/* LTC_DES includes EDE triple-DES */ +#define LTC_DES +#define LTC_CAST5 +#define LTC_NOEKEON +#define LTC_SKIPJACK +#define LTC_SAFER +#define LTC_KHAZAD +#define LTC_ANUBIS +#define LTC_ANUBIS_TWEAK +#define LTC_KSEED +#define LTC_KASUMI +#define LTC_MULTI2 +#define LTC_CAMELLIA +#define LTC_IDEA +#define LTC_SERPENT +#define LTC_TEA + +/* stream ciphers */ +#define LTC_CHACHA +#define LTC_SALSA20 +#define LTC_XSALSA20 +#define LTC_SOSEMANUK +#define LTC_RABBIT +#define LTC_RC4_STREAM +#define LTC_SOBER128_STREAM + +#endif /* LTC_NO_CIPHERS */ + + +/* ---> Block Cipher Modes of Operation <--- */ +#ifndef LTC_NO_MODES + +#define LTC_CFB_MODE +#define LTC_OFB_MODE +#define LTC_ECB_MODE +#define LTC_CBC_MODE +#define LTC_CTR_MODE + +/* F8 chaining mode */ +#define LTC_F8_MODE + +/* LRW mode */ +#define LTC_LRW_MODE +#ifndef LTC_NO_TABLES + /* like GCM mode this will enable 16 8x128 tables [64KB] that make + * seeking very fast. + */ + #define LTC_LRW_TABLES +#endif + +/* XTS mode */ +#define LTC_XTS_MODE + +#endif /* LTC_NO_MODES */ + +/* ---> One-Way Hash Functions <--- */ +#ifndef LTC_NO_HASHES + +#define LTC_CHC_HASH +#define LTC_WHIRLPOOL +#define LTC_SHA3 +#define LTC_KECCAK +#define LTC_SHA512 +#define LTC_SHA512_256 +#define LTC_SHA512_224 +#define LTC_SHA384 +#define LTC_SHA256 +#define LTC_SHA224 +#define LTC_TIGER +#define LTC_SHA1 +#define LTC_MD5 +#define LTC_MD4 +#define LTC_MD2 +#define LTC_RIPEMD128 +#define LTC_RIPEMD160 +#define LTC_RIPEMD256 +#define LTC_RIPEMD320 +#define LTC_BLAKE2S +#define LTC_BLAKE2B + +#define LTC_HASH_HELPERS + +#endif /* LTC_NO_HASHES */ + + +/* ---> MAC functions <--- */ +#ifndef LTC_NO_MACS + +#define LTC_HMAC +#define LTC_OMAC +#define LTC_PMAC +#define LTC_XCBC +#define LTC_F9_MODE +#define LTC_PELICAN +#define LTC_POLY1305 +#define LTC_BLAKE2SMAC +#define LTC_BLAKE2BMAC + +/* ---> Encrypt + Authenticate Modes <--- */ + +#define LTC_EAX_MODE + +#define LTC_OCB_MODE +#define LTC_OCB3_MODE +#define LTC_CCM_MODE +#define LTC_GCM_MODE +#define LTC_CHACHA20POLY1305_MODE + +/* Use 64KiB tables */ +#ifndef LTC_NO_TABLES + #define LTC_GCM_TABLES +#endif + +/* USE SSE2? requires GCC works on x86_32 and x86_64*/ +#ifdef LTC_GCM_TABLES +/* #define LTC_GCM_TABLES_SSE2 */ +#endif + +#endif /* LTC_NO_MACS */ + + +/* --> Pseudo Random Number Generators <--- */ +#ifndef LTC_NO_PRNGS + +/* Yarrow */ +#define LTC_YARROW + +/* a PRNG that simply reads from an available system source */ +#define LTC_SPRNG + +/* The RC4 stream cipher based PRNG */ +#define LTC_RC4 + +/* The ChaCha20 stream cipher based PRNG */ +#define LTC_CHACHA20_PRNG + +/* Fortuna PRNG */ +#define LTC_FORTUNA + +/* Greg's SOBER128 stream cipher based PRNG */ +#define LTC_SOBER128 + +/* the *nix style /dev/random device */ +#define LTC_DEVRANDOM +/* try /dev/urandom before trying /dev/random + * are you sure you want to disable this? http://www.2uo.de/myths-about-urandom/ */ +#define LTC_TRY_URANDOM_FIRST +/* rng_get_bytes() */ +#define LTC_RNG_GET_BYTES +/* rng_make_prng() */ +#define LTC_RNG_MAKE_PRNG + +/* enable the ltc_rng hook to integrate e.g. embedded hardware RNG's easily */ +/* #define LTC_PRNG_ENABLE_LTC_RNG */ + +#endif /* LTC_NO_PRNGS */ + +#ifdef LTC_YARROW + +/* which descriptor of AES to use? */ +/* 0 = rijndael_enc 1 = aes_enc, 2 = rijndael [full], 3 = aes [full] */ +#ifdef ENCRYPT_ONLY + #define LTC_YARROW_AES 0 +#else + #define LTC_YARROW_AES 2 +#endif + +#endif /* LTC_YARROW */ + +#ifdef LTC_FORTUNA + +#if !defined(LTC_FORTUNA_RESEED_RATELIMIT_STATIC) && \ + ((defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || defined(_WIN32)) + +/* time-based rate limit of the reseeding */ +#define LTC_FORTUNA_RESEED_RATELIMIT_TIMED + +/* with non-glibc or glibc 2.17+ prefer clock_gettime over gettimeofday */ +#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) +#if __GLIBC_PREREQ(2, 17) + #define LTC_CLOCK_GETTIME +#endif +#elif defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L + #define LTC_CLOCK_GETTIME +#endif + +#else + +#ifndef LTC_FORTUNA_WD +/* reseed every N calls to the read function */ +#define LTC_FORTUNA_WD 10 +#endif + +#ifdef LTC_FORTUNA_RESEED_RATELIMIT_TIMED +/* make sure only one of + * LTC_FORTUNA_RESEED_RATELIMIT_STATIC + * and + * LTC_FORTUNA_RESEED_RATELIMIT_TIMED + * is defined. + */ +#undef LTC_FORTUNA_RESEED_RATELIMIT_TIMED +#warning "undef'ed LTC_FORTUNA_RESEED_RATELIMIT_TIMED, looks like your architecture doesn't support it" +#endif + +#endif + +#ifndef LTC_FORTUNA_POOLS +/* number of pools (4..32) can save a bit of ram by lowering the count */ +#define LTC_FORTUNA_POOLS 32 +#endif + +#endif /* LTC_FORTUNA */ + + +/* ---> Public Key Crypto <--- */ +#ifndef LTC_NO_PK + +/* Include RSA support */ +#define LTC_MRSA + +/* Include Diffie-Hellman support */ +/* is_prime fails for GMP */ +#define LTC_MDH +/* Supported Key Sizes */ +#define LTC_DH768 +#define LTC_DH1024 +#define LTC_DH1536 +#define LTC_DH2048 + +#if defined(LTM_DESC) || defined(GMP_DESC) +/* tfm has a problem in fp_isprime for larger key sizes */ +#define LTC_DH3072 +#define LTC_DH4096 +#define LTC_DH6144 +#define LTC_DH8192 +#endif + +/* Digital Signature Algorithm */ +#define LTC_MDSA + +/* Ed25519 & X25519 */ +#define LTC_CURVE25519 + +/* ECC */ +#define LTC_MECC + +/* use Shamir's trick for point mul (speeds up signature verification) */ +#define LTC_ECC_SHAMIR + +#if defined(TFM_DESC) && defined(LTC_MECC) + #define LTC_MECC_ACCEL +#endif + +/* do we want fixed point ECC */ +/* #define LTC_MECC_FP */ + +#endif /* LTC_NO_PK */ + +#if defined(LTC_MRSA) && !defined(LTC_NO_RSA_BLINDING) +/* Enable RSA blinding when doing private key operations by default */ +#define LTC_RSA_BLINDING +#endif /* LTC_NO_RSA_BLINDING */ + +#if defined(LTC_MRSA) && !defined(LTC_NO_RSA_CRT_HARDENING) +/* Enable RSA CRT hardening when doing private key operations by default */ +#define LTC_RSA_CRT_HARDENING +#endif /* LTC_NO_RSA_CRT_HARDENING */ + +#if defined(LTC_MECC) && !defined(LTC_NO_ECC_TIMING_RESISTANT) +/* Enable ECC timing resistant version by default */ +#define LTC_ECC_TIMING_RESISTANT +#endif + +/* PKCS #1 (RSA) and #5 (Password Handling) stuff */ +#ifndef LTC_NO_PKCS + +#define LTC_PKCS_1 +#define LTC_PKCS_5 +#define LTC_PKCS_8 +#define LTC_PKCS_12 + +/* Include ASN.1 DER (required by DSA/RSA) */ +#define LTC_DER + +#define LTC_MPI + +#endif /* LTC_NO_PKCS */ + +/* misc stuff */ +#ifndef LTC_NO_MISC + +/* Various tidbits of modern neatoness */ +#define LTC_BASE64 +/* ... and it's URL safe version */ +#define LTC_BASE64_URL +/* Base32 encoding/decoding */ +#define LTC_BASE32 +/* Base16/hex encoding/decoding */ +#define LTC_BASE16 + +#define LTC_BCRYPT + +#ifndef LTC_BCRYPT_DEFAULT_ROUNDS +#define LTC_BCRYPT_DEFAULT_ROUNDS 10 +#endif + +/* Keep LTC_NO_HKDF for compatibility reasons + * superseeded by LTC_NO_MISC*/ +#ifndef LTC_NO_HKDF +/* HKDF Key Derivation/Expansion stuff */ +#define LTC_HKDF +#endif /* LTC_NO_HKDF */ + +#define LTC_ADLER32 + +#define LTC_CRC32 + +#define LTC_SSH + +#define LTC_PADDING + +#define LTC_PBES + +#endif /* LTC_NO_MISC */ + +/* cleanup */ + +#ifdef LTC_MECC +/* Supported ECC Key Sizes */ +#ifndef LTC_NO_CURVES + #define LTC_ECC_BRAINPOOLP160R1 + #define LTC_ECC_BRAINPOOLP160T1 + #define LTC_ECC_BRAINPOOLP192R1 + #define LTC_ECC_BRAINPOOLP192T1 + #define LTC_ECC_BRAINPOOLP224R1 + #define LTC_ECC_BRAINPOOLP224T1 + #define LTC_ECC_BRAINPOOLP256R1 + #define LTC_ECC_BRAINPOOLP256T1 + #define LTC_ECC_BRAINPOOLP320R1 + #define LTC_ECC_BRAINPOOLP320T1 + #define LTC_ECC_BRAINPOOLP384R1 + #define LTC_ECC_BRAINPOOLP384T1 + #define LTC_ECC_BRAINPOOLP512R1 + #define LTC_ECC_BRAINPOOLP512T1 + #define LTC_ECC_PRIME192V2 + #define LTC_ECC_PRIME192V3 + #define LTC_ECC_PRIME239V1 + #define LTC_ECC_PRIME239V2 + #define LTC_ECC_PRIME239V3 + #define LTC_ECC_SECP112R1 + #define LTC_ECC_SECP112R2 + #define LTC_ECC_SECP128R1 + #define LTC_ECC_SECP128R2 + #define LTC_ECC_SECP160K1 + #define LTC_ECC_SECP160R1 + #define LTC_ECC_SECP160R2 + #define LTC_ECC_SECP192K1 + #define LTC_ECC_SECP192R1 + #define LTC_ECC_SECP224K1 + #define LTC_ECC_SECP224R1 + #define LTC_ECC_SECP256K1 + #define LTC_ECC_SECP256R1 + #define LTC_ECC_SECP384R1 + #define LTC_ECC_SECP521R1 +#endif +#endif /* LTC_MECC */ + +#if defined(LTC_DER) + #ifndef LTC_DER_MAX_RECURSION + /* Maximum recursion limit when processing nested ASN.1 types. */ + #define LTC_DER_MAX_RECURSION 30 + #endif +#endif + +#if defined(LTC_MECC) || defined(LTC_MRSA) || defined(LTC_MDSA) || defined(LTC_SSH) + #ifndef LTC_PK_MAX_RETRIES + /* iterations limit for retry-loops */ + #define LTC_PK_MAX_RETRIES 20 + #endif +#endif + +#ifdef LTC_MRSA + #define LTC_PKCS_1 +#endif + +#if defined(LTC_MRSA) || defined(LTC_MECC) + #define LTC_PKCS_8 +#endif + +#ifdef LTC_PKCS_8 + #define LTC_PADDING + #define LTC_PBES +#endif + +#if defined(LTC_CLEAN_STACK) +/* if you're sure that you want to use it, remove the line below */ + #error LTC_CLEAN_STACK is considered as broken +#endif + +#if defined(LTC_PBES) && !defined(LTC_PKCS_5) + #error LTC_PBES requires LTC_PKCS_5 +#endif + +#if defined(LTC_PBES) && !defined(LTC_PKCS_12) + #error LTC_PBES requires LTC_PKCS_12 +#endif + +#if defined(LTC_PKCS_5) && !defined(LTC_HMAC) + #error LTC_PKCS_5 requires LTC_HMAC +#endif + +#if defined(LTC_PKCS_5) && !defined(LTC_HASH_HELPERS) + #error LTC_PKCS_5 requires LTC_HASH_HELPERS +#endif + +#if defined(LTC_PELICAN) && !defined(LTC_RIJNDAEL) + #error Pelican-MAC requires LTC_RIJNDAEL +#endif + +#if defined(LTC_EAX_MODE) && !(defined(LTC_CTR_MODE) && defined(LTC_OMAC)) + #error LTC_EAX_MODE requires CTR and LTC_OMAC mode +#endif + +#if defined(LTC_YARROW) && !defined(LTC_CTR_MODE) + #error LTC_YARROW requires LTC_CTR_MODE chaining mode to be defined! +#endif + +#if defined(LTC_DER) && !defined(LTC_MPI) + #error ASN.1 DER requires MPI functionality +#endif + +#if (defined(LTC_MDSA) || defined(LTC_MRSA) || defined(LTC_MECC)) && !defined(LTC_DER) + #error PK requires ASN.1 DER functionality, make sure LTC_DER is enabled +#endif + +#if defined(LTC_BCRYPT) && !defined(LTC_BLOWFISH) + #error LTC_BCRYPT requires LTC_BLOWFISH +#endif + +#if defined(LTC_CHACHA20POLY1305_MODE) && (!defined(LTC_CHACHA) || !defined(LTC_POLY1305)) + #error LTC_CHACHA20POLY1305_MODE requires LTC_CHACHA + LTC_POLY1305 +#endif + +#if defined(LTC_CHACHA20_PRNG) && !defined(LTC_CHACHA) + #error LTC_CHACHA20_PRNG requires LTC_CHACHA +#endif + +#if defined(LTC_XSALSA20) && !defined(LTC_SALSA20) + #error LTC_XSALSA20 requires LTC_SALSA20 +#endif + +#if defined(LTC_RC4) && !defined(LTC_RC4_STREAM) + #error LTC_RC4 requires LTC_RC4_STREAM +#endif + +#if defined(LTC_SOBER128) && !defined(LTC_SOBER128_STREAM) + #error LTC_SOBER128 requires LTC_SOBER128_STREAM +#endif + +#if defined(LTC_BLAKE2SMAC) && !defined(LTC_BLAKE2S) + #error LTC_BLAKE2SMAC requires LTC_BLAKE2S +#endif + +#if defined(LTC_BLAKE2BMAC) && !defined(LTC_BLAKE2B) + #error LTC_BLAKE2BMAC requires LTC_BLAKE2B +#endif + +#if defined(LTC_SPRNG) && !defined(LTC_RNG_GET_BYTES) + #error LTC_SPRNG requires LTC_RNG_GET_BYTES +#endif + +#if defined(LTC_NO_MATH) && (defined(LTM_DESC) || defined(TFM_DESC) || defined(GMP_DESC)) + #error LTC_NO_MATH defined, but also a math descriptor +#endif + +/* THREAD management */ +#if defined(_CFG_CORE_LTC_OPTEE_THREAD) + +#include + +#define LTC_MUTEX_GLOBAL(x) struct mutex x = MUTEX_INITIALIZER; +#define LTC_MUTEX_PROTO(x) extern struct mutex x; +#define LTC_MUTEX_TYPE(x) struct mutex x; +#define LTC_MUTEX_INIT(x) mutex_init(x); +#define LTC_MUTEX_LOCK(x) mutex_lock(x); +#define LTC_MUTEX_UNLOCK(x) mutex_unlock(x); + +#elif defined(LTC_PTHREAD) + +#include + +#define LTC_MUTEX_GLOBAL(x) pthread_mutex_t x = PTHREAD_MUTEX_INITIALIZER; +#define LTC_MUTEX_PROTO(x) extern pthread_mutex_t x; +#define LTC_MUTEX_TYPE(x) pthread_mutex_t x; +#define LTC_MUTEX_INIT(x) LTC_ARGCHK(pthread_mutex_init(x, NULL) == 0); +#define LTC_MUTEX_LOCK(x) LTC_ARGCHK(pthread_mutex_lock(x) == 0); +#define LTC_MUTEX_UNLOCK(x) LTC_ARGCHK(pthread_mutex_unlock(x) == 0); +#define LTC_MUTEX_DESTROY(x) LTC_ARGCHK(pthread_mutex_destroy(x) == 0); + +#else + +/* default no functions */ +#define LTC_MUTEX_GLOBAL(x) +#define LTC_MUTEX_PROTO(x) +#define LTC_MUTEX_TYPE(x) +#define LTC_MUTEX_INIT(x) +#define LTC_MUTEX_LOCK(x) +#define LTC_MUTEX_UNLOCK(x) +#define LTC_MUTEX_DESTROY(x) + +#endif /* LTC_PTHREAD */ + +/* Debuggers */ + +/* define this if you use Valgrind, note: it CHANGES the way SOBER-128 and RC4 work (see the code) */ +/* #define LTC_VALGRIND */ + +#ifndef LTC_NO_FILE + /* buffer size for reading from a file via fread(..) */ + #ifndef LTC_FILE_READ_BUFSIZE + #define LTC_FILE_READ_BUFSIZE 8192 + #endif +#endif + +/* ECC backwards compatibility */ +#if !defined(LTC_ECC_SECP112R1) && defined(LTC_ECC112) +#define LTC_ECC_SECP112R1 +#undef LTC_ECC112 +#endif +#if !defined(LTC_ECC_SECP128R1) && defined(LTC_ECC128) +#define LTC_ECC_SECP128R1 +#undef LTC_ECC128 +#endif +#if !defined(LTC_ECC_SECP160R1) && defined(LTC_ECC160) +#define LTC_ECC_SECP160R1 +#undef LTC_ECC160 +#endif +#if !defined(LTC_ECC_SECP192R1) && defined(LTC_ECC192) +#define LTC_ECC_SECP192R1 +#undef LTC_ECC192 +#endif +#if !defined(LTC_ECC_SECP224R1) && defined(LTC_ECC224) +#define LTC_ECC_SECP224R1 +#undef LTC_ECC224 +#endif +#if !defined(LTC_ECC_SECP256R1) && defined(LTC_ECC256) +#define LTC_ECC_SECP256R1 +#undef LTC_ECC256 +#endif +#if !defined(LTC_ECC_SECP384R1) && defined(LTC_ECC384) +#define LTC_ECC_SECP384R1 +#undef LTC_ECC384 +#endif +#if !defined(LTC_ECC_SECP512R1) && defined(LTC_ECC521) +#define LTC_ECC_SECP521R1 +#undef LTC_ECC521 +#endif + +#endif /* TOMCRYPT_CUSTOM_H_ */ diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_hash.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_hash.h new file mode 100644 index 0000000..bc18b63 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_hash.h @@ -0,0 +1,503 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* ---- HASH FUNCTIONS ---- */ +#if defined(LTC_SHA3) || defined(LTC_KECCAK) +struct sha3_state { + ulong64 saved; /* the portion of the input message that we didn't consume yet */ + ulong64 s[25]; + unsigned char sb[25 * 8]; /* used for storing `ulong64 s[25]` as little-endian bytes */ + unsigned short byte_index; /* 0..7--the next byte after the set one (starts from 0; 0--none are buffered) */ + unsigned short word_index; /* 0..24--the next word to integrate input (starts from 0) */ + unsigned short capacity_words; /* the double size of the hash output in words (e.g. 16 for Keccak 512) */ + unsigned short xof_flag; +}; +#endif + +#ifdef LTC_SHA512 +struct sha512_state { + ulong64 length, state[8]; + unsigned long curlen; + unsigned char buf[128]; +}; +#endif + +#ifdef LTC_SHA256 +struct sha256_state { + ulong64 length; + ulong32 state[8], curlen; + unsigned char buf[64]; +}; +#endif + +#ifdef LTC_SHA1 +struct sha1_state { + ulong64 length; + ulong32 state[5], curlen; + unsigned char buf[64]; +}; +#endif + +#ifdef LTC_MD5 +struct md5_state { + ulong64 length; + ulong32 state[4], curlen; + unsigned char buf[64]; +}; +#endif + +#ifdef LTC_MD4 +struct md4_state { + ulong64 length; + ulong32 state[4], curlen; + unsigned char buf[64]; +}; +#endif + +#ifdef LTC_TIGER +struct tiger_state { + ulong64 state[3], length; + unsigned long curlen; + unsigned char buf[64]; +}; +#endif + +#ifdef LTC_MD2 +struct md2_state { + unsigned char chksum[16], X[48], buf[16]; + unsigned long curlen; +}; +#endif + +#ifdef LTC_RIPEMD128 +struct rmd128_state { + ulong64 length; + unsigned char buf[64]; + ulong32 curlen, state[4]; +}; +#endif + +#ifdef LTC_RIPEMD160 +struct rmd160_state { + ulong64 length; + unsigned char buf[64]; + ulong32 curlen, state[5]; +}; +#endif + +#ifdef LTC_RIPEMD256 +struct rmd256_state { + ulong64 length; + unsigned char buf[64]; + ulong32 curlen, state[8]; +}; +#endif + +#ifdef LTC_RIPEMD320 +struct rmd320_state { + ulong64 length; + unsigned char buf[64]; + ulong32 curlen, state[10]; +}; +#endif + +#ifdef LTC_WHIRLPOOL +struct whirlpool_state { + ulong64 length, state[8]; + unsigned char buf[64]; + ulong32 curlen; +}; +#endif + +#ifdef LTC_CHC_HASH +struct chc_state { + ulong64 length; + unsigned char state[MAXBLOCKSIZE], buf[MAXBLOCKSIZE]; + ulong32 curlen; +}; +#endif + +#ifdef LTC_BLAKE2S +struct blake2s_state { + ulong32 h[8]; + ulong32 t[2]; + ulong32 f[2]; + unsigned char buf[64]; + unsigned long curlen; + unsigned long outlen; + unsigned char last_node; +}; +#endif + +#ifdef LTC_BLAKE2B +struct blake2b_state { + ulong64 h[8]; + ulong64 t[2]; + ulong64 f[2]; + unsigned char buf[128]; + unsigned long curlen; + unsigned long outlen; + unsigned char last_node; +}; +#endif + +typedef union Hash_state { + char dummy[1]; +#ifdef LTC_CHC_HASH + struct chc_state chc; +#endif +#ifdef LTC_WHIRLPOOL + struct whirlpool_state whirlpool; +#endif +#if defined(LTC_SHA3) || defined(LTC_KECCAK) + struct sha3_state sha3; +#endif +#ifdef LTC_SHA512 + struct sha512_state sha512; +#endif +#ifdef LTC_SHA256 + struct sha256_state sha256; +#endif +#ifdef LTC_SHA1 + struct sha1_state sha1; +#endif +#ifdef LTC_MD5 + struct md5_state md5; +#endif +#ifdef LTC_MD4 + struct md4_state md4; +#endif +#ifdef LTC_MD2 + struct md2_state md2; +#endif +#ifdef LTC_TIGER + struct tiger_state tiger; +#endif +#ifdef LTC_RIPEMD128 + struct rmd128_state rmd128; +#endif +#ifdef LTC_RIPEMD160 + struct rmd160_state rmd160; +#endif +#ifdef LTC_RIPEMD256 + struct rmd256_state rmd256; +#endif +#ifdef LTC_RIPEMD320 + struct rmd320_state rmd320; +#endif +#ifdef LTC_BLAKE2S + struct blake2s_state blake2s; +#endif +#ifdef LTC_BLAKE2B + struct blake2b_state blake2b; +#endif + + void *data; +} hash_state; + +/** hash descriptor */ +extern const struct ltc_hash_descriptor { + /** name of hash */ + const char *name; + /** internal ID */ + unsigned char ID; + /** Size of digest in octets */ + unsigned long hashsize; + /** Input block size in octets */ + unsigned long blocksize; + /** ASN.1 OID */ + unsigned long OID[16]; + /** Length of DER encoding */ + unsigned long OIDlen; + + /** Init a hash state + @param hash The hash to initialize + @return CRYPT_OK if successful + */ + int (*init)(hash_state *hash); + /** Process a block of data + @param hash The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful + */ + int (*process)(hash_state *hash, const unsigned char *in, unsigned long inlen); + /** Produce the digest and store it + @param hash The hash state + @param out [out] The destination of the digest + @return CRYPT_OK if successful + */ + int (*done)(hash_state *hash, unsigned char *out); + /** Self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled + */ + int (*test)(void); + + /* accelerated hmac callback: if you need to-do multiple packets just use the generic hmac_memory and provide a hash callback */ + int (*hmac_block)(const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +} *hash_descriptor[]; + +#ifdef LTC_CHC_HASH +int chc_register(int cipher); +int chc_init(hash_state * md); +int chc_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int chc_done(hash_state * md, unsigned char *out); +int chc_test(void); +extern const struct ltc_hash_descriptor chc_desc; +#endif + +#ifdef LTC_WHIRLPOOL +int whirlpool_init(hash_state * md); +int whirlpool_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int whirlpool_done(hash_state * md, unsigned char *out); +int whirlpool_test(void); +extern const struct ltc_hash_descriptor whirlpool_desc; +#endif + +#if defined(LTC_SHA3) || defined(LTC_KECCAK) +/* sha3_NNN_init are shared by SHA3 and KECCAK */ +int sha3_512_init(hash_state * md); +int sha3_384_init(hash_state * md); +int sha3_256_init(hash_state * md); +int sha3_224_init(hash_state * md); +/* sha3_process is the same for all variants of SHA3 + KECCAK */ +int sha3_process(hash_state * md, const unsigned char *in, unsigned long inlen); +#endif + +#ifdef LTC_SHA3 +int sha3_512_test(void); +extern const struct ltc_hash_descriptor sha3_512_desc; +int sha3_384_test(void); +extern const struct ltc_hash_descriptor sha3_384_desc; +int sha3_256_test(void); +extern const struct ltc_hash_descriptor sha3_256_desc; +int sha3_224_test(void); +extern const struct ltc_hash_descriptor sha3_224_desc; +int sha3_done(hash_state *md, unsigned char *out); +/* SHAKE128 + SHAKE256 */ +int sha3_shake_init(hash_state *md, int num); +#define sha3_shake_process(a,b,c) sha3_process(a,b,c) +int sha3_shake_done(hash_state *md, unsigned char *out, unsigned long outlen); +int sha3_shake_test(void); +int sha3_shake_memory(int num, const unsigned char *in, unsigned long inlen, unsigned char *out, const unsigned long *outlen); +#endif + +#ifdef LTC_KECCAK +#define keccak_512_init(a) sha3_512_init(a) +#define keccak_384_init(a) sha3_384_init(a) +#define keccak_256_init(a) sha3_256_init(a) +#define keccak_224_init(a) sha3_224_init(a) +#define keccak_process(a,b,c) sha3_process(a,b,c) +extern const struct ltc_hash_descriptor keccak_512_desc; +int keccak_512_test(void); +extern const struct ltc_hash_descriptor keccak_384_desc; +int keccak_384_test(void); +extern const struct ltc_hash_descriptor keccak_256_desc; +int keccak_256_test(void); +extern const struct ltc_hash_descriptor keccak_224_desc; +int keccak_224_test(void); +int keccak_done(hash_state *md, unsigned char *out); +#endif + +#ifdef LTC_SHA512 +int sha512_init(hash_state * md); +int sha512_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int sha512_done(hash_state * md, unsigned char *out); +int sha512_test(void); +extern const struct ltc_hash_descriptor sha512_desc; +#endif + +#ifdef LTC_SHA384 +#ifndef LTC_SHA512 + #error LTC_SHA512 is required for LTC_SHA384 +#endif +int sha384_init(hash_state * md); +#define sha384_process sha512_process +int sha384_done(hash_state * md, unsigned char *out); +int sha384_test(void); +extern const struct ltc_hash_descriptor sha384_desc; +#endif + +#ifdef LTC_SHA512_256 +#ifndef LTC_SHA512 + #error LTC_SHA512 is required for LTC_SHA512_256 +#endif +int sha512_256_init(hash_state * md); +#define sha512_256_process sha512_process +int sha512_256_done(hash_state * md, unsigned char *out); +int sha512_256_test(void); +extern const struct ltc_hash_descriptor sha512_256_desc; +#endif + +#ifdef LTC_SHA512_224 +#ifndef LTC_SHA512 + #error LTC_SHA512 is required for LTC_SHA512_224 +#endif +int sha512_224_init(hash_state * md); +#define sha512_224_process sha512_process +int sha512_224_done(hash_state * md, unsigned char *out); +int sha512_224_test(void); +extern const struct ltc_hash_descriptor sha512_224_desc; +#endif + +#ifdef LTC_SHA256 +int sha256_init(hash_state * md); +int sha256_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int sha256_done(hash_state * md, unsigned char *out); +int sha256_test(void); +extern const struct ltc_hash_descriptor sha256_desc; + +#ifdef LTC_SHA224 +#ifndef LTC_SHA256 + #error LTC_SHA256 is required for LTC_SHA224 +#endif +int sha224_init(hash_state * md); +#define sha224_process sha256_process +int sha224_done(hash_state * md, unsigned char *out); +int sha224_test(void); +extern const struct ltc_hash_descriptor sha224_desc; +#endif +#endif + +#ifdef LTC_SHA1 +int sha1_init(hash_state * md); +int sha1_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int sha1_done(hash_state * md, unsigned char *out); +int sha1_test(void); +extern const struct ltc_hash_descriptor sha1_desc; +#endif + +#ifdef LTC_BLAKE2S +extern const struct ltc_hash_descriptor blake2s_256_desc; +int blake2s_256_init(hash_state * md); +int blake2s_256_test(void); + +extern const struct ltc_hash_descriptor blake2s_224_desc; +int blake2s_224_init(hash_state * md); +int blake2s_224_test(void); + +extern const struct ltc_hash_descriptor blake2s_160_desc; +int blake2s_160_init(hash_state * md); +int blake2s_160_test(void); + +extern const struct ltc_hash_descriptor blake2s_128_desc; +int blake2s_128_init(hash_state * md); +int blake2s_128_test(void); + +int blake2s_init(hash_state * md, unsigned long outlen, const unsigned char *key, unsigned long keylen); +int blake2s_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int blake2s_done(hash_state * md, unsigned char *out); +#endif + +#ifdef LTC_BLAKE2B +extern const struct ltc_hash_descriptor blake2b_512_desc; +int blake2b_512_init(hash_state * md); +int blake2b_512_test(void); + +extern const struct ltc_hash_descriptor blake2b_384_desc; +int blake2b_384_init(hash_state * md); +int blake2b_384_test(void); + +extern const struct ltc_hash_descriptor blake2b_256_desc; +int blake2b_256_init(hash_state * md); +int blake2b_256_test(void); + +extern const struct ltc_hash_descriptor blake2b_160_desc; +int blake2b_160_init(hash_state * md); +int blake2b_160_test(void); + +int blake2b_init(hash_state * md, unsigned long outlen, const unsigned char *key, unsigned long keylen); +int blake2b_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int blake2b_done(hash_state * md, unsigned char *out); +#endif + +#ifdef LTC_MD5 +int md5_init(hash_state * md); +int md5_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int md5_done(hash_state * md, unsigned char *out); +int md5_test(void); +extern const struct ltc_hash_descriptor md5_desc; +#endif + +#ifdef LTC_MD4 +int md4_init(hash_state * md); +int md4_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int md4_done(hash_state * md, unsigned char *out); +int md4_test(void); +extern const struct ltc_hash_descriptor md4_desc; +#endif + +#ifdef LTC_MD2 +int md2_init(hash_state * md); +int md2_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int md2_done(hash_state * md, unsigned char *out); +int md2_test(void); +extern const struct ltc_hash_descriptor md2_desc; +#endif + +#ifdef LTC_TIGER +int tiger_init(hash_state * md); +int tiger_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int tiger_done(hash_state * md, unsigned char *out); +int tiger_test(void); +extern const struct ltc_hash_descriptor tiger_desc; +#endif + +#ifdef LTC_RIPEMD128 +int rmd128_init(hash_state * md); +int rmd128_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int rmd128_done(hash_state * md, unsigned char *out); +int rmd128_test(void); +extern const struct ltc_hash_descriptor rmd128_desc; +#endif + +#ifdef LTC_RIPEMD160 +int rmd160_init(hash_state * md); +int rmd160_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int rmd160_done(hash_state * md, unsigned char *out); +int rmd160_test(void); +extern const struct ltc_hash_descriptor rmd160_desc; +#endif + +#ifdef LTC_RIPEMD256 +int rmd256_init(hash_state * md); +int rmd256_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int rmd256_done(hash_state * md, unsigned char *out); +int rmd256_test(void); +extern const struct ltc_hash_descriptor rmd256_desc; +#endif + +#ifdef LTC_RIPEMD320 +int rmd320_init(hash_state * md); +int rmd320_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int rmd320_done(hash_state * md, unsigned char *out); +int rmd320_test(void); +extern const struct ltc_hash_descriptor rmd320_desc; +#endif + + +int find_hash(const char *name); +int find_hash_id(unsigned char ID); +int find_hash_oid(const unsigned long *ID, unsigned long IDlen); +int find_hash_any(const char *name, int digestlen); +int register_hash(const struct ltc_hash_descriptor *hash); +int unregister_hash(const struct ltc_hash_descriptor *hash); +int register_all_hashes(void); +int hash_is_valid(int idx); + +LTC_MUTEX_PROTO(ltc_hash_mutex) + +int hash_memory(int hash, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int hash_memory_multi(int hash, unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; + +#ifndef LTC_NO_FILE +int hash_filehandle(int hash, FILE *in, unsigned char *out, unsigned long *outlen); +int hash_file(int hash, const char *fname, unsigned char *out, unsigned long *outlen); +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_mac.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_mac.h new file mode 100644 index 0000000..c8f4a32 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_mac.h @@ -0,0 +1,567 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#ifdef LTC_HMAC +typedef struct Hmac_state { + hash_state md; + int hash; + unsigned char key[MAXBLOCKSIZE]; +} hmac_state; + +int hmac_init(hmac_state *hmac, int hash, const unsigned char *key, unsigned long keylen); +int hmac_process(hmac_state *hmac, const unsigned char *in, unsigned long inlen); +int hmac_done(hmac_state *hmac, unsigned char *out, unsigned long *outlen); +int hmac_test(void); +int hmac_memory(int hash, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int hmac_memory_multi(int hash, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; +int hmac_file(int hash, const char *fname, const unsigned char *key, + unsigned long keylen, + unsigned char *out, unsigned long *outlen); +#endif + +#ifdef LTC_OMAC + +typedef struct { + int cipher_idx, + buflen, + blklen; + unsigned char block[MAXBLOCKSIZE], + prev[MAXBLOCKSIZE], + Lu[2][MAXBLOCKSIZE]; + symmetric_key key; +} omac_state; + +int omac_init(omac_state *omac, int cipher, const unsigned char *key, unsigned long keylen); +int omac_process(omac_state *omac, const unsigned char *in, unsigned long inlen); +int omac_done(omac_state *omac, unsigned char *out, unsigned long *outlen); +int omac_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int omac_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; +int omac_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *filename, + unsigned char *out, unsigned long *outlen); +int omac_test(void); +#endif /* LTC_OMAC */ + +#ifdef LTC_PMAC + +typedef struct { + unsigned char Ls[32][MAXBLOCKSIZE], /* L shifted by i bits to the left */ + Li[MAXBLOCKSIZE], /* value of Li [current value, we calc from previous recall] */ + Lr[MAXBLOCKSIZE], /* L * x^-1 */ + block[MAXBLOCKSIZE], /* currently accumulated block */ + checksum[MAXBLOCKSIZE]; /* current checksum */ + + symmetric_key key; /* scheduled key for cipher */ + unsigned long block_index; /* index # for current block */ + int cipher_idx, /* cipher idx */ + block_len, /* length of block */ + buflen; /* number of bytes in the buffer */ +} pmac_state; + +int pmac_init(pmac_state *pmac, int cipher, const unsigned char *key, unsigned long keylen); +int pmac_process(pmac_state *pmac, const unsigned char *in, unsigned long inlen); +int pmac_done(pmac_state *pmac, unsigned char *out, unsigned long *outlen); + +int pmac_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +int pmac_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; + +int pmac_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *filename, + unsigned char *out, unsigned long *outlen); + +int pmac_test(void); + +/* internal functions */ +int pmac_ntz(unsigned long x); +void pmac_shift_xor(pmac_state *pmac); + +#endif /* PMAC */ + +#ifdef LTC_POLY1305 +typedef struct { + ulong32 r[5]; + ulong32 h[5]; + ulong32 pad[4]; + unsigned long leftover; + unsigned char buffer[16]; + int final; +} poly1305_state; + +int poly1305_init(poly1305_state *st, const unsigned char *key, unsigned long keylen); +int poly1305_process(poly1305_state *st, const unsigned char *in, unsigned long inlen); +int poly1305_done(poly1305_state *st, unsigned char *mac, unsigned long *maclen); +int poly1305_memory(const unsigned char *key, unsigned long keylen, const unsigned char *in, unsigned long inlen, unsigned char *mac, unsigned long *maclen); +int poly1305_memory_multi(const unsigned char *key, unsigned long keylen, + unsigned char *mac, unsigned long *maclen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; +int poly1305_file(const char *fname, const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen); +int poly1305_test(void); +#endif /* LTC_POLY1305 */ + +#ifdef LTC_BLAKE2SMAC +typedef hash_state blake2smac_state; +int blake2smac_init(blake2smac_state *st, unsigned long outlen, const unsigned char *key, unsigned long keylen); +int blake2smac_process(blake2smac_state *st, const unsigned char *in, unsigned long inlen); +int blake2smac_done(blake2smac_state *st, unsigned char *mac, unsigned long *maclen); +int blake2smac_memory(const unsigned char *key, unsigned long keylen, const unsigned char *in, unsigned long inlen, unsigned char *mac, unsigned long *maclen); +int blake2smac_memory_multi(const unsigned char *key, unsigned long keylen, + unsigned char *mac, unsigned long *maclen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; +int blake2smac_file(const char *fname, const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen); +int blake2smac_test(void); +#endif /* LTC_BLAKE2SMAC */ + +#ifdef LTC_BLAKE2BMAC +typedef hash_state blake2bmac_state; +int blake2bmac_init(blake2bmac_state *st, unsigned long outlen, const unsigned char *key, unsigned long keylen); +int blake2bmac_process(blake2bmac_state *st, const unsigned char *in, unsigned long inlen); +int blake2bmac_done(blake2bmac_state *st, unsigned char *mac, unsigned long *maclen); +int blake2bmac_memory(const unsigned char *key, unsigned long keylen, const unsigned char *in, unsigned long inlen, unsigned char *mac, unsigned long *maclen); +int blake2bmac_memory_multi(const unsigned char *key, unsigned long keylen, + unsigned char *mac, unsigned long *maclen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; +int blake2bmac_file(const char *fname, const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen); +int blake2bmac_test(void); +#endif /* LTC_BLAKE2BMAC */ + + +#ifdef LTC_PELICAN + +typedef struct pelican_state +{ + symmetric_key K; + unsigned char state[16]; + int buflen; +} pelican_state; + +int pelican_init(pelican_state *pelmac, const unsigned char *key, unsigned long keylen); +int pelican_process(pelican_state *pelmac, const unsigned char *in, unsigned long inlen); +int pelican_done(pelican_state *pelmac, unsigned char *out); +int pelican_test(void); + +int pelican_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out); + +#endif + +#ifdef LTC_XCBC + +/* add this to "keylen" to xcbc_init to use a pure three-key XCBC MAC */ +#define LTC_XCBC_PURE 0x8000UL + +typedef struct { + unsigned char K[3][MAXBLOCKSIZE], + IV[MAXBLOCKSIZE]; + + symmetric_key key; + + int cipher, + buflen, + blocksize; +} xcbc_state; + +int xcbc_init(xcbc_state *xcbc, int cipher, const unsigned char *key, unsigned long keylen); +int xcbc_process(xcbc_state *xcbc, const unsigned char *in, unsigned long inlen); +int xcbc_done(xcbc_state *xcbc, unsigned char *out, unsigned long *outlen); +int xcbc_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int xcbc_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; +int xcbc_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *filename, + unsigned char *out, unsigned long *outlen); +int xcbc_test(void); + +#endif + +#ifdef LTC_F9_MODE + +typedef struct { + unsigned char akey[MAXBLOCKSIZE], + ACC[MAXBLOCKSIZE], + IV[MAXBLOCKSIZE]; + + symmetric_key key; + + int cipher, + buflen, + keylen, + blocksize; +} f9_state; + +int f9_init(f9_state *f9, int cipher, const unsigned char *key, unsigned long keylen); +int f9_process(f9_state *f9, const unsigned char *in, unsigned long inlen); +int f9_done(f9_state *f9, unsigned char *out, unsigned long *outlen); +int f9_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int f9_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) + LTC_NULL_TERMINATED; +int f9_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *fname, + unsigned char *out, unsigned long *outlen); +int f9_test(void); + +#endif + +/* + * ENC+AUTH modes + */ + +#ifdef LTC_EAX_MODE + +#if !(defined(LTC_OMAC) && defined(LTC_CTR_MODE)) + #error LTC_EAX_MODE requires LTC_OMAC and CTR +#endif + +typedef struct { + unsigned char N[MAXBLOCKSIZE]; + symmetric_CTR ctr; + omac_state headeromac, ctomac; +} eax_state; + +int eax_init(eax_state *eax, int cipher, const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen); + +int eax_encrypt(eax_state *eax, const unsigned char *pt, unsigned char *ct, unsigned long length); +int eax_decrypt(eax_state *eax, const unsigned char *ct, unsigned char *pt, unsigned long length); +int eax_addheader(eax_state *eax, const unsigned char *header, unsigned long length); +int eax_done(eax_state *eax, unsigned char *tag, unsigned long *taglen); + +int eax_encrypt_authenticate_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen, + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen); + +int eax_decrypt_verify_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, + int *stat); + + int eax_test(void); +#endif /* EAX MODE */ + +#ifdef LTC_OCB_MODE +typedef struct { + unsigned char L[MAXBLOCKSIZE], /* L value */ + Ls[32][MAXBLOCKSIZE], /* L shifted by i bits to the left */ + Li[MAXBLOCKSIZE], /* value of Li [current value, we calc from previous recall] */ + Lr[MAXBLOCKSIZE], /* L * x^-1 */ + R[MAXBLOCKSIZE], /* R value */ + checksum[MAXBLOCKSIZE]; /* current checksum */ + + symmetric_key key; /* scheduled key for cipher */ + unsigned long block_index; /* index # for current block */ + int cipher, /* cipher idx */ + block_len; /* length of block */ +} ocb_state; + +int ocb_init(ocb_state *ocb, int cipher, + const unsigned char *key, unsigned long keylen, const unsigned char *nonce); + +int ocb_encrypt(ocb_state *ocb, const unsigned char *pt, unsigned char *ct); +int ocb_decrypt(ocb_state *ocb, const unsigned char *ct, unsigned char *pt); + +int ocb_done_encrypt(ocb_state *ocb, + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen); + +int ocb_done_decrypt(ocb_state *ocb, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, int *stat); + +int ocb_encrypt_authenticate_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen); + +int ocb_decrypt_verify_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, + int *stat); + +int ocb_test(void); + +/* internal functions */ +void ocb_shift_xor(ocb_state *ocb, unsigned char *Z); +int ocb_ntz(unsigned long x); +int s_ocb_done(ocb_state *ocb, const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, unsigned char *tag, unsigned long *taglen, int mode); + +#endif /* LTC_OCB_MODE */ + +#ifdef LTC_OCB3_MODE +typedef struct { + unsigned char Offset_0[MAXBLOCKSIZE], /* Offset_0 value */ + Offset_current[MAXBLOCKSIZE], /* Offset_{current_block_index} value */ + L_dollar[MAXBLOCKSIZE], /* L_$ value */ + L_star[MAXBLOCKSIZE], /* L_* value */ + L_[32][MAXBLOCKSIZE], /* L_{i} values */ + tag_part[MAXBLOCKSIZE], /* intermediate result of tag calculation */ + checksum[MAXBLOCKSIZE]; /* current checksum */ + + /* AAD related members */ + unsigned char aSum_current[MAXBLOCKSIZE], /* AAD related helper variable */ + aOffset_current[MAXBLOCKSIZE], /* AAD related helper variable */ + adata_buffer[MAXBLOCKSIZE]; /* AAD buffer */ + int adata_buffer_bytes; /* bytes in AAD buffer */ + unsigned long ablock_index; /* index # for current adata (AAD) block */ + + symmetric_key key; /* scheduled key for cipher */ + unsigned long block_index; /* index # for current data block */ + int cipher, /* cipher idx */ + tag_len, /* length of tag */ + block_len; /* length of block */ +} ocb3_state; + +int ocb3_init(ocb3_state *ocb, int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + unsigned long taglen); + +int ocb3_encrypt(ocb3_state *ocb, const unsigned char *pt, unsigned long ptlen, unsigned char *ct); +int ocb3_decrypt(ocb3_state *ocb, const unsigned char *ct, unsigned long ctlen, unsigned char *pt); +int ocb3_encrypt_last(ocb3_state *ocb, const unsigned char *pt, unsigned long ptlen, unsigned char *ct); +int ocb3_decrypt_last(ocb3_state *ocb, const unsigned char *ct, unsigned long ctlen, unsigned char *pt); +int ocb3_add_aad(ocb3_state *ocb, const unsigned char *aad, unsigned long aadlen); +int ocb3_done(ocb3_state *ocb, unsigned char *tag, unsigned long *taglen); + +int ocb3_encrypt_authenticate_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *adata, unsigned long adatalen, + const unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen); + +int ocb3_decrypt_verify_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *adata, unsigned long adatalen, + const unsigned char *ct, unsigned long ctlen, + unsigned char *pt, + const unsigned char *tag, unsigned long taglen, + int *stat); + +int ocb3_test(void); + +#endif /* LTC_OCB3_MODE */ + +#ifdef LTC_CCM_MODE + +#define CCM_ENCRYPT LTC_ENCRYPT +#define CCM_DECRYPT LTC_DECRYPT + +typedef struct { + symmetric_key K; + int cipher, /* which cipher */ + taglen, /* length of the tag (encoded in M value) */ + x; /* index in PAD */ + + unsigned long L, /* L value */ + ptlen, /* length that will be enc / dec */ + current_ptlen, /* current processed length */ + aadlen, /* length of the aad */ + current_aadlen, /* length of the currently provided add */ + noncelen; /* length of the nonce */ + + unsigned char PAD[16], /* flags | Nonce N | l(m) */ + ctr[16], + CTRPAD[16], + CTRlen; +} ccm_state; + +int ccm_init(ccm_state *ccm, int cipher, + const unsigned char *key, int keylen, int ptlen, int taglen, int aadlen); + +int ccm_reset(ccm_state *ccm); + +int ccm_add_nonce(ccm_state *ccm, + const unsigned char *nonce, unsigned long noncelen); + +int ccm_add_aad(ccm_state *ccm, + const unsigned char *adata, unsigned long adatalen); + +int ccm_process(ccm_state *ccm, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + int direction); + +int ccm_done(ccm_state *ccm, + unsigned char *tag, unsigned long *taglen); + +int ccm_memory(int cipher, + const unsigned char *key, unsigned long keylen, + symmetric_key *uskey, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *header, unsigned long headerlen, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen, + int direction); + +int ccm_test(void); + +#endif /* LTC_CCM_MODE */ + +#if defined(LRW_MODE) || defined(LTC_GCM_MODE) +void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c); +#endif + + +/* table shared between GCM and LRW */ +#if defined(LTC_GCM_TABLES) || defined(LTC_LRW_TABLES) || ((defined(LTC_GCM_MODE) || defined(LTC_GCM_MODE)) && defined(LTC_FAST)) +extern const unsigned char gcm_shift_table[]; +#endif + +#ifdef LTC_GCM_MODE + +#define GCM_ENCRYPT LTC_ENCRYPT +#define GCM_DECRYPT LTC_DECRYPT + +#define LTC_GCM_MODE_IV 0 +#define LTC_GCM_MODE_AAD 1 +#define LTC_GCM_MODE_TEXT 2 + +typedef struct { + symmetric_key K; + unsigned char H[16], /* multiplier */ + X[16], /* accumulator */ + Y[16], /* counter */ + Y_0[16], /* initial counter */ + buf[16]; /* buffer for stuff */ + + int cipher, /* which cipher */ + ivmode, /* Which mode is the IV in? */ + mode, /* mode the GCM code is in */ + buflen; /* length of data in buf */ + + ulong64 totlen, /* 64-bit counter used for IV and AAD */ + pttotlen; /* 64-bit counter for the PT */ + +#ifdef LTC_GCM_TABLES + unsigned char PC[16][256][16] /* 16 tables of 8x128 */ +#ifdef LTC_GCM_TABLES_SSE2 +LTC_ALIGN(16) +#endif +; +#endif +} gcm_state; + +void gcm_mult_h(const gcm_state *gcm, unsigned char *I); + +int gcm_init(gcm_state *gcm, int cipher, + const unsigned char *key, int keylen); + +int gcm_reset(gcm_state *gcm); + +int gcm_add_iv(gcm_state *gcm, + const unsigned char *IV, unsigned long IVlen); + +int gcm_add_aad(gcm_state *gcm, + const unsigned char *adata, unsigned long adatalen); + +int gcm_process(gcm_state *gcm, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + int direction); + +int gcm_done(gcm_state *gcm, + unsigned char *tag, unsigned long *taglen); + +int gcm_memory( int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *IV, unsigned long IVlen, + const unsigned char *adata, unsigned long adatalen, + unsigned char *pt, unsigned long ptlen, + unsigned char *ct, + unsigned char *tag, unsigned long *taglen, + int direction); +int gcm_test(void); + +#endif /* LTC_GCM_MODE */ + +#ifdef LTC_CHACHA20POLY1305_MODE + +typedef struct { + poly1305_state poly; + chacha_state chacha; + ulong64 aadlen; + ulong64 ctlen; + int aadflg; +} chacha20poly1305_state; + +#define CHACHA20POLY1305_ENCRYPT LTC_ENCRYPT +#define CHACHA20POLY1305_DECRYPT LTC_DECRYPT + +int chacha20poly1305_init(chacha20poly1305_state *st, const unsigned char *key, unsigned long keylen); +int chacha20poly1305_setiv(chacha20poly1305_state *st, const unsigned char *iv, unsigned long ivlen); +int chacha20poly1305_setiv_rfc7905(chacha20poly1305_state *st, const unsigned char *iv, unsigned long ivlen, ulong64 sequence_number); +int chacha20poly1305_add_aad(chacha20poly1305_state *st, const unsigned char *in, unsigned long inlen); +int chacha20poly1305_encrypt(chacha20poly1305_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int chacha20poly1305_decrypt(chacha20poly1305_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out); +int chacha20poly1305_done(chacha20poly1305_state *st, unsigned char *tag, unsigned long *taglen); +int chacha20poly1305_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *aad, unsigned long aadlen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, + unsigned char *tag, unsigned long *taglen, + int direction); +int chacha20poly1305_test(void); + +#endif /* LTC_CHACHA20POLY1305_MODE */ diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_macros.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_macros.h new file mode 100644 index 0000000..94aa7c3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_macros.h @@ -0,0 +1,454 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* ---- HELPER MACROS ---- */ +#ifdef ENDIAN_NEUTRAL + +#define STORE32L(x, y) \ + do { (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ + (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } while(0) + +#define LOAD32L(x, y) \ + do { x = ((ulong32)((y)[3] & 255)<<24) | \ + ((ulong32)((y)[2] & 255)<<16) | \ + ((ulong32)((y)[1] & 255)<<8) | \ + ((ulong32)((y)[0] & 255)); } while(0) + +#define STORE64L(x, y) \ + do { (y)[7] = (unsigned char)(((x)>>56)&255); (y)[6] = (unsigned char)(((x)>>48)&255); \ + (y)[5] = (unsigned char)(((x)>>40)&255); (y)[4] = (unsigned char)(((x)>>32)&255); \ + (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ + (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } while(0) + +#define LOAD64L(x, y) \ + do { x = (((ulong64)((y)[7] & 255))<<56)|(((ulong64)((y)[6] & 255))<<48)| \ + (((ulong64)((y)[5] & 255))<<40)|(((ulong64)((y)[4] & 255))<<32)| \ + (((ulong64)((y)[3] & 255))<<24)|(((ulong64)((y)[2] & 255))<<16)| \ + (((ulong64)((y)[1] & 255))<<8)|(((ulong64)((y)[0] & 255))); } while(0) + +#define STORE32H(x, y) \ + do { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \ + (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } while(0) + +#define LOAD32H(x, y) \ + do { x = ((ulong32)((y)[0] & 255)<<24) | \ + ((ulong32)((y)[1] & 255)<<16) | \ + ((ulong32)((y)[2] & 255)<<8) | \ + ((ulong32)((y)[3] & 255)); } while(0) + +#define STORE64H(x, y) \ +do { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ + (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ + (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ + (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } while(0) + +#define LOAD64H(x, y) \ +do { x = (((ulong64)((y)[0] & 255))<<56)|(((ulong64)((y)[1] & 255))<<48) | \ + (((ulong64)((y)[2] & 255))<<40)|(((ulong64)((y)[3] & 255))<<32) | \ + (((ulong64)((y)[4] & 255))<<24)|(((ulong64)((y)[5] & 255))<<16) | \ + (((ulong64)((y)[6] & 255))<<8)|(((ulong64)((y)[7] & 255))); } while(0) + + +#elif defined(ENDIAN_LITTLE) + +#ifdef LTC_HAVE_BSWAP_BUILTIN + +#define STORE32H(x, y) \ +do { ulong32 ttt = __builtin_bswap32 ((x)); \ + XMEMCPY ((y), &ttt, 4); } while(0) + +#define LOAD32H(x, y) \ +do { XMEMCPY (&(x), (y), 4); \ + (x) = __builtin_bswap32 ((x)); } while(0) + +#elif !defined(LTC_NO_BSWAP) && (defined(INTEL_CC) || (defined(__GNUC__) && (defined(__DJGPP__) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__i386__) || defined(__x86_64__)))) + +#define STORE32H(x, y) \ +asm __volatile__ ( \ + "bswapl %0 \n\t" \ + "movl %0,(%1)\n\t" \ + "bswapl %0 \n\t" \ + ::"r"(x), "r"(y): "memory"); + +#define LOAD32H(x, y) \ +asm __volatile__ ( \ + "movl (%1),%0\n\t" \ + "bswapl %0\n\t" \ + :"=r"(x): "r"(y): "memory"); + +#else + +#define STORE32H(x, y) \ + do { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \ + (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } while(0) + +#define LOAD32H(x, y) \ + do { x = ((ulong32)((y)[0] & 255)<<24) | \ + ((ulong32)((y)[1] & 255)<<16) | \ + ((ulong32)((y)[2] & 255)<<8) | \ + ((ulong32)((y)[3] & 255)); } while(0) + +#endif + +#ifdef LTC_HAVE_BSWAP_BUILTIN + +#define STORE64H(x, y) \ +do { ulong64 ttt = __builtin_bswap64 ((x)); \ + XMEMCPY ((y), &ttt, 8); } while(0) + +#define LOAD64H(x, y) \ +do { XMEMCPY (&(x), (y), 8); \ + (x) = __builtin_bswap64 ((x)); } while(0) + +/* x86_64 processor */ +#elif !defined(LTC_NO_BSWAP) && (defined(__GNUC__) && defined(__x86_64__)) + +#define STORE64H(x, y) \ +asm __volatile__ ( \ + "bswapq %0 \n\t" \ + "movq %0,(%1)\n\t" \ + "bswapq %0 \n\t" \ + ::"r"(x), "r"(y): "memory"); + +#define LOAD64H(x, y) \ +asm __volatile__ ( \ + "movq (%1),%0\n\t" \ + "bswapq %0\n\t" \ + :"=r"(x): "r"(y): "memory"); + +#else + +#define STORE64H(x, y) \ +do { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ + (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ + (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ + (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } while(0) + +#define LOAD64H(x, y) \ +do { x = (((ulong64)((y)[0] & 255))<<56)|(((ulong64)((y)[1] & 255))<<48) | \ + (((ulong64)((y)[2] & 255))<<40)|(((ulong64)((y)[3] & 255))<<32) | \ + (((ulong64)((y)[4] & 255))<<24)|(((ulong64)((y)[5] & 255))<<16) | \ + (((ulong64)((y)[6] & 255))<<8)|(((ulong64)((y)[7] & 255))); } while(0) + +#endif + +#ifdef ENDIAN_32BITWORD + +#define STORE32L(x, y) \ + do { ulong32 ttt = (x); XMEMCPY(y, &ttt, 4); } while(0) + +#define LOAD32L(x, y) \ + do { XMEMCPY(&(x), y, 4); } while(0) + +#define STORE64L(x, y) \ + do { (y)[7] = (unsigned char)(((x)>>56)&255); (y)[6] = (unsigned char)(((x)>>48)&255); \ + (y)[5] = (unsigned char)(((x)>>40)&255); (y)[4] = (unsigned char)(((x)>>32)&255); \ + (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ + (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } while(0) + +#define LOAD64L(x, y) \ + do { x = (((ulong64)((y)[7] & 255))<<56)|(((ulong64)((y)[6] & 255))<<48)| \ + (((ulong64)((y)[5] & 255))<<40)|(((ulong64)((y)[4] & 255))<<32)| \ + (((ulong64)((y)[3] & 255))<<24)|(((ulong64)((y)[2] & 255))<<16)| \ + (((ulong64)((y)[1] & 255))<<8)|(((ulong64)((y)[0] & 255))); } while(0) + +#else /* 64-bit words then */ + +#define STORE32L(x, y) \ + do { ulong32 ttt = (x); XMEMCPY(y, &ttt, 4); } while(0) + +#define LOAD32L(x, y) \ + do { XMEMCPY(&(x), y, 4); x &= 0xFFFFFFFF; } while(0) + +#define STORE64L(x, y) \ + do { ulong64 ttt = (x); XMEMCPY(y, &ttt, 8); } while(0) + +#define LOAD64L(x, y) \ + do { XMEMCPY(&(x), y, 8); } while(0) + +#endif /* ENDIAN_64BITWORD */ + +#elif defined(ENDIAN_BIG) + +#define STORE32L(x, y) \ + do { (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ + (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } while(0) + +#define LOAD32L(x, y) \ + do { x = ((ulong32)((y)[3] & 255)<<24) | \ + ((ulong32)((y)[2] & 255)<<16) | \ + ((ulong32)((y)[1] & 255)<<8) | \ + ((ulong32)((y)[0] & 255)); } while(0) + +#define STORE64L(x, y) \ +do { (y)[7] = (unsigned char)(((x)>>56)&255); (y)[6] = (unsigned char)(((x)>>48)&255); \ + (y)[5] = (unsigned char)(((x)>>40)&255); (y)[4] = (unsigned char)(((x)>>32)&255); \ + (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ + (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } while(0) + +#define LOAD64L(x, y) \ +do { x = (((ulong64)((y)[7] & 255))<<56)|(((ulong64)((y)[6] & 255))<<48) | \ + (((ulong64)((y)[5] & 255))<<40)|(((ulong64)((y)[4] & 255))<<32) | \ + (((ulong64)((y)[3] & 255))<<24)|(((ulong64)((y)[2] & 255))<<16) | \ + (((ulong64)((y)[1] & 255))<<8)|(((ulong64)((y)[0] & 255))); } while(0) + +#ifdef ENDIAN_32BITWORD + +#define STORE32H(x, y) \ + do { ulong32 ttt = (x); XMEMCPY(y, &ttt, 4); } while(0) + +#define LOAD32H(x, y) \ + do { XMEMCPY(&(x), y, 4); } while(0) + +#define STORE64H(x, y) \ + do { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ + (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ + (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ + (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } while(0) + +#define LOAD64H(x, y) \ + do { x = (((ulong64)((y)[0] & 255))<<56)|(((ulong64)((y)[1] & 255))<<48)| \ + (((ulong64)((y)[2] & 255))<<40)|(((ulong64)((y)[3] & 255))<<32)| \ + (((ulong64)((y)[4] & 255))<<24)|(((ulong64)((y)[5] & 255))<<16)| \ + (((ulong64)((y)[6] & 255))<<8)| (((ulong64)((y)[7] & 255))); } while(0) + +#else /* 64-bit words then */ + +#define STORE32H(x, y) \ + do { ulong32 ttt = (x); XMEMCPY(y, &ttt, 4); } while(0) + +#define LOAD32H(x, y) \ + do { XMEMCPY(&(x), y, 4); x &= 0xFFFFFFFF; } while(0) + +#define STORE64H(x, y) \ + do { ulong64 ttt = (x); XMEMCPY(y, &ttt, 8); } while(0) + +#define LOAD64H(x, y) \ + do { XMEMCPY(&(x), y, 8); } while(0) + +#endif /* ENDIAN_64BITWORD */ +#endif /* ENDIAN_BIG */ + +#define BSWAP(x) ( ((x>>24)&0x000000FFUL) | ((x<<24)&0xFF000000UL) | \ + ((x>>8)&0x0000FF00UL) | ((x<<8)&0x00FF0000UL) ) + + +/* 32-bit Rotates */ +#if defined(_MSC_VER) +#define LTC_ROx_BUILTIN + +/* instrinsic rotate */ +#include +#pragma intrinsic(_rotr,_rotl) +#define ROR(x,n) _rotr(x,n) +#define ROL(x,n) _rotl(x,n) +#define RORc(x,n) ROR(x,n) +#define ROLc(x,n) ROL(x,n) + +#elif defined(LTC_HAVE_ROTATE_BUILTIN) +#define LTC_ROx_BUILTIN + +#define ROR(x,n) __builtin_rotateright32(x,n) +#define ROL(x,n) __builtin_rotateleft32(x,n) +#define ROLc(x,n) ROL(x,n) +#define RORc(x,n) ROR(x,n) + +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) && !defined(INTEL_CC) && !defined(LTC_NO_ASM) +#define LTC_ROx_ASM + +static inline ulong32 ROL(ulong32 word, int i) +{ + asm ("roll %%cl,%0" + :"=r" (word) + :"0" (word),"c" (i)); + return word; +} + +static inline ulong32 ROR(ulong32 word, int i) +{ + asm ("rorl %%cl,%0" + :"=r" (word) + :"0" (word),"c" (i)); + return word; +} + +#ifndef LTC_NO_ROLC + +#define ROLc(word,i) ({ \ + ulong32 ROLc_tmp = (word); \ + __asm__ ("roll %2, %0" : \ + "=r" (ROLc_tmp) : \ + "0" (ROLc_tmp), \ + "I" (i)); \ + ROLc_tmp; \ + }) +#define RORc(word,i) ({ \ + ulong32 RORc_tmp = (word); \ + __asm__ ("rorl %2, %0" : \ + "=r" (RORc_tmp) : \ + "0" (RORc_tmp), \ + "I" (i)); \ + RORc_tmp; \ + }) + +#else + +#define ROLc ROL +#define RORc ROR + +#endif + +#elif !defined(__STRICT_ANSI__) && defined(LTC_PPC32) +#define LTC_ROx_ASM + +static inline ulong32 ROL(ulong32 word, int i) +{ + asm ("rotlw %0,%0,%2" + :"=r" (word) + :"0" (word),"r" (i)); + return word; +} + +static inline ulong32 ROR(ulong32 word, int i) +{ + asm ("rotlw %0,%0,%2" + :"=r" (word) + :"0" (word),"r" (32-i)); + return word; +} + +#ifndef LTC_NO_ROLC + +static inline ulong32 ROLc(ulong32 word, const int i) +{ + asm ("rotlwi %0,%0,%2" + :"=r" (word) + :"0" (word),"I" (i)); + return word; +} + +static inline ulong32 RORc(ulong32 word, const int i) +{ + asm ("rotrwi %0,%0,%2" + :"=r" (word) + :"0" (word),"I" (i)); + return word; +} + +#else + +#define ROLc ROL +#define RORc ROR + +#endif + + +#else + +/* rotates the hard way */ +#define ROL(x, y) ( (((ulong32)(x)<<(ulong32)((y)&31)) | (((ulong32)(x)&0xFFFFFFFFUL)>>(ulong32)((32-((y)&31))&31))) & 0xFFFFFFFFUL) +#define ROR(x, y) ( ((((ulong32)(x)&0xFFFFFFFFUL)>>(ulong32)((y)&31)) | ((ulong32)(x)<<(ulong32)((32-((y)&31))&31))) & 0xFFFFFFFFUL) +#define ROLc(x, y) ( (((ulong32)(x)<<(ulong32)((y)&31)) | (((ulong32)(x)&0xFFFFFFFFUL)>>(ulong32)((32-((y)&31))&31))) & 0xFFFFFFFFUL) +#define RORc(x, y) ( ((((ulong32)(x)&0xFFFFFFFFUL)>>(ulong32)((y)&31)) | ((ulong32)(x)<<(ulong32)((32-((y)&31))&31))) & 0xFFFFFFFFUL) + +#endif + + +/* 64-bit Rotates */ +#if defined(_MSC_VER) + +/* instrinsic rotate */ +#include +#pragma intrinsic(_rotr64,_rotr64) +#define ROR64(x,n) _rotr64(x,n) +#define ROL64(x,n) _rotl64(x,n) +#define ROR64c(x,n) ROR64(x,n) +#define ROL64c(x,n) ROL64(x,n) + +#elif defined(LTC_HAVE_ROTATE_BUILTIN) + +#define ROR64(x,n) __builtin_rotateright64(x,n) +#define ROL64(x,n) __builtin_rotateleft64(x,n) +#define ROR64c(x,n) ROR64(x,n) +#define ROL64c(x,n) ROL64(x,n) + +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && defined(__x86_64__) && !defined(INTEL_CC) && !defined(LTC_NO_ASM) + +static inline ulong64 ROL64(ulong64 word, int i) +{ + asm("rolq %%cl,%0" + :"=r" (word) + :"0" (word),"c" (i)); + return word; +} + +static inline ulong64 ROR64(ulong64 word, int i) +{ + asm("rorq %%cl,%0" + :"=r" (word) + :"0" (word),"c" (i)); + return word; +} + +#ifndef LTC_NO_ROLC + +#define ROL64c(word,i) ({ \ + ulong64 ROL64c_tmp = word; \ + __asm__ ("rolq %2, %0" : \ + "=r" (ROL64c_tmp) : \ + "0" (ROL64c_tmp), \ + "J" (i)); \ + ROL64c_tmp; \ + }) +#define ROR64c(word,i) ({ \ + ulong64 ROR64c_tmp = word; \ + __asm__ ("rorq %2, %0" : \ + "=r" (ROR64c_tmp) : \ + "0" (ROR64c_tmp), \ + "J" (i)); \ + ROR64c_tmp; \ + }) + +#else /* LTC_NO_ROLC */ + +#define ROL64c ROL64 +#define ROR64c ROR64 + +#endif + +#else /* Not x86_64 */ + +#define ROL64(x, y) \ + ( (((x)<<((ulong64)(y)&63)) | \ + (((x)&CONST64(0xFFFFFFFFFFFFFFFF))>>(((ulong64)64-((y)&63))&63))) & CONST64(0xFFFFFFFFFFFFFFFF)) + +#define ROR64(x, y) \ + ( ((((x)&CONST64(0xFFFFFFFFFFFFFFFF))>>((ulong64)(y)&CONST64(63))) | \ + ((x)<<(((ulong64)64-((y)&63))&63))) & CONST64(0xFFFFFFFFFFFFFFFF)) + +#define ROL64c(x, y) \ + ( (((x)<<((ulong64)(y)&63)) | \ + (((x)&CONST64(0xFFFFFFFFFFFFFFFF))>>(((ulong64)64-((y)&63))&63))) & CONST64(0xFFFFFFFFFFFFFFFF)) + +#define ROR64c(x, y) \ + ( ((((x)&CONST64(0xFFFFFFFFFFFFFFFF))>>((ulong64)(y)&CONST64(63))) | \ + ((x)<<(((ulong64)64-((y)&63))&63))) & CONST64(0xFFFFFFFFFFFFFFFF)) + +#endif + +#ifndef MAX + #define MAX(x, y) ( ((x)>(y))?(x):(y) ) +#endif + +#ifndef MIN + #define MIN(x, y) ( ((x)<(y))?(x):(y) ) +#endif + +#ifndef LTC_UNUSED_PARAM + #define LTC_UNUSED_PARAM(x) (void)(x) +#endif + +/* there is no snprintf before Visual C++ 2015 */ +#if defined(_MSC_VER) && _MSC_VER < 1900 +#define snprintf _snprintf +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_math.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_math.h new file mode 100644 index 0000000..49fa660 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_math.h @@ -0,0 +1,527 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** math functions **/ + +#define LTC_MP_LT -1 +#define LTC_MP_EQ 0 +#define LTC_MP_GT 1 + +#define LTC_MP_NO 0 +#define LTC_MP_YES 1 + +#ifndef LTC_MECC + typedef void ecc_point; +#endif + +#ifndef LTC_MRSA + typedef void rsa_key; +#endif + +#ifndef LTC_MILLER_RABIN_REPS + /* Number of rounds of the Miller-Rabin test + * "Reasonable values of reps are between 15 and 50." c.f. gmp doc of mpz_probab_prime_p() + * As of https://security.stackexchange.com/a/4546 we should use 40 rounds */ + #define LTC_MILLER_RABIN_REPS 40 +#endif + +int radix_to_bin(const void *in, int radix, void *out, unsigned long *len); + +/** math descriptor */ +typedef struct { + /** Name of the math provider */ + const char *name; + + /** Bits per digit, amount of bits must fit in an unsigned long */ + int bits_per_digit; + +/* ---- init/deinit functions ---- */ + + /** initialize a bignum + @param a The number to initialize + @return CRYPT_OK on success + */ + int (*init)(void **a); + + /** initialize a bignum + @param size_bits The size of the number we compute on + @param a The number to initialize + @return CRYPT_OK on success + */ + int (*init_size)(int size_bits, void **a); + + /** init copy + @param dst The number to initialize and write to + @param src The number to copy from + @return CRYPT_OK on success + */ + int (*init_copy)(void **dst, void *src); + + /** deinit + @param a The number to free + @return CRYPT_OK on success + */ + void (*deinit)(void *a); + +/* ---- data movement ---- */ + + /** negate + @param src The number to negate + @param dst The destination + @return CRYPT_OK on success + */ + int (*neg)(void *src, void *dst); + + /** copy + @param src The number to copy from + @param dst The number to write to + @return CRYPT_OK on success + */ + int (*copy)(void *src, void *dst); + +/* ---- trivial low level functions ---- */ + + /** set small constant + @param a Number to write to + @param n Source upto bits_per_digit (actually meant for very small constants) + @return CRYPT_OK on success + */ + int (*set_int)(void *a, ltc_mp_digit n); + + /** get small constant + @param a Small number to read, + only fetches up to bits_per_digit from the number + @return The lower bits_per_digit of the integer (unsigned) + */ + unsigned long (*get_int)(void *a); + + /** get digit n + @param a The number to read from + @param n The number of the digit to fetch + @return The bits_per_digit sized n'th digit of a + */ + ltc_mp_digit (*get_digit)(void *a, int n); + + /** Get the number of digits that represent the number + @param a The number to count + @return The number of digits used to represent the number + */ + int (*get_digit_count)(void *a); + + /** compare two integers + @param a The left side integer + @param b The right side integer + @return LTC_MP_LT if a < b, + LTC_MP_GT if a > b and + LTC_MP_EQ otherwise. (signed comparison) + */ + int (*compare)(void *a, void *b); + + /** compare against int + @param a The left side integer + @param b The right side integer (upto bits_per_digit) + @return LTC_MP_LT if a < b, + LTC_MP_GT if a > b and + LTC_MP_EQ otherwise. (signed comparison) + */ + int (*compare_d)(void *a, ltc_mp_digit n); + + /** Count the number of bits used to represent the integer + @param a The integer to count + @return The number of bits required to represent the integer + */ + int (*count_bits)(void * a); + + /** Count the number of LSB bits which are zero + @param a The integer to count + @return The number of contiguous zero LSB bits + */ + int (*count_lsb_bits)(void *a); + + /** Compute a power of two + @param a The integer to store the power in + @param n The power of two you want to store (a = 2^n) + @return CRYPT_OK on success + */ + int (*twoexpt)(void *a , int n); + +/* ---- radix conversions ---- */ + + /** read ascii string + @param a The integer to store into + @param str The string to read + @param radix The radix the integer has been represented in (2-64) + @return CRYPT_OK on success + */ + int (*read_radix)(void *a, const char *str, int radix); + + /** write number to string + @param a The integer to store + @param str The destination for the string + @param radix The radix the integer is to be represented in (2-64) + @return CRYPT_OK on success + */ + int (*write_radix)(void *a, char *str, int radix); + + /** get size as unsigned char string + @param a The integer to get the size (when stored in array of octets) + @return The length of the integer in octets + */ + unsigned long (*unsigned_size)(void *a); + + /** store an integer as an array of octets + @param src The integer to store + @param dst The buffer to store the integer in + @return CRYPT_OK on success + */ + int (*unsigned_write)(void *src, unsigned char *dst); + + /** read an array of octets and store as integer + @param dst The integer to load + @param src The array of octets + @param len The number of octets + @return CRYPT_OK on success + */ + int (*unsigned_read)( void *dst, + unsigned char *src, + unsigned long len); + +/* ---- basic math ---- */ + + /** add two integers + @param a The first source integer + @param b The second source integer + @param c The destination of "a + b" + @return CRYPT_OK on success + */ + int (*add)(void *a, void *b, void *c); + + /** add two integers + @param a The first source integer + @param b The second source integer + (single digit of upto bits_per_digit in length) + @param c The destination of "a + b" + @return CRYPT_OK on success + */ + int (*addi)(void *a, ltc_mp_digit b, void *c); + + /** subtract two integers + @param a The first source integer + @param b The second source integer + @param c The destination of "a - b" + @return CRYPT_OK on success + */ + int (*sub)(void *a, void *b, void *c); + + /** subtract two integers + @param a The first source integer + @param b The second source integer + (single digit of upto bits_per_digit in length) + @param c The destination of "a - b" + @return CRYPT_OK on success + */ + int (*subi)(void *a, ltc_mp_digit b, void *c); + + /** multiply two integers + @param a The first source integer + @param b The second source integer + (single digit of upto bits_per_digit in length) + @param c The destination of "a * b" + @return CRYPT_OK on success + */ + int (*mul)(void *a, void *b, void *c); + + /** multiply two integers + @param a The first source integer + @param b The second source integer + (single digit of upto bits_per_digit in length) + @param c The destination of "a * b" + @return CRYPT_OK on success + */ + int (*muli)(void *a, ltc_mp_digit b, void *c); + + /** Square an integer + @param a The integer to square + @param b The destination + @return CRYPT_OK on success + */ + int (*sqr)(void *a, void *b); + + /** Square root (mod prime) + @param a The integer to compute square root mod prime from + @param b The prime + @param c The destination + @return CRYPT_OK on success + */ + int (*sqrtmod_prime)(void *a, void *b, void *c); + + /** Divide an integer + @param a The dividend + @param b The divisor + @param c The quotient (can be NULL to signify don't care) + @param d The remainder (can be NULL to signify don't care) + @return CRYPT_OK on success + */ + int (*mpdiv)(void *a, void *b, void *c, void *d); + + /** divide by two + @param a The integer to divide (shift right) + @param b The destination + @return CRYPT_OK on success + */ + int (*div_2)(void *a, void *b); + + /** Get remainder (small value) + @param a The integer to reduce + @param b The modulus (upto bits_per_digit in length) + @param c The destination for the residue + @return CRYPT_OK on success + */ + int (*modi)(void *a, ltc_mp_digit b, ltc_mp_digit *c); + + /** gcd + @param a The first integer + @param b The second integer + @param c The destination for (a, b) + @return CRYPT_OK on success + */ + int (*gcd)(void *a, void *b, void *c); + + /** lcm + @param a The first integer + @param b The second integer + @param c The destination for [a, b] + @return CRYPT_OK on success + */ + int (*lcm)(void *a, void *b, void *c); + + /** Modular multiplication + @param a The first source + @param b The second source + @param c The modulus + @param d The destination (a*b mod c) + @return CRYPT_OK on success + */ + int (*mulmod)(void *a, void *b, void *c, void *d); + + /** Modular squaring + @param a The first source + @param b The modulus + @param c The destination (a*a mod b) + @return CRYPT_OK on success + */ + int (*sqrmod)(void *a, void *b, void *c); + + /** Modular inversion + @param a The value to invert + @param b The modulus + @param c The destination (1/a mod b) + @return CRYPT_OK on success + */ + int (*invmod)(void *, void *, void *); + +/* ---- reduction ---- */ + + /** setup Montgomery + @param a The modulus + @param b The destination for the reduction digit + @return CRYPT_OK on success + */ + int (*montgomery_setup)(void *a, void **b); + + /** get normalization value + @param a The destination for the normalization value + @param b The modulus + @return CRYPT_OK on success + */ + int (*montgomery_normalization)(void *a, void *b); + + /** reduce a number + @param a The number [and dest] to reduce + @param b The modulus + @param c The value "b" from montgomery_setup() + @return CRYPT_OK on success + */ + int (*montgomery_reduce)(void *a, void *b, void *c); + + /** clean up (frees memory) + @param a The value "b" from montgomery_setup() + @return CRYPT_OK on success + */ + void (*montgomery_deinit)(void *a); + +/* ---- exponentiation ---- */ + + /** Modular exponentiation + @param a The base integer + @param b The power (can be negative) integer + @param c The modulus integer + @param d The destination + @return CRYPT_OK on success + */ + int (*exptmod)(void *a, void *b, void *c, void *d); + + /** Primality testing + @param a The integer to test + @param b The number of Miller-Rabin tests that shall be executed + @param c The destination of the result (FP_YES if prime) + @return CRYPT_OK on success + */ + int (*isprime)(void *a, int b, int *c); + +/* ---- (optional) ecc point math ---- */ + + /** ECC GF(p) point multiplication (from the NIST curves) + @param k The integer to multiply the point by + @param G The point to multiply + @param R The destination for kG + @param a ECC curve parameter a + @param modulus The modulus for the field + @param map Boolean indicated whether to map back to affine or not + (can be ignored if you work in affine only) + @return CRYPT_OK on success + */ + int (*ecc_ptmul)( void *k, + const ecc_point *G, + ecc_point *R, + void *a, + void *modulus, + int map); + + /** ECC GF(p) point addition + @param P The first point + @param Q The second point + @param R The destination of P + Q + @param ma The curve parameter "a" in montgomery form + @param modulus The modulus + @param mp The "b" value from montgomery_setup() + @return CRYPT_OK on success + */ + int (*ecc_ptadd)(const ecc_point *P, + const ecc_point *Q, + ecc_point *R, + void *ma, + void *modulus, + void *mp); + + /** ECC GF(p) point double + @param P The first point + @param R The destination of 2P + @param ma The curve parameter "a" in montgomery form + @param modulus The modulus + @param mp The "b" value from montgomery_setup() + @return CRYPT_OK on success + */ + int (*ecc_ptdbl)(const ecc_point *P, + ecc_point *R, + void *ma, + void *modulus, + void *mp); + + /** ECC mapping from projective to affine, + currently uses (x,y,z) => (x/z^2, y/z^3, 1) + @param P The point to map + @param modulus The modulus + @param mp The "b" value from montgomery_setup() + @return CRYPT_OK on success + @remark The mapping can be different but keep in mind a + ecc_point only has three integers (x,y,z) so if + you use a different mapping you have to make it fit. + */ + int (*ecc_map)(ecc_point *P, void *modulus, void *mp); + + /** Computes kA*A + kB*B = C using Shamir's Trick + @param A First point to multiply + @param kA What to multiple A by + @param B Second point to multiply + @param kB What to multiple B by + @param C [out] Destination point (can overlap with A or B) + @param ma The curve parameter "a" in montgomery form + @param modulus Modulus for curve + @return CRYPT_OK on success + */ + int (*ecc_mul2add)(const ecc_point *A, void *kA, + const ecc_point *B, void *kB, + ecc_point *C, + void *ma, + void *modulus); + +/* ---- (optional) rsa optimized math (for internal CRT) ---- */ + + /** RSA Key Generation + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param size The size of the key in octets + @param e The "e" value (public key). + e==65537 is a good choice + @param key [out] Destination of a newly created private key pair + @return CRYPT_OK if successful, upon error all allocated ram is freed + */ + int (*rsa_keygen)(prng_state *prng, + int wprng, + int size, + long e, + rsa_key *key); + + /** RSA exponentiation + @param in The octet array representing the base + @param inlen The length of the input + @param out The destination (to be stored in an octet array format) + @param outlen The length of the output buffer and the resulting size + (zero padded to the size of the modulus) + @param which PK_PUBLIC for public RSA and PK_PRIVATE for private RSA + @param key The RSA key to use + @return CRYPT_OK on success + */ + int (*rsa_me)(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, int which, + const rsa_key *key); + +/* ---- basic math continued ---- */ + + /** Modular addition + @param a The first source + @param b The second source + @param c The modulus + @param d The destination (a + b mod c) + @return CRYPT_OK on success + */ + int (*addmod)(void *a, void *b, void *c, void *d); + + /** Modular substraction + @param a The first source + @param b The second source + @param c The modulus + @param d The destination (a - b mod c) + @return CRYPT_OK on success + */ + int (*submod)(void *a, void *b, void *c, void *d); + +/* ---- misc stuff ---- */ + + /** Make a pseudo-random mpi + @param a The mpi to make random + @param size The desired length + @return CRYPT_OK on success + */ + int (*rand)(void *a, int size); +} ltc_math_descriptor; + +extern ltc_math_descriptor ltc_mp; + +int ltc_init_multi(void **a, ...) LTC_NULL_TERMINATED; +int ltc_init_multi_size(int size_bits, void **a, ...) LTC_NULL_TERMINATED; +void ltc_deinit_multi(void *a, ...) LTC_NULL_TERMINATED; +void ltc_cleanup_multi(void **a, ...) LTC_NULL_TERMINATED; + +#ifdef LTM_DESC +extern const ltc_math_descriptor ltm_desc; +#endif + +#ifdef TFM_DESC +extern const ltc_math_descriptor tfm_desc; +#endif + +#ifdef GMP_DESC +extern const ltc_math_descriptor gmp_desc; +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_misc.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_misc.h new file mode 100644 index 0000000..3a2b7b1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_misc.h @@ -0,0 +1,179 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* ---- LTC_BASE64 Routines ---- */ +#ifdef LTC_BASE64 +int base64_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen); + +int base64_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int base64_strict_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int base64_sane_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +#endif + +#ifdef LTC_BASE64_URL +int base64url_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen); +int base64url_strict_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen); + +int base64url_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int base64url_strict_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int base64url_sane_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +#endif + +/* ---- BASE32 Routines ---- */ +#ifdef LTC_BASE32 +typedef enum { + BASE32_RFC4648 = 0, + BASE32_BASE32HEX = 1, + BASE32_ZBASE32 = 2, + BASE32_CROCKFORD = 3 +} base32_alphabet; +int base32_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen, + base32_alphabet id); +int base32_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + base32_alphabet id); +#endif + +/* ---- BASE16 Routines ---- */ +#ifdef LTC_BASE16 +int base16_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen, + unsigned int options); +int base16_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +#endif + +#ifdef LTC_BCRYPT +int bcrypt_pbkdf_openbsd(const void *secret, unsigned long secret_len, + const unsigned char *salt, unsigned long salt_len, + unsigned int rounds, int hash_idx, + unsigned char *out, unsigned long *outlen); +#endif + +/* ===> LTC_HKDF -- RFC5869 HMAC-based Key Derivation Function <=== */ +#ifdef LTC_HKDF + +int hkdf_test(void); + +int hkdf_extract(int hash_idx, + const unsigned char *salt, unsigned long saltlen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +int hkdf_expand(int hash_idx, + const unsigned char *info, unsigned long infolen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long outlen); + +int hkdf(int hash_idx, + const unsigned char *salt, unsigned long saltlen, + const unsigned char *info, unsigned long infolen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long outlen); + +#endif /* LTC_HKDF */ + +/* ---- MEM routines ---- */ +int mem_neq(const void *a, const void *b, size_t len); +void zeromem(volatile void *out, size_t outlen); +void burn_stack(unsigned long len); + +const char *error_to_string(int err); + +extern const char *crypt_build_settings; + +/* ---- HMM ---- */ +int crypt_fsa(void *mp, ...) LTC_NULL_TERMINATED; + +/* ---- Dynamic language support ---- */ +int crypt_get_constant(const char* namein, int *valueout); +int crypt_list_all_constants(char *names_list, unsigned int *names_list_size); + +int crypt_get_size(const char* namein, unsigned int *sizeout); +int crypt_list_all_sizes(char *names_list, unsigned int *names_list_size); + +#ifdef LTM_DESC +LTC_DEPRECATED(crypt_mp_init) void init_LTM(void); +#endif +#ifdef TFM_DESC +LTC_DEPRECATED(crypt_mp_init) void init_TFM(void); +#endif +#ifdef GMP_DESC +LTC_DEPRECATED(crypt_mp_init) void init_GMP(void); +#endif +int crypt_mp_init(const char* mpi); + +#ifdef LTC_ADLER32 +typedef struct adler32_state_s +{ + unsigned short s[2]; +} adler32_state; + +void adler32_init(adler32_state *ctx); +void adler32_update(adler32_state *ctx, const unsigned char *input, unsigned long length); +void adler32_finish(const adler32_state *ctx, void *hash, unsigned long size); +int adler32_test(void); +#endif + +#ifdef LTC_CRC32 +typedef struct crc32_state_s +{ + ulong32 crc; +} crc32_state; + +void crc32_init(crc32_state *ctx); +void crc32_update(crc32_state *ctx, const unsigned char *input, unsigned long length); +void crc32_finish(const crc32_state *ctx, void *hash, unsigned long size); +int crc32_test(void); +#endif + + +#ifdef LTC_PADDING + +enum padding_type { + LTC_PAD_PKCS7 = 0x0000U, +#ifdef LTC_RNG_GET_BYTES + LTC_PAD_ISO_10126 = 0x1000U, +#endif + LTC_PAD_ANSI_X923 = 0x2000U, + LTC_PAD_SSH = 0x3000U, + /* The following padding modes don't contain the padding + * length as last byte of the padding. + */ + LTC_PAD_ONE_AND_ZERO = 0x8000U, + LTC_PAD_ZERO = 0x9000U, + LTC_PAD_ZERO_ALWAYS = 0xA000U, +}; + +int padding_pad(unsigned char *data, unsigned long length, unsigned long* padded_length, unsigned long mode); +int padding_depad(const unsigned char *data, unsigned long *length, unsigned long mode); +#endif /* LTC_PADDING */ + +#ifdef LTC_SSH +typedef enum ssh_data_type_ { + LTC_SSHDATA_EOL, + LTC_SSHDATA_BYTE, + LTC_SSHDATA_BOOLEAN, + LTC_SSHDATA_UINT32, + LTC_SSHDATA_UINT64, + LTC_SSHDATA_STRING, + LTC_SSHDATA_MPINT, + LTC_SSHDATA_NAMELIST, +} ssh_data_type; + +/* VA list handy helpers with tuples of */ +int ssh_encode_sequence_multi(unsigned char *out, unsigned long *outlen, ...) LTC_NULL_TERMINATED; +int ssh_decode_sequence_multi(const unsigned char *in, unsigned long *inlen, ...) LTC_NULL_TERMINATED; +#endif /* LTC_SSH */ + +int compare_testvector(const void* is, const unsigned long is_len, const void* should, const unsigned long should_len, const char* what, int which); diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_pk.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_pk.h new file mode 100644 index 0000000..0ee3ac8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_pk.h @@ -0,0 +1,803 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* ---- NUMBER THEORY ---- */ + +enum public_key_type { + /* Refers to the public key */ + PK_PUBLIC = 0x0000, + /* Refers to the private key */ + PK_PRIVATE = 0x0001, + + /* Indicates standard output formats that can be read e.g. by OpenSSL or GnuTLS */ + PK_STD = 0x1000, + /* Indicates compressed public ECC key */ + PK_COMPRESSED = 0x2000, + /* Indicates ECC key with the curve specified by OID */ + PK_CURVEOID = 0x4000 +}; + +int rand_prime(void *N, long len, prng_state *prng, int wprng); + +/* ---- RSA ---- */ +#ifdef LTC_MRSA + +/** RSA PKCS style key */ +typedef struct Rsa_key { + /** Type of key, PK_PRIVATE or PK_PUBLIC */ + int type; + /** The public exponent */ + void *e; + /** The private exponent */ + void *d; + /** The modulus */ + void *N; + /** The p factor of N */ + void *p; + /** The q factor of N */ + void *q; + /** The 1/q mod p CRT param */ + void *qP; + /** The d mod (p - 1) CRT param */ + void *dP; + /** The d mod (q - 1) CRT param */ + void *dQ; +} rsa_key; + +int rsa_make_key(prng_state *prng, int wprng, int size, long e, rsa_key *key); +int rsa_make_key_ubin_e(prng_state *prng, int wprng, int size, + const unsigned char *e, unsigned long elen, rsa_key *key); +int rsa_get_size(const rsa_key *key); + +int rsa_exptmod(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, int which, + const rsa_key *key); + +void rsa_free(rsa_key *key); + +/* These use PKCS #1 v2.0 padding */ +#define rsa_encrypt_key(in, inlen, out, outlen, lparam, lparamlen, prng, prng_idx, hash_idx, key) \ + rsa_encrypt_key_ex(in, inlen, out, outlen, lparam, lparamlen, prng, prng_idx, hash_idx, LTC_PKCS_1_OAEP, key) + +#define rsa_decrypt_key(in, inlen, out, outlen, lparam, lparamlen, hash_idx, stat, key) \ + rsa_decrypt_key_ex(in, inlen, out, outlen, lparam, lparamlen, hash_idx, LTC_PKCS_1_OAEP, stat, key) + +#define rsa_sign_hash(in, inlen, out, outlen, prng, prng_idx, hash_idx, saltlen, key) \ + rsa_sign_hash_ex(in, inlen, out, outlen, LTC_PKCS_1_PSS, prng, prng_idx, hash_idx, saltlen, key) + +#define rsa_verify_hash(sig, siglen, hash, hashlen, hash_idx, saltlen, stat, key) \ + rsa_verify_hash_ex(sig, siglen, hash, hashlen, LTC_PKCS_1_PSS, hash_idx, saltlen, stat, key) + +#define rsa_sign_saltlen_get_max(hash_idx, key) \ + rsa_sign_saltlen_get_max_ex(LTC_PKCS_1_PSS, hash_idx, key) + +/* These can be switched between PKCS #1 v2.x and PKCS #1 v1.5 paddings */ +int rsa_encrypt_key_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const unsigned char *lparam, unsigned long lparamlen, + prng_state *prng, int prng_idx, + int hash_idx, int padding, + const rsa_key *key); + +int rsa_decrypt_key_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const unsigned char *lparam, unsigned long lparamlen, + int hash_idx, int padding, + int *stat, const rsa_key *key); + +int rsa_sign_hash_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + int padding, + prng_state *prng, int prng_idx, + int hash_idx, unsigned long saltlen, + const rsa_key *key); + +int rsa_verify_hash_ex(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + int padding, + int hash_idx, unsigned long saltlen, + int *stat, const rsa_key *key); + +int rsa_sign_saltlen_get_max_ex(int padding, int hash_idx, const rsa_key *key); + +/* PKCS #1 import/export */ +int rsa_export(unsigned char *out, unsigned long *outlen, int type, const rsa_key *key); +int rsa_import(const unsigned char *in, unsigned long inlen, rsa_key *key); + +int rsa_import_x509(const unsigned char *in, unsigned long inlen, rsa_key *key); +int rsa_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *passwd, unsigned long passwdlen, rsa_key *key); + +int rsa_set_key(const unsigned char *N, unsigned long Nlen, + const unsigned char *e, unsigned long elen, + const unsigned char *d, unsigned long dlen, + rsa_key *key); +int rsa_set_factors(const unsigned char *p, unsigned long plen, + const unsigned char *q, unsigned long qlen, + rsa_key *key); +int rsa_set_crt_params(const unsigned char *dP, unsigned long dPlen, + const unsigned char *dQ, unsigned long dQlen, + const unsigned char *qP, unsigned long qPlen, + rsa_key *key); +#endif + +/* ---- DH Routines ---- */ +#ifdef LTC_MDH + +typedef struct { + int type; + void *x; + void *y; + void *base; + void *prime; +} dh_key; + +int dh_get_groupsize(const dh_key *key); + +int dh_export(unsigned char *out, unsigned long *outlen, int type, const dh_key *key); +int dh_import(const unsigned char *in, unsigned long inlen, dh_key *key); + +int dh_set_pg(const unsigned char *p, unsigned long plen, + const unsigned char *g, unsigned long glen, + dh_key *key); +int dh_set_pg_dhparam(const unsigned char *dhparam, unsigned long dhparamlen, dh_key *key); +int dh_set_pg_groupsize(int groupsize, dh_key *key); + +int dh_set_key(const unsigned char *in, unsigned long inlen, int type, dh_key *key); +int dh_generate_key(prng_state *prng, int wprng, dh_key *key); +int dh_make_key(prng_state *prng, int wprng, void *q, int xbits, dh_key *key); /* OP-TEE */ +int dh_shared_secret(const dh_key *private_key, const dh_key *public_key, + unsigned char *out, unsigned long *outlen); + +void dh_free(dh_key *key); + +int dh_export_key(void *out, unsigned long *outlen, int type, const dh_key *key); +#endif /* LTC_MDH */ + + +/* ---- ECC Routines ---- */ +#ifdef LTC_MECC + +/* size of our temp buffers for exported keys */ +#define ECC_BUF_SIZE 256 + +/* max private key size */ +#define ECC_MAXSIZE 66 + +/** Structure defines a GF(p) curve */ +typedef struct { + /** The prime that defines the field the curve is in (encoded in hex) */ + const char *prime; + + /** The fields A param (hex) */ + const char *A; + + /** The fields B param (hex) */ + const char *B; + + /** The order of the curve (hex) */ + const char *order; + + /** The x co-ordinate of the base point on the curve (hex) */ + const char *Gx; + + /** The y co-ordinate of the base point on the curve (hex) */ + const char *Gy; + + /** The co-factor */ + unsigned long cofactor; + + /** The OID */ + const char *OID; +} ltc_ecc_curve; + +/** A point on a ECC curve, stored in Jacbobian format such that (x,y,z) => (x/z^2, y/z^3, 1) when interpretted as affine */ +typedef struct { + /** The x co-ordinate */ + void *x; + + /** The y co-ordinate */ + void *y; + + /** The z co-ordinate */ + void *z; +} ecc_point; + +/** ECC key's domain parameters */ +typedef struct { + /** The size of the curve in octets */ + int size; + /** The prime that defines the field the curve is in */ + void *prime; + /** The fields A param */ + void *A; + /** The fields B param */ + void *B; + /** The order of the curve */ + void *order; + /** The base point G on the curve */ + ecc_point base; + /** The co-factor */ + unsigned long cofactor; + /** The OID */ + unsigned long oid[16]; + unsigned long oidlen; +} ltc_ecc_dp; + +/** An ECC key */ +typedef struct { + /** Type of key, PK_PRIVATE or PK_PUBLIC */ + int type; + + /** Structure with domain parameters */ + ltc_ecc_dp dp; + + /** Structure with the public key */ + ecc_point pubkey; + + /** The private key */ + void *k; +} ecc_key; + +/** Formats of ECC signatures */ +typedef enum ecc_signature_type_ { + /* ASN.1 encoded, ANSI X9.62 */ + LTC_ECCSIG_ANSIX962 = 0x0, + /* raw R, S values */ + LTC_ECCSIG_RFC7518 = 0x1, + /* raw R, S, V (+27) values */ + LTC_ECCSIG_ETH27 = 0x2, + /* SSH + ECDSA signature format defined by RFC5656 */ + LTC_ECCSIG_RFC5656 = 0x3, +} ecc_signature_type; + +/** the ECC params provided */ +extern const ltc_ecc_curve ltc_ecc_curves[]; + +void ecc_sizes(int *low, int *high); +int ecc_get_size(const ecc_key *key); + +int ecc_find_curve(const char* name_or_oid, const ltc_ecc_curve** cu); +int ecc_set_curve(const ltc_ecc_curve *cu, ecc_key *key); +int ecc_generate_key(prng_state *prng, int wprng, ecc_key *key); +int ecc_set_key(const unsigned char *in, unsigned long inlen, int type, ecc_key *key); +int ecc_get_key(unsigned char *out, unsigned long *outlen, int type, const ecc_key *key); +int ecc_get_oid_str(char *out, unsigned long *outlen, const ecc_key *key); + +int ecc_make_key(prng_state *prng, int wprng, int keysize, ecc_key *key); +int ecc_make_key_ex(prng_state *prng, int wprng, ecc_key *key, const ltc_ecc_curve *cu); +void ecc_free(ecc_key *key); + +int ecc_export(unsigned char *out, unsigned long *outlen, int type, const ecc_key *key); +int ecc_import(const unsigned char *in, unsigned long inlen, ecc_key *key); +int ecc_import_ex(const unsigned char *in, unsigned long inlen, ecc_key *key, const ltc_ecc_curve *cu); + +int ecc_ansi_x963_export(const ecc_key *key, unsigned char *out, unsigned long *outlen); +int ecc_ansi_x963_import(const unsigned char *in, unsigned long inlen, ecc_key *key); +int ecc_ansi_x963_import_ex(const unsigned char *in, unsigned long inlen, ecc_key *key, const ltc_ecc_curve *cu); + +int ecc_export_openssl(unsigned char *out, unsigned long *outlen, int type, const ecc_key *key); +int ecc_import_openssl(const unsigned char *in, unsigned long inlen, ecc_key *key); +int ecc_import_pkcs8(const unsigned char *in, unsigned long inlen, const void *pwd, unsigned long pwdlen, ecc_key *key); +int ecc_import_x509(const unsigned char *in, unsigned long inlen, ecc_key *key); + +int ecc_shared_secret(const ecc_key *private_key, const ecc_key *public_key, + unsigned char *out, unsigned long *outlen); + +int ecc_encrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, int hash, + const ecc_key *key); + +int ecc_decrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const ecc_key *key); + +#define ecc_sign_hash_rfc7518(in_, inlen_, out_, outlen_, prng_, wprng_, key_) \ + ecc_sign_hash_ex(in_, inlen_, out_, outlen_, prng_, wprng_, LTC_ECCSIG_RFC7518, NULL, key_) + +#define ecc_sign_hash(in_, inlen_, out_, outlen_, prng_, wprng_, key_) \ + ecc_sign_hash_ex(in_, inlen_, out_, outlen_, prng_, wprng_, LTC_ECCSIG_ANSIX962, NULL, key_) + +#define ecc_verify_hash_rfc7518(sig_, siglen_, hash_, hashlen_, stat_, key_) \ + ecc_verify_hash_ex(sig_, siglen_, hash_, hashlen_, LTC_ECCSIG_RFC7518, stat_, key_) + +#define ecc_verify_hash(sig_, siglen_, hash_, hashlen_, stat_, key_) \ + ecc_verify_hash_ex(sig_, siglen_, hash_, hashlen_, LTC_ECCSIG_ANSIX962, stat_, key_) + +int ecc_sign_hash_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, ecc_signature_type sigformat, + int *recid, const ecc_key *key); + +int ecc_verify_hash_ex(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + ecc_signature_type sigformat, int *stat, const ecc_key *key); + +int ecc_recover_key(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + int recid, ecc_signature_type sigformat, ecc_key *key); + +#endif + +#ifdef LTC_CURVE25519 + +typedef struct { + /** The key type, PK_PRIVATE or PK_PUBLIC */ + enum public_key_type type; + + /** The PK-algorithm, PKA_ED25519 or PKA_X25519 */ + /** This was supposed to be: + * enum public_key_algorithms algo; + * but that enum is now in tomcrypt_private.h + */ + int algo; + + /** The private key */ + unsigned char priv[32]; + + /** The public key */ + unsigned char pub[32]; +} curve25519_key; + + +/** Ed25519 Signature API */ +int ed25519_make_key(prng_state *prng, int wprng, curve25519_key *key); + +int ed25519_export( unsigned char *out, unsigned long *outlen, + int which, + const curve25519_key *key); + +int ed25519_import(const unsigned char *in, unsigned long inlen, curve25519_key *key); +int ed25519_import_raw(const unsigned char *in, unsigned long inlen, int which, curve25519_key *key); +int ed25519_import_x509(const unsigned char *in, unsigned long inlen, curve25519_key *key); +int ed25519_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + curve25519_key *key); + +int ed25519_sign(const unsigned char *msg, unsigned long msglen, + unsigned char *sig, unsigned long *siglen, + const curve25519_key *private_key); +int ed25519ctx_sign(const unsigned char *msg, unsigned long msglen, + unsigned char *sig, unsigned long *siglen, + const unsigned char *ctx, unsigned long ctxlen, + const curve25519_key *private_key); +int ed25519ph_sign(const unsigned char *msg, unsigned long msglen, + unsigned char *sig, unsigned long *siglen, + const unsigned char *ctx, unsigned long ctxlen, + const curve25519_key *private_key); +int ed25519_verify(const unsigned char *msg, unsigned long msglen, + const unsigned char *sig, unsigned long siglen, + int *stat, + const curve25519_key *public_key); +int ed25519ctx_verify(const unsigned char *msg, unsigned long msglen, + const unsigned char *sig, unsigned long siglen, + const unsigned char *ctx, unsigned long ctxlen, + int *stat, + const curve25519_key *public_key); +int ed25519ph_verify(const unsigned char *msg, unsigned long msglen, + const unsigned char *sig, unsigned long siglen, + const unsigned char *ctx, unsigned long ctxlen, + int *stat, + const curve25519_key *public_key); + +/** X25519 Key-Exchange API */ +int x25519_make_key(prng_state *prng, int wprng, curve25519_key *key); + +int x25519_export( unsigned char *out, unsigned long *outlen, + int which, + const curve25519_key *key); + +int x25519_import(const unsigned char *in, unsigned long inlen, curve25519_key *key); +int x25519_import_raw(const unsigned char *in, unsigned long inlen, int which, curve25519_key *key); +int x25519_import_x509(const unsigned char *in, unsigned long inlen, curve25519_key *key); +int x25519_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + curve25519_key *key); + +int x25519_shared_secret(const curve25519_key *private_key, + const curve25519_key *public_key, + unsigned char *out, unsigned long *outlen); + +#endif /* LTC_CURVE25519 */ + +#ifdef LTC_MDSA + +/* Max diff between group and modulus size in bytes (max case: L=8192bits, N=256bits) */ +#define LTC_MDSA_DELTA 992 + +/* Max DSA group size in bytes */ +#define LTC_MDSA_MAX_GROUP 64 + +/* Max DSA modulus size in bytes (the actual DSA size, max 8192 bits) */ +#define LTC_MDSA_MAX_MODULUS 1024 + +/** DSA key structure */ +typedef struct { + /** The key type, PK_PRIVATE or PK_PUBLIC */ + int type; + + /** The order of the sub-group used in octets */ + int qord; + + /** The generator */ + void *g; + + /** The prime used to generate the sub-group */ + void *q; + + /** The large prime that generats the field the contains the sub-group */ + void *p; + + /** The private key */ + void *x; + + /** The public key */ + void *y; +} dsa_key; + +int dsa_make_key(prng_state *prng, int wprng, int group_size, int modulus_size, dsa_key *key); + +int dsa_set_pqg(const unsigned char *p, unsigned long plen, + const unsigned char *q, unsigned long qlen, + const unsigned char *g, unsigned long glen, + dsa_key *key); +int dsa_set_pqg_dsaparam(const unsigned char *dsaparam, unsigned long dsaparamlen, dsa_key *key); +int dsa_generate_pqg(prng_state *prng, int wprng, int group_size, int modulus_size, dsa_key *key); + +int dsa_set_key(const unsigned char *in, unsigned long inlen, int type, dsa_key *key); +int dsa_generate_key(prng_state *prng, int wprng, dsa_key *key); + +void dsa_free(dsa_key *key); + +int dsa_sign_hash_raw(const unsigned char *in, unsigned long inlen, + void *r, void *s, + prng_state *prng, int wprng, const dsa_key *key); + +int dsa_sign_hash(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, const dsa_key *key); + +int dsa_verify_hash_raw( void *r, void *s, + const unsigned char *hash, unsigned long hashlen, + int *stat, const dsa_key *key); + +int dsa_verify_hash(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + int *stat, const dsa_key *key); + +int dsa_encrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, int hash, + const dsa_key *key); + +int dsa_decrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const dsa_key *key); + +int dsa_import(const unsigned char *in, unsigned long inlen, dsa_key *key); +int dsa_export(unsigned char *out, unsigned long *outlen, int type, const dsa_key *key); +int dsa_verify_key(const dsa_key *key, int *stat); +int dsa_shared_secret(void *private_key, void *base, + const dsa_key *public_key, + unsigned char *out, unsigned long *outlen); +#endif /* LTC_MDSA */ + +#ifdef LTC_DER +/* DER handling */ + +typedef enum ltc_asn1_type_ { + /* 0 */ + LTC_ASN1_EOL, + LTC_ASN1_BOOLEAN, + LTC_ASN1_INTEGER, + LTC_ASN1_SHORT_INTEGER, + LTC_ASN1_BIT_STRING, + /* 5 */ + LTC_ASN1_OCTET_STRING, + LTC_ASN1_NULL, + LTC_ASN1_OBJECT_IDENTIFIER, + LTC_ASN1_IA5_STRING, + LTC_ASN1_PRINTABLE_STRING, + /* 10 */ + LTC_ASN1_UTF8_STRING, + LTC_ASN1_UTCTIME, + LTC_ASN1_CHOICE, + LTC_ASN1_SEQUENCE, + LTC_ASN1_SET, + /* 15 */ + LTC_ASN1_SETOF, + LTC_ASN1_RAW_BIT_STRING, + LTC_ASN1_TELETEX_STRING, + LTC_ASN1_GENERALIZEDTIME, + LTC_ASN1_CUSTOM_TYPE, +} ltc_asn1_type; + +typedef enum { + LTC_ASN1_CL_UNIVERSAL = 0x0, + LTC_ASN1_CL_APPLICATION = 0x1, + LTC_ASN1_CL_CONTEXT_SPECIFIC = 0x2, + LTC_ASN1_CL_PRIVATE = 0x3, +} ltc_asn1_class; + +typedef enum { + LTC_ASN1_PC_PRIMITIVE = 0x0, + LTC_ASN1_PC_CONSTRUCTED = 0x1, +} ltc_asn1_pc; + +/** A LTC ASN.1 list type */ +typedef struct ltc_asn1_list_ { + /** The LTC ASN.1 enumerated type identifier */ + ltc_asn1_type type; + /** The data to encode or place for decoding */ + void *data; + /** The size of the input or resulting output */ + unsigned long size; + /** The used flag + * 1. This is used by the CHOICE ASN.1 type to indicate which choice was made + * 2. This is used by the ASN.1 decoder to indicate if an element is used + * 3. This is used by the flexi-decoder to indicate the first byte of the identifier */ + int used; + /** Flag used to indicate optional items in ASN.1 sequences */ + int optional; + /** ASN.1 identifier */ + ltc_asn1_class klass; + ltc_asn1_pc pc; + ulong64 tag; + /** prev/next entry in the list */ + struct ltc_asn1_list_ *prev, *next, *child, *parent; +} ltc_asn1_list; + +#define LTC_SET_ASN1(list, index, Type, Data, Size) \ + do { \ + int LTC_MACRO_temp = (index); \ + ltc_asn1_list *LTC_MACRO_list = (list); \ + LTC_MACRO_list[LTC_MACRO_temp].type = (Type); \ + LTC_MACRO_list[LTC_MACRO_temp].data = (void*)(Data); \ + LTC_MACRO_list[LTC_MACRO_temp].size = (Size); \ + LTC_MACRO_list[LTC_MACRO_temp].used = 0; \ + LTC_MACRO_list[LTC_MACRO_temp].optional = 0; \ + LTC_MACRO_list[LTC_MACRO_temp].klass = 0; \ + LTC_MACRO_list[LTC_MACRO_temp].pc = 0; \ + LTC_MACRO_list[LTC_MACRO_temp].tag = 0; \ + } while (0) + +#define LTC_SET_ASN1_IDENTIFIER(list, index, Class, Pc, Tag) \ + do { \ + int LTC_MACRO_temp = (index); \ + ltc_asn1_list *LTC_MACRO_list = (list); \ + LTC_MACRO_list[LTC_MACRO_temp].type = LTC_ASN1_CUSTOM_TYPE; \ + LTC_MACRO_list[LTC_MACRO_temp].klass = (Class); \ + LTC_MACRO_list[LTC_MACRO_temp].pc = (Pc); \ + LTC_MACRO_list[LTC_MACRO_temp].tag = (Tag); \ + } while (0) + +#define LTC_SET_ASN1_CUSTOM_CONSTRUCTED(list, index, Class, Tag, Data) \ + do { \ + int LTC_MACRO_temp##__LINE__ = (index); \ + LTC_SET_ASN1(list, LTC_MACRO_temp##__LINE__, LTC_ASN1_CUSTOM_TYPE, Data, 1); \ + LTC_SET_ASN1_IDENTIFIER(list, LTC_MACRO_temp##__LINE__, Class, LTC_ASN1_PC_CONSTRUCTED, Tag); \ + } while (0) + +#define LTC_SET_ASN1_CUSTOM_PRIMITIVE(list, index, Class, Tag, Type, Data, Size) \ + do { \ + int LTC_MACRO_temp##__LINE__ = (index); \ + LTC_SET_ASN1(list, LTC_MACRO_temp##__LINE__, LTC_ASN1_CUSTOM_TYPE, Data, Size); \ + LTC_SET_ASN1_IDENTIFIER(list, LTC_MACRO_temp##__LINE__, Class, LTC_ASN1_PC_PRIMITIVE, Tag); \ + list[LTC_MACRO_temp##__LINE__].used = (int)(Type); \ + } while (0) + +extern const char* der_asn1_class_to_string_map[]; +extern const unsigned long der_asn1_class_to_string_map_sz; + +extern const char* der_asn1_pc_to_string_map[]; +extern const unsigned long der_asn1_pc_to_string_map_sz; + +extern const char* der_asn1_tag_to_string_map[]; +extern const unsigned long der_asn1_tag_to_string_map_sz; + +/* SEQUENCE */ +int der_encode_sequence_ex(const ltc_asn1_list *list, unsigned long inlen, + unsigned char *out, unsigned long *outlen, int type_of); + +#define der_encode_sequence(list, inlen, out, outlen) der_encode_sequence_ex(list, inlen, out, outlen, LTC_ASN1_SEQUENCE) + +/** The supported bitmap for all the + * decoders with a `flags` argument. + */ +enum ltc_der_seq { + LTC_DER_SEQ_ZERO = 0x0u, + + /** Bit0 - [0]=Unordered (SET or SETOF) + * [1]=Ordered (SEQUENCE) */ + LTC_DER_SEQ_UNORDERED = LTC_DER_SEQ_ZERO, + LTC_DER_SEQ_ORDERED = 0x1u, + + /** Bit1 - [0]=Relaxed + * [1]=Strict */ + LTC_DER_SEQ_RELAXED = LTC_DER_SEQ_ZERO, + LTC_DER_SEQ_STRICT = 0x2u, + + /** Alternative naming */ + LTC_DER_SEQ_SET = LTC_DER_SEQ_UNORDERED, + LTC_DER_SEQ_SEQUENCE = LTC_DER_SEQ_ORDERED, +}; + +int der_decode_sequence_ex(const unsigned char *in, unsigned long inlen, + ltc_asn1_list *list, unsigned long outlen, unsigned int flags); + +#define der_decode_sequence(in, inlen, list, outlen) der_decode_sequence_ex(in, inlen, list, outlen, LTC_DER_SEQ_SEQUENCE | LTC_DER_SEQ_RELAXED) +#define der_decode_sequence_strict(in, inlen, list, outlen) der_decode_sequence_ex(in, inlen, list, outlen, LTC_DER_SEQ_SEQUENCE | LTC_DER_SEQ_STRICT) + +int der_length_sequence(const ltc_asn1_list *list, unsigned long inlen, + unsigned long *outlen); + + +/* Custom-types */ +int der_encode_custom_type(const ltc_asn1_list *root, + unsigned char *out, unsigned long *outlen); + +int der_decode_custom_type(const unsigned char *in, unsigned long inlen, + ltc_asn1_list *root); + +int der_length_custom_type(const ltc_asn1_list *root, + unsigned long *outlen, + unsigned long *payloadlen); + +/* SET */ +#define der_decode_set(in, inlen, list, outlen) der_decode_sequence_ex(in, inlen, list, outlen, LTC_DER_SEQ_SET) +#define der_length_set der_length_sequence +int der_encode_set(const ltc_asn1_list *list, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +int der_encode_setof(const ltc_asn1_list *list, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +/* VA list handy helpers with triplets of */ +int der_encode_sequence_multi(unsigned char *out, unsigned long *outlen, ...) LTC_NULL_TERMINATED; +int der_decode_sequence_multi(const unsigned char *in, unsigned long inlen, ...) LTC_NULL_TERMINATED; + +/* FLEXI DECODER handle unknown list decoder */ +int der_decode_sequence_flexi(const unsigned char *in, unsigned long *inlen, ltc_asn1_list **out); +#define der_free_sequence_flexi der_sequence_free +void der_sequence_free(ltc_asn1_list *in); +void der_sequence_shrink(ltc_asn1_list *in); + +/* BOOLEAN */ +int der_length_boolean(unsigned long *outlen); +int der_encode_boolean(int in, + unsigned char *out, unsigned long *outlen); +int der_decode_boolean(const unsigned char *in, unsigned long inlen, + int *out); +/* INTEGER */ +int der_encode_integer(void *num, unsigned char *out, unsigned long *outlen); +int der_decode_integer(const unsigned char *in, unsigned long inlen, void *num); +int der_length_integer(void *num, unsigned long *outlen); + +/* INTEGER -- handy for 0..2^32-1 values */ +int der_decode_short_integer(const unsigned char *in, unsigned long inlen, unsigned long *num); +int der_encode_short_integer(unsigned long num, unsigned char *out, unsigned long *outlen); +int der_length_short_integer(unsigned long num, unsigned long *outlen); + +/* BIT STRING */ +int der_encode_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_decode_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_encode_raw_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_decode_raw_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_length_bit_string(unsigned long nbits, unsigned long *outlen); + +/* OCTET STRING */ +int der_encode_octet_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_decode_octet_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_length_octet_string(unsigned long noctets, unsigned long *outlen); + +/* OBJECT IDENTIFIER */ +int der_encode_object_identifier(const unsigned long *words, unsigned long nwords, + unsigned char *out, unsigned long *outlen); +int der_decode_object_identifier(const unsigned char *in, unsigned long inlen, + unsigned long *words, unsigned long *outlen); +int der_length_object_identifier(const unsigned long *words, unsigned long nwords, unsigned long *outlen); +unsigned long der_object_identifier_bits(unsigned long x); + +/* IA5 STRING */ +int der_encode_ia5_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_decode_ia5_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_length_ia5_string(const unsigned char *octets, unsigned long noctets, unsigned long *outlen); + +int der_ia5_char_encode(int c); +int der_ia5_value_decode(int v); + +/* TELETEX STRING */ +int der_decode_teletex_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_length_teletex_string(const unsigned char *octets, unsigned long noctets, unsigned long *outlen); + +/* PRINTABLE STRING */ +int der_encode_printable_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_decode_printable_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int der_length_printable_string(const unsigned char *octets, unsigned long noctets, unsigned long *outlen); + +int der_printable_char_encode(int c); +int der_printable_value_decode(int v); + +/* UTF-8 */ +#if (defined(SIZE_MAX) || __STDC_VERSION__ >= 199901L || defined(WCHAR_MAX) || defined(__WCHAR_MAX__) || defined(_WCHAR_T) || defined(_WCHAR_T_DEFINED) || defined (__WCHAR_TYPE__)) && !defined(LTC_NO_WCHAR) + #if defined(__WCHAR_MAX__) + #define LTC_WCHAR_MAX __WCHAR_MAX__ + #else + #include + #define LTC_WCHAR_MAX WCHAR_MAX + #endif +/* please note that it might happen that LTC_WCHAR_MAX is undefined */ +#else + typedef ulong32 wchar_t; + #define LTC_WCHAR_MAX 0xFFFFFFFF +#endif + +int der_encode_utf8_string(const wchar_t *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +int der_decode_utf8_string(const unsigned char *in, unsigned long inlen, + wchar_t *out, unsigned long *outlen); +unsigned long der_utf8_charsize(const wchar_t c); +int der_length_utf8_string(const wchar_t *in, unsigned long noctets, unsigned long *outlen); + + +/* CHOICE */ +int der_decode_choice(const unsigned char *in, unsigned long *inlen, + ltc_asn1_list *list, unsigned long outlen); + +/* UTCTime */ +typedef struct { + unsigned YY, /* year */ + MM, /* month */ + DD, /* day */ + hh, /* hour */ + mm, /* minute */ + ss, /* second */ + off_dir, /* timezone offset direction 0 == +, 1 == - */ + off_hh, /* timezone offset hours */ + off_mm; /* timezone offset minutes */ +} ltc_utctime; + +int der_encode_utctime(const ltc_utctime *utctime, + unsigned char *out, unsigned long *outlen); + +int der_decode_utctime(const unsigned char *in, unsigned long *inlen, + ltc_utctime *out); + +int der_length_utctime(const ltc_utctime *utctime, unsigned long *outlen); + +/* GeneralizedTime */ +typedef struct { + unsigned YYYY, /* year */ + MM, /* month */ + DD, /* day */ + hh, /* hour */ + mm, /* minute */ + ss, /* second */ + fs, /* fractional seconds */ + off_dir, /* timezone offset direction 0 == +, 1 == - */ + off_hh, /* timezone offset hours */ + off_mm; /* timezone offset minutes */ +} ltc_generalizedtime; + +int der_encode_generalizedtime(const ltc_generalizedtime *gtime, + unsigned char *out, unsigned long *outlen); + +int der_decode_generalizedtime(const unsigned char *in, unsigned long *inlen, + ltc_generalizedtime *out); + +int der_length_generalizedtime(const ltc_generalizedtime *gtime, unsigned long *outlen); + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_pkcs.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_pkcs.h new file mode 100644 index 0000000..a0aa892 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_pkcs.h @@ -0,0 +1,99 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* PKCS Header Info */ + +/* ===> PKCS #1 -- RSA Cryptography <=== */ +#ifdef LTC_PKCS_1 + +enum ltc_pkcs_1_v1_5_blocks +{ + LTC_PKCS_1_EMSA = 1, /* Block type 1 (PKCS #1 v1.5 signature padding) */ + LTC_PKCS_1_EME = 2 /* Block type 2 (PKCS #1 v1.5 encryption padding) */ +}; + +enum ltc_pkcs_1_paddings +{ + LTC_PKCS_1_V1_5 = 1, /* PKCS #1 v1.5 padding (\sa ltc_pkcs_1_v1_5_blocks) */ + LTC_PKCS_1_OAEP = 2, /* PKCS #1 v2.0 encryption padding */ + LTC_PKCS_1_PSS = 3, /* PKCS #1 v2.1 signature padding */ + LTC_PKCS_1_V1_5_NA1 = 4 /* PKCS #1 v1.5 padding - No ASN.1 (\sa ltc_pkcs_1_v1_5_blocks) */ +}; + +int pkcs_1_mgf1( int hash_idx, + const unsigned char *seed, unsigned long seedlen, + unsigned char *mask, unsigned long masklen); + +int pkcs_1_i2osp(void *n, unsigned long modulus_len, unsigned char *out); +int pkcs_1_os2ip(void *n, unsigned char *in, unsigned long inlen); + +/* *** v1.5 padding */ +int pkcs_1_v1_5_encode(const unsigned char *msg, + unsigned long msglen, + int block_type, + unsigned long modulus_bitlen, + prng_state *prng, + int prng_idx, + unsigned char *out, + unsigned long *outlen); + +int pkcs_1_v1_5_decode(const unsigned char *msg, + unsigned long msglen, + int block_type, + unsigned long modulus_bitlen, + unsigned char *out, + unsigned long *outlen, + int *is_valid); + +/* *** v2.1 padding */ +int pkcs_1_oaep_encode(const unsigned char *msg, unsigned long msglen, + const unsigned char *lparam, unsigned long lparamlen, + unsigned long modulus_bitlen, prng_state *prng, + int prng_idx, int hash_idx, + unsigned char *out, unsigned long *outlen); + +int pkcs_1_oaep_decode(const unsigned char *msg, unsigned long msglen, + const unsigned char *lparam, unsigned long lparamlen, + unsigned long modulus_bitlen, int hash_idx, + unsigned char *out, unsigned long *outlen, + int *res); + +int pkcs_1_pss_encode(const unsigned char *msghash, unsigned long msghashlen, + unsigned long saltlen, prng_state *prng, + int prng_idx, int hash_idx, + unsigned long modulus_bitlen, + unsigned char *out, unsigned long *outlen); + +int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen, + const unsigned char *sig, unsigned long siglen, + unsigned long saltlen, int hash_idx, + unsigned long modulus_bitlen, int *res); + +#endif /* LTC_PKCS_1 */ + +/* ===> PKCS #5 -- Password Based Cryptography <=== */ +#ifdef LTC_PKCS_5 + +/* Algorithm #1 (PBKDF1) */ +int pkcs_5_alg1(const unsigned char *password, unsigned long password_len, + const unsigned char *salt, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen); + +/* Algorithm #1 (PBKDF1) - OpenSSL-compatible variant for arbitrarily-long keys. + Compatible with EVP_BytesToKey() */ +int pkcs_5_alg1_openssl(const unsigned char *password, + unsigned long password_len, + const unsigned char *salt, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen); + +/* Algorithm #2 (PBKDF2) */ +int pkcs_5_alg2(const unsigned char *password, unsigned long password_len, + const unsigned char *salt, unsigned long salt_len, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen); + +int pkcs_5_test (void); +#endif /* LTC_PKCS_5 */ + diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_private.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_private.h new file mode 100644 index 0000000..a65c83e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_private.h @@ -0,0 +1,500 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt.h" + +/* + * Internal Macros + */ + +#define LTC_PAD_MASK (0xF000U) + +/* `NULL` as defined by the standard is not guaranteed to be of a pointer + * type. In order to make sure that in vararg API's a pointer type is used, + * define our own version and use that one internally. + */ +#ifndef LTC_NULL + #define LTC_NULL ((void *)0) +#endif + +/* + * Internal Enums + */ + +enum ltc_oid_id { + LTC_OID_RSA, + LTC_OID_DSA, + LTC_OID_EC, + LTC_OID_EC_PRIMEF, + LTC_OID_X25519, + LTC_OID_ED25519, +}; + +/* + * Internal Types + */ + +typedef struct { + int size; + const char *name, *base, *prime; +} ltc_dh_set_type; + + +typedef int (*fn_kdf_t)(const unsigned char *password, unsigned long password_len, + const unsigned char *salt, unsigned long salt_len, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen); + +typedef struct { + /* KDF */ + fn_kdf_t kdf; + /* Hash or HMAC */ + const char* h; + /* cipher */ + const char* c; + unsigned long keylen; + /* not used for pbkdf2 */ + unsigned long blocklen; +} pbes_properties; + +typedef struct +{ + pbes_properties type; + const void *pwd; + unsigned long pwdlen; + ltc_asn1_list *enc_data; + ltc_asn1_list *salt; + ltc_asn1_list *iv; + unsigned long iterations; + /* only used for RC2 */ + unsigned long key_bits; +} pbes_arg; + +/* + * Internal functions + */ + + +/* tomcrypt_cipher.h */ + +void blowfish_enc(ulong32 *data, unsigned long blocks, const symmetric_key *skey); +int blowfish_expand(const unsigned char *key, int keylen, + const unsigned char *data, int datalen, + symmetric_key *skey); +int blowfish_setup_with_data(const unsigned char *key, int keylen, + const unsigned char *data, int datalen, + symmetric_key *skey); + +/* tomcrypt_hash.h */ + +/* a macro for making hash "process" functions */ +#define HASH_PROCESS_(func_name, compress_name, compress_n_name, state_var, block_size) \ +int func_name (hash_state * md, const unsigned char *in, unsigned long inlen) \ +{ \ + unsigned long n, blocks; \ + int err; \ + int (*compress)(hash_state *, const unsigned char *) = compress_name; \ + int (*compress_n)(hash_state *, const unsigned char *, int) = compress_n_name;\ + LTC_ARGCHK(md != NULL); \ + LTC_ARGCHK(in != NULL); \ + if (md-> state_var .curlen > sizeof(md-> state_var .buf)) { \ + return CRYPT_INVALID_ARG; \ + } \ + if (((md-> state_var .length + inlen * 8) < md-> state_var .length) \ + || ((inlen * 8) < inlen)) { \ + return CRYPT_HASH_OVERFLOW; \ + } \ + while (inlen > 0) { \ + if (md-> state_var .curlen == 0 && inlen >= block_size) { \ + if (compress_n) { \ + blocks = inlen / block_size; \ + err = compress_n (md, in, blocks); \ + } else { \ + blocks = 1; \ + err = compress (md, in); \ + } \ + if (err != CRYPT_OK) \ + return err; \ + md-> state_var .length += blocks * block_size * 8; \ + in += blocks * block_size; \ + inlen -= blocks * block_size; \ + } else { \ + n = MIN(inlen, (block_size - md-> state_var .curlen)); \ + XMEMCPY(md-> state_var .buf + md-> state_var.curlen, in, (size_t)n); \ + md-> state_var .curlen += n; \ + in += n; \ + inlen -= n; \ + if (md-> state_var .curlen == block_size) { \ + if (compress_n) { \ + err = compress_n (md, md-> state_var .buf, 1); \ + } else { \ + err = compress (md, md-> state_var .buf); \ + } \ + if (err != CRYPT_OK) { \ + return err; \ + } \ + md-> state_var .length += 8*block_size; \ + md-> state_var .curlen = 0; \ + } \ + } \ + } \ + return CRYPT_OK; \ +} + +/* define a hash "process" function based on a 1-block compress function */ +#define HASH_PROCESS(func_name, compress_name, state_var, block_size) \ + HASH_PROCESS_(func_name, compress_name, NULL, state_var, block_size) + +/* define a hash "process" function based on a n-block compress function */ +#define HASH_PROCESS_NBLOCKS(func_name, compress_n_name, state_var, block_size) \ + HASH_PROCESS_(func_name, NULL, compress_n_name, state_var, block_size) + + +/* tomcrypt_mac.h */ + +int ocb3_int_ntz(unsigned long x); +void ocb3_int_xor_blocks(unsigned char *out, const unsigned char *block_a, const unsigned char *block_b, unsigned long block_len); + + +/* tomcrypt_math.h */ + +#if !defined(DESC_DEF_ONLY) + +#define MP_DIGIT_BIT ltc_mp.bits_per_digit + +/* some handy macros */ +#define mp_init(a) ltc_mp.init(a) +#define mp_init_multi ltc_init_multi +#define mp_init_size(a, b) ltc_mp.init_size(a, b) +#define mp_init_multi_size ltc_init_multi_size +#define mp_clear(a) ltc_mp.deinit(a) +#define mp_clear_multi ltc_deinit_multi +#define mp_cleanup_multi ltc_cleanup_multi +#define mp_init_copy(a, b) ltc_mp.init_copy(a, b) + +#define mp_neg(a, b) ltc_mp.neg(a, b) +#define mp_copy(a, b) ltc_mp.copy(a, b) + +#define mp_set(a, b) ltc_mp.set_int(a, b) +#define mp_set_int(a, b) ltc_mp.set_int(a, b) +#define mp_get_int(a) ltc_mp.get_int(a) +#define mp_get_digit(a, n) ltc_mp.get_digit(a, n) +#define mp_get_digit_count(a) ltc_mp.get_digit_count(a) +#define mp_cmp(a, b) ltc_mp.compare(a, b) +#define mp_cmp_d(a, b) ltc_mp.compare_d(a, b) +#define mp_count_bits(a) ltc_mp.count_bits(a) +#define mp_cnt_lsb(a) ltc_mp.count_lsb_bits(a) +#define mp_2expt(a, b) ltc_mp.twoexpt(a, b) + +#define mp_read_radix(a, b, c) ltc_mp.read_radix(a, b, c) +#define mp_toradix(a, b, c) ltc_mp.write_radix(a, b, c) +#define mp_unsigned_bin_size(a) ltc_mp.unsigned_size(a) +#define mp_to_unsigned_bin(a, b) ltc_mp.unsigned_write(a, b) +#define mp_read_unsigned_bin(a, b, c) ltc_mp.unsigned_read(a, b, c) + +#define mp_add(a, b, c) ltc_mp.add(a, b, c) +#define mp_add_d(a, b, c) ltc_mp.addi(a, b, c) +#define mp_sub(a, b, c) ltc_mp.sub(a, b, c) +#define mp_sub_d(a, b, c) ltc_mp.subi(a, b, c) +#define mp_mul(a, b, c) ltc_mp.mul(a, b, c) +#define mp_mul_d(a, b, c) ltc_mp.muli(a, b, c) +#define mp_sqr(a, b) ltc_mp.sqr(a, b) +#define mp_sqrtmod_prime(a, b, c) ltc_mp.sqrtmod_prime(a, b, c) +#define mp_div(a, b, c, d) ltc_mp.mpdiv(a, b, c, d) +#define mp_div_2(a, b) ltc_mp.div_2(a, b) +#define mp_mod(a, b, c) ltc_mp.mpdiv(a, b, NULL, c) +#define mp_mod_d(a, b, c) ltc_mp.modi(a, b, c) +#define mp_gcd(a, b, c) ltc_mp.gcd(a, b, c) +#define mp_lcm(a, b, c) ltc_mp.lcm(a, b, c) + +#define mp_addmod(a, b, c, d) ltc_mp.addmod(a, b, c, d) +#define mp_submod(a, b, c, d) ltc_mp.submod(a, b, c, d) +#define mp_mulmod(a, b, c, d) ltc_mp.mulmod(a, b, c, d) +#define mp_sqrmod(a, b, c) ltc_mp.sqrmod(a, b, c) +#define mp_invmod(a, b, c) ltc_mp.invmod(a, b, c) + +#define mp_montgomery_setup(a, b) ltc_mp.montgomery_setup(a, b) +#define mp_montgomery_normalization(a, b) ltc_mp.montgomery_normalization(a, b) +#define mp_montgomery_reduce(a, b, c) ltc_mp.montgomery_reduce(a, b, c) +#define mp_montgomery_free(a) ltc_mp.montgomery_deinit(a) + +#define mp_exptmod(a,b,c,d) ltc_mp.exptmod(a,b,c,d) +#define mp_prime_is_prime(a, b, c) ltc_mp.isprime(a, b, c) + +#define mp_iszero(a) (mp_cmp_d(a, 0) == LTC_MP_EQ ? LTC_MP_YES : LTC_MP_NO) +#define mp_isodd(a) (mp_get_digit_count(a) > 0 ? (mp_get_digit(a, 0) & 1 ? LTC_MP_YES : LTC_MP_NO) : LTC_MP_NO) +#define mp_exch(a, b) do { void *ABC__tmp = a; a = b; b = ABC__tmp; } while(0) + +#define mp_tohex(a, b) mp_toradix(a, b, 16) + +#define mp_rand(a, b) ltc_mp.rand(a, b) + +#endif + + +/* tomcrypt_misc.h */ + +typedef enum { + /** Use `\r\n` as line separator */ + BASE64_PEM_CRLF = 1, + /** Create output with 72 chars line length */ + BASE64_PEM_SSH = 2, +} base64_pem_flags; + +int base64_encode_pem(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen, + unsigned int flags); + +void copy_or_zeromem(const unsigned char* src, unsigned char* dest, unsigned long len, int coz); + +int pbes_decrypt(const pbes_arg *arg, unsigned char *dec_data, unsigned long *dec_size); + +int pbes1_extract(const ltc_asn1_list *s, pbes_arg *res); +int pbes2_extract(const ltc_asn1_list *s, pbes_arg *res); + + +/* tomcrypt_pk.h */ + +int rand_bn_bits(void *N, int bits, prng_state *prng, int wprng); +int rand_bn_upto(void *N, void *limit, prng_state *prng, int wprng); + +int pk_get_oid(enum ltc_oid_id id, const char **st); +int pk_oid_str_to_num(const char *OID, unsigned long *oid, unsigned long *oidlen); +int pk_oid_num_to_str(const unsigned long *oid, unsigned long oidlen, char *OID, unsigned long *outlen); + +/* ---- DH Routines ---- */ +#ifdef LTC_MRSA +int rsa_init(rsa_key *key); +void rsa_shrink_key(rsa_key *key); +int rsa_make_key_bn_e(prng_state *prng, int wprng, int size, void *e, + rsa_key *key); /* used by op-tee */ +int rsa_import_pkcs1(const unsigned char *in, unsigned long inlen, rsa_key *key); +#endif /* LTC_MRSA */ + +/* ---- DH Routines ---- */ +#ifdef LTC_MDH +extern const ltc_dh_set_type ltc_dh_sets[]; + +int dh_check_pubkey(const dh_key *key); +#endif /* LTC_MDH */ + +/* ---- ECC Routines ---- */ +#ifdef LTC_MECC +int ecc_set_curve_from_mpis(void *a, void *b, void *prime, void *order, void *gx, void *gy, unsigned long cofactor, ecc_key *key); +int ecc_copy_curve(const ecc_key *srckey, ecc_key *key); +int ecc_set_curve_by_size(int size, ecc_key *key); +int ecc_import_subject_public_key_info(const unsigned char *in, unsigned long inlen, ecc_key *key); + +#ifdef LTC_SSH +int ecc_ssh_ecdsa_encode_name(char *buffer, unsigned long *buflen, const ecc_key *key); +#endif + +/* low level functions */ +ecc_point *ltc_ecc_new_point(void); +void ltc_ecc_del_point(ecc_point *p); +int ltc_ecc_set_point_xyz(ltc_mp_digit x, ltc_mp_digit y, ltc_mp_digit z, ecc_point *p); +int ltc_ecc_copy_point(const ecc_point *src, ecc_point *dst); +int ltc_ecc_is_point(const ltc_ecc_dp *dp, void *x, void *y); +int ltc_ecc_is_point_at_infinity(const ecc_point *P, void *modulus, int *retval); +int ltc_ecc_import_point(const unsigned char *in, unsigned long inlen, void *prime, void *a, void *b, void *x, void *y); +int ltc_ecc_export_point(unsigned char *out, unsigned long *outlen, void *x, void *y, unsigned long size, int compressed); +int ltc_ecc_verify_key(const ecc_key *key); + +/* point ops (mp == montgomery digit) */ +#if !defined(LTC_MECC_ACCEL) || defined(LTM_DESC) || defined(GMP_DESC) +/* R = 2P */ +int ltc_ecc_projective_dbl_point(const ecc_point *P, ecc_point *R, void *ma, void *modulus, void *mp); + +/* R = P + Q */ +int ltc_ecc_projective_add_point(const ecc_point *P, const ecc_point *Q, ecc_point *R, void *ma, void *modulus, void *mp); +#endif + +#if defined(LTC_MECC_FP) +/* optimized point multiplication using fixed point cache (HAC algorithm 14.117) */ +int ltc_ecc_fp_mulmod(void *k, ecc_point *G, ecc_point *R, void *a, void *modulus, int map); + +/* functions for saving/loading/freeing/adding to fixed point cache */ +int ltc_ecc_fp_save_state(unsigned char **out, unsigned long *outlen); +int ltc_ecc_fp_restore_state(unsigned char *in, unsigned long inlen); +void ltc_ecc_fp_free(void); +int ltc_ecc_fp_add_point(ecc_point *g, void *modulus, int lock); + +/* lock/unlock all points currently in fixed point cache */ +void ltc_ecc_fp_tablelock(int lock); +#endif + +/* R = kG */ +int ltc_ecc_mulmod(void *k, const ecc_point *G, ecc_point *R, void *a, void *modulus, int map); + +#ifdef LTC_ECC_SHAMIR +/* kA*A + kB*B = C */ +int ltc_ecc_mul2add(const ecc_point *A, void *kA, + const ecc_point *B, void *kB, + ecc_point *C, + void *ma, + void *modulus); + +#ifdef LTC_MECC_FP +/* Shamir's trick with optimized point multiplication using fixed point cache */ +int ltc_ecc_fp_mul2add(const ecc_point *A, void *kA, + const ecc_point *B, void *kB, + ecc_point *C, + void *ma, + void *modulus); +#endif + +#endif + + +/* map P to affine from projective */ +int ltc_ecc_map(ecc_point *P, void *modulus, void *mp); +#endif /* LTC_MECC */ + +#ifdef LTC_MDSA +int dsa_int_validate_xy(const dsa_key *key, int *stat); +int dsa_int_validate_pqg(const dsa_key *key, int *stat); +int dsa_int_validate_primes(const dsa_key *key, int *stat); +#endif /* LTC_MDSA */ + + +#ifdef LTC_CURVE25519 + +int tweetnacl_crypto_sign( + unsigned char *sm,unsigned long long *smlen, + const unsigned char *m,unsigned long long mlen, + const unsigned char *sk,const unsigned char *pk, + const unsigned char *ctx,unsigned long long cs); +int tweetnacl_crypto_sign_open( + int *stat, + unsigned char *m,unsigned long long *mlen, + const unsigned char *sm,unsigned long long smlen, + const unsigned char *ctx, unsigned long long cs, + const unsigned char *pk); +int tweetnacl_crypto_sign_keypair(prng_state *prng, int wprng, unsigned char *pk,unsigned char *sk); +int tweetnacl_crypto_sk_to_pk(unsigned char *pk, const unsigned char *sk); +int tweetnacl_crypto_scalarmult(unsigned char *q, const unsigned char *n, const unsigned char *p); +int tweetnacl_crypto_scalarmult_base(unsigned char *q,const unsigned char *n); +int tweetnacl_crypto_ph(unsigned char *out, const unsigned char *msg, unsigned long long msglen); + +typedef int (*sk_to_pk)(unsigned char *pk ,const unsigned char *sk); +int ec25519_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + enum ltc_oid_id id, sk_to_pk fp, + curve25519_key *key); +int ec25519_export( unsigned char *out, unsigned long *outlen, + int which, + const curve25519_key *key); +int ec25519_crypto_ctx( unsigned char *out, unsigned long *outlen, + unsigned char flag, + const unsigned char *ctx, unsigned long ctxlen); +#endif /* LTC_CURVE25519 */ + +#ifdef LTC_DER + +#define LTC_ASN1_IS_TYPE(e, t) (((e) != NULL) && ((e)->type == (t))) + +/* DER handling */ +int der_decode_custom_type_ex(const unsigned char *in, unsigned long inlen, + ltc_asn1_list *root, + ltc_asn1_list *list, unsigned long outlen, unsigned int flags); + +int der_encode_asn1_identifier(const ltc_asn1_list *id, unsigned char *out, unsigned long *outlen); +int der_decode_asn1_identifier(const unsigned char *in, unsigned long *inlen, ltc_asn1_list *id); +int der_length_asn1_identifier(const ltc_asn1_list *id, unsigned long *idlen); + +int der_encode_asn1_length(unsigned long len, unsigned char* out, unsigned long* outlen); +int der_decode_asn1_length(const unsigned char *in, unsigned long *inlen, unsigned long *outlen); +int der_length_asn1_length(unsigned long len, unsigned long *outlen); + +int der_length_sequence_ex(const ltc_asn1_list *list, unsigned long inlen, + unsigned long *outlen, unsigned long *payloadlen); + +extern const ltc_asn1_type der_asn1_tag_to_type_map[]; +extern const unsigned long der_asn1_tag_to_type_map_sz; + +extern const int der_asn1_type_to_identifier_map[]; +extern const unsigned long der_asn1_type_to_identifier_map_sz; + +int der_decode_sequence_multi_ex(const unsigned char *in, unsigned long inlen, unsigned int flags, ...) + LTC_NULL_TERMINATED; + +int der_teletex_char_encode(int c); +int der_teletex_value_decode(int v); + +int der_utf8_valid_char(const wchar_t c); + +typedef int (*public_key_decode_cb)(const unsigned char *in, unsigned long inlen, void *ctx); + +int x509_decode_public_key_from_certificate(const unsigned char *in, unsigned long inlen, + enum ltc_oid_id algorithm, ltc_asn1_type param_type, + ltc_asn1_list* parameters, unsigned long *parameters_len, + public_key_decode_cb callback, void *ctx); + +/* SUBJECT PUBLIC KEY INFO */ +int x509_encode_subject_public_key_info(unsigned char *out, unsigned long *outlen, + unsigned int algorithm, const void* public_key, unsigned long public_key_len, + ltc_asn1_type parameters_type, ltc_asn1_list* parameters, unsigned long parameters_len); + +int x509_decode_subject_public_key_info(const unsigned char *in, unsigned long inlen, + unsigned int algorithm, void* public_key, unsigned long* public_key_len, + ltc_asn1_type parameters_type, ltc_asn1_list* parameters, unsigned long *parameters_len); + +int pk_oid_cmp_with_ulong(const char *o1, const unsigned long *o2, unsigned long o2size); +int pk_oid_cmp_with_asn1(const char *o1, const ltc_asn1_list *o2); + +#endif /* LTC_DER */ + +/* tomcrypt_pkcs.h */ + +#ifdef LTC_PKCS_8 + +int pkcs8_decode_flexi(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + ltc_asn1_list **decoded_list); + +#endif /* LTC_PKCS_8 */ + + +#ifdef LTC_PKCS_12 + +int pkcs12_utf8_to_utf16(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +int pkcs12_kdf( int hash_id, + const unsigned char *pw, unsigned long pwlen, + const unsigned char *salt, unsigned long saltlen, + unsigned int iterations, unsigned char purpose, + unsigned char *out, unsigned long outlen); + +#endif /* LTC_PKCS_12 */ + +/* tomcrypt_prng.h */ + +#define LTC_PRNG_EXPORT(which) \ +int which ## _export(unsigned char *out, unsigned long *outlen, prng_state *prng) \ +{ \ + unsigned long len = which ## _desc.export_size; \ + \ + LTC_ARGCHK(prng != NULL); \ + LTC_ARGCHK(out != NULL); \ + LTC_ARGCHK(outlen != NULL); \ + \ + if (*outlen < len) { \ + *outlen = len; \ + return CRYPT_BUFFER_OVERFLOW; \ + } \ + \ + if (which ## _read(out, len, prng) != len) { \ + return CRYPT_ERROR_READPRNG; \ + } \ + \ + *outlen = len; \ + return CRYPT_OK; \ +} + +/* extract a byte portably */ +#ifdef _MSC_VER + #define LTC_BYTE(x, n) ((unsigned char)((x) >> (8 * (n)))) +#else + #define LTC_BYTE(x, n) (((x) >> (8 * (n))) & 255) +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_prng.h b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_prng.h new file mode 100644 index 0000000..e7e1716 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/headers/tomcrypt_prng.h @@ -0,0 +1,223 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* ---- PRNG Stuff ---- */ +#ifdef LTC_YARROW +struct yarrow_prng { + int cipher, hash; + unsigned char pool[MAXBLOCKSIZE]; + symmetric_CTR ctr; +}; +#endif + +#ifdef LTC_RC4 +struct rc4_prng { + rc4_state s; +}; +#endif + +#ifdef LTC_CHACHA20_PRNG +struct chacha20_prng { + chacha_state s; /* chacha state */ + unsigned char ent[40]; /* entropy buffer */ + unsigned long idx; /* entropy counter */ +}; +#endif + +#ifdef LTC_FORTUNA +struct fortuna_prng { + hash_state pool[LTC_FORTUNA_POOLS]; /* the pools */ + + symmetric_key skey; + + unsigned char K[32], /* the current key */ + IV[16]; /* IV for CTR mode */ + + unsigned long pool_idx, /* current pool we will add to */ + pool0_len; /* length of 0'th pool */ + ulong64 wd; + ulong64 reset_cnt; /* number of times we have reseeded */ +}; +#endif + +#ifdef LTC_SOBER128 +struct sober128_prng { + sober128_state s; /* sober128 state */ + unsigned char ent[40]; /* entropy buffer */ + unsigned long idx; /* entropy counter */ +}; +#endif + +typedef struct { + union { + char dummy[1]; +#ifdef LTC_YARROW + struct yarrow_prng yarrow; +#endif +#ifdef LTC_RC4 + struct rc4_prng rc4; +#endif +#ifdef LTC_CHACHA20_PRNG + struct chacha20_prng chacha; +#endif +#ifdef LTC_FORTUNA + struct fortuna_prng fortuna; +#endif +#ifdef LTC_SOBER128 + struct sober128_prng sober128; +#endif + } u; + short ready; /* ready flag 0-1 */ + LTC_MUTEX_TYPE(lock) /* lock */ +} prng_state; + +/** PRNG descriptor */ +extern const struct ltc_prng_descriptor { + /** Name of the PRNG */ + const char *name; + /** size in bytes of exported state */ + int export_size; + /** Start a PRNG state + @param prng [out] The state to initialize + @return CRYPT_OK if successful + */ + int (*start)(prng_state *prng); + /** Add entropy to the PRNG + @param in The entropy + @param inlen Length of the entropy (octets)\ + @param prng The PRNG state + @return CRYPT_OK if successful + */ + int (*add_entropy)(const unsigned char *in, unsigned long inlen, prng_state *prng); + /** Ready a PRNG state to read from + @param prng The PRNG state to ready + @return CRYPT_OK if successful + */ + int (*ready)(prng_state *prng); + /** Read from the PRNG + @param out [out] Where to store the data + @param outlen Length of data desired (octets) + @param prng The PRNG state to read from + @return Number of octets read + */ + unsigned long (*read)(unsigned char *out, unsigned long outlen, prng_state *prng); + /** Terminate a PRNG state + @param prng The PRNG state to terminate + @return CRYPT_OK if successful + */ + int (*done)(prng_state *prng); + /** Export a PRNG state + @param out [out] The destination for the state + @param outlen [in/out] The max size and resulting size of the PRNG state + @param prng The PRNG to export + @return CRYPT_OK if successful + */ + int (*pexport)(unsigned char *out, unsigned long *outlen, prng_state *prng); + /** Import a PRNG state + @param in The data to import + @param inlen The length of the data to import (octets) + @param prng The PRNG to initialize/import + @return CRYPT_OK if successful + */ + int (*pimport)(const unsigned char *in, unsigned long inlen, prng_state *prng); + /** Self-test the PRNG + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled + */ + int (*test)(void); +} *prng_descriptor[]; + +#ifdef LTC_YARROW +int yarrow_start(prng_state *prng); +int yarrow_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng); +int yarrow_ready(prng_state *prng); +unsigned long yarrow_read(unsigned char *out, unsigned long outlen, prng_state *prng); +int yarrow_done(prng_state *prng); +int yarrow_export(unsigned char *out, unsigned long *outlen, prng_state *prng); +int yarrow_import(const unsigned char *in, unsigned long inlen, prng_state *prng); +int yarrow_test(void); +extern const struct ltc_prng_descriptor yarrow_desc; +#endif + +#ifdef LTC_FORTUNA +int fortuna_start(prng_state *prng); +int fortuna_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng); +int fortuna_add_random_event(unsigned long source, unsigned long pool, const unsigned char *in, unsigned long inlen, prng_state *prng); +int fortuna_ready(prng_state *prng); +unsigned long fortuna_read(unsigned char *out, unsigned long outlen, prng_state *prng); +int fortuna_done(prng_state *prng); +int fortuna_export(unsigned char *out, unsigned long *outlen, prng_state *prng); +int fortuna_import(const unsigned char *in, unsigned long inlen, prng_state *prng); +int fortuna_update_seed(const unsigned char *in, unsigned long inlen, prng_state *prng); +int fortuna_test(void); +extern const struct ltc_prng_descriptor fortuna_desc; +#endif + +#ifdef LTC_RC4 +int rc4_start(prng_state *prng); +int rc4_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng); +int rc4_ready(prng_state *prng); +unsigned long rc4_read(unsigned char *out, unsigned long outlen, prng_state *prng); +int rc4_done(prng_state *prng); +int rc4_export(unsigned char *out, unsigned long *outlen, prng_state *prng); +int rc4_import(const unsigned char *in, unsigned long inlen, prng_state *prng); +int rc4_test(void); +extern const struct ltc_prng_descriptor rc4_desc; +#endif + +#ifdef LTC_CHACHA20_PRNG +int chacha20_prng_start(prng_state *prng); +int chacha20_prng_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng); +int chacha20_prng_ready(prng_state *prng); +unsigned long chacha20_prng_read(unsigned char *out, unsigned long outlen, prng_state *prng); +int chacha20_prng_done(prng_state *prng); +int chacha20_prng_export(unsigned char *out, unsigned long *outlen, prng_state *prng); +int chacha20_prng_import(const unsigned char *in, unsigned long inlen, prng_state *prng); +int chacha20_prng_test(void); +extern const struct ltc_prng_descriptor chacha20_prng_desc; +#endif + +#ifdef LTC_SPRNG +int sprng_start(prng_state *prng); +int sprng_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng); +int sprng_ready(prng_state *prng); +unsigned long sprng_read(unsigned char *out, unsigned long outlen, prng_state *prng); +int sprng_done(prng_state *prng); +int sprng_export(unsigned char *out, unsigned long *outlen, prng_state *prng); +int sprng_import(const unsigned char *in, unsigned long inlen, prng_state *prng); +int sprng_test(void); +extern const struct ltc_prng_descriptor sprng_desc; +#endif + +#ifdef LTC_SOBER128 +int sober128_start(prng_state *prng); +int sober128_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng); +int sober128_ready(prng_state *prng); +unsigned long sober128_read(unsigned char *out, unsigned long outlen, prng_state *prng); +int sober128_done(prng_state *prng); +int sober128_export(unsigned char *out, unsigned long *outlen, prng_state *prng); +int sober128_import(const unsigned char *in, unsigned long inlen, prng_state *prng); +int sober128_test(void); +extern const struct ltc_prng_descriptor sober128_desc; +#endif + +int find_prng(const char *name); +int register_prng(const struct ltc_prng_descriptor *prng); +int unregister_prng(const struct ltc_prng_descriptor *prng); +int register_all_prngs(void); +int prng_is_valid(int idx); +LTC_MUTEX_PROTO(ltc_prng_mutex) + +/* Slow RNG you **might** be able to use to seed a PRNG with. Be careful as this + * might not work on all platforms as planned + */ +unsigned long rng_get_bytes(unsigned char *out, + unsigned long outlen, + void (*callback)(void)); + +int rng_make_prng(int bits, int wprng, prng_state *prng, void (*callback)(void)); + +#ifdef LTC_PRNG_ENABLE_LTC_RNG +extern unsigned long (*ltc_rng)(unsigned char *out, unsigned long outlen, + void (*callback)(void)); +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac.c new file mode 100644 index 0000000..ac26e10 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac.c @@ -0,0 +1,56 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2BMAC + +/** + Initialize an BLAKE2B MAC context. + @param st The BLAKE2B MAC state + @param outlen The size of the MAC output (octets) + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int blake2bmac_init(blake2bmac_state *st, unsigned long outlen, const unsigned char *key, unsigned long keylen) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + return blake2b_init(st, outlen, key, keylen); +} + +/** + Process data through BLAKE2B MAC + @param st The BLAKE2B MAC state + @param in The data to send through HMAC + @param inlen The length of the data to HMAC (octets) + @return CRYPT_OK if successful +*/ +int blake2bmac_process(blake2bmac_state *st, const unsigned char *in, unsigned long inlen) +{ + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + return blake2b_process(st, in, inlen); +} + +/** + Terminate a BLAKE2B MAC session + @param st The BLAKE2B MAC state + @param mac [out] The destination of the BLAKE2B MAC authentication tag + @param maclen [in/out] The max size and resulting size of the BLAKE2B MAC authentication tag + @return CRYPT_OK if successful +*/ +int blake2bmac_done(blake2bmac_state *st, unsigned char *mac, unsigned long *maclen) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + LTC_ARGCHK(*maclen >= st->blake2b.outlen); + + *maclen = st->blake2b.outlen; + return blake2b_done(st, mac); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_file.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_file.c new file mode 100644 index 0000000..831e845 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_file.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2BMAC + +/** + BLAKE2B MAC a file + @param fname The name of the file you wish to BLAKE2B MAC + @param key The secret key + @param keylen The length of the secret key + @param mac [out] The BLAKE2B MAC authentication tag + @param maclen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int blake2bmac_file(const char *fname, const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(fname); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(mac); + LTC_UNUSED_PARAM(maclen); + return CRYPT_NOP; +#else + blake2bmac_state st; + FILE *in; + unsigned char *buf; + size_t x; + int err; + + LTC_ARGCHK(fname != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = blake2bmac_init(&st, *maclen, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(fname, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = blake2bmac_process(&st, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = blake2bmac_done(&st, mac, maclen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(blake2bmac_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_memory.c new file mode 100644 index 0000000..3f5e309 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_memory.c @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2BMAC + +/** + BLAKE2B MAC a block of memory to produce the authentication tag + @param key The secret key + @param keylen The length of the secret key (octets) + @param in The data to BLAKE2B MAC + @param inlen The length of the data to BLAKE2B MAC (octets) + @param mac [out] Destination of the authentication tag + @param maclen [in/out] Max size and resulting size of authentication tag + @return CRYPT_OK if successful +*/ +int blake2bmac_memory(const unsigned char *key, unsigned long keylen, const unsigned char *in, unsigned long inlen, unsigned char *mac, unsigned long *maclen) +{ + blake2bmac_state st; + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + if ((err = blake2bmac_init(&st, *maclen, key, keylen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = blake2bmac_process(&st, in, inlen)) != CRYPT_OK) { goto LBL_ERR; } + err = blake2bmac_done(&st, mac, maclen); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(blake2bmac_state)); +#endif + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_memory_multi.c new file mode 100644 index 0000000..403fba3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_memory_multi.c @@ -0,0 +1,52 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" +#include + +#ifdef LTC_BLAKE2BMAC + +/** + BLAKE2B MAC multiple blocks of memory to produce the authentication tag + @param key The secret key + @param keylen The length of the secret key (octets) + @param mac [out] Destination of the authentication tag + @param maclen [in/out] Max size and resulting size of authentication tag + @param in The data to BLAKE2B MAC + @param inlen The length of the data to BLAKE2B MAC (octets) + @param ... tuples of (data,len) pairs to BLAKE2B MAC, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int blake2bmac_memory_multi(const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen, const unsigned char *in, unsigned long inlen, ...) +{ + blake2bmac_state st; + int err; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + va_start(args, inlen); + curptr = in; + curlen = inlen; + if ((err = blake2bmac_init(&st, *maclen, key, keylen)) != CRYPT_OK) { goto LBL_ERR; } + for (;;) { + if ((err = blake2bmac_process(&st, curptr, curlen)) != CRYPT_OK) { goto LBL_ERR; } + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) break; + curlen = va_arg(args, unsigned long); + } + err = blake2bmac_done(&st, mac, maclen); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(blake2bmac_state)); +#endif + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_test.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_test.c new file mode 100644 index 0000000..b2651cc --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2bmac_test.c @@ -0,0 +1,304 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2BMAC + +int blake2bmac_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const unsigned char tests[256][64] = { + /* source: https://github.com/BLAKE2/BLAKE2/blob/master/testvectors/blake2b-kat.txt */ + { 0x10, 0xeb, 0xb6, 0x77, 0x00, 0xb1, 0x86, 0x8e, 0xfb, 0x44, 0x17, 0x98, 0x7a, 0xcf, 0x46, 0x90, 0xae, 0x9d, 0x97, 0x2f, 0xb7, 0xa5, 0x90, 0xc2, 0xf0, 0x28, 0x71, 0x79, 0x9a, 0xaa, 0x47, 0x86, 0xb5, 0xe9, 0x96, 0xe8, 0xf0, 0xf4, 0xeb, 0x98, 0x1f, 0xc2, 0x14, 0xb0, 0x05, 0xf4, 0x2d, 0x2f, 0xf4, 0x23, 0x34, 0x99, 0x39, 0x16, 0x53, 0xdf, 0x7a, 0xef, 0xcb, 0xc1, 0x3f, 0xc5, 0x15, 0x68 }, + { 0x96, 0x1f, 0x6d, 0xd1, 0xe4, 0xdd, 0x30, 0xf6, 0x39, 0x01, 0x69, 0x0c, 0x51, 0x2e, 0x78, 0xe4, 0xb4, 0x5e, 0x47, 0x42, 0xed, 0x19, 0x7c, 0x3c, 0x5e, 0x45, 0xc5, 0x49, 0xfd, 0x25, 0xf2, 0xe4, 0x18, 0x7b, 0x0b, 0xc9, 0xfe, 0x30, 0x49, 0x2b, 0x16, 0xb0, 0xd0, 0xbc, 0x4e, 0xf9, 0xb0, 0xf3, 0x4c, 0x70, 0x03, 0xfa, 0xc0, 0x9a, 0x5e, 0xf1, 0x53, 0x2e, 0x69, 0x43, 0x02, 0x34, 0xce, 0xbd }, + { 0xda, 0x2c, 0xfb, 0xe2, 0xd8, 0x40, 0x9a, 0x0f, 0x38, 0x02, 0x61, 0x13, 0x88, 0x4f, 0x84, 0xb5, 0x01, 0x56, 0x37, 0x1a, 0xe3, 0x04, 0xc4, 0x43, 0x01, 0x73, 0xd0, 0x8a, 0x99, 0xd9, 0xfb, 0x1b, 0x98, 0x31, 0x64, 0xa3, 0x77, 0x07, 0x06, 0xd5, 0x37, 0xf4, 0x9e, 0x0c, 0x91, 0x6d, 0x9f, 0x32, 0xb9, 0x5c, 0xc3, 0x7a, 0x95, 0xb9, 0x9d, 0x85, 0x74, 0x36, 0xf0, 0x23, 0x2c, 0x88, 0xa9, 0x65 }, + { 0x33, 0xd0, 0x82, 0x5d, 0xdd, 0xf7, 0xad, 0xa9, 0x9b, 0x0e, 0x7e, 0x30, 0x71, 0x04, 0xad, 0x07, 0xca, 0x9c, 0xfd, 0x96, 0x92, 0x21, 0x4f, 0x15, 0x61, 0x35, 0x63, 0x15, 0xe7, 0x84, 0xf3, 0xe5, 0xa1, 0x7e, 0x36, 0x4a, 0xe9, 0xdb, 0xb1, 0x4c, 0xb2, 0x03, 0x6d, 0xf9, 0x32, 0xb7, 0x7f, 0x4b, 0x29, 0x27, 0x61, 0x36, 0x5f, 0xb3, 0x28, 0xde, 0x7a, 0xfd, 0xc6, 0xd8, 0x99, 0x8f, 0x5f, 0xc1 }, + { 0xbe, 0xaa, 0x5a, 0x3d, 0x08, 0xf3, 0x80, 0x71, 0x43, 0xcf, 0x62, 0x1d, 0x95, 0xcd, 0x69, 0x05, 0x14, 0xd0, 0xb4, 0x9e, 0xff, 0xf9, 0xc9, 0x1d, 0x24, 0xb5, 0x92, 0x41, 0xec, 0x0e, 0xef, 0xa5, 0xf6, 0x01, 0x96, 0xd4, 0x07, 0x04, 0x8b, 0xba, 0x8d, 0x21, 0x46, 0x82, 0x8e, 0xbc, 0xb0, 0x48, 0x8d, 0x88, 0x42, 0xfd, 0x56, 0xbb, 0x4f, 0x6d, 0xf8, 0xe1, 0x9c, 0x4b, 0x4d, 0xaa, 0xb8, 0xac }, + { 0x09, 0x80, 0x84, 0xb5, 0x1f, 0xd1, 0x3d, 0xea, 0xe5, 0xf4, 0x32, 0x0d, 0xe9, 0x4a, 0x68, 0x8e, 0xe0, 0x7b, 0xae, 0xa2, 0x80, 0x04, 0x86, 0x68, 0x9a, 0x86, 0x36, 0x11, 0x7b, 0x46, 0xc1, 0xf4, 0xc1, 0xf6, 0xaf, 0x7f, 0x74, 0xae, 0x7c, 0x85, 0x76, 0x00, 0x45, 0x6a, 0x58, 0xa3, 0xaf, 0x25, 0x1d, 0xc4, 0x72, 0x3a, 0x64, 0xcc, 0x7c, 0x0a, 0x5a, 0xb6, 0xd9, 0xca, 0xc9, 0x1c, 0x20, 0xbb }, + { 0x60, 0x44, 0x54, 0x0d, 0x56, 0x08, 0x53, 0xeb, 0x1c, 0x57, 0xdf, 0x00, 0x77, 0xdd, 0x38, 0x10, 0x94, 0x78, 0x1c, 0xdb, 0x90, 0x73, 0xe5, 0xb1, 0xb3, 0xd3, 0xf6, 0xc7, 0x82, 0x9e, 0x12, 0x06, 0x6b, 0xba, 0xca, 0x96, 0xd9, 0x89, 0xa6, 0x90, 0xde, 0x72, 0xca, 0x31, 0x33, 0xa8, 0x36, 0x52, 0xba, 0x28, 0x4a, 0x6d, 0x62, 0x94, 0x2b, 0x27, 0x1f, 0xfa, 0x26, 0x20, 0xc9, 0xe7, 0x5b, 0x1f }, + { 0x7a, 0x8c, 0xfe, 0x9b, 0x90, 0xf7, 0x5f, 0x7e, 0xcb, 0x3a, 0xcc, 0x05, 0x3a, 0xae, 0xd6, 0x19, 0x31, 0x12, 0xb6, 0xf6, 0xa4, 0xae, 0xeb, 0x3f, 0x65, 0xd3, 0xde, 0x54, 0x19, 0x42, 0xde, 0xb9, 0xe2, 0x22, 0x81, 0x52, 0xa3, 0xc4, 0xbb, 0xbe, 0x72, 0xfc, 0x3b, 0x12, 0x62, 0x95, 0x28, 0xcf, 0xbb, 0x09, 0xfe, 0x63, 0x0f, 0x04, 0x74, 0x33, 0x9f, 0x54, 0xab, 0xf4, 0x53, 0xe2, 0xed, 0x52 }, + { 0x38, 0x0b, 0xea, 0xf6, 0xea, 0x7c, 0xc9, 0x36, 0x5e, 0x27, 0x0e, 0xf0, 0xe6, 0xf3, 0xa6, 0x4f, 0xb9, 0x02, 0xac, 0xae, 0x51, 0xdd, 0x55, 0x12, 0xf8, 0x42, 0x59, 0xad, 0x2c, 0x91, 0xf4, 0xbc, 0x41, 0x08, 0xdb, 0x73, 0x19, 0x2a, 0x5b, 0xbf, 0xb0, 0xcb, 0xcf, 0x71, 0xe4, 0x6c, 0x3e, 0x21, 0xae, 0xe1, 0xc5, 0xe8, 0x60, 0xdc, 0x96, 0xe8, 0xeb, 0x0b, 0x7b, 0x84, 0x26, 0xe6, 0xab, 0xe9 }, + { 0x60, 0xfe, 0x3c, 0x45, 0x35, 0xe1, 0xb5, 0x9d, 0x9a, 0x61, 0xea, 0x85, 0x00, 0xbf, 0xac, 0x41, 0xa6, 0x9d, 0xff, 0xb1, 0xce, 0xad, 0xd9, 0xac, 0xa3, 0x23, 0xe9, 0xa6, 0x25, 0xb6, 0x4d, 0xa5, 0x76, 0x3b, 0xad, 0x72, 0x26, 0xda, 0x02, 0xb9, 0xc8, 0xc4, 0xf1, 0xa5, 0xde, 0x14, 0x0a, 0xc5, 0xa6, 0xc1, 0x12, 0x4e, 0x4f, 0x71, 0x8c, 0xe0, 0xb2, 0x8e, 0xa4, 0x73, 0x93, 0xaa, 0x66, 0x37 }, + { 0x4f, 0xe1, 0x81, 0xf5, 0x4a, 0xd6, 0x3a, 0x29, 0x83, 0xfe, 0xaa, 0xf7, 0x7d, 0x1e, 0x72, 0x35, 0xc2, 0xbe, 0xb1, 0x7f, 0xa3, 0x28, 0xb6, 0xd9, 0x50, 0x5b, 0xda, 0x32, 0x7d, 0xf1, 0x9f, 0xc3, 0x7f, 0x02, 0xc4, 0xb6, 0xf0, 0x36, 0x8c, 0xe2, 0x31, 0x47, 0x31, 0x3a, 0x8e, 0x57, 0x38, 0xb5, 0xfa, 0x2a, 0x95, 0xb2, 0x9d, 0xe1, 0xc7, 0xf8, 0x26, 0x4e, 0xb7, 0x7b, 0x69, 0xf5, 0x85, 0xcd }, + { 0xf2, 0x28, 0x77, 0x3c, 0xe3, 0xf3, 0xa4, 0x2b, 0x5f, 0x14, 0x4d, 0x63, 0x23, 0x7a, 0x72, 0xd9, 0x96, 0x93, 0xad, 0xb8, 0x83, 0x7d, 0x0e, 0x11, 0x2a, 0x8a, 0x0f, 0x8f, 0xff, 0xf2, 0xc3, 0x62, 0x85, 0x7a, 0xc4, 0x9c, 0x11, 0xec, 0x74, 0x0d, 0x15, 0x00, 0x74, 0x9d, 0xac, 0x9b, 0x1f, 0x45, 0x48, 0x10, 0x8b, 0xf3, 0x15, 0x57, 0x94, 0xdc, 0xc9, 0xe4, 0x08, 0x28, 0x49, 0xe2, 0xb8, 0x5b }, + { 0x96, 0x24, 0x52, 0xa8, 0x45, 0x5c, 0xc5, 0x6c, 0x85, 0x11, 0x31, 0x7e, 0x3b, 0x1f, 0x3b, 0x2c, 0x37, 0xdf, 0x75, 0xf5, 0x88, 0xe9, 0x43, 0x25, 0xfd, 0xd7, 0x70, 0x70, 0x35, 0x9c, 0xf6, 0x3a, 0x9a, 0xe6, 0xe9, 0x30, 0x93, 0x6f, 0xdf, 0x8e, 0x1e, 0x08, 0xff, 0xca, 0x44, 0x0c, 0xfb, 0x72, 0xc2, 0x8f, 0x06, 0xd8, 0x9a, 0x21, 0x51, 0xd1, 0xc4, 0x6c, 0xd5, 0xb2, 0x68, 0xef, 0x85, 0x63 }, + { 0x43, 0xd4, 0x4b, 0xfa, 0x18, 0x76, 0x8c, 0x59, 0x89, 0x6b, 0xf7, 0xed, 0x17, 0x65, 0xcb, 0x2d, 0x14, 0xaf, 0x8c, 0x26, 0x02, 0x66, 0x03, 0x90, 0x99, 0xb2, 0x5a, 0x60, 0x3e, 0x4d, 0xdc, 0x50, 0x39, 0xd6, 0xef, 0x3a, 0x91, 0x84, 0x7d, 0x10, 0x88, 0xd4, 0x01, 0xc0, 0xc7, 0xe8, 0x47, 0x78, 0x1a, 0x8a, 0x59, 0x0d, 0x33, 0xa3, 0xc6, 0xcb, 0x4d, 0xf0, 0xfa, 0xb1, 0xc2, 0xf2, 0x23, 0x55 }, + { 0xdc, 0xff, 0xa9, 0xd5, 0x8c, 0x2a, 0x4c, 0xa2, 0xcd, 0xbb, 0x0c, 0x7a, 0xa4, 0xc4, 0xc1, 0xd4, 0x51, 0x65, 0x19, 0x00, 0x89, 0xf4, 0xe9, 0x83, 0xbb, 0x1c, 0x2c, 0xab, 0x4a, 0xae, 0xff, 0x1f, 0xa2, 0xb5, 0xee, 0x51, 0x6f, 0xec, 0xd7, 0x80, 0x54, 0x02, 0x40, 0xbf, 0x37, 0xe5, 0x6c, 0x8b, 0xcc, 0xa7, 0xfa, 0xb9, 0x80, 0xe1, 0xe6, 0x1c, 0x94, 0x00, 0xd8, 0xa9, 0xa5, 0xb1, 0x4a, 0xc6 }, + { 0x6f, 0xbf, 0x31, 0xb4, 0x5a, 0xb0, 0xc0, 0xb8, 0xda, 0xd1, 0xc0, 0xf5, 0xf4, 0x06, 0x13, 0x79, 0x91, 0x2d, 0xde, 0x5a, 0xa9, 0x22, 0x09, 0x9a, 0x03, 0x0b, 0x72, 0x5c, 0x73, 0x34, 0x6c, 0x52, 0x42, 0x91, 0xad, 0xef, 0x89, 0xd2, 0xf6, 0xfd, 0x8d, 0xfc, 0xda, 0x6d, 0x07, 0xda, 0xd8, 0x11, 0xa9, 0x31, 0x45, 0x36, 0xc2, 0x91, 0x5e, 0xd4, 0x5d, 0xa3, 0x49, 0x47, 0xe8, 0x3d, 0xe3, 0x4e }, + { 0xa0, 0xc6, 0x5b, 0xdd, 0xde, 0x8a, 0xde, 0xf5, 0x72, 0x82, 0xb0, 0x4b, 0x11, 0xe7, 0xbc, 0x8a, 0xab, 0x10, 0x5b, 0x99, 0x23, 0x1b, 0x75, 0x0c, 0x02, 0x1f, 0x4a, 0x73, 0x5c, 0xb1, 0xbc, 0xfa, 0xb8, 0x75, 0x53, 0xbb, 0xa3, 0xab, 0xb0, 0xc3, 0xe6, 0x4a, 0x0b, 0x69, 0x55, 0x28, 0x51, 0x85, 0xa0, 0xbd, 0x35, 0xfb, 0x8c, 0xfd, 0xe5, 0x57, 0x32, 0x9b, 0xeb, 0xb1, 0xf6, 0x29, 0xee, 0x93 }, + { 0xf9, 0x9d, 0x81, 0x55, 0x50, 0x55, 0x8e, 0x81, 0xec, 0xa2, 0xf9, 0x67, 0x18, 0xae, 0xd1, 0x0d, 0x86, 0xf3, 0xf1, 0xcf, 0xb6, 0x75, 0xcc, 0xe0, 0x6b, 0x0e, 0xff, 0x02, 0xf6, 0x17, 0xc5, 0xa4, 0x2c, 0x5a, 0xa7, 0x60, 0x27, 0x0f, 0x26, 0x79, 0xda, 0x26, 0x77, 0xc5, 0xae, 0xb9, 0x4f, 0x11, 0x42, 0x27, 0x7f, 0x21, 0xc7, 0xf7, 0x9f, 0x3c, 0x4f, 0x0c, 0xce, 0x4e, 0xd8, 0xee, 0x62, 0xb1 }, + { 0x95, 0x39, 0x1d, 0xa8, 0xfc, 0x7b, 0x91, 0x7a, 0x20, 0x44, 0xb3, 0xd6, 0xf5, 0x37, 0x4e, 0x1c, 0xa0, 0x72, 0xb4, 0x14, 0x54, 0xd5, 0x72, 0xc7, 0x35, 0x6c, 0x05, 0xfd, 0x4b, 0xc1, 0xe0, 0xf4, 0x0b, 0x8b, 0xb8, 0xb4, 0xa9, 0xf6, 0xbc, 0xe9, 0xbe, 0x2c, 0x46, 0x23, 0xc3, 0x99, 0xb0, 0xdc, 0xa0, 0xda, 0xb0, 0x5c, 0xb7, 0x28, 0x1b, 0x71, 0xa2, 0x1b, 0x0e, 0xbc, 0xd9, 0xe5, 0x56, 0x70 }, + { 0x04, 0xb9, 0xcd, 0x3d, 0x20, 0xd2, 0x21, 0xc0, 0x9a, 0xc8, 0x69, 0x13, 0xd3, 0xdc, 0x63, 0x04, 0x19, 0x89, 0xa9, 0xa1, 0xe6, 0x94, 0xf1, 0xe6, 0x39, 0xa3, 0xba, 0x7e, 0x45, 0x18, 0x40, 0xf7, 0x50, 0xc2, 0xfc, 0x19, 0x1d, 0x56, 0xad, 0x61, 0xf2, 0xe7, 0x93, 0x6b, 0xc0, 0xac, 0x8e, 0x09, 0x4b, 0x60, 0xca, 0xee, 0xd8, 0x78, 0xc1, 0x87, 0x99, 0x04, 0x54, 0x02, 0xd6, 0x1c, 0xea, 0xf9 }, + { 0xec, 0x0e, 0x0e, 0xf7, 0x07, 0xe4, 0xed, 0x6c, 0x0c, 0x66, 0xf9, 0xe0, 0x89, 0xe4, 0x95, 0x4b, 0x05, 0x80, 0x30, 0xd2, 0xdd, 0x86, 0x39, 0x8f, 0xe8, 0x40, 0x59, 0x63, 0x1f, 0x9e, 0xe5, 0x91, 0xd9, 0xd7, 0x73, 0x75, 0x35, 0x51, 0x49, 0x17, 0x8c, 0x0c, 0xf8, 0xf8, 0xe7, 0xc4, 0x9e, 0xd2, 0xa5, 0xe4, 0xf9, 0x54, 0x88, 0xa2, 0x24, 0x70, 0x67, 0xc2, 0x08, 0x51, 0x0f, 0xad, 0xc4, 0x4c }, + { 0x9a, 0x37, 0xcc, 0xe2, 0x73, 0xb7, 0x9c, 0x09, 0x91, 0x36, 0x77, 0x51, 0x0e, 0xaf, 0x76, 0x88, 0xe8, 0x9b, 0x33, 0x14, 0xd3, 0x53, 0x2f, 0xd2, 0x76, 0x4c, 0x39, 0xde, 0x02, 0x2a, 0x29, 0x45, 0xb5, 0x71, 0x0d, 0x13, 0x51, 0x7a, 0xf8, 0xdd, 0xc0, 0x31, 0x66, 0x24, 0xe7, 0x3b, 0xec, 0x1c, 0xe6, 0x7d, 0xf1, 0x52, 0x28, 0x30, 0x20, 0x36, 0xf3, 0x30, 0xab, 0x0c, 0xb4, 0xd2, 0x18, 0xdd }, + { 0x4c, 0xf9, 0xbb, 0x8f, 0xb3, 0xd4, 0xde, 0x8b, 0x38, 0xb2, 0xf2, 0x62, 0xd3, 0xc4, 0x0f, 0x46, 0xdf, 0xe7, 0x47, 0xe8, 0xfc, 0x0a, 0x41, 0x4c, 0x19, 0x3d, 0x9f, 0xcf, 0x75, 0x31, 0x06, 0xce, 0x47, 0xa1, 0x8f, 0x17, 0x2f, 0x12, 0xe8, 0xa2, 0xf1, 0xc2, 0x67, 0x26, 0x54, 0x53, 0x58, 0xe5, 0xee, 0x28, 0xc9, 0xe2, 0x21, 0x3a, 0x87, 0x87, 0xaa, 0xfb, 0xc5, 0x16, 0xd2, 0x34, 0x31, 0x52 }, + { 0x64, 0xe0, 0xc6, 0x3a, 0xf9, 0xc8, 0x08, 0xfd, 0x89, 0x31, 0x37, 0x12, 0x98, 0x67, 0xfd, 0x91, 0x93, 0x9d, 0x53, 0xf2, 0xaf, 0x04, 0xbe, 0x4f, 0xa2, 0x68, 0x00, 0x61, 0x00, 0x06, 0x9b, 0x2d, 0x69, 0xda, 0xa5, 0xc5, 0xd8, 0xed, 0x7f, 0xdd, 0xcb, 0x2a, 0x70, 0xee, 0xec, 0xdf, 0x2b, 0x10, 0x5d, 0xd4, 0x6a, 0x1e, 0x3b, 0x73, 0x11, 0x72, 0x8f, 0x63, 0x9a, 0xb4, 0x89, 0x32, 0x6b, 0xc9 }, + { 0x5e, 0x9c, 0x93, 0x15, 0x8d, 0x65, 0x9b, 0x2d, 0xef, 0x06, 0xb0, 0xc3, 0xc7, 0x56, 0x50, 0x45, 0x54, 0x26, 0x62, 0xd6, 0xee, 0xe8, 0xa9, 0x6a, 0x89, 0xb7, 0x8a, 0xde, 0x09, 0xfe, 0x8b, 0x3d, 0xcc, 0x09, 0x6d, 0x4f, 0xe4, 0x88, 0x15, 0xd8, 0x8d, 0x8f, 0x82, 0x62, 0x01, 0x56, 0x60, 0x2a, 0xf5, 0x41, 0x95, 0x5e, 0x1f, 0x6c, 0xa3, 0x0d, 0xce, 0x14, 0xe2, 0x54, 0xc3, 0x26, 0xb8, 0x8f }, + { 0x77, 0x75, 0xdf, 0xf8, 0x89, 0x45, 0x8d, 0xd1, 0x1a, 0xef, 0x41, 0x72, 0x76, 0x85, 0x3e, 0x21, 0x33, 0x5e, 0xb8, 0x8e, 0x4d, 0xec, 0x9c, 0xfb, 0x4e, 0x9e, 0xdb, 0x49, 0x82, 0x00, 0x88, 0x55, 0x1a, 0x2c, 0xa6, 0x03, 0x39, 0xf1, 0x20, 0x66, 0x10, 0x11, 0x69, 0xf0, 0xdf, 0xe8, 0x4b, 0x09, 0x8f, 0xdd, 0xb1, 0x48, 0xd9, 0xda, 0x6b, 0x3d, 0x61, 0x3d, 0xf2, 0x63, 0x88, 0x9a, 0xd6, 0x4b }, + { 0xf0, 0xd2, 0x80, 0x5a, 0xfb, 0xb9, 0x1f, 0x74, 0x39, 0x51, 0x35, 0x1a, 0x6d, 0x02, 0x4f, 0x93, 0x53, 0xa2, 0x3c, 0x7c, 0xe1, 0xfc, 0x2b, 0x05, 0x1b, 0x3a, 0x8b, 0x96, 0x8c, 0x23, 0x3f, 0x46, 0xf5, 0x0f, 0x80, 0x6e, 0xcb, 0x15, 0x68, 0xff, 0xaa, 0x0b, 0x60, 0x66, 0x1e, 0x33, 0x4b, 0x21, 0xdd, 0xe0, 0x4f, 0x8f, 0xa1, 0x55, 0xac, 0x74, 0x0e, 0xeb, 0x42, 0xe2, 0x0b, 0x60, 0xd7, 0x64 }, + { 0x86, 0xa2, 0xaf, 0x31, 0x6e, 0x7d, 0x77, 0x54, 0x20, 0x1b, 0x94, 0x2e, 0x27, 0x53, 0x64, 0xac, 0x12, 0xea, 0x89, 0x62, 0xab, 0x5b, 0xd8, 0xd7, 0xfb, 0x27, 0x6d, 0xc5, 0xfb, 0xff, 0xc8, 0xf9, 0xa2, 0x8c, 0xae, 0x4e, 0x48, 0x67, 0xdf, 0x67, 0x80, 0xd9, 0xb7, 0x25, 0x24, 0x16, 0x09, 0x27, 0xc8, 0x55, 0xda, 0x5b, 0x60, 0x78, 0xe0, 0xb5, 0x54, 0xaa, 0x91, 0xe3, 0x1c, 0xb9, 0xca, 0x1d }, + { 0x10, 0xbd, 0xf0, 0xca, 0xa0, 0x80, 0x27, 0x05, 0xe7, 0x06, 0x36, 0x9b, 0xaf, 0x8a, 0x3f, 0x79, 0xd7, 0x2c, 0x0a, 0x03, 0xa8, 0x06, 0x75, 0xa7, 0xbb, 0xb0, 0x0b, 0xe3, 0xa4, 0x5e, 0x51, 0x64, 0x24, 0xd1, 0xee, 0x88, 0xef, 0xb5, 0x6f, 0x6d, 0x57, 0x77, 0x54, 0x5a, 0xe6, 0xe2, 0x77, 0x65, 0xc3, 0xa8, 0xf5, 0xe4, 0x93, 0xfc, 0x30, 0x89, 0x15, 0x63, 0x89, 0x33, 0xa1, 0xdf, 0xee, 0x55 }, + { 0xb0, 0x17, 0x81, 0x09, 0x2b, 0x17, 0x48, 0x45, 0x9e, 0x2e, 0x4e, 0xc1, 0x78, 0x69, 0x66, 0x27, 0xbf, 0x4e, 0xba, 0xfe, 0xbb, 0xa7, 0x74, 0xec, 0xf0, 0x18, 0xb7, 0x9a, 0x68, 0xae, 0xb8, 0x49, 0x17, 0xbf, 0x0b, 0x84, 0xbb, 0x79, 0xd1, 0x7b, 0x74, 0x31, 0x51, 0x14, 0x4c, 0xd6, 0x6b, 0x7b, 0x33, 0xa4, 0xb9, 0xe5, 0x2c, 0x76, 0xc4, 0xe1, 0x12, 0x05, 0x0f, 0xf5, 0x38, 0x5b, 0x7f, 0x0b }, + { 0xc6, 0xdb, 0xc6, 0x1d, 0xec, 0x6e, 0xae, 0xac, 0x81, 0xe3, 0xd5, 0xf7, 0x55, 0x20, 0x3c, 0x8e, 0x22, 0x05, 0x51, 0x53, 0x4a, 0x0b, 0x2f, 0xd1, 0x05, 0xa9, 0x18, 0x89, 0x94, 0x5a, 0x63, 0x85, 0x50, 0x20, 0x4f, 0x44, 0x09, 0x3d, 0xd9, 0x98, 0xc0, 0x76, 0x20, 0x5d, 0xff, 0xad, 0x70, 0x3a, 0x0e, 0x5c, 0xd3, 0xc7, 0xf4, 0x38, 0xa7, 0xe6, 0x34, 0xcd, 0x59, 0xfe, 0xde, 0xdb, 0x53, 0x9e }, + { 0xeb, 0xa5, 0x1a, 0xcf, 0xfb, 0x4c, 0xea, 0x31, 0xdb, 0x4b, 0x8d, 0x87, 0xe9, 0xbf, 0x7d, 0xd4, 0x8f, 0xe9, 0x7b, 0x02, 0x53, 0xae, 0x67, 0xaa, 0x58, 0x0f, 0x9a, 0xc4, 0xa9, 0xd9, 0x41, 0xf2, 0xbe, 0xa5, 0x18, 0xee, 0x28, 0x68, 0x18, 0xcc, 0x9f, 0x63, 0x3f, 0x2a, 0x3b, 0x9f, 0xb6, 0x8e, 0x59, 0x4b, 0x48, 0xcd, 0xd6, 0xd5, 0x15, 0xbf, 0x1d, 0x52, 0xba, 0x6c, 0x85, 0xa2, 0x03, 0xa7 }, + { 0x86, 0x22, 0x1f, 0x3a, 0xda, 0x52, 0x03, 0x7b, 0x72, 0x22, 0x4f, 0x10, 0x5d, 0x79, 0x99, 0x23, 0x1c, 0x5e, 0x55, 0x34, 0xd0, 0x3d, 0xa9, 0xd9, 0xc0, 0xa1, 0x2a, 0xcb, 0x68, 0x46, 0x0c, 0xd3, 0x75, 0xda, 0xf8, 0xe2, 0x43, 0x86, 0x28, 0x6f, 0x96, 0x68, 0xf7, 0x23, 0x26, 0xdb, 0xf9, 0x9b, 0xa0, 0x94, 0x39, 0x24, 0x37, 0xd3, 0x98, 0xe9, 0x5b, 0xb8, 0x16, 0x1d, 0x71, 0x7f, 0x89, 0x91 }, + { 0x55, 0x95, 0xe0, 0x5c, 0x13, 0xa7, 0xec, 0x4d, 0xc8, 0xf4, 0x1f, 0xb7, 0x0c, 0xb5, 0x0a, 0x71, 0xbc, 0xe1, 0x7c, 0x02, 0x4f, 0xf6, 0xde, 0x7a, 0xf6, 0x18, 0xd0, 0xcc, 0x4e, 0x9c, 0x32, 0xd9, 0x57, 0x0d, 0x6d, 0x3e, 0xa4, 0x5b, 0x86, 0x52, 0x54, 0x91, 0x03, 0x0c, 0x0d, 0x8f, 0x2b, 0x18, 0x36, 0xd5, 0x77, 0x8c, 0x1c, 0xe7, 0x35, 0xc1, 0x77, 0x07, 0xdf, 0x36, 0x4d, 0x05, 0x43, 0x47 }, + { 0xce, 0x0f, 0x4f, 0x6a, 0xca, 0x89, 0x59, 0x0a, 0x37, 0xfe, 0x03, 0x4d, 0xd7, 0x4d, 0xd5, 0xfa, 0x65, 0xeb, 0x1c, 0xbd, 0x0a, 0x41, 0x50, 0x8a, 0xad, 0xdc, 0x09, 0x35, 0x1a, 0x3c, 0xea, 0x6d, 0x18, 0xcb, 0x21, 0x89, 0xc5, 0x4b, 0x70, 0x0c, 0x00, 0x9f, 0x4c, 0xbf, 0x05, 0x21, 0xc7, 0xea, 0x01, 0xbe, 0x61, 0xc5, 0xae, 0x09, 0xcb, 0x54, 0xf2, 0x7b, 0xc1, 0xb4, 0x4d, 0x65, 0x8c, 0x82 }, + { 0x7e, 0xe8, 0x0b, 0x06, 0xa2, 0x15, 0xa3, 0xbc, 0xa9, 0x70, 0xc7, 0x7c, 0xda, 0x87, 0x61, 0x82, 0x2b, 0xc1, 0x03, 0xd4, 0x4f, 0xa4, 0xb3, 0x3f, 0x4d, 0x07, 0xdc, 0xb9, 0x97, 0xe3, 0x6d, 0x55, 0x29, 0x8b, 0xce, 0xae, 0x12, 0x24, 0x1b, 0x3f, 0xa0, 0x7f, 0xa6, 0x3b, 0xe5, 0x57, 0x60, 0x68, 0xda, 0x38, 0x7b, 0x8d, 0x58, 0x59, 0xae, 0xab, 0x70, 0x13, 0x69, 0x84, 0x8b, 0x17, 0x6d, 0x42 }, + { 0x94, 0x0a, 0x84, 0xb6, 0xa8, 0x4d, 0x10, 0x9a, 0xab, 0x20, 0x8c, 0x02, 0x4c, 0x6c, 0xe9, 0x64, 0x76, 0x76, 0xba, 0x0a, 0xaa, 0x11, 0xf8, 0x6d, 0xbb, 0x70, 0x18, 0xf9, 0xfd, 0x22, 0x20, 0xa6, 0xd9, 0x01, 0xa9, 0x02, 0x7f, 0x9a, 0xbc, 0xf9, 0x35, 0x37, 0x27, 0x27, 0xcb, 0xf0, 0x9e, 0xbd, 0x61, 0xa2, 0xa2, 0xee, 0xb8, 0x76, 0x53, 0xe8, 0xec, 0xad, 0x1b, 0xab, 0x85, 0xdc, 0x83, 0x27 }, + { 0x20, 0x20, 0xb7, 0x82, 0x64, 0xa8, 0x2d, 0x9f, 0x41, 0x51, 0x14, 0x1a, 0xdb, 0xa8, 0xd4, 0x4b, 0xf2, 0x0c, 0x5e, 0xc0, 0x62, 0xee, 0xe9, 0xb5, 0x95, 0xa1, 0x1f, 0x9e, 0x84, 0x90, 0x1b, 0xf1, 0x48, 0xf2, 0x98, 0xe0, 0xc9, 0xf8, 0x77, 0x7d, 0xcd, 0xbc, 0x7c, 0xc4, 0x67, 0x0a, 0xac, 0x35, 0x6c, 0xc2, 0xad, 0x8c, 0xcb, 0x16, 0x29, 0xf1, 0x6f, 0x6a, 0x76, 0xbc, 0xef, 0xbe, 0xe7, 0x60 }, + { 0xd1, 0xb8, 0x97, 0xb0, 0xe0, 0x75, 0xba, 0x68, 0xab, 0x57, 0x2a, 0xdf, 0x9d, 0x9c, 0x43, 0x66, 0x63, 0xe4, 0x3e, 0xb3, 0xd8, 0xe6, 0x2d, 0x92, 0xfc, 0x49, 0xc9, 0xbe, 0x21, 0x4e, 0x6f, 0x27, 0x87, 0x3f, 0xe2, 0x15, 0xa6, 0x51, 0x70, 0xe6, 0xbe, 0xa9, 0x02, 0x40, 0x8a, 0x25, 0xb4, 0x95, 0x06, 0xf4, 0x7b, 0xab, 0xd0, 0x7c, 0xec, 0xf7, 0x11, 0x3e, 0xc1, 0x0c, 0x5d, 0xd3, 0x12, 0x52 }, + { 0xb1, 0x4d, 0x0c, 0x62, 0xab, 0xfa, 0x46, 0x9a, 0x35, 0x71, 0x77, 0xe5, 0x94, 0xc1, 0x0c, 0x19, 0x42, 0x43, 0xed, 0x20, 0x25, 0xab, 0x8a, 0xa5, 0xad, 0x2f, 0xa4, 0x1a, 0xd3, 0x18, 0xe0, 0xff, 0x48, 0xcd, 0x5e, 0x60, 0xbe, 0xc0, 0x7b, 0x13, 0x63, 0x4a, 0x71, 0x1d, 0x23, 0x26, 0xe4, 0x88, 0xa9, 0x85, 0xf3, 0x1e, 0x31, 0x15, 0x33, 0x99, 0xe7, 0x30, 0x88, 0xef, 0xc8, 0x6a, 0x5c, 0x55 }, + { 0x41, 0x69, 0xc5, 0xcc, 0x80, 0x8d, 0x26, 0x97, 0xdc, 0x2a, 0x82, 0x43, 0x0d, 0xc2, 0x3e, 0x3c, 0xd3, 0x56, 0xdc, 0x70, 0xa9, 0x45, 0x66, 0x81, 0x05, 0x02, 0xb8, 0xd6, 0x55, 0xb3, 0x9a, 0xbf, 0x9e, 0x7f, 0x90, 0x2f, 0xe7, 0x17, 0xe0, 0x38, 0x92, 0x19, 0x85, 0x9e, 0x19, 0x45, 0xdf, 0x1a, 0xf6, 0xad, 0xa4, 0x2e, 0x4c, 0xcd, 0xa5, 0x5a, 0x19, 0x7b, 0x71, 0x00, 0xa3, 0x0c, 0x30, 0xa1 }, + { 0x25, 0x8a, 0x4e, 0xdb, 0x11, 0x3d, 0x66, 0xc8, 0x39, 0xc8, 0xb1, 0xc9, 0x1f, 0x15, 0xf3, 0x5a, 0xde, 0x60, 0x9f, 0x11, 0xcd, 0x7f, 0x86, 0x81, 0xa4, 0x04, 0x5b, 0x9f, 0xef, 0x7b, 0x0b, 0x24, 0xc8, 0x2c, 0xda, 0x06, 0xa5, 0xf2, 0x06, 0x7b, 0x36, 0x88, 0x25, 0xe3, 0x91, 0x4e, 0x53, 0xd6, 0x94, 0x8e, 0xde, 0x92, 0xef, 0xd6, 0xe8, 0x38, 0x7f, 0xa2, 0xe5, 0x37, 0x23, 0x9b, 0x5b, 0xee }, + { 0x79, 0xd2, 0xd8, 0x69, 0x6d, 0x30, 0xf3, 0x0f, 0xb3, 0x46, 0x57, 0x76, 0x11, 0x71, 0xa1, 0x1e, 0x6c, 0x3f, 0x1e, 0x64, 0xcb, 0xe7, 0xbe, 0xbe, 0xe1, 0x59, 0xcb, 0x95, 0xbf, 0xaf, 0x81, 0x2b, 0x4f, 0x41, 0x1e, 0x2f, 0x26, 0xd9, 0xc4, 0x21, 0xdc, 0x2c, 0x28, 0x4a, 0x33, 0x42, 0xd8, 0x23, 0xec, 0x29, 0x38, 0x49, 0xe4, 0x2d, 0x1e, 0x46, 0xb0, 0xa4, 0xac, 0x1e, 0x3c, 0x86, 0xab, 0xaa }, + { 0x8b, 0x94, 0x36, 0x01, 0x0d, 0xc5, 0xde, 0xe9, 0x92, 0xae, 0x38, 0xae, 0xa9, 0x7f, 0x2c, 0xd6, 0x3b, 0x94, 0x6d, 0x94, 0xfe, 0xdd, 0x2e, 0xc9, 0x67, 0x1d, 0xcd, 0xe3, 0xbd, 0x4c, 0xe9, 0x56, 0x4d, 0x55, 0x5c, 0x66, 0xc1, 0x5b, 0xb2, 0xb9, 0x00, 0xdf, 0x72, 0xed, 0xb6, 0xb8, 0x91, 0xeb, 0xca, 0xdf, 0xef, 0xf6, 0x3c, 0x9e, 0xa4, 0x03, 0x6a, 0x99, 0x8b, 0xe7, 0x97, 0x39, 0x81, 0xe7 }, + { 0xc8, 0xf6, 0x8e, 0x69, 0x6e, 0xd2, 0x82, 0x42, 0xbf, 0x99, 0x7f, 0x5b, 0x3b, 0x34, 0x95, 0x95, 0x08, 0xe4, 0x2d, 0x61, 0x38, 0x10, 0xf1, 0xe2, 0xa4, 0x35, 0xc9, 0x6e, 0xd2, 0xff, 0x56, 0x0c, 0x70, 0x22, 0xf3, 0x61, 0xa9, 0x23, 0x4b, 0x98, 0x37, 0xfe, 0xee, 0x90, 0xbf, 0x47, 0x92, 0x2e, 0xe0, 0xfd, 0x5f, 0x8d, 0xdf, 0x82, 0x37, 0x18, 0xd8, 0x6d, 0x1e, 0x16, 0xc6, 0x09, 0x00, 0x71 }, + { 0xb0, 0x2d, 0x3e, 0xee, 0x48, 0x60, 0xd5, 0x86, 0x8b, 0x2c, 0x39, 0xce, 0x39, 0xbf, 0xe8, 0x10, 0x11, 0x29, 0x05, 0x64, 0xdd, 0x67, 0x8c, 0x85, 0xe8, 0x78, 0x3f, 0x29, 0x30, 0x2d, 0xfc, 0x13, 0x99, 0xba, 0x95, 0xb6, 0xb5, 0x3c, 0xd9, 0xeb, 0xbf, 0x40, 0x0c, 0xca, 0x1d, 0xb0, 0xab, 0x67, 0xe1, 0x9a, 0x32, 0x5f, 0x2d, 0x11, 0x58, 0x12, 0xd2, 0x5d, 0x00, 0x97, 0x8a, 0xd1, 0xbc, 0xa4 }, + { 0x76, 0x93, 0xea, 0x73, 0xaf, 0x3a, 0xc4, 0xda, 0xd2, 0x1c, 0xa0, 0xd8, 0xda, 0x85, 0xb3, 0x11, 0x8a, 0x7d, 0x1c, 0x60, 0x24, 0xcf, 0xaf, 0x55, 0x76, 0x99, 0x86, 0x82, 0x17, 0xbc, 0x0c, 0x2f, 0x44, 0xa1, 0x99, 0xbc, 0x6c, 0x0e, 0xdd, 0x51, 0x97, 0x98, 0xba, 0x05, 0xbd, 0x5b, 0x1b, 0x44, 0x84, 0x34, 0x6a, 0x47, 0xc2, 0xca, 0xdf, 0x6b, 0xf3, 0x0b, 0x78, 0x5c, 0xc8, 0x8b, 0x2b, 0xaf }, + { 0xa0, 0xe5, 0xc1, 0xc0, 0x03, 0x1c, 0x02, 0xe4, 0x8b, 0x7f, 0x09, 0xa5, 0xe8, 0x96, 0xee, 0x9a, 0xef, 0x2f, 0x17, 0xfc, 0x9e, 0x18, 0xe9, 0x97, 0xd7, 0xf6, 0xca, 0xc7, 0xae, 0x31, 0x64, 0x22, 0xc2, 0xb1, 0xe7, 0x79, 0x84, 0xe5, 0xf3, 0xa7, 0x3c, 0xb4, 0x5d, 0xee, 0xd5, 0xd3, 0xf8, 0x46, 0x00, 0x10, 0x5e, 0x6e, 0xe3, 0x8f, 0x2d, 0x09, 0x0c, 0x7d, 0x04, 0x42, 0xea, 0x34, 0xc4, 0x6d }, + { 0x41, 0xda, 0xa6, 0xad, 0xcf, 0xdb, 0x69, 0xf1, 0x44, 0x0c, 0x37, 0xb5, 0x96, 0x44, 0x01, 0x65, 0xc1, 0x5a, 0xda, 0x59, 0x68, 0x13, 0xe2, 0xe2, 0x2f, 0x06, 0x0f, 0xcd, 0x55, 0x1f, 0x24, 0xde, 0xe8, 0xe0, 0x4b, 0xa6, 0x89, 0x03, 0x87, 0x88, 0x6c, 0xee, 0xc4, 0xa7, 0xa0, 0xd7, 0xfc, 0x6b, 0x44, 0x50, 0x63, 0x92, 0xec, 0x38, 0x22, 0xc0, 0xd8, 0xc1, 0xac, 0xfc, 0x7d, 0x5a, 0xeb, 0xe8 }, + { 0x14, 0xd4, 0xd4, 0x0d, 0x59, 0x84, 0xd8, 0x4c, 0x5c, 0xf7, 0x52, 0x3b, 0x77, 0x98, 0xb2, 0x54, 0xe2, 0x75, 0xa3, 0xa8, 0xcc, 0x0a, 0x1b, 0xd0, 0x6e, 0xbc, 0x0b, 0xee, 0x72, 0x68, 0x56, 0xac, 0xc3, 0xcb, 0xf5, 0x16, 0xff, 0x66, 0x7c, 0xda, 0x20, 0x58, 0xad, 0x5c, 0x34, 0x12, 0x25, 0x44, 0x60, 0xa8, 0x2c, 0x92, 0x18, 0x70, 0x41, 0x36, 0x3c, 0xc7, 0x7a, 0x4d, 0xc2, 0x15, 0xe4, 0x87 }, + { 0xd0, 0xe7, 0xa1, 0xe2, 0xb9, 0xa4, 0x47, 0xfe, 0xe8, 0x3e, 0x22, 0x77, 0xe9, 0xff, 0x80, 0x10, 0xc2, 0xf3, 0x75, 0xae, 0x12, 0xfa, 0x7a, 0xaa, 0x8c, 0xa5, 0xa6, 0x31, 0x78, 0x68, 0xa2, 0x6a, 0x36, 0x7a, 0x0b, 0x69, 0xfb, 0xc1, 0xcf, 0x32, 0xa5, 0x5d, 0x34, 0xeb, 0x37, 0x06, 0x63, 0x01, 0x6f, 0x3d, 0x21, 0x10, 0x23, 0x0e, 0xba, 0x75, 0x40, 0x28, 0xa5, 0x6f, 0x54, 0xac, 0xf5, 0x7c }, + { 0xe7, 0x71, 0xaa, 0x8d, 0xb5, 0xa3, 0xe0, 0x43, 0xe8, 0x17, 0x8f, 0x39, 0xa0, 0x85, 0x7b, 0xa0, 0x4a, 0x3f, 0x18, 0xe4, 0xaa, 0x05, 0x74, 0x3c, 0xf8, 0xd2, 0x22, 0xb0, 0xb0, 0x95, 0x82, 0x53, 0x50, 0xba, 0x42, 0x2f, 0x63, 0x38, 0x2a, 0x23, 0xd9, 0x2e, 0x41, 0x49, 0x07, 0x4e, 0x81, 0x6a, 0x36, 0xc1, 0xcd, 0x28, 0x28, 0x4d, 0x14, 0x62, 0x67, 0x94, 0x0b, 0x31, 0xf8, 0x81, 0x8e, 0xa2 }, + { 0xfe, 0xb4, 0xfd, 0x6f, 0x9e, 0x87, 0xa5, 0x6b, 0xef, 0x39, 0x8b, 0x32, 0x84, 0xd2, 0xbd, 0xa5, 0xb5, 0xb0, 0xe1, 0x66, 0x58, 0x3a, 0x66, 0xb6, 0x1e, 0x53, 0x84, 0x57, 0xff, 0x05, 0x84, 0x87, 0x2c, 0x21, 0xa3, 0x29, 0x62, 0xb9, 0x92, 0x8f, 0xfa, 0xb5, 0x8d, 0xe4, 0xaf, 0x2e, 0xdd, 0x4e, 0x15, 0xd8, 0xb3, 0x55, 0x70, 0x52, 0x32, 0x07, 0xff, 0x4e, 0x2a, 0x5a, 0xa7, 0x75, 0x4c, 0xaa }, + { 0x46, 0x2f, 0x17, 0xbf, 0x00, 0x5f, 0xb1, 0xc1, 0xb9, 0xe6, 0x71, 0x77, 0x9f, 0x66, 0x52, 0x09, 0xec, 0x28, 0x73, 0xe3, 0xe4, 0x11, 0xf9, 0x8d, 0xab, 0xf2, 0x40, 0xa1, 0xd5, 0xec, 0x3f, 0x95, 0xce, 0x67, 0x96, 0xb6, 0xfc, 0x23, 0xfe, 0x17, 0x19, 0x03, 0xb5, 0x02, 0x02, 0x34, 0x67, 0xde, 0xc7, 0x27, 0x3f, 0xf7, 0x48, 0x79, 0xb9, 0x29, 0x67, 0xa2, 0xa4, 0x3a, 0x5a, 0x18, 0x3d, 0x33 }, + { 0xd3, 0x33, 0x81, 0x93, 0xb6, 0x45, 0x53, 0xdb, 0xd3, 0x8d, 0x14, 0x4b, 0xea, 0x71, 0xc5, 0x91, 0x5b, 0xb1, 0x10, 0xe2, 0xd8, 0x81, 0x80, 0xdb, 0xc5, 0xdb, 0x36, 0x4f, 0xd6, 0x17, 0x1d, 0xf3, 0x17, 0xfc, 0x72, 0x68, 0x83, 0x1b, 0x5a, 0xef, 0x75, 0xe4, 0x34, 0x2b, 0x2f, 0xad, 0x87, 0x97, 0xba, 0x39, 0xed, 0xdc, 0xef, 0x80, 0xe6, 0xec, 0x08, 0x15, 0x93, 0x50, 0xb1, 0xad, 0x69, 0x6d }, + { 0xe1, 0x59, 0x0d, 0x58, 0x5a, 0x3d, 0x39, 0xf7, 0xcb, 0x59, 0x9a, 0xbd, 0x47, 0x90, 0x70, 0x96, 0x64, 0x09, 0xa6, 0x84, 0x6d, 0x43, 0x77, 0xac, 0xf4, 0x47, 0x1d, 0x06, 0x5d, 0x5d, 0xb9, 0x41, 0x29, 0xcc, 0x9b, 0xe9, 0x25, 0x73, 0xb0, 0x5e, 0xd2, 0x26, 0xbe, 0x1e, 0x9b, 0x7c, 0xb0, 0xca, 0xbe, 0x87, 0x91, 0x85, 0x89, 0xf8, 0x0d, 0xad, 0xd4, 0xef, 0x5e, 0xf2, 0x5a, 0x93, 0xd2, 0x8e }, + { 0xf8, 0xf3, 0x72, 0x6a, 0xc5, 0xa2, 0x6c, 0xc8, 0x01, 0x32, 0x49, 0x3a, 0x6f, 0xed, 0xcb, 0x0e, 0x60, 0x76, 0x0c, 0x09, 0xcf, 0xc8, 0x4c, 0xad, 0x17, 0x81, 0x75, 0x98, 0x68, 0x19, 0x66, 0x5e, 0x76, 0x84, 0x2d, 0x7b, 0x9f, 0xed, 0xf7, 0x6d, 0xdd, 0xeb, 0xf5, 0xd3, 0xf5, 0x6f, 0xaa, 0xad, 0x44, 0x77, 0x58, 0x7a, 0xf2, 0x16, 0x06, 0xd3, 0x96, 0xae, 0x57, 0x0d, 0x8e, 0x71, 0x9a, 0xf2 }, + { 0x30, 0x18, 0x60, 0x55, 0xc0, 0x79, 0x49, 0x94, 0x81, 0x83, 0xc8, 0x50, 0xe9, 0xa7, 0x56, 0xcc, 0x09, 0x93, 0x7e, 0x24, 0x7d, 0x9d, 0x92, 0x8e, 0x86, 0x9e, 0x20, 0xba, 0xfc, 0x3c, 0xd9, 0x72, 0x17, 0x19, 0xd3, 0x4e, 0x04, 0xa0, 0x89, 0x9b, 0x92, 0xc7, 0x36, 0x08, 0x45, 0x50, 0x18, 0x68, 0x86, 0xef, 0xba, 0x2e, 0x79, 0x0d, 0x8b, 0xe6, 0xeb, 0xf0, 0x40, 0xb2, 0x09, 0xc4, 0x39, 0xa4 }, + { 0xf3, 0xc4, 0x27, 0x6c, 0xb8, 0x63, 0x63, 0x77, 0x12, 0xc2, 0x41, 0xc4, 0x44, 0xc5, 0xcc, 0x1e, 0x35, 0x54, 0xe0, 0xfd, 0xdb, 0x17, 0x4d, 0x03, 0x58, 0x19, 0xdd, 0x83, 0xeb, 0x70, 0x0b, 0x4c, 0xe8, 0x8d, 0xf3, 0xab, 0x38, 0x41, 0xba, 0x02, 0x08, 0x5e, 0x1a, 0x99, 0xb4, 0xe1, 0x73, 0x10, 0xc5, 0x34, 0x10, 0x75, 0xc0, 0x45, 0x8b, 0xa3, 0x76, 0xc9, 0x5a, 0x68, 0x18, 0xfb, 0xb3, 0xe2 }, + { 0x0a, 0xa0, 0x07, 0xc4, 0xdd, 0x9d, 0x58, 0x32, 0x39, 0x30, 0x40, 0xa1, 0x58, 0x3c, 0x93, 0x0b, 0xca, 0x7d, 0xc5, 0xe7, 0x7e, 0xa5, 0x3a, 0xdd, 0x7e, 0x2b, 0x3f, 0x7c, 0x8e, 0x23, 0x13, 0x68, 0x04, 0x35, 0x20, 0xd4, 0xa3, 0xef, 0x53, 0xc9, 0x69, 0xb6, 0xbb, 0xfd, 0x02, 0x59, 0x46, 0xf6, 0x32, 0xbd, 0x7f, 0x76, 0x5d, 0x53, 0xc2, 0x10, 0x03, 0xb8, 0xf9, 0x83, 0xf7, 0x5e, 0x2a, 0x6a }, + { 0x08, 0xe9, 0x46, 0x47, 0x20, 0x53, 0x3b, 0x23, 0xa0, 0x4e, 0xc2, 0x4f, 0x7a, 0xe8, 0xc1, 0x03, 0x14, 0x5f, 0x76, 0x53, 0x87, 0xd7, 0x38, 0x77, 0x7d, 0x3d, 0x34, 0x34, 0x77, 0xfd, 0x1c, 0x58, 0xdb, 0x05, 0x21, 0x42, 0xca, 0xb7, 0x54, 0xea, 0x67, 0x43, 0x78, 0xe1, 0x87, 0x66, 0xc5, 0x35, 0x42, 0xf7, 0x19, 0x70, 0x17, 0x1c, 0xc4, 0xf8, 0x16, 0x94, 0x24, 0x6b, 0x71, 0x7d, 0x75, 0x64 }, + { 0xd3, 0x7f, 0xf7, 0xad, 0x29, 0x79, 0x93, 0xe7, 0xec, 0x21, 0xe0, 0xf1, 0xb4, 0xb5, 0xae, 0x71, 0x9c, 0xdc, 0x83, 0xc5, 0xdb, 0x68, 0x75, 0x27, 0xf2, 0x75, 0x16, 0xcb, 0xff, 0xa8, 0x22, 0x88, 0x8a, 0x68, 0x10, 0xee, 0x5c, 0x1c, 0xa7, 0xbf, 0xe3, 0x32, 0x11, 0x19, 0xbe, 0x1a, 0xb7, 0xbf, 0xa0, 0xa5, 0x02, 0x67, 0x1c, 0x83, 0x29, 0x49, 0x4d, 0xf7, 0xad, 0x6f, 0x52, 0x2d, 0x44, 0x0f }, + { 0xdd, 0x90, 0x42, 0xf6, 0xe4, 0x64, 0xdc, 0xf8, 0x6b, 0x12, 0x62, 0xf6, 0xac, 0xcf, 0xaf, 0xbd, 0x8c, 0xfd, 0x90, 0x2e, 0xd3, 0xed, 0x89, 0xab, 0xf7, 0x8f, 0xfa, 0x48, 0x2d, 0xbd, 0xee, 0xb6, 0x96, 0x98, 0x42, 0x39, 0x4c, 0x9a, 0x11, 0x68, 0xae, 0x3d, 0x48, 0x1a, 0x01, 0x78, 0x42, 0xf6, 0x60, 0x00, 0x2d, 0x42, 0x44, 0x7c, 0x6b, 0x22, 0xf7, 0xb7, 0x2f, 0x21, 0xaa, 0xe0, 0x21, 0xc9 }, + { 0xbd, 0x96, 0x5b, 0xf3, 0x1e, 0x87, 0xd7, 0x03, 0x27, 0x53, 0x6f, 0x2a, 0x34, 0x1c, 0xeb, 0xc4, 0x76, 0x8e, 0xca, 0x27, 0x5f, 0xa0, 0x5e, 0xf9, 0x8f, 0x7f, 0x1b, 0x71, 0xa0, 0x35, 0x12, 0x98, 0xde, 0x00, 0x6f, 0xba, 0x73, 0xfe, 0x67, 0x33, 0xed, 0x01, 0xd7, 0x58, 0x01, 0xb4, 0xa9, 0x28, 0xe5, 0x42, 0x31, 0xb3, 0x8e, 0x38, 0xc5, 0x62, 0xb2, 0xe3, 0x3e, 0xa1, 0x28, 0x49, 0x92, 0xfa }, + { 0x65, 0x67, 0x6d, 0x80, 0x06, 0x17, 0x97, 0x2f, 0xbd, 0x87, 0xe4, 0xb9, 0x51, 0x4e, 0x1c, 0x67, 0x40, 0x2b, 0x7a, 0x33, 0x10, 0x96, 0xd3, 0xbf, 0xac, 0x22, 0xf1, 0xab, 0xb9, 0x53, 0x74, 0xab, 0xc9, 0x42, 0xf1, 0x6e, 0x9a, 0xb0, 0xea, 0xd3, 0x3b, 0x87, 0xc9, 0x19, 0x68, 0xa6, 0xe5, 0x09, 0xe1, 0x19, 0xff, 0x07, 0x78, 0x7b, 0x3e, 0xf4, 0x83, 0xe1, 0xdc, 0xdc, 0xcf, 0x6e, 0x30, 0x22 }, + { 0x93, 0x9f, 0xa1, 0x89, 0x69, 0x9c, 0x5d, 0x2c, 0x81, 0xdd, 0xd1, 0xff, 0xc1, 0xfa, 0x20, 0x7c, 0x97, 0x0b, 0x6a, 0x36, 0x85, 0xbb, 0x29, 0xce, 0x1d, 0x3e, 0x99, 0xd4, 0x2f, 0x2f, 0x74, 0x42, 0xda, 0x53, 0xe9, 0x5a, 0x72, 0x90, 0x73, 0x14, 0xf4, 0x58, 0x83, 0x99, 0xa3, 0xff, 0x5b, 0x0a, 0x92, 0xbe, 0xb3, 0xf6, 0xbe, 0x26, 0x94, 0xf9, 0xf8, 0x6e, 0xcf, 0x29, 0x52, 0xd5, 0xb4, 0x1c }, + { 0xc5, 0x16, 0x54, 0x17, 0x01, 0x86, 0x3f, 0x91, 0x00, 0x5f, 0x31, 0x41, 0x08, 0xce, 0xec, 0xe3, 0xc6, 0x43, 0xe0, 0x4f, 0xc8, 0xc4, 0x2f, 0xd2, 0xff, 0x55, 0x62, 0x20, 0xe6, 0x16, 0xaa, 0xa6, 0xa4, 0x8a, 0xeb, 0x97, 0xa8, 0x4b, 0xad, 0x74, 0x78, 0x2e, 0x8d, 0xff, 0x96, 0xa1, 0xa2, 0xfa, 0x94, 0x93, 0x39, 0xd7, 0x22, 0xed, 0xca, 0xa3, 0x2b, 0x57, 0x06, 0x70, 0x41, 0xdf, 0x88, 0xcc }, + { 0x98, 0x7f, 0xd6, 0xe0, 0xd6, 0x85, 0x7c, 0x55, 0x3e, 0xae, 0xbb, 0x3d, 0x34, 0x97, 0x0a, 0x2c, 0x2f, 0x6e, 0x89, 0xa3, 0x54, 0x8f, 0x49, 0x25, 0x21, 0x72, 0x2b, 0x80, 0xa1, 0xc2, 0x1a, 0x15, 0x38, 0x92, 0x34, 0x6d, 0x2c, 0xba, 0x64, 0x44, 0x21, 0x2d, 0x56, 0xda, 0x9a, 0x26, 0xe3, 0x24, 0xdc, 0xcb, 0xc0, 0xdc, 0xde, 0x85, 0xd4, 0xd2, 0xee, 0x43, 0x99, 0xee, 0xc5, 0xa6, 0x4e, 0x8f }, + { 0xae, 0x56, 0xde, 0xb1, 0xc2, 0x32, 0x8d, 0x9c, 0x40, 0x17, 0x70, 0x6b, 0xce, 0x6e, 0x99, 0xd4, 0x13, 0x49, 0x05, 0x3b, 0xa9, 0xd3, 0x36, 0xd6, 0x77, 0xc4, 0xc2, 0x7d, 0x9f, 0xd5, 0x0a, 0xe6, 0xae, 0xe1, 0x7e, 0x85, 0x31, 0x54, 0xe1, 0xf4, 0xfe, 0x76, 0x72, 0x34, 0x6d, 0xa2, 0xea, 0xa3, 0x1e, 0xea, 0x53, 0xfc, 0xf2, 0x4a, 0x22, 0x80, 0x4f, 0x11, 0xd0, 0x3d, 0xa6, 0xab, 0xfc, 0x2b }, + { 0x49, 0xd6, 0xa6, 0x08, 0xc9, 0xbd, 0xe4, 0x49, 0x18, 0x70, 0x49, 0x85, 0x72, 0xac, 0x31, 0xaa, 0xc3, 0xfa, 0x40, 0x93, 0x8b, 0x38, 0xa7, 0x81, 0x8f, 0x72, 0x38, 0x3e, 0xb0, 0x40, 0xad, 0x39, 0x53, 0x2b, 0xc0, 0x65, 0x71, 0xe1, 0x3d, 0x76, 0x7e, 0x69, 0x45, 0xab, 0x77, 0xc0, 0xbd, 0xc3, 0xb0, 0x28, 0x42, 0x53, 0x34, 0x3f, 0x9f, 0x6c, 0x12, 0x44, 0xeb, 0xf2, 0xff, 0x0d, 0xf8, 0x66 }, + { 0xda, 0x58, 0x2a, 0xd8, 0xc5, 0x37, 0x0b, 0x44, 0x69, 0xaf, 0x86, 0x2a, 0xa6, 0x46, 0x7a, 0x22, 0x93, 0xb2, 0xb2, 0x8b, 0xd8, 0x0a, 0xe0, 0xe9, 0x1f, 0x42, 0x5a, 0xd3, 0xd4, 0x72, 0x49, 0xfd, 0xf9, 0x88, 0x25, 0xcc, 0x86, 0xf1, 0x40, 0x28, 0xc3, 0x30, 0x8c, 0x98, 0x04, 0xc7, 0x8b, 0xfe, 0xee, 0xee, 0x46, 0x14, 0x44, 0xce, 0x24, 0x36, 0x87, 0xe1, 0xa5, 0x05, 0x22, 0x45, 0x6a, 0x1d }, + { 0xd5, 0x26, 0x6a, 0xa3, 0x33, 0x11, 0x94, 0xae, 0xf8, 0x52, 0xee, 0xd8, 0x6d, 0x7b, 0x5b, 0x26, 0x33, 0xa0, 0xaf, 0x1c, 0x73, 0x59, 0x06, 0xf2, 0xe1, 0x32, 0x79, 0xf1, 0x49, 0x31, 0xa9, 0xfc, 0x3b, 0x0e, 0xac, 0x5c, 0xe9, 0x24, 0x52, 0x73, 0xbd, 0x1a, 0xa9, 0x29, 0x05, 0xab, 0xe1, 0x62, 0x78, 0xef, 0x7e, 0xfd, 0x47, 0x69, 0x47, 0x89, 0xa7, 0x28, 0x3b, 0x77, 0xda, 0x3c, 0x70, 0xf8 }, + { 0x29, 0x62, 0x73, 0x4c, 0x28, 0x25, 0x21, 0x86, 0xa9, 0xa1, 0x11, 0x1c, 0x73, 0x2a, 0xd4, 0xde, 0x45, 0x06, 0xd4, 0xb4, 0x48, 0x09, 0x16, 0x30, 0x3e, 0xb7, 0x99, 0x1d, 0x65, 0x9c, 0xcd, 0xa0, 0x7a, 0x99, 0x11, 0x91, 0x4b, 0xc7, 0x5c, 0x41, 0x8a, 0xb7, 0xa4, 0x54, 0x17, 0x57, 0xad, 0x05, 0x47, 0x96, 0xe2, 0x67, 0x97, 0xfe, 0xaf, 0x36, 0xe9, 0xf6, 0xad, 0x43, 0xf1, 0x4b, 0x35, 0xa4 }, + { 0xe8, 0xb7, 0x9e, 0xc5, 0xd0, 0x6e, 0x11, 0x1b, 0xdf, 0xaf, 0xd7, 0x1e, 0x9f, 0x57, 0x60, 0xf0, 0x0a, 0xc8, 0xac, 0x5d, 0x8b, 0xf7, 0x68, 0xf9, 0xff, 0x6f, 0x08, 0xb8, 0xf0, 0x26, 0x09, 0x6b, 0x1c, 0xc3, 0xa4, 0xc9, 0x73, 0x33, 0x30, 0x19, 0xf1, 0xe3, 0x55, 0x3e, 0x77, 0xda, 0x3f, 0x98, 0xcb, 0x9f, 0x54, 0x2e, 0x0a, 0x90, 0xe5, 0xf8, 0xa9, 0x40, 0xcc, 0x58, 0xe5, 0x98, 0x44, 0xb3 }, + { 0xdf, 0xb3, 0x20, 0xc4, 0x4f, 0x9d, 0x41, 0xd1, 0xef, 0xdc, 0xc0, 0x15, 0xf0, 0x8d, 0xd5, 0x53, 0x9e, 0x52, 0x6e, 0x39, 0xc8, 0x7d, 0x50, 0x9a, 0xe6, 0x81, 0x2a, 0x96, 0x9e, 0x54, 0x31, 0xbf, 0x4f, 0xa7, 0xd9, 0x1f, 0xfd, 0x03, 0xb9, 0x81, 0xe0, 0xd5, 0x44, 0xcf, 0x72, 0xd7, 0xb1, 0xc0, 0x37, 0x4f, 0x88, 0x01, 0x48, 0x2e, 0x6d, 0xea, 0x2e, 0xf9, 0x03, 0x87, 0x7e, 0xba, 0x67, 0x5e }, + { 0xd8, 0x86, 0x75, 0x11, 0x8f, 0xdb, 0x55, 0xa5, 0xfb, 0x36, 0x5a, 0xc2, 0xaf, 0x1d, 0x21, 0x7b, 0xf5, 0x26, 0xce, 0x1e, 0xe9, 0xc9, 0x4b, 0x2f, 0x00, 0x90, 0xb2, 0xc5, 0x8a, 0x06, 0xca, 0x58, 0x18, 0x7d, 0x7f, 0xe5, 0x7c, 0x7b, 0xed, 0x9d, 0x26, 0xfc, 0xa0, 0x67, 0xb4, 0x11, 0x0e, 0xef, 0xcd, 0x9a, 0x0a, 0x34, 0x5d, 0xe8, 0x72, 0xab, 0xe2, 0x0d, 0xe3, 0x68, 0x00, 0x1b, 0x07, 0x45 }, + { 0xb8, 0x93, 0xf2, 0xfc, 0x41, 0xf7, 0xb0, 0xdd, 0x6e, 0x2f, 0x6a, 0xa2, 0xe0, 0x37, 0x0c, 0x0c, 0xff, 0x7d, 0xf0, 0x9e, 0x3a, 0xcf, 0xcc, 0x0e, 0x92, 0x0b, 0x6e, 0x6f, 0xad, 0x0e, 0xf7, 0x47, 0xc4, 0x06, 0x68, 0x41, 0x7d, 0x34, 0x2b, 0x80, 0xd2, 0x35, 0x1e, 0x8c, 0x17, 0x5f, 0x20, 0x89, 0x7a, 0x06, 0x2e, 0x97, 0x65, 0xe6, 0xc6, 0x7b, 0x53, 0x9b, 0x6b, 0xa8, 0xb9, 0x17, 0x05, 0x45 }, + { 0x6c, 0x67, 0xec, 0x56, 0x97, 0xac, 0xcd, 0x23, 0x5c, 0x59, 0xb4, 0x86, 0xd7, 0xb7, 0x0b, 0xae, 0xed, 0xcb, 0xd4, 0xaa, 0x64, 0xeb, 0xd4, 0xee, 0xf3, 0xc7, 0xea, 0xc1, 0x89, 0x56, 0x1a, 0x72, 0x62, 0x50, 0xae, 0xc4, 0xd4, 0x8c, 0xad, 0xca, 0xfb, 0xbe, 0x2c, 0xe3, 0xc1, 0x6c, 0xe2, 0xd6, 0x91, 0xa8, 0xcc, 0xe0, 0x6e, 0x88, 0x79, 0x55, 0x6d, 0x44, 0x83, 0xed, 0x71, 0x65, 0xc0, 0x63 }, + { 0xf1, 0xaa, 0x2b, 0x04, 0x4f, 0x8f, 0x0c, 0x63, 0x8a, 0x3f, 0x36, 0x2e, 0x67, 0x7b, 0x5d, 0x89, 0x1d, 0x6f, 0xd2, 0xab, 0x07, 0x65, 0xf6, 0xee, 0x1e, 0x49, 0x87, 0xde, 0x05, 0x7e, 0xad, 0x35, 0x78, 0x83, 0xd9, 0xb4, 0x05, 0xb9, 0xd6, 0x09, 0xee, 0xa1, 0xb8, 0x69, 0xd9, 0x7f, 0xb1, 0x6d, 0x9b, 0x51, 0x01, 0x7c, 0x55, 0x3f, 0x3b, 0x93, 0xc0, 0xa1, 0xe0, 0xf1, 0x29, 0x6f, 0xed, 0xcd }, + { 0xcb, 0xaa, 0x25, 0x95, 0x72, 0xd4, 0xae, 0xbf, 0xc1, 0x91, 0x7a, 0xcd, 0xdc, 0x58, 0x2b, 0x9f, 0x8d, 0xfa, 0xa9, 0x28, 0xa1, 0x98, 0xca, 0x7a, 0xcd, 0x0f, 0x2a, 0xa7, 0x6a, 0x13, 0x4a, 0x90, 0x25, 0x2e, 0x62, 0x98, 0xa6, 0x5b, 0x08, 0x18, 0x6a, 0x35, 0x0d, 0x5b, 0x76, 0x26, 0x69, 0x9f, 0x8c, 0xb7, 0x21, 0xa3, 0xea, 0x59, 0x21, 0xb7, 0x53, 0xae, 0x3a, 0x2d, 0xce, 0x24, 0xba, 0x3a }, + { 0xfa, 0x15, 0x49, 0xc9, 0x79, 0x6c, 0xd4, 0xd3, 0x03, 0xdc, 0xf4, 0x52, 0xc1, 0xfb, 0xd5, 0x74, 0x4f, 0xd9, 0xb9, 0xb4, 0x70, 0x03, 0xd9, 0x20, 0xb9, 0x2d, 0xe3, 0x48, 0x39, 0xd0, 0x7e, 0xf2, 0xa2, 0x9d, 0xed, 0x68, 0xf6, 0xfc, 0x9e, 0x6c, 0x45, 0xe0, 0x71, 0xa2, 0xe4, 0x8b, 0xd5, 0x0c, 0x50, 0x84, 0xe9, 0x6b, 0x65, 0x7d, 0xd0, 0x40, 0x40, 0x45, 0xa1, 0xdd, 0xef, 0xe2, 0x82, 0xed }, + { 0x5c, 0xf2, 0xac, 0x89, 0x7a, 0xb4, 0x44, 0xdc, 0xb5, 0xc8, 0xd8, 0x7c, 0x49, 0x5d, 0xbd, 0xb3, 0x4e, 0x18, 0x38, 0xb6, 0xb6, 0x29, 0x42, 0x7c, 0xaa, 0x51, 0x70, 0x2a, 0xd0, 0xf9, 0x68, 0x85, 0x25, 0xf1, 0x3b, 0xec, 0x50, 0x3a, 0x3c, 0x3a, 0x2c, 0x80, 0xa6, 0x5e, 0x0b, 0x57, 0x15, 0xe8, 0xaf, 0xab, 0x00, 0xff, 0xa5, 0x6e, 0xc4, 0x55, 0xa4, 0x9a, 0x1a, 0xd3, 0x0a, 0xa2, 0x4f, 0xcd }, + { 0x9a, 0xaf, 0x80, 0x20, 0x7b, 0xac, 0xe1, 0x7b, 0xb7, 0xab, 0x14, 0x57, 0x57, 0xd5, 0x69, 0x6b, 0xde, 0x32, 0x40, 0x6e, 0xf2, 0x2b, 0x44, 0x29, 0x2e, 0xf6, 0x5d, 0x45, 0x19, 0xc3, 0xbb, 0x2a, 0xd4, 0x1a, 0x59, 0xb6, 0x2c, 0xc3, 0xe9, 0x4b, 0x6f, 0xa9, 0x6d, 0x32, 0xa7, 0xfa, 0xad, 0xae, 0x28, 0xaf, 0x7d, 0x35, 0x09, 0x72, 0x19, 0xaa, 0x3f, 0xd8, 0xcd, 0xa3, 0x1e, 0x40, 0xc2, 0x75 }, + { 0xaf, 0x88, 0xb1, 0x63, 0x40, 0x2c, 0x86, 0x74, 0x5c, 0xb6, 0x50, 0xc2, 0x98, 0x8f, 0xb9, 0x52, 0x11, 0xb9, 0x4b, 0x03, 0xef, 0x29, 0x0e, 0xed, 0x96, 0x62, 0x03, 0x42, 0x41, 0xfd, 0x51, 0xcf, 0x39, 0x8f, 0x80, 0x73, 0xe3, 0x69, 0x35, 0x4c, 0x43, 0xea, 0xe1, 0x05, 0x2f, 0x9b, 0x63, 0xb0, 0x81, 0x91, 0xca, 0xa1, 0x38, 0xaa, 0x54, 0xfe, 0xa8, 0x89, 0xcc, 0x70, 0x24, 0x23, 0x68, 0x97 }, + { 0x48, 0xfa, 0x7d, 0x64, 0xe1, 0xce, 0xee, 0x27, 0xb9, 0x86, 0x4d, 0xb5, 0xad, 0xa4, 0xb5, 0x3d, 0x00, 0xc9, 0xbc, 0x76, 0x26, 0x55, 0x58, 0x13, 0xd3, 0xcd, 0x67, 0x30, 0xab, 0x3c, 0xc0, 0x6f, 0xf3, 0x42, 0xd7, 0x27, 0x90, 0x5e, 0x33, 0x17, 0x1b, 0xde, 0x6e, 0x84, 0x76, 0xe7, 0x7f, 0xb1, 0x72, 0x08, 0x61, 0xe9, 0x4b, 0x73, 0xa2, 0xc5, 0x38, 0xd2, 0x54, 0x74, 0x62, 0x85, 0xf4, 0x30 }, + { 0x0e, 0x6f, 0xd9, 0x7a, 0x85, 0xe9, 0x04, 0xf8, 0x7b, 0xfe, 0x85, 0xbb, 0xeb, 0x34, 0xf6, 0x9e, 0x1f, 0x18, 0x10, 0x5c, 0xf4, 0xed, 0x4f, 0x87, 0xae, 0xc3, 0x6c, 0x6e, 0x8b, 0x5f, 0x68, 0xbd, 0x2a, 0x6f, 0x3d, 0xc8, 0xa9, 0xec, 0xb2, 0xb6, 0x1d, 0xb4, 0xee, 0xdb, 0x6b, 0x2e, 0xa1, 0x0b, 0xf9, 0xcb, 0x02, 0x51, 0xfb, 0x0f, 0x8b, 0x34, 0x4a, 0xbf, 0x7f, 0x36, 0x6b, 0x6d, 0xe5, 0xab }, + { 0x06, 0x62, 0x2d, 0xa5, 0x78, 0x71, 0x76, 0x28, 0x7f, 0xdc, 0x8f, 0xed, 0x44, 0x0b, 0xad, 0x18, 0x7d, 0x83, 0x00, 0x99, 0xc9, 0x4e, 0x6d, 0x04, 0xc8, 0xe9, 0xc9, 0x54, 0xcd, 0xa7, 0x0c, 0x8b, 0xb9, 0xe1, 0xfc, 0x4a, 0x6d, 0x0b, 0xaa, 0x83, 0x1b, 0x9b, 0x78, 0xef, 0x66, 0x48, 0x68, 0x1a, 0x48, 0x67, 0xa1, 0x1d, 0xa9, 0x3e, 0xe3, 0x6e, 0x5e, 0x6a, 0x37, 0xd8, 0x7f, 0xc6, 0x3f, 0x6f }, + { 0x1d, 0xa6, 0x77, 0x2b, 0x58, 0xfa, 0xbf, 0x9c, 0x61, 0xf6, 0x8d, 0x41, 0x2c, 0x82, 0xf1, 0x82, 0xc0, 0x23, 0x6d, 0x7d, 0x57, 0x5e, 0xf0, 0xb5, 0x8d, 0xd2, 0x24, 0x58, 0xd6, 0x43, 0xcd, 0x1d, 0xfc, 0x93, 0xb0, 0x38, 0x71, 0xc3, 0x16, 0xd8, 0x43, 0x0d, 0x31, 0x29, 0x95, 0xd4, 0x19, 0x7f, 0x08, 0x74, 0xc9, 0x91, 0x72, 0xba, 0x00, 0x4a, 0x01, 0xee, 0x29, 0x5a, 0xba, 0xc2, 0x4e, 0x46 }, + { 0x3c, 0xd2, 0xd9, 0x32, 0x0b, 0x7b, 0x1d, 0x5f, 0xb9, 0xaa, 0xb9, 0x51, 0xa7, 0x60, 0x23, 0xfa, 0x66, 0x7b, 0xe1, 0x4a, 0x91, 0x24, 0xe3, 0x94, 0x51, 0x39, 0x18, 0xa3, 0xf4, 0x40, 0x96, 0xae, 0x49, 0x04, 0xba, 0x0f, 0xfc, 0x15, 0x0b, 0x63, 0xbc, 0x7a, 0xb1, 0xee, 0xb9, 0xa6, 0xe2, 0x57, 0xe5, 0xc8, 0xf0, 0x00, 0xa7, 0x03, 0x94, 0xa5, 0xaf, 0xd8, 0x42, 0x71, 0x5d, 0xe1, 0x5f, 0x29 }, + { 0x04, 0xcd, 0xc1, 0x4f, 0x74, 0x34, 0xe0, 0xb4, 0xbe, 0x70, 0xcb, 0x41, 0xdb, 0x4c, 0x77, 0x9a, 0x88, 0xea, 0xef, 0x6a, 0xcc, 0xeb, 0xcb, 0x41, 0xf2, 0xd4, 0x2f, 0xff, 0xe7, 0xf3, 0x2a, 0x8e, 0x28, 0x1b, 0x5c, 0x10, 0x3a, 0x27, 0x02, 0x1d, 0x0d, 0x08, 0x36, 0x22, 0x50, 0x75, 0x3c, 0xdf, 0x70, 0x29, 0x21, 0x95, 0xa5, 0x3a, 0x48, 0x72, 0x8c, 0xeb, 0x58, 0x44, 0xc2, 0xd9, 0x8b, 0xab }, + { 0x90, 0x71, 0xb7, 0xa8, 0xa0, 0x75, 0xd0, 0x09, 0x5b, 0x8f, 0xb3, 0xae, 0x51, 0x13, 0x78, 0x57, 0x35, 0xab, 0x98, 0xe2, 0xb5, 0x2f, 0xaf, 0x91, 0xd5, 0xb8, 0x9e, 0x44, 0xaa, 0xc5, 0xb5, 0xd4, 0xeb, 0xbf, 0x91, 0x22, 0x3b, 0x0f, 0xf4, 0xc7, 0x19, 0x05, 0xda, 0x55, 0x34, 0x2e, 0x64, 0x65, 0x5d, 0x6e, 0xf8, 0xc8, 0x9a, 0x47, 0x68, 0xc3, 0xf9, 0x3a, 0x6d, 0xc0, 0x36, 0x6b, 0x5b, 0xc8 }, + { 0xeb, 0xb3, 0x02, 0x40, 0xdd, 0x96, 0xc7, 0xbc, 0x8d, 0x0a, 0xbe, 0x49, 0xaa, 0x4e, 0xdc, 0xbb, 0x4a, 0xfd, 0xc5, 0x1f, 0xf9, 0xaa, 0xf7, 0x20, 0xd3, 0xf9, 0xe7, 0xfb, 0xb0, 0xf9, 0xc6, 0xd6, 0x57, 0x13, 0x50, 0x50, 0x17, 0x69, 0xfc, 0x4e, 0xbd, 0x0b, 0x21, 0x41, 0x24, 0x7f, 0xf4, 0x00, 0xd4, 0xfd, 0x4b, 0xe4, 0x14, 0xed, 0xf3, 0x77, 0x57, 0xbb, 0x90, 0xa3, 0x2a, 0xc5, 0xc6, 0x5a }, + { 0x85, 0x32, 0xc5, 0x8b, 0xf3, 0xc8, 0x01, 0x5d, 0x9d, 0x1c, 0xbe, 0x00, 0xee, 0xf1, 0xf5, 0x08, 0x2f, 0x8f, 0x36, 0x32, 0xfb, 0xe9, 0xf1, 0xed, 0x4f, 0x9d, 0xfb, 0x1f, 0xa7, 0x9e, 0x82, 0x83, 0x06, 0x6d, 0x77, 0xc4, 0x4c, 0x4a, 0xf9, 0x43, 0xd7, 0x6b, 0x30, 0x03, 0x64, 0xae, 0xcb, 0xd0, 0x64, 0x8c, 0x8a, 0x89, 0x39, 0xbd, 0x20, 0x41, 0x23, 0xf4, 0xb5, 0x62, 0x60, 0x42, 0x2d, 0xec }, + { 0xfe, 0x98, 0x46, 0xd6, 0x4f, 0x7c, 0x77, 0x08, 0x69, 0x6f, 0x84, 0x0e, 0x2d, 0x76, 0xcb, 0x44, 0x08, 0xb6, 0x59, 0x5c, 0x2f, 0x81, 0xec, 0x6a, 0x28, 0xa7, 0xf2, 0xf2, 0x0c, 0xb8, 0x8c, 0xfe, 0x6a, 0xc0, 0xb9, 0xe9, 0xb8, 0x24, 0x4f, 0x08, 0xbd, 0x70, 0x95, 0xc3, 0x50, 0xc1, 0xd0, 0x84, 0x2f, 0x64, 0xfb, 0x01, 0xbb, 0x7f, 0x53, 0x2d, 0xfc, 0xd4, 0x73, 0x71, 0xb0, 0xae, 0xeb, 0x79 }, + { 0x28, 0xf1, 0x7e, 0xa6, 0xfb, 0x6c, 0x42, 0x09, 0x2d, 0xc2, 0x64, 0x25, 0x7e, 0x29, 0x74, 0x63, 0x21, 0xfb, 0x5b, 0xda, 0xea, 0x98, 0x73, 0xc2, 0xa7, 0xfa, 0x9d, 0x8f, 0x53, 0x81, 0x8e, 0x89, 0x9e, 0x16, 0x1b, 0xc7, 0x7d, 0xfe, 0x80, 0x90, 0xaf, 0xd8, 0x2b, 0xf2, 0x26, 0x6c, 0x5c, 0x1b, 0xc9, 0x30, 0xa8, 0xd1, 0x54, 0x76, 0x24, 0x43, 0x9e, 0x66, 0x2e, 0xf6, 0x95, 0xf2, 0x6f, 0x24 }, + { 0xec, 0x6b, 0x7d, 0x7f, 0x03, 0x0d, 0x48, 0x50, 0xac, 0xae, 0x3c, 0xb6, 0x15, 0xc2, 0x1d, 0xd2, 0x52, 0x06, 0xd6, 0x3e, 0x84, 0xd1, 0xdb, 0x8d, 0x95, 0x73, 0x70, 0x73, 0x7b, 0xa0, 0xe9, 0x84, 0x67, 0xea, 0x0c, 0xe2, 0x74, 0xc6, 0x61, 0x99, 0x90, 0x1e, 0xae, 0xc1, 0x8a, 0x08, 0x52, 0x57, 0x15, 0xf5, 0x3b, 0xfd, 0xb0, 0xaa, 0xcb, 0x61, 0x3d, 0x34, 0x2e, 0xbd, 0xce, 0xed, 0xdc, 0x3b }, + { 0xb4, 0x03, 0xd3, 0x69, 0x1c, 0x03, 0xb0, 0xd3, 0x41, 0x8d, 0xf3, 0x27, 0xd5, 0x86, 0x0d, 0x34, 0xbb, 0xfc, 0xc4, 0x51, 0x9b, 0xfb, 0xce, 0x36, 0xbf, 0x33, 0xb2, 0x08, 0x38, 0x5f, 0xad, 0xb9, 0x18, 0x6b, 0xc7, 0x8a, 0x76, 0xc4, 0x89, 0xd8, 0x9f, 0xd5, 0x7e, 0x7d, 0xc7, 0x54, 0x12, 0xd2, 0x3b, 0xcd, 0x1d, 0xae, 0x84, 0x70, 0xce, 0x92, 0x74, 0x75, 0x4b, 0xb8, 0x58, 0x5b, 0x13, 0xc5 }, + { 0x31, 0xfc, 0x79, 0x73, 0x8b, 0x87, 0x72, 0xb3, 0xf5, 0x5c, 0xd8, 0x17, 0x88, 0x13, 0xb3, 0xb5, 0x2d, 0x0d, 0xb5, 0xa4, 0x19, 0xd3, 0x0b, 0xa9, 0x49, 0x5c, 0x4b, 0x9d, 0xa0, 0x21, 0x9f, 0xac, 0x6d, 0xf8, 0xe7, 0xc2, 0x3a, 0x81, 0x15, 0x51, 0xa6, 0x2b, 0x82, 0x7f, 0x25, 0x6e, 0xcd, 0xb8, 0x12, 0x4a, 0xc8, 0xa6, 0x79, 0x2c, 0xcf, 0xec, 0xc3, 0xb3, 0x01, 0x27, 0x22, 0xe9, 0x44, 0x63 }, + { 0xbb, 0x20, 0x39, 0xec, 0x28, 0x70, 0x91, 0xbc, 0xc9, 0x64, 0x2f, 0xc9, 0x00, 0x49, 0xe7, 0x37, 0x32, 0xe0, 0x2e, 0x57, 0x7e, 0x28, 0x62, 0xb3, 0x22, 0x16, 0xae, 0x9b, 0xed, 0xcd, 0x73, 0x0c, 0x4c, 0x28, 0x4e, 0xf3, 0x96, 0x8c, 0x36, 0x8b, 0x7d, 0x37, 0x58, 0x4f, 0x97, 0xbd, 0x4b, 0x4d, 0xc6, 0xef, 0x61, 0x27, 0xac, 0xfe, 0x2e, 0x6a, 0xe2, 0x50, 0x91, 0x24, 0xe6, 0x6c, 0x8a, 0xf4 }, + { 0xf5, 0x3d, 0x68, 0xd1, 0x3f, 0x45, 0xed, 0xfc, 0xb9, 0xbd, 0x41, 0x5e, 0x28, 0x31, 0xe9, 0x38, 0x35, 0x0d, 0x53, 0x80, 0xd3, 0x43, 0x22, 0x78, 0xfc, 0x1c, 0x0c, 0x38, 0x1f, 0xcb, 0x7c, 0x65, 0xc8, 0x2d, 0xaf, 0xe0, 0x51, 0xd8, 0xc8, 0xb0, 0xd4, 0x4e, 0x09, 0x74, 0xa0, 0xe5, 0x9e, 0xc7, 0xbf, 0x7e, 0xd0, 0x45, 0x9f, 0x86, 0xe9, 0x6f, 0x32, 0x9f, 0xc7, 0x97, 0x52, 0x51, 0x0f, 0xd3 }, + { 0x8d, 0x56, 0x8c, 0x79, 0x84, 0xf0, 0xec, 0xdf, 0x76, 0x40, 0xfb, 0xc4, 0x83, 0xb5, 0xd8, 0xc9, 0xf8, 0x66, 0x34, 0xf6, 0xf4, 0x32, 0x91, 0x84, 0x1b, 0x30, 0x9a, 0x35, 0x0a, 0xb9, 0xc1, 0x13, 0x7d, 0x24, 0x06, 0x6b, 0x09, 0xda, 0x99, 0x44, 0xba, 0xc5, 0x4d, 0x5b, 0xb6, 0x58, 0x0d, 0x83, 0x60, 0x47, 0xaa, 0xc7, 0x4a, 0xb7, 0x24, 0xb8, 0x87, 0xeb, 0xf9, 0x3d, 0x4b, 0x32, 0xec, 0xa9 }, + { 0xc0, 0xb6, 0x5c, 0xe5, 0xa9, 0x6f, 0xf7, 0x74, 0xc4, 0x56, 0xca, 0xc3, 0xb5, 0xf2, 0xc4, 0xcd, 0x35, 0x9b, 0x4f, 0xf5, 0x3e, 0xf9, 0x3a, 0x3d, 0xa0, 0x77, 0x8b, 0xe4, 0x90, 0x0d, 0x1e, 0x8d, 0xa1, 0x60, 0x1e, 0x76, 0x9e, 0x8f, 0x1b, 0x02, 0xd2, 0xa2, 0xf8, 0xc5, 0xb9, 0xfa, 0x10, 0xb4, 0x4f, 0x1c, 0x18, 0x69, 0x85, 0x46, 0x8f, 0xee, 0xb0, 0x08, 0x73, 0x02, 0x83, 0xa6, 0x65, 0x7d }, + { 0x49, 0x00, 0xbb, 0xa6, 0xf5, 0xfb, 0x10, 0x3e, 0xce, 0x8e, 0xc9, 0x6a, 0xda, 0x13, 0xa5, 0xc3, 0xc8, 0x54, 0x88, 0xe0, 0x55, 0x51, 0xda, 0x6b, 0x6b, 0x33, 0xd9, 0x88, 0xe6, 0x11, 0xec, 0x0f, 0xe2, 0xe3, 0xc2, 0xaa, 0x48, 0xea, 0x6a, 0xe8, 0x98, 0x6a, 0x3a, 0x23, 0x1b, 0x22, 0x3c, 0x5d, 0x27, 0xce, 0xc2, 0xea, 0xdd, 0xe9, 0x1c, 0xe0, 0x79, 0x81, 0xee, 0x65, 0x28, 0x62, 0xd1, 0xe4 }, + { 0xc7, 0xf5, 0xc3, 0x7c, 0x72, 0x85, 0xf9, 0x27, 0xf7, 0x64, 0x43, 0x41, 0x4d, 0x43, 0x57, 0xff, 0x78, 0x96, 0x47, 0xd7, 0xa0, 0x05, 0xa5, 0xa7, 0x87, 0xe0, 0x3c, 0x34, 0x6b, 0x57, 0xf4, 0x9f, 0x21, 0xb6, 0x4f, 0xa9, 0xcf, 0x4b, 0x7e, 0x45, 0x57, 0x3e, 0x23, 0x04, 0x90, 0x17, 0x56, 0x71, 0x21, 0xa9, 0xc3, 0xd4, 0xb2, 0xb7, 0x3e, 0xc5, 0xe9, 0x41, 0x35, 0x77, 0x52, 0x5d, 0xb4, 0x5a }, + { 0xec, 0x70, 0x96, 0x33, 0x07, 0x36, 0xfd, 0xb2, 0xd6, 0x4b, 0x56, 0x53, 0xe7, 0x47, 0x5d, 0xa7, 0x46, 0xc2, 0x3a, 0x46, 0x13, 0xa8, 0x26, 0x87, 0xa2, 0x80, 0x62, 0xd3, 0x23, 0x63, 0x64, 0x28, 0x4a, 0xc0, 0x17, 0x20, 0xff, 0xb4, 0x06, 0xcf, 0xe2, 0x65, 0xc0, 0xdf, 0x62, 0x6a, 0x18, 0x8c, 0x9e, 0x59, 0x63, 0xac, 0xe5, 0xd3, 0xd5, 0xbb, 0x36, 0x3e, 0x32, 0xc3, 0x8c, 0x21, 0x90, 0xa6 }, + { 0x82, 0xe7, 0x44, 0xc7, 0x5f, 0x46, 0x49, 0xec, 0x52, 0xb8, 0x07, 0x71, 0xa7, 0x7d, 0x47, 0x5a, 0x3b, 0xc0, 0x91, 0x98, 0x95, 0x56, 0x96, 0x0e, 0x27, 0x6a, 0x5f, 0x9e, 0xad, 0x92, 0xa0, 0x3f, 0x71, 0x87, 0x42, 0xcd, 0xcf, 0xea, 0xee, 0x5c, 0xb8, 0x5c, 0x44, 0xaf, 0x19, 0x8a, 0xdc, 0x43, 0xa4, 0xa4, 0x28, 0xf5, 0xf0, 0xc2, 0xdd, 0xb0, 0xbe, 0x36, 0x05, 0x9f, 0x06, 0xd7, 0xdf, 0x73 }, + { 0x28, 0x34, 0xb7, 0xa7, 0x17, 0x0f, 0x1f, 0x5b, 0x68, 0x55, 0x9a, 0xb7, 0x8c, 0x10, 0x50, 0xec, 0x21, 0xc9, 0x19, 0x74, 0x0b, 0x78, 0x4a, 0x90, 0x72, 0xf6, 0xe5, 0xd6, 0x9f, 0x82, 0x8d, 0x70, 0xc9, 0x19, 0xc5, 0x03, 0x9f, 0xb1, 0x48, 0xe3, 0x9e, 0x2c, 0x8a, 0x52, 0x11, 0x83, 0x78, 0xb0, 0x64, 0xca, 0x8d, 0x50, 0x01, 0xcd, 0x10, 0xa5, 0x47, 0x83, 0x87, 0xb9, 0x66, 0x71, 0x5e, 0xd6 }, + { 0x16, 0xb4, 0xad, 0xa8, 0x83, 0xf7, 0x2f, 0x85, 0x3b, 0xb7, 0xef, 0x25, 0x3e, 0xfc, 0xab, 0x0c, 0x3e, 0x21, 0x61, 0x68, 0x7a, 0xd6, 0x15, 0x43, 0xa0, 0xd2, 0x82, 0x4f, 0x91, 0xc1, 0xf8, 0x13, 0x47, 0xd8, 0x6b, 0xe7, 0x09, 0xb1, 0x69, 0x96, 0xe1, 0x7f, 0x2d, 0xd4, 0x86, 0x92, 0x7b, 0x02, 0x88, 0xad, 0x38, 0xd1, 0x30, 0x63, 0xc4, 0xa9, 0x67, 0x2c, 0x39, 0x39, 0x7d, 0x37, 0x89, 0xb6 }, + { 0x78, 0xd0, 0x48, 0xf3, 0xa6, 0x9d, 0x8b, 0x54, 0xae, 0x0e, 0xd6, 0x3a, 0x57, 0x3a, 0xe3, 0x50, 0xd8, 0x9f, 0x7c, 0x6c, 0xf1, 0xf3, 0x68, 0x89, 0x30, 0xde, 0x89, 0x9a, 0xfa, 0x03, 0x76, 0x97, 0x62, 0x9b, 0x31, 0x4e, 0x5c, 0xd3, 0x03, 0xaa, 0x62, 0xfe, 0xea, 0x72, 0xa2, 0x5b, 0xf4, 0x2b, 0x30, 0x4b, 0x6c, 0x6b, 0xcb, 0x27, 0xfa, 0xe2, 0x1c, 0x16, 0xd9, 0x25, 0xe1, 0xfb, 0xda, 0xc3 }, + { 0x0f, 0x74, 0x6a, 0x48, 0x74, 0x92, 0x87, 0xad, 0xa7, 0x7a, 0x82, 0x96, 0x1f, 0x05, 0xa4, 0xda, 0x4a, 0xbd, 0xb7, 0xd7, 0x7b, 0x12, 0x20, 0xf8, 0x36, 0xd0, 0x9e, 0xc8, 0x14, 0x35, 0x9c, 0x0e, 0xc0, 0x23, 0x9b, 0x8c, 0x7b, 0x9f, 0xf9, 0xe0, 0x2f, 0x56, 0x9d, 0x1b, 0x30, 0x1e, 0xf6, 0x7c, 0x46, 0x12, 0xd1, 0xde, 0x4f, 0x73, 0x0f, 0x81, 0xc1, 0x2c, 0x40, 0xcc, 0x06, 0x3c, 0x5c, 0xaa }, + { 0xf0, 0xfc, 0x85, 0x9d, 0x3b, 0xd1, 0x95, 0xfb, 0xdc, 0x2d, 0x59, 0x1e, 0x4c, 0xda, 0xc1, 0x51, 0x79, 0xec, 0x0f, 0x1d, 0xc8, 0x21, 0xc1, 0x1d, 0xf1, 0xf0, 0xc1, 0xd2, 0x6e, 0x62, 0x60, 0xaa, 0xa6, 0x5b, 0x79, 0xfa, 0xfa, 0xca, 0xfd, 0x7d, 0x3a, 0xd6, 0x1e, 0x60, 0x0f, 0x25, 0x09, 0x05, 0xf5, 0x87, 0x8c, 0x87, 0x45, 0x28, 0x97, 0x64, 0x7a, 0x35, 0xb9, 0x95, 0xbc, 0xad, 0xc3, 0xa3 }, + { 0x26, 0x20, 0xf6, 0x87, 0xe8, 0x62, 0x5f, 0x6a, 0x41, 0x24, 0x60, 0xb4, 0x2e, 0x2c, 0xef, 0x67, 0x63, 0x42, 0x08, 0xce, 0x10, 0xa0, 0xcb, 0xd4, 0xdf, 0xf7, 0x04, 0x4a, 0x41, 0xb7, 0x88, 0x00, 0x77, 0xe9, 0xf8, 0xdc, 0x3b, 0x8d, 0x12, 0x16, 0xd3, 0x37, 0x6a, 0x21, 0xe0, 0x15, 0xb5, 0x8f, 0xb2, 0x79, 0xb5, 0x21, 0xd8, 0x3f, 0x93, 0x88, 0xc7, 0x38, 0x2c, 0x85, 0x05, 0x59, 0x0b, 0x9b }, + { 0x22, 0x7e, 0x3a, 0xed, 0x8d, 0x2c, 0xb1, 0x0b, 0x91, 0x8f, 0xcb, 0x04, 0xf9, 0xde, 0x3e, 0x6d, 0x0a, 0x57, 0xe0, 0x84, 0x76, 0xd9, 0x37, 0x59, 0xcd, 0x7b, 0x2e, 0xd5, 0x4a, 0x1c, 0xbf, 0x02, 0x39, 0xc5, 0x28, 0xfb, 0x04, 0xbb, 0xf2, 0x88, 0x25, 0x3e, 0x60, 0x1d, 0x3b, 0xc3, 0x8b, 0x21, 0x79, 0x4a, 0xfe, 0xf9, 0x0b, 0x17, 0x09, 0x4a, 0x18, 0x2c, 0xac, 0x55, 0x77, 0x45, 0xe7, 0x5f }, + { 0x1a, 0x92, 0x99, 0x01, 0xb0, 0x9c, 0x25, 0xf2, 0x7d, 0x6b, 0x35, 0xbe, 0x7b, 0x2f, 0x1c, 0x47, 0x45, 0x13, 0x1f, 0xde, 0xbc, 0xa7, 0xf3, 0xe2, 0x45, 0x19, 0x26, 0x72, 0x04, 0x34, 0xe0, 0xdb, 0x6e, 0x74, 0xfd, 0x69, 0x3a, 0xd2, 0x9b, 0x77, 0x7d, 0xc3, 0x35, 0x5c, 0x59, 0x2a, 0x36, 0x1c, 0x48, 0x73, 0xb0, 0x11, 0x33, 0xa5, 0x7c, 0x2e, 0x3b, 0x70, 0x75, 0xcb, 0xdb, 0x86, 0xf4, 0xfc }, + { 0x5f, 0xd7, 0x96, 0x8b, 0xc2, 0xfe, 0x34, 0xf2, 0x20, 0xb5, 0xe3, 0xdc, 0x5a, 0xf9, 0x57, 0x17, 0x42, 0xd7, 0x3b, 0x7d, 0x60, 0x81, 0x9f, 0x28, 0x88, 0xb6, 0x29, 0x07, 0x2b, 0x96, 0xa9, 0xd8, 0xab, 0x2d, 0x91, 0xb8, 0x2d, 0x0a, 0x9a, 0xab, 0xa6, 0x1b, 0xbd, 0x39, 0x95, 0x81, 0x32, 0xfc, 0xc4, 0x25, 0x70, 0x23, 0xd1, 0xec, 0xa5, 0x91, 0xb3, 0x05, 0x4e, 0x2d, 0xc8, 0x1c, 0x82, 0x00 }, + { 0xdf, 0xcc, 0xe8, 0xcf, 0x32, 0x87, 0x0c, 0xc6, 0xa5, 0x03, 0xea, 0xda, 0xfc, 0x87, 0xfd, 0x6f, 0x78, 0x91, 0x8b, 0x9b, 0x4d, 0x07, 0x37, 0xdb, 0x68, 0x10, 0xbe, 0x99, 0x6b, 0x54, 0x97, 0xe7, 0xe5, 0xcc, 0x80, 0xe3, 0x12, 0xf6, 0x1e, 0x71, 0xff, 0x3e, 0x96, 0x24, 0x43, 0x60, 0x73, 0x15, 0x64, 0x03, 0xf7, 0x35, 0xf5, 0x6b, 0x0b, 0x01, 0x84, 0x5c, 0x18, 0xf6, 0xca, 0xf7, 0x72, 0xe6 }, + { 0x02, 0xf7, 0xef, 0x3a, 0x9c, 0xe0, 0xff, 0xf9, 0x60, 0xf6, 0x70, 0x32, 0xb2, 0x96, 0xef, 0xca, 0x30, 0x61, 0xf4, 0x93, 0x4d, 0x69, 0x07, 0x49, 0xf2, 0xd0, 0x1c, 0x35, 0xc8, 0x1c, 0x14, 0xf3, 0x9a, 0x67, 0xfa, 0x35, 0x0b, 0xc8, 0xa0, 0x35, 0x9b, 0xf1, 0x72, 0x4b, 0xff, 0xc3, 0xbc, 0xa6, 0xd7, 0xc7, 0xbb, 0xa4, 0x79, 0x1f, 0xd5, 0x22, 0xa3, 0xad, 0x35, 0x3c, 0x02, 0xec, 0x5a, 0xa8 }, + { 0x64, 0xbe, 0x5c, 0x6a, 0xba, 0x65, 0xd5, 0x94, 0x84, 0x4a, 0xe7, 0x8b, 0xb0, 0x22, 0xe5, 0xbe, 0xbe, 0x12, 0x7f, 0xd6, 0xb6, 0xff, 0xa5, 0xa1, 0x37, 0x03, 0x85, 0x5a, 0xb6, 0x3b, 0x62, 0x4d, 0xcd, 0x1a, 0x36, 0x3f, 0x99, 0x20, 0x3f, 0x63, 0x2e, 0xc3, 0x86, 0xf3, 0xea, 0x76, 0x7f, 0xc9, 0x92, 0xe8, 0xed, 0x96, 0x86, 0x58, 0x6a, 0xa2, 0x75, 0x55, 0xa8, 0x59, 0x9d, 0x5b, 0x80, 0x8f }, + { 0xf7, 0x85, 0x85, 0x50, 0x5c, 0x4e, 0xaa, 0x54, 0xa8, 0xb5, 0xbe, 0x70, 0xa6, 0x1e, 0x73, 0x5e, 0x0f, 0xf9, 0x7a, 0xf9, 0x44, 0xdd, 0xb3, 0x00, 0x1e, 0x35, 0xd8, 0x6c, 0x4e, 0x21, 0x99, 0xd9, 0x76, 0x10, 0x4b, 0x6a, 0xe3, 0x17, 0x50, 0xa3, 0x6a, 0x72, 0x6e, 0xd2, 0x85, 0x06, 0x4f, 0x59, 0x81, 0xb5, 0x03, 0x88, 0x9f, 0xef, 0x82, 0x2f, 0xcd, 0xc2, 0x89, 0x8d, 0xdd, 0xb7, 0x88, 0x9a }, + { 0xe4, 0xb5, 0x56, 0x60, 0x33, 0x86, 0x95, 0x72, 0xed, 0xfd, 0x87, 0x47, 0x9a, 0x5b, 0xb7, 0x3c, 0x80, 0xe8, 0x75, 0x9b, 0x91, 0x23, 0x28, 0x79, 0xd9, 0x6b, 0x1d, 0xda, 0x36, 0xc0, 0x12, 0x07, 0x6e, 0xe5, 0xa2, 0xed, 0x7a, 0xe2, 0xde, 0x63, 0xef, 0x84, 0x06, 0xa0, 0x6a, 0xea, 0x82, 0xc1, 0x88, 0x03, 0x1b, 0x56, 0x0b, 0xea, 0xfb, 0x58, 0x3f, 0xb3, 0xde, 0x9e, 0x57, 0x95, 0x2a, 0x7e }, + { 0xe1, 0xb3, 0xe7, 0xed, 0x86, 0x7f, 0x6c, 0x94, 0x84, 0xa2, 0xa9, 0x7f, 0x77, 0x15, 0xf2, 0x5e, 0x25, 0x29, 0x4e, 0x99, 0x2e, 0x41, 0xf6, 0xa7, 0xc1, 0x61, 0xff, 0xc2, 0xad, 0xc6, 0xda, 0xae, 0xb7, 0x11, 0x31, 0x02, 0xd5, 0xe6, 0x09, 0x02, 0x87, 0xfe, 0x6a, 0xd9, 0x4c, 0xe5, 0xd6, 0xb7, 0x39, 0xc6, 0xca, 0x24, 0x0b, 0x05, 0xc7, 0x6f, 0xb7, 0x3f, 0x25, 0xdd, 0x02, 0x4b, 0xf9, 0x35 }, + { 0x85, 0xfd, 0x08, 0x5f, 0xdc, 0x12, 0xa0, 0x80, 0x98, 0x3d, 0xf0, 0x7b, 0xd7, 0x01, 0x2b, 0x0d, 0x40, 0x2a, 0x0f, 0x40, 0x43, 0xfc, 0xb2, 0x77, 0x5a, 0xdf, 0x0b, 0xad, 0x17, 0x4f, 0x9b, 0x08, 0xd1, 0x67, 0x6e, 0x47, 0x69, 0x85, 0x78, 0x5c, 0x0a, 0x5d, 0xcc, 0x41, 0xdb, 0xff, 0x6d, 0x95, 0xef, 0x4d, 0x66, 0xa3, 0xfb, 0xdc, 0x4a, 0x74, 0xb8, 0x2b, 0xa5, 0x2d, 0xa0, 0x51, 0x2b, 0x74 }, + { 0xae, 0xd8, 0xfa, 0x76, 0x4b, 0x0f, 0xbf, 0xf8, 0x21, 0xe0, 0x52, 0x33, 0xd2, 0xf7, 0xb0, 0x90, 0x0e, 0xc4, 0x4d, 0x82, 0x6f, 0x95, 0xe9, 0x3c, 0x34, 0x3c, 0x1b, 0xc3, 0xba, 0x5a, 0x24, 0x37, 0x4b, 0x1d, 0x61, 0x6e, 0x7e, 0x7a, 0xba, 0x45, 0x3a, 0x0a, 0xda, 0x5e, 0x4f, 0xab, 0x53, 0x82, 0x40, 0x9e, 0x0d, 0x42, 0xce, 0x9c, 0x2b, 0xc7, 0xfb, 0x39, 0xa9, 0x9c, 0x34, 0x0c, 0x20, 0xf0 }, + { 0x7b, 0xa3, 0xb2, 0xe2, 0x97, 0x23, 0x35, 0x22, 0xee, 0xb3, 0x43, 0xbd, 0x3e, 0xbc, 0xfd, 0x83, 0x5a, 0x04, 0x00, 0x77, 0x35, 0xe8, 0x7f, 0x0c, 0xa3, 0x00, 0xcb, 0xee, 0x6d, 0x41, 0x65, 0x65, 0x16, 0x21, 0x71, 0x58, 0x1e, 0x40, 0x20, 0xff, 0x4c, 0xf1, 0x76, 0x45, 0x0f, 0x12, 0x91, 0xea, 0x22, 0x85, 0xcb, 0x9e, 0xbf, 0xfe, 0x4c, 0x56, 0x66, 0x06, 0x27, 0x68, 0x51, 0x45, 0x05, 0x1c }, + { 0xde, 0x74, 0x8b, 0xcf, 0x89, 0xec, 0x88, 0x08, 0x47, 0x21, 0xe1, 0x6b, 0x85, 0xf3, 0x0a, 0xdb, 0x1a, 0x61, 0x34, 0xd6, 0x64, 0xb5, 0x84, 0x35, 0x69, 0xba, 0xbc, 0x5b, 0xbd, 0x1a, 0x15, 0xca, 0x9b, 0x61, 0x80, 0x3c, 0x90, 0x1a, 0x4f, 0xef, 0x32, 0x96, 0x5a, 0x17, 0x49, 0xc9, 0xf3, 0xa4, 0xe2, 0x43, 0xe1, 0x73, 0x93, 0x9d, 0xc5, 0xa8, 0xdc, 0x49, 0x5c, 0x67, 0x1a, 0xb5, 0x21, 0x45 }, + { 0xaa, 0xf4, 0xd2, 0xbd, 0xf2, 0x00, 0xa9, 0x19, 0x70, 0x6d, 0x98, 0x42, 0xdc, 0xe1, 0x6c, 0x98, 0x14, 0x0d, 0x34, 0xbc, 0x43, 0x3d, 0xf3, 0x20, 0xab, 0xa9, 0xbd, 0x42, 0x9e, 0x54, 0x9a, 0xa7, 0xa3, 0x39, 0x76, 0x52, 0xa4, 0xd7, 0x68, 0x27, 0x77, 0x86, 0xcf, 0x99, 0x3c, 0xde, 0x23, 0x38, 0x67, 0x3e, 0xd2, 0xe6, 0xb6, 0x6c, 0x96, 0x1f, 0xef, 0xb8, 0x2c, 0xd2, 0x0c, 0x93, 0x33, 0x8f }, + { 0xc4, 0x08, 0x21, 0x89, 0x68, 0xb7, 0x88, 0xbf, 0x86, 0x4f, 0x09, 0x97, 0xe6, 0xbc, 0x4c, 0x3d, 0xba, 0x68, 0xb2, 0x76, 0xe2, 0x12, 0x5a, 0x48, 0x43, 0x29, 0x60, 0x52, 0xff, 0x93, 0xbf, 0x57, 0x67, 0xb8, 0xcd, 0xce, 0x71, 0x31, 0xf0, 0x87, 0x64, 0x30, 0xc1, 0x16, 0x5f, 0xec, 0x6c, 0x4f, 0x47, 0xad, 0xaa, 0x4f, 0xd8, 0xbc, 0xfa, 0xce, 0xf4, 0x63, 0xb5, 0xd3, 0xd0, 0xfa, 0x61, 0xa0 }, + { 0x76, 0xd2, 0xd8, 0x19, 0xc9, 0x2b, 0xce, 0x55, 0xfa, 0x8e, 0x09, 0x2a, 0xb1, 0xbf, 0x9b, 0x9e, 0xab, 0x23, 0x7a, 0x25, 0x26, 0x79, 0x86, 0xca, 0xcf, 0x2b, 0x8e, 0xe1, 0x4d, 0x21, 0x4d, 0x73, 0x0d, 0xc9, 0xa5, 0xaa, 0x2d, 0x7b, 0x59, 0x6e, 0x86, 0xa1, 0xfd, 0x8f, 0xa0, 0x80, 0x4c, 0x77, 0x40, 0x2d, 0x2f, 0xcd, 0x45, 0x08, 0x36, 0x88, 0xb2, 0x18, 0xb1, 0xcd, 0xfa, 0x0d, 0xcb, 0xcb }, + { 0x72, 0x06, 0x5e, 0xe4, 0xdd, 0x91, 0xc2, 0xd8, 0x50, 0x9f, 0xa1, 0xfc, 0x28, 0xa3, 0x7c, 0x7f, 0xc9, 0xfa, 0x7d, 0x5b, 0x3f, 0x8a, 0xd3, 0xd0, 0xd7, 0xa2, 0x56, 0x26, 0xb5, 0x7b, 0x1b, 0x44, 0x78, 0x8d, 0x4c, 0xaf, 0x80, 0x62, 0x90, 0x42, 0x5f, 0x98, 0x90, 0xa3, 0xa2, 0xa3, 0x5a, 0x90, 0x5a, 0xb4, 0xb3, 0x7a, 0xcf, 0xd0, 0xda, 0x6e, 0x45, 0x17, 0xb2, 0x52, 0x5c, 0x96, 0x51, 0xe4 }, + { 0x64, 0x47, 0x5d, 0xfe, 0x76, 0x00, 0xd7, 0x17, 0x1b, 0xea, 0x0b, 0x39, 0x4e, 0x27, 0xc9, 0xb0, 0x0d, 0x8e, 0x74, 0xdd, 0x1e, 0x41, 0x6a, 0x79, 0x47, 0x36, 0x82, 0xad, 0x3d, 0xfd, 0xbb, 0x70, 0x66, 0x31, 0x55, 0x80, 0x55, 0xcf, 0xc8, 0xa4, 0x0e, 0x07, 0xbd, 0x01, 0x5a, 0x45, 0x40, 0xdc, 0xde, 0xa1, 0x58, 0x83, 0xcb, 0xbf, 0x31, 0x41, 0x2d, 0xf1, 0xde, 0x1c, 0xd4, 0x15, 0x2b, 0x91 }, + { 0x12, 0xcd, 0x16, 0x74, 0xa4, 0x48, 0x8a, 0x5d, 0x7c, 0x2b, 0x31, 0x60, 0xd2, 0xe2, 0xc4, 0xb5, 0x83, 0x71, 0xbe, 0xda, 0xd7, 0x93, 0x41, 0x8d, 0x6f, 0x19, 0xc6, 0xee, 0x38, 0x5d, 0x70, 0xb3, 0xe0, 0x67, 0x39, 0x36, 0x9d, 0x4d, 0xf9, 0x10, 0xed, 0xb0, 0xb0, 0xa5, 0x4c, 0xbf, 0xf4, 0x3d, 0x54, 0x54, 0x4c, 0xd3, 0x7a, 0xb3, 0xa0, 0x6c, 0xfa, 0x0a, 0x3d, 0xda, 0xc8, 0xb6, 0x6c, 0x89 }, + { 0x60, 0x75, 0x69, 0x66, 0x47, 0x9d, 0xed, 0xc6, 0xdd, 0x4b, 0xcf, 0xf8, 0xea, 0x7d, 0x1d, 0x4c, 0xe4, 0xd4, 0xaf, 0x2e, 0x7b, 0x09, 0x7e, 0x32, 0xe3, 0x76, 0x35, 0x18, 0x44, 0x11, 0x47, 0xcc, 0x12, 0xb3, 0xc0, 0xee, 0x6d, 0x2e, 0xca, 0xbf, 0x11, 0x98, 0xce, 0xc9, 0x2e, 0x86, 0xa3, 0x61, 0x6f, 0xba, 0x4f, 0x4e, 0x87, 0x2f, 0x58, 0x25, 0x33, 0x0a, 0xdb, 0xb4, 0xc1, 0xde, 0xe4, 0x44 }, + { 0xa7, 0x80, 0x3b, 0xcb, 0x71, 0xbc, 0x1d, 0x0f, 0x43, 0x83, 0xdd, 0xe1, 0xe0, 0x61, 0x2e, 0x04, 0xf8, 0x72, 0xb7, 0x15, 0xad, 0x30, 0x81, 0x5c, 0x22, 0x49, 0xcf, 0x34, 0xab, 0xb8, 0xb0, 0x24, 0x91, 0x5c, 0xb2, 0xfc, 0x9f, 0x4e, 0x7c, 0xc4, 0xc8, 0xcf, 0xd4, 0x5b, 0xe2, 0xd5, 0xa9, 0x1e, 0xab, 0x09, 0x41, 0xc7, 0xd2, 0x70, 0xe2, 0xda, 0x4c, 0xa4, 0xa9, 0xf7, 0xac, 0x68, 0x66, 0x3a }, + { 0xb8, 0x4e, 0xf6, 0xa7, 0x22, 0x9a, 0x34, 0xa7, 0x50, 0xd9, 0xa9, 0x8e, 0xe2, 0x52, 0x98, 0x71, 0x81, 0x6b, 0x87, 0xfb, 0xe3, 0xbc, 0x45, 0xb4, 0x5f, 0xa5, 0xae, 0x82, 0xd5, 0x14, 0x15, 0x40, 0x21, 0x11, 0x65, 0xc3, 0xc5, 0xd7, 0xa7, 0x47, 0x6b, 0xa5, 0xa4, 0xaa, 0x06, 0xd6, 0x64, 0x76, 0xf0, 0xd9, 0xdc, 0x49, 0xa3, 0xf1, 0xee, 0x72, 0xc3, 0xac, 0xab, 0xd4, 0x98, 0x96, 0x74, 0x14 }, + { 0xfa, 0xe4, 0xb6, 0xd8, 0xef, 0xc3, 0xf8, 0xc8, 0xe6, 0x4d, 0x00, 0x1d, 0xab, 0xec, 0x3a, 0x21, 0xf5, 0x44, 0xe8, 0x27, 0x14, 0x74, 0x52, 0x51, 0xb2, 0xb4, 0xb3, 0x93, 0xf2, 0xf4, 0x3e, 0x0d, 0xa3, 0xd4, 0x03, 0xc6, 0x4d, 0xb9, 0x5a, 0x2c, 0xb6, 0xe2, 0x3e, 0xbb, 0x7b, 0x9e, 0x94, 0xcd, 0xd5, 0xdd, 0xac, 0x54, 0xf0, 0x7c, 0x4a, 0x61, 0xbd, 0x3c, 0xb1, 0x0a, 0xa6, 0xf9, 0x3b, 0x49 }, + { 0x34, 0xf7, 0x28, 0x66, 0x05, 0xa1, 0x22, 0x36, 0x95, 0x40, 0x14, 0x1d, 0xed, 0x79, 0xb8, 0x95, 0x72, 0x55, 0xda, 0x2d, 0x41, 0x55, 0xab, 0xbf, 0x5a, 0x8d, 0xbb, 0x89, 0xc8, 0xeb, 0x7e, 0xde, 0x8e, 0xee, 0xf1, 0xda, 0xa4, 0x6d, 0xc2, 0x9d, 0x75, 0x1d, 0x04, 0x5d, 0xc3, 0xb1, 0xd6, 0x58, 0xbb, 0x64, 0xb8, 0x0f, 0xf8, 0x58, 0x9e, 0xdd, 0xb3, 0x82, 0x4b, 0x13, 0xda, 0x23, 0x5a, 0x6b }, + { 0x3b, 0x3b, 0x48, 0x43, 0x4b, 0xe2, 0x7b, 0x9e, 0xab, 0xab, 0xba, 0x43, 0xbf, 0x6b, 0x35, 0xf1, 0x4b, 0x30, 0xf6, 0xa8, 0x8d, 0xc2, 0xe7, 0x50, 0xc3, 0x58, 0x47, 0x0d, 0x6b, 0x3a, 0xa3, 0xc1, 0x8e, 0x47, 0xdb, 0x40, 0x17, 0xfa, 0x55, 0x10, 0x6d, 0x82, 0x52, 0xf0, 0x16, 0x37, 0x1a, 0x00, 0xf5, 0xf8, 0xb0, 0x70, 0xb7, 0x4b, 0xa5, 0xf2, 0x3c, 0xff, 0xc5, 0x51, 0x1c, 0x9f, 0x09, 0xf0 }, + { 0xba, 0x28, 0x9e, 0xbd, 0x65, 0x62, 0xc4, 0x8c, 0x3e, 0x10, 0xa8, 0xad, 0x6c, 0xe0, 0x2e, 0x73, 0x43, 0x3d, 0x1e, 0x93, 0xd7, 0xc9, 0x27, 0x9d, 0x4d, 0x60, 0xa7, 0xe8, 0x79, 0xee, 0x11, 0xf4, 0x41, 0xa0, 0x00, 0xf4, 0x8e, 0xd9, 0xf7, 0xc4, 0xed, 0x87, 0xa4, 0x51, 0x36, 0xd7, 0xdc, 0xcd, 0xca, 0x48, 0x21, 0x09, 0xc7, 0x8a, 0x51, 0x06, 0x2b, 0x3b, 0xa4, 0x04, 0x4a, 0xda, 0x24, 0x69 }, + { 0x02, 0x29, 0x39, 0xe2, 0x38, 0x6c, 0x5a, 0x37, 0x04, 0x98, 0x56, 0xc8, 0x50, 0xa2, 0xbb, 0x10, 0xa1, 0x3d, 0xfe, 0xa4, 0x21, 0x2b, 0x4c, 0x73, 0x2a, 0x88, 0x40, 0xa9, 0xff, 0xa5, 0xfa, 0xf5, 0x48, 0x75, 0xc5, 0x44, 0x88, 0x16, 0xb2, 0x78, 0x5a, 0x00, 0x7d, 0xa8, 0xa8, 0xd2, 0xbc, 0x7d, 0x71, 0xa5, 0x4e, 0x4e, 0x65, 0x71, 0xf1, 0x0b, 0x60, 0x0c, 0xbd, 0xb2, 0x5d, 0x13, 0xed, 0xe3 }, + { 0xe6, 0xfe, 0xc1, 0x9d, 0x89, 0xce, 0x87, 0x17, 0xb1, 0xa0, 0x87, 0x02, 0x46, 0x70, 0xfe, 0x02, 0x6f, 0x6c, 0x7c, 0xbd, 0xa1, 0x1c, 0xae, 0xf9, 0x59, 0xbb, 0x2d, 0x35, 0x1b, 0xf8, 0x56, 0xf8, 0x05, 0x5d, 0x1c, 0x0e, 0xbd, 0xaa, 0xa9, 0xd1, 0xb1, 0x78, 0x86, 0xfc, 0x2c, 0x56, 0x2b, 0x5e, 0x99, 0x64, 0x2f, 0xc0, 0x64, 0x71, 0x0c, 0x0d, 0x34, 0x88, 0xa0, 0x2b, 0x5e, 0xd7, 0xf6, 0xfd }, + { 0x94, 0xc9, 0x6f, 0x02, 0xa8, 0xf5, 0x76, 0xac, 0xa3, 0x2b, 0xa6, 0x1c, 0x2b, 0x20, 0x6f, 0x90, 0x72, 0x85, 0xd9, 0x29, 0x9b, 0x83, 0xac, 0x17, 0x5c, 0x20, 0x9a, 0x8d, 0x43, 0xd5, 0x3b, 0xfe, 0x68, 0x3d, 0xd1, 0xd8, 0x3e, 0x75, 0x49, 0xcb, 0x90, 0x6c, 0x28, 0xf5, 0x9a, 0xb7, 0xc4, 0x6f, 0x87, 0x51, 0x36, 0x6a, 0x28, 0xc3, 0x9d, 0xd5, 0xfe, 0x26, 0x93, 0xc9, 0x01, 0x96, 0x66, 0xc8 }, + { 0x31, 0xa0, 0xcd, 0x21, 0x5e, 0xbd, 0x2c, 0xb6, 0x1d, 0xe5, 0xb9, 0xed, 0xc9, 0x1e, 0x61, 0x95, 0xe3, 0x1c, 0x59, 0xa5, 0x64, 0x8d, 0x5c, 0x9f, 0x73, 0x7e, 0x12, 0x5b, 0x26, 0x05, 0x70, 0x8f, 0x2e, 0x32, 0x5a, 0xb3, 0x38, 0x1c, 0x8d, 0xce, 0x1a, 0x3e, 0x95, 0x88, 0x86, 0xf1, 0xec, 0xdc, 0x60, 0x31, 0x8f, 0x88, 0x2c, 0xfe, 0x20, 0xa2, 0x41, 0x91, 0x35, 0x2e, 0x61, 0x7b, 0x0f, 0x21 }, + { 0x91, 0xab, 0x50, 0x4a, 0x52, 0x2d, 0xce, 0x78, 0x77, 0x9f, 0x4c, 0x6c, 0x6b, 0xa2, 0xe6, 0xb6, 0xdb, 0x55, 0x65, 0xc7, 0x6d, 0x3e, 0x7e, 0x7c, 0x92, 0x0c, 0xaf, 0x7f, 0x75, 0x7e, 0xf9, 0xdb, 0x7c, 0x8f, 0xcf, 0x10, 0xe5, 0x7f, 0x03, 0x37, 0x9e, 0xa9, 0xbf, 0x75, 0xeb, 0x59, 0x89, 0x5d, 0x96, 0xe1, 0x49, 0x80, 0x0b, 0x6a, 0xae, 0x01, 0xdb, 0x77, 0x8b, 0xb9, 0x0a, 0xfb, 0xc9, 0x89 }, + { 0xd8, 0x5c, 0xab, 0xc6, 0xbd, 0x5b, 0x1a, 0x01, 0xa5, 0xaf, 0xd8, 0xc6, 0x73, 0x47, 0x40, 0xda, 0x9f, 0xd1, 0xc1, 0xac, 0xc6, 0xdb, 0x29, 0xbf, 0xc8, 0xa2, 0xe5, 0xb6, 0x68, 0xb0, 0x28, 0xb6, 0xb3, 0x15, 0x4b, 0xfb, 0x87, 0x03, 0xfa, 0x31, 0x80, 0x25, 0x1d, 0x58, 0x9a, 0xd3, 0x80, 0x40, 0xce, 0xb7, 0x07, 0xc4, 0xba, 0xd1, 0xb5, 0x34, 0x3c, 0xb4, 0x26, 0xb6, 0x1e, 0xaa, 0x49, 0xc1 }, + { 0xd6, 0x2e, 0xfb, 0xec, 0x2c, 0xa9, 0xc1, 0xf8, 0xbd, 0x66, 0xce, 0x8b, 0x3f, 0x6a, 0x89, 0x8c, 0xb3, 0xf7, 0x56, 0x6b, 0xa6, 0x56, 0x8c, 0x61, 0x8a, 0xd1, 0xfe, 0xb2, 0xb6, 0x5b, 0x76, 0xc3, 0xce, 0x1d, 0xd2, 0x0f, 0x73, 0x95, 0x37, 0x2f, 0xaf, 0x28, 0x42, 0x7f, 0x61, 0xc9, 0x27, 0x80, 0x49, 0xcf, 0x01, 0x40, 0xdf, 0x43, 0x4f, 0x56, 0x33, 0x04, 0x8c, 0x86, 0xb8, 0x1e, 0x03, 0x99 }, + { 0x7c, 0x8f, 0xdc, 0x61, 0x75, 0x43, 0x9e, 0x2c, 0x3d, 0xb1, 0x5b, 0xaf, 0xa7, 0xfb, 0x06, 0x14, 0x3a, 0x6a, 0x23, 0xbc, 0x90, 0xf4, 0x49, 0xe7, 0x9d, 0xee, 0xf7, 0x3c, 0x3d, 0x49, 0x2a, 0x67, 0x17, 0x15, 0xc1, 0x93, 0xb6, 0xfe, 0xa9, 0xf0, 0x36, 0x05, 0x0b, 0x94, 0x60, 0x69, 0x85, 0x6b, 0x89, 0x7e, 0x08, 0xc0, 0x07, 0x68, 0xf5, 0xee, 0x5d, 0xdc, 0xf7, 0x0b, 0x7c, 0xd6, 0xd0, 0xe0 }, + { 0x58, 0x60, 0x2e, 0xe7, 0x46, 0x8e, 0x6b, 0xc9, 0xdf, 0x21, 0xbd, 0x51, 0xb2, 0x3c, 0x00, 0x5f, 0x72, 0xd6, 0xcb, 0x01, 0x3f, 0x0a, 0x1b, 0x48, 0xcb, 0xec, 0x5e, 0xca, 0x29, 0x92, 0x99, 0xf9, 0x7f, 0x09, 0xf5, 0x4a, 0x9a, 0x01, 0x48, 0x3e, 0xae, 0xb3, 0x15, 0xa6, 0x47, 0x8b, 0xad, 0x37, 0xba, 0x47, 0xca, 0x13, 0x47, 0xc7, 0xc8, 0xfc, 0x9e, 0x66, 0x95, 0x59, 0x2c, 0x91, 0xd7, 0x23 }, + { 0x27, 0xf5, 0xb7, 0x9e, 0xd2, 0x56, 0xb0, 0x50, 0x99, 0x3d, 0x79, 0x34, 0x96, 0xed, 0xf4, 0x80, 0x7c, 0x1d, 0x85, 0xa7, 0xb0, 0xa6, 0x7c, 0x9c, 0x4f, 0xa9, 0x98, 0x60, 0x75, 0x0b, 0x0a, 0xe6, 0x69, 0x89, 0x67, 0x0a, 0x8f, 0xfd, 0x78, 0x56, 0xd7, 0xce, 0x41, 0x15, 0x99, 0xe5, 0x8c, 0x4d, 0x77, 0xb2, 0x32, 0xa6, 0x2b, 0xef, 0x64, 0xd1, 0x52, 0x75, 0xbe, 0x46, 0xa6, 0x82, 0x35, 0xff }, + { 0x39, 0x57, 0xa9, 0x76, 0xb9, 0xf1, 0x88, 0x7b, 0xf0, 0x04, 0xa8, 0xdc, 0xa9, 0x42, 0xc9, 0x2d, 0x2b, 0x37, 0xea, 0x52, 0x60, 0x0f, 0x25, 0xe0, 0xc9, 0xbc, 0x57, 0x07, 0xd0, 0x27, 0x9c, 0x00, 0xc6, 0xe8, 0x5a, 0x83, 0x9b, 0x0d, 0x2d, 0x8e, 0xb5, 0x9c, 0x51, 0xd9, 0x47, 0x88, 0xeb, 0xe6, 0x24, 0x74, 0xa7, 0x91, 0xca, 0xdf, 0x52, 0xcc, 0xcf, 0x20, 0xf5, 0x07, 0x0b, 0x65, 0x73, 0xfc }, + { 0xea, 0xa2, 0x37, 0x6d, 0x55, 0x38, 0x0b, 0xf7, 0x72, 0xec, 0xca, 0x9c, 0xb0, 0xaa, 0x46, 0x68, 0xc9, 0x5c, 0x70, 0x71, 0x62, 0xfa, 0x86, 0xd5, 0x18, 0xc8, 0xce, 0x0c, 0xa9, 0xbf, 0x73, 0x62, 0xb9, 0xf2, 0xa0, 0xad, 0xc3, 0xff, 0x59, 0x92, 0x2d, 0xf9, 0x21, 0xb9, 0x45, 0x67, 0xe8, 0x1e, 0x45, 0x2f, 0x6c, 0x1a, 0x07, 0xfc, 0x81, 0x7c, 0xeb, 0xe9, 0x96, 0x04, 0xb3, 0x50, 0x5d, 0x38 }, + { 0xc1, 0xe2, 0xc7, 0x8b, 0x6b, 0x27, 0x34, 0xe2, 0x48, 0x0e, 0xc5, 0x50, 0x43, 0x4c, 0xb5, 0xd6, 0x13, 0x11, 0x1a, 0xdc, 0xc2, 0x1d, 0x47, 0x55, 0x45, 0xc3, 0xb1, 0xb7, 0xe6, 0xff, 0x12, 0x44, 0x44, 0x76, 0xe5, 0xc0, 0x55, 0x13, 0x2e, 0x22, 0x29, 0xdc, 0x0f, 0x80, 0x70, 0x44, 0xbb, 0x91, 0x9b, 0x1a, 0x56, 0x62, 0xdd, 0x38, 0xa9, 0xee, 0x65, 0xe2, 0x43, 0xa3, 0x91, 0x1a, 0xed, 0x1a }, + { 0x8a, 0xb4, 0x87, 0x13, 0x38, 0x9d, 0xd0, 0xfc, 0xf9, 0xf9, 0x65, 0xd3, 0xce, 0x66, 0xb1, 0xe5, 0x59, 0xa1, 0xf8, 0xc5, 0x87, 0x41, 0xd6, 0x76, 0x83, 0xcd, 0x97, 0x13, 0x54, 0xf4, 0x52, 0xe6, 0x2d, 0x02, 0x07, 0xa6, 0x5e, 0x43, 0x6c, 0x5d, 0x5d, 0x8f, 0x8e, 0xe7, 0x1c, 0x6a, 0xbf, 0xe5, 0x0e, 0x66, 0x90, 0x04, 0xc3, 0x02, 0xb3, 0x1a, 0x7e, 0xa8, 0x31, 0x1d, 0x4a, 0x91, 0x60, 0x51 }, + { 0x24, 0xce, 0x0a, 0xdd, 0xaa, 0x4c, 0x65, 0x03, 0x8b, 0xd1, 0xb1, 0xc0, 0xf1, 0x45, 0x2a, 0x0b, 0x12, 0x87, 0x77, 0xaa, 0xbc, 0x94, 0xa2, 0x9d, 0xf2, 0xfd, 0x6c, 0x7e, 0x2f, 0x85, 0xf8, 0xab, 0x9a, 0xc7, 0xef, 0xf5, 0x16, 0xb0, 0xe0, 0xa8, 0x25, 0xc8, 0x4a, 0x24, 0xcf, 0xe4, 0x92, 0xea, 0xad, 0x0a, 0x63, 0x08, 0xe4, 0x6d, 0xd4, 0x2f, 0xe8, 0x33, 0x3a, 0xb9, 0x71, 0xbb, 0x30, 0xca }, + { 0x51, 0x54, 0xf9, 0x29, 0xee, 0x03, 0x04, 0x5b, 0x6b, 0x0c, 0x00, 0x04, 0xfa, 0x77, 0x8e, 0xde, 0xe1, 0xd1, 0x39, 0x89, 0x32, 0x67, 0xcc, 0x84, 0x82, 0x5a, 0xd7, 0xb3, 0x6c, 0x63, 0xde, 0x32, 0x79, 0x8e, 0x4a, 0x16, 0x6d, 0x24, 0x68, 0x65, 0x61, 0x35, 0x4f, 0x63, 0xb0, 0x07, 0x09, 0xa1, 0x36, 0x4b, 0x3c, 0x24, 0x1d, 0xe3, 0xfe, 0xbf, 0x07, 0x54, 0x04, 0x58, 0x97, 0x46, 0x7c, 0xd4 }, + { 0xe7, 0x4e, 0x90, 0x79, 0x20, 0xfd, 0x87, 0xbd, 0x5a, 0xd6, 0x36, 0xdd, 0x11, 0x08, 0x5e, 0x50, 0xee, 0x70, 0x45, 0x9c, 0x44, 0x3e, 0x1c, 0xe5, 0x80, 0x9a, 0xf2, 0xbc, 0x2e, 0xba, 0x39, 0xf9, 0xe6, 0xd7, 0x12, 0x8e, 0x0e, 0x37, 0x12, 0xc3, 0x16, 0xda, 0x06, 0xf4, 0x70, 0x5d, 0x78, 0xa4, 0x83, 0x8e, 0x28, 0x12, 0x1d, 0x43, 0x44, 0xa2, 0xc7, 0x9c, 0x5e, 0x0d, 0xb3, 0x07, 0xa6, 0x77 }, + { 0xbf, 0x91, 0xa2, 0x23, 0x34, 0xba, 0xc2, 0x0f, 0x3f, 0xd8, 0x06, 0x63, 0xb3, 0xcd, 0x06, 0xc4, 0xe8, 0x80, 0x2f, 0x30, 0xe6, 0xb5, 0x9f, 0x90, 0xd3, 0x03, 0x5c, 0xc9, 0x79, 0x8a, 0x21, 0x7e, 0xd5, 0xa3, 0x1a, 0xbb, 0xda, 0x7f, 0xa6, 0x84, 0x28, 0x27, 0xbd, 0xf2, 0xa7, 0xa1, 0xc2, 0x1f, 0x6f, 0xcf, 0xcc, 0xbb, 0x54, 0xc6, 0xc5, 0x29, 0x26, 0xf3, 0x2d, 0xa8, 0x16, 0x26, 0x9b, 0xe1 }, + { 0xd9, 0xd5, 0xc7, 0x4b, 0xe5, 0x12, 0x1b, 0x0b, 0xd7, 0x42, 0xf2, 0x6b, 0xff, 0xb8, 0xc8, 0x9f, 0x89, 0x17, 0x1f, 0x3f, 0x93, 0x49, 0x13, 0x49, 0x2b, 0x09, 0x03, 0xc2, 0x71, 0xbb, 0xe2, 0xb3, 0x39, 0x5e, 0xf2, 0x59, 0x66, 0x9b, 0xef, 0x43, 0xb5, 0x7f, 0x7f, 0xcc, 0x30, 0x27, 0xdb, 0x01, 0x82, 0x3f, 0x6b, 0xae, 0xe6, 0x6e, 0x4f, 0x9f, 0xea, 0xd4, 0xd6, 0x72, 0x6c, 0x74, 0x1f, 0xce }, + { 0x50, 0xc8, 0xb8, 0xcf, 0x34, 0xcd, 0x87, 0x9f, 0x80, 0xe2, 0xfa, 0xab, 0x32, 0x30, 0xb0, 0xc0, 0xe1, 0xcc, 0x3e, 0x9d, 0xca, 0xde, 0xb1, 0xb9, 0xd9, 0x7a, 0xb9, 0x23, 0x41, 0x5d, 0xd9, 0xa1, 0xfe, 0x38, 0xad, 0xdd, 0x5c, 0x11, 0x75, 0x6c, 0x67, 0x99, 0x0b, 0x25, 0x6e, 0x95, 0xad, 0x6d, 0x8f, 0x9f, 0xed, 0xce, 0x10, 0xbf, 0x1c, 0x90, 0x67, 0x9c, 0xde, 0x0e, 0xcf, 0x1b, 0xe3, 0x47 }, + { 0x0a, 0x38, 0x6e, 0x7c, 0xd5, 0xdd, 0x9b, 0x77, 0xa0, 0x35, 0xe0, 0x9f, 0xe6, 0xfe, 0xe2, 0xc8, 0xce, 0x61, 0xb5, 0x38, 0x3c, 0x87, 0xea, 0x43, 0x20, 0x50, 0x59, 0xc5, 0xe4, 0xcd, 0x4f, 0x44, 0x08, 0x31, 0x9b, 0xb0, 0xa8, 0x23, 0x60, 0xf6, 0xa5, 0x8e, 0x6c, 0x9c, 0xe3, 0xf4, 0x87, 0xc4, 0x46, 0x06, 0x3b, 0xf8, 0x13, 0xbc, 0x6b, 0xa5, 0x35, 0xe1, 0x7f, 0xc1, 0x82, 0x6c, 0xfc, 0x91 }, + { 0x1f, 0x14, 0x59, 0xcb, 0x6b, 0x61, 0xcb, 0xac, 0x5f, 0x0e, 0xfe, 0x8f, 0xc4, 0x87, 0x53, 0x8f, 0x42, 0x54, 0x89, 0x87, 0xfc, 0xd5, 0x62, 0x21, 0xcf, 0xa7, 0xbe, 0xb2, 0x25, 0x04, 0x76, 0x9e, 0x79, 0x2c, 0x45, 0xad, 0xfb, 0x1d, 0x6b, 0x3d, 0x60, 0xd7, 0xb7, 0x49, 0xc8, 0xa7, 0x5b, 0x0b, 0xdf, 0x14, 0xe8, 0xea, 0x72, 0x1b, 0x95, 0xdc, 0xa5, 0x38, 0xca, 0x6e, 0x25, 0x71, 0x12, 0x09 }, + { 0xe5, 0x8b, 0x38, 0x36, 0xb7, 0xd8, 0xfe, 0xdb, 0xb5, 0x0c, 0xa5, 0x72, 0x5c, 0x65, 0x71, 0xe7, 0x4c, 0x07, 0x85, 0xe9, 0x78, 0x21, 0xda, 0xb8, 0xb6, 0x29, 0x8c, 0x10, 0xe4, 0xc0, 0x79, 0xd4, 0xa6, 0xcd, 0xf2, 0x2f, 0x0f, 0xed, 0xb5, 0x50, 0x32, 0x92, 0x5c, 0x16, 0x74, 0x81, 0x15, 0xf0, 0x1a, 0x10, 0x5e, 0x77, 0xe0, 0x0c, 0xee, 0x3d, 0x07, 0x92, 0x4d, 0xc0, 0xd8, 0xf9, 0x06, 0x59 }, + { 0xb9, 0x29, 0xcc, 0x65, 0x05, 0xf0, 0x20, 0x15, 0x86, 0x72, 0xde, 0xda, 0x56, 0xd0, 0xdb, 0x08, 0x1a, 0x2e, 0xe3, 0x4c, 0x00, 0xc1, 0x10, 0x00, 0x29, 0xbd, 0xf8, 0xea, 0x98, 0x03, 0x4f, 0xa4, 0xbf, 0x3e, 0x86, 0x55, 0xec, 0x69, 0x7f, 0xe3, 0x6f, 0x40, 0x55, 0x3c, 0x5b, 0xb4, 0x68, 0x01, 0x64, 0x4a, 0x62, 0x7d, 0x33, 0x42, 0xf4, 0xfc, 0x92, 0xb6, 0x1f, 0x03, 0x29, 0x0f, 0xb3, 0x81 }, + { 0x72, 0xd3, 0x53, 0x99, 0x4b, 0x49, 0xd3, 0xe0, 0x31, 0x53, 0x92, 0x9a, 0x1e, 0x4d, 0x4f, 0x18, 0x8e, 0xe5, 0x8a, 0xb9, 0xe7, 0x2e, 0xe8, 0xe5, 0x12, 0xf2, 0x9b, 0xc7, 0x73, 0x91, 0x38, 0x19, 0xce, 0x05, 0x7d, 0xdd, 0x70, 0x02, 0xc0, 0x43, 0x3e, 0xe0, 0xa1, 0x61, 0x14, 0xe3, 0xd1, 0x56, 0xdd, 0x2c, 0x4a, 0x7e, 0x80, 0xee, 0x53, 0x37, 0x8b, 0x86, 0x70, 0xf2, 0x3e, 0x33, 0xef, 0x56 }, + { 0xc7, 0x0e, 0xf9, 0xbf, 0xd7, 0x75, 0xd4, 0x08, 0x17, 0x67, 0x37, 0xa0, 0x73, 0x6d, 0x68, 0x51, 0x7c, 0xe1, 0xaa, 0xad, 0x7e, 0x81, 0xa9, 0x3c, 0x8c, 0x1e, 0xd9, 0x67, 0xea, 0x21, 0x4f, 0x56, 0xc8, 0xa3, 0x77, 0xb1, 0x76, 0x3e, 0x67, 0x66, 0x15, 0xb6, 0x0f, 0x39, 0x88, 0x24, 0x1e, 0xae, 0x6e, 0xab, 0x96, 0x85, 0xa5, 0x12, 0x49, 0x29, 0xd2, 0x81, 0x88, 0xf2, 0x9e, 0xab, 0x06, 0xf7 }, + { 0xc2, 0x30, 0xf0, 0x80, 0x26, 0x79, 0xcb, 0x33, 0x82, 0x2e, 0xf8, 0xb3, 0xb2, 0x1b, 0xf7, 0xa9, 0xa2, 0x89, 0x42, 0x09, 0x29, 0x01, 0xd7, 0xda, 0xc3, 0x76, 0x03, 0x00, 0x83, 0x10, 0x26, 0xcf, 0x35, 0x4c, 0x92, 0x32, 0xdf, 0x3e, 0x08, 0x4d, 0x99, 0x03, 0x13, 0x0c, 0x60, 0x1f, 0x63, 0xc1, 0xf4, 0xa4, 0xa4, 0xb8, 0x10, 0x6e, 0x46, 0x8c, 0xd4, 0x43, 0xbb, 0xe5, 0xa7, 0x34, 0xf4, 0x5f }, + { 0x6f, 0x43, 0x09, 0x4c, 0xaf, 0xb5, 0xeb, 0xf1, 0xf7, 0xa4, 0x93, 0x7e, 0xc5, 0x0f, 0x56, 0xa4, 0xc9, 0xda, 0x30, 0x3c, 0xbb, 0x55, 0xac, 0x1f, 0x27, 0xf1, 0xf1, 0x97, 0x6c, 0xd9, 0x6b, 0xed, 0xa9, 0x46, 0x4f, 0x0e, 0x7b, 0x9c, 0x54, 0x62, 0x0b, 0x8a, 0x9f, 0xba, 0x98, 0x31, 0x64, 0xb8, 0xbe, 0x35, 0x78, 0x42, 0x5a, 0x02, 0x4f, 0x5f, 0xe1, 0x99, 0xc3, 0x63, 0x56, 0xb8, 0x89, 0x72 }, + { 0x37, 0x45, 0x27, 0x3f, 0x4c, 0x38, 0x22, 0x5d, 0xb2, 0x33, 0x73, 0x81, 0x87, 0x1a, 0x0c, 0x6a, 0xaf, 0xd3, 0xaf, 0x9b, 0x01, 0x8c, 0x88, 0xaa, 0x02, 0x02, 0x58, 0x50, 0xa5, 0xdc, 0x3a, 0x42, 0xa1, 0xa3, 0xe0, 0x3e, 0x56, 0xcb, 0xf1, 0xb0, 0x87, 0x6d, 0x63, 0xa4, 0x41, 0xf1, 0xd2, 0x85, 0x6a, 0x39, 0xb8, 0x80, 0x1e, 0xb5, 0xaf, 0x32, 0x52, 0x01, 0xc4, 0x15, 0xd6, 0x5e, 0x97, 0xfe }, + { 0xc5, 0x0c, 0x44, 0xcc, 0xa3, 0xec, 0x3e, 0xda, 0xae, 0x77, 0x9a, 0x7e, 0x17, 0x94, 0x50, 0xeb, 0xdd, 0xa2, 0xf9, 0x70, 0x67, 0xc6, 0x90, 0xaa, 0x6c, 0x5a, 0x4a, 0xc7, 0xc3, 0x01, 0x39, 0xbb, 0x27, 0xc0, 0xdf, 0x4d, 0xb3, 0x22, 0x0e, 0x63, 0xcb, 0x11, 0x0d, 0x64, 0xf3, 0x7f, 0xfe, 0x07, 0x8d, 0xb7, 0x26, 0x53, 0xe2, 0xda, 0xac, 0xf9, 0x3a, 0xe3, 0xf0, 0xa2, 0xd1, 0xa7, 0xeb, 0x2e }, + { 0x8a, 0xef, 0x26, 0x3e, 0x38, 0x5c, 0xbc, 0x61, 0xe1, 0x9b, 0x28, 0x91, 0x42, 0x43, 0x26, 0x2a, 0xf5, 0xaf, 0xe8, 0x72, 0x6a, 0xf3, 0xce, 0x39, 0xa7, 0x9c, 0x27, 0x02, 0x8c, 0xf3, 0xec, 0xd3, 0xf8, 0xd2, 0xdf, 0xd9, 0xcf, 0xc9, 0xad, 0x91, 0xb5, 0x8f, 0x6f, 0x20, 0x77, 0x8f, 0xd5, 0xf0, 0x28, 0x94, 0xa3, 0xd9, 0x1c, 0x7d, 0x57, 0xd1, 0xe4, 0xb8, 0x66, 0xa7, 0xf3, 0x64, 0xb6, 0xbe }, + { 0x28, 0x69, 0x61, 0x41, 0xde, 0x6e, 0x2d, 0x9b, 0xcb, 0x32, 0x35, 0x57, 0x8a, 0x66, 0x16, 0x6c, 0x14, 0x48, 0xd3, 0xe9, 0x05, 0xa1, 0xb4, 0x82, 0xd4, 0x23, 0xbe, 0x4b, 0xc5, 0x36, 0x9b, 0xc8, 0xc7, 0x4d, 0xae, 0x0a, 0xcc, 0x9c, 0xc1, 0x23, 0xe1, 0xd8, 0xdd, 0xce, 0x9f, 0x97, 0x91, 0x7e, 0x8c, 0x01, 0x9c, 0x55, 0x2d, 0xa3, 0x2d, 0x39, 0xd2, 0x21, 0x9b, 0x9a, 0xbf, 0x0f, 0xa8, 0xc8 }, + { 0x2f, 0xb9, 0xeb, 0x20, 0x85, 0x83, 0x01, 0x81, 0x90, 0x3a, 0x9d, 0xaf, 0xe3, 0xdb, 0x42, 0x8e, 0xe1, 0x5b, 0xe7, 0x66, 0x22, 0x24, 0xef, 0xd6, 0x43, 0x37, 0x1f, 0xb2, 0x56, 0x46, 0xae, 0xe7, 0x16, 0xe5, 0x31, 0xec, 0xa6, 0x9b, 0x2b, 0xdc, 0x82, 0x33, 0xf1, 0xa8, 0x08, 0x1f, 0xa4, 0x3d, 0xa1, 0x50, 0x03, 0x02, 0x97, 0x5a, 0x77, 0xf4, 0x2f, 0xa5, 0x92, 0x13, 0x67, 0x10, 0xe9, 0xdc }, + { 0x66, 0xf9, 0xa7, 0x14, 0x3f, 0x7a, 0x33, 0x14, 0xa6, 0x69, 0xbf, 0x2e, 0x24, 0xbb, 0xb3, 0x50, 0x14, 0x26, 0x1d, 0x63, 0x9f, 0x49, 0x5b, 0x6c, 0x9c, 0x1f, 0x10, 0x4f, 0xe8, 0xe3, 0x20, 0xac, 0xa6, 0x0d, 0x45, 0x50, 0xd6, 0x9d, 0x52, 0xed, 0xbd, 0x5a, 0x3c, 0xde, 0xb4, 0x01, 0x4a, 0xe6, 0x5b, 0x1d, 0x87, 0xaa, 0x77, 0x0b, 0x69, 0xae, 0x5c, 0x15, 0xf4, 0x33, 0x0b, 0x0b, 0x0a, 0xd8 }, + { 0xf4, 0xc4, 0xdd, 0x1d, 0x59, 0x4c, 0x35, 0x65, 0xe3, 0xe2, 0x5c, 0xa4, 0x3d, 0xad, 0x82, 0xf6, 0x2a, 0xbe, 0xa4, 0x83, 0x5e, 0xd4, 0xcd, 0x81, 0x1b, 0xcd, 0x97, 0x5e, 0x46, 0x27, 0x98, 0x28, 0xd4, 0x4d, 0x4c, 0x62, 0xc3, 0x67, 0x9f, 0x1b, 0x7f, 0x7b, 0x9d, 0xd4, 0x57, 0x1d, 0x7b, 0x49, 0x55, 0x73, 0x47, 0xb8, 0xc5, 0x46, 0x0c, 0xbd, 0xc1, 0xbe, 0xf6, 0x90, 0xfb, 0x2a, 0x08, 0xc0 }, + { 0x8f, 0x1d, 0xc9, 0x64, 0x9c, 0x3a, 0x84, 0x55, 0x1f, 0x8f, 0x6e, 0x91, 0xca, 0xc6, 0x82, 0x42, 0xa4, 0x3b, 0x1f, 0x8f, 0x32, 0x8e, 0xe9, 0x22, 0x80, 0x25, 0x73, 0x87, 0xfa, 0x75, 0x59, 0xaa, 0x6d, 0xb1, 0x2e, 0x4a, 0xea, 0xdc, 0x2d, 0x26, 0x09, 0x91, 0x78, 0x74, 0x9c, 0x68, 0x64, 0xb3, 0x57, 0xf3, 0xf8, 0x3b, 0x2f, 0xb3, 0xef, 0xa8, 0xd2, 0xa8, 0xdb, 0x05, 0x6b, 0xed, 0x6b, 0xcc }, + { 0x31, 0x39, 0xc1, 0xa7, 0xf9, 0x7a, 0xfd, 0x16, 0x75, 0xd4, 0x60, 0xeb, 0xbc, 0x07, 0xf2, 0x72, 0x8a, 0xa1, 0x50, 0xdf, 0x84, 0x96, 0x24, 0x51, 0x1e, 0xe0, 0x4b, 0x74, 0x3b, 0xa0, 0xa8, 0x33, 0x09, 0x2f, 0x18, 0xc1, 0x2d, 0xc9, 0x1b, 0x4d, 0xd2, 0x43, 0xf3, 0x33, 0x40, 0x2f, 0x59, 0xfe, 0x28, 0xab, 0xdb, 0xbb, 0xae, 0x30, 0x1e, 0x7b, 0x65, 0x9c, 0x7a, 0x26, 0xd5, 0xc0, 0xf9, 0x79 }, + { 0x06, 0xf9, 0x4a, 0x29, 0x96, 0x15, 0x8a, 0x81, 0x9f, 0xe3, 0x4c, 0x40, 0xde, 0x3c, 0xf0, 0x37, 0x9f, 0xd9, 0xfb, 0x85, 0xb3, 0xe3, 0x63, 0xba, 0x39, 0x26, 0xa0, 0xe7, 0xd9, 0x60, 0xe3, 0xf4, 0xc2, 0xe0, 0xc7, 0x0c, 0x7c, 0xe0, 0xcc, 0xb2, 0xa6, 0x4f, 0xc2, 0x98, 0x69, 0xf6, 0xe7, 0xab, 0x12, 0xbd, 0x4d, 0x3f, 0x14, 0xfc, 0xe9, 0x43, 0x27, 0x90, 0x27, 0xe7, 0x85, 0xfb, 0x5c, 0x29 }, + { 0xc2, 0x9c, 0x39, 0x9e, 0xf3, 0xee, 0xe8, 0x96, 0x1e, 0x87, 0x56, 0x5c, 0x1c, 0xe2, 0x63, 0x92, 0x5f, 0xc3, 0xd0, 0xce, 0x26, 0x7d, 0x13, 0xe4, 0x8d, 0xd9, 0xe7, 0x32, 0xee, 0x67, 0xb0, 0xf6, 0x9f, 0xad, 0x56, 0x40, 0x1b, 0x0f, 0x10, 0xfc, 0xaa, 0xc1, 0x19, 0x20, 0x10, 0x46, 0xcc, 0xa2, 0x8c, 0x5b, 0x14, 0xab, 0xde, 0xa3, 0x21, 0x2a, 0xe6, 0x55, 0x62, 0xf7, 0xf1, 0x38, 0xdb, 0x3d }, + { 0x4c, 0xec, 0x4c, 0x9d, 0xf5, 0x2e, 0xef, 0x05, 0xc3, 0xf6, 0xfa, 0xaa, 0x97, 0x91, 0xbc, 0x74, 0x45, 0x93, 0x71, 0x83, 0x22, 0x4e, 0xcc, 0x37, 0xa1, 0xe5, 0x8d, 0x01, 0x32, 0xd3, 0x56, 0x17, 0x53, 0x1d, 0x7e, 0x79, 0x5f, 0x52, 0xaf, 0x7b, 0x1e, 0xb9, 0xd1, 0x47, 0xde, 0x12, 0x92, 0xd3, 0x45, 0xfe, 0x34, 0x18, 0x23, 0xf8, 0xe6, 0xbc, 0x1e, 0x5b, 0xad, 0xca, 0x5c, 0x65, 0x61, 0x08 }, + { 0x89, 0x8b, 0xfb, 0xae, 0x93, 0xb3, 0xe1, 0x8d, 0x00, 0x69, 0x7e, 0xab, 0x7d, 0x97, 0x04, 0xfa, 0x36, 0xec, 0x33, 0x9d, 0x07, 0x61, 0x31, 0xce, 0xfd, 0xf3, 0x0e, 0xdb, 0xe8, 0xd9, 0xcc, 0x81, 0xc3, 0xa8, 0x0b, 0x12, 0x96, 0x59, 0xb1, 0x63, 0xa3, 0x23, 0xba, 0xb9, 0x79, 0x3d, 0x4f, 0xee, 0xd9, 0x2d, 0x54, 0xda, 0xe9, 0x66, 0xc7, 0x75, 0x29, 0x76, 0x4a, 0x09, 0xbe, 0x88, 0xdb, 0x45 }, + { 0xee, 0x9b, 0xd0, 0x46, 0x9d, 0x3a, 0xaf, 0x4f, 0x14, 0x03, 0x5b, 0xe4, 0x8a, 0x2c, 0x3b, 0x84, 0xd9, 0xb4, 0xb1, 0xff, 0xf1, 0xd9, 0x45, 0xe1, 0xf1, 0xc1, 0xd3, 0x89, 0x80, 0xa9, 0x51, 0xbe, 0x19, 0x7b, 0x25, 0xfe, 0x22, 0xc7, 0x31, 0xf2, 0x0a, 0xea, 0xcc, 0x93, 0x0b, 0xa9, 0xc4, 0xa1, 0xf4, 0x76, 0x22, 0x27, 0x61, 0x7a, 0xd3, 0x50, 0xfd, 0xab, 0xb4, 0xe8, 0x02, 0x73, 0xa0, 0xf4 }, + { 0x3d, 0x4d, 0x31, 0x13, 0x30, 0x05, 0x81, 0xcd, 0x96, 0xac, 0xbf, 0x09, 0x1c, 0x3d, 0x0f, 0x3c, 0x31, 0x01, 0x38, 0xcd, 0x69, 0x79, 0xe6, 0x02, 0x6c, 0xde, 0x62, 0x3e, 0x2d, 0xd1, 0xb2, 0x4d, 0x4a, 0x86, 0x38, 0xbe, 0xd1, 0x07, 0x33, 0x44, 0x78, 0x3a, 0xd0, 0x64, 0x9c, 0xc6, 0x30, 0x5c, 0xce, 0xc0, 0x4b, 0xeb, 0x49, 0xf3, 0x1c, 0x63, 0x30, 0x88, 0xa9, 0x9b, 0x65, 0x13, 0x02, 0x67 }, + { 0x95, 0xc0, 0x59, 0x1a, 0xd9, 0x1f, 0x92, 0x1a, 0xc7, 0xbe, 0x6d, 0x9c, 0xe3, 0x7e, 0x06, 0x63, 0xed, 0x80, 0x11, 0xc1, 0xcf, 0xd6, 0xd0, 0x16, 0x2a, 0x55, 0x72, 0xe9, 0x43, 0x68, 0xba, 0xc0, 0x20, 0x24, 0x48, 0x5e, 0x6a, 0x39, 0x85, 0x4a, 0xa4, 0x6f, 0xe3, 0x8e, 0x97, 0xd6, 0xc6, 0xb1, 0x94, 0x7c, 0xd2, 0x72, 0xd8, 0x6b, 0x06, 0xbb, 0x5b, 0x2f, 0x78, 0xb9, 0xb6, 0x8d, 0x55, 0x9d }, + { 0x22, 0x7b, 0x79, 0xde, 0xd3, 0x68, 0x15, 0x3b, 0xf4, 0x6c, 0x0a, 0x3c, 0xa9, 0x78, 0xbf, 0xdb, 0xef, 0x31, 0xf3, 0x02, 0x4a, 0x56, 0x65, 0x84, 0x24, 0x68, 0x49, 0x0b, 0x0f, 0xf7, 0x48, 0xae, 0x04, 0xe7, 0x83, 0x2e, 0xd4, 0xc9, 0xf4, 0x9d, 0xe9, 0xb1, 0x70, 0x67, 0x09, 0xd6, 0x23, 0xe5, 0xc8, 0xc1, 0x5e, 0x3c, 0xae, 0xca, 0xe8, 0xd5, 0xe4, 0x33, 0x43, 0x0f, 0xf7, 0x2f, 0x20, 0xeb }, + { 0x5d, 0x34, 0xf3, 0x95, 0x2f, 0x01, 0x05, 0xee, 0xf8, 0x8a, 0xe8, 0xb6, 0x4c, 0x6c, 0xe9, 0x5e, 0xbf, 0xad, 0xe0, 0xe0, 0x2c, 0x69, 0xb0, 0x87, 0x62, 0xa8, 0x71, 0x2d, 0x2e, 0x49, 0x11, 0xad, 0x3f, 0x94, 0x1f, 0xc4, 0x03, 0x4d, 0xc9, 0xb2, 0xe4, 0x79, 0xfd, 0xbc, 0xd2, 0x79, 0xb9, 0x02, 0xfa, 0xf5, 0xd8, 0x38, 0xbb, 0x2e, 0x0c, 0x64, 0x95, 0xd3, 0x72, 0xb5, 0xb7, 0x02, 0x98, 0x13 }, + { 0x7f, 0x93, 0x9b, 0xf8, 0x35, 0x3a, 0xbc, 0xe4, 0x9e, 0x77, 0xf1, 0x4f, 0x37, 0x50, 0xaf, 0x20, 0xb7, 0xb0, 0x39, 0x02, 0xe1, 0xa1, 0xe7, 0xfb, 0x6a, 0xaf, 0x76, 0xd0, 0x25, 0x9c, 0xd4, 0x01, 0xa8, 0x31, 0x90, 0xf1, 0x56, 0x40, 0xe7, 0x4f, 0x3e, 0x6c, 0x5a, 0x90, 0xe8, 0x39, 0xc7, 0x82, 0x1f, 0x64, 0x74, 0x75, 0x7f, 0x75, 0xc7, 0xbf, 0x90, 0x02, 0x08, 0x4d, 0xdc, 0x7a, 0x62, 0xdc }, + { 0x06, 0x2b, 0x61, 0xa2, 0xf9, 0xa3, 0x3a, 0x71, 0xd7, 0xd0, 0xa0, 0x61, 0x19, 0x64, 0x4c, 0x70, 0xb0, 0x71, 0x6a, 0x50, 0x4d, 0xe7, 0xe5, 0xe1, 0xbe, 0x49, 0xbd, 0x7b, 0x86, 0xe7, 0xed, 0x68, 0x17, 0x71, 0x4f, 0x9f, 0x0f, 0xc3, 0x13, 0xd0, 0x61, 0x29, 0x59, 0x7e, 0x9a, 0x22, 0x35, 0xec, 0x85, 0x21, 0xde, 0x36, 0xf7, 0x29, 0x0a, 0x90, 0xcc, 0xfc, 0x1f, 0xfa, 0x6d, 0x0a, 0xee, 0x29 }, + { 0xf2, 0x9e, 0x01, 0xee, 0xae, 0x64, 0x31, 0x1e, 0xb7, 0xf1, 0xc6, 0x42, 0x2f, 0x94, 0x6b, 0xf7, 0xbe, 0xa3, 0x63, 0x79, 0x52, 0x3e, 0x7b, 0x2b, 0xba, 0xba, 0x7d, 0x1d, 0x34, 0xa2, 0x2d, 0x5e, 0xa5, 0xf1, 0xc5, 0xa0, 0x9d, 0x5c, 0xe1, 0xfe, 0x68, 0x2c, 0xce, 0xd9, 0xa4, 0x79, 0x8d, 0x1a, 0x05, 0xb4, 0x6c, 0xd7, 0x2d, 0xff, 0x5c, 0x1b, 0x35, 0x54, 0x40, 0xb2, 0xa2, 0xd4, 0x76, 0xbc }, + { 0xec, 0x38, 0xcd, 0x3b, 0xba, 0xb3, 0xef, 0x35, 0xd7, 0xcb, 0x6d, 0x5c, 0x91, 0x42, 0x98, 0x35, 0x1d, 0x8a, 0x9d, 0xc9, 0x7f, 0xce, 0xe0, 0x51, 0xa8, 0xa0, 0x2f, 0x58, 0xe3, 0xed, 0x61, 0x84, 0xd0, 0xb7, 0x81, 0x0a, 0x56, 0x15, 0x41, 0x1a, 0xb1, 0xb9, 0x52, 0x09, 0xc3, 0xc8, 0x10, 0x11, 0x4f, 0xde, 0xb2, 0x24, 0x52, 0x08, 0x4e, 0x77, 0xf3, 0xf8, 0x47, 0xc6, 0xdb, 0xaa, 0xfe, 0x16 }, + { 0xc2, 0xae, 0xf5, 0xe0, 0xca, 0x43, 0xe8, 0x26, 0x41, 0x56, 0x5b, 0x8c, 0xb9, 0x43, 0xaa, 0x8b, 0xa5, 0x35, 0x50, 0xca, 0xef, 0x79, 0x3b, 0x65, 0x32, 0xfa, 0xfa, 0xd9, 0x4b, 0x81, 0x60, 0x82, 0xf0, 0x11, 0x3a, 0x3e, 0xa2, 0xf6, 0x36, 0x08, 0xab, 0x40, 0x43, 0x7e, 0xcc, 0x0f, 0x02, 0x29, 0xcb, 0x8f, 0xa2, 0x24, 0xdc, 0xf1, 0xc4, 0x78, 0xa6, 0x7d, 0x9b, 0x64, 0x16, 0x2b, 0x92, 0xd1 }, + { 0x15, 0xf5, 0x34, 0xef, 0xff, 0x71, 0x05, 0xcd, 0x1c, 0x25, 0x4d, 0x07, 0x4e, 0x27, 0xd5, 0x89, 0x8b, 0x89, 0x31, 0x3b, 0x7d, 0x36, 0x6d, 0xc2, 0xd7, 0xd8, 0x71, 0x13, 0xfa, 0x7d, 0x53, 0xaa, 0xe1, 0x3f, 0x6d, 0xba, 0x48, 0x7a, 0xd8, 0x10, 0x3d, 0x5e, 0x85, 0x4c, 0x91, 0xfd, 0xb6, 0xe1, 0xe7, 0x4b, 0x2e, 0xf6, 0xd1, 0x43, 0x17, 0x69, 0xc3, 0x07, 0x67, 0xdd, 0xe0, 0x67, 0xa3, 0x5c }, + { 0x89, 0xac, 0xbc, 0xa0, 0xb1, 0x69, 0x89, 0x7a, 0x0a, 0x27, 0x14, 0xc2, 0xdf, 0x8c, 0x95, 0xb5, 0xb7, 0x9c, 0xb6, 0x93, 0x90, 0x14, 0x2b, 0x7d, 0x60, 0x18, 0xbb, 0x3e, 0x30, 0x76, 0xb0, 0x99, 0xb7, 0x9a, 0x96, 0x41, 0x52, 0xa9, 0xd9, 0x12, 0xb1, 0xb8, 0x64, 0x12, 0xb7, 0xe3, 0x72, 0xe9, 0xce, 0xca, 0xd7, 0xf2, 0x5d, 0x4c, 0xba, 0xb8, 0xa3, 0x17, 0xbe, 0x36, 0x49, 0x2a, 0x67, 0xd7 }, + { 0xe3, 0xc0, 0x73, 0x91, 0x90, 0xed, 0x84, 0x9c, 0x9c, 0x96, 0x2f, 0xd9, 0xdb, 0xb5, 0x5e, 0x20, 0x7e, 0x62, 0x4f, 0xca, 0xc1, 0xeb, 0x41, 0x76, 0x91, 0x51, 0x54, 0x99, 0xee, 0xa8, 0xd8, 0x26, 0x7b, 0x7e, 0x8f, 0x12, 0x87, 0xa6, 0x36, 0x33, 0xaf, 0x50, 0x11, 0xfd, 0xe8, 0xc4, 0xdd, 0xf5, 0x5b, 0xfd, 0xf7, 0x22, 0xed, 0xf8, 0x88, 0x31, 0x41, 0x4f, 0x2c, 0xfa, 0xed, 0x59, 0xcb, 0x9a }, + { 0x8d, 0x6c, 0xf8, 0x7c, 0x08, 0x38, 0x0d, 0x2d, 0x15, 0x06, 0xee, 0xe4, 0x6f, 0xd4, 0x22, 0x2d, 0x21, 0xd8, 0xc0, 0x4e, 0x58, 0x5f, 0xbf, 0xd0, 0x82, 0x69, 0xc9, 0x8f, 0x70, 0x28, 0x33, 0xa1, 0x56, 0x32, 0x6a, 0x07, 0x24, 0x65, 0x64, 0x00, 0xee, 0x09, 0x35, 0x1d, 0x57, 0xb4, 0x40, 0x17, 0x5e, 0x2a, 0x5d, 0xe9, 0x3c, 0xc5, 0xf8, 0x0d, 0xb6, 0xda, 0xf8, 0x35, 0x76, 0xcf, 0x75, 0xfa }, + { 0xda, 0x24, 0xbe, 0xde, 0x38, 0x36, 0x66, 0xd5, 0x63, 0xee, 0xed, 0x37, 0xf6, 0x31, 0x9b, 0xaf, 0x20, 0xd5, 0xc7, 0x5d, 0x16, 0x35, 0xa6, 0xba, 0x5e, 0xf4, 0xcf, 0xa1, 0xac, 0x95, 0x48, 0x7e, 0x96, 0xf8, 0xc0, 0x8a, 0xf6, 0x00, 0xaa, 0xb8, 0x7c, 0x98, 0x6e, 0xba, 0xd4, 0x9f, 0xc7, 0x0a, 0x58, 0xb4, 0x89, 0x0b, 0x9c, 0x87, 0x6e, 0x09, 0x10, 0x16, 0xda, 0xf4, 0x9e, 0x1d, 0x32, 0x2e }, + { 0xf9, 0xd1, 0xd1, 0xb1, 0xe8, 0x7e, 0xa7, 0xae, 0x75, 0x3a, 0x02, 0x97, 0x50, 0xcc, 0x1c, 0xf3, 0xd0, 0x15, 0x7d, 0x41, 0x80, 0x5e, 0x24, 0x5c, 0x56, 0x17, 0xbb, 0x93, 0x4e, 0x73, 0x2f, 0x0a, 0xe3, 0x18, 0x0b, 0x78, 0xe0, 0x5b, 0xfe, 0x76, 0xc7, 0xc3, 0x05, 0x1e, 0x3e, 0x3a, 0xc7, 0x8b, 0x9b, 0x50, 0xc0, 0x51, 0x42, 0x65, 0x7e, 0x1e, 0x03, 0x21, 0x5d, 0x6e, 0xc7, 0xbf, 0xd0, 0xfc }, + { 0x11, 0xb7, 0xbc, 0x16, 0x68, 0x03, 0x20, 0x48, 0xaa, 0x43, 0x34, 0x3d, 0xe4, 0x76, 0x39, 0x5e, 0x81, 0x4b, 0xbb, 0xc2, 0x23, 0x67, 0x8d, 0xb9, 0x51, 0xa1, 0xb0, 0x3a, 0x02, 0x1e, 0xfa, 0xc9, 0x48, 0xcf, 0xbe, 0x21, 0x5f, 0x97, 0xfe, 0x9a, 0x72, 0xa2, 0xf6, 0xbc, 0x03, 0x9e, 0x39, 0x56, 0xbf, 0xa4, 0x17, 0xc1, 0xa9, 0xf1, 0x0d, 0x6d, 0x7b, 0xa5, 0xd3, 0xd3, 0x2f, 0xf3, 0x23, 0xe5 }, + { 0xb8, 0xd9, 0x00, 0x0e, 0x4f, 0xc2, 0xb0, 0x66, 0xed, 0xb9, 0x1a, 0xfe, 0xe8, 0xe7, 0xeb, 0x0f, 0x24, 0xe3, 0xa2, 0x01, 0xdb, 0x8b, 0x67, 0x93, 0xc0, 0x60, 0x85, 0x81, 0xe6, 0x28, 0xed, 0x0b, 0xcc, 0x4e, 0x5a, 0xa6, 0x78, 0x79, 0x92, 0xa4, 0xbc, 0xc4, 0x4e, 0x28, 0x80, 0x93, 0xe6, 0x3e, 0xe8, 0x3a, 0xbd, 0x0b, 0xc3, 0xec, 0x6d, 0x09, 0x34, 0xa6, 0x74, 0xa4, 0xda, 0x13, 0x83, 0x8a }, + { 0xce, 0x32, 0x5e, 0x29, 0x4f, 0x9b, 0x67, 0x19, 0xd6, 0xb6, 0x12, 0x78, 0x27, 0x6a, 0xe0, 0x6a, 0x25, 0x64, 0xc0, 0x3b, 0xb0, 0xb7, 0x83, 0xfa, 0xfe, 0x78, 0x5b, 0xdf, 0x89, 0xc7, 0xd5, 0xac, 0xd8, 0x3e, 0x78, 0x75, 0x6d, 0x30, 0x1b, 0x44, 0x56, 0x99, 0x02, 0x4e, 0xae, 0xb7, 0x7b, 0x54, 0xd4, 0x77, 0x33, 0x6e, 0xc2, 0xa4, 0xf3, 0x32, 0xf2, 0xb3, 0xf8, 0x87, 0x65, 0xdd, 0xb0, 0xc3 }, + { 0x29, 0xac, 0xc3, 0x0e, 0x96, 0x03, 0xae, 0x2f, 0xcc, 0xf9, 0x0b, 0xf9, 0x7e, 0x6c, 0xc4, 0x63, 0xeb, 0xe2, 0x8c, 0x1b, 0x2f, 0x9b, 0x4b, 0x76, 0x5e, 0x70, 0x53, 0x7c, 0x25, 0xc7, 0x02, 0xa2, 0x9d, 0xcb, 0xfb, 0xf1, 0x4c, 0x99, 0xc5, 0x43, 0x45, 0xba, 0x2b, 0x51, 0xf1, 0x7b, 0x77, 0xb5, 0xf1, 0x5d, 0xb9, 0x2b, 0xba, 0xd8, 0xfa, 0x95, 0xc4, 0x71, 0xf5, 0xd0, 0x70, 0xa1, 0x37, 0xcc }, + { 0x33, 0x79, 0xcb, 0xaa, 0xe5, 0x62, 0xa8, 0x7b, 0x4c, 0x04, 0x25, 0x55, 0x0f, 0xfd, 0xd6, 0xbf, 0xe1, 0x20, 0x3f, 0x0d, 0x66, 0x6c, 0xc7, 0xea, 0x09, 0x5b, 0xe4, 0x07, 0xa5, 0xdf, 0xe6, 0x1e, 0xe9, 0x14, 0x41, 0xcd, 0x51, 0x54, 0xb3, 0xe5, 0x3b, 0x4f, 0x5f, 0xb3, 0x1a, 0xd4, 0xc7, 0xa9, 0xad, 0x5c, 0x7a, 0xf4, 0xae, 0x67, 0x9a, 0xa5, 0x1a, 0x54, 0x00, 0x3a, 0x54, 0xca, 0x6b, 0x2d }, + { 0x30, 0x95, 0xa3, 0x49, 0xd2, 0x45, 0x70, 0x8c, 0x7c, 0xf5, 0x50, 0x11, 0x87, 0x03, 0xd7, 0x30, 0x2c, 0x27, 0xb6, 0x0a, 0xf5, 0xd4, 0xe6, 0x7f, 0xc9, 0x78, 0xf8, 0xa4, 0xe6, 0x09, 0x53, 0xc7, 0xa0, 0x4f, 0x92, 0xfc, 0xf4, 0x1a, 0xee, 0x64, 0x32, 0x1c, 0xcb, 0x70, 0x7a, 0x89, 0x58, 0x51, 0x55, 0x2b, 0x1e, 0x37, 0xb0, 0x0b, 0xc5, 0xe6, 0xb7, 0x2f, 0xa5, 0xbc, 0xef, 0x9e, 0x3f, 0xff }, + { 0x07, 0x26, 0x2d, 0x73, 0x8b, 0x09, 0x32, 0x1f, 0x4d, 0xbc, 0xce, 0xc4, 0xbb, 0x26, 0xf4, 0x8c, 0xb0, 0xf0, 0xed, 0x24, 0x6c, 0xe0, 0xb3, 0x1b, 0x9a, 0x6e, 0x7b, 0xc6, 0x83, 0x04, 0x9f, 0x1f, 0x3e, 0x55, 0x45, 0xf2, 0x8c, 0xe9, 0x32, 0xdd, 0x98, 0x5c, 0x5a, 0xb0, 0xf4, 0x3b, 0xd6, 0xde, 0x07, 0x70, 0x56, 0x0a, 0xf3, 0x29, 0x06, 0x5e, 0xd2, 0xe4, 0x9d, 0x34, 0x62, 0x4c, 0x2c, 0xbb }, + { 0xb6, 0x40, 0x5e, 0xca, 0x8e, 0xe3, 0x31, 0x6c, 0x87, 0x06, 0x1c, 0xc6, 0xec, 0x18, 0xdb, 0xa5, 0x3e, 0x6c, 0x25, 0x0c, 0x63, 0xba, 0x1f, 0x3b, 0xae, 0x9e, 0x55, 0xdd, 0x34, 0x98, 0x03, 0x6a, 0xf0, 0x8c, 0xd2, 0x72, 0xaa, 0x24, 0xd7, 0x13, 0xc6, 0x02, 0x0d, 0x77, 0xab, 0x2f, 0x39, 0x19, 0xaf, 0x1a, 0x32, 0xf3, 0x07, 0x42, 0x06, 0x18, 0xab, 0x97, 0xe7, 0x39, 0x53, 0x99, 0x4f, 0xb4 }, + { 0x7e, 0xe6, 0x82, 0xf6, 0x31, 0x48, 0xee, 0x45, 0xf6, 0xe5, 0x31, 0x5d, 0xa8, 0x1e, 0x5c, 0x6e, 0x55, 0x7c, 0x2c, 0x34, 0x64, 0x1f, 0xc5, 0x09, 0xc7, 0xa5, 0x70, 0x10, 0x88, 0xc3, 0x8a, 0x74, 0x75, 0x61, 0x68, 0xe2, 0xcd, 0x8d, 0x35, 0x1e, 0x88, 0xfd, 0x1a, 0x45, 0x1f, 0x36, 0x0a, 0x01, 0xf5, 0xb2, 0x58, 0x0f, 0x9b, 0x5a, 0x2e, 0x8c, 0xfc, 0x13, 0x8f, 0x3d, 0xd5, 0x9a, 0x3f, 0xfc }, + { 0x1d, 0x26, 0x3c, 0x17, 0x9d, 0x6b, 0x26, 0x8f, 0x6f, 0xa0, 0x16, 0xf3, 0xa4, 0xf2, 0x9e, 0x94, 0x38, 0x91, 0x12, 0x5e, 0xd8, 0x59, 0x3c, 0x81, 0x25, 0x60, 0x59, 0xf5, 0xa7, 0xb4, 0x4a, 0xf2, 0xdc, 0xb2, 0x03, 0x0d, 0x17, 0x5c, 0x00, 0xe6, 0x2e, 0xca, 0xf7, 0xee, 0x96, 0x68, 0x2a, 0xa0, 0x7a, 0xb2, 0x0a, 0x61, 0x10, 0x24, 0xa2, 0x85, 0x32, 0xb1, 0xc2, 0x5b, 0x86, 0x65, 0x79, 0x02 }, + { 0x10, 0x6d, 0x13, 0x2c, 0xbd, 0xb4, 0xcd, 0x25, 0x97, 0x81, 0x28, 0x46, 0xe2, 0xbc, 0x1b, 0xf7, 0x32, 0xfe, 0xc5, 0xf0, 0xa5, 0xf6, 0x5d, 0xbb, 0x39, 0xec, 0x4e, 0x6d, 0xc6, 0x4a, 0xb2, 0xce, 0x6d, 0x24, 0x63, 0x0d, 0x0f, 0x15, 0xa8, 0x05, 0xc3, 0x54, 0x00, 0x25, 0xd8, 0x4a, 0xfa, 0x98, 0xe3, 0x67, 0x03, 0xc3, 0xdb, 0xee, 0x71, 0x3e, 0x72, 0xdd, 0xe8, 0x46, 0x5b, 0xc1, 0xbe, 0x7e }, + { 0x0e, 0x79, 0x96, 0x82, 0x26, 0x65, 0x06, 0x67, 0xa8, 0xd8, 0x62, 0xea, 0x8d, 0xa4, 0x89, 0x1a, 0xf5, 0x6a, 0x4e, 0x3a, 0x8b, 0x6d, 0x17, 0x50, 0xe3, 0x94, 0xf0, 0xde, 0xa7, 0x6d, 0x64, 0x0d, 0x85, 0x07, 0x7b, 0xce, 0xc2, 0xcc, 0x86, 0x88, 0x6e, 0x50, 0x67, 0x51, 0xb4, 0xf6, 0xa5, 0x83, 0x8f, 0x7f, 0x0b, 0x5f, 0xef, 0x76, 0x5d, 0x9d, 0xc9, 0x0d, 0xcd, 0xcb, 0xaf, 0x07, 0x9f, 0x08 }, + { 0x52, 0x11, 0x56, 0xa8, 0x2a, 0xb0, 0xc4, 0xe5, 0x66, 0xe5, 0x84, 0x4d, 0x5e, 0x31, 0xad, 0x9a, 0xaf, 0x14, 0x4b, 0xbd, 0x5a, 0x46, 0x4f, 0xdc, 0xa3, 0x4d, 0xbd, 0x57, 0x17, 0xe8, 0xff, 0x71, 0x1d, 0x3f, 0xfe, 0xbb, 0xfa, 0x08, 0x5d, 0x67, 0xfe, 0x99, 0x6a, 0x34, 0xf6, 0xd3, 0xe4, 0xe6, 0x0b, 0x13, 0x96, 0xbf, 0x4b, 0x16, 0x10, 0xc2, 0x63, 0xbd, 0xbb, 0x83, 0x4d, 0x56, 0x08, 0x16 }, + { 0x1a, 0xba, 0x88, 0xbe, 0xfc, 0x55, 0xbc, 0x25, 0xef, 0xbc, 0xe0, 0x2d, 0xb8, 0xb9, 0x93, 0x3e, 0x46, 0xf5, 0x76, 0x61, 0xba, 0xea, 0xbe, 0xb2, 0x1c, 0xc2, 0x57, 0x4d, 0x2a, 0x51, 0x8a, 0x3c, 0xba, 0x5d, 0xc5, 0xa3, 0x8e, 0x49, 0x71, 0x34, 0x40, 0xb2, 0x5f, 0x9c, 0x74, 0x4e, 0x75, 0xf6, 0xb8, 0x5c, 0x9d, 0x8f, 0x46, 0x81, 0xf6, 0x76, 0x16, 0x0f, 0x61, 0x05, 0x35, 0x7b, 0x84, 0x06 }, + { 0x5a, 0x99, 0x49, 0xfc, 0xb2, 0xc4, 0x73, 0xcd, 0xa9, 0x68, 0xac, 0x1b, 0x5d, 0x08, 0x56, 0x6d, 0xc2, 0xd8, 0x16, 0xd9, 0x60, 0xf5, 0x7e, 0x63, 0xb8, 0x98, 0xfa, 0x70, 0x1c, 0xf8, 0xeb, 0xd3, 0xf5, 0x9b, 0x12, 0x4d, 0x95, 0xbf, 0xbb, 0xed, 0xc5, 0xf1, 0xcf, 0x0e, 0x17, 0xd5, 0xea, 0xed, 0x0c, 0x02, 0xc5, 0x0b, 0x69, 0xd8, 0xa4, 0x02, 0xca, 0xbc, 0xca, 0x44, 0x33, 0xb5, 0x1f, 0xd4 }, + { 0xb0, 0xce, 0xad, 0x09, 0x80, 0x7c, 0x67, 0x2a, 0xf2, 0xeb, 0x2b, 0x0f, 0x06, 0xdd, 0xe4, 0x6c, 0xf5, 0x37, 0x0e, 0x15, 0xa4, 0x09, 0x6b, 0x1a, 0x7d, 0x7c, 0xbb, 0x36, 0xec, 0x31, 0xc2, 0x05, 0xfb, 0xef, 0xca, 0x00, 0xb7, 0xa4, 0x16, 0x2f, 0xa8, 0x9f, 0xb4, 0xfb, 0x3e, 0xb7, 0x8d, 0x79, 0x77, 0x0c, 0x23, 0xf4, 0x4e, 0x72, 0x06, 0x66, 0x4c, 0xe3, 0xcd, 0x93, 0x1c, 0x29, 0x1e, 0x5d }, + { 0xbb, 0x66, 0x64, 0x93, 0x1e, 0xc9, 0x70, 0x44, 0xe4, 0x5b, 0x2a, 0xe4, 0x20, 0xae, 0x1c, 0x55, 0x1a, 0x88, 0x74, 0xbc, 0x93, 0x7d, 0x08, 0xe9, 0x69, 0x39, 0x9c, 0x39, 0x64, 0xeb, 0xdb, 0xa8, 0x34, 0x6c, 0xdd, 0x5d, 0x09, 0xca, 0xaf, 0xe4, 0xc2, 0x8b, 0xa7, 0xec, 0x78, 0x81, 0x91, 0xce, 0xca, 0x65, 0xdd, 0xd6, 0xf9, 0x5f, 0x18, 0x58, 0x3e, 0x04, 0x0d, 0x0f, 0x30, 0xd0, 0x36, 0x4d }, + { 0x65, 0xbc, 0x77, 0x0a, 0x5f, 0xaa, 0x37, 0x92, 0x36, 0x98, 0x03, 0x68, 0x3e, 0x84, 0x4b, 0x0b, 0xe7, 0xee, 0x96, 0xf2, 0x9f, 0x6d, 0x6a, 0x35, 0x56, 0x80, 0x06, 0xbd, 0x55, 0x90, 0xf9, 0xa4, 0xef, 0x63, 0x9b, 0x7a, 0x80, 0x61, 0xc7, 0xb0, 0x42, 0x4b, 0x66, 0xb6, 0x0a, 0xc3, 0x4a, 0xf3, 0x11, 0x99, 0x05, 0xf3, 0x3a, 0x9d, 0x8c, 0x3a, 0xe1, 0x83, 0x82, 0xca, 0x9b, 0x68, 0x99, 0x00 }, + { 0xea, 0x9b, 0x4d, 0xca, 0x33, 0x33, 0x36, 0xaa, 0xf8, 0x39, 0xa4, 0x5c, 0x6e, 0xaa, 0x48, 0xb8, 0xcb, 0x4c, 0x7d, 0xda, 0xbf, 0xfe, 0xa4, 0xf6, 0x43, 0xd6, 0x35, 0x7e, 0xa6, 0x62, 0x8a, 0x48, 0x0a, 0x5b, 0x45, 0xf2, 0xb0, 0x52, 0xc1, 0xb0, 0x7d, 0x1f, 0xed, 0xca, 0x91, 0x8b, 0x6f, 0x11, 0x39, 0xd8, 0x0f, 0x74, 0xc2, 0x45, 0x10, 0xdc, 0xba, 0xa4, 0xbe, 0x70, 0xea, 0xcc, 0x1b, 0x06 }, + { 0xe6, 0x34, 0x2f, 0xb4, 0xa7, 0x80, 0xad, 0x97, 0x5d, 0x0e, 0x24, 0xbc, 0xe1, 0x49, 0x98, 0x9b, 0x91, 0xd3, 0x60, 0x55, 0x7e, 0x87, 0x99, 0x4f, 0x6b, 0x45, 0x7b, 0x89, 0x55, 0x75, 0xcc, 0x02, 0xd0, 0xc1, 0x5b, 0xad, 0x3c, 0xe7, 0x57, 0x7f, 0x4c, 0x63, 0x92, 0x7f, 0xf1, 0x3f, 0x3e, 0x38, 0x1f, 0xf7, 0xe7, 0x2b, 0xdb, 0xe7, 0x45, 0x32, 0x48, 0x44, 0xa9, 0xd2, 0x7e, 0x3f, 0x1c, 0x01 }, + { 0x3e, 0x20, 0x9c, 0x9b, 0x33, 0xe8, 0xe4, 0x61, 0x17, 0x8a, 0xb4, 0x6b, 0x1c, 0x64, 0xb4, 0x9a, 0x07, 0xfb, 0x74, 0x5f, 0x1c, 0x8b, 0xc9, 0x5f, 0xbf, 0xb9, 0x4c, 0x6b, 0x87, 0xc6, 0x95, 0x16, 0x65, 0x1b, 0x26, 0x4e, 0xf9, 0x80, 0x93, 0x7f, 0xad, 0x41, 0x23, 0x8b, 0x91, 0xdd, 0xc0, 0x11, 0xa5, 0xdd, 0x77, 0x7c, 0x7e, 0xfd, 0x44, 0x94, 0xb4, 0xb6, 0xec, 0xd3, 0xa9, 0xc2, 0x2a, 0xc0 }, + { 0xfd, 0x6a, 0x3d, 0x5b, 0x18, 0x75, 0xd8, 0x04, 0x86, 0xd6, 0xe6, 0x96, 0x94, 0xa5, 0x6d, 0xbb, 0x04, 0xa9, 0x9a, 0x4d, 0x05, 0x1f, 0x15, 0xdb, 0x26, 0x89, 0x77, 0x6b, 0xa1, 0xc4, 0x88, 0x2e, 0x6d, 0x46, 0x2a, 0x60, 0x3b, 0x70, 0x15, 0xdc, 0x9f, 0x4b, 0x74, 0x50, 0xf0, 0x53, 0x94, 0x30, 0x3b, 0x86, 0x52, 0xcf, 0xb4, 0x04, 0xa2, 0x66, 0x96, 0x2c, 0x41, 0xba, 0xe6, 0xe1, 0x8a, 0x94 }, + { 0x95, 0x1e, 0x27, 0x51, 0x7e, 0x6b, 0xad, 0x9e, 0x41, 0x95, 0xfc, 0x86, 0x71, 0xde, 0xe3, 0xe7, 0xe9, 0xbe, 0x69, 0xce, 0xe1, 0x42, 0x2c, 0xb9, 0xfe, 0xcf, 0xce, 0x0d, 0xba, 0x87, 0x5f, 0x7b, 0x31, 0x0b, 0x93, 0xee, 0x3a, 0x3d, 0x55, 0x8f, 0x94, 0x1f, 0x63, 0x5f, 0x66, 0x8f, 0xf8, 0x32, 0xd2, 0xc1, 0xd0, 0x33, 0xc5, 0xe2, 0xf0, 0x99, 0x7e, 0x4c, 0x66, 0xf1, 0x47, 0x34, 0x4e, 0x02 }, + { 0x8e, 0xba, 0x2f, 0x87, 0x4f, 0x1a, 0xe8, 0x40, 0x41, 0x90, 0x3c, 0x7c, 0x42, 0x53, 0xc8, 0x22, 0x92, 0x53, 0x0f, 0xc8, 0x50, 0x95, 0x50, 0xbf, 0xdc, 0x34, 0xc9, 0x5c, 0x7e, 0x28, 0x89, 0xd5, 0x65, 0x0b, 0x0a, 0xd8, 0xcb, 0x98, 0x8e, 0x5c, 0x48, 0x94, 0xcb, 0x87, 0xfb, 0xfb, 0xb1, 0x96, 0x12, 0xea, 0x93, 0xcc, 0xc4, 0xc5, 0xca, 0xd1, 0x71, 0x58, 0xb9, 0x76, 0x34, 0x64, 0xb4, 0x92 }, + { 0x16, 0xf7, 0x12, 0xea, 0xa1, 0xb7, 0xc6, 0x35, 0x47, 0x19, 0xa8, 0xe7, 0xdb, 0xdf, 0xaf, 0x55, 0xe4, 0x06, 0x3a, 0x4d, 0x27, 0x7d, 0x94, 0x75, 0x50, 0x01, 0x9b, 0x38, 0xdf, 0xb5, 0x64, 0x83, 0x09, 0x11, 0x05, 0x7d, 0x50, 0x50, 0x61, 0x36, 0xe2, 0x39, 0x4c, 0x3b, 0x28, 0x94, 0x5c, 0xc9, 0x64, 0x96, 0x7d, 0x54, 0xe3, 0x00, 0x0c, 0x21, 0x81, 0x62, 0x6c, 0xfb, 0x9b, 0x73, 0xef, 0xd2 }, + { 0xc3, 0x96, 0x39, 0xe7, 0xd5, 0xc7, 0xfb, 0x8c, 0xdd, 0x0f, 0xd3, 0xe6, 0xa5, 0x20, 0x96, 0x03, 0x94, 0x37, 0x12, 0x2f, 0x21, 0xc7, 0x8f, 0x16, 0x79, 0xce, 0xa9, 0xd7, 0x8a, 0x73, 0x4c, 0x56, 0xec, 0xbe, 0xb2, 0x86, 0x54, 0xb4, 0xf1, 0x8e, 0x34, 0x2c, 0x33, 0x1f, 0x6f, 0x72, 0x29, 0xec, 0x4b, 0x4b, 0xc2, 0x81, 0xb2, 0xd8, 0x0a, 0x6e, 0xb5, 0x00, 0x43, 0xf3, 0x17, 0x96, 0xc8, 0x8c }, + { 0x72, 0xd0, 0x81, 0xaf, 0x99, 0xf8, 0xa1, 0x73, 0xdc, 0xc9, 0xa0, 0xac, 0x4e, 0xb3, 0x55, 0x74, 0x05, 0x63, 0x9a, 0x29, 0x08, 0x4b, 0x54, 0xa4, 0x01, 0x72, 0x91, 0x2a, 0x2f, 0x8a, 0x39, 0x51, 0x29, 0xd5, 0x53, 0x6f, 0x09, 0x18, 0xe9, 0x02, 0xf9, 0xe8, 0xfa, 0x60, 0x00, 0x99, 0x5f, 0x41, 0x68, 0xdd, 0xc5, 0xf8, 0x93, 0x01, 0x1b, 0xe6, 0xa0, 0xdb, 0xc9, 0xb8, 0xa1, 0xa3, 0xf5, 0xbb }, + { 0xc1, 0x1a, 0xa8, 0x1e, 0x5e, 0xfd, 0x24, 0xd5, 0xfc, 0x27, 0xee, 0x58, 0x6c, 0xfd, 0x88, 0x47, 0xfb, 0xb0, 0xe2, 0x76, 0x01, 0xcc, 0xec, 0xe5, 0xec, 0xca, 0x01, 0x98, 0xe3, 0xc7, 0x76, 0x53, 0x93, 0xbb, 0x74, 0x45, 0x7c, 0x7e, 0x7a, 0x27, 0xeb, 0x91, 0x70, 0x35, 0x0e, 0x1f, 0xb5, 0x38, 0x57, 0x17, 0x75, 0x06, 0xbe, 0x3e, 0x76, 0x2c, 0xc0, 0xf1, 0x4d, 0x8c, 0x3a, 0xfe, 0x90, 0x77 }, + { 0xc2, 0x8f, 0x21, 0x50, 0xb4, 0x52, 0xe6, 0xc0, 0xc4, 0x24, 0xbc, 0xde, 0x6f, 0x8d, 0x72, 0x00, 0x7f, 0x93, 0x10, 0xfe, 0xd7, 0xf2, 0xf8, 0x7d, 0xe0, 0xdb, 0xb6, 0x4f, 0x44, 0x79, 0xd6, 0xc1, 0x44, 0x1b, 0xa6, 0x6f, 0x44, 0xb2, 0xac, 0xce, 0xe6, 0x16, 0x09, 0x17, 0x7e, 0xd3, 0x40, 0x12, 0x8b, 0x40, 0x7e, 0xce, 0xc7, 0xc6, 0x4b, 0xbe, 0x50, 0xd6, 0x3d, 0x22, 0xd8, 0x62, 0x77, 0x27 }, + { 0xf6, 0x3d, 0x88, 0x12, 0x28, 0x77, 0xec, 0x30, 0xb8, 0xc8, 0xb0, 0x0d, 0x22, 0xe8, 0x90, 0x00, 0xa9, 0x66, 0x42, 0x61, 0x12, 0xbd, 0x44, 0x16, 0x6e, 0x2f, 0x52, 0x5b, 0x76, 0x9c, 0xcb, 0xe9, 0xb2, 0x86, 0xd4, 0x37, 0xa0, 0x12, 0x91, 0x30, 0xdd, 0xe1, 0xa8, 0x6c, 0x43, 0xe0, 0x4b, 0xed, 0xb5, 0x94, 0xe6, 0x71, 0xd9, 0x82, 0x83, 0xaf, 0xe6, 0x4c, 0xe3, 0x31, 0xde, 0x98, 0x28, 0xfd }, + { 0x34, 0x8b, 0x05, 0x32, 0x88, 0x0b, 0x88, 0xa6, 0x61, 0x4a, 0x8d, 0x74, 0x08, 0xc3, 0xf9, 0x13, 0x35, 0x7f, 0xbb, 0x60, 0xe9, 0x95, 0xc6, 0x02, 0x05, 0xbe, 0x91, 0x39, 0xe7, 0x49, 0x98, 0xae, 0xde, 0x7f, 0x45, 0x81, 0xe4, 0x2f, 0x6b, 0x52, 0x69, 0x8f, 0x7f, 0xa1, 0x21, 0x97, 0x08, 0xc1, 0x44, 0x98, 0x06, 0x7f, 0xd1, 0xe0, 0x95, 0x02, 0xde, 0x83, 0xa7, 0x7d, 0xd2, 0x81, 0x15, 0x0c }, + { 0x51, 0x33, 0xdc, 0x8b, 0xef, 0x72, 0x53, 0x59, 0xdf, 0xf5, 0x97, 0x92, 0xd8, 0x5e, 0xaf, 0x75, 0xb7, 0xe1, 0xdc, 0xd1, 0x97, 0x8b, 0x01, 0xc3, 0x5b, 0x1b, 0x85, 0xfc, 0xeb, 0xc6, 0x33, 0x88, 0xad, 0x99, 0xa1, 0x7b, 0x63, 0x46, 0xa2, 0x17, 0xdc, 0x1a, 0x96, 0x22, 0xeb, 0xd1, 0x22, 0xec, 0xf6, 0x91, 0x3c, 0x4d, 0x31, 0xa6, 0xb5, 0x2a, 0x69, 0x5b, 0x86, 0xaf, 0x00, 0xd7, 0x41, 0xa0 }, + { 0x27, 0x53, 0xc4, 0xc0, 0xe9, 0x8e, 0xca, 0xd8, 0x06, 0xe8, 0x87, 0x80, 0xec, 0x27, 0xfc, 0xcd, 0x0f, 0x5c, 0x1a, 0xb5, 0x47, 0xf9, 0xe4, 0xbf, 0x16, 0x59, 0xd1, 0x92, 0xc2, 0x3a, 0xa2, 0xcc, 0x97, 0x1b, 0x58, 0xb6, 0x80, 0x25, 0x80, 0xba, 0xef, 0x8a, 0xdc, 0x3b, 0x77, 0x6e, 0xf7, 0x08, 0x6b, 0x25, 0x45, 0xc2, 0x98, 0x7f, 0x34, 0x8e, 0xe3, 0x71, 0x9c, 0xde, 0xf2, 0x58, 0xc4, 0x03 }, + { 0xb1, 0x66, 0x35, 0x73, 0xce, 0x4b, 0x9d, 0x8c, 0xae, 0xfc, 0x86, 0x50, 0x12, 0xf3, 0xe3, 0x97, 0x14, 0xb9, 0x89, 0x8a, 0x5d, 0xa6, 0xce, 0x17, 0xc2, 0x5a, 0x6a, 0x47, 0x93, 0x1a, 0x9d, 0xdb, 0x9b, 0xbe, 0x98, 0xad, 0xaa, 0x55, 0x3b, 0xee, 0xd4, 0x36, 0xe8, 0x95, 0x78, 0x45, 0x54, 0x16, 0xc2, 0xa5, 0x2a, 0x52, 0x5c, 0xf2, 0x86, 0x2b, 0x8d, 0x1d, 0x49, 0xa2, 0x53, 0x1b, 0x73, 0x91 }, + { 0x64, 0xf5, 0x8b, 0xd6, 0xbf, 0xc8, 0x56, 0xf5, 0xe8, 0x73, 0xb2, 0xa2, 0x95, 0x6e, 0xa0, 0xed, 0xa0, 0xd6, 0xdb, 0x0d, 0xa3, 0x9c, 0x8c, 0x7f, 0xc6, 0x7c, 0x9f, 0x9f, 0xee, 0xfc, 0xff, 0x30, 0x72, 0xcd, 0xf9, 0xe6, 0xea, 0x37, 0xf6, 0x9a, 0x44, 0xf0, 0xc6, 0x1a, 0xa0, 0xda, 0x36, 0x93, 0xc2, 0xdb, 0x5b, 0x54, 0x96, 0x0c, 0x02, 0x81, 0xa0, 0x88, 0x15, 0x1d, 0xb4, 0x2b, 0x11, 0xe8 }, + { 0x07, 0x64, 0xc7, 0xbe, 0x28, 0x12, 0x5d, 0x90, 0x65, 0xc4, 0xb9, 0x8a, 0x69, 0xd6, 0x0a, 0xed, 0xe7, 0x03, 0x54, 0x7c, 0x66, 0xa1, 0x2e, 0x17, 0xe1, 0xc6, 0x18, 0x99, 0x41, 0x32, 0xf5, 0xef, 0x82, 0x48, 0x2c, 0x1e, 0x3f, 0xe3, 0x14, 0x6c, 0xc6, 0x53, 0x76, 0xcc, 0x10, 0x9f, 0x01, 0x38, 0xed, 0x9a, 0x80, 0xe4, 0x9f, 0x1f, 0x3c, 0x7d, 0x61, 0x0d, 0x2f, 0x24, 0x32, 0xf2, 0x06, 0x05 }, + { 0xf7, 0x48, 0x78, 0x43, 0x98, 0xa2, 0xff, 0x03, 0xeb, 0xeb, 0x07, 0xe1, 0x55, 0xe6, 0x61, 0x16, 0xa8, 0x39, 0x74, 0x1a, 0x33, 0x6e, 0x32, 0xda, 0x71, 0xec, 0x69, 0x60, 0x01, 0xf0, 0xad, 0x1b, 0x25, 0xcd, 0x48, 0xc6, 0x9c, 0xfc, 0xa7, 0x26, 0x5e, 0xca, 0x1d, 0xd7, 0x19, 0x04, 0xa0, 0xce, 0x74, 0x8a, 0xc4, 0x12, 0x4f, 0x35, 0x71, 0x07, 0x6d, 0xfa, 0x71, 0x16, 0xa9, 0xcf, 0x00, 0xe9 }, + { 0x3f, 0x0d, 0xbc, 0x01, 0x86, 0xbc, 0xeb, 0x6b, 0x78, 0x5b, 0xa7, 0x8d, 0x2a, 0x2a, 0x01, 0x3c, 0x91, 0x0b, 0xe1, 0x57, 0xbd, 0xaf, 0xfa, 0xe8, 0x1b, 0xb6, 0x66, 0x3b, 0x1a, 0x73, 0x72, 0x2f, 0x7f, 0x12, 0x28, 0x79, 0x5f, 0x3e, 0xca, 0xda, 0x87, 0xcf, 0x6e, 0xf0, 0x07, 0x84, 0x74, 0xaf, 0x73, 0xf3, 0x1e, 0xca, 0x0c, 0xc2, 0x00, 0xed, 0x97, 0x5b, 0x68, 0x93, 0xf7, 0x61, 0xcb, 0x6d }, + { 0xd4, 0x76, 0x2c, 0xd4, 0x59, 0x98, 0x76, 0xca, 0x75, 0xb2, 0xb8, 0xfe, 0x24, 0x99, 0x44, 0xdb, 0xd2, 0x7a, 0xce, 0x74, 0x1f, 0xda, 0xb9, 0x36, 0x16, 0xcb, 0xc6, 0xe4, 0x25, 0x46, 0x0f, 0xeb, 0x51, 0xd4, 0xe7, 0xad, 0xcc, 0x38, 0x18, 0x0e, 0x7f, 0xc4, 0x7c, 0x89, 0x02, 0x4a, 0x7f, 0x56, 0x19, 0x1a, 0xdb, 0x87, 0x8d, 0xfd, 0xe4, 0xea, 0xd6, 0x22, 0x23, 0xf5, 0xa2, 0x61, 0x0e, 0xfe }, + { 0xcd, 0x36, 0xb3, 0xd5, 0xb4, 0xc9, 0x1b, 0x90, 0xfc, 0xbb, 0xa7, 0x95, 0x13, 0xcf, 0xee, 0x19, 0x07, 0xd8, 0x64, 0x5a, 0x16, 0x2a, 0xfd, 0x0c, 0xd4, 0xcf, 0x41, 0x92, 0xd4, 0xa5, 0xf4, 0xc8, 0x92, 0x18, 0x3a, 0x8e, 0xac, 0xdb, 0x2b, 0x6b, 0x6a, 0x9d, 0x9a, 0xa8, 0xc1, 0x1a, 0xc1, 0xb2, 0x61, 0xb3, 0x80, 0xdb, 0xee, 0x24, 0xca, 0x46, 0x8f, 0x1b, 0xfd, 0x04, 0x3c, 0x58, 0xee, 0xfe }, + { 0x98, 0x59, 0x34, 0x52, 0x28, 0x16, 0x61, 0xa5, 0x3c, 0x48, 0xa9, 0xd8, 0xcd, 0x79, 0x08, 0x26, 0xc1, 0xa1, 0xce, 0x56, 0x77, 0x38, 0x05, 0x3d, 0x0b, 0xee, 0x4a, 0x91, 0xa3, 0xd5, 0xbd, 0x92, 0xee, 0xfd, 0xba, 0xbe, 0xbe, 0x32, 0x04, 0xf2, 0x03, 0x1c, 0xa5, 0xf7, 0x81, 0xbd, 0xa9, 0x9e, 0xf5, 0xd8, 0xae, 0x56, 0xe5, 0xb0, 0x4a, 0x9e, 0x1e, 0xcd, 0x21, 0xb0, 0xeb, 0x05, 0xd3, 0xe1 }, + { 0x77, 0x1f, 0x57, 0xdd, 0x27, 0x75, 0xcc, 0xda, 0xb5, 0x59, 0x21, 0xd3, 0xe8, 0xe3, 0x0c, 0xcf, 0x48, 0x4d, 0x61, 0xfe, 0x1c, 0x1b, 0x9c, 0x2a, 0xe8, 0x19, 0xd0, 0xfb, 0x2a, 0x12, 0xfa, 0xb9, 0xbe, 0x70, 0xc4, 0xa7, 0xa1, 0x38, 0xda, 0x84, 0xe8, 0x28, 0x04, 0x35, 0xda, 0xad, 0xe5, 0xbb, 0xe6, 0x6a, 0xf0, 0x83, 0x6a, 0x15, 0x4f, 0x81, 0x7f, 0xb1, 0x7f, 0x33, 0x97, 0xe7, 0x25, 0xa3 }, + { 0xc6, 0x08, 0x97, 0xc6, 0xf8, 0x28, 0xe2, 0x1f, 0x16, 0xfb, 0xb5, 0xf1, 0x5b, 0x32, 0x3f, 0x87, 0xb6, 0xc8, 0x95, 0x5e, 0xab, 0xf1, 0xd3, 0x80, 0x61, 0xf7, 0x07, 0xf6, 0x08, 0xab, 0xdd, 0x99, 0x3f, 0xac, 0x30, 0x70, 0x63, 0x3e, 0x28, 0x6c, 0xf8, 0x33, 0x9c, 0xe2, 0x95, 0xdd, 0x35, 0x2d, 0xf4, 0xb4, 0xb4, 0x0b, 0x2f, 0x29, 0xda, 0x1d, 0xd5, 0x0b, 0x3a, 0x05, 0xd0, 0x79, 0xe6, 0xbb }, + { 0x82, 0x10, 0xcd, 0x2c, 0x2d, 0x3b, 0x13, 0x5c, 0x2c, 0xf0, 0x7f, 0xa0, 0xd1, 0x43, 0x3c, 0xd7, 0x71, 0xf3, 0x25, 0xd0, 0x75, 0xc6, 0x46, 0x9d, 0x9c, 0x7f, 0x1b, 0xa0, 0x94, 0x3c, 0xd4, 0xab, 0x09, 0x80, 0x8c, 0xab, 0xf4, 0xac, 0xb9, 0xce, 0x5b, 0xb8, 0x8b, 0x49, 0x89, 0x29, 0xb4, 0xb8, 0x47, 0xf6, 0x81, 0xad, 0x2c, 0x49, 0x0d, 0x04, 0x2d, 0xb2, 0xae, 0xc9, 0x42, 0x14, 0xb0, 0x6b }, + { 0x1d, 0x4e, 0xdf, 0xff, 0xd8, 0xfd, 0x80, 0xf7, 0xe4, 0x10, 0x78, 0x40, 0xfa, 0x3a, 0xa3, 0x1e, 0x32, 0x59, 0x84, 0x91, 0xe4, 0xaf, 0x70, 0x13, 0xc1, 0x97, 0xa6, 0x5b, 0x7f, 0x36, 0xdd, 0x3a, 0xc4, 0xb4, 0x78, 0x45, 0x61, 0x11, 0xcd, 0x43, 0x09, 0xd9, 0x24, 0x35, 0x10, 0x78, 0x2f, 0xa3, 0x1b, 0x7c, 0x4c, 0x95, 0xfa, 0x95, 0x15, 0x20, 0xd0, 0x20, 0xeb, 0x7e, 0x5c, 0x36, 0xe4, 0xef }, + { 0xaf, 0x8e, 0x6e, 0x91, 0xfa, 0xb4, 0x6c, 0xe4, 0x87, 0x3e, 0x1a, 0x50, 0xa8, 0xef, 0x44, 0x8c, 0xc2, 0x91, 0x21, 0xf7, 0xf7, 0x4d, 0xee, 0xf3, 0x4a, 0x71, 0xef, 0x89, 0xcc, 0x00, 0xd9, 0x27, 0x4b, 0xc6, 0xc2, 0x45, 0x4b, 0xbb, 0x32, 0x30, 0xd8, 0xb2, 0xec, 0x94, 0xc6, 0x2b, 0x1d, 0xec, 0x85, 0xf3, 0x59, 0x3b, 0xfa, 0x30, 0xea, 0x6f, 0x7a, 0x44, 0xd7, 0xc0, 0x94, 0x65, 0xa2, 0x53 }, + { 0x29, 0xfd, 0x38, 0x4e, 0xd4, 0x90, 0x6f, 0x2d, 0x13, 0xaa, 0x9f, 0xe7, 0xaf, 0x90, 0x59, 0x90, 0x93, 0x8b, 0xed, 0x80, 0x7f, 0x18, 0x32, 0x45, 0x4a, 0x37, 0x2a, 0xb4, 0x12, 0xee, 0xa1, 0xf5, 0x62, 0x5a, 0x1f, 0xcc, 0x9a, 0xc8, 0x34, 0x3b, 0x7c, 0x67, 0xc5, 0xab, 0xa6, 0xe0, 0xb1, 0xcc, 0x46, 0x44, 0x65, 0x49, 0x13, 0x69, 0x2c, 0x6b, 0x39, 0xeb, 0x91, 0x87, 0xce, 0xac, 0xd3, 0xec }, + { 0xa2, 0x68, 0xc7, 0x88, 0x5d, 0x98, 0x74, 0xa5, 0x1c, 0x44, 0xdf, 0xfe, 0xd8, 0xea, 0x53, 0xe9, 0x4f, 0x78, 0x45, 0x6e, 0x0b, 0x2e, 0xd9, 0x9f, 0xf5, 0xa3, 0x92, 0x47, 0x60, 0x81, 0x38, 0x26, 0xd9, 0x60, 0xa1, 0x5e, 0xdb, 0xed, 0xbb, 0x5d, 0xe5, 0x22, 0x6b, 0xa4, 0xb0, 0x74, 0xe7, 0x1b, 0x05, 0xc5, 0x5b, 0x97, 0x56, 0xbb, 0x79, 0xe5, 0x5c, 0x02, 0x75, 0x4c, 0x2c, 0x7b, 0x6c, 0x8a }, + { 0x0c, 0xf8, 0x54, 0x54, 0x88, 0xd5, 0x6a, 0x86, 0x81, 0x7c, 0xd7, 0xec, 0xb1, 0x0f, 0x71, 0x16, 0xb7, 0xea, 0x53, 0x0a, 0x45, 0xb6, 0xea, 0x49, 0x7b, 0x6c, 0x72, 0xc9, 0x97, 0xe0, 0x9e, 0x3d, 0x0d, 0xa8, 0x69, 0x8f, 0x46, 0xbb, 0x00, 0x6f, 0xc9, 0x77, 0xc2, 0xcd, 0x3d, 0x11, 0x77, 0x46, 0x3a, 0xc9, 0x05, 0x7f, 0xdd, 0x16, 0x62, 0xc8, 0x5d, 0x0c, 0x12, 0x64, 0x43, 0xc1, 0x04, 0x73 }, + { 0xb3, 0x96, 0x14, 0x26, 0x8f, 0xdd, 0x87, 0x81, 0x51, 0x5e, 0x2c, 0xfe, 0xbf, 0x89, 0xb4, 0xd5, 0x40, 0x2b, 0xab, 0x10, 0xc2, 0x26, 0xe6, 0x34, 0x4e, 0x6b, 0x9a, 0xe0, 0x00, 0xfb, 0x0d, 0x6c, 0x79, 0xcb, 0x2f, 0x3e, 0xc8, 0x0e, 0x80, 0xea, 0xeb, 0x19, 0x80, 0xd2, 0xf8, 0x69, 0x89, 0x16, 0xbd, 0x2e, 0x9f, 0x74, 0x72, 0x36, 0x65, 0x51, 0x16, 0x64, 0x9c, 0xd3, 0xca, 0x23, 0xa8, 0x37 }, + { 0x74, 0xbe, 0xf0, 0x92, 0xfc, 0x6f, 0x1e, 0x5d, 0xba, 0x36, 0x63, 0xa3, 0xfb, 0x00, 0x3b, 0x2a, 0x5b, 0xa2, 0x57, 0x49, 0x65, 0x36, 0xd9, 0x9f, 0x62, 0xb9, 0xd7, 0x3f, 0x8f, 0x9e, 0xb3, 0xce, 0x9f, 0xf3, 0xee, 0xc7, 0x09, 0xeb, 0x88, 0x36, 0x55, 0xec, 0x9e, 0xb8, 0x96, 0xb9, 0x12, 0x8f, 0x2a, 0xfc, 0x89, 0xcf, 0x7d, 0x1a, 0xb5, 0x8a, 0x72, 0xf4, 0xa3, 0xbf, 0x03, 0x4d, 0x2b, 0x4a }, + { 0x3a, 0x98, 0x8d, 0x38, 0xd7, 0x56, 0x11, 0xf3, 0xef, 0x38, 0xb8, 0x77, 0x49, 0x80, 0xb3, 0x3e, 0x57, 0x3b, 0x6c, 0x57, 0xbe, 0xe0, 0x46, 0x9b, 0xa5, 0xee, 0xd9, 0xb4, 0x4f, 0x29, 0x94, 0x5e, 0x73, 0x47, 0x96, 0x7f, 0xba, 0x2c, 0x16, 0x2e, 0x1c, 0x3b, 0xe7, 0xf3, 0x10, 0xf2, 0xf7, 0x5e, 0xe2, 0x38, 0x1e, 0x7b, 0xfd, 0x6b, 0x3f, 0x0b, 0xae, 0xa8, 0xd9, 0x5d, 0xfb, 0x1d, 0xaf, 0xb1 }, + { 0x58, 0xae, 0xdf, 0xce, 0x6f, 0x67, 0xdd, 0xc8, 0x5a, 0x28, 0xc9, 0x92, 0xf1, 0xc0, 0xbd, 0x09, 0x69, 0xf0, 0x41, 0xe6, 0x6f, 0x1e, 0xe8, 0x80, 0x20, 0xa1, 0x25, 0xcb, 0xfc, 0xfe, 0xbc, 0xd6, 0x17, 0x09, 0xc9, 0xc4, 0xeb, 0xa1, 0x92, 0xc1, 0x5e, 0x69, 0xf0, 0x20, 0xd4, 0x62, 0x48, 0x60, 0x19, 0xfa, 0x8d, 0xea, 0x0c, 0xd7, 0xa4, 0x29, 0x21, 0xa1, 0x9d, 0x2f, 0xe5, 0x46, 0xd4, 0x3d }, + { 0x93, 0x47, 0xbd, 0x29, 0x14, 0x73, 0xe6, 0xb4, 0xe3, 0x68, 0x43, 0x7b, 0x8e, 0x56, 0x1e, 0x06, 0x5f, 0x64, 0x9a, 0x6d, 0x8a, 0xda, 0x47, 0x9a, 0xd0, 0x9b, 0x19, 0x99, 0xa8, 0xf2, 0x6b, 0x91, 0xcf, 0x61, 0x20, 0xfd, 0x3b, 0xfe, 0x01, 0x4e, 0x83, 0xf2, 0x3a, 0xcf, 0xa4, 0xc0, 0xad, 0x7b, 0x37, 0x12, 0xb2, 0xc3, 0xc0, 0x73, 0x32, 0x70, 0x66, 0x31, 0x12, 0xcc, 0xd9, 0x28, 0x5c, 0xd9 }, + { 0xb3, 0x21, 0x63, 0xe7, 0xc5, 0xdb, 0xb5, 0xf5, 0x1f, 0xdc, 0x11, 0xd2, 0xea, 0xc8, 0x75, 0xef, 0xbb, 0xcb, 0x7e, 0x76, 0x99, 0x09, 0x0a, 0x7e, 0x7f, 0xf8, 0xa8, 0xd5, 0x07, 0x95, 0xaf, 0x5d, 0x74, 0xd9, 0xff, 0x98, 0x54, 0x3e, 0xf8, 0xcd, 0xf8, 0x9a, 0xc1, 0x3d, 0x04, 0x85, 0x27, 0x87, 0x56, 0xe0, 0xef, 0x00, 0xc8, 0x17, 0x74, 0x56, 0x61, 0xe1, 0xd5, 0x9f, 0xe3, 0x8e, 0x75, 0x37 }, + { 0x10, 0x85, 0xd7, 0x83, 0x07, 0xb1, 0xc4, 0xb0, 0x08, 0xc5, 0x7a, 0x2e, 0x7e, 0x5b, 0x23, 0x46, 0x58, 0xa0, 0xa8, 0x2e, 0x4f, 0xf1, 0xe4, 0xaa, 0xac, 0x72, 0xb3, 0x12, 0xfd, 0xa0, 0xfe, 0x27, 0xd2, 0x33, 0xbc, 0x5b, 0x10, 0xe9, 0xcc, 0x17, 0xfd, 0xc7, 0x69, 0x7b, 0x54, 0x0c, 0x7d, 0x95, 0xeb, 0x21, 0x5a, 0x19, 0xa1, 0xa0, 0xe2, 0x0e, 0x1a, 0xbf, 0xa1, 0x26, 0xef, 0xd5, 0x68, 0xc7 }, + { 0x4e, 0x5c, 0x73, 0x4c, 0x7d, 0xde, 0x01, 0x1d, 0x83, 0xea, 0xc2, 0xb7, 0x34, 0x7b, 0x37, 0x35, 0x94, 0xf9, 0x2d, 0x70, 0x91, 0xb9, 0xca, 0x34, 0xcb, 0x9c, 0x6f, 0x39, 0xbd, 0xf5, 0xa8, 0xd2, 0xf1, 0x34, 0x37, 0x9e, 0x16, 0xd8, 0x22, 0xf6, 0x52, 0x21, 0x70, 0xcc, 0xf2, 0xdd, 0xd5, 0x5c, 0x84, 0xb9, 0xe6, 0xc6, 0x4f, 0xc9, 0x27, 0xac, 0x4c, 0xf8, 0xdf, 0xb2, 0xa1, 0x77, 0x01, 0xf2 }, + { 0x69, 0x5d, 0x83, 0xbd, 0x99, 0x0a, 0x11, 0x17, 0xb3, 0xd0, 0xce, 0x06, 0xcc, 0x88, 0x80, 0x27, 0xd1, 0x2a, 0x05, 0x4c, 0x26, 0x77, 0xfd, 0x82, 0xf0, 0xd4, 0xfb, 0xfc, 0x93, 0x57, 0x55, 0x23, 0xe7, 0x99, 0x1a, 0x5e, 0x35, 0xa3, 0x75, 0x2e, 0x9b, 0x70, 0xce, 0x62, 0x99, 0x2e, 0x26, 0x8a, 0x87, 0x77, 0x44, 0xcd, 0xd4, 0x35, 0xf5, 0xf1, 0x30, 0x86, 0x9c, 0x9a, 0x20, 0x74, 0xb3, 0x38 }, + { 0xa6, 0x21, 0x37, 0x43, 0x56, 0x8e, 0x3b, 0x31, 0x58, 0xb9, 0x18, 0x43, 0x01, 0xf3, 0x69, 0x08, 0x47, 0x55, 0x4c, 0x68, 0x45, 0x7c, 0xb4, 0x0f, 0xc9, 0xa4, 0xb8, 0xcf, 0xd8, 0xd4, 0xa1, 0x18, 0xc3, 0x01, 0xa0, 0x77, 0x37, 0xae, 0xda, 0x0f, 0x92, 0x9c, 0x68, 0x91, 0x3c, 0x5f, 0x51, 0xc8, 0x03, 0x94, 0xf5, 0x3b, 0xff, 0x1c, 0x3e, 0x83, 0xb2, 0xe4, 0x0c, 0xa9, 0x7e, 0xba, 0x9e, 0x15 }, + { 0xd4, 0x44, 0xbf, 0xa2, 0x36, 0x2a, 0x96, 0xdf, 0x21, 0x3d, 0x07, 0x0e, 0x33, 0xfa, 0x84, 0x1f, 0x51, 0x33, 0x4e, 0x4e, 0x76, 0x86, 0x6b, 0x81, 0x39, 0xe8, 0xaf, 0x3b, 0xb3, 0x39, 0x8b, 0xe2, 0xdf, 0xad, 0xdc, 0xbc, 0x56, 0xb9, 0x14, 0x6d, 0xe9, 0xf6, 0x81, 0x18, 0xdc, 0x58, 0x29, 0xe7, 0x4b, 0x0c, 0x28, 0xd7, 0x71, 0x19, 0x07, 0xb1, 0x21, 0xf9, 0x16, 0x1c, 0xb9, 0x2b, 0x69, 0xa9 }, + { 0x14, 0x27, 0x09, 0xd6, 0x2e, 0x28, 0xfc, 0xcc, 0xd0, 0xaf, 0x97, 0xfa, 0xd0, 0xf8, 0x46, 0x5b, 0x97, 0x1e, 0x82, 0x20, 0x1d, 0xc5, 0x10, 0x70, 0xfa, 0xa0, 0x37, 0x2a, 0xa4, 0x3e, 0x92, 0x48, 0x4b, 0xe1, 0xc1, 0xe7, 0x3b, 0xa1, 0x09, 0x06, 0xd5, 0xd1, 0x85, 0x3d, 0xb6, 0xa4, 0x10, 0x6e, 0x0a, 0x7b, 0xf9, 0x80, 0x0d, 0x37, 0x3d, 0x6d, 0xee, 0x2d, 0x46, 0xd6, 0x2e, 0xf2, 0xa4, 0x61 }, + }; + unsigned char inp[1000], out[1000]; + unsigned char key[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f }; + unsigned long ilen, klen = sizeof(key), mlen = 64; + blake2bmac_state st; + + for (ilen = 0; ilen < 256; ilen++) inp[ilen] = (unsigned char)ilen; + + for (ilen = 0; ilen < 256; ilen++) { + const unsigned char *mac = tests[ilen]; + unsigned long olen = mlen; + /* process piece by piece */ + if (ilen > 15) { + blake2bmac_init(&st, olen, key, klen); + blake2bmac_process(&st, (unsigned char*)inp, 5); + blake2bmac_process(&st, (unsigned char*)inp + 5, 4); + blake2bmac_process(&st, (unsigned char*)inp + 9, 3); + blake2bmac_process(&st, (unsigned char*)inp + 12, 2); + blake2bmac_process(&st, (unsigned char*)inp + 14, 1); + blake2bmac_process(&st, (unsigned char*)inp + 15, ilen - 15); + blake2bmac_done(&st, out, &olen); + if (compare_testvector(out, olen, mac, mlen, "BLAKE2B MAC multi", ilen) != 0) return CRYPT_FAIL_TESTVECTOR; + } + /* process in one go */ + blake2bmac_init(&st, olen, key, klen); + blake2bmac_process(&st, (unsigned char*)inp, ilen); + blake2bmac_done(&st, out, &olen); + if (compare_testvector(out, olen, mac, mlen, "BLAKE2B MAC single", ilen) != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac.c new file mode 100644 index 0000000..f42c7ae --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac.c @@ -0,0 +1,56 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2SMAC + +/** + Initialize an BLAKE2S MAC context. + @param st The BLAKE2S MAC state + @param outlen The size of the MAC output (octets) + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int blake2smac_init(blake2smac_state *st, unsigned long outlen, const unsigned char *key, unsigned long keylen) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + return blake2s_init(st, outlen, key, keylen); +} + +/** + Process data through BLAKE2S MAC + @param st The BLAKE2S MAC state + @param in The data to send through HMAC + @param inlen The length of the data to HMAC (octets) + @return CRYPT_OK if successful +*/ +int blake2smac_process(blake2smac_state *st, const unsigned char *in, unsigned long inlen) +{ + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + return blake2s_process(st, in, inlen); +} + +/** + Terminate a BLAKE2S MAC session + @param st The BLAKE2S MAC state + @param mac [out] The destination of the BLAKE2S MAC authentication tag + @param maclen [in/out] The max size and resulting size of the BLAKE2S MAC authentication tag + @return CRYPT_OK if successful +*/ +int blake2smac_done(blake2smac_state *st, unsigned char *mac, unsigned long *maclen) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + LTC_ARGCHK(*maclen >= st->blake2s.outlen); + + *maclen = st->blake2s.outlen; + return blake2s_done(st, mac); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_file.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_file.c new file mode 100644 index 0000000..bcfbe06 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_file.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2SMAC + +/** + BLAKE2S MAC a file + @param fname The name of the file you wish to BLAKE2S MAC + @param key The secret key + @param keylen The length of the secret key + @param mac [out] The BLAKE2S MAC authentication tag + @param maclen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int blake2smac_file(const char *fname, const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(fname); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(mac); + LTC_UNUSED_PARAM(maclen); + return CRYPT_NOP; +#else + blake2smac_state st; + FILE *in; + unsigned char *buf; + size_t x; + int err; + + LTC_ARGCHK(fname != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = blake2smac_init(&st, *maclen, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(fname, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = blake2smac_process(&st, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = blake2smac_done(&st, mac, maclen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(blake2smac_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_memory.c new file mode 100644 index 0000000..755267b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_memory.c @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2SMAC + +/** + BLAKE2S MAC a block of memory to produce the authentication tag + @param key The secret key + @param keylen The length of the secret key (octets) + @param in The data to BLAKE2S MAC + @param inlen The length of the data to BLAKE2S MAC (octets) + @param mac [out] Destination of the authentication tag + @param maclen [in/out] Max size and resulting size of authentication tag + @return CRYPT_OK if successful +*/ +int blake2smac_memory(const unsigned char *key, unsigned long keylen, const unsigned char *in, unsigned long inlen, unsigned char *mac, unsigned long *maclen) +{ + blake2smac_state st; + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + if ((err = blake2smac_init(&st, *maclen, key, keylen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = blake2smac_process(&st, in, inlen)) != CRYPT_OK) { goto LBL_ERR; } + err = blake2smac_done(&st, mac, maclen); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(blake2smac_state)); +#endif + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_memory_multi.c new file mode 100644 index 0000000..34fb5e1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_memory_multi.c @@ -0,0 +1,52 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" +#include + +#ifdef LTC_BLAKE2SMAC + +/** + BLAKE2S MAC multiple blocks of memory to produce the authentication tag + @param key The secret key + @param keylen The length of the secret key (octets) + @param mac [out] Destination of the authentication tag + @param maclen [in/out] Max size and resulting size of authentication tag + @param in The data to BLAKE2S MAC + @param inlen The length of the data to BLAKE2S MAC (octets) + @param ... tuples of (data,len) pairs to BLAKE2S MAC, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int blake2smac_memory_multi(const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen, const unsigned char *in, unsigned long inlen, ...) +{ + blake2smac_state st; + int err; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + va_start(args, inlen); + curptr = in; + curlen = inlen; + if ((err = blake2smac_init(&st, *maclen, key, keylen)) != CRYPT_OK) { goto LBL_ERR; } + for (;;) { + if ((err = blake2smac_process(&st, curptr, curlen)) != CRYPT_OK) { goto LBL_ERR; } + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) break; + curlen = va_arg(args, unsigned long); + } + err = blake2smac_done(&st, mac, maclen); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(blake2smac_state)); +#endif + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_test.c b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_test.c new file mode 100644 index 0000000..acfb030 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/blake2/blake2smac_test.c @@ -0,0 +1,304 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BLAKE2SMAC + +int blake2smac_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const unsigned char tests[256][32] = { + /* source: https://github.com/BLAKE2/BLAKE2/blob/master/testvectors/blake2s-kat.txt */ + { 0x48, 0xa8, 0x99, 0x7d, 0xa4, 0x07, 0x87, 0x6b, 0x3d, 0x79, 0xc0, 0xd9, 0x23, 0x25, 0xad, 0x3b, 0x89, 0xcb, 0xb7, 0x54, 0xd8, 0x6a, 0xb7, 0x1a, 0xee, 0x04, 0x7a, 0xd3, 0x45, 0xfd, 0x2c, 0x49 }, + { 0x40, 0xd1, 0x5f, 0xee, 0x7c, 0x32, 0x88, 0x30, 0x16, 0x6a, 0xc3, 0xf9, 0x18, 0x65, 0x0f, 0x80, 0x7e, 0x7e, 0x01, 0xe1, 0x77, 0x25, 0x8c, 0xdc, 0x0a, 0x39, 0xb1, 0x1f, 0x59, 0x80, 0x66, 0xf1 }, + { 0x6b, 0xb7, 0x13, 0x00, 0x64, 0x4c, 0xd3, 0x99, 0x1b, 0x26, 0xcc, 0xd4, 0xd2, 0x74, 0xac, 0xd1, 0xad, 0xea, 0xb8, 0xb1, 0xd7, 0x91, 0x45, 0x46, 0xc1, 0x19, 0x8b, 0xbe, 0x9f, 0xc9, 0xd8, 0x03 }, + { 0x1d, 0x22, 0x0d, 0xbe, 0x2e, 0xe1, 0x34, 0x66, 0x1f, 0xdf, 0x6d, 0x9e, 0x74, 0xb4, 0x17, 0x04, 0x71, 0x05, 0x56, 0xf2, 0xf6, 0xe5, 0xa0, 0x91, 0xb2, 0x27, 0x69, 0x74, 0x45, 0xdb, 0xea, 0x6b }, + { 0xf6, 0xc3, 0xfb, 0xad, 0xb4, 0xcc, 0x68, 0x7a, 0x00, 0x64, 0xa5, 0xbe, 0x6e, 0x79, 0x1b, 0xec, 0x63, 0xb8, 0x68, 0xad, 0x62, 0xfb, 0xa6, 0x1b, 0x37, 0x57, 0xef, 0x9c, 0xa5, 0x2e, 0x05, 0xb2 }, + { 0x49, 0xc1, 0xf2, 0x11, 0x88, 0xdf, 0xd7, 0x69, 0xae, 0xa0, 0xe9, 0x11, 0xdd, 0x6b, 0x41, 0xf1, 0x4d, 0xab, 0x10, 0x9d, 0x2b, 0x85, 0x97, 0x7a, 0xa3, 0x08, 0x8b, 0x5c, 0x70, 0x7e, 0x85, 0x98 }, + { 0xfd, 0xd8, 0x99, 0x3d, 0xcd, 0x43, 0xf6, 0x96, 0xd4, 0x4f, 0x3c, 0xea, 0x0f, 0xf3, 0x53, 0x45, 0x23, 0x4e, 0xc8, 0xee, 0x08, 0x3e, 0xb3, 0xca, 0xda, 0x01, 0x7c, 0x7f, 0x78, 0xc1, 0x71, 0x43 }, + { 0xe6, 0xc8, 0x12, 0x56, 0x37, 0x43, 0x8d, 0x09, 0x05, 0xb7, 0x49, 0xf4, 0x65, 0x60, 0xac, 0x89, 0xfd, 0x47, 0x1c, 0xf8, 0x69, 0x2e, 0x28, 0xfa, 0xb9, 0x82, 0xf7, 0x3f, 0x01, 0x9b, 0x83, 0xa9 }, + { 0x19, 0xfc, 0x8c, 0xa6, 0x97, 0x9d, 0x60, 0xe6, 0xed, 0xd3, 0xb4, 0x54, 0x1e, 0x2f, 0x96, 0x7c, 0xed, 0x74, 0x0d, 0xf6, 0xec, 0x1e, 0xae, 0xbb, 0xfe, 0x81, 0x38, 0x32, 0xe9, 0x6b, 0x29, 0x74 }, + { 0xa6, 0xad, 0x77, 0x7c, 0xe8, 0x81, 0xb5, 0x2b, 0xb5, 0xa4, 0x42, 0x1a, 0xb6, 0xcd, 0xd2, 0xdf, 0xba, 0x13, 0xe9, 0x63, 0x65, 0x2d, 0x4d, 0x6d, 0x12, 0x2a, 0xee, 0x46, 0x54, 0x8c, 0x14, 0xa7 }, + { 0xf5, 0xc4, 0xb2, 0xba, 0x1a, 0x00, 0x78, 0x1b, 0x13, 0xab, 0xa0, 0x42, 0x52, 0x42, 0xc6, 0x9c, 0xb1, 0x55, 0x2f, 0x3f, 0x71, 0xa9, 0xa3, 0xbb, 0x22, 0xb4, 0xa6, 0xb4, 0x27, 0x7b, 0x46, 0xdd }, + { 0xe3, 0x3c, 0x4c, 0x9b, 0xd0, 0xcc, 0x7e, 0x45, 0xc8, 0x0e, 0x65, 0xc7, 0x7f, 0xa5, 0x99, 0x7f, 0xec, 0x70, 0x02, 0x73, 0x85, 0x41, 0x50, 0x9e, 0x68, 0xa9, 0x42, 0x38, 0x91, 0xe8, 0x22, 0xa3 }, + { 0xfb, 0xa1, 0x61, 0x69, 0xb2, 0xc3, 0xee, 0x10, 0x5b, 0xe6, 0xe1, 0xe6, 0x50, 0xe5, 0xcb, 0xf4, 0x07, 0x46, 0xb6, 0x75, 0x3d, 0x03, 0x6a, 0xb5, 0x51, 0x79, 0x01, 0x4a, 0xd7, 0xef, 0x66, 0x51 }, + { 0xf5, 0xc4, 0xbe, 0xc6, 0xd6, 0x2f, 0xc6, 0x08, 0xbf, 0x41, 0xcc, 0x11, 0x5f, 0x16, 0xd6, 0x1c, 0x7e, 0xfd, 0x3f, 0xf6, 0xc6, 0x56, 0x92, 0xbb, 0xe0, 0xaf, 0xff, 0xb1, 0xfe, 0xde, 0x74, 0x75 }, + { 0xa4, 0x86, 0x2e, 0x76, 0xdb, 0x84, 0x7f, 0x05, 0xba, 0x17, 0xed, 0xe5, 0xda, 0x4e, 0x7f, 0x91, 0xb5, 0x92, 0x5c, 0xf1, 0xad, 0x4b, 0xa1, 0x27, 0x32, 0xc3, 0x99, 0x57, 0x42, 0xa5, 0xcd, 0x6e }, + { 0x65, 0xf4, 0xb8, 0x60, 0xcd, 0x15, 0xb3, 0x8e, 0xf8, 0x14, 0xa1, 0xa8, 0x04, 0x31, 0x4a, 0x55, 0xbe, 0x95, 0x3c, 0xaa, 0x65, 0xfd, 0x75, 0x8a, 0xd9, 0x89, 0xff, 0x34, 0xa4, 0x1c, 0x1e, 0xea }, + { 0x19, 0xba, 0x23, 0x4f, 0x0a, 0x4f, 0x38, 0x63, 0x7d, 0x18, 0x39, 0xf9, 0xd9, 0xf7, 0x6a, 0xd9, 0x1c, 0x85, 0x22, 0x30, 0x71, 0x43, 0xc9, 0x7d, 0x5f, 0x93, 0xf6, 0x92, 0x74, 0xce, 0xc9, 0xa7 }, + { 0x1a, 0x67, 0x18, 0x6c, 0xa4, 0xa5, 0xcb, 0x8e, 0x65, 0xfc, 0xa0, 0xe2, 0xec, 0xbc, 0x5d, 0xdc, 0x14, 0xae, 0x38, 0x1b, 0xb8, 0xbf, 0xfe, 0xb9, 0xe0, 0xa1, 0x03, 0x44, 0x9e, 0x3e, 0xf0, 0x3c }, + { 0xaf, 0xbe, 0xa3, 0x17, 0xb5, 0xa2, 0xe8, 0x9c, 0x0b, 0xd9, 0x0c, 0xcf, 0x5d, 0x7f, 0xd0, 0xed, 0x57, 0xfe, 0x58, 0x5e, 0x4b, 0xe3, 0x27, 0x1b, 0x0a, 0x6b, 0xf0, 0xf5, 0x78, 0x6b, 0x0f, 0x26 }, + { 0xf1, 0xb0, 0x15, 0x58, 0xce, 0x54, 0x12, 0x62, 0xf5, 0xec, 0x34, 0x29, 0x9d, 0x6f, 0xb4, 0x09, 0x00, 0x09, 0xe3, 0x43, 0x4b, 0xe2, 0xf4, 0x91, 0x05, 0xcf, 0x46, 0xaf, 0x4d, 0x2d, 0x41, 0x24 }, + { 0x13, 0xa0, 0xa0, 0xc8, 0x63, 0x35, 0x63, 0x5e, 0xaa, 0x74, 0xca, 0x2d, 0x5d, 0x48, 0x8c, 0x79, 0x7b, 0xbb, 0x4f, 0x47, 0xdc, 0x07, 0x10, 0x50, 0x15, 0xed, 0x6a, 0x1f, 0x33, 0x09, 0xef, 0xce }, + { 0x15, 0x80, 0xaf, 0xee, 0xbe, 0xbb, 0x34, 0x6f, 0x94, 0xd5, 0x9f, 0xe6, 0x2d, 0xa0, 0xb7, 0x92, 0x37, 0xea, 0xd7, 0xb1, 0x49, 0x1f, 0x56, 0x67, 0xa9, 0x0e, 0x45, 0xed, 0xf6, 0xca, 0x8b, 0x03 }, + { 0x20, 0xbe, 0x1a, 0x87, 0x5b, 0x38, 0xc5, 0x73, 0xdd, 0x7f, 0xaa, 0xa0, 0xde, 0x48, 0x9d, 0x65, 0x5c, 0x11, 0xef, 0xb6, 0xa5, 0x52, 0x69, 0x8e, 0x07, 0xa2, 0xd3, 0x31, 0xb5, 0xf6, 0x55, 0xc3 }, + { 0xbe, 0x1f, 0xe3, 0xc4, 0xc0, 0x40, 0x18, 0xc5, 0x4c, 0x4a, 0x0f, 0x6b, 0x9a, 0x2e, 0xd3, 0xc5, 0x3a, 0xbe, 0x3a, 0x9f, 0x76, 0xb4, 0xd2, 0x6d, 0xe5, 0x6f, 0xc9, 0xae, 0x95, 0x05, 0x9a, 0x99 }, + { 0xe3, 0xe3, 0xac, 0xe5, 0x37, 0xeb, 0x3e, 0xdd, 0x84, 0x63, 0xd9, 0xad, 0x35, 0x82, 0xe1, 0x3c, 0xf8, 0x65, 0x33, 0xff, 0xde, 0x43, 0xd6, 0x68, 0xdd, 0x2e, 0x93, 0xbb, 0xdb, 0xd7, 0x19, 0x5a }, + { 0x11, 0x0c, 0x50, 0xc0, 0xbf, 0x2c, 0x6e, 0x7a, 0xeb, 0x7e, 0x43, 0x5d, 0x92, 0xd1, 0x32, 0xab, 0x66, 0x55, 0x16, 0x8e, 0x78, 0xa2, 0xde, 0xcd, 0xec, 0x33, 0x30, 0x77, 0x76, 0x84, 0xd9, 0xc1 }, + { 0xe9, 0xba, 0x8f, 0x50, 0x5c, 0x9c, 0x80, 0xc0, 0x86, 0x66, 0xa7, 0x01, 0xf3, 0x36, 0x7e, 0x6c, 0xc6, 0x65, 0xf3, 0x4b, 0x22, 0xe7, 0x3c, 0x3c, 0x04, 0x17, 0xeb, 0x1c, 0x22, 0x06, 0x08, 0x2f }, + { 0x26, 0xcd, 0x66, 0xfc, 0xa0, 0x23, 0x79, 0xc7, 0x6d, 0xf1, 0x23, 0x17, 0x05, 0x2b, 0xca, 0xfd, 0x6c, 0xd8, 0xc3, 0xa7, 0xb8, 0x90, 0xd8, 0x05, 0xf3, 0x6c, 0x49, 0x98, 0x97, 0x82, 0x43, 0x3a }, + { 0x21, 0x3f, 0x35, 0x96, 0xd6, 0xe3, 0xa5, 0xd0, 0xe9, 0x93, 0x2c, 0xd2, 0x15, 0x91, 0x46, 0x01, 0x5e, 0x2a, 0xbc, 0x94, 0x9f, 0x47, 0x29, 0xee, 0x26, 0x32, 0xfe, 0x1e, 0xdb, 0x78, 0xd3, 0x37 }, + { 0x10, 0x15, 0xd7, 0x01, 0x08, 0xe0, 0x3b, 0xe1, 0xc7, 0x02, 0xfe, 0x97, 0x25, 0x36, 0x07, 0xd1, 0x4a, 0xee, 0x59, 0x1f, 0x24, 0x13, 0xea, 0x67, 0x87, 0x42, 0x7b, 0x64, 0x59, 0xff, 0x21, 0x9a }, + { 0x3c, 0xa9, 0x89, 0xde, 0x10, 0xcf, 0xe6, 0x09, 0x90, 0x94, 0x72, 0xc8, 0xd3, 0x56, 0x10, 0x80, 0x5b, 0x2f, 0x97, 0x77, 0x34, 0xcf, 0x65, 0x2c, 0xc6, 0x4b, 0x3b, 0xfc, 0x88, 0x2d, 0x5d, 0x89 }, + { 0xb6, 0x15, 0x6f, 0x72, 0xd3, 0x80, 0xee, 0x9e, 0xa6, 0xac, 0xd1, 0x90, 0x46, 0x4f, 0x23, 0x07, 0xa5, 0xc1, 0x79, 0xef, 0x01, 0xfd, 0x71, 0xf9, 0x9f, 0x2d, 0x0f, 0x7a, 0x57, 0x36, 0x0a, 0xea }, + { 0xc0, 0x3b, 0xc6, 0x42, 0xb2, 0x09, 0x59, 0xcb, 0xe1, 0x33, 0xa0, 0x30, 0x3e, 0x0c, 0x1a, 0xbf, 0xf3, 0xe3, 0x1e, 0xc8, 0xe1, 0xa3, 0x28, 0xec, 0x85, 0x65, 0xc3, 0x6d, 0xec, 0xff, 0x52, 0x65 }, + { 0x2c, 0x3e, 0x08, 0x17, 0x6f, 0x76, 0x0c, 0x62, 0x64, 0xc3, 0xa2, 0xcd, 0x66, 0xfe, 0xc6, 0xc3, 0xd7, 0x8d, 0xe4, 0x3f, 0xc1, 0x92, 0x45, 0x7b, 0x2a, 0x4a, 0x66, 0x0a, 0x1e, 0x0e, 0xb2, 0x2b }, + { 0xf7, 0x38, 0xc0, 0x2f, 0x3c, 0x1b, 0x19, 0x0c, 0x51, 0x2b, 0x1a, 0x32, 0xde, 0xab, 0xf3, 0x53, 0x72, 0x8e, 0x0e, 0x9a, 0xb0, 0x34, 0x49, 0x0e, 0x3c, 0x34, 0x09, 0x94, 0x6a, 0x97, 0xae, 0xec }, + { 0x8b, 0x18, 0x80, 0xdf, 0x30, 0x1c, 0xc9, 0x63, 0x41, 0x88, 0x11, 0x08, 0x89, 0x64, 0x83, 0x92, 0x87, 0xff, 0x7f, 0xe3, 0x1c, 0x49, 0xea, 0x6e, 0xbd, 0x9e, 0x48, 0xbd, 0xee, 0xe4, 0x97, 0xc5 }, + { 0x1e, 0x75, 0xcb, 0x21, 0xc6, 0x09, 0x89, 0x02, 0x03, 0x75, 0xf1, 0xa7, 0xa2, 0x42, 0x83, 0x9f, 0x0b, 0x0b, 0x68, 0x97, 0x3a, 0x4c, 0x2a, 0x05, 0xcf, 0x75, 0x55, 0xed, 0x5a, 0xae, 0xc4, 0xc1 }, + { 0x62, 0xbf, 0x8a, 0x9c, 0x32, 0xa5, 0xbc, 0xcf, 0x29, 0x0b, 0x6c, 0x47, 0x4d, 0x75, 0xb2, 0xa2, 0xa4, 0x09, 0x3f, 0x1a, 0x9e, 0x27, 0x13, 0x94, 0x33, 0xa8, 0xf2, 0xb3, 0xbc, 0xe7, 0xb8, 0xd7 }, + { 0x16, 0x6c, 0x83, 0x50, 0xd3, 0x17, 0x3b, 0x5e, 0x70, 0x2b, 0x78, 0x3d, 0xfd, 0x33, 0xc6, 0x6e, 0xe0, 0x43, 0x27, 0x42, 0xe9, 0xb9, 0x2b, 0x99, 0x7f, 0xd2, 0x3c, 0x60, 0xdc, 0x67, 0x56, 0xca }, + { 0x04, 0x4a, 0x14, 0xd8, 0x22, 0xa9, 0x0c, 0xac, 0xf2, 0xf5, 0xa1, 0x01, 0x42, 0x8a, 0xdc, 0x8f, 0x41, 0x09, 0x38, 0x6c, 0xcb, 0x15, 0x8b, 0xf9, 0x05, 0xc8, 0x61, 0x8b, 0x8e, 0xe2, 0x4e, 0xc3 }, + { 0x38, 0x7d, 0x39, 0x7e, 0xa4, 0x3a, 0x99, 0x4b, 0xe8, 0x4d, 0x2d, 0x54, 0x4a, 0xfb, 0xe4, 0x81, 0xa2, 0x00, 0x0f, 0x55, 0x25, 0x26, 0x96, 0xbb, 0xa2, 0xc5, 0x0c, 0x8e, 0xbd, 0x10, 0x13, 0x47 }, + { 0x56, 0xf8, 0xcc, 0xf1, 0xf8, 0x64, 0x09, 0xb4, 0x6c, 0xe3, 0x61, 0x66, 0xae, 0x91, 0x65, 0x13, 0x84, 0x41, 0x57, 0x75, 0x89, 0xdb, 0x08, 0xcb, 0xc5, 0xf6, 0x6c, 0xa2, 0x97, 0x43, 0xb9, 0xfd }, + { 0x97, 0x06, 0xc0, 0x92, 0xb0, 0x4d, 0x91, 0xf5, 0x3d, 0xff, 0x91, 0xfa, 0x37, 0xb7, 0x49, 0x3d, 0x28, 0xb5, 0x76, 0xb5, 0xd7, 0x10, 0x46, 0x9d, 0xf7, 0x94, 0x01, 0x66, 0x22, 0x36, 0xfc, 0x03 }, + { 0x87, 0x79, 0x68, 0x68, 0x6c, 0x06, 0x8c, 0xe2, 0xf7, 0xe2, 0xad, 0xcf, 0xf6, 0x8b, 0xf8, 0x74, 0x8e, 0xdf, 0x3c, 0xf8, 0x62, 0xcf, 0xb4, 0xd3, 0x94, 0x7a, 0x31, 0x06, 0x95, 0x80, 0x54, 0xe3 }, + { 0x88, 0x17, 0xe5, 0x71, 0x98, 0x79, 0xac, 0xf7, 0x02, 0x47, 0x87, 0xec, 0xcd, 0xb2, 0x71, 0x03, 0x55, 0x66, 0xcf, 0xa3, 0x33, 0xe0, 0x49, 0x40, 0x7c, 0x01, 0x78, 0xcc, 0xc5, 0x7a, 0x5b, 0x9f }, + { 0x89, 0x38, 0x24, 0x9e, 0x4b, 0x50, 0xca, 0xda, 0xcc, 0xdf, 0x5b, 0x18, 0x62, 0x13, 0x26, 0xcb, 0xb1, 0x52, 0x53, 0xe3, 0x3a, 0x20, 0xf5, 0x63, 0x6e, 0x99, 0x5d, 0x72, 0x47, 0x8d, 0xe4, 0x72 }, + { 0xf1, 0x64, 0xab, 0xba, 0x49, 0x63, 0xa4, 0x4d, 0x10, 0x72, 0x57, 0xe3, 0x23, 0x2d, 0x90, 0xac, 0xa5, 0xe6, 0x6a, 0x14, 0x08, 0x24, 0x8c, 0x51, 0x74, 0x1e, 0x99, 0x1d, 0xb5, 0x22, 0x77, 0x56 }, + { 0xd0, 0x55, 0x63, 0xe2, 0xb1, 0xcb, 0xa0, 0xc4, 0xa2, 0xa1, 0xe8, 0xbd, 0xe3, 0xa1, 0xa0, 0xd9, 0xf5, 0xb4, 0x0c, 0x85, 0xa0, 0x70, 0xd6, 0xf5, 0xfb, 0x21, 0x06, 0x6e, 0xad, 0x5d, 0x06, 0x01 }, + { 0x03, 0xfb, 0xb1, 0x63, 0x84, 0xf0, 0xa3, 0x86, 0x6f, 0x4c, 0x31, 0x17, 0x87, 0x76, 0x66, 0xef, 0xbf, 0x12, 0x45, 0x97, 0x56, 0x4b, 0x29, 0x3d, 0x4a, 0xab, 0x0d, 0x26, 0x9f, 0xab, 0xdd, 0xfa }, + { 0x5f, 0xa8, 0x48, 0x6a, 0xc0, 0xe5, 0x29, 0x64, 0xd1, 0x88, 0x1b, 0xbe, 0x33, 0x8e, 0xb5, 0x4b, 0xe2, 0xf7, 0x19, 0x54, 0x92, 0x24, 0x89, 0x20, 0x57, 0xb4, 0xda, 0x04, 0xba, 0x8b, 0x34, 0x75 }, + { 0xcd, 0xfa, 0xbc, 0xee, 0x46, 0x91, 0x11, 0x11, 0x23, 0x6a, 0x31, 0x70, 0x8b, 0x25, 0x39, 0xd7, 0x1f, 0xc2, 0x11, 0xd9, 0xb0, 0x9c, 0x0d, 0x85, 0x30, 0xa1, 0x1e, 0x1d, 0xbf, 0x6e, 0xed, 0x01 }, + { 0x4f, 0x82, 0xde, 0x03, 0xb9, 0x50, 0x47, 0x93, 0xb8, 0x2a, 0x07, 0xa0, 0xbd, 0xcd, 0xff, 0x31, 0x4d, 0x75, 0x9e, 0x7b, 0x62, 0xd2, 0x6b, 0x78, 0x49, 0x46, 0xb0, 0xd3, 0x6f, 0x91, 0x6f, 0x52 }, + { 0x25, 0x9e, 0xc7, 0xf1, 0x73, 0xbc, 0xc7, 0x6a, 0x09, 0x94, 0xc9, 0x67, 0xb4, 0xf5, 0xf0, 0x24, 0xc5, 0x60, 0x57, 0xfb, 0x79, 0xc9, 0x65, 0xc4, 0xfa, 0xe4, 0x18, 0x75, 0xf0, 0x6a, 0x0e, 0x4c }, + { 0x19, 0x3c, 0xc8, 0xe7, 0xc3, 0xe0, 0x8b, 0xb3, 0x0f, 0x54, 0x37, 0xaa, 0x27, 0xad, 0xe1, 0xf1, 0x42, 0x36, 0x9b, 0x24, 0x6a, 0x67, 0x5b, 0x23, 0x83, 0xe6, 0xda, 0x9b, 0x49, 0xa9, 0x80, 0x9e }, + { 0x5c, 0x10, 0x89, 0x6f, 0x0e, 0x28, 0x56, 0xb2, 0xa2, 0xee, 0xe0, 0xfe, 0x4a, 0x2c, 0x16, 0x33, 0x56, 0x5d, 0x18, 0xf0, 0xe9, 0x3e, 0x1f, 0xab, 0x26, 0xc3, 0x73, 0xe8, 0xf8, 0x29, 0x65, 0x4d }, + { 0xf1, 0x60, 0x12, 0xd9, 0x3f, 0x28, 0x85, 0x1a, 0x1e, 0xb9, 0x89, 0xf5, 0xd0, 0xb4, 0x3f, 0x3f, 0x39, 0xca, 0x73, 0xc9, 0xa6, 0x2d, 0x51, 0x81, 0xbf, 0xf2, 0x37, 0x53, 0x6b, 0xd3, 0x48, 0xc3 }, + { 0x29, 0x66, 0xb3, 0xcf, 0xae, 0x1e, 0x44, 0xea, 0x99, 0x6d, 0xc5, 0xd6, 0x86, 0xcf, 0x25, 0xfa, 0x05, 0x3f, 0xb6, 0xf6, 0x72, 0x01, 0xb9, 0xe4, 0x6e, 0xad, 0xe8, 0x5d, 0x0a, 0xd6, 0xb8, 0x06 }, + { 0xdd, 0xb8, 0x78, 0x24, 0x85, 0xe9, 0x00, 0xbc, 0x60, 0xbc, 0xf4, 0xc3, 0x3a, 0x6f, 0xd5, 0x85, 0x68, 0x0c, 0xc6, 0x83, 0xd5, 0x16, 0xef, 0xa0, 0x3e, 0xb9, 0x98, 0x5f, 0xad, 0x87, 0x15, 0xfb }, + { 0x4c, 0x4d, 0x6e, 0x71, 0xae, 0xa0, 0x57, 0x86, 0x41, 0x31, 0x48, 0xfc, 0x7a, 0x78, 0x6b, 0x0e, 0xca, 0xf5, 0x82, 0xcf, 0xf1, 0x20, 0x9f, 0x5a, 0x80, 0x9f, 0xba, 0x85, 0x04, 0xce, 0x66, 0x2c }, + { 0xfb, 0x4c, 0x5e, 0x86, 0xd7, 0xb2, 0x22, 0x9b, 0x99, 0xb8, 0xba, 0x6d, 0x94, 0xc2, 0x47, 0xef, 0x96, 0x4a, 0xa3, 0xa2, 0xba, 0xe8, 0xed, 0xc7, 0x75, 0x69, 0xf2, 0x8d, 0xbb, 0xff, 0x2d, 0x4e }, + { 0xe9, 0x4f, 0x52, 0x6d, 0xe9, 0x01, 0x96, 0x33, 0xec, 0xd5, 0x4a, 0xc6, 0x12, 0x0f, 0x23, 0x95, 0x8d, 0x77, 0x18, 0xf1, 0xe7, 0x71, 0x7b, 0xf3, 0x29, 0x21, 0x1a, 0x4f, 0xae, 0xed, 0x4e, 0x6d }, + { 0xcb, 0xd6, 0x66, 0x0a, 0x10, 0xdb, 0x3f, 0x23, 0xf7, 0xa0, 0x3d, 0x4b, 0x9d, 0x40, 0x44, 0xc7, 0x93, 0x2b, 0x28, 0x01, 0xac, 0x89, 0xd6, 0x0b, 0xc9, 0xeb, 0x92, 0xd6, 0x5a, 0x46, 0xc2, 0xa0 }, + { 0x88, 0x18, 0xbb, 0xd3, 0xdb, 0x4d, 0xc1, 0x23, 0xb2, 0x5c, 0xbb, 0xa5, 0xf5, 0x4c, 0x2b, 0xc4, 0xb3, 0xfc, 0xf9, 0xbf, 0x7d, 0x7a, 0x77, 0x09, 0xf4, 0xae, 0x58, 0x8b, 0x26, 0x7c, 0x4e, 0xce }, + { 0xc6, 0x53, 0x82, 0x51, 0x3f, 0x07, 0x46, 0x0d, 0xa3, 0x98, 0x33, 0xcb, 0x66, 0x6c, 0x5e, 0xd8, 0x2e, 0x61, 0xb9, 0xe9, 0x98, 0xf4, 0xb0, 0xc4, 0x28, 0x7c, 0xee, 0x56, 0xc3, 0xcc, 0x9b, 0xcd }, + { 0x89, 0x75, 0xb0, 0x57, 0x7f, 0xd3, 0x55, 0x66, 0xd7, 0x50, 0xb3, 0x62, 0xb0, 0x89, 0x7a, 0x26, 0xc3, 0x99, 0x13, 0x6d, 0xf0, 0x7b, 0xab, 0xab, 0xbd, 0xe6, 0x20, 0x3f, 0xf2, 0x95, 0x4e, 0xd4 }, + { 0x21, 0xfe, 0x0c, 0xeb, 0x00, 0x52, 0xbe, 0x7f, 0xb0, 0xf0, 0x04, 0x18, 0x7c, 0xac, 0xd7, 0xde, 0x67, 0xfa, 0x6e, 0xb0, 0x93, 0x8d, 0x92, 0x76, 0x77, 0xf2, 0x39, 0x8c, 0x13, 0x23, 0x17, 0xa8 }, + { 0x2e, 0xf7, 0x3f, 0x3c, 0x26, 0xf1, 0x2d, 0x93, 0x88, 0x9f, 0x3c, 0x78, 0xb6, 0xa6, 0x6c, 0x1d, 0x52, 0xb6, 0x49, 0xdc, 0x9e, 0x85, 0x6e, 0x2c, 0x17, 0x2e, 0xa7, 0xc5, 0x8a, 0xc2, 0xb5, 0xe3 }, + { 0x38, 0x8a, 0x3c, 0xd5, 0x6d, 0x73, 0x86, 0x7a, 0xbb, 0x5f, 0x84, 0x01, 0x49, 0x2b, 0x6e, 0x26, 0x81, 0xeb, 0x69, 0x85, 0x1e, 0x76, 0x7f, 0xd8, 0x42, 0x10, 0xa5, 0x60, 0x76, 0xfb, 0x3d, 0xd3 }, + { 0xaf, 0x53, 0x3e, 0x02, 0x2f, 0xc9, 0x43, 0x9e, 0x4e, 0x3c, 0xb8, 0x38, 0xec, 0xd1, 0x86, 0x92, 0x23, 0x2a, 0xdf, 0x6f, 0xe9, 0x83, 0x95, 0x26, 0xd3, 0xc3, 0xdd, 0x1b, 0x71, 0x91, 0x0b, 0x1a }, + { 0x75, 0x1c, 0x09, 0xd4, 0x1a, 0x93, 0x43, 0x88, 0x2a, 0x81, 0xcd, 0x13, 0xee, 0x40, 0x81, 0x8d, 0x12, 0xeb, 0x44, 0xc6, 0xc7, 0xf4, 0x0d, 0xf1, 0x6e, 0x4a, 0xea, 0x8f, 0xab, 0x91, 0x97, 0x2a }, + { 0x5b, 0x73, 0xdd, 0xb6, 0x8d, 0x9d, 0x2b, 0x0a, 0xa2, 0x65, 0xa0, 0x79, 0x88, 0xd6, 0xb8, 0x8a, 0xe9, 0xaa, 0xc5, 0x82, 0xaf, 0x83, 0x03, 0x2f, 0x8a, 0x9b, 0x21, 0xa2, 0xe1, 0xb7, 0xbf, 0x18 }, + { 0x3d, 0xa2, 0x91, 0x26, 0xc7, 0xc5, 0xd7, 0xf4, 0x3e, 0x64, 0x24, 0x2a, 0x79, 0xfe, 0xaa, 0x4e, 0xf3, 0x45, 0x9c, 0xde, 0xcc, 0xc8, 0x98, 0xed, 0x59, 0xa9, 0x7f, 0x6e, 0xc9, 0x3b, 0x9d, 0xab }, + { 0x56, 0x6d, 0xc9, 0x20, 0x29, 0x3d, 0xa5, 0xcb, 0x4f, 0xe0, 0xaa, 0x8a, 0xbd, 0xa8, 0xbb, 0xf5, 0x6f, 0x55, 0x23, 0x13, 0xbf, 0xf1, 0x90, 0x46, 0x64, 0x1e, 0x36, 0x15, 0xc1, 0xe3, 0xed, 0x3f }, + { 0x41, 0x15, 0xbe, 0xa0, 0x2f, 0x73, 0xf9, 0x7f, 0x62, 0x9e, 0x5c, 0x55, 0x90, 0x72, 0x0c, 0x01, 0xe7, 0xe4, 0x49, 0xae, 0x2a, 0x66, 0x97, 0xd4, 0xd2, 0x78, 0x33, 0x21, 0x30, 0x36, 0x92, 0xf9 }, + { 0x4c, 0xe0, 0x8f, 0x47, 0x62, 0x46, 0x8a, 0x76, 0x70, 0x01, 0x21, 0x64, 0x87, 0x8d, 0x68, 0x34, 0x0c, 0x52, 0xa3, 0x5e, 0x66, 0xc1, 0x88, 0x4d, 0x5c, 0x86, 0x48, 0x89, 0xab, 0xc9, 0x66, 0x77 }, + { 0x81, 0xea, 0x0b, 0x78, 0x04, 0x12, 0x4e, 0x0c, 0x22, 0xea, 0x5f, 0xc7, 0x11, 0x04, 0xa2, 0xaf, 0xcb, 0x52, 0xa1, 0xfa, 0x81, 0x6f, 0x3e, 0xcb, 0x7d, 0xcb, 0x5d, 0x9d, 0xea, 0x17, 0x86, 0xd0 }, + { 0xfe, 0x36, 0x27, 0x33, 0xb0, 0x5f, 0x6b, 0xed, 0xaf, 0x93, 0x79, 0xd7, 0xf7, 0x93, 0x6e, 0xde, 0x20, 0x9b, 0x1f, 0x83, 0x23, 0xc3, 0x92, 0x25, 0x49, 0xd9, 0xe7, 0x36, 0x81, 0xb5, 0xdb, 0x7b }, + { 0xef, 0xf3, 0x7d, 0x30, 0xdf, 0xd2, 0x03, 0x59, 0xbe, 0x4e, 0x73, 0xfd, 0xf4, 0x0d, 0x27, 0x73, 0x4b, 0x3d, 0xf9, 0x0a, 0x97, 0xa5, 0x5e, 0xd7, 0x45, 0x29, 0x72, 0x94, 0xca, 0x85, 0xd0, 0x9f }, + { 0x17, 0x2f, 0xfc, 0x67, 0x15, 0x3d, 0x12, 0xe0, 0xca, 0x76, 0xa8, 0xb6, 0xcd, 0x5d, 0x47, 0x31, 0x88, 0x5b, 0x39, 0xce, 0x0c, 0xac, 0x93, 0xa8, 0x97, 0x2a, 0x18, 0x00, 0x6c, 0x8b, 0x8b, 0xaf }, + { 0xc4, 0x79, 0x57, 0xf1, 0xcc, 0x88, 0xe8, 0x3e, 0xf9, 0x44, 0x58, 0x39, 0x70, 0x9a, 0x48, 0x0a, 0x03, 0x6b, 0xed, 0x5f, 0x88, 0xac, 0x0f, 0xcc, 0x8e, 0x1e, 0x70, 0x3f, 0xfa, 0xac, 0x13, 0x2c }, + { 0x30, 0xf3, 0x54, 0x83, 0x70, 0xcf, 0xdc, 0xed, 0xa5, 0xc3, 0x7b, 0x56, 0x9b, 0x61, 0x75, 0xe7, 0x99, 0xee, 0xf1, 0xa6, 0x2a, 0xaa, 0x94, 0x32, 0x45, 0xae, 0x76, 0x69, 0xc2, 0x27, 0xa7, 0xb5 }, + { 0xc9, 0x5d, 0xcb, 0x3c, 0xf1, 0xf2, 0x7d, 0x0e, 0xef, 0x2f, 0x25, 0xd2, 0x41, 0x38, 0x70, 0x90, 0x4a, 0x87, 0x7c, 0x4a, 0x56, 0xc2, 0xde, 0x1e, 0x83, 0xe2, 0xbc, 0x2a, 0xe2, 0xe4, 0x68, 0x21 }, + { 0xd5, 0xd0, 0xb5, 0xd7, 0x05, 0x43, 0x4c, 0xd4, 0x6b, 0x18, 0x57, 0x49, 0xf6, 0x6b, 0xfb, 0x58, 0x36, 0xdc, 0xdf, 0x6e, 0xe5, 0x49, 0xa2, 0xb7, 0xa4, 0xae, 0xe7, 0xf5, 0x80, 0x07, 0xca, 0xaf }, + { 0xbb, 0xc1, 0x24, 0xa7, 0x12, 0xf1, 0x5d, 0x07, 0xc3, 0x00, 0xe0, 0x5b, 0x66, 0x83, 0x89, 0xa4, 0x39, 0xc9, 0x17, 0x77, 0xf7, 0x21, 0xf8, 0x32, 0x0c, 0x1c, 0x90, 0x78, 0x06, 0x6d, 0x2c, 0x7e }, + { 0xa4, 0x51, 0xb4, 0x8c, 0x35, 0xa6, 0xc7, 0x85, 0x4c, 0xfa, 0xae, 0x60, 0x26, 0x2e, 0x76, 0x99, 0x08, 0x16, 0x38, 0x2a, 0xc0, 0x66, 0x7e, 0x5a, 0x5c, 0x9e, 0x1b, 0x46, 0xc4, 0x34, 0x2d, 0xdf }, + { 0xb0, 0xd1, 0x50, 0xfb, 0x55, 0xe7, 0x78, 0xd0, 0x11, 0x47, 0xf0, 0xb5, 0xd8, 0x9d, 0x99, 0xec, 0xb2, 0x0f, 0xf0, 0x7e, 0x5e, 0x67, 0x60, 0xd6, 0xb6, 0x45, 0xeb, 0x5b, 0x65, 0x4c, 0x62, 0x2b }, + { 0x34, 0xf7, 0x37, 0xc0, 0xab, 0x21, 0x99, 0x51, 0xee, 0xe8, 0x9a, 0x9f, 0x8d, 0xac, 0x29, 0x9c, 0x9d, 0x4c, 0x38, 0xf3, 0x3f, 0xa4, 0x94, 0xc5, 0xc6, 0xee, 0xfc, 0x92, 0xb6, 0xdb, 0x08, 0xbc }, + { 0x1a, 0x62, 0xcc, 0x3a, 0x00, 0x80, 0x0d, 0xcb, 0xd9, 0x98, 0x91, 0x08, 0x0c, 0x1e, 0x09, 0x84, 0x58, 0x19, 0x3a, 0x8c, 0xc9, 0xf9, 0x70, 0xea, 0x99, 0xfb, 0xef, 0xf0, 0x03, 0x18, 0xc2, 0x89 }, + { 0xcf, 0xce, 0x55, 0xeb, 0xaf, 0xc8, 0x40, 0xd7, 0xae, 0x48, 0x28, 0x1c, 0x7f, 0xd5, 0x7e, 0xc8, 0xb4, 0x82, 0xd4, 0xb7, 0x04, 0x43, 0x74, 0x95, 0x49, 0x5a, 0xc4, 0x14, 0xcf, 0x4a, 0x37, 0x4b }, + { 0x67, 0x46, 0xfa, 0xcf, 0x71, 0x14, 0x6d, 0x99, 0x9d, 0xab, 0xd0, 0x5d, 0x09, 0x3a, 0xe5, 0x86, 0x64, 0x8d, 0x1e, 0xe2, 0x8e, 0x72, 0x61, 0x7b, 0x99, 0xd0, 0xf0, 0x08, 0x6e, 0x1e, 0x45, 0xbf }, + { 0x57, 0x1c, 0xed, 0x28, 0x3b, 0x3f, 0x23, 0xb4, 0xe7, 0x50, 0xbf, 0x12, 0xa2, 0xca, 0xf1, 0x78, 0x18, 0x47, 0xbd, 0x89, 0x0e, 0x43, 0x60, 0x3c, 0xdc, 0x59, 0x76, 0x10, 0x2b, 0x7b, 0xb1, 0x1b }, + { 0xcf, 0xcb, 0x76, 0x5b, 0x04, 0x8e, 0x35, 0x02, 0x2c, 0x5d, 0x08, 0x9d, 0x26, 0xe8, 0x5a, 0x36, 0xb0, 0x05, 0xa2, 0xb8, 0x04, 0x93, 0xd0, 0x3a, 0x14, 0x4e, 0x09, 0xf4, 0x09, 0xb6, 0xaf, 0xd1 }, + { 0x40, 0x50, 0xc7, 0xa2, 0x77, 0x05, 0xbb, 0x27, 0xf4, 0x20, 0x89, 0xb2, 0x99, 0xf3, 0xcb, 0xe5, 0x05, 0x4e, 0xad, 0x68, 0x72, 0x7e, 0x8e, 0xf9, 0x31, 0x8c, 0xe6, 0xf2, 0x5c, 0xd6, 0xf3, 0x1d }, + { 0x18, 0x40, 0x70, 0xbd, 0x5d, 0x26, 0x5f, 0xbd, 0xc1, 0x42, 0xcd, 0x1c, 0x5c, 0xd0, 0xd7, 0xe4, 0x14, 0xe7, 0x03, 0x69, 0xa2, 0x66, 0xd6, 0x27, 0xc8, 0xfb, 0xa8, 0x4f, 0xa5, 0xe8, 0x4c, 0x34 }, + { 0x9e, 0xdd, 0xa9, 0xa4, 0x44, 0x39, 0x02, 0xa9, 0x58, 0x8c, 0x0d, 0x0c, 0xcc, 0x62, 0xb9, 0x30, 0x21, 0x84, 0x79, 0xa6, 0x84, 0x1e, 0x6f, 0xe7, 0xd4, 0x30, 0x03, 0xf0, 0x4b, 0x1f, 0xd6, 0x43 }, + { 0xe4, 0x12, 0xfe, 0xef, 0x79, 0x08, 0x32, 0x4a, 0x6d, 0xa1, 0x84, 0x16, 0x29, 0xf3, 0x5d, 0x3d, 0x35, 0x86, 0x42, 0x01, 0x93, 0x10, 0xec, 0x57, 0xc6, 0x14, 0x83, 0x6b, 0x63, 0xd3, 0x07, 0x63 }, + { 0x1a, 0x2b, 0x8e, 0xdf, 0xf3, 0xf9, 0xac, 0xc1, 0x55, 0x4f, 0xcb, 0xae, 0x3c, 0xf1, 0xd6, 0x29, 0x8c, 0x64, 0x62, 0xe2, 0x2e, 0x5e, 0xb0, 0x25, 0x96, 0x84, 0xf8, 0x35, 0x01, 0x2b, 0xd1, 0x3f }, + { 0x28, 0x8c, 0x4a, 0xd9, 0xb9, 0x40, 0x97, 0x62, 0xea, 0x07, 0xc2, 0x4a, 0x41, 0xf0, 0x4f, 0x69, 0xa7, 0xd7, 0x4b, 0xee, 0x2d, 0x95, 0x43, 0x53, 0x74, 0xbd, 0xe9, 0x46, 0xd7, 0x24, 0x1c, 0x7b }, + { 0x80, 0x56, 0x91, 0xbb, 0x28, 0x67, 0x48, 0xcf, 0xb5, 0x91, 0xd3, 0xae, 0xbe, 0x7e, 0x6f, 0x4e, 0x4d, 0xc6, 0xe2, 0x80, 0x8c, 0x65, 0x14, 0x3c, 0xc0, 0x04, 0xe4, 0xeb, 0x6f, 0xd0, 0x9d, 0x43 }, + { 0xd4, 0xac, 0x8d, 0x3a, 0x0a, 0xfc, 0x6c, 0xfa, 0x7b, 0x46, 0x0a, 0xe3, 0x00, 0x1b, 0xae, 0xb3, 0x6d, 0xad, 0xb3, 0x7d, 0xa0, 0x7d, 0x2e, 0x8a, 0xc9, 0x18, 0x22, 0xdf, 0x34, 0x8a, 0xed, 0x3d }, + { 0xc3, 0x76, 0x61, 0x70, 0x14, 0xd2, 0x01, 0x58, 0xbc, 0xed, 0x3d, 0x3b, 0xa5, 0x52, 0xb6, 0xec, 0xcf, 0x84, 0xe6, 0x2a, 0xa3, 0xeb, 0x65, 0x0e, 0x90, 0x02, 0x9c, 0x84, 0xd1, 0x3e, 0xea, 0x69 }, + { 0xc4, 0x1f, 0x09, 0xf4, 0x3c, 0xec, 0xae, 0x72, 0x93, 0xd6, 0x00, 0x7c, 0xa0, 0xa3, 0x57, 0x08, 0x7d, 0x5a, 0xe5, 0x9b, 0xe5, 0x00, 0xc1, 0xcd, 0x5b, 0x28, 0x9e, 0xe8, 0x10, 0xc7, 0xb0, 0x82 }, + { 0x03, 0xd1, 0xce, 0xd1, 0xfb, 0xa5, 0xc3, 0x91, 0x55, 0xc4, 0x4b, 0x77, 0x65, 0xcb, 0x76, 0x0c, 0x78, 0x70, 0x8d, 0xcf, 0xc8, 0x0b, 0x0b, 0xd8, 0xad, 0xe3, 0xa5, 0x6d, 0xa8, 0x83, 0x0b, 0x29 }, + { 0x09, 0xbd, 0xe6, 0xf1, 0x52, 0x21, 0x8d, 0xc9, 0x2c, 0x41, 0xd7, 0xf4, 0x53, 0x87, 0xe6, 0x3e, 0x58, 0x69, 0xd8, 0x07, 0xec, 0x70, 0xb8, 0x21, 0x40, 0x5d, 0xbd, 0x88, 0x4b, 0x7f, 0xcf, 0x4b }, + { 0x71, 0xc9, 0x03, 0x6e, 0x18, 0x17, 0x9b, 0x90, 0xb3, 0x7d, 0x39, 0xe9, 0xf0, 0x5e, 0xb8, 0x9c, 0xc5, 0xfc, 0x34, 0x1f, 0xd7, 0xc4, 0x77, 0xd0, 0xd7, 0x49, 0x32, 0x85, 0xfa, 0xca, 0x08, 0xa4 }, + { 0x59, 0x16, 0x83, 0x3e, 0xbb, 0x05, 0xcd, 0x91, 0x9c, 0xa7, 0xfe, 0x83, 0xb6, 0x92, 0xd3, 0x20, 0x5b, 0xef, 0x72, 0x39, 0x2b, 0x2c, 0xf6, 0xbb, 0x0a, 0x6d, 0x43, 0xf9, 0x94, 0xf9, 0x5f, 0x11 }, + { 0xf6, 0x3a, 0xab, 0x3e, 0xc6, 0x41, 0xb3, 0xb0, 0x24, 0x96, 0x4c, 0x2b, 0x43, 0x7c, 0x04, 0xf6, 0x04, 0x3c, 0x4c, 0x7e, 0x02, 0x79, 0x23, 0x99, 0x95, 0x40, 0x19, 0x58, 0xf8, 0x6b, 0xbe, 0x54 }, + { 0xf1, 0x72, 0xb1, 0x80, 0xbf, 0xb0, 0x97, 0x40, 0x49, 0x31, 0x20, 0xb6, 0x32, 0x6c, 0xbd, 0xc5, 0x61, 0xe4, 0x77, 0xde, 0xf9, 0xbb, 0xcf, 0xd2, 0x8c, 0xc8, 0xc1, 0xc5, 0xe3, 0x37, 0x9a, 0x31 }, + { 0xcb, 0x9b, 0x89, 0xcc, 0x18, 0x38, 0x1d, 0xd9, 0x14, 0x1a, 0xde, 0x58, 0x86, 0x54, 0xd4, 0xe6, 0xa2, 0x31, 0xd5, 0xbf, 0x49, 0xd4, 0xd5, 0x9a, 0xc2, 0x7d, 0x86, 0x9c, 0xbe, 0x10, 0x0c, 0xf3 }, + { 0x7b, 0xd8, 0x81, 0x50, 0x46, 0xfd, 0xd8, 0x10, 0xa9, 0x23, 0xe1, 0x98, 0x4a, 0xae, 0xbd, 0xcd, 0xf8, 0x4d, 0x87, 0xc8, 0x99, 0x2d, 0x68, 0xb5, 0xee, 0xb4, 0x60, 0xf9, 0x3e, 0xb3, 0xc8, 0xd7 }, + { 0x60, 0x7b, 0xe6, 0x68, 0x62, 0xfd, 0x08, 0xee, 0x5b, 0x19, 0xfa, 0xca, 0xc0, 0x9d, 0xfd, 0xbc, 0xd4, 0x0c, 0x31, 0x21, 0x01, 0xd6, 0x6e, 0x6e, 0xbd, 0x2b, 0x84, 0x1f, 0x1b, 0x9a, 0x93, 0x25 }, + { 0x9f, 0xe0, 0x3b, 0xbe, 0x69, 0xab, 0x18, 0x34, 0xf5, 0x21, 0x9b, 0x0d, 0xa8, 0x8a, 0x08, 0xb3, 0x0a, 0x66, 0xc5, 0x91, 0x3f, 0x01, 0x51, 0x96, 0x3c, 0x36, 0x05, 0x60, 0xdb, 0x03, 0x87, 0xb3 }, + { 0x90, 0xa8, 0x35, 0x85, 0x71, 0x7b, 0x75, 0xf0, 0xe9, 0xb7, 0x25, 0xe0, 0x55, 0xee, 0xee, 0xb9, 0xe7, 0xa0, 0x28, 0xea, 0x7e, 0x6c, 0xbc, 0x07, 0xb2, 0x09, 0x17, 0xec, 0x03, 0x63, 0xe3, 0x8c }, + { 0x33, 0x6e, 0xa0, 0x53, 0x0f, 0x4a, 0x74, 0x69, 0x12, 0x6e, 0x02, 0x18, 0x58, 0x7e, 0xbb, 0xde, 0x33, 0x58, 0xa0, 0xb3, 0x1c, 0x29, 0xd2, 0x00, 0xf7, 0xdc, 0x7e, 0xb1, 0x5c, 0x6a, 0xad, 0xd8 }, + { 0xa7, 0x9e, 0x76, 0xdc, 0x0a, 0xbc, 0xa4, 0x39, 0x6f, 0x07, 0x47, 0xcd, 0x7b, 0x74, 0x8d, 0xf9, 0x13, 0x00, 0x76, 0x26, 0xb1, 0xd6, 0x59, 0xda, 0x0c, 0x1f, 0x78, 0xb9, 0x30, 0x3d, 0x01, 0xa3 }, + { 0x44, 0xe7, 0x8a, 0x77, 0x37, 0x56, 0xe0, 0x95, 0x15, 0x19, 0x50, 0x4d, 0x70, 0x38, 0xd2, 0x8d, 0x02, 0x13, 0xa3, 0x7e, 0x0c, 0xe3, 0x75, 0x37, 0x17, 0x57, 0xbc, 0x99, 0x63, 0x11, 0xe3, 0xb8 }, + { 0x77, 0xac, 0x01, 0x2a, 0x3f, 0x75, 0x4d, 0xcf, 0xea, 0xb5, 0xeb, 0x99, 0x6b, 0xe9, 0xcd, 0x2d, 0x1f, 0x96, 0x11, 0x1b, 0x6e, 0x49, 0xf3, 0x99, 0x4d, 0xf1, 0x81, 0xf2, 0x85, 0x69, 0xd8, 0x25 }, + { 0xce, 0x5a, 0x10, 0xdb, 0x6f, 0xcc, 0xda, 0xf1, 0x40, 0xaa, 0xa4, 0xde, 0xd6, 0x25, 0x0a, 0x9c, 0x06, 0xe9, 0x22, 0x2b, 0xc9, 0xf9, 0xf3, 0x65, 0x8a, 0x4a, 0xff, 0x93, 0x5f, 0x2b, 0x9f, 0x3a }, + { 0xec, 0xc2, 0x03, 0xa7, 0xfe, 0x2b, 0xe4, 0xab, 0xd5, 0x5b, 0xb5, 0x3e, 0x6e, 0x67, 0x35, 0x72, 0xe0, 0x07, 0x8d, 0xa8, 0xcd, 0x37, 0x5e, 0xf4, 0x30, 0xcc, 0x97, 0xf9, 0xf8, 0x00, 0x83, 0xaf }, + { 0x14, 0xa5, 0x18, 0x6d, 0xe9, 0xd7, 0xa1, 0x8b, 0x04, 0x12, 0xb8, 0x56, 0x3e, 0x51, 0xcc, 0x54, 0x33, 0x84, 0x0b, 0x4a, 0x12, 0x9a, 0x8f, 0xf9, 0x63, 0xb3, 0x3a, 0x3c, 0x4a, 0xfe, 0x8e, 0xbb }, + { 0x13, 0xf8, 0xef, 0x95, 0xcb, 0x86, 0xe6, 0xa6, 0x38, 0x93, 0x1c, 0x8e, 0x10, 0x76, 0x73, 0xeb, 0x76, 0xba, 0x10, 0xd7, 0xc2, 0xcd, 0x70, 0xb9, 0xd9, 0x92, 0x0b, 0xbe, 0xed, 0x92, 0x94, 0x09 }, + { 0x0b, 0x33, 0x8f, 0x4e, 0xe1, 0x2f, 0x2d, 0xfc, 0xb7, 0x87, 0x13, 0x37, 0x79, 0x41, 0xe0, 0xb0, 0x63, 0x21, 0x52, 0x58, 0x1d, 0x13, 0x32, 0x51, 0x6e, 0x4a, 0x2c, 0xab, 0x19, 0x42, 0xcc, 0xa4 }, + { 0xea, 0xab, 0x0e, 0xc3, 0x7b, 0x3b, 0x8a, 0xb7, 0x96, 0xe9, 0xf5, 0x72, 0x38, 0xde, 0x14, 0xa2, 0x64, 0xa0, 0x76, 0xf3, 0x88, 0x7d, 0x86, 0xe2, 0x9b, 0xb5, 0x90, 0x6d, 0xb5, 0xa0, 0x0e, 0x02 }, + { 0x23, 0xcb, 0x68, 0xb8, 0xc0, 0xe6, 0xdc, 0x26, 0xdc, 0x27, 0x76, 0x6d, 0xdc, 0x0a, 0x13, 0xa9, 0x94, 0x38, 0xfd, 0x55, 0x61, 0x7a, 0xa4, 0x09, 0x5d, 0x8f, 0x96, 0x97, 0x20, 0xc8, 0x72, 0xdf }, + { 0x09, 0x1d, 0x8e, 0xe3, 0x0d, 0x6f, 0x29, 0x68, 0xd4, 0x6b, 0x68, 0x7d, 0xd6, 0x52, 0x92, 0x66, 0x57, 0x42, 0xde, 0x0b, 0xb8, 0x3d, 0xcc, 0x00, 0x04, 0xc7, 0x2c, 0xe1, 0x00, 0x07, 0xa5, 0x49 }, + { 0x7f, 0x50, 0x7a, 0xbc, 0x6d, 0x19, 0xba, 0x00, 0xc0, 0x65, 0xa8, 0x76, 0xec, 0x56, 0x57, 0x86, 0x88, 0x82, 0xd1, 0x8a, 0x22, 0x1b, 0xc4, 0x6c, 0x7a, 0x69, 0x12, 0x54, 0x1f, 0x5b, 0xc7, 0xba }, + { 0xa0, 0x60, 0x7c, 0x24, 0xe1, 0x4e, 0x8c, 0x22, 0x3d, 0xb0, 0xd7, 0x0b, 0x4d, 0x30, 0xee, 0x88, 0x01, 0x4d, 0x60, 0x3f, 0x43, 0x7e, 0x9e, 0x02, 0xaa, 0x7d, 0xaf, 0xa3, 0xcd, 0xfb, 0xad, 0x94 }, + { 0xdd, 0xbf, 0xea, 0x75, 0xcc, 0x46, 0x78, 0x82, 0xeb, 0x34, 0x83, 0xce, 0x5e, 0x2e, 0x75, 0x6a, 0x4f, 0x47, 0x01, 0xb7, 0x6b, 0x44, 0x55, 0x19, 0xe8, 0x9f, 0x22, 0xd6, 0x0f, 0xa8, 0x6e, 0x06 }, + { 0x0c, 0x31, 0x1f, 0x38, 0xc3, 0x5a, 0x4f, 0xb9, 0x0d, 0x65, 0x1c, 0x28, 0x9d, 0x48, 0x68, 0x56, 0xcd, 0x14, 0x13, 0xdf, 0x9b, 0x06, 0x77, 0xf5, 0x3e, 0xce, 0x2c, 0xd9, 0xe4, 0x77, 0xc6, 0x0a }, + { 0x46, 0xa7, 0x3a, 0x8d, 0xd3, 0xe7, 0x0f, 0x59, 0xd3, 0x94, 0x2c, 0x01, 0xdf, 0x59, 0x9d, 0xef, 0x78, 0x3c, 0x9d, 0xa8, 0x2f, 0xd8, 0x32, 0x22, 0xcd, 0x66, 0x2b, 0x53, 0xdc, 0xe7, 0xdb, 0xdf }, + { 0xad, 0x03, 0x8f, 0xf9, 0xb1, 0x4d, 0xe8, 0x4a, 0x80, 0x1e, 0x4e, 0x62, 0x1c, 0xe5, 0xdf, 0x02, 0x9d, 0xd9, 0x35, 0x20, 0xd0, 0xc2, 0xfa, 0x38, 0xbf, 0xf1, 0x76, 0xa8, 0xb1, 0xd1, 0x69, 0x8c }, + { 0xab, 0x70, 0xc5, 0xdf, 0xbd, 0x1e, 0xa8, 0x17, 0xfe, 0xd0, 0xcd, 0x06, 0x72, 0x93, 0xab, 0xf3, 0x19, 0xe5, 0xd7, 0x90, 0x1c, 0x21, 0x41, 0xd5, 0xd9, 0x9b, 0x23, 0xf0, 0x3a, 0x38, 0xe7, 0x48 }, + { 0x1f, 0xff, 0xda, 0x67, 0x93, 0x2b, 0x73, 0xc8, 0xec, 0xaf, 0x00, 0x9a, 0x34, 0x91, 0xa0, 0x26, 0x95, 0x3b, 0xab, 0xfe, 0x1f, 0x66, 0x3b, 0x06, 0x97, 0xc3, 0xc4, 0xae, 0x8b, 0x2e, 0x7d, 0xcb }, + { 0xb0, 0xd2, 0xcc, 0x19, 0x47, 0x2d, 0xd5, 0x7f, 0x2b, 0x17, 0xef, 0xc0, 0x3c, 0x8d, 0x58, 0xc2, 0x28, 0x3d, 0xbb, 0x19, 0xda, 0x57, 0x2f, 0x77, 0x55, 0x85, 0x5a, 0xa9, 0x79, 0x43, 0x17, 0xa0 }, + { 0xa0, 0xd1, 0x9a, 0x6e, 0xe3, 0x39, 0x79, 0xc3, 0x25, 0x51, 0x0e, 0x27, 0x66, 0x22, 0xdf, 0x41, 0xf7, 0x15, 0x83, 0xd0, 0x75, 0x01, 0xb8, 0x70, 0x71, 0x12, 0x9a, 0x0a, 0xd9, 0x47, 0x32, 0xa5 }, + { 0x72, 0x46, 0x42, 0xa7, 0x03, 0x2d, 0x10, 0x62, 0xb8, 0x9e, 0x52, 0xbe, 0xa3, 0x4b, 0x75, 0xdf, 0x7d, 0x8f, 0xe7, 0x72, 0xd9, 0xfe, 0x3c, 0x93, 0xdd, 0xf3, 0xc4, 0x54, 0x5a, 0xb5, 0xa9, 0x9b }, + { 0xad, 0xe5, 0xea, 0xa7, 0xe6, 0x1f, 0x67, 0x2d, 0x58, 0x7e, 0xa0, 0x3d, 0xae, 0x7d, 0x7b, 0x55, 0x22, 0x9c, 0x01, 0xd0, 0x6b, 0xc0, 0xa5, 0x70, 0x14, 0x36, 0xcb, 0xd1, 0x83, 0x66, 0xa6, 0x26 }, + { 0x01, 0x3b, 0x31, 0xeb, 0xd2, 0x28, 0xfc, 0xdd, 0xa5, 0x1f, 0xab, 0xb0, 0x3b, 0xb0, 0x2d, 0x60, 0xac, 0x20, 0xca, 0x21, 0x5a, 0xaf, 0xa8, 0x3b, 0xdd, 0x85, 0x5e, 0x37, 0x55, 0xa3, 0x5f, 0x0b }, + { 0x33, 0x2e, 0xd4, 0x0b, 0xb1, 0x0d, 0xde, 0x3c, 0x95, 0x4a, 0x75, 0xd7, 0xb8, 0x99, 0x9d, 0x4b, 0x26, 0xa1, 0xc0, 0x63, 0xc1, 0xdc, 0x6e, 0x32, 0xc1, 0xd9, 0x1b, 0xab, 0x7b, 0xbb, 0x7d, 0x16 }, + { 0xc7, 0xa1, 0x97, 0xb3, 0xa0, 0x5b, 0x56, 0x6b, 0xcc, 0x9f, 0xac, 0xd2, 0x0e, 0x44, 0x1d, 0x6f, 0x6c, 0x28, 0x60, 0xac, 0x96, 0x51, 0xcd, 0x51, 0xd6, 0xb9, 0xd2, 0xcd, 0xee, 0xea, 0x03, 0x90 }, + { 0xbd, 0x9c, 0xf6, 0x4e, 0xa8, 0x95, 0x3c, 0x03, 0x71, 0x08, 0xe6, 0xf6, 0x54, 0x91, 0x4f, 0x39, 0x58, 0xb6, 0x8e, 0x29, 0xc1, 0x67, 0x00, 0xdc, 0x18, 0x4d, 0x94, 0xa2, 0x17, 0x08, 0xff, 0x60 }, + { 0x88, 0x35, 0xb0, 0xac, 0x02, 0x11, 0x51, 0xdf, 0x71, 0x64, 0x74, 0xce, 0x27, 0xce, 0x4d, 0x3c, 0x15, 0xf0, 0xb2, 0xda, 0xb4, 0x80, 0x03, 0xcf, 0x3f, 0x3e, 0xfd, 0x09, 0x45, 0x10, 0x6b, 0x9a }, + { 0x3b, 0xfe, 0xfa, 0x33, 0x01, 0xaa, 0x55, 0xc0, 0x80, 0x19, 0x0c, 0xff, 0xda, 0x8e, 0xae, 0x51, 0xd9, 0xaf, 0x48, 0x8b, 0x4c, 0x1f, 0x24, 0xc3, 0xd9, 0xa7, 0x52, 0x42, 0xfd, 0x8e, 0xa0, 0x1d }, + { 0x08, 0x28, 0x4d, 0x14, 0x99, 0x3c, 0xd4, 0x7d, 0x53, 0xeb, 0xae, 0xcf, 0x0d, 0xf0, 0x47, 0x8c, 0xc1, 0x82, 0xc8, 0x9c, 0x00, 0xe1, 0x85, 0x9c, 0x84, 0x85, 0x16, 0x86, 0xdd, 0xf2, 0xc1, 0xb7 }, + { 0x1e, 0xd7, 0xef, 0x9f, 0x04, 0xc2, 0xac, 0x8d, 0xb6, 0xa8, 0x64, 0xdb, 0x13, 0x10, 0x87, 0xf2, 0x70, 0x65, 0x09, 0x8e, 0x69, 0xc3, 0xfe, 0x78, 0x71, 0x8d, 0x9b, 0x94, 0x7f, 0x4a, 0x39, 0xd0 }, + { 0xc1, 0x61, 0xf2, 0xdc, 0xd5, 0x7e, 0x9c, 0x14, 0x39, 0xb3, 0x1a, 0x9d, 0xd4, 0x3d, 0x8f, 0x3d, 0x7d, 0xd8, 0xf0, 0xeb, 0x7c, 0xfa, 0xc6, 0xfb, 0x25, 0xa0, 0xf2, 0x8e, 0x30, 0x6f, 0x06, 0x61 }, + { 0xc0, 0x19, 0x69, 0xad, 0x34, 0xc5, 0x2c, 0xaf, 0x3d, 0xc4, 0xd8, 0x0d, 0x19, 0x73, 0x5c, 0x29, 0x73, 0x1a, 0xc6, 0xe7, 0xa9, 0x20, 0x85, 0xab, 0x92, 0x50, 0xc4, 0x8d, 0xea, 0x48, 0xa3, 0xfc }, + { 0x17, 0x20, 0xb3, 0x65, 0x56, 0x19, 0xd2, 0xa5, 0x2b, 0x35, 0x21, 0xae, 0x0e, 0x49, 0xe3, 0x45, 0xcb, 0x33, 0x89, 0xeb, 0xd6, 0x20, 0x8a, 0xca, 0xf9, 0xf1, 0x3f, 0xda, 0xcc, 0xa8, 0xbe, 0x49 }, + { 0x75, 0x62, 0x88, 0x36, 0x1c, 0x83, 0xe2, 0x4c, 0x61, 0x7c, 0xf9, 0x5c, 0x90, 0x5b, 0x22, 0xd0, 0x17, 0xcd, 0xc8, 0x6f, 0x0b, 0xf1, 0xd6, 0x58, 0xf4, 0x75, 0x6c, 0x73, 0x79, 0x87, 0x3b, 0x7f }, + { 0xe7, 0xd0, 0xed, 0xa3, 0x45, 0x26, 0x93, 0xb7, 0x52, 0xab, 0xcd, 0xa1, 0xb5, 0x5e, 0x27, 0x6f, 0x82, 0x69, 0x8f, 0x5f, 0x16, 0x05, 0x40, 0x3e, 0xff, 0x83, 0x0b, 0xea, 0x00, 0x71, 0xa3, 0x94 }, + { 0x2c, 0x82, 0xec, 0xaa, 0x6b, 0x84, 0x80, 0x3e, 0x04, 0x4a, 0xf6, 0x31, 0x18, 0xaf, 0xe5, 0x44, 0x68, 0x7c, 0xb6, 0xe6, 0xc7, 0xdf, 0x49, 0xed, 0x76, 0x2d, 0xfd, 0x7c, 0x86, 0x93, 0xa1, 0xbc }, + { 0x61, 0x36, 0xcb, 0xf4, 0xb4, 0x41, 0x05, 0x6f, 0xa1, 0xe2, 0x72, 0x24, 0x98, 0x12, 0x5d, 0x6d, 0xed, 0x45, 0xe1, 0x7b, 0x52, 0x14, 0x39, 0x59, 0xc7, 0xf4, 0xd4, 0xe3, 0x95, 0x21, 0x8a, 0xc2 }, + { 0x72, 0x1d, 0x32, 0x45, 0xaa, 0xfe, 0xf2, 0x7f, 0x6a, 0x62, 0x4f, 0x47, 0x95, 0x4b, 0x6c, 0x25, 0x50, 0x79, 0x52, 0x6f, 0xfa, 0x25, 0xe9, 0xff, 0x77, 0xe5, 0xdc, 0xff, 0x47, 0x3b, 0x15, 0x97 }, + { 0x9d, 0xd2, 0xfb, 0xd8, 0xce, 0xf1, 0x6c, 0x35, 0x3c, 0x0a, 0xc2, 0x11, 0x91, 0xd5, 0x09, 0xeb, 0x28, 0xdd, 0x9e, 0x3e, 0x0d, 0x8c, 0xea, 0x5d, 0x26, 0xca, 0x83, 0x93, 0x93, 0x85, 0x1c, 0x3a }, + { 0xb2, 0x39, 0x4c, 0xea, 0xcd, 0xeb, 0xf2, 0x1b, 0xf9, 0xdf, 0x2c, 0xed, 0x98, 0xe5, 0x8f, 0x1c, 0x3a, 0x4b, 0xbb, 0xff, 0x66, 0x0d, 0xd9, 0x00, 0xf6, 0x22, 0x02, 0xd6, 0x78, 0x5c, 0xc4, 0x6e }, + { 0x57, 0x08, 0x9f, 0x22, 0x27, 0x49, 0xad, 0x78, 0x71, 0x76, 0x5f, 0x06, 0x2b, 0x11, 0x4f, 0x43, 0xba, 0x20, 0xec, 0x56, 0x42, 0x2a, 0x8b, 0x1e, 0x3f, 0x87, 0x19, 0x2c, 0x0e, 0xa7, 0x18, 0xc6 }, + { 0xe4, 0x9a, 0x94, 0x59, 0x96, 0x1c, 0xd3, 0x3c, 0xdf, 0x4a, 0xae, 0x1b, 0x10, 0x78, 0xa5, 0xde, 0xa7, 0xc0, 0x40, 0xe0, 0xfe, 0xa3, 0x40, 0xc9, 0x3a, 0x72, 0x48, 0x72, 0xfc, 0x4a, 0xf8, 0x06 }, + { 0xed, 0xe6, 0x7f, 0x72, 0x0e, 0xff, 0xd2, 0xca, 0x9c, 0x88, 0x99, 0x41, 0x52, 0xd0, 0x20, 0x1d, 0xee, 0x6b, 0x0a, 0x2d, 0x2c, 0x07, 0x7a, 0xca, 0x6d, 0xae, 0x29, 0xf7, 0x3f, 0x8b, 0x63, 0x09 }, + { 0xe0, 0xf4, 0x34, 0xbf, 0x22, 0xe3, 0x08, 0x80, 0x39, 0xc2, 0x1f, 0x71, 0x9f, 0xfc, 0x67, 0xf0, 0xf2, 0xcb, 0x5e, 0x98, 0xa7, 0xa0, 0x19, 0x4c, 0x76, 0xe9, 0x6b, 0xf4, 0xe8, 0xe1, 0x7e, 0x61 }, + { 0x27, 0x7c, 0x04, 0xe2, 0x85, 0x34, 0x84, 0xa4, 0xeb, 0xa9, 0x10, 0xad, 0x33, 0x6d, 0x01, 0xb4, 0x77, 0xb6, 0x7c, 0xc2, 0x00, 0xc5, 0x9f, 0x3c, 0x8d, 0x77, 0xee, 0xf8, 0x49, 0x4f, 0x29, 0xcd }, + { 0x15, 0x6d, 0x57, 0x47, 0xd0, 0xc9, 0x9c, 0x7f, 0x27, 0x09, 0x7d, 0x7b, 0x7e, 0x00, 0x2b, 0x2e, 0x18, 0x5c, 0xb7, 0x2d, 0x8d, 0xd7, 0xeb, 0x42, 0x4a, 0x03, 0x21, 0x52, 0x81, 0x61, 0x21, 0x9f }, + { 0x20, 0xdd, 0xd1, 0xed, 0x9b, 0x1c, 0xa8, 0x03, 0x94, 0x6d, 0x64, 0xa8, 0x3a, 0xe4, 0x65, 0x9d, 0xa6, 0x7f, 0xba, 0x7a, 0x1a, 0x3e, 0xdd, 0xb1, 0xe1, 0x03, 0xc0, 0xf5, 0xe0, 0x3e, 0x3a, 0x2c }, + { 0xf0, 0xaf, 0x60, 0x4d, 0x3d, 0xab, 0xbf, 0x9a, 0x0f, 0x2a, 0x7d, 0x3d, 0xda, 0x6b, 0xd3, 0x8b, 0xba, 0x72, 0xc6, 0xd0, 0x9b, 0xe4, 0x94, 0xfc, 0xef, 0x71, 0x3f, 0xf1, 0x01, 0x89, 0xb6, 0xe6 }, + { 0x98, 0x02, 0xbb, 0x87, 0xde, 0xf4, 0xcc, 0x10, 0xc4, 0xa5, 0xfd, 0x49, 0xaa, 0x58, 0xdf, 0xe2, 0xf3, 0xfd, 0xdb, 0x46, 0xb4, 0x70, 0x88, 0x14, 0xea, 0xd8, 0x1d, 0x23, 0xba, 0x95, 0x13, 0x9b }, + { 0x4f, 0x8c, 0xe1, 0xe5, 0x1d, 0x2f, 0xe7, 0xf2, 0x40, 0x43, 0xa9, 0x04, 0xd8, 0x98, 0xeb, 0xfc, 0x91, 0x97, 0x54, 0x18, 0x75, 0x34, 0x13, 0xaa, 0x09, 0x9b, 0x79, 0x5e, 0xcb, 0x35, 0xce, 0xdb }, + { 0xbd, 0xdc, 0x65, 0x14, 0xd7, 0xee, 0x6a, 0xce, 0x0a, 0x4a, 0xc1, 0xd0, 0xe0, 0x68, 0x11, 0x22, 0x88, 0xcb, 0xcf, 0x56, 0x04, 0x54, 0x64, 0x27, 0x05, 0x63, 0x01, 0x77, 0xcb, 0xa6, 0x08, 0xbd }, + { 0xd6, 0x35, 0x99, 0x4f, 0x62, 0x91, 0x51, 0x7b, 0x02, 0x81, 0xff, 0xdd, 0x49, 0x6a, 0xfa, 0x86, 0x27, 0x12, 0xe5, 0xb3, 0xc4, 0xe5, 0x2e, 0x4c, 0xd5, 0xfd, 0xae, 0x8c, 0x0e, 0x72, 0xfb, 0x08 }, + { 0x87, 0x8d, 0x9c, 0xa6, 0x00, 0xcf, 0x87, 0xe7, 0x69, 0xcc, 0x30, 0x5c, 0x1b, 0x35, 0x25, 0x51, 0x86, 0x61, 0x5a, 0x73, 0xa0, 0xda, 0x61, 0x3b, 0x5f, 0x1c, 0x98, 0xdb, 0xf8, 0x12, 0x83, 0xea }, + { 0xa6, 0x4e, 0xbe, 0x5d, 0xc1, 0x85, 0xde, 0x9f, 0xdd, 0xe7, 0x60, 0x7b, 0x69, 0x98, 0x70, 0x2e, 0xb2, 0x34, 0x56, 0x18, 0x49, 0x57, 0x30, 0x7d, 0x2f, 0xa7, 0x2e, 0x87, 0xa4, 0x77, 0x02, 0xd6 }, + { 0xce, 0x50, 0xea, 0xb7, 0xb5, 0xeb, 0x52, 0xbd, 0xc9, 0xad, 0x8e, 0x5a, 0x48, 0x0a, 0xb7, 0x80, 0xca, 0x93, 0x20, 0xe4, 0x43, 0x60, 0xb1, 0xfe, 0x37, 0xe0, 0x3f, 0x2f, 0x7a, 0xd7, 0xde, 0x01 }, + { 0xee, 0xdd, 0xb7, 0xc0, 0xdb, 0x6e, 0x30, 0xab, 0xe6, 0x6d, 0x79, 0xe3, 0x27, 0x51, 0x1e, 0x61, 0xfc, 0xeb, 0xbc, 0x29, 0xf1, 0x59, 0xb4, 0x0a, 0x86, 0xb0, 0x46, 0xec, 0xf0, 0x51, 0x38, 0x23 }, + { 0x78, 0x7f, 0xc9, 0x34, 0x40, 0xc1, 0xec, 0x96, 0xb5, 0xad, 0x01, 0xc1, 0x6c, 0xf7, 0x79, 0x16, 0xa1, 0x40, 0x5f, 0x94, 0x26, 0x35, 0x6e, 0xc9, 0x21, 0xd8, 0xdf, 0xf3, 0xea, 0x63, 0xb7, 0xe0 }, + { 0x7f, 0x0d, 0x5e, 0xab, 0x47, 0xee, 0xfd, 0xa6, 0x96, 0xc0, 0xbf, 0x0f, 0xbf, 0x86, 0xab, 0x21, 0x6f, 0xce, 0x46, 0x1e, 0x93, 0x03, 0xab, 0xa6, 0xac, 0x37, 0x41, 0x20, 0xe8, 0x90, 0xe8, 0xdf }, + { 0xb6, 0x80, 0x04, 0xb4, 0x2f, 0x14, 0xad, 0x02, 0x9f, 0x4c, 0x2e, 0x03, 0xb1, 0xd5, 0xeb, 0x76, 0xd5, 0x71, 0x60, 0xe2, 0x64, 0x76, 0xd2, 0x11, 0x31, 0xbe, 0xf2, 0x0a, 0xda, 0x7d, 0x27, 0xf4 }, + { 0xb0, 0xc4, 0xeb, 0x18, 0xae, 0x25, 0x0b, 0x51, 0xa4, 0x13, 0x82, 0xea, 0xd9, 0x2d, 0x0d, 0xc7, 0x45, 0x5f, 0x93, 0x79, 0xfc, 0x98, 0x84, 0x42, 0x8e, 0x47, 0x70, 0x60, 0x8d, 0xb0, 0xfa, 0xec }, + { 0xf9, 0x2b, 0x7a, 0x87, 0x0c, 0x05, 0x9f, 0x4d, 0x46, 0x46, 0x4c, 0x82, 0x4e, 0xc9, 0x63, 0x55, 0x14, 0x0b, 0xdc, 0xe6, 0x81, 0x32, 0x2c, 0xc3, 0xa9, 0x92, 0xff, 0x10, 0x3e, 0x3f, 0xea, 0x52 }, + { 0x53, 0x64, 0x31, 0x26, 0x14, 0x81, 0x33, 0x98, 0xcc, 0x52, 0x5d, 0x4c, 0x4e, 0x14, 0x6e, 0xde, 0xb3, 0x71, 0x26, 0x5f, 0xba, 0x19, 0x13, 0x3a, 0x2c, 0x3d, 0x21, 0x59, 0x29, 0x8a, 0x17, 0x42 }, + { 0xf6, 0x62, 0x0e, 0x68, 0xd3, 0x7f, 0xb2, 0xaf, 0x50, 0x00, 0xfc, 0x28, 0xe2, 0x3b, 0x83, 0x22, 0x97, 0xec, 0xd8, 0xbc, 0xe9, 0x9e, 0x8b, 0xe4, 0xd0, 0x4e, 0x85, 0x30, 0x9e, 0x3d, 0x33, 0x74 }, + { 0x53, 0x16, 0xa2, 0x79, 0x69, 0xd7, 0xfe, 0x04, 0xff, 0x27, 0xb2, 0x83, 0x96, 0x1b, 0xff, 0xc3, 0xbf, 0x5d, 0xfb, 0x32, 0xfb, 0x6a, 0x89, 0xd1, 0x01, 0xc6, 0xc3, 0xb1, 0x93, 0x7c, 0x28, 0x71 }, + { 0x81, 0xd1, 0x66, 0x4f, 0xdf, 0x3c, 0xb3, 0x3c, 0x24, 0xee, 0xba, 0xc0, 0xbd, 0x64, 0x24, 0x4b, 0x77, 0xc4, 0xab, 0xea, 0x90, 0xbb, 0xe8, 0xb5, 0xee, 0x0b, 0x2a, 0xaf, 0xcf, 0x2d, 0x6a, 0x53 }, + { 0x34, 0x57, 0x82, 0xf2, 0x95, 0xb0, 0x88, 0x03, 0x52, 0xe9, 0x24, 0xa0, 0x46, 0x7b, 0x5f, 0xbc, 0x3e, 0x8f, 0x3b, 0xfb, 0xc3, 0xc7, 0xe4, 0x8b, 0x67, 0x09, 0x1f, 0xb5, 0xe8, 0x0a, 0x94, 0x42 }, + { 0x79, 0x41, 0x11, 0xea, 0x6c, 0xd6, 0x5e, 0x31, 0x1f, 0x74, 0xee, 0x41, 0xd4, 0x76, 0xcb, 0x63, 0x2c, 0xe1, 0xe4, 0xb0, 0x51, 0xdc, 0x1d, 0x9e, 0x9d, 0x06, 0x1a, 0x19, 0xe1, 0xd0, 0xbb, 0x49 }, + { 0x2a, 0x85, 0xda, 0xf6, 0x13, 0x88, 0x16, 0xb9, 0x9b, 0xf8, 0xd0, 0x8b, 0xa2, 0x11, 0x4b, 0x7a, 0xb0, 0x79, 0x75, 0xa7, 0x84, 0x20, 0xc1, 0xa3, 0xb0, 0x6a, 0x77, 0x7c, 0x22, 0xdd, 0x8b, 0xcb }, + { 0x89, 0xb0, 0xd5, 0xf2, 0x89, 0xec, 0x16, 0x40, 0x1a, 0x06, 0x9a, 0x96, 0x0d, 0x0b, 0x09, 0x3e, 0x62, 0x5d, 0xa3, 0xcf, 0x41, 0xee, 0x29, 0xb5, 0x9b, 0x93, 0x0c, 0x58, 0x20, 0x14, 0x54, 0x55 }, + { 0xd0, 0xfd, 0xcb, 0x54, 0x39, 0x43, 0xfc, 0x27, 0xd2, 0x08, 0x64, 0xf5, 0x21, 0x81, 0x47, 0x1b, 0x94, 0x2c, 0xc7, 0x7c, 0xa6, 0x75, 0xbc, 0xb3, 0x0d, 0xf3, 0x1d, 0x35, 0x8e, 0xf7, 0xb1, 0xeb }, + { 0xb1, 0x7e, 0xa8, 0xd7, 0x70, 0x63, 0xc7, 0x09, 0xd4, 0xdc, 0x6b, 0x87, 0x94, 0x13, 0xc3, 0x43, 0xe3, 0x79, 0x0e, 0x9e, 0x62, 0xca, 0x85, 0xb7, 0x90, 0x0b, 0x08, 0x6f, 0x6b, 0x75, 0xc6, 0x72 }, + { 0xe7, 0x1a, 0x3e, 0x2c, 0x27, 0x4d, 0xb8, 0x42, 0xd9, 0x21, 0x14, 0xf2, 0x17, 0xe2, 0xc0, 0xea, 0xc8, 0xb4, 0x50, 0x93, 0xfd, 0xfd, 0x9d, 0xf4, 0xca, 0x71, 0x62, 0x39, 0x48, 0x62, 0xd5, 0x01 }, + { 0xc0, 0x47, 0x67, 0x59, 0xab, 0x7a, 0xa3, 0x33, 0x23, 0x4f, 0x6b, 0x44, 0xf5, 0xfd, 0x85, 0x83, 0x90, 0xec, 0x23, 0x69, 0x4c, 0x62, 0x2c, 0xb9, 0x86, 0xe7, 0x69, 0xc7, 0x8e, 0xdd, 0x73, 0x3e }, + { 0x9a, 0xb8, 0xea, 0xbb, 0x14, 0x16, 0x43, 0x4d, 0x85, 0x39, 0x13, 0x41, 0xd5, 0x69, 0x93, 0xc5, 0x54, 0x58, 0x16, 0x7d, 0x44, 0x18, 0xb1, 0x9a, 0x0f, 0x2a, 0xd8, 0xb7, 0x9a, 0x83, 0xa7, 0x5b }, + { 0x79, 0x92, 0xd0, 0xbb, 0xb1, 0x5e, 0x23, 0x82, 0x6f, 0x44, 0x3e, 0x00, 0x50, 0x5d, 0x68, 0xd3, 0xed, 0x73, 0x72, 0x99, 0x5a, 0x5c, 0x3e, 0x49, 0x86, 0x54, 0x10, 0x2f, 0xbc, 0xd0, 0x96, 0x4e }, + { 0xc0, 0x21, 0xb3, 0x00, 0x85, 0x15, 0x14, 0x35, 0xdf, 0x33, 0xb0, 0x07, 0xcc, 0xec, 0xc6, 0x9d, 0xf1, 0x26, 0x9f, 0x39, 0xba, 0x25, 0x09, 0x2b, 0xed, 0x59, 0xd9, 0x32, 0xac, 0x0f, 0xdc, 0x28 }, + { 0x91, 0xa2, 0x5e, 0xc0, 0xec, 0x0d, 0x9a, 0x56, 0x7f, 0x89, 0xc4, 0xbf, 0xe1, 0xa6, 0x5a, 0x0e, 0x43, 0x2d, 0x07, 0x06, 0x4b, 0x41, 0x90, 0xe2, 0x7d, 0xfb, 0x81, 0x90, 0x1f, 0xd3, 0x13, 0x9b }, + { 0x59, 0x50, 0xd3, 0x9a, 0x23, 0xe1, 0x54, 0x5f, 0x30, 0x12, 0x70, 0xaa, 0x1a, 0x12, 0xf2, 0xe6, 0xc4, 0x53, 0x77, 0x6e, 0x4d, 0x63, 0x55, 0xde, 0x42, 0x5c, 0xc1, 0x53, 0xf9, 0x81, 0x88, 0x67 }, + { 0xd7, 0x9f, 0x14, 0x72, 0x0c, 0x61, 0x0a, 0xf1, 0x79, 0xa3, 0x76, 0x5d, 0x4b, 0x7c, 0x09, 0x68, 0xf9, 0x77, 0x96, 0x2d, 0xbf, 0x65, 0x5b, 0x52, 0x12, 0x72, 0xb6, 0xf1, 0xe1, 0x94, 0x48, 0x8e }, + { 0xe9, 0x53, 0x1b, 0xfc, 0x8b, 0x02, 0x99, 0x5a, 0xea, 0xa7, 0x5b, 0xa2, 0x70, 0x31, 0xfa, 0xdb, 0xcb, 0xf4, 0xa0, 0xda, 0xb8, 0x96, 0x1d, 0x92, 0x96, 0xcd, 0x7e, 0x84, 0xd2, 0x5d, 0x60, 0x06 }, + { 0x34, 0xe9, 0xc2, 0x6a, 0x01, 0xd7, 0xf1, 0x61, 0x81, 0xb4, 0x54, 0xa9, 0xd1, 0x62, 0x3c, 0x23, 0x3c, 0xb9, 0x9d, 0x31, 0xc6, 0x94, 0x65, 0x6e, 0x94, 0x13, 0xac, 0xa3, 0xe9, 0x18, 0x69, 0x2f }, + { 0xd9, 0xd7, 0x42, 0x2f, 0x43, 0x7b, 0xd4, 0x39, 0xdd, 0xd4, 0xd8, 0x83, 0xda, 0xe2, 0xa0, 0x83, 0x50, 0x17, 0x34, 0x14, 0xbe, 0x78, 0x15, 0x51, 0x33, 0xff, 0xf1, 0x96, 0x4c, 0x3d, 0x79, 0x72 }, + { 0x4a, 0xee, 0x0c, 0x7a, 0xaf, 0x07, 0x54, 0x14, 0xff, 0x17, 0x93, 0xea, 0xd7, 0xea, 0xca, 0x60, 0x17, 0x75, 0xc6, 0x15, 0xdb, 0xd6, 0x0b, 0x64, 0x0b, 0x0a, 0x9f, 0x0c, 0xe5, 0x05, 0xd4, 0x35 }, + { 0x6b, 0xfd, 0xd1, 0x54, 0x59, 0xc8, 0x3b, 0x99, 0xf0, 0x96, 0xbf, 0xb4, 0x9e, 0xe8, 0x7b, 0x06, 0x3d, 0x69, 0xc1, 0x97, 0x4c, 0x69, 0x28, 0xac, 0xfc, 0xfb, 0x40, 0x99, 0xf8, 0xc4, 0xef, 0x67 }, + { 0x9f, 0xd1, 0xc4, 0x08, 0xfd, 0x75, 0xc3, 0x36, 0x19, 0x3a, 0x2a, 0x14, 0xd9, 0x4f, 0x6a, 0xf5, 0xad, 0xf0, 0x50, 0xb8, 0x03, 0x87, 0xb4, 0xb0, 0x10, 0xfb, 0x29, 0xf4, 0xcc, 0x72, 0x70, 0x7c }, + { 0x13, 0xc8, 0x84, 0x80, 0xa5, 0xd0, 0x0d, 0x6c, 0x8c, 0x7a, 0xd2, 0x11, 0x0d, 0x76, 0xa8, 0x2d, 0x9b, 0x70, 0xf4, 0xfa, 0x66, 0x96, 0xd4, 0xe5, 0xdd, 0x42, 0xa0, 0x66, 0xdc, 0xaf, 0x99, 0x20 }, + { 0x82, 0x0e, 0x72, 0x5e, 0xe2, 0x5f, 0xe8, 0xfd, 0x3a, 0x8d, 0x5a, 0xbe, 0x4c, 0x46, 0xc3, 0xba, 0x88, 0x9d, 0xe6, 0xfa, 0x91, 0x91, 0xaa, 0x22, 0xba, 0x67, 0xd5, 0x70, 0x54, 0x21, 0x54, 0x2b }, + { 0x32, 0xd9, 0x3a, 0x0e, 0xb0, 0x2f, 0x42, 0xfb, 0xbc, 0xaf, 0x2b, 0xad, 0x00, 0x85, 0xb2, 0x82, 0xe4, 0x60, 0x46, 0xa4, 0xdf, 0x7a, 0xd1, 0x06, 0x57, 0xc9, 0xd6, 0x47, 0x63, 0x75, 0xb9, 0x3e }, + { 0xad, 0xc5, 0x18, 0x79, 0x05, 0xb1, 0x66, 0x9c, 0xd8, 0xec, 0x9c, 0x72, 0x1e, 0x19, 0x53, 0x78, 0x6b, 0x9d, 0x89, 0xa9, 0xba, 0xe3, 0x07, 0x80, 0xf1, 0xe1, 0xea, 0xb2, 0x4a, 0x00, 0x52, 0x3c }, + { 0xe9, 0x07, 0x56, 0xff, 0x7f, 0x9a, 0xd8, 0x10, 0xb2, 0x39, 0xa1, 0x0c, 0xed, 0x2c, 0xf9, 0xb2, 0x28, 0x43, 0x54, 0xc1, 0xf8, 0xc7, 0xe0, 0xac, 0xcc, 0x24, 0x61, 0xdc, 0x79, 0x6d, 0x6e, 0x89 }, + { 0x12, 0x51, 0xf7, 0x6e, 0x56, 0x97, 0x84, 0x81, 0x87, 0x53, 0x59, 0x80, 0x1d, 0xb5, 0x89, 0xa0, 0xb2, 0x2f, 0x86, 0xd8, 0xd6, 0x34, 0xdc, 0x04, 0x50, 0x6f, 0x32, 0x2e, 0xd7, 0x8f, 0x17, 0xe8 }, + { 0x3a, 0xfa, 0x89, 0x9f, 0xd9, 0x80, 0xe7, 0x3e, 0xcb, 0x7f, 0x4d, 0x8b, 0x8f, 0x29, 0x1d, 0xc9, 0xaf, 0x79, 0x6b, 0xc6, 0x5d, 0x27, 0xf9, 0x74, 0xc6, 0xf1, 0x93, 0xc9, 0x19, 0x1a, 0x09, 0xfd }, + { 0xaa, 0x30, 0x5b, 0xe2, 0x6e, 0x5d, 0xed, 0xdc, 0x3c, 0x10, 0x10, 0xcb, 0xc2, 0x13, 0xf9, 0x5f, 0x05, 0x1c, 0x78, 0x5c, 0x5b, 0x43, 0x1e, 0x6a, 0x7c, 0xd0, 0x48, 0xf1, 0x61, 0x78, 0x75, 0x28 }, + { 0x8e, 0xa1, 0x88, 0x4f, 0xf3, 0x2e, 0x9d, 0x10, 0xf0, 0x39, 0xb4, 0x07, 0xd0, 0xd4, 0x4e, 0x7e, 0x67, 0x0a, 0xbd, 0x88, 0x4a, 0xee, 0xe0, 0xfb, 0x75, 0x7a, 0xe9, 0x4e, 0xaa, 0x97, 0x37, 0x3d }, + { 0xd4, 0x82, 0xb2, 0x15, 0x5d, 0x4d, 0xec, 0x6b, 0x47, 0x36, 0xa1, 0xf1, 0x61, 0x7b, 0x53, 0xaa, 0xa3, 0x73, 0x10, 0x27, 0x7d, 0x3f, 0xef, 0x0c, 0x37, 0xad, 0x41, 0x76, 0x8f, 0xc2, 0x35, 0xb4 }, + { 0x4d, 0x41, 0x39, 0x71, 0x38, 0x7e, 0x7a, 0x88, 0x98, 0xa8, 0xdc, 0x2a, 0x27, 0x50, 0x07, 0x78, 0x53, 0x9e, 0xa2, 0x14, 0xa2, 0xdf, 0xe9, 0xb3, 0xd7, 0xe8, 0xeb, 0xdc, 0xe5, 0xcf, 0x3d, 0xb3 }, + { 0x69, 0x6e, 0x5d, 0x46, 0xe6, 0xc5, 0x7e, 0x87, 0x96, 0xe4, 0x73, 0x5d, 0x08, 0x91, 0x6e, 0x0b, 0x79, 0x29, 0xb3, 0xcf, 0x29, 0x8c, 0x29, 0x6d, 0x22, 0xe9, 0xd3, 0x01, 0x96, 0x53, 0x37, 0x1c }, + { 0x1f, 0x56, 0x47, 0xc1, 0xd3, 0xb0, 0x88, 0x22, 0x88, 0x85, 0x86, 0x5c, 0x89, 0x40, 0x90, 0x8b, 0xf4, 0x0d, 0x1a, 0x82, 0x72, 0x82, 0x19, 0x73, 0xb1, 0x60, 0x00, 0x8e, 0x7a, 0x3c, 0xe2, 0xeb }, + { 0xb6, 0xe7, 0x6c, 0x33, 0x0f, 0x02, 0x1a, 0x5b, 0xda, 0x65, 0x87, 0x50, 0x10, 0xb0, 0xed, 0xf0, 0x91, 0x26, 0xc0, 0xf5, 0x10, 0xea, 0x84, 0x90, 0x48, 0x19, 0x20, 0x03, 0xae, 0xf4, 0xc6, 0x1c }, + { 0x3c, 0xd9, 0x52, 0xa0, 0xbe, 0xad, 0xa4, 0x1a, 0xbb, 0x42, 0x4c, 0xe4, 0x7f, 0x94, 0xb4, 0x2b, 0xe6, 0x4e, 0x1f, 0xfb, 0x0f, 0xd0, 0x78, 0x22, 0x76, 0x80, 0x79, 0x46, 0xd0, 0xd0, 0xbc, 0x55 }, + { 0x98, 0xd9, 0x26, 0x77, 0x43, 0x9b, 0x41, 0xb7, 0xbb, 0x51, 0x33, 0x12, 0xaf, 0xb9, 0x2b, 0xcc, 0x8e, 0xe9, 0x68, 0xb2, 0xe3, 0xb2, 0x38, 0xce, 0xcb, 0x9b, 0x0f, 0x34, 0xc9, 0xbb, 0x63, 0xd0 }, + { 0xec, 0xbc, 0xa2, 0xcf, 0x08, 0xae, 0x57, 0xd5, 0x17, 0xad, 0x16, 0x15, 0x8a, 0x32, 0xbf, 0xa7, 0xdc, 0x03, 0x82, 0xea, 0xed, 0xa1, 0x28, 0xe9, 0x18, 0x86, 0x73, 0x4c, 0x24, 0xa0, 0xb2, 0x9d }, + { 0x94, 0x2c, 0xc7, 0xc0, 0xb5, 0x2e, 0x2b, 0x16, 0xa4, 0xb8, 0x9f, 0xa4, 0xfc, 0x7e, 0x0b, 0xf6, 0x09, 0xe2, 0x9a, 0x08, 0xc1, 0xa8, 0x54, 0x34, 0x52, 0xb7, 0x7c, 0x7b, 0xfd, 0x11, 0xbb, 0x28 }, + { 0x8a, 0x06, 0x5d, 0x8b, 0x61, 0xa0, 0xdf, 0xfb, 0x17, 0x0d, 0x56, 0x27, 0x73, 0x5a, 0x76, 0xb0, 0xe9, 0x50, 0x60, 0x37, 0x80, 0x8c, 0xba, 0x16, 0xc3, 0x45, 0x00, 0x7c, 0x9f, 0x79, 0xcf, 0x8f }, + { 0x1b, 0x9f, 0xa1, 0x97, 0x14, 0x65, 0x9c, 0x78, 0xff, 0x41, 0x38, 0x71, 0x84, 0x92, 0x15, 0x36, 0x10, 0x29, 0xac, 0x80, 0x2b, 0x1c, 0xbc, 0xd5, 0x4e, 0x40, 0x8b, 0xd8, 0x72, 0x87, 0xf8, 0x1f }, + { 0x8d, 0xab, 0x07, 0x1b, 0xcd, 0x6c, 0x72, 0x92, 0xa9, 0xef, 0x72, 0x7b, 0x4a, 0xe0, 0xd8, 0x67, 0x13, 0x30, 0x1d, 0xa8, 0x61, 0x8d, 0x9a, 0x48, 0xad, 0xce, 0x55, 0xf3, 0x03, 0xa8, 0x69, 0xa1 }, + { 0x82, 0x53, 0xe3, 0xe7, 0xc7, 0xb6, 0x84, 0xb9, 0xcb, 0x2b, 0xeb, 0x01, 0x4c, 0xe3, 0x30, 0xff, 0x3d, 0x99, 0xd1, 0x7a, 0xbb, 0xdb, 0xab, 0xe4, 0xf4, 0xd6, 0x74, 0xde, 0xd5, 0x3f, 0xfc, 0x6b }, + { 0xf1, 0x95, 0xf3, 0x21, 0xe9, 0xe3, 0xd6, 0xbd, 0x7d, 0x07, 0x45, 0x04, 0xdd, 0x2a, 0xb0, 0xe6, 0x24, 0x1f, 0x92, 0xe7, 0x84, 0xb1, 0xaa, 0x27, 0x1f, 0xf6, 0x48, 0xb1, 0xca, 0xb6, 0xd7, 0xf6 }, + { 0x27, 0xe4, 0xcc, 0x72, 0x09, 0x0f, 0x24, 0x12, 0x66, 0x47, 0x6a, 0x7c, 0x09, 0x49, 0x5f, 0x2d, 0xb1, 0x53, 0xd5, 0xbc, 0xbd, 0x76, 0x19, 0x03, 0xef, 0x79, 0x27, 0x5e, 0xc5, 0x6b, 0x2e, 0xd8 }, + { 0x89, 0x9c, 0x24, 0x05, 0x78, 0x8e, 0x25, 0xb9, 0x9a, 0x18, 0x46, 0x35, 0x5e, 0x64, 0x6d, 0x77, 0xcf, 0x40, 0x00, 0x83, 0x41, 0x5f, 0x7d, 0xc5, 0xaf, 0xe6, 0x9d, 0x6e, 0x17, 0xc0, 0x00, 0x23 }, + { 0xa5, 0x9b, 0x78, 0xc4, 0x90, 0x57, 0x44, 0x07, 0x6b, 0xfe, 0xe8, 0x94, 0xde, 0x70, 0x7d, 0x4f, 0x12, 0x0b, 0x5c, 0x68, 0x93, 0xea, 0x04, 0x00, 0x29, 0x7d, 0x0b, 0xb8, 0x34, 0x72, 0x76, 0x32 }, + { 0x59, 0xdc, 0x78, 0xb1, 0x05, 0x64, 0x97, 0x07, 0xa2, 0xbb, 0x44, 0x19, 0xc4, 0x8f, 0x00, 0x54, 0x00, 0xd3, 0x97, 0x3d, 0xe3, 0x73, 0x66, 0x10, 0x23, 0x04, 0x35, 0xb1, 0x04, 0x24, 0xb2, 0x4f }, + { 0xc0, 0x14, 0x9d, 0x1d, 0x7e, 0x7a, 0x63, 0x53, 0xa6, 0xd9, 0x06, 0xef, 0xe7, 0x28, 0xf2, 0xf3, 0x29, 0xfe, 0x14, 0xa4, 0x14, 0x9a, 0x3e, 0xa7, 0x76, 0x09, 0xbc, 0x42, 0xb9, 0x75, 0xdd, 0xfa }, + { 0xa3, 0x2f, 0x24, 0x14, 0x74, 0xa6, 0xc1, 0x69, 0x32, 0xe9, 0x24, 0x3b, 0xe0, 0xcf, 0x09, 0xbc, 0xdc, 0x7e, 0x0c, 0xa0, 0xe7, 0xa6, 0xa1, 0xb9, 0xb1, 0xa0, 0xf0, 0x1e, 0x41, 0x50, 0x23, 0x77 }, + { 0xb2, 0x39, 0xb2, 0xe4, 0xf8, 0x18, 0x41, 0x36, 0x1c, 0x13, 0x39, 0xf6, 0x8e, 0x2c, 0x35, 0x9f, 0x92, 0x9a, 0xf9, 0xad, 0x9f, 0x34, 0xe0, 0x1a, 0xab, 0x46, 0x31, 0xad, 0x6d, 0x55, 0x00, 0xb0 }, + { 0x85, 0xfb, 0x41, 0x9c, 0x70, 0x02, 0xa3, 0xe0, 0xb4, 0xb6, 0xea, 0x09, 0x3b, 0x4c, 0x1a, 0xc6, 0x93, 0x66, 0x45, 0xb6, 0x5d, 0xac, 0x5a, 0xc1, 0x5a, 0x85, 0x28, 0xb7, 0xb9, 0x4c, 0x17, 0x54 }, + { 0x96, 0x19, 0x72, 0x06, 0x25, 0xf1, 0x90, 0xb9, 0x3a, 0x3f, 0xad, 0x18, 0x6a, 0xb3, 0x14, 0x18, 0x96, 0x33, 0xc0, 0xd3, 0xa0, 0x1e, 0x6f, 0x9b, 0xc8, 0xc4, 0xa8, 0xf8, 0x2f, 0x38, 0x3d, 0xbf }, + { 0x7d, 0x62, 0x0d, 0x90, 0xfe, 0x69, 0xfa, 0x46, 0x9a, 0x65, 0x38, 0x38, 0x89, 0x70, 0xa1, 0xaa, 0x09, 0xbb, 0x48, 0xa2, 0xd5, 0x9b, 0x34, 0x7b, 0x97, 0xe8, 0xce, 0x71, 0xf4, 0x8c, 0x7f, 0x46 }, + { 0x29, 0x43, 0x83, 0x56, 0x85, 0x96, 0xfb, 0x37, 0xc7, 0x5b, 0xba, 0xcd, 0x97, 0x9c, 0x5f, 0xf6, 0xf2, 0x0a, 0x55, 0x6b, 0xf8, 0x87, 0x9c, 0xc7, 0x29, 0x24, 0x85, 0x5d, 0xf9, 0xb8, 0x24, 0x0e }, + { 0x16, 0xb1, 0x8a, 0xb3, 0x14, 0x35, 0x9c, 0x2b, 0x83, 0x3c, 0x1c, 0x69, 0x86, 0xd4, 0x8c, 0x55, 0xa9, 0xfc, 0x97, 0xcd, 0xe9, 0xa3, 0xc1, 0xf1, 0x0a, 0x31, 0x77, 0x14, 0x0f, 0x73, 0xf7, 0x38 }, + { 0x8c, 0xbb, 0xdd, 0x14, 0xbc, 0x33, 0xf0, 0x4c, 0xf4, 0x58, 0x13, 0xe4, 0xa1, 0x53, 0xa2, 0x73, 0xd3, 0x6a, 0xda, 0xd5, 0xce, 0x71, 0xf4, 0x99, 0xee, 0xb8, 0x7f, 0xb8, 0xac, 0x63, 0xb7, 0x29 }, + { 0x69, 0xc9, 0xa4, 0x98, 0xdb, 0x17, 0x4e, 0xca, 0xef, 0xcc, 0x5a, 0x3a, 0xc9, 0xfd, 0xed, 0xf0, 0xf8, 0x13, 0xa5, 0xbe, 0xc7, 0x27, 0xf1, 0xe7, 0x75, 0xba, 0xbd, 0xec, 0x77, 0x18, 0x81, 0x6e }, + { 0xb4, 0x62, 0xc3, 0xbe, 0x40, 0x44, 0x8f, 0x1d, 0x4f, 0x80, 0x62, 0x62, 0x54, 0xe5, 0x35, 0xb0, 0x8b, 0xc9, 0xcd, 0xcf, 0xf5, 0x99, 0xa7, 0x68, 0x57, 0x8d, 0x4b, 0x28, 0x81, 0xa8, 0xe3, 0xf0 }, + { 0x55, 0x3e, 0x9d, 0x9c, 0x5f, 0x36, 0x0a, 0xc0, 0xb7, 0x4a, 0x7d, 0x44, 0xe5, 0xa3, 0x91, 0xda, 0xd4, 0xce, 0xd0, 0x3e, 0x0c, 0x24, 0x18, 0x3b, 0x7e, 0x8e, 0xca, 0xbd, 0xf1, 0x71, 0x5a, 0x64 }, + { 0x7a, 0x7c, 0x55, 0xa5, 0x6f, 0xa9, 0xae, 0x51, 0xe6, 0x55, 0xe0, 0x19, 0x75, 0xd8, 0xa6, 0xff, 0x4a, 0xe9, 0xe4, 0xb4, 0x86, 0xfc, 0xbe, 0x4e, 0xac, 0x04, 0x45, 0x88, 0xf2, 0x45, 0xeb, 0xea }, + { 0x2a, 0xfd, 0xf3, 0xc8, 0x2a, 0xbc, 0x48, 0x67, 0xf5, 0xde, 0x11, 0x12, 0x86, 0xc2, 0xb3, 0xbe, 0x7d, 0x6e, 0x48, 0x65, 0x7b, 0xa9, 0x23, 0xcf, 0xbf, 0x10, 0x1a, 0x6d, 0xfc, 0xf9, 0xdb, 0x9a }, + { 0x41, 0x03, 0x7d, 0x2e, 0xdc, 0xdc, 0xe0, 0xc4, 0x9b, 0x7f, 0xb4, 0xa6, 0xaa, 0x09, 0x99, 0xca, 0x66, 0x97, 0x6c, 0x74, 0x83, 0xaf, 0xe6, 0x31, 0xd4, 0xed, 0xa2, 0x83, 0x14, 0x4f, 0x6d, 0xfc }, + { 0xc4, 0x46, 0x6f, 0x84, 0x97, 0xca, 0x2e, 0xeb, 0x45, 0x83, 0xa0, 0xb0, 0x8e, 0x9d, 0x9a, 0xc7, 0x43, 0x95, 0x70, 0x9f, 0xda, 0x10, 0x9d, 0x24, 0xf2, 0xe4, 0x46, 0x21, 0x96, 0x77, 0x9c, 0x5d }, + { 0x75, 0xf6, 0x09, 0x33, 0x8a, 0xa6, 0x7d, 0x96, 0x9a, 0x2a, 0xe2, 0xa2, 0x36, 0x2b, 0x2d, 0xa9, 0xd7, 0x7c, 0x69, 0x5d, 0xfd, 0x1d, 0xf7, 0x22, 0x4a, 0x69, 0x01, 0xdb, 0x93, 0x2c, 0x33, 0x64 }, + { 0x68, 0x60, 0x6c, 0xeb, 0x98, 0x9d, 0x54, 0x88, 0xfc, 0x7c, 0xf6, 0x49, 0xf3, 0xd7, 0xc2, 0x72, 0xef, 0x05, 0x5d, 0xa1, 0xa9, 0x3f, 0xae, 0xcd, 0x55, 0xfe, 0x06, 0xf6, 0x96, 0x70, 0x98, 0xca }, + { 0x44, 0x34, 0x6b, 0xde, 0xb7, 0xe0, 0x52, 0xf6, 0x25, 0x50, 0x48, 0xf0, 0xd9, 0xb4, 0x2c, 0x42, 0x5b, 0xab, 0x9c, 0x3d, 0xd2, 0x41, 0x68, 0x21, 0x2c, 0x3e, 0xcf, 0x1e, 0xbf, 0x34, 0xe6, 0xae }, + { 0x8e, 0x9c, 0xf6, 0xe1, 0xf3, 0x66, 0x47, 0x1f, 0x2a, 0xc7, 0xd2, 0xee, 0x9b, 0x5e, 0x62, 0x66, 0xfd, 0xa7, 0x1f, 0x8f, 0x2e, 0x41, 0x09, 0xf2, 0x23, 0x7e, 0xd5, 0xf8, 0x81, 0x3f, 0xc7, 0x18 }, + { 0x84, 0xbb, 0xeb, 0x84, 0x06, 0xd2, 0x50, 0x95, 0x1f, 0x8c, 0x1b, 0x3e, 0x86, 0xa7, 0xc0, 0x10, 0x08, 0x29, 0x21, 0x83, 0x3d, 0xfd, 0x95, 0x55, 0xa2, 0xf9, 0x09, 0xb1, 0x08, 0x6e, 0xb4, 0xb8 }, + { 0xee, 0x66, 0x6f, 0x3e, 0xef, 0x0f, 0x7e, 0x2a, 0x9c, 0x22, 0x29, 0x58, 0xc9, 0x7e, 0xaf, 0x35, 0xf5, 0x1c, 0xed, 0x39, 0x3d, 0x71, 0x44, 0x85, 0xab, 0x09, 0xa0, 0x69, 0x34, 0x0f, 0xdf, 0x88 }, + { 0xc1, 0x53, 0xd3, 0x4a, 0x65, 0xc4, 0x7b, 0x4a, 0x62, 0xc5, 0xca, 0xcf, 0x24, 0x01, 0x09, 0x75, 0xd0, 0x35, 0x6b, 0x2f, 0x32, 0xc8, 0xf5, 0xda, 0x53, 0x0d, 0x33, 0x88, 0x16, 0xad, 0x5d, 0xe6 }, + { 0x9f, 0xc5, 0x45, 0x01, 0x09, 0xe1, 0xb7, 0x79, 0xf6, 0xc7, 0xae, 0x79, 0xd5, 0x6c, 0x27, 0x63, 0x5c, 0x8d, 0xd4, 0x26, 0xc5, 0xa9, 0xd5, 0x4e, 0x25, 0x78, 0xdb, 0x98, 0x9b, 0x8c, 0x3b, 0x4e }, + { 0xd1, 0x2b, 0xf3, 0x73, 0x2e, 0xf4, 0xaf, 0x5c, 0x22, 0xfa, 0x90, 0x35, 0x6a, 0xf8, 0xfc, 0x50, 0xfc, 0xb4, 0x0f, 0x8f, 0x2e, 0xa5, 0xc8, 0x59, 0x47, 0x37, 0xa3, 0xb3, 0xd5, 0xab, 0xdb, 0xd7 }, + { 0x11, 0x03, 0x0b, 0x92, 0x89, 0xbb, 0xa5, 0xaf, 0x65, 0x26, 0x06, 0x72, 0xab, 0x6f, 0xee, 0x88, 0xb8, 0x74, 0x20, 0xac, 0xef, 0x4a, 0x17, 0x89, 0xa2, 0x07, 0x3b, 0x7e, 0xc2, 0xf2, 0xa0, 0x9e }, + { 0x69, 0xcb, 0x19, 0x2b, 0x84, 0x44, 0x00, 0x5c, 0x8c, 0x0c, 0xeb, 0x12, 0xc8, 0x46, 0x86, 0x07, 0x68, 0x18, 0x8c, 0xda, 0x0a, 0xec, 0x27, 0xa9, 0xc8, 0xa5, 0x5c, 0xde, 0xe2, 0x12, 0x36, 0x32 }, + { 0xdb, 0x44, 0x4c, 0x15, 0x59, 0x7b, 0x5f, 0x1a, 0x03, 0xd1, 0xf9, 0xed, 0xd1, 0x6e, 0x4a, 0x9f, 0x43, 0xa6, 0x67, 0xcc, 0x27, 0x51, 0x75, 0xdf, 0xa2, 0xb7, 0x04, 0xe3, 0xbb, 0x1a, 0x9b, 0x83 }, + { 0x3f, 0xb7, 0x35, 0x06, 0x1a, 0xbc, 0x51, 0x9d, 0xfe, 0x97, 0x9e, 0x54, 0xc1, 0xee, 0x5b, 0xfa, 0xd0, 0xa9, 0xd8, 0x58, 0xb3, 0x31, 0x5b, 0xad, 0x34, 0xbd, 0xe9, 0x99, 0xef, 0xd7, 0x24, 0xdd }, + }; + unsigned char inp[1000], out[1000]; + unsigned char key[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }; + unsigned long ilen, klen = sizeof(key), mlen = 32; + blake2smac_state st; + + for (ilen = 0; ilen < 256; ilen++) inp[ilen] = (unsigned char)ilen; + + for (ilen = 0; ilen < 256; ilen++) { + const unsigned char *mac = tests[ilen]; + unsigned long olen = mlen; + /* process piece by piece */ + if (ilen > 15) { + blake2smac_init(&st, olen, key, klen); + blake2smac_process(&st, (unsigned char*)inp, 5); + blake2smac_process(&st, (unsigned char*)inp + 5, 4); + blake2smac_process(&st, (unsigned char*)inp + 9, 3); + blake2smac_process(&st, (unsigned char*)inp + 12, 2); + blake2smac_process(&st, (unsigned char*)inp + 14, 1); + blake2smac_process(&st, (unsigned char*)inp + 15, ilen - 15); + blake2smac_done(&st, out, &olen); + if (compare_testvector(out, olen, mac, mlen, "BLAKE2S MAC multi", ilen) != 0) return CRYPT_FAIL_TESTVECTOR; + } + /* process in one go */ + blake2smac_init(&st, olen, key, klen); + blake2smac_process(&st, (unsigned char*)inp, ilen); + blake2smac_done(&st, out, &olen); + if (compare_testvector(out, olen, mac, mlen, "BLAKE2S MAC single", ilen) != 0) return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_done.c b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_done.c new file mode 100644 index 0000000..53ed13f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_done.c @@ -0,0 +1,65 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f9_done.c + f9 Support, terminate the state +*/ + +#ifdef LTC_F9_MODE + +/** Terminate the f9-MAC state + @param f9 f9 state to terminate + @param out [out] Destination for the MAC tag + @param outlen [in/out] Destination size and final tag size + Return CRYPT_OK on success +*/ +int f9_done(f9_state *f9, unsigned char *out, unsigned long *outlen) +{ + int err, x; + LTC_ARGCHK(f9 != NULL); + LTC_ARGCHK(out != NULL); + + /* check structure */ + if ((err = cipher_is_valid(f9->cipher)) != CRYPT_OK) { + return err; + } + + if ((f9->blocksize > cipher_descriptor[f9->cipher]->block_length) || (f9->blocksize < 0) || + (f9->buflen > f9->blocksize) || (f9->buflen < 0)) { + return CRYPT_INVALID_ARG; + } + + if (f9->buflen != 0) { + /* encrypt */ + cipher_descriptor[f9->cipher]->ecb_encrypt(f9->IV, f9->IV, &f9->key); + f9->buflen = 0; + for (x = 0; x < f9->blocksize; x++) { + f9->ACC[x] ^= f9->IV[x]; + } + } + + /* schedule modified key */ + if ((err = cipher_descriptor[f9->cipher]->setup(f9->akey, f9->keylen, 0, &f9->key)) != CRYPT_OK) { + return err; + } + + /* encrypt the ACC */ + cipher_descriptor[f9->cipher]->ecb_encrypt(f9->ACC, f9->ACC, &f9->key); + cipher_descriptor[f9->cipher]->done(&f9->key); + + /* extract tag */ + for (x = 0; x < f9->blocksize && (unsigned long)x < *outlen; x++) { + out[x] = f9->ACC[x]; + } + *outlen = x; + +#ifdef LTC_CLEAN_STACK + zeromem(f9, sizeof(*f9)); +#endif + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_file.c b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_file.c new file mode 100644 index 0000000..c06f548 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_file.c @@ -0,0 +1,87 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f9_file.c + f9 support, process a file, Tom St Denis +*/ + +#ifdef LTC_F9_MODE + +/** + f9 a file + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param fname The name of the file you wish to f9 + @param out [out] Where the authentication tag is to be stored + @param outlen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int f9_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *fname, + unsigned char *out, unsigned long *outlen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(cipher); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(fname); + LTC_UNUSED_PARAM(out); + LTC_UNUSED_PARAM(outlen); + return CRYPT_NOP; +#else + size_t x; + int err; + f9_state f9; + FILE *in; + unsigned char *buf; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(fname != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = f9_init(&f9, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(fname, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = f9_process(&f9, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = f9_done(&f9, out, outlen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&f9, sizeof(f9_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_init.c b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_init.c new file mode 100644 index 0000000..7a9ed55 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_init.c @@ -0,0 +1,58 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f9_init.c + F9 Support, start an F9 state +*/ + +#ifdef LTC_F9_MODE + +/** Initialize F9-MAC state + @param f9 [out] f9 state to initialize + @param cipher Index of cipher to use + @param key [in] Secret key + @param keylen Length of secret key in octets + Return CRYPT_OK on success +*/ +int f9_init(f9_state *f9, int cipher, const unsigned char *key, unsigned long keylen) +{ + int x, err; + + LTC_ARGCHK(f9 != NULL); + LTC_ARGCHK(key != NULL); + + /* schedule the key */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + +#ifdef LTC_FAST + if (cipher_descriptor[cipher]->block_length % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, &f9->key)) != CRYPT_OK) { + goto done; + } + + /* make the second key */ + for (x = 0; (unsigned)x < keylen; x++) { + f9->akey[x] = key[x] ^ 0xAA; + } + + /* setup struct */ + zeromem(f9->IV, cipher_descriptor[cipher]->block_length); + zeromem(f9->ACC, cipher_descriptor[cipher]->block_length); + f9->blocksize = cipher_descriptor[cipher]->block_length; + f9->cipher = cipher; + f9->buflen = 0; + f9->keylen = keylen; +done: + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_memory.c new file mode 100644 index 0000000..e55efcd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_memory.c @@ -0,0 +1,59 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f9_process.c + f9 Support, Process a block through F9-MAC +*/ + +#ifdef LTC_F9_MODE + +/** f9-MAC a block of memory + @param cipher Index of cipher to use + @param key [in] Secret key + @param keylen Length of key in octets + @param in [in] Message to MAC + @param inlen Length of input in octets + @param out [out] Destination for the MAC tag + @param outlen [in/out] Output size and final tag size + Return CRYPT_OK on success. +*/ +int f9_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + f9_state *f9; + int err; + + /* is the cipher valid? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* Use accelerator if found */ + if (cipher_descriptor[cipher]->f9_memory != NULL) { + return cipher_descriptor[cipher]->f9_memory(key, keylen, in, inlen, out, outlen); + } + + f9 = XCALLOC(1, sizeof(*f9)); + if (f9 == NULL) { + return CRYPT_MEM; + } + + if ((err = f9_init(f9, cipher, key, keylen)) != CRYPT_OK) { + goto done; + } + + if ((err = f9_process(f9, in, inlen)) != CRYPT_OK) { + goto done; + } + + err = f9_done(f9, out, outlen); +done: + XFREE(f9); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_memory_multi.c new file mode 100644 index 0000000..c97ffd3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_memory_multi.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file f9_memory_multi.c + f9 support, process multiple blocks of memory, Tom St Denis +*/ + +#ifdef LTC_F9_MODE + +/** + f9 multiple blocks of memory + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @param out [out] The destination of the authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag (octets) + @param in The data to send through f9 + @param inlen The length of the data to send through f9 (octets) + @param ... tuples of (data,len) pairs to f9, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int f9_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) +{ + int err; + f9_state *f9; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* allocate ram for f9 state */ + f9 = XMALLOC(sizeof(f9_state)); + if (f9 == NULL) { + return CRYPT_MEM; + } + + /* f9 process the message */ + if ((err = f9_init(f9, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + va_start(args, inlen); + curptr = in; + curlen = inlen; + for (;;) { + /* process buf */ + if ((err = f9_process(f9, curptr, curlen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* step to next */ + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) { + break; + } + curlen = va_arg(args, unsigned long); + } + if ((err = f9_done(f9, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(f9, sizeof(f9_state)); +#endif + XFREE(f9); + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_process.c b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_process.c new file mode 100644 index 0000000..c6ad743 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_process.c @@ -0,0 +1,66 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f9_process.c + f9 Support, process blocks with f9 +*/ + +#ifdef LTC_F9_MODE + +/** Process data through f9-MAC + @param f9 The f9-MAC state + @param in Input data to process + @param inlen Length of input in octets + Return CRYPT_OK on success +*/ +int f9_process(f9_state *f9, const unsigned char *in, unsigned long inlen) +{ + int err, x; + + LTC_ARGCHK(f9 != NULL); + LTC_ARGCHK(in != NULL); + + /* check structure */ + if ((err = cipher_is_valid(f9->cipher)) != CRYPT_OK) { + return err; + } + + if ((f9->blocksize > cipher_descriptor[f9->cipher]->block_length) || (f9->blocksize < 0) || + (f9->buflen > f9->blocksize) || (f9->buflen < 0)) { + return CRYPT_INVALID_ARG; + } + +#ifdef LTC_FAST + if (f9->buflen == 0) { + while (inlen >= (unsigned long)f9->blocksize) { + for (x = 0; x < f9->blocksize; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&(f9->IV[x]))) ^= *(LTC_FAST_TYPE_PTR_CAST(&(in[x]))); + } + cipher_descriptor[f9->cipher]->ecb_encrypt(f9->IV, f9->IV, &f9->key); + for (x = 0; x < f9->blocksize; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&(f9->ACC[x]))) ^= *(LTC_FAST_TYPE_PTR_CAST(&(f9->IV[x]))); + } + in += f9->blocksize; + inlen -= f9->blocksize; + } + } +#endif + + while (inlen) { + if (f9->buflen == f9->blocksize) { + cipher_descriptor[f9->cipher]->ecb_encrypt(f9->IV, f9->IV, &f9->key); + for (x = 0; x < f9->blocksize; x++) { + f9->ACC[x] ^= f9->IV[x]; + } + f9->buflen = 0; + } + f9->IV[f9->buflen++] ^= *in++; + --inlen; + } + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_test.c b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_test.c new file mode 100644 index 0000000..779fdf1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/f9/f9_test.c @@ -0,0 +1,66 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f9_test.c + f9 Support, Test F9 mode +*/ + +#ifdef LTC_F9_MODE + +/** Test f9-MAC mode + Return CRYPT_OK on success +*/ +int f9_test(void) +{ +#ifdef LTC_NO_TEST + return CRYPT_NOP; +#else + static const struct { + int msglen; + unsigned char K[16], M[128], T[4]; + } tests[] = { +{ + 20, + { 0x2B, 0xD6, 0x45, 0x9F, 0x82, 0xC5, 0xB3, 0x00, 0x95, 0x2C, 0x49, 0x10, 0x48, 0x81, 0xFF, 0x48 }, + { 0x38, 0xA6, 0xF0, 0x56, 0xB8, 0xAE, 0xFD, 0xA9, 0x33, 0x32, 0x34, 0x62, 0x63, 0x39, 0x38, 0x61, 0x37, 0x34, 0x79, 0x40 }, + { 0x46, 0xE0, 0x0D, 0x4B } +}, + +{ + 105, + { 0x83, 0xFD, 0x23, 0xA2, 0x44, 0xA7, 0x4C, 0xF3, 0x58, 0xDA, 0x30, 0x19, 0xF1, 0x72, 0x26, 0x35 }, + { 0x36, 0xAF, 0x61, 0x44, 0x4F, 0x30, 0x2A, 0xD2, + 0x35, 0xC6, 0x87, 0x16, 0x63, 0x3C, 0x66, 0xFB, 0x75, 0x0C, 0x26, 0x68, 0x65, 0xD5, 0x3C, 0x11, 0xEA, 0x05, 0xB1, 0xE9, 0xFA, 0x49, 0xC8, 0x39, 0x8D, 0x48, 0xE1, 0xEF, 0xA5, 0x90, 0x9D, 0x39, + 0x47, 0x90, 0x28, 0x37, 0xF5, 0xAE, 0x96, 0xD5, 0xA0, 0x5B, 0xC8, 0xD6, 0x1C, 0xA8, 0xDB, 0xEF, 0x1B, 0x13, 0xA4, 0xB4, 0xAB, 0xFE, 0x4F, 0xB1, 0x00, 0x60, 0x45, 0xB6, 0x74, 0xBB, 0x54, 0x72, + 0x93, 0x04, 0xC3, 0x82, 0xBE, 0x53, 0xA5, 0xAF, 0x05, 0x55, 0x61, 0x76, 0xF6, 0xEA, 0xA2, 0xEF, 0x1D, 0x05, 0xE4, 0xB0, 0x83, 0x18, 0x1E, 0xE6, 0x74, 0xCD, 0xA5, 0xA4, 0x85, 0xF7, 0x4D, 0x7A, + 0x40|0x80 }, + { 0x95, 0xAE, 0x41, 0xBA }, +} +}; + unsigned char T[16]; + unsigned long taglen; + int err, x, idx; + + /* find kasumi */ + if ((idx = find_cipher("kasumi")) == -1) { + return CRYPT_NOP; + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + taglen = 4; + if ((err = f9_memory(idx, tests[x].K, 16, tests[x].M, tests[x].msglen, T, &taglen)) != CRYPT_OK) { + return err; + } + if (compare_testvector(T, taglen, tests[x].T, 4, "F9", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + return CRYPT_OK; +#endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_done.c b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_done.c new file mode 100644 index 0000000..60df551 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_done.c @@ -0,0 +1,96 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hmac_done.c + HMAC support, terminate stream, Tom St Denis/Dobes Vandermeer +*/ + +#ifdef LTC_HMAC + +#define LTC_HMAC_BLOCKSIZE hash_descriptor[hash]->blocksize + +/** + Terminate an HMAC session + @param hmac The HMAC state + @param out [out] The destination of the HMAC authentication tag + @param outlen [in/out] The max size and resulting size of the HMAC authentication tag + @return CRYPT_OK if successful +*/ +int hmac_done(hmac_state *hmac, unsigned char *out, unsigned long *outlen) +{ + unsigned char *buf, *isha; + unsigned long hashsize, i; + int hash, err; + + LTC_ARGCHK(hmac != NULL); + LTC_ARGCHK(out != NULL); + + /* test hash */ + hash = hmac->hash; + if((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + + /* get the hash message digest size */ + hashsize = hash_descriptor[hash]->hashsize; + + /* allocate buffers */ + buf = XMALLOC(LTC_HMAC_BLOCKSIZE); + isha = XMALLOC(hashsize); + if (buf == NULL || isha == NULL) { + if (buf != NULL) { + XFREE(buf); + } + if (isha != NULL) { + XFREE(isha); + } + return CRYPT_MEM; + } + + /* Get the hash of the first HMAC vector plus the data */ + if ((err = hash_descriptor[hash]->done(&hmac->md, isha)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* Create the second HMAC vector vector for step (3) */ + for(i=0; i < LTC_HMAC_BLOCKSIZE; i++) { + buf[i] = hmac->key[i] ^ 0x5C; + } + + /* Now calculate the "outer" hash for step (5), (6), and (7) */ + if ((err = hash_descriptor[hash]->init(&hmac->md)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash]->process(&hmac->md, buf, LTC_HMAC_BLOCKSIZE)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash]->process(&hmac->md, isha, hashsize)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash]->done(&hmac->md, buf)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* copy to output */ + for (i = 0; i < hashsize && i < *outlen; i++) { + out[i] = buf[i]; + } + *outlen = i; + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(isha, hashsize); + zeromem(buf, hashsize); + zeromem(hmac, sizeof(*hmac)); +#endif + + XFREE(isha); + XFREE(buf); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_file.c b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_file.c new file mode 100644 index 0000000..c944a28 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_file.c @@ -0,0 +1,90 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hmac_file.c + HMAC support, process a file, Tom St Denis/Dobes Vandermeer +*/ + +#ifdef LTC_HMAC + +/** + HMAC a file + @param hash The index of the hash you wish to use + @param fname The name of the file you wish to HMAC + @param key The secret key + @param keylen The length of the secret key + @param out [out] The HMAC authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int hmac_file(int hash, const char *fname, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(hash); + LTC_UNUSED_PARAM(fname); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(out); + LTC_UNUSED_PARAM(outlen); + return CRYPT_NOP; +#else + hmac_state hmac; + FILE *in; + unsigned char *buf; + size_t x; + int err; + + LTC_ARGCHK(fname != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = hmac_init(&hmac, hash, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(fname, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = hmac_process(&hmac, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); /* we don't trap this error since we're already returning an error! */ + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = hmac_done(&hmac, out, outlen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&hmac, sizeof(hmac_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_init.c b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_init.c new file mode 100644 index 0000000..2904299 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_init.c @@ -0,0 +1,94 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hmac_init.c + HMAC support, initialize state, Tom St Denis/Dobes Vandermeer +*/ + +#ifdef LTC_HMAC + +#define LTC_HMAC_BLOCKSIZE hash_descriptor[hash]->blocksize + +/** + Initialize an HMAC context. + @param hmac The HMAC state + @param hash The index of the hash you want to use + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int hmac_init(hmac_state *hmac, int hash, const unsigned char *key, unsigned long keylen) +{ + unsigned char *buf; + unsigned long hashsize; + unsigned long i, z; + int err; + + LTC_ARGCHK(hmac != NULL); + LTC_ARGCHK(key != NULL); + + /* valid hash? */ + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + hmac->hash = hash; + hashsize = hash_descriptor[hash]->hashsize; + + /* valid key length? */ + if (keylen == 0) { + return CRYPT_INVALID_KEYSIZE; + } + + /* allocate ram for buf */ + buf = XMALLOC(LTC_HMAC_BLOCKSIZE); + if (buf == NULL) { + return CRYPT_MEM; + } + + /* check hash block fits */ + if (sizeof(hmac->key) < LTC_HMAC_BLOCKSIZE) { + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + /* (1) make sure we have a large enough key */ + if(keylen > LTC_HMAC_BLOCKSIZE) { + z = LTC_HMAC_BLOCKSIZE; + if ((err = hash_memory(hash, key, keylen, hmac->key, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + keylen = hashsize; + } else { + XMEMCPY(hmac->key, key, (size_t)keylen); + } + + if(keylen < LTC_HMAC_BLOCKSIZE) { + zeromem((hmac->key) + keylen, (size_t)(LTC_HMAC_BLOCKSIZE - keylen)); + } + + /* Create the initialization vector for step (3) */ + for(i=0; i < LTC_HMAC_BLOCKSIZE; i++) { + buf[i] = hmac->key[i] ^ 0x36; + } + + /* Pre-pend that to the hash data */ + if ((err = hash_descriptor[hash]->init(&hmac->md)) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = hash_descriptor[hash]->process(&hmac->md, buf, LTC_HMAC_BLOCKSIZE)) != CRYPT_OK) { + goto LBL_ERR; + } + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(buf, LTC_HMAC_BLOCKSIZE); +#endif + + XFREE(buf); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_memory.c new file mode 100644 index 0000000..5e7b847 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_memory.c @@ -0,0 +1,76 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hmac_memory.c + HMAC support, process a block of memory, Tom St Denis/Dobes Vandermeer +*/ + +#ifdef LTC_HMAC + +/** + HMAC a block of memory to produce the authentication tag + @param hash The index of the hash to use + @param key The secret key + @param keylen The length of the secret key (octets) + @param in The data to HMAC + @param inlen The length of the data to HMAC (octets) + @param out [out] Destination of the authentication tag + @param outlen [in/out] Max size and resulting size of authentication tag + @return CRYPT_OK if successful +*/ +int hmac_memory(int hash, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + hmac_state *hmac; + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* make sure hash descriptor is valid */ + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + + /* is there a descriptor? */ + if (hash_descriptor[hash]->hmac_block != NULL) { + return hash_descriptor[hash]->hmac_block(key, keylen, in, inlen, out, outlen); + } + + /* nope, so call the hmac functions */ + /* allocate ram for hmac state */ + hmac = XMALLOC(sizeof(hmac_state)); + if (hmac == NULL) { + return CRYPT_MEM; + } + + if ((err = hmac_init(hmac, hash, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = hmac_process(hmac, in, inlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = hmac_done(hmac, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(hmac, sizeof(hmac_state)); +#endif + + XFREE(hmac); + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_memory_multi.c new file mode 100644 index 0000000..2809d71 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_memory_multi.c @@ -0,0 +1,80 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file hmac_memory_multi.c + HMAC support, process multiple blocks of memory, Tom St Denis/Dobes Vandermeer +*/ + +#ifdef LTC_HMAC + +/** + HMAC multiple blocks of memory to produce the authentication tag + @param hash The index of the hash to use + @param key The secret key + @param keylen The length of the secret key (octets) + @param out [out] Destination of the authentication tag + @param outlen [in/out] Max size and resulting size of authentication tag + @param in The data to HMAC + @param inlen The length of the data to HMAC (octets) + @param ... tuples of (data,len) pairs to HMAC, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int hmac_memory_multi(int hash, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) + +{ + hmac_state *hmac; + int err; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* allocate ram for hmac state */ + hmac = XMALLOC(sizeof(hmac_state)); + if (hmac == NULL) { + return CRYPT_MEM; + } + + if ((err = hmac_init(hmac, hash, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + va_start(args, inlen); + curptr = in; + curlen = inlen; + for (;;) { + /* process buf */ + if ((err = hmac_process(hmac, curptr, curlen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* step to next */ + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) { + break; + } + curlen = va_arg(args, unsigned long); + } + if ((err = hmac_done(hmac, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(hmac, sizeof(hmac_state)); +#endif + XFREE(hmac); + va_end(args); + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_process.c b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_process.c new file mode 100644 index 0000000..85dda20 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_process.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hmac_process.c + HMAC support, process data, Tom St Denis/Dobes Vandermeer +*/ + +#ifdef LTC_HMAC + +/** + Process data through HMAC + @param hmac The hmac state + @param in The data to send through HMAC + @param inlen The length of the data to HMAC (octets) + @return CRYPT_OK if successful +*/ +int hmac_process(hmac_state *hmac, const unsigned char *in, unsigned long inlen) +{ + int err; + LTC_ARGCHK(hmac != NULL); + LTC_ARGCHK(in != NULL); + if ((err = hash_is_valid(hmac->hash)) != CRYPT_OK) { + return err; + } + return hash_descriptor[hmac->hash]->process(&hmac->md, in, inlen); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_test.c b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_test.c new file mode 100644 index 0000000..1226590 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/hmac_test.c @@ -0,0 +1,618 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hmac_test.c + HMAC support, self-test, Tom St Denis/Dobes Vandermeer/Steffen Jaeckel +*/ + +#ifdef LTC_HMAC + +/* + TEST CASES SOURCE: + +Network Working Group P. Cheng +Request for Comments: 2202 IBM +Category: Informational R. Glenn + NIST + September 1997 + + Test Cases for HMAC-MD5 and HMAC-SHA-1 + +******************************************************************************* + +Network Working Group J. Kapp +Request for Comments: 2286 Reaper Technologies +Category: Informational February 1998 + + Test Cases for HMAC-RIPEMD160 and HMAC-RIPEMD128 + +******************************************************************************* + +Network Working Group M. Nystrom +Request for Comments: 4231 RSA Security +Category: Standards Track December 2005 + + Identifiers and Test Vectors for HMAC-SHA-224, HMAC-SHA-256, + HMAC-SHA-384, and HMAC-SHA-512 +*/ + +/** + HMAC self-test + @return CRYPT_OK if successful, CRYPT_NOP if tests have been disabled. +*/ +int hmac_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + unsigned char digest[MAXBLOCKSIZE]; + int i; + + static const unsigned char hmac_test_case_keys[][136] = { + { /* 1 */ + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b + }, +#ifdef LTC_TEST_EXT + { /* 2 */ + 0x4a, 0x65, 0x66, 0x65 + }, + { /* 4 */ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19 + }, + { /* 5 */ + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c + }, + { /* 3, 6, 7 */ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa + } +#endif /* LTC_TEST_EXT */ + }; + + + static const unsigned char hmac_test_case_data[][153] = { + { + "Hi There" + }, +#ifdef LTC_TEST_EXT + { + "what do ya want for nothing?" + }, + { + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd + }, + { + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd + }, + { + "Test With Truncation" + }, + { + "Test Using Larger Than Block-Size Key - Hash Key First" + }, + { + "Test Using Larger Than Block-Size Key and Larger Than One Block-Size Data" + }, + { + "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm." + } +#endif /* LTC_TEST_EXT */ + }; + + static const struct hmac_test_case { + const char *num; + const char *algo; + const unsigned char *key; + unsigned long keylen; + const unsigned char *data; + unsigned long datalen; + unsigned char digest[MAXBLOCKSIZE]; + } cases[] = { + /* + RFC 2202 3. Test Cases for HMAC-SHA-1 + */ + { "rfc2202 3.1", "sha1", + hmac_test_case_keys[0], 20, + hmac_test_case_data[0], 8, + {0xb6, 0x17, 0x31, 0x86, 0x55, 0x05, 0x72, 0x64, + 0xe2, 0x8b, 0xc0, 0xb6, 0xfb, 0x37, 0x8c, 0x8e, + 0xf1, 0x46, 0xbe, 0x00} }, + +#ifdef LTC_TEST_EXT + { "rfc2202 3.2", "sha1", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0xef, 0xfc, 0xdf, 0x6a, 0xe5, 0xeb, 0x2f, 0xa2, + 0xd2, 0x74, 0x16, 0xd5, 0xf1, 0x84, 0xdf, 0x9c, + 0x25, 0x9a, 0x7c, 0x79} }, + + { "rfc2202 3.3", "sha1", + hmac_test_case_keys[4], 20, + hmac_test_case_data[2], 50, + {0x12, 0x5d, 0x73, 0x42, 0xb9, 0xac, 0x11, 0xcd, + 0x91, 0xa3, 0x9a, 0xf4, 0x8a, 0xa1, 0x7b, 0x4f, + 0x63, 0xf1, 0x75, 0xd3} }, + + { "rfc2202 3.4", "sha1", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0x4c, 0x90, 0x07, 0xf4, 0x02, 0x62, 0x50, 0xc6, + 0xbc, 0x84, 0x14, 0xf9, 0xbf, 0x50, 0xc8, 0x6c, + 0x2d, 0x72, 0x35, 0xda} }, + + { "rfc2202 3.5", "sha1", + hmac_test_case_keys[3], 20, + hmac_test_case_data[4], 20, + {0x4c, 0x1a, 0x03, 0x42, 0x4b, 0x55, 0xe0, 0x7f, 0xe7, 0xf2, + 0x7b, 0xe1, 0xd5, 0x8b, 0xb9, 0x32, 0x4a, 0x9a, 0x5a, 0x04} }, + + { "rfc2202 3.6", "sha1", + hmac_test_case_keys[4], 80, + hmac_test_case_data[5], 54, + {0xaa, 0x4a, 0xe5, 0xe1, 0x52, 0x72, 0xd0, 0x0e, + 0x95, 0x70, 0x56, 0x37, 0xce, 0x8a, 0x3b, 0x55, + 0xed, 0x40, 0x21, 0x12} }, + + { "rfc2202 3.7", "sha1", + hmac_test_case_keys[4], 80, + hmac_test_case_data[6], 73, + {0xe8, 0xe9, 0x9d, 0x0f, 0x45, 0x23, 0x7d, 0x78, 0x6d, + 0x6b, 0xba, 0xa7, 0x96, 0x5c, 0x78, 0x08, 0xbb, 0xff, 0x1a, 0x91} }, +#endif /* LTC_TEST_EXT */ + + /* + RFC 2202 2. Test Cases for HMAC-MD5 + */ + { "rfc2202 2.1", "md5", + hmac_test_case_keys[0], 16, + hmac_test_case_data[0], 8, + {0x92, 0x94, 0x72, 0x7a, 0x36, 0x38, 0xbb, 0x1c, + 0x13, 0xf4, 0x8e, 0xf8, 0x15, 0x8b, 0xfc, 0x9d} }, + +#ifdef LTC_TEST_EXT + { "rfc2202 2.2", "md5", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0x75, 0x0c, 0x78, 0x3e, 0x6a, 0xb0, 0xb5, 0x03, + 0xea, 0xa8, 0x6e, 0x31, 0x0a, 0x5d, 0xb7, 0x38} }, + + { "rfc2202 2.3", "md5", + hmac_test_case_keys[4], 16, + hmac_test_case_data[2], 50, + {0x56, 0xbe, 0x34, 0x52, 0x1d, 0x14, 0x4c, 0x88, + 0xdb, 0xb8, 0xc7, 0x33, 0xf0, 0xe8, 0xb3, 0xf6} }, + + { "rfc2202 2.4", "md5", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0x69, 0x7e, 0xaf, 0x0a, 0xca, 0x3a, 0x3a, 0xea, + 0x3a, 0x75, 0x16, 0x47, 0x46, 0xff, 0xaa, 0x79} }, + + { "rfc2202 2.5", "md5", + hmac_test_case_keys[3], 16, + hmac_test_case_data[4], 20, + {0x56, 0x46, 0x1e, 0xf2, 0x34, 0x2e, 0xdc, 0x00, + 0xf9, 0xba, 0xb9, 0x95, 0x69, 0x0e, 0xfd, 0x4c} }, + + { "rfc2202 2.6", "md5", + hmac_test_case_keys[4], 80, + hmac_test_case_data[5], 54, + {0x6b, 0x1a, 0xb7, 0xfe, 0x4b, 0xd7, 0xbf, 0x8f, + 0x0b, 0x62, 0xe6, 0xce, 0x61, 0xb9, 0xd0, 0xcd} }, + + { "rfc2202 2.7", "md5", + hmac_test_case_keys[4], 80, + hmac_test_case_data[6], 73, + {0x6f, 0x63, 0x0f, 0xad, 0x67, 0xcd, 0xa0, 0xee, + 0x1f, 0xb1, 0xf5, 0x62, 0xdb, 0x3a, 0xa5, 0x3e} }, +#endif /* LTC_TEST_EXT */ + + /* + RFC 2286 2. Test Cases for HMAC-RIPEMD160 + */ + { "rfc2286 2.1", "rmd160", + hmac_test_case_keys[0], 20, + hmac_test_case_data[0], 8, + {0x24, 0xcb, 0x4b, 0xd6, 0x7d, 0x20, 0xfc, 0x1a, + 0x5d, 0x2e, 0xd7, 0x73, 0x2d, 0xcc, 0x39, 0x37, + 0x7f, 0x0a, 0x56, 0x68} }, + +#ifdef LTC_TEST_EXT + { "rfc2286 2.2", "rmd160", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0xdd, 0xa6, 0xc0, 0x21, 0x3a, 0x48, 0x5a, 0x9e, + 0x24, 0xf4, 0x74, 0x20, 0x64, 0xa7, 0xf0, 0x33, + 0xb4, 0x3c, 0x40, 0x69} }, + + { "rfc2286 2.3", "rmd160", + hmac_test_case_keys[4], 20, + hmac_test_case_data[2], 50, + {0xb0, 0xb1, 0x05, 0x36, 0x0d, 0xe7, 0x59, 0x96, + 0x0a, 0xb4, 0xf3, 0x52, 0x98, 0xe1, 0x16, 0xe2, + 0x95, 0xd8, 0xe7, 0xc1} }, + + { "rfc2286 2.4", "rmd160", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0xd5, 0xca, 0x86, 0x2f, 0x4d, 0x21, 0xd5, 0xe6, + 0x10, 0xe1, 0x8b, 0x4c, 0xf1, 0xbe, 0xb9, 0x7a, + 0x43, 0x65, 0xec, 0xf4} }, + + { "rfc2286 2.5", "rmd160", + hmac_test_case_keys[3], 20, + hmac_test_case_data[4], 20, + {0x76, 0x19, 0x69, 0x39, 0x78, 0xf9, 0x1d, 0x90, + 0x53, 0x9a, 0xe7, 0x86, 0x50, 0x0f, 0xf3, 0xd8, + 0xe0, 0x51, 0x8e, 0x39} }, + + { "rfc2286 2.6", "rmd160", + hmac_test_case_keys[4], 80, + hmac_test_case_data[5], 54, + {0x64, 0x66, 0xca, 0x07, 0xac, 0x5e, 0xac, 0x29, + 0xe1, 0xbd, 0x52, 0x3e, 0x5a, 0xda, 0x76, 0x05, + 0xb7, 0x91, 0xfd, 0x8b} }, + + { "rfc2286 2.7", "rmd160", + hmac_test_case_keys[4], 80, + hmac_test_case_data[6], 73, + {0x69, 0xea, 0x60, 0x79, 0x8d, 0x71, 0x61, 0x6c, + 0xce, 0x5f, 0xd0, 0x87, 0x1e, 0x23, 0x75, 0x4c, + 0xd7, 0x5d, 0x5a, 0x0a} }, +#endif /* LTC_TEST_EXT */ + + /* + RFC 2286 3. Test Cases for HMAC-RIPEMD128 + */ + { "rfc2286 3.1", "rmd128", + hmac_test_case_keys[0], 16, + hmac_test_case_data[0], 8, + {0xfb, 0xf6, 0x1f, 0x94, 0x92, 0xaa, 0x4b, 0xbf, + 0x81, 0xc1, 0x72, 0xe8, 0x4e, 0x07, 0x34, 0xdb} }, + +#ifdef LTC_TEST_EXT + { "rfc2286 3.2", "rmd128", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0x87, 0x5f, 0x82, 0x88, 0x62, 0xb6, 0xb3, 0x34, + 0xb4, 0x27, 0xc5, 0x5f, 0x9f, 0x7f, 0xf0, 0x9b} }, + + { "rfc2286 3.3", "rmd128", + hmac_test_case_keys[4], 16, + hmac_test_case_data[2], 50, + {0x09, 0xf0, 0xb2, 0x84, 0x6d, 0x2f, 0x54, 0x3d, + 0xa3, 0x63, 0xcb, 0xec, 0x8d, 0x62, 0xa3, 0x8d} }, + + { "rfc2286 3.4", "rmd128", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0xbd, 0xbb, 0xd7, 0xcf, 0x03, 0xe4, 0x4b, 0x5a, + 0xa6, 0x0a, 0xf8, 0x15, 0xbe, 0x4d, 0x22, 0x94} }, + + { "rfc2286 3.5", "rmd128", + hmac_test_case_keys[3], 16, + hmac_test_case_data[4], 20, + {0xe7, 0x98, 0x08, 0xf2, 0x4b, 0x25, 0xfd, 0x03, + 0x1c, 0x15, 0x5f, 0x0d, 0x55, 0x1d, 0x9a, 0x3a} }, + + { "rfc2286 3.6", "rmd128", + hmac_test_case_keys[4], 80, + hmac_test_case_data[5], 54, + {0xdc, 0x73, 0x29, 0x28, 0xde, 0x98, 0x10, 0x4a, + 0x1f, 0x59, 0xd3, 0x73, 0xc1, 0x50, 0xac, 0xbb} }, + + { "rfc2286 3.7", "rmd128", + hmac_test_case_keys[4], 80, + hmac_test_case_data[6], 73, + {0x5c, 0x6b, 0xec, 0x96, 0x79, 0x3e, 0x16, 0xd4, + 0x06, 0x90, 0xc2, 0x37, 0x63, 0x5f, 0x30, 0xc5} }, +#endif /* LTC_TEST_EXT */ + + /* + RFC 4231 4. Test Vectors + Ch. 4.6 with truncated output left out to simplify tests + */ + { "rfc4231 4.2", "sha224", + hmac_test_case_keys[0], 20, + hmac_test_case_data[0], 8, + {0x89, 0x6f, 0xb1, 0x12, 0x8a, 0xbb, 0xdf, 0x19, + 0x68, 0x32, 0x10, 0x7c, 0xd4, 0x9d, 0xf3, 0x3f, + 0x47, 0xb4, 0xb1, 0x16, 0x99, 0x12, 0xba, 0x4f, + 0x53, 0x68, 0x4b, 0x22} }, + +#ifdef LTC_TEST_EXT + { "rfc4231 4.3", "sha224", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0xa3, 0x0e, 0x01, 0x09, 0x8b, 0xc6, 0xdb, 0xbf, + 0x45, 0x69, 0x0f, 0x3a, 0x7e, 0x9e, 0x6d, 0x0f, + 0x8b, 0xbe, 0xa2, 0xa3, 0x9e, 0x61, 0x48, 0x00, + 0x8f, 0xd0, 0x5e, 0x44} }, + + { "rfc4231 4.4", "sha224", + hmac_test_case_keys[4], 20, + hmac_test_case_data[2], 50, + {0x7f, 0xb3, 0xcb, 0x35, 0x88, 0xc6, 0xc1, 0xf6, + 0xff, 0xa9, 0x69, 0x4d, 0x7d, 0x6a, 0xd2, 0x64, + 0x93, 0x65, 0xb0, 0xc1, 0xf6, 0x5d, 0x69, 0xd1, + 0xec, 0x83, 0x33, 0xea} }, + + { "rfc4231 4.5", "sha224", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0x6c, 0x11, 0x50, 0x68, 0x74, 0x01, 0x3c, 0xac, + 0x6a, 0x2a, 0xbc, 0x1b, 0xb3, 0x82, 0x62, 0x7c, + 0xec, 0x6a, 0x90, 0xd8, 0x6e, 0xfc, 0x01, 0x2d, + 0xe7, 0xaf, 0xec, 0x5a} }, + + { "rfc4231 4.7", "sha224", + hmac_test_case_keys[4], 131, + hmac_test_case_data[5], 54, + {0x95, 0xe9, 0xa0, 0xdb, 0x96, 0x20, 0x95, 0xad, + 0xae, 0xbe, 0x9b, 0x2d, 0x6f, 0x0d, 0xbc, 0xe2, + 0xd4, 0x99, 0xf1, 0x12, 0xf2, 0xd2, 0xb7, 0x27, + 0x3f, 0xa6, 0x87, 0x0e} }, + + { "rfc4231 4.8", "sha224", + hmac_test_case_keys[4], 131, + hmac_test_case_data[7], 152, + {0x3a, 0x85, 0x41, 0x66, 0xac, 0x5d, 0x9f, 0x02, + 0x3f, 0x54, 0xd5, 0x17, 0xd0, 0xb3, 0x9d, 0xbd, + 0x94, 0x67, 0x70, 0xdb, 0x9c, 0x2b, 0x95, 0xc9, + 0xf6, 0xf5, 0x65, 0xd1} }, +#endif /* LTC_TEST_EXT */ + + { "rfc4231 4.2", "sha256", + hmac_test_case_keys[0], 20, + hmac_test_case_data[0], 8, + {0xb0, 0x34, 0x4c, 0x61, 0xd8, 0xdb, 0x38, 0x53, + 0x5c, 0xa8, 0xaf, 0xce, 0xaf, 0x0b, 0xf1, 0x2b, + 0x88, 0x1d, 0xc2, 0x00, 0xc9, 0x83, 0x3d, 0xa7, + 0x26, 0xe9, 0x37, 0x6c, 0x2e, 0x32, 0xcf, 0xf7} }, + +#ifdef LTC_TEST_EXT + { "rfc4231 4.3", "sha256", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0x5b, 0xdc, 0xc1, 0x46, 0xbf, 0x60, 0x75, 0x4e, + 0x6a, 0x04, 0x24, 0x26, 0x08, 0x95, 0x75, 0xc7, + 0x5a, 0x00, 0x3f, 0x08, 0x9d, 0x27, 0x39, 0x83, + 0x9d, 0xec, 0x58, 0xb9, 0x64, 0xec, 0x38, 0x43} }, + + { "rfc4231 4.4", "sha256", + hmac_test_case_keys[4], 20, + hmac_test_case_data[2], 50, + {0x77, 0x3e, 0xa9, 0x1e, 0x36, 0x80, 0x0e, 0x46, + 0x85, 0x4d, 0xb8, 0xeb, 0xd0, 0x91, 0x81, 0xa7, + 0x29, 0x59, 0x09, 0x8b, 0x3e, 0xf8, 0xc1, 0x22, + 0xd9, 0x63, 0x55, 0x14, 0xce, 0xd5, 0x65, 0xfe} }, + + { "rfc4231 4.5", "sha256", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0x82, 0x55, 0x8a, 0x38, 0x9a, 0x44, 0x3c, 0x0e, + 0xa4, 0xcc, 0x81, 0x98, 0x99, 0xf2, 0x08, 0x3a, + 0x85, 0xf0, 0xfa, 0xa3, 0xe5, 0x78, 0xf8, 0x07, + 0x7a, 0x2e, 0x3f, 0xf4, 0x67, 0x29, 0x66, 0x5b} }, + + { "rfc4231 4.7", "sha256", + hmac_test_case_keys[4], 131, + hmac_test_case_data[5], 54, + {0x60, 0xe4, 0x31, 0x59, 0x1e, 0xe0, 0xb6, 0x7f, + 0x0d, 0x8a, 0x26, 0xaa, 0xcb, 0xf5, 0xb7, 0x7f, + 0x8e, 0x0b, 0xc6, 0x21, 0x37, 0x28, 0xc5, 0x14, + 0x05, 0x46, 0x04, 0x0f, 0x0e, 0xe3, 0x7f, 0x54} }, + + { "rfc4231 4.8", "sha256", + hmac_test_case_keys[4], 131, + hmac_test_case_data[7], 152, + {0x9b, 0x09, 0xff, 0xa7, 0x1b, 0x94, 0x2f, 0xcb, + 0x27, 0x63, 0x5f, 0xbc, 0xd5, 0xb0, 0xe9, 0x44, + 0xbf, 0xdc, 0x63, 0x64, 0x4f, 0x07, 0x13, 0x93, + 0x8a, 0x7f, 0x51, 0x53, 0x5c, 0x3a, 0x35, 0xe2} }, +#endif /* LTC_TEST_EXT */ + + { "rfc4231 4.2", "sha384", + hmac_test_case_keys[0], 20, + hmac_test_case_data[0], 8, + {0xaf, 0xd0, 0x39, 0x44, 0xd8, 0x48, 0x95, 0x62, + 0x6b, 0x08, 0x25, 0xf4, 0xab, 0x46, 0x90, 0x7f, + 0x15, 0xf9, 0xda, 0xdb, 0xe4, 0x10, 0x1e, 0xc6, + 0x82, 0xaa, 0x03, 0x4c, 0x7c, 0xeb, 0xc5, 0x9c, + 0xfa, 0xea, 0x9e, 0xa9, 0x07, 0x6e, 0xde, 0x7f, + 0x4a, 0xf1, 0x52, 0xe8, 0xb2, 0xfa, 0x9c, 0xb6} }, + +#ifdef LTC_TEST_EXT + { "rfc4231 4.3", "sha384", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0xaf, 0x45, 0xd2, 0xe3, 0x76, 0x48, 0x40, 0x31, + 0x61, 0x7f, 0x78, 0xd2, 0xb5, 0x8a, 0x6b, 0x1b, + 0x9c, 0x7e, 0xf4, 0x64, 0xf5, 0xa0, 0x1b, 0x47, + 0xe4, 0x2e, 0xc3, 0x73, 0x63, 0x22, 0x44, 0x5e, + 0x8e, 0x22, 0x40, 0xca, 0x5e, 0x69, 0xe2, 0xc7, + 0x8b, 0x32, 0x39, 0xec, 0xfa, 0xb2, 0x16, 0x49} }, + + { "rfc4231 4.4", "sha384", + hmac_test_case_keys[4], 20, + hmac_test_case_data[2], 50, + {0x88, 0x06, 0x26, 0x08, 0xd3, 0xe6, 0xad, 0x8a, + 0x0a, 0xa2, 0xac, 0xe0, 0x14, 0xc8, 0xa8, 0x6f, + 0x0a, 0xa6, 0x35, 0xd9, 0x47, 0xac, 0x9f, 0xeb, + 0xe8, 0x3e, 0xf4, 0xe5, 0x59, 0x66, 0x14, 0x4b, + 0x2a, 0x5a, 0xb3, 0x9d, 0xc1, 0x38, 0x14, 0xb9, + 0x4e, 0x3a, 0xb6, 0xe1, 0x01, 0xa3, 0x4f, 0x27} }, + + { "rfc4231 4.5", "sha384", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0x3e, 0x8a, 0x69, 0xb7, 0x78, 0x3c, 0x25, 0x85, + 0x19, 0x33, 0xab, 0x62, 0x90, 0xaf, 0x6c, 0xa7, + 0x7a, 0x99, 0x81, 0x48, 0x08, 0x50, 0x00, 0x9c, + 0xc5, 0x57, 0x7c, 0x6e, 0x1f, 0x57, 0x3b, 0x4e, + 0x68, 0x01, 0xdd, 0x23, 0xc4, 0xa7, 0xd6, 0x79, + 0xcc, 0xf8, 0xa3, 0x86, 0xc6, 0x74, 0xcf, 0xfb} }, + + { "rfc4231 4.7", "sha384", + hmac_test_case_keys[4], 131, + hmac_test_case_data[5], 54, + {0x4e, 0xce, 0x08, 0x44, 0x85, 0x81, 0x3e, 0x90, + 0x88, 0xd2, 0xc6, 0x3a, 0x04, 0x1b, 0xc5, 0xb4, + 0x4f, 0x9e, 0xf1, 0x01, 0x2a, 0x2b, 0x58, 0x8f, + 0x3c, 0xd1, 0x1f, 0x05, 0x03, 0x3a, 0xc4, 0xc6, + 0x0c, 0x2e, 0xf6, 0xab, 0x40, 0x30, 0xfe, 0x82, + 0x96, 0x24, 0x8d, 0xf1, 0x63, 0xf4, 0x49, 0x52} }, + + { "rfc4231 4.8", "sha384", + hmac_test_case_keys[4], 131, + hmac_test_case_data[7], 152, + {0x66, 0x17, 0x17, 0x8e, 0x94, 0x1f, 0x02, 0x0d, + 0x35, 0x1e, 0x2f, 0x25, 0x4e, 0x8f, 0xd3, 0x2c, + 0x60, 0x24, 0x20, 0xfe, 0xb0, 0xb8, 0xfb, 0x9a, + 0xdc, 0xce, 0xbb, 0x82, 0x46, 0x1e, 0x99, 0xc5, + 0xa6, 0x78, 0xcc, 0x31, 0xe7, 0x99, 0x17, 0x6d, + 0x38, 0x60, 0xe6, 0x11, 0x0c, 0x46, 0x52, 0x3e} }, +#endif /* LTC_TEST_EXT */ + + { "rfc4231 4.2", "sha512", + hmac_test_case_keys[0], 20, + hmac_test_case_data[0], 8, + {0x87, 0xaa, 0x7c, 0xde, 0xa5, 0xef, 0x61, 0x9d, + 0x4f, 0xf0, 0xb4, 0x24, 0x1a, 0x1d, 0x6c, 0xb0, + 0x23, 0x79, 0xf4, 0xe2, 0xce, 0x4e, 0xc2, 0x78, + 0x7a, 0xd0, 0xb3, 0x05, 0x45, 0xe1, 0x7c, 0xde, + 0xda, 0xa8, 0x33, 0xb7, 0xd6, 0xb8, 0xa7, 0x02, + 0x03, 0x8b, 0x27, 0x4e, 0xae, 0xa3, 0xf4, 0xe4, + 0xbe, 0x9d, 0x91, 0x4e, 0xeb, 0x61, 0xf1, 0x70, + 0x2e, 0x69, 0x6c, 0x20, 0x3a, 0x12, 0x68, 0x54} }, + +#ifdef LTC_TEST_EXT + { "rfc4231 4.3", "sha512", + hmac_test_case_keys[1], 4, + hmac_test_case_data[1], 28, + {0x16, 0x4b, 0x7a, 0x7b, 0xfc, 0xf8, 0x19, 0xe2, + 0xe3, 0x95, 0xfb, 0xe7, 0x3b, 0x56, 0xe0, 0xa3, + 0x87, 0xbd, 0x64, 0x22, 0x2e, 0x83, 0x1f, 0xd6, + 0x10, 0x27, 0x0c, 0xd7, 0xea, 0x25, 0x05, 0x54, + 0x97, 0x58, 0xbf, 0x75, 0xc0, 0x5a, 0x99, 0x4a, + 0x6d, 0x03, 0x4f, 0x65, 0xf8, 0xf0, 0xe6, 0xfd, + 0xca, 0xea, 0xb1, 0xa3, 0x4d, 0x4a, 0x6b, 0x4b, + 0x63, 0x6e, 0x07, 0x0a, 0x38, 0xbc, 0xe7, 0x37} }, + + { "rfc4231 4.4", "sha512", + hmac_test_case_keys[4], 20, + hmac_test_case_data[2], 50, + {0xfa, 0x73, 0xb0, 0x08, 0x9d, 0x56, 0xa2, 0x84, + 0xef, 0xb0, 0xf0, 0x75, 0x6c, 0x89, 0x0b, 0xe9, + 0xb1, 0xb5, 0xdb, 0xdd, 0x8e, 0xe8, 0x1a, 0x36, + 0x55, 0xf8, 0x3e, 0x33, 0xb2, 0x27, 0x9d, 0x39, + 0xbf, 0x3e, 0x84, 0x82, 0x79, 0xa7, 0x22, 0xc8, + 0x06, 0xb4, 0x85, 0xa4, 0x7e, 0x67, 0xc8, 0x07, + 0xb9, 0x46, 0xa3, 0x37, 0xbe, 0xe8, 0x94, 0x26, + 0x74, 0x27, 0x88, 0x59, 0xe1, 0x32, 0x92, 0xfb} }, + + { "rfc4231 4.5", "sha512", + hmac_test_case_keys[2], 25, + hmac_test_case_data[3], 50, + {0xb0, 0xba, 0x46, 0x56, 0x37, 0x45, 0x8c, 0x69, + 0x90, 0xe5, 0xa8, 0xc5, 0xf6, 0x1d, 0x4a, 0xf7, + 0xe5, 0x76, 0xd9, 0x7f, 0xf9, 0x4b, 0x87, 0x2d, + 0xe7, 0x6f, 0x80, 0x50, 0x36, 0x1e, 0xe3, 0xdb, + 0xa9, 0x1c, 0xa5, 0xc1, 0x1a, 0xa2, 0x5e, 0xb4, + 0xd6, 0x79, 0x27, 0x5c, 0xc5, 0x78, 0x80, 0x63, + 0xa5, 0xf1, 0x97, 0x41, 0x12, 0x0c, 0x4f, 0x2d, + 0xe2, 0xad, 0xeb, 0xeb, 0x10, 0xa2, 0x98, 0xdd} }, + + { "rfc4231 4.7", "sha512", + hmac_test_case_keys[4], 131, + hmac_test_case_data[5], 54, + {0x80, 0xb2, 0x42, 0x63, 0xc7, 0xc1, 0xa3, 0xeb, + 0xb7, 0x14, 0x93, 0xc1, 0xdd, 0x7b, 0xe8, 0xb4, + 0x9b, 0x46, 0xd1, 0xf4, 0x1b, 0x4a, 0xee, 0xc1, + 0x12, 0x1b, 0x01, 0x37, 0x83, 0xf8, 0xf3, 0x52, + 0x6b, 0x56, 0xd0, 0x37, 0xe0, 0x5f, 0x25, 0x98, + 0xbd, 0x0f, 0xd2, 0x21, 0x5d, 0x6a, 0x1e, 0x52, + 0x95, 0xe6, 0x4f, 0x73, 0xf6, 0x3f, 0x0a, 0xec, + 0x8b, 0x91, 0x5a, 0x98, 0x5d, 0x78, 0x65, 0x98} }, + + { "rfc4231 4.8", "sha512", + hmac_test_case_keys[4], 131, + hmac_test_case_data[7], 152, + {0xe3, 0x7b, 0x6a, 0x77, 0x5d, 0xc8, 0x7d, 0xba, + 0xa4, 0xdf, 0xa9, 0xf9, 0x6e, 0x5e, 0x3f, 0xfd, + 0xde, 0xbd, 0x71, 0xf8, 0x86, 0x72, 0x89, 0x86, + 0x5d, 0xf5, 0xa3, 0x2d, 0x20, 0xcd, 0xc9, 0x44, + 0xb6, 0x02, 0x2c, 0xac, 0x3c, 0x49, 0x82, 0xb1, + 0x0d, 0x5e, 0xeb, 0x55, 0xc3, 0xe4, 0xde, 0x15, + 0x13, 0x46, 0x76, 0xfb, 0x6d, 0xe0, 0x44, 0x60, + 0x65, 0xc9, 0x74, 0x40, 0xfa, 0x8c, 0x6a, 0x58} }, +#endif /* LTC_TEST_EXT */ + + }; + + unsigned long outlen; + int err; + int tested=0,failed=0; + for(i=0; i < (int)(sizeof(cases) / sizeof(cases[0])); i++) { + int hash = find_hash(cases[i].algo); + if (hash == -1) continue; + ++tested; + outlen = sizeof(digest); + if((err = hmac_memory(hash, cases[i].key, cases[i].keylen, cases[i].data, cases[i].datalen, digest, &outlen)) != CRYPT_OK) { +#ifdef LTC_TEST_DBG + printf("HMAC-%s test %s, %s\n", cases[i].algo, cases[i].num, error_to_string(err)); +#endif + return err; + } + + if(compare_testvector(digest, outlen, cases[i].digest, (size_t)hash_descriptor[hash]->hashsize, cases[i].num, i)) { + failed++; + } + } + + if (failed != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + if (tested == 0) { + return CRYPT_NOP; + } + return CRYPT_OK; + #endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/hmac/sub.mk b/optee_os/core/lib/libtomcrypt/src/mac/hmac/sub.mk new file mode 100644 index 0000000..2b8702f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/hmac/sub.mk @@ -0,0 +1,7 @@ +cflags-y += -Wno-unused-parameter + +srcs-y += hmac_done.c +srcs-y += hmac_init.c +srcs-y += hmac_memory.c +srcs-y += hmac_memory_multi.c +srcs-y += hmac_process.c diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_done.c b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_done.c new file mode 100644 index 0000000..2f36a88 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_done.c @@ -0,0 +1,74 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file omac_done.c + OMAC1 support, terminate a stream, Tom St Denis +*/ + +#ifdef LTC_OMAC + +/** + Terminate an OMAC stream + @param omac The OMAC state + @param out [out] Destination for the authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful +*/ +int omac_done(omac_state *omac, unsigned char *out, unsigned long *outlen) +{ + int err, mode; + unsigned x; + + LTC_ARGCHK(omac != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + if ((err = cipher_is_valid(omac->cipher_idx)) != CRYPT_OK) { + return err; + } + + if ((omac->buflen > (int)sizeof(omac->block)) || (omac->buflen < 0) || + (omac->blklen > (int)sizeof(omac->block)) || (omac->buflen > omac->blklen)) { + return CRYPT_INVALID_ARG; + } + + /* figure out mode */ + if (omac->buflen != omac->blklen) { + /* add the 0x80 byte */ + omac->block[omac->buflen++] = 0x80; + + /* pad with 0x00 */ + while (omac->buflen < omac->blklen) { + omac->block[omac->buflen++] = 0x00; + } + mode = 1; + } else { + mode = 0; + } + + /* now xor prev + Lu[mode] */ + for (x = 0; x < (unsigned)omac->blklen; x++) { + omac->block[x] ^= omac->prev[x] ^ omac->Lu[mode][x]; + } + + /* encrypt it */ + if ((err = cipher_descriptor[omac->cipher_idx]->ecb_encrypt(omac->block, omac->block, &omac->key)) != CRYPT_OK) { + return err; + } + cipher_descriptor[omac->cipher_idx]->done(&omac->key); + + /* output it */ + for (x = 0; x < (unsigned)omac->blklen && x < *outlen; x++) { + out[x] = omac->block[x]; + } + *outlen = x; + +#ifdef LTC_CLEAN_STACK + zeromem(omac, sizeof(*omac)); +#endif + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_file.c b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_file.c new file mode 100644 index 0000000..012958a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_file.c @@ -0,0 +1,87 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file omac_file.c + OMAC1 support, process a file, Tom St Denis +*/ + +#ifdef LTC_OMAC + +/** + OMAC a file + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param filename The name of the file you wish to OMAC + @param out [out] Where the authentication tag is to be stored + @param outlen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int omac_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *filename, + unsigned char *out, unsigned long *outlen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(cipher); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(filename); + LTC_UNUSED_PARAM(out); + LTC_UNUSED_PARAM(outlen); + return CRYPT_NOP; +#else + size_t x; + int err; + omac_state omac; + FILE *in; + unsigned char *buf; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(filename != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = omac_init(&omac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(filename, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = omac_process(&omac, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = omac_done(&omac, out, outlen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&omac, sizeof(omac_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_init.c b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_init.c new file mode 100644 index 0000000..a287a5e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_init.c @@ -0,0 +1,89 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file omac_init.c + OMAC1 support, initialize state, by Tom St Denis +*/ + + +#ifdef LTC_OMAC + +/** + Initialize an OMAC state + @param omac The OMAC state to initialize + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int omac_init(omac_state *omac, int cipher, const unsigned char *key, unsigned long keylen) +{ + int err, x, y, mask, msb, len; + + LTC_ARGCHK(omac != NULL); + LTC_ARGCHK(key != NULL); + + /* schedule the key */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + +#ifdef LTC_FAST + if (cipher_descriptor[cipher]->block_length % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + /* now setup the system */ + switch (cipher_descriptor[cipher]->block_length) { + case 8: mask = 0x1B; + len = 8; + break; + case 16: mask = 0x87; + len = 16; + break; + default: return CRYPT_INVALID_ARG; + } + + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, &omac->key)) != CRYPT_OK) { + return err; + } + + /* ok now we need Lu and Lu^2 [calc one from the other] */ + + /* first calc L which is Ek(0) */ + zeromem(omac->Lu[0], cipher_descriptor[cipher]->block_length); + if ((err = cipher_descriptor[cipher]->ecb_encrypt(omac->Lu[0], omac->Lu[0], &omac->key)) != CRYPT_OK) { + return err; + } + + /* now do the mults, whoopy! */ + for (x = 0; x < 2; x++) { + /* if msb(L * u^(x+1)) = 0 then just shift, otherwise shift and xor constant mask */ + msb = omac->Lu[x][0] >> 7; + + /* shift left */ + for (y = 0; y < (len - 1); y++) { + omac->Lu[x][y] = ((omac->Lu[x][y] << 1) | (omac->Lu[x][y+1] >> 7)) & 255; + } + omac->Lu[x][len - 1] = ((omac->Lu[x][len - 1] << 1) ^ (msb ? mask : 0)) & 255; + + /* copy up as require */ + if (x == 0) { + XMEMCPY(omac->Lu[1], omac->Lu[0], sizeof(omac->Lu[0])); + } + } + + /* setup state */ + omac->cipher_idx = cipher; + omac->buflen = 0; + omac->blklen = len; + zeromem(omac->prev, sizeof(omac->prev)); + zeromem(omac->block, sizeof(omac->block)); + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_memory.c new file mode 100644 index 0000000..c3ae9f9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_memory.c @@ -0,0 +1,73 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file omac_memory.c + OMAC1 support, process a block of memory, Tom St Denis +*/ + +#ifdef LTC_OMAC + +/** + OMAC a block of memory + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @param in The data to send through OMAC + @param inlen The length of the data to send through OMAC (octets) + @param out [out] The destination of the authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag (octets) + @return CRYPT_OK if successful +*/ +int omac_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + int err; + omac_state *omac; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* is the cipher valid? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* Use accelerator if found */ + if (cipher_descriptor[cipher]->omac_memory != NULL) { + return cipher_descriptor[cipher]->omac_memory(key, keylen, in, inlen, out, outlen); + } + + /* allocate ram for omac state */ + omac = XMALLOC(sizeof(omac_state)); + if (omac == NULL) { + return CRYPT_MEM; + } + + /* omac process the message */ + if ((err = omac_init(omac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = omac_process(omac, in, inlen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = omac_done(omac, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(omac, sizeof(omac_state)); +#endif + + XFREE(omac); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_memory_multi.c new file mode 100644 index 0000000..4f3b708 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_memory_multi.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file omac_memory_multi.c + OMAC1 support, process multiple blocks of memory, Tom St Denis +*/ + +#ifdef LTC_OMAC + +/** + OMAC multiple blocks of memory + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @param out [out] The destination of the authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag (octets) + @param in The data to send through OMAC + @param inlen The length of the data to send through OMAC (octets) + @param ... tuples of (data,len) pairs to OMAC, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int omac_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) +{ + int err; + omac_state *omac; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* allocate ram for omac state */ + omac = XMALLOC(sizeof(omac_state)); + if (omac == NULL) { + return CRYPT_MEM; + } + + /* omac process the message */ + if ((err = omac_init(omac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + va_start(args, inlen); + curptr = in; + curlen = inlen; + for (;;) { + /* process buf */ + if ((err = omac_process(omac, curptr, curlen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* step to next */ + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) { + break; + } + curlen = va_arg(args, unsigned long); + } + if ((err = omac_done(omac, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(omac, sizeof(omac_state)); +#endif + XFREE(omac); + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_process.c b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_process.c new file mode 100644 index 0000000..e065eb4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_process.c @@ -0,0 +1,80 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file omac_process.c + OMAC1 support, process data, Tom St Denis +*/ + + +#ifdef LTC_OMAC + +/** + Process data through OMAC + @param omac The OMAC state + @param in The input data to send through OMAC + @param inlen The length of the input (octets) + @return CRYPT_OK if successful +*/ +int omac_process(omac_state *omac, const unsigned char *in, unsigned long inlen) +{ + unsigned long n, x; + int err; + + LTC_ARGCHK(omac != NULL); + LTC_ARGCHK(in != NULL); + if ((err = cipher_is_valid(omac->cipher_idx)) != CRYPT_OK) { + return err; + } + + if ((omac->buflen > (int)sizeof(omac->block)) || (omac->buflen < 0) || + (omac->blklen > (int)sizeof(omac->block)) || (omac->buflen > omac->blklen)) { + return CRYPT_INVALID_ARG; + } + +#ifdef LTC_FAST + { + unsigned long blklen = cipher_descriptor[omac->cipher_idx]->block_length; + + if (omac->buflen == 0 && inlen > blklen) { + unsigned long y; + for (x = 0; x < (inlen - blklen); x += blklen) { + for (y = 0; y < blklen; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&omac->prev[y])) ^= *(LTC_FAST_TYPE_PTR_CAST(&in[y])); + } + in += blklen; + if ((err = cipher_descriptor[omac->cipher_idx]->ecb_encrypt(omac->prev, omac->prev, &omac->key)) != CRYPT_OK) { + return err; + } + } + inlen -= x; + } + } +#endif + + while (inlen != 0) { + /* ok if the block is full we xor in prev, encrypt and replace prev */ + if (omac->buflen == omac->blklen) { + for (x = 0; x < (unsigned long)omac->blklen; x++) { + omac->block[x] ^= omac->prev[x]; + } + if ((err = cipher_descriptor[omac->cipher_idx]->ecb_encrypt(omac->block, omac->prev, &omac->key)) != CRYPT_OK) { + return err; + } + omac->buflen = 0; + } + + /* add bytes */ + n = MIN(inlen, (unsigned long)(omac->blklen - omac->buflen)); + XMEMCPY(omac->block + omac->buflen, in, n); + omac->buflen += n; + inlen -= n; + in += n; + } + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_test.c b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_test.c new file mode 100644 index 0000000..bca6d9c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/omac_test.c @@ -0,0 +1,93 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file omac_test.c + OMAC1 support, self-test, by Tom St Denis +*/ + +#ifdef LTC_OMAC + +/** + Test the OMAC setup + @return CRYPT_OK if successful, CRYPT_NOP if tests have been disabled +*/ +int omac_test(void) +{ +#if !defined(LTC_TEST) + return CRYPT_NOP; +#else + static const struct { + int keylen, msglen; + unsigned char key[16], msg[64], tag[16]; + } tests[] = { + { 16, 0, + { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }, + { 0x00 }, + { 0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28, + 0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46 } + }, + { 16, 16, + { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }, + { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a }, + { 0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, + 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c } + }, + { 16, 40, + { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }, + { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11 }, + { 0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30, + 0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27 } + }, + { 16, 64, + { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }, + { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 }, + { 0x51, 0xf0, 0xbe, 0xbf, 0x7e, 0x3b, 0x9d, 0x92, + 0xfc, 0x49, 0x74, 0x17, 0x79, 0x36, 0x3c, 0xfe } + } + + }; + unsigned char out[16]; + int x, err, idx; + unsigned long len; + + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + len = sizeof(out); + if ((err = omac_memory(idx, tests[x].key, tests[x].keylen, tests[x].msg, tests[x].msglen, out, &len)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(out, len, tests[x].tag, sizeof(tests[x].tag), "OMAC", x) != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/omac/sub.mk b/optee_os/core/lib/libtomcrypt/src/mac/omac/sub.mk new file mode 100644 index 0000000..c0de70b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/omac/sub.mk @@ -0,0 +1,7 @@ +cflags-y += -Wno-unused-parameter + +srcs-y += omac_done.c +srcs-y += omac_init.c +srcs-y += omac_memory.c +srcs-y += omac_memory_multi.c +srcs-y += omac_process.c diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican.c b/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican.c new file mode 100644 index 0000000..25592cc --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican.c @@ -0,0 +1,154 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pelican.c + Pelican MAC, initialize state, by Tom St Denis +*/ + +#ifdef LTC_PELICAN + +#define LTC_AES_TAB_C +#define ENCRYPT_ONLY +#define PELI_TAB +#include "../../ciphers/aes/aes_tab.c" + +/** + Initialize a Pelican state + @param pelmac The Pelican state to initialize + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int pelican_init(pelican_state *pelmac, const unsigned char *key, unsigned long keylen) +{ + int err; + + LTC_ARGCHK(pelmac != NULL); + LTC_ARGCHK(key != NULL); + +#ifdef LTC_FAST + if (16 % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + if ((err = aes_setup(key, keylen, 0, &pelmac->K)) != CRYPT_OK) { + return err; + } + + zeromem(pelmac->state, 16); + aes_ecb_encrypt(pelmac->state, pelmac->state, &pelmac->K); + pelmac->buflen = 0; + + return CRYPT_OK; +} + +static void s_four_rounds(pelican_state *pelmac) +{ + ulong32 s0, s1, s2, s3, t0, t1, t2, t3; + int r; + + LOAD32H(s0, pelmac->state ); + LOAD32H(s1, pelmac->state + 4); + LOAD32H(s2, pelmac->state + 8); + LOAD32H(s3, pelmac->state + 12); + for (r = 0; r < 4; r++) { + t0 = + Te0(LTC_BYTE(s0, 3)) ^ + Te1(LTC_BYTE(s1, 2)) ^ + Te2(LTC_BYTE(s2, 1)) ^ + Te3(LTC_BYTE(s3, 0)); + t1 = + Te0(LTC_BYTE(s1, 3)) ^ + Te1(LTC_BYTE(s2, 2)) ^ + Te2(LTC_BYTE(s3, 1)) ^ + Te3(LTC_BYTE(s0, 0)); + t2 = + Te0(LTC_BYTE(s2, 3)) ^ + Te1(LTC_BYTE(s3, 2)) ^ + Te2(LTC_BYTE(s0, 1)) ^ + Te3(LTC_BYTE(s1, 0)); + t3 = + Te0(LTC_BYTE(s3, 3)) ^ + Te1(LTC_BYTE(s0, 2)) ^ + Te2(LTC_BYTE(s1, 1)) ^ + Te3(LTC_BYTE(s2, 0)); + s0 = t0; s1 = t1; s2 = t2; s3 = t3; + } + STORE32H(s0, pelmac->state ); + STORE32H(s1, pelmac->state + 4); + STORE32H(s2, pelmac->state + 8); + STORE32H(s3, pelmac->state + 12); +} + +/** + Process a block of text through Pelican + @param pelmac The Pelican MAC state + @param in The input + @param inlen The length input (octets) + @return CRYPT_OK on success + */ +int pelican_process(pelican_state *pelmac, const unsigned char *in, unsigned long inlen) +{ + + LTC_ARGCHK(pelmac != NULL); + LTC_ARGCHK(in != NULL); + + /* check range */ + if (pelmac->buflen < 0 || pelmac->buflen > 15) { + return CRYPT_INVALID_ARG; + } + +#ifdef LTC_FAST + if (pelmac->buflen == 0) { + while (inlen & ~15) { + int x; + for (x = 0; x < 16; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)pelmac->state + x)) ^= *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)in + x)); + } + s_four_rounds(pelmac); + in += 16; + inlen -= 16; + } + } +#endif + + while (inlen--) { + pelmac->state[pelmac->buflen++] ^= *in++; + if (pelmac->buflen == 16) { + s_four_rounds(pelmac); + pelmac->buflen = 0; + } + } + return CRYPT_OK; +} + +/** + Terminate Pelican MAC + @param pelmac The Pelican MAC state + @param out [out] The TAG + @return CRYPT_OK on sucess +*/ +int pelican_done(pelican_state *pelmac, unsigned char *out) +{ + LTC_ARGCHK(pelmac != NULL); + LTC_ARGCHK(out != NULL); + + /* check range */ + if (pelmac->buflen < 0 || pelmac->buflen > 16) { + return CRYPT_INVALID_ARG; + } + + if (pelmac->buflen == 16) { + s_four_rounds(pelmac); + pelmac->buflen = 0; + } + pelmac->state[pelmac->buflen++] ^= 0x80; + aes_ecb_encrypt(pelmac->state, out, &pelmac->K); + aes_done(&pelmac->K); + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican_memory.c new file mode 100644 index 0000000..f67eaee --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican_memory.c @@ -0,0 +1,47 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pelican_memory.c + Pelican MAC, MAC a block of memory, by Tom St Denis +*/ + +#ifdef LTC_PELICAN + +/** + Pelican block of memory + @param key The key for the MAC + @param keylen The length of the key (octets) + @param in The input to MAC + @param inlen The length of the input (octets) + @param out [out] The output TAG + @return CRYPT_OK on success +*/ +int pelican_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out) +{ + pelican_state *pel; + int err; + + pel = XMALLOC(sizeof(*pel)); + if (pel == NULL) { + return CRYPT_MEM; + } + + if ((err = pelican_init(pel, key, keylen)) != CRYPT_OK) { + XFREE(pel); + return err; + } + if ((err = pelican_process(pel, in ,inlen)) != CRYPT_OK) { + XFREE(pel); + return err; + } + err = pelican_done(pel, out); + XFREE(pel); + return err; +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican_test.c b/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican_test.c new file mode 100644 index 0000000..2ec696a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pelican/pelican_test.c @@ -0,0 +1,103 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pelican_test.c + Pelican MAC, test, by Tom St Denis +*/ + +#ifdef LTC_PELICAN + +int pelican_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char K[32], MSG[64], T[16]; + int keylen, ptlen; + } tests[] = { +/* K=16, M=0 */ +{ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }, + { 0 }, + { 0xeb, 0x58, 0x37, 0x15, 0xf8, 0x34, 0xde, 0xe5, + 0xa4, 0xd1, 0x6e, 0xe4, 0xb9, 0xd7, 0x76, 0x0e, }, + 16, 0 +}, + +/* K=16, M=3 */ +{ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }, + { 0x00, 0x01, 0x02 }, + { 0x1c, 0x97, 0x40, 0x60, 0x6c, 0x58, 0x17, 0x2d, + 0x03, 0x94, 0x19, 0x70, 0x81, 0xc4, 0x38, 0x54, }, + 16, 3 +}, + +/* K=16, M=16 */ +{ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }, + { 0x03, 0xcc, 0x46, 0xb8, 0xac, 0xa7, 0x9c, 0x36, + 0x1e, 0x8c, 0x6e, 0xa6, 0x7b, 0x89, 0x32, 0x49, }, + 16, 16 +}, + +/* K=16, M=32 */ +{ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F }, + { 0x89, 0xcc, 0x36, 0x58, 0x1b, 0xdd, 0x4d, 0xb5, + 0x78, 0xbb, 0xac, 0xf0, 0xff, 0x8b, 0x08, 0x15, }, + 16, 32 +}, + +/* K=16, M=35 */ +{ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F }, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, + 0x20, 0x21, 0x23 }, + { 0x4a, 0x7d, 0x45, 0x4d, 0xcd, 0xb5, 0xda, 0x8d, + 0x48, 0x78, 0x16, 0x48, 0x5d, 0x45, 0x95, 0x99, }, + 16, 35 +}, +}; + int x, err; + unsigned char out[16]; + pelican_state pel; + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + if ((err = pelican_init(&pel, tests[x].K, tests[x].keylen)) != CRYPT_OK) { + return err; + } + if ((err = pelican_process(&pel, tests[x].MSG, tests[x].ptlen)) != CRYPT_OK) { + return err; + } + if ((err = pelican_done(&pel, out)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(out, 16, tests[x].T, 16, "PELICAN", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_done.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_done.c new file mode 100644 index 0000000..bdccc4e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_done.c @@ -0,0 +1,62 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_done.c + PMAC implementation, terminate a session, by Tom St Denis +*/ + +#ifdef LTC_PMAC + +int pmac_done(pmac_state *pmac, unsigned char *out, unsigned long *outlen) +{ + int err, x; + + LTC_ARGCHK(pmac != NULL); + LTC_ARGCHK(out != NULL); + if ((err = cipher_is_valid(pmac->cipher_idx)) != CRYPT_OK) { + return err; + } + + if ((pmac->buflen > (int)sizeof(pmac->block)) || (pmac->buflen < 0) || + (pmac->block_len > (int)sizeof(pmac->block)) || (pmac->buflen > pmac->block_len)) { + return CRYPT_INVALID_ARG; + } + + + /* handle padding. If multiple xor in L/x */ + + if (pmac->buflen == pmac->block_len) { + /* xor Lr against the checksum */ + for (x = 0; x < pmac->block_len; x++) { + pmac->checksum[x] ^= pmac->block[x] ^ pmac->Lr[x]; + } + } else { + /* otherwise xor message bytes then the 0x80 byte */ + for (x = 0; x < pmac->buflen; x++) { + pmac->checksum[x] ^= pmac->block[x]; + } + pmac->checksum[x] ^= 0x80; + } + + /* encrypt it */ + if ((err = cipher_descriptor[pmac->cipher_idx]->ecb_encrypt(pmac->checksum, pmac->checksum, &pmac->key)) != CRYPT_OK) { + return err; + } + cipher_descriptor[pmac->cipher_idx]->done(&pmac->key); + + /* store it */ + for (x = 0; x < pmac->block_len && x < (int)*outlen; x++) { + out[x] = pmac->checksum[x]; + } + *outlen = x; + +#ifdef LTC_CLEAN_STACK + zeromem(pmac, sizeof(*pmac)); +#endif + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_file.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_file.c new file mode 100644 index 0000000..fca9b00 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_file.c @@ -0,0 +1,88 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_file.c + PMAC implementation, process a file, by Tom St Denis +*/ + +#ifdef LTC_PMAC + +/** + PMAC a file + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param filename The name of the file to send through PMAC + @param out [out] Destination for the authentication tag + @param outlen [in/out] Max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int pmac_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *filename, + unsigned char *out, unsigned long *outlen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(cipher); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(filename); + LTC_UNUSED_PARAM(out); + LTC_UNUSED_PARAM(outlen); + return CRYPT_NOP; +#else + size_t x; + int err; + pmac_state pmac; + FILE *in; + unsigned char *buf; + + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(filename != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = pmac_init(&pmac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(filename, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = pmac_process(&pmac, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = pmac_done(&pmac, out, outlen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&pmac, sizeof(pmac_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_init.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_init.c new file mode 100644 index 0000000..fd91292 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_init.c @@ -0,0 +1,138 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_init.c + PMAC implementation, initialize state, by Tom St Denis +*/ + +#ifdef LTC_PMAC + +static const struct { + int len; + unsigned char poly_div[MAXBLOCKSIZE], + poly_mul[MAXBLOCKSIZE]; +} polys[] = { +{ + 8, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0D }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1B } +}, { + 16, + { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87 } +} +}; + +/** + Initialize a PMAC state + @param pmac The PMAC state to initialize + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int pmac_init(pmac_state *pmac, int cipher, const unsigned char *key, unsigned long keylen) +{ + int poly, x, y, m, err; + unsigned char *L; + + LTC_ARGCHK(pmac != NULL); + LTC_ARGCHK(key != NULL); + + /* valid cipher? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* determine which polys to use */ + pmac->block_len = cipher_descriptor[cipher]->block_length; + for (poly = 0; poly < (int)(sizeof(polys)/sizeof(polys[0])); poly++) { + if (polys[poly].len == pmac->block_len) { + break; + } + } + if (poly >= (int)(sizeof(polys)/sizeof(polys[0]))) { + return CRYPT_INVALID_ARG; + } + if (polys[poly].len != pmac->block_len) { + return CRYPT_INVALID_ARG; + } + +#ifdef LTC_FAST + if (pmac->block_len % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + + /* schedule the key */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, &pmac->key)) != CRYPT_OK) { + return err; + } + + /* allocate L */ + L = XMALLOC(pmac->block_len); + if (L == NULL) { + return CRYPT_MEM; + } + + /* find L = E[0] */ + zeromem(L, pmac->block_len); + if ((err = cipher_descriptor[cipher]->ecb_encrypt(L, L, &pmac->key)) != CRYPT_OK) { + goto error; + } + + /* find Ls[i] = L << i for i == 0..31 */ + XMEMCPY(pmac->Ls[0], L, pmac->block_len); + for (x = 1; x < 32; x++) { + m = pmac->Ls[x-1][0] >> 7; + for (y = 0; y < pmac->block_len-1; y++) { + pmac->Ls[x][y] = ((pmac->Ls[x-1][y] << 1) | (pmac->Ls[x-1][y+1] >> 7)) & 255; + } + pmac->Ls[x][pmac->block_len-1] = (pmac->Ls[x-1][pmac->block_len-1] << 1) & 255; + + if (m == 1) { + for (y = 0; y < pmac->block_len; y++) { + pmac->Ls[x][y] ^= polys[poly].poly_mul[y]; + } + } + } + + /* find Lr = L / x */ + m = L[pmac->block_len-1] & 1; + + /* shift right */ + for (x = pmac->block_len - 1; x > 0; x--) { + pmac->Lr[x] = ((L[x] >> 1) | (L[x-1] << 7)) & 255; + } + pmac->Lr[0] = L[0] >> 1; + + if (m == 1) { + for (x = 0; x < pmac->block_len; x++) { + pmac->Lr[x] ^= polys[poly].poly_div[x]; + } + } + + /* zero buffer, counters, etc... */ + pmac->block_index = 1; + pmac->cipher_idx = cipher; + pmac->buflen = 0; + zeromem(pmac->block, sizeof(pmac->block)); + zeromem(pmac->Li, sizeof(pmac->Li)); + zeromem(pmac->checksum, sizeof(pmac->checksum)); + err = CRYPT_OK; +error: +#ifdef LTC_CLEAN_STACK + zeromem(L, pmac->block_len); +#endif + + XFREE(L); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_memory.c new file mode 100644 index 0000000..4671547 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_memory.c @@ -0,0 +1,62 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_memory.c + PMAC implementation, process a block of memory, by Tom St Denis +*/ + +#ifdef LTC_PMAC + +/** + PMAC a block of memory + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param in The data you wish to send through PMAC + @param inlen The length of data you wish to send through PMAC (octets) + @param out [out] Destination for the authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful +*/ +int pmac_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + int err; + pmac_state *pmac; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* allocate ram for pmac state */ + pmac = XMALLOC(sizeof(pmac_state)); + if (pmac == NULL) { + return CRYPT_MEM; + } + + if ((err = pmac_init(pmac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = pmac_process(pmac, in, inlen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = pmac_done(pmac, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(pmac, sizeof(pmac_state)); +#endif + + XFREE(pmac); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_memory_multi.c new file mode 100644 index 0000000..94d450c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_memory_multi.c @@ -0,0 +1,77 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file pmac_memory_multi.c + PMAC implementation, process multiple blocks of memory, by Tom St Denis +*/ + +#ifdef LTC_PMAC + +/** + PMAC multiple blocks of memory + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param out [out] Destination for the authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag + @param in The data you wish to send through PMAC + @param inlen The length of data you wish to send through PMAC (octets) + @param ... tuples of (data,len) pairs to PMAC, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int pmac_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) +{ + int err; + pmac_state *pmac; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* allocate ram for pmac state */ + pmac = XMALLOC(sizeof(pmac_state)); + if (pmac == NULL) { + return CRYPT_MEM; + } + + if ((err = pmac_init(pmac, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + va_start(args, inlen); + curptr = in; + curlen = inlen; + for (;;) { + /* process buf */ + if ((err = pmac_process(pmac, curptr, curlen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* step to next */ + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) { + break; + } + curlen = va_arg(args, unsigned long); + } + if ((err = pmac_done(pmac, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(pmac, sizeof(pmac_state)); +#endif + XFREE(pmac); + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_ntz.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_ntz.c new file mode 100644 index 0000000..ed71f33 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_ntz.c @@ -0,0 +1,27 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_ntz.c + PMAC implementation, internal function, by Tom St Denis +*/ + +#ifdef LTC_PMAC + +/** + Internal PMAC function +*/ +int pmac_ntz(unsigned long x) +{ + int c; + x &= 0xFFFFFFFFUL; + c = 0; + while ((x & 1) == 0) { + ++c; + x >>= 1; + } + return c; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_process.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_process.c new file mode 100644 index 0000000..c277afa --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_process.c @@ -0,0 +1,88 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_process.c + PMAC implementation, process data, by Tom St Denis +*/ + + +#ifdef LTC_PMAC + +/** + Process data in a PMAC stream + @param pmac The PMAC state + @param in The data to send through PMAC + @param inlen The length of the data to send through PMAC + @return CRYPT_OK if successful +*/ +int pmac_process(pmac_state *pmac, const unsigned char *in, unsigned long inlen) +{ + int err, n; + unsigned long x; + unsigned char Z[MAXBLOCKSIZE]; + + LTC_ARGCHK(pmac != NULL); + LTC_ARGCHK(in != NULL); + if ((err = cipher_is_valid(pmac->cipher_idx)) != CRYPT_OK) { + return err; + } + + if ((pmac->buflen > (int)sizeof(pmac->block)) || (pmac->buflen < 0) || + (pmac->block_len > (int)sizeof(pmac->block)) || (pmac->buflen > pmac->block_len)) { + return CRYPT_INVALID_ARG; + } + +#ifdef LTC_FAST + if (pmac->buflen == 0 && inlen > 16) { + unsigned long y; + for (x = 0; x < (inlen - 16); x += 16) { + pmac_shift_xor(pmac); + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&Z[y])) = *(LTC_FAST_TYPE_PTR_CAST(&in[y])) ^ *(LTC_FAST_TYPE_PTR_CAST(&pmac->Li[y])); + } + if ((err = cipher_descriptor[pmac->cipher_idx]->ecb_encrypt(Z, Z, &pmac->key)) != CRYPT_OK) { + return err; + } + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&pmac->checksum[y])) ^= *(LTC_FAST_TYPE_PTR_CAST(&Z[y])); + } + in += 16; + } + inlen -= x; + } +#endif + + while (inlen != 0) { + /* ok if the block is full we xor in prev, encrypt and replace prev */ + if (pmac->buflen == pmac->block_len) { + pmac_shift_xor(pmac); + for (x = 0; x < (unsigned long)pmac->block_len; x++) { + Z[x] = pmac->Li[x] ^ pmac->block[x]; + } + if ((err = cipher_descriptor[pmac->cipher_idx]->ecb_encrypt(Z, Z, &pmac->key)) != CRYPT_OK) { + return err; + } + for (x = 0; x < (unsigned long)pmac->block_len; x++) { + pmac->checksum[x] ^= Z[x]; + } + pmac->buflen = 0; + } + + /* add bytes */ + n = MIN(inlen, (unsigned long)(pmac->block_len - pmac->buflen)); + XMEMCPY(pmac->block + pmac->buflen, in, n); + pmac->buflen += n; + inlen -= n; + in += n; + } + +#ifdef LTC_CLEAN_STACK + zeromem(Z, sizeof(Z)); +#endif + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_shift_xor.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_shift_xor.c new file mode 100644 index 0000000..ad97fa8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_shift_xor.c @@ -0,0 +1,32 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_shift_xor.c + PMAC implementation, internal function, by Tom St Denis +*/ + +#ifdef LTC_PMAC + +/** + Internal function. Performs the state update (adding correct multiple) + @param pmac The PMAC state. +*/ +void pmac_shift_xor(pmac_state *pmac) +{ + int x, y; + y = pmac_ntz(pmac->block_index++); +#ifdef LTC_FAST + for (x = 0; x < pmac->block_len; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)pmac->Li + x)) ^= + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)pmac->Ls[y] + x)); + } +#else + for (x = 0; x < pmac->block_len; x++) { + pmac->Li[x] ^= pmac->Ls[y][x]; + } +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_test.c b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_test.c new file mode 100644 index 0000000..3ccf06e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/pmac/pmac_test.c @@ -0,0 +1,144 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pmac_test.c + PMAC implementation, self-test, by Tom St Denis +*/ + + +#ifdef LTC_PMAC + +/** + Test the LTC_OMAC implementation + @return CRYPT_OK if successful, CRYPT_NOP if testing has been disabled +*/ +int pmac_test(void) +{ +#if !defined(LTC_TEST) + return CRYPT_NOP; +#else + static const struct { + int msglen; + unsigned char key[16], msg[34], tag[16]; + } tests[] = { + + /* PMAC-AES-128-0B */ +{ + 0, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* msg */ + { 0x00 }, + /* tag */ + { 0x43, 0x99, 0x57, 0x2c, 0xd6, 0xea, 0x53, 0x41, + 0xb8, 0xd3, 0x58, 0x76, 0xa7, 0x09, 0x8a, 0xf7 } +}, + + /* PMAC-AES-128-3B */ +{ + 3, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* msg */ + { 0x00, 0x01, 0x02 }, + /* tag */ + { 0x25, 0x6b, 0xa5, 0x19, 0x3c, 0x1b, 0x99, 0x1b, + 0x4d, 0xf0, 0xc5, 0x1f, 0x38, 0x8a, 0x9e, 0x27 } +}, + + /* PMAC-AES-128-16B */ +{ + 16, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* msg */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* tag */ + { 0xeb, 0xbd, 0x82, 0x2f, 0xa4, 0x58, 0xda, 0xf6, + 0xdf, 0xda, 0xd7, 0xc2, 0x7d, 0xa7, 0x63, 0x38 } +}, + + /* PMAC-AES-128-20B */ +{ + 20, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* msg */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13 }, + /* tag */ + { 0x04, 0x12, 0xca, 0x15, 0x0b, 0xbf, 0x79, 0x05, + 0x8d, 0x8c, 0x75, 0xa5, 0x8c, 0x99, 0x3f, 0x55 } +}, + + /* PMAC-AES-128-32B */ +{ + 32, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* msg */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + /* tag */ + { 0xe9, 0x7a, 0xc0, 0x4e, 0x9e, 0x5e, 0x33, 0x99, + 0xce, 0x53, 0x55, 0xcd, 0x74, 0x07, 0xbc, 0x75 } +}, + + /* PMAC-AES-128-34B */ +{ + 34, + /* key */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + /* msg */ + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21 }, + /* tag */ + { 0x5c, 0xba, 0x7d, 0x5e, 0xb2, 0x4f, 0x7c, 0x86, + 0xcc, 0xc5, 0x46, 0x04, 0xe5, 0x3d, 0x55, 0x12 } +} + +}; + int err, x, idx; + unsigned long len; + unsigned char outtag[MAXBLOCKSIZE]; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + len = sizeof(outtag); + if ((err = pmac_memory(idx, tests[x].key, 16, tests[x].msg, tests[x].msglen, outtag, &len)) != CRYPT_OK) { + return err; + } + + if (compare_testvector(outtag, len, tests[x].tag, sizeof(tests[x].tag), "PMAC", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif /* LTC_TEST */ +} + +#endif /* PMAC_MODE */ + + + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305.c b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305.c new file mode 100644 index 0000000..32cdb55 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305.c @@ -0,0 +1,258 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * Public Domain poly1305 from Andrew Moon + * https://github.com/floodyberry/poly1305-donna + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_POLY1305 + +/* internal only */ +static void s_poly1305_block(poly1305_state *st, const unsigned char *in, unsigned long inlen) +{ + const unsigned long hibit = (st->final) ? 0 : (1UL << 24); /* 1 << 128 */ + ulong32 r0,r1,r2,r3,r4; + ulong32 s1,s2,s3,s4; + ulong32 h0,h1,h2,h3,h4; + ulong32 tmp; + ulong64 d0,d1,d2,d3,d4; + ulong32 c; + + r0 = st->r[0]; + r1 = st->r[1]; + r2 = st->r[2]; + r3 = st->r[3]; + r4 = st->r[4]; + + s1 = r1 * 5; + s2 = r2 * 5; + s3 = r3 * 5; + s4 = r4 * 5; + + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + while (inlen >= 16) { + /* h += in[i] */ + LOAD32L(tmp, in+ 0); h0 += (tmp ) & 0x3ffffff; + LOAD32L(tmp, in+ 3); h1 += (tmp >> 2) & 0x3ffffff; + LOAD32L(tmp, in+ 6); h2 += (tmp >> 4) & 0x3ffffff; + LOAD32L(tmp, in+ 9); h3 += (tmp >> 6) & 0x3ffffff; + LOAD32L(tmp, in+12); h4 += (tmp >> 8) | hibit; + + /* h *= r */ + d0 = ((ulong64)h0 * r0) + ((ulong64)h1 * s4) + ((ulong64)h2 * s3) + ((ulong64)h3 * s2) + ((ulong64)h4 * s1); + d1 = ((ulong64)h0 * r1) + ((ulong64)h1 * r0) + ((ulong64)h2 * s4) + ((ulong64)h3 * s3) + ((ulong64)h4 * s2); + d2 = ((ulong64)h0 * r2) + ((ulong64)h1 * r1) + ((ulong64)h2 * r0) + ((ulong64)h3 * s4) + ((ulong64)h4 * s3); + d3 = ((ulong64)h0 * r3) + ((ulong64)h1 * r2) + ((ulong64)h2 * r1) + ((ulong64)h3 * r0) + ((ulong64)h4 * s4); + d4 = ((ulong64)h0 * r4) + ((ulong64)h1 * r3) + ((ulong64)h2 * r2) + ((ulong64)h3 * r1) + ((ulong64)h4 * r0); + + /* (partial) h %= p */ + c = (ulong32)(d0 >> 26); h0 = (ulong32)d0 & 0x3ffffff; + d1 += c; c = (ulong32)(d1 >> 26); h1 = (ulong32)d1 & 0x3ffffff; + d2 += c; c = (ulong32)(d2 >> 26); h2 = (ulong32)d2 & 0x3ffffff; + d3 += c; c = (ulong32)(d3 >> 26); h3 = (ulong32)d3 & 0x3ffffff; + d4 += c; c = (ulong32)(d4 >> 26); h4 = (ulong32)d4 & 0x3ffffff; + h0 += c * 5; c = (h0 >> 26); h0 = h0 & 0x3ffffff; + h1 += c; + + in += 16; + inlen -= 16; + } + + st->h[0] = h0; + st->h[1] = h1; + st->h[2] = h2; + st->h[3] = h3; + st->h[4] = h4; +} + +/** + Initialize an POLY1305 context. + @param st The POLY1305 state + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int poly1305_init(poly1305_state *st, const unsigned char *key, unsigned long keylen) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen == 32); + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + LOAD32L(st->r[0], key + 0); st->r[0] = (st->r[0] ) & 0x3ffffff; + LOAD32L(st->r[1], key + 3); st->r[1] = (st->r[1] >> 2) & 0x3ffff03; + LOAD32L(st->r[2], key + 6); st->r[2] = (st->r[2] >> 4) & 0x3ffc0ff; + LOAD32L(st->r[3], key + 9); st->r[3] = (st->r[3] >> 6) & 0x3f03fff; + LOAD32L(st->r[4], key + 12); st->r[4] = (st->r[4] >> 8) & 0x00fffff; + + /* h = 0 */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; + st->h[3] = 0; + st->h[4] = 0; + + /* save pad for later */ + LOAD32L(st->pad[0], key + 16); + LOAD32L(st->pad[1], key + 20); + LOAD32L(st->pad[2], key + 24); + LOAD32L(st->pad[3], key + 28); + + st->leftover = 0; + st->final = 0; + return CRYPT_OK; +} + +/** + Process data through POLY1305 + @param st The POLY1305 state + @param in The data to send through HMAC + @param inlen The length of the data to HMAC (octets) + @return CRYPT_OK if successful +*/ +int poly1305_process(poly1305_state *st, const unsigned char *in, unsigned long inlen) +{ + unsigned long i; + + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + + /* handle leftover */ + if (st->leftover) { + unsigned long want = (16 - st->leftover); + if (want > inlen) want = inlen; + for (i = 0; i < want; i++) st->buffer[st->leftover + i] = in[i]; + inlen -= want; + in += want; + st->leftover += want; + if (st->leftover < 16) return CRYPT_OK; + s_poly1305_block(st, st->buffer, 16); + st->leftover = 0; + } + + /* process full blocks */ + if (inlen >= 16) { + unsigned long want = (inlen & ~(16 - 1)); + s_poly1305_block(st, in, want); + in += want; + inlen -= want; + } + + /* store leftover */ + if (inlen) { + for (i = 0; i < inlen; i++) st->buffer[st->leftover + i] = in[i]; + st->leftover += inlen; + } + return CRYPT_OK; +} + +/** + Terminate a POLY1305 session + @param st The POLY1305 state + @param mac [out] The destination of the POLY1305 authentication tag + @param maclen [in/out] The max size and resulting size of the POLY1305 authentication tag + @return CRYPT_OK if successful +*/ +int poly1305_done(poly1305_state *st, unsigned char *mac, unsigned long *maclen) +{ + ulong32 h0,h1,h2,h3,h4,c; + ulong32 g0,g1,g2,g3,g4; + ulong64 f; + ulong32 mask; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + LTC_ARGCHK(*maclen >= 16); + + /* process the remaining block */ + if (st->leftover) { + unsigned long i = st->leftover; + st->buffer[i++] = 1; + for (; i < 16; i++) st->buffer[i] = 0; + st->final = 1; + s_poly1305_block(st, st->buffer, 16); + } + + /* fully carry h */ + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + c = h1 >> 26; h1 = h1 & 0x3ffffff; + h2 += c; c = h2 >> 26; h2 = h2 & 0x3ffffff; + h3 += c; c = h3 >> 26; h3 = h3 & 0x3ffffff; + h4 += c; c = h4 >> 26; h4 = h4 & 0x3ffffff; + h0 += c * 5; c = h0 >> 26; h0 = h0 & 0x3ffffff; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; c = g0 >> 26; g0 &= 0x3ffffff; + g1 = h1 + c; c = g1 >> 26; g1 &= 0x3ffffff; + g2 = h2 + c; c = g2 >> 26; g2 &= 0x3ffffff; + g3 = h3 + c; c = g3 >> 26; g3 &= 0x3ffffff; + g4 = h4 + c - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> 31) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + h0 = ((h0 ) | (h1 << 26)) & 0xffffffff; + h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; + h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; + h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; + + /* mac = (h + pad) % (2^128) */ + f = (ulong64)h0 + st->pad[0] ; h0 = (ulong32)f; + f = (ulong64)h1 + st->pad[1] + (f >> 32); h1 = (ulong32)f; + f = (ulong64)h2 + st->pad[2] + (f >> 32); h2 = (ulong32)f; + f = (ulong64)h3 + st->pad[3] + (f >> 32); h3 = (ulong32)f; + + STORE32L(h0, mac + 0); + STORE32L(h1, mac + 4); + STORE32L(h2, mac + 8); + STORE32L(h3, mac + 12); + + /* zero out the state */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; + st->h[3] = 0; + st->h[4] = 0; + st->r[0] = 0; + st->r[1] = 0; + st->r[2] = 0; + st->r[3] = 0; + st->r[4] = 0; + st->pad[0] = 0; + st->pad[1] = 0; + st->pad[2] = 0; + st->pad[3] = 0; + + *maclen = 16; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_file.c b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_file.c new file mode 100644 index 0000000..e09c994 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_file.c @@ -0,0 +1,83 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * Public Domain poly1305 from Andrew Moon + * https://github.com/floodyberry/poly1305-donna + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_POLY1305 + +/** + POLY1305 a file + @param fname The name of the file you wish to POLY1305 + @param key The secret key + @param keylen The length of the secret key + @param mac [out] The POLY1305 authentication tag + @param maclen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int poly1305_file(const char *fname, const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(fname); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(mac); + LTC_UNUSED_PARAM(maclen); + return CRYPT_NOP; +#else + poly1305_state st; + FILE *in; + unsigned char *buf; + size_t x; + int err; + + LTC_ARGCHK(fname != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = poly1305_init(&st, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(fname, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = poly1305_process(&st, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = poly1305_done(&st, mac, maclen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(poly1305_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_memory.c new file mode 100644 index 0000000..8a9fea6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_memory.c @@ -0,0 +1,43 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * Public Domain poly1305 from Andrew Moon + * https://github.com/floodyberry/poly1305-donna + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_POLY1305 + +/** + POLY1305 a block of memory to produce the authentication tag + @param key The secret key + @param keylen The length of the secret key (octets) + @param in The data to POLY1305 + @param inlen The length of the data to POLY1305 (octets) + @param mac [out] Destination of the authentication tag + @param maclen [in/out] Max size and resulting size of authentication tag + @return CRYPT_OK if successful +*/ +int poly1305_memory(const unsigned char *key, unsigned long keylen, const unsigned char *in, unsigned long inlen, unsigned char *mac, unsigned long *maclen) +{ + poly1305_state st; + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + if ((err = poly1305_init(&st, key, keylen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = poly1305_process(&st, in, inlen)) != CRYPT_OK) { goto LBL_ERR; } + err = poly1305_done(&st, mac, maclen); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(poly1305_state)); +#endif + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_memory_multi.c new file mode 100644 index 0000000..6bd61df --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_memory_multi.c @@ -0,0 +1,57 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * Public Domain poly1305 from Andrew Moon + * https://github.com/floodyberry/poly1305-donna + */ + +#include "tomcrypt_private.h" +#include + +#ifdef LTC_POLY1305 + +/** + POLY1305 multiple blocks of memory to produce the authentication tag + @param key The secret key + @param keylen The length of the secret key (octets) + @param mac [out] Destination of the authentication tag + @param maclen [in/out] Max size and resulting size of authentication tag + @param in The data to POLY1305 + @param inlen The length of the data to POLY1305 (octets) + @param ... tuples of (data,len) pairs to POLY1305, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int poly1305_memory_multi(const unsigned char *key, unsigned long keylen, unsigned char *mac, unsigned long *maclen, const unsigned char *in, unsigned long inlen, ...) +{ + poly1305_state st; + int err; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(mac != NULL); + LTC_ARGCHK(maclen != NULL); + + va_start(args, inlen); + curptr = in; + curlen = inlen; + if ((err = poly1305_init(&st, key, keylen)) != CRYPT_OK) { goto LBL_ERR; } + for (;;) { + if ((err = poly1305_process(&st, curptr, curlen)) != CRYPT_OK) { goto LBL_ERR; } + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) break; + curlen = va_arg(args, unsigned long); + } + err = poly1305_done(&st, mac, maclen); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&st, sizeof(poly1305_state)); +#endif + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_test.c b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_test.c new file mode 100644 index 0000000..f9b3f75 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/poly1305/poly1305_test.c @@ -0,0 +1,46 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * Public Domain poly1305 from Andrew Moon + * https://github.com/floodyberry/poly1305-donna + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_POLY1305 + +int poly1305_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + /* https://tools.ietf.org/html/rfc7539#section-2.5.2 */ + unsigned char k[] = { 0x85, 0xd6, 0xbe, 0x78, 0x57, 0x55, 0x6d, 0x33, 0x7f, 0x44, 0x52, 0xfe, 0x42, 0xd5, 0x06, 0xa8, 0x01, 0x03, 0x80, 0x8a, 0xfb, 0x0d, 0xb2, 0xfd, 0x4a, 0xbf, 0xf6, 0xaf, 0x41, 0x49, 0xf5, 0x1b }; + unsigned char tag[] = { 0xA8, 0x06, 0x1D, 0xC1, 0x30, 0x51, 0x36, 0xC6, 0xC2, 0x2B, 0x8B, 0xAF, 0x0C, 0x01, 0x27, 0xA9 }; + char m[] = "Cryptographic Forum Research Group"; + unsigned long len = 16, mlen = XSTRLEN(m); + unsigned char out[1000]; + poly1305_state st; + int err; + + /* process piece by piece */ + if ((err = poly1305_init(&st, k, 32)) != CRYPT_OK) return err; + if ((err = poly1305_process(&st, (unsigned char*)m, 5)) != CRYPT_OK) return err; + if ((err = poly1305_process(&st, (unsigned char*)m + 5, 4)) != CRYPT_OK) return err; + if ((err = poly1305_process(&st, (unsigned char*)m + 9, 3)) != CRYPT_OK) return err; + if ((err = poly1305_process(&st, (unsigned char*)m + 12, 2)) != CRYPT_OK) return err; + if ((err = poly1305_process(&st, (unsigned char*)m + 14, 1)) != CRYPT_OK) return err; + if ((err = poly1305_process(&st, (unsigned char*)m + 15, mlen - 15)) != CRYPT_OK) return err; + if ((err = poly1305_done(&st, out, &len)) != CRYPT_OK) return err; + if (compare_testvector(out, len, tag, sizeof(tag), "POLY1305-TV1", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + /* process in one go */ + if ((err = poly1305_init(&st, k, 32)) != CRYPT_OK) return err; + if ((err = poly1305_process(&st, (unsigned char*)m, mlen)) != CRYPT_OK) return err; + if ((err = poly1305_done(&st, out, &len)) != CRYPT_OK) return err; + if (compare_testvector(out, len, tag, sizeof(tag), "POLY1305-TV2", 1) != 0) return CRYPT_FAIL_TESTVECTOR; + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/sub.mk b/optee_os/core/lib/libtomcrypt/src/mac/sub.mk new file mode 100644 index 0000000..e8690ab --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/sub.mk @@ -0,0 +1,2 @@ +subdirs-$(_CFG_CORE_LTC_HMAC) += hmac +subdirs-$(_CFG_CORE_LTC_CMAC) += omac diff --git a/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_done.c b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_done.c new file mode 100644 index 0000000..8f60bba --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_done.c @@ -0,0 +1,65 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file xcbc_done.c + XCBC Support, terminate the state +*/ + +#ifdef LTC_XCBC + +/** Terminate the XCBC-MAC state + @param xcbc XCBC state to terminate + @param out [out] Destination for the MAC tag + @param outlen [in/out] Destination size and final tag size + Return CRYPT_OK on success +*/ +int xcbc_done(xcbc_state *xcbc, unsigned char *out, unsigned long *outlen) +{ + int err, x; + LTC_ARGCHK(xcbc != NULL); + LTC_ARGCHK(out != NULL); + + /* check structure */ + if ((err = cipher_is_valid(xcbc->cipher)) != CRYPT_OK) { + return err; + } + + if ((xcbc->blocksize > cipher_descriptor[xcbc->cipher]->block_length) || (xcbc->blocksize < 0) || + (xcbc->buflen > xcbc->blocksize) || (xcbc->buflen < 0)) { + return CRYPT_INVALID_ARG; + } + + /* which key do we use? */ + if (xcbc->buflen == xcbc->blocksize) { + /* k2 */ + for (x = 0; x < xcbc->blocksize; x++) { + xcbc->IV[x] ^= xcbc->K[1][x]; + } + } else { + xcbc->IV[xcbc->buflen] ^= 0x80; + /* k3 */ + for (x = 0; x < xcbc->blocksize; x++) { + xcbc->IV[x] ^= xcbc->K[2][x]; + } + } + + /* encrypt */ + cipher_descriptor[xcbc->cipher]->ecb_encrypt(xcbc->IV, xcbc->IV, &xcbc->key); + cipher_descriptor[xcbc->cipher]->done(&xcbc->key); + + /* extract tag */ + for (x = 0; x < xcbc->blocksize && (unsigned long)x < *outlen; x++) { + out[x] = xcbc->IV[x]; + } + *outlen = x; + +#ifdef LTC_CLEAN_STACK + zeromem(xcbc, sizeof(*xcbc)); +#endif + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_file.c b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_file.c new file mode 100644 index 0000000..d8612df --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_file.c @@ -0,0 +1,87 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file xcbc_file.c + XCBC support, process a file, Tom St Denis +*/ + +#ifdef LTC_XCBC + +/** + XCBC a file + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param filename The name of the file you wish to XCBC + @param out [out] Where the authentication tag is to be stored + @param outlen [in/out] The max size and resulting size of the authentication tag + @return CRYPT_OK if successful, CRYPT_NOP if file support has been disabled +*/ +int xcbc_file(int cipher, + const unsigned char *key, unsigned long keylen, + const char *filename, + unsigned char *out, unsigned long *outlen) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(cipher); + LTC_UNUSED_PARAM(key); + LTC_UNUSED_PARAM(keylen); + LTC_UNUSED_PARAM(filename); + LTC_UNUSED_PARAM(out); + LTC_UNUSED_PARAM(outlen); + return CRYPT_NOP; +#else + size_t x; + int err; + xcbc_state xcbc; + FILE *in; + unsigned char *buf; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(filename != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((buf = XMALLOC(LTC_FILE_READ_BUFSIZE)) == NULL) { + return CRYPT_MEM; + } + + if ((err = xcbc_init(&xcbc, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + + in = fopen(filename, "rb"); + if (in == NULL) { + err = CRYPT_FILE_NOTFOUND; + goto LBL_ERR; + } + + do { + x = fread(buf, 1, LTC_FILE_READ_BUFSIZE, in); + if ((err = xcbc_process(&xcbc, buf, (unsigned long)x)) != CRYPT_OK) { + fclose(in); + goto LBL_CLEANBUF; + } + } while (x == LTC_FILE_READ_BUFSIZE); + + if (fclose(in) != 0) { + err = CRYPT_ERROR; + goto LBL_CLEANBUF; + } + + err = xcbc_done(&xcbc, out, outlen); + +LBL_CLEANBUF: + zeromem(buf, LTC_FILE_READ_BUFSIZE); +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(&xcbc, sizeof(xcbc_state)); +#endif + XFREE(buf); + return err; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_init.c b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_init.c new file mode 100644 index 0000000..6eca4a4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_init.c @@ -0,0 +1,96 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file xcbc_init.c + XCBC Support, start an XCBC state +*/ + +#ifdef LTC_XCBC + +/** Initialize XCBC-MAC state + @param xcbc [out] XCBC state to initialize + @param cipher Index of cipher to use + @param key [in] Secret key + @param keylen Length of secret key in octets + Return CRYPT_OK on success +*/ +int xcbc_init(xcbc_state *xcbc, int cipher, const unsigned char *key, unsigned long keylen) +{ + int x, y, err; + symmetric_key *skey; + unsigned long k1; + + LTC_ARGCHK(xcbc != NULL); + LTC_ARGCHK(key != NULL); + + /* schedule the key */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + +#ifdef LTC_FAST + if (cipher_descriptor[cipher]->block_length % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + skey = NULL; + + /* are we in pure XCBC mode with three keys? */ + if (keylen & LTC_XCBC_PURE) { + keylen &= ~LTC_XCBC_PURE; + + if (keylen < 2UL*cipher_descriptor[cipher]->block_length) { + return CRYPT_INVALID_ARG; + } + + k1 = keylen - 2*cipher_descriptor[cipher]->block_length; + XMEMCPY(xcbc->K[0], key, k1); + XMEMCPY(xcbc->K[1], key+k1, cipher_descriptor[cipher]->block_length); + XMEMCPY(xcbc->K[2], key+k1 + cipher_descriptor[cipher]->block_length, cipher_descriptor[cipher]->block_length); + } else { + /* use the key expansion */ + k1 = cipher_descriptor[cipher]->block_length; + + /* schedule the user key */ + skey = XCALLOC(1, sizeof(*skey)); + if (skey == NULL) { + return CRYPT_MEM; + } + + if ((err = cipher_descriptor[cipher]->setup(key, keylen, 0, skey)) != CRYPT_OK) { + goto done; + } + + /* make the three keys */ + for (y = 0; y < 3; y++) { + for (x = 0; x < cipher_descriptor[cipher]->block_length; x++) { + xcbc->K[y][x] = y + 1; + } + cipher_descriptor[cipher]->ecb_encrypt(xcbc->K[y], xcbc->K[y], skey); + } + } + + /* setup K1 */ + err = cipher_descriptor[cipher]->setup(xcbc->K[0], k1, 0, &xcbc->key); + + /* setup struct */ + zeromem(xcbc->IV, cipher_descriptor[cipher]->block_length); + xcbc->blocksize = cipher_descriptor[cipher]->block_length; + xcbc->cipher = cipher; + xcbc->buflen = 0; +done: + cipher_descriptor[cipher]->done(skey); + if (skey != NULL) { +#ifdef LTC_CLEAN_STACK + zeromem(skey, sizeof(*skey)); +#endif + XFREE(skey); + } + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_memory.c b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_memory.c new file mode 100644 index 0000000..73f78d2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_memory.c @@ -0,0 +1,59 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file xcbc_process.c + XCBC Support, XCBC-MAC a block of memory +*/ + +#ifdef LTC_XCBC + +/** XCBC-MAC a block of memory + @param cipher Index of cipher to use + @param key [in] Secret key + @param keylen Length of key in octets + @param in [in] Message to MAC + @param inlen Length of input in octets + @param out [out] Destination for the MAC tag + @param outlen [in/out] Output size and final tag size + Return CRYPT_OK on success. +*/ +int xcbc_memory(int cipher, + const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + xcbc_state *xcbc; + int err; + + /* is the cipher valid? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* Use accelerator if found */ + if (cipher_descriptor[cipher]->xcbc_memory != NULL) { + return cipher_descriptor[cipher]->xcbc_memory(key, keylen, in, inlen, out, outlen); + } + + xcbc = XCALLOC(1, sizeof(*xcbc)); + if (xcbc == NULL) { + return CRYPT_MEM; + } + + if ((err = xcbc_init(xcbc, cipher, key, keylen)) != CRYPT_OK) { + goto done; + } + + if ((err = xcbc_process(xcbc, in, inlen)) != CRYPT_OK) { + goto done; + } + + err = xcbc_done(xcbc, out, outlen); +done: + XFREE(xcbc); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_memory_multi.c b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_memory_multi.c new file mode 100644 index 0000000..d9e48d9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_memory_multi.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file xcbc_memory_multi.c + XCBC support, process multiple blocks of memory, Tom St Denis +*/ + +#ifdef LTC_XCBC + +/** + XCBC multiple blocks of memory + @param cipher The index of the desired cipher + @param key The secret key + @param keylen The length of the secret key (octets) + @param out [out] The destination of the authentication tag + @param outlen [in/out] The max size and resulting size of the authentication tag (octets) + @param in The data to send through XCBC + @param inlen The length of the data to send through XCBC (octets) + @param ... tuples of (data,len) pairs to XCBC, terminated with a (NULL,x) (x=don't care) + @return CRYPT_OK if successful +*/ +int xcbc_memory_multi(int cipher, + const unsigned char *key, unsigned long keylen, + unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...) +{ + int err; + xcbc_state *xcbc; + va_list args; + const unsigned char *curptr; + unsigned long curlen; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* allocate ram for xcbc state */ + xcbc = XMALLOC(sizeof(xcbc_state)); + if (xcbc == NULL) { + return CRYPT_MEM; + } + + /* xcbc process the message */ + if ((err = xcbc_init(xcbc, cipher, key, keylen)) != CRYPT_OK) { + goto LBL_ERR; + } + va_start(args, inlen); + curptr = in; + curlen = inlen; + for (;;) { + /* process buf */ + if ((err = xcbc_process(xcbc, curptr, curlen)) != CRYPT_OK) { + goto LBL_ERR; + } + /* step to next */ + curptr = va_arg(args, const unsigned char*); + if (curptr == NULL) { + break; + } + curlen = va_arg(args, unsigned long); + } + if ((err = xcbc_done(xcbc, out, outlen)) != CRYPT_OK) { + goto LBL_ERR; + } +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(xcbc, sizeof(xcbc_state)); +#endif + XFREE(xcbc); + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_process.c b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_process.c new file mode 100644 index 0000000..be7c617 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_process.c @@ -0,0 +1,63 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file xcbc_process.c + XCBC Support, process blocks with XCBC +*/ + +#ifdef LTC_XCBC + +/** Process data through XCBC-MAC + @param xcbc The XCBC-MAC state + @param in Input data to process + @param inlen Length of input in octets + Return CRYPT_OK on success +*/ +int xcbc_process(xcbc_state *xcbc, const unsigned char *in, unsigned long inlen) +{ + int err; +#ifdef LTC_FAST + int x; +#endif + + LTC_ARGCHK(xcbc != NULL); + LTC_ARGCHK(in != NULL); + + /* check structure */ + if ((err = cipher_is_valid(xcbc->cipher)) != CRYPT_OK) { + return err; + } + + if ((xcbc->blocksize > cipher_descriptor[xcbc->cipher]->block_length) || (xcbc->blocksize < 0) || + (xcbc->buflen > xcbc->blocksize) || (xcbc->buflen < 0)) { + return CRYPT_INVALID_ARG; + } + +#ifdef LTC_FAST + if (xcbc->buflen == 0) { + while (inlen > (unsigned long)xcbc->blocksize) { + for (x = 0; x < xcbc->blocksize; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&(xcbc->IV[x]))) ^= *(LTC_FAST_TYPE_PTR_CAST(&(in[x]))); + } + cipher_descriptor[xcbc->cipher]->ecb_encrypt(xcbc->IV, xcbc->IV, &xcbc->key); + in += xcbc->blocksize; + inlen -= xcbc->blocksize; + } + } +#endif + + while (inlen) { + if (xcbc->buflen == xcbc->blocksize) { + cipher_descriptor[xcbc->cipher]->ecb_encrypt(xcbc->IV, xcbc->IV, &xcbc->key); + xcbc->buflen = 0; + } + xcbc->IV[xcbc->buflen++] ^= *in++; + --inlen; + } + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_test.c b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_test.c new file mode 100644 index 0000000..23555de --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/mac/xcbc/xcbc_test.c @@ -0,0 +1,116 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file xcbc_test.c + XCBC Support, Test XCBC-MAC mode +*/ + +#ifdef LTC_XCBC + +/** Test XCBC-MAC mode + Return CRYPT_OK on success +*/ +int xcbc_test(void) +{ +#ifdef LTC_NO_TEST + return CRYPT_NOP; +#else + static const struct { + int msglen; + unsigned char K[16], M[34], T[16]; + } tests[] = { +{ + 0, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + + { 0 }, + + { 0x75, 0xf0, 0x25, 0x1d, 0x52, 0x8a, 0xc0, 0x1c, + 0x45, 0x73, 0xdf, 0xd5, 0x84, 0xd7, 0x9f, 0x29 } +}, + +{ + 3, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + + { 0x00, 0x01, 0x02 }, + + { 0x5b, 0x37, 0x65, 0x80, 0xae, 0x2f, 0x19, 0xaf, + 0xe7, 0x21, 0x9c, 0xee, 0xf1, 0x72, 0x75, 0x6f } +}, + +{ + 16, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + + { 0xd2, 0xa2, 0x46, 0xfa, 0x34, 0x9b, 0x68, 0xa7, + 0x99, 0x98, 0xa4, 0x39, 0x4f, 0xf7, 0xa2, 0x63 } +}, + +{ + 32, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + + { 0xf5, 0x4f, 0x0e, 0xc8, 0xd2, 0xb9, 0xf3, 0xd3, + 0x68, 0x07, 0x73, 0x4b, 0xd5, 0x28, 0x3f, 0xd4 } +}, + +{ + 34, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21 }, + + { 0xbe, 0xcb, 0xb3, 0xbc, 0xcd, 0xb5, 0x18, 0xa3, + 0x06, 0x77, 0xd5, 0x48, 0x1f, 0xb6, 0xb4, 0xd8 }, +}, + + + +}; + unsigned char T[16]; + unsigned long taglen; + int err, x, idx; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + taglen = 16; + if ((err = xcbc_memory(idx, tests[x].K, 16, tests[x].M, tests[x].msglen, T, &taglen)) != CRYPT_OK) { + return err; + } + if (compare_testvector(T, taglen, tests[x].T, 16, "XCBC", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + + return CRYPT_OK; +#endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/math/fp/ltc_ecc_fp_mulmod.c b/optee_os/core/lib/libtomcrypt/src/math/fp/ltc_ecc_fp_mulmod.c new file mode 100644 index 0000000..37cee9f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/fp/ltc_ecc_fp_mulmod.c @@ -0,0 +1,1580 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ltc_ecc_fp_mulmod.c + ECC Crypto, Tom St Denis +*/ + +#if defined(LTC_MECC) && defined(LTC_MECC_FP) +#include + +/* number of entries in the cache */ +#ifndef FP_ENTRIES +#define FP_ENTRIES 16 +#endif + +/* number of bits in LUT */ +#ifndef FP_LUT +#define FP_LUT 8U +#endif + +#if (FP_LUT > 12) || (FP_LUT < 2) + #error FP_LUT must be between 2 and 12 inclusively +#endif + +/** Our FP cache */ +static struct { + ecc_point *g, /* cached COPY of base point */ + *LUT[1U< 6 + { 1, 0, 0 }, { 2, 1, 64 }, { 2, 2, 64 }, { 3, 3, 64 }, { 2, 4, 64 }, { 3, 5, 64 }, { 3, 6, 64 }, { 4, 7, 64 }, + { 2, 8, 64 }, { 3, 9, 64 }, { 3, 10, 64 }, { 4, 11, 64 }, { 3, 12, 64 }, { 4, 13, 64 }, { 4, 14, 64 }, { 5, 15, 64 }, + { 2, 16, 64 }, { 3, 17, 64 }, { 3, 18, 64 }, { 4, 19, 64 }, { 3, 20, 64 }, { 4, 21, 64 }, { 4, 22, 64 }, { 5, 23, 64 }, + { 3, 24, 64 }, { 4, 25, 64 }, { 4, 26, 64 }, { 5, 27, 64 }, { 4, 28, 64 }, { 5, 29, 64 }, { 5, 30, 64 }, { 6, 31, 64 }, + { 2, 32, 64 }, { 3, 33, 64 }, { 3, 34, 64 }, { 4, 35, 64 }, { 3, 36, 64 }, { 4, 37, 64 }, { 4, 38, 64 }, { 5, 39, 64 }, + { 3, 40, 64 }, { 4, 41, 64 }, { 4, 42, 64 }, { 5, 43, 64 }, { 4, 44, 64 }, { 5, 45, 64 }, { 5, 46, 64 }, { 6, 47, 64 }, + { 3, 48, 64 }, { 4, 49, 64 }, { 4, 50, 64 }, { 5, 51, 64 }, { 4, 52, 64 }, { 5, 53, 64 }, { 5, 54, 64 }, { 6, 55, 64 }, + { 4, 56, 64 }, { 5, 57, 64 }, { 5, 58, 64 }, { 6, 59, 64 }, { 5, 60, 64 }, { 6, 61, 64 }, { 6, 62, 64 }, { 7, 63, 64 }, +#if FP_LUT > 7 + { 1, 0, 0 }, { 2, 1, 128 }, { 2, 2, 128 }, { 3, 3, 128 }, { 2, 4, 128 }, { 3, 5, 128 }, { 3, 6, 128 }, { 4, 7, 128 }, + { 2, 8, 128 }, { 3, 9, 128 }, { 3, 10, 128 }, { 4, 11, 128 }, { 3, 12, 128 }, { 4, 13, 128 }, { 4, 14, 128 }, { 5, 15, 128 }, + { 2, 16, 128 }, { 3, 17, 128 }, { 3, 18, 128 }, { 4, 19, 128 }, { 3, 20, 128 }, { 4, 21, 128 }, { 4, 22, 128 }, { 5, 23, 128 }, + { 3, 24, 128 }, { 4, 25, 128 }, { 4, 26, 128 }, { 5, 27, 128 }, { 4, 28, 128 }, { 5, 29, 128 }, { 5, 30, 128 }, { 6, 31, 128 }, + { 2, 32, 128 }, { 3, 33, 128 }, { 3, 34, 128 }, { 4, 35, 128 }, { 3, 36, 128 }, { 4, 37, 128 }, { 4, 38, 128 }, { 5, 39, 128 }, + { 3, 40, 128 }, { 4, 41, 128 }, { 4, 42, 128 }, { 5, 43, 128 }, { 4, 44, 128 }, { 5, 45, 128 }, { 5, 46, 128 }, { 6, 47, 128 }, + { 3, 48, 128 }, { 4, 49, 128 }, { 4, 50, 128 }, { 5, 51, 128 }, { 4, 52, 128 }, { 5, 53, 128 }, { 5, 54, 128 }, { 6, 55, 128 }, + { 4, 56, 128 }, { 5, 57, 128 }, { 5, 58, 128 }, { 6, 59, 128 }, { 5, 60, 128 }, { 6, 61, 128 }, { 6, 62, 128 }, { 7, 63, 128 }, + { 2, 64, 128 }, { 3, 65, 128 }, { 3, 66, 128 }, { 4, 67, 128 }, { 3, 68, 128 }, { 4, 69, 128 }, { 4, 70, 128 }, { 5, 71, 128 }, + { 3, 72, 128 }, { 4, 73, 128 }, { 4, 74, 128 }, { 5, 75, 128 }, { 4, 76, 128 }, { 5, 77, 128 }, { 5, 78, 128 }, { 6, 79, 128 }, + { 3, 80, 128 }, { 4, 81, 128 }, { 4, 82, 128 }, { 5, 83, 128 }, { 4, 84, 128 }, { 5, 85, 128 }, { 5, 86, 128 }, { 6, 87, 128 }, + { 4, 88, 128 }, { 5, 89, 128 }, { 5, 90, 128 }, { 6, 91, 128 }, { 5, 92, 128 }, { 6, 93, 128 }, { 6, 94, 128 }, { 7, 95, 128 }, + { 3, 96, 128 }, { 4, 97, 128 }, { 4, 98, 128 }, { 5, 99, 128 }, { 4, 100, 128 }, { 5, 101, 128 }, { 5, 102, 128 }, { 6, 103, 128 }, + { 4, 104, 128 }, { 5, 105, 128 }, { 5, 106, 128 }, { 6, 107, 128 }, { 5, 108, 128 }, { 6, 109, 128 }, { 6, 110, 128 }, { 7, 111, 128 }, + { 4, 112, 128 }, { 5, 113, 128 }, { 5, 114, 128 }, { 6, 115, 128 }, { 5, 116, 128 }, { 6, 117, 128 }, { 6, 118, 128 }, { 7, 119, 128 }, + { 5, 120, 128 }, { 6, 121, 128 }, { 6, 122, 128 }, { 7, 123, 128 }, { 6, 124, 128 }, { 7, 125, 128 }, { 7, 126, 128 }, { 8, 127, 128 }, +#if FP_LUT > 8 + { 1, 0, 0 }, { 2, 1, 256 }, { 2, 2, 256 }, { 3, 3, 256 }, { 2, 4, 256 }, { 3, 5, 256 }, { 3, 6, 256 }, { 4, 7, 256 }, + { 2, 8, 256 }, { 3, 9, 256 }, { 3, 10, 256 }, { 4, 11, 256 }, { 3, 12, 256 }, { 4, 13, 256 }, { 4, 14, 256 }, { 5, 15, 256 }, + { 2, 16, 256 }, { 3, 17, 256 }, { 3, 18, 256 }, { 4, 19, 256 }, { 3, 20, 256 }, { 4, 21, 256 }, { 4, 22, 256 }, { 5, 23, 256 }, + { 3, 24, 256 }, { 4, 25, 256 }, { 4, 26, 256 }, { 5, 27, 256 }, { 4, 28, 256 }, { 5, 29, 256 }, { 5, 30, 256 }, { 6, 31, 256 }, + { 2, 32, 256 }, { 3, 33, 256 }, { 3, 34, 256 }, { 4, 35, 256 }, { 3, 36, 256 }, { 4, 37, 256 }, { 4, 38, 256 }, { 5, 39, 256 }, + { 3, 40, 256 }, { 4, 41, 256 }, { 4, 42, 256 }, { 5, 43, 256 }, { 4, 44, 256 }, { 5, 45, 256 }, { 5, 46, 256 }, { 6, 47, 256 }, + { 3, 48, 256 }, { 4, 49, 256 }, { 4, 50, 256 }, { 5, 51, 256 }, { 4, 52, 256 }, { 5, 53, 256 }, { 5, 54, 256 }, { 6, 55, 256 }, + { 4, 56, 256 }, { 5, 57, 256 }, { 5, 58, 256 }, { 6, 59, 256 }, { 5, 60, 256 }, { 6, 61, 256 }, { 6, 62, 256 }, { 7, 63, 256 }, + { 2, 64, 256 }, { 3, 65, 256 }, { 3, 66, 256 }, { 4, 67, 256 }, { 3, 68, 256 }, { 4, 69, 256 }, { 4, 70, 256 }, { 5, 71, 256 }, + { 3, 72, 256 }, { 4, 73, 256 }, { 4, 74, 256 }, { 5, 75, 256 }, { 4, 76, 256 }, { 5, 77, 256 }, { 5, 78, 256 }, { 6, 79, 256 }, + { 3, 80, 256 }, { 4, 81, 256 }, { 4, 82, 256 }, { 5, 83, 256 }, { 4, 84, 256 }, { 5, 85, 256 }, { 5, 86, 256 }, { 6, 87, 256 }, + { 4, 88, 256 }, { 5, 89, 256 }, { 5, 90, 256 }, { 6, 91, 256 }, { 5, 92, 256 }, { 6, 93, 256 }, { 6, 94, 256 }, { 7, 95, 256 }, + { 3, 96, 256 }, { 4, 97, 256 }, { 4, 98, 256 }, { 5, 99, 256 }, { 4, 100, 256 }, { 5, 101, 256 }, { 5, 102, 256 }, { 6, 103, 256 }, + { 4, 104, 256 }, { 5, 105, 256 }, { 5, 106, 256 }, { 6, 107, 256 }, { 5, 108, 256 }, { 6, 109, 256 }, { 6, 110, 256 }, { 7, 111, 256 }, + { 4, 112, 256 }, { 5, 113, 256 }, { 5, 114, 256 }, { 6, 115, 256 }, { 5, 116, 256 }, { 6, 117, 256 }, { 6, 118, 256 }, { 7, 119, 256 }, + { 5, 120, 256 }, { 6, 121, 256 }, { 6, 122, 256 }, { 7, 123, 256 }, { 6, 124, 256 }, { 7, 125, 256 }, { 7, 126, 256 }, { 8, 127, 256 }, + { 2, 128, 256 }, { 3, 129, 256 }, { 3, 130, 256 }, { 4, 131, 256 }, { 3, 132, 256 }, { 4, 133, 256 }, { 4, 134, 256 }, { 5, 135, 256 }, + { 3, 136, 256 }, { 4, 137, 256 }, { 4, 138, 256 }, { 5, 139, 256 }, { 4, 140, 256 }, { 5, 141, 256 }, { 5, 142, 256 }, { 6, 143, 256 }, + { 3, 144, 256 }, { 4, 145, 256 }, { 4, 146, 256 }, { 5, 147, 256 }, { 4, 148, 256 }, { 5, 149, 256 }, { 5, 150, 256 }, { 6, 151, 256 }, + { 4, 152, 256 }, { 5, 153, 256 }, { 5, 154, 256 }, { 6, 155, 256 }, { 5, 156, 256 }, { 6, 157, 256 }, { 6, 158, 256 }, { 7, 159, 256 }, + { 3, 160, 256 }, { 4, 161, 256 }, { 4, 162, 256 }, { 5, 163, 256 }, { 4, 164, 256 }, { 5, 165, 256 }, { 5, 166, 256 }, { 6, 167, 256 }, + { 4, 168, 256 }, { 5, 169, 256 }, { 5, 170, 256 }, { 6, 171, 256 }, { 5, 172, 256 }, { 6, 173, 256 }, { 6, 174, 256 }, { 7, 175, 256 }, + { 4, 176, 256 }, { 5, 177, 256 }, { 5, 178, 256 }, { 6, 179, 256 }, { 5, 180, 256 }, { 6, 181, 256 }, { 6, 182, 256 }, { 7, 183, 256 }, + { 5, 184, 256 }, { 6, 185, 256 }, { 6, 186, 256 }, { 7, 187, 256 }, { 6, 188, 256 }, { 7, 189, 256 }, { 7, 190, 256 }, { 8, 191, 256 }, + { 3, 192, 256 }, { 4, 193, 256 }, { 4, 194, 256 }, { 5, 195, 256 }, { 4, 196, 256 }, { 5, 197, 256 }, { 5, 198, 256 }, { 6, 199, 256 }, + { 4, 200, 256 }, { 5, 201, 256 }, { 5, 202, 256 }, { 6, 203, 256 }, { 5, 204, 256 }, { 6, 205, 256 }, { 6, 206, 256 }, { 7, 207, 256 }, + { 4, 208, 256 }, { 5, 209, 256 }, { 5, 210, 256 }, { 6, 211, 256 }, { 5, 212, 256 }, { 6, 213, 256 }, { 6, 214, 256 }, { 7, 215, 256 }, + { 5, 216, 256 }, { 6, 217, 256 }, { 6, 218, 256 }, { 7, 219, 256 }, { 6, 220, 256 }, { 7, 221, 256 }, { 7, 222, 256 }, { 8, 223, 256 }, + { 4, 224, 256 }, { 5, 225, 256 }, { 5, 226, 256 }, { 6, 227, 256 }, { 5, 228, 256 }, { 6, 229, 256 }, { 6, 230, 256 }, { 7, 231, 256 }, + { 5, 232, 256 }, { 6, 233, 256 }, { 6, 234, 256 }, { 7, 235, 256 }, { 6, 236, 256 }, { 7, 237, 256 }, { 7, 238, 256 }, { 8, 239, 256 }, + { 5, 240, 256 }, { 6, 241, 256 }, { 6, 242, 256 }, { 7, 243, 256 }, { 6, 244, 256 }, { 7, 245, 256 }, { 7, 246, 256 }, { 8, 247, 256 }, + { 6, 248, 256 }, { 7, 249, 256 }, { 7, 250, 256 }, { 8, 251, 256 }, { 7, 252, 256 }, { 8, 253, 256 }, { 8, 254, 256 }, { 9, 255, 256 }, +#if FP_LUT > 9 + { 1, 0, 0 }, { 2, 1, 512 }, { 2, 2, 512 }, { 3, 3, 512 }, { 2, 4, 512 }, { 3, 5, 512 }, { 3, 6, 512 }, { 4, 7, 512 }, + { 2, 8, 512 }, { 3, 9, 512 }, { 3, 10, 512 }, { 4, 11, 512 }, { 3, 12, 512 }, { 4, 13, 512 }, { 4, 14, 512 }, { 5, 15, 512 }, + { 2, 16, 512 }, { 3, 17, 512 }, { 3, 18, 512 }, { 4, 19, 512 }, { 3, 20, 512 }, { 4, 21, 512 }, { 4, 22, 512 }, { 5, 23, 512 }, + { 3, 24, 512 }, { 4, 25, 512 }, { 4, 26, 512 }, { 5, 27, 512 }, { 4, 28, 512 }, { 5, 29, 512 }, { 5, 30, 512 }, { 6, 31, 512 }, + { 2, 32, 512 }, { 3, 33, 512 }, { 3, 34, 512 }, { 4, 35, 512 }, { 3, 36, 512 }, { 4, 37, 512 }, { 4, 38, 512 }, { 5, 39, 512 }, + { 3, 40, 512 }, { 4, 41, 512 }, { 4, 42, 512 }, { 5, 43, 512 }, { 4, 44, 512 }, { 5, 45, 512 }, { 5, 46, 512 }, { 6, 47, 512 }, + { 3, 48, 512 }, { 4, 49, 512 }, { 4, 50, 512 }, { 5, 51, 512 }, { 4, 52, 512 }, { 5, 53, 512 }, { 5, 54, 512 }, { 6, 55, 512 }, + { 4, 56, 512 }, { 5, 57, 512 }, { 5, 58, 512 }, { 6, 59, 512 }, { 5, 60, 512 }, { 6, 61, 512 }, { 6, 62, 512 }, { 7, 63, 512 }, + { 2, 64, 512 }, { 3, 65, 512 }, { 3, 66, 512 }, { 4, 67, 512 }, { 3, 68, 512 }, { 4, 69, 512 }, { 4, 70, 512 }, { 5, 71, 512 }, + { 3, 72, 512 }, { 4, 73, 512 }, { 4, 74, 512 }, { 5, 75, 512 }, { 4, 76, 512 }, { 5, 77, 512 }, { 5, 78, 512 }, { 6, 79, 512 }, + { 3, 80, 512 }, { 4, 81, 512 }, { 4, 82, 512 }, { 5, 83, 512 }, { 4, 84, 512 }, { 5, 85, 512 }, { 5, 86, 512 }, { 6, 87, 512 }, + { 4, 88, 512 }, { 5, 89, 512 }, { 5, 90, 512 }, { 6, 91, 512 }, { 5, 92, 512 }, { 6, 93, 512 }, { 6, 94, 512 }, { 7, 95, 512 }, + { 3, 96, 512 }, { 4, 97, 512 }, { 4, 98, 512 }, { 5, 99, 512 }, { 4, 100, 512 }, { 5, 101, 512 }, { 5, 102, 512 }, { 6, 103, 512 }, + { 4, 104, 512 }, { 5, 105, 512 }, { 5, 106, 512 }, { 6, 107, 512 }, { 5, 108, 512 }, { 6, 109, 512 }, { 6, 110, 512 }, { 7, 111, 512 }, + { 4, 112, 512 }, { 5, 113, 512 }, { 5, 114, 512 }, { 6, 115, 512 }, { 5, 116, 512 }, { 6, 117, 512 }, { 6, 118, 512 }, { 7, 119, 512 }, + { 5, 120, 512 }, { 6, 121, 512 }, { 6, 122, 512 }, { 7, 123, 512 }, { 6, 124, 512 }, { 7, 125, 512 }, { 7, 126, 512 }, { 8, 127, 512 }, + { 2, 128, 512 }, { 3, 129, 512 }, { 3, 130, 512 }, { 4, 131, 512 }, { 3, 132, 512 }, { 4, 133, 512 }, { 4, 134, 512 }, { 5, 135, 512 }, + { 3, 136, 512 }, { 4, 137, 512 }, { 4, 138, 512 }, { 5, 139, 512 }, { 4, 140, 512 }, { 5, 141, 512 }, { 5, 142, 512 }, { 6, 143, 512 }, + { 3, 144, 512 }, { 4, 145, 512 }, { 4, 146, 512 }, { 5, 147, 512 }, { 4, 148, 512 }, { 5, 149, 512 }, { 5, 150, 512 }, { 6, 151, 512 }, + { 4, 152, 512 }, { 5, 153, 512 }, { 5, 154, 512 }, { 6, 155, 512 }, { 5, 156, 512 }, { 6, 157, 512 }, { 6, 158, 512 }, { 7, 159, 512 }, + { 3, 160, 512 }, { 4, 161, 512 }, { 4, 162, 512 }, { 5, 163, 512 }, { 4, 164, 512 }, { 5, 165, 512 }, { 5, 166, 512 }, { 6, 167, 512 }, + { 4, 168, 512 }, { 5, 169, 512 }, { 5, 170, 512 }, { 6, 171, 512 }, { 5, 172, 512 }, { 6, 173, 512 }, { 6, 174, 512 }, { 7, 175, 512 }, + { 4, 176, 512 }, { 5, 177, 512 }, { 5, 178, 512 }, { 6, 179, 512 }, { 5, 180, 512 }, { 6, 181, 512 }, { 6, 182, 512 }, { 7, 183, 512 }, + { 5, 184, 512 }, { 6, 185, 512 }, { 6, 186, 512 }, { 7, 187, 512 }, { 6, 188, 512 }, { 7, 189, 512 }, { 7, 190, 512 }, { 8, 191, 512 }, + { 3, 192, 512 }, { 4, 193, 512 }, { 4, 194, 512 }, { 5, 195, 512 }, { 4, 196, 512 }, { 5, 197, 512 }, { 5, 198, 512 }, { 6, 199, 512 }, + { 4, 200, 512 }, { 5, 201, 512 }, { 5, 202, 512 }, { 6, 203, 512 }, { 5, 204, 512 }, { 6, 205, 512 }, { 6, 206, 512 }, { 7, 207, 512 }, + { 4, 208, 512 }, { 5, 209, 512 }, { 5, 210, 512 }, { 6, 211, 512 }, { 5, 212, 512 }, { 6, 213, 512 }, { 6, 214, 512 }, { 7, 215, 512 }, + { 5, 216, 512 }, { 6, 217, 512 }, { 6, 218, 512 }, { 7, 219, 512 }, { 6, 220, 512 }, { 7, 221, 512 }, { 7, 222, 512 }, { 8, 223, 512 }, + { 4, 224, 512 }, { 5, 225, 512 }, { 5, 226, 512 }, { 6, 227, 512 }, { 5, 228, 512 }, { 6, 229, 512 }, { 6, 230, 512 }, { 7, 231, 512 }, + { 5, 232, 512 }, { 6, 233, 512 }, { 6, 234, 512 }, { 7, 235, 512 }, { 6, 236, 512 }, { 7, 237, 512 }, { 7, 238, 512 }, { 8, 239, 512 }, + { 5, 240, 512 }, { 6, 241, 512 }, { 6, 242, 512 }, { 7, 243, 512 }, { 6, 244, 512 }, { 7, 245, 512 }, { 7, 246, 512 }, { 8, 247, 512 }, + { 6, 248, 512 }, { 7, 249, 512 }, { 7, 250, 512 }, { 8, 251, 512 }, { 7, 252, 512 }, { 8, 253, 512 }, { 8, 254, 512 }, { 9, 255, 512 }, + { 2, 256, 512 }, { 3, 257, 512 }, { 3, 258, 512 }, { 4, 259, 512 }, { 3, 260, 512 }, { 4, 261, 512 }, { 4, 262, 512 }, { 5, 263, 512 }, + { 3, 264, 512 }, { 4, 265, 512 }, { 4, 266, 512 }, { 5, 267, 512 }, { 4, 268, 512 }, { 5, 269, 512 }, { 5, 270, 512 }, { 6, 271, 512 }, + { 3, 272, 512 }, { 4, 273, 512 }, { 4, 274, 512 }, { 5, 275, 512 }, { 4, 276, 512 }, { 5, 277, 512 }, { 5, 278, 512 }, { 6, 279, 512 }, + { 4, 280, 512 }, { 5, 281, 512 }, { 5, 282, 512 }, { 6, 283, 512 }, { 5, 284, 512 }, { 6, 285, 512 }, { 6, 286, 512 }, { 7, 287, 512 }, + { 3, 288, 512 }, { 4, 289, 512 }, { 4, 290, 512 }, { 5, 291, 512 }, { 4, 292, 512 }, { 5, 293, 512 }, { 5, 294, 512 }, { 6, 295, 512 }, + { 4, 296, 512 }, { 5, 297, 512 }, { 5, 298, 512 }, { 6, 299, 512 }, { 5, 300, 512 }, { 6, 301, 512 }, { 6, 302, 512 }, { 7, 303, 512 }, + { 4, 304, 512 }, { 5, 305, 512 }, { 5, 306, 512 }, { 6, 307, 512 }, { 5, 308, 512 }, { 6, 309, 512 }, { 6, 310, 512 }, { 7, 311, 512 }, + { 5, 312, 512 }, { 6, 313, 512 }, { 6, 314, 512 }, { 7, 315, 512 }, { 6, 316, 512 }, { 7, 317, 512 }, { 7, 318, 512 }, { 8, 319, 512 }, + { 3, 320, 512 }, { 4, 321, 512 }, { 4, 322, 512 }, { 5, 323, 512 }, { 4, 324, 512 }, { 5, 325, 512 }, { 5, 326, 512 }, { 6, 327, 512 }, + { 4, 328, 512 }, { 5, 329, 512 }, { 5, 330, 512 }, { 6, 331, 512 }, { 5, 332, 512 }, { 6, 333, 512 }, { 6, 334, 512 }, { 7, 335, 512 }, + { 4, 336, 512 }, { 5, 337, 512 }, { 5, 338, 512 }, { 6, 339, 512 }, { 5, 340, 512 }, { 6, 341, 512 }, { 6, 342, 512 }, { 7, 343, 512 }, + { 5, 344, 512 }, { 6, 345, 512 }, { 6, 346, 512 }, { 7, 347, 512 }, { 6, 348, 512 }, { 7, 349, 512 }, { 7, 350, 512 }, { 8, 351, 512 }, + { 4, 352, 512 }, { 5, 353, 512 }, { 5, 354, 512 }, { 6, 355, 512 }, { 5, 356, 512 }, { 6, 357, 512 }, { 6, 358, 512 }, { 7, 359, 512 }, + { 5, 360, 512 }, { 6, 361, 512 }, { 6, 362, 512 }, { 7, 363, 512 }, { 6, 364, 512 }, { 7, 365, 512 }, { 7, 366, 512 }, { 8, 367, 512 }, + { 5, 368, 512 }, { 6, 369, 512 }, { 6, 370, 512 }, { 7, 371, 512 }, { 6, 372, 512 }, { 7, 373, 512 }, { 7, 374, 512 }, { 8, 375, 512 }, + { 6, 376, 512 }, { 7, 377, 512 }, { 7, 378, 512 }, { 8, 379, 512 }, { 7, 380, 512 }, { 8, 381, 512 }, { 8, 382, 512 }, { 9, 383, 512 }, + { 3, 384, 512 }, { 4, 385, 512 }, { 4, 386, 512 }, { 5, 387, 512 }, { 4, 388, 512 }, { 5, 389, 512 }, { 5, 390, 512 }, { 6, 391, 512 }, + { 4, 392, 512 }, { 5, 393, 512 }, { 5, 394, 512 }, { 6, 395, 512 }, { 5, 396, 512 }, { 6, 397, 512 }, { 6, 398, 512 }, { 7, 399, 512 }, + { 4, 400, 512 }, { 5, 401, 512 }, { 5, 402, 512 }, { 6, 403, 512 }, { 5, 404, 512 }, { 6, 405, 512 }, { 6, 406, 512 }, { 7, 407, 512 }, + { 5, 408, 512 }, { 6, 409, 512 }, { 6, 410, 512 }, { 7, 411, 512 }, { 6, 412, 512 }, { 7, 413, 512 }, { 7, 414, 512 }, { 8, 415, 512 }, + { 4, 416, 512 }, { 5, 417, 512 }, { 5, 418, 512 }, { 6, 419, 512 }, { 5, 420, 512 }, { 6, 421, 512 }, { 6, 422, 512 }, { 7, 423, 512 }, + { 5, 424, 512 }, { 6, 425, 512 }, { 6, 426, 512 }, { 7, 427, 512 }, { 6, 428, 512 }, { 7, 429, 512 }, { 7, 430, 512 }, { 8, 431, 512 }, + { 5, 432, 512 }, { 6, 433, 512 }, { 6, 434, 512 }, { 7, 435, 512 }, { 6, 436, 512 }, { 7, 437, 512 }, { 7, 438, 512 }, { 8, 439, 512 }, + { 6, 440, 512 }, { 7, 441, 512 }, { 7, 442, 512 }, { 8, 443, 512 }, { 7, 444, 512 }, { 8, 445, 512 }, { 8, 446, 512 }, { 9, 447, 512 }, + { 4, 448, 512 }, { 5, 449, 512 }, { 5, 450, 512 }, { 6, 451, 512 }, { 5, 452, 512 }, { 6, 453, 512 }, { 6, 454, 512 }, { 7, 455, 512 }, + { 5, 456, 512 }, { 6, 457, 512 }, { 6, 458, 512 }, { 7, 459, 512 }, { 6, 460, 512 }, { 7, 461, 512 }, { 7, 462, 512 }, { 8, 463, 512 }, + { 5, 464, 512 }, { 6, 465, 512 }, { 6, 466, 512 }, { 7, 467, 512 }, { 6, 468, 512 }, { 7, 469, 512 }, { 7, 470, 512 }, { 8, 471, 512 }, + { 6, 472, 512 }, { 7, 473, 512 }, { 7, 474, 512 }, { 8, 475, 512 }, { 7, 476, 512 }, { 8, 477, 512 }, { 8, 478, 512 }, { 9, 479, 512 }, + { 5, 480, 512 }, { 6, 481, 512 }, { 6, 482, 512 }, { 7, 483, 512 }, { 6, 484, 512 }, { 7, 485, 512 }, { 7, 486, 512 }, { 8, 487, 512 }, + { 6, 488, 512 }, { 7, 489, 512 }, { 7, 490, 512 }, { 8, 491, 512 }, { 7, 492, 512 }, { 8, 493, 512 }, { 8, 494, 512 }, { 9, 495, 512 }, + { 6, 496, 512 }, { 7, 497, 512 }, { 7, 498, 512 }, { 8, 499, 512 }, { 7, 500, 512 }, { 8, 501, 512 }, { 8, 502, 512 }, { 9, 503, 512 }, + { 7, 504, 512 }, { 8, 505, 512 }, { 8, 506, 512 }, { 9, 507, 512 }, { 8, 508, 512 }, { 9, 509, 512 }, { 9, 510, 512 }, { 10, 511, 512 }, +#if FP_LUT > 10 + { 1, 0, 0 }, { 2, 1, 1024 }, { 2, 2, 1024 }, { 3, 3, 1024 }, { 2, 4, 1024 }, { 3, 5, 1024 }, { 3, 6, 1024 }, { 4, 7, 1024 }, + { 2, 8, 1024 }, { 3, 9, 1024 }, { 3, 10, 1024 }, { 4, 11, 1024 }, { 3, 12, 1024 }, { 4, 13, 1024 }, { 4, 14, 1024 }, { 5, 15, 1024 }, + { 2, 16, 1024 }, { 3, 17, 1024 }, { 3, 18, 1024 }, { 4, 19, 1024 }, { 3, 20, 1024 }, { 4, 21, 1024 }, { 4, 22, 1024 }, { 5, 23, 1024 }, + { 3, 24, 1024 }, { 4, 25, 1024 }, { 4, 26, 1024 }, { 5, 27, 1024 }, { 4, 28, 1024 }, { 5, 29, 1024 }, { 5, 30, 1024 }, { 6, 31, 1024 }, + { 2, 32, 1024 }, { 3, 33, 1024 }, { 3, 34, 1024 }, { 4, 35, 1024 }, { 3, 36, 1024 }, { 4, 37, 1024 }, { 4, 38, 1024 }, { 5, 39, 1024 }, + { 3, 40, 1024 }, { 4, 41, 1024 }, { 4, 42, 1024 }, { 5, 43, 1024 }, { 4, 44, 1024 }, { 5, 45, 1024 }, { 5, 46, 1024 }, { 6, 47, 1024 }, + { 3, 48, 1024 }, { 4, 49, 1024 }, { 4, 50, 1024 }, { 5, 51, 1024 }, { 4, 52, 1024 }, { 5, 53, 1024 }, { 5, 54, 1024 }, { 6, 55, 1024 }, + { 4, 56, 1024 }, { 5, 57, 1024 }, { 5, 58, 1024 }, { 6, 59, 1024 }, { 5, 60, 1024 }, { 6, 61, 1024 }, { 6, 62, 1024 }, { 7, 63, 1024 }, + { 2, 64, 1024 }, { 3, 65, 1024 }, { 3, 66, 1024 }, { 4, 67, 1024 }, { 3, 68, 1024 }, { 4, 69, 1024 }, { 4, 70, 1024 }, { 5, 71, 1024 }, + { 3, 72, 1024 }, { 4, 73, 1024 }, { 4, 74, 1024 }, { 5, 75, 1024 }, { 4, 76, 1024 }, { 5, 77, 1024 }, { 5, 78, 1024 }, { 6, 79, 1024 }, + { 3, 80, 1024 }, { 4, 81, 1024 }, { 4, 82, 1024 }, { 5, 83, 1024 }, { 4, 84, 1024 }, { 5, 85, 1024 }, { 5, 86, 1024 }, { 6, 87, 1024 }, + { 4, 88, 1024 }, { 5, 89, 1024 }, { 5, 90, 1024 }, { 6, 91, 1024 }, { 5, 92, 1024 }, { 6, 93, 1024 }, { 6, 94, 1024 }, { 7, 95, 1024 }, + { 3, 96, 1024 }, { 4, 97, 1024 }, { 4, 98, 1024 }, { 5, 99, 1024 }, { 4, 100, 1024 }, { 5, 101, 1024 }, { 5, 102, 1024 }, { 6, 103, 1024 }, + { 4, 104, 1024 }, { 5, 105, 1024 }, { 5, 106, 1024 }, { 6, 107, 1024 }, { 5, 108, 1024 }, { 6, 109, 1024 }, { 6, 110, 1024 }, { 7, 111, 1024 }, + { 4, 112, 1024 }, { 5, 113, 1024 }, { 5, 114, 1024 }, { 6, 115, 1024 }, { 5, 116, 1024 }, { 6, 117, 1024 }, { 6, 118, 1024 }, { 7, 119, 1024 }, + { 5, 120, 1024 }, { 6, 121, 1024 }, { 6, 122, 1024 }, { 7, 123, 1024 }, { 6, 124, 1024 }, { 7, 125, 1024 }, { 7, 126, 1024 }, { 8, 127, 1024 }, + { 2, 128, 1024 }, { 3, 129, 1024 }, { 3, 130, 1024 }, { 4, 131, 1024 }, { 3, 132, 1024 }, { 4, 133, 1024 }, { 4, 134, 1024 }, { 5, 135, 1024 }, + { 3, 136, 1024 }, { 4, 137, 1024 }, { 4, 138, 1024 }, { 5, 139, 1024 }, { 4, 140, 1024 }, { 5, 141, 1024 }, { 5, 142, 1024 }, { 6, 143, 1024 }, + { 3, 144, 1024 }, { 4, 145, 1024 }, { 4, 146, 1024 }, { 5, 147, 1024 }, { 4, 148, 1024 }, { 5, 149, 1024 }, { 5, 150, 1024 }, { 6, 151, 1024 }, + { 4, 152, 1024 }, { 5, 153, 1024 }, { 5, 154, 1024 }, { 6, 155, 1024 }, { 5, 156, 1024 }, { 6, 157, 1024 }, { 6, 158, 1024 }, { 7, 159, 1024 }, + { 3, 160, 1024 }, { 4, 161, 1024 }, { 4, 162, 1024 }, { 5, 163, 1024 }, { 4, 164, 1024 }, { 5, 165, 1024 }, { 5, 166, 1024 }, { 6, 167, 1024 }, + { 4, 168, 1024 }, { 5, 169, 1024 }, { 5, 170, 1024 }, { 6, 171, 1024 }, { 5, 172, 1024 }, { 6, 173, 1024 }, { 6, 174, 1024 }, { 7, 175, 1024 }, + { 4, 176, 1024 }, { 5, 177, 1024 }, { 5, 178, 1024 }, { 6, 179, 1024 }, { 5, 180, 1024 }, { 6, 181, 1024 }, { 6, 182, 1024 }, { 7, 183, 1024 }, + { 5, 184, 1024 }, { 6, 185, 1024 }, { 6, 186, 1024 }, { 7, 187, 1024 }, { 6, 188, 1024 }, { 7, 189, 1024 }, { 7, 190, 1024 }, { 8, 191, 1024 }, + { 3, 192, 1024 }, { 4, 193, 1024 }, { 4, 194, 1024 }, { 5, 195, 1024 }, { 4, 196, 1024 }, { 5, 197, 1024 }, { 5, 198, 1024 }, { 6, 199, 1024 }, + { 4, 200, 1024 }, { 5, 201, 1024 }, { 5, 202, 1024 }, { 6, 203, 1024 }, { 5, 204, 1024 }, { 6, 205, 1024 }, { 6, 206, 1024 }, { 7, 207, 1024 }, + { 4, 208, 1024 }, { 5, 209, 1024 }, { 5, 210, 1024 }, { 6, 211, 1024 }, { 5, 212, 1024 }, { 6, 213, 1024 }, { 6, 214, 1024 }, { 7, 215, 1024 }, + { 5, 216, 1024 }, { 6, 217, 1024 }, { 6, 218, 1024 }, { 7, 219, 1024 }, { 6, 220, 1024 }, { 7, 221, 1024 }, { 7, 222, 1024 }, { 8, 223, 1024 }, + { 4, 224, 1024 }, { 5, 225, 1024 }, { 5, 226, 1024 }, { 6, 227, 1024 }, { 5, 228, 1024 }, { 6, 229, 1024 }, { 6, 230, 1024 }, { 7, 231, 1024 }, + { 5, 232, 1024 }, { 6, 233, 1024 }, { 6, 234, 1024 }, { 7, 235, 1024 }, { 6, 236, 1024 }, { 7, 237, 1024 }, { 7, 238, 1024 }, { 8, 239, 1024 }, + { 5, 240, 1024 }, { 6, 241, 1024 }, { 6, 242, 1024 }, { 7, 243, 1024 }, { 6, 244, 1024 }, { 7, 245, 1024 }, { 7, 246, 1024 }, { 8, 247, 1024 }, + { 6, 248, 1024 }, { 7, 249, 1024 }, { 7, 250, 1024 }, { 8, 251, 1024 }, { 7, 252, 1024 }, { 8, 253, 1024 }, { 8, 254, 1024 }, { 9, 255, 1024 }, + { 2, 256, 1024 }, { 3, 257, 1024 }, { 3, 258, 1024 }, { 4, 259, 1024 }, { 3, 260, 1024 }, { 4, 261, 1024 }, { 4, 262, 1024 }, { 5, 263, 1024 }, + { 3, 264, 1024 }, { 4, 265, 1024 }, { 4, 266, 1024 }, { 5, 267, 1024 }, { 4, 268, 1024 }, { 5, 269, 1024 }, { 5, 270, 1024 }, { 6, 271, 1024 }, + { 3, 272, 1024 }, { 4, 273, 1024 }, { 4, 274, 1024 }, { 5, 275, 1024 }, { 4, 276, 1024 }, { 5, 277, 1024 }, { 5, 278, 1024 }, { 6, 279, 1024 }, + { 4, 280, 1024 }, { 5, 281, 1024 }, { 5, 282, 1024 }, { 6, 283, 1024 }, { 5, 284, 1024 }, { 6, 285, 1024 }, { 6, 286, 1024 }, { 7, 287, 1024 }, + { 3, 288, 1024 }, { 4, 289, 1024 }, { 4, 290, 1024 }, { 5, 291, 1024 }, { 4, 292, 1024 }, { 5, 293, 1024 }, { 5, 294, 1024 }, { 6, 295, 1024 }, + { 4, 296, 1024 }, { 5, 297, 1024 }, { 5, 298, 1024 }, { 6, 299, 1024 }, { 5, 300, 1024 }, { 6, 301, 1024 }, { 6, 302, 1024 }, { 7, 303, 1024 }, + { 4, 304, 1024 }, { 5, 305, 1024 }, { 5, 306, 1024 }, { 6, 307, 1024 }, { 5, 308, 1024 }, { 6, 309, 1024 }, { 6, 310, 1024 }, { 7, 311, 1024 }, + { 5, 312, 1024 }, { 6, 313, 1024 }, { 6, 314, 1024 }, { 7, 315, 1024 }, { 6, 316, 1024 }, { 7, 317, 1024 }, { 7, 318, 1024 }, { 8, 319, 1024 }, + { 3, 320, 1024 }, { 4, 321, 1024 }, { 4, 322, 1024 }, { 5, 323, 1024 }, { 4, 324, 1024 }, { 5, 325, 1024 }, { 5, 326, 1024 }, { 6, 327, 1024 }, + { 4, 328, 1024 }, { 5, 329, 1024 }, { 5, 330, 1024 }, { 6, 331, 1024 }, { 5, 332, 1024 }, { 6, 333, 1024 }, { 6, 334, 1024 }, { 7, 335, 1024 }, + { 4, 336, 1024 }, { 5, 337, 1024 }, { 5, 338, 1024 }, { 6, 339, 1024 }, { 5, 340, 1024 }, { 6, 341, 1024 }, { 6, 342, 1024 }, { 7, 343, 1024 }, + { 5, 344, 1024 }, { 6, 345, 1024 }, { 6, 346, 1024 }, { 7, 347, 1024 }, { 6, 348, 1024 }, { 7, 349, 1024 }, { 7, 350, 1024 }, { 8, 351, 1024 }, + { 4, 352, 1024 }, { 5, 353, 1024 }, { 5, 354, 1024 }, { 6, 355, 1024 }, { 5, 356, 1024 }, { 6, 357, 1024 }, { 6, 358, 1024 }, { 7, 359, 1024 }, + { 5, 360, 1024 }, { 6, 361, 1024 }, { 6, 362, 1024 }, { 7, 363, 1024 }, { 6, 364, 1024 }, { 7, 365, 1024 }, { 7, 366, 1024 }, { 8, 367, 1024 }, + { 5, 368, 1024 }, { 6, 369, 1024 }, { 6, 370, 1024 }, { 7, 371, 1024 }, { 6, 372, 1024 }, { 7, 373, 1024 }, { 7, 374, 1024 }, { 8, 375, 1024 }, + { 6, 376, 1024 }, { 7, 377, 1024 }, { 7, 378, 1024 }, { 8, 379, 1024 }, { 7, 380, 1024 }, { 8, 381, 1024 }, { 8, 382, 1024 }, { 9, 383, 1024 }, + { 3, 384, 1024 }, { 4, 385, 1024 }, { 4, 386, 1024 }, { 5, 387, 1024 }, { 4, 388, 1024 }, { 5, 389, 1024 }, { 5, 390, 1024 }, { 6, 391, 1024 }, + { 4, 392, 1024 }, { 5, 393, 1024 }, { 5, 394, 1024 }, { 6, 395, 1024 }, { 5, 396, 1024 }, { 6, 397, 1024 }, { 6, 398, 1024 }, { 7, 399, 1024 }, + { 4, 400, 1024 }, { 5, 401, 1024 }, { 5, 402, 1024 }, { 6, 403, 1024 }, { 5, 404, 1024 }, { 6, 405, 1024 }, { 6, 406, 1024 }, { 7, 407, 1024 }, + { 5, 408, 1024 }, { 6, 409, 1024 }, { 6, 410, 1024 }, { 7, 411, 1024 }, { 6, 412, 1024 }, { 7, 413, 1024 }, { 7, 414, 1024 }, { 8, 415, 1024 }, + { 4, 416, 1024 }, { 5, 417, 1024 }, { 5, 418, 1024 }, { 6, 419, 1024 }, { 5, 420, 1024 }, { 6, 421, 1024 }, { 6, 422, 1024 }, { 7, 423, 1024 }, + { 5, 424, 1024 }, { 6, 425, 1024 }, { 6, 426, 1024 }, { 7, 427, 1024 }, { 6, 428, 1024 }, { 7, 429, 1024 }, { 7, 430, 1024 }, { 8, 431, 1024 }, + { 5, 432, 1024 }, { 6, 433, 1024 }, { 6, 434, 1024 }, { 7, 435, 1024 }, { 6, 436, 1024 }, { 7, 437, 1024 }, { 7, 438, 1024 }, { 8, 439, 1024 }, + { 6, 440, 1024 }, { 7, 441, 1024 }, { 7, 442, 1024 }, { 8, 443, 1024 }, { 7, 444, 1024 }, { 8, 445, 1024 }, { 8, 446, 1024 }, { 9, 447, 1024 }, + { 4, 448, 1024 }, { 5, 449, 1024 }, { 5, 450, 1024 }, { 6, 451, 1024 }, { 5, 452, 1024 }, { 6, 453, 1024 }, { 6, 454, 1024 }, { 7, 455, 1024 }, + { 5, 456, 1024 }, { 6, 457, 1024 }, { 6, 458, 1024 }, { 7, 459, 1024 }, { 6, 460, 1024 }, { 7, 461, 1024 }, { 7, 462, 1024 }, { 8, 463, 1024 }, + { 5, 464, 1024 }, { 6, 465, 1024 }, { 6, 466, 1024 }, { 7, 467, 1024 }, { 6, 468, 1024 }, { 7, 469, 1024 }, { 7, 470, 1024 }, { 8, 471, 1024 }, + { 6, 472, 1024 }, { 7, 473, 1024 }, { 7, 474, 1024 }, { 8, 475, 1024 }, { 7, 476, 1024 }, { 8, 477, 1024 }, { 8, 478, 1024 }, { 9, 479, 1024 }, + { 5, 480, 1024 }, { 6, 481, 1024 }, { 6, 482, 1024 }, { 7, 483, 1024 }, { 6, 484, 1024 }, { 7, 485, 1024 }, { 7, 486, 1024 }, { 8, 487, 1024 }, + { 6, 488, 1024 }, { 7, 489, 1024 }, { 7, 490, 1024 }, { 8, 491, 1024 }, { 7, 492, 1024 }, { 8, 493, 1024 }, { 8, 494, 1024 }, { 9, 495, 1024 }, + { 6, 496, 1024 }, { 7, 497, 1024 }, { 7, 498, 1024 }, { 8, 499, 1024 }, { 7, 500, 1024 }, { 8, 501, 1024 }, { 8, 502, 1024 }, { 9, 503, 1024 }, + { 7, 504, 1024 }, { 8, 505, 1024 }, { 8, 506, 1024 }, { 9, 507, 1024 }, { 8, 508, 1024 }, { 9, 509, 1024 }, { 9, 510, 1024 }, { 10, 511, 1024 }, + { 2, 512, 1024 }, { 3, 513, 1024 }, { 3, 514, 1024 }, { 4, 515, 1024 }, { 3, 516, 1024 }, { 4, 517, 1024 }, { 4, 518, 1024 }, { 5, 519, 1024 }, + { 3, 520, 1024 }, { 4, 521, 1024 }, { 4, 522, 1024 }, { 5, 523, 1024 }, { 4, 524, 1024 }, { 5, 525, 1024 }, { 5, 526, 1024 }, { 6, 527, 1024 }, + { 3, 528, 1024 }, { 4, 529, 1024 }, { 4, 530, 1024 }, { 5, 531, 1024 }, { 4, 532, 1024 }, { 5, 533, 1024 }, { 5, 534, 1024 }, { 6, 535, 1024 }, + { 4, 536, 1024 }, { 5, 537, 1024 }, { 5, 538, 1024 }, { 6, 539, 1024 }, { 5, 540, 1024 }, { 6, 541, 1024 }, { 6, 542, 1024 }, { 7, 543, 1024 }, + { 3, 544, 1024 }, { 4, 545, 1024 }, { 4, 546, 1024 }, { 5, 547, 1024 }, { 4, 548, 1024 }, { 5, 549, 1024 }, { 5, 550, 1024 }, { 6, 551, 1024 }, + { 4, 552, 1024 }, { 5, 553, 1024 }, { 5, 554, 1024 }, { 6, 555, 1024 }, { 5, 556, 1024 }, { 6, 557, 1024 }, { 6, 558, 1024 }, { 7, 559, 1024 }, + { 4, 560, 1024 }, { 5, 561, 1024 }, { 5, 562, 1024 }, { 6, 563, 1024 }, { 5, 564, 1024 }, { 6, 565, 1024 }, { 6, 566, 1024 }, { 7, 567, 1024 }, + { 5, 568, 1024 }, { 6, 569, 1024 }, { 6, 570, 1024 }, { 7, 571, 1024 }, { 6, 572, 1024 }, { 7, 573, 1024 }, { 7, 574, 1024 }, { 8, 575, 1024 }, + { 3, 576, 1024 }, { 4, 577, 1024 }, { 4, 578, 1024 }, { 5, 579, 1024 }, { 4, 580, 1024 }, { 5, 581, 1024 }, { 5, 582, 1024 }, { 6, 583, 1024 }, + { 4, 584, 1024 }, { 5, 585, 1024 }, { 5, 586, 1024 }, { 6, 587, 1024 }, { 5, 588, 1024 }, { 6, 589, 1024 }, { 6, 590, 1024 }, { 7, 591, 1024 }, + { 4, 592, 1024 }, { 5, 593, 1024 }, { 5, 594, 1024 }, { 6, 595, 1024 }, { 5, 596, 1024 }, { 6, 597, 1024 }, { 6, 598, 1024 }, { 7, 599, 1024 }, + { 5, 600, 1024 }, { 6, 601, 1024 }, { 6, 602, 1024 }, { 7, 603, 1024 }, { 6, 604, 1024 }, { 7, 605, 1024 }, { 7, 606, 1024 }, { 8, 607, 1024 }, + { 4, 608, 1024 }, { 5, 609, 1024 }, { 5, 610, 1024 }, { 6, 611, 1024 }, { 5, 612, 1024 }, { 6, 613, 1024 }, { 6, 614, 1024 }, { 7, 615, 1024 }, + { 5, 616, 1024 }, { 6, 617, 1024 }, { 6, 618, 1024 }, { 7, 619, 1024 }, { 6, 620, 1024 }, { 7, 621, 1024 }, { 7, 622, 1024 }, { 8, 623, 1024 }, + { 5, 624, 1024 }, { 6, 625, 1024 }, { 6, 626, 1024 }, { 7, 627, 1024 }, { 6, 628, 1024 }, { 7, 629, 1024 }, { 7, 630, 1024 }, { 8, 631, 1024 }, + { 6, 632, 1024 }, { 7, 633, 1024 }, { 7, 634, 1024 }, { 8, 635, 1024 }, { 7, 636, 1024 }, { 8, 637, 1024 }, { 8, 638, 1024 }, { 9, 639, 1024 }, + { 3, 640, 1024 }, { 4, 641, 1024 }, { 4, 642, 1024 }, { 5, 643, 1024 }, { 4, 644, 1024 }, { 5, 645, 1024 }, { 5, 646, 1024 }, { 6, 647, 1024 }, + { 4, 648, 1024 }, { 5, 649, 1024 }, { 5, 650, 1024 }, { 6, 651, 1024 }, { 5, 652, 1024 }, { 6, 653, 1024 }, { 6, 654, 1024 }, { 7, 655, 1024 }, + { 4, 656, 1024 }, { 5, 657, 1024 }, { 5, 658, 1024 }, { 6, 659, 1024 }, { 5, 660, 1024 }, { 6, 661, 1024 }, { 6, 662, 1024 }, { 7, 663, 1024 }, + { 5, 664, 1024 }, { 6, 665, 1024 }, { 6, 666, 1024 }, { 7, 667, 1024 }, { 6, 668, 1024 }, { 7, 669, 1024 }, { 7, 670, 1024 }, { 8, 671, 1024 }, + { 4, 672, 1024 }, { 5, 673, 1024 }, { 5, 674, 1024 }, { 6, 675, 1024 }, { 5, 676, 1024 }, { 6, 677, 1024 }, { 6, 678, 1024 }, { 7, 679, 1024 }, + { 5, 680, 1024 }, { 6, 681, 1024 }, { 6, 682, 1024 }, { 7, 683, 1024 }, { 6, 684, 1024 }, { 7, 685, 1024 }, { 7, 686, 1024 }, { 8, 687, 1024 }, + { 5, 688, 1024 }, { 6, 689, 1024 }, { 6, 690, 1024 }, { 7, 691, 1024 }, { 6, 692, 1024 }, { 7, 693, 1024 }, { 7, 694, 1024 }, { 8, 695, 1024 }, + { 6, 696, 1024 }, { 7, 697, 1024 }, { 7, 698, 1024 }, { 8, 699, 1024 }, { 7, 700, 1024 }, { 8, 701, 1024 }, { 8, 702, 1024 }, { 9, 703, 1024 }, + { 4, 704, 1024 }, { 5, 705, 1024 }, { 5, 706, 1024 }, { 6, 707, 1024 }, { 5, 708, 1024 }, { 6, 709, 1024 }, { 6, 710, 1024 }, { 7, 711, 1024 }, + { 5, 712, 1024 }, { 6, 713, 1024 }, { 6, 714, 1024 }, { 7, 715, 1024 }, { 6, 716, 1024 }, { 7, 717, 1024 }, { 7, 718, 1024 }, { 8, 719, 1024 }, + { 5, 720, 1024 }, { 6, 721, 1024 }, { 6, 722, 1024 }, { 7, 723, 1024 }, { 6, 724, 1024 }, { 7, 725, 1024 }, { 7, 726, 1024 }, { 8, 727, 1024 }, + { 6, 728, 1024 }, { 7, 729, 1024 }, { 7, 730, 1024 }, { 8, 731, 1024 }, { 7, 732, 1024 }, { 8, 733, 1024 }, { 8, 734, 1024 }, { 9, 735, 1024 }, + { 5, 736, 1024 }, { 6, 737, 1024 }, { 6, 738, 1024 }, { 7, 739, 1024 }, { 6, 740, 1024 }, { 7, 741, 1024 }, { 7, 742, 1024 }, { 8, 743, 1024 }, + { 6, 744, 1024 }, { 7, 745, 1024 }, { 7, 746, 1024 }, { 8, 747, 1024 }, { 7, 748, 1024 }, { 8, 749, 1024 }, { 8, 750, 1024 }, { 9, 751, 1024 }, + { 6, 752, 1024 }, { 7, 753, 1024 }, { 7, 754, 1024 }, { 8, 755, 1024 }, { 7, 756, 1024 }, { 8, 757, 1024 }, { 8, 758, 1024 }, { 9, 759, 1024 }, + { 7, 760, 1024 }, { 8, 761, 1024 }, { 8, 762, 1024 }, { 9, 763, 1024 }, { 8, 764, 1024 }, { 9, 765, 1024 }, { 9, 766, 1024 }, { 10, 767, 1024 }, + { 3, 768, 1024 }, { 4, 769, 1024 }, { 4, 770, 1024 }, { 5, 771, 1024 }, { 4, 772, 1024 }, { 5, 773, 1024 }, { 5, 774, 1024 }, { 6, 775, 1024 }, + { 4, 776, 1024 }, { 5, 777, 1024 }, { 5, 778, 1024 }, { 6, 779, 1024 }, { 5, 780, 1024 }, { 6, 781, 1024 }, { 6, 782, 1024 }, { 7, 783, 1024 }, + { 4, 784, 1024 }, { 5, 785, 1024 }, { 5, 786, 1024 }, { 6, 787, 1024 }, { 5, 788, 1024 }, { 6, 789, 1024 }, { 6, 790, 1024 }, { 7, 791, 1024 }, + { 5, 792, 1024 }, { 6, 793, 1024 }, { 6, 794, 1024 }, { 7, 795, 1024 }, { 6, 796, 1024 }, { 7, 797, 1024 }, { 7, 798, 1024 }, { 8, 799, 1024 }, + { 4, 800, 1024 }, { 5, 801, 1024 }, { 5, 802, 1024 }, { 6, 803, 1024 }, { 5, 804, 1024 }, { 6, 805, 1024 }, { 6, 806, 1024 }, { 7, 807, 1024 }, + { 5, 808, 1024 }, { 6, 809, 1024 }, { 6, 810, 1024 }, { 7, 811, 1024 }, { 6, 812, 1024 }, { 7, 813, 1024 }, { 7, 814, 1024 }, { 8, 815, 1024 }, + { 5, 816, 1024 }, { 6, 817, 1024 }, { 6, 818, 1024 }, { 7, 819, 1024 }, { 6, 820, 1024 }, { 7, 821, 1024 }, { 7, 822, 1024 }, { 8, 823, 1024 }, + { 6, 824, 1024 }, { 7, 825, 1024 }, { 7, 826, 1024 }, { 8, 827, 1024 }, { 7, 828, 1024 }, { 8, 829, 1024 }, { 8, 830, 1024 }, { 9, 831, 1024 }, + { 4, 832, 1024 }, { 5, 833, 1024 }, { 5, 834, 1024 }, { 6, 835, 1024 }, { 5, 836, 1024 }, { 6, 837, 1024 }, { 6, 838, 1024 }, { 7, 839, 1024 }, + { 5, 840, 1024 }, { 6, 841, 1024 }, { 6, 842, 1024 }, { 7, 843, 1024 }, { 6, 844, 1024 }, { 7, 845, 1024 }, { 7, 846, 1024 }, { 8, 847, 1024 }, + { 5, 848, 1024 }, { 6, 849, 1024 }, { 6, 850, 1024 }, { 7, 851, 1024 }, { 6, 852, 1024 }, { 7, 853, 1024 }, { 7, 854, 1024 }, { 8, 855, 1024 }, + { 6, 856, 1024 }, { 7, 857, 1024 }, { 7, 858, 1024 }, { 8, 859, 1024 }, { 7, 860, 1024 }, { 8, 861, 1024 }, { 8, 862, 1024 }, { 9, 863, 1024 }, + { 5, 864, 1024 }, { 6, 865, 1024 }, { 6, 866, 1024 }, { 7, 867, 1024 }, { 6, 868, 1024 }, { 7, 869, 1024 }, { 7, 870, 1024 }, { 8, 871, 1024 }, + { 6, 872, 1024 }, { 7, 873, 1024 }, { 7, 874, 1024 }, { 8, 875, 1024 }, { 7, 876, 1024 }, { 8, 877, 1024 }, { 8, 878, 1024 }, { 9, 879, 1024 }, + { 6, 880, 1024 }, { 7, 881, 1024 }, { 7, 882, 1024 }, { 8, 883, 1024 }, { 7, 884, 1024 }, { 8, 885, 1024 }, { 8, 886, 1024 }, { 9, 887, 1024 }, + { 7, 888, 1024 }, { 8, 889, 1024 }, { 8, 890, 1024 }, { 9, 891, 1024 }, { 8, 892, 1024 }, { 9, 893, 1024 }, { 9, 894, 1024 }, { 10, 895, 1024 }, + { 4, 896, 1024 }, { 5, 897, 1024 }, { 5, 898, 1024 }, { 6, 899, 1024 }, { 5, 900, 1024 }, { 6, 901, 1024 }, { 6, 902, 1024 }, { 7, 903, 1024 }, + { 5, 904, 1024 }, { 6, 905, 1024 }, { 6, 906, 1024 }, { 7, 907, 1024 }, { 6, 908, 1024 }, { 7, 909, 1024 }, { 7, 910, 1024 }, { 8, 911, 1024 }, + { 5, 912, 1024 }, { 6, 913, 1024 }, { 6, 914, 1024 }, { 7, 915, 1024 }, { 6, 916, 1024 }, { 7, 917, 1024 }, { 7, 918, 1024 }, { 8, 919, 1024 }, + { 6, 920, 1024 }, { 7, 921, 1024 }, { 7, 922, 1024 }, { 8, 923, 1024 }, { 7, 924, 1024 }, { 8, 925, 1024 }, { 8, 926, 1024 }, { 9, 927, 1024 }, + { 5, 928, 1024 }, { 6, 929, 1024 }, { 6, 930, 1024 }, { 7, 931, 1024 }, { 6, 932, 1024 }, { 7, 933, 1024 }, { 7, 934, 1024 }, { 8, 935, 1024 }, + { 6, 936, 1024 }, { 7, 937, 1024 }, { 7, 938, 1024 }, { 8, 939, 1024 }, { 7, 940, 1024 }, { 8, 941, 1024 }, { 8, 942, 1024 }, { 9, 943, 1024 }, + { 6, 944, 1024 }, { 7, 945, 1024 }, { 7, 946, 1024 }, { 8, 947, 1024 }, { 7, 948, 1024 }, { 8, 949, 1024 }, { 8, 950, 1024 }, { 9, 951, 1024 }, + { 7, 952, 1024 }, { 8, 953, 1024 }, { 8, 954, 1024 }, { 9, 955, 1024 }, { 8, 956, 1024 }, { 9, 957, 1024 }, { 9, 958, 1024 }, { 10, 959, 1024 }, + { 5, 960, 1024 }, { 6, 961, 1024 }, { 6, 962, 1024 }, { 7, 963, 1024 }, { 6, 964, 1024 }, { 7, 965, 1024 }, { 7, 966, 1024 }, { 8, 967, 1024 }, + { 6, 968, 1024 }, { 7, 969, 1024 }, { 7, 970, 1024 }, { 8, 971, 1024 }, { 7, 972, 1024 }, { 8, 973, 1024 }, { 8, 974, 1024 }, { 9, 975, 1024 }, + { 6, 976, 1024 }, { 7, 977, 1024 }, { 7, 978, 1024 }, { 8, 979, 1024 }, { 7, 980, 1024 }, { 8, 981, 1024 }, { 8, 982, 1024 }, { 9, 983, 1024 }, + { 7, 984, 1024 }, { 8, 985, 1024 }, { 8, 986, 1024 }, { 9, 987, 1024 }, { 8, 988, 1024 }, { 9, 989, 1024 }, { 9, 990, 1024 }, { 10, 991, 1024 }, + { 6, 992, 1024 }, { 7, 993, 1024 }, { 7, 994, 1024 }, { 8, 995, 1024 }, { 7, 996, 1024 }, { 8, 997, 1024 }, { 8, 998, 1024 }, { 9, 999, 1024 }, + { 7, 1000, 1024 }, { 8, 1001, 1024 }, { 8, 1002, 1024 }, { 9, 1003, 1024 }, { 8, 1004, 1024 }, { 9, 1005, 1024 }, { 9, 1006, 1024 }, { 10, 1007, 1024 }, + { 7, 1008, 1024 }, { 8, 1009, 1024 }, { 8, 1010, 1024 }, { 9, 1011, 1024 }, { 8, 1012, 1024 }, { 9, 1013, 1024 }, { 9, 1014, 1024 }, { 10, 1015, 1024 }, + { 8, 1016, 1024 }, { 9, 1017, 1024 }, { 9, 1018, 1024 }, { 10, 1019, 1024 }, { 9, 1020, 1024 }, { 10, 1021, 1024 }, { 10, 1022, 1024 }, { 11, 1023, 1024 }, +#if FP_LUT > 11 + { 1, 0, 0 }, { 2, 1, 2048 }, { 2, 2, 2048 }, { 3, 3, 2048 }, { 2, 4, 2048 }, { 3, 5, 2048 }, { 3, 6, 2048 }, { 4, 7, 2048 }, + { 2, 8, 2048 }, { 3, 9, 2048 }, { 3, 10, 2048 }, { 4, 11, 2048 }, { 3, 12, 2048 }, { 4, 13, 2048 }, { 4, 14, 2048 }, { 5, 15, 2048 }, + { 2, 16, 2048 }, { 3, 17, 2048 }, { 3, 18, 2048 }, { 4, 19, 2048 }, { 3, 20, 2048 }, { 4, 21, 2048 }, { 4, 22, 2048 }, { 5, 23, 2048 }, + { 3, 24, 2048 }, { 4, 25, 2048 }, { 4, 26, 2048 }, { 5, 27, 2048 }, { 4, 28, 2048 }, { 5, 29, 2048 }, { 5, 30, 2048 }, { 6, 31, 2048 }, + { 2, 32, 2048 }, { 3, 33, 2048 }, { 3, 34, 2048 }, { 4, 35, 2048 }, { 3, 36, 2048 }, { 4, 37, 2048 }, { 4, 38, 2048 }, { 5, 39, 2048 }, + { 3, 40, 2048 }, { 4, 41, 2048 }, { 4, 42, 2048 }, { 5, 43, 2048 }, { 4, 44, 2048 }, { 5, 45, 2048 }, { 5, 46, 2048 }, { 6, 47, 2048 }, + { 3, 48, 2048 }, { 4, 49, 2048 }, { 4, 50, 2048 }, { 5, 51, 2048 }, { 4, 52, 2048 }, { 5, 53, 2048 }, { 5, 54, 2048 }, { 6, 55, 2048 }, + { 4, 56, 2048 }, { 5, 57, 2048 }, { 5, 58, 2048 }, { 6, 59, 2048 }, { 5, 60, 2048 }, { 6, 61, 2048 }, { 6, 62, 2048 }, { 7, 63, 2048 }, + { 2, 64, 2048 }, { 3, 65, 2048 }, { 3, 66, 2048 }, { 4, 67, 2048 }, { 3, 68, 2048 }, { 4, 69, 2048 }, { 4, 70, 2048 }, { 5, 71, 2048 }, + { 3, 72, 2048 }, { 4, 73, 2048 }, { 4, 74, 2048 }, { 5, 75, 2048 }, { 4, 76, 2048 }, { 5, 77, 2048 }, { 5, 78, 2048 }, { 6, 79, 2048 }, + { 3, 80, 2048 }, { 4, 81, 2048 }, { 4, 82, 2048 }, { 5, 83, 2048 }, { 4, 84, 2048 }, { 5, 85, 2048 }, { 5, 86, 2048 }, { 6, 87, 2048 }, + { 4, 88, 2048 }, { 5, 89, 2048 }, { 5, 90, 2048 }, { 6, 91, 2048 }, { 5, 92, 2048 }, { 6, 93, 2048 }, { 6, 94, 2048 }, { 7, 95, 2048 }, + { 3, 96, 2048 }, { 4, 97, 2048 }, { 4, 98, 2048 }, { 5, 99, 2048 }, { 4, 100, 2048 }, { 5, 101, 2048 }, { 5, 102, 2048 }, { 6, 103, 2048 }, + { 4, 104, 2048 }, { 5, 105, 2048 }, { 5, 106, 2048 }, { 6, 107, 2048 }, { 5, 108, 2048 }, { 6, 109, 2048 }, { 6, 110, 2048 }, { 7, 111, 2048 }, + { 4, 112, 2048 }, { 5, 113, 2048 }, { 5, 114, 2048 }, { 6, 115, 2048 }, { 5, 116, 2048 }, { 6, 117, 2048 }, { 6, 118, 2048 }, { 7, 119, 2048 }, + { 5, 120, 2048 }, { 6, 121, 2048 }, { 6, 122, 2048 }, { 7, 123, 2048 }, { 6, 124, 2048 }, { 7, 125, 2048 }, { 7, 126, 2048 }, { 8, 127, 2048 }, + { 2, 128, 2048 }, { 3, 129, 2048 }, { 3, 130, 2048 }, { 4, 131, 2048 }, { 3, 132, 2048 }, { 4, 133, 2048 }, { 4, 134, 2048 }, { 5, 135, 2048 }, + { 3, 136, 2048 }, { 4, 137, 2048 }, { 4, 138, 2048 }, { 5, 139, 2048 }, { 4, 140, 2048 }, { 5, 141, 2048 }, { 5, 142, 2048 }, { 6, 143, 2048 }, + { 3, 144, 2048 }, { 4, 145, 2048 }, { 4, 146, 2048 }, { 5, 147, 2048 }, { 4, 148, 2048 }, { 5, 149, 2048 }, { 5, 150, 2048 }, { 6, 151, 2048 }, + { 4, 152, 2048 }, { 5, 153, 2048 }, { 5, 154, 2048 }, { 6, 155, 2048 }, { 5, 156, 2048 }, { 6, 157, 2048 }, { 6, 158, 2048 }, { 7, 159, 2048 }, + { 3, 160, 2048 }, { 4, 161, 2048 }, { 4, 162, 2048 }, { 5, 163, 2048 }, { 4, 164, 2048 }, { 5, 165, 2048 }, { 5, 166, 2048 }, { 6, 167, 2048 }, + { 4, 168, 2048 }, { 5, 169, 2048 }, { 5, 170, 2048 }, { 6, 171, 2048 }, { 5, 172, 2048 }, { 6, 173, 2048 }, { 6, 174, 2048 }, { 7, 175, 2048 }, + { 4, 176, 2048 }, { 5, 177, 2048 }, { 5, 178, 2048 }, { 6, 179, 2048 }, { 5, 180, 2048 }, { 6, 181, 2048 }, { 6, 182, 2048 }, { 7, 183, 2048 }, + { 5, 184, 2048 }, { 6, 185, 2048 }, { 6, 186, 2048 }, { 7, 187, 2048 }, { 6, 188, 2048 }, { 7, 189, 2048 }, { 7, 190, 2048 }, { 8, 191, 2048 }, + { 3, 192, 2048 }, { 4, 193, 2048 }, { 4, 194, 2048 }, { 5, 195, 2048 }, { 4, 196, 2048 }, { 5, 197, 2048 }, { 5, 198, 2048 }, { 6, 199, 2048 }, + { 4, 200, 2048 }, { 5, 201, 2048 }, { 5, 202, 2048 }, { 6, 203, 2048 }, { 5, 204, 2048 }, { 6, 205, 2048 }, { 6, 206, 2048 }, { 7, 207, 2048 }, + { 4, 208, 2048 }, { 5, 209, 2048 }, { 5, 210, 2048 }, { 6, 211, 2048 }, { 5, 212, 2048 }, { 6, 213, 2048 }, { 6, 214, 2048 }, { 7, 215, 2048 }, + { 5, 216, 2048 }, { 6, 217, 2048 }, { 6, 218, 2048 }, { 7, 219, 2048 }, { 6, 220, 2048 }, { 7, 221, 2048 }, { 7, 222, 2048 }, { 8, 223, 2048 }, + { 4, 224, 2048 }, { 5, 225, 2048 }, { 5, 226, 2048 }, { 6, 227, 2048 }, { 5, 228, 2048 }, { 6, 229, 2048 }, { 6, 230, 2048 }, { 7, 231, 2048 }, + { 5, 232, 2048 }, { 6, 233, 2048 }, { 6, 234, 2048 }, { 7, 235, 2048 }, { 6, 236, 2048 }, { 7, 237, 2048 }, { 7, 238, 2048 }, { 8, 239, 2048 }, + { 5, 240, 2048 }, { 6, 241, 2048 }, { 6, 242, 2048 }, { 7, 243, 2048 }, { 6, 244, 2048 }, { 7, 245, 2048 }, { 7, 246, 2048 }, { 8, 247, 2048 }, + { 6, 248, 2048 }, { 7, 249, 2048 }, { 7, 250, 2048 }, { 8, 251, 2048 }, { 7, 252, 2048 }, { 8, 253, 2048 }, { 8, 254, 2048 }, { 9, 255, 2048 }, + { 2, 256, 2048 }, { 3, 257, 2048 }, { 3, 258, 2048 }, { 4, 259, 2048 }, { 3, 260, 2048 }, { 4, 261, 2048 }, { 4, 262, 2048 }, { 5, 263, 2048 }, + { 3, 264, 2048 }, { 4, 265, 2048 }, { 4, 266, 2048 }, { 5, 267, 2048 }, { 4, 268, 2048 }, { 5, 269, 2048 }, { 5, 270, 2048 }, { 6, 271, 2048 }, + { 3, 272, 2048 }, { 4, 273, 2048 }, { 4, 274, 2048 }, { 5, 275, 2048 }, { 4, 276, 2048 }, { 5, 277, 2048 }, { 5, 278, 2048 }, { 6, 279, 2048 }, + { 4, 280, 2048 }, { 5, 281, 2048 }, { 5, 282, 2048 }, { 6, 283, 2048 }, { 5, 284, 2048 }, { 6, 285, 2048 }, { 6, 286, 2048 }, { 7, 287, 2048 }, + { 3, 288, 2048 }, { 4, 289, 2048 }, { 4, 290, 2048 }, { 5, 291, 2048 }, { 4, 292, 2048 }, { 5, 293, 2048 }, { 5, 294, 2048 }, { 6, 295, 2048 }, + { 4, 296, 2048 }, { 5, 297, 2048 }, { 5, 298, 2048 }, { 6, 299, 2048 }, { 5, 300, 2048 }, { 6, 301, 2048 }, { 6, 302, 2048 }, { 7, 303, 2048 }, + { 4, 304, 2048 }, { 5, 305, 2048 }, { 5, 306, 2048 }, { 6, 307, 2048 }, { 5, 308, 2048 }, { 6, 309, 2048 }, { 6, 310, 2048 }, { 7, 311, 2048 }, + { 5, 312, 2048 }, { 6, 313, 2048 }, { 6, 314, 2048 }, { 7, 315, 2048 }, { 6, 316, 2048 }, { 7, 317, 2048 }, { 7, 318, 2048 }, { 8, 319, 2048 }, + { 3, 320, 2048 }, { 4, 321, 2048 }, { 4, 322, 2048 }, { 5, 323, 2048 }, { 4, 324, 2048 }, { 5, 325, 2048 }, { 5, 326, 2048 }, { 6, 327, 2048 }, + { 4, 328, 2048 }, { 5, 329, 2048 }, { 5, 330, 2048 }, { 6, 331, 2048 }, { 5, 332, 2048 }, { 6, 333, 2048 }, { 6, 334, 2048 }, { 7, 335, 2048 }, + { 4, 336, 2048 }, { 5, 337, 2048 }, { 5, 338, 2048 }, { 6, 339, 2048 }, { 5, 340, 2048 }, { 6, 341, 2048 }, { 6, 342, 2048 }, { 7, 343, 2048 }, + { 5, 344, 2048 }, { 6, 345, 2048 }, { 6, 346, 2048 }, { 7, 347, 2048 }, { 6, 348, 2048 }, { 7, 349, 2048 }, { 7, 350, 2048 }, { 8, 351, 2048 }, + { 4, 352, 2048 }, { 5, 353, 2048 }, { 5, 354, 2048 }, { 6, 355, 2048 }, { 5, 356, 2048 }, { 6, 357, 2048 }, { 6, 358, 2048 }, { 7, 359, 2048 }, + { 5, 360, 2048 }, { 6, 361, 2048 }, { 6, 362, 2048 }, { 7, 363, 2048 }, { 6, 364, 2048 }, { 7, 365, 2048 }, { 7, 366, 2048 }, { 8, 367, 2048 }, + { 5, 368, 2048 }, { 6, 369, 2048 }, { 6, 370, 2048 }, { 7, 371, 2048 }, { 6, 372, 2048 }, { 7, 373, 2048 }, { 7, 374, 2048 }, { 8, 375, 2048 }, + { 6, 376, 2048 }, { 7, 377, 2048 }, { 7, 378, 2048 }, { 8, 379, 2048 }, { 7, 380, 2048 }, { 8, 381, 2048 }, { 8, 382, 2048 }, { 9, 383, 2048 }, + { 3, 384, 2048 }, { 4, 385, 2048 }, { 4, 386, 2048 }, { 5, 387, 2048 }, { 4, 388, 2048 }, { 5, 389, 2048 }, { 5, 390, 2048 }, { 6, 391, 2048 }, + { 4, 392, 2048 }, { 5, 393, 2048 }, { 5, 394, 2048 }, { 6, 395, 2048 }, { 5, 396, 2048 }, { 6, 397, 2048 }, { 6, 398, 2048 }, { 7, 399, 2048 }, + { 4, 400, 2048 }, { 5, 401, 2048 }, { 5, 402, 2048 }, { 6, 403, 2048 }, { 5, 404, 2048 }, { 6, 405, 2048 }, { 6, 406, 2048 }, { 7, 407, 2048 }, + { 5, 408, 2048 }, { 6, 409, 2048 }, { 6, 410, 2048 }, { 7, 411, 2048 }, { 6, 412, 2048 }, { 7, 413, 2048 }, { 7, 414, 2048 }, { 8, 415, 2048 }, + { 4, 416, 2048 }, { 5, 417, 2048 }, { 5, 418, 2048 }, { 6, 419, 2048 }, { 5, 420, 2048 }, { 6, 421, 2048 }, { 6, 422, 2048 }, { 7, 423, 2048 }, + { 5, 424, 2048 }, { 6, 425, 2048 }, { 6, 426, 2048 }, { 7, 427, 2048 }, { 6, 428, 2048 }, { 7, 429, 2048 }, { 7, 430, 2048 }, { 8, 431, 2048 }, + { 5, 432, 2048 }, { 6, 433, 2048 }, { 6, 434, 2048 }, { 7, 435, 2048 }, { 6, 436, 2048 }, { 7, 437, 2048 }, { 7, 438, 2048 }, { 8, 439, 2048 }, + { 6, 440, 2048 }, { 7, 441, 2048 }, { 7, 442, 2048 }, { 8, 443, 2048 }, { 7, 444, 2048 }, { 8, 445, 2048 }, { 8, 446, 2048 }, { 9, 447, 2048 }, + { 4, 448, 2048 }, { 5, 449, 2048 }, { 5, 450, 2048 }, { 6, 451, 2048 }, { 5, 452, 2048 }, { 6, 453, 2048 }, { 6, 454, 2048 }, { 7, 455, 2048 }, + { 5, 456, 2048 }, { 6, 457, 2048 }, { 6, 458, 2048 }, { 7, 459, 2048 }, { 6, 460, 2048 }, { 7, 461, 2048 }, { 7, 462, 2048 }, { 8, 463, 2048 }, + { 5, 464, 2048 }, { 6, 465, 2048 }, { 6, 466, 2048 }, { 7, 467, 2048 }, { 6, 468, 2048 }, { 7, 469, 2048 }, { 7, 470, 2048 }, { 8, 471, 2048 }, + { 6, 472, 2048 }, { 7, 473, 2048 }, { 7, 474, 2048 }, { 8, 475, 2048 }, { 7, 476, 2048 }, { 8, 477, 2048 }, { 8, 478, 2048 }, { 9, 479, 2048 }, + { 5, 480, 2048 }, { 6, 481, 2048 }, { 6, 482, 2048 }, { 7, 483, 2048 }, { 6, 484, 2048 }, { 7, 485, 2048 }, { 7, 486, 2048 }, { 8, 487, 2048 }, + { 6, 488, 2048 }, { 7, 489, 2048 }, { 7, 490, 2048 }, { 8, 491, 2048 }, { 7, 492, 2048 }, { 8, 493, 2048 }, { 8, 494, 2048 }, { 9, 495, 2048 }, + { 6, 496, 2048 }, { 7, 497, 2048 }, { 7, 498, 2048 }, { 8, 499, 2048 }, { 7, 500, 2048 }, { 8, 501, 2048 }, { 8, 502, 2048 }, { 9, 503, 2048 }, + { 7, 504, 2048 }, { 8, 505, 2048 }, { 8, 506, 2048 }, { 9, 507, 2048 }, { 8, 508, 2048 }, { 9, 509, 2048 }, { 9, 510, 2048 }, { 10, 511, 2048 }, + { 2, 512, 2048 }, { 3, 513, 2048 }, { 3, 514, 2048 }, { 4, 515, 2048 }, { 3, 516, 2048 }, { 4, 517, 2048 }, { 4, 518, 2048 }, { 5, 519, 2048 }, + { 3, 520, 2048 }, { 4, 521, 2048 }, { 4, 522, 2048 }, { 5, 523, 2048 }, { 4, 524, 2048 }, { 5, 525, 2048 }, { 5, 526, 2048 }, { 6, 527, 2048 }, + { 3, 528, 2048 }, { 4, 529, 2048 }, { 4, 530, 2048 }, { 5, 531, 2048 }, { 4, 532, 2048 }, { 5, 533, 2048 }, { 5, 534, 2048 }, { 6, 535, 2048 }, + { 4, 536, 2048 }, { 5, 537, 2048 }, { 5, 538, 2048 }, { 6, 539, 2048 }, { 5, 540, 2048 }, { 6, 541, 2048 }, { 6, 542, 2048 }, { 7, 543, 2048 }, + { 3, 544, 2048 }, { 4, 545, 2048 }, { 4, 546, 2048 }, { 5, 547, 2048 }, { 4, 548, 2048 }, { 5, 549, 2048 }, { 5, 550, 2048 }, { 6, 551, 2048 }, + { 4, 552, 2048 }, { 5, 553, 2048 }, { 5, 554, 2048 }, { 6, 555, 2048 }, { 5, 556, 2048 }, { 6, 557, 2048 }, { 6, 558, 2048 }, { 7, 559, 2048 }, + { 4, 560, 2048 }, { 5, 561, 2048 }, { 5, 562, 2048 }, { 6, 563, 2048 }, { 5, 564, 2048 }, { 6, 565, 2048 }, { 6, 566, 2048 }, { 7, 567, 2048 }, + { 5, 568, 2048 }, { 6, 569, 2048 }, { 6, 570, 2048 }, { 7, 571, 2048 }, { 6, 572, 2048 }, { 7, 573, 2048 }, { 7, 574, 2048 }, { 8, 575, 2048 }, + { 3, 576, 2048 }, { 4, 577, 2048 }, { 4, 578, 2048 }, { 5, 579, 2048 }, { 4, 580, 2048 }, { 5, 581, 2048 }, { 5, 582, 2048 }, { 6, 583, 2048 }, + { 4, 584, 2048 }, { 5, 585, 2048 }, { 5, 586, 2048 }, { 6, 587, 2048 }, { 5, 588, 2048 }, { 6, 589, 2048 }, { 6, 590, 2048 }, { 7, 591, 2048 }, + { 4, 592, 2048 }, { 5, 593, 2048 }, { 5, 594, 2048 }, { 6, 595, 2048 }, { 5, 596, 2048 }, { 6, 597, 2048 }, { 6, 598, 2048 }, { 7, 599, 2048 }, + { 5, 600, 2048 }, { 6, 601, 2048 }, { 6, 602, 2048 }, { 7, 603, 2048 }, { 6, 604, 2048 }, { 7, 605, 2048 }, { 7, 606, 2048 }, { 8, 607, 2048 }, + { 4, 608, 2048 }, { 5, 609, 2048 }, { 5, 610, 2048 }, { 6, 611, 2048 }, { 5, 612, 2048 }, { 6, 613, 2048 }, { 6, 614, 2048 }, { 7, 615, 2048 }, + { 5, 616, 2048 }, { 6, 617, 2048 }, { 6, 618, 2048 }, { 7, 619, 2048 }, { 6, 620, 2048 }, { 7, 621, 2048 }, { 7, 622, 2048 }, { 8, 623, 2048 }, + { 5, 624, 2048 }, { 6, 625, 2048 }, { 6, 626, 2048 }, { 7, 627, 2048 }, { 6, 628, 2048 }, { 7, 629, 2048 }, { 7, 630, 2048 }, { 8, 631, 2048 }, + { 6, 632, 2048 }, { 7, 633, 2048 }, { 7, 634, 2048 }, { 8, 635, 2048 }, { 7, 636, 2048 }, { 8, 637, 2048 }, { 8, 638, 2048 }, { 9, 639, 2048 }, + { 3, 640, 2048 }, { 4, 641, 2048 }, { 4, 642, 2048 }, { 5, 643, 2048 }, { 4, 644, 2048 }, { 5, 645, 2048 }, { 5, 646, 2048 }, { 6, 647, 2048 }, + { 4, 648, 2048 }, { 5, 649, 2048 }, { 5, 650, 2048 }, { 6, 651, 2048 }, { 5, 652, 2048 }, { 6, 653, 2048 }, { 6, 654, 2048 }, { 7, 655, 2048 }, + { 4, 656, 2048 }, { 5, 657, 2048 }, { 5, 658, 2048 }, { 6, 659, 2048 }, { 5, 660, 2048 }, { 6, 661, 2048 }, { 6, 662, 2048 }, { 7, 663, 2048 }, + { 5, 664, 2048 }, { 6, 665, 2048 }, { 6, 666, 2048 }, { 7, 667, 2048 }, { 6, 668, 2048 }, { 7, 669, 2048 }, { 7, 670, 2048 }, { 8, 671, 2048 }, + { 4, 672, 2048 }, { 5, 673, 2048 }, { 5, 674, 2048 }, { 6, 675, 2048 }, { 5, 676, 2048 }, { 6, 677, 2048 }, { 6, 678, 2048 }, { 7, 679, 2048 }, + { 5, 680, 2048 }, { 6, 681, 2048 }, { 6, 682, 2048 }, { 7, 683, 2048 }, { 6, 684, 2048 }, { 7, 685, 2048 }, { 7, 686, 2048 }, { 8, 687, 2048 }, + { 5, 688, 2048 }, { 6, 689, 2048 }, { 6, 690, 2048 }, { 7, 691, 2048 }, { 6, 692, 2048 }, { 7, 693, 2048 }, { 7, 694, 2048 }, { 8, 695, 2048 }, + { 6, 696, 2048 }, { 7, 697, 2048 }, { 7, 698, 2048 }, { 8, 699, 2048 }, { 7, 700, 2048 }, { 8, 701, 2048 }, { 8, 702, 2048 }, { 9, 703, 2048 }, + { 4, 704, 2048 }, { 5, 705, 2048 }, { 5, 706, 2048 }, { 6, 707, 2048 }, { 5, 708, 2048 }, { 6, 709, 2048 }, { 6, 710, 2048 }, { 7, 711, 2048 }, + { 5, 712, 2048 }, { 6, 713, 2048 }, { 6, 714, 2048 }, { 7, 715, 2048 }, { 6, 716, 2048 }, { 7, 717, 2048 }, { 7, 718, 2048 }, { 8, 719, 2048 }, + { 5, 720, 2048 }, { 6, 721, 2048 }, { 6, 722, 2048 }, { 7, 723, 2048 }, { 6, 724, 2048 }, { 7, 725, 2048 }, { 7, 726, 2048 }, { 8, 727, 2048 }, + { 6, 728, 2048 }, { 7, 729, 2048 }, { 7, 730, 2048 }, { 8, 731, 2048 }, { 7, 732, 2048 }, { 8, 733, 2048 }, { 8, 734, 2048 }, { 9, 735, 2048 }, + { 5, 736, 2048 }, { 6, 737, 2048 }, { 6, 738, 2048 }, { 7, 739, 2048 }, { 6, 740, 2048 }, { 7, 741, 2048 }, { 7, 742, 2048 }, { 8, 743, 2048 }, + { 6, 744, 2048 }, { 7, 745, 2048 }, { 7, 746, 2048 }, { 8, 747, 2048 }, { 7, 748, 2048 }, { 8, 749, 2048 }, { 8, 750, 2048 }, { 9, 751, 2048 }, + { 6, 752, 2048 }, { 7, 753, 2048 }, { 7, 754, 2048 }, { 8, 755, 2048 }, { 7, 756, 2048 }, { 8, 757, 2048 }, { 8, 758, 2048 }, { 9, 759, 2048 }, + { 7, 760, 2048 }, { 8, 761, 2048 }, { 8, 762, 2048 }, { 9, 763, 2048 }, { 8, 764, 2048 }, { 9, 765, 2048 }, { 9, 766, 2048 }, { 10, 767, 2048 }, + { 3, 768, 2048 }, { 4, 769, 2048 }, { 4, 770, 2048 }, { 5, 771, 2048 }, { 4, 772, 2048 }, { 5, 773, 2048 }, { 5, 774, 2048 }, { 6, 775, 2048 }, + { 4, 776, 2048 }, { 5, 777, 2048 }, { 5, 778, 2048 }, { 6, 779, 2048 }, { 5, 780, 2048 }, { 6, 781, 2048 }, { 6, 782, 2048 }, { 7, 783, 2048 }, + { 4, 784, 2048 }, { 5, 785, 2048 }, { 5, 786, 2048 }, { 6, 787, 2048 }, { 5, 788, 2048 }, { 6, 789, 2048 }, { 6, 790, 2048 }, { 7, 791, 2048 }, + { 5, 792, 2048 }, { 6, 793, 2048 }, { 6, 794, 2048 }, { 7, 795, 2048 }, { 6, 796, 2048 }, { 7, 797, 2048 }, { 7, 798, 2048 }, { 8, 799, 2048 }, + { 4, 800, 2048 }, { 5, 801, 2048 }, { 5, 802, 2048 }, { 6, 803, 2048 }, { 5, 804, 2048 }, { 6, 805, 2048 }, { 6, 806, 2048 }, { 7, 807, 2048 }, + { 5, 808, 2048 }, { 6, 809, 2048 }, { 6, 810, 2048 }, { 7, 811, 2048 }, { 6, 812, 2048 }, { 7, 813, 2048 }, { 7, 814, 2048 }, { 8, 815, 2048 }, + { 5, 816, 2048 }, { 6, 817, 2048 }, { 6, 818, 2048 }, { 7, 819, 2048 }, { 6, 820, 2048 }, { 7, 821, 2048 }, { 7, 822, 2048 }, { 8, 823, 2048 }, + { 6, 824, 2048 }, { 7, 825, 2048 }, { 7, 826, 2048 }, { 8, 827, 2048 }, { 7, 828, 2048 }, { 8, 829, 2048 }, { 8, 830, 2048 }, { 9, 831, 2048 }, + { 4, 832, 2048 }, { 5, 833, 2048 }, { 5, 834, 2048 }, { 6, 835, 2048 }, { 5, 836, 2048 }, { 6, 837, 2048 }, { 6, 838, 2048 }, { 7, 839, 2048 }, + { 5, 840, 2048 }, { 6, 841, 2048 }, { 6, 842, 2048 }, { 7, 843, 2048 }, { 6, 844, 2048 }, { 7, 845, 2048 }, { 7, 846, 2048 }, { 8, 847, 2048 }, + { 5, 848, 2048 }, { 6, 849, 2048 }, { 6, 850, 2048 }, { 7, 851, 2048 }, { 6, 852, 2048 }, { 7, 853, 2048 }, { 7, 854, 2048 }, { 8, 855, 2048 }, + { 6, 856, 2048 }, { 7, 857, 2048 }, { 7, 858, 2048 }, { 8, 859, 2048 }, { 7, 860, 2048 }, { 8, 861, 2048 }, { 8, 862, 2048 }, { 9, 863, 2048 }, + { 5, 864, 2048 }, { 6, 865, 2048 }, { 6, 866, 2048 }, { 7, 867, 2048 }, { 6, 868, 2048 }, { 7, 869, 2048 }, { 7, 870, 2048 }, { 8, 871, 2048 }, + { 6, 872, 2048 }, { 7, 873, 2048 }, { 7, 874, 2048 }, { 8, 875, 2048 }, { 7, 876, 2048 }, { 8, 877, 2048 }, { 8, 878, 2048 }, { 9, 879, 2048 }, + { 6, 880, 2048 }, { 7, 881, 2048 }, { 7, 882, 2048 }, { 8, 883, 2048 }, { 7, 884, 2048 }, { 8, 885, 2048 }, { 8, 886, 2048 }, { 9, 887, 2048 }, + { 7, 888, 2048 }, { 8, 889, 2048 }, { 8, 890, 2048 }, { 9, 891, 2048 }, { 8, 892, 2048 }, { 9, 893, 2048 }, { 9, 894, 2048 }, { 10, 895, 2048 }, + { 4, 896, 2048 }, { 5, 897, 2048 }, { 5, 898, 2048 }, { 6, 899, 2048 }, { 5, 900, 2048 }, { 6, 901, 2048 }, { 6, 902, 2048 }, { 7, 903, 2048 }, + { 5, 904, 2048 }, { 6, 905, 2048 }, { 6, 906, 2048 }, { 7, 907, 2048 }, { 6, 908, 2048 }, { 7, 909, 2048 }, { 7, 910, 2048 }, { 8, 911, 2048 }, + { 5, 912, 2048 }, { 6, 913, 2048 }, { 6, 914, 2048 }, { 7, 915, 2048 }, { 6, 916, 2048 }, { 7, 917, 2048 }, { 7, 918, 2048 }, { 8, 919, 2048 }, + { 6, 920, 2048 }, { 7, 921, 2048 }, { 7, 922, 2048 }, { 8, 923, 2048 }, { 7, 924, 2048 }, { 8, 925, 2048 }, { 8, 926, 2048 }, { 9, 927, 2048 }, + { 5, 928, 2048 }, { 6, 929, 2048 }, { 6, 930, 2048 }, { 7, 931, 2048 }, { 6, 932, 2048 }, { 7, 933, 2048 }, { 7, 934, 2048 }, { 8, 935, 2048 }, + { 6, 936, 2048 }, { 7, 937, 2048 }, { 7, 938, 2048 }, { 8, 939, 2048 }, { 7, 940, 2048 }, { 8, 941, 2048 }, { 8, 942, 2048 }, { 9, 943, 2048 }, + { 6, 944, 2048 }, { 7, 945, 2048 }, { 7, 946, 2048 }, { 8, 947, 2048 }, { 7, 948, 2048 }, { 8, 949, 2048 }, { 8, 950, 2048 }, { 9, 951, 2048 }, + { 7, 952, 2048 }, { 8, 953, 2048 }, { 8, 954, 2048 }, { 9, 955, 2048 }, { 8, 956, 2048 }, { 9, 957, 2048 }, { 9, 958, 2048 }, { 10, 959, 2048 }, + { 5, 960, 2048 }, { 6, 961, 2048 }, { 6, 962, 2048 }, { 7, 963, 2048 }, { 6, 964, 2048 }, { 7, 965, 2048 }, { 7, 966, 2048 }, { 8, 967, 2048 }, + { 6, 968, 2048 }, { 7, 969, 2048 }, { 7, 970, 2048 }, { 8, 971, 2048 }, { 7, 972, 2048 }, { 8, 973, 2048 }, { 8, 974, 2048 }, { 9, 975, 2048 }, + { 6, 976, 2048 }, { 7, 977, 2048 }, { 7, 978, 2048 }, { 8, 979, 2048 }, { 7, 980, 2048 }, { 8, 981, 2048 }, { 8, 982, 2048 }, { 9, 983, 2048 }, + { 7, 984, 2048 }, { 8, 985, 2048 }, { 8, 986, 2048 }, { 9, 987, 2048 }, { 8, 988, 2048 }, { 9, 989, 2048 }, { 9, 990, 2048 }, { 10, 991, 2048 }, + { 6, 992, 2048 }, { 7, 993, 2048 }, { 7, 994, 2048 }, { 8, 995, 2048 }, { 7, 996, 2048 }, { 8, 997, 2048 }, { 8, 998, 2048 }, { 9, 999, 2048 }, + { 7, 1000, 2048 }, { 8, 1001, 2048 }, { 8, 1002, 2048 }, { 9, 1003, 2048 }, { 8, 1004, 2048 }, { 9, 1005, 2048 }, { 9, 1006, 2048 }, { 10, 1007, 2048 }, + { 7, 1008, 2048 }, { 8, 1009, 2048 }, { 8, 1010, 2048 }, { 9, 1011, 2048 }, { 8, 1012, 2048 }, { 9, 1013, 2048 }, { 9, 1014, 2048 }, { 10, 1015, 2048 }, + { 8, 1016, 2048 }, { 9, 1017, 2048 }, { 9, 1018, 2048 }, { 10, 1019, 2048 }, { 9, 1020, 2048 }, { 10, 1021, 2048 }, { 10, 1022, 2048 }, { 11, 1023, 2048 }, + { 2, 1024, 2048 }, { 3, 1025, 2048 }, { 3, 1026, 2048 }, { 4, 1027, 2048 }, { 3, 1028, 2048 }, { 4, 1029, 2048 }, { 4, 1030, 2048 }, { 5, 1031, 2048 }, + { 3, 1032, 2048 }, { 4, 1033, 2048 }, { 4, 1034, 2048 }, { 5, 1035, 2048 }, { 4, 1036, 2048 }, { 5, 1037, 2048 }, { 5, 1038, 2048 }, { 6, 1039, 2048 }, + { 3, 1040, 2048 }, { 4, 1041, 2048 }, { 4, 1042, 2048 }, { 5, 1043, 2048 }, { 4, 1044, 2048 }, { 5, 1045, 2048 }, { 5, 1046, 2048 }, { 6, 1047, 2048 }, + { 4, 1048, 2048 }, { 5, 1049, 2048 }, { 5, 1050, 2048 }, { 6, 1051, 2048 }, { 5, 1052, 2048 }, { 6, 1053, 2048 }, { 6, 1054, 2048 }, { 7, 1055, 2048 }, + { 3, 1056, 2048 }, { 4, 1057, 2048 }, { 4, 1058, 2048 }, { 5, 1059, 2048 }, { 4, 1060, 2048 }, { 5, 1061, 2048 }, { 5, 1062, 2048 }, { 6, 1063, 2048 }, + { 4, 1064, 2048 }, { 5, 1065, 2048 }, { 5, 1066, 2048 }, { 6, 1067, 2048 }, { 5, 1068, 2048 }, { 6, 1069, 2048 }, { 6, 1070, 2048 }, { 7, 1071, 2048 }, + { 4, 1072, 2048 }, { 5, 1073, 2048 }, { 5, 1074, 2048 }, { 6, 1075, 2048 }, { 5, 1076, 2048 }, { 6, 1077, 2048 }, { 6, 1078, 2048 }, { 7, 1079, 2048 }, + { 5, 1080, 2048 }, { 6, 1081, 2048 }, { 6, 1082, 2048 }, { 7, 1083, 2048 }, { 6, 1084, 2048 }, { 7, 1085, 2048 }, { 7, 1086, 2048 }, { 8, 1087, 2048 }, + { 3, 1088, 2048 }, { 4, 1089, 2048 }, { 4, 1090, 2048 }, { 5, 1091, 2048 }, { 4, 1092, 2048 }, { 5, 1093, 2048 }, { 5, 1094, 2048 }, { 6, 1095, 2048 }, + { 4, 1096, 2048 }, { 5, 1097, 2048 }, { 5, 1098, 2048 }, { 6, 1099, 2048 }, { 5, 1100, 2048 }, { 6, 1101, 2048 }, { 6, 1102, 2048 }, { 7, 1103, 2048 }, + { 4, 1104, 2048 }, { 5, 1105, 2048 }, { 5, 1106, 2048 }, { 6, 1107, 2048 }, { 5, 1108, 2048 }, { 6, 1109, 2048 }, { 6, 1110, 2048 }, { 7, 1111, 2048 }, + { 5, 1112, 2048 }, { 6, 1113, 2048 }, { 6, 1114, 2048 }, { 7, 1115, 2048 }, { 6, 1116, 2048 }, { 7, 1117, 2048 }, { 7, 1118, 2048 }, { 8, 1119, 2048 }, + { 4, 1120, 2048 }, { 5, 1121, 2048 }, { 5, 1122, 2048 }, { 6, 1123, 2048 }, { 5, 1124, 2048 }, { 6, 1125, 2048 }, { 6, 1126, 2048 }, { 7, 1127, 2048 }, + { 5, 1128, 2048 }, { 6, 1129, 2048 }, { 6, 1130, 2048 }, { 7, 1131, 2048 }, { 6, 1132, 2048 }, { 7, 1133, 2048 }, { 7, 1134, 2048 }, { 8, 1135, 2048 }, + { 5, 1136, 2048 }, { 6, 1137, 2048 }, { 6, 1138, 2048 }, { 7, 1139, 2048 }, { 6, 1140, 2048 }, { 7, 1141, 2048 }, { 7, 1142, 2048 }, { 8, 1143, 2048 }, + { 6, 1144, 2048 }, { 7, 1145, 2048 }, { 7, 1146, 2048 }, { 8, 1147, 2048 }, { 7, 1148, 2048 }, { 8, 1149, 2048 }, { 8, 1150, 2048 }, { 9, 1151, 2048 }, + { 3, 1152, 2048 }, { 4, 1153, 2048 }, { 4, 1154, 2048 }, { 5, 1155, 2048 }, { 4, 1156, 2048 }, { 5, 1157, 2048 }, { 5, 1158, 2048 }, { 6, 1159, 2048 }, + { 4, 1160, 2048 }, { 5, 1161, 2048 }, { 5, 1162, 2048 }, { 6, 1163, 2048 }, { 5, 1164, 2048 }, { 6, 1165, 2048 }, { 6, 1166, 2048 }, { 7, 1167, 2048 }, + { 4, 1168, 2048 }, { 5, 1169, 2048 }, { 5, 1170, 2048 }, { 6, 1171, 2048 }, { 5, 1172, 2048 }, { 6, 1173, 2048 }, { 6, 1174, 2048 }, { 7, 1175, 2048 }, + { 5, 1176, 2048 }, { 6, 1177, 2048 }, { 6, 1178, 2048 }, { 7, 1179, 2048 }, { 6, 1180, 2048 }, { 7, 1181, 2048 }, { 7, 1182, 2048 }, { 8, 1183, 2048 }, + { 4, 1184, 2048 }, { 5, 1185, 2048 }, { 5, 1186, 2048 }, { 6, 1187, 2048 }, { 5, 1188, 2048 }, { 6, 1189, 2048 }, { 6, 1190, 2048 }, { 7, 1191, 2048 }, + { 5, 1192, 2048 }, { 6, 1193, 2048 }, { 6, 1194, 2048 }, { 7, 1195, 2048 }, { 6, 1196, 2048 }, { 7, 1197, 2048 }, { 7, 1198, 2048 }, { 8, 1199, 2048 }, + { 5, 1200, 2048 }, { 6, 1201, 2048 }, { 6, 1202, 2048 }, { 7, 1203, 2048 }, { 6, 1204, 2048 }, { 7, 1205, 2048 }, { 7, 1206, 2048 }, { 8, 1207, 2048 }, + { 6, 1208, 2048 }, { 7, 1209, 2048 }, { 7, 1210, 2048 }, { 8, 1211, 2048 }, { 7, 1212, 2048 }, { 8, 1213, 2048 }, { 8, 1214, 2048 }, { 9, 1215, 2048 }, + { 4, 1216, 2048 }, { 5, 1217, 2048 }, { 5, 1218, 2048 }, { 6, 1219, 2048 }, { 5, 1220, 2048 }, { 6, 1221, 2048 }, { 6, 1222, 2048 }, { 7, 1223, 2048 }, + { 5, 1224, 2048 }, { 6, 1225, 2048 }, { 6, 1226, 2048 }, { 7, 1227, 2048 }, { 6, 1228, 2048 }, { 7, 1229, 2048 }, { 7, 1230, 2048 }, { 8, 1231, 2048 }, + { 5, 1232, 2048 }, { 6, 1233, 2048 }, { 6, 1234, 2048 }, { 7, 1235, 2048 }, { 6, 1236, 2048 }, { 7, 1237, 2048 }, { 7, 1238, 2048 }, { 8, 1239, 2048 }, + { 6, 1240, 2048 }, { 7, 1241, 2048 }, { 7, 1242, 2048 }, { 8, 1243, 2048 }, { 7, 1244, 2048 }, { 8, 1245, 2048 }, { 8, 1246, 2048 }, { 9, 1247, 2048 }, + { 5, 1248, 2048 }, { 6, 1249, 2048 }, { 6, 1250, 2048 }, { 7, 1251, 2048 }, { 6, 1252, 2048 }, { 7, 1253, 2048 }, { 7, 1254, 2048 }, { 8, 1255, 2048 }, + { 6, 1256, 2048 }, { 7, 1257, 2048 }, { 7, 1258, 2048 }, { 8, 1259, 2048 }, { 7, 1260, 2048 }, { 8, 1261, 2048 }, { 8, 1262, 2048 }, { 9, 1263, 2048 }, + { 6, 1264, 2048 }, { 7, 1265, 2048 }, { 7, 1266, 2048 }, { 8, 1267, 2048 }, { 7, 1268, 2048 }, { 8, 1269, 2048 }, { 8, 1270, 2048 }, { 9, 1271, 2048 }, + { 7, 1272, 2048 }, { 8, 1273, 2048 }, { 8, 1274, 2048 }, { 9, 1275, 2048 }, { 8, 1276, 2048 }, { 9, 1277, 2048 }, { 9, 1278, 2048 }, { 10, 1279, 2048 }, + { 3, 1280, 2048 }, { 4, 1281, 2048 }, { 4, 1282, 2048 }, { 5, 1283, 2048 }, { 4, 1284, 2048 }, { 5, 1285, 2048 }, { 5, 1286, 2048 }, { 6, 1287, 2048 }, + { 4, 1288, 2048 }, { 5, 1289, 2048 }, { 5, 1290, 2048 }, { 6, 1291, 2048 }, { 5, 1292, 2048 }, { 6, 1293, 2048 }, { 6, 1294, 2048 }, { 7, 1295, 2048 }, + { 4, 1296, 2048 }, { 5, 1297, 2048 }, { 5, 1298, 2048 }, { 6, 1299, 2048 }, { 5, 1300, 2048 }, { 6, 1301, 2048 }, { 6, 1302, 2048 }, { 7, 1303, 2048 }, + { 5, 1304, 2048 }, { 6, 1305, 2048 }, { 6, 1306, 2048 }, { 7, 1307, 2048 }, { 6, 1308, 2048 }, { 7, 1309, 2048 }, { 7, 1310, 2048 }, { 8, 1311, 2048 }, + { 4, 1312, 2048 }, { 5, 1313, 2048 }, { 5, 1314, 2048 }, { 6, 1315, 2048 }, { 5, 1316, 2048 }, { 6, 1317, 2048 }, { 6, 1318, 2048 }, { 7, 1319, 2048 }, + { 5, 1320, 2048 }, { 6, 1321, 2048 }, { 6, 1322, 2048 }, { 7, 1323, 2048 }, { 6, 1324, 2048 }, { 7, 1325, 2048 }, { 7, 1326, 2048 }, { 8, 1327, 2048 }, + { 5, 1328, 2048 }, { 6, 1329, 2048 }, { 6, 1330, 2048 }, { 7, 1331, 2048 }, { 6, 1332, 2048 }, { 7, 1333, 2048 }, { 7, 1334, 2048 }, { 8, 1335, 2048 }, + { 6, 1336, 2048 }, { 7, 1337, 2048 }, { 7, 1338, 2048 }, { 8, 1339, 2048 }, { 7, 1340, 2048 }, { 8, 1341, 2048 }, { 8, 1342, 2048 }, { 9, 1343, 2048 }, + { 4, 1344, 2048 }, { 5, 1345, 2048 }, { 5, 1346, 2048 }, { 6, 1347, 2048 }, { 5, 1348, 2048 }, { 6, 1349, 2048 }, { 6, 1350, 2048 }, { 7, 1351, 2048 }, + { 5, 1352, 2048 }, { 6, 1353, 2048 }, { 6, 1354, 2048 }, { 7, 1355, 2048 }, { 6, 1356, 2048 }, { 7, 1357, 2048 }, { 7, 1358, 2048 }, { 8, 1359, 2048 }, + { 5, 1360, 2048 }, { 6, 1361, 2048 }, { 6, 1362, 2048 }, { 7, 1363, 2048 }, { 6, 1364, 2048 }, { 7, 1365, 2048 }, { 7, 1366, 2048 }, { 8, 1367, 2048 }, + { 6, 1368, 2048 }, { 7, 1369, 2048 }, { 7, 1370, 2048 }, { 8, 1371, 2048 }, { 7, 1372, 2048 }, { 8, 1373, 2048 }, { 8, 1374, 2048 }, { 9, 1375, 2048 }, + { 5, 1376, 2048 }, { 6, 1377, 2048 }, { 6, 1378, 2048 }, { 7, 1379, 2048 }, { 6, 1380, 2048 }, { 7, 1381, 2048 }, { 7, 1382, 2048 }, { 8, 1383, 2048 }, + { 6, 1384, 2048 }, { 7, 1385, 2048 }, { 7, 1386, 2048 }, { 8, 1387, 2048 }, { 7, 1388, 2048 }, { 8, 1389, 2048 }, { 8, 1390, 2048 }, { 9, 1391, 2048 }, + { 6, 1392, 2048 }, { 7, 1393, 2048 }, { 7, 1394, 2048 }, { 8, 1395, 2048 }, { 7, 1396, 2048 }, { 8, 1397, 2048 }, { 8, 1398, 2048 }, { 9, 1399, 2048 }, + { 7, 1400, 2048 }, { 8, 1401, 2048 }, { 8, 1402, 2048 }, { 9, 1403, 2048 }, { 8, 1404, 2048 }, { 9, 1405, 2048 }, { 9, 1406, 2048 }, { 10, 1407, 2048 }, + { 4, 1408, 2048 }, { 5, 1409, 2048 }, { 5, 1410, 2048 }, { 6, 1411, 2048 }, { 5, 1412, 2048 }, { 6, 1413, 2048 }, { 6, 1414, 2048 }, { 7, 1415, 2048 }, + { 5, 1416, 2048 }, { 6, 1417, 2048 }, { 6, 1418, 2048 }, { 7, 1419, 2048 }, { 6, 1420, 2048 }, { 7, 1421, 2048 }, { 7, 1422, 2048 }, { 8, 1423, 2048 }, + { 5, 1424, 2048 }, { 6, 1425, 2048 }, { 6, 1426, 2048 }, { 7, 1427, 2048 }, { 6, 1428, 2048 }, { 7, 1429, 2048 }, { 7, 1430, 2048 }, { 8, 1431, 2048 }, + { 6, 1432, 2048 }, { 7, 1433, 2048 }, { 7, 1434, 2048 }, { 8, 1435, 2048 }, { 7, 1436, 2048 }, { 8, 1437, 2048 }, { 8, 1438, 2048 }, { 9, 1439, 2048 }, + { 5, 1440, 2048 }, { 6, 1441, 2048 }, { 6, 1442, 2048 }, { 7, 1443, 2048 }, { 6, 1444, 2048 }, { 7, 1445, 2048 }, { 7, 1446, 2048 }, { 8, 1447, 2048 }, + { 6, 1448, 2048 }, { 7, 1449, 2048 }, { 7, 1450, 2048 }, { 8, 1451, 2048 }, { 7, 1452, 2048 }, { 8, 1453, 2048 }, { 8, 1454, 2048 }, { 9, 1455, 2048 }, + { 6, 1456, 2048 }, { 7, 1457, 2048 }, { 7, 1458, 2048 }, { 8, 1459, 2048 }, { 7, 1460, 2048 }, { 8, 1461, 2048 }, { 8, 1462, 2048 }, { 9, 1463, 2048 }, + { 7, 1464, 2048 }, { 8, 1465, 2048 }, { 8, 1466, 2048 }, { 9, 1467, 2048 }, { 8, 1468, 2048 }, { 9, 1469, 2048 }, { 9, 1470, 2048 }, { 10, 1471, 2048 }, + { 5, 1472, 2048 }, { 6, 1473, 2048 }, { 6, 1474, 2048 }, { 7, 1475, 2048 }, { 6, 1476, 2048 }, { 7, 1477, 2048 }, { 7, 1478, 2048 }, { 8, 1479, 2048 }, + { 6, 1480, 2048 }, { 7, 1481, 2048 }, { 7, 1482, 2048 }, { 8, 1483, 2048 }, { 7, 1484, 2048 }, { 8, 1485, 2048 }, { 8, 1486, 2048 }, { 9, 1487, 2048 }, + { 6, 1488, 2048 }, { 7, 1489, 2048 }, { 7, 1490, 2048 }, { 8, 1491, 2048 }, { 7, 1492, 2048 }, { 8, 1493, 2048 }, { 8, 1494, 2048 }, { 9, 1495, 2048 }, + { 7, 1496, 2048 }, { 8, 1497, 2048 }, { 8, 1498, 2048 }, { 9, 1499, 2048 }, { 8, 1500, 2048 }, { 9, 1501, 2048 }, { 9, 1502, 2048 }, { 10, 1503, 2048 }, + { 6, 1504, 2048 }, { 7, 1505, 2048 }, { 7, 1506, 2048 }, { 8, 1507, 2048 }, { 7, 1508, 2048 }, { 8, 1509, 2048 }, { 8, 1510, 2048 }, { 9, 1511, 2048 }, + { 7, 1512, 2048 }, { 8, 1513, 2048 }, { 8, 1514, 2048 }, { 9, 1515, 2048 }, { 8, 1516, 2048 }, { 9, 1517, 2048 }, { 9, 1518, 2048 }, { 10, 1519, 2048 }, + { 7, 1520, 2048 }, { 8, 1521, 2048 }, { 8, 1522, 2048 }, { 9, 1523, 2048 }, { 8, 1524, 2048 }, { 9, 1525, 2048 }, { 9, 1526, 2048 }, { 10, 1527, 2048 }, + { 8, 1528, 2048 }, { 9, 1529, 2048 }, { 9, 1530, 2048 }, { 10, 1531, 2048 }, { 9, 1532, 2048 }, { 10, 1533, 2048 }, { 10, 1534, 2048 }, { 11, 1535, 2048 }, + { 3, 1536, 2048 }, { 4, 1537, 2048 }, { 4, 1538, 2048 }, { 5, 1539, 2048 }, { 4, 1540, 2048 }, { 5, 1541, 2048 }, { 5, 1542, 2048 }, { 6, 1543, 2048 }, + { 4, 1544, 2048 }, { 5, 1545, 2048 }, { 5, 1546, 2048 }, { 6, 1547, 2048 }, { 5, 1548, 2048 }, { 6, 1549, 2048 }, { 6, 1550, 2048 }, { 7, 1551, 2048 }, + { 4, 1552, 2048 }, { 5, 1553, 2048 }, { 5, 1554, 2048 }, { 6, 1555, 2048 }, { 5, 1556, 2048 }, { 6, 1557, 2048 }, { 6, 1558, 2048 }, { 7, 1559, 2048 }, + { 5, 1560, 2048 }, { 6, 1561, 2048 }, { 6, 1562, 2048 }, { 7, 1563, 2048 }, { 6, 1564, 2048 }, { 7, 1565, 2048 }, { 7, 1566, 2048 }, { 8, 1567, 2048 }, + { 4, 1568, 2048 }, { 5, 1569, 2048 }, { 5, 1570, 2048 }, { 6, 1571, 2048 }, { 5, 1572, 2048 }, { 6, 1573, 2048 }, { 6, 1574, 2048 }, { 7, 1575, 2048 }, + { 5, 1576, 2048 }, { 6, 1577, 2048 }, { 6, 1578, 2048 }, { 7, 1579, 2048 }, { 6, 1580, 2048 }, { 7, 1581, 2048 }, { 7, 1582, 2048 }, { 8, 1583, 2048 }, + { 5, 1584, 2048 }, { 6, 1585, 2048 }, { 6, 1586, 2048 }, { 7, 1587, 2048 }, { 6, 1588, 2048 }, { 7, 1589, 2048 }, { 7, 1590, 2048 }, { 8, 1591, 2048 }, + { 6, 1592, 2048 }, { 7, 1593, 2048 }, { 7, 1594, 2048 }, { 8, 1595, 2048 }, { 7, 1596, 2048 }, { 8, 1597, 2048 }, { 8, 1598, 2048 }, { 9, 1599, 2048 }, + { 4, 1600, 2048 }, { 5, 1601, 2048 }, { 5, 1602, 2048 }, { 6, 1603, 2048 }, { 5, 1604, 2048 }, { 6, 1605, 2048 }, { 6, 1606, 2048 }, { 7, 1607, 2048 }, + { 5, 1608, 2048 }, { 6, 1609, 2048 }, { 6, 1610, 2048 }, { 7, 1611, 2048 }, { 6, 1612, 2048 }, { 7, 1613, 2048 }, { 7, 1614, 2048 }, { 8, 1615, 2048 }, + { 5, 1616, 2048 }, { 6, 1617, 2048 }, { 6, 1618, 2048 }, { 7, 1619, 2048 }, { 6, 1620, 2048 }, { 7, 1621, 2048 }, { 7, 1622, 2048 }, { 8, 1623, 2048 }, + { 6, 1624, 2048 }, { 7, 1625, 2048 }, { 7, 1626, 2048 }, { 8, 1627, 2048 }, { 7, 1628, 2048 }, { 8, 1629, 2048 }, { 8, 1630, 2048 }, { 9, 1631, 2048 }, + { 5, 1632, 2048 }, { 6, 1633, 2048 }, { 6, 1634, 2048 }, { 7, 1635, 2048 }, { 6, 1636, 2048 }, { 7, 1637, 2048 }, { 7, 1638, 2048 }, { 8, 1639, 2048 }, + { 6, 1640, 2048 }, { 7, 1641, 2048 }, { 7, 1642, 2048 }, { 8, 1643, 2048 }, { 7, 1644, 2048 }, { 8, 1645, 2048 }, { 8, 1646, 2048 }, { 9, 1647, 2048 }, + { 6, 1648, 2048 }, { 7, 1649, 2048 }, { 7, 1650, 2048 }, { 8, 1651, 2048 }, { 7, 1652, 2048 }, { 8, 1653, 2048 }, { 8, 1654, 2048 }, { 9, 1655, 2048 }, + { 7, 1656, 2048 }, { 8, 1657, 2048 }, { 8, 1658, 2048 }, { 9, 1659, 2048 }, { 8, 1660, 2048 }, { 9, 1661, 2048 }, { 9, 1662, 2048 }, { 10, 1663, 2048 }, + { 4, 1664, 2048 }, { 5, 1665, 2048 }, { 5, 1666, 2048 }, { 6, 1667, 2048 }, { 5, 1668, 2048 }, { 6, 1669, 2048 }, { 6, 1670, 2048 }, { 7, 1671, 2048 }, + { 5, 1672, 2048 }, { 6, 1673, 2048 }, { 6, 1674, 2048 }, { 7, 1675, 2048 }, { 6, 1676, 2048 }, { 7, 1677, 2048 }, { 7, 1678, 2048 }, { 8, 1679, 2048 }, + { 5, 1680, 2048 }, { 6, 1681, 2048 }, { 6, 1682, 2048 }, { 7, 1683, 2048 }, { 6, 1684, 2048 }, { 7, 1685, 2048 }, { 7, 1686, 2048 }, { 8, 1687, 2048 }, + { 6, 1688, 2048 }, { 7, 1689, 2048 }, { 7, 1690, 2048 }, { 8, 1691, 2048 }, { 7, 1692, 2048 }, { 8, 1693, 2048 }, { 8, 1694, 2048 }, { 9, 1695, 2048 }, + { 5, 1696, 2048 }, { 6, 1697, 2048 }, { 6, 1698, 2048 }, { 7, 1699, 2048 }, { 6, 1700, 2048 }, { 7, 1701, 2048 }, { 7, 1702, 2048 }, { 8, 1703, 2048 }, + { 6, 1704, 2048 }, { 7, 1705, 2048 }, { 7, 1706, 2048 }, { 8, 1707, 2048 }, { 7, 1708, 2048 }, { 8, 1709, 2048 }, { 8, 1710, 2048 }, { 9, 1711, 2048 }, + { 6, 1712, 2048 }, { 7, 1713, 2048 }, { 7, 1714, 2048 }, { 8, 1715, 2048 }, { 7, 1716, 2048 }, { 8, 1717, 2048 }, { 8, 1718, 2048 }, { 9, 1719, 2048 }, + { 7, 1720, 2048 }, { 8, 1721, 2048 }, { 8, 1722, 2048 }, { 9, 1723, 2048 }, { 8, 1724, 2048 }, { 9, 1725, 2048 }, { 9, 1726, 2048 }, { 10, 1727, 2048 }, + { 5, 1728, 2048 }, { 6, 1729, 2048 }, { 6, 1730, 2048 }, { 7, 1731, 2048 }, { 6, 1732, 2048 }, { 7, 1733, 2048 }, { 7, 1734, 2048 }, { 8, 1735, 2048 }, + { 6, 1736, 2048 }, { 7, 1737, 2048 }, { 7, 1738, 2048 }, { 8, 1739, 2048 }, { 7, 1740, 2048 }, { 8, 1741, 2048 }, { 8, 1742, 2048 }, { 9, 1743, 2048 }, + { 6, 1744, 2048 }, { 7, 1745, 2048 }, { 7, 1746, 2048 }, { 8, 1747, 2048 }, { 7, 1748, 2048 }, { 8, 1749, 2048 }, { 8, 1750, 2048 }, { 9, 1751, 2048 }, + { 7, 1752, 2048 }, { 8, 1753, 2048 }, { 8, 1754, 2048 }, { 9, 1755, 2048 }, { 8, 1756, 2048 }, { 9, 1757, 2048 }, { 9, 1758, 2048 }, { 10, 1759, 2048 }, + { 6, 1760, 2048 }, { 7, 1761, 2048 }, { 7, 1762, 2048 }, { 8, 1763, 2048 }, { 7, 1764, 2048 }, { 8, 1765, 2048 }, { 8, 1766, 2048 }, { 9, 1767, 2048 }, + { 7, 1768, 2048 }, { 8, 1769, 2048 }, { 8, 1770, 2048 }, { 9, 1771, 2048 }, { 8, 1772, 2048 }, { 9, 1773, 2048 }, { 9, 1774, 2048 }, { 10, 1775, 2048 }, + { 7, 1776, 2048 }, { 8, 1777, 2048 }, { 8, 1778, 2048 }, { 9, 1779, 2048 }, { 8, 1780, 2048 }, { 9, 1781, 2048 }, { 9, 1782, 2048 }, { 10, 1783, 2048 }, + { 8, 1784, 2048 }, { 9, 1785, 2048 }, { 9, 1786, 2048 }, { 10, 1787, 2048 }, { 9, 1788, 2048 }, { 10, 1789, 2048 }, { 10, 1790, 2048 }, { 11, 1791, 2048 }, + { 4, 1792, 2048 }, { 5, 1793, 2048 }, { 5, 1794, 2048 }, { 6, 1795, 2048 }, { 5, 1796, 2048 }, { 6, 1797, 2048 }, { 6, 1798, 2048 }, { 7, 1799, 2048 }, + { 5, 1800, 2048 }, { 6, 1801, 2048 }, { 6, 1802, 2048 }, { 7, 1803, 2048 }, { 6, 1804, 2048 }, { 7, 1805, 2048 }, { 7, 1806, 2048 }, { 8, 1807, 2048 }, + { 5, 1808, 2048 }, { 6, 1809, 2048 }, { 6, 1810, 2048 }, { 7, 1811, 2048 }, { 6, 1812, 2048 }, { 7, 1813, 2048 }, { 7, 1814, 2048 }, { 8, 1815, 2048 }, + { 6, 1816, 2048 }, { 7, 1817, 2048 }, { 7, 1818, 2048 }, { 8, 1819, 2048 }, { 7, 1820, 2048 }, { 8, 1821, 2048 }, { 8, 1822, 2048 }, { 9, 1823, 2048 }, + { 5, 1824, 2048 }, { 6, 1825, 2048 }, { 6, 1826, 2048 }, { 7, 1827, 2048 }, { 6, 1828, 2048 }, { 7, 1829, 2048 }, { 7, 1830, 2048 }, { 8, 1831, 2048 }, + { 6, 1832, 2048 }, { 7, 1833, 2048 }, { 7, 1834, 2048 }, { 8, 1835, 2048 }, { 7, 1836, 2048 }, { 8, 1837, 2048 }, { 8, 1838, 2048 }, { 9, 1839, 2048 }, + { 6, 1840, 2048 }, { 7, 1841, 2048 }, { 7, 1842, 2048 }, { 8, 1843, 2048 }, { 7, 1844, 2048 }, { 8, 1845, 2048 }, { 8, 1846, 2048 }, { 9, 1847, 2048 }, + { 7, 1848, 2048 }, { 8, 1849, 2048 }, { 8, 1850, 2048 }, { 9, 1851, 2048 }, { 8, 1852, 2048 }, { 9, 1853, 2048 }, { 9, 1854, 2048 }, { 10, 1855, 2048 }, + { 5, 1856, 2048 }, { 6, 1857, 2048 }, { 6, 1858, 2048 }, { 7, 1859, 2048 }, { 6, 1860, 2048 }, { 7, 1861, 2048 }, { 7, 1862, 2048 }, { 8, 1863, 2048 }, + { 6, 1864, 2048 }, { 7, 1865, 2048 }, { 7, 1866, 2048 }, { 8, 1867, 2048 }, { 7, 1868, 2048 }, { 8, 1869, 2048 }, { 8, 1870, 2048 }, { 9, 1871, 2048 }, + { 6, 1872, 2048 }, { 7, 1873, 2048 }, { 7, 1874, 2048 }, { 8, 1875, 2048 }, { 7, 1876, 2048 }, { 8, 1877, 2048 }, { 8, 1878, 2048 }, { 9, 1879, 2048 }, + { 7, 1880, 2048 }, { 8, 1881, 2048 }, { 8, 1882, 2048 }, { 9, 1883, 2048 }, { 8, 1884, 2048 }, { 9, 1885, 2048 }, { 9, 1886, 2048 }, { 10, 1887, 2048 }, + { 6, 1888, 2048 }, { 7, 1889, 2048 }, { 7, 1890, 2048 }, { 8, 1891, 2048 }, { 7, 1892, 2048 }, { 8, 1893, 2048 }, { 8, 1894, 2048 }, { 9, 1895, 2048 }, + { 7, 1896, 2048 }, { 8, 1897, 2048 }, { 8, 1898, 2048 }, { 9, 1899, 2048 }, { 8, 1900, 2048 }, { 9, 1901, 2048 }, { 9, 1902, 2048 }, { 10, 1903, 2048 }, + { 7, 1904, 2048 }, { 8, 1905, 2048 }, { 8, 1906, 2048 }, { 9, 1907, 2048 }, { 8, 1908, 2048 }, { 9, 1909, 2048 }, { 9, 1910, 2048 }, { 10, 1911, 2048 }, + { 8, 1912, 2048 }, { 9, 1913, 2048 }, { 9, 1914, 2048 }, { 10, 1915, 2048 }, { 9, 1916, 2048 }, { 10, 1917, 2048 }, { 10, 1918, 2048 }, { 11, 1919, 2048 }, + { 5, 1920, 2048 }, { 6, 1921, 2048 }, { 6, 1922, 2048 }, { 7, 1923, 2048 }, { 6, 1924, 2048 }, { 7, 1925, 2048 }, { 7, 1926, 2048 }, { 8, 1927, 2048 }, + { 6, 1928, 2048 }, { 7, 1929, 2048 }, { 7, 1930, 2048 }, { 8, 1931, 2048 }, { 7, 1932, 2048 }, { 8, 1933, 2048 }, { 8, 1934, 2048 }, { 9, 1935, 2048 }, + { 6, 1936, 2048 }, { 7, 1937, 2048 }, { 7, 1938, 2048 }, { 8, 1939, 2048 }, { 7, 1940, 2048 }, { 8, 1941, 2048 }, { 8, 1942, 2048 }, { 9, 1943, 2048 }, + { 7, 1944, 2048 }, { 8, 1945, 2048 }, { 8, 1946, 2048 }, { 9, 1947, 2048 }, { 8, 1948, 2048 }, { 9, 1949, 2048 }, { 9, 1950, 2048 }, { 10, 1951, 2048 }, + { 6, 1952, 2048 }, { 7, 1953, 2048 }, { 7, 1954, 2048 }, { 8, 1955, 2048 }, { 7, 1956, 2048 }, { 8, 1957, 2048 }, { 8, 1958, 2048 }, { 9, 1959, 2048 }, + { 7, 1960, 2048 }, { 8, 1961, 2048 }, { 8, 1962, 2048 }, { 9, 1963, 2048 }, { 8, 1964, 2048 }, { 9, 1965, 2048 }, { 9, 1966, 2048 }, { 10, 1967, 2048 }, + { 7, 1968, 2048 }, { 8, 1969, 2048 }, { 8, 1970, 2048 }, { 9, 1971, 2048 }, { 8, 1972, 2048 }, { 9, 1973, 2048 }, { 9, 1974, 2048 }, { 10, 1975, 2048 }, + { 8, 1976, 2048 }, { 9, 1977, 2048 }, { 9, 1978, 2048 }, { 10, 1979, 2048 }, { 9, 1980, 2048 }, { 10, 1981, 2048 }, { 10, 1982, 2048 }, { 11, 1983, 2048 }, + { 6, 1984, 2048 }, { 7, 1985, 2048 }, { 7, 1986, 2048 }, { 8, 1987, 2048 }, { 7, 1988, 2048 }, { 8, 1989, 2048 }, { 8, 1990, 2048 }, { 9, 1991, 2048 }, + { 7, 1992, 2048 }, { 8, 1993, 2048 }, { 8, 1994, 2048 }, { 9, 1995, 2048 }, { 8, 1996, 2048 }, { 9, 1997, 2048 }, { 9, 1998, 2048 }, { 10, 1999, 2048 }, + { 7, 2000, 2048 }, { 8, 2001, 2048 }, { 8, 2002, 2048 }, { 9, 2003, 2048 }, { 8, 2004, 2048 }, { 9, 2005, 2048 }, { 9, 2006, 2048 }, { 10, 2007, 2048 }, + { 8, 2008, 2048 }, { 9, 2009, 2048 }, { 9, 2010, 2048 }, { 10, 2011, 2048 }, { 9, 2012, 2048 }, { 10, 2013, 2048 }, { 10, 2014, 2048 }, { 11, 2015, 2048 }, + { 7, 2016, 2048 }, { 8, 2017, 2048 }, { 8, 2018, 2048 }, { 9, 2019, 2048 }, { 8, 2020, 2048 }, { 9, 2021, 2048 }, { 9, 2022, 2048 }, { 10, 2023, 2048 }, + { 8, 2024, 2048 }, { 9, 2025, 2048 }, { 9, 2026, 2048 }, { 10, 2027, 2048 }, { 9, 2028, 2048 }, { 10, 2029, 2048 }, { 10, 2030, 2048 }, { 11, 2031, 2048 }, + { 8, 2032, 2048 }, { 9, 2033, 2048 }, { 9, 2034, 2048 }, { 10, 2035, 2048 }, { 9, 2036, 2048 }, { 10, 2037, 2048 }, { 10, 2038, 2048 }, { 11, 2039, 2048 }, + { 9, 2040, 2048 }, { 10, 2041, 2048 }, { 10, 2042, 2048 }, { 11, 2043, 2048 }, { 10, 2044, 2048 }, { 11, 2045, 2048 }, { 11, 2046, 2048 }, { 12, 2047, 2048 }, +#endif +#endif +#endif +#endif +#endif +#endif +}; + +/* find a hole and free as required, return -1 if no hole found */ +static int s_find_hole(void) +{ + unsigned x; + int y, z; + for (z = -1, y = INT_MAX, x = 0; x < FP_ENTRIES; x++) { + if (fp_cache[x].lru_count < y && fp_cache[x].lock == 0) { + z = x; + y = fp_cache[x].lru_count; + } + } + + /* decrease all */ + for (x = 0; x < FP_ENTRIES; x++) { + if (fp_cache[x].lru_count > 3) { + --(fp_cache[x].lru_count); + } + } + + /* free entry z */ + if (z >= 0 && fp_cache[z].g) { + if (fp_cache[z].mu != NULL) { + mp_clear(fp_cache[z].mu); + fp_cache[z].mu = NULL; + } + ltc_ecc_del_point(fp_cache[z].g); + fp_cache[z].g = NULL; + for (x = 0; x < (1U<x, g->x) == LTC_MP_EQ && + mp_cmp(fp_cache[x].g->y, g->y) == LTC_MP_EQ && + mp_cmp(fp_cache[x].g->z, g->z) == LTC_MP_EQ) { + break; + } + } + if (x == FP_ENTRIES) { + x = -1; + } + return x; +} + +/* add a new base to the cache */ +static int s_add_entry(int idx, ecc_point *g) +{ + unsigned x, y; + + /* allocate base and LUT */ + fp_cache[idx].g = ltc_ecc_new_point(); + if (fp_cache[idx].g == NULL) { + return CRYPT_MEM; + } + + /* copy x and y */ + if ((mp_copy(g->x, fp_cache[idx].g->x) != CRYPT_OK) || + (mp_copy(g->y, fp_cache[idx].g->y) != CRYPT_OK) || + (mp_copy(g->z, fp_cache[idx].g->z) != CRYPT_OK)) { + ltc_ecc_del_point(fp_cache[idx].g); + fp_cache[idx].g = NULL; + return CRYPT_MEM; + } + + for (x = 0; x < (1U<x, mu, modulus, fp_cache[idx].LUT[1]->x) != CRYPT_OK) || + (mp_mulmod(fp_cache[idx].g->y, mu, modulus, fp_cache[idx].LUT[1]->y) != CRYPT_OK) || + (mp_mulmod(fp_cache[idx].g->z, mu, modulus, fp_cache[idx].LUT[1]->z) != CRYPT_OK)) { goto ERR; } + + /* make all single bit entries */ + for (x = 1; x < FP_LUT; x++) { + if ((mp_copy(fp_cache[idx].LUT[1<<(x-1)]->x, fp_cache[idx].LUT[1<x) != CRYPT_OK) || + (mp_copy(fp_cache[idx].LUT[1<<(x-1)]->y, fp_cache[idx].LUT[1<y) != CRYPT_OK) || + (mp_copy(fp_cache[idx].LUT[1<<(x-1)]->z, fp_cache[idx].LUT[1<z) != CRYPT_OK)) { goto ERR; } + + /* now double it bitlen/FP_LUT times */ + for (y = 0; y < lut_gap; y++) { + if ((err = ltc_mp.ecc_ptdbl(fp_cache[idx].LUT[1<z, modulus, mp)) != CRYPT_OK) { goto ERR; } + + /* invert it */ + if ((err = mp_invmod(fp_cache[idx].LUT[x]->z, modulus, fp_cache[idx].LUT[x]->z)) != CRYPT_OK) { goto ERR; } + + /* now square it */ + if ((err = mp_sqrmod(fp_cache[idx].LUT[x]->z, modulus, tmp)) != CRYPT_OK) { goto ERR; } + + /* fix x */ + if ((err = mp_mulmod(fp_cache[idx].LUT[x]->x, tmp, modulus, fp_cache[idx].LUT[x]->x)) != CRYPT_OK) { goto ERR; } + + /* get 1/z^3 */ + if ((err = mp_mulmod(tmp, fp_cache[idx].LUT[x]->z, modulus, tmp)) != CRYPT_OK) { goto ERR; } + + /* fix y */ + if ((err = mp_mulmod(fp_cache[idx].LUT[x]->y, tmp, modulus, fp_cache[idx].LUT[x]->y)) != CRYPT_OK) { goto ERR; } + + /* free z */ + mp_clear(fp_cache[idx].LUT[x]->z); + fp_cache[idx].LUT[x]->z = NULL; + } + mp_clear(tmp); + + return CRYPT_OK; +ERR: + err = CRYPT_MEM; +DONE: + for (y = 0; y < (1U< mp_unsigned_bin_size(modulus)) { + /* find order */ + y = mp_unsigned_bin_size(modulus); + for (x = 0; ltc_ecc_sets[x].size; x++) { + if (y <= (unsigned)ltc_ecc_sets[x].size) break; + } + + /* back off if we are on the 521 bit curve */ + if (y == 66) --x; + + if ((err = mp_init(&order)) != CRYPT_OK) { + return err; + } + if ((err = mp_read_radix(order, ltc_ecc_sets[x].order, 16)) != CRYPT_OK) { + mp_clear(&order); + return err; + } + + /* k must be less than modulus */ + if (mp_cmp(k, order) != LTC_MP_LT) { + if ((err = mp_init(&tk)) != CRYPT_OK) { + mp_clear(order); + return err; + } + if ((err = mp_mod(k, order, tk)) != CRYPT_OK) { + mp_clear(tk); + mp_clear(order); + return err; + } + } else { + tk = k; + } + mp_clear(order); + } else { + tk = k; + } + + /* get bitlen and round up to next multiple of FP_LUT */ + bitlen = mp_unsigned_bin_size(modulus) << 3; + x = bitlen % FP_LUT; + if (x) { + bitlen += FP_LUT - x; + } + lut_gap = bitlen / FP_LUT; + + /* get the k value */ + if (mp_unsigned_bin_size(tk) > (sizeof(kb) - 2)) { + if (tk != k) { + mp_clear(tk); + } + return CRYPT_BUFFER_OVERFLOW; + } + + /* store k */ + zeromem(kb, sizeof(kb)); + if ((err = mp_to_unsigned_bin(tk, kb)) != CRYPT_OK) { + if (tk != k) { + mp_clear(tk); + } + return err; + } + + /* let's reverse kb so it's little endian */ + x = 0; + y = mp_unsigned_bin_size(tk) - 1; + if (tk != k) { + mp_clear(tk); + } + while ((unsigned)x < y) { + z = kb[x]; kb[x] = kb[y]; kb[y] = z; + ++x; --y; + } + + /* at this point we can start, yipee */ + first = 1; + for (x = lut_gap-1; x >= 0; x--) { + /* extract FP_LUT bits from kb spread out by lut_gap bits and offset by x bits from the start */ + bitpos = x; + for (y = z = 0; y < FP_LUT; y++) { + z |= ((kb[bitpos>>3] >> (bitpos&7)) & 1) << y; + bitpos += lut_gap; /* it's y*lut_gap + x, but here we can avoid the mult in each loop */ + } + + /* double if not first */ + if (!first) { + if ((err = ltc_mp.ecc_ptdbl(R, R, a, modulus, mp)) != CRYPT_OK) { + return err; + } + } + + /* add if not first, otherwise copy */ + if (!first && z) { + if ((err = ltc_mp.ecc_ptadd(R, fp_cache[idx].LUT[z], R, a, modulus, mp)) != CRYPT_OK) { + return err; + } + } else if (z) { + if ((mp_copy(fp_cache[idx].LUT[z]->x, R->x) != CRYPT_OK) || + (mp_copy(fp_cache[idx].LUT[z]->y, R->y) != CRYPT_OK) || + (mp_copy(fp_cache[idx].mu, R->z) != CRYPT_OK)) { return CRYPT_MEM; } + first = 0; + } + } + z = 0; + zeromem(kb, sizeof(kb)); + /* map R back from projective space */ + if (map) { + err = ltc_ecc_map(R, modulus, mp); + } else { + err = CRYPT_OK; + } + return err; +} + +#ifdef LTC_ECC_SHAMIR +/* perform a fixed point ECC mulmod */ +static int ss_accel_fp_mul2add(int idx1, int idx2, + void *kA, void *kB, + ecc_point *R, void *a, void *modulus, void *mp) +{ + unsigned char kb[2][128]; + int x; + unsigned y, z, err, bitlen, bitpos, lut_gap, first, zA, zB; + void *tka, *tkb, *order; + + /* if it's smaller than modulus we fine */ + if (mp_unsigned_bin_size(kA) > mp_unsigned_bin_size(modulus)) { + /* find order */ + y = mp_unsigned_bin_size(modulus); + for (x = 0; ltc_ecc_sets[x].size; x++) { + if (y <= (unsigned)ltc_ecc_sets[x].size) break; + } + + /* back off if we are on the 521 bit curve */ + if (y == 66) --x; + + if ((err = mp_init(&order)) != CRYPT_OK) { + return err; + } + if ((err = mp_read_radix(order, ltc_ecc_sets[x].order, 16)) != CRYPT_OK) { + mp_clear(&order); + return err; + } + + /* kA must be less than modulus */ + if (mp_cmp(kA, order) != LTC_MP_LT) { + if ((err = mp_init(&tka)) != CRYPT_OK) { + mp_clear(order); + return err; + } + if ((err = mp_mod(kA, order, tka)) != CRYPT_OK) { + mp_clear(tka); + mp_clear(order); + return err; + } + } else { + tka = kA; + } + mp_clear(order); + } else { + tka = kA; + } + + /* if it's smaller than modulus we fine */ + if (mp_unsigned_bin_size(kB) > mp_unsigned_bin_size(modulus)) { + /* find order */ + y = mp_unsigned_bin_size(modulus); + for (x = 0; ltc_ecc_sets[x].size; x++) { + if (y <= (unsigned)ltc_ecc_sets[x].size) break; + } + + /* back off if we are on the 521 bit curve */ + if (y == 66) --x; + + if ((err = mp_init(&order)) != CRYPT_OK) { + return err; + } + if ((err = mp_read_radix(order, ltc_ecc_sets[x].order, 16)) != CRYPT_OK) { + mp_clear(&order); + return err; + } + + /* kB must be less than modulus */ + if (mp_cmp(kB, order) != LTC_MP_LT) { + if ((err = mp_init(&tkb)) != CRYPT_OK) { + mp_clear(order); + return err; + } + if ((err = mp_mod(kB, order, tkb)) != CRYPT_OK) { + mp_clear(tkb); + mp_clear(order); + return err; + } + } else { + tkb = kB; + } + mp_clear(order); + } else { + tkb = kB; + } + + /* get bitlen and round up to next multiple of FP_LUT */ + bitlen = mp_unsigned_bin_size(modulus) << 3; + x = bitlen % FP_LUT; + if (x) { + bitlen += FP_LUT - x; + } + lut_gap = bitlen / FP_LUT; + + /* get the k value */ + if ((mp_unsigned_bin_size(tka) > (sizeof(kb[0]) - 2)) || (mp_unsigned_bin_size(tkb) > (sizeof(kb[0]) - 2)) ) { + if (tka != kA) { + mp_clear(tka); + } + if (tkb != kB) { + mp_clear(tkb); + } + return CRYPT_BUFFER_OVERFLOW; + } + + /* store k */ + zeromem(kb, sizeof(kb)); + if ((err = mp_to_unsigned_bin(tka, kb[0])) != CRYPT_OK) { + if (tka != kA) { + mp_clear(tka); + } + if (tkb != kB) { + mp_clear(tkb); + } + return err; + } + + /* let's reverse kb so it's little endian */ + x = 0; + y = mp_unsigned_bin_size(tka) - 1; + if (tka != kA) { + mp_clear(tka); + } + while ((unsigned)x < y) { + z = kb[0][x]; kb[0][x] = kb[0][y]; kb[0][y] = z; + ++x; --y; + } + + /* store b */ + if ((err = mp_to_unsigned_bin(tkb, kb[1])) != CRYPT_OK) { + if (tkb != kB) { + mp_clear(tkb); + } + return err; + } + + x = 0; + y = mp_unsigned_bin_size(tkb) - 1; + if (tkb != kB) { + mp_clear(tkb); + } + while ((unsigned)x < y) { + z = kb[1][x]; kb[1][x] = kb[1][y]; kb[1][y] = z; + ++x; --y; + } + + /* at this point we can start, yipee */ + first = 1; + for (x = lut_gap-1; x >= 0; x--) { + /* extract FP_LUT bits from kb spread out by lut_gap bits and offset by x bits from the start */ + bitpos = x; + for (y = zA = zB = 0; y < FP_LUT; y++) { + zA |= ((kb[0][bitpos>>3] >> (bitpos&7)) & 1) << y; + zB |= ((kb[1][bitpos>>3] >> (bitpos&7)) & 1) << y; + bitpos += lut_gap; /* it's y*lut_gap + x, but here we can avoid the mult in each loop */ + } + + /* double if not first */ + if (!first) { + if ((err = ltc_mp.ecc_ptdbl(R, R, a, modulus, mp)) != CRYPT_OK) { + return err; + } + } + + /* add if not first, otherwise copy */ + if (!first) { + if (zA) { + if ((err = ltc_mp.ecc_ptadd(R, fp_cache[idx1].LUT[zA], R, a, modulus, mp)) != CRYPT_OK) { + return err; + } + } + if (zB) { + if ((err = ltc_mp.ecc_ptadd(R, fp_cache[idx2].LUT[zB], R, a, modulus, mp)) != CRYPT_OK) { + return err; + } + } + } else { + if (zA) { + if ((mp_copy(fp_cache[idx1].LUT[zA]->x, R->x) != CRYPT_OK) || + (mp_copy(fp_cache[idx1].LUT[zA]->y, R->y) != CRYPT_OK) || + (mp_copy(fp_cache[idx1].mu, R->z) != CRYPT_OK)) { return CRYPT_MEM; } + first = 0; + } + if (zB && first == 0) { + if (zB) { + if ((err = ltc_mp.ecc_ptadd(R, fp_cache[idx2].LUT[zB], R, a, modulus, mp)) != CRYPT_OK) { + return err; + } + } + } else if (zB && first == 1) { + if ((mp_copy(fp_cache[idx2].LUT[zB]->x, R->x) != CRYPT_OK) || + (mp_copy(fp_cache[idx2].LUT[zB]->y, R->y) != CRYPT_OK) || + (mp_copy(fp_cache[idx2].mu, R->z) != CRYPT_OK)) { return CRYPT_MEM; } + first = 0; + } + } + } + zeromem(kb, sizeof(kb)); + return ltc_ecc_map(R, modulus, mp); +} + +/** ECC Fixed Point mulmod global + Computes kA*A + kB*B = C using Shamir's Trick + @param A First point to multiply + @param kA What to multiple A by + @param B Second point to multiply + @param kB What to multiple B by + @param C [out] Destination point (can overlap with A or B) + @param modulus Modulus for curve + @return CRYPT_OK on success +*/ +int ltc_ecc_fp_mul2add(ecc_point *A, void *kA, + ecc_point *B, void *kB, + ecc_point *C, + void *a, + void *modulus) +{ + int idx1, idx2, err; + void *mp, *mu; + + mp = NULL; + mu = NULL; + LTC_MUTEX_LOCK(<c_ecc_fp_lock); + /* find point */ + idx1 = s_find_base(A); + + /* no entry? */ + if (idx1 == -1) { + /* find hole and add it */ + if ((idx1 = s_find_hole()) >= 0) { + if ((err = s_add_entry(idx1, A)) != CRYPT_OK) { + goto LBL_ERR; + } + } + } + if (idx1 != -1) { + /* increment LRU */ + ++(fp_cache[idx1].lru_count); + } + + /* find point */ + idx2 = s_find_base(B); + + /* no entry? */ + if (idx2 == -1) { + /* find hole and add it */ + if ((idx2 = s_find_hole()) >= 0) { + if ((err = s_add_entry(idx2, B)) != CRYPT_OK) { + goto LBL_ERR; + } + } + } + if (idx2 != -1) { + /* increment LRU */ + ++(fp_cache[idx2].lru_count); + } + + /* if it's 2 build the LUT, if it's higher just use the LUT */ + if (idx1 >= 0 && fp_cache[idx1].lru_count == 2) { + /* compute mp */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { goto LBL_ERR; } + + /* compute mu */ + if ((err = mp_init(&mu)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = mp_montgomery_normalization(mu, modulus)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* build the LUT */ + if ((err = s_build_lut(idx1, a, modulus, mp, mu)) != CRYPT_OK) { + goto LBL_ERR;; + } + } + + /* if it's 2 build the LUT, if it's higher just use the LUT */ + if (idx2 >= 0 && fp_cache[idx2].lru_count == 2) { + if (mp == NULL) { + /* compute mp */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { goto LBL_ERR; } + + /* compute mu */ + if ((err = mp_init(&mu)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = mp_montgomery_normalization(mu, modulus)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + /* build the LUT */ + if ((err = s_build_lut(idx2, a, modulus, mp, mu)) != CRYPT_OK) { + goto LBL_ERR;; + } + } + + + if (idx1 >=0 && idx2 >= 0 && fp_cache[idx1].lru_count >= 2 && fp_cache[idx2].lru_count >= 2) { + if (mp == NULL) { + /* compute mp */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { goto LBL_ERR; } + } + err = ss_accel_fp_mul2add(idx1, idx2, kA, kB, C, a, modulus, mp); + } else { + err = ltc_ecc_mul2add(A, kA, B, kB, C, a, modulus); + } +LBL_ERR: + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); + if (mp != NULL) { + mp_montgomery_free(mp); + } + if (mu != NULL) { + mp_clear(mu); + } + return err; +} +#endif + +/** ECC Fixed Point mulmod global + @param k The multiplicand + @param G Base point to multiply + @param R [out] Destination of product + @param a ECC curve parameter a + @param modulus The modulus for the curve + @param map [boolean] If non-zero maps the point back to affine co-ordinates, otherwise it's left in jacobian-montgomery form + @return CRYPT_OK if successful +*/ +int ltc_ecc_fp_mulmod(void *k, ecc_point *G, ecc_point *R, void *a, void *modulus, int map) +{ + int idx, err; + void *mp, *mu; + + mp = NULL; + mu = NULL; + LTC_MUTEX_LOCK(<c_ecc_fp_lock); + /* find point */ + idx = s_find_base(G); + + /* no entry? */ + if (idx == -1) { + /* find hole and add it */ + idx = s_find_hole(); + + if (idx >= 0) { + if ((err = s_add_entry(idx, G)) != CRYPT_OK) { + goto LBL_ERR; + } + } + } + if (idx != -1) { + /* increment LRU */ + ++(fp_cache[idx].lru_count); + } + + + /* if it's 2 build the LUT, if it's higher just use the LUT */ + if (idx >= 0 && fp_cache[idx].lru_count == 2) { + /* compute mp */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { goto LBL_ERR; } + + /* compute mu */ + if ((err = mp_init(&mu)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = mp_montgomery_normalization(mu, modulus)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* build the LUT */ + if ((err = s_build_lut(idx, a, modulus, mp, mu)) != CRYPT_OK) { + goto LBL_ERR;; + } + } + + if (idx >= 0 && fp_cache[idx].lru_count >= 2) { + if (mp == NULL) { + /* compute mp */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { goto LBL_ERR; } + } + err = s_accel_fp_mul(idx, k, R, a, modulus, mp, map); + } else { + err = ltc_ecc_mulmod(k, G, R, a, modulus, map); + } +LBL_ERR: + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); + if (mp != NULL) { + mp_montgomery_free(mp); + } + if (mu != NULL) { + mp_clear(mu); + } + return err; +} + +/* helper function for freeing the cache ... must be called with the cache mutex locked */ +static void s_ltc_ecc_fp_free_cache(void) +{ + unsigned x, y; + for (x = 0; x < FP_ENTRIES; x++) { + if (fp_cache[x].g != NULL) { + for (y = 0; y < (1U<= 0) { + /* it is already in the cache ... just check that the LUT is initialized */ + if(fp_cache[idx].lru_count >= 2) { + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); + return CRYPT_OK; + } + } + + if(idx == -1 && (idx = s_find_hole()) == -1) { + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + if ((err = s_add_entry(idx, g)) != CRYPT_OK) { + goto LBL_ERR; + } + /* compute mp */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* compute mu */ + if ((err = mp_init(&mu)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = mp_montgomery_normalization(mu, modulus)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* build the LUT */ + if ((err = s_build_lut(idx, a, modulus, mp, mu)) != CRYPT_OK) { + goto LBL_ERR; + } + fp_cache[idx].lru_count = 2; + fp_cache[idx].lock = lock; +LBL_ERR: + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); + if (mp != NULL) { + mp_montgomery_free(mp); + } + if (mu != NULL) { + mp_clear(mu); + } + return err; +} + +/** Prevent/permit the FP cache from being updated + @param flag If flag is 0, remove cache lock (unlock), otherwise lock it +*/ +void ltc_ecc_fp_tablelock(int lock) +{ + int i; + + LTC_MUTEX_LOCK(<c_ecc_fp_lock); + for (i = 0; i < FP_ENTRIES; i++) { + fp_cache[i].lock = lock; + } + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); +} + +/** Export the current cache as a binary packet + @param out [out] pointer to malloc'ed space containing the packet + @param outlen [out] size of exported packet + @return CRYPT_OK if successful +*/ +int ltc_ecc_fp_save_state(unsigned char **out, unsigned long *outlen) +{ + ltc_asn1_list *cache_entry; + unsigned int i, j, k; + unsigned long fp_entries, fp_lut, num_entries; + int err; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + fp_entries = FP_ENTRIES; + fp_lut = FP_LUT; + num_entries = 0; + + LTC_MUTEX_LOCK(<c_ecc_fp_lock); + /* + * build the list; + Cache DEFINITIONS ::= + BEGIN + CacheDump ::= SEQUENCE { + numEntries SHORTINTEGER, + maxEntries SHORTINTEGER, + numLUT SHORTINTEGER, + cache SEQUENCE OF INTEGER + } + END + * + */ + /* + * The cache itself is a point (3 INTEGERS), + * the LUT as pairs of INTEGERS (2 * 1<x, 1); + LTC_SET_ASN1(cache_entry, j++, LTC_ASN1_INTEGER, fp_cache[i].g->y, 1); + LTC_SET_ASN1(cache_entry, j++, LTC_ASN1_INTEGER, fp_cache[i].g->z, 1); + for (k = 0; k < (1U<x, 1); + LTC_SET_ASN1(cache_entry, j++, LTC_ASN1_INTEGER, fp_cache[i].LUT[k]->y, 1); + } + LTC_SET_ASN1(cache_entry, j++, LTC_ASN1_INTEGER, fp_cache[i].mu, 1); + } + LTC_SET_ASN1(cache_entry, j++, LTC_ASN1_EOL, 0, 0); + + LTC_SET_ASN1(cache_entry, 0, LTC_ASN1_SHORT_INTEGER, &num_entries, 1); + + if ((err = der_length_sequence(cache_entry, j, outlen)) != CRYPT_OK) { + goto save_err; + } + if ((*out = XMALLOC(*outlen)) == NULL) { + err = CRYPT_MEM; + goto save_err; + } + err = der_encode_sequence(cache_entry, j, *out, outlen); +save_err: + XFREE(cache_entry); + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); + return err; +} + +/** Import a binary packet into the current cache + @param in [in] pointer to packet + @param inlen [in] size of packet (bytes) + @return CRYPT_OK if successful +*/ +int ltc_ecc_fp_restore_state(unsigned char *in, unsigned long inlen) +{ + int err; + ltc_asn1_list *asn1_list; + unsigned long num_entries, fp_entries, fp_lut; + unsigned long i, j; + unsigned int x; + + LTC_ARGCHK(in != NULL); + if (inlen == 0) { + return CRYPT_INVALID_ARG; + } + + /* zero indecies */ + i = 0; + j = 0; + asn1_list = NULL; + + LTC_MUTEX_LOCK(<c_ecc_fp_lock); + /* + * start with an empty cache + */ + s_ltc_ecc_fp_free_cache(); + + /* + * decode the input packet: It consists of a sequence with a few + * integers (including the FP_ENTRIES and FP_LUT sizes), followed by a + * SEQUENCE which is the cache itself. + * + * use standard decoding for the first part, then flexible for the second + */ + if((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_SHORT_INTEGER, 1, &num_entries, + LTC_ASN1_SHORT_INTEGER, 1, &fp_entries, + LTC_ASN1_SHORT_INTEGER, 1, &fp_lut, + LTC_ASN1_EOL, 0, 0)) != CRYPT_OK) { + goto ERR_OUT; + } + if (fp_entries != FP_ENTRIES || fp_lut != FP_LUT || num_entries > fp_entries) { + err = CRYPT_INVALID_PACKET; + goto ERR_OUT; + } + if ((asn1_list = XCALLOC(3+num_entries*(4+2*(1<x, 1); + LTC_SET_ASN1(asn1_list, j++, LTC_ASN1_INTEGER, fp_cache[i].g->y, 1); + LTC_SET_ASN1(asn1_list, j++, LTC_ASN1_INTEGER, fp_cache[i].g->z, 1); + for (x = 0; x < (1U<x, &p->y, LTC_NULL)) != CRYPT_OK) { + goto ERR_OUT; + } + p->z = NULL; + LTC_SET_ASN1(asn1_list, j++, LTC_ASN1_INTEGER, p->x, 1); + LTC_SET_ASN1(asn1_list, j++, LTC_ASN1_INTEGER, p->y, 1); + } + if((err = mp_init(&fp_cache[i].mu)) != CRYPT_OK) { + goto ERR_OUT; + } + LTC_SET_ASN1(asn1_list, j++, LTC_ASN1_INTEGER, fp_cache[i].mu, 1); + fp_cache[i].lru_count = 3; + fp_cache[i].lock = 1; + } + + if ((err = der_decode_sequence(in, inlen, asn1_list, j)) != CRYPT_OK) { + goto ERR_OUT; + } + XFREE(asn1_list); + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); + return CRYPT_OK; +ERR_OUT: + if(asn1_list) + XFREE(asn1_list); + s_ltc_ecc_fp_free_cache(); + LTC_MUTEX_UNLOCK(<c_ecc_fp_lock); + return err; +} + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/math/fp/sub.mk b/optee_os/core/lib/libtomcrypt/src/math/fp/sub.mk new file mode 100644 index 0000000..fe79656 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/fp/sub.mk @@ -0,0 +1 @@ +srcs-$(_CFG_CORE_LTC_ECC) += ltc_ecc_fp_mulmod.c diff --git a/optee_os/core/lib/libtomcrypt/src/math/gmp_desc.c b/optee_os/core/lib/libtomcrypt/src/math/gmp_desc.c new file mode 100644 index 0000000..2db699d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/gmp_desc.c @@ -0,0 +1,658 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#define DESC_DEF_ONLY +#include "tomcrypt_private.h" + +#ifdef GMP_DESC + +#include +#include + +static int init(void **a) +{ + LTC_ARGCHK(a != NULL); + + *a = XCALLOC(1, sizeof(__mpz_struct)); + if (*a == NULL) { + return CRYPT_MEM; + } + mpz_init(((__mpz_struct *)*a)); + return CRYPT_OK; +} + +static void deinit(void *a) +{ + LTC_ARGCHKVD(a != NULL); + mpz_clear(a); + XFREE(a); +} + +static int neg(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + mpz_neg(b, a); + return CRYPT_OK; +} + +static int copy(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + mpz_set(b, a); + return CRYPT_OK; +} + +static int init_copy(void **a, void *b) +{ + if (init(a) != CRYPT_OK) { + return CRYPT_MEM; + } + return copy(b, *a); +} + +/* ---- trivial ---- */ +static int set_int(void *a, ltc_mp_digit b) +{ + LTC_ARGCHK(a != NULL); + mpz_set_ui(((__mpz_struct *)a), b); + return CRYPT_OK; +} + +static unsigned long get_int(void *a) +{ + LTC_ARGCHK(a != NULL); + return mpz_get_ui(a); +} + +static ltc_mp_digit get_digit(void *a, int n) +{ + LTC_ARGCHK(a != NULL); + return mpz_getlimbn(a, n); +} + +static int get_digit_count(void *a) +{ + LTC_ARGCHK(a != NULL); + return mpz_size(a); +} + +static int compare(void *a, void *b) +{ + int ret; + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + ret = mpz_cmp(a, b); + if (ret < 0) { + return LTC_MP_LT; + } else if (ret > 0) { + return LTC_MP_GT; + } else { + return LTC_MP_EQ; + } +} + +static int compare_d(void *a, ltc_mp_digit b) +{ + int ret; + LTC_ARGCHK(a != NULL); + ret = mpz_cmp_ui(((__mpz_struct *)a), b); + if (ret < 0) { + return LTC_MP_LT; + } else if (ret > 0) { + return LTC_MP_GT; + } else { + return LTC_MP_EQ; + } +} + +static int count_bits(void *a) +{ + LTC_ARGCHK(a != NULL); + return mpz_sizeinbase(a, 2); +} + +static int count_lsb_bits(void *a) +{ + LTC_ARGCHK(a != NULL); + return mpz_scan1(a, 0); +} + + +static int twoexpt(void *a, int n) +{ + LTC_ARGCHK(a != NULL); + mpz_set_ui(a, 0); + mpz_setbit(a, n); + return CRYPT_OK; +} + +/* ---- conversions ---- */ + +static const char rmap[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/"; + +/* read ascii string */ +static int read_radix(void *a, const char *b, int radix) +{ + int ret; + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + if (radix == 64) { + /* Sadly, GMP only supports radixes up to 62, but we need 64. + * So, although this is not the most elegant or efficient way, + * let's just convert the base 64 string (6 bits per digit) to + * an octal string (3 bits per digit) that's twice as long. */ + char c, *tmp, *q; + const char *p; + int i; + tmp = XMALLOC (1 + 2 * XSTRLEN (b)); + if (tmp == NULL) { + return CRYPT_MEM; + } + p = b; + q = tmp; + while ((c = *p++) != 0) { + for (i = 0; i < 64; i++) { + if (c == rmap[i]) + break; + } + if (i == 64) { + XFREE (tmp); + /* printf ("c = '%c'\n", c); */ + return CRYPT_ERROR; + } + *q++ = '0' + (i / 8); + *q++ = '0' + (i % 8); + } + *q = 0; + ret = mpz_set_str(a, tmp, 8); + /* printf ("ret = %d for '%s'\n", ret, tmp); */ + XFREE (tmp); + } else { + ret = mpz_set_str(a, b, radix); + } + return (ret == 0 ? CRYPT_OK : CRYPT_ERROR); +} + +/* write one */ +static int write_radix(void *a, char *b, int radix) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + if (radix >= 11 && radix <= 36) + /* If radix is positive, GMP uses lowercase, and if negative, uppercase. + * We want it to use uppercase, to match the test vectors (presumably + * generated with LibTomMath). */ + radix = -radix; + mpz_get_str(b, radix, a); + return CRYPT_OK; +} + +/* get size as unsigned char string */ +static unsigned long unsigned_size(void *a) +{ + unsigned long t; + LTC_ARGCHK(a != NULL); + t = mpz_sizeinbase(a, 2); + if (mpz_cmp_ui(((__mpz_struct *)a), 0) == 0) return 0; + return (t>>3) + ((t&7)?1:0); +} + +/* store */ +static int unsigned_write(void *a, unsigned char *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + mpz_export(b, NULL, 1, 1, 1, 0, ((__mpz_struct*)a)); + return CRYPT_OK; +} + +/* read */ +static int unsigned_read(void *a, unsigned char *b, unsigned long len) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + mpz_import(a, len, 1, 1, 1, 0, b); + return CRYPT_OK; +} + +/* add */ +static int add(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_add(c, a, b); + return CRYPT_OK; +} + +static int addi(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + mpz_add_ui(c, a, b); + return CRYPT_OK; +} + +/* sub */ +static int sub(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_sub(c, a, b); + return CRYPT_OK; +} + +static int subi(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + mpz_sub_ui(c, a, b); + return CRYPT_OK; +} + +/* mul */ +static int mul(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_mul(c, a, b); + return CRYPT_OK; +} + +static int muli(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + mpz_mul_ui(c, a, b); + return CRYPT_OK; +} + +/* sqr */ +static int sqr(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + mpz_mul(b, a, a); + return CRYPT_OK; +} + +/* sqrtmod_prime */ +static int sqrtmod_prime(void *n, void *prime, void *ret) +{ + int res, legendre, i; + mpz_t t1, C, Q, S, Z, M, T, R, two; + + LTC_ARGCHK(n != NULL); + LTC_ARGCHK(prime != NULL); + LTC_ARGCHK(ret != NULL); + + /* first handle the simple cases */ + if (mpz_cmp_ui(((__mpz_struct *)n), 0) == 0) { + mpz_set_ui(ret, 0); + return CRYPT_OK; + } + if (mpz_cmp_ui(((__mpz_struct *)prime), 2) == 0) return CRYPT_ERROR; /* prime must be odd */ + legendre = mpz_legendre(n, prime); + if (legendre == -1) return CRYPT_ERROR; /* quadratic non-residue mod prime */ + + mpz_init(t1); mpz_init(C); mpz_init(Q); + mpz_init(S); mpz_init(Z); mpz_init(M); + mpz_init(T); mpz_init(R); mpz_init(two); + + /* SPECIAL CASE: if prime mod 4 == 3 + * compute directly: res = n^(prime+1)/4 mod prime + * Handbook of Applied Cryptography algorithm 3.36 + */ + i = mpz_mod_ui(t1, prime, 4); /* t1 is ignored here */ + if (i == 3) { + mpz_add_ui(t1, prime, 1); + mpz_fdiv_q_2exp(t1, t1, 2); + mpz_powm(ret, n, t1, prime); + res = CRYPT_OK; + goto cleanup; + } + + /* NOW: Tonelli-Shanks algorithm */ + + /* factor out powers of 2 from prime-1, defining Q and S as: prime-1 = Q*2^S */ + mpz_set(Q, prime); + mpz_sub_ui(Q, Q, 1); + /* Q = prime - 1 */ + mpz_set_ui(S, 0); + /* S = 0 */ + while (mpz_even_p(Q)) { + mpz_fdiv_q_2exp(Q, Q, 1); + /* Q = Q / 2 */ + mpz_add_ui(S, S, 1); + /* S = S + 1 */ + } + + /* find a Z such that the Legendre symbol (Z|prime) == -1 */ + mpz_set_ui(Z, 2); + /* Z = 2 */ + while(1) { + legendre = mpz_legendre(Z, prime); + if (legendre == -1) break; + mpz_add_ui(Z, Z, 1); + /* Z = Z + 1 */ + } + + mpz_powm(C, Z, Q, prime); + /* C = Z ^ Q mod prime */ + mpz_add_ui(t1, Q, 1); + mpz_fdiv_q_2exp(t1, t1, 1); + /* t1 = (Q + 1) / 2 */ + mpz_powm(R, n, t1, prime); + /* R = n ^ ((Q + 1) / 2) mod prime */ + mpz_powm(T, n, Q, prime); + /* T = n ^ Q mod prime */ + mpz_set(M, S); + /* M = S */ + mpz_set_ui(two, 2); + + while (1) { + mpz_set(t1, T); + i = 0; + while (1) { + if (mpz_cmp_ui(((__mpz_struct *)t1), 1) == 0) break; + mpz_powm(t1, t1, two, prime); + i++; + } + if (i == 0) { + mpz_set(ret, R); + res = CRYPT_OK; + goto cleanup; + } + mpz_sub_ui(t1, M, i); + mpz_sub_ui(t1, t1, 1); + mpz_powm(t1, two, t1, prime); + /* t1 = 2 ^ (M - i - 1) */ + mpz_powm(t1, C, t1, prime); + /* t1 = C ^ (2 ^ (M - i - 1)) mod prime */ + mpz_mul(C, t1, t1); + mpz_mod(C, C, prime); + /* C = (t1 * t1) mod prime */ + mpz_mul(R, R, t1); + mpz_mod(R, R, prime); + /* R = (R * t1) mod prime */ + mpz_mul(T, T, C); + mpz_mod(T, T, prime); + /* T = (T * C) mod prime */ + mpz_set_ui(M, i); + /* M = i */ + } + +cleanup: + mpz_clear(t1); mpz_clear(C); mpz_clear(Q); + mpz_clear(S); mpz_clear(Z); mpz_clear(M); + mpz_clear(T); mpz_clear(R); mpz_clear(two); + return res; +} + +/* div */ +static int divide(void *a, void *b, void *c, void *d) +{ + mpz_t tmp; + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + if (c != NULL) { + mpz_init(tmp); + mpz_divexact(tmp, a, b); + } + if (d != NULL) { + mpz_mod(d, a, b); + } + if (c != NULL) { + mpz_set(c, tmp); + mpz_clear(tmp); + } + return CRYPT_OK; +} + +static int div_2(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + mpz_divexact_ui(b, a, 2); + return CRYPT_OK; +} + +/* modi */ +static int modi(void *a, ltc_mp_digit b, ltc_mp_digit *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + + *c = mpz_fdiv_ui(a, b); + return CRYPT_OK; +} + +/* gcd */ +static int gcd(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_gcd(c, a, b); + return CRYPT_OK; +} + +/* lcm */ +static int lcm(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_lcm(c, a, b); + return CRYPT_OK; +} + +static int addmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + mpz_add(d, a, b); + mpz_mod(d, d, c); + return CRYPT_OK; +} + +static int submod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + mpz_sub(d, a, b); + mpz_mod(d, d, c); + return CRYPT_OK; +} + +static int mulmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + mpz_mul(d, a, b); + mpz_mod(d, d, c); + return CRYPT_OK; +} + +static int sqrmod(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_mul(c, a, a); + mpz_mod(c, c, b); + return CRYPT_OK; +} + +/* invmod */ +static int invmod(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_invert(c, a, b); + return CRYPT_OK; +} + +/* setup */ +static int montgomery_setup(void *a, void **b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + *b = (void *)1; + return CRYPT_OK; +} + +/* get normalization value */ +static int montgomery_normalization(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + mpz_set_ui(a, 1); + return CRYPT_OK; +} + +/* reduce */ +static int montgomery_reduce(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + mpz_mod(a, a, b); + return CRYPT_OK; +} + +/* clean up */ +static void montgomery_deinit(void *a) +{ + LTC_UNUSED_PARAM(a); +} + +static int exptmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + mpz_powm(d, a, b, c); + return CRYPT_OK; +} + +static int isprime(void *a, int b, int *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + if (b == 0) { + b = LTC_MILLER_RABIN_REPS; + } /* if */ + *c = mpz_probab_prime_p(a, b) > 0 ? LTC_MP_YES : LTC_MP_NO; + return CRYPT_OK; +} + +static int set_rand(void *a, int size) +{ + LTC_ARGCHK(a != NULL); + mpz_random(a, size); + return CRYPT_OK; +} + +const ltc_math_descriptor gmp_desc = { + "GNU MP", + sizeof(mp_limb_t) * CHAR_BIT - GMP_NAIL_BITS, + + &init, + &init_copy, + &deinit, + + &neg, + ©, + + &set_int, + &get_int, + &get_digit, + &get_digit_count, + &compare, + &compare_d, + &count_bits, + &count_lsb_bits, + &twoexpt, + + &read_radix, + &write_radix, + &unsigned_size, + &unsigned_write, + &unsigned_read, + + &add, + &addi, + &sub, + &subi, + &mul, + &muli, + &sqr, + &sqrtmod_prime, + ÷, + &div_2, + &modi, + &gcd, + &lcm, + + &mulmod, + &sqrmod, + &invmod, + + &montgomery_setup, + &montgomery_normalization, + &montgomery_reduce, + &montgomery_deinit, + + &exptmod, + &isprime, + +#ifdef LTC_MECC +#ifdef LTC_MECC_FP + <c_ecc_fp_mulmod, +#else + <c_ecc_mulmod, +#endif /* LTC_MECC_FP */ + <c_ecc_projective_add_point, + <c_ecc_projective_dbl_point, + <c_ecc_map, +#ifdef LTC_ECC_SHAMIR +#ifdef LTC_MECC_FP + <c_ecc_fp_mul2add, +#else + <c_ecc_mul2add, +#endif /* LTC_MECC_FP */ +#else + NULL, +#endif /* LTC_ECC_SHAMIR */ +#else + NULL, NULL, NULL, NULL, NULL, +#endif /* LTC_MECC */ + +#ifdef LTC_MRSA + &rsa_make_key, + &rsa_exptmod, +#else + NULL, NULL, +#endif + &addmod, + &submod, + + &set_rand, + +}; + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/math/ltm_desc.c b/optee_os/core/lib/libtomcrypt/src/math/ltm_desc.c new file mode 100644 index 0000000..b49b30e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/ltm_desc.c @@ -0,0 +1,560 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#define DESC_DEF_ONLY +#include "tomcrypt_private.h" + +#ifdef LTM_DESC + +#include +#if !defined(PRIVATE_MP_WARRAY) && !defined(BN_MP_PRIME_IS_PRIME_C) +#include +#endif + +static const struct { + mp_err mpi_code; + int ltc_code; +} mpi_to_ltc_codes[] = { + { MP_OKAY , CRYPT_OK}, + { MP_MEM , CRYPT_MEM}, + { MP_VAL , CRYPT_INVALID_ARG}, +#if defined(MP_BUF) || defined(MP_USE_ENUMS) + { MP_ITER , CRYPT_INVALID_PACKET}, + { MP_BUF , CRYPT_BUFFER_OVERFLOW}, +#endif +}; + +/** + Convert a MPI error to a LTC error (Possibly the most powerful function ever! Oh wait... no) + @param err The error to convert + @return The equivalent LTC error code or CRYPT_ERROR if none found +*/ +static int mpi_to_ltc_error(mp_err err) +{ + size_t x; + + for (x = 0; x < sizeof(mpi_to_ltc_codes)/sizeof(mpi_to_ltc_codes[0]); x++) { + if (err == mpi_to_ltc_codes[x].mpi_code) { + return mpi_to_ltc_codes[x].ltc_code; + } + } + return CRYPT_ERROR; +} + +static int init_mpi(void **a) +{ + LTC_ARGCHK(a != NULL); + + *a = XCALLOC(1, sizeof(mp_int)); + if (*a == NULL) { + return CRYPT_MEM; + } else { + return CRYPT_OK; + } +} + +static int init(void **a) +{ + int err; + + LTC_ARGCHK(a != NULL); + + if ((err = init_mpi(a)) != CRYPT_OK) { + return err; + } + if ((err = mpi_to_ltc_error(mp_init(*a))) != CRYPT_OK) { + XFREE(*a); + } + return err; +} + +static void deinit(void *a) +{ + LTC_ARGCHKVD(a != NULL); + mp_clear(a); + XFREE(a); +} + +static int neg(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return mpi_to_ltc_error(mp_neg(a, b)); +} + +static int copy(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return mpi_to_ltc_error(mp_copy(a, b)); +} + +static int init_copy(void **a, void *b) +{ + int err; + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + if ((err = init_mpi(a)) != CRYPT_OK) return err; + return mpi_to_ltc_error(mp_init_copy(*a, b)); +} + +/* ---- trivial ---- */ +static int set_int(void *a, ltc_mp_digit b) +{ + LTC_ARGCHK(a != NULL); +#ifdef BN_MP_SET_INT_C + return mpi_to_ltc_error(mp_set_int(a, b)); +#else + mp_set_u32(a, b); + return CRYPT_OK; +#endif +} + +static unsigned long get_int(void *a) +{ + LTC_ARGCHK(a != NULL); +#ifdef BN_MP_GET_INT_C + return mp_get_int(a); +#else + return mp_get_ul(a); +#endif +} + +static ltc_mp_digit get_digit(void *a, int n) +{ + mp_int *A; + LTC_ARGCHK(a != NULL); + A = a; + return (n >= A->used || n < 0) ? 0 : A->dp[n]; +} + +static int get_digit_count(void *a) +{ + mp_int *A; + LTC_ARGCHK(a != NULL); + A = a; + return A->used; +} + +static int compare(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + switch (mp_cmp(a, b)) { + case MP_LT: return LTC_MP_LT; + case MP_EQ: return LTC_MP_EQ; + case MP_GT: return LTC_MP_GT; + default: return 0; + } +} + +static int compare_d(void *a, ltc_mp_digit b) +{ + LTC_ARGCHK(a != NULL); + switch (mp_cmp_d(a, b)) { + case MP_LT: return LTC_MP_LT; + case MP_EQ: return LTC_MP_EQ; + case MP_GT: return LTC_MP_GT; + default: return 0; + } +} + +static int count_bits(void *a) +{ + LTC_ARGCHK(a != NULL); + return mp_count_bits(a); +} + +static int count_lsb_bits(void *a) +{ + LTC_ARGCHK(a != NULL); + return mp_cnt_lsb(a); +} + + +static int twoexpt(void *a, int n) +{ + LTC_ARGCHK(a != NULL); + return mpi_to_ltc_error(mp_2expt(a, n)); +} + +/* ---- conversions ---- */ + +/* read ascii string */ +static int read_radix(void *a, const char *b, int radix) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return mpi_to_ltc_error(mp_read_radix(a, b, radix)); +} + +/* write one */ +static int write_radix(void *a, char *b, int radix) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); +#ifdef BN_MP_TORADIX_C + return mpi_to_ltc_error(mp_toradix(a, b, radix)); +#else + return mpi_to_ltc_error(mp_to_radix(a, b, SIZE_MAX, NULL, radix)); +#endif +} + +/* get size as unsigned char string */ +static unsigned long unsigned_size(void *a) +{ + LTC_ARGCHK(a != NULL); +#ifdef BN_MP_UNSIGNED_BIN_SIZE_C + return mp_unsigned_bin_size(a); +#else + return (unsigned long)mp_ubin_size(a); +#endif +} + +/* store */ +static int unsigned_write(void *a, unsigned char *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); +#ifdef BN_MP_TO_UNSIGNED_BIN_C + return mpi_to_ltc_error(mp_to_unsigned_bin(a, b)); +#else + return mpi_to_ltc_error(mp_to_ubin(a, b, SIZE_MAX, NULL)); +#endif +} + +/* read */ +static int unsigned_read(void *a, unsigned char *b, unsigned long len) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); +#ifdef BN_MP_READ_UNSIGNED_BIN_C + return mpi_to_ltc_error(mp_read_unsigned_bin(a, b, len)); +#else + return mpi_to_ltc_error(mp_from_ubin(a, b, (size_t)len)); +#endif +} + +/* add */ +static int add(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_add(a, b, c)); +} + +static int addi(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_add_d(a, b, c)); +} + +/* sub */ +static int sub(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_sub(a, b, c)); +} + +static int subi(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_sub_d(a, b, c)); +} + +/* mul */ +static int mul(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_mul(a, b, c)); +} + +static int muli(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_mul_d(a, b, c)); +} + +/* sqr */ +static int sqr(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return mpi_to_ltc_error(mp_sqr(a, b)); +} + +/* sqrtmod_prime */ +static int sqrtmod_prime(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_sqrtmod_prime(a, b, c)); +} + +/* div */ +static int divide(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return mpi_to_ltc_error(mp_div(a, b, c, d)); +} + +static int div_2(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return mpi_to_ltc_error(mp_div_2(a, b)); +} + +/* modi */ +static int modi(void *a, ltc_mp_digit b, ltc_mp_digit *c) +{ + mp_digit tmp; + int err; + + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + + if ((err = mpi_to_ltc_error(mp_mod_d(a, b, &tmp))) != CRYPT_OK) { + return err; + } + *c = tmp; + return CRYPT_OK; +} + +/* gcd */ +static int gcd(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_gcd(a, b, c)); +} + +/* lcm */ +static int lcm(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_lcm(a, b, c)); +} + +static int addmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return mpi_to_ltc_error(mp_addmod(a,b,c,d)); +} + +static int submod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return mpi_to_ltc_error(mp_submod(a,b,c,d)); +} + +static int mulmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return mpi_to_ltc_error(mp_mulmod(a,b,c,d)); +} + +static int sqrmod(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_sqrmod(a,b,c)); +} + +/* invmod */ +static int invmod(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_invmod(a, b, c)); +} + +/* setup */ +static int montgomery_setup(void *a, void **b) +{ + int err; + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + *b = XCALLOC(1, sizeof(mp_digit)); + if (*b == NULL) { + return CRYPT_MEM; + } + if ((err = mpi_to_ltc_error(mp_montgomery_setup(a, (mp_digit *)*b))) != CRYPT_OK) { + XFREE(*b); + } + return err; +} + +/* get normalization value */ +static int montgomery_normalization(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return mpi_to_ltc_error(mp_montgomery_calc_normalization(a, b)); +} + +/* reduce */ +static int montgomery_reduce(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return mpi_to_ltc_error(mp_montgomery_reduce(a, b, *((mp_digit *)c))); +} + +/* clean up */ +static void montgomery_deinit(void *a) +{ + XFREE(a); +} + +static int exptmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return mpi_to_ltc_error(mp_exptmod(a,b,c,d)); +} + +static int isprime(void *a, int b, int *c) +{ + int err; +#if defined(PRIVATE_MP_WARRAY) || defined(BN_MP_PRIME_IS_PRIME_C) + int res; +#else + bool res; +#endif + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + b = mp_prime_rabin_miller_trials(mp_count_bits(a)); + err = mpi_to_ltc_error(mp_prime_is_prime(a, b, &res)); + *c = res ? LTC_MP_YES : LTC_MP_NO; + return err; +} + +static int set_rand(void *a, int size) +{ + LTC_ARGCHK(a != NULL); + return mpi_to_ltc_error(mp_rand(a, size)); +} + +#ifndef MP_DIGIT_BIT +#define MP_DIGIT_BIT DIGIT_BIT +#endif + +const ltc_math_descriptor ltm_desc = { + + "LibTomMath", + (int)MP_DIGIT_BIT, + + &init, + &init_copy, + &deinit, + + &neg, + ©, + + &set_int, + &get_int, + &get_digit, + &get_digit_count, + &compare, + &compare_d, + &count_bits, + &count_lsb_bits, + &twoexpt, + + &read_radix, + &write_radix, + &unsigned_size, + &unsigned_write, + &unsigned_read, + + &add, + &addi, + &sub, + &subi, + &mul, + &muli, + &sqr, + &sqrtmod_prime, + ÷, + &div_2, + &modi, + &gcd, + &lcm, + + &mulmod, + &sqrmod, + &invmod, + + &montgomery_setup, + &montgomery_normalization, + &montgomery_reduce, + &montgomery_deinit, + + &exptmod, + &isprime, + +#ifdef LTC_MECC +#ifdef LTC_MECC_FP + <c_ecc_fp_mulmod, +#else + <c_ecc_mulmod, +#endif + <c_ecc_projective_add_point, + <c_ecc_projective_dbl_point, + <c_ecc_map, +#ifdef LTC_ECC_SHAMIR +#ifdef LTC_MECC_FP + <c_ecc_fp_mul2add, +#else + <c_ecc_mul2add, +#endif /* LTC_MECC_FP */ +#else + NULL, +#endif /* LTC_ECC_SHAMIR */ +#else + NULL, NULL, NULL, NULL, NULL, +#endif /* LTC_MECC */ + +#ifdef LTC_MRSA + &rsa_make_key, + &rsa_exptmod, +#else + NULL, NULL, +#endif + &addmod, + &submod, + + &set_rand, + +}; + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/math/multi.c b/optee_os/core/lib/libtomcrypt/src/math/multi.c new file mode 100644 index 0000000..cf2ea4d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/multi.c @@ -0,0 +1,94 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_MPI +#include + +int ltc_init_multi(void **a, ...) +{ + void **cur = a; + int np = 0; + va_list args; + + va_start(args, a); + while (cur != NULL) { + if (mp_init(cur) != CRYPT_OK) { + /* failed */ + va_list clean_list; + + va_start(clean_list, a); + cur = a; + while (np--) { + mp_clear(*cur); + cur = va_arg(clean_list, void**); + } + va_end(clean_list); + va_end(args); + return CRYPT_MEM; + } + ++np; + cur = va_arg(args, void**); + } + va_end(args); + return CRYPT_OK; +} + +int ltc_init_multi_size(int size_bits, void **a, ...) +{ + void **cur = a; + int np = 0; + va_list args; + + va_start(args, a); + while (cur != NULL) { + if (mp_init_size(size_bits, cur) != CRYPT_OK) { + /* failed */ + va_list clean_list; + + va_start(clean_list, a); + cur = a; + while (np--) { + mp_clear(*cur); + cur = va_arg(clean_list, void**); + } + va_end(clean_list); + return CRYPT_MEM; + } + ++np; + cur = va_arg(args, void**); + } + va_end(args); + return CRYPT_OK; +} + +void ltc_deinit_multi(void *a, ...) +{ + void *cur = a; + va_list args; + + va_start(args, a); + while (cur != NULL) { + mp_clear(cur); + cur = va_arg(args, void *); + } + va_end(args); +} + +void ltc_cleanup_multi(void **a, ...) +{ + void **cur = a; + va_list args; + + va_start(args, a); + while (cur != NULL) { + if (*cur != NULL) { + mp_clear(*cur); + *cur = NULL; + } + cur = va_arg(args, void**); + } + va_end(args); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/math/radix_to_bin.c b/optee_os/core/lib/libtomcrypt/src/math/radix_to_bin.c new file mode 100644 index 0000000..5c17f0d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/radix_to_bin.c @@ -0,0 +1,52 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file radix_to_bin.c + Convert data from a specific radix to binary. + Steffen Jaeckel +*/ + +/** + Convert data from a specific radix to binary + + The default MPI descriptors #ltm_desc, #tfm_desc and #gmp_desc + have the following restrictions on parameters: + + \p in - NUL-terminated char buffer + + \p radix - 2..64 + + @param in The input + @param radix The radix of the input + @param out The output buffer + @param len [in/out] The length of the output buffer + + @return CRYPT_OK on success. +*/ +int radix_to_bin(const void *in, int radix, void *out, unsigned long *len) +{ + unsigned long l; + void* mpi; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(len != NULL); + + if ((err = mp_init(&mpi)) != CRYPT_OK) return err; + if ((err = mp_read_radix(mpi, in, radix)) != CRYPT_OK) goto LBL_ERR; + + if ((l = mp_unsigned_bin_size(mpi)) > *len) { + *len = l; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + *len = l; + + if ((err = mp_to_unsigned_bin(mpi, out)) != CRYPT_OK) goto LBL_ERR; + +LBL_ERR: + mp_clear(mpi); + return err; +} diff --git a/optee_os/core/lib/libtomcrypt/src/math/rand_bn.c b/optee_os/core/lib/libtomcrypt/src/math/rand_bn.c new file mode 100644 index 0000000..137586c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/rand_bn.c @@ -0,0 +1,65 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#if defined(LTC_MDSA) || defined(LTC_MECC) +/** + Generate a random number N with given bitlength (note: MSB can be 0) +*/ + +int rand_bn_bits(void *N, int bits, prng_state *prng, int wprng) +{ + int res, bytes; + unsigned char *buf, mask; + + LTC_ARGCHK(N != NULL); + LTC_ARGCHK(bits > 1); + + /* check PRNG */ + if ((res = prng_is_valid(wprng)) != CRYPT_OK) return res; + + bytes = (bits+7) >> 3; + mask = 0xff >> (bits % 8 == 0 ? 0 : 8 - bits % 8); + + /* allocate buffer */ + if ((buf = XCALLOC(1, bytes)) == NULL) return CRYPT_MEM; + + /* generate random bytes */ + if (prng_descriptor[wprng]->read(buf, bytes, prng) != (unsigned long)bytes) { + res = CRYPT_ERROR_READPRNG; + goto cleanup; + } + /* mask bits */ + buf[0] &= mask; + /* load value */ + if ((res = mp_read_unsigned_bin(N, buf, bytes)) != CRYPT_OK) goto cleanup; + + res = CRYPT_OK; + +cleanup: +#ifdef LTC_CLEAN_STACK + zeromem(buf, bytes); +#endif + XFREE(buf); + return res; +} + +/** + Generate a random number N in a range: 1 <= N < limit +*/ +int rand_bn_upto(void *N, void *limit, prng_state *prng, int wprng) +{ + int res, bits; + + LTC_ARGCHK(N != NULL); + LTC_ARGCHK(limit != NULL); + + bits = mp_count_bits(limit); + do { + res = rand_bn_bits(N, bits, prng, wprng); + if (res != CRYPT_OK) return res; + } while (mp_cmp_d(N, 0) != LTC_MP_GT || mp_cmp(N, limit) != LTC_MP_LT); + + return CRYPT_OK; +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/math/rand_prime.c b/optee_os/core/lib/libtomcrypt/src/math/rand_prime.c new file mode 100644 index 0000000..b50ac8c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/rand_prime.c @@ -0,0 +1,78 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#if defined(LTC_MRSA) || (!defined(LTC_NO_MATH) && !defined(LTC_NO_PRNGS)) + +/** + @file rand_prime.c + Generate a random prime, Tom St Denis +*/ + +#define USE_BBS 1 + +int rand_prime(void *N, long len, prng_state *prng, int wprng) +{ + int err, res, type; + unsigned char *buf; + + LTC_ARGCHK(N != NULL); + + /* get type */ + if (len < 0) { + type = USE_BBS; + len = -len; + } else { + type = 0; + } + + /* allow sizes between 2 and 512 bytes for a prime size */ + if (len < 2 || len > 512) { + return CRYPT_INVALID_PRIME_SIZE; + } + + /* valid PRNG? Better be! */ + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + + /* allocate buffer to work with */ + buf = XCALLOC(1, len); + if (buf == NULL) { + return CRYPT_MEM; + } + + do { + /* generate value */ + if (prng_descriptor[wprng]->read(buf, len, prng) != (unsigned long)len) { + XFREE(buf); + return CRYPT_ERROR_READPRNG; + } + + /* munge bits */ + buf[0] |= 0x80 | 0x40; + buf[len-1] |= 0x01 | ((type & USE_BBS) ? 0x02 : 0x00); + + /* load value */ + if ((err = mp_read_unsigned_bin(N, buf, len)) != CRYPT_OK) { + XFREE(buf); + return err; + } + + /* test */ + if ((err = mp_prime_is_prime(N, LTC_MILLER_RABIN_REPS, &res)) != CRYPT_OK) { + XFREE(buf); + return err; + } + } while (res == LTC_MP_NO); + +#ifdef LTC_CLEAN_STACK + zeromem(buf, len); +#endif + + XFREE(buf); + return CRYPT_OK; +} + +#endif /* LTC_NO_MATH */ + diff --git a/optee_os/core/lib/libtomcrypt/src/math/sub.mk b/optee_os/core/lib/libtomcrypt/src/math/sub.mk new file mode 100644 index 0000000..4a38a32 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/sub.mk @@ -0,0 +1,4 @@ +subdirs-y += fp +srcs-y += multi.c +srcs-y += rand_prime.c +srcs-y += rand_bn.c diff --git a/optee_os/core/lib/libtomcrypt/src/math/tfm_desc.c b/optee_os/core/lib/libtomcrypt/src/math/tfm_desc.c new file mode 100644 index 0000000..cee753a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/math/tfm_desc.c @@ -0,0 +1,857 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#define DESC_DEF_ONLY +#include "tomcrypt_private.h" + +#ifdef TFM_DESC + +#include + +static const struct { + int tfm_code, ltc_code; +} tfm_to_ltc_codes[] = { + { FP_OKAY , CRYPT_OK}, + { FP_MEM , CRYPT_MEM}, + { FP_VAL , CRYPT_INVALID_ARG}, +}; + +/** + Convert a tfm error to a LTC error (Possibly the most powerful function ever! Oh wait... no) + @param err The error to convert + @return The equivalent LTC error code or CRYPT_ERROR if none found +*/ +static int tfm_to_ltc_error(int err) +{ + int x; + + for (x = 0; x < (int)(sizeof(tfm_to_ltc_codes)/sizeof(tfm_to_ltc_codes[0])); x++) { + if (err == tfm_to_ltc_codes[x].tfm_code) { + return tfm_to_ltc_codes[x].ltc_code; + } + } + return CRYPT_ERROR; +} + +static int init(void **a) +{ + LTC_ARGCHK(a != NULL); + + *a = XCALLOC(1, sizeof(fp_int)); + if (*a == NULL) { + return CRYPT_MEM; + } + fp_init(*a); + return CRYPT_OK; +} + +static void deinit(void *a) +{ + LTC_ARGCHKVD(a != NULL); + XFREE(a); +} + +static int neg(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + fp_neg(((fp_int*)a), ((fp_int*)b)); + return CRYPT_OK; +} + +static int copy(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + fp_copy(a, b); + return CRYPT_OK; +} + +static int init_copy(void **a, void *b) +{ + if (init(a) != CRYPT_OK) { + return CRYPT_MEM; + } + return copy(b, *a); +} + +/* ---- trivial ---- */ +static int set_int(void *a, ltc_mp_digit b) +{ + LTC_ARGCHK(a != NULL); + fp_set(a, b); + return CRYPT_OK; +} + +static unsigned long get_int(void *a) +{ + fp_int *A; + LTC_ARGCHK(a != NULL); + A = a; + return A->used > 0 ? A->dp[0] : 0; +} + +static ltc_mp_digit get_digit(void *a, int n) +{ + fp_int *A; + LTC_ARGCHK(a != NULL); + A = a; + return (n >= A->used || n < 0) ? 0 : A->dp[n]; +} + +static int get_digit_count(void *a) +{ + fp_int *A; + LTC_ARGCHK(a != NULL); + A = a; + return A->used; +} + +static int compare(void *a, void *b) +{ + int ret; + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + ret = fp_cmp(a, b); + switch (ret) { + case FP_LT: return LTC_MP_LT; + case FP_EQ: return LTC_MP_EQ; + case FP_GT: return LTC_MP_GT; + } + return 0; +} + +static int compare_d(void *a, ltc_mp_digit b) +{ + int ret; + LTC_ARGCHK(a != NULL); + ret = fp_cmp_d(a, b); + switch (ret) { + case FP_LT: return LTC_MP_LT; + case FP_EQ: return LTC_MP_EQ; + case FP_GT: return LTC_MP_GT; + } + return 0; +} + +static int count_bits(void *a) +{ + LTC_ARGCHK(a != NULL); + return fp_count_bits(a); +} + +static int count_lsb_bits(void *a) +{ + LTC_ARGCHK(a != NULL); + return fp_cnt_lsb(a); +} + +static int twoexpt(void *a, int n) +{ + LTC_ARGCHK(a != NULL); + fp_2expt(a, n); + return CRYPT_OK; +} + +/* ---- conversions ---- */ + +/* read ascii string */ +static int read_radix(void *a, const char *b, int radix) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return tfm_to_ltc_error(fp_read_radix(a, (char *)b, radix)); +} + +/* write one */ +static int write_radix(void *a, char *b, int radix) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return tfm_to_ltc_error(fp_toradix(a, b, radix)); +} + +/* get size as unsigned char string */ +static unsigned long unsigned_size(void *a) +{ + LTC_ARGCHK(a != NULL); + return fp_unsigned_bin_size(a); +} + +/* store */ +static int unsigned_write(void *a, unsigned char *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + fp_to_unsigned_bin(a, b); + return CRYPT_OK; +} + +/* read */ +static int unsigned_read(void *a, unsigned char *b, unsigned long len) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + fp_read_unsigned_bin(a, b, len); + return CRYPT_OK; +} + +/* add */ +static int add(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + fp_add(a, b, c); + return CRYPT_OK; +} + +static int addi(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + fp_add_d(a, b, c); + return CRYPT_OK; +} + +/* sub */ +static int sub(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + fp_sub(a, b, c); + return CRYPT_OK; +} + +static int subi(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + fp_sub_d(a, b, c); + return CRYPT_OK; +} + +/* mul */ +static int mul(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + fp_mul(a, b, c); + return CRYPT_OK; +} + +static int muli(void *a, ltc_mp_digit b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + fp_mul_d(a, b, c); + return CRYPT_OK; +} + +/* sqr */ +static int sqr(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + fp_sqr(a, b); + return CRYPT_OK; +} + +/* sqrtmod_prime - NOT SUPPORTED */ + +/* div */ +static int divide(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + return tfm_to_ltc_error(fp_div(a, b, c, d)); +} + +static int div_2(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + fp_div_2(a, b); + return CRYPT_OK; +} + +/* modi */ +static int modi(void *a, ltc_mp_digit b, ltc_mp_digit *c) +{ + fp_digit tmp; + int err; + + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + + if ((err = tfm_to_ltc_error(fp_mod_d(a, b, &tmp))) != CRYPT_OK) { + return err; + } + *c = tmp; + return CRYPT_OK; +} + +/* gcd */ +static int gcd(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + fp_gcd(a, b, c); + return CRYPT_OK; +} + +/* lcm */ +static int lcm(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + fp_lcm(a, b, c); + return CRYPT_OK; +} + +static int addmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return tfm_to_ltc_error(fp_addmod(a,b,c,d)); +} + +static int submod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return tfm_to_ltc_error(fp_submod(a,b,c,d)); +} + +static int mulmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return tfm_to_ltc_error(fp_mulmod(a,b,c,d)); +} + +static int sqrmod(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return tfm_to_ltc_error(fp_sqrmod(a,b,c)); +} + +/* invmod */ +static int invmod(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + return tfm_to_ltc_error(fp_invmod(a, b, c)); +} + +/* setup */ +static int montgomery_setup(void *a, void **b) +{ + int err; + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + *b = XCALLOC(1, sizeof(fp_digit)); + if (*b == NULL) { + return CRYPT_MEM; + } + if ((err = tfm_to_ltc_error(fp_montgomery_setup(a, (fp_digit *)*b))) != CRYPT_OK) { + XFREE(*b); + } + return err; +} + +/* get normalization value */ +static int montgomery_normalization(void *a, void *b) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + fp_montgomery_calc_normalization(a, b); + return CRYPT_OK; +} + +/* reduce */ +static int montgomery_reduce(void *a, void *b, void *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + fp_montgomery_reduce(a, b, *((fp_digit *)c)); + return CRYPT_OK; +} + +/* clean up */ +static void montgomery_deinit(void *a) +{ + XFREE(a); +} + +static int exptmod(void *a, void *b, void *c, void *d) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(c != NULL); + LTC_ARGCHK(d != NULL); + return tfm_to_ltc_error(fp_exptmod(a,b,c,d)); +} + +static int isprime(void *a, int b, int *c) +{ + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(c != NULL); + if (b == 0) { + b = LTC_MILLER_RABIN_REPS; + } /* if */ + *c = (fp_isprime_ex(a, b) == FP_YES) ? LTC_MP_YES : LTC_MP_NO; + return CRYPT_OK; +} + +#if defined(LTC_MECC) && defined(LTC_MECC_ACCEL) + +static int tfm_ecc_projective_dbl_point(const ecc_point *P, ecc_point *R, void *ma, void *modulus, void *Mp) +{ + fp_int t1, t2; + fp_digit mp; + int err, inf; + + LTC_ARGCHK(P != NULL); + LTC_ARGCHK(R != NULL); + LTC_ARGCHK(modulus != NULL); + LTC_ARGCHK(Mp != NULL); + + mp = *((fp_digit*)Mp); + + fp_init(&t1); + fp_init(&t2); + + if (P != R) { + fp_copy(P->x, R->x); + fp_copy(P->y, R->y); + fp_copy(P->z, R->z); + } + + if ((err = ltc_ecc_is_point_at_infinity(P, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* if P is point at infinity >> Result = point at infinity */ + ltc_mp.set_int(R->x, 1); + ltc_mp.set_int(R->y, 1); + ltc_mp.set_int(R->z, 0); + return CRYPT_OK; + } + + /* t1 = Z * Z */ + fp_sqr(R->z, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + /* Z = Y * Z */ + fp_mul(R->z, R->y, R->z); + fp_montgomery_reduce(R->z, modulus, mp); + /* Z = 2Z */ + fp_add(R->z, R->z, R->z); + if (fp_cmp(R->z, modulus) != FP_LT) { + fp_sub(R->z, modulus, R->z); + } + + if (ma == NULL) { /* special case for curves with a == -3 (10% faster than general case) */ + /* T2 = X - T1 */ + fp_sub(R->x, &t1, &t2); + if (fp_cmp_d(&t2, 0) == LTC_MP_LT) { + fp_add(&t2, modulus, &t2); + } + /* T1 = X + T1 */ + fp_add(&t1, R->x, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + /* T2 = T1 * T2 */ + fp_mul(&t1, &t2, &t2); + fp_montgomery_reduce(&t2, modulus, mp); + /* T1 = 2T2 */ + fp_add(&t2, &t2, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + /* T1 = T1 + T2 */ + fp_add(&t1, &t2, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + } + else { + /* T2 = T1 * T1 */ + fp_sqr(&t1, &t2); + fp_montgomery_reduce(&t2, modulus, mp); + /* T1 = T2 * a */ + fp_mul(&t2, ma, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + /* T2 = X * X */ + fp_sqr(R->x, &t2); + fp_montgomery_reduce(&t2, modulus, mp); + /* T1 = T1 + T2 */ + fp_add(&t1, &t2, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + /* T1 = T1 + T2 */ + fp_add(&t1, &t2, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + /* T1 = T1 + T2 */ + fp_add(&t1, &t2, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + } + + /* Y = 2Y */ + fp_add(R->y, R->y, R->y); + if (fp_cmp(R->y, modulus) != FP_LT) { + fp_sub(R->y, modulus, R->y); + } + /* Y = Y * Y */ + fp_sqr(R->y, R->y); + fp_montgomery_reduce(R->y, modulus, mp); + /* T2 = Y * Y */ + fp_sqr(R->y, &t2); + fp_montgomery_reduce(&t2, modulus, mp); + /* T2 = T2/2 */ + if (fp_isodd(&t2)) { + fp_add(&t2, modulus, &t2); + } + fp_div_2(&t2, &t2); + /* Y = Y * X */ + fp_mul(R->y, R->x, R->y); + fp_montgomery_reduce(R->y, modulus, mp); + + /* X = T1 * T1 */ + fp_sqr(&t1, R->x); + fp_montgomery_reduce(R->x, modulus, mp); + /* X = X - Y */ + fp_sub(R->x, R->y, R->x); + if (fp_cmp_d(R->x, 0) == FP_LT) { + fp_add(R->x, modulus, R->x); + } + /* X = X - Y */ + fp_sub(R->x, R->y, R->x); + if (fp_cmp_d(R->x, 0) == FP_LT) { + fp_add(R->x, modulus, R->x); + } + + /* Y = Y - X */ + fp_sub(R->y, R->x, R->y); + if (fp_cmp_d(R->y, 0) == FP_LT) { + fp_add(R->y, modulus, R->y); + } + /* Y = Y * T1 */ + fp_mul(R->y, &t1, R->y); + fp_montgomery_reduce(R->y, modulus, mp); + /* Y = Y - T2 */ + fp_sub(R->y, &t2, R->y); + if (fp_cmp_d(R->y, 0) == FP_LT) { + fp_add(R->y, modulus, R->y); + } + + return CRYPT_OK; +} + +/** + Add two ECC points + @param P The point to add + @param Q The point to add + @param R [out] The destination of the double + @param modulus The modulus of the field the ECC curve is in + @param Mp The "b" value from montgomery_setup() + @return CRYPT_OK on success +*/ +static int tfm_ecc_projective_add_point(const ecc_point *P, const ecc_point *Q, ecc_point *R, void *ma, void *modulus, void *Mp) +{ + fp_int t1, t2, x, y, z; + fp_digit mp; + int err, inf; + + LTC_ARGCHK(P != NULL); + LTC_ARGCHK(Q != NULL); + LTC_ARGCHK(R != NULL); + LTC_ARGCHK(modulus != NULL); + LTC_ARGCHK(Mp != NULL); + + mp = *((fp_digit*)Mp); + + fp_init(&t1); + fp_init(&t2); + fp_init(&x); + fp_init(&y); + fp_init(&z); + + if ((err = ltc_ecc_is_point_at_infinity(P, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* P is point at infinity >> Result = Q */ + ltc_mp.copy(Q->x, R->x); + ltc_mp.copy(Q->y, R->y); + ltc_mp.copy(Q->z, R->z); + return CRYPT_OK; + } + + if ((err = ltc_ecc_is_point_at_infinity(Q, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* Q is point at infinity >> Result = P */ + ltc_mp.copy(P->x, R->x); + ltc_mp.copy(P->y, R->y); + ltc_mp.copy(P->z, R->z); + return CRYPT_OK; + } + + /* should we dbl instead? */ + fp_sub(modulus, Q->y, &t1); + if ( (fp_cmp(P->x, Q->x) == FP_EQ) && + (Q->z != NULL && fp_cmp(P->z, Q->z) == FP_EQ) && + (fp_cmp(P->y, Q->y) == FP_EQ || fp_cmp(P->y, &t1) == FP_EQ)) { + return tfm_ecc_projective_dbl_point(P, R, ma, modulus, Mp); + } + + fp_copy(P->x, &x); + fp_copy(P->y, &y); + fp_copy(P->z, &z); + + /* if Z is one then these are no-operations */ + if (Q->z != NULL) { + /* T1 = Z' * Z' */ + fp_sqr(Q->z, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + /* X = X * T1 */ + fp_mul(&t1, &x, &x); + fp_montgomery_reduce(&x, modulus, mp); + /* T1 = Z' * T1 */ + fp_mul(Q->z, &t1, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + /* Y = Y * T1 */ + fp_mul(&t1, &y, &y); + fp_montgomery_reduce(&y, modulus, mp); + } + + /* T1 = Z*Z */ + fp_sqr(&z, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + /* T2 = X' * T1 */ + fp_mul(Q->x, &t1, &t2); + fp_montgomery_reduce(&t2, modulus, mp); + /* T1 = Z * T1 */ + fp_mul(&z, &t1, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + /* T1 = Y' * T1 */ + fp_mul(Q->y, &t1, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + + /* Y = Y - T1 */ + fp_sub(&y, &t1, &y); + if (fp_cmp_d(&y, 0) == FP_LT) { + fp_add(&y, modulus, &y); + } + /* T1 = 2T1 */ + fp_add(&t1, &t1, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + /* T1 = Y + T1 */ + fp_add(&t1, &y, &t1); + if (fp_cmp(&t1, modulus) != FP_LT) { + fp_sub(&t1, modulus, &t1); + } + /* X = X - T2 */ + fp_sub(&x, &t2, &x); + if (fp_cmp_d(&x, 0) == FP_LT) { + fp_add(&x, modulus, &x); + } + /* T2 = 2T2 */ + fp_add(&t2, &t2, &t2); + if (fp_cmp(&t2, modulus) != FP_LT) { + fp_sub(&t2, modulus, &t2); + } + /* T2 = X + T2 */ + fp_add(&t2, &x, &t2); + if (fp_cmp(&t2, modulus) != FP_LT) { + fp_sub(&t2, modulus, &t2); + } + + /* if Z' != 1 */ + if (Q->z != NULL) { + /* Z = Z * Z' */ + fp_mul(&z, Q->z, &z); + fp_montgomery_reduce(&z, modulus, mp); + } + + /* Z = Z * X */ + fp_mul(&z, &x, &z); + fp_montgomery_reduce(&z, modulus, mp); + + /* T1 = T1 * X */ + fp_mul(&t1, &x, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + /* X = X * X */ + fp_sqr(&x, &x); + fp_montgomery_reduce(&x, modulus, mp); + /* T2 = T2 * x */ + fp_mul(&t2, &x, &t2); + fp_montgomery_reduce(&t2, modulus, mp); + /* T1 = T1 * X */ + fp_mul(&t1, &x, &t1); + fp_montgomery_reduce(&t1, modulus, mp); + + /* X = Y*Y */ + fp_sqr(&y, &x); + fp_montgomery_reduce(&x, modulus, mp); + /* X = X - T2 */ + fp_sub(&x, &t2, &x); + if (fp_cmp_d(&x, 0) == FP_LT) { + fp_add(&x, modulus, &x); + } + + /* T2 = T2 - X */ + fp_sub(&t2, &x, &t2); + if (fp_cmp_d(&t2, 0) == FP_LT) { + fp_add(&t2, modulus, &t2); + } + /* T2 = T2 - X */ + fp_sub(&t2, &x, &t2); + if (fp_cmp_d(&t2, 0) == FP_LT) { + fp_add(&t2, modulus, &t2); + } + /* T2 = T2 * Y */ + fp_mul(&t2, &y, &t2); + fp_montgomery_reduce(&t2, modulus, mp); + /* Y = T2 - T1 */ + fp_sub(&t2, &t1, &y); + if (fp_cmp_d(&y, 0) == FP_LT) { + fp_add(&y, modulus, &y); + } + /* Y = Y/2 */ + if (fp_isodd(&y)) { + fp_add(&y, modulus, &y); + } + fp_div_2(&y, &y); + + fp_copy(&x, R->x); + fp_copy(&y, R->y); + fp_copy(&z, R->z); + + return CRYPT_OK; +} + + +#endif + +static int set_rand(void *a, int size) +{ + LTC_ARGCHK(a != NULL); + fp_rand(a, size); + return CRYPT_OK; +} + +const ltc_math_descriptor tfm_desc = { + + "TomsFastMath", + (int)DIGIT_BIT, + + &init, + &init_copy, + &deinit, + + &neg, + ©, + + &set_int, + &get_int, + &get_digit, + &get_digit_count, + &compare, + &compare_d, + &count_bits, + &count_lsb_bits, + &twoexpt, + + &read_radix, + &write_radix, + &unsigned_size, + &unsigned_write, + &unsigned_read, + + &add, + &addi, + &sub, + &subi, + &mul, + &muli, + &sqr, + NULL, /* TODO: &sqrtmod_prime */ + ÷, + &div_2, + &modi, + &gcd, + &lcm, + + &mulmod, + &sqrmod, + &invmod, + + &montgomery_setup, + &montgomery_normalization, + &montgomery_reduce, + &montgomery_deinit, + + &exptmod, + &isprime, + +#ifdef LTC_MECC +#ifdef LTC_MECC_FP + <c_ecc_fp_mulmod, +#else + <c_ecc_mulmod, +#endif /* LTC_MECC_FP */ +#ifdef LTC_MECC_ACCEL + &tfm_ecc_projective_add_point, + &tfm_ecc_projective_dbl_point, +#else + <c_ecc_projective_add_point, + <c_ecc_projective_dbl_point, +#endif /* LTC_MECC_ACCEL */ + <c_ecc_map, +#ifdef LTC_ECC_SHAMIR +#ifdef LTC_MECC_FP + <c_ecc_fp_mul2add, +#else + <c_ecc_mul2add, +#endif /* LTC_MECC_FP */ +#else + NULL, +#endif /* LTC_ECC_SHAMIR */ +#else + NULL, NULL, NULL, NULL, NULL, +#endif /* LTC_MECC */ + +#ifdef LTC_MRSA + &rsa_make_key, + &rsa_exptmod, +#else + NULL, NULL, +#endif + &addmod, + &submod, + + set_rand, + +}; + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/adler32.c b/optee_os/core/lib/libtomcrypt/src/misc/adler32.c new file mode 100644 index 0000000..8c2953d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/adler32.c @@ -0,0 +1,123 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file adler32.c + Adler-32 checksum algorithm + Written and placed in the public domain by Wei Dai + Adapted for libtomcrypt by Steffen Jaeckel +*/ +#ifdef LTC_ADLER32 + +static const unsigned long s_adler32_base = 65521; + +void adler32_init(adler32_state *ctx) +{ + LTC_ARGCHKVD(ctx != NULL); + ctx->s[0] = 1; + ctx->s[1] = 0; +} + +void adler32_update(adler32_state *ctx, const unsigned char *input, unsigned long length) +{ + unsigned long s1, s2; + + LTC_ARGCHKVD(ctx != NULL); + LTC_ARGCHKVD(input != NULL); + s1 = ctx->s[0]; + s2 = ctx->s[1]; + + if (length % 8 != 0) { + do { + s1 += *input++; + s2 += s1; + length--; + } while (length % 8 != 0); + + if (s1 >= s_adler32_base) { + s1 -= s_adler32_base; + } + s2 %= s_adler32_base; + } + + while (length > 0) { + s1 += input[0]; + s2 += s1; + s1 += input[1]; + s2 += s1; + s1 += input[2]; + s2 += s1; + s1 += input[3]; + s2 += s1; + s1 += input[4]; + s2 += s1; + s1 += input[5]; + s2 += s1; + s1 += input[6]; + s2 += s1; + s1 += input[7]; + s2 += s1; + + length -= 8; + input += 8; + + if (s1 >= s_adler32_base) { + s1 -= s_adler32_base; + } + s2 %= s_adler32_base; + } + + LTC_ARGCHKVD(s1 < s_adler32_base); + LTC_ARGCHKVD(s2 < s_adler32_base); + + ctx->s[0] = (unsigned short)s1; + ctx->s[1] = (unsigned short)s2; +} + +void adler32_finish(const adler32_state *ctx, void *hash, unsigned long size) +{ + unsigned char* h; + + LTC_ARGCHKVD(ctx != NULL); + LTC_ARGCHKVD(hash != NULL); + + h = hash; + + switch (size) { + default: + h[3] = ctx->s[0] & 0x0ff; + /* FALLTHROUGH */ + case 3: + h[2] = (ctx->s[0] >> 8) & 0x0ff; + /* FALLTHROUGH */ + case 2: + h[1] = ctx->s[1] & 0x0ff; + /* FALLTHROUGH */ + case 1: + h[0] = (ctx->s[1] >> 8) & 0x0ff; + /* FALLTHROUGH */ + case 0: + ; + } +} + +int adler32_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + const void* in = "libtomcrypt"; + const unsigned char adler32[] = { 0x1b, 0xe8, 0x04, 0xba }; + unsigned char out[4]; + adler32_state ctx; + adler32_init(&ctx); + adler32_update(&ctx, in, XSTRLEN(in)); + adler32_finish(&ctx, out, 4); + if (compare_testvector(adler32, 4, out, 4, "adler32", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; +#endif +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/base16/base16_decode.c b/optee_os/core/lib/libtomcrypt/src/misc/base16/base16_decode.c new file mode 100644 index 0000000..8ebc9ea --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/base16/base16_decode.c @@ -0,0 +1,65 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file base16_decode.c + Base16/Hex decode a string. + Based on https://stackoverflow.com/a/23898449 + Adapted for libtomcrypt by Steffen Jaeckel +*/ + +#ifdef LTC_BASE16 + +/** + Base16 decode a string + @param in The Base16 string to decode + @param inlen The length of the Base16 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @return CRYPT_OK if successful +*/ +int base16_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long pos, out_len; + unsigned char idx0, idx1; + char in0, in1; + + const unsigned char hashmap[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 01234567 */ + 0x08, 0x09, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 89:;<=>? */ + 0xff, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, /* @ABCDEFG */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* HIJKLMNO */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* PQRSTUVW */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* XYZ[\]^_ */ + 0xff, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, /* `abcdefg */ + }; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((inlen % 2) == 1) return CRYPT_INVALID_PACKET; + out_len = *outlen * 2; + for (pos = 0; ((pos + 1 < out_len) && (pos + 1 < inlen)); pos += 2) { + in0 = in[pos + 0]; + in1 = in[pos + 1]; + + if ((in0 < '0') || (in0 > 'g')) return CRYPT_INVALID_PACKET; + if ((in1 < '0') || (in1 > 'g')) return CRYPT_INVALID_PACKET; + + idx0 = (unsigned char) (in0 & 0x1F) ^ 0x10; + idx1 = (unsigned char) (in1 & 0x1F) ^ 0x10; + + if (hashmap[idx0] == 0xff) return CRYPT_INVALID_PACKET; + if (hashmap[idx1] == 0xff) return CRYPT_INVALID_PACKET; + + out[pos / 2] = (unsigned char) (hashmap[idx0] << 4) | hashmap[idx1]; + } + *outlen = pos / 2; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/base16/base16_encode.c b/optee_os/core/lib/libtomcrypt/src/misc/base16/base16_encode.c new file mode 100644 index 0000000..649a4d8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/base16/base16_encode.c @@ -0,0 +1,64 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file base16_encode.c + Base16/Hex encode a string, Steffen Jaeckel +*/ + +#ifdef LTC_BASE16 + +/** + Base16 encode a buffer + @param in The input buffer to encode + @param inlen The length of the input buffer + @param out [out] The destination of the Base16 encoded data + @param outlen [in/out] The max size and resulting size of the encoded data + @param options Output 'a-f' on 0 and 'A-F' otherwise. + @return CRYPT_OK if successful +*/ +int base16_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen, + unsigned int options) +{ + unsigned long i, x; + const char *alphabet; + const char *alphabets[2] = { + "0123456789abcdef", + "0123456789ABCDEF", + }; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* check the sizes */ + x = inlen * 2 + 1; + + if (x < inlen) return CRYPT_OVERFLOW; + + if (*outlen < x) { + *outlen = x; + return CRYPT_BUFFER_OVERFLOW; + } + x--; + *outlen = x; /* returning the length without terminating NUL */ + + if (options == 0) { + alphabet = alphabets[0]; + } else { + alphabet = alphabets[1]; + } + + for (i = 0; i < x; i += 2) { + out[i] = alphabet[(in[i/2] >> 4) & 0x0f]; + out[i+1] = alphabet[in[i/2] & 0x0f]; + } + out[x] = '\0'; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/base32/base32_decode.c b/optee_os/core/lib/libtomcrypt/src/misc/base32/base32_decode.c new file mode 100644 index 0000000..fc98cce --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/base32/base32_decode.c @@ -0,0 +1,111 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BASE32 + +/** + Base32 decode a buffer + @param in The Base32 data to decode + @param inlen The length of the Base32 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @param id Alphabet to use BASE32_RFC4648, BASE32_BASE32HEX, BASE32_ZBASE32 or BASE32_CROCKFORD + @return CRYPT_OK if successful +*/ +int base32_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + base32_alphabet id) +{ + unsigned long x; + int y = 0; + ulong64 t = 0; + char c; + const unsigned char *map; + const unsigned char tables[4][43] = { + { /* id = BASE32_RFC4648 : ABCDEFGHIJKLMNOPQRSTUVWXYZ234567 */ + 99/*0*/,99/*1*/,26/*2*/,27/*3*/,28/*4*/,29/*5*/,30/*6*/,31/*7*/,99/*8*/,99/*9*/, + 99/*:*/,99/*;*/,99/*<*/,99/*=*/,99/*>*/,99/*?*/,99/*@*/, + 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, 7/*H*/, 8/*I*/, 9/*J*/,10/*K*/,11/*L*/,12/*M*/, + 13/*N*/,14/*O*/,15/*P*/,16/*Q*/,17/*R*/,18/*S*/,19/*T*/,20/*U*/,21/*V*/,22/*W*/,23/*X*/,24/*Y*/,25/*Z*/ + }, + { /* id = BASE32_BASE32HEX : 0123456789ABCDEFGHIJKLMNOPQRSTUV */ + 0/*0*/, 1/*1*/, 2/*2*/, 3/*3*/, 4/*4*/, 5/*5*/, 6/*6*/, 7/*7*/, 8/*8*/, 9/*9*/, + 99/*:*/,99/*;*/,99/*<*/,99/*=*/,99/*>*/,99/*?*/,99/*@*/, + 10/*A*/,11/*B*/,12/*C*/,13/*D*/,14/*E*/,15/*F*/,16/*G*/,17/*H*/,18/*I*/,19/*J*/,20/*K*/,21/*L*/,22/*M*/, + 23/*N*/,24/*O*/,25/*P*/,26/*Q*/,27/*R*/,28/*S*/,29/*T*/,30/*U*/,31/*V*/,99/*W*/,99/*X*/,99/*Y*/,99/*Z*/ + }, + { /* id = BASE32_ZBASE32 : YBNDRFG8EJKMCPQXOT1UWISZA345H769 */ + 99/*0*/,18/*1*/,99/*2*/,25/*3*/,26/*4*/,27/*5*/,30/*6*/,29/*7*/, 7/*8*/,31/*9*/, + 99/*:*/,99/*;*/,99/*<*/,99/*=*/,99/*>*/,99/*?*/,99/*@*/, + 24/*A*/, 1/*B*/,12/*C*/, 3/*D*/, 8/*E*/, 5/*F*/, 6/*G*/,28/*H*/,21/*I*/, 9/*J*/,10/*K*/,99/*L*/,11/*M*/, + 2/*N*/,16/*O*/,13/*P*/,14/*Q*/, 4/*R*/,22/*S*/,17/*T*/,19/*U*/,99/*V*/,20/*W*/,15/*X*/, 0/*Y*/,23/*Z*/ + }, + { /* id = BASE32_CROCKFORD : 0123456789ABCDEFGHJKMNPQRSTVWXYZ + O=>0 + IL=>1 */ + 0/*0*/, 1/*1*/, 2/*2*/, 3/*3*/, 4/*4*/, 5/*5*/, 6/*6*/, 7/*7*/, 8/*8*/, 9/*9*/, + 99/*:*/,99/*;*/,99/*<*/,99/*=*/,99/*>*/,99/*?*/,99/*@*/, + 10/*A*/,11/*B*/,12/*C*/,13/*D*/,14/*E*/,15/*F*/,16/*G*/,17/*H*/, 1/*I*/,18/*J*/,19/*K*/, 1/*L*/,20/*M*/, + 21/*N*/, 0/*O*/,22/*P*/,23/*Q*/,24/*R*/,25/*S*/,26/*T*/,99/*U*/,27/*V*/,28/*W*/,29/*X*/,30/*Y*/,31/*Z*/ + } + }; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(id >= BASE32_RFC4648); + LTC_ARGCHK(id <= BASE32_CROCKFORD); + + /* ignore all trailing = */ + while (inlen > 0 && in[inlen-1] == '=') inlen--; + + /* no input, nothing to do */ + if (inlen == 0) { + *outlen = 0; + return CRYPT_OK; + } + + /* check the size of output buffer */ + x = (inlen * 5) / 8; + if (*outlen < x) { + *outlen = x; + return CRYPT_BUFFER_OVERFLOW; + } + *outlen = x; + + /* check input data length */ + x = inlen % 8; + if (x == 1 || x == 3 || x == 6) { + return CRYPT_INVALID_PACKET; + } + + map = tables[id]; + for (x = 0; x < inlen; x++) { + c = in[x]; + /* convert to upper case */ + if ((c >= 'a') && (c <= 'z')) c -= 32; + if (c < '0' || c > 'Z' || map[c-'0'] > 31) { + return CRYPT_INVALID_PACKET; + } + t = (t<<5) | map[c-'0']; + if (++y == 8) { + *out++ = (unsigned char)((t>>32) & 255); + *out++ = (unsigned char)((t>>24) & 255); + *out++ = (unsigned char)((t>>16) & 255); + *out++ = (unsigned char)((t>> 8) & 255); + *out++ = (unsigned char)( t & 255); + y = 0; + t = 0; + } + } + if (y > 0) { + t = t << (5 * (8 - y)); + if (y >= 2) *out++ = (unsigned char)((t>>32) & 255); + if (y >= 4) *out++ = (unsigned char)((t>>24) & 255); + if (y >= 5) *out++ = (unsigned char)((t>>16) & 255); + if (y >= 7) *out++ = (unsigned char)((t>> 8) & 255); + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/base32/base32_encode.c b/optee_os/core/lib/libtomcrypt/src/misc/base32/base32_encode.c new file mode 100644 index 0000000..29519bf --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/base32/base32_encode.c @@ -0,0 +1,86 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_BASE32 + +/** + Base32 encode a buffer + @param in The input buffer to encode + @param inlen The length of the input buffer + @param out [out] The destination of the Base32 encoded data + @param outlen [in/out] The max size and resulting size of the encoded data + @param id Alphabet to use BASE32_RFC4648, BASE32_BASE32HEX, BASE32_ZBASE32 or BASE32_CROCKFORD + @return CRYPT_OK if successful +*/ +int base32_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen, + base32_alphabet id) +{ + unsigned long i, x; + const char *codes; + const char *alphabet[4] = { + "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", /* id = BASE32_RFC4648 */ + "0123456789ABCDEFGHIJKLMNOPQRSTUV", /* id = BASE32_BASE32HEX */ + "ybndrfg8ejkmcpqxot1uwisza345h769", /* id = BASE32_ZBASE32 */ + "0123456789ABCDEFGHJKMNPQRSTVWXYZ" /* id = BASE32_CROCKFORD */ + }; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(id >= BASE32_RFC4648); + LTC_ARGCHK(id <= BASE32_CROCKFORD); + + /* check the size of output buffer +1 byte for terminating NUL */ + x = (8 * inlen + 4) / 5 + 1; + if (*outlen < x) { + *outlen = x; + return CRYPT_BUFFER_OVERFLOW; + } + *outlen = x - 1; /* returning the length without terminating NUL */ + + /* no input, nothing to do */ + if (inlen == 0) { + *out = '\0'; + return CRYPT_OK; + } + + codes = alphabet[id]; + x = 5 * (inlen / 5); + for (i = 0; i < x; i += 5) { + *out++ = codes[(in[0] >> 3) & 0x1F]; + *out++ = codes[(((in[0] & 0x7) << 2) + (in[1] >> 6)) & 0x1F]; + *out++ = codes[(in[1] >> 1) & 0x1F]; + *out++ = codes[(((in[1] & 0x1) << 4) + (in[2] >> 4)) & 0x1F]; + *out++ = codes[(((in[2] & 0xF) << 1) + (in[3] >> 7)) & 0x1F]; + *out++ = codes[(in[3] >> 2) & 0x1F]; + *out++ = codes[(((in[3] & 0x3) << 3) + (in[4] >> 5)) & 0x1F]; + *out++ = codes[in[4] & 0x1F]; + in += 5; + } + if (i < inlen) { + unsigned a = in[0]; + unsigned b = (i+1 < inlen) ? in[1] : 0; + unsigned c = (i+2 < inlen) ? in[2] : 0; + unsigned d = (i+3 < inlen) ? in[3] : 0; + *out++ = codes[(a >> 3) & 0x1F]; + *out++ = codes[(((a & 0x7) << 2) + (b >> 6)) & 0x1F]; + if (i+1 < inlen) { + *out++ = codes[(b >> 1) & 0x1F]; + *out++ = codes[(((b & 0x1) << 4) + (c >> 4)) & 0x1F]; + } + if (i+2 < inlen) { + *out++ = codes[(((c & 0xF) << 1) + (d >> 7)) & 0x1F]; + } + if (i+3 < inlen) { + *out++ = codes[(d >> 2) & 0x1F]; + *out++ = codes[((d & 0x3) << 3) & 0x1F]; + } + } + *out = '\0'; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/base64/base64_decode.c b/optee_os/core/lib/libtomcrypt/src/misc/base64/base64_decode.c new file mode 100644 index 0000000..5859111 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/base64/base64_decode.c @@ -0,0 +1,229 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file base64_decode.c + Compliant base64 code donated by Wayne Scott (wscott@bitmover.com) + base64 URL Safe variant (RFC 4648 section 5) by Karel Miko +*/ + + +#if defined(LTC_BASE64) || defined (LTC_BASE64_URL) + +/* 253 - ignored in "relaxed" + "insane" mode: TAB(9), CR(13), LF(10), space(32) + * 254 - padding character '=' (allowed only at the end) + * 255 - ignored in "insane" mode, but not allowed in "relaxed" + "strict" mode + */ + +#if defined(LTC_BASE64) +static const unsigned char map_base64[256] = { +255, 255, 255, 255, 255, 255, 255, 255, 255, 253, 253, 255, +255, 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, +255, 254, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, +255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255 }; +#endif /* LTC_BASE64 */ + +static const unsigned char map_base64url[] = { +#if defined(LTC_BASE64_URL) +255, 255, 255, 255, 255, 255, 255, 255, 255, 253, 253, 255, +255, 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, +255, 254, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 63, +255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +255, 255, 255, 255 +#endif /* LTC_BASE64_URL */ +}; + +enum { + insane = 0, + strict = 1, + relaxed = 2 +}; + +static int s_base64_decode_internal(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const unsigned char *map, int mode) +{ + unsigned long t, x, y, z; + unsigned char c; + int g; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + g = 0; /* '=' counter */ + for (x = y = z = t = 0; x < inlen; x++) { + if ((in[x] == 0) && (x == (inlen - 1)) && (mode != strict)) { + continue; /* allow the last byte to be NUL (relaxed+insane) */ + } + c = map[(unsigned char)in[x]&0xFF]; + if (c == 254) { + g++; + continue; + } + if (c == 253) { + if (mode == strict) { + return CRYPT_INVALID_PACKET; + } + continue; /* allow to ignore white-spaces (relaxed+insane) */ + } + if (c == 255) { + if (mode == insane) { + continue; /* allow to ignore invalid garbage (insane) */ + } + return CRYPT_INVALID_PACKET; + } + if ((g > 0) && (mode != insane)) { + /* we only allow '=' to be at the end (strict+relaxed) */ + return CRYPT_INVALID_PACKET; + } + + t = (t<<6)|c; + + if (++y == 4) { + if (z + 3 > *outlen) return CRYPT_BUFFER_OVERFLOW; + out[z++] = (unsigned char)((t>>16)&255); + out[z++] = (unsigned char)((t>>8)&255); + out[z++] = (unsigned char)(t&255); + y = t = 0; + } + } + + if (y != 0) { + if (y == 1) return CRYPT_INVALID_PACKET; + if (((y + g) != 4) && (mode == strict) && (map != map_base64url)) return CRYPT_INVALID_PACKET; + t = t << (6 * (4 - y)); + if (z + y - 1 > *outlen) return CRYPT_BUFFER_OVERFLOW; + if (y >= 2) out[z++] = (unsigned char) ((t >> 16) & 255); + if (y == 3) out[z++] = (unsigned char) ((t >> 8) & 255); + } + *outlen = z; + return CRYPT_OK; +} + +#if defined(LTC_BASE64) +/** + Dangerously relaxed base64 decode a block of memory + @param in The base64 data to decode + @param inlen The length of the base64 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @return CRYPT_OK if successful +*/ +int base64_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + return s_base64_decode_internal(in, inlen, out, outlen, map_base64, insane); +} + +/** + Strict base64 decode a block of memory + @param in The base64 data to decode + @param inlen The length of the base64 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @return CRYPT_OK if successful +*/ +int base64_strict_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + return s_base64_decode_internal(in, inlen, out, outlen, map_base64, strict); +} + +/** + Sane base64 decode a block of memory + @param in The base64 data to decode + @param inlen The length of the base64 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @return CRYPT_OK if successful +*/ +int base64_sane_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + return s_base64_decode_internal(in, inlen, out, outlen, map_base64, relaxed); +} +#endif /* LTC_BASE64 */ + +#if defined(LTC_BASE64_URL) +/** + Dangerously relaxed base64 (URL Safe, RFC 4648 section 5) decode a block of memory + @param in The base64 data to decode + @param inlen The length of the base64 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @return CRYPT_OK if successful +*/ +int base64url_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + return s_base64_decode_internal(in, inlen, out, outlen, map_base64url, insane); +} + +/** + Strict base64 (URL Safe, RFC 4648 section 5) decode a block of memory + @param in The base64 data to decode + @param inlen The length of the base64 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @return CRYPT_OK if successful +*/ +int base64url_strict_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + return s_base64_decode_internal(in, inlen, out, outlen, map_base64url, strict); +} + +/** + Sane base64 (URL Safe, RFC 4648 section 5) decode a block of memory + @param in The base64 data to decode + @param inlen The length of the base64 data + @param out [out] The destination of the binary decoded data + @param outlen [in/out] The max size and resulting size of the decoded data + @return CRYPT_OK if successful +*/ +int base64url_sane_decode(const char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + return s_base64_decode_internal(in, inlen, out, outlen, map_base64url, relaxed); +} +#endif /* LTC_BASE64_URL */ + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/base64/base64_encode.c b/optee_os/core/lib/libtomcrypt/src/misc/base64/base64_encode.c new file mode 100644 index 0000000..ef8aee9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/base64/base64_encode.c @@ -0,0 +1,159 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file base64_encode.c + Compliant base64 encoder donated by Wayne Scott (wscott@bitmover.com) + base64 URL Safe variant (RFC 4648 section 5) by Karel Miko +*/ + + +#if defined(LTC_BASE64) || defined (LTC_BASE64_URL) + +#if defined(LTC_BASE64) +static const char * const codes_base64 = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +#endif /* LTC_BASE64 */ + +#if defined(LTC_BASE64_URL) +static const char * const codes_base64url = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; +#endif /* LTC_BASE64_URL */ + +enum mode { + nopad = 0, + pad = 1, + lf = 2, + cr = 4, + ssh = 8, + crlf = lf | cr, +}; + +static int s_base64_encode_internal(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen, + const char *codes, unsigned int mode) +{ + unsigned long i, len2, leven, linelen; + char *p; + + LTC_ARGCHK(outlen != NULL); + + linelen = (mode & ssh) ? 72 : 64; + + /* valid output size ? */ + len2 = 4 * ((inlen + 2) / 3); + if ((mode & crlf) == lf) { + len2 += len2 / linelen; + } else if ((mode & crlf) == crlf) { + len2 += (len2 / linelen) * 2; + } + if (*outlen < len2 + 1) { + *outlen = len2 + 1; + return CRYPT_BUFFER_OVERFLOW; + } + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + + if ((void*)in == out) { + return CRYPT_INVALID_ARG; + } + + p = out; + leven = 3*(inlen / 3); + for (i = 0; i < leven; i += 3) { + *p++ = codes[(in[0] >> 2) & 0x3F]; + *p++ = codes[(((in[0] & 3) << 4) + (in[1] >> 4)) & 0x3F]; + *p++ = codes[(((in[1] & 0xf) << 2) + (in[2] >> 6)) & 0x3F]; + *p++ = codes[in[2] & 0x3F]; + in += 3; + if ((p - out) % linelen == 0) { + if (mode & cr) *p++ = '\r'; + if (mode & lf) *p++ = '\n'; + } + } + /* Pad it if necessary... */ + if (i < inlen) { + unsigned a = in[0]; + unsigned b = (i+1 < inlen) ? in[1] : 0; + + *p++ = codes[(a >> 2) & 0x3F]; + *p++ = codes[(((a & 3) << 4) + (b >> 4)) & 0x3F]; + if (mode & pad) { + *p++ = (i+1 < inlen) ? codes[(((b & 0xf) << 2)) & 0x3F] : '='; + *p++ = '='; + } + else { + if (i+1 < inlen) *p++ = codes[(((b & 0xf) << 2)) & 0x3F]; + } + } + + /* append a NULL byte */ + *p = '\0'; + + /* return ok */ + *outlen = (unsigned long)(p - out); /* the length without terminating NUL */ + return CRYPT_OK; +} + +#if defined(LTC_BASE64) +/** + base64 Encode a buffer (NUL terminated) + @param in The input buffer to encode + @param inlen The length of the input buffer + @param out [out] The destination of the base64 encoded data + @param outlen [in/out] The max size and resulting size + @return CRYPT_OK if successful +*/ +int base64_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen) +{ + return s_base64_encode_internal(in, inlen, out, outlen, codes_base64, pad); +} + +/** + base64 Encode a buffer for PEM output + (NUL terminated with line-break at 64 chars) + @param in The input buffer to encode + @param inlen The length of the input buffer + @param out [out] The destination of the base64 encoded data + @param outlen [in/out] The max size and resulting size + @param flags \ref base64_pem_flags + @return CRYPT_OK if successful +*/ +int base64_encode_pem(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen, + unsigned int flags) +{ + int use_crlf = flags & BASE64_PEM_CRLF ? pad | crlf : pad | lf; + int ssh_style = flags & BASE64_PEM_SSH ? ssh : 0; + return s_base64_encode_internal(in, inlen, out, outlen, codes_base64, ssh_style | use_crlf); +} +#endif /* LTC_BASE64 */ + + +#if defined(LTC_BASE64_URL) +/** + base64 (URL Safe, RFC 4648 section 5) Encode a buffer (NUL terminated) + @param in The input buffer to encode + @param inlen The length of the input buffer + @param out [out] The destination of the base64 encoded data + @param outlen [in/out] The max size and resulting size + @return CRYPT_OK if successful +*/ +int base64url_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen) +{ + return s_base64_encode_internal(in, inlen, out, outlen, codes_base64url, nopad); +} + +int base64url_strict_encode(const unsigned char *in, unsigned long inlen, + char *out, unsigned long *outlen) +{ + return s_base64_encode_internal(in, inlen, out, outlen, codes_base64url, pad); +} +#endif /* LTC_BASE64_URL */ + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/base64/sub.mk b/optee_os/core/lib/libtomcrypt/src/misc/base64/sub.mk new file mode 100644 index 0000000..9191690 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/base64/sub.mk @@ -0,0 +1,2 @@ +srcs-y += base64_decode.c +srcs-y += base64_encode.c diff --git a/optee_os/core/lib/libtomcrypt/src/misc/bcrypt/bcrypt.c b/optee_os/core/lib/libtomcrypt/src/misc/bcrypt/bcrypt.c new file mode 100644 index 0000000..1bebdff --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/bcrypt/bcrypt.c @@ -0,0 +1,191 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file bcrypt.c + bcrypt pbkdf, Steffen Jaeckel +*/ +#ifdef LTC_BCRYPT + +#define BCRYPT_WORDS 8 +#define BCRYPT_HASHSIZE (BCRYPT_WORDS * 4) + +static int s_bcrypt_hash(const unsigned char *pt, + const unsigned char *pass, unsigned long passlen, + const unsigned char *salt, unsigned long saltlen, + unsigned char *out, unsigned long *outlen) +{ + symmetric_key key; + int err, n; + ulong32 ct[BCRYPT_WORDS]; + + if ((err = blowfish_setup_with_data(pass, passlen, salt, saltlen, &key)) != CRYPT_OK) { + return err; + } + for (n = 0; n < 64; ++n) { + if ((err = blowfish_expand(salt, saltlen, NULL, 0, &key)) != CRYPT_OK) { + return err; + } + if ((err = blowfish_expand(pass, passlen, NULL, 0, &key)) != CRYPT_OK) { + return err; + } + } + + for (n = 0; n < BCRYPT_WORDS; ++n) { + LOAD32H(ct[n], &pt[n*4]); + } + + for (n = 0; n < 64; ++n) { + blowfish_enc(ct, BCRYPT_WORDS/2, &key); + } + + for (n = 0; n < BCRYPT_WORDS; ++n) { + STORE32L(ct[n], &out[4 * n]); + } + *outlen = sizeof(ct); +#ifdef LTC_CLEAN_STACK + zeromem(&key, sizeof(key)); + zeromem(ct, sizeof(ct)); +#endif + + return CRYPT_OK; +} + +static int s_bcrypt_pbkdf_hash(const unsigned char *pass, unsigned long passlen, + const unsigned char *salt, unsigned long saltlen, + unsigned char *out, unsigned long *outlen) +{ + const unsigned char pt[] = "OxychromaticBlowfishSwatDynamite"; + return s_bcrypt_hash(pt, pass, passlen, salt, saltlen, out, outlen); +} + +/** + Compatible to bcrypt_pbkdf() as provided in OpenBSD + @param password The input password (or key) + @param password_len The length of the password (octets) + @param salt The salt (or nonce) + @param salt_len The length of the salt (octets) + @param rounds # of iterations desired [read specs for more] + @param hash_idx The index of the hash desired + @param out [out] The destination for this algorithm + @param outlen [in/out] The desired size of the algorithm output + @return CRYPT_OK if successful +*/ +int bcrypt_pbkdf_openbsd(const void *secret, unsigned long secret_len, + const unsigned char *salt, unsigned long salt_len, + unsigned int rounds, int hash_idx, + unsigned char *out, unsigned long *outlen) +{ + int err; + ulong32 blkno; + unsigned long left, itts, x, y, hashed_pass_len, step_size, steps, dest, used_rounds; + unsigned char *buf[3], blkbuf[4]; + unsigned char *hashed_pass; + + LTC_ARGCHK(secret != NULL); + LTC_ARGCHK(salt != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((secret_len == 0) || (salt_len == 0) || (*outlen == 0)) { + return CRYPT_INVALID_ARG; + } + /* test hash IDX */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + /* set default value for rounds if not given */ + if (rounds == 0) { + used_rounds = LTC_BCRYPT_DEFAULT_ROUNDS; + } else { + used_rounds = rounds; + } + + buf[0] = XMALLOC(MAXBLOCKSIZE * 3); + hashed_pass = XMALLOC(MAXBLOCKSIZE); + if (buf[0] == NULL || hashed_pass == NULL) { + if (hashed_pass != NULL) { + XFREE(hashed_pass); + } + if (buf[0] != NULL) { + XFREE(buf[0]); + } + return CRYPT_MEM; + } + /* buf[1] points to the second block of MAXBLOCKSIZE bytes */ + buf[1] = buf[0] + MAXBLOCKSIZE; + buf[2] = buf[1] + MAXBLOCKSIZE; + + step_size = (*outlen + BCRYPT_HASHSIZE - 1) / BCRYPT_HASHSIZE; + steps = (*outlen + step_size - 1) / step_size; + + hashed_pass_len = MAXBLOCKSIZE; + if ((err = hash_memory(hash_idx, (unsigned char*)secret, secret_len, hashed_pass, &hashed_pass_len)) != CRYPT_OK) { + goto LBL_ERR; + } + + left = *outlen; + blkno = 0; + while (left != 0) { + /* increment and store current block number */ + ++blkno; + STORE32H(blkno, blkbuf); + + /* process block number blkno */ + zeromem(buf[0], MAXBLOCKSIZE*2); + + x = MAXBLOCKSIZE; + if ((err = hash_memory_multi(hash_idx, buf[0], &x, + salt, salt_len, + blkbuf, 4uL, + LTC_NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + y = MAXBLOCKSIZE; + if ((err = s_bcrypt_pbkdf_hash(hashed_pass, hashed_pass_len, buf[0], x, buf[1], &y)) != CRYPT_OK) { + goto LBL_ERR; + } + XMEMCPY(buf[2], buf[1], y); + + /* now compute repeated and XOR it in buf[2] */ + for (itts = 1; itts < used_rounds; ++itts) { + x = MAXBLOCKSIZE; + if ((err = hash_memory(hash_idx, buf[1], y, buf[0], &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y = MAXBLOCKSIZE; + if ((err = s_bcrypt_pbkdf_hash(hashed_pass, hashed_pass_len, buf[0], x, buf[1], &y)) != CRYPT_OK) { + goto LBL_ERR; + } + for (x = 0; x < y; x++) { + buf[2][x] ^= buf[1][x]; + } + } + + /* now emit upto `steps` bytes of buf[2] to output */ + steps = MIN(steps, left); + for (y = 0; y < steps; ++y) { + dest = y * step_size + (blkno - 1); + if (dest >= *outlen) + break; + out[dest] = buf[2][y]; + } + left -= y; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(buf[0], MAXBLOCKSIZE*3); + zeromem(hashed_pass, MAXBLOCKSIZE); +#endif + + XFREE(hashed_pass); + XFREE(buf[0]); + + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/burn_stack.c b/optee_os/core/lib/libtomcrypt/src/misc/burn_stack.c new file mode 100644 index 0000000..5d276a4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/burn_stack.c @@ -0,0 +1,23 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file burn_stack.c + Burn stack, Tom St Denis +*/ + +/** + Burn some stack memory + @param len amount of stack to burn in bytes +*/ +void burn_stack(unsigned long len) +{ + unsigned char buf[32]; + zeromem(buf, sizeof(buf)); + if (len > (unsigned long)sizeof(buf)) { + burn_stack(len - sizeof(buf)); + } +} + + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/compare_testvector.c b/optee_os/core/lib/libtomcrypt/src/misc/compare_testvector.c new file mode 100644 index 0000000..bb3c8cb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/compare_testvector.c @@ -0,0 +1,81 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file compare_testvector.c + Function to compare two testvectors and print a (detailed) error-message if required, Steffen Jaeckel +*/ + +#if defined(LTC_TEST) && defined(LTC_TEST_DBG) +static void s_print_hex(const char* what, const void* v, const unsigned long l) +{ + const unsigned char* p = v; + unsigned long x, y = 0, z; + fprintf(stderr, "%s contents: \n", what); + for (x = 0; x < l; ) { + fprintf(stderr, "%02X ", p[x]); + if (!(++x % 16) || x == l) { + if((x % 16) != 0) { + z = 16 - (x % 16); + if(z >= 8) + fprintf(stderr, " "); + for (; z != 0; --z) { + fprintf(stderr, " "); + } + } + fprintf(stderr, " | "); + for(; y < x; y++) { + if((y % 8) == 0) + fprintf(stderr, " "); + if(isgraph(p[y])) + fprintf(stderr, "%c", p[y]); + else + fprintf(stderr, "."); + } + fprintf(stderr, "\n"); + } + else if((x % 8) == 0) { + fprintf(stderr, " "); + } + } +} +#endif + +/** + Compare two test-vectors + + @param is The data as it is + @param is_len The length of is + @param should The data as it should + @param should_len The length of should + @param what The type of the data + @param which The iteration count + @return 0 on equality, -1 or 1 on difference +*/ +int compare_testvector(const void* is, const unsigned long is_len, const void* should, const unsigned long should_len, const char* what, int which) +{ + int res = 0; + if(is_len != should_len) { + res = is_len > should_len ? -1 : 1; + } else { + res = XMEMCMP(is, should, is_len); + } +#if defined(LTC_TEST) && defined(LTC_TEST_DBG) + if (res != 0) { + fprintf(stderr, "Testvector #%i(0x%x) of %s failed:\n", which, which, what); + s_print_hex("SHOULD", should, should_len); + s_print_hex("IS ", is, is_len); +#if LTC_TEST_DBG > 1 + } else { + fprintf(stderr, "Testvector #%i(0x%x) of %s passed!\n", which, which, what); +#endif + } +#else + LTC_UNUSED_PARAM(which); + LTC_UNUSED_PARAM(what); +#endif + + return res; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/copy_or_zeromem.c b/optee_os/core/lib/libtomcrypt/src/misc/copy_or_zeromem.c new file mode 100644 index 0000000..a05eac6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/copy_or_zeromem.c @@ -0,0 +1,51 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file copy_or_zeromem.c + Either copy or zero a block of memory in constant time, Steffen Jaeckel +*/ + +/** + Either copy or zero a block of memory in constant time + @param src The source where to read from + @param dest The destination where to write to + @param len The length of the area to process (octets) + @param coz Copy (on 0) Or Zero (> 0) +*/ +void copy_or_zeromem(const unsigned char* src, unsigned char* dest, unsigned long len, int coz) +{ + unsigned long y; +#ifdef LTC_FAST + unsigned long z; + LTC_FAST_TYPE fastMask = ~(LTC_FAST_TYPE)0; /* initialize fastMask at all ones */ +#endif + unsigned char mask = 0xff; /* initialize mask at all ones */ + + LTC_ARGCHKVD(src != NULL); + LTC_ARGCHKVD(dest != NULL); + + if (coz != 0) coz = 1; + y = 0; + mask *= 1 - coz; /* mask = ( coz ? 0 : 0xff ) */ +#ifdef LTC_FAST + fastMask *= 1 - coz; + if (len & ~15) { + for (; y < (len & ~15); y += 16) { + for (z = 0; z < 16; z += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&dest[y+z])) = *(LTC_FAST_TYPE_PTR_CAST(&src[y+z])) & fastMask; + } + } + } +#endif + for (; y < len; y++) { + dest[y] = src[y] & mask; + } +#ifdef LTC_CLEAN_STACK +#ifdef LTC_FAST + fastMask = 0; +#endif + mask = 0; +#endif +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crc32.c b/optee_os/core/lib/libtomcrypt/src/misc/crc32.c new file mode 100644 index 0000000..d90e830 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crc32.c @@ -0,0 +1,193 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crc32.c + CRC-32 checksum algorithm + Written and placed in the public domain by Wei Dai + Adapted for libtomcrypt by Steffen Jaeckel +*/ +#ifdef LTC_CRC32 + +static const ulong32 CRC32_NEGL = 0xffffffffUL; + +#if defined(ENDIAN_LITTLE) +#define CRC32_INDEX(c) (c & 0xff) +#define CRC32_SHIFTED(c) (c >> 8) +#elif defined(ENDIAN_BIG) +#define CRC32_INDEX(c) (c >> 24) +#define CRC32_SHIFTED(c) (c << 8) +#else +#error The existing CRC32 implementation only works properly when the endianness of the target platform is known. +#endif + +/* Table of CRC-32's of all single byte values (made by makecrc.c) */ +static const ulong32 crc32_m_tab[] = +{ +#if defined(ENDIAN_LITTLE) + 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, + 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, + 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, + 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, + 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, + 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, + 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, + 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, + 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, + 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, + 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, + 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, + 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, + 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, + 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, + 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, + 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, + 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, + 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, + 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, + 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, + 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, + 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, + 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, + 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, + 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, + 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, + 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, + 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, + 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, + 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, + 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, + 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, + 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, + 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, + 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, + 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, + 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, + 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, + 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, + 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, + 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, + 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, + 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, + 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, + 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, + 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, + 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, + 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, + 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, + 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, + 0x2d02ef8dL +#else + 0x00000000L, 0x96300777L, 0x2c610eeeL, 0xba510999L, 0x19c46d07L, + 0x8ff46a70L, 0x35a563e9L, 0xa395649eL, 0x3288db0eL, 0xa4b8dc79L, + 0x1ee9d5e0L, 0x88d9d297L, 0x2b4cb609L, 0xbd7cb17eL, 0x072db8e7L, + 0x911dbf90L, 0x6410b71dL, 0xf220b06aL, 0x4871b9f3L, 0xde41be84L, + 0x7dd4da1aL, 0xebe4dd6dL, 0x51b5d4f4L, 0xc785d383L, 0x56986c13L, + 0xc0a86b64L, 0x7af962fdL, 0xecc9658aL, 0x4f5c0114L, 0xd96c0663L, + 0x633d0ffaL, 0xf50d088dL, 0xc8206e3bL, 0x5e10694cL, 0xe44160d5L, + 0x727167a2L, 0xd1e4033cL, 0x47d4044bL, 0xfd850dd2L, 0x6bb50aa5L, + 0xfaa8b535L, 0x6c98b242L, 0xd6c9bbdbL, 0x40f9bcacL, 0xe36cd832L, + 0x755cdf45L, 0xcf0dd6dcL, 0x593dd1abL, 0xac30d926L, 0x3a00de51L, + 0x8051d7c8L, 0x1661d0bfL, 0xb5f4b421L, 0x23c4b356L, 0x9995bacfL, + 0x0fa5bdb8L, 0x9eb80228L, 0x0888055fL, 0xb2d90cc6L, 0x24e90bb1L, + 0x877c6f2fL, 0x114c6858L, 0xab1d61c1L, 0x3d2d66b6L, 0x9041dc76L, + 0x0671db01L, 0xbc20d298L, 0x2a10d5efL, 0x8985b171L, 0x1fb5b606L, + 0xa5e4bf9fL, 0x33d4b8e8L, 0xa2c90778L, 0x34f9000fL, 0x8ea80996L, + 0x18980ee1L, 0xbb0d6a7fL, 0x2d3d6d08L, 0x976c6491L, 0x015c63e6L, + 0xf4516b6bL, 0x62616c1cL, 0xd8306585L, 0x4e0062f2L, 0xed95066cL, + 0x7ba5011bL, 0xc1f40882L, 0x57c40ff5L, 0xc6d9b065L, 0x50e9b712L, + 0xeab8be8bL, 0x7c88b9fcL, 0xdf1ddd62L, 0x492dda15L, 0xf37cd38cL, + 0x654cd4fbL, 0x5861b24dL, 0xce51b53aL, 0x7400bca3L, 0xe230bbd4L, + 0x41a5df4aL, 0xd795d83dL, 0x6dc4d1a4L, 0xfbf4d6d3L, 0x6ae96943L, + 0xfcd96e34L, 0x468867adL, 0xd0b860daL, 0x732d0444L, 0xe51d0333L, + 0x5f4c0aaaL, 0xc97c0dddL, 0x3c710550L, 0xaa410227L, 0x10100bbeL, + 0x86200cc9L, 0x25b56857L, 0xb3856f20L, 0x09d466b9L, 0x9fe461ceL, + 0x0ef9de5eL, 0x98c9d929L, 0x2298d0b0L, 0xb4a8d7c7L, 0x173db359L, + 0x810db42eL, 0x3b5cbdb7L, 0xad6cbac0L, 0x2083b8edL, 0xb6b3bf9aL, + 0x0ce2b603L, 0x9ad2b174L, 0x3947d5eaL, 0xaf77d29dL, 0x1526db04L, + 0x8316dc73L, 0x120b63e3L, 0x843b6494L, 0x3e6a6d0dL, 0xa85a6a7aL, + 0x0bcf0ee4L, 0x9dff0993L, 0x27ae000aL, 0xb19e077dL, 0x44930ff0L, + 0xd2a30887L, 0x68f2011eL, 0xfec20669L, 0x5d5762f7L, 0xcb676580L, + 0x71366c19L, 0xe7066b6eL, 0x761bd4feL, 0xe02bd389L, 0x5a7ada10L, + 0xcc4add67L, 0x6fdfb9f9L, 0xf9efbe8eL, 0x43beb717L, 0xd58eb060L, + 0xe8a3d6d6L, 0x7e93d1a1L, 0xc4c2d838L, 0x52f2df4fL, 0xf167bbd1L, + 0x6757bca6L, 0xdd06b53fL, 0x4b36b248L, 0xda2b0dd8L, 0x4c1b0aafL, + 0xf64a0336L, 0x607a0441L, 0xc3ef60dfL, 0x55df67a8L, 0xef8e6e31L, + 0x79be6946L, 0x8cb361cbL, 0x1a8366bcL, 0xa0d26f25L, 0x36e26852L, + 0x95770cccL, 0x03470bbbL, 0xb9160222L, 0x2f260555L, 0xbe3bbac5L, + 0x280bbdb2L, 0x925ab42bL, 0x046ab35cL, 0xa7ffd7c2L, 0x31cfd0b5L, + 0x8b9ed92cL, 0x1daede5bL, 0xb0c2649bL, 0x26f263ecL, 0x9ca36a75L, + 0x0a936d02L, 0xa906099cL, 0x3f360eebL, 0x85670772L, 0x13570005L, + 0x824abf95L, 0x147ab8e2L, 0xae2bb17bL, 0x381bb60cL, 0x9b8ed292L, + 0x0dbed5e5L, 0xb7efdc7cL, 0x21dfdb0bL, 0xd4d2d386L, 0x42e2d4f1L, + 0xf8b3dd68L, 0x6e83da1fL, 0xcd16be81L, 0x5b26b9f6L, 0xe177b06fL, + 0x7747b718L, 0xe65a0888L, 0x706a0fffL, 0xca3b0666L, 0x5c0b0111L, + 0xff9e658fL, 0x69ae62f8L, 0xd3ff6b61L, 0x45cf6c16L, 0x78e20aa0L, + 0xeed20dd7L, 0x5483044eL, 0xc2b30339L, 0x612667a7L, 0xf71660d0L, + 0x4d476949L, 0xdb776e3eL, 0x4a6ad1aeL, 0xdc5ad6d9L, 0x660bdf40L, + 0xf03bd837L, 0x53aebca9L, 0xc59ebbdeL, 0x7fcfb247L, 0xe9ffb530L, + 0x1cf2bdbdL, 0x8ac2bacaL, 0x3093b353L, 0xa6a3b424L, 0x0536d0baL, + 0x9306d7cdL, 0x2957de54L, 0xbf67d923L, 0x2e7a66b3L, 0xb84a61c4L, + 0x021b685dL, 0x942b6f2aL, 0x37be0bb4L, 0xa18e0cc3L, 0x1bdf055aL, + 0x8def022dL +#endif +}; + +void crc32_init(crc32_state *ctx) +{ + LTC_ARGCHKVD(ctx != NULL); + ctx->crc = CRC32_NEGL; +} + +void crc32_update(crc32_state *ctx, const unsigned char *input, unsigned long length) +{ + ulong32 crc; + LTC_ARGCHKVD(ctx != NULL); + LTC_ARGCHKVD(input != NULL); + crc = ctx->crc; + + while (length--) { + crc = crc32_m_tab[CRC32_INDEX(crc) ^ *input++] ^ CRC32_SHIFTED(crc); + } + + ctx->crc = crc; +} + +void crc32_finish(const crc32_state *ctx, void *hash, unsigned long size) +{ + unsigned long i; + unsigned char* h; + ulong32 crc; + LTC_ARGCHKVD(ctx != NULL); + LTC_ARGCHKVD(hash != NULL); + + h = hash; + crc = ctx->crc; + crc ^= CRC32_NEGL; + + if (size > 4) size = 4; + for (i = 0; i < size; i++) { + h[i] = ((unsigned char*)&(crc))[size-i-1]; + } +} + +int crc32_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + const void* in = "libtomcrypt"; + const unsigned char crc32[] = { 0xb3, 0x73, 0x76, 0xef }; + unsigned char out[4]; + crc32_state ctx; + crc32_init(&ctx); + crc32_update(&ctx, in, XSTRLEN(in)); + crc32_finish(&ctx, out, 4); + if (compare_testvector(crc32, 4, out, 4, "CRC32", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + return CRYPT_OK; +#endif +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt.c new file mode 100644 index 0000000..14576fa --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt.c @@ -0,0 +1,552 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt.c + Build strings, Tom St Denis +*/ +#define NAME_VALUE(s) #s"="NAME(s) +#define NAME(s) #s + +const char *crypt_build_settings = + "LibTomCrypt " SCRYPT " (www.libtom.net)\n" + "LibTomCrypt is public domain software.\n" +#if defined(INCLUDE_BUILD_DATE) + "Built on " __DATE__ " at " __TIME__ "\n" +#endif + "\n\nEndianness: " +#if defined(ENDIAN_NEUTRAL) + "neutral/" +#endif +#if defined(ENDIAN_LITTLE) + "little" +#elif defined(ENDIAN_BIG) + "big" +#endif + #if defined(ENDIAN_32BITWORD) + " (32-bit words)\n" + #elif defined(ENDIAN_64BITWORD) + " (64-bit words)\n" + #else + " (no wordsize defined)\n" + #endif + "Clean stack: " +#if defined(LTC_CLEAN_STACK) + "enabled\n" +#else + "disabled\n" +#endif + "\nCiphers built-in:\n" +#if defined(LTC_BLOWFISH) + " Blowfish\n" +#endif +#if defined(LTC_RC2) + " RC2\n" +#endif +#if defined(LTC_RC5) + " RC5\n" +#endif +#if defined(LTC_RC6) + " RC6\n" +#endif +#if defined(LTC_SAFERP) + " Safer+\n" +#endif +#if defined(LTC_SAFER) + " Safer\n" +#endif +#if defined(LTC_RIJNDAEL) + " Rijndael\n" +#endif +#if defined(LTC_XTEA) + " XTEA\n" +#endif +#if defined(LTC_TWOFISH) + " Twofish " + #if defined(LTC_TWOFISH_SMALL) && defined(LTC_TWOFISH_TABLES) && defined(LTC_TWOFISH_ALL_TABLES) + "(small, tables, all_tables)\n" + #elif defined(LTC_TWOFISH_SMALL) && defined(LTC_TWOFISH_TABLES) + "(small, tables)\n" + #elif defined(LTC_TWOFISH_SMALL) && defined(LTC_TWOFISH_ALL_TABLES) + "(small, all_tables)\n" + #elif defined(LTC_TWOFISH_TABLES) && defined(LTC_TWOFISH_ALL_TABLES) + "(tables, all_tables)\n" + #elif defined(LTC_TWOFISH_SMALL) + "(small)\n" + #elif defined(LTC_TWOFISH_TABLES) + "(tables)\n" + #elif defined(LTC_TWOFISH_ALL_TABLES) + "(all_tables)\n" + #else + "\n" + #endif +#endif +#if defined(LTC_DES) + " DES\n" +#endif +#if defined(LTC_CAST5) + " CAST5\n" +#endif +#if defined(LTC_NOEKEON) + " Noekeon\n" +#endif +#if defined(LTC_SKIPJACK) + " Skipjack\n" +#endif +#if defined(LTC_KHAZAD) + " Khazad\n" +#endif +#if defined(LTC_ANUBIS) + " Anubis " +#endif +#if defined(LTC_ANUBIS_TWEAK) + " (tweaked)" +#endif + "\n" +#if defined(LTC_KSEED) + " KSEED\n" +#endif +#if defined(LTC_KASUMI) + " KASUMI\n" +#endif +#if defined(LTC_MULTI2) + " MULTI2\n" +#endif +#if defined(LTC_CAMELLIA) + " Camellia\n" +#endif +#if defined(LTC_IDEA) + " IDEA\n" +#endif +#if defined(LTC_SERPENT) + " Serpent\n" +#endif +#if defined(LTC_TEA) + " TEA\n" +#endif + "Stream ciphers built-in:\n" +#if defined(LTC_CHACHA) + " ChaCha\n" +#endif +#if defined(LTC_SALSA20) + " Salsa20\n" +#endif +#if defined(LTC_XSALSA20) + " XSalsa20\n" +#endif +#if defined(LTC_SOSEMANUK) + " Sosemanuk\n" +#endif +#if defined(LTC_RABBIT) + " Rabbit\n" +#endif +#if defined(LTC_RC4_STREAM) + " RC4\n" +#endif +#if defined(LTC_SOBER128_STREAM) + " SOBER128\n" +#endif + + "\nHashes built-in:\n" +#if defined(LTC_SHA3) + " SHA3\n" +#endif +#if defined(LTC_KECCAK) + " KECCAK\n" +#endif +#if defined(LTC_SHA512) + " SHA-512\n" +#endif +#if defined(LTC_SHA384) + " SHA-384\n" +#endif +#if defined(LTC_SHA512_256) + " SHA-512/256\n" +#endif +#if defined(LTC_SHA256) + " SHA-256\n" +#endif +#if defined(LTC_SHA512_224) + " SHA-512/224\n" +#endif +#if defined(LTC_SHA224) + " SHA-224\n" +#endif +#if defined(LTC_TIGER) + " TIGER\n" +#endif +#if defined(LTC_SHA1) + " SHA1\n" +#endif +#if defined(LTC_MD5) + " MD5\n" +#endif +#if defined(LTC_MD4) + " MD4\n" +#endif +#if defined(LTC_MD2) + " MD2\n" +#endif +#if defined(LTC_RIPEMD128) + " RIPEMD128\n" +#endif +#if defined(LTC_RIPEMD160) + " RIPEMD160\n" +#endif +#if defined(LTC_RIPEMD256) + " RIPEMD256\n" +#endif +#if defined(LTC_RIPEMD320) + " RIPEMD320\n" +#endif +#if defined(LTC_WHIRLPOOL) + " WHIRLPOOL\n" +#endif +#if defined(LTC_BLAKE2S) + " BLAKE2S\n" +#endif +#if defined(LTC_BLAKE2B) + " BLAKE2B\n" +#endif +#if defined(LTC_CHC_HASH) + " CHC_HASH\n" +#endif + + "\nBlock Chaining Modes:\n" +#if defined(LTC_CFB_MODE) + " CFB\n" +#endif +#if defined(LTC_OFB_MODE) + " OFB\n" +#endif +#if defined(LTC_ECB_MODE) + " ECB\n" +#endif +#if defined(LTC_CBC_MODE) + " CBC\n" +#endif +#if defined(LTC_CTR_MODE) + " CTR\n" +#endif +#if defined(LTC_LRW_MODE) + " LRW" +#if defined(LTC_LRW_TABLES) + " (tables) " +#endif + "\n" +#endif +#if defined(LTC_F8_MODE) + " F8\n" +#endif +#if defined(LTC_XTS_MODE) + " XTS\n" +#endif + + "\nMACs:\n" +#if defined(LTC_HMAC) + " HMAC\n" +#endif +#if defined(LTC_OMAC) + " OMAC\n" +#endif +#if defined(LTC_PMAC) + " PMAC\n" +#endif +#if defined(LTC_PELICAN) + " PELICAN\n" +#endif +#if defined(LTC_XCBC) + " XCBC\n" +#endif +#if defined(LTC_F9_MODE) + " F9\n" +#endif +#if defined(LTC_POLY1305) + " POLY1305\n" +#endif +#if defined(LTC_BLAKE2SMAC) + " BLAKE2S MAC\n" +#endif +#if defined(LTC_BLAKE2BMAC) + " BLAKE2B MAC\n" +#endif + + "\nENC + AUTH modes:\n" +#if defined(LTC_EAX_MODE) + " EAX\n" +#endif +#if defined(LTC_OCB_MODE) + " OCB\n" +#endif +#if defined(LTC_OCB3_MODE) + " OCB3\n" +#endif +#if defined(LTC_CCM_MODE) + " CCM\n" +#endif +#if defined(LTC_GCM_MODE) + " GCM" +#if defined(LTC_GCM_TABLES) + " (tables) " +#endif +#if defined(LTC_GCM_TABLES_SSE2) + " (SSE2) " +#endif + "\n" +#endif +#if defined(LTC_CHACHA20POLY1305_MODE) + " CHACHA20POLY1305\n" +#endif + + "\nPRNG:\n" +#if defined(LTC_YARROW) + " Yarrow ("NAME_VALUE(LTC_YARROW_AES)")\n" +#endif +#if defined(LTC_SPRNG) + " SPRNG\n" +#endif +#if defined(LTC_RC4) + " RC4\n" +#endif +#if defined(LTC_CHACHA20_PRNG) + " ChaCha20\n" +#endif +#if defined(LTC_FORTUNA) + " Fortuna (" NAME_VALUE(LTC_FORTUNA_POOLS) ", " +#if defined(LTC_FORTUNA_RESEED_RATELIMIT_TIMED) + "LTC_FORTUNA_RESEED_RATELIMIT_TIMED, " +#else + "LTC_FORTUNA_RESEED_RATELIMIT_STATIC, " NAME_VALUE(LTC_FORTUNA_WD) +#endif + ")\n" +#endif +#if defined(LTC_SOBER128) + " SOBER128\n" +#endif + + "\nPK Crypto:\n" +#if defined(LTC_MRSA) + " RSA" +#if defined(LTC_RSA_BLINDING) && defined(LTC_RSA_CRT_HARDENING) + " (with blinding and CRT hardening)" +#elif defined(LTC_RSA_BLINDING) + " (with blinding)" +#elif defined(LTC_RSA_CRT_HARDENING) + " (with CRT hardening)" +#endif + "\n" +#endif +#if defined(LTC_MDH) + " DH\n" +#endif +#if defined(LTC_MECC) + " ECC" +#if defined(LTC_ECC_TIMING_RESISTANT) + " (with blinding)" +#endif + "\n" +#endif +#if defined(LTC_MDSA) + " DSA\n" +#endif +#if defined(LTC_CURVE25519) +#if defined(LTC_CURVE25519) + " Ed25519\n" +#endif +#if defined(LTC_CURVE25519) + " X25519\n" +#endif +#endif +#if defined(LTC_PK_MAX_RETRIES) + " "NAME_VALUE(LTC_PK_MAX_RETRIES)"\n" +#endif + + "\nMPI (Math):\n" +#if defined(LTC_MPI) + " LTC_MPI\n" +#endif +#if defined(LTM_DESC) + " LTM_DESC\n" +#endif +#if defined(TFM_DESC) + " TFM_DESC\n" +#endif +#if defined(GMP_DESC) + " GMP_DESC\n" +#endif +#if defined(LTC_MILLER_RABIN_REPS) + " "NAME_VALUE(LTC_MILLER_RABIN_REPS)"\n" +#endif + + "\nCompiler:\n" +#if defined(_WIN64) + " WIN64 platform detected.\n" +#elif defined(_WIN32) + " WIN32 platform detected.\n" +#endif +#if defined(__CYGWIN__) + " CYGWIN Detected.\n" +#endif +#if defined(__DJGPP__) + " DJGPP Detected.\n" +#endif +#if defined(_MSC_VER) + " MSVC compiler detected.\n" +#endif +#if defined(__clang_version__) + " Clang compiler " __clang_version__ ".\n" +#elif defined(INTEL_CC) + " Intel C Compiler " __VERSION__ ".\n" +#elif defined(__GNUC__) /* clang and icc also define __GNUC__ */ + " GCC compiler " __VERSION__ ".\n" +#endif + +#if defined(__x86_64__) + " x86-64 detected.\n" +#endif +#if defined(LTC_PPC32) + " PPC32 detected.\n" +#endif + + "\nVarious others: " +#if defined(ARGTYPE) + " " NAME_VALUE(ARGTYPE) " " +#endif +#if defined(LTC_ADLER32) + " ADLER32 " +#endif +#if defined(LTC_BASE64) + " BASE64 " +#endif +#if defined(LTC_BASE64_URL) + " BASE64-URL-SAFE " +#endif +#if defined(LTC_BASE32) + " BASE32 " +#endif +#if defined(LTC_BASE16) + " BASE16 " +#endif +#if defined(LTC_BCRYPT) + " BCRYPT " + " " NAME_VALUE(LTC_BCRYPT_DEFAULT_ROUNDS) " " +#endif +#if defined(LTC_CRC32) + " CRC32 " +#endif +#if defined(LTC_DER) + " DER " + " " NAME_VALUE(LTC_DER_MAX_RECURSION) " " +#endif +#if defined(LTC_PKCS_1) + " PKCS#1 " +#endif +#if defined(LTC_PKCS_5) + " PKCS#5 " +#endif +#if defined(LTC_PKCS_8) + " PKCS#8 " +#endif +#if defined(LTC_PKCS_12) + " PKCS#12 " +#endif +#if defined(LTC_PADDING) + " PADDING " +#endif +#if defined(LTC_HKDF) + " HKDF " +#endif +#if defined(LTC_PBES) + " PBES1 " + " PBES2 " +#endif +#if defined(LTC_SSH) + " SSH " +#endif +#if defined(LTC_DEVRANDOM) + " LTC_DEVRANDOM " +#endif +#if defined(LTC_TRY_URANDOM_FIRST) + " LTC_TRY_URANDOM_FIRST " +#endif +#if defined(LTC_RNG_GET_BYTES) + " LTC_RNG_GET_BYTES " +#endif +#if defined(LTC_RNG_MAKE_PRNG) + " LTC_RNG_MAKE_PRNG " +#endif +#if defined(LTC_PRNG_ENABLE_LTC_RNG) + " LTC_PRNG_ENABLE_LTC_RNG " +#endif +#if defined(LTC_HASH_HELPERS) + " LTC_HASH_HELPERS " +#endif +#if defined(LTC_VALGRIND) + " LTC_VALGRIND " +#endif +#if defined(LTC_TEST) + " LTC_TEST " +#endif +#if defined(LTC_TEST_DBG) + " " NAME_VALUE(LTC_TEST_DBG) " " +#endif +#if defined(LTC_TEST_EXT) + " LTC_TEST_EXT " +#endif +#if defined(LTC_SMALL_CODE) + " LTC_SMALL_CODE " +#endif +#if defined(LTC_NO_FILE) + " LTC_NO_FILE " +#endif +#if defined(LTC_FILE_READ_BUFSIZE) + " " NAME_VALUE(LTC_FILE_READ_BUFSIZE) " " +#endif +#if defined(LTC_FAST) + " LTC_FAST " +#endif +#if defined(LTC_NO_FAST) + " LTC_NO_FAST " +#endif +#if defined(LTC_NO_BSWAP) + " LTC_NO_BSWAP " +#endif +#if defined(LTC_NO_ASM) + " LTC_NO_ASM " +#endif +#if defined(LTC_ROx_BUILTIN) + " LTC_ROx_BUILTIN " +#elif defined(LTC_ROx_ASM) + " LTC_ROx_ASM " +#if defined(LTC_NO_ROLC) + " LTC_NO_ROLC " +#endif +#endif +#if defined(LTC_NO_TEST) + " LTC_NO_TEST " +#endif +#if defined(LTC_NO_TABLES) + " LTC_NO_TABLES " +#endif +#if defined(LTC_PTHREAD) + " LTC_PTHREAD " +#endif +#if defined(LTC_EASY) + " LTC_EASY " +#endif +#if defined(LTC_MECC_ACCEL) + " LTC_MECC_ACCEL " +#endif +#if defined(LTC_MECC_FP) + " LTC_MECC_FP " +#endif +#if defined(LTC_ECC_SHAMIR) + " LTC_ECC_SHAMIR " +#endif +#if defined(LTC_CLOCK_GETTIME) + " LTC_CLOCK_GETTIME " +#endif + "\n" + ; + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_argchk.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_argchk.c new file mode 100644 index 0000000..85b562a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_argchk.c @@ -0,0 +1,17 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_argchk.c + Perform argument checking, Tom St Denis +*/ + +#if (ARGTYPE == 0) +void crypt_argchk(const char *v, const char *s, int d) +{ + fprintf(stderr, "LTC_ARGCHK '%s' failure on line %d of file %s\n", + v, d, s); + abort(); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_cipher_descriptor.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_cipher_descriptor.c new file mode 100644 index 0000000..777870f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_cipher_descriptor.c @@ -0,0 +1,13 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_cipher_descriptor.c + Stores the cipher descriptor table, Tom St Denis +*/ + +const struct ltc_cipher_descriptor *cipher_descriptor[TAB_SIZE]; + +LTC_MUTEX_GLOBAL(ltc_cipher_mutex) + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_cipher_is_valid.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_cipher_is_valid.c new file mode 100644 index 0000000..ada1fbf --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_cipher_is_valid.c @@ -0,0 +1,24 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_cipher_is_valid.c + Determine if cipher is valid, Tom St Denis +*/ + +/* + Test if a cipher index is valid + @param idx The index of the cipher to search for + @return CRYPT_OK if valid +*/ +int cipher_is_valid(int idx) +{ + LTC_MUTEX_LOCK(<c_cipher_mutex); + if (idx < 0 || idx >= TAB_SIZE || cipher_descriptor[idx] == NULL) { + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return CRYPT_INVALID_CIPHER; + } + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return CRYPT_OK; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_constants.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_constants.c new file mode 100644 index 0000000..eac6dae --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_constants.c @@ -0,0 +1,290 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_constants.c + + Make various constants available to dynamic languages + like Python - Larry Bugbee, February 2013 + + LB - Dec 2013 - revised to include compiler define options + LB - Mar 2014 - added endianness and word size +*/ + +typedef struct { + const char *name; + const int value; +} crypt_constant; + +#define C_STRINGIFY(s) { #s, s } + +static const crypt_constant s_crypt_constants[] = { + + C_STRINGIFY(CRYPT_OK), + C_STRINGIFY(CRYPT_ERROR), + C_STRINGIFY(CRYPT_NOP), + C_STRINGIFY(CRYPT_INVALID_KEYSIZE), + C_STRINGIFY(CRYPT_INVALID_ROUNDS), + C_STRINGIFY(CRYPT_FAIL_TESTVECTOR), + C_STRINGIFY(CRYPT_BUFFER_OVERFLOW), + C_STRINGIFY(CRYPT_INVALID_PACKET), + C_STRINGIFY(CRYPT_INVALID_PRNGSIZE), + C_STRINGIFY(CRYPT_ERROR_READPRNG), + C_STRINGIFY(CRYPT_INVALID_CIPHER), + C_STRINGIFY(CRYPT_INVALID_HASH), + C_STRINGIFY(CRYPT_INVALID_PRNG), + C_STRINGIFY(CRYPT_MEM), + C_STRINGIFY(CRYPT_PK_TYPE_MISMATCH), + C_STRINGIFY(CRYPT_PK_NOT_PRIVATE), + C_STRINGIFY(CRYPT_INVALID_ARG), + C_STRINGIFY(CRYPT_FILE_NOTFOUND), + C_STRINGIFY(CRYPT_PK_INVALID_TYPE), + C_STRINGIFY(CRYPT_OVERFLOW), + C_STRINGIFY(CRYPT_PK_ASN1_ERROR), + C_STRINGIFY(CRYPT_INPUT_TOO_LONG), + C_STRINGIFY(CRYPT_PK_INVALID_SIZE), + C_STRINGIFY(CRYPT_INVALID_PRIME_SIZE), + C_STRINGIFY(CRYPT_PK_INVALID_PADDING), + C_STRINGIFY(CRYPT_HASH_OVERFLOW), + + C_STRINGIFY(PK_PUBLIC), + C_STRINGIFY(PK_PRIVATE), + + C_STRINGIFY(LTC_ENCRYPT), + C_STRINGIFY(LTC_DECRYPT), + +#ifdef LTC_PKCS_1 + {"LTC_PKCS_1", 1}, + /* Block types */ + C_STRINGIFY(LTC_PKCS_1_EMSA), + C_STRINGIFY(LTC_PKCS_1_EME), + + /* Padding types */ + C_STRINGIFY(LTC_PKCS_1_V1_5), + C_STRINGIFY(LTC_PKCS_1_OAEP), + C_STRINGIFY(LTC_PKCS_1_PSS), + C_STRINGIFY(LTC_PKCS_1_V1_5_NA1), +#else + {"LTC_PKCS_1", 0}, +#endif + +#ifdef LTC_PADDING + {"LTC_PADDING", 1}, + + C_STRINGIFY(LTC_PAD_PKCS7), +#ifdef LTC_RNG_GET_BYTES + C_STRINGIFY(LTC_PAD_ISO_10126), +#endif + C_STRINGIFY(LTC_PAD_ANSI_X923), + C_STRINGIFY(LTC_PAD_ONE_AND_ZERO), + C_STRINGIFY(LTC_PAD_ZERO), + C_STRINGIFY(LTC_PAD_ZERO_ALWAYS), +#else + {"LTC_PADDING", 0}, +#endif + +#ifdef LTC_MRSA + {"LTC_MRSA", 1}, +#else + {"LTC_MRSA", 0}, +#endif + +#ifdef LTC_MECC + {"LTC_MECC", 1}, + C_STRINGIFY(ECC_BUF_SIZE), + C_STRINGIFY(ECC_MAXSIZE), +#else + {"LTC_MECC", 0}, +#endif + +#ifdef LTC_MDSA + {"LTC_MDSA", 1}, + C_STRINGIFY(LTC_MDSA_DELTA), + C_STRINGIFY(LTC_MDSA_MAX_GROUP), + C_STRINGIFY(LTC_MDSA_MAX_MODULUS), +#else + {"LTC_MDSA", 0}, +#endif + +#ifdef LTC_MILLER_RABIN_REPS + C_STRINGIFY(LTC_MILLER_RABIN_REPS), +#endif + +#ifdef LTC_DER +/* DER handling */ + {"LTC_DER", 1}, + C_STRINGIFY(LTC_ASN1_EOL), + C_STRINGIFY(LTC_ASN1_BOOLEAN), + C_STRINGIFY(LTC_ASN1_INTEGER), + C_STRINGIFY(LTC_ASN1_SHORT_INTEGER), + C_STRINGIFY(LTC_ASN1_BIT_STRING), + C_STRINGIFY(LTC_ASN1_OCTET_STRING), + C_STRINGIFY(LTC_ASN1_NULL), + C_STRINGIFY(LTC_ASN1_OBJECT_IDENTIFIER), + C_STRINGIFY(LTC_ASN1_IA5_STRING), + C_STRINGIFY(LTC_ASN1_PRINTABLE_STRING), + C_STRINGIFY(LTC_ASN1_UTF8_STRING), + C_STRINGIFY(LTC_ASN1_UTCTIME), + C_STRINGIFY(LTC_ASN1_CHOICE), + C_STRINGIFY(LTC_ASN1_SEQUENCE), + C_STRINGIFY(LTC_ASN1_SET), + C_STRINGIFY(LTC_ASN1_SETOF), + C_STRINGIFY(LTC_ASN1_RAW_BIT_STRING), + C_STRINGIFY(LTC_ASN1_TELETEX_STRING), + C_STRINGIFY(LTC_ASN1_GENERALIZEDTIME), + C_STRINGIFY(LTC_ASN1_CUSTOM_TYPE), + C_STRINGIFY(LTC_DER_MAX_RECURSION), +#else + {"LTC_DER", 0}, +#endif + +#ifdef LTC_CTR_MODE + {"LTC_CTR_MODE", 1}, + C_STRINGIFY(CTR_COUNTER_LITTLE_ENDIAN), + C_STRINGIFY(CTR_COUNTER_BIG_ENDIAN), + C_STRINGIFY(LTC_CTR_RFC3686), +#else + {"LTC_CTR_MODE", 0}, +#endif +#ifdef LTC_GCM_MODE + C_STRINGIFY(LTC_GCM_MODE_IV), + C_STRINGIFY(LTC_GCM_MODE_AAD), + C_STRINGIFY(LTC_GCM_MODE_TEXT), +#endif + + C_STRINGIFY(LTC_MP_LT), + C_STRINGIFY(LTC_MP_EQ), + C_STRINGIFY(LTC_MP_GT), + + C_STRINGIFY(LTC_MP_NO), + C_STRINGIFY(LTC_MP_YES), + + C_STRINGIFY(MAXBLOCKSIZE), + C_STRINGIFY(TAB_SIZE), + C_STRINGIFY(ARGTYPE), + +#ifdef LTM_DESC + {"LTM_DESC", 1}, +#else + {"LTM_DESC", 0}, +#endif +#ifdef TFM_DESC + {"TFM_DESC", 1}, +#else + {"TFM_DESC", 0}, +#endif +#ifdef GMP_DESC + {"GMP_DESC", 1}, +#else + {"GMP_DESC", 0}, +#endif + +#ifdef LTC_FAST + {"LTC_FAST", 1}, +#else + {"LTC_FAST", 0}, +#endif + +#ifdef LTC_NO_FILE + {"LTC_NO_FILE", 1}, +#else + {"LTC_NO_FILE", 0}, +#endif + +#ifdef ENDIAN_LITTLE + {"ENDIAN_LITTLE", 1}, +#else + {"ENDIAN_LITTLE", 0}, +#endif + +#ifdef ENDIAN_BIG + {"ENDIAN_BIG", 1}, +#else + {"ENDIAN_BIG", 0}, +#endif + +#ifdef ENDIAN_32BITWORD + {"ENDIAN_32BITWORD", 1}, +#else + {"ENDIAN_32BITWORD", 0}, +#endif + +#ifdef ENDIAN_64BITWORD + {"ENDIAN_64BITWORD", 1}, +#else + {"ENDIAN_64BITWORD", 0}, +#endif + +#ifdef ENDIAN_NEUTRAL + {"ENDIAN_NEUTRAL", 1}, +#else + {"ENDIAN_NEUTRAL", 0}, +#endif +}; + + +/* crypt_get_constant() + * valueout will be the value of the named constant + * return -1 if named item not found + */ +int crypt_get_constant(const char* namein, int *valueout) { + int i; + int count = sizeof(s_crypt_constants) / sizeof(s_crypt_constants[0]); + for (i=0; i *names_list_size) { + return -1; + } + /* build the names list */ + ptr = names_list; + for (i=0; i total_len) return -1; + total_len -= number_len; + ptr += number_len; + } + /* to remove the trailing new-line */ + ptr -= 1; + *ptr = 0; + } + return 0; +} + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher.c new file mode 100644 index 0000000..d1ad352 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher.c @@ -0,0 +1,29 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_cipher.c + Find a cipher in the descriptor tables, Tom St Denis +*/ + +/** + Find a registered cipher by name + @param name The name of the cipher to look for + @return >= 0 if found, -1 if not present +*/ +int find_cipher(const char *name) +{ + int x; + LTC_ARGCHK(name != NULL); + LTC_MUTEX_LOCK(<c_cipher_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (cipher_descriptor[x] != NULL && !XSTRCMP(cipher_descriptor[x]->name, name)) { + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return x; + } + } + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return -1; +} + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher_any.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher_any.c new file mode 100644 index 0000000..9a9925f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher_any.c @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_cipher_any.c + Find a cipher in the descriptor tables, Tom St Denis +*/ + +/** + Find a cipher flexibly. First by name then if not present by block and key size + @param name The name of the cipher desired + @param blocklen The minimum length of the block cipher desired (octets) + @param keylen The minimum length of the key size desired (octets) + @return >= 0 if found, -1 if not present +*/ +int find_cipher_any(const char *name, int blocklen, int keylen) +{ + int x; + + if(name != NULL) { + x = find_cipher(name); + if (x != -1) return x; + } + + LTC_MUTEX_LOCK(<c_cipher_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (cipher_descriptor[x] == NULL) { + continue; + } + if (blocklen <= (int)cipher_descriptor[x]->block_length && keylen <= (int)cipher_descriptor[x]->max_key_length) { + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return x; + } + } + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher_id.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher_id.c new file mode 100644 index 0000000..8907055 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_cipher_id.c @@ -0,0 +1,27 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_cipher_id.c + Find cipher by ID, Tom St Denis +*/ + +/** + Find a cipher by ID number + @param ID The ID (not same as index) of the cipher to find + @return >= 0 if found, -1 if not present +*/ +int find_cipher_id(unsigned char ID) +{ + int x; + LTC_MUTEX_LOCK(<c_cipher_mutex); + for (x = 0; x < TAB_SIZE && cipher_descriptor[x] != NULL; x++) { + if (cipher_descriptor[x]->ID == ID) { + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return x; + } + } + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash.c new file mode 100644 index 0000000..31866fb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash.c @@ -0,0 +1,28 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_hash.c + Find a hash, Tom St Denis +*/ + +/** + Find a registered hash by name + @param name The name of the hash to look for + @return >= 0 if found, -1 if not present +*/ +int find_hash(const char *name) +{ + int x; + LTC_ARGCHK(name != NULL); + LTC_MUTEX_LOCK(<c_hash_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (hash_descriptor[x] != NULL && XSTRCMP(hash_descriptor[x]->name, name) == 0) { + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return x; + } + } + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_any.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_any.c new file mode 100644 index 0000000..5f0e55c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_any.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_hash_any.c + Find a hash, Tom St Denis +*/ + +/** + Find a hash flexibly. First by name then if not present by digest size + @param name The name of the hash desired + @param digestlen The minimum length of the digest size (octets) + @return >= 0 if found, -1 if not present +*/int find_hash_any(const char *name, int digestlen) +{ + int x, y, z; + LTC_ARGCHK(name != NULL); + + x = find_hash(name); + if (x != -1) return x; + + LTC_MUTEX_LOCK(<c_hash_mutex); + y = MAXBLOCKSIZE+1; + z = -1; + for (x = 0; x < TAB_SIZE; x++) { + if (hash_descriptor[x] == NULL) { + continue; + } + if ((int)hash_descriptor[x]->hashsize >= digestlen && (int)hash_descriptor[x]->hashsize < y) { + z = x; + y = hash_descriptor[x]->hashsize; + } + } + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return z; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_id.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_id.c new file mode 100644 index 0000000..e960af0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_id.c @@ -0,0 +1,27 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_hash_id.c + Find hash by ID, Tom St Denis +*/ + +/** + Find a hash by ID number + @param ID The ID (not same as index) of the hash to find + @return >= 0 if found, -1 if not present +*/ +int find_hash_id(unsigned char ID) +{ + int x; + LTC_MUTEX_LOCK(<c_hash_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (hash_descriptor[x] && hash_descriptor[x]->ID == ID) { + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return x; + } + } + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_oid.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_oid.c new file mode 100644 index 0000000..b764938 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_hash_oid.c @@ -0,0 +1,23 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_hash_oid.c + Find a hash, Tom St Denis +*/ + +int find_hash_oid(const unsigned long *ID, unsigned long IDlen) +{ + int x; + LTC_ARGCHK(ID != NULL); + LTC_MUTEX_LOCK(<c_hash_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (hash_descriptor[x] != NULL && hash_descriptor[x]->OIDlen == IDlen && !XMEMCMP(hash_descriptor[x]->OID, ID, sizeof(unsigned long) * IDlen)) { + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return x; + } + } + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_prng.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_prng.c new file mode 100644 index 0000000..b221f9e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_find_prng.c @@ -0,0 +1,29 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_find_prng.c + Find a PRNG, Tom St Denis +*/ + +/** + Find a registered PRNG by name + @param name The name of the PRNG to look for + @return >= 0 if found, -1 if not present +*/ +int find_prng(const char *name) +{ + int x; + LTC_ARGCHK(name != NULL); + LTC_MUTEX_LOCK(<c_prng_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if ((prng_descriptor[x] != NULL) && XSTRCMP(prng_descriptor[x]->name, name) == 0) { + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return x; + } + } + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return -1; +} + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_fsa.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_fsa.c new file mode 100644 index 0000000..a23216d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_fsa.c @@ -0,0 +1,46 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file crypt_fsa.c + LibTomCrypt FULL SPEED AHEAD!, Tom St Denis +*/ + +/* format is ltc_mp, cipher_desc, [cipher_desc], NULL, hash_desc, [hash_desc], NULL, prng_desc, [prng_desc], NULL */ +int crypt_fsa(void *mp, ...) +{ + va_list args; + void *p; + + va_start(args, mp); + if (mp != NULL) { + XMEMCPY(<c_mp, mp, sizeof(ltc_mp)); + } + + while ((p = va_arg(args, void*)) != NULL) { + if (register_cipher(p) == -1) { + va_end(args); + return CRYPT_INVALID_CIPHER; + } + } + + while ((p = va_arg(args, void*)) != NULL) { + if (register_hash(p) == -1) { + va_end(args); + return CRYPT_INVALID_HASH; + } + } + + while ((p = va_arg(args, void*)) != NULL) { + if (register_prng(p) == -1) { + va_end(args); + return CRYPT_INVALID_PRNG; + } + } + + va_end(args); + return CRYPT_OK; +} + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_hash_descriptor.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_hash_descriptor.c new file mode 100644 index 0000000..1a826c0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_hash_descriptor.c @@ -0,0 +1,13 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_hash_descriptor.c + Stores the hash descriptor table, Tom St Denis +*/ + +const struct ltc_hash_descriptor *hash_descriptor[TAB_SIZE]; + +LTC_MUTEX_GLOBAL(ltc_hash_mutex) + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_hash_is_valid.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_hash_is_valid.c new file mode 100644 index 0000000..d98f715 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_hash_is_valid.c @@ -0,0 +1,24 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_hash_is_valid.c + Determine if hash is valid, Tom St Denis +*/ + +/* + Test if a hash index is valid + @param idx The index of the hash to search for + @return CRYPT_OK if valid +*/ +int hash_is_valid(int idx) +{ + LTC_MUTEX_LOCK(<c_hash_mutex); + if (idx < 0 || idx >= TAB_SIZE || hash_descriptor[idx] == NULL) { + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return CRYPT_INVALID_HASH; + } + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return CRYPT_OK; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_inits.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_inits.c new file mode 100644 index 0000000..f7af2b8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_inits.c @@ -0,0 +1,81 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_inits.c + + Provide math library functions for dynamic languages + like Python - Larry Bugbee, February 2013 +*/ + + +#ifdef LTM_DESC +void init_LTM(void) +{ + ltc_mp = ltm_desc; +} +#endif + +#ifdef TFM_DESC +void init_TFM(void) +{ + ltc_mp = tfm_desc; +} +#endif + +#ifdef GMP_DESC +void init_GMP(void) +{ + ltc_mp = gmp_desc; +} +#endif + +int crypt_mp_init(const char* mpi) +{ + if (mpi == NULL) return CRYPT_ERROR; + switch (mpi[0]) { +#ifdef LTM_DESC + case 'l': + case 'L': + ltc_mp = ltm_desc; + return CRYPT_OK; +#endif +#ifdef TFM_DESC + case 't': + case 'T': + ltc_mp = tfm_desc; + return CRYPT_OK; +#endif +#ifdef GMP_DESC + case 'g': + case 'G': + ltc_mp = gmp_desc; + return CRYPT_OK; +#endif +#ifdef EXT_MATH_LIB + case 'e': + case 'E': + { + extern ltc_math_descriptor EXT_MATH_LIB; + ltc_mp = EXT_MATH_LIB; + } + +#if defined(LTC_TEST_DBG) +#define NAME_VALUE(s) #s"="NAME(s) +#define NAME(s) #s + printf("EXT_MATH_LIB = %s\n", NAME_VALUE(EXT_MATH_LIB)); +#undef NAME_VALUE +#undef NAME +#endif + + return CRYPT_OK; +#endif + default: +#if defined(LTC_TEST_DBG) + printf("Unknown/Invalid MPI provider: %s\n", mpi); +#endif + return CRYPT_ERROR; + } +} + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_ltc_mp_descriptor.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_ltc_mp_descriptor.c new file mode 100644 index 0000000..760ba63 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_ltc_mp_descriptor.c @@ -0,0 +1,6 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/* Initialize ltc_mp to nulls, to force allocation on all platforms, including macOS. */ +ltc_math_descriptor ltc_mp = { 0 }; diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_descriptor.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_descriptor.c new file mode 100644 index 0000000..56ccb6c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_descriptor.c @@ -0,0 +1,12 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_prng_descriptor.c + Stores the PRNG descriptors, Tom St Denis +*/ +const struct ltc_prng_descriptor *prng_descriptor[TAB_SIZE]; + +LTC_MUTEX_GLOBAL(ltc_prng_mutex) + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_is_valid.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_is_valid.c new file mode 100644 index 0000000..dea9cf6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_is_valid.c @@ -0,0 +1,24 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_prng_is_valid.c + Determine if PRNG is valid, Tom St Denis +*/ + +/* + Test if a PRNG index is valid + @param idx The index of the PRNG to search for + @return CRYPT_OK if valid +*/ +int prng_is_valid(int idx) +{ + LTC_MUTEX_LOCK(<c_prng_mutex); + if (idx < 0 || idx >= TAB_SIZE || prng_descriptor[idx] == NULL) { + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return CRYPT_INVALID_PRNG; + } + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return CRYPT_OK; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_rng_descriptor.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_rng_descriptor.c new file mode 100644 index 0000000..49a456f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_prng_rng_descriptor.c @@ -0,0 +1,7 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_PRNG_ENABLE_LTC_RNG +unsigned long (*ltc_rng)(unsigned char *out, unsigned long outlen, void (*callback)(void)); +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_ciphers.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_ciphers.c new file mode 100644 index 0000000..bb2b873 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_ciphers.c @@ -0,0 +1,99 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file crypt_register_all_ciphers.c + + Steffen Jaeckel +*/ + +#define REGISTER_CIPHER(h) do {\ + LTC_ARGCHK(register_cipher(h) != -1); \ +} while(0) + +int register_all_ciphers(void) +{ +#ifdef LTC_RIJNDAEL +#ifdef ENCRYPT_ONLY + /* alternative would be + * register_cipher(&rijndael_enc_desc); + */ + REGISTER_CIPHER(&aes_enc_desc); +#else + /* alternative would be + * register_cipher(&rijndael_desc); + */ + REGISTER_CIPHER(&aes_desc); +#endif +#endif +#ifdef LTC_BLOWFISH + REGISTER_CIPHER(&blowfish_desc); +#endif +#ifdef LTC_XTEA + REGISTER_CIPHER(&xtea_desc); +#endif +#ifdef LTC_RC5 + REGISTER_CIPHER(&rc5_desc); +#endif +#ifdef LTC_RC6 + REGISTER_CIPHER(&rc6_desc); +#endif +#ifdef LTC_SAFERP + REGISTER_CIPHER(&saferp_desc); +#endif +#ifdef LTC_TWOFISH + REGISTER_CIPHER(&twofish_desc); +#endif +#ifdef LTC_SAFER + REGISTER_CIPHER(&safer_k64_desc); + REGISTER_CIPHER(&safer_sk64_desc); + REGISTER_CIPHER(&safer_k128_desc); + REGISTER_CIPHER(&safer_sk128_desc); +#endif +#ifdef LTC_RC2 + REGISTER_CIPHER(&rc2_desc); +#endif +#ifdef LTC_DES + REGISTER_CIPHER(&des_desc); + REGISTER_CIPHER(&des3_desc); +#endif +#ifdef LTC_CAST5 + REGISTER_CIPHER(&cast5_desc); +#endif +#ifdef LTC_NOEKEON + REGISTER_CIPHER(&noekeon_desc); +#endif +#ifdef LTC_SKIPJACK + REGISTER_CIPHER(&skipjack_desc); +#endif +#ifdef LTC_ANUBIS + REGISTER_CIPHER(&anubis_desc); +#endif +#ifdef LTC_KHAZAD + REGISTER_CIPHER(&khazad_desc); +#endif +#ifdef LTC_KSEED + REGISTER_CIPHER(&kseed_desc); +#endif +#ifdef LTC_KASUMI + REGISTER_CIPHER(&kasumi_desc); +#endif +#ifdef LTC_MULTI2 + REGISTER_CIPHER(&multi2_desc); +#endif +#ifdef LTC_CAMELLIA + REGISTER_CIPHER(&camellia_desc); +#endif +#ifdef LTC_IDEA + REGISTER_CIPHER(&idea_desc); +#endif +#ifdef LTC_SERPENT + REGISTER_CIPHER(&serpent_desc); +#endif +#ifdef LTC_TEA + REGISTER_CIPHER(&tea_desc); +#endif + return CRYPT_OK; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_hashes.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_hashes.c new file mode 100644 index 0000000..68a64ae --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_hashes.c @@ -0,0 +1,95 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file crypt_register_all_hashes.c + + Steffen Jaeckel +*/ + +#define REGISTER_HASH(h) do {\ + LTC_ARGCHK(register_hash(h) != -1); \ +} while(0) + +int register_all_hashes(void) +{ +#ifdef LTC_TIGER + REGISTER_HASH(&tiger_desc); +#endif +#ifdef LTC_MD2 + REGISTER_HASH(&md2_desc); +#endif +#ifdef LTC_MD4 + REGISTER_HASH(&md4_desc); +#endif +#ifdef LTC_MD5 + REGISTER_HASH(&md5_desc); +#endif +#ifdef LTC_SHA1 + REGISTER_HASH(&sha1_desc); +#endif +#ifdef LTC_SHA224 + REGISTER_HASH(&sha224_desc); +#endif +#ifdef LTC_SHA256 + REGISTER_HASH(&sha256_desc); +#endif +#ifdef LTC_SHA384 + REGISTER_HASH(&sha384_desc); +#endif +#ifdef LTC_SHA512 + REGISTER_HASH(&sha512_desc); +#endif +#ifdef LTC_SHA512_224 + REGISTER_HASH(&sha512_224_desc); +#endif +#ifdef LTC_SHA512_256 + REGISTER_HASH(&sha512_256_desc); +#endif +#ifdef LTC_SHA3 + REGISTER_HASH(&sha3_224_desc); + REGISTER_HASH(&sha3_256_desc); + REGISTER_HASH(&sha3_384_desc); + REGISTER_HASH(&sha3_512_desc); +#endif +#ifdef LTC_KECCAK + REGISTER_HASH(&keccak_224_desc); + REGISTER_HASH(&keccak_256_desc); + REGISTER_HASH(&keccak_384_desc); + REGISTER_HASH(&keccak_512_desc); +#endif +#ifdef LTC_RIPEMD128 + REGISTER_HASH(&rmd128_desc); +#endif +#ifdef LTC_RIPEMD160 + REGISTER_HASH(&rmd160_desc); +#endif +#ifdef LTC_RIPEMD256 + REGISTER_HASH(&rmd256_desc); +#endif +#ifdef LTC_RIPEMD320 + REGISTER_HASH(&rmd320_desc); +#endif +#ifdef LTC_WHIRLPOOL + REGISTER_HASH(&whirlpool_desc); +#endif +#ifdef LTC_BLAKE2S + REGISTER_HASH(&blake2s_128_desc); + REGISTER_HASH(&blake2s_160_desc); + REGISTER_HASH(&blake2s_224_desc); + REGISTER_HASH(&blake2s_256_desc); +#endif +#ifdef LTC_BLAKE2S + REGISTER_HASH(&blake2b_160_desc); + REGISTER_HASH(&blake2b_256_desc); + REGISTER_HASH(&blake2b_384_desc); + REGISTER_HASH(&blake2b_512_desc); +#endif +#ifdef LTC_CHC_HASH + REGISTER_HASH(&chc_desc); + LTC_ARGCHK(chc_register(find_cipher_any("aes", 8, 16)) == CRYPT_OK); +#endif + return CRYPT_OK; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_prngs.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_prngs.c new file mode 100644 index 0000000..4a2e2ef --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_all_prngs.c @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file crypt_register_all_prngs.c + + Steffen Jaeckel +*/ + +#define REGISTER_PRNG(h) do {\ + LTC_ARGCHK(register_prng(h) != -1); \ +} while(0) + +int register_all_prngs(void) +{ +#ifdef LTC_YARROW + REGISTER_PRNG(&yarrow_desc); +#endif +#ifdef LTC_FORTUNA + REGISTER_PRNG(&fortuna_desc); +#endif +#ifdef LTC_RC4 + REGISTER_PRNG(&rc4_desc); +#endif +#ifdef LTC_CHACHA20_PRNG + REGISTER_PRNG(&chacha20_prng_desc); +#endif +#ifdef LTC_SOBER128 + REGISTER_PRNG(&sober128_desc); +#endif +#ifdef LTC_SPRNG + REGISTER_PRNG(&sprng_desc); +#endif + + return CRYPT_OK; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_cipher.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_cipher.c new file mode 100644 index 0000000..ee0439d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_cipher.c @@ -0,0 +1,42 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_register_cipher.c + Register a cipher, Tom St Denis +*/ + +/** + Register a cipher with the descriptor table + @param cipher The cipher you wish to register + @return value >= 0 if successfully added (or already present), -1 if unsuccessful +*/ +int register_cipher(const struct ltc_cipher_descriptor *cipher) +{ + int x; + + LTC_ARGCHK(cipher != NULL); + + /* is it already registered? */ + LTC_MUTEX_LOCK(<c_cipher_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (cipher_descriptor[x] != NULL && cipher_descriptor[x]->ID == cipher->ID) { + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return x; + } + } + + /* find a blank spot */ + for (x = 0; x < TAB_SIZE; x++) { + if (cipher_descriptor[x] == NULL) { + cipher_descriptor[x] = cipher; + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return x; + } + } + + /* no spot */ + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_hash.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_hash.c new file mode 100644 index 0000000..742985c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_hash.c @@ -0,0 +1,42 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_register_hash.c + Register a HASH, Tom St Denis +*/ + +/** + Register a hash with the descriptor table + @param hash The hash you wish to register + @return value >= 0 if successfully added (or already present), -1 if unsuccessful +*/ +int register_hash(const struct ltc_hash_descriptor *hash) +{ + int x; + + LTC_ARGCHK(hash != NULL); + + /* is it already registered? */ + LTC_MUTEX_LOCK(<c_hash_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (hash_descriptor[x] == hash) { + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return x; + } + } + + /* find a blank spot */ + for (x = 0; x < TAB_SIZE; x++) { + if (hash_descriptor[x] == NULL) { + hash_descriptor[x] = hash; + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return x; + } + } + + /* no spot */ + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_prng.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_prng.c new file mode 100644 index 0000000..bcb0d26 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_register_prng.c @@ -0,0 +1,42 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_register_prng.c + Register a PRNG, Tom St Denis +*/ + +/** + Register a PRNG with the descriptor table + @param prng The PRNG you wish to register + @return value >= 0 if successfully added (or already present), -1 if unsuccessful +*/ +int register_prng(const struct ltc_prng_descriptor *prng) +{ + int x; + + LTC_ARGCHK(prng != NULL); + + /* is it already registered? */ + LTC_MUTEX_LOCK(<c_prng_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (prng_descriptor[x] == prng) { + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return x; + } + } + + /* find a blank spot */ + for (x = 0; x < TAB_SIZE; x++) { + if (prng_descriptor[x] == NULL) { + prng_descriptor[x] = prng; + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return x; + } + } + + /* no spot */ + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return -1; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_sizes.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_sizes.c new file mode 100644 index 0000000..7545aa8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_sizes.c @@ -0,0 +1,351 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_sizes.c + + Make various struct sizes available to dynamic languages + like Python - Larry Bugbee, February 2013 + + LB - Dec 2013 - revised to include compiler define options +*/ + + +typedef struct { + const char *name; + const unsigned int size; +} crypt_size; + +#define SZ_STRINGIFY_S(s) { #s, sizeof(struct s) } +#define SZ_STRINGIFY_T(s) { #s, sizeof(s) } + +static const crypt_size s_crypt_sizes[] = { + /* hash state sizes */ + SZ_STRINGIFY_S(ltc_hash_descriptor), + SZ_STRINGIFY_T(hash_state), +#ifdef LTC_CHC_HASH + SZ_STRINGIFY_S(chc_state), +#endif +#ifdef LTC_WHIRLPOOL + SZ_STRINGIFY_S(whirlpool_state), +#endif +#ifdef LTC_SHA3 + SZ_STRINGIFY_S(sha3_state), +#endif +#ifdef LTC_SHA512 + SZ_STRINGIFY_S(sha512_state), +#endif +#ifdef LTC_SHA256 + SZ_STRINGIFY_S(sha256_state), +#endif +#ifdef LTC_SHA1 + SZ_STRINGIFY_S(sha1_state), +#endif +#ifdef LTC_MD5 + SZ_STRINGIFY_S(md5_state), +#endif +#ifdef LTC_MD4 + SZ_STRINGIFY_S(md4_state), +#endif +#ifdef LTC_MD2 + SZ_STRINGIFY_S(md2_state), +#endif +#ifdef LTC_TIGER + SZ_STRINGIFY_S(tiger_state), +#endif +#ifdef LTC_RIPEMD128 + SZ_STRINGIFY_S(rmd128_state), +#endif +#ifdef LTC_RIPEMD160 + SZ_STRINGIFY_S(rmd160_state), +#endif +#ifdef LTC_RIPEMD256 + SZ_STRINGIFY_S(rmd256_state), +#endif +#ifdef LTC_RIPEMD320 + SZ_STRINGIFY_S(rmd320_state), +#endif +#ifdef LTC_BLAKE2S + SZ_STRINGIFY_S(blake2s_state), +#endif +#ifdef LTC_BLAKE2B + SZ_STRINGIFY_S(blake2b_state), +#endif + + /* block cipher key sizes */ + SZ_STRINGIFY_S(ltc_cipher_descriptor), + SZ_STRINGIFY_T(symmetric_key), +#ifdef LTC_ANUBIS + SZ_STRINGIFY_S(anubis_key), +#endif +#ifdef LTC_CAMELLIA + SZ_STRINGIFY_S(camellia_key), +#endif +#ifdef LTC_BLOWFISH + SZ_STRINGIFY_S(blowfish_key), +#endif +#ifdef LTC_CAST5 + SZ_STRINGIFY_S(cast5_key), +#endif +#ifdef LTC_DES + SZ_STRINGIFY_S(des_key), + SZ_STRINGIFY_S(des3_key), +#endif +#ifdef LTC_IDEA + SZ_STRINGIFY_S(idea_key), +#endif +#ifdef LTC_KASUMI + SZ_STRINGIFY_S(kasumi_key), +#endif +#ifdef LTC_KHAZAD + SZ_STRINGIFY_S(khazad_key), +#endif +#ifdef LTC_KSEED + SZ_STRINGIFY_S(kseed_key), +#endif +#ifdef LTC_MULTI2 + SZ_STRINGIFY_S(multi2_key), +#endif +#ifdef LTC_NOEKEON + SZ_STRINGIFY_S(noekeon_key), +#endif +#ifdef LTC_RC2 + SZ_STRINGIFY_S(rc2_key), +#endif +#ifdef LTC_RC5 + SZ_STRINGIFY_S(rc5_key), +#endif +#ifdef LTC_RC6 + SZ_STRINGIFY_S(rc6_key), +#endif +#ifdef LTC_SERPENT + SZ_STRINGIFY_S(serpent_key), +#endif +#ifdef LTC_SKIPJACK + SZ_STRINGIFY_S(skipjack_key), +#endif +#ifdef LTC_XTEA + SZ_STRINGIFY_S(xtea_key), +#endif +#ifdef LTC_RIJNDAEL + SZ_STRINGIFY_S(rijndael_key), +#endif +#ifdef LTC_SAFER + SZ_STRINGIFY_S(safer_key), +#endif +#ifdef LTC_SAFERP + SZ_STRINGIFY_S(saferp_key), +#endif +#ifdef LTC_TWOFISH + SZ_STRINGIFY_S(twofish_key), +#endif + + /* mode sizes */ +#ifdef LTC_ECB_MODE + SZ_STRINGIFY_T(symmetric_ECB), +#endif +#ifdef LTC_CFB_MODE + SZ_STRINGIFY_T(symmetric_CFB), +#endif +#ifdef LTC_OFB_MODE + SZ_STRINGIFY_T(symmetric_OFB), +#endif +#ifdef LTC_CBC_MODE + SZ_STRINGIFY_T(symmetric_CBC), +#endif +#ifdef LTC_CTR_MODE + SZ_STRINGIFY_T(symmetric_CTR), +#endif +#ifdef LTC_LRW_MODE + SZ_STRINGIFY_T(symmetric_LRW), +#endif +#ifdef LTC_F8_MODE + SZ_STRINGIFY_T(symmetric_F8), +#endif +#ifdef LTC_XTS_MODE + SZ_STRINGIFY_T(symmetric_xts), +#endif + + /* stream cipher sizes */ +#ifdef LTC_CHACHA + SZ_STRINGIFY_T(chacha_state), +#endif +#ifdef LTC_SALSA20 + SZ_STRINGIFY_T(salsa20_state), +#endif +#ifdef LTC_SOSEMANUK + SZ_STRINGIFY_T(sosemanuk_state), +#endif +#ifdef LTC_RABBIT + SZ_STRINGIFY_T(rabbit_state), +#endif +#ifdef LTC_RC4_STREAM + SZ_STRINGIFY_T(rc4_state), +#endif +#ifdef LTC_SOBER128_STREAM + SZ_STRINGIFY_T(sober128_state), +#endif + + /* MAC sizes -- no states for ccm, lrw */ +#ifdef LTC_HMAC + SZ_STRINGIFY_T(hmac_state), +#endif +#ifdef LTC_OMAC + SZ_STRINGIFY_T(omac_state), +#endif +#ifdef LTC_PMAC + SZ_STRINGIFY_T(pmac_state), +#endif +#ifdef LTC_POLY1305 + SZ_STRINGIFY_T(poly1305_state), +#endif +#ifdef LTC_EAX_MODE + SZ_STRINGIFY_T(eax_state), +#endif +#ifdef LTC_OCB_MODE + SZ_STRINGIFY_T(ocb_state), +#endif +#ifdef LTC_OCB3_MODE + SZ_STRINGIFY_T(ocb3_state), +#endif +#ifdef LTC_CCM_MODE + SZ_STRINGIFY_T(ccm_state), +#endif +#ifdef LTC_GCM_MODE + SZ_STRINGIFY_T(gcm_state), +#endif +#ifdef LTC_PELICAN + SZ_STRINGIFY_T(pelican_state), +#endif +#ifdef LTC_XCBC + SZ_STRINGIFY_T(xcbc_state), +#endif +#ifdef LTC_F9_MODE + SZ_STRINGIFY_T(f9_state), +#endif +#ifdef LTC_CHACHA20POLY1305_MODE + SZ_STRINGIFY_T(chacha20poly1305_state), +#endif + + /* asymmetric keys */ +#ifdef LTC_MRSA + SZ_STRINGIFY_T(rsa_key), +#endif +#ifdef LTC_MDSA + SZ_STRINGIFY_T(dsa_key), +#endif +#ifdef LTC_MDH + SZ_STRINGIFY_T(dh_key), +#endif +#ifdef LTC_MECC + SZ_STRINGIFY_T(ltc_ecc_curve), + SZ_STRINGIFY_T(ecc_point), + SZ_STRINGIFY_T(ecc_key), +#endif + + /* DER handling */ +#ifdef LTC_DER + SZ_STRINGIFY_T(ltc_asn1_list), /* a list entry */ + SZ_STRINGIFY_T(ltc_utctime), + SZ_STRINGIFY_T(ltc_generalizedtime), +#endif + + /* prng state sizes */ + SZ_STRINGIFY_S(ltc_prng_descriptor), + SZ_STRINGIFY_T(prng_state), +#ifdef LTC_FORTUNA + SZ_STRINGIFY_S(fortuna_prng), +#endif +#ifdef LTC_CHACHA20_PRNG + SZ_STRINGIFY_S(chacha20_prng), +#endif +#ifdef LTC_RC4 + SZ_STRINGIFY_S(rc4_prng), +#endif +#ifdef LTC_SOBER128 + SZ_STRINGIFY_S(sober128_prng), +#endif +#ifdef LTC_YARROW + SZ_STRINGIFY_S(yarrow_prng), +#endif + /* sprng has no state as it uses other potentially available sources */ + /* like /dev/random. See Developers Guide for more info. */ + +#ifdef LTC_ADLER32 + SZ_STRINGIFY_T(adler32_state), +#endif +#ifdef LTC_CRC32 + SZ_STRINGIFY_T(crc32_state), +#endif + + SZ_STRINGIFY_T(ltc_mp_digit), + SZ_STRINGIFY_T(ltc_math_descriptor) + +}; + +/* crypt_get_size() + * sizeout will be the size (bytes) of the named struct or union + * return -1 if named item not found + */ +int crypt_get_size(const char* namein, unsigned int *sizeout) { + int i; + int count = sizeof(s_crypt_sizes) / sizeof(s_crypt_sizes[0]); + for (i=0; i *names_list_size) { + return -1; + } + /* build the names list */ + ptr = names_list; + for (i=0; i total_len) return -1; + total_len -= number_len; + ptr += number_len; + } + /* to remove the trailing new-line */ + ptr -= 1; + *ptr = 0; + } + return 0; +} + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_cipher.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_cipher.c new file mode 100644 index 0000000..f63a355 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_cipher.c @@ -0,0 +1,32 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_unregister_cipher.c + Unregister a cipher, Tom St Denis +*/ + +/** + Unregister a cipher from the descriptor table + @param cipher The cipher descriptor to remove + @return CRYPT_OK on success +*/ +int unregister_cipher(const struct ltc_cipher_descriptor *cipher) +{ + int x; + + LTC_ARGCHK(cipher != NULL); + + /* is it already registered? */ + LTC_MUTEX_LOCK(<c_cipher_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (cipher_descriptor[x] == cipher) { + cipher_descriptor[x] = NULL; + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return CRYPT_OK; + } + } + LTC_MUTEX_UNLOCK(<c_cipher_mutex); + return CRYPT_ERROR; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_hash.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_hash.c new file mode 100644 index 0000000..c82d296 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_hash.c @@ -0,0 +1,32 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_unregister_hash.c + Unregister a hash, Tom St Denis +*/ + +/** + Unregister a hash from the descriptor table + @param hash The hash descriptor to remove + @return CRYPT_OK on success +*/ +int unregister_hash(const struct ltc_hash_descriptor *hash) +{ + int x; + + LTC_ARGCHK(hash != NULL); + + /* is it already registered? */ + LTC_MUTEX_LOCK(<c_hash_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (hash_descriptor[x] == hash) { + hash_descriptor[x] = NULL; + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return CRYPT_OK; + } + } + LTC_MUTEX_UNLOCK(<c_hash_mutex); + return CRYPT_ERROR; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_prng.c b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_prng.c new file mode 100644 index 0000000..ece0630 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/crypt_unregister_prng.c @@ -0,0 +1,32 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file crypt_unregister_prng.c + Unregister a PRNG, Tom St Denis +*/ + +/** + Unregister a PRNG from the descriptor table + @param prng The PRNG descriptor to remove + @return CRYPT_OK on success +*/ +int unregister_prng(const struct ltc_prng_descriptor *prng) +{ + int x; + + LTC_ARGCHK(prng != NULL); + + /* is it already registered? */ + LTC_MUTEX_LOCK(<c_prng_mutex); + for (x = 0; x < TAB_SIZE; x++) { + if (prng_descriptor[x] == prng) { + prng_descriptor[x] = NULL; + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return CRYPT_OK; + } + } + LTC_MUTEX_UNLOCK(<c_prng_mutex); + return CRYPT_ERROR; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/crypt/sub.mk b/optee_os/core/lib/libtomcrypt/src/misc/crypt/sub.mk new file mode 100644 index 0000000..9880a9b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/crypt/sub.mk @@ -0,0 +1,22 @@ +srcs-y += crypt.c +srcs-y += crypt_cipher_descriptor.c +srcs-y += crypt_cipher_is_valid.c +srcs-y += crypt_find_cipher_any.c +srcs-y += crypt_find_cipher.c +srcs-y += crypt_find_cipher_id.c +srcs-y += crypt_find_hash_any.c +srcs-y += crypt_find_hash.c +srcs-y += crypt_find_hash_id.c +srcs-y += crypt_find_hash_oid.c +srcs-y += crypt_find_prng.c +srcs-y += crypt_fsa.c +srcs-y += crypt_hash_descriptor.c +srcs-y += crypt_hash_is_valid.c +srcs-y += crypt_prng_descriptor.c +srcs-y += crypt_prng_is_valid.c +srcs-y += crypt_register_cipher.c +srcs-y += crypt_register_hash.c +srcs-y += crypt_register_prng.c +srcs-y += crypt_unregister_cipher.c +srcs-y += crypt_unregister_hash.c +srcs-y += crypt_unregister_prng.c diff --git a/optee_os/core/lib/libtomcrypt/src/misc/error_to_string.c b/optee_os/core/lib/libtomcrypt/src/misc/error_to_string.c new file mode 100644 index 0000000..5afac84 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/error_to_string.c @@ -0,0 +1,68 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file error_to_string.c + Convert error codes to ASCII strings, Tom St Denis +*/ + +static const char * const err_2_str[] = +{ + "CRYPT_OK", + "CRYPT_ERROR", + "Non-fatal 'no-operation' requested.", + + "Invalid key size.", + "Invalid number of rounds for block cipher.", + "Algorithm failed test vectors.", + + "Buffer overflow.", + "Invalid input packet.", + + "Invalid number of bits for a PRNG.", + "Error reading the PRNG.", + + "Invalid cipher specified.", + "Invalid hash specified.", + "Invalid PRNG specified.", + + "Out of memory.", + + "Invalid PK key or key type specified for function.", + "A private PK key is required.", + + "Invalid argument provided.", + "File Not Found", + + "Invalid PK type.", + + "An overflow of a value was detected/prevented.", + + "An ASN.1 decoding error occurred.", + + "The input was longer than expected.", + + "Invalid sized parameter.", + + "Invalid size for prime.", + + "Invalid padding.", + + "Hash applied to too many bits.", +}; + +/** + Convert an LTC error code to ASCII + @param err The error code + @return A pointer to the ASCII NUL terminated string for the error or "Invalid error code." if the err code was not valid. +*/ +const char *error_to_string(int err) +{ + if (err < 0 || err >= (int)(sizeof(err_2_str)/sizeof(err_2_str[0]))) { + return "Invalid error code."; + } + return err_2_str[err]; +} + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/hkdf/hkdf.c b/optee_os/core/lib/libtomcrypt/src/misc/hkdf/hkdf.c new file mode 100644 index 0000000..eb79b9c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/hkdf/hkdf.c @@ -0,0 +1,135 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include +#include +#include + +#include "tomcrypt_private.h" + +#ifdef LTC_HKDF + +/* This is mostly just a wrapper around hmac_memory */ +int hkdf_extract(int hash_idx, const unsigned char *salt, unsigned long saltlen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + /* libtomcrypt chokes on a zero length HMAC key, so we need to check for + that. HMAC specifies that keys shorter than the hash's blocksize are + 0 padded to the block size. HKDF specifies that a NULL salt is to be + substituted with a salt comprised of hashLen 0 bytes. HMAC's padding + means that in either case the HMAC is actually using a blocksize long + zero filled key. Unless blocksize < hashLen (which wouldn't make any + sense), we can use a single 0 byte as the HMAC key and still generate + valid results for HKDF. */ + if (salt == NULL || saltlen == 0) { + return hmac_memory(hash_idx, (const unsigned char *)"", 1, in, inlen, out, outlen); + } + return hmac_memory(hash_idx, salt, saltlen, in, inlen, out, outlen); +} + +int hkdf_expand(int hash_idx, const unsigned char *info, unsigned long infolen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long outlen) +{ + unsigned long hashsize; + int err; + unsigned char N; + unsigned long Noutlen, outoff; + + unsigned char *T, *dat; + unsigned long Tlen, datlen; + + /* make sure hash descriptor is valid */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + + hashsize = hash_descriptor[hash_idx]->hashsize; + + /* RFC5869 parameter restrictions */ + if (inlen < hashsize || outlen > hashsize * 255) { + return CRYPT_INVALID_ARG; + } + if (info == NULL && infolen != 0) { + return CRYPT_INVALID_ARG; + } + LTC_ARGCHK(out != NULL); + + Tlen = hashsize + infolen + 1; + T = XMALLOC(Tlen); /* Replace with static buffer? */ + if (T == NULL) { + return CRYPT_MEM; + } + if (info != NULL) { + XMEMCPY(T + hashsize, info, infolen); + } + + /* HMAC data T(1) doesn't include a previous hash value */ + dat = T + hashsize; + datlen = Tlen - hashsize; + + N = 0; + outoff = 0; /* offset in out to write to */ + while (1) { /* an exit condition breaks mid-loop */ + Noutlen = MIN(hashsize, outlen - outoff); + T[Tlen - 1] = ++N; + if ((err = hmac_memory(hash_idx, in, inlen, dat, datlen, + out + outoff, &Noutlen)) != CRYPT_OK) { + zeromem(T, Tlen); + XFREE(T); + return err; + } + outoff += Noutlen; + + if (outoff >= outlen) { /* loop exit condition */ + break; + } + + /* All subsequent HMAC data T(N) DOES include the previous hash value */ + XMEMCPY(T, out + hashsize * (N-1), hashsize); + if (N == 1) { + dat = T; + datlen = Tlen; + } + } + zeromem(T, Tlen); + XFREE(T); + return CRYPT_OK; +} + +/* all in one step */ +int hkdf(int hash_idx, const unsigned char *salt, unsigned long saltlen, + const unsigned char *info, unsigned long infolen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long outlen) +{ + unsigned long hashsize; + int err; + unsigned char *extracted; + + /* make sure hash descriptor is valid */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + + hashsize = hash_descriptor[hash_idx]->hashsize; + + extracted = XMALLOC(hashsize); /* replace with static buffer? */ + if (extracted == NULL) { + return CRYPT_MEM; + } + if ((err = hkdf_extract(hash_idx, salt, saltlen, in, inlen, extracted, &hashsize)) != 0) { + zeromem(extracted, hashsize); + XFREE(extracted); + return err; + } + err = hkdf_expand(hash_idx, info, infolen, extracted, hashsize, out, outlen); + zeromem(extracted, hashsize); + XFREE(extracted); + return err; +} +#endif /* LTC_HKDF */ + + +/* vim: set ts=2 sw=2 et ai si: */ diff --git a/optee_os/core/lib/libtomcrypt/src/misc/hkdf/hkdf_test.c b/optee_os/core/lib/libtomcrypt/src/misc/hkdf/hkdf_test.c new file mode 100644 index 0000000..99a970c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/hkdf/hkdf_test.c @@ -0,0 +1,284 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hkdf_test.c + LTC_HKDF support, self-test, Steffen Jaeckel +*/ + +#ifdef LTC_HKDF + +/* + TEST CASES SOURCE: + +Internet Engineering Task Force (IETF) H. Krawczyk +Request for Comments: 5869 IBM Research +Category: Informational P. Eronen +ISSN: 2070-1721 Nokia + May 2010 +Appendix A. Test Vectors +*/ + +/** + LTC_HKDF self-test + @return CRYPT_OK if successful, CRYPT_NOP if tests have been disabled. +*/ +int hkdf_test(void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + unsigned char OKM[82]; + int i; + + static const struct hkdf_test_case { + int num; + const char* Hash; + unsigned char IKM[80]; + unsigned long IKM_l; + unsigned char salt[80]; + unsigned long salt_l; + unsigned char info[80]; + unsigned long info_l; + unsigned char PRK[32]; + unsigned long PRK_l; + unsigned char OKM[82]; + unsigned long OKM_l; + } cases[] = { +#ifdef LTC_SHA256 + /* + Basic test case with SHA-256 + + Hash = SHA-256 + IKM = 0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b (22 octets) + salt = 0x000102030405060708090a0b0c (13 octets) + info = 0xf0f1f2f3f4f5f6f7f8f9 (10 octets) + L = 42 + + PRK = 0x077709362c2e32df0ddc3f0dc47bba63 + 90b6c73bb50f9c3122ec844ad7c2b3e5 (32 octets) + OKM = 0x3cb25f25faacd57a90434f64d0362f2a + 2d2d0a90cf1a5a4c5db02d56ecc4c5bf + 34007208d5b887185865 (42 octets) + */ + {1, "sha256", + {0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b}, 22, + {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c}, 13, + {0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9}, 10, + {0x07, 0x77, 0x09, 0x36, 0x2c, 0x2e, 0x32, 0xdf, + 0x0d, 0xdc, 0x3f, 0x0d, 0xc4, 0x7b, 0xba, 0x63, + 0x90, 0xb6, 0xc7, 0x3b, 0xb5, 0x0f, 0x9c, 0x31, + 0x22, 0xec, 0x84, 0x4a, 0xd7, 0xc2, 0xb3, 0xe5}, 32, + {0x3c, 0xb2, 0x5f, 0x25, 0xfa, 0xac, 0xd5, 0x7a, + 0x90, 0x43, 0x4f, 0x64, 0xd0, 0x36, 0x2f, 0x2a, + 0x2d, 0x2d, 0x0a, 0x90, 0xcf, 0x1a, 0x5a, 0x4c, + 0x5d, 0xb0, 0x2d, 0x56, 0xec, 0xc4, 0xc5, 0xbf, + 0x34, 0x00, 0x72, 0x08, 0xd5, 0xb8, 0x87, 0x18, + 0x58, 0x65}, 42}, +#ifdef LTC_TEST_EXT + /* Test with SHA-256 and longer inputs/outputs */ + {2, "sha256", + {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f}, 80, + {0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf}, 80, + {0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff}, 80, + {0x06, 0xa6, 0xb8, 0x8c, 0x58, 0x53, 0x36, 0x1a, + 0x06, 0x10, 0x4c, 0x9c, 0xeb, 0x35, 0xb4, 0x5c, + 0xef, 0x76, 0x00, 0x14, 0x90, 0x46, 0x71, 0x01, + 0x4a, 0x19, 0x3f, 0x40, 0xc1, 0x5f, 0xc2, 0x44}, 32, + {0xb1, 0x1e, 0x39, 0x8d, 0xc8, 0x03, 0x27, 0xa1, + 0xc8, 0xe7, 0xf7, 0x8c, 0x59, 0x6a, 0x49, 0x34, + 0x4f, 0x01, 0x2e, 0xda, 0x2d, 0x4e, 0xfa, 0xd8, + 0xa0, 0x50, 0xcc, 0x4c, 0x19, 0xaf, 0xa9, 0x7c, + 0x59, 0x04, 0x5a, 0x99, 0xca, 0xc7, 0x82, 0x72, + 0x71, 0xcb, 0x41, 0xc6, 0x5e, 0x59, 0x0e, 0x09, + 0xda, 0x32, 0x75, 0x60, 0x0c, 0x2f, 0x09, 0xb8, + 0x36, 0x77, 0x93, 0xa9, 0xac, 0xa3, 0xdb, 0x71, + 0xcc, 0x30, 0xc5, 0x81, 0x79, 0xec, 0x3e, 0x87, + 0xc1, 0x4c, 0x01, 0xd5, 0xc1, 0xf3, 0x43, 0x4f, + 0x1d, 0x87}, 82}, + /* Test with SHA-256 and zero length salt/info */ + {3, "sha256", + {0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b}, 22, + {0}, 0, + {0}, 0, + {0x19, 0xef, 0x24, 0xa3, 0x2c, 0x71, 0x7b, 0x16, + 0x7f, 0x33, 0xa9, 0x1d, 0x6f, 0x64, 0x8b, 0xdf, + 0x96, 0x59, 0x67, 0x76, 0xaf, 0xdb, 0x63, 0x77, + 0xac, 0x43, 0x4c, 0x1c, 0x29, 0x3c, 0xcb, 0x04}, 32, + {0x8d, 0xa4, 0xe7, 0x75, 0xa5, 0x63, 0xc1, 0x8f, + 0x71, 0x5f, 0x80, 0x2a, 0x06, 0x3c, 0x5a, 0x31, + 0xb8, 0xa1, 0x1f, 0x5c, 0x5e, 0xe1, 0x87, 0x9e, + 0xc3, 0x45, 0x4e, 0x5f, 0x3c, 0x73, 0x8d, 0x2d, + 0x9d, 0x20, 0x13, 0x95, 0xfa, 0xa4, 0xb6, 0x1a, + 0x96, 0xc8}, 42}, +#endif /* LTC_TEST_EXT */ +#endif /* LTC_SHA256 */ +#ifdef LTC_SHA1 + /* Basic test case with SHA-1 */ + {4, "sha1", + {0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b}, 11, + {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c}, 13, + {0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9}, 10, + {0x9b, 0x6c, 0x18, 0xc4, 0x32, 0xa7, 0xbf, 0x8f, + 0x0e, 0x71, 0xc8, 0xeb, 0x88, 0xf4, 0xb3, 0x0b, + 0xaa, 0x2b, 0xa2, 0x43}, 20, + {0x08, 0x5a, 0x01, 0xea, 0x1b, 0x10, 0xf3, 0x69, + 0x33, 0x06, 0x8b, 0x56, 0xef, 0xa5, 0xad, 0x81, + 0xa4, 0xf1, 0x4b, 0x82, 0x2f, 0x5b, 0x09, 0x15, + 0x68, 0xa9, 0xcd, 0xd4, 0xf1, 0x55, 0xfd, 0xa2, + 0xc2, 0x2e, 0x42, 0x24, 0x78, 0xd3, 0x05, 0xf3, + 0xf8, 0x96}, 42}, +#ifdef LTC_TEST_EXT + /* Test with SHA-1 and longer inputs/outputs */ + {5, "sha1", + {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f}, 80, + {0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf}, 80, + {0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff}, 80, + {0x8a, 0xda, 0xe0, 0x9a, 0x2a, 0x30, 0x70, 0x59, + 0x47, 0x8d, 0x30, 0x9b, 0x26, 0xc4, 0x11, 0x5a, + 0x22, 0x4c, 0xfa, 0xf6}, 20, + {0x0b, 0xd7, 0x70, 0xa7, 0x4d, 0x11, 0x60, 0xf7, + 0xc9, 0xf1, 0x2c, 0xd5, 0x91, 0x2a, 0x06, 0xeb, + 0xff, 0x6a, 0xdc, 0xae, 0x89, 0x9d, 0x92, 0x19, + 0x1f, 0xe4, 0x30, 0x56, 0x73, 0xba, 0x2f, 0xfe, + 0x8f, 0xa3, 0xf1, 0xa4, 0xe5, 0xad, 0x79, 0xf3, + 0xf3, 0x34, 0xb3, 0xb2, 0x02, 0xb2, 0x17, 0x3c, + 0x48, 0x6e, 0xa3, 0x7c, 0xe3, 0xd3, 0x97, 0xed, + 0x03, 0x4c, 0x7f, 0x9d, 0xfe, 0xb1, 0x5c, 0x5e, + 0x92, 0x73, 0x36, 0xd0, 0x44, 0x1f, 0x4c, 0x43, + 0x00, 0xe2, 0xcf, 0xf0, 0xd0, 0x90, 0x0b, 0x52, + 0xd3, 0xb4}, 82}, + /* Test with SHA-1 and zero-length salt/info */ + {6, "sha1", + {0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b}, 22, + {0}, 0, + {0}, 0, + {0xda, 0x8c, 0x8a, 0x73, 0xc7, 0xfa, 0x77, 0x28, + 0x8e, 0xc6, 0xf5, 0xe7, 0xc2, 0x97, 0x78, 0x6a, + 0xa0, 0xd3, 0x2d, 0x01}, 20, + {0x0a, 0xc1, 0xaf, 0x70, 0x02, 0xb3, 0xd7, 0x61, + 0xd1, 0xe5, 0x52, 0x98, 0xda, 0x9d, 0x05, 0x06, + 0xb9, 0xae, 0x52, 0x05, 0x72, 0x20, 0xa3, 0x06, + 0xe0, 0x7b, 0x6b, 0x87, 0xe8, 0xdf, 0x21, 0xd0, + 0xea, 0x00, 0x03, 0x3d, 0xe0, 0x39, 0x84, 0xd3, + 0x49, 0x18}, 42}, + /* Test with SHA-1, salt not provided (defaults to HashLen zero octets), + zero-length info */ + {7, "sha1", + {0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c}, 22, + {0}, 0, /* pass a null pointer */ + {0}, 0, + {0x2a, 0xdc, 0xca, 0xda, 0x18, 0x77, 0x9e, 0x7c, + 0x20, 0x77, 0xad, 0x2e, 0xb1, 0x9d, 0x3f, 0x3e, + 0x73, 0x13, 0x85, 0xdd}, 20, + {0x2c, 0x91, 0x11, 0x72, 0x04, 0xd7, 0x45, 0xf3, + 0x50, 0x0d, 0x63, 0x6a, 0x62, 0xf6, 0x4f, 0x0a, + 0xb3, 0xba, 0xe5, 0x48, 0xaa, 0x53, 0xd4, 0x23, + 0xb0, 0xd1, 0xf2, 0x7e, 0xbb, 0xa6, 0xf5, 0xe5, + 0x67, 0x3a, 0x08, 0x1d, 0x70, 0xcc, 0xe7, 0xac, + 0xfc, 0x48}, 42}, +#endif /* LTC_TEST_EXT */ +#endif /* LTC_SHA1 */ + }; + + int err; + int tested=0,failed=0; + for(i=0; i < (int)(sizeof(cases) / sizeof(cases[0])); i++) { + int hash = find_hash(cases[i].Hash); + if (hash == -1) continue; + ++tested; + if((err = hkdf(hash, cases[i].salt, cases[i].salt_l, + cases[i].info, cases[i].info_l, + cases[i].IKM, cases[i].IKM_l, + OKM, cases[i].OKM_l)) != CRYPT_OK) { +#if defined(LTC_TEST_DBG) && (LTC_TEST_DBG > 1) + printf("LTC_HKDF-%s test #%d, %s\n", cases[i].Hash, i, error_to_string(err)); +#endif + return err; + } + + if(compare_testvector(OKM, cases[i].OKM_l, cases[i].OKM, (size_t)cases[i].OKM_l, "HKDF", cases[i].num)) { + failed++; + } + } + + if (failed != 0) { + return CRYPT_FAIL_TESTVECTOR; + } + if (tested == 0) { + return CRYPT_NOP; + } + return CRYPT_OK; + #endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/mem_neq.c b/optee_os/core/lib/libtomcrypt/src/misc/mem_neq.c new file mode 100644 index 0000000..8078a0a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/mem_neq.c @@ -0,0 +1,53 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file mem_neq.c + Compare two blocks of memory for inequality in constant time. + Steffen Jaeckel +*/ + +/** + Compare two blocks of memory for inequality in constant time. + + The usage is similar to that of standard memcmp, but you can only test + if the memory is equal or not - you can not determine by how much the + first different byte differs. + + This function shall be used to compare results of cryptographic + operations where inequality means most likely usage of a wrong key. + The execution time has therefore to be constant as otherwise + timing attacks could be possible. + + @param a The first memory region + @param b The second memory region + @param len The length of the area to compare (octets) + + @return 0 when a and b are equal for len bytes, 1 they are not equal. +*/ +int mem_neq(const void *a, const void *b, size_t len) +{ + unsigned char ret = 0; + const unsigned char* pa; + const unsigned char* pb; + + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + + pa = a; + pb = b; + + while (len-- > 0) { + ret |= *pa ^ *pb; + ++pa; + ++pb; + } + + ret |= ret >> 4; + ret |= ret >> 2; + ret |= ret >> 1; + ret &= 1; + + return ret; +} diff --git a/optee_os/core/lib/libtomcrypt/src/misc/padding/padding_depad.c b/optee_os/core/lib/libtomcrypt/src/misc/padding/padding_depad.c new file mode 100644 index 0000000..e3f7151 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/padding/padding_depad.c @@ -0,0 +1,90 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_PADDING + +/** + Remove padding from your data + + This depads your data. + + @param data The data to depad + @param length [in/out] The size of the data before/after (removing padding) + @param mode One of the LTC_PAD_xx flags + @return CRYPT_OK on success +*/ +int padding_depad(const unsigned char *data, unsigned long *length, unsigned long mode) +{ + unsigned long padded_length, unpadded_length, n; + unsigned char pad; + enum padding_type type; + + LTC_ARGCHK(data != NULL); + LTC_ARGCHK(length != NULL); + + padded_length = *length; + + type = mode & LTC_PAD_MASK; + + if (type < LTC_PAD_ONE_AND_ZERO) { + pad = data[padded_length - 1]; + + if (pad > padded_length || pad == 0) return CRYPT_INVALID_ARG; + + unpadded_length = padded_length - pad; + } else { + /* init pad to calm old compilers */ + pad = 0x0; + unpadded_length = padded_length; + } + + switch (type) { + case LTC_PAD_ANSI_X923: + pad = 0x0; + /* FALLTHROUGH */ + case LTC_PAD_PKCS7: + for (n = unpadded_length; n < padded_length - 1; ++n) { + if (data[n] != pad) return CRYPT_INVALID_PACKET; + } + break; +#ifdef LTC_RNG_GET_BYTES + case LTC_PAD_ISO_10126: + /* nop */ + break; +#endif + case LTC_PAD_SSH: + pad = 0x1; + for (n = unpadded_length; n < padded_length; ++n) { + if (data[n] != pad++) return CRYPT_INVALID_PACKET; + } + break; + case LTC_PAD_ONE_AND_ZERO: + while (unpadded_length > 0 && data[unpadded_length - 1] != 0x80) { + if (data[unpadded_length - 1] != 0x0) return CRYPT_INVALID_PACKET; + unpadded_length--; + } + if (unpadded_length == 0) return CRYPT_INVALID_PACKET; + unpadded_length--; + if (data[unpadded_length] != 0x80) return CRYPT_INVALID_PACKET; + break; + case LTC_PAD_ZERO: + case LTC_PAD_ZERO_ALWAYS: + while (unpadded_length > 0 && data[unpadded_length - 1] == 0x0) { + unpadded_length--; + } + if (type == LTC_PAD_ZERO_ALWAYS) { + if (unpadded_length == padded_length) return CRYPT_INVALID_PACKET; + if (data[unpadded_length] != 0x0) return CRYPT_INVALID_PACKET; + } + break; + default: + return CRYPT_INVALID_ARG; + } + + *length = unpadded_length; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/padding/padding_pad.c b/optee_os/core/lib/libtomcrypt/src/misc/padding/padding_pad.c new file mode 100644 index 0000000..7d8bbba --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/padding/padding_pad.c @@ -0,0 +1,151 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_PADDING + +/** + Determine the to-be-padded length. + + @param length [in/out] The size of the data before/after padding + @param mode Mask of (LTC_PAD_xxx | block_length) + @return CRYPT_OK on success +*/ +static int s_padding_padded_length(unsigned long *length, unsigned long mode) +{ + enum padding_type padding; + unsigned char pad, block_length, r, t; + + LTC_ARGCHK(length != NULL); + + block_length = mode & 0xff; + padding = mode & LTC_PAD_MASK; + r = *length % block_length; + + switch (padding) { + case LTC_PAD_ZERO: + if (r == 0) { + t = 0; + break; + } + /* FALLTHROUGH */ + case LTC_PAD_PKCS7: + case LTC_PAD_ONE_AND_ZERO: + case LTC_PAD_ZERO_ALWAYS: + case LTC_PAD_SSH: + t = 1; + break; +#ifdef LTC_RNG_GET_BYTES + case LTC_PAD_ISO_10126: + do { + if (rng_get_bytes(&t, sizeof(t), NULL) != sizeof(t)) { + return CRYPT_ERROR_READPRNG; + } + t %= (256 / block_length); + } while (t == 0); + break; +#endif + case LTC_PAD_ANSI_X923: + if (block_length != 16) { + return CRYPT_INVALID_ARG; + } + t = 1; + break; + default: + return CRYPT_INVALID_ARG; + } + + pad = (t * block_length) - r; + + if ((pad == 0) && (padding != LTC_PAD_ZERO)) { + pad = block_length; + } + + *length += pad; + + return CRYPT_OK; +} + +/** + Add padding to data. + + This pads your data. + + @param data The data to depad + @param length The size of the data before padding + @param padded_length [in/out] The size of the data available/after padding + @param mode One of the LTC_PAD_xx flags + @return CRYPT_OK on success +*/ +int padding_pad(unsigned char *data, unsigned long length, unsigned long* padded_length, unsigned long mode) +{ + unsigned long l, n; + enum padding_type type; + int err; + unsigned char diff, pad; + + LTC_ARGCHK(data != NULL); + LTC_ARGCHK(padded_length != NULL); + + l = length; + if ((err = s_padding_padded_length(&l, mode)) != CRYPT_OK) { + return err; + } + + type = mode & LTC_PAD_MASK; + + if (*padded_length < l) { +#ifdef LTC_RNG_GET_BYTES + if (type != LTC_PAD_ISO_10126) { + *padded_length = l; + } else { + *padded_length = length + 256; + } +#else + *padded_length = l; +#endif + return CRYPT_BUFFER_OVERFLOW; + } + + if (l - length > 255) return CRYPT_INVALID_ARG; + diff = (unsigned char)(l - length); + + switch (type) { + case LTC_PAD_PKCS7: + XMEMSET(&data[length], diff, diff); + break; +#ifdef LTC_RNG_GET_BYTES + case LTC_PAD_ISO_10126: + if (rng_get_bytes(&data[length], diff-1u, NULL) != diff-1u) { + return CRYPT_ERROR_READPRNG; + } + data[l-1] = diff; + break; +#endif + case LTC_PAD_ANSI_X923: + XMEMSET(&data[length], 0, diff-1); + data[l-1] = diff; + break; + case LTC_PAD_SSH: + pad = 0x1; + for (n = length; n < l; ++n) { + data[n] = pad++; + } + break; + case LTC_PAD_ONE_AND_ZERO: + XMEMSET(&data[length + 1], 0, diff); + data[length] = 0x80; + break; + case LTC_PAD_ZERO: + case LTC_PAD_ZERO_ALWAYS: + XMEMSET(&data[length], 0, diff); + break; + default: + return CRYPT_INVALID_ARG; + } + *padded_length = l; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes.c b/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes.c new file mode 100644 index 0000000..f332623 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes.c @@ -0,0 +1,73 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_PBES + +/** + Decrypt Data encrypted via either PBES1 or PBES2 + + @param arg The according PBES parameters + @param dec_data [out] The decrypted data + @param dec_size [in/out] The length of the encrypted resp. decrypted data + @return CRYPT_OK on success +*/ +int pbes_decrypt(const pbes_arg *arg, unsigned char *dec_data, unsigned long *dec_size) +{ + int err, hid, cid; + unsigned char k[32], *iv; + unsigned long klen, keylen, dlen; + long diff; + symmetric_CBC cbc; + + LTC_ARGCHK(arg != NULL); + LTC_ARGCHK(arg->type.kdf != NULL); + LTC_ARGCHK(dec_data != NULL); + LTC_ARGCHK(dec_size != NULL); + + hid = find_hash(arg->type.h); + if (hid == -1) return CRYPT_INVALID_HASH; + cid = find_cipher(arg->type.c); + if (cid == -1) return CRYPT_INVALID_CIPHER; + + klen = arg->type.keylen; + + /* RC2 special case */ + if (arg->key_bits != 0) { + /* We can't handle odd lengths of Key Bits */ + if ((arg->key_bits % 8) != 0) return CRYPT_INVALID_KEYSIZE; + /* Internally we use bytes, not bits */ + klen = arg->key_bits / 8; + } + keylen = klen; + + if (arg->iv != NULL) { + iv = arg->iv->data; + } else { + iv = k + klen; + klen += arg->type.blocklen; + } + + if (klen > sizeof(k)) return CRYPT_INVALID_ARG; + + if ((err = arg->type.kdf(arg->pwd, arg->pwdlen, arg->salt->data, arg->salt->size, arg->iterations, hid, k, &klen)) != CRYPT_OK) goto LBL_ERROR; + if ((err = cbc_start(cid, iv, k, keylen, 0, &cbc)) != CRYPT_OK) goto LBL_ERROR; + if ((err = cbc_decrypt(arg->enc_data->data, dec_data, arg->enc_data->size, &cbc)) != CRYPT_OK) goto LBL_ERROR; + if ((err = cbc_done(&cbc)) != CRYPT_OK) goto LBL_ERROR; + dlen = arg->enc_data->size; + if ((err = padding_depad(dec_data, &dlen, LTC_PAD_PKCS7)) != CRYPT_OK) goto LBL_ERROR; + diff = (long)arg->enc_data->size - (long)dlen; + if ((diff <= 0) || (diff > cipher_descriptor[cid]->block_length)) { + err = CRYPT_PK_INVALID_PADDING; + goto LBL_ERROR; + } + *dec_size = dlen; + return CRYPT_OK; + +LBL_ERROR: + zeromem(k, sizeof(k)); + zeromem(dec_data, *dec_size); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes1.c b/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes1.c new file mode 100644 index 0000000..f33a009 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes1.c @@ -0,0 +1,117 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_PBES + +static int s_pkcs_5_alg1_wrap(const unsigned char *password, unsigned long password_len, + const unsigned char *salt, unsigned long salt_len, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen) +{ + LTC_UNUSED_PARAM(salt_len); + return pkcs_5_alg1(password, password_len, salt, iteration_count, hash_idx, out, outlen); +} + +static int s_pkcs_12_wrap(const unsigned char *password, unsigned long password_len, + const unsigned char *salt, unsigned long salt_len, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen) +{ + int err; + /* convert password to unicode/utf16-be */ + unsigned long pwlen = password_len * 2; + unsigned char* pw; + if (*outlen < 32) return CRYPT_INVALID_ARG; + pw = XMALLOC(pwlen + 2); + if (pw == NULL) return CRYPT_MEM; + if ((err = pkcs12_utf8_to_utf16(password, password_len, pw, &pwlen)) != CRYPT_OK) goto LBL_ERROR; + pw[pwlen++] = 0; + pw[pwlen++] = 0; + /* derive KEY */ + if ((err = pkcs12_kdf(hash_idx, pw, pwlen, salt, salt_len, iteration_count, 1, out, 24)) != CRYPT_OK) goto LBL_ERROR; + /* derive IV */ + if ((err = pkcs12_kdf(hash_idx, pw, pwlen, salt, salt_len, iteration_count, 2, out+24, 8)) != CRYPT_OK) goto LBL_ERROR; + + *outlen = 32; +LBL_ERROR: + zeromem(pw, pwlen); + XFREE(pw); + return err; +} + +static const pbes_properties s_pbes1_types[] = { + { s_pkcs_5_alg1_wrap, "md2", "des", 8, 8 }, + { s_pkcs_5_alg1_wrap, "md2", "rc2", 8, 8 }, + { s_pkcs_5_alg1_wrap, "md5", "des", 8, 8 }, + { s_pkcs_5_alg1_wrap, "md5", "rc2", 8, 8 }, + { s_pkcs_5_alg1_wrap, "sha1", "des", 8, 8 }, + { s_pkcs_5_alg1_wrap, "sha1", "rc2", 8, 8 }, + { s_pkcs_12_wrap, "sha1", "3des", 24, 8 }, +}; + +typedef struct { + const pbes_properties *data; + const char *oid; +} oid_to_pbes; + +static const oid_to_pbes s_pbes1_list[] = { + { &s_pbes1_types[0], "1.2.840.113549.1.5.1" }, /* http://www.oid-info.com/get/1.2.840.113549.1.5.1 pbeWithMD2AndDES-CBC */ + { &s_pbes1_types[1], "1.2.840.113549.1.5.4" }, /* http://www.oid-info.com/get/1.2.840.113549.1.5.4 pbeWithMD2AndRC2-CBC */ + { &s_pbes1_types[2], "1.2.840.113549.1.5.3" }, /* http://www.oid-info.com/get/1.2.840.113549.1.5.3 pbeWithMD5AndDES-CBC */ + { &s_pbes1_types[3], "1.2.840.113549.1.5.6" }, /* http://www.oid-info.com/get/1.2.840.113549.1.5.6 pbeWithMD5AndRC2-CBC */ + { &s_pbes1_types[4], "1.2.840.113549.1.5.10" }, /* http://www.oid-info.com/get/1.2.840.113549.1.5.10 pbeWithSHA1AndDES-CBC */ + { &s_pbes1_types[5], "1.2.840.113549.1.5.11" }, /* http://www.oid-info.com/get/1.2.840.113549.1.5.11 pbeWithSHA1AndRC2-CBC */ + { &s_pbes1_types[6], "1.2.840.113549.1.12.1.3" }, /* http://www.oid-info.com/get/1.2.840.113549.1.12.1.3 pbeWithSHAAnd3-KeyTripleDES-CBC */ + { 0 }, +}; + +static int s_pbes1_from_oid(const ltc_asn1_list *oid, pbes_properties *res) +{ + unsigned int i; + for (i = 0; s_pbes1_list[i].data != NULL; ++i) { + if (pk_oid_cmp_with_asn1(s_pbes1_list[i].oid, oid) == CRYPT_OK) { + if (res != NULL) *res = *s_pbes1_list[i].data; + return CRYPT_OK; + } + } + return CRYPT_INVALID_ARG; +} + +/** + Extract PBES1 parameters + + @param s The start of the sequence with potential PBES1 parameters + @param res Pointer to where the extracted parameters should be stored + @return CRYPT_OK on success +*/ +int pbes1_extract(const ltc_asn1_list *s, pbes_arg *res) +{ + int err; + + LTC_ARGCHK(s != NULL); + LTC_ARGCHK(res != NULL); + + if ((err = s_pbes1_from_oid(s, &res->type)) != CRYPT_OK) return err; + + if (!LTC_ASN1_IS_TYPE(s->next, LTC_ASN1_SEQUENCE) || + !LTC_ASN1_IS_TYPE(s->next->child, LTC_ASN1_OCTET_STRING) || + !LTC_ASN1_IS_TYPE(s->next->child->next, LTC_ASN1_INTEGER)) { + return CRYPT_INVALID_PACKET; + } + /* PBES1: encrypted pkcs8 - pbeWithMD5AndDES-CBC: + * 0:d=0 hl=4 l= 329 cons: SEQUENCE + * 4:d=1 hl=2 l= 27 cons: SEQUENCE + * 6:d=2 hl=2 l= 9 prim: OBJECT :pbeWithMD5AndDES-CBC (== 1.2.840.113549.1.5.3) (== *s) + * 17:d=2 hl=2 l= 14 cons: SEQUENCE (== *lalgparam) + * 19:d=3 hl=2 l= 8 prim: OCTET STRING [HEX DUMP]:8EDF749A06CCDE51 (== salt) + * 29:d=3 hl=2 l= 2 prim: INTEGER :0800 (== iterations) + * 33:d=1 hl=4 l= 296 prim: OCTET STRING :bytes (== encrypted data) + */ + res->salt = s->next->child; + res->iterations = mp_get_int(s->next->child->next->data); + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes2.c b/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes2.c new file mode 100644 index 0000000..3378cd6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pbes/pbes2.c @@ -0,0 +1,198 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_PBES + +static const char * const s_oid_pbes2 = "1.2.840.113549.1.5.13"; +static const char * const s_oid_pbkdf2 = "1.2.840.113549.1.5.12"; + +typedef struct { + const char *oid; + const char *id; +} oid_id_st; + +static const oid_id_st s_hmac_oid_names[] = { + { "1.2.840.113549.2.7", "sha1" }, + { "1.2.840.113549.2.8", "sha224" }, + { "1.2.840.113549.2.9", "sha256" }, + { "1.2.840.113549.2.10", "sha384" }, + { "1.2.840.113549.2.11", "sha512" }, + { "1.2.840.113549.2.12", "sha512-224" }, + { "1.2.840.113549.2.13", "sha512-256" }, +}; + +static const pbes_properties s_pbes2_default_types[] = { + { pkcs_5_alg2, "sha1", "des", 8, 0 }, + { pkcs_5_alg2, "sha1", "rc2", 4, 0 }, + { pkcs_5_alg2, "sha1", "3des", 24, 0 }, + { pkcs_5_alg2, "sha1", "aes", 16, 0 }, + { pkcs_5_alg2, "sha1", "aes", 24, 0 }, + { pkcs_5_alg2, "sha1", "aes", 32, 0 }, +}; + +typedef struct { + const pbes_properties *data; + const char* oid; +} oid_to_pbes; + +static const oid_to_pbes s_pbes2_list[] = { + { &s_pbes2_default_types[0], "1.3.14.3.2.7" }, /* http://www.oid-info.com/get/1.3.14.3.2.7 desCBC */ + { &s_pbes2_default_types[1], "1.2.840.113549.3.2" }, /* http://www.oid-info.com/get/1.2.840.113549.3.2 rc2CBC */ + { &s_pbes2_default_types[2], "1.2.840.113549.3.7" }, /* http://www.oid-info.com/get/1.2.840.113549.3.7 des-EDE3-CBC */ + { &s_pbes2_default_types[3], "2.16.840.1.101.3.4.1.2" }, /* http://www.oid-info.com/get/2.16.840.1.101.3.4.1.2 aes128-CBC */ + { &s_pbes2_default_types[4], "2.16.840.1.101.3.4.1.22" }, /* http://www.oid-info.com/get/2.16.840.1.101.3.4.1.22 aes192-CBC */ + { &s_pbes2_default_types[5], "2.16.840.1.101.3.4.1.42" }, /* http://www.oid-info.com/get/2.16.840.1.101.3.4.1.42 aes256-CBC */ +}; + +static int s_pbes2_from_oid(const ltc_asn1_list *cipher_oid, const ltc_asn1_list *hmac_oid, pbes_properties *res) +{ + unsigned int i; + for (i = 0; i < sizeof(s_pbes2_list)/sizeof(s_pbes2_list[0]); ++i) { + if (pk_oid_cmp_with_asn1(s_pbes2_list[i].oid, cipher_oid) == CRYPT_OK) { + *res = *s_pbes2_list[i].data; + break; + } + } + if (res->c == NULL) return CRYPT_INVALID_CIPHER; + if (hmac_oid != NULL) { + for (i = 0; i < sizeof(s_hmac_oid_names)/sizeof(s_hmac_oid_names[0]); ++i) { + if (pk_oid_cmp_with_asn1(s_hmac_oid_names[i].oid, hmac_oid) == CRYPT_OK) { + res->h = s_hmac_oid_names[i].id; + return CRYPT_OK; + } + } + return CRYPT_INVALID_HASH; + } + return CRYPT_OK; +} + + +/** + Extract PBES2 parameters + + @param s The start of the sequence with potential PBES2 parameters + @param res Pointer to where the extracted parameters should be stored + @return CRYPT_OK on success +*/ +int pbes2_extract(const ltc_asn1_list *s, pbes_arg *res) +{ + unsigned long klen; + ltc_asn1_list *lkdf, *lenc, *loptseq, *liter, *lhmac; + int err; + + LTC_ARGCHK(s != NULL); + LTC_ARGCHK(res != NULL); + + if ((err = pk_oid_cmp_with_asn1(s_oid_pbes2, s)) != CRYPT_OK) return err; + + if (!LTC_ASN1_IS_TYPE(s->next, LTC_ASN1_SEQUENCE) || + !LTC_ASN1_IS_TYPE(s->next->child, LTC_ASN1_SEQUENCE) || + !LTC_ASN1_IS_TYPE(s->next->child->child, LTC_ASN1_OBJECT_IDENTIFIER) || + !LTC_ASN1_IS_TYPE(s->next->child->child->next, LTC_ASN1_SEQUENCE) || + !LTC_ASN1_IS_TYPE(s->next->child->next, LTC_ASN1_SEQUENCE) || + !LTC_ASN1_IS_TYPE(s->next->child->next->child, LTC_ASN1_OBJECT_IDENTIFIER)) { + return CRYPT_INVALID_PACKET; + } + /* PBES2: encrypted pkcs8 - PBES2+PBKDF2+des-ede3-cbc: + * 0:d=0 hl=4 l= 380 cons: SEQUENCE + * 4:d=1 hl=2 l= 78 cons: SEQUENCE + * 6:d=2 hl=2 l= 9 prim: OBJECT :PBES2 (== 1.2.840.113549.1.5.13) (== *s) + * 17:d=2 hl=2 l= 65 cons: SEQUENCE + * 19:d=3 hl=2 l= 41 cons: SEQUENCE + * 21:d=4 hl=2 l= 9 prim: OBJECT :PBKDF2 (== *lkdf) + * 32:d=4 hl=2 l= 28 cons: SEQUENCE + * 34:d=5 hl=2 l= 8 prim: OCTET STRING [HEX DUMP]:28BA4ABF6AA76A3D (== res->salt) + * 44:d=5 hl=2 l= 2 prim: INTEGER :0800 (== res->iterations, *liter) + * 48:d=5 hl=2 l= 12 cons: SEQUENCE (== *loptseq - this sequence is optional, may be missing) + * 50:d=6 hl=2 l= 8 prim: OBJECT :hmacWithSHA256 (== *lhmac) + * 60:d=6 hl=2 l= 0 prim: NULL + * 62:d=3 hl=2 l= 20 cons: SEQUENCE + * 64:d=4 hl=2 l= 8 prim: OBJECT :des-ede3-cbc (== *lenc) + * 74:d=4 hl=2 l= 8 prim: OCTET STRING [HEX DUMP]:B1404C4688DC9A5A + * 84:d=1 hl=4 l= 296 prim: OCTET STRING :bytes (== encrypted data) + */ + lkdf = s->next->child->child; + lenc = s->next->child->next->child; + + if ((err = pk_oid_cmp_with_asn1(s_oid_pbkdf2, lkdf)) != CRYPT_OK) return err; + + if (!LTC_ASN1_IS_TYPE(lkdf->next, LTC_ASN1_SEQUENCE) || + !LTC_ASN1_IS_TYPE(lkdf->next->child, LTC_ASN1_OCTET_STRING) || + !LTC_ASN1_IS_TYPE(lkdf->next->child->next, LTC_ASN1_INTEGER)) { + return CRYPT_INVALID_PACKET; + } + + liter = lkdf->next->child->next; + loptseq = liter->next; + res->salt = lkdf->next->child; + res->iterations = mp_get_int(liter->data); + + /* There's an optional INTEGER keyLength after the iterations, skip that if it's there. + * c.f. RFC 2898 A.2 PBKDF2 */ + if(LTC_ASN1_IS_TYPE(loptseq, LTC_ASN1_INTEGER)) { + loptseq = loptseq->next; + } + + /* this sequence is optional */ + lhmac = NULL; + if (LTC_ASN1_IS_TYPE(loptseq, LTC_ASN1_SEQUENCE) && + LTC_ASN1_IS_TYPE(loptseq->child, LTC_ASN1_OBJECT_IDENTIFIER)) { + lhmac = loptseq->child; + } + if ((err = s_pbes2_from_oid(lenc, lhmac, &res->type)) != CRYPT_OK) return err; + + if (LTC_ASN1_IS_TYPE(lenc->next, LTC_ASN1_OCTET_STRING)) { + /* 'NON-RC2'-CBC */ + res->iv = lenc->next; + } else if (LTC_ASN1_IS_TYPE(lenc->next, LTC_ASN1_SEQUENCE)) { + /* RC2-CBC is a bit special ... + * + * RC2-CBC-Parameter ::= SEQUENCE { + * rc2ParameterVersion INTEGER OPTIONAL, + * iv OCTET STRING (SIZE(8)) } + */ + if (LTC_ASN1_IS_TYPE(lenc->next->child, LTC_ASN1_INTEGER) && + LTC_ASN1_IS_TYPE(lenc->next->child->next, LTC_ASN1_OCTET_STRING)) { + klen = mp_get_int(lenc->next->child->data); + res->iv = lenc->next->child->next; + /* + * Effective Key Bits Encoding + * 40 160 + * 64 120 + * 128 58 + * b >= 256 b + */ + switch (klen) { + case 160: + res->key_bits = 40; + break; + case 120: + res->key_bits = 64; + break; + case 58: + res->key_bits = 128; + break; + default: + /* We don't handle undefined Key Bits */ + if (klen < 256) return CRYPT_INVALID_KEYSIZE; + + res->key_bits = klen; + break; + } + } else if (LTC_ASN1_IS_TYPE(lenc->next->child, LTC_ASN1_OCTET_STRING)) { + res->iv = lenc->next->child; + /* + * If the rc2ParameterVersion field is omitted, the "effective key bits" + * defaults to 32. + */ + res->key_bits = 32; + } else { + return CRYPT_INVALID_PACKET; + } + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pbes/sub.mk b/optee_os/core/lib/libtomcrypt/src/misc/pbes/sub.mk new file mode 100644 index 0000000..46557aa --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pbes/sub.mk @@ -0,0 +1,3 @@ +srcs-y += pbes.c +srcs-y += pbes1.c +srcs-y += pbes2.c diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/pkcs12_kdf.c b/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/pkcs12_kdf.c new file mode 100644 index 0000000..e278792 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/pkcs12_kdf.c @@ -0,0 +1,82 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_PKCS_12 + +int pkcs12_kdf( int hash_id, + const unsigned char *pw, unsigned long pwlen, + const unsigned char *salt, unsigned long saltlen, + unsigned int iterations, unsigned char purpose, + unsigned char *out, unsigned long outlen) +{ + unsigned long u = hash_descriptor[hash_id]->hashsize; + unsigned long v = hash_descriptor[hash_id]->blocksize; + unsigned long c = (outlen + u - 1) / u; + unsigned long Slen = ((saltlen + v - 1) / v) * v; + unsigned long Plen = ((pwlen + v - 1) / v) * v; + unsigned long k = (Plen + Slen) / v; + unsigned long Alen, keylen = 0; + unsigned int tmp, i, j, n; + unsigned char ch; + unsigned char D[MAXBLOCKSIZE], A[MAXBLOCKSIZE], B[MAXBLOCKSIZE]; + unsigned char *I, *key; + int err = CRYPT_ERROR; + + LTC_ARGCHK(pw != NULL); + LTC_ARGCHK(salt != NULL); + LTC_ARGCHK(out != NULL); + + key = XMALLOC(u * c); + I = XMALLOC(Plen + Slen); + if (key == NULL || I == NULL) goto DONE; + zeromem(key, u * c); + + for (i = 0; i < v; i++) D[i] = purpose; /* D - diversifier */ + for (i = 0; i < Slen; i++) I[i] = salt[i % saltlen]; + for (i = 0; i < Plen; i++) I[Slen + i] = pw[i % pwlen]; /* I = Salt || Pass */ + + for (i = 0; i < c; i++) { + Alen = sizeof(A); + err = hash_memory_multi(hash_id, A, &Alen, D, v, I, Slen + Plen, LTC_NULL); /* A = HASH(D || I) */ + if (err != CRYPT_OK) goto DONE; + for (j = 1; j < iterations; j++) { + err = hash_memory(hash_id, A, Alen, A, &Alen); /* A = HASH(A) */ + if (err != CRYPT_OK) goto DONE; + } + /* fill buffer B with A */ + for (j = 0; j < v; j++) B[j] = A[j % Alen]; + /* B += 1 */ + for (j = v; j > 0; j--) { + if (++B[j - 1] != 0) break; + } + /* I_n += B */ + for (n = 0; n < k; n++) { + ch = 0; + for (j = v; j > 0; j--) { + tmp = I[n * v + j - 1] + B[j - 1] + ch; + ch = (unsigned char)((tmp >> 8) & 0xFF); + I[n * v + j - 1] = (unsigned char)(tmp & 0xFF); + } + } + /* store derived key block */ + XMEMCPY(&key[keylen], A, Alen); + keylen += Alen; + } + + XMEMCPY(out, key, outlen); + err = CRYPT_OK; +DONE: + if (I) { + zeromem(I, Plen + Slen); + XFREE(I); + } + if (key) { + zeromem(key, u * c); + XFREE(key); + } + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/pkcs12_utf8_to_utf16.c b/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/pkcs12_utf8_to_utf16.c new file mode 100644 index 0000000..b84fe93 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/pkcs12_utf8_to_utf16.c @@ -0,0 +1,60 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_PKCS_12 + +int pkcs12_utf8_to_utf16(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) { + unsigned long len = 0; + const unsigned char* in_end = in + inlen; + const ulong32 offset[6] = { + 0x00000000UL, 0x00003080UL, 0x000E2080UL, + 0x03C82080UL, 0xFA082080UL, 0x82082080UL + }; + int err = CRYPT_ERROR; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + while (in < in_end) { + ulong32 ch = 0; + unsigned short extra = 0; /* 0 */ + if (*in >= 192) extra++; /* 1 */ + if (*in >= 224) extra++; /* 2 */ + if (*in >= 240) extra++; /* 3 */ + if (*in >= 248) extra++; /* 4 */ + if (*in >= 252) extra++; /* 5 */ + if (in + extra >= in_end) goto ERROR; + switch (extra) { + case 5: ch += *in++; ch <<= 6; + /* FALLTHROUGH */ + case 4: ch += *in++; ch <<= 6; + /* FALLTHROUGH */ + case 3: ch += *in++; ch <<= 6; + /* FALLTHROUGH */ + case 2: ch += *in++; ch <<= 6; + /* FALLTHROUGH */ + case 1: ch += *in++; ch <<= 6; + /* FALLTHROUGH */ + case 0: ch += *in++; + default: break; + } + ch -= offset[extra]; + if (ch > 0xFFFF) goto ERROR; + if (*outlen >= len + 2) { + out[len] = (unsigned short)((ch >> 8) & 0xFF); + out[len + 1] = (unsigned char)(ch & 0xFF); + } + len += 2; + } + + err = len > *outlen ? CRYPT_BUFFER_OVERFLOW : CRYPT_OK; + *outlen = len; +ERROR: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/sub.mk b/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/sub.mk new file mode 100644 index 0000000..92f782e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pkcs12/sub.mk @@ -0,0 +1,2 @@ +srcs-y += pkcs12_kdf.c +srcs-y += pkcs12_utf8_to_utf16.c diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_1.c b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_1.c new file mode 100644 index 0000000..a33c2a2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_1.c @@ -0,0 +1,185 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_5_1.c + PKCS #5, Algorithm #1, Tom St Denis +*/ +#ifdef LTC_PKCS_5 +/** + Execute PKCS #5 v1 in strict or OpenSSL EVP_BytesToKey()-compat mode. + + PKCS#5 v1 specifies that the output key length can be no larger than + the hash output length. OpenSSL unilaterally extended that by repeating + the hash process on a block-by-block basis for as long as needed to make + bigger keys. If you want to be compatible with KDF for e.g. "openssl enc", + you'll want that. + + If you want strict PKCS behavior, turn openssl_compat off. Or (more + likely), use one of the convenience functions below. + + @param password The password (or key) + @param password_len The length of the password (octet) + @param salt The salt (or nonce) which is 8 octets long + @param iteration_count The PKCS #5 v1 iteration count + @param hash_idx The index of the hash desired + @param out [out] The destination for this algorithm + @param outlen [in/out] The max size and resulting size of the algorithm output + @param openssl_compat [in] Whether or not to grow the key to the buffer size ala OpenSSL + @return CRYPT_OK if successful +*/ +static int s_pkcs_5_alg1_common(const unsigned char *password, + unsigned long password_len, + const unsigned char *salt, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen, + int openssl_compat) +{ + int err; + unsigned long x; + hash_state *md; + unsigned char *buf; + /* Storage vars in case we need to support > hashsize (OpenSSL compat) */ + unsigned long block = 0, iter; + /* How many bytes to put in the outbut buffer (convenience calc) */ + unsigned long outidx = 0, nb = 0; + + LTC_ARGCHK(password != NULL); + LTC_ARGCHK(salt != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if (iteration_count <= 0) { + return CRYPT_INVALID_ARG; + } + + /* test hash IDX */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + + /* allocate memory */ + md = XMALLOC(sizeof(hash_state)); + buf = XMALLOC(MAXBLOCKSIZE); + if (md == NULL || buf == NULL) { + if (md != NULL) { + XFREE(md); + } + if (buf != NULL) { + XFREE(buf); + } + return CRYPT_MEM; + } + + while(block * hash_descriptor[hash_idx]->hashsize < *outlen) { + + /* hash initial (maybe previous hash) + password + salt */ + if ((err = hash_descriptor[hash_idx]->init(md)) != CRYPT_OK) { + goto LBL_ERR; + } + /* in OpenSSL mode, we first hash the previous result for blocks 2-n */ + if (openssl_compat && block) { + if ((err = hash_descriptor[hash_idx]->process(md, buf, hash_descriptor[hash_idx]->hashsize)) != CRYPT_OK) { + goto LBL_ERR; + } + } + if ((err = hash_descriptor[hash_idx]->process(md, password, password_len)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->process(md, salt, 8)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->done(md, buf)) != CRYPT_OK) { + goto LBL_ERR; + } + + iter = iteration_count; + while (--iter) { + /* code goes here. */ + x = MAXBLOCKSIZE; + if ((err = hash_memory(hash_idx, buf, hash_descriptor[hash_idx]->hashsize, buf, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + /* limit the size of the copy to however many bytes we have left in + the output buffer (and how many bytes we have to copy) */ + outidx = block*hash_descriptor[hash_idx]->hashsize; + nb = hash_descriptor[hash_idx]->hashsize; + if(outidx+nb > *outlen) { + nb = *outlen - outidx; + } + if(nb > 0) { + XMEMCPY(out+outidx, buf, nb); + } + + block++; + if (!openssl_compat) { + break; + } + } + /* In strict mode, we always return the hashsize, in compat we filled it + as much as was requested, so we leave it alone. */ + if(!openssl_compat) { + *outlen = hash_descriptor[hash_idx]->hashsize; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(buf, MAXBLOCKSIZE); + zeromem(md, sizeof(hash_state)); +#endif + + XFREE(buf); + XFREE(md); + + return err; +} + +/** + Execute PKCS #5 v1 - Strict mode (no OpenSSL-compatible extension) + @param password The password (or key) + @param password_len The length of the password (octet) + @param salt The salt (or nonce) which is 8 octets long + @param iteration_count The PKCS #5 v1 iteration count + @param hash_idx The index of the hash desired + @param out [out] The destination for this algorithm + @param outlen [in/out] The max size and resulting size of the algorithm output + @return CRYPT_OK if successful +*/ +int pkcs_5_alg1(const unsigned char *password, unsigned long password_len, + const unsigned char *salt, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen) +{ + return s_pkcs_5_alg1_common(password, password_len, salt, iteration_count, + hash_idx, out, outlen, 0); +} + +/** + Execute PKCS #5 v1 - OpenSSL-extension-compatible mode + + Use this one if you need to derive keys as "openssl enc" does by default. + OpenSSL (for better or worse), uses MD5 as the hash and iteration_count=1. + @param password The password (or key) + @param password_len The length of the password (octet) + @param salt The salt (or nonce) which is 8 octets long + @param iteration_count The PKCS #5 v1 iteration count + @param hash_idx The index of the hash desired + @param out [out] The destination for this algorithm + @param outlen [in/out] The max size and resulting size of the algorithm output + @return CRYPT_OK if successful +*/ +int pkcs_5_alg1_openssl(const unsigned char *password, + unsigned long password_len, + const unsigned char *salt, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen) +{ + return s_pkcs_5_alg1_common(password, password_len, salt, iteration_count, + hash_idx, out, outlen, 1); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_2.c b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_2.c new file mode 100644 index 0000000..61ebd00 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_2.c @@ -0,0 +1,121 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_5_2.c + PKCS #5, Algorithm #2, Tom St Denis +*/ +#ifdef LTC_PKCS_5 + +/** + Execute PKCS #5 v2 + @param password The input password (or key) + @param password_len The length of the password (octets) + @param salt The salt (or nonce) + @param salt_len The length of the salt (octets) + @param iteration_count # of iterations desired for PKCS #5 v2 [read specs for more] + @param hash_idx The index of the hash desired + @param out [out] The destination for this algorithm + @param outlen [in/out] The max size and resulting size of the algorithm output + @return CRYPT_OK if successful +*/ +int pkcs_5_alg2(const unsigned char *password, unsigned long password_len, + const unsigned char *salt, unsigned long salt_len, + int iteration_count, int hash_idx, + unsigned char *out, unsigned long *outlen) +{ + int err, itts; + ulong32 blkno; + unsigned long stored, left, x, y; + unsigned char *buf[2]; + hmac_state *hmac; + + LTC_ARGCHK(password != NULL); + LTC_ARGCHK(salt != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if (iteration_count <= 0) { + return CRYPT_INVALID_ARG; + } + + /* test hash IDX */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + + buf[0] = XMALLOC(MAXBLOCKSIZE * 2); + hmac = XMALLOC(sizeof(hmac_state)); + if (hmac == NULL || buf[0] == NULL) { + if (hmac != NULL) { + XFREE(hmac); + } + if (buf[0] != NULL) { + XFREE(buf[0]); + } + return CRYPT_MEM; + } + /* buf[1] points to the second block of MAXBLOCKSIZE bytes */ + buf[1] = buf[0] + MAXBLOCKSIZE; + + left = *outlen; + blkno = 1; + stored = 0; + while (left != 0) { + /* process block number blkno */ + zeromem(buf[0], MAXBLOCKSIZE*2); + + /* store current block number and increment for next pass */ + STORE32H(blkno, buf[1]); + ++blkno; + + /* get PRF(P, S||int(blkno)) */ + if ((err = hmac_init(hmac, hash_idx, password, password_len)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hmac_process(hmac, salt, salt_len)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hmac_process(hmac, buf[1], 4)) != CRYPT_OK) { + goto LBL_ERR; + } + x = MAXBLOCKSIZE; + if ((err = hmac_done(hmac, buf[0], &x)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* now compute repeated and XOR it in buf[1] */ + XMEMCPY(buf[1], buf[0], x); + for (itts = 1; itts < iteration_count; ++itts) { + if ((err = hmac_memory(hash_idx, password, password_len, buf[0], x, buf[0], &x)) != CRYPT_OK) { + goto LBL_ERR; + } + for (y = 0; y < x; y++) { + buf[1][y] ^= buf[0][y]; + } + } + + /* now emit upto x bytes of buf[1] to output */ + for (y = 0; y < x && left != 0; ++y) { + out[stored++] = buf[1][y]; + --left; + } + } + *outlen = stored; + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(buf[0], MAXBLOCKSIZE*2); + zeromem(hmac, sizeof(hmac_state)); +#endif + + XFREE(hmac); + XFREE(buf[0]); + + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_test.c b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_test.c new file mode 100644 index 0000000..73bc4a1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/pkcs_5_test.c @@ -0,0 +1,221 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file hkdf_test.c + PKCS #5 support, self-test, Steffen Jaeckel +*/ + +#ifdef LTC_PKCS_5 + +/* + TEST CASES SOURCE: + +Internet Engineering Task Force (IETF) S. Josefsson +Request for Comments: 6070 SJD AB +Category: Informational January 2011 +ISSN: 2070-1721 +*/ + +/** + PKCS #5 self-test + @return CRYPT_OK if successful, CRYPT_NOP if tests have been disabled. +*/ +int pkcs_5_test (void) +{ + #ifndef LTC_TEST + return CRYPT_NOP; + #else + + typedef struct { + const char* P; + unsigned long P_len; + const char* S; + unsigned long S_len; + int c; + unsigned long dkLen; + unsigned char DK[40]; + } case_item; + + static const case_item cases_5_2[] = { + { + "password", + 8, + "salt", + 4, + 1, + 20, + { 0x0c, 0x60, 0xc8, 0x0f, 0x96, 0x1f, 0x0e, 0x71, + 0xf3, 0xa9, 0xb5, 0x24, 0xaf, 0x60, 0x12, 0x06, + 0x2f, 0xe0, 0x37, 0xa6 } + }, + { + "password", + 8, + "salt", + 4, + 2, + 20, + { 0xea, 0x6c, 0x01, 0x4d, 0xc7, 0x2d, 0x6f, 0x8c, + 0xcd, 0x1e, 0xd9, 0x2a, 0xce, 0x1d, 0x41, 0xf0, + 0xd8, 0xde, 0x89, 0x57 } + }, +#ifdef LTC_TEST_EXT + { + "password", + 8, + "salt", + 4, + 4096, + 20, + { 0x4b, 0x00, 0x79, 0x01, 0xb7, 0x65, 0x48, 0x9a, + 0xbe, 0xad, 0x49, 0xd9, 0x26, 0xf7, 0x21, 0xd0, + 0x65, 0xa4, 0x29, 0xc1 } + }, + { + "password", + 8, + "salt", + 4, + 16777216, + 20, + { 0xee, 0xfe, 0x3d, 0x61, 0xcd, 0x4d, 0xa4, 0xe4, + 0xe9, 0x94, 0x5b, 0x3d, 0x6b, 0xa2, 0x15, 0x8c, + 0x26, 0x34, 0xe9, 0x84 } + }, + { + "passwordPASSWORDpassword", + 25, + "saltSALTsaltSALTsaltSALTsaltSALTsalt", + 36, + 4096, + 25, + { 0x3d, 0x2e, 0xec, 0x4f, 0xe4, 0x1c, 0x84, 0x9b, + 0x80, 0xc8, 0xd8, 0x36, 0x62, 0xc0, 0xe4, 0x4a, + 0x8b, 0x29, 0x1a, 0x96, 0x4c, 0xf2, 0xf0, 0x70, + 0x38 } + }, + { + "pass\0word", + 9, + "sa\0lt", + 5, + 4096, + 16, + { 0x56, 0xfa, 0x6a, 0xa7, 0x55, 0x48, 0x09, 0x9d, + 0xcc, 0x37, 0xd7, 0xf0, 0x34, 0x25, 0xe0, 0xc3 } + }, +#endif /* LTC_TEST_EXT */ + }; + + static const case_item cases_5_1[] = { + { + "password", + 8, + "saltsalt", /* must be 8 octects */ + 8, /* ignored by alg1 */ + 1, + 20, + { 0xca, 0xb8, 0x6d, 0xd6, 0x26, 0x17, 0x10, 0x89, 0x1e, 0x8c, + 0xb5, 0x6e, 0xe3, 0x62, 0x56, 0x91, 0xa7, 0x5d, 0xf3, 0x44 } + }, + }; + + static const case_item cases_5_1o[] = { + { + "password", + 8, + "saltsalt", /* must be 8 octects */ + 8, /* ignored by alg1_openssl */ + 1, + 20, + { 0xca, 0xb8, 0x6d, 0xd6, 0x26, 0x17, 0x10, 0x89, 0x1e, 0x8c, + 0xb5, 0x6e, 0xe3, 0x62, 0x56, 0x91, 0xa7, 0x5d, 0xf3, 0x44 } + + }, + { + "password", + 8, + "saltsalt", /* must be 8 octects */ + 8, /* ignored by alg1_openssl */ + 1, + 30, + { 0xca, 0xb8, 0x6d, 0xd6, 0x26, 0x17, 0x10, 0x89, 0x1e, 0x8c, + 0xb5, 0x6e, 0xe3, 0x62, 0x56, 0x91, 0xa7, 0x5d, 0xf3, 0x44, + 0xf0, 0xbf, 0xf4, 0xc1, 0x2c, 0xf3, 0x59, 0x6f, 0xc0, 0x0b } + + } + }; + + unsigned char DK[40]; + unsigned long dkLen; + int i, err; + int tested=0, failed=0; + int hash = find_hash("sha1"); + if (hash == -1) + { +#ifdef LTC_TEST_DBG + printf("PKCS#5 test failed: 'sha1' hash not found\n"); +#endif + return CRYPT_ERROR; + } + + /* testing alg 2 */ + for(i=0; i < (int)(sizeof(cases_5_2) / sizeof(cases_5_2[0])); i++) { + ++tested; + dkLen = cases_5_2[i].dkLen; + if((err = pkcs_5_alg2((unsigned char*)cases_5_2[i].P, cases_5_2[i].P_len, + (unsigned char*)cases_5_2[i].S, cases_5_2[i].S_len, + cases_5_2[i].c, hash, + DK, &dkLen)) != CRYPT_OK) { +#ifdef LTC_TEST_DBG + printf("\npkcs_5_alg2() #%d: Failed/1 (%s)\n", i, error_to_string(err)); +#endif + ++failed; + } + else if (compare_testvector(DK, dkLen, cases_5_2[i].DK, cases_5_2[i].dkLen, "PKCS#5_2", i)) { + ++failed; + } + } + + /* testing alg 1 */ + for(i=0; i < (int)(sizeof(cases_5_1) / sizeof(case_item)); i++, tested++) { + dkLen = cases_5_1[i].dkLen; + if((err = pkcs_5_alg1((unsigned char*)cases_5_1[i].P, cases_5_1[i].P_len, + (unsigned char*)cases_5_1[i].S, + cases_5_1[i].c, hash, + DK, &dkLen)) != CRYPT_OK) { +#ifdef LTC_TEST_DBG + printf("\npkcs_5_alg1() #%d: Failed/1 (%s)\n", i, error_to_string(err)); +#endif + ++failed; + } + else if (compare_testvector(DK, dkLen, cases_5_1[i].DK, cases_5_1[i].dkLen, "PKCS#5_1", i)) { + ++failed; + } + } + + /* testing alg 1_openssl */ + for(i = 0; i < (int)(sizeof(cases_5_1o) / sizeof(cases_5_1o[0])); i++, tested++) { + dkLen = cases_5_1o[i].dkLen; + if ((err = pkcs_5_alg1_openssl((unsigned char*)cases_5_1o[i].P, cases_5_1o[i].P_len, + (unsigned char*)cases_5_1o[i].S, + cases_5_1o[i].c, hash, + DK, &dkLen)) != CRYPT_OK) { +#ifdef LTC_TEST_DBG + printf("\npkcs_5_alg1_openssl() #%d: Failed/1 (%s)\n", i, error_to_string(err)); +#endif + ++failed; + } + else if (compare_testvector(DK, dkLen, cases_5_1o[i].DK, cases_5_1o[i].dkLen, "PKCS#5_1o", i)) { + ++failed; + } + } + + return (failed != 0) ? CRYPT_FAIL_TESTVECTOR : CRYPT_OK; + #endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/sub.mk b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/sub.mk new file mode 100644 index 0000000..7f8ccfe --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/pkcs5/sub.mk @@ -0,0 +1,2 @@ +srcs-y += pkcs_5_1.c +srcs-y += pkcs_5_2.c diff --git a/optee_os/core/lib/libtomcrypt/src/misc/ssh/ssh_decode_sequence_multi.c b/optee_os/core/lib/libtomcrypt/src/misc/ssh/ssh_decode_sequence_multi.c new file mode 100644 index 0000000..ff6fe3c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/ssh/ssh_decode_sequence_multi.c @@ -0,0 +1,167 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file ssh_decode_sequence_multi.c + SSH data type representation as per RFC4251, Russ Williams +*/ + +#ifdef LTC_SSH + +/** + Decode a SSH sequence using a VA list + @param in The input buffer + @param inlen [in/out] The length of the input buffer and on output the amount of decoded data + @remark <...> is of the form (int, ) except for string&name-list (int, void*, unsigned long*) + @return CRYPT_OK on success +*/ +int ssh_decode_sequence_multi(const unsigned char *in, unsigned long *inlen, ...) +{ + int err; + va_list args; + ssh_data_type type; + void *vdata; + unsigned char *cdata; + char *sdata; + ulong32 *u32data; + ulong64 *u64data; + unsigned long *bufsize; + ulong32 size; + unsigned long remaining; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != NULL); + + remaining = *inlen; + /* Decode values from buffer */ + va_start(args, inlen); + while ((type = (ssh_data_type)va_arg(args, int)) != LTC_SSHDATA_EOL) { + /* Size of length field */ + if (type == LTC_SSHDATA_STRING || + type == LTC_SSHDATA_NAMELIST || + type == LTC_SSHDATA_MPINT) + { + /* Check we'll not read too far */ + if (remaining < 4) { + err = CRYPT_BUFFER_OVERFLOW; + goto error; + } + } + + /* Calculate (or read) length of data */ + size = 0xFFFFFFFFU; + switch (type) { + case LTC_SSHDATA_BYTE: + case LTC_SSHDATA_BOOLEAN: + size = 1; + break; + case LTC_SSHDATA_UINT32: + size = 4; + break; + case LTC_SSHDATA_UINT64: + size = 8; + break; + case LTC_SSHDATA_STRING: + case LTC_SSHDATA_NAMELIST: + case LTC_SSHDATA_MPINT: + LOAD32H(size, in); + in += 4; + remaining -= 4; + break; + + case LTC_SSHDATA_EOL: + default: + /* Should never get here */ + err = CRYPT_INVALID_ARG; + goto error; + } + + /* Check we'll not read too far */ + if (remaining < size) { + err = CRYPT_BUFFER_OVERFLOW; + goto error; + } else { + remaining -= size; + } + + vdata = va_arg(args, void*); + if (vdata == NULL) { + err = CRYPT_INVALID_ARG; + goto error; + } + + /* Read data */ + switch (type) { + case LTC_SSHDATA_BYTE: + cdata = vdata; + *cdata = *in++; + break; + case LTC_SSHDATA_BOOLEAN: + cdata = vdata; + /* + The value 0 represents FALSE, and the value 1 represents TRUE. All non-zero values MUST be + interpreted as TRUE; however, applications MUST NOT store values other than 0 and 1. + */ + *cdata = (*in++)?1:0; + break; + case LTC_SSHDATA_UINT32: + u32data = vdata; + LOAD32H(*u32data, in); + in += 4; + break; + case LTC_SSHDATA_UINT64: + u64data = vdata; + LOAD64H(*u64data, in); + in += 8; + break; + case LTC_SSHDATA_STRING: + case LTC_SSHDATA_NAMELIST: + sdata = vdata; + bufsize = va_arg(args, unsigned long*); + if (bufsize == NULL) { + err = CRYPT_INVALID_ARG; + goto error; + } + if (size + 1 >= *bufsize) { + err = CRYPT_BUFFER_OVERFLOW; + goto error; + } + if (size > 0) { + XMEMCPY(sdata, (const char *)in, size); + } + sdata[size] = '\0'; + *bufsize = size; + in += size; + break; + case LTC_SSHDATA_MPINT: + if (size == 0) { + if ((err = mp_set(vdata, 0)) != CRYPT_OK) { goto error; } + } else if ((in[0] & 0x80) != 0) { + /* Negative number - not supported */ + err = CRYPT_INVALID_PACKET; + goto error; + } else { + if ((err = mp_read_unsigned_bin(vdata, (unsigned char *)in, size)) != CRYPT_OK) { goto error; } + } + in += size; + break; + + case LTC_SSHDATA_EOL: + default: + /* Should never get here */ + err = CRYPT_INVALID_ARG; + goto error; + } + } + err = CRYPT_OK; + + *inlen -= remaining; + +error: + va_end(args); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/ssh/ssh_encode_sequence_multi.c b/optee_os/core/lib/libtomcrypt/src/misc/ssh/ssh_encode_sequence_multi.c new file mode 100644 index 0000000..c65402c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/ssh/ssh_encode_sequence_multi.c @@ -0,0 +1,160 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file ssh_encode_sequence_multi.c + SSH data type representation as per RFC4251, Russ Williams +*/ + +#ifdef LTC_SSH + +/** + Encode a SSH sequence using a VA list + @param out [out] Destination for data + @param outlen [in/out] Length of buffer and resulting length of output + @remark <...> is of the form (int, ) except for string&name-list (int, void*, unsigned long) + @return CRYPT_OK on success +*/ +int ssh_encode_sequence_multi(unsigned char *out, unsigned long *outlen, ...) +{ + int err; + va_list args; + ulong32 size; + ssh_data_type type; + void *vdata; + const char *sdata; + int idata; + ulong32 u32data; + ulong64 u64data; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* Check values and calculate output size */ + size = 0; + va_start(args, outlen); + while ((type = (ssh_data_type)va_arg(args, int)) != LTC_SSHDATA_EOL) { + switch (type) { + case LTC_SSHDATA_BYTE: + case LTC_SSHDATA_BOOLEAN: /* Both stored as 1 byte */ + LTC_UNUSED_PARAM( va_arg(args, int) ); + size++; + break; + case LTC_SSHDATA_UINT32: + LTC_UNUSED_PARAM( va_arg(args, ulong32) ); + size += 4; + break; + case LTC_SSHDATA_UINT64: + LTC_UNUSED_PARAM( va_arg(args, ulong64) ); + size += 8; + break; + case LTC_SSHDATA_STRING: + case LTC_SSHDATA_NAMELIST: + LTC_UNUSED_PARAM( va_arg(args, char*) ); + size += va_arg(args, unsigned long); + size += 4; + break; + case LTC_SSHDATA_MPINT: + vdata = va_arg(args, void*); + /* Calculate size */ + size += 4; + if (mp_iszero(vdata) != LTC_MP_YES) { + size += mp_unsigned_bin_size(vdata); + if ((mp_count_bits(vdata) & 7) == 0) size++; /* Zero padding if high bit set */ + } + break; + + case LTC_SSHDATA_EOL: /* Should never get here */ + default: + err = CRYPT_INVALID_ARG; + goto error; + } + } + va_end(args); + + /* Check we have sufficient space */ + if (*outlen < size) { + *outlen = size; + err = CRYPT_BUFFER_OVERFLOW; + goto errornoargs; + } + *outlen = size; + + /* Encode values into buffer */ + va_start(args, outlen); + while ((type = (ssh_data_type)va_arg(args, int)) != LTC_SSHDATA_EOL) { + switch (type) { + case LTC_SSHDATA_BYTE: + idata = va_arg(args, int); + + *out++ = (unsigned char)(idata & 255); + break; + case LTC_SSHDATA_BOOLEAN: + idata = va_arg(args, int); + + /* + The value 0 represents FALSE, and the value 1 represents TRUE. All non-zero values MUST be + interpreted as TRUE; however, applications MUST NOT store values other than 0 and 1. + */ + *out++ = (idata)?1:0; + break; + case LTC_SSHDATA_UINT32: + u32data = va_arg(args, ulong32); + STORE32H(u32data, out); + out += 4; + break; + case LTC_SSHDATA_UINT64: + u64data = va_arg(args, ulong64); + STORE64H(u64data, out); + out += 8; + break; + case LTC_SSHDATA_STRING: + case LTC_SSHDATA_NAMELIST: + sdata = va_arg(args, char*); + size = va_arg(args, unsigned long); + STORE32H(size, out); + out += 4; + XMEMCPY(out, sdata, size); + out += size; + break; + case LTC_SSHDATA_MPINT: + vdata = va_arg(args, void*); + if (mp_iszero(vdata) == LTC_MP_YES) { + STORE32H(0, out); + out += 4; + } else { + size = mp_unsigned_bin_size(vdata); + if ((mp_count_bits(vdata) & 7) == 0) { + /* Zero padding if high bit set */ + STORE32H(size+1, out); + out += 4; + *out++ = 0; + } else { + STORE32H(size, out); + out += 4; + } + if ((err = mp_to_unsigned_bin(vdata, out)) != CRYPT_OK) { + err = CRYPT_ERROR; + goto error; + } + out += size; + } + break; + + case LTC_SSHDATA_EOL: /* Should never get here */ + default: + err = CRYPT_INVALID_ARG; + goto error; + } + } + err = CRYPT_OK; + +error: + va_end(args); +errornoargs: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/misc/ssh/sub.mk b/optee_os/core/lib/libtomcrypt/src/misc/ssh/sub.mk new file mode 100644 index 0000000..b10fb7c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/ssh/sub.mk @@ -0,0 +1,2 @@ +srcs-y += ssh_decode_sequence_multi.c +srcs-y += ssh_encode_sequence_multi.c diff --git a/optee_os/core/lib/libtomcrypt/src/misc/sub.mk b/optee_os/core/lib/libtomcrypt/src/misc/sub.mk new file mode 100644 index 0000000..fc55c8e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/sub.mk @@ -0,0 +1,9 @@ +srcs-y += burn_stack.c +srcs-y += error_to_string.c +srcs-y += mem_neq.c +srcs-y += zeromem.c +subdirs-y += base64 +subdirs-y += crypt +subdirs-y += pkcs5 +subdirs-y += pkcs12 +subdirs-y += ssh diff --git a/optee_os/core/lib/libtomcrypt/src/misc/zeromem.c b/optee_os/core/lib/libtomcrypt/src/misc/zeromem.c new file mode 100644 index 0000000..8bb64ec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/misc/zeromem.c @@ -0,0 +1,20 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + +/** + @file zeromem.c + Zero a block of memory, Tom St Denis +*/ + +/** + Zero a block of memory + @param out The destination of the area to zero + @param outlen The length of the area to zero (octets) +*/ +void zeromem(volatile void *out, size_t outlen) +{ + LTC_ARGCHKVD(out != NULL); + memzero_explicit((void *)out, outlen); +} diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_decrypt.c new file mode 100644 index 0000000..4a7517a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_decrypt.c @@ -0,0 +1,84 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cbc_decrypt.c + CBC implementation, encrypt block, Tom St Denis +*/ + + +#ifdef LTC_CBC_MODE + +/** + CBC decrypt + @param ct Ciphertext + @param pt [out] Plaintext + @param len The number of bytes to process (must be multiple of block length) + @param cbc CBC state + @return CRYPT_OK if successful +*/ +int cbc_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_CBC *cbc) +{ + int x, err; + unsigned char tmp[16]; +#ifdef LTC_FAST + LTC_FAST_TYPE tmpy; +#else + unsigned char tmpy; +#endif + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(cbc != NULL); + + if ((err = cipher_is_valid(cbc->cipher)) != CRYPT_OK) { + return err; + } + + /* is blocklen valid? */ + if (cbc->blocklen < 1 || cbc->blocklen > (int)sizeof(cbc->IV) || cbc->blocklen > (int)sizeof(tmp)) { + return CRYPT_INVALID_ARG; + } + + if (len % cbc->blocklen) { + return CRYPT_INVALID_ARG; + } +#ifdef LTC_FAST + if (cbc->blocklen % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + if (cipher_descriptor[cbc->cipher]->accel_cbc_decrypt != NULL) { + return cipher_descriptor[cbc->cipher]->accel_cbc_decrypt(ct, pt, len / cbc->blocklen, cbc->IV, &cbc->key); + } + while (len) { + /* decrypt */ + if ((err = cipher_descriptor[cbc->cipher]->ecb_decrypt(ct, tmp, &cbc->key)) != CRYPT_OK) { + return err; + } + + /* xor IV against plaintext */ +#if defined(LTC_FAST) + for (x = 0; x < cbc->blocklen; x += sizeof(LTC_FAST_TYPE)) { + tmpy = *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)cbc->IV + x)) ^ *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)tmp + x)); + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)cbc->IV + x)) = *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)ct + x)); + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)pt + x)) = tmpy; + } +#else + for (x = 0; x < cbc->blocklen; x++) { + tmpy = tmp[x] ^ cbc->IV[x]; + cbc->IV[x] = ct[x]; + pt[x] = tmpy; + } +#endif + + ct += cbc->blocklen; + pt += cbc->blocklen; + len -= cbc->blocklen; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_done.c b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_done.c new file mode 100644 index 0000000..2b842f0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_done.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cbc_done.c + CBC implementation, finish chain, Tom St Denis +*/ + +#ifdef LTC_CBC_MODE + +/** Terminate the chain + @param cbc The CBC chain to terminate + @return CRYPT_OK on success +*/ +int cbc_done(symmetric_CBC *cbc) +{ + int err; + LTC_ARGCHK(cbc != NULL); + + if ((err = cipher_is_valid(cbc->cipher)) != CRYPT_OK) { + return err; + } + cipher_descriptor[cbc->cipher]->done(&cbc->key); + return CRYPT_OK; +} + + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_encrypt.c new file mode 100644 index 0000000..86e1481 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_encrypt.c @@ -0,0 +1,85 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cbc_encrypt.c + CBC implementation, encrypt block, Tom St Denis +*/ + + +#ifdef LTC_CBC_MODE + +/** + CBC encrypt + @param pt Plaintext + @param ct [out] Ciphertext + @param len The number of bytes to process (must be multiple of block length) + @param cbc CBC state + @return CRYPT_OK if successful +*/ +int cbc_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CBC *cbc) +{ + int x, err; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(cbc != NULL); + + if ((err = cipher_is_valid(cbc->cipher)) != CRYPT_OK) { + return err; + } + + /* is blocklen valid? */ + if (cbc->blocklen < 1 || cbc->blocklen > (int)sizeof(cbc->IV)) { + return CRYPT_INVALID_ARG; + } + + if (len % cbc->blocklen) { + return CRYPT_INVALID_ARG; + } +#ifdef LTC_FAST + if (cbc->blocklen % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + if (cipher_descriptor[cbc->cipher]->accel_cbc_encrypt != NULL) { + return cipher_descriptor[cbc->cipher]->accel_cbc_encrypt(pt, ct, len / cbc->blocklen, cbc->IV, &cbc->key); + } + while (len) { + /* xor IV against plaintext */ +#if defined(LTC_FAST) + for (x = 0; x < cbc->blocklen; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)cbc->IV + x)) ^= *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)pt + x)); + } +#else + for (x = 0; x < cbc->blocklen; x++) { + cbc->IV[x] ^= pt[x]; + } +#endif + + /* encrypt */ + if ((err = cipher_descriptor[cbc->cipher]->ecb_encrypt(cbc->IV, ct, &cbc->key)) != CRYPT_OK) { + return err; + } + + /* store IV [ciphertext] for a future block */ +#if defined(LTC_FAST) + for (x = 0; x < cbc->blocklen; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)cbc->IV + x)) = *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)ct + x)); + } +#else + for (x = 0; x < cbc->blocklen; x++) { + cbc->IV[x] = ct[x]; + } +#endif + + ct += cbc->blocklen; + pt += cbc->blocklen; + len -= cbc->blocklen; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_getiv.c b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_getiv.c new file mode 100644 index 0000000..7af2cf1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_getiv.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cbc_getiv.c + CBC implementation, get IV, Tom St Denis +*/ + +#ifdef LTC_CBC_MODE + +/** + Get the current initialization vector + @param IV [out] The destination of the initialization vector + @param len [in/out] The max size and resulting size of the initialization vector + @param cbc The CBC state + @return CRYPT_OK if successful +*/ +int cbc_getiv(unsigned char *IV, unsigned long *len, const symmetric_CBC *cbc) +{ + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(len != NULL); + LTC_ARGCHK(cbc != NULL); + if ((unsigned long)cbc->blocklen > *len) { + *len = cbc->blocklen; + return CRYPT_BUFFER_OVERFLOW; + } + XMEMCPY(IV, cbc->IV, cbc->blocklen); + *len = cbc->blocklen; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_setiv.c b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_setiv.c new file mode 100644 index 0000000..a9e91c3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_setiv.c @@ -0,0 +1,32 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cbc_setiv.c + CBC implementation, set IV, Tom St Denis +*/ + + +#ifdef LTC_CBC_MODE + +/** + Set an initialization vector + @param IV The initialization vector + @param len The length of the vector (in octets) + @param cbc The CBC state + @return CRYPT_OK if successful +*/ +int cbc_setiv(const unsigned char *IV, unsigned long len, symmetric_CBC *cbc) +{ + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(cbc != NULL); + if (len != (unsigned long)cbc->blocklen) { + return CRYPT_INVALID_ARG; + } + XMEMCPY(cbc->IV, IV, len); + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_start.c b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_start.c new file mode 100644 index 0000000..c0cec09 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cbc/cbc_start.c @@ -0,0 +1,50 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cbc_start.c + CBC implementation, start chain, Tom St Denis +*/ + +#ifdef LTC_CBC_MODE + +/** + Initialize a CBC context + @param cipher The index of the cipher desired + @param IV The initialization vector + @param key The secret key + @param keylen The length of the secret key (octets) + @param num_rounds Number of rounds in the cipher desired (0 for default) + @param cbc The CBC state to initialize + @return CRYPT_OK if successful +*/ +int cbc_start(int cipher, const unsigned char *IV, const unsigned char *key, + int keylen, int num_rounds, symmetric_CBC *cbc) +{ + int x, err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(cbc != NULL); + + /* bad param? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* setup cipher */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, num_rounds, &cbc->key)) != CRYPT_OK) { + return err; + } + + /* copy IV */ + cbc->blocklen = cipher_descriptor[cipher]->block_length; + cbc->cipher = cipher; + for (x = 0; x < cbc->blocklen; x++) { + cbc->IV[x] = IV[x]; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cbc/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/cbc/sub.mk new file mode 100644 index 0000000..1ce3e77 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cbc/sub.mk @@ -0,0 +1,6 @@ +srcs-y += cbc_decrypt.c +srcs-y += cbc_done.c +srcs-y += cbc_encrypt.c +srcs-y += cbc_getiv.c +srcs-y += cbc_setiv.c +srcs-y += cbc_start.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_decrypt.c new file mode 100644 index 0000000..ff735e3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_decrypt.c @@ -0,0 +1,55 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cfb_decrypt.c + CFB implementation, decrypt data, Tom St Denis +*/ + +#ifdef LTC_CFB_MODE + +/** + CFB decrypt + @param ct Ciphertext + @param pt [out] Plaintext + @param len Length of ciphertext (octets) + @param cfb CFB state + @return CRYPT_OK if successful +*/ +int cfb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_CFB *cfb) +{ + int err; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(cfb != NULL); + + if ((err = cipher_is_valid(cfb->cipher)) != CRYPT_OK) { + return err; + } + + /* is blocklen/padlen valid? */ + if (cfb->blocklen < 0 || cfb->blocklen > (int)sizeof(cfb->IV) || + cfb->padlen < 0 || cfb->padlen > (int)sizeof(cfb->pad)) { + return CRYPT_INVALID_ARG; + } + + while (len-- > 0) { + if (cfb->padlen == cfb->blocklen) { + if ((err = cipher_descriptor[cfb->cipher]->ecb_encrypt(cfb->pad, cfb->IV, &cfb->key)) != CRYPT_OK) { + return err; + } + cfb->padlen = 0; + } + cfb->pad[cfb->padlen] = *ct; + *pt = *ct ^ cfb->IV[cfb->padlen]; + ++pt; + ++ct; + ++(cfb->padlen); + } + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_done.c b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_done.c new file mode 100644 index 0000000..28fe514 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_done.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cfb_done.c + CFB implementation, finish chain, Tom St Denis +*/ + +#ifdef LTC_CFB_MODE + +/** Terminate the chain + @param cfb The CFB chain to terminate + @return CRYPT_OK on success +*/ +int cfb_done(symmetric_CFB *cfb) +{ + int err; + LTC_ARGCHK(cfb != NULL); + + if ((err = cipher_is_valid(cfb->cipher)) != CRYPT_OK) { + return err; + } + cipher_descriptor[cfb->cipher]->done(&cfb->key); + return CRYPT_OK; +} + + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_encrypt.c new file mode 100644 index 0000000..b03b036 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_encrypt.c @@ -0,0 +1,53 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cfb_encrypt.c + CFB implementation, encrypt data, Tom St Denis +*/ + +#ifdef LTC_CFB_MODE + +/** + CFB encrypt + @param pt Plaintext + @param ct [out] Ciphertext + @param len Length of plaintext (octets) + @param cfb CFB state + @return CRYPT_OK if successful +*/ +int cfb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CFB *cfb) +{ + int err; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(cfb != NULL); + + if ((err = cipher_is_valid(cfb->cipher)) != CRYPT_OK) { + return err; + } + + /* is blocklen/padlen valid? */ + if (cfb->blocklen < 0 || cfb->blocklen > (int)sizeof(cfb->IV) || + cfb->padlen < 0 || cfb->padlen > (int)sizeof(cfb->pad)) { + return CRYPT_INVALID_ARG; + } + + while (len-- > 0) { + if (cfb->padlen == cfb->blocklen) { + if ((err = cipher_descriptor[cfb->cipher]->ecb_encrypt(cfb->pad, cfb->IV, &cfb->key)) != CRYPT_OK) { + return err; + } + cfb->padlen = 0; + } + cfb->pad[cfb->padlen] = (*ct = *pt ^ cfb->IV[cfb->padlen]); + ++pt; + ++ct; + ++(cfb->padlen); + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_getiv.c b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_getiv.c new file mode 100644 index 0000000..9dc2e86 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_getiv.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cfb_getiv.c + CFB implementation, get IV, Tom St Denis +*/ + +#ifdef LTC_CFB_MODE + +/** + Get the current initialization vector + @param IV [out] The destination of the initialization vector + @param len [in/out] The max size and resulting size of the initialization vector + @param cfb The CFB state + @return CRYPT_OK if successful +*/ +int cfb_getiv(unsigned char *IV, unsigned long *len, const symmetric_CFB *cfb) +{ + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(len != NULL); + LTC_ARGCHK(cfb != NULL); + if ((unsigned long)cfb->blocklen > *len) { + *len = cfb->blocklen; + return CRYPT_BUFFER_OVERFLOW; + } + XMEMCPY(IV, cfb->IV, cfb->blocklen); + *len = cfb->blocklen; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_setiv.c b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_setiv.c new file mode 100644 index 0000000..04306f5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_setiv.c @@ -0,0 +1,40 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cfb_setiv.c + CFB implementation, set IV, Tom St Denis +*/ + +#ifdef LTC_CFB_MODE + +/** + Set an initialization vector + @param IV The initialization vector + @param len The length of the vector (in octets) + @param cfb The CFB state + @return CRYPT_OK if successful +*/ +int cfb_setiv(const unsigned char *IV, unsigned long len, symmetric_CFB *cfb) +{ + int err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(cfb != NULL); + + if ((err = cipher_is_valid(cfb->cipher)) != CRYPT_OK) { + return err; + } + + if (len != (unsigned long)cfb->blocklen) { + return CRYPT_INVALID_ARG; + } + + /* force next block */ + cfb->padlen = 0; + return cipher_descriptor[cfb->cipher]->ecb_encrypt(IV, cfb->IV, &cfb->key); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_start.c b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_start.c new file mode 100644 index 0000000..fd924e0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cfb/cfb_start.c @@ -0,0 +1,54 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file cfb_start.c + CFB implementation, start chain, Tom St Denis +*/ + + +#ifdef LTC_CFB_MODE + +/** + Initialize a CFB context + @param cipher The index of the cipher desired + @param IV The initialization vector + @param key The secret key + @param keylen The length of the secret key (octets) + @param num_rounds Number of rounds in the cipher desired (0 for default) + @param cfb The CFB state to initialize + @return CRYPT_OK if successful +*/ +int cfb_start(int cipher, const unsigned char *IV, const unsigned char *key, + int keylen, int num_rounds, symmetric_CFB *cfb) +{ + int x, err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(cfb != NULL); + + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + + /* copy data */ + cfb->cipher = cipher; + cfb->blocklen = cipher_descriptor[cipher]->block_length; + for (x = 0; x < cfb->blocklen; x++) { + cfb->IV[x] = IV[x]; + } + + /* init the cipher */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, num_rounds, &cfb->key)) != CRYPT_OK) { + return err; + } + + /* encrypt the IV */ + cfb->padlen = 0; + return cipher_descriptor[cfb->cipher]->ecb_encrypt(cfb->IV, cfb->IV, &cfb->key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/cfb/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/cfb/sub.mk new file mode 100644 index 0000000..7a92b01 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/cfb/sub.mk @@ -0,0 +1,6 @@ +srcs-y += cfb_decrypt.c +srcs-y += cfb_done.c +srcs-y += cfb_encrypt.c +srcs-y += cfb_getiv.c +srcs-y += cfb_setiv.c +srcs-y += cfb_start.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_decrypt.c new file mode 100644 index 0000000..a55a08f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_decrypt.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ctr_decrypt.c + CTR implementation, decrypt data, Tom St Denis +*/ + +#ifdef LTC_CTR_MODE + +/** + CTR decrypt + @param ct Ciphertext + @param pt [out] Plaintext + @param len Length of ciphertext (octets) + @param ctr CTR state + @return CRYPT_OK if successful +*/ +int ctr_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_CTR *ctr) +{ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(ctr != NULL); + + return ctr_encrypt(ct, pt, len, ctr); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_done.c b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_done.c new file mode 100644 index 0000000..3dbea53 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_done.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ctr_done.c + CTR implementation, finish chain, Tom St Denis +*/ + +#ifdef LTC_CTR_MODE + +/** Terminate the chain + @param ctr The CTR chain to terminate + @return CRYPT_OK on success +*/ +int ctr_done(symmetric_CTR *ctr) +{ + int err; + LTC_ARGCHK(ctr != NULL); + + if ((err = cipher_is_valid(ctr->cipher)) != CRYPT_OK) { + return err; + } + cipher_descriptor[ctr->cipher]->done(&ctr->key); + return CRYPT_OK; +} + + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_encrypt.c new file mode 100644 index 0000000..5ef6915 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_encrypt.c @@ -0,0 +1,140 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ctr_encrypt.c + CTR implementation, encrypt data, Tom St Denis +*/ + + +#ifdef LTC_CTR_MODE + +static void s_ctr_increment_counter(symmetric_CTR *ctr) +{ + int x; + + if (ctr->mode == CTR_COUNTER_LITTLE_ENDIAN) { + for (x = 0; x < ctr->ctrlen; x++) { + ctr->ctr[x] = (ctr->ctr[x] + 1) & 0xff; + if (ctr->ctr[x]) + return; + } + } else { + for (x = ctr->blocklen - 1; x >= ctr->ctrlen; x--) { + ctr->ctr[x] = (ctr->ctr[x] + 1) & 0xff; + if (ctr->ctr[x]) { + return; + } + } + } +} + +/** + CTR encrypt software implementation + @param pt Plaintext + @param ct [out] Ciphertext + @param len Length of plaintext (octets) + @param ctr CTR state + @return CRYPT_OK if successful +*/ +static int s_ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CTR *ctr) +{ + int err; + + while (len) { + /* is the pad empty? */ + if (ctr->padlen == ctr->blocklen) { + /* encrypt counter into pad */ + if ((err = cipher_descriptor[ctr->cipher]->ecb_encrypt(ctr->ctr, ctr->pad, &ctr->key)) != CRYPT_OK) { + return err; + } + ctr->padlen = 0; + } +#ifdef LTC_FAST + if ((ctr->padlen == 0) && (len >= (unsigned long)ctr->blocklen)) { + for (x = 0; x < ctr->blocklen; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)ct + x)) = *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)pt + x)) ^ + *(LTC_FAST_TYPE_PTR_CAST((unsigned char *)ctr->pad + x)); + } + pt += ctr->blocklen; + ct += ctr->blocklen; + len -= ctr->blocklen; + ctr->padlen = ctr->blocklen; + continue; + } +#endif + *ct++ = *pt++ ^ ctr->pad[ctr->padlen++]; + --len; + + /* done with one full block? if so, set counter for next block. */ + if (ctr->padlen == ctr->blocklen) { + s_ctr_increment_counter(ctr); + } + } + return CRYPT_OK; +} + +/** + CTR encrypt + @param pt Plaintext + @param ct [out] Ciphertext + @param len Length of plaintext (octets) + @param ctr CTR state + @return CRYPT_OK if successful +*/ +int ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CTR *ctr) +{ + unsigned long incr; + int err; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(ctr != NULL); + + if ((err = cipher_is_valid(ctr->cipher)) != CRYPT_OK) { + return err; + } + + /* is blocklen/padlen valid? */ + if ((ctr->blocklen < 1) || (ctr->blocklen > (int)sizeof(ctr->ctr)) || + (ctr->padlen < 0) || (ctr->padlen > (int)sizeof(ctr->pad))) { + return CRYPT_INVALID_ARG; + } + +#ifdef LTC_FAST + if (ctr->blocklen % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + if (cipher_descriptor[ctr->cipher]->accel_ctr_encrypt != NULL ) { + /* handle acceleration only if not in the middle of a block, accelerator is present and length is >= a block size */ + if ((ctr->padlen == 0 || ctr->padlen == ctr->blocklen) && len >= (unsigned long)ctr->blocklen) { + if ((err = cipher_descriptor[ctr->cipher]->accel_ctr_encrypt(pt, ct, len/ctr->blocklen, ctr->ctr, ctr->mode, &ctr->key)) != CRYPT_OK) { + return err; + } + pt += (len / ctr->blocklen) * ctr->blocklen; + ct += (len / ctr->blocklen) * ctr->blocklen; + len %= ctr->blocklen; + /* counter was changed by accelerator so mark pad empty (will need updating in s_ctr_encrypt()) */ + ctr->padlen = ctr->blocklen; + } + + /* try to re-synchronize on a block boundary for maximum use of acceleration */ + incr = ctr->blocklen - ctr->padlen; + if (len >= incr + (unsigned long)ctr->blocklen) { + if ((err = s_ctr_encrypt(pt, ct, incr, ctr)) != CRYPT_OK) { + return err; + } + pt += incr; + ct += incr; + len -= incr; + return ctr_encrypt(pt, ct, len, ctr); + } + } + + return s_ctr_encrypt(pt, ct, len, ctr); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_getiv.c b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_getiv.c new file mode 100644 index 0000000..05277fa --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_getiv.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ctr_getiv.c + CTR implementation, get IV, Tom St Denis +*/ + +#ifdef LTC_CTR_MODE + +/** + Get the current initialization vector + @param IV [out] The destination of the initialization vector + @param len [in/out] The max size and resulting size of the initialization vector + @param ctr The CTR state + @return CRYPT_OK if successful +*/ +int ctr_getiv(unsigned char *IV, unsigned long *len, const symmetric_CTR *ctr) +{ + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(len != NULL); + LTC_ARGCHK(ctr != NULL); + if ((unsigned long)ctr->blocklen > *len) { + *len = ctr->blocklen; + return CRYPT_BUFFER_OVERFLOW; + } + XMEMCPY(IV, ctr->ctr, ctr->blocklen); + *len = ctr->blocklen; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_setiv.c b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_setiv.c new file mode 100644 index 0000000..7bdccf2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_setiv.c @@ -0,0 +1,44 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ctr_setiv.c + CTR implementation, set IV, Tom St Denis +*/ + +#ifdef LTC_CTR_MODE + +/** + Set an initialization vector + @param IV The initialization vector + @param len The length of the vector (in octets) + @param ctr The CTR state + @return CRYPT_OK if successful +*/ +int ctr_setiv(const unsigned char *IV, unsigned long len, symmetric_CTR *ctr) +{ + int err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(ctr != NULL); + + /* bad param? */ + if ((err = cipher_is_valid(ctr->cipher)) != CRYPT_OK) { + return err; + } + + if (len != (unsigned long)ctr->blocklen) { + return CRYPT_INVALID_ARG; + } + + /* set IV */ + XMEMCPY(ctr->ctr, IV, len); + + /* force next block */ + ctr->padlen = 0; + return cipher_descriptor[ctr->cipher]->ecb_encrypt(IV, ctr->pad, &ctr->key); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_start.c b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_start.c new file mode 100644 index 0000000..df8117a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_start.c @@ -0,0 +1,89 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ctr_start.c + CTR implementation, start chain, Tom St Denis +*/ + + +#ifdef LTC_CTR_MODE + +/** + Initialize a CTR context + @param cipher The index of the cipher desired + @param IV The initialization vector + @param key The secret key + @param keylen The length of the secret key (octets) + @param num_rounds Number of rounds in the cipher desired (0 for default) + @param ctr_mode The counter mode (CTR_COUNTER_LITTLE_ENDIAN or CTR_COUNTER_BIG_ENDIAN) + @param ctr The CTR state to initialize + @return CRYPT_OK if successful +*/ +int ctr_start( int cipher, + const unsigned char *IV, + const unsigned char *key, int keylen, + int num_rounds, int ctr_mode, + symmetric_CTR *ctr) +{ + int x, err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ctr != NULL); + + /* bad param? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* ctrlen == counter width */ + ctr->ctrlen = (ctr_mode & 255) ? (ctr_mode & 255) : cipher_descriptor[cipher]->block_length; + if (ctr->ctrlen > cipher_descriptor[cipher]->block_length) { + return CRYPT_INVALID_ARG; + } + + if ((ctr_mode & 0x1000) == CTR_COUNTER_BIG_ENDIAN) { + ctr->ctrlen = cipher_descriptor[cipher]->block_length - ctr->ctrlen; + } + + /* setup cipher */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, num_rounds, &ctr->key)) != CRYPT_OK) { + return err; + } + + /* copy ctr */ + ctr->blocklen = cipher_descriptor[cipher]->block_length; + ctr->cipher = cipher; + ctr->padlen = 0; + ctr->mode = ctr_mode & 0x1000; + for (x = 0; x < ctr->blocklen; x++) { + ctr->ctr[x] = IV[x]; + } + + if (ctr_mode & LTC_CTR_RFC3686) { + /* increment the IV as per RFC 3686 */ + if (ctr->mode == CTR_COUNTER_LITTLE_ENDIAN) { + /* little-endian */ + for (x = 0; x < ctr->ctrlen; x++) { + ctr->ctr[x] = (ctr->ctr[x] + (unsigned char)1) & (unsigned char)255; + if (ctr->ctr[x] != (unsigned char)0) { + break; + } + } + } else { + /* big-endian */ + for (x = ctr->blocklen-1; x >= ctr->ctrlen; x--) { + ctr->ctr[x] = (ctr->ctr[x] + (unsigned char)1) & (unsigned char)255; + if (ctr->ctr[x] != (unsigned char)0) { + break; + } + } + } + } + + return cipher_descriptor[ctr->cipher]->ecb_encrypt(ctr->ctr, ctr->pad, &ctr->key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_test.c b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_test.c new file mode 100644 index 0000000..df7e649 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/ctr_test.c @@ -0,0 +1,73 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ctr_test.c + CTR implementation, Tests again RFC 3686, Tom St Denis +*/ + +#ifdef LTC_CTR_MODE + +int ctr_test(void) +{ +#ifdef LTC_NO_TEST + return CRYPT_NOP; +#else + static const struct { + int keylen, msglen; + unsigned char key[32], IV[16], pt[64], ct[64]; + } tests[] = { +/* 128-bit key, 16-byte pt */ +{ + 16, 16, + {0xAE,0x68,0x52,0xF8,0x12,0x10,0x67,0xCC,0x4B,0xF7,0xA5,0x76,0x55,0x77,0xF3,0x9E }, + {0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 }, + {0x53,0x69,0x6E,0x67,0x6C,0x65,0x20,0x62,0x6C,0x6F,0x63,0x6B,0x20,0x6D,0x73,0x67 }, + {0xE4,0x09,0x5D,0x4F,0xB7,0xA7,0xB3,0x79,0x2D,0x61,0x75,0xA3,0x26,0x13,0x11,0xB8 }, +}, + +/* 128-bit key, 36-byte pt */ +{ + 16, 36, + {0x76,0x91,0xBE,0x03,0x5E,0x50,0x20,0xA8,0xAC,0x6E,0x61,0x85,0x29,0xF9,0xA0,0xDC }, + {0x00,0xE0,0x01,0x7B,0x27,0x77,0x7F,0x3F,0x4A,0x17,0x86,0xF0,0x00,0x00,0x00,0x00 }, + {0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F, + 0x20,0x21,0x22,0x23}, + {0xC1,0xCF,0x48,0xA8,0x9F,0x2F,0xFD,0xD9,0xCF,0x46,0x52,0xE9,0xEF,0xDB,0x72,0xD7, + 0x45,0x40,0xA4,0x2B,0xDE,0x6D,0x78,0x36,0xD5,0x9A,0x5C,0xEA,0xAE,0xF3,0x10,0x53, + 0x25,0xB2,0x07,0x2F }, +}, +}; + int idx, err, x; + unsigned char buf[64]; + symmetric_CTR ctr; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + if ((err = ctr_start(idx, tests[x].IV, tests[x].key, tests[x].keylen, 0, CTR_COUNTER_BIG_ENDIAN|LTC_CTR_RFC3686, &ctr)) != CRYPT_OK) { + return err; + } + if ((err = ctr_encrypt(tests[x].pt, buf, tests[x].msglen, &ctr)) != CRYPT_OK) { + return err; + } + ctr_done(&ctr); + if (compare_testvector(buf, tests[x].msglen, tests[x].ct, tests[x].msglen, "CTR", x)) { + return CRYPT_FAIL_TESTVECTOR; + } + } + return CRYPT_OK; +#endif +} + +#endif + + + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ctr/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/ctr/sub.mk new file mode 100644 index 0000000..1541501 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ctr/sub.mk @@ -0,0 +1,6 @@ +srcs-y += ctr_decrypt.c +srcs-y += ctr_done.c +srcs-y += ctr_encrypt.c +srcs-y += ctr_getiv.c +srcs-y += ctr_setiv.c +srcs-y += ctr_start.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_decrypt.c new file mode 100644 index 0000000..1c73f59 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_decrypt.c @@ -0,0 +1,48 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ecb_decrypt.c + ECB implementation, decrypt a block, Tom St Denis +*/ + +#ifdef LTC_ECB_MODE + +/** + ECB decrypt + @param ct Ciphertext + @param pt [out] Plaintext + @param len The number of octets to process (must be multiple of the cipher block size) + @param ecb ECB state + @return CRYPT_OK if successful +*/ +int ecb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_ECB *ecb) +{ + int err; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(ecb != NULL); + if ((err = cipher_is_valid(ecb->cipher)) != CRYPT_OK) { + return err; + } + if (len % cipher_descriptor[ecb->cipher]->block_length) { + return CRYPT_INVALID_ARG; + } + + /* check for accel */ + if (cipher_descriptor[ecb->cipher]->accel_ecb_decrypt != NULL) { + return cipher_descriptor[ecb->cipher]->accel_ecb_decrypt(ct, pt, len / cipher_descriptor[ecb->cipher]->block_length, &ecb->key); + } + while (len) { + if ((err = cipher_descriptor[ecb->cipher]->ecb_decrypt(ct, pt, &ecb->key)) != CRYPT_OK) { + return err; + } + pt += cipher_descriptor[ecb->cipher]->block_length; + ct += cipher_descriptor[ecb->cipher]->block_length; + len -= cipher_descriptor[ecb->cipher]->block_length; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_done.c b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_done.c new file mode 100644 index 0000000..56126ab --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_done.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ecb_done.c + ECB implementation, finish chain, Tom St Denis +*/ + +#ifdef LTC_ECB_MODE + +/** Terminate the chain + @param ecb The ECB chain to terminate + @return CRYPT_OK on success +*/ +int ecb_done(symmetric_ECB *ecb) +{ + int err; + LTC_ARGCHK(ecb != NULL); + + if ((err = cipher_is_valid(ecb->cipher)) != CRYPT_OK) { + return err; + } + cipher_descriptor[ecb->cipher]->done(&ecb->key); + return CRYPT_OK; +} + + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_encrypt.c new file mode 100644 index 0000000..0ac8de6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_encrypt.c @@ -0,0 +1,48 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ecb_encrypt.c + ECB implementation, encrypt a block, Tom St Denis +*/ + +#ifdef LTC_ECB_MODE + +/** + ECB encrypt + @param pt Plaintext + @param ct [out] Ciphertext + @param len The number of octets to process (must be multiple of the cipher block size) + @param ecb ECB state + @return CRYPT_OK if successful +*/ +int ecb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_ECB *ecb) +{ + int err; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(ecb != NULL); + if ((err = cipher_is_valid(ecb->cipher)) != CRYPT_OK) { + return err; + } + if (len % cipher_descriptor[ecb->cipher]->block_length) { + return CRYPT_INVALID_ARG; + } + + /* check for accel */ + if (cipher_descriptor[ecb->cipher]->accel_ecb_encrypt != NULL) { + return cipher_descriptor[ecb->cipher]->accel_ecb_encrypt(pt, ct, len / cipher_descriptor[ecb->cipher]->block_length, &ecb->key); + } + while (len) { + if ((err = cipher_descriptor[ecb->cipher]->ecb_encrypt(pt, ct, &ecb->key)) != CRYPT_OK) { + return err; + } + pt += cipher_descriptor[ecb->cipher]->block_length; + ct += cipher_descriptor[ecb->cipher]->block_length; + len -= cipher_descriptor[ecb->cipher]->block_length; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_start.c b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_start.c new file mode 100644 index 0000000..6d34409 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ecb/ecb_start.c @@ -0,0 +1,36 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ecb_start.c + ECB implementation, start chain, Tom St Denis +*/ + + +#ifdef LTC_ECB_MODE + +/** + Initialize a ECB context + @param cipher The index of the cipher desired + @param key The secret key + @param keylen The length of the secret key (octets) + @param num_rounds Number of rounds in the cipher desired (0 for default) + @param ecb The ECB state to initialize + @return CRYPT_OK if successful +*/ +int ecb_start(int cipher, const unsigned char *key, int keylen, int num_rounds, symmetric_ECB *ecb) +{ + int err; + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ecb != NULL); + + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + ecb->cipher = cipher; + ecb->blocklen = cipher_descriptor[cipher]->block_length; + return cipher_descriptor[cipher]->setup(key, keylen, num_rounds, &ecb->key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ecb/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/ecb/sub.mk new file mode 100644 index 0000000..c47c061 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ecb/sub.mk @@ -0,0 +1,4 @@ +srcs-y += ecb_decrypt.c +srcs-y += ecb_done.c +srcs-y += ecb_encrypt.c +srcs-y += ecb_start.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_decrypt.c new file mode 100644 index 0000000..f9e3a54 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_decrypt.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f8_decrypt.c + F8 implementation, decrypt data, Tom St Denis +*/ + +#ifdef LTC_F8_MODE + +/** + F8 decrypt + @param ct Ciphertext + @param pt [out] Plaintext + @param len Length of ciphertext (octets) + @param f8 F8 state + @return CRYPT_OK if successful +*/ +int f8_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_F8 *f8) +{ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(f8 != NULL); + return f8_encrypt(ct, pt, len, f8); +} + + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_done.c b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_done.c new file mode 100644 index 0000000..0f501f8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_done.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f8_done.c + F8 implementation, finish chain, Tom St Denis +*/ + +#ifdef LTC_F8_MODE + +/** Terminate the chain + @param f8 The F8 chain to terminate + @return CRYPT_OK on success +*/ +int f8_done(symmetric_F8 *f8) +{ + int err; + LTC_ARGCHK(f8 != NULL); + + if ((err = cipher_is_valid(f8->cipher)) != CRYPT_OK) { + return err; + } + cipher_descriptor[f8->cipher]->done(&f8->key); + return CRYPT_OK; +} + + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_encrypt.c new file mode 100644 index 0000000..8a32ad6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_encrypt.c @@ -0,0 +1,91 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f8_encrypt.c + F8 implementation, encrypt data, Tom St Denis +*/ + +#ifdef LTC_F8_MODE + +/** + F8 encrypt + @param pt Plaintext + @param ct [out] Ciphertext + @param len Length of plaintext (octets) + @param f8 F8 state + @return CRYPT_OK if successful +*/ +int f8_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_F8 *f8) +{ + int err, x; + unsigned char buf[MAXBLOCKSIZE]; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(f8 != NULL); + if ((err = cipher_is_valid(f8->cipher)) != CRYPT_OK) { + return err; + } + + /* is blocklen/padlen valid? */ + if (f8->blocklen < 0 || f8->blocklen > (int)sizeof(f8->IV) || + f8->padlen < 0 || f8->padlen > (int)sizeof(f8->IV)) { + return CRYPT_INVALID_ARG; + } + + zeromem(buf, sizeof(buf)); + + /* make sure the pad is empty */ + if (f8->padlen == f8->blocklen) { + /* xor of IV, MIV and blockcnt == what goes into cipher */ + STORE32H(f8->blockcnt, (buf+(f8->blocklen-4))); + ++(f8->blockcnt); + for (x = 0; x < f8->blocklen; x++) { + f8->IV[x] ^= f8->MIV[x] ^ buf[x]; + } + if ((err = cipher_descriptor[f8->cipher]->ecb_encrypt(f8->IV, f8->IV, &f8->key)) != CRYPT_OK) { + return err; + } + f8->padlen = 0; + } + +#ifdef LTC_FAST + if (f8->padlen == 0) { + while (len >= (unsigned long)f8->blocklen) { + STORE32H(f8->blockcnt, (buf+(f8->blocklen-4))); + ++(f8->blockcnt); + for (x = 0; x < f8->blocklen; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&ct[x])) = *(LTC_FAST_TYPE_PTR_CAST(&pt[x])) ^ *(LTC_FAST_TYPE_PTR_CAST(&f8->IV[x])); + *(LTC_FAST_TYPE_PTR_CAST(&f8->IV[x])) ^= *(LTC_FAST_TYPE_PTR_CAST(&f8->MIV[x])) ^ *(LTC_FAST_TYPE_PTR_CAST(&buf[x])); + } + if ((err = cipher_descriptor[f8->cipher]->ecb_encrypt(f8->IV, f8->IV, &f8->key)) != CRYPT_OK) { + return err; + } + len -= x; + pt += x; + ct += x; + } + } +#endif + + while (len > 0) { + if (f8->padlen == f8->blocklen) { + /* xor of IV, MIV and blockcnt == what goes into cipher */ + STORE32H(f8->blockcnt, (buf+(f8->blocklen-4))); + ++(f8->blockcnt); + for (x = 0; x < f8->blocklen; x++) { + f8->IV[x] ^= f8->MIV[x] ^ buf[x]; + } + if ((err = cipher_descriptor[f8->cipher]->ecb_encrypt(f8->IV, f8->IV, &f8->key)) != CRYPT_OK) { + return err; + } + f8->padlen = 0; + } + *ct++ = *pt++ ^ f8->IV[f8->padlen++]; + --len; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_getiv.c b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_getiv.c new file mode 100644 index 0000000..1a4e53f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_getiv.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ofb_getiv.c + F8 implementation, get IV, Tom St Denis +*/ + +#ifdef LTC_F8_MODE + +/** + Get the current initialization vector + @param IV [out] The destination of the initialization vector + @param len [in/out] The max size and resulting size of the initialization vector + @param f8 The F8 state + @return CRYPT_OK if successful +*/ +int f8_getiv(unsigned char *IV, unsigned long *len, const symmetric_F8 *f8) +{ + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(len != NULL); + LTC_ARGCHK(f8 != NULL); + if ((unsigned long)f8->blocklen > *len) { + *len = f8->blocklen; + return CRYPT_BUFFER_OVERFLOW; + } + XMEMCPY(IV, f8->IV, f8->blocklen); + *len = f8->blocklen; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_setiv.c b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_setiv.c new file mode 100644 index 0000000..45d2239 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_setiv.c @@ -0,0 +1,40 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f8_setiv.c + F8 implementation, set IV, Tom St Denis +*/ + +#ifdef LTC_F8_MODE + +/** + Set an initialization vector + @param IV The initialization vector + @param len The length of the vector (in octets) + @param f8 The F8 state + @return CRYPT_OK if successful +*/ +int f8_setiv(const unsigned char *IV, unsigned long len, symmetric_F8 *f8) +{ + int err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(f8 != NULL); + + if ((err = cipher_is_valid(f8->cipher)) != CRYPT_OK) { + return err; + } + + if (len != (unsigned long)f8->blocklen) { + return CRYPT_INVALID_ARG; + } + + /* force next block */ + f8->padlen = 0; + return cipher_descriptor[f8->cipher]->ecb_encrypt(IV, f8->IV, &f8->key); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_start.c b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_start.c new file mode 100644 index 0000000..f23c141 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_start.c @@ -0,0 +1,86 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f8_start.c + F8 implementation, start chain, Tom St Denis +*/ + + +#ifdef LTC_F8_MODE + +/** + Initialize an F8 context + @param cipher The index of the cipher desired + @param IV The initialization vector + @param key The secret key + @param keylen The length of the secret key (octets) + @param salt_key The salting key for the IV + @param skeylen The length of the salting key (octets) + @param num_rounds Number of rounds in the cipher desired (0 for default) + @param f8 The F8 state to initialize + @return CRYPT_OK if successful +*/ +int f8_start( int cipher, const unsigned char *IV, + const unsigned char *key, int keylen, + const unsigned char *salt_key, int skeylen, + int num_rounds, symmetric_F8 *f8) +{ + int x, err; + unsigned char tkey[MAXBLOCKSIZE]; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(salt_key != NULL); + LTC_ARGCHK(f8 != NULL); + + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + +#ifdef LTC_FAST + if (cipher_descriptor[cipher]->block_length % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + /* copy details */ + f8->blockcnt = 0; + f8->cipher = cipher; + f8->blocklen = cipher_descriptor[cipher]->block_length; + f8->padlen = f8->blocklen; + + /* now get key ^ salt_key [extend salt_ket with 0x55 as required to match length] */ + zeromem(tkey, sizeof(tkey)); + for (x = 0; x < keylen && x < (int)sizeof(tkey); x++) { + tkey[x] = key[x]; + } + for (x = 0; x < skeylen && x < (int)sizeof(tkey); x++) { + tkey[x] ^= salt_key[x]; + } + for (; x < keylen && x < (int)sizeof(tkey); x++) { + tkey[x] ^= 0x55; + } + + /* now encrypt with tkey[0..keylen-1] the IV and use that as the IV */ + if ((err = cipher_descriptor[cipher]->setup(tkey, keylen, num_rounds, &f8->key)) != CRYPT_OK) { + return err; + } + + /* encrypt IV */ + if ((err = cipher_descriptor[f8->cipher]->ecb_encrypt(IV, f8->MIV, &f8->key)) != CRYPT_OK) { + cipher_descriptor[f8->cipher]->done(&f8->key); + return err; + } + zeromem(tkey, sizeof(tkey)); + zeromem(f8->IV, sizeof(f8->IV)); + + /* terminate this cipher */ + cipher_descriptor[f8->cipher]->done(&f8->key); + + /* init the cipher */ + return cipher_descriptor[cipher]->setup(key, keylen, num_rounds, &f8->key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_test_mode.c b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_test_mode.c new file mode 100644 index 0000000..7b25175 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/f8_test_mode.c @@ -0,0 +1,64 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file f8_test_mode.c + F8 implementation, test, Tom St Denis +*/ + + +#ifdef LTC_F8_MODE + +int f8_test_mode(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const unsigned char key[16] = { 0x23, 0x48, 0x29, 0x00, 0x84, 0x67, 0xbe, 0x18, + 0x6c, 0x3d, 0xe1, 0x4a, 0xae, 0x72, 0xd6, 0x2c }; + static const unsigned char salt[4] = { 0x32, 0xf2, 0x87, 0x0d }; + static const unsigned char IV[16] = { 0x00, 0x6e, 0x5c, 0xba, 0x50, 0x68, 0x1d, 0xe5, + 0x5c, 0x62, 0x15, 0x99, 0xd4, 0x62, 0x56, 0x4a }; + static const unsigned char pt[39] = { 0x70, 0x73, 0x65, 0x75, 0x64, 0x6f, 0x72, 0x61, + 0x6e, 0x64, 0x6f, 0x6d, 0x6e, 0x65, 0x73, 0x73, + 0x20, 0x69, 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, + 0x6e, 0x65, 0x78, 0x74, 0x20, 0x62, 0x65, 0x73, + 0x74, 0x20, 0x74, 0x68, 0x69, 0x6e, 0x67 }; + static const unsigned char ct[39] = { 0x01, 0x9c, 0xe7, 0xa2, 0x6e, 0x78, 0x54, 0x01, + 0x4a, 0x63, 0x66, 0xaa, 0x95, 0xd4, 0xee, 0xfd, + 0x1a, 0xd4, 0x17, 0x2a, 0x14, 0xf9, 0xfa, 0xf4, + 0x55, 0xb7, 0xf1, 0xd4, 0xb6, 0x2b, 0xd0, 0x8f, + 0x56, 0x2c, 0x0e, 0xef, 0x7c, 0x48, 0x02 }; + unsigned char buf[39]; + symmetric_F8 f8; + int err, idx; + + idx = find_cipher("aes"); + if (idx == -1) { + idx = find_cipher("rijndael"); + if (idx == -1) return CRYPT_NOP; + } + + /* initialize the context */ + if ((err = f8_start(idx, IV, key, sizeof(key), salt, sizeof(salt), 0, &f8)) != CRYPT_OK) { + return err; + } + + /* encrypt block */ + if ((err = f8_encrypt(pt, buf, sizeof(pt), &f8)) != CRYPT_OK) { + f8_done(&f8); + return err; + } + f8_done(&f8); + + /* compare */ + if (compare_testvector(buf, sizeof(ct), ct, sizeof(ct), "f8", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/f8/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/f8/sub.mk new file mode 100644 index 0000000..32e1381 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/f8/sub.mk @@ -0,0 +1,6 @@ +srcs-y += f8_decrypt.c +srcs-y += f8_done.c +srcs-y += f8_encrypt.c +srcs-y += f8_getiv.c +srcs-y += f8_setiv.c +srcs-y += f8_start.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_decrypt.c new file mode 100644 index 0000000..4e880a4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_decrypt.c @@ -0,0 +1,39 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_decrypt.c + LRW_MODE implementation, Decrypt blocks, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + LRW decrypt blocks + @param ct The ciphertext + @param pt [out] The plaintext + @param len The length in octets, must be a multiple of 16 + @param lrw The LRW state +*/ +int lrw_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_LRW *lrw) +{ + int err; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(lrw != NULL); + + if ((err = cipher_is_valid(lrw->cipher)) != CRYPT_OK) { + return err; + } + + if (cipher_descriptor[lrw->cipher]->accel_lrw_decrypt != NULL) { + return cipher_descriptor[lrw->cipher]->accel_lrw_decrypt(ct, pt, len, lrw->IV, lrw->tweak, &lrw->key); + } + + return lrw_process(ct, pt, len, LRW_DECRYPT, lrw); +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_done.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_done.c new file mode 100644 index 0000000..d216528 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_done.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_done.c + LRW_MODE implementation, Free resources, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + Terminate a LRW state + @param lrw The state to terminate + @return CRYPT_OK if successful +*/ +int lrw_done(symmetric_LRW *lrw) +{ + int err; + + LTC_ARGCHK(lrw != NULL); + + if ((err = cipher_is_valid(lrw->cipher)) != CRYPT_OK) { + return err; + } + cipher_descriptor[lrw->cipher]->done(&lrw->key); + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_encrypt.c new file mode 100644 index 0000000..e175911 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_encrypt.c @@ -0,0 +1,39 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_encrypt.c + LRW_MODE implementation, Encrypt blocks, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + LRW encrypt blocks + @param pt The plaintext + @param ct [out] The ciphertext + @param len The length in octets, must be a multiple of 16 + @param lrw The LRW state +*/ +int lrw_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_LRW *lrw) +{ + int err; + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(lrw != NULL); + + if ((err = cipher_is_valid(lrw->cipher)) != CRYPT_OK) { + return err; + } + + if (cipher_descriptor[lrw->cipher]->accel_lrw_encrypt != NULL) { + return cipher_descriptor[lrw->cipher]->accel_lrw_encrypt(pt, ct, len, lrw->IV, lrw->tweak, &lrw->key); + } + + return lrw_process(pt, ct, len, LRW_ENCRYPT, lrw); +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_getiv.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_getiv.c new file mode 100644 index 0000000..9da1aab --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_getiv.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_getiv.c + LRW_MODE implementation, Retrieve the current IV, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + Get the IV for LRW + @param IV [out] The IV, must be 16 octets + @param len Length ... must be at least 16 :-) + @param lrw The LRW state to read + @return CRYPT_OK if successful +*/ +int lrw_getiv(unsigned char *IV, unsigned long *len, const symmetric_LRW *lrw) +{ + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(len != NULL); + LTC_ARGCHK(lrw != NULL); + if (*len < 16) { + *len = 16; + return CRYPT_BUFFER_OVERFLOW; + } + + XMEMCPY(IV, lrw->IV, 16); + *len = 16; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_process.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_process.c new file mode 100644 index 0000000..4ee0b41 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_process.c @@ -0,0 +1,109 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_process.c + LRW_MODE implementation, Encrypt/decrypt blocks, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + Process blocks with LRW, since decrypt/encrypt are largely the same they share this code. + @param pt The "input" data + @param ct [out] The "output" data + @param len The length of the input, must be a multiple of 128-bits (16 octets) + @param mode LRW_ENCRYPT or LRW_DECRYPT + @param lrw The LRW state + @return CRYPT_OK if successful +*/ +int lrw_process(const unsigned char *pt, unsigned char *ct, unsigned long len, int mode, symmetric_LRW *lrw) +{ + unsigned char prod[16]; + int x, err; +#ifdef LTC_LRW_TABLES + int y; +#endif + + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(lrw != NULL); + + if (len & 15) { + return CRYPT_INVALID_ARG; + } + + while (len) { + /* copy pad */ + XMEMCPY(prod, lrw->pad, 16); + + /* increment IV */ + for (x = 15; x >= 0; x--) { + lrw->IV[x] = (lrw->IV[x] + 1) & 255; + if (lrw->IV[x]) { + break; + } + } + + /* update pad */ +#ifdef LTC_LRW_TABLES + /* for each byte changed we undo it's affect on the pad then add the new product */ + for (; x < 16; x++) { +#ifdef LTC_FAST + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(lrw->pad + y)) ^= *(LTC_FAST_TYPE_PTR_CAST(&lrw->PC[x][lrw->IV[x]][y])) ^ *(LTC_FAST_TYPE_PTR_CAST(&lrw->PC[x][(lrw->IV[x]-1)&255][y])); + } +#else + for (y = 0; y < 16; y++) { + lrw->pad[y] ^= lrw->PC[x][lrw->IV[x]][y] ^ lrw->PC[x][(lrw->IV[x]-1)&255][y]; + } +#endif + } +#else + gcm_gf_mult(lrw->tweak, lrw->IV, lrw->pad); +#endif + + /* xor prod */ +#ifdef LTC_FAST + for (x = 0; x < 16; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(ct + x)) = *(LTC_FAST_TYPE_PTR_CAST(pt + x)) ^ *(LTC_FAST_TYPE_PTR_CAST(prod + x)); + } +#else + for (x = 0; x < 16; x++) { + ct[x] = pt[x] ^ prod[x]; + } +#endif + + /* send through cipher */ + if (mode == LRW_ENCRYPT) { + if ((err = cipher_descriptor[lrw->cipher]->ecb_encrypt(ct, ct, &lrw->key)) != CRYPT_OK) { + return err; + } + } else { + if ((err = cipher_descriptor[lrw->cipher]->ecb_decrypt(ct, ct, &lrw->key)) != CRYPT_OK) { + return err; + } + } + + /* xor prod */ +#ifdef LTC_FAST + for (x = 0; x < 16; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(ct + x)) = *(LTC_FAST_TYPE_PTR_CAST(ct + x)) ^ *(LTC_FAST_TYPE_PTR_CAST(prod + x)); + } +#else + for (x = 0; x < 16; x++) { + ct[x] = ct[x] ^ prod[x]; + } +#endif + + /* move to next */ + pt += 16; + ct += 16; + len -= 16; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_setiv.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_setiv.c new file mode 100644 index 0000000..a1a355a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_setiv.c @@ -0,0 +1,68 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_setiv.c + LRW_MODE implementation, Set the current IV, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + Set the IV for LRW + @param IV The IV, must be 16 octets + @param len Length ... must be 16 :-) + @param lrw The LRW state to update + @return CRYPT_OK if successful +*/ +int lrw_setiv(const unsigned char *IV, unsigned long len, symmetric_LRW *lrw) +{ + int err; +#ifdef LTC_LRW_TABLES + unsigned char T[16]; + int x, y; +#endif + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(lrw != NULL); + + if (len != 16) { + return CRYPT_INVALID_ARG; + } + + if ((err = cipher_is_valid(lrw->cipher)) != CRYPT_OK) { + return err; + } + + /* copy the IV */ + XMEMCPY(lrw->IV, IV, 16); + + /* check if we have to actually do work */ + if (cipher_descriptor[lrw->cipher]->accel_lrw_encrypt != NULL && cipher_descriptor[lrw->cipher]->accel_lrw_decrypt != NULL) { + /* we have accelerators, let's bail since they don't use lrw->pad anyways */ + return CRYPT_OK; + } + +#ifdef LTC_LRW_TABLES + XMEMCPY(T, &lrw->PC[0][IV[0]][0], 16); + for (x = 1; x < 16; x++) { +#ifdef LTC_FAST + for (y = 0; y < 16; y += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(T + y)) ^= *(LTC_FAST_TYPE_PTR_CAST(&lrw->PC[x][IV[x]][y])); + } +#else + for (y = 0; y < 16; y++) { + T[y] ^= lrw->PC[x][IV[x]][y]; + } +#endif + } + XMEMCPY(lrw->pad, T, 16); +#else + gcm_gf_mult(lrw->tweak, IV, lrw->pad); +#endif + + return CRYPT_OK; +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_start.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_start.c new file mode 100644 index 0000000..ec2cdc0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_start.c @@ -0,0 +1,92 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_start.c + LRW_MODE implementation, start mode, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + Initialize the LRW context + @param cipher The cipher desired, must be a 128-bit block cipher + @param IV The index value, must be 128-bits + @param key The cipher key + @param keylen The length of the cipher key in octets + @param tweak The tweak value (second key), must be 128-bits + @param num_rounds The number of rounds for the cipher (0 == default) + @param lrw [out] The LRW state + @return CRYPT_OK on success. +*/ +int lrw_start( int cipher, + const unsigned char *IV, + const unsigned char *key, int keylen, + const unsigned char *tweak, + int num_rounds, + symmetric_LRW *lrw) +{ + int err; +#ifdef LTC_LRW_TABLES + unsigned char B[16]; + int x, y, z, t; +#endif + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(tweak != NULL); + LTC_ARGCHK(lrw != NULL); + +#ifdef LTC_FAST + if (16 % sizeof(LTC_FAST_TYPE)) { + return CRYPT_INVALID_ARG; + } +#endif + + /* is cipher valid? */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + if (cipher_descriptor[cipher]->block_length != 16) { + return CRYPT_INVALID_CIPHER; + } + + /* schedule key */ + if ((err = cipher_descriptor[cipher]->setup(key, keylen, num_rounds, &lrw->key)) != CRYPT_OK) { + return err; + } + lrw->cipher = cipher; + + /* copy the IV and tweak */ + XMEMCPY(lrw->tweak, tweak, 16); + +#ifdef LTC_LRW_TABLES + /* setup tables */ + /* generate the first table as it has no shifting (from which we make the other tables) */ + zeromem(B, 16); + for (y = 0; y < 256; y++) { + B[0] = y; + gcm_gf_mult(tweak, B, &lrw->PC[0][y][0]); + } + + /* now generate the rest of the tables based the previous table */ + for (x = 1; x < 16; x++) { + for (y = 0; y < 256; y++) { + /* now shift it right by 8 bits */ + t = lrw->PC[x-1][y][15]; + for (z = 15; z > 0; z--) { + lrw->PC[x][y][z] = lrw->PC[x-1][y][z-1]; + } + lrw->PC[x][y][0] = gcm_shift_table[t<<1]; + lrw->PC[x][y][1] ^= gcm_shift_table[(t<<1)+1]; + } + } +#endif + + /* generate first pad */ + return lrw_setiv(IV, 16, lrw); +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_test.c b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_test.c new file mode 100644 index 0000000..3d9015b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/lrw_test.c @@ -0,0 +1,124 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file lrw_test.c + LRW_MODE implementation, test LRW, Tom St Denis +*/ + +#ifdef LTC_LRW_MODE + +/** + Test LRW against specs + @return CRYPT_OK if goodly +*/ +int lrw_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + static const struct { + unsigned char key[16], tweak[16], IV[16], P[16], expected_tweak[16], C[16]; + } tests[] = { + +{ +{ 0x45, 0x62, 0xac, 0x25, 0xf8, 0x28, 0x17, 0x6d, 0x4c, 0x26, 0x84, 0x14, 0xb5, 0x68, 0x01, 0x85 }, +{ 0x25, 0x8e, 0x2a, 0x05, 0xe7, 0x3e, 0x9d, 0x03, 0xee, 0x5a, 0x83, 0x0c, 0xcc, 0x09, 0x4c, 0x87 }, +{ 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +{ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46 }, +{ 0x25, 0x8e, 0x2a, 0x05, 0xe7, 0x3e, 0x9d, 0x03, 0xee, 0x5a, 0x83, 0x0c, 0xcc, 0x09, 0x4c, 0x87 }, +{ 0xf1, 0xb2, 0x73, 0xcd, 0x65, 0xa3, 0xdf, 0x5f, 0xe9, 0x5d, 0x48, 0x92, 0x54, 0x63, 0x4e, 0xb8 } +}, + +{ +{ 0x59, 0x70, 0x47, 0x14, 0xf5, 0x57, 0x47, 0x8c, 0xd7, 0x79, 0xe8, 0x0f, 0x54, 0x88, 0x79, 0x44 }, +{ 0x35, 0x23, 0xc2, 0xde, 0xc5, 0x69, 0x4f, 0xa8, 0x72, 0xa9, 0xac, 0xa7, 0x0b, 0x2b, 0xee, 0xbc }, +{ 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +{ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46 }, +{ 0x1a, 0x91, 0xe1, 0x6f, 0x62, 0xb4, 0xa7, 0xd4, 0x39, 0x54, 0xd6, 0x53, 0x85, 0x95, 0xf7, 0x5e }, +{ 0x00, 0xc8, 0x2b, 0xae, 0x95, 0xbb, 0xcd, 0xe5, 0x27, 0x4f, 0x07, 0x69, 0xb2, 0x60, 0xe1, 0x36 }, +}, + +{ +{ 0x59, 0x70, 0x47, 0x14, 0xf5, 0x57, 0x47, 0x8c, 0xd7, 0x79, 0xe8, 0x0f, 0x54, 0x88, 0x79, 0x44 }, +{ 0x67, 0x53, 0xc9, 0x0c, 0xb7, 0xd8, 0xcd, 0xe5, 0x06, 0xa0, 0x47, 0x78, 0x1a, 0xad, 0x85, 0x11 }, +{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 }, +{ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46 }, +{ 0x1a, 0x91, 0xe1, 0x6f, 0x62, 0xb4, 0xa7, 0xd4, 0x39, 0x54, 0xd6, 0x53, 0x85, 0x95, 0xf7, 0x5e }, +{ 0x00, 0xc8, 0x2b, 0xae, 0x95, 0xbb, 0xcd, 0xe5, 0x27, 0x4f, 0x07, 0x69, 0xb2, 0x60, 0xe1, 0x36 }, +}, + +{ + +{ 0xd8, 0x2a, 0x91, 0x34, 0xb2, 0x6a, 0x56, 0x50, 0x30, 0xfe, 0x69, 0xe2, 0x37, 0x7f, 0x98, 0x47 }, +{ 0x4e, 0xb5, 0x5d, 0x31, 0x05, 0x97, 0x3a, 0x3f, 0x5e, 0x23, 0xda, 0xfb, 0x5a, 0x45, 0xd6, 0xc0 }, +{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00 }, +{ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46 }, +{ 0x18, 0xc9, 0x1f, 0x6d, 0x60, 0x1a, 0x1a, 0x37, 0x5d, 0x0b, 0x0e, 0xf7, 0x3a, 0xd5, 0x74, 0xc4 }, +{ 0x76, 0x32, 0x21, 0x83, 0xed, 0x8f, 0xf1, 0x82, 0xf9, 0x59, 0x62, 0x03, 0x69, 0x0e, 0x5e, 0x01 }, + +} +}; + + int idx, err, x; + symmetric_LRW lrw; + unsigned char buf[2][16]; + + idx = find_cipher("aes"); + if (idx == -1) { + idx = find_cipher("rijndael"); + if (idx == -1) { + return CRYPT_NOP; + } + } + + for (x = 0; x < (int)(sizeof(tests)/sizeof(tests[0])); x++) { + /* schedule it */ + if ((err = lrw_start(idx, tests[x].IV, tests[x].key, 16, tests[x].tweak, 0, &lrw)) != CRYPT_OK) { + return err; + } + + /* check pad against expected tweak */ + if (compare_testvector(tests[x].expected_tweak, 16, lrw.pad, 16, "LRW Tweak", x)) { + lrw_done(&lrw); + return CRYPT_FAIL_TESTVECTOR; + } + + /* process block */ + if ((err = lrw_encrypt(tests[x].P, buf[0], 16, &lrw)) != CRYPT_OK) { + lrw_done(&lrw); + return err; + } + + if (compare_testvector(buf[0], 16, tests[x].C, 16, "LRW Encrypt", x)) { + lrw_done(&lrw); + return CRYPT_FAIL_TESTVECTOR; + } + + /* process block */ + if ((err = lrw_setiv(tests[x].IV, 16, &lrw)) != CRYPT_OK) { + lrw_done(&lrw); + return err; + } + + if ((err = lrw_decrypt(buf[0], buf[1], 16, &lrw)) != CRYPT_OK) { + lrw_done(&lrw); + return err; + } + + if (compare_testvector(buf[1], 16, tests[x].P, 16, "LRW Decrypt", x)) { + lrw_done(&lrw); + return CRYPT_FAIL_TESTVECTOR; + } + if ((err = lrw_done(&lrw)) != CRYPT_OK) { + return err; + } + } + return CRYPT_OK; +#endif +} + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/lrw/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/lrw/sub.mk new file mode 100644 index 0000000..b464c41 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/lrw/sub.mk @@ -0,0 +1,7 @@ +srcs-y += lrw_decrypt.c +srcs-y += lrw_done.c +srcs-y += lrw_encrypt.c +srcs-y += lrw_getiv.c +srcs-y += lrw_process.c +srcs-y += lrw_setiv.c +srcs-y += lrw_start.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_decrypt.c new file mode 100644 index 0000000..cebcd7b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_decrypt.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ofb_decrypt.c + OFB implementation, decrypt data, Tom St Denis +*/ + +#ifdef LTC_OFB_MODE + +/** + OFB decrypt + @param ct Ciphertext + @param pt [out] Plaintext + @param len Length of ciphertext (octets) + @param ofb OFB state + @return CRYPT_OK if successful +*/ +int ofb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, symmetric_OFB *ofb) +{ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(ofb != NULL); + return ofb_encrypt(ct, pt, len, ofb); +} + + +#endif + + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_done.c b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_done.c new file mode 100644 index 0000000..1a728e6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_done.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ofb_done.c + OFB implementation, finish chain, Tom St Denis +*/ + +#ifdef LTC_OFB_MODE + +/** Terminate the chain + @param ofb The OFB chain to terminate + @return CRYPT_OK on success +*/ +int ofb_done(symmetric_OFB *ofb) +{ + int err; + LTC_ARGCHK(ofb != NULL); + + if ((err = cipher_is_valid(ofb->cipher)) != CRYPT_OK) { + return err; + } + cipher_descriptor[ofb->cipher]->done(&ofb->key); + return CRYPT_OK; +} + + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_encrypt.c new file mode 100644 index 0000000..d07b570 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_encrypt.c @@ -0,0 +1,48 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ofb_encrypt.c + OFB implementation, encrypt data, Tom St Denis +*/ + +#ifdef LTC_OFB_MODE + +/** + OFB encrypt + @param pt Plaintext + @param ct [out] Ciphertext + @param len Length of plaintext (octets) + @param ofb OFB state + @return CRYPT_OK if successful +*/ +int ofb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_OFB *ofb) +{ + int err; + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(ofb != NULL); + if ((err = cipher_is_valid(ofb->cipher)) != CRYPT_OK) { + return err; + } + + /* is blocklen/padlen valid? */ + if (ofb->blocklen < 0 || ofb->blocklen > (int)sizeof(ofb->IV) || + ofb->padlen < 0 || ofb->padlen > (int)sizeof(ofb->IV)) { + return CRYPT_INVALID_ARG; + } + + while (len-- > 0) { + if (ofb->padlen == ofb->blocklen) { + if ((err = cipher_descriptor[ofb->cipher]->ecb_encrypt(ofb->IV, ofb->IV, &ofb->key)) != CRYPT_OK) { + return err; + } + ofb->padlen = 0; + } + *ct++ = *pt++ ^ ofb->IV[(ofb->padlen)++]; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_getiv.c b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_getiv.c new file mode 100644 index 0000000..0a799f0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_getiv.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ofb_getiv.c + OFB implementation, get IV, Tom St Denis +*/ + +#ifdef LTC_OFB_MODE + +/** + Get the current initialization vector + @param IV [out] The destination of the initialization vector + @param len [in/out] The max size and resulting size of the initialization vector + @param ofb The OFB state + @return CRYPT_OK if successful +*/ +int ofb_getiv(unsigned char *IV, unsigned long *len, const symmetric_OFB *ofb) +{ + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(len != NULL); + LTC_ARGCHK(ofb != NULL); + if ((unsigned long)ofb->blocklen > *len) { + *len = ofb->blocklen; + return CRYPT_BUFFER_OVERFLOW; + } + XMEMCPY(IV, ofb->IV, ofb->blocklen); + *len = ofb->blocklen; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_setiv.c b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_setiv.c new file mode 100644 index 0000000..8fcb392 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_setiv.c @@ -0,0 +1,40 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ofb_setiv.c + OFB implementation, set IV, Tom St Denis +*/ + +#ifdef LTC_OFB_MODE + +/** + Set an initialization vector + @param IV The initialization vector + @param len The length of the vector (in octets) + @param ofb The OFB state + @return CRYPT_OK if successful +*/ +int ofb_setiv(const unsigned char *IV, unsigned long len, symmetric_OFB *ofb) +{ + int err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(ofb != NULL); + + if ((err = cipher_is_valid(ofb->cipher)) != CRYPT_OK) { + return err; + } + + if (len != (unsigned long)ofb->blocklen) { + return CRYPT_INVALID_ARG; + } + + /* force next block */ + ofb->padlen = 0; + return cipher_descriptor[ofb->cipher]->ecb_encrypt(IV, ofb->IV, &ofb->key); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_start.c b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_start.c new file mode 100644 index 0000000..f0a0f04 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ofb/ofb_start.c @@ -0,0 +1,48 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ofb_start.c + OFB implementation, start chain, Tom St Denis +*/ + + +#ifdef LTC_OFB_MODE + +/** + Initialize a OFB context + @param cipher The index of the cipher desired + @param IV The initialization vector + @param key The secret key + @param keylen The length of the secret key (octets) + @param num_rounds Number of rounds in the cipher desired (0 for default) + @param ofb The OFB state to initialize + @return CRYPT_OK if successful +*/ +int ofb_start(int cipher, const unsigned char *IV, const unsigned char *key, + int keylen, int num_rounds, symmetric_OFB *ofb) +{ + int x, err; + + LTC_ARGCHK(IV != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ofb != NULL); + + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + /* copy details */ + ofb->cipher = cipher; + ofb->blocklen = cipher_descriptor[cipher]->block_length; + for (x = 0; x < ofb->blocklen; x++) { + ofb->IV[x] = IV[x]; + } + + /* init the cipher */ + ofb->padlen = ofb->blocklen; + return cipher_descriptor[cipher]->setup(key, keylen, num_rounds, &ofb->key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/ofb/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/ofb/sub.mk new file mode 100644 index 0000000..f1fceeb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/ofb/sub.mk @@ -0,0 +1,6 @@ +srcs-y += ofb_decrypt.c +srcs-y += ofb_done.c +srcs-y += ofb_encrypt.c +srcs-y += ofb_getiv.c +srcs-y += ofb_setiv.c +srcs-y += ofb_start.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/sub.mk new file mode 100644 index 0000000..45b14a8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/sub.mk @@ -0,0 +1,4 @@ +subdirs-$(_CFG_CORE_LTC_CBC) += cbc +subdirs-$(_CFG_CORE_LTC_CTR) += ctr +subdirs-$(_CFG_CORE_LTC_ECB) += ecb +subdirs-$(_CFG_CORE_LTC_XTS) += xts diff --git a/optee_os/core/lib/libtomcrypt/src/modes/xts/sub.mk b/optee_os/core/lib/libtomcrypt/src/modes/xts/sub.mk new file mode 100644 index 0000000..d8d9f18 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/xts/sub.mk @@ -0,0 +1,5 @@ +srcs-y += xts_decrypt.c +srcs-y += xts_done.c +srcs-y += xts_encrypt.c +srcs-y += xts_init.c +srcs-y += xts_mult_x.c diff --git a/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_decrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_decrypt.c new file mode 100644 index 0000000..c3adf9c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_decrypt.c @@ -0,0 +1,146 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + Source donated by Elliptic Semiconductor Inc (www.ellipticsemi.com) to the LibTom Projects + */ + +#ifdef LTC_XTS_MODE + +static int s_tweak_uncrypt(const unsigned char *C, unsigned char *P, unsigned char *T, const symmetric_xts *xts) +{ + unsigned long x; + int err; + + /* tweak encrypt block i */ +#ifdef LTC_FAST + for (x = 0; x < 16; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&P[x])) = *(LTC_FAST_TYPE_PTR_CAST(&C[x])) ^ *(LTC_FAST_TYPE_PTR_CAST(&T[x])); + } +#else + for (x = 0; x < 16; x++) { + P[x] = C[x] ^ T[x]; + } +#endif + + err = cipher_descriptor[xts->cipher]->ecb_decrypt(P, P, &xts->key1); + +#ifdef LTC_FAST + for (x = 0; x < 16; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&P[x])) ^= *(LTC_FAST_TYPE_PTR_CAST(&T[x])); + } +#else + for (x = 0; x < 16; x++) { + P[x] = P[x] ^ T[x]; + } +#endif + + /* LFSR the tweak */ + xts_mult_x(T); + + return err; +} + +/** XTS Decryption + @param ct [in] Ciphertext + @param ptlen Length of plaintext (and ciphertext) + @param pt [out] Plaintext + @param tweak [in] The 128--bit encryption tweak (e.g. sector number) + @param xts The XTS structure + Returns CRYPT_OK upon success + */ +int xts_decrypt(const unsigned char *ct, unsigned long ptlen, unsigned char *pt, unsigned char *tweak, + const symmetric_xts *xts) +{ + unsigned char PP[16], CC[16], T[16]; + unsigned long i, m, mo, lim; + int err; + + /* check inputs */ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tweak != NULL); + LTC_ARGCHK(xts != NULL); + + /* check if valid */ + if ((err = cipher_is_valid(xts->cipher)) != CRYPT_OK) { + return err; + } + + /* get number of blocks */ + m = ptlen >> 4; + mo = ptlen & 15; + + /* must have at least one full block */ + if (m == 0) { + return CRYPT_INVALID_ARG; + } + + if (mo == 0) { + lim = m; + } else { + lim = m - 1; + } + + if (cipher_descriptor[xts->cipher]->accel_xts_decrypt && lim > 0) { + + /* use accelerated decryption for whole blocks */ + if ((err = cipher_descriptor[xts->cipher]->accel_xts_decrypt(ct, pt, lim, tweak, &xts->key1, &xts->key2)) != + CRYPT_OK) { + return err; + } + ct += lim * 16; + pt += lim * 16; + + /* tweak is encrypted on output */ + XMEMCPY(T, tweak, sizeof(T)); + } else { + /* encrypt the tweak */ + if ((err = cipher_descriptor[xts->cipher]->ecb_encrypt(tweak, T, &xts->key2)) != CRYPT_OK) { + return err; + } + + for (i = 0; i < lim; i++) { + if ((err = s_tweak_uncrypt(ct, pt, T, xts)) != CRYPT_OK) { + return err; + } + ct += 16; + pt += 16; + } + } + + /* if ptlen not divide 16 then */ + if (mo > 0) { + XMEMCPY(CC, T, 16); + xts_mult_x(CC); + + /* PP = tweak decrypt block m-1 */ + if ((err = s_tweak_uncrypt(ct, PP, CC, xts)) != CRYPT_OK) { + return err; + } + + /* Pm = first ptlen % 16 bytes of PP */ + for (i = 0; i < mo; i++) { + CC[i] = ct[16 + i]; + pt[16 + i] = PP[i]; + } + for (; i < 16; i++) { + CC[i] = PP[i]; + } + + /* Pm-1 = Tweak uncrypt CC */ + if ((err = s_tweak_uncrypt(CC, pt, T, xts)) != CRYPT_OK) { + return err; + } + } + + /* Decrypt the tweak back */ + if ((err = cipher_descriptor[xts->cipher]->ecb_decrypt(T, tweak, &xts->key2)) != CRYPT_OK) { + return err; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_done.c b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_done.c new file mode 100644 index 0000000..66491fa --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_done.c @@ -0,0 +1,21 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + Source donated by Elliptic Semiconductor Inc (www.ellipticsemi.com) to the LibTom Projects +*/ + +#ifdef LTC_XTS_MODE + +/** Terminate XTS state + @param xts The state to terminate +*/ +void xts_done(symmetric_xts *xts) +{ + LTC_ARGCHKVD(xts != NULL); + cipher_descriptor[xts->cipher]->done(&xts->key1); + cipher_descriptor[xts->cipher]->done(&xts->key2); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_encrypt.c b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_encrypt.c new file mode 100644 index 0000000..08acb45 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_encrypt.c @@ -0,0 +1,147 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + Source donated by Elliptic Semiconductor Inc (www.ellipticsemi.com) to the LibTom Projects + */ + +#ifdef LTC_XTS_MODE + +static int s_tweak_crypt(const unsigned char *P, unsigned char *C, unsigned char *T, const symmetric_xts *xts) +{ + unsigned long x; + int err; + + /* tweak encrypt block i */ +#ifdef LTC_FAST + for (x = 0; x < 16; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&C[x])) = *(LTC_FAST_TYPE_PTR_CAST(&P[x])) ^ *(LTC_FAST_TYPE_PTR_CAST(&T[x])); + } +#else + for (x = 0; x < 16; x++) { + C[x] = P[x] ^ T[x]; + } +#endif + + if ((err = cipher_descriptor[xts->cipher]->ecb_encrypt(C, C, &xts->key1)) != CRYPT_OK) { + return err; + } + +#ifdef LTC_FAST + for (x = 0; x < 16; x += sizeof(LTC_FAST_TYPE)) { + *(LTC_FAST_TYPE_PTR_CAST(&C[x])) ^= *(LTC_FAST_TYPE_PTR_CAST(&T[x])); + } +#else + for (x = 0; x < 16; x++) { + C[x] = C[x] ^ T[x]; + } +#endif + + /* LFSR the tweak */ + xts_mult_x(T); + + return CRYPT_OK; +} + +/** XTS Encryption + @param pt [in] Plaintext + @param ptlen Length of plaintext (and ciphertext) + @param ct [out] Ciphertext + @param tweak [in] The 128--bit encryption tweak (e.g. sector number) + @param xts The XTS structure + Returns CRYPT_OK upon success + */ +int xts_encrypt(const unsigned char *pt, unsigned long ptlen, unsigned char *ct, unsigned char *tweak, + const symmetric_xts *xts) +{ + unsigned char PP[16], CC[16], T[16]; + unsigned long i, m, mo, lim; + int err; + + /* check inputs */ + LTC_ARGCHK(pt != NULL); + LTC_ARGCHK(ct != NULL); + LTC_ARGCHK(tweak != NULL); + LTC_ARGCHK(xts != NULL); + + /* check if valid */ + if ((err = cipher_is_valid(xts->cipher)) != CRYPT_OK) { + return err; + } + + /* get number of blocks */ + m = ptlen >> 4; + mo = ptlen & 15; + + /* must have at least one full block */ + if (m == 0) { + return CRYPT_INVALID_ARG; + } + + if (mo == 0) { + lim = m; + } else { + lim = m - 1; + } + + if (cipher_descriptor[xts->cipher]->accel_xts_encrypt && lim > 0) { + + /* use accelerated encryption for whole blocks */ + if ((err = cipher_descriptor[xts->cipher]->accel_xts_encrypt(pt, ct, lim, tweak, &xts->key1, &xts->key2)) != + CRYPT_OK) { + return err; + } + ct += lim * 16; + pt += lim * 16; + + /* tweak is encrypted on output */ + XMEMCPY(T, tweak, sizeof(T)); + } else { + + /* encrypt the tweak */ + if ((err = cipher_descriptor[xts->cipher]->ecb_encrypt(tweak, T, &xts->key2)) != CRYPT_OK) { + return err; + } + + for (i = 0; i < lim; i++) { + if ((err = s_tweak_crypt(pt, ct, T, xts)) != CRYPT_OK) { + return err; + } + ct += 16; + pt += 16; + } + } + + /* if ptlen not divide 16 then */ + if (mo > 0) { + /* CC = tweak encrypt block m-1 */ + if ((err = s_tweak_crypt(pt, CC, T, xts)) != CRYPT_OK) { + return err; + } + + /* Cm = first ptlen % 16 bytes of CC */ + for (i = 0; i < mo; i++) { + PP[i] = pt[16 + i]; + ct[16 + i] = CC[i]; + } + + for (; i < 16; i++) { + PP[i] = CC[i]; + } + + /* Cm-1 = Tweak encrypt PP */ + if ((err = s_tweak_crypt(PP, ct, T, xts)) != CRYPT_OK) { + return err; + } + } + + /* Decrypt the tweak back */ + if ((err = cipher_descriptor[xts->cipher]->ecb_decrypt(T, tweak, &xts->key2)) != CRYPT_OK) { + return err; + } + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_init.c b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_init.c new file mode 100644 index 0000000..224df32 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_init.c @@ -0,0 +1,51 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + Source donated by Elliptic Semiconductor Inc (www.ellipticsemi.com) to the LibTom Projects +*/ + +#ifdef LTC_XTS_MODE + +/** Start XTS mode + @param cipher The index of the cipher to use + @param key1 The encrypt key + @param key2 The tweak encrypt key + @param keylen The length of the keys (each) in octets + @param num_rounds The number of rounds for the cipher (0 == default) + @param xts [out] XTS structure + Returns CRYPT_OK upon success. +*/ +int xts_start(int cipher, const unsigned char *key1, const unsigned char *key2, unsigned long keylen, int num_rounds, + symmetric_xts *xts) +{ + int err; + + /* check inputs */ + LTC_ARGCHK(key1 != NULL); + LTC_ARGCHK(key2 != NULL); + LTC_ARGCHK(xts != NULL); + + /* check if valid */ + if ((err = cipher_is_valid(cipher)) != CRYPT_OK) { + return err; + } + + if (cipher_descriptor[cipher]->block_length != 16) { + return CRYPT_INVALID_ARG; + } + + /* schedule the two ciphers */ + if ((err = cipher_descriptor[cipher]->setup(key1, keylen, num_rounds, &xts->key1)) != CRYPT_OK) { + return err; + } + if ((err = cipher_descriptor[cipher]->setup(key2, keylen, num_rounds, &xts->key2)) != CRYPT_OK) { + return err; + } + xts->cipher = cipher; + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_mult_x.c b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_mult_x.c new file mode 100644 index 0000000..20947a5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_mult_x.c @@ -0,0 +1,29 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + Source donated by Elliptic Semiconductor Inc (www.ellipticsemi.com) to the LibTom Projects +*/ + +#ifdef LTC_XTS_MODE + +/** multiply by x + @param I The value to multiply by x (LFSR shift) +*/ +void xts_mult_x(unsigned char *I) +{ + int x; + unsigned char t, tt; + + for (x = t = 0; x < 16; x++) { + tt = I[x] >> 7; + I[x] = ((I[x] << 1) | t) & 0xFF; + t = tt; + } + if (tt) { + I[0] ^= 0x87; + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_test.c b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_test.c new file mode 100644 index 0000000..208853a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/modes/xts/xts_test.c @@ -0,0 +1,296 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_XTS_MODE + +#ifndef LTC_NO_TEST +static int s_xts_test_accel_xts_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long blocks, + unsigned char *tweak, const symmetric_key *skey1, const symmetric_key *skey2) +{ + int ret; + symmetric_xts xts; + int (*orig)(const unsigned char *, unsigned char *, + unsigned long , unsigned char *, + const symmetric_key *, const symmetric_key *); + + /* AES can be under rijndael or aes... try to find it */ + if ((xts.cipher = find_cipher("aes")) == -1) { + if ((xts.cipher = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + orig = cipher_descriptor[xts.cipher]->accel_xts_encrypt; + cipher_descriptor[xts.cipher]->accel_xts_encrypt = NULL; + + XMEMCPY(&xts.key1, skey1, sizeof(symmetric_key)); + XMEMCPY(&xts.key2, skey2, sizeof(symmetric_key)); + + ret = xts_encrypt(pt, blocks << 4, ct, tweak, &xts); + cipher_descriptor[xts.cipher]->accel_xts_encrypt = orig; + + return ret; +} + +static int s_xts_test_accel_xts_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long blocks, + unsigned char *tweak, const symmetric_key *skey1, const symmetric_key *skey2) +{ + int ret; + symmetric_xts xts; + int (*orig)(const unsigned char *, unsigned char *, + unsigned long , unsigned char *, + const symmetric_key *, const symmetric_key *); + + /* AES can be under rijndael or aes... try to find it */ + if ((xts.cipher = find_cipher("aes")) == -1) { + if ((xts.cipher = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + orig = cipher_descriptor[xts.cipher]->accel_xts_decrypt; + cipher_descriptor[xts.cipher]->accel_xts_decrypt = NULL; + + XMEMCPY(&xts.key1, skey1, sizeof(symmetric_key)); + XMEMCPY(&xts.key2, skey2, sizeof(symmetric_key)); + + ret = xts_decrypt(ct, blocks << 4, pt, tweak, &xts); + cipher_descriptor[xts.cipher]->accel_xts_decrypt = orig; + + return ret; +} +#endif + +/** + Source donated by Elliptic Semiconductor Inc (www.ellipticsemi.com) to the LibTom Projects + + Returns CRYPT_OK upon success. +*/ +int xts_test(void) +{ +#ifdef LTC_NO_TEST + return CRYPT_NOP; +#else + static const struct + { + int keylen; + unsigned char key1[32]; + unsigned char key2[32]; + ulong64 seqnum; + unsigned long PTLEN; + unsigned char PTX[512], CTX[512]; + } tests[] = { + +/* #1 32 byte key, 32 byte PTX */ +{ + 32, + { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 }, + { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 }, + 0, + 32, + { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 }, + { 0x91,0x7c,0xf6,0x9e,0xbd,0x68,0xb2,0xec,0x9b,0x9f,0xe9,0xa3,0xea,0xdd,0xa6,0x92,0xcd,0x43,0xd2,0xf5,0x95,0x98,0xed,0x85,0x8c,0x02,0xc2,0x65,0x2f,0xbf,0x92,0x2e }, +}, + +/* #2, 32 byte key, 32 byte PTX */ +{ + 32, + { 0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11 }, + { 0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22 }, + CONST64(0x3333333333), + 32, + { 0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44 }, + { 0xc4,0x54,0x18,0x5e,0x6a,0x16,0x93,0x6e,0x39,0x33,0x40,0x38,0xac,0xef,0x83,0x8b,0xfb,0x18,0x6f,0xff,0x74,0x80,0xad,0xc4,0x28,0x93,0x82,0xec,0xd6,0xd3,0x94,0xf0 }, +}, + +/* #5 from xts.7, 32 byte key, 32 byte PTX */ +{ + 32, + { 0xff,0xfe,0xfd,0xfc,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf5,0xf4,0xf3,0xf2,0xf1,0xf0 }, + { 0xbf,0xbe,0xbd,0xbc,0xbb,0xba,0xb9,0xb8,0xb7,0xb6,0xb5,0xb4,0xb3,0xb2,0xb1,0xb0 }, + CONST64(0x123456789a), + 32, + { 0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44 }, + { 0xb0,0x1f,0x86,0xf8,0xed,0xc1,0x86,0x37,0x06,0xfa,0x8a,0x42,0x53,0xe3,0x4f,0x28,0xaf,0x31,0x9d,0xe3,0x83,0x34,0x87,0x0f,0x4d,0xd1,0xf9,0x4c,0xbe,0x98,0x32,0xf1 }, +}, + +/* #4, 32 byte key, 512 byte PTX */ +{ + 32, + { 0x27,0x18,0x28,0x18,0x28,0x45,0x90,0x45,0x23,0x53,0x60,0x28,0x74,0x71,0x35,0x26 }, + { 0x31,0x41,0x59,0x26,0x53,0x58,0x97,0x93,0x23,0x84,0x62,0x64,0x33,0x83,0x27,0x95 }, + 0, + 512, + { +0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f, +0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f, +0x40,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f, +0x60,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f, +0x80,0x81,0x82,0x83,0x84,0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f, +0xa0,0xa1,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf, +0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf, +0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff, +0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f, +0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f, +0x40,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f, +0x60,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f, +0x80,0x81,0x82,0x83,0x84,0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f, +0xa0,0xa1,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf, +0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf, +0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff, + }, + { +0x27,0xa7,0x47,0x9b,0xef,0xa1,0xd4,0x76,0x48,0x9f,0x30,0x8c,0xd4,0xcf,0xa6,0xe2,0xa9,0x6e,0x4b,0xbe,0x32,0x08,0xff,0x25,0x28,0x7d,0xd3,0x81,0x96,0x16,0xe8,0x9c, +0xc7,0x8c,0xf7,0xf5,0xe5,0x43,0x44,0x5f,0x83,0x33,0xd8,0xfa,0x7f,0x56,0x00,0x00,0x05,0x27,0x9f,0xa5,0xd8,0xb5,0xe4,0xad,0x40,0xe7,0x36,0xdd,0xb4,0xd3,0x54,0x12, +0x32,0x80,0x63,0xfd,0x2a,0xab,0x53,0xe5,0xea,0x1e,0x0a,0x9f,0x33,0x25,0x00,0xa5,0xdf,0x94,0x87,0xd0,0x7a,0x5c,0x92,0xcc,0x51,0x2c,0x88,0x66,0xc7,0xe8,0x60,0xce, +0x93,0xfd,0xf1,0x66,0xa2,0x49,0x12,0xb4,0x22,0x97,0x61,0x46,0xae,0x20,0xce,0x84,0x6b,0xb7,0xdc,0x9b,0xa9,0x4a,0x76,0x7a,0xae,0xf2,0x0c,0x0d,0x61,0xad,0x02,0x65, +0x5e,0xa9,0x2d,0xc4,0xc4,0xe4,0x1a,0x89,0x52,0xc6,0x51,0xd3,0x31,0x74,0xbe,0x51,0xa1,0x0c,0x42,0x11,0x10,0xe6,0xd8,0x15,0x88,0xed,0xe8,0x21,0x03,0xa2,0x52,0xd8, +0xa7,0x50,0xe8,0x76,0x8d,0xef,0xff,0xed,0x91,0x22,0x81,0x0a,0xae,0xb9,0x9f,0x91,0x72,0xaf,0x82,0xb6,0x04,0xdc,0x4b,0x8e,0x51,0xbc,0xb0,0x82,0x35,0xa6,0xf4,0x34, +0x13,0x32,0xe4,0xca,0x60,0x48,0x2a,0x4b,0xa1,0xa0,0x3b,0x3e,0x65,0x00,0x8f,0xc5,0xda,0x76,0xb7,0x0b,0xf1,0x69,0x0d,0xb4,0xea,0xe2,0x9c,0x5f,0x1b,0xad,0xd0,0x3c, +0x5c,0xcf,0x2a,0x55,0xd7,0x05,0xdd,0xcd,0x86,0xd4,0x49,0x51,0x1c,0xeb,0x7e,0xc3,0x0b,0xf1,0x2b,0x1f,0xa3,0x5b,0x91,0x3f,0x9f,0x74,0x7a,0x8a,0xfd,0x1b,0x13,0x0e, +0x94,0xbf,0xf9,0x4e,0xff,0xd0,0x1a,0x91,0x73,0x5c,0xa1,0x72,0x6a,0xcd,0x0b,0x19,0x7c,0x4e,0x5b,0x03,0x39,0x36,0x97,0xe1,0x26,0x82,0x6f,0xb6,0xbb,0xde,0x8e,0xcc, +0x1e,0x08,0x29,0x85,0x16,0xe2,0xc9,0xed,0x03,0xff,0x3c,0x1b,0x78,0x60,0xf6,0xde,0x76,0xd4,0xce,0xcd,0x94,0xc8,0x11,0x98,0x55,0xef,0x52,0x97,0xca,0x67,0xe9,0xf3, +0xe7,0xff,0x72,0xb1,0xe9,0x97,0x85,0xca,0x0a,0x7e,0x77,0x20,0xc5,0xb3,0x6d,0xc6,0xd7,0x2c,0xac,0x95,0x74,0xc8,0xcb,0xbc,0x2f,0x80,0x1e,0x23,0xe5,0x6f,0xd3,0x44, +0xb0,0x7f,0x22,0x15,0x4b,0xeb,0xa0,0xf0,0x8c,0xe8,0x89,0x1e,0x64,0x3e,0xd9,0x95,0xc9,0x4d,0x9a,0x69,0xc9,0xf1,0xb5,0xf4,0x99,0x02,0x7a,0x78,0x57,0x2a,0xee,0xbd, +0x74,0xd2,0x0c,0xc3,0x98,0x81,0xc2,0x13,0xee,0x77,0x0b,0x10,0x10,0xe4,0xbe,0xa7,0x18,0x84,0x69,0x77,0xae,0x11,0x9f,0x7a,0x02,0x3a,0xb5,0x8c,0xca,0x0a,0xd7,0x52, +0xaf,0xe6,0x56,0xbb,0x3c,0x17,0x25,0x6a,0x9f,0x6e,0x9b,0xf1,0x9f,0xdd,0x5a,0x38,0xfc,0x82,0xbb,0xe8,0x72,0xc5,0x53,0x9e,0xdb,0x60,0x9e,0xf4,0xf7,0x9c,0x20,0x3e, +0xbb,0x14,0x0f,0x2e,0x58,0x3c,0xb2,0xad,0x15,0xb4,0xaa,0x5b,0x65,0x50,0x16,0xa8,0x44,0x92,0x77,0xdb,0xd4,0x77,0xef,0x2c,0x8d,0x6c,0x01,0x7d,0xb7,0x38,0xb1,0x8d, +0xeb,0x4a,0x42,0x7d,0x19,0x23,0xce,0x3f,0xf2,0x62,0x73,0x57,0x79,0xa4,0x18,0xf2,0x0a,0x28,0x2d,0xf9,0x20,0x14,0x7b,0xea,0xbe,0x42,0x1e,0xe5,0x31,0x9d,0x05,0x68, + } +}, + +/* #7, 32 byte key, 17 byte PTX */ +{ + 32, + { 0xff,0xfe,0xfd,0xfc,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf5,0xf4,0xf3,0xf2,0xf1,0xf0 }, + { 0xbf,0xbe,0xbd,0xbc,0xbb,0xba,0xb9,0xb8,0xb7,0xb6,0xb5,0xb4,0xb3,0xb2,0xb1,0xb0 }, + CONST64(0x123456789a), + 17, + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10 }, + { 0x6c,0x16,0x25,0xdb,0x46,0x71,0x52,0x2d,0x3d,0x75,0x99,0x60,0x1d,0xe7,0xca,0x09,0xed }, +}, + +/* #15, 32 byte key, 25 byte PTX */ +{ + 32, + { 0xff,0xfe,0xfd,0xfc,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf5,0xf4,0xf3,0xf2,0xf1,0xf0 }, + { 0xbf,0xbe,0xbd,0xbc,0xbb,0xba,0xb9,0xb8,0xb7,0xb6,0xb5,0xb4,0xb3,0xb2,0xb1,0xb0 }, + CONST64(0x123456789a), + 25, + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18 }, + { 0x8f,0x4d,0xcb,0xad,0x55,0x55,0x8d,0x7b,0x4e,0x01,0xd9,0x37,0x9c,0xd4,0xea,0x22,0xed,0xbf,0x9d,0xac,0xe4,0x5d,0x6f,0x6a,0x73 }, +}, + +/* #21, 32 byte key, 31 byte PTX */ +{ + 32, + { 0xff,0xfe,0xfd,0xfc,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf5,0xf4,0xf3,0xf2,0xf1,0xf0 }, + { 0xbf,0xbe,0xbd,0xbc,0xbb,0xba,0xb9,0xb8,0xb7,0xb6,0xb5,0xb4,0xb3,0xb2,0xb1,0xb0 }, + CONST64(0x123456789a), + 31, + { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e }, + { 0xd0,0x5b,0xc0,0x90,0xa8,0xe0,0x4f,0x1b,0x3d,0x3e,0xcd,0xd5,0xba,0xec,0x0f,0xd4,0xed,0xbf,0x9d,0xac,0xe4,0x5d,0x6f,0x6a,0x73,0x06,0xe6,0x4b,0xe5,0xdd,0x82 }, +}, + +}; + unsigned char OUT[512], Torg[16], T[16]; + ulong64 seq; + symmetric_xts xts; + int i, j, k, err, idx; + unsigned long len; + + /* AES can be under rijndael or aes... try to find it */ + if ((idx = find_cipher("aes")) == -1) { + if ((idx = find_cipher("rijndael")) == -1) { + return CRYPT_NOP; + } + } + for (k = 0; k < 4; ++k) { + cipher_descriptor[idx]->accel_xts_encrypt = NULL; + cipher_descriptor[idx]->accel_xts_decrypt = NULL; + if (k & 0x1) { + cipher_descriptor[idx]->accel_xts_encrypt = s_xts_test_accel_xts_encrypt; + } + if (k & 0x2) { + cipher_descriptor[idx]->accel_xts_decrypt = s_xts_test_accel_xts_decrypt; + } + for (j = 0; j < 2; j++) { + for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { + /* skip the cases where + * the length is smaller than 2*blocklen + * or the length is not a multiple of 32 + */ + if ((j == 1) && ((tests[i].PTLEN < 32) || (tests[i].PTLEN % 32))) { + continue; + } + if ((k > 0) && (j == 1)) { + continue; + } + len = tests[i].PTLEN / 2; + + err = xts_start(idx, tests[i].key1, tests[i].key2, tests[i].keylen / 2, 0, &xts); + if (err != CRYPT_OK) { + return err; + } + + seq = tests[i].seqnum; + STORE64L(seq, Torg); + XMEMSET(Torg + 8, 0, 8); + + XMEMCPY(T, Torg, sizeof(T)); + if (j == 0) { + err = xts_encrypt(tests[i].PTX, tests[i].PTLEN, OUT, T, &xts); + if (err != CRYPT_OK) { + xts_done(&xts); + return err; + } + } else { + err = xts_encrypt(tests[i].PTX, len, OUT, T, &xts); + if (err != CRYPT_OK) { + xts_done(&xts); + return err; + } + err = xts_encrypt(&tests[i].PTX[len], len, &OUT[len], T, &xts); + if (err != CRYPT_OK) { + xts_done(&xts); + return err; + } + } + + if (compare_testvector(OUT, tests[i].PTLEN, tests[i].CTX, tests[i].PTLEN, "XTS encrypt", i)) { + xts_done(&xts); + return CRYPT_FAIL_TESTVECTOR; + } + + XMEMCPY(T, Torg, sizeof(T)); + if (j == 0) { + err = xts_decrypt(tests[i].CTX, tests[i].PTLEN, OUT, T, &xts); + if (err != CRYPT_OK) { + xts_done(&xts); + return err; + } + } else { + err = xts_decrypt(tests[i].CTX, len, OUT, T, &xts); + if (err != CRYPT_OK) { + xts_done(&xts); + return err; + } + err = xts_decrypt(&tests[i].CTX[len], len, &OUT[len], T, &xts); + if (err != CRYPT_OK) { + xts_done(&xts); + return err; + } + } + + if (compare_testvector(OUT, tests[i].PTLEN, tests[i].PTX, tests[i].PTLEN, "XTS decrypt", i)) { + xts_done(&xts); + return CRYPT_FAIL_TESTVECTOR; + } + xts_done(&xts); + } + } + } + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_decode_bit_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_decode_bit_string.c new file mode 100644 index 0000000..83c915d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_decode_bit_string.c @@ -0,0 +1,77 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_bit_string.c + ASN.1 DER, encode a BIT STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store a BIT STRING + @param in The DER encoded BIT STRING + @param inlen The size of the DER BIT STRING + @param out [out] The array of bits stored (one per char) + @param outlen [in/out] The number of bits stored + @return CRYPT_OK if successful +*/ +int der_decode_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long dlen, blen, x, y; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* packet must be at least 4 bytes */ + if (inlen < 4) { + return CRYPT_INVALID_ARG; + } + + /* check for 0x03 */ + if ((in[0]&0x1F) != 0x03) { + return CRYPT_INVALID_PACKET; + } + + /* offset in the data */ + x = 1; + + /* get the length of the data */ + y = inlen - 1; + if ((err = der_decode_asn1_length(in + x, &y, &dlen)) != CRYPT_OK) { + return err; + } + x += y; + /* is the data len too long or too short? */ + if ((dlen == 0) || (dlen > (inlen - x))) { + return CRYPT_INVALID_PACKET; + } + + /* get padding count */ + blen = ((dlen - 1) << 3) - (in[x++] & 7); + + /* too many bits? */ + if (blen > *outlen) { + *outlen = blen; + return CRYPT_BUFFER_OVERFLOW; + } + + /* decode/store the bits */ + for (y = 0; y < blen; y++) { + out[y] = (in[x] & (1 << (7 - (y & 7)))) ? 1 : 0; + if ((y & 7) == 7) { + ++x; + } + } + + /* we done */ + *outlen = blen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_decode_raw_bit_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_decode_raw_bit_string.c new file mode 100644 index 0000000..1bc74f6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_decode_raw_bit_string.c @@ -0,0 +1,84 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_bit_string.c + ASN.1 DER, encode a BIT STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +#define SETBIT(v, n) (v=((unsigned char)(v) | (1U << (unsigned char)(n)))) +#define CLRBIT(v, n) (v=((unsigned char)(v) & ~(1U << (unsigned char)(n)))) + +/** + Store a BIT STRING + @param in The DER encoded BIT STRING + @param inlen The size of the DER BIT STRING + @param out [out] The array of bits stored (8 per char) + @param outlen [in/out] The number of bits stored + @return CRYPT_OK if successful +*/ +int der_decode_raw_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long dlen, blen, x, y; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* packet must be at least 4 bytes */ + if (inlen < 4) { + return CRYPT_INVALID_ARG; + } + + /* check for 0x03 */ + if ((in[0]&0x1F) != 0x03) { + return CRYPT_INVALID_PACKET; + } + + /* offset in the data */ + x = 1; + + /* get the length of the data */ + y = inlen - 1; + if ((err = der_decode_asn1_length(in + x, &y, &dlen)) != CRYPT_OK) { + return err; + } + x += y; + /* is the data len too long or too short? */ + if ((dlen == 0) || (dlen > (inlen - x))) { + return CRYPT_INVALID_PACKET; + } + + /* get padding count */ + blen = ((dlen - 1) << 3) - (in[x++] & 7); + + /* too many bits? */ + if (blen > *outlen) { + *outlen = blen; + return CRYPT_BUFFER_OVERFLOW; + } + + /* decode/store the bits */ + for (y = 0; y < blen; y++) { + if (in[x] & (1 << (7 - (y & 7)))) { + SETBIT(out[y/8], 7-(y%8)); + } else { + CLRBIT(out[y/8], 7-(y%8)); + } + if ((y & 7) == 7) { + ++x; + } + } + + /* we done */ + *outlen = blen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_encode_bit_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_encode_bit_string.c new file mode 100644 index 0000000..0c2cd16 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_encode_bit_string.c @@ -0,0 +1,72 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_bit_string.c + ASN.1 DER, encode a BIT STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store a BIT STRING + @param in The array of bits to store (one per char) + @param inlen The number of bits tostore + @param out [out] The destination for the DER encoded BIT STRING + @param outlen [in/out] The max size and resulting size of the DER BIT STRING + @return CRYPT_OK if successful +*/ +int der_encode_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long len, x, y; + unsigned char buf; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* avoid overflows */ + if ((err = der_length_bit_string(inlen, &len)) != CRYPT_OK) { + return err; + } + + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + /* store header (include bit padding count in length) */ + x = 0; + y = ((inlen + 7) >> 3) + 1; + + out[x++] = 0x03; + len = *outlen - x; + if ((err = der_encode_asn1_length(y, out + x, &len)) != CRYPT_OK) { + return err; + } + x += len; + + /* store number of zero padding bits */ + out[x++] = (unsigned char)((8 - inlen) & 7); + + /* store the bits in big endian format */ + for (y = buf = 0; y < inlen; y++) { + buf |= (in[y] ? 1 : 0) << (7 - (y & 7)); + if ((y & 7) == 7) { + out[x++] = buf; + buf = 0; + } + } + /* store last byte */ + if (inlen & 7) { + out[x++] = buf; + } + *outlen = x; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_encode_raw_bit_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_encode_raw_bit_string.c new file mode 100644 index 0000000..9cc23ae --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_encode_raw_bit_string.c @@ -0,0 +1,75 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_bit_string.c + ASN.1 DER, encode a BIT STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +#define getbit(n, k) (((n) & ( 1 << (k) )) >> (k)) + +/** + Store a BIT STRING + @param in The array of bits to store (8 per char) + @param inlen The number of bits to store + @param out [out] The destination for the DER encoded BIT STRING + @param outlen [in/out] The max size and resulting size of the DER BIT STRING + @return CRYPT_OK if successful +*/ +int der_encode_raw_bit_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long len, x, y; + unsigned char buf; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* avoid overflows */ + if ((err = der_length_bit_string(inlen, &len)) != CRYPT_OK) { + return err; + } + + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + /* store header (include bit padding count in length) */ + x = 0; + y = ((inlen + 7) >> 3) + 1; + + out[x++] = 0x03; + len = *outlen - x; + if ((err = der_encode_asn1_length(y, out + x, &len)) != CRYPT_OK) { + return err; + } + x += len; + + /* store number of zero padding bits */ + out[x++] = (unsigned char)((8 - inlen) & 7); + + /* store the bits in big endian format */ + for (y = buf = 0; y < inlen; y++) { + buf |= (getbit(in[y/8],7-y%8)?1:0) << (7 - (y & 7)); + if ((y & 7) == 7) { + out[x++] = buf; + buf = 0; + } + } + /* store last byte */ + if (inlen & 7) { + out[x++] = buf; + } + + *outlen = x; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_length_bit_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_length_bit_string.c new file mode 100644 index 0000000..4554a80 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/der_length_bit_string.c @@ -0,0 +1,36 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_bit_string.c + ASN.1 DER, get length of BIT STRING, Tom St Denis +*/ + +#ifdef LTC_DER +/** + Gets length of DER encoding of BIT STRING + @param nbits The number of bits in the string to encode + @param outlen [out] The length of the DER encoding for the given string + @return CRYPT_OK if successful +*/ +int der_length_bit_string(unsigned long nbits, unsigned long *outlen) +{ + unsigned long nbytes, x; + int err; + + LTC_ARGCHK(outlen != NULL); + + /* get the number of the bytes */ + nbytes = (nbits >> 3) + ((nbits & 7) ? 1 : 0) + 1; + + if ((err = der_length_asn1_length(nbytes, &x)) != CRYPT_OK) { + return err; + } + *outlen = 1 + x + nbytes; + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/sub.mk new file mode 100644 index 0000000..4cf829e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/bit/sub.mk @@ -0,0 +1,5 @@ +srcs-y += der_decode_bit_string.c +srcs-y += der_encode_bit_string.c +srcs-y += der_length_bit_string.c +srcs-y += der_decode_raw_bit_string.c +srcs-y += der_encode_raw_bit_string.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_decode_boolean.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_decode_boolean.c new file mode 100644 index 0000000..35ac7ec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_decode_boolean.c @@ -0,0 +1,35 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_boolean.c + ASN.1 DER, decode a BOOLEAN, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Read a BOOLEAN + @param in The destination for the DER encoded BOOLEAN + @param inlen The size of the DER BOOLEAN + @param out [out] The boolean to decode + @return CRYPT_OK if successful +*/ +int der_decode_boolean(const unsigned char *in, unsigned long inlen, + int *out) +{ + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + + if (inlen < 3 || in[0] != 0x01 || in[1] != 0x01 || (in[2] != 0x00 && in[2] != 0xFF)) { + return CRYPT_INVALID_ARG; + } + + *out = (in[2]==0xFF) ? 1 : 0; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_encode_boolean.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_encode_boolean.c new file mode 100644 index 0000000..a77aefb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_encode_boolean.c @@ -0,0 +1,39 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_boolean.c + ASN.1 DER, encode a BOOLEAN, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store a BOOLEAN + @param in The boolean to encode + @param out [out] The destination for the DER encoded BOOLEAN + @param outlen [in/out] The max size and resulting size of the DER BOOLEAN + @return CRYPT_OK if successful +*/ +int der_encode_boolean(int in, + unsigned char *out, unsigned long *outlen) +{ + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(out != NULL); + + if (*outlen < 3) { + *outlen = 3; + return CRYPT_BUFFER_OVERFLOW; + } + + *outlen = 3; + out[0] = 0x01; + out[1] = 0x01; + out[2] = in ? 0xFF : 0x00; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_length_boolean.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_length_boolean.c new file mode 100644 index 0000000..f1b9e57 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/der_length_boolean.c @@ -0,0 +1,23 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_boolean.c + ASN.1 DER, get length of a BOOLEAN, Tom St Denis +*/ + +#ifdef LTC_DER +/** + Gets length of DER encoding of a BOOLEAN + @param outlen [out] The length of the DER encoding + @return CRYPT_OK if successful +*/ +int der_length_boolean(unsigned long *outlen) +{ + LTC_ARGCHK(outlen != NULL); + *outlen = 3; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/sub.mk new file mode 100644 index 0000000..3761249 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/boolean/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_boolean.c +srcs-y += der_encode_boolean.c +srcs-y += der_length_boolean.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/choice/der_decode_choice.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/choice/der_decode_choice.c new file mode 100644 index 0000000..7ff7809 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/choice/der_decode_choice.c @@ -0,0 +1,222 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_choice.c + ASN.1 DER, decode a CHOICE, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Decode a CHOICE + @param in The DER encoded input + @param inlen [in/out] The size of the input and resulting size of read type + @param list The list of items to decode + @param outlen The number of items in the list + @return CRYPT_OK on success +*/ +int der_decode_choice(const unsigned char *in, unsigned long *inlen, + ltc_asn1_list *list, unsigned long outlen) +{ + unsigned long size, x, z; + void *data; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != NULL); + LTC_ARGCHK(list != NULL); + + /* get blk size */ + if (*inlen < 2) { + return CRYPT_INVALID_PACKET; + } + + /* set all of the "used" flags to zero */ + for (x = 0; x < outlen; x++) { + list[x].used = 0; + } + + /* now scan until we have a winner */ + for (x = 0; x < outlen; x++) { + size = list[x].size; + data = list[x].data; + + switch (list[x].type) { + case LTC_ASN1_BOOLEAN: + if (der_decode_boolean(in, *inlen, data) == CRYPT_OK) { + if (der_length_boolean(&z) == CRYPT_OK) { + list[x].used = 1; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_INTEGER: + if (der_decode_integer(in, *inlen, data) == CRYPT_OK) { + if (der_length_integer(data, &z) == CRYPT_OK) { + list[x].used = 1; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_SHORT_INTEGER: + if (der_decode_short_integer(in, *inlen, data) == CRYPT_OK) { + if (der_length_short_integer(size, &z) == CRYPT_OK) { + list[x].used = 1; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_BIT_STRING: + if (der_decode_bit_string(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_bit_string(size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_RAW_BIT_STRING: + if (der_decode_raw_bit_string(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_bit_string(size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_OCTET_STRING: + if (der_decode_octet_string(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_octet_string(size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_NULL: + if (*inlen == 2 && in[x] == 0x05 && in[x+1] == 0x00) { + *inlen = 2; + list[x].used = 1; + return CRYPT_OK; + } + break; + + case LTC_ASN1_OBJECT_IDENTIFIER: + if (der_decode_object_identifier(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_object_identifier(data, size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_TELETEX_STRING: + if (der_decode_teletex_string(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_teletex_string(data, size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_IA5_STRING: + if (der_decode_ia5_string(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_ia5_string(data, size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_PRINTABLE_STRING: + if (der_decode_printable_string(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_printable_string(data, size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_UTF8_STRING: + if (der_decode_utf8_string(in, *inlen, data, &size) == CRYPT_OK) { + if (der_length_utf8_string(data, size, &z) == CRYPT_OK) { + list[x].used = 1; + list[x].size = size; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_UTCTIME: + z = *inlen; + if (der_decode_utctime(in, &z, data) == CRYPT_OK) { + list[x].used = 1; + *inlen = z; + return CRYPT_OK; + } + break; + + case LTC_ASN1_GENERALIZEDTIME: + z = *inlen; + if (der_decode_generalizedtime(in, &z, data) == CRYPT_OK) { + list[x].used = 1; + *inlen = z; + return CRYPT_OK; + } + break; + + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_SEQUENCE: + if (der_decode_sequence(in, *inlen, data, size) == CRYPT_OK) { + if (der_length_sequence(data, size, &z) == CRYPT_OK) { + list[x].used = 1; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_CUSTOM_TYPE: + if (der_decode_custom_type(in, *inlen, &list[x]) == CRYPT_OK) { + if (der_length_custom_type(&list[x], &z, NULL) == CRYPT_OK) { + list[x].used = 1; + *inlen = z; + return CRYPT_OK; + } + } + break; + + case LTC_ASN1_CHOICE: + case LTC_ASN1_EOL: + default: + return CRYPT_INVALID_ARG; + } + } + + return CRYPT_INVALID_PACKET; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/choice/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/choice/sub.mk new file mode 100644 index 0000000..d3b95ec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/choice/sub.mk @@ -0,0 +1 @@ +srcs-y += der_decode_choice.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_decode_custom_type.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_decode_custom_type.c new file mode 100644 index 0000000..59a98cc --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_decode_custom_type.c @@ -0,0 +1,427 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +/** + @file der_decode_custom_type.c + ASN.1 DER, decode a Custom type, Steffen Jaeckel +*/ + +#ifdef LTC_DER + +/** + Decode a Custom type + @param in The DER encoded input + @param inlen The size of the input + @param root The item that defines the custom type to decode + @return CRYPT_OK on success +*/ +int der_decode_custom_type(const unsigned char *in, unsigned long inlen, + ltc_asn1_list *root) +{ + LTC_ARGCHK(root != NULL); + return der_decode_custom_type_ex(in, inlen, root, NULL, 0, LTC_DER_SEQ_ORDERED | LTC_DER_SEQ_RELAXED); +} + +/** + Extended-decode a Custom type + + This function is used to decode custom types and sequences/sets + For custom types root is used + For sequences/sets list and outlen are used + + @param in The DER encoded input + @param inlen The size of the input + @param root The item that defines the custom type to decode + @param list The list of items to decode + @param outlen The number of items in the list + @param flags c.f. enum ltc_der_seq + @return CRYPT_OK on success +*/ +int der_decode_custom_type_ex(const unsigned char *in, unsigned long inlen, + ltc_asn1_list *root, + ltc_asn1_list *list, unsigned long outlen, + unsigned int flags) +{ + int err, seq_err, i, ordered; + ltc_asn1_type type; + ltc_asn1_list ident; + unsigned long size, x, y, z, blksize; + unsigned char* in_new = NULL; + void *data; + + LTC_ARGCHK(in != NULL); + + /* get blk size */ + if (inlen < 2) { + return CRYPT_INVALID_PACKET; + } + x = 0; + + if (root == NULL) { + LTC_ARGCHK(list != NULL); + + /* sequence type? We allow 0x30 SEQUENCE and 0x31 SET since fundamentally they're the same structure */ + if (in[x] != 0x30 && in[x] != 0x31) { + return CRYPT_INVALID_PACKET; + } + ++x; + } else { + if (root->type != LTC_ASN1_CUSTOM_TYPE) { + return CRYPT_INVALID_PACKET; + } + + /* Alloc a copy of the data for primitive handling. */ + if (root->pc == LTC_ASN1_PC_PRIMITIVE) { + in_new = XMALLOC(inlen); + if (in_new == NULL) { + return CRYPT_MEM; + } + XMEMCPY(in_new, in, inlen); + in = in_new; + } + + y = inlen; + if ((err = der_decode_asn1_identifier(in, &y, &ident)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((ident.type != root->type) || + (ident.klass != root->klass) || + (ident.pc != root->pc) || + (ident.tag != root->tag)) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + x += y; + + list = root->data; + outlen = root->size; + } + + if (root != NULL && root->pc == LTC_ASN1_PC_PRIMITIVE) { + if (((unsigned long)root->used >= der_asn1_type_to_identifier_map_sz) || + (der_asn1_type_to_identifier_map[root->used] == -1)) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + root->type = (ltc_asn1_type)root->used; + list = root; + outlen = 1; + + x -= 1; + in_new[x] = (unsigned char)der_asn1_type_to_identifier_map[list[0].type]; + blksize = inlen - x; + } else { + + y = inlen - x; + if ((err = der_decode_asn1_length(&in[x], &y, &blksize)) != CRYPT_OK) { + goto LBL_ERR; + } + x += y; + } + + /* would this blksize overflow? */ + if (blksize > (inlen - x)) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + /* mark all as unused */ + for (i = 0; i < (int)outlen; i++) { + list[i].used = 0; + } + ordered = flags & LTC_DER_SEQ_ORDERED; + + /* ok read data */ + seq_err = CRYPT_OK; + blksize += x; + inlen -= x; + for (i = 0; i < (int)outlen; i++) { + z = 0; + type = list[i].type; + size = list[i].size; + data = list[i].data; + if (!ordered && list[i].used == 1) { continue; } + + if (type == LTC_ASN1_EOL) { + break; + } + + if (root != NULL && root->pc == LTC_ASN1_PC_PRIMITIVE && i != 0) { + err = CRYPT_PK_ASN1_ERROR; + goto LBL_ERR; + } + + switch (type) { + case LTC_ASN1_CUSTOM_TYPE: + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_SEQUENCE: + break; + default: + /* Verify that all basic types are indeed UNIVERSAL&PRIMITIVE */ + if (((flags & LTC_DER_SEQ_STRICT) == LTC_DER_SEQ_STRICT) && (inlen > 0)) { + if (in[x] & 0xE0u) { + err = CRYPT_PK_ASN1_ERROR; + goto LBL_ERR; + } + } + } + + switch (type) { + case LTC_ASN1_BOOLEAN: + z = inlen; + if ((err = der_decode_boolean(in + x, z, ((int *)data))) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + if ((err = der_length_boolean(&z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_INTEGER: + z = inlen; + if ((err = der_decode_integer(in + x, z, data)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + if ((err = der_length_integer(data, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SHORT_INTEGER: + z = inlen; + if ((err = der_decode_short_integer(in + x, z, data)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + if ((err = der_length_short_integer(((unsigned long*)data)[0], &z)) != CRYPT_OK) { + goto LBL_ERR; + } + + break; + + case LTC_ASN1_BIT_STRING: + z = inlen; + if ((err = der_decode_bit_string(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_bit_string(size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_RAW_BIT_STRING: + z = inlen; + if ((err = der_decode_raw_bit_string(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_bit_string(size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_OCTET_STRING: + z = inlen; + if ((err = der_decode_octet_string(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_octet_string(size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_NULL: + if (inlen < 2 || in[x] != 0x05 || in[x+1] != 0x00) { + if (!ordered || list[i].optional) { continue; } + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + z = 2; + break; + + case LTC_ASN1_OBJECT_IDENTIFIER: + z = inlen; + if ((err = der_decode_object_identifier(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_object_identifier(data, size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_TELETEX_STRING: + z = inlen; + if ((err = der_decode_teletex_string(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_teletex_string(data, size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_IA5_STRING: + z = inlen; + if ((err = der_decode_ia5_string(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_ia5_string(data, size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_PRINTABLE_STRING: + z = inlen; + if ((err = der_decode_printable_string(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_printable_string(data, size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_UTF8_STRING: + z = inlen; + if ((err = der_decode_utf8_string(in + x, z, data, &size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + list[i].size = size; + if ((err = der_length_utf8_string(data, size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_UTCTIME: + z = inlen; + if ((err = der_decode_utctime(in + x, &z, data)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + break; + + case LTC_ASN1_GENERALIZEDTIME: + z = inlen; + if ((err = der_decode_generalizedtime(in + x, &z, data)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + break; + + case LTC_ASN1_SET: + z = inlen; + if ((err = der_decode_set(in + x, z, data, size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + if ((err = der_length_sequence(data, size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SETOF: + case LTC_ASN1_SEQUENCE: + /* detect if we have the right type */ + if ((type == LTC_ASN1_SETOF && (in[x] & 0x3F) != 0x31) || (type == LTC_ASN1_SEQUENCE && (in[x] & 0x3F) != 0x30)) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + z = inlen; + err = der_decode_sequence_ex(in + x, z, data, size, flags); + if (err == CRYPT_INPUT_TOO_LONG) { + seq_err = CRYPT_INPUT_TOO_LONG; + err = CRYPT_OK; + } + if (err != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + if ((err = der_length_sequence(data, size, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_CUSTOM_TYPE: + z = inlen; + err = der_decode_custom_type(in + x, z, &list[i]); + if (err == CRYPT_INPUT_TOO_LONG) { + seq_err = CRYPT_INPUT_TOO_LONG; + err = CRYPT_OK; + } + if (err != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + if ((err = der_length_custom_type(&list[i], &z, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_CHOICE: + z = inlen; + if ((err = der_decode_choice(in + x, &z, data, size)) != CRYPT_OK) { + if (!ordered || list[i].optional) { continue; } + goto LBL_ERR; + } + break; + + case LTC_ASN1_EOL: + default: + err = CRYPT_INVALID_ARG; + goto LBL_ERR; + } + x += z; + inlen -= z; + list[i].used = 1; + if (!ordered) { + /* restart the decoder */ + i = -1; + } + } + + for (i = 0; i < (int)outlen; i++) { + if (list[i].used == 0 && list[i].optional == 0) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + } + + if (blksize == x && seq_err == CRYPT_OK && inlen == 0) { + /* everything decoded and no errors in nested sequences */ + err = CRYPT_OK; + } else if (blksize == x && seq_err == CRYPT_INPUT_TOO_LONG && inlen == 0) { + /* a sequence reported too-long input, but now we've decoded everything */ + err = CRYPT_OK; + } else if (blksize != x && ((flags & LTC_DER_SEQ_STRICT) == LTC_DER_SEQ_STRICT)) { + err = CRYPT_INVALID_PACKET; + } else { + err = CRYPT_INPUT_TOO_LONG; + } + +LBL_ERR: + if (in_new != NULL) { + XFREE(in_new); + } + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_encode_custom_type.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_encode_custom_type.c new file mode 100644 index 0000000..f5cc93e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_encode_custom_type.c @@ -0,0 +1,229 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +/** + @file der_encode_custom_type.c + ASN.1 DER, encode a Custom Type, Steffen Jaeckel +*/ + +#ifdef LTC_DER + +/** + Encode a Custom Type + + This function is a bit special compared to the others, as it requires the + root-ltc_asn1_list where the type is defined. + + @param root The root of the list of items to encode + @param out [out] The destination + @param outlen [in/out] The size of the output + @return CRYPT_OK on success +*/ +int der_encode_custom_type(const ltc_asn1_list *root, + unsigned char *out, unsigned long *outlen) +{ + int err; + ltc_asn1_type type; + const ltc_asn1_list *list; + unsigned long size, x, y, z, i, inlen, id_len; + void *data; + + LTC_ARGCHK(root != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get size of output that will be required */ + y = 0; z = 0; + if ((err = der_length_custom_type(root, &y, &z)) != CRYPT_OK) return CRYPT_INVALID_ARG; + + /* too big ? */ + if (*outlen < y) { + *outlen = y; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + /* get length of the identifier, so we know the offset where to start writing */ + if ((err = der_length_asn1_identifier(root, &id_len)) != CRYPT_OK) return CRYPT_INVALID_ARG; + x = id_len; + + + if (root->pc == LTC_ASN1_PC_PRIMITIVE) { + list = root; + inlen = 1; + /* In case it's a PRIMITIVE type we encode directly to the output + * but leave space for a potentially longer identifier as it will + * simply be replaced afterwards. + */ + x -= 1; + } else { + list = root->data; + inlen = root->size; + /* store length, identifier will be added later */ + y = *outlen - x; + if ((err = der_encode_asn1_length(z, &out[x], &y)) != CRYPT_OK) { + goto LBL_ERR; + } + x += y; + } + + /* store data */ + *outlen -= x; + for (i = 0; i < inlen; i++) { + if (root->pc == LTC_ASN1_PC_PRIMITIVE) { + type = (ltc_asn1_type)list[i].used; + } else { + type = list[i].type; + } + size = list[i].size; + data = list[i].data; + + if (type == LTC_ASN1_EOL) { + break; + } + + switch (type) { + case LTC_ASN1_BOOLEAN: + z = *outlen; + if ((err = der_encode_boolean(*((int *)data), out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_INTEGER: + z = *outlen; + if ((err = der_encode_integer(data, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SHORT_INTEGER: + z = *outlen; + if ((err = der_encode_short_integer(*((unsigned long*)data), out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_BIT_STRING: + z = *outlen; + if ((err = der_encode_bit_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_RAW_BIT_STRING: + z = *outlen; + if ((err = der_encode_raw_bit_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_OCTET_STRING: + z = *outlen; + if ((err = der_encode_octet_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_NULL: + out[x] = 0x05; + out[x+1] = 0x00; + z = 2; + break; + + case LTC_ASN1_OBJECT_IDENTIFIER: + z = *outlen; + if ((err = der_encode_object_identifier(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_IA5_STRING: + z = *outlen; + if ((err = der_encode_ia5_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_PRINTABLE_STRING: + z = *outlen; + if ((err = der_encode_printable_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_UTF8_STRING: + z = *outlen; + if ((err = der_encode_utf8_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_UTCTIME: + z = *outlen; + if ((err = der_encode_utctime(data, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_GENERALIZEDTIME: + z = *outlen; + if ((err = der_encode_generalizedtime(data, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SET: + z = *outlen; + if ((err = der_encode_set(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SETOF: + z = *outlen; + if ((err = der_encode_setof(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SEQUENCE: + z = *outlen; + if ((err = der_encode_sequence_ex(data, size, out + x, &z, type)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_CUSTOM_TYPE: + z = *outlen; + if ((err = der_encode_custom_type(&list[i], out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_CHOICE: + case LTC_ASN1_EOL: + case LTC_ASN1_TELETEX_STRING: + default: + err = CRYPT_INVALID_ARG; + goto LBL_ERR; + } + + + x += z; + *outlen -= z; + } + + if ((err = der_encode_asn1_identifier(root, out, &id_len)) != CRYPT_OK) { + goto LBL_ERR; + } + *outlen = x; + err = CRYPT_OK; + +LBL_ERR: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_length_custom_type.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_length_custom_type.c new file mode 100644 index 0000000..958badf --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/der_length_custom_type.c @@ -0,0 +1,204 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_custom_type.c + ASN.1 DER, length of a custom type, Steffen Jaeckel +*/ + +#ifdef LTC_DER + +/** + Get the length of a DER custom type + + This function is a bit special compared to the others, as it requires the + root-ltc_asn1_list where the type is defined. + + @param root The root of the struct to encode + @param outlen [out] The length required in octets to store it + @param payloadlen [out] The length of the payload in octets + @return CRYPT_OK on success +*/ +int der_length_custom_type(const ltc_asn1_list *root, unsigned long *outlen, unsigned long *payloadlen) +{ + int err; + const ltc_asn1_list *list; + ltc_asn1_type type; + unsigned long size, x, y, i, inlen, id_len; + void *data; + + LTC_ARGCHK(root != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get size of output that will be required */ + if ((err = der_length_asn1_identifier(root, &id_len)) != CRYPT_OK) { + return err; + } + y = id_len; + + if (root->pc == LTC_ASN1_PC_PRIMITIVE) { + list = root; + inlen = 1; + } else { + list = root->data; + inlen = root->size; + } + for (i = 0; i < inlen; i++) { + if (root->pc == LTC_ASN1_PC_PRIMITIVE) { + type = (ltc_asn1_type)list[i].used; + } else { + type = list[i].type; + } + size = list[i].size; + data = list[i].data; + + if (type == LTC_ASN1_EOL) { + break; + } + + /* some items may be optional during import */ + if (!list[i].used && list[i].optional) continue; + + switch (type) { + case LTC_ASN1_BOOLEAN: + if ((err = der_length_boolean(&x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_INTEGER: + if ((err = der_length_integer(data, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_SHORT_INTEGER: + if ((err = der_length_short_integer(*((unsigned long *)data), &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_BIT_STRING: + case LTC_ASN1_RAW_BIT_STRING: + if ((err = der_length_bit_string(size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_OCTET_STRING: + if ((err = der_length_octet_string(size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_NULL: + y += 2; + break; + + case LTC_ASN1_OBJECT_IDENTIFIER: + if ((err = der_length_object_identifier(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_IA5_STRING: + if ((err = der_length_ia5_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_TELETEX_STRING: + if ((err = der_length_teletex_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_PRINTABLE_STRING: + if ((err = der_length_printable_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_UTCTIME: + if ((err = der_length_utctime(data, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_GENERALIZEDTIME: + if ((err = der_length_generalizedtime(data, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_UTF8_STRING: + if ((err = der_length_utf8_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_CUSTOM_TYPE: + if ((err = der_length_custom_type(&list[i], &x, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_SEQUENCE: + if ((err = der_length_sequence(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_CHOICE: + case LTC_ASN1_EOL: + default: + err = CRYPT_INVALID_ARG; + goto LBL_ERR; + } + } + + if (root->pc == LTC_ASN1_PC_PRIMITIVE) { + /* In case it's a PRIMITIVE element we're going + * to only replace the identifier of the one element + * by the custom identifier. + */ + y -= 1; + if (payloadlen != NULL) { + *payloadlen = y - id_len; + } + } else { + /* calc length of length */ + if ((err = der_length_asn1_length(y - id_len, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + if (payloadlen != NULL) { + *payloadlen = y - id_len; + } + y += x; + } + + /* store size */ + *outlen = y; + +LBL_ERR: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/sub.mk new file mode 100644 index 0000000..5ba17b7 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/custom_type/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_custom_type.c +srcs-y += der_encode_custom_type.c +srcs-y += der_length_custom_type.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_asn1_maps.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_asn1_maps.c new file mode 100644 index 0000000..8f54f09 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_asn1_maps.c @@ -0,0 +1,157 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_asn1_maps.c + ASN.1 DER, a collection of maps to convert between different representations, Steffen Jaeckel +*/ + +#ifdef LTC_DER + +/** + A Map from ltc_asn1_type to the regularly used ASN.1 identifier +*/ +const int der_asn1_type_to_identifier_map[] = +{ + /* 0 */ + -1, /* LTC_ASN1_EOL, */ + 1, /* LTC_ASN1_BOOLEAN, */ + 2, /* LTC_ASN1_INTEGER, */ + 2, /* LTC_ASN1_SHORT_INTEGER, */ + 3, /* LTC_ASN1_BIT_STRING, */ + /* 5 */ + 4, /* LTC_ASN1_OCTET_STRING, */ + 5, /* LTC_ASN1_NULL, */ + 6, /* LTC_ASN1_OBJECT_IDENTIFIER, */ + 22, /* LTC_ASN1_IA5_STRING, */ + 19, /* LTC_ASN1_PRINTABLE_STRING, */ + /* 10 */ + 12, /* LTC_ASN1_UTF8_STRING, */ + 23, /* LTC_ASN1_UTCTIME, */ + -1, /* LTC_ASN1_CHOICE, */ + 48, /* LTC_ASN1_SEQUENCE, */ + 49, /* LTC_ASN1_SET, */ + /* 15 */ + 49, /* LTC_ASN1_SETOF, */ + 3, /* LTC_ASN1_RAW_BIT_STRING, */ + 20, /* LTC_ASN1_TELETEX_STRING, */ + 24, /* LTC_ASN1_GENERALIZEDTIME, */ + -1, /* LTC_ASN1_CUSTOM_TYPE, */ +}; +const unsigned long der_asn1_type_to_identifier_map_sz = sizeof(der_asn1_type_to_identifier_map)/sizeof(der_asn1_type_to_identifier_map[0]); + +/** + A Map from the ASN.1 Class to its string +*/ +const char* der_asn1_class_to_string_map[] = +{ + "UNIVERSAL", + "APPLICATION", + "CONTEXT-SPECIFIC", + "PRIVATE", +}; +const unsigned long der_asn1_class_to_string_map_sz = sizeof(der_asn1_class_to_string_map)/sizeof(der_asn1_class_to_string_map[0]); + +/** + A Map from the ASN.1 P/C-bit to its string +*/ +const char* der_asn1_pc_to_string_map[] = +{ + "PRIMITIVE", + "CONSTRUCTED", +}; +const unsigned long der_asn1_pc_to_string_map_sz = sizeof(der_asn1_pc_to_string_map)/sizeof(der_asn1_pc_to_string_map[0]); + +/** + A Map from the ASN.1 tag to its string +*/ +const char* der_asn1_tag_to_string_map[] = +{ + "Reserved for use by the encoding rules", + "Boolean type", + "Integer type", + "Bitstring type", + "Octetstring type", + "Null type", + "Object identifier type", + "Object descriptor type", + "External type and Instance-of type", + "Real type", + "Enumerated type", + "Embedded-pdv type", + "UTF8String type", + "Relative object identifier type", + "The time type", + "Reserved for future editions of this Recommendation | International Standard", + "Sequence and Sequence-of types", + "Set and Set-of types", + "NumericString type", + "PrintableString type", + "TeletexString (T61String) type", + "VideotexString type", + "IA5String type", + "UTCTime type", + "GeneralizedTime type", + "GraphicString type", + "VisibleString (ISO646String) type", + "GeneralString type", + "UniversalString type", + "UnrestrictedCharacterString type", + "BMPString type", + "Date type", + "TimeOfDay type", + "DateTime type", + "Duration type", + "OID internationalized resource identifier type", + "Relative OID internationalized resource identifier type", +}; +const unsigned long der_asn1_tag_to_string_map_sz = sizeof(der_asn1_tag_to_string_map)/sizeof(der_asn1_tag_to_string_map[0]); + +/** + A Map from ASN.1 Tags to ltc_asn1_type +*/ +const ltc_asn1_type der_asn1_tag_to_type_map[] = +{ + /* 0 */ + LTC_ASN1_EOL, /* Reserved for use by the encoding rules */ + LTC_ASN1_BOOLEAN, /* Boolean type */ + LTC_ASN1_INTEGER, /* Integer type */ + LTC_ASN1_BIT_STRING, /* Bitstring type */ + LTC_ASN1_OCTET_STRING, /* Octetstring type */ + /* 5 */ + LTC_ASN1_NULL, /* Null type */ + LTC_ASN1_OBJECT_IDENTIFIER, /* Object identifier type */ + LTC_ASN1_CUSTOM_TYPE, /* Object descriptor type */ + LTC_ASN1_CUSTOM_TYPE, /* External type and Instance-of type */ + LTC_ASN1_CUSTOM_TYPE, /* Real type */ + /* 10 */ + LTC_ASN1_CUSTOM_TYPE, /* Enumerated type */ + LTC_ASN1_CUSTOM_TYPE, /* Embedded-pdv type */ + LTC_ASN1_UTF8_STRING, /* UTF8String type */ + LTC_ASN1_CUSTOM_TYPE, /* Relative object identifier type */ + LTC_ASN1_CUSTOM_TYPE, /* The time type */ + /* 15 */ + LTC_ASN1_EOL, /* Reserved for future editions of this Recommendation | International Standard */ + LTC_ASN1_SEQUENCE, /* Sequence and Sequence-of types */ + LTC_ASN1_SET, /* Set and Set-of types */ + LTC_ASN1_CUSTOM_TYPE, /* NumericString types */ + LTC_ASN1_PRINTABLE_STRING, /* PrintableString types */ + /* 20 */ + LTC_ASN1_TELETEX_STRING, /* TeletexString (T61String) types */ + LTC_ASN1_CUSTOM_TYPE, /* VideotexString types */ + LTC_ASN1_IA5_STRING, /* IA5String types */ + LTC_ASN1_UTCTIME, /* UTCTime types */ + LTC_ASN1_GENERALIZEDTIME, /* GeneralizedTime types */ + /* 25 */ + LTC_ASN1_CUSTOM_TYPE, /* GraphicString types */ + LTC_ASN1_CUSTOM_TYPE, /* VisibleString (ISO646String) types */ + LTC_ASN1_CUSTOM_TYPE, /* GeneralString types */ + LTC_ASN1_CUSTOM_TYPE, /* UniversalString types */ + LTC_ASN1_CUSTOM_TYPE, /* UnrestrictedCharacterString types */ + /* 30 */ + LTC_ASN1_CUSTOM_TYPE, /* BMPString types */ +}; +const unsigned long der_asn1_tag_to_type_map_sz = sizeof(der_asn1_tag_to_type_map)/sizeof(der_asn1_tag_to_type_map[0]); + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_decode_asn1_identifier.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_decode_asn1_identifier.c new file mode 100644 index 0000000..ad4dc7b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_decode_asn1_identifier.c @@ -0,0 +1,123 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_asn1_identifier.c + ASN.1 DER, decode the ASN.1 Identifier, Steffen Jaeckel +*/ + +#ifdef LTC_DER +/* c.f. X.680 & X.690, some decisions backed by X.690 ch. 10.2 */ +static const unsigned char tag_constructed_map[] = +{ + /* 0 */ + 255, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + /* 5 */ + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + /* 10 */ + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + /* 15 */ + 255, + LTC_ASN1_PC_CONSTRUCTED, + LTC_ASN1_PC_CONSTRUCTED, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + /* 20 */ + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + /* 25 */ + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, + LTC_ASN1_PC_PRIMITIVE, +}; + static const unsigned long tag_constructed_map_sz = sizeof(tag_constructed_map)/sizeof(tag_constructed_map[0]); + +/** + Decode the ASN.1 Identifier + @param id Where to store the decoded Identifier + @param in Where to read the Identifier from + @param inlen [in/out] The size of in available/read + @return CRYPT_OK if successful +*/ +int der_decode_asn1_identifier(const unsigned char *in, unsigned long *inlen, ltc_asn1_list *id) +{ + ulong64 tmp; + unsigned long tag_len; + int err; + + LTC_ARGCHK(id != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != NULL); + + if (*inlen == 0) { + return CRYPT_BUFFER_OVERFLOW; + } + + tag_len = 1; + id->klass = (in[0] >> 6) & 0x3; + id->pc = (in[0] >> 5) & 0x1; + id->tag = in[0] & 0x1f; + + err = CRYPT_OK; + if (id->tag == 0x1f) { + id->tag = 0; + do { + if (*inlen < tag_len) { + /* break the loop and trigger the BOF error-code */ + tmp = 0xff; + break; + } + id->tag <<= 7; + id->tag |= in[tag_len] & 0x7f; + tmp = in[tag_len] & 0x80; + tag_len++; + } while ((tmp != 0) && (tag_len < 10)); + + if (tmp != 0) { + err = CRYPT_BUFFER_OVERFLOW; + } else if (id->tag < 0x1f) { + err = CRYPT_PK_ASN1_ERROR; + } + } + + if (err != CRYPT_OK) { + id->pc = 0; + id->klass = 0; + id->tag = 0; + } else { + *inlen = tag_len; + if ((id->klass == LTC_ASN1_CL_UNIVERSAL) && + (id->tag < der_asn1_tag_to_type_map_sz) && + (id->tag < tag_constructed_map_sz) && + (id->pc == tag_constructed_map[id->tag])) { + id->type = der_asn1_tag_to_type_map[id->tag]; + } else { + if ((id->klass == LTC_ASN1_CL_UNIVERSAL) && (id->tag == 0)) { + id->type = LTC_ASN1_EOL; + } else { + id->type = LTC_ASN1_CUSTOM_TYPE; + } + } + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_decode_asn1_length.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_decode_asn1_length.c new file mode 100644 index 0000000..0e5a393 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_decode_asn1_length.c @@ -0,0 +1,59 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_asn1_length.c + ASN.1 DER, decode the ASN.1 Length field, Steffen Jaeckel +*/ + +#ifdef LTC_DER +/** + Decode the ASN.1 Length field + @param in Where to read the length field from + @param inlen [in/out] The size of in available/read + @param outlen [out] The decoded ASN.1 length + @return CRYPT_OK if successful +*/ +int der_decode_asn1_length(const unsigned char *in, unsigned long *inlen, unsigned long *outlen) +{ + unsigned long real_len, decoded_len, offset, i; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != NULL); + + if (*inlen < 1) { + return CRYPT_BUFFER_OVERFLOW; + } + + real_len = in[0]; + + if (real_len < 128) { + decoded_len = real_len; + offset = 1; + } else { + real_len &= 0x7F; + if (real_len == 0) { + return CRYPT_PK_ASN1_ERROR; + } + if (real_len > sizeof(decoded_len)) { + return CRYPT_OVERFLOW; + } + if (real_len > (*inlen - 1)) { + return CRYPT_BUFFER_OVERFLOW; + } + decoded_len = 0; + offset = 1 + real_len; + for (i = 0; i < real_len; i++) { + decoded_len = (decoded_len << 8) | in[1 + i]; + } + } + + if (outlen != NULL) *outlen = decoded_len; + if (decoded_len > (*inlen - offset)) return CRYPT_OVERFLOW; + *inlen = offset; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_encode_asn1_identifier.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_encode_asn1_identifier.c new file mode 100644 index 0000000..51d57ec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_encode_asn1_identifier.c @@ -0,0 +1,86 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_asn1_identifier.c + ASN.1 DER, encode the ASN.1 Identifier, Steffen Jaeckel +*/ + +#ifdef LTC_DER +/** + Encode the ASN.1 Identifier + @param id The ASN.1 Identifer to encode + @param out Where to write the identifier to + @param outlen [in/out] The size of out available/written + @return CRYPT_OK if successful +*/ +int der_encode_asn1_identifier(const ltc_asn1_list *id, unsigned char *out, unsigned long *outlen) +{ + ulong64 tmp; + unsigned long tag_len; + + LTC_ARGCHK(id != NULL); + LTC_ARGCHK(outlen != NULL); + + if (id->type != LTC_ASN1_CUSTOM_TYPE) { + if ((unsigned)id->type >= der_asn1_type_to_identifier_map_sz) { + return CRYPT_INVALID_ARG; + } + if (der_asn1_type_to_identifier_map[id->type] == -1) { + return CRYPT_INVALID_ARG; + } + if (out != NULL) { + *out = der_asn1_type_to_identifier_map[id->type]; + } + *outlen = 1; + return CRYPT_OK; + } + if (id->klass < LTC_ASN1_CL_UNIVERSAL || id->klass > LTC_ASN1_CL_PRIVATE) { + return CRYPT_INVALID_ARG; + } + if (id->pc < LTC_ASN1_PC_PRIMITIVE || id->pc > LTC_ASN1_PC_CONSTRUCTED) { + return CRYPT_INVALID_ARG; + } + if (id->tag > (ULONG_MAX >> (8 + 7))) { + return CRYPT_INVALID_ARG; + } + + if (out != NULL) { + if (*outlen < 1) { + return CRYPT_BUFFER_OVERFLOW; + } + + out[0] = id->klass << 6 | id->pc << 5; + } + + if (id->tag < 0x1f) { + if (out != NULL) { + out[0] |= id->tag & 0x1f; + } + *outlen = 1; + } else { + tag_len = 0; + tmp = id->tag; + do { + tag_len++; + tmp >>= 7; + } while (tmp); + + if (out != NULL) { + if (*outlen < tag_len + 1) { + return CRYPT_BUFFER_OVERFLOW; + } + out[0] |= 0x1f; + for (tmp = 1; tmp <= tag_len; ++tmp) { + out[tmp] = ((id->tag >> (7 * (tag_len - tmp))) & 0x7f) | 0x80; + } + out[tag_len] &= ~0x80; + } + *outlen = tag_len + 1; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_encode_asn1_length.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_encode_asn1_length.c new file mode 100644 index 0000000..7892eea --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_encode_asn1_length.c @@ -0,0 +1,117 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_asn1_length.c + ASN.1 DER, encode the ASN.1 length field, Steffen Jaeckel +*/ + +#ifdef LTC_DER +/** + Encode the ASN.1 length field + @param len The length to encode + @param out Where to write the length field to + @param outlen [in/out] The size of out available/written + @return CRYPT_OK if successful +*/ +int der_encode_asn1_length(unsigned long len, unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y; + + LTC_ARGCHK(outlen != NULL); + + x = len; + y = 0; + + while(x != 0) { + y++; + x >>= 8; + } + if (y == 0) { + return CRYPT_PK_ASN1_ERROR; + } + + if (out == NULL) { + if (len < 128) { + x = y; + } else { + x = y + 1; + } + } else { + if (*outlen < y) { + return CRYPT_BUFFER_OVERFLOW; + } + x = 0; + if (len < 128) { + out[x++] = (unsigned char)len; + } else if (len <= 0xffUL) { + out[x++] = 0x81; + out[x++] = (unsigned char)len; + } else if (len <= 0xffffUL) { + out[x++] = 0x82; + out[x++] = (unsigned char)((len>>8UL)&255); + out[x++] = (unsigned char)(len&255); + } else if (len <= 0xffffffUL) { + out[x++] = 0x83; + out[x++] = (unsigned char)((len>>16UL)&255); + out[x++] = (unsigned char)((len>>8UL)&255); + out[x++] = (unsigned char)(len&255); + #if ULONG_MAX != ULLONG_MAX + } else { + out[x++] = 0x84; + out[x++] = (unsigned char)((len>>24UL)&255); + out[x++] = (unsigned char)((len>>16UL)&255); + out[x++] = (unsigned char)((len>>8UL)&255); + out[x++] = (unsigned char)(len&255); + } + #else + } else if (len <= 0xffffffffUL) { + out[x++] = 0x84; + out[x++] = (unsigned char)((len>>24UL)&255); + out[x++] = (unsigned char)((len>>16UL)&255); + out[x++] = (unsigned char)((len>>8UL)&255); + out[x++] = (unsigned char)(len&255); + } else if (len <= 0xffffffffffULL) { + out[x++] = 0x85; + out[x++] = (unsigned char)((len>>32ULL)&255); + out[x++] = (unsigned char)((len>>24ULL)&255); + out[x++] = (unsigned char)((len>>16ULL)&255); + out[x++] = (unsigned char)((len>>8ULL)&255); + out[x++] = (unsigned char)(len&255); + } else if (len <= 0xffffffffffffULL) { + out[x++] = 0x86; + out[x++] = (unsigned char)((len>>40ULL)&255); + out[x++] = (unsigned char)((len>>32ULL)&255); + out[x++] = (unsigned char)((len>>24ULL)&255); + out[x++] = (unsigned char)((len>>16ULL)&255); + out[x++] = (unsigned char)((len>>8ULL)&255); + out[x++] = (unsigned char)(len&255); + } else if (len <= 0xffffffffffffffULL) { + out[x++] = 0x87; + out[x++] = (unsigned char)((len>>48ULL)&255); + out[x++] = (unsigned char)((len>>40ULL)&255); + out[x++] = (unsigned char)((len>>32ULL)&255); + out[x++] = (unsigned char)((len>>24ULL)&255); + out[x++] = (unsigned char)((len>>16ULL)&255); + out[x++] = (unsigned char)((len>>8ULL)&255); + out[x++] = (unsigned char)(len&255); + } else { + out[x++] = 0x88; + out[x++] = (unsigned char)((len>>56ULL)&255); + out[x++] = (unsigned char)((len>>48ULL)&255); + out[x++] = (unsigned char)((len>>40ULL)&255); + out[x++] = (unsigned char)((len>>32ULL)&255); + out[x++] = (unsigned char)((len>>24ULL)&255); + out[x++] = (unsigned char)((len>>16ULL)&255); + out[x++] = (unsigned char)((len>>8ULL)&255); + out[x++] = (unsigned char)(len&255); + } + #endif + } + *outlen = x; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_length_asn1_identifier.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_length_asn1_identifier.c new file mode 100644 index 0000000..63e6bdd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_length_asn1_identifier.c @@ -0,0 +1,23 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_asn1_identifier.c + ASN.1 DER, determine the length when encoding the ASN.1 Identifier, Steffen Jaeckel +*/ + +#ifdef LTC_DER +/** + Determine the length required when encoding the ASN.1 Identifier + @param id The ASN.1 identifier to encode + @param idlen [out] The required length to encode list + @return CRYPT_OK if successful +*/ + +int der_length_asn1_identifier(const ltc_asn1_list *id, unsigned long *idlen) +{ + return der_encode_asn1_identifier(id, NULL, idlen); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_length_asn1_length.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_length_asn1_length.c new file mode 100644 index 0000000..1c81adb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/der_length_asn1_length.c @@ -0,0 +1,22 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_asn1_length.c + ASN.1 DER, determine the length of the ASN.1 length field, Steffen Jaeckel +*/ + +#ifdef LTC_DER +/** + Determine the length required to encode len in the ASN.1 length field + @param len The length to encode + @param outlen [out] The length that's required to store len + @return CRYPT_OK if successful +*/ +int der_length_asn1_length(unsigned long len, unsigned long *outlen) +{ + return der_encode_asn1_length(len, NULL, outlen); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/sub.mk new file mode 100644 index 0000000..63de0ef --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/general/sub.mk @@ -0,0 +1,7 @@ +srcs-y += der_asn1_maps.c +srcs-y += der_decode_asn1_length.c +srcs-y += der_decode_asn1_identifier.c +srcs-y += der_encode_asn1_identifier.c +srcs-y += der_encode_asn1_length.c +srcs-y += der_length_asn1_identifier.c +srcs-y += der_length_asn1_length.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_decode_generalizedtime.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_decode_generalizedtime.c new file mode 100644 index 0000000..7d2f296 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_decode_generalizedtime.c @@ -0,0 +1,135 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_generalizedtime.c + ASN.1 DER, decode a GeneralizedTime, Steffen Jaeckel + Based on der_decode_utctime.c +*/ + +#ifdef LTC_DER + +static int s_char_to_int(unsigned char x) +{ + switch (x) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + default: return 100; + } +} + +#define DECODE_V(y, max) do {\ + y = s_char_to_int(buf[x])*10 + s_char_to_int(buf[x+1]); \ + if (y >= max) return CRYPT_INVALID_PACKET; \ + x += 2; \ +} while(0) + +#define DECODE_V4(y, max) do {\ + y = s_char_to_int(buf[x])*1000 + s_char_to_int(buf[x+1])*100 + s_char_to_int(buf[x+2])*10 + s_char_to_int(buf[x+3]); \ + if (y >= max) return CRYPT_INVALID_PACKET; \ + x += 4; \ +} while(0) + +/** + Decodes a Generalized time structure in DER format (reads all 6 valid encoding formats) + @param in Input buffer + @param inlen Length of input buffer in octets + @param out [out] Destination of Generalized time structure + @return CRYPT_OK if successful +*/ +int der_decode_generalizedtime(const unsigned char *in, unsigned long *inlen, + ltc_generalizedtime *out) +{ + unsigned char buf[32]; + unsigned long x; + int y; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != NULL); + LTC_ARGCHK(out != NULL); + + /* check header */ + if (*inlen < 2UL || (in[1] >= sizeof(buf)) || ((in[1] + 2UL) > *inlen)) { + return CRYPT_INVALID_PACKET; + } + + /* decode the string */ + for (x = 0; x < in[1]; x++) { + y = der_ia5_value_decode(in[x+2]); + if (y == -1) { + return CRYPT_INVALID_PACKET; + } + if (!((y >= '0' && y <= '9') + || y == 'Z' || y == '.' + || y == '+' || y == '-')) { + return CRYPT_INVALID_PACKET; + } + buf[x] = y; + } + *inlen = 2 + x; + + if (x < 15) { + return CRYPT_INVALID_PACKET; + } + + /* possible encodings are +YYYYMMDDhhmmssZ +YYYYMMDDhhmmss+hh'mm' +YYYYMMDDhhmmss-hh'mm' +YYYYMMDDhhmmss.fsZ +YYYYMMDDhhmmss.fs+hh'mm' +YYYYMMDDhhmmss.fs-hh'mm' + + So let's do a trivial decode upto [including] ss + */ + + x = 0; + DECODE_V4(out->YYYY, 10000); + DECODE_V(out->MM, 13); + DECODE_V(out->DD, 32); + DECODE_V(out->hh, 24); + DECODE_V(out->mm, 60); + DECODE_V(out->ss, 60); + + /* clear fractional seconds info */ + out->fs = 0; + + /* now is it Z or . */ + if (buf[x] == 'Z') { + return CRYPT_OK; + } + if (buf[x] == '.') { + x++; + while (buf[x] >= '0' && buf[x] <= '9') { + unsigned fs = out->fs; + if (x >= sizeof(buf)) return CRYPT_INVALID_PACKET; + out->fs *= 10; + out->fs += s_char_to_int(buf[x]); + if (fs > out->fs) return CRYPT_OVERFLOW; + x++; + } + } + + /* now is it Z, +, - */ + if (buf[x] == 'Z') { + return CRYPT_OK; + } + if (buf[x] == '+' || buf[x] == '-') { + out->off_dir = (buf[x++] == '+') ? 0 : 1; + DECODE_V(out->off_hh, 24); + DECODE_V(out->off_mm, 60); + return CRYPT_OK; + } + return CRYPT_INVALID_PACKET; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_encode_generalizedtime.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_encode_generalizedtime.c new file mode 100644 index 0000000..2378eeb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_encode_generalizedtime.c @@ -0,0 +1,98 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_utctime.c + ASN.1 DER, encode a GeneralizedTime, Steffen Jaeckel + Based on der_encode_utctime.c +*/ + +#ifdef LTC_DER + +static const char * const baseten = "0123456789"; + +#define STORE_V(y) do {\ + out[x++] = der_ia5_char_encode(baseten[(y/10) % 10]); \ + out[x++] = der_ia5_char_encode(baseten[y % 10]); \ +} while(0) + +#define STORE_V4(y) do {\ + out[x++] = der_ia5_char_encode(baseten[(y/1000) % 10]); \ + out[x++] = der_ia5_char_encode(baseten[(y/100) % 10]); \ + out[x++] = der_ia5_char_encode(baseten[(y/10) % 10]); \ + out[x++] = der_ia5_char_encode(baseten[y % 10]); \ +} while(0) + +/** + Encodes a Generalized time structure in DER format + @param gtime The GeneralizedTime structure to encode + @param out The destination of the DER encoding of the GeneralizedTime structure + @param outlen [in/out] The length of the DER encoding + @return CRYPT_OK if successful +*/ +int der_encode_generalizedtime(const ltc_generalizedtime *gtime, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, tmplen; + int err; + + LTC_ARGCHK(gtime != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((err = der_length_generalizedtime(gtime, &tmplen)) != CRYPT_OK) { + return err; + } + if (tmplen > *outlen) { + *outlen = tmplen; + return CRYPT_BUFFER_OVERFLOW; + } + + /* store header */ + out[0] = 0x18; + + /* store values */ + x = 2; + STORE_V4(gtime->YYYY); + STORE_V(gtime->MM); + STORE_V(gtime->DD); + STORE_V(gtime->hh); + STORE_V(gtime->mm); + STORE_V(gtime->ss); + + if (gtime->fs) { + unsigned long divisor; + unsigned fs = gtime->fs; + unsigned len = 0; + out[x++] = der_ia5_char_encode('.'); + divisor = 1; + do { + fs /= 10; + divisor *= 10; + len++; + } while(fs != 0); + while (len-- > 1) { + divisor /= 10; + out[x++] = der_ia5_char_encode(baseten[(gtime->fs/divisor) % 10]); + } + out[x++] = der_ia5_char_encode(baseten[gtime->fs % 10]); + } + + if (gtime->off_mm || gtime->off_hh) { + out[x++] = der_ia5_char_encode(gtime->off_dir ? '-' : '+'); + STORE_V(gtime->off_hh); + STORE_V(gtime->off_mm); + } else { + out[x++] = der_ia5_char_encode('Z'); + } + + /* store length */ + out[1] = (unsigned char)(x - 2); + + /* all good let's return */ + *outlen = x; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_length_generalizedtime.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_length_generalizedtime.c new file mode 100644 index 0000000..dc237d3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/der_length_generalizedtime.c @@ -0,0 +1,48 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_utctime.c + ASN.1 DER, get length of GeneralizedTime, Steffen Jaeckel + Based on der_length_utctime.c +*/ + +#ifdef LTC_DER + +/** + Gets length of DER encoding of GeneralizedTime + @param gtime The GeneralizedTime structure to get the size of + @param outlen [out] The length of the DER encoding + @return CRYPT_OK if successful +*/ +int der_length_generalizedtime(const ltc_generalizedtime *gtime, unsigned long *outlen) +{ + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(gtime != NULL); + + if (gtime->fs == 0) { + /* we encode as YYYYMMDDhhmmssZ */ + *outlen = 2 + 14 + 1; + } else { + unsigned long len = 2 + 14 + 1; + unsigned fs = gtime->fs; + do { + fs /= 10; + len++; + } while(fs != 0); + if (gtime->off_hh == 0 && gtime->off_mm == 0) { + /* we encode as YYYYMMDDhhmmss.fsZ */ + len += 1; + } + else { + /* we encode as YYYYMMDDhhmmss.fs{+|-}hh'mm' */ + len += 5; + } + *outlen = len; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/sub.mk new file mode 100644 index 0000000..312e613 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/generalizedtime/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_encode_generalizedtime.c +srcs-y += der_decode_generalizedtime.c +srcs-y += der_length_generalizedtime.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_decode_ia5_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_decode_ia5_string.c new file mode 100644 index 0000000..8fd80cf --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_decode_ia5_string.c @@ -0,0 +1,73 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_ia5_string.c + ASN.1 DER, encode a IA5 STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store a IA5 STRING + @param in The DER encoded IA5 STRING + @param inlen The size of the DER IA5 STRING + @param out [out] The array of octets stored (one per char) + @param outlen [in/out] The number of octets stored + @return CRYPT_OK if successful +*/ +int der_decode_ia5_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int t, err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* must have header at least */ + if (inlen < 2) { + return CRYPT_INVALID_PACKET; + } + + /* check for 0x16 */ + if ((in[0] & 0x1F) != 0x16) { + return CRYPT_INVALID_PACKET; + } + x = 1; + + /* get the length of the data */ + y = inlen - x; + if ((err = der_decode_asn1_length(in + x, &y, &len)) != CRYPT_OK) { + return err; + } + x += y; + + /* is it too long? */ + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + if (len > (inlen - x)) { + return CRYPT_INVALID_PACKET; + } + + /* read the data */ + for (y = 0; y < len; y++) { + t = der_ia5_value_decode(in[x++]); + if (t == -1) { + return CRYPT_INVALID_ARG; + } + out[y] = t; + } + + *outlen = y; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_encode_ia5_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_encode_ia5_string.c new file mode 100644 index 0000000..8b8bd36 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_encode_ia5_string.c @@ -0,0 +1,61 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_ia5_string.c + ASN.1 DER, encode a IA5 STRING, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Store an IA5 STRING + @param in The array of IA5 to store (one per char) + @param inlen The number of IA5 to store + @param out [out] The destination for the DER encoded IA5 STRING + @param outlen [in/out] The max size and resulting size of the DER IA5 STRING + @return CRYPT_OK if successful +*/ +int der_encode_ia5_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get the size */ + if ((err = der_length_ia5_string(in, inlen, &len)) != CRYPT_OK) { + return err; + } + + /* too big? */ + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + /* encode the header+len */ + x = 0; + out[x++] = 0x16; + len = *outlen - x; + if ((err = der_encode_asn1_length(inlen, out + x, &len)) != CRYPT_OK) { + return err; + } + x += len; + + /* store octets */ + for (y = 0; y < inlen; y++) { + out[x++] = der_ia5_char_encode(in[y]); + } + + /* retun length */ + *outlen = x; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_length_ia5_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_length_ia5_string.c new file mode 100644 index 0000000..e397b1c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/der_length_ia5_string.c @@ -0,0 +1,172 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_ia5_string.c + ASN.1 DER, get length of IA5 STRING, Tom St Denis +*/ + +#ifdef LTC_DER + +static const struct { + int code, value; +} ia5_table[] = { +{ '\0', 0 }, +{ '\a', 7 }, +{ '\b', 8 }, +{ '\t', 9 }, +{ '\n', 10 }, +{ '\f', 12 }, +{ '\r', 13 }, +{ ' ', 32 }, +{ '!', 33 }, +{ '"', 34 }, +{ '#', 35 }, +{ '$', 36 }, +{ '%', 37 }, +{ '&', 38 }, +{ '\'', 39 }, +{ '(', 40 }, +{ ')', 41 }, +{ '*', 42 }, +{ '+', 43 }, +{ ',', 44 }, +{ '-', 45 }, +{ '.', 46 }, +{ '/', 47 }, +{ '0', 48 }, +{ '1', 49 }, +{ '2', 50 }, +{ '3', 51 }, +{ '4', 52 }, +{ '5', 53 }, +{ '6', 54 }, +{ '7', 55 }, +{ '8', 56 }, +{ '9', 57 }, +{ ':', 58 }, +{ ';', 59 }, +{ '<', 60 }, +{ '=', 61 }, +{ '>', 62 }, +{ '?', 63 }, +{ '@', 64 }, +{ 'A', 65 }, +{ 'B', 66 }, +{ 'C', 67 }, +{ 'D', 68 }, +{ 'E', 69 }, +{ 'F', 70 }, +{ 'G', 71 }, +{ 'H', 72 }, +{ 'I', 73 }, +{ 'J', 74 }, +{ 'K', 75 }, +{ 'L', 76 }, +{ 'M', 77 }, +{ 'N', 78 }, +{ 'O', 79 }, +{ 'P', 80 }, +{ 'Q', 81 }, +{ 'R', 82 }, +{ 'S', 83 }, +{ 'T', 84 }, +{ 'U', 85 }, +{ 'V', 86 }, +{ 'W', 87 }, +{ 'X', 88 }, +{ 'Y', 89 }, +{ 'Z', 90 }, +{ '[', 91 }, +{ '\\', 92 }, +{ ']', 93 }, +{ '^', 94 }, +{ '_', 95 }, +{ '`', 96 }, +{ 'a', 97 }, +{ 'b', 98 }, +{ 'c', 99 }, +{ 'd', 100 }, +{ 'e', 101 }, +{ 'f', 102 }, +{ 'g', 103 }, +{ 'h', 104 }, +{ 'i', 105 }, +{ 'j', 106 }, +{ 'k', 107 }, +{ 'l', 108 }, +{ 'm', 109 }, +{ 'n', 110 }, +{ 'o', 111 }, +{ 'p', 112 }, +{ 'q', 113 }, +{ 'r', 114 }, +{ 's', 115 }, +{ 't', 116 }, +{ 'u', 117 }, +{ 'v', 118 }, +{ 'w', 119 }, +{ 'x', 120 }, +{ 'y', 121 }, +{ 'z', 122 }, +{ '{', 123 }, +{ '|', 124 }, +{ '}', 125 }, +{ '~', 126 } +}; + +int der_ia5_char_encode(int c) +{ + int x; + for (x = 0; x < (int)(sizeof(ia5_table)/sizeof(ia5_table[0])); x++) { + if (ia5_table[x].code == c) { + return ia5_table[x].value; + } + } + return -1; +} + +int der_ia5_value_decode(int v) +{ + int x; + for (x = 0; x < (int)(sizeof(ia5_table)/sizeof(ia5_table[0])); x++) { + if (ia5_table[x].value == v) { + return ia5_table[x].code; + } + } + return -1; +} + +/** + Gets length of DER encoding of IA5 STRING + @param octets The values you want to encode + @param noctets The number of octets in the string to encode + @param outlen [out] The length of the DER encoding for the given string + @return CRYPT_OK if successful +*/ +int der_length_ia5_string(const unsigned char *octets, unsigned long noctets, unsigned long *outlen) +{ + unsigned long x; + int err; + + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(octets != NULL); + + /* scan string for validity */ + for (x = 0; x < noctets; x++) { + if (der_ia5_char_encode(octets[x]) == -1) { + return CRYPT_INVALID_ARG; + } + } + + if ((err = der_length_asn1_length(noctets, &x)) != CRYPT_OK) { + return err; + } + *outlen = 1 + x + noctets; + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/sub.mk new file mode 100644 index 0000000..dc32a5d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/ia5/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_ia5_string.c +srcs-y += der_encode_ia5_string.c +srcs-y += der_length_ia5_string.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_decode_integer.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_decode_integer.c new file mode 100644 index 0000000..2c9bf7b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_decode_integer.c @@ -0,0 +1,68 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_integer.c + ASN.1 DER, decode an integer, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Read a mp_int integer + @param in The DER encoded data + @param inlen Size of DER encoded data + @param num The first mp_int to decode + @return CRYPT_OK if successful +*/ +int der_decode_integer(const unsigned char *in, unsigned long inlen, void *num) +{ + unsigned long x, y; + int err; + + LTC_ARGCHK(num != NULL); + LTC_ARGCHK(in != NULL); + + /* min DER INTEGER is 0x02 01 00 == 0 */ + if (inlen < (1 + 1 + 1)) { + return CRYPT_INVALID_PACKET; + } + + /* ok expect 0x02 when we AND with 0001 1111 [1F] */ + x = 0; + if ((in[x++] & 0x1F) != 0x02) { + return CRYPT_INVALID_PACKET; + } + + /* get the length of the data */ + inlen -= x; + if ((err = der_decode_asn1_length(in + x, &inlen, &y)) != CRYPT_OK) { + return err; + } + x += inlen; + + if ((err = mp_read_unsigned_bin(num, (unsigned char *)in + x, y)) != CRYPT_OK) { + return err; + } + + /* see if it's negative */ + if (in[x] & 0x80) { + void *tmp; + if (mp_init(&tmp) != CRYPT_OK) { + return CRYPT_MEM; + } + + if (mp_2expt(tmp, mp_count_bits(num)) != CRYPT_OK || mp_sub(num, tmp, num) != CRYPT_OK) { + mp_clear(tmp); + return CRYPT_MEM; + } + mp_clear(tmp); + } + + return CRYPT_OK; + +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_encode_integer.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_encode_integer.c new file mode 100644 index 0000000..7261149 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_encode_integer.c @@ -0,0 +1,105 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_integer.c + ASN.1 DER, encode an integer, Tom St Denis +*/ + + +#ifdef LTC_DER + +/* Exports a positive bignum as DER format (upto 2^32 bytes in size) */ +/** + Store a mp_int integer + @param num The first mp_int to encode + @param out [out] The destination for the DER encoded integers + @param outlen [in/out] The max size and resulting size of the DER encoded integers + @return CRYPT_OK if successful +*/ +int der_encode_integer(void *num, unsigned char *out, unsigned long *outlen) +{ + unsigned long tmplen, y, len; + int err, leading_zero; + + LTC_ARGCHK(num != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* find out how big this will be */ + if ((err = der_length_integer(num, &tmplen)) != CRYPT_OK) { + return err; + } + + if (*outlen < tmplen) { + *outlen = tmplen; + return CRYPT_BUFFER_OVERFLOW; + } + + if (mp_cmp_d(num, 0) != LTC_MP_LT) { + /* we only need a leading zero if the msb of the first byte is one */ + if ((mp_count_bits(num) & 7) == 0 || mp_iszero(num) == LTC_MP_YES) { + leading_zero = 1; + } else { + leading_zero = 0; + } + + /* get length of num in bytes (plus 1 since we force the msbyte to zero) */ + y = mp_unsigned_bin_size(num) + leading_zero; + } else { + leading_zero = 0; + y = mp_count_bits(num); + y = y + (8 - (y & 7)); + y = y >> 3; + if (((mp_cnt_lsb(num)+1)==mp_count_bits(num)) && ((mp_count_bits(num)&7)==0)) --y; + } + + /* now store initial data */ + *out++ = 0x02; + len = *outlen - 1; + if ((err = der_encode_asn1_length(y, out, &len)) != CRYPT_OK) { + return err; + } + out += len; + + /* now store msbyte of zero if num is non-zero */ + if (leading_zero) { + *out++ = 0x00; + } + + /* if it's not zero store it as big endian */ + if (mp_cmp_d(num, 0) == LTC_MP_GT) { + /* now store the mpint */ + if ((err = mp_to_unsigned_bin(num, out)) != CRYPT_OK) { + return err; + } + } else if (mp_iszero(num) != LTC_MP_YES) { + void *tmp; + + /* negative */ + if (mp_init(&tmp) != CRYPT_OK) { + return CRYPT_MEM; + } + + /* 2^roundup and subtract */ + y = mp_count_bits(num); + y = y + (8 - (y & 7)); + if (((mp_cnt_lsb(num)+1)==mp_count_bits(num)) && ((mp_count_bits(num)&7)==0)) y -= 8; + if (mp_2expt(tmp, y) != CRYPT_OK || mp_add(tmp, num, tmp) != CRYPT_OK) { + mp_clear(tmp); + return CRYPT_MEM; + } + if ((err = mp_to_unsigned_bin(tmp, out)) != CRYPT_OK) { + mp_clear(tmp); + return err; + } + mp_clear(tmp); + } + + /* we good */ + *outlen = tmplen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_length_integer.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_length_integer.c new file mode 100644 index 0000000..90e2de3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/der_length_integer.c @@ -0,0 +1,55 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_integer.c + ASN.1 DER, get length of encoding, Tom St Denis +*/ + + +#ifdef LTC_DER +/** + Gets length of DER encoding of num + @param num The int to get the size of + @param outlen [out] The length of the DER encoding for the given integer + @return CRYPT_OK if successful +*/ +int der_length_integer(void *num, unsigned long *outlen) +{ + unsigned long z, len; + int leading_zero, err; + + LTC_ARGCHK(num != NULL); + LTC_ARGCHK(outlen != NULL); + + if (mp_cmp_d(num, 0) != LTC_MP_LT) { + /* positive */ + + /* we only need a leading zero if the msb of the first byte is one */ + if ((mp_count_bits(num) & 7) == 0 || mp_iszero(num) == LTC_MP_YES) { + leading_zero = 1; + } else { + leading_zero = 0; + } + + /* size for bignum */ + len = leading_zero + mp_unsigned_bin_size(num); + } else { + /* it's negative */ + /* find power of 2 that is a multiple of eight and greater than count bits */ + z = mp_count_bits(num); + z = z + (8 - (z & 7)); + if (((mp_cnt_lsb(num)+1)==mp_count_bits(num)) && ((mp_count_bits(num)&7)==0)) --z; + len = z >> 3; + } + + if ((err = der_length_asn1_length(len, &z)) != CRYPT_OK) { + return err; + } + *outlen = 1 + z + len; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/sub.mk new file mode 100644 index 0000000..d319f91 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/integer/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_integer.c +srcs-y += der_encode_integer.c +srcs-y += der_length_integer.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_decode_object_identifier.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_decode_object_identifier.c new file mode 100644 index 0000000..7251247 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_decode_object_identifier.c @@ -0,0 +1,94 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_object_identifier.c + ASN.1 DER, Decode Object Identifier, Tom St Denis +*/ + +#ifdef LTC_DER +/** + Decode OID data and store the array of integers in words + @param in The OID DER encoded data + @param inlen The length of the OID data + @param words [out] The destination of the OID words + @param outlen [in/out] The number of OID words + @return CRYPT_OK if successful +*/ +int der_decode_object_identifier(const unsigned char *in, unsigned long inlen, + unsigned long *words, unsigned long *outlen) +{ + unsigned long x, y, t, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(words != NULL); + LTC_ARGCHK(outlen != NULL); + + /* header is at least 3 bytes */ + if (inlen < 3) { + return CRYPT_INVALID_PACKET; + } + + /* must be room for at least two words */ + if (*outlen < 2) { + *outlen = 2; + return CRYPT_BUFFER_OVERFLOW; + } + + /* decode the packet header */ + x = 0; + if ((in[x++] & 0x1F) != 0x06) { + return CRYPT_INVALID_PACKET; + } + + /* get the length of the data */ + y = inlen - x; + if ((err = der_decode_asn1_length(in + x, &y, &len)) != CRYPT_OK) { + return err; + } + x += y; + + if ((len == 0) || (len > (inlen - x))) { + return CRYPT_INVALID_PACKET; + } + + /* decode words */ + y = 0; + t = 0; + while (len--) { + t = (t << 7) | (in[x] & 0x7F); + if (!(in[x++] & 0x80)) { + /* store t */ + if (y >= *outlen) { + y++; + } else { + if (y == 0) { + if (t <= 79) { + words[0] = t / 40; + words[1] = t % 40; + } else { + words[0] = 2; + words[1] = t - 80; + } + y = 2; + } else { + words[y++] = t; + } + } + t = 0; + } + } + + if (y > *outlen) { + err = CRYPT_BUFFER_OVERFLOW; + } else { + err = CRYPT_OK; + } + + *outlen = y; + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_encode_object_identifier.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_encode_object_identifier.c new file mode 100644 index 0000000..9a9c62d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_encode_object_identifier.c @@ -0,0 +1,92 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_object_identifier.c + ASN.1 DER, Encode Object Identifier, Tom St Denis +*/ + +#ifdef LTC_DER +/** + Encode an OID + @param words The words to encode (upto 32-bits each) + @param nwords The number of words in the OID + @param out [out] Destination of OID data + @param outlen [in/out] The max and resulting size of the OID + @return CRYPT_OK if successful +*/ +int der_encode_object_identifier(const unsigned long *words, unsigned long nwords, + unsigned char *out, unsigned long *outlen) +{ + unsigned long i, x, y, z, t, mask, wordbuf; + int err; + + LTC_ARGCHK(words != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* check length */ + if ((err = der_length_object_identifier(words, nwords, &x)) != CRYPT_OK) { + return err; + } + if (x > *outlen) { + *outlen = x; + return CRYPT_BUFFER_OVERFLOW; + } + + /* compute length to store OID data */ + z = 0; + wordbuf = words[0] * 40 + words[1]; + for (y = 1; y < nwords; y++) { + t = der_object_identifier_bits(wordbuf); + z += t/7 + ((t%7) ? 1 : 0) + (wordbuf == 0 ? 1 : 0); + if (y < nwords - 1) { + wordbuf = words[y + 1]; + } + } + + /* store header + length */ + x = 0; + out[x++] = 0x06; + y = *outlen - x; + if ((err = der_encode_asn1_length(z, out + x, &y)) != CRYPT_OK) { + return err; + } + x += y; + + /* store first byte */ + wordbuf = words[0] * 40 + words[1]; + for (i = 1; i < nwords; i++) { + /* store 7 bit words in little endian */ + t = wordbuf & 0xFFFFFFFF; + if (t) { + y = x; + mask = 0; + while (t) { + out[x++] = (unsigned char)((t & 0x7F) | mask); + t >>= 7; + mask |= 0x80; /* upper bit is set on all but the last byte */ + } + /* now swap bytes y...x-1 */ + z = x - 1; + while (y < z) { + t = out[y]; out[y] = out[z]; out[z] = (unsigned char)t; + ++y; + --z; + } + } else { + /* zero word */ + out[x++] = 0x00; + } + + if (i < nwords - 1) { + wordbuf = words[i + 1]; + } + } + + *outlen = x; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_length_object_identifier.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_length_object_identifier.c new file mode 100644 index 0000000..d9ded02 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/der_length_object_identifier.c @@ -0,0 +1,77 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_object_identifier.c + ASN.1 DER, get length of Object Identifier, Tom St Denis +*/ + +#ifdef LTC_DER + +unsigned long der_object_identifier_bits(unsigned long x) +{ + unsigned long c; + x &= 0xFFFFFFFF; + c = 0; + while (x) { + ++c; + x >>= 1; + } + return c; +} + + +/** + Gets length of DER encoding of Object Identifier + @param nwords The number of OID words + @param words The actual OID words to get the size of + @param outlen [out] The length of the DER encoding for the given string + @return CRYPT_OK if successful +*/ +int der_length_object_identifier(const unsigned long *words, unsigned long nwords, unsigned long *outlen) +{ + unsigned long y, z, t, wordbuf; + + LTC_ARGCHK(words != NULL); + LTC_ARGCHK(outlen != NULL); + + + /* must be >= 2 words */ + if (nwords < 2) { + return CRYPT_INVALID_ARG; + } + + /* word1 = 0,1,2 and word2 0..39 */ + if (words[0] > 2 || (words[0] < 2 && words[1] > 39)) { + return CRYPT_INVALID_ARG; + } + + /* leading word is the first two */ + z = 0; + wordbuf = words[0] * 40 + words[1]; + for (y = 1; y < nwords; y++) { + t = der_object_identifier_bits(wordbuf); + z += t/7 + ((t%7) ? 1 : 0) + (wordbuf == 0 ? 1 : 0); + if (y < nwords - 1) { + /* grab next word */ + wordbuf = words[y+1]; + } + } + + /* now depending on the length our length encoding changes */ + if (z < 128) { + z += 2; + } else if (z < 256) { + z += 3; + } else if (z < 65536UL) { + z += 4; + } else { + return CRYPT_INVALID_ARG; + } + + *outlen = z; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/sub.mk new file mode 100644 index 0000000..b56efb0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/object_identifier/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_object_identifier.c +srcs-y += der_encode_object_identifier.c +srcs-y += der_length_object_identifier.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_decode_octet_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_decode_octet_string.c new file mode 100644 index 0000000..df84721 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_decode_octet_string.c @@ -0,0 +1,69 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_octet_string.c + ASN.1 DER, encode a OCTET STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store a OCTET STRING + @param in The DER encoded OCTET STRING + @param inlen The size of the DER OCTET STRING + @param out [out] The array of octets stored (one per char) + @param outlen [in/out] The number of octets stored + @return CRYPT_OK if successful +*/ +int der_decode_octet_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* must have header at least */ + if (inlen < 2) { + return CRYPT_INVALID_PACKET; + } + + /* check for 0x04 */ + if ((in[0] & 0x1F) != 0x04) { + return CRYPT_INVALID_PACKET; + } + x = 1; + + /* get the length of the data */ + y = inlen - x; + if ((err = der_decode_asn1_length(in + x, &y, &len)) != CRYPT_OK) { + return err; + } + x += y; + + /* is it too long? */ + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + if (len > (inlen - x)) { + return CRYPT_INVALID_PACKET; + } + + /* read the data */ + for (y = 0; y < len; y++) { + out[y] = in[x++]; + } + + *outlen = y; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_encode_octet_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_encode_octet_string.c new file mode 100644 index 0000000..73f1ee7 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_encode_octet_string.c @@ -0,0 +1,62 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_octet_string.c + ASN.1 DER, encode a OCTET STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store an OCTET STRING + @param in The array of OCTETS to store (one per char) + @param inlen The number of OCTETS to store + @param out [out] The destination for the DER encoded OCTET STRING + @param outlen [in/out] The max size and resulting size of the DER OCTET STRING + @return CRYPT_OK if successful +*/ +int der_encode_octet_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get the size */ + if ((err = der_length_octet_string(inlen, &len)) != CRYPT_OK) { + return err; + } + + /* too big? */ + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + /* encode the header+len */ + x = 0; + out[x++] = 0x04; + len = *outlen - x; + if ((err = der_encode_asn1_length(inlen, out + x, &len)) != CRYPT_OK) { + return err; + } + x += len; + + /* store octets */ + for (y = 0; y < inlen; y++) { + out[x++] = in[y]; + } + + /* retun length */ + *outlen = x; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_length_octet_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_length_octet_string.c new file mode 100644 index 0000000..67b071e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/der_length_octet_string.c @@ -0,0 +1,33 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_octet_string.c + ASN.1 DER, get length of OCTET STRING, Tom St Denis +*/ + +#ifdef LTC_DER +/** + Gets length of DER encoding of OCTET STRING + @param noctets The number of octets in the string to encode + @param outlen [out] The length of the DER encoding for the given string + @return CRYPT_OK if successful +*/ +int der_length_octet_string(unsigned long noctets, unsigned long *outlen) +{ + unsigned long x; + int err; + + LTC_ARGCHK(outlen != NULL); + + if ((err = der_length_asn1_length(noctets, &x)) != CRYPT_OK) { + return err; + } + *outlen = 1 + x + noctets; + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/sub.mk new file mode 100644 index 0000000..04607ee --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/octet/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_octet_string.c +srcs-y += der_encode_octet_string.c +srcs-y += der_length_octet_string.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_decode_printable_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_decode_printable_string.c new file mode 100644 index 0000000..3f9b533 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_decode_printable_string.c @@ -0,0 +1,73 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_printable_string.c + ASN.1 DER, encode a printable STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store a printable STRING + @param in The DER encoded printable STRING + @param inlen The size of the DER printable STRING + @param out [out] The array of octets stored (one per char) + @param outlen [in/out] The number of octets stored + @return CRYPT_OK if successful +*/ +int der_decode_printable_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int t, err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* must have header at least */ + if (inlen < 2) { + return CRYPT_INVALID_PACKET; + } + + /* check for 0x13 */ + if ((in[0] & 0x1F) != 0x13) { + return CRYPT_INVALID_PACKET; + } + x = 1; + + /* get the length of the data */ + y = inlen - x; + if ((err = der_decode_asn1_length(in + x, &y, &len)) != CRYPT_OK) { + return err; + } + x += y; + + /* is it too long? */ + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + if (len > (inlen - x)) { + return CRYPT_INVALID_PACKET; + } + + /* read the data */ + for (y = 0; y < len; y++) { + t = der_printable_value_decode(in[x++]); + if (t == -1) { + return CRYPT_INVALID_ARG; + } + out[y] = t; + } + + *outlen = y; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_encode_printable_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_encode_printable_string.c new file mode 100644 index 0000000..7b3aefd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_encode_printable_string.c @@ -0,0 +1,61 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_printable_string.c + ASN.1 DER, encode a printable STRING, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Store an printable STRING + @param in The array of printable to store (one per char) + @param inlen The number of printable to store + @param out [out] The destination for the DER encoded printable STRING + @param outlen [in/out] The max size and resulting size of the DER printable STRING + @return CRYPT_OK if successful +*/ +int der_encode_printable_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get the size */ + if ((err = der_length_printable_string(in, inlen, &len)) != CRYPT_OK) { + return err; + } + + /* too big? */ + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + /* encode the header+len */ + x = 0; + out[x++] = 0x13; + len = *outlen - x; + if ((err = der_encode_asn1_length(inlen, out + x, &len)) != CRYPT_OK) { + return err; + } + x += len; + + /* store octets */ + for (y = 0; y < inlen; y++) { + out[x++] = der_printable_char_encode(in[y]); + } + + /* retun length */ + *outlen = x; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_length_printable_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_length_printable_string.c new file mode 100644 index 0000000..c52e36d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/der_length_printable_string.c @@ -0,0 +1,144 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_printable_string.c + ASN.1 DER, get length of Printable STRING, Tom St Denis +*/ + +#ifdef LTC_DER + +static const struct { + int code, value; +} printable_table[] = { +{ ' ', 32 }, +{ '\'', 39 }, +{ '(', 40 }, +{ ')', 41 }, +{ '+', 43 }, +{ ',', 44 }, +{ '-', 45 }, +{ '.', 46 }, +{ '/', 47 }, +{ '0', 48 }, +{ '1', 49 }, +{ '2', 50 }, +{ '3', 51 }, +{ '4', 52 }, +{ '5', 53 }, +{ '6', 54 }, +{ '7', 55 }, +{ '8', 56 }, +{ '9', 57 }, +{ ':', 58 }, +{ '=', 61 }, +{ '?', 63 }, +{ 'A', 65 }, +{ 'B', 66 }, +{ 'C', 67 }, +{ 'D', 68 }, +{ 'E', 69 }, +{ 'F', 70 }, +{ 'G', 71 }, +{ 'H', 72 }, +{ 'I', 73 }, +{ 'J', 74 }, +{ 'K', 75 }, +{ 'L', 76 }, +{ 'M', 77 }, +{ 'N', 78 }, +{ 'O', 79 }, +{ 'P', 80 }, +{ 'Q', 81 }, +{ 'R', 82 }, +{ 'S', 83 }, +{ 'T', 84 }, +{ 'U', 85 }, +{ 'V', 86 }, +{ 'W', 87 }, +{ 'X', 88 }, +{ 'Y', 89 }, +{ 'Z', 90 }, +{ 'a', 97 }, +{ 'b', 98 }, +{ 'c', 99 }, +{ 'd', 100 }, +{ 'e', 101 }, +{ 'f', 102 }, +{ 'g', 103 }, +{ 'h', 104 }, +{ 'i', 105 }, +{ 'j', 106 }, +{ 'k', 107 }, +{ 'l', 108 }, +{ 'm', 109 }, +{ 'n', 110 }, +{ 'o', 111 }, +{ 'p', 112 }, +{ 'q', 113 }, +{ 'r', 114 }, +{ 's', 115 }, +{ 't', 116 }, +{ 'u', 117 }, +{ 'v', 118 }, +{ 'w', 119 }, +{ 'x', 120 }, +{ 'y', 121 }, +{ 'z', 122 }, +}; + +int der_printable_char_encode(int c) +{ + int x; + for (x = 0; x < (int)(sizeof(printable_table)/sizeof(printable_table[0])); x++) { + if (printable_table[x].code == c) { + return printable_table[x].value; + } + } + return -1; +} + +int der_printable_value_decode(int v) +{ + int x; + for (x = 0; x < (int)(sizeof(printable_table)/sizeof(printable_table[0])); x++) { + if (printable_table[x].value == v) { + return printable_table[x].code; + } + } + return -1; +} + +/** + Gets length of DER encoding of Printable STRING + @param octets The values you want to encode + @param noctets The number of octets in the string to encode + @param outlen [out] The length of the DER encoding for the given string + @return CRYPT_OK if successful +*/ +int der_length_printable_string(const unsigned char *octets, unsigned long noctets, unsigned long *outlen) +{ + unsigned long x; + int err; + + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(octets != NULL); + + /* scan string for validity */ + for (x = 0; x < noctets; x++) { + if (der_printable_char_encode(octets[x]) == -1) { + return CRYPT_INVALID_ARG; + } + } + + if ((err = der_length_asn1_length(noctets, &x)) != CRYPT_OK) { + return err; + } + *outlen = 1 + x + noctets; + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/sub.mk new file mode 100644 index 0000000..b509209 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/printable_string/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_printable_string.c +srcs-y += der_encode_printable_string.c +srcs-y += der_length_printable_string.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_ex.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_ex.c new file mode 100644 index 0000000..6d7f846 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_ex.c @@ -0,0 +1,28 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +/** + @file der_decode_sequence_ex.c + ASN.1 DER, decode a SEQUENCE, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Decode a SEQUENCE + @param in The DER encoded input + @param inlen The size of the input + @param list The list of items to decode + @param outlen The number of items in the list + @param flags c.f. enum ltc_der_seq + @return CRYPT_OK on success +*/ +int der_decode_sequence_ex(const unsigned char *in, unsigned long inlen, + ltc_asn1_list *list, unsigned long outlen, unsigned int flags) +{ + return der_decode_custom_type_ex(in, inlen, NULL, list, outlen, flags); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_flexi.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_flexi.c new file mode 100644 index 0000000..2a0841d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_flexi.c @@ -0,0 +1,541 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_sequence_flexi.c + ASN.1 DER, decode an array of ASN.1 types with a flexi parser, Tom St Denis +*/ + +#ifdef LTC_DER + +static int s_new_element(ltc_asn1_list **l) +{ + /* alloc new link */ + if (*l == NULL) { + *l = XCALLOC(1, sizeof(ltc_asn1_list)); + if (*l == NULL) { + return CRYPT_MEM; + } + } else { + (*l)->next = XCALLOC(1, sizeof(ltc_asn1_list)); + if ((*l)->next == NULL) { + return CRYPT_MEM; + } + (*l)->next->prev = *l; + *l = (*l)->next; + } + return CRYPT_OK; +} + +/** + ASN.1 DER Flexi(ble) decoder will decode arbitrary DER packets and create a linked list of the decoded elements. + @param in The input buffer + @param inlen [in/out] The length of the input buffer and on output the amount of decoded data + @param out [out] A pointer to the linked list + @param depth The depth/level of decoding recursion we've already reached + @return CRYPT_OK on success. +*/ +static int s_der_decode_sequence_flexi(const unsigned char *in, unsigned long *inlen, ltc_asn1_list **out, unsigned long depth) +{ + ltc_asn1_list *l; + unsigned long err, identifier, len, totlen, data_offset, id_len, len_len; + void *realloc_tmp; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != NULL); + LTC_ARGCHK(out != NULL); + + l = NULL; + totlen = 0; + + if (*inlen == 0) { + /* alloc new link */ + if ((err = s_new_element(&l)) != CRYPT_OK) { + goto error; + } + } + + /* scan the input and and get lengths and what not */ + while (*inlen) { + /* alloc new link */ + if ((err = s_new_element(&l)) != CRYPT_OK) { + goto error; + } + + id_len = *inlen; + if ((err = der_decode_asn1_identifier(in, &id_len, l)) != CRYPT_OK) { + goto error; + } + /* read the type byte */ + identifier = *in; + + if (l->type != LTC_ASN1_EOL) { + /* fetch length */ + len_len = *inlen - id_len; +#if defined(LTC_TEST_DBG) + data_offset = 666; + len = 0; +#endif + if ((err = der_decode_asn1_length(&in[id_len], &len_len, &len)) != CRYPT_OK) { +#if defined(LTC_TEST_DBG) + fprintf(stderr, "E1 %02lx: hl=%4lu l=%4lu - %s (%s)\n", identifier, data_offset, len, der_asn1_tag_to_string_map[l->tag], error_to_string(err)); +#endif + goto error; + } else if (len > (*inlen - id_len - len_len)) { + err = CRYPT_INVALID_PACKET; +#if defined(LTC_TEST_DBG) + fprintf(stderr, "E2 %02lx: hl=%4lu l=%4lu - %s (%s)\n", identifier, data_offset, len, der_asn1_tag_to_string_map[l->tag], error_to_string(err)); +#endif + goto error; + } + data_offset = id_len + len_len; +#if defined(LTC_TEST_DBG) && LTC_TEST_DBG > 1 + if (l->type == LTC_ASN1_CUSTOM_TYPE && l->klass == LTC_ASN1_CL_CONTEXT_SPECIFIC) { + fprintf(stderr, "OK %02lx: hl=%4lu l=%4lu - Context Specific[%s %llu]\n", identifier, data_offset, len, der_asn1_pc_to_string_map[l->pc], l->tag); + } else { + fprintf(stderr, "OK %02lx: hl=%4lu l=%4lu - %s\n", identifier, data_offset, len, der_asn1_tag_to_string_map[l->tag]); + } +#endif + len += data_offset; + + if (l->type == LTC_ASN1_CUSTOM_TYPE) { + /* Custom type, use the 'used' field to store the original identifier */ + l->used = identifier; + if (l->pc == LTC_ASN1_PC_CONSTRUCTED) { + /* treat constructed elements like SEQUENCEs */ + identifier = 0x20; + } else { + /* primitive elements are treated as opaque data */ + identifier = 0x80; + } + } + } else { + /* Init this so gcc won't complain, + * as this case will only be hit when we + * can't decode the identifier so the + * switch-case should go to default anyway... + */ + data_offset = 0; + len = 0; + } + + /* now switch on type */ + switch (identifier) { + case 0x01: /* BOOLEAN */ + if (l->type != LTC_ASN1_BOOLEAN) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = 1; + l->data = XCALLOC(1, sizeof(int)); + + if ((err = der_decode_boolean(in, *inlen, l->data)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_boolean(&len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x02: /* INTEGER */ + if (l->type != LTC_ASN1_INTEGER) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = 1; + if ((err = mp_init(&l->data)) != CRYPT_OK) { + goto error; + } + + /* decode field */ + if ((err = der_decode_integer(in, *inlen, l->data)) != CRYPT_OK) { + goto error; + } + + /* calc length of object */ + if ((err = der_length_integer(l->data, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x03: /* BIT */ + if (l->type != LTC_ASN1_BIT_STRING) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = len * 8; /* *8 because we store decoded bits one per char and they are encoded 8 per char. */ + + if ((l->data = XCALLOC(1, l->size)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_bit_string(in, *inlen, l->data, &l->size)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_bit_string(l->size, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x04: /* OCTET */ + if (l->type != LTC_ASN1_OCTET_STRING) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = len; + + if ((l->data = XCALLOC(1, l->size)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_octet_string(in, *inlen, l->data, &l->size)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_octet_string(l->size, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x05: /* NULL */ + if (l->type != LTC_ASN1_NULL) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* valid NULL is 0x05 0x00 */ + if (in[0] != 0x05 || in[1] != 0x00) { + err = CRYPT_INVALID_PACKET; + goto error; + } + + /* simple to store ;-) */ + l->data = NULL; + l->size = 0; + len = 2; + + break; + + case 0x06: /* OID */ + if (l->type != LTC_ASN1_OBJECT_IDENTIFIER) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = len; + + if ((l->data = XCALLOC(len, sizeof(unsigned long))) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_object_identifier(in, *inlen, l->data, &l->size)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_object_identifier(l->data, l->size, &len)) != CRYPT_OK) { + goto error; + } + + /* resize it to save a bunch of mem */ + if ((realloc_tmp = XREALLOC(l->data, l->size * sizeof(unsigned long))) == NULL) { + /* out of heap but this is not an error */ + break; + } + l->data = realloc_tmp; + break; + + case 0x0C: /* UTF8 */ + + /* init field */ + if (l->type != LTC_ASN1_UTF8_STRING) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + l->size = len; + + if ((l->data = XCALLOC(sizeof(wchar_t), l->size)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_utf8_string(in, *inlen, l->data, &l->size)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_utf8_string(l->data, l->size, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x13: /* PRINTABLE */ + if (l->type != LTC_ASN1_PRINTABLE_STRING) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = len; + + if ((l->data = XCALLOC(1, l->size)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_printable_string(in, *inlen, l->data, &l->size)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_printable_string(l->data, l->size, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x14: /* TELETEXT */ + if (l->type != LTC_ASN1_TELETEX_STRING) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = len; + + if ((l->data = XCALLOC(1, l->size)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_teletex_string(in, *inlen, l->data, &l->size)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_teletex_string(l->data, l->size, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x16: /* IA5 */ + if (l->type != LTC_ASN1_IA5_STRING) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = len; + + if ((l->data = XCALLOC(1, l->size)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_ia5_string(in, *inlen, l->data, &l->size)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_ia5_string(l->data, l->size, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x17: /* UTC TIME */ + if (l->type != LTC_ASN1_UTCTIME) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = 1; + + if ((l->data = XCALLOC(1, sizeof(ltc_utctime))) == NULL) { + err = CRYPT_MEM; + goto error; + } + + len = *inlen; + if ((err = der_decode_utctime(in, &len, l->data)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_utctime(l->data, &len)) != CRYPT_OK) { + goto error; + } + break; + + case 0x18: + if (l->type != LTC_ASN1_GENERALIZEDTIME) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* init field */ + l->size = len; + + if ((l->data = XCALLOC(1, sizeof(ltc_generalizedtime))) == NULL) { + err = CRYPT_MEM; + goto error; + } + + if ((err = der_decode_generalizedtime(in, &len, l->data)) != CRYPT_OK) { + goto error; + } + + if ((err = der_length_generalizedtime(l->data, &len)) != CRYPT_OK) { + goto error; + } + + break; + + case 0x20: /* Any CONSTRUCTED element that is neither SEQUENCE nor SET */ + case 0x30: /* SEQUENCE */ + case 0x31: /* SET */ + + /* init field */ + if (identifier == 0x20) { + if (l->type != LTC_ASN1_CUSTOM_TYPE) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + } + else if (identifier == 0x30) { + if (l->type != LTC_ASN1_SEQUENCE) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + } + else { + if (l->type != LTC_ASN1_SET) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + } + + /* check that we don't go over the recursion limit */ + if (depth > LTC_DER_MAX_RECURSION) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + if ((l->data = XMALLOC(len)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + XMEMCPY(l->data, in, len); + l->size = len; + + + /* jump to the start of the data */ + in += data_offset; + *inlen -= data_offset; + len -= data_offset; + + /* save the decoded ASN.1 len */ + len_len = len; + + /* Sequence elements go as child */ + if ((err = s_der_decode_sequence_flexi(in, &len, &(l->child), depth+1)) != CRYPT_OK) { + goto error; + } + if (len_len != len) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + /* len update */ + totlen += data_offset; + + /* the flexi decoder can also do nothing, so make sure a child has been allocated */ + if (l->child) { + /* link them up y0 */ + l->child->parent = l; + } + + break; + + case 0x80: /* Context-specific */ + if (l->type != LTC_ASN1_CUSTOM_TYPE) { + err = CRYPT_PK_ASN1_ERROR; + goto error; + } + + if ((l->data = XCALLOC(1, len - data_offset)) == NULL) { + err = CRYPT_MEM; + goto error; + } + + XMEMCPY(l->data, in + data_offset, len - data_offset); + l->size = len - data_offset; + + break; + + default: + /* invalid byte ... this is a soft error */ + /* remove link */ + if (l->prev) { + l = l->prev; + XFREE(l->next); + l->next = NULL; + } + goto outside; + } + + /* advance pointers */ + totlen += len; + in += len; + *inlen -= len; + } + +outside: + + /* in case we processed anything */ + if (totlen) { + /* rewind l please */ + while (l->prev != NULL || l->parent != NULL) { + if (l->parent != NULL) { + l = l->parent; + } else { + l = l->prev; + } + } + } + + /* return */ + *out = l; + *inlen = totlen; + return CRYPT_OK; + +error: + /* free list */ + der_sequence_free(l); + + return err; +} + +/** + ASN.1 DER Flexi(ble) decoder will decode arbitrary DER packets and create a linked list of the decoded elements. + @param in The input buffer + @param inlen [in/out] The length of the input buffer and on output the amount of decoded data + @param out [out] A pointer to the linked list + @return CRYPT_OK on success. +*/ +int der_decode_sequence_flexi(const unsigned char *in, unsigned long *inlen, ltc_asn1_list **out) +{ + return s_der_decode_sequence_flexi(in, inlen, out, 0); +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_multi.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_multi.c new file mode 100644 index 0000000..429b540 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_decode_sequence_multi.c @@ -0,0 +1,183 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + + +/** + @file der_decode_sequence_multi.c + ASN.1 DER, decode a SEQUENCE, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Decode a SEQUENCE type using a VA list + @param in Input buffer + @param inlen Length of input in octets + @param a1 Initialized argument list #1 + @param a2 Initialized argument list #2 (copy of #1) + @param flags c.f. enum ltc_der_seq + @return CRYPT_OK on success +*/ +static int s_der_decode_sequence_va(const unsigned char *in, unsigned long inlen, va_list a1, va_list a2, unsigned int flags) +{ + int err; + ltc_asn1_type type; + unsigned long size, x; + void *data; + ltc_asn1_list *list; + + LTC_ARGCHK(in != NULL); + + /* get size of output that will be required */ + x = 0; + for (;;) { + type = (ltc_asn1_type)va_arg(a1, int); + + if (type == LTC_ASN1_EOL) { + break; + } + + size = va_arg(a1, unsigned long); + data = va_arg(a1, void*); + LTC_UNUSED_PARAM(size); + LTC_UNUSED_PARAM(data); + + switch (type) { + case LTC_ASN1_BOOLEAN: + case LTC_ASN1_INTEGER: + case LTC_ASN1_SHORT_INTEGER: + case LTC_ASN1_BIT_STRING: + case LTC_ASN1_OCTET_STRING: + case LTC_ASN1_NULL: + case LTC_ASN1_OBJECT_IDENTIFIER: + case LTC_ASN1_IA5_STRING: + case LTC_ASN1_PRINTABLE_STRING: + case LTC_ASN1_UTF8_STRING: + case LTC_ASN1_UTCTIME: + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_SEQUENCE: + case LTC_ASN1_CHOICE: + case LTC_ASN1_RAW_BIT_STRING: + case LTC_ASN1_TELETEX_STRING: + case LTC_ASN1_GENERALIZEDTIME: + ++x; + break; + + case LTC_ASN1_EOL: + case LTC_ASN1_CUSTOM_TYPE: + default: + return CRYPT_INVALID_ARG; + } + } + + /* allocate structure for x elements */ + if (x == 0) { + return CRYPT_NOP; + } + + list = XCALLOC(sizeof(*list), x); + if (list == NULL) { + return CRYPT_MEM; + } + + /* fill in the structure */ + x = 0; + for (;;) { + type = (ltc_asn1_type)va_arg(a2, int); + size = va_arg(a2, unsigned long); + data = va_arg(a2, void*); + + if (type == LTC_ASN1_EOL) { + break; + } + + switch (type) { + case LTC_ASN1_BOOLEAN: + case LTC_ASN1_INTEGER: + case LTC_ASN1_SHORT_INTEGER: + case LTC_ASN1_BIT_STRING: + case LTC_ASN1_OCTET_STRING: + case LTC_ASN1_NULL: + case LTC_ASN1_OBJECT_IDENTIFIER: + case LTC_ASN1_IA5_STRING: + case LTC_ASN1_PRINTABLE_STRING: + case LTC_ASN1_UTF8_STRING: + case LTC_ASN1_UTCTIME: + case LTC_ASN1_SEQUENCE: + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_CHOICE: + case LTC_ASN1_RAW_BIT_STRING: + case LTC_ASN1_TELETEX_STRING: + case LTC_ASN1_GENERALIZEDTIME: + LTC_SET_ASN1(list, x++, type, data, size); + break; + /* coverity[dead_error_line] */ + case LTC_ASN1_EOL: + case LTC_ASN1_CUSTOM_TYPE: + default: + break; + } + } + + err = der_decode_sequence_ex(in, inlen, list, x, flags); + XFREE(list); + return err; +} + +/** + Decode a SEQUENCE type using a VA list + @param in Input buffer + @param inlen Length of input in octets + @remark <...> is of the form (int, unsigned long, void*) + @return CRYPT_OK on success +*/ +int der_decode_sequence_multi(const unsigned char *in, unsigned long inlen, ...) +{ + va_list a1, a2; + int err; + + LTC_ARGCHK(in != NULL); + + va_start(a1, inlen); + va_start(a2, inlen); + + err = s_der_decode_sequence_va(in, inlen, a1, a2, LTC_DER_SEQ_SEQUENCE | LTC_DER_SEQ_RELAXED); + + va_end(a2); + va_end(a1); + + return err; +} + +/** + Decode a SEQUENCE type using a VA list + @param in Input buffer + @param inlen Length of input in octets + @param flags c.f. enum ltc_der_seq + @remark <...> is of the form (int, unsigned long, void*) + @return CRYPT_OK on success +*/ +int der_decode_sequence_multi_ex(const unsigned char *in, unsigned long inlen, unsigned int flags, ...) +{ + va_list a1, a2; + int err; + + LTC_ARGCHK(in != NULL); + + va_start(a1, flags); + va_start(a2, flags); + + err = s_der_decode_sequence_va(in, inlen, a1, a2, flags); + + va_end(a2); + va_end(a1); + + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_encode_sequence_ex.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_encode_sequence_ex.c new file mode 100644 index 0000000..d7f3c88 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_encode_sequence_ex.c @@ -0,0 +1,203 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +/** + @file der_encode_sequence_ex.c + ASN.1 DER, encode a SEQUENCE, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Encode a SEQUENCE + @param list The list of items to encode + @param inlen The number of items in the list + @param out [out] The destination + @param outlen [in/out] The size of the output + @param type_of LTC_ASN1_SEQUENCE or LTC_ASN1_SET/LTC_ASN1_SETOF + @return CRYPT_OK on success +*/ +int der_encode_sequence_ex(const ltc_asn1_list *list, unsigned long inlen, + unsigned char *out, unsigned long *outlen, int type_of) +{ + int err; + ltc_asn1_type type; + unsigned long size, x, y, z, i; + void *data; + + LTC_ARGCHK(list != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get size of output that will be required */ + y = 0; z = 0; + if ((err = der_length_sequence_ex(list, inlen, &y, &z)) != CRYPT_OK) return CRYPT_INVALID_ARG; + + /* too big ? */ + if (*outlen < y) { + *outlen = y; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + /* store header */ + x = 0; + out[x++] = (type_of == LTC_ASN1_SEQUENCE) ? 0x30 : 0x31; + + y = *outlen - x; + if ((err = der_encode_asn1_length(z, &out[x], &y)) != CRYPT_OK) { + goto LBL_ERR; + } + x += y; + + /* store data */ + *outlen -= x; + for (i = 0; i < inlen; i++) { + type = list[i].type; + size = list[i].size; + data = list[i].data; + + if (type == LTC_ASN1_EOL) { + break; + } + + switch (type) { + case LTC_ASN1_BOOLEAN: + z = *outlen; + if ((err = der_encode_boolean(*((int *)data), out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_INTEGER: + z = *outlen; + if ((err = der_encode_integer(data, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SHORT_INTEGER: + z = *outlen; + if ((err = der_encode_short_integer(*((unsigned long*)data), out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_BIT_STRING: + z = *outlen; + if ((err = der_encode_bit_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_RAW_BIT_STRING: + z = *outlen; + if ((err = der_encode_raw_bit_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_OCTET_STRING: + z = *outlen; + if ((err = der_encode_octet_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_NULL: + out[x] = 0x05; + out[x+1] = 0x00; + z = 2; + break; + + case LTC_ASN1_OBJECT_IDENTIFIER: + z = *outlen; + if ((err = der_encode_object_identifier(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_IA5_STRING: + z = *outlen; + if ((err = der_encode_ia5_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_PRINTABLE_STRING: + z = *outlen; + if ((err = der_encode_printable_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_UTF8_STRING: + z = *outlen; + if ((err = der_encode_utf8_string(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_UTCTIME: + z = *outlen; + if ((err = der_encode_utctime(data, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_GENERALIZEDTIME: + z = *outlen; + if ((err = der_encode_generalizedtime(data, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SET: + z = *outlen; + if ((err = der_encode_set(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SETOF: + z = *outlen; + if ((err = der_encode_setof(data, size, out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_SEQUENCE: + z = *outlen; + if ((err = der_encode_sequence_ex(data, size, out + x, &z, type)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_CUSTOM_TYPE: + z = *outlen; + if ((err = der_encode_custom_type(&list[i], out + x, &z)) != CRYPT_OK) { + goto LBL_ERR; + } + break; + + case LTC_ASN1_CHOICE: + case LTC_ASN1_EOL: + case LTC_ASN1_TELETEX_STRING: + default: + err = CRYPT_INVALID_ARG; + goto LBL_ERR; + } + + x += z; + *outlen -= z; + } + *outlen = x; + err = CRYPT_OK; + +LBL_ERR: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_encode_sequence_multi.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_encode_sequence_multi.c new file mode 100644 index 0000000..a0877fc --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_encode_sequence_multi.c @@ -0,0 +1,141 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" +#include + + +/** + @file der_encode_sequence_multi.c + ASN.1 DER, encode a SEQUENCE, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Encode a SEQUENCE type using a VA list + @param out [out] Destination for data + @param outlen [in/out] Length of buffer and resulting length of output + @remark <...> is of the form (int, unsigned long, void*) + @return CRYPT_OK on success +*/ +int der_encode_sequence_multi(unsigned char *out, unsigned long *outlen, ...) +{ + int err; + ltc_asn1_type type; + unsigned long size, x; + void *data; + va_list args; + ltc_asn1_list *list; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get size of output that will be required */ + va_start(args, outlen); + x = 0; + for (;;) { + type = (ltc_asn1_type)va_arg(args, int); + + if (type == LTC_ASN1_EOL) { + break; + } + + size = va_arg(args, unsigned long); + data = va_arg(args, void*); + LTC_UNUSED_PARAM(size); + LTC_UNUSED_PARAM(data); + + switch (type) { + case LTC_ASN1_BOOLEAN: + case LTC_ASN1_INTEGER: + case LTC_ASN1_SHORT_INTEGER: + case LTC_ASN1_BIT_STRING: + case LTC_ASN1_OCTET_STRING: + case LTC_ASN1_NULL: + case LTC_ASN1_OBJECT_IDENTIFIER: + case LTC_ASN1_IA5_STRING: + case LTC_ASN1_PRINTABLE_STRING: + case LTC_ASN1_UTF8_STRING: + case LTC_ASN1_UTCTIME: + case LTC_ASN1_SEQUENCE: + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_RAW_BIT_STRING: + case LTC_ASN1_GENERALIZEDTIME: + ++x; + break; + + case LTC_ASN1_CHOICE: + case LTC_ASN1_CUSTOM_TYPE: + case LTC_ASN1_EOL: + case LTC_ASN1_TELETEX_STRING: + default: + va_end(args); + return CRYPT_INVALID_ARG; + } + } + va_end(args); + + /* allocate structure for x elements */ + if (x == 0) { + return CRYPT_NOP; + } + + list = XCALLOC(sizeof(*list), x); + if (list == NULL) { + return CRYPT_MEM; + } + + /* fill in the structure */ + va_start(args, outlen); + x = 0; + for (;;) { + type = (ltc_asn1_type)va_arg(args, int); + + if (type == LTC_ASN1_EOL) { + break; + } + + size = va_arg(args, unsigned long); + data = va_arg(args, void*); + + switch (type) { + case LTC_ASN1_BOOLEAN: + case LTC_ASN1_INTEGER: + case LTC_ASN1_SHORT_INTEGER: + case LTC_ASN1_BIT_STRING: + case LTC_ASN1_OCTET_STRING: + case LTC_ASN1_NULL: + case LTC_ASN1_OBJECT_IDENTIFIER: + case LTC_ASN1_IA5_STRING: + case LTC_ASN1_PRINTABLE_STRING: + case LTC_ASN1_UTF8_STRING: + case LTC_ASN1_UTCTIME: + case LTC_ASN1_SEQUENCE: + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_RAW_BIT_STRING: + case LTC_ASN1_GENERALIZEDTIME: + LTC_SET_ASN1(list, x++, type, data, size); + break; + + case LTC_ASN1_CHOICE: + case LTC_ASN1_CUSTOM_TYPE: + case LTC_ASN1_EOL: + case LTC_ASN1_TELETEX_STRING: + default: + va_end(args); + err = CRYPT_INVALID_ARG; + goto LBL_ERR; + } + } + va_end(args); + + err = der_encode_sequence(list, x, out, outlen); +LBL_ERR: + XFREE(list); + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_length_sequence.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_length_sequence.c new file mode 100644 index 0000000..fe7dd0e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_length_sequence.c @@ -0,0 +1,180 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_sequence.c + ASN.1 DER, length a SEQUENCE, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Get the length of a DER sequence + @param list The sequences of items in the SEQUENCE + @param inlen The number of items + @param outlen [out] The length required in octets to store it + @return CRYPT_OK on success +*/ +int der_length_sequence(const ltc_asn1_list *list, unsigned long inlen, + unsigned long *outlen) +{ + return der_length_sequence_ex(list, inlen, outlen, NULL); +} + +int der_length_sequence_ex(const ltc_asn1_list *list, unsigned long inlen, + unsigned long *outlen, unsigned long *payloadlen) +{ + int err; + ltc_asn1_type type; + unsigned long size, x, y, i; + void *data; + + LTC_ARGCHK(list != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get size of output that will be required */ + y = 0; + for (i = 0; i < inlen; i++) { + type = list[i].type; + size = list[i].size; + data = list[i].data; + + if (type == LTC_ASN1_EOL) { + break; + } + + /* some items may be optional during import */ + if (!list[i].used && list[i].optional) continue; + + switch (type) { + case LTC_ASN1_BOOLEAN: + if ((err = der_length_boolean(&x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_INTEGER: + if ((err = der_length_integer(data, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_SHORT_INTEGER: + if ((err = der_length_short_integer(*((unsigned long *)data), &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_BIT_STRING: + case LTC_ASN1_RAW_BIT_STRING: + if ((err = der_length_bit_string(size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_OCTET_STRING: + if ((err = der_length_octet_string(size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_NULL: + y += 2; + break; + + case LTC_ASN1_OBJECT_IDENTIFIER: + if ((err = der_length_object_identifier(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_IA5_STRING: + if ((err = der_length_ia5_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_TELETEX_STRING: + if ((err = der_length_teletex_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_PRINTABLE_STRING: + if ((err = der_length_printable_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_UTCTIME: + if ((err = der_length_utctime(data, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_GENERALIZEDTIME: + if ((err = der_length_generalizedtime(data, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_UTF8_STRING: + if ((err = der_length_utf8_string(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_CUSTOM_TYPE: + if ((err = der_length_custom_type(&list[i], &x, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_SET: + case LTC_ASN1_SETOF: + case LTC_ASN1_SEQUENCE: + if ((err = der_length_sequence(data, size, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + y += x; + break; + + case LTC_ASN1_CHOICE: + case LTC_ASN1_EOL: + default: + err = CRYPT_INVALID_ARG; + goto LBL_ERR; + } + } + + if ((err = der_length_asn1_length(y, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + + if (payloadlen != NULL) { + *payloadlen = y; + } + + /* store size */ + *outlen = y + x + 1; + err = CRYPT_OK; + +LBL_ERR: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_sequence_free.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_sequence_free.c new file mode 100644 index 0000000..b7cbd07 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_sequence_free.c @@ -0,0 +1,53 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_sequence_free.c + ASN.1 DER, free's a structure allocated by der_decode_sequence_flexi(), Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Free memory allocated by der_decode_sequence_flexi() + @param in The list to free +*/ +void der_sequence_free(ltc_asn1_list *in) +{ + ltc_asn1_list *l; + + if (!in) return; + + /* walk to the start of the chain */ + while (in->prev != NULL || in->parent != NULL) { + if (in->parent != NULL) { + in = in->parent; + } else { + in = in->prev; + } + } + + /* now walk the list and free stuff */ + while (in != NULL) { + /* is there a child? */ + if (in->child) { + /* disconnect */ + in->child->parent = NULL; + der_sequence_free(in->child); + } + + switch (in->type) { + case LTC_ASN1_SETOF: break; + case LTC_ASN1_INTEGER : if (in->data != NULL) { mp_clear(in->data); } break; + default : if (in->data != NULL) { XFREE(in->data); } + } + + /* move to next and free current */ + l = in->next; + XFREE(in); + in = l; + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_sequence_shrink.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_sequence_shrink.c new file mode 100644 index 0000000..22634a3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/der_sequence_shrink.c @@ -0,0 +1,40 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_sequence_shrink.c + Free memory allocated for CONSTRUCTED, SET or SEQUENCE elements by der_decode_sequence_flexi(), Steffen Jaeckel +*/ + +#ifdef LTC_DER + +/** + Free memory allocated for CONSTRUCTED, + SET or SEQUENCE elements by der_decode_sequence_flexi() + @param in The list to shrink +*/ +void der_sequence_shrink(ltc_asn1_list *in) +{ + if (!in) return; + + /* now walk the list and free stuff */ + while (in != NULL) { + /* is there a child? */ + if (in->child) { + der_sequence_shrink(in->child); + } + + switch (in->type) { + case LTC_ASN1_CUSTOM_TYPE: + case LTC_ASN1_SET: + case LTC_ASN1_SEQUENCE : if (in->data != NULL) { XFREE(in->data); in->data = NULL; } break; + default: break; + } + + /* move to next and free current */ + in = in->next; + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/sub.mk new file mode 100644 index 0000000..d44420b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sequence/sub.mk @@ -0,0 +1,7 @@ +srcs-y += der_decode_sequence_ex.c +srcs-y += der_decode_sequence_flexi.c +srcs-y += der_decode_sequence_multi.c +srcs-y += der_encode_sequence_ex.c +srcs-y += der_encode_sequence_multi.c +srcs-y += der_length_sequence.c +srcs-y += der_sequence_free.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/der_encode_set.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/der_encode_set.c new file mode 100644 index 0000000..1e90a1f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/der_encode_set.c @@ -0,0 +1,74 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_set.c + ASN.1 DER, Encode a SET, Tom St Denis +*/ + +#ifdef LTC_DER + +/* LTC define to ASN.1 TAG */ +static int s_ltc_to_asn1(ltc_asn1_type v) +{ + return der_asn1_type_to_identifier_map[v]; +} + + +static int s_qsort_helper(const void *a, const void *b) +{ + ltc_asn1_list *A = (ltc_asn1_list *)a, *B = (ltc_asn1_list *)b; + int r; + + r = s_ltc_to_asn1(A->type) - s_ltc_to_asn1(B->type); + + /* for QSORT the order is UNDEFINED if they are "equal" which means it is NOT DETERMINISTIC. So we force it to be :-) */ + if (r == 0) { + /* their order in the original list now determines the position */ + return A->used - B->used; + } + return r; +} + +/* + Encode a SET type + @param list The list of items to encode + @param inlen The number of items in the list + @param out [out] The destination + @param outlen [in/out] The size of the output + @return CRYPT_OK on success +*/ +int der_encode_set(const ltc_asn1_list *list, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + ltc_asn1_list *copy; + unsigned long x; + int err; + + /* make copy of list */ + copy = XCALLOC(inlen, sizeof(*copy)); + if (copy == NULL) { + return CRYPT_MEM; + } + + /* fill in used member with index so we can fully sort it */ + for (x = 0; x < inlen; x++) { + copy[x] = list[x]; + copy[x].used = x; + } + + /* sort it by the "type" field */ + XQSORT(copy, inlen, sizeof(*copy), &s_qsort_helper); + + /* call der_encode_sequence_ex() */ + err = der_encode_sequence_ex(copy, inlen, out, outlen, LTC_ASN1_SET); + + /* free list */ + XFREE(copy); + + return err; +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/der_encode_setof.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/der_encode_setof.c new file mode 100644 index 0000000..184b6ec --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/der_encode_setof.c @@ -0,0 +1,151 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_setof.c + ASN.1 DER, Encode SET OF, Tom St Denis +*/ + +#ifdef LTC_DER + +struct edge { + unsigned char *start; + unsigned long size; +}; + +static int s_qsort_helper(const void *a, const void *b) +{ + struct edge *A = (struct edge *)a, *B = (struct edge *)b; + int r; + unsigned long x; + + /* compare min length */ + r = XMEMCMP(A->start, B->start, MIN(A->size, B->size)); + + if (r == 0 && A->size != B->size) { + if (A->size > B->size) { + for (x = B->size; x < A->size; x++) { + if (A->start[x]) { + return 1; + } + } + } else { + for (x = A->size; x < B->size; x++) { + if (B->start[x]) { + return -1; + } + } + } + } + + return r; +} + +/** + Encode a SETOF stucture + @param list The list of items to encode + @param inlen The number of items in the list + @param out [out] The destination + @param outlen [in/out] The size of the output + @return CRYPT_OK on success +*/ +int der_encode_setof(const ltc_asn1_list *list, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, z; + ptrdiff_t hdrlen; + int err; + struct edge *edges; + unsigned char *ptr, *buf; + + /* check that they're all the same type */ + for (x = 1; x < inlen; x++) { + if (list[x].type != list[x-1].type) { + return CRYPT_INVALID_ARG; + } + } + + /* alloc buffer to store copy of output */ + buf = XCALLOC(1, *outlen); + if (buf == NULL) { + return CRYPT_MEM; + } + + /* encode list */ + if ((err = der_encode_sequence_ex(list, inlen, buf, outlen, LTC_ASN1_SETOF)) != CRYPT_OK) { + XFREE(buf); + return err; + } + + /* allocate edges */ + edges = XCALLOC(inlen, sizeof(*edges)); + if (edges == NULL) { + XFREE(buf); + return CRYPT_MEM; + } + + /* skip header */ + ptr = buf + 1; + + /* now skip length data */ + x = *ptr++; + if (x >= 0x80) { + ptr += (x & 0x7F); + } + + /* get the size of the static header */ + hdrlen = ptr - buf; + + + /* scan for edges */ + x = 0; + while (ptr < (buf + *outlen)) { + /* store start */ + edges[x].start = ptr; + + /* skip type */ + z = 1; + + /* parse length */ + y = ptr[z++]; + if (y < 128) { + edges[x].size = y; + } else { + y &= 0x7F; + edges[x].size = 0; + while (y--) { + edges[x].size = (edges[x].size << 8) | ((unsigned long)ptr[z++]); + } + } + + /* skip content */ + edges[x].size += z; + ptr += edges[x].size; + ++x; + } + + /* sort based on contents (using edges) */ + XQSORT(edges, inlen, sizeof(*edges), &s_qsort_helper); + + /* copy static header */ + XMEMCPY(out, buf, hdrlen); + + /* copy+sort using edges+indecies to output from buffer */ + for (y = (unsigned long)hdrlen, x = 0; x < inlen; x++) { + XMEMCPY(out+y, edges[x].start, edges[x].size); + y += edges[x].size; + } + +#ifdef LTC_CLEAN_STACK + zeromem(buf, *outlen); +#endif + + /* free buffers */ + XFREE(edges); + XFREE(buf); + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/sub.mk new file mode 100644 index 0000000..52525d6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/set/sub.mk @@ -0,0 +1,2 @@ +srcs-y += der_encode_set.c +srcs-y += der_encode_setof.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_decode_short_integer.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_decode_short_integer.c new file mode 100644 index 0000000..e00702d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_decode_short_integer.c @@ -0,0 +1,60 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_short_integer.c + ASN.1 DER, decode an integer, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Read a short integer + @param in The DER encoded data + @param inlen Size of data + @param num [out] The integer to decode + @return CRYPT_OK if successful +*/ +int der_decode_short_integer(const unsigned char *in, unsigned long inlen, unsigned long *num) +{ + unsigned long len, x, y; + + LTC_ARGCHK(num != NULL); + LTC_ARGCHK(in != NULL); + + /* check length */ + if (inlen < 2) { + return CRYPT_INVALID_PACKET; + } + + /* check header */ + x = 0; + if ((in[x++] & 0x1F) != 0x02) { + return CRYPT_INVALID_PACKET; + } + + /* get the packet len */ + len = in[x++]; + + if (x + len > inlen) { + return CRYPT_INVALID_PACKET; + } + + if (len > sizeof(unsigned long)) { + return CRYPT_OVERFLOW; + } + + /* read number */ + y = 0; + while (len--) { + y = (y<<8) | (unsigned long)in[x++]; + } + *num = y; + + return CRYPT_OK; + +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_encode_short_integer.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_encode_short_integer.c new file mode 100644 index 0000000..3af47c5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_encode_short_integer.c @@ -0,0 +1,85 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_short_integer.c + ASN.1 DER, encode an integer, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store a short integer in the range (0,2^32-1) + @param num The integer to encode + @param out [out] The destination for the DER encoded integers + @param outlen [in/out] The max size and resulting size of the DER encoded integers + @return CRYPT_OK if successful +*/ +int der_encode_short_integer(unsigned long num, unsigned char *out, unsigned long *outlen) +{ + unsigned long len, x, y, z; + int err; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* force to 32 bits */ + num &= 0xFFFFFFFFUL; + + /* find out how big this will be */ + if ((err = der_length_short_integer(num, &len)) != CRYPT_OK) { + return err; + } + + if (*outlen < len) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + /* get len of output */ + z = 0; + y = num; + while (y) { + ++z; + y >>= 8; + } + + /* handle zero */ + if (z == 0) { + z = 1; + } + + /* see if msb is set */ + z += (num&(1UL<<((z<<3) - 1))) ? 1 : 0; + + /* adjust the number so the msB is non-zero */ + for (x = 0; (z <= 4) && (x < (4 - z)); x++) { + num <<= 8; + } + + /* store header */ + x = 0; + out[x++] = 0x02; + out[x++] = (unsigned char)z; + + /* if 31st bit is set output a leading zero and decrement count */ + if (z == 5) { + out[x++] = 0; + --z; + } + + /* store values */ + for (y = 0; y < z; y++) { + out[x++] = (unsigned char)((num >> 24) & 0xFF); + num <<= 8; + } + + /* we good */ + *outlen = x; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_length_short_integer.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_length_short_integer.c new file mode 100644 index 0000000..aeee948 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/der_length_short_integer.c @@ -0,0 +1,52 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_short_integer.c + ASN.1 DER, get length of encoding, Tom St Denis +*/ + + +#ifdef LTC_DER +/** + Gets length of DER encoding of num + @param num The integer to get the size of + @param outlen [out] The length of the DER encoding for the given integer + @return CRYPT_OK if successful +*/ +int der_length_short_integer(unsigned long num, unsigned long *outlen) +{ + unsigned long z, y; + int err; + + LTC_ARGCHK(outlen != NULL); + + /* force to 32 bits */ + num &= 0xFFFFFFFFUL; + + /* get the number of bytes */ + z = 0; + y = num; + while (y) { + ++z; + y >>= 8; + } + + /* handle zero */ + if (z == 0) { + z = 1; + } else if ((num&(1UL<<((z<<3) - 1))) != 0) { + /* in case msb is set */ + ++z; + } + + if ((err = der_length_asn1_length(z, &y)) != CRYPT_OK) { + return err; + } + *outlen = 1 + y + z; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/sub.mk new file mode 100644 index 0000000..fae1f08 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/short_integer/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_short_integer.c +srcs-y += der_encode_short_integer.c +srcs-y += der_length_short_integer.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sub.mk new file mode 100644 index 0000000..2951f19 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/sub.mk @@ -0,0 +1,17 @@ +subdirs-y += bit +subdirs-y += boolean +subdirs-y += choice +subdirs-y += custom_type +subdirs-y += general +subdirs-y += generalizedtime +subdirs-y += ia5 +subdirs-y += integer +subdirs-y += object_identifier +subdirs-y += octet +subdirs-y += printable_string +subdirs-y += sequence +subdirs-y += set +subdirs-y += short_integer +subdirs-y += utctime +subdirs-y += utf8 +subdirs-y += teletex_string diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/der_decode_teletex_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/der_decode_teletex_string.c new file mode 100644 index 0000000..6779cd5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/der_decode_teletex_string.c @@ -0,0 +1,72 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_teletex_string.c + ASN.1 DER, encode a teletex STRING +*/ + +#ifdef LTC_DER + +/** + Store a teletex STRING + @param in The DER encoded teletex STRING + @param inlen The size of the DER teletex STRING + @param out [out] The array of octets stored (one per char) + @param outlen [in/out] The number of octets stored + @return CRYPT_OK if successful +*/ +int der_decode_teletex_string(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int t, err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* must have header at least */ + if (inlen < 2) { + return CRYPT_INVALID_PACKET; + } + + /* check for 0x14 */ + if ((in[0] & 0x1F) != 0x14) { + return CRYPT_INVALID_PACKET; + } + x = 1; + + /* get the length of the data */ + y = inlen - x; + if ((err = der_decode_asn1_length(in + x, &y, &len)) != CRYPT_OK) { + return err; + } + x += y; + + /* is it too long? */ + if (len > *outlen) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + + if (len > (inlen - x)) { + return CRYPT_INVALID_PACKET; + } + + /* read the data */ + for (y = 0; y < len; y++) { + t = der_teletex_value_decode(in[x++]); + if (t == -1) { + return CRYPT_INVALID_ARG; + } + out[y] = t; + } + + *outlen = y; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/der_length_teletex_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/der_length_teletex_string.c new file mode 100644 index 0000000..05dc4b0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/der_length_teletex_string.c @@ -0,0 +1,188 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_teletex_string.c + ASN.1 DER, get length of teletex STRING +*/ + +#ifdef LTC_DER + +static const struct { + int code, value; +} teletex_table[] = { +{ '\0', 0 }, +{ '\a', 7 }, +{ '\b', 8 }, +{ '\t', 9 }, +{ '\n', 10 }, +{ '\v', 11 }, +{ '\f', 12 }, +{ '\r', 13 }, +{ ' ', 32 }, +{ '!', 33 }, +{ '"', 34 }, +{ '%', 37 }, +{ '&', 38 }, +{ '\'', 39 }, +{ '(', 40 }, +{ ')', 41 }, +{ '+', 43 }, +{ ',', 44 }, +{ '-', 45 }, +{ '.', 46 }, +{ '/', 47 }, +{ '0', 48 }, +{ '1', 49 }, +{ '2', 50 }, +{ '3', 51 }, +{ '4', 52 }, +{ '5', 53 }, +{ '6', 54 }, +{ '7', 55 }, +{ '8', 56 }, +{ '9', 57 }, +{ ':', 58 }, +{ ';', 59 }, +{ '<', 60 }, +{ '=', 61 }, +{ '>', 62 }, +{ '?', 63 }, +{ '@', 64 }, +{ 'A', 65 }, +{ 'B', 66 }, +{ 'C', 67 }, +{ 'D', 68 }, +{ 'E', 69 }, +{ 'F', 70 }, +{ 'G', 71 }, +{ 'H', 72 }, +{ 'I', 73 }, +{ 'J', 74 }, +{ 'K', 75 }, +{ 'L', 76 }, +{ 'M', 77 }, +{ 'N', 78 }, +{ 'O', 79 }, +{ 'P', 80 }, +{ 'Q', 81 }, +{ 'R', 82 }, +{ 'S', 83 }, +{ 'T', 84 }, +{ 'U', 85 }, +{ 'V', 86 }, +{ 'W', 87 }, +{ 'X', 88 }, +{ 'Y', 89 }, +{ 'Z', 90 }, +{ '[', 91 }, +{ ']', 93 }, +{ '_', 95 }, +{ 'a', 97 }, +{ 'b', 98 }, +{ 'c', 99 }, +{ 'd', 100 }, +{ 'e', 101 }, +{ 'f', 102 }, +{ 'g', 103 }, +{ 'h', 104 }, +{ 'i', 105 }, +{ 'j', 106 }, +{ 'k', 107 }, +{ 'l', 108 }, +{ 'm', 109 }, +{ 'n', 110 }, +{ 'o', 111 }, +{ 'p', 112 }, +{ 'q', 113 }, +{ 'r', 114 }, +{ 's', 115 }, +{ 't', 116 }, +{ 'u', 117 }, +{ 'v', 118 }, +{ 'w', 119 }, +{ 'x', 120 }, +{ 'y', 121 }, +{ 'z', 122 }, +{ '|', 124 }, +{ ' ', 160 }, +{ 0xa1, 161 }, +{ 0xa2, 162 }, +{ 0xa3, 163 }, +{ '$', 164 }, +{ 0xa5, 165 }, +{ '#', 166 }, +{ 0xa7, 167 }, +{ 0xa4, 168 }, +{ 0xab, 171 }, +{ 0xb0, 176 }, +{ 0xb1, 177 }, +{ 0xb2, 178 }, +{ 0xb3, 179 }, +{ 0xd7, 180 }, +{ 0xb5, 181 }, +{ 0xb6, 182 }, +{ 0xb7, 183 }, +{ 0xf7, 184 }, +{ 0xbb, 187 }, +{ 0xbc, 188 }, +{ 0xbd, 189 }, +{ 0xbe, 190 }, +{ 0xbf, 191 }, +}; + +int der_teletex_char_encode(int c) +{ + int x; + for (x = 0; x < (int)(sizeof(teletex_table)/sizeof(teletex_table[0])); x++) { + if (teletex_table[x].code == c) { + return teletex_table[x].value; + } + } + return -1; +} + +int der_teletex_value_decode(int v) +{ + int x; + for (x = 0; x < (int)(sizeof(teletex_table)/sizeof(teletex_table[0])); x++) { + if (teletex_table[x].value == v) { + return teletex_table[x].code; + } + } + return -1; +} + +/** + Gets length of DER encoding of teletex STRING + @param octets The values you want to encode + @param noctets The number of octets in the string to encode + @param outlen [out] The length of the DER encoding for the given string + @return CRYPT_OK if successful +*/ +int der_length_teletex_string(const unsigned char *octets, unsigned long noctets, unsigned long *outlen) +{ + unsigned long x; + int err; + + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(octets != NULL); + + /* scan string for validity */ + for (x = 0; x < noctets; x++) { + if (der_teletex_char_encode(octets[x]) == -1) { + return CRYPT_INVALID_ARG; + } + } + + if ((err = der_length_asn1_length(noctets, &x)) != CRYPT_OK) { + return err; + } + *outlen = 1 + x + noctets; + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/sub.mk new file mode 100644 index 0000000..8f47f07 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/teletex_string/sub.mk @@ -0,0 +1,2 @@ +srcs-y += der_decode_teletex_string.c +srcs-y += der_length_teletex_string.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_decode_utctime.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_decode_utctime.c new file mode 100644 index 0000000..6149bfe --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_decode_utctime.c @@ -0,0 +1,116 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_utctime.c + ASN.1 DER, decode a UTCTIME, Tom St Denis +*/ + +#ifdef LTC_DER + +static int s_char_to_int(unsigned char x) +{ + switch (x) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + default: return 100; + } +} + +#define DECODE_V(y, max) \ + y = s_char_to_int(buf[x])*10 + s_char_to_int(buf[x+1]); \ + if (y >= max) return CRYPT_INVALID_PACKET; \ + x += 2; + +/** + Decodes a UTC time structure in DER format (reads all 6 valid encoding formats) + @param in Input buffer + @param inlen Length of input buffer in octets + @param out [out] Destination of UTC time structure + @return CRYPT_OK if successful +*/ +int der_decode_utctime(const unsigned char *in, unsigned long *inlen, + ltc_utctime *out) +{ + unsigned char buf[32] = { 0 }; /* initialize as all zeroes */ + unsigned long x; + int y; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != NULL); + LTC_ARGCHK(out != NULL); + + /* check header */ + if (*inlen < 2UL || (in[1] >= sizeof(buf)) || ((in[1] + 2UL) > *inlen)) { + return CRYPT_INVALID_PACKET; + } + + /* decode the string */ + for (x = 0; x < in[1]; x++) { + y = der_ia5_value_decode(in[x+2]); + if (y == -1) { + return CRYPT_INVALID_PACKET; + } + buf[x] = y; + } + *inlen = 2 + x; + + + /* possible encodings are +YYMMDDhhmmZ +YYMMDDhhmm+hh'mm' +YYMMDDhhmm-hh'mm' +YYMMDDhhmmssZ +YYMMDDhhmmss+hh'mm' +YYMMDDhhmmss-hh'mm' + + So let's do a trivial decode upto [including] mm + */ + + x = 0; + DECODE_V(out->YY, 100); + DECODE_V(out->MM, 13); + DECODE_V(out->DD, 32); + DECODE_V(out->hh, 24); + DECODE_V(out->mm, 60); + + /* clear timezone and seconds info */ + out->off_dir = out->off_hh = out->off_mm = out->ss = 0; + + /* now is it Z, +, - or 0-9 */ + if (buf[x] == 'Z') { + return CRYPT_OK; + } + if (buf[x] == '+' || buf[x] == '-') { + out->off_dir = (buf[x++] == '+') ? 0 : 1; + DECODE_V(out->off_hh, 24); + DECODE_V(out->off_mm, 60); + return CRYPT_OK; + } + + /* decode seconds */ + DECODE_V(out->ss, 60); + + /* now is it Z, +, - */ + if (buf[x] == 'Z') { + return CRYPT_OK; + } + if (buf[x] == '+' || buf[x] == '-') { + out->off_dir = (buf[x++] == '+') ? 0 : 1; + DECODE_V(out->off_hh, 24); + DECODE_V(out->off_mm, 60); + return CRYPT_OK; + } + return CRYPT_INVALID_PACKET; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_encode_utctime.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_encode_utctime.c new file mode 100644 index 0000000..b1c6fbe --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_encode_utctime.c @@ -0,0 +1,71 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_utctime.c + ASN.1 DER, encode a UTCTIME, Tom St Denis +*/ + +#ifdef LTC_DER + +static const char * const baseten = "0123456789"; + +#define STORE_V(y) \ + out[x++] = der_ia5_char_encode(baseten[(y/10) % 10]); \ + out[x++] = der_ia5_char_encode(baseten[y % 10]); + +/** + Encodes a UTC time structure in DER format + @param utctime The UTC time structure to encode + @param out The destination of the DER encoding of the UTC time structure + @param outlen [in/out] The length of the DER encoding + @return CRYPT_OK if successful +*/ +int der_encode_utctime(const ltc_utctime *utctime, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, tmplen; + int err; + + LTC_ARGCHK(utctime != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((err = der_length_utctime(utctime, &tmplen)) != CRYPT_OK) { + return err; + } + if (tmplen > *outlen) { + *outlen = tmplen; + return CRYPT_BUFFER_OVERFLOW; + } + + /* store header */ + out[0] = 0x17; + + /* store values */ + x = 2; + STORE_V(utctime->YY); + STORE_V(utctime->MM); + STORE_V(utctime->DD); + STORE_V(utctime->hh); + STORE_V(utctime->mm); + STORE_V(utctime->ss); + + if (utctime->off_mm || utctime->off_hh) { + out[x++] = der_ia5_char_encode(utctime->off_dir ? '-' : '+'); + STORE_V(utctime->off_hh); + STORE_V(utctime->off_mm); + } else { + out[x++] = der_ia5_char_encode('Z'); + } + + /* store length */ + out[1] = (unsigned char)(x - 2); + + /* all good let's return */ + *outlen = x; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_length_utctime.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_length_utctime.c new file mode 100644 index 0000000..b93c25f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/der_length_utctime.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_utctime.c + ASN.1 DER, get length of UTCTIME, Tom St Denis +*/ + +#ifdef LTC_DER + +/** + Gets length of DER encoding of UTCTIME + @param utctime The UTC time structure to get the size of + @param outlen [out] The length of the DER encoding + @return CRYPT_OK if successful +*/ +int der_length_utctime(const ltc_utctime *utctime, unsigned long *outlen) +{ + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(utctime != NULL); + + if (utctime->off_hh == 0 && utctime->off_mm == 0) { + /* we encode as YYMMDDhhmmssZ */ + *outlen = 2 + 13; + } else { + /* we encode as YYMMDDhhmmss{+|-}hh'mm' */ + *outlen = 2 + 17; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/sub.mk new file mode 100644 index 0000000..afb3e1b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utctime/sub.mk @@ -0,0 +1,3 @@ +srcs-y += der_decode_utctime.c +srcs-y += der_encode_utctime.c +srcs-y += der_length_utctime.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_decode_utf8_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_decode_utf8_string.c new file mode 100644 index 0000000..93a5e5e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_decode_utf8_string.c @@ -0,0 +1,117 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_decode_utf8_string.c + ASN.1 DER, encode a UTF8 STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Decode a UTF8 STRING and recover an array of unicode characters. + @param in The DER encoded UTF8 STRING + @param inlen The size of the DER UTF8 STRING + @param out [out] The array of unicode characters (wchar_t*) + @param outlen [in/out] The number of unicode characters in the array + @return CRYPT_OK if successful +*/ +int der_decode_utf8_string(const unsigned char *in, unsigned long inlen, + wchar_t *out, unsigned long *outlen) +{ + wchar_t tmp; + unsigned long x, y, z, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* must have header at least */ + if (inlen < 2) { + return CRYPT_INVALID_PACKET; + } + + /* check for 0x0C */ + if ((in[0] & 0x1F) != 0x0C) { + return CRYPT_INVALID_PACKET; + } + x = 1; + + /* get the length of the data */ + y = inlen - x; + if ((err = der_decode_asn1_length(in + x, &y, &len)) != CRYPT_OK) { + return err; + } + x += y; + + if (len > (inlen - x)) { + return CRYPT_INVALID_PACKET; + } + + /* proceed to recover unicode characters from utf8 data. + for reference see Section 3 of RFC 3629: + + https://tools.ietf.org/html/rfc3629#section-3 + */ + for (y = 0; x < inlen; ) { + /* read first byte */ + tmp = in[x++]; + + /* a unicode character is recovered from a sequence of 1 to 4 utf8 bytes. + the form of those bytes must match a row in the following table: + + 0xxxxxxx + 110xxxxx 10xxxxxx + 1110xxxx 10xxxxxx 10xxxxxx + 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + + the number of leading ones in the first byte (0,2,3,4) determines the + number of remaining bytes to read (0,1,2,3) + */ + + /* determine z, the number of leading ones. + this is done by left-shifting tmp, which clears the ms-bits */ + for (z = 0; (tmp & 0x80) && (z <= 4); z++, tmp = (tmp << 1) & 0xFF); + + /* z should be in {0,2,3,4} */ + if (z == 1 || z > 4) { + return CRYPT_INVALID_PACKET; + } + + /* right-shift tmp to restore least-sig bits */ + tmp >>= z; + + /* now update z so it equals the number of additional bytes to read */ + if (z > 0) { --z; } + + if (x + z > inlen) { + return CRYPT_INVALID_PACKET; + } + + /* read remaining bytes */ + while (z-- != 0) { + if ((in[x] & 0xC0) != 0x80) { + return CRYPT_INVALID_PACKET; + } + tmp = (tmp << 6) | ((wchar_t)in[x++] & 0x3F); + } + + if (y < *outlen) { + out[y] = tmp; + } + y++; + } + if (y > *outlen) { + err = CRYPT_BUFFER_OVERFLOW; + } else { + err = CRYPT_OK; + } + *outlen = y; + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_encode_utf8_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_encode_utf8_string.c new file mode 100644 index 0000000..a8154cd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_encode_utf8_string.c @@ -0,0 +1,76 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_encode_utf8_string.c + ASN.1 DER, encode a UTF8 STRING, Tom St Denis +*/ + + +#ifdef LTC_DER + +/** + Store an UTF8 STRING + @param in The array of UTF8 to store (one per wchar_t) + @param inlen The number of UTF8 to store + @param out [out] The destination for the DER encoded UTF8 STRING + @param outlen [in/out] The max size and resulting size of the DER UTF8 STRING + @return CRYPT_OK if successful +*/ +int der_encode_utf8_string(const wchar_t *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x, y, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* get the size */ + for (x = len = 0; x < inlen; x++) { + if (!der_utf8_valid_char(in[x])) return CRYPT_INVALID_ARG; + len += der_utf8_charsize(in[x]); + } + if ((err = der_length_asn1_length(len, &x)) != CRYPT_OK) { + return err; + } + x += len + 1; + + /* too big? */ + if (x > *outlen) { + *outlen = x; + return CRYPT_BUFFER_OVERFLOW; + } + + /* encode the header+len */ + x = 0; + out[x++] = 0x0C; + + y = *outlen - x; + if ((err = der_encode_asn1_length(len, out + x, &y)) != CRYPT_OK) { + return err; + } + x += y; + + /* store UTF8 */ + for (y = 0; y < inlen; y++) { + switch (der_utf8_charsize(in[y])) { + case 1: out[x++] = (unsigned char)in[y]; break; + case 2: out[x++] = 0xC0 | ((in[y] >> 6) & 0x1F); out[x++] = 0x80 | (in[y] & 0x3F); break; + case 3: out[x++] = 0xE0 | ((in[y] >> 12) & 0x0F); out[x++] = 0x80 | ((in[y] >> 6) & 0x3F); out[x++] = 0x80 | (in[y] & 0x3F); break; +#if !defined(LTC_WCHAR_MAX) || LTC_WCHAR_MAX > 0xFFFF + case 4: out[x++] = 0xF0 | ((in[y] >> 18) & 0x07); out[x++] = 0x80 | ((in[y] >> 12) & 0x3F); out[x++] = 0x80 | ((in[y] >> 6) & 0x3F); out[x++] = 0x80 | (in[y] & 0x3F); break; +#endif + default: break; + } + } + + /* return length */ + *outlen = x; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_length_utf8_string.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_length_utf8_string.c new file mode 100644 index 0000000..424002b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/der_length_utf8_string.c @@ -0,0 +1,81 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file der_length_utf8_string.c + ASN.1 DER, get length of UTF8 STRING, Tom St Denis +*/ + +#ifdef LTC_DER + +/** Return the size in bytes of a UTF-8 character + @param c The UTF-8 character to measure + @return The size in bytes +*/ +unsigned long der_utf8_charsize(const wchar_t c) +{ + if (c <= 0x7F) { + return 1; + } + if (c <= 0x7FF) { + return 2; + } +#if LTC_WCHAR_MAX == 0xFFFF + return 3; +#else + if (c <= 0xFFFF) { + return 3; + } + return 4; +#endif +} + +/** + Test whether the given code point is valid character + @param c The UTF-8 character to test + @return 1 - valid, 0 - invalid +*/ +int der_utf8_valid_char(const wchar_t c) +{ + LTC_UNUSED_PARAM(c); +#if !defined(LTC_WCHAR_MAX) || LTC_WCHAR_MAX > 0xFFFF + if (c > 0x10FFFF) return 0; +#endif +#if LTC_WCHAR_MAX != 0xFFFF && LTC_WCHAR_MAX != 0xFFFFFFFF + if (c < 0) return 0; +#endif + return 1; +} + +/** + Gets length of DER encoding of UTF8 STRING + @param in The characters to measure the length of + @param noctets The number of octets in the string to encode + @param outlen [out] The length of the DER encoding for the given string + @return CRYPT_OK if successful +*/ +int der_length_utf8_string(const wchar_t *in, unsigned long noctets, unsigned long *outlen) +{ + unsigned long x, len; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(outlen != NULL); + + len = 0; + for (x = 0; x < noctets; x++) { + if (!der_utf8_valid_char(in[x])) return CRYPT_INVALID_ARG; + len += der_utf8_charsize(in[x]); + } + + if ((err = der_length_asn1_length(len, &x)) != CRYPT_OK) { + return err; + } + *outlen = 1 + x + len; + + return CRYPT_OK; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/sub.mk new file mode 100644 index 0000000..3538929 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/der/utf8/sub.mk @@ -0,0 +1,4 @@ +cflags-remove-y += -Wextra +srcs-y += der_decode_utf8_string.c +srcs-y += der_encode_utf8_string.c +srcs-y += der_length_utf8_string.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_get_oid.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_get_oid.c new file mode 100644 index 0000000..b3e1ee2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_get_oid.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_DER + +typedef struct { + enum ltc_oid_id id; + const char* oid; +} oid_table_entry; + +static const oid_table_entry pka_oids[] = { + { LTC_OID_RSA, "1.2.840.113549.1.1.1" }, + { LTC_OID_DSA, "1.2.840.10040.4.1" }, + { LTC_OID_EC, "1.2.840.10045.2.1" }, + { LTC_OID_EC_PRIMEF, "1.2.840.10045.1.1" }, + { LTC_OID_X25519, "1.3.101.110" }, + { LTC_OID_ED25519, "1.3.101.112" }, +}; + +/* + Returns the OID requested. + @return CRYPT_OK if valid +*/ +int pk_get_oid(enum ltc_oid_id id, const char **st) +{ + unsigned int i; + LTC_ARGCHK(st != NULL); + for (i = 0; i < sizeof(pka_oids)/sizeof(pka_oids[0]); ++i) { + if (pka_oids[i].id == id) { + *st = pka_oids[i].oid; + return CRYPT_OK; + } + } + return CRYPT_INVALID_ARG; +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_oid_cmp.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_oid_cmp.c new file mode 100644 index 0000000..f842bc5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_oid_cmp.c @@ -0,0 +1,44 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_DER + +/* + Compare an OID string to an array of `unsigned long`. + @return CRYPT_OK if equal +*/ +int pk_oid_cmp_with_ulong(const char *o1, const unsigned long *o2, unsigned long o2size) +{ + unsigned long i; + char tmp[256] = { 0 }; + int err; + + if (o1 == NULL || o2 == NULL) return CRYPT_ERROR; + + i = sizeof(tmp); + if ((err = pk_oid_num_to_str(o2, o2size, tmp, &i)) != CRYPT_OK) { + return err; + } + + if (XSTRCMP(o1, tmp) != 0) { + return CRYPT_PK_INVALID_TYPE; + } + + return CRYPT_OK; +} + +/* + Compare an OID string to an OID element decoded from ASN.1. + @return CRYPT_OK if equal +*/ +int pk_oid_cmp_with_asn1(const char *o1, const ltc_asn1_list *o2) +{ + if (o1 == NULL || o2 == NULL) return CRYPT_ERROR; + + if (o2->type != LTC_ASN1_OBJECT_IDENTIFIER) return CRYPT_INVALID_ARG; + + return pk_oid_cmp_with_ulong(o1, o2->data, o2->size); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_oid_str.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_oid_str.c new file mode 100644 index 0000000..bc21a6f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/pk_oid_str.c @@ -0,0 +1,82 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +int pk_oid_str_to_num(const char *OID, unsigned long *oid, unsigned long *oidlen) +{ + unsigned long i, j, limit, oid_j; + size_t OID_len; + + LTC_ARGCHK(oidlen != NULL); + + limit = *oidlen; + *oidlen = 0; /* make sure that we return zero oidlen on error */ + for (i = 0; i < limit; i++) oid[i] = 0; + + if (OID == NULL) return CRYPT_OK; + + OID_len = XSTRLEN(OID); + if (OID_len == 0) return CRYPT_OK; + + for (i = 0, j = 0; i < OID_len; i++) { + if (OID[i] == '.') { + if (++j >= limit) continue; + } + else if ((OID[i] >= '0') && (OID[i] <= '9')) { + if ((j >= limit) || (oid == NULL)) continue; + oid_j = oid[j]; + oid[j] = oid[j] * 10 + (OID[i] - '0'); + if (oid[j] < oid_j) return CRYPT_OVERFLOW; + } + else { + return CRYPT_ERROR; + } + } + if (j == 0) return CRYPT_ERROR; + if (j >= limit) { + *oidlen = j; + return CRYPT_BUFFER_OVERFLOW; + } + *oidlen = j + 1; + return CRYPT_OK; +} + +int pk_oid_num_to_str(const unsigned long *oid, unsigned long oidlen, char *OID, unsigned long *outlen) +{ + int i; + unsigned long j, k; + char tmp[256] = { 0 }; + + LTC_ARGCHK(oid != NULL); + LTC_ARGCHK(oidlen < INT_MAX); + LTC_ARGCHK(outlen != NULL); + + for (i = oidlen - 1, k = 0; i >= 0; i--) { + j = oid[i]; + if (j == 0) { + tmp[k] = '0'; + if (++k >= sizeof(tmp)) return CRYPT_ERROR; + } + else { + while (j > 0) { + tmp[k] = '0' + (j % 10); + if (++k >= sizeof(tmp)) return CRYPT_ERROR; + j /= 10; + } + } + if (i > 0) { + tmp[k] = '.'; + if (++k >= sizeof(tmp)) return CRYPT_ERROR; + } + } + if (*outlen < k + 1) { + *outlen = k + 1; + return CRYPT_BUFFER_OVERFLOW; + } + LTC_ARGCHK(OID != NULL); + for (j = 0; j < k; j++) OID[j] = tmp[k - j - 1]; + OID[k] = '\0'; + *outlen = k; /* the length without terminating NUL byte */ + return CRYPT_OK; +} diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/sub.mk new file mode 100644 index 0000000..867ffad --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/oid/sub.mk @@ -0,0 +1,2 @@ +srcs-y += pk_oid_cmp.c +srcs-y += pk_oid_str.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/pkcs8/pkcs8_decode_flexi.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/pkcs8/pkcs8_decode_flexi.c new file mode 100644 index 0000000..e715f11 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/pkcs8/pkcs8_decode_flexi.c @@ -0,0 +1,97 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_PKCS_8 + +/** + PKCS#8 decrypt if necessary & flexi-decode + + @param in Pointer to the ASN.1 encoded input data + @param inlen Length of the input data + @param pwd Pointer to the password that was used when encrypting + @param pwdlen Length of the password + @param decoded_list Pointer to a pointer for the flexi-decoded list + @return CRYPT_OK on success +*/ +int pkcs8_decode_flexi(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + ltc_asn1_list **decoded_list) +{ + unsigned long len = inlen; + unsigned long dec_size; + unsigned char *dec_data = NULL; + ltc_asn1_list *l = NULL; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(decoded_list != NULL); + + *decoded_list = NULL; + if ((err = der_decode_sequence_flexi(in, &len, &l)) == CRYPT_OK) { + /* the following "if" detects whether it is encrypted or not */ + /* PKCS8 Setup + * 0:d=0 hl=4 l= 380 cons: SEQUENCE + * 4:d=1 hl=2 l= 78 cons: SEQUENCE + * 6:d=2 hl=2 l= 9 prim: OBJECT :OID indicating PBES1 or PBES2 (== *lalgoid) + * 17:d=2 hl=2 l= 65 cons: SEQUENCE + * Stuff in between is dependent on whether it's PBES1 or PBES2 + * 84:d=1 hl=4 l= 296 prim: OCTET STRING :bytes (== encrypted data) + */ + if (l->type == LTC_ASN1_SEQUENCE && + LTC_ASN1_IS_TYPE(l->child, LTC_ASN1_SEQUENCE) && + LTC_ASN1_IS_TYPE(l->child->child, LTC_ASN1_OBJECT_IDENTIFIER) && + LTC_ASN1_IS_TYPE(l->child->child->next, LTC_ASN1_SEQUENCE) && + LTC_ASN1_IS_TYPE(l->child->next, LTC_ASN1_OCTET_STRING)) { + ltc_asn1_list *lalgoid = l->child->child; + pbes_arg pbes; + + XMEMSET(&pbes, 0, sizeof(pbes)); + + if (pbes1_extract(lalgoid, &pbes) == CRYPT_OK) { + /* Successfully extracted PBES1 parameters */ + } else if (pbes2_extract(lalgoid, &pbes) == CRYPT_OK) { + /* Successfully extracted PBES2 parameters */ + } else { + /* unsupported encryption */ + err = CRYPT_INVALID_PACKET; + goto LBL_DONE; + } + + pbes.enc_data = l->child->next; + pbes.pwd = pwd; + pbes.pwdlen = pwdlen; + + dec_size = pbes.enc_data->size; + if ((dec_data = XMALLOC(dec_size)) == NULL) { + err = CRYPT_MEM; + goto LBL_DONE; + } + + if ((err = pbes_decrypt(&pbes, dec_data, &dec_size)) != CRYPT_OK) goto LBL_DONE; + + der_free_sequence_flexi(l); + l = NULL; + err = der_decode_sequence_flexi(dec_data, &dec_size, &l); + if (err != CRYPT_OK) goto LBL_DONE; + *decoded_list = l; + } + else { + /* not encrypted */ + err = CRYPT_OK; + *decoded_list = l; + } + /* Set l to NULL so it won't be free'd */ + l = NULL; + } + +LBL_DONE: + if (l) der_free_sequence_flexi(l); + if (dec_data) { + zeromem(dec_data, dec_size); + XFREE(dec_data); + } + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/asn1/sub.mk new file mode 100644 index 0000000..86d77d4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/sub.mk @@ -0,0 +1,2 @@ +subdirs-y += der +subdirs-y += oid diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_decode_public_key_from_certificate.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_decode_public_key_from_certificate.c new file mode 100644 index 0000000..feff868 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_decode_public_key_from_certificate.c @@ -0,0 +1,109 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x509_decode_public_key_from_certificate.c + ASN.1 DER/X.509, decode a certificate +*/ + +#ifdef LTC_DER + +/* Check if it looks like a SubjectPublicKeyInfo */ +#define LOOKS_LIKE_SPKI(l) ((l) != NULL) \ +&& ((l)->type == LTC_ASN1_SEQUENCE) \ +&& ((l)->child != NULL) \ +&& ((l)->child->type == LTC_ASN1_OBJECT_IDENTIFIER) \ +&& ((l)->next != NULL) \ +&& ((l)->next->type == LTC_ASN1_BIT_STRING) + +/** + Try to decode the public key from a X.509 certificate + @param in The input buffer + @param inlen The length of the input buffer + @param algorithm One out of the enum #public_key_algorithms + @param param_type The parameters' type out of the enum ltc_asn1_type + @param parameters The parameters to include + @param parameters_len [in/out] The number of parameters to include + @param callback The callback + @param ctx The context passed to the callback + @return CRYPT_OK on success, CRYPT_NOP if no SubjectPublicKeyInfo was found +*/ +int x509_decode_public_key_from_certificate(const unsigned char *in, unsigned long inlen, + enum ltc_oid_id algorithm, ltc_asn1_type param_type, + ltc_asn1_list* parameters, unsigned long *parameters_len, + public_key_decode_cb callback, void *ctx) +{ + int err; + unsigned char *tmpbuf; + unsigned long tmpbuf_len, tmp_inlen; + ltc_asn1_list *decoded_list = NULL, *l; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != 0); + LTC_ARGCHK(callback != NULL); + + tmpbuf_len = inlen; + tmpbuf = XCALLOC(1, tmpbuf_len); + if (tmpbuf == NULL) { + err = CRYPT_MEM; + goto LBL_OUT; + } + + tmp_inlen = inlen; + if ((err = der_decode_sequence_flexi(in, &tmp_inlen, &decoded_list)) == CRYPT_OK) { + l = decoded_list; + + err = CRYPT_NOP; + + /* Move 2 levels up in the tree + SEQUENCE + SEQUENCE + ... + */ + if ((l->type == LTC_ASN1_SEQUENCE) && (l->child != NULL)) { + l = l->child; + if ((l->type == LTC_ASN1_SEQUENCE) && (l->child != NULL)) { + l = l->child; + + /* Move forward in the tree until we find this combination + ... + SEQUENCE + SEQUENCE + OBJECT IDENTIFIER + NULL + BIT STRING + */ + do { + /* The additional check for l->data is there to make sure + * we won't try to decode a list that has been 'shrunk' + */ + if ((l->type == LTC_ASN1_SEQUENCE) + && (l->data != NULL) + && LOOKS_LIKE_SPKI(l->child)) { + if (algorithm == LTC_OID_EC) { + err = callback(l->data, l->size, ctx); + } else { + err = x509_decode_subject_public_key_info(l->data, l->size, + algorithm, tmpbuf, &tmpbuf_len, + param_type, parameters, parameters_len); + if (err == CRYPT_OK) { + err = callback(tmpbuf, tmpbuf_len, ctx); + goto LBL_OUT; + } + } + } + l = l->next; + } while(l); + } + } + } + +LBL_OUT: + if (decoded_list) der_free_sequence_flexi(decoded_list); + if (tmpbuf != NULL) XFREE(tmpbuf); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_decode_subject_public_key_info.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_decode_subject_public_key_info.c new file mode 100644 index 0000000..a94362d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_decode_subject_public_key_info.c @@ -0,0 +1,119 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x509_decode_subject_public_key_info.c + ASN.1 DER/X.509, encode a SubjectPublicKeyInfo structure --nmav +*/ + +#ifdef LTC_DER + +/* AlgorithmIdentifier := SEQUENCE { + * algorithm OBJECT IDENTIFIER, + * parameters ANY DEFINED BY algorithm + * } + * + * SubjectPublicKeyInfo := SEQUENCE { + * algorithm AlgorithmIdentifier, + * subjectPublicKey BIT STRING + * } + */ +/** + Decode a SubjectPublicKeyInfo + @param in The input buffer + @param inlen The length of the input buffer + @param algorithm One out of the enum #public_key_algorithms + @param public_key The buffer for the public key + @param public_key_len [in/out] The length of the public key buffer and the written length + @param parameters_type The parameters' type out of the enum ltc_asn1_type + @param parameters The parameters to include + @param parameters_len [in/out] The number of parameters to include + @return CRYPT_OK on success +*/ +int x509_decode_subject_public_key_info(const unsigned char *in, unsigned long inlen, + unsigned int algorithm, void* public_key, unsigned long* public_key_len, + ltc_asn1_type parameters_type, ltc_asn1_list* parameters, unsigned long *parameters_len) +{ + int err; + unsigned long len, alg_id_num, tmplen; + const char* oid; + unsigned char *tmpbuf; + unsigned long tmpoid[16]; + unsigned long *_parameters_len; + ltc_asn1_list alg_id[2]; + ltc_asn1_list subject_pubkey[2]; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen != 0); + LTC_ARGCHK(public_key_len != NULL); + + if (parameters_type != LTC_ASN1_EOL) { + if ((parameters == NULL) || (parameters_len == NULL)) { + tmplen = 0; + _parameters_len = &tmplen; + } else { + _parameters_len = parameters_len; + } + } + + err = pk_get_oid(algorithm, &oid); + if (err != CRYPT_OK) { + return err; + } + + /* see if the OpenSSL DER format RSA public key will work */ + tmpbuf = XCALLOC(1, inlen); + if (tmpbuf == NULL) { + err = CRYPT_MEM; + goto LBL_ERR; + } + + /* this includes the internal hash ID and optional params (NULL in this case) */ + LTC_SET_ASN1(alg_id, 0, LTC_ASN1_OBJECT_IDENTIFIER, tmpoid, sizeof(tmpoid)/sizeof(tmpoid[0])); + if (parameters_type == LTC_ASN1_EOL) { + alg_id_num = 1; + } else { + LTC_SET_ASN1(alg_id, 1, parameters_type, parameters, *_parameters_len); + alg_id_num = 2; + } + + /* the actual format of the SSL DER key is odd, it stores a RSAPublicKey + * in a **BIT** string ... so we have to extract it then proceed to convert bit to octet + */ + LTC_SET_ASN1(subject_pubkey, 0, LTC_ASN1_SEQUENCE, alg_id, alg_id_num); + LTC_SET_ASN1(subject_pubkey, 1, LTC_ASN1_RAW_BIT_STRING, tmpbuf, inlen*8U); + + err=der_decode_sequence(in, inlen, subject_pubkey, 2UL); + if (err != CRYPT_OK) { + goto LBL_ERR; + } + if (parameters_type != LTC_ASN1_EOL) { + *_parameters_len = alg_id[1].size; + } + + if ((err = pk_oid_cmp_with_asn1(oid, &alg_id[0])) != CRYPT_OK) { + /* OID mismatch */ + goto LBL_ERR; + } + + len = subject_pubkey[1].size/8; + if (*public_key_len >= len) { + XMEMCPY(public_key, subject_pubkey[1].data, len); + *public_key_len = len; + } else { + *public_key_len = len; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + err = CRYPT_OK; + +LBL_ERR: + + XFREE(tmpbuf); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_encode_subject_public_key_info.c b/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_encode_subject_public_key_info.c new file mode 100644 index 0000000..fd0a5d0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/asn1/x509/x509_encode_subject_public_key_info.c @@ -0,0 +1,66 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x509_encode_subject_public_key_info.c + ASN.1 DER/X.509, encode a SubjectPublicKeyInfo structure --nmav +*/ + +#ifdef LTC_DER + +/* AlgorithmIdentifier := SEQUENCE { + * algorithm OBJECT IDENTIFIER, + * parameters ANY DEFINED BY algorithm + * } + * + * SubjectPublicKeyInfo := SEQUENCE { + * algorithm AlgorithmIdentifier, + * subjectPublicKey BIT STRING + * } + */ +/** + Encode a SubjectPublicKeyInfo + @param out The output buffer + @param outlen [in/out] Length of buffer and resulting length of output + @param algorithm One out of the enum #public_key_algorithms + @param public_key The buffer for the public key + @param public_key_len The length of the public key buffer + @param parameters_type The parameters' type out of the enum ltc_asn1_type + @param parameters The parameters to include + @param parameters_len The number of parameters to include + @return CRYPT_OK on success +*/ +int x509_encode_subject_public_key_info(unsigned char *out, unsigned long *outlen, + unsigned int algorithm, const void* public_key, unsigned long public_key_len, + ltc_asn1_type parameters_type, ltc_asn1_list* parameters, unsigned long parameters_len) +{ + int err; + ltc_asn1_list alg_id[2]; + const char *OID; + unsigned long oid[16], oidlen; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if ((err = pk_get_oid(algorithm, &OID)) != CRYPT_OK) { + return err; + } + + oidlen = sizeof(oid)/sizeof(oid[0]); + if ((err = pk_oid_str_to_num(OID, oid, &oidlen)) != CRYPT_OK) { + return err; + } + + LTC_SET_ASN1(alg_id, 0, LTC_ASN1_OBJECT_IDENTIFIER, oid, oidlen); + LTC_SET_ASN1(alg_id, 1, parameters_type, parameters, parameters_len); + + return der_encode_sequence_multi(out, outlen, + LTC_ASN1_SEQUENCE, (unsigned long)sizeof(alg_id)/sizeof(alg_id[0]), alg_id, + LTC_ASN1_RAW_BIT_STRING, public_key_len*8U, public_key, + LTC_ASN1_EOL, 0UL, NULL); + +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh.c new file mode 100644 index 0000000..277ce30 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh.c @@ -0,0 +1,227 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/* This holds the key settings. ***MUST*** be organized by size from smallest to largest. */ +const ltc_dh_set_type ltc_dh_sets[] = { +#ifdef LTC_DH768 +{ /* 768-bit MODP Group 1 - https://tools.ietf.org/html/rfc7296#appendix-B.1 */ + 96, + "DH-768", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A63A3620FFFFFFFFFFFFFFFF" +}, +#endif +#ifdef LTC_DH1024 +{ /* 1024-bit MODP Group 2 - https://tools.ietf.org/html/rfc7296#appendix-B.2 */ + 128, + "DH-1024", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" + "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381" + "FFFFFFFFFFFFFFFF" +}, +#endif +#ifdef LTC_DH1536 +{ /* 1536-bit MODP Group 5 - https://tools.ietf.org/html/rfc3526#section-2 */ + 192, + "DH-1536", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" + "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" + "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" + "83655D23DCA3AD961C62F356208552BB9ED529077096966D" + "670C354E4ABC9804F1746C08CA237327FFFFFFFFFFFFFFFF" +}, +#endif +#ifdef LTC_DH2048 +{ /* 2048-bit MODP Group 14 - https://tools.ietf.org/html/rfc3526#section-3 */ + 256, + "DH-2048", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" + "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" + "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" + "83655D23DCA3AD961C62F356208552BB9ED529077096966D" + "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" + "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" + "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" + "15728E5A8AACAA68FFFFFFFFFFFFFFFF" +}, +#endif +#ifdef LTC_DH3072 +{ /* 3072-bit MODP Group 15 - https://tools.ietf.org/html/rfc3526#section-4 */ + 384, + "DH-3072", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" + "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" + "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" + "83655D23DCA3AD961C62F356208552BB9ED529077096966D" + "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" + "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" + "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" + "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" + "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" + "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" + "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" + "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" + "43DB5BFCE0FD108E4B82D120A93AD2CAFFFFFFFFFFFFFFFF" +}, +#endif +#ifdef LTC_DH4096 +{ /* 4096-bit MODP Group 16 - https://tools.ietf.org/html/rfc3526#section-5 */ + 512, + "DH-4096", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" + "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" + "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" + "83655D23DCA3AD961C62F356208552BB9ED529077096966D" + "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" + "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" + "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" + "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" + "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" + "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" + "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" + "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" + "43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7" + "88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA" + "2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6" + "287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED" + "1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9" + "93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934063199" + "FFFFFFFFFFFFFFFF" +}, +#endif +#ifdef LTC_DH6144 +{ /* 6144-bit MODP Group 17 - https://tools.ietf.org/html/rfc3526#section-6 */ + 768, + "DH-6144", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" + "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" + "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" + "83655D23DCA3AD961C62F356208552BB9ED529077096966D" + "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" + "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" + "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" + "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" + "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" + "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" + "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" + "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" + "43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7" + "88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA" + "2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6" + "287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED" + "1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9" + "93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934028492" + "36C3FAB4D27C7026C1D4DCB2602646DEC9751E763DBA37BD" + "F8FF9406AD9E530EE5DB382F413001AEB06A53ED9027D831" + "179727B0865A8918DA3EDBEBCF9B14ED44CE6CBACED4BB1B" + "DB7F1447E6CC254B332051512BD7AF426FB8F401378CD2BF" + "5983CA01C64B92ECF032EA15D1721D03F482D7CE6E74FEF6" + "D55E702F46980C82B5A84031900B1C9E59E7C97FBEC7E8F3" + "23A97A7E36CC88BE0F1D45B7FF585AC54BD407B22B4154AA" + "CC8F6D7EBF48E1D814CC5ED20F8037E0A79715EEF29BE328" + "06A1D58BB7C5DA76F550AA3D8A1FBFF0EB19CCB1A313D55C" + "DA56C9EC2EF29632387FE8D76E3C0468043E8F663F4860EE" + "12BF2D5B0B7474D6E694F91E6DCC4024FFFFFFFFFFFFFFFF" +}, +#endif +#ifdef LTC_DH8192 +{ /* 8192-bit MODP Group 18 - https://tools.ietf.org/html/rfc3526#section-7 */ + 1024, + "DH-8192", + "2", + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" + "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" + "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" + "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" + "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" + "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" + "83655D23DCA3AD961C62F356208552BB9ED529077096966D" + "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" + "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" + "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" + "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" + "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" + "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" + "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" + "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" + "43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7" + "88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA" + "2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6" + "287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED" + "1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9" + "93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934028492" + "36C3FAB4D27C7026C1D4DCB2602646DEC9751E763DBA37BD" + "F8FF9406AD9E530EE5DB382F413001AEB06A53ED9027D831" + "179727B0865A8918DA3EDBEBCF9B14ED44CE6CBACED4BB1B" + "DB7F1447E6CC254B332051512BD7AF426FB8F401378CD2BF" + "5983CA01C64B92ECF032EA15D1721D03F482D7CE6E74FEF6" + "D55E702F46980C82B5A84031900B1C9E59E7C97FBEC7E8F3" + "23A97A7E36CC88BE0F1D45B7FF585AC54BD407B22B4154AA" + "CC8F6D7EBF48E1D814CC5ED20F8037E0A79715EEF29BE328" + "06A1D58BB7C5DA76F550AA3D8A1FBFF0EB19CCB1A313D55C" + "DA56C9EC2EF29632387FE8D76E3C0468043E8F663F4860EE" + "12BF2D5B0B7474D6E694F91E6DBE115974A3926F12FEE5E4" + "38777CB6A932DF8CD8BEC4D073B931BA3BC832B68D9DD300" + "741FA7BF8AFC47ED2576F6936BA424663AAB639C5AE4F568" + "3423B4742BF1C978238F16CBE39D652DE3FDB8BEFC848AD9" + "22222E04A4037C0713EB57A81A23F0C73473FC646CEA306B" + "4BCBC8862F8385DDFA9D4B7FA2C087E879683303ED5BDD3A" + "062B3CF5B3A278A66D2A13F83F44F82DDF310EE074AB6A36" + "4597E899A0255DC164F31CC50846851DF9AB48195DED7EA1" + "B1D510BD7EE74D73FAF36BC31ECFA268359046F4EB879F92" + "4009438B481C6CD7889A002ED5EE382BC9190DA6FC026E47" + "9558E4475677E9AA9E3050E2765694DFC81F56E880B96E71" + "60C980DD98EDD3DFFFFFFFFFFFFFFFFF" +}, +#endif +{ + 0, + NULL, + NULL, + NULL +} +}; + +/** + Returns the DH group size (octets) for given key + @param key The DH key to get the size of + @return The group size in octets (0 on error) + */ +int dh_get_groupsize(const dh_key *key) +{ + if (key == NULL) return 0; + return mp_unsigned_bin_size(key->prime); +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_check_pubkey.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_check_pubkey.c new file mode 100644 index 0000000..7dd128e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_check_pubkey.c @@ -0,0 +1,55 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Check DH public key (INTERNAL ONLY, not part of public API) + @param key The key you wish to test + @return CRYPT_OK if successful +*/ +int dh_check_pubkey(const dh_key *key) +{ + void *p_minus1; + ltc_mp_digit digit; + int i, digit_count, bits_set = 0, err; + + LTC_ARGCHK(key != NULL); + + if ((err = mp_init(&p_minus1)) != CRYPT_OK) { + return err; + } + + /* avoid: y <= 1 OR y >= p-1 */ + if ((err = mp_sub_d(key->prime, 1, p_minus1)) != CRYPT_OK) { + goto error; + } + if (mp_cmp(key->y, p_minus1) != LTC_MP_LT || mp_cmp_d(key->y, 1) != LTC_MP_GT) { + err = CRYPT_INVALID_ARG; + goto error; + } + + /* public key must have more than one bit set */ + digit_count = mp_get_digit_count(key->y); + for (i = 0; i < digit_count && bits_set < 2; i++) { + digit = mp_get_digit(key->y, i); + while (digit > 0) { + if (digit & 1) bits_set++; + digit >>= 1; + } + } + if (bits_set > 1) { + err = CRYPT_OK; + } + else { + err = CRYPT_INVALID_ARG; + } + +error: + mp_clear(p_minus1); + return err; +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_export.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_export.c new file mode 100644 index 0000000..19112c4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_export.c @@ -0,0 +1,52 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Export a DH key to a binary packet + @param out [out] The destination for the key + @param outlen [in/out] The max size and resulting size of the DH key + @param type Which type of key (PK_PRIVATE or PK_PUBLIC) + @param key The key you wish to export + @return CRYPT_OK if successful +*/ +int dh_export(unsigned char *out, unsigned long *outlen, int type, const dh_key *key) +{ + unsigned char flags[1]; + int err; + unsigned long version = 0; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + if (type == PK_PRIVATE) { + /* export x - private key */ + flags[0] = 1; + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &version, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->prime, + LTC_ASN1_INTEGER, 1UL, key->base, + LTC_ASN1_INTEGER, 1UL, key->x, + LTC_ASN1_EOL, 0UL, NULL); + } + else { + /* export y - public key */ + flags[0] = 0; + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &version, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->prime, + LTC_ASN1_INTEGER, 1UL, key->base, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_EOL, 0UL, NULL); + } + + return err; +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_export_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_export_key.c new file mode 100644 index 0000000..374635c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_export_key.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Binary export a DH key to a buffer + @param out [out] The destination for the key + @param outlen [in/out] The max size and resulting size of the DH key + @param type Which type of key (PK_PRIVATE or PK_PUBLIC) + @param key The key you wish to export + @return CRYPT_OK if successful +*/ +int dh_export_key(void *out, unsigned long *outlen, int type, const dh_key *key) +{ + unsigned long len; + void *k; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + k = (type == PK_PRIVATE) ? key->x : key->y; + len = mp_unsigned_bin_size(k); + + if (*outlen < len) { + *outlen = len; + return CRYPT_BUFFER_OVERFLOW; + } + *outlen = len; + + return mp_to_unsigned_bin(k, out); +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_free.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_free.c new file mode 100644 index 0000000..ff56b00 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_free.c @@ -0,0 +1,18 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Free the allocated ram for a DH key + @param key The key which you wish to free +*/ +void dh_free(dh_key *key) +{ + LTC_ARGCHKVD(key != NULL); + mp_cleanup_multi(&key->prime, &key->base, &key->y, &key->x, LTC_NULL); +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_generate_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_generate_key.c new file mode 100644 index 0000000..c28246f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_generate_key.c @@ -0,0 +1,90 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +static int s_dh_groupsize_to_keysize(int groupsize) +{ + /* The strength estimates from https://tools.ietf.org/html/rfc3526#section-8 + * We use "Estimate 2" to get an appropriate private key (exponent) size. + */ + if (groupsize <= 0) { + return 0; + } + if (groupsize <= 192) { + return 30; /* 1536-bit => key size 240-bit */ + } + if (groupsize <= 256) { + return 40; /* 2048-bit => key size 320-bit */ + } + if (groupsize <= 384) { + return 52; /* 3072-bit => key size 416-bit */ + } + if (groupsize <= 512) { + return 60; /* 4096-bit => key size 480-bit */ + } + if (groupsize <= 768) { + return 67; /* 6144-bit => key size 536-bit */ + } + if (groupsize <= 1024) { + return 77; /* 8192-bit => key size 616-bit */ + } + return 0; +} + +int dh_generate_key(prng_state *prng, int wprng, dh_key *key) +{ + unsigned char *buf; + unsigned long keysize; + int err, max_iterations = LTC_PK_MAX_RETRIES; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* good prng? */ + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + + keysize = s_dh_groupsize_to_keysize(mp_unsigned_bin_size(key->prime)); + if (keysize == 0) { + err = CRYPT_INVALID_KEYSIZE; + goto freemp; + } + + /* allocate buffer */ + buf = XMALLOC(keysize); + if (buf == NULL) { + err = CRYPT_MEM; + goto freemp; + } + + key->type = PK_PRIVATE; + do { + /* make up random buf */ + if (prng_descriptor[wprng]->read(buf, keysize, prng) != keysize) { + err = CRYPT_ERROR_READPRNG; + goto freebuf; + } + /* load the x value - private key */ + if ((err = mp_read_unsigned_bin(key->x, buf, keysize)) != CRYPT_OK) { + goto freebuf; + } + /* compute the y value - public key */ + if ((err = mp_exptmod(key->base, key->x, key->prime, key->y)) != CRYPT_OK) { + goto freebuf; + } + err = dh_check_pubkey(key); + } while (err != CRYPT_OK && max_iterations-- > 0); + +freebuf: + zeromem(buf, keysize); + XFREE(buf); +freemp: + if (err != CRYPT_OK) dh_free(key); + return err; +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_import.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_import.c new file mode 100644 index 0000000..a067a32 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_import.c @@ -0,0 +1,89 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Import a DH key from a binary packet + @param in The packet to read + @param inlen The length of the input packet + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int dh_import(const unsigned char *in, unsigned long inlen, dh_key *key) +{ + unsigned char flags[1]; + int err; + unsigned long version; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + + /* init */ + if ((err = mp_init_multi(&key->x, &key->y, &key->base, &key->prime, LTC_NULL)) != CRYPT_OK) { + return err; + } + + /* find out what type of key it is */ + err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &version, + LTC_ASN1_BIT_STRING, 1UL, &flags, + LTC_ASN1_EOL, 0UL, NULL); + if (err != CRYPT_OK && err != CRYPT_INPUT_TOO_LONG) { + goto error; + } + + if (version == 0) { + if (flags[0] == 1) { + key->type = PK_PRIVATE; + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &version, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->prime, + LTC_ASN1_INTEGER, 1UL, key->base, + LTC_ASN1_INTEGER, 1UL, key->x, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto error; + } + /* compute public key: y = (base ^ x) mod prime */ + if ((err = mp_exptmod(key->base, key->x, key->prime, key->y)) != CRYPT_OK) { + goto error; + } + } + else if (flags[0] == 0) { + key->type = PK_PUBLIC; + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &version, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->prime, + LTC_ASN1_INTEGER, 1UL, key->base, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto error; + } + } + else { + err = CRYPT_INVALID_PACKET; + goto error; + } + } + else { + err = CRYPT_INVALID_PACKET; + goto error; + } + + /* check public key */ + if ((err = dh_check_pubkey(key)) != CRYPT_OK) { + goto error; + } + + return CRYPT_OK; + +error: + dh_free(key); + return err; +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_make_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_make_key.c new file mode 100644 index 0000000..e122985 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_make_key.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2001-2007, Tom St Denis + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tomcrypt_private.h" +#include + +/* + * Make a DH key [private key pair] + * @param prng An active PRNG state + * @param wprng The index for the PRNG you desire to use + * @param keysize The key size (octets) desired of the private key + * @param q If not null, then the private key is in the range + * [2, q-2] where q is called the subprime + * @param xbits If not 0, then the private key has 'xbits' bits + * @note The private key must always be less than p-1 + * @param key [in/out] Where the newly created DH key will be stored + * g and p are provided as input in the key + * type, x and y are output of this function + * @return CRYPT_OK if successful, note: on error all allocated memory will be + * freed automatically. +*/ + +int dh_make_key(prng_state *prng, int wprng, void *q, int xbits, dh_key *key) +{ + int err = 0; + int key_size = 0; /* max key size, in bytes */ + int key_size_p = 0; /* key size of p */ + int key_size_q = 0; /* key size of p */ + void *arg_mod = 0; + uint8_t *buf = 0; /* intermediate buffer to have a raw random */ + + /* + * Check the arguments + */ + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(key->base != NULL); + LTC_ARGCHK(key->prime != NULL); + err = prng_is_valid(wprng); + if (err != CRYPT_OK) + return err; + + /* + * Set the key size and check constraints + */ + if (xbits) { + LTC_ARGCHK((xbits % 8) == 0); + key_size = xbits / 8; + } + key_size_p = mp_unsigned_bin_size(key->prime); + if (q) + key_size_q = mp_unsigned_bin_size(q); + if (key_size) { + /* check the constraints */ + LTC_ARGCHK(key_size <= key_size_p); + LTC_ARGCHK((q == NULL) || (key_size <= key_size_q)); + } else { + if (q) + key_size = MIN(key_size_p, key_size_q); + else + key_size =key_size_p; + } + + /* Set the argument we will make the modulo against to */ + if ((q != NULL) && (key_size_q < key_size_p)) + arg_mod = q; + else + arg_mod = key->prime; + + /* initialize the key */ + key->x = NULL; + key->y = NULL; + err = mp_init_multi(&key->x, &key->y, NULL); + if (err != CRYPT_OK) + goto error; + + /* Initialize the buffer used to store the random number */ + buf = XMALLOC(key_size); + if (buf == NULL) { + err = CRYPT_MEM; + goto error; + } + + /* generate the private key in a raw-buffer */ + if (prng_descriptor[wprng]->read(buf, key_size, prng) != + (unsigned long)key_size) { + err = CRYPT_ERROR_READPRNG; + goto error; + } + + /* + * Transform it as a Big Number compatible with p and q + */ + err = mp_read_unsigned_bin(key->y, buf, key_size); + if (err != CRYPT_OK) + goto error; + err = mp_mod(key->y, arg_mod, key->x); + if (err != CRYPT_OK) + goto error; + + /* generate the public key key->y */ + err = mp_exptmod(key->base, key->x, key->prime, key->y); + if (err != CRYPT_OK) + goto error; + + /* no error */ + err = CRYPT_OK; + +error: + if (err != CRYPT_OK) + mp_clear_multi(key->x, key->y, NULL); + if (buf) + XFREE(buf); + + return err; +} diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_set.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_set.c new file mode 100644 index 0000000..e25db08 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_set.c @@ -0,0 +1,114 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Import DH key parts p and g from raw numbers + + @param p DH's p (prime) + @param plen DH's p's length + @param g DH's g (group) + @param glen DH's g's length + @param key [out] the destination for the imported key + @return CRYPT_OK if successful +*/ +int dh_set_pg(const unsigned char *p, unsigned long plen, + const unsigned char *g, unsigned long glen, + dh_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(p != NULL); + LTC_ARGCHK(g != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + if ((err = mp_init_multi(&key->x, &key->y, &key->base, &key->prime, LTC_NULL)) != CRYPT_OK) { + return err; + } + + if ((err = mp_read_unsigned_bin(key->base, (unsigned char*)g, glen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_unsigned_bin(key->prime, (unsigned char*)p, plen)) != CRYPT_OK) { goto LBL_ERR; } + + return CRYPT_OK; + +LBL_ERR: + dh_free(key); + return err; +} + +/** + Import DH key parts p and g from built-in DH groups + + @param groupsize The size of the DH group to use + @param key [out] Where the newly created DH key will be stored + @return CRYPT_OK if successful, note: on error all allocated memory will be freed automatically. +*/ +int dh_set_pg_groupsize(int groupsize, dh_key *key) +{ + int err, i; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + LTC_ARGCHK(groupsize > 0); + + for (i = 0; (groupsize > ltc_dh_sets[i].size) && (ltc_dh_sets[i].size != 0); i++); + if (ltc_dh_sets[i].size == 0) return CRYPT_INVALID_KEYSIZE; + + if ((err = mp_init_multi(&key->x, &key->y, &key->base, &key->prime, LTC_NULL)) != CRYPT_OK) { + return err; + } + if ((err = mp_read_radix(key->base, ltc_dh_sets[i].base, 16)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_radix(key->prime, ltc_dh_sets[i].prime, 16)) != CRYPT_OK) { goto LBL_ERR; } + + return CRYPT_OK; + +LBL_ERR: + dh_free(key); + return err; +} + +/** + Import DH public or private key part from raw numbers + + NB: The p & g parts must be set beforehand + + @param in The key-part to import, either public or private. + @param inlen The key-part's length + @param type Which type of key (PK_PRIVATE or PK_PUBLIC) + @param key [out] the destination for the imported key + @return CRYPT_OK if successful +*/ +int dh_set_key(const unsigned char *in, unsigned long inlen, int type, dh_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + if (type == PK_PRIVATE) { + key->type = PK_PRIVATE; + if ((err = mp_read_unsigned_bin(key->x, (unsigned char*)in, inlen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_exptmod(key->base, key->x, key->prime, key->y)) != CRYPT_OK) { goto LBL_ERR; } + } + else { + key->type = PK_PUBLIC; + if ((err = mp_read_unsigned_bin(key->y, (unsigned char*)in, inlen)) != CRYPT_OK) { goto LBL_ERR; } + } + + /* check public key */ + if ((err = dh_check_pubkey(key)) != CRYPT_OK) { + goto LBL_ERR; + } + + return CRYPT_OK; + +LBL_ERR: + dh_free(key); + return err; +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_set_pg_dhparam.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_set_pg_dhparam.c new file mode 100644 index 0000000..1f2fb3b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_set_pg_dhparam.c @@ -0,0 +1,44 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Import DH key parts p and g from dhparam + + dhparam data: openssl dhparam -outform DER -out dhparam.der 2048 + + @param dhparam The DH param DER encoded data + @param dhparamlen The length of dhparam data + @param key [out] Where the newly created DH key will be stored + @return CRYPT_OK if successful, note: on error all allocated memory will be freed automatically. +*/ +int dh_set_pg_dhparam(const unsigned char *dhparam, unsigned long dhparamlen, dh_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + LTC_ARGCHK(dhparam != NULL); + LTC_ARGCHK(dhparamlen > 0); + + if ((err = mp_init_multi(&key->x, &key->y, &key->base, &key->prime, LTC_NULL)) != CRYPT_OK) { + return err; + } + if ((err = der_decode_sequence_multi(dhparam, dhparamlen, + LTC_ASN1_INTEGER, 1UL, key->prime, + LTC_ASN1_INTEGER, 1UL, key->base, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + + return CRYPT_OK; + +LBL_ERR: + dh_free(key); + return err; +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_shared_secret.c b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_shared_secret.c new file mode 100644 index 0000000..0c4dfc9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/dh_shared_secret.c @@ -0,0 +1,70 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MDH + +/** + Create a DH shared secret. + @param private_key The private DH key in the pair + @param public_key The public DH key in the pair + @param out [out] The destination of the shared data + @param outlen [in/out] The max size and resulting size of the shared data. + @return CRYPT_OK if successful +*/ +int dh_shared_secret(const dh_key *private_key, const dh_key *public_key, + unsigned char *out, unsigned long *outlen) +{ + void *tmp; + unsigned long x; + int err; + + LTC_ARGCHK(private_key != NULL); + LTC_ARGCHK(public_key != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* types valid? */ + if (private_key->type != PK_PRIVATE) { + return CRYPT_PK_NOT_PRIVATE; + } + + /* same DH group? */ + if (mp_cmp(private_key->prime, public_key->prime) != LTC_MP_EQ) { return CRYPT_PK_TYPE_MISMATCH; } + if (mp_cmp(private_key->base, public_key->base) != LTC_MP_EQ) { return CRYPT_PK_TYPE_MISMATCH; } + + /* init big numbers */ + if ((err = mp_init(&tmp)) != CRYPT_OK) { + return err; + } + + /* check public key */ + if ((err = dh_check_pubkey(public_key)) != CRYPT_OK) { + goto error; + } + + /* compute tmp = y^x mod p */ + if ((err = mp_exptmod(public_key->y, private_key->x, private_key->prime, tmp)) != CRYPT_OK) { + goto error; + } + + /* enough space for output? */ + x = (unsigned long)mp_unsigned_bin_size(tmp); + if (*outlen < x) { + *outlen = x; + err = CRYPT_BUFFER_OVERFLOW; + goto error; + } + if ((err = mp_to_unsigned_bin(tmp, out)) != CRYPT_OK) { + goto error; + } + *outlen = x; + err = CRYPT_OK; + +error: + mp_clear(tmp); + return err; +} + +#endif /* LTC_MDH */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dh/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/dh/sub.mk new file mode 100644 index 0000000..86f2d76 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dh/sub.mk @@ -0,0 +1,13 @@ +srcs-y += dh.c +cflags-dh.c-y += -Wno-unused-variable +srcs-y += dh_check_pubkey.c +srcs-y += dh_export.c +srcs-y += dh_export_key.c +srcs-y += dh_free.c +srcs-y += dh_generate_key.c +srcs-y += dh_import.c +srcs-y += dh_make_key.c +srcs-y += dh_set.c +srcs-y += dh_set_pg_dhparam.c +srcs-y += dh_shared_secret.c + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_decrypt_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_decrypt_key.c new file mode 100644 index 0000000..c897023 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_decrypt_key.c @@ -0,0 +1,129 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_decrypt_key.c + DSA Crypto, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Decrypt an DSA encrypted key + @param in The ciphertext + @param inlen The length of the ciphertext (octets) + @param out [out] The plaintext + @param outlen [in/out] The max size and resulting size of the plaintext + @param key The corresponding private DSA key + @return CRYPT_OK if successful +*/ +int dsa_decrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const dsa_key *key) +{ + unsigned char *skey, *expt; + void *g_pub; + unsigned long x, y; + unsigned long hashOID[32] = { 0 }; + int hash, err; + ltc_asn1_list decode[3]; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* right key type? */ + if (key->type != PK_PRIVATE) { + return CRYPT_PK_NOT_PRIVATE; + } + + /* decode to find out hash */ + LTC_SET_ASN1(decode, 0, LTC_ASN1_OBJECT_IDENTIFIER, hashOID, sizeof(hashOID)/sizeof(hashOID[0])); + err = der_decode_sequence(in, inlen, decode, 1); + if (err != CRYPT_OK && err != CRYPT_INPUT_TOO_LONG) { + return err; + } + + hash = find_hash_oid(hashOID, decode[0].size); + if (hash_is_valid(hash) != CRYPT_OK) { + return CRYPT_INVALID_PACKET; + } + + /* we now have the hash! */ + + if ((err = mp_init(&g_pub)) != CRYPT_OK) { + return err; + } + + /* allocate memory */ + expt = XMALLOC(mp_unsigned_bin_size(key->p) + 1); + skey = XMALLOC(MAXBLOCKSIZE); + if (expt == NULL || skey == NULL) { + if (expt != NULL) { + XFREE(expt); + } + if (skey != NULL) { + XFREE(skey); + } + mp_clear(g_pub); + return CRYPT_MEM; + } + + LTC_SET_ASN1(decode, 1, LTC_ASN1_INTEGER, g_pub, 1UL); + LTC_SET_ASN1(decode, 2, LTC_ASN1_OCTET_STRING, skey, MAXBLOCKSIZE); + + /* read the structure in now */ + if ((err = der_decode_sequence(in, inlen, decode, 3)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* make shared key */ + x = mp_unsigned_bin_size(key->p) + 1; + if ((err = dsa_shared_secret(key->x, g_pub, key, expt, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + + y = mp_unsigned_bin_size(key->p) + 1; + y = MIN(y, MAXBLOCKSIZE); + if ((err = hash_memory(hash, expt, x, expt, &y)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* ensure the hash of the shared secret is at least as big as the encrypt itself */ + if (decode[2].size > y) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + /* avoid buffer overflow */ + if (*outlen < decode[2].size) { + *outlen = decode[2].size; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + /* Decrypt the key */ + for (x = 0; x < decode[2].size; x++) { + out[x] = expt[x] ^ skey[x]; + } + *outlen = x; + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(expt, mp_unsigned_bin_size(key->p) + 1); + zeromem(skey, MAXBLOCKSIZE); +#endif + + XFREE(expt); + XFREE(skey); + + mp_clear(g_pub); + + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_encrypt_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_encrypt_key.c new file mode 100644 index 0000000..eb49d1b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_encrypt_key.c @@ -0,0 +1,118 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_encrypt_key.c + DSA Crypto, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Encrypt a symmetric key with DSA + @param in The symmetric key you want to encrypt + @param inlen The length of the key to encrypt (octets) + @param out [out] The destination for the ciphertext + @param outlen [in/out] The max size and resulting size of the ciphertext + @param prng An active PRNG state + @param wprng The index of the PRNG you wish to use + @param hash The index of the hash you want to use + @param key The DSA key you want to encrypt to + @return CRYPT_OK if successful +*/ +int dsa_encrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, int hash, + const dsa_key *key) +{ + unsigned char *expt, *skey; + void *g_pub, *g_priv; + unsigned long x, y; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* check that wprng/cipher/hash are not invalid */ + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + + if (inlen > hash_descriptor[hash]->hashsize) { + return CRYPT_INVALID_HASH; + } + + /* make a random key and export the public copy */ + if ((err = mp_init_multi(&g_pub, &g_priv, LTC_NULL)) != CRYPT_OK) { + return err; + } + + expt = XMALLOC(mp_unsigned_bin_size(key->p) + 1); + skey = XMALLOC(MAXBLOCKSIZE); + if (expt == NULL || skey == NULL) { + if (expt != NULL) { + XFREE(expt); + } + if (skey != NULL) { + XFREE(skey); + } + mp_clear_multi(g_pub, g_priv, LTC_NULL); + return CRYPT_MEM; + } + + /* make a random g_priv, g_pub = g^x pair + private key x should be in range: 1 <= x <= q-1 (see FIPS 186-4 B.1.2) + */ + if ((err = rand_bn_upto(g_priv, key->q, prng, wprng)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* compute y */ + if ((err = mp_exptmod(key->g, g_priv, key->p, g_pub)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* make random key */ + x = mp_unsigned_bin_size(key->p) + 1; + if ((err = dsa_shared_secret(g_priv, key->y, key, expt, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + + y = MAXBLOCKSIZE; + if ((err = hash_memory(hash, expt, x, skey, &y)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* Encrypt key */ + for (x = 0; x < inlen; x++) { + skey[x] ^= in[x]; + } + + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_OBJECT_IDENTIFIER, hash_descriptor[hash]->OIDlen, hash_descriptor[hash]->OID, + LTC_ASN1_INTEGER, 1UL, g_pub, + LTC_ASN1_OCTET_STRING, inlen, skey, + LTC_ASN1_EOL, 0UL, NULL); + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + /* clean up */ + zeromem(expt, mp_unsigned_bin_size(key->p) + 1); + zeromem(skey, MAXBLOCKSIZE); +#endif + + XFREE(skey); + XFREE(expt); + + mp_clear_multi(g_pub, g_priv, LTC_NULL); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_export.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_export.c new file mode 100644 index 0000000..3550cbe --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_export.c @@ -0,0 +1,100 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_export.c + DSA implementation, export key, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Export a DSA key to a binary packet + @param out [out] Where to store the packet + @param outlen [in/out] The max size and resulting size of the packet + @param type The type of key to export (PK_PRIVATE or PK_PUBLIC) + @param key The key to export + @return CRYPT_OK if successful +*/ +int dsa_export(unsigned char *out, unsigned long *outlen, int type, const dsa_key *key) +{ + unsigned long zero=0; + unsigned char flags[1]; + int err, std; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + std = type & PK_STD; + type &= ~PK_STD; + + if (type == PK_PRIVATE && key->type != PK_PRIVATE) { + return CRYPT_PK_TYPE_MISMATCH; + } + + if (type == PK_PRIVATE) { + if (std) { + return der_encode_sequence_multi(out, outlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &zero, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->g, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_INTEGER, 1UL, key->x, + LTC_ASN1_EOL, 0UL, NULL); + } + flags[0] = 1; + return der_encode_sequence_multi(out, outlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->g, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_INTEGER, 1UL, key->x, + LTC_ASN1_EOL, 0UL, NULL); + } + + if (type == PK_PUBLIC) { + if (std) { + unsigned long tmplen = (unsigned long)(mp_count_bits(key->y) / 8) + 8; + unsigned char* tmp = XMALLOC(tmplen); + ltc_asn1_list int_list[3]; + + if (tmp == NULL) { + return CRYPT_MEM; + } + + err = der_encode_integer(key->y, tmp, &tmplen); + if (err != CRYPT_OK) { + goto error; + } + + LTC_SET_ASN1(int_list, 0, LTC_ASN1_INTEGER, key->p, 1UL); + LTC_SET_ASN1(int_list, 1, LTC_ASN1_INTEGER, key->q, 1UL); + LTC_SET_ASN1(int_list, 2, LTC_ASN1_INTEGER, key->g, 1UL); + + err = x509_encode_subject_public_key_info(out, outlen, LTC_OID_DSA, tmp, + tmplen, LTC_ASN1_SEQUENCE, int_list, + sizeof(int_list) / sizeof(int_list[0])); + +error: + XFREE(tmp); + return err; + } + flags[0] = 0; + return der_encode_sequence_multi(out, outlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->g, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_EOL, 0UL, NULL); + } + + return CRYPT_INVALID_ARG; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_free.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_free.c new file mode 100644 index 0000000..dbe8625 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_free.c @@ -0,0 +1,23 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_free.c + DSA implementation, free a DSA key, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Free a DSA key + @param key The key to free from memory +*/ +void dsa_free(dsa_key *key) +{ + LTC_ARGCHKVD(key != NULL); + mp_cleanup_multi(&key->y, &key->x, &key->q, &key->g, &key->p, LTC_NULL); + key->type = key->qord = 0; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_generate_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_generate_key.c new file mode 100644 index 0000000..bc83c0e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_generate_key.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_make_key.c + DSA implementation, generate a DSA key +*/ + +#ifdef LTC_MDSA + +/** + Create a DSA key + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param key [in/out] Where to store the created key + @return CRYPT_OK if successful. +*/ +int dsa_generate_key(prng_state *prng, int wprng, dsa_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* so now we have our DH structure, generator g, order q, modulus p + Now we need a random exponent [mod q] and it's power g^x mod p + */ + /* private key x should be from range: 1 <= x <= q-1 (see FIPS 186-4 B.1.2) */ + if ((err = rand_bn_upto(key->x, key->q, prng, wprng)) != CRYPT_OK) { return err; } + if ((err = mp_exptmod(key->g, key->x, key->p, key->y)) != CRYPT_OK) { return err; } + key->type = PK_PRIVATE; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_generate_pqg.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_generate_pqg.c new file mode 100644 index 0000000..954362e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_generate_pqg.c @@ -0,0 +1,234 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_generate_pqg.c + DSA implementation - generate DSA parameters p, q & g +*/ + +#ifdef LTC_MDSA + +/** + Create DSA parameters (INTERNAL ONLY, not part of public API) + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param group_size Size of the multiplicative group (octets) + @param modulus_size Size of the modulus (octets) + @param p [out] bignum where generated 'p' is stored (must be initialized by caller) + @param q [out] bignum where generated 'q' is stored (must be initialized by caller) + @param g [out] bignum where generated 'g' is stored (must be initialized by caller) + @return CRYPT_OK if successful, upon error this function will free all allocated memory +*/ +static int s_dsa_make_params(prng_state *prng, int wprng, int group_size, int modulus_size, void *p, void *q, void *g) +{ + unsigned long L, N, n, outbytes, seedbytes, counter, j, i; + int err, res, mr_tests_q, mr_tests_p, found_p, found_q, hash; + unsigned char *wbuf, *sbuf, digest[MAXBLOCKSIZE]; + void *t2L1, *t2N1, *t2q, *t2seedlen, *U, *W, *X, *c, *h, *e, *seedinc; + const char *accepted_hashes[] = { "sha3-512", "sha512", "sha3-384", "sha384", "sha3-256", "sha256" }; + + /* check size */ + if (group_size > LTC_MDSA_MAX_GROUP || group_size < 1 || group_size >= modulus_size || modulus_size > LTC_MDSA_MAX_MODULUS) { + return CRYPT_INVALID_ARG; + } + + /* FIPS-186-4 A.1.1.2 Generation of the Probable Primes p and q Using an Approved Hash Function + * + * L = The desired length of the prime p (in bits e.g. L = 1024) + * N = The desired length of the prime q (in bits e.g. N = 160) + * seedlen = The desired bit length of the domain parameter seed; seedlen shallbe equal to or greater than N + * outlen = The bit length of Hash function + * + * 1. Check that the (L, N) + * 2. If (seedlen = 2^(L-1)) { + * Test whether or not p is prime as specified in Appendix C.3. + * If p is determined to be prime, then return VALID and the values of p, qand (optionally) the values of domain_parameter_seed and counter + * } + * offset = offset + n + 1 Comment: Increment offset + * } + */ + + seedbytes = group_size; + L = (unsigned long)modulus_size * 8; + N = (unsigned long)group_size * 8; + + /* XXX-TODO no Lucas test */ +#ifdef LTC_MPI_HAS_LUCAS_TEST + /* M-R tests (when followed by one Lucas test) according FIPS-186-4 - Appendix C.3 - table C.1 */ + mr_tests_p = (L <= 2048) ? 3 : 2; + if (N <= 160) { mr_tests_q = 19; } + else if (N <= 224) { mr_tests_q = 24; } + else { mr_tests_q = 27; } +#else + /* M-R tests (without Lucas test) according FIPS-186-4 - Appendix C.3 - table C.1 */ + if (L <= 1024) { mr_tests_p = 40; } + else if (L <= 2048) { mr_tests_p = 56; } + else { mr_tests_p = 64; } + + if (N <= 160) { mr_tests_q = 40; } + else if (N <= 224) { mr_tests_q = 56; } + else { mr_tests_q = 64; } +#endif + + hash = -1; + for (i = 0; i < sizeof(accepted_hashes)/sizeof(accepted_hashes[0]); ++i) { + hash = find_hash(accepted_hashes[i]); + if (hash != -1) break; + } + if (hash == -1) { + return CRYPT_INVALID_ARG; /* no appropriate hash function found */ + } + if (N > hash_descriptor[hash]->hashsize * 8) { + return CRYPT_INVALID_ARG; /* group_size too big */ + } + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { return err; } + outbytes = hash_descriptor[hash]->hashsize; + + n = ((L + outbytes*8 - 1) / (outbytes*8)) - 1; + + if ((wbuf = XMALLOC((n+1)*outbytes)) == NULL) { err = CRYPT_MEM; goto cleanup3; } + if ((sbuf = XMALLOC(seedbytes)) == NULL) { err = CRYPT_MEM; goto cleanup2; } + + err = mp_init_multi(&t2L1, &t2N1, &t2q, &t2seedlen, &U, &W, &X, &c, &h, &e, &seedinc, LTC_NULL); + if (err != CRYPT_OK) { goto cleanup1; } + + if ((err = mp_2expt(t2L1, L-1)) != CRYPT_OK) { goto cleanup; } + /* t2L1 = 2^(L-1) */ + if ((err = mp_2expt(t2N1, N-1)) != CRYPT_OK) { goto cleanup; } + /* t2N1 = 2^(N-1) */ + if ((err = mp_2expt(t2seedlen, seedbytes*8)) != CRYPT_OK) { goto cleanup; } + /* t2seedlen = 2^seedlen */ + + for(found_p=0; !found_p;) { + /* q */ + for(found_q=0; !found_q;) { + if (prng_descriptor[wprng]->read(sbuf, seedbytes, prng) != seedbytes) { err = CRYPT_ERROR_READPRNG; goto cleanup; } + i = outbytes; + if ((err = hash_memory(hash, sbuf, seedbytes, digest, &i)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_read_unsigned_bin(U, digest, outbytes)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_mod(U, t2N1, U)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_add(t2N1, U, q)) != CRYPT_OK) { goto cleanup; } + if (!mp_isodd(q)) mp_add_d(q, 1, q); + if ((err = mp_prime_is_prime(q, mr_tests_q, &res)) != CRYPT_OK) { goto cleanup; } + if (res == LTC_MP_YES) found_q = 1; + } + + /* p */ + if ((err = mp_read_unsigned_bin(seedinc, sbuf, seedbytes)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_add(q, q, t2q)) != CRYPT_OK) { goto cleanup; } + for(counter=0; counter < 4*L && !found_p; counter++) { + for(j=0; j<=n; j++) { + if ((err = mp_add_d(seedinc, 1, seedinc)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_mod(seedinc, t2seedlen, seedinc)) != CRYPT_OK) { goto cleanup; } + /* seedinc = (seedinc+1) % 2^seed_bitlen */ + if ((i = mp_unsigned_bin_size(seedinc)) > seedbytes) { err = CRYPT_INVALID_ARG; goto cleanup; } + zeromem(sbuf, seedbytes); + if ((err = mp_to_unsigned_bin(seedinc, sbuf + seedbytes-i)) != CRYPT_OK) { goto cleanup; } + i = outbytes; + err = hash_memory(hash, sbuf, seedbytes, wbuf+(n-j)*outbytes, &i); + if (err != CRYPT_OK) { goto cleanup; } + } + if ((err = mp_read_unsigned_bin(W, wbuf, (n+1)*outbytes)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_mod(W, t2L1, W)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_add(W, t2L1, X)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_mod(X, t2q, c)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_sub_d(c, 1, p)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_sub(X, p, p)) != CRYPT_OK) { goto cleanup; } + if (mp_cmp(p, t2L1) != LTC_MP_LT) { + /* p >= 2^(L-1) */ + if ((err = mp_prime_is_prime(p, mr_tests_p, &res)) != CRYPT_OK) { goto cleanup; } + if (res == LTC_MP_YES) { + found_p = 1; + } + } + } + } + + /* FIPS-186-4 A.2.1 Unverifiable Generation of the Generator g + * 1. e = (p - 1)/q + * 2. h = any integer satisfying: 1 < h < (p - 1) + * h could be obtained from a random number generator or from a counter that changes after each use + * 3. g = h^e mod p + * 4. if (g == 1), then go to step 2. + * + */ + + if ((err = mp_sub_d(p, 1, e)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_div(e, q, e, c)) != CRYPT_OK) { goto cleanup; } + /* e = (p - 1)/q */ + i = mp_count_bits(p); + do { + do { + if ((err = rand_bn_bits(h, i, prng, wprng)) != CRYPT_OK) { goto cleanup; } + } while (mp_cmp(h, p) != LTC_MP_LT || mp_cmp_d(h, 2) != LTC_MP_GT); + if ((err = mp_sub_d(h, 1, h)) != CRYPT_OK) { goto cleanup; } + /* h is randon and 1 < h < (p-1) */ + if ((err = mp_exptmod(h, e, p, g)) != CRYPT_OK) { goto cleanup; } + } while (mp_cmp_d(g, 1) == LTC_MP_EQ); + + err = CRYPT_OK; +cleanup: + mp_clear_multi(t2L1, t2N1, t2q, t2seedlen, U, W, X, c, h, e, seedinc, LTC_NULL); +cleanup1: + XFREE(sbuf); +cleanup2: + XFREE(wbuf); +cleanup3: + return err; +} + +/** + Generate DSA parameters p, q & g + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param group_size Size of the multiplicative group (octets) + @param modulus_size Size of the modulus (octets) + @param key [out] Where to store the created key + @return CRYPT_OK if successful. +*/ +int dsa_generate_pqg(prng_state *prng, int wprng, int group_size, int modulus_size, dsa_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* init mp_ints */ + if ((err = mp_init_multi(&key->p, &key->g, &key->q, &key->x, &key->y, LTC_NULL)) != CRYPT_OK) { + return err; + } + /* generate params */ + err = s_dsa_make_params(prng, wprng, group_size, modulus_size, key->p, key->q, key->g); + if (err != CRYPT_OK) { + goto cleanup; + } + + key->qord = group_size; + + return CRYPT_OK; + +cleanup: + dsa_free(key); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_import.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_import.c new file mode 100644 index 0000000..995e0d2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_import.c @@ -0,0 +1,143 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_import.c + DSA implementation, import a DSA key, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Import a DSA key + @param in The binary packet to import from + @param inlen The length of the binary packet + @param key [out] Where to store the imported key + @return CRYPT_OK if successful, upon error this function will free all allocated memory +*/ +int dsa_import(const unsigned char *in, unsigned long inlen, dsa_key *key) +{ + int err, stat; + unsigned long zero = 0, len; + unsigned char* tmpbuf = NULL; + unsigned char flags[1]; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* init key */ + if (mp_init_multi(&key->p, &key->g, &key->q, &key->x, &key->y, LTC_NULL) != CRYPT_OK) { + return CRYPT_MEM; + } + + /* try to match the old libtomcrypt format */ + err = der_decode_sequence_multi(in, inlen, LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_EOL, 0UL, NULL); + + if (err == CRYPT_OK || err == CRYPT_INPUT_TOO_LONG) { + /* private key */ + if (flags[0] == 1) { + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->g, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_INTEGER, 1UL, key->x, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + key->type = PK_PRIVATE; + goto LBL_OK; + } + /* public key */ + else if (flags[0] == 0) { + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_INTEGER, 1UL, key->g, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + key->type = PK_PUBLIC; + goto LBL_OK; + } + else { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + } + /* get key type */ + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &zero, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->g, + LTC_ASN1_INTEGER, 1UL, key->y, + LTC_ASN1_INTEGER, 1UL, key->x, + LTC_ASN1_EOL, 0UL, NULL)) == CRYPT_OK) { + + key->type = PK_PRIVATE; + } else { /* public */ + ltc_asn1_list params[3]; + unsigned long tmpbuf_len = inlen; + + LTC_SET_ASN1(params, 0, LTC_ASN1_INTEGER, key->p, 1UL); + LTC_SET_ASN1(params, 1, LTC_ASN1_INTEGER, key->q, 1UL); + LTC_SET_ASN1(params, 2, LTC_ASN1_INTEGER, key->g, 1UL); + + tmpbuf = XCALLOC(1, tmpbuf_len); + if (tmpbuf == NULL) { + err = CRYPT_MEM; + goto LBL_ERR; + } + + len = 3; + err = x509_decode_subject_public_key_info(in, inlen, LTC_OID_DSA, + tmpbuf, &tmpbuf_len, + LTC_ASN1_SEQUENCE, params, &len); + if (err != CRYPT_OK) { + XFREE(tmpbuf); + goto LBL_ERR; + } + + if ((err=der_decode_integer(tmpbuf, tmpbuf_len, key->y)) != CRYPT_OK) { + XFREE(tmpbuf); + goto LBL_ERR; + } + + XFREE(tmpbuf); + key->type = PK_PUBLIC; + } + +LBL_OK: + key->qord = mp_unsigned_bin_size(key->q); + + /* quick p, q, g validation, without primality testing */ + if ((err = dsa_int_validate_pqg(key, &stat)) != CRYPT_OK) { + goto LBL_ERR; + } + if (stat == 0) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + /* validate x, y */ + if ((err = dsa_int_validate_xy(key, &stat)) != CRYPT_OK) { + goto LBL_ERR; + } + if (stat == 0) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + return CRYPT_OK; +LBL_ERR: + dsa_free(key); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_make_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_make_key.c new file mode 100644 index 0000000..5a205b2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_make_key.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_make_key.c + DSA implementation, generate a DSA key +*/ + +#ifdef LTC_MDSA + +/** + Old-style creation of a DSA key + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param group_size Size of the multiplicative group (octets) + @param modulus_size Size of the modulus (octets) + @param key [out] Where to store the created key + @return CRYPT_OK if successful. +*/ +int dsa_make_key(prng_state *prng, int wprng, int group_size, int modulus_size, dsa_key *key) +{ + int err; + + if ((err = dsa_generate_pqg(prng, wprng, group_size, modulus_size, key)) != CRYPT_OK) { return err; } + if ((err = dsa_generate_key(prng, wprng, key)) != CRYPT_OK) { return err; } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_set.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_set.c new file mode 100644 index 0000000..82b6033 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_set.c @@ -0,0 +1,102 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +#ifdef LTC_MDSA + +/** + Import DSA's p, q & g from raw numbers + @param p DSA's p in binary representation + @param plen The length of p + @param q DSA's q in binary representation + @param qlen The length of q + @param g DSA's g in binary representation + @param glen The length of g + @param key [out] the destination for the imported key + @return CRYPT_OK if successful. +*/ +int dsa_set_pqg(const unsigned char *p, unsigned long plen, + const unsigned char *q, unsigned long qlen, + const unsigned char *g, unsigned long glen, + dsa_key *key) +{ + int err, stat; + + LTC_ARGCHK(p != NULL); + LTC_ARGCHK(q != NULL); + LTC_ARGCHK(g != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* init key */ + err = mp_init_multi(&key->p, &key->g, &key->q, &key->x, &key->y, LTC_NULL); + if (err != CRYPT_OK) return err; + + if ((err = mp_read_unsigned_bin(key->p, (unsigned char *)p , plen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_unsigned_bin(key->g, (unsigned char *)g , glen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_unsigned_bin(key->q, (unsigned char *)q , qlen)) != CRYPT_OK) { goto LBL_ERR; } + + key->qord = mp_unsigned_bin_size(key->q); + + /* do only a quick validation, without primality testing */ + if ((err = dsa_int_validate_pqg(key, &stat)) != CRYPT_OK) { goto LBL_ERR; } + if (stat == 0) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + return CRYPT_OK; + +LBL_ERR: + dsa_free(key); + return err; +} + +/** + Import DSA public or private key-part from raw numbers + + NB: The p, q & g parts must be set beforehand + + @param in The key-part to import, either public or private. + @param inlen The key-part's length + @param type Which type of key (PK_PRIVATE or PK_PUBLIC) + @param key [out] the destination for the imported key + @return CRYPT_OK if successful. +*/ +int dsa_set_key(const unsigned char *in, unsigned long inlen, int type, dsa_key *key) +{ + int err, stat = 0; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(key->x != NULL); + LTC_ARGCHK(key->y != NULL); + LTC_ARGCHK(key->p != NULL); + LTC_ARGCHK(key->g != NULL); + LTC_ARGCHK(key->q != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + if (type == PK_PRIVATE) { + key->type = PK_PRIVATE; + if ((err = mp_read_unsigned_bin(key->x, (unsigned char *)in, inlen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_exptmod(key->g, key->x, key->p, key->y)) != CRYPT_OK) { goto LBL_ERR; } + } + else { + key->type = PK_PUBLIC; + if ((err = mp_read_unsigned_bin(key->y, (unsigned char *)in, inlen)) != CRYPT_OK) { goto LBL_ERR; } + } + + if ((err = dsa_int_validate_xy(key, &stat)) != CRYPT_OK) { goto LBL_ERR; } + if (stat == 0) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + return CRYPT_OK; + +LBL_ERR: + dsa_free(key); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_set_pqg_dsaparam.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_set_pqg_dsaparam.c new file mode 100644 index 0000000..97c71f1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_set_pqg_dsaparam.c @@ -0,0 +1,57 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +#ifdef LTC_MDSA + +/** + Import DSA's p, q & g from dsaparam + + dsaparam data: openssl dsaparam -outform DER -out dsaparam.der 2048 + + @param dsaparam The DSA param DER encoded data + @param dsaparamlen The length of dhparam data + @param key [out] the destination for the imported key + @return CRYPT_OK if successful. +*/ +int dsa_set_pqg_dsaparam(const unsigned char *dsaparam, unsigned long dsaparamlen, + dsa_key *key) +{ + int err, stat; + + LTC_ARGCHK(dsaparam != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* init key */ + err = mp_init_multi(&key->p, &key->g, &key->q, &key->x, &key->y, LTC_NULL); + if (err != CRYPT_OK) return err; + + if ((err = der_decode_sequence_multi(dsaparam, dsaparamlen, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->g, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + + key->qord = mp_unsigned_bin_size(key->q); + + /* quick p, q, g validation, without primality testing */ + if ((err = dsa_int_validate_pqg(key, &stat)) != CRYPT_OK) { + goto LBL_ERR; + } + if (stat == 0) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + return CRYPT_OK; + +LBL_ERR: + dsa_free(key); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_shared_secret.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_shared_secret.c new file mode 100644 index 0000000..df11c17 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_shared_secret.c @@ -0,0 +1,60 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_shared_secret.c + DSA Crypto, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Create a DSA shared secret between two keys + @param private_key The private DSA key (the exponent) + @param base The base of the exponentiation (allows this to be used for both encrypt and decrypt) + @param public_key The public key + @param out [out] Destination of the shared secret + @param outlen [in/out] The max size and resulting size of the shared secret + @return CRYPT_OK if successful +*/ +int dsa_shared_secret(void *private_key, void *base, + const dsa_key *public_key, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x; + void *res; + int err; + + LTC_ARGCHK(private_key != NULL); + LTC_ARGCHK(public_key != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* make new point */ + if ((err = mp_init(&res)) != CRYPT_OK) { + return err; + } + + if ((err = mp_exptmod(base, private_key, public_key->p, res)) != CRYPT_OK) { + mp_clear(res); + return err; + } + + x = (unsigned long)mp_unsigned_bin_size(res); + if (*outlen < x) { + *outlen = x; + err = CRYPT_BUFFER_OVERFLOW; + goto done; + } + zeromem(out, x); + if ((err = mp_to_unsigned_bin(res, out + (x - mp_unsigned_bin_size(res)))) != CRYPT_OK) { goto done; } + + err = CRYPT_OK; + *outlen = x; +done: + mp_clear(res); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_sign_hash.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_sign_hash.c new file mode 100644 index 0000000..56baa80 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_sign_hash.c @@ -0,0 +1,142 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_sign_hash.c + DSA implementation, sign a hash, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Sign a hash with DSA + @param in The hash to sign + @param inlen The length of the hash to sign + @param r The "r" integer of the signature (caller must initialize with mp_init() first) + @param s The "s" integer of the signature (caller must initialize with mp_init() first) + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param key A private DSA key + @return CRYPT_OK if successful +*/ +int dsa_sign_hash_raw(const unsigned char *in, unsigned long inlen, + void *r, void *s, + prng_state *prng, int wprng, const dsa_key *key) +{ + void *k, *kinv, *tmp; + unsigned char *buf; + int err, qbits; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(r != NULL); + LTC_ARGCHK(s != NULL); + LTC_ARGCHK(key != NULL); + + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + if (key->type != PK_PRIVATE) { + return CRYPT_PK_NOT_PRIVATE; + } + + /* check group order size */ + if (key->qord >= LTC_MDSA_MAX_GROUP) { + return CRYPT_INVALID_ARG; + } + + buf = XMALLOC(LTC_MDSA_MAX_GROUP); + if (buf == NULL) { + return CRYPT_MEM; + } + + /* Init our temps */ + if ((err = mp_init_multi(&k, &kinv, &tmp, LTC_NULL)) != CRYPT_OK) { goto ERRBUF; } + + qbits = mp_count_bits(key->q); +retry: + + do { + /* gen random k */ + if ((err = rand_bn_bits(k, qbits, prng, wprng)) != CRYPT_OK) { goto error; } + + /* k should be from range: 1 <= k <= q-1 (see FIPS 186-4 B.2.2) */ + if (mp_cmp_d(k, 0) != LTC_MP_GT || mp_cmp(k, key->q) != LTC_MP_LT) { goto retry; } + + /* test gcd */ + if ((err = mp_gcd(k, key->q, tmp)) != CRYPT_OK) { goto error; } + } while (mp_cmp_d(tmp, 1) != LTC_MP_EQ); + + /* now find 1/k mod q */ + if ((err = mp_invmod(k, key->q, kinv)) != CRYPT_OK) { goto error; } + + /* now find r = g^k mod p mod q */ + if ((err = mp_exptmod(key->g, k, key->p, r)) != CRYPT_OK) { goto error; } + if ((err = mp_mod(r, key->q, r)) != CRYPT_OK) { goto error; } + + if (mp_iszero(r) == LTC_MP_YES) { goto retry; } + + /* FIPS 186-4 4.6: use leftmost min(bitlen(q), bitlen(hash)) bits of 'hash'*/ + inlen = MIN(inlen, (unsigned long)(key->qord)); + + /* now find s = (in + xr)/k mod q */ + if ((err = mp_read_unsigned_bin(tmp, (unsigned char *)in, inlen)) != CRYPT_OK) { goto error; } + if ((err = mp_mul(key->x, r, s)) != CRYPT_OK) { goto error; } + if ((err = mp_add(s, tmp, s)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(s, kinv, key->q, s)) != CRYPT_OK) { goto error; } + + if (mp_iszero(s) == LTC_MP_YES) { goto retry; } + + err = CRYPT_OK; +error: + mp_clear_multi(k, kinv, tmp, LTC_NULL); +ERRBUF: +#ifdef LTC_CLEAN_STACK + zeromem(buf, LTC_MDSA_MAX_GROUP); +#endif + XFREE(buf); + return err; +} + +/** + Sign a hash with DSA + @param in The hash to sign + @param inlen The length of the hash to sign + @param out [out] Where to store the signature + @param outlen [in/out] The max size and resulting size of the signature + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param key A private DSA key + @return CRYPT_OK if successful +*/ +int dsa_sign_hash(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, const dsa_key *key) +{ + void *r, *s; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + if (mp_init_multi(&r, &s, LTC_NULL) != CRYPT_OK) { + return CRYPT_MEM; + } + + if ((err = dsa_sign_hash_raw(in, inlen, r, s, prng, wprng, key)) != CRYPT_OK) { + goto error; + } + + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_INTEGER, 1UL, r, + LTC_ASN1_INTEGER, 1UL, s, + LTC_ASN1_EOL, 0UL, NULL); + +error: + mp_clear_multi(r, s, LTC_NULL); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_verify_hash.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_verify_hash.c new file mode 100644 index 0000000..500feda --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_verify_hash.c @@ -0,0 +1,127 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_verify_hash.c + DSA implementation, verify a signature, Tom St Denis +*/ + + +#ifdef LTC_MDSA + +/** + Verify a DSA signature + @param r DSA "r" parameter + @param s DSA "s" parameter + @param hash The hash that was signed + @param hashlen The length of the hash that was signed + @param stat [out] The result of the signature verification, 1==valid, 0==invalid + @param key The corresponding public DSA key + @return CRYPT_OK if successful (even if the signature is invalid) +*/ +int dsa_verify_hash_raw( void *r, void *s, + const unsigned char *hash, unsigned long hashlen, + int *stat, const dsa_key *key) +{ + void *w, *v, *u1, *u2; + int err; + + LTC_ARGCHK(r != NULL); + LTC_ARGCHK(s != NULL); + LTC_ARGCHK(stat != NULL); + LTC_ARGCHK(key != NULL); + + /* default to invalid signature */ + *stat = 0; + + /* init our variables */ + if ((err = mp_init_multi(&w, &v, &u1, &u2, LTC_NULL)) != CRYPT_OK) { + return err; + } + + /* neither r or s can be null or >q*/ + if (mp_cmp_d(r, 0) != LTC_MP_GT || mp_cmp_d(s, 0) != LTC_MP_GT || mp_cmp(r, key->q) != LTC_MP_LT || mp_cmp(s, key->q) != LTC_MP_LT) { + err = CRYPT_INVALID_PACKET; + goto error; + } + + /* FIPS 186-4 4.7: use leftmost min(bitlen(q), bitlen(hash)) bits of 'hash' */ + hashlen = MIN(hashlen, (unsigned long)(key->qord)); + + /* w = 1/s mod q */ + if ((err = mp_invmod(s, key->q, w)) != CRYPT_OK) { goto error; } + + /* u1 = m * w mod q */ + if ((err = mp_read_unsigned_bin(u1, (unsigned char *)hash, hashlen)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(u1, w, key->q, u1)) != CRYPT_OK) { goto error; } + + /* u2 = r*w mod q */ + if ((err = mp_mulmod(r, w, key->q, u2)) != CRYPT_OK) { goto error; } + + /* v = g^u1 * y^u2 mod p mod q */ + if ((err = mp_exptmod(key->g, u1, key->p, u1)) != CRYPT_OK) { goto error; } + if ((err = mp_exptmod(key->y, u2, key->p, u2)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(u1, u2, key->p, v)) != CRYPT_OK) { goto error; } + if ((err = mp_mod(v, key->q, v)) != CRYPT_OK) { goto error; } + + /* if r = v then we're set */ + if (mp_cmp(r, v) == LTC_MP_EQ) { + *stat = 1; + } + + err = CRYPT_OK; +error: + mp_clear_multi(w, v, u1, u2, LTC_NULL); + return err; +} + +/** + Verify a DSA signature + @param sig The signature + @param siglen The length of the signature (octets) + @param hash The hash that was signed + @param hashlen The length of the hash that was signed + @param stat [out] The result of the signature verification, 1==valid, 0==invalid + @param key The corresponding public DSA key + @return CRYPT_OK if successful (even if the signature is invalid) +*/ +int dsa_verify_hash(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + int *stat, const dsa_key *key) +{ + int err; + void *r, *s; + ltc_asn1_list sig_seq[2]; + unsigned long reallen = 0; + + LTC_ARGCHK(stat != NULL); + *stat = 0; /* must be set before the first return */ + + if ((err = mp_init_multi(&r, &s, LTC_NULL)) != CRYPT_OK) { + return err; + } + + LTC_SET_ASN1(sig_seq, 0, LTC_ASN1_INTEGER, r, 1UL); + LTC_SET_ASN1(sig_seq, 1, LTC_ASN1_INTEGER, s, 1UL); + + err = der_decode_sequence_strict(sig, siglen, sig_seq, 2); + if (err != CRYPT_OK) { + goto LBL_ERR; + } + + err = der_length_sequence(sig_seq, 2, &reallen); + if (err != CRYPT_OK || reallen != siglen) { + goto LBL_ERR; + } + + /* do the op */ + err = dsa_verify_hash_raw(r, s, hash, hashlen, stat, key); + +LBL_ERR: + mp_clear_multi(r, s, LTC_NULL); + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_verify_key.c b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_verify_key.c new file mode 100644 index 0000000..50d566a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/dsa_verify_key.c @@ -0,0 +1,189 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file dsa_verify_key.c + DSA implementation, verify a key, Tom St Denis +*/ + +#ifdef LTC_MDSA + +/** + Validate a DSA key + + Yeah, this function should've been called dsa_validate_key() + in the first place and for compat-reasons we keep it + as it was (for now). + + @param key The key to validate + @param stat [out] Result of test, 1==valid, 0==invalid + @return CRYPT_OK if successful +*/ +int dsa_verify_key(const dsa_key *key, int *stat) +{ + int err; + + err = dsa_int_validate_primes(key, stat); + if (err != CRYPT_OK || *stat == 0) return err; + + err = dsa_int_validate_pqg(key, stat); + if (err != CRYPT_OK || *stat == 0) return err; + + return dsa_int_validate_xy(key, stat); +} + +/** + Non-complex part (no primality testing) of the validation + of DSA params (p, q, g) + + @param key The key to validate + @param stat [out] Result of test, 1==valid, 0==invalid + @return CRYPT_OK if successful +*/ +int dsa_int_validate_pqg(const dsa_key *key, int *stat) +{ + void *tmp1, *tmp2; + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(stat != NULL); + *stat = 0; + + /* check q-order */ + if ( key->qord >= LTC_MDSA_MAX_GROUP || key->qord <= 15 || + (unsigned long)key->qord >= mp_unsigned_bin_size(key->p) || + (mp_unsigned_bin_size(key->p) - key->qord) >= LTC_MDSA_DELTA ) { + return CRYPT_OK; + } + + /* FIPS 186-4 chapter 4.1: 1 < g < p */ + if (mp_cmp_d(key->g, 1) != LTC_MP_GT || mp_cmp(key->g, key->p) != LTC_MP_LT) { + return CRYPT_OK; + } + + if ((err = mp_init_multi(&tmp1, &tmp2, LTC_NULL)) != CRYPT_OK) { return err; } + + /* FIPS 186-4 chapter 4.1: q is a divisor of (p - 1) */ + if ((err = mp_sub_d(key->p, 1, tmp1)) != CRYPT_OK) { goto error; } + if ((err = mp_div(tmp1, key->q, tmp1, tmp2)) != CRYPT_OK) { goto error; } + if (mp_iszero(tmp2) != LTC_MP_YES) { + err = CRYPT_OK; + goto error; + } + + /* FIPS 186-4 chapter 4.1: g is a generator of a subgroup of order q in + * the multiplicative group of GF(p) - so we make sure that g^q mod p = 1 + */ + if ((err = mp_exptmod(key->g, key->q, key->p, tmp1)) != CRYPT_OK) { goto error; } + if (mp_cmp_d(tmp1, 1) != LTC_MP_EQ) { + err = CRYPT_OK; + goto error; + } + + err = CRYPT_OK; + *stat = 1; +error: + mp_clear_multi(tmp2, tmp1, LTC_NULL); + return err; +} + +/** + Primality testing of DSA params p and q + + @param key The key to validate + @param stat [out] Result of test, 1==valid, 0==invalid + @return CRYPT_OK if successful +*/ +int dsa_int_validate_primes(const dsa_key *key, int *stat) +{ + int err, res; + + *stat = 0; + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(stat != NULL); + + /* key->q prime? */ + if ((err = mp_prime_is_prime(key->q, LTC_MILLER_RABIN_REPS, &res)) != CRYPT_OK) { + return err; + } + if (res == LTC_MP_NO) { + return CRYPT_OK; + } + + /* key->p prime? */ + if ((err = mp_prime_is_prime(key->p, LTC_MILLER_RABIN_REPS, &res)) != CRYPT_OK) { + return err; + } + if (res == LTC_MP_NO) { + return CRYPT_OK; + } + + *stat = 1; + return CRYPT_OK; +} + +/** + Validation of a DSA key (x and y values) + + @param key The key to validate + @param stat [out] Result of test, 1==valid, 0==invalid + @return CRYPT_OK if successful +*/ +int dsa_int_validate_xy(const dsa_key *key, int *stat) +{ + void *tmp; + int err; + + *stat = 0; + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(stat != NULL); + + /* 1 < y < p-1 */ + if ((err = mp_init(&tmp)) != CRYPT_OK) { + return err; + } + if ((err = mp_sub_d(key->p, 1, tmp)) != CRYPT_OK) { + goto error; + } + if (mp_cmp_d(key->y, 1) != LTC_MP_GT || mp_cmp(key->y, tmp) != LTC_MP_LT) { + err = CRYPT_OK; + goto error; + } + + if (key->type == PK_PRIVATE) { + /* FIPS 186-4 chapter 4.1: 0 < x < q */ + if (mp_cmp_d(key->x, 0) != LTC_MP_GT || mp_cmp(key->x, key->q) != LTC_MP_LT) { + err = CRYPT_OK; + goto error; + } + /* FIPS 186-4 chapter 4.1: y = g^x mod p */ + if ((err = mp_exptmod(key->g, key->x, key->p, tmp)) != CRYPT_OK) { + goto error; + } + if (mp_cmp(tmp, key->y) != LTC_MP_EQ) { + err = CRYPT_OK; + goto error; + } + } + else { + /* with just a public key we cannot test y = g^x mod p therefore we + * only test that y^q mod p = 1, which makes sure y is in g^x mod p + */ + if ((err = mp_exptmod(key->y, key->q, key->p, tmp)) != CRYPT_OK) { + goto error; + } + if (mp_cmp_d(tmp, 1) != LTC_MP_EQ) { + err = CRYPT_OK; + goto error; + } + } + + err = CRYPT_OK; + *stat = 1; +error: + mp_clear(tmp); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/dsa/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/dsa/sub.mk new file mode 100644 index 0000000..d0b29e5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/dsa/sub.mk @@ -0,0 +1,12 @@ +srcs-y += dsa_decrypt_key.c +srcs-y += dsa_encrypt_key.c +srcs-y += dsa_export.c +srcs-y += dsa_free.c +srcs-y += dsa_generate_key.c +srcs-y += dsa_generate_pqg.c +srcs-y += dsa_import.c +srcs-y += dsa_make_key.c +srcs-y += dsa_shared_secret.c +srcs-y += dsa_sign_hash.c +srcs-y += dsa_verify_hash.c +srcs-y += dsa_verify_key.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_crypto_ctx.c b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_crypto_ctx.c new file mode 100644 index 0000000..e1efb30 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_crypto_ctx.c @@ -0,0 +1,41 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ec25519_crypto_ctx.c + curve25519 crypto context helper +*/ + +#ifdef LTC_CURVE25519 + +int ec25519_crypto_ctx(unsigned char *out, unsigned long *outlen, unsigned char flag, const unsigned char *ctx, unsigned long ctxlen) +{ + unsigned char *buf = out; + + const char *prefix = "SigEd25519 no Ed25519 collisions"; + const unsigned long prefix_len = XSTRLEN(prefix); + const unsigned char ctxlen8 = (unsigned char)ctxlen; + + if (ctxlen > 255u) return CRYPT_INPUT_TOO_LONG; + if (*outlen < prefix_len + 2u + ctxlen) return CRYPT_BUFFER_OVERFLOW; + + XMEMCPY(buf, prefix, prefix_len); + buf += prefix_len; + XMEMCPY(buf, &flag, 1); + buf++; + XMEMCPY(buf, &ctxlen8, 1); + buf++; + + if (ctxlen > 0u) { + LTC_ARGCHK(ctx != NULL); + XMEMCPY(buf, ctx, ctxlen); + buf += ctxlen; + } + + *outlen = buf-out; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_export.c b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_export.c new file mode 100644 index 0000000..42f4c4b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_export.c @@ -0,0 +1,90 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ec25519_export.c + Generic export of a Curve/Ed25519 key to a binary packet, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Generic export of a Curve/Ed25519 key to a binary packet + @param out [out] The destination for the key + @param outlen [in/out] The max size and resulting size of the Ed25519 key + @param type Which type of key (PK_PRIVATE, PK_PUBLIC|PK_STD or PK_PUBLIC) + @param key The key you wish to export + @return CRYPT_OK if successful +*/ +int ec25519_export( unsigned char *out, unsigned long *outlen, + int which, + const curve25519_key *key) +{ + int err, std; + const char* OID; + unsigned long oid[16], oidlen; + ltc_asn1_list alg_id[1]; + unsigned char private_key[34]; + unsigned long version, private_key_len = sizeof(private_key); + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + std = which & PK_STD; + which &= ~PK_STD; + + if (which == PK_PRIVATE) { + if(key->type != PK_PRIVATE) return CRYPT_PK_INVALID_TYPE; + + if (std == PK_STD) { + if ((err = pk_get_oid(key->algo, &OID)) != CRYPT_OK) { + return err; + } + oidlen = sizeof(oid)/sizeof(oid[0]); + if ((err = pk_oid_str_to_num(OID, oid, &oidlen)) != CRYPT_OK) { + return err; + } + + LTC_SET_ASN1(alg_id, 0, LTC_ASN1_OBJECT_IDENTIFIER, oid, oidlen); + + /* encode private key as PKCS#8 */ + if ((err = der_encode_octet_string(key->priv, 32uL, private_key, &private_key_len)) != CRYPT_OK) { + return err; + } + + version = 0; + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_SHORT_INTEGER, 1uL, &version, + LTC_ASN1_SEQUENCE, 1uL, alg_id, + LTC_ASN1_OCTET_STRING, private_key_len, private_key, + LTC_ASN1_EOL, 0uL, NULL); + } else { + if (*outlen < sizeof(key->priv)) { + err = CRYPT_BUFFER_OVERFLOW; + } else { + XMEMCPY(out, key->priv, sizeof(key->priv)); + err = CRYPT_OK; + } + *outlen = sizeof(key->priv); + } + } else { + if (std == PK_STD) { + /* encode public key as SubjectPublicKeyInfo */ + err = x509_encode_subject_public_key_info(out, outlen, key->algo, key->pub, 32uL, LTC_ASN1_EOL, NULL, 0); + } else { + if (*outlen < sizeof(key->pub)) { + err = CRYPT_BUFFER_OVERFLOW; + } else { + XMEMCPY(out, key->pub, sizeof(key->pub)); + err = CRYPT_OK; + } + *outlen = sizeof(key->pub); + } + } + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_import_pkcs8.c b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_import_pkcs8.c new file mode 100644 index 0000000..13807ab --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/ec25519_import_pkcs8.c @@ -0,0 +1,87 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ec25519_import_pkcs8.c + Generic import of a Curve/Ed25519 private key in PKCS#8 format, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Generic import of a Curve/Ed25519 private key in PKCS#8 format + @param in The DER-encoded PKCS#8-formatted private key + @param inlen The length of the input data + @param passwd The password to decrypt the private key + @param passwdlen Password's length (octets) + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int ec25519_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + enum ltc_oid_id id, sk_to_pk fp, + curve25519_key *key) +{ + int err; + ltc_asn1_list *l = NULL; + const char *oid; + ltc_asn1_list alg_id[1]; + unsigned char private_key[34]; + unsigned long version, key_len; + unsigned long tmpoid[16]; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(fp != NULL); + + if ((err = pkcs8_decode_flexi(in, inlen, pwd, pwdlen, &l)) == CRYPT_OK) { + + LTC_SET_ASN1(alg_id, 0, LTC_ASN1_OBJECT_IDENTIFIER, tmpoid, sizeof(tmpoid) / sizeof(tmpoid[0])); + + key_len = sizeof(private_key); + if ((err = der_decode_sequence_multi(l->data, l->size, + LTC_ASN1_SHORT_INTEGER, 1uL, &version, + LTC_ASN1_SEQUENCE, 1uL, alg_id, + LTC_ASN1_OCTET_STRING, key_len, private_key, + LTC_ASN1_EOL, 0uL, NULL)) + != CRYPT_OK) { + /* If there are attributes added after the private_key it is tagged with version 1 and + * we get an 'input too long' error but the rest is already decoded and can be + * handled the same as for version 0 + */ + if ((err == CRYPT_INPUT_TOO_LONG) && (version == 1)) { + version = 0; + } else { + goto out; + } + } + + if ((err = pk_get_oid(id, &oid)) != CRYPT_OK) { + goto out; + } + if ((err = pk_oid_cmp_with_asn1(oid, &alg_id[0])) != CRYPT_OK) { + goto out; + } + + if (version == 0) { + key_len = sizeof(key->priv); + if ((err = der_decode_octet_string(private_key, sizeof(private_key), key->priv, &key_len)) == CRYPT_OK) { + fp(key->pub, key->priv); + key->type = PK_PRIVATE; + key->algo = id; + } + } else { + err = CRYPT_PK_INVALID_TYPE; + } + } +out: + if (l) der_free_sequence_flexi(l); +#ifdef LTC_CLEAN_STACK + zeromem(private_key, sizeof(private_key)); +#endif + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ec25519/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/sub.mk new file mode 100644 index 0000000..5040c39 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/sub.mk @@ -0,0 +1,3 @@ +srcs-y += ec25519_crypto_ctx.c +srcs-y += ec25519_export.c +srcs-y += tweetnacl.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ec25519/tweetnacl.c b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/tweetnacl.c new file mode 100644 index 0000000..ad0a7c6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ec25519/tweetnacl.c @@ -0,0 +1,491 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/* automatically generated file, do not edit */ + +#define FOR(i,n) for (i = 0;i < n;++i) +#define sv static void + +typedef unsigned char u8; +typedef ulong32 u32; +typedef ulong64 u64; +typedef long64 i64; +typedef i64 gf[16]; + +static const u8 + nine[32] = {9}; +static const gf + gf0, + gf1 = {1}, + gf121665 = {0xDB41,1}, + D = {0x78a3, 0x1359, 0x4dca, 0x75eb, 0xd8ab, 0x4141, 0x0a4d, 0x0070, 0xe898, 0x7779, 0x4079, 0x8cc7, 0xfe73, 0x2b6f, 0x6cee, 0x5203}, + D2 = {0xf159, 0x26b2, 0x9b94, 0xebd6, 0xb156, 0x8283, 0x149a, 0x00e0, 0xd130, 0xeef3, 0x80f2, 0x198e, 0xfce7, 0x56df, 0xd9dc, 0x2406}, + X = {0xd51a, 0x8f25, 0x2d60, 0xc956, 0xa7b2, 0x9525, 0xc760, 0x692c, 0xdc5c, 0xfdd6, 0xe231, 0xc0a4, 0x53fe, 0xcd6e, 0x36d3, 0x2169}, + Y = {0x6658, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666, 0x6666}, + I = {0xa0b0, 0x4a0e, 0x1b27, 0xc4ee, 0xe478, 0xad2f, 0x1806, 0x2f43, 0xd7a7, 0x3dfb, 0x0099, 0x2b4d, 0xdf0b, 0x4fc1, 0x2480, 0x2b83}; + +static int vn(const u8 *x,const u8 *y,int n) +{ + int i; + u32 d = 0; + FOR(i,n) d |= x[i]^y[i]; + return (1 & ((d - 1) >> 8)) - 1; +} + +static int tweetnacl_crypto_verify_32(const u8 *x,const u8 *y) +{ + return vn(x,y,32); +} + +sv set25519(gf r, const gf a) +{ + int i; + FOR(i,16) r[i]=a[i]; +} + +sv car25519(gf o) +{ + int i; + i64 c; + FOR(i,16) { + o[i]+=(1LL<<16); + c=o[i]>>16; + o[(i+1)*(i<15)]+=c-1+37*(c-1)*(i==15); + o[i]-=c<<16; + } +} + +sv sel25519(gf p,gf q,int b) +{ + i64 t,i,c=~(b-1); + FOR(i,16) { + t= c&(p[i]^q[i]); + p[i]^=t; + q[i]^=t; + } +} + +sv pack25519(u8 *o,const gf n) +{ + int i,j,b; + gf m,t; + FOR(i,16) t[i]=n[i]; + car25519(t); + car25519(t); + car25519(t); + FOR(j,2) { + m[0]=t[0]-0xffed; + for(i=1;i<15;i++) { + m[i]=t[i]-0xffff-((m[i-1]>>16)&1); + m[i-1]&=0xffff; + } + m[15]=t[15]-0x7fff-((m[14]>>16)&1); + b=(m[15]>>16)&1; + m[14]&=0xffff; + sel25519(t,m,1-b); + } + FOR(i,16) { + o[2*i]=t[i]&0xff; + o[2*i+1]=t[i]>>8; + } +} + +static int neq25519(const gf a, const gf b) +{ + u8 c[32],d[32]; + pack25519(c,a); + pack25519(d,b); + return tweetnacl_crypto_verify_32(c,d); +} + +static u8 par25519(const gf a) +{ + u8 d[32]; + pack25519(d,a); + return d[0]&1; +} + +sv unpack25519(gf o, const u8 *n) +{ + int i; + FOR(i,16) o[i]=n[2*i]+((i64)n[2*i+1]<<8); + o[15]&=0x7fff; +} + +sv A(gf o,const gf a,const gf b) +{ + int i; + FOR(i,16) o[i]=a[i]+b[i]; +} + +sv Z(gf o,const gf a,const gf b) +{ + int i; + FOR(i,16) o[i]=a[i]-b[i]; +} + +sv M(gf o,const gf a,const gf b) +{ + i64 i,j,t[31]; + FOR(i,31) t[i]=0; + FOR(i,16) FOR(j,16) t[i+j]+=a[i]*b[j]; + FOR(i,15) t[i]+=38*t[i+16]; + FOR(i,16) o[i]=t[i]; + car25519(o); + car25519(o); +} + +sv S(gf o,const gf a) +{ + M(o,a,a); +} + +sv inv25519(gf o,const gf i) +{ + gf c; + int a; + FOR(a,16) c[a]=i[a]; + for(a=253;a>=0;a--) { + S(c,c); + if(a!=2&&a!=4) M(c,c,i); + } + FOR(a,16) o[a]=c[a]; +} + +sv pow2523(gf o,const gf i) +{ + gf c; + int a; + FOR(a,16) c[a]=i[a]; + for(a=250;a>=0;a--) { + S(c,c); + if(a!=1) M(c,c,i); + } + FOR(a,16) o[a]=c[a]; +} + +int tweetnacl_crypto_scalarmult(u8 *q,const u8 *n,const u8 *p) +{ + u8 z[32]; + i64 x[80],r,i; + gf a,b,c,d,e,f; + FOR(i,31) z[i]=n[i]; + z[31]=(n[31]&127)|64; + z[0]&=248; + unpack25519(x,p); + FOR(i,16) { + b[i]=x[i]; + d[i]=a[i]=c[i]=0; + } + a[0]=d[0]=1; + for(i=254;i>=0;--i) { + r=(z[i>>3]>>(i&7))&1; + sel25519(a,b,r); + sel25519(c,d,r); + A(e,a,c); + Z(a,a,c); + A(c,b,d); + Z(b,b,d); + S(d,e); + S(f,a); + M(a,c,a); + M(c,b,e); + A(e,a,c); + Z(a,a,c); + S(b,a); + Z(c,d,f); + M(a,c,gf121665); + A(a,a,d); + M(c,c,a); + M(a,d,f); + M(d,b,x); + S(b,e); + sel25519(a,b,r); + sel25519(c,d,r); + } + FOR(i,16) { + x[i+16]=a[i]; + x[i+32]=c[i]; + x[i+48]=b[i]; + x[i+64]=d[i]; + } + inv25519(x+32,x+32); + M(x+16,x+16,x+32); + pack25519(q,x+16); + return 0; +} + +int tweetnacl_crypto_scalarmult_base(u8 *q,const u8 *n) +{ + return tweetnacl_crypto_scalarmult(q,n,nine); +} + +static LTC_INLINE int tweetnacl_crypto_hash_ctx(u8 *out,const u8 *m,u64 n,const u8 *ctx,u32 cs) +{ + unsigned long len = 64; + int hash_idx = find_hash("sha512"); + + if (n > ULONG_MAX) return CRYPT_OVERFLOW; + + if(cs == 0) + return hash_memory(hash_idx, m, n, out, &len); + + return hash_memory_multi(hash_idx, out, &len, ctx, cs, m, n, LTC_NULL); +} + +static LTC_INLINE int tweetnacl_crypto_hash(u8 *out,const u8 *m,u64 n) +{ + return tweetnacl_crypto_hash_ctx(out, m, n, NULL, 0); +} + +sv add(gf p[4],gf q[4]) +{ + gf a,b,c,d,t,e,f,g,h; + + Z(a, p[1], p[0]); + Z(t, q[1], q[0]); + M(a, a, t); + A(b, p[0], p[1]); + A(t, q[0], q[1]); + M(b, b, t); + M(c, p[3], q[3]); + M(c, c, D2); + M(d, p[2], q[2]); + A(d, d, d); + Z(e, b, a); + Z(f, d, c); + A(g, d, c); + A(h, b, a); + + M(p[0], e, f); + M(p[1], h, g); + M(p[2], g, f); + M(p[3], e, h); +} + +sv cswap(gf p[4],gf q[4],u8 b) +{ + int i; + FOR(i,4) + sel25519(p[i],q[i],b); +} + +sv pack(u8 *r,gf p[4]) +{ + gf tx, ty, zi; + inv25519(zi, p[2]); + M(tx, p[0], zi); + M(ty, p[1], zi); + pack25519(r, ty); + r[31] ^= par25519(tx) << 7; +} + +sv scalarmult(gf p[4],gf q[4],const u8 *s) +{ + int i; + set25519(p[0],gf0); + set25519(p[1],gf1); + set25519(p[2],gf1); + set25519(p[3],gf0); + for (i = 255;i >= 0;--i) { + u8 b = (s[i/8]>>(i&7))&1; + cswap(p,q,b); + add(q,p); + add(p,p); + cswap(p,q,b); + } +} + +sv scalarbase(gf p[4],const u8 *s) +{ + gf q[4]; + set25519(q[0],X); + set25519(q[1],Y); + set25519(q[2],gf1); + M(q[3],X,Y); + scalarmult(p,q,s); +} + +int tweetnacl_crypto_sk_to_pk(u8 *pk, const u8 *sk) +{ + u8 d[64]; + gf p[4]; + tweetnacl_crypto_hash(d, sk, 32); + d[0] &= 248; + d[31] &= 127; + d[31] |= 64; + + scalarbase(p,d); + pack(pk,p); + + return 0; +} + +int tweetnacl_crypto_sign_keypair(prng_state *prng, int wprng, u8 *pk, u8 *sk) +{ + int err; + + /* randombytes(sk,32); */ + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + + if (prng_descriptor[wprng]->read(sk,32, prng) != 32) { + return CRYPT_ERROR_READPRNG; + } + + if ((err = tweetnacl_crypto_sk_to_pk(pk, sk)) != CRYPT_OK) { + return err; + } + + /* FOR(i,32) sk[32 + i] = pk[i]; + * we don't copy the pk in the sk */ + return CRYPT_OK; +} + +static const u64 L[32] = {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10}; + +sv modL(u8 *r,i64 x[64]) +{ + i64 carry,i,j; + for (i = 63;i >= 32;--i) { + carry = 0; + for (j = i - 32;j < i - 12;++j) { + x[j] += carry - 16 * x[i] * L[j - (i - 32)]; + carry = (x[j] + 128) >> 8; + x[j] -= carry << 8; + } + x[j] += carry; + x[i] = 0; + } + carry = 0; + FOR(j,32) { + x[j] += carry - (x[31] >> 4) * L[j]; + carry = x[j] >> 8; + x[j] &= 255; + } + FOR(j,32) x[j] -= carry * L[j]; + FOR(i,32) { + x[i+1] += x[i] >> 8; + r[i] = x[i] & 255; + } +} + +sv reduce(u8 *r) +{ + i64 x[64],i; + FOR(i,64) x[i] = (u64) r[i]; + FOR(i,64) r[i] = 0; + modL(r,x); +} + +int tweetnacl_crypto_sign(u8 *sm,u64 *smlen,const u8 *m,u64 mlen,const u8 *sk,const u8 *pk, const u8 *ctx, u64 cs) +{ + u8 d[64],h[64],r[64]; + i64 i,j,x[64]; + gf p[4]; + + tweetnacl_crypto_hash(d, sk, 32); + d[0] &= 248; + d[31] &= 127; + d[31] |= 64; + + *smlen = mlen+64; + FOR(i,(i64)mlen) sm[64 + i] = m[i]; + FOR(i,32) sm[32 + i] = d[32 + i]; + + tweetnacl_crypto_hash_ctx(r, sm+32, mlen+32,ctx,cs); + reduce(r); + scalarbase(p,r); + pack(sm,p); + + FOR(i,32) sm[i+32] = pk[i]; + tweetnacl_crypto_hash_ctx(h,sm,mlen + 64,ctx,cs); + reduce(h); + + FOR(i,64) x[i] = 0; + FOR(i,32) x[i] = (u64) r[i]; + FOR(i,32) FOR(j,32) x[i+j] += h[i] * (u64) d[j]; + modL(sm + 32,x); + + return 0; +} + +static int unpackneg(gf r[4],const u8 p[32]) +{ + gf t, chk, num, den, den2, den4, den6; + set25519(r[2],gf1); + unpack25519(r[1],p); + S(num,r[1]); + M(den,num,D); + Z(num,num,r[2]); + A(den,r[2],den); + + S(den2,den); + S(den4,den2); + M(den6,den4,den2); + M(t,den6,num); + M(t,t,den); + + pow2523(t,t); + M(t,t,num); + M(t,t,den); + M(t,t,den); + M(r[0],t,den); + + S(chk,r[0]); + M(chk,chk,den); + if (neq25519(chk, num)) M(r[0],r[0],I); + + S(chk,r[0]); + M(chk,chk,den); + if (neq25519(chk, num)) return -1; + + if (par25519(r[0]) == (p[31]>>7)) Z(r[0],gf0,r[0]); + + M(r[3],r[0],r[1]); + return 0; +} + +int tweetnacl_crypto_sign_open(int *stat, u8 *m,u64 *mlen,const u8 *sm,u64 smlen,const u8 *ctx,u64 cs,const u8 *pk) +{ + u64 i; + u8 s[32],t[32],h[64]; + gf p[4],q[4]; + + *stat = 0; + if (*mlen < smlen) return CRYPT_BUFFER_OVERFLOW; + *mlen = -1; + if (smlen < 64) return CRYPT_INVALID_ARG; + + if (unpackneg(q,pk)) return CRYPT_ERROR; + + XMEMMOVE(m,sm,smlen); + XMEMMOVE(s,m + 32,32); + XMEMMOVE(m + 32,pk,32); + tweetnacl_crypto_hash_ctx(h,m,smlen,ctx,cs); + reduce(h); + scalarmult(p,q,h); + + scalarbase(q,s); + add(p,q); + pack(t,p); + + smlen -= 64; + if (tweetnacl_crypto_verify_32(sm, t)) { + FOR(i,smlen) m[i] = 0; + zeromem(m, smlen); + return CRYPT_OK; + } + + *stat = 1; + XMEMMOVE(m,m + 64,smlen); + *mlen = smlen; + return CRYPT_OK; +} + +int tweetnacl_crypto_ph(u8 *out,const u8 *msg,u64 msglen) +{ + return tweetnacl_crypto_hash(out, msg, msglen); +} diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc.c new file mode 100644 index 0000000..5132e0c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc.c @@ -0,0 +1,447 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/* This array holds the curve parameters. + * Curves (prime field only) are taken from: + * - http://www.secg.org/collateral/sec2_final.pdf (named: SECP*) + * - http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf (named: NISTP*) + * - ANS X9.62 (named: PRIMEP*) + * - http://www.ecc-brainpool.org/download/Domain-parameters.pdf (named: BRAINPOOLP*) + */ +const ltc_ecc_curve ltc_ecc_curves[] = { +#ifdef LTC_ECC_SECP112R1 +{ + /* prime */ "DB7C2ABF62E35E668076BEAD208B", + /* A */ "DB7C2ABF62E35E668076BEAD2088", + /* B */ "659EF8BA043916EEDE8911702B22", + /* order */ "DB7C2ABF62E35E7628DFAC6561C5", + /* Gx */ "09487239995A5EE76B55F9C2F098", + /* Gy */ "A89CE5AF8724C0A23E0E0FF77500", + /* cofactor */ 1, + /* OID */ "1.3.132.0.6" +}, +#endif +#ifdef LTC_ECC_SECP112R2 +{ + /* prime */ "DB7C2ABF62E35E668076BEAD208B", + /* A */ "6127C24C05F38A0AAAF65C0EF02C", + /* B */ "51DEF1815DB5ED74FCC34C85D709", + /* order */ "36DF0AAFD8B8D7597CA10520D04B", + /* Gx */ "4BA30AB5E892B4E1649DD0928643", + /* Gy */ "ADCD46F5882E3747DEF36E956E97", + /* cofactor */ 4, + /* OID */ "1.3.132.0.7" +}, +#endif +#ifdef LTC_ECC_SECP128R1 +{ + /* prime */ "FFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFF", + /* A */ "FFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFC", + /* B */ "E87579C11079F43DD824993C2CEE5ED3", + /* order */ "FFFFFFFE0000000075A30D1B9038A115", + /* Gx */ "161FF7528B899B2D0C28607CA52C5B86", + /* Gy */ "CF5AC8395BAFEB13C02DA292DDED7A83", + /* cofactor */ 1, + /* OID */ "1.3.132.0.28" +}, +#endif +#ifdef LTC_ECC_SECP128R2 +{ + /* prime */ "FFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFF", + /* A */ "D6031998D1B3BBFEBF59CC9BBFF9AEE1", + /* B */ "5EEEFCA380D02919DC2C6558BB6D8A5D", + /* order */ "3FFFFFFF7FFFFFFFBE0024720613B5A3", + /* Gx */ "7B6AA5D85E572983E6FB32A7CDEBC140", + /* Gy */ "27B6916A894D3AEE7106FE805FC34B44", + /* cofactor */ 4, + /* OID */ "1.3.132.0.29" +}, +#endif +#ifdef LTC_ECC_SECP160R1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF7FFFFFFF", + /* A */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF7FFFFFFC", + /* B */ "1C97BEFC54BD7A8B65ACF89F81D4D4ADC565FA45", + /* order */ "0100000000000000000001F4C8F927AED3CA752257", + /* Gx */ "4A96B5688EF573284664698968C38BB913CBFC82", + /* Gy */ "23A628553168947D59DCC912042351377AC5FB32", + /* cofactor */ 1, + /* OID */ "1.3.132.0.8" +}, +#endif +#ifdef LTC_ECC_SECP160R2 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFAC73", + /* A */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFAC70", + /* B */ "B4E134D3FB59EB8BAB57274904664D5AF50388BA", + /* order */ "0100000000000000000000351EE786A818F3A1A16B", + /* Gx */ "52DCB034293A117E1F4FF11B30F7199D3144CE6D", + /* Gy */ "FEAFFEF2E331F296E071FA0DF9982CFEA7D43F2E", + /* cofactor */ 1, + /* OID */ "1.3.132.0.30" +}, +#endif +#ifdef LTC_ECC_SECP160K1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFAC73", + /* A */ "0000000000000000000000000000000000000000", + /* B */ "0000000000000000000000000000000000000007", + /* order */ "0100000000000000000001B8FA16DFAB9ACA16B6B3", + /* Gx */ "3B4C382CE37AA192A4019E763036F4F5DD4D7EBB", + /* Gy */ "938CF935318FDCED6BC28286531733C3F03C4FEE", + /* cofactor */ 1, + /* OID */ "1.3.132.0.9" +}, +#endif +#ifdef LTC_ECC_SECP192R1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFF", + /* A */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFC", + /* B */ "64210519E59C80E70FA7E9AB72243049FEB8DEECC146B9B1", + /* order */ "FFFFFFFFFFFFFFFFFFFFFFFF99DEF836146BC9B1B4D22831", + /* Gx */ "188DA80EB03090F67CBF20EB43A18800F4FF0AFD82FF1012", + /* Gy */ "07192B95FFC8DA78631011ED6B24CDD573F977A11E794811", + /* cofactor */ 1, + /* OID */ "1.2.840.10045.3.1.1" +}, +#endif +#ifdef LTC_ECC_PRIME192V2 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFF", + /* A */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFC", + /* B */ "CC22D6DFB95C6B25E49C0D6364A4E5980C393AA21668D953", + /* order */ "FFFFFFFFFFFFFFFFFFFFFFFE5FB1A724DC80418648D8DD31", + /* Gx */ "EEA2BAE7E1497842F2DE7769CFE9C989C072AD696F48034A", + /* Gy */ "6574D11D69B6EC7A672BB82A083DF2F2B0847DE970B2DE15", + /* cofactor */ 1, + /* OID */ "1.2.840.10045.3.1.2" +}, +#endif +#ifdef LTC_ECC_PRIME192V3 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFF", + /* A */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFC", + /* B */ "22123DC2395A05CAA7423DAECCC94760A7D462256BD56916", + /* order */ "FFFFFFFFFFFFFFFFFFFFFFFF7A62D031C83F4294F640EC13", + /* Gx */ "7D29778100C65A1DA1783716588DCE2B8B4AEE8E228F1896", + /* Gy */ "38A90F22637337334B49DCB66A6DC8F9978ACA7648A943B0", + /* cofactor */ 1, + /* OID */ "1.2.840.10045.3.1.3" +}, +#endif +#ifdef LTC_ECC_SECP192K1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFEE37", + /* A */ "000000000000000000000000000000000000000000000000", + /* B */ "000000000000000000000000000000000000000000000003", + /* order */ "FFFFFFFFFFFFFFFFFFFFFFFE26F2FC170F69466A74DEFD8D", + /* Gx */ "DB4FF10EC057E9AE26B07D0280B7F4341DA5D1B1EAE06C7D", + /* Gy */ "9B2F2F6D9C5628A7844163D015BE86344082AA88D95E2F9D", + /* cofactor */ 1, + /* OID */ "1.3.132.0.31" +}, +#endif +#ifdef LTC_ECC_SECP224R1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF000000000000000000000001", + /* A */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFE", + /* B */ "B4050A850C04B3ABF54132565044B0B7D7BFD8BA270B39432355FFB4", + /* order */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFF16A2E0B8F03E13DD29455C5C2A3D", + /* Gx */ "B70E0CBD6BB4BF7F321390B94A03C1D356C21122343280D6115C1D21", + /* Gy */ "BD376388B5F723FB4C22DFE6CD4375A05A07476444D5819985007E34", + /* cofactor */ 1, + /* OID */ "1.3.132.0.33" +}, +#endif +#ifdef LTC_ECC_SECP224K1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFE56D", + /* A */ "00000000000000000000000000000000000000000000000000000000", + /* B */ "00000000000000000000000000000000000000000000000000000005", + /* order */ "010000000000000000000000000001DCE8D2EC6184CAF0A971769FB1F7", + /* Gx */ "A1455B334DF099DF30FC28A169A467E9E47075A90F7E650EB6B7A45C", + /* Gy */ "7E089FED7FBA344282CAFBD6F7E319F7C0B0BD59E2CA4BDB556D61A5", + /* cofactor */ 1, + /* OID */ "1.3.132.0.32" +}, +#endif +#ifdef LTC_ECC_SECP256R1 +{ + /* prime */ "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF", + /* A */ "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC", + /* B */ "5AC635D8AA3A93E7B3EBBD55769886BC651D06B0CC53B0F63BCE3C3E27D2604B", + /* order */ "FFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551", + /* Gx */ "6B17D1F2E12C4247F8BCE6E563A440F277037D812DEB33A0F4A13945D898C296", + /* Gy */ "4FE342E2FE1A7F9B8EE7EB4A7C0F9E162BCE33576B315ECECBB6406837BF51F5", + /* cofactor */ 1, + /* OID */ "1.2.840.10045.3.1.7" +}, +#endif +#ifdef LTC_ECC_SECP256K1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", + /* A */ "0000000000000000000000000000000000000000000000000000000000000000", + /* B */ "0000000000000000000000000000000000000000000000000000000000000007", + /* order */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", + /* Gx */ "79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", + /* Gy */ "483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", + /* cofactor */ 1, + /* OID */ "1.3.132.0.10" +}, +#endif +#ifdef LTC_ECC_SECP384R1 +{ + /* prime */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFF0000000000000000FFFFFFFF", + /* A */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFF0000000000000000FFFFFFFC", + /* B */ "B3312FA7E23EE7E4988E056BE3F82D19181D9C6EFE8141120314088F5013875AC656398D8A2ED19D2A85C8EDD3EC2AEF", + /* order */ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC7634D81F4372DDF581A0DB248B0A77AECEC196ACCC52973", + /* Gx */ "AA87CA22BE8B05378EB1C71EF320AD746E1D3B628BA79B9859F741E082542A385502F25DBF55296C3A545E3872760AB7", + /* Gy */ "3617DE4A96262C6F5D9E98BF9292DC29F8F41DBD289A147CE9DA3113B5F0B8C00A60B1CE1D7E819D7A431D7C90EA0E5F", + /* cofactor */ 1, + /* OID */ "1.3.132.0.34" +}, +#endif +#ifdef LTC_ECC_SECP521R1 +{ + /* prime */ "01FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + /* A */ "01FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC", + /* B */ "0051953EB9618E1C9A1F929A21A0B68540EEA2DA725B99B315F3B8B489918EF109E156193951EC7E937B1652C0BD3BB1BF073573DF883D2C34F1EF451FD46B503F00", + /* order */ "01FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFA51868783BF2F966B7FCC0148F709A5D03BB5C9B8899C47AEBB6FB71E91386409", + /* Gx */ "00C6858E06B70404E9CD9E3ECB662395B4429C648139053FB521F828AF606B4D3DBAA14B5E77EFE75928FE1DC127A2FFA8DE3348B3C1856A429BF97E7E31C2E5BD66", + /* Gy */ "011839296A789A3BC0045C8A5FB42C7D1BD998F54449579B446817AFBD17273E662C97EE72995EF42640C550B9013FAD0761353C7086A272C24088BE94769FD16650", + /* cofactor */ 1, + /* OID */ "1.3.132.0.35" +}, +#endif +#ifdef LTC_ECC_PRIME239V1 +{ + /* prime */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFFFFFFFF8000000000007FFFFFFFFFFF", + /* A */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFFFFFFFF8000000000007FFFFFFFFFFC", + /* B */ "6B016C3BDCF18941D0D654921475CA71A9DB2FB27D1D37796185C2942C0A", + /* order */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFF9E5E9A9F5D9071FBD1522688909D0B", + /* Gx */ "0FFA963CDCA8816CCC33B8642BEDF905C3D358573D3F27FBBD3B3CB9AAAF", + /* Gy */ "7DEBE8E4E90A5DAE6E4054CA530BA04654B36818CE226B39FCCB7B02F1AE", + /* cofactor */ 1, + /* OID */ "1.2.840.10045.3.1.4" +}, +#endif +#ifdef LTC_ECC_PRIME239V2 +{ + /* prime */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFFFFFFFF8000000000007FFFFFFFFFFF", + /* A */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFFFFFFFF8000000000007FFFFFFFFFFC", + /* B */ "617FAB6832576CBBFED50D99F0249C3FEE58B94BA0038C7AE84C8C832F2C", + /* order */ "7FFFFFFFFFFFFFFFFFFFFFFF800000CFA7E8594377D414C03821BC582063", + /* Gx */ "38AF09D98727705120C921BB5E9E26296A3CDCF2F35757A0EAFD87B830E7", + /* Gy */ "5B0125E4DBEA0EC7206DA0FC01D9B081329FB555DE6EF460237DFF8BE4BA", + /* cofactor */ 1, + /* OID */ "1.2.840.10045.3.1.5" +}, +#endif +#ifdef LTC_ECC_PRIME239V3 +{ + /* prime */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFFFFFFFF8000000000007FFFFFFFFFFF", + /* A */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFFFFFFFF8000000000007FFFFFFFFFFC", + /* B */ "255705FA2A306654B1F4CB03D6A750A30C250102D4988717D9BA15AB6D3E", + /* order */ "7FFFFFFFFFFFFFFFFFFFFFFF7FFFFF975DEB41B3A6057C3C432146526551", + /* Gx */ "6768AE8E18BB92CFCF005C949AA2C6D94853D0E660BBF854B1C9505FE95A", + /* Gy */ "1607E6898F390C06BC1D552BAD226F3B6FCFE48B6E818499AF18E3ED6CF3", + /* cofactor */ 1, + /* OID */ "1.2.840.10045.3.1.6" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP160R1 +{ + /* prime */ "E95E4A5F737059DC60DFC7AD95B3D8139515620F", + /* A */ "340E7BE2A280EB74E2BE61BADA745D97E8F7C300", + /* B */ "1E589A8595423412134FAA2DBDEC95C8D8675E58", + /* order */ "E95E4A5F737059DC60DF5991D45029409E60FC09", + /* Gx */ "BED5AF16EA3F6A4F62938C4631EB5AF7BDBCDBC3", + /* Gy */ "1667CB477A1A8EC338F94741669C976316DA6321", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.1" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP192R1 +{ + /* prime */ "C302F41D932A36CDA7A3463093D18DB78FCE476DE1A86297", + /* A */ "6A91174076B1E0E19C39C031FE8685C1CAE040E5C69A28EF", + /* B */ "469A28EF7C28CCA3DC721D044F4496BCCA7EF4146FBF25C9", + /* order */ "C302F41D932A36CDA7A3462F9E9E916B5BE8F1029AC4ACC1", + /* Gx */ "C0A0647EAAB6A48753B033C56CB0F0900A2F5C4853375FD6", + /* Gy */ "14B690866ABD5BB88B5F4828C1490002E6773FA2FA299B8F", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.3" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP224R1 +{ + /* prime */ "D7C134AA264366862A18302575D1D787B09F075797DA89F57EC8C0FF", + /* A */ "68A5E62CA9CE6C1C299803A6C1530B514E182AD8B0042A59CAD29F43", + /* B */ "2580F63CCFE44138870713B1A92369E33E2135D266DBB372386C400B", + /* order */ "D7C134AA264366862A18302575D0FB98D116BC4B6DDEBCA3A5A7939F", + /* Gx */ "0D9029AD2C7E5CF4340823B2A87DC68C9E4CE3174C1E6EFDEE12C07D", + /* Gy */ "58AA56F772C0726F24C6B89E4ECDAC24354B9E99CAA3F6D3761402CD", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.5" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP256R1 +{ + /* prime */ "A9FB57DBA1EEA9BC3E660A909D838D726E3BF623D52620282013481D1F6E5377", + /* A */ "7D5A0975FC2C3057EEF67530417AFFE7FB8055C126DC5C6CE94A4B44F330B5D9", + /* B */ "26DC5C6CE94A4B44F330B5D9BBD77CBF958416295CF7E1CE6BCCDC18FF8C07B6", + /* order */ "A9FB57DBA1EEA9BC3E660A909D838D718C397AA3B561A6F7901E0E82974856A7", + /* Gx */ "8BD2AEB9CB7E57CB2C4B482FFC81B7AFB9DE27E1E3BD23C23A4453BD9ACE3262", + /* Gy */ "547EF835C3DAC4FD97F8461A14611DC9C27745132DED8E545C1D54C72F046997", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.7" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP320R1 +{ + /* prime */ "D35E472036BC4FB7E13C785ED201E065F98FCFA6F6F40DEF4F92B9EC7893EC28FCD412B1F1B32E27", + /* A */ "3EE30B568FBAB0F883CCEBD46D3F3BB8A2A73513F5EB79DA66190EB085FFA9F492F375A97D860EB4", + /* B */ "520883949DFDBC42D3AD198640688A6FE13F41349554B49ACC31DCCD884539816F5EB4AC8FB1F1A6", + /* order */ "D35E472036BC4FB7E13C785ED201E065F98FCFA5B68F12A32D482EC7EE8658E98691555B44C59311", + /* Gx */ "43BD7E9AFB53D8B85289BCC48EE5BFE6F20137D10A087EB6E7871E2A10A599C710AF8D0D39E20611", + /* Gy */ "14FDD05545EC1CC8AB4093247F77275E0743FFED117182EAA9C77877AAAC6AC7D35245D1692E8EE1", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.9" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP384R1 +{ + /* prime */ "8CB91E82A3386D280F5D6F7E50E641DF152F7109ED5456B412B1DA197FB71123ACD3A729901D1A71874700133107EC53", + /* A */ "7BC382C63D8C150C3C72080ACE05AFA0C2BEA28E4FB22787139165EFBA91F90F8AA5814A503AD4EB04A8C7DD22CE2826", + /* B */ "04A8C7DD22CE28268B39B55416F0447C2FB77DE107DCD2A62E880EA53EEB62D57CB4390295DBC9943AB78696FA504C11", + /* order */ "8CB91E82A3386D280F5D6F7E50E641DF152F7109ED5456B31F166E6CAC0425A7CF3AB6AF6B7FC3103B883202E9046565", + /* Gx */ "1D1C64F068CF45FFA2A63A81B7C13F6B8847A3E77EF14FE3DB7FCAFE0CBD10E8E826E03436D646AAEF87B2E247D4AF1E", + /* Gy */ "8ABE1D7520F9C2A45CB1EB8E95CFD55262B70B29FEEC5864E19C054FF99129280E4646217791811142820341263C5315", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.11" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP512R1 +{ + /* prime */ "AADD9DB8DBE9C48B3FD4E6AE33C9FC07CB308DB3B3C9D20ED6639CCA703308717D4D9B009BC66842AECDA12AE6A380E62881FF2F2D82C68528AA6056583A48F3", + /* A */ "7830A3318B603B89E2327145AC234CC594CBDD8D3DF91610A83441CAEA9863BC2DED5D5AA8253AA10A2EF1C98B9AC8B57F1117A72BF2C7B9E7C1AC4D77FC94CA", + /* B */ "3DF91610A83441CAEA9863BC2DED5D5AA8253AA10A2EF1C98B9AC8B57F1117A72BF2C7B9E7C1AC4D77FC94CADC083E67984050B75EBAE5DD2809BD638016F723", + /* order */ "AADD9DB8DBE9C48B3FD4E6AE33C9FC07CB308DB3B3C9D20ED6639CCA70330870553E5C414CA92619418661197FAC10471DB1D381085DDADDB58796829CA90069", + /* Gx */ "81AEE4BDD82ED9645A21322E9C4C6A9385ED9F70B5D916C1B43B62EEF4D0098EFF3B1F78E2D0D48D50D1687B93B97D5F7C6D5047406A5E688B352209BCB9F822", + /* Gy */ "7DDE385D566332ECC0EABFA9CF7822FDF209F70024A57B1AA000C55B881F8111B2DCDE494A5F485E5BCA4BD88A2763AED1CA2B2FA8F0540678CD1E0F3AD80892", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.13" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP160T1 +{ + /* prime */ "E95E4A5F737059DC60DFC7AD95B3D8139515620F", + /* A */ "E95E4A5F737059DC60DFC7AD95B3D8139515620C", + /* B */ "7A556B6DAE535B7B51ED2C4D7DAA7A0B5C55F380", + /* order */ "E95E4A5F737059DC60DF5991D45029409E60FC09", + /* Gx */ "B199B13B9B34EFC1397E64BAEB05ACC265FF2378", + /* Gy */ "ADD6718B7C7C1961F0991B842443772152C9E0AD", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.2" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP192T1 +{ + /* prime */ "C302F41D932A36CDA7A3463093D18DB78FCE476DE1A86297", + /* A */ "C302F41D932A36CDA7A3463093D18DB78FCE476DE1A86294", + /* B */ "13D56FFAEC78681E68F9DEB43B35BEC2FB68542E27897B79", + /* order */ "C302F41D932A36CDA7A3462F9E9E916B5BE8F1029AC4ACC1", + /* Gx */ "3AE9E58C82F63C30282E1FE7BBF43FA72C446AF6F4618129", + /* Gy */ "097E2C5667C2223A902AB5CA449D0084B7E5B3DE7CCC01C9", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.4" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP224T1 +{ + /* prime */ "D7C134AA264366862A18302575D1D787B09F075797DA89F57EC8C0FF", + /* A */ "D7C134AA264366862A18302575D1D787B09F075797DA89F57EC8C0FC", + /* B */ "4B337D934104CD7BEF271BF60CED1ED20DA14C08B3BB64F18A60888D", + /* order */ "D7C134AA264366862A18302575D0FB98D116BC4B6DDEBCA3A5A7939F", + /* Gx */ "6AB1E344CE25FF3896424E7FFE14762ECB49F8928AC0C76029B4D580", + /* Gy */ "0374E9F5143E568CD23F3F4D7C0D4B1E41C8CC0D1C6ABD5F1A46DB4C", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.6" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP256T1 +{ + /* prime */ "A9FB57DBA1EEA9BC3E660A909D838D726E3BF623D52620282013481D1F6E5377", + /* A */ "A9FB57DBA1EEA9BC3E660A909D838D726E3BF623D52620282013481D1F6E5374", + /* B */ "662C61C430D84EA4FE66A7733D0B76B7BF93EBC4AF2F49256AE58101FEE92B04", + /* order */ "A9FB57DBA1EEA9BC3E660A909D838D718C397AA3B561A6F7901E0E82974856A7", + /* Gx */ "A3E8EB3CC1CFE7B7732213B23A656149AFA142C47AAFBC2B79A191562E1305F4", + /* Gy */ "2D996C823439C56D7F7B22E14644417E69BCB6DE39D027001DABE8F35B25C9BE", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.8" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP320T1 +{ + /* prime */ "D35E472036BC4FB7E13C785ED201E065F98FCFA6F6F40DEF4F92B9EC7893EC28FCD412B1F1B32E27", + /* A */ "D35E472036BC4FB7E13C785ED201E065F98FCFA6F6F40DEF4F92B9EC7893EC28FCD412B1F1B32E24", + /* B */ "A7F561E038EB1ED560B3D147DB782013064C19F27ED27C6780AAF77FB8A547CEB5B4FEF422340353", + /* order */ "D35E472036BC4FB7E13C785ED201E065F98FCFA5B68F12A32D482EC7EE8658E98691555B44C59311", + /* Gx */ "925BE9FB01AFC6FB4D3E7D4990010F813408AB106C4F09CB7EE07868CC136FFF3357F624A21BED52", + /* Gy */ "63BA3A7A27483EBF6671DBEF7ABB30EBEE084E58A0B077AD42A5A0989D1EE71B1B9BC0455FB0D2C3", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.10" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP384T1 +{ + /* prime */ "8CB91E82A3386D280F5D6F7E50E641DF152F7109ED5456B412B1DA197FB71123ACD3A729901D1A71874700133107EC53", + /* A */ "8CB91E82A3386D280F5D6F7E50E641DF152F7109ED5456B412B1DA197FB71123ACD3A729901D1A71874700133107EC50", + /* B */ "7F519EADA7BDA81BD826DBA647910F8C4B9346ED8CCDC64E4B1ABD11756DCE1D2074AA263B88805CED70355A33B471EE", + /* order */ "8CB91E82A3386D280F5D6F7E50E641DF152F7109ED5456B31F166E6CAC0425A7CF3AB6AF6B7FC3103B883202E9046565", + /* Gx */ "18DE98B02DB9A306F2AFCD7235F72A819B80AB12EBD653172476FECD462AABFFC4FF191B946A5F54D8D0AA2F418808CC", + /* Gy */ "25AB056962D30651A114AFD2755AD336747F93475B7A1FCA3B88F2B6A208CCFE469408584DC2B2912675BF5B9E582928", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.12" +}, +#endif +#ifdef LTC_ECC_BRAINPOOLP512T1 +{ + /* prime */ "AADD9DB8DBE9C48B3FD4E6AE33C9FC07CB308DB3B3C9D20ED6639CCA703308717D4D9B009BC66842AECDA12AE6A380E62881FF2F2D82C68528AA6056583A48F3", + /* A */ "AADD9DB8DBE9C48B3FD4E6AE33C9FC07CB308DB3B3C9D20ED6639CCA703308717D4D9B009BC66842AECDA12AE6A380E62881FF2F2D82C68528AA6056583A48F0", + /* B */ "7CBBBCF9441CFAB76E1890E46884EAE321F70C0BCB4981527897504BEC3E36A62BCDFA2304976540F6450085F2DAE145C22553B465763689180EA2571867423E", + /* order */ "AADD9DB8DBE9C48B3FD4E6AE33C9FC07CB308DB3B3C9D20ED6639CCA70330870553E5C414CA92619418661197FAC10471DB1D381085DDADDB58796829CA90069", + /* Gx */ "640ECE5C12788717B9C1BA06CBC2A6FEBA85842458C56DDE9DB1758D39C0313D82BA51735CDB3EA499AA77A7D6943A64F7A3F25FE26F06B51BAA2696FA9035DA", + /* Gy */ "5B534BD595F5AF0FA2C892376C84ACE1BB4E3019B71634C01131159CAE03CEE9D9932184BEEF216BD71DF2DADF86A627306ECFF96DBB8BACE198B61E00F8B332", + /* cofactor */ 1, + /* OID */ "1.3.36.3.3.2.8.1.1.14" +}, +#endif +#ifdef LTC_ECC_SM2 +{ + /* prime */ "FFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00000000FFFFFFFFFFFFFFFF", + /* A */ "FFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00000000FFFFFFFFFFFFFFFC", + /* B */ "28E9FA9E9D9F5E344D5A9E4BCF6509A7F39789F515AB8F92DDBCBD414D940E93", + /* order */ "FFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFF7203DF6B21C6052B53BBF40939D54123", + /* Gx */ "32C4AE2C1F1981195F9904466A39C9948FE30BBFF2660BE1715A4589334C74C7", + /* Gy */ "BC3736A2F4F6779C59BDCEE36B692153D0A9877CC62A474002DF32E52139F0A0", + /* cofactor */ 1, + /* OID */ "1.2.156.10197.1.301" +}, +#endif +{ + NULL, NULL, NULL, NULL, NULL, NULL, 0, NULL +} +}; + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ansi_x963_export.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ansi_x963_export.c new file mode 100644 index 0000000..472e1b5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ansi_x963_export.c @@ -0,0 +1,24 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_ansi_x963_export.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** ECC X9.63 (Sec. 4.3.6) uncompressed export + @param key Key to export + @param out [out] destination of export + @param outlen [in/out] Length of destination and final output size + Return CRYPT_OK on success +*/ +int ecc_ansi_x963_export(const ecc_key *key, unsigned char *out, unsigned long *outlen) +{ + return ecc_get_key(out, outlen, PK_PUBLIC, key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ansi_x963_import.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ansi_x963_import.c new file mode 100644 index 0000000..58eb27d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ansi_x963_import.c @@ -0,0 +1,52 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_ansi_x963_import.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** Import an ANSI X9.63 format public key + @param in The input data to read + @param inlen The length of the input data + @param key [out] destination to store imported key \ +*/ +int ecc_ansi_x963_import(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + return ecc_ansi_x963_import_ex(in, inlen, key, NULL); +} + +int ecc_ansi_x963_import_ex(const unsigned char *in, unsigned long inlen, ecc_key *key, const ltc_ecc_curve *cu) +{ + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + + /* must be odd */ + if ((inlen & 1) == 0) { + return CRYPT_INVALID_ARG; + } + + /* initialize key->dp */ + if (cu == NULL) { + /* this case works only for uncompressed public keys */ + if ((err = ecc_set_curve_by_size((inlen-1)>>1, key)) != CRYPT_OK) { return err; } + } + else { + /* this one works for both compressed / uncompressed pubkeys */ + if ((err = ecc_set_curve(cu, key)) != CRYPT_OK) { return err; } + } + + /* load public key */ + if ((err = ecc_set_key((unsigned char *)in, inlen, PK_PUBLIC, key)) != CRYPT_OK) { return err; } + + /* we're done */ + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_decrypt_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_decrypt_key.c new file mode 100644 index 0000000..ad2d9b1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_decrypt_key.c @@ -0,0 +1,133 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_decrypt_key.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Decrypt an ECC encrypted key + @param in The ciphertext + @param inlen The length of the ciphertext (octets) + @param out [out] The plaintext + @param outlen [in/out] The max size and resulting size of the plaintext + @param key The corresponding private ECC key + @return CRYPT_OK if successful +*/ +int ecc_decrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const ecc_key *key) +{ + unsigned char *ecc_shared, *skey, *pub_expt; + unsigned long x, y; + unsigned long hashOID[32] = { 0 }; + int hash, err; + ecc_key pubkey; + ltc_asn1_list decode[3]; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* right key type? */ + if (key->type != PK_PRIVATE) { + return CRYPT_PK_NOT_PRIVATE; + } + + /* decode to find out hash */ + LTC_SET_ASN1(decode, 0, LTC_ASN1_OBJECT_IDENTIFIER, hashOID, sizeof(hashOID)/sizeof(hashOID[0])); + err = der_decode_sequence(in, inlen, decode, 1); + if (err != CRYPT_OK && err != CRYPT_INPUT_TOO_LONG) { + return err; + } + + hash = find_hash_oid(hashOID, decode[0].size); + if (hash_is_valid(hash) != CRYPT_OK) { + return CRYPT_INVALID_PACKET; + } + + /* we now have the hash! */ + + /* allocate memory */ + pub_expt = XMALLOC(ECC_BUF_SIZE); + ecc_shared = XMALLOC(ECC_BUF_SIZE); + skey = XMALLOC(MAXBLOCKSIZE); + if (pub_expt == NULL || ecc_shared == NULL || skey == NULL) { + if (pub_expt != NULL) { + XFREE(pub_expt); + } + if (ecc_shared != NULL) { + XFREE(ecc_shared); + } + if (skey != NULL) { + XFREE(skey); + } + return CRYPT_MEM; + } + LTC_SET_ASN1(decode, 1, LTC_ASN1_OCTET_STRING, pub_expt, ECC_BUF_SIZE); + LTC_SET_ASN1(decode, 2, LTC_ASN1_OCTET_STRING, skey, MAXBLOCKSIZE); + + /* read the structure in now */ + if ((err = der_decode_sequence(in, inlen, decode, 3)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* import ECC key from packet */ + if ((err = ecc_copy_curve(key, &pubkey)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = ecc_set_key(decode[1].data, decode[1].size, PK_PUBLIC, &pubkey)) != CRYPT_OK) { goto LBL_ERR; } + + /* make shared key */ + x = ECC_BUF_SIZE; + if ((err = ecc_shared_secret(key, &pubkey, ecc_shared, &x)) != CRYPT_OK) { + ecc_free(&pubkey); + goto LBL_ERR; + } + ecc_free(&pubkey); + + y = MIN(ECC_BUF_SIZE, MAXBLOCKSIZE); + if ((err = hash_memory(hash, ecc_shared, x, ecc_shared, &y)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* ensure the hash of the shared secret is at least as big as the encrypt itself */ + if (decode[2].size > y) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + /* avoid buffer overflow */ + if (*outlen < decode[2].size) { + *outlen = decode[2].size; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + /* Decrypt the key */ + for (x = 0; x < decode[2].size; x++) { + out[x] = skey[x] ^ ecc_shared[x]; + } + *outlen = x; + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(pub_expt, ECC_BUF_SIZE); + zeromem(ecc_shared, ECC_BUF_SIZE); + zeromem(skey, MAXBLOCKSIZE); +#endif + + XFREE(pub_expt); + XFREE(ecc_shared); + XFREE(skey); + + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_encrypt_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_encrypt_key.c new file mode 100644 index 0000000..be24253 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_encrypt_key.c @@ -0,0 +1,120 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_encrypt_key.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Encrypt a symmetric key with ECC + @param in The symmetric key you want to encrypt + @param inlen The length of the key to encrypt (octets) + @param out [out] The destination for the ciphertext + @param outlen [in/out] The max size and resulting size of the ciphertext + @param prng An active PRNG state + @param wprng The index of the PRNG you wish to use + @param hash The index of the hash you want to use + @param key The ECC key you want to encrypt to + @return CRYPT_OK if successful +*/ +int ecc_encrypt_key(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, int hash, + const ecc_key *key) +{ + unsigned char *pub_expt, *ecc_shared, *skey; + ecc_key pubkey; + unsigned long x, y, pubkeysize; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + if ((err = hash_is_valid(hash)) != CRYPT_OK) { + return err; + } + + if (inlen > hash_descriptor[hash]->hashsize) { + return CRYPT_INVALID_HASH; + } + + /* make a random key and export the public copy */ + if ((err = ecc_copy_curve(key, &pubkey)) != CRYPT_OK) { return err; } + if ((err = ecc_generate_key(prng, wprng, &pubkey)) != CRYPT_OK) { return err; } + + pub_expt = XMALLOC(ECC_BUF_SIZE); + ecc_shared = XMALLOC(ECC_BUF_SIZE); + skey = XMALLOC(MAXBLOCKSIZE); + if (pub_expt == NULL || ecc_shared == NULL || skey == NULL) { + if (pub_expt != NULL) { + XFREE(pub_expt); + } + if (ecc_shared != NULL) { + XFREE(ecc_shared); + } + if (skey != NULL) { + XFREE(skey); + } + ecc_free(&pubkey); + return CRYPT_MEM; + } + + pubkeysize = ECC_BUF_SIZE; + if (ltc_mp.sqrtmod_prime != NULL) { + /* PK_COMPRESSED requires sqrtmod_prime */ + err = ecc_get_key(pub_expt, &pubkeysize, PK_PUBLIC|PK_COMPRESSED, &pubkey); + } + else { + err = ecc_get_key(pub_expt, &pubkeysize, PK_PUBLIC, &pubkey); + } + if (err != CRYPT_OK) { + ecc_free(&pubkey); + goto LBL_ERR; + } + + /* make random key */ + x = ECC_BUF_SIZE; + if ((err = ecc_shared_secret(&pubkey, key, ecc_shared, &x)) != CRYPT_OK) { + ecc_free(&pubkey); + goto LBL_ERR; + } + ecc_free(&pubkey); + y = MAXBLOCKSIZE; + if ((err = hash_memory(hash, ecc_shared, x, skey, &y)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* Encrypt key */ + for (x = 0; x < inlen; x++) { + skey[x] ^= in[x]; + } + + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_OBJECT_IDENTIFIER, hash_descriptor[hash]->OIDlen, hash_descriptor[hash]->OID, + LTC_ASN1_OCTET_STRING, pubkeysize, pub_expt, + LTC_ASN1_OCTET_STRING, inlen, skey, + LTC_ASN1_EOL, 0UL, NULL); + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + /* clean up */ + zeromem(pub_expt, ECC_BUF_SIZE); + zeromem(ecc_shared, ECC_BUF_SIZE); + zeromem(skey, MAXBLOCKSIZE); +#endif + + XFREE(skey); + XFREE(ecc_shared); + XFREE(pub_expt); + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_export.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_export.c new file mode 100644 index 0000000..edbe4c3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_export.c @@ -0,0 +1,61 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_export.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Export an ECC key as a binary packet + @param out [out] Destination for the key + @param outlen [in/out] Max size and resulting size of the exported key + @param type The type of key you want to export (PK_PRIVATE or PK_PUBLIC) + @param key The key to export + @return CRYPT_OK if successful +*/ +int ecc_export(unsigned char *out, unsigned long *outlen, int type, const ecc_key *key) +{ + int err; + unsigned char flags[1]; + unsigned long key_size; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* type valid? */ + if (key->type != PK_PRIVATE && type == PK_PRIVATE) { + return CRYPT_PK_TYPE_MISMATCH; + } + + /* we store the NIST byte size */ + key_size = key->dp.size; + + if (type == PK_PRIVATE) { + flags[0] = 1; + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_SHORT_INTEGER, 1UL, &key_size, + LTC_ASN1_INTEGER, 1UL, key->pubkey.x, + LTC_ASN1_INTEGER, 1UL, key->pubkey.y, + LTC_ASN1_INTEGER, 1UL, key->k, + LTC_ASN1_EOL, 0UL, NULL); + } else { + flags[0] = 0; + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_SHORT_INTEGER, 1UL, &key_size, + LTC_ASN1_INTEGER, 1UL, key->pubkey.x, + LTC_ASN1_INTEGER, 1UL, key->pubkey.y, + LTC_ASN1_EOL, 0UL, NULL); + } + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_export_openssl.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_export_openssl.c new file mode 100644 index 0000000..2e89996 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_export_openssl.c @@ -0,0 +1,167 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +/** + Export an ECC key as a binary packet + @param out [out] Destination for the key + @param outlen [in/out] Max size and resulting size of the exported key + @param type The type of key you want to export (PK_PRIVATE or PK_PUBLIC) + @param key The key to export + @return CRYPT_OK if successful +*/ + +int ecc_export_openssl(unsigned char *out, unsigned long *outlen, int type, const ecc_key *key) +{ + int err; + void *prime, *order, *a, *b, *gx, *gy; + unsigned char bin_a[256], bin_b[256], bin_k[256], bin_g[512], bin_xy[512]; + unsigned long len_a, len_b, len_k, len_g, len_xy; + unsigned long cofactor, one = 1; + const char *OID; + unsigned long oid[16], oidlen; + ltc_asn1_list seq_fieldid[2], seq_curve[2], seq_ecparams[6], seq_priv[4], pub_xy, ecparams; + int flag_oid = type & PK_CURVEOID ? 1 : 0; + int flag_com = type & PK_COMPRESSED ? 1 : 0; + int flag_pri = type & PK_PRIVATE ? 1 : 0; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + if (key->type != PK_PRIVATE && flag_pri) return CRYPT_PK_TYPE_MISMATCH; + + if (flag_oid) { + /* http://tools.ietf.org/html/rfc5912 + ECParameters ::= CHOICE { + namedCurve CURVE.&id({NamedCurve}) # OBJECT + } + */ + if (key->dp.oidlen == 0) { err = CRYPT_INVALID_ARG; goto error; } + LTC_SET_ASN1(&ecparams, 0, LTC_ASN1_OBJECT_IDENTIFIER, key->dp.oid, key->dp.oidlen); + } + else { + prime = key->dp.prime; + order = key->dp.order; + a = key->dp.A; + b = key->dp.B; + gx = key->dp.base.x; + gy = key->dp.base.y; + cofactor = key->dp.cofactor; + + /* curve param a */ + len_a = mp_unsigned_bin_size(a); + if (len_a > sizeof(bin_a)) { err = CRYPT_BUFFER_OVERFLOW; goto error; } + if ((err = mp_to_unsigned_bin(a, bin_a)) != CRYPT_OK) { goto error; } + if (len_a == 0) { len_a = 1; bin_a[0] = 0; } /* handle case a == 0 */ + + /* curve param b */ + len_b = mp_unsigned_bin_size(b); + if (len_b > sizeof(bin_b)) { err = CRYPT_BUFFER_OVERFLOW; goto error; } + if ((err = mp_to_unsigned_bin(b, bin_b)) != CRYPT_OK) { goto error; } + if (len_b == 0) { len_b = 1; bin_b[0] = 0; } /* handle case b == 0 */ + + /* base point - (un)compressed based on flag_com */ + len_g = sizeof(bin_g); + err = ltc_ecc_export_point(bin_g, &len_g, gx, gy, key->dp.size, flag_com); + if (err != CRYPT_OK) { goto error; } + + /* we support only prime-field EC */ + if ((err = pk_get_oid(LTC_OID_EC_PRIMEF, &OID)) != CRYPT_OK) { goto error; } + + /* http://tools.ietf.org/html/rfc3279 + ECParameters ::= SEQUENCE { # SEQUENCE + version INTEGER { ecpVer1(1) } (ecpVer1) # INTEGER :01 + FieldID ::= SEQUENCE { # SEQUENCE + fieldType FIELD-ID.&id({IOSet}), # OBJECT :prime-field + parameters FIELD-ID.&Type({IOSet}{@fieldType}) # INTEGER + } + Curve ::= SEQUENCE { # SEQUENCE + a FieldElement ::= OCTET STRING # OCTET STRING + b FieldElement ::= OCTET STRING # OCTET STRING + seed BIT STRING OPTIONAL + } + base ECPoint ::= OCTET STRING # OCTET STRING + order INTEGER, # INTEGER + cofactor INTEGER OPTIONAL # INTEGER + } + */ + + oidlen = sizeof(oid)/sizeof(oid[0]); + if ((err = pk_oid_str_to_num(OID, oid, &oidlen)) != CRYPT_OK) { + goto error; + } + + /* FieldID SEQUENCE */ + LTC_SET_ASN1(seq_fieldid, 0, LTC_ASN1_OBJECT_IDENTIFIER, oid, oidlen); + LTC_SET_ASN1(seq_fieldid, 1, LTC_ASN1_INTEGER, prime, 1UL); + + /* Curve SEQUENCE */ + LTC_SET_ASN1(seq_curve, 0, LTC_ASN1_OCTET_STRING, bin_a, len_a); + LTC_SET_ASN1(seq_curve, 1, LTC_ASN1_OCTET_STRING, bin_b, len_b); + + /* ECParameters SEQUENCE */ + LTC_SET_ASN1(seq_ecparams, 0, LTC_ASN1_SHORT_INTEGER, &one, 1UL); + LTC_SET_ASN1(seq_ecparams, 1, LTC_ASN1_SEQUENCE, seq_fieldid, 2UL); + LTC_SET_ASN1(seq_ecparams, 2, LTC_ASN1_SEQUENCE, seq_curve, 2UL); + LTC_SET_ASN1(seq_ecparams, 3, LTC_ASN1_OCTET_STRING, bin_g, len_g); + LTC_SET_ASN1(seq_ecparams, 4, LTC_ASN1_INTEGER, order, 1UL); + LTC_SET_ASN1(seq_ecparams, 5, LTC_ASN1_SHORT_INTEGER, &cofactor, 1UL); + + /* ECParameters used by ECPrivateKey or SubjectPublicKeyInfo below */ + LTC_SET_ASN1(&ecparams, 0, LTC_ASN1_SEQUENCE, seq_ecparams, 6UL); + } + + /* public key - (un)compressed based on flag_com */ + len_xy = sizeof(bin_xy); + err = ltc_ecc_export_point(bin_xy, &len_xy, key->pubkey.x, key->pubkey.y, key->dp.size, flag_com); + if (err != CRYPT_OK) { + goto error; + } + + if (flag_pri) { + + /* http://tools.ietf.org/html/rfc5915 + ECPrivateKey ::= SEQUENCE { # SEQUENCE + version INTEGER { ecPrivkeyVer1(1) } (ecPrivkeyVer1) # INTEGER :01 + privateKey OCTET STRING, # OCTET STRING + [0] ECParameters # see above + [1] publicKey # BIT STRING + } + */ + + /* private key */ + len_k = mp_unsigned_bin_size(key->k); + if (len_k > sizeof(bin_k)) { err = CRYPT_BUFFER_OVERFLOW; goto error; } + if ((err = mp_to_unsigned_bin(key->k, bin_k)) != CRYPT_OK) { goto error; } + + LTC_SET_ASN1(&pub_xy, 0, LTC_ASN1_RAW_BIT_STRING, bin_xy, 8*len_xy); + LTC_SET_ASN1(seq_priv, 0, LTC_ASN1_SHORT_INTEGER, &one, 1); + LTC_SET_ASN1(seq_priv, 1, LTC_ASN1_OCTET_STRING, bin_k, len_k); + LTC_SET_ASN1_CUSTOM_CONSTRUCTED(seq_priv, 2, LTC_ASN1_CL_CONTEXT_SPECIFIC, 0, &ecparams); /* context specific 0 */ + LTC_SET_ASN1_CUSTOM_CONSTRUCTED(seq_priv, 3, LTC_ASN1_CL_CONTEXT_SPECIFIC, 1, &pub_xy); /* context specific 1 */ + + err = der_encode_sequence(seq_priv, 4, out, outlen); + } + else { + /* http://tools.ietf.org/html/rfc5480 + SubjectPublicKeyInfo ::= SEQUENCE { # SEQUENCE + AlgorithmIdentifier ::= SEQUENCE { # SEQUENCE + algorithm OBJECT IDENTIFIER # OBJECT :id-ecPublicKey + ECParameters # see above + } + subjectPublicKey BIT STRING # BIT STRING + } + */ + err = x509_encode_subject_public_key_info( out, outlen, LTC_OID_EC, bin_xy, len_xy, + ecparams.type, ecparams.data, ecparams.size ); + } + +error: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_find_curve.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_find_curve.c new file mode 100644 index 0000000..3a01f9f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_find_curve.c @@ -0,0 +1,247 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +static const struct { + const char *OID; + const char *names[6]; +} s_curve_names[] = { +#ifdef LTC_ECC_SECP112R1 + { + "1.3.132.0.6", { "SECP112R1", "ECC-112", NULL } + }, +#endif +#ifdef LTC_ECC_SECP112R2 + { + "1.3.132.0.7", { "SECP112R2", NULL } + }, +#endif +#ifdef LTC_ECC_SECP128R1 + { + "1.3.132.0.28", { "SECP128R1", "ECC-128", NULL } + }, +#endif +#ifdef LTC_ECC_SECP128R2 + { + "1.3.132.0.29", { "SECP128R2", NULL } + }, +#endif +#ifdef LTC_ECC_SECP160R1 + { + "1.3.132.0.8", { "SECP160R1", "ECC-160", NULL } + }, +#endif +#ifdef LTC_ECC_SECP160R2 + { + "1.3.132.0.30", { "SECP160R2", NULL } + }, +#endif +#ifdef LTC_ECC_SECP160K1 + { + "1.3.132.0.9", { "SECP160K1", NULL } + }, +#endif +#ifdef LTC_ECC_SECP192R1 + { + "1.2.840.10045.3.1.1", { "SECP192R1", "NISTP192", "PRIME192V1", "ECC-192", "P-192", NULL } + }, +#endif +#ifdef LTC_ECC_PRIME192V2 + { + "1.2.840.10045.3.1.2", { "PRIME192V2", NULL } + }, +#endif +#ifdef LTC_ECC_PRIME192V3 + { + "1.2.840.10045.3.1.3", { "PRIME192V3", NULL } + }, +#endif +#ifdef LTC_ECC_SECP192K1 + { + "1.3.132.0.31", { "SECP192K1", NULL } + }, +#endif +#ifdef LTC_ECC_SECP224R1 + { + "1.3.132.0.33", { "SECP224R1", "NISTP224", "ECC-224", "P-224", NULL } + }, +#endif +#ifdef LTC_ECC_SECP224K1 + { + "1.3.132.0.32", { "SECP224K1", NULL } + }, +#endif +#ifdef LTC_ECC_SECP256R1 + { + "1.2.840.10045.3.1.7", { "SECP256R1", "NISTP256", "PRIME256V1", "ECC-256", "P-256", NULL } + }, +#endif +#ifdef LTC_ECC_SECP256K1 + { + "1.3.132.0.10", { "SECP256K1", NULL } + }, +#endif +#ifdef LTC_ECC_SECP384R1 + { + "1.3.132.0.34", { "SECP384R1", "NISTP384", "ECC-384", "P-384", NULL } + }, +#endif +#ifdef LTC_ECC_SECP521R1 + { + "1.3.132.0.35", { "SECP521R1", "NISTP521", "ECC-521", "P-521", NULL } + }, +#endif +#ifdef LTC_ECC_PRIME239V1 + { + "1.2.840.10045.3.1.4", { "PRIME239V1", NULL } + }, +#endif +#ifdef LTC_ECC_PRIME239V2 + { + "1.2.840.10045.3.1.5", { "PRIME239V2", NULL } + }, +#endif +#ifdef LTC_ECC_PRIME239V3 + { + "1.2.840.10045.3.1.6", { "PRIME239V3", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP160R1 + { + "1.3.36.3.3.2.8.1.1.1", { "BRAINPOOLP160R1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP192R1 + { + "1.3.36.3.3.2.8.1.1.3", { "BRAINPOOLP192R1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP224R1 + { + "1.3.36.3.3.2.8.1.1.5", { "BRAINPOOLP224R1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP256R1 + { + "1.3.36.3.3.2.8.1.1.7", { "BRAINPOOLP256R1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP320R1 + { + "1.3.36.3.3.2.8.1.1.9", { "BRAINPOOLP320R1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP384R1 + { + "1.3.36.3.3.2.8.1.1.11", { "BRAINPOOLP384R1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP512R1 + { + "1.3.36.3.3.2.8.1.1.13", { "BRAINPOOLP512R1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP160T1 + { + "1.3.36.3.3.2.8.1.1.2", { "BRAINPOOLP160T1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP192T1 + { + "1.3.36.3.3.2.8.1.1.4", { "BRAINPOOLP192T1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP224T1 + { + "1.3.36.3.3.2.8.1.1.6", { "BRAINPOOLP224T1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP256T1 + { + "1.3.36.3.3.2.8.1.1.8", { "BRAINPOOLP256T1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP320T1 + { + "1.3.36.3.3.2.8.1.1.10", { "BRAINPOOLP320T1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP384T1 + { + "1.3.36.3.3.2.8.1.1.12", { "BRAINPOOLP384T1", NULL } + }, +#endif +#ifdef LTC_ECC_BRAINPOOLP512T1 + { + "1.3.36.3.3.2.8.1.1.14", { "BRAINPOOLP512T1", NULL } + }, +#endif +#ifdef LTC_ECC_SM2 + { + "1.2.156.10197.1.301", { "SM2", NULL } + }, +#endif + { + NULL, { NULL } + } +}; + +/* case-insensitive match + ignore '-', '_', ' ' */ +static int s_name_match(const char *left, const char *right) +{ + char lc_r, lc_l; + + while ((*left != '\0') && (*right != '\0')) { + while ((*left == ' ') || (*left == '-') || (*left == '_')) left++; + while ((*right == ' ') || (*right == '-') || (*right == '_')) right++; + if (*left == '\0' || *right == '\0') break; + lc_r = *right; + lc_l = *left; + if ((lc_r >= 'A') && (lc_r <= 'Z')) lc_r += 32; + if ((lc_l >= 'A') && (lc_l <= 'Z')) lc_l += 32; + if (lc_l != lc_r) return 0; + left++; + right++; + } + + if ((*left == '\0') && (*right == '\0')) return 1; + return 0; +} + +int ecc_find_curve(const char *name_or_oid, const ltc_ecc_curve **cu) +{ + int i, j; + const char *OID = NULL; + + LTC_ARGCHK(cu != NULL); + LTC_ARGCHK(name_or_oid != NULL); + + *cu = NULL; + + for (i = 0; s_curve_names[i].OID != NULL && !OID; i++) { + if (XSTRCMP(s_curve_names[i].OID, name_or_oid) == 0) { + OID = s_curve_names[i].OID; + } + for (j = 0; s_curve_names[i].names[j] != NULL && !OID; j++) { + if (s_name_match(s_curve_names[i].names[j], name_or_oid)) { + OID = s_curve_names[i].OID; + } + } + } + + if (OID != NULL) { + for (i = 0; ltc_ecc_curves[i].prime != NULL; i++) { + if (XSTRCMP(ltc_ecc_curves[i].OID, OID) == 0) { + *cu = <c_ecc_curves[i]; + return CRYPT_OK; + } + } + } + + return CRYPT_INVALID_ARG; /* not found */ +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_free.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_free.c new file mode 100644 index 0000000..6133109 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_free.c @@ -0,0 +1,28 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_free.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Free an ECC key from memory + @param key The key you wish to free +*/ +void ecc_free(ecc_key *key) +{ + LTC_ARGCHKVD(key != NULL); + + mp_cleanup_multi(&key->dp.prime, &key->dp.order, + &key->dp.A, &key->dp.B, + &key->dp.base.x, &key->dp.base.y, &key->dp.base.z, + &key->pubkey.x, &key->pubkey.y, &key->pubkey.z, + &key->k, NULL); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_key.c new file mode 100644 index 0000000..d30fd06 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_key.c @@ -0,0 +1,50 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +/** Export raw public or private key (public keys = ANS X9.63 compressed or uncompressed; private keys = raw bytes) + @param out [out] destination of export + @param outlen [in/out] Length of destination and final output size + @param type PK_PRIVATE, PK_PUBLIC or PK_PUBLIC|PK_COMPRESSED + @param key Key to export + Return CRYPT_OK on success +*/ + +int ecc_get_key(unsigned char *out, unsigned long *outlen, int type, const ecc_key *key) +{ + unsigned long size, ksize; + int err, compressed; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + size = key->dp.size; + compressed = type & PK_COMPRESSED ? 1 : 0; + type &= ~PK_COMPRESSED; + + if (type == PK_PUBLIC) { + if ((err = ltc_ecc_export_point(out, outlen, key->pubkey.x, key->pubkey.y, size, compressed)) != CRYPT_OK) { + return err; + } + } + else if (type == PK_PRIVATE) { + if (key->type != PK_PRIVATE) return CRYPT_PK_TYPE_MISMATCH; + *outlen = size; + if (size > *outlen) return CRYPT_BUFFER_OVERFLOW; + if ((ksize = mp_unsigned_bin_size(key->k)) > size) return CRYPT_BUFFER_OVERFLOW; + /* pad and store k */ + if ((err = mp_to_unsigned_bin(key->k, out + (size - ksize))) != CRYPT_OK) return err; + zeromem(out, size - ksize); + } + else { + return CRYPT_INVALID_ARG; + } + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_oid_str.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_oid_str.c new file mode 100644 index 0000000..756f641 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_oid_str.c @@ -0,0 +1,22 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +/** Extract OID as a string from ECC key + @param out [out] destination buffer + @param outlen [in/out] Length of destination buffer and final output size (without terminating NUL byte) + @param key The ECC key + Return CRYPT_OK on success +*/ + +int ecc_get_oid_str(char *out, unsigned long *outlen, const ecc_key *key) +{ + LTC_ARGCHK(key != NULL); + + return pk_oid_num_to_str(key->dp.oid, key->dp.oidlen, out, outlen); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_size.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_size.c new file mode 100644 index 0000000..e563339 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_get_size.c @@ -0,0 +1,26 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_get_size.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Get the size of an ECC key + @param key The key to get the size of + @return The size (octets) of the key or INT_MAX on error +*/ +int ecc_get_size(const ecc_key *key) +{ + if (key == NULL) { + return INT_MAX; + } + return key->dp.size; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import.c new file mode 100644 index 0000000..64bed21 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import.c @@ -0,0 +1,100 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_import.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Import an ECC key from a binary packet + @param in The packet to import + @param inlen The length of the packet + @param key [out] The destination of the import + @return CRYPT_OK if successful, upon error all allocated memory will be freed +*/ +int ecc_import(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + return ecc_import_ex(in, inlen, key, NULL); +} + +/** + Import an ECC key from a binary packet, using user supplied domain params rather than one of the NIST ones + @param in The packet to import + @param inlen The length of the packet + @param key [out] The destination of the import + @param cu pointer to user supplied params; must be the same as the params used when exporting + @return CRYPT_OK if successful, upon error all allocated memory will be freed +*/ +int ecc_import_ex(const unsigned char *in, unsigned long inlen, ecc_key *key, const ltc_ecc_curve *cu) +{ + unsigned long key_size; + unsigned char flags[1]; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* find out what type of key it is */ + err = der_decode_sequence_multi(in, inlen, LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_SHORT_INTEGER, 1UL, &key_size, + LTC_ASN1_EOL, 0UL, NULL); + if (err != CRYPT_OK && err != CRYPT_INPUT_TOO_LONG) { + return err; + } + + /* allocate & initialize the key */ + if (cu == NULL) { + if ((err = ecc_set_curve_by_size(key_size, key)) != CRYPT_OK) { goto done; } + } else { + if ((err = ecc_set_curve(cu, key)) != CRYPT_OK) { goto done; } + } + + if (flags[0] == 1) { + /* private key */ + key->type = PK_PRIVATE; + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_SHORT_INTEGER, 1UL, &key_size, + LTC_ASN1_INTEGER, 1UL, key->pubkey.x, + LTC_ASN1_INTEGER, 1UL, key->pubkey.y, + LTC_ASN1_INTEGER, 1UL, key->k, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto done; + } + } else if (flags[0] == 0) { + /* public key */ + key->type = PK_PUBLIC; + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_BIT_STRING, 1UL, flags, + LTC_ASN1_SHORT_INTEGER, 1UL, &key_size, + LTC_ASN1_INTEGER, 1UL, key->pubkey.x, + LTC_ASN1_INTEGER, 1UL, key->pubkey.y, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto done; + } + } + else { + err = CRYPT_INVALID_PACKET; + goto done; + } + + /* set z */ + if ((err = mp_set(key->pubkey.z, 1)) != CRYPT_OK) { goto done; } + + /* point on the curve + other checks */ + if ((err = ltc_ecc_verify_key(key)) != CRYPT_OK) { goto done; } + + /* we're good */ + return CRYPT_OK; + +done: + ecc_free(key); + return err; +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_openssl.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_openssl.c new file mode 100644 index 0000000..325b0b0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_openssl.c @@ -0,0 +1,120 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +static int s_ecc_import_private_with_oid(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + ltc_asn1_list seq_priv[4], custom[2]; + unsigned char bin_xy[2*ECC_MAXSIZE+2], bin_k[ECC_MAXSIZE]; + unsigned long len, pkver = 0, curveoid[16]; + char OID[256]; + const ltc_ecc_curve *curve; + int err; + + /* ### try to load private key - no curve parameters just curve OID */ + + /* ECPrivateKey SEQUENCE */ + LTC_SET_ASN1(custom, 0, LTC_ASN1_OBJECT_IDENTIFIER, curveoid, 16UL); + LTC_SET_ASN1(custom, 1, LTC_ASN1_RAW_BIT_STRING, bin_xy, 8UL*sizeof(bin_xy)); + LTC_SET_ASN1(seq_priv, 0, LTC_ASN1_SHORT_INTEGER, &pkver, 1UL); + LTC_SET_ASN1(seq_priv, 1, LTC_ASN1_OCTET_STRING, bin_k, sizeof(bin_k)); + LTC_SET_ASN1_CUSTOM_CONSTRUCTED(seq_priv, 2, LTC_ASN1_CL_CONTEXT_SPECIFIC, 0, custom); /* context specific 0 */ + LTC_SET_ASN1_CUSTOM_CONSTRUCTED(seq_priv, 3, LTC_ASN1_CL_CONTEXT_SPECIFIC, 1, custom + 1); /* context specific 1 */ + + /* try to load private key */ + err = der_decode_sequence(in, inlen, seq_priv, 4); + if (err == CRYPT_OK) { + /* load curve parameters for given curve OID */ + len = sizeof(OID); + if ((err = pk_oid_num_to_str(curveoid, custom[0].size, OID, &len)) != CRYPT_OK) { goto error; } + if ((err = ecc_find_curve(OID, &curve)) != CRYPT_OK) { goto error; } + if ((err = ecc_set_curve(curve, key)) != CRYPT_OK) { goto error; } + /* load private+public key */ + err = ecc_set_key(bin_k, seq_priv[1].size, PK_PRIVATE, key); + } +error: + return err; +} + +static int s_ecc_import_private_with_curve(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + void *prime, *order, *a, *b, *gx, *gy; + ltc_asn1_list seq_fieldid[2], seq_curve[3], seq_ecparams[6], seq_priv[4], custom[2]; + unsigned char bin_a[ECC_MAXSIZE], bin_b[ECC_MAXSIZE], bin_k[ECC_MAXSIZE]; + unsigned char bin_g[2*ECC_MAXSIZE+1], bin_xy[2*ECC_MAXSIZE+2], bin_seed[128]; + unsigned long len_a, len_b, len_k, len_g; + unsigned long cofactor = 0, ecver = 0, pkver = 0, tmpoid[16]; + int err; + + if ((err = mp_init_multi(&prime, &order, &a, &b, &gx, &gy, LTC_NULL)) != CRYPT_OK) { + return err; + } + + /* ### try to load private key - curve parameters included */ + + /* ECPrivateKey SEQUENCE */ + LTC_SET_ASN1(custom, 0, LTC_ASN1_SEQUENCE, seq_ecparams, 6UL); + LTC_SET_ASN1(custom, 1, LTC_ASN1_RAW_BIT_STRING, bin_xy, 8UL*sizeof(bin_xy)); + LTC_SET_ASN1(seq_priv, 0, LTC_ASN1_SHORT_INTEGER, &pkver, 1UL); + LTC_SET_ASN1(seq_priv, 1, LTC_ASN1_OCTET_STRING, bin_k, sizeof(bin_k)); + LTC_SET_ASN1_CUSTOM_CONSTRUCTED(seq_priv, 2, LTC_ASN1_CL_CONTEXT_SPECIFIC, 0, custom); /* context specific 0 */ + LTC_SET_ASN1_CUSTOM_CONSTRUCTED(seq_priv, 3, LTC_ASN1_CL_CONTEXT_SPECIFIC, 1, custom + 1); /* context specific 1 */ + /* ECParameters SEQUENCE */ + LTC_SET_ASN1(seq_ecparams, 0, LTC_ASN1_SHORT_INTEGER, &ecver, 1UL); + LTC_SET_ASN1(seq_ecparams, 1, LTC_ASN1_SEQUENCE, seq_fieldid, 2UL); + LTC_SET_ASN1(seq_ecparams, 2, LTC_ASN1_SEQUENCE, seq_curve, 3UL); + LTC_SET_ASN1(seq_ecparams, 3, LTC_ASN1_OCTET_STRING, bin_g, sizeof(bin_g)); + LTC_SET_ASN1(seq_ecparams, 4, LTC_ASN1_INTEGER, order, 1UL); + LTC_SET_ASN1(seq_ecparams, 5, LTC_ASN1_SHORT_INTEGER, &cofactor, 1UL); + seq_ecparams[5].optional = 1; + /* FieldID SEQUENCE */ + LTC_SET_ASN1(seq_fieldid, 0, LTC_ASN1_OBJECT_IDENTIFIER, tmpoid, 16UL); + LTC_SET_ASN1(seq_fieldid, 1, LTC_ASN1_INTEGER, prime, 1UL); + /* Curve SEQUENCE */ + LTC_SET_ASN1(seq_curve, 0, LTC_ASN1_OCTET_STRING, bin_a, sizeof(bin_a)); + LTC_SET_ASN1(seq_curve, 1, LTC_ASN1_OCTET_STRING, bin_b, sizeof(bin_b)); + LTC_SET_ASN1(seq_curve, 2, LTC_ASN1_RAW_BIT_STRING, bin_seed, 8UL*sizeof(bin_seed)); + seq_curve[2].optional = 1; + /* try to load private key */ + err = der_decode_sequence(in, inlen, seq_priv, 4); + if (err == CRYPT_OK) { + len_k = seq_priv[1].size; + len_a = seq_curve[0].size; + len_b = seq_curve[1].size; + len_g = seq_ecparams[3].size; + /* create bignums */ + if ((err = mp_read_unsigned_bin(a, bin_a, len_a)) != CRYPT_OK) { goto error; } + if ((err = mp_read_unsigned_bin(b, bin_b, len_b)) != CRYPT_OK) { goto error; } + if ((err = ltc_ecc_import_point(bin_g, len_g, prime, a, b, gx, gy)) != CRYPT_OK) { goto error; } + /* load curve parameters */ + if ((err = ecc_set_curve_from_mpis(a, b, prime, order, gx, gy, cofactor, key)) != CRYPT_OK) { goto error; } + /* load private+public key */ + err = ecc_set_key(bin_k, len_k, PK_PRIVATE, key); + } +error: + mp_clear_multi(prime, order, a, b, gx, gy, LTC_NULL); + return err; +} + +int ecc_import_openssl(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + int err; + + if ((err = ecc_import_subject_public_key_info(in, inlen, key)) == CRYPT_OK) { + goto success; + } + + if ((err = s_ecc_import_private_with_oid(in, inlen, key)) == CRYPT_OK) { + goto success; + } + + err = s_ecc_import_private_with_curve(in, inlen, key); + +success: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_pkcs8.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_pkcs8.c new file mode 100644 index 0000000..a6fd7bf --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_pkcs8.c @@ -0,0 +1,186 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +typedef struct { + ltc_asn1_type t; + ltc_asn1_list **pp; +} der_flexi_check; + +#define LTC_SET_DER_FLEXI_CHECK(list, index, Type, P) \ + do { \ + int LTC_SDFC_temp##__LINE__ = (index); \ + list[LTC_SDFC_temp##__LINE__].t = Type; \ + list[LTC_SDFC_temp##__LINE__].pp = P; \ + } while (0) + +static int s_der_flexi_sequence_cmp(const ltc_asn1_list *flexi, der_flexi_check *check) +{ + const ltc_asn1_list *cur; + if (flexi->type != LTC_ASN1_SEQUENCE) { + return CRYPT_INVALID_PACKET; + } + cur = flexi->child; + while(check->t != LTC_ASN1_EOL) { + if (!LTC_ASN1_IS_TYPE(cur, check->t)) { + return CRYPT_INVALID_PACKET; + } + if (check->pp != NULL) *check->pp = (ltc_asn1_list*)cur; + cur = cur->next; + check++; + } + return CRYPT_OK; +} + +/* NOTE: s_der_decode_pkcs8_flexi & related stuff can be shared with rsa_import_pkcs8() */ + +int ecc_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + ecc_key *key) +{ + void *a, *b, *gx, *gy; + unsigned long len, cofactor, n; + const char *pka_ec_oid; + int err; + char OID[256]; + const ltc_ecc_curve *curve; + ltc_asn1_list *p = NULL, *l = NULL; + der_flexi_check flexi_should[7]; + ltc_asn1_list *seq, *priv_key; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* get EC alg oid */ + err = pk_get_oid(LTC_OID_EC, &pka_ec_oid); + if (err != CRYPT_OK) return err; + + /* init key */ + err = mp_init_multi(&a, &b, &gx, &gy, LTC_NULL); + if (err != CRYPT_OK) return err; + + + if ((err = pkcs8_decode_flexi(in, inlen, pwd, pwdlen, &l)) == CRYPT_OK) { + + /* Setup for basic structure */ + n=0; + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_INTEGER, NULL); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_SEQUENCE, &seq); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_OCTET_STRING, &priv_key); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n, LTC_ASN1_EOL, NULL); + + if (((err = s_der_flexi_sequence_cmp(l, flexi_should)) == CRYPT_OK) && + (pk_oid_cmp_with_asn1(pka_ec_oid, seq->child) == CRYPT_OK)) { + ltc_asn1_list *version, *field, *point, *point_g, *order, *p_cofactor; + + /* Setup for CASE 2 */ + n=0; + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_INTEGER, &version); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_SEQUENCE, &field); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_SEQUENCE, &point); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_OCTET_STRING, &point_g); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_INTEGER, &order); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n++, LTC_ASN1_INTEGER, &p_cofactor); + LTC_SET_DER_FLEXI_CHECK(flexi_should, n, LTC_ASN1_EOL, NULL); + + if (LTC_ASN1_IS_TYPE(seq->child->next, LTC_ASN1_OBJECT_IDENTIFIER)) { + /* CASE 1: curve by OID (AKA short variant): + * 0:d=0 hl=2 l= 100 cons: SEQUENCE + * 2:d=1 hl=2 l= 1 prim: INTEGER :00 + * 5:d=1 hl=2 l= 16 cons: SEQUENCE (== *seq) + * 7:d=2 hl=2 l= 7 prim: OBJECT :id-ecPublicKey + * 16:d=2 hl=2 l= 5 prim: OBJECT :(== *curve_oid (e.g. secp256k1 (== 1.3.132.0.10))) + * 23:d=1 hl=2 l= 77 prim: OCTET STRING :bytes (== *priv_key) + */ + ltc_asn1_list *curve_oid = seq->child->next; + len = sizeof(OID); + if ((err = pk_oid_num_to_str(curve_oid->data, curve_oid->size, OID, &len)) != CRYPT_OK) { goto LBL_DONE; } + if ((err = ecc_find_curve(OID, &curve)) != CRYPT_OK) { goto LBL_DONE; } + if ((err = ecc_set_curve(curve, key)) != CRYPT_OK) { goto LBL_DONE; } + } + else if ((err = s_der_flexi_sequence_cmp(seq->child->next, flexi_should)) == CRYPT_OK) { + /* CASE 2: explicit curve parameters (AKA long variant): + * 0:d=0 hl=3 l= 227 cons: SEQUENCE + * 3:d=1 hl=2 l= 1 prim: INTEGER :00 + * 6:d=1 hl=3 l= 142 cons: SEQUENCE (== *seq) + * 9:d=2 hl=2 l= 7 prim: OBJECT :id-ecPublicKey + * 18:d=2 hl=3 l= 130 cons: SEQUENCE + * 21:d=3 hl=2 l= 1 prim: INTEGER :01 + * 24:d=3 hl=2 l= 44 cons: SEQUENCE (== *field) + * 26:d=4 hl=2 l= 7 prim: OBJECT :prime-field + * 35:d=4 hl=2 l= 33 prim: INTEGER :(== *prime / curve.prime) + * 70:d=3 hl=2 l= 6 cons: SEQUENCE (== *point) + * 72:d=4 hl=2 l= 1 prim: OCTET STRING :bytes (== curve.A) + * 75:d=4 hl=2 l= 1 prim: OCTET STRING :bytes (== curve.B) + * 78:d=3 hl=2 l= 33 prim: OCTET STRING :bytes (== *g_point / curve.G-point) + * 113:d=3 hl=2 l= 33 prim: INTEGER :(== *order / curve.order) + * 148:d=3 hl=2 l= 1 prim: INTEGER :(== curve.cofactor) + * 151:d=1 hl=2 l= 77 prim: OCTET STRING :bytes (== *priv_key) + */ + + if (mp_get_int(version->data) != 1) { + goto LBL_DONE; + } + cofactor = mp_get_int(p_cofactor->data); + + if (LTC_ASN1_IS_TYPE(field->child, LTC_ASN1_OBJECT_IDENTIFIER) && + LTC_ASN1_IS_TYPE(field->child->next, LTC_ASN1_INTEGER) && + LTC_ASN1_IS_TYPE(point->child, LTC_ASN1_OCTET_STRING) && + LTC_ASN1_IS_TYPE(point->child->next, LTC_ASN1_OCTET_STRING)) { + + ltc_asn1_list *prime = field->child->next; + if ((err = mp_read_unsigned_bin(a, point->child->data, point->child->size)) != CRYPT_OK) { + goto LBL_DONE; + } + if ((err = mp_read_unsigned_bin(b, point->child->next->data, point->child->next->size)) != CRYPT_OK) { + goto LBL_DONE; + } + if ((err = ltc_ecc_import_point(point_g->data, point_g->size, prime->data, a, b, gx, gy)) != CRYPT_OK) { + goto LBL_DONE; + } + if ((err = ecc_set_curve_from_mpis(a, b, prime->data, order->data, gx, gy, cofactor, key)) != CRYPT_OK) { + goto LBL_DONE; + } + } + } + else { + err = CRYPT_INVALID_PACKET; + goto LBL_DONE; + } + + /* load private key value 'k' */ + len = priv_key->size; + if ((err = der_decode_sequence_flexi(priv_key->data, &len, &p)) == CRYPT_OK) { + if (p->type == LTC_ASN1_SEQUENCE && + LTC_ASN1_IS_TYPE(p->child, LTC_ASN1_INTEGER) && + LTC_ASN1_IS_TYPE(p->child->next, LTC_ASN1_OCTET_STRING)) { + ltc_asn1_list *lk = p->child->next; + if (mp_cmp_d(p->child->data, 1) != LTC_MP_EQ) { + err = CRYPT_INVALID_PACKET; + goto LBL_ECCFREE; + } + if ((err = ecc_set_key(lk->data, lk->size, PK_PRIVATE, key)) != CRYPT_OK) { + goto LBL_ECCFREE; + } + goto LBL_DONE; /* success */ + } + } + } + } + err = CRYPT_INVALID_PACKET; + goto LBL_DONE; + +LBL_ECCFREE: + ecc_free(key); +LBL_DONE: + mp_clear_multi(a, b, gx, gy, LTC_NULL); + if (l) der_free_sequence_flexi(l); + if (p) der_free_sequence_flexi(p); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_x509.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_x509.c new file mode 100644 index 0000000..e4ba5e3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_import_x509.c @@ -0,0 +1,116 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +static int s_ecc_import_x509_with_oid(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + unsigned char bin_xy[2*ECC_MAXSIZE+2]; + unsigned long curveoid[16]; + unsigned long len_xy, len_oid, len; + char OID[256]; + const ltc_ecc_curve *curve; + int err; + + len_xy = sizeof(bin_xy); + len_oid = 16; + err = x509_decode_subject_public_key_info(in, inlen, LTC_OID_EC, bin_xy, &len_xy, + LTC_ASN1_OBJECT_IDENTIFIER, (void *)curveoid, &len_oid); + if (err == CRYPT_OK) { + /* load curve parameters for given curve OID */ + len = sizeof(OID); + if ((err = pk_oid_num_to_str(curveoid, len_oid, OID, &len)) != CRYPT_OK) { goto error; } + if ((err = ecc_find_curve(OID, &curve)) != CRYPT_OK) { goto error; } + if ((err = ecc_set_curve(curve, key)) != CRYPT_OK) { goto error; } + /* load public key */ + err = ecc_set_key(bin_xy, len_xy, PK_PUBLIC, key); + } +error: + return err; +} + +static int s_ecc_import_x509_with_curve(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + void *prime, *order, *a, *b, *gx, *gy; + ltc_asn1_list seq_fieldid[2], seq_curve[3], seq_ecparams[6]; + unsigned char bin_a[ECC_MAXSIZE], bin_b[ECC_MAXSIZE]; + unsigned char bin_g[2*ECC_MAXSIZE+1], bin_xy[2*ECC_MAXSIZE+2], bin_seed[128]; + unsigned long len_a, len_b, len_g, len_xy, len; + unsigned long cofactor = 0, ecver = 0, tmpoid[16]; + int err; + + if ((err = mp_init_multi(&prime, &order, &a, &b, &gx, &gy, LTC_NULL)) != CRYPT_OK) { + return err; + } + + /* ECParameters SEQUENCE */ + LTC_SET_ASN1(seq_ecparams, 0, LTC_ASN1_SHORT_INTEGER, &ecver, 1UL); + LTC_SET_ASN1(seq_ecparams, 1, LTC_ASN1_SEQUENCE, seq_fieldid, 2UL); + LTC_SET_ASN1(seq_ecparams, 2, LTC_ASN1_SEQUENCE, seq_curve, 3UL); + LTC_SET_ASN1(seq_ecparams, 3, LTC_ASN1_OCTET_STRING, bin_g, sizeof(bin_g)); + LTC_SET_ASN1(seq_ecparams, 4, LTC_ASN1_INTEGER, order, 1UL); + LTC_SET_ASN1(seq_ecparams, 5, LTC_ASN1_SHORT_INTEGER, &cofactor, 1UL); + seq_ecparams[5].optional = 1; + /* FieldID SEQUENCE */ + LTC_SET_ASN1(seq_fieldid, 0, LTC_ASN1_OBJECT_IDENTIFIER, tmpoid, 16UL); + LTC_SET_ASN1(seq_fieldid, 1, LTC_ASN1_INTEGER, prime, 1UL); + /* Curve SEQUENCE */ + LTC_SET_ASN1(seq_curve, 0, LTC_ASN1_OCTET_STRING, bin_a, sizeof(bin_a)); + LTC_SET_ASN1(seq_curve, 1, LTC_ASN1_OCTET_STRING, bin_b, sizeof(bin_b)); + LTC_SET_ASN1(seq_curve, 2, LTC_ASN1_RAW_BIT_STRING, bin_seed, 8u*sizeof(bin_seed)); + seq_curve[2].optional = 1; + /* try to load public key */ + len_xy = sizeof(bin_xy); + len = 6; + err = x509_decode_subject_public_key_info(in, inlen, LTC_OID_EC, bin_xy, &len_xy, LTC_ASN1_SEQUENCE, seq_ecparams, &len); + + if (err == CRYPT_OK) { + len_a = seq_curve[0].size; + len_b = seq_curve[1].size; + len_g = seq_ecparams[3].size; + /* create bignums */ + if ((err = mp_read_unsigned_bin(a, bin_a, len_a)) != CRYPT_OK) { goto error; } + if ((err = mp_read_unsigned_bin(b, bin_b, len_b)) != CRYPT_OK) { goto error; } + if ((err = ltc_ecc_import_point(bin_g, len_g, prime, a, b, gx, gy)) != CRYPT_OK) { goto error; } + /* load curve parameters */ + if ((err = ecc_set_curve_from_mpis(a, b, prime, order, gx, gy, cofactor, key)) != CRYPT_OK) { goto error; } + /* load public key */ + err = ecc_set_key(bin_xy, len_xy, PK_PUBLIC, key); + } +error: + mp_clear_multi(prime, order, a, b, gx, gy, LTC_NULL); + return err; +} + +int ecc_import_subject_public_key_info(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + int err; + + if ((err = s_ecc_import_x509_with_oid(in, inlen, key)) == CRYPT_OK) { + goto success; + } + + err = s_ecc_import_x509_with_curve(in, inlen, key); + +success: + return err; +} + +/** + Import an ECC key from a X.509 certificate + @param in The packet to import from + @param inlen It's length (octets) + @param key [out] Destination for newly imported key + @return CRYPT_OK if successful, upon error allocated memory is freed +*/ +int ecc_import_x509(const unsigned char *in, unsigned long inlen, ecc_key *key) +{ + return x509_decode_public_key_from_certificate(in, inlen, + LTC_OID_EC, + LTC_ASN1_EOL, NULL, NULL, + (public_key_decode_cb)ecc_import_subject_public_key_info, key); +} + +#endif /* LTC_MECC */ + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_make_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_make_key.c new file mode 100644 index 0000000..1b04741 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_make_key.c @@ -0,0 +1,73 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_make_key.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Make a new ECC key + @param prng An active PRNG state + @param wprng The index of the PRNG you wish to use + @param keysize The keysize for the new key (in octets from 20 to 65 bytes) + @param key [out] Destination of the newly created key + @return CRYPT_OK if successful, upon error all allocated memory will be freed +*/ +int ecc_make_key(prng_state *prng, int wprng, int keysize, ecc_key *key) +{ + int err; + + if ((err = ecc_set_curve_by_size(keysize, key)) != CRYPT_OK) { return err; } + if ((err = ecc_generate_key(prng, wprng, key)) != CRYPT_OK) { return err; } + return CRYPT_OK; +} + +int ecc_make_key_ex(prng_state *prng, int wprng, ecc_key *key, const ltc_ecc_curve *cu) +{ + int err; + if ((err = ecc_set_curve(cu, key)) != CRYPT_OK) { return err; } + if ((err = ecc_generate_key(prng, wprng, key)) != CRYPT_OK) { return err; } + return CRYPT_OK; +} + +int ecc_generate_key(prng_state *prng, int wprng, ecc_key *key) +{ + int err; + + LTC_ARGCHK(ltc_mp.name != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(key->dp.size > 0); + + /* ECC key pair generation according to FIPS-186-4 (B.4.2 Key Pair Generation by Testing Candidates): + * the generated private key k should be the range [1, order-1] + * a/ N = bitlen(order) + * b/ generate N random bits and convert them into big integer k + * c/ if k not in [1, order-1] go to b/ + * e/ Q = k*G + */ + if ((err = rand_bn_upto(key->k, key->dp.order, prng, wprng)) != CRYPT_OK) { + goto error; + } + + /* make the public key */ + if ((err = ltc_mp.ecc_ptmul(key->k, &key->dp.base, &key->pubkey, key->dp.A, key->dp.prime, 1)) != CRYPT_OK) { + goto error; + } + key->type = PK_PRIVATE; + + /* success */ + err = CRYPT_OK; + goto cleanup; + +error: + ecc_free(key); +cleanup: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_recover_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_recover_key.c new file mode 100644 index 0000000..b6ae644 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_recover_key.c @@ -0,0 +1,260 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +#ifdef LTC_ECC_SHAMIR + +/** + @file ecc_recover_key.c + ECC Crypto, Russ Williams +*/ + +/** + Recover ECC public key from signature and hash + @param sig The signature to verify + @param siglen The length of the signature (octets) + @param hash The hash (message digest) that was signed + @param hashlen The length of the hash (octets) + @param recid The recovery ID ("v"), can be -1 if signature contains it + @param sigformat The format of the signature (ecc_signature_type) + @param key The recovered public ECC key + @return CRYPT_OK if successful (even if the signature is not valid) +*/ +int ecc_recover_key(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + int recid, ecc_signature_type sigformat, ecc_key *key) +{ + ecc_point *mG = NULL, *mQ = NULL, *mR = NULL; + void *p, *m, *a, *b; + void *r, *s, *v, *w, *t1, *t2, *u1, *u2, *v1, *v2, *e, *x, *y, *a_plus3; + void *mu = NULL, *ma = NULL; + void *mp = NULL; + int err; + unsigned long pbits, pbytes, i, shift_right; + unsigned char ch, buf[MAXBLOCKSIZE]; + + LTC_ARGCHK(sig != NULL); + LTC_ARGCHK(hash != NULL); + LTC_ARGCHK(key != NULL); + + /* BEWARE: requires sqrtmod_prime */ + if (ltc_mp.sqrtmod_prime == NULL) { + return CRYPT_ERROR; + } + + /* allocate ints */ + if ((err = mp_init_multi(&r, &s, &v, &w, &t1, &t2, &u1, &u2, &v1, &v2, &e, &x, &y, &a_plus3, LTC_NULL)) != CRYPT_OK) { + return err; + } + + p = key->dp.order; + m = key->dp.prime; + a = key->dp.A; + b = key->dp.B; + if ((err = mp_add_d(a, 3, a_plus3)) != CRYPT_OK) { + goto error; + } + + /* allocate points */ + mG = ltc_ecc_new_point(); + mQ = ltc_ecc_new_point(); + mR = ltc_ecc_new_point(); + if (mR == NULL || mQ == NULL || mG == NULL) { + err = CRYPT_MEM; + goto error; + } + + if (sigformat == LTC_ECCSIG_ANSIX962) { + /* ANSI X9.62 format - ASN.1 encoded SEQUENCE{ INTEGER(r), INTEGER(s) } */ + if ((err = der_decode_sequence_multi_ex(sig, siglen, LTC_DER_SEQ_SEQUENCE | LTC_DER_SEQ_STRICT, + LTC_ASN1_INTEGER, 1UL, r, + LTC_ASN1_INTEGER, 1UL, s, + LTC_ASN1_EOL, 0UL, LTC_NULL)) != CRYPT_OK) { goto error; } + } + else if (sigformat == LTC_ECCSIG_RFC7518) { + /* RFC7518 format - raw (r,s) */ + i = mp_unsigned_bin_size(key->dp.order); + if (siglen != (2*i)) { + err = CRYPT_INVALID_PACKET; + goto error; + } + if ((err = mp_read_unsigned_bin(r, (unsigned char *)sig, i)) != CRYPT_OK) { goto error; } + if ((err = mp_read_unsigned_bin(s, (unsigned char *)sig+i, i)) != CRYPT_OK) { goto error; } + } + else if (sigformat == LTC_ECCSIG_ETH27) { + /* Ethereum (v,r,s) format */ + if (pk_oid_cmp_with_ulong("1.3.132.0.10", key->dp.oid, key->dp.oidlen) != CRYPT_OK) { + /* Only valid for secp256k1 - OID 1.3.132.0.10 */ + err = CRYPT_ERROR; goto error; + } + if (siglen != 65) { /* Only secp256k1 curves use this format, so must be 65 bytes long */ + err = CRYPT_INVALID_PACKET; + goto error; + } + i = (unsigned long)sig[64]; + if ((i>=27) && (i<31)) i -= 27; /* Ethereum adds 27 to recovery ID */ + if (recid >= 0 && ((unsigned long)recid != i)) { + /* Recovery ID specified, but doesn't match signature */ + err = CRYPT_INVALID_PACKET; + goto error; + } + recid = i; + if ((err = mp_read_unsigned_bin(r, (unsigned char *)sig, 32)) != CRYPT_OK) { goto error; } + if ((err = mp_read_unsigned_bin(s, (unsigned char *)sig+32, 32)) != CRYPT_OK) { goto error; } + } +#ifdef LTC_SSH + else if (sigformat == LTC_ECCSIG_RFC5656) { + char name[64], name2[64]; + unsigned long namelen = sizeof(name); + unsigned long name2len = sizeof(name2); + + /* Decode as SSH data sequence, per RFC4251 */ + if ((err = ssh_decode_sequence_multi(sig, &siglen, + LTC_SSHDATA_STRING, name, &namelen, + LTC_SSHDATA_MPINT, r, + LTC_SSHDATA_MPINT, s, + LTC_SSHDATA_EOL, NULL)) != CRYPT_OK) { goto error; } + + + /* Check curve matches identifier string */ + if ((err = ecc_ssh_ecdsa_encode_name(name2, &name2len, key)) != CRYPT_OK) { goto error; } + if ((namelen != name2len) || (XSTRCMP(name, name2) != 0)) { + err = CRYPT_INVALID_ARG; + goto error; + } + } +#endif + else { + /* Unknown signature format */ + err = CRYPT_ERROR; + goto error; + } + + if (recid < 0 || (unsigned long)recid >= 2*(key->dp.cofactor+1)) { + /* Recovery ID is out of range, reject it */ + err = CRYPT_INVALID_ARG; + goto error; + } + + /* check for zero */ + if (mp_cmp_d(r, 0) != LTC_MP_GT || mp_cmp_d(s, 0) != LTC_MP_GT || + mp_cmp(r, p) != LTC_MP_LT || mp_cmp(s, p) != LTC_MP_LT) { + err = CRYPT_INVALID_PACKET; + goto error; + } + + /* read hash - truncate if needed */ + pbits = mp_count_bits(p); + pbytes = (pbits+7) >> 3; + if (pbits > hashlen*8) { + if ((err = mp_read_unsigned_bin(e, (unsigned char *)hash, hashlen)) != CRYPT_OK) { goto error; } + } + else if (pbits % 8 == 0) { + if ((err = mp_read_unsigned_bin(e, (unsigned char *)hash, pbytes)) != CRYPT_OK) { goto error; } + } + else { + shift_right = 8 - pbits % 8; + for (i=0, ch=0; i> shift_right); + } + if ((err = mp_read_unsigned_bin(e, (unsigned char *)buf, pbytes)) != CRYPT_OK) { goto error; } + } + + /* decompress point from r=(x mod p) - BEWARE: requires sqrtmod_prime */ + /* x = r + p*(recid/2) */ + if ((err = mp_set(x, recid/2)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(p, x, m, x)) != CRYPT_OK) { goto error; } + if ((err = mp_add(x, r, x)) != CRYPT_OK) { goto error; } + /* compute x^3 */ + if ((err = mp_sqr(x, t1)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(t1, x, m, t1)) != CRYPT_OK) { goto error; } + /* compute x^3 + a*x */ + if ((err = mp_mulmod(a, x, m, t2)) != CRYPT_OK) { goto error; } + if ((err = mp_add(t1, t2, t1)) != CRYPT_OK) { goto error; } + /* compute x^3 + a*x + b */ + if ((err = mp_add(t1, b, t1)) != CRYPT_OK) { goto error; } + /* compute sqrt(x^3 + a*x + b) */ + if ((err = mp_sqrtmod_prime(t1, m, t2)) != CRYPT_OK) { goto error; } + + /* fill in mR */ + if ((err = mp_copy(x, mR->x)) != CRYPT_OK) { goto error; } + if ((mp_isodd(t2) && (recid%2)) || (!mp_isodd(t2) && !(recid%2))) { + if ((err = mp_mod(t2, m, mR->y)) != CRYPT_OK) { goto error; } + } + else { + if ((err = mp_submod(m, t2, m, mR->y)) != CRYPT_OK) { goto error; } + } + if ((err = mp_set(mR->z, 1)) != CRYPT_OK) { goto error; } + + /* w = r^-1 mod n */ + if ((err = mp_invmod(r, p, w)) != CRYPT_OK) { goto error; } + /* v1 = sw */ + if ((err = mp_mulmod(s, w, p, v1)) != CRYPT_OK) { goto error; } + /* v2 = -ew */ + if ((err = mp_mulmod(e, w, p, v2)) != CRYPT_OK) { goto error; } + if ((err = mp_submod(p, v2, p, v2)) != CRYPT_OK) { goto error; } + + /* w = s^-1 mod n */ + if ((err = mp_invmod(s, p, w)) != CRYPT_OK) { goto error; } + /* u1 = ew */ + if ((err = mp_mulmod(e, w, p, u1)) != CRYPT_OK) { goto error; } + /* u2 = rw */ + if ((err = mp_mulmod(r, w, p, u2)) != CRYPT_OK) { goto error; } + + /* find mG */ + if ((err = ltc_ecc_copy_point(&key->dp.base, mG)) != CRYPT_OK) { goto error; } + + /* find the montgomery mp */ + if ((err = mp_montgomery_setup(m, &mp)) != CRYPT_OK) { goto error; } + + /* for curves with a == -3 keep ma == NULL */ + if (mp_cmp(a_plus3, m) != LTC_MP_EQ) { + if ((err = mp_init_multi(&mu, &ma, LTC_NULL)) != CRYPT_OK) { goto error; } + if ((err = mp_montgomery_normalization(mu, m)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(a, mu, m, ma)) != CRYPT_OK) { goto error; } + } + + /* recover mQ from mR */ + /* compute v1*mR + v2*mG = mQ using Shamir's trick */ + if ((err = ltc_mp.ecc_mul2add(mR, v1, mG, v2, mQ, ma, m)) != CRYPT_OK) { goto error; } + + /* compute u1*mG + u2*mQ = mG using Shamir's trick */ + if ((err = ltc_mp.ecc_mul2add(mG, u1, mQ, u2, mG, ma, m)) != CRYPT_OK) { goto error; } + + /* v = X_x1 mod n */ + if ((err = mp_mod(mG->x, p, v)) != CRYPT_OK) { goto error; } + + /* does v == r */ + if (mp_cmp(v, r) == LTC_MP_EQ) { + /* found public key which verifies signature */ + if ((err = ltc_ecc_copy_point(mQ, &key->pubkey)) != CRYPT_OK) { goto error; } + /* point on the curve + other checks */ + if ((err = ltc_ecc_verify_key(key)) != CRYPT_OK) { goto error; } + + key->type = PK_PUBLIC; + + err = CRYPT_OK; + } + else { + /* not found - recid is wrong or we're unable to calculate public key for some other reason */ + err = CRYPT_INVALID_ARG; + } + +error: + if (ma != NULL) mp_clear(ma); + if (mu != NULL) mp_clear(mu); + if (mp != NULL) mp_montgomery_free(mp); + if (mR != NULL) ltc_ecc_del_point(mR); + if (mQ != NULL) ltc_ecc_del_point(mQ); + if (mG != NULL) ltc_ecc_del_point(mG); + mp_clear_multi(a_plus3, y, x, e, v2, v1, u2, u1, t2, t1, w, v, s, r, LTC_NULL); + return err; +} + +#endif +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_curve.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_curve.c new file mode 100644 index 0000000..427a593 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_curve.c @@ -0,0 +1,80 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +int ecc_set_curve(const ltc_ecc_curve *cu, ecc_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(cu != NULL); + + if ((err = mp_init_multi(&key->dp.prime, &key->dp.order, &key->dp.A, &key->dp.B, + &key->dp.base.x, &key->dp.base.y, &key->dp.base.z, + &key->pubkey.x, &key->pubkey.y, &key->pubkey.z, &key->k, + NULL)) != CRYPT_OK) { + return err; + } + + /* A, B, order, prime, Gx, Gy */ + if ((err = mp_read_radix(key->dp.prime, cu->prime, 16)) != CRYPT_OK) { goto error; } + if ((err = mp_read_radix(key->dp.order, cu->order, 16)) != CRYPT_OK) { goto error; } + if ((err = mp_read_radix(key->dp.A, cu->A, 16)) != CRYPT_OK) { goto error; } + if ((err = mp_read_radix(key->dp.B, cu->B, 16)) != CRYPT_OK) { goto error; } + if ((err = mp_read_radix(key->dp.base.x, cu->Gx, 16)) != CRYPT_OK) { goto error; } + if ((err = mp_read_radix(key->dp.base.y, cu->Gy, 16)) != CRYPT_OK) { goto error; } + if ((err = mp_set(key->dp.base.z, 1)) != CRYPT_OK) { goto error; } + /* cofactor & size */ + key->dp.cofactor = cu->cofactor; + key->dp.size = mp_unsigned_bin_size(key->dp.prime); + /* OID string >> unsigned long oid[16] + oidlen */ + key->dp.oidlen = 16; + if ((err = pk_oid_str_to_num(cu->OID, key->dp.oid, &key->dp.oidlen)) != CRYPT_OK) { goto error; } + /* success */ + return CRYPT_OK; + +error: + ecc_free(key); + return err; +} + +int ecc_set_curve_by_size(int size, ecc_key *key) +{ + const ltc_ecc_curve *cu = NULL; + int err = CRYPT_ERROR; + + /* for compatibility with libtomcrypt-1.17 the sizes below must match the specific curves */ + if (size <= 14) { + err = ecc_find_curve("SECP112R1", &cu); + } + else if (size <= 16) { + err = ecc_find_curve("SECP128R1", &cu); + } + else if (size <= 20) { + err = ecc_find_curve("SECP160R1", &cu); + } + else if (size <= 24) { + err = ecc_find_curve("SECP192R1", &cu); + } + else if (size <= 28) { + err = ecc_find_curve("SECP224R1", &cu); + } + else if (size <= 32) { + err = ecc_find_curve("SECP256R1", &cu); + } + else if (size <= 48) { + err = ecc_find_curve("SECP384R1", &cu); + } + else if (size <= 66) { + err = ecc_find_curve("SECP521R1", &cu); + } + + if (err == CRYPT_OK && cu != NULL) return ecc_set_curve(cu, key); + + return CRYPT_INVALID_ARG; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_curve_internal.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_curve_internal.c new file mode 100644 index 0000000..4f43914 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_curve_internal.c @@ -0,0 +1,119 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +static int s_ecc_cmp_hex_bn(const char *left_hex, void *right_bn, void *tmp_bn) +{ + if (mp_read_radix(tmp_bn, left_hex, 16) != CRYPT_OK) return 0; + if (mp_cmp(tmp_bn, right_bn) != LTC_MP_EQ) return 0; + return 1; +} + +static void s_ecc_oid_lookup(ecc_key *key) +{ + void *bn; + const ltc_ecc_curve *curve; + + key->dp.oidlen = 0; + if (mp_init(&bn) != CRYPT_OK) return; + for (curve = ltc_ecc_curves; curve->prime != NULL; curve++) { + if (s_ecc_cmp_hex_bn(curve->prime, key->dp.prime, bn) != 1) continue; + if (s_ecc_cmp_hex_bn(curve->order, key->dp.order, bn) != 1) continue; + if (s_ecc_cmp_hex_bn(curve->A, key->dp.A, bn) != 1) continue; + if (s_ecc_cmp_hex_bn(curve->B, key->dp.B, bn) != 1) continue; + if (s_ecc_cmp_hex_bn(curve->Gx, key->dp.base.x, bn) != 1) continue; + if (s_ecc_cmp_hex_bn(curve->Gy, key->dp.base.y, bn) != 1) continue; + if (key->dp.cofactor != curve->cofactor) continue; + break; /* found */ + } + mp_clear(bn); + if (curve->prime && curve->OID) { + key->dp.oidlen = 16; /* size of key->dp.oid */ + pk_oid_str_to_num(curve->OID, key->dp.oid, &key->dp.oidlen); + } +} + +int ecc_copy_curve(const ecc_key *srckey, ecc_key *key) +{ + unsigned long i; + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(srckey != NULL); + + if ((err = mp_init_multi(&key->dp.prime, &key->dp.order, &key->dp.A, &key->dp.B, + &key->dp.base.x, &key->dp.base.y, &key->dp.base.z, + &key->pubkey.x, &key->pubkey.y, &key->pubkey.z, &key->k, + NULL)) != CRYPT_OK) { + return err; + } + + /* A, B, order, prime, Gx, Gy */ + if ((err = mp_copy(srckey->dp.prime, key->dp.prime )) != CRYPT_OK) { goto error; } + if ((err = mp_copy(srckey->dp.order, key->dp.order )) != CRYPT_OK) { goto error; } + if ((err = mp_copy(srckey->dp.A, key->dp.A )) != CRYPT_OK) { goto error; } + if ((err = mp_copy(srckey->dp.B, key->dp.B )) != CRYPT_OK) { goto error; } + if ((err = ltc_ecc_copy_point(&srckey->dp.base, &key->dp.base)) != CRYPT_OK) { goto error; } + /* cofactor & size */ + key->dp.cofactor = srckey->dp.cofactor; + key->dp.size = srckey->dp.size; + /* OID */ + if (srckey->dp.oidlen > 0) { + key->dp.oidlen = srckey->dp.oidlen; + for (i = 0; i < key->dp.oidlen; i++) key->dp.oid[i] = srckey->dp.oid[i]; + } + else { + s_ecc_oid_lookup(key); /* try to find OID in ltc_ecc_curves */ + } + /* success */ + return CRYPT_OK; + +error: + ecc_free(key); + return err; +} + +int ecc_set_curve_from_mpis(void *a, void *b, void *prime, void *order, void *gx, void *gy, unsigned long cofactor, ecc_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(a != NULL); + LTC_ARGCHK(b != NULL); + LTC_ARGCHK(prime != NULL); + LTC_ARGCHK(order != NULL); + LTC_ARGCHK(gx != NULL); + LTC_ARGCHK(gy != NULL); + + if ((err = mp_init_multi(&key->dp.prime, &key->dp.order, &key->dp.A, &key->dp.B, + &key->dp.base.x, &key->dp.base.y, &key->dp.base.z, + &key->pubkey.x, &key->pubkey.y, &key->pubkey.z, &key->k, + NULL)) != CRYPT_OK) { + return err; + } + + /* A, B, order, prime, Gx, Gy */ + if ((err = mp_copy(prime, key->dp.prime )) != CRYPT_OK) { goto error; } + if ((err = mp_copy(order, key->dp.order )) != CRYPT_OK) { goto error; } + if ((err = mp_copy(a, key->dp.A )) != CRYPT_OK) { goto error; } + if ((err = mp_copy(b, key->dp.B )) != CRYPT_OK) { goto error; } + if ((err = mp_copy(gx, key->dp.base.x)) != CRYPT_OK) { goto error; } + if ((err = mp_copy(gy, key->dp.base.y)) != CRYPT_OK) { goto error; } + if ((err = mp_set(key->dp.base.z, 1)) != CRYPT_OK) { goto error; } + /* cofactor & size */ + key->dp.cofactor = cofactor; + key->dp.size = mp_unsigned_bin_size(prime); + /* try to find OID in ltc_ecc_curves */ + s_ecc_oid_lookup(key); + /* success */ + return CRYPT_OK; + +error: + ecc_free(key); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_key.c new file mode 100644 index 0000000..3cf5488 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_set_key.c @@ -0,0 +1,56 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +int ecc_set_key(const unsigned char *in, unsigned long inlen, int type, ecc_key *key) +{ + int err; + void *prime, *a, *b; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen > 0); + + prime = key->dp.prime; + a = key->dp.A; + b = key->dp.B; + + if (type == PK_PRIVATE) { + /* load private key */ + if ((err = mp_read_unsigned_bin(key->k, (unsigned char *)in, inlen)) != CRYPT_OK) { + goto error; + } + if (mp_iszero(key->k) || (mp_cmp(key->k, key->dp.order) != LTC_MP_LT)) { + err = CRYPT_INVALID_PACKET; + goto error; + } + /* compute public key */ + if ((err = ltc_mp.ecc_ptmul(key->k, &key->dp.base, &key->pubkey, a, prime, 1)) != CRYPT_OK) { goto error; } + } + else if (type == PK_PUBLIC) { + /* load public key */ + if ((err = ltc_ecc_import_point(in, inlen, prime, a, b, key->pubkey.x, key->pubkey.y)) != CRYPT_OK) { goto error; } + if ((err = mp_set(key->pubkey.z, 1)) != CRYPT_OK) { goto error; } + } + else { + err = CRYPT_INVALID_PACKET; + goto error; + } + + /* point on the curve + other checks */ + if ((err = ltc_ecc_verify_key(key)) != CRYPT_OK) { + goto error; + } + + key->type = type; + return CRYPT_OK; + +error: + ecc_free(key); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_shared_secret.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_shared_secret.c new file mode 100644 index 0000000..21ac913 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_shared_secret.c @@ -0,0 +1,66 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_shared_secret.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Create an ECC shared secret between two keys + @param private_key The private ECC key + @param public_key The public key + @param out [out] Destination of the shared secret (Conforms to EC-DH from ANSI X9.63) + @param outlen [in/out] The max size and resulting size of the shared secret + @return CRYPT_OK if successful +*/ +int ecc_shared_secret(const ecc_key *private_key, const ecc_key *public_key, + unsigned char *out, unsigned long *outlen) +{ + unsigned long x; + ecc_point *result; + void *prime, *a; + int err; + + LTC_ARGCHK(private_key != NULL); + LTC_ARGCHK(public_key != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* type valid? */ + if (private_key->type != PK_PRIVATE) { + return CRYPT_PK_NOT_PRIVATE; + } + + /* make new point */ + result = ltc_ecc_new_point(); + if (result == NULL) { + return CRYPT_MEM; + } + + prime = private_key->dp.prime; + a = private_key->dp.A; + + if ((err = ltc_mp.ecc_ptmul(private_key->k, &public_key->pubkey, result, a, prime, 1)) != CRYPT_OK) { goto done; } + + x = (unsigned long)mp_unsigned_bin_size(prime); + if (*outlen < x) { + *outlen = x; + err = CRYPT_BUFFER_OVERFLOW; + goto done; + } + zeromem(out, x); + if ((err = mp_to_unsigned_bin(result->x, out + (x - mp_unsigned_bin_size(result->x)))) != CRYPT_OK) { goto done; } + + err = CRYPT_OK; + *outlen = x; +done: + ltc_ecc_del_point(result); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_sign_hash.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_sign_hash.c new file mode 100644 index 0000000..229ced1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_sign_hash.c @@ -0,0 +1,181 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +/** + @file ecc_sign_hash.c + ECC Crypto, Tom St Denis +*/ + +/** + Sign a message digest + @param in The message digest to sign + @param inlen The length of the digest + @param out [out] The destination for the signature + @param outlen [in/out] The max size and resulting size of the signature + @param prng An active PRNG state + @param wprng The index of the PRNG you wish to use + @param sigformat The format of the signature to generate (ecc_signature_type) + @param recid [out] The recovery ID for this signature (optional) + @param key A private ECC key + @return CRYPT_OK if successful +*/ +int ecc_sign_hash_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + prng_state *prng, int wprng, ecc_signature_type sigformat, + int *recid, const ecc_key *key) +{ + ecc_key pubkey; + void *r, *s, *e, *p, *b; + int v = 0; + int err, max_iterations = LTC_PK_MAX_RETRIES; + unsigned long pbits, pbytes, i, shift_right; + unsigned char ch, buf[MAXBLOCKSIZE]; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* is this a private key? */ + if (key->type != PK_PRIVATE) { + return CRYPT_PK_NOT_PRIVATE; + } + + /* init the bignums */ + if ((err = mp_init_multi(&r, &s, &e, &b, LTC_NULL)) != CRYPT_OK) { + return err; + } + + /* get the hash and load it as a bignum into 'e' */ + p = key->dp.order; + pbits = mp_count_bits(p); + pbytes = (pbits+7) >> 3; + if (pbits > inlen*8) { + if ((err = mp_read_unsigned_bin(e, (unsigned char *)in, inlen)) != CRYPT_OK) { goto errnokey; } + } + else if (pbits % 8 == 0) { + if ((err = mp_read_unsigned_bin(e, (unsigned char *)in, pbytes)) != CRYPT_OK) { goto errnokey; } + } + else { + shift_right = 8 - pbits % 8; + for (i=0, ch=0; i> shift_right); + } + if ((err = mp_read_unsigned_bin(e, (unsigned char *)buf, pbytes)) != CRYPT_OK) { goto errnokey; } + } + + /* make up a key and export the public copy */ + do { + if ((err = ecc_copy_curve(key, &pubkey)) != CRYPT_OK) { goto errnokey; } + if ((err = ecc_generate_key(prng, wprng, &pubkey)) != CRYPT_OK) { goto errnokey; } + + /* find r = x1 mod n */ + if ((err = mp_mod(pubkey.pubkey.x, p, r)) != CRYPT_OK) { goto error; } + + if (recid || sigformat==LTC_ECCSIG_ETH27) { + /* find recovery ID (if needed) */ + v = 0; + if (mp_copy(pubkey.pubkey.x, s) != CRYPT_OK) { goto error; } + while (mp_cmp_d(s, 0) == LTC_MP_GT && mp_cmp(s, p) != LTC_MP_LT) { + /* Compute x1 div n... this will almost never be reached for curves with order 1 */ + v += 2; + if ((err = mp_sub(s, p, s)) != CRYPT_OK) { goto error; } + } + if (mp_isodd(pubkey.pubkey.y)) v += 1; + } + + if (mp_iszero(r) == LTC_MP_YES) { + ecc_free(&pubkey); + } else { + if ((err = rand_bn_upto(b, p, prng, wprng)) != CRYPT_OK) { goto error; } /* b = blinding value */ + /* find s = (e + xr)/k */ + if ((err = mp_mulmod(pubkey.k, b, p, pubkey.k)) != CRYPT_OK) { goto error; } /* k = kb */ + if ((err = mp_invmod(pubkey.k, p, pubkey.k)) != CRYPT_OK) { goto error; } /* k = 1/kb */ + if ((err = mp_mulmod(key->k, r, p, s)) != CRYPT_OK) { goto error; } /* s = xr */ + if ((err = mp_mulmod(pubkey.k, s, p, s)) != CRYPT_OK) { goto error; } /* s = xr/kb */ + if ((err = mp_mulmod(pubkey.k, e, p, e)) != CRYPT_OK) { goto error; } /* e = e/kb */ + if ((err = mp_add(e, s, s)) != CRYPT_OK) { goto error; } /* s = e/kb + xr/kb */ + if ((err = mp_mulmod(s, b, p, s)) != CRYPT_OK) { goto error; } /* s = b(e/kb + xr/kb) = (e + xr)/k */ + ecc_free(&pubkey); + if (mp_iszero(s) == LTC_MP_NO) { + break; + } + } + } while (--max_iterations > 0); + + if (max_iterations == 0) { + goto errnokey; + } + + if (recid) *recid = v; + + if (sigformat == LTC_ECCSIG_ANSIX962) { + /* store as ASN.1 SEQUENCE { r, s -- integer } */ + err = der_encode_sequence_multi(out, outlen, + LTC_ASN1_INTEGER, 1UL, r, + LTC_ASN1_INTEGER, 1UL, s, + LTC_ASN1_EOL, 0UL, NULL); + } + else if (sigformat == LTC_ECCSIG_RFC7518) { + /* RFC7518 format - raw (r,s) */ + if (*outlen < 2*pbytes) { err = CRYPT_MEM; goto errnokey; } + zeromem(out, 2*pbytes); + i = mp_unsigned_bin_size(r); + if ((err = mp_to_unsigned_bin(r, out + (pbytes - i))) != CRYPT_OK) { goto errnokey; } + i = mp_unsigned_bin_size(s); + if ((err = mp_to_unsigned_bin(s, out + (2*pbytes - i))) != CRYPT_OK) { goto errnokey; } + *outlen = 2*pbytes; + err = CRYPT_OK; + } + else if (sigformat == LTC_ECCSIG_ETH27) { + /* Ethereum (v,r,s) format */ + if (pk_oid_cmp_with_ulong("1.3.132.0.10", key->dp.oid, key->dp.oidlen) != CRYPT_OK) { + /* Only valid for secp256k1 - OID 1.3.132.0.10 */ + err = CRYPT_ERROR; goto errnokey; + } + if (*outlen < 65) { err = CRYPT_MEM; goto errnokey; } + zeromem(out, 65); + i = mp_unsigned_bin_size(r); + if ((err = mp_to_unsigned_bin(r, out + 32 - i)) != CRYPT_OK) { goto errnokey; } + i = mp_unsigned_bin_size(s); + if ((err = mp_to_unsigned_bin(s, out + 64 - i)) != CRYPT_OK) { goto errnokey; } + out[64] = (unsigned char)(v + 27); /* Recovery ID is 27/28 for Ethereum */ + *outlen = 65; + err = CRYPT_OK; + } +#ifdef LTC_SSH + else if (sigformat == LTC_ECCSIG_RFC5656) { + /* Get identifier string */ + char name[64]; + unsigned long namelen = sizeof(name); + if ((err = ecc_ssh_ecdsa_encode_name(name, &namelen, key)) != CRYPT_OK) { goto errnokey; } + + /* Store as SSH data sequence, per RFC4251 */ + err = ssh_encode_sequence_multi(out, outlen, + LTC_SSHDATA_STRING, name, namelen, + LTC_SSHDATA_MPINT, r, + LTC_SSHDATA_MPINT, s, + LTC_SSHDATA_EOL, NULL); + } +#endif + else { + /* Unknown signature format */ + err = CRYPT_ERROR; + goto error; + } + + goto errnokey; +error: + ecc_free(&pubkey); +errnokey: + mp_clear_multi(r, s, e, b, LTC_NULL); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_sizes.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_sizes.c new file mode 100644 index 0000000..07379b6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_sizes.c @@ -0,0 +1,36 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ecc_sizes.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +void ecc_sizes(int *low, int *high) +{ + int i, size; + void *prime; + + LTC_ARGCHKVD(low != NULL); + LTC_ARGCHKVD(high != NULL); + + *low = INT_MAX; + *high = 0; + + if (mp_init(&prime) == CRYPT_OK) { + for (i = 0; ltc_ecc_curves[i].prime != NULL; i++) { + if (mp_read_radix(prime, ltc_ecc_curves[i].prime, 16) == CRYPT_OK) { + size = mp_unsigned_bin_size(prime); + if (size < *low) *low = size; + if (size > *high) *high = size; + } + } + mp_clear(prime); + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ssh_ecdsa_encode_name.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ssh_ecdsa_encode_name.c new file mode 100644 index 0000000..4b8d554 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_ssh_ecdsa_encode_name.c @@ -0,0 +1,65 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ecc_ssh_ecdsa_encode_name.c + Curve/OID to SSH+ECDSA name string mapping per RFC5656 + Russ Williams +*/ + +#ifdef LTC_SSH + +/** + Curve/OID to SSH+ECDSA name string mapping + @param buffer [out] The destination for the name + @param buflen [in/out] The max size and resulting size (including terminator) of the name + @param key A public or private ECC key + @return CRYPT_OK if successful +*/ +int ecc_ssh_ecdsa_encode_name(char *buffer, unsigned long *buflen, const ecc_key *key) +{ + char oidstr[64] = {0}; + unsigned long oidlen = sizeof(oidstr); + int err, size = 0; + + LTC_ARGCHK(buffer != NULL); + LTC_ARGCHK(buflen != NULL); + LTC_ARGCHK(key != NULL); + + /* Get the OID of the curve */ + if ((err = ecc_get_oid_str(oidstr, &oidlen, key)) != CRYPT_OK) goto error; + + /* Check for three named curves: nistp256, nistp384, nistp521 */ + if (XSTRCMP("1.2.840.10045.3.1.7", oidstr) == 0) { + /* nistp256 - secp256r1 - OID 1.2.840.10045.3.1.7 */ + size = snprintf(buffer, *buflen, "ecdsa-sha2-nistp256"); + } + else if (XSTRCMP("1.3.132.0.34", oidstr) == 0) { + /* nistp384 - secp384r1 - OID 1.3.132.0.34 */ + size = snprintf(buffer, *buflen, "ecdsa-sha2-nistp384"); + } + else if (XSTRCMP("1.3.132.0.35", oidstr) == 0) { + /* nistp521 - secp521r1 - OID 1.3.132.0.35 */ + size = snprintf(buffer, *buflen, "ecdsa-sha2-nistp521"); + } else { + /* Otherwise we use the OID... */ + size = snprintf(buffer, *buflen, "ecdsa-sha2-%s", oidstr); + } + + /* snprintf returns a negative value on error + * or the size that would have been written, but limits to buflen-1 chars plus terminator */ + if (size < 0) { + err = CRYPT_ERROR; + } else if ((unsigned)size >= *buflen) { + err = CRYPT_BUFFER_OVERFLOW; + } else { + err = CRYPT_OK; + } + *buflen = size + 1; /* the string length + NUL byte */ + +error: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_verify_hash.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_verify_hash.c new file mode 100644 index 0000000..4480ff4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ecc_verify_hash.c @@ -0,0 +1,206 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +/** + @file ecc_verify_hash.c + ECC Crypto, Tom St Denis +*/ + +/** + Verify an ECC signature in RFC7518 format + @param sig The signature to verify + @param siglen The length of the signature (octets) + @param hash The hash (message digest) that was signed + @param hashlen The length of the hash (octets) + @param sigformat The format of the signature (ecc_signature_type) + @param stat Result of signature, 1==valid, 0==invalid + @param key The corresponding public ECC key + @return CRYPT_OK if successful (even if the signature is not valid) +*/ +int ecc_verify_hash_ex(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + ecc_signature_type sigformat, int *stat, const ecc_key *key) +{ + ecc_point *mG = NULL, *mQ = NULL; + void *r, *s, *v, *w, *u1, *u2, *e, *p, *m, *a, *a_plus3; + void *mu = NULL, *ma = NULL; + void *mp = NULL; + int err; + unsigned long pbits, pbytes, i, shift_right; + unsigned char ch, buf[MAXBLOCKSIZE]; + + LTC_ARGCHK(sig != NULL); + LTC_ARGCHK(hash != NULL); + LTC_ARGCHK(stat != NULL); + LTC_ARGCHK(key != NULL); + + /* default to invalid signature */ + *stat = 0; + + /* allocate ints */ + if ((err = mp_init_multi(&r, &s, &v, &w, &u1, &u2, &e, &a_plus3, LTC_NULL)) != CRYPT_OK) { + return err; + } + + p = key->dp.order; + m = key->dp.prime; + a = key->dp.A; + if ((err = mp_add_d(a, 3, a_plus3)) != CRYPT_OK) { + goto error; + } + + /* allocate points */ + mG = ltc_ecc_new_point(); + mQ = ltc_ecc_new_point(); + if (mQ == NULL || mG == NULL) { + err = CRYPT_MEM; + goto error; + } + + if (sigformat == LTC_ECCSIG_ANSIX962) { + /* ANSI X9.62 format - ASN.1 encoded SEQUENCE{ INTEGER(r), INTEGER(s) } */ + if ((err = der_decode_sequence_multi_ex(sig, siglen, LTC_DER_SEQ_SEQUENCE | LTC_DER_SEQ_STRICT, + LTC_ASN1_INTEGER, 1UL, r, + LTC_ASN1_INTEGER, 1UL, s, + LTC_ASN1_EOL, 0UL, LTC_NULL)) != CRYPT_OK) { goto error; } + } + else if (sigformat == LTC_ECCSIG_RFC7518) { + /* RFC7518 format - raw (r,s) */ + i = mp_unsigned_bin_size(key->dp.order); + if (siglen != (2 * i)) { + err = CRYPT_INVALID_PACKET; + goto error; + } + if ((err = mp_read_unsigned_bin(r, (unsigned char *)sig, i)) != CRYPT_OK) { goto error; } + if ((err = mp_read_unsigned_bin(s, (unsigned char *)sig+i, i)) != CRYPT_OK) { goto error; } + } + else if (sigformat == LTC_ECCSIG_ETH27) { + /* Ethereum (v,r,s) format */ + if (pk_oid_cmp_with_ulong("1.3.132.0.10", key->dp.oid, key->dp.oidlen) != CRYPT_OK) { + /* Only valid for secp256k1 - OID 1.3.132.0.10 */ + err = CRYPT_ERROR; goto error; + } + if (siglen != 65) { /* Only secp256k1 curves use this format, so must be 65 bytes long */ + err = CRYPT_INVALID_PACKET; + goto error; + } + if ((err = mp_read_unsigned_bin(r, (unsigned char *)sig, 32)) != CRYPT_OK) { goto error; } + if ((err = mp_read_unsigned_bin(s, (unsigned char *)sig+32, 32)) != CRYPT_OK) { goto error; } + } +#ifdef LTC_SSH + else if (sigformat == LTC_ECCSIG_RFC5656) { + char name[64], name2[64]; + unsigned long namelen = sizeof(name); + unsigned long name2len = sizeof(name2); + + /* Decode as SSH data sequence, per RFC4251 */ + if ((err = ssh_decode_sequence_multi(sig, &siglen, + LTC_SSHDATA_STRING, name, &namelen, + LTC_SSHDATA_MPINT, r, + LTC_SSHDATA_MPINT, s, + LTC_SSHDATA_EOL, NULL)) != CRYPT_OK) { goto error; } + + + /* Check curve matches identifier string */ + if ((err = ecc_ssh_ecdsa_encode_name(name2, &name2len, key)) != CRYPT_OK) { goto error; } + if ((namelen != name2len) || (XSTRCMP(name, name2) != 0)) { + err = CRYPT_INVALID_ARG; + goto error; + } + } +#endif + else { + /* Unknown signature format */ + err = CRYPT_ERROR; + goto error; + } + + /* check for zero */ + if (mp_cmp_d(r, 0) != LTC_MP_GT || mp_cmp_d(s, 0) != LTC_MP_GT || + mp_cmp(r, p) != LTC_MP_LT || mp_cmp(s, p) != LTC_MP_LT) { + err = CRYPT_INVALID_PACKET; + goto error; + } + + /* read hash - truncate if needed */ + pbits = mp_count_bits(p); + pbytes = (pbits+7) >> 3; + if (pbits > hashlen*8) { + if ((err = mp_read_unsigned_bin(e, (unsigned char *)hash, hashlen)) != CRYPT_OK) { goto error; } + } + else if (pbits % 8 == 0) { + if ((err = mp_read_unsigned_bin(e, (unsigned char *)hash, pbytes)) != CRYPT_OK) { goto error; } + } + else { + shift_right = 8 - pbits % 8; + for (i=0, ch=0; i> shift_right); + } + if ((err = mp_read_unsigned_bin(e, (unsigned char *)buf, pbytes)) != CRYPT_OK) { goto error; } + } + + /* w = s^-1 mod n */ + if ((err = mp_invmod(s, p, w)) != CRYPT_OK) { goto error; } + + /* u1 = ew */ + if ((err = mp_mulmod(e, w, p, u1)) != CRYPT_OK) { goto error; } + + /* u2 = rw */ + if ((err = mp_mulmod(r, w, p, u2)) != CRYPT_OK) { goto error; } + + /* find mG and mQ */ + if ((err = ltc_ecc_copy_point(&key->dp.base, mG)) != CRYPT_OK) { goto error; } + if ((err = ltc_ecc_copy_point(&key->pubkey, mQ)) != CRYPT_OK) { goto error; } + + /* find the montgomery mp */ + if ((err = mp_montgomery_setup(m, &mp)) != CRYPT_OK) { goto error; } + + /* for curves with a == -3 keep ma == NULL */ + if (mp_cmp(a_plus3, m) != LTC_MP_EQ) { + if ((err = mp_init_multi(&mu, &ma, LTC_NULL)) != CRYPT_OK) { goto error; } + if ((err = mp_montgomery_normalization(mu, m)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(a, mu, m, ma)) != CRYPT_OK) { goto error; } + } + + /* compute u1*mG + u2*mQ = mG */ + if (ltc_mp.ecc_mul2add == NULL) { + if ((err = ltc_mp.ecc_ptmul(u1, mG, mG, a, m, 0)) != CRYPT_OK) { goto error; } + if ((err = ltc_mp.ecc_ptmul(u2, mQ, mQ, a, m, 0)) != CRYPT_OK) { goto error; } + + /* add them */ + if ((err = ltc_mp.ecc_ptadd(mQ, mG, mG, ma, m, mp)) != CRYPT_OK) { goto error; } + + /* reduce */ + if ((err = ltc_mp.ecc_map(mG, m, mp)) != CRYPT_OK) { goto error; } + } else { + /* use Shamir's trick to compute u1*mG + u2*mQ using half of the doubles */ + if ((err = ltc_mp.ecc_mul2add(mG, u1, mQ, u2, mG, ma, m)) != CRYPT_OK) { goto error; } + } + + /* v = X_x1 mod n */ + if ((err = mp_mod(mG->x, p, v)) != CRYPT_OK) { goto error; } + + /* does v == r */ + if (mp_cmp(v, r) == LTC_MP_EQ) { + *stat = 1; + } + + /* clear up and return */ + err = CRYPT_OK; +error: + if (mG != NULL) ltc_ecc_del_point(mG); + if (mQ != NULL) ltc_ecc_del_point(mQ); + if (mu != NULL) mp_clear(mu); + if (ma != NULL) mp_clear(ma); + mp_clear_multi(r, s, v, w, u1, u2, e, a_plus3, LTC_NULL); + if (mp != NULL) mp_montgomery_free(mp); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_export_point.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_export_point.c new file mode 100644 index 0000000..022c6bc --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_export_point.c @@ -0,0 +1,53 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +int ltc_ecc_export_point(unsigned char *out, unsigned long *outlen, void *x, void *y, unsigned long size, int compressed) +{ + int err; + unsigned char buf[ECC_BUF_SIZE]; + unsigned long xsize, ysize; + + if (size > sizeof(buf)) return CRYPT_BUFFER_OVERFLOW; + if ((xsize = mp_unsigned_bin_size(x)) > size) return CRYPT_BUFFER_OVERFLOW; + if ((ysize = mp_unsigned_bin_size(y)) > size) return CRYPT_BUFFER_OVERFLOW; + + if(compressed) { + if (*outlen < (1 + size)) { + *outlen = 1 + size; + return CRYPT_BUFFER_OVERFLOW; + } + /* store first byte */ + out[0] = mp_isodd(y) ? 0x03 : 0x02; + /* pad and store x */ + zeromem(buf, sizeof(buf)); + if ((err = mp_to_unsigned_bin(x, buf + (size - xsize))) != CRYPT_OK) return err; + XMEMCPY(out+1, buf, size); + /* adjust outlen */ + *outlen = 1 + size; + } + else { + if (*outlen < (1 + 2*size)) { + *outlen = 1 + 2*size; + return CRYPT_BUFFER_OVERFLOW; + } + /* store byte 0x04 */ + out[0] = 0x04; + /* pad and store x */ + zeromem(buf, sizeof(buf)); + if ((err = mp_to_unsigned_bin(x, buf + (size - xsize))) != CRYPT_OK) return err; + XMEMCPY(out+1, buf, size); + /* pad and store y */ + zeromem(buf, sizeof(buf)); + if ((err = mp_to_unsigned_bin(y, buf + (size - ysize))) != CRYPT_OK) return err; + XMEMCPY(out+1+size, buf, size); + /* adjust outlen */ + *outlen = 1 + 2*size; + } + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_import_point.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_import_point.c new file mode 100644 index 0000000..6250fca --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_import_point.c @@ -0,0 +1,61 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +int ltc_ecc_import_point(const unsigned char *in, unsigned long inlen, void *prime, void *a, void *b, void *x, void *y) +{ + int err; + unsigned long size; + void *t1, *t2; + + /* init key + temporary numbers */ + if (mp_init_multi(&t1, &t2, LTC_NULL) != CRYPT_OK) { + return CRYPT_MEM; + } + + size = mp_unsigned_bin_size(prime); + + if (in[0] == 0x04 && (inlen&1) && ((inlen-1)>>1) == size) { + /* read uncompressed point */ + /* load x */ + if ((err = mp_read_unsigned_bin(x, (unsigned char *)in+1, size)) != CRYPT_OK) { goto cleanup; } + /* load y */ + if ((err = mp_read_unsigned_bin(y, (unsigned char *)in+1+size, size)) != CRYPT_OK) { goto cleanup; } + } + else if ((in[0] == 0x02 || in[0] == 0x03) && (inlen-1) == size && ltc_mp.sqrtmod_prime != NULL) { + /* read compressed point - BEWARE: requires sqrtmod_prime */ + /* load x */ + if ((err = mp_read_unsigned_bin(x, (unsigned char *)in+1, size)) != CRYPT_OK) { goto cleanup; } + /* compute x^3 */ + if ((err = mp_sqr(x, t1)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_mulmod(t1, x, prime, t1)) != CRYPT_OK) { goto cleanup; } + /* compute x^3 + a*x */ + if ((err = mp_mulmod(a, x, prime, t2)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_add(t1, t2, t1)) != CRYPT_OK) { goto cleanup; } + /* compute x^3 + a*x + b */ + if ((err = mp_add(t1, b, t1)) != CRYPT_OK) { goto cleanup; } + /* compute sqrt(x^3 + a*x + b) */ + if ((err = mp_sqrtmod_prime(t1, prime, t2)) != CRYPT_OK) { goto cleanup; } + /* adjust y */ + if ((mp_isodd(t2) && in[0] == 0x03) || (!mp_isodd(t2) && in[0] == 0x02)) { + if ((err = mp_mod(t2, prime, y)) != CRYPT_OK) { goto cleanup; } + } + else { + if ((err = mp_submod(prime, t2, prime, y)) != CRYPT_OK) { goto cleanup; } + } + } + else { + err = CRYPT_INVALID_PACKET; + goto cleanup; + } + + err = CRYPT_OK; +cleanup: + mp_clear_multi(t1, t2, LTC_NULL); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_is_point.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_is_point.c new file mode 100644 index 0000000..56a9537 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_is_point.c @@ -0,0 +1,62 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +/** Returns whether [x,y] is a point on curve defined by dp + @param dp curve parameters + @param x x point coordinate + @param y y point coordinate + @return CRYPT_OK if valid +*/ + +int ltc_ecc_is_point(const ltc_ecc_dp *dp, void *x, void *y) +{ + void *prime, *a, *b, *t1, *t2; + int err; + + prime = dp->prime; + b = dp->B; + a = dp->A; + + if ((err = mp_init_multi(&t1, &t2, LTC_NULL)) != CRYPT_OK) return err; + + /* compute y^2 */ + if ((err = mp_sqr(y, t1)) != CRYPT_OK) goto cleanup; + + /* compute x^3 */ + if ((err = mp_sqr(x, t2)) != CRYPT_OK) goto cleanup; + if ((err = mp_mod(t2, prime, t2)) != CRYPT_OK) goto cleanup; + if ((err = mp_mul(x, t2, t2)) != CRYPT_OK) goto cleanup; + + /* compute y^2 - x^3 */ + if ((err = mp_sub(t1, t2, t1)) != CRYPT_OK) goto cleanup; + + /* compute y^2 - x^3 - a*x */ + if ((err = mp_submod(prime, a, prime, t2)) != CRYPT_OK) goto cleanup; + if ((err = mp_mulmod(t2, x, prime, t2)) != CRYPT_OK) goto cleanup; + if ((err = mp_addmod(t1, t2, prime, t1)) != CRYPT_OK) goto cleanup; + + /* adjust range (0, prime) */ + while (mp_cmp_d(t1, 0) == LTC_MP_LT) { + if ((err = mp_add(t1, prime, t1)) != CRYPT_OK) goto cleanup; + } + while (mp_cmp(t1, prime) != LTC_MP_LT) { + if ((err = mp_sub(t1, prime, t1)) != CRYPT_OK) goto cleanup; + } + + /* compare to b */ + if (mp_cmp(t1, b) != LTC_MP_EQ) { + err = CRYPT_INVALID_PACKET; + } else { + err = CRYPT_OK; + } + +cleanup: + mp_clear_multi(t1, t2, LTC_NULL); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_is_point_at_infinity.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_is_point_at_infinity.c new file mode 100644 index 0000000..2afc4d4 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_is_point_at_infinity.c @@ -0,0 +1,53 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_MECC + +/* http://crypto.stackexchange.com/questions/41468/point-at-infinity-for-jacobian-coordinates + * a point at infinity is any point (x,y,0) such that y^2 == x^3, except (0,0,0) + */ + +int ltc_ecc_is_point_at_infinity(const ecc_point *P, void *modulus, int *retval) +{ + int err; + void *x3, *y2; + + /* trivial case */ + if (!mp_iszero(P->z)) { + *retval = 0; + return CRYPT_OK; + } + + /* point (0,0,0) is not at infinity */ + if (mp_iszero(P->x) && mp_iszero(P->y)) { + *retval = 0; + return CRYPT_OK; + } + + /* initialize */ + if ((err = mp_init_multi(&x3, &y2, LTC_NULL)) != CRYPT_OK) goto done; + + /* compute y^2 */ + if ((err = mp_mulmod(P->y, P->y, modulus, y2)) != CRYPT_OK) goto cleanup; + + /* compute x^3 */ + if ((err = mp_mulmod(P->x, P->x, modulus, x3)) != CRYPT_OK) goto cleanup; + if ((err = mp_mulmod(P->x, x3, modulus, x3)) != CRYPT_OK) goto cleanup; + + /* test y^2 == x^3 */ + err = CRYPT_OK; + if ((mp_cmp(x3, y2) == LTC_MP_EQ) && !mp_iszero(y2)) { + *retval = 1; + } else { + *retval = 0; + } + +cleanup: + mp_clear_multi(x3, y2, LTC_NULL); +done: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_map.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_map.c new file mode 100644 index 0000000..163fb41 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_map.c @@ -0,0 +1,63 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ltc_ecc_map.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Map a projective jacbobian point back to affine space + @param P [in/out] The point to map + @param modulus The modulus of the field the ECC curve is in + @param mp The "b" value from montgomery_setup() + @return CRYPT_OK on success +*/ +int ltc_ecc_map(ecc_point *P, void *modulus, void *mp) +{ + void *t1, *t2; + int err; + + LTC_ARGCHK(P != NULL); + LTC_ARGCHK(modulus != NULL); + LTC_ARGCHK(mp != NULL); + + if (mp_iszero(P->z)) { + return ltc_ecc_set_point_xyz(0, 0, 1, P); + } + + if ((err = mp_init_multi(&t1, &t2, LTC_NULL)) != CRYPT_OK) { + return err; + } + + /* first map z back to normal */ + if ((err = mp_montgomery_reduce(P->z, modulus, mp)) != CRYPT_OK) { goto done; } + + /* get 1/z */ + if ((err = mp_invmod(P->z, modulus, t1)) != CRYPT_OK) { goto done; } + + /* get 1/z^2 and 1/z^3 */ + if ((err = mp_sqr(t1, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_mod(t2, modulus, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_mul(t1, t2, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_mod(t1, modulus, t1)) != CRYPT_OK) { goto done; } + + /* multiply against x/y */ + if ((err = mp_mul(P->x, t2, P->x)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(P->x, modulus, mp)) != CRYPT_OK) { goto done; } + if ((err = mp_mul(P->y, t1, P->y)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(P->y, modulus, mp)) != CRYPT_OK) { goto done; } + if ((err = mp_set(P->z, 1)) != CRYPT_OK) { goto done; } + + err = CRYPT_OK; +done: + mp_clear_multi(t1, t2, LTC_NULL); + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mul2add.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mul2add.c new file mode 100644 index 0000000..a0188e3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mul2add.c @@ -0,0 +1,192 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ltc_ecc_mul2add.c + ECC Crypto, Shamir's Trick, Tom St Denis +*/ + +#ifdef LTC_MECC + +#ifdef LTC_ECC_SHAMIR + +/** Computes kA*A + kB*B = C using Shamir's Trick + @param A First point to multiply + @param kA What to multiple A by + @param B Second point to multiply + @param kB What to multiple B by + @param C [out] Destination point (can overlap with A or B) + @param ma ECC curve parameter a in montgomery form + @param modulus Modulus for curve + @return CRYPT_OK on success +*/ +int ltc_ecc_mul2add(const ecc_point *A, void *kA, + const ecc_point *B, void *kB, + ecc_point *C, + void *ma, + void *modulus) +{ + ecc_point *precomp[16]; + unsigned bitbufA, bitbufB, lenA, lenB, len, nA, nB, nibble; + unsigned x, y; + unsigned char *tA, *tB; + int err, first; + void *mp, *mu; + + /* argchks */ + LTC_ARGCHK(A != NULL); + LTC_ARGCHK(B != NULL); + LTC_ARGCHK(C != NULL); + LTC_ARGCHK(kA != NULL); + LTC_ARGCHK(kB != NULL); + LTC_ARGCHK(modulus != NULL); + + /* allocate memory */ + tA = XCALLOC(1, ECC_BUF_SIZE); + if (tA == NULL) { + return CRYPT_MEM; + } + tB = XCALLOC(1, ECC_BUF_SIZE); + if (tB == NULL) { + XFREE(tA); + return CRYPT_MEM; + } + + /* get sizes */ + lenA = mp_unsigned_bin_size(kA); + lenB = mp_unsigned_bin_size(kB); + len = MAX(lenA, lenB); + + /* sanity check */ + if ((lenA > ECC_BUF_SIZE) || (lenB > ECC_BUF_SIZE)) { + err = CRYPT_INVALID_ARG; + goto ERR_T; + } + + /* extract and justify kA */ + mp_to_unsigned_bin(kA, (len - lenA) + tA); + + /* extract and justify kB */ + mp_to_unsigned_bin(kB, (len - lenB) + tB); + + /* allocate the table */ + for (x = 0; x < 16; x++) { + precomp[x] = ltc_ecc_new_point(); + if (precomp[x] == NULL) { + for (y = 0; y < x; ++y) { + ltc_ecc_del_point(precomp[y]); + } + err = CRYPT_MEM; + goto ERR_T; + } + } + + /* init montgomery reduction */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { + goto ERR_P; + } + if ((err = mp_init(&mu)) != CRYPT_OK) { + goto ERR_MP; + } + if ((err = mp_montgomery_normalization(mu, modulus)) != CRYPT_OK) { + goto ERR_MU; + } + + /* copy ones ... */ + if ((err = mp_mulmod(A->x, mu, modulus, precomp[1]->x)) != CRYPT_OK) { goto ERR_MU; } + if ((err = mp_mulmod(A->y, mu, modulus, precomp[1]->y)) != CRYPT_OK) { goto ERR_MU; } + if ((err = mp_mulmod(A->z, mu, modulus, precomp[1]->z)) != CRYPT_OK) { goto ERR_MU; } + + if ((err = mp_mulmod(B->x, mu, modulus, precomp[1<<2]->x)) != CRYPT_OK) { goto ERR_MU; } + if ((err = mp_mulmod(B->y, mu, modulus, precomp[1<<2]->y)) != CRYPT_OK) { goto ERR_MU; } + if ((err = mp_mulmod(B->z, mu, modulus, precomp[1<<2]->z)) != CRYPT_OK) { goto ERR_MU; } + + /* precomp [i,0](A + B) table */ + if ((err = ltc_mp.ecc_ptdbl(precomp[1], precomp[2], ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + if ((err = ltc_mp.ecc_ptadd(precomp[1], precomp[2], precomp[3], ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + + /* precomp [0,i](A + B) table */ + if ((err = ltc_mp.ecc_ptdbl(precomp[1<<2], precomp[2<<2], ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + if ((err = ltc_mp.ecc_ptadd(precomp[1<<2], precomp[2<<2], precomp[3<<2], ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + + /* precomp [i,j](A + B) table (i != 0, j != 0) */ + for (x = 1; x < 4; x++) { + for (y = 1; y < 4; y++) { + if ((err = ltc_mp.ecc_ptadd(precomp[x], precomp[(y<<2)], precomp[x+(y<<2)], ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + } + } + + nibble = 3; + first = 1; + bitbufA = tA[0]; + bitbufB = tB[0]; + + /* for every byte of the multiplicands */ + for (x = 0;; ) { + /* grab a nibble */ + if (++nibble == 4) { + if (x == len) break; + bitbufA = tA[x]; + bitbufB = tB[x]; + nibble = 0; + ++x; + } + + /* extract two bits from both, shift/update */ + nA = (bitbufA >> 6) & 0x03; + nB = (bitbufB >> 6) & 0x03; + bitbufA = (bitbufA << 2) & 0xFF; + bitbufB = (bitbufB << 2) & 0xFF; + + /* if both zero, if first, continue */ + if ((nA == 0) && (nB == 0) && (first == 1)) { + continue; + } + + /* double twice, only if this isn't the first */ + if (first == 0) { + /* double twice */ + if ((err = ltc_mp.ecc_ptdbl(C, C, ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + if ((err = ltc_mp.ecc_ptdbl(C, C, ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + } + + /* if not both zero */ + if ((nA != 0) || (nB != 0)) { + if (first == 1) { + /* if first, copy from table */ + first = 0; + if ((err = ltc_ecc_copy_point(precomp[nA + (nB<<2)], C)) != CRYPT_OK) { goto ERR_MU; } + } else { + /* if not first, add from table */ + if ((err = ltc_mp.ecc_ptadd(C, precomp[nA + (nB<<2)], C, ma, modulus, mp)) != CRYPT_OK) { goto ERR_MU; } + } + } + } + + /* reduce to affine */ + err = ltc_ecc_map(C, modulus, mp); + + /* clean up */ +ERR_MU: + mp_clear(mu); +ERR_MP: + mp_montgomery_free(mp); +ERR_P: + for (x = 0; x < 16; x++) { + ltc_ecc_del_point(precomp[x]); + } +ERR_T: +#ifdef LTC_CLEAN_STACK + zeromem(tA, ECC_BUF_SIZE); + zeromem(tB, ECC_BUF_SIZE); +#endif + XFREE(tA); + XFREE(tB); + + return err; +} + +#endif +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mulmod.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mulmod.c new file mode 100644 index 0000000..cabe266 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mulmod.c @@ -0,0 +1,204 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ltc_ecc_mulmod.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC +#ifndef LTC_ECC_TIMING_RESISTANT + +/* size of sliding window, don't change this! */ +#define WINSIZE 4 + +/** + Perform a point multiplication + @param k The scalar to multiply by + @param G The base point + @param R [out] Destination for kG + @param modulus The modulus of the field the ECC curve is in + @param map Boolean whether to map back to affine or not (1==map, 0 == leave in projective) + @return CRYPT_OK on success +*/ +int ltc_ecc_mulmod(void *k, const ecc_point *G, ecc_point *R, void *a, void *modulus, int map) +{ + ecc_point *tG, *M[8]; + int i, j, err, inf; + void *mp = NULL, *mu = NULL, *ma = NULL, *a_plus3 = NULL; + ltc_mp_digit buf; + int first, bitbuf, bitcpy, bitcnt, mode, digidx; + + LTC_ARGCHK(k != NULL); + LTC_ARGCHK(G != NULL); + LTC_ARGCHK(R != NULL); + LTC_ARGCHK(modulus != NULL); + + if ((err = ltc_ecc_is_point_at_infinity(G, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* return the point at infinity */ + return ltc_ecc_set_point_xyz(1, 1, 0, R); + } + + /* init montgomery reduction */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { goto error; } + if ((err = mp_init(&mu)) != CRYPT_OK) { goto error; } + if ((err = mp_montgomery_normalization(mu, modulus)) != CRYPT_OK) { goto error; } + + /* for curves with a == -3 keep ma == NULL */ + if ((err = mp_init(&a_plus3)) != CRYPT_OK) { goto error; } + if ((err = mp_add_d(a, 3, a_plus3)) != CRYPT_OK) { goto error; } + if (mp_cmp(a_plus3, modulus) != LTC_MP_EQ) { + if ((err = mp_init(&ma)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(a, mu, modulus, ma)) != CRYPT_OK) { goto error; } + } + + /* alloc ram for window temps */ + for (i = 0; i < 8; i++) { + M[i] = ltc_ecc_new_point(); + if (M[i] == NULL) { + for (j = 0; j < i; j++) { + ltc_ecc_del_point(M[j]); + } + err = CRYPT_MEM; + goto error; + } + } + + /* make a copy of G incase R==G */ + tG = ltc_ecc_new_point(); + if (tG == NULL) { err = CRYPT_MEM; goto done; } + + /* tG = G and convert to montgomery */ + if (mp_cmp_d(mu, 1) == LTC_MP_EQ) { + if ((err = ltc_ecc_copy_point(G, tG)) != CRYPT_OK) { goto done; } + } else { + if ((err = mp_mulmod(G->x, mu, modulus, tG->x)) != CRYPT_OK) { goto done; } + if ((err = mp_mulmod(G->y, mu, modulus, tG->y)) != CRYPT_OK) { goto done; } + if ((err = mp_mulmod(G->z, mu, modulus, tG->z)) != CRYPT_OK) { goto done; } + } + mp_clear(mu); + mu = NULL; + + /* calc the M tab, which holds kG for k==8..15 */ + /* M[0] == 8G */ + if ((err = ltc_mp.ecc_ptdbl(tG, M[0], ma, modulus, mp)) != CRYPT_OK) { goto done; } + if ((err = ltc_mp.ecc_ptdbl(M[0], M[0], ma, modulus, mp)) != CRYPT_OK) { goto done; } + if ((err = ltc_mp.ecc_ptdbl(M[0], M[0], ma, modulus, mp)) != CRYPT_OK) { goto done; } + + /* now find (8+k)G for k=1..7 */ + for (j = 9; j < 16; j++) { + if ((err = ltc_mp.ecc_ptadd(M[j-9], tG, M[j-8], ma, modulus, mp)) != CRYPT_OK) { goto done; } + } + + /* setup sliding window */ + mode = 0; + bitcnt = 1; + buf = 0; + digidx = mp_get_digit_count(k) - 1; + bitcpy = bitbuf = 0; + first = 1; + + /* perform ops */ + for (;;) { + /* grab next digit as required */ + if (--bitcnt == 0) { + if (digidx == -1) { + break; + } + buf = mp_get_digit(k, digidx); + bitcnt = (int) ltc_mp.bits_per_digit; + --digidx; + } + + /* grab the next msb from the ltiplicand */ + i = (buf >> (ltc_mp.bits_per_digit - 1)) & 1; + buf <<= 1; + + /* skip leading zero bits */ + if (mode == 0 && i == 0) { + continue; + } + + /* if the bit is zero and mode == 1 then we double */ + if (mode == 1 && i == 0) { + if ((err = ltc_mp.ecc_ptdbl(R, R, ma, modulus, mp)) != CRYPT_OK) { goto done; } + continue; + } + + /* else we add it to the window */ + bitbuf |= (i << (WINSIZE - ++bitcpy)); + mode = 2; + + if (bitcpy == WINSIZE) { + /* if this is the first window we do a simple copy */ + if (first == 1) { + /* R = kG [k = first window] */ + if ((err = ltc_ecc_copy_point(M[bitbuf-8], R)) != CRYPT_OK) { goto done; } + first = 0; + } else { + /* normal window */ + /* ok window is filled so double as required and add */ + /* double first */ + for (j = 0; j < WINSIZE; j++) { + if ((err = ltc_mp.ecc_ptdbl(R, R, ma, modulus, mp)) != CRYPT_OK) { goto done; } + } + + /* then add, bitbuf will be 8..15 [8..2^WINSIZE] guaranteed */ + if ((err = ltc_mp.ecc_ptadd(R, M[bitbuf-8], R, ma, modulus, mp)) != CRYPT_OK) { goto done; } + } + /* empty window and reset */ + bitcpy = bitbuf = 0; + mode = 1; + } + } + + /* if bits remain then double/add */ + if (mode == 2 && bitcpy > 0) { + /* double then add */ + for (j = 0; j < bitcpy; j++) { + /* only double if we have had at least one add first */ + if (first == 0) { + if ((err = ltc_mp.ecc_ptdbl(R, R, ma, modulus, mp)) != CRYPT_OK) { goto done; } + } + + bitbuf <<= 1; + if ((bitbuf & (1 << WINSIZE)) != 0) { + if (first == 1){ + /* first add, so copy */ + if ((err = ltc_ecc_copy_point(tG, R)) != CRYPT_OK) { goto done; } + first = 0; + } else { + /* then add */ + if ((err = ltc_mp.ecc_ptadd(R, tG, R, ma, modulus, mp)) != CRYPT_OK) { goto done; } + } + } + } + } + + /* map R back from projective space */ + if (map) { + err = ltc_ecc_map(R, modulus, mp); + } else { + err = CRYPT_OK; + } +done: + ltc_ecc_del_point(tG); + for (i = 0; i < 8; i++) { + ltc_ecc_del_point(M[i]); + } +error: + if (ma != NULL) mp_clear(ma); + if (a_plus3 != NULL) mp_clear(a_plus3); + if (mu != NULL) mp_clear(mu); + if (mp != NULL) mp_montgomery_free(mp); + return err; +} + +#endif + +#undef WINSIZE + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mulmod_timing.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mulmod_timing.c new file mode 100644 index 0000000..96f50f9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_mulmod_timing.c @@ -0,0 +1,151 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ltc_ecc_mulmod_timing.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +#ifdef LTC_ECC_TIMING_RESISTANT + +/** + Perform a point multiplication (timing resistant) + @param k The scalar to multiply by + @param G The base point + @param R [out] Destination for kG + @param a ECC curve parameter a + @param modulus The modulus of the field the ECC curve is in + @param map Boolean whether to map back to affine or not (1==map, 0 == leave in projective) + @return CRYPT_OK on success +*/ +int ltc_ecc_mulmod(void *k, const ecc_point *G, ecc_point *R, void *a, void *modulus, int map) +{ + ecc_point *tG, *M[3]; + int i, j, err, inf; + void *mp = NULL, *mu = NULL, *ma = NULL, *a_plus3 = NULL; + ltc_mp_digit buf; + int bitcnt, mode, digidx; + + LTC_ARGCHK(k != NULL); + LTC_ARGCHK(G != NULL); + LTC_ARGCHK(R != NULL); + LTC_ARGCHK(modulus != NULL); + + if ((err = ltc_ecc_is_point_at_infinity(G, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* return the point at infinity */ + return ltc_ecc_set_point_xyz(1, 1, 0, R); + } + + /* init montgomery reduction */ + if ((err = mp_montgomery_setup(modulus, &mp)) != CRYPT_OK) { goto error; } + if ((err = mp_init(&mu)) != CRYPT_OK) { goto error; } + if ((err = mp_montgomery_normalization(mu, modulus)) != CRYPT_OK) { goto error; } + + /* for curves with a == -3 keep ma == NULL */ + if ((err = mp_init(&a_plus3)) != CRYPT_OK) { goto error; } + if ((err = mp_add_d(a, 3, a_plus3)) != CRYPT_OK) { goto error; } + if (mp_cmp(a_plus3, modulus) != LTC_MP_EQ) { + if ((err = mp_init(&ma)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(a, mu, modulus, ma)) != CRYPT_OK) { goto error; } + } + + /* alloc ram for window temps */ + for (i = 0; i < 3; i++) { + M[i] = ltc_ecc_new_point(); + if (M[i] == NULL) { + for (j = 0; j < i; j++) { + ltc_ecc_del_point(M[j]); + } + mp_clear(mu); + mp_montgomery_free(mp); + return CRYPT_MEM; + } + } + + /* make a copy of G incase R==G */ + tG = ltc_ecc_new_point(); + if (tG == NULL) { err = CRYPT_MEM; goto done; } + + /* tG = G and convert to montgomery */ + if ((err = mp_mulmod(G->x, mu, modulus, tG->x)) != CRYPT_OK) { goto done; } + if ((err = mp_mulmod(G->y, mu, modulus, tG->y)) != CRYPT_OK) { goto done; } + if ((err = mp_mulmod(G->z, mu, modulus, tG->z)) != CRYPT_OK) { goto done; } + mp_clear(mu); + mu = NULL; + + /* calc the M tab */ + /* M[0] == G */ + if ((err = ltc_ecc_copy_point(tG, M[0])) != CRYPT_OK) { goto done; } + /* M[1] == 2G */ + if ((err = ltc_mp.ecc_ptdbl(tG, M[1], ma, modulus, mp)) != CRYPT_OK) { goto done; } + + /* setup sliding window */ + mode = 0; + bitcnt = 1; + buf = 0; + digidx = mp_get_digit_count(k) - 1; + + /* perform ops */ + for (;;) { + /* grab next digit as required */ + if (--bitcnt == 0) { + if (digidx == -1) { + break; + } + buf = mp_get_digit(k, digidx); + bitcnt = (int) MP_DIGIT_BIT; + --digidx; + } + + /* grab the next msb from the ltiplicand */ + i = (int)((buf >> (MP_DIGIT_BIT - 1)) & 1); + buf <<= 1; + + if (mode == 0 && i == 0) { + /* dummy operations */ + if ((err = ltc_mp.ecc_ptadd(M[0], M[1], M[2], ma, modulus, mp)) != CRYPT_OK) { goto done; } + if ((err = ltc_mp.ecc_ptdbl(M[1], M[2], ma, modulus, mp)) != CRYPT_OK) { goto done; } + continue; + } + + if (mode == 0 && i == 1) { + mode = 1; + /* dummy operations */ + if ((err = ltc_mp.ecc_ptadd(M[0], M[1], M[2], ma, modulus, mp)) != CRYPT_OK) { goto done; } + if ((err = ltc_mp.ecc_ptdbl(M[1], M[2], ma, modulus, mp)) != CRYPT_OK) { goto done; } + continue; + } + + if ((err = ltc_mp.ecc_ptadd(M[0], M[1], M[i^1], ma, modulus, mp)) != CRYPT_OK) { goto done; } + if ((err = ltc_mp.ecc_ptdbl(M[i], M[i], ma, modulus, mp)) != CRYPT_OK) { goto done; } + } + + /* copy result out */ + if ((err = ltc_ecc_copy_point(M[0], R)) != CRYPT_OK) { goto done; } + + /* map R back from projective space */ + if (map) { + err = ltc_ecc_map(R, modulus, mp); + } else { + err = CRYPT_OK; + } +done: + ltc_ecc_del_point(tG); + for (i = 0; i < 3; i++) { + ltc_ecc_del_point(M[i]); + } +error: + if (ma != NULL) mp_clear(ma); + if (a_plus3 != NULL) mp_clear(a_plus3); + if (mu != NULL) mp_clear(mu); + if (mp != NULL) mp_montgomery_free(mp); + return err; +} + +#endif +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_points.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_points.c new file mode 100644 index 0000000..b6872d3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_points.c @@ -0,0 +1,62 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ltc_ecc_points.c + ECC Crypto, Tom St Denis +*/ + +#ifdef LTC_MECC + +/** + Allocate a new ECC point + @return A newly allocated point or NULL on error +*/ +ecc_point *ltc_ecc_new_point(void) +{ + ecc_point *p; + p = XCALLOC(1, sizeof(*p)); + if (p == NULL) { + return NULL; + } + if (mp_init_multi_size(LTC_MAX_ECC * 2, + &p->x, &p->y, &p->z, LTC_NULL) != CRYPT_OK) { + XFREE(p); + return NULL; + } + return p; +} + +/** Free an ECC point from memory + @param p The point to free +*/ +void ltc_ecc_del_point(ecc_point *p) +{ + /* prevents free'ing null arguments */ + if (p != NULL) { + mp_clear_multi(p->x, p->y, p->z, LTC_NULL); /* note: p->z may be NULL but that's ok with this function anyways */ + XFREE(p); + } +} + +int ltc_ecc_set_point_xyz(ltc_mp_digit x, ltc_mp_digit y, ltc_mp_digit z, ecc_point *p) +{ + int err; + if ((err = ltc_mp.set_int(p->x, x)) != CRYPT_OK) return err; + if ((err = ltc_mp.set_int(p->y, y)) != CRYPT_OK) return err; + if ((err = ltc_mp.set_int(p->z, z)) != CRYPT_OK) return err; + return CRYPT_OK; +} + +int ltc_ecc_copy_point(const ecc_point *src, ecc_point *dst) +{ + int err; + if ((err = ltc_mp.copy(src->x, dst->x)) != CRYPT_OK) return err; + if ((err = ltc_mp.copy(src->y, dst->y)) != CRYPT_OK) return err; + if ((err = ltc_mp.copy(src->z, dst->z)) != CRYPT_OK) return err; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_projective_add_point.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_projective_add_point.c new file mode 100644 index 0000000..448c2bb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_projective_add_point.c @@ -0,0 +1,198 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file ltc_ecc_projective_add_point.c + ECC Crypto, Tom St Denis +*/ + +#if defined(LTC_MECC) && (!defined(LTC_MECC_ACCEL) || defined(LTM_DESC)) + +/** + Add two ECC points + @param P The point to add + @param Q The point to add + @param R [out] The destination of the double + @param ma ECC curve parameter a in montgomery form + @param modulus The modulus of the field the ECC curve is in + @param mp The "b" value from montgomery_setup() + @return CRYPT_OK on success +*/ +int ltc_ecc_projective_add_point(const ecc_point *P, const ecc_point *Q, ecc_point *R, void *ma, void *modulus, void *mp) +{ + void *t1, *t2, *x, *y, *z; + int err, inf; + + LTC_ARGCHK(P != NULL); + LTC_ARGCHK(Q != NULL); + LTC_ARGCHK(R != NULL); + LTC_ARGCHK(modulus != NULL); + LTC_ARGCHK(mp != NULL); + + if ((err = mp_init_multi(&t1, &t2, &x, &y, &z, LTC_NULL)) != CRYPT_OK) { + return err; + } + + if ((err = ltc_ecc_is_point_at_infinity(P, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* P is point at infinity >> Result = Q */ + err = ltc_ecc_copy_point(Q, R); + goto done; + } + + if ((err = ltc_ecc_is_point_at_infinity(Q, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* Q is point at infinity >> Result = P */ + err = ltc_ecc_copy_point(P, R); + goto done; + } + + if ((mp_cmp(P->x, Q->x) == LTC_MP_EQ) && (mp_cmp(P->z, Q->z) == LTC_MP_EQ)) { + if (mp_cmp(P->y, Q->y) == LTC_MP_EQ) { + /* here P = Q >> Result = 2 * P (use doubling) */ + mp_clear_multi(t1, t2, x, y, z, LTC_NULL); + return ltc_ecc_projective_dbl_point(P, R, ma, modulus, mp); + } + if ((err = mp_sub(modulus, Q->y, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(P->y, t1) == LTC_MP_EQ) { + /* here Q = -P >>> Result = the point at infinity */ + err = ltc_ecc_set_point_xyz(1, 1, 0, R); + goto done; + } + } + + if ((err = mp_copy(P->x, x)) != CRYPT_OK) { goto done; } + if ((err = mp_copy(P->y, y)) != CRYPT_OK) { goto done; } + if ((err = mp_copy(P->z, z)) != CRYPT_OK) { goto done; } + + /* if Z is one then these are no-operations */ + if (Q->z != NULL) { + /* T1 = Z' * Z' */ + if ((err = mp_sqr(Q->z, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + /* X = X * T1 */ + if ((err = mp_mul(t1, x, x)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(x, modulus, mp)) != CRYPT_OK) { goto done; } + /* T1 = Z' * T1 */ + if ((err = mp_mul(Q->z, t1, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + /* Y = Y * T1 */ + if ((err = mp_mul(t1, y, y)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(y, modulus, mp)) != CRYPT_OK) { goto done; } + } + + /* T1 = Z*Z */ + if ((err = mp_sqr(z, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + /* T2 = X' * T1 */ + if ((err = mp_mul(Q->x, t1, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t2, modulus, mp)) != CRYPT_OK) { goto done; } + /* T1 = Z * T1 */ + if ((err = mp_mul(z, t1, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + /* T1 = Y' * T1 */ + if ((err = mp_mul(Q->y, t1, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + + /* Y = Y - T1 */ + if ((err = mp_sub(y, t1, y)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(y, 0) == LTC_MP_LT) { + if ((err = mp_add(y, modulus, y)) != CRYPT_OK) { goto done; } + } + /* T1 = 2T1 */ + if ((err = mp_add(t1, t1, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + /* T1 = Y + T1 */ + if ((err = mp_add(t1, y, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + /* X = X - T2 */ + if ((err = mp_sub(x, t2, x)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(x, 0) == LTC_MP_LT) { + if ((err = mp_add(x, modulus, x)) != CRYPT_OK) { goto done; } + } + /* T2 = 2T2 */ + if ((err = mp_add(t2, t2, t2)) != CRYPT_OK) { goto done; } + if (mp_cmp(t2, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t2, modulus, t2)) != CRYPT_OK) { goto done; } + } + /* T2 = X + T2 */ + if ((err = mp_add(t2, x, t2)) != CRYPT_OK) { goto done; } + if (mp_cmp(t2, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t2, modulus, t2)) != CRYPT_OK) { goto done; } + } + + /* if Z' != 1 */ + if (Q->z != NULL) { + /* Z = Z * Z' */ + if ((err = mp_mul(z, Q->z, z)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(z, modulus, mp)) != CRYPT_OK) { goto done; } + } + + /* Z = Z * X */ + if ((err = mp_mul(z, x, z)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(z, modulus, mp)) != CRYPT_OK) { goto done; } + + /* T1 = T1 * X */ + if ((err = mp_mul(t1, x, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + /* X = X * X */ + if ((err = mp_sqr(x, x)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(x, modulus, mp)) != CRYPT_OK) { goto done; } + /* T2 = T2 * x */ + if ((err = mp_mul(t2, x, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t2, modulus, mp)) != CRYPT_OK) { goto done; } + /* T1 = T1 * X */ + if ((err = mp_mul(t1, x, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + + /* X = Y*Y */ + if ((err = mp_sqr(y, x)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(x, modulus, mp)) != CRYPT_OK) { goto done; } + /* X = X - T2 */ + if ((err = mp_sub(x, t2, x)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(x, 0) == LTC_MP_LT) { + if ((err = mp_add(x, modulus, x)) != CRYPT_OK) { goto done; } + } + + /* T2 = T2 - X */ + if ((err = mp_sub(t2, x, t2)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(t2, 0) == LTC_MP_LT) { + if ((err = mp_add(t2, modulus, t2)) != CRYPT_OK) { goto done; } + } + /* T2 = T2 - X */ + if ((err = mp_sub(t2, x, t2)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(t2, 0) == LTC_MP_LT) { + if ((err = mp_add(t2, modulus, t2)) != CRYPT_OK) { goto done; } + } + /* T2 = T2 * Y */ + if ((err = mp_mul(t2, y, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t2, modulus, mp)) != CRYPT_OK) { goto done; } + /* Y = T2 - T1 */ + if ((err = mp_sub(t2, t1, y)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(y, 0) == LTC_MP_LT) { + if ((err = mp_add(y, modulus, y)) != CRYPT_OK) { goto done; } + } + /* Y = Y/2 */ + if (mp_isodd(y)) { + if ((err = mp_add(y, modulus, y)) != CRYPT_OK) { goto done; } + } + if ((err = mp_div_2(y, y)) != CRYPT_OK) { goto done; } + + if ((err = mp_copy(x, R->x)) != CRYPT_OK) { goto done; } + if ((err = mp_copy(y, R->y)) != CRYPT_OK) { goto done; } + if ((err = mp_copy(z, R->z)) != CRYPT_OK) { goto done; } + + err = CRYPT_OK; +done: + mp_clear_multi(t1, t2, x, y, z, LTC_NULL); + return err; +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_projective_dbl_point.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_projective_dbl_point.c new file mode 100644 index 0000000..c99a267 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_projective_dbl_point.c @@ -0,0 +1,182 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/* ### Point doubling in Jacobian coordinate system ### + * + * let us have a curve: y^2 = x^3 + a*x + b + * in Jacobian coordinates it becomes: y^2 = x^3 + a*x*z^4 + b*z^6 + * + * The doubling of P = (Xp, Yp, Zp) is given by R = (Xr, Yr, Zr) where: + * Xr = M^2 - 2*S + * Yr = M * (S - Xr) - 8*T + * Zr = 2 * Yp * Zp + * + * M = 3 * Xp^2 + a*Zp^4 + * T = Yp^4 + * S = 4 * Xp * Yp^2 + * + * SPECIAL CASE: when a == -3 we can compute M as + * M = 3 * (Xp^2 - Zp^4) = 3 * (Xp + Zp^2) * (Xp - Zp^2) + */ + +/** + @file ltc_ecc_projective_dbl_point.c + ECC Crypto, Tom St Denis +*/ + +#if defined(LTC_MECC) && (!defined(LTC_MECC_ACCEL) || defined(LTM_DESC)) + +/** + Double an ECC point + @param P The point to double + @param R [out] The destination of the double + @param ma ECC curve parameter a in montgomery form + @param modulus The modulus of the field the ECC curve is in + @param mp The "b" value from montgomery_setup() + @return CRYPT_OK on success +*/ +int ltc_ecc_projective_dbl_point(const ecc_point *P, ecc_point *R, void *ma, void *modulus, void *mp) +{ + void *t1, *t2; + int err, inf; + + LTC_ARGCHK(P != NULL); + LTC_ARGCHK(R != NULL); + LTC_ARGCHK(modulus != NULL); + LTC_ARGCHK(mp != NULL); + + if ((err = mp_init_multi(&t1, &t2, LTC_NULL)) != CRYPT_OK) { + return err; + } + + if (P != R) { + if ((err = ltc_ecc_copy_point(P, R)) != CRYPT_OK) { goto done; } + } + + if ((err = ltc_ecc_is_point_at_infinity(P, modulus, &inf)) != CRYPT_OK) return err; + if (inf) { + /* if P is point at infinity >> Result = point at infinity */ + err = ltc_ecc_set_point_xyz(1, 1, 0, R); + goto done; + } + + /* t1 = Z * Z */ + if ((err = mp_sqr(R->z, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + /* Z = Y * Z */ + if ((err = mp_mul(R->z, R->y, R->z)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(R->z, modulus, mp)) != CRYPT_OK) { goto done; } + /* Z = 2Z */ + if ((err = mp_add(R->z, R->z, R->z)) != CRYPT_OK) { goto done; } + if (mp_cmp(R->z, modulus) != LTC_MP_LT) { + if ((err = mp_sub(R->z, modulus, R->z)) != CRYPT_OK) { goto done; } + } + + if (ma == NULL) { /* special case for curves with a == -3 (10% faster than general case) */ + /* T2 = X - T1 */ + if ((err = mp_sub(R->x, t1, t2)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(t2, 0) == LTC_MP_LT) { + if ((err = mp_add(t2, modulus, t2)) != CRYPT_OK) { goto done; } + } + /* T1 = X + T1 */ + if ((err = mp_add(t1, R->x, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + /* T2 = T1 * T2 */ + if ((err = mp_mul(t1, t2, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t2, modulus, mp)) != CRYPT_OK) { goto done; } + /* T1 = 2T2 */ + if ((err = mp_add(t2, t2, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + /* T1 = T1 + T2 */ + if ((err = mp_add(t1, t2, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + } + else { + /* T2 = T1 * T1 */ + if ((err = mp_sqr(t1, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t2, modulus, mp)) != CRYPT_OK) { goto done; } + /* T1 = T2 * a */ + if ((err = mp_mul(t2, ma, t1)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t1, modulus, mp)) != CRYPT_OK) { goto done; } + /* T2 = X * X */ + if ((err = mp_sqr(R->x, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t2, modulus, mp)) != CRYPT_OK) { goto done; } + /* T1 = T2 + T1 */ + if ((err = mp_add(t1, t2, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + /* T1 = T2 + T1 */ + if ((err = mp_add(t1, t2, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + /* T1 = T2 + T1 */ + if ((err = mp_add(t1, t2, t1)) != CRYPT_OK) { goto done; } + if (mp_cmp(t1, modulus) != LTC_MP_LT) { + if ((err = mp_sub(t1, modulus, t1)) != CRYPT_OK) { goto done; } + } + } + + /* Y = 2Y */ + if ((err = mp_add(R->y, R->y, R->y)) != CRYPT_OK) { goto done; } + if (mp_cmp(R->y, modulus) != LTC_MP_LT) { + if ((err = mp_sub(R->y, modulus, R->y)) != CRYPT_OK) { goto done; } + } + /* Y = Y * Y */ + if ((err = mp_sqr(R->y, R->y)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(R->y, modulus, mp)) != CRYPT_OK) { goto done; } + /* T2 = Y * Y */ + if ((err = mp_sqr(R->y, t2)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(t2, modulus, mp)) != CRYPT_OK) { goto done; } + /* T2 = T2/2 */ + if (mp_isodd(t2)) { + if ((err = mp_add(t2, modulus, t2)) != CRYPT_OK) { goto done; } + } + if ((err = mp_div_2(t2, t2)) != CRYPT_OK) { goto done; } + /* Y = Y * X */ + if ((err = mp_mul(R->y, R->x, R->y)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(R->y, modulus, mp)) != CRYPT_OK) { goto done; } + + /* X = T1 * T1 */ + if ((err = mp_sqr(t1, R->x)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(R->x, modulus, mp)) != CRYPT_OK) { goto done; } + /* X = X - Y */ + if ((err = mp_sub(R->x, R->y, R->x)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(R->x, 0) == LTC_MP_LT) { + if ((err = mp_add(R->x, modulus, R->x)) != CRYPT_OK) { goto done; } + } + /* X = X - Y */ + if ((err = mp_sub(R->x, R->y, R->x)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(R->x, 0) == LTC_MP_LT) { + if ((err = mp_add(R->x, modulus, R->x)) != CRYPT_OK) { goto done; } + } + + /* Y = Y - X */ + if ((err = mp_sub(R->y, R->x, R->y)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(R->y, 0) == LTC_MP_LT) { + if ((err = mp_add(R->y, modulus, R->y)) != CRYPT_OK) { goto done; } + } + /* Y = Y * T1 */ + if ((err = mp_mul(R->y, t1, R->y)) != CRYPT_OK) { goto done; } + if ((err = mp_montgomery_reduce(R->y, modulus, mp)) != CRYPT_OK) { goto done; } + /* Y = Y - T2 */ + if ((err = mp_sub(R->y, t2, R->y)) != CRYPT_OK) { goto done; } + if (mp_cmp_d(R->y, 0) == LTC_MP_LT) { + if ((err = mp_add(R->y, modulus, R->y)) != CRYPT_OK) { goto done; } + } + + err = CRYPT_OK; +done: + mp_clear_multi(t2, t1, LTC_NULL); + return err; +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_verify_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_verify_key.c new file mode 100644 index 0000000..26c8383 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/ltc_ecc_verify_key.c @@ -0,0 +1,59 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/* origin of this code - OLPC */ + +#ifdef LTC_MECC + +/** + Verify a key according to ANSI spec + @param key The key to validate + @return CRYPT_OK if successful +*/ + +int ltc_ecc_verify_key(const ecc_key *key) +{ + int err, inf; + ecc_point *point; + void *prime = key->dp.prime; + void *order = key->dp.order; + void *a = key->dp.A; + + /* Test 1: Are the x and y points of the public key in the field? */ + if (ltc_mp.compare_d(key->pubkey.z, 1) == LTC_MP_EQ) { + if ((ltc_mp.compare(key->pubkey.x, prime) != LTC_MP_LT) || + (ltc_mp.compare(key->pubkey.y, prime) != LTC_MP_LT) || + (ltc_mp.compare_d(key->pubkey.x, 0) == LTC_MP_LT) || + (ltc_mp.compare_d(key->pubkey.y, 0) == LTC_MP_LT) || + (mp_iszero(key->pubkey.x) && mp_iszero(key->pubkey.y)) + ) + { + err = CRYPT_INVALID_PACKET; + goto done2; + } + } + + /* Test 2: is the public key on the curve? */ + if ((err = ltc_ecc_is_point(&key->dp, key->pubkey.x, key->pubkey.y)) != CRYPT_OK) { goto done2; } + + /* Test 3: does nG = O? (n = order, O = point at infinity, G = public key) */ + point = ltc_ecc_new_point(); + if ((err = ltc_ecc_mulmod(order, &(key->pubkey), point, a, prime, 1)) != CRYPT_OK) { goto done1; } + + err = ltc_ecc_is_point_at_infinity(point, prime, &inf); + if (err != CRYPT_OK || inf) { + err = CRYPT_ERROR; + } + else { + err = CRYPT_OK; + } + +done1: + ltc_ecc_del_point(point); +done2: + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ecc/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/ecc/sub.mk new file mode 100644 index 0000000..8567283 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ecc/sub.mk @@ -0,0 +1,20 @@ +srcs-y += ecc.c +srcs-y += ecc_find_curve.c +srcs-y += ecc_free.c +srcs-y += ecc_get_oid_str.c +srcs-y += ecc_make_key.c +srcs-y += ecc_set_curve.c +srcs-y += ecc_set_curve_internal.c +srcs-y += ecc_shared_secret.c +srcs-y += ecc_sign_hash.c +srcs-y += ecc_ssh_ecdsa_encode_name.c +srcs-y += ecc_verify_hash.c +srcs-y += ltc_ecc_is_point.c +srcs-y += ltc_ecc_is_point_at_infinity.c +srcs-y += ltc_ecc_map.c +srcs-y += ltc_ecc_mulmod.c +srcs-y += ltc_ecc_mulmod_timing.c +srcs-y += ltc_ecc_mul2add.c +srcs-y += ltc_ecc_points.c +srcs-y += ltc_ecc_projective_add_point.c +srcs-y += ltc_ecc_projective_dbl_point.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_export.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_export.c new file mode 100644 index 0000000..2b710e5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_export.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_export.c + Export an Ed25519 key to a binary packet, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Export an Ed25519 key to a binary packet + @param out [out] The destination for the key + @param outlen [in/out] The max size and resulting size of the Ed25519 key + @param type Which type of key (PK_PRIVATE, PK_PUBLIC|PK_STD or PK_PUBLIC) + @param key The key you wish to export + @return CRYPT_OK if successful +*/ +int ed25519_export( unsigned char *out, unsigned long *outlen, + int which, + const curve25519_key *key) +{ + LTC_ARGCHK(key != NULL); + + if (key->algo != LTC_OID_ED25519) return CRYPT_PK_INVALID_TYPE; + + return ec25519_export(out, outlen, which, key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import.c new file mode 100644 index 0000000..f197d59 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import.c @@ -0,0 +1,35 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_import.c + Import a Ed25519 key from a SubjectPublicKeyInfo, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Import an Ed25519 public key + @param in The packet to read + @param inlen The length of the input packet + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int ed25519_import(const unsigned char *in, unsigned long inlen, curve25519_key *key) +{ + int err; + unsigned long key_len; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + + key_len = sizeof(key->pub); + if ((err = x509_decode_subject_public_key_info(in, inlen, LTC_OID_ED25519, key->pub, &key_len, LTC_ASN1_EOL, NULL, 0uL)) == CRYPT_OK) { + key->type = PK_PUBLIC; + key->algo = LTC_OID_ED25519; + } + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_pkcs8.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_pkcs8.c new file mode 100644 index 0000000..71f12de --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_pkcs8.c @@ -0,0 +1,28 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_import_pkcs8.c + Import an Ed25519 key in PKCS#8 format, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Import an Ed25519 private key in PKCS#8 format + @param in The DER-encoded PKCS#8-formatted private key + @param inlen The length of the input data + @param passwd The password to decrypt the private key + @param passwdlen Password's length (octets) + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int ed25519_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + curve25519_key *key) +{ + return ec25519_import_pkcs8(in, inlen, pwd, pwdlen, LTC_OID_ED25519, tweetnacl_crypto_sk_to_pk, key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_raw.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_raw.c new file mode 100644 index 0000000..19955d1 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_raw.c @@ -0,0 +1,41 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_import_raw.c + Set the parameters of an Ed25519 key, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Set the parameters of an Ed25519 key + + @param in The key + @param inlen The length of the key + @param which Which type of key (PK_PRIVATE or PK_PUBLIC) + @param key [out] Destination of the key + @return CRYPT_OK if successful +*/ +int ed25519_import_raw(const unsigned char *in, unsigned long inlen, int which, curve25519_key *key) +{ + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen == 32uL); + LTC_ARGCHK(key != NULL); + + if (which == PK_PRIVATE) { + XMEMCPY(key->priv, in, sizeof(key->priv)); + tweetnacl_crypto_sk_to_pk(key->pub, key->priv); + } else if (which == PK_PUBLIC) { + XMEMCPY(key->pub, in, sizeof(key->pub)); + } else { + return CRYPT_INVALID_ARG; + } + key->algo = LTC_OID_ED25519; + key->type = which; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_x509.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_x509.c new file mode 100644 index 0000000..44978ac --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_import_x509.c @@ -0,0 +1,45 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_import_x509.c + Import an Ed25519 key from a X.509 certificate, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +static int s_ed25519_decode(const unsigned char *in, unsigned long inlen, curve25519_key *key) +{ + if (inlen != sizeof(key->pub)) return CRYPT_PK_INVALID_SIZE; + XMEMCPY(key->pub, in, sizeof(key->pub)); + return CRYPT_OK; +} + +/** + Import an Ed25519 public key from a X.509 certificate + @param in The DER encoded X.509 certificate + @param inlen The length of the certificate + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int ed25519_import_x509(const unsigned char *in, unsigned long inlen, curve25519_key *key) +{ + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + + if ((err = x509_decode_public_key_from_certificate(in, inlen, + LTC_OID_ED25519, + LTC_ASN1_EOL, NULL, NULL, + (public_key_decode_cb)s_ed25519_decode, key)) != CRYPT_OK) { + return err; + } + key->type = PK_PUBLIC; + key->algo = LTC_OID_ED25519; + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_make_key.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_make_key.c new file mode 100644 index 0000000..b5fe919 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_make_key.c @@ -0,0 +1,35 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_make_key.c + Create an Ed25519 key, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Create an Ed25519 key + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param key [out] Destination of a newly created private key pair + @return CRYPT_OK if successful +*/ +int ed25519_make_key(prng_state *prng, int wprng, curve25519_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + + if ((err = tweetnacl_crypto_sign_keypair(prng, wprng, key->pub, key->priv)) != CRYPT_OK) { + return err; + } + + key->type = PK_PRIVATE; + key->algo = LTC_OID_ED25519; + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_sign.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_sign.c new file mode 100644 index 0000000..d5bf364 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_sign.c @@ -0,0 +1,126 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_shared_secret.c + Create an Ed25519 signature, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +static int s_ed25519_sign(const unsigned char *msg, unsigned long msglen, + unsigned char *sig, unsigned long *siglen, + const unsigned char *ctx, unsigned long ctxlen, + const curve25519_key *private_key) +{ + unsigned char *s; + unsigned long long smlen; + int err; + + LTC_ARGCHK(msg != NULL); + LTC_ARGCHK(sig != NULL); + LTC_ARGCHK(siglen != NULL); + LTC_ARGCHK(private_key != NULL); + + if (private_key->algo != LTC_OID_ED25519) return CRYPT_PK_INVALID_TYPE; + if (private_key->type != PK_PRIVATE) return CRYPT_PK_INVALID_TYPE; + + if (*siglen < 64uL) { + *siglen = 64uL; + return CRYPT_BUFFER_OVERFLOW; + } + + smlen = msglen + 64; + s = XMALLOC(smlen); + if (s == NULL) return CRYPT_MEM; + + err = tweetnacl_crypto_sign(s, &smlen, + msg, msglen, + private_key->priv, private_key->pub, + ctx, ctxlen); + + XMEMCPY(sig, s, 64uL); + *siglen = 64uL; + +#ifdef LTC_CLEAN_STACK + zeromem(s, smlen); +#endif + XFREE(s); + + return err; +} + +/** + Create an Ed25519ctx signature. + @param msg The data to be signed + @param msglen [in] The size of the date to be signed + @param sig [out] The destination of the shared data + @param siglen [in/out] The max size and resulting size of the shared data. + @param ctx [in] The context is a constant null terminated string + @param private_key The private Ed25519 key in the pair + @return CRYPT_OK if successful +*/ +int ed25519ctx_sign(const unsigned char *msg, unsigned long msglen, + unsigned char *sig, unsigned long *siglen, + const unsigned char *ctx, unsigned long ctxlen, + const curve25519_key *private_key) +{ + int err; + unsigned char ctx_prefix[292]; + unsigned long ctx_prefix_size = sizeof(ctx_prefix); + + LTC_ARGCHK(ctx != NULL); + + if ((err = ec25519_crypto_ctx(ctx_prefix, &ctx_prefix_size, 0, ctx, ctxlen)) != CRYPT_OK) + return err; + + return s_ed25519_sign(msg, msglen, sig, siglen, ctx_prefix, ctx_prefix_size, private_key); +} + +/** + Create an Ed25519ph signature. + @param msg The data to be signed + @param msglen [in] The size of the date to be signed + @param sig [out] The destination of the shared data + @param siglen [in/out] The max size and resulting size of the shared data. + @param ctx [in] The context is a constant null terminated string + @param private_key The private Ed25519 key in the pair + @return CRYPT_OK if successful +*/ +int ed25519ph_sign(const unsigned char *msg, unsigned long msglen, + unsigned char *sig, unsigned long *siglen, + const unsigned char *ctx, unsigned long ctxlen, + const curve25519_key *private_key) +{ + int err; + unsigned char msg_hash[64]; + unsigned char ctx_prefix[292]; + unsigned long ctx_prefix_size = sizeof(ctx_prefix); + + if ((err = ec25519_crypto_ctx(ctx_prefix, &ctx_prefix_size, 1, ctx, ctxlen)) != CRYPT_OK) + return err; + + if ((err = tweetnacl_crypto_ph(msg_hash, msg, msglen)) != CRYPT_OK) + return err; + + return s_ed25519_sign(msg_hash, sizeof(msg_hash), sig, siglen, ctx_prefix, ctx_prefix_size, private_key); +} + +/** + Create an Ed25519 signature. + @param msg The data to be signed + @param msglen [in] The size of the date to be signed + @param sig [out] The destination of the shared data + @param siglen [in/out] The max size and resulting size of the shared data. + @param private_key The private Ed25519 key in the pair + @return CRYPT_OK if successful +*/ +int ed25519_sign(const unsigned char *msg, unsigned long msglen, + unsigned char *sig, unsigned long *siglen, + const curve25519_key *private_key) +{ + return s_ed25519_sign(msg, msglen, sig, siglen, NULL, 0, private_key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_verify.c b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_verify.c new file mode 100644 index 0000000..e7dcc30 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/ed25519_verify.c @@ -0,0 +1,134 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file ed25519_verify.c + Verify an Ed25519 signature, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +static int s_ed25519_verify(const unsigned char *msg, unsigned long msglen, + const unsigned char *sig, unsigned long siglen, + const unsigned char *ctx, unsigned long ctxlen, + int *stat, + const curve25519_key *public_key) +{ + unsigned char* m; + unsigned long long mlen; + int err; + + LTC_ARGCHK(msg != NULL); + LTC_ARGCHK(sig != NULL); + LTC_ARGCHK(stat != NULL); + LTC_ARGCHK(public_key != NULL); + + *stat = 0; + + if (siglen != 64uL) return CRYPT_INVALID_ARG; + if (public_key->algo != LTC_OID_ED25519) return CRYPT_PK_INVALID_TYPE; + + mlen = msglen + siglen; + if ((mlen < msglen) || (mlen < siglen)) return CRYPT_OVERFLOW; + + m = XMALLOC(mlen); + if (m == NULL) return CRYPT_MEM; + + XMEMCPY(m, sig, siglen); + XMEMCPY(m + siglen, msg, msglen); + + err = tweetnacl_crypto_sign_open(stat, + m, &mlen, + m, mlen, + ctx, ctxlen, + public_key->pub); + +#ifdef LTC_CLEAN_STACK + zeromem(m, msglen + siglen); +#endif + XFREE(m); + + return err; +} + +/** + Verify an Ed25519ctx signature. + @param msg [in] The data to be verified + @param msglen [in] The size of the data to be verified + @param sig [in] The signature to be verified + @param siglen [in] The size of the signature to be verified + @param ctx [in] The context + @param ctxlen [in] The size of the context + @param stat [out] The result of the signature verification, 1==valid, 0==invalid + @param public_key [in] The public Ed25519 key in the pair + @return CRYPT_OK if successful +*/ +int ed25519ctx_verify(const unsigned char *msg, unsigned long msglen, + const unsigned char *sig, unsigned long siglen, + const unsigned char *ctx, unsigned long ctxlen, + int *stat, + const curve25519_key *public_key) +{ + unsigned char ctx_prefix[292]; + unsigned long ctx_prefix_size = sizeof(ctx_prefix); + + LTC_ARGCHK(ctx != NULL); + + if (ec25519_crypto_ctx(ctx_prefix, &ctx_prefix_size, 0, ctx, ctxlen) != CRYPT_OK) + return CRYPT_INVALID_ARG; + + return s_ed25519_verify(msg, msglen, sig, siglen, ctx_prefix, ctx_prefix_size, stat, public_key); +} + +/** + Verify an Ed25519ph signature. + @param msg [in] The data to be verified + @param msglen [in] The size of the data to be verified + @param sig [in] The signature to be verified + @param siglen [in] The size of the signature to be verified + @param ctx [in] The context + @param ctxlen [in] The size of the context + @param stat [out] The result of the signature verification, 1==valid, 0==invalid + @param public_key [in] The public Ed25519 key in the pair + @return CRYPT_OK if successful +*/ +int ed25519ph_verify(const unsigned char *msg, unsigned long msglen, + const unsigned char *sig, unsigned long siglen, + const unsigned char *ctx, unsigned long ctxlen, + int *stat, + const curve25519_key *public_key) +{ + int err; + unsigned char msg_hash[64]; + unsigned char ctx_prefix[292]; + unsigned long ctx_prefix_size = sizeof(ctx_prefix); + + if ((err = ec25519_crypto_ctx(ctx_prefix, &ctx_prefix_size, 1, ctx, ctxlen)) != CRYPT_OK) + return err; + + if ((err = tweetnacl_crypto_ph(msg_hash, msg, msglen)) != CRYPT_OK) + return err; + + return s_ed25519_verify(msg_hash, sizeof(msg_hash), sig, siglen, ctx_prefix, ctx_prefix_size, stat, public_key); +} + +/** + Verify an Ed25519 signature. + @param msg [in] The data to be verified + @param msglen [in] The size of the data to be verified + @param sig [in] The signature to be verified + @param siglen [in] The size of the signature to be verified + @param stat [out] The result of the signature verification, 1==valid, 0==invalid + @param public_key [in] The public Ed25519 key in the pair + @return CRYPT_OK if successful +*/ +int ed25519_verify(const unsigned char *msg, unsigned long msglen, + const unsigned char *sig, unsigned long siglen, + int *stat, + const curve25519_key *public_key) +{ + return s_ed25519_verify(msg, msglen, sig, siglen, NULL, 0, stat, public_key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/ed25519/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/sub.mk new file mode 100644 index 0000000..cb2206d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/ed25519/sub.mk @@ -0,0 +1,7 @@ +srcs-y += ed25519_export.c +srcs-y += ed25519_import.c +srcs-y += ed25519_import_pkcs8.c +srcs-y += ed25519_import_x509.c +srcs-y += ed25519_make_key.c +srcs-y += ed25519_sign.c +srcs-y += ed25519_verify.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_i2osp.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_i2osp.c new file mode 100644 index 0000000..a62dd86 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_i2osp.c @@ -0,0 +1,39 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_1_i2osp.c + Integer to Octet I2OSP, Tom St Denis +*/ + +#ifdef LTC_PKCS_1 + +/* always stores the same # of bytes, pads with leading zero bytes + as required + */ + +/** + PKCS #1 Integer to binary + @param n The integer to store + @param modulus_len The length of the RSA modulus + @param out [out] The destination for the integer + @return CRYPT_OK if successful +*/ +int pkcs_1_i2osp(void *n, unsigned long modulus_len, unsigned char *out) +{ + unsigned long size; + + size = mp_unsigned_bin_size(n); + + if (size > modulus_len) { + return CRYPT_BUFFER_OVERFLOW; + } + + /* store it */ + zeromem(out, modulus_len); + return mp_to_unsigned_bin(n, out+(modulus_len-size)); +} + +#endif /* LTC_PKCS_1 */ + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_mgf1.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_mgf1.c new file mode 100644 index 0000000..d65f003 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_mgf1.c @@ -0,0 +1,96 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_1_mgf1.c + The Mask Generation Function (MGF1) for PKCS #1, Tom St Denis +*/ + +#ifdef LTC_PKCS_1 + +/** + Perform PKCS #1 MGF1 (internal) + @param hash_idx The index of the hash desired + @param seed The seed for MGF1 + @param seedlen The length of the seed + @param mask [out] The destination + @param masklen The length of the mask desired + @return CRYPT_OK if successful +*/ +int pkcs_1_mgf1(int hash_idx, + const unsigned char *seed, unsigned long seedlen, + unsigned char *mask, unsigned long masklen) +{ + unsigned long hLen, x; + ulong32 counter; + int err; + hash_state *md; + unsigned char *buf; + + LTC_ARGCHK(seed != NULL); + LTC_ARGCHK(mask != NULL); + + /* ensure valid hash */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + + /* get hash output size */ + hLen = hash_descriptor[hash_idx]->hashsize; + + /* allocate memory */ + md = XMALLOC(sizeof(hash_state)); + buf = XMALLOC(hLen); + if (md == NULL || buf == NULL) { + if (md != NULL) { + XFREE(md); + } + if (buf != NULL) { + XFREE(buf); + } + return CRYPT_MEM; + } + + /* start counter */ + counter = 0; + + while (masklen > 0) { + /* handle counter */ + STORE32H(counter, buf); + ++counter; + + /* get hash of seed || counter */ + if ((err = hash_descriptor[hash_idx]->init(md)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->process(md, seed, seedlen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->process(md, buf, 4)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->done(md, buf)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* store it */ + for (x = 0; x < hLen && masklen > 0; x++, masklen--) { + *mask++ = buf[x]; + } + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(buf, hLen); + zeromem(md, sizeof(hash_state)); +#endif + + XFREE(buf); + XFREE(md); + + return err; +} + +#endif /* LTC_PKCS_1 */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_oaep_decode.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_oaep_decode.c new file mode 100644 index 0000000..5bb3918 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_oaep_decode.c @@ -0,0 +1,175 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_1_oaep_decode.c + OAEP Padding for PKCS #1, Tom St Denis +*/ + +#ifdef LTC_PKCS_1 + +/** + PKCS #1 v2.00 OAEP decode + @param msg The encoded data to decode + @param msglen The length of the encoded data (octets) + @param lparam The session or system data (can be NULL) + @param lparamlen The length of the lparam + @param modulus_bitlen The bit length of the RSA modulus + @param hash_idx The index of the hash desired + @param out [out] Destination of decoding + @param outlen [in/out] The max size and resulting size of the decoding + @param res [out] Result of decoding, 1==valid, 0==invalid + @return CRYPT_OK if successful +*/ +int pkcs_1_oaep_decode(const unsigned char *msg, unsigned long msglen, + const unsigned char *lparam, unsigned long lparamlen, + unsigned long modulus_bitlen, int hash_idx, + unsigned char *out, unsigned long *outlen, + int *res) +{ + unsigned char *DB, *seed, *mask; + unsigned long hLen, x, y, modulus_len; + int err, ret; + + LTC_ARGCHK(msg != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(res != NULL); + + /* default to invalid packet */ + *res = 0; + + /* test valid hash */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + hLen = hash_descriptor[hash_idx]->hashsize; + modulus_len = (modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0); + + /* test hash/message size */ + if ((2*hLen >= (modulus_len - 2)) || (msglen != modulus_len)) { + return CRYPT_PK_INVALID_SIZE; + } + + /* allocate ram for DB/mask/salt of size modulus_len */ + DB = XMALLOC(modulus_len); + mask = XMALLOC(modulus_len); + seed = XMALLOC(hLen); + if (DB == NULL || mask == NULL || seed == NULL) { + if (DB != NULL) { + XFREE(DB); + } + if (mask != NULL) { + XFREE(mask); + } + if (seed != NULL) { + XFREE(seed); + } + return CRYPT_MEM; + } + + /* ok so it's now in the form + + 0x00 || maskedseed || maskedDB + + 1 || hLen || modulus_len - hLen - 1 + + */ + + ret = CRYPT_OK; + + /* must have leading 0x00 byte */ + if (msg[0] != 0x00) { + ret = CRYPT_INVALID_PACKET; + } + + /* now read the masked seed */ + x = 1; + XMEMCPY(seed, msg + x, hLen); + x += hLen; + + /* now read the masked DB */ + XMEMCPY(DB, msg + x, modulus_len - hLen - 1); + x += modulus_len - hLen - 1; + + /* compute MGF1 of maskedDB (hLen) */ + if ((err = pkcs_1_mgf1(hash_idx, DB, modulus_len - hLen - 1, mask, hLen)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* XOR against seed */ + for (y = 0; y < hLen; y++) { + seed[y] ^= mask[y]; + } + + /* compute MGF1 of seed (k - hlen - 1) */ + if ((err = pkcs_1_mgf1(hash_idx, seed, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* xor against DB */ + for (y = 0; y < (modulus_len - hLen - 1); y++) { + DB[y] ^= mask[y]; + } + + /* now DB == lhash || PS || 0x01 || M, PS == k - mlen - 2hlen - 2 zeroes */ + + /* compute lhash and store it in seed [reuse temps!] */ + x = modulus_len; + if (lparam != NULL) { + if ((err = hash_memory(hash_idx, lparam, lparamlen, seed, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + } else { + /* can't pass hash_memory a NULL so use DB with zero length */ + if ((err = hash_memory(hash_idx, DB, 0, seed, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + /* compare the lhash'es */ + if (XMEM_NEQ(seed, DB, hLen) != 0) { + ret = CRYPT_INVALID_PACKET; + } + + /* now zeroes before a 0x01 */ + for (x = hLen; x < (modulus_len - hLen - 1) && DB[x] == 0x00; x++) { + /* step... */ + } + + /* error if wasn't 0x01 */ + if (x == (modulus_len - hLen - 1) || DB[x] != 0x01) { + ret = CRYPT_INVALID_PACKET; + } + + /* rest is the message (and skip 0x01) */ + if ((modulus_len - hLen - 1 - ++x) > *outlen) { + ret = CRYPT_INVALID_PACKET; + } + + if (ret == CRYPT_OK) { + /* copy message */ + *outlen = modulus_len - hLen - 1 - x; + XMEMCPY(out, DB + x, modulus_len - hLen - 1 - x); + + /* valid packet */ + *res = 1; + } + err = ret; + +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(DB, modulus_len); + zeromem(seed, hLen); + zeromem(mask, modulus_len); +#endif + + XFREE(seed); + XFREE(mask); + XFREE(DB); + + return err; +} + +#endif /* LTC_PKCS_1 */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_oaep_encode.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_oaep_encode.c new file mode 100644 index 0000000..93ee1db --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_oaep_encode.c @@ -0,0 +1,161 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_1_oaep_encode.c + OAEP Padding for PKCS #1, Tom St Denis +*/ + +#ifdef LTC_PKCS_1 + +/** + PKCS #1 v2.00 OAEP encode + @param msg The data to encode + @param msglen The length of the data to encode (octets) + @param lparam A session or system parameter (can be NULL) + @param lparamlen The length of the lparam data + @param modulus_bitlen The bit length of the RSA modulus + @param prng An active PRNG state + @param prng_idx The index of the PRNG desired + @param hash_idx The index of the hash desired + @param out [out] The destination for the encoded data + @param outlen [in/out] The max size and resulting size of the encoded data + @return CRYPT_OK if successful +*/ +int pkcs_1_oaep_encode(const unsigned char *msg, unsigned long msglen, + const unsigned char *lparam, unsigned long lparamlen, + unsigned long modulus_bitlen, prng_state *prng, + int prng_idx, int hash_idx, + unsigned char *out, unsigned long *outlen) +{ + unsigned char *DB, *seed, *mask; + unsigned long hLen, x, y, modulus_len; + int err; + + LTC_ARGCHK(msg != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* test valid hash */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + + /* valid prng */ + if ((err = prng_is_valid(prng_idx)) != CRYPT_OK) { + return err; + } + + hLen = hash_descriptor[hash_idx]->hashsize; + modulus_len = (modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0); + + /* test message size */ + if ((2*hLen >= (modulus_len - 2)) || (msglen > (modulus_len - 2*hLen - 2))) { + return CRYPT_PK_INVALID_SIZE; + } + + /* allocate ram for DB/mask/salt of size modulus_len */ + DB = XMALLOC(modulus_len); + mask = XMALLOC(modulus_len); + seed = XMALLOC(hLen); + if (DB == NULL || mask == NULL || seed == NULL) { + if (DB != NULL) { + XFREE(DB); + } + if (mask != NULL) { + XFREE(mask); + } + if (seed != NULL) { + XFREE(seed); + } + return CRYPT_MEM; + } + + /* get lhash */ + /* DB == lhash || PS || 0x01 || M, PS == k - mlen - 2hlen - 2 zeroes */ + x = modulus_len; + if (lparam != NULL) { + if ((err = hash_memory(hash_idx, lparam, lparamlen, DB, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + } else { + /* can't pass hash_memory a NULL so use DB with zero length */ + if ((err = hash_memory(hash_idx, DB, 0, DB, &x)) != CRYPT_OK) { + goto LBL_ERR; + } + } + + /* append PS then 0x01 (to lhash) */ + x = hLen; + y = modulus_len - msglen - 2*hLen - 2; + XMEMSET(DB+x, 0, y); + x += y; + + /* 0x01 byte */ + DB[x++] = 0x01; + + /* message (length = msglen) */ + XMEMCPY(DB+x, msg, msglen); + x += msglen; + + /* now choose a random seed */ + if (prng_descriptor[prng_idx]->read(seed, hLen, prng) != hLen) { + err = CRYPT_ERROR_READPRNG; + goto LBL_ERR; + } + + /* compute MGF1 of seed (k - hlen - 1) */ + if ((err = pkcs_1_mgf1(hash_idx, seed, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* xor against DB */ + for (y = 0; y < (modulus_len - hLen - 1); y++) { + DB[y] ^= mask[y]; + } + + /* compute MGF1 of maskedDB (hLen) */ + if ((err = pkcs_1_mgf1(hash_idx, DB, modulus_len - hLen - 1, mask, hLen)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* XOR against seed */ + for (y = 0; y < hLen; y++) { + seed[y] ^= mask[y]; + } + + /* create string of length modulus_len */ + if (*outlen < modulus_len) { + *outlen = modulus_len; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + /* start output which is 0x00 || maskedSeed || maskedDB */ + x = 0; + out[x++] = 0x00; + XMEMCPY(out+x, seed, hLen); + x += hLen; + XMEMCPY(out+x, DB, modulus_len - hLen - 1); + x += modulus_len - hLen - 1; + + *outlen = x; + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(DB, modulus_len); + zeromem(seed, hLen); + zeromem(mask, modulus_len); +#endif + + XFREE(seed); + XFREE(mask); + XFREE(DB); + + return err; +} + +#endif /* LTC_PKCS_1 */ + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_os2ip.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_os2ip.c new file mode 100644 index 0000000..438030e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_os2ip.c @@ -0,0 +1,24 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_1_os2ip.c + Octet to Integer OS2IP, Tom St Denis +*/ +#ifdef LTC_PKCS_1 + +/** + Read a binary string into an mp_int + @param n [out] The mp_int destination + @param in The binary string to read + @param inlen The length of the binary string + @return CRYPT_OK if successful +*/ +int pkcs_1_os2ip(void *n, unsigned char *in, unsigned long inlen) +{ + return mp_read_unsigned_bin(n, in, inlen); +} + +#endif /* LTC_PKCS_1 */ + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_pss_decode.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_pss_decode.c new file mode 100644 index 0000000..9718196 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_pss_decode.c @@ -0,0 +1,167 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include +#include "tomcrypt_private.h" + +/** + @file pkcs_1_pss_decode.c + PKCS #1 PSS Signature Padding, Tom St Denis +*/ + +#ifdef LTC_PKCS_1 + +/** + PKCS #1 v2.00 PSS decode + @param msghash The hash to verify + @param msghashlen The length of the hash (octets) + @param sig The signature data (encoded data) + @param siglen The length of the signature data (octets) + @param saltlen The length of the salt used (octets) + @param hash_idx The index of the hash desired + @param modulus_bitlen The bit length of the RSA modulus + @param res [out] The result of the comparison, 1==valid, 0==invalid + @return CRYPT_OK if successful (even if the comparison failed) +*/ +int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen, + const unsigned char *sig, unsigned long siglen, + unsigned long saltlen, int hash_idx, + unsigned long modulus_bitlen, int *res) +{ + unsigned char *DB, *mask, *salt, *hash; + unsigned long x, y, hLen, modulus_len; + int err; + hash_state md; + + LTC_ARGCHK(msghash != NULL); + LTC_ARGCHK(res != NULL); + + /* default to invalid */ + *res = 0; + + /* ensure hash is valid */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + + hLen = hash_descriptor[hash_idx]->hashsize; + modulus_bitlen--; + modulus_len = (modulus_bitlen>>3) + (modulus_bitlen & 7 ? 1 : 0); + + /* check sizes */ + if ((saltlen > modulus_len) || + (modulus_len < hLen + saltlen + 2)) { + return CRYPT_PK_INVALID_SIZE; + } + + /* allocate ram for DB/mask/salt/hash of size modulus_len */ + DB = XMALLOC(modulus_len); + mask = XMALLOC(modulus_len); + salt = XMALLOC(modulus_len); + hash = XMALLOC(modulus_len); + if (DB == NULL || mask == NULL || salt == NULL || hash == NULL) { + if (DB != NULL) { + XFREE(DB); + } + if (mask != NULL) { + XFREE(mask); + } + if (salt != NULL) { + XFREE(salt); + } + if (hash != NULL) { + XFREE(hash); + } + return CRYPT_MEM; + } + + /* ensure the 0xBC byte */ + if (sig[siglen-1] != 0xBC) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + /* copy out the DB */ + x = 0; + XMEMCPY(DB, sig + x, modulus_len - hLen - 1); + x += modulus_len - hLen - 1; + + /* copy out the hash */ + XMEMCPY(hash, sig + x, hLen); + /* x += hLen; */ + + /* check the MSB */ + if ((sig[0] & ~(0xFF >> ((modulus_len<<3) - (modulus_bitlen)))) != 0) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + /* generate mask of length modulus_len - hLen - 1 from hash */ + if ((err = pkcs_1_mgf1(hash_idx, hash, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* xor against DB */ + for (y = 0; y < (modulus_len - hLen - 1); y++) { + DB[y] ^= mask[y]; + } + + /* now clear the first byte [make sure smaller than modulus] */ + DB[0] &= 0xFF >> ((modulus_len<<3) - (modulus_bitlen)); + + /* DB = PS || 0x01 || salt, PS == modulus_len - saltlen - hLen - 2 zero bytes */ + + /* check for zeroes and 0x01 */ + for (x = 0; x < modulus_len - saltlen - hLen - 2; x++) { + if (DB[x] != 0x00) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + } + + /* check for the 0x01 */ + if (DB[x++] != 0x01) { + err = CRYPT_INVALID_PACKET; + goto LBL_ERR; + } + + /* M = (eight) 0x00 || msghash || salt, mask = H(M) */ + if ((err = hash_descriptor[hash_idx]->init(&md)) != CRYPT_OK) { + goto LBL_ERR; + } + zeromem(mask, 8); + if ((err = hash_descriptor[hash_idx]->process(&md, mask, 8)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->process(&md, msghash, msghashlen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->process(&md, DB+x, saltlen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->done(&md, mask)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* mask == hash means valid signature */ + if (FTMN_CALLEE_DONE_MEMCMP(XMEM_NEQ, mask, hash, hLen) == 0) { + *res = 1; + } + + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(DB, modulus_len); + zeromem(mask, modulus_len); + zeromem(salt, modulus_len); + zeromem(hash, modulus_len); +#endif + + XFREE(hash); + XFREE(salt); + XFREE(mask); + XFREE(DB); + + return err; +} + +#endif /* LTC_PKCS_1 */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_pss_encode.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_pss_encode.c new file mode 100644 index 0000000..00e3bce --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_pss_encode.c @@ -0,0 +1,164 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file pkcs_1_pss_encode.c + PKCS #1 PSS Signature Padding, Tom St Denis +*/ + +#ifdef LTC_PKCS_1 + +/** + PKCS #1 v2.00 Signature Encoding + @param msghash The hash to encode + @param msghashlen The length of the hash (octets) + @param saltlen The length of the salt desired (octets) + @param prng An active PRNG context + @param prng_idx The index of the PRNG desired + @param hash_idx The index of the hash desired + @param modulus_bitlen The bit length of the RSA modulus + @param out [out] The destination of the encoding + @param outlen [in/out] The max size and resulting size of the encoded data + @return CRYPT_OK if successful +*/ +int pkcs_1_pss_encode(const unsigned char *msghash, unsigned long msghashlen, + unsigned long saltlen, prng_state *prng, + int prng_idx, int hash_idx, + unsigned long modulus_bitlen, + unsigned char *out, unsigned long *outlen) +{ + unsigned char *DB, *mask, *salt, *hash; + unsigned long x, y, hLen, modulus_len; + int err; + hash_state md; + + LTC_ARGCHK(msghash != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + /* ensure hash and PRNG are valid */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + if ((err = prng_is_valid(prng_idx)) != CRYPT_OK) { + return err; + } + + hLen = hash_descriptor[hash_idx]->hashsize; + modulus_bitlen--; + modulus_len = (modulus_bitlen>>3) + (modulus_bitlen & 7 ? 1 : 0); + + /* check sizes */ + if ((saltlen > modulus_len) || (modulus_len < hLen + saltlen + 2)) { + return CRYPT_PK_INVALID_SIZE; + } + + /* allocate ram for DB/mask/salt/hash of size modulus_len */ + DB = XMALLOC(modulus_len); + mask = XMALLOC(modulus_len); + salt = XMALLOC(modulus_len); + hash = XMALLOC(modulus_len); + if (DB == NULL || mask == NULL || salt == NULL || hash == NULL) { + if (DB != NULL) { + XFREE(DB); + } + if (mask != NULL) { + XFREE(mask); + } + if (salt != NULL) { + XFREE(salt); + } + if (hash != NULL) { + XFREE(hash); + } + return CRYPT_MEM; + } + + + /* generate random salt */ + if (saltlen > 0) { + if (prng_descriptor[prng_idx]->read(salt, saltlen, prng) != saltlen) { + err = CRYPT_ERROR_READPRNG; + goto LBL_ERR; + } + } + + /* M = (eight) 0x00 || msghash || salt, hash = H(M) */ + if ((err = hash_descriptor[hash_idx]->init(&md)) != CRYPT_OK) { + goto LBL_ERR; + } + zeromem(DB, 8); + if ((err = hash_descriptor[hash_idx]->process(&md, DB, 8)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->process(&md, msghash, msghashlen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->process(&md, salt, saltlen)) != CRYPT_OK) { + goto LBL_ERR; + } + if ((err = hash_descriptor[hash_idx]->done(&md, hash)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* generate DB = PS || 0x01 || salt, PS == modulus_len - saltlen - hLen - 2 zero bytes */ + x = 0; + XMEMSET(DB + x, 0, modulus_len - saltlen - hLen - 2); + x += modulus_len - saltlen - hLen - 2; + DB[x++] = 0x01; + XMEMCPY(DB + x, salt, saltlen); + /* x += saltlen; */ + + /* generate mask of length modulus_len - hLen - 1 from hash */ + if ((err = pkcs_1_mgf1(hash_idx, hash, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) { + goto LBL_ERR; + } + + /* xor against DB */ + for (y = 0; y < (modulus_len - hLen - 1); y++) { + DB[y] ^= mask[y]; + } + + /* output is DB || hash || 0xBC */ + if (*outlen < modulus_len) { + *outlen = modulus_len; + err = CRYPT_BUFFER_OVERFLOW; + goto LBL_ERR; + } + + /* DB len = modulus_len - hLen - 1 */ + y = 0; + XMEMCPY(out + y, DB, modulus_len - hLen - 1); + y += modulus_len - hLen - 1; + + /* hash */ + XMEMCPY(out + y, hash, hLen); + y += hLen; + + /* 0xBC */ + out[y] = 0xBC; + + /* now clear the 8*modulus_len - modulus_bitlen most significant bits */ + out[0] &= 0xFF >> ((modulus_len<<3) - modulus_bitlen); + + /* store output size */ + *outlen = modulus_len; + err = CRYPT_OK; +LBL_ERR: +#ifdef LTC_CLEAN_STACK + zeromem(DB, modulus_len); + zeromem(mask, modulus_len); + zeromem(salt, modulus_len); + zeromem(hash, modulus_len); +#endif + + XFREE(hash); + XFREE(salt); + XFREE(mask); + XFREE(DB); + + return err; +} + +#endif /* LTC_PKCS_1 */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_v1_5_decode.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_v1_5_decode.c new file mode 100644 index 0000000..e988060 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_v1_5_decode.c @@ -0,0 +1,102 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** @file pkcs_1_v1_5_decode.c + * + * PKCS #1 v1.5 Padding. (Andreas Lange) + */ + +#ifdef LTC_PKCS_1 + +/** @brief PKCS #1 v1.5 decode. + * + * @param msg The encoded data to decode + * @param msglen The length of the encoded data (octets) + * @param block_type Block type to use in padding (\sa ltc_pkcs_1_v1_5_blocks) + * @param modulus_bitlen The bit length of the RSA modulus + * @param out [out] Destination of decoding + * @param outlen [in/out] The max size and resulting size of the decoding + * @param is_valid [out] Boolean whether the padding was valid + * + * @return CRYPT_OK if successful + */ +int pkcs_1_v1_5_decode(const unsigned char *msg, + unsigned long msglen, + int block_type, + unsigned long modulus_bitlen, + unsigned char *out, + unsigned long *outlen, + int *is_valid) +{ + unsigned long modulus_len, ps_len, i; + int result; + + /* default to invalid packet */ + *is_valid = 0; + + modulus_len = (modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0); + + /* test message size */ + + if ((msglen > modulus_len) || (modulus_len < 11)) { + return CRYPT_PK_INVALID_SIZE; + } + + result = CRYPT_OK; + + /* separate encoded message */ + + if ((msg[0] != 0x00) || (msg[1] != (unsigned char)block_type)) { + result = CRYPT_INVALID_PACKET; + } + + if (block_type == LTC_PKCS_1_EME) { + for (i = 2; i < modulus_len; i++) { + /* separator */ + if (msg[i] == 0x00) { break; } + } + ps_len = i++ - 2; + + if (i > modulus_len) { + /* There was no octet with hexadecimal value 0x00 to separate ps from m. + */ + result = CRYPT_INVALID_PACKET; + } + } else { + for (i = 2; i < modulus_len - 1; i++) { + if (msg[i] != 0xFF) { break; } + } + + /* separator check */ + if (msg[i] != 0) { + /* There was no octet with hexadecimal value 0x00 to separate ps from m. */ + result = CRYPT_INVALID_PACKET; + } + + ps_len = i - 2; + } + + if (ps_len < 8) + { + /* The length of ps is less than 8 octets. + */ + result = CRYPT_INVALID_PACKET; + } + + if (*outlen < (msglen - (2 + ps_len + 1))) { + result = CRYPT_INVALID_PACKET; + } + + if (result == CRYPT_OK) { + *outlen = (msglen - (2 + ps_len + 1)); + XMEMCPY(out, &msg[2 + ps_len + 1], *outlen); + + /* valid packet */ + *is_valid = 1; + } + + return result; +} /* pkcs_1_v1_5_decode */ + +#endif /* #ifdef LTC_PKCS_1 */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_v1_5_encode.c b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_v1_5_encode.c new file mode 100644 index 0000000..5172425 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/pkcs_1_v1_5_encode.c @@ -0,0 +1,99 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/*! \file pkcs_1_v1_5_encode.c + * + * PKCS #1 v1.5 Padding (Andreas Lange) + */ + +#ifdef LTC_PKCS_1 + +/*! \brief PKCS #1 v1.5 encode. + * + * \param msg The data to encode + * \param msglen The length of the data to encode (octets) + * \param block_type Block type to use in padding (\sa ltc_pkcs_1_v1_5_blocks) + * \param modulus_bitlen The bit length of the RSA modulus + * \param prng An active PRNG state (only for LTC_PKCS_1_EME) + * \param prng_idx The index of the PRNG desired (only for LTC_PKCS_1_EME) + * \param out [out] The destination for the encoded data + * \param outlen [in/out] The max size and resulting size of the encoded data + * + * \return CRYPT_OK if successful + */ +int pkcs_1_v1_5_encode(const unsigned char *msg, + unsigned long msglen, + int block_type, + unsigned long modulus_bitlen, + prng_state *prng, + int prng_idx, + unsigned char *out, + unsigned long *outlen) +{ + unsigned long modulus_len, ps_len, i; + unsigned char *ps; + int result; + + /* valid block_type? */ + if ((block_type != LTC_PKCS_1_EMSA) && + (block_type != LTC_PKCS_1_EME)) { + return CRYPT_PK_INVALID_PADDING; + } + + if (block_type == LTC_PKCS_1_EME) { /* encryption padding, we need a valid PRNG */ + if ((result = prng_is_valid(prng_idx)) != CRYPT_OK) { + return result; + } + } + + modulus_len = (modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0); + + /* test message size */ + if ((msglen + 11) > modulus_len) { + return CRYPT_PK_INVALID_SIZE; + } + + if (*outlen < modulus_len) { + *outlen = modulus_len; + result = CRYPT_BUFFER_OVERFLOW; + goto bail; + } + + /* generate an octets string PS */ + ps = &out[2]; + ps_len = modulus_len - msglen - 3; + + if (block_type == LTC_PKCS_1_EME) { + /* now choose a random ps */ + if (prng_descriptor[prng_idx]->read(ps, ps_len, prng) != ps_len) { + result = CRYPT_ERROR_READPRNG; + goto bail; + } + + /* transform zero bytes (if any) to non-zero random bytes */ + for (i = 0; i < ps_len; i++) { + while (ps[i] == 0) { + if (prng_descriptor[prng_idx]->read(&ps[i], 1, prng) != 1) { + result = CRYPT_ERROR_READPRNG; + goto bail; + } + } + } + } else { + XMEMSET(ps, 0xFF, ps_len); + } + + /* create string of length modulus_len */ + out[0] = 0x00; + out[1] = (unsigned char)block_type; /* block_type 1 or 2 */ + out[2 + ps_len] = 0x00; + XMEMCPY(&out[2 + ps_len + 1], msg, msglen); + *outlen = modulus_len; + + result = CRYPT_OK; +bail: + return result; +} /* pkcs_1_v1_5_encode */ + +#endif /* #ifdef LTC_PKCS_1 */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/sub.mk new file mode 100644 index 0000000..43f96b9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/pkcs1/sub.mk @@ -0,0 +1,9 @@ +srcs-y += pkcs_1_i2osp.c +srcs-y += pkcs_1_mgf1.c +srcs-y += pkcs_1_oaep_decode.c +srcs-y += pkcs_1_oaep_encode.c +srcs-y += pkcs_1_os2ip.c +srcs-y += pkcs_1_pss_decode.c +srcs-y += pkcs_1_pss_encode.c +srcs-y += pkcs_1_v1_5_decode.c +srcs-y += pkcs_1_v1_5_encode.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_decrypt_key.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_decrypt_key.c new file mode 100644 index 0000000..34a11f9 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_decrypt_key.c @@ -0,0 +1,93 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_decrypt_key.c + RSA PKCS #1 Decryption, Tom St Denis and Andreas Lange +*/ + +#ifdef LTC_MRSA + +/** + PKCS #1 decrypt then v1.5 or OAEP depad + @param in The ciphertext + @param inlen The length of the ciphertext (octets) + @param out [out] The plaintext + @param outlen [in/out] The max size and resulting size of the plaintext (octets) + @param lparam The system "lparam" value + @param lparamlen The length of the lparam value (octets) + @param hash_idx The index of the hash desired + @param padding Type of padding (LTC_PKCS_1_OAEP or LTC_PKCS_1_V1_5) + @param stat [out] Result of the decryption, 1==valid, 0==invalid + @param key The corresponding private RSA key + @return CRYPT_OK if succcessul (even if invalid) +*/ +int rsa_decrypt_key_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const unsigned char *lparam, unsigned long lparamlen, + int hash_idx, int padding, + int *stat, const rsa_key *key) +{ + unsigned long modulus_bitlen, modulus_bytelen, x; + int err; + unsigned char *tmp; + + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(stat != NULL); + + /* default to invalid */ + *stat = 0; + + /* valid padding? */ + + if ((padding != LTC_PKCS_1_V1_5) && + (padding != LTC_PKCS_1_OAEP)) { + return CRYPT_PK_INVALID_PADDING; + } + + if (padding == LTC_PKCS_1_OAEP) { + /* valid hash ? */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + } + + /* get modulus len in bits */ + modulus_bitlen = mp_count_bits( (key->N)); + + /* outlen must be at least the size of the modulus */ + modulus_bytelen = mp_unsigned_bin_size( (key->N)); + if (modulus_bytelen != inlen) { + return CRYPT_INVALID_PACKET; + } + + /* allocate ram */ + tmp = XMALLOC(inlen); + if (tmp == NULL) { + return CRYPT_MEM; + } + + /* rsa decode the packet */ + x = inlen; + if ((err = ltc_mp.rsa_me(in, inlen, tmp, &x, PK_PRIVATE, key)) != CRYPT_OK) { + XFREE(tmp); + return err; + } + + if (padding == LTC_PKCS_1_OAEP) { + /* now OAEP decode the packet */ + err = pkcs_1_oaep_decode(tmp, x, lparam, lparamlen, modulus_bitlen, hash_idx, + out, outlen, stat); + } else { + /* now PKCS #1 v1.5 depad the packet */ + err = pkcs_1_v1_5_decode(tmp, x, LTC_PKCS_1_EME, modulus_bitlen, out, outlen, stat); + } + + XFREE(tmp); + return err; +} + +#endif /* LTC_MRSA */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_encrypt_key.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_encrypt_key.c new file mode 100644 index 0000000..6694260 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_encrypt_key.c @@ -0,0 +1,92 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_encrypt_key.c + RSA PKCS #1 encryption, Tom St Denis and Andreas Lange +*/ + +#ifdef LTC_MRSA + +/** + (PKCS #1 v2.0) OAEP pad then encrypt + @param in The plaintext + @param inlen The length of the plaintext (octets) + @param out [out] The ciphertext + @param outlen [in/out] The max size and resulting size of the ciphertext + @param lparam The system "lparam" for the encryption + @param lparamlen The length of lparam (octets) + @param prng An active PRNG + @param prng_idx The index of the desired prng + @param hash_idx The index of the desired hash + @param padding Type of padding (LTC_PKCS_1_OAEP or LTC_PKCS_1_V1_5) + @param key The RSA key to encrypt to + @return CRYPT_OK if successful +*/ +int rsa_encrypt_key_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + const unsigned char *lparam, unsigned long lparamlen, + prng_state *prng, int prng_idx, + int hash_idx, int padding, + const rsa_key *key) +{ + unsigned long modulus_bitlen, modulus_bytelen, x; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* valid padding? */ + if ((padding != LTC_PKCS_1_V1_5) && + (padding != LTC_PKCS_1_OAEP)) { + return CRYPT_PK_INVALID_PADDING; + } + + /* valid prng? */ + if ((err = prng_is_valid(prng_idx)) != CRYPT_OK) { + return err; + } + + if (padding == LTC_PKCS_1_OAEP) { + /* valid hash? */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + } + + /* get modulus len in bits */ + modulus_bitlen = mp_count_bits( (key->N)); + + /* outlen must be at least the size of the modulus */ + modulus_bytelen = mp_unsigned_bin_size( (key->N)); + if (modulus_bytelen > *outlen) { + *outlen = modulus_bytelen; + return CRYPT_BUFFER_OVERFLOW; + } + + if (padding == LTC_PKCS_1_OAEP) { + /* OAEP pad the key */ + x = *outlen; + if ((err = pkcs_1_oaep_encode(in, inlen, lparam, + lparamlen, modulus_bitlen, prng, prng_idx, hash_idx, + out, &x)) != CRYPT_OK) { + return err; + } + } else { + /* PKCS #1 v1.5 pad the key */ + x = *outlen; + if ((err = pkcs_1_v1_5_encode(in, inlen, LTC_PKCS_1_EME, + modulus_bitlen, prng, prng_idx, + out, &x)) != CRYPT_OK) { + return err; + } + } + + /* rsa exptmod the OAEP or PKCS #1 v1.5 pad */ + return ltc_mp.rsa_me(out, x, out, outlen, PK_PUBLIC, key); +} + +#endif /* LTC_MRSA */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_export.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_export.c new file mode 100644 index 0000000..225224d --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_export.c @@ -0,0 +1,91 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_export.c + Export RSA PKCS keys, Tom St Denis +*/ + +#ifdef LTC_MRSA + +/** + This will export either an RSAPublicKey or RSAPrivateKey [defined in PKCS #1 v2.1] + @param out [out] Destination of the packet + @param outlen [in/out] The max size and resulting size of the packet + @param type The type of exported key (PK_PRIVATE or PK_PUBLIC) + @param key The RSA key to export + @return CRYPT_OK if successful +*/ +int rsa_export(unsigned char *out, unsigned long *outlen, int type, const rsa_key *key) +{ + unsigned long zero=0; + int err, std; + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + std = type & PK_STD; + type &= ~PK_STD; + + if (type == PK_PRIVATE && key->type != PK_PRIVATE) { + return CRYPT_PK_TYPE_MISMATCH; + } + + if (type == PK_PRIVATE) { + /* private key */ + /* output is + Version, n, e, d, p, q, d mod (p-1), d mod (q - 1), 1/q mod p + */ + return der_encode_sequence_multi(out, outlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &zero, + LTC_ASN1_INTEGER, 1UL, key->N, + LTC_ASN1_INTEGER, 1UL, key->e, + LTC_ASN1_INTEGER, 1UL, key->d, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->dP, + LTC_ASN1_INTEGER, 1UL, key->dQ, + LTC_ASN1_INTEGER, 1UL, key->qP, + LTC_ASN1_EOL, 0UL, NULL); + } + + if (type == PK_PUBLIC) { + /* public key */ + unsigned long tmplen, *ptmplen; + unsigned char* tmp = NULL; + + if (std) { + tmplen = (unsigned long)(mp_count_bits(key->N) / 8) * 2 + 8; + tmp = XMALLOC(tmplen); + ptmplen = &tmplen; + if (tmp == NULL) { + return CRYPT_MEM; + } + } + else { + tmp = out; + ptmplen = outlen; + } + + err = der_encode_sequence_multi(tmp, ptmplen, + LTC_ASN1_INTEGER, 1UL, key->N, + LTC_ASN1_INTEGER, 1UL, key->e, + LTC_ASN1_EOL, 0UL, NULL); + + if ((err != CRYPT_OK) || !std) { + goto finish; + } + + err = x509_encode_subject_public_key_info(out, outlen, + LTC_OID_RSA, tmp, tmplen, LTC_ASN1_NULL, NULL, 0); + +finish: + if (tmp != out) XFREE(tmp); + return err; + } + + return CRYPT_INVALID_ARG; +} + +#endif /* LTC_MRSA */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_exptmod.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_exptmod.c new file mode 100644 index 0000000..73daee5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_exptmod.c @@ -0,0 +1,172 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_exptmod.c + RSA PKCS exptmod, Tom St Denis + Added RSA blinding --nmav +*/ + +#ifdef LTC_MRSA + +/** + Compute an RSA modular exponentiation + @param in The input data to send into RSA + @param inlen The length of the input (octets) + @param out [out] The destination + @param outlen [in/out] The max size and resulting size of the output + @param which Which exponent to use, e.g. PK_PRIVATE or PK_PUBLIC + @param key The RSA key to use + @return CRYPT_OK if successful +*/ +int rsa_exptmod(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, int which, + const rsa_key *key) +{ + void *tmp, *tmpa, *tmpb; + #ifdef LTC_RSA_BLINDING + void *rnd, *rndi /* inverse of rnd */; + #endif + unsigned long x; + int err, has_crt_parameters; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* is the key of the right type for the operation? */ + if (which == PK_PRIVATE && (key->type != PK_PRIVATE)) { + return CRYPT_PK_NOT_PRIVATE; + } + + /* must be a private or public operation */ + if (which != PK_PRIVATE && which != PK_PUBLIC) { + return CRYPT_PK_INVALID_TYPE; + } + + /* init and copy into tmp */ + if ((err = mp_init_multi(&tmp, &tmpa, &tmpb, +#ifdef LTC_RSA_BLINDING + &rnd, &rndi, +#endif /* LTC_RSA_BLINDING */ + NULL)) != CRYPT_OK) + { return err; } + if ((err = mp_read_unsigned_bin(tmp, (unsigned char *)in, (int)inlen)) != CRYPT_OK) + { goto error; } + + + /* sanity check on the input */ + if (mp_cmp(key->N, tmp) == LTC_MP_LT) { + err = CRYPT_PK_INVALID_SIZE; + goto error; + } + + /* are we using the private exponent and is the key optimized? */ + if (which == PK_PRIVATE) { + #ifdef LTC_RSA_BLINDING + /* do blinding */ + err = mp_rand(rnd, mp_get_digit_count(key->N)); + if (err != CRYPT_OK) { + goto error; + } + + /* rndi = 1/rnd mod N */ + err = mp_invmod(rnd, key->N, rndi); + if (err != CRYPT_OK) { + goto error; + } + + /* rnd = rnd^e */ + err = mp_exptmod( rnd, key->e, key->N, rnd); + if (err != CRYPT_OK) { + goto error; + } + + /* tmp = tmp*rnd mod N */ + err = mp_mulmod( tmp, rnd, key->N, tmp); + if (err != CRYPT_OK) { + goto error; + } + #endif /* LTC_RSA_BLINDING */ + + has_crt_parameters = (key->p != NULL) && (mp_get_digit_count(key->p) != 0) && + (key->q != NULL) && (mp_get_digit_count(key->q) != 0) && + (key->dP != NULL) && (mp_get_digit_count(key->dP) != 0) && + (key->dQ != NULL) && (mp_get_digit_count(key->dQ) != 0) && + (key->qP != NULL) && (mp_get_digit_count(key->qP) != 0); + + if (!has_crt_parameters) { + /* + * In case CRT optimization parameters are not provided, + * the private key is directly used to exptmod it + */ + if ((err = mp_exptmod(tmp, key->d, key->N, tmp)) != CRYPT_OK) { goto error; } + } else { + /* tmpa = tmp^dP mod p */ + if ((err = mp_exptmod(tmp, key->dP, key->p, tmpa)) != CRYPT_OK) { goto error; } + + /* tmpb = tmp^dQ mod q */ + if ((err = mp_exptmod(tmp, key->dQ, key->q, tmpb)) != CRYPT_OK) { goto error; } + + /* tmp = (tmpa - tmpb) * qInv (mod p) */ + if ((err = mp_sub(tmpa, tmpb, tmp)) != CRYPT_OK) { goto error; } + if ((err = mp_mulmod(tmp, key->qP, key->p, tmp)) != CRYPT_OK) { goto error; } + + /* tmp = tmpb + q * tmp */ + if ((err = mp_mul(tmp, key->q, tmp)) != CRYPT_OK) { goto error; } + if ((err = mp_add(tmp, tmpb, tmp)) != CRYPT_OK) { goto error; } + } + + #ifdef LTC_RSA_BLINDING + /* unblind */ + err = mp_mulmod( tmp, rndi, key->N, tmp); + if (err != CRYPT_OK) { + goto error; + } + #endif + + #ifdef LTC_RSA_CRT_HARDENING + if (has_crt_parameters) { + if ((err = mp_exptmod(tmp, key->e, key->N, tmpa)) != CRYPT_OK) { goto error; } + if ((err = mp_read_unsigned_bin(tmpb, (unsigned char *)in, (int)inlen)) != CRYPT_OK) { goto error; } + if (mp_cmp(tmpa, tmpb) != LTC_MP_EQ) { err = CRYPT_ERROR; goto error; } + } + #endif + } else { + /* exptmod it */ + if ((err = mp_exptmod(tmp, key->e, key->N, tmp)) != CRYPT_OK) { goto error; } + } + + /* read it back */ + x = (unsigned long)mp_unsigned_bin_size(key->N); + if (x > *outlen) { + *outlen = x; + err = CRYPT_BUFFER_OVERFLOW; + goto error; + } + + /* this should never happen ... */ + if (mp_unsigned_bin_size(tmp) > mp_unsigned_bin_size(key->N)) { + err = CRYPT_ERROR; + goto error; + } + *outlen = x; + + /* convert it */ + zeromem(out, x); + if ((err = mp_to_unsigned_bin(tmp, out+(x-mp_unsigned_bin_size(tmp)))) != CRYPT_OK) { goto error; } + + /* clean up and return */ + err = CRYPT_OK; +error: + mp_clear_multi( +#ifdef LTC_RSA_BLINDING + rndi, rnd, +#endif /* LTC_RSA_BLINDING */ + tmpb, tmpa, tmp, NULL); + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_get_size.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_get_size.c new file mode 100644 index 0000000..72d3f7a --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_get_size.c @@ -0,0 +1,30 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_get_size.c + Retrieve the size of an RSA key, Steffen Jaeckel. +*/ + +#ifdef LTC_MRSA + +/** + Retrieve the size in bytes of an RSA key. + @param key The RSA key + @return The size in bytes of the RSA key or INT_MAX on error. +*/ +int rsa_get_size(const rsa_key *key) +{ + int ret = INT_MAX; + LTC_ARGCHK(key != NULL); + + if (key) + { + ret = mp_unsigned_bin_size(key->N); + } /* if */ + + return ret; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import.c new file mode 100644 index 0000000..1240a77 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import.c @@ -0,0 +1,143 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_import.c + Import a PKCS RSA key, Tom St Denis +*/ + +#ifdef LTC_MRSA + + +/** + Import an RSAPublicKey or RSAPrivateKey as defined in PKCS #1 v2.1 [two-prime only] + + The `key` passed into this function has to be already initialized and will + NOT be free'd on error! + + @param in The packet to import from + @param inlen It's length (octets) + @param key [out] Destination for newly imported key + @return CRYPT_OK if successful +*/ +int rsa_import_pkcs1(const unsigned char *in, unsigned long inlen, rsa_key *key) +{ + int err; + unsigned long version = -1; + + err = der_decode_sequence_multi(in, inlen, LTC_ASN1_SHORT_INTEGER, 1UL, &version, + LTC_ASN1_EOL, 0UL, NULL); + + if (err == CRYPT_OVERFLOW) { + /* the version would fit into an LTC_ASN1_SHORT_INTEGER + * so we try to decode as a public key + */ + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_INTEGER, 1UL, key->N, + LTC_ASN1_INTEGER, 1UL, key->e, + LTC_ASN1_EOL, 0UL, NULL)) == CRYPT_OK) { + key->type = PK_PUBLIC; + } + goto LBL_OUT; + } else if (err != CRYPT_INPUT_TOO_LONG) { + /* couldn't decode the version, so error out */ + goto LBL_OUT; + } + + if (version == 0) { + /* it's a private key */ + if ((err = der_decode_sequence_multi(in, inlen, + LTC_ASN1_SHORT_INTEGER, 1UL, &version, + LTC_ASN1_INTEGER, 1UL, key->N, + LTC_ASN1_INTEGER, 1UL, key->e, + LTC_ASN1_INTEGER, 1UL, key->d, + LTC_ASN1_INTEGER, 1UL, key->p, + LTC_ASN1_INTEGER, 1UL, key->q, + LTC_ASN1_INTEGER, 1UL, key->dP, + LTC_ASN1_INTEGER, 1UL, key->dQ, + LTC_ASN1_INTEGER, 1UL, key->qP, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto LBL_OUT; + } + key->type = PK_PRIVATE; + } else if (version == 1) { + /* we don't support multi-prime RSA */ + err = CRYPT_PK_INVALID_TYPE; + goto LBL_OUT; + } + err = CRYPT_OK; +LBL_OUT: + return err; +} + +/** + Import multiple formats of RSA public and private keys. + + RSAPublicKey or RSAPrivateKey as defined in PKCS #1 v2.1 [two-prime only] + SubjectPublicKeyInfo formatted public keys + + @param in The packet to import from + @param inlen It's length (octets) + @param key [out] Destination for newly imported key + @return CRYPT_OK if successful, upon error allocated memory is freed +*/ +int rsa_import(const unsigned char *in, unsigned long inlen, rsa_key *key) +{ + int err; + unsigned char *tmpbuf=NULL; + unsigned long tmpbuf_len, len; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* init key */ + if ((err = rsa_init(key)) != CRYPT_OK) { + return err; + } + + /* see if the OpenSSL DER format RSA public key will work */ + tmpbuf_len = inlen; + tmpbuf = XCALLOC(1, tmpbuf_len); + if (tmpbuf == NULL) { + err = CRYPT_MEM; + goto LBL_ERR; + } + + len = 0; + err = x509_decode_subject_public_key_info(in, inlen, + LTC_OID_RSA, tmpbuf, &tmpbuf_len, + LTC_ASN1_NULL, NULL, &len); + + if (err == CRYPT_OK) { /* SubjectPublicKeyInfo format */ + + /* now it should be SEQUENCE { INTEGER, INTEGER } */ + if ((err = der_decode_sequence_multi(tmpbuf, tmpbuf_len, + LTC_ASN1_INTEGER, 1UL, key->N, + LTC_ASN1_INTEGER, 1UL, key->e, + LTC_ASN1_EOL, 0UL, NULL)) != CRYPT_OK) { + goto LBL_ERR; + } + key->type = PK_PUBLIC; + err = CRYPT_OK; + goto LBL_FREE; + } + + /* not SSL public key, try to match against PKCS #1 standards */ + if ((err = rsa_import_pkcs1(in, inlen, key)) == CRYPT_OK) { + goto LBL_FREE; + } + +LBL_ERR: + rsa_free(key); + +LBL_FREE: + if (tmpbuf != NULL) { + XFREE(tmpbuf); + } + return err; +} + +#endif /* LTC_MRSA */ + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import_pkcs8.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import_pkcs8.c new file mode 100644 index 0000000..9e02585 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import_pkcs8.c @@ -0,0 +1,116 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_import_pkcs8.c + Import a PKCS RSA key +*/ + +#ifdef LTC_MRSA + +/* Public-Key Cryptography Standards (PKCS) #8: + * Private-Key Information Syntax Specification Version 1.2 + * https://tools.ietf.org/html/rfc5208 + * + * PrivateKeyInfo ::= SEQUENCE { + * version Version, + * privateKeyAlgorithm PrivateKeyAlgorithmIdentifier, + * privateKey PrivateKey, + * attributes [0] IMPLICIT Attributes OPTIONAL } + * where: + * - Version ::= INTEGER + * - PrivateKeyAlgorithmIdentifier ::= AlgorithmIdentifier + * - PrivateKey ::= OCTET STRING + * - Attributes ::= SET OF Attribute + * + * EncryptedPrivateKeyInfo ::= SEQUENCE { + * encryptionAlgorithm EncryptionAlgorithmIdentifier, + * encryptedData EncryptedData } + * where: + * - EncryptionAlgorithmIdentifier ::= AlgorithmIdentifier + * - EncryptedData ::= OCTET STRING + */ + +/** + Import an RSAPrivateKey in PKCS#8 format + @param in The packet to import from + @param inlen It's length (octets) + @param passwd The password for decrypting privkey + @param passwdlen Password's length (octets) + @param key [out] Destination for newly imported key + @return CRYPT_OK if successful, upon error allocated memory is freed +*/ +int rsa_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *passwd, unsigned long passwdlen, + rsa_key *key) +{ + int err; + unsigned char *buf1 = NULL, *buf2 = NULL; + unsigned long buf1len, buf2len; + unsigned long oid[16], version; + const char *rsaoid; + ltc_asn1_list alg_seq[2], top_seq[3]; + ltc_asn1_list *l = NULL; + unsigned char *decrypted = NULL; + unsigned long decryptedlen; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* get RSA alg oid */ + err = pk_get_oid(LTC_OID_RSA, &rsaoid); + if (err != CRYPT_OK) { goto LBL_NOFREE; } + + /* alloc buffers */ + buf1len = inlen; /* approx. */ + buf1 = XMALLOC(buf1len); + if (buf1 == NULL) { err = CRYPT_MEM; goto LBL_NOFREE; } + buf2len = inlen; /* approx. */ + buf2 = XMALLOC(buf2len); + if (buf2 == NULL) { err = CRYPT_MEM; goto LBL_FREE1; } + + /* init key */ + if ((err = rsa_init(key)) != CRYPT_OK) { goto LBL_FREE2; } + + /* try to decode encrypted priv key */ + if ((err = pkcs8_decode_flexi(in, inlen, passwd, passwdlen, &l)) != CRYPT_OK) { + goto LBL_ERR; + } + decrypted = l->data; + decryptedlen = l->size; + + /* try to decode unencrypted priv key */ + LTC_SET_ASN1(alg_seq, 0, LTC_ASN1_OBJECT_IDENTIFIER, oid, 16UL); + LTC_SET_ASN1(alg_seq, 1, LTC_ASN1_NULL, NULL, 0UL); + LTC_SET_ASN1(top_seq, 0, LTC_ASN1_SHORT_INTEGER, &version, 1UL); + LTC_SET_ASN1(top_seq, 1, LTC_ASN1_SEQUENCE, alg_seq, 2UL); + LTC_SET_ASN1(top_seq, 2, LTC_ASN1_OCTET_STRING, buf1, buf1len); + err=der_decode_sequence(decrypted, decryptedlen, top_seq, 3UL); + if (err != CRYPT_OK) { goto LBL_ERR; } + + /* check alg oid */ + if ((err = pk_oid_cmp_with_asn1(rsaoid, &alg_seq[0])) != CRYPT_OK) { + goto LBL_ERR; + } + + if ((err = rsa_import_pkcs1(buf1, top_seq[2].size, key)) != CRYPT_OK) { + goto LBL_ERR; + } + key->type = PK_PRIVATE; + err = CRYPT_OK; + goto LBL_FREE2; + +LBL_ERR: + rsa_free(key); +LBL_FREE2: + if (l) der_free_sequence_flexi(l); + XFREE(buf2); +LBL_FREE1: + XFREE(buf1); +LBL_NOFREE: + return err; +} + +#endif /* LTC_MRSA */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import_x509.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import_x509.c new file mode 100644 index 0000000..e921aae --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_import_x509.c @@ -0,0 +1,54 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_import.c + Import an RSA key from a X.509 certificate, Steffen Jaeckel +*/ + +#ifdef LTC_MRSA + +static int s_rsa_decode(const unsigned char *in, unsigned long inlen, rsa_key *key) +{ + /* now it should be SEQUENCE { INTEGER, INTEGER } */ + return der_decode_sequence_multi(in, inlen, + LTC_ASN1_INTEGER, 1UL, key->N, + LTC_ASN1_INTEGER, 1UL, key->e, + LTC_ASN1_EOL, 0UL, NULL); +} + +/** + Import an RSA key from a X.509 certificate + @param in The packet to import from + @param inlen It's length (octets) + @param key [out] Destination for newly imported key + @return CRYPT_OK if successful, upon error allocated memory is freed +*/ +int rsa_import_x509(const unsigned char *in, unsigned long inlen, rsa_key *key) +{ + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + /* init key */ + if ((err = rsa_init(key)) != CRYPT_OK) { + return err; + } + + if ((err = x509_decode_public_key_from_certificate(in, inlen, + LTC_OID_RSA, LTC_ASN1_NULL, + NULL, NULL, + (public_key_decode_cb)s_rsa_decode, key)) != CRYPT_OK) { + rsa_free(key); + } else { + key->type = PK_PUBLIC; + } + + return err; +} + +#endif /* LTC_MRSA */ + diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_key.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_key.c new file mode 100644 index 0000000..2d0712f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_key.c @@ -0,0 +1,103 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_key.c + Free an RSA key, Tom St Denis + Basic operations on an RSA key, Steffen Jaeckel +*/ + +#ifdef LTC_MRSA +#include + +static void s_mpi_shrink_multi(void **a, ...) +{ + void **cur; + unsigned n; + int err; + va_list args; + void *tmp[10] = { 0 }; + void **arg[10] = { 0 }; + + /* We re-allocate in the order that we received the varargs */ + n = 0; + err = CRYPT_ERROR; + cur = a; + va_start(args, a); + while (cur != NULL) { + if (n >= sizeof(tmp)/sizeof(tmp[0])) { + goto out; + } + if (*cur != NULL) { + arg[n] = cur; + if ((err = mp_init_copy(&tmp[n], *arg[n])) != CRYPT_OK) { + goto out; + } + n++; + } + cur = va_arg(args, void**); + } + va_end(args); + + /* but we clear the old values in the reverse order */ + while (n != 0 && arg[--n] != NULL) { + mp_clear(*arg[n]); + *arg[n] = tmp[n]; + } +out: + va_end(args); + /* clean-up after an error + * or after this was called with too many args + */ + if ((err != CRYPT_OK) || + (n >= sizeof(tmp)/sizeof(tmp[0]))) { + for (n = 0; n < sizeof(tmp)/sizeof(tmp[0]); ++n) { + if (tmp[n] != NULL) { + mp_clear(tmp[n]); + } + } + } +} + +/** + This shrinks the allocated memory of a RSA key + + It will use up some more memory temporarily, + but then it will free-up the entire sequence that + was once allocated when the key was created/populated. + + This only works with libtommath >= 1.2.0 in earlier versions + it has the inverse effect due to the way it worked internally. + Also works for GNU MP, tomsfastmath naturally shows no effect. + + @param key The RSA key to shrink +*/ +void rsa_shrink_key(rsa_key *key) +{ + LTC_ARGCHKVD(key != NULL); + s_mpi_shrink_multi(&key->e, &key->d, &key->N, &key->dQ, &key->dP, &key->qP, &key->p, &key->q, NULL); +} + +/** + Init an RSA key + @param key The RSA key to free + @return CRYPT_OK if successful +*/ +int rsa_init(rsa_key *key) +{ + LTC_ARGCHK(key != NULL); + return mp_init_multi(&key->e, &key->d, &key->N, &key->dQ, &key->dP, &key->qP, &key->p, &key->q, LTC_NULL); +} + +/** + Free an RSA key from memory + @param key The RSA key to free +*/ +void rsa_free(rsa_key *key) +{ + LTC_ARGCHKVD(key != NULL); + mp_cleanup_multi(&key->q, &key->p, &key->qP, &key->dP, &key->dQ, &key->N, &key->d, &key->e, LTC_NULL); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_make_key.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_make_key.c new file mode 100644 index 0000000..6bfc041 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_make_key.c @@ -0,0 +1,165 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_make_key.c + RSA key generation, Tom St Denis +*/ + +#ifdef LTC_MRSA + +static int s_rsa_make_key(prng_state *prng, int wprng, int size, void *e, rsa_key *key) +{ + void *p, *q, *tmp1, *tmp2; + int err; + + LTC_ARGCHK(ltc_mp.name != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(size > 0); + + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + + if ((err = mp_init_multi(&p, &q, &tmp1, &tmp2, LTC_NULL)) != CRYPT_OK) { + return err; + } + + /* make primes p and q (optimization provided by Wayne Scott) */ + + /* make prime "p" */ + do { + if ((err = rand_prime( p, size/2, prng, wprng)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_sub_d( p, 1, tmp1)) != CRYPT_OK) { goto cleanup; } /* tmp1 = p-1 */ + if ((err = mp_gcd( tmp1, e, tmp2)) != CRYPT_OK) { goto cleanup; } /* tmp2 = gcd(p-1, e) */ + } while (mp_cmp_d( tmp2, 1) != 0); /* while e divides p-1 */ + + /* make prime "q" */ + do { + if ((err = rand_prime( q, size/2, prng, wprng)) != CRYPT_OK) { goto cleanup; } + if ((err = mp_sub_d( q, 1, tmp1)) != CRYPT_OK) { goto cleanup; } /* tmp1 = q-1 */ + if ((err = mp_gcd( tmp1, e, tmp2)) != CRYPT_OK) { goto cleanup; } /* tmp2 = gcd(q-1, e) */ + } while (mp_cmp_d( tmp2, 1) != 0); /* while e divides q-1 */ + + /* tmp1 = lcm(p-1, q-1) */ + if ((err = mp_sub_d( p, 1, tmp2)) != CRYPT_OK) { goto cleanup; } /* tmp2 = p-1 */ + /* tmp1 = q-1 (previous do/while loop) */ + if ((err = mp_lcm( tmp1, tmp2, tmp1)) != CRYPT_OK) { goto cleanup; } /* tmp1 = lcm(p-1, q-1) */ + + /* make key */ + if ((err = rsa_init(key)) != CRYPT_OK) { + goto errkey; + } + + if ((err = mp_copy( e, key->e)) != CRYPT_OK) { goto errkey; } /* key->e = e */ + if ((err = mp_invmod( key->e, tmp1, key->d)) != CRYPT_OK) { goto errkey; } /* key->d = 1/e mod lcm(p-1,q-1) */ + if ((err = mp_mul( p, q, key->N)) != CRYPT_OK) { goto errkey; } /* key->N = pq */ + + /* optimize for CRT now */ + /* find d mod q-1 and d mod p-1 */ + if ((err = mp_sub_d( p, 1, tmp1)) != CRYPT_OK) { goto errkey; } /* tmp1 = p-1 */ + if ((err = mp_sub_d( q, 1, tmp2)) != CRYPT_OK) { goto errkey; } /* tmp2 = q-1 */ + if ((err = mp_mod( key->d, tmp1, key->dP)) != CRYPT_OK) { goto errkey; } /* dP = d mod p-1 */ + if ((err = mp_mod( key->d, tmp2, key->dQ)) != CRYPT_OK) { goto errkey; } /* dQ = d mod q-1 */ + if ((err = mp_invmod( q, p, key->qP)) != CRYPT_OK) { goto errkey; } /* qP = 1/q mod p */ + + if ((err = mp_copy( p, key->p)) != CRYPT_OK) { goto errkey; } + if ((err = mp_copy( q, key->q)) != CRYPT_OK) { goto errkey; } + + /* set key type (in this case it's CRT optimized) */ + key->type = PK_PRIVATE; + + /* return ok and free temps */ + err = CRYPT_OK; + goto cleanup; +errkey: + rsa_free(key); +cleanup: + mp_clear_multi(tmp2, tmp1, q, p, LTC_NULL); + return err; +} + +/** + Create an RSA key based on a long public exponent type + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param size The size of the modulus (key size) desired (octets) + @param e The "e" value (public key). e==65537 is a good choice + @param key [out] Destination of a newly created private key pair + @return CRYPT_OK if successful, upon error all allocated ram is freed +*/ +int rsa_make_key(prng_state *prng, int wprng, int size, long e, rsa_key *key) +{ + void *tmp_e; + int err; + + if ((e < 3) || ((e & 1) == 0)) { + return CRYPT_INVALID_ARG; + } + + if ((err = mp_init(&tmp_e)) != CRYPT_OK) { + return err; + } + + if ((err = mp_set_int(tmp_e, e)) == CRYPT_OK) + err = s_rsa_make_key(prng, wprng, size, tmp_e, key); + + mp_clear(tmp_e); + + return err; +} + +/** + Create an RSA key based on a hexadecimal public exponent type + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param size The size of the modulus (key size) desired (octets) + @param e The "e" value (public key). e==65537 is a good choice + @param elen The length of e (octets) + @param key [out] Destination of a newly created private key pair + @return CRYPT_OK if successful, upon error all allocated ram is freed +*/ +int rsa_make_key_ubin_e(prng_state *prng, int wprng, int size, + const unsigned char *e, unsigned long elen, rsa_key *key) +{ + int err; + void *tmp_e; + + if ((err = mp_init(&tmp_e)) != CRYPT_OK) { + return err; + } + + if ((err = mp_read_unsigned_bin(tmp_e, (unsigned char *)e, elen)) == CRYPT_OK) + err = rsa_make_key_bn_e(prng, wprng, size, tmp_e, key); + + mp_clear(tmp_e); + + return err; +} + +/** + Create an RSA key based on a bignumber public exponent type + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param size The size of the modulus (key size) desired (octets) + @param e The "e" value (public key). e==65537 is a good choice + @param key [out] Destination of a newly created private key pair + @return CRYPT_OK if successful, upon error all allocated ram is freed +*/ +int rsa_make_key_bn_e(prng_state *prng, int wprng, int size, void *e, rsa_key *key) +{ + int err; + int e_bits; + + e_bits = mp_count_bits(e); + if ((e_bits > 1 && e_bits < 256) && (mp_get_digit(e, 0) & 1)) { + err = s_rsa_make_key(prng, wprng, size, e, key); + } else { + err = CRYPT_INVALID_ARG; + } + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_set.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_set.c new file mode 100644 index 0000000..d4dc6ed --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_set.c @@ -0,0 +1,123 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + + +#ifdef LTC_MRSA + +/** + Import RSA key from raw numbers + + @param N RSA's N + @param Nlen RSA's N's length + @param e RSA's e + @param elen RSA's e's length + @param d RSA's d (only private key, NULL for public key) + @param dlen RSA's d's length + @param key [out] the destination for the imported key + @return CRYPT_OK if successful +*/ +int rsa_set_key(const unsigned char *N, unsigned long Nlen, + const unsigned char *e, unsigned long elen, + const unsigned char *d, unsigned long dlen, + rsa_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(N != NULL); + LTC_ARGCHK(e != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + if ((err = rsa_init(key)) != CRYPT_OK) return err; + + if ((err = mp_read_unsigned_bin(key->N , (unsigned char *)N , Nlen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_unsigned_bin(key->e , (unsigned char *)e , elen)) != CRYPT_OK) { goto LBL_ERR; } + if (d && dlen) { + if ((err = mp_read_unsigned_bin(key->d , (unsigned char *)d , dlen)) != CRYPT_OK) { goto LBL_ERR; } + key->type = PK_PRIVATE; + } + else { + key->type = PK_PUBLIC; + } + return CRYPT_OK; + +LBL_ERR: + rsa_free(key); + return err; +} + +/** + Import factors of an RSA key from raw numbers + + Only for private keys. + + @param p RSA's p + @param plen RSA's p's length + @param q RSA's q + @param qlen RSA's q's length + @param key [out] the destination for the imported key + @return CRYPT_OK if successful +*/ +int rsa_set_factors(const unsigned char *p, unsigned long plen, + const unsigned char *q, unsigned long qlen, + rsa_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(p != NULL); + LTC_ARGCHK(q != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + if (key->type != PK_PRIVATE) return CRYPT_PK_TYPE_MISMATCH; + + if ((err = mp_read_unsigned_bin(key->p , (unsigned char *)p , plen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_unsigned_bin(key->q , (unsigned char *)q , qlen)) != CRYPT_OK) { goto LBL_ERR; } + return CRYPT_OK; + +LBL_ERR: + rsa_free(key); + return err; +} + +/** + Import CRT parameters of an RSA key from raw numbers + + Only for private keys. + + @param dP RSA's dP + @param dPlen RSA's dP's length + @param dQ RSA's dQ + @param dQlen RSA's dQ's length + @param qP RSA's qP + @param qPlen RSA's qP's length + @param key [out] the destination for the imported key + @return CRYPT_OK if successful +*/ +int rsa_set_crt_params(const unsigned char *dP, unsigned long dPlen, + const unsigned char *dQ, unsigned long dQlen, + const unsigned char *qP, unsigned long qPlen, + rsa_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(dP != NULL); + LTC_ARGCHK(dQ != NULL); + LTC_ARGCHK(qP != NULL); + LTC_ARGCHK(ltc_mp.name != NULL); + + if (key->type != PK_PRIVATE) return CRYPT_PK_TYPE_MISMATCH; + + if ((err = mp_read_unsigned_bin(key->dP, (unsigned char *)dP, dPlen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_unsigned_bin(key->dQ, (unsigned char *)dQ, dQlen)) != CRYPT_OK) { goto LBL_ERR; } + if ((err = mp_read_unsigned_bin(key->qP, (unsigned char *)qP, qPlen)) != CRYPT_OK) { goto LBL_ERR; } + return CRYPT_OK; + +LBL_ERR: + rsa_free(key); + return err; +} + +#endif /* LTC_MRSA */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_sign_hash.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_sign_hash.c new file mode 100644 index 0000000..d7bc5af --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_sign_hash.c @@ -0,0 +1,136 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_sign_hash.c + RSA PKCS #1 v1.5 and v2 PSS sign hash, Tom St Denis and Andreas Lange +*/ + +#ifdef LTC_MRSA + +/** + PKCS #1 pad then sign + @param in The hash to sign + @param inlen The length of the hash to sign (octets) + @param out [out] The signature + @param outlen [in/out] The max size and resulting size of the signature + @param padding Type of padding (LTC_PKCS_1_PSS, LTC_PKCS_1_V1_5 or LTC_PKCS_1_V1_5_NA1) + @param prng An active PRNG state + @param prng_idx The index of the PRNG desired + @param hash_idx The index of the hash desired + @param saltlen The length of the salt desired (octets) + @param key The private RSA key to use + @return CRYPT_OK if successful +*/ +int rsa_sign_hash_ex(const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen, + int padding, + prng_state *prng, int prng_idx, + int hash_idx, unsigned long saltlen, + const rsa_key *key) +{ + unsigned long modulus_bitlen, modulus_bytelen, x, y; + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + LTC_ARGCHK(key != NULL); + + /* valid padding? */ + if ((padding != LTC_PKCS_1_V1_5) && + (padding != LTC_PKCS_1_PSS) && + (padding != LTC_PKCS_1_V1_5_NA1)) { + return CRYPT_PK_INVALID_PADDING; + } + + if (padding == LTC_PKCS_1_PSS) { + /* valid prng ? */ + if ((err = prng_is_valid(prng_idx)) != CRYPT_OK) { + return err; + } + } + + if (padding != LTC_PKCS_1_V1_5_NA1) { + /* valid hash ? */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + } + + /* get modulus len in bits */ + modulus_bitlen = mp_count_bits((key->N)); + + /* outlen must be at least the size of the modulus */ + modulus_bytelen = mp_unsigned_bin_size((key->N)); + if (modulus_bytelen > *outlen) { + *outlen = modulus_bytelen; + return CRYPT_BUFFER_OVERFLOW; + } + + if (padding == LTC_PKCS_1_PSS) { + /* PSS pad the key */ + x = *outlen; + if ((err = pkcs_1_pss_encode(in, inlen, saltlen, prng, prng_idx, + hash_idx, modulus_bitlen, out, &x)) != CRYPT_OK) { + return err; + } + } else { + /* PKCS #1 v1.5 pad the hash */ + unsigned char *tmpin; + + if (padding == LTC_PKCS_1_V1_5) { + ltc_asn1_list digestinfo[2], siginfo[2]; + /* not all hashes have OIDs... so sad */ + if (hash_descriptor[hash_idx]->OIDlen == 0) { + return CRYPT_INVALID_ARG; + } + + /* construct the SEQUENCE + SEQUENCE { + SEQUENCE {hashoid OID + blah NULL + } + hash OCTET STRING + } + */ + LTC_SET_ASN1(digestinfo, 0, LTC_ASN1_OBJECT_IDENTIFIER, hash_descriptor[hash_idx]->OID, hash_descriptor[hash_idx]->OIDlen); + LTC_SET_ASN1(digestinfo, 1, LTC_ASN1_NULL, NULL, 0); + LTC_SET_ASN1(siginfo, 0, LTC_ASN1_SEQUENCE, digestinfo, 2); + LTC_SET_ASN1(siginfo, 1, LTC_ASN1_OCTET_STRING, in, inlen); + + /* allocate memory for the encoding */ + y = mp_unsigned_bin_size(key->N); + tmpin = XMALLOC(y); + if (tmpin == NULL) { + return CRYPT_MEM; + } + + if ((err = der_encode_sequence(siginfo, 2, tmpin, &y)) != CRYPT_OK) { + XFREE(tmpin); + return err; + } + } else { + /* set the pointer and data-length to the input values */ + tmpin = (unsigned char *)in; + y = inlen; + } + + x = *outlen; + err = pkcs_1_v1_5_encode(tmpin, y, LTC_PKCS_1_EMSA, modulus_bitlen, NULL, 0, out, &x); + + if (padding == LTC_PKCS_1_V1_5) { + XFREE(tmpin); + } + + if (err != CRYPT_OK) { + return err; + } + } + + /* RSA encode it */ + return ltc_mp.rsa_me(out, x, out, outlen, PK_PRIVATE, key); +} + +#endif /* LTC_MRSA */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_sign_saltlen_get.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_sign_saltlen_get.c new file mode 100644 index 0000000..802aac3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_sign_saltlen_get.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file rsa_sign_saltlen_get.c + Retrieve the maximum size of the salt, Steffen Jaeckel. +*/ + +#ifdef LTC_MRSA + +/** + Retrieve the maximum possible size of the salt when creating a PKCS#1 PSS signature. + @param padding Type of padding (LTC_PKCS_1_PSS only) + @param hash_idx The index of the desired hash + @param key The RSA key + @return The maximum salt length in bytes or INT_MAX on error. +*/ +int rsa_sign_saltlen_get_max_ex(int padding, int hash_idx, const rsa_key *key) +{ + int ret = INT_MAX; + LTC_ARGCHK(key != NULL); + + if ((hash_is_valid(hash_idx) == CRYPT_OK) && + (padding == LTC_PKCS_1_PSS)) + { + ret = rsa_get_size(key); + if (ret < INT_MAX) + { + ret -= (hash_descriptor[hash_idx]->hashsize + 2); + } /* if */ + } /* if */ + + return ret; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_verify_hash.c b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_verify_hash.c new file mode 100644 index 0000000..f035f6f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/rsa_verify_hash.c @@ -0,0 +1,208 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include +#include "tomcrypt_private.h" + +/** + @file rsa_verify_hash.c + RSA PKCS #1 v1.5 or v2 PSS signature verification, Tom St Denis and Andreas Lange +*/ + +#ifdef LTC_MRSA + +/** + PKCS #1 de-sign then v1.5 or PSS depad + @param sig The signature data + @param siglen The length of the signature data (octets) + @param hash The hash of the message that was signed + @param hashlen The length of the hash of the message that was signed (octets) + @param padding Type of padding (LTC_PKCS_1_PSS, LTC_PKCS_1_V1_5 or LTC_PKCS_1_V1_5_NA1) + @param hash_idx The index of the desired hash + @param saltlen The length of the salt used during signature + @param stat [out] The result of the signature comparison, 1==valid, 0==invalid + @param key The public RSA key corresponding to the key that performed the signature + @return CRYPT_OK on success (even if the signature is invalid) +*/ +int rsa_verify_hash_ex(const unsigned char *sig, unsigned long siglen, + const unsigned char *hash, unsigned long hashlen, + int padding, + int hash_idx, unsigned long saltlen, + int *stat, const rsa_key *key) +{ + unsigned long modulus_bitlen, modulus_bytelen, x; + int err; + unsigned int inc1 = 0; + unsigned char *tmpbuf; + struct ftmn ftmn = { }; + + LTC_ARGCHK(hash != NULL); + LTC_ARGCHK(sig != NULL); + LTC_ARGCHK(stat != NULL); + LTC_ARGCHK(key != NULL); + + /* default to invalid */ + *stat = 0; + FTMN_SET_CHECK_RES(&ftmn, FTMN_INCR0, 1); + + /* valid padding? */ + + if ((padding != LTC_PKCS_1_V1_5) && + (padding != LTC_PKCS_1_PSS) && + (padding != LTC_PKCS_1_V1_5_NA1)) { + return CRYPT_PK_INVALID_PADDING; + } + + if (padding != LTC_PKCS_1_V1_5_NA1) { + /* valid hash ? */ + if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { + return err; + } + } + + /* get modulus len in bits */ + modulus_bitlen = mp_count_bits( (key->N)); + + /* outlen must be at least the size of the modulus */ + modulus_bytelen = mp_unsigned_bin_size( (key->N)); + if (modulus_bytelen != siglen) { + return CRYPT_INVALID_PACKET; + } + + /* allocate temp buffer for decoded sig */ + tmpbuf = XMALLOC(siglen); + if (tmpbuf == NULL) { + return CRYPT_MEM; + } + + /* RSA decode it */ + x = siglen; + if ((err = ltc_mp.rsa_me(sig, siglen, tmpbuf, &x, PK_PUBLIC, key)) != CRYPT_OK) { + XFREE(tmpbuf); + return err; + } + + /* make sure the output is the right size */ + if (x != siglen) { + XFREE(tmpbuf); + return CRYPT_INVALID_PACKET; + } + + if (padding == LTC_PKCS_1_PSS) { + /* PSS decode and verify it */ + + FTMN_PUSH_LINKED_CALL(&ftmn, FTMN_FUNC_HASH("pkcs_1_pss_decode")); + if(modulus_bitlen%8 == 1){ + err = pkcs_1_pss_decode(hash, hashlen, tmpbuf+1, x-1, saltlen, hash_idx, modulus_bitlen, stat); + } + else{ + err = pkcs_1_pss_decode(hash, hashlen, tmpbuf, x, saltlen, hash_idx, modulus_bitlen, stat); + } + if (*stat) { + FTMN_SET_CHECK_RES_FROM_CALL(&ftmn, FTMN_INCR1, 0); + inc1 = 1; + } + FTMN_POP_LINKED_CALL(&ftmn); + + } else { + /* PKCS #1 v1.5 decode it */ + unsigned char *out; + unsigned long outlen; + int decoded; + + /* allocate temp buffer for decoded hash */ + outlen = ((modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0)) - 3; + out = XMALLOC(outlen); + if (out == NULL) { + err = CRYPT_MEM; + goto bail_2; + } + + if ((err = pkcs_1_v1_5_decode(tmpbuf, x, LTC_PKCS_1_EMSA, modulus_bitlen, out, &outlen, &decoded)) != CRYPT_OK) { + XFREE(out); + goto bail_2; + } + + if (padding == LTC_PKCS_1_V1_5) { + unsigned long loid[16], reallen; + ltc_asn1_list digestinfo[2], siginfo[2]; + + /* not all hashes have OIDs... so sad */ + if (hash_descriptor[hash_idx]->OIDlen == 0) { + err = CRYPT_INVALID_ARG; + goto bail_2; + } + + /* now we must decode out[0...outlen-1] using ASN.1, test the OID and then test the hash */ + /* construct the SEQUENCE + SEQUENCE { + SEQUENCE {hashoid OID + blah NULL + } + hash OCTET STRING + } + */ + LTC_SET_ASN1(digestinfo, 0, LTC_ASN1_OBJECT_IDENTIFIER, loid, sizeof(loid)/sizeof(loid[0])); + LTC_SET_ASN1(digestinfo, 1, LTC_ASN1_NULL, NULL, 0); + LTC_SET_ASN1(siginfo, 0, LTC_ASN1_SEQUENCE, digestinfo, 2); + LTC_SET_ASN1(siginfo, 1, LTC_ASN1_OCTET_STRING, tmpbuf, siglen); + + if ((err = der_decode_sequence_strict(out, outlen, siginfo, 2)) != CRYPT_OK) { + /* fallback to Legacy:missing NULL */ + LTC_SET_ASN1(siginfo, 0, LTC_ASN1_SEQUENCE, digestinfo, 1); + if ((err = der_decode_sequence_strict(out, outlen, siginfo, 2)) != CRYPT_OK) { + XFREE(out); + goto bail_2; + } + } + + if ((err = der_length_sequence(siginfo, 2, &reallen)) != CRYPT_OK) { + XFREE(out); + goto bail_2; + } + + /* test OID */ + if ((reallen == outlen) && + (digestinfo[0].size == hash_descriptor[hash_idx]->OIDlen) && + (XMEMCMP(digestinfo[0].data, hash_descriptor[hash_idx]->OID, sizeof(unsigned long) * hash_descriptor[hash_idx]->OIDlen) == 0) && + (siginfo[1].size == hashlen) && + (ftmn_set_check_res_memcmp(&ftmn, FTMN_INCR1, XMEMCMP, + siginfo[1].data, hash, hashlen) == 0)) { + *stat = 1; + } + inc1 = 1; + } else { + /* only check if the hash is equal */ + if ((hashlen == outlen) && + (ftmn_set_check_res_memcmp(&ftmn, FTMN_INCR1, XMEMCMP, + out, hash, hashlen) == 0)) { + *stat = 1; + } + inc1 = 1; + } + + if (!*stat) { + /* + * The call to ftmn_set_check_res_memcmp() above might have failed, + * since memcmp() may set any non-zero result force it 1 to match the + * check with FTMN_CALLEE_DONE_CHECK() below. + */ + FTMN_SET_CHECK_RES_NOT_ZERO(&ftmn, FTMN_INCR1, 1); + inc1++; + } + +#ifdef LTC_CLEAN_STACK + zeromem(out, outlen); +#endif + XFREE(out); + } + +bail_2: +#ifdef LTC_CLEAN_STACK + zeromem(tmpbuf, siglen); +#endif + XFREE(tmpbuf); + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR0, FTMN_STEP_COUNT(1, inc1), !*stat); + return err; +} + +#endif /* LTC_MRSA */ diff --git a/optee_os/core/lib/libtomcrypt/src/pk/rsa/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/rsa/sub.mk new file mode 100644 index 0000000..f0b8975 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/rsa/sub.mk @@ -0,0 +1,9 @@ +srcs-y += rsa_decrypt_key.c +srcs-y += rsa_encrypt_key.c +srcs-y += rsa_export.c +srcs-y += rsa_exptmod.c +srcs-y += rsa_import.c +srcs-y += rsa_key.c +srcs-y += rsa_make_key.c +srcs-y += rsa_sign_hash.c +srcs-y += rsa_verify_hash.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/sub.mk new file mode 100644 index 0000000..dca26a5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/sub.mk @@ -0,0 +1,11 @@ +subdirs-$(_CFG_CORE_LTC_ASN1) += asn1 +subdirs-$(_CFG_CORE_LTC_DSA) += dsa +# PKCS1 paddings are used with RSA only +subdirs-$(_CFG_CORE_LTC_RSA) += pkcs1 +subdirs-$(_CFG_CORE_LTC_RSA) += rsa +subdirs-$(_CFG_CORE_LTC_DH) += dh +subdirs-$(_CFG_CORE_LTC_ECC) += ecc +subdirs-$(_CFG_CORE_LTC_X25519) += ec25519 +subdirs-$(_CFG_CORE_LTC_X25519) += x25519 +subdirs-$(_CFG_CORE_LTC_ED25519) += ec25519 +subdirs-$(_CFG_CORE_LTC_ED25519) += ed25519 diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/sub.mk b/optee_os/core/lib/libtomcrypt/src/pk/x25519/sub.mk new file mode 100644 index 0000000..853371e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/sub.mk @@ -0,0 +1,4 @@ +srcs-y += x25519_export.c +srcs-y += x25519_import.c +srcs-y += x25519_make_key.c +srcs-y += x25519_shared_secret.c diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_export.c b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_export.c new file mode 100644 index 0000000..0687c13 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_export.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x25519_export.c + Export a X25519 key to a binary packet, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Export a X25519 key to a binary packet + @param out [out] The destination for the key + @param outlen [in/out] The max size and resulting size of the X25519 key + @param type Which type of key (PK_PRIVATE, PK_PUBLIC|PK_STD or PK_PUBLIC) + @param key The key you wish to export + @return CRYPT_OK if successful +*/ +int x25519_export( unsigned char *out, unsigned long *outlen, + int which, + const curve25519_key *key) +{ + LTC_ARGCHK(key != NULL); + + if (key->algo != LTC_OID_X25519) return CRYPT_PK_INVALID_TYPE; + + return ec25519_export(out, outlen, which, key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import.c b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import.c new file mode 100644 index 0000000..247885f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import.c @@ -0,0 +1,35 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x25519_import.c + Import a X25519 key from a SubjectPublicKeyInfo, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Import a X25519 key + @param in The packet to read + @param inlen The length of the input packet + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int x25519_import(const unsigned char *in, unsigned long inlen, curve25519_key *key) +{ + int err; + unsigned long key_len; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + + key_len = sizeof(key->pub); + if ((err = x509_decode_subject_public_key_info(in, inlen, LTC_OID_X25519, key->pub, &key_len, LTC_ASN1_EOL, NULL, 0uL)) == CRYPT_OK) { + key->type = PK_PUBLIC; + key->algo = LTC_OID_X25519; + } + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_pkcs8.c b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_pkcs8.c new file mode 100644 index 0000000..8b577c3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_pkcs8.c @@ -0,0 +1,28 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x25519_import_pkcs8.c + Import a X25519 key in PKCS#8 format, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Import a X25519 private key in PKCS#8 format + @param in The DER-encoded PKCS#8-formatted private key + @param inlen The length of the input data + @param passwd The password to decrypt the private key + @param passwdlen Password's length (octets) + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int x25519_import_pkcs8(const unsigned char *in, unsigned long inlen, + const void *pwd, unsigned long pwdlen, + curve25519_key *key) +{ + return ec25519_import_pkcs8(in, inlen, pwd, pwdlen, LTC_OID_X25519, tweetnacl_crypto_scalarmult_base, key); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_raw.c b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_raw.c new file mode 100644 index 0000000..e86e8c6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_raw.c @@ -0,0 +1,41 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x25519_import_raw.c + Set the parameters of a X25519 key, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Set the parameters of a X25519 key + + @param in The key + @param inlen The length of the key + @param which Which type of key (PK_PRIVATE or PK_PUBLIC) + @param key [out] Destination of the key + @return CRYPT_OK if successful +*/ +int x25519_import_raw(const unsigned char *in, unsigned long inlen, int which, curve25519_key *key) +{ + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen == 32uL); + LTC_ARGCHK(key != NULL); + + if (which == PK_PRIVATE) { + XMEMCPY(key->priv, in, sizeof(key->priv)); + tweetnacl_crypto_scalarmult_base(key->pub, key->priv); + } else if (which == PK_PUBLIC) { + XMEMCPY(key->pub, in, sizeof(key->pub)); + } else { + return CRYPT_INVALID_ARG; + } + key->algo = LTC_OID_X25519; + key->type = which; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_x509.c b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_x509.c new file mode 100644 index 0000000..043b6ad --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_import_x509.c @@ -0,0 +1,45 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x25519_import_x509.c + Import a X25519 key from a X.509 certificate, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +static int s_x25519_decode(const unsigned char *in, unsigned long inlen, curve25519_key *key) +{ + if (inlen != sizeof(key->pub)) return CRYPT_PK_INVALID_SIZE; + XMEMCPY(key->pub, in, sizeof(key->pub)); + return CRYPT_OK; +} + +/** + Import a X25519 public key from a X.509 certificate + @param in The DER encoded X.509 certificate + @param inlen The length of the certificate + @param key [out] Where to import the key to + @return CRYPT_OK if successful, on error all allocated memory is freed automatically +*/ +int x25519_import_x509(const unsigned char *in, unsigned long inlen, curve25519_key *key) +{ + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(key != NULL); + + if ((err = x509_decode_public_key_from_certificate(in, inlen, + LTC_OID_X25519, + LTC_ASN1_EOL, NULL, NULL, + (public_key_decode_cb)s_x25519_decode, key)) != CRYPT_OK) { + return err; + } + key->type = PK_PUBLIC; + key->algo = LTC_OID_X25519; + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_make_key.c b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_make_key.c new file mode 100644 index 0000000..9080cbd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_make_key.c @@ -0,0 +1,41 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x25519_make_key.c + Create a X25519 key, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Create a X25519 key + @param prng An active PRNG state + @param wprng The index of the PRNG desired + @param key [out] Destination of a newly created private key pair + @return CRYPT_OK if successful +*/ +int x25519_make_key(prng_state *prng, int wprng, curve25519_key *key) +{ + int err; + + LTC_ARGCHK(key != NULL); + + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + + if (prng_descriptor[wprng]->read(key->priv, sizeof(key->priv), prng) != sizeof(key->priv)) { + return CRYPT_ERROR_READPRNG; + } + + tweetnacl_crypto_scalarmult_base(key->pub, key->priv); + + key->type = PK_PRIVATE; + key->algo = LTC_OID_X25519; + + return err; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_shared_secret.c b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_shared_secret.c new file mode 100644 index 0000000..eaea1c0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/pk/x25519/x25519_shared_secret.c @@ -0,0 +1,42 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file x25519_shared_secret.c + Create a X25519 shared secret, Steffen Jaeckel +*/ + +#ifdef LTC_CURVE25519 + +/** + Create a X25519 shared secret. + @param private_key The private X25519 key in the pair + @param public_key The public X25519 key in the pair + @param out [out] The destination of the shared data + @param outlen [in/out] The max size and resulting size of the shared data. + @return CRYPT_OK if successful +*/ +int x25519_shared_secret(const curve25519_key *private_key, + const curve25519_key *public_key, + unsigned char *out, unsigned long *outlen) +{ + LTC_ARGCHK(private_key != NULL); + LTC_ARGCHK(public_key != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(outlen != NULL); + + if(private_key->type != PK_PRIVATE) return CRYPT_PK_INVALID_TYPE; + + if(*outlen < 32uL) { + *outlen = 32uL; + return CRYPT_BUFFER_OVERFLOW; + } + + tweetnacl_crypto_scalarmult(out, private_key->priv, public_key->pub); + *outlen = 32uL; + + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/chacha20.c b/optee_os/core/lib/libtomcrypt/src/prngs/chacha20.c new file mode 100644 index 0000000..d9bd7b0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/chacha20.c @@ -0,0 +1,218 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + + /* the idea of re-keying loosely follows the approach used in: + * http://bxr.su/OpenBSD/lib/libc/crypt/arc4random.c + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA20_PRNG + +const struct ltc_prng_descriptor chacha20_prng_desc = +{ + "chacha20", + 40, + &chacha20_prng_start, + &chacha20_prng_add_entropy, + &chacha20_prng_ready, + &chacha20_prng_read, + &chacha20_prng_done, + &chacha20_prng_export, + &chacha20_prng_import, + &chacha20_prng_test +}; + +/** + Start the PRNG + @param prng The PRNG state to initialize + @return CRYPT_OK if successful +*/ +int chacha20_prng_start(prng_state *prng) +{ + LTC_ARGCHK(prng != NULL); + prng->ready = 0; + XMEMSET(&prng->u.chacha.ent, 0, sizeof(prng->u.chacha.ent)); + prng->u.chacha.idx = 0; + LTC_MUTEX_INIT(&prng->lock) + return CRYPT_OK; +} + +/** + Add entropy to the PRNG state + @param in The data to add + @param inlen Length of the data to add + @param prng PRNG state to update + @return CRYPT_OK if successful +*/ +int chacha20_prng_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + unsigned char buf[40]; + unsigned long i; + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen > 0); + + LTC_MUTEX_LOCK(&prng->lock); + if (prng->ready) { + /* chacha20_prng_ready() was already called, do "rekey" operation */ + if ((err = chacha_keystream(&prng->u.chacha.s, buf, sizeof(buf))) != CRYPT_OK) goto LBL_UNLOCK; + for(i = 0; i < inlen; i++) buf[i % sizeof(buf)] ^= in[i]; + /* key 32 bytes, 20 rounds */ + if ((err = chacha_setup(&prng->u.chacha.s, buf, 32, 20)) != CRYPT_OK) goto LBL_UNLOCK; + /* iv 8 bytes */ + if ((err = chacha_ivctr64(&prng->u.chacha.s, buf + 32, 8, 0)) != CRYPT_OK) goto LBL_UNLOCK; + /* clear KEY + IV */ + zeromem(buf, sizeof(buf)); + } + else { + /* chacha20_prng_ready() was not called yet, add entropy to ent buffer */ + while (inlen--) prng->u.chacha.ent[prng->u.chacha.idx++ % sizeof(prng->u.chacha.ent)] ^= *in++; + } + err = CRYPT_OK; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Make the PRNG ready to read from + @param prng The PRNG to make active + @return CRYPT_OK if successful +*/ +int chacha20_prng_ready(prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + + LTC_MUTEX_LOCK(&prng->lock); + if (prng->ready) { err = CRYPT_OK; goto LBL_UNLOCK; } + /* key 32 bytes, 20 rounds */ + if ((err = chacha_setup(&prng->u.chacha.s, prng->u.chacha.ent, 32, 20)) != CRYPT_OK) goto LBL_UNLOCK; + /* iv 8 bytes */ + if ((err = chacha_ivctr64(&prng->u.chacha.s, prng->u.chacha.ent + 32, 8, 0)) != CRYPT_OK) goto LBL_UNLOCK; + XMEMSET(&prng->u.chacha.ent, 0, sizeof(prng->u.chacha.ent)); + prng->u.chacha.idx = 0; + prng->ready = 1; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Read from the PRNG + @param out Destination + @param outlen Length of output + @param prng The active PRNG to read from + @return Number of octets read +*/ +unsigned long chacha20_prng_read(unsigned char *out, unsigned long outlen, prng_state *prng) +{ + if (outlen == 0 || prng == NULL || out == NULL) return 0; + LTC_MUTEX_LOCK(&prng->lock); + if (!prng->ready) { outlen = 0; goto LBL_UNLOCK; } + if (chacha_keystream(&prng->u.chacha.s, out, outlen) != CRYPT_OK) outlen = 0; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return outlen; +} + +/** + Terminate the PRNG + @param prng The PRNG to terminate + @return CRYPT_OK if successful +*/ +int chacha20_prng_done(prng_state *prng) +{ + int err; + LTC_ARGCHK(prng != NULL); + LTC_MUTEX_LOCK(&prng->lock); + prng->ready = 0; + err = chacha_done(&prng->u.chacha.s); + LTC_MUTEX_UNLOCK(&prng->lock); + LTC_MUTEX_DESTROY(&prng->lock); + return err; +} + +/** + Export the PRNG state + @param out [out] Destination + @param outlen [in/out] Max size and resulting size of the state + @param prng The PRNG to export + @return CRYPT_OK if successful +*/ +LTC_PRNG_EXPORT(chacha20_prng) + +/** + Import a PRNG state + @param in The PRNG state + @param inlen Size of the state + @param prng The PRNG to import + @return CRYPT_OK if successful +*/ +int chacha20_prng_import(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + if (inlen < (unsigned long)chacha20_prng_desc.export_size) return CRYPT_INVALID_ARG; + + if ((err = chacha20_prng_start(prng)) != CRYPT_OK) return err; + if ((err = chacha20_prng_add_entropy(in, inlen, prng)) != CRYPT_OK) return err; + return CRYPT_OK; +} + +/** + PRNG self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled +*/ +int chacha20_prng_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + prng_state st; + unsigned char en[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, + 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32 }; + unsigned char dmp[300]; + unsigned long dmplen = sizeof(dmp); + unsigned char out[500]; + unsigned char t1[] = { 0x59, 0xB2, 0x26, 0x95, 0x2B, 0x01, 0x8F, 0x05, 0xBE, 0xD8 }; + unsigned char t2[] = { 0x47, 0xC9, 0x0D, 0x03, 0xE4, 0x75, 0x34, 0x27, 0xBD, 0xDE }; + unsigned char t3[] = { 0xBC, 0xFA, 0xEF, 0x59, 0x37, 0x7F, 0x1A, 0x91, 0x1A, 0xA6 }; + int err; + + if ((err = chacha20_prng_start(&st)) != CRYPT_OK) return err; + /* add entropy to uninitialized prng */ + if ((err = chacha20_prng_add_entropy(en, sizeof(en), &st)) != CRYPT_OK) return err; + if ((err = chacha20_prng_ready(&st)) != CRYPT_OK) return err; + if (chacha20_prng_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t1, sizeof(t1), "CHACHA-PRNG", 1)) return CRYPT_FAIL_TESTVECTOR; + if (chacha20_prng_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + /* add entropy to already initialized prng */ + if ((err = chacha20_prng_add_entropy(en, sizeof(en), &st)) != CRYPT_OK) return err; + if (chacha20_prng_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if ((err = chacha20_prng_export(dmp, &dmplen, &st)) != CRYPT_OK) return err; + if (chacha20_prng_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if (chacha20_prng_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t2, sizeof(t2), "CHACHA-PRNG", 2)) return CRYPT_FAIL_TESTVECTOR; + if ((err = chacha20_prng_done(&st)) != CRYPT_OK) return err; + if ((err = chacha20_prng_import(dmp, dmplen, &st)) != CRYPT_OK) return err; + if ((err = chacha20_prng_ready(&st)) != CRYPT_OK) return err; + if (chacha20_prng_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if (chacha20_prng_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t3, sizeof(t3), "CHACHA-PRNG", 3)) return CRYPT_FAIL_TESTVECTOR; + if ((err = chacha20_prng_done(&st)) != CRYPT_OK) return err; + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/fortuna.c b/optee_os/core/lib/libtomcrypt/src/prngs/fortuna.c new file mode 100644 index 0000000..6f50098 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/fortuna.c @@ -0,0 +1,520 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_FORTUNA_RESEED_RATELIMIT_TIMED +#if defined(_WIN32) + #include +#elif defined(LTC_CLOCK_GETTIME) + #include /* struct timespec + clock_gettime */ +#else + #include /* struct timeval + gettimeofday */ +#endif +#endif + +/** + @file fortuna.c + Fortuna PRNG, Tom St Denis +*/ + +/* Implementation of Fortuna by Tom St Denis + +We deviate slightly here for reasons of simplicity [and to fit in the API]. First all "sources" +in the AddEntropy function are fixed to 0. Second since no reliable timer is provided +we reseed automatically when len(pool0) >= 64 or every LTC_FORTUNA_WD calls to the read function */ + +#ifdef LTC_FORTUNA + +/* requries LTC_SHA256 and AES */ +#if !(defined(LTC_RIJNDAEL) && defined(LTC_SHA256)) + #error LTC_FORTUNA requires LTC_SHA256 and LTC_RIJNDAEL (AES) +#endif + +#ifndef LTC_FORTUNA_POOLS + #warning LTC_FORTUNA_POOLS was not previously defined (old headers?) + #define LTC_FORTUNA_POOLS 32 +#endif + +#if LTC_FORTUNA_POOLS < 4 || LTC_FORTUNA_POOLS > 32 + #error LTC_FORTUNA_POOLS must be in [4..32] +#endif + +const struct ltc_prng_descriptor fortuna_desc = { + "fortuna", + 64, + &fortuna_start, + &fortuna_add_entropy, + &fortuna_ready, + &fortuna_read, + &fortuna_done, + &fortuna_export, + &fortuna_import, + &fortuna_test +}; + +/* update the IV */ +static void s_fortuna_update_iv(prng_state *prng) +{ + int x; + unsigned char *IV; + /* update IV */ + IV = prng->u.fortuna.IV; + for (x = 0; x < 16; x++) { + IV[x] = (IV[x] + 1) & 255; + if (IV[x] != 0) break; + } +} + +#ifdef LTC_FORTUNA_RESEED_RATELIMIT_TIMED +/* get the current time in 100ms steps */ +static ulong64 s_fortuna_current_time(void) +{ + ulong64 cur_time; +#if defined(_WIN32) + FILETIME CurrentTime; + ULARGE_INTEGER ul; + GetSystemTimeAsFileTime(&CurrentTime); + ul.LowPart = CurrentTime.dwLowDateTime; + ul.HighPart = CurrentTime.dwHighDateTime; + cur_time = ul.QuadPart; /* now we have 100ns intervals since 1 January 1601 */ + cur_time -= CONST64(116444736000000000); /* subtract 100ns intervals between 1601-1970 */ + cur_time /= 10; /* 100ns intervals > microseconds */ +#elif defined(LTC_CLOCK_GETTIME) + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + cur_time = (ulong64)(ts.tv_sec) * 1000000 + (ulong64)(ts.tv_nsec) / 1000; /* get microseconds */ +#else + struct timeval tv; + gettimeofday(&tv, NULL); + cur_time = (ulong64)(tv.tv_sec) * 1000000 + (ulong64)(tv.tv_usec); /* get microseconds */ +#endif + return cur_time / 100; +} +#endif + +/* reseed the PRNG */ +static int s_fortuna_reseed(prng_state *prng) +{ + unsigned char tmp[MAXBLOCKSIZE]; + hash_state md; + ulong64 reset_cnt; + int err, x; + +#ifdef LTC_FORTUNA_RESEED_RATELIMIT_TIMED + ulong64 now = s_fortuna_current_time(); + if (now == prng->u.fortuna.wd) { + return CRYPT_OK; + } +#else + if (++prng->u.fortuna.wd < LTC_FORTUNA_WD) { + return CRYPT_OK; + } +#endif + + /* new K == LTC_SHA256(K || s) where s == LTC_SHA256(P0) || LTC_SHA256(P1) ... */ + sha256_init(&md); + if ((err = sha256_process(&md, prng->u.fortuna.K, 32)) != CRYPT_OK) { + sha256_done(&md, tmp); + return err; + } + + reset_cnt = prng->u.fortuna.reset_cnt + 1; + + for (x = 0; x < LTC_FORTUNA_POOLS; x++) { + if (x == 0 || ((reset_cnt >> (x-1)) & 1) == 0) { + /* terminate this hash */ + if ((err = sha256_done(&prng->u.fortuna.pool[x], tmp)) != CRYPT_OK) { + sha256_done(&md, tmp); + return err; + } + /* add it to the string */ + if ((err = sha256_process(&md, tmp, 32)) != CRYPT_OK) { + sha256_done(&md, tmp); + return err; + } + /* reset this pool */ + if ((err = sha256_init(&prng->u.fortuna.pool[x])) != CRYPT_OK) { + sha256_done(&md, tmp); + return err; + } + } else { + break; + } + } + + /* finish key */ + if ((err = sha256_done(&md, prng->u.fortuna.K)) != CRYPT_OK) { + return err; + } + if ((err = rijndael_setup(prng->u.fortuna.K, 32, 0, &prng->u.fortuna.skey)) != CRYPT_OK) { + return err; + } + s_fortuna_update_iv(prng); + + /* reset/update internals */ + prng->u.fortuna.pool0_len = 0; +#ifdef LTC_FORTUNA_RESEED_RATELIMIT_TIMED + prng->u.fortuna.wd = now; +#else + prng->u.fortuna.wd = 0; +#endif + prng->u.fortuna.reset_cnt = reset_cnt; + + +#ifdef LTC_CLEAN_STACK + zeromem(&md, sizeof(md)); + zeromem(tmp, sizeof(tmp)); +#endif + + return CRYPT_OK; +} + +/** + "Update Seed File"-compliant update of K + + @param in The PRNG state + @param inlen Size of the state + @param prng The PRNG to import + @return CRYPT_OK if successful +*/ +int fortuna_update_seed(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + unsigned char tmp[MAXBLOCKSIZE]; + hash_state md; + + LTC_MUTEX_LOCK(&prng->lock); + /* new K = LTC_SHA256(K || in) */ + sha256_init(&md); + if ((err = sha256_process(&md, prng->u.fortuna.K, 32)) != CRYPT_OK) { + sha256_done(&md, tmp); + goto LBL_UNLOCK; + } + if ((err = sha256_process(&md, in, inlen)) != CRYPT_OK) { + sha256_done(&md, tmp); + goto LBL_UNLOCK; + } + /* finish key */ + if ((err = sha256_done(&md, prng->u.fortuna.K)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + s_fortuna_update_iv(prng); + +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); +#ifdef LTC_CLEAN_STACK + zeromem(&md, sizeof(md)); +#endif + + return err; +} + +/** + Start the PRNG + @param prng [out] The PRNG state to initialize + @return CRYPT_OK if successful +*/ +int fortuna_start(prng_state *prng) +{ + int err, x, y; + unsigned char tmp[MAXBLOCKSIZE]; + + LTC_ARGCHK(prng != NULL); + prng->ready = 0; + + /* initialize the pools */ + for (x = 0; x < LTC_FORTUNA_POOLS; x++) { + if ((err = sha256_init(&prng->u.fortuna.pool[x])) != CRYPT_OK) { + for (y = 0; y < x; y++) { + sha256_done(&prng->u.fortuna.pool[y], tmp); + } + return err; + } + } + prng->u.fortuna.pool_idx = prng->u.fortuna.pool0_len = 0; + prng->u.fortuna.reset_cnt = prng->u.fortuna.wd = 0; + + /* reset bufs */ + zeromem(prng->u.fortuna.K, 32); + if ((err = rijndael_setup(prng->u.fortuna.K, 32, 0, &prng->u.fortuna.skey)) != CRYPT_OK) { + for (x = 0; x < LTC_FORTUNA_POOLS; x++) { + sha256_done(&prng->u.fortuna.pool[x], tmp); + } + return err; + } + zeromem(prng->u.fortuna.IV, 16); + + LTC_MUTEX_INIT(&prng->lock) + + return CRYPT_OK; +} + +static int s_fortuna_add(unsigned long source, unsigned long pool, const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + unsigned char tmp[2]; + int err; + + /* ensure inlen <= 32 */ + if (inlen > 32) { + inlen = 32; + } + + /* add s || length(in) || in to pool[pool_idx] */ + tmp[0] = (unsigned char)source; + tmp[1] = (unsigned char)inlen; + + if ((err = sha256_process(&prng->u.fortuna.pool[pool], tmp, 2)) != CRYPT_OK) { + return err; + } + if ((err = sha256_process(&prng->u.fortuna.pool[pool], in, inlen)) != CRYPT_OK) { + return err; + } + if (pool == 0) { + prng->u.fortuna.pool0_len += inlen; + } + return CRYPT_OK; /* success */ +} + +/** + Add random event to the PRNG state as proposed by the original paper. + @param source The source this random event comes from (0 .. 255) + @param pool The pool where to add the data to (0 .. LTC_FORTUNA_POOLS) + @param in The data to add + @param inlen Length of the data to add + @param prng PRNG state to update + @return CRYPT_OK if successful +*/ +int fortuna_add_random_event(unsigned long source, unsigned long pool, const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen > 0); + LTC_ARGCHK(source <= 255); + LTC_ARGCHK(pool < LTC_FORTUNA_POOLS); + + LTC_MUTEX_LOCK(&prng->lock); + + err = s_fortuna_add(source, pool, in, inlen, prng); + + LTC_MUTEX_UNLOCK(&prng->lock); + + return err; +} + +/** + Add entropy to the PRNG state + @param in The data to add + @param inlen Length of the data to add + @param prng PRNG state to update + @return CRYPT_OK if successful +*/ +int fortuna_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen > 0); + + LTC_MUTEX_LOCK(&prng->lock); + + err = s_fortuna_add(0, prng->u.fortuna.pool_idx, in, inlen, prng); + + if (err == CRYPT_OK) { + ++(prng->u.fortuna.pool_idx); + prng->u.fortuna.pool_idx %= LTC_FORTUNA_POOLS; + } + + LTC_MUTEX_UNLOCK(&prng->lock); + + return err; +} + +/** + Make the PRNG ready to read from + @param prng The PRNG to make active + @return CRYPT_OK if successful +*/ +int fortuna_ready(prng_state *prng) +{ + int err; + LTC_ARGCHK(prng != NULL); + + LTC_MUTEX_LOCK(&prng->lock); + /* make sure the reseed doesn't fail because + * of the chosen rate limit */ +#ifdef LTC_FORTUNA_RESEED_RATELIMIT_TIMED + prng->u.fortuna.wd = s_fortuna_current_time() - 1; +#else + prng->u.fortuna.wd = LTC_FORTUNA_WD; +#endif + err = s_fortuna_reseed(prng); + prng->ready = (err == CRYPT_OK) ? 1 : 0; + + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Read from the PRNG + @param out Destination + @param outlen Length of output + @param prng The active PRNG to read from + @return Number of octets read +*/ +unsigned long fortuna_read(unsigned char *out, unsigned long outlen, prng_state *prng) +{ + unsigned char tmp[16]; + unsigned long tlen = 0; + + if (outlen == 0 || prng == NULL || out == NULL) return 0; + + LTC_MUTEX_LOCK(&prng->lock); + + if (!prng->ready) { + goto LBL_UNLOCK; + } + + /* do we have to reseed? */ + if (prng->u.fortuna.pool0_len >= 64) { + if (s_fortuna_reseed(prng) != CRYPT_OK) { + goto LBL_UNLOCK; + } + } + + /* ensure that one reseed happened before allowing to read */ + if (prng->u.fortuna.reset_cnt == 0) { + goto LBL_UNLOCK; + } + + /* now generate the blocks required */ + tlen = outlen; + + /* handle whole blocks without the extra XMEMCPY */ + while (outlen >= 16) { + /* encrypt the IV and store it */ + rijndael_ecb_encrypt(prng->u.fortuna.IV, out, &prng->u.fortuna.skey); + out += 16; + outlen -= 16; + s_fortuna_update_iv(prng); + } + + /* left over bytes? */ + if (outlen > 0) { + rijndael_ecb_encrypt(prng->u.fortuna.IV, tmp, &prng->u.fortuna.skey); + XMEMCPY(out, tmp, outlen); + s_fortuna_update_iv(prng); + } + + /* generate new key */ + rijndael_ecb_encrypt(prng->u.fortuna.IV, prng->u.fortuna.K , &prng->u.fortuna.skey); + s_fortuna_update_iv(prng); + + rijndael_ecb_encrypt(prng->u.fortuna.IV, prng->u.fortuna.K+16, &prng->u.fortuna.skey); + s_fortuna_update_iv(prng); + + if (rijndael_setup(prng->u.fortuna.K, 32, 0, &prng->u.fortuna.skey) != CRYPT_OK) { + tlen = 0; + } + +LBL_UNLOCK: +#ifdef LTC_CLEAN_STACK + zeromem(tmp, sizeof(tmp)); +#endif + LTC_MUTEX_UNLOCK(&prng->lock); + return tlen; +} + +/** + Terminate the PRNG + @param prng The PRNG to terminate + @return CRYPT_OK if successful +*/ +int fortuna_done(prng_state *prng) +{ + int err, x; + unsigned char tmp[32]; + + LTC_ARGCHK(prng != NULL); + + LTC_MUTEX_LOCK(&prng->lock); + prng->ready = 0; + + /* terminate all the hashes */ + for (x = 0; x < LTC_FORTUNA_POOLS; x++) { + if ((err = sha256_done(&(prng->u.fortuna.pool[x]), tmp)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + } + /* call cipher done when we invent one ;-) */ + err = CRYPT_OK; /* success */ + +LBL_UNLOCK: +#ifdef LTC_CLEAN_STACK + zeromem(tmp, sizeof(tmp)); +#endif + LTC_MUTEX_UNLOCK(&prng->lock); + LTC_MUTEX_DESTROY(&prng->lock); + return err; +} + +/** + Export the PRNG state + @param out [out] Destination + @param outlen [in/out] Max size and resulting size of the state + @param prng The PRNG to export + @return CRYPT_OK if successful +*/ +LTC_PRNG_EXPORT(fortuna) + +/** + Import a PRNG state + @param in The PRNG state + @param inlen Size of the state + @param prng The PRNG to import + @return CRYPT_OK if successful +*/ +int fortuna_import(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(prng != NULL); + + if (inlen < (unsigned long)fortuna_desc.export_size) { + return CRYPT_INVALID_ARG; + } + + if ((err = fortuna_start(prng)) != CRYPT_OK) { + return err; + } + + if ((err = fortuna_update_seed(in, inlen, prng)) != CRYPT_OK) { + return err; + } + + return err; +} + +/** + PRNG self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled +*/ +int fortuna_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + int err; + + if ((err = sha256_test()) != CRYPT_OK) { + return err; + } + return rijndael_test(); +#endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/rc4.c b/optee_os/core/lib/libtomcrypt/src/prngs/rc4.c new file mode 100644 index 0000000..edcd73f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/rc4.c @@ -0,0 +1,221 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file prngs/rc4.c + RC4 PRNG, Tom St Denis +*/ + +#ifdef LTC_RC4 + +const struct ltc_prng_descriptor rc4_desc = +{ + "rc4", + 32, + &rc4_start, + &rc4_add_entropy, + &rc4_ready, + &rc4_read, + &rc4_done, + &rc4_export, + &rc4_import, + &rc4_test +}; + +/** + Start the PRNG + @param prng [out] The PRNG state to initialize + @return CRYPT_OK if successful +*/ +int rc4_start(prng_state *prng) +{ + LTC_ARGCHK(prng != NULL); + prng->ready = 0; + /* set entropy (key) size to zero */ + prng->u.rc4.s.x = 0; + /* clear entropy (key) buffer */ + XMEMSET(&prng->u.rc4.s.buf, 0, sizeof(prng->u.rc4.s.buf)); + LTC_MUTEX_INIT(&prng->lock) + return CRYPT_OK; +} + +/** + Add entropy to the PRNG state + @param in The data to add + @param inlen Length of the data to add + @param prng PRNG state to update + @return CRYPT_OK if successful +*/ +int rc4_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + unsigned char buf[256]; + unsigned long i; + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen > 0); + + LTC_MUTEX_LOCK(&prng->lock); + if (prng->ready) { + /* rc4_ready() was already called, do "rekey" operation */ + if ((err = rc4_stream_keystream(&prng->u.rc4.s, buf, sizeof(buf))) != CRYPT_OK) goto LBL_UNLOCK; + for(i = 0; i < inlen; i++) buf[i % sizeof(buf)] ^= in[i]; + /* initialize RC4 */ + if ((err = rc4_stream_setup(&prng->u.rc4.s, buf, sizeof(buf))) != CRYPT_OK) goto LBL_UNLOCK; + /* drop first 3072 bytes - https://en.wikipedia.org/wiki/RC4#Fluhrer.2C_Mantin_and_Shamir_attack */ + for (i = 0; i < 12; i++) rc4_stream_keystream(&prng->u.rc4.s, buf, sizeof(buf)); + zeromem(buf, sizeof(buf)); + } + else { + /* rc4_ready() was not called yet, add entropy to the buffer */ + while (inlen--) prng->u.rc4.s.buf[prng->u.rc4.s.x++ % sizeof(prng->u.rc4.s.buf)] ^= *in++; + } + err = CRYPT_OK; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Make the PRNG ready to read from + @param prng The PRNG to make active + @return CRYPT_OK if successful +*/ +int rc4_ready(prng_state *prng) +{ + unsigned char buf[256] = { 0 }; + unsigned long len; + int err, i; + + LTC_ARGCHK(prng != NULL); + + LTC_MUTEX_LOCK(&prng->lock); + if (prng->ready) { err = CRYPT_OK; goto LBL_UNLOCK; } + XMEMCPY(buf, prng->u.rc4.s.buf, sizeof(buf)); + /* initialize RC4 */ + len = MIN(prng->u.rc4.s.x, 256); /* TODO: we can perhaps always use all 256 bytes */ + if ((err = rc4_stream_setup(&prng->u.rc4.s, buf, len)) != CRYPT_OK) goto LBL_UNLOCK; + /* drop first 3072 bytes - https://en.wikipedia.org/wiki/RC4#Fluhrer.2C_Mantin_and_Shamir_attack */ + for (i = 0; i < 12; i++) rc4_stream_keystream(&prng->u.rc4.s, buf, sizeof(buf)); + prng->ready = 1; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Read from the PRNG + @param out Destination + @param outlen Length of output + @param prng The active PRNG to read from + @return Number of octets read +*/ +unsigned long rc4_read(unsigned char *out, unsigned long outlen, prng_state *prng) +{ + if (outlen == 0 || prng == NULL || out == NULL) return 0; + LTC_MUTEX_LOCK(&prng->lock); + if (!prng->ready) { outlen = 0; goto LBL_UNLOCK; } + if (rc4_stream_keystream(&prng->u.rc4.s, out, outlen) != CRYPT_OK) outlen = 0; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return outlen; +} + +/** + Terminate the PRNG + @param prng The PRNG to terminate + @return CRYPT_OK if successful +*/ +int rc4_done(prng_state *prng) +{ + int err; + LTC_ARGCHK(prng != NULL); + LTC_MUTEX_LOCK(&prng->lock); + prng->ready = 0; + err = rc4_stream_done(&prng->u.rc4.s); + LTC_MUTEX_UNLOCK(&prng->lock); + LTC_MUTEX_DESTROY(&prng->lock); + return err; +} + +/** + Export the PRNG state + @param out [out] Destination + @param outlen [in/out] Max size and resulting size of the state + @param prng The PRNG to export + @return CRYPT_OK if successful +*/ +LTC_PRNG_EXPORT(rc4) + +/** + Import a PRNG state + @param in The PRNG state + @param inlen Size of the state + @param prng The PRNG to import + @return CRYPT_OK if successful +*/ +int rc4_import(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + if (inlen < (unsigned long)rc4_desc.export_size) return CRYPT_INVALID_ARG; + + if ((err = rc4_start(prng)) != CRYPT_OK) return err; + if ((err = rc4_add_entropy(in, inlen, prng)) != CRYPT_OK) return err; + return CRYPT_OK; +} + +/** + PRNG self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled +*/ +int rc4_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + prng_state st; + unsigned char en[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, + 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32 }; + unsigned char dmp[500]; + unsigned long dmplen = sizeof(dmp); + unsigned char out[1000]; + unsigned char t1[] = { 0xE0, 0x4D, 0x9A, 0xF6, 0xA8, 0x9D, 0x77, 0x53, 0xAE, 0x09 }; + unsigned char t2[] = { 0xEF, 0x80, 0xA2, 0xE6, 0x50, 0x91, 0xF3, 0x17, 0x4A, 0x8A }; + unsigned char t3[] = { 0x4B, 0xD6, 0x5C, 0x67, 0x99, 0x03, 0x56, 0x12, 0x80, 0x48 }; + int err; + + if ((err = rc4_start(&st)) != CRYPT_OK) return err; + /* add entropy to uninitialized prng */ + if ((err = rc4_add_entropy(en, sizeof(en), &st)) != CRYPT_OK) return err; + if ((err = rc4_ready(&st)) != CRYPT_OK) return err; + if (rc4_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t1, sizeof(t1), "RC4-PRNG", 1)) return CRYPT_FAIL_TESTVECTOR; + if (rc4_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + /* add entropy to already initialized prng */ + if ((err = rc4_add_entropy(en, sizeof(en), &st)) != CRYPT_OK) return err; + if (rc4_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if ((err = rc4_export(dmp, &dmplen, &st)) != CRYPT_OK) return err; + if (rc4_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if (rc4_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t2, sizeof(t2), "RC4-PRNG", 2)) return CRYPT_FAIL_TESTVECTOR; + if ((err = rc4_done(&st)) != CRYPT_OK) return err; + if ((err = rc4_import(dmp, dmplen, &st)) != CRYPT_OK) return err; + if ((err = rc4_ready(&st)) != CRYPT_OK) return err; + if (rc4_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if (rc4_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t3, sizeof(t3), "RC4-PRNG", 3)) return CRYPT_FAIL_TESTVECTOR; + if ((err = rc4_done(&st)) != CRYPT_OK) return err; + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/rng_get_bytes.c b/optee_os/core/lib/libtomcrypt/src/prngs/rng_get_bytes.c new file mode 100644 index 0000000..b07238e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/rng_get_bytes.c @@ -0,0 +1,152 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_RNG_GET_BYTES +/** + @file rng_get_bytes.c + portable way to get secure random bits to feed a PRNG (Tom St Denis) +*/ + +#if defined(LTC_DEVRANDOM) && !defined(_WIN32) +/* on *NIX read /dev/random */ +static unsigned long s_rng_nix(unsigned char *buf, unsigned long len, + void (*callback)(void)) +{ +#ifdef LTC_NO_FILE + LTC_UNUSED_PARAM(callback); + LTC_UNUSED_PARAM(buf); + LTC_UNUSED_PARAM(len); + return 0; +#else + FILE *f; + unsigned long x; + LTC_UNUSED_PARAM(callback); +#ifdef LTC_TRY_URANDOM_FIRST + f = fopen("/dev/urandom", "rb"); + if (f == NULL) { + f = fopen("/dev/random", "rb"); + } +#else + f = fopen("/dev/random", "rb"); +#endif /* LTC_TRY_URANDOM_FIRST */ + + if (f == NULL) { + return 0; + } + + /* disable buffering */ + if (setvbuf(f, NULL, _IONBF, 0) != 0) { + fclose(f); + return 0; + } + + x = (unsigned long)fread(buf, 1, (size_t)len, f); + fclose(f); + return x; +#endif /* LTC_NO_FILE */ +} + +#endif /* LTC_DEVRANDOM */ + +#if !defined(_WIN32_WCE) + +#define ANSI_RNG + +static unsigned long s_rng_ansic(unsigned char *buf, unsigned long len, + void (*callback)(void)) +{ + clock_t t1; + int l, acc, bits, a, b; + + l = len; + bits = 8; + acc = a = b = 0; + while (len--) { + if (callback != NULL) callback(); + while (bits--) { + do { + t1 = XCLOCK(); while (t1 == XCLOCK()) a ^= 1; + t1 = XCLOCK(); while (t1 == XCLOCK()) b ^= 1; + } while (a == b); + acc = (acc << 1) | a; + } + *buf++ = acc; + acc = 0; + bits = 8; + } + return l; +} + +#endif + +/* Try the Microsoft CSP */ +#if defined(_WIN32) || defined(_WIN32_WCE) +#ifndef _WIN32_WINNT + #define _WIN32_WINNT 0x0400 +#endif +#ifdef _WIN32_WCE + #define UNDER_CE + #define ARM +#endif + +#define WIN32_LEAN_AND_MEAN +#include +#include + +static unsigned long s_rng_win32(unsigned char *buf, unsigned long len, + void (*callback)(void)) +{ + HCRYPTPROV hProv = 0; + LTC_UNUSED_PARAM(callback); + if (!CryptAcquireContext(&hProv, NULL, MS_DEF_PROV, PROV_RSA_FULL, + (CRYPT_VERIFYCONTEXT | CRYPT_MACHINE_KEYSET)) && + !CryptAcquireContext (&hProv, NULL, MS_DEF_PROV, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT | CRYPT_MACHINE_KEYSET | CRYPT_NEWKEYSET)) + return 0; + + if (CryptGenRandom(hProv, len, buf) == TRUE) { + CryptReleaseContext(hProv, 0); + return len; + } else { + CryptReleaseContext(hProv, 0); + return 0; + } +} + +#endif /* WIN32 */ + +/** + Read the system RNG + @param out Destination + @param outlen Length desired (octets) + @param callback Pointer to void function to act as "callback" when RNG is slow. This can be NULL + @return Number of octets read +*/ +unsigned long rng_get_bytes(unsigned char *out, unsigned long outlen, + void (*callback)(void)) +{ + unsigned long x; + + LTC_ARGCHK(out != NULL); + +#ifdef LTC_PRNG_ENABLE_LTC_RNG + if (ltc_rng) { + x = ltc_rng(out, outlen, callback); + if (x != 0) { + return x; + } + } +#endif + +#if defined(_WIN32) || defined(_WIN32_WCE) + x = s_rng_win32(out, outlen, callback); if (x != 0) { return x; } +#elif defined(LTC_DEVRANDOM) + x = s_rng_nix(out, outlen, callback); if (x != 0) { return x; } +#endif +#ifdef ANSI_RNG + x = s_rng_ansic(out, outlen, callback); if (x != 0) { return x; } +#endif + return 0; +} +#endif /* #ifdef LTC_RNG_GET_BYTES */ diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/rng_make_prng.c b/optee_os/core/lib/libtomcrypt/src/prngs/rng_make_prng.c new file mode 100644 index 0000000..e70c1cd --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/rng_make_prng.c @@ -0,0 +1,81 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +#ifdef LTC_RNG_MAKE_PRNG +/** + @file rng_make_prng.c + portable way to get secure random bits to feed a PRNG (Tom St Denis) +*/ + +/** + Create a PRNG from a RNG + + In case you pass bits as '-1' the PRNG will be setup + as if the export/import functionality has been used, + but the imported data comes directly from the RNG. + + @param bits Number of bits of entropy desired (-1 or 64 ... 1024) + @param wprng Index of which PRNG to setup + @param prng [out] PRNG state to initialize + @param callback A pointer to a void function for when the RNG is slow, this can be NULL + @return CRYPT_OK if successful +*/ +int rng_make_prng(int bits, int wprng, prng_state *prng, + void (*callback)(void)) +{ + unsigned char* buf; + unsigned long bytes; + int err; + + LTC_ARGCHK(prng != NULL); + + /* check parameter */ + if ((err = prng_is_valid(wprng)) != CRYPT_OK) { + return err; + } + + if (bits == -1) { + bytes = prng_descriptor[wprng]->export_size; + } else if (bits < 64 || bits > 1024) { + return CRYPT_INVALID_PRNGSIZE; + } else { + bytes = (unsigned long)((bits+7)/8) * 2; + } + + if ((err = prng_descriptor[wprng]->start(prng)) != CRYPT_OK) { + return err; + } + + buf = XMALLOC(bytes); + if (buf == NULL) { + return CRYPT_MEM; + } + + if (rng_get_bytes(buf, bytes, callback) != bytes) { + err = CRYPT_ERROR_READPRNG; + goto LBL_ERR; + } + + if (bits == -1) { + if ((err = prng_descriptor[wprng]->pimport(buf, bytes, prng)) != CRYPT_OK) { + goto LBL_ERR; + } + } else { + if ((err = prng_descriptor[wprng]->add_entropy(buf, bytes, prng)) != CRYPT_OK) { + goto LBL_ERR; + } + } + if ((err = prng_descriptor[wprng]->ready(prng)) != CRYPT_OK) { + goto LBL_ERR; + } + +LBL_ERR: + #ifdef LTC_CLEAN_STACK + zeromem(buf, bytes); + #endif + XFREE(buf); + return err; +} +#endif /* #ifdef LTC_RNG_MAKE_PRNG */ + diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/sober128.c b/optee_os/core/lib/libtomcrypt/src/prngs/sober128.c new file mode 100644 index 0000000..6454578 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/sober128.c @@ -0,0 +1,220 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +/** + @file prngs/sober128.c + Implementation of SOBER-128 by Tom St Denis. + Based on s128fast.c reference code supplied by Greg Rose of QUALCOMM. +*/ + +#ifdef LTC_SOBER128 + +const struct ltc_prng_descriptor sober128_desc = +{ + "sober128", + 40, + &sober128_start, + &sober128_add_entropy, + &sober128_ready, + &sober128_read, + &sober128_done, + &sober128_export, + &sober128_import, + &sober128_test +}; + +/** + Start the PRNG + @param prng [out] The PRNG state to initialize + @return CRYPT_OK if successful +*/ +int sober128_start(prng_state *prng) +{ + LTC_ARGCHK(prng != NULL); + prng->ready = 0; + XMEMSET(&prng->u.sober128.ent, 0, sizeof(prng->u.sober128.ent)); + prng->u.sober128.idx = 0; + LTC_MUTEX_INIT(&prng->lock) + return CRYPT_OK; +} + +/** + Add entropy to the PRNG state + @param in The data to add + @param inlen Length of the data to add + @param prng PRNG state to update + @return CRYPT_OK if successful +*/ +int sober128_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + unsigned char buf[40]; + unsigned long i; + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen > 0); + + LTC_MUTEX_LOCK(&prng->lock); + if (prng->ready) { + /* sober128_ready() was already called, do "rekey" operation */ + if ((err = sober128_stream_keystream(&prng->u.sober128.s, buf, sizeof(buf))) != CRYPT_OK) goto LBL_UNLOCK; + for(i = 0; i < inlen; i++) buf[i % sizeof(buf)] ^= in[i]; + /* key 32 bytes, 20 rounds */ + if ((err = sober128_stream_setup(&prng->u.sober128.s, buf, 32)) != CRYPT_OK) goto LBL_UNLOCK; + /* iv 8 bytes */ + if ((err = sober128_stream_setiv(&prng->u.sober128.s, buf + 32, 8)) != CRYPT_OK) goto LBL_UNLOCK; + /* clear KEY + IV */ + zeromem(buf, sizeof(buf)); + } + else { + /* sober128_ready() was not called yet, add entropy to ent buffer */ + while (inlen--) prng->u.sober128.ent[prng->u.sober128.idx++ % sizeof(prng->u.sober128.ent)] ^= *in++; + } + err = CRYPT_OK; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Make the PRNG ready to read from + @param prng The PRNG to make active + @return CRYPT_OK if successful +*/ +int sober128_ready(prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + + LTC_MUTEX_LOCK(&prng->lock); + if (prng->ready) { err = CRYPT_OK; goto LBL_UNLOCK; } + /* key 32 bytes, 20 rounds */ + if ((err = sober128_stream_setup(&prng->u.sober128.s, prng->u.sober128.ent, 32)) != CRYPT_OK) goto LBL_UNLOCK; + /* iv 8 bytes */ + if ((err = sober128_stream_setiv(&prng->u.sober128.s, prng->u.sober128.ent + 32, 8)) != CRYPT_OK) goto LBL_UNLOCK; + XMEMSET(&prng->u.sober128.ent, 0, sizeof(prng->u.sober128.ent)); + prng->u.sober128.idx = 0; + prng->ready = 1; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Read from the PRNG + @param out Destination + @param outlen Length of output + @param prng The active PRNG to read from + @return Number of octets read +*/ +unsigned long sober128_read(unsigned char *out, unsigned long outlen, prng_state *prng) +{ + if (outlen == 0 || prng == NULL || out == NULL) return 0; + LTC_MUTEX_LOCK(&prng->lock); + if (!prng->ready) { outlen = 0; goto LBL_UNLOCK; } + if (sober128_stream_keystream(&prng->u.sober128.s, out, outlen) != CRYPT_OK) outlen = 0; +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return outlen; +} + +/** + Terminate the PRNG + @param prng The PRNG to terminate + @return CRYPT_OK if successful +*/ +int sober128_done(prng_state *prng) +{ + int err; + LTC_ARGCHK(prng != NULL); + LTC_MUTEX_LOCK(&prng->lock); + prng->ready = 0; + err = sober128_stream_done(&prng->u.sober128.s); + LTC_MUTEX_UNLOCK(&prng->lock); + LTC_MUTEX_DESTROY(&prng->lock); + return err; +} + +/** + Export the PRNG state + @param out [out] Destination + @param outlen [in/out] Max size and resulting size of the state + @param prng The PRNG to export + @return CRYPT_OK if successful +*/ +LTC_PRNG_EXPORT(sober128) + +/** + Import a PRNG state + @param in The PRNG state + @param inlen Size of the state + @param prng The PRNG to import + @return CRYPT_OK if successful +*/ +int sober128_import(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + if (inlen < (unsigned long)sober128_desc.export_size) return CRYPT_INVALID_ARG; + + if ((err = sober128_start(prng)) != CRYPT_OK) return err; + if ((err = sober128_add_entropy(in, inlen, prng)) != CRYPT_OK) return err; + return CRYPT_OK; +} + +/** + PRNG self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled +*/ +int sober128_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + prng_state st; + unsigned char en[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, + 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32 }; + unsigned char dmp[300]; + unsigned long dmplen = sizeof(dmp); + unsigned char out[500]; + unsigned char t1[] = { 0x31, 0x82, 0xA7, 0xA5, 0x8B, 0xD7, 0xCB, 0x39, 0x86, 0x1A }; + unsigned char t2[] = { 0x6B, 0x43, 0x9E, 0xBC, 0xE7, 0x62, 0x9B, 0xE6, 0x9B, 0x83 }; + unsigned char t3[] = { 0x4A, 0x0E, 0x6C, 0xC1, 0xCF, 0xB4, 0x73, 0x49, 0x99, 0x05 }; + int err; + + if ((err = sober128_start(&st)) != CRYPT_OK) return err; + /* add entropy to uninitialized prng */ + if ((err = sober128_add_entropy(en, sizeof(en), &st)) != CRYPT_OK) return err; + if ((err = sober128_ready(&st)) != CRYPT_OK) return err; + if (sober128_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t1, sizeof(t1), "SOBER128-PRNG", 1)) return CRYPT_FAIL_TESTVECTOR; + if (sober128_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + /* add entropy to already initialized prng */ + if ((err = sober128_add_entropy(en, sizeof(en), &st)) != CRYPT_OK) return err; + if (sober128_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if ((err = sober128_export(dmp, &dmplen, &st)) != CRYPT_OK) return err; + if (sober128_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if (sober128_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t2, sizeof(t2), "SOBER128-PRNG", 2)) return CRYPT_FAIL_TESTVECTOR; + if ((err = sober128_done(&st)) != CRYPT_OK) return err; + if ((err = sober128_import(dmp, dmplen, &st)) != CRYPT_OK) return err; + if ((err = sober128_ready(&st)) != CRYPT_OK) return err; + if (sober128_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if (sober128_read(out, 10, &st) != 10) return CRYPT_ERROR_READPRNG; /* 10 bytes for testing */ + if (compare_testvector(out, 10, t3, sizeof(t3), "SOBER128-PRNG", 3)) return CRYPT_FAIL_TESTVECTOR; + if ((err = sober128_done(&st)) != CRYPT_OK) return err; + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/sprng.c b/optee_os/core/lib/libtomcrypt/src/prngs/sprng.c new file mode 100644 index 0000000..e404718 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/sprng.c @@ -0,0 +1,152 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file sprng.c + Secure PRNG, Tom St Denis +*/ + +/* A secure PRNG using the RNG functions. Basically this is a + * wrapper that allows you to use a secure RNG as a PRNG + * in the various other functions. + */ + +#ifdef LTC_SPRNG + +const struct ltc_prng_descriptor sprng_desc = +{ + "sprng", 0, + &sprng_start, + &sprng_add_entropy, + &sprng_ready, + &sprng_read, + &sprng_done, + &sprng_export, + &sprng_import, + &sprng_test +}; + +/** + Start the PRNG + @param prng [out] The PRNG state to initialize + @return CRYPT_OK if successful +*/ +int sprng_start(prng_state *prng) +{ + LTC_UNUSED_PARAM(prng); + return CRYPT_OK; +} + +/** + Add entropy to the PRNG state + @param in The data to add + @param inlen Length of the data to add + @param prng PRNG state to update + @return CRYPT_OK if successful +*/ +int sprng_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + LTC_UNUSED_PARAM(in); + LTC_UNUSED_PARAM(inlen); + LTC_UNUSED_PARAM(prng); + return CRYPT_OK; +} + +/** + Make the PRNG ready to read from + @param prng The PRNG to make active + @return CRYPT_OK if successful +*/ +int sprng_ready(prng_state *prng) +{ + LTC_UNUSED_PARAM(prng); + return CRYPT_OK; +} + +/** + Read from the PRNG + @param out Destination + @param outlen Length of output + @param prng The active PRNG to read from + @return Number of octets read +*/ +unsigned long sprng_read(unsigned char *out, unsigned long outlen, prng_state *prng) +{ + LTC_ARGCHK(out != NULL); + LTC_UNUSED_PARAM(prng); + return rng_get_bytes(out, outlen, NULL); +} + +/** + Terminate the PRNG + @param prng The PRNG to terminate + @return CRYPT_OK if successful +*/ +int sprng_done(prng_state *prng) +{ + LTC_UNUSED_PARAM(prng); + return CRYPT_OK; +} + +/** + Export the PRNG state + @param out [out] Destination + @param outlen [in/out] Max size and resulting size of the state + @param prng The PRNG to export + @return CRYPT_OK if successful +*/ +/* NOLINTNEXTLINE(readability-non-const-parameter) - silence clang-tidy warning */ +int sprng_export(unsigned char *out, unsigned long *outlen, prng_state *prng) +{ + LTC_ARGCHK(outlen != NULL); + LTC_UNUSED_PARAM(out); + LTC_UNUSED_PARAM(prng); + + *outlen = 0; + return CRYPT_OK; +} + +/** + Import a PRNG state + @param in The PRNG state + @param inlen Size of the state + @param prng The PRNG to import + @return CRYPT_OK if successful +*/ +int sprng_import(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + LTC_UNUSED_PARAM(in); + LTC_UNUSED_PARAM(inlen); + LTC_UNUSED_PARAM(prng); + return CRYPT_OK; +} + +/** + PRNG self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled +*/ +int sprng_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + prng_state st; + unsigned char en[] = { 0x01, 0x02, 0x03, 0x04 }; + unsigned char out[1000]; + int err; + + if ((err = sprng_start(&st)) != CRYPT_OK) return err; + if ((err = sprng_add_entropy(en, sizeof(en), &st)) != CRYPT_OK) return err; + if ((err = sprng_ready(&st)) != CRYPT_OK) return err; + if (sprng_read(out, 500, &st) != 500) return CRYPT_ERROR_READPRNG; /* skip 500 bytes */ + if ((err = sprng_done(&st)) != CRYPT_OK) return err; + + return CRYPT_OK; +#endif +} + +#endif + + + diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/sub.mk b/optee_os/core/lib/libtomcrypt/src/prngs/sub.mk new file mode 100644 index 0000000..d8a8e51 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/sub.mk @@ -0,0 +1,7 @@ +cflags-y += -Wno-unused-parameter -Wno-unused-variable + +srcs-y += rng_get_bytes.c +srcs-y += rng_make_prng.c +srcs-y += sprng.c +srcs-y += rc4.c +srcs-$(_CFG_CORE_LTC_FORTUNA_PRNG) += fortuna.c diff --git a/optee_os/core/lib/libtomcrypt/src/prngs/yarrow.c b/optee_os/core/lib/libtomcrypt/src/prngs/yarrow.c new file mode 100644 index 0000000..6131043 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/prngs/yarrow.c @@ -0,0 +1,323 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file yarrow.c + Yarrow PRNG, Tom St Denis +*/ + +#ifdef LTC_YARROW + +const struct ltc_prng_descriptor yarrow_desc = +{ + "yarrow", 64, + &yarrow_start, + &yarrow_add_entropy, + &yarrow_ready, + &yarrow_read, + &yarrow_done, + &yarrow_export, + &yarrow_import, + &yarrow_test +}; + +/** + Start the PRNG + @param prng [out] The PRNG state to initialize + @return CRYPT_OK if successful +*/ +int yarrow_start(prng_state *prng) +{ + int err; + + LTC_ARGCHK(prng != NULL); + prng->ready = 0; + + /* these are the default hash/cipher combo used */ +#ifdef LTC_RIJNDAEL +#if LTC_YARROW_AES==0 + prng->u.yarrow.cipher = register_cipher(&rijndael_enc_desc); +#elif LTC_YARROW_AES==1 + prng->u.yarrow.cipher = register_cipher(&aes_enc_desc); +#elif LTC_YARROW_AES==2 + prng->u.yarrow.cipher = register_cipher(&rijndael_desc); +#elif LTC_YARROW_AES==3 + prng->u.yarrow.cipher = register_cipher(&aes_desc); +#endif +#elif defined(LTC_BLOWFISH) + prng->u.yarrow.cipher = register_cipher(&blowfish_desc); +#elif defined(LTC_TWOFISH) + prng->u.yarrow.cipher = register_cipher(&twofish_desc); +#elif defined(LTC_RC6) + prng->u.yarrow.cipher = register_cipher(&rc6_desc); +#elif defined(LTC_RC5) + prng->u.yarrow.cipher = register_cipher(&rc5_desc); +#elif defined(LTC_SAFERP) + prng->u.yarrow.cipher = register_cipher(&saferp_desc); +#elif defined(LTC_RC2) + prng->u.yarrow.cipher = register_cipher(&rc2_desc); +#elif defined(LTC_NOEKEON) + prng->u.yarrow.cipher = register_cipher(&noekeon_desc); +#elif defined(LTC_ANUBIS) + prng->u.yarrow.cipher = register_cipher(&anubis_desc); +#elif defined(LTC_KSEED) + prng->u.yarrow.cipher = register_cipher(&kseed_desc); +#elif defined(LTC_KHAZAD) + prng->u.yarrow.cipher = register_cipher(&khazad_desc); +#elif defined(LTC_CAST5) + prng->u.yarrow.cipher = register_cipher(&cast5_desc); +#elif defined(LTC_XTEA) + prng->u.yarrow.cipher = register_cipher(&xtea_desc); +#elif defined(LTC_SAFER) + prng->u.yarrow.cipher = register_cipher(&safer_sk128_desc); +#elif defined(LTC_DES) + prng->u.yarrow.cipher = register_cipher(&des3_desc); +#else + #error LTC_YARROW needs at least one CIPHER +#endif + if ((err = cipher_is_valid(prng->u.yarrow.cipher)) != CRYPT_OK) { + return err; + } + +#ifdef LTC_SHA256 + prng->u.yarrow.hash = register_hash(&sha256_desc); +#elif defined(LTC_SHA512) + prng->u.yarrow.hash = register_hash(&sha512_desc); +#elif defined(LTC_TIGER) + prng->u.yarrow.hash = register_hash(&tiger_desc); +#elif defined(LTC_SHA1) + prng->u.yarrow.hash = register_hash(&sha1_desc); +#elif defined(LTC_RIPEMD320) + prng->u.yarrow.hash = register_hash(&rmd320_desc); +#elif defined(LTC_RIPEMD256) + prng->u.yarrow.hash = register_hash(&rmd256_desc); +#elif defined(LTC_RIPEMD160) + prng->u.yarrow.hash = register_hash(&rmd160_desc); +#elif defined(LTC_RIPEMD128) + prng->u.yarrow.hash = register_hash(&rmd128_desc); +#elif defined(LTC_MD5) + prng->u.yarrow.hash = register_hash(&md5_desc); +#elif defined(LTC_MD4) + prng->u.yarrow.hash = register_hash(&md4_desc); +#elif defined(LTC_MD2) + prng->u.yarrow.hash = register_hash(&md2_desc); +#elif defined(LTC_WHIRLPOOL) + prng->u.yarrow.hash = register_hash(&whirlpool_desc); +#else + #error LTC_YARROW needs at least one HASH +#endif + if ((err = hash_is_valid(prng->u.yarrow.hash)) != CRYPT_OK) { + return err; + } + + /* zero the memory used */ + zeromem(prng->u.yarrow.pool, sizeof(prng->u.yarrow.pool)); + LTC_MUTEX_INIT(&prng->lock) + + return CRYPT_OK; +} + +/** + Add entropy to the PRNG state + @param in The data to add + @param inlen Length of the data to add + @param prng PRNG state to update + @return CRYPT_OK if successful +*/ +int yarrow_add_entropy(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + hash_state md; + int err; + + LTC_ARGCHK(prng != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(inlen > 0); + + LTC_MUTEX_LOCK(&prng->lock); + + if ((err = hash_is_valid(prng->u.yarrow.hash)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + + /* start the hash */ + if ((err = hash_descriptor[prng->u.yarrow.hash]->init(&md)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + + /* hash the current pool */ + if ((err = hash_descriptor[prng->u.yarrow.hash]->process(&md, prng->u.yarrow.pool, + hash_descriptor[prng->u.yarrow.hash]->hashsize)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + + /* add the new entropy */ + if ((err = hash_descriptor[prng->u.yarrow.hash]->process(&md, in, inlen)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + + /* store result */ + err = hash_descriptor[prng->u.yarrow.hash]->done(&md, prng->u.yarrow.pool); + +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Make the PRNG ready to read from + @param prng The PRNG to make active + @return CRYPT_OK if successful +*/ +int yarrow_ready(prng_state *prng) +{ + int ks, err; + + LTC_ARGCHK(prng != NULL); + + LTC_MUTEX_LOCK(&prng->lock); + + if ((err = hash_is_valid(prng->u.yarrow.hash)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + + if ((err = cipher_is_valid(prng->u.yarrow.cipher)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + + /* setup CTR mode using the "pool" as the key */ + ks = (int)hash_descriptor[prng->u.yarrow.hash]->hashsize; + if ((err = cipher_descriptor[prng->u.yarrow.cipher]->keysize(&ks)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + + if ((err = ctr_start(prng->u.yarrow.cipher, /* what cipher to use */ + prng->u.yarrow.pool, /* IV */ + prng->u.yarrow.pool, ks, /* KEY and key size */ + 0, /* number of rounds */ + CTR_COUNTER_LITTLE_ENDIAN, /* little endian counter */ + &prng->u.yarrow.ctr)) != CRYPT_OK) { + goto LBL_UNLOCK; + } + prng->ready = 1; + +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return err; +} + +/** + Read from the PRNG + @param out Destination + @param outlen Length of output + @param prng The active PRNG to read from + @return Number of octets read +*/ +unsigned long yarrow_read(unsigned char *out, unsigned long outlen, prng_state *prng) +{ + if (outlen == 0 || prng == NULL || out == NULL) return 0; + + LTC_MUTEX_LOCK(&prng->lock); + + if (!prng->ready) { + outlen = 0; + goto LBL_UNLOCK; + } + + /* put out in predictable state first */ + zeromem(out, outlen); + + /* now randomize it */ + if (ctr_encrypt(out, out, outlen, &prng->u.yarrow.ctr) != CRYPT_OK) { + outlen = 0; + } + +LBL_UNLOCK: + LTC_MUTEX_UNLOCK(&prng->lock); + return outlen; +} + +/** + Terminate the PRNG + @param prng The PRNG to terminate + @return CRYPT_OK if successful +*/ +int yarrow_done(prng_state *prng) +{ + int err; + LTC_ARGCHK(prng != NULL); + + LTC_MUTEX_LOCK(&prng->lock); + prng->ready = 0; + + /* call cipher done when we invent one ;-) */ + + /* we invented one */ + err = ctr_done(&prng->u.yarrow.ctr); + + LTC_MUTEX_UNLOCK(&prng->lock); + LTC_MUTEX_DESTROY(&prng->lock); + return err; +} + +/** + Export the PRNG state + @param out [out] Destination + @param outlen [in/out] Max size and resulting size of the state + @param prng The PRNG to export + @return CRYPT_OK if successful +*/ +LTC_PRNG_EXPORT(yarrow) + +/** + Import a PRNG state + @param in The PRNG state + @param inlen Size of the state + @param prng The PRNG to import + @return CRYPT_OK if successful +*/ +int yarrow_import(const unsigned char *in, unsigned long inlen, prng_state *prng) +{ + int err; + + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(prng != NULL); + if (inlen < (unsigned long)yarrow_desc.export_size) return CRYPT_INVALID_ARG; + + if ((err = yarrow_start(prng)) != CRYPT_OK) return err; + if ((err = yarrow_add_entropy(in, inlen, prng)) != CRYPT_OK) return err; + return CRYPT_OK; +} + +/** + PRNG self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-testing has been disabled +*/ +int yarrow_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + int err; + prng_state prng; + + if ((err = yarrow_start(&prng)) != CRYPT_OK) { + return err; + } + + /* now let's test the hash/cipher that was chosen */ + if (cipher_descriptor[prng.u.yarrow.cipher]->test && + ((err = cipher_descriptor[prng.u.yarrow.cipher]->test()) != CRYPT_OK)) { + return err; + } + if (hash_descriptor[prng.u.yarrow.hash]->test && + ((err = hash_descriptor[prng.u.yarrow.hash]->test()) != CRYPT_OK)) { + return err; + } + + return CRYPT_OK; +#endif +} + +#endif + diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_crypt.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_crypt.c new file mode 100644 index 0000000..2798800 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_crypt.c @@ -0,0 +1,91 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * chacha-ref.c version 20080118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +#define QUARTERROUND(a,b,c,d) \ + x[a] += x[b]; x[d] = ROL(x[d] ^ x[a], 16); \ + x[c] += x[d]; x[b] = ROL(x[b] ^ x[c], 12); \ + x[a] += x[b]; x[d] = ROL(x[d] ^ x[a], 8); \ + x[c] += x[d]; x[b] = ROL(x[b] ^ x[c], 7); + +static void s_chacha_block(unsigned char *output, const ulong32 *input, int rounds) +{ + ulong32 x[16]; + int i; + XMEMCPY(x, input, sizeof(x)); + for (i = rounds; i > 0; i -= 2) { + QUARTERROUND(0, 4, 8,12) + QUARTERROUND(1, 5, 9,13) + QUARTERROUND(2, 6,10,14) + QUARTERROUND(3, 7,11,15) + QUARTERROUND(0, 5,10,15) + QUARTERROUND(1, 6,11,12) + QUARTERROUND(2, 7, 8,13) + QUARTERROUND(3, 4, 9,14) + } + for (i = 0; i < 16; ++i) { + x[i] += input[i]; + STORE32L(x[i], output + 4 * i); + } +} + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with ChaCha + @param st The ChaCha state + @param in The plaintext (or ciphertext) + @param inlen The length of the input (octets) + @param out [out] The ciphertext (or plaintext), length inlen + @return CRYPT_OK if successful +*/ +int chacha_crypt(chacha_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + unsigned char buf[64]; + unsigned long i, j; + + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(st->ivlen != 0); + + if (st->ksleft > 0) { + j = MIN(st->ksleft, inlen); + for (i = 0; i < j; ++i, st->ksleft--) out[i] = in[i] ^ st->kstream[64 - st->ksleft]; + inlen -= j; + if (inlen == 0) return CRYPT_OK; + out += j; + in += j; + } + for (;;) { + s_chacha_block(buf, st->input, st->rounds); + if (st->ivlen == 8) { + /* IV-64bit, increment 64bit counter */ + if (0 == ++st->input[12] && 0 == ++st->input[13]) return CRYPT_OVERFLOW; + } + else { + /* IV-96bit, increment 32bit counter */ + if (0 == ++st->input[12]) return CRYPT_OVERFLOW; + } + if (inlen <= 64) { + for (i = 0; i < inlen; ++i) out[i] = in[i] ^ buf[i]; + st->ksleft = 64 - inlen; + for (i = inlen; i < 64; ++i) st->kstream[i] = buf[i]; + return CRYPT_OK; + } + for (i = 0; i < 64; ++i) out[i] = in[i] ^ buf[i]; + inlen -= 64; + out += 64; + in += 64; + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_done.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_done.c new file mode 100644 index 0000000..1dc641e --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_done.c @@ -0,0 +1,20 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +/** + Terminate and clear ChaCha state + @param st The ChaCha state + @return CRYPT_OK on success +*/ +int chacha_done(chacha_state *st) +{ + LTC_ARGCHK(st != NULL); + zeromem(st, sizeof(chacha_state)); + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_ivctr32.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_ivctr32.c new file mode 100644 index 0000000..08416f5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_ivctr32.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * chacha-ref.c version 20080118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +/** + Set IV + counter data to the ChaCha state + @param st The ChaCha20 state + @param iv The IV data to add + @param ivlen The length of the IV (must be 12) + @param counter 32bit (unsigned) initial counter value + @return CRYPT_OK on success + */ +int chacha_ivctr32(chacha_state *st, const unsigned char *iv, unsigned long ivlen, ulong32 counter) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(iv != NULL); + /* 96bit IV + 32bit counter */ + LTC_ARGCHK(ivlen == 12); + + st->input[12] = counter; + LOAD32L(st->input[13], iv + 0); + LOAD32L(st->input[14], iv + 4); + LOAD32L(st->input[15], iv + 8); + st->ksleft = 0; + st->ivlen = ivlen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_ivctr64.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_ivctr64.c new file mode 100644 index 0000000..49c5459 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_ivctr64.c @@ -0,0 +1,37 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * chacha-ref.c version 20080118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +/** + Set IV + counter data to the ChaCha state + @param st The ChaCha20 state + @param iv The IV data to add + @param ivlen The length of the IV (must be 8) + @param counter 64bit (unsigned) initial counter value + @return CRYPT_OK on success + */ +int chacha_ivctr64(chacha_state *st, const unsigned char *iv, unsigned long ivlen, ulong64 counter) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(iv != NULL); + /* 64bit IV + 64bit counter */ + LTC_ARGCHK(ivlen == 8); + + st->input[12] = (ulong32)(counter & 0xFFFFFFFF); + st->input[13] = (ulong32)(counter >> 32); + LOAD32L(st->input[14], iv + 0); + LOAD32L(st->input[15], iv + 4); + st->ksleft = 0; + st->ivlen = ivlen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_keystream.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_keystream.c new file mode 100644 index 0000000..b7fe688 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_keystream.c @@ -0,0 +1,28 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * chacha-ref.c version 20080118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +/** + Generate a stream of random bytes via ChaCha + @param st The ChaCha20 state + @param out [out] The output buffer + @param outlen The output length + @return CRYPT_OK on success + */ +int chacha_keystream(chacha_state *st, unsigned char *out, unsigned long outlen) +{ + if (outlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(out != NULL); + XMEMSET(out, 0, outlen); + return chacha_crypt(st, out, outlen, out); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_memory.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_memory.c new file mode 100644 index 0000000..e73a038 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_memory.c @@ -0,0 +1,41 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with ChaCha + @param key The key + @param keylen The key length + @param iv The initial vector + @param ivlen The initial vector length + @param datain The plaintext (or ciphertext) + @param datalen The length of the input and output (octets) + @param rounds The number of rounds + @param dataout [out] The ciphertext (or plaintext) + @return CRYPT_OK if successful +*/ +int chacha_memory(const unsigned char *key, unsigned long keylen, unsigned long rounds, + const unsigned char *iv, unsigned long ivlen, ulong64 counter, + const unsigned char *datain, unsigned long datalen, unsigned char *dataout) +{ + chacha_state st; + int err; + + LTC_ARGCHK(ivlen <= 8 || counter < 4294967296); /* 2**32 */ + + if ((err = chacha_setup(&st, key, keylen, rounds)) != CRYPT_OK) goto WIPE_KEY; + if (ivlen > 8) { + if ((err = chacha_ivctr32(&st, iv, ivlen, (ulong32)counter)) != CRYPT_OK) goto WIPE_KEY; + } else { + if ((err = chacha_ivctr64(&st, iv, ivlen, counter)) != CRYPT_OK) goto WIPE_KEY; + } + err = chacha_crypt(&st, datain, datalen, dataout); +WIPE_KEY: + chacha_done(&st); + return err; +} + +#endif /* LTC_CHACHA */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_setup.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_setup.c new file mode 100644 index 0000000..6089607 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_setup.c @@ -0,0 +1,57 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * chacha-ref.c version 20080118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +static const char * const sigma = "expand 32-byte k"; +static const char * const tau = "expand 16-byte k"; + +/** + Initialize an ChaCha context (only the key) + @param st [out] The destination of the ChaCha state + @param key The secret key + @param keylen The length of the secret key (octets) + @param rounds Number of rounds (e.g. 20 for ChaCha20) + @return CRYPT_OK if successful +*/ +int chacha_setup(chacha_state *st, const unsigned char *key, unsigned long keylen, int rounds) +{ + const char *constants; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen == 32 || keylen == 16); + + if (rounds == 0) rounds = 20; + + LOAD32L(st->input[4], key + 0); + LOAD32L(st->input[5], key + 4); + LOAD32L(st->input[6], key + 8); + LOAD32L(st->input[7], key + 12); + if (keylen == 32) { /* 256bit */ + key += 16; + constants = sigma; + } else { /* 128bit */ + constants = tau; + } + LOAD32L(st->input[8], key + 0); + LOAD32L(st->input[9], key + 4); + LOAD32L(st->input[10], key + 8); + LOAD32L(st->input[11], key + 12); + LOAD32L(st->input[0], constants + 0); + LOAD32L(st->input[1], constants + 4); + LOAD32L(st->input[2], constants + 8); + LOAD32L(st->input[3], constants + 12); + st->rounds = rounds; /* e.g. 20 for chacha20 */ + st->ivlen = 0; /* will be set later by chacha_ivctr(32|64) */ + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_test.c b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_test.c new file mode 100644 index 0000000..cdd5f4f --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/chacha/chacha_test.c @@ -0,0 +1,74 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * chacha-ref.c version 20080118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_CHACHA + +int chacha_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + unsigned long len; + unsigned char out[1000]; + /* https://tools.ietf.org/html/rfc7539#section-2.4.2 */ + unsigned char k[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }; + unsigned char n[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x00 }; + unsigned char ct[] = { 0x6E, 0x2E, 0x35, 0x9A, 0x25, 0x68, 0xF9, 0x80, 0x41, 0xBA, 0x07, 0x28, 0xDD, 0x0D, 0x69, 0x81, + 0xE9, 0x7E, 0x7A, 0xEC, 0x1D, 0x43, 0x60, 0xC2, 0x0A, 0x27, 0xAF, 0xCC, 0xFD, 0x9F, 0xAE, 0x0B, + 0xF9, 0x1B, 0x65, 0xC5, 0x52, 0x47, 0x33, 0xAB, 0x8F, 0x59, 0x3D, 0xAB, 0xCD, 0x62, 0xB3, 0x57, + 0x16, 0x39, 0xD6, 0x24, 0xE6, 0x51, 0x52, 0xAB, 0x8F, 0x53, 0x0C, 0x35, 0x9F, 0x08, 0x61, 0xD8, + 0x07, 0xCA, 0x0D, 0xBF, 0x50, 0x0D, 0x6A, 0x61, 0x56, 0xA3, 0x8E, 0x08, 0x8A, 0x22, 0xB6, 0x5E, + 0x52, 0xBC, 0x51, 0x4D, 0x16, 0xCC, 0xF8, 0x06, 0x81, 0x8C, 0xE9, 0x1A, 0xB7, 0x79, 0x37, 0x36, + 0x5A, 0xF9, 0x0B, 0xBF, 0x74, 0xA3, 0x5B, 0xE6, 0xB4, 0x0B, 0x8E, 0xED, 0xF2, 0x78, 0x5E, 0x42, + 0x87, 0x4D }; + char pt[] = "Ladies and Gentlemen of the class of '99: If I could offer you only one tip for the future, sunscreen would be it."; + chacha_state st; + int err; + + len = XSTRLEN(pt); + + /* crypt piece by piece - using chacha_ivctr32() */ + if ((err = chacha_setup(&st, k, sizeof(k), 20)) != CRYPT_OK) return err; + if ((err = chacha_ivctr32(&st, n, sizeof(n), 1)) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st, (unsigned char*)pt, 35, out )) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st, (unsigned char*)pt + 35, 35, out + 35)) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st, (unsigned char*)pt + 70, 5, out + 70)) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st, (unsigned char*)pt + 75, 5, out + 75)) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st, (unsigned char*)pt + 80, len - 80, out + 80)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "CHACHA-TV1", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in one go - using chacha_ivctr32() */ + if ((err = chacha_setup(&st, k, sizeof(k), 20)) != CRYPT_OK) return err; + if ((err = chacha_ivctr32(&st, n, sizeof(n), 1)) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st, (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "CHACHA-TV2", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in one go - using chacha_ivctr64() */ + if ((err = chacha_setup(&st, k, sizeof(k), 20)) != CRYPT_OK) return err; + if ((err = chacha_ivctr64(&st, n + 4, sizeof(n) - 4, 1)) != CRYPT_OK) return err; + if ((err = chacha_crypt(&st, (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "CHACHA-TV3", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in a single call using 32-bit counter with a value of 1 */ + if ((err = chacha_memory(k, sizeof(k), 20, + n, sizeof(n), 1, (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "CHACHA-TV4", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in a single call using 64-bit counter with a value of 1 */ + if ((err = chacha_memory(k, sizeof(k), 20, + n + 4, sizeof(n) - 4, 1, (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "CHACHA-TV5", 1)) return CRYPT_FAIL_TESTVECTOR; + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/rabbit/rabbit.c b/optee_os/core/lib/libtomcrypt/src/stream/rabbit/rabbit.c new file mode 100644 index 0000000..075f4c0 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/rabbit/rabbit.c @@ -0,0 +1,447 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/****************************************************************************** + * This Rabbit C source code was morphed fm the EU eSTREAM ECRYPT submission + * and should run on any conforming C implementation (C90 or later). + * + * This implementation supports any key length up to 128 bits (16 bytes) and + * works in increments of 8-bit bytes. Keys must be submitted as whole bytes + * and shorter keys will be right-null-padded to 16 bytes. Likewise, an iv + * may be any length up to 8 bytes and will be padded out to 8 bytes. + * + * The eSTREAM submission was rather picky about the calling sequence of + * ECRYPT_process_blocks() and ECRYPT_process_bytes(). That version allowed + * calling ECRYPT_process_blocks() multiple times for a multiple of whole + * 16-byte blocks, but once ECRYPT_process_bytes() was called. no more calls + * were supported correctly. This implementation handles the keystream + * differently and rabbit_crypt() may be called as many times as desired, + * crypting any number of bytes each time. + * + * http://www.ecrypt.eu.org/stream/e2-rabbit.html + * + * NB: One of the test vectors distributed by the eSTREAM site in the file + * "rabbit_p3source.zip" is in error. Referring to "test-vectors.txt" + * in that ZIP file, the 3rd line in "out1" should be + * "96 D6 73 16 88 D1 68 DA 51 D4 0C 70 C3 A1 16 F4". + * + * Here is the original legal notice accompanying the Rabbit submission + * to the EU eSTREAM competition. + *--------------------------------------------------------------------------- + * Copyright (C) Cryptico A/S. All rights reserved. + * + * YOU SHOULD CAREFULLY READ THIS LEGAL NOTICE BEFORE USING THIS SOFTWARE. + * + * This software is developed by Cryptico A/S and/or its suppliers. + * All title and intellectual property rights in and to the software, + * including but not limited to patent rights and copyrights, are owned + * by Cryptico A/S and/or its suppliers. + * + * The software may be used solely for non-commercial purposes + * without the prior written consent of Cryptico A/S. For further + * information on licensing terms and conditions please contact + * Cryptico A/S at info@cryptico.com + * + * Cryptico, CryptiCore, the Cryptico logo and "Re-thinking encryption" + * are either trademarks or registered trademarks of Cryptico A/S. + * + * Cryptico A/S shall not in any way be liable for any use of this + * software. The software is provided "as is" without any express or + * implied warranty. + *--------------------------------------------------------------------------- + * On October 6, 2008, Rabbit was "released into the public domain and + * may be used freely for any purpose." + * http://www.ecrypt.eu.org/stream/rabbitpf.html + * https://web.archive.org/web/20090630021733/http://www.ecrypt.eu.org/stream/phorum/read.php?1,1244 + ******************************************************************************/ + + +#include "tomcrypt_private.h" + +#ifdef LTC_RABBIT + +/* local/private prototypes (NB: rabbit_ctx and rabbit_state are different) */ +static LTC_INLINE ulong32 ss_rabbit_g_func(ulong32 x); +static LTC_INLINE void ss_rabbit_next_state(rabbit_ctx *p_instance); +static LTC_INLINE void ss_rabbit_gen_1_block(rabbit_state* st, unsigned char *out); + +/* -------------------------------------------------------------------------- */ + +/* Square a 32-bit unsigned integer to obtain the 64-bit result and return */ +/* the upper 32 bits XOR the lower 32 bits */ +static LTC_INLINE ulong32 ss_rabbit_g_func(ulong32 x) +{ + ulong32 a, b, h, l; + + /* Construct high and low argument for squaring */ + a = x & 0xFFFF; + b = x >> 16; + + /* Calculate high and low result of squaring */ + h = ((((ulong32)(a*a)>>17) + (ulong32)(a*b))>>15) + b*b; + l = x * x; + + /* Return high XOR low */ + return (ulong32)(h^l); +} + +/* -------------------------------------------------------------------------- */ + +/* Calculate the next internal state */ +static LTC_INLINE void ss_rabbit_next_state(rabbit_ctx *p_instance) +{ + ulong32 g[8], c_old[8], i; + + /* Save old counter values */ + for (i=0; i<8; i++) { + c_old[i] = p_instance->c[i]; + } + + /* Calculate new counter values */ + p_instance->c[0] = (ulong32)(p_instance->c[0] + 0x4D34D34D + p_instance->carry); + p_instance->c[1] = (ulong32)(p_instance->c[1] + 0xD34D34D3 + (p_instance->c[0] < c_old[0])); + p_instance->c[2] = (ulong32)(p_instance->c[2] + 0x34D34D34 + (p_instance->c[1] < c_old[1])); + p_instance->c[3] = (ulong32)(p_instance->c[3] + 0x4D34D34D + (p_instance->c[2] < c_old[2])); + p_instance->c[4] = (ulong32)(p_instance->c[4] + 0xD34D34D3 + (p_instance->c[3] < c_old[3])); + p_instance->c[5] = (ulong32)(p_instance->c[5] + 0x34D34D34 + (p_instance->c[4] < c_old[4])); + p_instance->c[6] = (ulong32)(p_instance->c[6] + 0x4D34D34D + (p_instance->c[5] < c_old[5])); + p_instance->c[7] = (ulong32)(p_instance->c[7] + 0xD34D34D3 + (p_instance->c[6] < c_old[6])); + p_instance->carry = (p_instance->c[7] < c_old[7]); + + /* Calculate the g-values */ + for (i=0;i<8;i++) { + g[i] = ss_rabbit_g_func((ulong32)(p_instance->x[i] + p_instance->c[i])); + } + + /* Calculate new state values */ + p_instance->x[0] = (ulong32)(g[0] + ROLc(g[7],16) + ROLc(g[6], 16)); + p_instance->x[1] = (ulong32)(g[1] + ROLc(g[0], 8) + g[7]); + p_instance->x[2] = (ulong32)(g[2] + ROLc(g[1],16) + ROLc(g[0], 16)); + p_instance->x[3] = (ulong32)(g[3] + ROLc(g[2], 8) + g[1]); + p_instance->x[4] = (ulong32)(g[4] + ROLc(g[3],16) + ROLc(g[2], 16)); + p_instance->x[5] = (ulong32)(g[5] + ROLc(g[4], 8) + g[3]); + p_instance->x[6] = (ulong32)(g[6] + ROLc(g[5],16) + ROLc(g[4], 16)); + p_instance->x[7] = (ulong32)(g[7] + ROLc(g[6], 8) + g[5]); +} + +/* ------------------------------------------------------------------------- */ + +static LTC_INLINE void ss_rabbit_gen_1_block(rabbit_state* st, unsigned char *out) +{ + ulong32 *ptr; + + /* Iterate the work context once */ + ss_rabbit_next_state(&(st->work_ctx)); + + /* Generate 16 bytes of pseudo-random data */ + ptr = (ulong32*)&(st->work_ctx.x); + STORE32L((ptr[0] ^ (ptr[5]>>16) ^ (ulong32)(ptr[3]<<16)), out+ 0); + STORE32L((ptr[2] ^ (ptr[7]>>16) ^ (ulong32)(ptr[5]<<16)), out+ 4); + STORE32L((ptr[4] ^ (ptr[1]>>16) ^ (ulong32)(ptr[7]<<16)), out+ 8); + STORE32L((ptr[6] ^ (ptr[3]>>16) ^ (ulong32)(ptr[1]<<16)), out+12); +} + +/* -------------------------------------------------------------------------- */ + +/* Key setup */ +int rabbit_setup(rabbit_state* st, const unsigned char *key, unsigned long keylen) +{ + ulong32 k0, k1, k2, k3, i; + unsigned char tmpkey[16] = {0}; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen <= 16); + + /* init state */ + XMEMSET(st, 0, sizeof(rabbit_state)); + + /* pad key in tmpkey */ + XMEMCPY(tmpkey, key, keylen); + + /* Generate four subkeys */ + LOAD32L(k0, tmpkey+ 0); + LOAD32L(k1, tmpkey+ 4); + LOAD32L(k2, tmpkey+ 8); + LOAD32L(k3, tmpkey+12); + +#ifdef LTC_CLEAN_STACK + /* done with tmpkey, wipe it */ + zeromem(tmpkey, sizeof(tmpkey)); +#endif + + /* Generate initial state variables */ + st->master_ctx.x[0] = k0; + st->master_ctx.x[2] = k1; + st->master_ctx.x[4] = k2; + st->master_ctx.x[6] = k3; + st->master_ctx.x[1] = (ulong32)(k3<<16) | (k2>>16); + st->master_ctx.x[3] = (ulong32)(k0<<16) | (k3>>16); + st->master_ctx.x[5] = (ulong32)(k1<<16) | (k0>>16); + st->master_ctx.x[7] = (ulong32)(k2<<16) | (k1>>16); + + /* Generate initial counter values */ + st->master_ctx.c[0] = ROLc(k2, 16); + st->master_ctx.c[2] = ROLc(k3, 16); + st->master_ctx.c[4] = ROLc(k0, 16); + st->master_ctx.c[6] = ROLc(k1, 16); + st->master_ctx.c[1] = (k0&0xFFFF0000) | (k1&0xFFFF); + st->master_ctx.c[3] = (k1&0xFFFF0000) | (k2&0xFFFF); + st->master_ctx.c[5] = (k2&0xFFFF0000) | (k3&0xFFFF); + st->master_ctx.c[7] = (k3&0xFFFF0000) | (k0&0xFFFF); + + /* Clear carry bit */ + st->master_ctx.carry = 0; + + /* Iterate the master context four times */ + for (i=0; i<4; i++) { + ss_rabbit_next_state(&(st->master_ctx)); + } + + /* Modify the counters */ + for (i=0; i<8; i++) { + st->master_ctx.c[i] ^= st->master_ctx.x[(i+4)&0x7]; + } + + /* Copy master instance to work instance */ + for (i=0; i<8; i++) { + st->work_ctx.x[i] = st->master_ctx.x[i]; + st->work_ctx.c[i] = st->master_ctx.c[i]; + } + st->work_ctx.carry = st->master_ctx.carry; + /* ...and prepare block for crypt() */ + XMEMSET(&(st->block), 0, sizeof(st->block)); + st->unused = 0; + + return CRYPT_OK; +} + +/* -------------------------------------------------------------------------- */ + +/* IV setup */ +int rabbit_setiv(rabbit_state* st, const unsigned char *iv, unsigned long ivlen) +{ + ulong32 i0, i1, i2, i3, i; + unsigned char tmpiv[8] = {0}; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(iv != NULL || ivlen == 0); + LTC_ARGCHK(ivlen <= 8); + + /* pad iv in tmpiv */ + if (iv && ivlen > 0) XMEMCPY(tmpiv, iv, ivlen); + + /* Generate four subvectors */ + LOAD32L(i0, tmpiv+0); + LOAD32L(i2, tmpiv+4); + i1 = (i0>>16) | (i2&0xFFFF0000); + i3 = (i2<<16) | (i0&0x0000FFFF); + + /* Modify counter values */ + st->work_ctx.c[0] = st->master_ctx.c[0] ^ i0; + st->work_ctx.c[1] = st->master_ctx.c[1] ^ i1; + st->work_ctx.c[2] = st->master_ctx.c[2] ^ i2; + st->work_ctx.c[3] = st->master_ctx.c[3] ^ i3; + st->work_ctx.c[4] = st->master_ctx.c[4] ^ i0; + st->work_ctx.c[5] = st->master_ctx.c[5] ^ i1; + st->work_ctx.c[6] = st->master_ctx.c[6] ^ i2; + st->work_ctx.c[7] = st->master_ctx.c[7] ^ i3; + + /* Copy state variables */ + for (i=0; i<8; i++) { + st->work_ctx.x[i] = st->master_ctx.x[i]; + } + st->work_ctx.carry = st->master_ctx.carry; + + /* Iterate the work context four times */ + for (i=0; i<4; i++) { + ss_rabbit_next_state(&(st->work_ctx)); + } + + /* reset keystream buffer and unused count */ + XMEMSET(&(st->block), 0, sizeof(st->block)); + st->unused = 0; + + return CRYPT_OK; +} + +/* ------------------------------------------------------------------------- */ + +/* Crypt a chunk of any size (encrypt/decrypt) */ +int rabbit_crypt(rabbit_state* st, const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + unsigned char buf[16]; + unsigned long i, j; + + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + + if (st->unused > 0) { + j = MIN(st->unused, inlen); + for (i = 0; i < j; ++i, st->unused--) out[i] = in[i] ^ st->block[16 - st->unused]; + inlen -= j; + if (inlen == 0) return CRYPT_OK; + out += j; + in += j; + } + for (;;) { + /* gen a block for buf */ + ss_rabbit_gen_1_block(st, buf); + if (inlen <= 16) { + /* XOR and send to out */ + for (i = 0; i < inlen; ++i) out[i] = in[i] ^ buf[i]; + st->unused = 16 - inlen; + /* copy remainder to block */ + for (i = inlen; i < 16; ++i) st->block[i] = buf[i]; + return CRYPT_OK; + } + /* XOR entire buf and send to out */ + for (i = 0; i < 16; ++i) out[i] = in[i] ^ buf[i]; + inlen -= 16; + out += 16; + in += 16; + } +} + +/* ------------------------------------------------------------------------- */ + +int rabbit_keystream(rabbit_state *st, unsigned char *out, unsigned long outlen) +{ + if (outlen == 0) return CRYPT_OK; /* nothing to do */ + + LTC_ARGCHK(out != NULL); + + XMEMSET(out, 0, outlen); + return rabbit_crypt(st, out, outlen, out); +} + +/* -------------------------------------------------------------------------- */ + +int rabbit_done(rabbit_state *st) +{ + LTC_ARGCHK(st != NULL); + + zeromem(st, sizeof(rabbit_state)); + return CRYPT_OK; +} + +/* -------------------------------------------------------------------------- */ + +int rabbit_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + rabbit_state st; + int err; + unsigned char out[1000] = { 0 }; + { + /* all 3 tests use key and iv fm set 6, vector 3, the last vector in: + http://www.ecrypt.eu.org/stream/svn/viewcvs.cgi/ecrypt/trunk/submissions/rabbit/verified.test-vectors?rev=210&view=log + */ + + /* --- Test 1 (generate whole blocks) --------------------------------- */ + + { + unsigned char k[] = { 0x0F, 0x62, 0xB5, 0x08, 0x5B, 0xAE, 0x01, 0x54, + 0xA7, 0xFA, 0x4D, 0xA0, 0xF3, 0x46, 0x99, 0xEC }; + unsigned char iv[] = { 0x28, 0x8F, 0xF6, 0x5D, 0xC4, 0x2B, 0x92, 0xF9 }; + char pt[64] = { 0 }; + unsigned char ct[] = { 0x61, 0x3C, 0xB0, 0xBA, 0x96, 0xAF, 0xF6, 0xCA, + 0xCF, 0x2A, 0x45, 0x9A, 0x10, 0x2A, 0x7F, 0x78, + 0xCA, 0x98, 0x5C, 0xF8, 0xFD, 0xD1, 0x47, 0x40, + 0x18, 0x75, 0x8E, 0x36, 0xAE, 0x99, 0x23, 0xF5, + 0x19, 0xD1, 0x3D, 0x71, 0x8D, 0xAF, 0x8D, 0x7C, + 0x0C, 0x10, 0x9B, 0x79, 0xD5, 0x74, 0x94, 0x39, + 0xB7, 0xEF, 0xA4, 0xC4, 0xC9, 0xC8, 0xD2, 0x9D, + 0xC5, 0xB3, 0x88, 0x83, 0x14, 0xA6, 0x81, 0x6F }; + unsigned long ptlen = sizeof(pt); + + /* crypt 64 nulls */ + if ((err = rabbit_setup(&st, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = rabbit_setiv(&st, iv, sizeof(iv))) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt, ptlen, out)) != CRYPT_OK) return err; + if (compare_testvector(out, ptlen, ct, ptlen, "RABBIT-TV1", 1)) return CRYPT_FAIL_TESTVECTOR; + } + + /* --- Test 2 (generate unusual number of bytes each time) ------------ */ + + { + unsigned char k[] = { 0x0F, 0x62, 0xB5, 0x08, 0x5B, 0xAE, 0x01, 0x54, + 0xA7, 0xFA, 0x4D, 0xA0, 0xF3, 0x46, 0x99, 0xEC }; + unsigned char iv[] = { 0x28, 0x8F, 0xF6, 0x5D, 0xC4, 0x2B, 0x92, 0xF9 }; + char pt[39] = { 0 }; + unsigned char ct[] = { 0x61, 0x3C, 0xB0, 0xBA, 0x96, 0xAF, 0xF6, 0xCA, + 0xCF, 0x2A, 0x45, 0x9A, 0x10, 0x2A, 0x7F, 0x78, + 0xCA, 0x98, 0x5C, 0xF8, 0xFD, 0xD1, 0x47, 0x40, + 0x18, 0x75, 0x8E, 0x36, 0xAE, 0x99, 0x23, 0xF5, + 0x19, 0xD1, 0x3D, 0x71, 0x8D, 0xAF, 0x8D }; + unsigned long ptlen = sizeof(pt); + + /* crypt piece by piece (hit at least one 16-byte boundary) */ + if ((err = rabbit_setup(&st, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = rabbit_setiv(&st, iv, sizeof(iv))) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt, 5, out)) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt + 5, 11, out + 5)) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt + 16, 14, out + 16)) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt + 30, 2, out + 30)) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt + 32, 7, out + 32)) != CRYPT_OK) return err; + if (compare_testvector(out, ptlen, ct, ptlen, "RABBIT-TV2", 1)) return CRYPT_FAIL_TESTVECTOR; + } + + /* --- Test 3 (use non-null data) ------------------------------------- */ + + { + unsigned char k[] = { 0x0F, 0x62, 0xB5, 0x08, 0x5B, 0xAE, 0x01, 0x54, + 0xA7, 0xFA, 0x4D, 0xA0, 0xF3, 0x46, 0x99, 0xEC }; + unsigned char iv[] = { 0x28, 0x8F, 0xF6, 0x5D, 0xC4, 0x2B, 0x92, 0xF9 }; + char pt[] = "Kilroy was here, there, and everywhere!"; + unsigned char ct[] = { 0x2a, 0x55, 0xdc, 0xc8, 0xf9, 0xd6, 0xd6, 0xbd, + 0xae, 0x59, 0x65, 0xf2, 0x75, 0x58, 0x1a, 0x54, + 0xea, 0xec, 0x34, 0x9d, 0x8f, 0xb4, 0x6b, 0x60, + 0x79, 0x1b, 0xea, 0x16, 0xcb, 0xef, 0x46, 0x87, + 0x60, 0xa6, 0x55, 0x14, 0xff, 0xca, 0xac }; + unsigned long ptlen = XSTRLEN(pt); + unsigned char out2[1000] = { 0 }; + unsigned char nulls[1000] = { 0 }; + + /* crypt piece by piece */ + if ((err = rabbit_setup(&st, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = rabbit_setiv(&st, iv, sizeof(iv))) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt, 5, out)) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt + 5, 29, out + 5)) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, (unsigned char*)pt + 34, 5, out + 34)) != CRYPT_OK) return err; + if (compare_testvector(out, ptlen, ct, ptlen, "RABBIT-TV3", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* --- Test 4 (crypt in a single call) ------------------------------------ */ + + if ((err = rabbit_memory(k, sizeof(k), iv, sizeof(iv), + (unsigned char*)pt, sizeof(pt), out)) != CRYPT_OK) return err; + if (compare_testvector(out, ptlen, ct, ptlen, "RABBIT-TV4", 1)) return CRYPT_FAIL_TESTVECTOR; + /* use 'out' (ciphertext) in the next decryption test */ + + /* --- Test 5 (decrypt ciphertext) ------------------------------------ */ + + /* decrypt ct (out) and compare with pt (start with only setiv() to reset) */ + if ((err = rabbit_setiv(&st, iv, sizeof(iv))) != CRYPT_OK) return err; + if ((err = rabbit_crypt(&st, out, ptlen, out2)) != CRYPT_OK) return err; + if (compare_testvector(out2, ptlen, pt, ptlen, "RABBIT-TV5", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* --- Test 6 (wipe state, incl key) ---------------------------------- */ + + if ((err = rabbit_done(&st)) != CRYPT_OK) return err; + if (compare_testvector(&st, sizeof(st), nulls, sizeof(st), "RABBIT-TV6", 1)) return CRYPT_FAIL_TESTVECTOR; + + } + + return CRYPT_OK; + } +#endif +} + +/* -------------------------------------------------------------------------- */ + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/rabbit/rabbit_memory.c b/optee_os/core/lib/libtomcrypt/src/stream/rabbit/rabbit_memory.c new file mode 100644 index 0000000..fee0234 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/rabbit/rabbit_memory.c @@ -0,0 +1,40 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * chacha-ref.c version 20080118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_RABBIT + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with Rabbit + @param key The key + @param keylen The key length + @param iv The initial vector + @param ivlen The initial vector length + @param datain The plaintext (or ciphertext) + @param datalen The length of the input and output (octets) + @param dataout [out] The ciphertext (or plaintext) + @return CRYPT_OK if successful +*/ +int rabbit_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout) +{ + rabbit_state st; + int err; + + if ((err = rabbit_setup(&st, key, keylen)) != CRYPT_OK) goto WIPE_KEY; + if ((err = rabbit_setiv(&st, iv, ivlen)) != CRYPT_OK) goto WIPE_KEY; + err = rabbit_crypt(&st, datain, datalen, dataout); +WIPE_KEY: + rabbit_done(&st); + return err; +} + +#endif /* LTC_RABBIT */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_stream.c b/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_stream.c new file mode 100644 index 0000000..2587764 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_stream.c @@ -0,0 +1,101 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_RC4_STREAM + +/** + Initialize an RC4 context (only the key) + @param st [out] The destination of the RC4 state + @param key The secret key + @param keylen The length of the secret key (8 - 256 bytes) + @return CRYPT_OK if successful +*/ +int rc4_stream_setup(rc4_state *st, const unsigned char *key, unsigned long keylen) +{ + unsigned char tmp, *s; + int x, y; + unsigned long j; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen >= 5); /* 40-2048 bits */ + + s = st->buf; + for (x = 0; x < 256; x++) { + s[x] = x; + } + + for (j = x = y = 0; x < 256; x++) { + y = (y + s[x] + key[j++]) & 255; + if (j == keylen) { + j = 0; + } + tmp = s[x]; s[x] = s[y]; s[y] = tmp; + } + st->x = 0; + st->y = 0; + + return CRYPT_OK; +} + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with RC4 + @param st The RC4 state + @param in The plaintext (or ciphertext) + @param inlen The length of the input (octets) + @param out [out] The ciphertext (or plaintext), length inlen + @return CRYPT_OK if successful +*/ +int rc4_stream_crypt(rc4_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + unsigned char x, y, *s, tmp; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + + x = st->x; + y = st->y; + s = st->buf; + while (inlen--) { + x = (x + 1) & 255; + y = (y + s[x]) & 255; + tmp = s[x]; s[x] = s[y]; s[y] = tmp; + tmp = (s[x] + s[y]) & 255; + *out++ = *in++ ^ s[tmp]; + } + st->x = x; + st->y = y; + return CRYPT_OK; +} + +/** + Generate a stream of random bytes via RC4 + @param st The RC420 state + @param out [out] The output buffer + @param outlen The output length + @return CRYPT_OK on success + */ +int rc4_stream_keystream(rc4_state *st, unsigned char *out, unsigned long outlen) +{ + if (outlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(out != NULL); + XMEMSET(out, 0, outlen); + return rc4_stream_crypt(st, out, outlen, out); +} + +/** + Terminate and clear RC4 state + @param st The RC4 state + @return CRYPT_OK on success +*/ +int rc4_stream_done(rc4_state *st) +{ + LTC_ARGCHK(st != NULL); + zeromem(st, sizeof(rc4_state)); + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_stream_memory.c b/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_stream_memory.c new file mode 100644 index 0000000..efe4888 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_stream_memory.c @@ -0,0 +1,31 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_RC4_STREAM + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with RC4 + @param key The key + @param keylen The key length + @param datain The plaintext (or ciphertext) + @param datalen The length of the input and output (octets) + @param dataout [out] The ciphertext (or plaintext) + @return CRYPT_OK if successful +*/ +int rc4_stream_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout) +{ + rc4_state st; + int err; + + if ((err = rc4_stream_setup(&st, key, keylen)) != CRYPT_OK) goto WIPE_KEY; + err = rc4_stream_crypt(&st, datain, datalen, dataout); +WIPE_KEY: + rc4_stream_done(&st); + return err; +} + +#endif /* LTC_RC4_STREAM */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_test.c b/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_test.c new file mode 100644 index 0000000..d78022b --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/rc4/rc4_test.c @@ -0,0 +1,33 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_RC4_STREAM + +int rc4_stream_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + rc4_state st; + int err; + const unsigned char key[] = { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef }; + const unsigned char pt[] = { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef }; + const unsigned char ct[] = { 0x75, 0xb7, 0x87, 0x80, 0x99, 0xe0, 0xc5, 0x96 }; + unsigned char buf[10]; + + if ((err = rc4_stream_setup(&st, key, sizeof(key))) != CRYPT_OK) return err; + if ((err = rc4_stream_crypt(&st, pt, sizeof(pt), buf)) != CRYPT_OK) return err; + if (compare_testvector(buf, sizeof(ct), ct, sizeof(ct), "RC4-TV1", 0)) return CRYPT_FAIL_TESTVECTOR; + if ((err = rc4_stream_done(&st)) != CRYPT_OK) return err; + + /* crypt in a single call */ + if ((err = rc4_stream_memory(key, sizeof(key), pt, sizeof(pt), buf)) != CRYPT_OK) return err; + if (compare_testvector(buf, sizeof(ct), ct, sizeof(ct), "RC4-TV2", 0)) return CRYPT_FAIL_TESTVECTOR; + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_crypt.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_crypt.c new file mode 100644 index 0000000..3058372 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_crypt.c @@ -0,0 +1,86 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * "Salsa20 specification", http://cr.yp.to/snuffle/spec.pdf + * and salsa20-ref.c version 20051118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SALSA20 + +#define QUARTERROUND(a,b,c,d) \ + x[b] ^= (ROL((x[a] + x[d]), 7)); \ + x[c] ^= (ROL((x[b] + x[a]), 9)); \ + x[d] ^= (ROL((x[c] + x[b]), 13)); \ + x[a] ^= (ROL((x[d] + x[c]), 18)); + +static void s_salsa20_block(unsigned char *output, const ulong32 *input, int rounds) +{ + ulong32 x[16]; + int i; + XMEMCPY(x, input, sizeof(x)); + for (i = rounds; i > 0; i -= 2) { + QUARTERROUND( 0, 4, 8,12) + QUARTERROUND( 5, 9,13, 1) + QUARTERROUND(10,14, 2, 6) + QUARTERROUND(15, 3, 7,11) + QUARTERROUND( 0, 1, 2, 3) + QUARTERROUND( 5, 6, 7, 4) + QUARTERROUND(10,11, 8, 9) + QUARTERROUND(15,12,13,14) + } + for (i = 0; i < 16; ++i) { + x[i] += input[i]; + STORE32L(x[i], output + 4 * i); + } +} + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with Salsa20 + @param st The Salsa20 state + @param in The plaintext (or ciphertext) + @param inlen The length of the input (octets) + @param out [out] The ciphertext (or plaintext), length inlen + @return CRYPT_OK if successful +*/ +int salsa20_crypt(salsa20_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + unsigned char buf[64]; + unsigned long i, j; + + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(st->ivlen == 8 || st->ivlen == 24); + + if (st->ksleft > 0) { + j = MIN(st->ksleft, inlen); + for (i = 0; i < j; ++i, st->ksleft--) out[i] = in[i] ^ st->kstream[64 - st->ksleft]; + inlen -= j; + if (inlen == 0) return CRYPT_OK; + out += j; + in += j; + } + for (;;) { + s_salsa20_block(buf, st->input, st->rounds); + /* Salsa20: 64-bit IV, increment 64-bit counter */ + if (0 == ++st->input[8] && 0 == ++st->input[9]) return CRYPT_OVERFLOW; + if (inlen <= 64) { + for (i = 0; i < inlen; ++i) out[i] = in[i] ^ buf[i]; + st->ksleft = 64 - inlen; + for (i = inlen; i < 64; ++i) st->kstream[i] = buf[i]; + return CRYPT_OK; + } + for (i = 0; i < 64; ++i) out[i] = in[i] ^ buf[i]; + inlen -= 64; + out += 64; + in += 64; + } +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_done.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_done.c new file mode 100644 index 0000000..9b8cbb6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_done.c @@ -0,0 +1,20 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SALSA20 + +/** + Terminate and clear Salsa20 state + @param st The Salsa20 state + @return CRYPT_OK on success +*/ +int salsa20_done(salsa20_state *st) +{ + LTC_ARGCHK(st != NULL); + zeromem(st, sizeof(salsa20_state)); + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_ivctr64.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_ivctr64.c new file mode 100644 index 0000000..8b25dfb --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_ivctr64.c @@ -0,0 +1,38 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * "Salsa20 specification", http://cr.yp.to/snuffle/spec.pdf + * and salsa20-ref.c version 20051118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SALSA20 + +/** + Set IV + counter data to the Salsa20 state + @param st The Salsa20 state + @param iv The IV data to add + @param ivlen The length of the IV (must be 8) + @param counter 64bit (unsigned) initial counter value + @return CRYPT_OK on success + */ +int salsa20_ivctr64(salsa20_state *st, const unsigned char *iv, unsigned long ivlen, ulong64 counter) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(iv != NULL); + /* Salsa20: 64-bit IV (nonce) + 64-bit counter */ + LTC_ARGCHK(ivlen == 8); + + LOAD32L(st->input[6], iv + 0); + LOAD32L(st->input[7], iv + 4); + st->input[8] = (ulong32)(counter & 0xFFFFFFFF); + st->input[9] = (ulong32)(counter >> 32); + st->ksleft = 0; + st->ivlen = ivlen; + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_keystream.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_keystream.c new file mode 100644 index 0000000..c51d0d6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_keystream.c @@ -0,0 +1,29 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * "Salsa20 specification", http://cr.yp.to/snuffle/spec.pdf + * and salsa20-ref.c version 20051118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SALSA20 + +/** + Generate a stream of random bytes via Salsa20 + @param st The Salsa20 state + @param out [out] The output buffer + @param outlen The output length + @return CRYPT_OK on success + */ +int salsa20_keystream(salsa20_state *st, unsigned char *out, unsigned long outlen) +{ + if (outlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(out != NULL); + XMEMSET(out, 0, outlen); + return salsa20_crypt(st, out, outlen, out); +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_memory.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_memory.c new file mode 100644 index 0000000..9408eb2 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_memory.c @@ -0,0 +1,35 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SALSA20 + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with Salsa20 + @param key The key + @param keylen The key length + @param iv The initial vector + @param ivlen The initial vector length + @param datain The plaintext (or ciphertext) + @param datalen The length of the input and output (octets) + @param rounds The number of rounds + @param dataout [out] The ciphertext (or plaintext) + @return CRYPT_OK if successful +*/ +int salsa20_memory(const unsigned char *key, unsigned long keylen, unsigned long rounds, + const unsigned char *iv, unsigned long ivlen, ulong64 counter, + const unsigned char *datain, unsigned long datalen, unsigned char *dataout) +{ + salsa20_state st; + int err; + + if ((err = salsa20_setup(&st, key, keylen, rounds)) != CRYPT_OK) goto WIPE_KEY; + if ((err = salsa20_ivctr64(&st, iv, ivlen, counter)) != CRYPT_OK) goto WIPE_KEY; + err = salsa20_crypt(&st, datain, datalen, dataout); +WIPE_KEY: + salsa20_done(&st); + return err; +} + +#endif /* LTC_SALSA20 */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_setup.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_setup.c new file mode 100644 index 0000000..012dc95 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_setup.c @@ -0,0 +1,59 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * "Salsa20 specification", http://cr.yp.to/snuffle/spec.pdf + * and salsa20-ref.c version 20051118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SALSA20 + +static const char * const sigma = "expand 32-byte k"; +static const char * const tau = "expand 16-byte k"; + +/** + Initialize an Salsa20 context (only the key) + @param st [out] The destination of the Salsa20 state + @param key The secret key + @param keylen The length of the secret key (octets) + @param rounds Number of rounds (e.g. 20 for Salsa20) + @return CRYPT_OK if successful +*/ +int salsa20_setup(salsa20_state *st, const unsigned char *key, unsigned long keylen, int rounds) +{ + const char *constants; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen == 32 || keylen == 16); + + if (rounds == 0) rounds = 20; + LTC_ARGCHK(rounds % 2 == 0); /* number of rounds must be evenly divisible by 2 */ + + LOAD32L(st->input[1], key + 0); + LOAD32L(st->input[2], key + 4); + LOAD32L(st->input[3], key + 8); + LOAD32L(st->input[4], key + 12); + if (keylen == 32) { /* 256bit */ + key += 16; + constants = sigma; + } else { /* 128bit */ + constants = tau; + } + LOAD32L(st->input[11], key + 0); + LOAD32L(st->input[12], key + 4); + LOAD32L(st->input[13], key + 8); + LOAD32L(st->input[14], key + 12); + LOAD32L(st->input[ 0], constants + 0); + LOAD32L(st->input[ 5], constants + 4); + LOAD32L(st->input[10], constants + 8); + LOAD32L(st->input[15], constants + 12); + st->rounds = rounds; /* default is 20 for salsa20 */ + st->ivlen = 0; /* will be set later by salsa20_ivctr(32|64) */ + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_test.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_test.c new file mode 100644 index 0000000..c924dbe --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/salsa20_test.c @@ -0,0 +1,86 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * "Salsa20 specification", http://cr.yp.to/snuffle/spec.pdf + * and salsa20-ref.c version 20051118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SALSA20 + +int salsa20_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + salsa20_state st; + unsigned char k[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }; + unsigned char n[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a }; + unsigned char ct[] = { 0x37, 0x37, 0x2e, 0x60, 0xb8, 0xae, 0x88, 0x1f, 0xf8, 0xdf, 0x00, 0x26, 0x6c, 0x30, 0x34, 0x2d, + 0xa1, 0xd7, 0x79, 0x60, 0x67, 0x72, 0xe0, 0x67, 0x26, 0x22, 0xad, 0x00, 0x9e, 0xd5, 0x59, 0x44, + 0x51, 0xd9, 0xe6, 0xaa, 0xc9, 0x59, 0x9e, 0x60, 0xff, 0x87, 0x90, 0xc1, 0xc9, 0x1e }; + unsigned char ct2[] = { 0xec, 0x06, 0x32, 0xb3, 0x83, 0x5c, 0xae, 0x91, 0x01, 0x82, 0x7a, 0x71, 0xd9, 0x7d, 0x45, 0xd7, + 0xa6, 0x5b, 0xa0, 0x89, 0x9d, 0xd2, 0x6c, 0xaa, 0xbb, 0x2f, 0x5f, 0x30, 0x89, 0x54, 0xff, 0x3e, + 0x83, 0xc3, 0x34, 0x10, 0xb6, 0xe1, 0xab, 0xe7, 0xf5, 0xab, 0xab, 0xed, 0xa4, 0xff }; + char pt[] = "Kilroy was here, and there. ...and everywhere!"; /* len = 46 bytes */ + unsigned long len; + unsigned char out[1000]; + int counter; + int rounds; + int err; + len = XSTRLEN(pt); + + /* crypt piece by piece */ + counter = 0; + rounds = 12; + if ((err = salsa20_setup(&st, k, sizeof(k), rounds)) != CRYPT_OK) return err; + if ((err = salsa20_ivctr64(&st, n, sizeof(n), counter)) != CRYPT_OK) return err; + if ((err = salsa20_crypt(&st, (unsigned char*)pt, 5, out)) != CRYPT_OK) return err; + if ((err = salsa20_crypt(&st, (unsigned char*)pt + 5, 25, out + 5)) != CRYPT_OK) return err; + if ((err = salsa20_crypt(&st, (unsigned char*)pt + 30, 10, out + 30)) != CRYPT_OK) return err; + if ((err = salsa20_crypt(&st, (unsigned char*)pt + 40, len - 40, out + 40)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "SALSA20-TV1", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in one go - using salsa20_ivctr64() */ + counter = 0; + rounds = 20; + if ((err = salsa20_setup(&st, k, sizeof(k), rounds)) != CRYPT_OK) return err; + if ((err = salsa20_ivctr64(&st, n, sizeof(n), counter)) != CRYPT_OK) return err; + if ((err = salsa20_crypt(&st, (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct2, sizeof(ct2), "SALSA20-TV2", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in a single call */ + if ((err = salsa20_memory(k, sizeof(k), rounds, n, sizeof(n), counter, + (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct2, sizeof(ct2), "SALSA20-TV3", 1)) return CRYPT_FAIL_TESTVECTOR; + + { + /* keystream + * http://www.ecrypt.eu.org/stream/svn/viewcvs.cgi/ecrypt/trunk/submissions/salsa20/full/verified.test-vectors?rev=161&view=markup + * Set 6, vector 0 + */ + unsigned char k3[] = { 0x00, 0x53, 0xA6, 0xF9, 0x4C, 0x9F, 0xF2, 0x45, 0x98, 0xEB, 0x3E, 0x91, 0xE4, 0x37, 0x8A, 0xDD, + 0x30, 0x83, 0xD6, 0x29, 0x7C, 0xCF, 0x22, 0x75, 0xC8, 0x1B, 0x6E, 0xC1, 0x14, 0x67, 0xBA, 0x0D }; + unsigned char n3[] = { 0x0D, 0x74, 0xDB, 0x42, 0xA9, 0x10, 0x77, 0xDE }; + unsigned char ct3[] = { 0xF5, 0xFA, 0xD5, 0x3F, 0x79, 0xF9, 0xDF, 0x58, 0xC4, 0xAE, 0xA0, 0xD0, 0xED, 0x9A, 0x96, 0x01, + 0xF2, 0x78, 0x11, 0x2C, 0xA7, 0x18, 0x0D, 0x56, 0x5B, 0x42, 0x0A, 0x48, 0x01, 0x96, 0x70, 0xEA, + 0xF2, 0x4C, 0xE4, 0x93, 0xA8, 0x62, 0x63, 0xF6, 0x77, 0xB4, 0x6A, 0xCE, 0x19, 0x24, 0x77, 0x3D, + 0x2B, 0xB2, 0x55, 0x71, 0xE1, 0xAA, 0x85, 0x93, 0x75, 0x8F, 0xC3, 0x82, 0xB1, 0x28, 0x0B, 0x71 }; + int counter3 = 0; + int rounds3 = 20; + if ((err = salsa20_setup(&st, k3, sizeof(k3), rounds3)) != CRYPT_OK) return err; + if ((err = salsa20_ivctr64(&st, n3, sizeof(n3), counter3)) != CRYPT_OK) return err; + if ((err = salsa20_keystream(&st, out, 64)) != CRYPT_OK) return err; + if ((err = salsa20_done(&st)) != CRYPT_OK) return err; + if (compare_testvector(out, 64, ct3, sizeof(ct3), "SALSA20-TV4", 1)) return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_memory.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_memory.c new file mode 100644 index 0000000..1908cd5 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_memory.c @@ -0,0 +1,34 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_XSALSA20 + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with XSalsa20 + @param key The key + @param keylen The key length + @param nonce The initial vector + @param noncelen The initial vector length + @param datain The plaintext (or ciphertext) + @param datalen The length of the input and output (octets) + @param rounds The number of rounds + @param dataout [out] The ciphertext (or plaintext) + @return CRYPT_OK if successful +*/ +int xsalsa20_memory(const unsigned char *key, unsigned long keylen, unsigned long rounds, + const unsigned char *nonce, unsigned long noncelen, + const unsigned char *datain, unsigned long datalen, unsigned char *dataout) +{ + salsa20_state st; + int err; + + if ((err = xsalsa20_setup(&st, key, keylen, nonce, noncelen, rounds)) != CRYPT_OK) goto WIPE_KEY; + err = salsa20_crypt(&st, datain, datalen, dataout); +WIPE_KEY: + salsa20_done(&st); + return err; +} + +#endif /* LTC_XSALSA20 */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_setup.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_setup.c new file mode 100644 index 0000000..6bfac09 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_setup.c @@ -0,0 +1,127 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * "Extending the Salsa20 nonce", https://cr.yp.to/snuffle/xsalsa-20081128.pdf + * "Salsa20 specification", http://cr.yp.to/snuffle/spec.pdf + * and salsa20-ref.c version 20051118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt.h" + +#ifdef LTC_XSALSA20 + +static const char * const constants = "expand 32-byte k"; + +#define QUARTERROUND(a,b,c,d) \ + x[b] ^= (ROL((x[a] + x[d]), 7)); \ + x[c] ^= (ROL((x[b] + x[a]), 9)); \ + x[d] ^= (ROL((x[c] + x[b]), 13)); \ + x[a] ^= (ROL((x[d] + x[c]), 18)); + +/* use modified salsa20 doubleround (no final addition as in salsa20) */ +static void s_xsalsa20_doubleround(ulong32 *x, int rounds) +{ + int i; + + for (i = rounds; i > 0; i -= 2) { + /* columnround */ + QUARTERROUND( 0, 4, 8,12) + QUARTERROUND( 5, 9,13, 1) + QUARTERROUND(10,14, 2, 6) + QUARTERROUND(15, 3, 7,11) + /* rowround */ + QUARTERROUND( 0, 1, 2, 3) + QUARTERROUND( 5, 6, 7, 4) + QUARTERROUND(10,11, 8, 9) + QUARTERROUND(15,12,13,14) + } +} + +#undef QUARTERROUND + +/** + Initialize an XSalsa20 context + @param st [out] The destination of the XSalsa20 state + @param key The secret key + @param keylen The length of the secret key, must be 32 (octets) + @param nonce The nonce + @param noncelen The length of the nonce, must be 24 (octets) + @param rounds Number of rounds (must be evenly divisible by 2, default is 20) + @return CRYPT_OK if successful +*/ +int xsalsa20_setup(salsa20_state *st, const unsigned char *key, unsigned long keylen, + const unsigned char *nonce, unsigned long noncelen, + int rounds) +{ + const int sti[] = {0, 5, 10, 15, 6, 7, 8, 9}; /* indices used to build subkey fm x */ + ulong32 x[64]; /* input to & output fm doubleround */ + unsigned char subkey[32]; + int i; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen == 32); + LTC_ARGCHK(nonce != NULL); + LTC_ARGCHK(noncelen == 24); + if (rounds == 0) rounds = 20; + LTC_ARGCHK(rounds % 2 == 0); /* number of rounds must be evenly divisible by 2 */ + + /* load the state to "hash" the key */ + LOAD32L(x[ 0], constants + 0); + LOAD32L(x[ 5], constants + 4); + LOAD32L(x[10], constants + 8); + LOAD32L(x[15], constants + 12); + LOAD32L(x[ 1], key + 0); + LOAD32L(x[ 2], key + 4); + LOAD32L(x[ 3], key + 8); + LOAD32L(x[ 4], key + 12); + LOAD32L(x[11], key + 16); + LOAD32L(x[12], key + 20); + LOAD32L(x[13], key + 24); + LOAD32L(x[14], key + 28); + LOAD32L(x[ 6], nonce + 0); + LOAD32L(x[ 7], nonce + 4); + LOAD32L(x[ 8], nonce + 8); + LOAD32L(x[ 9], nonce + 12); + + /* use modified salsa20 doubleround (no final addition) */ + s_xsalsa20_doubleround(x, rounds); + + /* extract the subkey */ + for (i = 0; i < 8; ++i) { + STORE32L(x[sti[i]], subkey + 4 * i); + } + + /* load the final initial state */ + LOAD32L(st->input[ 0], constants + 0); + LOAD32L(st->input[ 5], constants + 4); + LOAD32L(st->input[10], constants + 8); + LOAD32L(st->input[15], constants + 12); + LOAD32L(st->input[ 1], subkey + 0); + LOAD32L(st->input[ 2], subkey + 4); + LOAD32L(st->input[ 3], subkey + 8); + LOAD32L(st->input[ 4], subkey + 12); + LOAD32L(st->input[11], subkey + 16); + LOAD32L(st->input[12], subkey + 20); + LOAD32L(st->input[13], subkey + 24); + LOAD32L(st->input[14], subkey + 28); + LOAD32L(st->input[ 6], &(nonce[16]) + 0); + LOAD32L(st->input[ 7], &(nonce[16]) + 4); + st->input[ 8] = 0; + st->input[ 9] = 0; + st->rounds = rounds; + st->ksleft = 0; + st->ivlen = 24; /* set switch to say nonce/IV has been loaded */ + +#ifdef LTC_CLEAN_STACK + zeromem(x, sizeof(x)); + zeromem(subkey, sizeof(subkey)); +#endif + + return CRYPT_OK; +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_test.c b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_test.c new file mode 100644 index 0000000..31522fe --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/salsa20/xsalsa20_test.c @@ -0,0 +1,90 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* The implementation is based on: + * "Extending the Salsa20 nonce", https://cr.yp.to/snuffle/xsalsa-20081128.pdf + * "Salsa20 specification", http://cr.yp.to/snuffle/spec.pdf + * and salsa20-ref.c version 20051118 + * Public domain from D. J. Bernstein + */ + +#include "tomcrypt.h" + +#ifdef LTC_XSALSA20 + +#if defined(LTC_SHA256) && defined(LTC_TEST) +static int s_sha256(unsigned char *hash, const unsigned char *data, const int datalen) { + hash_state md; + sha256_init(&md); + sha256_process(&md, data, datalen); + sha256_done(&md, hash); + return CRYPT_OK; +} +#endif + +int xsalsa20_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + + /*************************************************************************** + * verify a round trip: + */ + { + const unsigned char key[] = {0x1b,0x27,0x55,0x64,0x73,0xe9,0x85,0xd4,0x62,0xcd,0x51,0x19,0x7a,0x9a,0x46,0xc7,0x60,0x09,0x54,0x9e,0xac,0x64,0x74,0xf2,0x06,0xc4,0xee,0x08,0x44,0xf6,0x83,0x89}; + const unsigned char nonce[] = {0x69,0x69,0x6e,0xe9,0x55,0xb6,0x2b,0x73,0xcd,0x62,0xbd,0xa8,0x75,0xfc,0x73,0xd6,0x82,0x19,0xe0,0x03,0x6b,0x7a,0x0b,0x37}; + const void *msg = "Kilroy was here!"; + unsigned char msglen = 17; /* includes trailing NULL */ + int rounds = 20; + unsigned char ciphertext[17]; + unsigned char msg2[17]; + salsa20_state st; + int err; + + if ((err = xsalsa20_setup(&st, key, 32, nonce, 24, rounds)) != CRYPT_OK) return err; + if ((err = salsa20_crypt(&st, msg, msglen, ciphertext)) != CRYPT_OK) return err; + if ((err = salsa20_done(&st)) != CRYPT_OK) return err; + + if ((err = xsalsa20_setup(&st, key, 32, nonce, 24, rounds)) != CRYPT_OK) return err; + if ((err = salsa20_crypt(&st, ciphertext, msglen, msg2)) != CRYPT_OK) return err; + if ((err = salsa20_done(&st)) != CRYPT_OK) return err; + + if (compare_testvector(msg, msglen, msg2, msglen, "XSALSA20-TV1", 1)) return CRYPT_FAIL_TESTVECTOR; + + + /* round trip with two single function calls */ + if ((err = xsalsa20_memory(key, sizeof(key), 20, nonce, sizeof(nonce), msg, msglen, ciphertext)) != CRYPT_OK) return err; + if ((err = xsalsa20_memory(key, sizeof(key), 20, nonce, sizeof(nonce), ciphertext, msglen, msg2)) != CRYPT_OK) return err; + if (compare_testvector(msg, msglen, msg2, msglen, "XSALSA20-TV2", 1)) return CRYPT_FAIL_TESTVECTOR; + } + +#ifdef LTC_SHA256 + /*************************************************************************** + * verify correct generation of a keystream + */ + { + const unsigned char key[] = {0x1b,0x27,0x55,0x64,0x73,0xe9,0x85,0xd4,0x62,0xcd,0x51,0x19,0x7a,0x9a,0x46,0xc7,0x60,0x09,0x54,0x9e,0xac,0x64,0x74,0xf2,0x06,0xc4,0xee,0x08,0x44,0xf6,0x83,0x89}; + const unsigned char nonce[] = {0x69,0x69,0x6e,0xe9,0x55,0xb6,0x2b,0x73,0xcd,0x62,0xbd,0xa8,0x75,0xfc,0x73,0xd6,0x82,0x19,0xe0,0x03,0x6b,0x7a,0x0b,0x37}; + const unsigned char expecthash[] = {0x6a,0x60,0x57,0x65,0x27,0xe0,0x00,0x51,0x6d,0xb0,0xda,0x60,0x46,0x20,0xf6,0xd0,0x95,0x65,0x45,0x39,0xf4,0x86,0x83,0x43,0x64,0xdf,0xd9,0x5a,0x6f,0x3f,0xbe,0xb7}; + int rounds = 20; + unsigned char keystream[91101]; + unsigned long keystreamlen = 91101; + unsigned char hash[32]; + salsa20_state st; + int err; + + if ((err = xsalsa20_setup(&st, key, 32, nonce, 24, rounds)) != CRYPT_OK) return err; + if ((err = salsa20_keystream(&st, keystream, keystreamlen)) != CRYPT_OK) return err; + if ((err = salsa20_done(&st)) != CRYPT_OK) return err; + if ((err = s_sha256(hash, keystream, keystreamlen)) != CRYPT_OK) return err; + if (compare_testvector(hash, sizeof(hash), expecthash, sizeof(expecthash), "XSALSA20-TV3", 1)) return CRYPT_FAIL_TESTVECTOR; + } +#endif + + return CRYPT_OK; + +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_stream.c b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_stream.c new file mode 100644 index 0000000..4445151 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_stream.c @@ -0,0 +1,333 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ +#include "tomcrypt_private.h" + +/** + @file sober128_stream.c + Implementation of SOBER-128 by Tom St Denis. + Based on s128fast.c reference code supplied by Greg Rose of QUALCOMM. +*/ + +#ifdef LTC_SOBER128 + +#define LTC_SOBER128TAB_C +#include "sober128tab.c" + +/* don't change these... */ +#define N 17 +#define INITKONST 0x6996c53a /* value of KONST to use during key loading */ +#define KEYP 15 /* where to insert key words */ +#define FOLDP 4 /* where to insert non-linear feedback */ + +static ulong32 BYTE2WORD(const unsigned char *b) +{ + ulong32 t; + LOAD32L(t, b); + return t; +} + +static void XORWORD(ulong32 w, const unsigned char *in, unsigned char *out) +{ + ulong32 t; + LOAD32L(t, in); + t ^= w; + STORE32L(t, out); +} + +/* give correct offset for the current position of the register, + * where logically R[0] is at position "zero". + */ +#define OFF(zero, i) (((zero)+(i)) % N) + +/* step the LFSR */ +/* After stepping, "zero" moves right one place */ +#define STEP(R,z) \ + R[OFF(z,0)] = R[OFF(z,15)] ^ R[OFF(z,4)] ^ (R[OFF(z,0)] << 8) ^ Multab[(R[OFF(z,0)] >> 24) & 0xFF]; + +static void cycle(ulong32 *R) +{ + ulong32 t; + int i; + + STEP(R,0); + t = R[0]; + for (i = 1; i < N; ++i) { + R[i-1] = R[i]; + } + R[N-1] = t; +} + +/* Return a non-linear function of some parts of the register. + */ +#define NLFUNC(st,z) \ +{ \ + t = st->R[OFF(z,0)] + st->R[OFF(z,16)]; \ + t ^= Sbox[(t >> 24) & 0xFF]; \ + t = RORc(t, 8); \ + t = ((t + st->R[OFF(z,1)]) ^ st->konst) + st->R[OFF(z,6)]; \ + t ^= Sbox[(t >> 24) & 0xFF]; \ + t = t + st->R[OFF(z,13)]; \ +} + +static ulong32 nltap(const sober128_state *st) +{ + ulong32 t; + NLFUNC(st, 0); + return t; +} + +/* Save the current register state + */ +static void s128_savestate(sober128_state *st) +{ + int i; + for (i = 0; i < N; ++i) { + st->initR[i] = st->R[i]; + } +} + +/* initialise to previously saved register state + */ +static void s128_reloadstate(sober128_state *st) +{ + int i; + + for (i = 0; i < N; ++i) { + st->R[i] = st->initR[i]; + } +} + +/* Initialise "konst" + */ +static void s128_genkonst(sober128_state *st) +{ + ulong32 newkonst; + + do { + cycle(st->R); + newkonst = nltap(st); + } while ((newkonst & 0xFF000000) == 0); + st->konst = newkonst; +} + +/* Load key material into the register + */ +#define ADDKEY(k) \ + st->R[KEYP] += (k); + +#define XORNL(nl) \ + st->R[FOLDP] ^= (nl); + +/* nonlinear diffusion of register for key */ +#define DROUND(z) STEP(st->R,z); NLFUNC(st,(z+1)); st->R[OFF((z+1),FOLDP)] ^= t; +static void s128_diffuse(sober128_state *st) +{ + ulong32 t; + /* relies on FOLD == N == 17! */ + DROUND(0); + DROUND(1); + DROUND(2); + DROUND(3); + DROUND(4); + DROUND(5); + DROUND(6); + DROUND(7); + DROUND(8); + DROUND(9); + DROUND(10); + DROUND(11); + DROUND(12); + DROUND(13); + DROUND(14); + DROUND(15); + DROUND(16); +} + +/** + Initialize an Sober128 context (only the key) + @param st [out] The destination of the Sober128 state + @param key The secret key + @param keylen The length of the secret key (octets) + @return CRYPT_OK if successful +*/ +int sober128_stream_setup(sober128_state *st, const unsigned char *key, unsigned long keylen) +{ + ulong32 i, k; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen > 0); + + /* keylen must be multiple of 4 bytes */ + if ((keylen & 3) != 0) { + return CRYPT_INVALID_KEYSIZE; + } + + /* Register initialised to Fibonacci numbers */ + st->R[0] = 1; + st->R[1] = 1; + for (i = 2; i < N; ++i) { + st->R[i] = st->R[i-1] + st->R[i-2]; + } + st->konst = INITKONST; + + for (i = 0; i < keylen; i += 4) { + k = BYTE2WORD((unsigned char *)&key[i]); + ADDKEY(k); + cycle(st->R); + XORNL(nltap(st)); + } + + /* also fold in the length of the key */ + ADDKEY(keylen); + + /* now diffuse */ + s128_diffuse(st); + s128_genkonst(st); + s128_savestate(st); + st->nbuf = 0; + + return CRYPT_OK; +} + +/** + Set IV to the Sober128 state + @param st The Sober12820 state + @param iv The IV data to add + @param ivlen The length of the IV (must be 12) + @return CRYPT_OK on success + */ +int sober128_stream_setiv(sober128_state *st, const unsigned char *iv, unsigned long ivlen) +{ + ulong32 i, k; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(iv != NULL); + LTC_ARGCHK(ivlen > 0); + + /* ok we are adding an IV then... */ + s128_reloadstate(st); + + /* ivlen must be multiple of 4 bytes */ + if ((ivlen & 3) != 0) { + return CRYPT_INVALID_KEYSIZE; + } + + for (i = 0; i < ivlen; i += 4) { + k = BYTE2WORD((unsigned char *)&iv[i]); + ADDKEY(k); + cycle(st->R); + XORNL(nltap(st)); + } + + /* also fold in the length of the key */ + ADDKEY(ivlen); + + /* now diffuse */ + s128_diffuse(st); + st->nbuf = 0; + + return CRYPT_OK; +} + +/* XOR pseudo-random bytes into buffer + */ +#define SROUND(z) STEP(st->R,z); NLFUNC(st,(z+1)); XORWORD(t, in+(z*4), out+(z*4)); + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with Sober128 + @param st The Sober128 state + @param in The plaintext (or ciphertext) + @param inlen The length of the input (octets) + @param out [out] The ciphertext (or plaintext), length inlen + @return CRYPT_OK if successful +*/ +int sober128_stream_crypt(sober128_state *st, const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + ulong32 t; + + if (inlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(out != NULL); + LTC_ARGCHK(st != NULL); + + /* handle any previously buffered bytes */ + while (st->nbuf != 0 && inlen != 0) { + *out++ = *in++ ^ (unsigned char)(st->sbuf & 0xFF); + st->sbuf >>= 8; + st->nbuf -= 8; + --inlen; + } + +#ifndef LTC_SMALL_CODE + /* do lots at a time, if there's enough to do */ + while (inlen >= N*4) { + SROUND(0); + SROUND(1); + SROUND(2); + SROUND(3); + SROUND(4); + SROUND(5); + SROUND(6); + SROUND(7); + SROUND(8); + SROUND(9); + SROUND(10); + SROUND(11); + SROUND(12); + SROUND(13); + SROUND(14); + SROUND(15); + SROUND(16); + out += 4*N; + in += 4*N; + inlen -= 4*N; + } +#endif + + /* do small or odd size buffers the slow way */ + while (4 <= inlen) { + cycle(st->R); + t = nltap(st); + XORWORD(t, in, out); + out += 4; + in += 4; + inlen -= 4; + } + + /* handle any trailing bytes */ + if (inlen != 0) { + cycle(st->R); + st->sbuf = nltap(st); + st->nbuf = 32; + while (st->nbuf != 0 && inlen != 0) { + *out++ = *in++ ^ (unsigned char)(st->sbuf & 0xFF); + st->sbuf >>= 8; + st->nbuf -= 8; + --inlen; + } + } + + return CRYPT_OK; +} + +int sober128_stream_keystream(sober128_state *st, unsigned char *out, unsigned long outlen) +{ + if (outlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(out != NULL); + XMEMSET(out, 0, outlen); + return sober128_stream_crypt(st, out, outlen, out); +} + +/** + Terminate and clear Sober128 state + @param st The Sober128 state + @return CRYPT_OK on success +*/ +int sober128_stream_done(sober128_state *st) +{ + LTC_ARGCHK(st != NULL); + zeromem(st, sizeof(sober128_state)); + return CRYPT_OK; +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_stream_memory.c b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_stream_memory.c new file mode 100644 index 0000000..5150b82 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_stream_memory.c @@ -0,0 +1,35 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SOBER128_STREAM + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with SOBER128 + @param key The key + @param keylen The key length + @param iv The initial vector + @param ivlen The initial vector length + @param datain The plaintext (or ciphertext) + @param datalen The length of the input and output (octets) + @param dataout [out] The ciphertext (or plaintext) + @return CRYPT_OK if successful +*/ +int sober128_stream_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout) +{ + sober128_state st; + int err; + + if ((err = sober128_stream_setup(&st, key, keylen)) != CRYPT_OK) goto WIPE_KEY; + if ((err = sober128_stream_setiv(&st, iv, ivlen)) != CRYPT_OK) goto WIPE_KEY; + err = sober128_stream_crypt(&st, datain, datalen, dataout); +WIPE_KEY: + sober128_stream_done(&st); + return err; +} + +#endif /* LTC_SOBER128_STREAM */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_test.c b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_test.c new file mode 100644 index 0000000..52ea3a8 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128_test.c @@ -0,0 +1,43 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SOBER128 + +int sober128_stream_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + unsigned char key[16] = { 0x74, 0x65, 0x73, 0x74, 0x20, 0x6b, 0x65, 0x79, + 0x20, 0x31, 0x32, 0x38, 0x62, 0x69, 0x74, 0x73 }; + unsigned char iv[4] = { 0x00, 0x00, 0x00, 0x00 }; + unsigned char out[20] = { 0x43, 0x50, 0x0c, 0xcf, 0x89, 0x91, 0x9f, 0x1d, + 0xaa, 0x37, 0x74, 0x95, 0xf4, 0xb4, 0x58, 0xc2, + 0x40, 0x37, 0x8b, 0xbb }; + int err, len = 20; + unsigned char src[20], dst[20]; + sober128_state st; + + XMEMSET(src, 0, len); /* input */ + if ((err = sober128_stream_setup(&st, key, sizeof(key))) != CRYPT_OK) return err; + if ((err = sober128_stream_setiv(&st, iv, sizeof(iv))) != CRYPT_OK) return err; + if ((err = sober128_stream_crypt(&st, src, len, dst)) != CRYPT_OK) return err; + if ((err = sober128_stream_done(&st)) != CRYPT_OK) return err; + if (compare_testvector(dst, len, out, len, "SOBER-128-TV1", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + /* crypt in a single call */ + if ((err = sober128_stream_memory(key, sizeof(key), iv, sizeof(iv), + src, len, dst)) != CRYPT_OK) return err; + if (compare_testvector(dst, len, out, len, "SOBER-128-TV2", 0)) { + return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128tab.c b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128tab.c new file mode 100644 index 0000000..e067bb6 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/sober128/sober128tab.c @@ -0,0 +1,166 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/** + @file sober128tab.c + SOBER-128 Tables +*/ + +#ifdef LTC_SOBER128TAB_C + +/* $ID$ */ +/* @(#)TuringMultab.h 1.3 (QUALCOMM) 02/09/03 */ +/* Multiplication table for Turing using 0xD02B4367 */ +static const ulong32 Multab[256] = { + 0x00000000, 0xD02B4367, 0xED5686CE, 0x3D7DC5A9, + 0x97AC41D1, 0x478702B6, 0x7AFAC71F, 0xAAD18478, + 0x631582EF, 0xB33EC188, 0x8E430421, 0x5E684746, + 0xF4B9C33E, 0x24928059, 0x19EF45F0, 0xC9C40697, + 0xC62A4993, 0x16010AF4, 0x2B7CCF5D, 0xFB578C3A, + 0x51860842, 0x81AD4B25, 0xBCD08E8C, 0x6CFBCDEB, + 0xA53FCB7C, 0x7514881B, 0x48694DB2, 0x98420ED5, + 0x32938AAD, 0xE2B8C9CA, 0xDFC50C63, 0x0FEE4F04, + 0xC154926B, 0x117FD10C, 0x2C0214A5, 0xFC2957C2, + 0x56F8D3BA, 0x86D390DD, 0xBBAE5574, 0x6B851613, + 0xA2411084, 0x726A53E3, 0x4F17964A, 0x9F3CD52D, + 0x35ED5155, 0xE5C61232, 0xD8BBD79B, 0x089094FC, + 0x077EDBF8, 0xD755989F, 0xEA285D36, 0x3A031E51, + 0x90D29A29, 0x40F9D94E, 0x7D841CE7, 0xADAF5F80, + 0x646B5917, 0xB4401A70, 0x893DDFD9, 0x59169CBE, + 0xF3C718C6, 0x23EC5BA1, 0x1E919E08, 0xCEBADD6F, + 0xCFA869D6, 0x1F832AB1, 0x22FEEF18, 0xF2D5AC7F, + 0x58042807, 0x882F6B60, 0xB552AEC9, 0x6579EDAE, + 0xACBDEB39, 0x7C96A85E, 0x41EB6DF7, 0x91C02E90, + 0x3B11AAE8, 0xEB3AE98F, 0xD6472C26, 0x066C6F41, + 0x09822045, 0xD9A96322, 0xE4D4A68B, 0x34FFE5EC, + 0x9E2E6194, 0x4E0522F3, 0x7378E75A, 0xA353A43D, + 0x6A97A2AA, 0xBABCE1CD, 0x87C12464, 0x57EA6703, + 0xFD3BE37B, 0x2D10A01C, 0x106D65B5, 0xC04626D2, + 0x0EFCFBBD, 0xDED7B8DA, 0xE3AA7D73, 0x33813E14, + 0x9950BA6C, 0x497BF90B, 0x74063CA2, 0xA42D7FC5, + 0x6DE97952, 0xBDC23A35, 0x80BFFF9C, 0x5094BCFB, + 0xFA453883, 0x2A6E7BE4, 0x1713BE4D, 0xC738FD2A, + 0xC8D6B22E, 0x18FDF149, 0x258034E0, 0xF5AB7787, + 0x5F7AF3FF, 0x8F51B098, 0xB22C7531, 0x62073656, + 0xABC330C1, 0x7BE873A6, 0x4695B60F, 0x96BEF568, + 0x3C6F7110, 0xEC443277, 0xD139F7DE, 0x0112B4B9, + 0xD31DD2E1, 0x03369186, 0x3E4B542F, 0xEE601748, + 0x44B19330, 0x949AD057, 0xA9E715FE, 0x79CC5699, + 0xB008500E, 0x60231369, 0x5D5ED6C0, 0x8D7595A7, + 0x27A411DF, 0xF78F52B8, 0xCAF29711, 0x1AD9D476, + 0x15379B72, 0xC51CD815, 0xF8611DBC, 0x284A5EDB, + 0x829BDAA3, 0x52B099C4, 0x6FCD5C6D, 0xBFE61F0A, + 0x7622199D, 0xA6095AFA, 0x9B749F53, 0x4B5FDC34, + 0xE18E584C, 0x31A51B2B, 0x0CD8DE82, 0xDCF39DE5, + 0x1249408A, 0xC26203ED, 0xFF1FC644, 0x2F348523, + 0x85E5015B, 0x55CE423C, 0x68B38795, 0xB898C4F2, + 0x715CC265, 0xA1778102, 0x9C0A44AB, 0x4C2107CC, + 0xE6F083B4, 0x36DBC0D3, 0x0BA6057A, 0xDB8D461D, + 0xD4630919, 0x04484A7E, 0x39358FD7, 0xE91ECCB0, + 0x43CF48C8, 0x93E40BAF, 0xAE99CE06, 0x7EB28D61, + 0xB7768BF6, 0x675DC891, 0x5A200D38, 0x8A0B4E5F, + 0x20DACA27, 0xF0F18940, 0xCD8C4CE9, 0x1DA70F8E, + 0x1CB5BB37, 0xCC9EF850, 0xF1E33DF9, 0x21C87E9E, + 0x8B19FAE6, 0x5B32B981, 0x664F7C28, 0xB6643F4F, + 0x7FA039D8, 0xAF8B7ABF, 0x92F6BF16, 0x42DDFC71, + 0xE80C7809, 0x38273B6E, 0x055AFEC7, 0xD571BDA0, + 0xDA9FF2A4, 0x0AB4B1C3, 0x37C9746A, 0xE7E2370D, + 0x4D33B375, 0x9D18F012, 0xA06535BB, 0x704E76DC, + 0xB98A704B, 0x69A1332C, 0x54DCF685, 0x84F7B5E2, + 0x2E26319A, 0xFE0D72FD, 0xC370B754, 0x135BF433, + 0xDDE1295C, 0x0DCA6A3B, 0x30B7AF92, 0xE09CECF5, + 0x4A4D688D, 0x9A662BEA, 0xA71BEE43, 0x7730AD24, + 0xBEF4ABB3, 0x6EDFE8D4, 0x53A22D7D, 0x83896E1A, + 0x2958EA62, 0xF973A905, 0xC40E6CAC, 0x14252FCB, + 0x1BCB60CF, 0xCBE023A8, 0xF69DE601, 0x26B6A566, + 0x8C67211E, 0x5C4C6279, 0x6131A7D0, 0xB11AE4B7, + 0x78DEE220, 0xA8F5A147, 0x958864EE, 0x45A32789, + 0xEF72A3F1, 0x3F59E096, 0x0224253F, 0xD20F6658, +}; + +/* $ID$ */ +/* Sbox for SOBER-128 */ +/* + * This is really the combination of two SBoxes; the least significant + * 24 bits comes from: + * 8->32 Sbox generated by Millan et. al. at Queensland University of + * Technology. See: E. Dawson, W. Millan, L. Burnett, G. Carter, + * "On the Design of 8*32 S-boxes". Unpublished report, by the + * Information Systems Research Centre, + * Queensland University of Technology, 1999. + * + * The most significant 8 bits are the Skipjack "F table", which can be + * found at http://csrc.nist.gov/CryptoToolkit/skipjack/skipjack.pdf . + * In this optimised table, though, the intent is to XOR the word from + * the table selected by the high byte with the input word. Thus, the + * high byte is actually the Skipjack F-table entry XORED with its + * table index. + */ +static const ulong32 Sbox[256] = { + 0xa3aa1887, 0xd65e435c, 0x0b65c042, 0x800e6ef4, + 0xfc57ee20, 0x4d84fed3, 0xf066c502, 0xf354e8ae, + 0xbb2ee9d9, 0x281f38d4, 0x1f829b5d, 0x735cdf3c, + 0x95864249, 0xbc2e3963, 0xa1f4429f, 0xf6432c35, + 0xf7f40325, 0x3cc0dd70, 0x5f973ded, 0x9902dc5e, + 0xda175b42, 0x590012bf, 0xdc94d78c, 0x39aab26b, + 0x4ac11b9a, 0x8c168146, 0xc3ea8ec5, 0x058ac28f, + 0x52ed5c0f, 0x25b4101c, 0x5a2db082, 0x370929e1, + 0x2a1843de, 0xfe8299fc, 0x202fbc4b, 0x833915dd, + 0x33a803fa, 0xd446b2de, 0x46233342, 0x4fcee7c3, + 0x3ad607ef, 0x9e97ebab, 0x507f859b, 0xe81f2e2f, + 0xc55b71da, 0xd7e2269a, 0x1339c3d1, 0x7ca56b36, + 0xa6c9def2, 0xb5c9fc5f, 0x5927b3a3, 0x89a56ddf, + 0xc625b510, 0x560f85a7, 0xace82e71, 0x2ecb8816, + 0x44951e2a, 0x97f5f6af, 0xdfcbc2b3, 0xce4ff55d, + 0xcb6b6214, 0x2b0b83e3, 0x549ea6f5, 0x9de041af, + 0x792f1f17, 0xf73b99ee, 0x39a65ec0, 0x4c7016c6, + 0x857709a4, 0xd6326e01, 0xc7b280d9, 0x5cfb1418, + 0xa6aff227, 0xfd548203, 0x506b9d96, 0xa117a8c0, + 0x9cd5bf6e, 0xdcee7888, 0x61fcfe64, 0xf7a193cd, + 0x050d0184, 0xe8ae4930, 0x88014f36, 0xd6a87088, + 0x6bad6c2a, 0x1422c678, 0xe9204de7, 0xb7c2e759, + 0x0200248e, 0x013b446b, 0xda0d9fc2, 0x0414a895, + 0x3a6cc3a1, 0x56fef170, 0x86c19155, 0xcf7b8a66, + 0x551b5e69, 0xb4a8623e, 0xa2bdfa35, 0xc4f068cc, + 0x573a6acd, 0x6355e936, 0x03602db9, 0x0edf13c1, + 0x2d0bb16d, 0x6980b83c, 0xfeb23763, 0x3dd8a911, + 0x01b6bc13, 0xf55579d7, 0xf55c2fa8, 0x19f4196e, + 0xe7db5476, 0x8d64a866, 0xc06e16ad, 0xb17fc515, + 0xc46feb3c, 0x8bc8a306, 0xad6799d9, 0x571a9133, + 0x992466dd, 0x92eb5dcd, 0xac118f50, 0x9fafb226, + 0xa1b9cef3, 0x3ab36189, 0x347a19b1, 0x62c73084, + 0xc27ded5c, 0x6c8bc58f, 0x1cdde421, 0xed1e47fb, + 0xcdcc715e, 0xb9c0ff99, 0x4b122f0f, 0xc4d25184, + 0xaf7a5e6c, 0x5bbf18bc, 0x8dd7c6e0, 0x5fb7e420, + 0x521f523f, 0x4ad9b8a2, 0xe9da1a6b, 0x97888c02, + 0x19d1e354, 0x5aba7d79, 0xa2cc7753, 0x8c2d9655, + 0x19829da1, 0x531590a7, 0x19c1c149, 0x3d537f1c, + 0x50779b69, 0xed71f2b7, 0x463c58fa, 0x52dc4418, + 0xc18c8c76, 0xc120d9f0, 0xafa80d4d, 0x3b74c473, + 0xd09410e9, 0x290e4211, 0xc3c8082b, 0x8f6b334a, + 0x3bf68ed2, 0xa843cc1b, 0x8d3c0ff3, 0x20e564a0, + 0xf8f55a4f, 0x2b40f8e7, 0xfea7f15f, 0xcf00fe21, + 0x8a6d37d6, 0xd0d506f1, 0xade00973, 0xefbbde36, + 0x84670fa8, 0xfa31ab9e, 0xaedab618, 0xc01f52f5, + 0x6558eb4f, 0x71b9e343, 0x4b8d77dd, 0x8cb93da6, + 0x740fd52d, 0x425412f8, 0xc5a63360, 0x10e53ad0, + 0x5a700f1c, 0x8324ed0b, 0xe53dc1ec, 0x1a366795, + 0x6d549d15, 0xc5ce46d7, 0xe17abe76, 0x5f48e0a0, + 0xd0f07c02, 0x941249b7, 0xe49ed6ba, 0x37a47f78, + 0xe1cfffbd, 0xb007ca84, 0xbb65f4da, 0xb59f35da, + 0x33d2aa44, 0x417452ac, 0xc0d674a7, 0x2d61a46a, + 0xdc63152a, 0x3e12b7aa, 0x6e615927, 0xa14fb118, + 0xa151758d, 0xba81687b, 0xe152f0b3, 0x764254ed, + 0x34c77271, 0x0a31acab, 0x54f94aec, 0xb9e994cd, + 0x574d9e81, 0x5b623730, 0xce8a21e8, 0x37917f0b, + 0xe8a9b5d6, 0x9697adf8, 0xf3d30431, 0x5dcac921, + 0x76b35d46, 0xaa430a36, 0xc2194022, 0x22bca65e, + 0xdaec70ba, 0xdfaea8cc, 0x777bae8b, 0x242924d5, + 0x1f098a5a, 0x4b396b81, 0x55de2522, 0x435c1cb8, + 0xaeb8fe1d, 0x9db3c697, 0x5b164f83, 0xe0c16376, + 0xa319224c, 0xd0203b35, 0x433ac0fe, 0x1466a19a, + 0x45f0b24f, 0x51fda998, 0xc0d52d71, 0xfa0896a8, + 0xf9e6053f, 0xa4b0d300, 0xd499cbcc, 0xb95e3d40, +}; + +#endif /* LTC_SOBER128TAB_C */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk.c b/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk.c new file mode 100644 index 0000000..6733450 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk.c @@ -0,0 +1,807 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +/* + * This LTC implementation was adapted from: + * http://www.ecrypt.eu.org/stream/e2-sosemanuk.html + */ + +/* + * SOSEMANUK reference implementation. + * + * This code is supposed to run on any conforming C implementation (C90 + * or later). + * + * (c) 2005 X-CRYPT project. This software is provided 'as-is', without + * any express or implied warranty. In no event will the authors be held + * liable for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to no restriction. + * + * Technical remarks and questions can be addressed to + * + */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SOSEMANUK + +/* ======================================================================== */ + +/* + * We want (and sometimes need) to perform explicit truncations to 32 bits. + */ +#define T32(x) ((x) & (ulong32)0xFFFFFFFF) + +/* + * Some of our functions will be tagged as "inline" to help the compiler + * optimize things. We use "inline" only if the compiler is advanced + * enough to understand it; C99 compilers, and pre-C99 versions of gcc, + * understand enough "inline" for our purposes. + */ + +/* ======================================================================== */ + +/* + * Serpent S-boxes, implemented in bitslice mode. These circuits have + * been published by Dag Arne Osvik ("Speeding up Serpent", published in + * the 3rd AES Candidate Conference) and work on five 32-bit registers: + * the four inputs, and a fifth scratch register. There are meant to be + * quite fast on Pentium-class processors. These are not the fastest + * published, but they are "fast enough" and they are unencumbered as + * far as intellectual property is concerned (note: these are rewritten + * from the article itself, and hence are not covered by the GPL on + * Dag's code, which was not used here). + * + * The output bits are permuted. Here is the correspondance: + * S0: 1420 + * S1: 2031 + * S2: 2314 + * S3: 1234 + * S4: 1403 + * S5: 1302 + * S6: 0142 + * S7: 4310 + * (for instance, the output of S0 is in "r1, r4, r2, r0"). + */ + +#define S0(r0, r1, r2, r3, r4) do { \ + r3 ^= r0; r4 = r1; \ + r1 &= r3; r4 ^= r2; \ + r1 ^= r0; r0 |= r3; \ + r0 ^= r4; r4 ^= r3; \ + r3 ^= r2; r2 |= r1; \ + r2 ^= r4; r4 = ~r4; \ + r4 |= r1; r1 ^= r3; \ + r1 ^= r4; r3 |= r0; \ + r1 ^= r3; r4 ^= r3; \ + } while (0) + +#define S1(r0, r1, r2, r3, r4) do { \ + r0 = ~r0; r2 = ~r2; \ + r4 = r0; r0 &= r1; \ + r2 ^= r0; r0 |= r3; \ + r3 ^= r2; r1 ^= r0; \ + r0 ^= r4; r4 |= r1; \ + r1 ^= r3; r2 |= r0; \ + r2 &= r4; r0 ^= r1; \ + r1 &= r2; \ + r1 ^= r0; r0 &= r2; \ + r0 ^= r4; \ + } while (0) + +#define S2(r0, r1, r2, r3, r4) do { \ + r4 = r0; r0 &= r2; \ + r0 ^= r3; r2 ^= r1; \ + r2 ^= r0; r3 |= r4; \ + r3 ^= r1; r4 ^= r2; \ + r1 = r3; r3 |= r4; \ + r3 ^= r0; r0 &= r1; \ + r4 ^= r0; r1 ^= r3; \ + r1 ^= r4; r4 = ~r4; \ + } while (0) + +#define S3(r0, r1, r2, r3, r4) do { \ + r4 = r0; r0 |= r3; \ + r3 ^= r1; r1 &= r4; \ + r4 ^= r2; r2 ^= r3; \ + r3 &= r0; r4 |= r1; \ + r3 ^= r4; r0 ^= r1; \ + r4 &= r0; r1 ^= r3; \ + r4 ^= r2; r1 |= r0; \ + r1 ^= r2; r0 ^= r3; \ + r2 = r1; r1 |= r3; \ + r1 ^= r0; \ + } while (0) + +#define S4(r0, r1, r2, r3, r4) do { \ + r1 ^= r3; r3 = ~r3; \ + r2 ^= r3; r3 ^= r0; \ + r4 = r1; r1 &= r3; \ + r1 ^= r2; r4 ^= r3; \ + r0 ^= r4; r2 &= r4; \ + r2 ^= r0; r0 &= r1; \ + r3 ^= r0; r4 |= r1; \ + r4 ^= r0; r0 |= r3; \ + r0 ^= r2; r2 &= r3; \ + r0 = ~r0; r4 ^= r2; \ + } while (0) + +#define S5(r0, r1, r2, r3, r4) do { \ + r0 ^= r1; r1 ^= r3; \ + r3 = ~r3; r4 = r1; \ + r1 &= r0; r2 ^= r3; \ + r1 ^= r2; r2 |= r4; \ + r4 ^= r3; r3 &= r1; \ + r3 ^= r0; r4 ^= r1; \ + r4 ^= r2; r2 ^= r0; \ + r0 &= r3; r2 = ~r2; \ + r0 ^= r4; r4 |= r3; \ + r2 ^= r4; \ + } while (0) + +#define S6(r0, r1, r2, r3, r4) do { \ + r2 = ~r2; r4 = r3; \ + r3 &= r0; r0 ^= r4; \ + r3 ^= r2; r2 |= r4; \ + r1 ^= r3; r2 ^= r0; \ + r0 |= r1; r2 ^= r1; \ + r4 ^= r0; r0 |= r3; \ + r0 ^= r2; r4 ^= r3; \ + r4 ^= r0; r3 = ~r3; \ + r2 &= r4; \ + r2 ^= r3; \ + } while (0) + +#define S7(r0, r1, r2, r3, r4) do { \ + r4 = r1; r1 |= r2; \ + r1 ^= r3; r4 ^= r2; \ + r2 ^= r1; r3 |= r4; \ + r3 &= r0; r4 ^= r2; \ + r3 ^= r1; r1 |= r4; \ + r1 ^= r0; r0 |= r4; \ + r0 ^= r2; r1 ^= r4; \ + r2 ^= r1; r1 &= r0; \ + r1 ^= r4; r2 = ~r2; \ + r2 |= r0; \ + r4 ^= r2; \ + } while (0) + +/* + * The Serpent linear transform. + */ +#define SERPENT_LT(x0, x1, x2, x3) do { \ + x0 = ROLc(x0, 13); \ + x2 = ROLc(x2, 3); \ + x1 = x1 ^ x0 ^ x2; \ + x3 = x3 ^ x2 ^ T32(x0 << 3); \ + x1 = ROLc(x1, 1); \ + x3 = ROLc(x3, 7); \ + x0 = x0 ^ x1 ^ x3; \ + x2 = x2 ^ x3 ^ T32(x1 << 7); \ + x0 = ROLc(x0, 5); \ + x2 = ROLc(x2, 22); \ + } while (0) + +/* ======================================================================== */ + +/* + * Initialize Sosemanuk's state by providing a key. The key is an array of + * 1 to 32 bytes. + * @param st The Sosemanuk state + * @param key Key + * @param keylen Length of key in bytes + * @return CRYPT_OK on success + */ +int sosemanuk_setup(sosemanuk_state *st, const unsigned char *key, unsigned long keylen) +{ + /* + * This key schedule is actually a truncated Serpent key schedule. + * The key-derived words (w_i) are computed within the eight + * local variables w0 to w7, which are reused again and again. + */ + +#define SKS(S, o0, o1, o2, o3, d0, d1, d2, d3) do { \ + ulong32 r0, r1, r2, r3, r4; \ + r0 = w ## o0; \ + r1 = w ## o1; \ + r2 = w ## o2; \ + r3 = w ## o3; \ + S(r0, r1, r2, r3, r4); \ + st->kc[i ++] = r ## d0; \ + st->kc[i ++] = r ## d1; \ + st->kc[i ++] = r ## d2; \ + st->kc[i ++] = r ## d3; \ + } while (0) + +#define SKS0 SKS(S0, 4, 5, 6, 7, 1, 4, 2, 0) +#define SKS1 SKS(S1, 0, 1, 2, 3, 2, 0, 3, 1) +#define SKS2 SKS(S2, 4, 5, 6, 7, 2, 3, 1, 4) +#define SKS3 SKS(S3, 0, 1, 2, 3, 1, 2, 3, 4) +#define SKS4 SKS(S4, 4, 5, 6, 7, 1, 4, 0, 3) +#define SKS5 SKS(S5, 0, 1, 2, 3, 1, 3, 0, 2) +#define SKS6 SKS(S6, 4, 5, 6, 7, 0, 1, 4, 2) +#define SKS7 SKS(S7, 0, 1, 2, 3, 4, 3, 1, 0) + +#define WUP(wi, wi5, wi3, wi1, cc) do { \ + ulong32 tt = (wi) ^ (wi5) ^ (wi3) \ + ^ (wi1) ^ (0x9E3779B9 ^ (ulong32)(cc)); \ + (wi) = ROLc(tt, 11); \ + } while (0) + +#define WUP0(cc) do { \ + WUP(w0, w3, w5, w7, cc); \ + WUP(w1, w4, w6, w0, cc + 1); \ + WUP(w2, w5, w7, w1, cc + 2); \ + WUP(w3, w6, w0, w2, cc + 3); \ + } while (0) + +#define WUP1(cc) do { \ + WUP(w4, w7, w1, w3, cc); \ + WUP(w5, w0, w2, w4, cc + 1); \ + WUP(w6, w1, w3, w5, cc + 2); \ + WUP(w7, w2, w4, w6, cc + 3); \ + } while (0) + + unsigned char wbuf[32]; + ulong32 w0, w1, w2, w3, w4, w5, w6, w7; + int i = 0; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(key != NULL); + LTC_ARGCHK(keylen > 0 && keylen <= 32); + + /* + * The key is copied into the wbuf[] buffer and padded to 256 bits + * as described in the Serpent specification. + */ + XMEMCPY(wbuf, key, keylen); + if (keylen < 32) { + wbuf[keylen] = 0x01; + if (keylen < 31) { + XMEMSET(wbuf + keylen + 1, 0, 31 - keylen); + } + } + + LOAD32L(w0, wbuf); + LOAD32L(w1, wbuf + 4); + LOAD32L(w2, wbuf + 8); + LOAD32L(w3, wbuf + 12); + LOAD32L(w4, wbuf + 16); + LOAD32L(w5, wbuf + 20); + LOAD32L(w6, wbuf + 24); + LOAD32L(w7, wbuf + 28); + + WUP0(0); SKS3; + WUP1(4); SKS2; + WUP0(8); SKS1; + WUP1(12); SKS0; + WUP0(16); SKS7; + WUP1(20); SKS6; + WUP0(24); SKS5; + WUP1(28); SKS4; + WUP0(32); SKS3; + WUP1(36); SKS2; + WUP0(40); SKS1; + WUP1(44); SKS0; + WUP0(48); SKS7; + WUP1(52); SKS6; + WUP0(56); SKS5; + WUP1(60); SKS4; + WUP0(64); SKS3; + WUP1(68); SKS2; + WUP0(72); SKS1; + WUP1(76); SKS0; + WUP0(80); SKS7; + WUP1(84); SKS6; + WUP0(88); SKS5; + WUP1(92); SKS4; + WUP0(96); SKS3; + +#undef SKS +#undef SKS0 +#undef SKS1 +#undef SKS2 +#undef SKS3 +#undef SKS4 +#undef SKS5 +#undef SKS6 +#undef SKS7 +#undef WUP +#undef WUP0 +#undef WUP1 + + return CRYPT_OK; +} + + +/* + * Initialization continues by setting the IV. The IV length is up to 16 bytes. + * If "ivlen" is 0 (no IV), then the "iv" parameter can be NULL. If multiple + * encryptions/decryptions are to be performed with the same key and + * sosemanuk_done() has not been called, only sosemanuk_setiv() need be called + * to set the state. + * @param st The Sosemanuk state + * @param iv Initialization vector + * @param ivlen Length of iv in bytes + * @return CRYPT_OK on success + */ +int sosemanuk_setiv(sosemanuk_state *st, const unsigned char *iv, unsigned long ivlen) +{ + + /* + * The Serpent key addition step. + */ +#define KA(zc, x0, x1, x2, x3) do { \ + x0 ^= st->kc[(zc)]; \ + x1 ^= st->kc[(zc) + 1]; \ + x2 ^= st->kc[(zc) + 2]; \ + x3 ^= st->kc[(zc) + 3]; \ + } while (0) + + /* + * One Serpent round. + * zc = current subkey counter + * S = S-box macro for this round + * i0 to i4 = input register numbers (the fifth is a scratch register) + * o0 to o3 = output register numbers + */ +#define FSS(zc, S, i0, i1, i2, i3, i4, o0, o1, o2, o3) do { \ + KA(zc, r ## i0, r ## i1, r ## i2, r ## i3); \ + S(r ## i0, r ## i1, r ## i2, r ## i3, r ## i4); \ + SERPENT_LT(r ## o0, r ## o1, r ## o2, r ## o3); \ + } while (0) + + /* + * Last Serpent round. Contrary to the "true" Serpent, we keep + * the linear transformation for that last round. + */ +#define FSF(zc, S, i0, i1, i2, i3, i4, o0, o1, o2, o3) do { \ + KA(zc, r ## i0, r ## i1, r ## i2, r ## i3); \ + S(r ## i0, r ## i1, r ## i2, r ## i3, r ## i4); \ + SERPENT_LT(r ## o0, r ## o1, r ## o2, r ## o3); \ + KA(zc + 4, r ## o0, r ## o1, r ## o2, r ## o3); \ + } while (0) + + ulong32 r0, r1, r2, r3, r4; + unsigned char ivtmp[16] = {0}; + + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(ivlen <= 16); + LTC_ARGCHK(iv != NULL || ivlen == 0); + + if (ivlen > 0) XMEMCPY(ivtmp, iv, ivlen); + + /* + * Decode IV into four 32-bit words (little-endian). + */ + LOAD32L(r0, ivtmp); + LOAD32L(r1, ivtmp + 4); + LOAD32L(r2, ivtmp + 8); + LOAD32L(r3, ivtmp + 12); + + /* + * Encrypt IV with Serpent24. Some values are extracted from the + * output of the twelfth, eighteenth and twenty-fourth rounds. + */ + FSS(0, S0, 0, 1, 2, 3, 4, 1, 4, 2, 0); + FSS(4, S1, 1, 4, 2, 0, 3, 2, 1, 0, 4); + FSS(8, S2, 2, 1, 0, 4, 3, 0, 4, 1, 3); + FSS(12, S3, 0, 4, 1, 3, 2, 4, 1, 3, 2); + FSS(16, S4, 4, 1, 3, 2, 0, 1, 0, 4, 2); + FSS(20, S5, 1, 0, 4, 2, 3, 0, 2, 1, 4); + FSS(24, S6, 0, 2, 1, 4, 3, 0, 2, 3, 1); + FSS(28, S7, 0, 2, 3, 1, 4, 4, 1, 2, 0); + FSS(32, S0, 4, 1, 2, 0, 3, 1, 3, 2, 4); + FSS(36, S1, 1, 3, 2, 4, 0, 2, 1, 4, 3); + FSS(40, S2, 2, 1, 4, 3, 0, 4, 3, 1, 0); + FSS(44, S3, 4, 3, 1, 0, 2, 3, 1, 0, 2); + st->s09 = r3; + st->s08 = r1; + st->s07 = r0; + st->s06 = r2; + + FSS(48, S4, 3, 1, 0, 2, 4, 1, 4, 3, 2); + FSS(52, S5, 1, 4, 3, 2, 0, 4, 2, 1, 3); + FSS(56, S6, 4, 2, 1, 3, 0, 4, 2, 0, 1); + FSS(60, S7, 4, 2, 0, 1, 3, 3, 1, 2, 4); + FSS(64, S0, 3, 1, 2, 4, 0, 1, 0, 2, 3); + FSS(68, S1, 1, 0, 2, 3, 4, 2, 1, 3, 0); + st->r1 = r2; + st->s04 = r1; + st->r2 = r3; + st->s05 = r0; + + FSS(72, S2, 2, 1, 3, 0, 4, 3, 0, 1, 4); + FSS(76, S3, 3, 0, 1, 4, 2, 0, 1, 4, 2); + FSS(80, S4, 0, 1, 4, 2, 3, 1, 3, 0, 2); + FSS(84, S5, 1, 3, 0, 2, 4, 3, 2, 1, 0); + FSS(88, S6, 3, 2, 1, 0, 4, 3, 2, 4, 1); + FSF(92, S7, 3, 2, 4, 1, 0, 0, 1, 2, 3); + st->s03 = r0; + st->s02 = r1; + st->s01 = r2; + st->s00 = r3; + + st->ptr = sizeof(st->buf); + +#undef KA +#undef FSS +#undef FSF + + return CRYPT_OK; +} + +/* + * Multiplication by alpha: alpha * x = T32(x << 8) ^ mul_a[x >> 24] + */ +static const ulong32 mul_a[] = { + 0x00000000, 0xE19FCF13, 0x6B973726, 0x8A08F835, + 0xD6876E4C, 0x3718A15F, 0xBD10596A, 0x5C8F9679, + 0x05A7DC98, 0xE438138B, 0x6E30EBBE, 0x8FAF24AD, + 0xD320B2D4, 0x32BF7DC7, 0xB8B785F2, 0x59284AE1, + 0x0AE71199, 0xEB78DE8A, 0x617026BF, 0x80EFE9AC, + 0xDC607FD5, 0x3DFFB0C6, 0xB7F748F3, 0x566887E0, + 0x0F40CD01, 0xEEDF0212, 0x64D7FA27, 0x85483534, + 0xD9C7A34D, 0x38586C5E, 0xB250946B, 0x53CF5B78, + 0x1467229B, 0xF5F8ED88, 0x7FF015BD, 0x9E6FDAAE, + 0xC2E04CD7, 0x237F83C4, 0xA9777BF1, 0x48E8B4E2, + 0x11C0FE03, 0xF05F3110, 0x7A57C925, 0x9BC80636, + 0xC747904F, 0x26D85F5C, 0xACD0A769, 0x4D4F687A, + 0x1E803302, 0xFF1FFC11, 0x75170424, 0x9488CB37, + 0xC8075D4E, 0x2998925D, 0xA3906A68, 0x420FA57B, + 0x1B27EF9A, 0xFAB82089, 0x70B0D8BC, 0x912F17AF, + 0xCDA081D6, 0x2C3F4EC5, 0xA637B6F0, 0x47A879E3, + 0x28CE449F, 0xC9518B8C, 0x435973B9, 0xA2C6BCAA, + 0xFE492AD3, 0x1FD6E5C0, 0x95DE1DF5, 0x7441D2E6, + 0x2D699807, 0xCCF65714, 0x46FEAF21, 0xA7616032, + 0xFBEEF64B, 0x1A713958, 0x9079C16D, 0x71E60E7E, + 0x22295506, 0xC3B69A15, 0x49BE6220, 0xA821AD33, + 0xF4AE3B4A, 0x1531F459, 0x9F390C6C, 0x7EA6C37F, + 0x278E899E, 0xC611468D, 0x4C19BEB8, 0xAD8671AB, + 0xF109E7D2, 0x109628C1, 0x9A9ED0F4, 0x7B011FE7, + 0x3CA96604, 0xDD36A917, 0x573E5122, 0xB6A19E31, + 0xEA2E0848, 0x0BB1C75B, 0x81B93F6E, 0x6026F07D, + 0x390EBA9C, 0xD891758F, 0x52998DBA, 0xB30642A9, + 0xEF89D4D0, 0x0E161BC3, 0x841EE3F6, 0x65812CE5, + 0x364E779D, 0xD7D1B88E, 0x5DD940BB, 0xBC468FA8, + 0xE0C919D1, 0x0156D6C2, 0x8B5E2EF7, 0x6AC1E1E4, + 0x33E9AB05, 0xD2766416, 0x587E9C23, 0xB9E15330, + 0xE56EC549, 0x04F10A5A, 0x8EF9F26F, 0x6F663D7C, + 0x50358897, 0xB1AA4784, 0x3BA2BFB1, 0xDA3D70A2, + 0x86B2E6DB, 0x672D29C8, 0xED25D1FD, 0x0CBA1EEE, + 0x5592540F, 0xB40D9B1C, 0x3E056329, 0xDF9AAC3A, + 0x83153A43, 0x628AF550, 0xE8820D65, 0x091DC276, + 0x5AD2990E, 0xBB4D561D, 0x3145AE28, 0xD0DA613B, + 0x8C55F742, 0x6DCA3851, 0xE7C2C064, 0x065D0F77, + 0x5F754596, 0xBEEA8A85, 0x34E272B0, 0xD57DBDA3, + 0x89F22BDA, 0x686DE4C9, 0xE2651CFC, 0x03FAD3EF, + 0x4452AA0C, 0xA5CD651F, 0x2FC59D2A, 0xCE5A5239, + 0x92D5C440, 0x734A0B53, 0xF942F366, 0x18DD3C75, + 0x41F57694, 0xA06AB987, 0x2A6241B2, 0xCBFD8EA1, + 0x977218D8, 0x76EDD7CB, 0xFCE52FFE, 0x1D7AE0ED, + 0x4EB5BB95, 0xAF2A7486, 0x25228CB3, 0xC4BD43A0, + 0x9832D5D9, 0x79AD1ACA, 0xF3A5E2FF, 0x123A2DEC, + 0x4B12670D, 0xAA8DA81E, 0x2085502B, 0xC11A9F38, + 0x9D950941, 0x7C0AC652, 0xF6023E67, 0x179DF174, + 0x78FBCC08, 0x9964031B, 0x136CFB2E, 0xF2F3343D, + 0xAE7CA244, 0x4FE36D57, 0xC5EB9562, 0x24745A71, + 0x7D5C1090, 0x9CC3DF83, 0x16CB27B6, 0xF754E8A5, + 0xABDB7EDC, 0x4A44B1CF, 0xC04C49FA, 0x21D386E9, + 0x721CDD91, 0x93831282, 0x198BEAB7, 0xF81425A4, + 0xA49BB3DD, 0x45047CCE, 0xCF0C84FB, 0x2E934BE8, + 0x77BB0109, 0x9624CE1A, 0x1C2C362F, 0xFDB3F93C, + 0xA13C6F45, 0x40A3A056, 0xCAAB5863, 0x2B349770, + 0x6C9CEE93, 0x8D032180, 0x070BD9B5, 0xE69416A6, + 0xBA1B80DF, 0x5B844FCC, 0xD18CB7F9, 0x301378EA, + 0x693B320B, 0x88A4FD18, 0x02AC052D, 0xE333CA3E, + 0xBFBC5C47, 0x5E239354, 0xD42B6B61, 0x35B4A472, + 0x667BFF0A, 0x87E43019, 0x0DECC82C, 0xEC73073F, + 0xB0FC9146, 0x51635E55, 0xDB6BA660, 0x3AF46973, + 0x63DC2392, 0x8243EC81, 0x084B14B4, 0xE9D4DBA7, + 0xB55B4DDE, 0x54C482CD, 0xDECC7AF8, 0x3F53B5EB +}; + +/* + * Multiplication by 1/alpha: 1/alpha * x = (x >> 8) ^ mul_ia[x & 0xFF] + */ +static const ulong32 mul_ia[] = { + 0x00000000, 0x180F40CD, 0x301E8033, 0x2811C0FE, + 0x603CA966, 0x7833E9AB, 0x50222955, 0x482D6998, + 0xC078FBCC, 0xD877BB01, 0xF0667BFF, 0xE8693B32, + 0xA04452AA, 0xB84B1267, 0x905AD299, 0x88559254, + 0x29F05F31, 0x31FF1FFC, 0x19EEDF02, 0x01E19FCF, + 0x49CCF657, 0x51C3B69A, 0x79D27664, 0x61DD36A9, + 0xE988A4FD, 0xF187E430, 0xD99624CE, 0xC1996403, + 0x89B40D9B, 0x91BB4D56, 0xB9AA8DA8, 0xA1A5CD65, + 0x5249BE62, 0x4A46FEAF, 0x62573E51, 0x7A587E9C, + 0x32751704, 0x2A7A57C9, 0x026B9737, 0x1A64D7FA, + 0x923145AE, 0x8A3E0563, 0xA22FC59D, 0xBA208550, + 0xF20DECC8, 0xEA02AC05, 0xC2136CFB, 0xDA1C2C36, + 0x7BB9E153, 0x63B6A19E, 0x4BA76160, 0x53A821AD, + 0x1B854835, 0x038A08F8, 0x2B9BC806, 0x339488CB, + 0xBBC11A9F, 0xA3CE5A52, 0x8BDF9AAC, 0x93D0DA61, + 0xDBFDB3F9, 0xC3F2F334, 0xEBE333CA, 0xF3EC7307, + 0xA492D5C4, 0xBC9D9509, 0x948C55F7, 0x8C83153A, + 0xC4AE7CA2, 0xDCA13C6F, 0xF4B0FC91, 0xECBFBC5C, + 0x64EA2E08, 0x7CE56EC5, 0x54F4AE3B, 0x4CFBEEF6, + 0x04D6876E, 0x1CD9C7A3, 0x34C8075D, 0x2CC74790, + 0x8D628AF5, 0x956DCA38, 0xBD7C0AC6, 0xA5734A0B, + 0xED5E2393, 0xF551635E, 0xDD40A3A0, 0xC54FE36D, + 0x4D1A7139, 0x551531F4, 0x7D04F10A, 0x650BB1C7, + 0x2D26D85F, 0x35299892, 0x1D38586C, 0x053718A1, + 0xF6DB6BA6, 0xEED42B6B, 0xC6C5EB95, 0xDECAAB58, + 0x96E7C2C0, 0x8EE8820D, 0xA6F942F3, 0xBEF6023E, + 0x36A3906A, 0x2EACD0A7, 0x06BD1059, 0x1EB25094, + 0x569F390C, 0x4E9079C1, 0x6681B93F, 0x7E8EF9F2, + 0xDF2B3497, 0xC724745A, 0xEF35B4A4, 0xF73AF469, + 0xBF179DF1, 0xA718DD3C, 0x8F091DC2, 0x97065D0F, + 0x1F53CF5B, 0x075C8F96, 0x2F4D4F68, 0x37420FA5, + 0x7F6F663D, 0x676026F0, 0x4F71E60E, 0x577EA6C3, + 0xE18D0321, 0xF98243EC, 0xD1938312, 0xC99CC3DF, + 0x81B1AA47, 0x99BEEA8A, 0xB1AF2A74, 0xA9A06AB9, + 0x21F5F8ED, 0x39FAB820, 0x11EB78DE, 0x09E43813, + 0x41C9518B, 0x59C61146, 0x71D7D1B8, 0x69D89175, + 0xC87D5C10, 0xD0721CDD, 0xF863DC23, 0xE06C9CEE, + 0xA841F576, 0xB04EB5BB, 0x985F7545, 0x80503588, + 0x0805A7DC, 0x100AE711, 0x381B27EF, 0x20146722, + 0x68390EBA, 0x70364E77, 0x58278E89, 0x4028CE44, + 0xB3C4BD43, 0xABCBFD8E, 0x83DA3D70, 0x9BD57DBD, + 0xD3F81425, 0xCBF754E8, 0xE3E69416, 0xFBE9D4DB, + 0x73BC468F, 0x6BB30642, 0x43A2C6BC, 0x5BAD8671, + 0x1380EFE9, 0x0B8FAF24, 0x239E6FDA, 0x3B912F17, + 0x9A34E272, 0x823BA2BF, 0xAA2A6241, 0xB225228C, + 0xFA084B14, 0xE2070BD9, 0xCA16CB27, 0xD2198BEA, + 0x5A4C19BE, 0x42435973, 0x6A52998D, 0x725DD940, + 0x3A70B0D8, 0x227FF015, 0x0A6E30EB, 0x12617026, + 0x451FD6E5, 0x5D109628, 0x750156D6, 0x6D0E161B, + 0x25237F83, 0x3D2C3F4E, 0x153DFFB0, 0x0D32BF7D, + 0x85672D29, 0x9D686DE4, 0xB579AD1A, 0xAD76EDD7, + 0xE55B844F, 0xFD54C482, 0xD545047C, 0xCD4A44B1, + 0x6CEF89D4, 0x74E0C919, 0x5CF109E7, 0x44FE492A, + 0x0CD320B2, 0x14DC607F, 0x3CCDA081, 0x24C2E04C, + 0xAC977218, 0xB49832D5, 0x9C89F22B, 0x8486B2E6, + 0xCCABDB7E, 0xD4A49BB3, 0xFCB55B4D, 0xE4BA1B80, + 0x17566887, 0x0F59284A, 0x2748E8B4, 0x3F47A879, + 0x776AC1E1, 0x6F65812C, 0x477441D2, 0x5F7B011F, + 0xD72E934B, 0xCF21D386, 0xE7301378, 0xFF3F53B5, + 0xB7123A2D, 0xAF1D7AE0, 0x870CBA1E, 0x9F03FAD3, + 0x3EA637B6, 0x26A9777B, 0x0EB8B785, 0x16B7F748, + 0x5E9A9ED0, 0x4695DE1D, 0x6E841EE3, 0x768B5E2E, + 0xFEDECC7A, 0xE6D18CB7, 0xCEC04C49, 0xD6CF0C84, + 0x9EE2651C, 0x86ED25D1, 0xAEFCE52F, 0xB6F3A5E2 +}; + + +/* + * Compute the next block of bits of output stream. This is equivalent + * to one full rotation of the shift register. + */ +static LTC_INLINE void s_sosemanuk_internal(sosemanuk_state *st) +{ + /* + * MUL_A(x) computes alpha * x (in F_{2^32}). + * MUL_G(x) computes 1/alpha * x (in F_{2^32}). + */ +#define MUL_A(x) (T32((x) << 8) ^ mul_a[(x) >> 24]) +#define MUL_G(x) (((x) >> 8) ^ mul_ia[(x) & 0xFF]) + + /* + * This macro computes the special multiplexer, which chooses + * between "x" and "x xor y", depending on the least significant + * bit of the control word. We use the C "?:" selection operator + * (which most compilers know how to optimise) except for Alpha, + * where the manual sign extension seems to perform equally well + * with DEC/Compaq/HP compiler, and much better with gcc. + */ +#ifdef __alpha +#define XMUX(c, x, y) ((((signed int)((c) << 31) >> 31) & (y)) ^ (x)) +#else +#define XMUX(c, x, y) (((c) & 0x1) ? ((x) ^ (y)) : (x)) +#endif + + /* + * FSM() updates the finite state machine. + */ +#define FSM(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9) do { \ + ulong32 tt, or1; \ + tt = XMUX(r1, s ## x1, s ## x8); \ + or1 = r1; \ + r1 = T32(r2 + tt); \ + tt = T32(or1 * 0x54655307); \ + r2 = ROLc(tt, 7); \ + } while (0) + + /* + * LRU updates the shift register; the dropped value is stored + * in variable "dd". + */ +#define LRU(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, dd) do { \ + dd = s ## x0; \ + s ## x0 = MUL_A(s ## x0) ^ MUL_G(s ## x3) ^ s ## x9; \ + } while (0) + + /* + * CC1 stores into variable "ee" the next intermediate word + * (combination of the new states of the LFSR and the FSM). + */ +#define CC1(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, ee) do { \ + ee = T32(s ## x9 + r1) ^ r2; \ + } while (0) + + /* + * STEP computes one internal round. "dd" receives the "s_t" + * value (dropped from the LFSR) and "ee" gets the value computed + * from the LFSR and FSM. + */ +#define STEP(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, dd, ee) do { \ + FSM(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9); \ + LRU(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, dd); \ + CC1(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, ee); \ + } while (0) + + /* + * Apply one Serpent round (with the provided S-box macro), XOR + * the result with the "v" values, and encode the result into + * the destination buffer, at the provided offset. The "x*" + * arguments encode the output permutation of the "S" macro. + */ +#define SRD(S, x0, x1, x2, x3, ooff) do { \ + S(u0, u1, u2, u3, u4); \ + STORE32L(u ## x0 ^ v0, st->buf + ooff); \ + STORE32L(u ## x1 ^ v1, st->buf + ooff + 4); \ + STORE32L(u ## x2 ^ v2, st->buf + ooff + 8); \ + STORE32L(u ## x3 ^ v3, st->buf + ooff + 12); \ + } while (0) + + ulong32 s00 = st->s00; + ulong32 s01 = st->s01; + ulong32 s02 = st->s02; + ulong32 s03 = st->s03; + ulong32 s04 = st->s04; + ulong32 s05 = st->s05; + ulong32 s06 = st->s06; + ulong32 s07 = st->s07; + ulong32 s08 = st->s08; + ulong32 s09 = st->s09; + ulong32 r1 = st->r1; + ulong32 r2 = st->r2; + ulong32 u0, u1, u2, u3, u4; + ulong32 v0, v1, v2, v3; + + STEP(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, v0, u0); + STEP(01, 02, 03, 04, 05, 06, 07, 08, 09, 00, v1, u1); + STEP(02, 03, 04, 05, 06, 07, 08, 09, 00, 01, v2, u2); + STEP(03, 04, 05, 06, 07, 08, 09, 00, 01, 02, v3, u3); + SRD(S2, 2, 3, 1, 4, 0); + STEP(04, 05, 06, 07, 08, 09, 00, 01, 02, 03, v0, u0); + STEP(05, 06, 07, 08, 09, 00, 01, 02, 03, 04, v1, u1); + STEP(06, 07, 08, 09, 00, 01, 02, 03, 04, 05, v2, u2); + STEP(07, 08, 09, 00, 01, 02, 03, 04, 05, 06, v3, u3); + SRD(S2, 2, 3, 1, 4, 16); + STEP(08, 09, 00, 01, 02, 03, 04, 05, 06, 07, v0, u0); + STEP(09, 00, 01, 02, 03, 04, 05, 06, 07, 08, v1, u1); + STEP(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, v2, u2); + STEP(01, 02, 03, 04, 05, 06, 07, 08, 09, 00, v3, u3); + SRD(S2, 2, 3, 1, 4, 32); + STEP(02, 03, 04, 05, 06, 07, 08, 09, 00, 01, v0, u0); + STEP(03, 04, 05, 06, 07, 08, 09, 00, 01, 02, v1, u1); + STEP(04, 05, 06, 07, 08, 09, 00, 01, 02, 03, v2, u2); + STEP(05, 06, 07, 08, 09, 00, 01, 02, 03, 04, v3, u3); + SRD(S2, 2, 3, 1, 4, 48); + STEP(06, 07, 08, 09, 00, 01, 02, 03, 04, 05, v0, u0); + STEP(07, 08, 09, 00, 01, 02, 03, 04, 05, 06, v1, u1); + STEP(08, 09, 00, 01, 02, 03, 04, 05, 06, 07, v2, u2); + STEP(09, 00, 01, 02, 03, 04, 05, 06, 07, 08, v3, u3); + SRD(S2, 2, 3, 1, 4, 64); + + st->s00 = s00; + st->s01 = s01; + st->s02 = s02; + st->s03 = s03; + st->s04 = s04; + st->s05 = s05; + st->s06 = s06; + st->s07 = s07; + st->s08 = s08; + st->s09 = s09; + st->r1 = r1; + st->r2 = r2; +} + +/* + * Combine buffers in1[] and in2[] by XOR, result in out[]. The length + * is "datalen" (in bytes). Partial overlap of out[] with either in1[] + * or in2[] is not allowed. Total overlap (out == in1 and/or out == in2) + * is allowed. + */ +static LTC_INLINE void s_xorbuf(const unsigned char *in1, const unsigned char *in2, + unsigned char *out, unsigned long datalen) +{ + while (datalen -- > 0) { + *out ++ = *in1 ++ ^ *in2 ++; + } +} + + +/* + * Cipher operation, as a stream cipher: data is read from the "in" + * buffer, combined by XOR with the stream, and the result is written + * in the "out" buffer. "in" and "out" must be either equal, or + * reference distinct buffers (no partial overlap is allowed). + * @param st The Sosemanuk state + * @param in Data in + * @param inlen Length of data in bytes + * @param out Data out + * @return CRYPT_OK on success + */ +int sosemanuk_crypt(sosemanuk_state *st, + const unsigned char *in, unsigned long inlen, unsigned char *out) +{ + LTC_ARGCHK(st != NULL); + LTC_ARGCHK(in != NULL); + LTC_ARGCHK(out != NULL); + + if (st->ptr < (sizeof(st->buf))) { + unsigned long rlen = (sizeof(st->buf)) - st->ptr; + + if (rlen > inlen) { + rlen = inlen; + } + s_xorbuf(st->buf + st->ptr, in, out, rlen); + in += rlen; + out += rlen; + inlen -= rlen; + st->ptr += rlen; + } + while (inlen > 0) { + s_sosemanuk_internal(st); + if (inlen >= sizeof(st->buf)) { + s_xorbuf(st->buf, in, out, sizeof(st->buf)); + in += sizeof(st->buf); + out += sizeof(st->buf); + inlen -= sizeof(st->buf); + } else { + s_xorbuf(st->buf, in, out, inlen); + st->ptr = inlen; + inlen = 0; + } + } + return CRYPT_OK; +} + + + +/* + * Cipher operation, as a PRNG: the provided output buffer is filled with + * pseudo-random bytes as output from the stream cipher. + * @param st The Sosemanuk state + * @param out Data out + * @param outlen Length of output in bytes + * @return CRYPT_OK on success + */ +int sosemanuk_keystream(sosemanuk_state *st, unsigned char *out, unsigned long outlen) +{ + if (outlen == 0) return CRYPT_OK; /* nothing to do */ + LTC_ARGCHK(out != NULL); + XMEMSET(out, 0, outlen); + return sosemanuk_crypt(st, out, outlen, out); +} + + +/* + * Terminate and clear Sosemanuk key context + * @param st The Sosemanuk state + * @return CRYPT_OK on success + */ +int sosemanuk_done(sosemanuk_state *st) +{ + LTC_ARGCHK(st != NULL); + zeromem(st, sizeof(sosemanuk_state)); + return CRYPT_OK; +} + + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk_memory.c b/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk_memory.c new file mode 100644 index 0000000..c0fac25 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk_memory.c @@ -0,0 +1,35 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SOSEMANUK + +/** + Encrypt (or decrypt) bytes of ciphertext (or plaintext) with Sosemanuk + @param key The key + @param keylen The key length + @param iv The initial vector + @param ivlen The initial vector length + @param datain The plaintext (or ciphertext) + @param datalen The length of the input and output (octets) + @param dataout [out] The ciphertext (or plaintext) + @return CRYPT_OK if successful +*/ +int sosemanuk_memory(const unsigned char *key, unsigned long keylen, + const unsigned char *iv, unsigned long ivlen, + const unsigned char *datain, unsigned long datalen, + unsigned char *dataout) +{ + sosemanuk_state st; + int err; + + if ((err = sosemanuk_setup(&st, key, keylen)) != CRYPT_OK) goto WIPE_KEY; + if ((err = sosemanuk_setiv(&st, iv, ivlen)) != CRYPT_OK) goto WIPE_KEY; + err = sosemanuk_crypt(&st, datain, datalen, dataout); +WIPE_KEY: + sosemanuk_done(&st); + return err; +} + +#endif /* LTC_SOSEMANUK */ diff --git a/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk_test.c b/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk_test.c new file mode 100644 index 0000000..2c0a887 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/stream/sosemanuk/sosemanuk_test.c @@ -0,0 +1,79 @@ +/* LibTomCrypt, modular cryptographic library -- Tom St Denis */ +/* SPDX-License-Identifier: Unlicense */ + +#include "tomcrypt_private.h" + +#ifdef LTC_SOSEMANUK +int sosemanuk_test(void) +{ +#ifndef LTC_TEST + return CRYPT_NOP; +#else + sosemanuk_state st; + int err; + unsigned char out[1000]; + + { + unsigned char k[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }; + unsigned char n[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + unsigned char ct[] = { 0x7e, 0xfe, 0x2e, 0x6f, 0x8f, 0x77, 0x15, 0x72, 0x6a, 0x88, 0x14, 0xa6, 0x56, 0x88, 0x29, 0x9a, + 0x86, 0x32, 0x7f, 0x14, 0xd6, 0xb1, 0x94, 0x90, 0x25, 0xbc, 0x73, 0xfd, 0x02, 0x6c, 0x6a, 0xb8, + 0xda, 0x8e, 0x7f, 0x61, 0x70, 0x81, 0xe3, 0xbb, 0x99, 0xaf, 0x19, 0x9f, 0x20, 0x45 }; + char pt[] = "Kilroy was here, and there. ...and everywhere!"; /* len = 46 bytes */ + unsigned long len; + len = XSTRLEN(pt); + /* crypt piece by piece */ + if ((err = sosemanuk_setup(&st, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = sosemanuk_setiv(&st, n, sizeof(n))) != CRYPT_OK) return err; + if ((err = sosemanuk_crypt(&st, (unsigned char*)pt, 5, out)) != CRYPT_OK) return err; + if ((err = sosemanuk_crypt(&st, (unsigned char*)pt + 5, 25, out + 5)) != CRYPT_OK) return err; + if ((err = sosemanuk_crypt(&st, (unsigned char*)pt + 30, 10, out + 30)) != CRYPT_OK) return err; + if ((err = sosemanuk_crypt(&st, (unsigned char*)pt + 40, len - 40, out + 40)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "SOSEMANUK-TV1", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in one go - using sosemanuk_ivctr64() */ + if ((err = sosemanuk_setup(&st, k, sizeof(k))) != CRYPT_OK) return err; + if ((err = sosemanuk_setiv(&st, n, sizeof(n))) != CRYPT_OK) return err; + if ((err = sosemanuk_crypt(&st, (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "SOSEMANUK-TV2", 1)) return CRYPT_FAIL_TESTVECTOR; + + /* crypt in a single call */ + if ((err = sosemanuk_memory(k, sizeof(k), n, sizeof(n), + (unsigned char*)pt, len, out)) != CRYPT_OK) return err; + if (compare_testvector(out, len, ct, sizeof(ct), "SOSEMANUK-TV3", 1)) return CRYPT_FAIL_TESTVECTOR; + + } + { + /* keystream + * http://www.ecrypt.eu.org/stream/svn/viewcvs.cgi/ecrypt/trunk/submissions/sosemanuk/unverified.test-vectors?rev=210&view=auto + * Set 6, vector 0 + * key = 0053A6F94C9FF24598EB3E91E4378ADD + * 3083D6297CCF2275C81B6EC11467BA0D + * IV = 0D74DB42A91077DE45AC137AE148AF16 + * stream[0..63] = 55EB8D174C2E0351E5A53C90E84740EB + * 0F5A24AAFEC8E0C9F9D2CE48B2ADB0A3 + * 4D2E8C4E016102607368FFA43A0F9155 + * 0706E3548AD9E5EA15A53EB6F0EDE9DC + * + */ + + unsigned char k3[] = { 0x00, 0x53, 0xA6, 0xF9, 0x4C, 0x9F, 0xF2, 0x45, 0x98, 0xEB, 0x3E, 0x91, 0xE4, 0x37, 0x8A, 0xDD, + 0x30, 0x83, 0xD6, 0x29, 0x7C, 0xCF, 0x22, 0x75, 0xC8, 0x1B, 0x6E, 0xC1, 0x14, 0x67, 0xBA, 0x0D }; + unsigned char n3[] = { 0x0D, 0x74, 0xDB, 0x42, 0xA9, 0x10, 0x77, 0xDE, 0x45, 0xAC, 0x13, 0x7A, 0xE1, 0x48, 0xAF, 0x16 }; + unsigned char ct3[] = { 0x55, 0xEB, 0x8D, 0x17, 0x4C, 0x2E, 0x03, 0x51, 0xE5, 0xA5, 0x3C, 0x90, 0xE8, 0x47, 0x40, 0xEB, + 0x0F, 0x5A, 0x24, 0xAA, 0xFE, 0xC8, 0xE0, 0xC9, 0xF9, 0xD2, 0xCE, 0x48, 0xB2, 0xAD, 0xB0, 0xA3, + 0x4D, 0x2E, 0x8C, 0x4E, 0x01, 0x61, 0x02, 0x60, 0x73, 0x68, 0xFF, 0xA4, 0x3A, 0x0F, 0x91, 0x55, + 0x07, 0x06, 0xE3, 0x54, 0x8A, 0xD9, 0xE5, 0xEA, 0x15, 0xA5, 0x3E, 0xB6, 0xF0, 0xED, 0xE9, 0xDC }; + if ((err = sosemanuk_setup(&st, k3, sizeof(k3))) != CRYPT_OK) return err; + if ((err = sosemanuk_setiv(&st, n3, sizeof(n3))) != CRYPT_OK) return err; + if ((err = sosemanuk_keystream(&st, out, 64)) != CRYPT_OK) return err; + if ((err = sosemanuk_done(&st)) != CRYPT_OK) return err; + if (compare_testvector(out, 64, ct3, sizeof(ct3), "SOSEMANUK-TV4", 1)) return CRYPT_FAIL_TESTVECTOR; + } + + return CRYPT_OK; +#endif +} + +#endif diff --git a/optee_os/core/lib/libtomcrypt/src/sub.mk b/optee_os/core/lib/libtomcrypt/src/sub.mk new file mode 100644 index 0000000..d622204 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/src/sub.mk @@ -0,0 +1,9 @@ +subdirs-$(_CFG_CORE_LTC_CIPHER) += ciphers +subdirs-$(_CFG_CORE_LTC_AUTHENC) += encauth +subdirs-y += hashes +subdirs-$(_CFG_CORE_LTC_MAC) += mac +subdirs-$(_CFG_CORE_LTC_ACIPHER) += math +subdirs-y += misc +subdirs-y += modes +subdirs-$(_CFG_CORE_LTC_ACIPHER) += pk +subdirs-$(_CFG_CORE_LTC_EC25519) += pk diff --git a/optee_os/core/lib/libtomcrypt/sub.mk b/optee_os/core/lib/libtomcrypt/sub.mk new file mode 100644 index 0000000..de11ccc --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/sub.mk @@ -0,0 +1,155 @@ +global-incdirs-y += include +global-incdirs-y += src/headers + +cflags-lib-y += -Wno-declaration-after-statement + +cppflags-lib-y += -DARGTYPE=4 # Make LTC_ARGCHK() return on error +cppflags-lib-y += -DLTC_NO_TEST -DLTC_NO_PROTOTYPES +cppflags-lib-y += -DLTC_NO_TABLES -DLTC_HASH_HELPERS +cppflags-lib-y += -DLTC_NO_MISC +cppflags-lib-y += -DLTC_HMAC +cppflags-lib-$(_CFG_CORE_LTC_SIZE_OPTIMIZATION) += -DLTC_SMALL_CODE + +cppflags-lib-y += -DLTC_NO_CIPHERS + +ifeq ($(_CFG_CORE_LTC_AES_DESC),y) + cppflags-lib-y += -DLTC_RIJNDAEL +endif +ifeq ($(_CFG_CORE_LTC_DES),y) + cppflags-lib-y += -DLTC_DES +endif + +cppflags-lib-y += -DLTC_NO_MODES + +ifeq ($(_CFG_CORE_LTC_ECB),y) + cppflags-lib-y += -DLTC_ECB_MODE +endif +ifeq ($(_CFG_CORE_LTC_CBC),y) + cppflags-lib-y += -DLTC_CBC_MODE +endif +ifeq ($(_CFG_CORE_LTC_CTR),y) + cppflags-lib-y += -DLTC_CTR_MODE +endif +ifeq ($(_CFG_CORE_LTC_XTS),y) + cppflags-lib-y += -DLTC_XTS_MODE +endif + +cppflags-lib-y += -DLTC_NO_HASHES + +ifeq ($(_CFG_CORE_LTC_MD5_DESC),y) + cppflags-lib-y += -DLTC_MD5 +endif +ifeq ($(_CFG_CORE_LTC_SHA1_DESC),y) + cppflags-lib-y += -DLTC_SHA1 +endif +ifeq ($(_CFG_CORE_LTC_SHA224_DESC),y) + cppflags-lib-y += -DLTC_SHA224 +endif +ifeq ($(_CFG_CORE_LTC_SHA256_DESC),y) + cppflags-lib-y += -DLTC_SHA256 +endif +ifeq ($(_CFG_CORE_LTC_SHA384_DESC),y) + cppflags-lib-y += -DLTC_SHA384 +endif +ifeq ($(_CFG_CORE_LTC_SHA512_DESC),y) + cppflags-lib-y += -DLTC_SHA512 +endif +ifeq ($(_CFG_CORE_LTC_SHA512_256),y) + cppflags-lib-y += -DLTC_SHA512_256 +endif +cppflags-lib-$(_CFG_CORE_LTC_SHA3_DESC) += -DLTC_SHA3 + + +cppflags-lib-y += -DLTC_NO_MACS + +ifeq ($(_CFG_CORE_LTC_HMAC),y) + cppflags-lib-y += -DLTC_HMAC +endif +ifeq ($(_CFG_CORE_LTC_CMAC),y) + cppflags-lib-y += -DLTC_OMAC +endif +ifeq ($(_CFG_CORE_LTC_CCM),y) + cppflags-lib-y += -DLTC_CCM_MODE +endif +ifeq ($(_CFG_CORE_LTC_GCM),y) + cppflags-lib-y += -DLTC_GCM_MODE +endif + +cppflags-lib-y += -DLTC_NO_PK + +ifeq ($(_CFG_CORE_LTC_RSA),y) + cppflags-lib-y += -DLTC_MRSA +endif +ifeq ($(_CFG_CORE_LTC_DSA),y) + cppflags-lib-y += -DLTC_MDSA +endif +ifeq ($(_CFG_CORE_LTC_DH),y) + cppflags-lib-y += -DLTC_MDH +endif +ifeq ($(_CFG_CORE_LTC_ECC),y) + cppflags-lib-y += -DLTC_MECC + + # use Shamir's trick for point mul (speeds up signature verification) + cppflags-lib-y += -DLTC_ECC_SHAMIR + + cppflags-lib-y += -DLTC_ECC192 + cppflags-lib-y += -DLTC_ECC224 + cppflags-lib-y += -DLTC_ECC256 + cppflags-lib-y += -DLTC_ECC384 + cppflags-lib-y += -DLTC_ECC521 + cppflags-lib-y += -DLTC_CURVE25519 + + # ECC 521 bits is the max supported key size + cppflags-lib-y += -DLTC_MAX_ECC=521 +endif +ifneq (,$(filter y,$(_CFG_CORE_LTC_SM2_DSA) $(_CFG_CORE_LTC_SM2_PKE))) + cppflags-lib-y += -DLTC_ECC_SM2 +endif + +cppflags-lib-$(_CFG_CORE_LTC_X25519) += -DLTC_CURVE25519 +cppflags-lib-$(_CFG_CORE_LTC_ED25519) += -DLTC_CURVE25519 + +cppflags-lib-y += -DLTC_NO_PRNGS -DLTC_FORTUNA + +cflags-lib-$(_CFG_CORE_LTC_SIZE_OPTIMIZATION) += -Os + +subdirs-y += src + +srcs-$(_CFG_CORE_LTC_HASH) += hash.c +srcs-$(_CFG_CORE_LTC_HMAC) += hmac.c +srcs-$(_CFG_CORE_LTC_CMAC) += cmac.c +srcs-$(_CFG_CORE_LTC_ECB) += ecb.c +srcs-$(_CFG_CORE_LTC_CBC) += cbc.c +srcs-$(_CFG_CORE_LTC_CTR) += ctr.c +srcs-$(_CFG_CORE_LTC_XTS) += xts.c +srcs-$(_CFG_CORE_LTC_CCM) += ccm.c +srcs-$(_CFG_CORE_LTC_GCM) += gcm.c +srcs-$(_CFG_CORE_LTC_DSA) += dsa.c +srcs-$(_CFG_CORE_LTC_ECC) += ecc.c +srcs-$(_CFG_CORE_LTC_RSA) += rsa.c +srcs-$(_CFG_CORE_LTC_DH) += dh.c +srcs-$(_CFG_CORE_LTC_AES) += aes.c +srcs-$(_CFG_CORE_LTC_AES_ACCEL) += aes_accel.c +srcs-$(_CFG_CORE_LTC_SHA1_ACCEL) += sha1_accel.c +ifeq ($(_CFG_CORE_LTC_SHA256_DESC),y) +srcs-$(_CFG_CORE_LTC_SHA256_ACCEL) += sha256_accel.c +endif +ifeq ($(_CFG_CORE_LTC_SHA512_DESC),y) +srcs-$(_CFG_CORE_LTC_SHA512_ACCEL) += sha512_accel.c +endif +ifeq ($(_CFG_CORE_LTC_SHA3_DESC),y) +srcs-y += shake.c +srcs-$(_CFG_CORE_LTC_SHA3_ACCEL) += sha3_accel.c +endif +srcs-$(_CFG_CORE_LTC_SM2_DSA) += sm2-dsa.c +srcs-$(_CFG_CORE_LTC_SM2_PKE) += sm2-pke.c +srcs-$(_CFG_CORE_LTC_SM2_KEP) += sm2-kep.c +srcs-$(_CFG_CORE_LTC_X25519) += x25519.c +srcs-$(_CFG_CORE_LTC_ED25519) += ed25519.c +ifeq ($(_CFG_CORE_LTC_ACIPHER),y) +srcs-y += mpi_desc.c +cppflags-mpi_desc.c-y += -DMBEDTLS_ALLOW_PRIVATE_ACCESS +endif + +srcs-y += tomcrypt.c + diff --git a/optee_os/core/lib/libtomcrypt/tomcrypt.c b/optee_os/core/lib/libtomcrypt/tomcrypt.c new file mode 100644 index 0000000..cdff8f3 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/tomcrypt.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2023, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include "tomcrypt_mp.h" +#include + +#if defined(_CFG_CORE_LTC_VFP) +#include +#include +#endif + +#if defined(_CFG_CORE_LTC_ACIPHER) || defined(_CFG_CORE_LTC_EC25519) +/* Random generator */ +static int prng_crypto_start(prng_state *prng __unused) +{ + return CRYPT_OK; +} + +static int prng_crypto_add_entropy(const unsigned char *in __unused, + unsigned long inlen __unused, + prng_state *prng __unused) +{ + /* No entropy is required */ + return CRYPT_OK; +} + +static int prng_crypto_ready(prng_state *prng __unused) +{ + return CRYPT_OK; +} + +static unsigned long prng_crypto_read(unsigned char *out, unsigned long outlen, + prng_state *prng __unused) +{ + if (crypto_rng_read(out, outlen)) + return 0; + + return outlen; +} + +static int prng_crypto_done(prng_state *prng __unused) +{ + return CRYPT_OK; +} + +static int prng_crypto_export(unsigned char *out __unused, + unsigned long *outlen __unused, + prng_state *prng __unused) +{ + return CRYPT_OK; +} + +static int prng_crypto_import(const unsigned char *in __unused, + unsigned long inlen __unused, + prng_state *prng __unused) +{ + return CRYPT_OK; +} + +static int prng_crypto_test(void) +{ + return CRYPT_OK; +} + +static const struct ltc_prng_descriptor prng_crypto_desc = { + .name = "prng_crypto", + .export_size = 64, + .start = prng_crypto_start, + .add_entropy = prng_crypto_add_entropy, + .ready = prng_crypto_ready, + .read = prng_crypto_read, + .done = prng_crypto_done, + .pexport = prng_crypto_export, + .pimport = prng_crypto_import, + .test = prng_crypto_test, +}; +#endif /*_CFG_CORE_LTC_ACIPHER*/ + +/* + * tee_ltc_reg_algs(): Registers + * - algorithms + * - hash + * - prng (pseudo random generator) + */ + +static void tee_ltc_reg_algs(void) +{ +#if defined(_CFG_CORE_LTC_AES) || defined(_CFG_CORE_LTC_AES_DESC) + register_cipher(&aes_desc); +#endif +#if defined(_CFG_CORE_LTC_DES) + register_cipher(&des_desc); + register_cipher(&des3_desc); +#endif +#if defined(_CFG_CORE_LTC_MD5_DESC) + register_hash(&md5_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA1) || defined(_CFG_CORE_LTC_SHA1_DESC) + register_hash(&sha1_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA224) || defined(_CFG_CORE_LTC_SHA224_DESC) + register_hash(&sha224_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA256) || defined(_CFG_CORE_LTC_SHA256_DESC) + register_hash(&sha256_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA384) || defined(_CFG_CORE_LTC_SHA384_DESC) + register_hash(&sha384_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA512) || defined(_CFG_CORE_LTC_SHA512_DESC) + register_hash(&sha512_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA3_224) || defined(_CFG_CORE_LTC_SHA3_224_DESC) + register_hash(&sha3_224_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA3_256) || defined(_CFG_CORE_LTC_SHA3_256_DESC) + register_hash(&sha3_256_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA3_384) || defined(_CFG_CORE_LTC_SHA3_384_DESC) + register_hash(&sha3_384_desc); +#endif +#if defined(_CFG_CORE_LTC_SHA3_512) || defined(_CFG_CORE_LTC_SHA3_512_DESC) + register_hash(&sha3_512_desc); +#endif +#if defined(_CFG_CORE_LTC_ACIPHER) || defined(_CFG_CORE_LTC_EC25519) + register_prng(&prng_crypto_desc); +#endif +} + +static void ltc_init(void) +{ +#if defined(_CFG_CORE_LTC_ACIPHER) + init_mp_tomcrypt(); +#endif + tee_ltc_reg_algs(); +} + +#if defined(CFG_CRYPTOLIB_NAME_tomcrypt) +TEE_Result crypto_init(void) +{ + ltc_init(); + + return TEE_SUCCESS; +} +#else +void tomcrypt_init(void) +{ + ltc_init(); +} +#endif + +#if defined(CFG_WITH_VFP) +void tomcrypt_arm_neon_enable(struct tomcrypt_arm_neon_state *state) +{ + state->state = thread_kernel_enable_vfp(); +} + +void tomcrypt_arm_neon_disable(struct tomcrypt_arm_neon_state *state) +{ + thread_kernel_disable_vfp(state->state); +} +#endif diff --git a/optee_os/core/lib/libtomcrypt/x25519.c b/optee_os/core/lib/libtomcrypt/x25519.c new file mode 100644 index 0000000..6098a46 --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/x25519.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Technology Innovation Institute (TII) + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "acipher_helpers.h" + +/* X25519 key is an octet string of 32 bytes */ +#define X25519_KEY_SIZE_BYTES UL(32) + +TEE_Result crypto_acipher_alloc_x25519_keypair(struct montgomery_keypair *key, + size_t key_size) +{ + size_t key_size_bytes = key_size / 8; + + if (!key) + return TEE_ERROR_BAD_PARAMETERS; + + memset(key, 0, sizeof(*key)); + + if (key_size_bytes != X25519_KEY_SIZE_BYTES) + return TEE_ERROR_BAD_PARAMETERS; + + key->priv = calloc(1, key_size_bytes); + key->pub = calloc(1, key_size_bytes); + + if (!key->priv || !key->pub) { + free(key->priv); + free(key->pub); + return TEE_ERROR_OUT_OF_MEMORY; + } + + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_gen_x25519_key(struct montgomery_keypair *key, + size_t key_size) +{ + curve25519_key ltc_tmp_key = { }; + size_t key_size_bytes = key_size / 8; + + if (key_size_bytes != X25519_KEY_SIZE_BYTES) + return TEE_ERROR_BAD_PARAMETERS; + + if (x25519_make_key(NULL, find_prng("prng_crypto"), <c_tmp_key) != + CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + if (key_size_bytes < sizeof(ltc_tmp_key.pub) || + key_size_bytes < sizeof(ltc_tmp_key.priv)) + return TEE_ERROR_BAD_PARAMETERS; + + memcpy(key->pub, ltc_tmp_key.pub, sizeof(ltc_tmp_key.pub)); + memcpy(key->priv, ltc_tmp_key.priv, sizeof(ltc_tmp_key.priv)); + memzero_explicit(<c_tmp_key, sizeof(ltc_tmp_key)); + + return TEE_SUCCESS; +} + +TEE_Result crypto_acipher_x25519_shared_secret(struct montgomery_keypair + *private_key, + void *public_key, + void *secret, + unsigned long *secret_len) +{ + curve25519_key ltc_private_key = { + .type = PK_PRIVATE, + .algo = LTC_OID_X25519, + }; + curve25519_key ltc_public_key = { + .type = PK_PUBLIC, + .algo = LTC_OID_X25519, + }; + + if (!private_key || !public_key || !secret || !secret_len) + return TEE_ERROR_BAD_PARAMETERS; + + static_assert(sizeof(ltc_public_key.pub) == X25519_KEY_SIZE_BYTES && + sizeof(ltc_public_key.priv) == X25519_KEY_SIZE_BYTES); + + memcpy(ltc_public_key.pub, public_key, X25519_KEY_SIZE_BYTES); + memcpy(ltc_private_key.priv, private_key->priv, X25519_KEY_SIZE_BYTES); + + if (x25519_shared_secret(<c_private_key, <c_public_key, + secret, secret_len) != CRYPT_OK) + return TEE_ERROR_BAD_PARAMETERS; + + /* Clear private key from the stack */ + memzero_explicit(<c_private_key, sizeof(ltc_private_key)); + + /* + * RFC 7748, sec 6.1, check for all zero shared secret output to reject + * input points of low order. + */ + if (*secret_len != X25519_KEY_SIZE_BYTES || + !consttime_memcmp(secret, ltc_private_key.pub, + X25519_KEY_SIZE_BYTES)) + return TEE_ERROR_SECURITY; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/lib/libtomcrypt/xts.c b/optee_os/core/lib/libtomcrypt/xts.c new file mode 100644 index 0000000..cd65d6c --- /dev/null +++ b/optee_os/core/lib/libtomcrypt/xts.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ltc_xts_ctx { + struct crypto_cipher_ctx ctx; + int cipher_idx; + int (*update)(const unsigned char *src, unsigned long len, + unsigned char *dst, unsigned char *tweak, + const symmetric_xts *xts); + symmetric_xts state; + uint8_t tweak[TEE_AES_BLOCK_SIZE]; +}; + +static const struct crypto_cipher_ops ltc_xts_ops; + +static struct ltc_xts_ctx *to_xts_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == <c_xts_ops); + + return container_of(ctx, struct ltc_xts_ctx, ctx); +} + +static TEE_Result ltc_xts_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct ltc_xts_ctx *c = to_xts_ctx(ctx); + + if (key1_len != key2_len) + return TEE_ERROR_BAD_PARAMETERS; + if (iv) { + if (iv_len != sizeof(c->tweak)) + return TEE_ERROR_BAD_PARAMETERS; + memcpy(c->tweak, iv, sizeof(c->tweak)); + } else { + memset(c->tweak, 0, sizeof(c->tweak)); + } + + if ((int)iv_len != cipher_descriptor[c->cipher_idx]->block_length) + return TEE_ERROR_BAD_PARAMETERS; + + if (mode == TEE_MODE_ENCRYPT) + c->update = xts_encrypt; + else + c->update = xts_decrypt; + + + if (xts_start(c->cipher_idx, key1, key2, key1_len, 0, + &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static TEE_Result ltc_xts_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, uint8_t *dst) +{ + struct ltc_xts_ctx *c = to_xts_ctx(ctx); + + if (c->update && c->update(data, len, dst, c->tweak, + &c->state) == CRYPT_OK) + return TEE_SUCCESS; + else + return TEE_ERROR_BAD_STATE; +} + +static void ltc_xts_final(struct crypto_cipher_ctx *ctx) +{ + xts_done(&to_xts_ctx(ctx)->state); +} + +static void ltc_xts_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_xts_ctx(ctx)); +} + +static void ltc_xts_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct ltc_xts_ctx *src = to_xts_ctx(src_ctx); + struct ltc_xts_ctx *dst = to_xts_ctx(dst_ctx); + + assert(src->cipher_idx == dst->cipher_idx); + dst->update = src->update; + memcpy(dst->tweak, src->tweak, sizeof(src->tweak)); + dst->state = src->state; +} + +static const struct crypto_cipher_ops ltc_xts_ops = { + .init = ltc_xts_init, + .update = ltc_xts_update, + .final = ltc_xts_final, + .free_ctx = ltc_xts_free_ctx, + .copy_state = ltc_xts_copy_state, +}; + +TEE_Result crypto_aes_xts_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct ltc_xts_ctx *c = NULL; + int cipher_idx = find_cipher("aes"); + + if (cipher_idx < 0) + return TEE_ERROR_NOT_SUPPORTED; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = <c_xts_ops; + c->cipher_idx = cipher_idx; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/lib/scmi-server/conf-optee-fvp.mk b/optee_os/core/lib/scmi-server/conf-optee-fvp.mk new file mode 100644 index 0000000..0b060c4 --- /dev/null +++ b/optee_os/core/lib/scmi-server/conf-optee-fvp.mk @@ -0,0 +1,23 @@ +$(call force,CFG_SCPFW_MOD_CLOCK,y) +$(call force,CFG_SCPFW_MOD_DVFS,y) +$(call force,CFG_SCPFW_MOD_MSG_SMT,y) +$(call force,CFG_SCPFW_MOD_MOCK_CLOCK,y) +$(call force,CFG_SCPFW_MOD_MOCK_PPU,y) +$(call force,CFG_SCPFW_MOD_MOCK_PSU,y) +$(call force,CFG_SCPFW_MOD_OPTEE_MBX,y) +$(call force,CFG_SCPFW_MOD_POWER_DOMAIN,y) +$(call force,CFG_SCPFW_MOD_PSU,y) +$(call force,CFG_SCPFW_MOD_REG_SENSOR,y) +$(call force,CFG_SCPFW_MOD_SCMI,y) +$(call force,CFG_SCPFW_MOD_SCMI_CLOCK,y) +$(call force,CFG_SCPFW_MOD_SCMI_PERF,y) +$(call force,CFG_SCPFW_SCMI_PERF_PROTOCOL_OPS,y) +$(call force,CFG_SCPFW_MOD_SCMI_POWER_DOMAIN,y) +$(call force,CFG_SCPFW_MOD_SCMI_SENSOR,y) +$(call force,CFG_SCPFW_MOD_SYSTEM_PLL,y) +$(call force,CFG_SCPFW_MOD_SENSOR,y) +$(call force,CFG_SCPFW_MOD_VPLL,y) + +$(call force,CFG_SCPFW_NOTIFICATION,y) +$(call force,CFG_SCPFW_SCMI_SENSOR_EVENTS,y) +$(call force,CFG_SCPFW_FAST_CHANNEL,n) diff --git a/optee_os/core/lib/scmi-server/conf-optee-stm32mp1.mk b/optee_os/core/lib/scmi-server/conf-optee-stm32mp1.mk new file mode 100644 index 0000000..34ae07f --- /dev/null +++ b/optee_os/core/lib/scmi-server/conf-optee-stm32mp1.mk @@ -0,0 +1,18 @@ +$(call force,CFG_SCPFW_MOD_RESET_DOMAIN,y) +$(call force,CFG_SCPFW_MOD_CLOCK,y) +$(call force,CFG_SCPFW_MOD_OPTEE_CLOCK,y) +$(call force,CFG_SCPFW_MOD_OPTEE_CONSOLE,y) +$(call force,CFG_SCPFW_MOD_OPTEE_MBX,y) +$(call force,CFG_SCPFW_MOD_OPTEE_RESET,y) +$(call force,CFG_SCPFW_MOD_OPTEE_SMT,y) +$(call force,CFG_SCPFW_MOD_SCMI,y) +$(call force,CFG_SCPFW_MOD_SCMI_CLOCK,y) +$(call force,CFG_SCPFW_MOD_SCMI_RESET_DOMAIN,y) +$(call force,CFG_SCPFW_MOD_SCMI_VOLTAGE_DOMAIN,y) +$(call force,CFG_SCPFW_MOD_VOLTAGE_DOMAIN,y) + +$(call force,CFG_SCPFW_MOD_STM32_PMIC_REGU,$(CFG_STPMIC1)) +$(call force,CFG_SCPFW_MOD_STM32_PWR_REGU,y) + +$(call force,CFG_SCPFW_NOTIFICATION,n) +$(call force,CFG_SCPFW_FAST_CHANNEL,n) diff --git a/optee_os/core/lib/scmi-server/conf.mk b/optee_os/core/lib/scmi-server/conf.mk new file mode 100644 index 0000000..ba49748 --- /dev/null +++ b/optee_os/core/lib/scmi-server/conf.mk @@ -0,0 +1,78 @@ +# Configuration switches for SCP-firmware: +# +# CFG_SCMI_SCPFW Boolean switch, embeds SCP-firmware SCMI stack when enabled. +# CFG_SCMI_SCPFW_PRODUCT Name of the SCP-firmware product to build. +# CFG_SCP_FIRMWARE Directory path to SCP-firmware source tree +# CFG_SCPFW_LOG_LEVEL SCP-firmware log level +# CFG_SCPFW_MOD_x Boolean switch, embeds SCP-firmware module x when enabled +# CFG_SCPFW_NOTIFICATION Boolean switch, embeds SCMI notification when enabled. +# CFG_SCPFW_FAST_CHANNELS Boolean switch, embeds SCMI fast channel when enabled. +# +# Boolean y|n switches for module features +# CFG_SCPFW_CLOCK_TREE_MGMT +# CFG_SCPFW_SCMI_PERF_FAST_CHANNELS +# CFG_SCPFW_SCMI_SENSOR_EVENTS +# CFG_SCPFW_SCMI_SENSOR_V2 +# CFG_SCPFW_SENSOR_TIMESTAMP +# CFG_SCPFW_SENSOR_MULTI_AXIS +# CFG_SCPFW_SENSOR_EXT_ATTRIBS +# CFG_SCPFW_SENSOR_SIGNED_VALUE + +include core/lib/scmi-server/conf-$(CFG_SCMI_SCPFW_PRODUCT).mk + +# SCP-fmw log level: 0 trace/verbose, 1 info, 2 warning, 3 error, 4 critical +ifeq ($(CFG_TEE_CORE_LOG_LEVEL),0) +CFG_SCPFW_LOG_LEVEL ?= 3 +else ifeq ($(CFG_TEE_CORE_LOG_LEVEL),1) +CFG_SCPFW_LOG_LEVEL ?= 3 +else ifeq ($(CFG_TEE_CORE_LOG_LEVEL),2) +CFG_SCPFW_LOG_LEVEL ?= 1 +else +CFG_SCPFW_LOG_LEVEL ?= 0 +endif + +CFG_SCPFW_MOD_DVFS ?= n +CFG_SCPFW_MOD_RESET_DOMAIN ?= n +CFG_SCPFW_MOD_CLOCK ?= n +CFG_SCPFW_MOD_MSG_SMT ?= n +CFG_SCPFW_MOD_MOCK_CLOCK ?= n +CFG_SCPFW_MOD_MOCK_PPU ?= n +CFG_SCPFW_MOD_MOCK_PSU ?= n +CFG_SCPFW_MOD_OPTEE_CLOCK ?= n +CFG_SCPFW_MOD_OPTEE_CONSOLE ?= y +CFG_SCPFW_MOD_OPTEE_MBX ?= n +CFG_SCPFW_MOD_OPTEE_RESET ?= n +CFG_SCPFW_MOD_OPTEE_SMT ?= n +CFG_SCPFW_MOD_POWER_DOMAIN ?= n +CFG_SCPFW_MOD_PSU ?= n +CFG_SCPFW_MOD_REG_SENSOR ?= n +CFG_SCPFW_MOD_SENSOR ?= n +CFG_SCPFW_MOD_SCMI ?= y +CFG_SCPFW_MOD_SCMI_APCORE ?= n +CFG_SCPFW_MOD_SCMI_CLOCK ?= n +CFG_SCPFW_MOD_SCMI_PERF ?= n +CFG_SCPFW_MOD_SCMI_POWER_DOMAIN ?= n +CFG_SCPFW_MOD_SCMI_RESET_DOMAIN ?= n +CFG_SCPFW_MOD_SCMI_SENSOR ?= n +CFG_SCPFW_MOD_SCMI_VOLTAGE_DOMAIN ?= n +CFG_SCPFW_MOD_SYSTEM_PLL ?= n +CFG_SCPFW_MOD_VOLTAGE_DOMAIN ?= n + +CFG_SCPFW_NOTIFICATION ?= n +CFG_SCPFW_FAST_CHANNELS ?= n + +CFG_SCPFW_CLOCK_TREE_MGMT ?= n +CFG_SCPFW_SCMI_PERF_FAST_CHANNELS ?= n +CFG_SCPFW_SCMI_SENSOR_EVENTS ?= n +CFG_SCPFW_SCMI_SENSOR_V2 ?= n +CFG_SCPFW_SENSOR_TIMESTAMP ?= n +CFG_SCPFW_SENSOR_MULTI_AXIS ?= n +CFG_SCPFW_SENSOR_EXT_ATTRIBS ?= n +CFG_SCPFW_SENSOR_SIGNED_VALUE ?= n + +ifeq ($(CFG_SCPFW_MOD_OPTEE_SMT),y) +_CFG_SCMI_PTA_SMT_HEADER := y +endif +ifeq ($(CFG_SCPFW_MOD_MSG_SMT),y) +_CFG_SCMI_PTA_MSG_HEADER := y +endif diff --git a/optee_os/core/lib/scmi-server/include/optee_scmi.h b/optee_os/core/lib/scmi-server/include/optee_scmi.h new file mode 100644 index 0000000..a15f1f2 --- /dev/null +++ b/optee_os/core/lib/scmi-server/include/optee_scmi.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019-2022, Linaro Limited + */ +#ifndef OPTEE_SCMI_H +#define OPTEE_SCMI_H + +#include +#include + +/* + * Return virtual address mapped for target SMT IOMEM address range + * + * @pa: Target address range base physical address + * @sz: Target address range byte size + * @shmem_is_secure: True if memory is secure, false otherwise + * Return a virtual address or 0 is memory is not mapped + */ +uintptr_t smt_phys_to_virt(uintptr_t pa, size_t sz, bool shmem_is_secure); + +#endif /* OPTEE_SCMI_H */ diff --git a/optee_os/core/lib/scmi-server/scmi_server.c b/optee_os/core/lib/scmi-server/scmi_server.c new file mode 100644 index 0000000..f0bb256 --- /dev/null +++ b/optee_os/core/lib/scmi-server/scmi_server.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +/* + * OP-TEE helper function exported to SCP-firmware + */ +uintptr_t smt_phys_to_virt(uintptr_t pa, size_t sz, bool shmem_is_secure) +{ + if (shmem_is_secure) + return (uintptr_t)phys_to_virt(pa, MEM_AREA_IO_SEC, sz); + else + return (uintptr_t)phys_to_virt(pa, MEM_AREA_IO_NSEC, sz); +} + +/* + * SCMI server APIs exported to OP-TEE core + */ +int scmi_server_get_channels_count(void) +{ + return scmi_get_devices_count(); +} + +TEE_Result scmi_server_get_channel(unsigned int channel_id, int *handle) +{ + int fwk_id = 0; + + fwk_id = scmi_get_device(channel_id); + if (fwk_id < 0) + return TEE_ERROR_BAD_PARAMETERS; + + if (handle) + *handle = fwk_id; + + return TEE_SUCCESS; +} + +TEE_Result scmi_server_smt_process_thread(unsigned int channel_id) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int fwk_id = 0; + + res = scmi_server_get_channel(channel_id, &fwk_id); + if (!res) + scmi_process_mbx_smt(fwk_id); + + return res; +} + +TEE_Result scmi_server_msg_process_thread(unsigned int channel_id, + void *in_buf, size_t in_sz, + void *out_buf, size_t *out_sz) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int fwk_id = 0; + + res = scmi_server_get_channel(channel_id, &fwk_id); + if (!res) + scmi_process_mbx_msg(fwk_id, in_buf, in_sz, out_buf, out_sz); + + return res; +} + +static TEE_Result scmi_server_initialize(void) +{ + int rc = 0; + + rc = scmi_arch_init(); + if (rc < 0) { + EMSG("SCMI server init failed: %d", rc); + panic(); + } + + return TEE_SUCCESS; +} + +boot_final(scmi_server_initialize); diff --git a/optee_os/core/lib/scmi-server/sub-optee-fvp.mk b/optee_os/core/lib/scmi-server/sub-optee-fvp.mk new file mode 100644 index 0000000..6600a61 --- /dev/null +++ b/optee_os/core/lib/scmi-server/sub-optee-fvp.mk @@ -0,0 +1,17 @@ +incdirs_ext-y += $(scpfw-path)/product/optee-fvp/fw +incdirs_ext-y += $(scpfw-path)/product/optee-fvp/include + +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_clock.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_dvfs.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_mbx_smt.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_mock_clock.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_mock_ppu.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_mock_psu.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_power_domain.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_psu.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_scmi.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_scmi_clock.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_scmi_perf.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_scmi_power_domain.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_sensor.c +srcs-y += $(scpfw-path)/product/optee-fvp/fw/config_vpll.c diff --git a/optee_os/core/lib/scmi-server/sub-optee-stm32mp1.mk b/optee_os/core/lib/scmi-server/sub-optee-stm32mp1.mk new file mode 100644 index 0000000..b0cddd6 --- /dev/null +++ b/optee_os/core/lib/scmi-server/sub-optee-stm32mp1.mk @@ -0,0 +1,10 @@ +incdirs_ext-y += $(scpfw-path)/product/optee-stm32mp1/include + +srcs-y += $(scpfw-path)/product/optee-stm32mp1/fw/config_mbx_smt.c +srcs-y += $(scpfw-path)/product/optee-stm32mp1/fw/config_scmi.c +srcs-y += $(scpfw-path)/product/optee-stm32mp1/fw/config_scmi_clocks.c +srcs-y += $(scpfw-path)/product/optee-stm32mp1/fw/config_scmi_reset_domains.c +srcs-y += $(scpfw-path)/product/optee-stm32mp1/fw/config_scmi_voltage_domains.c + +$(eval $(call scpfw-embed-product-module,stm32_pmic_regu)) +$(eval $(call scpfw-embed-product-module,stm32_pwr_regu)) diff --git a/optee_os/core/lib/scmi-server/sub.mk b/optee_os/core/lib/scmi-server/sub.mk new file mode 100644 index 0000000..d719a1c --- /dev/null +++ b/optee_os/core/lib/scmi-server/sub.mk @@ -0,0 +1,226 @@ +# SCMI server library is built from SCP-firmware source tree. +# The firmware is made of a framework, a product and modules. +# Only modules used by firmware must be built, as stated by +# CFG_SCPFW_MOD_* swtches. SCP-firmware needs a C source and +# a header file to be generated to describe embedded modules. +# This is done through cmake configuration of the package. +# The toolchain build directive must also match the list of +# embedded modules. + +scpfw-path = $(CFG_SCP_FIRMWARE) +scpfw-product = $(CFG_SCMI_SCPFW_PRODUCT) +scpfw-out-path := $(out-dir)/$(libdir) + +# This script was validated against SCP-firmware 2.11.0 development branch, +# from commit f1d894921d76 ("product/optee-fvp: Add new OPTEE FVP product"). +scpfw-integ-version-maj = 2 +scpfw-integ-version-min = 11 +scpfw-integ-version-pat = 0 +scpfw-integ-version = $(scpfw-integ-version-maj).$(scpfw-integ-version-min).$(scpfw-integ-version-pat) + +srcs-y += scmi_server.c +incdirs-y += include + +# SCP-firmware cmake configuration generates header fwk_module_idx.h and +# source files fwk_module_list.c needed for scp-firmware compilation. +scpfw-cmake-flags-y = -DSCP_FIRMWARE_SOURCE_DIR:PATH=$(scpfw-product)/fw \ + -DSCP_LOG_LEVEL="TRACE" \ + -DDISABLE_CPPCHECK=1 \ + -DCFG_NUM_THREADS=$(CFG_NUM_THREADS) \ + -DSCP_OPTEE_DIR:PATH=$(CURDIR) \ + -DCFG_CROSS_COMPILE=$(lastword $(CROSS_COMPILE_core)) + +ifeq ($(cmd-echo-silent),true) +scpfw-cmake-redirect = >/dev/null +endif + +gensrcs-y += fwk_module_list +force-gensrc-fwk_module_list := y +produce-fwk_module_list = build/framework/src/fwk_module_list.c +recipe-fwk_module_list = cmake -S $(scpfw-path) -B $(scpfw-out-path)/build \ + $(scpfw-cmake-flags-y) --log-level=WARNING $(scpfw-cmake-redirect) +depends-fwk_module_list = $(scpfw-path)/product/$(scpfw-product)/fw/Firmware.cmake $(conf-file) +# Include path of generated header file fwk_module_idx.h +incdirs_ext-y += $(scpfw-out-path)/build/framework/include + +cppflags-lib-y += -DBUILD_VERSION_MAJOR=$(scpfw-integ-version-maj) \ + -DBUILD_VERSION_MINOR=$(scpfw-integ-version-min) \ + -DBUILD_VERSION_PATCH=$(scpfw-integ-version-pat) + +scpfw-impl-version := $(shell git -C $(scpfw-path) describe --tags --always --dirty=-dev 2>/dev/null || \ + echo Unknown_$(scpfw-integ-version)) +cppflags-lib-y += -DBUILD_VERSION_DESCRIBE_STRING=\"$(scpfw-impl-version)\" + +cppflags-lib-y += -DFWK_LOG_LEVEL=$(CFG_SCPFW_LOG_LEVEL) +ifneq ($(CFG_SCPFW_LOG_LEVEL),0) +cppflags-lib-y += -DFMW_LOG_MINIMAL_BANNER=1 +endif + +cflags-lib-y += -Wno-cast-align \ + -Wno-nonnull-compare \ + -Wno-unused-parameter \ + -Wno-suggest-attribute=format \ + -Wno-declaration-after-statement + +# The below directives will be removed once SCP-firmware pull requests +# 728 and 732 are merged. +cflags-lib-y += -Wno-undef \ + -Wno-missing-prototypes \ + -Wno-missing-declarations \ + -Wno-unused-but-set-variable \ + -Wno-suggest-attribute=format + +# Notification implementation has strict aliasing issues +cflags-lib-$(CFG_SCPFW_NOTIFICATION) += -Wno-strict-aliasing + +cppflags-lib-y += -DBUILD_HAS_SUB_SYSTEM_MODE=1 \ + -DBUILD_HAS_BASE_PROTOCOL + +cppflags-lib-$(CFG_SCPFW_NOTIFICATION) += -DBUILD_HAS_NOTIFICATION \ + -DBUILD_HAS_SCMI_NOTIFICATIONS + +cppflags-lib-$(CFG_SCPFW_FAST_CHANNELS) += -DBUILD_HAS_FAST_CHANNELS \ + -DBUILD_HAS_SCMI_FAST_CHANNELS + +cppflags-lib-$(CFG_SCPFW_CLOCK_TREE_MGMT) += -DBUILD_HAS_CLOCK_TREE_MGMT + +cppflags-lib-$(CFG_SCPFW_SCMI_PERF_FAST_CHANNELS) += -DBUILD_HAS_SCMI_PERF_FAST_CHANNELS + +cppflags-lib-$(CFG_SCPFW_SCMI_PERF_PROTOCOL_OPS) \ + += -DBUILD_HAS_SCMI_PERF_PROTOCOL_OPS + +cppflags-lib-$(CFG_SCPFW_SCMI_SENSOR_EVENTS) += -DBUILD_HAS_SCMI_SENSOR_EVENTS +cppflags-lib-$(CFG_SCPFW_SCMI_SENSOR_V2) += -DBUILD_HAS_SCMI_SENSOR_V2 \ + -DBUILD_HAS_SENSOR_TIMESTAMP \ + -DBUILD_HAS_SENSOR_MULTI_AXIS \ + -DBUILD_HAS_SENSOR_EXT_ATTRIBS \ + -DBUILD_HAS_SENSOR_SIGNED_VALUE + +cppflags-lib-$(CFG_SCPFW_SENSOR_TIMESTAMP) += -DBUILD_HAS_SENSOR_TIMESTAMP +cppflags-lib-$(CFG_SCPFW_SENSOR_MULTI_AXIS) += -DBUILD_HAS_SENSOR_MULTI_AXI +cppflags-lib-$(CFG_SCPFW_SENSOR_EXT_ATTRIBS) += -DBUILD_HAS_SENSOR_EXT_ATTRIBS +cppflags-lib-$(CFG_SCPFW_SENSOR_SIGNED_VALUE) += -DBUILD_HAS_SENSOR_SIGNED_VALUE +cppflags-lib-$(CFG_SCPFW_INBAND_MSG_SUPPORT) += -DBUILD_HAS_INBAND_MSG_SUPPORT + +incdirs_ext-y += $(scpfw-path)/arch/none/optee/include +srcs-y += $(scpfw-path)/arch/none/optee/src/arch_interrupt.c +srcs-y += $(scpfw-path)/arch/none/optee/src/arch_main.c + +incdirs_ext-y += $(scpfw-path)/framework/include +srcs-y += $(scpfw-path)/framework/src/fwk_arch.c +srcs-y += $(scpfw-path)/framework/src/fwk_dlist.c +srcs-y += $(scpfw-path)/framework/src/fwk_id.c +srcs-y += $(scpfw-path)/framework/src/fwk_interrupt.c +srcs-y += $(scpfw-path)/framework/src/fwk_io.c +srcs-y += $(scpfw-path)/framework/src/fwk_log.c +srcs-y += $(scpfw-path)/framework/src/fwk_mm.c +srcs-y += $(scpfw-path)/framework/src/fwk_module.c +srcs-y += $(scpfw-path)/framework/src/fwk_ring.c +srcs-y += $(scpfw-path)/framework/src/fwk_slist.c +srcs-y += $(scpfw-path)/framework/src/fwk_status.c +srcs-y += $(scpfw-path)/framework/src/fwk_string.c +srcs-y += $(scpfw-path)/framework/src/fwk_delayed_resp.c +srcs-y += $(scpfw-path)/framework/src/fwk_time.c +srcs-y += $(scpfw-path)/framework/src/fwk_core.c +srcs-y += $(scpfw-path)/framework/src/assert.c +srcs-y += $(scpfw-path)/framework/src/stdlib.c +srcs-$(CFG_SCPFW_NOTIFICATION) += $(scpfw-path)/framework/src/fwk_notification.c + +# Helper macros for listing SCP-firmware modules source files (in srcs-y) +# and header include paths (in incdirs_ext-y). Each module provides a C source +# file named mod_.c and possibly an include directory. Build +# directive BUILD_HAS_MOD_ must be set for each embedded module. +# +# Standard modules source tree: /module//src/mod_.c +# Optee modules source tree: /module/optee//src/mod_.c +# Product modules source tree: /product//module//src/mod_.c +# +# scpfw-embed-generic-module is to be used for standard modules. +# scpfw-embed-optee-module is to be used for optee modules. +# scpfw-embed-product-module is to be used for product modules. +# For modules that implement other C source files aside mandatory mod_.c we must +# add to srcs-y the required source file paths. +# +# scpfw-embed-mod takes 4 arguments: +# $1 module name, lowercase +# $2 module directory name +# $3 module parent directory relative path in scpfw tree +# $4 module name, uppercase, relates to CFG_SCPFW_MOD_$4 +define scpfw-embed-mod +ifneq (,$$(wildcard $(scpfw-path)/$3/$2/include/*)) +incdirs_ext-y += $(scpfw-path)/$3/$2/include +endif +srcs-$(CFG_SCPFW_MOD_$4) += $(scpfw-path)/$3/$2/src/mod_$1.c + +# SCMI_Perf in SCP-firmware has components that can be added conditionally at +# build time. +ifeq ($(1), scmi_perf) + +ifeq ($(CFG_SCPFW_SCMI_PERF_PROTOCOL_OPS),y) +srcs-$(CFG_SCPFW_MOD_SCMI_PERF) += $(scpfw-path)/$3/$2/src/scmi_perf_protocol_ops.c +endif + +ifeq ($(CFG_SCPFW_SCMI_PERF_FAST_CHANNELS),y) +srcs-$(CFG_SCPFW_MOD_SCMI_PERF) += $(scpfw-path)/$3/$2/src/scmi_perf_fastchannels.c +endif + +endif + +cflags-lib-$(CFG_SCPFW_MOD_$4) += -DBUILD_HAS_MOD_$4 +endef + +define scpfw-embed-generic-module +$(eval $(call scpfw-embed-mod,$1,$1,module,$(shell echo $1 | tr a-z A-Z))) +endef + +define scpfw-embed-optee-module +$(eval $(call scpfw-embed-mod,optee_$1,$1,module/optee,OPTEE_$(shell echo $1 | tr a-z A-Z))) +endef + +define scpfw-embed-product-module +$(eval $(call scpfw-embed-mod,$1,$1,product/$(scpfw-product)/module,$(shell echo $1 | tr a-z A-Z))) +endef + +$(eval $(call scpfw-embed-generic-module,clock)) +$(eval $(call scpfw-embed-generic-module,dvfs)) +$(eval $(call scpfw-embed-generic-module,mock_clock)) +$(eval $(call scpfw-embed-generic-module,mock_ppu)) +$(eval $(call scpfw-embed-generic-module,mock_psu)) +$(eval $(call scpfw-embed-generic-module,msg_smt)) +$(eval $(call scpfw-embed-generic-module,power_domain)) +$(eval $(call scpfw-embed-generic-module,psu)) +$(eval $(call scpfw-embed-generic-module,reg_sensor)) +$(eval $(call scpfw-embed-generic-module,reset_domain)) +$(eval $(call scpfw-embed-generic-module,sensor)) +$(eval $(call scpfw-embed-generic-module,scmi)) +$(eval $(call scpfw-embed-generic-module,scmi_apcore)) +$(eval $(call scpfw-embed-generic-module,scmi_clock)) +$(eval $(call scpfw-embed-generic-module,scmi_perf)) +$(eval $(call scpfw-embed-generic-module,scmi_power_domain)) +$(eval $(call scpfw-embed-generic-module,scmi_reset_domain)) +$(eval $(call scpfw-embed-generic-module,scmi_sensor)) +$(eval $(call scpfw-embed-generic-module,scmi_voltage_domain)) +$(eval $(call scpfw-embed-generic-module,system_pll)) +$(eval $(call scpfw-embed-generic-module,voltage_domain)) +$(eval $(call scpfw-embed-optee-module,clock)) +$(eval $(call scpfw-embed-optee-module,console)) +$(eval $(call scpfw-embed-optee-module,mbx)) +$(eval $(call scpfw-embed-optee-module,reset)) +$(eval $(call scpfw-embed-optee-module,smt)) + +srcs-$(CFG_SCPFW_MOD_CLOCK) += $(scpfw-path)/module/clock/src/clock_tree_management.c +srcs-$(CFG_SCPFW_MOD_POWER_DOMAIN) += $(scpfw-path)/module/power_domain/src/power_domain_utils.c +srcs-$(CFG_SCPFW_MOD_SCMI) += $(scpfw-path)/module/scmi/src/mod_scmi_base.c +srcs-$(CFG_SCPFW_MOD_SCMI_SENSOR) += $(scpfw-path)/module/scmi_sensor/src/mod_scmi_ext_attrib.c +srcs-$(CFG_SCPFW_MOD_SENSOR) += $(scpfw-path)/module/sensor/src/sensor_extended.c + +# Architecture arch/none/optee requires optee mbx header file +incdirs_ext-y += $(scpfw-path)/module/optee/mbx/include +# Some modules require header files from module that are not embedded +ifneq (,$(filter y, $(CFG_SCPFW_MOD_DVFS) $(CFG_SCPFW_MOD_MOCK_PSU) $(CFG_SCPFW_MOD_SCMI_PERF))) +incdirs_ext-y += $(scpfw-path)/module/timer/include +endif +incdirs_ext-$(CFG_SCPFW_MOD_OPTEE_MBX) += $(scpfw-path)/module/msg_smt/include +incdirs_ext-$(CFG_SCPFW_MOD_SCMI) += $(scpfw-path)/module/power_domain/include + +include core/lib/scmi-server/sub-$(CFG_SCMI_SCPFW_PRODUCT).mk diff --git a/optee_os/core/lib/zlib/adler32.c b/optee_os/core/lib/zlib/adler32.c new file mode 100644 index 0000000..fbaec27 --- /dev/null +++ b/optee_os/core/lib/zlib/adler32.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Zlib +/* adler32.c -- compute the Adler-32 checksum of a data stream + * Copyright (C) 1995-2011, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#include "zutil.h" + +local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); + +#define BASE 65521U /* largest prime smaller than 65536 */ +#define NMAX 5552 +/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ + +#define DO1(buf,i) {adler += (buf)[i]; sum2 += adler;} +#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1); +#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2); +#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); +#define DO16(buf) DO8(buf,0); DO8(buf,8); + +/* use NO_DIVIDE if your processor does not do division in hardware -- + try it both ways to see which is faster */ +#ifdef NO_DIVIDE +/* note that this assumes BASE is 65521, where 65536 % 65521 == 15 + (thank you to John Reiser for pointing this out) */ +# define CHOP(a) \ + do { \ + unsigned long tmp = a >> 16; \ + a &= 0xffffUL; \ + a += (tmp << 4) - tmp; \ + } while (0) +# define MOD28(a) \ + do { \ + CHOP(a); \ + if (a >= BASE) a -= BASE; \ + } while (0) +# define MOD(a) \ + do { \ + CHOP(a); \ + MOD28(a); \ + } while (0) +# define MOD63(a) \ + do { /* this assumes a is not negative */ \ + z_off64_t tmp = a >> 32; \ + a &= 0xffffffffL; \ + a += (tmp << 8) - (tmp << 5) + tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + if (a >= BASE) a -= BASE; \ + } while (0) +#else +# define MOD(a) a %= BASE +# define MOD28(a) a %= BASE +# define MOD63(a) a %= BASE +#endif + +/* ========================================================================= */ +uLong ZEXPORT adler32_z(adler, buf, len) + uLong adler; + const Bytef *buf; + z_size_t len; +{ + unsigned long sum2; + unsigned n; + + /* split Adler-32 into component sums */ + sum2 = (adler >> 16) & 0xffff; + adler &= 0xffff; + + /* in case user likes doing a byte at a time, keep it fast */ + if (len == 1) { + adler += buf[0]; + if (adler >= BASE) + adler -= BASE; + sum2 += adler; + if (sum2 >= BASE) + sum2 -= BASE; + return adler | (sum2 << 16); + } + + /* initial Adler-32 value (deferred check for len == 1 speed) */ + if (buf == Z_NULL) + return 1L; + + /* in case short lengths are provided, keep it somewhat fast */ + if (len < 16) { + while (len--) { + adler += *buf++; + sum2 += adler; + } + if (adler >= BASE) + adler -= BASE; + MOD28(sum2); /* only added so many BASE's */ + return adler | (sum2 << 16); + } + + /* do length NMAX blocks -- requires just one modulo operation */ + while (len >= NMAX) { + len -= NMAX; + n = NMAX / 16; /* NMAX is divisible by 16 */ + do { + DO16(buf); /* 16 sums unrolled */ + buf += 16; + } while (--n); + MOD(adler); + MOD(sum2); + } + + /* do remaining bytes (less than NMAX, still just one modulo) */ + if (len) { /* avoid modulos if none remaining */ + while (len >= 16) { + len -= 16; + DO16(buf); + buf += 16; + } + while (len--) { + adler += *buf++; + sum2 += adler; + } + MOD(adler); + MOD(sum2); + } + + /* return recombined sums */ + return adler | (sum2 << 16); +} + +/* ========================================================================= */ +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ + return adler32_z(adler, buf, len); +} + +/* ========================================================================= */ +local uLong adler32_combine_(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ + unsigned long sum1; + unsigned long sum2; + unsigned rem; + + /* for negative len, return invalid adler32 as a clue for debugging */ + if (len2 < 0) + return 0xffffffffUL; + + /* the derivation of this formula is left as an exercise for the reader */ + MOD63(len2); /* assumes len2 >= 0 */ + rem = (unsigned)len2; + sum1 = adler1 & 0xffff; + sum2 = rem * sum1; + MOD(sum2); + sum1 += (adler2 & 0xffff) + BASE - 1; + sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; + if (sum1 >= BASE) sum1 -= BASE; + if (sum1 >= BASE) sum1 -= BASE; + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); + if (sum2 >= BASE) sum2 -= BASE; + return sum1 | (sum2 << 16); +} + +/* ========================================================================= */ +uLong ZEXPORT adler32_combine(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off_t len2; +{ + return adler32_combine_(adler1, adler2, len2); +} + +uLong ZEXPORT adler32_combine64(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ + return adler32_combine_(adler1, adler2, len2); +} diff --git a/optee_os/core/lib/zlib/gzguts.h b/optee_os/core/lib/zlib/gzguts.h new file mode 100644 index 0000000..9c5eb41 --- /dev/null +++ b/optee_os/core/lib/zlib/gzguts.h @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: Zlib */ +/* gzguts.h -- zlib internal header definitions for gz* operations + * Copyright (C) 2004-2019 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifdef _LARGEFILE64_SOURCE +# ifndef _LARGEFILE_SOURCE +# define _LARGEFILE_SOURCE 1 +# endif +# ifdef _FILE_OFFSET_BITS +# undef _FILE_OFFSET_BITS +# endif +#endif + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include +#include "zlib.h" +#ifdef STDC +# include +# include +# include +#endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif +#include + +#ifdef _WIN32 +# include +#endif + +#if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) +# include +#endif + +#if defined(_WIN32) +# define WIDECHAR +#endif + +#ifdef WINAPI_FAMILY +# define open _open +# define read _read +# define write _write +# define close _close +#endif + +#ifdef NO_DEFLATE /* for compatibility with old definition */ +# define NO_GZCOMPRESS +#endif + +#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(__CYGWIN__) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#ifndef HAVE_VSNPRINTF +# ifdef MSDOS +/* vsnprintf may exist on some MS-DOS compilers (DJGPP?), + but for now we just assume it doesn't. */ +# define NO_vsnprintf +# endif +# ifdef __TURBOC__ +# define NO_vsnprintf +# endif +# ifdef WIN32 +/* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ +# if !defined(vsnprintf) && !defined(NO_vsnprintf) +# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) +# define vsnprintf _vsnprintf +# endif +# endif +# endif +# ifdef __SASC +# define NO_vsnprintf +# endif +# ifdef VMS +# define NO_vsnprintf +# endif +# ifdef __OS400__ +# define NO_vsnprintf +# endif +# ifdef __MVS__ +# define NO_vsnprintf +# endif +#endif + +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where + the result is assured to fit in the space provided */ +#if defined(_MSC_VER) && _MSC_VER < 1900 +# define snprintf _snprintf +#endif + +#ifndef local +# define local static +#endif +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ + +/* gz* functions always use library allocation functions */ +#ifndef STDC + extern voidp malloc OF((uInt size)); + extern void free OF((voidpf ptr)); +#endif + +/* get errno and strerror definition */ +#if defined UNDER_CE +# include +# define zstrerror() gz_strwinerror((DWORD)GetLastError()) +#else +# ifndef NO_STRERROR +# include +# define zstrerror() strerror(errno) +# else +# define zstrerror() "stdio error (consult errno)" +# endif +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); +#endif + +/* default memLevel */ +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif + +/* default i/o buffer size -- double this for output when reading (this and + twice this must be able to fit in an unsigned type) */ +#define GZBUFSIZE 8192 + +/* gzip modes, also provide a little integrity check on the passed structure */ +#define GZ_NONE 0 +#define GZ_READ 7247 +#define GZ_WRITE 31153 +#define GZ_APPEND 1 /* mode set to GZ_WRITE after the file is opened */ + +/* values for gz_state how */ +#define LOOK 0 /* look for a gzip header */ +#define COPY 1 /* copy input directly */ +#define GZIP 2 /* decompress a gzip stream */ + +/* internal gzip file state data structure */ +typedef struct { + /* exposed contents for gzgetc() macro */ + struct gzFile_s x; /* "x" for exposed */ + /* x.have: number of bytes available at x.next */ + /* x.next: next output data to deliver or write */ + /* x.pos: current position in uncompressed data */ + /* used for both reading and writing */ + int mode; /* see gzip modes above */ + int fd; /* file descriptor */ + char *path; /* path or fd for error messages */ + unsigned size; /* buffer size, zero if not allocated yet */ + unsigned want; /* requested buffer size, default is GZBUFSIZE */ + unsigned char *in; /* input buffer (double-sized when writing) */ + unsigned char *out; /* output buffer (double-sized when reading) */ + int direct; /* 0 if processing gzip, 1 if transparent */ + /* just for reading */ + int how; /* 0: get header, 1: copy, 2: decompress */ + z_off64_t start; /* where the gzip data started, for rewinding */ + int eof; /* true if end of input file reached */ + int past; /* true if read requested past end */ + /* just for writing */ + int level; /* compression level */ + int strategy; /* compression strategy */ + int reset; /* true if a reset is pending after a Z_FINISH */ + /* seek request */ + z_off64_t skip; /* amount to skip (already rewound if backwards) */ + int seek; /* true if seek request pending */ + /* error information */ + int err; /* error code */ + char *msg; /* error message */ + /* zlib inflate or deflate stream */ + z_stream strm; /* stream structure in-place (not a pointer) */ +} gz_state; +typedef gz_state FAR *gz_statep; + +/* shared functions */ +void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); +#if defined UNDER_CE +char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); +#endif + +/* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t + value -- needed when comparing unsigned to z_off64_t, which is signed + (possible z_off64_t types off_t, off64_t, and long are all signed) */ +#ifdef INT_MAX +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) +#else +unsigned ZLIB_INTERNAL gz_intmax OF((void)); +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) +#endif diff --git a/optee_os/core/lib/zlib/inffast.c b/optee_os/core/lib/zlib/inffast.c new file mode 100644 index 0000000..aa5ea80 --- /dev/null +++ b/optee_os/core/lib/zlib/inffast.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: Zlib +/* inffast.c -- fast decoding + * Copyright (C) 1995-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "inffast.h" + +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") +#else + +/* + Decode literal, length, and distance codes and write out the resulting + literal and match bytes until either not enough input or output is + available, an end-of-block is encountered, or a data error is encountered. + When large enough input and output buffers are supplied to inflate(), for + example, a 16K input buffer and a 64K output buffer, more than 95% of the + inflate execution time is spent in this routine. + + Entry assumptions: + + state->mode == LEN + strm->avail_in >= 6 + strm->avail_out >= 258 + start >= strm->avail_out + state->bits < 8 + + On return, state->mode is one of: + + LEN -- ran out of enough output space or enough available input + TYPE -- reached end of block code, inflate() to interpret next block + BAD -- error in block data + + Notes: + + - The maximum input bits used by a length/distance pair is 15 bits for the + length code, 5 bits for the length extra, 15 bits for the distance code, + and 13 bits for the distance extra. This totals 48 bits, or six bytes. + Therefore if strm->avail_in >= 6, then there is enough input to avoid + checking for available input while decoding. + + - The maximum bytes that a single length/distance pair can output is 258 + bytes, which is the maximum length that can be coded. inflate_fast() + requires strm->avail_out >= 258 for each loop to avoid checking for + output space. + */ +void ZLIB_INTERNAL inflate_fast(strm, start) +z_streamp strm; +unsigned start; /* inflate()'s starting value for strm->avail_out */ +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ + unsigned char FAR *out; /* local strm->next_out */ + unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ + unsigned char FAR *end; /* while out < end, enough space available */ +#ifdef INFLATE_STRICT + unsigned dmax; /* maximum distance from zlib header */ +#endif + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ + unsigned long hold; /* local strm->hold */ + unsigned bits; /* local strm->bits */ + code const FAR *lcode; /* local strm->lencode */ + code const FAR *dcode; /* local strm->distcode */ + unsigned lmask; /* mask for first level of length codes */ + unsigned dmask; /* mask for first level of distance codes */ + code const *here; /* retrieved table entry */ + unsigned op; /* code bits, operation, extra bits, or */ + /* window position, window bytes to copy */ + unsigned len; /* match length, unused bytes */ + unsigned dist; /* match distance */ + unsigned char FAR *from; /* where to copy match from */ + + /* copy state to local variables */ + state = (struct inflate_state FAR *)strm->state; + in = strm->next_in; + last = in + (strm->avail_in - 5); + out = strm->next_out; + beg = out - (start - strm->avail_out); + end = out + (strm->avail_out - 257); +#ifdef INFLATE_STRICT + dmax = state->dmax; +#endif + wsize = state->wsize; + whave = state->whave; + wnext = state->wnext; + window = state->window; + hold = state->hold; + bits = state->bits; + lcode = state->lencode; + dcode = state->distcode; + lmask = (1U << state->lenbits) - 1; + dmask = (1U << state->distbits) - 1; + + /* decode literals and length/distances until end-of-block or not enough + input data or output space */ + do { + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + here = lcode + (hold & lmask); + dolen: + op = (unsigned)(here->bits); + hold >>= op; + bits -= op; + op = (unsigned)(here->op); + if (op == 0) { /* literal */ + Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here->val)); + *out++ = (unsigned char)(here->val); + } + else if (op & 16) { /* length base */ + len = (unsigned)(here->val); + op &= 15; /* number of extra bits */ + if (op) { + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + len += (unsigned)hold & ((1U << op) - 1); + hold >>= op; + bits -= op; + } + Tracevv((stderr, "inflate: length %u\n", len)); + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + here = dcode + (hold & dmask); + dodist: + op = (unsigned)(here->bits); + hold >>= op; + bits -= op; + op = (unsigned)(here->op); + if (op & 16) { /* distance base */ + dist = (unsigned)(here->val); + op &= 15; /* number of extra bits */ + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + } + dist += (unsigned)hold & ((1U << op) - 1); +#ifdef INFLATE_STRICT + if (dist > dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + hold >>= op; + bits -= op; + Tracevv((stderr, "inflate: distance %u\n", dist)); + op = (unsigned)(out - beg); /* max distance in output */ + if (dist > op) { /* see if copy from window */ + op = dist - op; /* distance back in window */ + if (op > whave) { + if (state->sane) { + strm->msg = + (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + *out++ = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + *out++ = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + *out++ = *from++; + } while (--len); + continue; + } +#endif + } + from = window; + if (wnext == 0) { /* very common case */ + from += wsize - op; + if (op < len) { /* some from window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + else if (wnext < op) { /* wrap around window */ + from += wsize + wnext - op; + op -= wnext; + if (op < len) { /* some from end of window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = window; + if (wnext < len) { /* some from start of window */ + op = wnext; + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + } + else { /* contiguous in window */ + from += wnext - op; + if (op < len) { /* some from window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + while (len > 2) { + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; + len -= 3; + } + if (len) { + *out++ = *from++; + if (len > 1) + *out++ = *from++; + } + } + else { + from = out - dist; /* copy direct from output */ + do { /* minimum length is three */ + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; + len -= 3; + } while (len > 2); + if (len) { + *out++ = *from++; + if (len > 1) + *out++ = *from++; + } + } + } + else if ((op & 64) == 0) { /* 2nd level distance code */ + here = dcode + here->val + (hold & ((1U << op) - 1)); + goto dodist; + } + else { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + } + else if ((op & 64) == 0) { /* 2nd level length code */ + here = lcode + here->val + (hold & ((1U << op) - 1)); + goto dolen; + } + else if (op & 32) { /* end-of-block */ + Tracevv((stderr, "inflate: end of block\n")); + state->mode = TYPE; + break; + } + else { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + } while (in < last && out < end); + + /* return unused bytes (on entry, bits < 8, so in won't go too far back) */ + len = bits >> 3; + in -= len; + bits -= len << 3; + hold &= (1U << bits) - 1; + + /* update state and return */ + strm->next_in = in; + strm->next_out = out; + strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); + strm->avail_out = (unsigned)(out < end ? + 257 + (end - out) : 257 - (out - end)); + state->hold = hold; + state->bits = bits; + return; +} + +/* + inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): + - Using bit fields for code structure + - Different op definition to avoid & for extra bits (do & for table bits) + - Three separate decoding do-loops for direct, window, and wnext == 0 + - Special case for distance > 1 copies to do overlapped load and store copy + - Explicit branch predictions (based on measured branch probabilities) + - Deferring match copy and interspersed it with decoding subsequent codes + - Swapping literal/length else + - Swapping window/direct else + - Larger unrolled copy loops (three is about right) + - Moving len -= 3 statement into middle of loop + */ + +#endif /* !ASMINF */ diff --git a/optee_os/core/lib/zlib/inffast.h b/optee_os/core/lib/zlib/inffast.h new file mode 100644 index 0000000..7bace8d --- /dev/null +++ b/optee_os/core/lib/zlib/inffast.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: Zlib */ +/* inffast.h -- header to use inffast.c + * Copyright (C) 1995-2003, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); diff --git a/optee_os/core/lib/zlib/inffixed.h b/optee_os/core/lib/zlib/inffixed.h new file mode 100644 index 0000000..d628327 --- /dev/null +++ b/optee_os/core/lib/zlib/inffixed.h @@ -0,0 +1,94 @@ + /* inffixed.h -- table for decoding fixed codes + * Generated automatically by makefixed(). + */ + + /* WARNING: this file should *not* be used by applications. + It is part of the implementation of this library and is + subject to change. Applications should only use zlib.h. + */ + + static const code lenfix[512] = { + {96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48}, + {0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128}, + {0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59}, + {0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176}, + {0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20}, + {21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100}, + {0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8}, + {0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216}, + {18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76}, + {0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114}, + {0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2}, + {0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148}, + {20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42}, + {0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86}, + {0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15}, + {0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236}, + {16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62}, + {0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142}, + {0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31}, + {0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162}, + {0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25}, + {0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105}, + {0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4}, + {0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202}, + {17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69}, + {0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125}, + {0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13}, + {0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195}, + {19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35}, + {0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91}, + {0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19}, + {0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246}, + {16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55}, + {0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135}, + {0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99}, + {0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190}, + {0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16}, + {20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96}, + {0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6}, + {0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209}, + {17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72}, + {0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116}, + {0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4}, + {0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153}, + {20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44}, + {0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82}, + {0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11}, + {0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229}, + {16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58}, + {0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138}, + {0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51}, + {0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173}, + {0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30}, + {0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110}, + {0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0}, + {0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195}, + {16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65}, + {0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121}, + {0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9}, + {0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258}, + {19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37}, + {0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93}, + {0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23}, + {0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251}, + {16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51}, + {0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131}, + {0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67}, + {0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183}, + {0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23}, + {64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103}, + {0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9}, + {0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223}, + {18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79}, + {0,9,255} + }; + + static const code distfix[32] = { + {16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025}, + {21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193}, + {18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385}, + {19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577}, + {16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073}, + {22,5,193},{64,5,0} + }; diff --git a/optee_os/core/lib/zlib/inflate.c b/optee_os/core/lib/zlib/inflate.c new file mode 100644 index 0000000..c494d06 --- /dev/null +++ b/optee_os/core/lib/zlib/inflate.c @@ -0,0 +1,1593 @@ +// SPDX-License-Identifier: Zlib +/* inflate.c -- zlib decompression + * Copyright (C) 1995-2022 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * Change history: + * + * 1.2.beta0 24 Nov 2002 + * - First version -- complete rewrite of inflate to simplify code, avoid + * creation of window when not needed, minimize use of window when it is + * needed, make inffast.c even faster, implement gzip decoding, and to + * improve code readability and style over the previous zlib inflate code + * + * 1.2.beta1 25 Nov 2002 + * - Use pointers for available input and output checking in inffast.c + * - Remove input and output counters in inffast.c + * - Change inffast.c entry and loop from avail_in >= 7 to >= 6 + * - Remove unnecessary second byte pull from length extra in inffast.c + * - Unroll direct copy to three copies per loop in inffast.c + * + * 1.2.beta2 4 Dec 2002 + * - Change external routine names to reduce potential conflicts + * - Correct filename to inffixed.h for fixed tables in inflate.c + * - Make hbuf[] unsigned char to match parameter type in inflate.c + * - Change strm->next_out[-state->offset] to *(strm->next_out - state->offset) + * to avoid negation problem on Alphas (64 bit) in inflate.c + * + * 1.2.beta3 22 Dec 2002 + * - Add comments on state->bits assertion in inffast.c + * - Add comments on op field in inftrees.h + * - Fix bug in reuse of allocated window after inflateReset() + * - Remove bit fields--back to byte structure for speed + * - Remove distance extra == 0 check in inflate_fast()--only helps for lengths + * - Change post-increments to pre-increments in inflate_fast(), PPC biased? + * - Add compile time option, POSTINC, to use post-increments instead (Intel?) + * - Make MATCH copy in inflate() much faster for when inflate_fast() not used + * - Use local copies of stream next and avail values, as well as local bit + * buffer and bit count in inflate()--for speed when inflate_fast() not used + * + * 1.2.beta4 1 Jan 2003 + * - Split ptr - 257 statements in inflate_table() to avoid compiler warnings + * - Move a comment on output buffer sizes from inffast.c to inflate.c + * - Add comments in inffast.c to introduce the inflate_fast() routine + * - Rearrange window copies in inflate_fast() for speed and simplification + * - Unroll last copy for window match in inflate_fast() + * - Use local copies of window variables in inflate_fast() for speed + * - Pull out common wnext == 0 case for speed in inflate_fast() + * - Make op and len in inflate_fast() unsigned for consistency + * - Add FAR to lcode and dcode declarations in inflate_fast() + * - Simplified bad distance check in inflate_fast() + * - Added inflateBackInit(), inflateBack(), and inflateBackEnd() in new + * source file infback.c to provide a call-back interface to inflate for + * programs like gzip and unzip -- uses window as output buffer to avoid + * window copying + * + * 1.2.beta5 1 Jan 2003 + * - Improved inflateBack() interface to allow the caller to provide initial + * input in strm. + * - Fixed stored blocks bug in inflateBack() + * + * 1.2.beta6 4 Jan 2003 + * - Added comments in inffast.c on effectiveness of POSTINC + * - Typecasting all around to reduce compiler warnings + * - Changed loops from while (1) or do {} while (1) to for (;;), again to + * make compilers happy + * - Changed type of window in inflateBackInit() to unsigned char * + * + * 1.2.beta7 27 Jan 2003 + * - Changed many types to unsigned or unsigned short to avoid warnings + * - Added inflateCopy() function + * + * 1.2.0 9 Mar 2003 + * - Changed inflateBack() interface to provide separate opaque descriptors + * for the in() and out() functions + * - Changed inflateBack() argument and in_func typedef to swap the length + * and buffer address return values for the input function + * - Check next_in and next_out for Z_NULL on entry to inflate() + * + * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "inffast.h" + +#ifdef MAKEFIXED +# ifndef BUILDFIXED +# define BUILDFIXED +# endif +#endif + +/* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); +local void fixedtables OF((struct inflate_state FAR *state)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); +#ifdef BUILDFIXED + void makefixed OF((void)); +#endif +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, + unsigned len)); + +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->flags = -1; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + +int ZEXPORT inflateReset(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + state->wsize = 0; + state->whave = 0; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(strm, windowBits) +z_streamp strm; +int windowBits; +{ + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 5; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) +z_streamp strm; +int windowBits; +const char *version; +int stream_size; +{ + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(strm, version, stream_size) +z_streamp strm; +const char *version; +int stream_size; +{ + return inflateInit2_(strm, DEF_WBITS, version, stream_size); +} + +int ZEXPORT inflatePrime(strm, bits, value) +z_streamp strm; +int bits; +int value; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; + value &= (1L << bits) - 1; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; + return Z_OK; +} + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications, since the rewriting of the tables and virgin + may not be thread-safe. + */ +local void fixedtables(state) +struct inflate_state FAR *state; +{ +#ifdef BUILDFIXED + static int virgin = 1; + static code *lenfix, *distfix; + static code fixed[544]; + + /* build fixed huffman tables if first call (may not be thread safe) */ + if (virgin) { + unsigned sym, bits; + static code *next; + + /* literal/length table */ + sym = 0; + while (sym < 144) state->lens[sym++] = 8; + while (sym < 256) state->lens[sym++] = 9; + while (sym < 280) state->lens[sym++] = 7; + while (sym < 288) state->lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); + + /* distance table */ + sym = 0; + while (sym < 32) state->lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); + + /* do this just once */ + virgin = 0; + } +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +#ifdef MAKEFIXED +#include + +/* + Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also + defines BUILDFIXED, so the tables are built on the fly. makefixed() writes + those tables to stdout, which would be piped to inffixed.h. A small program + can simply call makefixed to do this: + + void makefixed(void); + + int main(void) + { + makefixed(); + return 0; + } + + Then that can be linked with zlib built with MAKEFIXED defined and run: + + a.out > inffixed.h + */ +void makefixed() +{ + unsigned low, size; + struct inflate_state state; + + fixedtables(&state); + puts(" /* inffixed.h -- table for decoding fixed codes"); + puts(" * Generated automatically by makefixed()."); + puts(" */"); + puts(""); + puts(" /* WARNING: this file should *not* be used by applications."); + puts(" It is part of the implementation of this library and is"); + puts(" subject to change. Applications should only use zlib.h."); + puts(" */"); + puts(""); + size = 1U << 9; + printf(" static const code lenfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 7) == 0) printf("\n "); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); + size = 1U << 5; + printf("\n static const code distfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 6) == 0) printf("\n "); + printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, + state.distcode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); +} +#endif /* MAKEFIXED */ + +/* + Update the window with the last wsize (normally 32K) bytes written before + returning. If window does not exist yet, create it. This is only called + when a window is already in use, or when output has been written during this + inflate call, but the end of the deflate stream has not been reached yet. + It is also called to create a window for dictionary data when a dictionary + is loaded. + + Providing output buffers larger than 32K to inflate() should provide a speed + advantage, since only the last 32K of output is copied to the sliding window + upon return from inflate(), and since all distances after the first 32K of + output will fall in the output data, making match copies simpler and faster. + The advantage may be dependent on the size of the processor's data caches. + */ +local int updatewindow(strm, end, copy) +z_streamp strm; +const Bytef *end; +unsigned copy; +{ + struct inflate_state FAR *state; + unsigned dist; + + state = (struct inflate_state FAR *)strm->state; + + /* if it hasn't been done already, allocate space for the window */ + if (state->window == Z_NULL) { + state->window = (unsigned char FAR *) + ZALLOC(strm, 1U << state->wbits, + sizeof(unsigned char)); + if (state->window == Z_NULL) return 1; + } + + /* if window not in use yet, initialize */ + if (state->wsize == 0) { + state->wsize = 1U << state->wbits; + state->wnext = 0; + state->whave = 0; + } + + /* copy state->wsize or less output bytes into the circular window */ + if (copy >= state->wsize) { + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; + state->whave = state->wsize; + } + else { + dist = state->wsize - state->wnext; + if (dist > copy) dist = copy; + zmemcpy(state->window + state->wnext, end - copy, dist); + copy -= dist; + if (copy) { + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; + state->whave = state->wsize; + } + else { + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; + if (state->whave < state->wsize) state->whave += dist; + } + } + return 0; +} + +/* Macros for inflate(): */ + +/* check function to use adler32() for zlib or crc32() for gzip */ +#ifdef GUNZIP +# define UPDATE_CHECK(check, buf, len) \ + (state->flags ? crc32(check, buf, len) : adler32(check, buf, len)) +#else +# define UPDATE_CHECK(check, buf, len) adler32(check, buf, len) +#endif + +/* check macros for header crc */ +#ifdef GUNZIP +# define CRC2(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + check = crc32(check, hbuf, 2); \ + } while (0) + +# define CRC4(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + hbuf[2] = (unsigned char)((word) >> 16); \ + hbuf[3] = (unsigned char)((word) >> 24); \ + check = crc32(check, hbuf, 4); \ + } while (0) +#endif + +/* Load registers with state in inflate() for speed */ +#define LOAD() \ + do { \ + put = strm->next_out; \ + left = strm->avail_out; \ + next = strm->next_in; \ + have = strm->avail_in; \ + hold = state->hold; \ + bits = state->bits; \ + } while (0) + +/* Restore state from registers in inflate() */ +#define RESTORE() \ + do { \ + strm->next_out = put; \ + strm->avail_out = left; \ + strm->next_in = next; \ + strm->avail_in = have; \ + state->hold = hold; \ + state->bits = bits; \ + } while (0) + +/* Clear the input bit accumulator */ +#define INITBITS() \ + do { \ + hold = 0; \ + bits = 0; \ + } while (0) + +/* Get a byte of input into the bit accumulator, or return from inflate() + if there is no input available. */ +#define PULLBYTE() \ + do { \ + if (have == 0) goto inf_leave; \ + have--; \ + hold += (unsigned long)(*next++) << bits; \ + bits += 8; \ + } while (0) + +/* Assure that there are at least n bits in the bit accumulator. If there is + not enough available input to do that, then return from inflate(). */ +#define NEEDBITS(n) \ + do { \ + while (bits < (unsigned)(n)) \ + PULLBYTE(); \ + } while (0) + +/* Return the low n bits of the bit accumulator (n < 16) */ +#define BITS(n) \ + ((unsigned)hold & ((1U << (n)) - 1)) + +/* Remove n bits from the bit accumulator */ +#define DROPBITS(n) \ + do { \ + hold >>= (n); \ + bits -= (unsigned)(n); \ + } while (0) + +/* Remove zero to seven bits as needed to go to a byte boundary */ +#define BYTEBITS() \ + do { \ + hold >>= bits & 7; \ + bits -= bits & 7; \ + } while (0) + +/* + inflate() uses a state machine to process as much input data and generate as + much output data as possible before returning. The state machine is + structured roughly as follows: + + for (;;) switch (state) { + ... + case STATEn: + if (not enough input data or output space to make progress) + return; + ... make progress ... + state = STATEm; + break; + ... + } + + so when inflate() is called again, the same case is attempted again, and + if the appropriate resources are provided, the machine proceeds to the + next state. The NEEDBITS() macro is usually the way the state evaluates + whether it can proceed or should return. NEEDBITS() does the return if + the requested bits are not available. The typical use of the BITS macros + is: + + NEEDBITS(n); + ... do something with BITS(n) ... + DROPBITS(n); + + where NEEDBITS(n) either returns from inflate() if there isn't enough + input left to load n bits into the accumulator, or it continues. BITS(n) + gives the low n bits in the accumulator. When done, DROPBITS(n) drops + the low n bits off the accumulator. INITBITS() clears the accumulator + and sets the number of available bits to zero. BYTEBITS() discards just + enough bits to put the accumulator on a byte boundary. After BYTEBITS() + and a NEEDBITS(8), then BITS(8) would return the next byte in the stream. + + NEEDBITS(n) uses PULLBYTE() to get an available byte of input, or to return + if there is no input available. The decoding of variable length codes uses + PULLBYTE() directly in order to pull just enough bytes to decode the next + code, and no more. + + Some states loop until they get enough input, making sure that enough + state information is maintained to continue the loop where it left off + if NEEDBITS() returns in the loop. For example, want, need, and keep + would all have to actually be part of the saved state in case NEEDBITS() + returns: + + case STATEw: + while (want < need) { + NEEDBITS(n); + keep[want++] = BITS(n); + DROPBITS(n); + } + state = STATEx; + case STATEx: + + As shown above, if the next state is also the next case, then the break + is omitted. + + A state may also return if there is not enough output space available to + complete that state. Those states are copying stored data, writing a + literal byte, and copying a matching string. + + When returning, a "goto inf_leave" is used to update the total counters, + update the check value, and determine whether any progress has been made + during that inflate() call in order to return the proper return code. + Progress is defined as a change in either strm->avail_in or strm->avail_out. + When there is a window, goto inf_leave will update the window with the last + output written. If a goto inf_leave occurs in the middle of decompression + and there is no window currently, goto inf_leave will create one and copy + output to the window for the next call of inflate(). + + In this implementation, the flush parameter of inflate() only affects the + return code (per zlib.h). inflate() always writes as much as possible to + strm->next_out, given the space available and the provided input--the effect + documented in zlib.h of Z_SYNC_FLUSH. Furthermore, inflate() always defers + the allocation of and copying into a sliding window until necessary, which + provides the effect documented in zlib.h for Z_FINISH when the entire input + stream available. So the only thing the flush parameter actually does is: + when flush is set to Z_FINISH, inflate() cannot return Z_OK. Instead it + will return Z_BUF_ERROR if it has not reached the end of the stream. + */ + +int ZEXPORT inflate(strm, flush) +z_streamp strm; +int flush; +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *next; /* next input */ + unsigned char FAR *put; /* next output */ + unsigned have, left; /* available input and output */ + unsigned long hold; /* bit buffer */ + unsigned bits; /* bits in bit buffer */ + unsigned in, out; /* save starting available input and output */ + unsigned copy; /* number of stored or match bytes to copy */ + unsigned char FAR *from; /* where to copy match bytes from */ + code here; /* current decoding table entry */ + code last; /* parent table entry */ + unsigned len; /* length to copy for repeats, bits to drop */ + int ret; /* return code */ +#ifdef GUNZIP + unsigned char hbuf[4]; /* buffer for gzip header crc calculation */ +#endif + static const unsigned short order[19] = /* permutation of code lengths */ + {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || + (strm->next_in == Z_NULL && strm->avail_in != 0)) + return Z_STREAM_ERROR; + + state = (struct inflate_state FAR *)strm->state; + if (state->mode == TYPE) state->mode = TYPEDO; /* skip check */ + LOAD(); + in = have; + out = left; + ret = Z_OK; + for (;;) + switch (state->mode) { + case HEAD: + if (state->wrap == 0) { + state->mode = TYPEDO; + break; + } + NEEDBITS(16); +#ifdef GUNZIP + if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; + state->check = crc32(0L, Z_NULL, 0); + CRC2(state->check, hold); + INITBITS(); + state->mode = FLAGS; + break; + } + if (state->head != Z_NULL) + state->head->done = -1; + if (!(state->wrap & 1) || /* check if zlib header allowed */ +#else + if ( +#endif + ((BITS(8) << 8) + (hold >> 8)) % 31) { + strm->msg = (char *)"incorrect header check"; + state->mode = BAD; + break; + } + if (BITS(4) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + DROPBITS(4); + len = BITS(4) + 8; + if (state->wbits == 0) + state->wbits = len; + if (len > 15 || len > state->wbits) { + strm->msg = (char *)"invalid window size"; + state->mode = BAD; + break; + } + state->dmax = 1U << len; + state->flags = 0; /* indicate zlib header */ + Tracev((stderr, "inflate: zlib header ok\n")); + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = hold & 0x200 ? DICTID : TYPE; + INITBITS(); + break; +#ifdef GUNZIP + case FLAGS: + NEEDBITS(16); + state->flags = (int)(hold); + if ((state->flags & 0xff) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + if (state->flags & 0xe000) { + strm->msg = (char *)"unknown header flags set"; + state->mode = BAD; + break; + } + if (state->head != Z_NULL) + state->head->text = (int)((hold >> 8) & 1); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = TIME; + /* fallthrough */ + case TIME: + NEEDBITS(32); + if (state->head != Z_NULL) + state->head->time = hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); + INITBITS(); + state->mode = OS; + /* fallthrough */ + case OS: + NEEDBITS(16); + if (state->head != Z_NULL) { + state->head->xflags = (int)(hold & 0xff); + state->head->os = (int)(hold >> 8); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = EXLEN; + /* fallthrough */ + case EXLEN: + if (state->flags & 0x0400) { + NEEDBITS(16); + state->length = (unsigned)(hold); + if (state->head != Z_NULL) + state->head->extra_len = (unsigned)hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + } + else if (state->head != Z_NULL) + state->head->extra = Z_NULL; + state->mode = EXTRA; + /* fallthrough */ + case EXTRA: + if (state->flags & 0x0400) { + copy = state->length; + if (copy > have) copy = have; + if (copy) { + if (state->head != Z_NULL && + state->head->extra != Z_NULL) { + len = state->head->extra_len - state->length; + zmemcpy(state->head->extra + len, next, + len + copy > state->head->extra_max ? + state->head->extra_max - len : copy); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + state->length -= copy; + } + if (state->length) goto inf_leave; + } + state->length = 0; + state->mode = NAME; + /* fallthrough */ + case NAME: + if (state->flags & 0x0800) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->name != Z_NULL && + state->length < state->head->name_max) + state->head->name[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->name = Z_NULL; + state->length = 0; + state->mode = COMMENT; + /* fallthrough */ + case COMMENT: + if (state->flags & 0x1000) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->comment != Z_NULL && + state->length < state->head->comm_max) + state->head->comment[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->comment = Z_NULL; + state->mode = HCRC; + /* fallthrough */ + case HCRC: + if (state->flags & 0x0200) { + NEEDBITS(16); + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { + strm->msg = (char *)"header crc mismatch"; + state->mode = BAD; + break; + } + INITBITS(); + } + if (state->head != Z_NULL) { + state->head->hcrc = (int)((state->flags >> 9) & 1); + state->head->done = 1; + } + strm->adler = state->check = crc32(0L, Z_NULL, 0); + state->mode = TYPE; + break; +#endif + case DICTID: + NEEDBITS(32); + strm->adler = state->check = ZSWAP32(hold); + INITBITS(); + state->mode = DICT; + /* fallthrough */ + case DICT: + if (state->havedict == 0) { + RESTORE(); + return Z_NEED_DICT; + } + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = TYPE; + /* fallthrough */ + case TYPE: + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case TYPEDO: + if (state->last) { + BYTEBITS(); + state->mode = CHECK; + break; + } + NEEDBITS(3); + state->last = BITS(1); + DROPBITS(1); + switch (BITS(2)) { + case 0: /* stored block */ + Tracev((stderr, "inflate: stored block%s\n", + state->last ? " (last)" : "")); + state->mode = STORED; + break; + case 1: /* fixed block */ + fixedtables(state); + Tracev((stderr, "inflate: fixed codes block%s\n", + state->last ? " (last)" : "")); + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } + break; + case 2: /* dynamic block */ + Tracev((stderr, "inflate: dynamic codes block%s\n", + state->last ? " (last)" : "")); + state->mode = TABLE; + break; + case 3: + strm->msg = (char *)"invalid block type"; + state->mode = BAD; + } + DROPBITS(2); + break; + case STORED: + BYTEBITS(); /* go to byte boundary */ + NEEDBITS(32); + if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { + strm->msg = (char *)"invalid stored block lengths"; + state->mode = BAD; + break; + } + state->length = (unsigned)hold & 0xffff; + Tracev((stderr, "inflate: stored length %u\n", + state->length)); + INITBITS(); + state->mode = COPY_; + if (flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case COPY_: + state->mode = COPY; + /* fallthrough */ + case COPY: + copy = state->length; + if (copy) { + if (copy > have) copy = have; + if (copy > left) copy = left; + if (copy == 0) goto inf_leave; + zmemcpy(put, next, copy); + have -= copy; + next += copy; + left -= copy; + put += copy; + state->length -= copy; + break; + } + Tracev((stderr, "inflate: stored end\n")); + state->mode = TYPE; + break; + case TABLE: + NEEDBITS(14); + state->nlen = BITS(5) + 257; + DROPBITS(5); + state->ndist = BITS(5) + 1; + DROPBITS(5); + state->ncode = BITS(4) + 4; + DROPBITS(4); +#ifndef PKZIP_BUG_WORKAROUND + if (state->nlen > 286 || state->ndist > 30) { + strm->msg = (char *)"too many length or distance symbols"; + state->mode = BAD; + break; + } +#endif + Tracev((stderr, "inflate: table sizes ok\n")); + state->have = 0; + state->mode = LENLENS; + /* fallthrough */ + case LENLENS: + while (state->have < state->ncode) { + NEEDBITS(3); + state->lens[order[state->have++]] = (unsigned short)BITS(3); + DROPBITS(3); + } + while (state->have < 19) + state->lens[order[state->have++]] = 0; + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 7; + ret = inflate_table(CODES, state->lens, 19, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid code lengths set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: code lengths ok\n")); + state->have = 0; + state->mode = CODELENS; + /* fallthrough */ + case CODELENS: + while (state->have < state->nlen + state->ndist) { + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; + } + else { + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); + if (state->have == 0) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + len = state->lens[state->have - 1]; + copy = 3 + BITS(2); + DROPBITS(2); + } + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); + len = 0; + copy = 3 + BITS(3); + DROPBITS(3); + } + else { + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); + len = 0; + copy = 11 + BITS(7); + DROPBITS(7); + } + if (state->have + copy > state->nlen + state->ndist) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + while (copy--) + state->lens[state->have++] = (unsigned short)len; + } + } + + /* handle error breaks in while */ + if (state->mode == BAD) break; + + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 9; + ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid literal/lengths set"; + state->mode = BAD; + break; + } + state->distcode = (const code FAR *)(state->next); + state->distbits = 6; + ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, + &(state->next), &(state->distbits), state->work); + if (ret) { + strm->msg = (char *)"invalid distances set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN_; + if (flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case LEN_: + state->mode = LEN; + /* fallthrough */ + case LEN: + if (have >= 6 && left >= 258) { + RESTORE(); + inflate_fast(strm, out); + LOAD(); + if (state->mode == TYPE) + state->back = -1; + break; + } + state->back = 0; + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.op && (here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->lencode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + state->length = (unsigned)here.val; + if ((int)(here.op) == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + state->mode = LIT; + break; + } + if (here.op & 32) { + Tracevv((stderr, "inflate: end of block\n")); + state->back = -1; + state->mode = TYPE; + break; + } + if (here.op & 64) { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + state->extra = (unsigned)(here.op) & 15; + state->mode = LENEXT; + /* fallthrough */ + case LENEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->length += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } + Tracevv((stderr, "inflate: length %u\n", state->length)); + state->was = state->length; + state->mode = DIST; + /* fallthrough */ + case DIST: + for (;;) { + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if ((here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->distcode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + if (here.op & 64) { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + state->offset = (unsigned)here.val; + state->extra = (unsigned)(here.op) & 15; + state->mode = DISTEXT; + /* fallthrough */ + case DISTEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->offset += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } +#ifdef INFLATE_STRICT + if (state->offset > state->dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + Tracevv((stderr, "inflate: distance %u\n", state->offset)); + state->mode = MATCH; + /* fallthrough */ + case MATCH: + if (left == 0) goto inf_leave; + copy = out - left; + if (state->offset > copy) { /* copy from window */ + copy = state->offset - copy; + if (copy > state->whave) { + if (state->sane) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + Trace((stderr, "inflate.c too far\n")); + copy -= state->whave; + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = 0; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; +#endif + } + if (copy > state->wnext) { + copy -= state->wnext; + from = state->window + (state->wsize - copy); + } + else + from = state->window + (state->wnext - copy); + if (copy > state->length) copy = state->length; + } + else { /* copy from output */ + from = put - state->offset; + copy = state->length; + } + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = *from++; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; + case LIT: + if (left == 0) goto inf_leave; + *put++ = (unsigned char)(state->length); + left--; + state->mode = LEN; + break; + case CHECK: + if (state->wrap) { + NEEDBITS(32); + out -= left; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE_CHECK(state->check, put - out, out); + out = left; + if ((state->wrap & 4) && ( +#ifdef GUNZIP + state->flags ? hold : +#endif + ZSWAP32(hold)) != state->check) { + strm->msg = (char *)"incorrect data check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: check matches trailer\n")); + } +#ifdef GUNZIP + state->mode = LENGTH; + /* fallthrough */ + case LENGTH: + if (state->wrap && state->flags) { + NEEDBITS(32); + if ((state->wrap & 4) && hold != (state->total & 0xffffffff)) { + strm->msg = (char *)"incorrect length check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: length matches trailer\n")); + } +#endif + state->mode = DONE; + /* fallthrough */ + case DONE: + ret = Z_STREAM_END; + goto inf_leave; + case BAD: + ret = Z_DATA_ERROR; + goto inf_leave; + case MEM: + return Z_MEM_ERROR; + case SYNC: + /* fallthrough */ + default: + return Z_STREAM_ERROR; + } + + /* + Return from inflate(), updating the total counts and the check value. + If there was no progress during the inflate() call, return a buffer + error. Call updatewindow() to create and/or update the window state. + Note: a memory error from inflate() is non-recoverable. + */ + inf_leave: + RESTORE(); + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { + state->mode = MEM; + return Z_MEM_ERROR; + } + in -= strm->avail_in; + out -= strm->avail_out; + strm->total_in += in; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE_CHECK(state->check, strm->next_out - out, out); + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + + (state->mode == TYPE ? 128 : 0) + + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); + if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) + ret = Z_BUF_ERROR; + return ret; +} + +int ZEXPORT inflateEnd(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->window != Z_NULL) ZFREE(strm, state->window); + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); + return Z_OK; +} + +int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) +z_streamp strm; +Bytef *dictionary; +uInt *dictLength; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + +int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) +z_streamp strm; +const Bytef *dictionary; +uInt dictLength; +{ + struct inflate_state FAR *state; + unsigned long dictid; + int ret; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->wrap != 0 && state->mode != DICT) + return Z_STREAM_ERROR; + + /* check for correct dictionary identifier */ + if (state->mode == DICT) { + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) + return Z_DATA_ERROR; + } + + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { + state->mode = MEM; + return Z_MEM_ERROR; + } + state->havedict = 1; + Tracev((stderr, "inflate: dictionary set\n")); + return Z_OK; +} + +int ZEXPORT inflateGetHeader(strm, head) +z_streamp strm; +gz_headerp head; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; + + /* save header structure */ + state->head = head; + head->done = 0; + return Z_OK; +} + +/* + Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found + or when out of input. When called, *have is the number of pattern bytes + found in order so far, in 0..3. On return *have is updated to the new + state. If on return *have equals four, then the pattern was found and the + return value is how many bytes were read including the last byte of the + pattern. If *have is less than four, then the pattern has not been found + yet and the return value is len. In the latter case, syncsearch() can be + called again with more data and the *have state. *have is initialized to + zero for the first call. + */ +local unsigned syncsearch(have, buf, len) +unsigned FAR *have; +const unsigned char FAR *buf; +unsigned len; +{ + unsigned got; + unsigned next; + + got = *have; + next = 0; + while (next < len && got < 4) { + if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) + got++; + else if (buf[next]) + got = 0; + else + got = 4 - got; + next++; + } + *have = got; + return next; +} + +int ZEXPORT inflateSync(strm) +z_streamp strm; +{ + unsigned len; /* number of bytes to look at or looked at */ + int flags; /* temporary to save header status */ + unsigned long in, out; /* temporary to save total_in and total_out */ + unsigned char buf[4]; /* to restore bit buffer to byte string */ + struct inflate_state FAR *state; + + /* check parameters */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; + + /* if first time, start search in bit buffer */ + if (state->mode != SYNC) { + state->mode = SYNC; + state->hold <<= state->bits & 7; + state->bits -= state->bits & 7; + len = 0; + while (state->bits >= 8) { + buf[len++] = (unsigned char)(state->hold); + state->hold >>= 8; + state->bits -= 8; + } + state->have = 0; + syncsearch(&(state->have), buf, len); + } + + /* search available input */ + len = syncsearch(&(state->have), strm->next_in, strm->avail_in); + strm->avail_in -= len; + strm->next_in += len; + strm->total_in += len; + + /* return no joy or set up to restart inflate() on a new block */ + if (state->have != 4) return Z_DATA_ERROR; + if (state->flags == -1) + state->wrap = 0; /* if no header yet, treat as raw */ + else + state->wrap &= ~4; /* no point in computing a check value now */ + flags = state->flags; + in = strm->total_in; out = strm->total_out; + inflateReset(strm); + strm->total_in = in; strm->total_out = out; + state->flags = flags; + state->mode = TYPE; + return Z_OK; +} + +/* + Returns true if inflate is currently at the end of a block generated by + Z_SYNC_FLUSH or Z_FULL_FLUSH. This function is used by one PPP + implementation to provide an additional safety check. PPP uses + Z_SYNC_FLUSH but removes the length bytes of the resulting empty stored + block. When decompressing, PPP checks that at the end of input packet, + inflate is waiting for these length bytes. + */ +int ZEXPORT inflateSyncPoint(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + return state->mode == STORED && state->bits == 0; +} + +int ZEXPORT inflateCopy(dest, source) +z_streamp dest; +z_streamp source; +{ + struct inflate_state FAR *state; + struct inflate_state FAR *copy; + unsigned char FAR *window; + unsigned wsize; + + /* check input */ + if (inflateStateCheck(source) || dest == Z_NULL) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)source->state; + + /* allocate space */ + copy = (struct inflate_state FAR *) + ZALLOC(source, 1, sizeof(struct inflate_state)); + if (copy == Z_NULL) return Z_MEM_ERROR; + window = Z_NULL; + if (state->window != Z_NULL) { + window = (unsigned char FAR *) + ZALLOC(source, 1U << state->wbits, sizeof(unsigned char)); + if (window == Z_NULL) { + ZFREE(source, copy); + return Z_MEM_ERROR; + } + } + + /* copy state */ + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; + if (state->lencode >= state->codes && + state->lencode <= state->codes + ENOUGH - 1) { + copy->lencode = copy->codes + (state->lencode - state->codes); + copy->distcode = copy->codes + (state->distcode - state->codes); + } + copy->next = copy->codes + (state->next - state->codes); + if (window != Z_NULL) { + wsize = 1U << state->wbits; + zmemcpy(window, state->window, wsize); + } + copy->window = window; + dest->state = (struct internal_state FAR *)copy; + return Z_OK; +} + +int ZEXPORT inflateUndermine(strm, subvert) +z_streamp strm; +int subvert; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; + return Z_OK; +#else + (void)subvert; + state->sane = 1; + return Z_DATA_ERROR; +#endif +} + +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check && state->wrap) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + +long ZEXPORT inflateMark(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) + return -(1L << 16); + state = (struct inflate_state FAR *)strm->state; + return (long)(((unsigned long)((long)state->back)) << 16) + + (state->mode == COPY ? state->length : + (state->mode == MATCH ? state->was - state->length : 0)); +} + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/optee_os/core/lib/zlib/inflate.h b/optee_os/core/lib/zlib/inflate.h new file mode 100644 index 0000000..f912e57 --- /dev/null +++ b/optee_os/core/lib/zlib/inflate.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: Zlib */ +/* inflate.h -- internal inflate state definition + * Copyright (C) 1995-2019 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* define NO_GZIP when compiling if you want to disable gzip header and + trailer decoding by inflate(). NO_GZIP would be used to avoid linking in + the crc code when it is not needed. For shared libraries, gzip decoding + should be left enabled. */ +#ifndef NO_GZIP +# define GUNZIP +#endif + +/* Possible inflate modes between inflate() calls */ +typedef enum { + HEAD = 16180, /* i: waiting for magic header */ + FLAGS, /* i: waiting for method and flags (gzip) */ + TIME, /* i: waiting for modification time (gzip) */ + OS, /* i: waiting for extra flags and operating system (gzip) */ + EXLEN, /* i: waiting for extra length (gzip) */ + EXTRA, /* i: waiting for extra bytes (gzip) */ + NAME, /* i: waiting for end of file name (gzip) */ + COMMENT, /* i: waiting for end of comment (gzip) */ + HCRC, /* i: waiting for header crc (gzip) */ + DICTID, /* i: waiting for dictionary check value */ + DICT, /* waiting for inflateSetDictionary() call */ + TYPE, /* i: waiting for type bits, including last-flag bit */ + TYPEDO, /* i: same, but skip check to exit inflate on new block */ + STORED, /* i: waiting for stored size (length and complement) */ + COPY_, /* i/o: same as COPY below, but only first time in */ + COPY, /* i/o: waiting for input or output to copy stored block */ + TABLE, /* i: waiting for dynamic block table lengths */ + LENLENS, /* i: waiting for code length code lengths */ + CODELENS, /* i: waiting for length/lit and distance code lengths */ + LEN_, /* i: same as LEN below, but only first time in */ + LEN, /* i: waiting for length/lit/eob code */ + LENEXT, /* i: waiting for length extra bits */ + DIST, /* i: waiting for distance code */ + DISTEXT, /* i: waiting for distance extra bits */ + MATCH, /* o: waiting for output space to copy string */ + LIT, /* o: waiting for output space to write literal */ + CHECK, /* i: waiting for 32-bit check value */ + LENGTH, /* i: waiting for 32-bit length (gzip) */ + DONE, /* finished check, done -- remain here until reset */ + BAD, /* got a data error -- remain here until reset */ + MEM, /* got an inflate() memory error -- remain here until reset */ + SYNC /* looking for synchronization bytes to restart inflate() */ +} inflate_mode; + +/* + State transitions between above modes - + + (most modes can go to BAD or MEM on error -- not shown for clarity) + + Process header: + HEAD -> (gzip) or (zlib) or (raw) + (gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME -> COMMENT -> + HCRC -> TYPE + (zlib) -> DICTID or TYPE + DICTID -> DICT -> TYPE + (raw) -> TYPEDO + Read deflate blocks: + TYPE -> TYPEDO -> STORED or TABLE or LEN_ or CHECK + STORED -> COPY_ -> COPY -> TYPE + TABLE -> LENLENS -> CODELENS -> LEN_ + LEN_ -> LEN + Read deflate codes in fixed or dynamic block: + LEN -> LENEXT or LIT or TYPE + LENEXT -> DIST -> DISTEXT -> MATCH -> LEN + LIT -> LEN + Process trailer: + CHECK -> LENGTH -> DONE + */ + +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ +struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ + inflate_mode mode; /* current inflate mode */ + int last; /* true if processing last block */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ + int havedict; /* true if dictionary provided */ + int flags; /* gzip header method and flags, 0 if zlib, or + -1 if raw or no header yet */ + unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ + unsigned long check; /* protected copy of check value */ + unsigned long total; /* protected copy of output count */ + gz_headerp head; /* where to save gzip header information */ + /* sliding window */ + unsigned wbits; /* log base 2 of requested window size */ + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if needed */ + /* bit accumulator */ + unsigned long hold; /* input bit accumulator */ + unsigned bits; /* number of bits in "in" */ + /* for string and stored block copying */ + unsigned length; /* literal or length of data to copy */ + unsigned offset; /* distance back to copy string from */ + /* for table and code decoding */ + unsigned extra; /* extra bits needed */ + /* fixed and dynamic code tables */ + code const FAR *lencode; /* starting table for length/literal codes */ + code const FAR *distcode; /* starting table for distance codes */ + unsigned lenbits; /* index bits for lencode */ + unsigned distbits; /* index bits for distcode */ + /* dynamic table building */ + unsigned ncode; /* number of code length code lengths */ + unsigned nlen; /* number of length code lengths */ + unsigned ndist; /* number of distance code lengths */ + unsigned have; /* number of code lengths in lens[] */ + code FAR *next; /* next available space in codes[] */ + unsigned short lens[320]; /* temporary storage for code lengths */ + unsigned short work[288]; /* work area for code table building */ + code codes[ENOUGH]; /* space for code tables */ + int sane; /* if false, allow invalid distance too far */ + int back; /* bits back of last unprocessed length/lit */ + unsigned was; /* initial length of match */ +}; diff --git a/optee_os/core/lib/zlib/inftrees.c b/optee_os/core/lib/zlib/inftrees.c new file mode 100644 index 0000000..b1e83e8 --- /dev/null +++ b/optee_os/core/lib/zlib/inftrees.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: Zlib +/* inftrees.c -- generate Huffman trees for efficient decoding + * Copyright (C) 1995-2022 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" + +#define MAXBITS 15 + +const char inflate_copyright[] = + " inflate 1.2.12 Copyright 1995-2022 Mark Adler "; +/* + If you use the zlib library in a product, an acknowledgment is welcome + in the documentation of your product. If for some reason you cannot + include such an acknowledgment, I would appreciate that you keep this + copyright string in the executable of your product. + */ + +/* + Build a set of tables to decode the provided canonical Huffman code. + The code lengths are lens[0..codes-1]. The result starts at *table, + whose indices are 0..2^bits-1. work is a writable array of at least + lens shorts, which is used as a work area. type is the type of code + to be generated, CODES, LENS, or DISTS. On return, zero is success, + -1 is an invalid code, and +1 means that ENOUGH isn't enough. table + on return points to the next available entry's address. bits is the + requested root table index bits, and on return it is the actual root + table index bits. It will differ if the request is greater than the + longest code or if it is less than the shortest code. + */ +int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work) +codetype type; +unsigned short FAR *lens; +unsigned codes; +code FAR * FAR *table; +unsigned FAR *bits; +unsigned short FAR *work; +{ + unsigned len; /* a code's length in bits */ + unsigned sym; /* index of code symbols */ + unsigned min, max; /* minimum and maximum code lengths */ + unsigned root; /* number of index bits for root table */ + unsigned curr; /* number of index bits for current table */ + unsigned drop; /* code bits to drop for sub-table */ + int left; /* number of prefix codes available */ + unsigned used; /* code entries in table used */ + unsigned huff; /* Huffman code */ + unsigned incr; /* for incrementing code, index */ + unsigned fill; /* index for replicating entries */ + unsigned low; /* low bits for current root entry */ + unsigned mask; /* mask for low root bits */ + code here; /* table entry for duplication */ + code FAR *next; /* next available space in table */ + const unsigned short FAR *base; /* base value table to use */ + const unsigned short FAR *extra; /* extra bits table to use */ + unsigned match; /* use base and extra for symbol >= match */ + unsigned short count[MAXBITS+1]; /* number of codes of each length */ + unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ + static const unsigned short lbase[31] = { /* Length codes 257..285 base */ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; + static const unsigned short lext[31] = { /* Length codes 257..285 extra */ + 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 199, 202}; + static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, + 8193, 12289, 16385, 24577, 0, 0}; + static const unsigned short dext[32] = { /* Distance codes 0..29 extra */ + 16, 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, + 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, + 28, 28, 29, 29, 64, 64}; + + /* + Process a set of code lengths to create a canonical Huffman code. The + code lengths are lens[0..codes-1]. Each length corresponds to the + symbols 0..codes-1. The Huffman code is generated by first sorting the + symbols by length from short to long, and retaining the symbol order + for codes with equal lengths. Then the code starts with all zero bits + for the first code of the shortest length, and the codes are integer + increments for the same length, and zeros are appended as the length + increases. For the deflate format, these bits are stored backwards + from their more natural integer increment ordering, and so when the + decoding tables are built in the large loop below, the integer codes + are incremented backwards. + + This routine assumes, but does not check, that all of the entries in + lens[] are in the range 0..MAXBITS. The caller must assure this. + 1..MAXBITS is interpreted as that code length. zero means that that + symbol does not occur in this code. + + The codes are sorted by computing a count of codes for each length, + creating from that a table of starting indices for each length in the + sorted table, and then entering the symbols in order in the sorted + table. The sorted table is work[], with that space being provided by + the caller. + + The length counts are used for other purposes as well, i.e. finding + the minimum and maximum length codes, determining if there are any + codes at all, checking for a valid set of lengths, and looking ahead + at length counts to determine sub-table sizes when building the + decoding tables. + */ + + /* accumulate lengths for codes (assumes lens[] all in 0..MAXBITS) */ + for (len = 0; len <= MAXBITS; len++) + count[len] = 0; + for (sym = 0; sym < codes; sym++) + count[lens[sym]]++; + + /* bound code lengths, force root to be within code lengths */ + root = *bits; + for (max = MAXBITS; max >= 1; max--) + if (count[max] != 0) break; + if (root > max) root = max; + if (max == 0) { /* no symbols to code at all */ + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)1; + here.val = (unsigned short)0; + *(*table)++ = here; /* make a table to force an error */ + *(*table)++ = here; + *bits = 1; + return 0; /* no symbols, but wait for decoding to report error */ + } + for (min = 1; min < max; min++) + if (count[min] != 0) break; + if (root < min) root = min; + + /* check for an over-subscribed or incomplete set of lengths */ + left = 1; + for (len = 1; len <= MAXBITS; len++) { + left <<= 1; + left -= count[len]; + if (left < 0) return -1; /* over-subscribed */ + } + if (left > 0 && (type == CODES || max != 1)) + return -1; /* incomplete set */ + + /* generate offsets into symbol table for each length for sorting */ + offs[1] = 0; + for (len = 1; len < MAXBITS; len++) + offs[len + 1] = offs[len] + count[len]; + + /* sort symbols by length, by symbol order within each length */ + for (sym = 0; sym < codes; sym++) + if (lens[sym] != 0) work[offs[lens[sym]]++] = (unsigned short)sym; + + /* + Create and fill in decoding tables. In this loop, the table being + filled is at next and has curr index bits. The code being used is huff + with length len. That code is converted to an index by dropping drop + bits off of the bottom. For codes where len is less than drop + curr, + those top drop + curr - len bits are incremented through all values to + fill the table with replicated entries. + + root is the number of index bits for the root table. When len exceeds + root, sub-tables are created pointed to by the root entry with an index + of the low root bits of huff. This is saved in low to check for when a + new sub-table should be started. drop is zero when the root table is + being filled, and drop is root when sub-tables are being filled. + + When a new sub-table is needed, it is necessary to look ahead in the + code lengths to determine what size sub-table is needed. The length + counts are used for this, and so count[] is decremented as codes are + entered in the tables. + + used keeps track of how many table entries have been allocated from the + provided *table space. It is checked for LENS and DIST tables against + the constants ENOUGH_LENS and ENOUGH_DISTS to guard against changes in + the initial root table size constants. See the comments in inftrees.h + for more information. + + sym increments through all symbols, and the loop terminates when + all codes of length max, i.e. all codes, have been processed. This + routine permits incomplete codes, so another loop after this one fills + in the rest of the decoding tables with invalid code markers. + */ + + /* set up for code type */ + switch (type) { + case CODES: + base = extra = work; /* dummy value--not used */ + match = 20; + break; + case LENS: + base = lbase; + extra = lext; + match = 257; + break; + default: /* DISTS */ + base = dbase; + extra = dext; + match = 0; + } + + /* initialize state for loop */ + huff = 0; /* starting code */ + sym = 0; /* starting code symbol */ + len = min; /* starting code length */ + next = *table; /* current table to fill in */ + curr = root; /* current table index bits */ + drop = 0; /* current bits to drop from code for index */ + low = (unsigned)(-1); /* trigger new sub-table when len > root */ + used = 1U << root; /* use root table entries */ + mask = used - 1; /* mask for comparing low */ + + /* check available table space */ + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) + return 1; + + /* process all codes and make table entries */ + for (;;) { + /* create table entry */ + here.bits = (unsigned char)(len - drop); + if (work[sym] + 1U < match) { + here.op = (unsigned char)0; + here.val = work[sym]; + } + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; + } + else { + here.op = (unsigned char)(32 + 64); /* end of block */ + here.val = 0; + } + + /* replicate for those indices with low len bits equal to huff */ + incr = 1U << (len - drop); + fill = 1U << curr; + min = fill; /* save offset to next table */ + do { + fill -= incr; + next[(huff >> drop) + fill] = here; + } while (fill != 0); + + /* backwards increment the len-bit code huff */ + incr = 1U << (len - 1); + while (huff & incr) + incr >>= 1; + if (incr != 0) { + huff &= incr - 1; + huff += incr; + } + else + huff = 0; + + /* go to next symbol, update count, len */ + sym++; + if (--(count[len]) == 0) { + if (len == max) break; + len = lens[work[sym]]; + } + + /* create new sub-table if needed */ + if (len > root && (huff & mask) != low) { + /* if first time, transition to sub-tables */ + if (drop == 0) + drop = root; + + /* increment past last table */ + next += min; /* here min is 1 << curr */ + + /* determine length of next table */ + curr = len - drop; + left = (int)(1 << curr); + while (curr + drop < max) { + left -= count[curr + drop]; + if (left <= 0) break; + curr++; + left <<= 1; + } + + /* check for enough space */ + used += 1U << curr; + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) + return 1; + + /* point entry in root table to sub-table */ + low = huff & mask; + (*table)[low].op = (unsigned char)curr; + (*table)[low].bits = (unsigned char)root; + (*table)[low].val = (unsigned short)(next - *table); + } + } + + /* fill in remaining table entry if code is incomplete (guaranteed to have + at most one remaining entry, since if the code is incomplete, the + maximum code length that was allowed to get this far is one bit) */ + if (huff != 0) { + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)(len - drop); + here.val = (unsigned short)0; + next[huff] = here; + } + + /* set return parameters */ + *table += used; + *bits = root; + return 0; +} diff --git a/optee_os/core/lib/zlib/inftrees.h b/optee_os/core/lib/zlib/inftrees.h new file mode 100644 index 0000000..cb1678d --- /dev/null +++ b/optee_os/core/lib/zlib/inftrees.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: Zlib */ +/* inftrees.h -- header to use inftrees.c + * Copyright (C) 1995-2005, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* Structure for decoding tables. Each entry provides either the + information needed to do the operation requested by the code that + indexed that table entry, or it provides a pointer to another + table that indexes more bits of the code. op indicates whether + the entry is a pointer to another table, a literal, a length or + distance, an end-of-block, or an invalid code. For a table + pointer, the low four bits of op is the number of index bits of + that table. For a length or distance, the low four bits of op + is the number of extra bits to get after the code. bits is + the number of bits in this code or part of the code to drop off + of the bit buffer. val is the actual byte to output in the case + of a literal, the base length or distance, or the offset from + the current table to the next table. Each entry is four bytes. */ +typedef struct { + unsigned char op; /* operation, extra bits, table bits */ + unsigned char bits; /* bits in this part of the code */ + unsigned short val; /* offset in table or code value */ +} code; + +/* op values as set by inflate_table(): + 00000000 - literal + 0000tttt - table link, tttt != 0 is the number of table index bits + 0001eeee - length or distance, eeee is the number of extra bits + 01100000 - end of block + 01000000 - invalid code + */ + +/* Maximum size of the dynamic table. The maximum number of code structures is + 1444, which is the sum of 852 for literal/length codes and 592 for distance + codes. These values were found by exhaustive searches using the program + examples/enough.c found in the zlib distribtution. The arguments to that + program are the number of symbols, the initial root table size, and the + maximum bit length of a code. "enough 286 9 15" for literal/length codes + returns returns 852, and "enough 30 6 15" for distance codes returns 592. + The initial root table size (9 or 6) is found in the fifth argument of the + inflate_table() calls in inflate.c and infback.c. If the root table size is + changed, then these maximum sizes would be need to be recalculated and + updated. */ +#define ENOUGH_LENS 852 +#define ENOUGH_DISTS 592 +#define ENOUGH (ENOUGH_LENS+ENOUGH_DISTS) + +/* Type of code to build for inflate_table() */ +typedef enum { + CODES, + LENS, + DISTS +} codetype; + +int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens, + unsigned codes, code FAR * FAR *table, + unsigned FAR *bits, unsigned short FAR *work)); diff --git a/optee_os/core/lib/zlib/sub.mk b/optee_os/core/lib/zlib/sub.mk new file mode 100644 index 0000000..d4f225d --- /dev/null +++ b/optee_os/core/lib/zlib/sub.mk @@ -0,0 +1,8 @@ +global-incdirs-y += . +srcs-y += adler32.c +srcs-y += inffast.c +srcs-y += inflate.c +srcs-y += inftrees.c +srcs-y += zutil.c +cflags-remove-y += -Wold-style-definition +cflags-remove-y += -Wswitch-default diff --git a/optee_os/core/lib/zlib/zconf.h b/optee_os/core/lib/zlib/zconf.h new file mode 100644 index 0000000..a7d1374 --- /dev/null +++ b/optee_os/core/lib/zlib/zconf.h @@ -0,0 +1,544 @@ +/* SPDX-License-Identifier: Zlib */ +/* zconf.h -- configuration of the zlib compression library + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#ifndef ZCONF_H +#define ZCONF_H + +#define Z_SOLO +#define ZLIB_CONST +#define NO_GZIP + +/* + * If you *really* need a unique prefix for all types and library functions, + * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. + * Even better than compiling with -DZ_PREFIX would be to use configure to set + * this permanently in zconf.h using "./configure --zprefix". + */ +#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ +# define Z_PREFIX_SET + +/* all linked symbols and init macros */ +# define _dist_code z__dist_code +# define _length_code z__length_code +# define _tr_align z__tr_align +# define _tr_flush_bits z__tr_flush_bits +# define _tr_flush_block z__tr_flush_block +# define _tr_init z__tr_init +# define _tr_stored_block z__tr_stored_block +# define _tr_tally z__tr_tally +# define adler32 z_adler32 +# define adler32_combine z_adler32_combine +# define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z +# ifndef Z_SOLO +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# endif +# define crc32 z_crc32 +# define crc32_combine z_crc32_combine +# define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z +# define deflate z_deflate +# define deflateBound z_deflateBound +# define deflateCopy z_deflateCopy +# define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 +# define deflateInit2_ z_deflateInit2_ +# define deflateInit_ z_deflateInit_ +# define deflateParams z_deflateParams +# define deflatePending z_deflatePending +# define deflatePrime z_deflatePrime +# define deflateReset z_deflateReset +# define deflateResetKeep z_deflateResetKeep +# define deflateSetDictionary z_deflateSetDictionary +# define deflateSetHeader z_deflateSetHeader +# define deflateTune z_deflateTune +# define deflate_copyright z_deflate_copyright +# define get_crc_table z_get_crc_table +# ifndef Z_SOLO +# define gz_error z_gz_error +# define gz_intmax z_gz_intmax +# define gz_strwinerror z_gz_strwinerror +# define gzbuffer z_gzbuffer +# define gzclearerr z_gzclearerr +# define gzclose z_gzclose +# define gzclose_r z_gzclose_r +# define gzclose_w z_gzclose_w +# define gzdirect z_gzdirect +# define gzdopen z_gzdopen +# define gzeof z_gzeof +# define gzerror z_gzerror +# define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite +# define gzgetc z_gzgetc +# define gzgetc_ z_gzgetc_ +# define gzgets z_gzgets +# define gzoffset z_gzoffset +# define gzoffset64 z_gzoffset64 +# define gzopen z_gzopen +# define gzopen64 z_gzopen64 +# ifdef _WIN32 +# define gzopen_w z_gzopen_w +# endif +# define gzprintf z_gzprintf +# define gzputc z_gzputc +# define gzputs z_gzputs +# define gzread z_gzread +# define gzrewind z_gzrewind +# define gzseek z_gzseek +# define gzseek64 z_gzseek64 +# define gzsetparams z_gzsetparams +# define gztell z_gztell +# define gztell64 z_gztell64 +# define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf +# define gzwrite z_gzwrite +# endif +# define inflate z_inflate +# define inflateBack z_inflateBack +# define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit +# define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed +# define inflateCopy z_inflateCopy +# define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary +# define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 +# define inflateInit2_ z_inflateInit2_ +# define inflateInit_ z_inflateInit_ +# define inflateMark z_inflateMark +# define inflatePrime z_inflatePrime +# define inflateReset z_inflateReset +# define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep +# define inflateSetDictionary z_inflateSetDictionary +# define inflateSync z_inflateSync +# define inflateSyncPoint z_inflateSyncPoint +# define inflateUndermine z_inflateUndermine +# define inflateValidate z_inflateValidate +# define inflate_copyright z_inflate_copyright +# define inflate_fast z_inflate_fast +# define inflate_table z_inflate_table +# ifndef Z_SOLO +# define uncompress z_uncompress +# define uncompress2 z_uncompress2 +# endif +# define zError z_zError +# ifndef Z_SOLO +# define zcalloc z_zcalloc +# define zcfree z_zcfree +# endif +# define zlibCompileFlags z_zlibCompileFlags +# define zlibVersion z_zlibVersion + +/* all zlib typedefs in zlib.h and zconf.h */ +# define Byte z_Byte +# define Bytef z_Bytef +# define alloc_func z_alloc_func +# define charf z_charf +# define free_func z_free_func +# ifndef Z_SOLO +# define gzFile z_gzFile +# endif +# define gz_header z_gz_header +# define gz_headerp z_gz_headerp +# define in_func z_in_func +# define intf z_intf +# define out_func z_out_func +# define uInt z_uInt +# define uIntf z_uIntf +# define uLong z_uLong +# define uLongf z_uLongf +# define voidp z_voidp +# define voidpc z_voidpc +# define voidpf z_voidpf + +/* all zlib structs in zlib.h and zconf.h */ +# define gz_header_s z_gz_header_s +# define internal_state z_internal_state + +#endif + +#if defined(__MSDOS__) && !defined(MSDOS) +# define MSDOS +#endif +#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) +# define OS2 +#endif +#if defined(_WINDOWS) && !defined(WINDOWS) +# define WINDOWS +#endif +#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__) +# ifndef WIN32 +# define WIN32 +# endif +#endif +#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) +# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) +# ifndef SYS16BIT +# define SYS16BIT +# endif +# endif +#endif + +/* + * Compile with -DMAXSEG_64K if the alloc function cannot allocate more + * than 64k bytes at a time (needed on systems with 16-bit int). + */ +#ifdef SYS16BIT +# define MAXSEG_64K +#endif +#ifdef MSDOS +# define UNALIGNED_OK +#endif + +#ifdef __STDC_VERSION__ +# ifndef STDC +# define STDC +# endif +# if __STDC_VERSION__ >= 199901L +# ifndef STDC99 +# define STDC99 +# endif +# endif +#endif +#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) +# define STDC +#endif +#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) +# define STDC +#endif +#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) +# define STDC +#endif +#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) +# define STDC +#endif + +#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ +# define STDC +#endif + +#ifndef STDC +# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ +# define const /* note: need a more gentle solution here */ +# endif +#endif + +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong +#endif + +/* Maximum value for memLevel in deflateInit2 */ +#ifndef MAX_MEM_LEVEL +# ifdef MAXSEG_64K +# define MAX_MEM_LEVEL 8 +# else +# define MAX_MEM_LEVEL 9 +# endif +#endif + +/* Maximum value for windowBits in deflateInit2 and inflateInit2. + * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files + * created by gzip. (Files created by minigzip can still be extracted by + * gzip.) + */ +#ifndef MAX_WBITS +# define MAX_WBITS 15 /* 32K LZ77 window */ +#endif + +/* The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) + plus a few kilobytes for small objects. For example, if you want to reduce + the default memory requirements from 256K to 128K, compile with + make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" + Of course this will generally degrade compression (there's no free lunch). + + The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes + for small objects. +*/ + + /* Type declarations */ + +#ifndef OF /* function prototypes */ +# ifdef STDC +# define OF(args) args +# else +# define OF(args) () +# endif +#endif + +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + +/* The following definitions for FAR are needed only for MSDOS mixed + * model programming (small or medium model with some far allocations). + * This was tested only with MSC; for other MSDOS compilers you may have + * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, + * just define FAR to be empty. + */ +#ifdef SYS16BIT +# if defined(M_I86SM) || defined(M_I86MM) + /* MSC small or medium model */ +# define SMALL_MEDIUM +# ifdef _MSC_VER +# define FAR _far +# else +# define FAR far +# endif +# endif +# if (defined(__SMALL__) || defined(__MEDIUM__)) + /* Turbo C small or medium model */ +# define SMALL_MEDIUM +# ifdef __BORLANDC__ +# define FAR _far +# else +# define FAR far +# endif +# endif +#endif + +#if defined(WINDOWS) || defined(WIN32) + /* If building or using zlib as a DLL, define ZLIB_DLL. + * This is not mandatory, but it offers a little performance increase. + */ +# ifdef ZLIB_DLL +# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) +# ifdef ZLIB_INTERNAL +# define ZEXTERN extern __declspec(dllexport) +# else +# define ZEXTERN extern __declspec(dllimport) +# endif +# endif +# endif /* ZLIB_DLL */ + /* If building or using zlib with the WINAPI/WINAPIV calling convention, + * define ZLIB_WINAPI. + * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. + */ +# ifdef ZLIB_WINAPI +# ifdef FAR +# undef FAR +# endif +# include + /* No need for _export, use ZLIB.DEF instead. */ + /* For complete Windows compatibility, use WINAPI, not __stdcall. */ +# define ZEXPORT WINAPI +# ifdef WIN32 +# define ZEXPORTVA WINAPIV +# else +# define ZEXPORTVA FAR CDECL +# endif +# endif +#endif + +#if defined (__BEOS__) +# ifdef ZLIB_DLL +# ifdef ZLIB_INTERNAL +# define ZEXPORT __declspec(dllexport) +# define ZEXPORTVA __declspec(dllexport) +# else +# define ZEXPORT __declspec(dllimport) +# define ZEXPORTVA __declspec(dllimport) +# endif +# endif +#endif + +#ifndef ZEXTERN +# define ZEXTERN extern +#endif +#ifndef ZEXPORT +# define ZEXPORT +#endif +#ifndef ZEXPORTVA +# define ZEXPORTVA +#endif + +#ifndef FAR +# define FAR +#endif + +#if !defined(__MACTYPES__) +typedef unsigned char Byte; /* 8 bits */ +#endif +typedef unsigned int uInt; /* 16 bits or more */ +typedef unsigned long uLong; /* 32 bits or more */ + +#ifdef SMALL_MEDIUM + /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ +# define Bytef Byte FAR +#else + typedef Byte FAR Bytef; +#endif +typedef char FAR charf; +typedef int FAR intf; +typedef uInt FAR uIntf; +typedef uLong FAR uLongf; + +#ifdef STDC + typedef void const *voidpc; + typedef void FAR *voidpf; + typedef void *voidp; +#else + typedef Byte const *voidpc; + typedef Byte FAR *voidpf; + typedef Byte *voidp; +#endif + +#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) +# include +# if (UINT_MAX == 0xffffffffUL) +# define Z_U4 unsigned +# elif (ULONG_MAX == 0xffffffffUL) +# define Z_U4 unsigned long +# elif (USHRT_MAX == 0xffffffffUL) +# define Z_U4 unsigned short +# endif +#endif + +#ifdef Z_U4 + typedef Z_U4 z_crc_t; +#else + typedef unsigned long z_crc_t; +#endif + +#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_UNISTD_H +#endif + +#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_STDARG_H +#endif + +#ifdef STDC +# ifndef Z_SOLO +# include /* for off_t */ +# endif +#endif + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# include /* for va_list */ +# endif +#endif + +#ifdef _WIN32 +# ifndef Z_SOLO +# include /* for wchar_t */ +# endif +#endif + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H) +# define Z_HAVE_UNISTD_H +#endif +#ifndef Z_SOLO +# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifdef VMS +# include /* for off_t */ +# endif +# ifndef z_off_t +# define z_off_t off_t +# endif +# endif +#endif + +/* Other places expect _LFS64_LARGEFILE to be defined with a valid value */ +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && !defined(Z_SOLO) +# define SEEK_SET 0 /* Seek from beginning of file. */ +# define SEEK_CUR 1 /* Seek from current position. */ +# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ +#endif + +#ifndef z_off_t +# define z_off_t long +#endif + +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif +#endif + +/* MVS linker does not support external names larger than 8 bytes */ +#if defined(__MVS__) + #pragma map(deflateInit_,"DEIN") + #pragma map(deflateInit2_,"DEIN2") + #pragma map(deflateEnd,"DEEND") + #pragma map(deflateBound,"DEBND") + #pragma map(inflateInit_,"ININ") + #pragma map(inflateInit2_,"ININ2") + #pragma map(inflateEnd,"INEND") + #pragma map(inflateSync,"INSY") + #pragma map(inflateSetDictionary,"INSEDI") + #pragma map(compressBound,"CMBND") + #pragma map(inflate_table,"INTABL") + #pragma map(inflate_fast,"INFA") + #pragma map(inflate_copyright,"INCOPY") +#endif + +#endif /* ZCONF_H */ diff --git a/optee_os/core/lib/zlib/zlib.h b/optee_os/core/lib/zlib/zlib.h new file mode 100644 index 0000000..f0c53df --- /dev/null +++ b/optee_os/core/lib/zlib/zlib.h @@ -0,0 +1,1936 @@ +/* SPDX-License-Identifier: Zlib */ +/* zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.12, March 11th, 2022 + + Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + + The data format used by the zlib library is described by RFCs (Request for + Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 + (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format). +*/ + +#ifndef ZLIB_H +#define ZLIB_H + +#include "zconf.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ZLIB_VERSION "1.2.12" +#define ZLIB_VERNUM 0x12c0 +#define ZLIB_VER_MAJOR 1 +#define ZLIB_VER_MINOR 2 +#define ZLIB_VER_REVISION 12 +#define ZLIB_VER_SUBREVISION 0 + +/* + The 'zlib' compression library provides in-memory compression and + decompression functions, including integrity checks of the uncompressed data. + This version of the library supports only one compression method (deflation) + but other algorithms will be added later and will have the same stream + interface. + + Compression can be done in a single step if the buffers are large enough, + or can be done by repeated calls of the compression function. In the latter + case, the application must provide more input and/or consume the output + (providing more output space) before each call. + + The compressed data format used by default by the in-memory functions is + the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped + around a deflate stream, which is itself documented in RFC 1951. + + The library also supports reading and writing files in gzip (.gz) format + with an interface similar to that of stdio using the functions that start + with "gz". The gzip format is different from the zlib format. gzip is a + gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. + + This library can optionally read and write gzip and raw deflate streams in + memory as well. + + The zlib format was designed to be compact and fast for use in memory + and on communications channels. The gzip format was designed for single- + file compression on file systems, has a larger header than zlib to maintain + directory information, and uses a different, slower check method than zlib. + + The library does not install any signal handler. The decoder checks + the consistency of the compressed data, so the library should never crash + even in the case of corrupted input. +*/ + +typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); +typedef void (*free_func) OF((voidpf opaque, voidpf address)); + +struct internal_state; + +typedef struct z_stream_s { + z_const Bytef *next_in; /* next input byte */ + uInt avail_in; /* number of bytes available at next_in */ + uLong total_in; /* total number of input bytes read so far */ + + Bytef *next_out; /* next output byte will go here */ + uInt avail_out; /* remaining free space at next_out */ + uLong total_out; /* total number of bytes output so far */ + + z_const char *msg; /* last error message, NULL if no error */ + struct internal_state FAR *state; /* not visible by applications */ + + alloc_func zalloc; /* used to allocate the internal state */ + free_func zfree; /* used to free the internal state */ + voidpf opaque; /* private data object passed to zalloc and zfree */ + + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ + uLong reserved; /* reserved for future use */ +} z_stream; + +typedef z_stream FAR *z_streamp; + +/* + gzip header information passed to and from zlib routines. See RFC 1952 + for more details on the meanings of these fields. +*/ +typedef struct gz_header_s { + int text; /* true if compressed data believed to be text */ + uLong time; /* modification time */ + int xflags; /* extra flags (not used when writing a gzip file) */ + int os; /* operating system */ + Bytef *extra; /* pointer to extra field or Z_NULL if none */ + uInt extra_len; /* extra field length (valid if extra != Z_NULL) */ + uInt extra_max; /* space at extra (only when reading header) */ + Bytef *name; /* pointer to zero-terminated file name or Z_NULL */ + uInt name_max; /* space at name (only when reading header) */ + Bytef *comment; /* pointer to zero-terminated comment or Z_NULL */ + uInt comm_max; /* space at comment (only when reading header) */ + int hcrc; /* true if there was or will be a header crc */ + int done; /* true when done reading gzip header (not used + when writing a gzip file) */ +} gz_header; + +typedef gz_header FAR *gz_headerp; + +/* + The application must update next_in and avail_in when avail_in has dropped + to zero. It must update next_out and avail_out when avail_out has dropped + to zero. The application must initialize zalloc, zfree and opaque before + calling the init function. All other fields are set by the compression + library and must not be updated by the application. + + The opaque value provided by the application will be passed as the first + parameter for calls of zalloc and zfree. This can be useful for custom + memory management. The compression library attaches no meaning to the + opaque value. + + zalloc must return Z_NULL if there is not enough memory for the object. + If zlib is used in a multi-threaded application, zalloc and zfree must be + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). + + On 16-bit systems, the functions zalloc and zfree must be able to allocate + exactly 65536 bytes, but will not be required to allocate more than this if + the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, pointers + returned by zalloc for objects of exactly 65536 bytes *must* have their + offset normalized to zero. The default allocation function provided by this + library ensures this (see zutil.c). To reduce memory requirements and avoid + any allocation of 64K objects, at the expense of compression ratio, compile + the library with -DMAX_WBITS=14 (see zconf.h). + + The fields total_in and total_out can be used for statistics or progress + reports. After compression, total_in holds the total size of the + uncompressed data and may be saved for use by the decompressor (particularly + if the decompressor wants to decompress everything in a single step). +*/ + + /* constants */ + +#define Z_NO_FLUSH 0 +#define Z_PARTIAL_FLUSH 1 +#define Z_SYNC_FLUSH 2 +#define Z_FULL_FLUSH 3 +#define Z_FINISH 4 +#define Z_BLOCK 5 +#define Z_TREES 6 +/* Allowed flush values; see deflate() and inflate() below for details */ + +#define Z_OK 0 +#define Z_STREAM_END 1 +#define Z_NEED_DICT 2 +#define Z_ERRNO (-1) +#define Z_STREAM_ERROR (-2) +#define Z_DATA_ERROR (-3) +#define Z_MEM_ERROR (-4) +#define Z_BUF_ERROR (-5) +#define Z_VERSION_ERROR (-6) +/* Return codes for the compression/decompression functions. Negative values + * are errors, positive values are used for special but normal events. + */ + +#define Z_NO_COMPRESSION 0 +#define Z_BEST_SPEED 1 +#define Z_BEST_COMPRESSION 9 +#define Z_DEFAULT_COMPRESSION (-1) +/* compression levels */ + +#define Z_FILTERED 1 +#define Z_HUFFMAN_ONLY 2 +#define Z_RLE 3 +#define Z_FIXED 4 +#define Z_DEFAULT_STRATEGY 0 +/* compression strategy; see deflateInit2() below for details */ + +#define Z_BINARY 0 +#define Z_TEXT 1 +#define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ +#define Z_UNKNOWN 2 +/* Possible values of the data_type field for deflate() */ + +#define Z_DEFLATED 8 +/* The deflate compression method (the only one supported in this version) */ + +#define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ + +#define zlib_version zlibVersion() +/* for compatibility with versions < 1.0.2 */ + + + /* basic functions */ + +ZEXTERN const char * ZEXPORT zlibVersion OF((void)); +/* The application can compare zlibVersion and ZLIB_VERSION for consistency. + If the first character differs, the library code actually used is not + compatible with the zlib.h header file used by the application. This check + is automatically made by deflateInit and inflateInit. + */ + +/* +ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); + + Initializes the internal stream state for compression. The fields + zalloc, zfree and opaque must be initialized before by the caller. If + zalloc and zfree are set to Z_NULL, deflateInit updates them to use default + allocation functions. + + The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: + 1 gives best speed, 9 gives best compression, 0 gives no compression at all + (the input data is simply copied a block at a time). Z_DEFAULT_COMPRESSION + requests a default compromise between speed and compression (currently + equivalent to level 6). + + deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if level is not a valid compression level, or + Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible + with the version assumed by the caller (ZLIB_VERSION). msg is set to null + if there is no error message. deflateInit does not perform any compression: + this will be done by deflate(). +*/ + + +ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); +/* + deflate compresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. deflate performs one or both of the + following actions: + + - Compress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), next_in and avail_in are updated and + processing will resume at this point for the next call of deflate(). + + - Generate more output starting at next_out and update next_out and avail_out + accordingly. This action is forced if the parameter flush is non zero. + Forcing flush frequently degrades the compression ratio, so this parameter + should be set only when necessary. Some output may be provided even if + flush is zero. + + Before the call of deflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating avail_in or avail_out accordingly; avail_out should + never be zero before the call. The application can consume the compressed + output when it wants, for example when the output buffer is full (avail_out + == 0), or after each call of deflate(). If deflate returns Z_OK and with + zero avail_out, it must be called again after making room in the output + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more ouput + in that case. + + Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to + decide how much data to accumulate before producing output, in order to + maximize compression. + + If the parameter flush is set to Z_SYNC_FLUSH, all pending output is + flushed to the output buffer and the output is aligned on a byte boundary, so + that the decompressor can get all input data available so far. (In + particular avail_in is zero after the call if enough output space has been + provided before the call.) Flushing may degrade compression for some + compression algorithms and so it should be used only when necessary. This + completes the current deflate block and follows it with an empty stored block + that is three bits plus filler bits to the next byte, followed by four bytes + (00 00 ff ff). + + If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the + output buffer, but the output is not aligned to a byte boundary. All of the + input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. + This completes the current deflate block and follows it with an empty fixed + codes block that is 10 bits long. This assures that enough bytes are output + in order for the decompressor to finish the block before the empty fixed + codes block. + + If flush is set to Z_BLOCK, a deflate block is completed and emitted, as + for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to + seven bits of the current block are held to be written as the next byte after + the next deflate block is completed. In this case, the decompressor may not + be provided enough bits at this point in order to complete decompression of + the data provided so far to the compressor. It may need to wait for the next + block to be emitted. This is for advanced applications that need to control + the emission of deflate blocks. + + If flush is set to Z_FULL_FLUSH, all output is flushed as with + Z_SYNC_FLUSH, and the compression state is reset so that decompression can + restart from this point if previous compressed data has been damaged or if + random access is desired. Using Z_FULL_FLUSH too often can seriously degrade + compression. + + If deflate returns with avail_out == 0, this function must be called again + with the same value of the flush parameter and more output space (updated + avail_out), until the flush is complete (deflate returns with non-zero + avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that + avail_out is greater than six to avoid repeated flush markers due to + avail_out == 0 on return. + + If the parameter flush is set to Z_FINISH, pending input is processed, + pending output is flushed and deflate returns with Z_STREAM_END if there was + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. + + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. + + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) + + deflate() may update strm->data_type if it can make a good guess about + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. + + deflate() returns Z_OK if some progress has been made (more input + processed or more output produced), Z_STREAM_END if all input has been + consumed and all output has been produced (only when flush is set to + Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. +*/ + + +ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any pending + output. + + deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the + stream state was inconsistent, Z_DATA_ERROR if the stream was freed + prematurely (some input or output was discarded). In the error case, msg + may be set but then points to a static string (which must not be + deallocated). +*/ + + +/* +ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); + + Initializes the internal stream state for decompression. The fields + next_in, avail_in, zalloc, zfree and opaque must be initialized before by + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. + + inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. +*/ + + +ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); +/* + inflate decompresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. inflate performs one or both of the + following actions: + + - Decompress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). + + - Generate more output starting at next_out and update next_out and avail_out + accordingly. inflate() provides as much output as possible, until there is + no more input data or no more space in the output buffer (see below about + the flush parameter). + + Before the call of inflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The + application can consume the uncompressed output when it wants, for example + when the output buffer is full (avail_out == 0), or after each call of + inflate(). If inflate returns Z_OK and with zero avail_out, it must be + called again after making room in the output buffer because there might be + more output pending. + + The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH, + Z_BLOCK, or Z_TREES. Z_SYNC_FLUSH requests that inflate() flush as much + output as possible to the output buffer. Z_BLOCK requests that inflate() + stop if and when it gets to the next deflate block boundary. When decoding + the zlib or gzip format, this will cause inflate() to return immediately + after the header and before the first block. When doing a raw inflate, + inflate() will go ahead and process the first block, and will return when it + gets to the end of that block, or when it runs out of data. + + The Z_BLOCK option assists in appending to or combining deflate streams. + To assist in this, on return inflate() always sets strm->data_type to the + number of unused bits in the last byte taken from strm->next_in, plus 64 if + inflate() is currently decoding the last block in the deflate stream, plus + 128 if inflate() returned immediately after decoding an end-of-block code or + decoding the complete header up to just before the first byte of the deflate + stream. The end-of-block will not be indicated until all of the uncompressed + data from that block has been written to strm->next_out. The number of + unused bits may in general be greater than seven, except when bit 7 of + data_type is set, in which case the number of unused bits will be less than + eight. data_type is set as noted here every time inflate() returns for all + flush options, and so can be used to determine the amount of currently + consumed input in bits. + + The Z_TREES option behaves as Z_BLOCK does, but it also returns when the + end of each deflate block header is reached, before any actual data in that + block is decoded. This allows the caller to determine the length of the + deflate block header for later use in random access within a deflate block. + 256 is added to the value of strm->data_type when inflate() returns + immediately after reaching the end of the deflate block header. + + inflate() should normally be called until it returns Z_STREAM_END or an + error. However if all decompression is to be performed in a single step (a + single call of inflate), the parameter flush should be set to Z_FINISH. In + this case all pending input is processed and all pending output is flushed; + avail_out must be large enough to hold all of the uncompressed data for the + operation to complete. (The size of the uncompressed data may have been + saved by the compressor for this purpose.) The use of Z_FINISH is not + required to perform an inflation in one step. However it may be used to + inform inflate that a faster approach can be used for the single inflate() + call. Z_FINISH also informs inflate to not maintain a sliding window if the + stream completes, which reduces inflate's memory footprint. If the stream + does not complete, either because not all of the stream is provided or not + enough output space is provided, then a sliding window will be allocated and + inflate() can be called again to continue the operation as if Z_NO_FLUSH had + been used. + + In this implementation, inflate() always flushes as much output as + possible to the output buffer, and always uses the faster approach on the + first call. So the effects of the flush parameter in this implementation are + on the return value of inflate() as noted below, when inflate() returns early + when Z_BLOCK or Z_TREES is used, and when inflate() avoids the allocation of + memory for a sliding window when Z_FINISH is used. + + If a preset dictionary is needed after this call (see inflateSetDictionary + below), inflate sets strm->adler to the Adler-32 checksum of the dictionary + chosen by the compressor and returns Z_NEED_DICT; otherwise it sets + strm->adler to the Adler-32 checksum of all output produced so far (that is, + total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described + below. At the end of the stream, inflate() checks that its computed Adler-32 + checksum is equal to that saved by the compressor and returns Z_STREAM_END + only if the checksum is correct. + + inflate() can decompress and check either zlib-wrapped or gzip-wrapped + deflate data. The header type is detected automatically, if requested when + initializing with inflateInit2(). Any information contained in the gzip + header is not retained unless inflateGetHeader() is used. When processing + gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. + + inflate() returns Z_OK if some progress has been made (more input processed + or more output produced), Z_STREAM_END if the end of the compressed data has + been reached and all uncompressed output has been produced, Z_NEED_DICT if a + preset dictionary is needed at this point, Z_DATA_ERROR if the input data was + corrupted (input stream not conforming to the zlib format or incorrect check + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + inflate() can be called again with more input and more output space to + continue decompressing. If Z_DATA_ERROR is returned, the application may + then call inflateSync() to look for a good compression block if a partial + recovery of the data is to be attempted. +*/ + + +ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any pending + output. + + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. +*/ + + + /* Advanced functions */ + +/* + The following functions are needed only in some special applications. +*/ + +/* +ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, + int level, + int method, + int windowBits, + int memLevel, + int strategy)); + + This is another version of deflateInit with more compression options. The + fields zalloc, zfree and opaque must be initialized before by the caller. + + The method parameter is the compression method. It must be Z_DEFLATED in + this version of the library. + + The windowBits parameter is the base two logarithm of the window size + (the size of the history buffer). It should be in the range 8..15 for this + version of the library. Larger values of this parameter result in better + compression at the expense of memory usage. The default value is 15 if + deflateInit is used instead. + + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). + + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits + determines the window size. deflate() will then generate raw deflate data + with no zlib header or trailer, and will not compute a check value. + + windowBits can also be greater than 15 for optional gzip encoding. Add + 16 to windowBits to write a simple gzip header and trailer around the + compressed data instead of a zlib wrapper. The gzip header will have no + file name, no extra data, no comment, no modification time (set to zero), no + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. + + The memLevel parameter specifies how much memory should be allocated + for the internal compression state. memLevel=1 uses minimum memory but is + slow and reduces compression ratio; memLevel=9 uses maximum memory for + optimal speed. The default value is 8. See zconf.h for total memory usage + as a function of windowBits and memLevel. + + The strategy parameter is used to tune the compression algorithm. Use the + value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a + filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no + string match), or Z_RLE to limit match distances to one (run-length + encoding). Filtered data consists mostly of small values with a somewhat + random distribution. In this case, the compression algorithm is tuned to + compress them better. The effect of Z_FILTERED is to force more Huffman + coding and less string matching; it is somewhat intermediate between + Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as + fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data. The + strategy parameter only affects the compression ratio but not the + correctness of the compressed output even if it is not set appropriately. + Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler + decoder for special applications. + + deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid + method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is + incompatible with the version assumed by the caller (ZLIB_VERSION). msg is + set to null if there is no error message. deflateInit2 does not perform any + compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); +/* + Initializes the compression dictionary from the given byte sequence + without producing any compressed output. When using the zlib format, this + function must be called immediately after deflateInit, deflateInit2 or + deflateReset, and before any call of deflate. When doing raw deflate, this + function must be called either before any call of deflate, or immediately + after the completion of a deflate block, i.e. after all input has been + consumed and all output has been delivered when using any of the flush + options Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, or Z_FULL_FLUSH. The + compressor and decompressor must use exactly the same dictionary (see + inflateSetDictionary). + + The dictionary should consist of strings (byte sequences) that are likely + to be encountered later in the data to be compressed, with the most commonly + used strings preferably put towards the end of the dictionary. Using a + dictionary is most useful when the data to be compressed is short and can be + predicted with good accuracy; the data can then be compressed better than + with the default empty dictionary. + + Depending on the size of the compression data structures selected by + deflateInit or deflateInit2, a part of the dictionary may in effect be + discarded, for example if the dictionary is larger than the window size + provided in deflateInit or deflateInit2. Thus the strings most likely to be + useful should be put at the end of the dictionary, not at the front. In + addition, the current implementation of deflate will use at most the window + size minus 262 bytes of the provided dictionary. + + Upon return of this function, strm->adler is set to the Adler-32 value + of the dictionary; the decompressor may later use this value to determine + which dictionary has been used by the compressor. (The Adler-32 value + applies to the whole dictionary even if only a subset of the dictionary is + actually used by the compressor.) If a raw deflate was requested, then the + Adler-32 value is not computed and strm->adler is not set. + + deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is + inconsistent (for example if deflate has already been called for this stream + or if not at a block boundary for raw deflate). deflateSetDictionary does + not perform any compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + +ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, + z_streamp source)); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when several compression strategies will be + tried, for example when there are several ways of pre-processing the input + data with a filter. The streams that will be discarded should then be freed + by calling deflateEnd. Note that deflateCopy duplicates the internal + compression state which can be quite large, so this strategy is slow and can + consume lots of memory. + + deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being Z_NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); +/* + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. + + deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). +*/ + +ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, + int level, + int strategy)); +/* + Dynamically update the compression level and compression strategy. The + interpretation of level and strategy is as in deflateInit2(). This can be + used to switch between compression and straight copy of the input data, or + to switch to a different kind of input data requiring a different strategy. + If the compression approach (which is a function of the level) or the + strategy is changed, and if there have been any deflate() calls since the + state was initialized or reset, then the input available so far is + compressed with the old level and strategy using deflate(strm, Z_BLOCK). + There are three approaches for the compression levels 0, 1..3, and 4..9 + respectively. The new level and strategy will take effect at the next call + of deflate(). + + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. + + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. +*/ + +ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, + int good_length, + int max_lazy, + int nice_length, + int max_chain)); +/* + Fine tune deflate's internal compression parameters. This should only be + used by someone who understands the algorithm used by zlib's deflate for + searching for the best matching string, and even then only by the most + fanatic optimizer trying to squeeze out the last compressed bit for their + specific input data. Read the deflate.c source code for the meaning of the + max_lazy, good_length, nice_length, and max_chain parameters. + + deflateTune() can be called after deflateInit() or deflateInit2(), and + returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. + */ + +ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, + uLong sourceLen)); +/* + deflateBound() returns an upper bound on the compressed size after + deflation of sourceLen bytes. It must be called after deflateInit() or + deflateInit2(), and after deflateSetHeader(), if used. This would be used + to allocate an output buffer for deflation in a single pass, and so would be + called before deflate(). If that first deflate() call is provided the + sourceLen input bytes, an output buffer allocated to the size returned by + deflateBound(), and the flush value Z_FINISH, then deflate() is guaranteed + to return Z_STREAM_END. Note that it is possible for the compressed size to + be larger than the value returned by deflateBound() if flush options other + than Z_FINISH or Z_NO_FLUSH are used. +*/ + +ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, + unsigned *pending, + int *bits)); +/* + deflatePending() returns the number of bytes and bits of output that have + been generated, but not yet provided in the available output. The bytes not + provided would be due to the available output space having being consumed. + The number of bits of output not provided are between 0 and 7, where they + await more bits to join them in order to fill out a full byte. If pending + or bits are Z_NULL, then those values are not set. + + deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. + */ + +ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, + int bits, + int value)); +/* + deflatePrime() inserts bits in the deflate output stream. The intent + is that this function is used to start off the deflate output with the bits + leftover from a previous deflate stream when appending to it. As such, this + function can only be used for raw deflate, and must be used before the first + deflate() call after a deflateInit2() or deflateReset(). bits must be less + than or equal to 16, and that many of the least significant bits of value + will be inserted in the output. + + deflatePrime returns Z_OK if success, Z_BUF_ERROR if there was not enough + room in the internal buffer to insert the bits, or Z_STREAM_ERROR if the + source stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, + gz_headerp head)); +/* + deflateSetHeader() provides gzip header information for when a gzip + stream is requested by deflateInit2(). deflateSetHeader() may be called + after deflateInit2() or deflateReset() and before the first call of + deflate(). The text, time, os, extra field, name, and comment information + in the provided gz_header structure are written to the gzip header (xflag is + ignored -- the extra flags are set according to the compression level). The + caller must assure that, if not Z_NULL, name and comment are terminated with + a zero byte, and that if extra is not Z_NULL, that extra_len bytes are + available there. If hcrc is true, a gzip header crc is included. Note that + the current versions of the command-line version of gzip (up through version + 1.3.x) do not support header crc's, and will report that it is a "multi-part + gzip file" and give up. + + If deflateSetHeader is not used, the default gzip header has text false, + the time set to zero, and os set to 255, with no extra, name, or comment + fields. The gzip header is returned to the default state by deflateReset(). + + deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, + int windowBits)); + + This is another version of inflateInit with an extra parameter. The + fields next_in, avail_in, zalloc, zfree and opaque must be initialized + before by the caller. + + The windowBits parameter is the base two logarithm of the maximum window + size (the size of the history buffer). It should be in the range 8..15 for + this version of the library. The default value is 15 if inflateInit is used + instead. windowBits must be greater than or equal to the windowBits value + provided to deflateInit2() while compressing, or it must be equal to 15 if + deflateInit2() was not used. If a compressed stream with a larger window + size is given as input, inflate() will return with the error code + Z_DATA_ERROR instead of trying to allocate a larger window. + + windowBits can also be zero to request that inflate use the window size in + the zlib header of the compressed stream. + + windowBits can also be -8..-15 for raw inflate. In this case, -windowBits + determines the window size. inflate() will then process raw deflate data, + not looking for a zlib or gzip header, not generating a check value, and not + looking for any check values for comparison at the end of the stream. This + is for use with other formats that use the deflate compressed data format + such as zip. Those formats provide their own check values. If a custom + format is developed using the raw deflate format for compressed data, it is + recommended that a check value such as an Adler-32 or a CRC-32 be applied to + the uncompressed data as is done in the zlib, gzip, and zip formats. For + most applications, the zlib format should be used as is. Note that comments + above on the use in deflateInit2() applies to the magnitude of windowBits. + + windowBits can also be greater than 15 for optional gzip decoding. Add + 32 to windowBits to enable zlib and gzip decoding with automatic header + detection, or add 16 to decode only the gzip format (the zlib format will + return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will *not* automatically decode concatenated gzip members. + inflate() will return Z_STREAM_END at the end of the gzip member. The state + would need to be reset to continue decoding a subsequent gzip member. This + *must* be done if there is more data after a gzip member, in order for the + decompression to be compliant with the gzip standard (RFC 1952). + + inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit2 does not perform any decompression + apart from possibly reading the zlib header if present: actual decompression + will be done by inflate(). (So next_in and avail_in may be modified, but + next_out and avail_out are unused and unchanged.) The current implementation + of inflateInit2() does not process any header information -- that is + deferred until inflate() is called. +*/ + +ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); +/* + Initializes the decompression dictionary from the given uncompressed byte + sequence. This function must be called immediately after a call of inflate, + if that call returned Z_NEED_DICT. The dictionary chosen by the compressor + can be determined from the Adler-32 value returned by that call of inflate. + The compressor and decompressor must use exactly the same dictionary (see + deflateSetDictionary). For raw inflate, this function can be called at any + time to set the dictionary. If the provided dictionary is smaller than the + window and there is already data in the window, then the provided dictionary + will amend what's there. The application must insure that the dictionary + that was used for compression is provided. + + inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is + inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the + expected one (incorrect Adler-32 value). inflateSetDictionary does not + perform any decompression: this will be done by subsequent calls of + inflate(). +*/ + +ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by inflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If inflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + +ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); +/* + Skips invalid compressed data until a possible full flush point (see above + for the description of deflate with Z_FULL_FLUSH) can be found, or until all + available input is skipped. No output is provided. + + inflateSync searches for a 00 00 FF FF pattern in the compressed data. + All full flush points have this pattern, but not all occurrences of this + pattern are full flush points. + + inflateSync returns Z_OK if a possible full flush point has been found, + Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point + has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. + In the success case, the application may save the current current value of + total_in which indicates where valid compressed data was found. In the + error case, the application may repeatedly call inflateSync, providing more + input each time, until success or end of the input data. +*/ + +ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, + z_streamp source)); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when randomly accessing a large stream. The + first pass through the stream can periodically record the inflate state, + allowing restarting inflate at those points when randomly accessing the + stream. + + inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being Z_NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); +/* + This function is equivalent to inflateEnd followed by inflateInit, + but does not free and reallocate the internal decompression state. The + stream will keep attributes that may have been set by inflateInit2. + + inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). +*/ + +ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, + int windowBits)); +/* + This function is the same as inflateReset, but it also permits changing + the wrap and window size requests. The windowBits parameter is interpreted + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. + + inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL), or if + the windowBits parameter is invalid. +*/ + +ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, + int bits, + int value)); +/* + This function inserts bits in the inflate input stream. The intent is + that this function is used to start inflating at a bit position in the + middle of a byte. The provided bits will be used before any bytes are used + from next_in. This function should only be used with raw inflate, and + should be used before the first inflate() call after inflateInit2() or + inflateReset(). bits must be less than or equal to 16, and that many of the + least significant bits of value will be inserted in the input. + + If bits is negative, then the input stream bit buffer is emptied. Then + inflatePrime() can be called again to put bits in the buffer. This is used + to clear out bits leftover after feeding inflate a block description prior + to feeding inflate codes. + + inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); +/* + This function returns two values, one in the lower 16 bits of the return + value, and the other in the remaining upper bits, obtained by shifting the + return value down 16 bits. If the upper value is -1 and the lower value is + zero, then inflate() is currently decoding information outside of a block. + If the upper value is -1 and the lower value is non-zero, then inflate is in + the middle of a stored block, with the lower value equaling the number of + bytes from the input remaining to copy. If the upper value is not -1, then + it is the number of bits back from the current bit position in the input of + the code (literal or length/distance pair) currently being processed. In + that case the lower value is the number of bytes already emitted for that + code. + + A code is being processed if inflate is waiting for more input to complete + decoding of the code, or if it has completed decoding but is waiting for + more output space to write the literal or match data. + + inflateMark() is used to mark locations in the input data for random + access, which may be at bit positions, and to note those cases where the + output of a code may span boundaries of random access blocks. The current + location in the input stream can be determined from avail_in and data_type + as noted in the description for the Z_BLOCK flush parameter for inflate. + + inflateMark returns the value noted above, or -65536 if the provided + source stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, + gz_headerp head)); +/* + inflateGetHeader() requests that gzip header information be stored in the + provided gz_header structure. inflateGetHeader() may be called after + inflateInit2() or inflateReset(), and before the first call of inflate(). + As inflate() processes the gzip stream, head->done is zero until the header + is completed, at which time head->done is set to one. If a zlib stream is + being decoded, then head->done is set to -1 to indicate that there will be + no gzip header information forthcoming. Note that Z_BLOCK or Z_TREES can be + used to force inflate() to return immediately after header processing is + complete and before any actual data is decompressed. + + The text, time, xflags, and os fields are filled in with the gzip header + contents. hcrc is set to true if there is a header CRC. (The header CRC + was valid if done is set to one.) If extra is not Z_NULL, then extra_max + contains the maximum number of bytes to write to extra. Once done is true, + extra_len contains the actual extra field length, and extra contains the + extra field, or that field truncated if extra_max is less than extra_len. + If name is not Z_NULL, then up to name_max characters are written there, + terminated with a zero unless the length is greater than name_max. If + comment is not Z_NULL, then up to comm_max characters are written there, + terminated with a zero unless the length is greater than comm_max. When any + of extra, name, or comment are not Z_NULL and the respective field is not + present in the header, then that field is set to Z_NULL to signal its + absence. This allows the use of deflateSetHeader() with the returned + structure to duplicate the header. However if those fields are set to + allocated memory, then the application will need to save those pointers + elsewhere so that they can be eventually freed. + + If inflateGetHeader is not used, then the header information is simply + discarded. The header is always checked for validity, including the header + CRC if present. inflateReset() will reset the process to discard the header + information. The application would need to call inflateGetHeader() again to + retrieve the header from the next gzip stream. + + inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, + unsigned char FAR *window)); + + Initialize the internal stream state for decompression using inflateBack() + calls. The fields zalloc, zfree and opaque in strm must be initialized + before the call. If zalloc and zfree are Z_NULL, then the default library- + derived memory allocation routines are used. windowBits is the base two + logarithm of the window size, in the range 8..15. window is a caller + supplied buffer of that size. Except for special applications where it is + assured that deflate was used with small window sizes, windowBits must be 15 + and a 32K byte window must be supplied to be able to decompress general + deflate streams. + + See inflateBack() for the usage of these routines. + + inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of + the parameters are invalid, Z_MEM_ERROR if the internal state could not be + allocated, or Z_VERSION_ERROR if the version of the library does not match + the version of the header file. +*/ + +typedef unsigned (*in_func) OF((void FAR *, + z_const unsigned char FAR * FAR *)); +typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); + +ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, + in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc)); +/* + inflateBack() does a raw inflate with a single call using a call-back + interface for input and output. This is potentially more efficient than + inflate() for file i/o applications, in that it avoids copying between the + output and the sliding window by simply making the window itself the output + buffer. inflate() can be faster on modern CPUs when used with large + buffers. inflateBack() trusts the application to not change the output + buffer passed by the output function, at least until inflateBack() returns. + + inflateBackInit() must be called first to allocate the internal state + and to initialize the state with the user-provided window buffer. + inflateBack() may then be used multiple times to inflate a complete, raw + deflate stream with each call. inflateBackEnd() is then called to free the + allocated state. + + A raw deflate stream is one with no zlib or gzip header or trailer. + This routine would normally be used in a utility that reads zip or gzip + files and writes out uncompressed files. The utility would decode the + header and process the trailer on its own, hence this routine expects only + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. + + inflateBack() uses two subroutines supplied by the caller that are then + called by inflateBack() for input and output. inflateBack() calls those + routines until it reads a complete deflate stream and writes out all of the + uncompressed data, or until it encounters an error. The function's + parameters and return types are defined above in the in_func and out_func + typedefs. inflateBack() will call in(in_desc, &buf) which should return the + number of bytes of provided input, and a pointer to that input in buf. If + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to + inflateBackInit(), which is also the buffer that out() uses to write from. + The length written by out() will be at most the window size. Any non-zero + amount of input may be provided by in(). + + For convenience, inflateBack() can be provided input on the first call by + setting strm->next_in and strm->avail_in. If that input is exhausted, then + in() will be called. Therefore strm->next_in must be initialized before + calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called + immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in + must also be initialized, and then if strm->avail_in is not zero, input will + initially be taken from strm->next_in[0 .. strm->avail_in - 1]. + + The in_desc and out_desc parameters of inflateBack() is passed as the + first parameter of in() and out() respectively when they are called. These + descriptors can be optionally used to pass any information that the caller- + supplied in() and out() functions need to do their job. + + On return, inflateBack() will set strm->next_in and strm->avail_in to + pass back any unused input that was provided by the last in() call. The + return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR + if in() or out() returned an error, Z_DATA_ERROR if there was a format error + in the deflate stream (in which case strm->msg is set to indicate the nature + of the error), or Z_STREAM_ERROR if the stream was not properly initialized. + In the case of Z_BUF_ERROR, an input or output error can be distinguished + using strm->next_in which will be Z_NULL only if in() returned an error. If + strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning + non-zero. (in() will always be called before out(), so strm->next_in is + assured to be defined if out() returns non-zero.) Note that inflateBack() + cannot return Z_OK. +*/ + +ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); +/* + All memory allocated by inflateBackInit() is freed. + + inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream + state was inconsistent. +*/ + +ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); +/* Return flags indicating compile-time options. + + Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other: + 1.0: size of uInt + 3.2: size of uLong + 5.4: size of voidpf (pointer) + 7.6: size of z_off_t + + Compiler, assembler, and debug options: + 8: ZLIB_DEBUG + 9: ASMV or ASMINF -- use ASM code + 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention + 11: 0 (reserved) + + One-time table building (smaller code, but not thread-safe if true): + 12: BUILDFIXED -- build static block decoding tables when needed + 13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed + 14,15: 0 (reserved) + + Library content (indicates missing functionality): + 16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking + deflate code when not needed) + 17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect + and decode gzip streams (to avoid linking crc code) + 18-19: 0 (reserved) + + Operation variations (changes in library functionality): + 20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate + 21: FASTEST -- deflate algorithm with only one, lowest compression level + 22,23: 0 (reserved) + + The sprintf variant used by gzprintf (zero is best): + 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format + 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure! + 26: 0 = returns value, 1 = void -- 1 means inferred string length returned + + Remainder: + 27-31: 0 (reserved) + */ + +#ifndef Z_SOLO + + /* utility functions */ + +/* + The following utility functions are implemented on top of the basic + stream-oriented functions. To simplify the interface, some default options + are assumed (compression level and memory usage, standard memory allocation + functions). The source code of these utility functions can be modified if + you need special options. +*/ + +ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); +/* + Compresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. + + compress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer. +*/ + +ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level)); +/* + Compresses the source buffer into the destination buffer. The level + parameter has the same meaning as in deflateInit. sourceLen is the byte + length of the source buffer. Upon entry, destLen is the total size of the + destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. + + compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, + Z_STREAM_ERROR if the level parameter is invalid. +*/ + +ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen)); +/* + compressBound() returns an upper bound on the compressed size after + compress() or compress2() on sourceLen bytes. It would be used before a + compress() or compress2() call to allocate the destination buffer. +*/ + +ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); +/* + Decompresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, destLen + is the actual size of the uncompressed data. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In + the case where there is not enough room, uncompress() will fill the output + buffer with the uncompressed data up to that point. +*/ + +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); +/* + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ + + /* gzip file access functions */ + +/* + This library supports reading and writing files in gzip (.gz) format with + an interface similar to that of stdio, using the functions that start with + "gz". The gzip format is different from the zlib format. gzip is a gzip + wrapper, documented in RFC 1952, wrapped around a deflate stream. +*/ + +typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ + +/* +ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); + + Open the gzip (.gz) file at path for reading and decompressing, or + compressing and writing. The mode parameter is as in fopen ("rb" or "wb") + but can also include a compression level ("wb9") or a strategy: 'f' for + filtered data as in "wb6f", 'h' for Huffman-only compression as in "wb1h", + 'R' for run-length encoding as in "wb1R", or 'F' for fixed code compression + as in "wb9F". (See the description of deflateInit2 for more information + about the strategy parameter.) 'T' will request transparent writing or + appending with no compression and not using the gzip format. + + "a" can be used instead of "w" to request that the gzip stream that will + be written be appended to the file. "+" will result in an error, since + reading and writing to the same gzip file is not supported. The addition of + "x" when writing will create the file exclusively, which fails if the file + already exists. On systems that support it, the addition of "e" when + reading or writing will set the flag to close the file on an execve() call. + + These functions, as well as gzip, will read and decode a sequence of gzip + streams in a file. The append function of gzopen() can be used to create + such a file. (Also see gzflush() for another way to do this.) When + appending, gzopen does not test whether the file begins with a gzip stream, + nor does it look for the end of the gzip streams to begin appending. gzopen + will simply append a gzip stream to the existing file. + + gzopen can be used to read a file which is not in gzip format; in this + case gzread will directly read from the file without decompression. When + reading, this will be detected automatically by looking for the magic two- + byte gzip header. + + gzopen returns NULL if the file could not be opened, if there was + insufficient memory to allocate the gzFile state, or if an invalid mode was + specified (an 'r', 'w', or 'a' was not provided, or '+' was provided). + errno can be checked to determine if the reason gzopen failed was that the + file could not be opened. +*/ + +ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); +/* + Associate a gzFile with the file descriptor fd. File descriptors are + obtained from calls like open, dup, creat, pipe or fileno (if the file has + been previously opened with fopen). The mode parameter is as in gzopen. + + The next call of gzclose on the returned gzFile will also close the file + descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor + fd. If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd, + mode);. The duplicated descriptor should be saved to avoid a leak, since + gzdopen does not close fd if it fails. If you are using fileno() to get the + file descriptor from a FILE *, then you will have to use dup() to avoid + double-close()ing the file descriptor. Both gzclose() and fclose() will + close the associated file descriptor, so they need to have different file + descriptors. + + gzdopen returns NULL if there was insufficient memory to allocate the + gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not + provided, or '+' was provided), or if fd is -1. The file descriptor is not + used until the next gz* read, write, seek, or close operation, so gzdopen + will not detect if fd is invalid (unless fd is -1). +*/ + +ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); +/* + Set the internal buffer size used by this library's functions for file to + size. The default buffer size is 8192 bytes. This function must be called + after gzopen() or gzdopen(), and before any other calls that read or write + the file. The buffer memory allocation is always deferred to the first read + or write. Three times that size in buffer space is allocated. A larger + buffer size of, for example, 64K or 128K bytes will noticeably increase the + speed of decompression (reading). + + The new buffer size also affects the maximum length for gzprintf(). + + gzbuffer() returns 0 on success, or -1 on failure, such as being called + too late. +*/ + +ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); +/* + Dynamically update the compression level and strategy for file. See the + description of deflateInit2 for the meaning of these parameters. Previously + provided data is flushed before applying the parameter changes. + + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. +*/ + +ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); +/* + Read and decompress up to len uncompressed bytes from file into buf. If + the input file is not in gzip format, gzread copies the given number of + bytes into the buffer directly from the file. + + After reaching the end of a gzip stream in the input, gzread will continue + to read, looking for another gzip stream. Any number of gzip streams may be + concatenated in the input file, and will all be decompressed by gzread(). + If something other than a gzip stream is encountered after a gzip stream, + that remaining trailing garbage is ignored (and no error is returned). + + gzread can be used to read a gzip file that is being concurrently written. + Upon reaching the end of the input, gzread will return with the available + data. If the error code returned by gzerror is Z_OK or Z_BUF_ERROR, then + gzclearerr can be used to clear the end of file indicator in order to permit + gzread to be tried again. Z_OK indicates that a gzip stream was completed + on the last gzread. Z_BUF_ERROR indicates that the input file ended in the + middle of a gzip stream. Note that gzread does not return -1 in the event + of an incomplete gzip stream. This error is deferred until gzclose(), which + will return Z_BUF_ERROR if the last gzread ended in the middle of a gzip + stream. Alternatively, gzerror can be used before gzclose to detect this + case. + + gzread returns the number of uncompressed bytes actually read, less than + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +/* + Read and decompress up to nitems items of size size from file into buf, + otherwise operating as gzread() does. This duplicates the interface of + stdio's fread(), with size_t request and return types. If the library + defines size_t, then z_size_t is identical to size_t. If not, then z_size_t + is an unsigned integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevetheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, reseting and retrying on end-of-file, when size is not 1. +*/ + +ZEXTERN int ZEXPORT gzwrite OF((gzFile file, voidpc buf, unsigned len)); +/* + Compress and write the len uncompressed bytes at buf to file. gzwrite + returns the number of uncompressed bytes written or 0 in case of error. +*/ + +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); +/* + Compress and write nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + +ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); +/* + Convert, format, compress, and write the arguments (...) to file under + control of the string format, as in fprintf. gzprintf returns the number of + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf(), + because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). +*/ + +ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); +/* + Compress and write the given null-terminated string s to file, excluding + the terminating null character. + + gzputs returns the number of characters written, or -1 in case of error. +*/ + +ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); +/* + Read and decompress bytes from file into buf, until len-1 characters are + read, or until a newline character is read and transferred to buf, or an + end-of-file condition is encountered. If any characters are read or if len + is one, the string is terminated with a null character. If no characters + are read due to an end-of-file or len is less than one, then the buffer is + left untouched. + + gzgets returns buf which is a null-terminated string, or it returns NULL + for end-of-file or in case of error. If there was an error, the contents at + buf are indeterminate. +*/ + +ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); +/* + Compress and write c, converted to an unsigned char, into file. gzputc + returns the value that was written, or -1 in case of error. +*/ + +ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); +/* + Read and decompress one byte from file. gzgetc returns this byte or -1 + in case of end of file or error. This is implemented as a macro for speed. + As such, it does not do all of the checking the other functions do. I.e. + it does not check to see if file is NULL, nor whether the structure file + points to has been clobbered or not. +*/ + +ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); +/* + Push c back onto the stream for file to be read as the first character on + the next read. At least one character of push-back is always allowed. + gzungetc() returns the character pushed, or -1 on failure. gzungetc() will + fail if c is -1, and may fail if a character has been pushed but not read + yet. If gzungetc is used immediately after gzopen or gzdopen, at least the + output buffer size of pushed characters is allowed. (See gzbuffer above.) + The pushed character will be discarded if the stream is repositioned with + gzseek() or gzrewind(). +*/ + +ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); +/* + Flush all pending output to file. The parameter flush is as in the + deflate() function. The return value is the zlib error number (see function + gzerror below). gzflush is only permitted when writing. + + If the flush parameter is Z_FINISH, the remaining data is written and the + gzip stream is completed in the output. If gzwrite() is called again, a new + gzip stream will be started in the output. gzread() is able to read such + concatenated gzip streams. + + gzflush should be called only when strictly necessary because it will + degrade compression if called too often. +*/ + +/* +ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, + z_off_t offset, int whence)); + + Set the starting position to offset relative to whence for the next gzread + or gzwrite on file. The offset represents a number of bytes in the + uncompressed data stream. The whence parameter is defined as in lseek(2); + the value SEEK_END is not supported. + + If the file is opened for reading, this function is emulated but can be + extremely slow. If the file is opened for writing, only forward seeks are + supported; gzseek then compresses a sequence of zeroes up to the new + starting position. + + gzseek returns the resulting offset location as measured in bytes from + the beginning of the uncompressed stream, or -1 in case of error, in + particular if the file is opened for writing and the new starting position + would be before the current position. +*/ + +ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); +/* + Rewind file. This function is supported only for reading. + + gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET). +*/ + +/* +ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); + + Return the starting position for the next gzread or gzwrite on file. + This position represents a number of bytes in the uncompressed data stream, + and is zero when starting, even if appending or reading a gzip stream from + the middle of a file using gzdopen(). + + gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR) +*/ + +/* +ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); + + Return the current compressed (actual) read or write offset of file. This + offset includes the count of bytes that precede the gzip stream, for example + when appending or when using gzdopen() for reading. When reading, the + offset does not include as yet unused buffered input. This information can + be used for a progress indicator. On error, gzoffset() returns -1. +*/ + +ZEXTERN int ZEXPORT gzeof OF((gzFile file)); +/* + Return true (1) if the end-of-file indicator for file has been set while + reading, false (0) otherwise. Note that the end-of-file indicator is set + only if the read tried to go past the end of the input, but came up short. + Therefore, just like feof(), gzeof() may return false even if there is no + more data to read, in the event that the last read request was for the exact + number of bytes remaining in the input file. This will happen if the input + file size is an exact multiple of the buffer size. + + If gzeof() returns true, then the read functions will return no more data, + unless the end-of-file indicator is reset by gzclearerr() and the input file + has grown since the previous end of file was detected. +*/ + +ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); +/* + Return true (1) if file is being copied directly while reading, or false + (0) if file is a gzip stream being decompressed. + + If the input file is empty, gzdirect() will return true, since the input + does not contain a gzip stream. + + If gzdirect() is used immediately after gzopen() or gzdopen() it will + cause buffers to be allocated to allow reading the file to determine if it + is a gzip file. Therefore if gzbuffer() is used, it should be called before + gzdirect(). + + When writing, gzdirect() returns true (1) if transparent writing was + requested ("wT" for the gzopen() mode), or false (0) otherwise. (Note: + gzdirect() is not needed when writing. Transparent writing must be + explicitly requested, so the application already knows the answer. When + linking statically, using gzdirect() will include all of the zlib code for + gzip file reading and decompression, which may not be desired.) +*/ + +ZEXTERN int ZEXPORT gzclose OF((gzFile file)); +/* + Flush all pending output for file, if necessary, close file and + deallocate the (de)compression state. Note that once file is closed, you + cannot call gzerror with file, since its structures have been deallocated. + gzclose must not be called more than once on the same file, just as free + must not be called more than once on the same allocation. + + gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a + file operation error, Z_MEM_ERROR if out of memory, Z_BUF_ERROR if the + last read ended in the middle of a gzip stream, or Z_OK on success. +*/ + +ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); +ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); +/* + Same as gzclose(), but gzclose_r() is only for use when reading, and + gzclose_w() is only for use when writing or appending. The advantage to + using these instead of gzclose() is that they avoid linking in zlib + compression or decompression code that is not used when only reading or only + writing respectively. If gzclose() is used, then both compression and + decompression code will be included the application when linking to a static + zlib library. +*/ + +ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); +/* + Return the error message for the last error which occurred on file. + errnum is set to zlib error number. If an error occurred in the file system + and not in the compression library, errnum is set to Z_ERRNO and the + application may consult errno to get the exact error code. + + The application must not modify the returned string. Future calls to + this function may invalidate the previously returned string. If file is + closed, then the string previously returned by gzerror will no longer be + available. + + gzerror() should be used to distinguish errors from end-of-file for those + functions above that do not distinguish those cases in their return values. +*/ + +ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); +/* + Clear the error and end-of-file flags for file. This is analogous to the + clearerr() function in stdio. This is useful for continuing to read a gzip + file that is being written concurrently. +*/ + +#endif /* !Z_SOLO */ + + /* checksum functions */ + +/* + These functions are not related to compression but are exported + anyway because they might be useful in applications using the compression + library. +*/ + +ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); +/* + Update a running Adler-32 checksum with the bytes buf[0..len-1] and + return the updated checksum. An Adler-32 value is in the range of a 32-bit + unsigned integer. If buf is Z_NULL, this function returns the required + initial value for the checksum. + + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed + much faster. + + Usage example: + + uLong adler = adler32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + adler = adler32(adler, buffer, length); + } + if (adler != original_adler) error(); +*/ + +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as adler32(), but with a size_t length. +*/ + +/* +ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, + z_off_t len2)); + + Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 + and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for + each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of + seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. Note + that the z_off_t type (like off_t) is a signed integer. If len2 is + negative, the result has no meaning or utility. +*/ + +ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); +/* + Update a running CRC-32 with the bytes buf[0..len-1] and return the + updated CRC-32. A CRC-32 value is in the range of a 32-bit unsigned integer. + If buf is Z_NULL, this function returns the required initial value for the + crc. Pre- and post-conditioning (one's complement) is performed within this + function so it shouldn't be done by the application. + + Usage example: + + uLong crc = crc32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + crc = crc32(crc, buffer, length); + } + if (crc != original_crc) error(); +*/ + +ZEXTERN uLong ZEXPORT crc32_z OF((uLong crc, const Bytef *buf, + z_size_t len)); +/* + Same as crc32(), but with a size_t length. +*/ + +/* +ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); + + Combine two CRC-32 check values into one. For two sequences of bytes, + seq1 and seq2 with lengths len1 and len2, CRC-32 check values were + calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 + check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and + len2. +*/ + +/* +ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t len2)); + + Return the operator corresponding to length len2, to be used with + crc32_combine_op(). +*/ + +ZEXTERN uLong ZEXPORT crc32_combine_op OF((uLong crc1, uLong crc2, uLong op)); +/* + Give the same result as crc32_combine(), using op in place of len2. op is + is generated from len2 by crc32_combine_gen(). This will be faster than + crc32_combine() if the generated op is used more than once. +*/ + + + /* various hacks, don't look :) */ + +/* deflateInit and inflateInit are macros to allow checking the zlib version + * and the compiler's view of z_stream: + */ +ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method, + int windowBits, int memLevel, + int strategy, const char *version, + int stream_size)); +ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, + unsigned char FAR *window, + const char *version, + int stream_size)); +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#endif + +#ifndef Z_SOLO + +/* gzgetc() macro and its supporting function and exposed data structure. Note + * that the real internal state is much larger than the exposed structure. + * This abbreviated structure exposes just enough for the gzgetc() macro. The + * user should not mess with these exposed elements, since their names or + * behavior could change in the future, perhaps even capriciously. They can + * only be used by the gzgetc() macro. You have been warned. + */ +struct gzFile_s { + unsigned have; + unsigned char *next; + z_off64_t pos; +}; +ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +# define z_gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#else +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#endif + +/* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or + * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if + * both are true, the application gets the *64 functions, and the regular + * functions are changed to 64 bits) -- in case these are set on systems + * without large file support, _LFS64_LARGEFILE must also be true + */ +#ifdef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off64_t)); +#endif + +#if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) +# ifdef Z_PREFIX_SET +# define z_gzopen z_gzopen64 +# define z_gzseek z_gzseek64 +# define z_gztell z_gztell64 +# define z_gzoffset z_gzoffset64 +# define z_adler32_combine z_adler32_combine64 +# define z_crc32_combine z_crc32_combine64 +# define z_crc32_combine_gen z_crc32_combine_gen64 +# else +# define gzopen gzopen64 +# define gzseek gzseek64 +# define gztell gztell64 +# define gzoffset gzoffset64 +# define adler32_combine adler32_combine64 +# define crc32_combine crc32_combine64 +# define crc32_combine_gen crc32_combine_gen64 +# endif +# ifndef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off_t)); +# endif +#else + ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t)); +#endif + +#else /* Z_SOLO */ + + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t)); + +#endif /* !Z_SOLO */ + +/* undocumented functions */ +ZEXTERN const char * ZEXPORT zError OF((int)); +ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); +ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); +ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); +ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); +ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); +#if defined(_WIN32) && !defined(Z_SOLO) +ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, + const char *mode)); +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, + const char *format, + va_list va)); +# endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* ZLIB_H */ diff --git a/optee_os/core/lib/zlib/zutil.c b/optee_os/core/lib/zlib/zutil.c new file mode 100644 index 0000000..8466f6d --- /dev/null +++ b/optee_os/core/lib/zlib/zutil.c @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: Zlib +/* zutil.c -- target dependent utility functions for the compression library + * Copyright (C) 1995-2017 Jean-loup Gailly + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#include "zutil.h" +#ifndef Z_SOLO +# include "gzguts.h" +#endif + +z_const char * const z_errmsg[10] = { + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; + + +const char * ZEXPORT zlibVersion() +{ + return ZLIB_VERSION; +} + +uLong ZEXPORT zlibCompileFlags() +{ + uLong flags; + + flags = 0; + switch ((int)(sizeof(uInt))) { + case 2: break; + case 4: flags += 1; break; + case 8: flags += 2; break; + default: flags += 3; + } + switch ((int)(sizeof(uLong))) { + case 2: break; + case 4: flags += 1 << 2; break; + case 8: flags += 2 << 2; break; + default: flags += 3 << 2; + } + switch ((int)(sizeof(voidpf))) { + case 2: break; + case 4: flags += 1 << 4; break; + case 8: flags += 2 << 4; break; + default: flags += 3 << 4; + } + switch ((int)(sizeof(z_off_t))) { + case 2: break; + case 4: flags += 1 << 6; break; + case 8: flags += 2 << 6; break; + default: flags += 3 << 6; + } +#ifdef ZLIB_DEBUG + flags += 1 << 8; +#endif +#if defined(ASMV) || defined(ASMINF) + flags += 1 << 9; +#endif +#ifdef ZLIB_WINAPI + flags += 1 << 10; +#endif +#ifdef BUILDFIXED + flags += 1 << 12; +#endif +#ifdef DYNAMIC_CRC_TABLE + flags += 1 << 13; +#endif +#ifdef NO_GZCOMPRESS + flags += 1L << 16; +#endif +#ifdef NO_GZIP + flags += 1L << 17; +#endif +#ifdef PKZIP_BUG_WORKAROUND + flags += 1L << 20; +#endif +#ifdef FASTEST + flags += 1L << 21; +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifdef NO_vsnprintf + flags += 1L << 25; +# ifdef HAS_vsprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_vsnprintf_void + flags += 1L << 26; +# endif +# endif +#else + flags += 1L << 24; +# ifdef NO_snprintf + flags += 1L << 25; +# ifdef HAS_sprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_snprintf_void + flags += 1L << 26; +# endif +# endif +#endif + return flags; +} + +#ifdef ZLIB_DEBUG +#include +# ifndef verbose +# define verbose 0 +# endif +int ZLIB_INTERNAL z_verbose = verbose; + +void ZLIB_INTERNAL z_error (m) + char *m; +{ + fprintf(stderr, "%s\n", m); + exit(1); +} +#endif + +/* exported to allow conversion of error code to string for compress() and + * uncompress() + */ +const char * ZEXPORT zError(err) + int err; +{ + return ERR_MSG(err); +} + +#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 + /* The older Microsoft C Run-Time Library for Windows CE doesn't have + * errno. We define it as a global variable to simplify porting. + * Its value is always 0 and should not be used. + */ + int errno = 0; +#endif + +#ifndef HAVE_MEMCPY + +void ZLIB_INTERNAL zmemcpy(dest, source, len) + Bytef* dest; + const Bytef* source; + uInt len; +{ + if (len == 0) return; + do { + *dest++ = *source++; /* ??? to be unrolled */ + } while (--len != 0); +} + +int ZLIB_INTERNAL zmemcmp(s1, s2, len) + const Bytef* s1; + const Bytef* s2; + uInt len; +{ + uInt j; + + for (j = 0; j < len; j++) { + if (s1[j] != s2[j]) return 2*(s1[j] > s2[j])-1; + } + return 0; +} + +void ZLIB_INTERNAL zmemzero(dest, len) + Bytef* dest; + uInt len; +{ + if (len == 0) return; + do { + *dest++ = 0; /* ??? to be unrolled */ + } while (--len != 0); +} +#endif + +#ifndef Z_SOLO + +#ifdef SYS16BIT + +#ifdef __TURBOC__ +/* Turbo C in 16-bit mode */ + +# define MY_ZCALLOC + +/* Turbo C malloc() does not allow dynamic allocation of 64K bytes + * and farmalloc(64K) returns a pointer with an offset of 8, so we + * must fix the pointer. Warning: the pointer must be put back to its + * original form in order to free it, use zcfree(). + */ + +#define MAX_PTR 10 +/* 10*64K = 640K */ + +local int next_ptr = 0; + +typedef struct ptr_table_s { + voidpf org_ptr; + voidpf new_ptr; +} ptr_table; + +local ptr_table table[MAX_PTR]; +/* This table is used to remember the original form of pointers + * to large buffers (64K). Such pointers are normalized with a zero offset. + * Since MSDOS is not a preemptive multitasking OS, this table is not + * protected from concurrent access. This hack doesn't work anyway on + * a protected system like OS/2. Use Microsoft C instead. + */ + +voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) +{ + voidpf buf; + ulg bsize = (ulg)items*size; + + (void)opaque; + + /* If we allocate less than 65520 bytes, we assume that farmalloc + * will return a usable pointer which doesn't have to be normalized. + */ + if (bsize < 65520L) { + buf = farmalloc(bsize); + if (*(ush*)&buf != 0) return buf; + } else { + buf = farmalloc(bsize + 16L); + } + if (buf == NULL || next_ptr >= MAX_PTR) return NULL; + table[next_ptr].org_ptr = buf; + + /* Normalize the pointer to seg:0 */ + *((ush*)&buf+1) += ((ush)((uch*)buf-0) + 15) >> 4; + *(ush*)&buf = 0; + table[next_ptr++].new_ptr = buf; + return buf; +} + +void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) +{ + int n; + + (void)opaque; + + if (*(ush*)&ptr != 0) { /* object < 64K */ + farfree(ptr); + return; + } + /* Find the original pointer */ + for (n = 0; n < next_ptr; n++) { + if (ptr != table[n].new_ptr) continue; + + farfree(table[n].org_ptr); + while (++n < next_ptr) { + table[n-1] = table[n]; + } + next_ptr--; + return; + } + Assert(0, "zcfree: ptr not found"); +} + +#endif /* __TURBOC__ */ + + +#ifdef M_I86 +/* Microsoft C in 16-bit mode */ + +# define MY_ZCALLOC + +#if (!defined(_MSC_VER) || (_MSC_VER <= 600)) +# define _halloc halloc +# define _hfree hfree +#endif + +voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) +{ + (void)opaque; + return _halloc((long)items, size); +} + +void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) +{ + (void)opaque; + _hfree(ptr); +} + +#endif /* M_I86 */ + +#endif /* SYS16BIT */ + + +#ifndef MY_ZCALLOC /* Any system without a special alloc function */ + +#ifndef STDC +extern voidp malloc OF((uInt size)); +extern voidp calloc OF((uInt items, uInt size)); +extern void free OF((voidpf ptr)); +#endif + +voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) + voidpf opaque; + unsigned items; + unsigned size; +{ + (void)opaque; + return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : + (voidpf)calloc(items, size); +} + +void ZLIB_INTERNAL zcfree (opaque, ptr) + voidpf opaque; + voidpf ptr; +{ + (void)opaque; + free(ptr); +} + +#endif /* MY_ZCALLOC */ + +#endif /* !Z_SOLO */ diff --git a/optee_os/core/lib/zlib/zutil.h b/optee_os/core/lib/zlib/zutil.h new file mode 100644 index 0000000..fcf8ef5 --- /dev/null +++ b/optee_os/core/lib/zlib/zutil.h @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: Zlib */ +/* zutil.h -- internal interface and configuration of the compression library + * Copyright (C) 1995-2022 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* @(#) $Id$ */ + +#ifndef ZUTIL_H +#define ZUTIL_H + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include "zlib.h" + +#if defined(STDC) && !defined(Z_SOLO) +# if !(defined(_WIN32_WCE) && defined(_MSC_VER)) +# include +# endif +# include +# include +#endif + +#ifndef local +# define local static +#endif +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ + +typedef unsigned char uch; +typedef uch FAR uchf; +typedef unsigned short ush; +typedef ush FAR ushf; +typedef unsigned long ulg; + +#if !defined(Z_U8) && !defined(Z_SOLO) && defined(STDC) +# include +# if (ULONG_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned long +# elif (ULLONG_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned long long +# elif (UINT_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned +# endif +#endif + +extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ +/* (size given to avoid silly warnings with Visual C++) */ + +#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] + +#define ERR_RETURN(strm,err) \ + return (strm->msg = ERR_MSG(err), (err)) +/* To be used only when the state is known to be valid */ + + /* common constants */ + +#ifndef DEF_WBITS +# define DEF_WBITS MAX_WBITS +#endif +/* default windowBits for decompression. MAX_WBITS is for compression only */ + +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +/* default memLevel */ + +#define STORED_BLOCK 0 +#define STATIC_TREES 1 +#define DYN_TREES 2 +/* The three kinds of block type */ + +#define MIN_MATCH 3 +#define MAX_MATCH 258 +/* The minimum and maximum match lengths */ + +#define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */ + + /* target dependencies */ + +#if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) +# define OS_CODE 0x00 +# ifndef Z_SOLO +# if defined(__TURBOC__) || defined(__BORLANDC__) +# if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) + /* Allow compilation with ANSI keywords only enabled */ + void _Cdecl farfree( void *block ); + void *_Cdecl farmalloc( unsigned long nbytes ); +# else +# include +# endif +# else /* MSC or DJGPP */ +# include +# endif +# endif +#endif + +#ifdef AMIGA +# define OS_CODE 1 +#endif + +#if defined(VAXC) || defined(VMS) +# define OS_CODE 2 +# define F_OPEN(name, mode) \ + fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") +#endif + +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + +#if defined(ATARI) || defined(atarist) +# define OS_CODE 5 +#endif + +#ifdef OS2 +# define OS_CODE 6 +# if defined(M_I86) && !defined(Z_SOLO) +# include +# endif +#endif + +#if defined(MACOS) || defined(TARGET_OS_MAC) +# define OS_CODE 7 +# ifndef Z_SOLO +# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include /* for fdopen */ +# else +# ifndef fdopen +# define fdopen(fd,mode) NULL /* No fdopen() */ +# endif +# endif +# endif +#endif + +#ifdef __acorn +# define OS_CODE 13 +#endif + +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 +#endif + +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 +#endif + +#ifdef __APPLE__ +# define OS_CODE 19 +#endif + +#if defined(_BEOS_) || defined(RISCOS) +# define fdopen(fd,mode) NULL /* No fdopen() */ +#endif + +#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX +# if defined(_WIN32_WCE) +# define fdopen(fd,mode) NULL /* No fdopen() */ +# else +# define fdopen(fd,type) _fdopen(fd,type) +# endif +#endif + +#if defined(__BORLANDC__) && !defined(MSDOS) + #pragma warn -8004 + #pragma warn -8008 + #pragma warn -8066 +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_WIN32) && \ + (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); +#endif + + /* common defaults */ + +#ifndef OS_CODE +# define OS_CODE 3 /* assume Unix */ +#endif + +#ifndef F_OPEN +# define F_OPEN(name, mode) fopen((name), (mode)) +#endif + + /* functions */ + +#if defined(pyr) || defined(Z_SOLO) +# define NO_MEMCPY +#endif +#if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__) + /* Use our own functions for small and medium model with MSC <= 5.0. + * You may have to use the same strategy for Borland C (untested). + * The __SC__ check is for Symantec. + */ +# define NO_MEMCPY +#endif +#if defined(STDC) && !defined(HAVE_MEMCPY) && !defined(NO_MEMCPY) +# define HAVE_MEMCPY +#endif +#ifdef HAVE_MEMCPY +# ifdef SMALL_MEDIUM /* MSDOS small or medium model */ +# define zmemcpy _fmemcpy +# define zmemcmp _fmemcmp +# define zmemzero(dest, len) _fmemset(dest, 0, len) +# else +# define zmemcpy memcpy +# define zmemcmp memcmp +# define zmemzero(dest, len) memset(dest, 0, len) +# endif +#else + void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); + int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); + void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len)); +#endif + +/* Diagnostic functions */ +#ifdef ZLIB_DEBUG +# include + extern int ZLIB_INTERNAL z_verbose; + extern void ZLIB_INTERNAL z_error OF((char *m)); +# define Assert(cond,msg) {if(!(cond)) z_error(msg);} +# define Trace(x) {if (z_verbose>=0) fprintf x ;} +# define Tracev(x) {if (z_verbose>0) fprintf x ;} +# define Tracevv(x) {if (z_verbose>1) fprintf x ;} +# define Tracec(c,x) {if (z_verbose>0 && (c)) fprintf x ;} +# define Tracecv(c,x) {if (z_verbose>1 && (c)) fprintf x ;} +#else +# define Assert(cond,msg) +# define Trace(x) +# define Tracev(x) +# define Tracevv(x) +# define Tracec(c,x) +# define Tracecv(c,x) +#endif + +#ifndef Z_SOLO + voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, + unsigned size)); + void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); +#endif + +#define ZALLOC(strm, items, size) \ + (*((strm)->zalloc))((strm)->opaque, (items), (size)) +#define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) +#define TRY_FREE(s, p) {if (p) ZFREE(s, p);} + +/* Reverse the bytes in a 32-bit value */ +#define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + +#endif /* ZUTIL_H */ diff --git a/optee_os/core/mm/core_mmu.c b/optee_os/core/mm/core_mmu.c new file mode 100644 index 0000000..584ff78 --- /dev/null +++ b/optee_os/core/mm/core_mmu.c @@ -0,0 +1,2579 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, 2022 Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2022, Arm Limited and Contributors. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef DEBUG_XLAT_TABLE +#define DEBUG_XLAT_TABLE 0 +#endif + +#define SHM_VASPACE_SIZE (1024 * 1024 * 32) + +#ifdef CFG_CORE_PHYS_RELOCATABLE +unsigned long core_mmu_tee_load_pa __nex_bss; +#else +const unsigned long core_mmu_tee_load_pa = TEE_LOAD_ADDR; +#endif + +/* + * These variables are initialized before .bss is cleared. To avoid + * resetting them when .bss is cleared we're storing them in .data instead, + * even if they initially are zero. + */ + +#ifdef CFG_CORE_RESERVED_SHM +/* Default NSec shared memory allocated from NSec world */ +unsigned long default_nsec_shm_size __nex_bss; +unsigned long default_nsec_shm_paddr __nex_bss; +#endif + +static struct tee_mmap_region static_memory_map[CFG_MMAP_REGIONS +#if defined(CFG_CORE_ASLR) || defined(CFG_CORE_PHYS_RELOCATABLE) + + 1 +#endif + + 1] __nex_bss; + +/* Define the platform's memory layout. */ +struct memaccess_area { + paddr_t paddr; + size_t size; +}; + +#define MEMACCESS_AREA(a, s) { .paddr = a, .size = s } + +static struct memaccess_area secure_only[] __nex_data = { +#ifdef CFG_CORE_PHYS_RELOCATABLE + MEMACCESS_AREA(0, 0), +#else +#ifdef TRUSTED_SRAM_BASE + MEMACCESS_AREA(TRUSTED_SRAM_BASE, TRUSTED_SRAM_SIZE), +#endif + MEMACCESS_AREA(TRUSTED_DRAM_BASE, TRUSTED_DRAM_SIZE), +#endif +}; + +static struct memaccess_area nsec_shared[] __nex_data = { +#ifdef CFG_CORE_RESERVED_SHM + MEMACCESS_AREA(TEE_SHMEM_START, TEE_SHMEM_SIZE), +#endif +}; + +#if defined(CFG_SECURE_DATA_PATH) +static const char *tz_sdp_match = "linaro,secure-heap"; +static struct memaccess_area sec_sdp; +#ifdef CFG_TEE_SDP_MEM_BASE +register_sdp_mem(CFG_TEE_SDP_MEM_BASE, CFG_TEE_SDP_MEM_SIZE); +#endif +#ifdef TEE_SDP_TEST_MEM_BASE +register_sdp_mem(TEE_SDP_TEST_MEM_BASE, TEE_SDP_TEST_MEM_SIZE); +#endif +#endif + +#ifdef CFG_CORE_RESERVED_SHM +register_phys_mem(MEM_AREA_NSEC_SHM, TEE_SHMEM_START, TEE_SHMEM_SIZE); +#endif +static unsigned int mmu_spinlock; + +static uint32_t mmu_lock(void) +{ + return cpu_spin_lock_xsave(&mmu_spinlock); +} + +static void mmu_unlock(uint32_t exceptions) +{ + cpu_spin_unlock_xrestore(&mmu_spinlock, exceptions); +} + +void core_mmu_get_secure_memory(paddr_t *base, paddr_size_t *size) +{ + /* + * The first range is always used to cover OP-TEE core memory, but + * depending on configuration it may cover more than that. + */ + *base = secure_only[0].paddr; + *size = secure_only[0].size; +} + +void core_mmu_set_secure_memory(paddr_t base, size_t size) +{ +#ifdef CFG_CORE_PHYS_RELOCATABLE + static_assert(ARRAY_SIZE(secure_only) == 1); +#endif + runtime_assert(IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE)); + assert(!secure_only[0].size); + assert(base && size); + + DMSG("Physical secure memory base %#"PRIxPA" size %#zx", base, size); + secure_only[0].paddr = base; + secure_only[0].size = size; +} + +void core_mmu_get_ta_range(paddr_t *base, size_t *size) +{ + paddr_t b = 0; + size_t s = 0; + + static_assert(!(TEE_RAM_VA_SIZE % SMALL_PAGE_SIZE)); +#ifdef TA_RAM_START + b = TA_RAM_START; + s = TA_RAM_SIZE; +#else + static_assert(ARRAY_SIZE(secure_only) <= 2); + if (ARRAY_SIZE(secure_only) == 1) { + vaddr_t load_offs = 0; + + assert(core_mmu_tee_load_pa >= secure_only[0].paddr); + load_offs = core_mmu_tee_load_pa - secure_only[0].paddr; + + assert(secure_only[0].size > + load_offs + TEE_RAM_VA_SIZE + TEE_SDP_TEST_MEM_SIZE); + b = secure_only[0].paddr + load_offs + TEE_RAM_VA_SIZE; + s = secure_only[0].size - load_offs - TEE_RAM_VA_SIZE - + TEE_SDP_TEST_MEM_SIZE; + } else { + assert(secure_only[1].size > TEE_SDP_TEST_MEM_SIZE); + b = secure_only[1].paddr; + s = secure_only[1].size - TEE_SDP_TEST_MEM_SIZE; + } +#endif + if (base) + *base = b; + if (size) + *size = s; +} + +static struct tee_mmap_region *get_memory_map(void) +{ + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + struct tee_mmap_region *map = virt_get_memory_map(); + + if (map) + return map; + } + + return static_memory_map; +} + +static bool _pbuf_intersects(struct memaccess_area *a, size_t alen, + paddr_t pa, size_t size) +{ + size_t n; + + for (n = 0; n < alen; n++) + if (core_is_buffer_intersect(pa, size, a[n].paddr, a[n].size)) + return true; + return false; +} + +#define pbuf_intersects(a, pa, size) \ + _pbuf_intersects((a), ARRAY_SIZE(a), (pa), (size)) + +static bool _pbuf_is_inside(struct memaccess_area *a, size_t alen, + paddr_t pa, size_t size) +{ + size_t n; + + for (n = 0; n < alen; n++) + if (core_is_buffer_inside(pa, size, a[n].paddr, a[n].size)) + return true; + return false; +} + +#define pbuf_is_inside(a, pa, size) \ + _pbuf_is_inside((a), ARRAY_SIZE(a), (pa), (size)) + +static bool pa_is_in_map(struct tee_mmap_region *map, paddr_t pa, size_t len) +{ + paddr_t end_pa = 0; + + if (!map) + return false; + + if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) + return false; + + return (pa >= map->pa && end_pa <= map->pa + map->size - 1); +} + +static bool va_is_in_map(struct tee_mmap_region *map, vaddr_t va) +{ + if (!map) + return false; + return (va >= map->va && va <= (map->va + map->size - 1)); +} + +/* check if target buffer fits in a core default map area */ +static bool pbuf_inside_map_area(unsigned long p, size_t l, + struct tee_mmap_region *map) +{ + return core_is_buffer_inside(p, l, map->pa, map->size); +} + +static struct tee_mmap_region *find_map_by_type(enum teecore_memtypes type) +{ + struct tee_mmap_region *map; + + for (map = get_memory_map(); !core_mmap_is_end_of_table(map); map++) + if (map->type == type) + return map; + return NULL; +} + +static struct tee_mmap_region * +find_map_by_type_and_pa(enum teecore_memtypes type, paddr_t pa, size_t len) +{ + struct tee_mmap_region *map; + + for (map = get_memory_map(); !core_mmap_is_end_of_table(map); map++) { + if (map->type != type) + continue; + if (pa_is_in_map(map, pa, len)) + return map; + } + return NULL; +} + +static struct tee_mmap_region *find_map_by_va(void *va) +{ + struct tee_mmap_region *map = get_memory_map(); + unsigned long a = (unsigned long)va; + + while (!core_mmap_is_end_of_table(map)) { + if (a >= map->va && a <= (map->va - 1 + map->size)) + return map; + map++; + } + return NULL; +} + +static struct tee_mmap_region *find_map_by_pa(unsigned long pa) +{ + struct tee_mmap_region *map = get_memory_map(); + + while (!core_mmap_is_end_of_table(map)) { + if (pa >= map->pa && pa <= (map->pa + map->size - 1)) + return map; + map++; + } + return NULL; +} + +#if defined(CFG_SECURE_DATA_PATH) +static bool dtb_get_sdp_region(void) +{ + void *fdt = NULL; + int node = 0; + int tmp_node = 0; + paddr_t tmp_addr = 0; + size_t tmp_size = 0; + + if (!IS_ENABLED(CFG_EMBED_DTB)) + return false; + + fdt = get_embedded_dt(); + if (!fdt) + panic("No DTB found"); + + node = fdt_node_offset_by_compatible(fdt, -1, tz_sdp_match); + if (node < 0) { + DMSG("No %s compatible node found", tz_sdp_match); + return false; + } + tmp_node = node; + while (tmp_node >= 0) { + tmp_node = fdt_node_offset_by_compatible(fdt, tmp_node, + tz_sdp_match); + if (tmp_node >= 0) + DMSG("Ignore SDP pool node %s, supports only 1 node", + fdt_get_name(fdt, tmp_node, NULL)); + } + + tmp_addr = fdt_reg_base_address(fdt, node); + if (tmp_addr == DT_INFO_INVALID_REG) { + EMSG("%s: Unable to get base addr from DT", tz_sdp_match); + return false; + } + + tmp_size = fdt_reg_size(fdt, node); + if (tmp_size == DT_INFO_INVALID_REG_SIZE) { + EMSG("%s: Unable to get size of base addr from DT", + tz_sdp_match); + return false; + } + + sec_sdp.paddr = tmp_addr; + sec_sdp.size = tmp_size; + + return true; +} +#endif + +#if defined(CFG_CORE_DYN_SHM) || defined(CFG_SECURE_DATA_PATH) +static bool pbuf_is_special_mem(paddr_t pbuf, size_t len, + const struct core_mmu_phys_mem *start, + const struct core_mmu_phys_mem *end) +{ + const struct core_mmu_phys_mem *mem; + + for (mem = start; mem < end; mem++) { + if (core_is_buffer_inside(pbuf, len, mem->addr, mem->size)) + return true; + } + + return false; +} +#endif + +#ifdef CFG_CORE_DYN_SHM +static void carve_out_phys_mem(struct core_mmu_phys_mem **mem, size_t *nelems, + paddr_t pa, size_t size) +{ + struct core_mmu_phys_mem *m = *mem; + size_t n = 0; + + while (true) { + if (n >= *nelems) { + DMSG("No need to carve out %#" PRIxPA " size %#zx", + pa, size); + return; + } + if (core_is_buffer_inside(pa, size, m[n].addr, m[n].size)) + break; + if (!core_is_buffer_outside(pa, size, m[n].addr, m[n].size)) + panic(); + n++; + } + + if (pa == m[n].addr && size == m[n].size) { + /* Remove this entry */ + (*nelems)--; + memmove(m + n, m + n + 1, sizeof(*m) * (*nelems - n)); + m = nex_realloc(m, sizeof(*m) * *nelems); + if (!m) + panic(); + *mem = m; + } else if (pa == m[n].addr) { + m[n].addr += size; + m[n].size -= size; + } else if ((pa + size) == (m[n].addr + m[n].size)) { + m[n].size -= size; + } else { + /* Need to split the memory entry */ + m = nex_realloc(m, sizeof(*m) * (*nelems + 1)); + if (!m) + panic(); + *mem = m; + memmove(m + n + 1, m + n, sizeof(*m) * (*nelems - n)); + (*nelems)++; + m[n].size = pa - m[n].addr; + m[n + 1].size -= size + m[n].size; + m[n + 1].addr = pa + size; + } +} + +static void check_phys_mem_is_outside(struct core_mmu_phys_mem *start, + size_t nelems, + struct tee_mmap_region *map) +{ + size_t n; + + for (n = 0; n < nelems; n++) { + if (!core_is_buffer_outside(start[n].addr, start[n].size, + map->pa, map->size)) { + EMSG("Non-sec mem (%#" PRIxPA ":%#" PRIxPASZ + ") overlaps map (type %d %#" PRIxPA ":%#zx)", + start[n].addr, start[n].size, + map->type, map->pa, map->size); + panic(); + } + } +} + +static const struct core_mmu_phys_mem *discovered_nsec_ddr_start __nex_bss; +static size_t discovered_nsec_ddr_nelems __nex_bss; + +static int cmp_pmem_by_addr(const void *a, const void *b) +{ + const struct core_mmu_phys_mem *pmem_a = a; + const struct core_mmu_phys_mem *pmem_b = b; + + return CMP_TRILEAN(pmem_a->addr, pmem_b->addr); +} + +void core_mmu_set_discovered_nsec_ddr(struct core_mmu_phys_mem *start, + size_t nelems) +{ + struct core_mmu_phys_mem *m = start; + size_t num_elems = nelems; + struct tee_mmap_region *map = static_memory_map; + const struct core_mmu_phys_mem __maybe_unused *pmem; + size_t n = 0; + + assert(!discovered_nsec_ddr_start); + assert(m && num_elems); + + qsort(m, num_elems, sizeof(*m), cmp_pmem_by_addr); + + /* + * Non-secure shared memory and also secure data + * path memory are supposed to reside inside + * non-secure memory. Since NSEC_SHM and SDP_MEM + * are used for a specific purpose make holes for + * those memory in the normal non-secure memory. + * + * This has to be done since for instance QEMU + * isn't aware of which memory range in the + * non-secure memory is used for NSEC_SHM. + */ + +#ifdef CFG_SECURE_DATA_PATH + if (dtb_get_sdp_region()) + carve_out_phys_mem(&m, &num_elems, sec_sdp.paddr, sec_sdp.size); + + for (pmem = phys_sdp_mem_begin; pmem < phys_sdp_mem_end; pmem++) + carve_out_phys_mem(&m, &num_elems, pmem->addr, pmem->size); +#endif + + for (n = 0; n < ARRAY_SIZE(secure_only); n++) + carve_out_phys_mem(&m, &num_elems, secure_only[n].paddr, + secure_only[n].size); + + for (map = static_memory_map; !core_mmap_is_end_of_table(map); map++) { + switch (map->type) { + case MEM_AREA_NSEC_SHM: + carve_out_phys_mem(&m, &num_elems, map->pa, map->size); + break; + case MEM_AREA_EXT_DT: + case MEM_AREA_MANIFEST_DT: + case MEM_AREA_RES_VASPACE: + case MEM_AREA_SHM_VASPACE: + case MEM_AREA_TS_VASPACE: + case MEM_AREA_PAGER_VASPACE: + break; + default: + check_phys_mem_is_outside(m, num_elems, map); + } + } + + discovered_nsec_ddr_start = m; + discovered_nsec_ddr_nelems = num_elems; + + if (!core_mmu_check_end_pa(m[num_elems - 1].addr, + m[num_elems - 1].size)) + panic(); +} + +static bool get_discovered_nsec_ddr(const struct core_mmu_phys_mem **start, + const struct core_mmu_phys_mem **end) +{ + if (!discovered_nsec_ddr_start) + return false; + + *start = discovered_nsec_ddr_start; + *end = discovered_nsec_ddr_start + discovered_nsec_ddr_nelems; + + return true; +} + +static bool pbuf_is_nsec_ddr(paddr_t pbuf, size_t len) +{ + const struct core_mmu_phys_mem *start; + const struct core_mmu_phys_mem *end; + + if (!get_discovered_nsec_ddr(&start, &end)) + return false; + + return pbuf_is_special_mem(pbuf, len, start, end); +} + +bool core_mmu_nsec_ddr_is_defined(void) +{ + const struct core_mmu_phys_mem *start; + const struct core_mmu_phys_mem *end; + + if (!get_discovered_nsec_ddr(&start, &end)) + return false; + + return start != end; +} +#else +static bool pbuf_is_nsec_ddr(paddr_t pbuf __unused, size_t len __unused) +{ + return false; +} +#endif /*CFG_CORE_DYN_SHM*/ + +#define MSG_MEM_INSTERSECT(pa1, sz1, pa2, sz2) \ + EMSG("[%" PRIxPA " %" PRIx64 "] intersects [%" PRIxPA " %" PRIx64 "]", \ + pa1, (uint64_t)pa1 + (sz1), pa2, (uint64_t)pa2 + (sz2)) + +#ifdef CFG_SECURE_DATA_PATH +static bool pbuf_is_sdp_mem(paddr_t pbuf, size_t len) +{ + bool is_sdp_mem = false; + + if (sec_sdp.size) + is_sdp_mem = core_is_buffer_inside(pbuf, len, sec_sdp.paddr, + sec_sdp.size); + + if (!is_sdp_mem) + is_sdp_mem = pbuf_is_special_mem(pbuf, len, phys_sdp_mem_begin, + phys_sdp_mem_end); + + return is_sdp_mem; +} + +static struct mobj *core_sdp_mem_alloc_mobj(paddr_t pa, size_t size) +{ + struct mobj *mobj = mobj_phys_alloc(pa, size, TEE_MATTR_MEM_TYPE_CACHED, + CORE_MEM_SDP_MEM); + + if (!mobj) + panic("can't create SDP physical memory object"); + + return mobj; +} + +struct mobj **core_sdp_mem_create_mobjs(void) +{ + const struct core_mmu_phys_mem *mem = NULL; + struct mobj **mobj_base = NULL; + struct mobj **mobj = NULL; + int cnt = phys_sdp_mem_end - phys_sdp_mem_begin; + + if (sec_sdp.size) + cnt++; + + /* SDP mobjs table must end with a NULL entry */ + mobj_base = calloc(cnt + 1, sizeof(struct mobj *)); + if (!mobj_base) + panic("Out of memory"); + + mobj = mobj_base; + + for (mem = phys_sdp_mem_begin; mem < phys_sdp_mem_end; mem++, mobj++) + *mobj = core_sdp_mem_alloc_mobj(mem->addr, mem->size); + + if (sec_sdp.size) + *mobj = core_sdp_mem_alloc_mobj(sec_sdp.paddr, sec_sdp.size); + + return mobj_base; +} + +#else /* CFG_SECURE_DATA_PATH */ +static bool pbuf_is_sdp_mem(paddr_t pbuf __unused, size_t len __unused) +{ + return false; +} + +#endif /* CFG_SECURE_DATA_PATH */ + +/* Check special memories comply with registered memories */ +static void verify_special_mem_areas(struct tee_mmap_region *mem_map, + size_t len, + const struct core_mmu_phys_mem *start, + const struct core_mmu_phys_mem *end, + const char *area_name __maybe_unused) +{ + const struct core_mmu_phys_mem *mem; + const struct core_mmu_phys_mem *mem2; + struct tee_mmap_region *mmap; + size_t n; + + if (start == end) { + DMSG("No %s memory area defined", area_name); + return; + } + + for (mem = start; mem < end; mem++) + DMSG("%s memory [%" PRIxPA " %" PRIx64 "]", + area_name, mem->addr, (uint64_t)mem->addr + mem->size); + + /* Check memories do not intersect each other */ + for (mem = start; mem + 1 < end; mem++) { + for (mem2 = mem + 1; mem2 < end; mem2++) { + if (core_is_buffer_intersect(mem2->addr, mem2->size, + mem->addr, mem->size)) { + MSG_MEM_INSTERSECT(mem2->addr, mem2->size, + mem->addr, mem->size); + panic("Special memory intersection"); + } + } + } + + /* + * Check memories do not intersect any mapped memory. + * This is called before reserved VA space is loaded in mem_map. + */ + for (mem = start; mem < end; mem++) { + for (mmap = mem_map, n = 0; n < len; mmap++, n++) { + if (core_is_buffer_intersect(mem->addr, mem->size, + mmap->pa, mmap->size)) { + MSG_MEM_INSTERSECT(mem->addr, mem->size, + mmap->pa, mmap->size); + panic("Special memory intersection"); + } + } + } +} + +static void add_phys_mem(struct tee_mmap_region *memory_map, size_t num_elems, + const char *mem_name __maybe_unused, + enum teecore_memtypes mem_type, + paddr_t mem_addr, paddr_size_t mem_size, size_t *last) +{ + size_t n = 0; + paddr_t pa; + paddr_size_t size; + + if (!mem_size) /* Discard null size entries */ + return; + /* + * If some ranges of memory of the same type do overlap + * each others they are coalesced into one entry. To help this + * added entries are sorted by increasing physical. + * + * Note that it's valid to have the same physical memory as several + * different memory types, for instance the same device memory + * mapped as both secure and non-secure. This will probably not + * happen often in practice. + */ + DMSG("%s type %s 0x%08" PRIxPA " size 0x%08" PRIxPASZ, + mem_name, teecore_memtype_name(mem_type), mem_addr, mem_size); + while (true) { + if (n >= (num_elems - 1)) { + EMSG("Out of entries (%zu) in memory_map", num_elems); + panic(); + } + if (n == *last) + break; + pa = memory_map[n].pa; + size = memory_map[n].size; + if (mem_type == memory_map[n].type && + ((pa <= (mem_addr + (mem_size - 1))) && + (mem_addr <= (pa + (size - 1))))) { + DMSG("Physical mem map overlaps 0x%" PRIxPA, mem_addr); + memory_map[n].pa = MIN(pa, mem_addr); + memory_map[n].size = MAX(size, mem_size) + + (pa - memory_map[n].pa); + return; + } + if (mem_type < memory_map[n].type || + (mem_type == memory_map[n].type && mem_addr < pa)) + break; /* found the spot where to insert this memory */ + n++; + } + + memmove(memory_map + n + 1, memory_map + n, + sizeof(struct tee_mmap_region) * (*last - n)); + (*last)++; + memset(memory_map + n, 0, sizeof(memory_map[0])); + memory_map[n].type = mem_type; + memory_map[n].pa = mem_addr; + memory_map[n].size = mem_size; +} + +static void add_va_space(struct tee_mmap_region *memory_map, size_t num_elems, + enum teecore_memtypes type, size_t size, size_t *last) +{ + size_t n = 0; + + DMSG("type %s size 0x%08zx", teecore_memtype_name(type), size); + while (true) { + if (n >= (num_elems - 1)) { + EMSG("Out of entries (%zu) in memory_map", num_elems); + panic(); + } + if (n == *last) + break; + if (type < memory_map[n].type) + break; + n++; + } + + memmove(memory_map + n + 1, memory_map + n, + sizeof(struct tee_mmap_region) * (*last - n)); + (*last)++; + memset(memory_map + n, 0, sizeof(memory_map[0])); + memory_map[n].type = type; + memory_map[n].size = size; +} + +uint32_t core_mmu_type_to_attr(enum teecore_memtypes t) +{ + const uint32_t attr = TEE_MATTR_VALID_BLOCK; + const uint32_t tagged = TEE_MATTR_MEM_TYPE_TAGGED << + TEE_MATTR_MEM_TYPE_SHIFT; + const uint32_t cached = TEE_MATTR_MEM_TYPE_CACHED << + TEE_MATTR_MEM_TYPE_SHIFT; + const uint32_t noncache = TEE_MATTR_MEM_TYPE_DEV << + TEE_MATTR_MEM_TYPE_SHIFT; + + switch (t) { + case MEM_AREA_TEE_RAM: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | tagged; + case MEM_AREA_TEE_RAM_RX: + case MEM_AREA_INIT_RAM_RX: + case MEM_AREA_IDENTITY_MAP_RX: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRX | tagged; + case MEM_AREA_TEE_RAM_RO: + case MEM_AREA_INIT_RAM_RO: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | tagged; + case MEM_AREA_TEE_RAM_RW: + case MEM_AREA_NEX_RAM_RO: /* This has to be r/w during init runtime */ + case MEM_AREA_NEX_RAM_RW: + case MEM_AREA_TEE_ASAN: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; + case MEM_AREA_TEE_COHERENT: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | noncache; + case MEM_AREA_TA_RAM: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; + case MEM_AREA_NSEC_SHM: + case MEM_AREA_NEX_NSEC_SHM: + return attr | TEE_MATTR_PRW | cached; + case MEM_AREA_MANIFEST_DT: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; + case MEM_AREA_EXT_DT: + /* + * If CFG_MAP_EXT_DT_SECURE is enabled map the external device + * tree as secure non-cached memory, otherwise, fall back to + * non-secure mapping. + */ + if (IS_ENABLED(CFG_MAP_EXT_DT_SECURE)) + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | + noncache; + fallthrough; + case MEM_AREA_IO_NSEC: + return attr | TEE_MATTR_PRW | noncache; + case MEM_AREA_IO_SEC: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | noncache; + case MEM_AREA_RAM_NSEC: + return attr | TEE_MATTR_PRW | cached; + case MEM_AREA_RAM_SEC: + case MEM_AREA_SEC_RAM_OVERALL: + return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; + case MEM_AREA_RES_VASPACE: + case MEM_AREA_SHM_VASPACE: + return 0; + case MEM_AREA_PAGER_VASPACE: + return TEE_MATTR_SECURE; + default: + panic("invalid type"); + } +} + +static bool __maybe_unused map_is_tee_ram(const struct tee_mmap_region *mm) +{ + switch (mm->type) { + case MEM_AREA_TEE_RAM: + case MEM_AREA_TEE_RAM_RX: + case MEM_AREA_TEE_RAM_RO: + case MEM_AREA_TEE_RAM_RW: + case MEM_AREA_INIT_RAM_RX: + case MEM_AREA_INIT_RAM_RO: + case MEM_AREA_NEX_RAM_RW: + case MEM_AREA_NEX_RAM_RO: + case MEM_AREA_TEE_ASAN: + return true; + default: + return false; + } +} + +static bool __maybe_unused map_is_secure(const struct tee_mmap_region *mm) +{ + return !!(core_mmu_type_to_attr(mm->type) & TEE_MATTR_SECURE); +} + +static bool __maybe_unused map_is_pgdir(const struct tee_mmap_region *mm) +{ + return mm->region_size == CORE_MMU_PGDIR_SIZE; +} + +static int cmp_mmap_by_lower_va(const void *a, const void *b) +{ + const struct tee_mmap_region *mm_a = a; + const struct tee_mmap_region *mm_b = b; + + return CMP_TRILEAN(mm_a->va, mm_b->va); +} + +static void dump_mmap_table(struct tee_mmap_region *memory_map) +{ + struct tee_mmap_region *map; + + for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { + vaddr_t __maybe_unused vstart; + + vstart = map->va + ((vaddr_t)map->pa & (map->region_size - 1)); + DMSG("type %-12s va 0x%08" PRIxVA "..0x%08" PRIxVA + " pa 0x%08" PRIxPA "..0x%08" PRIxPA " size 0x%08zx (%s)", + teecore_memtype_name(map->type), vstart, + vstart + map->size - 1, map->pa, + (paddr_t)(map->pa + map->size - 1), map->size, + map->region_size == SMALL_PAGE_SIZE ? "smallpg" : "pgdir"); + } +} + +#if DEBUG_XLAT_TABLE + +static void dump_xlat_table(vaddr_t va, unsigned int level) +{ + struct core_mmu_table_info tbl_info; + unsigned int idx = 0; + paddr_t pa; + uint32_t attr; + + core_mmu_find_table(NULL, va, level, &tbl_info); + va = tbl_info.va_base; + for (idx = 0; idx < tbl_info.num_entries; idx++) { + core_mmu_get_entry(&tbl_info, idx, &pa, &attr); + if (attr || level > CORE_MMU_BASE_TABLE_LEVEL) { + const char *security_bit = ""; + + if (core_mmu_entry_have_security_bit(attr)) { + if (attr & TEE_MATTR_SECURE) + security_bit = "S"; + else + security_bit = "NS"; + } + + if (attr & TEE_MATTR_TABLE) { + DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA + " TBL:0x%010" PRIxPA " %s", + level * 2, "", level, va, pa, + security_bit); + dump_xlat_table(va, level + 1); + } else if (attr) { + DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA + " PA:0x%010" PRIxPA " %s-%s-%s-%s", + level * 2, "", level, va, pa, + mattr_is_cached(attr) ? "MEM" : + "DEV", + attr & TEE_MATTR_PW ? "RW" : "RO", + attr & TEE_MATTR_PX ? "X " : "XN", + security_bit); + } else { + DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA + " INVALID\n", + level * 2, "", level, va); + } + } + va += BIT64(tbl_info.shift); + } +} + +#else + +static void dump_xlat_table(vaddr_t va __unused, int level __unused) +{ +} + +#endif + +/* + * Reserves virtual memory space for pager usage. + * + * From the start of the first memory used by the link script + + * TEE_RAM_VA_SIZE should be covered, either with a direct mapping or empty + * mapping for pager usage. This adds translation tables as needed for the + * pager to operate. + */ +static void add_pager_vaspace(struct tee_mmap_region *mmap, size_t num_elems, + size_t *last) +{ + paddr_t begin = 0; + paddr_t end = 0; + size_t size = 0; + size_t pos = 0; + size_t n = 0; + + if (*last >= (num_elems - 1)) { + EMSG("Out of entries (%zu) in memory map", num_elems); + panic(); + } + + for (n = 0; !core_mmap_is_end_of_table(mmap + n); n++) { + if (map_is_tee_ram(mmap + n)) { + if (!begin) + begin = mmap[n].pa; + pos = n + 1; + } + } + + end = mmap[pos - 1].pa + mmap[pos - 1].size; + assert(end - begin < TEE_RAM_VA_SIZE); + size = TEE_RAM_VA_SIZE - (end - begin); + + assert(pos <= *last); + memmove(mmap + pos + 1, mmap + pos, + sizeof(struct tee_mmap_region) * (*last - pos)); + (*last)++; + memset(mmap + pos, 0, sizeof(mmap[0])); + mmap[pos].type = MEM_AREA_PAGER_VASPACE; + mmap[pos].va = 0; + mmap[pos].size = size; + mmap[pos].region_size = SMALL_PAGE_SIZE; + mmap[pos].attr = core_mmu_type_to_attr(MEM_AREA_PAGER_VASPACE); +} + +static void check_sec_nsec_mem_config(void) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(secure_only); n++) { + if (pbuf_intersects(nsec_shared, secure_only[n].paddr, + secure_only[n].size)) + panic("Invalid memory access config: sec/nsec"); + } +} + +static void collect_device_mem_ranges(struct tee_mmap_region *memory_map, + size_t num_elems, size_t *last) +{ + const char *compatible = "arm,ffa-manifest-device-regions"; + void *fdt = get_manifest_dt(); + const char *name = NULL; + uint64_t page_count = 0; + uint64_t base = 0; + int subnode = 0; + int node = 0; + + node = fdt_node_offset_by_compatible(fdt, 0, compatible); + if (node < 0) + return; + + fdt_for_each_subnode(subnode, fdt, node) { + name = fdt_get_name(fdt, subnode, NULL); + if (!name) + continue; + + if (dt_getprop_as_number(fdt, subnode, "base-address", + &base)) { + EMSG("Mandatory field is missing: base-address"); + continue; + } + + if (base & SMALL_PAGE_MASK) { + EMSG("base-address is not page aligned"); + continue; + } + + if (dt_getprop_as_number(fdt, subnode, "pages-count", + &page_count)) { + EMSG("Mandatory field is missing: pages-count"); + continue; + } + + add_phys_mem(memory_map, num_elems, name, MEM_AREA_IO_SEC, + base, base + page_count * SMALL_PAGE_SIZE, last); + } +} + +static size_t collect_mem_ranges(struct tee_mmap_region *memory_map, + size_t num_elems) +{ + const struct core_mmu_phys_mem *mem = NULL; + vaddr_t ram_start = secure_only[0].paddr; + size_t last = 0; + + +#define ADD_PHYS_MEM(_type, _addr, _size) \ + add_phys_mem(memory_map, num_elems, #_addr, (_type), \ + (_addr), (_size), &last) + + if (IS_ENABLED(CFG_CORE_RWDATA_NOEXEC)) { + ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, ram_start, + VCORE_UNPG_RX_PA - ram_start); + ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RX, VCORE_UNPG_RX_PA, + VCORE_UNPG_RX_SZ); + ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, VCORE_UNPG_RO_PA, + VCORE_UNPG_RO_SZ); + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RO, VCORE_UNPG_RW_PA, + VCORE_UNPG_RW_SZ); + ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RW, VCORE_NEX_RW_PA, + VCORE_NEX_RW_SZ); + } else { + ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RW, VCORE_UNPG_RW_PA, + VCORE_UNPG_RW_SZ); + } + + if (IS_ENABLED(CFG_WITH_PAGER)) { + ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RX, VCORE_INIT_RX_PA, + VCORE_INIT_RX_SZ); + ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RO, VCORE_INIT_RO_PA, + VCORE_INIT_RO_SZ); + } + } else { + ADD_PHYS_MEM(MEM_AREA_TEE_RAM, TEE_RAM_START, TEE_RAM_PH_SIZE); + } + + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { + ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, TRUSTED_DRAM_BASE, + TRUSTED_DRAM_SIZE); + } else { + /* + * Every guest will have own TA RAM if virtualization + * support is enabled. + */ + paddr_t ta_base = 0; + size_t ta_size = 0; + + core_mmu_get_ta_range(&ta_base, &ta_size); + ADD_PHYS_MEM(MEM_AREA_TA_RAM, ta_base, ta_size); + } + + if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS) && + IS_ENABLED(CFG_WITH_PAGER)) { + /* + * Asan ram is part of MEM_AREA_TEE_RAM_RW when pager is + * disabled. + */ + ADD_PHYS_MEM(MEM_AREA_TEE_ASAN, ASAN_MAP_PA, ASAN_MAP_SZ); + } + +#undef ADD_PHYS_MEM + + /* Collect device memory info from SP manifest */ + if (IS_ENABLED(CFG_CORE_SEL2_SPMC)) + collect_device_mem_ranges(memory_map, num_elems, &last); + + for (mem = phys_mem_map_begin; mem < phys_mem_map_end; mem++) { + /* Only unmapped virtual range may have a null phys addr */ + assert(mem->addr || !core_mmu_type_to_attr(mem->type)); + + add_phys_mem(memory_map, num_elems, mem->name, mem->type, + mem->addr, mem->size, &last); + } + + if (IS_ENABLED(CFG_SECURE_DATA_PATH)) + verify_special_mem_areas(memory_map, num_elems, + phys_sdp_mem_begin, + phys_sdp_mem_end, "SDP"); + + add_va_space(memory_map, num_elems, MEM_AREA_RES_VASPACE, + CFG_RESERVED_VASPACE_SIZE, &last); + + add_va_space(memory_map, num_elems, MEM_AREA_SHM_VASPACE, + SHM_VASPACE_SIZE, &last); + + memory_map[last].type = MEM_AREA_END; + + return last; +} + +static void assign_mem_granularity(struct tee_mmap_region *memory_map) +{ + struct tee_mmap_region *map = NULL; + + /* + * Assign region sizes, note that MEM_AREA_TEE_RAM always uses + * SMALL_PAGE_SIZE. + */ + for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { + paddr_t mask = map->pa | map->size; + + if (!(mask & CORE_MMU_PGDIR_MASK)) + map->region_size = CORE_MMU_PGDIR_SIZE; + else if (!(mask & SMALL_PAGE_MASK)) + map->region_size = SMALL_PAGE_SIZE; + else + panic("Impossible memory alignment"); + + if (map_is_tee_ram(map)) + map->region_size = SMALL_PAGE_SIZE; + } +} + +static bool place_tee_ram_at_top(paddr_t paddr) +{ + return paddr > BIT64(core_mmu_get_va_width()) / 2; +} + +/* + * MMU arch driver shall override this function if it helps + * optimizing the memory footprint of the address translation tables. + */ +bool __weak core_mmu_prefer_tee_ram_at_top(paddr_t paddr) +{ + return place_tee_ram_at_top(paddr); +} + +static bool assign_mem_va_dir(vaddr_t tee_ram_va, + struct tee_mmap_region *memory_map, + bool tee_ram_at_top) +{ + struct tee_mmap_region *map = NULL; + vaddr_t va = 0; + bool va_is_secure = true; + + /* + * tee_ram_va might equals 0 when CFG_CORE_ASLR=y. + * 0 is by design an invalid va, so return false directly. + */ + if (!tee_ram_va) + return false; + + /* Clear eventual previous assignments */ + for (map = memory_map; !core_mmap_is_end_of_table(map); map++) + map->va = 0; + + /* + * TEE RAM regions are always aligned with region_size. + * + * Note that MEM_AREA_PAGER_VASPACE also counts as TEE RAM here + * since it handles virtual memory which covers the part of the ELF + * that cannot fit directly into memory. + */ + va = tee_ram_va; + for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { + if (map_is_tee_ram(map) || + map->type == MEM_AREA_PAGER_VASPACE) { + assert(!(va & (map->region_size - 1))); + assert(!(map->size & (map->region_size - 1))); + map->va = va; + if (ADD_OVERFLOW(va, map->size, &va)) + return false; + if (va >= BIT64(core_mmu_get_va_width())) + return false; + } + } + + if (tee_ram_at_top) { + /* + * Map non-tee ram regions at addresses lower than the tee + * ram region. + */ + va = tee_ram_va; + for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { + map->attr = core_mmu_type_to_attr(map->type); + if (map->va) + continue; + + if (!IS_ENABLED(CFG_WITH_LPAE) && + va_is_secure != map_is_secure(map)) { + va_is_secure = !va_is_secure; + va = ROUNDDOWN(va, CORE_MMU_PGDIR_SIZE); + } + + if (SUB_OVERFLOW(va, map->size, &va)) + return false; + va = ROUNDDOWN(va, map->region_size); + /* + * Make sure that va is aligned with pa for + * efficient pgdir mapping. Basically pa & + * pgdir_mask should be == va & pgdir_mask + */ + if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { + if (SUB_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, &va)) + return false; + va += (map->pa - va) & CORE_MMU_PGDIR_MASK; + } + map->va = va; + } + } else { + /* + * Map non-tee ram regions at addresses higher than the tee + * ram region. + */ + for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { + map->attr = core_mmu_type_to_attr(map->type); + if (map->va) + continue; + + if (!IS_ENABLED(CFG_WITH_LPAE) && + va_is_secure != map_is_secure(map)) { + va_is_secure = !va_is_secure; + if (ROUNDUP_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, + &va)) + return false; + } + + if (ROUNDUP_OVERFLOW(va, map->region_size, &va)) + return false; + /* + * Make sure that va is aligned with pa for + * efficient pgdir mapping. Basically pa & + * pgdir_mask should be == va & pgdir_mask + */ + if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { + vaddr_t offs = (map->pa - va) & + CORE_MMU_PGDIR_MASK; + + if (ADD_OVERFLOW(va, offs, &va)) + return false; + } + + map->va = va; + if (ADD_OVERFLOW(va, map->size, &va)) + return false; + if (va >= BIT64(core_mmu_get_va_width())) + return false; + } + } + + return true; +} + +static bool assign_mem_va(vaddr_t tee_ram_va, + struct tee_mmap_region *memory_map) +{ + bool tee_ram_at_top = place_tee_ram_at_top(tee_ram_va); + + /* + * Check that we're not overlapping with the user VA range. + */ + if (IS_ENABLED(CFG_WITH_LPAE)) { + /* + * User VA range is supposed to be defined after these + * mappings have been established. + */ + assert(!core_mmu_user_va_range_is_defined()); + } else { + vaddr_t user_va_base = 0; + size_t user_va_size = 0; + + assert(core_mmu_user_va_range_is_defined()); + core_mmu_get_user_va_range(&user_va_base, &user_va_size); + if (tee_ram_va < (user_va_base + user_va_size)) + return false; + } + + if (IS_ENABLED(CFG_WITH_PAGER)) { + bool prefered_dir = core_mmu_prefer_tee_ram_at_top(tee_ram_va); + + /* Try whole mapping covered by a single base xlat entry */ + if (prefered_dir != tee_ram_at_top && + assign_mem_va_dir(tee_ram_va, memory_map, prefered_dir)) + return true; + } + + return assign_mem_va_dir(tee_ram_va, memory_map, tee_ram_at_top); +} + +static int cmp_init_mem_map(const void *a, const void *b) +{ + const struct tee_mmap_region *mm_a = a; + const struct tee_mmap_region *mm_b = b; + int rc = 0; + + rc = CMP_TRILEAN(mm_a->region_size, mm_b->region_size); + if (!rc) + rc = CMP_TRILEAN(mm_a->pa, mm_b->pa); + /* + * 32bit MMU descriptors cannot mix secure and non-secure mapping in + * the same level2 table. Hence sort secure mapping from non-secure + * mapping. + */ + if (!rc && !IS_ENABLED(CFG_WITH_LPAE)) + rc = CMP_TRILEAN(map_is_secure(mm_a), map_is_secure(mm_b)); + + return rc; +} + +static bool mem_map_add_id_map(struct tee_mmap_region *memory_map, + size_t num_elems, size_t *last, + vaddr_t id_map_start, vaddr_t id_map_end) +{ + struct tee_mmap_region *map = NULL; + vaddr_t start = ROUNDDOWN(id_map_start, SMALL_PAGE_SIZE); + vaddr_t end = ROUNDUP(id_map_end, SMALL_PAGE_SIZE); + size_t len = end - start; + + if (*last >= num_elems - 1) { + EMSG("Out of entries (%zu) in memory map", num_elems); + panic(); + } + + for (map = memory_map; !core_mmap_is_end_of_table(map); map++) + if (core_is_buffer_intersect(map->va, map->size, start, len)) + return false; + + *map = (struct tee_mmap_region){ + .type = MEM_AREA_IDENTITY_MAP_RX, + /* + * Could use CORE_MMU_PGDIR_SIZE to potentially save a + * translation table, at the increased risk of clashes with + * the rest of the memory map. + */ + .region_size = SMALL_PAGE_SIZE, + .pa = start, + .va = start, + .size = len, + .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), + }; + + (*last)++; + + return true; +} + +static unsigned long init_mem_map(struct tee_mmap_region *memory_map, + size_t num_elems, unsigned long seed) +{ + /* + * @id_map_start and @id_map_end describes a physical memory range + * that must be mapped Read-Only eXecutable at identical virtual + * addresses. + */ + vaddr_t id_map_start = (vaddr_t)__identity_map_init_start; + vaddr_t id_map_end = (vaddr_t)__identity_map_init_end; + vaddr_t start_addr = secure_only[0].paddr; + unsigned long offs = 0; + size_t last = 0; + + last = collect_mem_ranges(memory_map, num_elems); + assign_mem_granularity(memory_map); + + /* + * To ease mapping and lower use of xlat tables, sort mapping + * description moving small-page regions after the pgdir regions. + */ + qsort(memory_map, last, sizeof(struct tee_mmap_region), + cmp_init_mem_map); + + if (IS_ENABLED(CFG_WITH_PAGER)) + add_pager_vaspace(memory_map, num_elems, &last); + + if (IS_ENABLED(CFG_CORE_ASLR) && seed) { + vaddr_t base_addr = start_addr + seed; + const unsigned int va_width = core_mmu_get_va_width(); + const vaddr_t va_mask = GENMASK_64(va_width - 1, + SMALL_PAGE_SHIFT); + vaddr_t ba = base_addr; + size_t n = 0; + + for (n = 0; n < 3; n++) { + if (n) + ba = base_addr ^ BIT64(va_width - n); + ba &= va_mask; + if (assign_mem_va(ba, memory_map) && + mem_map_add_id_map(memory_map, num_elems, &last, + id_map_start, id_map_end)) { + offs = ba - start_addr; + DMSG("Mapping core at %#"PRIxVA" offs %#lx", + ba, offs); + goto out; + } else { + DMSG("Failed to map core at %#"PRIxVA, ba); + } + } + EMSG("Failed to map core with seed %#lx", seed); + } + + if (!assign_mem_va(start_addr, memory_map)) + panic(); + +out: + qsort(memory_map, last, sizeof(struct tee_mmap_region), + cmp_mmap_by_lower_va); + + dump_mmap_table(memory_map); + + return offs; +} + +static void check_mem_map(struct tee_mmap_region *map) +{ + struct tee_mmap_region *m = NULL; + + for (m = map; !core_mmap_is_end_of_table(m); m++) { + switch (m->type) { + case MEM_AREA_TEE_RAM: + case MEM_AREA_TEE_RAM_RX: + case MEM_AREA_TEE_RAM_RO: + case MEM_AREA_TEE_RAM_RW: + case MEM_AREA_INIT_RAM_RX: + case MEM_AREA_INIT_RAM_RO: + case MEM_AREA_NEX_RAM_RW: + case MEM_AREA_NEX_RAM_RO: + case MEM_AREA_IDENTITY_MAP_RX: + if (!pbuf_is_inside(secure_only, m->pa, m->size)) + panic("TEE_RAM can't fit in secure_only"); + break; + case MEM_AREA_TA_RAM: + if (!pbuf_is_inside(secure_only, m->pa, m->size)) + panic("TA_RAM can't fit in secure_only"); + break; + case MEM_AREA_NSEC_SHM: + if (!pbuf_is_inside(nsec_shared, m->pa, m->size)) + panic("NS_SHM can't fit in nsec_shared"); + break; + case MEM_AREA_SEC_RAM_OVERALL: + case MEM_AREA_TEE_COHERENT: + case MEM_AREA_TEE_ASAN: + case MEM_AREA_IO_SEC: + case MEM_AREA_IO_NSEC: + case MEM_AREA_EXT_DT: + case MEM_AREA_MANIFEST_DT: + case MEM_AREA_RAM_SEC: + case MEM_AREA_RAM_NSEC: + case MEM_AREA_RES_VASPACE: + case MEM_AREA_SHM_VASPACE: + case MEM_AREA_PAGER_VASPACE: + break; + default: + EMSG("Uhandled memtype %d", m->type); + panic(); + } + } +} + +static struct tee_mmap_region *get_tmp_mmap(void) +{ + struct tee_mmap_region *tmp_mmap = (void *)__heap1_start; + +#ifdef CFG_WITH_PAGER + if (__heap1_end - __heap1_start < (ptrdiff_t)sizeof(static_memory_map)) + tmp_mmap = (void *)__heap2_start; +#endif + + memset(tmp_mmap, 0, sizeof(static_memory_map)); + + return tmp_mmap; +} + +/* + * core_init_mmu_map() - init tee core default memory mapping + * + * This routine sets the static default TEE core mapping. If @seed is > 0 + * and configured with CFG_CORE_ASLR it will map tee core at a location + * based on the seed and return the offset from the link address. + * + * If an error happened: core_init_mmu_map is expected to panic. + * + * Note: this function is weak just to make it possible to exclude it from + * the unpaged area. + */ +void __weak core_init_mmu_map(unsigned long seed, struct core_mmu_config *cfg) +{ +#ifndef CFG_NS_VIRTUALIZATION + vaddr_t start = ROUNDDOWN((vaddr_t)__nozi_start, SMALL_PAGE_SIZE); +#else + vaddr_t start = ROUNDDOWN((vaddr_t)__vcore_nex_rw_start, + SMALL_PAGE_SIZE); +#endif + vaddr_t len = ROUNDUP((vaddr_t)__nozi_end, SMALL_PAGE_SIZE) - start; + struct tee_mmap_region *tmp_mmap = get_tmp_mmap(); + unsigned long offs = 0; + + if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) && + (core_mmu_tee_load_pa & SMALL_PAGE_MASK)) + panic("OP-TEE load address is not page aligned"); + + check_sec_nsec_mem_config(); + + /* + * Add a entry covering the translation tables which will be + * involved in some virt_to_phys() and phys_to_virt() conversions. + */ + static_memory_map[0] = (struct tee_mmap_region){ + .type = MEM_AREA_TEE_RAM, + .region_size = SMALL_PAGE_SIZE, + .pa = start, + .va = start, + .size = len, + .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), + }; + + COMPILE_TIME_ASSERT(CFG_MMAP_REGIONS >= 13); + offs = init_mem_map(tmp_mmap, ARRAY_SIZE(static_memory_map), seed); + + check_mem_map(tmp_mmap); + core_init_mmu(tmp_mmap); + dump_xlat_table(0x0, CORE_MMU_BASE_TABLE_LEVEL); + core_init_mmu_regs(cfg); + cfg->map_offset = offs; + memcpy(static_memory_map, tmp_mmap, sizeof(static_memory_map)); +} + +bool core_mmu_mattr_is_ok(uint32_t mattr) +{ + /* + * Keep in sync with core_mmu_lpae.c:mattr_to_desc and + * core_mmu_v7.c:mattr_to_texcb + */ + + switch ((mattr >> TEE_MATTR_MEM_TYPE_SHIFT) & TEE_MATTR_MEM_TYPE_MASK) { + case TEE_MATTR_MEM_TYPE_DEV: + case TEE_MATTR_MEM_TYPE_STRONGLY_O: + case TEE_MATTR_MEM_TYPE_CACHED: + case TEE_MATTR_MEM_TYPE_TAGGED: + return true; + default: + return false; + } +} + +/* + * test attributes of target physical buffer + * + * Flags: pbuf_is(SECURE, NOT_SECURE, RAM, IOMEM, KEYVAULT). + * + */ +bool core_pbuf_is(uint32_t attr, paddr_t pbuf, size_t len) +{ + paddr_t ta_base = 0; + size_t ta_size = 0; + struct tee_mmap_region *map; + + /* Empty buffers complies with anything */ + if (len == 0) + return true; + + switch (attr) { + case CORE_MEM_SEC: + return pbuf_is_inside(secure_only, pbuf, len); + case CORE_MEM_NON_SEC: + return pbuf_is_inside(nsec_shared, pbuf, len) || + pbuf_is_nsec_ddr(pbuf, len); + case CORE_MEM_TEE_RAM: + return core_is_buffer_inside(pbuf, len, TEE_RAM_START, + TEE_RAM_PH_SIZE); + case CORE_MEM_TA_RAM: + core_mmu_get_ta_range(&ta_base, &ta_size); + return core_is_buffer_inside(pbuf, len, ta_base, ta_size); +#ifdef CFG_CORE_RESERVED_SHM + case CORE_MEM_NSEC_SHM: + return core_is_buffer_inside(pbuf, len, TEE_SHMEM_START, + TEE_SHMEM_SIZE); +#endif + case CORE_MEM_SDP_MEM: + return pbuf_is_sdp_mem(pbuf, len); + case CORE_MEM_CACHED: + map = find_map_by_pa(pbuf); + if (!map || !pbuf_inside_map_area(pbuf, len, map)) + return false; + return mattr_is_cached(map->attr); + default: + return false; + } +} + +/* test attributes of target virtual buffer (in core mapping) */ +bool core_vbuf_is(uint32_t attr, const void *vbuf, size_t len) +{ + paddr_t p; + + /* Empty buffers complies with anything */ + if (len == 0) + return true; + + p = virt_to_phys((void *)vbuf); + if (!p) + return false; + + return core_pbuf_is(attr, p, len); +} + +/* core_va2pa - teecore exported service */ +static int __maybe_unused core_va2pa_helper(void *va, paddr_t *pa) +{ + struct tee_mmap_region *map; + + map = find_map_by_va(va); + if (!va_is_in_map(map, (vaddr_t)va)) + return -1; + + /* + * We can calculate PA for static map. Virtual address ranges + * reserved to core dynamic mapping return a 'match' (return 0;) + * together with an invalid null physical address. + */ + if (map->pa) + *pa = map->pa + (vaddr_t)va - map->va; + else + *pa = 0; + + return 0; +} + +static void *map_pa2va(struct tee_mmap_region *map, paddr_t pa, size_t len) +{ + if (!pa_is_in_map(map, pa, len)) + return NULL; + + return (void *)(vaddr_t)(map->va + pa - map->pa); +} + +/* + * teecore gets some memory area definitions + */ +void core_mmu_get_mem_by_type(enum teecore_memtypes type, vaddr_t *s, + vaddr_t *e) +{ + struct tee_mmap_region *map = find_map_by_type(type); + + if (map) { + *s = map->va; + *e = map->va + map->size; + } else { + *s = 0; + *e = 0; + } +} + +enum teecore_memtypes core_mmu_get_type_by_pa(paddr_t pa) +{ + struct tee_mmap_region *map = find_map_by_pa(pa); + + if (!map) + return MEM_AREA_MAXTYPE; + return map->type; +} + +void core_mmu_set_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, + paddr_t pa, uint32_t attr) +{ + assert(idx < tbl_info->num_entries); + core_mmu_set_entry_primitive(tbl_info->table, tbl_info->level, + idx, pa, attr); +} + +void core_mmu_get_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, + paddr_t *pa, uint32_t *attr) +{ + assert(idx < tbl_info->num_entries); + core_mmu_get_entry_primitive(tbl_info->table, tbl_info->level, + idx, pa, attr); +} + +static void clear_region(struct core_mmu_table_info *tbl_info, + struct tee_mmap_region *region) +{ + unsigned int end = 0; + unsigned int idx = 0; + + /* va, len and pa should be block aligned */ + assert(!core_mmu_get_block_offset(tbl_info, region->va)); + assert(!core_mmu_get_block_offset(tbl_info, region->size)); + assert(!core_mmu_get_block_offset(tbl_info, region->pa)); + + idx = core_mmu_va2idx(tbl_info, region->va); + end = core_mmu_va2idx(tbl_info, region->va + region->size); + + while (idx < end) { + core_mmu_set_entry(tbl_info, idx, 0, 0); + idx++; + } +} + +static void set_region(struct core_mmu_table_info *tbl_info, + struct tee_mmap_region *region) +{ + unsigned int end; + unsigned int idx; + paddr_t pa; + + /* va, len and pa should be block aligned */ + assert(!core_mmu_get_block_offset(tbl_info, region->va)); + assert(!core_mmu_get_block_offset(tbl_info, region->size)); + assert(!core_mmu_get_block_offset(tbl_info, region->pa)); + + idx = core_mmu_va2idx(tbl_info, region->va); + end = core_mmu_va2idx(tbl_info, region->va + region->size); + pa = region->pa; + + while (idx < end) { + core_mmu_set_entry(tbl_info, idx, pa, region->attr); + idx++; + pa += BIT64(tbl_info->shift); + } +} + +static void set_pg_region(struct core_mmu_table_info *dir_info, + struct vm_region *region, struct pgt **pgt, + struct core_mmu_table_info *pg_info) +{ + struct tee_mmap_region r = { + .va = region->va, + .size = region->size, + .attr = region->attr, + }; + vaddr_t end = r.va + r.size; + uint32_t pgt_attr = (r.attr & TEE_MATTR_SECURE) | TEE_MATTR_TABLE; + + while (r.va < end) { + if (!pg_info->table || + r.va >= (pg_info->va_base + CORE_MMU_PGDIR_SIZE)) { + /* + * We're assigning a new translation table. + */ + unsigned int idx; + + /* Virtual addresses must grow */ + assert(r.va > pg_info->va_base); + + idx = core_mmu_va2idx(dir_info, r.va); + pg_info->va_base = core_mmu_idx2va(dir_info, idx); + + /* + * Advance pgt to va_base, note that we may need to + * skip multiple page tables if there are large + * holes in the vm map. + */ + while ((*pgt)->vabase < pg_info->va_base) { + *pgt = SLIST_NEXT(*pgt, link); + /* We should have allocated enough */ + assert(*pgt); + } + assert((*pgt)->vabase == pg_info->va_base); + pg_info->table = (*pgt)->tbl; + + core_mmu_set_entry(dir_info, idx, + virt_to_phys(pg_info->table), + pgt_attr); + } + + r.size = MIN(CORE_MMU_PGDIR_SIZE - (r.va - pg_info->va_base), + end - r.va); + + if (!(*pgt)->populated && !mobj_is_paged(region->mobj)) { + size_t granule = BIT(pg_info->shift); + size_t offset = r.va - region->va + region->offset; + + r.size = MIN(r.size, + mobj_get_phys_granule(region->mobj)); + r.size = ROUNDUP(r.size, SMALL_PAGE_SIZE); + + if (mobj_get_pa(region->mobj, offset, granule, + &r.pa) != TEE_SUCCESS) + panic("Failed to get PA of unpaged mobj"); + set_region(pg_info, &r); + } + r.va += r.size; + } +} + +static bool can_map_at_level(paddr_t paddr, vaddr_t vaddr, + size_t size_left, paddr_t block_size, + struct tee_mmap_region *mm __maybe_unused) +{ + /* VA and PA are aligned to block size at current level */ + if ((vaddr | paddr) & (block_size - 1)) + return false; + + /* Remainder fits into block at current level */ + if (size_left < block_size) + return false; + +#ifdef CFG_WITH_PAGER + /* + * If pager is enabled, we need to map tee ram + * regions with small pages only + */ + if (map_is_tee_ram(mm) && block_size != SMALL_PAGE_SIZE) + return false; +#endif + + return true; +} + +void core_mmu_map_region(struct mmu_partition *prtn, struct tee_mmap_region *mm) +{ + struct core_mmu_table_info tbl_info; + unsigned int idx; + vaddr_t vaddr = mm->va; + paddr_t paddr = mm->pa; + ssize_t size_left = mm->size; + unsigned int level; + bool table_found; + uint32_t old_attr; + + assert(!((vaddr | paddr) & SMALL_PAGE_MASK)); + + while (size_left > 0) { + level = CORE_MMU_BASE_TABLE_LEVEL; + + while (true) { + paddr_t block_size = 0; + + assert(core_mmu_level_in_range(level)); + + table_found = core_mmu_find_table(prtn, vaddr, level, + &tbl_info); + if (!table_found) + panic("can't find table for mapping"); + + block_size = BIT64(tbl_info.shift); + + idx = core_mmu_va2idx(&tbl_info, vaddr); + if (!can_map_at_level(paddr, vaddr, size_left, + block_size, mm)) { + bool secure = mm->attr & TEE_MATTR_SECURE; + + /* + * This part of the region can't be mapped at + * this level. Need to go deeper. + */ + if (!core_mmu_entry_to_finer_grained(&tbl_info, + idx, + secure)) + panic("Can't divide MMU entry"); + level = tbl_info.next_level; + continue; + } + + /* We can map part of the region at current level */ + core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); + if (old_attr) + panic("Page is already mapped"); + + core_mmu_set_entry(&tbl_info, idx, paddr, mm->attr); + paddr += block_size; + vaddr += block_size; + size_left -= block_size; + + break; + } + } +} + +TEE_Result core_mmu_map_pages(vaddr_t vstart, paddr_t *pages, size_t num_pages, + enum teecore_memtypes memtype) +{ + TEE_Result ret; + struct core_mmu_table_info tbl_info; + struct tee_mmap_region *mm; + unsigned int idx; + uint32_t old_attr; + uint32_t exceptions; + vaddr_t vaddr = vstart; + size_t i; + bool secure; + + assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); + + secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; + + if (vaddr & SMALL_PAGE_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + exceptions = mmu_lock(); + + mm = find_map_by_va((void *)vaddr); + if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) + panic("VA does not belong to any known mm region"); + + if (!core_mmu_is_dynamic_vaspace(mm)) + panic("Trying to map into static region"); + + for (i = 0; i < num_pages; i++) { + if (pages[i] & SMALL_PAGE_MASK) { + ret = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + while (true) { + if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, + &tbl_info)) + panic("Can't find pagetable for vaddr "); + + idx = core_mmu_va2idx(&tbl_info, vaddr); + if (tbl_info.shift == SMALL_PAGE_SHIFT) + break; + + /* This is supertable. Need to divide it. */ + if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, + secure)) + panic("Failed to spread pgdir on small tables"); + } + + core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); + if (old_attr) + panic("Page is already mapped"); + + core_mmu_set_entry(&tbl_info, idx, pages[i], + core_mmu_type_to_attr(memtype)); + vaddr += SMALL_PAGE_SIZE; + } + + /* + * Make sure all the changes to translation tables are visible + * before returning. TLB doesn't need to be invalidated as we are + * guaranteed that there's no valid mapping in this range. + */ + core_mmu_table_write_barrier(); + mmu_unlock(exceptions); + + return TEE_SUCCESS; +err: + mmu_unlock(exceptions); + + if (i) + core_mmu_unmap_pages(vstart, i); + + return ret; +} + +TEE_Result core_mmu_map_contiguous_pages(vaddr_t vstart, paddr_t pstart, + size_t num_pages, + enum teecore_memtypes memtype) +{ + struct core_mmu_table_info tbl_info = { }; + struct tee_mmap_region *mm = NULL; + unsigned int idx = 0; + uint32_t old_attr = 0; + uint32_t exceptions = 0; + vaddr_t vaddr = vstart; + paddr_t paddr = pstart; + size_t i = 0; + bool secure = false; + + assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); + + secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; + + if ((vaddr | paddr) & SMALL_PAGE_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + exceptions = mmu_lock(); + + mm = find_map_by_va((void *)vaddr); + if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) + panic("VA does not belong to any known mm region"); + + if (!core_mmu_is_dynamic_vaspace(mm)) + panic("Trying to map into static region"); + + for (i = 0; i < num_pages; i++) { + while (true) { + if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, + &tbl_info)) + panic("Can't find pagetable for vaddr "); + + idx = core_mmu_va2idx(&tbl_info, vaddr); + if (tbl_info.shift == SMALL_PAGE_SHIFT) + break; + + /* This is supertable. Need to divide it. */ + if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, + secure)) + panic("Failed to spread pgdir on small tables"); + } + + core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); + if (old_attr) + panic("Page is already mapped"); + + core_mmu_set_entry(&tbl_info, idx, paddr, + core_mmu_type_to_attr(memtype)); + paddr += SMALL_PAGE_SIZE; + vaddr += SMALL_PAGE_SIZE; + } + + /* + * Make sure all the changes to translation tables are visible + * before returning. TLB doesn't need to be invalidated as we are + * guaranteed that there's no valid mapping in this range. + */ + core_mmu_table_write_barrier(); + mmu_unlock(exceptions); + + return TEE_SUCCESS; +} + +void core_mmu_unmap_pages(vaddr_t vstart, size_t num_pages) +{ + struct core_mmu_table_info tbl_info; + struct tee_mmap_region *mm; + size_t i; + unsigned int idx; + uint32_t exceptions; + + exceptions = mmu_lock(); + + mm = find_map_by_va((void *)vstart); + if (!mm || !va_is_in_map(mm, vstart + num_pages * SMALL_PAGE_SIZE - 1)) + panic("VA does not belong to any known mm region"); + + if (!core_mmu_is_dynamic_vaspace(mm)) + panic("Trying to unmap static region"); + + for (i = 0; i < num_pages; i++, vstart += SMALL_PAGE_SIZE) { + if (!core_mmu_find_table(NULL, vstart, UINT_MAX, &tbl_info)) + panic("Can't find pagetable"); + + if (tbl_info.shift != SMALL_PAGE_SHIFT) + panic("Invalid pagetable level"); + + idx = core_mmu_va2idx(&tbl_info, vstart); + core_mmu_set_entry(&tbl_info, idx, 0, 0); + } + tlbi_all(); + + mmu_unlock(exceptions); +} + +void core_mmu_populate_user_map(struct core_mmu_table_info *dir_info, + struct user_mode_ctx *uctx) +{ + struct core_mmu_table_info pg_info = { }; + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct pgt *pgt = NULL; + struct pgt *p = NULL; + struct vm_region *r = NULL; + + if (TAILQ_EMPTY(&uctx->vm_info.regions)) + return; /* Nothing to map */ + + /* + * Allocate all page tables in advance. + */ + pgt_get_all(uctx); + pgt = SLIST_FIRST(pgt_cache); + + core_mmu_set_info_table(&pg_info, dir_info->next_level, 0, NULL); + + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) + set_pg_region(dir_info, r, &pgt, &pg_info); + /* Record that the translation tables now are populated. */ + SLIST_FOREACH(p, pgt_cache, link) { + p->populated = true; + if (p == pgt) + break; + } + assert(p == pgt); +} + +TEE_Result core_mmu_remove_mapping(enum teecore_memtypes type, void *addr, + size_t len) +{ + struct core_mmu_table_info tbl_info = { }; + struct tee_mmap_region *res_map = NULL; + struct tee_mmap_region *map = NULL; + paddr_t pa = virt_to_phys(addr); + size_t granule = 0; + ptrdiff_t i = 0; + paddr_t p = 0; + size_t l = 0; + + map = find_map_by_type_and_pa(type, pa, len); + if (!map) + return TEE_ERROR_GENERIC; + + res_map = find_map_by_type(MEM_AREA_RES_VASPACE); + if (!res_map) + return TEE_ERROR_GENERIC; + if (!core_mmu_find_table(NULL, res_map->va, UINT_MAX, &tbl_info)) + return TEE_ERROR_GENERIC; + granule = BIT(tbl_info.shift); + + if (map < static_memory_map || + map >= static_memory_map + ARRAY_SIZE(static_memory_map)) + return TEE_ERROR_GENERIC; + i = map - static_memory_map; + + /* Check that we have a full match */ + p = ROUNDDOWN(pa, granule); + l = ROUNDUP(len + pa - p, granule); + if (map->pa != p || map->size != l) + return TEE_ERROR_GENERIC; + + clear_region(&tbl_info, map); + tlbi_all(); + + /* If possible remove the va range from res_map */ + if (res_map->va - map->size == map->va) { + res_map->va -= map->size; + res_map->size += map->size; + } + + /* Remove the entry. */ + memmove(map, map + 1, + (ARRAY_SIZE(static_memory_map) - i - 1) * sizeof(*map)); + + /* Clear the last new entry in case it was used */ + memset(static_memory_map + ARRAY_SIZE(static_memory_map) - 1, + 0, sizeof(*map)); + + return TEE_SUCCESS; +} + +struct tee_mmap_region * +core_mmu_find_mapping_exclusive(enum teecore_memtypes type, size_t len) +{ + struct tee_mmap_region *map = NULL; + struct tee_mmap_region *map_found = NULL; + + if (!len) + return NULL; + + for (map = get_memory_map(); !core_mmap_is_end_of_table(map); map++) { + if (map->type != type) + continue; + + if (map_found) + return NULL; + + map_found = map; + } + + if (!map_found || map_found->size < len) + return NULL; + + return map_found; +} + +void *core_mmu_add_mapping(enum teecore_memtypes type, paddr_t addr, size_t len) +{ + struct core_mmu_table_info tbl_info; + struct tee_mmap_region *map; + size_t n; + size_t granule; + paddr_t p; + size_t l; + + if (!len) + return NULL; + + if (!core_mmu_check_end_pa(addr, len)) + return NULL; + + /* Check if the memory is already mapped */ + map = find_map_by_type_and_pa(type, addr, len); + if (map && pbuf_inside_map_area(addr, len, map)) + return (void *)(vaddr_t)(map->va + addr - map->pa); + + /* Find the reserved va space used for late mappings */ + map = find_map_by_type(MEM_AREA_RES_VASPACE); + if (!map) + return NULL; + + if (!core_mmu_find_table(NULL, map->va, UINT_MAX, &tbl_info)) + return NULL; + + granule = BIT64(tbl_info.shift); + p = ROUNDDOWN(addr, granule); + l = ROUNDUP(len + addr - p, granule); + + /* Ban overflowing virtual addresses */ + if (map->size < l) + return NULL; + + /* + * Something is wrong, we can't fit the va range into the selected + * table. The reserved va range is possibly missaligned with + * granule. + */ + if (core_mmu_va2idx(&tbl_info, map->va + len) >= tbl_info.num_entries) + return NULL; + + /* Find end of the memory map */ + n = 0; + while (!core_mmap_is_end_of_table(static_memory_map + n)) + n++; + + if (n < (ARRAY_SIZE(static_memory_map) - 1)) { + /* There's room for another entry */ + static_memory_map[n].va = map->va; + static_memory_map[n].size = l; + static_memory_map[n + 1].type = MEM_AREA_END; + map->va += l; + map->size -= l; + map = static_memory_map + n; + } else { + /* + * There isn't room for another entry, steal the reserved + * entry as it's not useful for anything else any longer. + */ + map->size = l; + } + map->type = type; + map->region_size = granule; + map->attr = core_mmu_type_to_attr(type); + map->pa = p; + + set_region(&tbl_info, map); + + /* Make sure the new entry is visible before continuing. */ + core_mmu_table_write_barrier(); + + return (void *)(vaddr_t)(map->va + addr - map->pa); +} + +#ifdef CFG_WITH_PAGER +static vaddr_t get_linear_map_end_va(void) +{ + /* this is synced with the generic linker file kern.ld.S */ + return (vaddr_t)__heap2_end; +} + +static paddr_t get_linear_map_end_pa(void) +{ + return get_linear_map_end_va() - boot_mmu_config.map_offset; +} +#endif + +#if defined(CFG_TEE_CORE_DEBUG) +static void check_pa_matches_va(void *va, paddr_t pa) +{ + TEE_Result res = TEE_ERROR_GENERIC; + vaddr_t v = (vaddr_t)va; + paddr_t p = 0; + struct core_mmu_table_info ti __maybe_unused = { }; + + if (core_mmu_user_va_range_is_defined()) { + vaddr_t user_va_base = 0; + size_t user_va_size = 0; + + core_mmu_get_user_va_range(&user_va_base, &user_va_size); + if (v >= user_va_base && + v <= (user_va_base - 1 + user_va_size)) { + if (!core_mmu_user_mapping_is_active()) { + if (pa) + panic("issue in linear address space"); + return; + } + + res = vm_va2pa(to_user_mode_ctx(thread_get_tsd()->ctx), + va, &p); + if (res == TEE_ERROR_NOT_SUPPORTED) + return; + if (res == TEE_SUCCESS && pa != p) + panic("bad pa"); + if (res != TEE_SUCCESS && pa) + panic("false pa"); + return; + } + } +#ifdef CFG_WITH_PAGER + if (is_unpaged(va)) { + if (v - boot_mmu_config.map_offset != pa) + panic("issue in linear address space"); + return; + } + + if (tee_pager_get_table_info(v, &ti)) { + uint32_t a; + + /* + * Lookups in the page table managed by the pager is + * dangerous for addresses in the paged area as those pages + * changes all the time. But some ranges are safe, + * rw-locked areas when the page is populated for instance. + */ + core_mmu_get_entry(&ti, core_mmu_va2idx(&ti, v), &p, &a); + if (a & TEE_MATTR_VALID_BLOCK) { + paddr_t mask = BIT64(ti.shift) - 1; + + p |= v & mask; + if (pa != p) + panic(); + } else { + if (pa) + panic(); + } + return; + } +#endif + + if (!core_va2pa_helper(va, &p)) { + /* Verfiy only the static mapping (case non null phys addr) */ + if (p && pa != p) { + DMSG("va %p maps 0x%" PRIxPA ", expect 0x%" PRIxPA, + va, p, pa); + panic(); + } + } else { + if (pa) { + DMSG("va %p unmapped, expect 0x%" PRIxPA, va, pa); + panic(); + } + } +} +#else +static void check_pa_matches_va(void *va __unused, paddr_t pa __unused) +{ +} +#endif + +paddr_t virt_to_phys(void *va) +{ + paddr_t pa = 0; + + if (!arch_va2pa_helper(va, &pa)) + pa = 0; + check_pa_matches_va(va, pa); + return pa; +} + +#if defined(CFG_TEE_CORE_DEBUG) +static void check_va_matches_pa(paddr_t pa, void *va) +{ + paddr_t p = 0; + + if (!va) + return; + + p = virt_to_phys(va); + if (p != pa) { + DMSG("va %p maps 0x%" PRIxPA " expect 0x%" PRIxPA, va, p, pa); + panic(); + } +} +#else +static void check_va_matches_pa(paddr_t pa __unused, void *va __unused) +{ +} +#endif + +static void *phys_to_virt_ts_vaspace(paddr_t pa, size_t len) +{ + if (!core_mmu_user_mapping_is_active()) + return NULL; + + return vm_pa2va(to_user_mode_ctx(thread_get_tsd()->ctx), pa, len); +} + +#ifdef CFG_WITH_PAGER +static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) +{ + paddr_t end_pa = 0; + + if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) + return NULL; + + if (pa >= TEE_LOAD_ADDR && pa < get_linear_map_end_pa()) { + if (end_pa > get_linear_map_end_pa()) + return NULL; + return (void *)(vaddr_t)(pa + boot_mmu_config.map_offset); + } + + return tee_pager_phys_to_virt(pa, len); +} +#else +static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) +{ + struct tee_mmap_region *mmap = NULL; + + mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM, pa, len); + if (!mmap) + mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RW, pa, len); + if (!mmap) + mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RO, pa, len); + if (!mmap) + mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RW, pa, len); + if (!mmap) + mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RO, pa, len); + if (!mmap) + mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RX, pa, len); + /* + * Note that MEM_AREA_INIT_RAM_RO and MEM_AREA_INIT_RAM_RX are only + * used with pager and not needed here. + */ + return map_pa2va(mmap, pa, len); +} +#endif + +void *phys_to_virt(paddr_t pa, enum teecore_memtypes m, size_t len) +{ + void *va = NULL; + + switch (m) { + case MEM_AREA_TS_VASPACE: + va = phys_to_virt_ts_vaspace(pa, len); + break; + case MEM_AREA_TEE_RAM: + case MEM_AREA_TEE_RAM_RX: + case MEM_AREA_TEE_RAM_RO: + case MEM_AREA_TEE_RAM_RW: + case MEM_AREA_NEX_RAM_RO: + case MEM_AREA_NEX_RAM_RW: + va = phys_to_virt_tee_ram(pa, len); + break; + case MEM_AREA_SHM_VASPACE: + /* Find VA from PA in dynamic SHM is not yet supported */ + va = NULL; + break; + default: + va = map_pa2va(find_map_by_type_and_pa(m, pa, len), pa, len); + } + if (m != MEM_AREA_SEC_RAM_OVERALL) + check_va_matches_pa(pa, va); + return va; +} + +void *phys_to_virt_io(paddr_t pa, size_t len) +{ + struct tee_mmap_region *map = NULL; + void *va = NULL; + + map = find_map_by_type_and_pa(MEM_AREA_IO_SEC, pa, len); + if (!map) + map = find_map_by_type_and_pa(MEM_AREA_IO_NSEC, pa, len); + if (!map) + return NULL; + va = map_pa2va(map, pa, len); + check_va_matches_pa(pa, va); + return va; +} + +vaddr_t core_mmu_get_va(paddr_t pa, enum teecore_memtypes type, size_t len) +{ + if (cpu_mmu_enabled()) + return (vaddr_t)phys_to_virt(pa, type, len); + + return (vaddr_t)pa; +} + +#ifdef CFG_WITH_PAGER +bool is_unpaged(void *va) +{ + vaddr_t v = (vaddr_t)va; + + return v >= VCORE_START_VA && v < get_linear_map_end_va(); +} +#else +bool is_unpaged(void *va __unused) +{ + return true; +} +#endif + +void core_mmu_init_virtualization(void) +{ + paddr_t b1 = 0; + paddr_size_t s1 = 0; + + static_assert(ARRAY_SIZE(secure_only) <= 2); + if (ARRAY_SIZE(secure_only) == 2) { + b1 = secure_only[1].paddr; + s1 = secure_only[1].size; + } + virt_init_memory(static_memory_map, secure_only[0].paddr, + secure_only[0].size, b1, s1); +} + +vaddr_t io_pa_or_va(struct io_pa_va *p, size_t len) +{ + assert(p->pa); + if (cpu_mmu_enabled()) { + if (!p->va) + p->va = (vaddr_t)phys_to_virt_io(p->pa, len); + assert(p->va); + return p->va; + } + return p->pa; +} + +vaddr_t io_pa_or_va_secure(struct io_pa_va *p, size_t len) +{ + assert(p->pa); + if (cpu_mmu_enabled()) { + if (!p->va) + p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_SEC, + len); + assert(p->va); + return p->va; + } + return p->pa; +} + +vaddr_t io_pa_or_va_nsec(struct io_pa_va *p, size_t len) +{ + assert(p->pa); + if (cpu_mmu_enabled()) { + if (!p->va) + p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_NSEC, + len); + assert(p->va); + return p->va; + } + return p->pa; +} + +#ifdef CFG_CORE_RESERVED_SHM +static TEE_Result teecore_init_pub_ram(void) +{ + vaddr_t s = 0; + vaddr_t e = 0; + + /* get virtual addr/size of NSec shared mem allocated from teecore */ + core_mmu_get_mem_by_type(MEM_AREA_NSEC_SHM, &s, &e); + + if (s >= e || s & SMALL_PAGE_MASK || e & SMALL_PAGE_MASK) + panic("invalid PUB RAM"); + + /* extra check: we could rely on core_mmu_get_mem_by_type() */ + if (!tee_vbuf_is_non_sec(s, e - s)) + panic("PUB RAM is not non-secure"); + +#ifdef CFG_PL310 + /* Allocate statically the l2cc mutex */ + tee_l2cc_store_mutex_boot_pa(virt_to_phys((void *)s)); + s += sizeof(uint32_t); /* size of a pl310 mutex */ + s = ROUNDUP(s, SMALL_PAGE_SIZE); /* keep required alignment */ +#endif + + default_nsec_shm_paddr = virt_to_phys((void *)s); + default_nsec_shm_size = e - s; + + return TEE_SUCCESS; +} +early_init(teecore_init_pub_ram); +#endif /*CFG_CORE_RESERVED_SHM*/ + +void core_mmu_init_ta_ram(void) +{ + vaddr_t s = 0; + vaddr_t e = 0; + paddr_t ps = 0; + size_t size = 0; + + /* + * Get virtual addr/size of RAM where TA are loaded/executedNSec + * shared mem allocated from teecore. + */ + if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) + virt_get_ta_ram(&s, &e); + else + core_mmu_get_mem_by_type(MEM_AREA_TA_RAM, &s, &e); + + ps = virt_to_phys((void *)s); + size = e - s; + + if (!ps || (ps & CORE_MMU_USER_CODE_MASK) || + !size || (size & CORE_MMU_USER_CODE_MASK)) + panic("invalid TA RAM"); + + /* extra check: we could rely on core_mmu_get_mem_by_type() */ + if (!tee_pbuf_is_sec(ps, size)) + panic("TA RAM is not secure"); + + if (!tee_mm_is_empty(&tee_mm_sec_ddr)) + panic("TA RAM pool is not empty"); + + /* remove previous config and init TA ddr memory pool */ + tee_mm_final(&tee_mm_sec_ddr); + tee_mm_init(&tee_mm_sec_ddr, ps, size, CORE_MMU_USER_CODE_SHIFT, + TEE_MM_POOL_NO_FLAGS); +} diff --git a/optee_os/core/mm/file.c b/optee_os/core/mm/file.c new file mode 100644 index 0000000..1f6a987 --- /dev/null +++ b/optee_os/core/mm/file.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct file_slice_elem { + struct file_slice slice; + SLIST_ENTRY(file_slice_elem) link; +}; + +/* + * struct file - file resources + * @tag: Tag or hash uniquely identifying a file + * @taglen: Byte length of @tag + * @refc: Reference counter + * @link: Linked list element + * @num_slices: Number of elements in the @slices array below + * @slices: Array of file slices holding the fobjs of this file + * + * A file is constructed of slices which may be shared in different + * mappings/contexts. There may be holes in the file for ranges of the file + * that can't be shared. + */ +struct file { + uint8_t tag[FILE_TAG_SIZE]; + unsigned int taglen; + struct refcount refc; + TAILQ_ENTRY(file) link; + struct mutex mu; + SLIST_HEAD(, file_slice_elem) slice_head; +}; + +static struct mutex file_mu = MUTEX_INITIALIZER; +static TAILQ_HEAD(, file) file_head = TAILQ_HEAD_INITIALIZER(file_head); + +static int file_tag_cmp(const struct file *f, const uint8_t *tag, + unsigned int taglen) +{ + if (f->taglen != taglen) + return -1; + return memcmp(tag, f->tag, taglen); +} + +static struct file *file_find_tag_unlocked(const uint8_t *tag, + unsigned int taglen) +{ + struct file *f = NULL; + + TAILQ_FOREACH(f, &file_head, link) + if (!file_tag_cmp(f, tag, taglen)) + return f; + + return NULL; +} + +static void file_free(struct file *f) +{ + mutex_destroy(&f->mu); + + while (!SLIST_EMPTY(&f->slice_head)) { + struct file_slice_elem *fse = SLIST_FIRST(&f->slice_head); + + SLIST_REMOVE_HEAD(&f->slice_head, link); + fobj_put(fse->slice.fobj); + free(fse); + } + + free(f); +} + +TEE_Result file_add_slice(struct file *f, struct fobj *fobj, + unsigned int page_offset) +{ + struct file_slice_elem *fse = NULL; + unsigned int s = 0; + + /* Check for conflicts */ + if (file_find_slice(f, page_offset)) + return TEE_ERROR_BAD_PARAMETERS; + + fse = calloc(1, sizeof(*fse)); + if (!fse) + return TEE_ERROR_OUT_OF_MEMORY; + + fse->slice.fobj = fobj_get(fobj); + if (!fse->slice.fobj || + ADD_OVERFLOW(page_offset, fse->slice.fobj->num_pages, &s)) { + fobj_put(fse->slice.fobj); + free(fse); + return TEE_ERROR_BAD_PARAMETERS; + } + + fse->slice.page_offset = page_offset; + SLIST_INSERT_HEAD(&f->slice_head, fse, link); + + return TEE_SUCCESS; +} + +struct file *file_get(struct file *f) +{ + if (f && !refcount_inc(&f->refc)) + panic(); + + return f; +} + +struct file *file_get_by_tag(const uint8_t *tag, unsigned int taglen) +{ + struct file *f = NULL; + + if (taglen > sizeof(f->tag)) + return NULL; + + mutex_lock(&file_mu); + + /* + * If file is found and reference counter can be increased, we're done. + * If file can't be found, it doesn't exist so it has to be added. + * If it's found but reference counter is 0, the situation is + * a bit complicated: + * - file_put() is about to free the file as soon as it can obtain the + * mutex. + * - Unless there's a mismatch between file_get() and file_put() only + * one thread calling file_put() is about to free the file. + * + * There's a window of opportunity where file_put() is called + * (without a mutex being held, which is quite OK) while we're + * holding the mutex here and are searching for the file and it's + * found, but just after file_put() has decreased the reference + * counter. + * + * To keep it simple we're adding a new file at the head (so new + * searches finds this file instead of the old being freed) instead + * of complicating file_put() by trying to rescue the file and + * possibly hiding a case of mismatching file_put() and file_get(). + */ + f = file_find_tag_unlocked(tag, taglen); + if (f && refcount_inc(&f->refc)) + goto out; + + f = calloc(1, sizeof(*f)); + if (!f) + goto out; + memcpy(f->tag, tag, taglen); + f->taglen = taglen; + refcount_set(&f->refc, 1); + mutex_init(&f->mu); + SLIST_INIT(&f->slice_head); + TAILQ_INSERT_HEAD(&file_head, f, link); + +out: + mutex_unlock(&file_mu); + + return f; +} + +void file_put(struct file *f) +{ + if (f && refcount_dec(&f->refc)) { + mutex_lock(&file_mu); + TAILQ_REMOVE(&file_head, f, link); + mutex_unlock(&file_mu); + + file_free(f); + } + +} + +struct file_slice *file_find_slice(struct file *f, unsigned int page_offset) +{ + struct file_slice_elem *fse = NULL; + + assert(f->mu.state); + + SLIST_FOREACH(fse, &f->slice_head, link) { + struct file_slice *fs = &fse->slice; + + if (page_offset >= fs->page_offset && + page_offset < fs->page_offset + fs->fobj->num_pages) + return fs; + } + + return NULL; +} + +void file_lock(struct file *f) +{ + mutex_lock(&f->mu); +} + +bool file_trylock(struct file *f) +{ + return mutex_trylock(&f->mu); +} + +void file_unlock(struct file *f) +{ + mutex_unlock(&f->mu); +} diff --git a/optee_os/core/mm/fobj.c b/optee_os/core/mm/fobj.c new file mode 100644 index 0000000..e1397cc --- /dev/null +++ b/optee_os/core/mm/fobj.c @@ -0,0 +1,818 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_WITH_PAGER + +#define RWP_AE_KEY_BITS 256 + +struct rwp_aes_gcm_iv { + uint32_t iv[3]; +}; + +#define RWP_AES_GCM_TAG_LEN 16 + +struct rwp_state { + uint64_t iv; + uint8_t tag[RWP_AES_GCM_TAG_LEN]; +}; + +/* + * Note that this struct is padded to a size which is a power of 2, this + * guarantees that this state will not span two pages. This avoids a corner + * case in the pager when making the state available. + */ +struct rwp_state_padded { + struct rwp_state state; + uint64_t pad; +}; + +struct fobj_rwp_unpaged_iv { + uint8_t *store; + struct rwp_state *state; + struct fobj fobj; +}; + +struct fobj_rwp_paged_iv { + size_t idx; + struct fobj fobj; +}; + +const struct fobj_ops ops_rwp_paged_iv; +const struct fobj_ops ops_rwp_unpaged_iv; + +static struct internal_aes_gcm_key rwp_ae_key; + +static struct rwp_state_padded *rwp_state_base; +static uint8_t *rwp_store_base; + +static void fobj_init(struct fobj *fobj, const struct fobj_ops *ops, + unsigned int num_pages) +{ + fobj->ops = ops; + fobj->num_pages = num_pages; + refcount_set(&fobj->refc, 1); + TAILQ_INIT(&fobj->regions); +} + +static void fobj_uninit(struct fobj *fobj) +{ + assert(!refcount_val(&fobj->refc)); + assert(TAILQ_EMPTY(&fobj->regions)); + tee_pager_invalidate_fobj(fobj); +} + +static TEE_Result rwp_load_page(void *va, struct rwp_state *state, + const uint8_t *src) +{ + struct rwp_aes_gcm_iv iv = { + .iv = { (vaddr_t)state, state->iv >> 32, state->iv } + }; + + if (!state->iv) { + /* + * IV still zero which means that this is previously unused + * page. + */ + memset(va, 0, SMALL_PAGE_SIZE); + return TEE_SUCCESS; + } + + return internal_aes_gcm_dec(&rwp_ae_key, &iv, sizeof(iv), + NULL, 0, src, SMALL_PAGE_SIZE, va, + state->tag, sizeof(state->tag)); +} + +static TEE_Result rwp_save_page(const void *va, struct rwp_state *state, + uint8_t *dst) +{ + size_t tag_len = sizeof(state->tag); + struct rwp_aes_gcm_iv iv = { }; + + assert(state->iv + 1 > state->iv); + + state->iv++; + + /* + * IV is constructed as recommended in section "8.2.1 Deterministic + * Construction" of "Recommendation for Block Cipher Modes of + * Operation: Galois/Counter Mode (GCM) and GMAC", + * http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf + */ + iv.iv[0] = (vaddr_t)state; + iv.iv[1] = state->iv >> 32; + iv.iv[2] = state->iv; + + return internal_aes_gcm_enc(&rwp_ae_key, &iv, sizeof(iv), + NULL, 0, va, SMALL_PAGE_SIZE, dst, + state->tag, &tag_len); +} + +static struct rwp_state_padded *idx_to_state_padded(size_t idx) +{ + assert(rwp_state_base); + return rwp_state_base + idx; +} + +static uint8_t *idx_to_store(size_t idx) +{ + assert(rwp_store_base); + return rwp_store_base + idx * SMALL_PAGE_SIZE; +} + +static struct fobj *rwp_paged_iv_alloc(unsigned int num_pages) +{ + struct fobj_rwp_paged_iv *rwp = NULL; + tee_mm_entry_t *mm = NULL; + size_t size = 0; + + COMPILE_TIME_ASSERT(IS_POWER_OF_TWO(sizeof(struct rwp_state_padded))); + + rwp = calloc(1, sizeof(*rwp)); + if (!rwp) + return NULL; + + if (MUL_OVERFLOW(num_pages, SMALL_PAGE_SIZE, &size)) + goto err; + mm = tee_mm_alloc(&tee_mm_sec_ddr, size); + if (!mm) + goto err; + rwp->idx = (tee_mm_get_smem(mm) - tee_mm_sec_ddr.lo) / SMALL_PAGE_SIZE; + + memset(idx_to_state_padded(rwp->idx), 0, + num_pages * sizeof(struct rwp_state_padded)); + + fobj_init(&rwp->fobj, &ops_rwp_paged_iv, num_pages); + + return &rwp->fobj; +err: + tee_mm_free(mm); + free(rwp); + + return NULL; +} + +static struct fobj_rwp_paged_iv *to_rwp_paged_iv(struct fobj *fobj) +{ + assert(fobj->ops == &ops_rwp_paged_iv); + + return container_of(fobj, struct fobj_rwp_paged_iv, fobj); +} + +static TEE_Result rwp_paged_iv_load_page(struct fobj *fobj, + unsigned int page_idx, void *va) +{ + struct fobj_rwp_paged_iv *rwp = to_rwp_paged_iv(fobj); + uint8_t *src = idx_to_store(rwp->idx) + page_idx * SMALL_PAGE_SIZE; + struct rwp_state_padded *st = idx_to_state_padded(rwp->idx + page_idx); + + assert(refcount_val(&fobj->refc)); + assert(page_idx < fobj->num_pages); + + return rwp_load_page(va, &st->state, src); +} +DECLARE_KEEP_PAGER(rwp_paged_iv_load_page); + +static TEE_Result rwp_paged_iv_save_page(struct fobj *fobj, + unsigned int page_idx, const void *va) +{ + struct fobj_rwp_paged_iv *rwp = to_rwp_paged_iv(fobj); + uint8_t *dst = idx_to_store(rwp->idx) + page_idx * SMALL_PAGE_SIZE; + struct rwp_state_padded *st = idx_to_state_padded(rwp->idx + page_idx); + + assert(page_idx < fobj->num_pages); + + if (!refcount_val(&fobj->refc)) { + /* + * This fobj is being teared down, it just hasn't had the time + * to call tee_pager_invalidate_fobj() yet. + */ + assert(TAILQ_EMPTY(&fobj->regions)); + return TEE_SUCCESS; + } + + return rwp_save_page(va, &st->state, dst); +} +DECLARE_KEEP_PAGER(rwp_paged_iv_save_page); + +static void rwp_paged_iv_free(struct fobj *fobj) +{ + struct fobj_rwp_paged_iv *rwp = to_rwp_paged_iv(fobj); + paddr_t pa = rwp->idx * SMALL_PAGE_SIZE + tee_mm_sec_ddr.lo; + tee_mm_entry_t *mm = tee_mm_find(&tee_mm_sec_ddr, pa); + + assert(mm); + + fobj_uninit(fobj); + tee_mm_free(mm); + free(rwp); +} + +static vaddr_t rwp_paged_iv_get_iv_vaddr(struct fobj *fobj, + unsigned int page_idx) +{ + struct fobj_rwp_paged_iv *rwp = to_rwp_paged_iv(fobj); + struct rwp_state_padded *st = idx_to_state_padded(rwp->idx + page_idx); + + assert(page_idx < fobj->num_pages); + return (vaddr_t)&st->state & ~SMALL_PAGE_MASK; +} +DECLARE_KEEP_PAGER(rwp_paged_iv_get_iv_vaddr); + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct fobj_ops ops_rwp_paged_iv +__weak __relrodata_unpaged("ops_rwp_paged_iv") = { + .free = rwp_paged_iv_free, + .load_page = rwp_paged_iv_load_page, + .save_page = rwp_paged_iv_save_page, + .get_iv_vaddr = rwp_paged_iv_get_iv_vaddr, +}; + +static struct fobj *rwp_unpaged_iv_alloc(unsigned int num_pages) +{ + struct fobj_rwp_unpaged_iv *rwp = NULL; + tee_mm_entry_t *mm = NULL; + size_t size = 0; + + rwp = calloc(1, sizeof(*rwp)); + if (!rwp) + return NULL; + + rwp->state = calloc(num_pages, sizeof(*rwp->state)); + if (!rwp->state) + goto err_free_rwp; + + if (MUL_OVERFLOW(num_pages, SMALL_PAGE_SIZE, &size)) + goto err_free_state; + mm = tee_mm_alloc(&tee_mm_sec_ddr, size); + if (!mm) + goto err_free_state; + rwp->store = phys_to_virt(tee_mm_get_smem(mm), MEM_AREA_TA_RAM, size); + assert(rwp->store); + + fobj_init(&rwp->fobj, &ops_rwp_unpaged_iv, num_pages); + + return &rwp->fobj; + +err_free_state: + free(rwp->state); +err_free_rwp: + free(rwp); + return NULL; +} + +static struct fobj_rwp_unpaged_iv *to_rwp_unpaged_iv(struct fobj *fobj) +{ + assert(fobj->ops == &ops_rwp_unpaged_iv); + + return container_of(fobj, struct fobj_rwp_unpaged_iv, fobj); +} + +static TEE_Result rwp_unpaged_iv_load_page(struct fobj *fobj, + unsigned int page_idx, void *va) +{ + struct fobj_rwp_unpaged_iv *rwp = to_rwp_unpaged_iv(fobj); + uint8_t *src = rwp->store + page_idx * SMALL_PAGE_SIZE; + + assert(refcount_val(&fobj->refc)); + assert(page_idx < fobj->num_pages); + + return rwp_load_page(va, rwp->state + page_idx, src); +} +DECLARE_KEEP_PAGER(rwp_unpaged_iv_load_page); + +static TEE_Result rwp_unpaged_iv_save_page(struct fobj *fobj, + unsigned int page_idx, + const void *va) +{ + struct fobj_rwp_unpaged_iv *rwp = to_rwp_unpaged_iv(fobj); + uint8_t *dst = rwp->store + page_idx * SMALL_PAGE_SIZE; + + assert(page_idx < fobj->num_pages); + + if (!refcount_val(&fobj->refc)) { + /* + * This fobj is being teared down, it just hasn't had the time + * to call tee_pager_invalidate_fobj() yet. + */ + assert(TAILQ_EMPTY(&fobj->regions)); + return TEE_SUCCESS; + } + + return rwp_save_page(va, rwp->state + page_idx, dst); +} +DECLARE_KEEP_PAGER(rwp_unpaged_iv_save_page); + +static void rwp_unpaged_iv_free(struct fobj *fobj) +{ + struct fobj_rwp_unpaged_iv *rwp = NULL; + tee_mm_entry_t *mm = NULL; + + if (IS_ENABLED(CFG_CORE_PAGE_TAG_AND_IV)) + panic(); + + rwp = to_rwp_unpaged_iv(fobj); + mm = tee_mm_find(&tee_mm_sec_ddr, virt_to_phys(rwp->store)); + + assert(mm); + + fobj_uninit(fobj); + tee_mm_free(mm); + free(rwp->state); + free(rwp); +} + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct fobj_ops ops_rwp_unpaged_iv +__weak __relrodata_unpaged("ops_rwp_unpaged_iv") = { + .free = rwp_unpaged_iv_free, + .load_page = rwp_unpaged_iv_load_page, + .save_page = rwp_unpaged_iv_save_page, +}; + +static TEE_Result rwp_init(void) +{ + uint8_t key[RWP_AE_KEY_BITS / 8] = { 0 }; + struct fobj *fobj = NULL; + size_t num_pool_pages = 0; + size_t num_fobj_pages = 0; + + if (crypto_rng_read(key, sizeof(key)) != TEE_SUCCESS) + panic("failed to generate random"); + if (crypto_aes_expand_enc_key(key, sizeof(key), rwp_ae_key.data, + sizeof(rwp_ae_key.data), + &rwp_ae_key.rounds)) + panic("failed to expand key"); + + if (!IS_ENABLED(CFG_CORE_PAGE_TAG_AND_IV)) + return TEE_SUCCESS; + + assert(tee_mm_sec_ddr.size && !(tee_mm_sec_ddr.size & SMALL_PAGE_SIZE)); + + num_pool_pages = tee_mm_sec_ddr.size / SMALL_PAGE_SIZE; + num_fobj_pages = ROUNDUP(num_pool_pages * sizeof(*rwp_state_base), + SMALL_PAGE_SIZE) / SMALL_PAGE_SIZE; + + /* + * Each page in the pool needs a struct rwp_state. + * + * This isn't entirely true, the pages not used by + * fobj_rw_paged_alloc() don't need any. A future optimization + * may try to avoid allocating for such pages. + */ + fobj = rwp_unpaged_iv_alloc(num_fobj_pages); + if (!fobj) + panic(); + + rwp_state_base = (void *)tee_pager_init_iv_region(fobj); + assert(rwp_state_base); + + rwp_store_base = phys_to_virt(tee_mm_sec_ddr.lo, MEM_AREA_TA_RAM, + tee_mm_sec_ddr.size); + assert(rwp_store_base); + + return TEE_SUCCESS; +} +driver_init_late(rwp_init); + +struct fobj *fobj_rw_paged_alloc(unsigned int num_pages) +{ + assert(num_pages); + + if (IS_ENABLED(CFG_CORE_PAGE_TAG_AND_IV)) + return rwp_paged_iv_alloc(num_pages); + else + return rwp_unpaged_iv_alloc(num_pages); +} + +struct fobj_rop { + uint8_t *hashes; + uint8_t *store; + struct fobj fobj; +}; + +const struct fobj_ops ops_ro_paged; + +static void rop_init(struct fobj_rop *rop, const struct fobj_ops *ops, + unsigned int num_pages, void *hashes, void *store) +{ + rop->hashes = hashes; + rop->store = store; + fobj_init(&rop->fobj, ops, num_pages); +} + +struct fobj *fobj_ro_paged_alloc(unsigned int num_pages, void *hashes, + void *store) +{ + struct fobj_rop *rop = NULL; + + assert(num_pages && hashes && store); + + rop = calloc(1, sizeof(*rop)); + if (!rop) + return NULL; + + rop_init(rop, &ops_ro_paged, num_pages, hashes, store); + + return &rop->fobj; +} + +static struct fobj_rop *to_rop(struct fobj *fobj) +{ + assert(fobj->ops == &ops_ro_paged); + + return container_of(fobj, struct fobj_rop, fobj); +} + +static void rop_uninit(struct fobj_rop *rop) +{ + fobj_uninit(&rop->fobj); + tee_mm_free(tee_mm_find(&tee_mm_sec_ddr, virt_to_phys(rop->store))); + free(rop->hashes); +} + +static void rop_free(struct fobj *fobj) +{ + struct fobj_rop *rop = to_rop(fobj); + + rop_uninit(rop); + free(rop); +} + +static TEE_Result rop_load_page_helper(struct fobj_rop *rop, + unsigned int page_idx, void *va) +{ + const uint8_t *hash = rop->hashes + page_idx * TEE_SHA256_HASH_SIZE; + const uint8_t *src = rop->store + page_idx * SMALL_PAGE_SIZE; + + assert(refcount_val(&rop->fobj.refc)); + assert(page_idx < rop->fobj.num_pages); + memcpy(va, src, SMALL_PAGE_SIZE); + + return hash_sha256_check(hash, va, SMALL_PAGE_SIZE); +} + +static TEE_Result rop_load_page(struct fobj *fobj, unsigned int page_idx, + void *va) +{ + return rop_load_page_helper(to_rop(fobj), page_idx, va); +} +DECLARE_KEEP_PAGER(rop_load_page); + +static TEE_Result rop_save_page(struct fobj *fobj __unused, + unsigned int page_idx __unused, + const void *va __unused) +{ + return TEE_ERROR_GENERIC; +} +DECLARE_KEEP_PAGER(rop_save_page); + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct fobj_ops ops_ro_paged +__weak __relrodata_unpaged("ops_ro_paged") = { + .free = rop_free, + .load_page = rop_load_page, + .save_page = rop_save_page, +}; + +#ifdef CFG_CORE_ASLR +/* + * When using relocated pages the relocation information must be applied + * before the pages can be used. With read-only paging the content is only + * integrity protected so relocation cannot be applied on pages in the less + * secure "store" or the load_address selected by ASLR could be given away. + * This means that each time a page has been loaded and verified it has to + * have its relocation information applied before it can be used. + * + * Only the relative relocations are supported, this allows a rather compact + * represenation of the needed relocation information in this struct. + * r_offset is replaced with the offset into the page that need to be updated, + * this number can never be larger than SMALL_PAGE_SIZE so a uint16_t can be + * used to represent it. + * + * All relocations are converted and stored in @relocs. @page_reloc_idx is + * an array of length @rop.fobj.num_pages with an entry for each page. If + * @page_reloc_idx[page_idx] isn't UINT16_MAX it's an index into @relocs. + */ +struct fobj_ro_reloc_paged { + uint16_t *page_reloc_idx; + uint16_t *relocs; + unsigned int num_relocs; + struct fobj_rop rop; +}; + +const struct fobj_ops ops_ro_reloc_paged; + +static unsigned int get_num_rels(unsigned int num_pages, + unsigned int reloc_offs, + const uint32_t *reloc, unsigned int num_relocs) +{ + const unsigned int align_mask __maybe_unused = sizeof(long) - 1; + unsigned int nrels = 0; + unsigned int n = 0; + vaddr_t offs = 0; + + /* + * Count the number of relocations which are needed for these + * pages. Also check that the data is well formed, only expected + * relocations and sorted in order of address which it applies to. + */ + for (; n < num_relocs; n++) { + assert(IS_ALIGNED_WITH_TYPE(reloc[n], unsigned long)); + assert(offs < reloc[n]); /* check that it's sorted */ + offs = reloc[n]; + if (offs >= reloc_offs && + offs <= reloc_offs + num_pages * SMALL_PAGE_SIZE) + nrels++; + } + + return nrels; +} + +static void init_rels(struct fobj_ro_reloc_paged *rrp, unsigned int reloc_offs, + const uint32_t *reloc, unsigned int num_relocs) +{ + unsigned int npg = rrp->rop.fobj.num_pages; + unsigned int pg_idx = 0; + unsigned int reln = 0; + unsigned int n = 0; + uint32_t r = 0; + + for (n = 0; n < npg; n++) + rrp->page_reloc_idx[n] = UINT16_MAX; + + for (n = 0; n < num_relocs ; n++) { + if (reloc[n] < reloc_offs) + continue; + + /* r is the offset from beginning of this fobj */ + r = reloc[n] - reloc_offs; + + pg_idx = r / SMALL_PAGE_SIZE; + if (pg_idx >= npg) + break; + + if (rrp->page_reloc_idx[pg_idx] == UINT16_MAX) + rrp->page_reloc_idx[pg_idx] = reln; + rrp->relocs[reln] = r - pg_idx * SMALL_PAGE_SIZE; + reln++; + } + + assert(reln == rrp->num_relocs); +} + +struct fobj *fobj_ro_reloc_paged_alloc(unsigned int num_pages, void *hashes, + unsigned int reloc_offs, + const void *reloc, + unsigned int reloc_len, void *store) +{ + struct fobj_ro_reloc_paged *rrp = NULL; + const unsigned int num_relocs = reloc_len / sizeof(uint32_t); + unsigned int nrels = 0; + + assert(IS_ALIGNED_WITH_TYPE(reloc, uint32_t)); + assert(IS_ALIGNED_WITH_TYPE(reloc_len, uint32_t)); + assert(num_pages && hashes && store); + if (!reloc_len) { + assert(!reloc); + return fobj_ro_paged_alloc(num_pages, hashes, store); + } + assert(reloc); + + nrels = get_num_rels(num_pages, reloc_offs, reloc, num_relocs); + if (!nrels) + return fobj_ro_paged_alloc(num_pages, hashes, store); + + rrp = calloc(1, sizeof(*rrp) + num_pages * sizeof(uint16_t) + + nrels * sizeof(uint16_t)); + if (!rrp) + return NULL; + rop_init(&rrp->rop, &ops_ro_reloc_paged, num_pages, hashes, store); + rrp->page_reloc_idx = (uint16_t *)(rrp + 1); + rrp->relocs = rrp->page_reloc_idx + num_pages; + rrp->num_relocs = nrels; + init_rels(rrp, reloc_offs, reloc, num_relocs); + + return &rrp->rop.fobj; +} + +static struct fobj_ro_reloc_paged *to_rrp(struct fobj *fobj) +{ + assert(fobj->ops == &ops_ro_reloc_paged); + + return container_of(fobj, struct fobj_ro_reloc_paged, rop.fobj); +} + +static void rrp_free(struct fobj *fobj) +{ + struct fobj_ro_reloc_paged *rrp = to_rrp(fobj); + + rop_uninit(&rrp->rop); + free(rrp); +} + +static TEE_Result rrp_load_page(struct fobj *fobj, unsigned int page_idx, + void *va) +{ + struct fobj_ro_reloc_paged *rrp = to_rrp(fobj); + unsigned int end_rel = rrp->num_relocs; + TEE_Result res = TEE_SUCCESS; + unsigned long *where = NULL; + unsigned int n = 0; + + res = rop_load_page_helper(&rrp->rop, page_idx, va); + if (res) + return res; + + /* Find the reloc index of the next page to tell when we're done */ + for (n = page_idx + 1; n < fobj->num_pages; n++) { + if (rrp->page_reloc_idx[n] != UINT16_MAX) { + end_rel = rrp->page_reloc_idx[n]; + break; + } + } + + for (n = rrp->page_reloc_idx[page_idx]; n < end_rel; n++) { + where = (void *)((vaddr_t)va + rrp->relocs[n]); + *where += boot_mmu_config.map_offset; + } + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(rrp_load_page); + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct fobj_ops ops_ro_reloc_paged +__weak __relrodata_unpaged("ops_ro_reloc_paged") = { + .free = rrp_free, + .load_page = rrp_load_page, + .save_page = rop_save_page, /* Direct reuse */ +}; +#endif /*CFG_CORE_ASLR*/ + +const struct fobj_ops ops_locked_paged; + +struct fobj *fobj_locked_paged_alloc(unsigned int num_pages) +{ + struct fobj *f = NULL; + + assert(num_pages); + + f = calloc(1, sizeof(*f)); + if (!f) + return NULL; + + fobj_init(f, &ops_locked_paged, num_pages); + + return f; +} + +static void lop_free(struct fobj *fobj) +{ + assert(fobj->ops == &ops_locked_paged); + fobj_uninit(fobj); + free(fobj); +} + +static TEE_Result lop_load_page(struct fobj *fobj __maybe_unused, + unsigned int page_idx __maybe_unused, + void *va) +{ + assert(fobj->ops == &ops_locked_paged); + assert(refcount_val(&fobj->refc)); + assert(page_idx < fobj->num_pages); + + memset(va, 0, SMALL_PAGE_SIZE); + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(lop_load_page); + +static TEE_Result lop_save_page(struct fobj *fobj __unused, + unsigned int page_idx __unused, + const void *va __unused) +{ + return TEE_ERROR_GENERIC; +} +DECLARE_KEEP_PAGER(lop_save_page); + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct fobj_ops ops_locked_paged +__weak __relrodata_unpaged("ops_locked_paged") = { + .free = lop_free, + .load_page = lop_load_page, + .save_page = lop_save_page, +}; +#endif /*CFG_WITH_PAGER*/ + +#ifndef CFG_PAGED_USER_TA + +struct fobj_sec_mem { + tee_mm_entry_t *mm; + struct fobj fobj; +}; + +const struct fobj_ops ops_sec_mem; + +struct fobj *fobj_sec_mem_alloc(unsigned int num_pages) +{ + struct fobj_sec_mem *f = calloc(1, sizeof(*f)); + size_t size = 0; + void *va = NULL; + + if (!f) + return NULL; + + if (MUL_OVERFLOW(num_pages, SMALL_PAGE_SIZE, &size)) + goto err; + + f->mm = tee_mm_alloc(&tee_mm_sec_ddr, size); + if (!f->mm) + goto err; + + va = phys_to_virt(tee_mm_get_smem(f->mm), MEM_AREA_TA_RAM, size); + if (!va) + goto err; + + memtag_clear_mem(va, size); + f->fobj.ops = &ops_sec_mem; + f->fobj.num_pages = num_pages; + refcount_set(&f->fobj.refc, 1); + + return &f->fobj; +err: + tee_mm_free(f->mm); + free(f); + + return NULL; +} + +static struct fobj_sec_mem *to_sec_mem(struct fobj *fobj) +{ + assert(fobj->ops == &ops_sec_mem); + + return container_of(fobj, struct fobj_sec_mem, fobj); +} + +static void sec_mem_free(struct fobj *fobj) +{ + struct fobj_sec_mem *f = to_sec_mem(fobj); + + assert(!refcount_val(&fobj->refc)); + tee_mm_free(f->mm); + free(f); +} + +static paddr_t sec_mem_get_pa(struct fobj *fobj, unsigned int page_idx) +{ + struct fobj_sec_mem *f = to_sec_mem(fobj); + + assert(refcount_val(&fobj->refc)); + assert(page_idx < fobj->num_pages); + + return tee_mm_get_smem(f->mm) + page_idx * SMALL_PAGE_SIZE; +} + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct fobj_ops ops_sec_mem __weak __relrodata_unpaged("ops_sec_mem") = { + .free = sec_mem_free, + .get_pa = sec_mem_get_pa, +}; + +#endif /*PAGED_USER_TA*/ diff --git a/optee_os/core/mm/mobj.c b/optee_os/core/mm/mobj.c new file mode 100644 index 0000000..afc8176 --- /dev/null +++ b/optee_os/core/mm/mobj.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mobj *mobj_sec_ddr; +struct mobj *mobj_tee_ram_rx; +struct mobj *mobj_tee_ram_rw; + +/* + * mobj_phys implementation + */ + +struct mobj_phys { + struct mobj mobj; + enum buf_is_attr battr; + /* Defined by TEE_MATTR_MEM_TYPE_* in tee_mmu_types.h */ + uint32_t mem_type; + vaddr_t va; + paddr_t pa; +}; + +static struct mobj_phys *to_mobj_phys(struct mobj *mobj); + +static void *mobj_phys_get_va(struct mobj *mobj, size_t offset, size_t len) +{ + struct mobj_phys *moph = to_mobj_phys(mobj); + + if (!moph->va || !mobj_check_offset_and_len(mobj, offset, len)) + return NULL; + + return (void *)(moph->va + offset); +} + +static TEE_Result mobj_phys_get_pa(struct mobj *mobj, size_t offs, + size_t granule, paddr_t *pa) +{ + struct mobj_phys *moph = to_mobj_phys(mobj); + paddr_t p; + + if (!pa) + return TEE_ERROR_GENERIC; + + p = moph->pa + offs; + + if (granule) { + if (granule != SMALL_PAGE_SIZE && + granule != CORE_MMU_PGDIR_SIZE) + return TEE_ERROR_GENERIC; + p &= ~(granule - 1); + } + + *pa = p; + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(mobj_phys_get_pa); + +static TEE_Result mobj_phys_get_mem_type(struct mobj *mobj, uint32_t *mem_type) +{ + struct mobj_phys *moph = to_mobj_phys(mobj); + + if (!mem_type) + return TEE_ERROR_GENERIC; + + *mem_type = moph->mem_type; + return TEE_SUCCESS; +} + +static bool mobj_phys_matches(struct mobj *mobj, enum buf_is_attr attr) +{ + struct mobj_phys *moph = to_mobj_phys(mobj); + enum buf_is_attr a; + + a = moph->battr; + + switch (attr) { + case CORE_MEM_SEC: + return a == CORE_MEM_SEC || a == CORE_MEM_TEE_RAM || + a == CORE_MEM_TA_RAM || a == CORE_MEM_SDP_MEM; + case CORE_MEM_NON_SEC: + return a == CORE_MEM_NSEC_SHM; + case CORE_MEM_TEE_RAM: + case CORE_MEM_TA_RAM: + case CORE_MEM_NSEC_SHM: + case CORE_MEM_SDP_MEM: + return attr == a; + default: + return false; + } +} + +static void mobj_phys_free(struct mobj *mobj) +{ + struct mobj_phys *moph = to_mobj_phys(mobj); + + free(moph); +} + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct mobj_ops mobj_phys_ops +__weak __relrodata_unpaged("mobj_phys_ops") = { + .get_va = mobj_phys_get_va, + .get_pa = mobj_phys_get_pa, + .get_phys_offs = NULL, /* only offset 0 */ + .get_mem_type = mobj_phys_get_mem_type, + .matches = mobj_phys_matches, + .free = mobj_phys_free, +}; + +static struct mobj_phys *to_mobj_phys(struct mobj *mobj) +{ + assert(mobj->ops == &mobj_phys_ops); + return container_of(mobj, struct mobj_phys, mobj); +} + +static struct mobj *mobj_phys_init(paddr_t pa, size_t size, uint32_t mem_type, + enum buf_is_attr battr, + enum teecore_memtypes area_type) +{ + void *va = NULL; + struct mobj_phys *moph = NULL; + struct tee_mmap_region *map = NULL; + + if ((pa & CORE_MMU_USER_PARAM_MASK) || + (size & CORE_MMU_USER_PARAM_MASK)) { + DMSG("Expect %#x alignment", CORE_MMU_USER_PARAM_SIZE); + return NULL; + } + + if (pa) { + va = phys_to_virt(pa, area_type, size); + } else { + map = core_mmu_find_mapping_exclusive(area_type, size); + if (!map) + return NULL; + + pa = map->pa; + va = (void *)map->va; + } + + /* Only SDP memory may not have a virtual address */ + if (!va && battr != CORE_MEM_SDP_MEM) + return NULL; + + moph = calloc(1, sizeof(*moph)); + if (!moph) + return NULL; + + moph->battr = battr; + moph->mem_type = mem_type; + moph->mobj.size = size; + moph->mobj.ops = &mobj_phys_ops; + refcount_set(&moph->mobj.refc, 1); + moph->pa = pa; + moph->va = (vaddr_t)va; + + return &moph->mobj; +} + +struct mobj *mobj_phys_alloc(paddr_t pa, size_t size, uint32_t mem_type, + enum buf_is_attr battr) +{ + enum teecore_memtypes area_type; + + switch (battr) { + case CORE_MEM_TEE_RAM: + area_type = MEM_AREA_TEE_RAM_RW_DATA; + break; + case CORE_MEM_TA_RAM: + area_type = MEM_AREA_TA_RAM; + break; + case CORE_MEM_NSEC_SHM: + area_type = MEM_AREA_NSEC_SHM; + break; + case CORE_MEM_SDP_MEM: + area_type = MEM_AREA_SDP_MEM; + break; + default: + DMSG("can't allocate with specified attribute"); + return NULL; + } + + return mobj_phys_init(pa, size, mem_type, battr, area_type); +} + +/* + * mobj_virt implementation + */ + +static void mobj_virt_assert_type(struct mobj *mobj); + +static void *mobj_virt_get_va(struct mobj *mobj, size_t offset, + size_t len __maybe_unused) +{ + mobj_virt_assert_type(mobj); + assert(mobj_check_offset_and_len(mobj, offset, len)); + + return (void *)(vaddr_t)offset; +} + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct mobj_ops mobj_virt_ops +__weak __relrodata_unpaged("mobj_virt_ops") = { + .get_va = mobj_virt_get_va, +}; + +static void mobj_virt_assert_type(struct mobj *mobj __maybe_unused) +{ + assert(mobj->ops == &mobj_virt_ops); +} + +struct mobj mobj_virt = { .ops = &mobj_virt_ops, .size = SIZE_MAX }; + +/* + * mobj_shm implementation. mobj_shm represents buffer in predefined shm region + * - it is physically contiguous. + * - it is identified in static physical layout as MEM_AREA_NSEC_SHM. + * - it creates mobjs that match specific CORE_MEM_NSEC_SHM and non secure + * generic CORE_MEM_NON_SEC. + */ + +struct mobj_shm { + struct mobj mobj; + paddr_t pa; + uint64_t cookie; +}; + +static struct mobj_shm *to_mobj_shm(struct mobj *mobj); + +static void *mobj_shm_get_va(struct mobj *mobj, size_t offset, size_t len) +{ + struct mobj_shm *m = to_mobj_shm(mobj); + + if (!mobj_check_offset_and_len(mobj, offset, len)) + return NULL; + + return phys_to_virt(m->pa + offset, MEM_AREA_NSEC_SHM, + mobj->size - offset); +} + +static TEE_Result mobj_shm_get_pa(struct mobj *mobj, size_t offs, + size_t granule, paddr_t *pa) +{ + struct mobj_shm *m = to_mobj_shm(mobj); + paddr_t p; + + if (!pa || offs >= mobj->size) + return TEE_ERROR_GENERIC; + + p = m->pa + offs; + + if (granule) { + if (granule != SMALL_PAGE_SIZE && + granule != CORE_MMU_PGDIR_SIZE) + return TEE_ERROR_GENERIC; + p &= ~(granule - 1); + } + + *pa = p; + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(mobj_shm_get_pa); + +static size_t mobj_shm_get_phys_offs(struct mobj *mobj, size_t granule) +{ + assert(IS_POWER_OF_TWO(granule)); + return to_mobj_shm(mobj)->pa & (granule - 1); +} + +static bool mobj_shm_matches(struct mobj *mobj __unused, enum buf_is_attr attr) +{ + return attr == CORE_MEM_NSEC_SHM || attr == CORE_MEM_NON_SEC; +} + +static TEE_Result mobj_shm_get_mem_type(struct mobj *mobj __unused, + uint32_t *mem_type) +{ + if (!mem_type) + return TEE_ERROR_GENERIC; + + *mem_type = TEE_MATTR_MEM_TYPE_CACHED; + + return TEE_SUCCESS; +} + +static void mobj_shm_free(struct mobj *mobj) +{ + struct mobj_shm *m = to_mobj_shm(mobj); + + free(m); +} + +static uint64_t mobj_shm_get_cookie(struct mobj *mobj) +{ + return to_mobj_shm(mobj)->cookie; +} + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct mobj_ops mobj_shm_ops +__weak __relrodata_unpaged("mobj_shm_ops") = { + .get_va = mobj_shm_get_va, + .get_pa = mobj_shm_get_pa, + .get_phys_offs = mobj_shm_get_phys_offs, + .get_mem_type = mobj_shm_get_mem_type, + .matches = mobj_shm_matches, + .free = mobj_shm_free, + .get_cookie = mobj_shm_get_cookie, +}; + +static struct mobj_shm *to_mobj_shm(struct mobj *mobj) +{ + assert(mobj->ops == &mobj_shm_ops); + return container_of(mobj, struct mobj_shm, mobj); +} + +struct mobj *mobj_shm_alloc(paddr_t pa, size_t size, uint64_t cookie) +{ + struct mobj_shm *m; + + if (!core_pbuf_is(CORE_MEM_NSEC_SHM, pa, size)) + return NULL; + + m = calloc(1, sizeof(*m)); + if (!m) + return NULL; + + m->mobj.size = size; + m->mobj.ops = &mobj_shm_ops; + m->mobj.phys_granule = SMALL_PAGE_SIZE; + refcount_set(&m->mobj.refc, 1); + m->pa = pa; + m->cookie = cookie; + + return &m->mobj; +} + +struct mobj_with_fobj { + struct fobj *fobj; + struct file *file; + struct mobj mobj; + uint8_t mem_type; +}; + +const struct mobj_ops mobj_with_fobj_ops; + +struct mobj *mobj_with_fobj_alloc(struct fobj *fobj, struct file *file, + uint32_t mem_type) +{ + struct mobj_with_fobj *m = NULL; + + assert(!(mem_type & ~TEE_MATTR_MEM_TYPE_MASK)); + + if (!fobj) + return NULL; + if (mem_type > UINT8_MAX) + return NULL; + + m = calloc(1, sizeof(*m)); + if (!m) + return NULL; + + m->mobj.ops = &mobj_with_fobj_ops; + refcount_set(&m->mobj.refc, 1); + m->mobj.size = fobj->num_pages * SMALL_PAGE_SIZE; + m->mobj.phys_granule = SMALL_PAGE_SIZE; + m->fobj = fobj_get(fobj); + m->file = file_get(file); + m->mem_type = mem_type; + + return &m->mobj; +} + +static struct mobj_with_fobj *to_mobj_with_fobj(struct mobj *mobj) +{ + assert(mobj && mobj->ops == &mobj_with_fobj_ops); + + return container_of(mobj, struct mobj_with_fobj, mobj); +} + +static bool mobj_with_fobj_matches(struct mobj *mobj __maybe_unused, + enum buf_is_attr attr) +{ + assert(to_mobj_with_fobj(mobj)); + + /* + * All fobjs are supposed to be mapped secure so classify it as + * CORE_MEM_SEC. Stay out of CORE_MEM_TEE_RAM etc, if that information + * needed it can probably be carried in another way than to put the + * burden directly on fobj. + */ + return attr == CORE_MEM_SEC; +} + +static void mobj_with_fobj_free(struct mobj *mobj) +{ + struct mobj_with_fobj *m = to_mobj_with_fobj(mobj); + + fobj_put(m->fobj); + file_put(m->file); + free(m); +} + +static struct fobj *mobj_with_fobj_get_fobj(struct mobj *mobj) +{ + return fobj_get(to_mobj_with_fobj(mobj)->fobj); +} + +static TEE_Result mobj_with_fobj_get_mem_type(struct mobj *mobj, + uint32_t *mem_type) +{ + struct mobj_with_fobj *m = to_mobj_with_fobj(mobj); + + if (!mem_type) + return TEE_ERROR_GENERIC; + + *mem_type = m->mem_type; + + return TEE_SUCCESS; +} + +static TEE_Result mobj_with_fobj_get_pa(struct mobj *mobj, size_t offs, + size_t granule, paddr_t *pa) +{ + struct mobj_with_fobj *f = to_mobj_with_fobj(mobj); + paddr_t p = 0; + + if (!f->fobj->ops->get_pa) { + assert(mobj_is_paged(mobj)); + return TEE_ERROR_NOT_SUPPORTED; + } + + p = f->fobj->ops->get_pa(f->fobj, offs / SMALL_PAGE_SIZE) + + offs % SMALL_PAGE_SIZE; + + if (granule) { + if (granule != SMALL_PAGE_SIZE && + granule != CORE_MMU_PGDIR_SIZE) + return TEE_ERROR_GENERIC; + p &= ~(granule - 1); + } + + *pa = p; + + return TEE_SUCCESS; +} +DECLARE_KEEP_PAGER(mobj_with_fobj_get_pa); + +/* + * Note: this variable is weak just to ease breaking its dependency chain + * when added to the unpaged area. + */ +const struct mobj_ops mobj_with_fobj_ops +__weak __relrodata_unpaged("mobj_with_fobj_ops") = { + .matches = mobj_with_fobj_matches, + .free = mobj_with_fobj_free, + .get_fobj = mobj_with_fobj_get_fobj, + .get_mem_type = mobj_with_fobj_get_mem_type, + .get_pa = mobj_with_fobj_get_pa, +}; + +#ifdef CFG_PAGED_USER_TA +bool mobj_is_paged(struct mobj *mobj) +{ + if (mobj->ops == &mobj_with_fobj_ops && + !to_mobj_with_fobj(mobj)->fobj->ops->get_pa) + return true; + + return false; +} +#endif /*CFG_PAGED_USER_TA*/ + +static TEE_Result mobj_init(void) +{ + mobj_sec_ddr = mobj_phys_alloc(tee_mm_sec_ddr.lo, + tee_mm_sec_ddr.size, + TEE_MATTR_MEM_TYPE_CACHED, + CORE_MEM_TA_RAM); + if (!mobj_sec_ddr) + panic("Failed to register secure ta ram"); + + if (IS_ENABLED(CFG_CORE_RWDATA_NOEXEC)) { + mobj_tee_ram_rx = mobj_phys_init(0, + VCORE_UNPG_RX_SZ, + TEE_MATTR_MEM_TYPE_CACHED, + CORE_MEM_TEE_RAM, + MEM_AREA_TEE_RAM_RX); + if (!mobj_tee_ram_rx) + panic("Failed to register tee ram rx"); + + mobj_tee_ram_rw = mobj_phys_init(0, + VCORE_UNPG_RW_SZ, + TEE_MATTR_MEM_TYPE_CACHED, + CORE_MEM_TEE_RAM, + MEM_AREA_TEE_RAM_RW_DATA); + if (!mobj_tee_ram_rw) + panic("Failed to register tee ram rw"); + } else { + mobj_tee_ram_rw = mobj_phys_init(TEE_RAM_START, + VCORE_UNPG_RW_PA + + VCORE_UNPG_RW_SZ - + TEE_RAM_START, + TEE_MATTR_MEM_TYPE_CACHED, + CORE_MEM_TEE_RAM, + MEM_AREA_TEE_RAM_RW_DATA); + if (!mobj_tee_ram_rw) + panic("Failed to register tee ram"); + + mobj_tee_ram_rx = mobj_tee_ram_rw; + } + + return TEE_SUCCESS; +} + +driver_init_late(mobj_init); diff --git a/optee_os/core/mm/pgt_cache.c b/optee_os/core/mm/pgt_cache.c new file mode 100644 index 0000000..79553c6 --- /dev/null +++ b/optee_os/core/mm/pgt_cache.c @@ -0,0 +1,873 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, 2022 Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * With pager enabled we allocate page table from the pager. + * + * For LPAE each page table is a complete page which is allocated and freed + * using the interface provided by the pager. + * + * For compat v7 page tables there's room for four page table in one page + * so we need to keep track of how much of an allocated page is used. When + * a page is completely unused it's returned to the pager. + * + * With pager disabled we have a static allocation of page tables instead. + * + * In all cases we limit the number of active page tables to + * PGT_CACHE_SIZE. This pool of page tables are shared between all + * threads. In case a thread can't allocate the needed number of pager + * tables it will release all its current tables and wait for some more to + * be freed. A threads allocated tables are freed each time a TA is + * unmapped so each thread should be able to allocate the needed tables in + * turn if needed. + */ + +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) || \ + (defined(CFG_WITH_PAGER) && !defined(CFG_WITH_LPAE)) +struct pgt_parent { + size_t num_used; + struct pgt_cache pgt_cache; +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) + tee_mm_entry_t *mm; + SLIST_ENTRY(pgt_parent) link; +#endif +}; +#endif + +#if defined(CFG_CORE_PREALLOC_EL0_TBLS) + +/* + * Pick something large enough that tee_mm_alloc() doesn't have to be + * called for each needed translation table. + */ +#define PGT_PARENT_SIZE (4 * SMALL_PAGE_SIZE) +#define PGT_PARENT_TBL_COUNT (PGT_PARENT_SIZE / PGT_SIZE) + +SLIST_HEAD(pgt_parent_list, pgt_parent); +static struct pgt_parent_list parent_list = SLIST_HEAD_INITIALIZER(parent_list); +static unsigned int parent_spinlock = SPINLOCK_UNLOCK; + +static void free_pgt(struct pgt *pgt) +{ + struct pgt_parent *parent = NULL; + uint32_t exceptions = 0; + + exceptions = cpu_spin_lock_xsave(&parent_spinlock); + + assert(pgt && pgt->parent); + parent = pgt->parent; + assert(parent->num_used <= PGT_PARENT_TBL_COUNT && + parent->num_used > 0); + if (parent->num_used == PGT_PARENT_TBL_COUNT) + SLIST_INSERT_HEAD(&parent_list, parent, link); + parent->num_used--; + + if (!parent->num_used && SLIST_NEXT(SLIST_FIRST(&parent_list), link)) { + /* + * If this isn't the last pgt_parent with free entries we + * can free this. + */ + SLIST_REMOVE(&parent_list, parent, pgt_parent, link); + tee_mm_free(parent->mm); + free(parent); + } else { + SLIST_INSERT_HEAD(&parent->pgt_cache, pgt, link); + pgt->vabase = 0; + pgt->populated = false; + } + + cpu_spin_unlock_xrestore(&parent_spinlock, exceptions); +} + +static struct pgt_parent *alloc_pgt_parent(void) +{ + struct pgt_parent *parent = NULL; + struct pgt *pgt = NULL; + uint8_t *tbl = NULL; + size_t sz = 0; + size_t n = 0; + + sz = sizeof(*parent) + sizeof(*pgt) * PGT_PARENT_TBL_COUNT; + parent = calloc(1, sz); + if (!parent) + return NULL; + parent->mm = tee_mm_alloc(&tee_mm_sec_ddr, PGT_PARENT_SIZE); + if (!parent->mm) { + free(parent); + return NULL; + } + tbl = phys_to_virt(tee_mm_get_smem(parent->mm), MEM_AREA_TA_RAM, + PGT_PARENT_SIZE); + assert(tbl); /* "can't fail" */ + + SLIST_INIT(&parent->pgt_cache); + pgt = (struct pgt *)(parent + 1); + for (n = 0; n < PGT_PARENT_TBL_COUNT; n++) { + pgt[n].parent = parent; + pgt[n].tbl = tbl + n * PGT_SIZE; + SLIST_INSERT_HEAD(&parent->pgt_cache, pgt + n, link); + } + + return parent; +} + +static struct pgt *alloc_pgt(vaddr_t vabase) +{ + struct pgt_parent *parent = NULL; + uint32_t exceptions = 0; + struct pgt *pgt = NULL; + + exceptions = cpu_spin_lock_xsave(&parent_spinlock); + + parent = SLIST_FIRST(&parent_list); + if (!parent) { + parent = alloc_pgt_parent(); + if (!parent) + goto out; + + SLIST_INSERT_HEAD(&parent_list, parent, link); + } + + pgt = SLIST_FIRST(&parent->pgt_cache); + SLIST_REMOVE_HEAD(&parent->pgt_cache, link); + parent->num_used++; + assert(pgt && parent->num_used <= PGT_PARENT_TBL_COUNT); + if (parent->num_used == PGT_PARENT_TBL_COUNT) + SLIST_REMOVE_HEAD(&parent_list, link); + + pgt->vabase = vabase; +out: + cpu_spin_unlock_xrestore(&parent_spinlock, exceptions); + return pgt; +} + +static bool pgt_entry_matches(struct pgt *p, vaddr_t begin, vaddr_t last) +{ + if (!p) + return false; + if (last <= begin) + return false; + return core_is_buffer_inside(p->vabase, CORE_MMU_PGDIR_SIZE, begin, + last - begin); +} + +void pgt_flush_range(struct user_mode_ctx *uctx, vaddr_t begin, vaddr_t last) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct pgt *next_p = NULL; + struct pgt *p = NULL; + + /* + * Do the special case where the first element in the list is + * removed first. + */ + p = SLIST_FIRST(pgt_cache); + while (pgt_entry_matches(p, begin, last)) { + SLIST_REMOVE_HEAD(pgt_cache, link); + free_pgt(p); + p = SLIST_FIRST(pgt_cache); + } + + /* + * p either points to the first element in the list or it's NULL, + * if NULL the list is empty and we're done. + */ + if (!p) + return; + + /* + * Do the common case where the next element in the list is + * removed. + */ + while (true) { + next_p = SLIST_NEXT(p, link); + if (!next_p) + break; + if (pgt_entry_matches(next_p, begin, last)) { + SLIST_REMOVE_AFTER(p, link); + free_pgt(next_p); + continue; + } + + p = SLIST_NEXT(p, link); + } +} + +void pgt_flush(struct user_mode_ctx *uctx) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct pgt *p = NULL; + + while (true) { + p = SLIST_FIRST(pgt_cache); + if (!p) + break; + SLIST_REMOVE_HEAD(pgt_cache, link); + free_pgt(p); + } +} + +void pgt_clear_range(struct user_mode_ctx *uctx, vaddr_t begin, vaddr_t end) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct pgt *p = NULL; +#ifdef CFG_WITH_LPAE + uint64_t *tbl = NULL; +#else + uint32_t *tbl = NULL; +#endif + unsigned int idx = 0; + unsigned int n = 0; + + SLIST_FOREACH(p, pgt_cache, link) { + vaddr_t b = MAX(p->vabase, begin); + vaddr_t e = MIN(p->vabase + CORE_MMU_PGDIR_SIZE, end); + + if (b >= e) + continue; + + tbl = p->tbl; + idx = (b - p->vabase) / SMALL_PAGE_SIZE; + n = (e - b) / SMALL_PAGE_SIZE; + memset(tbl + idx, 0, n * sizeof(*tbl)); + } +} + +static struct pgt *prune_before_va(struct pgt_cache *pgt_cache, struct pgt *p, + struct pgt *pp, vaddr_t va) +{ + while (p && p->vabase < va) { + if (pp) { + assert(p == SLIST_NEXT(pp, link)); + SLIST_REMOVE_AFTER(pp, link); + free_pgt(p); + p = SLIST_NEXT(pp, link); + } else { + assert(p == SLIST_FIRST(pgt_cache)); + SLIST_REMOVE_HEAD(pgt_cache, link); + free_pgt(p); + p = SLIST_FIRST(pgt_cache); + } + } + + return p; +} + +bool pgt_check_avail(struct user_mode_ctx *uctx) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct vm_info *vm_info = &uctx->vm_info; + struct pgt *p = SLIST_FIRST(pgt_cache); + struct vm_region *r = NULL; + struct pgt *pp = NULL; + vaddr_t va = 0; + bool p_used = false; + + /* + * Prune unused tables. This is normally not needed since + * pgt_flush_range() does this too, but in the error path of for + * instance vm_remap() such calls may not be done. So for increased + * robustness remove all unused translation tables before we may + * allocate new ones. + */ + TAILQ_FOREACH(r, &vm_info->regions, link) { + for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); + va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { + if (!p_used) + p = prune_before_va(pgt_cache, p, pp, va); + if (!p) + goto prune_done; + + if (p->vabase < va) { + pp = p; + p = SLIST_NEXT(pp, link); + if (!p) + goto prune_done; + p_used = false; + } + + if (p->vabase == va) + p_used = true; + } + } +prune_done: + + p = SLIST_FIRST(pgt_cache); + pp = NULL; + TAILQ_FOREACH(r, &vm_info->regions, link) { + for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); + va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { + if (p && p->vabase < va) { + pp = p; + p = SLIST_NEXT(pp, link); + } + + if (p) { + if (p->vabase == va) + continue; + assert(p->vabase > va); + } + + p = alloc_pgt(va); + if (!p) + return false; + + if (pp) + SLIST_INSERT_AFTER(pp, p, link); + else + SLIST_INSERT_HEAD(pgt_cache, p, link); + } + } + + return true; +} +#else /* !CFG_CORE_PREALLOC_EL0_TBLS */ + +#if defined(CFG_WITH_PAGER) && !defined(CFG_WITH_LPAE) +static struct pgt_parent pgt_parents[PGT_CACHE_SIZE / PGT_NUM_PGT_PER_PAGE]; +#else + +static struct pgt_cache pgt_free_list = SLIST_HEAD_INITIALIZER(pgt_free_list); +#endif + +/* + * When a user TA context is temporarily unmapped the used struct pgt's of + * the context (page tables holding valid physical pages) are saved in this + * cache in the hope that it will remain in the cache when the context is + * mapped again. + */ +static struct pgt_cache pgt_cache_list = SLIST_HEAD_INITIALIZER(pgt_cache_list); + +static struct pgt pgt_entries[PGT_CACHE_SIZE]; + +static struct mutex pgt_mu = MUTEX_INITIALIZER; +static struct condvar pgt_cv = CONDVAR_INITIALIZER; + +#if defined(CFG_WITH_PAGER) && defined(CFG_WITH_LPAE) +/* + * Simple allocation of translation tables from pager, one translation + * table is one page. + */ +void pgt_init(void) +{ + size_t n; + + for (n = 0; n < PGT_CACHE_SIZE; n++) { + struct pgt *p = pgt_entries + n; + + p->tbl = tee_pager_alloc(PGT_SIZE); + SLIST_INSERT_HEAD(&pgt_free_list, p, link); + } +} +#elif defined(CFG_WITH_PAGER) && !defined(CFG_WITH_LPAE) +/* + * Four translation tables per page -> need to keep track of the page + * allocated from the pager. + */ +void pgt_init(void) +{ + size_t n; + size_t m; + + COMPILE_TIME_ASSERT(PGT_CACHE_SIZE % PGT_NUM_PGT_PER_PAGE == 0); + COMPILE_TIME_ASSERT(PGT_SIZE * PGT_NUM_PGT_PER_PAGE == SMALL_PAGE_SIZE); + + for (n = 0; n < ARRAY_SIZE(pgt_parents); n++) { + uint8_t *tbl = tee_pager_alloc(SMALL_PAGE_SIZE); + + SLIST_INIT(&pgt_parents[n].pgt_cache); + for (m = 0; m < PGT_NUM_PGT_PER_PAGE; m++) { + struct pgt *p = pgt_entries + + n * PGT_NUM_PGT_PER_PAGE + m; + + p->tbl = tbl + m * PGT_SIZE; + p->parent = &pgt_parents[n]; + SLIST_INSERT_HEAD(&pgt_parents[n].pgt_cache, p, link); + } + } +} +#else +/* Static allocation of translation tables */ +void pgt_init(void) +{ + /* + * We're putting this in .nozi.* instead of .bss because .nozi.* already + * has a large alignment, while .bss has a small alignment. The current + * link script is optimized for small alignment in .bss + */ + static uint8_t pgt_tables[PGT_CACHE_SIZE][PGT_SIZE] + __aligned(PGT_SIZE) __section(".nozi.pgt_cache"); + size_t n; + + for (n = 0; n < ARRAY_SIZE(pgt_tables); n++) { + struct pgt *p = pgt_entries + n; + + p->tbl = pgt_tables[n]; + SLIST_INSERT_HEAD(&pgt_free_list, p, link); + } +} +#endif + +#if defined(CFG_WITH_LPAE) || !defined(CFG_WITH_PAGER) +/* Simple allocation of translation tables from pager or static allocation */ +static struct pgt *pop_from_free_list(void) +{ + struct pgt *p = SLIST_FIRST(&pgt_free_list); + + if (p) { + SLIST_REMOVE_HEAD(&pgt_free_list, link); + memset(p->tbl, 0, PGT_SIZE); + p->populated = false; + } + return p; +} + +static void push_to_free_list(struct pgt *p) +{ + SLIST_INSERT_HEAD(&pgt_free_list, p, link); +#if defined(CFG_WITH_PAGER) + tee_pager_release_phys(p->tbl, PGT_SIZE); +#endif +} +#else +/* + * Four translation tables per page -> need to keep track of the page + * allocated from the pager. + */ +static struct pgt *pop_from_free_list(void) +{ + size_t n; + + for (n = 0; n < ARRAY_SIZE(pgt_parents); n++) { + struct pgt *p = SLIST_FIRST(&pgt_parents[n].pgt_cache); + + if (p) { + SLIST_REMOVE_HEAD(&pgt_parents[n].pgt_cache, link); + pgt_parents[n].num_used++; + memset(p->tbl, 0, PGT_SIZE); + p->populated = false; + return p; + } + } + return NULL; +} + +static void push_to_free_list(struct pgt *p) +{ + SLIST_INSERT_HEAD(&p->parent->pgt_cache, p, link); + assert(p->parent->num_used > 0); + p->parent->num_used--; + if (!p->parent->num_used) { + vaddr_t va = (vaddr_t)p->tbl & ~SMALL_PAGE_MASK; + + tee_pager_release_phys((void *)va, SMALL_PAGE_SIZE); + } +} +#endif + +static void push_to_cache_list(struct pgt *pgt) +{ + SLIST_INSERT_HEAD(&pgt_cache_list, pgt, link); +} + +static bool match_pgt(struct pgt *pgt, vaddr_t vabase, void *ctx) +{ + return pgt->ctx == ctx && pgt->vabase == vabase; +} + +static struct pgt *pop_from_cache_list(vaddr_t vabase, void *ctx) +{ + struct pgt *pgt; + struct pgt *p; + + pgt = SLIST_FIRST(&pgt_cache_list); + if (!pgt) + return NULL; + if (match_pgt(pgt, vabase, ctx)) { + SLIST_REMOVE_HEAD(&pgt_cache_list, link); + return pgt; + } + + while (true) { + p = SLIST_NEXT(pgt, link); + if (!p) + break; + if (match_pgt(p, vabase, ctx)) { + SLIST_REMOVE_AFTER(pgt, link); + break; + } + pgt = p; + } + return p; +} + +static uint16_t get_num_used_entries(struct pgt *pgt __maybe_unused) +{ +#ifdef CFG_PAGED_USER_TA + return pgt->num_used_entries; +#else + return 0; +#endif +} + +static struct pgt *pop_least_used_from_cache_list(void) +{ + struct pgt *pgt = NULL; + struct pgt *p_prev = NULL; + size_t least_used = 0; + size_t next_used = 0; + + pgt = SLIST_FIRST(&pgt_cache_list); + if (!pgt) + return NULL; + least_used = get_num_used_entries(pgt); + + while (true) { + if (!SLIST_NEXT(pgt, link)) + break; + next_used = get_num_used_entries(SLIST_NEXT(pgt, link)); + if (next_used <= least_used) { + p_prev = pgt; + least_used = next_used; + } + pgt = SLIST_NEXT(pgt, link); + } + + if (p_prev) { + pgt = SLIST_NEXT(p_prev, link); + SLIST_REMOVE_AFTER(p_prev, link); + } else { + pgt = SLIST_FIRST(&pgt_cache_list); + SLIST_REMOVE_HEAD(&pgt_cache_list, link); + } + return pgt; +} + +static void pgt_free_unlocked(struct pgt_cache *pgt_cache) +{ + while (!SLIST_EMPTY(pgt_cache)) { + struct pgt *p = SLIST_FIRST(pgt_cache); + + SLIST_REMOVE_HEAD(pgt_cache, link); + + /* + * With paging enabled we free all tables which doesn't + * refer to any paged pages any longer. This reduces the + * pressure the pool of physical pages. + */ + if (IS_ENABLED(CFG_PAGED_USER_TA) && !get_num_used_entries(p)) { + tee_pager_pgt_save_and_release_entries(p); + p->ctx = NULL; + p->vabase = 0; + + push_to_free_list(p); + continue; + } + + push_to_cache_list(p); + } +} + +static struct pgt *pop_from_some_list(vaddr_t vabase, void *ctx) +{ + struct pgt *p = pop_from_cache_list(vabase, ctx); + + if (p) + return p; + p = pop_from_free_list(); + if (!p) { + p = pop_least_used_from_cache_list(); + if (!p) + return NULL; + tee_pager_pgt_save_and_release_entries(p); + memset(p->tbl, 0, PGT_SIZE); + p->populated = false; + } + p->ctx = ctx; + p->vabase = vabase; + return p; +} + +void pgt_flush(struct user_mode_ctx *uctx) +{ + struct ts_ctx *ctx = uctx->ts_ctx; + struct pgt *pp = NULL; + struct pgt *p = NULL; + + mutex_lock(&pgt_mu); + + while (true) { + p = SLIST_FIRST(&pgt_cache_list); + if (!p) + goto out; + if (p->ctx != ctx) + break; + SLIST_REMOVE_HEAD(&pgt_cache_list, link); + tee_pager_pgt_save_and_release_entries(p); + p->ctx = NULL; + p->vabase = 0; + push_to_free_list(p); + } + + pp = p; + while (true) { + p = SLIST_NEXT(pp, link); + if (!p) + break; + if (p->ctx == ctx) { + SLIST_REMOVE_AFTER(pp, link); + tee_pager_pgt_save_and_release_entries(p); + p->ctx = NULL; + p->vabase = 0; + push_to_free_list(p); + } else { + pp = p; + } + } + +out: + mutex_unlock(&pgt_mu); +} + +static void flush_pgt_entry(struct pgt *p) +{ + tee_pager_pgt_save_and_release_entries(p); + p->ctx = NULL; + p->vabase = 0; +} + +static bool pgt_entry_matches(struct pgt *p, void *ctx, vaddr_t begin, + vaddr_t last) +{ + if (!p) + return false; + if (p->ctx != ctx) + return false; + if (last <= begin) + return false; + if (!core_is_buffer_inside(p->vabase, CORE_MMU_PGDIR_SIZE, begin, + last - begin)) + return false; + + return true; +} + +static void flush_ctx_range_from_list(struct pgt_cache *pgt_cache, void *ctx, + vaddr_t begin, vaddr_t last) +{ + struct pgt *p; + struct pgt *next_p; + + /* + * Do the special case where the first element in the list is + * removed first. + */ + p = SLIST_FIRST(pgt_cache); + while (pgt_entry_matches(p, ctx, begin, last)) { + flush_pgt_entry(p); + SLIST_REMOVE_HEAD(pgt_cache, link); + push_to_free_list(p); + p = SLIST_FIRST(pgt_cache); + } + + /* + * p either points to the first element in the list or it's NULL, + * if NULL the list is empty and we're done. + */ + if (!p) + return; + + /* + * Do the common case where the next element in the list is + * removed. + */ + while (true) { + next_p = SLIST_NEXT(p, link); + if (!next_p) + break; + if (pgt_entry_matches(next_p, ctx, begin, last)) { + flush_pgt_entry(next_p); + SLIST_REMOVE_AFTER(p, link); + push_to_free_list(next_p); + continue; + } + + p = SLIST_NEXT(p, link); + } +} + +void pgt_flush_range(struct user_mode_ctx *uctx, vaddr_t begin, vaddr_t last) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct ts_ctx *ctx = uctx->ts_ctx; + + mutex_lock(&pgt_mu); + + flush_ctx_range_from_list(pgt_cache, ctx, begin, last); + flush_ctx_range_from_list(&pgt_cache_list, ctx, begin, last); + + condvar_broadcast(&pgt_cv); + mutex_unlock(&pgt_mu); +} + +static void clear_ctx_range_from_list(struct pgt_cache *pgt_cache, + void *ctx, vaddr_t begin, vaddr_t end) +{ + struct pgt *p = NULL; +#ifdef CFG_WITH_LPAE + uint64_t *tbl = NULL; +#else + uint32_t *tbl = NULL; +#endif + unsigned int idx = 0; + unsigned int n = 0; + + SLIST_FOREACH(p, pgt_cache, link) { + vaddr_t b = MAX(p->vabase, begin); + vaddr_t e = MIN(p->vabase + CORE_MMU_PGDIR_SIZE, end); + + if (p->ctx != ctx) + continue; + if (b >= e) + continue; + + tbl = p->tbl; + idx = (b - p->vabase) / SMALL_PAGE_SIZE; + n = (e - b) / SMALL_PAGE_SIZE; + memset(tbl + idx, 0, n * sizeof(*tbl)); + } +} + +void pgt_clear_range(struct user_mode_ctx *uctx, vaddr_t begin, vaddr_t end) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct ts_ctx *ctx = uctx->ts_ctx; + + mutex_lock(&pgt_mu); + + clear_ctx_range_from_list(pgt_cache, ctx, begin, end); + clear_ctx_range_from_list(&pgt_cache_list, ctx, begin, end); + + mutex_unlock(&pgt_mu); +} + +static bool pgt_alloc_unlocked(struct pgt_cache *pgt_cache, struct ts_ctx *ctx, + struct vm_info *vm_info) +{ + struct vm_region *r = NULL; + struct pgt *pp = NULL; + struct pgt *p = NULL; + vaddr_t va = 0; + + TAILQ_FOREACH(r, &vm_info->regions, link) { + for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); + va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { + if (p && p->vabase == va) + continue; + p = pop_from_some_list(va, ctx); + if (!p) { + pgt_free_unlocked(pgt_cache); + return false; + } + if (pp) + SLIST_INSERT_AFTER(pp, p, link); + else + SLIST_INSERT_HEAD(pgt_cache, p, link); + pp = p; + } + } + + return true; +} + +bool pgt_check_avail(struct user_mode_ctx *uctx) +{ + struct vm_info *vm_info = &uctx->vm_info; + struct vm_region *r = NULL; + size_t tbl_count = 0; + vaddr_t last_va = 0; + vaddr_t va = 0; + + TAILQ_FOREACH(r, &vm_info->regions, link) { + for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); + va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { + if (va == last_va) + continue; + tbl_count++; + last_va = va; + } + } + + return tbl_count <= PGT_CACHE_SIZE; +} + +void pgt_get_all(struct user_mode_ctx *uctx) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + struct vm_info *vm_info = &uctx->vm_info; + + if (TAILQ_EMPTY(&vm_info->regions)) + return; + + mutex_lock(&pgt_mu); + + pgt_free_unlocked(pgt_cache); + while (!pgt_alloc_unlocked(pgt_cache, uctx->ts_ctx, vm_info)) { + assert(pgt_check_avail(uctx)); + DMSG("Waiting for page tables"); + condvar_broadcast(&pgt_cv); + condvar_wait(&pgt_cv, &pgt_mu); + } + + mutex_unlock(&pgt_mu); +} + +void pgt_put_all(struct user_mode_ctx *uctx) +{ + struct pgt_cache *pgt_cache = &uctx->pgt_cache; + + if (SLIST_EMPTY(pgt_cache)) + return; + + mutex_lock(&pgt_mu); + + pgt_free_unlocked(pgt_cache); + + condvar_broadcast(&pgt_cv); + mutex_unlock(&pgt_mu); +} + +struct pgt *pgt_pop_from_cache_list(vaddr_t vabase, struct ts_ctx *ctx) +{ + struct pgt *pgt = NULL; + + mutex_lock(&pgt_mu); + pgt = pop_from_cache_list(vabase, ctx); + mutex_unlock(&pgt_mu); + + return pgt; +} + +void pgt_push_to_cache_list(struct pgt *pgt) +{ + mutex_lock(&pgt_mu); + push_to_cache_list(pgt); + mutex_unlock(&pgt_mu); +} + +#endif /* !CFG_CORE_PREALLOC_EL0_TBLS */ diff --git a/optee_os/core/mm/sub.mk b/optee_os/core/mm/sub.mk new file mode 100644 index 0000000..7d3c5ea --- /dev/null +++ b/optee_os/core/mm/sub.mk @@ -0,0 +1,9 @@ +srcs-y += mobj.c +srcs-y += fobj.c +cflags-fobj.c-$(CFG_CORE_PAGE_TAG_AND_IV) := -Wno-missing-noreturn +srcs-y += file.c +srcs-y += vm.c +srcs-y += core_mmu.c +srcs-y += pgt_cache.c +srcs-y += tee_mm.c + diff --git a/optee_os/core/mm/tee_mm.c b/optee_os/core/mm/tee_mm.c new file mode 100644 index 0000000..c61306d --- /dev/null +++ b/optee_os/core/mm/tee_mm.c @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include + +static void *pmalloc(tee_mm_pool_t *pool, size_t size) +{ + if (pool->flags & TEE_MM_POOL_NEX_MALLOC) + return nex_malloc(size); + else + return malloc(size); +} + +static void *pcalloc(tee_mm_pool_t *pool, size_t num_el, size_t size) +{ + if (pool->flags & TEE_MM_POOL_NEX_MALLOC) + return nex_calloc(num_el, size); + else + return calloc(num_el, size); +} + +static void pfree(tee_mm_pool_t *pool, void *ptr) +{ + if (pool->flags & TEE_MM_POOL_NEX_MALLOC) + nex_free(ptr); + else + free(ptr); +} + +bool tee_mm_init(tee_mm_pool_t *pool, paddr_t lo, paddr_size_t size, + uint8_t shift, uint32_t flags) +{ + paddr_size_t rounded = 0; + paddr_t initial_lo = lo; + + if (pool == NULL) + return false; + + lo = ROUNDUP(lo, 1 << shift); + rounded = lo - initial_lo; + size = ROUNDDOWN(size - rounded, 1 << shift); + + assert(((uint64_t)size >> shift) < (uint64_t)UINT32_MAX); + + pool->lo = lo; + pool->size = size; + pool->shift = shift; + pool->flags = flags; + pool->entry = pcalloc(pool, 1, sizeof(tee_mm_entry_t)); + + if (pool->entry == NULL) + return false; + + if (pool->flags & TEE_MM_POOL_HI_ALLOC) + pool->entry->offset = ((size - 1) >> shift) + 1; + + pool->entry->pool = pool; + pool->lock = SPINLOCK_UNLOCK; + + return true; +} + +void tee_mm_final(tee_mm_pool_t *pool) +{ + if (pool == NULL || pool->entry == NULL) + return; + + while (pool->entry->next != NULL) + tee_mm_free(pool->entry->next); + pfree(pool, pool->entry); + pool->entry = NULL; +} + +static void tee_mm_add(tee_mm_entry_t *p, tee_mm_entry_t *nn) +{ + /* add to list */ + nn->next = p->next; + p->next = nn; +} + +#ifdef CFG_WITH_STATS +static size_t tee_mm_stats_allocated(tee_mm_pool_t *pool) +{ + tee_mm_entry_t *entry; + uint32_t sz = 0; + + if (!pool) + return 0; + + entry = pool->entry; + while (entry) { + sz += entry->size; + entry = entry->next; + } + + return sz << pool->shift; +} + +void tee_mm_get_pool_stats(tee_mm_pool_t *pool, struct malloc_stats *stats, + bool reset) +{ + uint32_t exceptions; + + if (!pool) + return; + + memset(stats, 0, sizeof(*stats)); + + exceptions = cpu_spin_lock_xsave(&pool->lock); + + stats->size = pool->size; + stats->max_allocated = pool->max_allocated; + stats->allocated = tee_mm_stats_allocated(pool); + + if (reset) + pool->max_allocated = 0; + cpu_spin_unlock_xrestore(&pool->lock, exceptions); +} + +static void update_max_allocated(tee_mm_pool_t *pool) +{ + size_t sz = tee_mm_stats_allocated(pool); + + if (sz > pool->max_allocated) + pool->max_allocated = sz; +} +#else /* CFG_WITH_STATS */ +static inline void update_max_allocated(tee_mm_pool_t *pool __unused) +{ +} +#endif /* CFG_WITH_STATS */ + +tee_mm_entry_t *tee_mm_alloc(tee_mm_pool_t *pool, size_t size) +{ + size_t psize; + tee_mm_entry_t *entry; + tee_mm_entry_t *nn; + size_t remaining; + uint32_t exceptions; + + /* Check that pool is initialized */ + if (!pool || !pool->entry) + return NULL; + + nn = pmalloc(pool, sizeof(tee_mm_entry_t)); + if (!nn) + return NULL; + + exceptions = cpu_spin_lock_xsave(&pool->lock); + + entry = pool->entry; + if (!size) + psize = 0; + else + psize = ((size - 1) >> pool->shift) + 1; + + /* find free slot */ + if (pool->flags & TEE_MM_POOL_HI_ALLOC) { + while (entry->next != NULL && psize > + (entry->offset - entry->next->offset - + entry->next->size)) + entry = entry->next; + } else { + while (entry->next != NULL && psize > + (entry->next->offset - entry->size - entry->offset)) + entry = entry->next; + } + + /* check if we have enough memory */ + if (entry->next == NULL) { + if (pool->flags & TEE_MM_POOL_HI_ALLOC) { + /* + * entry->offset is a "block count" offset from + * pool->lo. The byte offset is + * (entry->offset << pool->shift). + * In the HI_ALLOC allocation scheme the memory is + * allocated from the end of the segment, thus to + * validate there is sufficient memory validate that + * (entry->offset << pool->shift) > size. + */ + if ((entry->offset << pool->shift) < size) { + /* out of memory */ + goto err; + } + } else { + if (!pool->size) + panic("invalid pool"); + + remaining = pool->size; + remaining -= ((entry->offset + entry->size) << + pool->shift); + + if (remaining < size) { + /* out of memory */ + goto err; + } + } + } + + tee_mm_add(entry, nn); + + if (pool->flags & TEE_MM_POOL_HI_ALLOC) + nn->offset = entry->offset - psize; + else + nn->offset = entry->offset + entry->size; + nn->size = psize; + nn->pool = pool; + + update_max_allocated(pool); + + cpu_spin_unlock_xrestore(&pool->lock, exceptions); + return nn; +err: + cpu_spin_unlock_xrestore(&pool->lock, exceptions); + pfree(pool, nn); + return NULL; +} + +static inline bool fit_in_gap(tee_mm_pool_t *pool, tee_mm_entry_t *e, + paddr_t offslo, paddr_t offshi) +{ + if (pool->flags & TEE_MM_POOL_HI_ALLOC) { + if (offshi > e->offset || + (e->next != NULL && + (offslo < e->next->offset + e->next->size)) || + (offshi << pool->shift) - 1 > pool->size) + /* memory not available */ + return false; + } else { + if (offslo < (e->offset + e->size) || + (e->next != NULL && (offshi > e->next->offset)) || + (offshi << pool->shift) > pool->size) + /* memory not available */ + return false; + } + + return true; +} + +tee_mm_entry_t *tee_mm_alloc2(tee_mm_pool_t *pool, paddr_t base, size_t size) +{ + tee_mm_entry_t *entry; + paddr_t offslo; + paddr_t offshi; + tee_mm_entry_t *mm; + uint32_t exceptions; + + /* Check that pool is initialized */ + if (!pool || !pool->entry) + return NULL; + + /* Wrapping and sanity check */ + if ((base + size) < base || base < pool->lo) + return NULL; + + mm = pmalloc(pool, sizeof(tee_mm_entry_t)); + if (!mm) + return NULL; + + exceptions = cpu_spin_lock_xsave(&pool->lock); + + entry = pool->entry; + offslo = (base - pool->lo) >> pool->shift; + offshi = ((base - pool->lo + size - 1) >> pool->shift) + 1; + + /* find slot */ + if (pool->flags & TEE_MM_POOL_HI_ALLOC) { + while (entry->next != NULL && + offshi < entry->next->offset + entry->next->size) + entry = entry->next; + } else { + while (entry->next != NULL && offslo > entry->next->offset) + entry = entry->next; + } + + /* Check that memory is available */ + if (!fit_in_gap(pool, entry, offslo, offshi)) + goto err; + + tee_mm_add(entry, mm); + + mm->offset = offslo; + mm->size = offshi - offslo; + mm->pool = pool; + + update_max_allocated(pool); + cpu_spin_unlock_xrestore(&pool->lock, exceptions); + return mm; +err: + cpu_spin_unlock_xrestore(&pool->lock, exceptions); + pfree(pool, mm); + return NULL; +} + +void tee_mm_free(tee_mm_entry_t *p) +{ + tee_mm_entry_t *entry; + uint32_t exceptions; + + if (!p || !p->pool) + return; + + exceptions = cpu_spin_lock_xsave(&p->pool->lock); + entry = p->pool->entry; + + /* remove entry from list */ + while (entry->next != NULL && entry->next != p) + entry = entry->next; + + if (!entry->next) + panic("invalid mm_entry"); + + entry->next = entry->next->next; + cpu_spin_unlock_xrestore(&p->pool->lock, exceptions); + + pfree(p->pool, p); +} + +size_t tee_mm_get_bytes(const tee_mm_entry_t *mm) +{ + if (!mm || !mm->pool) + return 0; + else + return mm->size << mm->pool->shift; +} + +bool tee_mm_addr_is_within_range(const tee_mm_pool_t *pool, paddr_t addr) +{ + return pool && addr >= pool->lo && + addr <= (pool->lo + (pool->size - 1)); +} + +bool tee_mm_is_empty(tee_mm_pool_t *pool) +{ + bool ret; + uint32_t exceptions; + + if (pool == NULL || pool->entry == NULL) + return true; + + exceptions = cpu_spin_lock_xsave(&pool->lock); + ret = pool->entry == NULL || pool->entry->next == NULL; + cpu_spin_unlock_xrestore(&pool->lock, exceptions); + + return ret; +} + +/* Physical Secure DDR pool */ +tee_mm_pool_t tee_mm_sec_ddr; + +/* Virtual eSRAM pool */ +tee_mm_pool_t tee_mm_vcore; + +/* Shared memory pool */ +tee_mm_pool_t tee_mm_shm; + +tee_mm_entry_t *tee_mm_find(const tee_mm_pool_t *pool, paddr_t addr) +{ + tee_mm_entry_t *entry = pool->entry; + uint16_t offset = (addr - pool->lo) >> pool->shift; + uint32_t exceptions; + + if (!tee_mm_addr_is_within_range(pool, addr)) + return NULL; + + exceptions = cpu_spin_lock_xsave(&((tee_mm_pool_t *)pool)->lock); + + while (entry->next != NULL) { + entry = entry->next; + + if ((offset >= entry->offset) && + (offset < (entry->offset + entry->size))) { + cpu_spin_unlock_xrestore(&((tee_mm_pool_t *)pool)->lock, + exceptions); + return entry; + } + } + + cpu_spin_unlock_xrestore(&((tee_mm_pool_t *)pool)->lock, exceptions); + return NULL; +} + +uintptr_t tee_mm_get_smem(const tee_mm_entry_t *mm) +{ + return (mm->offset << mm->pool->shift) + mm->pool->lo; +} diff --git a/optee_os/core/mm/vm.c b/optee_os/core/mm/vm.c new file mode 100644 index 0000000..2863925 --- /dev/null +++ b/optee_os/core/mm/vm.c @@ -0,0 +1,1345 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2021, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2021, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CFG_PL310 +#include +#endif + +#define TEE_MMU_UDATA_ATTR (TEE_MATTR_VALID_BLOCK | \ + TEE_MATTR_PRW | TEE_MATTR_URW | \ + TEE_MATTR_SECURE) +#define TEE_MMU_UCODE_ATTR (TEE_MATTR_VALID_BLOCK | \ + TEE_MATTR_PRW | TEE_MATTR_URWX | \ + TEE_MATTR_SECURE) + +#define TEE_MMU_UCACHE_DEFAULT_ATTR (TEE_MATTR_MEM_TYPE_CACHED << \ + TEE_MATTR_MEM_TYPE_SHIFT) + +static vaddr_t select_va_in_range(const struct vm_region *prev_reg, + const struct vm_region *next_reg, + const struct vm_region *reg, + size_t pad_begin, size_t pad_end, + size_t granul) +{ + const uint32_t f = VM_FLAG_EPHEMERAL | VM_FLAG_PERMANENT | + VM_FLAG_SHAREABLE; + vaddr_t begin_va = 0; + vaddr_t end_va = 0; + size_t pad = 0; + + /* + * Insert an unmapped entry to separate regions with differing + * VM_FLAG_EPHEMERAL, VM_FLAG_PERMANENT or VM_FLAG_SHAREABLE + * bits as they never are to be contiguous with another region. + */ + if (prev_reg->flags && (prev_reg->flags & f) != (reg->flags & f)) + pad = SMALL_PAGE_SIZE; + else + pad = 0; + +#ifndef CFG_WITH_LPAE + if ((prev_reg->attr & TEE_MATTR_SECURE) != + (reg->attr & TEE_MATTR_SECURE)) + granul = CORE_MMU_PGDIR_SIZE; +#endif + + if (ADD_OVERFLOW(prev_reg->va, prev_reg->size, &begin_va) || + ADD_OVERFLOW(begin_va, pad_begin, &begin_va) || + ADD_OVERFLOW(begin_va, pad, &begin_va) || + ROUNDUP_OVERFLOW(begin_va, granul, &begin_va)) + return 0; + + if (reg->va) { + if (reg->va < begin_va) + return 0; + begin_va = reg->va; + } + + if (next_reg->flags && (next_reg->flags & f) != (reg->flags & f)) + pad = SMALL_PAGE_SIZE; + else + pad = 0; + +#ifndef CFG_WITH_LPAE + if ((next_reg->attr & TEE_MATTR_SECURE) != + (reg->attr & TEE_MATTR_SECURE)) + granul = CORE_MMU_PGDIR_SIZE; +#endif + if (ADD_OVERFLOW(begin_va, reg->size, &end_va) || + ADD_OVERFLOW(end_va, pad_end, &end_va) || + ADD_OVERFLOW(end_va, pad, &end_va) || + ROUNDUP_OVERFLOW(end_va, granul, &end_va)) + return 0; + + if (end_va <= next_reg->va) { + assert(!reg->va || reg->va == begin_va); + return begin_va; + } + + return 0; +} + +static TEE_Result alloc_pgt(struct user_mode_ctx *uctx) +{ + struct thread_specific_data *tsd __maybe_unused; + + if (!pgt_check_avail(uctx)) { + EMSG("Page tables are not available"); + return TEE_ERROR_OUT_OF_MEMORY; + } + +#ifdef CFG_PAGED_USER_TA + tsd = thread_get_tsd(); + if (uctx->ts_ctx == tsd->ctx) { + /* + * The supplied utc is the current active utc, allocate the + * page tables too as the pager needs to use them soon. + */ + pgt_get_all(uctx); + } +#endif + + return TEE_SUCCESS; +} + +static void rem_um_region(struct user_mode_ctx *uctx, struct vm_region *r) +{ + vaddr_t begin = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); + vaddr_t last = ROUNDUP(r->va + r->size, CORE_MMU_PGDIR_SIZE); + struct vm_region *r2 = NULL; + + if (mobj_is_paged(r->mobj)) { + tee_pager_rem_um_region(uctx, r->va, r->size); + } else { + pgt_clear_range(uctx, r->va, r->va + r->size); + tlbi_va_range_asid(r->va, r->size, SMALL_PAGE_SIZE, + uctx->vm_info.asid); + } + + /* + * Figure out how much virtual memory on a CORE_MMU_PGDIR_SIZE + * grunalarity can be freed. Only completely unused + * CORE_MMU_PGDIR_SIZE ranges can be supplied to pgt_flush_range(). + * + * Note that there's is no margin for error here, both flushing too + * many or too few translation tables can be fatal. + */ + r2 = TAILQ_NEXT(r, link); + if (r2) + last = MIN(last, ROUNDDOWN(r2->va, CORE_MMU_PGDIR_SIZE)); + + r2 = TAILQ_PREV(r, vm_region_head, link); + if (r2) + begin = MAX(begin, + ROUNDUP(r2->va + r2->size, CORE_MMU_PGDIR_SIZE)); + + if (begin < last) + pgt_flush_range(uctx, begin, last); +} + +static void set_pa_range(struct core_mmu_table_info *ti, vaddr_t va, + paddr_t pa, size_t size, uint32_t attr) +{ + unsigned int end = core_mmu_va2idx(ti, va + size); + unsigned int idx = core_mmu_va2idx(ti, va); + + while (idx < end) { + core_mmu_set_entry(ti, idx, pa, attr); + idx++; + pa += BIT64(ti->shift); + } +} + +static void set_reg_in_table(struct core_mmu_table_info *ti, + struct vm_region *r) +{ + vaddr_t va = MAX(r->va, ti->va_base); + vaddr_t end = MIN(r->va + r->size, ti->va_base + CORE_MMU_PGDIR_SIZE); + size_t sz = MIN(end - va, mobj_get_phys_granule(r->mobj)); + size_t granule = BIT(ti->shift); + size_t offset = 0; + paddr_t pa = 0; + + while (va < end) { + offset = va - r->va + r->offset; + if (mobj_get_pa(r->mobj, offset, granule, &pa)) + panic("Failed to get PA"); + set_pa_range(ti, va, pa, sz, r->attr); + va += sz; + } +} + +static void set_um_region(struct user_mode_ctx *uctx, struct vm_region *r) +{ + struct pgt *p = SLIST_FIRST(&uctx->pgt_cache); + struct core_mmu_table_info ti = { }; + + assert(!mobj_is_paged(r->mobj)); + + core_mmu_set_info_table(&ti, CORE_MMU_PGDIR_LEVEL, 0, NULL); + + if (p) { + /* All the pgts are already allocated, update in place */ + do { + ti.va_base = p->vabase; + ti.table = p->tbl; + set_reg_in_table(&ti, r); + p = SLIST_NEXT(p, link); + } while (p); + } else { + /* + * We may have a few pgts in the cache list, update the + * ones found. + */ + for (ti.va_base = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); + ti.va_base < r->va + r->size; + ti.va_base += CORE_MMU_PGDIR_SIZE) { + p = pgt_pop_from_cache_list(ti.va_base, uctx->ts_ctx); + if (!p) + continue; + ti.table = p->tbl; + set_reg_in_table(&ti, r); + pgt_push_to_cache_list(p); + } + } +} + +static TEE_Result umap_add_region(struct vm_info *vmi, struct vm_region *reg, + size_t pad_begin, size_t pad_end, + size_t align) +{ + struct vm_region dummy_first_reg = { }; + struct vm_region dummy_last_reg = { }; + struct vm_region *r = NULL; + struct vm_region *prev_r = NULL; + vaddr_t va_range_base = 0; + size_t va_range_size = 0; + size_t granul; + vaddr_t va = 0; + size_t offs_plus_size = 0; + + core_mmu_get_user_va_range(&va_range_base, &va_range_size); + dummy_first_reg.va = va_range_base; + dummy_last_reg.va = va_range_base + va_range_size; + + /* Check alignment, it has to be at least SMALL_PAGE based */ + if ((reg->va | reg->size | pad_begin | pad_end) & SMALL_PAGE_MASK) + return TEE_ERROR_ACCESS_CONFLICT; + + /* Check that the mobj is defined for the entire range */ + if (ADD_OVERFLOW(reg->offset, reg->size, &offs_plus_size)) + return TEE_ERROR_BAD_PARAMETERS; + if (offs_plus_size > ROUNDUP(reg->mobj->size, SMALL_PAGE_SIZE)) + return TEE_ERROR_BAD_PARAMETERS; + + granul = MAX(align, SMALL_PAGE_SIZE); + if (!IS_POWER_OF_TWO(granul)) + return TEE_ERROR_BAD_PARAMETERS; + + prev_r = &dummy_first_reg; + TAILQ_FOREACH(r, &vmi->regions, link) { + va = select_va_in_range(prev_r, r, reg, pad_begin, pad_end, + granul); + if (va) { + reg->va = va; + TAILQ_INSERT_BEFORE(r, reg, link); + return TEE_SUCCESS; + } + prev_r = r; + } + + r = TAILQ_LAST(&vmi->regions, vm_region_head); + if (!r) + r = &dummy_first_reg; + va = select_va_in_range(r, &dummy_last_reg, reg, pad_begin, pad_end, + granul); + if (va) { + reg->va = va; + TAILQ_INSERT_TAIL(&vmi->regions, reg, link); + return TEE_SUCCESS; + } + + return TEE_ERROR_ACCESS_CONFLICT; +} + +TEE_Result vm_map_pad(struct user_mode_ctx *uctx, vaddr_t *va, size_t len, + uint32_t prot, uint32_t flags, struct mobj *mobj, + size_t offs, size_t pad_begin, size_t pad_end, + size_t align) +{ + TEE_Result res = TEE_SUCCESS; + struct vm_region *reg = NULL; + uint32_t attr = 0; + + if (prot & ~TEE_MATTR_PROT_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + reg = calloc(1, sizeof(*reg)); + if (!reg) + return TEE_ERROR_OUT_OF_MEMORY; + + if (!mobj_is_paged(mobj)) { + uint32_t mem_type = 0; + + res = mobj_get_mem_type(mobj, &mem_type); + if (res) + goto err_free_reg; + attr |= mem_type << TEE_MATTR_MEM_TYPE_SHIFT; + } + attr |= TEE_MATTR_VALID_BLOCK; + if (mobj_is_secure(mobj)) + attr |= TEE_MATTR_SECURE; + + reg->mobj = mobj_get(mobj); + reg->offset = offs; + reg->va = *va; + reg->size = ROUNDUP(len, SMALL_PAGE_SIZE); + reg->attr = attr | prot; + reg->flags = flags; + + res = umap_add_region(&uctx->vm_info, reg, pad_begin, pad_end, align); + if (res) + goto err_put_mobj; + + res = alloc_pgt(uctx); + if (res) + goto err_rem_reg; + + if (mobj_is_paged(mobj)) { + struct fobj *fobj = mobj_get_fobj(mobj); + + if (!fobj) { + res = TEE_ERROR_GENERIC; + goto err_rem_reg; + } + + res = tee_pager_add_um_region(uctx, reg->va, fobj, prot); + fobj_put(fobj); + if (res) + goto err_rem_reg; + } else { + set_um_region(uctx, reg); + } + + /* + * If the context currently is active set it again to update + * the mapping. + */ + if (thread_get_tsd()->ctx == uctx->ts_ctx) + vm_set_ctx(uctx->ts_ctx); + + *va = reg->va; + + return TEE_SUCCESS; + +err_rem_reg: + TAILQ_REMOVE(&uctx->vm_info.regions, reg, link); +err_put_mobj: + mobj_put(reg->mobj); +err_free_reg: + free(reg); + return res; +} + +static struct vm_region *find_vm_region(struct vm_info *vm_info, vaddr_t va) +{ + struct vm_region *r = NULL; + + TAILQ_FOREACH(r, &vm_info->regions, link) + if (va >= r->va && va < r->va + r->size) + return r; + + return NULL; +} + +static bool va_range_is_contiguous(struct vm_region *r0, vaddr_t va, + size_t len, + bool (*cmp_regs)(const struct vm_region *r0, + const struct vm_region *r, + const struct vm_region *rn)) +{ + struct vm_region *r = r0; + vaddr_t end_va = 0; + + if (ADD_OVERFLOW(va, len, &end_va)) + return false; + + while (true) { + struct vm_region *r_next = TAILQ_NEXT(r, link); + vaddr_t r_end_va = r->va + r->size; + + if (r_end_va >= end_va) + return true; + if (!r_next) + return false; + if (r_end_va != r_next->va) + return false; + if (cmp_regs && !cmp_regs(r0, r, r_next)) + return false; + r = r_next; + } +} + +static TEE_Result split_vm_region(struct user_mode_ctx *uctx, + struct vm_region *r, vaddr_t va) +{ + struct vm_region *r2 = NULL; + size_t diff = va - r->va; + + assert(diff && diff < r->size); + + r2 = calloc(1, sizeof(*r2)); + if (!r2) + return TEE_ERROR_OUT_OF_MEMORY; + + if (mobj_is_paged(r->mobj)) { + TEE_Result res = tee_pager_split_um_region(uctx, va); + + if (res) { + free(r2); + return res; + } + } + + r2->mobj = mobj_get(r->mobj); + r2->offset = r->offset + diff; + r2->va = va; + r2->size = r->size - diff; + r2->attr = r->attr; + r2->flags = r->flags; + + r->size = diff; + + TAILQ_INSERT_AFTER(&uctx->vm_info.regions, r, r2, link); + + return TEE_SUCCESS; +} + +static TEE_Result split_vm_range(struct user_mode_ctx *uctx, vaddr_t va, + size_t len, + bool (*cmp_regs)(const struct vm_region *r0, + const struct vm_region *r, + const struct vm_region *rn), + struct vm_region **r0_ret) +{ + TEE_Result res = TEE_SUCCESS; + struct vm_region *r = NULL; + vaddr_t end_va = 0; + + if ((va | len) & SMALL_PAGE_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + if (ADD_OVERFLOW(va, len, &end_va)) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * Find first vm_region in range and check that the entire range is + * contiguous. + */ + r = find_vm_region(&uctx->vm_info, va); + if (!r || !va_range_is_contiguous(r, va, len, cmp_regs)) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * If needed split regions so that va and len covers only complete + * regions. + */ + if (va != r->va) { + res = split_vm_region(uctx, r, va); + if (res) + return res; + r = TAILQ_NEXT(r, link); + } + + *r0_ret = r; + r = find_vm_region(&uctx->vm_info, va + len - 1); + if (!r) + return TEE_ERROR_BAD_PARAMETERS; + if (end_va != r->va + r->size) { + res = split_vm_region(uctx, r, end_va); + if (res) + return res; + } + + return TEE_SUCCESS; +} + +static void merge_vm_range(struct user_mode_ctx *uctx, vaddr_t va, size_t len) +{ + struct vm_region *r_next = NULL; + struct vm_region *r = NULL; + vaddr_t end_va = 0; + + if (ADD_OVERFLOW(va, len, &end_va)) + return; + + tee_pager_merge_um_region(uctx, va, len); + + for (r = TAILQ_FIRST(&uctx->vm_info.regions);; r = r_next) { + r_next = TAILQ_NEXT(r, link); + if (!r_next) + return; + + /* Try merging with the region just before va */ + if (r->va + r->size < va) + continue; + + /* + * If r->va is well past our range we're done. + * Note that if it's just the page after our range we'll + * try to merge. + */ + if (r->va > end_va) + return; + + if (r->va + r->size != r_next->va) + continue; + if (r->mobj != r_next->mobj || + r->flags != r_next->flags || + r->attr != r_next->attr) + continue; + if (r->offset + r->size != r_next->offset) + continue; + + TAILQ_REMOVE(&uctx->vm_info.regions, r_next, link); + r->size += r_next->size; + mobj_put(r_next->mobj); + free(r_next); + r_next = r; + } +} + +static bool cmp_region_for_remap(const struct vm_region *r0, + const struct vm_region *r, + const struct vm_region *rn) +{ + /* + * All the essentionals has to match for remap to make sense. The + * essentials are, mobj/fobj, attr, flags and the offset should be + * contiguous. + * + * Note that vm_remap() depends on mobj/fobj to be the same. + */ + return r0->flags == r->flags && r0->attr == r->attr && + r0->mobj == r->mobj && rn->offset == r->offset + r->size; +} + +TEE_Result vm_remap(struct user_mode_ctx *uctx, vaddr_t *new_va, vaddr_t old_va, + size_t len, size_t pad_begin, size_t pad_end) +{ + struct vm_region_head regs = TAILQ_HEAD_INITIALIZER(regs); + TEE_Result res = TEE_SUCCESS; + struct vm_region *r0 = NULL; + struct vm_region *r = NULL; + struct vm_region *r_next = NULL; + struct vm_region *r_last = NULL; + struct vm_region *r_first = NULL; + struct fobj *fobj = NULL; + vaddr_t next_va = 0; + + assert(thread_get_tsd()->ctx == uctx->ts_ctx); + + if (!len || ((len | old_va) & SMALL_PAGE_MASK)) + return TEE_ERROR_BAD_PARAMETERS; + + res = split_vm_range(uctx, old_va, len, cmp_region_for_remap, &r0); + if (res) + return res; + + if (mobj_is_paged(r0->mobj)) { + fobj = mobj_get_fobj(r0->mobj); + if (!fobj) + panic(); + } + + for (r = r0; r; r = r_next) { + if (r->va + r->size > old_va + len) + break; + r_next = TAILQ_NEXT(r, link); + rem_um_region(uctx, r); + TAILQ_REMOVE(&uctx->vm_info.regions, r, link); + TAILQ_INSERT_TAIL(®s, r, link); + } + + /* + * Synchronize change to translation tables. Even though the pager + * case unmaps immediately we may still free a translation table. + */ + vm_set_ctx(uctx->ts_ctx); + + r_first = TAILQ_FIRST(®s); + while (!TAILQ_EMPTY(®s)) { + r = TAILQ_FIRST(®s); + TAILQ_REMOVE(®s, r, link); + if (r_last) { + r->va = r_last->va + r_last->size; + res = umap_add_region(&uctx->vm_info, r, 0, 0, 0); + } else { + r->va = *new_va; + res = umap_add_region(&uctx->vm_info, r, pad_begin, + pad_end + len - r->size, 0); + } + if (!res) { + r_last = r; + res = alloc_pgt(uctx); + } + if (!res) { + if (!fobj) + set_um_region(uctx, r); + else + res = tee_pager_add_um_region(uctx, r->va, fobj, + r->attr); + } + + if (res) { + /* + * Something went wrong move all the recently added + * regions back to regs for later reinsertion at + * the original spot. + */ + struct vm_region *r_tmp = NULL; + struct vm_region *r_stop = NULL; + + if (r != r_last) { + /* + * umap_add_region() failed, move r back to + * regs before all the rest are moved back. + */ + TAILQ_INSERT_HEAD(®s, r, link); + } + if (r_last) + r_stop = TAILQ_NEXT(r_last, link); + for (r = r_first; r != r_stop; r = r_next) { + r_next = TAILQ_NEXT(r, link); + TAILQ_REMOVE(&uctx->vm_info.regions, r, link); + if (r_tmp) + TAILQ_INSERT_AFTER(®s, r_tmp, r, + link); + else + TAILQ_INSERT_HEAD(®s, r, link); + r_tmp = r; + } + + goto err_restore_map; + } + } + + fobj_put(fobj); + + vm_set_ctx(uctx->ts_ctx); + *new_va = r_first->va; + + return TEE_SUCCESS; + +err_restore_map: + next_va = old_va; + while (!TAILQ_EMPTY(®s)) { + r = TAILQ_FIRST(®s); + TAILQ_REMOVE(®s, r, link); + r->va = next_va; + next_va += r->size; + if (umap_add_region(&uctx->vm_info, r, 0, 0, 0)) + panic("Cannot restore mapping"); + if (alloc_pgt(uctx)) + panic("Cannot restore mapping"); + if (fobj) { + if (tee_pager_add_um_region(uctx, r->va, fobj, r->attr)) + panic("Cannot restore mapping"); + } else { + set_um_region(uctx, r); + } + } + fobj_put(fobj); + vm_set_ctx(uctx->ts_ctx); + + return res; +} + +static bool cmp_region_for_get_flags(const struct vm_region *r0, + const struct vm_region *r, + const struct vm_region *rn __unused) +{ + return r0->flags == r->flags; +} + +TEE_Result vm_get_flags(struct user_mode_ctx *uctx, vaddr_t va, size_t len, + uint32_t *flags) +{ + struct vm_region *r = NULL; + + if (!len || ((len | va) & SMALL_PAGE_MASK)) + return TEE_ERROR_BAD_PARAMETERS; + + r = find_vm_region(&uctx->vm_info, va); + if (!r) + return TEE_ERROR_BAD_PARAMETERS; + + if (!va_range_is_contiguous(r, va, len, cmp_region_for_get_flags)) + return TEE_ERROR_BAD_PARAMETERS; + + *flags = r->flags; + + return TEE_SUCCESS; +} + +static bool cmp_region_for_get_prot(const struct vm_region *r0, + const struct vm_region *r, + const struct vm_region *rn __unused) +{ + return (r0->attr & TEE_MATTR_PROT_MASK) == + (r->attr & TEE_MATTR_PROT_MASK); +} + +TEE_Result vm_get_prot(struct user_mode_ctx *uctx, vaddr_t va, size_t len, + uint16_t *prot) +{ + struct vm_region *r = NULL; + + if (!len || ((len | va) & SMALL_PAGE_MASK)) + return TEE_ERROR_BAD_PARAMETERS; + + r = find_vm_region(&uctx->vm_info, va); + if (!r) + return TEE_ERROR_BAD_PARAMETERS; + + if (!va_range_is_contiguous(r, va, len, cmp_region_for_get_prot)) + return TEE_ERROR_BAD_PARAMETERS; + + *prot = r->attr & TEE_MATTR_PROT_MASK; + + return TEE_SUCCESS; +} + +TEE_Result vm_set_prot(struct user_mode_ctx *uctx, vaddr_t va, size_t len, + uint32_t prot) +{ + TEE_Result res = TEE_SUCCESS; + struct vm_region *r0 = NULL; + struct vm_region *r = NULL; + bool was_writeable = false; + bool need_sync = false; + + assert(thread_get_tsd()->ctx == uctx->ts_ctx); + + if (prot & ~TEE_MATTR_PROT_MASK || !len) + return TEE_ERROR_BAD_PARAMETERS; + + res = split_vm_range(uctx, va, len, NULL, &r0); + if (res) + return res; + + for (r = r0; r; r = TAILQ_NEXT(r, link)) { + if (r->va + r->size > va + len) + break; + if (r->attr & (TEE_MATTR_UW | TEE_MATTR_PW)) + was_writeable = true; + + r->attr &= ~TEE_MATTR_PROT_MASK; + r->attr |= prot; + + if (!mobj_is_paged(r->mobj)) { + need_sync = true; + set_um_region(uctx, r); + /* + * Normally when set_um_region() is called we + * change from no mapping to some mapping, but in + * this case we change the permissions on an + * already present mapping so some TLB invalidation + * is needed. We also depend on the dsb() performed + * as part of the TLB invalidation. + */ + tlbi_va_range_asid(r->va, r->size, SMALL_PAGE_SIZE, + uctx->vm_info.asid); + } + } + + for (r = r0; r; r = TAILQ_NEXT(r, link)) { + if (r->va + r->size > va + len) + break; + if (mobj_is_paged(r->mobj)) { + if (!tee_pager_set_um_region_attr(uctx, r->va, r->size, + prot)) + panic(); + } else if (was_writeable) { + cache_op_inner(DCACHE_AREA_CLEAN, (void *)r->va, + r->size); + } + + } + if (need_sync && was_writeable) + cache_op_inner(ICACHE_INVALIDATE, NULL, 0); + + merge_vm_range(uctx, va, len); + + return TEE_SUCCESS; +} + +static void umap_remove_region(struct vm_info *vmi, struct vm_region *reg) +{ + TAILQ_REMOVE(&vmi->regions, reg, link); + mobj_put(reg->mobj); + free(reg); +} + +TEE_Result vm_unmap(struct user_mode_ctx *uctx, vaddr_t va, size_t len) +{ + TEE_Result res = TEE_SUCCESS; + struct vm_region *r = NULL; + struct vm_region *r_next = NULL; + size_t end_va = 0; + size_t unmap_end_va = 0; + size_t l = 0; + + assert(thread_get_tsd()->ctx == uctx->ts_ctx); + + if (ROUNDUP_OVERFLOW(len, SMALL_PAGE_SIZE, &l)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!l || (va & SMALL_PAGE_MASK)) + return TEE_ERROR_BAD_PARAMETERS; + + if (ADD_OVERFLOW(va, l, &end_va)) + return TEE_ERROR_BAD_PARAMETERS; + + res = split_vm_range(uctx, va, l, NULL, &r); + if (res) + return res; + + while (true) { + r_next = TAILQ_NEXT(r, link); + unmap_end_va = r->va + r->size; + rem_um_region(uctx, r); + umap_remove_region(&uctx->vm_info, r); + if (!r_next || unmap_end_va == end_va) + break; + r = r_next; + } + + return TEE_SUCCESS; +} + +static TEE_Result map_kinit(struct user_mode_ctx *uctx) +{ + TEE_Result res = TEE_SUCCESS; + struct mobj *mobj = NULL; + size_t offs = 0; + vaddr_t va = 0; + size_t sz = 0; + uint32_t prot = 0; + + thread_get_user_kcode(&mobj, &offs, &va, &sz); + if (sz) { + prot = TEE_MATTR_PRX; + if (IS_ENABLED(CFG_CORE_BTI)) + prot |= TEE_MATTR_GUARDED; + res = vm_map(uctx, &va, sz, prot, VM_FLAG_PERMANENT, + mobj, offs); + if (res) + return res; + } + + thread_get_user_kdata(&mobj, &offs, &va, &sz); + if (sz) + return vm_map(uctx, &va, sz, TEE_MATTR_PRW, VM_FLAG_PERMANENT, + mobj, offs); + + return TEE_SUCCESS; +} + +TEE_Result vm_info_init(struct user_mode_ctx *uctx, struct ts_ctx *ts_ctx) +{ + TEE_Result res; + uint32_t asid = asid_alloc(); + + if (!asid) { + DMSG("Failed to allocate ASID"); + return TEE_ERROR_GENERIC; + } + + memset(uctx, 0, sizeof(*uctx)); + TAILQ_INIT(&uctx->vm_info.regions); + SLIST_INIT(&uctx->pgt_cache); + uctx->vm_info.asid = asid; + uctx->ts_ctx = ts_ctx; + + res = map_kinit(uctx); + if (res) + vm_info_final(uctx); + return res; +} + +void vm_clean_param(struct user_mode_ctx *uctx) +{ + struct vm_region *next_r; + struct vm_region *r; + + TAILQ_FOREACH_SAFE(r, &uctx->vm_info.regions, link, next_r) { + if (r->flags & VM_FLAG_EPHEMERAL) { + rem_um_region(uctx, r); + umap_remove_region(&uctx->vm_info, r); + } + } +} + +static void check_param_map_empty(struct user_mode_ctx *uctx __maybe_unused) +{ + struct vm_region *r = NULL; + + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) + assert(!(r->flags & VM_FLAG_EPHEMERAL)); +} + +static TEE_Result param_mem_to_user_va(struct user_mode_ctx *uctx, + struct param_mem *mem, void **user_va) +{ + struct vm_region *region = NULL; + + TAILQ_FOREACH(region, &uctx->vm_info.regions, link) { + vaddr_t va = 0; + size_t phys_offs = 0; + + if (!(region->flags & VM_FLAG_EPHEMERAL)) + continue; + if (mem->mobj != region->mobj) + continue; + + phys_offs = mobj_get_phys_offs(mem->mobj, + CORE_MMU_USER_PARAM_SIZE); + phys_offs += mem->offs; + if (phys_offs < region->offset) + continue; + if (phys_offs >= (region->offset + region->size)) + continue; + va = region->va + phys_offs - region->offset; + *user_va = (void *)va; + return TEE_SUCCESS; + } + return TEE_ERROR_GENERIC; +} + +static int cmp_param_mem(const void *a0, const void *a1) +{ + const struct param_mem *m1 = a1; + const struct param_mem *m0 = a0; + int ret; + + /* Make sure that invalid param_mem are placed last in the array */ + if (!m0->mobj && !m1->mobj) + return 0; + if (!m0->mobj) + return 1; + if (!m1->mobj) + return -1; + + ret = CMP_TRILEAN(mobj_is_secure(m0->mobj), mobj_is_secure(m1->mobj)); + if (ret) + return ret; + + ret = CMP_TRILEAN((vaddr_t)m0->mobj, (vaddr_t)m1->mobj); + if (ret) + return ret; + + ret = CMP_TRILEAN(m0->offs, m1->offs); + if (ret) + return ret; + + return CMP_TRILEAN(m0->size, m1->size); +} + +TEE_Result vm_map_param(struct user_mode_ctx *uctx, struct tee_ta_param *param, + void *param_va[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + size_t n; + size_t m; + struct param_mem mem[TEE_NUM_PARAMS]; + + memset(mem, 0, sizeof(mem)); + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uint32_t param_type = TEE_PARAM_TYPE_GET(param->types, n); + size_t phys_offs; + + if (param_type != TEE_PARAM_TYPE_MEMREF_INPUT && + param_type != TEE_PARAM_TYPE_MEMREF_OUTPUT && + param_type != TEE_PARAM_TYPE_MEMREF_INOUT) + continue; + phys_offs = mobj_get_phys_offs(param->u[n].mem.mobj, + CORE_MMU_USER_PARAM_SIZE); + mem[n].mobj = param->u[n].mem.mobj; + mem[n].offs = ROUNDDOWN(phys_offs + param->u[n].mem.offs, + CORE_MMU_USER_PARAM_SIZE); + mem[n].size = ROUNDUP(phys_offs + param->u[n].mem.offs - + mem[n].offs + param->u[n].mem.size, + CORE_MMU_USER_PARAM_SIZE); + /* + * For size 0 (raw pointer parameter), add minimum size + * value to allow address to be mapped + */ + if (!mem[n].size) + mem[n].size = CORE_MMU_USER_PARAM_SIZE; + } + + /* + * Sort arguments so NULL mobj is last, secure mobjs first, then by + * mobj pointer value since those entries can't be merged either, + * finally by offset. + * + * This should result in a list where all mergeable entries are + * next to each other and unused/invalid entries are at the end. + */ + qsort(mem, TEE_NUM_PARAMS, sizeof(struct param_mem), cmp_param_mem); + + for (n = 1, m = 0; n < TEE_NUM_PARAMS && mem[n].mobj; n++) { + if (mem[n].mobj == mem[m].mobj && + (mem[n].offs == (mem[m].offs + mem[m].size) || + core_is_buffer_intersect(mem[m].offs, mem[m].size, + mem[n].offs, mem[n].size))) { + mem[m].size = mem[n].offs + mem[n].size - mem[m].offs; + continue; + } + m++; + if (n != m) + mem[m] = mem[n]; + } + /* + * We'd like 'm' to be the number of valid entries. Here 'm' is the + * index of the last valid entry if the first entry is valid, else + * 0. + */ + if (mem[0].mobj) + m++; + + check_param_map_empty(uctx); + + for (n = 0; n < m; n++) { + vaddr_t va = 0; + + res = vm_map(uctx, &va, mem[n].size, + TEE_MATTR_PRW | TEE_MATTR_URW, + VM_FLAG_EPHEMERAL | VM_FLAG_SHAREABLE, + mem[n].mobj, mem[n].offs); + if (res) + goto out; + } + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uint32_t param_type = TEE_PARAM_TYPE_GET(param->types, n); + + if (param_type != TEE_PARAM_TYPE_MEMREF_INPUT && + param_type != TEE_PARAM_TYPE_MEMREF_OUTPUT && + param_type != TEE_PARAM_TYPE_MEMREF_INOUT) + continue; + if (!param->u[n].mem.mobj) + continue; + + res = param_mem_to_user_va(uctx, ¶m->u[n].mem, + param_va + n); + if (res != TEE_SUCCESS) + goto out; + } + + res = alloc_pgt(uctx); +out: + if (res) + vm_clean_param(uctx); + + return res; +} + +void vm_info_final(struct user_mode_ctx *uctx) +{ + if (!uctx->vm_info.asid) + return; + + pgt_flush(uctx); + tee_pager_rem_um_regions(uctx); + + /* clear MMU entries to avoid clash when asid is reused */ + tlbi_asid(uctx->vm_info.asid); + + asid_free(uctx->vm_info.asid); + uctx->vm_info.asid = 0; + + while (!TAILQ_EMPTY(&uctx->vm_info.regions)) + umap_remove_region(&uctx->vm_info, + TAILQ_FIRST(&uctx->vm_info.regions)); +} + +/* return true only if buffer fits inside TA private memory */ +bool vm_buf_is_inside_um_private(const struct user_mode_ctx *uctx, + const void *va, size_t size) +{ + struct vm_region *r = NULL; + + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) { + if (r->flags & VM_FLAGS_NONPRIV) + continue; + if (core_is_buffer_inside((vaddr_t)va, size, r->va, r->size)) + return true; + } + + return false; +} + +/* return true only if buffer intersects TA private memory */ +bool vm_buf_intersects_um_private(const struct user_mode_ctx *uctx, + const void *va, size_t size) +{ + struct vm_region *r = NULL; + + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) { + if (r->attr & VM_FLAGS_NONPRIV) + continue; + if (core_is_buffer_intersect((vaddr_t)va, size, r->va, r->size)) + return true; + } + + return false; +} + +TEE_Result vm_buf_to_mboj_offs(const struct user_mode_ctx *uctx, + const void *va, size_t size, + struct mobj **mobj, size_t *offs) +{ + struct vm_region *r = NULL; + + TAILQ_FOREACH(r, &uctx->vm_info.regions, link) { + if (!r->mobj) + continue; + if (core_is_buffer_inside((vaddr_t)va, size, r->va, r->size)) { + size_t poffs; + + poffs = mobj_get_phys_offs(r->mobj, + CORE_MMU_USER_PARAM_SIZE); + *mobj = r->mobj; + *offs = (vaddr_t)va - r->va + r->offset - poffs; + return TEE_SUCCESS; + } + } + + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result tee_mmu_user_va2pa_attr(const struct user_mode_ctx *uctx, + void *ua, paddr_t *pa, uint32_t *attr) +{ + struct vm_region *region = NULL; + + TAILQ_FOREACH(region, &uctx->vm_info.regions, link) { + if (!core_is_buffer_inside((vaddr_t)ua, 1, region->va, + region->size)) + continue; + + if (pa) { + TEE_Result res; + paddr_t p; + size_t offset; + size_t granule; + + /* + * mobj and input user address may each include + * a specific offset-in-granule position. + * Drop both to get target physical page base + * address then apply only user address + * offset-in-granule. + * Mapping lowest granule is the small page. + */ + granule = MAX(region->mobj->phys_granule, + (size_t)SMALL_PAGE_SIZE); + assert(!granule || IS_POWER_OF_TWO(granule)); + + offset = region->offset + + ROUNDDOWN((vaddr_t)ua - region->va, granule); + + res = mobj_get_pa(region->mobj, offset, granule, &p); + if (res != TEE_SUCCESS) + return res; + + *pa = p | ((vaddr_t)ua & (granule - 1)); + } + if (attr) + *attr = region->attr; + + return TEE_SUCCESS; + } + + return TEE_ERROR_ACCESS_DENIED; +} + +TEE_Result vm_va2pa(const struct user_mode_ctx *uctx, void *ua, paddr_t *pa) +{ + return tee_mmu_user_va2pa_attr(uctx, ua, pa, NULL); +} + +void *vm_pa2va(const struct user_mode_ctx *uctx, paddr_t pa, size_t pa_size) +{ + paddr_t p = 0; + struct vm_region *region = NULL; + + TAILQ_FOREACH(region, &uctx->vm_info.regions, link) { + size_t granule = 0; + size_t size = 0; + size_t ofs = 0; + + /* pa2va is expected only for memory tracked through mobj */ + if (!region->mobj) + continue; + + /* Physically granulated memory object must be scanned */ + granule = region->mobj->phys_granule; + assert(!granule || IS_POWER_OF_TWO(granule)); + + for (ofs = region->offset; ofs < region->size; ofs += size) { + + if (granule) { + /* From current offset to buffer/granule end */ + size = granule - (ofs & (granule - 1)); + + if (size > (region->size - ofs)) + size = region->size - ofs; + } else { + size = region->size; + } + + if (mobj_get_pa(region->mobj, ofs, granule, &p)) + continue; + + if (core_is_buffer_inside(pa, pa_size, p, size)) { + /* Remove region offset (mobj phys offset) */ + ofs -= region->offset; + /* Get offset-in-granule */ + p = pa - p; + + return (void *)(region->va + ofs + (vaddr_t)p); + } + } + } + + return NULL; +} + +TEE_Result vm_check_access_rights(const struct user_mode_ctx *uctx, + uint32_t flags, uaddr_t uaddr, size_t len) +{ + uaddr_t a = 0; + uaddr_t end_addr = 0; + size_t addr_incr = MIN(CORE_MMU_USER_CODE_SIZE, + CORE_MMU_USER_PARAM_SIZE); + + if (ADD_OVERFLOW(uaddr, len, &end_addr)) + return TEE_ERROR_ACCESS_DENIED; + + if ((flags & TEE_MEMORY_ACCESS_NONSECURE) && + (flags & TEE_MEMORY_ACCESS_SECURE)) + return TEE_ERROR_ACCESS_DENIED; + + /* + * Rely on TA private memory test to check if address range is private + * to TA or not. + */ + if (!(flags & TEE_MEMORY_ACCESS_ANY_OWNER) && + !vm_buf_is_inside_um_private(uctx, (void *)uaddr, len)) + return TEE_ERROR_ACCESS_DENIED; + + for (a = ROUNDDOWN(uaddr, addr_incr); a < end_addr; a += addr_incr) { + uint32_t attr; + TEE_Result res; + + res = tee_mmu_user_va2pa_attr(uctx, (void *)a, NULL, &attr); + if (res != TEE_SUCCESS) + return res; + + if ((flags & TEE_MEMORY_ACCESS_NONSECURE) && + (attr & TEE_MATTR_SECURE)) + return TEE_ERROR_ACCESS_DENIED; + + if ((flags & TEE_MEMORY_ACCESS_SECURE) && + !(attr & TEE_MATTR_SECURE)) + return TEE_ERROR_ACCESS_DENIED; + + if ((flags & TEE_MEMORY_ACCESS_WRITE) && !(attr & TEE_MATTR_UW)) + return TEE_ERROR_ACCESS_DENIED; + if ((flags & TEE_MEMORY_ACCESS_READ) && !(attr & TEE_MATTR_UR)) + return TEE_ERROR_ACCESS_DENIED; + } + + return TEE_SUCCESS; +} + +void vm_set_ctx(struct ts_ctx *ctx) +{ + struct thread_specific_data *tsd = thread_get_tsd(); + struct user_mode_ctx *uctx = NULL; + + core_mmu_set_user_map(NULL); + + if (is_user_mode_ctx(tsd->ctx)) { + /* + * We're coming from a user mode context so we must make + * the pgts available for reuse. + */ + uctx = to_user_mode_ctx(tsd->ctx); + pgt_put_all(uctx); + } + + if (is_user_mode_ctx(ctx)) { + struct core_mmu_user_map map = { }; + + uctx = to_user_mode_ctx(ctx); + core_mmu_create_user_map(uctx, &map); + core_mmu_set_user_map(&map); + tee_pager_assign_um_tables(uctx); + } + tsd->ctx = ctx; +} + +struct mobj *vm_get_mobj(struct user_mode_ctx *uctx, vaddr_t va, size_t *len, + uint16_t *prot, size_t *offs) +{ + struct vm_region *r = NULL; + size_t r_offs = 0; + + if (!len || ((*len | va) & SMALL_PAGE_MASK)) + return NULL; + + r = find_vm_region(&uctx->vm_info, va); + if (!r) + return NULL; + + r_offs = va - r->va; + + *len = MIN(r->size - r_offs, *len); + *offs = r->offset + r_offs; + *prot = r->attr & TEE_MATTR_PROT_MASK; + return mobj_get(r->mobj); +} diff --git a/optee_os/core/pta/apdu.c b/optee_os/core/pta/apdu.c new file mode 100644 index 0000000..940bdb5 --- /dev/null +++ b/optee_os/core/pta/apdu.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) Foundries Ltd. 2021 - All Rights Reserved + * Author: Jorge Ramirez + */ +#include +#include +#include + +#define PTA_NAME "pta.apdu" + +static TEE_Result get_apdu_type(uint32_t val, enum crypto_apdu_type *type) +{ + switch (val) { + case PTA_APDU_TXRX_CASE_NO_HINT: + *type = CRYPTO_APDU_CASE_NO_HINT; + break; + case PTA_APDU_TXRX_CASE_1: + *type = CRYPTO_APDU_CASE_1; + break; + case PTA_APDU_TXRX_CASE_2: + *type = CRYPTO_APDU_CASE_2; + break; + case PTA_APDU_TXRX_CASE_2E: + *type = CRYPTO_APDU_CASE_2E; + break; + case PTA_APDU_TXRX_CASE_3: + *type = CRYPTO_APDU_CASE_3; + break; + case PTA_APDU_TXRX_CASE_3E: + *type = CRYPTO_APDU_CASE_3E; + break; + case PTA_APDU_TXRX_CASE_4: + *type = CRYPTO_APDU_CASE_4; + break; + case PTA_APDU_TXRX_CASE_4E: + *type = CRYPTO_APDU_CASE_4E; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t command_id, uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT); + enum crypto_apdu_type type = CRYPTO_APDU_CASE_NO_HINT; + TEE_Result ret = TEE_ERROR_NOT_IMPLEMENTED; + size_t len = 0; + + FMSG("command entry point for pseudo-TA \"%s\"", PTA_NAME); + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + switch (command_id) { + case PTA_CMD_TXRX_APDU_RAW_FRAME: + ret = get_apdu_type(params[0].value.a, &type); + if (ret) + return ret; + + len = params[3].memref.size; + ret = crypto_se_do_apdu(type, + params[1].memref.buffer, + params[1].memref.size, + params[2].memref.buffer, + params[2].memref.size, + params[3].memref.buffer, + &len); + if (!ret) + params[3].memref.size = len; + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + + return ret; +} + +pseudo_ta_register(.uuid = PTA_APDU_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/attestation.c b/optee_os/core/pta/attestation.c new file mode 100644 index 0000000..ab5a3bc --- /dev/null +++ b/optee_os/core/pta/attestation.c @@ -0,0 +1,802 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2021, Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PTA_NAME "attestation.pta" + +#define MAX_KEY_SIZE 4096 + +static TEE_UUID pta_uuid = PTA_ATTESTATION_UUID; + +static struct rsa_keypair *key; + +static const uint8_t key_file_name[] = "key"; + +static TEE_Result allocate_key(void) +{ + assert(!key); + + key = calloc(1, sizeof(*key)); + if (!key) + return TEE_ERROR_OUT_OF_MEMORY; + + COMPILE_TIME_ASSERT(CFG_ATTESTATION_PTA_KEY_SIZE <= MAX_KEY_SIZE); + return crypto_acipher_alloc_rsa_keypair(key, MAX_KEY_SIZE); +} + +static void free_key(void) +{ + crypto_acipher_free_rsa_keypair(key); + free(key); + key = NULL; +} + +static TEE_Result generate_key(void) +{ + uint32_t e = TEE_U32_TO_BIG_ENDIAN(65537); + TEE_Result res = TEE_ERROR_GENERIC; + + res = allocate_key(); + if (res) + goto err; + + res = crypto_bignum_bin2bn((uint8_t *)&e, sizeof(e), key->e); + if (res) + goto err; + + /* + * For security reasons, the RSA modulus size has to be at least the + * size of the data to be signed. + */ + DMSG("Generating %u bit RSA key pair", CFG_ATTESTATION_PTA_KEY_SIZE); + COMPILE_TIME_ASSERT(CFG_ATTESTATION_PTA_KEY_SIZE >= + TEE_SHA256_HASH_SIZE); + res = crypto_acipher_gen_rsa_key(key, CFG_ATTESTATION_PTA_KEY_SIZE); + if (!res) + goto out; + +err: + free_key(); +out: + return res; +} + +/* + * Return values: + * > 0 : Number of bytes written to buf + * 0 : @sz too large (> UINT16_MAX) or @buf_sz too small + */ +static size_t serialize_bignum(uint8_t *buf, size_t buf_sz, struct bignum *bn) +{ + uint8_t *p = buf; + size_t sz = crypto_bignum_num_bytes(bn); + uint16_t val = TEE_U16_TO_BIG_ENDIAN(sz); + size_t total_sz = sizeof(val) + sz; + + if (sz > UINT16_MAX || total_sz > buf_sz) + return 0; + + memcpy(p, &val, sizeof(val)); + p += sizeof(val); + + crypto_bignum_bn2bin(bn, p); + + return total_sz; +} + +static size_t bufsize(size_t e_sz, size_t d_sz, size_t n_sz) +{ + /* + * Serialized key pair is 3 bignums (e, p and n) plus their sizes + * encoded as uint16_t. + */ + return e_sz + d_sz + n_sz + 3 * sizeof(uint16_t); +} + +static TEE_Result serialize_key(uint8_t *buf, size_t size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *p = buf; + size_t needed_sz = 0; + size_t e_sz = 0; + size_t d_sz = 0; + size_t n_sz = 0; + size_t sz = 0; + + assert(key); + + e_sz = crypto_bignum_num_bytes(key->e); + d_sz = crypto_bignum_num_bytes(key->d); + n_sz = crypto_bignum_num_bytes(key->n); + if (e_sz > UINT16_MAX || d_sz > UINT16_MAX || n_sz > UINT16_MAX) + goto err; + + needed_sz = bufsize(e_sz, d_sz, n_sz); + if (size < needed_sz) + goto err; + + sz = serialize_bignum(p, needed_sz, key->e); + if (!sz) + goto err; + p += sz; + needed_sz -= sz; + sz = serialize_bignum(p, needed_sz, key->d); + if (!sz) + goto err; + p += sz; + needed_sz -= sz; + sz = serialize_bignum(p, needed_sz, key->n); + if (!sz) + goto err; + needed_sz -= sz; + assert(!needed_sz); + + return TEE_SUCCESS; +err: + return res; +} + +static size_t deserialize_bignum(uint8_t *buf, size_t max_sz, struct bignum *bn) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *p = buf; + uint16_t val = 0; + size_t sz = 0; + + if (max_sz < sizeof(val)) + return 0; + + memcpy(&val, p, sizeof(val)); + sz = TEE_U16_FROM_BIG_ENDIAN(val); + p += sizeof(val); + max_sz -= sizeof(val); + if (max_sz < sz) + return 0; + + res = crypto_bignum_bin2bn(p, sz, bn); + if (res) + return 0; + + return sizeof(val) + sz; +} + +static TEE_Result deserialize_key(uint8_t *buf, size_t buf_sz) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *p = buf; + size_t sz = 0; + + res = allocate_key(); + if (res) + return res; + + sz = deserialize_bignum(p, buf_sz, key->e); + if (!sz) + goto err; + p += sz; + buf_sz -= sz; + sz = deserialize_bignum(p, buf_sz, key->d); + if (!sz) + goto err; + p += sz; + buf_sz -= sz; + sz = deserialize_bignum(p, buf_sz, key->n); + if (!sz) + goto err; + + return TEE_SUCCESS; +err: + free_key(); + return TEE_ERROR_GENERIC; +} + +static TEE_Result sec_storage_obj_read(TEE_UUID *uuid, uint32_t storage_id, + const uint8_t *obj_id, + size_t obj_id_len, + uint8_t *data, size_t *len, + size_t offset, uint32_t flags) +{ + const struct tee_file_operations *fops = NULL; + TEE_Result res = TEE_ERROR_BAD_STATE; + struct tee_file_handle *fh = NULL; + struct tee_pobj *po = NULL; + size_t file_size = 0; + size_t read_len = 0; + + fops = tee_svc_storage_file_ops(storage_id); + if (!fops) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (obj_id_len > TEE_OBJECT_ID_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + res = tee_pobj_get(uuid, (void *)obj_id, obj_id_len, flags, false, fops, + &po); + if (res) + return res; + + res = po->fops->open(po, &file_size, &fh); + if (res) + goto out; + + read_len = *len; + res = po->fops->read(fh, offset, data, NULL, &read_len); + if (res == TEE_ERROR_CORRUPT_OBJECT) { + EMSG("Object corrupt"); + po->fops->remove(po); + } else if (!res) { + *len = read_len; + } + + po->fops->close(&fh); +out: + tee_pobj_release(po); + + return res; +} + +static TEE_Result sec_storage_obj_write(TEE_UUID *uuid, uint32_t storage_id, + const uint8_t *obj_id, + size_t obj_id_len, + const uint8_t *data, size_t len, + size_t offset, uint32_t flags) + +{ + const struct tee_file_operations *fops = NULL; + struct tee_file_handle *fh = NULL; + TEE_Result res = TEE_SUCCESS; + struct tee_pobj *po = NULL; + + fops = tee_svc_storage_file_ops(storage_id); + if (!fops) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (obj_id_len > TEE_OBJECT_ID_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + res = tee_pobj_get(uuid, (void *)obj_id, obj_id_len, flags, false, + fops, &po); + if (res) + return res; + + res = po->fops->open(po, NULL, &fh); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = po->fops->create(po, false, NULL, 0, NULL, 0, + NULL, NULL, 0, &fh); + if (!res) { + res = po->fops->write(fh, offset, data, NULL, len); + po->fops->close(&fh); + } + + tee_pobj_release(po); + + return res; +} + +static TEE_Result load_key(uint8_t *buf, size_t size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + DMSG("Loading RSA key pair from secure storage"); + res = sec_storage_obj_read(&pta_uuid, TEE_STORAGE_PRIVATE, + key_file_name, sizeof(key_file_name) - 1, + buf, &size, 0, TEE_DATA_FLAG_ACCESS_READ); + if (res) + return res; + DMSG("Read %zu bytes", size); + res = deserialize_key(buf, size); + if (!res) + DMSG("Loaded %zu bit key pair", crypto_bignum_num_bits(key->n)); + + return res; +} + +static TEE_Result write_key(uint8_t *buf, size_t size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + DMSG("Saving key pair"); + res = serialize_key(buf, size); + if (res) + return res; + + res = sec_storage_obj_write(&pta_uuid, TEE_STORAGE_PRIVATE, + key_file_name, sizeof(key_file_name) - 1, + buf, size, 0, TEE_DATA_FLAG_ACCESS_WRITE); + if (!res) + DMSG("Wrote %zu bytes", size); + return res; +} + +static TEE_Result init_key(void) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *buf = NULL; + size_t size = 0; + + if (!key) { + /* + * e is 65537 so its bignum size is 3 bytes. d and n can be up + * to MAX_KEY_SIZE bits. + */ + size = bufsize(3, MAX_KEY_SIZE / 8, MAX_KEY_SIZE / 8); + buf = calloc(1, size); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + res = load_key(buf, size); + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + res = generate_key(); + if (res) + goto out; + res = write_key(buf, size); + } + } +out: + free(buf); + return res; +} + +static TEE_Result cmd_get_pubkey(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *e = params[0].memref.buffer; + size_t *e_out_sz = ¶ms[0].memref.size; + uint8_t *n = params[1].memref.buffer; + size_t *n_out_sz = ¶ms[1].memref.size; + size_t sz = 0; + + if (param_types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + res = init_key(); + if (res) + return res; + + sz = crypto_bignum_num_bytes(key->e); + if (*e_out_sz >= sz) + crypto_bignum_bn2bin(key->e, e); + else + res = TEE_ERROR_SHORT_BUFFER; + *e_out_sz = sz; + + sz = crypto_bignum_num_bytes(key->n); + if (*n_out_sz >= sz) + crypto_bignum_bn2bin(key->n, n); + else + res = TEE_ERROR_SHORT_BUFFER; + *n_out_sz = sz; + + params[2].value.a = TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256; + + return res; +} + +static TEE_Result hash_binary(const TEE_UUID *uuid, uint8_t *hash) +{ + TEE_Result res = TEE_ERROR_ITEM_NOT_FOUND; + unsigned int tag_len = FILE_TAG_SIZE; + const struct ts_store_ops *ops = NULL; + struct ts_store_handle *h = NULL; + + SCATTERED_ARRAY_FOREACH(ops, ta_stores, struct ts_store_ops) { + res = ops->open(uuid, &h); + if (!res) + break; /* TA found */ + } + if (res) + return res; + + /* + * Output hash size is assumed to be the same size as the file tag + * size which is the size of the digest in the TA shdr. If one or the + * other changes, additional hashing will be needed. + */ + COMPILE_TIME_ASSERT(FILE_TAG_SIZE == TEE_SHA256_HASH_SIZE); + assert(ops); + res = ops->get_tag(h, hash, &tag_len); + if (res) + goto out; + + DMSG("TA %pUl hash:", uuid); + DHEXDUMP(hash, TEE_SHA256_HASH_SIZE); +out: + ops->close(h); + return res; +} + +/* Hash @nonce and @hash into @digest */ +static TEE_Result digest_nonce_and_hash(uint8_t *digest, uint8_t *nonce, + size_t nonce_sz, uint8_t *hash) +{ + TEE_Result res = TEE_SUCCESS; + void *ctx = NULL; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SHA256); + if (res) + return res; + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, nonce, nonce_sz); + if (res) + goto out; + res = crypto_hash_update(ctx, hash, TEE_SHA256_HASH_SIZE); + if (res) + goto out; + res = crypto_hash_final(ctx, digest, TEE_SHA256_HASH_SIZE); +out: + crypto_hash_free_ctx(ctx); + return res; +} + +static TEE_Result sign_digest(uint8_t *sig, size_t sig_len, + const uint8_t *digest) +{ + return crypto_acipher_rsassa_sign(TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256, + key, + TEE_SHA256_HASH_SIZE, /* salt len */ + digest, TEE_SHA256_HASH_SIZE, + sig, &sig_len); +} + +/* + * Sign the first 32 bytes contained in @buf and append signature + * out = [ hash | sig(sha256(nonce | hash)) ] + * ^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^ + * 32B modulus size + */ +static TEE_Result sign_buffer(uint8_t *buf, size_t buf_sz, uint8_t *nonce, + size_t nonce_sz) +{ + uint8_t digest[TEE_SHA256_HASH_SIZE] = { }; + TEE_Result res = TEE_SUCCESS; + + res = digest_nonce_and_hash(digest, nonce, nonce_sz, buf); + if (res) + return res; + return sign_digest(buf + TEE_SHA256_HASH_SIZE, + buf_sz - TEE_SHA256_HASH_SIZE, digest); +} + +/* + * Is region valid for hashing? + * Exclude writable regions as well as those that are not specific to the TA + * (ldelf, kernel or temporary mappings). + */ +static bool is_region_valid(struct vm_region *r) +{ + uint32_t dontwant = VM_FLAG_EPHEMERAL | VM_FLAG_PERMANENT | + VM_FLAG_LDELF; + uint32_t want = VM_FLAG_READONLY; + + return ((r->flags & want) == want && !(r->flags & dontwant)); +} + +/* + * With this comparison function, we're hashing the smaller regions first. + * Regions of equal size are ordered based on their content (memcmp()). + * Identical regions can be in any order since they will yield the same hash + * anyways. + */ +static int cmp_regions(const void *a, const void *b) +{ + const struct vm_region *r1 = *(const struct vm_region **)a; + const struct vm_region *r2 = *(const struct vm_region **)b; + + if (r1->size < r2->size) + return -1; + + if (r1->size > r2->size) + return 1; + + return memcmp((void *)r1->va, (void *)r2->va, r1->size); +} + +static TEE_Result hash_regions(struct vm_info *vm_info, uint8_t *hash) +{ + TEE_Result res = TEE_SUCCESS; + struct vm_region *r = NULL; + struct vm_region **regions = NULL; + size_t nregions = 0; + void *ctx = NULL; + size_t i = 0; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SHA256); + if (res) + return res; + + res = crypto_hash_init(ctx); + if (res) + goto out; + + /* + * Make an array of region pointers so we can use qsort() to order it. + */ + + TAILQ_FOREACH(r, &vm_info->regions, link) + if (is_region_valid(r)) + nregions++; + + regions = malloc(nregions * sizeof(*regions)); + if (!regions) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + TAILQ_FOREACH(r, &vm_info->regions, link) + if (is_region_valid(r)) + regions[i++] = r; + + enter_user_access(); + + /* + * Sort regions so that they are in a consistent order even when TA ASLR + * is enabled. + */ + qsort(regions, nregions, sizeof(*regions), cmp_regions); + + /* Hash regions in order */ + for (i = 0; i < nregions; i++) { + r = regions[i]; + DMSG("va %p size %zu", (void *)r->va, r->size); + res = crypto_hash_update(ctx, (uint8_t *)r->va, r->size); + if (res) + break; + } + + exit_user_access(); + + if (res) + goto out; + + res = crypto_hash_final(ctx, hash, TEE_SHA256_HASH_SIZE); +out: + free(regions); + crypto_hash_free_ctx(ctx); + return res; +} + +static TEE_Result cmd_get_ta_shdr_digest(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_UUID *uuid = params[0].memref.buffer; + size_t uuid_sz = params[0].memref.size; + uint8_t *nonce = params[1].memref.buffer; + size_t nonce_sz = params[1].memref.size; + uint8_t *out = params[2].memref.buffer; + size_t out_sz = params[2].memref.size; + size_t min_out_sz = 0; + TEE_Result res = TEE_SUCCESS; + + if (param_types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + if (uuid_sz != sizeof(*uuid)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!nonce || !nonce_sz) + return TEE_ERROR_BAD_PARAMETERS; + + if (!out && out_sz) + return TEE_ERROR_BAD_PARAMETERS; + + res = init_key(); + if (res) + return res; + + min_out_sz = TEE_SHA256_HASH_SIZE + crypto_bignum_num_bytes(key->n); + params[2].memref.size = min_out_sz; + if (out_sz < min_out_sz) + return TEE_ERROR_SHORT_BUFFER; + out_sz = min_out_sz; + + res = hash_binary(uuid, out); + if (res) + return res; + return sign_buffer(out, out_sz, nonce, nonce_sz); +} + +static TEE_Result cmd_hash_ta_memory(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint8_t *nonce = params[0].memref.buffer; + size_t nonce_sz = params[0].memref.size; + uint8_t *out = params[1].memref.buffer; + size_t out_sz = params[1].memref.size; + struct user_mode_ctx *uctx = NULL; + TEE_Result res = TEE_SUCCESS; + struct ts_session *s = NULL; + size_t min_out_sz = 0; + + if (param_types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!nonce || !nonce_sz) + return TEE_ERROR_BAD_PARAMETERS; + + if (!out && out_sz) + return TEE_ERROR_BAD_PARAMETERS; + + /* Check that we're called from a user TA */ + s = ts_get_calling_session(); + if (!s) + return TEE_ERROR_ACCESS_DENIED; + uctx = to_user_mode_ctx(s->ctx); + if (!uctx) + return TEE_ERROR_ACCESS_DENIED; + + res = init_key(); + if (res) + return res; + + min_out_sz = TEE_SHA256_HASH_SIZE + crypto_bignum_num_bytes(key->n); + params[1].memref.size = min_out_sz; + if (out_sz < min_out_sz) + return TEE_ERROR_SHORT_BUFFER; + out_sz = min_out_sz; + + s = ts_pop_current_session(); + res = hash_regions(&uctx->vm_info, out); + ts_push_current_session(s); + if (res) + return res; + + return sign_buffer(out, out_sz, nonce, nonce_sz); +} + +static TEE_Result cmd_hash_tee_memory(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint8_t *nonce = params[0].memref.buffer; + size_t nonce_sz = params[0].memref.size; + uint8_t *out = params[1].memref.buffer; + size_t out_sz = params[1].memref.size; + TEE_Result res = TEE_SUCCESS; + size_t min_out_sz = 0; + void *ctx = NULL; + + if (param_types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!nonce || !nonce_sz) + return TEE_ERROR_BAD_PARAMETERS; + + if (!out && out_sz) + return TEE_ERROR_BAD_PARAMETERS; + + res = init_key(); + if (res) + return res; + + min_out_sz = TEE_SHA256_HASH_SIZE + crypto_bignum_num_bytes(key->n); + params[1].memref.size = min_out_sz; + if (out_sz < min_out_sz) + return TEE_ERROR_SHORT_BUFFER; + out_sz = min_out_sz; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SHA256); + if (res) + return res; + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, __text_start, + __text_data_start - __text_start); + if (res) + goto out; + res = crypto_hash_update(ctx, __text_data_end, + __text_end - __text_data_end); + if (IS_ENABLED(CFG_WITH_PAGER)) { + res = crypto_hash_update(ctx, __text_init_start, + __text_init_end - __text_init_start); + if (res) + goto out; + res = crypto_hash_update(ctx, __text_pageable_start, + __text_pageable_end - + __text_pageable_start); + if (res) + goto out; + } + if (res) + goto out; + res = crypto_hash_update(ctx, __rodata_start, + __rodata_end - __rodata_start); + if (res) + goto out; + if (IS_ENABLED(CFG_WITH_PAGER)) { + res = crypto_hash_update(ctx, __rodata_init_start, + __rodata_init_end - + __rodata_init_start); + if (res) + goto out; + res = crypto_hash_update(ctx, __rodata_pageable_start, + __rodata_pageable_end - + __rodata_pageable_start); + if (res) + goto out; + } + res = crypto_hash_final(ctx, out, TEE_SHA256_HASH_SIZE); + if (res) + goto out; + + DHEXDUMP(out, TEE_SHA256_HASH_SIZE); + + res = sign_buffer(out, out_sz, nonce, nonce_sz); +out: + crypto_hash_free_ctx(ctx); + return res; +} + +static TEE_Result invoke_command(void *sess_ctx __unused, uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_ERROR_BAD_PARAMETERS; + TEE_Result res2 = TEE_ERROR_GENERIC; + TEE_Param bparams[TEE_NUM_PARAMS] = { }; + TEE_Param *eparams = NULL; + + res = to_bounce_params(param_types, params, bparams, &eparams); + if (res) + return res; + + switch (cmd_id) { + case PTA_ATTESTATION_GET_PUBKEY: + res = cmd_get_pubkey(param_types, eparams); + break; + case PTA_ATTESTATION_GET_TA_SHDR_DIGEST: + res = cmd_get_ta_shdr_digest(param_types, eparams); + break; + case PTA_ATTESTATION_HASH_TA_MEMORY: + res = cmd_hash_ta_memory(param_types, eparams); + break; + case PTA_ATTESTATION_HASH_TEE_MEMORY: + res = cmd_hash_tee_memory(param_types, eparams); + break; + default: + break; + } + + res2 = from_bounce_params(param_types, params, bparams, eparams); + if (!res && res2) + res = res2; + + return res; +} + +pseudo_ta_register(.uuid = PTA_ATTESTATION_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/bcm/bnxt.c b/optee_os/core/pta/bcm/bnxt.c new file mode 100644 index 0000000..e753305 --- /dev/null +++ b/optee_os/core/pta/bcm/bnxt.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define BNXT_SERVICE_UUID \ + {0x6272636D, 0x2019, 0x0716, \ + {0x42, 0x43, 0x4D, 0x5F, 0x53, 0x43, 0x48, 0x49} } + +/* + * enum pta_bnxt_cmd - commands supported by this PTA + * PTA_BNXT_FASTBOOT: boot bnxt device by copying f/w into sram + * + * PTA_BNXT_HEALTH_STATUS: check health of bnxt device + * [out] value[0].a - health status + * + * PTA_BNXT_HANDSHAKE_STATUS: check bnxt device is booted + * [inout] value[0].a - max timeout value + * value[0].a - boot status + * + * PTA_BNXT_CRASH_DUMP_COPY: copy the core dump into shm + * [inout] memref[0].buf: destination addr + * [in] value[1].a: offset + * [in] value[1].b: size + */ +enum pta_bnxt_cmd { + PTA_BNXT_FASTBOOT = 0, + PTA_BNXT_HEALTH_STATUS, + PTA_BNXT_HANDSHAKE_STATUS, + PTA_BNXT_CRASH_DUMP_COPY, +}; + +#define BNXT_TA_NAME "pta_bnxt.ta" + +static TEE_Result get_bnxt_status(uint32_t type, TEE_Param p[TEE_NUM_PARAMS]) +{ + if (type != TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + p[0].value.a = bnxt_health_status(); + + return TEE_SUCCESS; +} + +static TEE_Result get_bnxt_handshake_status(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS]) +{ + if (type != TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + p[0].value.a = bnxt_wait_handshake(p[0].value.a); + + return TEE_SUCCESS; +} + +static TEE_Result copy_bnxt_crash_dump(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t *d = NULL; + uint32_t offset = 0; + uint32_t req_len = 0; + TEE_Result res = TEE_SUCCESS; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) { + DMSG("bad parameters types: 0x%" PRIx32, types); + return TEE_ERROR_BAD_PARAMETERS; + } + + d = (uint32_t *)params[0].memref.buffer; + offset = params[1].value.a; + req_len = params[1].value.b; + + if (!d || params[0].memref.size < req_len) + return TEE_ERROR_BAD_PARAMETERS; + + res = bnxt_copy_crash_dump((uint8_t *)d, offset, req_len); + + return res; +} + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t cmd_id, + uint32_t param_types __unused, + TEE_Param params[TEE_NUM_PARAMS] __unused) +{ + TEE_Result res = TEE_SUCCESS; + + DMSG("command entry point[%d] for \"%s\"", cmd_id, BNXT_TA_NAME); + + switch (cmd_id) { + case PTA_BNXT_FASTBOOT: + DMSG("bnxt fastboot"); + if (bnxt_load_fw(1) != BNXT_SUCCESS) + return TEE_ERROR_TARGET_DEAD; + break; + case PTA_BNXT_HEALTH_STATUS: + DMSG("bnxt health status"); + return get_bnxt_status(param_types, params); + case PTA_BNXT_HANDSHAKE_STATUS: + DMSG("bnxt handshake status"); + return get_bnxt_handshake_status(param_types, params); + case PTA_BNXT_CRASH_DUMP_COPY: + DMSG("bnxt copy crash dump data"); + return copy_bnxt_crash_dump(param_types, params); + default: + DMSG("cmd: %d Not supported %s", cmd_id, BNXT_TA_NAME); + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + + return res; +} + +pseudo_ta_register(.uuid = BNXT_SERVICE_UUID, + .name = BNXT_TA_NAME, + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_DEVICE_ENUM, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/bcm/elog.c b/optee_os/core/pta/bcm/elog.c new file mode 100644 index 0000000..869e9ea --- /dev/null +++ b/optee_os/core/pta/bcm/elog.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define ELOG_SERVICE_UUID \ + { 0x6272636D, 0x2019, 0x0701, \ + { 0x42, 0x43, 0x4D, 0x5F, 0x45, 0x4C, 0x4F, 0x47 } } + +#define ELOG_TA_NAME "pta_bcm_elog.ta" + +#define BCM_NITRO_FW_LOAD_ADDR 0x8ae00000 +#define BCM_NITRO_CRASH_DUMP_BASE_ADDR 0x8b000000 + +/* Default ELOG buffer size 1MB */ +#define DEFAULT_ELOG_BUFFER_SIZE 0x100000U + +/* + * Get Error log memory dump + * + * [out] memref[0]: Destination + * [in] value[1].a: Offset + */ +#define PTA_BCM_ELOG_CMD_GET_ELOG_MEM 1 + +/* + * Get nitro crash_dump memory + * + * [out] memref[0]: Destination + * [in] value[1].a: Offset + */ +#define PTA_BCM_ELOG_CMD_GET_NITRO_CRASH_DUMP 2 + +/* + * Load nitro firmware memory + * + * [in] memref[0]: Nitro f/w image data + * [in] value[1].a: Offset for loading f/w image + * [in] value[2].a: Firmware image size + */ +#define PTA_BCM_ELOG_CMD_LOAD_NITRO_FW 3 + +#define BCM_ELOG_GLOBAL_METADATA_SIG 0x45524c47 + +#define MAX_NITRO_CRASH_DUMP_MEM_SIZE 0x2000000 +#define MAX_NITRO_FW_LOAD_MEM_SIZE 0x200000 + +/* Load Nitro fw image to SEC DDR memory */ +static TEE_Result pta_elog_load_nitro_fw(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + paddr_t src_paddr = BCM_NITRO_FW_LOAD_ADDR + BNXT_IMG_SECMEM_OFFSET; + vaddr_t src_vaddr = 0; + uint32_t offset = 0; + size_t end_offs = 0; + size_t sz = 0; + char *buf = NULL; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Check if firmware file size exceeds reserved memory size */ + if (params[2].value.a > MAX_NITRO_FW_LOAD_MEM_SIZE) { + EMSG("Invalid access"); + return TEE_ERROR_EXCESS_DATA; + } + + offset = params[1].value.a; + buf = params[0].memref.buffer; + sz = params[0].memref.size; + + /* + * Check that we under no circumstances will attempt to write + * beyond BCM_NITRO_FW_LOAD_ADDR + MAX_NITRO_FW_LOAD_MEM_SIZE. + */ + if (ADD_OVERFLOW(sz, offset, &end_offs) || + end_offs > MAX_NITRO_FW_LOAD_MEM_SIZE - BNXT_IMG_SECMEM_OFFSET) { + EMSG("Invalid access"); + return TEE_ERROR_ACCESS_DENIED; + } + + src_vaddr = (vaddr_t)phys_to_virt((uintptr_t)src_paddr + offset, + MEM_AREA_RAM_SEC, sz); + if (!src_vaddr) { + EMSG("Not enough memory mapped"); + return TEE_ERROR_BAD_PARAMETERS; + } + + memcpy((char *)src_vaddr, buf, sz); + + cache_op_inner(DCACHE_AREA_CLEAN, (void *)src_vaddr, sz); + + return res; +} + +static uint32_t get_dump_data(vaddr_t src, TEE_Param params[TEE_NUM_PARAMS]) +{ + char *buf = NULL; + uint32_t sz = 0; + + buf = params[0].memref.buffer; + sz = params[0].memref.size; + + /* + * If request size exceeds default buf size + * override request size to default DEFAULT_ELOG_BUFFER_SIZE + */ + if (sz > DEFAULT_ELOG_BUFFER_SIZE) + sz = DEFAULT_ELOG_BUFFER_SIZE; + + DMSG("buf %p sz 0x%x", buf, sz); + + memcpy(buf, (char *)src, sz); + + params[0].memref.size = sz; + + return sz; +} + +/* Copy nitro crash dump data */ +static TEE_Result pta_elog_nitro_crash_dump(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + paddr_t src_paddr = BCM_NITRO_CRASH_DUMP_BASE_ADDR; + vaddr_t src_vaddr = 0; + uint32_t offset = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + uint32_t sz = 0; + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + offset = params[1].value.a; + + /* + * Check if offset is within memory range reserved for nitro crash dump + * minus default size of buffer + */ + if (offset > MAX_NITRO_CRASH_DUMP_MEM_SIZE - DEFAULT_ELOG_BUFFER_SIZE) { + EMSG("Invalid access"); + return TEE_ERROR_ACCESS_DENIED; + } + + sz = MIN(params[0].memref.size, DEFAULT_ELOG_BUFFER_SIZE); + src_vaddr = (vaddr_t)phys_to_virt((uintptr_t)src_paddr + offset, + MEM_AREA_RAM_SEC, sz); + if (!src_vaddr) { + EMSG("Not enough memory mapped"); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* TODO : check if NITRO_CRASH_DUMP is available */ + + cache_op_inner(DCACHE_AREA_INVALIDATE, (void *)src_vaddr, + DEFAULT_ELOG_BUFFER_SIZE); + + get_dump_data(src_vaddr, params); + + return res; +} + +/* Copy soc error log data */ +static TEE_Result pta_elog_dump(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + paddr_t src_paddr = CFG_BCM_ELOG_BASE; + vaddr_t src_vaddr = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + uint32_t sz = 0; + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + sz = MIN(params[0].memref.size, DEFAULT_ELOG_BUFFER_SIZE); + src_vaddr = (vaddr_t)phys_to_virt(src_paddr, MEM_AREA_RAM_NSEC, sz); + if (!src_vaddr) { + EMSG("Not enough memory mapped"); + return TEE_ERROR_BAD_PARAMETERS; + } + + /* Validate if Error logs are present */ + if ((*(uint32_t *)src_vaddr) != BCM_ELOG_GLOBAL_METADATA_SIG) { + EMSG("Elog Not setup"); + return TEE_ERROR_NOT_SUPPORTED; + } + + get_dump_data(src_vaddr, params); + + return res; +} + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + + DMSG("command entry point[%d] for \"%s\"", cmd_id, ELOG_TA_NAME); + + switch (cmd_id) { + case PTA_BCM_ELOG_CMD_GET_ELOG_MEM: + res = pta_elog_dump(param_types, params); + break; + case PTA_BCM_ELOG_CMD_GET_NITRO_CRASH_DUMP: + res = pta_elog_nitro_crash_dump(param_types, params); + break; + case PTA_BCM_ELOG_CMD_LOAD_NITRO_FW: + res = pta_elog_load_nitro_fw(param_types, params); + break; + default: + EMSG("cmd: %d Not supported %s", cmd_id, ELOG_TA_NAME); + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + + return res; +} + +pseudo_ta_register(.uuid = ELOG_SERVICE_UUID, + .name = ELOG_TA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/bcm/gpio.c b/optee_os/core/pta/bcm/gpio.c new file mode 100644 index 0000000..43ed935 --- /dev/null +++ b/optee_os/core/pta/bcm/gpio.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019 Broadcom. + */ + +#include +#include +#include +#include + +#define GPIO_SERVICE_UUID \ + { 0x6272636D, 0x2018, 0x1101, \ + { 0x42, 0x43, 0x4D, 0x5F, 0x47, 0x50, 0x49, 0x4F } } + +/* + * Configure GPIO Pin + * + * [in] value[0].a: gpio pin number + * [in] value[0].b: direction to configure + */ +#define PTA_BCM_GPIO_CMD_CFG 0 + +/* + * Set GPIO pin + * + * [in] value[0].a: gpio pin number + * [in] value[0].b: value drive on pin + */ +#define PTA_BCM_GPIO_CMD_SET 1 + +/* + * Get GPIO pin + * + * [in] value[0].a: gpio pin number + * [out] value[1].a: value read from gpio pin + */ +#define PTA_BCM_GPIO_CMD_GET 2 + +#define GPIO_TA_NAME "pta_bcm_gpio.ta" + +static TEE_Result pta_gpio_config(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t gpio_num = 0; + struct bcm_gpio_chip *bcm_gc = NULL; + struct gpio_chip *gc = NULL; + bool dir = false; + TEE_Result res = TEE_SUCCESS; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + gpio_num = params[0].value.a; + dir = params[0].value.b; + + bcm_gc = bcm_gpio_pin_to_chip(gpio_num); + if (!bcm_gc) { + EMSG("GPIO %u not supported", gpio_num); + return TEE_ERROR_NOT_SUPPORTED; + } + + gc = &bcm_gc->chip; + + /* Make gpio secure. */ + iproc_gpio_set_secure(gpio_num); + + if (dir) { + /* Set GPIO to output with default value to 0 */ + gc->ops->set_direction(NULL, gpio_num, GPIO_DIR_OUT); + gc->ops->set_value(NULL, gpio_num, 0); + } else { + gc->ops->set_direction(NULL, gpio_num, GPIO_DIR_IN); + } + + return res; +} + +static TEE_Result pta_gpio_set(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t gpio_num = 0; + uint32_t val = 0; + TEE_Result res = TEE_SUCCESS; + struct bcm_gpio_chip *bcm_gc = NULL; + struct gpio_chip *gc = NULL; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + gpio_num = params[0].value.a; + val = !!params[0].value.b; + + bcm_gc = bcm_gpio_pin_to_chip(gpio_num); + if (!bcm_gc) { + EMSG("GPIO %u not supported", gpio_num); + return TEE_ERROR_NOT_SUPPORTED; + } + + gc = &bcm_gc->chip; + + /* + * For setting a value to GPIO Pin, + * need to make sure the PIN is configured in + * output direction. + */ + if (gc->ops->get_direction(NULL, gpio_num) != GPIO_DIR_OUT) { + EMSG("gpio pin %u is configured as INPUT", gpio_num); + return TEE_ERROR_ACCESS_DENIED; + } + + gc->ops->set_value(NULL, gpio_num, val); + + DMSG("GPIO(%d) value = 0x%08x", gpio_num, + gc->ops->get_value(NULL, gpio_num)); + + return res; +} + +static TEE_Result pta_gpio_get(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t gpio_num = 0; + struct bcm_gpio_chip *bcm_gc = NULL; + struct gpio_chip *gc = NULL; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + gpio_num = params[0].value.a; + + bcm_gc = bcm_gpio_pin_to_chip(gpio_num); + if (!bcm_gc) { + EMSG("GPIO %u not supported", gpio_num); + return TEE_ERROR_NOT_SUPPORTED; + } + + gc = &bcm_gc->chip; + + params[1].value.a = gc->ops->get_value(NULL, gpio_num); + + DMSG("gpio(%d) value = 0x%08x", gpio_num, params[1].value.a); + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + + DMSG("command entry point[%d] for \"%s\"", cmd_id, GPIO_TA_NAME); + + switch (cmd_id) { + case PTA_BCM_GPIO_CMD_CFG: + res = pta_gpio_config(param_types, params); + break; + case PTA_BCM_GPIO_CMD_SET: + res = pta_gpio_set(param_types, params); + break; + case PTA_BCM_GPIO_CMD_GET: + res = pta_gpio_get(param_types, params); + break; + default: + EMSG("cmd: %d Not supported %s\n", cmd_id, GPIO_TA_NAME); + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + + return res; +} + +pseudo_ta_register(.uuid = GPIO_SERVICE_UUID, + .name = GPIO_TA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/bcm/hwrng.c b/optee_os/core/pta/bcm/hwrng.c new file mode 100644 index 0000000..daaae5a --- /dev/null +++ b/optee_os/core/pta/bcm/hwrng.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Broadcom + */ + +#include +#include +#include +#include + +#define HWRNG_SERVICE_UUID \ + { 0x6272636D, 0x2019, 0x0201, \ + { 0x42, 0x43, 0x4D, 0x5F, 0x52, 0x4E, 0x47, 0x30 } } + +/* + * Get a HW generated random number + * + * [out] value[0].a: Generated 32-bit random number + */ +#define PTA_BCM_HWRNG_CMD_GET 0 + +#define HWRNG_TA_NAME "pta_hwrng.ta" + +static TEE_Result pta_hwrng_get(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t num_words = 0; + uint32_t rnd_num = 0; + uint32_t res = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + num_words = bcm_hwrng_read_rng(&rnd_num, 1); + + if (num_words < 1) { + res = TEE_ERROR_NO_DATA; + } else { + DMSG("Random Value is: 0x%08x", rnd_num); + params[0].value.a = rnd_num; + res = TEE_SUCCESS; + } + + return res; +} + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + + DMSG("command entry point[%d] for \"%s\"", cmd_id, HWRNG_TA_NAME); + + switch (cmd_id) { + case PTA_BCM_HWRNG_CMD_GET: + res = pta_hwrng_get(param_types, params); + break; + default: + EMSG("cmd: %d Not supported %s", cmd_id, HWRNG_TA_NAME); + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + + return res; +} + +pseudo_ta_register(.uuid = HWRNG_SERVICE_UUID, + .name = HWRNG_TA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/bcm/sotp.c b/optee_os/core/pta/bcm/sotp.c new file mode 100644 index 0000000..9b97265 --- /dev/null +++ b/optee_os/core/pta/bcm/sotp.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Broadcom + */ + +#include +#include +#include +#include +#include + +#define SOTP_SERVICE_UUID \ + {0x6272636D, 0x2018, 0x1101, \ + {0x42, 0x43, 0x4D, 0x5F, 0x53, 0x4F, 0x54, 0x50} } + +enum pta_bcm_sotp_cmd { + PTA_BCM_SOTP_CMD_READ = 0, + PTA_BCM_SOTP_CMD_WRITE, +}; + +#define SOTP_TA_NAME "pta_bcm_sotp.ta" + +static bool sotp_access_disabled; + +/** + * close_session() - Print a debug message when closing a session and set the + * driver to disallow any more pta sessions to connect. + * @pSessionContext Unused. + */ +static void close_session(void *pSessionContext __unused) +{ + DMSG("close entry point for \"%s\"", SOTP_TA_NAME); + if (IS_ENABLED(CFG_BCM_SOTP_SINGLE_SESSION)) + sotp_access_disabled = true; +} + +static TEE_Result pta_sotp_read(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint64_t sotp_row_value = 0; + uint32_t val = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + val = params[0].value.a; + + bcm_iproc_sotp_mem_read(val, true, &sotp_row_value); + reg_pair_from_64(sotp_row_value, ¶ms[1].value.a, + ¶ms[1].value.b); + + return TEE_SUCCESS; +} + +static TEE_Result pta_sotp_write(uint32_t param_types __unused, + TEE_Param params[TEE_NUM_PARAMS] __unused) +{ + /* Nothing as of now */ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + + DMSG("command entry point[%d] for \"%s\"", cmd_id, SOTP_TA_NAME); + + if (IS_ENABLED(CFG_BCM_SOTP_SINGLE_SESSION) && sotp_access_disabled) { + DMSG("bcm sotp pta access disabled"); + return TEE_ERROR_ACCESS_DENIED; + } + + switch (cmd_id) { + case PTA_BCM_SOTP_CMD_READ: + res = pta_sotp_read(param_types, params); + break; + case PTA_BCM_SOTP_CMD_WRITE: + res = pta_sotp_write(param_types, params); + break; + default: + EMSG("cmd %d Not supported %s", cmd_id, SOTP_TA_NAME); + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + + return res; +} + +pseudo_ta_register(.uuid = SOTP_SERVICE_UUID, + .name = SOTP_TA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .close_session_entry_point = close_session, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/bcm/sub.mk b/optee_os/core/pta/bcm/sub.mk new file mode 100644 index 0000000..63a29cb --- /dev/null +++ b/optee_os/core/pta/bcm/sub.mk @@ -0,0 +1,6 @@ +srcs-$(CFG_BNXT_FW) += bnxt.c +srcs-$(CFG_SP805_WDT) += wdt.c +srcs-$(CFG_BCM_HWRNG) += hwrng.c +srcs-$(CFG_BCM_SOTP) += sotp.c +srcs-$(CFG_BCM_GPIO) += gpio.c +srcs-$(CFG_BCM_ELOG_DUMP) += elog.c diff --git a/optee_os/core/pta/bcm/wdt.c b/optee_os/core/pta/bcm/wdt.c new file mode 100644 index 0000000..dceaa13 --- /dev/null +++ b/optee_os/core/pta/bcm/wdt.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Broadcom + */ + +#include +#include +#include +#include +#include +#include +#include + +#define SEC_WD_SERVICE_UUID \ + { 0x6272636D, 0x2019, 0x0801, \ + { 0x42, 0x43, 0x4D, 0x5F, 0x57, 0x44, 0x54, 0x30 } } + +#define PTA_BCM_SEC_WD_CMD_CONFIG 0 +#define PTA_BCM_SEC_WD_CMD_START 1 +#define PTA_BCM_SEC_WD_CMD_PING 2 +#define PTA_BCM_SEC_WD_CMD_STOP 3 +#define PTA_BCM_SEC_WD_CMD_SET_TIMEOUT 4 + +#define SEC_WD_TA_NAME "pta_bcm_sec_wd.ta" + +static struct sp805_wdt_data wd_pd; + +static void wd_isr_handler(struct wdt_chip *chip __unused) +{ + /* Do nothing */ + DMSG("Watchdog ISR handler !!!"); +} + +static TEE_Result pta_wd_config(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t timeout = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + timeout = params[0].value.a; + + sp805_wdt_init(&wd_pd, SEC_WDT_BASE, SEC_WDT_CLK_HZ, timeout); + + sp805_register_itr_handler(&wd_pd, GIC_SPI_TO_ITNUM(SEC_WDT_INTR), + ITRF_TRIGGER_LEVEL, wd_isr_handler); + wd_pd.chip.ops->start(&wd_pd.chip); + + return TEE_SUCCESS; +} + +static TEE_Result pta_wd_start(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS] __unused) +{ + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + wd_pd.chip.ops->start(&wd_pd.chip); + + return TEE_SUCCESS; +} + +static TEE_Result pta_wd_ping(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS] __unused) +{ + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + wd_pd.chip.ops->ping(&wd_pd.chip); + + return TEE_SUCCESS; +} + +static TEE_Result pta_wd_stop(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS] __unused) +{ + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + wd_pd.chip.ops->stop(&wd_pd.chip); + + return TEE_SUCCESS; +} + +static TEE_Result pta_wd_set_timeout(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t timeout = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_param_types != param_types) { + EMSG("Invalid Param types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + timeout = params[0].value.a; + + wd_pd.chip.ops->set_timeout(&wd_pd.chip, timeout); + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + + DMSG("command entry point[%d] for \"%s\"", cmd_id, SEC_WD_TA_NAME); + + switch (cmd_id) { + case PTA_BCM_SEC_WD_CMD_CONFIG: + res = pta_wd_config(param_types, params); + break; + case PTA_BCM_SEC_WD_CMD_START: + res = pta_wd_start(param_types, params); + break; + case PTA_BCM_SEC_WD_CMD_PING: + res = pta_wd_ping(param_types, params); + break; + case PTA_BCM_SEC_WD_CMD_STOP: + res = pta_wd_stop(param_types, params); + break; + case PTA_BCM_SEC_WD_CMD_SET_TIMEOUT: + res = pta_wd_set_timeout(param_types, params); + break; + default: + EMSG("cmd: %d Not supported %s", cmd_id, SEC_WD_TA_NAME); + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + + return res; +} + +pseudo_ta_register(.uuid = SEC_WD_SERVICE_UUID, + .name = SEC_WD_TA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/benchmark.c b/optee_os/core/pta/benchmark.c new file mode 100644 index 0000000..bfaa426 --- /dev/null +++ b/optee_os/core/pta/benchmark.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TA_NAME "benchmark.ta" +#define TA_PRINT_PREFIX "Benchmark: " + +static struct tee_ts_global *bench_ts_global; +static size_t bench_ts_size; + +static struct mutex bench_reg_mu = MUTEX_INITIALIZER; +static struct mobj *bench_mobj; + +static TEE_Result rpc_reg_global_buf(uint64_t type, paddr_t phta, size_t size) +{ + struct thread_param tpm = THREAD_PARAM_VALUE(IN, type, phta, size); + + return thread_rpc_cmd(OPTEE_RPC_CMD_BENCH_REG, 1, &tpm); +} + +static TEE_Result alloc_benchmark_buffer(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS]) +{ + TEE_Result res; + + if ((TEE_PARAM_TYPE_GET(type, 0) != TEE_PARAM_TYPE_VALUE_INOUT) || + (TEE_PARAM_TYPE_GET(type, 1) != TEE_PARAM_TYPE_VALUE_INPUT) || + (TEE_PARAM_TYPE_GET(type, 2) != TEE_PARAM_TYPE_NONE) || + (TEE_PARAM_TYPE_GET(type, 3) != TEE_PARAM_TYPE_NONE)) { + return TEE_ERROR_BAD_PARAMETERS; + } + + mutex_lock(&bench_reg_mu); + + /* Check if we have already registered buffer */ + if (bench_ts_global) { + EMSG(TA_PRINT_PREFIX + "timestamp buffer was already registered"); + mutex_unlock(&bench_reg_mu); + return TEE_ERROR_BAD_STATE; + } + + bench_ts_size = sizeof(struct tee_ts_global) + + p[1].value.a * sizeof(struct tee_ts_cpu_buf); + if (!bench_ts_size) { + EMSG(TA_PRINT_PREFIX + "invalid timestamp buffer size"); + mutex_unlock(&bench_reg_mu); + return TEE_ERROR_BAD_STATE; + } + + bench_mobj = thread_rpc_alloc_global_payload(bench_ts_size); + if (!bench_mobj) { + EMSG(TA_PRINT_PREFIX + "can't create mobj for timestamp buffer"); + mutex_unlock(&bench_reg_mu); + return TEE_ERROR_OUT_OF_MEMORY; + } + + bench_ts_global = (struct tee_ts_global *)mobj_get_va(bench_mobj, 0, + bench_ts_size); + if (!bench_ts_global) { + thread_rpc_free_global_payload(bench_mobj); + bench_mobj = NULL; + + mutex_unlock(&bench_reg_mu); + return TEE_ERROR_BAD_STATE; + } + + memset((void *)bench_ts_global, 0, bench_ts_size); + bench_ts_global->cores = p[1].value.a; + + DMSG(TA_PRINT_PREFIX + "allocated timestamp buffer, addr = %p", + (void *)bench_ts_global); + + mutex_unlock(&bench_reg_mu); + + /* Send back to the optee linux kernel module */ + res = rpc_reg_global_buf(OPTEE_MSG_RPC_CMD_BENCH_REG_NEW, + virt_to_phys((void *)bench_ts_global), + bench_ts_size); + + p[0].value.a = virt_to_phys((void *)bench_ts_global); + p[0].value.b = bench_ts_size; + + return res; +} + +static TEE_Result get_benchmark_memref(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS]) +{ + if ((TEE_PARAM_TYPE_GET(type, 0) != TEE_PARAM_TYPE_VALUE_OUTPUT) || + (TEE_PARAM_TYPE_GET(type, 1) != TEE_PARAM_TYPE_NONE) || + (TEE_PARAM_TYPE_GET(type, 2) != TEE_PARAM_TYPE_NONE) || + (TEE_PARAM_TYPE_GET(type, 3) != TEE_PARAM_TYPE_NONE)) { + return TEE_ERROR_BAD_PARAMETERS; + } + + mutex_lock(&bench_reg_mu); + + DMSG(TA_PRINT_PREFIX "Sending back timestamp buffer paddr = %p", + (void *)virt_to_phys((void *)bench_ts_global)); + + if (bench_ts_global) { + p[0].value.a = virt_to_phys((void *)bench_ts_global); + p[0].value.b = bench_ts_size; + } else { + p[0].value.a = 0; + p[0].value.b = 0; + } + + mutex_unlock(&bench_reg_mu); + + return TEE_SUCCESS; +} + +static TEE_Result unregister_benchmark(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS] __unused) +{ + TEE_Result res; + + if ((TEE_PARAM_TYPE_GET(type, 0) != TEE_PARAM_TYPE_NONE) || + (TEE_PARAM_TYPE_GET(type, 1) != TEE_PARAM_TYPE_NONE) || + (TEE_PARAM_TYPE_GET(type, 2) != TEE_PARAM_TYPE_NONE) || + (TEE_PARAM_TYPE_GET(type, 3) != TEE_PARAM_TYPE_NONE)) { + return TEE_ERROR_BAD_PARAMETERS; + } + mutex_lock(&bench_reg_mu); + + DMSG(TA_PRINT_PREFIX "Unregister benchmark ts buffer paddr = %p", + (void *)virt_to_phys((void *)bench_ts_global)); + bench_ts_global = NULL; + + mutex_unlock(&bench_reg_mu); + + res = rpc_reg_global_buf(OPTEE_MSG_RPC_CMD_BENCH_REG_DEL, 0, 0); + + thread_rpc_free_global_payload(bench_mobj); + bench_mobj = NULL; + + return res; +} + +static TEE_Result invoke_command(void *session_ctx __unused, + uint32_t cmd_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd_id) { + case BENCHMARK_CMD_ALLOCATE_BUF: + return alloc_benchmark_buffer(param_types, params); + case BENCHMARK_CMD_GET_MEMREF: + return get_benchmark_memref(param_types, params); + case BENCHMARK_CMD_UNREGISTER: + return unregister_benchmark(param_types, params); + default: + break; + } + + return TEE_ERROR_BAD_PARAMETERS; +} + +pseudo_ta_register(.uuid = BENCHMARK_UUID, .name = TA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); + +void bm_timestamp(void) +{ + struct tee_ts_cpu_buf *cpu_buf; + struct tee_time_st ts_data; + uint64_t ts_i; + void *ret_addr; + uint32_t cur_cpu; + uint32_t exceptions; + + if (!bench_ts_global) + return; + + exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); + cur_cpu = get_core_pos(); + + if (cur_cpu >= bench_ts_global->cores) { + thread_unmask_exceptions(exceptions); + return; + } + + ret_addr = __builtin_return_address(0); + + cpu_buf = &bench_ts_global->cpu_buf[cur_cpu]; + ts_i = cpu_buf->head++; + ts_data.cnt = read_pmccntr() * TEE_BENCH_DIVIDER; + ts_data.addr = (uintptr_t)ret_addr; + ts_data.src = TEE_BENCH_CORE; + cpu_buf->stamps[ts_i & TEE_BENCH_MAX_MASK] = ts_data; + + thread_unmask_exceptions(exceptions); +} diff --git a/optee_os/core/pta/device.c b/optee_os/core/pta/device.c new file mode 100644 index 0000000..1a026a6 --- /dev/null +++ b/optee_os/core/pta/device.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + */ + +/* + * This pseudo TA is used by normal world OS TEE driver to fetch pseudo TA's + * UUIDs which can act as TEE bus devices. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PTA_NAME "device.pta" + +static void add_ta(uint32_t flags, const TEE_UUID *uuid, uint8_t *buf, + uint32_t blen, uint32_t *pos, uint32_t rflags) +{ + if ((flags & TA_FLAG_DEVICE_ENUM) && + (flags & TA_FLAG_DEVICE_ENUM_SUPP)) { + EMSG(PTA_NAME ": skipping TA %pUl, inconsistent flags", uuid); + return; + } + + if (flags & rflags) { + if (*pos + sizeof(*uuid) <= blen) + tee_uuid_to_octets(buf + *pos, uuid); + + *pos += sizeof(*uuid); + } +} + +static TEE_Result get_devices(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS], + uint32_t rflags) +{ + const struct pseudo_ta_head *ta = NULL; + const struct embedded_ts *eta = NULL; + void *buf = NULL; + uint32_t blen = 0; + uint32_t pos = 0; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!params[0].memref.buffer && (params[0].memref.size > 0)) + return TEE_ERROR_BAD_PARAMETERS; + + buf = params[0].memref.buffer; + blen = params[0].memref.size; + + SCATTERED_ARRAY_FOREACH(ta, pseudo_tas, struct pseudo_ta_head) + add_ta(ta->flags, &ta->uuid, buf, blen, &pos, rflags); + + if (stmm_get_uuid()) + add_ta(TA_FLAG_DEVICE_ENUM_SUPP, stmm_get_uuid(), buf, blen, + &pos, rflags); + + if (IS_ENABLED(CFG_EARLY_TA)) + for_each_early_ta(eta) + add_ta(eta->flags, &eta->uuid, buf, blen, &pos, + rflags); + + params[0].memref.size = pos; + if (pos > blen) + return TEE_ERROR_SHORT_BUFFER; + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *pSessionContext __unused, + uint32_t nCommandID, uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]) +{ + switch (nCommandID) { + case PTA_CMD_GET_DEVICES: + return get_devices(nParamTypes, pParams, + TA_FLAG_DEVICE_ENUM); + case PTA_CMD_GET_DEVICES_SUPP: + return get_devices(nParamTypes, pParams, + TA_FLAG_DEVICE_ENUM_SUPP); + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_DEVICE_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/gprof.c b/optee_os/core/pta/gprof.c new file mode 100644 index 0000000..d999687 --- /dev/null +++ b/optee_os/core/pta/gprof.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static TEE_Result gprof_send_rpc(TEE_UUID *uuid, void *buf, size_t len, + uint32_t *id) +{ + struct thread_param params[3] = { }; + struct mobj *mobj; + TEE_Result res = TEE_ERROR_GENERIC; + char *va; + + mobj = thread_rpc_alloc_payload(sizeof(*uuid) + len); + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + + va = mobj_get_va(mobj, 0, sizeof(*uuid) + len); + if (!va) + goto exit; + + memcpy(va, uuid, sizeof(*uuid)); + memcpy(va + sizeof(*uuid), buf, len); + + params[0] = THREAD_PARAM_VALUE(INOUT, *id, 0, 0); + params[1] = THREAD_PARAM_MEMREF(IN, mobj, 0, sizeof(*uuid)); + params[2] = THREAD_PARAM_MEMREF(IN, mobj, sizeof(*uuid), len); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_GPROF, 3, params); + if (res != TEE_SUCCESS) + goto exit; + + *id = (uint32_t)params[0].u.value.a; +exit: + thread_rpc_free_payload(mobj); + return res; +} + +static TEE_Result gprof_send(struct ts_session *s, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INOUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + return gprof_send_rpc(&s->ctx->uuid, params[1].memref.buffer, + params[1].memref.size, ¶ms[0].value.a); +} + +static TEE_Result gprof_start_pc_sampling(struct ts_session *s, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + struct sample_buf *sbuf = NULL; + uint32_t offset = 0; + uint32_t scale = 0; + uint32_t len = 0; + uaddr_t buf = 0; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (s->sbuf) { + DMSG("PC sampling already started"); + return TEE_ERROR_BAD_STATE; + } + + buf = (uaddr_t)params[0].memref.buffer; + len = params[0].memref.size; + offset = params[1].value.a; + scale = params[1].value.b; + + sbuf = calloc(1, sizeof(*sbuf)); + if (!sbuf) + return TEE_ERROR_OUT_OF_MEMORY; + + sbuf->samples = (uint16_t *)buf; + sbuf->nsamples = len / sizeof(*sbuf->samples); + sbuf->offset = offset; + sbuf->scale = scale; + sbuf->freq = read_cntfrq(); + sbuf->enabled = true; + s->sbuf = sbuf; + + return TEE_SUCCESS; +} + +static TEE_Result gprof_stop_pc_sampling(struct ts_session *s, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + struct sample_buf *sbuf = NULL; + uint32_t rate = 0; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + sbuf = s->sbuf; + if (!sbuf) + return TEE_ERROR_BAD_STATE; + assert(sbuf->samples); + + /* Stop sampling */ + if (sbuf->enabled) + sbuf->enabled = false; + + rate = ((uint64_t)sbuf->count * sbuf->freq) / sbuf->usr; + params[0].value.a = rate; + + DMSG("TA sampling stats: sample count=%" PRIu32 " user time=%" PRIu64 + " cntfrq=%" PRIu32 " rate=%" PRIu32, sbuf->count, sbuf->usr, + sbuf->freq, rate); + + free(sbuf); + s->sbuf = NULL; + + return TEE_SUCCESS; +} + +/* + * Trusted Application Entry Points + */ + +static TEE_Result open_session(uint32_t param_types __unused, + TEE_Param params[TEE_NUM_PARAMS] __unused, + void **sess_ctx __unused) +{ + struct ts_session *s = ts_get_calling_session(); + + /* Check that we're called from a user TA */ + if (!s) + return TEE_ERROR_ACCESS_DENIED; + if (!is_user_ta_ctx(s->ctx)) + return TEE_ERROR_ACCESS_DENIED; + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *sess_ctx __unused, uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct ts_session *s = ts_get_calling_session(); + + switch (cmd_id) { + case PTA_GPROF_SEND: + return gprof_send(s, param_types, params); + case PTA_GPROF_START_PC_SAMPLING: + return gprof_start_pc_sampling(s, param_types, params); + case PTA_GPROF_STOP_PC_SAMPLING: + return gprof_stop_pc_sampling(s, param_types, params); + default: + break; + } + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_GPROF_UUID, .name = "gprof", + .flags = PTA_DEFAULT_FLAGS, + .open_session_entry_point = open_session, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/hwrng.c b/optee_os/core/pta/hwrng.c new file mode 100644 index 0000000..ed089a1 --- /dev/null +++ b/optee_os/core/pta/hwrng.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, Linaro Limited + * Copyright (c) 2021, EPAM Systems. All rights reserved. + * + * Based on plat-synquacer/rng_pta.c + * + */ + +#include +#include +#include +#include +#include +#include + +#define PTA_NAME "rng.pta" + +/* This PTA only works with hardware random number generators */ +static_assert(!IS_ENABLED(CFG_WITH_SOFTWARE_PRNG)); + +static TEE_Result rng_get_entropy(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint8_t *e = NULL; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) { + DMSG("bad parameters types: 0x%" PRIx32, types); + return TEE_ERROR_BAD_PARAMETERS; + } + + e = (uint8_t *)params[0].memref.buffer; + if (!e) + return TEE_ERROR_BAD_PARAMETERS; + + return crypto_rng_read(e, params[0].memref.size); +} + +static TEE_Result rng_get_info(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) { + DMSG("bad parameters types: 0x%" PRIx32, types); + return TEE_ERROR_BAD_PARAMETERS; + } + + params[0].value.a = CFG_HWRNG_RATE; + params[0].value.b = CFG_HWRNG_QUALITY; + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *session __unused, + uint32_t cmd, uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + FMSG(PTA_NAME" command %#"PRIx32" ptypes %#"PRIx32, cmd, ptypes); + + switch (cmd) { + case PTA_CMD_GET_ENTROPY: + return rng_get_entropy(ptypes, params); + case PTA_CMD_GET_RNG_INFO: + return rng_get_info(ptypes, params); + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_RNG_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_CONCURRENT | + TA_FLAG_DEVICE_ENUM, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/imx/dek_blob.c b/optee_os/core/pta/imx/dek_blob.c new file mode 100644 index 0000000..b7e311b --- /dev/null +++ b/optee_os/core/pta/imx/dek_blob.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2019, 2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PTA_NAME "dek_blob.pta" + +/* Blob size padding in bytes */ +#define BLOB_PAD_SIZE 48 + +/* HAB Blob header values */ +#define HAB_HDR_TAG 0x81 +#define HAB_HDR_V4 0x43 +#define HAB_HDR_MODE_CCM 0x66 +#define HAB_HDR_ALG_AES 0x55 + +/* + * DEK blobs are stored by the HAB in a secret key blob data structure. Notice + * that the HAB supports a set of encryption algorithms, but the encrypted boot + * protocol expects AES. The key length is a variable; it can be 128-bit, + * 192-bit, or 256-bit. + * For more info, see NXP application note AN12056 + */ +struct dek_blob_header { + uint8_t tag; /* Constant identifying HAB struct: 0x81 */ + uint8_t len_msb; /* Struct length in 8-bit msb */ + uint8_t len_lsb; /* Struct length in 8-bit lsb */ + uint8_t par; /* Constant value, HAB version: 0x43 */ + uint8_t mode; /* AES encryption CCM mode: 0x66 */ + uint8_t alg; /* AES encryption alg: 0x55 */ + uint8_t size; /* Unwrapped key value size in bytes */ + uint8_t flg; /* Key flags */ +}; + +/* + * Generate HAB DEK blob for encrypted boot + * + * payload [in] Plain text key to encapsulate. + * payload_size [in] Plain text key size. Must be 128, 192 or 256 bits. + * blob [out] DEK blob. + * blob_size [in/out] DEK blob size. + */ +static TEE_Result do_generate(const uint8_t *payload, size_t payload_size, + uint8_t *blob, size_t *blob_size) +{ + struct dek_blob_header *header = NULL; + size_t expected_blob_size = 0; + size_t dek_size = 0; + + /* + * Prevent against an unexpected padding of dek_blob_header structure + * that must remain packed. This structure will be seriailized to a + * buffer along the DEK blob. + */ + static_assert(sizeof(struct dek_blob_header) == 8 * sizeof(uint8_t)); + + assert(payload && blob && payload_size && blob_size); + assert(payload_size == (128 / 8) || payload_size == (192 / 8) || + payload_size == (256 / 8)); + + /* + * The DEK size is equals to input key size plus the required blob + * padding. The total output size is the DEK size plus its header + */ + dek_size = payload_size + BLOB_PAD_SIZE; + expected_blob_size = sizeof(*header) + dek_size; + + /* Check that the output buffer has the required size */ + if (*blob_size < expected_blob_size) { + *blob_size = expected_blob_size; + return TEE_ERROR_SHORT_BUFFER; + } + *blob_size = expected_blob_size; + + /* Defined header */ + header = (struct dek_blob_header *)blob; + header->tag = HAB_HDR_TAG; + header->len_msb = 0; + header->len_lsb = expected_blob_size; + header->par = HAB_HDR_V4; + header->mode = HAB_HDR_MODE_CCM; + header->alg = HAB_HDR_ALG_AES; + header->size = payload_size; + header->flg = 0; + + /* Generate DEK */ + return caam_dek_generate(payload, payload_size, blob + sizeof(*header), + dek_size); +} + +static TEE_Result cmd_dek_generate(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (params[0].memref.size != (128 / 8) && + params[0].memref.size != (192 / 8) && + params[0].memref.size != (256 / 8)) + return TEE_ERROR_BAD_PARAMETERS; + + memset(params[1].memref.buffer, 0, params[1].memref.size); + + return do_generate(params[0].memref.buffer, params[0].memref.size, + params[1].memref.buffer, ¶ms[1].memref.size); +} + +static TEE_Result invoke_command(void *sess_ctx __unused, uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd_id) { + case PTA_IMX_DEK_BLOB_CMD_GENERATE: + return cmd_dek_generate(param_types, params); + default: + return TEE_ERROR_BAD_PARAMETERS; + } +} + +pseudo_ta_register(.uuid = PTA_DEK_BLOB_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/imx/digprog.c b/optee_os/core/pta/imx/digprog.c new file mode 100644 index 0000000..ae569cc --- /dev/null +++ b/optee_os/core/pta/imx/digprog.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + */ +#include +#include +#include + +#define DIGPROG_PTA_NAME "digprog.pta" + +static TEE_Result invokeCommandEntryPoint(void *sess_ctx __unused, + uint32_t cmd_id __unused, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + params[0].value.a = imx_get_digprog(); + params[0].value.b = 0; + + return TEE_SUCCESS; +} + +pseudo_ta_register(.uuid = PTA_DIGPROG_UUID, .name = DIGPROG_PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invokeCommandEntryPoint); diff --git a/optee_os/core/pta/imx/manufacturing_protection.c b/optee_os/core/pta/imx/manufacturing_protection.c new file mode 100644 index 0000000..22aa41c --- /dev/null +++ b/optee_os/core/pta/imx/manufacturing_protection.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2018-2019, 2023 NXP + */ +#include +#include +#include +#include +#include +#include +#include + +#define PTA_NAME "manufacturing_protection.pta" + +static TEE_Result mp_get_public_key(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint8_t *data = NULL; + size_t size = 0; + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + data = params[0].memref.buffer; + size = params[0].memref.size; + + res = caam_mp_export_publickey(data, &size); + if (res != TEE_SUCCESS) + EMSG("MP public key export failed with code 0x%" PRIx32, res); + + params[0].memref.size = size; + return res; +} + +static TEE_Result mp_signature(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *msg = NULL; + uint8_t *sig = NULL; + uint8_t *mpmr = NULL; + size_t msg_size = 0; + size_t sig_size = 0; + size_t mpmr_size = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + DMSG("MPSign function"); + + msg = params[0].memref.buffer; + msg_size = params[0].memref.size; + sig = params[1].memref.buffer; + sig_size = params[1].memref.size; + mpmr = params[2].memref.buffer; + mpmr_size = params[2].memref.size; + + memset(sig, 0, sig_size); + memset(mpmr, 0, mpmr_size); + + res = caam_mp_sign(msg, &msg_size, sig, &sig_size); + + params[1].memref.size = sig_size; + + if (res != TEE_SUCCESS) { + EMSG("Manufacturing Protection signature failed 0x%" PRIx32, + res); + return res; + } + + res = caam_mp_export_mpmr(mpmr, &mpmr_size); + + params[2].memref.size = mpmr_size; + + if (res != TEE_SUCCESS) + EMSG("Manufacturing Protection export MPRM failed 0x%" PRIx32, + res); + + return res; +} + +static TEE_Result +pta_mp_open_session(uint32_t param_types __unused, + TEE_Param params[TEE_NUM_PARAMS] __unused, + void **sess_ctx __unused) +{ + struct ts_session *s = NULL; + + if (IS_ENABLED(CFG_NXP_CAAM_MP_NO_ACCESS_CTRL)) + return TEE_SUCCESS; + + s = ts_get_calling_session(); + if (!s || !is_user_ta_ctx(s->ctx)) + return TEE_ERROR_ACCESS_DENIED; + + return TEE_SUCCESS; +} + +static TEE_Result pta_mp_invoke_cmd(void *sess_ctx __unused, + uint32_t cmd_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd_id) { + case PTA_IMX_MP_CMD_SIGNATURE_MPMR: + return mp_signature(param_types, params); + case PTA_IMX_MP_CMD_GET_PUBLIC_KEY: + return mp_get_public_key(param_types, params); + default: + return TEE_ERROR_BAD_PARAMETERS; + } +} + +pseudo_ta_register(.uuid = PTA_MANUFACT_PROTEC_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .open_session_entry_point = pta_mp_open_session, + .invoke_command_entry_point = pta_mp_invoke_cmd); diff --git a/optee_os/core/pta/imx/ocotp.c b/optee_os/core/pta/imx/ocotp.c new file mode 100644 index 0000000..97dc4df --- /dev/null +++ b/optee_os/core/pta/imx/ocotp.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2021 NXP + */ +#include +#include +#include +#include + +#define OCOTP_PTA_NAME "ocotp.pta" + +static TEE_Result chip_uid(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint8_t val[IMX_UID_SIZE] = { }; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + /* On i.MX platforms, the chip UID is 64 bits long */ + if (params[0].memref.size != sizeof(uint64_t)) + return TEE_ERROR_BAD_PARAMETERS; + + if (tee_otp_get_die_id(val, IMX_UID_SIZE)) + return TEE_ERROR_GENERIC; + + memcpy(params[0].memref.buffer, val, IMX_UID_SIZE); + + return TEE_SUCCESS; +} + +static TEE_Result read_fuse(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result ret = TEE_ERROR_GENERIC; + uint32_t val = 0; + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + params[1].value.a = 0; + params[1].value.b = 0; + + ret = imx_ocotp_read(params[0].value.a, params[0].value.b, &val); + if (!ret) + params[1].value.a = val; + + return ret; +} + +static TEE_Result invokeCommandEntryPoint(void *sess_ctx __unused, + uint32_t cmd_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd_id) { + case PTA_OCOTP_CMD_CHIP_UID: + return chip_uid(param_types, params); + case PTA_OCOTP_CMD_READ_FUSE: + return read_fuse(param_types, params); + default: + return TEE_ERROR_BAD_PARAMETERS; + } +} + +pseudo_ta_register(.uuid = PTA_OCOTP_UUID, .name = OCOTP_PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invokeCommandEntryPoint); diff --git a/optee_os/core/pta/imx/sub.mk b/optee_os/core/pta/imx/sub.mk new file mode 100644 index 0000000..6209e97 --- /dev/null +++ b/optee_os/core/pta/imx/sub.mk @@ -0,0 +1,4 @@ +srcs-$(CFG_IMX_DIGPROG) += digprog.c +srcs-$(CFG_IMX_OCOTP) += ocotp.c +srcs-$(CFG_NXP_CAAM_MP_DRV) += manufacturing_protection.c +srcs-$(CFG_NXP_CAAM_DEK_DRV) += dek_blob.c diff --git a/optee_os/core/pta/k3/otp.c b/optee_os/core/pta/k3/otp.c new file mode 100644 index 0000000..bfe87a3 --- /dev/null +++ b/optee_os/core/pta/k3/otp.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Texas Instruments System Control Interface Driver + * + * Copyright (C) 2023 Texas Instruments Incorporated - https://www.ti.com/ + * Manorit Chawdhry + */ + +#include +#include +#include +#include + +static TEE_Result write_otp_row(uint32_t param_types, TEE_Param params[4]) +{ + TEE_Result ret = TEE_SUCCESS; + const uint32_t exp_param_types = + TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + /* + * Safely get the invocation parameters + */ + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + ret = ti_sci_write_otp_row(params[0].value.a, params[1].value.a, + params[1].value.b); + if (ret) + return ret; + + DMSG("Written the value: 0x%08"PRIx32, params[1].value.a); + + return TEE_SUCCESS; +} + +static TEE_Result read_otp_mmr(uint32_t param_types, TEE_Param params[4]) +{ + TEE_Result ret = TEE_SUCCESS; + const uint32_t exp_param_types = + TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE); + + /* + * Safely get the invocation parameters + */ + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + ret = ti_sci_read_otp_mmr(params[0].value.a, ¶ms[1].value.a); + if (ret) + return ret; + + DMSG("Got the value: 0x%08"PRIx32, params[1].value.a); + + return TEE_SUCCESS; +} + +static TEE_Result lock_otp_row(uint32_t param_types, TEE_Param params[4]) +{ + TEE_Result ret = TEE_SUCCESS; + int hw_write_lock = 0; + int hw_read_lock = 0; + int soft_lock = 0; + const uint32_t exp_param_types = + TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE); + + /* + * Safely get the invocation parameters + */ + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (params[0].value.b & K3_OTP_KEYWRITING_SOFT_LOCK) + soft_lock = 0x5A; + if (params[0].value.b & K3_OTP_KEYWRITING_HW_READ_LOCK) + hw_read_lock = 0x5A; + if (params[0].value.b & K3_OTP_KEYWRITING_HW_WRITE_LOCK) + hw_write_lock = 0x5A; + + DMSG("hw_write_lock: 0x%#x", hw_write_lock); + DMSG("hw_read_lock: 0x%#x", hw_read_lock); + DMSG("soft_lock: 0x%#x", soft_lock); + + ret = ti_sci_lock_otp_row(params[0].value.a, hw_write_lock, + hw_read_lock, soft_lock); + + if (ret) + return ret; + + DMSG("Locked the row: 0x%08"PRIx32, params[1].value.a); + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *session __unused, + uint32_t command, uint32_t param_types, + TEE_Param params[4]) +{ + switch (command) { + case TA_OTP_KEYWRITING_CMD_READ_MMR: + return read_otp_mmr(param_types, params); + case TA_OTP_KEYWRITING_CMD_WRITE_ROW: + return write_otp_row(param_types, params); + case TA_OTP_KEYWRITING_CMD_LOCK_ROW: + return lock_otp_row(param_types, params); + default: + EMSG("Command ID 0x%"PRIx32" is not supported", command); + return TEE_ERROR_NOT_SUPPORTED; + } +} + +pseudo_ta_register(.uuid = PTA_K3_OTP_KEYWRITING_UUID, + .name = PTA_K3_OTP_KEYWRITING_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/k3/sub.mk b/optee_os/core/pta/k3/sub.mk new file mode 100644 index 0000000..3c6d8e2 --- /dev/null +++ b/optee_os/core/pta/k3/sub.mk @@ -0,0 +1 @@ +srcs-$(CFG_K3_OTP_KEYWRITING) += otp.c diff --git a/optee_os/core/pta/rtc.c b/optee_os/core/pta/rtc.c new file mode 100644 index 0000000..8cc6797 --- /dev/null +++ b/optee_os/core/pta/rtc.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, Linaro Limited + * Copyright (c) 2021, EPAM Systems. All rights reserved. + * Copyright (c) 2022, Microchip + * + */ + +#include +#include +#include +#include +#include +#include + +#define PTA_NAME "rtc.pta" + +static void rtc_pta_copy_time_from_optee(struct pta_rtc_time *pta_time, + struct optee_rtc_time *optee_time) +{ + pta_time->tm_sec = optee_time->tm_sec; + pta_time->tm_min = optee_time->tm_min; + pta_time->tm_hour = optee_time->tm_hour; + pta_time->tm_mday = optee_time->tm_mday; + pta_time->tm_mon = optee_time->tm_mon; + pta_time->tm_year = optee_time->tm_year; + pta_time->tm_wday = optee_time->tm_wday; +} + +static TEE_Result rtc_pta_get_time(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct optee_rtc_time time = { }; + struct pta_rtc_time *pta_time = NULL; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + pta_time = params[0].memref.buffer; + if (!pta_time || params[0].memref.size != sizeof(*pta_time)) + return TEE_ERROR_BAD_PARAMETERS; + + res = rtc_get_time(&time); + if (res) + return res; + + rtc_pta_copy_time_from_optee(pta_time, &time); + + return TEE_SUCCESS; +} + +static TEE_Result rtc_pta_set_time(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct optee_rtc_time time = { }; + struct pta_rtc_time *pta_time = NULL; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + pta_time = params[0].memref.buffer; + if (!pta_time || params[0].memref.size != sizeof(*pta_time)) + return TEE_ERROR_BAD_PARAMETERS; + + time.tm_sec = pta_time->tm_sec; + time.tm_min = pta_time->tm_min; + time.tm_hour = pta_time->tm_hour; + time.tm_mday = pta_time->tm_mday; + time.tm_mon = pta_time->tm_mon; + time.tm_year = pta_time->tm_year; + time.tm_wday = pta_time->tm_wday; + + return rtc_set_time(&time); +} + +static TEE_Result rtc_pta_set_offset(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + return rtc_set_offset((int32_t)params[0].value.a); +} + +static TEE_Result rtc_pta_get_offset(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_ERROR_GENERIC; + long offset = 0; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + res = rtc_get_offset(&offset); + if (res) + return res; + + if (offset > INT32_MAX || offset < INT32_MIN) + return TEE_ERROR_OVERFLOW; + + params[0].value.a = (uint32_t)offset; + + return res; +} + +static TEE_Result rtc_pta_get_info(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct pta_rtc_info *info = NULL; + struct optee_rtc_time range_min = { }; + struct optee_rtc_time range_max = { }; + uint64_t features = 0; + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + info = params[0].memref.buffer; + if (!info || params[0].memref.size != sizeof(*info)) + return TEE_ERROR_BAD_PARAMETERS; + + memset(info, 0, sizeof(*info)); + + res = rtc_get_info(&features, &range_min, &range_max); + if (res) + return res; + + info->version = PTA_RTC_INFO_VERSION; + + if (features & RTC_CORRECTION_FEATURE) + info->features |= PTA_RTC_FEATURE_CORRECTION; + + rtc_pta_copy_time_from_optee(&info->range_min, &range_min); + rtc_pta_copy_time_from_optee(&info->range_max, &range_max); + + return TEE_SUCCESS; +} + +static TEE_Result open_session(uint32_t ptypes __unused, + TEE_Param par[TEE_NUM_PARAMS] __unused, + void **session __unused) +{ + struct ts_session *ts = ts_get_current_session(); + struct tee_ta_session *ta_session = to_ta_session(ts); + + /* Only REE kernel is allowed to access RTC */ + if (ta_session->clnt_id.login != TEE_LOGIN_REE_KERNEL) + return TEE_ERROR_ACCESS_DENIED; + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *session __unused, + uint32_t cmd, uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd) { + case PTA_CMD_RTC_GET_INFO: + return rtc_pta_get_info(ptypes, params); + case PTA_CMD_RTC_GET_TIME: + return rtc_pta_get_time(ptypes, params); + case PTA_CMD_RTC_SET_TIME: + return rtc_pta_set_time(ptypes, params); + case PTA_CMD_RTC_GET_OFFSET: + return rtc_pta_get_offset(ptypes, params); + case PTA_CMD_RTC_SET_OFFSET: + return rtc_pta_set_offset(ptypes, params); + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_RTC_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_CONCURRENT | + TA_FLAG_DEVICE_ENUM, + .open_session_entry_point = open_session, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/scmi.c b/optee_os/core/pta/scmi.c new file mode 100644 index 0000000..7686435 --- /dev/null +++ b/optee_os/core/pta/scmi.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2021, Linaro Limited + * Copyright (c) 2019-2021, STMicroelectronics + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static uint32_t supported_caps(void) +{ + uint32_t caps = 0; + + if (IS_ENABLED2(_CFG_SCMI_PTA_SMT_HEADER)) + caps |= PTA_SCMI_CAPS_SMT_HEADER; + + if (IS_ENABLED2(_CFG_SCMI_PTA_MSG_HEADER)) + caps |= PTA_SCMI_CAPS_MSG_HEADER; + + return caps; +} + +static TEE_Result cmd_capabilities(uint32_t ptypes, + TEE_Param param[TEE_NUM_PARAMS]) +{ + const uint32_t exp_ptypes = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (ptypes != exp_ptypes) + return TEE_ERROR_BAD_PARAMETERS; + + param[0].value.a = supported_caps(); + param[0].value.b = 0; + + return TEE_SUCCESS; +} + +static TEE_Result cmd_process_smt_channel(uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_ptypes = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + unsigned int channel_id = params[0].value.a; + + if (ptypes != exp_ptypes) + return TEE_ERROR_BAD_PARAMETERS; + + if (IS_ENABLED(CFG_SCMI_MSG_SMT)) { + struct scmi_msg_channel *channel = NULL; + + channel = plat_scmi_get_channel(channel_id); + if (!channel) + return TEE_ERROR_BAD_PARAMETERS; + + scmi_smt_threaded_entry(channel_id); + + return TEE_SUCCESS; + } + + if (IS_ENABLED(CFG_SCMI_SCPFW)) + return scmi_server_smt_process_thread(channel_id); + + return TEE_ERROR_NOT_SUPPORTED; +} + +static TEE_Result cmd_process_smt_message(uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_ptypes = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + unsigned int channel_id = params[0].value.a; + TEE_Param *param1 = params + 1; + + if (ptypes != exp_ptypes) + return TEE_ERROR_BAD_PARAMETERS; + + if (IS_ENABLED(CFG_SCMI_MSG_SMT)) { + struct scmi_msg_channel *channel = NULL; + + if (param1->memref.size < SMT_BUF_SLOT_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + channel = plat_scmi_get_channel(channel_id); + if (!channel) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * Caller provides the buffer, we bind channel to that buffer. + * Once message is processed, unbind the buffer since it is + * valid only for the current invocation. + */ + scmi_smt_set_shared_buffer(channel, param1->memref.buffer); + scmi_smt_threaded_entry(channel_id); + scmi_smt_set_shared_buffer(channel, NULL); + + return TEE_SUCCESS; + } + + return TEE_ERROR_NOT_SUPPORTED; +} + +static TEE_Result cmd_process_msg_channel(uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_ERROR_GENERIC; + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + unsigned int channel_id = params[0].value.a; + void *in_buf = params[1].memref.buffer; + size_t in_size = params[1].memref.size; + void *out_buf = params[2].memref.buffer; + size_t out_size = params[2].memref.size; + + if (ptypes != exp_pt || !in_buf || !out_buf) + return TEE_ERROR_BAD_PARAMETERS; + + if (IS_ENABLED(CFG_SCMI_MSG_DRIVERS)) { + struct scmi_msg_channel *channel = NULL; + + if (!IS_ENABLED(CFG_SCMI_MSG_SHM_MSG)) + return TEE_ERROR_NOT_SUPPORTED; + + channel = plat_scmi_get_channel(channel_id); + if (!channel) + return TEE_ERROR_BAD_PARAMETERS; + + res = scmi_msg_threaded_entry(channel_id, in_buf, in_size, + out_buf, &out_size); + if (!res) + params[2].memref.size = out_size; + + return res; + } + + if (IS_ENABLED(CFG_SCMI_SCPFW)) { + if (!in_buf || !out_buf) + return TEE_ERROR_BAD_PARAMETERS; + + res = scmi_server_msg_process_thread(channel_id, in_buf, + in_size, out_buf, + &out_size); + if (!res) { + params[2].memref.size = (uint32_t)out_size; + IMSG("scmi optee shm: out %zu", out_size); + } + + return res; + } + + return TEE_ERROR_NOT_SUPPORTED; +} + +static TEE_Result cmd_get_channel_handle(uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_ptypes = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + unsigned int channel_id = params[0].value.a; + unsigned int caps = params[0].value.b; + + if (ptypes != exp_ptypes || caps & ~PTA_SCMI_CAPS_MASK) + return TEE_ERROR_BAD_PARAMETERS; + + if (!(caps & supported_caps())) + return TEE_ERROR_NOT_SUPPORTED; + + if (IS_ENABLED(CFG_SCMI_MSG_DRIVERS)) { + struct scmi_msg_channel *channel = NULL; + + channel = plat_scmi_get_channel(channel_id); + if (!channel) + return TEE_ERROR_BAD_PARAMETERS; + + channel->threaded = true; + params[0].value.a = scmi_smt_channel_handle(channel_id); + + return TEE_SUCCESS; + } + + if (IS_ENABLED(CFG_SCMI_SCPFW)) { + /* Only check the channel ID is known from SCP-firwmare */ + return scmi_server_get_channel(channel_id, NULL); + } + + return TEE_ERROR_NOT_SUPPORTED; +} + +static TEE_Result pta_scmi_open_session(uint32_t ptypes __unused, + TEE_Param par[TEE_NUM_PARAMS] __unused, + void **session __unused) +{ + struct ts_session *ts = ts_get_current_session(); + struct tee_ta_session *ta_session = to_ta_session(ts); + + /* Only REE kernel is allowed to access SCMI resources */ + if (ta_session->clnt_id.login != TEE_LOGIN_REE_KERNEL) { + DMSG("Expecting TEE_LOGIN_REE_KERNEL"); + return TEE_ERROR_ACCESS_DENIED; + } + + if (IS_ENABLED(CFG_SCMI_MSG_DRIVERS) || IS_ENABLED(CFG_SCMI_SCPFW)) + return TEE_SUCCESS; + + return TEE_ERROR_NOT_SUPPORTED; +} + +static TEE_Result pta_scmi_invoke_command(void *session __unused, uint32_t cmd, + uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + FMSG("SCMI command %#"PRIx32" ptypes %#"PRIx32, cmd, ptypes); + + switch (cmd) { + case PTA_SCMI_CMD_CAPABILITIES: + return cmd_capabilities(ptypes, params); + case PTA_SCMI_CMD_PROCESS_SMT_CHANNEL: + return cmd_process_smt_channel(ptypes, params); + case PTA_SCMI_CMD_PROCESS_SMT_CHANNEL_MESSAGE: + return cmd_process_smt_message(ptypes, params); + case PTA_SCMI_CMD_PROCESS_MSG_CHANNEL: + return cmd_process_msg_channel(ptypes, params); + case PTA_SCMI_CMD_GET_CHANNEL_HANDLE: + return cmd_get_channel_handle(ptypes, params); + default: + return TEE_ERROR_NOT_SUPPORTED; + } +} + +pseudo_ta_register(.uuid = PTA_SCMI_UUID, .name = PTA_SCMI_NAME, + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_CONCURRENT | + TA_FLAG_DEVICE_ENUM, + .open_session_entry_point = pta_scmi_open_session, + .invoke_command_entry_point = pta_scmi_invoke_command); diff --git a/optee_os/core/pta/scp03.c b/optee_os/core/pta/scp03.c new file mode 100644 index 0000000..b5b24ac --- /dev/null +++ b/optee_os/core/pta/scp03.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (C) Foundries Ltd. 2020 - All Rights Reserved + * Author: Jorge Ramirez + */ + +#include +#include +#include + +#define PTA_NAME "pta.scp03" + +static TEE_Result invoke_command(void *session_context __unused, + uint32_t command_id, uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + bool rotate_keys = false; + + FMSG("command entry point for pseudo-TA \"%s\"", PTA_NAME); + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + switch (command_id) { + case PTA_CMD_ENABLE_SCP03: + if (params[0].value.a == PTA_SCP03_SESSION_ROTATE_KEYS) + rotate_keys = true; + + return crypto_se_enable_scp03(rotate_keys); + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_SCP03_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/secstor_ta_mgmt.c b/optee_os/core/pta/secstor_ta_mgmt.c new file mode 100644 index 0000000..162de43 --- /dev/null +++ b/optee_os/core/pta/secstor_ta_mgmt.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static TEE_Result check_install_conflict(const struct shdr_bootstrap_ta *bs_ta) +{ + TEE_Result res; + struct tee_tadb_ta_read *ta; + TEE_UUID uuid; + const struct tee_tadb_property *prop; + + tee_uuid_from_octets(&uuid, bs_ta->uuid); + res = tee_tadb_ta_open(&uuid, &ta); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + return TEE_SUCCESS; + if (res) + return res; + + prop = tee_tadb_ta_get_property(ta); + if (prop->version > bs_ta->ta_version) + res = TEE_ERROR_ACCESS_CONFLICT; + + tee_tadb_ta_close(ta); + return res; +} + +static TEE_Result install_ta(struct shdr *shdr, const uint8_t *nw, + size_t nw_size) +{ + TEE_Result res; + struct tee_tadb_ta_write *ta; + void *hash_ctx = NULL; + size_t offs; + const size_t buf_size = 2 * 4096; + void *buf; + struct tee_tadb_property property; + struct shdr_bootstrap_ta bs_ta; + + if (shdr->img_type != SHDR_BOOTSTRAP_TA) + return TEE_ERROR_SECURITY; + + if (nw_size < (sizeof(struct shdr_bootstrap_ta) + SHDR_GET_SIZE(shdr))) + return TEE_ERROR_SECURITY; + + if (shdr->hash_size > buf_size) + return TEE_ERROR_SECURITY; + + buf = malloc(buf_size); + if (!buf) + return TEE_ERROR_OUT_OF_MEMORY; + + /* + * Initialize a hash context and run the algorithm over the signed + * header (less the final file hash and its signature of course) + */ + res = crypto_hash_alloc_ctx(&hash_ctx, + TEE_DIGEST_HASH_TO_ALGO(shdr->algo)); + if (res) + goto err; + res = crypto_hash_init(hash_ctx); + if (res) + goto err_free_hash_ctx; + res = crypto_hash_update(hash_ctx, (uint8_t *)shdr, sizeof(*shdr)); + if (res) + goto err_free_hash_ctx; + + offs = SHDR_GET_SIZE(shdr); + memcpy(&bs_ta, nw + offs, sizeof(bs_ta)); + + /* Check that we're not downgrading a TA */ + res = check_install_conflict(&bs_ta); + if (res) + goto err_free_hash_ctx; + + res = crypto_hash_update(hash_ctx, (uint8_t *)&bs_ta, sizeof(bs_ta)); + if (res) + goto err_free_hash_ctx; + offs += sizeof(bs_ta); + + memset(&property, 0, sizeof(property)); + COMPILE_TIME_ASSERT(sizeof(property.uuid) == sizeof(bs_ta.uuid)); + tee_uuid_from_octets(&property.uuid, bs_ta.uuid); + property.version = bs_ta.ta_version; + property.custom_size = 0; + property.bin_size = nw_size - offs; + DMSG("Installing %pUl", (void *)&property.uuid); + + res = tee_tadb_ta_create(&property, &ta); + if (res) + goto err_free_hash_ctx; + + while (offs < nw_size) { + size_t l = MIN(buf_size, nw_size - offs); + + memcpy(buf, nw + offs, l); + res = crypto_hash_update(hash_ctx, buf, l); + if (res) + goto err_ta_finalize; + res = tee_tadb_ta_write(ta, buf, l); + if (res) + goto err_ta_finalize; + offs += l; + } + + res = crypto_hash_final(hash_ctx, buf, shdr->hash_size); + if (res) + goto err_ta_finalize; + if (consttime_memcmp(buf, SHDR_GET_HASH(shdr), shdr->hash_size)) { + res = TEE_ERROR_SECURITY; + goto err_ta_finalize; + } + + crypto_hash_free_ctx(hash_ctx); + free(buf); + return tee_tadb_ta_close_and_commit(ta); + +err_ta_finalize: + tee_tadb_ta_close_and_delete(ta); +err_free_hash_ctx: + crypto_hash_free_ctx(hash_ctx); +err: + free(buf); + return res; +} + +static TEE_Result bootstrap(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res; + struct shdr *shdr; + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (param_types != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + shdr = shdr_alloc_and_copy(0, params->memref.buffer, + params->memref.size); + if (!shdr) + return TEE_ERROR_SECURITY; + + res = shdr_verify_signature(shdr); + if (res) + goto out; + + res = install_ta(shdr, params->memref.buffer, params->memref.size); +out: + shdr_free(shdr); + return res; +} + +static TEE_Result invoke_command(void *sess_ctx __unused, uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd_id) { + case PTA_SECSTOR_TA_MGMT_BOOTSTRAP: + return bootstrap(param_types, params); + default: + break; + } + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_SECSTOR_TA_MGMT_UUID, .name = "secstor_ta_mgmt", + /* + * TA_FLAG_SINGLE_INSTANCE and TA_FLAG_MULTI_SESSION are + * current part of PTA_DEFAULT_FLAGS, but as this TA + * depends on those two flags we add them explicitly + * too. + */ + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_SINGLE_INSTANCE | + TA_FLAG_MULTI_SESSION, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/stats.c b/optee_os/core/pta/stats.c new file mode 100644 index 0000000..933d8e1 --- /dev/null +++ b/optee_os/core/pta/stats.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TA_NAME "stats.ta" + +#define STATS_UUID \ + { 0xd96a5b40, 0xe2c7, 0xb1af, \ + { 0x87, 0x94, 0x10, 0x02, 0xa5, 0xd5, 0xc6, 0x1b } } + +#define STATS_CMD_PAGER_STATS 0 +#define STATS_CMD_ALLOC_STATS 1 +#define STATS_CMD_MEMLEAK_STATS 2 +/* + * UTEE_ENTRY_FUNC_DUMP_MEMSTATS + * [out] memref[0] Array of context information of loaded TAs + * + * Each cell of the TA information array contains: + * TEE_UUID TA UUID + * uint32_t Non zero if TA panicked, 0 otherwise + * uint32_t Number of sessions opened by the TA + * uint32_t Byte size currently allocated in TA heap + * uint32_t Max bytes allocated since last stats reset + * uint32_t TA heap pool byte size + * uint32_t Number of failed allocation requests + * uint32_t Biggest byte size which allocation failed + * uint32_t Biggest byte size which allocation succeeded + */ +#define STATS_CMD_TA_STATS 3 + +/* + * STATS_CMD_GET_TIME - Get both REE time and TEE time + * + * [out] value[0].a REE time as seen by OP-TEE in seconds + * [out] value[0].b REE time as seen by OP-TEE, milliseconds part + * [out] value[1].a TEE system time in seconds + * [out] value[1].b TEE system time, milliseconds part + */ +#define STATS_CMD_GET_TIME 4 + +#define STATS_NB_POOLS 4 + +static TEE_Result get_alloc_stats(uint32_t type, TEE_Param p[TEE_NUM_PARAMS]) +{ + struct malloc_stats *stats; + uint32_t size_to_retrieve; + uint32_t pool_id; + uint32_t i; + + /* + * p[0].value.a = pool id (from 0 to n) + * - 0 means all the pools to be retrieved + * - 1..n means pool id + * p[0].value.b = 0 if no reset of the stats + * p[1].memref.buffer = output buffer to struct malloc_stats + */ + if (TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE) != type) { + return TEE_ERROR_BAD_PARAMETERS; + } + + pool_id = p[0].value.a; + if (pool_id > STATS_NB_POOLS) + return TEE_ERROR_BAD_PARAMETERS; + + size_to_retrieve = sizeof(struct malloc_stats); + if (!pool_id) + size_to_retrieve *= STATS_NB_POOLS; + + if (p[1].memref.size < size_to_retrieve) { + p[1].memref.size = size_to_retrieve; + return TEE_ERROR_SHORT_BUFFER; + } + p[1].memref.size = size_to_retrieve; + stats = p[1].memref.buffer; + + for (i = 1; i <= STATS_NB_POOLS; i++) { + if ((pool_id) && (i != pool_id)) + continue; + + switch (i) { + case 1: + malloc_get_stats(stats); + strlcpy(stats->desc, "Heap", sizeof(stats->desc)); + if (p[0].value.b) + malloc_reset_stats(); + break; + + case 2: + EMSG("public DDR not managed by secure side anymore"); + break; + + case 3: + tee_mm_get_pool_stats(&tee_mm_sec_ddr, stats, + !!p[0].value.b); + strlcpy(stats->desc, "Secure DDR", sizeof(stats->desc)); + break; + +#ifdef CFG_NS_VIRTUALIZATION + case 4: + nex_malloc_get_stats(stats); + strlcpy(stats->desc, "KHeap", sizeof(stats->desc)); + if (p[0].value.b) + nex_malloc_reset_stats(); + break; +#endif + default: + EMSG("Wrong pool id"); + break; + } + + stats++; + } + + return TEE_SUCCESS; +} + +static TEE_Result get_pager_stats(uint32_t type, TEE_Param p[TEE_NUM_PARAMS]) +{ + struct tee_pager_stats stats; + + if (TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE) != type) { + EMSG("expect 3 output values as argument"); + return TEE_ERROR_BAD_PARAMETERS; + } + + tee_pager_get_stats(&stats); + p[0].value.a = stats.npages; + p[0].value.b = stats.npages_all; + p[1].value.a = stats.ro_hits; + p[1].value.b = stats.rw_hits; + p[2].value.a = stats.hidden_hits; + p[2].value.b = stats.zi_released; + + return TEE_SUCCESS; +} + +static TEE_Result get_memleak_stats(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS] __maybe_unused) +{ + + if (TEE_PARAM_TYPES(TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE) != type) + return TEE_ERROR_BAD_PARAMETERS; + + mdbg_check(1); + + return TEE_SUCCESS; +} + +static TEE_Result get_user_ta_stats(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS] __maybe_unused) +{ + uint32_t res = TEE_SUCCESS; + + if (TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE) != type) + return TEE_ERROR_BAD_PARAMETERS; + +#if defined(CFG_TA_STATS) + res = tee_ta_instance_stats(p[0].memref.buffer, + &p[0].memref.size); + if (res != TEE_SUCCESS) + DMSG("tee_ta_dump_stats return: 0x%"PRIx32, res); +#else + res = TEE_ERROR_NOT_SUPPORTED; +#endif + return res; +} + +static TEE_Result get_system_time(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS]) +{ + TEE_Time ree_time = { }; + TEE_Time tee_time = { }; + + if (TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE) != type) + return TEE_ERROR_BAD_PARAMETERS; + + tee_time_get_sys_time(&tee_time); + tee_time_get_ree_time(&ree_time); + + p[0].value.a = ree_time.seconds; + p[0].value.b = ree_time.millis; + p[1].value.a = tee_time.seconds; + p[1].value.b = tee_time.millis; + + return TEE_SUCCESS; +} + +/* + * Trusted Application Entry Points + */ + +static TEE_Result invoke_command(void *psess __unused, + uint32_t cmd, uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd) { + case STATS_CMD_PAGER_STATS: + return get_pager_stats(ptypes, params); + case STATS_CMD_ALLOC_STATS: + return get_alloc_stats(ptypes, params); + case STATS_CMD_MEMLEAK_STATS: + return get_memleak_stats(ptypes, params); + case STATS_CMD_TA_STATS: + return get_user_ta_stats(ptypes, params); + case STATS_CMD_GET_TIME: + return get_system_time(ptypes, params); + default: + break; + } + return TEE_ERROR_BAD_PARAMETERS; +} + +pseudo_ta_register(.uuid = STATS_UUID, .name = TA_NAME, + .flags = PTA_DEFAULT_FLAGS, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/stm32mp/bsec_pta.c b/optee_os/core/pta/stm32mp/bsec_pta.c new file mode 100644 index 0000000..107a935 --- /dev/null +++ b/optee_os/core/pta/stm32mp/bsec_pta.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2022, STMicroelectronics - All Rights Reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static_assert(IS_ENABLED(CFG_STM32_BSEC)); + +#define PTA_NAME "bsec.pta" + +static TEE_Result bsec_check_access(uint32_t otp_id) +{ + if (!stm32_bsec_nsec_can_access_otp(otp_id)) + return TEE_ERROR_ACCESS_DENIED; + + return TEE_SUCCESS; +} + +static TEE_Result bsec_read_mem(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + uint32_t *buf = (uint32_t *)params[1].memref.buffer; + uint32_t otp_start = 0; + size_t otp_length = 0; + uint32_t otp_id = 0; + TEE_Result res = TEE_ERROR_GENERIC; + size_t size = params[1].memref.size; + bool locked = false; + unsigned int otp_base_offset = params[0].value.a; + unsigned int bsec_command = params[0].value.b; + + if (pt != exp_pt || !buf || !size || + !IS_ALIGNED_WITH_TYPE(params[1].memref.buffer, uint32_t)) + return TEE_ERROR_BAD_PARAMETERS; + + /* Check 32bits alignment */ + if (otp_base_offset % BSEC_BYTES_PER_WORD || + size % BSEC_BYTES_PER_WORD) + return TEE_ERROR_BAD_PARAMETERS; + + otp_start = otp_base_offset / BSEC_BYTES_PER_WORD; + otp_length = size / BSEC_BYTES_PER_WORD; + + for (otp_id = otp_start; otp_id < otp_start + otp_length; + otp_id++, buf++) { + res = bsec_check_access(otp_id); + switch (bsec_command) { + case PTA_BSEC_SHADOW_ACCESS: + if (res) { + /* Force 0 when access is not allowed */ + *buf = 0x0; + continue; + } + /* Read shadow register */ + res = stm32_bsec_read_otp(buf, otp_id); + FMSG("Read shadow %"PRIu32" val: %#"PRIx32, otp_id, + *buf); + break; + case PTA_BSEC_FUSE_ACCESS: + /* Check access */ + if (res) + goto out; + /* Read fuse value */ + res = stm32_bsec_shadow_read_otp(buf, otp_id); + FMSG("Read fuse %"PRIu32" val: %#"PRIx32, otp_id, *buf); + break; + case PTA_BSEC_LOCKS_ACCESS: + if (res) { + /* Force error when access is not allowed */ + *buf = PTA_BSEC_LOCK_ERROR; + continue; + } + *buf = 0; + /* Read lock value */ + res = stm32_bsec_read_permanent_lock(otp_id, &locked); + if (res) + goto out; + + if (locked) + *buf |= PTA_BSEC_LOCK_PERM; + + res = stm32_bsec_read_sr_lock(otp_id, &locked); + if (res) + goto out; + + if (locked) + *buf |= PTA_BSEC_LOCK_SHADOW_R; + + res = stm32_bsec_read_sw_lock(otp_id, &locked); + if (res) + goto out; + + if (locked) + *buf |= PTA_BSEC_LOCK_SHADOW_W; + + res = stm32_bsec_read_sp_lock(otp_id, &locked); + if (res) + goto out; + + if (locked) + *buf |= PTA_BSEC_LOCK_SHADOW_P; + + FMSG("Read lock %"PRIu32" val: %#"PRIx32, otp_id, *buf); + break; + default: + FMSG("%"PRIu32" invalid operation: %"PRIu32, otp_id, + bsec_command); + res = TEE_ERROR_BAD_PARAMETERS; + } + + if (res) + goto out; + } + + FMSG("Buffer orig %p, size %zu", buf, size); + + res = TEE_SUCCESS; +out: + return res; +} + +static TEE_Result bsec_write_mem(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + uint32_t *buf = (uint32_t *)params[1].memref.buffer; + size_t size = params[1].memref.size; + uint32_t otp_start = 0; + size_t otp_length = 0; + uint32_t otp_id = 0; + TEE_Result res = TEE_ERROR_GENERIC; + unsigned int otp_base_offset = params[0].value.a; + unsigned int bsec_command = params[0].value.b; + + if (pt != exp_pt || !buf || !size || + !IS_ALIGNED_WITH_TYPE(params[1].memref.buffer, uint32_t)) + return TEE_ERROR_BAD_PARAMETERS; + + /* Check 32bits alignment */ + if (otp_base_offset % BSEC_BYTES_PER_WORD || + size % BSEC_BYTES_PER_WORD) + return TEE_ERROR_BAD_PARAMETERS; + + otp_start = otp_base_offset / BSEC_BYTES_PER_WORD; + otp_length = size / BSEC_BYTES_PER_WORD; + + /* Initial check to ensure that all BSEC words are available */ + for (otp_id = otp_start; otp_id < otp_start + otp_length; otp_id++) { + res = bsec_check_access(otp_id); + if (res) + return res; + } + + for (otp_id = otp_start; otp_id < otp_start + otp_length; + otp_id++, buf++) { + switch (bsec_command) { + case PTA_BSEC_SHADOW_ACCESS: + /* Write shadow register */ + FMSG("Write shadow %"PRIx32" : %"PRIx32, + otp_id, *buf); + res = stm32_bsec_write_otp(*buf, otp_id); + break; + + case PTA_BSEC_FUSE_ACCESS: + /* Write fuse value */ + FMSG("Write fuse %"PRIx32" : %08"PRIx32, + otp_id, *buf); + res = stm32_bsec_program_otp(*buf, otp_id); + break; + + case PTA_BSEC_LOCKS_ACCESS: + if (*buf & PTA_BSEC_LOCK_PERM) { + FMSG("Perm lock access OTP: %u", otp_id); + res = stm32_bsec_permanent_lock_otp(otp_id); + if (res) + break; + } + + if (*buf & PTA_BSEC_LOCK_SHADOW_R) { + FMSG("Shadow read lock"); + res = stm32_bsec_set_sr_lock(otp_id); + if (res) + break; + } + + if (*buf & PTA_BSEC_LOCK_SHADOW_W) { + FMSG("Shadow write lock detected"); + res = stm32_bsec_set_sw_lock(otp_id); + if (res) + break; + } + + if (*buf & PTA_BSEC_LOCK_SHADOW_P) { + FMSG("Shadow programming lock detected"); + res = stm32_bsec_set_sp_lock(otp_id); + } + + break; + + default: + FMSG("OTP %"PRIx32" invalid operation: %"PRIx32, + otp_id, bsec_command); + res = TEE_ERROR_BAD_PARAMETERS; + } + + if (res) + return res; + } + + return TEE_SUCCESS; +} + +static TEE_Result bsec_pta_state(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Result res = TEE_ERROR_GENERIC; + enum stm32_bsec_sec_state state = BSEC_STATE_INVALID; + enum stm32_bsec_pta_sec_state pta_state = PTA_BSEC_STATE_INVALID; + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + res = stm32_bsec_get_state(&state); + if (res) + return res; + + switch (state) { + case BSEC_STATE_SEC_CLOSED: + pta_state = PTA_BSEC_STATE_SEC_CLOSE; + break; + case BSEC_STATE_SEC_OPEN: + pta_state = PTA_BSEC_STATE_SEC_OPEN; + break; + default: + pta_state = PTA_BSEC_STATE_INVALID; + break; + } + + params[0].value.a = pta_state; + + return TEE_SUCCESS; +} + +static TEE_Result bsec_pta_invoke_command(void *pSessionContext __unused, + uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + FMSG(PTA_NAME" command %#"PRIx32" ptypes %#"PRIx32, + cmd_id, param_types); + + switch (cmd_id) { + case PTA_BSEC_CMD_READ_OTP: + return bsec_read_mem(param_types, params); + case PTA_BSEC_CMD_WRITE_OTP: + return bsec_write_mem(param_types, params); + case PTA_BSEC_CMD_GET_STATE: + return bsec_pta_state(param_types, params); + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static TEE_Result pta_bsec_open_session(uint32_t ptypes __unused, + TEE_Param par[TEE_NUM_PARAMS] __unused, + void **session __unused) +{ + uint32_t login = to_ta_session(ts_get_current_session())->clnt_id.login; + + if (login == TEE_LOGIN_REE_KERNEL) + return TEE_SUCCESS; + + return TEE_ERROR_ACCESS_DENIED; +} + +pseudo_ta_register(.uuid = PTA_BSEC_UUID, .name = PTA_NAME, + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_CONCURRENT | + TA_FLAG_DEVICE_ENUM, + .open_session_entry_point = pta_bsec_open_session, + .invoke_command_entry_point = bsec_pta_invoke_command); diff --git a/optee_os/core/pta/stm32mp/sub.mk b/optee_os/core/pta/stm32mp/sub.mk new file mode 100644 index 0000000..ed16c5f --- /dev/null +++ b/optee_os/core/pta/stm32mp/sub.mk @@ -0,0 +1 @@ +srcs-$(CFG_STM32_BSEC_PTA) += bsec_pta.c diff --git a/optee_os/core/pta/sub.mk b/optee_os/core/pta/sub.mk new file mode 100644 index 0000000..41212d3 --- /dev/null +++ b/optee_os/core/pta/sub.mk @@ -0,0 +1,21 @@ +subdirs-$(CFG_TEE_CORE_EMBED_INTERNAL_TESTS) += tests + +srcs-$(CFG_ATTESTATION_PTA) += attestation.c +srcs-$(CFG_TEE_BENCHMARK) += benchmark.c +srcs-$(CFG_DEVICE_ENUM_PTA) += device.c +srcs-$(CFG_TA_GPROF_SUPPORT) += gprof.c +ifeq ($(CFG_WITH_USER_TA),y) +srcs-$(CFG_SECSTOR_TA_MGMT_PTA) += secstor_ta_mgmt.c +endif +srcs-$(CFG_WITH_STATS) += stats.c +srcs-$(CFG_SYSTEM_PTA) += system.c +srcs-$(CFG_SCP03_PTA) += scp03.c +srcs-$(CFG_APDU_PTA) += apdu.c +srcs-$(CFG_SCMI_PTA) += scmi.c +srcs-$(CFG_HWRNG_PTA) += hwrng.c +srcs-$(CFG_RTC_PTA) += rtc.c + +subdirs-y += bcm +subdirs-y += stm32mp +subdirs-y += imx +subdirs-y += k3 diff --git a/optee_os/core/pta/system.c b/optee_os/core/pta/system.c new file mode 100644 index 0000000..07a2d58 --- /dev/null +++ b/optee_os/core/pta/system.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2022 Linaro Limited + * Copyright (c) 2020, Arm Limited. + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned int system_pnum; + +static TEE_Result system_rng_reseed(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t entropy_sz = 0; + uint8_t *entropy_input = NULL; + void *seed_bbuf = NULL; + TEE_Result res = TEE_SUCCESS; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + entropy_input = params[0].memref.buffer; + entropy_sz = params[0].memref.size; + + if (!entropy_sz || !entropy_input) + return TEE_ERROR_BAD_PARAMETERS; + + res = bb_memdup_user(entropy_input, entropy_sz, &seed_bbuf); + if (res) + return res; + + crypto_rng_add_event(CRYPTO_RNG_SRC_NONSECURE, &system_pnum, + seed_bbuf, entropy_sz); + + bb_free(seed_bbuf, entropy_sz); + + return TEE_SUCCESS; +} + +static TEE_Result system_derive_ta_unique_key(struct user_mode_ctx *uctx, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t data_len = sizeof(TEE_UUID); + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *data = NULL; + uint32_t access_flags = 0; + void *subkey_bbuf = NULL; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (params[0].memref.size > TA_DERIVED_EXTRA_DATA_MAX_SIZE || + params[1].memref.size < TA_DERIVED_KEY_MIN_SIZE || + params[1].memref.size > TA_DERIVED_KEY_MAX_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * The derived key shall not end up in non-secure memory by + * mistake. + * + * Note that we're allowing shared memory as long as it's + * secure. This is needed because a TA always uses shared memory + * when communicating with another TA. + */ + access_flags = TEE_MEMORY_ACCESS_WRITE | TEE_MEMORY_ACCESS_ANY_OWNER | + TEE_MEMORY_ACCESS_SECURE; + res = vm_check_access_rights(uctx, access_flags, + (uaddr_t)params[1].memref.buffer, + params[1].memref.size); + if (res != TEE_SUCCESS) + return TEE_ERROR_SECURITY; + + /* Take extra data into account. */ + if (ADD_OVERFLOW(data_len, params[0].memref.size, &data_len)) + return TEE_ERROR_SECURITY; + + data = bb_alloc(data_len); + if (!data) + return TEE_ERROR_OUT_OF_MEMORY; + + memcpy(data, &uctx->ts_ctx->uuid, sizeof(TEE_UUID)); + + /* Append the user provided data */ + res = copy_from_user(data + sizeof(TEE_UUID), params[0].memref.buffer, + params[0].memref.size); + if (res) + goto out; + + subkey_bbuf = bb_alloc(params[1].memref.size); + if (!subkey_bbuf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = huk_subkey_derive(HUK_SUBKEY_UNIQUE_TA, data, data_len, + subkey_bbuf, params[1].memref.size); + if (res) + goto out; + + res = copy_to_user(params[1].memref.buffer, subkey_bbuf, + params[1].memref.size); + +out: + bb_free_wipe(subkey_bbuf, params[1].memref.size); + bb_free_wipe(data, data_len); + return res; +} + +static TEE_Result system_map_zi(struct user_mode_ctx *uctx, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INOUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE); + uint32_t prot = TEE_MATTR_URW | TEE_MATTR_PRW; + TEE_Result res = TEE_ERROR_GENERIC; + struct mobj *mobj = NULL; + uint32_t pad_begin = 0; + uint32_t vm_flags = 0; + struct fobj *f = NULL; + uint32_t pad_end = 0; + size_t num_bytes = 0; + vaddr_t va = 0; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + if (params[0].value.b & ~PTA_SYSTEM_MAP_FLAG_SHAREABLE) + return TEE_ERROR_BAD_PARAMETERS; + + if (params[0].value.b & PTA_SYSTEM_MAP_FLAG_SHAREABLE) + vm_flags |= VM_FLAG_SHAREABLE; + + num_bytes = params[0].value.a; + va = reg_pair_to_64(params[1].value.a, params[1].value.b); + pad_begin = params[2].value.a; + pad_end = params[2].value.b; + + f = fobj_ta_mem_alloc(ROUNDUP_DIV(num_bytes, SMALL_PAGE_SIZE)); + if (!f) + return TEE_ERROR_OUT_OF_MEMORY; + mobj = mobj_with_fobj_alloc(f, NULL, TEE_MATTR_MEM_TYPE_TAGGED); + fobj_put(f); + if (!mobj) + return TEE_ERROR_OUT_OF_MEMORY; + res = vm_map_pad(uctx, &va, num_bytes, prot, vm_flags, + mobj, 0, pad_begin, pad_end, 0); + mobj_put(mobj); + if (!res) + reg_pair_from_64(va, ¶ms[1].value.a, ¶ms[1].value.b); + + return res; +} + +static TEE_Result system_unmap(struct user_mode_ctx *uctx, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Result res = TEE_SUCCESS; + uint32_t vm_flags = 0; + vaddr_t end_va = 0; + vaddr_t va = 0; + size_t sz = 0; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (params[0].value.b) + return TEE_ERROR_BAD_PARAMETERS; + + va = reg_pair_to_64(params[1].value.a, params[1].value.b); + sz = ROUNDUP(params[0].value.a, SMALL_PAGE_SIZE); + + /* + * The vm_get_flags() and vm_unmap() are supposed to detect or + * handle overflow directly or indirectly. However, this function + * an API function so an extra guard here is in order. If nothing + * else to make it easier to review the code. + */ + if (ADD_OVERFLOW(va, sz, &end_va)) + return TEE_ERROR_BAD_PARAMETERS; + + res = vm_get_flags(uctx, va, sz, &vm_flags); + if (res) + return res; + if (vm_flags & VM_FLAG_PERMANENT) + return TEE_ERROR_ACCESS_DENIED; + + return vm_unmap(uctx, va, sz); +} + +static TEE_Result system_dlopen(struct user_mode_ctx *uctx, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Result res = TEE_ERROR_GENERIC; + struct ts_session *s = NULL; + TEE_UUID uuid = { }; + uint32_t flags = 0; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (!params[0].memref.buffer || params[0].memref.size != sizeof(uuid)) + return TEE_ERROR_BAD_PARAMETERS; + + res = copy_from_user(&uuid, params[0].memref.buffer, sizeof(uuid)); + if (res) + return res; + + flags = params[1].value.a; + + s = ts_pop_current_session(); + res = ldelf_dlopen(uctx, &uuid, flags); + ts_push_current_session(s); + + return res; +} + +static TEE_Result system_dlsym(struct user_mode_ctx *uctx, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Result res = TEE_ERROR_GENERIC; + struct ts_session *s = NULL; + char *sym = NULL; + TEE_UUID uuid = { }; + size_t symlen = 0; + vaddr_t va = 0; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (!params[0].memref.buffer || params[0].memref.size != sizeof(uuid)) + return TEE_ERROR_BAD_PARAMETERS; + + res = copy_from_user(&uuid, params[0].memref.buffer, sizeof(uuid)); + if (res) + return res; + + if (!params[1].memref.buffer) + return TEE_ERROR_BAD_PARAMETERS; + res = bb_strndup_user(params[1].memref.buffer, params[1].memref.size, + &sym, &symlen); + if (res) + return res; + + s = ts_pop_current_session(); + res = ldelf_dlsym(uctx, &uuid, sym, symlen, &va); + ts_push_current_session(s); + + if (!res) + reg_pair_from_64(va, ¶ms[2].value.a, ¶ms[2].value.b); + + return res; +} + +static TEE_Result system_get_tpm_event_log(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + size_t size = 0; + TEE_Result res = TEE_SUCCESS; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + size = params[0].memref.size; + res = tpm_get_event_log(params[0].memref.buffer, &size); + params[0].memref.size = size; + + return res; +} + +static TEE_Result system_supp_plugin_invoke(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_VALUE_OUTPUT); + TEE_Result res = TEE_ERROR_GENERIC; + size_t outlen = 0; + TEE_UUID uuid = { }; + + if (exp_pt != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + if (!params[0].memref.buffer || params[0].memref.size != sizeof(uuid)) + return TEE_ERROR_BAD_PARAMETERS; + + res = copy_from_user(&uuid, params[0].memref.buffer, sizeof(uuid)); + if (res) + return res; + + res = tee_invoke_supp_plugin_rpc(&uuid, + params[1].value.a, /* cmd */ + params[1].value.b, /* sub_cmd */ + NULL, + params[2].memref.buffer, /* data */ + params[2].memref.size, /* in len */ + &outlen); + params[3].value.a = (uint32_t)outlen; + + return res; +} + +static TEE_Result open_session(uint32_t param_types __unused, + TEE_Param params[TEE_NUM_PARAMS] __unused, + void **sess_ctx __unused) +{ + struct ts_session *s = NULL; + + /* Check that we're called from a user TA */ + s = ts_get_calling_session(); + if (!s) + return TEE_ERROR_ACCESS_DENIED; + if (!is_user_ta_ctx(s->ctx)) + return TEE_ERROR_ACCESS_DENIED; + + return TEE_SUCCESS; +} + +static TEE_Result invoke_command(void *sess_ctx __unused, uint32_t cmd_id, + uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct ts_session *s = ts_get_calling_session(); + struct user_mode_ctx *uctx = to_user_mode_ctx(s->ctx); + + switch (cmd_id) { + case PTA_SYSTEM_ADD_RNG_ENTROPY: + return system_rng_reseed(param_types, params); + case PTA_SYSTEM_DERIVE_TA_UNIQUE_KEY: + return system_derive_ta_unique_key(uctx, param_types, params); + case PTA_SYSTEM_MAP_ZI: + return system_map_zi(uctx, param_types, params); + case PTA_SYSTEM_UNMAP: + return system_unmap(uctx, param_types, params); + case PTA_SYSTEM_DLOPEN: + return system_dlopen(uctx, param_types, params); + case PTA_SYSTEM_DLSYM: + return system_dlsym(uctx, param_types, params); + case PTA_SYSTEM_GET_TPM_EVENT_LOG: + return system_get_tpm_event_log(param_types, params); + case PTA_SYSTEM_SUPP_PLUGIN_INVOKE: + return system_supp_plugin_invoke(param_types, params); + default: + break; + } + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_SYSTEM_UUID, .name = "system.pta", + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_CONCURRENT, + .open_session_entry_point = open_session, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/tests/aes_perf.c b/optee_os/core/pta/tests/aes_perf.c new file mode 100644 index 0000000..84a48e8 --- /dev/null +++ b/optee_os/core/pta/tests/aes_perf.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "misc.h" + +/* + * These keys and iv are copied from optee_test/ta/aes_perf/ta_aes_perf.c, + * not because their actual values are important, rather that there's no + * reason to use different values. + */ + +static const uint8_t aes_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F +}; + +static const uint8_t aes_key2[] = { + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F +}; + +static uint8_t aes_iv[] = { + 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, + 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF +}; + +static void free_ctx(void **ctx, uint32_t algo) +{ + if (algo == TEE_ALG_AES_GCM) + crypto_authenc_free_ctx(*ctx); + else + crypto_cipher_free_ctx(*ctx); + + *ctx = NULL; +} + +static TEE_Result init_ctx(void **ctx, uint32_t algo, TEE_OperationMode mode, + size_t key_size_bits, size_t payload_len) +{ + TEE_Result res = TEE_SUCCESS; + const uint8_t *key2 = NULL; + const uint8_t *iv = NULL; + size_t key2_len = 0; + size_t key_len = 0; + size_t iv_len = 0; + + if (key_size_bits % 8) + return TEE_ERROR_BAD_PARAMETERS; + key_len = key_size_bits / 8; + if (key_len > sizeof(aes_key)) + return TEE_ERROR_BAD_PARAMETERS; + + /* Alloc ctx */ + switch (algo) { + case TEE_ALG_AES_XTS: + key2_len = key_len; + key2 = aes_key2; + fallthrough; + case TEE_ALG_AES_ECB_NOPAD: + case TEE_ALG_AES_CBC_NOPAD: + case TEE_ALG_AES_CTR: + res = crypto_cipher_alloc_ctx(ctx, algo); + break; + case TEE_ALG_AES_GCM: + res = crypto_authenc_alloc_ctx(ctx, algo); + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + if (res) + return res; + + /* Init ctx */ + switch (algo) { + case TEE_ALG_AES_CBC_NOPAD: + case TEE_ALG_AES_CTR: + case TEE_ALG_AES_XTS: + iv = aes_iv; + iv_len = sizeof(aes_iv); + fallthrough; + case TEE_ALG_AES_ECB_NOPAD: + res = crypto_cipher_init(*ctx, mode, aes_key, key_len, key2, + key2_len, iv, iv_len); + break; + case TEE_ALG_AES_GCM: + res = crypto_authenc_init(*ctx, mode, aes_key, key_len, aes_iv, + sizeof(aes_iv), TEE_AES_BLOCK_SIZE, + 0, payload_len); + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + if (res) + free_ctx(ctx, algo); + + return res; +} + +static TEE_Result update_ae(void *ctx, TEE_OperationMode mode, + const void *src, size_t len, void *dst) +{ + size_t dlen = len; + + return crypto_authenc_update_payload(ctx, mode, src, len, dst, &dlen); +} + +static TEE_Result update_cipher(void *ctx, TEE_OperationMode mode, + const void *src, size_t len, void *dst) +{ + return crypto_cipher_update(ctx, mode, false, src, len, dst); +} + +static TEE_Result do_update(void *ctx, uint32_t algo, TEE_OperationMode mode, + unsigned int rep_count, unsigned int unit_size, + const uint8_t *in, size_t sz, uint8_t *out) +{ + TEE_Result (*update_func)(void *ctx, TEE_OperationMode mode, + const void *src, size_t len, + void *dst) = NULL; + TEE_Result res = TEE_SUCCESS; + unsigned int n = 0; + unsigned int m = 0; + + if (algo == TEE_ALG_AES_GCM) + update_func = update_ae; + else + update_func = update_cipher; + + for (n = 0; n < rep_count; n++) { + for (m = 0; m < sz / unit_size; m++) { + res = update_func(ctx, mode, in + m * unit_size, + unit_size, out + m * unit_size); + if (res) + return res; + } + if (sz % unit_size) + res = update_func(ctx, mode, in + m * unit_size, + sz % unit_size, out + m * unit_size); + } + + return res; +} + +TEE_Result core_aes_perf_tests(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_MEMREF_INOUT); + TEE_Result res = TEE_SUCCESS; + TEE_OperationMode mode = 0; + unsigned int rep_count = 0; + unsigned int unit_size = 0; + size_t key_size_bits = 0; + uint32_t algo = 0; + void *ctx = NULL; + + if (param_types != exp_param_types) + return TEE_ERROR_BAD_PARAMETERS; + + switch (params[0].value.b) { + case PTA_INVOKE_TESTS_AES_ECB: + algo = TEE_ALG_AES_ECB_NOPAD; + break; + case PTA_INVOKE_TESTS_AES_CBC: + algo = TEE_ALG_AES_CBC_NOPAD; + break; + case PTA_INVOKE_TESTS_AES_CTR: + algo = TEE_ALG_AES_CTR; + break; + case PTA_INVOKE_TESTS_AES_XTS: + algo = TEE_ALG_AES_XTS; + break; + case PTA_INVOKE_TESTS_AES_GCM: + algo = TEE_ALG_AES_GCM; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + if (params[0].value.a >> 16) + mode = TEE_MODE_DECRYPT; + else + mode = TEE_MODE_ENCRYPT; + + key_size_bits = params[0].value.a & 0xffff; + + rep_count = params[1].value.a; + unit_size = params[1].value.b; + + if (params[2].memref.size > params[3].memref.size) + return TEE_ERROR_BAD_PARAMETERS; + + res = init_ctx(&ctx, algo, mode, key_size_bits, params[2].memref.size); + if (res) + return res; + + res = do_update(ctx, algo, mode, rep_count, unit_size, + params[2].memref.buffer, params[2].memref.size, + params[3].memref.buffer); + + free_ctx(&ctx, algo); + return res; +} diff --git a/optee_os/core/pta/tests/dt_driver_test.c b/optee_os/core/pta/tests/dt_driver_test.c new file mode 100644 index 0000000..6a5ac85 --- /dev/null +++ b/optee_os/core/pta/tests/dt_driver_test.c @@ -0,0 +1,768 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + * + * Tests introduce dummy test drivers and assiciated devices defined in + * dt_driver_test.dtsi file with device resource dependencies. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DT_TEST_MSG(...) FMSG("(dt-driver-test) " __VA_ARGS__) + +/* Test state IDs */ +enum dt_test_sid { DEFAULT = 0, IN_PROGRESS, SUCCESS, FAILED }; + +/* + * DT tests state to be reported from PTA_INVOKE_TESTS_CMD_DT_TEST_STATUS + * possibly printed to console. A test can be skipped (DEFAULT) or be + * successful (SUCCESS) orthewise it has failed (IN_PROGRESS, FAILED). + */ +struct dt_test_state { + enum dt_test_sid probe_deferral; + enum dt_test_sid probe_clocks; + enum dt_test_sid probe_gpios; + enum dt_test_sid probe_resets; + enum dt_test_sid crypto_dependencies; +}; + +/* + * References allocated from heap to be free once test completed + * dt_test_alloc(), dt_test_free(), dt_test_free_all() + */ +struct dt_test_free_ref { + void *p; + SLIST_ENTRY(dt_test_free_ref) link; +}; + +static struct dt_test_state dt_test_state; + +static const char __maybe_unused * const dt_test_str_sid[] = { + [DEFAULT] = "not run", + [IN_PROGRESS] = "in-progress", + [SUCCESS] = "successful", + [FAILED] = "failed", +}; + +/* Reference allocations during test for release_init_resource initcall level */ +static SLIST_HEAD(dt_test_free_refs, dt_test_free_ref) dt_test_free_list = + SLIST_HEAD_INITIALIZER(dt_test_free_list); + +static void __maybe_unused *dt_test_alloc(size_t size) +{ + struct dt_test_free_ref *ref = NULL; + + ref = calloc(1, sizeof(*ref) + size); + if (!ref) + return NULL; + + ref->p = ref + 1; + SLIST_INSERT_HEAD(&dt_test_free_list, ref, link); + + return ref->p; +} + +static void __maybe_unused dt_test_free(void *p) +{ + struct dt_test_free_ref *ref = NULL; + struct dt_test_free_ref *t_ref = NULL; + + if (!p) + return; + + SLIST_FOREACH_SAFE(ref, &dt_test_free_list, link, t_ref) { + if (ref->p == p) { + SLIST_REMOVE(&dt_test_free_list, ref, + dt_test_free_ref, link); + free(ref); + return; + } + } + + panic(); +} + +static void dt_test_free_all(void) +{ + while (!SLIST_EMPTY(&dt_test_free_list)) { + struct dt_test_free_ref *ref = SLIST_FIRST(&dt_test_free_list); + + SLIST_REMOVE(&dt_test_free_list, ref, dt_test_free_ref, link); + free(ref); + } +} + +static TEE_Result dt_test_release(void) +{ + dt_test_free_all(); + + DT_TEST_MSG("Probe deferral: %s", + dt_test_str_sid[dt_test_state.probe_deferral]); + DT_TEST_MSG("Clocks probe: %s", + dt_test_str_sid[dt_test_state.probe_clocks]); + DT_TEST_MSG("GPIO ctrl probe: %s", + dt_test_str_sid[dt_test_state.probe_gpios]); + DT_TEST_MSG("Reset ctrl probe: %s", + dt_test_str_sid[dt_test_state.probe_resets]); + DT_TEST_MSG("Crypto deps.: %s", + dt_test_str_sid[dt_test_state.crypto_dependencies]); + + return dt_driver_test_status(); +} + +release_init_resource(dt_test_release); + +TEE_Result dt_driver_test_status(void) +{ + TEE_Result res = TEE_SUCCESS; + + if (dt_test_state.probe_deferral != SUCCESS) { + EMSG("Probe deferral test failed"); + res = TEE_ERROR_GENERIC; + } + if (IS_ENABLED(CFG_DRIVERS_CLK) && + dt_test_state.probe_clocks != SUCCESS) { + EMSG("Clocks probing test failed"); + res = TEE_ERROR_GENERIC; + } + if (IS_ENABLED(CFG_DRIVERS_GPIOS) && + dt_test_state.probe_gpios != SUCCESS) { + EMSG("GPIO controllers probing test failed"); + res = TEE_ERROR_GENERIC; + } + if (IS_ENABLED(CFG_DRIVERS_RSTCTRL) && + dt_test_state.probe_resets != SUCCESS) { + EMSG("Reset controllers probing test failed"); + res = TEE_ERROR_GENERIC; + } + if (dt_test_state.crypto_dependencies != SUCCESS) { + EMSG("Probe deferral on crypto dependencies test failed"); + res = TEE_ERROR_GENERIC; + } + + return res; +} + +static TEE_Result probe_test_clocks(const void *fdt, int node) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct clk *clk0 = NULL; + struct clk *clk1 = NULL; + struct clk *clk = NULL; + + DT_TEST_MSG("Probe clocks"); + dt_test_state.probe_clocks = IN_PROGRESS; + + res = clk_dt_get_by_index(fdt, node, 0, &clk0); + if (res) + goto err; + + res = clk_dt_get_by_index(fdt, node, 1, &clk1); + if (res) + goto err; + + DT_TEST_MSG("Check valid clock references"); + + if (clk_enable(clk0)) { + DT_TEST_MSG("Can't enable %s", clk_get_name(clk0)); + res = TEE_ERROR_GENERIC; + goto err; + } + clk_disable(clk0); + + res = clk_dt_get_by_name(fdt, node, "clk0", &clk); + if (res || clk != clk0) { + DT_TEST_MSG("Unexpected clock reference"); + res = TEE_ERROR_GENERIC; + goto err; + } + + res = clk_dt_get_by_name(fdt, node, "clk1", &clk); + if (res || clk != clk1) { + DT_TEST_MSG("Unexpected clock reference"); + res = TEE_ERROR_GENERIC; + goto err; + } + + DT_TEST_MSG("Bad clock reference"); + + res = clk_dt_get_by_index(fdt, node, 3, &clk); + if (!res) { + DT_TEST_MSG("Unexpected clock found on invalid index"); + res = TEE_ERROR_GENERIC; + goto err; + } + + res = clk_dt_get_by_name(fdt, node, "clk2", &clk); + if (!res) { + DT_TEST_MSG("Unexpected clock found on invalid name"); + res = TEE_ERROR_GENERIC; + goto err; + } + + dt_test_state.probe_clocks = SUCCESS; + return TEE_SUCCESS; + +err: + if (res != TEE_ERROR_DEFER_DRIVER_INIT) + dt_test_state.probe_clocks = FAILED; + + return res; +} + +static TEE_Result probe_test_resets(const void *fdt, int node) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct rstctrl *rstctrl0 = NULL; + struct rstctrl *rstctrl1 = NULL; + struct rstctrl *rstctrl = NULL; + + DT_TEST_MSG("Probe reset controllers"); + dt_test_state.probe_resets = IN_PROGRESS; + + res = rstctrl_dt_get_by_index(fdt, node, 0, &rstctrl0); + if (res) + goto err; + + DT_TEST_MSG("Check valid reset controller"); + + if (rstctrl_assert(rstctrl0)) { + EMSG("Can't assert rstctrl %s", rstctrl_name(rstctrl0)); + res = TEE_ERROR_GENERIC; + goto err; + } + + res = rstctrl_dt_get_by_name(fdt, node, "rst0", &rstctrl); + if (res) + goto err; + + if (rstctrl != rstctrl0) { + EMSG("Unexpected reset controller reference"); + res = TEE_ERROR_GENERIC; + goto err; + } + + res = rstctrl_dt_get_by_name(fdt, node, "rst1", &rstctrl1); + if (res) + goto err; + + if (!rstctrl1 || rstctrl1 == rstctrl0) { + EMSG("Unexpected reset controller reference"); + res = TEE_ERROR_GENERIC; + goto err; + } + + dt_test_state.probe_resets = SUCCESS; + return TEE_SUCCESS; + +err: + if (res != TEE_ERROR_DEFER_DRIVER_INIT) + dt_test_state.probe_resets = FAILED; + + return res; +} + +static TEE_Result probe_test_gpios(const void *fdt, int node) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct gpio *gpio = NULL; + + DT_TEST_MSG("Probe GPIO controllers"); + dt_test_state.probe_gpios = IN_PROGRESS; + + res = gpio_dt_get_by_index(fdt, node, 0, "test", &gpio); + if (res) + goto err; + + if (gpio_get_direction(gpio) != GPIO_DIR_IN) { + EMSG("Unexpected gpio_get_direction() return value"); + res = TEE_ERROR_GENERIC; + goto err; + } + + /* GPIO is declared as ACTIVE_LOW in device-tree */ + if (gpio_get_value(gpio) != GPIO_LEVEL_LOW) { + EMSG("Unexpected gpio_get_value() return value"); + res = TEE_ERROR_GENERIC; + goto err; + } + + res = gpio_dt_get_by_index(fdt, node, 1, "test", &gpio); + if (res) + goto err; + + if (gpio_get_direction(gpio) != GPIO_DIR_IN) { + EMSG("Unexpected gpio_get_direction() return value"); + res = TEE_ERROR_GENERIC; + goto err; + } + + if (gpio_get_value(gpio) != GPIO_LEVEL_HIGH) { + EMSG("Unexpected gpio_get_value() return value"); + res = TEE_ERROR_GENERIC; + goto err; + } + + dt_test_state.probe_gpios = SUCCESS; + return TEE_SUCCESS; + +err: + if (res != TEE_ERROR_DEFER_DRIVER_INIT) + dt_test_state.probe_gpios = FAILED; + + return res; +} + +/* + * Consumer test driver: instance probed from the compatible + * node parsed in the DT. It consumes emulated resource obtained + * from DT references. Probe shall succeed only once all resources + * are found. + */ +static TEE_Result dt_test_consumer_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + if (IS_ENABLED(CFG_DRIVERS_CLK)) { + res = probe_test_clocks(fdt, node); + if (res) + goto err_probe; + } + + if (IS_ENABLED(CFG_DRIVERS_RSTCTRL)) { + res = probe_test_resets(fdt, node); + if (res) + goto err_probe; + } + + if (IS_ENABLED(CFG_DRIVERS_GPIO)) { + res = probe_test_gpios(fdt, node); + if (res) + goto err_probe; + } + + if (dt_test_state.probe_deferral != IN_PROGRESS) { + dt_test_state.probe_deferral = FAILED; + return TEE_ERROR_GENERIC; + } + + dt_test_state.probe_deferral = SUCCESS; + + return TEE_SUCCESS; + +err_probe: + assert(res); + + if (res == TEE_ERROR_DEFER_DRIVER_INIT && + dt_test_state.probe_deferral == DEFAULT) { + /* We expect at least a probe deferral */ + dt_test_state.probe_deferral = IN_PROGRESS; + } + + return res; +} + +static const struct dt_device_match dt_test_consumer_match_table[] = { + { .compatible = "linaro,dt-test-consumer", }, + { } +}; + +DEFINE_DT_DRIVER(dt_test_consumer_driver) = { + .name = "dt-test-consumer", + .match_table = dt_test_consumer_match_table, + .probe = dt_test_consumer_probe, +}; + +static TEE_Result dt_test_crypt_consumer_probe(const void *fdt __unused, + int node __unused, + const void *compat_data __unused) +{ + TEE_Result res = dt_driver_get_crypto(); + uint8_t __maybe_unused byte = 0; + + if (res == TEE_ERROR_DEFER_DRIVER_INIT && + dt_test_state.crypto_dependencies == DEFAULT) { + /* We expect to be deferred */ + dt_test_state.crypto_dependencies = IN_PROGRESS; + } + + if (res) + return res; + + if (dt_test_state.crypto_dependencies == DEFAULT) { + EMSG("Test expects at least a driver probe deferral"); + dt_test_state.crypto_dependencies = FAILED; + return TEE_ERROR_GENERIC; + } + + if (crypto_rng_read(&byte, sizeof(byte))) { + dt_test_state.crypto_dependencies = FAILED; + return TEE_ERROR_GENERIC; + } + + dt_test_state.crypto_dependencies = SUCCESS; + return TEE_SUCCESS; +} + +static const struct dt_device_match dt_test_crypt_consumer_match_table[] = { + { .compatible = "linaro,dt-test-crypt-consumer", }, + { } +}; + +DEFINE_DT_DRIVER(dt_test_consumer_driver) = { + .name = "dt-test-crypt-consumer", + .match_table = dt_test_crypt_consumer_match_table, + .probe = dt_test_crypt_consumer_probe, +}; + +#ifdef CFG_DRIVERS_CLK +#define DT_TEST_CLK_COUNT 2 + +#define DT_TEST_CLK0_BINDING_ID 3 +#define DT_TEST_CLK1_BINDING_ID 7 + +static const char *dt_test_clk_name[DT_TEST_CLK_COUNT] = { + "dt_test-clk3", + "dt_test-clk7", +}; + +/* Emulating a clock does not require operators */ +static const struct clk_ops dt_test_clock_provider_ops; + +static TEE_Result dt_test_get_clk(struct dt_pargs *args, void *data, + struct clk **out_device) +{ + struct clk *clk_ref = data; + struct clk *clk = NULL; + + if (args->args_count != 1) + return TEE_ERROR_BAD_PARAMETERS; + + switch (args->args[0]) { + case DT_TEST_CLK0_BINDING_ID: + clk = clk_ref; + break; + case DT_TEST_CLK1_BINDING_ID: + clk = clk_ref + 1; + break; + default: + EMSG("Unexpected binding ID %"PRIu32, args->args[0]); + return TEE_ERROR_BAD_PARAMETERS; + } + + DT_TEST_MSG("Providing clock %s", clk_get_name(clk)); + + *out_device = clk; + return TEE_SUCCESS; +} + +static TEE_Result dt_test_clock_provider_probe(const void *fdt, int node, + const void *compat_data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct clk *clk = NULL; + size_t n = 0; + + DT_TEST_MSG("Register clocks"); + + clk = dt_test_alloc(DT_TEST_CLK_COUNT * sizeof(*clk)); + if (!clk) + return TEE_ERROR_OUT_OF_MEMORY; + + for (n = 0; n < DT_TEST_CLK_COUNT; n++) { + clk[n].ops = &dt_test_clock_provider_ops; + clk[n].name = dt_test_clk_name[n]; + + res = clk_register(clk + n); + if (res) + goto err; + } + + res = clk_dt_register_clk_provider(fdt, node, dt_test_get_clk, clk); + if (res) + goto err; + + return TEE_SUCCESS; + +err: + dt_test_free(clk); + return res; +} + +CLK_DT_DECLARE(dt_test_clock_provider, "linaro,dt-test-provider", + dt_test_clock_provider_probe); +#endif /* CFG_DRIVERS_CLK */ + +#ifdef CFG_DRIVERS_RSTCTRL +#define DT_TEST_RSTCTRL_COUNT 2 + +#define DT_TEST_RSTCTRL0_BINDING_ID 5 +#define DT_TEST_RSTCTRL1_BINDING_ID 35 + +struct dt_test_rstctrl { + unsigned int dt_binding; + struct rstctrl rstctrl; +}; + +static struct dt_test_rstctrl *to_test_rstctrl(struct rstctrl *rstctrl) +{ + return container_of(rstctrl, struct dt_test_rstctrl, rstctrl); +} + +static TEE_Result dt_test_rstctrl_stub(struct rstctrl *rstctrl __maybe_unused, + unsigned int to_us __unused) +{ + struct dt_test_rstctrl *dev = to_test_rstctrl(rstctrl); + + switch (dev->dt_binding) { + case DT_TEST_RSTCTRL0_BINDING_ID: + case DT_TEST_RSTCTRL1_BINDING_ID: + return TEE_SUCCESS; + default: + EMSG("Unexpected rstctrl reference"); + return TEE_ERROR_GENERIC; + } +} + +static const char *dt_test_rstctrl_name(struct rstctrl *rstctrl __maybe_unused) +{ + static const char *rstctrl_name[DT_TEST_RSTCTRL_COUNT] = { + "dt_test-rstctrl5", + "dt_test-rstctrl35", + }; + struct dt_test_rstctrl *dev = to_test_rstctrl(rstctrl); + + switch (dev->dt_binding) { + case DT_TEST_RSTCTRL0_BINDING_ID: + return rstctrl_name[0]; + case DT_TEST_RSTCTRL1_BINDING_ID: + return rstctrl_name[1]; + default: + EMSG("Unexpected rstctrl reference"); + return NULL; + } +} + +const struct rstctrl_ops dt_test_rstctrl_ops = { + .assert_level = dt_test_rstctrl_stub, + .deassert_level = dt_test_rstctrl_stub, + .get_name = dt_test_rstctrl_name, +}; + +static TEE_Result dt_test_get_rstctrl(struct dt_pargs *args, void *data, + struct rstctrl **out_device) +{ + struct dt_test_rstctrl *ref = data; + struct rstctrl *rstctrl = NULL; + + if (args->args_count != 1) + return TEE_ERROR_BAD_PARAMETERS; + + switch (args->args[0]) { + case DT_TEST_RSTCTRL0_BINDING_ID: + rstctrl = &ref[0].rstctrl; + break; + case DT_TEST_RSTCTRL1_BINDING_ID: + rstctrl = &ref[1].rstctrl; + break; + default: + EMSG("Unexpected binding ID %"PRIu32, args->args[0]); + return TEE_ERROR_BAD_PARAMETERS; + } + + DT_TEST_MSG("Providing reset controller %s", rstctrl_name(rstctrl)); + + *out_device = rstctrl; + + return TEE_SUCCESS; +} + +static TEE_Result dt_test_rstctrl_provider_probe(const void *fdt, int offs, + const void *data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct dt_test_rstctrl *devices = NULL; + + DT_TEST_MSG("Register reset controllers"); + + assert(rstctrl_ops_is_valid(&dt_test_rstctrl_ops)); + + devices = dt_test_alloc(DT_TEST_RSTCTRL_COUNT * sizeof(*devices)); + if (!devices) + return TEE_ERROR_OUT_OF_MEMORY; + + devices[0].rstctrl.ops = &dt_test_rstctrl_ops; + devices[0].dt_binding = DT_TEST_RSTCTRL0_BINDING_ID; + + devices[1].rstctrl.ops = &dt_test_rstctrl_ops; + devices[1].dt_binding = DT_TEST_RSTCTRL1_BINDING_ID; + + res = rstctrl_register_provider(fdt, offs, dt_test_get_rstctrl, + devices); + if (res) { + dt_test_free(devices); + return res; + } + + return TEE_SUCCESS; +} + +RSTCTRL_DT_DECLARE(dt_test_rstctrl_provider, "linaro,dt-test-provider", + dt_test_rstctrl_provider_probe); +#endif /* CFG_DRIVERS_RSTCTRL */ + +#ifdef CFG_DRIVERS_GPIO +#define DT_TEST_GPIO_COUNT 2 + +#define DT_TEST_GPIO0_PIN 1 +#define DT_TEST_GPIO0_FLAGS GPIO_ACTIVE_LOW +#define DT_TEST_GPIO1_PIN 2 +#define DT_TEST_GPIO1_FLAGS GPIO_PULL_UP + +struct dt_test_gpio { + unsigned int pin; + unsigned int flags; + struct gpio_chip gpio_chip; +}; + +static struct dt_test_gpio *to_test_gpio(struct gpio_chip *chip) +{ + return container_of(chip, struct dt_test_gpio, gpio_chip); +} + +static enum gpio_dir dt_test_gpio_get_direction(struct gpio_chip *chip, + unsigned int gpio_pin) +{ + struct dt_test_gpio *dtg = to_test_gpio(chip); + + if (dtg->pin != gpio_pin) + panic("Invalid GPIO number"); + + return GPIO_DIR_IN; +} + +static void dt_test_gpio_set_direction(struct gpio_chip *chip, + unsigned int gpio_pin, + enum gpio_dir direction __unused) +{ + struct dt_test_gpio *dtg = to_test_gpio(chip); + + if (dtg->pin != gpio_pin) + panic("Invalid GPIO number"); +} + +static enum gpio_level dt_test_gpio_get_value(struct gpio_chip *chip, + unsigned int gpio_pin) +{ + struct dt_test_gpio *dtg = to_test_gpio(chip); + + if (dtg->pin != gpio_pin) + panic("Invalid GPIO number"); + + return GPIO_LEVEL_HIGH; +} + +static void dt_test_gpio_set_value(struct gpio_chip *chip, + unsigned int gpio_pin, + enum gpio_level value __unused) +{ + struct dt_test_gpio *dtg = to_test_gpio(chip); + + if (dtg->pin != gpio_pin) + panic("Invalid GPIO number"); +} + +static const struct gpio_ops dt_test_gpio_ops = { + .get_direction = dt_test_gpio_get_direction, + .set_direction = dt_test_gpio_set_direction, + .get_value = dt_test_gpio_get_value, + .set_value = dt_test_gpio_set_value, +}; + +static TEE_Result dt_test_gpio_get_dt(struct dt_pargs *args, void *data, + struct gpio **out_device) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct gpio *gpio = NULL; + struct dt_test_gpio *gpios = (struct dt_test_gpio *)data; + + res = gpio_dt_alloc_pin(args, &gpio); + if (res) + return res; + + switch (gpio->pin) { + case DT_TEST_GPIO0_PIN: + gpio->chip = &gpios[0].gpio_chip; + if (gpio->dt_flags != gpios[0].flags) { + EMSG("Unexpected dt_flags %#"PRIx32, gpio->dt_flags); + free(gpio); + return TEE_ERROR_GENERIC; + } + break; + case DT_TEST_GPIO1_PIN: + gpio->chip = &gpios[1].gpio_chip; + if (gpio->dt_flags != gpios[1].flags) { + EMSG("Unexpected dt_flags %#"PRIx32, gpio->dt_flags); + free(gpio); + return TEE_ERROR_GENERIC; + } + break; + default: + EMSG("Unexpected pin ID %u", gpio->pin); + free(gpio); + return TEE_ERROR_BAD_PARAMETERS; + }; + + *out_device = gpio; + + return TEE_SUCCESS; +} + +static TEE_Result dt_test_gpio_provider_probe(const void *fdt, int offs, + const void *data __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct dt_test_gpio *gpios = NULL; + + DT_TEST_MSG("Register GPIO controllers"); + + assert(gpio_ops_is_valid(&dt_test_gpio_ops)); + + gpios = dt_test_alloc(DT_TEST_GPIO_COUNT * sizeof(*gpios)); + if (!gpios) + return TEE_ERROR_OUT_OF_MEMORY; + + gpios[0].gpio_chip.ops = &dt_test_gpio_ops; + gpios[0].pin = DT_TEST_GPIO0_PIN; + gpios[0].flags = DT_TEST_GPIO0_FLAGS; + + gpios[1].gpio_chip.ops = &dt_test_gpio_ops; + gpios[1].pin = DT_TEST_GPIO1_PIN; + gpios[1].flags = DT_TEST_GPIO1_FLAGS; + + res = gpio_register_provider(fdt, offs, dt_test_gpio_get_dt, gpios); + if (res) { + dt_test_free(gpios); + return res; + } + + return TEE_SUCCESS; +} + +GPIO_DT_DECLARE(dt_test_gpio_provider, "linaro,dt-test-provider", + dt_test_gpio_provider_probe); +#endif /* CFG_DRIVERS_GPIO */ diff --git a/optee_os/core/pta/tests/fs_htree.c b/optee_os/core/pta/tests/fs_htree.c new file mode 100644 index 0000000..2a50045 --- /dev/null +++ b/optee_os/core/pta/tests/fs_htree.c @@ -0,0 +1,627 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "misc.h" + +/* + * The smallest blocks size that can hold two struct + * tee_fs_htree_node_image or two struct tee_fs_htree_image. + */ +#define TEST_BLOCK_SIZE 144 + +struct test_aux { + uint8_t *data; + size_t data_len; + size_t data_alloced; + uint8_t *block; +}; + +static TEE_Result test_get_offs_size(enum tee_fs_htree_type type, size_t idx, + uint8_t vers, size_t *offs, size_t *size) +{ + const size_t node_size = sizeof(struct tee_fs_htree_node_image); + const size_t block_nodes = TEST_BLOCK_SIZE / (node_size * 2); + size_t pbn = 0; + size_t bidx = 0; + + COMPILE_TIME_ASSERT(TEST_BLOCK_SIZE > + sizeof(struct tee_fs_htree_node_image) * 2); + COMPILE_TIME_ASSERT(TEST_BLOCK_SIZE > + sizeof(struct tee_fs_htree_image) * 2); + + assert(vers == 0 || vers == 1); + + /* + * File layout + * + * phys block 0: + * tee_fs_htree_image vers 0 @ offs = 0 + * tee_fs_htree_image vers 1 @ offs = sizeof(tee_fs_htree_image) + * + * phys block 1: + * tee_fs_htree_node_image 0 vers 0 @ offs = 0 + * tee_fs_htree_node_image 0 vers 1 @ offs = node_size + * + * phys block 2: + * data block 0 vers 0 + * + * phys block 3: + * tee_fs_htree_node_image 1 vers 0 @ offs = 0 + * tee_fs_htree_node_image 1 vers 1 @ offs = node_size + * + * phys block 4: + * data block 0 vers 1 + * + * ... + */ + + switch (type) { + case TEE_FS_HTREE_TYPE_HEAD: + *offs = sizeof(struct tee_fs_htree_image) * vers; + *size = sizeof(struct tee_fs_htree_image); + return TEE_SUCCESS; + case TEE_FS_HTREE_TYPE_NODE: + pbn = 1 + ((idx / block_nodes) * block_nodes * 2); + *offs = pbn * TEST_BLOCK_SIZE + + 2 * node_size * (idx % block_nodes) + + node_size * vers; + *size = node_size; + return TEE_SUCCESS; + case TEE_FS_HTREE_TYPE_BLOCK: + bidx = 2 * idx + vers; + pbn = 2 + bidx + bidx / (block_nodes * 2 - 1); + *offs = pbn * TEST_BLOCK_SIZE; + *size = TEST_BLOCK_SIZE; + return TEE_SUCCESS; + default: + return TEE_ERROR_GENERIC; + } +} + +static TEE_Result test_read_init(void *aux, struct tee_fs_rpc_operation *op, + enum tee_fs_htree_type type, size_t idx, + uint8_t vers, void **data) +{ + TEE_Result res = TEE_SUCCESS; + struct test_aux *a = aux; + size_t offs = 0; + size_t sz = 0; + + res = test_get_offs_size(type, idx, vers, &offs, &sz); + if (res == TEE_SUCCESS) { + memset(op, 0, sizeof(*op)); + op->params[0].u.value.a = (vaddr_t)aux; + op->params[0].u.value.b = offs; + op->params[0].u.value.c = sz; + *data = a->block; + } + + return res; +} + +static void *uint_to_ptr(uintptr_t p) +{ + return (void *)p; +} + +static TEE_Result test_read_final(struct tee_fs_rpc_operation *op, + size_t *bytes) +{ + struct test_aux *a = uint_to_ptr(op->params[0].u.value.a); + size_t offs = op->params[0].u.value.b; + size_t sz = op->params[0].u.value.c; + + if (offs + sz <= a->data_len) + *bytes = sz; + else if (offs <= a->data_len) + *bytes = a->data_len - offs; + else + *bytes = 0; + + memcpy(a->block, a->data + offs, *bytes); + return TEE_SUCCESS; +} + +static TEE_Result test_write_init(void *aux, struct tee_fs_rpc_operation *op, + enum tee_fs_htree_type type, size_t idx, + uint8_t vers, void **data) +{ + return test_read_init(aux, op, type, idx, vers, data); +} + +static TEE_Result test_write_final(struct tee_fs_rpc_operation *op) +{ + struct test_aux *a = uint_to_ptr(op->params[0].u.value.a); + size_t offs = op->params[0].u.value.b; + size_t sz = op->params[0].u.value.c; + size_t end = offs + sz; + + if (end > a->data_alloced) { + EMSG("out of bounds"); + return TEE_ERROR_GENERIC; + } + + memcpy(a->data + offs, a->block, sz); + if (end > a->data_len) + a->data_len = end; + return TEE_SUCCESS; + +} + +static const struct tee_fs_htree_storage test_htree_ops = { + .block_size = TEST_BLOCK_SIZE, + .rpc_read_init = test_read_init, + .rpc_read_final = test_read_final, + .rpc_write_init = test_write_init, + .rpc_write_final = test_write_final, +}; + +#define CHECK_RES(res, cleanup) \ + do { \ + TEE_Result _res = (res); \ + \ + if (_res != TEE_SUCCESS) { \ + EMSG("error: res = %#" PRIx32, _res); \ + { cleanup; } \ + } \ + } while (0) + +static uint32_t val_from_bn_n_salt(size_t bn, size_t n, uint8_t salt) +{ + assert(bn < UINT16_MAX); + assert(n < UINT8_MAX); + return SHIFT_U32(n, 16) | SHIFT_U32(bn, 8) | salt; +} + +static TEE_Result write_block(struct tee_fs_htree **ht, size_t bn, uint8_t salt) +{ + uint32_t b[TEST_BLOCK_SIZE / sizeof(uint32_t)] = { 0 }; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(b); n++) + b[n] = val_from_bn_n_salt(bn, n, salt); + + return tee_fs_htree_write_block(ht, bn, b); +} + +static TEE_Result read_block(struct tee_fs_htree **ht, size_t bn, uint8_t salt) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t b[TEST_BLOCK_SIZE / sizeof(uint32_t)] = { 0 }; + size_t n = 0; + + res = tee_fs_htree_read_block(ht, bn, b); + if (res != TEE_SUCCESS) + return res; + + for (n = 0; n < ARRAY_SIZE(b); n++) { + if (b[n] != val_from_bn_n_salt(bn, n, salt)) { + DMSG("Unpected b[%zu] %#" PRIx32 + "(expected %#" PRIx32 ")", + n, b[n], val_from_bn_n_salt(bn, n, salt)); + return TEE_ERROR_TIME_NOT_SET; + } + } + + return TEE_SUCCESS; +} + +static TEE_Result do_range(TEE_Result (*fn)(struct tee_fs_htree **ht, + size_t bn, uint8_t salt), + struct tee_fs_htree **ht, size_t begin, + size_t num_blocks, size_t salt) +{ + TEE_Result res = TEE_SUCCESS; + size_t n = 0; + + for (n = 0; n < num_blocks; n++) { + res = fn(ht, n + begin, salt); + CHECK_RES(res, goto out); + } + +out: + return res; +} + +static TEE_Result do_range_backwards(TEE_Result (*fn)(struct tee_fs_htree **ht, + size_t bn, uint8_t salt), + struct tee_fs_htree **ht, size_t begin, + size_t num_blocks, size_t salt) +{ + TEE_Result res = TEE_SUCCESS; + size_t n = 0; + + for (n = 0; n < num_blocks; n++) { + res = fn(ht, num_blocks - 1 - n + begin, salt); + CHECK_RES(res, goto out); + } + +out: + return res; +} + +static TEE_Result htree_test_rewrite(struct test_aux *aux, size_t num_blocks, + size_t w_unsync_begin, size_t w_unsync_num) +{ + struct ts_session *sess = ts_get_current_session(); + const TEE_UUID *uuid = &sess->ctx->uuid; + TEE_Result res = TEE_SUCCESS; + struct tee_fs_htree *ht = NULL; + size_t salt = 23; + uint8_t hash[TEE_FS_HTREE_HASH_SIZE] = { 0 }; + + assert((w_unsync_begin + w_unsync_num) <= num_blocks); + + aux->data_len = 0; + memset(aux->data, 0xce, aux->data_alloced); + + res = tee_fs_htree_open(true, hash, 0, uuid, &test_htree_ops, aux, &ht); + CHECK_RES(res, goto out); + + /* + * Intialize all blocks and verify that they read back as + * expected. + */ + res = do_range(write_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + /* + * Write all blocks again, but starting from the end using a new + * salt, then verify that that read back as expected. + */ + salt++; + res = do_range_backwards(write_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + /* + * Use a new salt to write all blocks once more and verify that + * they read back as expected. + */ + salt++; + res = do_range(write_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + /* + * Sync the changes of the nodes to memory, verify that all + * blocks are read back as expected. + */ + res = tee_fs_htree_sync_to_storage(&ht, hash, NULL); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + /* + * Close and reopen the hash-tree + */ + tee_fs_htree_close(&ht); + res = tee_fs_htree_open(false, hash, 0, uuid, &test_htree_ops, aux, + &ht); + CHECK_RES(res, goto out); + + /* + * Verify that all blocks are read as expected. + */ + res = do_range(read_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + /* + * Rewrite a few blocks and verify that all blocks are read as + * expected. + */ + res = do_range_backwards(write_block, &ht, w_unsync_begin, w_unsync_num, + salt + 1); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, w_unsync_begin, salt); + CHECK_RES(res, goto out); + res = do_range(read_block, &ht, w_unsync_begin, w_unsync_num, salt + 1); + CHECK_RES(res, goto out); + res = do_range(read_block, &ht, w_unsync_begin + w_unsync_num, + num_blocks - (w_unsync_begin + w_unsync_num), salt); + CHECK_RES(res, goto out); + + /* + * Rewrite the blocks from above again with another salt and + * verify that they are read back as expected. + */ + res = do_range(write_block, &ht, w_unsync_begin, w_unsync_num, + salt + 2); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, w_unsync_begin, salt); + CHECK_RES(res, goto out); + res = do_range(read_block, &ht, w_unsync_begin, w_unsync_num, salt + 2); + CHECK_RES(res, goto out); + res = do_range(read_block, &ht, w_unsync_begin + w_unsync_num, + num_blocks - (w_unsync_begin + w_unsync_num), salt); + CHECK_RES(res, goto out); + + /* + * Skip tee_fs_htree_sync_to_storage() and call + * tee_fs_htree_close() directly to undo the changes since last + * call to tee_fs_htree_sync_to_storage(). Reopen the hash-tree + * and verify that recent changes indeed was discarded. + */ + tee_fs_htree_close(&ht); + res = tee_fs_htree_open(false, hash, 0, uuid, &test_htree_ops, aux, + &ht); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + + /* + * Close, reopen and verify that all blocks are read as expected + * again but this time based on the counter value in struct + * tee_fs_htree_image. + */ + tee_fs_htree_close(&ht); + res = tee_fs_htree_open(false, NULL, 0, uuid, &test_htree_ops, aux, + &ht); + CHECK_RES(res, goto out); + + res = do_range(read_block, &ht, 0, num_blocks, salt); + CHECK_RES(res, goto out); + +out: + tee_fs_htree_close(&ht); + /* + * read_block() returns TEE_ERROR_TIME_NOT_SET in case unexpected + * data is read. + */ + if (res == TEE_ERROR_TIME_NOT_SET) + res = TEE_ERROR_SECURITY; + return res; +} + +static void aux_free(struct test_aux *aux) +{ + if (aux) { + free(aux->data); + free(aux->block); + free(aux); + } +} + +static struct test_aux *aux_alloc(size_t num_blocks) +{ + struct test_aux *aux = NULL; + size_t o = 0; + size_t sz = 0; + + if (test_get_offs_size(TEE_FS_HTREE_TYPE_BLOCK, num_blocks, 1, &o, &sz)) + return NULL; + + aux = calloc(1, sizeof(*aux)); + if (!aux) + return NULL; + + aux->data_alloced = o + sz; + aux->data = malloc(aux->data_alloced); + if (!aux->data) + goto err; + + aux->block = malloc(TEST_BLOCK_SIZE); + if (!aux->block) + goto err; + + return aux; +err: + aux_free(aux); + return NULL; + +} + +static TEE_Result test_write_read(size_t num_blocks) +{ + struct test_aux *aux = aux_alloc(num_blocks); + TEE_Result res = TEE_SUCCESS; + size_t n = 0; + size_t m = 0; + size_t o = 0; + + if (!aux) + return TEE_ERROR_OUT_OF_MEMORY; + + /* + * n is the number of block we're going to initialize/use. + * m is the offset from where we'll rewrite blocks and expect + * the changes to be visible until tee_fs_htree_close() is called + * without a call to tee_fs_htree_sync_to_storage() before. + * o is the number of blocks we're rewriting starting at m. + */ + for (n = 0; n < num_blocks; n += 3) { + for (m = 0; m < n; m += 3) { + for (o = 0; o < (n - m); o++) { + res = htree_test_rewrite(aux, n, m, o); + CHECK_RES(res, goto out); + o += 2; + } + } + } + +out: + aux_free(aux); + return res; +} + +static TEE_Result test_corrupt_type(const TEE_UUID *uuid, uint8_t *hash, + size_t num_blocks, struct test_aux *aux, + enum tee_fs_htree_type type, size_t idx) +{ + TEE_Result res = TEE_SUCCESS; + struct test_aux aux2 = *aux; + struct tee_fs_htree *ht = NULL; + size_t offs = 0; + size_t size = 0; + size_t size0 = 0; + size_t n = 0; + + res = test_get_offs_size(type, idx, 0, &offs, &size0); + CHECK_RES(res, return res); + + aux2.data = malloc(aux->data_alloced); + if (!aux2.data) + return TEE_ERROR_OUT_OF_MEMORY; + + n = 0; + while (true) { + memcpy(aux2.data, aux->data, aux->data_len); + + res = test_get_offs_size(type, idx, 0, &offs, &size); + CHECK_RES(res, goto out); + aux2.data[offs + n]++; + res = test_get_offs_size(type, idx, 1, &offs, &size); + CHECK_RES(res, goto out); + aux2.data[offs + n]++; + + /* + * Errors in head or node is detected by + * tee_fs_htree_open() errors in block is detected when + * actually read by do_range(read_block) + */ + res = tee_fs_htree_open(false, hash, 0, uuid, &test_htree_ops, + &aux2, &ht); + if (!res) { + res = do_range(read_block, &ht, 0, num_blocks, 1); + /* + * do_range(read_block,) is supposed to detect the + * error. If TEE_ERROR_TIME_NOT_SET is returned + * read_block() was acutally able to get some data, + * but the data was incorrect. + * + * If res == TEE_SUCCESS or + * res == TEE_ERROR_TIME_NOT_SET + * there's some problem with the htree + * implementation. + */ + if (res == TEE_ERROR_TIME_NOT_SET) { + EMSG("error: data silently corrupted"); + res = TEE_ERROR_SECURITY; + goto out; + } + if (!res) + break; + tee_fs_htree_close(&ht); + } + + /* We've tested the last byte, let's get out of here */ + if (n == size0 - 1) + break; + + /* Increase n exponentionally after 1 to skip some testing */ + if (n) + n += n; + else + n = 1; + + /* Make sure we test the last byte too */ + if (n >= size0) + n = size0 - 1; + } + + if (res) { + res = TEE_SUCCESS; + } else { + EMSG("error: data corruption undetected"); + res = TEE_ERROR_SECURITY; + } +out: + free(aux2.data); + tee_fs_htree_close(&ht); + return res; +} + + + +static TEE_Result test_corrupt(size_t num_blocks) +{ + struct ts_session *sess = ts_get_current_session(); + const TEE_UUID *uuid = &sess->ctx->uuid; + TEE_Result res = TEE_SUCCESS; + struct tee_fs_htree *ht = NULL; + uint8_t hash[TEE_FS_HTREE_HASH_SIZE] = { 0 }; + struct test_aux *aux = NULL; + size_t n = 0; + + aux = aux_alloc(num_blocks); + if (!aux) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + aux->data_len = 0; + memset(aux->data, 0xce, aux->data_alloced); + + /* Write the object and close it */ + res = tee_fs_htree_open(true, hash, 0, uuid, &test_htree_ops, aux, &ht); + CHECK_RES(res, goto out); + res = do_range(write_block, &ht, 0, num_blocks, 1); + CHECK_RES(res, goto out); + res = tee_fs_htree_sync_to_storage(&ht, hash, NULL); + CHECK_RES(res, goto out); + tee_fs_htree_close(&ht); + + /* Verify that the object can be read correctly */ + res = tee_fs_htree_open(false, hash, 0, uuid, &test_htree_ops, aux, + &ht); + CHECK_RES(res, goto out); + res = do_range(read_block, &ht, 0, num_blocks, 1); + CHECK_RES(res, goto out); + tee_fs_htree_close(&ht); + + res = test_corrupt_type(uuid, hash, num_blocks, aux, + TEE_FS_HTREE_TYPE_HEAD, 0); + CHECK_RES(res, goto out); + for (n = 0; n < num_blocks; n++) { + res = test_corrupt_type(uuid, hash, num_blocks, aux, + TEE_FS_HTREE_TYPE_NODE, n); + CHECK_RES(res, goto out); + } + for (n = 0; n < num_blocks; n++) { + res = test_corrupt_type(uuid, hash, num_blocks, aux, + TEE_FS_HTREE_TYPE_BLOCK, n); + CHECK_RES(res, goto out); + } + +out: + tee_fs_htree_close(&ht); + aux_free(aux); + return res; +} + +TEE_Result core_fs_htree_tests(uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS] __unused) +{ + TEE_Result res = TEE_SUCCESS; + + if (nParamTypes) + return TEE_ERROR_BAD_PARAMETERS; + + res = test_write_read(10); + if (res) + return res; + + return test_corrupt(5); +} diff --git a/optee_os/core/pta/tests/invoke.c b/optee_os/core/pta/tests/invoke.c new file mode 100644 index 0000000..bbc761f --- /dev/null +++ b/optee_os/core/pta/tests/invoke.c @@ -0,0 +1,449 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "misc.h" + +#define TA_NAME "invoke_tests.pta" + +static TEE_Result test_trace(uint32_t param_types __unused, + TEE_Param params[TEE_NUM_PARAMS] __unused) +{ + IMSG("pseudo TA \"%s\" says \"Hello world !\"", TA_NAME); + + return TEE_SUCCESS; +} + +static int test_v2p2v(void *va, size_t size) +{ + struct ts_session *session = NULL; + paddr_t p = 0; + void *v = NULL; + + if (!va) + return 0; + + session = ts_get_current_session(); + p = virt_to_phys(va); + + /* 0 is not a valid physical address */ + if (!p) + return 1; + + if (to_ta_session(session)->clnt_id.login == TEE_LOGIN_TRUSTED_APP) { + v = phys_to_virt(p, MEM_AREA_TS_VASPACE, size); + } else { + v = phys_to_virt(p, MEM_AREA_NSEC_SHM, size); + if (!v) + v = phys_to_virt(p, MEM_AREA_SDP_MEM, size); + if (!v) + v = phys_to_virt(p, MEM_AREA_SHM_VASPACE, size); + } + + /* + * Return an error only the vaddr found mismatches input address. + * Finding a virtual address from a physical address cannot be painful + * in some case (i.e pager). Moreover this operation is more debug + * related. Thus do not report error if phys_to_virt failed + */ + if (v && va != v) { + EMSG("Failed to p2v/v2p on caller TA memref arguments"); + EMSG("va %p -> pa 0x%" PRIxPA " -> va %p", va, p, v); + return 1; + } + + return 0; +} + +/* + * Check PTA can be invoked with a memory reference on a NULL buffer + */ +static TEE_Result test_entry_memref_null(uint32_t type, + TEE_Param p[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != type) + return TEE_ERROR_BAD_PARAMETERS; + + if (p[0].memref.buffer || p[0].memref.size) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; +} + +/* + * Supported tests on parameters + * (I, J, K, L refer to param index) + * + * Case 1: command parameters type are: 1 in/out value, 3 empty. + * => process outI.a = inI.a + inI.b + * Case 2: command parameters type are: 3 input value, 1 output value + * => process = outI.a = inJ.a + inK.a + inL.a + * Case 3: command parameters type are: 1 in/out memref, 3 empty. + * => process = outI[0] = sum(inI[0..len-1]) + */ +static TEE_Result test_entry_params(uint32_t type, TEE_Param p[TEE_NUM_PARAMS]) +{ + size_t i; + uint8_t d8, *in; + + /* case 1a: 1 input/output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_VALUE_INOUT) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_NONE)) { + p[0].value.a = p[0].value.a + p[0].value.b; + return TEE_SUCCESS; + } + /* case 1b: 1 input/output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_VALUE_INOUT) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_NONE)) { + p[1].value.a = p[1].value.a + p[1].value.b; + return TEE_SUCCESS; + } + /* case 1c: 1 input/output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_VALUE_INOUT) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_NONE)) { + p[2].value.a = p[2].value.a + p[2].value.b; + return TEE_SUCCESS; + } + /* case 1d: 1 input/output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_VALUE_INOUT)) { + p[3].value.a = p[3].value.a + p[3].value.b; + return TEE_SUCCESS; + } + + /* case 2a: 3 input value arguments, 1 output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_VALUE_OUTPUT) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_VALUE_INPUT)) { + p[0].value.a = p[1].value.a + p[2].value.a + p[3].value.a; + p[0].value.b = p[1].value.b + p[2].value.b + p[3].value.b; + return TEE_SUCCESS; + } + /* case 2a: 3 input value arguments, 1 output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_VALUE_OUTPUT) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_VALUE_INPUT)) { + p[1].value.a = p[0].value.a + p[2].value.a + p[3].value.a; + p[1].value.b = p[0].value.b + p[2].value.b + p[3].value.b; + return TEE_SUCCESS; + } + /* case 2a: 3 input value arguments, 1 output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_VALUE_OUTPUT) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_VALUE_INPUT)) { + p[2].value.a = p[0].value.a + p[1].value.a + p[3].value.a; + p[2].value.b = p[0].value.b + p[1].value.b + p[3].value.b; + return TEE_SUCCESS; + } + /* case 2a: 3 input value arguments, 1 output value argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_VALUE_INPUT) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_VALUE_OUTPUT)) { + p[3].value.a = p[0].value.a + p[1].value.a + p[2].value.a; + p[3].value.b = p[0].value.b + p[1].value.b + p[2].value.b; + return TEE_SUCCESS; + } + + DMSG("expect memref params: %p/%zu - %p/%zu - %p/%zu - %p/%zu", + p[0].memref.buffer, p[0].memref.size, p[1].memref.buffer, + p[1].memref.size, p[2].memref.buffer, p[2].memref.size, + p[3].memref.buffer, p[3].memref.size); + + /* case 3a: 1 in/out memref argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_MEMREF_INOUT) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_NONE)) { + in = (uint8_t *)p[0].memref.buffer; + if (test_v2p2v(in, p[0].memref.size)) + return TEE_ERROR_SECURITY; + d8 = 0; + for (i = 0; i < p[0].memref.size; i++) + d8 += in[i]; + *(uint8_t *)p[0].memref.buffer = d8; + return TEE_SUCCESS; + } + /* case 3b: 1 in/out memref argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_MEMREF_INOUT) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_NONE)) { + in = (uint8_t *)p[1].memref.buffer; + if (test_v2p2v(in, p[1].memref.size)) + return TEE_ERROR_SECURITY; + d8 = 0; + for (i = 0; i < p[1].memref.size; i++) + d8 += in[i]; + *(uint8_t *)p[1].memref.buffer = d8; + return TEE_SUCCESS; + } + /* case 3c: 1 in/out memref argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_MEMREF_INOUT) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_NONE)) { + in = (uint8_t *)p[2].memref.buffer; + if (test_v2p2v(in, p[2].memref.size)) + return TEE_ERROR_SECURITY; + d8 = 0; + for (i = 0; i < p[2].memref.size; i++) + d8 += in[i]; + *(uint8_t *)p[2].memref.buffer = d8; + return TEE_SUCCESS; + } + /* case 3d: 1 in/out memref argument */ + if ((TEE_PARAM_TYPE_GET(type, 0) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 1) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 2) == TEE_PARAM_TYPE_NONE) && + (TEE_PARAM_TYPE_GET(type, 3) == TEE_PARAM_TYPE_MEMREF_INOUT)) { + in = (uint8_t *)p[3].memref.buffer; + if (test_v2p2v(in, p[3].memref.size)) + return TEE_ERROR_SECURITY; + d8 = 0; + for (i = 0; i < p[3].memref.size; i++) + d8 += in[i]; + *(uint8_t *)p[3].memref.buffer = d8; + return TEE_SUCCESS; + } + + EMSG("unexpected parameters"); + return TEE_ERROR_BAD_PARAMETERS; +} + +/* + * Test access to Secure Data Path memory from pseudo TAs + */ + +static TEE_Result test_inject_sdp(uint32_t type, TEE_Param p[TEE_NUM_PARAMS]) +{ + char *src = p[0].memref.buffer; + char *dst = p[1].memref.buffer; + size_t sz = p[0].memref.size; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != type) { + DMSG("bad parameter types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (p[1].memref.size < sz) { + p[1].memref.size = sz; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!core_vbuf_is(CORE_MEM_NON_SEC, src, sz) || + !core_vbuf_is(CORE_MEM_SDP_MEM, dst, sz)) { + DMSG("bad memref secure attribute"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!sz) + return TEE_SUCCESS; + + /* Check that core can p2v and v2p over memory reference arguments */ + if (test_v2p2v(src, sz) || test_v2p2v(dst, sz)) + return TEE_ERROR_SECURITY; + + if (cache_operation(TEE_CACHEFLUSH, dst, sz) != TEE_SUCCESS) + return TEE_ERROR_GENERIC; + + memcpy(dst, src, sz); + + if (cache_operation(TEE_CACHEFLUSH, dst, sz) != TEE_SUCCESS) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result test_transform_sdp(uint32_t type, TEE_Param p[TEE_NUM_PARAMS]) +{ + char *buf = p[0].memref.buffer; + size_t sz = p[0].memref.size; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != type) { + DMSG("bad parameter types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!core_vbuf_is(CORE_MEM_SDP_MEM, buf, sz)) { + DMSG("bad memref secure attribute"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!sz) + return TEE_SUCCESS; + + /* Check that core can p2v and v2p over memory reference arguments */ + if (test_v2p2v(buf, sz)) + return TEE_ERROR_SECURITY; + + if (cache_operation(TEE_CACHEFLUSH, buf, sz) != TEE_SUCCESS) + return TEE_ERROR_GENERIC; + + for (; sz; sz--, buf++) + *buf = ~(*buf) + 1; + + if (cache_operation(TEE_CACHEFLUSH, buf, sz) != TEE_SUCCESS) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +static TEE_Result test_dump_sdp(uint32_t type, TEE_Param p[TEE_NUM_PARAMS]) +{ + char *src = p[0].memref.buffer; + char *dst = p[1].memref.buffer; + size_t sz = p[0].memref.size; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != type) { + DMSG("bad parameter types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (p[1].memref.size < sz) { + p[1].memref.size = sz; + return TEE_ERROR_SHORT_BUFFER; + } + + if (!core_vbuf_is(CORE_MEM_SDP_MEM, src, sz) || + !core_vbuf_is(CORE_MEM_NON_SEC, dst, sz)) { + DMSG("bad memref secure attribute"); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (!sz) + return TEE_SUCCESS; + + /* Check that core can p2v and v2p over memory reference arguments */ + if (test_v2p2v(src, sz) || test_v2p2v(dst, sz)) + return TEE_ERROR_SECURITY; + + if (cache_operation(TEE_CACHEFLUSH, dst, sz) != TEE_SUCCESS) + return TEE_ERROR_GENERIC; + + memcpy(dst, src, sz); + + if (cache_operation(TEE_CACHEFLUSH, dst, sz) != TEE_SUCCESS) + return TEE_ERROR_GENERIC; + + return TEE_SUCCESS; +} + +/* + * Trusted Application Entry Points + */ + +static TEE_Result create_ta(void) +{ + DMSG("create entry point for pseudo TA \"%s\"", TA_NAME); + return TEE_SUCCESS; +} + +static void destroy_ta(void) +{ + DMSG("destroy entry point for pseudo ta \"%s\"", TA_NAME); +} + +static TEE_Result open_session(uint32_t nParamTypes __unused, + TEE_Param pParams[TEE_NUM_PARAMS] __unused, + void **ppSessionContext __unused) +{ + DMSG("open entry point for pseudo ta \"%s\"", TA_NAME); + return TEE_SUCCESS; +} + +static void close_session(void *pSessionContext __unused) +{ + DMSG("close entry point for pseudo ta \"%s\"", TA_NAME); +} + +static TEE_Result invoke_command(void *pSessionContext __unused, + uint32_t nCommandID, uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]) +{ + FMSG("command entry point for pseudo ta \"%s\"", TA_NAME); + + switch (nCommandID) { + case PTA_INVOKE_TESTS_CMD_TRACE: + return test_trace(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_PARAMS: + return test_entry_params(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_MEMREF_NULL: + return test_entry_memref_null(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_COPY_NSEC_TO_SEC: + return test_inject_sdp(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_READ_MODIFY_SEC: + return test_transform_sdp(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_COPY_SEC_TO_NSEC: + return test_dump_sdp(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_SELF_TESTS: + return core_self_tests(nParamTypes, pParams); +#if defined(CFG_REE_FS) && defined(CFG_WITH_USER_TA) + case PTA_INVOKE_TESTS_CMD_FS_HTREE: + return core_fs_htree_tests(nParamTypes, pParams); +#endif + case PTA_INVOKE_TESTS_CMD_MUTEX: + return core_mutex_tests(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_LOCKDEP: + return core_lockdep_tests(nParamTypes, pParams); + case PTA_INVOKE_TEST_CMD_AES_PERF: + return core_aes_perf_tests(nParamTypes, pParams); + case PTA_INVOKE_TESTS_CMD_DT_DRIVER_TESTS: + return core_dt_driver_tests(nParamTypes, pParams); + default: + break; + } + return TEE_ERROR_BAD_PARAMETERS; +} + +pseudo_ta_register(.uuid = PTA_INVOKE_TESTS_UUID, .name = TA_NAME, + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_SECURE_DATA_PATH | + TA_FLAG_CONCURRENT | TA_FLAG_DEVICE_ENUM, + .create_entry_point = create_ta, + .destroy_entry_point = destroy_ta, + .open_session_entry_point = open_session, + .close_session_entry_point = close_session, + .invoke_command_entry_point = invoke_command); diff --git a/optee_os/core/pta/tests/lockdep.c b/optee_os/core/pta/tests/lockdep.c new file mode 100644 index 0000000..3cdca06 --- /dev/null +++ b/optee_os/core/pta/tests/lockdep.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ + +/* + * Test lockdep with hypothetical thread and lock objects + */ + +#include +#include + +#include "misc.h" + +static int self_test_lockdep1(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct lockdep_node_head graph; + struct lockdep_lock_head thread1; + int count = 1; + + DMSG(""); + + TAILQ_INIT(&thread1); + TAILQ_INIT(&graph); + + /* Not locked, expect failure */ + res = __lockdep_lock_release(&thread1, 1); + if (!res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread1, 1); + if (res) + return count; + count++; + + res = __lockdep_lock_release(&thread1, 1); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread1, 1); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread1, 3); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread1, 2); + if (res) + return count; + count++; + + res = __lockdep_lock_release(&thread1, 3); + if (res) + return count; + count++; + + /* Already locked */ + res = __lockdep_lock_acquire(&graph, &thread1, 2); + if (!res) + return count; + + lockdep_graph_delete(&graph); + lockdep_queue_delete(&thread1); + + return 0; +} + +static int self_test_lockdep2(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct lockdep_node_head graph; + struct lockdep_lock_head thread1; + struct lockdep_lock_head thread2; + struct lockdep_lock_head thread3; + int count = 1; + + DMSG(""); + + TAILQ_INIT(&thread1); + TAILQ_INIT(&thread2); + TAILQ_INIT(&thread3); + TAILQ_INIT(&graph); + + res = __lockdep_lock_acquire(&graph, &thread1, 1); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread2, 2); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread1, 2); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread3, 3); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread2, 3); + if (res) + return count; + count++; + + /* Deadlock 1-2-3 */ + res = __lockdep_lock_acquire(&graph, &thread3, 1); + if (!res) + return count; + + lockdep_graph_delete(&graph); + lockdep_queue_delete(&thread1); + lockdep_queue_delete(&thread2); + lockdep_queue_delete(&thread3); + + return 0; +} + +static int self_test_lockdep3(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct lockdep_node_head graph; + struct lockdep_lock_head thread1; + struct lockdep_lock_head thread2; + int count = 1; + + DMSG(""); + + TAILQ_INIT(&thread1); + TAILQ_INIT(&thread2); + TAILQ_INIT(&graph); + + res = __lockdep_lock_tryacquire(&graph, &thread1, 1); + if (res) + return count; + count++; + + res = __lockdep_lock_release(&thread1, 1); + if (res) + return count; + count++; + + res = __lockdep_lock_tryacquire(&graph, &thread1, 1); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread2, 2); + if (res) + return count; + count++; + + res = __lockdep_lock_acquire(&graph, &thread1, 2); + if (res) + return count; + count++; + + /* Deadlock 1-2 */ + res = __lockdep_lock_acquire(&graph, &thread2, 1); + if (!res) + return count; + + lockdep_graph_delete(&graph); + lockdep_queue_delete(&thread1); + lockdep_queue_delete(&thread2); + + return 0; +} + +TEE_Result core_lockdep_tests(uint32_t nParamTypes __unused, + TEE_Param pParams[TEE_NUM_PARAMS] __unused) + +{ + int count = 0; + + count = self_test_lockdep1(); + if (count) + goto out; + count = self_test_lockdep2(); + if (count) + goto out; + count = self_test_lockdep3(); + if (count) + goto out; +out: + if (count) { + DMSG("count=%d", count); + return TEE_ERROR_GENERIC; + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/pta/tests/misc.c b/optee_os/core/pta/tests/misc.c new file mode 100644 index 0000000..63dbea4 --- /dev/null +++ b/optee_os/core/pta/tests/misc.c @@ -0,0 +1,580 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "misc.h" + +/* + * Enable expect LOG macro to enable/disable self tests traces. + * + * #define LOG DMSG_RAW + * #define LOG(...) + */ +#define LOG(...) + +static int self_test_add_overflow(void) +{ + uint32_t r_u32; + int32_t r_s32; + uintmax_t r_um; + intmax_t r_sm; + + if (ADD_OVERFLOW(8U, 0U, &r_s32)) + return -1; + if (r_s32 != 8) + return -1; + if (ADD_OVERFLOW(32U, 30U, &r_u32)) + return -1; + if (r_u32 != 62) + return -1; + if (!ADD_OVERFLOW(UINT32_MAX, UINT32_MAX, &r_u32)) + return -1; + if (!ADD_OVERFLOW(UINT32_MAX / 2 + 1, UINT32_MAX / 2 + 1, &r_u32)) + return -1; + if (ADD_OVERFLOW(UINT32_MAX / 2, UINT32_MAX / 2 + 1, &r_u32)) + return -1; + if (r_u32 != UINT32_MAX) + return -1; + + if (ADD_OVERFLOW((uint32_t)30, (int32_t)-31, &r_s32)) + return -1; + if (r_s32 != -1) + return -1; + if (ADD_OVERFLOW((int32_t)30, (int32_t)-31, &r_s32)) + return -1; + if (r_s32 != -1) + return -1; + if (ADD_OVERFLOW((int32_t)-31, (uint32_t)30, &r_s32)) + return -1; + if (r_s32 != -1) + return -1; + + if (ADD_OVERFLOW(INT32_MIN + 1, -1, &r_s32)) + return -1; + if (r_s32 != INT32_MIN) + return -1; + if (!ADD_OVERFLOW(INT32_MIN, -1, &r_s32)) + return -1; + if (!ADD_OVERFLOW(INT32_MIN + 1, -2, &r_s32)) + return -1; + if (!ADD_OVERFLOW(INT32_MAX, INT32_MAX, &r_s32)) + return -1; + if (ADD_OVERFLOW(INT32_MAX, INT32_MAX, &r_u32)) + return -1; + if (!ADD_OVERFLOW(INTMAX_MAX, INTMAX_MAX, &r_sm)) + return -1; + if (ADD_OVERFLOW(INTMAX_MAX, INTMAX_MAX, &r_um)) + return -1; + if (!ADD_OVERFLOW(INT32_MAX / 2 + 1, INT32_MAX / 2 + 1, &r_s32)) + return -1; + if (ADD_OVERFLOW(INT32_MAX / 2, INT32_MAX / 2 + 1, &r_s32)) + return -1; + if (r_s32 != INT32_MAX) + return -1; + + return 0; +} + +static int self_test_sub_overflow(void) +{ + uint32_t r_u32; + int32_t r_s32; + intmax_t r_sm; + + if (SUB_OVERFLOW(8U, 1U, &r_s32)) + return -1; + if (r_s32 != 7) + return -1; + if (SUB_OVERFLOW(32U, 30U, &r_u32)) + return -1; + if (r_u32 != 2) + return -1; + if (!SUB_OVERFLOW(30U, 31U, &r_u32)) + return -1; + + if (SUB_OVERFLOW(30, 31, &r_s32)) + return -1; + if (r_s32 != -1) + return -1; + if (SUB_OVERFLOW(-1, INT32_MAX, &r_s32)) + return -1; + if (r_s32 != INT32_MIN) + return -1; + if (!SUB_OVERFLOW(-2, INT32_MAX, &r_s32)) + return -1; + + if (SUB_OVERFLOW((uint32_t)30, (int32_t)-31, &r_s32)) + return -1; + if (r_s32 != 61) + return -1; + if (SUB_OVERFLOW((int32_t)30, (int32_t)-31, &r_s32)) + return -1; + if (r_s32 != 61) + return -1; + if (SUB_OVERFLOW((int32_t)-31, (uint32_t)30, &r_s32)) + return -1; + if (r_s32 != -61) + return -1; + if (SUB_OVERFLOW((int32_t)-31, (int32_t)-30, &r_s32)) + return -1; + if (r_s32 != -1) + return -1; + + if (SUB_OVERFLOW((int32_t)31, -(INTMAX_MIN + 1), &r_sm)) + return -1; + if (r_sm != (INTMAX_MIN + 32)) + return -1; + + return 0; +} + +static int self_test_mul_unsigned_overflow(void) +{ + const size_t um_half_shift = sizeof(uintmax_t) * 8 / 2; + const uintmax_t um_half_mask = UINTMAX_MAX >> um_half_shift; + uint32_t r_u32; + uintmax_t r_um; + + if (MUL_OVERFLOW(32, 30, &r_u32)) + return -1; + if (r_u32 != 960) + return -1; + if (MUL_OVERFLOW(-32, -30, &r_u32)) + return -1; + if (r_u32 != 960) + return -1; + + if (MUL_OVERFLOW(UINTMAX_MAX, 1, &r_um)) + return -1; + if (r_um != UINTMAX_MAX) + return -1; + if (MUL_OVERFLOW(UINTMAX_MAX / 4, 4, &r_um)) + return -1; + if (r_um != (UINTMAX_MAX - 3)) + return -1; + if (!MUL_OVERFLOW(UINTMAX_MAX / 4 + 1, 4, &r_um)) + return -1; + if (!MUL_OVERFLOW(UINTMAX_MAX, UINTMAX_MAX, &r_um)) + return -1; + if (!MUL_OVERFLOW(um_half_mask << um_half_shift, + um_half_mask << um_half_shift, &r_um)) + return -1; + + return 0; +} + +static int self_test_mul_signed_overflow(void) +{ + intmax_t r; + + if (MUL_OVERFLOW(32, -30, &r)) + return -1; + if (r != -960) + return -1; + if (MUL_OVERFLOW(-32, 30, &r)) + return -1; + if (r != -960) + return -1; + if (MUL_OVERFLOW(32, 30, &r)) + return -1; + if (r != 960) + return -1; + + if (MUL_OVERFLOW(INTMAX_MAX, 1, &r)) + return -1; + if (r != INTMAX_MAX) + return -1; + if (MUL_OVERFLOW(INTMAX_MAX / 4, 4, &r)) + return -1; + if (r != (INTMAX_MAX - 3)) + return -1; + if (!MUL_OVERFLOW(INTMAX_MAX / 4 + 1, 4, &r)) + return -1; + if (!MUL_OVERFLOW(INTMAX_MAX, INTMAX_MAX, &r)) + return -1; + if (MUL_OVERFLOW(INTMAX_MIN + 1, 1, &r)) + return -1; + if (r != INTMAX_MIN + 1) + return -1; + if (MUL_OVERFLOW(1, INTMAX_MIN + 1, &r)) + return -1; + if (r != INTMAX_MIN + 1) + return -1; + if (MUL_OVERFLOW(0, INTMAX_MIN, &r)) + return -1; + if (r != 0) + return -1; + if (MUL_OVERFLOW(1, INTMAX_MIN, &r)) + return -1; + if (r != INTMAX_MIN) + return -1; + + return 0; +} + +/* test division support. resulting trace shall be manually checked */ +static int self_test_division(void) +{ + signed a, b, c, d; + bool r; + int ret = 0; + + LOG(""); + LOG("division tests (division and modulo):"); + /* get some unpredicted values to prevent compilation optimizations: */ + /* => use the stack address */ + + LOG("- test with unsigned small integers:"); + a = (signed)((unsigned)(vaddr_t)&a & 0xFFFFF); + b = (signed)((unsigned)(vaddr_t)&b & 0x00FFF) + 1; + c = a / b; + d = a % b; + r = ((b * c + d) == a); + if (!r) + ret = -1; + LOG(" 0x%08x / 0x%08x = %u / %u = %u = 0x%x)", + (unsigned)a, (unsigned)b, (unsigned)a, (unsigned)b, (unsigned)c, + (unsigned)c); + LOG(" 0x%08x %% 0x%08x = %u %% %u = %u = 0x%x)", (unsigned)a, + (unsigned)b, (unsigned)a, (unsigned)b, (unsigned)d, (unsigned)d); + LOG(" check results => %s", r ? "ok" : "FAILED !!!"); + LOG(""); + + LOG("- test with signed small integers, negative numerator:"); + a = (signed)(vaddr_t)&a; + b = (signed)((unsigned)(vaddr_t)&b & 0x00FFF) - 1; + c = a / b; + d = a % b; + r = ((b * c + d) == a); + if (!r) + ret = -1; + LOG(" 0x%08x / 0x%08x = %d / %d = %d = 0x%x)", + (unsigned)a, (unsigned)b, (signed)a, (signed)b, (signed)c, + (unsigned)c); + LOG(" 0x%08x %% 0x%08x = %d %% %d = %d = 0x%x)", (unsigned)a, + (unsigned)b, (signed)a, (signed)b, (signed)d, (unsigned)d); + LOG(" check results => %s", r ? "ok" : "FAILED !!!"); + LOG(""); + + LOG("- test with signed small integers, negative denominator:"); + a = (signed)((unsigned)(vaddr_t)&a & 0xFFFFF); + b = -(signed)((unsigned)(vaddr_t)&b & 0x00FFF) + 1; + c = a / b; + d = a % b; + + LOG("- test with unsigned integers, big numerator (> 0x80000000):"); + a = (signed)(vaddr_t)&a; + b = (signed)((unsigned)(vaddr_t)&b & 0x00FFF) + 1; + c = (signed)((unsigned)a / (unsigned)b); + d = (signed)((unsigned)a % (unsigned)b); + r = (((unsigned)b * (unsigned)c + (unsigned)d) == (unsigned)a); + if (!r) + ret = -1; + LOG(" 0x%08x / 0x%08x = %u / %u = %u = 0x%x)", + (unsigned)a, (unsigned)b, (unsigned)a, (unsigned)b, (unsigned)c, + (unsigned)c); + LOG(" 0x%08x %% 0x%08x = %u %% %u = %u = 0x%x)", (unsigned)a, + (unsigned)b, (unsigned)a, (unsigned)b, (unsigned)d, (unsigned)d); + LOG(" check results => %s", r ? "ok" : "FAILED !!!"); + LOG(""); + + LOG("- test with unsigned integers, big num. & denom. (> 0x80000000):"); + a = (signed)(vaddr_t)&a; + b = (signed)((unsigned)(vaddr_t)&a - 1); + c = (signed)((unsigned)a / (unsigned)b); + d = (signed)((unsigned)a % (unsigned)b); + r = (((unsigned)b * (unsigned)c + (unsigned)d) == (unsigned)a); + if (!r) + ret = -1; + LOG(" 0x%08x / 0x%08x = %u / %u = %u = 0x%x)", + (unsigned)a, (unsigned)b, (unsigned)a, (unsigned)b, (unsigned)c, + (unsigned)c); + LOG(" 0x%08x %% 0x%08x = %u %% %u = %u = 0x%x)", (unsigned)a, + (unsigned)b, (unsigned)a, (unsigned)b, (unsigned)d, (unsigned)d); + LOG(" check results => %s", r ? "ok" : "FAILED !!!"); + LOG(""); + + return ret; +} + +/* test malloc support. resulting trace shall be manually checked */ +static int self_test_malloc(void) +{ + char *p1 = NULL, *p2 = NULL; + int *p3 = NULL, *p4 = NULL; + bool r; + int ret = 0; + + LOG("malloc tests:"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + /* test malloc */ + p1 = malloc(1024); + LOG("- p1 = malloc(1024)"); + p2 = malloc(1024); + LOG("- p2 = malloc(1024)"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p1 && p2 && malloc_buffer_is_within_alloced(p1, 1024) && + !malloc_buffer_is_within_alloced(p1 + 25, 1000) && + !malloc_buffer_is_within_alloced(p1 - 25, 500) && + malloc_buffer_overlaps_heap(p1 - 25, 500)); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + + /* test realloc */ + p3 = realloc(p1, 3 * 1024); + if (p3) + p1 = NULL; + LOG("- p3 = realloc(p1, 3*1024)"); + LOG("- free p2"); + free(p2); + p2 = malloc(1024); + LOG("- p2 = malloc(1024)"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p2 && p3); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- free p1, p2, p3"); + free(p1); + free(p2); + free(p3); + p1 = NULL; + p2 = NULL; + p3 = NULL; + + /* test calloc */ + p3 = calloc(4, 1024); + p4 = calloc(0x100, 1024 * 1024); + LOG("- p3 = calloc(4, 1024)"); + LOG("- p4 = calloc(0x100, 1024*1024) too big: should fail!"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p3 && !p4); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- free p3, p4"); + free(p3); + free(p4); + p3 = NULL; + p4 = NULL; + + /* test memalign */ + p3 = memalign(0x1000, 1024); + LOG("- p3 = memalign(%d, 1024)", 0x1000); + p1 = malloc(1024); + LOG("- p1 = malloc(1024)"); + p4 = memalign(0x100, 512); + LOG("- p4 = memalign(%d, 512)", 0x100); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p1 && p3 && p4 && + !((vaddr_t)p3 % 0x1000) && !((vaddr_t)p4 % 0x100)); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- free p1, p3, p4"); + free(p1); + free(p3); + free(p4); + p1 = NULL; + p3 = NULL; + p4 = NULL; + + /* test memalign with invalid alignments */ + p3 = memalign(100, 1024); + LOG("- p3 = memalign(%d, 1024)", 100); + p4 = memalign(0, 1024); + LOG("- p4 = memalign(%d, 1024)", 0); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (!p3 && !p4); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- free p3, p4"); + free(p3); + free(p4); + p3 = NULL; + p4 = NULL; + + /* test free(NULL) */ + LOG("- free NULL"); + free(NULL); + LOG(""); + LOG("malloc test done"); + + return ret; +} + +#ifdef CFG_NS_VIRTUALIZATION +/* test nex_malloc support. resulting trace shall be manually checked */ +static int self_test_nex_malloc(void) +{ + char *p1 = NULL, *p2 = NULL; + int *p3 = NULL, *p4 = NULL; + bool r; + int ret = 0; + + LOG("nex_malloc tests:"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + /* test malloc */ + p1 = nex_malloc(1024); + LOG("- p1 = nex_malloc(1024)"); + p2 = nex_malloc(1024); + LOG("- p2 = nex_malloc(1024)"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p1 && p2 && nex_malloc_buffer_is_within_alloced(p1, 1024) && + !nex_malloc_buffer_is_within_alloced(p1 + 25, 1000) && + !nex_malloc_buffer_is_within_alloced(p1 - 25, 500) && + nex_malloc_buffer_overlaps_heap(p1 - 25, 500)); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + + /* test realloc */ + p3 = nex_realloc(p1, 3 * 1024); + if (p3) + p1 = NULL; + LOG("- p3 = nex_realloc(p1, 3*1024)"); + LOG("- nex_free p2"); + nex_free(p2); + p2 = nex_malloc(1024); + LOG("- p2 = nex_malloc(1024)"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p2 && p3); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- nex_free p1, p2, p3"); + nex_free(p1); + nex_free(p2); + nex_free(p3); + p1 = NULL; + p2 = NULL; + p3 = NULL; + + /* test calloc */ + p3 = nex_calloc(4, 1024); + p4 = nex_calloc(0x100, 1024 * 1024); + LOG("- p3 = nex_calloc(4, 1024)"); + LOG("- p4 = nex_calloc(0x100, 1024*1024) too big: should fail!"); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p3 && !p4); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- nex_free p3, p4"); + nex_free(p3); + nex_free(p4); + p3 = NULL; + p4 = NULL; + + /* test memalign */ + p3 = nex_memalign(0x1000, 1024); + LOG("- p3 = nex_memalign(%d, 1024)", 0x1000); + p1 = nex_malloc(1024); + LOG("- p1 = nex_malloc(1024)"); + p4 = nex_memalign(0x100, 512); + LOG("- p4 = nex_memalign(%d, 512)", 0x100); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (p1 && p3 && p4 && + !((vaddr_t)p3 % 0x1000) && !((vaddr_t)p4 % 0x100)); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- nex_free p1, p3, p4"); + nex_free(p1); + nex_free(p3); + nex_free(p4); + p1 = NULL; + p3 = NULL; + p4 = NULL; + + /* test memalign with invalid alignments */ + p3 = nex_memalign(100, 1024); + LOG("- p3 = nex_memalign(%d, 1024)", 100); + p4 = nex_memalign(0, 1024); + LOG("- p4 = nex_memalign(%d, 1024)", 0); + LOG(" p1=%p p2=%p p3=%p p4=%p", + (void *)p1, (void *)p2, (void *)p3, (void *)p4); + r = (!p3 && !p4); + if (!r) + ret = -1; + LOG(" => test %s", r ? "ok" : "FAILED"); + LOG(""); + LOG("- nex_free p3, p4"); + nex_free(p3); + nex_free(p4); + p3 = NULL; + p4 = NULL; + + /* test free(NULL) */ + LOG("- nex_free NULL"); + nex_free(NULL); + LOG(""); + LOG("nex_malloc test done"); + + return ret; +} +#else /* CFG_NS_VIRTUALIZATION */ +static int self_test_nex_malloc(void) +{ + return 0; +} +#endif + +/* exported entry points for some basic test */ +TEE_Result core_self_tests(uint32_t nParamTypes __unused, + TEE_Param pParams[TEE_NUM_PARAMS] __unused) +{ + if (self_test_mul_signed_overflow() || self_test_add_overflow() || + self_test_sub_overflow() || self_test_mul_unsigned_overflow() || + self_test_division() || self_test_malloc() || + self_test_nex_malloc()) { + EMSG("some self_test_xxx failed! you should enable local LOG"); + return TEE_ERROR_GENERIC; + } + return TEE_SUCCESS; +} + +/* Exported entrypoint for dt_driver tests */ +TEE_Result core_dt_driver_tests(uint32_t nParamTypes __unused, + TEE_Param pParams[TEE_NUM_PARAMS] __unused) +{ + if (IS_ENABLED(CFG_DT_DRIVER_EMBEDDED_TEST)) { + if (dt_driver_test_status()) + return TEE_ERROR_GENERIC; + } else { + IMSG("dt_driver tests are not embedded"); + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/pta/tests/misc.h b/optee_os/core/pta/tests/misc.h new file mode 100644 index 0000000..58aced4 --- /dev/null +++ b/optee_os/core/pta/tests/misc.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef CORE_PTA_TESTS_MISC_H +#define CORE_PTA_TESTS_MISC_H + +#include +#include +#include + +/* basic run-time tests */ +TEE_Result core_self_tests(uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]); + +TEE_Result core_fs_htree_tests(uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]); + +TEE_Result core_mutex_tests(uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]); + +#ifdef CFG_LOCKDEP +TEE_Result core_lockdep_tests(uint32_t nParamTypes, + TEE_Param pParams[TEE_NUM_PARAMS]); +#else +static inline TEE_Result core_lockdep_tests( + uint32_t nParamTypes __unused, + TEE_Param pParams[TEE_NUM_PARAMS] __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif + +TEE_Result core_aes_perf_tests(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]); + +TEE_Result core_dt_driver_tests(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]); + +#endif /*CORE_PTA_TESTS_MISC_H*/ diff --git a/optee_os/core/pta/tests/mutex.c b/optee_os/core/pta/tests/mutex.c new file mode 100644 index 0000000..f1020a1 --- /dev/null +++ b/optee_os/core/pta/tests/mutex.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include + +#include "misc.h" + +static uint32_t before_lock_readers; +static uint32_t before_lock_writers; +static uint32_t during_lock_readers; +static uint32_t during_lock_writers; + +static uint64_t val0; +static uint64_t val1; + +struct mutex test_mutex = MUTEX_INITIALIZER; + +static TEE_Result mutex_test_writer(TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t n; + + params[1].value.a = atomic_inc32(&before_lock_writers); + + mutex_lock(&test_mutex); + + atomic_dec32(&before_lock_writers); + + params[1].value.b = atomic_inc32(&during_lock_writers); + + for (n = 0; n < params[0].value.b; n++) { + val0++; + val1++; + val1++; + } + + atomic_dec32(&during_lock_writers); + mutex_unlock(&test_mutex); + + return TEE_SUCCESS; +} + +static TEE_Result mutex_test_reader(TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + size_t n; + + params[1].value.a = atomic_inc32(&before_lock_readers); + + mutex_read_lock(&test_mutex); + + atomic_dec32(&before_lock_readers); + + params[1].value.b = atomic_inc32(&during_lock_readers); + + for (n = 0; n < params[0].value.b; n++) { + if (val0 * 2 != val1) + res = TEE_ERROR_BAD_STATE; + } + + atomic_dec32(&during_lock_readers); + mutex_read_unlock(&test_mutex); + + return res; +} + +TEE_Result core_mutex_tests(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) { + DMSG("bad parameter types"); + return TEE_ERROR_BAD_PARAMETERS; + } + + switch (params[0].value.a) { + case PTA_MUTEX_TEST_WRITER: + return mutex_test_writer(params); + case PTA_MUTEX_TEST_READER: + return mutex_test_reader(params); + default: + return TEE_ERROR_BAD_PARAMETERS; + } +} diff --git a/optee_os/core/pta/tests/sub.mk b/optee_os/core/pta/tests/sub.mk new file mode 100644 index 0000000..cb3c29e --- /dev/null +++ b/optee_os/core/pta/tests/sub.mk @@ -0,0 +1,8 @@ +srcs-$(call cfg-all-enabled,CFG_REE_FS CFG_WITH_USER_TA) += fs_htree.c +srcs-y += invoke.c +srcs-$(CFG_LOCKDEP) += lockdep.c +srcs-y += misc.c +cflags-misc.c-y += -fno-builtin +srcs-y += mutex.c +srcs-y += aes_perf.c +srcs-$(CFG_DT_DRIVER_EMBEDDED_TEST) += dt_driver_test.c diff --git a/optee_os/core/sub.mk b/optee_os/core/sub.mk new file mode 100644 index 0000000..958524f --- /dev/null +++ b/optee_os/core/sub.mk @@ -0,0 +1,90 @@ +subdirs-y += crypto +subdirs-y += drivers +subdirs-y += kernel +subdirs-y += mm +subdirs-y += pta +subdirs-y += tee +subdirs-$(CFG_TEE_CORE_EMBED_INTERNAL_TESTS) += tests + +ifeq ($(CFG_WITH_USER_TA),y) +gensrcs-y += ta_pub_key +produce-ta_pub_key = ta_pub_key.c +depends-ta_pub_key = $(TA_PUBLIC_KEY) scripts/pem_to_pub_c.py +recipe-ta_pub_key = $(PYTHON3) scripts/pem_to_pub_c.py --prefix ta_pub_key \ + --key $(TA_PUBLIC_KEY) --out $(sub-dir-out)/ta_pub_key.c + +gensrcs-y += ldelf +produce-ldelf = ldelf_hex.c +depends-ldelf = scripts/gen_ldelf_hex.py $(out-dir)/ldelf/ldelf.elf +recipe-ldelf = $(PYTHON3) scripts/gen_ldelf_hex.py --input $(out-dir)/ldelf/ldelf.elf \ + --output $(sub-dir-out)/ldelf_hex.c +endif + +ifeq ($(CFG_WITH_USER_TA)-$(CFG_EARLY_TA),y-y) +ifeq ($(CFG_EARLY_TA_COMPRESS),y) +early-ta-compress = --compress +endif +define process_early_ta +early-ta-$1-uuid := $(firstword $(subst ., ,$(notdir $1))) +gensrcs-y += early-ta-$1 +produce-early-ta-$1 = early_ta_$$(early-ta-$1-uuid).c +depends-early-ta-$1 = $1 scripts/ts_bin_to_c.py +recipe-early-ta-$1 = $(PYTHON3) scripts/ts_bin_to_c.py $(early-ta-compress) \ + --ta $1 --out $(sub-dir-out)/early_ta_$$(early-ta-$1-uuid).c +endef +$(foreach f, $(EARLY_TA_PATHS), $(eval $(call process_early_ta,$(f)))) +$(foreach f, $(CFG_IN_TREE_EARLY_TAS), $(eval $(call \ + process_early_ta,$(out-dir)/ta/$(f).stripped.elf))) +endif + +define process_secure_partition +sp-$1-uuid := $(firstword $(subst ., ,$(notdir $1))) +gensrcs-y += sp-$1 +produce-sp-$1 = sp_$$(sp-$1-uuid).c +depends-sp-$1 = $1 scripts/ts_bin_to_c.py +dtb-$1-path = $(dir $1) +dtb-$1 = $$(dtb-$1-path)../manifest/$$(sp-$1-uuid).dtb +recipe-sp-$1 = $(PYTHON3) scripts/ts_bin_to_c.py --compress --sp $1 \ + --out $(sub-dir-out)/sp_$$(sp-$1-uuid).c \ + --manifest $$(dtb-$1) +endef +$(foreach f, $(SP_PATHS), $(eval $(call process_secure_partition,$(f)))) + +ifeq ($(CFG_EMBED_DTB),y) +core-embed-fdt-dts = $(arch-dir)/dts/$(CFG_EMBED_DTB_SOURCE_FILE) +core-embed-fdt-dtb = $(out-dir)/$(arch-dir)/dts/$(CFG_EMBED_DTB_SOURCE_FILE:.dts=.dtb) +core-embed-fdt-c = $(out-dir)/$(arch-dir)/dts/$(CFG_EMBED_DTB_SOURCE_FILE:.dts=.c) +gensrcs-y += embedded_secure_dtb +produce-embedded_secure_dtb = arch/$(ARCH)/dts/$(CFG_EMBED_DTB_SOURCE_FILE:.dts=.c) +depends-embedded_secure_dtb = $(core-embed-fdt-dtb) scripts/bin_to_c.py +recipe-embedded_secure_dtb = $(PYTHON3) scripts/bin_to_c.py \ + --bin $(core-embed-fdt-dtb) \ + --vname embedded_secure_dtb \ + --out $(core-embed-fdt-c) +$(eval $(call gen-dtb-file,$(core-embed-fdt-dts),$(core-embed-fdt-dtb))) +endif + +ifeq ($(CFG_SHOW_CONF_ON_BOOT),y) +conf-mk-xz-base64 := $(sub-dir-out)/conf.mk.xz.base64 +cleanfiles += $(conf-mk-xz-base64) + +$(conf-mk-xz-base64): $(conf-mk-file) + @$(cmd-echo-silent) ' GEN $@' + $(q)tail +3 $< | xz | base64 -w 100 >$@ + +gensrcs-y += conf_str +produce-conf_str = conf.mk.xz.base64.c +depends-conf_str = $(conf-mk-xz-base64) +recipe-conf_str = $(PYTHON3) scripts/bin_to_c.py --text --bin $(conf-mk-xz-base64) \ + --out $(sub-dir-out)/conf.mk.xz.base64.c \ + --vname conf_str +endif + +ifneq ($(CFG_STMM_PATH),) +gensrcs-y += stmm +produce-stmm = stmm_hex.c +depends-stmm = scripts/gen_stmm_hex.py $(CFG_STMM_PATH) +recipe-stmm = scripts/gen_stmm_hex.py --input $(CFG_STMM_PATH) \ + --output $(sub-dir-out)/stmm_hex.c +cleanfiles += $(sub-dir-out)/stmm_hex.c +endif diff --git a/optee_os/core/tee/entry_std.c b/optee_os/core/tee/entry_std.c new file mode 100644 index 0000000..d3ae624 --- /dev/null +++ b/optee_os/core/tee/entry_std.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015-2016, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SHM_CACHE_ATTRS \ + (uint32_t)(core_mmu_is_shm_cached() ? \ + TEE_MATTR_MEM_TYPE_CACHED : TEE_MATTR_MEM_TYPE_DEV) + +/* Sessions opened from normal world */ +static struct tee_ta_session_head tee_open_sessions = +TAILQ_HEAD_INITIALIZER(tee_open_sessions); + +#ifdef CFG_CORE_RESERVED_SHM +static struct mobj *shm_mobj; +#endif +#ifdef CFG_SECURE_DATA_PATH +static struct mobj **sdp_mem_mobjs; +#endif + +static unsigned int session_pnum; + +static bool __maybe_unused param_mem_from_mobj(struct param_mem *mem, + struct mobj *mobj, + const paddr_t pa, + const size_t sz) +{ + paddr_t b; + + if (mobj_get_pa(mobj, 0, 0, &b) != TEE_SUCCESS) + panic("mobj_get_pa failed"); + + if (!core_is_buffer_inside(pa, MAX(sz, 1UL), b, mobj->size)) + return false; + + mem->mobj = mobj_get(mobj); + mem->offs = pa - b; + mem->size = sz; + return true; +} + +#ifdef CFG_CORE_FFA +static TEE_Result set_fmem_param(const struct optee_msg_param_fmem *fmem, + struct param_mem *mem) +{ + size_t req_size = 0; + uint64_t global_id = READ_ONCE(fmem->global_id); + size_t sz = READ_ONCE(fmem->size); + + if (global_id == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID && !sz) { + mem->mobj = NULL; + mem->offs = 0; + mem->size = 0; + return TEE_SUCCESS; + } + mem->mobj = mobj_ffa_get_by_cookie(global_id, + READ_ONCE(fmem->internal_offs)); + if (!mem->mobj) + return TEE_ERROR_BAD_PARAMETERS; + + mem->offs = reg_pair_to_64(READ_ONCE(fmem->offs_high), + READ_ONCE(fmem->offs_low)); + mem->size = sz; + + /* + * Check that the supplied offset and size is covered by the + * previously verified MOBJ. + */ + if (ADD_OVERFLOW(mem->offs, mem->size, &req_size) || + mem->mobj->size < req_size) + return TEE_ERROR_SECURITY; + + return TEE_SUCCESS; +} +#else /*!CFG_CORE_FFA*/ +/* fill 'struct param_mem' structure if buffer matches a valid memory object */ +static TEE_Result set_tmem_param(const struct optee_msg_param_tmem *tmem, + uint32_t attr, struct param_mem *mem) +{ + struct mobj __maybe_unused **mobj; + paddr_t pa = READ_ONCE(tmem->buf_ptr); + size_t sz = READ_ONCE(tmem->size); + + /* + * Handle NULL memory reference + */ + if (!pa) { + mem->mobj = NULL; + mem->offs = 0; + mem->size = 0; + return TEE_SUCCESS; + } + + /* Handle non-contiguous reference from a shared memory area */ + if (attr & OPTEE_MSG_ATTR_NONCONTIG) { + uint64_t shm_ref = READ_ONCE(tmem->shm_ref); + + mem->mobj = msg_param_mobj_from_noncontig(pa, sz, shm_ref, + false); + if (!mem->mobj) + return TEE_ERROR_BAD_PARAMETERS; + mem->offs = 0; + mem->size = sz; + return TEE_SUCCESS; + } + +#ifdef CFG_CORE_RESERVED_SHM + /* Handle memory reference in the contiguous shared memory */ + if (param_mem_from_mobj(mem, shm_mobj, pa, sz)) + return TEE_SUCCESS; +#endif + +#ifdef CFG_SECURE_DATA_PATH + /* Handle memory reference to Secure Data Path memory areas */ + for (mobj = sdp_mem_mobjs; *mobj; mobj++) + if (param_mem_from_mobj(mem, *mobj, pa, sz)) + return TEE_SUCCESS; +#endif + + return TEE_ERROR_BAD_PARAMETERS; +} + +#ifdef CFG_CORE_DYN_SHM +static TEE_Result set_rmem_param(const struct optee_msg_param_rmem *rmem, + struct param_mem *mem) +{ + size_t req_size = 0; + uint64_t shm_ref = READ_ONCE(rmem->shm_ref); + size_t sz = READ_ONCE(rmem->size); + + mem->mobj = mobj_reg_shm_get_by_cookie(shm_ref); + if (!mem->mobj) + return TEE_ERROR_BAD_PARAMETERS; + + mem->offs = READ_ONCE(rmem->offs); + mem->size = sz; + + /* + * Check that the supplied offset and size is covered by the + * previously verified MOBJ. + */ + if (ADD_OVERFLOW(mem->offs, mem->size, &req_size) || + mem->mobj->size < req_size) + return TEE_ERROR_SECURITY; + + return TEE_SUCCESS; +} +#endif /*CFG_CORE_DYN_SHM*/ +#endif /*!CFG_CORE_FFA*/ + +static TEE_Result copy_in_params(const struct optee_msg_param *params, + uint32_t num_params, + struct tee_ta_param *ta_param, + uint64_t *saved_attr) +{ + TEE_Result res; + size_t n; + uint8_t pt[TEE_NUM_PARAMS] = { 0 }; + + if (num_params > TEE_NUM_PARAMS) + return TEE_ERROR_BAD_PARAMETERS; + + memset(ta_param, 0, sizeof(*ta_param)); + + for (n = 0; n < num_params; n++) { + uint32_t attr; + + saved_attr[n] = READ_ONCE(params[n].attr); + + if (saved_attr[n] & OPTEE_MSG_ATTR_META) + return TEE_ERROR_BAD_PARAMETERS; + + attr = saved_attr[n] & OPTEE_MSG_ATTR_TYPE_MASK; + switch (attr) { + case OPTEE_MSG_ATTR_TYPE_NONE: + pt[n] = TEE_PARAM_TYPE_NONE; + break; + case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT: + case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT: + pt[n] = TEE_PARAM_TYPE_VALUE_INPUT + attr - + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; + ta_param->u[n].val.a = READ_ONCE(params[n].u.value.a); + ta_param->u[n].val.b = READ_ONCE(params[n].u.value.b); + break; +#ifdef CFG_CORE_FFA + case OPTEE_MSG_ATTR_TYPE_FMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_FMEM_INOUT: + res = set_fmem_param(¶ms[n].u.fmem, + &ta_param->u[n].mem); + if (res) + return res; + pt[n] = TEE_PARAM_TYPE_MEMREF_INPUT + attr - + OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; + break; +#else /*!CFG_CORE_FFA*/ + case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: + res = set_tmem_param(¶ms[n].u.tmem, saved_attr[n], + &ta_param->u[n].mem); + if (res) + return res; + pt[n] = TEE_PARAM_TYPE_MEMREF_INPUT + attr - + OPTEE_MSG_ATTR_TYPE_TMEM_INPUT; + break; +#ifdef CFG_CORE_DYN_SHM + case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT: + res = set_rmem_param(¶ms[n].u.rmem, + &ta_param->u[n].mem); + if (res) + return res; + pt[n] = TEE_PARAM_TYPE_MEMREF_INPUT + attr - + OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; + break; +#endif /*CFG_CORE_DYN_SHM*/ +#endif /*!CFG_CORE_FFA*/ + default: + return TEE_ERROR_BAD_PARAMETERS; + } + } + + ta_param->types = TEE_PARAM_TYPES(pt[0], pt[1], pt[2], pt[3]); + + return TEE_SUCCESS; +} + +static void cleanup_shm_refs(const uint64_t *saved_attr, + struct tee_ta_param *param, uint32_t num_params) +{ + size_t n; + + for (n = 0; n < MIN((unsigned int)TEE_NUM_PARAMS, num_params); n++) { + switch (saved_attr[n]) { + case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: +#ifdef CFG_CORE_DYN_SHM + case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT: +#endif + mobj_put(param->u[n].mem.mobj); + break; + default: + break; + } + } +} + +static void copy_out_param(struct tee_ta_param *ta_param, uint32_t num_params, + struct optee_msg_param *params, uint64_t *saved_attr) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + switch (TEE_PARAM_TYPE_GET(ta_param->types, n)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + switch (saved_attr[n] & OPTEE_MSG_ATTR_TYPE_MASK) { + case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: + params[n].u.tmem.size = ta_param->u[n].mem.size; + break; + case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT: + params[n].u.rmem.size = ta_param->u[n].mem.size; + break; + default: + break; + } + break; + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + params[n].u.value.a = ta_param->u[n].val.a; + params[n].u.value.b = ta_param->u[n].val.b; + break; + default: + break; + } + } +} + +/* + * Extracts mandatory parameter for open session. + * + * Returns + * false : mandatory parameter wasn't found or malformatted + * true : paramater found and OK + */ +static TEE_Result get_open_session_meta(size_t num_params, + struct optee_msg_param *params, + size_t *num_meta, TEE_UUID *uuid, + TEE_Identity *clnt_id) +{ + const uint32_t req_attr = OPTEE_MSG_ATTR_META | + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; + + if (num_params < 2) + return TEE_ERROR_BAD_PARAMETERS; + + if (params[0].attr != req_attr || params[1].attr != req_attr) + return TEE_ERROR_BAD_PARAMETERS; + + tee_uuid_from_octets(uuid, (void *)¶ms[0].u.value); + clnt_id->login = params[1].u.value.c; + switch (clnt_id->login) { + case TEE_LOGIN_PUBLIC: + case TEE_LOGIN_REE_KERNEL: + memset(&clnt_id->uuid, 0, sizeof(clnt_id->uuid)); + break; + case TEE_LOGIN_USER: + case TEE_LOGIN_GROUP: + case TEE_LOGIN_APPLICATION: + case TEE_LOGIN_APPLICATION_USER: + case TEE_LOGIN_APPLICATION_GROUP: + tee_uuid_from_octets(&clnt_id->uuid, + (void *)¶ms[1].u.value); + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + *num_meta = 2; + return TEE_SUCCESS; +} + +static void entry_open_session(struct optee_msg_arg *arg, uint32_t num_params) +{ + TEE_Result res = TEE_ERROR_GENERIC; + TEE_ErrorOrigin err_orig = TEE_ORIGIN_TEE; + struct tee_ta_session *s = NULL; + TEE_Identity clnt_id = { }; + TEE_UUID uuid = { }; + struct tee_ta_param param = { }; + size_t num_meta = 0; + uint64_t saved_attr[TEE_NUM_PARAMS] = { 0 }; + + res = get_open_session_meta(num_params, arg->params, &num_meta, &uuid, + &clnt_id); + if (res != TEE_SUCCESS) + goto out; + + res = copy_in_params(arg->params + num_meta, num_params - num_meta, + ¶m, saved_attr); + if (res != TEE_SUCCESS) + goto cleanup_shm_refs; + + res = tee_ta_open_session(&err_orig, &s, &tee_open_sessions, &uuid, + &clnt_id, TEE_TIMEOUT_INFINITE, ¶m); + if (res != TEE_SUCCESS) + s = NULL; + copy_out_param(¶m, num_params - num_meta, arg->params + num_meta, + saved_attr); + + /* + * The occurrence of open/close session command is usually + * un-predictable, using this property to increase randomness + * of prng + */ + plat_prng_add_jitter_entropy(CRYPTO_RNG_SRC_JITTER_SESSION, + &session_pnum); + +cleanup_shm_refs: + cleanup_shm_refs(saved_attr, ¶m, num_params - num_meta); + +out: + if (s) + arg->session = s->id; + else + arg->session = 0; + arg->ret = res; + arg->ret_origin = err_orig; +} + +static void entry_close_session(struct optee_msg_arg *arg, uint32_t num_params) +{ + TEE_Result res; + struct tee_ta_session *s; + + if (num_params) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + plat_prng_add_jitter_entropy(CRYPTO_RNG_SRC_JITTER_SESSION, + &session_pnum); + + s = tee_ta_find_session(arg->session, &tee_open_sessions); + res = tee_ta_close_session(s, &tee_open_sessions, NSAPP_IDENTITY); +out: + arg->ret = res; + arg->ret_origin = TEE_ORIGIN_TEE; +} + +static void entry_invoke_command(struct optee_msg_arg *arg, uint32_t num_params) +{ + TEE_Result res; + TEE_ErrorOrigin err_orig = TEE_ORIGIN_TEE; + struct tee_ta_session *s; + struct tee_ta_param param = { 0 }; + uint64_t saved_attr[TEE_NUM_PARAMS] = { 0 }; + + bm_timestamp(); + + res = copy_in_params(arg->params, num_params, ¶m, saved_attr); + if (res != TEE_SUCCESS) + goto out; + + s = tee_ta_get_session(arg->session, true, &tee_open_sessions); + if (!s) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = tee_ta_invoke_command(&err_orig, s, NSAPP_IDENTITY, + TEE_TIMEOUT_INFINITE, arg->func, ¶m); + + bm_timestamp(); + + tee_ta_put_session(s); + + copy_out_param(¶m, num_params, arg->params, saved_attr); + +out: + cleanup_shm_refs(saved_attr, ¶m, num_params); + + arg->ret = res; + arg->ret_origin = err_orig; +} + +static void entry_cancel(struct optee_msg_arg *arg, uint32_t num_params) +{ + TEE_Result res; + TEE_ErrorOrigin err_orig = TEE_ORIGIN_TEE; + struct tee_ta_session *s; + + if (num_params) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + s = tee_ta_get_session(arg->session, false, &tee_open_sessions); + if (!s) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = tee_ta_cancel_command(&err_orig, s, NSAPP_IDENTITY); + tee_ta_put_session(s); + +out: + arg->ret = res; + arg->ret_origin = err_orig; +} + +#ifndef CFG_CORE_FFA +#ifdef CFG_CORE_DYN_SHM +static void register_shm(struct optee_msg_arg *arg, uint32_t num_params) +{ + struct optee_msg_param_tmem *tmem = NULL; + struct mobj *mobj = NULL; + + arg->ret = TEE_ERROR_BAD_PARAMETERS; + + if (num_params != 1 || + (arg->params[0].attr != + (OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | OPTEE_MSG_ATTR_NONCONTIG))) + return; + + tmem = &arg->params[0].u.tmem; + mobj = msg_param_mobj_from_noncontig(tmem->buf_ptr, tmem->size, + tmem->shm_ref, false); + + if (!mobj) + return; + + mobj_reg_shm_unguard(mobj); + arg->ret = TEE_SUCCESS; +} + +static void unregister_shm(struct optee_msg_arg *arg, uint32_t num_params) +{ + if (num_params == 1) { + uint64_t cookie = arg->params[0].u.rmem.shm_ref; + TEE_Result res = mobj_reg_shm_release_by_cookie(cookie); + + if (res) + EMSG("Can't find mapping with given cookie"); + arg->ret = res; + } else { + arg->ret = TEE_ERROR_BAD_PARAMETERS; + arg->ret_origin = TEE_ORIGIN_TEE; + } +} +#endif /*CFG_CORE_DYN_SHM*/ +#endif + +void nsec_sessions_list_head(struct tee_ta_session_head **open_sessions) +{ + *open_sessions = &tee_open_sessions; +} + +/* Note: this function is weak to let platforms add special handling */ +TEE_Result __weak tee_entry_std(struct optee_msg_arg *arg, uint32_t num_params) +{ + return __tee_entry_std(arg, num_params); +} + +/* + * If tee_entry_std() is overridden, it's still supposed to call this + * function. + */ +TEE_Result __tee_entry_std(struct optee_msg_arg *arg, uint32_t num_params) +{ + TEE_Result res = TEE_SUCCESS; + + /* Enable foreign interrupts for STD calls */ + thread_set_foreign_intr(true); + switch (arg->cmd) { + case OPTEE_MSG_CMD_OPEN_SESSION: + entry_open_session(arg, num_params); + break; + case OPTEE_MSG_CMD_CLOSE_SESSION: + entry_close_session(arg, num_params); + break; + case OPTEE_MSG_CMD_INVOKE_COMMAND: + entry_invoke_command(arg, num_params); + break; + case OPTEE_MSG_CMD_CANCEL: + entry_cancel(arg, num_params); + break; +#ifndef CFG_CORE_FFA +#ifdef CFG_CORE_DYN_SHM + case OPTEE_MSG_CMD_REGISTER_SHM: + register_shm(arg, num_params); + break; + case OPTEE_MSG_CMD_UNREGISTER_SHM: + unregister_shm(arg, num_params); + break; +#endif +#endif + + case OPTEE_MSG_CMD_DO_BOTTOM_HALF: + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) + notif_deliver_event(NOTIF_EVENT_DO_BOTTOM_HALF); + else + goto err; + break; + case OPTEE_MSG_CMD_STOP_ASYNC_NOTIF: + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) + notif_deliver_event(NOTIF_EVENT_STOPPED); + else + goto err; + break; + + default: +err: + EMSG("Unknown cmd 0x%x", arg->cmd); + res = TEE_ERROR_NOT_IMPLEMENTED; + } + + return res; +} + +static TEE_Result default_mobj_init(void) +{ +#ifdef CFG_CORE_RESERVED_SHM + shm_mobj = mobj_phys_alloc(default_nsec_shm_paddr, + default_nsec_shm_size, SHM_CACHE_ATTRS, + CORE_MEM_NSEC_SHM); + if (!shm_mobj) + panic("Failed to register shared memory"); +#endif + +#ifdef CFG_SECURE_DATA_PATH + sdp_mem_mobjs = core_sdp_mem_create_mobjs(); + if (!sdp_mem_mobjs) + panic("Failed to register SDP memory"); +#endif + + return TEE_SUCCESS; +} + +driver_init_late(default_mobj_init); diff --git a/optee_os/core/tee/fs_dirfile.c b/optee_os/core/tee/fs_dirfile.c new file mode 100644 index 0000000..0cf0fad --- /dev/null +++ b/optee_os/core/tee/fs_dirfile.c @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +struct tee_fs_dirfile_dirh { + const struct tee_fs_dirfile_operations *fops; + struct tee_file_handle *fh; + int nbits; + bitstr_t *files; + size_t ndents; +}; + +struct dirfile_entry { + TEE_UUID uuid; + uint8_t oid[TEE_OBJECT_ID_MAX_LEN]; + uint32_t oidlen; + uint8_t hash[TEE_FS_HTREE_HASH_SIZE]; + uint32_t file_number; +}; + +#define OID_EMPTY_NAME 1 + +/* + * An object can have an ID of size zero. This object is represented by + * oidlen == 0 and oid[0] == OID_EMPTY_NAME. When both are zero, the entry is + * not a valid object. + */ +static bool is_free(struct dirfile_entry *dent) +{ + assert(dent->oidlen || !dent->oid[0] || dent->oid[0] == OID_EMPTY_NAME); + + return !dent->oidlen && !dent->oid[0]; +} + +/* + * File layout + * + * dirfile_entry.0 + * ... + * dirfile_entry.n + * + * where n the index is disconnected from file_number in struct dirfile_entry + */ + +static TEE_Result maybe_grow_files(struct tee_fs_dirfile_dirh *dirh, int idx) +{ + void *p; + + if (idx < dirh->nbits) + return TEE_SUCCESS; + + p = realloc(dirh->files, bitstr_size(idx + 1)); + if (!p) + return TEE_ERROR_OUT_OF_MEMORY; + dirh->files = p; + + bit_nclear(dirh->files, dirh->nbits, idx); + dirh->nbits = idx + 1; + + return TEE_SUCCESS; +} + +static TEE_Result set_file(struct tee_fs_dirfile_dirh *dirh, int idx) +{ + TEE_Result res = maybe_grow_files(dirh, idx); + + if (!res) + bit_set(dirh->files, idx); + + return res; +} + +static void clear_file(struct tee_fs_dirfile_dirh *dirh, int idx) +{ + if (idx < dirh->nbits) + bit_clear(dirh->files, idx); +} + +static bool test_file(struct tee_fs_dirfile_dirh *dirh, int idx) +{ + if (idx < dirh->nbits) + return bit_test(dirh->files, idx); + + return false; +} + +static TEE_Result read_dent(struct tee_fs_dirfile_dirh *dirh, int idx, + struct dirfile_entry *dent) +{ + TEE_Result res; + size_t l; + + l = sizeof(*dent); + res = dirh->fops->read(dirh->fh, sizeof(struct dirfile_entry) * idx, + dent, &l); + if (!res && l != sizeof(*dent)) + res = TEE_ERROR_ITEM_NOT_FOUND; + + return res; +} + +static TEE_Result write_dent(struct tee_fs_dirfile_dirh *dirh, size_t n, + struct dirfile_entry *dent) +{ + TEE_Result res; + + res = dirh->fops->write(dirh->fh, sizeof(*dent) * n, dent, + sizeof(*dent)); + if (!res && n >= dirh->ndents) + dirh->ndents = n + 1; + + return res; +} + +TEE_Result tee_fs_dirfile_open(bool create, uint8_t *hash, uint32_t min_counter, + const struct tee_fs_dirfile_operations *fops, + struct tee_fs_dirfile_dirh **dirh_ret) +{ + TEE_Result res; + struct tee_fs_dirfile_dirh *dirh = calloc(1, sizeof(*dirh)); + size_t n; + + if (!dirh) + return TEE_ERROR_OUT_OF_MEMORY; + + dirh->fops = fops; + res = fops->open(create, hash, min_counter, NULL, NULL, &dirh->fh); + if (res) + goto out; + + for (n = 0;; n++) { + struct dirfile_entry dent = { }; + + res = read_dent(dirh, n, &dent); + if (res) { + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = TEE_SUCCESS; + goto out; + } + + if (is_free(&dent)) + continue; + + if (test_file(dirh, dent.file_number)) { + DMSG("clearing duplicate file number %" PRIu32, + dent.file_number); + memset(&dent, 0, sizeof(dent)); + res = write_dent(dirh, n, &dent); + if (res) + goto out; + continue; + } + + res = set_file(dirh, dent.file_number); + if (res != TEE_SUCCESS) + goto out; + } +out: + if (!res) { + dirh->ndents = n; + *dirh_ret = dirh; + } else { + tee_fs_dirfile_close(dirh); + } + return res; +} + +void tee_fs_dirfile_close(struct tee_fs_dirfile_dirh *dirh) +{ + if (dirh) { + dirh->fops->close(dirh->fh); + free(dirh->files); + free(dirh); + } +} + +TEE_Result tee_fs_dirfile_commit_writes(struct tee_fs_dirfile_dirh *dirh, + uint8_t *hash, uint32_t *counter) +{ + return dirh->fops->commit_writes(dirh->fh, hash, counter); +} + +TEE_Result tee_fs_dirfile_get_tmp(struct tee_fs_dirfile_dirh *dirh, + struct tee_fs_dirfile_fileh *dfh) +{ + TEE_Result res; + int i = 0; + + if (dirh->nbits) { + bit_ffc(dirh->files, dirh->nbits, &i); + if (i == -1) + i = dirh->nbits; + } + + res = set_file(dirh, i); + if (!res) + dfh->file_number = i; + + return res; +} + +TEE_Result tee_fs_dirfile_find(struct tee_fs_dirfile_dirh *dirh, + const TEE_UUID *uuid, const void *oid, + size_t oidlen, struct tee_fs_dirfile_fileh *dfh) +{ + TEE_Result res = TEE_SUCCESS; + struct dirfile_entry dent = { }; + int n = 0; + + for (n = 0;; n++) { + res = read_dent(dirh, n, &dent); + if (res) + return res; + + if (is_free(&dent)) + continue; + if (dent.oidlen != oidlen) + continue; + + assert(test_file(dirh, dent.file_number)); + + if (!memcmp(&dent.uuid, uuid, sizeof(dent.uuid)) && + !memcmp(&dent.oid, oid, oidlen)) + break; + } + + if (dfh) { + dfh->idx = n; + dfh->file_number = dent.file_number; + memcpy(dfh->hash, dent.hash, sizeof(dent.hash)); + } + + return TEE_SUCCESS; +} + +static TEE_Result find_empty_idx(struct tee_fs_dirfile_dirh *dh, int *idx) +{ + struct dirfile_entry dent = { }; + TEE_Result res = TEE_SUCCESS; + int n = 0; + + for (n = 0;; n++) { + res = read_dent(dh, n, &dent); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + break; + if (res) + return res; + if (is_free(&dent)) + break; + } + + *idx = n; + return TEE_SUCCESS; +} + +TEE_Result tee_fs_dirfile_fileh_to_fname(const struct tee_fs_dirfile_fileh *dfh, + char *fname, size_t *fnlen) +{ + int r; + size_t l = *fnlen; + + if (dfh) + r = snprintf(fname, l, "%" PRIx32, dfh->file_number); + else + r = snprintf(fname, l, "dirf.db"); + + if (r < 0) + return TEE_ERROR_GENERIC; + + *fnlen = r + 1; + if ((size_t)r >= l) + return TEE_ERROR_SHORT_BUFFER; + + return TEE_SUCCESS; +} + +TEE_Result tee_fs_dirfile_rename(struct tee_fs_dirfile_dirh *dirh, + const TEE_UUID *uuid, + struct tee_fs_dirfile_fileh *dfh, + const void *oid, size_t oidlen) +{ + TEE_Result res; + struct dirfile_entry dent = { }; + + if (oidlen > sizeof(dent.oid)) + return TEE_ERROR_BAD_PARAMETERS; + memset(&dent, 0, sizeof(dent)); + dent.uuid = *uuid; + if (oidlen) + memcpy(dent.oid, oid, oidlen); + else + dent.oid[0] = OID_EMPTY_NAME; + + dent.oidlen = oidlen; + memcpy(dent.hash, dfh->hash, sizeof(dent.hash)); + dent.file_number = dfh->file_number; + + if (dfh->idx < 0) { + struct tee_fs_dirfile_fileh dfh2; + + res = tee_fs_dirfile_find(dirh, uuid, oid, oidlen, &dfh2); + if (res) { + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = find_empty_idx(dirh, &dfh2.idx); + if (res) + return res; + } + dfh->idx = dfh2.idx; + } + + return write_dent(dirh, dfh->idx, &dent); +} + +TEE_Result tee_fs_dirfile_remove(struct tee_fs_dirfile_dirh *dirh, + const struct tee_fs_dirfile_fileh *dfh) +{ + TEE_Result res; + struct dirfile_entry dent = { }; + uint32_t file_number; + + res = read_dent(dirh, dfh->idx, &dent); + if (res) + return res; + + if (is_free(&dent)) + return TEE_SUCCESS; + + file_number = dent.file_number; + assert(dfh->file_number == file_number); + assert(test_file(dirh, file_number)); + + memset(&dent, 0, sizeof(dent)); + res = write_dent(dirh, dfh->idx, &dent); + if (!res) + clear_file(dirh, file_number); + + return res; +} + +TEE_Result tee_fs_dirfile_update_hash(struct tee_fs_dirfile_dirh *dirh, + const struct tee_fs_dirfile_fileh *dfh) +{ + TEE_Result res; + struct dirfile_entry dent = { }; + + res = read_dent(dirh, dfh->idx, &dent); + if (res) + return res; + assert(dent.file_number == dfh->file_number); + assert(test_file(dirh, dent.file_number)); + + memcpy(&dent.hash, dfh->hash, sizeof(dent.hash)); + + return write_dent(dirh, dfh->idx, &dent); +} + +TEE_Result tee_fs_dirfile_get_next(struct tee_fs_dirfile_dirh *dirh, + const TEE_UUID *uuid, int *idx, void *oid, + size_t *oidlen) +{ + TEE_Result res; + int i = *idx + 1; + struct dirfile_entry dent; + + if (i < 0) + i = 0; + + for (;; i++) { + res = read_dent(dirh, i, &dent); + if (res) + return res; + if (!memcmp(&dent.uuid, uuid, sizeof(dent.uuid)) && + !is_free(&dent)) + break; + } + + if (*oidlen < dent.oidlen) + return TEE_ERROR_SHORT_BUFFER; + + memcpy(oid, dent.oid, dent.oidlen); + *oidlen = dent.oidlen; + *idx = i; + + return TEE_SUCCESS; +} diff --git a/optee_os/core/tee/fs_htree.c b/optee_os/core/tee/fs_htree.c new file mode 100644 index 0000000..a65e6e6 --- /dev/null +++ b/optee_os/core/tee/fs_htree.c @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TEE_FS_HTREE_CHIP_ID_SIZE 32 +#define TEE_FS_HTREE_HASH_ALG TEE_ALG_SHA256 +#define TEE_FS_HTREE_TSK_SIZE TEE_FS_HTREE_HASH_SIZE +#define TEE_FS_HTREE_ENC_ALG TEE_ALG_AES_ECB_NOPAD +#define TEE_FS_HTREE_ENC_SIZE TEE_AES_BLOCK_SIZE +#define TEE_FS_HTREE_SSK_SIZE TEE_FS_HTREE_HASH_SIZE + +#define TEE_FS_HTREE_AUTH_ENC_ALG TEE_ALG_AES_GCM +#define TEE_FS_HTREE_HMAC_ALG TEE_ALG_HMAC_SHA256 + +#define BLOCK_NUM_TO_NODE_ID(num) ((num) + 1) + +#define NODE_ID_TO_BLOCK_NUM(id) ((id) - 1) + +/* + * The hash tree is implemented as a binary tree with the purpose to ensure + * integrity of the data in the nodes. The data in the nodes their turn + * provides both integrity and confidentiality of the data blocks. + * + * The hash tree is saved in a file as: + * +----------------------------+ + * | htree_image.0 | + * | htree_image.1 | + * +----------------------------+ + * | htree_node_image.1.0 | + * | htree_node_image.1.1 | + * +----------------------------+ + * | htree_node_image.2.0 | + * | htree_node_image.2.1 | + * +----------------------------+ + * | htree_node_image.3.0 | + * | htree_node_image.3.1 | + * +----------------------------+ + * | htree_node_image.4.0 | + * | htree_node_image.4.1 | + * +----------------------------+ + * ... + * + * htree_image is the header of the file, there's two instances of it. One + * which is committed and the other is used when updating the file. Which + * is committed is indicated by the "counter" field, the one with the + * largest value is selected. + * + * htree_node_image is a node in the hash tree, each node has two instances + * which is committed is decided by the parent node .flag bit + * HTREE_NODE_COMMITTED_CHILD. Which version is the committed version of + * node 1 is determined by the by the lowest bit of the counter field in + * the header. + * + * Note that nodes start counting at 1 while blocks at 0, this means that + * block 0 is represented by node 1. + * + * Where different elements are stored in the file is managed by the file + * system. + */ + +#define HTREE_NODE_COMMITTED_BLOCK BIT32(0) +/* n is 0 or 1 */ +#define HTREE_NODE_COMMITTED_CHILD(n) BIT32(1 + (n)) + +struct htree_node { + size_t id; + bool dirty; + bool block_updated; + struct tee_fs_htree_node_image node; + struct htree_node *parent; + struct htree_node *child[2]; +}; + +struct tee_fs_htree { + struct htree_node root; + struct tee_fs_htree_image head; + uint8_t fek[TEE_FS_HTREE_FEK_SIZE]; + struct tee_fs_htree_imeta imeta; + bool dirty; + const TEE_UUID *uuid; + const struct tee_fs_htree_storage *stor; + void *stor_aux; +}; + +struct traverse_arg; +typedef TEE_Result (*traverse_cb_t)(struct traverse_arg *targ, + struct htree_node *node); +struct traverse_arg { + struct tee_fs_htree *ht; + traverse_cb_t cb; + void *arg; +}; + +static TEE_Result rpc_read(struct tee_fs_htree *ht, enum tee_fs_htree_type type, + size_t idx, size_t vers, void *data, size_t dlen) +{ + TEE_Result res; + struct tee_fs_rpc_operation op; + size_t bytes; + void *p; + + res = ht->stor->rpc_read_init(ht->stor_aux, &op, type, idx, vers, &p); + if (res != TEE_SUCCESS) + return res; + + res = ht->stor->rpc_read_final(&op, &bytes); + if (res != TEE_SUCCESS) + return res; + + if (bytes != dlen) + return TEE_ERROR_CORRUPT_OBJECT; + + memcpy(data, p, dlen); + return TEE_SUCCESS; +} + +static TEE_Result rpc_read_head(struct tee_fs_htree *ht, size_t vers, + struct tee_fs_htree_image *head) +{ + return rpc_read(ht, TEE_FS_HTREE_TYPE_HEAD, 0, vers, + head, sizeof(*head)); +} + +static TEE_Result rpc_read_node(struct tee_fs_htree *ht, size_t node_id, + size_t vers, + struct tee_fs_htree_node_image *node) +{ + return rpc_read(ht, TEE_FS_HTREE_TYPE_NODE, node_id - 1, vers, + node, sizeof(*node)); +} + +static TEE_Result rpc_write(struct tee_fs_htree *ht, + enum tee_fs_htree_type type, size_t idx, + size_t vers, const void *data, size_t dlen) +{ + TEE_Result res; + struct tee_fs_rpc_operation op; + void *p; + + res = ht->stor->rpc_write_init(ht->stor_aux, &op, type, idx, vers, &p); + if (res != TEE_SUCCESS) + return res; + + memcpy(p, data, dlen); + return ht->stor->rpc_write_final(&op); +} + +static TEE_Result rpc_write_head(struct tee_fs_htree *ht, size_t vers, + const struct tee_fs_htree_image *head) +{ + return rpc_write(ht, TEE_FS_HTREE_TYPE_HEAD, 0, vers, + head, sizeof(*head)); +} + +static TEE_Result rpc_write_node(struct tee_fs_htree *ht, size_t node_id, + size_t vers, + const struct tee_fs_htree_node_image *node) +{ + return rpc_write(ht, TEE_FS_HTREE_TYPE_NODE, node_id - 1, vers, + node, sizeof(*node)); +} + +static TEE_Result traverse_post_order(struct traverse_arg *targ, + struct htree_node *node) +{ + TEE_Result res; + + /* + * This function is recursing but not very deep, only with Log(N) + * maximum depth. + */ + + if (!node) + return TEE_SUCCESS; + + res = traverse_post_order(targ, node->child[0]); + if (res != TEE_SUCCESS) + return res; + + res = traverse_post_order(targ, node->child[1]); + if (res != TEE_SUCCESS) + return res; + + return targ->cb(targ, node); +} + +static TEE_Result htree_traverse_post_order(struct tee_fs_htree *ht, + traverse_cb_t cb, void *arg) +{ + struct traverse_arg targ = { ht, cb, arg }; + + return traverse_post_order(&targ, &ht->root); +} + +static size_t node_id_to_level(size_t node_id) +{ + assert(node_id && node_id < UINT_MAX); + /* Calculate level of the node, root node (1) has level 1 */ + return sizeof(unsigned int) * 8 - __builtin_clz(node_id); +} + +static struct htree_node *find_closest_node(struct tee_fs_htree *ht, + size_t node_id) +{ + struct htree_node *node = &ht->root; + size_t level = node_id_to_level(node_id); + size_t n; + + /* n = 1 because root node is level 1 */ + for (n = 1; n < level; n++) { + struct htree_node *child; + size_t bit_idx; + + /* + * The difference between levels of the current node and + * the node we're looking for tells which bit decides + * direction in the tree. + * + * As the first bit has index 0 we'll subtract 1 + */ + bit_idx = level - n - 1; + child = node->child[((node_id >> bit_idx) & 1)]; + if (!child) + return node; + node = child; + } + + return node; +} + +static struct htree_node *find_node(struct tee_fs_htree *ht, size_t node_id) +{ + struct htree_node *node = find_closest_node(ht, node_id); + + if (node && node->id == node_id) + return node; + return NULL; +} + +static TEE_Result get_node(struct tee_fs_htree *ht, bool create, + size_t node_id, struct htree_node **node_ret) +{ + struct htree_node *node; + struct htree_node *nc; + size_t n; + + node = find_closest_node(ht, node_id); + if (!node) + return TEE_ERROR_GENERIC; + if (node->id == node_id) + goto ret_node; + + /* + * Trying to read beyond end of file should be caught earlier than + * here. + */ + if (!create) + return TEE_ERROR_GENERIC; + + /* + * Add missing nodes, some nodes may already be there. When we've + * processed the range all nodes up to node_id will be in the tree. + */ + for (n = node->id + 1; n <= node_id; n++) { + node = find_closest_node(ht, n); + if (node->id == n) + continue; + /* Node id n should be a child of node */ + assert((n >> 1) == node->id); + assert(!node->child[n & 1]); + + nc = calloc(1, sizeof(*nc)); + if (!nc) + return TEE_ERROR_OUT_OF_MEMORY; + nc->id = n; + nc->parent = node; + node->child[n & 1] = nc; + node = nc; + } + + if (node->id > ht->imeta.max_node_id) + ht->imeta.max_node_id = node->id; + +ret_node: + *node_ret = node; + return TEE_SUCCESS; +} + +static int get_idx_from_counter(uint32_t counter0, uint32_t counter1) +{ + if (!(counter0 & 1)) { + if (!(counter1 & 1)) + return 0; + if (counter0 > counter1) + return 0; + else + return 1; + } + + if (counter1 & 1) + return 1; + else + return -1; +} + +static TEE_Result init_head_from_data(struct tee_fs_htree *ht, + const uint8_t *hash, uint32_t min_counter) +{ + TEE_Result res; + int idx; + + if (hash) { + for (idx = 0;; idx++) { + res = rpc_read_node(ht, 1, idx, &ht->root.node); + if (res != TEE_SUCCESS) + return res; + + if (!memcmp(ht->root.node.hash, hash, + sizeof(ht->root.node.hash))) { + res = rpc_read_head(ht, idx, &ht->head); + if (res != TEE_SUCCESS) + return res; + break; + } + + if (idx) + return TEE_ERROR_CORRUPT_OBJECT; + } + } else { + struct tee_fs_htree_image head[2]; + + for (idx = 0; idx < 2; idx++) { + res = rpc_read_head(ht, idx, head + idx); + if (res != TEE_SUCCESS) + return res; + } + + idx = get_idx_from_counter(head[0].counter, head[1].counter); + if (idx < 0) + return TEE_ERROR_SECURITY; + + res = rpc_read_node(ht, 1, idx, &ht->root.node); + if (res != TEE_SUCCESS) + return res; + + ht->head = head[idx]; + } + + if (ht->head.counter < min_counter) + return TEE_ERROR_SECURITY; + + ht->root.id = 1; + + return TEE_SUCCESS; +} + +static TEE_Result init_tree_from_data(struct tee_fs_htree *ht) +{ + TEE_Result res; + struct tee_fs_htree_node_image node_image; + struct htree_node *node; + struct htree_node *nc; + size_t committed_version; + size_t node_id = 2; + + while (node_id <= ht->imeta.max_node_id) { + node = find_node(ht, node_id >> 1); + if (!node) + return TEE_ERROR_GENERIC; + committed_version = !!(node->node.flags & + HTREE_NODE_COMMITTED_CHILD(node_id & 1)); + + res = rpc_read_node(ht, node_id, committed_version, + &node_image); + if (res != TEE_SUCCESS) + return res; + + res = get_node(ht, true, node_id, &nc); + if (res != TEE_SUCCESS) + return res; + nc->node = node_image; + node_id++; + } + + return TEE_SUCCESS; +} + +static TEE_Result calc_node_hash(struct htree_node *node, + struct tee_fs_htree_meta *meta, void *ctx, + uint8_t *digest) +{ + TEE_Result res; + uint8_t *ndata = (uint8_t *)&node->node + sizeof(node->node.hash); + size_t nsize = sizeof(node->node) - sizeof(node->node.hash); + + res = crypto_hash_init(ctx); + if (res != TEE_SUCCESS) + return res; + + res = crypto_hash_update(ctx, ndata, nsize); + if (res != TEE_SUCCESS) + return res; + + if (meta) { + res = crypto_hash_update(ctx, (void *)meta, sizeof(*meta)); + if (res != TEE_SUCCESS) + return res; + } + + if (node->child[0]) { + res = crypto_hash_update(ctx, node->child[0]->node.hash, + sizeof(node->child[0]->node.hash)); + if (res != TEE_SUCCESS) + return res; + } + + if (node->child[1]) { + res = crypto_hash_update(ctx, node->child[1]->node.hash, + sizeof(node->child[1]->node.hash)); + if (res != TEE_SUCCESS) + return res; + } + + return crypto_hash_final(ctx, digest, TEE_FS_HTREE_HASH_SIZE); +} + +static TEE_Result authenc_init(void **ctx_ret, TEE_OperationMode mode, + struct tee_fs_htree *ht, + struct tee_fs_htree_node_image *ni, + size_t payload_len) +{ + TEE_Result res = TEE_SUCCESS; + const uint32_t alg = TEE_FS_HTREE_AUTH_ENC_ALG; + void *ctx; + size_t aad_len = TEE_FS_HTREE_FEK_SIZE + TEE_FS_HTREE_IV_SIZE; + uint8_t *iv; + + if (ni) { + iv = ni->iv; + } else { + iv = ht->head.iv; + aad_len += TEE_FS_HTREE_HASH_SIZE + sizeof(ht->head.counter); + } + + if (mode == TEE_MODE_ENCRYPT) { + res = crypto_rng_read(iv, TEE_FS_HTREE_IV_SIZE); + if (res != TEE_SUCCESS) + return res; + } + + res = crypto_authenc_alloc_ctx(&ctx, alg); + if (res != TEE_SUCCESS) + return res; + + res = crypto_authenc_init(ctx, mode, ht->fek, TEE_FS_HTREE_FEK_SIZE, iv, + TEE_FS_HTREE_IV_SIZE, TEE_FS_HTREE_TAG_SIZE, + aad_len, payload_len); + if (res != TEE_SUCCESS) + goto err_free; + + if (!ni) { + res = crypto_authenc_update_aad(ctx, mode, ht->root.node.hash, + TEE_FS_HTREE_FEK_SIZE); + if (res != TEE_SUCCESS) + goto err; + + res = crypto_authenc_update_aad(ctx, mode, + (void *)&ht->head.counter, + sizeof(ht->head.counter)); + if (res != TEE_SUCCESS) + goto err; + } + + res = crypto_authenc_update_aad(ctx, mode, ht->head.enc_fek, + TEE_FS_HTREE_FEK_SIZE); + if (res != TEE_SUCCESS) + goto err; + + res = crypto_authenc_update_aad(ctx, mode, iv, TEE_FS_HTREE_IV_SIZE); + if (res != TEE_SUCCESS) + goto err; + + *ctx_ret = ctx; + + return TEE_SUCCESS; +err: + crypto_authenc_final(ctx); +err_free: + crypto_authenc_free_ctx(ctx); + return res; +} + +static TEE_Result authenc_decrypt_final(void *ctx, const uint8_t *tag, + const void *crypt, size_t len, + void *plain) +{ + TEE_Result res; + size_t out_size = len; + + res = crypto_authenc_dec_final(ctx, crypt, len, plain, &out_size, tag, + TEE_FS_HTREE_TAG_SIZE); + crypto_authenc_final(ctx); + crypto_authenc_free_ctx(ctx); + + if (res == TEE_SUCCESS && out_size != len) + return TEE_ERROR_GENERIC; + if (res == TEE_ERROR_MAC_INVALID) + return TEE_ERROR_CORRUPT_OBJECT; + + return res; +} + +static TEE_Result authenc_encrypt_final(void *ctx, uint8_t *tag, + const void *plain, size_t len, + void *crypt) +{ + TEE_Result res; + size_t out_size = len; + size_t out_tag_size = TEE_FS_HTREE_TAG_SIZE; + + res = crypto_authenc_enc_final(ctx, plain, len, crypt, &out_size, tag, + &out_tag_size); + crypto_authenc_final(ctx); + crypto_authenc_free_ctx(ctx); + + if (res == TEE_SUCCESS && + (out_size != len || out_tag_size != TEE_FS_HTREE_TAG_SIZE)) + return TEE_ERROR_GENERIC; + + return res; +} + +static TEE_Result verify_root(struct tee_fs_htree *ht) +{ + TEE_Result res; + void *ctx; + + res = tee_fs_fek_crypt(ht->uuid, TEE_MODE_DECRYPT, ht->head.enc_fek, + sizeof(ht->fek), ht->fek); + if (res != TEE_SUCCESS) + return res; + + res = authenc_init(&ctx, TEE_MODE_DECRYPT, ht, NULL, sizeof(ht->imeta)); + if (res != TEE_SUCCESS) + return res; + + return authenc_decrypt_final(ctx, ht->head.tag, ht->head.imeta, + sizeof(ht->imeta), &ht->imeta); +} + +static TEE_Result verify_node(struct traverse_arg *targ, + struct htree_node *node) +{ + void *ctx = targ->arg; + TEE_Result res; + uint8_t digest[TEE_FS_HTREE_HASH_SIZE]; + + if (node->parent) + res = calc_node_hash(node, NULL, ctx, digest); + else + res = calc_node_hash(node, &targ->ht->imeta.meta, ctx, digest); + if (res == TEE_SUCCESS && + consttime_memcmp(digest, node->node.hash, sizeof(digest))) + return TEE_ERROR_CORRUPT_OBJECT; + + return res; +} + +static TEE_Result verify_tree(struct tee_fs_htree *ht) +{ + TEE_Result res; + void *ctx; + + res = crypto_hash_alloc_ctx(&ctx, TEE_FS_HTREE_HASH_ALG); + if (res != TEE_SUCCESS) + return res; + + res = htree_traverse_post_order(ht, verify_node, ctx); + crypto_hash_free_ctx(ctx); + + return res; +} + +static TEE_Result init_root_node(struct tee_fs_htree *ht) +{ + TEE_Result res; + void *ctx; + + res = crypto_hash_alloc_ctx(&ctx, TEE_FS_HTREE_HASH_ALG); + if (res != TEE_SUCCESS) + return res; + + ht->root.id = 1; + ht->root.dirty = true; + + res = calc_node_hash(&ht->root, &ht->imeta.meta, ctx, + ht->root.node.hash); + crypto_hash_free_ctx(ctx); + + return res; +} + +TEE_Result tee_fs_htree_open(bool create, uint8_t *hash, uint32_t min_counter, + const TEE_UUID *uuid, + const struct tee_fs_htree_storage *stor, + void *stor_aux, struct tee_fs_htree **ht_ret) +{ + TEE_Result res; + struct tee_fs_htree *ht = calloc(1, sizeof(*ht)); + + if (!ht) + return TEE_ERROR_OUT_OF_MEMORY; + + ht->uuid = uuid; + ht->stor = stor; + ht->stor_aux = stor_aux; + + if (create) { + const struct tee_fs_htree_image dummy_head = { + .counter = min_counter, + }; + + res = crypto_rng_read(ht->fek, sizeof(ht->fek)); + if (res != TEE_SUCCESS) + goto out; + + res = tee_fs_fek_crypt(ht->uuid, TEE_MODE_ENCRYPT, ht->fek, + sizeof(ht->fek), ht->head.enc_fek); + if (res != TEE_SUCCESS) + goto out; + + res = init_root_node(ht); + if (res != TEE_SUCCESS) + goto out; + + ht->dirty = true; + res = tee_fs_htree_sync_to_storage(&ht, hash, NULL); + if (res != TEE_SUCCESS) + goto out; + res = rpc_write_head(ht, 0, &dummy_head); + } else { + res = init_head_from_data(ht, hash, min_counter); + if (res != TEE_SUCCESS) + goto out; + + res = verify_root(ht); + if (res != TEE_SUCCESS) + goto out; + + res = init_tree_from_data(ht); + if (res != TEE_SUCCESS) + goto out; + + res = verify_tree(ht); + } +out: + if (res == TEE_SUCCESS) + *ht_ret = ht; + else + tee_fs_htree_close(&ht); + return res; +} + +struct tee_fs_htree_meta *tee_fs_htree_get_meta(struct tee_fs_htree *ht) +{ + return &ht->imeta.meta; +} + +void tee_fs_htree_meta_set_dirty(struct tee_fs_htree *ht) +{ + ht->dirty = true; + ht->root.dirty = true; +} + +static TEE_Result free_node(struct traverse_arg *targ __unused, + struct htree_node *node) +{ + if (node->parent) + free(node); + return TEE_SUCCESS; +} + +void tee_fs_htree_close(struct tee_fs_htree **ht) +{ + if (!*ht) + return; + htree_traverse_post_order(*ht, free_node, NULL); + free(*ht); + *ht = NULL; +} + +static TEE_Result htree_sync_node_to_storage(struct traverse_arg *targ, + struct htree_node *node) +{ + TEE_Result res; + uint8_t vers; + struct tee_fs_htree_meta *meta = NULL; + + /* + * The node can be dirty while the block isn't updated due to + * updated children, but if block is updated the node has to be + * dirty. + */ + assert(node->dirty >= node->block_updated); + + if (!node->dirty) + return TEE_SUCCESS; + + if (node->parent) { + uint32_t f = HTREE_NODE_COMMITTED_CHILD(node->id & 1); + + node->parent->dirty = true; + node->parent->node.flags ^= f; + vers = !!(node->parent->node.flags & f); + } else { + /* + * Counter isn't updated yet, it's increased just before + * writing the header. + */ + vers = !(targ->ht->head.counter & 1); + meta = &targ->ht->imeta.meta; + } + + res = calc_node_hash(node, meta, targ->arg, node->node.hash); + if (res != TEE_SUCCESS) + return res; + + node->dirty = false; + node->block_updated = false; + + return rpc_write_node(targ->ht, node->id, vers, &node->node); +} + +static TEE_Result update_root(struct tee_fs_htree *ht) +{ + TEE_Result res; + void *ctx; + + ht->head.counter++; + + res = authenc_init(&ctx, TEE_MODE_ENCRYPT, ht, NULL, sizeof(ht->imeta)); + if (res != TEE_SUCCESS) + return res; + + return authenc_encrypt_final(ctx, ht->head.tag, &ht->imeta, + sizeof(ht->imeta), &ht->head.imeta); +} + +TEE_Result tee_fs_htree_sync_to_storage(struct tee_fs_htree **ht_arg, + uint8_t *hash, uint32_t *counter) +{ + TEE_Result res; + struct tee_fs_htree *ht = *ht_arg; + void *ctx; + + if (!ht) + return TEE_ERROR_CORRUPT_OBJECT; + + if (!ht->dirty) + return TEE_SUCCESS; + + res = crypto_hash_alloc_ctx(&ctx, TEE_FS_HTREE_HASH_ALG); + if (res != TEE_SUCCESS) + return res; + + res = htree_traverse_post_order(ht, htree_sync_node_to_storage, ctx); + if (res != TEE_SUCCESS) + goto out; + + /* All the nodes are written to storage now. Time to update root. */ + res = update_root(ht); + if (res != TEE_SUCCESS) + goto out; + + res = rpc_write_head(ht, ht->head.counter & 1, &ht->head); + if (res != TEE_SUCCESS) + goto out; + + ht->dirty = false; + if (hash) + memcpy(hash, ht->root.node.hash, sizeof(ht->root.node.hash)); + if (counter) + *counter = ht->head.counter; +out: + crypto_hash_free_ctx(ctx); + if (res != TEE_SUCCESS) + tee_fs_htree_close(ht_arg); + return res; +} + +static TEE_Result get_block_node(struct tee_fs_htree *ht, bool create, + size_t block_num, struct htree_node **node) +{ + TEE_Result res; + struct htree_node *nd; + + res = get_node(ht, create, BLOCK_NUM_TO_NODE_ID(block_num), &nd); + if (res == TEE_SUCCESS) + *node = nd; + + return res; +} + +TEE_Result tee_fs_htree_write_block(struct tee_fs_htree **ht_arg, + size_t block_num, const void *block) +{ + struct tee_fs_htree *ht = *ht_arg; + TEE_Result res; + struct tee_fs_rpc_operation op; + struct htree_node *node = NULL; + uint8_t block_vers; + void *ctx; + void *enc_block; + + if (!ht) + return TEE_ERROR_CORRUPT_OBJECT; + + res = get_block_node(ht, true, block_num, &node); + if (res != TEE_SUCCESS) + goto out; + + if (!node->block_updated) + node->node.flags ^= HTREE_NODE_COMMITTED_BLOCK; + + block_vers = !!(node->node.flags & HTREE_NODE_COMMITTED_BLOCK); + res = ht->stor->rpc_write_init(ht->stor_aux, &op, + TEE_FS_HTREE_TYPE_BLOCK, block_num, + block_vers, &enc_block); + if (res != TEE_SUCCESS) + goto out; + + res = authenc_init(&ctx, TEE_MODE_ENCRYPT, ht, &node->node, + ht->stor->block_size); + if (res != TEE_SUCCESS) + goto out; + res = authenc_encrypt_final(ctx, node->node.tag, block, + ht->stor->block_size, enc_block); + if (res != TEE_SUCCESS) + goto out; + + res = ht->stor->rpc_write_final(&op); + if (res != TEE_SUCCESS) + goto out; + + node->block_updated = true; + node->dirty = true; + ht->dirty = true; +out: + if (res != TEE_SUCCESS) + tee_fs_htree_close(ht_arg); + return res; +} + +TEE_Result tee_fs_htree_read_block(struct tee_fs_htree **ht_arg, + size_t block_num, void *block) +{ + struct tee_fs_htree *ht = *ht_arg; + TEE_Result res; + struct tee_fs_rpc_operation op; + struct htree_node *node; + uint8_t block_vers; + size_t len; + void *ctx; + void *enc_block; + + if (!ht) + return TEE_ERROR_CORRUPT_OBJECT; + + res = get_block_node(ht, false, block_num, &node); + if (res != TEE_SUCCESS) + goto out; + + block_vers = !!(node->node.flags & HTREE_NODE_COMMITTED_BLOCK); + res = ht->stor->rpc_read_init(ht->stor_aux, &op, + TEE_FS_HTREE_TYPE_BLOCK, block_num, + block_vers, &enc_block); + if (res != TEE_SUCCESS) + goto out; + + res = ht->stor->rpc_read_final(&op, &len); + if (res != TEE_SUCCESS) + goto out; + if (len != ht->stor->block_size) { + res = TEE_ERROR_CORRUPT_OBJECT; + goto out; + } + + res = authenc_init(&ctx, TEE_MODE_DECRYPT, ht, &node->node, + ht->stor->block_size); + if (res != TEE_SUCCESS) + goto out; + + res = authenc_decrypt_final(ctx, node->node.tag, enc_block, + ht->stor->block_size, block); +out: + if (res != TEE_SUCCESS) + tee_fs_htree_close(ht_arg); + return res; +} + +TEE_Result tee_fs_htree_truncate(struct tee_fs_htree **ht_arg, size_t block_num) +{ + struct tee_fs_htree *ht = *ht_arg; + size_t node_id = BLOCK_NUM_TO_NODE_ID(block_num); + struct htree_node *node; + + if (!ht) + return TEE_ERROR_CORRUPT_OBJECT; + + while (node_id < ht->imeta.max_node_id) { + node = find_closest_node(ht, ht->imeta.max_node_id); + assert(node && node->id == ht->imeta.max_node_id); + assert(!node->child[0] && !node->child[1]); + assert(node->parent); + assert(node->parent->child[node->id & 1] == node); + node->parent->child[node->id & 1] = NULL; + free(node); + ht->imeta.max_node_id--; + ht->dirty = true; + } + + return TEE_SUCCESS; +} diff --git a/optee_os/core/tee/socket.c b/optee_os/core/tee/socket.c new file mode 100644 index 0000000..16af296 --- /dev/null +++ b/optee_os/core/tee/socket.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static uint32_t get_instance_id(struct ts_session *sess) +{ + return sess->ctx->ops->get_instance_id(sess->ctx); +} + +static TEE_Result socket_open(uint32_t instance_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct thread_param tpm[4] = { }; + struct mobj *mobj = NULL; + TEE_Result res = TEE_SUCCESS; + void *va = NULL; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT); + + if (exp_pt != param_types) { + DMSG("got param_types 0x%x, expected 0x%x", + param_types, exp_pt); + return TEE_ERROR_BAD_PARAMETERS; + } + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_SOCKET, + THREAD_SHM_TYPE_APPLICATION, + params[1].memref.size, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + res = copy_from_user(va, params[1].memref.buffer, + params[1].memref.size); + if (res) + return res; + + tpm[0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_SOCKET_OPEN, instance_id, 0); + tpm[1] = THREAD_PARAM_VALUE(IN, + params[0].value.b, /* server port number */ + params[2].value.a, /* protocol */ + params[0].value.a /* ip version */); + tpm[2] = THREAD_PARAM_MEMREF(IN, mobj, 0, params[1].memref.size); + tpm[3] = THREAD_PARAM_VALUE(OUT, 0, 0, 0); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_SOCKET, 4, tpm); + if (res == TEE_SUCCESS) + params[3].value.a = tpm[3].u.value.a; + + return res; +} + +static TEE_Result socket_close(uint32_t instance_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct thread_param tpm = { }; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) { + DMSG("got param_types 0x%x, expected 0x%x", + param_types, exp_pt); + return TEE_ERROR_BAD_PARAMETERS; + } + + tpm = THREAD_PARAM_VALUE(IN, OPTEE_RPC_SOCKET_CLOSE, instance_id, + params[0].value.a); + + return thread_rpc_cmd(OPTEE_RPC_CMD_SOCKET, 1, &tpm); +} + +static TEE_Result socket_send(uint32_t instance_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct thread_param tpm[3] = { }; + struct mobj *mobj = NULL; + TEE_Result res = TEE_SUCCESS; + void *va = NULL; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) { + DMSG("got param_types 0x%x, expected 0x%x", + param_types, exp_pt); + return TEE_ERROR_BAD_PARAMETERS; + } + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_SOCKET, + THREAD_SHM_TYPE_APPLICATION, + params[1].memref.size, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + res = copy_from_user(va, params[1].memref.buffer, + params[1].memref.size); + if (res) + return res; + + tpm[0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_SOCKET_SEND, instance_id, + params[0].value.a /* handle */); + tpm[1] = THREAD_PARAM_MEMREF(IN, mobj, 0, params[1].memref.size); + tpm[2] = THREAD_PARAM_VALUE(INOUT, params[0].value.b, /* timeout */ + 0, 0); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_SOCKET, 3, tpm); + params[2].value.a = tpm[2].u.value.b; /* transmitted bytes */ + + return res; +} + +static TEE_Result socket_recv(uint32_t instance_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct thread_param tpm[3] = { }; + struct mobj *mobj = NULL; + TEE_Result res = TEE_SUCCESS; + void *va = NULL; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) { + DMSG("got param_types 0x%x, expected 0x%x", + param_types, exp_pt); + return TEE_ERROR_BAD_PARAMETERS; + } + + if (params[1].memref.size) { + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_SOCKET, + THREAD_SHM_TYPE_APPLICATION, + params[1].memref.size, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + } + + tpm[0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_SOCKET_RECV, instance_id, + params[0].value.a /* handle */); + tpm[1] = THREAD_PARAM_MEMREF(OUT, mobj, 0, params[1].memref.size); + tpm[2] = THREAD_PARAM_VALUE(IN, params[0].value.b /* timeout */, 0, 0); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_SOCKET, 3, tpm); + + if (params[1].memref.size) { + TEE_Result res2 = TEE_SUCCESS; + + res2 = copy_to_user(params[1].memref.buffer, va, + MIN(params[1].memref.size, + tpm[1].u.memref.size)); + if (res2) + return res2; + } + params[1].memref.size = tpm[1].u.memref.size; + + return res; +} + +static TEE_Result socket_ioctl(uint32_t instance_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct thread_param tpm[3] = { }; + struct mobj *mobj = NULL; + TEE_Result res = TEE_SUCCESS; + void *va = NULL; + uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + if (exp_pt != param_types) { + DMSG("got param_types 0x%x, expected 0x%x", + param_types, exp_pt); + return TEE_ERROR_BAD_PARAMETERS; + } + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_SOCKET, + THREAD_SHM_TYPE_APPLICATION, + params[1].memref.size, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + res = copy_from_user(va, params[1].memref.buffer, + params[1].memref.size); + if (res) + return res; + + tpm[0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_SOCKET_IOCTL, instance_id, + params[0].value.a /* handle */); + tpm[1] = THREAD_PARAM_MEMREF(INOUT, mobj, 0, params[1].memref.size); + tpm[2] = THREAD_PARAM_VALUE(IN, params[0].value.b /* ioctl command */, + 0, 0); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_SOCKET, 3, tpm); + if (tpm[1].u.memref.size <= params[1].memref.size) { + TEE_Result res2 = TEE_SUCCESS; + + res2 = copy_to_user(params[1].memref.buffer, va, + tpm[1].u.memref.size); + if (res2) + return res2; + } + + params[1].memref.size = tpm[1].u.memref.size; + + return res; +} + +typedef TEE_Result (*ta_func)(uint32_t instance_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]); + +static const ta_func ta_funcs[] = { + [PTA_SOCKET_OPEN] = socket_open, + [PTA_SOCKET_CLOSE] = socket_close, + [PTA_SOCKET_SEND] = socket_send, + [PTA_SOCKET_RECV] = socket_recv, + [PTA_SOCKET_IOCTL] = socket_ioctl, +}; + +/* + * Trusted Application Entry Points + */ + +static TEE_Result pta_socket_open_session(uint32_t param_types __unused, + TEE_Param pParams[TEE_NUM_PARAMS] __unused, + void **sess_ctx) +{ + struct ts_session *s = ts_get_calling_session(); + + /* Check that we're called from a TA */ + if (!s || !is_user_ta_ctx(s->ctx)) + return TEE_ERROR_ACCESS_DENIED; + + *sess_ctx = (void *)(vaddr_t)get_instance_id(s); + + return TEE_SUCCESS; +} + +static void pta_socket_close_session(void *sess_ctx) +{ + TEE_Result res; + struct thread_param tpm = { + .attr = THREAD_PARAM_ATTR_VALUE_IN, .u.value = { + .a = OPTEE_RPC_SOCKET_CLOSE_ALL, .b = (vaddr_t)sess_ctx, + }, + }; + + res = thread_rpc_cmd(OPTEE_RPC_CMD_SOCKET, 1, &tpm); + if (res != TEE_SUCCESS) + DMSG("OPTEE_RPC_SOCKET_CLOSE_ALL failed: %#" PRIx32, res); +} + +static TEE_Result pta_socket_invoke_command(void *sess_ctx, uint32_t cmd_id, + uint32_t param_types, TEE_Param params[TEE_NUM_PARAMS]) +{ + if (cmd_id < ARRAY_SIZE(ta_funcs) && ta_funcs[cmd_id]) + return ta_funcs[cmd_id]((vaddr_t)sess_ctx, param_types, params); + + return TEE_ERROR_NOT_IMPLEMENTED; +} + +pseudo_ta_register(.uuid = PTA_SOCKET_UUID, .name = "socket", + .flags = PTA_DEFAULT_FLAGS | TA_FLAG_CONCURRENT, + .open_session_entry_point = pta_socket_open_session, + .close_session_entry_point = pta_socket_close_session, + .invoke_command_entry_point = pta_socket_invoke_command); diff --git a/optee_os/core/tee/sub.mk b/optee_os/core/tee/sub.mk new file mode 100644 index 0000000..908f44c --- /dev/null +++ b/optee_os/core/tee/sub.mk @@ -0,0 +1,52 @@ +CFG_CRYPTO ?= y + +ifeq (y,$(CFG_CRYPTO)) + +# HMAC-based Extract-and-Expand Key Derivation Function +# http://tools.ietf.org/html/rfc5869 +# This is an OP-TEE extension, not part of the GlobalPlatform Internal API v1.0 +CFG_CRYPTO_HKDF ?= y + +# NIST SP800-56A Concatenation Key Derivation Function +# This is an OP-TEE extension +CFG_CRYPTO_CONCAT_KDF ?= y + +# PKCS #5 v2.0 / RFC 2898 key derivation function 2 +# This is an OP-TEE extension +CFG_CRYPTO_PBKDF2 ?= y + +endif + +srcs-y += entry_std.c +srcs-y += tee_cryp_utl.c +srcs-$(CFG_CRYPTO_HKDF) += tee_cryp_hkdf.c +srcs-$(CFG_CRYPTO_CONCAT_KDF) += tee_cryp_concat_kdf.c +ifneq ($(CFG_CRYPTO_HW_PBKDF2),y) +srcs-$(CFG_CRYPTO_PBKDF2) += tee_cryp_pbkdf2.c +endif + +ifeq ($(CFG_WITH_USER_TA),y) +srcs-y += tee_obj.c +srcs-y += tee_svc.c +srcs-y += tee_svc_cryp.c +srcs-y += tee_svc_storage.c +cppflags-tee_svc.c-y += -DTEE_IMPL_VERSION=$(TEE_IMPL_VERSION) +srcs-y += tee_time_generic.c +srcs-$(CFG_SECSTOR_TA) += tadb.c +srcs-$(CFG_GP_SOCKETS) += socket.c +srcs-y += tee_ta_enc_manager.c +endif #CFG_WITH_USER_TA,y + +srcs-$(_CFG_WITH_SECURE_STORAGE) += tee_fs_key_manager.c +srcs-$(CFG_RPMB_FS) += tee_rpmb_fs.c +srcs-$(CFG_REE_FS) += tee_ree_fs.c +srcs-$(CFG_REE_FS) += fs_dirfile.c +srcs-$(CFG_REE_FS) += fs_htree.c +srcs-$(CFG_REE_FS) += tee_fs_rpc.c + +ifeq ($(call cfg-one-enabled,CFG_WITH_USER_TA _CFG_WITH_SECURE_STORAGE),y) +srcs-y += tee_pobj.c +endif + +srcs-y += uuid.c +srcs-y += tee_supp_plugin_rpc.c diff --git a/optee_os/core/tee/tadb.c b/optee_os/core/tee/tadb.c new file mode 100644 index 0000000..b5a787d --- /dev/null +++ b/optee_os/core/tee/tadb.c @@ -0,0 +1,797 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TADB_MAX_BUFFER_SIZE (64U * 1024) + +#define TADB_AUTH_ENC_ALG TEE_ALG_AES_GCM +#define TADB_IV_SIZE TEE_AES_BLOCK_SIZE +#define TADB_TAG_SIZE TEE_AES_BLOCK_SIZE +#define TADB_KEY_SIZE TEE_AES_MAX_KEY_SIZE + +struct tee_tadb_dir { + const struct tee_file_operations *ops; + struct tee_file_handle *fh; + int nbits; + bitstr_t *files; +}; + +/* + * struct tadb_entry - TA database entry + * @prop: properties of TA + * @file_number: encrypted TA is stored in .ta + * @iv: Initialization vector of the authentication crypto + * @tag: Tag used to validate the authentication encrypted TA + * @key: Key used to decrypt the TA + */ +struct tadb_entry { + struct tee_tadb_property prop; + uint32_t file_number; + uint8_t iv[TADB_IV_SIZE]; + uint8_t tag[TADB_TAG_SIZE]; + uint8_t key[TADB_KEY_SIZE]; +}; + +struct tadb_header { + uint32_t opaque_len; + uint8_t opaque[]; +}; + +struct tee_tadb_ta_write { + struct tee_tadb_dir *db; + int fd; + struct tadb_entry entry; + size_t pos; + void *ctx; +}; + +struct tee_tadb_ta_read { + struct tee_tadb_dir *db; + int fd; + struct tadb_entry entry; + size_t pos; + void *ctx; + struct mobj *ta_mobj; + uint8_t *ta_buf; +}; + +static const char tadb_obj_id[] = "ta.db"; +static struct tee_tadb_dir *tadb_db; +static unsigned int tadb_db_refc; +static struct mutex tadb_mutex = MUTEX_INITIALIZER; + +static void file_num_to_str(char *buf, size_t blen, uint32_t file_number) +{ + int rc __maybe_unused = 0; + + rc = snprintf(buf, blen, "%" PRIu32 ".ta", file_number); + assert(rc >= 0); +} + +static bool is_null_uuid(const TEE_UUID *uuid) +{ + const TEE_UUID null_uuid = { 0 }; + + return !memcmp(uuid, &null_uuid, sizeof(*uuid)); +} + +static TEE_Result ta_operation_open(unsigned int cmd, uint32_t file_number, + int *fd) +{ + struct thread_param params[3] = { }; + struct mobj *mobj = NULL; + TEE_Result res = TEE_SUCCESS; + void *va = NULL; + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_FS, + THREAD_SHM_TYPE_APPLICATION, + TEE_FS_NAME_MAX, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + file_num_to_str(va, TEE_FS_NAME_MAX, file_number); + + params[0] = THREAD_PARAM_VALUE(IN, cmd, 0, 0); + params[1] = THREAD_PARAM_MEMREF(IN, mobj, 0, TEE_FS_NAME_MAX); + params[2] = THREAD_PARAM_VALUE(OUT, 0, 0, 0); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_FS, ARRAY_SIZE(params), params); + if (!res) + *fd = params[2].u.value.a; + + return res; +} + +static TEE_Result ta_operation_remove(uint32_t file_number) +{ + struct thread_param params[2] = { }; + struct mobj *mobj = NULL; + void *va = NULL; + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_FS, + THREAD_SHM_TYPE_APPLICATION, + TEE_FS_NAME_MAX, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + file_num_to_str(va, TEE_FS_NAME_MAX, file_number); + + params[0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_FS_REMOVE, 0, 0); + params[1] = THREAD_PARAM_MEMREF(IN, mobj, 0, TEE_FS_NAME_MAX); + + return thread_rpc_cmd(OPTEE_RPC_CMD_FS, ARRAY_SIZE(params), params); +} + +static TEE_Result maybe_grow_files(struct tee_tadb_dir *db, int idx) +{ + void *p; + + if (idx < db->nbits) + return TEE_SUCCESS; + + p = realloc(db->files, bitstr_size(idx + 1)); + if (!p) + return TEE_ERROR_OUT_OF_MEMORY; + db->files = p; + + bit_nclear(db->files, db->nbits, idx); + db->nbits = idx + 1; + + return TEE_SUCCESS; +} + +static TEE_Result set_file(struct tee_tadb_dir *db, int idx) +{ + TEE_Result res = maybe_grow_files(db, idx); + + if (!res) + bit_set(db->files, idx); + + return res; +} + +static void clear_file(struct tee_tadb_dir *db, int idx) +{ + /* + * This is safe because db->nbits > 0 implies that + * db->files is non-NULL (see maybe_grow_files()). + */ + if (idx < db->nbits) + bit_clear(db->files, idx); +} + +static bool test_file(struct tee_tadb_dir *db, int idx) +{ + /* + * This is safe because db->nbits > 0 implies that + * db->files is non-NULL (see maybe_grow_files()). + */ + if (idx < db->nbits) + return bit_test(db->files, idx); + + return false; +} + +static TEE_Result read_ent(struct tee_tadb_dir *db, size_t idx, + struct tadb_entry *entry) +{ + size_t l = sizeof(*entry); + TEE_Result res = db->ops->read(db->fh, idx * l, entry, NULL, &l); + + if (!res && l != sizeof(*entry)) + return TEE_ERROR_ITEM_NOT_FOUND; + + return res; +} + +static TEE_Result write_ent(struct tee_tadb_dir *db, size_t idx, + const struct tadb_entry *entry) +{ + const size_t l = sizeof(*entry); + + return db->ops->write(db->fh, idx * l, entry, NULL, l); +} + +static TEE_Result tadb_open(struct tee_tadb_dir **db_ret) +{ + TEE_Result res; + struct tee_tadb_dir *db = calloc(1, sizeof(*db)); + struct tee_pobj po = { + .obj_id = (void *)tadb_obj_id, + .obj_id_len = sizeof(tadb_obj_id) + }; + + if (!db) + return TEE_ERROR_OUT_OF_MEMORY; + + db->ops = tee_svc_storage_file_ops(TEE_STORAGE_PRIVATE); + + res = db->ops->open(&po, NULL, &db->fh); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = db->ops->create(&po, false, NULL, 0, NULL, 0, + NULL, NULL, 0, &db->fh); + + if (res) + free(db); + else + *db_ret = db; + + return res; +} + +static TEE_Result tee_tadb_open(struct tee_tadb_dir **db) +{ + TEE_Result res = TEE_SUCCESS; + + mutex_lock(&tadb_mutex); + if (!tadb_db_refc) { + assert(!tadb_db); + res = tadb_open(&tadb_db); + if (res) + goto err; + } + tadb_db_refc++; + *db = tadb_db; +err: + mutex_unlock(&tadb_mutex); + return res; +} + +static void tadb_put(struct tee_tadb_dir *db) +{ + assert(db == tadb_db); + mutex_lock(&tadb_mutex); + assert(tadb_db_refc); + tadb_db_refc--; + if (!tadb_db_refc) { + db->ops->close(&db->fh); + free(db->files); + free(db); + tadb_db = NULL; + } + mutex_unlock(&tadb_mutex); +} + +static void tee_tadb_close(struct tee_tadb_dir *db) +{ + tadb_put(db); +} + +static TEE_Result tadb_authenc_init(TEE_OperationMode mode, + const struct tadb_entry *entry, + void **ctx_ret) +{ + TEE_Result res; + void *ctx; + const size_t enc_size = entry->prop.custom_size + entry->prop.bin_size; + + res = crypto_authenc_alloc_ctx(&ctx, TADB_AUTH_ENC_ALG); + if (res) + return res; + + res = crypto_authenc_init(ctx, mode, entry->key, sizeof(entry->key), + entry->iv, sizeof(entry->iv), + sizeof(entry->tag), 0, enc_size); + if (res) + crypto_authenc_free_ctx(ctx); + else + *ctx_ret = ctx; + + return res; +} + +static TEE_Result tadb_update_payload(void *ctx, TEE_OperationMode mode, + const void *src, size_t len, void *dst) +{ + TEE_Result res; + size_t sz = len; + + res = crypto_authenc_update_payload(ctx, mode, (const uint8_t *)src, + len, dst, &sz); + assert(res || sz == len); + return res; +} + +static TEE_Result populate_files(struct tee_tadb_dir *db) +{ + TEE_Result res; + size_t idx; + + /* + * If db->files isn't NULL the bitfield is already populated and + * there's nothing left to do here for now. + */ + if (db->nbits) + return TEE_SUCCESS; + + /* + * Iterate over the TA database and set the bits in the bit field + * for used file numbers. Note that set_file() will allocate and + * grow the bitfield as needed. + * + * At the same time clean out duplicate file numbers, the first + * entry with the file number has precedence. Duplicate entries is + * not supposed to be able to happen, but if it still does better + * to clean it out here instead of letting the error spread with + * unexpected side effects. + */ + for (idx = 0;; idx++) { + struct tadb_entry entry; + + res = read_ent(db, idx, &entry); + if (res) { + if (res == TEE_ERROR_ITEM_NOT_FOUND) + return TEE_SUCCESS; + goto err; + } + + if (is_null_uuid(&entry.prop.uuid)) + continue; + + if (test_file(db, entry.file_number)) { + IMSG("Clearing duplicate file number %" PRIu32, + entry.file_number); + memset(&entry, 0, sizeof(entry)); + res = write_ent(db, idx, &entry); + if (res) + goto err; + continue; + } + + res = set_file(db, entry.file_number); + if (res) + goto err; + } + +err: + free(db->files); + db->files = NULL; + db->nbits = 0; + + return res; +} + +TEE_Result tee_tadb_ta_create(const struct tee_tadb_property *property, + struct tee_tadb_ta_write **ta_ret) +{ + TEE_Result res; + struct tee_tadb_ta_write *ta; + int i = 0; + + if (is_null_uuid(&property->uuid)) + return TEE_ERROR_GENERIC; + + ta = calloc(1, sizeof(*ta)); + if (!ta) + return TEE_ERROR_OUT_OF_MEMORY; + + res = tee_tadb_open(&ta->db); + if (res) + goto err_free; + + mutex_lock(&tadb_mutex); + + /* + * Since we're going to search for next free file number below we + * need to populate the bitfield holding used file numbers. + */ + res = populate_files(ta->db); + if (res) + goto err_mutex; + + if (ta->db->files) { + bit_ffc(ta->db->files, ta->db->nbits, &i); + if (i == -1) + i = ta->db->nbits; + } + + res = set_file(ta->db, i); + if (res) + goto err_mutex; + + mutex_unlock(&tadb_mutex); + + ta->entry.file_number = i; + ta->entry.prop = *property; + + res = crypto_rng_read(ta->entry.iv, sizeof(ta->entry.iv)); + if (res) + goto err_put; + + res = crypto_rng_read(ta->entry.key, sizeof(ta->entry.key)); + if (res) + goto err_put; + + res = ta_operation_open(OPTEE_RPC_FS_CREATE, ta->entry.file_number, + &ta->fd); + if (res) + goto err_put; + + res = tadb_authenc_init(TEE_MODE_ENCRYPT, &ta->entry, &ta->ctx); + if (res) + goto err_put; + + *ta_ret = ta; + + return TEE_SUCCESS; + +err_mutex: + mutex_unlock(&tadb_mutex); +err_put: + tadb_put(ta->db); +err_free: + free(ta); + + return res; +} + +TEE_Result tee_tadb_ta_write(struct tee_tadb_ta_write *ta, const void *buf, + size_t len) +{ + TEE_Result res; + const uint8_t *rb = buf; + size_t rl = len; + struct tee_fs_rpc_operation op; + + while (rl) { + size_t wl = MIN(rl, TADB_MAX_BUFFER_SIZE); + void *wb; + + res = tee_fs_rpc_write_init(&op, OPTEE_RPC_CMD_FS, ta->fd, + ta->pos, wl, &wb); + if (res) + return res; + + res = tadb_update_payload(ta->ctx, TEE_MODE_ENCRYPT, + rb, wl, wb); + if (res) + return res; + + res = tee_fs_rpc_write_final(&op); + if (res) + return res; + + rl -= wl; + rb += wl; + ta->pos += wl; + } + + return TEE_SUCCESS; +} + +void tee_tadb_ta_close_and_delete(struct tee_tadb_ta_write *ta) +{ + crypto_authenc_final(ta->ctx); + crypto_authenc_free_ctx(ta->ctx); + tee_fs_rpc_close(OPTEE_RPC_CMD_FS, ta->fd); + ta_operation_remove(ta->entry.file_number); + + mutex_lock(&tadb_mutex); + clear_file(ta->db, ta->entry.file_number); + mutex_unlock(&tadb_mutex); + + tadb_put(ta->db); + free(ta); +} + +static TEE_Result find_ent(struct tee_tadb_dir *db, const TEE_UUID *uuid, + size_t *idx_ret, struct tadb_entry *entry_ret) +{ + TEE_Result res; + size_t idx; + + /* + * Search for the provided uuid, if it's found return the index it + * has together with TEE_SUCCESS. + * + * If the uuid can't be found return the number indexes together + * with TEE_ERROR_ITEM_NOT_FOUND. + */ + for (idx = 0;; idx++) { + struct tadb_entry entry = { }; + + res = read_ent(db, idx, &entry); + if (res) { + if (res == TEE_ERROR_ITEM_NOT_FOUND) + break; + return res; + } + + if (!memcmp(&entry.prop.uuid, uuid, sizeof(*uuid))) { + if (entry_ret) + *entry_ret = entry; + break; + } + } + + *idx_ret = idx; + return res; +} + +static TEE_Result find_free_ent_idx(struct tee_tadb_dir *db, size_t *idx) +{ + const TEE_UUID null_uuid = { 0 }; + TEE_Result res = find_ent(db, &null_uuid, idx, NULL); + + /* + * Note that *idx is set to the number of entries on + * TEE_ERROR_ITEM_NOT_FOUND. + */ + if (res == TEE_ERROR_ITEM_NOT_FOUND) + return TEE_SUCCESS; + return res; +} + +TEE_Result tee_tadb_ta_close_and_commit(struct tee_tadb_ta_write *ta) +{ + TEE_Result res; + size_t dsz = 0; + size_t sz = sizeof(ta->entry.tag); + size_t idx; + struct tadb_entry old_ent; + bool have_old_ent = false; + + res = crypto_authenc_enc_final(ta->ctx, NULL, 0, NULL, &dsz, + ta->entry.tag, &sz); + if (res) + goto err; + + tee_fs_rpc_close(OPTEE_RPC_CMD_FS, ta->fd); + + mutex_lock(&tadb_mutex); + /* + * First try to find an existing TA to replace. If there's one + * we'll use the entry, but we should also remove the old encrypted + * file. + * + * If there isn't an existing TA to replace, grab a new entry. + */ + res = find_ent(ta->db, &ta->entry.prop.uuid, &idx, &old_ent); + if (!res) { + have_old_ent = true; + } else { + res = find_free_ent_idx(ta->db, &idx); + if (res) + goto err_mutex; + } + res = write_ent(ta->db, idx, &ta->entry); + if (res) + goto err_mutex; + if (have_old_ent) + clear_file(ta->db, old_ent.file_number); + mutex_unlock(&tadb_mutex); + + crypto_authenc_final(ta->ctx); + crypto_authenc_free_ctx(ta->ctx); + tadb_put(ta->db); + free(ta); + if (have_old_ent) + ta_operation_remove(old_ent.file_number); + return TEE_SUCCESS; + +err_mutex: + mutex_unlock(&tadb_mutex); +err: + tee_tadb_ta_close_and_delete(ta); + return res; +} + +TEE_Result tee_tadb_ta_delete(const TEE_UUID *uuid) +{ + const struct tadb_entry null_entry = { { { 0 } } }; + struct tee_tadb_dir *db; + struct tadb_entry entry; + size_t idx; + TEE_Result res; + + if (is_null_uuid(uuid)) + return TEE_ERROR_GENERIC; + + res = tee_tadb_open(&db); + if (res) + return res; + + mutex_lock(&tadb_mutex); + res = find_ent(db, uuid, &idx, &entry); + if (res) { + mutex_unlock(&tadb_mutex); + tee_tadb_close(db); + return res; + } + + clear_file(db, entry.file_number); + res = write_ent(db, idx, &null_entry); + mutex_unlock(&tadb_mutex); + + tee_tadb_close(db); + if (res) + return res; + + ta_operation_remove(entry.file_number); + return TEE_SUCCESS; +} + +TEE_Result tee_tadb_ta_open(const TEE_UUID *uuid, + struct tee_tadb_ta_read **ta_ret) +{ + TEE_Result res = TEE_SUCCESS; + size_t idx = 0; + struct tee_tadb_ta_read *ta = NULL; + + if (is_null_uuid(uuid)) + return TEE_ERROR_GENERIC; + + ta = calloc(1, sizeof(*ta)); + if (!ta) + return TEE_ERROR_OUT_OF_MEMORY; + + res = tee_tadb_open(&ta->db); + if (res) + goto err_free; /* Mustn't call tadb_put() */ + + mutex_read_lock(&tadb_mutex); + res = find_ent(ta->db, uuid, &idx, &ta->entry); + mutex_read_unlock(&tadb_mutex); + if (res) + goto err; + + res = ta_operation_open(OPTEE_RPC_FS_OPEN, ta->entry.file_number, + &ta->fd); + if (res) + goto err; + + res = tadb_authenc_init(TEE_MODE_DECRYPT, &ta->entry, &ta->ctx); + if (res) + goto err; + + *ta_ret = ta; + + return TEE_SUCCESS; +err: + tadb_put(ta->db); +err_free: + free(ta); + return res; +} + +const struct tee_tadb_property * +tee_tadb_ta_get_property(struct tee_tadb_ta_read *ta) +{ + return &ta->entry.prop; +} + +TEE_Result tee_tadb_get_tag(struct tee_tadb_ta_read *ta, uint8_t *tag, + unsigned int *tag_len) +{ + if (!tag || *tag_len < sizeof(ta->entry.tag)) { + *tag_len = sizeof(ta->entry.tag); + return TEE_ERROR_SHORT_BUFFER; + } + *tag_len = sizeof(ta->entry.tag); + + memcpy(tag, ta->entry.tag, sizeof(ta->entry.tag)); + + return TEE_SUCCESS; +} + +static TEE_Result ta_load(struct tee_tadb_ta_read *ta) +{ + struct thread_param params[2] = { }; + TEE_Result res; + const size_t sz = ta->entry.prop.custom_size + ta->entry.prop.bin_size; + + if (ta->ta_mobj) + return TEE_SUCCESS; + + ta->ta_mobj = thread_rpc_alloc_payload(sz); + if (!ta->ta_mobj) + return TEE_ERROR_OUT_OF_MEMORY; + + ta->ta_buf = mobj_get_va(ta->ta_mobj, 0, sz); + assert(ta->ta_buf); + + params[0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_FS_READ, ta->fd, 0); + params[1] = THREAD_PARAM_MEMREF(OUT, ta->ta_mobj, 0, sz); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_FS, ARRAY_SIZE(params), params); + if (res) { + thread_rpc_free_payload(ta->ta_mobj); + ta->ta_mobj = NULL; + } + return res; +} + +TEE_Result tee_tadb_ta_read(struct tee_tadb_ta_read *ta, void *buf_core, + void *buf_user, size_t *len) +{ + TEE_Result res = TEE_SUCCESS; + const size_t sz = ta->entry.prop.custom_size + ta->entry.prop.bin_size; + size_t l = MIN(*len, sz - ta->pos); + size_t bb_len = MIN(1024U, l); + size_t num_bytes = 0; + size_t dst_len = 0; + void *dst = NULL; + void *bb = NULL; + + res = ta_load(ta); + if (res) + return res; + + if (buf_core) { + dst = buf_core; + dst_len = l; + } else { + bb = bb_alloc(bb_len); + if (!bb) + return TEE_ERROR_OUT_OF_MEMORY; + dst = bb; + dst_len = bb_len; + } + + /* + * This loop will only run once if buf_core is non-NULL, but as + * many times as needed if the bounce buffer bb is used. That's why + * dst doesn't need to be updated in the loop. + */ + while (num_bytes < l) { + size_t n = MIN(dst_len, l - num_bytes); + + res = tadb_update_payload(ta->ctx, TEE_MODE_DECRYPT, + ta->ta_buf + ta->pos + num_bytes, + n, dst); + if (res) + goto out; + + if (buf_user) { + res = copy_to_user((uint8_t *)buf_user + num_bytes, + dst, n); + if (res) + goto out; + } + num_bytes += n; + } + + ta->pos += l; + if (ta->pos == sz) { + size_t dl = 0; + + res = crypto_authenc_dec_final(ta->ctx, NULL, 0, NULL, &dl, + ta->entry.tag, TADB_TAG_SIZE); + if (res) + return res; + } + *len = l; +out: + bb_free(bb, bb_len); + return res; +} + +void tee_tadb_ta_close(struct tee_tadb_ta_read *ta) +{ + crypto_authenc_final(ta->ctx); + crypto_authenc_free_ctx(ta->ctx); + if (ta->ta_mobj) + thread_rpc_free_payload(ta->ta_mobj); + tee_fs_rpc_close(OPTEE_RPC_CMD_FS, ta->fd); + tadb_put(ta->db); + free(ta); +} diff --git a/optee_os/core/tee/tee_cryp_concat_kdf.c b/optee_os/core/tee/tee_cryp_concat_kdf.c new file mode 100644 index 0000000..8e74ec3 --- /dev/null +++ b/optee_os/core/tee/tee_cryp_concat_kdf.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +TEE_Result tee_cryp_concat_kdf(uint32_t hash_id, const uint8_t *shared_secret, + size_t shared_secret_len, + const uint8_t *other_info, + size_t other_info_len, uint8_t *derived_key, + size_t derived_key_len) +{ + TEE_Result res; + size_t hash_len, i, n, sz; + void *ctx = NULL; + uint8_t tmp[TEE_MAX_HASH_SIZE]; + uint32_t be_count; + uint8_t *out = derived_key; + uint32_t hash_algo = TEE_ALG_HASH_ALGO(hash_id); + + res = crypto_hash_alloc_ctx(&ctx, hash_algo); + if (res != TEE_SUCCESS) + return res; + + res = tee_alg_get_digest_size(hash_algo, &hash_len); + if (res != TEE_SUCCESS) + goto out; + + n = derived_key_len / hash_len; + sz = hash_len; + for (i = 1; i <= n + 1; i++) { + be_count = TEE_U32_TO_BIG_ENDIAN(i); + + res = crypto_hash_init(ctx); + if (res != TEE_SUCCESS) + goto out; + res = crypto_hash_update(ctx, (uint8_t *)&be_count, + sizeof(be_count)); + if (res != TEE_SUCCESS) + goto out; + res = crypto_hash_update(ctx, shared_secret, shared_secret_len); + if (res != TEE_SUCCESS) + goto out; + if (other_info && other_info_len) { + res = crypto_hash_update(ctx, other_info, + other_info_len); + if (res != TEE_SUCCESS) + goto out; + } + res = crypto_hash_final(ctx, tmp, sizeof(tmp)); + if (res != TEE_SUCCESS) + goto out; + + if (i == n + 1) + sz = derived_key_len % hash_len; + memcpy(out, tmp, sz); + out += sz; + } + res = TEE_SUCCESS; +out: + crypto_hash_free_ctx(ctx); + return res; +} diff --git a/optee_os/core/tee/tee_cryp_hkdf.c b/optee_os/core/tee/tee_cryp_hkdf.c new file mode 100644 index 0000000..c62f22b --- /dev/null +++ b/optee_os/core/tee/tee_cryp_hkdf.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + + +static const uint8_t zero_salt[TEE_MAX_HASH_SIZE]; + +static TEE_Result hkdf_extract(uint32_t hash_id, const uint8_t *ikm, + size_t ikm_len, const uint8_t *salt, + size_t salt_len, uint8_t *prk, size_t *prk_len) +{ + TEE_Result res; + void *ctx = NULL; + uint32_t hash_algo = TEE_ALG_HASH_ALGO(hash_id); + uint32_t hmac_algo = (TEE_OPERATION_MAC << 28) | hash_id; + + if (!salt || !salt_len) { + /* + * RFC 5869 section 2.2: + * If not provided, [the salt] is set to a string of HashLen + * zeros + */ + salt = zero_salt; + res = tee_alg_get_digest_size(hash_algo, &salt_len); + if (res != TEE_SUCCESS) + goto out; + } + + res = crypto_mac_alloc_ctx(&ctx, hmac_algo); + if (res) + goto out; + + /* + * RFC 5869 section 2.1: "Note that in the extract step, 'IKM' is used + * as the HMAC input, not as the HMAC key." + * Therefore, salt is the HMAC key in the formula from section 2.2: + * "PRK = HMAC-Hash(salt, IKM)" + */ + res = crypto_mac_init(ctx, salt, salt_len); + if (res != TEE_SUCCESS) + goto out; + + res = crypto_mac_update(ctx, ikm, ikm_len); + if (res != TEE_SUCCESS) + goto out; + + res = crypto_mac_final(ctx, prk, *prk_len); + if (res != TEE_SUCCESS) + goto out; + + res = tee_alg_get_digest_size(hash_algo, prk_len); +out: + crypto_mac_free_ctx(ctx); + return res; +} + +static TEE_Result hkdf_expand(uint32_t hash_id, const uint8_t *prk, + size_t prk_len, const uint8_t *info, + size_t info_len, uint8_t *okm, size_t okm_len) +{ + uint8_t tn[TEE_MAX_HASH_SIZE]; + size_t tn_len, hash_len, i, n, where; + TEE_Result res = TEE_SUCCESS; + void *ctx = NULL; + uint32_t hash_algo = TEE_ALG_HASH_ALGO(hash_id); + uint32_t hmac_algo = TEE_ALG_HMAC_ALGO(hash_id); + + res = tee_alg_get_digest_size(hash_algo, &hash_len); + if (res != TEE_SUCCESS) + goto out; + + if (!okm || prk_len < hash_len) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + if (!info) + info_len = 0; + + res = crypto_mac_alloc_ctx(&ctx, hmac_algo); + if (res) + goto out; + + /* N = ceil(L/HashLen) */ + n = okm_len / hash_len; + if ((okm_len % hash_len) != 0) + n++; + + if (n > 255) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + + /* + * RFC 5869 section 2.3 + * T = T(1) | T(2) | T(3) | ... | T(N) + * OKM = first L octets of T + * T(0) = empty string (zero length) + * T(1) = HMAC-Hash(PRK, T(0) | info | 0x01) + * T(2) = HMAC-Hash(PRK, T(1) | info | 0x02) + * T(3) = HMAC-Hash(PRK, T(2) | info | 0x03) + * ... + */ + tn_len = 0; + where = 0; + for (i = 1; i <= n; i++) { + uint8_t c = i; + + res = crypto_mac_init(ctx, prk, prk_len); + if (res != TEE_SUCCESS) + goto out; + res = crypto_mac_update(ctx, tn, tn_len); + if (res != TEE_SUCCESS) + goto out; + res = crypto_mac_update(ctx, info, info_len); + if (res != TEE_SUCCESS) + goto out; + res = crypto_mac_update(ctx, &c, 1); + if (res != TEE_SUCCESS) + goto out; + res = crypto_mac_final(ctx, tn, sizeof(tn)); + if (res != TEE_SUCCESS) + goto out; + + memcpy(okm + where, tn, (i < n) ? hash_len : (okm_len - where)); + where += hash_len; + tn_len = hash_len; + } + +out: + crypto_mac_free_ctx(ctx); + return res; +} + +TEE_Result tee_cryp_hkdf(uint32_t hash_id, const uint8_t *ikm, size_t ikm_len, + const uint8_t *salt, size_t salt_len, + const uint8_t *info, size_t info_len, uint8_t *okm, + size_t okm_len) +{ + TEE_Result res; + uint8_t prk[TEE_MAX_HASH_SIZE]; + size_t prk_len = sizeof(prk); + + res = hkdf_extract(hash_id, ikm, ikm_len, salt, salt_len, prk, + &prk_len); + if (res != TEE_SUCCESS) + return res; + res = hkdf_expand(hash_id, prk, prk_len, info, info_len, okm, + okm_len); + + return res; +} diff --git a/optee_os/core/tee/tee_cryp_pbkdf2.c b/optee_os/core/tee/tee_cryp_pbkdf2.c new file mode 100644 index 0000000..1a7dd57 --- /dev/null +++ b/optee_os/core/tee/tee_cryp_pbkdf2.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +struct hmac_parms { + uint32_t algo; + size_t hash_len; + void *ctx; +}; + +struct pbkdf2_parms { + const uint8_t *password; + size_t password_len; + const uint8_t *salt; + size_t salt_len; + uint32_t iteration_count; +}; + +static TEE_Result pbkdf2_f(uint8_t *out, size_t len, uint32_t idx, + struct hmac_parms *h, struct pbkdf2_parms *p) +{ + TEE_Result res; + uint8_t u[TEE_MAX_HASH_SIZE]; + uint32_t be_index; + size_t i, j; + + memset(out, 0, len); + for (i = 1; i <= p->iteration_count; i++) { + res = crypto_mac_init(h->ctx, p->password, p->password_len); + if (res != TEE_SUCCESS) + return res; + + if (i == 1) { + if (p->salt && p->salt_len) { + res = crypto_mac_update(h->ctx, p->salt, + p->salt_len); + if (res != TEE_SUCCESS) + return res; + } + + be_index = TEE_U32_TO_BIG_ENDIAN(idx); + + res = crypto_mac_update(h->ctx, (uint8_t *)&be_index, + sizeof(be_index)); + if (res != TEE_SUCCESS) + return res; + } else { + res = crypto_mac_update(h->ctx, u, h->hash_len); + if (res != TEE_SUCCESS) + return res; + } + + res = crypto_mac_final(h->ctx, u, sizeof(u)); + if (res != TEE_SUCCESS) + return res; + + for (j = 0; j < len; j++) + out[j] ^= u[j]; + } + return TEE_SUCCESS; +} + +TEE_Result tee_cryp_pbkdf2(uint32_t hash_id, const uint8_t *password, + size_t password_len, const uint8_t *salt, + size_t salt_len, uint32_t iteration_count, + uint8_t *derived_key, size_t derived_key_len) +{ + TEE_Result res; + size_t i, l, r; + uint8_t *out = derived_key; + struct pbkdf2_parms pbkdf2_parms; + struct hmac_parms hmac_parms = {0, }; + + hmac_parms.algo = TEE_ALG_HMAC_ALGO(hash_id); + + res = tee_alg_get_digest_size(hmac_parms.algo, &hmac_parms.hash_len); + if (res != TEE_SUCCESS) + return res; + + res = crypto_mac_alloc_ctx(&hmac_parms.ctx, hmac_parms.algo); + if (res != TEE_SUCCESS) + return res; + + pbkdf2_parms.password = password; + pbkdf2_parms.password_len = password_len; + pbkdf2_parms.salt = salt; + pbkdf2_parms.salt_len = salt_len; + pbkdf2_parms.iteration_count = iteration_count; + + l = derived_key_len / hmac_parms.hash_len; + r = derived_key_len % hmac_parms.hash_len; + + for (i = 1; i <= l; i++) { + res = pbkdf2_f(out, hmac_parms.hash_len, i, &hmac_parms, + &pbkdf2_parms); + if (res != TEE_SUCCESS) + goto out; + out += hmac_parms.hash_len; + } + if (r) + res = pbkdf2_f(out, r, i, &hmac_parms, &pbkdf2_parms); + +out: + crypto_mac_free_ctx(hmac_parms.ctx); + return res; +} diff --git a/optee_os/core/tee/tee_cryp_utl.c b/optee_os/core/tee/tee_cryp_utl.c new file mode 100644 index 0000000..3220c16 --- /dev/null +++ b/optee_os/core/tee/tee_cryp_utl.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2021, Linaro Limited + * Copyright (c) 2021, SumUp Services GmbH + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TEE_Result tee_alg_get_digest_size(uint32_t algo, size_t *size) +{ + size_t digest_size = TEE_ALG_GET_DIGEST_SIZE(algo); + + if (!digest_size) + return TEE_ERROR_NOT_SUPPORTED; + + *size = digest_size; + + return TEE_SUCCESS; +} + +TEE_Result tee_hash_createdigest(uint32_t algo, const uint8_t *data, + size_t datalen, uint8_t *digest, + size_t digestlen) +{ + TEE_Result res; + void *ctx = NULL; + + res = crypto_hash_alloc_ctx(&ctx, algo); + if (res) + return res; + + res = crypto_hash_init(ctx); + if (res) + goto out; + + if (datalen != 0) { + res = crypto_hash_update(ctx, data, datalen); + if (res) + goto out; + } + + res = crypto_hash_final(ctx, digest, digestlen); +out: + crypto_hash_free_ctx(ctx); + + return res; +} + +TEE_Result tee_cipher_get_block_size(uint32_t algo, size_t *size) +{ + switch (algo) { + case TEE_ALG_AES_CBC_MAC_NOPAD: + case TEE_ALG_AES_CBC_MAC_PKCS5: + case TEE_ALG_AES_CMAC: + case TEE_ALG_AES_ECB_NOPAD: + case TEE_ALG_AES_CBC_NOPAD: + case TEE_ALG_AES_CTR: + case TEE_ALG_AES_CTS: + case TEE_ALG_AES_XTS: + case TEE_ALG_AES_CCM: + case TEE_ALG_AES_GCM: + case TEE_ALG_SM4_ECB_NOPAD: + case TEE_ALG_SM4_CBC_NOPAD: + case TEE_ALG_SM4_XTS: + case TEE_ALG_SM4_CTR: + *size = 16; + break; + + case TEE_ALG_DES_CBC_MAC_NOPAD: + case TEE_ALG_DES_CBC_MAC_PKCS5: + case TEE_ALG_DES_ECB_NOPAD: + case TEE_ALG_DES_CBC_NOPAD: + case TEE_ALG_DES3_CBC_MAC_NOPAD: + case TEE_ALG_DES3_CBC_MAC_PKCS5: + case TEE_ALG_DES3_ECB_NOPAD: + case TEE_ALG_DES3_CBC_NOPAD: + case TEE_ALG_DES3_CMAC: + *size = 8; + break; + + default: + return TEE_ERROR_NOT_SUPPORTED; + } + + return TEE_SUCCESS; +} + +TEE_Result tee_do_cipher_update(void *ctx, uint32_t algo, + TEE_OperationMode mode, bool last_block, + const uint8_t *data, size_t len, uint8_t *dst) +{ + TEE_Result res; + size_t block_size; + + if (mode != TEE_MODE_ENCRYPT && mode != TEE_MODE_DECRYPT) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * Check that the block contains the correct number of data, apart + * for the last block in some XTS / CTR / XTS mode + */ + res = tee_cipher_get_block_size(algo, &block_size); + if (res != TEE_SUCCESS) + return res; + if ((len % block_size) != 0) { + if (!last_block && algo != TEE_ALG_AES_CTR) + return TEE_ERROR_BAD_PARAMETERS; + + switch (algo) { + case TEE_ALG_AES_ECB_NOPAD: + case TEE_ALG_DES_ECB_NOPAD: + case TEE_ALG_DES3_ECB_NOPAD: + case TEE_ALG_AES_CBC_NOPAD: + case TEE_ALG_DES_CBC_NOPAD: + case TEE_ALG_DES3_CBC_NOPAD: + case TEE_ALG_SM4_ECB_NOPAD: + case TEE_ALG_SM4_CBC_NOPAD: + return TEE_ERROR_BAD_PARAMETERS; + + case TEE_ALG_AES_CTR: + case TEE_ALG_AES_XTS: + case TEE_ALG_AES_CTS: + case TEE_ALG_SM4_XTS: + /* + * These modes doesn't require padding for the last + * block. + * + * This isn't entirely true, both XTS and CTS can only + * encrypt minimum one block and also they need at least + * one complete block in the last update to finish the + * encryption. The algorithms are supposed to detect + * that, we're only making sure that all data fed up to + * that point consists of complete blocks. + */ + break; + + default: + return TEE_ERROR_NOT_SUPPORTED; + } + } + + return crypto_cipher_update(ctx, mode, last_block, data, len, dst); +} + +/* + * Override this in your platform code to feed the PRNG platform-specific + * jitter entropy. This implementation does not efficiently deliver entropy + * and is here for backwards-compatibility. + */ +__weak void plat_prng_add_jitter_entropy(enum crypto_rng_src sid, + unsigned int *pnum) +{ + TEE_Time current; + +#ifdef CFG_SECURE_TIME_SOURCE_REE + if (CRYPTO_RNG_SRC_IS_QUICK(sid)) + return; /* Can't read REE time here */ +#endif + + if (tee_time_get_sys_time(¤t) == TEE_SUCCESS) + crypto_rng_add_event(sid, pnum, ¤t, sizeof(current)); +} + +__weak void plat_rng_init(void) +{ + TEE_Result res = TEE_SUCCESS; + TEE_Time t; + +#ifndef CFG_SECURE_TIME_SOURCE_REE + /* + * This isn't much of a seed. Ideally we should either get a seed from + * a hardware RNG or from a previously saved seed. + * + * Seeding with hardware RNG is currently up to the platform to + * override this function. + * + * Seeding with a saved seed will require cooperation from normal + * world, this is still TODO. + */ + res = tee_time_get_sys_time(&t); +#else + EMSG("Warning: seeding RNG with zeroes"); + memset(&t, 0, sizeof(t)); +#endif + if (!res) + res = crypto_rng_init(&t, sizeof(t)); + if (res) { + EMSG("Failed to initialize RNG: %#" PRIx32, res); + panic(); + } +} + +static TEE_Result tee_cryp_init(void) +{ + TEE_Result res = crypto_init(); + + if (res) { + EMSG("Failed to initialize crypto API: %#" PRIx32, res); + panic(); + } + plat_rng_init(); + + dt_driver_crypt_init_complete(); + + return TEE_SUCCESS; +} +service_init(tee_cryp_init); diff --git a/optee_os/core/tee/tee_fs_key_manager.c b/optee_os/core/tee/tee_fs_key_manager.c new file mode 100644 index 0000000..00bafca --- /dev/null +++ b/optee_os/core/tee/tee_fs_key_manager.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + + +/* + * Acronyms: + * + * FEK - File Encryption Key + * SSK - Secure Storage Key + * TSK - Trusted app Storage Key + * IV - Initial vector + * HUK - Hardware Unique Key + * RNG - Random Number Generator + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct tee_fs_ssk { + bool is_init; + uint8_t key[TEE_FS_KM_SSK_SIZE]; +}; + +static struct tee_fs_ssk tee_fs_ssk; + +static TEE_Result do_hmac(void *out_key, size_t out_key_size, + const void *in_key, size_t in_key_size, + const void *message, size_t message_size) +{ + TEE_Result res; + void *ctx = NULL; + + if (!out_key || !in_key || !message) + return TEE_ERROR_BAD_PARAMETERS; + + res = crypto_mac_alloc_ctx(&ctx, TEE_FS_KM_HMAC_ALG); + if (res != TEE_SUCCESS) + return res; + + res = crypto_mac_init(ctx, in_key, in_key_size); + if (res != TEE_SUCCESS) + goto exit; + + res = crypto_mac_update(ctx, message, message_size); + if (res != TEE_SUCCESS) + goto exit; + + res = crypto_mac_final(ctx, out_key, out_key_size); + if (res != TEE_SUCCESS) + goto exit; + + res = TEE_SUCCESS; + +exit: + crypto_mac_free_ctx(ctx); + return res; +} + +TEE_Result tee_fs_fek_crypt(const TEE_UUID *uuid, TEE_OperationMode mode, + const uint8_t *in_key, size_t size, + uint8_t *out_key) +{ + TEE_Result res; + void *ctx = NULL; + uint8_t tsk[TEE_FS_KM_TSK_SIZE]; + uint8_t dst_key[size]; + + if (!in_key || !out_key) + return TEE_ERROR_BAD_PARAMETERS; + + if (size != TEE_FS_KM_FEK_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + if (tee_fs_ssk.is_init == 0) + return TEE_ERROR_GENERIC; + + if (uuid) { + res = do_hmac(tsk, sizeof(tsk), tee_fs_ssk.key, + TEE_FS_KM_SSK_SIZE, uuid, sizeof(*uuid)); + if (res != TEE_SUCCESS) + return res; + } else { + /* + * Pick something of a different size than TEE_UUID to + * guarantee that there's never a conflict. + */ + uint8_t dummy[1] = { 0 }; + + res = do_hmac(tsk, sizeof(tsk), tee_fs_ssk.key, + TEE_FS_KM_SSK_SIZE, dummy, sizeof(dummy)); + if (res != TEE_SUCCESS) + return res; + } + + res = crypto_cipher_alloc_ctx(&ctx, TEE_FS_KM_ENC_FEK_ALG); + if (res != TEE_SUCCESS) + return res; + + res = crypto_cipher_init(ctx, mode, tsk, sizeof(tsk), NULL, 0, NULL, 0); + if (res != TEE_SUCCESS) + goto exit; + + res = crypto_cipher_update(ctx, mode, true, in_key, size, dst_key); + if (res != TEE_SUCCESS) + goto exit; + + crypto_cipher_final(ctx); + + memcpy(out_key, dst_key, sizeof(dst_key)); + +exit: + crypto_cipher_free_ctx(ctx); + memzero_explicit(tsk, sizeof(tsk)); + memzero_explicit(dst_key, sizeof(dst_key)); + + return res; +} + +static TEE_Result generate_fek(uint8_t *key, uint8_t len) +{ + return crypto_rng_read(key, len); +} + +static TEE_Result tee_fs_init_key_manager(void) +{ + TEE_Result res = TEE_SUCCESS; + + COMPILE_TIME_ASSERT(TEE_FS_KM_SSK_SIZE <= HUK_SUBKEY_MAX_LEN); + + res = huk_subkey_derive(HUK_SUBKEY_SSK, NULL, 0, + tee_fs_ssk.key, sizeof(tee_fs_ssk.key)); + if (res == TEE_SUCCESS) + tee_fs_ssk.is_init = 1; + else + memzero_explicit(&tee_fs_ssk, sizeof(tee_fs_ssk)); + + return res; +} + +TEE_Result tee_fs_generate_fek(const TEE_UUID *uuid, void *buf, size_t buf_size) +{ + TEE_Result res; + + if (buf_size != TEE_FS_KM_FEK_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + res = generate_fek(buf, TEE_FS_KM_FEK_SIZE); + if (res != TEE_SUCCESS) + return res; + + return tee_fs_fek_crypt(uuid, TEE_MODE_ENCRYPT, buf, + TEE_FS_KM_FEK_SIZE, buf); +} + +static TEE_Result sha256(uint8_t *out, size_t out_size, const uint8_t *in, + size_t in_size) +{ + return tee_hash_createdigest(TEE_ALG_SHA256, in, in_size, + out, out_size); +} + +static TEE_Result aes_ecb(uint8_t out[TEE_AES_BLOCK_SIZE], + const uint8_t in[TEE_AES_BLOCK_SIZE], + const uint8_t *key, size_t key_size) +{ + TEE_Result res; + void *ctx = NULL; + + res = crypto_cipher_alloc_ctx(&ctx, TEE_ALG_AES_ECB_NOPAD); + if (res != TEE_SUCCESS) + return res; + + res = crypto_cipher_init(ctx, TEE_MODE_ENCRYPT, key, + key_size, NULL, 0, NULL, 0); + if (res != TEE_SUCCESS) + goto out; + + res = crypto_cipher_update(ctx, TEE_MODE_ENCRYPT, true, in, + TEE_AES_BLOCK_SIZE, out); + if (res != TEE_SUCCESS) + goto out; + + crypto_cipher_final(ctx); + res = TEE_SUCCESS; + +out: + crypto_cipher_free_ctx(ctx); + return res; +} + +static TEE_Result essiv(uint8_t iv[TEE_AES_BLOCK_SIZE], + const uint8_t fek[TEE_FS_KM_FEK_SIZE], + uint16_t blk_idx) +{ + TEE_Result res; + uint8_t sha[TEE_SHA256_HASH_SIZE]; + uint8_t pad_blkid[TEE_AES_BLOCK_SIZE] = { 0, }; + + res = sha256(sha, sizeof(sha), fek, TEE_FS_KM_FEK_SIZE); + if (res != TEE_SUCCESS) + return res; + + pad_blkid[0] = (blk_idx & 0xFF); + pad_blkid[1] = (blk_idx & 0xFF00) >> 8; + + res = aes_ecb(iv, pad_blkid, sha, 16); + + memzero_explicit(sha, sizeof(sha)); + return res; +} + +/* + * Encryption/decryption of RPMB FS file data. This is AES CBC with ESSIV. + */ +TEE_Result tee_fs_crypt_block(const TEE_UUID *uuid, uint8_t *out, + const uint8_t *in, size_t size, + uint16_t blk_idx, const uint8_t *encrypted_fek, + TEE_OperationMode mode) +{ + TEE_Result res; + uint8_t fek[TEE_FS_KM_FEK_SIZE]; + uint8_t iv[TEE_AES_BLOCK_SIZE]; + void *ctx; + + DMSG("%scrypt block #%u", (mode == TEE_MODE_ENCRYPT) ? "En" : "De", + blk_idx); + + /* Decrypt FEK */ + res = tee_fs_fek_crypt(uuid, TEE_MODE_DECRYPT, encrypted_fek, + TEE_FS_KM_FEK_SIZE, fek); + if (res != TEE_SUCCESS) + goto wipe; + + /* Compute initialization vector for this block */ + res = essiv(iv, fek, blk_idx); + if (res != TEE_SUCCESS) + goto wipe; + + /* Run AES CBC */ + res = crypto_cipher_alloc_ctx(&ctx, TEE_ALG_AES_CBC_NOPAD); + if (res != TEE_SUCCESS) + goto wipe; + + res = crypto_cipher_init(ctx, mode, fek, sizeof(fek), NULL, + 0, iv, TEE_AES_BLOCK_SIZE); + if (res != TEE_SUCCESS) + goto exit; + res = crypto_cipher_update(ctx, mode, true, in, size, out); + if (res != TEE_SUCCESS) + goto exit; + + crypto_cipher_final(ctx); + +exit: + crypto_cipher_free_ctx(ctx); +wipe: + memzero_explicit(fek, sizeof(fek)); + memzero_explicit(iv, sizeof(iv)); + return res; +} + +service_init_late(tee_fs_init_key_manager); diff --git a/optee_os/core/tee/tee_fs_rpc.c b/optee_os/core/tee/tee_fs_rpc.c new file mode 100644 index 0000000..e0d54e1 --- /dev/null +++ b/optee_os/core/tee/tee_fs_rpc.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct tee_fs_dir { + int nw_dir; + struct tee_fs_dirent d; +}; + +/* "/dirf.db" or "/" */ +static TEE_Result create_filename(void *buf, size_t blen, + const struct tee_fs_dirfile_fileh *dfh) +{ + char *file = buf; + size_t pos = 0; + size_t l; + + if (pos >= blen) + return TEE_ERROR_SHORT_BUFFER; + + file[pos] = '/'; + pos++; + if (pos >= blen) + return TEE_ERROR_SHORT_BUFFER; + + l = blen - pos; + return tee_fs_dirfile_fileh_to_fname(dfh, file + pos, &l); +} + +static TEE_Result operation_commit(struct tee_fs_rpc_operation *op) +{ + return thread_rpc_cmd(op->id, op->num_params, op->params); +} + +static TEE_Result operation_open_dfh(uint32_t id, unsigned int cmd, + const struct tee_fs_dirfile_fileh *dfh, + int *fd) +{ + struct tee_fs_rpc_operation op = { }; + struct mobj *mobj = NULL; + TEE_Result res = TEE_SUCCESS; + void *va = NULL; + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_FS, + THREAD_SHM_TYPE_APPLICATION, + TEE_FS_NAME_MAX, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + res = create_filename(va, TEE_FS_NAME_MAX, dfh); + if (res != TEE_SUCCESS) + return res; + + op = (struct tee_fs_rpc_operation){ + .id = id, .num_params = 3, .params = { + [0] = THREAD_PARAM_VALUE(IN, cmd, 0, 0), + [1] = THREAD_PARAM_MEMREF(IN, mobj, 0, TEE_FS_NAME_MAX), + [2] = THREAD_PARAM_VALUE(OUT, 0, 0, 0), + } }; + + res = operation_commit(&op); + if (res == TEE_SUCCESS) + *fd = op.params[2].u.value.a; + + return res; +} + + + +TEE_Result tee_fs_rpc_open_dfh(uint32_t id, + const struct tee_fs_dirfile_fileh *dfh, int *fd) +{ + return operation_open_dfh(id, OPTEE_RPC_FS_OPEN, dfh, fd); +} + +TEE_Result tee_fs_rpc_create_dfh(uint32_t id, + const struct tee_fs_dirfile_fileh *dfh, + int *fd) +{ + return operation_open_dfh(id, OPTEE_RPC_FS_CREATE, dfh, fd); +} + +TEE_Result tee_fs_rpc_close(uint32_t id, int fd) +{ + struct tee_fs_rpc_operation op = { + .id = id, .num_params = 1, .params = { + [0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_FS_CLOSE, fd, 0), + }, + }; + + return operation_commit(&op); +} + +TEE_Result tee_fs_rpc_read_init(struct tee_fs_rpc_operation *op, + uint32_t id, int fd, tee_fs_off_t offset, + size_t data_len, void **out_data) +{ + struct mobj *mobj; + uint8_t *va; + + if (offset < 0) + return TEE_ERROR_BAD_PARAMETERS; + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_FS, + THREAD_SHM_TYPE_APPLICATION, + data_len, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + *op = (struct tee_fs_rpc_operation){ + .id = id, .num_params = 2, .params = { + [0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_FS_READ, fd, + offset), + [1] = THREAD_PARAM_MEMREF(OUT, mobj, 0, data_len), + }, + }; + + *out_data = va; + + return TEE_SUCCESS; +} + +TEE_Result tee_fs_rpc_read_final(struct tee_fs_rpc_operation *op, + size_t *data_len) +{ + TEE_Result res = operation_commit(op); + + if (res == TEE_SUCCESS) + *data_len = op->params[1].u.memref.size; + return res; +} + +TEE_Result tee_fs_rpc_write_init(struct tee_fs_rpc_operation *op, + uint32_t id, int fd, tee_fs_off_t offset, + size_t data_len, void **data) +{ + struct mobj *mobj; + uint8_t *va; + + if (offset < 0) + return TEE_ERROR_BAD_PARAMETERS; + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_FS, + THREAD_SHM_TYPE_APPLICATION, + data_len, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + *op = (struct tee_fs_rpc_operation){ + .id = id, .num_params = 2, .params = { + [0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_FS_WRITE, fd, + offset), + [1] = THREAD_PARAM_MEMREF(IN, mobj, 0, data_len), + }, + }; + + *data = va; + + return TEE_SUCCESS; +} + +TEE_Result tee_fs_rpc_write_final(struct tee_fs_rpc_operation *op) +{ + return operation_commit(op); +} + +TEE_Result tee_fs_rpc_truncate(uint32_t id, int fd, size_t len) +{ + struct tee_fs_rpc_operation op = { + .id = id, .num_params = 1, .params = { + [0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_FS_TRUNCATE, fd, + len), + } + }; + + return operation_commit(&op); +} + +TEE_Result tee_fs_rpc_remove_dfh(uint32_t id, + const struct tee_fs_dirfile_fileh *dfh) +{ + struct tee_fs_rpc_operation op = { }; + TEE_Result res = TEE_SUCCESS; + struct mobj *mobj = NULL; + void *va = NULL; + + va = thread_rpc_shm_cache_alloc(THREAD_SHM_CACHE_USER_FS, + THREAD_SHM_TYPE_APPLICATION, + TEE_FS_NAME_MAX, &mobj); + if (!va) + return TEE_ERROR_OUT_OF_MEMORY; + + + res = create_filename(va, TEE_FS_NAME_MAX, dfh); + if (res != TEE_SUCCESS) + return res; + + op = (struct tee_fs_rpc_operation){ + .id = id, .num_params = 2, .params = { + [0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_FS_REMOVE, 0, 0), + [1] = THREAD_PARAM_MEMREF(IN, mobj, 0, TEE_FS_NAME_MAX), + } + }; + + return operation_commit(&op); +} diff --git a/optee_os/core/tee/tee_obj.c b/optee_os/core/tee/tee_obj.c new file mode 100644 index 0000000..2d091e4 --- /dev/null +++ b/optee_os/core/tee/tee_obj.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void tee_obj_add(struct user_ta_ctx *utc, struct tee_obj *o) +{ + TAILQ_INSERT_TAIL(&utc->objects, o, link); +} + +TEE_Result tee_obj_get(struct user_ta_ctx *utc, vaddr_t obj_id, + struct tee_obj **obj) +{ + struct tee_obj *o; + + TAILQ_FOREACH(o, &utc->objects, link) { + if (obj_id == (vaddr_t)o) { + *obj = o; + return TEE_SUCCESS; + } + } + return TEE_ERROR_BAD_STATE; +} + +void tee_obj_close(struct user_ta_ctx *utc, struct tee_obj *o) +{ + TAILQ_REMOVE(&utc->objects, o, link); + + if ((o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT)) { + o->pobj->fops->close(&o->fh); + tee_pobj_release(o->pobj); + } + + tee_obj_free(o); +} + +void tee_obj_close_all(struct user_ta_ctx *utc) +{ + struct tee_obj_head *objects = &utc->objects; + + while (!TAILQ_EMPTY(objects)) + tee_obj_close(utc, TAILQ_FIRST(objects)); +} + +TEE_Result tee_obj_verify(struct tee_ta_session *sess, struct tee_obj *o) +{ + TEE_Result res; + const struct tee_file_operations *fops = o->pobj->fops; + struct tee_file_handle *fh = NULL; + + if (!fops) + return TEE_ERROR_STORAGE_NOT_AVAILABLE; + + res = fops->open(o->pobj, NULL, &fh); + if (res == TEE_ERROR_CORRUPT_OBJECT) { + EMSG("Object corrupt"); + fops->remove(o->pobj); + tee_obj_close(to_user_ta_ctx(sess->ts_sess.ctx), o); + } + + fops->close(&fh); + return res; +} + +struct tee_obj *tee_obj_alloc(void) +{ + return calloc(1, sizeof(struct tee_obj)); +} + +void tee_obj_free(struct tee_obj *o) +{ + if (o) { + tee_obj_attr_free(o); + free(o->attr); + free(o); + } +} diff --git a/optee_os/core/tee/tee_pobj.c b/optee_os/core/tee/tee_pobj.c new file mode 100644 index 0000000..d8a2d26 --- /dev/null +++ b/optee_os/core/tee/tee_pobj.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include + +static TAILQ_HEAD(tee_pobjs, tee_pobj) tee_pobjs = + TAILQ_HEAD_INITIALIZER(tee_pobjs); +static struct mutex pobjs_mutex = MUTEX_INITIALIZER; + +static TEE_Result tee_pobj_check_access(uint32_t oflags, uint32_t nflags) +{ + /* meta is exclusive */ + if ((oflags & TEE_DATA_FLAG_ACCESS_WRITE_META) || + (nflags & TEE_DATA_FLAG_ACCESS_WRITE_META)) + return TEE_ERROR_ACCESS_CONFLICT; + + /* + * Excerpt of TEE Internal Core API Specification v1.1: + * If more than one handle is opened on the same object, and if any + * of these object handles was opened with the flag + * TEE_DATA_FLAG_ACCESS_READ, then all the object handles MUST have been + * opened with the flag TEE_DATA_FLAG_SHARE_READ + */ + if (((oflags & TEE_DATA_FLAG_ACCESS_READ) || + (nflags & TEE_DATA_FLAG_ACCESS_READ)) && + !((nflags & TEE_DATA_FLAG_SHARE_READ) && + (oflags & TEE_DATA_FLAG_SHARE_READ))) + return TEE_ERROR_ACCESS_CONFLICT; + + /* + * Excerpt of TEE Internal Core API Specification v1.1: + * An object can be opened with only share flags, which locks the access + * to an object against a given mode. + * An object can be opened with no flag set, which completely locks all + * subsequent attempts to access the object + */ + if ((nflags & TEE_DATA_FLAG_SHARE_READ) != + (oflags & TEE_DATA_FLAG_SHARE_READ)) + return TEE_ERROR_ACCESS_CONFLICT; + + /* Same on WRITE access */ + if (((oflags & TEE_DATA_FLAG_ACCESS_WRITE) || + (nflags & TEE_DATA_FLAG_ACCESS_WRITE)) && + !((nflags & TEE_DATA_FLAG_SHARE_WRITE) && + (oflags & TEE_DATA_FLAG_SHARE_WRITE))) + return TEE_ERROR_ACCESS_CONFLICT; + if ((nflags & TEE_DATA_FLAG_SHARE_WRITE) != + (oflags & TEE_DATA_FLAG_SHARE_WRITE)) + return TEE_ERROR_ACCESS_CONFLICT; + + return TEE_SUCCESS; +} + +TEE_Result tee_pobj_get(TEE_UUID *uuid, void *obj_id, uint32_t obj_id_len, + uint32_t flags, enum tee_pobj_usage usage, + const struct tee_file_operations *fops, + struct tee_pobj **obj) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_pobj *o = NULL; + + *obj = NULL; + + mutex_lock(&pobjs_mutex); + /* Check if file is open */ + TAILQ_FOREACH(o, &tee_pobjs, link) { + if ((obj_id_len == o->obj_id_len) && + (memcmp(obj_id, o->obj_id, obj_id_len) == 0) && + (memcmp(uuid, &o->uuid, sizeof(TEE_UUID)) == 0) && + (fops == o->fops)) { + *obj = o; + } + } + + if (*obj) { + if (usage == TEE_POBJ_USAGE_ENUM) { + (*obj)->refcnt++; + goto out; + } + if ((*obj)->creating || (usage == TEE_POBJ_USAGE_CREATE && + !(flags & TEE_DATA_FLAG_OVERWRITE))) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto out; + } + res = tee_pobj_check_access((*obj)->flags, flags); + if (res == TEE_SUCCESS) + (*obj)->refcnt++; + goto out; + } + + /* new file */ + o = calloc(1, sizeof(struct tee_pobj)); + if (!o) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + o->refcnt = 1; + memcpy(&o->uuid, uuid, sizeof(TEE_UUID)); + o->flags = flags; + o->fops = fops; + + if (usage == TEE_POBJ_USAGE_CREATE) { + o->temporary = true; + o->creating = true; + } + + o->obj_id = malloc(obj_id_len); + if (o->obj_id == NULL) { + free(o); + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + memcpy(o->obj_id, obj_id, obj_id_len); + o->obj_id_len = obj_id_len; + + TAILQ_INSERT_TAIL(&tee_pobjs, o, link); + *obj = o; + + res = TEE_SUCCESS; +out: + if (res != TEE_SUCCESS) + *obj = NULL; + mutex_unlock(&pobjs_mutex); + return res; +} + +void tee_pobj_create_final(struct tee_pobj *po) +{ + mutex_lock(&pobjs_mutex); + po->temporary = false; + po->creating = false; + mutex_unlock(&pobjs_mutex); +} + +TEE_Result tee_pobj_release(struct tee_pobj *obj) +{ + if (obj == NULL) + return TEE_ERROR_BAD_PARAMETERS; + + mutex_lock(&pobjs_mutex); + obj->refcnt--; + if (obj->refcnt == 0) { + TAILQ_REMOVE(&tee_pobjs, obj, link); + free(obj->obj_id); + free(obj); + } + mutex_unlock(&pobjs_mutex); + + return TEE_SUCCESS; +} + +TEE_Result tee_pobj_rename(struct tee_pobj *obj, void *obj_id, + uint32_t obj_id_len) +{ + TEE_Result res = TEE_SUCCESS; + void *new_obj_id = NULL; + + if (obj == NULL || obj_id == NULL) + return TEE_ERROR_BAD_PARAMETERS; + + mutex_lock(&pobjs_mutex); + if (obj->refcnt != 1) { + res = TEE_ERROR_BAD_STATE; + goto exit; + } + + new_obj_id = malloc(obj_id_len); + if (new_obj_id == NULL) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto exit; + } + memcpy(new_obj_id, obj_id, obj_id_len); + + /* update internal data */ + free(obj->obj_id); + obj->obj_id = new_obj_id; + obj->obj_id_len = obj_id_len; + new_obj_id = NULL; + +exit: + mutex_unlock(&pobjs_mutex); + free(new_obj_id); + return res; +} diff --git a/optee_os/core/tee/tee_ree_fs.c b/optee_os/core/tee/tee_ree_fs.c new file mode 100644 index 0000000..71d5dcd --- /dev/null +++ b/optee_os/core/tee/tee_ree_fs.c @@ -0,0 +1,1096 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BLOCK_SHIFT 12 + +#define BLOCK_SIZE (1 << BLOCK_SHIFT) + +struct tee_fs_fd { + struct tee_fs_htree *ht; + int fd; + struct tee_fs_dirfile_fileh dfh; + const TEE_UUID *uuid; +}; + +struct tee_fs_dir { + struct tee_fs_dirfile_dirh *dirh; + int idx; + struct tee_fs_dirent d; + const TEE_UUID *uuid; +}; + +static int pos_to_block_num(int position) +{ + return position >> BLOCK_SHIFT; +} + +static struct mutex ree_fs_mutex = MUTEX_INITIALIZER; + +static void *get_tmp_block(void) +{ + return mempool_alloc(mempool_default, BLOCK_SIZE); +} + +static void put_tmp_block(void *tmp_block) +{ + mempool_free(mempool_default, tmp_block); +} + +static TEE_Result out_of_place_write(struct tee_fs_fd *fdp, size_t pos, + const void *buf_core, + const void *buf_user, size_t len) +{ + TEE_Result res; + size_t start_block_num = pos_to_block_num(pos); + size_t end_block_num = pos_to_block_num(pos + len - 1); + size_t remain_bytes = len; + uint8_t *data_core_ptr = (uint8_t *)buf_core; + uint8_t *data_user_ptr = (uint8_t *)buf_user; + uint8_t *block; + struct tee_fs_htree_meta *meta = tee_fs_htree_get_meta(fdp->ht); + + /* + * It doesn't make sense to call this function if nothing is to be + * written. This also guards against end_block_num getting an + * unexpected value when pos == 0 and len == 0. + */ + if (!len) + return TEE_ERROR_BAD_PARAMETERS; + + block = get_tmp_block(); + if (!block) + return TEE_ERROR_OUT_OF_MEMORY; + + while (start_block_num <= end_block_num) { + size_t offset = pos % BLOCK_SIZE; + size_t size_to_write = MIN(remain_bytes, (size_t)BLOCK_SIZE); + + if (size_to_write + offset > BLOCK_SIZE) + size_to_write = BLOCK_SIZE - offset; + + if (start_block_num * BLOCK_SIZE < + ROUNDUP(meta->length, BLOCK_SIZE)) { + res = tee_fs_htree_read_block(&fdp->ht, + start_block_num, block); + if (res != TEE_SUCCESS) + goto exit; + } else { + memset(block, 0, BLOCK_SIZE); + } + + if (data_core_ptr) { + memcpy(block + offset, data_core_ptr, size_to_write); + } else if (data_user_ptr) { + res = copy_from_user(block + offset, data_user_ptr, + size_to_write); + if (res) + return res; + } else { + memset(block + offset, 0, size_to_write); + } + + res = tee_fs_htree_write_block(&fdp->ht, start_block_num, + block); + if (res != TEE_SUCCESS) + goto exit; + + if (data_core_ptr) + data_core_ptr += size_to_write; + if (data_user_ptr) + data_user_ptr += size_to_write; + remain_bytes -= size_to_write; + start_block_num++; + pos += size_to_write; + } + + if (pos > meta->length) { + meta->length = pos; + tee_fs_htree_meta_set_dirty(fdp->ht); + } + +exit: + if (block) + put_tmp_block(block); + return res; +} + +static TEE_Result get_offs_size(enum tee_fs_htree_type type, size_t idx, + uint8_t vers, size_t *offs, size_t *size) +{ + const size_t node_size = sizeof(struct tee_fs_htree_node_image); + const size_t block_nodes = BLOCK_SIZE / (node_size * 2); + size_t pbn; + size_t bidx; + + assert(vers == 0 || vers == 1); + + /* + * File layout + * [demo with input: + * BLOCK_SIZE = 4096, + * node_size = 66, + * block_nodes = 4096/(66*2) = 31 ] + * + * phys block 0: + * tee_fs_htree_image vers 0 @ offs = 0 + * tee_fs_htree_image vers 1 @ offs = sizeof(tee_fs_htree_image) + * + * phys block 1: + * tee_fs_htree_node_image 0 vers 0 @ offs = 0 + * tee_fs_htree_node_image 0 vers 1 @ offs = node_size + * tee_fs_htree_node_image 1 vers 0 @ offs = node_size * 2 + * tee_fs_htree_node_image 1 vers 1 @ offs = node_size * 3 + * ... + * tee_fs_htree_node_image 30 vers 0 @ offs = node_size * 60 + * tee_fs_htree_node_image 30 vers 1 @ offs = node_size * 61 + * + * phys block 2: + * data block 0 vers 0 + * + * phys block 3: + * data block 0 vers 1 + * + * ... + * phys block 62: + * data block 30 vers 0 + * + * phys block 63: + * data block 30 vers 1 + * + * phys block 64: + * tee_fs_htree_node_image 31 vers 0 @ offs = 0 + * tee_fs_htree_node_image 31 vers 1 @ offs = node_size + * tee_fs_htree_node_image 32 vers 0 @ offs = node_size * 2 + * tee_fs_htree_node_image 32 vers 1 @ offs = node_size * 3 + * ... + * tee_fs_htree_node_image 61 vers 0 @ offs = node_size * 60 + * tee_fs_htree_node_image 61 vers 1 @ offs = node_size * 61 + * + * phys block 65: + * data block 31 vers 0 + * + * phys block 66: + * data block 31 vers 1 + * ... + */ + + switch (type) { + case TEE_FS_HTREE_TYPE_HEAD: + *offs = sizeof(struct tee_fs_htree_image) * vers; + *size = sizeof(struct tee_fs_htree_image); + return TEE_SUCCESS; + case TEE_FS_HTREE_TYPE_NODE: + pbn = 1 + ((idx / block_nodes) * block_nodes * 2); + *offs = pbn * BLOCK_SIZE + + 2 * node_size * (idx % block_nodes) + + node_size * vers; + *size = node_size; + return TEE_SUCCESS; + case TEE_FS_HTREE_TYPE_BLOCK: + bidx = 2 * idx + vers; + pbn = 2 + bidx + bidx / (block_nodes * 2 - 1); + *offs = pbn * BLOCK_SIZE; + *size = BLOCK_SIZE; + return TEE_SUCCESS; + default: + return TEE_ERROR_GENERIC; + } +} + +static TEE_Result ree_fs_rpc_read_init(void *aux, + struct tee_fs_rpc_operation *op, + enum tee_fs_htree_type type, size_t idx, + uint8_t vers, void **data) +{ + struct tee_fs_fd *fdp = aux; + TEE_Result res; + size_t offs; + size_t size; + + res = get_offs_size(type, idx, vers, &offs, &size); + if (res != TEE_SUCCESS) + return res; + + return tee_fs_rpc_read_init(op, OPTEE_RPC_CMD_FS, fdp->fd, + offs, size, data); +} + +static TEE_Result ree_fs_rpc_write_init(void *aux, + struct tee_fs_rpc_operation *op, + enum tee_fs_htree_type type, size_t idx, + uint8_t vers, void **data) +{ + struct tee_fs_fd *fdp = aux; + TEE_Result res; + size_t offs; + size_t size; + + res = get_offs_size(type, idx, vers, &offs, &size); + if (res != TEE_SUCCESS) + return res; + + return tee_fs_rpc_write_init(op, OPTEE_RPC_CMD_FS, fdp->fd, + offs, size, data); +} + +static const struct tee_fs_htree_storage ree_fs_storage_ops = { + .block_size = BLOCK_SIZE, + .rpc_read_init = ree_fs_rpc_read_init, + .rpc_read_final = tee_fs_rpc_read_final, + .rpc_write_init = ree_fs_rpc_write_init, + .rpc_write_final = tee_fs_rpc_write_final, +}; + +static TEE_Result ree_fs_ftruncate_internal(struct tee_fs_fd *fdp, + tee_fs_off_t new_file_len) +{ + TEE_Result res; + struct tee_fs_htree_meta *meta = tee_fs_htree_get_meta(fdp->ht); + + if ((size_t)new_file_len > meta->length) { + size_t ext_len = new_file_len - meta->length; + + res = out_of_place_write(fdp, meta->length, NULL, NULL, + ext_len); + if (res != TEE_SUCCESS) + return res; + } else { + size_t offs; + size_t sz; + + res = get_offs_size(TEE_FS_HTREE_TYPE_BLOCK, + ROUNDUP(new_file_len, BLOCK_SIZE) / + BLOCK_SIZE, 1, &offs, &sz); + if (res != TEE_SUCCESS) + return res; + + res = tee_fs_htree_truncate(&fdp->ht, + new_file_len / BLOCK_SIZE); + if (res != TEE_SUCCESS) + return res; + + res = tee_fs_rpc_truncate(OPTEE_RPC_CMD_FS, fdp->fd, + offs + sz); + if (res != TEE_SUCCESS) + return res; + + meta->length = new_file_len; + tee_fs_htree_meta_set_dirty(fdp->ht); + } + + return TEE_SUCCESS; +} + +static TEE_Result ree_fs_read_primitive(struct tee_file_handle *fh, size_t pos, + void *buf_core, void *buf_user, + size_t *len) +{ + TEE_Result res; + int start_block_num; + int end_block_num; + size_t remain_bytes; + uint8_t *data_core_ptr = buf_core; + uint8_t *data_user_ptr = buf_user; + uint8_t *block = NULL; + struct tee_fs_fd *fdp = (struct tee_fs_fd *)fh; + struct tee_fs_htree_meta *meta = tee_fs_htree_get_meta(fdp->ht); + + /* One of buf_core and buf_user must be NULL */ + assert(!buf_core || !buf_user); + + remain_bytes = *len; + if ((pos + remain_bytes) < remain_bytes || pos > meta->length) + remain_bytes = 0; + else if (pos + remain_bytes > meta->length) + remain_bytes = meta->length - pos; + + *len = remain_bytes; + + if (!remain_bytes) { + res = TEE_SUCCESS; + goto exit; + } + + start_block_num = pos_to_block_num(pos); + end_block_num = pos_to_block_num(pos + remain_bytes - 1); + + block = get_tmp_block(); + if (!block) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto exit; + } + + while (start_block_num <= end_block_num) { + size_t offset = pos % BLOCK_SIZE; + size_t size_to_read = MIN(remain_bytes, (size_t)BLOCK_SIZE); + + if (size_to_read + offset > BLOCK_SIZE) + size_to_read = BLOCK_SIZE - offset; + + res = tee_fs_htree_read_block(&fdp->ht, start_block_num, block); + if (res != TEE_SUCCESS) + goto exit; + + if (data_core_ptr) { + memcpy(data_core_ptr, block + offset, size_to_read); + data_core_ptr += size_to_read; + } else if (data_user_ptr) { + res = copy_to_user(data_user_ptr, block + offset, + size_to_read); + if (res) + goto exit; + data_user_ptr += size_to_read; + } + + remain_bytes -= size_to_read; + pos += size_to_read; + + start_block_num++; + } + res = TEE_SUCCESS; +exit: + if (block) + put_tmp_block(block); + return res; +} + +static TEE_Result ree_fs_read(struct tee_file_handle *fh, size_t pos, + void *buf_core, void *buf_user, size_t *len) +{ + TEE_Result res; + + mutex_lock(&ree_fs_mutex); + res = ree_fs_read_primitive(fh, pos, buf_core, buf_user, len); + mutex_unlock(&ree_fs_mutex); + + return res; +} + +static TEE_Result ree_fs_write_primitive(struct tee_file_handle *fh, size_t pos, + const void *buf_core, + const void *buf_user, size_t len) +{ + TEE_Result res; + struct tee_fs_fd *fdp = (struct tee_fs_fd *)fh; + size_t file_size; + + /* One of buf_core and buf_user must be NULL */ + assert(!buf_core || !buf_user); + + if (!len) + return TEE_SUCCESS; + + file_size = tee_fs_htree_get_meta(fdp->ht)->length; + + if ((pos + len) < len) + return TEE_ERROR_BAD_PARAMETERS; + + if (file_size < pos) { + res = ree_fs_ftruncate_internal(fdp, pos); + if (res != TEE_SUCCESS) + return res; + } + + return out_of_place_write(fdp, pos, buf_core, buf_user, len); +} + +static TEE_Result ree_fs_open_primitive(bool create, uint8_t *hash, + uint32_t min_counter, + const TEE_UUID *uuid, + struct tee_fs_dirfile_fileh *dfh, + struct tee_file_handle **fh) +{ + TEE_Result res; + struct tee_fs_fd *fdp; + + fdp = calloc(1, sizeof(struct tee_fs_fd)); + if (!fdp) + return TEE_ERROR_OUT_OF_MEMORY; + fdp->fd = -1; + fdp->uuid = uuid; + + if (create) + res = tee_fs_rpc_create_dfh(OPTEE_RPC_CMD_FS, + dfh, &fdp->fd); + else + res = tee_fs_rpc_open_dfh(OPTEE_RPC_CMD_FS, dfh, &fdp->fd); + + if (res != TEE_SUCCESS) + goto out; + + res = tee_fs_htree_open(create, hash, min_counter, uuid, + &ree_fs_storage_ops, fdp, &fdp->ht); +out: + if (res == TEE_SUCCESS) { + if (dfh) + fdp->dfh = *dfh; + else + fdp->dfh.idx = -1; + *fh = (struct tee_file_handle *)fdp; + } else { + if (res == TEE_ERROR_SECURITY) + DMSG("Secure storage corruption detected"); + if (fdp->fd != -1) + tee_fs_rpc_close(OPTEE_RPC_CMD_FS, fdp->fd); + if (create) + tee_fs_rpc_remove_dfh(OPTEE_RPC_CMD_FS, dfh); + free(fdp); + } + + return res; +} + +static void ree_fs_close_primitive(struct tee_file_handle *fh) +{ + struct tee_fs_fd *fdp = (struct tee_fs_fd *)fh; + + if (fdp) { + tee_fs_htree_close(&fdp->ht); + tee_fs_rpc_close(OPTEE_RPC_CMD_FS, fdp->fd); + free(fdp); + } +} + +static TEE_Result ree_dirf_commit_writes(struct tee_file_handle *fh, + uint8_t *hash, uint32_t *counter) +{ + TEE_Result res; + struct tee_fs_fd *fdp = (struct tee_fs_fd *)fh; + + res = tee_fs_htree_sync_to_storage(&fdp->ht, fdp->dfh.hash, counter); + + if (!res && hash) + memcpy(hash, fdp->dfh.hash, sizeof(fdp->dfh.hash)); + + return res; +} + +static TEE_Result dirf_read(struct tee_file_handle *fh, size_t pos, void *buf, + size_t *len) +{ + return ree_fs_read_primitive(fh, pos, buf, NULL, len); +} + +static TEE_Result dirf_write(struct tee_file_handle *fh, size_t pos, + const void *buf, size_t len) +{ + return ree_fs_write_primitive(fh, pos, buf, NULL, len); +} + +static const struct tee_fs_dirfile_operations ree_dirf_ops = { + .open = ree_fs_open_primitive, + .close = ree_fs_close_primitive, + .read = dirf_read, + .write = dirf_write, + .commit_writes = ree_dirf_commit_writes, +}; + +static struct tee_fs_dirfile_dirh *ree_fs_dirh; +static size_t ree_fs_dirh_refcount; + +#ifdef CFG_REE_FS_INTEGRITY_RPMB +static struct tee_file_handle *ree_fs_rpmb_fh; + +static TEE_Result open_dirh(struct tee_fs_dirfile_dirh **dirh) +{ + TEE_Result res; + uint8_t hash[TEE_FS_HTREE_HASH_SIZE]; + uint8_t *hashp = NULL; + const char fname[] = "dirfile.db.hash"; + + res = tee_rpmb_fs_raw_open(fname, false, &ree_fs_rpmb_fh); + if (!res) { + size_t l = sizeof(hash); + + res = rpmb_fs_ops.read(ree_fs_rpmb_fh, 0, hash, NULL, &l); + if (res) + return res; + if (l == sizeof(hash)) + hashp = hash; + } else if (res == TEE_ERROR_ITEM_NOT_FOUND) { + res = tee_rpmb_fs_raw_open(fname, true, &ree_fs_rpmb_fh); + } + if (res) + return res; + + res = tee_fs_dirfile_open(false, hashp, 0, &ree_dirf_ops, dirh); + + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + if (hashp) { + if (IS_ENABLED(CFG_REE_FS_ALLOW_RESET)) { + DMSG("dirf.db not found, clear hash in RPMB"); + res = rpmb_fs_ops.truncate(ree_fs_rpmb_fh, 0); + if (res) { + DMSG("Can't clear hash: %#"PRIx32, res); + res = TEE_ERROR_SECURITY; + goto out; + } + } else { + DMSG("dirf.db file not found"); + res = TEE_ERROR_SECURITY; + goto out; + } + } + + res = tee_fs_dirfile_open(true, NULL, 0, &ree_dirf_ops, dirh); + } + +out: + if (res) + rpmb_fs_ops.close(&ree_fs_rpmb_fh); + + return res; +} + +static TEE_Result commit_dirh_writes(struct tee_fs_dirfile_dirh *dirh) +{ + TEE_Result res; + uint8_t hash[TEE_FS_HTREE_HASH_SIZE]; + + res = tee_fs_dirfile_commit_writes(dirh, hash, NULL); + if (res) + return res; + return rpmb_fs_ops.write(ree_fs_rpmb_fh, 0, hash, NULL, sizeof(hash)); +} + +static void close_dirh(struct tee_fs_dirfile_dirh **dirh) +{ + tee_fs_dirfile_close(*dirh); + *dirh = NULL; + rpmb_fs_ops.close(&ree_fs_rpmb_fh); +} + +#else /*!CFG_REE_FS_INTEGRITY_RPMB*/ +static TEE_Result open_dirh(struct tee_fs_dirfile_dirh **dirh) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t min_counter = 0; + + res = nv_counter_get_ree_fs(&min_counter); + if (res) { + static bool once; + + if (res != TEE_ERROR_NOT_IMPLEMENTED || + !IS_ENABLED(CFG_WARN_INSECURE)) + return res; + + if (!once) { + IMSG("WARNING (insecure configuration): Failed to get monotonic counter for REE FS, using 0"); + once = true; + } + min_counter = 0; + } + res = tee_fs_dirfile_open(false, NULL, min_counter, &ree_dirf_ops, + dirh); + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + if (min_counter) { + if (!IS_ENABLED(CFG_REE_FS_ALLOW_RESET)) { + DMSG("dirf.db file not found"); + return TEE_ERROR_SECURITY; + } + DMSG("dirf.db not found, initializing with a non-zero monotonic counter"); + } + return tee_fs_dirfile_open(true, NULL, min_counter, + &ree_dirf_ops, dirh); + } + + return res; +} + +static TEE_Result commit_dirh_writes(struct tee_fs_dirfile_dirh *dirh) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t counter = 0; + + res = tee_fs_dirfile_commit_writes(dirh, NULL, &counter); + if (res) + return res; + res = nv_counter_incr_ree_fs_to(counter); + if (res == TEE_ERROR_NOT_IMPLEMENTED && IS_ENABLED(CFG_WARN_INSECURE)) { + static bool once; + + if (!once) { + IMSG("WARNING (insecure configuration): Failed to commit dirh counter %"PRIu32, counter); + once = true; + } + return TEE_SUCCESS; + } + return res; +} + +static void close_dirh(struct tee_fs_dirfile_dirh **dirh) +{ + tee_fs_dirfile_close(*dirh); + *dirh = NULL; +} +#endif /*!CFG_REE_FS_INTEGRITY_RPMB*/ + +static TEE_Result get_dirh(struct tee_fs_dirfile_dirh **dirh) +{ + if (!ree_fs_dirh) { + TEE_Result res = open_dirh(&ree_fs_dirh); + + if (res) { + *dirh = NULL; + return res; + } + } + ree_fs_dirh_refcount++; + assert(ree_fs_dirh); + assert(ree_fs_dirh_refcount); + *dirh = ree_fs_dirh; + return TEE_SUCCESS; +} + +static void put_dirh_primitive(bool close) +{ + assert(ree_fs_dirh_refcount); + + /* + * During the execution of one of the ree_fs_ops ree_fs_dirh is + * guareteed to be a valid pointer. But when the fop has returned + * another thread may get an error or something causing that fop + * to do a put with close=1. + * + * For all fops but ree_fs_close() there's a call to get_dirh() to + * get a new dirh which will open it again if it was closed before. + * But in the ree_fs_close() case there's no call to get_dirh() + * only to this function, put_dirh_primitive(), and in this case + * ree_fs_dirh may actually be NULL. + */ + ree_fs_dirh_refcount--; + if (ree_fs_dirh && (!ree_fs_dirh_refcount || close)) + close_dirh(&ree_fs_dirh); +} + +static void put_dirh(struct tee_fs_dirfile_dirh *dirh, bool close) +{ + if (dirh) { + assert(dirh == ree_fs_dirh); + put_dirh_primitive(close); + } +} + +static TEE_Result ree_fs_open(struct tee_pobj *po, size_t *size, + struct tee_file_handle **fh) +{ + TEE_Result res; + struct tee_fs_dirfile_dirh *dirh = NULL; + struct tee_fs_dirfile_fileh dfh; + + mutex_lock(&ree_fs_mutex); + + res = get_dirh(&dirh); + if (res != TEE_SUCCESS) + goto out; + + res = tee_fs_dirfile_find(dirh, &po->uuid, po->obj_id, po->obj_id_len, + &dfh); + if (res != TEE_SUCCESS) + goto out; + + res = ree_fs_open_primitive(false, dfh.hash, 0, &po->uuid, &dfh, fh); + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + /* + * If the object isn't found someone has tampered with it, + * treat it as corrupt. + */ + res = TEE_ERROR_CORRUPT_OBJECT; + } else if (!res && size) { + struct tee_fs_fd *fdp = (struct tee_fs_fd *)*fh; + + *size = tee_fs_htree_get_meta(fdp->ht)->length; + } + +out: + if (res) + put_dirh(dirh, true); + mutex_unlock(&ree_fs_mutex); + + return res; +} + +static TEE_Result set_name(struct tee_fs_dirfile_dirh *dirh, + struct tee_fs_fd *fdp, struct tee_pobj *po, + bool overwrite) +{ + TEE_Result res; + bool have_old_dfh = false; + struct tee_fs_dirfile_fileh old_dfh = { .idx = -1 }; + + res = tee_fs_dirfile_find(dirh, &po->uuid, po->obj_id, po->obj_id_len, + &old_dfh); + if (!overwrite && !res) + return TEE_ERROR_ACCESS_CONFLICT; + + if (!res) + have_old_dfh = true; + + /* + * If old_dfh wasn't found, the idx will be -1 and + * tee_fs_dirfile_rename() will allocate a new index. + */ + fdp->dfh.idx = old_dfh.idx; + old_dfh.idx = -1; + res = tee_fs_dirfile_rename(dirh, &po->uuid, &fdp->dfh, + po->obj_id, po->obj_id_len); + if (res) + return res; + + res = commit_dirh_writes(dirh); + if (res) + return res; + + if (have_old_dfh) + tee_fs_rpc_remove_dfh(OPTEE_RPC_CMD_FS, &old_dfh); + + return TEE_SUCCESS; +} + +static void ree_fs_close(struct tee_file_handle **fh) +{ + if (*fh) { + mutex_lock(&ree_fs_mutex); + put_dirh_primitive(false); + ree_fs_close_primitive(*fh); + *fh = NULL; + mutex_unlock(&ree_fs_mutex); + + } +} + +static TEE_Result ree_fs_create(struct tee_pobj *po, bool overwrite, + const void *head, size_t head_size, + const void *attr, size_t attr_size, + const void *data_core, const void *data_user, + size_t data_size, struct tee_file_handle **fh) +{ + struct tee_fs_fd *fdp; + struct tee_fs_dirfile_dirh *dirh = NULL; + struct tee_fs_dirfile_fileh dfh; + TEE_Result res; + size_t pos = 0; + + /* One of data_core and data_user must be NULL */ + assert(!data_core || !data_user); + + *fh = NULL; + mutex_lock(&ree_fs_mutex); + + res = get_dirh(&dirh); + if (res) + goto out; + + res = tee_fs_dirfile_get_tmp(dirh, &dfh); + if (res) + goto out; + + res = ree_fs_open_primitive(true, dfh.hash, 0, &po->uuid, &dfh, fh); + if (res) + goto out; + + if (head && head_size) { + res = ree_fs_write_primitive(*fh, pos, head, NULL, head_size); + if (res) + goto out; + pos += head_size; + } + + if (attr && attr_size) { + res = ree_fs_write_primitive(*fh, pos, attr, NULL, attr_size); + if (res) + goto out; + pos += attr_size; + } + + if ((data_core || data_user) && data_size) { + res = ree_fs_write_primitive(*fh, pos, data_core, data_user, + data_size); + if (res) + goto out; + } + + fdp = (struct tee_fs_fd *)*fh; + res = tee_fs_htree_sync_to_storage(&fdp->ht, fdp->dfh.hash, NULL); + if (res) + goto out; + + res = set_name(dirh, fdp, po, overwrite); +out: + if (res) { + put_dirh(dirh, true); + if (*fh) { + ree_fs_close_primitive(*fh); + *fh = NULL; + tee_fs_rpc_remove_dfh(OPTEE_RPC_CMD_FS, &dfh); + } + } + mutex_unlock(&ree_fs_mutex); + + return res; +} + +static TEE_Result ree_fs_write(struct tee_file_handle *fh, size_t pos, + const void *buf_core, const void *buf_user, + size_t len) +{ + TEE_Result res; + struct tee_fs_dirfile_dirh *dirh = NULL; + struct tee_fs_fd *fdp = (struct tee_fs_fd *)fh; + + /* One of buf_core and buf_user must be NULL */ + assert(!buf_core || !buf_user); + + mutex_lock(&ree_fs_mutex); + + res = get_dirh(&dirh); + if (res) + goto out; + + res = ree_fs_write_primitive(fh, pos, buf_core, buf_user, len); + if (res) + goto out; + + res = tee_fs_htree_sync_to_storage(&fdp->ht, fdp->dfh.hash, NULL); + if (res) + goto out; + + res = tee_fs_dirfile_update_hash(dirh, &fdp->dfh); + if (res) + goto out; + res = commit_dirh_writes(dirh); +out: + put_dirh(dirh, res); + mutex_unlock(&ree_fs_mutex); + + return res; +} + +static TEE_Result ree_fs_rename(struct tee_pobj *old, struct tee_pobj *new, + bool overwrite) +{ + TEE_Result res; + struct tee_fs_dirfile_dirh *dirh = NULL; + struct tee_fs_dirfile_fileh dfh; + struct tee_fs_dirfile_fileh remove_dfh = { .idx = -1 }; + + if (!new) + return TEE_ERROR_BAD_PARAMETERS; + + mutex_lock(&ree_fs_mutex); + res = get_dirh(&dirh); + if (res) + goto out; + + res = tee_fs_dirfile_find(dirh, &new->uuid, new->obj_id, + new->obj_id_len, &remove_dfh); + if (!res && !overwrite) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto out; + } + + res = tee_fs_dirfile_find(dirh, &old->uuid, old->obj_id, + old->obj_id_len, &dfh); + if (res) + goto out; + + res = tee_fs_dirfile_rename(dirh, &new->uuid, &dfh, new->obj_id, + new->obj_id_len); + if (res) + goto out; + + if (remove_dfh.idx != -1) { + res = tee_fs_dirfile_remove(dirh, &remove_dfh); + if (res) + goto out; + } + + res = commit_dirh_writes(dirh); + if (res) + goto out; + + if (remove_dfh.idx != -1) + tee_fs_rpc_remove_dfh(OPTEE_RPC_CMD_FS, &remove_dfh); + +out: + put_dirh(dirh, res); + mutex_unlock(&ree_fs_mutex); + + return res; + +} + +static TEE_Result ree_fs_remove(struct tee_pobj *po) +{ + TEE_Result res; + struct tee_fs_dirfile_dirh *dirh = NULL; + struct tee_fs_dirfile_fileh dfh; + + mutex_lock(&ree_fs_mutex); + res = get_dirh(&dirh); + if (res) + goto out; + + res = tee_fs_dirfile_find(dirh, &po->uuid, po->obj_id, po->obj_id_len, + &dfh); + if (res) + goto out; + + res = tee_fs_dirfile_remove(dirh, &dfh); + if (res) + goto out; + + res = commit_dirh_writes(dirh); + if (res) + goto out; + + tee_fs_rpc_remove_dfh(OPTEE_RPC_CMD_FS, &dfh); + + assert(tee_fs_dirfile_find(dirh, &po->uuid, po->obj_id, po->obj_id_len, + &dfh)); +out: + put_dirh(dirh, res); + mutex_unlock(&ree_fs_mutex); + + return res; +} + +static TEE_Result ree_fs_truncate(struct tee_file_handle *fh, size_t len) +{ + TEE_Result res; + struct tee_fs_dirfile_dirh *dirh = NULL; + struct tee_fs_fd *fdp = (struct tee_fs_fd *)fh; + + mutex_lock(&ree_fs_mutex); + + res = get_dirh(&dirh); + if (res) + goto out; + + res = ree_fs_ftruncate_internal(fdp, len); + if (res) + goto out; + + res = tee_fs_htree_sync_to_storage(&fdp->ht, fdp->dfh.hash, NULL); + if (res) + goto out; + + res = tee_fs_dirfile_update_hash(dirh, &fdp->dfh); + if (res) + goto out; + res = commit_dirh_writes(dirh); +out: + put_dirh(dirh, res); + mutex_unlock(&ree_fs_mutex); + + return res; +} + +static TEE_Result ree_fs_opendir_rpc(const TEE_UUID *uuid, + struct tee_fs_dir **dir) + +{ + TEE_Result res; + struct tee_fs_dir *d = calloc(1, sizeof(*d)); + + if (!d) + return TEE_ERROR_OUT_OF_MEMORY; + + d->uuid = uuid; + + mutex_lock(&ree_fs_mutex); + + res = get_dirh(&d->dirh); + if (res) + goto out; + + /* See that there's at least one file */ + d->idx = -1; + d->d.oidlen = sizeof(d->d.oid); + res = tee_fs_dirfile_get_next(d->dirh, d->uuid, &d->idx, d->d.oid, + &d->d.oidlen); + d->idx = -1; + +out: + if (!res) { + *dir = d; + } else { + if (d) + put_dirh(d->dirh, false); + free(d); + } + mutex_unlock(&ree_fs_mutex); + + return res; +} + +static void ree_fs_closedir_rpc(struct tee_fs_dir *d) +{ + if (d) { + mutex_lock(&ree_fs_mutex); + + put_dirh(d->dirh, false); + free(d); + + mutex_unlock(&ree_fs_mutex); + } +} + +static TEE_Result ree_fs_readdir_rpc(struct tee_fs_dir *d, + struct tee_fs_dirent **ent) +{ + TEE_Result res; + + mutex_lock(&ree_fs_mutex); + + d->d.oidlen = sizeof(d->d.oid); + res = tee_fs_dirfile_get_next(d->dirh, d->uuid, &d->idx, d->d.oid, + &d->d.oidlen); + if (res == TEE_SUCCESS) + *ent = &d->d; + + mutex_unlock(&ree_fs_mutex); + + return res; +} + +const struct tee_file_operations ree_fs_ops = { + .open = ree_fs_open, + .create = ree_fs_create, + .close = ree_fs_close, + .read = ree_fs_read, + .write = ree_fs_write, + .truncate = ree_fs_truncate, + .rename = ree_fs_rename, + .remove = ree_fs_remove, + .opendir = ree_fs_opendir_rpc, + .closedir = ree_fs_closedir_rpc, + .readdir = ree_fs_readdir_rpc, +}; diff --git a/optee_os/core/tee/tee_rpmb_fs.c b/optee_os/core/tee/tee_rpmb_fs.c new file mode 100644 index 0000000..516e3be --- /dev/null +++ b/optee_os/core/tee/tee_rpmb_fs.c @@ -0,0 +1,3152 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RPMB_STORAGE_START_ADDRESS 0 +#define RPMB_FS_FAT_START_ADDRESS 512 +#define RPMB_BLOCK_SIZE_SHIFT 8 +#define RPMB_CID_PRV_OFFSET 9 +#define RPMB_CID_CRC_OFFSET 15 + +#define RPMB_FS_MAGIC 0x52504D42 +#define FS_VERSION 2 + +#define FILE_IS_ACTIVE (1u << 0) +#define FILE_IS_LAST_ENTRY (1u << 1) + +#define TEE_RPMB_FS_FILENAME_LENGTH 224 + +#define TMP_BLOCK_SIZE 4096U + +#define RPMB_MAX_RETRIES 10 + +/** + * Utilized when caching is enabled, i.e., when CFG_RPMB_FS_CACHE_ENTRIES > 0. + * Cache size + the number of entries that are repeatedly read in and buffered + * once the cache is full. + */ +#define RPMB_BUF_MAX_ENTRIES (CFG_RPMB_FS_CACHE_ENTRIES + \ + CFG_RPMB_FS_RD_ENTRIES) + +/** + * FS parameters: Information often used by internal functions. + * fat_start_address will be set by rpmb_fs_setup(). + * rpmb_fs_parameters can be read by any other function. + */ +struct rpmb_fs_parameters { + uint32_t fat_start_address; + uint32_t max_rpmb_address; +}; + +/** + * File entry for a single file in a RPMB_FS partition. + */ +struct rpmb_fat_entry { + uint32_t start_address; + uint32_t data_size; + uint32_t flags; + uint32_t write_counter; + uint8_t fek[TEE_FS_KM_FEK_SIZE]; + char filename[TEE_RPMB_FS_FILENAME_LENGTH]; +}; + +/** + * Structure that describes buffered/cached FAT FS entries in RPMB storage. + * This structure is used in functions traversing the FAT FS. + */ +struct rpmb_fat_entry_dir { + /* + * Buffer storing the FAT FS entries read in from RPMB storage. It + * includes the optional cache entries (CFG_RPMB_FS_CACHE_ENTRIES) + * and entries temporary read for current FAT FS traversal + * (CFG_RPMB_FS_RD_ENTRIES) when not found from cached entries. + */ + struct rpmb_fat_entry *rpmb_fat_entry_buf; + /* Current index of FAT FS entry to read from buffer. */ + uint32_t idx_curr; + /* Total number of FAT FS entries in buffer. */ + uint32_t num_buffered; + /* Total number of FAT FS entries read during traversal. */ + uint32_t num_total_read; + /* Indicates that last FAT FS entry was read. */ + bool last_reached; +}; + +/** + * FAT entry context with reference to a FAT entry and its + * location in RPMB. + */ +struct rpmb_file_handle { + struct rpmb_fat_entry fat_entry; + const TEE_UUID *uuid; + char filename[TEE_RPMB_FS_FILENAME_LENGTH]; + /* Address for current entry in RPMB */ + uint32_t rpmb_fat_address; +}; + +/** + * RPMB_FS partition data + */ +struct rpmb_fs_partition { + uint32_t rpmb_fs_magic; + uint32_t fs_version; + uint32_t write_counter; + uint32_t fat_start_address; + /* Do not use reserved[] for other purpose than partition data. */ + uint8_t reserved[112]; +}; + +/** + * A node in a list of directory entries. + */ +struct tee_rpmb_fs_dirent { + struct tee_fs_dirent entry; + SIMPLEQ_ENTRY(tee_rpmb_fs_dirent) link; +}; + +/** + * The RPMB directory representation. It contains a queue of + * RPMB directory entries: 'next'. + * The current pointer points to the last directory entry + * returned by readdir(). + */ +struct tee_fs_dir { + struct tee_rpmb_fs_dirent *current; + /* */ + SIMPLEQ_HEAD(next_head, tee_rpmb_fs_dirent) next; +}; + +static struct rpmb_fs_parameters *fs_par; +static struct rpmb_fat_entry_dir *fat_entry_dir; + +/* + * Lower interface to RPMB device + */ + +#define RPMB_DATA_OFFSET (RPMB_STUFF_DATA_SIZE + RPMB_KEY_MAC_SIZE) +#define RPMB_MAC_PROTECT_DATA_SIZE (RPMB_DATA_FRAME_SIZE - RPMB_DATA_OFFSET) + +#define RPMB_MSG_TYPE_REQ_AUTH_KEY_PROGRAM 0x0001 +#define RPMB_MSG_TYPE_REQ_WRITE_COUNTER_VAL_READ 0x0002 +#define RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE 0x0003 +#define RPMB_MSG_TYPE_REQ_AUTH_DATA_READ 0x0004 +#define RPMB_MSG_TYPE_REQ_RESULT_READ 0x0005 +#define RPMB_MSG_TYPE_RESP_AUTH_KEY_PROGRAM 0x0100 +#define RPMB_MSG_TYPE_RESP_WRITE_COUNTER_VAL_READ 0x0200 +#define RPMB_MSG_TYPE_RESP_AUTH_DATA_WRITE 0x0300 +#define RPMB_MSG_TYPE_RESP_AUTH_DATA_READ 0x0400 + +#define RPMB_STUFF_DATA_SIZE 196 +#define RPMB_KEY_MAC_SIZE 32 +#define RPMB_DATA_SIZE 256 +#define RPMB_NONCE_SIZE 16 +#define RPMB_DATA_FRAME_SIZE 512 + +#define RPMB_RESULT_OK 0x00 +#define RPMB_RESULT_GENERAL_FAILURE 0x01 +#define RPMB_RESULT_AUTH_FAILURE 0x02 +#define RPMB_RESULT_COUNTER_FAILURE 0x03 +#define RPMB_RESULT_ADDRESS_FAILURE 0x04 +#define RPMB_RESULT_WRITE_FAILURE 0x05 +#define RPMB_RESULT_READ_FAILURE 0x06 +#define RPMB_RESULT_AUTH_KEY_NOT_PROGRAMMED 0x07 +#define RPMB_RESULT_MASK 0x3F +#define RPMB_RESULT_WR_CNT_EXPIRED 0x80 + +/* RPMB internal commands */ +#define RPMB_CMD_DATA_REQ 0x00 +#define RPMB_CMD_GET_DEV_INFO 0x01 + +#define RPMB_SIZE_SINGLE (128 * 1024) + +/* Error codes for get_dev_info request/response. */ +#define RPMB_CMD_GET_DEV_INFO_RET_OK 0x00 +#define RPMB_CMD_GET_DEV_INFO_RET_ERROR 0x01 + +struct rpmb_data_frame { + uint8_t stuff_bytes[RPMB_STUFF_DATA_SIZE]; + uint8_t key_mac[RPMB_KEY_MAC_SIZE]; + uint8_t data[RPMB_DATA_SIZE]; + uint8_t nonce[RPMB_NONCE_SIZE]; + uint8_t write_counter[4]; + uint8_t address[2]; + uint8_t block_count[2]; + uint8_t op_result[2]; + uint8_t msg_type[2]; +}; + +struct rpmb_req { + uint16_t cmd; + uint16_t dev_id; + uint16_t block_count; + /* variable length of data */ + /* uint8_t data[]; REMOVED! */ +}; + +#define TEE_RPMB_REQ_DATA(req) \ + ((void *)((struct rpmb_req *)(req) + 1)) + +struct rpmb_raw_data { + uint16_t msg_type; + uint16_t *op_result; + uint16_t *block_count; + uint16_t *blk_idx; + uint32_t *write_counter; + uint8_t *nonce; + uint8_t *key_mac; + uint8_t *data; + /* data length to read or write */ + uint32_t len; + /* Byte address offset in the first block involved */ + uint8_t byte_offset; +}; + +#define RPMB_EMMC_CID_SIZE 16 +struct rpmb_dev_info { + uint8_t cid[RPMB_EMMC_CID_SIZE]; + /* EXT CSD-slice 168 "RPMB Size" */ + uint8_t rpmb_size_mult; + /* EXT CSD-slice 222 "Reliable Write Sector Count" */ + uint8_t rel_wr_sec_c; + /* Check the ret code and accept the data only if it is OK. */ + uint8_t ret_code; +}; + +/* + * Struct for rpmb context data. + * + * @key RPMB key. + * @cid eMMC card ID. + * @wr_cnt Current write counter. + * @max_blk_idx The highest block index supported by current device. + * @rel_wr_blkcnt Max number of data blocks for each reliable write. + * @dev_id Device ID of the eMMC device. + * @wr_cnt_synced Flag indicating if write counter is synced to RPMB. + * @key_derived Flag indicating if key has been generated. + * @key_verified Flag indicating the key generated is verified ok. + * @dev_info_synced Flag indicating if dev info has been retrieved from RPMB. + */ +struct tee_rpmb_ctx { + uint8_t key[RPMB_KEY_MAC_SIZE]; + uint8_t cid[RPMB_EMMC_CID_SIZE]; + uint32_t wr_cnt; + uint16_t max_blk_idx; + uint16_t rel_wr_blkcnt; + uint16_t dev_id; + bool wr_cnt_synced; + bool key_derived; + bool key_verified; + bool dev_info_synced; +}; + +static struct tee_rpmb_ctx *rpmb_ctx; + +/* If set to true, don't try to access RPMB until rebooted */ +static bool rpmb_dead; + +/* + * Mutex to serialize the operations exported by this file. + * It protects rpmb_ctx and prevents overlapping operations on eMMC devices with + * different IDs. + */ +static struct mutex rpmb_mutex = MUTEX_INITIALIZER; + +#ifdef CFG_RPMB_TESTKEY + +static const uint8_t rpmb_test_key[RPMB_KEY_MAC_SIZE] = { + 0xD3, 0xEB, 0x3E, 0xC3, 0x6E, 0x33, 0x4C, 0x9F, + 0x98, 0x8C, 0xE2, 0xC0, 0xB8, 0x59, 0x54, 0x61, + 0x0D, 0x2B, 0xCF, 0x86, 0x64, 0x84, 0x4D, 0xF2, + 0xAB, 0x56, 0xE6, 0xC6, 0x1B, 0xB7, 0x01, 0xE4 +}; + +static TEE_Result tee_rpmb_key_gen(uint16_t dev_id __unused, + uint8_t *key, uint32_t len) +{ + TEE_Result res = TEE_SUCCESS; + + if (!key || RPMB_KEY_MAC_SIZE != len) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + DMSG("RPMB: Using test key"); + memcpy(key, rpmb_test_key, RPMB_KEY_MAC_SIZE); + +out: + return res; +} + +#else /* !CFG_RPMB_TESTKEY */ + +static TEE_Result tee_rpmb_key_gen(uint16_t dev_id __unused, + uint8_t *key, uint32_t len) +{ + uint8_t message[RPMB_EMMC_CID_SIZE]; + + if (!key || RPMB_KEY_MAC_SIZE != len) + return TEE_ERROR_BAD_PARAMETERS; + + IMSG("RPMB: Using generated key"); + + /* + * PRV/CRC would be changed when doing eMMC FFU + * The following fields should be masked off when deriving RPMB key + * + * CID [55: 48]: PRV (Product revision) + * CID [07: 01]: CRC (CRC7 checksum) + * CID [00]: not used + */ + memcpy(message, rpmb_ctx->cid, RPMB_EMMC_CID_SIZE); + memset(message + RPMB_CID_PRV_OFFSET, 0, 1); + memset(message + RPMB_CID_CRC_OFFSET, 0, 1); + return huk_subkey_derive(HUK_SUBKEY_RPMB, message, sizeof(message), + key, len); +} + +#endif /* !CFG_RPMB_TESTKEY */ + +static void u32_to_bytes(uint32_t u32, uint8_t *bytes) +{ + *bytes = (uint8_t) (u32 >> 24); + *(bytes + 1) = (uint8_t) (u32 >> 16); + *(bytes + 2) = (uint8_t) (u32 >> 8); + *(bytes + 3) = (uint8_t) u32; +} + +static void bytes_to_u32(uint8_t *bytes, uint32_t *u32) +{ + *u32 = (uint32_t) ((*(bytes) << 24) + + (*(bytes + 1) << 16) + + (*(bytes + 2) << 8) + (*(bytes + 3))); +} + +static void u16_to_bytes(uint16_t u16, uint8_t *bytes) +{ + *bytes = (uint8_t) (u16 >> 8); + *(bytes + 1) = (uint8_t) u16; +} + +static void bytes_to_u16(uint8_t *bytes, uint16_t *u16) +{ + *u16 = (uint16_t) ((*bytes << 8) + *(bytes + 1)); +} + +static void get_op_result_bits(uint8_t *bytes, uint8_t *res) +{ + *res = *(bytes + 1) & RPMB_RESULT_MASK; +} + +static TEE_Result tee_rpmb_mac_calc(uint8_t *mac, uint32_t macsize, + uint8_t *key, uint32_t keysize, + struct rpmb_data_frame *datafrms, + uint16_t blkcnt) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int i; + void *ctx = NULL; + + if (!mac || !key || !datafrms) + return TEE_ERROR_BAD_PARAMETERS; + + res = crypto_mac_alloc_ctx(&ctx, TEE_ALG_HMAC_SHA256); + if (res) + return res; + + res = crypto_mac_init(ctx, key, keysize); + if (res != TEE_SUCCESS) + goto func_exit; + + for (i = 0; i < blkcnt; i++) { + res = crypto_mac_update(ctx, datafrms[i].data, + RPMB_MAC_PROTECT_DATA_SIZE); + if (res != TEE_SUCCESS) + goto func_exit; + } + + res = crypto_mac_final(ctx, mac, macsize); + if (res != TEE_SUCCESS) + goto func_exit; + + res = TEE_SUCCESS; + +func_exit: + crypto_mac_free_ctx(ctx); + return res; +} + +struct tee_rpmb_mem { + struct mobj *phreq_mobj; + struct mobj *phresp_mobj; + size_t req_size; + size_t resp_size; +}; + +static void tee_rpmb_free(struct tee_rpmb_mem *mem) +{ + if (!mem) + return; + + if (mem->phreq_mobj) { + thread_rpc_free_payload(mem->phreq_mobj); + mem->phreq_mobj = NULL; + } + if (mem->phresp_mobj) { + thread_rpc_free_payload(mem->phresp_mobj); + mem->phresp_mobj = NULL; + } +} + + +static TEE_Result tee_rpmb_alloc(size_t req_size, size_t resp_size, + struct tee_rpmb_mem *mem, void **req, void **resp) +{ + TEE_Result res = TEE_SUCCESS; + size_t req_s = ROUNDUP(req_size, sizeof(uint32_t)); + size_t resp_s = ROUNDUP(resp_size, sizeof(uint32_t)); + + if (!mem) + return TEE_ERROR_BAD_PARAMETERS; + + memset(mem, 0, sizeof(*mem)); + + mem->phreq_mobj = thread_rpc_alloc_payload(req_s); + mem->phresp_mobj = thread_rpc_alloc_payload(resp_s); + + if (!mem->phreq_mobj || !mem->phresp_mobj) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + *req = mobj_get_va(mem->phreq_mobj, 0, req_s); + *resp = mobj_get_va(mem->phresp_mobj, 0, resp_s); + if (!*req || !*resp) { + res = TEE_ERROR_GENERIC; + goto out; + } + + mem->req_size = req_size; + mem->resp_size = resp_size; + +out: + if (res != TEE_SUCCESS) + tee_rpmb_free(mem); + return res; +} + +static TEE_Result tee_rpmb_invoke(struct tee_rpmb_mem *mem) +{ + struct thread_param params[2] = { + [0] = THREAD_PARAM_MEMREF(IN, mem->phreq_mobj, 0, + mem->req_size), + [1] = THREAD_PARAM_MEMREF(OUT, mem->phresp_mobj, 0, + mem->resp_size), + }; + + return thread_rpc_cmd(OPTEE_RPC_CMD_RPMB, 2, params); +} + +static bool is_zero(const uint8_t *buf, size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) + if (buf[i]) + return false; + return true; +} + +static TEE_Result encrypt_block(uint8_t *out, const uint8_t *in, + uint16_t blk_idx, const uint8_t *fek, + const TEE_UUID *uuid) +{ + return tee_fs_crypt_block(uuid, out, in, RPMB_DATA_SIZE, + blk_idx, fek, TEE_MODE_ENCRYPT); +} + +static TEE_Result decrypt_block(uint8_t *out, const uint8_t *in, + uint16_t blk_idx, const uint8_t *fek, + const TEE_UUID *uuid) +{ + return tee_fs_crypt_block(uuid, out, in, RPMB_DATA_SIZE, + blk_idx, fek, TEE_MODE_DECRYPT); +} + +/* Decrypt/copy at most one block of data */ +static TEE_Result decrypt(uint8_t *out, const struct rpmb_data_frame *frm, + size_t size, size_t offset, + uint16_t blk_idx __maybe_unused, const uint8_t *fek, + const TEE_UUID *uuid) +{ + uint8_t *tmp __maybe_unused; + TEE_Result res = TEE_SUCCESS; + + + if ((size + offset < size) || (size + offset > RPMB_DATA_SIZE)) + panic("invalid size or offset"); + + if (!fek) { + /* Block is not encrypted (not a file data block) */ + memcpy(out, frm->data + offset, size); + } else if (is_zero(fek, TEE_FS_KM_FEK_SIZE)) { + /* The file was created with encryption disabled */ + return TEE_ERROR_SECURITY; + } else { + /* Block is encrypted */ + if (size < RPMB_DATA_SIZE) { + /* + * Since output buffer is not large enough to hold one + * block we must allocate a temporary buffer. + */ + tmp = malloc(RPMB_DATA_SIZE); + if (!tmp) + return TEE_ERROR_OUT_OF_MEMORY; + res = decrypt_block(tmp, frm->data, blk_idx, fek, uuid); + if (res == TEE_SUCCESS) + memcpy(out, tmp + offset, size); + free(tmp); + } else { + res = decrypt_block(out, frm->data, blk_idx, fek, uuid); + } + } + + return res; +} + +static TEE_Result tee_rpmb_req_pack(struct rpmb_req *req, + struct rpmb_raw_data *rawdata, + uint16_t nbr_frms, uint16_t dev_id, + const uint8_t *fek, const TEE_UUID *uuid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int i; + struct rpmb_data_frame *datafrm; + + if (!req || !rawdata || !nbr_frms) + return TEE_ERROR_BAD_PARAMETERS; + + /* + * Check write blockcount is not bigger than reliable write + * blockcount. + */ + if ((rawdata->msg_type == RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE) && + (nbr_frms > rpmb_ctx->rel_wr_blkcnt)) { + DMSG("wr_blkcnt(%d) > rel_wr_blkcnt(%d)", nbr_frms, + rpmb_ctx->rel_wr_blkcnt); + return TEE_ERROR_GENERIC; + } + + req->cmd = RPMB_CMD_DATA_REQ; + req->dev_id = dev_id; + + /* Allocate memory for construct all data packets and calculate MAC. */ + datafrm = calloc(nbr_frms, RPMB_DATA_FRAME_SIZE); + if (!datafrm) + return TEE_ERROR_OUT_OF_MEMORY; + + for (i = 0; i < nbr_frms; i++) { + u16_to_bytes(rawdata->msg_type, datafrm[i].msg_type); + + if (rawdata->block_count) + u16_to_bytes(*rawdata->block_count, + datafrm[i].block_count); + + if (rawdata->blk_idx) { + /* Check the block index is within range. */ + if ((*rawdata->blk_idx + nbr_frms - 1) > + rpmb_ctx->max_blk_idx) { + res = TEE_ERROR_GENERIC; + goto func_exit; + } + u16_to_bytes(*rawdata->blk_idx, datafrm[i].address); + } + + if (rawdata->write_counter) + u32_to_bytes(*rawdata->write_counter, + datafrm[i].write_counter); + + if (rawdata->nonce) + memcpy(datafrm[i].nonce, rawdata->nonce, + RPMB_NONCE_SIZE); + + if (rawdata->data) { + if (fek) { + res = encrypt_block(datafrm[i].data, + rawdata->data + + (i * RPMB_DATA_SIZE), + *rawdata->blk_idx + i, + fek, uuid); + if (res != TEE_SUCCESS) + goto func_exit; + } else { + memcpy(datafrm[i].data, + rawdata->data + (i * RPMB_DATA_SIZE), + RPMB_DATA_SIZE); + } + } + } + + if (rawdata->key_mac) { + if (rawdata->msg_type == RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE) { + res = + tee_rpmb_mac_calc(rawdata->key_mac, + RPMB_KEY_MAC_SIZE, rpmb_ctx->key, + RPMB_KEY_MAC_SIZE, datafrm, + nbr_frms); + if (res != TEE_SUCCESS) + goto func_exit; + } + memcpy(datafrm[nbr_frms - 1].key_mac, + rawdata->key_mac, RPMB_KEY_MAC_SIZE); + } + + memcpy(TEE_RPMB_REQ_DATA(req), datafrm, + nbr_frms * RPMB_DATA_FRAME_SIZE); + + if (IS_ENABLED(CFG_RPMB_FS_DEBUG_DATA)) { + for (i = 0; i < nbr_frms; i++) { + DMSG("Dumping data frame %d:", i); + DHEXDUMP((uint8_t *)&datafrm[i] + RPMB_STUFF_DATA_SIZE, + 512 - RPMB_STUFF_DATA_SIZE); + } + } + + res = TEE_SUCCESS; +func_exit: + free(datafrm); + return res; +} + +static TEE_Result data_cpy_mac_calc_1b(struct rpmb_raw_data *rawdata, + struct rpmb_data_frame *frm, + const uint8_t *fek, const TEE_UUID *uuid) +{ + TEE_Result res; + uint8_t *data; + uint16_t idx; + + if (rawdata->len + rawdata->byte_offset > RPMB_DATA_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + res = tee_rpmb_mac_calc(rawdata->key_mac, RPMB_KEY_MAC_SIZE, + rpmb_ctx->key, RPMB_KEY_MAC_SIZE, frm, 1); + if (res != TEE_SUCCESS) + return res; + + data = rawdata->data; + bytes_to_u16(frm->address, &idx); + + res = decrypt(data, frm, rawdata->len, rawdata->byte_offset, idx, fek, + uuid); + return res; +} + +static TEE_Result tee_rpmb_data_cpy_mac_calc(struct rpmb_data_frame *datafrm, + struct rpmb_raw_data *rawdata, + uint16_t nbr_frms, + struct rpmb_data_frame *lastfrm, + const uint8_t *fek, + const TEE_UUID *uuid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + int i; + void *ctx = NULL; + uint16_t offset; + uint32_t size; + uint8_t *data; + uint16_t start_idx; + struct rpmb_data_frame localfrm; + + if (!datafrm || !rawdata || !nbr_frms || !lastfrm) + return TEE_ERROR_BAD_PARAMETERS; + + if (nbr_frms == 1) + return data_cpy_mac_calc_1b(rawdata, lastfrm, fek, uuid); + + /* nbr_frms > 1 */ + + data = rawdata->data; + + res = crypto_mac_alloc_ctx(&ctx, TEE_ALG_HMAC_SHA256); + if (res) + goto func_exit; + + res = crypto_mac_init(ctx, rpmb_ctx->key, RPMB_KEY_MAC_SIZE); + if (res != TEE_SUCCESS) + goto func_exit; + + /* + * Note: JEDEC JESD84-B51: "In every packet the address is the start + * address of the full access (not address of the individual half a + * sector)" + */ + bytes_to_u16(lastfrm->address, &start_idx); + + for (i = 0; i < (nbr_frms - 1); i++) { + + /* + * By working on a local copy of the RPMB frame, we ensure that + * the data can not be modified after the MAC is computed but + * before the payload is decrypted/copied to the output buffer. + */ + memcpy(&localfrm, &datafrm[i], RPMB_DATA_FRAME_SIZE); + + res = crypto_mac_update(ctx, localfrm.data, + RPMB_MAC_PROTECT_DATA_SIZE); + if (res != TEE_SUCCESS) + goto func_exit; + + if (i == 0) { + /* First block */ + offset = rawdata->byte_offset; + size = RPMB_DATA_SIZE - offset; + } else { + /* Middle blocks */ + size = RPMB_DATA_SIZE; + offset = 0; + } + + res = decrypt(data, &localfrm, size, offset, start_idx + i, + fek, uuid); + if (res != TEE_SUCCESS) + goto func_exit; + + data += size; + } + + /* Last block */ + size = (rawdata->len + rawdata->byte_offset) % RPMB_DATA_SIZE; + if (size == 0) + size = RPMB_DATA_SIZE; + res = decrypt(data, lastfrm, size, 0, start_idx + nbr_frms - 1, fek, + uuid); + if (res != TEE_SUCCESS) + goto func_exit; + + /* Update MAC against the last block */ + res = crypto_mac_update(ctx, lastfrm->data, RPMB_MAC_PROTECT_DATA_SIZE); + if (res != TEE_SUCCESS) + goto func_exit; + + res = crypto_mac_final(ctx, rawdata->key_mac, RPMB_KEY_MAC_SIZE); + if (res != TEE_SUCCESS) + goto func_exit; + + res = TEE_SUCCESS; + +func_exit: + crypto_mac_free_ctx(ctx); + return res; +} + +static TEE_Result tee_rpmb_resp_unpack_verify(struct rpmb_data_frame *datafrm, + struct rpmb_raw_data *rawdata, + uint16_t nbr_frms, + const uint8_t *fek, + const TEE_UUID *uuid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint16_t msg_type; + uint32_t wr_cnt; + uint16_t blk_idx; + uint8_t op_result; + struct rpmb_data_frame lastfrm; + + if (!datafrm || !rawdata || !nbr_frms) + return TEE_ERROR_BAD_PARAMETERS; + + if (IS_ENABLED(CFG_RPMB_FS_DEBUG_DATA)) { + size_t i = 0; + + for (i = 0; i < nbr_frms; i++) { + DMSG("Dumping data frame %zu:", i); + DHEXDUMP((uint8_t *)&datafrm[i] + RPMB_STUFF_DATA_SIZE, + 512 - RPMB_STUFF_DATA_SIZE); + } + } + + /* Make sure the last data packet can't be modified once verified */ + memcpy(&lastfrm, &datafrm[nbr_frms - 1], RPMB_DATA_FRAME_SIZE); + + /* Handle operation result and translate to TEEC error code. */ + get_op_result_bits(lastfrm.op_result, &op_result); + if (op_result == RPMB_RESULT_AUTH_KEY_NOT_PROGRAMMED) + return TEE_ERROR_ITEM_NOT_FOUND; + if (op_result != RPMB_RESULT_OK) + return TEE_ERROR_GENERIC; + + /* Check the response msg_type. */ + bytes_to_u16(lastfrm.msg_type, &msg_type); + if (msg_type != rawdata->msg_type) { + DMSG("Unexpected msg_type (0x%04x != 0x%04x)", msg_type, + rawdata->msg_type); + return TEE_ERROR_GENERIC; + } + + if (rawdata->blk_idx) { + bytes_to_u16(lastfrm.address, &blk_idx); + if (blk_idx != *rawdata->blk_idx) { + DMSG("Unexpected block index"); + return TEE_ERROR_GENERIC; + } + } + + if (rawdata->write_counter) { + wr_cnt = *rawdata->write_counter; + bytes_to_u32(lastfrm.write_counter, rawdata->write_counter); + if (msg_type == RPMB_MSG_TYPE_RESP_AUTH_DATA_WRITE) { + /* Verify the write counter is incremented by 1 */ + if (*rawdata->write_counter != wr_cnt + 1) { + DMSG("Counter mismatched (0x%04x/0x%04x)", + *rawdata->write_counter, wr_cnt + 1); + return TEE_ERROR_SECURITY; + } + rpmb_ctx->wr_cnt++; + } + } + + if (rawdata->nonce) { + if (buf_compare_ct(rawdata->nonce, lastfrm.nonce, + RPMB_NONCE_SIZE) != 0) { + DMSG("Nonce mismatched"); + return TEE_ERROR_SECURITY; + } + } + + if (rawdata->key_mac) { + if (msg_type == RPMB_MSG_TYPE_RESP_AUTH_DATA_READ) { + if (!rawdata->data) + return TEE_ERROR_GENERIC; + + res = tee_rpmb_data_cpy_mac_calc(datafrm, rawdata, + nbr_frms, &lastfrm, + fek, uuid); + + if (res != TEE_SUCCESS) + return res; + } else { + /* + * There should be only one data frame for + * other msg types. + */ + if (nbr_frms != 1) + return TEE_ERROR_GENERIC; + + res = tee_rpmb_mac_calc(rawdata->key_mac, + RPMB_KEY_MAC_SIZE, + rpmb_ctx->key, + RPMB_KEY_MAC_SIZE, + &lastfrm, 1); + + if (res != TEE_SUCCESS) + return res; + } + + if (consttime_memcmp(rawdata->key_mac, + (datafrm + nbr_frms - 1)->key_mac, + RPMB_KEY_MAC_SIZE) != 0) { + DMSG("MAC mismatched:"); + if (IS_ENABLED(CFG_RPMB_FS_DEBUG_DATA)) + DHEXDUMP(rawdata->key_mac, RPMB_KEY_MAC_SIZE); + + return TEE_ERROR_SECURITY; + } + } + + return TEE_SUCCESS; +} + +static TEE_Result tee_rpmb_get_dev_info(uint16_t dev_id, + struct rpmb_dev_info *dev_info) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct tee_rpmb_mem mem; + struct rpmb_dev_info *di; + struct rpmb_req *req = NULL; + uint8_t *resp = NULL; + uint32_t req_size; + uint32_t resp_size; + + if (!dev_info) + return TEE_ERROR_BAD_PARAMETERS; + + req_size = sizeof(struct rpmb_req); + resp_size = sizeof(struct rpmb_dev_info); + res = tee_rpmb_alloc(req_size, resp_size, &mem, + (void *)&req, (void *)&resp); + if (res != TEE_SUCCESS) + goto func_exit; + + req->cmd = RPMB_CMD_GET_DEV_INFO; + req->dev_id = dev_id; + + di = (struct rpmb_dev_info *)resp; + di->ret_code = RPMB_CMD_GET_DEV_INFO_RET_ERROR; + + res = tee_rpmb_invoke(&mem); + if (res != TEE_SUCCESS) + goto func_exit; + + if (di->ret_code != RPMB_CMD_GET_DEV_INFO_RET_OK) { + res = TEE_ERROR_GENERIC; + goto func_exit; + } + + memcpy((uint8_t *)dev_info, resp, sizeof(struct rpmb_dev_info)); + + if (IS_ENABLED(CFG_RPMB_FS_DEBUG_DATA)) { + DMSG("Dumping dev_info:"); + DHEXDUMP((uint8_t *)dev_info, sizeof(struct rpmb_dev_info)); + } + + res = TEE_SUCCESS; + +func_exit: + tee_rpmb_free(&mem); + return res; +} + +static TEE_Result tee_rpmb_init_read_wr_cnt(uint16_t dev_id, + uint32_t *wr_cnt, + uint16_t *op_result) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct tee_rpmb_mem mem; + uint16_t msg_type; + uint8_t nonce[RPMB_NONCE_SIZE]; + uint8_t hmac[RPMB_KEY_MAC_SIZE]; + struct rpmb_req *req = NULL; + struct rpmb_data_frame *resp = NULL; + struct rpmb_raw_data rawdata; + uint32_t req_size; + uint32_t resp_size; + + if (!wr_cnt) + return TEE_ERROR_BAD_PARAMETERS; + + req_size = sizeof(struct rpmb_req) + RPMB_DATA_FRAME_SIZE; + resp_size = RPMB_DATA_FRAME_SIZE; + res = tee_rpmb_alloc(req_size, resp_size, &mem, + (void *)&req, (void *)&resp); + if (res != TEE_SUCCESS) + goto func_exit; + + res = crypto_rng_read(nonce, RPMB_NONCE_SIZE); + if (res != TEE_SUCCESS) + goto func_exit; + + msg_type = RPMB_MSG_TYPE_REQ_WRITE_COUNTER_VAL_READ; + + memset(&rawdata, 0x00, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = msg_type; + rawdata.nonce = nonce; + + res = tee_rpmb_req_pack(req, &rawdata, 1, dev_id, NULL, NULL); + if (res != TEE_SUCCESS) + goto func_exit; + + res = tee_rpmb_invoke(&mem); + if (res != TEE_SUCCESS) + goto func_exit; + + msg_type = RPMB_MSG_TYPE_RESP_WRITE_COUNTER_VAL_READ; + + memset(&rawdata, 0x00, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = msg_type; + rawdata.op_result = op_result; + rawdata.write_counter = wr_cnt; + rawdata.nonce = nonce; + rawdata.key_mac = hmac; + + res = tee_rpmb_resp_unpack_verify(resp, &rawdata, 1, NULL, NULL); + if (res != TEE_SUCCESS) + goto func_exit; + + res = TEE_SUCCESS; + +func_exit: + tee_rpmb_free(&mem); + return res; +} + +static TEE_Result tee_rpmb_verify_key_sync_counter(uint16_t dev_id) +{ + uint16_t op_result = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + res = tee_rpmb_init_read_wr_cnt(dev_id, &rpmb_ctx->wr_cnt, + &op_result); + + if (res == TEE_SUCCESS) { + rpmb_ctx->key_verified = true; + rpmb_ctx->wr_cnt_synced = true; + } else + EMSG("Verify key returning 0x%x", res); + return res; +} + +#ifdef CFG_RPMB_WRITE_KEY +static TEE_Result tee_rpmb_write_key(uint16_t dev_id) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct tee_rpmb_mem mem = { 0 }; + uint16_t msg_type; + struct rpmb_req *req = NULL; + struct rpmb_data_frame *resp = NULL; + struct rpmb_raw_data rawdata; + uint32_t req_size; + uint32_t resp_size; + + req_size = sizeof(struct rpmb_req) + RPMB_DATA_FRAME_SIZE; + resp_size = RPMB_DATA_FRAME_SIZE; + res = tee_rpmb_alloc(req_size, resp_size, &mem, + (void *)&req, (void *)&resp); + if (res != TEE_SUCCESS) + goto func_exit; + + msg_type = RPMB_MSG_TYPE_REQ_AUTH_KEY_PROGRAM; + + memset(&rawdata, 0x00, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = msg_type; + rawdata.key_mac = rpmb_ctx->key; + + res = tee_rpmb_req_pack(req, &rawdata, 1, dev_id, NULL, NULL); + if (res != TEE_SUCCESS) + goto func_exit; + + res = tee_rpmb_invoke(&mem); + if (res != TEE_SUCCESS) + goto func_exit; + + msg_type = RPMB_MSG_TYPE_RESP_AUTH_KEY_PROGRAM; + + memset(&rawdata, 0x00, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = msg_type; + + res = tee_rpmb_resp_unpack_verify(resp, &rawdata, 1, NULL, NULL); + if (res != TEE_SUCCESS) + goto func_exit; + + res = TEE_SUCCESS; + +func_exit: + tee_rpmb_free(&mem); + return res; +} + +static TEE_Result tee_rpmb_write_and_verify_key(uint16_t dev_id) +{ + TEE_Result res; + + if (!plat_rpmb_key_is_ready()) { + DMSG("RPMB INIT: platform indicates RPMB key is not ready"); + return TEE_ERROR_BAD_STATE; + } + + DMSG("RPMB INIT: Writing Key value:"); + DHEXDUMP(rpmb_ctx->key, RPMB_KEY_MAC_SIZE); + + res = tee_rpmb_write_key(dev_id); + if (res == TEE_SUCCESS) { + DMSG("RPMB INIT: Verifying Key"); + res = tee_rpmb_verify_key_sync_counter(dev_id); + } + return res; +} +#else +static TEE_Result tee_rpmb_write_and_verify_key(uint16_t dev_id __unused) +{ + DMSG("RPMB INIT: CFG_RPMB_WRITE_KEY is not set"); + return TEE_ERROR_STORAGE_NOT_AVAILABLE; +} +#endif + +/* This function must never return TEE_SUCCESS if rpmb_ctx == NULL */ +static TEE_Result tee_rpmb_init(uint16_t dev_id) +{ + TEE_Result res = TEE_SUCCESS; + struct rpmb_dev_info dev_info = { }; + uint32_t nblocks = 0; + + if (rpmb_dead) + return TEE_ERROR_COMMUNICATION; + + if (!rpmb_ctx) { + rpmb_ctx = calloc(1, sizeof(struct tee_rpmb_ctx)); + if (!rpmb_ctx) + return TEE_ERROR_OUT_OF_MEMORY; + } else if (rpmb_ctx->dev_id != dev_id) { + memset(rpmb_ctx, 0x00, sizeof(struct tee_rpmb_ctx)); + } + + rpmb_ctx->dev_id = dev_id; + + if (!rpmb_ctx->dev_info_synced) { + DMSG("RPMB: Syncing device information"); + + dev_info.rpmb_size_mult = 0; + dev_info.rel_wr_sec_c = 0; + res = tee_rpmb_get_dev_info(dev_id, &dev_info); + if (res != TEE_SUCCESS) + goto func_exit; + + DMSG("RPMB: RPMB size is %d*128 KB", dev_info.rpmb_size_mult); + DMSG("RPMB: Reliable Write Sector Count is %d", + dev_info.rel_wr_sec_c); + + if (dev_info.rpmb_size_mult == 0) { + res = TEE_ERROR_GENERIC; + goto func_exit; + } + + if (MUL_OVERFLOW(dev_info.rpmb_size_mult, + RPMB_SIZE_SINGLE / RPMB_DATA_SIZE, &nblocks) || + SUB_OVERFLOW(nblocks, 1, &rpmb_ctx->max_blk_idx)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto func_exit; + } + + memcpy(rpmb_ctx->cid, dev_info.cid, RPMB_EMMC_CID_SIZE); + +#ifdef RPMB_DRIVER_MULTIPLE_WRITE_FIXED + rpmb_ctx->rel_wr_blkcnt = dev_info.rel_wr_sec_c * 2; +#else + rpmb_ctx->rel_wr_blkcnt = 1; +#endif + + rpmb_ctx->dev_info_synced = true; + } + + if (!rpmb_ctx->key_derived) { + DMSG("RPMB INIT: Deriving key"); + + res = tee_rpmb_key_gen(dev_id, rpmb_ctx->key, + RPMB_KEY_MAC_SIZE); + if (res != TEE_SUCCESS) { + EMSG("RPMB INIT: Deriving key failed with error 0x%x", + res); + goto func_exit; + } + + rpmb_ctx->key_derived = true; + } + + /* Perform a write counter read to verify if the key is ok. */ + if (!rpmb_ctx->wr_cnt_synced || !rpmb_ctx->key_verified) { + DMSG("RPMB INIT: Verifying Key"); + + res = tee_rpmb_verify_key_sync_counter(dev_id); + if (res == TEE_ERROR_ITEM_NOT_FOUND && + !rpmb_ctx->key_verified) { + /* + * Need to write the key here and verify it. + */ + DMSG("RPMB INIT: Auth key not yet written"); + res = tee_rpmb_write_and_verify_key(dev_id); + } else if (res != TEE_SUCCESS) { + EMSG("Verify key failed!"); + EMSG("Make sure key here matches device key"); + } + } + +func_exit: + return res; +} + +/* + * Read RPMB data in bytes. + * + * @dev_id Device ID of the eMMC device. + * @addr Byte address of data. + * @data Pointer to the data. + * @len Size of data in bytes. + * @fek Encrypted File Encryption Key or NULL. + */ +static TEE_Result tee_rpmb_read(uint16_t dev_id, uint32_t addr, uint8_t *data, + uint32_t len, const uint8_t *fek, + const TEE_UUID *uuid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct tee_rpmb_mem mem = { 0 }; + uint16_t msg_type; + uint8_t nonce[RPMB_NONCE_SIZE]; + uint8_t hmac[RPMB_KEY_MAC_SIZE]; + struct rpmb_req *req = NULL; + struct rpmb_data_frame *resp = NULL; + struct rpmb_raw_data rawdata; + uint32_t req_size; + uint32_t resp_size; + uint16_t blk_idx; + uint16_t blkcnt; + uint8_t byte_offset; + + if (!data || !len) + return TEE_ERROR_BAD_PARAMETERS; + + blk_idx = addr / RPMB_DATA_SIZE; + byte_offset = addr % RPMB_DATA_SIZE; + + if (len + byte_offset + RPMB_DATA_SIZE < RPMB_DATA_SIZE) { + /* Overflow */ + return TEE_ERROR_BAD_PARAMETERS; + } + blkcnt = + ROUNDUP(len + byte_offset, RPMB_DATA_SIZE) / RPMB_DATA_SIZE; + res = tee_rpmb_init(dev_id); + if (res != TEE_SUCCESS) + goto func_exit; + + req_size = sizeof(struct rpmb_req) + RPMB_DATA_FRAME_SIZE; + resp_size = RPMB_DATA_FRAME_SIZE * blkcnt; + res = tee_rpmb_alloc(req_size, resp_size, &mem, + (void *)&req, (void *)&resp); + if (res != TEE_SUCCESS) + goto func_exit; + + msg_type = RPMB_MSG_TYPE_REQ_AUTH_DATA_READ; + res = crypto_rng_read(nonce, RPMB_NONCE_SIZE); + if (res != TEE_SUCCESS) + goto func_exit; + + memset(&rawdata, 0x00, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = msg_type; + rawdata.nonce = nonce; + rawdata.blk_idx = &blk_idx; + res = tee_rpmb_req_pack(req, &rawdata, 1, dev_id, NULL, NULL); + if (res != TEE_SUCCESS) + goto func_exit; + + req->block_count = blkcnt; + + DMSG("Read %u block%s at index %u", blkcnt, ((blkcnt > 1) ? "s" : ""), + blk_idx); + + res = tee_rpmb_invoke(&mem); + if (res != TEE_SUCCESS) + goto func_exit; + + msg_type = RPMB_MSG_TYPE_RESP_AUTH_DATA_READ; + + memset(&rawdata, 0x00, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = msg_type; + rawdata.block_count = &blkcnt; + rawdata.blk_idx = &blk_idx; + rawdata.nonce = nonce; + rawdata.key_mac = hmac; + rawdata.data = data; + + rawdata.len = len; + rawdata.byte_offset = byte_offset; + + res = tee_rpmb_resp_unpack_verify(resp, &rawdata, blkcnt, fek, uuid); + if (res != TEE_SUCCESS) + goto func_exit; + + res = TEE_SUCCESS; + +func_exit: + tee_rpmb_free(&mem); + return res; +} + +static TEE_Result write_req(uint16_t dev_id, uint16_t blk_idx, + const void *data_blks, uint16_t blkcnt, + const uint8_t *fek, const TEE_UUID *uuid, + struct tee_rpmb_mem *mem, void *req, void *resp) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t hmac[RPMB_KEY_MAC_SIZE] = { }; + uint32_t wr_cnt = rpmb_ctx->wr_cnt; + struct rpmb_raw_data rawdata = { }; + size_t retry_count = 0; + + assert(mem->req_size <= + sizeof(struct rpmb_req) + blkcnt * RPMB_DATA_FRAME_SIZE); + assert(mem->resp_size <= RPMB_DATA_FRAME_SIZE); + + while (true) { + memset(req, 0, mem->req_size); + memset(resp, 0, mem->resp_size); + + memset(&rawdata, 0, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = RPMB_MSG_TYPE_REQ_AUTH_DATA_WRITE; + rawdata.block_count = &blkcnt; + rawdata.blk_idx = &blk_idx; + rawdata.write_counter = &wr_cnt; + rawdata.key_mac = hmac; + rawdata.data = (uint8_t *)data_blks; + + res = tee_rpmb_req_pack(req, &rawdata, blkcnt, dev_id, fek, + uuid); + if (res) { + /* + * If we haven't tried to send a request yet we can + * allow a failure here since there's no chance of + * an intercepted request with a valid write + * counter. + */ + if (!retry_count) + return res; + + retry_count++; + if (retry_count >= RPMB_MAX_RETRIES) + goto out_of_retries; + + DMSG("Request pack failed, retrying %zu", retry_count); + continue; + } + + res = tee_rpmb_invoke(mem); + if (res != TEE_SUCCESS) { + retry_count++; + if (retry_count >= RPMB_MAX_RETRIES) + goto out_of_retries; + /* + * To force wr_cnt sync next time, as it might get + * out of sync due to inconsistent operation result! + */ + rpmb_ctx->wr_cnt_synced = false; + DMSG("Write invoke failed, retrying %zu", retry_count); + continue; + } + + memset(&rawdata, 0, sizeof(struct rpmb_raw_data)); + rawdata.msg_type = RPMB_MSG_TYPE_RESP_AUTH_DATA_WRITE; + rawdata.block_count = &blkcnt; + rawdata.blk_idx = &blk_idx; + rawdata.write_counter = &wr_cnt; + rawdata.key_mac = hmac; + + res = tee_rpmb_resp_unpack_verify(resp, &rawdata, 1, NULL, + NULL); + if (res != TEE_SUCCESS) { + retry_count++; + if (retry_count >= RPMB_MAX_RETRIES) + goto out_of_retries; + /* + * To force wr_cnt sync next time, as it might get + * out of sync due to inconsistent operation result! + */ + rpmb_ctx->wr_cnt_synced = false; + DMSG("Write resp unpack verify failed, retrying %zu", + retry_count); + continue; + } + + return TEE_SUCCESS; + } + +out_of_retries: + rpmb_dead = true; + /* + * We're using this error code to cause an eventuall calling TA to + * panic since we don't know if the data to be written has been + * committed to storage or not. + */ + return TEE_ERROR_COMMUNICATION; +} + +static TEE_Result tee_rpmb_write_blk(uint16_t dev_id, uint16_t blk_idx, + const uint8_t *data_blks, uint16_t blkcnt, + const uint8_t *fek, const TEE_UUID *uuid) +{ + TEE_Result res; + struct tee_rpmb_mem mem; + struct rpmb_req *req = NULL; + struct rpmb_data_frame *resp = NULL; + uint32_t req_size; + uint32_t resp_size; + uint32_t nbr_writes; + uint16_t tmp_blkcnt; + uint16_t tmp_blk_idx; + uint16_t i; + + DMSG("Write %u block%s at index %u", blkcnt, ((blkcnt > 1) ? "s" : ""), + blk_idx); + + if (!data_blks || !blkcnt) + return TEE_ERROR_BAD_PARAMETERS; + + res = tee_rpmb_init(dev_id); + if (res != TEE_SUCCESS) + return res; + + /* + * We need to split data when block count + * is bigger than reliable block write count. + */ + if (blkcnt < rpmb_ctx->rel_wr_blkcnt) + req_size = sizeof(struct rpmb_req) + + RPMB_DATA_FRAME_SIZE * blkcnt; + else + req_size = sizeof(struct rpmb_req) + + RPMB_DATA_FRAME_SIZE * rpmb_ctx->rel_wr_blkcnt; + + resp_size = RPMB_DATA_FRAME_SIZE; + res = tee_rpmb_alloc(req_size, resp_size, &mem, + (void *)&req, (void *)&resp); + if (res != TEE_SUCCESS) + return res; + + nbr_writes = blkcnt / rpmb_ctx->rel_wr_blkcnt; + if (blkcnt % rpmb_ctx->rel_wr_blkcnt > 0) + nbr_writes += 1; + + tmp_blkcnt = rpmb_ctx->rel_wr_blkcnt; + tmp_blk_idx = blk_idx; + for (i = 0; i < nbr_writes; i++) { + size_t offs = i * rpmb_ctx->rel_wr_blkcnt * RPMB_DATA_SIZE; + + /* + * To handle the last write of block count which is + * equal or smaller than reliable write block count. + */ + if (i == nbr_writes - 1) + tmp_blkcnt = blkcnt - rpmb_ctx->rel_wr_blkcnt * + (nbr_writes - 1); + + res = write_req(dev_id, tmp_blk_idx, data_blks + offs, + tmp_blkcnt, fek, uuid, &mem, req, resp); + if (res) + goto out; + + + tmp_blk_idx += tmp_blkcnt; + } + +out: + tee_rpmb_free(&mem); + return res; +} + +static bool tee_rpmb_write_is_atomic(uint16_t dev_id __unused, uint32_t addr, + uint32_t len) +{ + uint8_t byte_offset = addr % RPMB_DATA_SIZE; + uint16_t blkcnt = ROUNDUP(len + byte_offset, + RPMB_DATA_SIZE) / RPMB_DATA_SIZE; + + return (blkcnt <= rpmb_ctx->rel_wr_blkcnt); +} + +/* + * Write RPMB data in bytes. + * + * @dev_id Device ID of the eMMC device. + * @addr Byte address of data. + * @data Pointer to the data. + * @len Size of data in bytes. + * @fek Encrypted File Encryption Key or NULL. + */ +static TEE_Result tee_rpmb_write(uint16_t dev_id, uint32_t addr, + const uint8_t *data, uint32_t len, + const uint8_t *fek, const TEE_UUID *uuid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t *data_tmp = NULL; + uint16_t blk_idx; + uint16_t blkcnt; + uint8_t byte_offset; + + blk_idx = addr / RPMB_DATA_SIZE; + byte_offset = addr % RPMB_DATA_SIZE; + + blkcnt = + ROUNDUP(len + byte_offset, RPMB_DATA_SIZE) / RPMB_DATA_SIZE; + + if (byte_offset == 0 && (len % RPMB_DATA_SIZE) == 0) { + res = tee_rpmb_write_blk(dev_id, blk_idx, data, blkcnt, fek, + uuid); + if (res != TEE_SUCCESS) + goto func_exit; + } else { + data_tmp = calloc(blkcnt, RPMB_DATA_SIZE); + if (!data_tmp) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto func_exit; + } + + /* Read the complete blocks */ + res = tee_rpmb_read(dev_id, blk_idx * RPMB_DATA_SIZE, data_tmp, + blkcnt * RPMB_DATA_SIZE, fek, uuid); + if (res != TEE_SUCCESS) + goto func_exit; + + /* Partial update of the data blocks */ + memcpy(data_tmp + byte_offset, data, len); + + res = tee_rpmb_write_blk(dev_id, blk_idx, data_tmp, blkcnt, + fek, uuid); + if (res != TEE_SUCCESS) + goto func_exit; + } + + res = TEE_SUCCESS; + +func_exit: + free(data_tmp); + return res; +} + +/* + * Read the RPMB write counter. + * + * @dev_id Device ID of the eMMC device. + * @counter Pointer to the counter. + */ +static TEE_Result tee_rpmb_get_write_counter(uint16_t dev_id, + uint32_t *counter) +{ + TEE_Result res = TEE_SUCCESS; + + if (!counter) + return TEE_ERROR_BAD_PARAMETERS; + + if (rpmb_dead) + return TEE_ERROR_COMMUNICATION; + + if (!rpmb_ctx || !rpmb_ctx->wr_cnt_synced) { + res = tee_rpmb_init(dev_id); + if (res != TEE_SUCCESS) + goto func_exit; + } + + *counter = rpmb_ctx->wr_cnt; + +func_exit: + return res; +} + +/* + * Read the RPMB max block. + * + * @dev_id Device ID of the eMMC device. + * @counter Pointer to receive the max block. + */ +static TEE_Result tee_rpmb_get_max_block(uint16_t dev_id, uint32_t *max_block) +{ + TEE_Result res = TEE_SUCCESS; + + if (!max_block) + return TEE_ERROR_BAD_PARAMETERS; + + if (rpmb_dead) + return TEE_ERROR_COMMUNICATION; + + if (!rpmb_ctx || !rpmb_ctx->dev_info_synced) { + res = tee_rpmb_init(dev_id); + if (res != TEE_SUCCESS) + goto func_exit; + } + + *max_block = rpmb_ctx->max_blk_idx; + +func_exit: + return res; +} + +/* + * End of lower interface to RPMB device + */ + +static TEE_Result get_fat_start_address(uint32_t *addr); +static TEE_Result rpmb_fs_setup(void); + +/** + * fat_entry_dir_free: Free the FAT entry dir. + */ +static void fat_entry_dir_free(void) +{ + if (fat_entry_dir) { + free(fat_entry_dir->rpmb_fat_entry_buf); + free(fat_entry_dir); + fat_entry_dir = NULL; + } +} + +/** + * fat_entry_dir_init: Initialize the FAT FS entry buffer/cache + * This function must be called before reading FAT FS entries using the + * function fat_entry_dir_get_next. This initializes the buffer/cache with the + * first FAT FS entries. + */ +static TEE_Result fat_entry_dir_init(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct rpmb_fat_entry *fe = NULL; + uint32_t fat_address = 0; + uint32_t num_elems_read = 0; + + if (fat_entry_dir) + return TEE_SUCCESS; + + res = rpmb_fs_setup(); + if (res) + return res; + + res = get_fat_start_address(&fat_address); + if (res) + return res; + + fat_entry_dir = calloc(1, sizeof(struct rpmb_fat_entry_dir)); + if (!fat_entry_dir) + return TEE_ERROR_OUT_OF_MEMORY; + + /* + * If caching is enabled, read in up to the maximum cache size, but + * never more than the single read in size. Otherwise, read in as many + * entries fit into the temporary buffer. + */ + if (CFG_RPMB_FS_CACHE_ENTRIES) + num_elems_read = MIN(CFG_RPMB_FS_CACHE_ENTRIES, + CFG_RPMB_FS_RD_ENTRIES); + else + num_elems_read = CFG_RPMB_FS_RD_ENTRIES; + + /* + * Allocate memory for the FAT FS entries to read in. + */ + fe = calloc(num_elems_read, sizeof(struct rpmb_fat_entry)); + if (!fe) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, fat_address, (uint8_t *)fe, + num_elems_read * sizeof(*fe), NULL, NULL); + if (res) + goto out; + + fat_entry_dir->rpmb_fat_entry_buf = fe; + + /* + * We use this variable when getting next entries from the buffer/cache + * to see whether we have to read in more entries from storage. + */ + fat_entry_dir->num_buffered = num_elems_read; + + return TEE_SUCCESS; +out: + fat_entry_dir_free(); + free(fe); + return res; +} + +/** + * fat_entry_dir_deinit: If caching is enabled, free the temporary buffer for + * FAT FS entries in case the cache was too small. Keep the elements in the + * cache. Reset the counter variables to start the next traversal from fresh + * from the first cached entry. If caching is disabled, just free the + * temporary buffer by calling fat_entry_dir_free and return. + */ +static void fat_entry_dir_deinit(void) +{ + struct rpmb_fat_entry *fe = NULL; + + if (!fat_entry_dir) + return; + + if (!CFG_RPMB_FS_CACHE_ENTRIES) { + fat_entry_dir_free(); + return; + } + + fe = fat_entry_dir->rpmb_fat_entry_buf; + fat_entry_dir->idx_curr = 0; + fat_entry_dir->num_total_read = 0; + fat_entry_dir->last_reached = false; + + if (fat_entry_dir->num_buffered > CFG_RPMB_FS_CACHE_ENTRIES) { + fat_entry_dir->num_buffered = CFG_RPMB_FS_CACHE_ENTRIES; + + fe = realloc(fe, fat_entry_dir->num_buffered * sizeof(*fe)); + + /* + * In case realloc fails, we are on the safe side if we destroy + * the whole structure. Upon the next init, the cache has to be + * re-established, but this case should not happen in practice. + */ + if (!fe) + fat_entry_dir_free(); + else + fat_entry_dir->rpmb_fat_entry_buf = fe; + } +} + +/** + * fat_entry_dir_update: Updates a persisted FAT FS entry in the cache. + * This function updates the FAT entry fat_entry that was written to address + * fat_address onto RPMB storage in the cache. + */ +static TEE_Result __maybe_unused fat_entry_dir_update + (struct rpmb_fat_entry *fat_entry, + uint32_t fat_address) +{ + uint32_t fat_entry_buf_idx = 0; + /* Use a temp var to avoid compiler warning if caching disabled. */ + uint32_t max_cache_entries = CFG_RPMB_FS_CACHE_ENTRIES; + + assert(!((fat_address - RPMB_FS_FAT_START_ADDRESS) % + sizeof(struct rpmb_fat_entry))); + + /* Nothing to update if the cache is not initialized. */ + if (!fat_entry_dir) + return TEE_SUCCESS; + + fat_entry_buf_idx = (fat_address - RPMB_FS_FAT_START_ADDRESS) / + sizeof(struct rpmb_fat_entry); + + /* Only need to write if index points to an entry in cache. */ + if (fat_entry_buf_idx < fat_entry_dir->num_buffered && + fat_entry_buf_idx < max_cache_entries) { + memcpy(fat_entry_dir->rpmb_fat_entry_buf + fat_entry_buf_idx, + fat_entry, sizeof(struct rpmb_fat_entry)); + } + + return TEE_SUCCESS; +} + +/** + * fat_entry_dir_get_next: Get next FAT FS entry. + * Read either from cache/buffer, or by reading from RPMB storage if the + * elements in the buffer/cache are fully read. When reading in from RPMB + * storage, the buffer is overwritten in case caching is disabled. + * In case caching is enabled, the cache is either further filled, or a + * temporary buffer populated if the cache is already full. + * The FAT FS entry is written to fat_entry. The respective address in RPMB + * storage is written to fat_address, if not NULL. When the last FAT FS entry + * was previously read, the function indicates this case by writing a NULL + * pointer to fat_entry. + * Returns a value different TEE_SUCCESS if the next FAT FS entry could not be + * retrieved. + */ +static TEE_Result fat_entry_dir_get_next(struct rpmb_fat_entry **fat_entry, + uint32_t *fat_address) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct rpmb_fat_entry *fe = NULL; + uint32_t num_elems_read = 0; + uint32_t fat_address_local = 0; + + assert(fat_entry_dir && fat_entry); + + /* Don't read further if we previously read the last FAT FS entry. */ + if (fat_entry_dir->last_reached) { + *fat_entry = NULL; + return TEE_SUCCESS; + } + + fe = fat_entry_dir->rpmb_fat_entry_buf; + + /* Determine address of FAT FS entry in RPMB storage. */ + fat_address_local = RPMB_FS_FAT_START_ADDRESS + + (fat_entry_dir->num_total_read * + sizeof(struct rpmb_fat_entry)); + + /* + * We've read all so-far buffered elements, so we need to + * read in more entries from RPMB storage. + */ + if (fat_entry_dir->idx_curr >= fat_entry_dir->num_buffered) { + /* + * This is the case where we do not cache entries, so just read + * in next set of FAT FS entries into the buffer. + * Goto the end of the when statement if that is done. + */ + if (!CFG_RPMB_FS_CACHE_ENTRIES) { + num_elems_read = CFG_RPMB_FS_RD_ENTRIES; + fat_entry_dir->idx_curr = 0; + + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, + fat_address_local, (uint8_t *)fe, + num_elems_read * sizeof(*fe), NULL, + NULL); + if (res) + return res; + goto post_read_in; + } + + /* + * We cache FAT FS entries, and the buffer is not completely + * filled. Further keep on extending the buffer up to its max + * size by reading in from RPMB. + */ + if (fat_entry_dir->num_total_read < RPMB_BUF_MAX_ENTRIES) { + /* + * Read at most as many elements as fit in the buffer + * and no more than the defined number of entries to + * read in at once. + */ + num_elems_read = MIN(RPMB_BUF_MAX_ENTRIES - + fat_entry_dir->num_total_read, + (uint32_t)CFG_RPMB_FS_RD_ENTRIES); + + /* + * Expand the buffer to fit in the additional entries. + */ + fe = realloc(fe, + (fat_entry_dir->num_buffered + + num_elems_read) * sizeof(*fe)); + if (!fe) + return TEE_ERROR_OUT_OF_MEMORY; + + fat_entry_dir->rpmb_fat_entry_buf = fe; + + /* Read in to the next free slot in the buffer/cache. */ + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, + fat_address_local, + (uint8_t *)(fe + + fat_entry_dir->num_total_read), + num_elems_read * sizeof(*fe), + NULL, NULL); + if (res) + return res; + + fat_entry_dir->num_buffered += num_elems_read; + } else { + /* + * This happens when we have read as many elements as + * can possibly fit into the buffer. + * As the first part of the buffer serves as our cache, + * we only overwrite the last part that serves as our + * temporary buffer used to iteratively read in entries + * when the cache is full. Read in the temporary buffer + * maximum size. + */ + num_elems_read = CFG_RPMB_FS_RD_ENTRIES; + /* Reset index to beginning of the temporary buffer. */ + fat_entry_dir->idx_curr = CFG_RPMB_FS_CACHE_ENTRIES; + + /* Read in elements after the end of the cache. */ + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, + fat_address_local, + (uint8_t *)(fe + + fat_entry_dir->idx_curr), + num_elems_read * sizeof(*fe), + NULL, NULL); + if (res) + return res; + } + } + +post_read_in: + if (fat_address) + *fat_address = fat_address_local; + + *fat_entry = fe + fat_entry_dir->idx_curr; + + fat_entry_dir->idx_curr++; + fat_entry_dir->num_total_read++; + + /* + * Indicate last entry was read. + * Ensures we return a zero value for fat_entry on next invocation. + */ + if ((*fat_entry)->flags & FILE_IS_LAST_ENTRY) + fat_entry_dir->last_reached = true; + + return TEE_SUCCESS; +} + +#if (TRACE_LEVEL >= TRACE_FLOW) +static void dump_fat(void) +{ + TEE_Result res = TEE_ERROR_SECURITY; + struct rpmb_fat_entry *fe = NULL; + + if (!fs_par) + return; + + if (fat_entry_dir_init()) + return; + + while (true) { + res = fat_entry_dir_get_next(&fe, NULL); + if (res || !fe) + break; + + FMSG("flags %#"PRIx32", size %"PRIu32", address %#"PRIx32 + ", filename '%s'", + fe->flags, fe->data_size, fe->start_address, fe->filename); + } + + fat_entry_dir_deinit(); +} +#else +static void dump_fat(void) +{ +} +#endif + +#if (TRACE_LEVEL >= TRACE_DEBUG) +static void dump_fh(struct rpmb_file_handle *fh) +{ + DMSG("fh->filename=%s", fh->filename); + DMSG("fh->rpmb_fat_address=%u", fh->rpmb_fat_address); + DMSG("fh->fat_entry.start_address=%u", fh->fat_entry.start_address); + DMSG("fh->fat_entry.data_size=%u", fh->fat_entry.data_size); +} +#else +static void dump_fh(struct rpmb_file_handle *fh __unused) +{ +} +#endif + +/* "/TA_uuid/object_id" or "/TA_uuid/.object_id" */ +static TEE_Result create_filename(void *buf, size_t blen, struct tee_pobj *po, + bool transient) +{ + uint8_t *file = buf; + uint32_t pos = 0; + uint32_t hslen = 1 /* Leading slash */ + + TEE_B2HS_HSBUF_SIZE(sizeof(TEE_UUID) + po->obj_id_len) + + 1; /* Intermediate slash */ + + /* +1 for the '.' (temporary persistent object) */ + if (transient) + hslen++; + + if (blen < hslen) + return TEE_ERROR_SHORT_BUFFER; + + file[pos++] = '/'; + pos += tee_b2hs((uint8_t *)&po->uuid, &file[pos], + sizeof(TEE_UUID), hslen); + file[pos++] = '/'; + + if (transient) + file[pos++] = '.'; + + tee_b2hs(po->obj_id, file + pos, po->obj_id_len, hslen - pos); + + return TEE_SUCCESS; +} + +/* "/TA_uuid" */ +static TEE_Result create_dirname(void *buf, size_t blen, const TEE_UUID *uuid) +{ + uint8_t *dir = buf; + uint32_t hslen = TEE_B2HS_HSBUF_SIZE(sizeof(TEE_UUID)) + 1; + + if (blen < hslen) + return TEE_ERROR_SHORT_BUFFER; + + dir[0] = '/'; + tee_b2hs((uint8_t *)uuid, dir + 1, sizeof(TEE_UUID), hslen); + + return TEE_SUCCESS; +} + +static struct rpmb_file_handle *alloc_file_handle(struct tee_pobj *po, + bool temporary) +{ + struct rpmb_file_handle *fh = NULL; + + fh = calloc(1, sizeof(struct rpmb_file_handle)); + if (!fh) + return NULL; + + if (po) + create_filename(fh->filename, sizeof(fh->filename), po, + temporary); + + return fh; +} + +/** + * write_fat_entry: Store info in a fat_entry to RPMB. + */ +static TEE_Result write_fat_entry(struct rpmb_file_handle *fh, + bool update_write_counter) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + /* Protect partition data. */ + if (fh->rpmb_fat_address < sizeof(struct rpmb_fs_partition)) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto out; + } + + if (fh->rpmb_fat_address % sizeof(struct rpmb_fat_entry) != 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (update_write_counter) { + res = tee_rpmb_get_write_counter(CFG_RPMB_FS_DEV_ID, + &fh->fat_entry.write_counter); + if (res) + goto out; + } + + res = tee_rpmb_write(CFG_RPMB_FS_DEV_ID, fh->rpmb_fat_address, + (uint8_t *)&fh->fat_entry, + sizeof(struct rpmb_fat_entry), NULL, NULL); + + dump_fat(); + + /* If caching enabled, update a successfully written entry in cache. */ + if (CFG_RPMB_FS_CACHE_ENTRIES && !res) + res = fat_entry_dir_update(&fh->fat_entry, + fh->rpmb_fat_address); + +out: + return res; +} + +/** + * rpmb_fs_setup: Setup RPMB FS. + * Set initial partition and FS values and write to RPMB. + * Store frequently used data in RAM. + */ +static TEE_Result rpmb_fs_setup(void) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct rpmb_fs_partition *partition_data = NULL; + struct rpmb_file_handle *fh = NULL; + uint32_t max_rpmb_block = 0; + + if (fs_par) { + res = TEE_SUCCESS; + goto out; + } + + res = tee_rpmb_get_max_block(CFG_RPMB_FS_DEV_ID, &max_rpmb_block); + if (res != TEE_SUCCESS) + goto out; + + /* + * We're going to read a full block in order to have a full block + * for the dummy write below. + */ + COMPILE_TIME_ASSERT(sizeof(struct rpmb_fs_partition) <= + RPMB_DATA_SIZE); + partition_data = calloc(1, RPMB_DATA_SIZE); + if (!partition_data) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, RPMB_STORAGE_START_ADDRESS, + (uint8_t *)partition_data, RPMB_DATA_SIZE, + NULL, NULL); + if (res != TEE_SUCCESS) + goto out; + /* + * Perform a write in order to increase the write counter. This + * prevents late usage (replay attack) of a previously blocked + * request with a valid write counter value. + */ + res = tee_rpmb_write(CFG_RPMB_FS_DEV_ID, RPMB_STORAGE_START_ADDRESS, + (uint8_t *)partition_data, RPMB_DATA_SIZE, + NULL, NULL); + if (res != TEE_SUCCESS) + goto out; + /* + * We're reading again in case a stale request was committed + * instead of the one issued above. If this succeeds we're in sync + * with the RPMB block since there are no other possible stale + * blocks with valid write counters available. + */ + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, RPMB_STORAGE_START_ADDRESS, + (uint8_t *)partition_data, + sizeof(struct rpmb_fs_partition), NULL, NULL); + if (res != TEE_SUCCESS) + goto out; + +#ifndef CFG_RPMB_RESET_FAT + if (partition_data->rpmb_fs_magic == RPMB_FS_MAGIC) { + if (partition_data->fs_version == FS_VERSION) { + res = TEE_SUCCESS; + goto store_fs_par; + } else { + EMSG("Wrong software is in use."); + res = TEE_ERROR_ACCESS_DENIED; + goto out; + } + } +#else + EMSG("**** Clearing Storage ****"); +#endif + + /* Setup new partition data. */ + partition_data->rpmb_fs_magic = RPMB_FS_MAGIC; + partition_data->fs_version = FS_VERSION; + partition_data->fat_start_address = RPMB_FS_FAT_START_ADDRESS; + + /* Initial FAT entry with FILE_IS_LAST_ENTRY flag set. */ + fh = alloc_file_handle(NULL, false); + if (!fh) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + fh->fat_entry.flags = FILE_IS_LAST_ENTRY; + fh->rpmb_fat_address = partition_data->fat_start_address; + + /* Write init FAT entry and partition data to RPMB. */ + res = write_fat_entry(fh, true); + if (res != TEE_SUCCESS) + goto out; + + res = + tee_rpmb_get_write_counter(CFG_RPMB_FS_DEV_ID, + &partition_data->write_counter); + if (res != TEE_SUCCESS) + goto out; + res = tee_rpmb_write(CFG_RPMB_FS_DEV_ID, RPMB_STORAGE_START_ADDRESS, + (uint8_t *)partition_data, + sizeof(struct rpmb_fs_partition), NULL, NULL); + +#ifndef CFG_RPMB_RESET_FAT +store_fs_par: +#endif + + /* Store FAT start address. */ + fs_par = calloc(1, sizeof(struct rpmb_fs_parameters)); + if (!fs_par) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + fs_par->fat_start_address = partition_data->fat_start_address; + fs_par->max_rpmb_address = max_rpmb_block << RPMB_BLOCK_SIZE_SHIFT; + + dump_fat(); + +out: + free(fh); + free(partition_data); + return res; +} + +/** + * get_fat_start_address: + * FAT start_address from fs_par. + */ +static TEE_Result get_fat_start_address(uint32_t *addr) +{ + if (!fs_par) + return TEE_ERROR_NO_DATA; + + *addr = fs_par->fat_start_address; + + return TEE_SUCCESS; +} + +/** + * read_fat: Read FAT entries + * Return matching FAT entry for read, rm rename and stat. + * Build up memory pool and return matching entry for write operation. + * "Last FAT entry" can be returned during write. + */ +static TEE_Result read_fat(struct rpmb_file_handle *fh, tee_mm_pool_t *p) +{ + TEE_Result res = TEE_ERROR_GENERIC; + tee_mm_entry_t *mm = NULL; + struct rpmb_fat_entry *fe = NULL; + uint32_t fat_address; + bool entry_found = false; + bool expand_fat = false; + struct rpmb_file_handle last_fh; + + DMSG("fat_address %d", fh->rpmb_fat_address); + + res = fat_entry_dir_init(); + if (res) + goto out; + + /* + * The pool is used to represent the current RPMB layout. To find + * a slot for the file tee_mm_alloc is called on the pool. Thus + * if it is not NULL the entire FAT must be traversed to fill in + * the pool. + */ + while (true) { + res = fat_entry_dir_get_next(&fe, &fat_address); + if (res || !fe) + break; + + /* + * Look for an entry, matching filenames. (read, rm, + * rename and stat.). Only store first filename match. + */ + if ((!strcmp(fh->filename, fe->filename)) && + (fe->flags & FILE_IS_ACTIVE) && !entry_found) { + entry_found = true; + fh->rpmb_fat_address = fat_address; + memcpy(&fh->fat_entry, fe, sizeof(*fe)); + if (!p) + break; + } + + /* Add existing files to memory pool. (write) */ + if (p) { + if ((fe->flags & FILE_IS_ACTIVE) && fe->data_size > 0) { + + mm = tee_mm_alloc2(p, fe->start_address, + fe->data_size); + if (!mm) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + } + + /* Unused FAT entries can be reused (write) */ + if (((fe->flags & FILE_IS_ACTIVE) == 0) && + fh->rpmb_fat_address == 0) { + fh->rpmb_fat_address = fat_address; + memcpy(&fh->fat_entry, fe, + sizeof(struct rpmb_fat_entry)); + } + + if (((fe->flags & FILE_IS_LAST_ENTRY) != 0) && + fh->rpmb_fat_address == fat_address) { + + /* + * If the last entry was reached and was chosen + * by the previous check, then the FAT needs to + * be expanded. + * fh->rpmb_fat_address is the address chosen + * to store the files FAT entry and fat_address + * is the current FAT entry address being + * compared. + */ + expand_fat = true; + } + } + } + + if (res) + goto out; + /* + * Represent the FAT table in the pool. + */ + if (p) { + /* + * Since fat_address is the start of the last entry it needs to + * be moved up by an entry. + */ + fat_address += sizeof(struct rpmb_fat_entry); + + /* Make room for yet a FAT entry and add to memory pool. */ + if (expand_fat) + fat_address += sizeof(struct rpmb_fat_entry); + + mm = tee_mm_alloc2(p, RPMB_STORAGE_START_ADDRESS, fat_address); + if (!mm) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + if (expand_fat) { + /* + * Point fat_address to the beginning of the new + * entry. + */ + fat_address -= sizeof(struct rpmb_fat_entry); + memset(&last_fh, 0, sizeof(last_fh)); + last_fh.fat_entry.flags = FILE_IS_LAST_ENTRY; + last_fh.rpmb_fat_address = fat_address; + res = write_fat_entry(&last_fh, true); + if (res != TEE_SUCCESS) + goto out; + } + } + + if (!fh->rpmb_fat_address) + res = TEE_ERROR_ITEM_NOT_FOUND; + +out: + fat_entry_dir_deinit(); + return res; +} + +static TEE_Result generate_fek(struct rpmb_fat_entry *fe, const TEE_UUID *uuid) +{ + TEE_Result res; + +again: + res = tee_fs_generate_fek(uuid, fe->fek, sizeof(fe->fek)); + if (res != TEE_SUCCESS) + return res; + + if (is_zero(fe->fek, sizeof(fe->fek))) + goto again; + + return res; +} + +static TEE_Result rpmb_fs_open_internal(struct rpmb_file_handle *fh, + const TEE_UUID *uuid, bool create) +{ + tee_mm_pool_t p; + bool pool_result; + paddr_size_t pool_sz = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + /* We need to do setup in order to make sure fs_par is filled in */ + res = rpmb_fs_setup(); + if (res != TEE_SUCCESS) + goto out; + + fh->uuid = uuid; + if (create) { + /* Upper memory allocation must be used for RPMB_FS. */ + pool_sz = fs_par->max_rpmb_address - RPMB_STORAGE_START_ADDRESS; + pool_result = tee_mm_init(&p, + RPMB_STORAGE_START_ADDRESS, + pool_sz, + RPMB_BLOCK_SIZE_SHIFT, + TEE_MM_POOL_HI_ALLOC); + + if (!pool_result) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = read_fat(fh, &p); + tee_mm_final(&p); + if (res != TEE_SUCCESS) + goto out; + } else { + res = read_fat(fh, NULL); + if (res != TEE_SUCCESS) + goto out; + } + + /* + * If this is opened with create and the entry found was not active + * then this is a new file and the FAT entry must be written + */ + if (create) { + if ((fh->fat_entry.flags & FILE_IS_ACTIVE) == 0) { + memset(&fh->fat_entry, 0, + sizeof(struct rpmb_fat_entry)); + memcpy(fh->fat_entry.filename, fh->filename, + strlen(fh->filename)); + /* Start address and size are 0 */ + fh->fat_entry.flags = FILE_IS_ACTIVE; + + res = generate_fek(&fh->fat_entry, uuid); + if (res != TEE_SUCCESS) + goto out; + DMSG("GENERATE FEK key: %p", + (void *)fh->fat_entry.fek); + DHEXDUMP(fh->fat_entry.fek, sizeof(fh->fat_entry.fek)); + + res = write_fat_entry(fh, true); + if (res != TEE_SUCCESS) + goto out; + } + } + + res = TEE_SUCCESS; + +out: + return res; +} + +static void rpmb_fs_close(struct tee_file_handle **tfh) +{ + struct rpmb_file_handle *fh = (struct rpmb_file_handle *)*tfh; + + free(fh); + *tfh = NULL; +} + +static TEE_Result rpmb_fs_read(struct tee_file_handle *tfh, size_t pos, + void *buf_core, void *buf_user, size_t *len) +{ + TEE_Result res; + struct rpmb_file_handle *fh = (struct rpmb_file_handle *)tfh; + size_t size = *len; + + /* One of buf_core and buf_user must be NULL */ + assert(!buf_core || !buf_user); + + if (!size) + return TEE_SUCCESS; + + mutex_lock(&rpmb_mutex); + + dump_fh(fh); + + res = read_fat(fh, NULL); + if (res != TEE_SUCCESS) + goto out; + + if (pos >= fh->fat_entry.data_size) { + *len = 0; + goto out; + } + + size = MIN(size, fh->fat_entry.data_size - pos); + if (size) { + if (buf_core) { + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, + fh->fat_entry.start_address + pos, + buf_core, size, fh->fat_entry.fek, + fh->uuid); + if (res != TEE_SUCCESS) + goto out; + } else if (buf_user) { + uint32_t f = TEE_MEMORY_ACCESS_WRITE; + + res = check_user_access(f, buf_user, size); + if (res) + goto out; + enter_user_access(); + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, + fh->fat_entry.start_address + pos, + buf_user, size, fh->fat_entry.fek, + fh->uuid); + exit_user_access(); + if (res) + goto out; + } + } + *len = size; + +out: + mutex_unlock(&rpmb_mutex); + return res; +} + +static TEE_Result update_write_helper(struct rpmb_file_handle *fh, + size_t pos, const void *buf, + size_t size, uintptr_t new_fat, + size_t new_size) +{ + uintptr_t old_fat = fh->fat_entry.start_address; + size_t old_size = fh->fat_entry.data_size; + const uint8_t *rem_buf = buf; + size_t rem_size = size; + uint8_t *blk_buf = NULL; + size_t blk_offset = 0; + size_t blk_size = 0; + TEE_Result res = TEE_SUCCESS; + + blk_buf = mempool_alloc(mempool_default, TMP_BLOCK_SIZE); + if (!blk_buf) + return TEE_ERROR_OUT_OF_MEMORY; + + while (blk_offset < new_size) { + uint8_t *copy_dst = blk_buf; + size_t copy_size = 0; + size_t rd_size = 0; + + blk_size = MIN(TMP_BLOCK_SIZE, new_size - blk_offset); + memset(blk_buf, 0, blk_size); + + /* Possibly read old RPMB data in temporary buffer */ + if (blk_offset < pos && blk_offset < old_size) { + rd_size = MIN(blk_size, old_size - blk_offset); + + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, + old_fat + blk_offset, blk_buf, + rd_size, fh->fat_entry.fek, + fh->uuid); + if (res != TEE_SUCCESS) + break; + } + + /* Possibly update data in temporary buffer */ + if ((blk_offset + TMP_BLOCK_SIZE > pos) && + (blk_offset < pos + size)) { + size_t offset = 0; + + copy_dst = blk_buf; + copy_size = TMP_BLOCK_SIZE; + + if (blk_offset < pos) { + offset = pos - blk_offset; + + copy_dst += offset; + copy_size -= offset; + } + copy_size = MIN(copy_size, rem_size); + + memcpy(copy_dst, rem_buf, copy_size); + rem_buf += copy_size; + rem_size -= copy_size; + } + + /* Write temporary buffer to new RPMB destination */ + res = tee_rpmb_write(CFG_RPMB_FS_DEV_ID, new_fat + blk_offset, + blk_buf, blk_size, + fh->fat_entry.fek, fh->uuid); + if (res != TEE_SUCCESS) + break; + + blk_offset += blk_size; + } + + mempool_free(mempool_default, blk_buf); + + return res; +} + +static TEE_Result rpmb_fs_write_primitive(struct rpmb_file_handle *fh, + size_t pos, const void *buf, + size_t size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + tee_mm_pool_t p = { }; + bool pool_result = false; + size_t end = 0; + uint32_t start_addr = 0; + paddr_size_t pool_sz = 0; + + if (!size) + return TEE_SUCCESS; + + if (!fs_par) { + res = TEE_ERROR_GENERIC; + goto out; + } + + dump_fh(fh); + + /* Upper memory allocation must be used for RPMB_FS. */ + pool_sz = fs_par->max_rpmb_address - RPMB_STORAGE_START_ADDRESS; + pool_result = tee_mm_init(&p, + RPMB_STORAGE_START_ADDRESS, + pool_sz, + RPMB_BLOCK_SIZE_SHIFT, + TEE_MM_POOL_HI_ALLOC); + if (!pool_result) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = read_fat(fh, &p); + if (res != TEE_SUCCESS) + goto out; + + if (fh->fat_entry.flags & FILE_IS_LAST_ENTRY) + panic("invalid last entry flag"); + + if (ADD_OVERFLOW(pos, size, &end)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + if (ADD_OVERFLOW(fh->fat_entry.start_address, pos, &start_addr)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (end <= fh->fat_entry.data_size && + tee_rpmb_write_is_atomic(CFG_RPMB_FS_DEV_ID, start_addr, size)) { + + DMSG("Updating data in-place"); + res = tee_rpmb_write(CFG_RPMB_FS_DEV_ID, start_addr, buf, + size, fh->fat_entry.fek, fh->uuid); + } else { + /* + * File must be extended, or update cannot be atomic: allocate, + * read, update, write. + */ + size_t new_size = MAX(end, fh->fat_entry.data_size); + tee_mm_entry_t *mm = tee_mm_alloc(&p, new_size); + uintptr_t new_fat_entry = 0; + + DMSG("Need to re-allocate"); + if (!mm) { + DMSG("RPMB: No space left"); + res = TEE_ERROR_STORAGE_NO_SPACE; + goto out; + } + + new_fat_entry = tee_mm_get_smem(mm); + + res = update_write_helper(fh, pos, buf, size, + new_fat_entry, new_size); + if (res == TEE_SUCCESS) { + fh->fat_entry.data_size = new_size; + fh->fat_entry.start_address = new_fat_entry; + + res = write_fat_entry(fh, true); + } + } + +out: + if (pool_result) + tee_mm_final(&p); + + return res; +} + +static TEE_Result rpmb_fs_write(struct tee_file_handle *tfh, size_t pos, + const void *buf_core, const void *buf_user, + size_t size) +{ + TEE_Result res = TEE_SUCCESS; + + /* One of buf_core and buf_user must be NULL */ + assert(!buf_core || !buf_user); + + if (!size) + return TEE_SUCCESS; + + mutex_lock(&rpmb_mutex); + if (buf_core) { + res = rpmb_fs_write_primitive((struct rpmb_file_handle *)tfh, + pos, buf_core, size); + } else if (buf_user) { + uint32_t f = TEE_MEMORY_ACCESS_READ; + + res = check_user_access(f, buf_user, size); + if (res) + goto out; + enter_user_access(); + res = rpmb_fs_write_primitive((struct rpmb_file_handle *)tfh, + pos, buf_user, size); + exit_user_access(); + } +out: + mutex_unlock(&rpmb_mutex); + + return res; +} + +static TEE_Result rpmb_fs_remove_internal(struct rpmb_file_handle *fh) +{ + TEE_Result res; + + res = read_fat(fh, NULL); + if (res) + return res; + + /* Clear this file entry. */ + memset(&fh->fat_entry, 0, sizeof(struct rpmb_fat_entry)); + return write_fat_entry(fh, false); +} + +static TEE_Result rpmb_fs_remove(struct tee_pobj *po) +{ + TEE_Result res; + struct rpmb_file_handle *fh = alloc_file_handle(po, po->temporary); + + if (!fh) + return TEE_ERROR_OUT_OF_MEMORY; + + mutex_lock(&rpmb_mutex); + + res = rpmb_fs_remove_internal(fh); + + mutex_unlock(&rpmb_mutex); + + free(fh); + return res; +} + +static TEE_Result rpmb_fs_rename_internal(struct tee_pobj *old, + struct tee_pobj *new, + bool overwrite) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct rpmb_file_handle *fh_old = NULL; + struct rpmb_file_handle *fh_new = NULL; + + if (!old) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (new) + fh_old = alloc_file_handle(old, old->temporary); + else + fh_old = alloc_file_handle(old, true); + if (!fh_old) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + if (new) + fh_new = alloc_file_handle(new, new->temporary); + else + fh_new = alloc_file_handle(old, false); + if (!fh_new) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = read_fat(fh_old, NULL); + if (res != TEE_SUCCESS) + goto out; + + res = read_fat(fh_new, NULL); + if (res == TEE_SUCCESS) { + if (!overwrite) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto out; + } + + /* Clear this file entry. */ + memset(&fh_new->fat_entry, 0, sizeof(struct rpmb_fat_entry)); + res = write_fat_entry(fh_new, false); + if (res != TEE_SUCCESS) + goto out; + } + + memset(fh_old->fat_entry.filename, 0, TEE_RPMB_FS_FILENAME_LENGTH); + memcpy(fh_old->fat_entry.filename, fh_new->filename, + strlen(fh_new->filename)); + + res = write_fat_entry(fh_old, false); + +out: + free(fh_old); + free(fh_new); + + return res; +} + +static TEE_Result rpmb_fs_rename(struct tee_pobj *old, struct tee_pobj *new, + bool overwrite) +{ + TEE_Result res; + + mutex_lock(&rpmb_mutex); + res = rpmb_fs_rename_internal(old, new, overwrite); + mutex_unlock(&rpmb_mutex); + + return res; +} + +static TEE_Result rpmb_fs_truncate(struct tee_file_handle *tfh, size_t length) +{ + struct rpmb_file_handle *fh = (struct rpmb_file_handle *)tfh; + tee_mm_pool_t p; + bool pool_result = false; + tee_mm_entry_t *mm; + uint32_t newsize; + uint8_t *newbuf = NULL; + uintptr_t newaddr; + TEE_Result res = TEE_ERROR_GENERIC; + paddr_size_t pool_sz = 0; + + mutex_lock(&rpmb_mutex); + + if (length > INT32_MAX) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + newsize = length; + + res = read_fat(fh, NULL); + if (res != TEE_SUCCESS) + goto out; + + if (newsize > fh->fat_entry.data_size) { + /* Extend file */ + + pool_sz = fs_par->max_rpmb_address - RPMB_STORAGE_START_ADDRESS; + pool_result = tee_mm_init(&p, + RPMB_STORAGE_START_ADDRESS, + pool_sz, + RPMB_BLOCK_SIZE_SHIFT, + TEE_MM_POOL_HI_ALLOC); + if (!pool_result) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + res = read_fat(fh, &p); + if (res != TEE_SUCCESS) + goto out; + + mm = tee_mm_alloc(&p, newsize); + newbuf = calloc(1, newsize); + if (!mm || !newbuf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + if (fh->fat_entry.data_size) { + res = tee_rpmb_read(CFG_RPMB_FS_DEV_ID, + fh->fat_entry.start_address, + newbuf, fh->fat_entry.data_size, + fh->fat_entry.fek, fh->uuid); + if (res != TEE_SUCCESS) + goto out; + } + + newaddr = tee_mm_get_smem(mm); + res = tee_rpmb_write(CFG_RPMB_FS_DEV_ID, newaddr, newbuf, + newsize, fh->fat_entry.fek, fh->uuid); + if (res != TEE_SUCCESS) + goto out; + + } else { + /* Don't change file location */ + newaddr = fh->fat_entry.start_address; + } + + /* fh->pos is unchanged */ + fh->fat_entry.data_size = newsize; + fh->fat_entry.start_address = newaddr; + res = write_fat_entry(fh, true); + +out: + mutex_unlock(&rpmb_mutex); + if (pool_result) + tee_mm_final(&p); + if (newbuf) + free(newbuf); + + return res; +} + +static void rpmb_fs_dir_free(struct tee_fs_dir *dir) +{ + struct tee_rpmb_fs_dirent *e; + + if (!dir) + return; + + free(dir->current); + + while ((e = SIMPLEQ_FIRST(&dir->next))) { + SIMPLEQ_REMOVE_HEAD(&dir->next, link); + free(e); + } +} + +static TEE_Result rpmb_fs_dir_populate(const char *path, + struct tee_fs_dir *dir) +{ + struct tee_rpmb_fs_dirent *current = NULL; + struct rpmb_fat_entry *fe = NULL; + uint32_t fat_address; + uint32_t filelen; + char *filename; + bool matched; + struct tee_rpmb_fs_dirent *next = NULL; + uint32_t pathlen; + TEE_Result res = TEE_ERROR_GENERIC; + char temp; + + mutex_lock(&rpmb_mutex); + + res = fat_entry_dir_init(); + if (res) + goto out; + + pathlen = strlen(path); + + while (true) { + res = fat_entry_dir_get_next(&fe, &fat_address); + if (res || !fe) + break; + + filename = fe->filename; + if (fe->flags & FILE_IS_ACTIVE) { + matched = false; + filelen = strlen(filename); + if (filelen > pathlen) { + temp = filename[pathlen]; + filename[pathlen] = '\0'; + if (strcmp(filename, path) == 0) + matched = true; + + filename[pathlen] = temp; + } + + if (matched) { + next = malloc(sizeof(*next)); + if (!next) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + next->entry.oidlen = tee_hs2b((uint8_t *) + &filename[pathlen], + next->entry.oid, + filelen - pathlen, + sizeof(next->entry.oid)); + if (next->entry.oidlen) { + SIMPLEQ_INSERT_TAIL(&dir->next, + next, link); + current = next; + } else { + free(next); + next = NULL; + } + } + } + } + + if (res) + goto out; + + if (current) + res = TEE_SUCCESS; + else + res = TEE_ERROR_ITEM_NOT_FOUND; /* No directories were found. */ + +out: + mutex_unlock(&rpmb_mutex); + fat_entry_dir_deinit(); + if (res) + rpmb_fs_dir_free(dir); + + return res; +} + +static TEE_Result rpmb_fs_opendir(const TEE_UUID *uuid, struct tee_fs_dir **dir) +{ + uint32_t len; + char path_local[TEE_RPMB_FS_FILENAME_LENGTH]; + TEE_Result res = TEE_ERROR_GENERIC; + struct tee_fs_dir *rpmb_dir = NULL; + + if (!uuid || !dir) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + memset(path_local, 0, sizeof(path_local)); + if (create_dirname(path_local, sizeof(path_local) - 1, uuid)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + len = strlen(path_local); + + /* Add a slash to correctly match the full directory name. */ + if (path_local[len - 1] != '/') + path_local[len] = '/'; + + rpmb_dir = calloc(1, sizeof(*rpmb_dir)); + if (!rpmb_dir) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + SIMPLEQ_INIT(&rpmb_dir->next); + + res = rpmb_fs_dir_populate(path_local, rpmb_dir); + if (res != TEE_SUCCESS) { + free(rpmb_dir); + rpmb_dir = NULL; + goto out; + } + + *dir = rpmb_dir; + +out: + return res; +} + +static TEE_Result rpmb_fs_readdir(struct tee_fs_dir *dir, + struct tee_fs_dirent **ent) +{ + if (!dir) + return TEE_ERROR_GENERIC; + + free(dir->current); + + dir->current = SIMPLEQ_FIRST(&dir->next); + if (!dir->current) + return TEE_ERROR_ITEM_NOT_FOUND; + + SIMPLEQ_REMOVE_HEAD(&dir->next, link); + + *ent = &dir->current->entry; + return TEE_SUCCESS; +} + +static void rpmb_fs_closedir(struct tee_fs_dir *dir) +{ + if (dir) { + rpmb_fs_dir_free(dir); + free(dir); + } +} + +static TEE_Result rpmb_fs_open(struct tee_pobj *po, size_t *size, + struct tee_file_handle **ret_fh) +{ + TEE_Result res; + struct rpmb_file_handle *fh = alloc_file_handle(po, po->temporary); + + if (!fh) + return TEE_ERROR_OUT_OF_MEMORY; + + mutex_lock(&rpmb_mutex); + + res = rpmb_fs_open_internal(fh, &po->uuid, false); + if (!res && size) + *size = fh->fat_entry.data_size; + + mutex_unlock(&rpmb_mutex); + + if (res) + free(fh); + else + *ret_fh = (struct tee_file_handle *)fh; + + return res; +} + +static TEE_Result rpmb_fs_create(struct tee_pobj *po, bool overwrite, + const void *head, size_t head_size, + const void *attr, size_t attr_size, + const void *data_core, const void *data_user, + size_t data_size, + struct tee_file_handle **ret_fh) +{ + TEE_Result res; + size_t pos = 0; + struct rpmb_file_handle *fh = alloc_file_handle(po, po->temporary); + + /* One of data_core and data_user must be NULL */ + assert(!data_core || !data_user); + + if (!fh) + return TEE_ERROR_OUT_OF_MEMORY; + + mutex_lock(&rpmb_mutex); + res = rpmb_fs_open_internal(fh, &po->uuid, true); + if (res) + goto out; + + if (head && head_size) { + res = rpmb_fs_write_primitive(fh, pos, head, head_size); + if (res) + goto out; + pos += head_size; + } + + if (attr && attr_size) { + res = rpmb_fs_write_primitive(fh, pos, attr, attr_size); + if (res) + goto out; + pos += attr_size; + } + + if (data_size) { + if (data_core) { + res = rpmb_fs_write_primitive(fh, pos, data_core, + data_size); + if (res) + goto out; + } else if (data_user) { + uint32_t f = TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER; + + res = check_user_access(f, data_user, data_size); + if (res) + goto out; + enter_user_access(); + res = rpmb_fs_write_primitive(fh, pos, data_user, + data_size); + exit_user_access(); + if (res) + goto out; + } + } + + if (po->temporary) { + /* + * If it's a temporary filename (which it normally is) + * rename into the final filename now that the file is + * fully initialized. + */ + po->temporary = false; + res = rpmb_fs_rename_internal(po, NULL, overwrite); + if (res) { + po->temporary = true; + goto out; + } + /* Update file handle after rename. */ + create_filename(fh->filename, sizeof(fh->filename), po, false); + } + +out: + if (res) { + rpmb_fs_remove_internal(fh); + free(fh); + } else { + *ret_fh = (struct tee_file_handle *)fh; + } + mutex_unlock(&rpmb_mutex); + + return res; +} + +const struct tee_file_operations rpmb_fs_ops = { + .open = rpmb_fs_open, + .create = rpmb_fs_create, + .close = rpmb_fs_close, + .read = rpmb_fs_read, + .write = rpmb_fs_write, + .truncate = rpmb_fs_truncate, + .rename = rpmb_fs_rename, + .remove = rpmb_fs_remove, + .opendir = rpmb_fs_opendir, + .closedir = rpmb_fs_closedir, + .readdir = rpmb_fs_readdir, +}; + +TEE_Result tee_rpmb_fs_raw_open(const char *fname, bool create, + struct tee_file_handle **ret_fh) +{ + TEE_Result res; + struct rpmb_file_handle *fh = calloc(1, sizeof(*fh)); + static const TEE_UUID uuid = { 0 }; + + if (!fh) + return TEE_ERROR_OUT_OF_MEMORY; + + snprintf(fh->filename, sizeof(fh->filename), "/%s", fname); + + mutex_lock(&rpmb_mutex); + + res = rpmb_fs_open_internal(fh, &uuid, create); + + mutex_unlock(&rpmb_mutex); + + if (res) { + if (create) + rpmb_fs_remove_internal(fh); + free(fh); + } else { + *ret_fh = (struct tee_file_handle *)fh; + } + + return res; +} + +bool __weak plat_rpmb_key_is_ready(void) +{ + return true; +} diff --git a/optee_os/core/tee/tee_supp_plugin_rpc.c b/optee_os/core/tee/tee_supp_plugin_rpc.c new file mode 100644 index 0000000..c91d1eb --- /dev/null +++ b/optee_os/core/tee/tee_supp_plugin_rpc.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TEE_Result tee_invoke_supp_plugin_rpc(const TEE_UUID *uuid, uint32_t cmd, + uint32_t sub_cmd, void *buf_core, + void *buf_user, size_t len, + size_t *outlen) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct thread_param params[THREAD_RPC_MAX_NUM_PARAMS]; + uint32_t uuid_words[4] = { }; + void *va = NULL; + struct mobj *mobj = NULL; + + /* + * sizeof 'TEE_UUID' and array 'uuid_words' must be same size, + * because 'tee_uuid_to_octets()' is used to copy variable + * with one type to another. + * + * Array 'uuid_words' is used just for convenient work with + * 'TEE_UUID' as with uint32_t values. + */ + COMPILE_TIME_ASSERT(sizeof(TEE_UUID) == sizeof(uuid_words)); + + if (!uuid || (len && !buf_core && !buf_user) || + (!len && (buf_core || buf_user)) || (buf_core && buf_user)) + return TEE_ERROR_BAD_PARAMETERS; + + if (len) { + mobj = thread_rpc_alloc_payload(len); + if (!mobj) { + EMSG("can't create mobj for plugin data"); + return TEE_ERROR_OUT_OF_MEMORY; + } + + va = mobj_get_va(mobj, 0, len); + if (!va) { + EMSG("can't get va from mobj"); + goto out; + } + + if (buf_core) + memcpy(va, buf_core, len); + if (buf_user) { + res = copy_from_user(va, buf_user, len); + if (res) + goto out; + } + } + + tee_uuid_to_octets((uint8_t *)uuid_words, uuid); + + params[0] = THREAD_PARAM_VALUE(IN, OPTEE_RPC_SUPP_PLUGIN_INVOKE, + uuid_words[0], uuid_words[1]); + params[1] = THREAD_PARAM_VALUE(IN, uuid_words[2], uuid_words[3], cmd); + params[2] = THREAD_PARAM_VALUE(INOUT, sub_cmd, 0, 0); + params[3] = THREAD_PARAM_MEMREF(INOUT, mobj, 0, len); + + res = thread_rpc_cmd(OPTEE_RPC_CMD_SUPP_PLUGIN, 4, params); + + if (outlen) + *outlen = params[2].u.value.b; + + if (len && outlen && *outlen) { + if (buf_core) + memcpy(buf_core, va, *outlen <= len ? *outlen : len); + if (buf_user) + res = copy_to_user(buf_user, va, len); + } + +out: + if (len) + thread_rpc_free_payload(mobj); + + return res; +} diff --git a/optee_os/core/tee/tee_svc.c b/optee_os/core/tee/tee_svc.c new file mode 100644 index 0000000..f4158dc --- /dev/null +++ b/optee_os/core/tee/tee_svc.c @@ -0,0 +1,1014 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020-2022 Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +vaddr_t tee_svc_uref_base; + +void syscall_log(const void *buf, size_t len) +{ + if (IS_ENABLED(CFG_TEE_CORE_TA_TRACE)) { + char *kbuf = NULL; + size_t sz = 0; + + if (!len || ADD_OVERFLOW(len, 1, &sz)) + return; + + kbuf = malloc(sz); + if (!kbuf) + return; + + if (copy_from_user(kbuf, buf, len) == TEE_SUCCESS) { + kbuf[len] = '\0'; + trace_ext_puts(kbuf); + } + + free_wipe(kbuf); + } +} + +TEE_Result syscall_not_supported(void) +{ + return TEE_ERROR_NOT_SUPPORTED; +} + +/* Configuration properties */ +/* API implementation version */ +static const char api_vers[] = TO_STR(CFG_TEE_API_VERSION); + +/* Implementation description (implementation-dependent) */ +static const char descr[] = TO_STR(CFG_TEE_IMPL_DESCR); + +/* + * TA persistent time protection level + * 100: Persistent time based on an REE-controlled real-time clock + * and on the TEE Trusted Storage for the storage of origins (default). + * 1000: Persistent time based on a TEE-controlled real-time clock + * and the TEE Trusted Storage. + * The real-time clock MUST be out of reach of software attacks + * from the REE. + */ +static const uint32_t ta_time_prot_lvl = 100; + +/* Elliptic Curve Cryptographic support */ +#ifdef CFG_CRYPTO_ECC +static const bool crypto_ecc_en = 1; +#else +static const bool crypto_ecc_en; +#endif + +/* + * Trusted storage anti rollback protection level + * 100: Antirollback enforced at REE level + * 1000: Antirollback TEE-controlled hardware + */ +#ifdef CFG_RPMB_FS +static const uint32_t ts_antiroll_prot_lvl = 1000; +#else +static const uint32_t ts_antiroll_prot_lvl = 100; +#endif + +/* Trusted OS implementation version */ +static const char trustedos_impl_version[] = TO_STR(TEE_IMPL_VERSION); + +/* Trusted OS implementation version (binary value) */ +static const uint32_t trustedos_impl_bin_version; /* 0 by default */ + +/* Trusted OS implementation manufacturer name */ +static const char trustedos_manufacturer[] = TO_STR(CFG_TEE_MANUFACTURER); + +/* Trusted firmware version */ +static const char fw_impl_version[] = TO_STR(CFG_TEE_FW_IMPL_VERSION); + +/* Trusted firmware version (binary value) */ +static const uint32_t fw_impl_bin_version; /* 0 by default */ + +/* Trusted firmware manufacturer name */ +static const char fw_manufacturer[] = TO_STR(CFG_TEE_FW_MANUFACTURER); + +static TEE_Result get_prop_tee_dev_id(struct ts_session *sess __unused, + void *buf, size_t *blen) +{ + TEE_Result res; + TEE_UUID uuid; + const size_t nslen = 5; + uint8_t data[5 + FVR_DIE_ID_NUM_REGS * sizeof(uint32_t)] = { + 'O', 'P', 'T', 'E', 'E' }; + + if (*blen < sizeof(uuid)) { + *blen = sizeof(uuid); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(uuid); + + if (tee_otp_get_die_id(data + nslen, sizeof(data) - nslen)) + return TEE_ERROR_BAD_STATE; + + res = tee_hash_createdigest(TEE_ALG_SHA256, data, sizeof(data), + (uint8_t *)&uuid, sizeof(uuid)); + if (res != TEE_SUCCESS) + return TEE_ERROR_BAD_STATE; + + /* + * Changes the random value into and UUID as specifiec + * in RFC 4122. The magic values are from the example + * code in the RFC. + * + * TEE_UUID is defined slightly different from the RFC, + * but close enough for our purpose. + */ + + uuid.timeHiAndVersion &= 0x0fff; + uuid.timeHiAndVersion |= 5 << 12; + + /* uuid.clock_seq_hi_and_reserved in the RFC */ + uuid.clockSeqAndNode[0] &= 0x3f; + uuid.clockSeqAndNode[0] |= 0x80; + + return copy_to_user(buf, &uuid, sizeof(TEE_UUID)); +} + +static TEE_Result +get_prop_tee_sys_time_prot_level(struct ts_session *sess __unused, + void *buf, size_t *blen) +{ + uint32_t prot; + + if (*blen < sizeof(prot)) { + *blen = sizeof(prot); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(prot); + prot = tee_time_get_sys_time_protection_level(); + return copy_to_user(buf, &prot, sizeof(prot)); +} + +static TEE_Result get_prop_client_id(struct ts_session *sess, + void *buf, size_t *blen) +{ + if (*blen < sizeof(TEE_Identity)) { + *blen = sizeof(TEE_Identity); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(TEE_Identity); + return copy_to_user(buf, &to_ta_session(sess)->clnt_id, + sizeof(TEE_Identity)); +} + +static TEE_Result get_prop_client_endian(struct ts_session *sess __unused, + void *buf, size_t *blen) +{ + const uint32_t endian = 0; /* assume little-endian */ + + if (*blen < sizeof(endian)) { + *blen = sizeof(endian); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(endian); + return copy_to_user(buf, &endian, sizeof(endian)); +} + +static TEE_Result get_prop_ta_app_id(struct ts_session *sess, + void *buf, size_t *blen) +{ + if (*blen < sizeof(TEE_UUID)) { + *blen = sizeof(TEE_UUID); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(TEE_UUID); + return copy_to_user(buf, &sess->ctx->uuid, sizeof(TEE_UUID)); +} + +#ifdef CFG_TA_BTI +static TEE_Result +get_prop_feat_bti_implemented(struct ts_session *sess __unused, void *buf, + size_t *blen) +{ + bool bti_impl = false; + + if (*blen < sizeof(bti_impl)) { + *blen = sizeof(bti_impl); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(bti_impl); + bti_impl = feat_bti_is_implemented(); + + return copy_to_user(buf, &bti_impl, sizeof(bti_impl)); +} +#endif + +#ifdef CFG_TA_PAUTH +static TEE_Result +get_prop_feat_pauth_implemented(struct ts_session *sess __unused, void *buf, + size_t *blen) +{ + bool pauth_impl = false; + + if (*blen < sizeof(pauth_impl)) { + *blen = sizeof(pauth_impl); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(pauth_impl); + pauth_impl = feat_pauth_is_implemented(); + + return copy_to_user(buf, &pauth_impl, sizeof(pauth_impl)); +} +#endif + +#if MEMTAG_IS_ENABLED +static TEE_Result +get_prop_feat_memtag_implemented(struct ts_session *sess __unused, void *buf, + size_t *blen) +{ + uint32_t v = 0; + + if (*blen < sizeof(v)) { + *blen = sizeof(v); + return TEE_ERROR_SHORT_BUFFER; + } + *blen = sizeof(v); + if (memtag_is_enabled()) + v = feat_mte_implemented(); + + return copy_to_user(buf, &v, sizeof(v)); +} +#endif + +/* Properties of the set TEE_PROPSET_CURRENT_CLIENT */ +const struct tee_props tee_propset_client[] = { + { + .name = "gpd.client.identity", + .prop_type = USER_TA_PROP_TYPE_IDENTITY, + .get_prop_func = get_prop_client_id + }, + { + .name = "gpd.client.endian", + .prop_type = USER_TA_PROP_TYPE_U32, + .get_prop_func = get_prop_client_endian + }, +}; + +/* Properties of the set TEE_PROPSET_CURRENT_TA */ +const struct tee_props tee_propset_ta[] = { + { + .name = "gpd.ta.appID", + .prop_type = USER_TA_PROP_TYPE_UUID, + .get_prop_func = get_prop_ta_app_id + }, + + /* + * Following properties are processed directly in libutee: + * TA_PROP_STR_SINGLE_INSTANCE + * TA_PROP_STR_MULTI_SESSION + * TA_PROP_STR_KEEP_ALIVE + * TA_PROP_STR_DATA_SIZE + * TA_PROP_STR_STACK_SIZE + * TA_PROP_STR_VERSION + * TA_PROP_STR_DESCRIPTION + * USER_TA_PROP_TYPE_STRING, + * TA_DESCRIPTION + */ +}; + +/* Properties of the set TEE_PROPSET_TEE_IMPLEMENTATION */ +const struct tee_props tee_propset_tee[] = { + { + .name = "gpd.tee.apiversion", + .prop_type = USER_TA_PROP_TYPE_STRING, + .data = api_vers, + .len = sizeof(api_vers), + }, + { + .name = "gpd.tee.description", + .prop_type = USER_TA_PROP_TYPE_STRING, + .data = descr, .len = sizeof(descr) + }, + { + .name = "gpd.tee.deviceID", + .prop_type = USER_TA_PROP_TYPE_UUID, + .get_prop_func = get_prop_tee_dev_id + }, + { + .name = "gpd.tee.systemTime.protectionLevel", + .prop_type = USER_TA_PROP_TYPE_U32, + .get_prop_func = get_prop_tee_sys_time_prot_level + }, + { + .name = "gpd.tee.TAPersistentTime.protectionLevel", + .prop_type = USER_TA_PROP_TYPE_U32, + .data = &ta_time_prot_lvl, + .len = sizeof(ta_time_prot_lvl) + }, + { + .name = "gpd.tee.cryptography.ecc", + .prop_type = USER_TA_PROP_TYPE_BOOL, + .data = &crypto_ecc_en, + .len = sizeof(crypto_ecc_en) + }, + { + .name = "gpd.tee.trustedStorage.antiRollback.protectionLevel", + .prop_type = USER_TA_PROP_TYPE_U32, + .data = &ts_antiroll_prot_lvl, + .len = sizeof(ts_antiroll_prot_lvl) + }, + { + .name = "gpd.tee.trustedos.implementation.version", + .prop_type = USER_TA_PROP_TYPE_STRING, + .data = trustedos_impl_version, + .len = sizeof(trustedos_impl_version) + }, + { + .name = "gpd.tee.trustedos.implementation.binaryversion", + .prop_type = USER_TA_PROP_TYPE_U32, + .data = &trustedos_impl_bin_version, + .len = sizeof(trustedos_impl_bin_version) + }, + { + .name = "gpd.tee.trustedos.manufacturer", + .prop_type = USER_TA_PROP_TYPE_STRING, + .data = trustedos_manufacturer, + .len = sizeof(trustedos_manufacturer) + }, + { + .name = "gpd.tee.firmware.implementation.version", + .prop_type = USER_TA_PROP_TYPE_STRING, + .data = fw_impl_version, + .len = sizeof(fw_impl_version) + }, + { + .name = "gpd.tee.firmware.implementation.binaryversion", + .prop_type = USER_TA_PROP_TYPE_U32, + .data = &fw_impl_bin_version, + .len = sizeof(fw_impl_bin_version) + }, + { + .name = "gpd.tee.firmware.manufacturer", + .prop_type = USER_TA_PROP_TYPE_STRING, + .data = fw_manufacturer, + .len = sizeof(fw_manufacturer) + }, +#ifdef CFG_TA_BTI + { + .name = "org.trustedfirmware.optee.cpu.feat_bti_implemented", + .prop_type = USER_TA_PROP_TYPE_BOOL, + .get_prop_func = get_prop_feat_bti_implemented + }, +#endif +#ifdef CFG_TA_PAUTH + { + .name = "org.trustedfirmware.optee.cpu.feat_pauth_implemented", + .prop_type = USER_TA_PROP_TYPE_BOOL, + .get_prop_func = get_prop_feat_pauth_implemented + }, +#endif +#if MEMTAG_IS_ENABLED + { + .name = "org.trustedfirmware.optee.cpu.feat_memtag_implemented", + .prop_type = USER_TA_PROP_TYPE_U32, + .get_prop_func = get_prop_feat_memtag_implemented + } +#endif + + /* + * Following properties are processed directly in libutee: + * gpd.tee.arith.maxBigIntSize + */ +}; + +__weak const struct tee_vendor_props vendor_props_client; +__weak const struct tee_vendor_props vendor_props_ta; +__weak const struct tee_vendor_props vendor_props_tee; + +static void get_prop_set(unsigned long prop_set, + const struct tee_props **props, + size_t *size, + const struct tee_props **vendor_props, + size_t *vendor_size) +{ + if ((TEE_PropSetHandle)prop_set == TEE_PROPSET_CURRENT_CLIENT) { + *props = tee_propset_client; + *size = ARRAY_SIZE(tee_propset_client); + *vendor_props = vendor_props_client.props; + *vendor_size = vendor_props_client.len; + } else if ((TEE_PropSetHandle)prop_set == TEE_PROPSET_CURRENT_TA) { + *props = tee_propset_ta; + *size = ARRAY_SIZE(tee_propset_ta); + *vendor_props = vendor_props_ta.props; + *vendor_size = vendor_props_ta.len; + } else if ((TEE_PropSetHandle)prop_set == + TEE_PROPSET_TEE_IMPLEMENTATION) { + *props = tee_propset_tee; + *size = ARRAY_SIZE(tee_propset_tee); + *vendor_props = vendor_props_tee.props; + *vendor_size = vendor_props_tee.len; + } else { + *props = NULL; + *size = 0; + *vendor_props = NULL; + *vendor_size = 0; + } +} + +static const struct tee_props *get_prop_struct(unsigned long prop_set, + unsigned long index) +{ + const struct tee_props *props; + const struct tee_props *vendor_props; + size_t size; + size_t vendor_size; + + get_prop_set(prop_set, &props, &size, &vendor_props, &vendor_size); + + if (index < size) + return &(props[index]); + index -= size; + + if (index < vendor_size) + return &(vendor_props[index]); + + return NULL; +} + +/* + * prop_set is part of TEE_PROPSET_xxx + * index is the index in the Property Set to retrieve + * if name is not NULL, the name of "index" property is returned + * if buf is not NULL, the property is returned + */ +TEE_Result syscall_get_property(unsigned long prop_set, + unsigned long index, + void *name, uint32_t *name_len, + void *buf, uint32_t *blen, + uint32_t *prop_type) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + TEE_Result res2 = TEE_SUCCESS; + const struct tee_props *prop = NULL; + uint32_t klen = 0; + size_t klen_size = 0; + uint32_t elen = 0; + + prop = get_prop_struct(prop_set, index); + if (!prop) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* Get the property type */ + if (prop_type) { + res = copy_to_user(prop_type, &prop->prop_type, + sizeof(*prop_type)); + if (res != TEE_SUCCESS) + return res; + } + + /* Get the property */ + if (buf && blen) { + res = copy_from_user(&klen, blen, sizeof(klen)); + if (res != TEE_SUCCESS) + return res; + + if (prop->get_prop_func) { + klen_size = klen; + res = prop->get_prop_func(sess, buf, &klen_size); + klen = klen_size; + res2 = copy_to_user(blen, &klen, sizeof(*blen)); + } else { + if (klen < prop->len) + res = TEE_ERROR_SHORT_BUFFER; + else + res = copy_to_user(buf, prop->data, prop->len); + res2 = copy_to_user(blen, &prop->len, sizeof(*blen)); + } + if (res2 != TEE_SUCCESS) + return res2; + if (res != TEE_SUCCESS) + return res; + } + + /* Get the property name */ + if (name && name_len) { + res = copy_from_user(&klen, name_len, sizeof(klen)); + if (res != TEE_SUCCESS) + return res; + + elen = strlen(prop->name) + 1; + + if (klen < elen) + res = TEE_ERROR_SHORT_BUFFER; + else + res = copy_to_user(name, prop->name, elen); + res2 = copy_to_user(name_len, &elen, sizeof(*name_len)); + if (res2 != TEE_SUCCESS) + return res2; + if (res != TEE_SUCCESS) + return res; + } + + return res; +} + +/* + * prop_set is part of TEE_PROPSET_xxx + */ +TEE_Result syscall_get_property_name_to_index(unsigned long prop_set, + void *name, + unsigned long name_len, + uint32_t *index) +{ + TEE_Result res = TEE_SUCCESS; + const struct tee_props *props = NULL; + size_t size = 0; + const struct tee_props *vendor_props = NULL; + size_t vendor_size = 0; + char *kname = NULL; + uint32_t i = 0; + + get_prop_set(prop_set, &props, &size, &vendor_props, &vendor_size); + if (!props) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (!name || !name_len) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + kname = malloc(name_len); + if (!kname) + return TEE_ERROR_OUT_OF_MEMORY; + res = copy_from_user(kname, name, name_len); + if (res != TEE_SUCCESS) + goto out; + kname[name_len - 1] = 0; + + res = TEE_ERROR_ITEM_NOT_FOUND; + for (i = 0; i < size; i++) { + if (!strcmp(kname, props[i].name)) { + res = copy_to_user(index, &i, sizeof(*index)); + goto out; + } + } + for (i = size; i < size + vendor_size; i++) { + if (!strcmp(kname, vendor_props[i - size].name)) { + res = copy_to_user(index, &i, sizeof(*index)); + goto out; + } + } + +out: + free_wipe(kname); + return res; +} + +static TEE_Result utee_param_to_param(struct user_ta_ctx *utc, + struct tee_ta_param *p, + struct utee_params *up) +{ + TEE_Result res = TEE_SUCCESS; + size_t n = 0; + uint64_t types = 0; + struct utee_params *up_bbuf = NULL; + + res = BB_MEMDUP_USER(up, sizeof(*up), &up_bbuf); + if (res) + goto out; + + types = up_bbuf->types; + + p->types = types; + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uintptr_t a = up_bbuf->vals[n * 2]; + size_t b = up_bbuf->vals[n * 2 + 1]; + uint32_t flags = TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER; + + switch (TEE_PARAM_TYPE_GET(types, n)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + flags |= TEE_MEMORY_ACCESS_WRITE; + fallthrough; + case TEE_PARAM_TYPE_MEMREF_INPUT: + p->u[n].mem.offs = memtag_strip_tag_vaddr((void *)a); + p->u[n].mem.size = b; + + if (!p->u[n].mem.offs) { + /* Allow NULL memrefs if of size 0 */ + if (p->u[n].mem.size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + p->u[n].mem.mobj = NULL; + break; + } + + p->u[n].mem.mobj = &mobj_virt; + + res = vm_check_access_rights(&utc->uctx, flags, a, b); + if (res) + goto out; + break; + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + p->u[n].val.a = a; + p->u[n].val.b = b; + break; + default: + memset(&p->u[n], 0, sizeof(p->u[n])); + break; + } + } + +out: + bb_free(up_bbuf, sizeof(struct utee_params)); + return res; +} + +/* + * TA invokes some TA with parameter. + * If some parameters are memory references: + * - either the memref is inside TA private RAM: TA is not allowed to expose + * its private RAM: use a temporary memory buffer and copy the data. + * - or the memref is not in the TA private RAM: + * - if the memref was mapped to the TA, TA is allowed to expose it. + * - if so, converts memref virtual address into a physical address. + */ +static TEE_Result tee_svc_copy_param(struct ts_session *sess, + struct ts_session *called_sess, + struct utee_params *callee_params, + struct tee_ta_param *param) +{ + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + void *va = NULL; + size_t n = 0; + size_t s = 0; + + callee_params = memtag_strip_tag(callee_params); + + /* fill 'param' input struct with caller params description buffer */ + if (!callee_params) { + memset(param, 0, sizeof(*param)); + } else { + uint32_t flags = TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER; + + res = vm_check_access_rights(&utc->uctx, flags, + (uaddr_t)callee_params, + sizeof(struct utee_params)); + if (res != TEE_SUCCESS) + return res; + res = utee_param_to_param(utc, param, callee_params); + if (res != TEE_SUCCESS) + return res; + } + + if (called_sess && is_pseudo_ta_ctx(called_sess->ctx)) { + /* pseudo TA borrows the mapping of the calling TA */ + return TEE_SUCCESS; + } + + /* All mobj in param are of type MOJB_TYPE_VIRT */ + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(param->types, n)) { + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + va = (void *)param->u[n].mem.offs; + s = param->u[n].mem.size; + if (!va) { + if (s) + return TEE_ERROR_BAD_PARAMETERS; + break; + } + /* uTA cannot expose its private memory */ + if (vm_buf_is_inside_um_private(&utc->uctx, va, s)) + return TEE_ERROR_BAD_PARAMETERS; + + res = vm_buf_to_mboj_offs(&utc->uctx, va, s, + ¶m->u[n].mem.mobj, + ¶m->u[n].mem.offs); + if (res != TEE_SUCCESS) + return res; + break; + default: + break; + } + } + + return TEE_SUCCESS; +} + +/* + * Back from execution of service: update parameters passed from TA: + * If some parameters were memory references: + * - either the memref was temporary: copy back data and update size + * - or it was the original TA memref: update only the size value. + */ +static TEE_Result tee_svc_update_out_param( + struct tee_ta_param *param, + struct utee_params *usr_param) +{ + size_t n = 0; + uint64_t *vals = usr_param->vals; + uint64_t sz = 0; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + TEE_Result res = TEE_SUCCESS; + uint64_t val_buf[2] = { }; + + switch (TEE_PARAM_TYPE_GET(param->types, n)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + sz = param->u[n].mem.size; + + res = PUT_USER_SCALAR(sz, &usr_param->vals[n * 2 + 1]); + if (res) + return res; + + break; + + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + val_buf[0] = param->u[n].val.a; + val_buf[1] = param->u[n].val.b; + + res = copy_to_user(&vals[n * 2], val_buf, + 2 * sizeof(uint64_t)); + if (res) + return res; + + break; + + default: + continue; + } + } + + return TEE_SUCCESS; +} + +/* Called when a TA calls an OpenSession on another TA */ +TEE_Result syscall_open_ta_session(const TEE_UUID *dest, + unsigned long cancel_req_to, + struct utee_params *usr_param, uint32_t *ta_sess, + uint32_t *ret_orig) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + uint32_t ret_o = TEE_ORIGIN_TEE; + struct tee_ta_session *s = NULL; + TEE_UUID *uuid = malloc(sizeof(TEE_UUID)); + struct tee_ta_param *param = malloc(sizeof(struct tee_ta_param)); + TEE_Identity *clnt_id = malloc(sizeof(TEE_Identity)); + + if (uuid == NULL || param == NULL || clnt_id == NULL) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out_free_only; + } + + memset(param, 0, sizeof(struct tee_ta_param)); + + res = copy_from_user_private(uuid, dest, sizeof(TEE_UUID)); + if (res != TEE_SUCCESS) + goto function_exit; + + clnt_id->login = TEE_LOGIN_TRUSTED_APP; + memcpy(&clnt_id->uuid, &sess->ctx->uuid, sizeof(TEE_UUID)); + + res = tee_svc_copy_param(sess, NULL, usr_param, param); + if (res != TEE_SUCCESS) + goto function_exit; + + res = tee_ta_open_session(&ret_o, &s, &utc->open_sessions, uuid, + clnt_id, cancel_req_to, param); + vm_set_ctx(&utc->ta_ctx.ts_ctx); + if (res != TEE_SUCCESS) + goto function_exit; + + res = tee_svc_update_out_param(param, usr_param); + +function_exit: + if (res == TEE_SUCCESS) + copy_to_user_private(ta_sess, &s->id, sizeof(s->id)); + copy_to_user_private(ret_orig, &ret_o, sizeof(ret_o)); + +out_free_only: + free_wipe(param); + free_wipe(uuid); + free_wipe(clnt_id); + return res; +} + +TEE_Result syscall_close_ta_session(unsigned long ta_sess) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Identity clnt_id = { }; + struct tee_ta_session *s = NULL; + + s = tee_ta_find_session(ta_sess, &utc->open_sessions); + + clnt_id.login = TEE_LOGIN_TRUSTED_APP; + memcpy(&clnt_id.uuid, &sess->ctx->uuid, sizeof(TEE_UUID)); + + return tee_ta_close_session(s, &utc->open_sessions, &clnt_id); +} + +TEE_Result syscall_invoke_ta_command(unsigned long ta_sess, + unsigned long cancel_req_to, unsigned long cmd_id, + struct utee_params *usr_param, uint32_t *ret_orig) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + TEE_Result res2 = TEE_SUCCESS; + uint32_t ret_o = TEE_ORIGIN_TEE; + struct tee_ta_param param = { 0 }; + TEE_Identity clnt_id = { }; + struct tee_ta_session *called_sess = NULL; + + called_sess = tee_ta_get_session((uint32_t)ta_sess, true, + &utc->open_sessions); + if (!called_sess) + return TEE_ERROR_BAD_PARAMETERS; + + clnt_id.login = TEE_LOGIN_TRUSTED_APP; + memcpy(&clnt_id.uuid, &sess->ctx->uuid, sizeof(TEE_UUID)); + + res = tee_svc_copy_param(sess, &called_sess->ts_sess, usr_param, + ¶m); + if (res != TEE_SUCCESS) + goto function_exit; + + res = tee_ta_invoke_command(&ret_o, called_sess, &clnt_id, + cancel_req_to, cmd_id, ¶m); + if (res == TEE_ERROR_TARGET_DEAD) + goto function_exit; + + res2 = tee_svc_update_out_param(¶m, usr_param); + if (res2 != TEE_SUCCESS) { + /* + * Spec for TEE_InvokeTACommand() says: + * "If the return origin is different from + * TEE_ORIGIN_TRUSTED_APP, then the function has failed + * before it could reach the destination Trusted + * Application." + * + * But if we can't update params to the caller we have no + * choice we need to return some error to indicate that + * parameters aren't updated as expected. + */ + ret_o = TEE_ORIGIN_TEE; + res = res2; + } + +function_exit: + tee_ta_put_session(called_sess); + copy_to_user_private(ret_orig, &ret_o, sizeof(ret_o)); + return res; +} + +TEE_Result syscall_check_access_rights(unsigned long flags, const void *buf, + size_t len) +{ + struct ts_session *s = ts_get_current_session(); + + return vm_check_access_rights(&to_user_ta_ctx(s->ctx)->uctx, flags, + memtag_strip_tag_vaddr(buf), len); +} + +TEE_Result syscall_get_cancellation_flag(uint32_t *cancel) +{ + struct ts_session *s = ts_get_current_session(); + uint32_t c = 0; + + c = tee_ta_session_is_cancelled(to_ta_session(s), NULL); + + return copy_to_user(cancel, &c, sizeof(c)); +} + +TEE_Result syscall_unmask_cancellation(uint32_t *old_mask) +{ + struct ts_session *s = ts_get_current_session(); + struct tee_ta_session *sess = NULL; + uint32_t m = 0; + + sess = to_ta_session(s); + m = sess->cancel_mask; + sess->cancel_mask = false; + return copy_to_user(old_mask, &m, sizeof(m)); +} + +TEE_Result syscall_mask_cancellation(uint32_t *old_mask) +{ + struct ts_session *s = ts_get_current_session(); + struct tee_ta_session *sess = NULL; + uint32_t m = 0; + + sess = to_ta_session(s); + m = sess->cancel_mask; + sess->cancel_mask = true; + return copy_to_user(old_mask, &m, sizeof(m)); +} + +TEE_Result syscall_wait(unsigned long timeout) +{ + struct ts_session *s = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + uint32_t mytime = 0; + TEE_Time base_time = { }; + TEE_Time current_time = { }; + + res = tee_time_get_sys_time(&base_time); + if (res != TEE_SUCCESS) + return res; + + while (true) { + res = tee_time_get_sys_time(¤t_time); + if (res != TEE_SUCCESS) + return res; + + if (tee_ta_session_is_cancelled(to_ta_session(s), + ¤t_time)) + return TEE_ERROR_CANCEL; + + mytime = (current_time.seconds - base_time.seconds) * 1000 + + (int)current_time.millis - (int)base_time.millis; + if (mytime >= timeout) + return TEE_SUCCESS; + + tee_time_wait(timeout - mytime); + } + + return res; +} + +TEE_Result syscall_get_time(unsigned long cat, TEE_Time *mytime) +{ + struct ts_session *s = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + TEE_Result res2 = TEE_SUCCESS; + TEE_Time t = { }; + + switch (cat) { + case UTEE_TIME_CAT_SYSTEM: + res = tee_time_get_sys_time(&t); + break; + case UTEE_TIME_CAT_TA_PERSISTENT: + res = tee_time_get_ta_time((const void *)&s->ctx->uuid, &t); + break; + case UTEE_TIME_CAT_REE: + res = tee_time_get_ree_time(&t); + break; + default: + res = TEE_ERROR_BAD_PARAMETERS; + break; + } + + if (res == TEE_SUCCESS || res == TEE_ERROR_OVERFLOW) { + res2 = copy_to_user_private(mytime, &t, sizeof(t)); + if (res2 != TEE_SUCCESS) + res = res2; + } + + return res; +} + +TEE_Result syscall_set_ta_time(const TEE_Time *mytime) +{ + struct ts_session *s = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + TEE_Time t = { }; + + res = copy_from_user_private(&t, mytime, sizeof(t)); + if (res != TEE_SUCCESS) + return res; + + return tee_time_set_ta_time((const void *)&s->ctx->uuid, &t); +} diff --git a/optee_os/core/tee/tee_svc_cryp.c b/optee_os/core/tee/tee_svc_cryp.c new file mode 100644 index 0000000..7250345 --- /dev/null +++ b/optee_os/core/tee/tee_svc_cryp.c @@ -0,0 +1,4752 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, 2022 Linaro Limited + * Copyright (c) 2022, Technology Innovation Institute (TII) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(CFG_CRYPTO_HKDF) +#include +#endif +#if defined(CFG_CRYPTO_CONCAT_KDF) +#include +#endif +#if defined(CFG_CRYPTO_PBKDF2) +#include +#endif + +enum cryp_state { + CRYP_STATE_INITIALIZED = 0, + CRYP_STATE_UNINITIALIZED +}; + +typedef void (*tee_cryp_ctx_finalize_func_t) (void *ctx); +struct tee_cryp_state { + TAILQ_ENTRY(tee_cryp_state) link; + uint32_t algo; + uint32_t mode; + vaddr_t key1; + vaddr_t key2; + void *ctx; + tee_cryp_ctx_finalize_func_t ctx_finalize; + enum cryp_state state; +}; + +struct tee_cryp_obj_secret { + uint32_t key_size; + uint32_t alloc_size; + + /* + * Pseudo code visualize layout of structure + * Next follows data, such as: + * uint8_t data[alloc_size] + * key_size must never exceed alloc_size + */ +}; + +#define TEE_TYPE_ATTR_OPTIONAL BIT(0) +#define TEE_TYPE_ATTR_REQUIRED BIT(1) +#define TEE_TYPE_ATTR_OPTIONAL_GROUP BIT(2) +#define TEE_TYPE_ATTR_SIZE_INDICATOR BIT(3) +#define TEE_TYPE_ATTR_GEN_KEY_OPT BIT(4) +#define TEE_TYPE_ATTR_GEN_KEY_REQ BIT(5) +#define TEE_TYPE_ATTR_BIGNUM_MAXBITS BIT(6) + + /* Handle storing of generic secret keys of varying lengths */ +#define ATTR_OPS_INDEX_SECRET 0 + /* Convert to/from big-endian byte array and provider-specific bignum */ +#define ATTR_OPS_INDEX_BIGNUM 1 + /* Convert to/from value attribute depending on direction */ +#define ATTR_OPS_INDEX_VALUE 2 + /* Convert to/from curve25519 attribute depending on direction */ +#define ATTR_OPS_INDEX_25519 3 +#define ATTR_OPS_INDEX_448 4 + + /* Curve25519 key bytes size is always 32 bytes*/ +#define KEY_SIZE_BYTES_25519 UL(32) +#define KEY_SIZE_BYTES_448 UL(56) + /* TEE Internal Core API v1.3.1, Table 6-8 */ +#define TEE_ED25519_CTX_MAX_LENGTH 255 + +struct tee_cryp_obj_type_attrs { + uint32_t attr_id; + uint16_t flags; + uint16_t ops_index; + uint16_t raw_offs; + uint16_t raw_size; +}; + +#define RAW_DATA(_x, _y) \ + .raw_offs = offsetof(_x, _y), .raw_size = MEMBER_SIZE(_x, _y) + +static const struct tee_cryp_obj_type_attrs + tee_cryp_obj_secret_value_attrs[] = { + { + .attr_id = TEE_ATTR_SECRET_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_SECRET, + .raw_offs = 0, + .raw_size = 0 + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_rsa_pub_key_attrs[] = { + { + .attr_id = TEE_ATTR_RSA_MODULUS, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_public_key, n) + }, + + { + .attr_id = TEE_ATTR_RSA_PUBLIC_EXPONENT, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_public_key, e) + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_rsa_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_RSA_MODULUS, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, n) + }, + + { + .attr_id = TEE_ATTR_RSA_PUBLIC_EXPONENT, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_GEN_KEY_OPT, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, e) + }, + + { + .attr_id = TEE_ATTR_RSA_PRIVATE_EXPONENT, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, d) + }, + + { + .attr_id = TEE_ATTR_RSA_PRIME1, + .flags = TEE_TYPE_ATTR_OPTIONAL_GROUP, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, p) + }, + + { + .attr_id = TEE_ATTR_RSA_PRIME2, + .flags = TEE_TYPE_ATTR_OPTIONAL_GROUP, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, q) + }, + + { + .attr_id = TEE_ATTR_RSA_EXPONENT1, + .flags = TEE_TYPE_ATTR_OPTIONAL_GROUP, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, dp) + }, + + { + .attr_id = TEE_ATTR_RSA_EXPONENT2, + .flags = TEE_TYPE_ATTR_OPTIONAL_GROUP, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, dq) + }, + + { + .attr_id = TEE_ATTR_RSA_COEFFICIENT, + .flags = TEE_TYPE_ATTR_OPTIONAL_GROUP, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct rsa_keypair, qp) + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_dsa_pub_key_attrs[] = { + { + .attr_id = TEE_ATTR_DSA_PRIME, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_BIGNUM_MAXBITS | + TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_public_key, p) + }, + + { + .attr_id = TEE_ATTR_DSA_SUBPRIME, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_public_key, q) + }, + + { + .attr_id = TEE_ATTR_DSA_BASE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_BIGNUM_MAXBITS, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_public_key, g) + }, + + { + .attr_id = TEE_ATTR_DSA_PUBLIC_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_BIGNUM_MAXBITS, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_public_key, y) + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_dsa_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_DSA_PRIME, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_GEN_KEY_REQ | + TEE_TYPE_ATTR_BIGNUM_MAXBITS | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_keypair, p) + }, + + { + .attr_id = TEE_ATTR_DSA_SUBPRIME, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_GEN_KEY_REQ, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_keypair, q) + }, + + { + .attr_id = TEE_ATTR_DSA_BASE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_GEN_KEY_REQ | + TEE_TYPE_ATTR_BIGNUM_MAXBITS, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_keypair, g) + }, + + { + .attr_id = TEE_ATTR_DSA_PRIVATE_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_BIGNUM_MAXBITS, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_keypair, x) + }, + + { + .attr_id = TEE_ATTR_DSA_PUBLIC_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_BIGNUM_MAXBITS, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dsa_keypair, y) + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_dh_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_DH_PRIME, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR | + TEE_TYPE_ATTR_GEN_KEY_REQ, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dh_keypair, p) + }, + + { + .attr_id = TEE_ATTR_DH_BASE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_GEN_KEY_REQ, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dh_keypair, g) + }, + + { + .attr_id = TEE_ATTR_DH_PUBLIC_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dh_keypair, y) + }, + + { + .attr_id = TEE_ATTR_DH_PRIVATE_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dh_keypair, x) + }, + + { + .attr_id = TEE_ATTR_DH_SUBPRIME, + .flags = TEE_TYPE_ATTR_OPTIONAL_GROUP | TEE_TYPE_ATTR_GEN_KEY_OPT, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct dh_keypair, q) + }, + + { + .attr_id = TEE_ATTR_DH_X_BITS, + .flags = TEE_TYPE_ATTR_GEN_KEY_OPT, + .ops_index = ATTR_OPS_INDEX_VALUE, + RAW_DATA(struct dh_keypair, xbits) + }, +}; + +#if defined(CFG_CRYPTO_HKDF) +static const struct tee_cryp_obj_type_attrs + tee_cryp_obj_hkdf_ikm_attrs[] = { + { + .attr_id = TEE_ATTR_HKDF_IKM, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_SECRET, + .raw_offs = 0, + .raw_size = 0 + }, +}; +#endif + +#if defined(CFG_CRYPTO_CONCAT_KDF) +static const struct tee_cryp_obj_type_attrs + tee_cryp_obj_concat_kdf_z_attrs[] = { + { + .attr_id = TEE_ATTR_CONCAT_KDF_Z, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_SECRET, + .raw_offs = 0, + .raw_size = 0 + }, +}; +#endif + +#if defined(CFG_CRYPTO_PBKDF2) +static const struct tee_cryp_obj_type_attrs + tee_cryp_obj_pbkdf2_passwd_attrs[] = { + { + .attr_id = TEE_ATTR_PBKDF2_PASSWORD, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_SECRET, + .raw_offs = 0, + .raw_size = 0 + }, +}; +#endif + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_ecc_pub_key_attrs[] = { + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_X, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_public_key, x) + }, + + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_Y, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_public_key, y) + }, + + { + .attr_id = TEE_ATTR_ECC_CURVE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR, + .ops_index = ATTR_OPS_INDEX_VALUE, + RAW_DATA(struct ecc_public_key, curve) + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_ecc_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_ECC_PRIVATE_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_keypair, d) + }, + + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_X, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_keypair, x) + }, + + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_Y, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_keypair, y) + }, + + { + .attr_id = TEE_ATTR_ECC_CURVE, + .flags = TEE_TYPE_ATTR_REQUIRED | TEE_TYPE_ATTR_SIZE_INDICATOR | + TEE_TYPE_ATTR_GEN_KEY_REQ, + .ops_index = ATTR_OPS_INDEX_VALUE, + RAW_DATA(struct ecc_keypair, curve) + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_sm2_pub_key_attrs[] = { + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_X, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_public_key, x) + }, + + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_Y, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_public_key, y) + }, +}; + +static const struct tee_cryp_obj_type_attrs tee_cryp_obj_sm2_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_ECC_PRIVATE_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_keypair, d) + }, + + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_X, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_keypair, x) + }, + + { + .attr_id = TEE_ATTR_ECC_PUBLIC_VALUE_Y, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_BIGNUM, + RAW_DATA(struct ecc_keypair, y) + }, +}; + +static +const struct tee_cryp_obj_type_attrs tee_cryp_obj_x25519_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_X25519_PRIVATE_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_25519, + RAW_DATA(struct montgomery_keypair, priv) + }, + + { + .attr_id = TEE_ATTR_X25519_PUBLIC_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_25519, + RAW_DATA(struct montgomery_keypair, pub) + }, +}; + +static +const struct tee_cryp_obj_type_attrs tee_cryp_obj_x448_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_X448_PRIVATE_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_448, + RAW_DATA(struct montgomery_keypair, priv) + }, + + { + .attr_id = TEE_ATTR_X448_PUBLIC_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_448, + RAW_DATA(struct montgomery_keypair, pub) + }, +}; + +static +const struct tee_cryp_obj_type_attrs tee_cryp_obj_ed25519_pub_key_attrs[] = { + { + .attr_id = TEE_ATTR_ED25519_PUBLIC_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_25519, + RAW_DATA(struct ed25519_public_key, pub) + }, +}; + +static +const struct tee_cryp_obj_type_attrs tee_cryp_obj_ed25519_keypair_attrs[] = { + { + .attr_id = TEE_ATTR_ED25519_PRIVATE_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_25519, + RAW_DATA(struct ed25519_keypair, priv) + }, + + { + .attr_id = TEE_ATTR_ED25519_PUBLIC_VALUE, + .flags = TEE_TYPE_ATTR_REQUIRED, + .ops_index = ATTR_OPS_INDEX_25519, + RAW_DATA(struct ed25519_keypair, pub) + }, +}; + +struct tee_cryp_obj_type_props { + TEE_ObjectType obj_type; + uint16_t min_size; /* may not be smaller than this */ + uint16_t max_size; /* may not be larger than this */ + uint16_t alloc_size; /* this many bytes are allocated to hold data */ + uint8_t quanta; /* may only be an multiple of this */ + + uint8_t num_type_attrs; + const struct tee_cryp_obj_type_attrs *type_attrs; +}; + +#define PROP(obj_type, quanta, min_size, max_size, alloc_size, type_attrs) \ + { (obj_type), (min_size), (max_size), (alloc_size), (quanta), \ + ARRAY_SIZE(type_attrs), (type_attrs) } + +static const struct tee_cryp_obj_type_props tee_cryp_obj_props[] = { + PROP(TEE_TYPE_AES, 64, 128, 256, /* valid sizes 128, 192, 256 */ + 256 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_DES, 64, 64, 64, + /* Valid size 64 with parity */ + 64 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_DES3, 64, 128, 192, + /* Valid sizes 128, 192 with parity */ + 192 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_SM4, 128, 128, 128, + 128 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_MD5, 8, 64, 512, + 512 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), +#if defined(CFG_HMAC_64_1024_RANGE) + PROP(TEE_TYPE_HMAC_SHA1, 8, 64, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA224, 8, 64, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA256, 8, 64, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA384, 8, 64, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA512, 8, 64, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), +#else + PROP(TEE_TYPE_HMAC_SHA1, 8, 80, 512, + 512 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA224, 8, 112, 512, + 512 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA256, 8, 192, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA384, 8, 256, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA512, 8, 256, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), +#endif + PROP(TEE_TYPE_HMAC_SHA3_224, 8, 192, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA3_256, 8, 256, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA3_384, 8, 256, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SHA3_512, 8, 256, 1024, + 1024 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_HMAC_SM3, 8, 80, 1024, + 512 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), + PROP(TEE_TYPE_GENERIC_SECRET, 8, 0, 4096, + 4096 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_secret_value_attrs), +#if defined(CFG_CRYPTO_HKDF) + PROP(TEE_TYPE_HKDF_IKM, 8, 0, 4096, + 4096 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_hkdf_ikm_attrs), +#endif +#if defined(CFG_CRYPTO_CONCAT_KDF) + PROP(TEE_TYPE_CONCAT_KDF_Z, 8, 0, 4096, + 4096 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_concat_kdf_z_attrs), +#endif +#if defined(CFG_CRYPTO_PBKDF2) + PROP(TEE_TYPE_PBKDF2_PASSWORD, 8, 0, 4096, + 4096 / 8 + sizeof(struct tee_cryp_obj_secret), + tee_cryp_obj_pbkdf2_passwd_attrs), +#endif + PROP(TEE_TYPE_RSA_PUBLIC_KEY, 1, 256, CFG_CORE_BIGNUM_MAX_BITS, + sizeof(struct rsa_public_key), + tee_cryp_obj_rsa_pub_key_attrs), + + PROP(TEE_TYPE_RSA_KEYPAIR, 1, 256, CFG_CORE_BIGNUM_MAX_BITS, + sizeof(struct rsa_keypair), + tee_cryp_obj_rsa_keypair_attrs), + + PROP(TEE_TYPE_DSA_PUBLIC_KEY, 64, 512, 3072, + sizeof(struct dsa_public_key), + tee_cryp_obj_dsa_pub_key_attrs), + + PROP(TEE_TYPE_DSA_KEYPAIR, 64, 512, 3072, + sizeof(struct dsa_keypair), + tee_cryp_obj_dsa_keypair_attrs), + + PROP(TEE_TYPE_DH_KEYPAIR, 1, 256, 2048, + sizeof(struct dh_keypair), + tee_cryp_obj_dh_keypair_attrs), + + PROP(TEE_TYPE_ECDSA_PUBLIC_KEY, 1, 192, 521, + sizeof(struct ecc_public_key), + tee_cryp_obj_ecc_pub_key_attrs), + + PROP(TEE_TYPE_ECDSA_KEYPAIR, 1, 192, 521, + sizeof(struct ecc_keypair), + tee_cryp_obj_ecc_keypair_attrs), + + PROP(TEE_TYPE_ECDH_PUBLIC_KEY, 1, 192, 521, + sizeof(struct ecc_public_key), + tee_cryp_obj_ecc_pub_key_attrs), + + PROP(TEE_TYPE_ECDH_KEYPAIR, 1, 192, 521, + sizeof(struct ecc_keypair), + tee_cryp_obj_ecc_keypair_attrs), + + PROP(TEE_TYPE_SM2_DSA_PUBLIC_KEY, 1, 256, 256, + sizeof(struct ecc_public_key), + tee_cryp_obj_sm2_pub_key_attrs), + + PROP(TEE_TYPE_SM2_DSA_KEYPAIR, 1, 256, 256, + sizeof(struct ecc_keypair), + tee_cryp_obj_sm2_keypair_attrs), + + PROP(TEE_TYPE_SM2_PKE_PUBLIC_KEY, 1, 256, 256, + sizeof(struct ecc_public_key), + tee_cryp_obj_sm2_pub_key_attrs), + + PROP(TEE_TYPE_SM2_PKE_KEYPAIR, 1, 256, 256, + sizeof(struct ecc_keypair), + tee_cryp_obj_sm2_keypair_attrs), + + PROP(TEE_TYPE_SM2_KEP_PUBLIC_KEY, 1, 256, 256, + sizeof(struct ecc_public_key), + tee_cryp_obj_sm2_pub_key_attrs), + + PROP(TEE_TYPE_SM2_KEP_KEYPAIR, 1, 256, 256, + sizeof(struct ecc_keypair), + tee_cryp_obj_sm2_keypair_attrs), + + PROP(TEE_TYPE_X25519_KEYPAIR, 1, 256, 256, + sizeof(struct montgomery_keypair), + tee_cryp_obj_x25519_keypair_attrs), + + PROP(TEE_TYPE_X448_KEYPAIR, 1, 448, 448, + sizeof(struct montgomery_keypair), + tee_cryp_obj_x448_keypair_attrs), + + PROP(TEE_TYPE_ED25519_PUBLIC_KEY, 1, 256, 256, + sizeof(struct ed25519_public_key), + tee_cryp_obj_ed25519_pub_key_attrs), + + PROP(TEE_TYPE_ED25519_KEYPAIR, 1, 256, 256, + sizeof(struct ed25519_keypair), + tee_cryp_obj_ed25519_keypair_attrs), +}; + +struct attr_ops { + TEE_Result (*from_user)(void *attr, const void *buffer, size_t size); + TEE_Result (*to_user)(void *attr, struct ts_session *sess, + void *buffer, uint64_t *size); + TEE_Result (*to_binary)(void *attr, void *data, size_t data_len, + size_t *offs); + bool (*from_binary)(void *attr, const void *data, size_t data_len, + size_t *offs); + TEE_Result (*from_obj)(void *attr, void *src_attr); + void (*free)(void *attr); + void (*clear)(void *attr); +}; + +static TEE_Result op_u32_to_binary_helper(uint32_t v, uint8_t *data, + size_t data_len, size_t *offs) +{ + uint32_t field; + size_t next_offs; + + if (ADD_OVERFLOW(*offs, sizeof(field), &next_offs)) + return TEE_ERROR_OVERFLOW; + + if (data && next_offs <= data_len) { + field = TEE_U32_TO_BIG_ENDIAN(v); + memcpy(data + *offs, &field, sizeof(field)); + } + (*offs) = next_offs; + + return TEE_SUCCESS; +} + +static bool op_u32_from_binary_helper(uint32_t *v, const uint8_t *data, + size_t data_len, size_t *offs) +{ + uint32_t field; + + if (!data || (*offs + sizeof(field)) > data_len) + return false; + + memcpy(&field, data + *offs, sizeof(field)); + *v = TEE_U32_FROM_BIG_ENDIAN(field); + (*offs) += sizeof(field); + return true; +} + +static TEE_Result op_attr_secret_value_from_user(void *attr, const void *buffer, + size_t size) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_cryp_obj_secret *key = attr; + + /* Data size has to fit in allocated buffer */ + if (size > key->alloc_size) + return TEE_ERROR_SECURITY; + res = copy_from_user(key + 1, buffer, size); + if (!res) + key->key_size = size; + + return res; +} + +static TEE_Result op_attr_secret_value_to_user(void *attr, + struct ts_session *sess __unused, + void *buffer, uint64_t *size) +{ + TEE_Result res; + struct tee_cryp_obj_secret *key = attr; + uint64_t s; + uint64_t key_size; + + res = copy_from_user(&s, size, sizeof(s)); + if (res != TEE_SUCCESS) + return res; + + key_size = key->key_size; + res = copy_to_user(size, &key_size, sizeof(key_size)); + if (res != TEE_SUCCESS) + return res; + + if (s < key->key_size || !buffer) + return TEE_ERROR_SHORT_BUFFER; + + return copy_to_user(buffer, key + 1, key->key_size); +} + +static TEE_Result op_attr_secret_value_to_binary(void *attr, void *data, + size_t data_len, size_t *offs) +{ + TEE_Result res; + struct tee_cryp_obj_secret *key = attr; + size_t next_offs; + + res = op_u32_to_binary_helper(key->key_size, data, data_len, offs); + if (res != TEE_SUCCESS) + return res; + + if (ADD_OVERFLOW(*offs, key->key_size, &next_offs)) + return TEE_ERROR_OVERFLOW; + + if (data && next_offs <= data_len) + memcpy((uint8_t *)data + *offs, key + 1, key->key_size); + (*offs) = next_offs; + + return TEE_SUCCESS; +} + +static bool op_attr_secret_value_from_binary(void *attr, const void *data, + size_t data_len, size_t *offs) +{ + struct tee_cryp_obj_secret *key = attr; + uint32_t s; + + if (!op_u32_from_binary_helper(&s, data, data_len, offs)) + return false; + + if ((*offs + s) > data_len) + return false; + + /* Data size has to fit in allocated buffer */ + if (s > key->alloc_size) + return false; + key->key_size = s; + memcpy(key + 1, (const uint8_t *)data + *offs, s); + (*offs) += s; + return true; +} + + +static TEE_Result op_attr_secret_value_from_obj(void *attr, void *src_attr) +{ + struct tee_cryp_obj_secret *key = attr; + struct tee_cryp_obj_secret *src_key = src_attr; + + if (src_key->key_size > key->alloc_size) + return TEE_ERROR_BAD_STATE; + memcpy(key + 1, src_key + 1, src_key->key_size); + key->key_size = src_key->key_size; + return TEE_SUCCESS; +} + +static void op_attr_secret_value_clear(void *attr) +{ + struct tee_cryp_obj_secret *key = attr; + + key->key_size = 0; + memzero_explicit(key + 1, key->alloc_size); +} + +static TEE_Result op_attr_bignum_from_user(void *attr, const void *buffer, + size_t size) +{ + TEE_Result res = TEE_SUCCESS; + struct bignum **bn = attr; + void *bbuf = NULL; + + res = bb_memdup_user(buffer, size, &bbuf); + if (res) + return res; + + res = crypto_bignum_bin2bn(bbuf, size, *bn); + + bb_free(bbuf, size); + + return res; +} + +static TEE_Result op_attr_bignum_to_user(void *attr, + struct ts_session *sess __unused, + void *buffer, uint64_t *size) +{ + TEE_Result res = TEE_SUCCESS; + struct bignum **bn = attr; + uint64_t req_size = 0; + uint64_t s = 0; + void *bbuf = NULL; + + res = copy_from_user(&s, size, sizeof(s)); + if (res != TEE_SUCCESS) + return res; + + req_size = crypto_bignum_num_bytes(*bn); + res = copy_to_user(size, &req_size, sizeof(req_size)); + if (res != TEE_SUCCESS) + return res; + if (!req_size) + return TEE_SUCCESS; + if (s < req_size || !buffer) + return TEE_ERROR_SHORT_BUFFER; + + bbuf = bb_alloc(req_size); + if (!bbuf) + return TEE_ERROR_OUT_OF_MEMORY; + + /* + * Write the bignum (wich raw data points to) into an array of + * bytes (stored in buffer) + */ + crypto_bignum_bn2bin(*bn, bbuf); + res = copy_to_user(buffer, bbuf, req_size); + + bb_free(bbuf, req_size); + return res; +} + +static TEE_Result op_attr_bignum_to_binary(void *attr, void *data, + size_t data_len, size_t *offs) +{ + TEE_Result res; + struct bignum **bn = attr; + uint32_t n = crypto_bignum_num_bytes(*bn); + size_t next_offs; + + res = op_u32_to_binary_helper(n, data, data_len, offs); + if (res != TEE_SUCCESS) + return res; + + if (ADD_OVERFLOW(*offs, n, &next_offs)) + return TEE_ERROR_OVERFLOW; + + if (data && next_offs <= data_len) + crypto_bignum_bn2bin(*bn, (uint8_t *)data + *offs); + (*offs) = next_offs; + + return TEE_SUCCESS; +} + +static bool op_attr_bignum_from_binary(void *attr, const void *data, + size_t data_len, size_t *offs) +{ + struct bignum **bn = attr; + uint32_t n; + + if (!op_u32_from_binary_helper(&n, data, data_len, offs)) + return false; + + if ((*offs + n) > data_len) + return false; + if (crypto_bignum_bin2bn((const uint8_t *)data + *offs, n, *bn)) + return false; + (*offs) += n; + return true; +} + +static TEE_Result op_attr_bignum_from_obj(void *attr, void *src_attr) +{ + struct bignum **bn = attr; + struct bignum **src_bn = src_attr; + + crypto_bignum_copy(*bn, *src_bn); + return TEE_SUCCESS; +} + +static void op_attr_bignum_clear(void *attr) +{ + struct bignum **bn = attr; + + crypto_bignum_clear(*bn); +} + +static void op_attr_bignum_free(void *attr) +{ + struct bignum **bn = attr; + + crypto_bignum_free(bn); +} + +static TEE_Result op_attr_value_from_user(void *attr, const void *buffer, + size_t size) +{ + uint32_t *v = attr; + + if (size != sizeof(uint32_t) * 2) + return TEE_ERROR_GENERIC; /* "can't happen */ + + /* Note that only the first value is copied */ + memcpy(v, buffer, sizeof(uint32_t)); + return TEE_SUCCESS; +} + +static TEE_Result op_attr_value_to_user(void *attr, + struct ts_session *sess __unused, + void *buffer, uint64_t *size) +{ + TEE_Result res; + uint32_t *v = attr; + uint64_t s; + uint32_t value[2] = { *v }; + uint64_t req_size = sizeof(value); + + res = copy_from_user(&s, size, sizeof(s)); + if (res != TEE_SUCCESS) + return res; + + if (s < req_size || !buffer) + return TEE_ERROR_SHORT_BUFFER; + + return copy_to_user(buffer, value, req_size); +} + +static TEE_Result op_attr_value_to_binary(void *attr, void *data, + size_t data_len, size_t *offs) +{ + uint32_t *v = attr; + + return op_u32_to_binary_helper(*v, data, data_len, offs); +} + +static bool op_attr_value_from_binary(void *attr, const void *data, + size_t data_len, size_t *offs) +{ + uint32_t *v = attr; + + return op_u32_from_binary_helper(v, data, data_len, offs); +} + +static TEE_Result op_attr_value_from_obj(void *attr, void *src_attr) +{ + uint32_t *v = attr; + uint32_t *src_v = src_attr; + + *v = *src_v; + return TEE_SUCCESS; +} + +static void op_attr_value_clear(void *attr) +{ + uint32_t *v = attr; + + *v = 0; +} + +static TEE_Result op_attr_25519_from_user(void *attr, const void *buffer, + size_t size) +{ + uint8_t **key = attr; + + if (size != KEY_SIZE_BYTES_25519 || !*key) + return TEE_ERROR_SECURITY; + + return copy_from_user(*key, buffer, size); +} + +static TEE_Result op_attr_25519_to_user(void *attr, + struct ts_session *sess __unused, + void *buffer, uint64_t *size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t **key = attr; + uint64_t s = 0; + uint64_t key_size = (uint64_t)KEY_SIZE_BYTES_25519; + + res = copy_from_user(&s, size, sizeof(s)); + if (res != TEE_SUCCESS) + return res; + + res = copy_to_user(size, &key_size, sizeof(key_size)); + if (res != TEE_SUCCESS) + return res; + + if (s < key_size || !buffer) + return TEE_ERROR_SHORT_BUFFER; + + return copy_to_user(buffer, *key, key_size); +} + +static TEE_Result op_attr_25519_to_binary(void *attr, void *data, + size_t data_len, size_t *offs) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t **key = attr; + size_t next_offs = 0; + uint64_t key_size = (uint64_t)KEY_SIZE_BYTES_25519; + + res = op_u32_to_binary_helper(key_size, data, data_len, offs); + if (res != TEE_SUCCESS) + return res; + + if (ADD_OVERFLOW(*offs, key_size, &next_offs)) + return TEE_ERROR_OVERFLOW; + + if (data && next_offs <= data_len) + memcpy((uint8_t *)data + *offs, *key, key_size); + *offs = next_offs; + + return TEE_SUCCESS; +} + +static bool op_attr_25519_from_binary(void *attr, const void *data, + size_t data_len, size_t *offs) +{ + uint8_t **key = attr; + uint32_t s = 0; + + if (!op_u32_from_binary_helper(&s, data, data_len, offs)) + return false; + + if (*offs + s > data_len) + return false; + + if (s > (uint32_t)KEY_SIZE_BYTES_25519) + return false; + + memcpy(*key, (const uint8_t *)data + *offs, s); + *offs += s; + return true; +} + +static TEE_Result op_attr_25519_from_obj(void *attr, void *src_attr) +{ + uint8_t **key = attr; + uint8_t **src_key = src_attr; + + if (!*key || !*src_key) + return TEE_ERROR_SECURITY; + + memcpy(*key, *src_key, KEY_SIZE_BYTES_25519); + + return TEE_SUCCESS; +} + +static void op_attr_25519_clear(void *attr) +{ + uint8_t **key = attr; + + assert(*key); + + memzero_explicit(*key, KEY_SIZE_BYTES_25519); +} + +static void op_attr_25519_free(void *attr) +{ + uint8_t **key = attr; + + op_attr_25519_clear(attr); + free(*key); +} + +static const struct attr_ops attr_ops[] = { + [ATTR_OPS_INDEX_SECRET] = { + .from_user = op_attr_secret_value_from_user, + .to_user = op_attr_secret_value_to_user, + .to_binary = op_attr_secret_value_to_binary, + .from_binary = op_attr_secret_value_from_binary, + .from_obj = op_attr_secret_value_from_obj, + .free = op_attr_secret_value_clear, /* not a typo */ + .clear = op_attr_secret_value_clear, + }, + [ATTR_OPS_INDEX_BIGNUM] = { + .from_user = op_attr_bignum_from_user, + .to_user = op_attr_bignum_to_user, + .to_binary = op_attr_bignum_to_binary, + .from_binary = op_attr_bignum_from_binary, + .from_obj = op_attr_bignum_from_obj, + .free = op_attr_bignum_free, + .clear = op_attr_bignum_clear, + }, + [ATTR_OPS_INDEX_VALUE] = { + .from_user = op_attr_value_from_user, + .to_user = op_attr_value_to_user, + .to_binary = op_attr_value_to_binary, + .from_binary = op_attr_value_from_binary, + .from_obj = op_attr_value_from_obj, + .free = op_attr_value_clear, /* not a typo */ + .clear = op_attr_value_clear, + }, + [ATTR_OPS_INDEX_25519] = { + .from_user = op_attr_25519_from_user, + .to_user = op_attr_25519_to_user, + .to_binary = op_attr_25519_to_binary, + .from_binary = op_attr_25519_from_binary, + .from_obj = op_attr_25519_from_obj, + .free = op_attr_25519_free, + .clear = op_attr_25519_clear, + }, +}; + +static TEE_Result get_user_u64_as_size_t(size_t *dst, uint64_t *src) +{ + uint64_t d = 0; + TEE_Result res = copy_from_user(&d, src, sizeof(d)); + + /* + * On 32-bit systems a size_t can't hold a uint64_t so we need to + * check that the value isn't too large. + */ + if (!res && ADD_OVERFLOW(0, d, dst)) + return TEE_ERROR_OVERFLOW; + + return res; +} + +static TEE_Result put_user_u64(uint64_t *dst, size_t value) +{ + uint64_t v = value; + + return copy_to_user(dst, &v, sizeof(v)); +} + +TEE_Result syscall_cryp_obj_get_info(unsigned long obj, + struct utee_object_info *info) +{ + struct ts_session *sess = ts_get_current_session(); + struct utee_object_info o_info = { }; + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), + uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + goto exit; + + o_info.obj_type = o->info.objectType; + o_info.obj_size = o->info.objectSize; + o_info.max_obj_size = o->info.maxObjectSize; + o_info.obj_usage = o->info.objectUsage; + o_info.data_size = o->info.dataSize; + o_info.data_pos = o->info.dataPosition; + o_info.handle_flags = o->info.handleFlags; + res = copy_to_user_private(info, &o_info, sizeof(o_info)); + +exit: + return res; +} + +TEE_Result syscall_cryp_obj_restrict_usage(unsigned long obj, + unsigned long usage) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + goto exit; + + o->info.objectUsage &= usage; + +exit: + return res; +} + +static int tee_svc_cryp_obj_find_type_attr_idx( + uint32_t attr_id, + const struct tee_cryp_obj_type_props *type_props) +{ + size_t n; + + for (n = 0; n < type_props->num_type_attrs; n++) { + if (attr_id == type_props->type_attrs[n].attr_id) + return n; + } + return -1; +} + +static const struct tee_cryp_obj_type_props *tee_svc_find_type_props( + TEE_ObjectType obj_type) +{ + size_t n; + + for (n = 0; n < ARRAY_SIZE(tee_cryp_obj_props); n++) { + if (tee_cryp_obj_props[n].obj_type == obj_type) + return tee_cryp_obj_props + n; + } + + return NULL; +} + +/* Set an attribute on an object */ +static void set_attribute(struct tee_obj *o, + const struct tee_cryp_obj_type_props *props, + uint32_t attr) +{ + int idx = tee_svc_cryp_obj_find_type_attr_idx(attr, props); + + if (idx < 0) + return; + o->have_attrs |= BIT(idx); +} + +/* Get an attribute on an object */ +static uint32_t get_attribute(const struct tee_obj *o, + const struct tee_cryp_obj_type_props *props, + uint32_t attr) +{ + int idx = tee_svc_cryp_obj_find_type_attr_idx(attr, props); + + if (idx < 0) + return 0; + return o->have_attrs & BIT(idx); +} + +TEE_Result syscall_cryp_obj_get_attr(unsigned long obj, unsigned long attr_id, + void *buffer, uint64_t *size) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + const struct tee_cryp_obj_type_props *type_props = NULL; + int idx = 0; + const struct attr_ops *ops = NULL; + void *attr = NULL; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* Check that the object is initialized */ + if (!(o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED)) + return TEE_ERROR_BAD_PARAMETERS; + + /* Check that getting the attribute is allowed */ + if (!(attr_id & TEE_ATTR_FLAG_PUBLIC) && + !(o->info.objectUsage & TEE_USAGE_EXTRACTABLE)) + return TEE_ERROR_BAD_PARAMETERS; + + type_props = tee_svc_find_type_props(o->info.objectType); + if (!type_props) { + /* Unknown object type, "can't happen" */ + return TEE_ERROR_BAD_STATE; + } + + idx = tee_svc_cryp_obj_find_type_attr_idx(attr_id, type_props); + if ((idx < 0) || ((o->have_attrs & (1 << idx)) == 0)) + return TEE_ERROR_ITEM_NOT_FOUND; + + ops = attr_ops + type_props->type_attrs[idx].ops_index; + attr = (uint8_t *)o->attr + type_props->type_attrs[idx].raw_offs; + return ops->to_user(attr, sess, buffer, size); +} + +void tee_obj_attr_free(struct tee_obj *o) +{ + const struct tee_cryp_obj_type_props *tp; + size_t n; + + if (!o->attr) + return; + tp = tee_svc_find_type_props(o->info.objectType); + if (!tp) + return; + + for (n = 0; n < tp->num_type_attrs; n++) { + const struct tee_cryp_obj_type_attrs *ta = tp->type_attrs + n; + + attr_ops[ta->ops_index].free((uint8_t *)o->attr + ta->raw_offs); + } +} + +void tee_obj_attr_clear(struct tee_obj *o) +{ + const struct tee_cryp_obj_type_props *tp; + size_t n; + + if (!o->attr) + return; + tp = tee_svc_find_type_props(o->info.objectType); + if (!tp) + return; + + for (n = 0; n < tp->num_type_attrs; n++) { + const struct tee_cryp_obj_type_attrs *ta = tp->type_attrs + n; + + attr_ops[ta->ops_index].clear((uint8_t *)o->attr + + ta->raw_offs); + } +} + +TEE_Result tee_obj_attr_to_binary(struct tee_obj *o, void *data, + size_t *data_len) +{ + const struct tee_cryp_obj_type_props *tp; + size_t n; + size_t offs = 0; + size_t len = data ? *data_len : 0; + TEE_Result res; + + if (o->info.objectType == TEE_TYPE_DATA) { + *data_len = 0; + return TEE_SUCCESS; /* pure data object */ + } + if (!o->attr) + return TEE_ERROR_BAD_STATE; + tp = tee_svc_find_type_props(o->info.objectType); + if (!tp) + return TEE_ERROR_BAD_STATE; + + for (n = 0; n < tp->num_type_attrs; n++) { + const struct tee_cryp_obj_type_attrs *ta = tp->type_attrs + n; + void *attr = (uint8_t *)o->attr + ta->raw_offs; + + res = attr_ops[ta->ops_index].to_binary(attr, data, len, &offs); + if (res != TEE_SUCCESS) + return res; + } + + *data_len = offs; + if (data && offs > len) + return TEE_ERROR_SHORT_BUFFER; + return TEE_SUCCESS; +} + +TEE_Result tee_obj_attr_from_binary(struct tee_obj *o, const void *data, + size_t data_len) +{ + const struct tee_cryp_obj_type_props *tp; + size_t n; + size_t offs = 0; + + if (o->info.objectType == TEE_TYPE_DATA) + return TEE_SUCCESS; /* pure data object */ + if (!o->attr) + return TEE_ERROR_BAD_STATE; + tp = tee_svc_find_type_props(o->info.objectType); + if (!tp) + return TEE_ERROR_BAD_STATE; + + for (n = 0; n < tp->num_type_attrs; n++) { + const struct tee_cryp_obj_type_attrs *ta = tp->type_attrs + n; + void *attr = (uint8_t *)o->attr + ta->raw_offs; + + if (!attr_ops[ta->ops_index].from_binary(attr, data, data_len, + &offs)) + return TEE_ERROR_CORRUPT_OBJECT; + } + return TEE_SUCCESS; +} + +TEE_Result tee_obj_attr_copy_from(struct tee_obj *o, const struct tee_obj *src) +{ + TEE_Result res; + const struct tee_cryp_obj_type_props *tp; + const struct tee_cryp_obj_type_attrs *ta; + size_t n; + uint32_t have_attrs = 0; + void *attr; + void *src_attr; + + if (o->info.objectType == TEE_TYPE_DATA) + return TEE_SUCCESS; /* pure data object */ + if (!o->attr) + return TEE_ERROR_BAD_STATE; + tp = tee_svc_find_type_props(o->info.objectType); + if (!tp) + return TEE_ERROR_BAD_STATE; + + if (o->info.objectType == src->info.objectType) { + have_attrs = src->have_attrs; + for (n = 0; n < tp->num_type_attrs; n++) { + ta = tp->type_attrs + n; + attr = (uint8_t *)o->attr + ta->raw_offs; + src_attr = (uint8_t *)src->attr + ta->raw_offs; + res = attr_ops[ta->ops_index].from_obj(attr, src_attr); + if (res != TEE_SUCCESS) + return res; + } + } else { + const struct tee_cryp_obj_type_props *tp_src; + int idx; + + if (o->info.objectType == TEE_TYPE_RSA_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_RSA_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_DSA_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_DSA_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_ECDSA_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_ECDSA_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_ECDH_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_ECDH_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_SM2_DSA_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_SM2_DSA_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_SM2_PKE_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_SM2_PKE_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_SM2_KEP_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_SM2_KEP_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_ED25519_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_ED25519_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_X25519_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_X25519_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else if (o->info.objectType == TEE_TYPE_X448_PUBLIC_KEY) { + if (src->info.objectType != TEE_TYPE_X448_KEYPAIR) + return TEE_ERROR_BAD_PARAMETERS; + } else { + return TEE_ERROR_BAD_PARAMETERS; + } + + tp_src = tee_svc_find_type_props(src->info.objectType); + if (!tp_src) + return TEE_ERROR_BAD_STATE; + + have_attrs = BIT32(tp->num_type_attrs) - 1; + for (n = 0; n < tp->num_type_attrs; n++) { + ta = tp->type_attrs + n; + + idx = tee_svc_cryp_obj_find_type_attr_idx(ta->attr_id, + tp_src); + if (idx < 0) + return TEE_ERROR_BAD_STATE; + + attr = (uint8_t *)o->attr + ta->raw_offs; + src_attr = (uint8_t *)src->attr + + tp_src->type_attrs[idx].raw_offs; + res = attr_ops[ta->ops_index].from_obj(attr, src_attr); + if (res != TEE_SUCCESS) + return res; + } + } + + o->have_attrs = have_attrs; + return TEE_SUCCESS; +} + +static bool is_gp_legacy_des_key_size(TEE_ObjectType type, size_t sz) +{ + return IS_ENABLED(CFG_COMPAT_GP10_DES) && + ((type == TEE_TYPE_DES && sz == 56) || + (type == TEE_TYPE_DES3 && (sz == 112 || sz == 168))); +} + +static TEE_Result check_key_size(const struct tee_cryp_obj_type_props *props, + size_t key_size) +{ + size_t sz = key_size; + + /* + * In GP Internal API Specification 1.0 the partity bits aren't + * counted when telling the size of the key in bits so add them + * here if missing. + */ + if (is_gp_legacy_des_key_size(props->obj_type, sz)) + sz += sz / 7; + + if (sz % props->quanta != 0) + return TEE_ERROR_NOT_SUPPORTED; + if (sz < props->min_size) + return TEE_ERROR_NOT_SUPPORTED; + if (sz > props->max_size) + return TEE_ERROR_NOT_SUPPORTED; + + return TEE_SUCCESS; +} + +TEE_Result tee_obj_set_type(struct tee_obj *o, uint32_t obj_type, + size_t max_key_size) +{ + TEE_Result res = TEE_SUCCESS; + const struct tee_cryp_obj_type_props *type_props; + + /* Can only set type for newly allocated objs */ + if (o->attr) + return TEE_ERROR_BAD_STATE; + + /* + * Verify that maxObjectSize is supported and find out how + * much should be allocated. + */ + + if (obj_type == TEE_TYPE_DATA) { + if (max_key_size) + return TEE_ERROR_NOT_SUPPORTED; + } else { + /* Find description of object */ + type_props = tee_svc_find_type_props(obj_type); + if (!type_props) + return TEE_ERROR_NOT_SUPPORTED; + + /* Check that max_key_size follows restrictions */ + res = check_key_size(type_props, max_key_size); + if (res) + return res; + + o->attr = calloc(1, type_props->alloc_size); + if (!o->attr) + return TEE_ERROR_OUT_OF_MEMORY; + } + + /* If we have a key structure, pre-allocate the bignums inside */ + switch (obj_type) { + case TEE_TYPE_RSA_PUBLIC_KEY: + res = crypto_acipher_alloc_rsa_public_key(o->attr, + max_key_size); + break; + case TEE_TYPE_RSA_KEYPAIR: + res = crypto_acipher_alloc_rsa_keypair(o->attr, max_key_size); + break; + case TEE_TYPE_DSA_PUBLIC_KEY: + res = crypto_acipher_alloc_dsa_public_key(o->attr, + max_key_size); + break; + case TEE_TYPE_DSA_KEYPAIR: + res = crypto_acipher_alloc_dsa_keypair(o->attr, max_key_size); + break; + case TEE_TYPE_DH_KEYPAIR: + res = crypto_acipher_alloc_dh_keypair(o->attr, max_key_size); + break; + case TEE_TYPE_ECDSA_PUBLIC_KEY: + case TEE_TYPE_ECDH_PUBLIC_KEY: + case TEE_TYPE_SM2_DSA_PUBLIC_KEY: + case TEE_TYPE_SM2_PKE_PUBLIC_KEY: + case TEE_TYPE_SM2_KEP_PUBLIC_KEY: + res = crypto_acipher_alloc_ecc_public_key(o->attr, obj_type, + max_key_size); + break; + case TEE_TYPE_ECDSA_KEYPAIR: + case TEE_TYPE_ECDH_KEYPAIR: + case TEE_TYPE_SM2_DSA_KEYPAIR: + case TEE_TYPE_SM2_PKE_KEYPAIR: + case TEE_TYPE_SM2_KEP_KEYPAIR: + res = crypto_acipher_alloc_ecc_keypair(o->attr, obj_type, + max_key_size); + break; + case TEE_TYPE_X25519_KEYPAIR: + res = crypto_acipher_alloc_x25519_keypair(o->attr, + max_key_size); + break; + case TEE_TYPE_X448_KEYPAIR: + res = crypto_acipher_alloc_x448_keypair(o->attr, + max_key_size); + break; + case TEE_TYPE_ED25519_KEYPAIR: + res = crypto_acipher_alloc_ed25519_keypair(o->attr, + max_key_size); + break; + case TEE_TYPE_ED25519_PUBLIC_KEY: + res = crypto_acipher_alloc_ed25519_public_key(o->attr, + max_key_size); + break; + default: + if (obj_type != TEE_TYPE_DATA) { + struct tee_cryp_obj_secret *key = o->attr; + + key->alloc_size = type_props->alloc_size - + sizeof(*key); + } + break; + } + + if (res != TEE_SUCCESS) + return res; + + o->info.objectType = obj_type; + o->info.maxObjectSize = max_key_size; + o->info.objectUsage = TEE_USAGE_DEFAULT; + + return TEE_SUCCESS; +} + +TEE_Result syscall_cryp_obj_alloc(unsigned long obj_type, + unsigned long max_key_size, uint32_t *obj) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + + + o = tee_obj_alloc(); + if (!o) + return TEE_ERROR_OUT_OF_MEMORY; + + res = tee_obj_set_type(o, obj_type, max_key_size); + if (res != TEE_SUCCESS) { + tee_obj_free(o); + return res; + } + + tee_obj_add(to_user_ta_ctx(sess->ctx), o); + + res = copy_kaddr_to_uref(obj, o); + if (res != TEE_SUCCESS) + tee_obj_close(to_user_ta_ctx(sess->ctx), o); + return res; +} + +TEE_Result syscall_cryp_obj_close(unsigned long obj) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return res; + + /* + * If it's busy it's used by an operation, a client should never have + * this handle. + */ + if (o->busy) + return TEE_ERROR_ITEM_NOT_FOUND; + + tee_obj_close(to_user_ta_ctx(sess->ctx), o); + return TEE_SUCCESS; +} + +TEE_Result syscall_cryp_obj_reset(unsigned long obj) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return res; + + if ((o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT) == 0) { + tee_obj_attr_clear(o); + o->info.objectSize = 0; + o->info.objectUsage = TEE_USAGE_DEFAULT; + } else { + return TEE_ERROR_BAD_PARAMETERS; + } + + /* the object is no more initialized */ + o->info.handleFlags &= ~TEE_HANDLE_FLAG_INITIALIZED; + + return TEE_SUCCESS; +} + +static TEE_Result copy_in_attrs(struct user_ta_ctx *utc, + const struct utee_attribute *usr_attrs, + uint32_t attr_count, TEE_Attribute *attrs) +{ + TEE_Result res = TEE_SUCCESS; + size_t size = 0; + uint32_t n = 0; + struct utee_attribute *usr_attrs_bbuf = NULL; + + if (MUL_OVERFLOW(sizeof(struct utee_attribute), attr_count, &size)) + return TEE_ERROR_OVERFLOW; + + usr_attrs_bbuf = bb_alloc(size); + if (!usr_attrs_bbuf) + return TEE_ERROR_OUT_OF_MEMORY; + + res = copy_from_user(usr_attrs_bbuf, usr_attrs, size); + if (res) + goto out; + + for (n = 0; n < attr_count; n++) { + attrs[n].attributeID = usr_attrs_bbuf[n].attribute_id; + if (attrs[n].attributeID & TEE_ATTR_FLAG_VALUE) { + attrs[n].content.value.a = usr_attrs_bbuf[n].a; + attrs[n].content.value.b = usr_attrs_bbuf[n].b; + } else { + uintptr_t buf = usr_attrs_bbuf[n].a; + size_t len = usr_attrs_bbuf[n].b; + uint32_t flags = TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER; + + buf = memtag_strip_tag_vaddr((void *)buf); + + res = vm_check_access_rights(&utc->uctx, flags, buf, + len); + if (res != TEE_SUCCESS) + goto out; + attrs[n].content.ref.buffer = (void *)buf; + attrs[n].content.ref.length = len; + } + } + +out: + bb_free(usr_attrs_bbuf, size); + return res; +} + +enum attr_usage { + ATTR_USAGE_POPULATE, + ATTR_USAGE_GENERATE_KEY +}; + +static TEE_Result tee_svc_cryp_check_attr(enum attr_usage usage, + const struct tee_cryp_obj_type_props + *type_props, + const TEE_Attribute *attrs, + uint32_t attr_count) +{ + uint32_t required_flag = 0; + uint32_t opt_flag = 0; + bool all_opt_needed = false; + uint32_t req_attrs = 0; + uint32_t opt_grp_attrs = 0; + uint32_t attrs_found = 0; + size_t n = 0; + uint32_t bit = 0; + uint32_t flags = 0; + int idx = 0; + + if (usage == ATTR_USAGE_POPULATE) { + required_flag = TEE_TYPE_ATTR_REQUIRED; + opt_flag = TEE_TYPE_ATTR_OPTIONAL_GROUP; + all_opt_needed = true; + } else { + required_flag = TEE_TYPE_ATTR_GEN_KEY_REQ; + opt_flag = TEE_TYPE_ATTR_GEN_KEY_OPT; + all_opt_needed = false; + } + + /* + * First find out which attributes are required and which belong to + * the optional group + */ + for (n = 0; n < type_props->num_type_attrs; n++) { + bit = 1 << n; + flags = type_props->type_attrs[n].flags; + + if (flags & required_flag) + req_attrs |= bit; + else if (flags & opt_flag) + opt_grp_attrs |= bit; + } + + /* + * Verify that all required attributes are in place and + * that the same attribute isn't repeated. + */ + for (n = 0; n < attr_count; n++) { + idx = tee_svc_cryp_obj_find_type_attr_idx( + attrs[n].attributeID, + type_props); + + /* attribute not defined in current object type */ + if (idx < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + bit = 1 << idx; + + /* attribute not repeated */ + if ((attrs_found & bit) != 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* + * Attribute not defined in current object type for this + * usage. + */ + if (!(bit & (req_attrs | opt_grp_attrs))) + return TEE_ERROR_ITEM_NOT_FOUND; + + attrs_found |= bit; + } + /* Required attribute missing */ + if ((attrs_found & req_attrs) != req_attrs) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* + * If the flag says that "if one of the optional attributes are included + * all of them has to be included" this must be checked. + */ + if (all_opt_needed && (attrs_found & opt_grp_attrs) != 0 && + (attrs_found & opt_grp_attrs) != opt_grp_attrs) + return TEE_ERROR_ITEM_NOT_FOUND; + + return TEE_SUCCESS; +} + +static TEE_Result get_ec_key_size(uint32_t curve, size_t *key_size) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + *key_size = 192; + break; + case TEE_ECC_CURVE_NIST_P224: + *key_size = 224; + break; + case TEE_ECC_CURVE_NIST_P256: + *key_size = 256; + break; + case TEE_ECC_CURVE_NIST_P384: + *key_size = 384; + break; + case TEE_ECC_CURVE_NIST_P521: + *key_size = 521; + break; + case TEE_ECC_CURVE_SM2: + case TEE_ECC_CURVE_25519: + *key_size = 256; + break; + default: + return TEE_ERROR_NOT_SUPPORTED; + } + + return TEE_SUCCESS; +} + +static TEE_Result tee_svc_cryp_obj_populate_type( + struct tee_obj *o, + const struct tee_cryp_obj_type_props *type_props, + const TEE_Attribute *attrs, + uint32_t attr_count) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t have_attrs = 0; + size_t obj_size = 0; + size_t n = 0; + int idx = 0; + const struct attr_ops *ops = NULL; + void *attr = NULL; + + for (n = 0; n < attr_count; n++) { + idx = tee_svc_cryp_obj_find_type_attr_idx( + attrs[n].attributeID, + type_props); + /* attribute not defined in current object type */ + if (idx < 0) + return TEE_ERROR_ITEM_NOT_FOUND; + + have_attrs |= BIT32(idx); + ops = attr_ops + type_props->type_attrs[idx].ops_index; + attr = (uint8_t *)o->attr + + type_props->type_attrs[idx].raw_offs; + if (attrs[n].attributeID & TEE_ATTR_FLAG_VALUE) + res = ops->from_user(attr, &attrs[n].content.value, + sizeof(attrs[n].content.value)); + else + res = ops->from_user(attr, attrs[n].content.ref.buffer, + attrs[n].content.ref.length); + if (res != TEE_SUCCESS) + return res; + + /* + * The attribute that gives the size of the object is + * flagged with TEE_TYPE_ATTR_SIZE_INDICATOR. + */ + if (type_props->type_attrs[idx].flags & + TEE_TYPE_ATTR_SIZE_INDICATOR) { + /* There should be only one */ + if (obj_size) + return TEE_ERROR_BAD_STATE; + + /* + * For ECDSA/ECDH we need to translate curve into + * object size + */ + if (attrs[n].attributeID == TEE_ATTR_ECC_CURVE) { + res = get_ec_key_size(attrs[n].content.value.a, + &obj_size); + if (res != TEE_SUCCESS) + return res; + } else { + TEE_ObjectType obj_type = o->info.objectType; + size_t sz = o->info.maxObjectSize; + + obj_size = attrs[n].content.ref.length * 8; + /* Drop the parity bits for legacy objects */ + if (is_gp_legacy_des_key_size(obj_type, sz)) + obj_size -= obj_size / 8; + } + if (obj_size > o->info.maxObjectSize) + return TEE_ERROR_BAD_STATE; + res = check_key_size(type_props, obj_size); + if (res != TEE_SUCCESS) + return TEE_ERROR_BAD_PARAMETERS; + } + + /* + * Bignum attributes limited by the number of bits in + * o->info.objectSize are flagged with + * TEE_TYPE_ATTR_BIGNUM_MAXBITS. + */ + if (type_props->type_attrs[idx].flags & + TEE_TYPE_ATTR_BIGNUM_MAXBITS) { + if (crypto_bignum_num_bits(*(struct bignum **)attr) > + o->info.maxObjectSize) + return TEE_ERROR_BAD_STATE; + } + } + + o->have_attrs = have_attrs; + o->info.objectSize = obj_size; + /* + * In GP Internal API Specification 1.0 the partity bits aren't + * counted when telling the size of the key in bits so remove the + * parity bits here. + */ + if (is_gp_legacy_des_key_size(o->info.objectType, + o->info.maxObjectSize)) + o->info.objectSize -= o->info.objectSize / 8; + + return TEE_SUCCESS; +} + +TEE_Result syscall_cryp_obj_populate(unsigned long obj, + struct utee_attribute *usr_attrs, + unsigned long attr_count) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + const struct tee_cryp_obj_type_props *type_props = NULL; + TEE_Attribute *attrs = NULL; + size_t alloc_size = 0; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return res; + + /* Must be a transient object */ + if ((o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT) != 0) + return TEE_ERROR_BAD_PARAMETERS; + + /* Must not be initialized already */ + if ((o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) != 0) + return TEE_ERROR_BAD_PARAMETERS; + + type_props = tee_svc_find_type_props(o->info.objectType); + if (!type_props) + return TEE_ERROR_NOT_IMPLEMENTED; + + if (MUL_OVERFLOW(sizeof(TEE_Attribute), attr_count, &alloc_size)) + return TEE_ERROR_OVERFLOW; + + attrs = malloc(alloc_size); + if (!attrs) + return TEE_ERROR_OUT_OF_MEMORY; + + res = copy_in_attrs(to_user_ta_ctx(sess->ctx), usr_attrs, attr_count, + attrs); + if (res != TEE_SUCCESS) + goto out; + + res = tee_svc_cryp_check_attr(ATTR_USAGE_POPULATE, type_props, + attrs, attr_count); + if (res != TEE_SUCCESS) + goto out; + + res = tee_svc_cryp_obj_populate_type(o, type_props, attrs, attr_count); + if (res == TEE_SUCCESS) + o->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + +out: + free_wipe(attrs); + return res; +} + +TEE_Result syscall_cryp_obj_copy(unsigned long dst, unsigned long src) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *dst_o = NULL; + struct tee_obj *src_o = NULL; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), + uref_to_vaddr(dst), &dst_o); + if (res != TEE_SUCCESS) + return res; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), + uref_to_vaddr(src), &src_o); + if (res != TEE_SUCCESS) + return res; + + if ((src_o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) == 0) + return TEE_ERROR_BAD_PARAMETERS; + if ((dst_o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT) != 0) + return TEE_ERROR_BAD_PARAMETERS; + if ((dst_o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) != 0) + return TEE_ERROR_BAD_PARAMETERS; + + res = tee_obj_attr_copy_from(dst_o, src_o); + if (res != TEE_SUCCESS) + return res; + + dst_o->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + dst_o->info.objectSize = src_o->info.objectSize; + dst_o->info.objectUsage = src_o->info.objectUsage; + return TEE_SUCCESS; +} + +static TEE_Result check_pub_rsa_key(struct bignum *e) +{ + size_t n = crypto_bignum_num_bytes(e); + uint8_t bin_key[256 / 8] = { 0 }; + + /* + * NIST SP800-56B requires public RSA key to be an odd integer in + * the range 65537 <= e < 2^256. + */ + + if (n > sizeof(bin_key) || n < 3) + return TEE_ERROR_BAD_PARAMETERS; + + crypto_bignum_bn2bin(e, bin_key); + + if (!(bin_key[n - 1] & 1)) /* key must be odd */ + return TEE_ERROR_BAD_PARAMETERS; + + if (n == 3) { + uint32_t key = 0; + + for (n = 0; n < 3; n++) { + key <<= 8; + key |= bin_key[n]; + } + + if (key < 65537) + return TEE_ERROR_BAD_PARAMETERS; + } + + /* key is larger than 65537 */ + return TEE_SUCCESS; +} + +static TEE_Result tee_svc_obj_generate_key_rsa( + struct tee_obj *o, const struct tee_cryp_obj_type_props *type_props, + uint32_t key_size, + const TEE_Attribute *params, uint32_t param_count) +{ + TEE_Result res = TEE_SUCCESS; + struct rsa_keypair *key = o->attr; + uint32_t e = TEE_U32_TO_BIG_ENDIAN(65537); + + /* Copy the present attributes into the obj before starting */ + res = tee_svc_cryp_obj_populate_type(o, type_props, params, + param_count); + if (res != TEE_SUCCESS) + return res; + if (get_attribute(o, type_props, TEE_ATTR_RSA_PUBLIC_EXPONENT)) { + res = check_pub_rsa_key(key->e); + if (res) + return res; + } else { + crypto_bignum_bin2bn((const uint8_t *)&e, sizeof(e), key->e); + } + res = crypto_acipher_gen_rsa_key(key, key_size); + if (res != TEE_SUCCESS) + return res; + + /* Set bits for all known attributes for this object type */ + o->have_attrs = (1 << type_props->num_type_attrs) - 1; + + return TEE_SUCCESS; +} + +static TEE_Result tee_svc_obj_generate_key_dsa( + struct tee_obj *o, const struct tee_cryp_obj_type_props *type_props, + uint32_t key_size, const TEE_Attribute *params, uint32_t param_count) +{ + TEE_Result res; + + /* Copy the present attributes into the obj before starting */ + res = tee_svc_cryp_obj_populate_type(o, type_props, params, + param_count); + if (res != TEE_SUCCESS) + return res; + + res = crypto_acipher_gen_dsa_key(o->attr, key_size); + if (res != TEE_SUCCESS) + return res; + + /* Set bits for all known attributes for this object type */ + o->have_attrs = (1 << type_props->num_type_attrs) - 1; + + return TEE_SUCCESS; +} + +static TEE_Result tee_svc_obj_generate_key_dh( + struct tee_obj *o, const struct tee_cryp_obj_type_props *type_props, + uint32_t key_size, const TEE_Attribute *params, uint32_t param_count) +{ + TEE_Result res; + struct dh_keypair *tee_dh_key; + struct bignum *dh_q = NULL; + uint32_t dh_xbits = 0; + + /* Copy the present attributes into the obj before starting */ + res = tee_svc_cryp_obj_populate_type(o, type_props, params, + param_count); + if (res != TEE_SUCCESS) + return res; + + tee_dh_key = (struct dh_keypair *)o->attr; + + if (get_attribute(o, type_props, TEE_ATTR_DH_SUBPRIME)) + dh_q = tee_dh_key->q; + if (get_attribute(o, type_props, TEE_ATTR_DH_X_BITS)) + dh_xbits = tee_dh_key->xbits; + res = crypto_acipher_gen_dh_key(tee_dh_key, dh_q, dh_xbits, key_size); + if (res != TEE_SUCCESS) + return res; + + /* Set bits for the generated public and private key */ + set_attribute(o, type_props, TEE_ATTR_DH_PUBLIC_VALUE); + set_attribute(o, type_props, TEE_ATTR_DH_PRIVATE_VALUE); + set_attribute(o, type_props, TEE_ATTR_DH_X_BITS); + return TEE_SUCCESS; +} + +static TEE_Result tee_svc_obj_generate_key_ecc( + struct tee_obj *o, const struct tee_cryp_obj_type_props *type_props, + uint32_t key_size, const TEE_Attribute *params, uint32_t param_count) +{ + TEE_Result res; + struct ecc_keypair *tee_ecc_key; + + /* Copy the present attributes into the obj before starting */ + res = tee_svc_cryp_obj_populate_type(o, type_props, params, + param_count); + if (res != TEE_SUCCESS) + return res; + + tee_ecc_key = (struct ecc_keypair *)o->attr; + + res = crypto_acipher_gen_ecc_key(tee_ecc_key, key_size); + if (res != TEE_SUCCESS) + return res; + + /* Set bits for the generated public and private key */ + set_attribute(o, type_props, TEE_ATTR_ECC_PRIVATE_VALUE); + set_attribute(o, type_props, TEE_ATTR_ECC_PUBLIC_VALUE_X); + set_attribute(o, type_props, TEE_ATTR_ECC_PUBLIC_VALUE_Y); + set_attribute(o, type_props, TEE_ATTR_ECC_CURVE); + return TEE_SUCCESS; +} + +static TEE_Result +tee_svc_obj_generate_key_x25519(struct tee_obj *o, + const struct tee_cryp_obj_type_props + *type_props, + uint32_t key_size, + const TEE_Attribute *params, + uint32_t param_count) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct montgomery_keypair *tee_x25519_key = NULL; + + /* Copy the present attributes into the obj before starting */ + res = tee_svc_cryp_obj_populate_type(o, type_props, params, + param_count); + if (res != TEE_SUCCESS) + return res; + + tee_x25519_key = (struct montgomery_keypair *)o->attr; + + res = crypto_acipher_gen_x25519_key(tee_x25519_key, key_size); + if (res != TEE_SUCCESS) + return res; + + /* Set bits for the generated public and private key */ + set_attribute(o, type_props, TEE_ATTR_X25519_PRIVATE_VALUE); + set_attribute(o, type_props, TEE_ATTR_X25519_PUBLIC_VALUE); + return TEE_SUCCESS; +} + +static TEE_Result +tee_svc_obj_generate_key_x448(struct tee_obj *o, + const struct tee_cryp_obj_type_props *type_props, + uint32_t key_size, const TEE_Attribute *params, + uint32_t param_count) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct montgomery_keypair *tee_x448_key = NULL; + + /* Copy the present attributes into the obj before starting */ + res = tee_svc_cryp_obj_populate_type(o, type_props, params, + param_count); + if (res != TEE_SUCCESS) + return res; + + tee_x448_key = (struct montgomery_keypair *)o->attr; + res = crypto_acipher_gen_x448_key(tee_x448_key, key_size); + if (res != TEE_SUCCESS) + return res; + + set_attribute(o, type_props, TEE_ATTR_X448_PRIVATE_VALUE); + set_attribute(o, type_props, TEE_ATTR_X448_PUBLIC_VALUE); + + return TEE_SUCCESS; +} + +static TEE_Result +tee_svc_obj_generate_key_ed25519(struct tee_obj *o, + const struct tee_cryp_obj_type_props + *type_props, + uint32_t key_size, + const TEE_Attribute *params, + uint32_t param_count) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct ed25519_keypair *key = NULL; + + /* Copy the present attributes into the obj before starting */ + res = tee_svc_cryp_obj_populate_type(o, type_props, params, + param_count); + if (res != TEE_SUCCESS) + return res; + + key = o->attr; + + res = crypto_acipher_gen_ed25519_key(key, key_size); + if (res != TEE_SUCCESS) + return res; + + /* Set bits for the generated public and private key */ + set_attribute(o, type_props, TEE_ATTR_ED25519_PRIVATE_VALUE); + set_attribute(o, type_props, TEE_ATTR_ED25519_PUBLIC_VALUE); + return TEE_SUCCESS; +} + +static TEE_Result +tee_svc_obj_ed25519_parse_params(const TEE_Attribute *params, size_t num_params, + bool *ph_flag, const uint8_t **ctx, + size_t *ctx_len) +{ + size_t n = 0; + + *ctx = NULL; + + for (n = 0; n < num_params; n++) { + switch (params[n].attributeID) { + case TEE_ATTR_EDDSA_PREHASH: + if (params[n].content.value.b) + return TEE_ERROR_BAD_PARAMETERS; + if (!params[n].content.value.a) + *ph_flag = false; + else if (params[n].content.value.a == 1) + *ph_flag = true; + else + return TEE_ERROR_BAD_PARAMETERS; + break; + + case TEE_ATTR_EDDSA_CTX: + /* several provided contexts are treated as error */ + if (*ctx) + return TEE_ERROR_BAD_PARAMETERS; + + *ctx_len = params[n].content.ref.length; + if (*ctx_len > TEE_ED25519_CTX_MAX_LENGTH) + return TEE_ERROR_BAD_PARAMETERS; + + if (!*ctx_len) + break; + + *ctx = params[n].content.ref.buffer; + if (!*ctx) + return TEE_ERROR_BAD_PARAMETERS; + break; + + default: + return TEE_ERROR_BAD_PARAMETERS; + } + } + + return TEE_SUCCESS; +} + +static TEE_Result +tee_svc_obj_ed25519_sign(struct ed25519_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len, + const TEE_Attribute *params, size_t num_params) +{ + TEE_Result err = TEE_ERROR_GENERIC; + size_t ctx_len = 0; + const uint8_t *ctx = NULL; + bool ph_flag = false; + + err = tee_svc_obj_ed25519_parse_params(params, num_params, &ph_flag, + &ctx, &ctx_len); + if (err != TEE_SUCCESS) + return err; + + if (ph_flag || ctx) { + return crypto_acipher_ed25519ctx_sign(key, msg, msg_len, sig, + sig_len, ph_flag, + ctx, ctx_len); + } + + return crypto_acipher_ed25519_sign(key, msg, msg_len, sig, sig_len); +} + +static TEE_Result +tee_svc_obj_ed25519_verify(struct ed25519_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len, + const TEE_Attribute *params, size_t num_params) +{ + TEE_Result err = TEE_ERROR_GENERIC; + size_t ctx_len = 0; + const uint8_t *ctx = NULL; + bool ph_flag = false; + + err = tee_svc_obj_ed25519_parse_params(params, num_params, &ph_flag, + &ctx, &ctx_len); + if (err) + return err; + + if (ph_flag || ctx) { + return crypto_acipher_ed25519ctx_verify(key, msg, msg_len, sig, + sig_len, ph_flag, + ctx, ctx_len); + } + + return crypto_acipher_ed25519_verify(key, msg, msg_len, sig, sig_len); +} + +TEE_Result syscall_obj_generate_key(unsigned long obj, unsigned long key_size, + const struct utee_attribute *usr_params, + unsigned long param_count) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + const struct tee_cryp_obj_type_props *type_props = NULL; + struct tee_obj *o = NULL; + struct tee_cryp_obj_secret *key = NULL; + size_t byte_size = 0; + TEE_Attribute *params = NULL; + size_t alloc_size = 0; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return res; + + /* Must be a transient object */ + if ((o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT) != 0) + return TEE_ERROR_BAD_STATE; + + /* Must not be initialized already */ + if ((o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) != 0) + return TEE_ERROR_BAD_STATE; + + /* Find description of object */ + type_props = tee_svc_find_type_props(o->info.objectType); + if (!type_props) + return TEE_ERROR_NOT_SUPPORTED; + + /* Check that key_size follows restrictions */ + res = check_key_size(type_props, key_size); + if (res) + return res; + + if (MUL_OVERFLOW(sizeof(TEE_Attribute), param_count, &alloc_size)) + return TEE_ERROR_OVERFLOW; + + params = malloc(alloc_size); + if (!params) + return TEE_ERROR_OUT_OF_MEMORY; + res = copy_in_attrs(to_user_ta_ctx(sess->ctx), usr_params, param_count, + params); + if (res != TEE_SUCCESS) + goto out; + + res = tee_svc_cryp_check_attr(ATTR_USAGE_GENERATE_KEY, type_props, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + + switch (o->info.objectType) { + case TEE_TYPE_AES: + case TEE_TYPE_DES: + case TEE_TYPE_DES3: + case TEE_TYPE_SM4: + case TEE_TYPE_HMAC_MD5: + case TEE_TYPE_HMAC_SHA1: + case TEE_TYPE_HMAC_SHA224: + case TEE_TYPE_HMAC_SHA256: + case TEE_TYPE_HMAC_SHA384: + case TEE_TYPE_HMAC_SHA512: + case TEE_TYPE_HMAC_SHA3_224: + case TEE_TYPE_HMAC_SHA3_256: + case TEE_TYPE_HMAC_SHA3_384: + case TEE_TYPE_HMAC_SHA3_512: + case TEE_TYPE_HMAC_SM3: + case TEE_TYPE_GENERIC_SECRET: + byte_size = key_size / 8; + + /* + * In GP Internal API Specification 1.0 the partity bits + * aren't counted when telling the size of the key in bits. + */ + if (is_gp_legacy_des_key_size(o->info.objectType, key_size)) + byte_size = (key_size + key_size / 7) / 8; + + key = (struct tee_cryp_obj_secret *)o->attr; + if (byte_size > key->alloc_size) { + res = TEE_ERROR_EXCESS_DATA; + goto out; + } + + res = crypto_rng_read((void *)(key + 1), byte_size); + if (res != TEE_SUCCESS) + goto out; + + key->key_size = byte_size; + + /* Set bits for all known attributes for this object type */ + o->have_attrs = (1 << type_props->num_type_attrs) - 1; + + break; + + case TEE_TYPE_RSA_KEYPAIR: + res = tee_svc_obj_generate_key_rsa(o, type_props, key_size, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + break; + + case TEE_TYPE_DSA_KEYPAIR: + res = tee_svc_obj_generate_key_dsa(o, type_props, key_size, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + break; + + case TEE_TYPE_DH_KEYPAIR: + res = tee_svc_obj_generate_key_dh(o, type_props, key_size, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + break; + + case TEE_TYPE_ECDSA_KEYPAIR: + case TEE_TYPE_ECDH_KEYPAIR: + case TEE_TYPE_SM2_DSA_KEYPAIR: + case TEE_TYPE_SM2_KEP_KEYPAIR: + case TEE_TYPE_SM2_PKE_KEYPAIR: + res = tee_svc_obj_generate_key_ecc(o, type_props, key_size, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + break; + + case TEE_TYPE_X25519_KEYPAIR: + res = tee_svc_obj_generate_key_x25519(o, type_props, key_size, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + break; + case TEE_TYPE_X448_KEYPAIR: + res = tee_svc_obj_generate_key_x448(o, type_props, key_size, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + break; + + case TEE_TYPE_ED25519_KEYPAIR: + res = tee_svc_obj_generate_key_ed25519(o, type_props, key_size, + params, param_count); + if (res != TEE_SUCCESS) + goto out; + break; + + default: + res = TEE_ERROR_BAD_FORMAT; + } + +out: + free_wipe(params); + if (res == TEE_SUCCESS) { + o->info.objectSize = key_size; + o->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + } + return res; +} + +static TEE_Result tee_svc_cryp_get_state(struct ts_session *sess, + vaddr_t state_id, + struct tee_cryp_state **state) +{ + struct tee_cryp_state *s; + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + + TAILQ_FOREACH(s, &utc->cryp_states, link) { + if (state_id == (vaddr_t)s) { + *state = s; + return TEE_SUCCESS; + } + } + return TEE_ERROR_BAD_PARAMETERS; +} + +static void cryp_state_free(struct user_ta_ctx *utc, struct tee_cryp_state *cs) +{ + struct tee_obj *o; + + if (tee_obj_get(utc, cs->key1, &o) == TEE_SUCCESS) + tee_obj_close(utc, o); + if (tee_obj_get(utc, cs->key2, &o) == TEE_SUCCESS) + tee_obj_close(utc, o); + + TAILQ_REMOVE(&utc->cryp_states, cs, link); + if (cs->ctx_finalize != NULL) + cs->ctx_finalize(cs->ctx); + + switch (TEE_ALG_GET_CLASS(cs->algo)) { + case TEE_OPERATION_CIPHER: + crypto_cipher_free_ctx(cs->ctx); + break; + case TEE_OPERATION_AE: + crypto_authenc_free_ctx(cs->ctx); + break; + case TEE_OPERATION_DIGEST: + crypto_hash_free_ctx(cs->ctx); + break; + case TEE_OPERATION_MAC: + crypto_mac_free_ctx(cs->ctx); + break; + default: + assert(!cs->ctx); + } + + free(cs); +} + +static TEE_Result tee_svc_cryp_check_key_type(const struct tee_obj *o, + uint32_t algo, + TEE_OperationMode mode) +{ + uint32_t req_key_type; + uint32_t req_key_type2 = 0; + + switch (TEE_ALG_GET_MAIN_ALG(algo)) { + case TEE_MAIN_ALGO_MD5: + req_key_type = TEE_TYPE_HMAC_MD5; + break; + case TEE_MAIN_ALGO_SHA1: + req_key_type = TEE_TYPE_HMAC_SHA1; + break; + case TEE_MAIN_ALGO_SHA224: + req_key_type = TEE_TYPE_HMAC_SHA224; + break; + case TEE_MAIN_ALGO_SHA256: + req_key_type = TEE_TYPE_HMAC_SHA256; + break; + case TEE_MAIN_ALGO_SHA384: + req_key_type = TEE_TYPE_HMAC_SHA384; + break; + case TEE_MAIN_ALGO_SHA512: + req_key_type = TEE_TYPE_HMAC_SHA512; + break; + case TEE_MAIN_ALGO_SHA3_224: + req_key_type = TEE_TYPE_HMAC_SHA3_224; + break; + case TEE_MAIN_ALGO_SHA3_256: + req_key_type = TEE_TYPE_HMAC_SHA3_256; + break; + case TEE_MAIN_ALGO_SHA3_384: + req_key_type = TEE_TYPE_HMAC_SHA3_384; + break; + case TEE_MAIN_ALGO_SHA3_512: + req_key_type = TEE_TYPE_HMAC_SHA3_512; + break; + case TEE_MAIN_ALGO_SM3: + req_key_type = TEE_TYPE_HMAC_SM3; + break; + case TEE_MAIN_ALGO_AES: + req_key_type = TEE_TYPE_AES; + break; + case TEE_MAIN_ALGO_DES: + req_key_type = TEE_TYPE_DES; + break; + case TEE_MAIN_ALGO_DES3: + req_key_type = TEE_TYPE_DES3; + break; + case TEE_MAIN_ALGO_SM4: + req_key_type = TEE_TYPE_SM4; + break; + case TEE_MAIN_ALGO_RSA: + req_key_type = TEE_TYPE_RSA_KEYPAIR; + if (mode == TEE_MODE_ENCRYPT || mode == TEE_MODE_VERIFY) + req_key_type2 = TEE_TYPE_RSA_PUBLIC_KEY; + break; + case TEE_MAIN_ALGO_DSA: + req_key_type = TEE_TYPE_DSA_KEYPAIR; + if (mode == TEE_MODE_ENCRYPT || mode == TEE_MODE_VERIFY) + req_key_type2 = TEE_TYPE_DSA_PUBLIC_KEY; + break; + case TEE_MAIN_ALGO_DH: + req_key_type = TEE_TYPE_DH_KEYPAIR; + break; + case TEE_MAIN_ALGO_ECDSA: + req_key_type = TEE_TYPE_ECDSA_KEYPAIR; + if (mode == TEE_MODE_VERIFY) + req_key_type2 = TEE_TYPE_ECDSA_PUBLIC_KEY; + break; + case TEE_MAIN_ALGO_ECDH: + req_key_type = TEE_TYPE_ECDH_KEYPAIR; + break; + case TEE_MAIN_ALGO_ED25519: + req_key_type = TEE_TYPE_ED25519_KEYPAIR; + if (mode == TEE_MODE_VERIFY) + req_key_type2 = TEE_TYPE_ED25519_PUBLIC_KEY; + break; + case TEE_MAIN_ALGO_SM2_PKE: + if (mode == TEE_MODE_ENCRYPT) + req_key_type = TEE_TYPE_SM2_PKE_PUBLIC_KEY; + else + req_key_type = TEE_TYPE_SM2_PKE_KEYPAIR; + break; + case TEE_MAIN_ALGO_SM2_DSA_SM3: + if (mode == TEE_MODE_VERIFY) + req_key_type = TEE_TYPE_SM2_DSA_PUBLIC_KEY; + else + req_key_type = TEE_TYPE_SM2_DSA_KEYPAIR; + break; +#if defined(CFG_CRYPTO_SM2_KEP) + case TEE_MAIN_ALGO_SM2_KEP: + req_key_type = TEE_TYPE_SM2_KEP_KEYPAIR; + req_key_type2 = TEE_TYPE_SM2_KEP_PUBLIC_KEY; + break; +#endif +#if defined(CFG_CRYPTO_HKDF) + case TEE_MAIN_ALGO_HKDF: + req_key_type = TEE_TYPE_HKDF_IKM; + break; +#endif +#if defined(CFG_CRYPTO_CONCAT_KDF) + case TEE_MAIN_ALGO_CONCAT_KDF: + req_key_type = TEE_TYPE_CONCAT_KDF_Z; + break; +#endif +#if defined(CFG_CRYPTO_PBKDF2) + case TEE_MAIN_ALGO_PBKDF2: + req_key_type = TEE_TYPE_PBKDF2_PASSWORD; + break; +#endif + case TEE_MAIN_ALGO_X25519: + req_key_type = TEE_TYPE_X25519_KEYPAIR; + break; + case TEE_MAIN_ALGO_X448: + req_key_type = TEE_TYPE_X448_KEYPAIR; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + if (req_key_type != o->info.objectType && + req_key_type2 != o->info.objectType) + return TEE_ERROR_BAD_PARAMETERS; + return TEE_SUCCESS; +} + +static uint32_t translate_compat_algo(uint32_t algo) +{ + switch (algo) { + case __OPTEE_ALG_ECDSA_P192: + return TEE_ALG_ECDSA_SHA1; + case __OPTEE_ALG_ECDSA_P224: + return TEE_ALG_ECDSA_SHA224; + case __OPTEE_ALG_ECDSA_P256: + return TEE_ALG_ECDSA_SHA256; + case __OPTEE_ALG_ECDSA_P384: + return TEE_ALG_ECDSA_SHA384; + case __OPTEE_ALG_ECDSA_P521: + return TEE_ALG_ECDSA_SHA512; + case __OPTEE_ALG_ECDH_P192: + case __OPTEE_ALG_ECDH_P224: + case __OPTEE_ALG_ECDH_P256: + case __OPTEE_ALG_ECDH_P384: + case __OPTEE_ALG_ECDH_P521: + return TEE_ALG_ECDH_DERIVE_SHARED_SECRET; + default: + return algo; + } +} + +TEE_Result syscall_cryp_state_alloc(unsigned long algo, unsigned long mode, + unsigned long key1, unsigned long key2, + uint32_t *state) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + struct tee_cryp_state *cs = NULL; + struct tee_obj *o1 = NULL; + struct tee_obj *o2 = NULL; + + algo = translate_compat_algo(algo); + + if (key1 != 0) { + res = tee_obj_get(utc, uref_to_vaddr(key1), &o1); + if (res != TEE_SUCCESS) + return res; + if (o1->busy) + return TEE_ERROR_BAD_PARAMETERS; + res = tee_svc_cryp_check_key_type(o1, algo, mode); + if (res != TEE_SUCCESS) + return res; + } + if (key2 != 0) { + res = tee_obj_get(utc, uref_to_vaddr(key2), &o2); + if (res != TEE_SUCCESS) + return res; + if (o2->busy) + return TEE_ERROR_BAD_PARAMETERS; + res = tee_svc_cryp_check_key_type(o2, algo, mode); + if (res != TEE_SUCCESS) + return res; + } + + cs = calloc(1, sizeof(struct tee_cryp_state)); + if (!cs) + return TEE_ERROR_OUT_OF_MEMORY; + TAILQ_INSERT_TAIL(&utc->cryp_states, cs, link); + cs->algo = algo; + cs->mode = mode; + cs->state = CRYP_STATE_UNINITIALIZED; + + switch (TEE_ALG_GET_CLASS(algo)) { + case TEE_OPERATION_CIPHER: + if ((TEE_ALG_GET_CHAIN_MODE(algo) == TEE_CHAIN_MODE_XTS && + (key1 == 0 || key2 == 0)) || + (TEE_ALG_GET_CHAIN_MODE(algo) != TEE_CHAIN_MODE_XTS && + (key1 == 0 || key2 != 0))) { + res = TEE_ERROR_BAD_PARAMETERS; + } else { + res = crypto_cipher_alloc_ctx(&cs->ctx, algo); + if (res != TEE_SUCCESS) + break; + } + break; + case TEE_OPERATION_AE: + if (key1 == 0 || key2 != 0) { + res = TEE_ERROR_BAD_PARAMETERS; + } else { + res = crypto_authenc_alloc_ctx(&cs->ctx, algo); + if (res != TEE_SUCCESS) + break; + } + break; + case TEE_OPERATION_MAC: + if (key1 == 0 || key2 != 0) { + res = TEE_ERROR_BAD_PARAMETERS; + } else { + res = crypto_mac_alloc_ctx(&cs->ctx, algo); + if (res != TEE_SUCCESS) + break; + } + break; + case TEE_OPERATION_DIGEST: + if (key1 != 0 || key2 != 0) { + res = TEE_ERROR_BAD_PARAMETERS; + } else { + res = crypto_hash_alloc_ctx(&cs->ctx, algo); + if (res != TEE_SUCCESS) + break; + } + break; + case TEE_OPERATION_ASYMMETRIC_CIPHER: + case TEE_OPERATION_ASYMMETRIC_SIGNATURE: + if (algo == TEE_ALG_RSASSA_PKCS1_V1_5 && + !IS_ENABLED(CFG_CRYPTO_RSASSA_NA1)) { + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + if (key1 == 0 || key2 != 0) + res = TEE_ERROR_BAD_PARAMETERS; + break; + case TEE_OPERATION_KEY_DERIVATION: + if (algo == TEE_ALG_SM2_KEP) { + if (key1 == 0 || key2 == 0) + res = TEE_ERROR_BAD_PARAMETERS; + } else { + if (key1 == 0 || key2 != 0) + res = TEE_ERROR_BAD_PARAMETERS; + } + break; + default: + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + if (res != TEE_SUCCESS) + goto out; + + res = copy_kaddr_to_uref(state, cs); + if (res != TEE_SUCCESS) + goto out; + + /* Register keys */ + if (o1 != NULL) { + o1->busy = true; + cs->key1 = (vaddr_t)o1; + } + if (o2 != NULL) { + o2->busy = true; + cs->key2 = (vaddr_t)o2; + } + +out: + if (res != TEE_SUCCESS) + cryp_state_free(utc, cs); + return res; +} + +TEE_Result syscall_cryp_state_copy(unsigned long dst, unsigned long src) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_cryp_state *cs_dst = NULL; + struct tee_cryp_state *cs_src = NULL; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(dst), &cs_dst); + if (res != TEE_SUCCESS) + return res; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(src), &cs_src); + if (res != TEE_SUCCESS) + return res; + if (cs_dst->algo != cs_src->algo || cs_dst->mode != cs_src->mode) + return TEE_ERROR_BAD_PARAMETERS; + + switch (TEE_ALG_GET_CLASS(cs_src->algo)) { + case TEE_OPERATION_CIPHER: + crypto_cipher_copy_state(cs_dst->ctx, cs_src->ctx); + break; + case TEE_OPERATION_AE: + crypto_authenc_copy_state(cs_dst->ctx, cs_src->ctx); + break; + case TEE_OPERATION_DIGEST: + crypto_hash_copy_state(cs_dst->ctx, cs_src->ctx); + break; + case TEE_OPERATION_MAC: + crypto_mac_copy_state(cs_dst->ctx, cs_src->ctx); + break; + default: + return TEE_ERROR_BAD_STATE; + } + + cs_dst->state = cs_src->state; + cs_dst->ctx_finalize = cs_src->ctx_finalize; + + return TEE_SUCCESS; +} + +void tee_svc_cryp_free_states(struct user_ta_ctx *utc) +{ + struct tee_cryp_state_head *states = &utc->cryp_states; + + while (!TAILQ_EMPTY(states)) + cryp_state_free(utc, TAILQ_FIRST(states)); +} + +TEE_Result syscall_cryp_state_free(unsigned long state) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_cryp_state *cs = NULL; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + cryp_state_free(to_user_ta_ctx(sess->ctx), cs); + return TEE_SUCCESS; +} + +TEE_Result syscall_hash_init(unsigned long state, + const void *iv __maybe_unused, + size_t iv_len __maybe_unused) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_cryp_state *cs = NULL; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + switch (TEE_ALG_GET_CLASS(cs->algo)) { + case TEE_OPERATION_DIGEST: + res = crypto_hash_init(cs->ctx); + if (res != TEE_SUCCESS) + return res; + break; + case TEE_OPERATION_MAC: + { + struct tee_obj *o; + struct tee_cryp_obj_secret *key; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), + cs->key1, &o); + if (res != TEE_SUCCESS) + return res; + if ((o->info.handleFlags & + TEE_HANDLE_FLAG_INITIALIZED) == 0) + return TEE_ERROR_BAD_PARAMETERS; + + key = (struct tee_cryp_obj_secret *)o->attr; + res = crypto_mac_init(cs->ctx, (void *)(key + 1), + key->key_size); + if (res != TEE_SUCCESS) + return res; + break; + } + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + cs->state = CRYP_STATE_INITIALIZED; + + return TEE_SUCCESS; +} + +TEE_Result syscall_hash_update(unsigned long state, const void *chunk, + size_t chunk_size) +{ + struct ts_session *sess = ts_get_current_session(); + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + + /* No data, but size provided isn't valid parameters. */ + if (!chunk && chunk_size) + return TEE_ERROR_BAD_PARAMETERS; + + /* Zero length hash is valid, but nothing we need to do. */ + if (!chunk_size) + return TEE_SUCCESS; + + chunk = memtag_strip_tag_const(chunk); + + res = vm_check_access_rights(&to_user_ta_ctx(sess->ctx)->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)chunk, chunk_size); + if (res != TEE_SUCCESS) + return res; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->state != CRYP_STATE_INITIALIZED) + return TEE_ERROR_BAD_STATE; + + switch (TEE_ALG_GET_CLASS(cs->algo)) { + case TEE_OPERATION_DIGEST: + enter_user_access(); + res = crypto_hash_update(cs->ctx, chunk, chunk_size); + exit_user_access(); + if (res != TEE_SUCCESS) + return res; + break; + case TEE_OPERATION_MAC: + enter_user_access(); + res = crypto_mac_update(cs->ctx, chunk, chunk_size); + exit_user_access(); + if (res != TEE_SUCCESS) + return res; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static bool is_xof_algo(uint32_t algo) +{ + return algo == TEE_ALG_SHAKE128 || algo == TEE_ALG_SHAKE256; +} + +TEE_Result syscall_hash_final(unsigned long state, const void *chunk, + size_t chunk_size, void *hash, uint64_t *hash_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct tee_cryp_state *cs = NULL; + TEE_Result res2 = TEE_SUCCESS; + TEE_Result res = TEE_SUCCESS; + size_t hash_size = 0; + size_t hlen = 0; + + /* No data, but size provided isn't valid parameters. */ + if (!chunk && chunk_size) + return TEE_ERROR_BAD_PARAMETERS; + + chunk = memtag_strip_tag_const(chunk); + hash = memtag_strip_tag(hash); + + res = vm_check_access_rights(&to_user_ta_ctx(sess->ctx)->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)chunk, chunk_size); + if (res != TEE_SUCCESS) + return res; + + res = get_user_u64_as_size_t(&hlen, hash_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(&to_user_ta_ctx(sess->ctx)->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)hash, hlen); + if (res != TEE_SUCCESS) + return res; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->state != CRYP_STATE_INITIALIZED) + return TEE_ERROR_BAD_STATE; + + switch (TEE_ALG_GET_CLASS(cs->algo)) { + case TEE_OPERATION_DIGEST: + if (is_xof_algo(cs->algo)) { + if (chunk_size) { + enter_user_access(); + res = crypto_hash_update(cs->ctx, chunk, + chunk_size); + exit_user_access(); + if (res) + return res; + } + + /* + * hash_size is supposed to be unchanged for XOF + * algorithms so return directly. + */ + enter_user_access(); + res = crypto_hash_final(cs->ctx, hash, hlen); + exit_user_access(); + return res; + } + + res = tee_alg_get_digest_size(cs->algo, &hash_size); + if (res != TEE_SUCCESS) + return res; + if (hlen < hash_size) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + if (chunk_size) { + enter_user_access(); + res = crypto_hash_update(cs->ctx, chunk, chunk_size); + exit_user_access(); + if (res != TEE_SUCCESS) + return res; + } + + enter_user_access(); + res = crypto_hash_final(cs->ctx, hash, hash_size); + exit_user_access(); + if (res != TEE_SUCCESS) + return res; + break; + + case TEE_OPERATION_MAC: + res = tee_alg_get_digest_size(cs->algo, &hash_size); + if (res != TEE_SUCCESS) + return res; + if (hlen < hash_size) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + if (chunk_size) { + enter_user_access(); + res = crypto_mac_update(cs->ctx, chunk, chunk_size); + exit_user_access(); + if (res != TEE_SUCCESS) + return res; + } + + enter_user_access(); + res = crypto_mac_final(cs->ctx, hash, hash_size); + exit_user_access(); + if (res != TEE_SUCCESS) + return res; + break; + + default: + return TEE_ERROR_BAD_PARAMETERS; + } +out: + res2 = put_user_u64(hash_len, hash_size); + if (res2 != TEE_SUCCESS) + return res2; + return res; +} + +TEE_Result syscall_cipher_init(unsigned long state, const void *iv, + size_t iv_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + struct tee_cryp_obj_secret *key1 = NULL; + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + void *iv_bbuf = NULL; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (TEE_ALG_GET_CLASS(cs->algo) != TEE_OPERATION_CIPHER) + return TEE_ERROR_BAD_STATE; + + res = tee_obj_get(utc, cs->key1, &o); + if (res != TEE_SUCCESS) + return res; + if ((o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) == 0) + return TEE_ERROR_BAD_PARAMETERS; + + key1 = o->attr; + + res = bb_memdup_user(iv, iv_len, &iv_bbuf); + if (res) + return res; + + if (tee_obj_get(utc, cs->key2, &o) == TEE_SUCCESS) { + struct tee_cryp_obj_secret *key2 = o->attr; + + if ((o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) == 0) + return TEE_ERROR_BAD_PARAMETERS; + + res = crypto_cipher_init(cs->ctx, cs->mode, + (uint8_t *)(key1 + 1), key1->key_size, + (uint8_t *)(key2 + 1), key2->key_size, + iv_bbuf, iv_len); + } else { + res = crypto_cipher_init(cs->ctx, cs->mode, + (uint8_t *)(key1 + 1), key1->key_size, + NULL, 0, iv_bbuf, iv_len); + } + if (res != TEE_SUCCESS) + return res; + + cs->ctx_finalize = crypto_cipher_final; + cs->state = CRYP_STATE_INITIALIZED; + + return TEE_SUCCESS; +} + +static TEE_Result tee_svc_cipher_update_helper(unsigned long state, + bool last_block, const void *src, size_t src_len, + void *dst, uint64_t *dst_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + size_t dlen = 0; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->state != CRYP_STATE_INITIALIZED) + return TEE_ERROR_BAD_STATE; + + src = memtag_strip_tag_const(src); + dst = memtag_strip_tag(dst); + + res = vm_check_access_rights(&to_user_ta_ctx(sess->ctx)->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)src, src_len); + if (res != TEE_SUCCESS) + return res; + + if (!dst_len) { + dlen = 0; + } else { + struct user_mode_ctx *uctx = &to_user_ta_ctx(sess->ctx)->uctx; + uint32_t flags = TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER; + + res = get_user_u64_as_size_t(&dlen, dst_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(uctx, flags, (uaddr_t)dst, dlen); + if (res != TEE_SUCCESS) + return res; + } + + if (dlen < src_len) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + if (src_len > 0) { + /* Permit src_len == 0 to finalize the operation */ + enter_user_access(); + res = tee_do_cipher_update(cs->ctx, cs->algo, cs->mode, + last_block, src, src_len, dst); + exit_user_access(); + } + + if (last_block && cs->ctx_finalize != NULL) { + cs->ctx_finalize(cs->ctx); + cs->ctx_finalize = NULL; + } + +out: + if ((res == TEE_SUCCESS || res == TEE_ERROR_SHORT_BUFFER) && + dst_len != NULL) { + TEE_Result res2; + + res2 = put_user_u64(dst_len, src_len); + if (res2 != TEE_SUCCESS) + res = res2; + } + + return res; +} + +TEE_Result syscall_cipher_update(unsigned long state, const void *src, + size_t src_len, void *dst, uint64_t *dst_len) +{ + return tee_svc_cipher_update_helper(state, false /* last_block */, + src, src_len, dst, dst_len); +} + +TEE_Result syscall_cipher_final(unsigned long state, const void *src, + size_t src_len, void *dst, uint64_t *dst_len) +{ + return tee_svc_cipher_update_helper(state, true /* last_block */, + src, src_len, dst, dst_len); +} + +#if defined(CFG_CRYPTO_HKDF) +static TEE_Result get_hkdf_params(uint32_t algo, const TEE_Attribute *params, + uint32_t param_count, + void **salt, size_t *salt_len, void **info, + size_t *info_len, size_t *okm_len, + uint32_t *hash_id) +{ + TEE_Result res = TEE_SUCCESS; + size_t n; + enum { SALT = 0x1, LENGTH = 0x2, INFO = 0x4, HASH = 0x8 }; + uint8_t found = 0; + + *salt = *info = NULL; + *salt_len = *info_len = *okm_len = 0; + + if (algo == TEE_ALG_HKDF) { + *hash_id = TEE_ALG_SHA256; + } else { + *hash_id = TEE_ALG_GET_DIGEST_HASH(algo); + found |= HASH; + } + + for (n = 0; n < param_count; n++) { + const TEE_Attribute *p = ¶ms[n]; + + switch (p->attributeID) { + case __OPTEE_TEE_ATTR_HKDF_SALT: + case TEE_ATTR_HKDF_SALT: + if (!(found & SALT)) { + *salt_len = p->content.ref.length; + res = bb_memdup_user(p->content.ref.buffer, + *salt_len, salt); + if (res) + return res; + + found |= SALT; + } + break; + case TEE_ATTR_KDF_KEY_SIZE: + case TEE_ATTR_HKDF_OKM_LENGTH: + if (!(found & LENGTH)) { + *okm_len = p->content.value.a; + found |= LENGTH; + } + break; + case __OPTEE_ATTR_HKDF_INFO: + case TEE_ATTR_HKDF_INFO: + if (!(found & INFO)) { + *info_len = p->content.ref.length; + res = bb_memdup_user(p->content.ref.buffer, + *info_len, info); + if (res) + return res; + + found |= INFO; + } + break; + case TEE_ATTR_HKDF_HASH_ALGORITHM: + if (!(found & HASH)) { + *hash_id = p->content.value.a; + found |= HASH; + } + break; + default: + /* Unexpected attribute */ + return TEE_ERROR_BAD_PARAMETERS; + } + + } + + if (!(found & LENGTH)) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; +} +#endif + +#if defined(CFG_CRYPTO_CONCAT_KDF) +static TEE_Result get_concat_kdf_params(const TEE_Attribute *params, + uint32_t param_count, + void **other_info, + size_t *other_info_len, + size_t *derived_key_len) +{ + size_t n; + enum { LENGTH = 0x1, INFO = 0x2 }; + uint8_t found = 0; + + *other_info = NULL; + *other_info_len = *derived_key_len = 0; + + for (n = 0; n < param_count; n++) { + const TEE_Attribute *p = ¶ms[n]; + + switch (p->attributeID) { + case TEE_ATTR_CONCAT_KDF_OTHER_INFO: + if (!(found & INFO)) { + TEE_Result res = TEE_SUCCESS; + + *other_info_len = p->content.ref.length; + res = bb_memdup_user(p->content.ref.buffer, + *other_info_len, + other_info); + if (res) + return res; + + found |= INFO; + } + break; + case TEE_ATTR_CONCAT_KDF_DKM_LENGTH: + if (!(found & LENGTH)) { + *derived_key_len = p->content.value.a; + found |= LENGTH; + } + break; + default: + /* Unexpected attribute */ + return TEE_ERROR_BAD_PARAMETERS; + } + } + + if (!(found & LENGTH)) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; +} +#endif + +#if defined(CFG_CRYPTO_PBKDF2) +static TEE_Result get_pbkdf2_params(const TEE_Attribute *params, + uint32_t param_count, void **salt, + size_t *salt_len, size_t *derived_key_len, + size_t *iteration_count) +{ + size_t n; + enum { SALT = 0x1, LENGTH = 0x2, COUNT = 0x4 }; + uint8_t found = 0; + + *salt = NULL; + *salt_len = *derived_key_len = *iteration_count = 0; + + for (n = 0; n < param_count; n++) { + const TEE_Attribute *p = ¶ms[n]; + + switch (p->attributeID) { + case TEE_ATTR_PBKDF2_SALT: + if (!(found & SALT)) { + TEE_Result res = TEE_SUCCESS; + + *salt_len = p->content.ref.length; + res = bb_memdup_user(p->content.ref.buffer, + *salt_len, salt); + if (res) + return res; + + found |= SALT; + } + break; + case TEE_ATTR_PBKDF2_DKM_LENGTH: + if (!(found & LENGTH)) { + *derived_key_len = p->content.value.a; + found |= LENGTH; + } + break; + case TEE_ATTR_PBKDF2_ITERATION_COUNT: + if (!(found & COUNT)) { + *iteration_count = p->content.value.a; + found |= COUNT; + } + break; + default: + /* Unexpected attribute */ + return TEE_ERROR_BAD_PARAMETERS; + } + } + + if ((found & (LENGTH|COUNT)) != (LENGTH|COUNT)) + return TEE_ERROR_BAD_PARAMETERS; + + return TEE_SUCCESS; +} +#endif + +#if defined(CFG_CRYPTO_SM2_KEP) +static TEE_Result get_sm2_kep_params(const TEE_Attribute *params, + uint32_t param_count, + struct ecc_public_key *peer_key, + struct ecc_public_key *peer_eph_key, + struct sm2_kep_parms *kep_parms) +{ + TEE_Result res = TEE_ERROR_GENERIC; + size_t n; + enum { + IS_INITIATOR, + PEER_KEY_X, + PEER_KEY_Y, + PEER_EPH_KEY_X, + PEER_EPH_KEY_Y, + INITIATOR_ID, + RESPONDER_ID, + }; + uint8_t mandatory = BIT(IS_INITIATOR) | BIT(PEER_KEY_X) | + BIT(PEER_KEY_Y) | BIT(PEER_EPH_KEY_X) | BIT(PEER_EPH_KEY_Y) | + BIT(INITIATOR_ID) | BIT(RESPONDER_ID); + uint8_t found = 0; + + res = crypto_acipher_alloc_ecc_public_key(peer_key, + TEE_TYPE_SM2_KEP_PUBLIC_KEY, + 256); + if (res) + return res; + + res = crypto_acipher_alloc_ecc_public_key(peer_eph_key, + TEE_TYPE_SM2_KEP_PUBLIC_KEY, + 256); + if (res) + goto out_p; + + peer_key->curve = TEE_ECC_CURVE_SM2; + peer_eph_key->curve = TEE_ECC_CURVE_SM2; + + for (n = 0; n < param_count; n++) { + const TEE_Attribute *p = ¶ms[n]; + void *bbuf = NULL; + + switch (p->attributeID) { + case TEE_ATTR_SM2_KEP_USER: + kep_parms->is_initiator = !p->content.value.a; + found |= BIT(IS_INITIATOR); + break; + case TEE_ATTR_ECC_PUBLIC_VALUE_X: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + crypto_bignum_bin2bn(bbuf, + p->content.ref.length, + peer_key->x); + found |= BIT(PEER_KEY_X); + bb_free(bbuf, p->content.ref.length); + break; + case TEE_ATTR_ECC_PUBLIC_VALUE_Y: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + crypto_bignum_bin2bn(bbuf, + p->content.ref.length, + peer_key->y); + found |= BIT(PEER_KEY_Y); + bb_free(bbuf, p->content.ref.length); + break; + case __OPTEE_SM2_KEP_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_X: + case TEE_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_X: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + crypto_bignum_bin2bn(bbuf, + p->content.ref.length, + peer_eph_key->x); + found |= BIT(PEER_EPH_KEY_X); + bb_free(bbuf, p->content.ref.length); + break; + case __OPTEE_SM2_KEP_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_Y: + case TEE_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_Y: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + crypto_bignum_bin2bn(bbuf, + p->content.ref.length, + peer_eph_key->y); + found |= BIT(PEER_EPH_KEY_Y); + bb_free(bbuf, p->content.ref.length); + break; + case TEE_ATTR_SM2_ID_INITIATOR: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + kep_parms->initiator_id = bbuf; + kep_parms->initiator_id_len = p->content.ref.length; + found |= BIT(INITIATOR_ID); + break; + case TEE_ATTR_SM2_ID_RESPONDER: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + kep_parms->responder_id = bbuf; + kep_parms->responder_id_len = p->content.ref.length; + found |= BIT(RESPONDER_ID); + break; + case TEE_ATTR_SM2_KEP_CONFIRMATION_IN: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + kep_parms->conf_in = bbuf; + kep_parms->conf_in_len = p->content.ref.length; + break; + case TEE_ATTR_SM2_KEP_CONFIRMATION_OUT: + res = bb_memdup_user(p->content.ref.buffer, + p->content.ref.length, + &bbuf); + if (res) + return res; + + kep_parms->conf_out = bbuf; + kep_parms->conf_out_len = p->content.ref.length; + break; + default: + /* Unexpected attribute */ + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + } + + if ((found & mandatory) != mandatory) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + return TEE_SUCCESS; +out: + crypto_acipher_free_ecc_public_key(peer_eph_key); +out_p: + crypto_acipher_free_ecc_public_key(peer_key); + return res; +} +#endif + +TEE_Result syscall_cryp_derive_key(unsigned long state, + const struct utee_attribute *usr_params, + unsigned long param_count, unsigned long derived_key) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_ERROR_NOT_SUPPORTED; + struct tee_obj *ko = NULL; + struct tee_obj *so = NULL; + struct tee_cryp_state *cs = NULL; + struct tee_cryp_obj_secret *sk = NULL; + const struct tee_cryp_obj_type_props *type_props = NULL; + TEE_Attribute *params = NULL; + size_t alloc_size = 0; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (MUL_OVERFLOW(sizeof(TEE_Attribute), param_count, &alloc_size)) + return TEE_ERROR_OVERFLOW; + + params = malloc(alloc_size); + if (!params) + return TEE_ERROR_OUT_OF_MEMORY; + res = copy_in_attrs(utc, usr_params, param_count, params); + if (res != TEE_SUCCESS) + goto out; + + /* Get key set in operation */ + res = tee_obj_get(utc, cs->key1, &ko); + if (res != TEE_SUCCESS) + goto out; + + res = tee_obj_get(utc, uref_to_vaddr(derived_key), &so); + if (res != TEE_SUCCESS) + goto out; + + /* Find information needed about the object to initialize */ + sk = so->attr; + + /* Find description of object */ + type_props = tee_svc_find_type_props(so->info.objectType); + if (!type_props) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + if (cs->algo == TEE_ALG_DH_DERIVE_SHARED_SECRET) { + struct bignum *pub = NULL; + struct bignum *ss = NULL; + size_t bin_size = 0; + void *bbuf = NULL; + + if (param_count != 1 || + params[0].attributeID != TEE_ATTR_DH_PUBLIC_VALUE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + bin_size = params[0].content.ref.length; + + if (MUL_OVERFLOW(bin_size, 8, &alloc_size)) { + res = TEE_ERROR_OVERFLOW; + goto out; + } + + res = bb_memdup_user(params[0].content.ref.buffer, bin_size, + &bbuf); + if (res) + goto out; + + pub = crypto_bignum_allocate(alloc_size); + ss = crypto_bignum_allocate(alloc_size); + if (pub && ss) { + crypto_bignum_bin2bn(bbuf, bin_size, pub); + res = crypto_acipher_dh_shared_secret(ko->attr, + pub, ss); + if (res == TEE_SUCCESS) { + sk->key_size = crypto_bignum_num_bytes(ss); + crypto_bignum_bn2bin(ss, (uint8_t *)(sk + 1)); + so->info.handleFlags |= + TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, + TEE_ATTR_SECRET_VALUE); + } + } else { + res = TEE_ERROR_OUT_OF_MEMORY; + } + crypto_bignum_free(&pub); + crypto_bignum_free(&ss); + } else if (cs->algo == TEE_ALG_ECDH_DERIVE_SHARED_SECRET) { + uint32_t curve = ((struct ecc_keypair *)ko->attr)->curve; + struct ecc_public_key key_public = { }; + uint8_t *pt_secret = NULL; + unsigned long pt_secret_len = 0; + uint32_t key_type = TEE_TYPE_ECDH_PUBLIC_KEY; + void *x_bbuf = NULL; + void *y_bbuf = NULL; + + if (param_count != 2 || + params[0].attributeID != TEE_ATTR_ECC_PUBLIC_VALUE_X || + params[1].attributeID != TEE_ATTR_ECC_PUBLIC_VALUE_Y) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + alloc_size = 192; + break; + case TEE_ECC_CURVE_NIST_P224: + alloc_size = 224; + break; + case TEE_ECC_CURVE_NIST_P256: + alloc_size = 256; + break; + case TEE_ECC_CURVE_NIST_P384: + alloc_size = 384; + break; + case TEE_ECC_CURVE_NIST_P521: + alloc_size = 521; + break; + default: + res = TEE_ERROR_NOT_IMPLEMENTED; + goto out; + } + + res = bb_memdup_user(params[0].content.ref.buffer, + params[0].content.ref.length, + &x_bbuf); + if (res) + goto out; + + res = bb_memdup_user(params[1].content.ref.buffer, + params[1].content.ref.length, + &y_bbuf); + if (res) + goto out; + + /* Create the public key */ + res = crypto_acipher_alloc_ecc_public_key(&key_public, key_type, + alloc_size); + if (res != TEE_SUCCESS) + goto out; + key_public.curve = curve; + crypto_bignum_bin2bn(x_bbuf, params[0].content.ref.length, + key_public.x); + crypto_bignum_bin2bn(y_bbuf, params[1].content.ref.length, + key_public.y); + + pt_secret = (uint8_t *)(sk + 1); + pt_secret_len = sk->alloc_size; + res = crypto_acipher_ecc_shared_secret(ko->attr, &key_public, + pt_secret, + &pt_secret_len); + + if (res == TEE_SUCCESS) { + sk->key_size = pt_secret_len; + so->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, TEE_ATTR_SECRET_VALUE); + } + + /* free the public key */ + crypto_acipher_free_ecc_public_key(&key_public); + } +#if defined(CFG_CRYPTO_HKDF) + else if (TEE_ALG_GET_MAIN_ALG(cs->algo) == TEE_MAIN_ALGO_HKDF) { + void *salt, *info; + size_t salt_len, info_len, okm_len; + uint32_t hash_id = 0; + struct tee_cryp_obj_secret *ik = ko->attr; + const uint8_t *ikm = (const uint8_t *)(ik + 1); + + res = get_hkdf_params(cs->algo, params, param_count, &salt, + &salt_len, &info, &info_len, &okm_len, + &hash_id); + if (res != TEE_SUCCESS) + goto out; + + /* Requested size must fit into the output object's buffer */ + if (okm_len > ik->alloc_size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = tee_cryp_hkdf(hash_id, ikm, ik->key_size, salt, salt_len, + info, info_len, (uint8_t *)(sk + 1), + okm_len); + if (res == TEE_SUCCESS) { + sk->key_size = okm_len; + so->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, TEE_ATTR_SECRET_VALUE); + } + } +#endif +#if defined(CFG_CRYPTO_CONCAT_KDF) + else if (TEE_ALG_GET_MAIN_ALG(cs->algo) == TEE_MAIN_ALGO_CONCAT_KDF) { + void *info; + size_t info_len, derived_key_len; + uint32_t hash_id = TEE_ALG_GET_DIGEST_HASH(cs->algo); + struct tee_cryp_obj_secret *ss = ko->attr; + const uint8_t *shared_secret = (const uint8_t *)(ss + 1); + + res = get_concat_kdf_params(params, param_count, &info, + &info_len, &derived_key_len); + if (res != TEE_SUCCESS) + goto out; + + /* Requested size must fit into the output object's buffer */ + if (derived_key_len > ss->alloc_size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = tee_cryp_concat_kdf(hash_id, shared_secret, ss->key_size, + info, info_len, (uint8_t *)(sk + 1), + derived_key_len); + if (res == TEE_SUCCESS) { + sk->key_size = derived_key_len; + so->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, TEE_ATTR_SECRET_VALUE); + } + } +#endif +#if defined(CFG_CRYPTO_PBKDF2) + else if (TEE_ALG_GET_MAIN_ALG(cs->algo) == TEE_MAIN_ALGO_PBKDF2) { + void *salt; + size_t salt_len, iteration_count, derived_key_len; + uint32_t hash_id = TEE_ALG_GET_DIGEST_HASH(cs->algo); + struct tee_cryp_obj_secret *ss = ko->attr; + const uint8_t *password = (const uint8_t *)(ss + 1); + + res = get_pbkdf2_params(params, param_count, &salt, &salt_len, + &derived_key_len, &iteration_count); + if (res != TEE_SUCCESS) + goto out; + + /* Requested size must fit into the output object's buffer */ + if (derived_key_len > ss->alloc_size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = tee_cryp_pbkdf2(hash_id, password, ss->key_size, salt, + salt_len, iteration_count, + (uint8_t *)(sk + 1), derived_key_len); + if (res == TEE_SUCCESS) { + sk->key_size = derived_key_len; + so->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, TEE_ATTR_SECRET_VALUE); + } + } +#endif +#if defined(CFG_CRYPTO_SM2_KEP) + else if (cs->algo == TEE_ALG_SM2_KEP) { + struct ecc_public_key peer_eph_key = { }; + struct ecc_public_key peer_key = { }; + struct sm2_kep_parms kep_parms = { + .out = (uint8_t *)(sk + 1), + .out_len = so->info.maxObjectSize, + }; + struct tee_obj *ko2 = NULL; + + res = tee_obj_get(utc, cs->key2, &ko2); + if (res != TEE_SUCCESS) + goto out; + + res = get_sm2_kep_params(params, param_count, &peer_key, + &peer_eph_key, &kep_parms); + if (res != TEE_SUCCESS) + goto out; + + /* + * key1 is our private keypair, key2 is our ephemeral public key + */ + res = crypto_acipher_sm2_kep_derive(ko->attr, /* key1 */ + ko2->attr, /* key2 */ + &peer_key, &peer_eph_key, + &kep_parms); + + if (res == TEE_SUCCESS) { + sk->key_size = kep_parms.out_len; + so->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, TEE_ATTR_SECRET_VALUE); + } + crypto_acipher_free_ecc_public_key(&peer_key); + crypto_acipher_free_ecc_public_key(&peer_eph_key); + } +#endif +#if defined(CFG_CRYPTO_X25519) + else if (cs->algo == TEE_ALG_X25519) { + uint8_t *x25519_pub_key = NULL; + uint8_t *pt_secret = NULL; + unsigned long pt_secret_len = 0; + void *bbuf = NULL; + + if (param_count != 1 || + params[0].attributeID != TEE_ATTR_X25519_PUBLIC_VALUE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* X25519 public key size is 32 bytes */ + if (params[0].content.ref.length != KEY_SIZE_BYTES_25519) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = bb_memdup_user(params[0].content.ref.buffer, + params[0].content.ref.length, + &bbuf); + if (res) + goto out; + + /* Set the public key */ + x25519_pub_key = bbuf; + + pt_secret = (uint8_t *)(sk + 1); + pt_secret_len = sk->alloc_size; + res = crypto_acipher_x25519_shared_secret(ko->attr, + x25519_pub_key, + pt_secret, + &pt_secret_len); + + if (res == TEE_SUCCESS) { + sk->key_size = pt_secret_len; + so->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, TEE_ATTR_SECRET_VALUE); + } + } +#endif +#if defined(CFG_CRYPTO_X448) + else if (cs->algo == TEE_ALG_X448) { + uint8_t *x448_pub_key = NULL; + uint8_t *pt_secret = NULL; + unsigned long pt_secret_len = 0; + void *bbuf = NULL; + + if (param_count != 1 || + params[0].attributeID != TEE_ATTR_X448_PUBLIC_VALUE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* X448 public key size is 56 bytes */ + if (params[0].content.ref.length != KEY_SIZE_BYTES_448) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = bb_memdup_user(params[0].content.ref.buffer, + params[0].content.ref.length, + &bbuf); + if (res) + goto out; + + /* Set the public key */ + x448_pub_key = bbuf; + + pt_secret = (uint8_t *)(sk + 1); + pt_secret_len = sk->alloc_size; + res = crypto_acipher_x448_shared_secret(ko->attr, + x448_pub_key, + pt_secret, + &pt_secret_len); + + if (res == TEE_SUCCESS) { + sk->key_size = pt_secret_len; + so->info.handleFlags |= TEE_HANDLE_FLAG_INITIALIZED; + set_attribute(so, type_props, TEE_ATTR_SECRET_VALUE); + } + } +#endif + else + res = TEE_ERROR_NOT_SUPPORTED; + +out: + free_wipe(params); + return res; +} + +TEE_Result syscall_cryp_random_number_generate(void *buf, size_t blen) +{ + TEE_Result res = TEE_SUCCESS; + void *bbuf = NULL; + + bbuf = bb_alloc(blen); + if (!bbuf) + return TEE_ERROR_OUT_OF_MEMORY; + + res = crypto_rng_read(bbuf, blen); + if (res != TEE_SUCCESS) + return res; + + res = copy_to_user(buf, bbuf, blen); + return res; +} + +TEE_Result syscall_authenc_init(unsigned long state, const void *nonce, + size_t nonce_len, size_t tag_len, + size_t aad_len, size_t payload_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct tee_cryp_obj_secret *key = NULL; + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + void *nonce_bbuf = NULL; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), cs->key1, &o); + if (res != TEE_SUCCESS) + return res; + if ((o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) == 0) + return TEE_ERROR_BAD_PARAMETERS; + + key = o->attr; + + res = bb_memdup_user(nonce, nonce_len, &nonce_bbuf); + if (res) + return res; + + res = crypto_authenc_init(cs->ctx, cs->mode, (uint8_t *)(key + 1), + key->key_size, nonce_bbuf, nonce_len, tag_len, + aad_len, payload_len); + if (res != TEE_SUCCESS) + return res; + + cs->ctx_finalize = crypto_authenc_final; + cs->state = CRYP_STATE_INITIALIZED; + + return TEE_SUCCESS; +} + +TEE_Result syscall_authenc_update_aad(unsigned long state, + const void *aad_data, size_t aad_data_len) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_cryp_state *cs = NULL; + + aad_data = memtag_strip_tag_const(aad_data); + + res = vm_check_access_rights(&to_user_ta_ctx(sess->ctx)->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)aad_data, aad_data_len); + if (res != TEE_SUCCESS) + return res; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->state != CRYP_STATE_INITIALIZED) + return TEE_ERROR_BAD_STATE; + + if (TEE_ALG_GET_CLASS(cs->algo) != TEE_OPERATION_AE) + return TEE_ERROR_BAD_STATE; + + enter_user_access(); + res = crypto_authenc_update_aad(cs->ctx, cs->mode, aad_data, + aad_data_len); + exit_user_access(); + if (res != TEE_SUCCESS) + return res; + + return TEE_SUCCESS; +} + +TEE_Result syscall_authenc_update_payload(unsigned long state, + const void *src_data, + size_t src_len, void *dst_data, + uint64_t *dst_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + size_t dlen = 0; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->state != CRYP_STATE_INITIALIZED) + return TEE_ERROR_BAD_STATE; + + if (TEE_ALG_GET_CLASS(cs->algo) != TEE_OPERATION_AE) + return TEE_ERROR_BAD_STATE; + + src_data = memtag_strip_tag_const(src_data); + dst_data = memtag_strip_tag(dst_data); + + res = vm_check_access_rights(&to_user_ta_ctx(sess->ctx)->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)src_data, src_len); + if (res != TEE_SUCCESS) + return res; + + res = get_user_u64_as_size_t(&dlen, dst_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(&to_user_ta_ctx(sess->ctx)->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)dst_data, dlen); + if (res != TEE_SUCCESS) + return res; + + if (dlen < src_len) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + enter_user_access(); + res = crypto_authenc_update_payload(cs->ctx, cs->mode, src_data, + src_len, dst_data, &dlen); + exit_user_access(); +out: + if (res == TEE_SUCCESS || res == TEE_ERROR_SHORT_BUFFER) { + TEE_Result res2 = put_user_u64(dst_len, dlen); + + if (res2 != TEE_SUCCESS) + res = res2; + } + + return res; +} + +TEE_Result syscall_authenc_enc_final(unsigned long state, const void *src_data, + size_t src_len, void *dst_data, + uint64_t *dst_len, void *tag, + uint64_t *tag_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = &to_user_ta_ctx(sess->ctx)->uctx; + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + size_t dlen = 0; + size_t tlen = 0; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->state != CRYP_STATE_INITIALIZED) + return TEE_ERROR_BAD_STATE; + + if (cs->mode != TEE_MODE_ENCRYPT) + return TEE_ERROR_BAD_PARAMETERS; + + if (TEE_ALG_GET_CLASS(cs->algo) != TEE_OPERATION_AE) + return TEE_ERROR_BAD_STATE; + + src_data = memtag_strip_tag_const(src_data); + dst_data = memtag_strip_tag(dst_data); + tag = memtag_strip_tag(tag); + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)src_data, src_len); + if (res != TEE_SUCCESS) + return res; + + if (!dst_len) { + dlen = 0; + } else { + res = get_user_u64_as_size_t(&dlen, dst_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)dst_data, dlen); + if (res != TEE_SUCCESS) + return res; + } + + if (dlen < src_len) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + res = get_user_u64_as_size_t(&tlen, tag_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)tag, tlen); + if (res != TEE_SUCCESS) + return res; + + enter_user_access(); + res = crypto_authenc_enc_final(cs->ctx, src_data, src_len, dst_data, + &dlen, tag, &tlen); + exit_user_access(); + +out: + if (res == TEE_SUCCESS || res == TEE_ERROR_SHORT_BUFFER) { + TEE_Result res2 = TEE_SUCCESS; + + if (dst_len != NULL) { + res2 = put_user_u64(dst_len, dlen); + if (res2 != TEE_SUCCESS) + return res2; + } + + res2 = put_user_u64(tag_len, tlen); + if (res2 != TEE_SUCCESS) + return res2; + } + + return res; +} + +TEE_Result syscall_authenc_dec_final(unsigned long state, + const void *src_data, size_t src_len, void *dst_data, + uint64_t *dst_len, const void *tag, size_t tag_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_mode_ctx *uctx = &to_user_ta_ctx(sess->ctx)->uctx; + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + size_t dlen = 0; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->state != CRYP_STATE_INITIALIZED) + return TEE_ERROR_BAD_STATE; + + if (cs->mode != TEE_MODE_DECRYPT) + return TEE_ERROR_BAD_PARAMETERS; + + if (TEE_ALG_GET_CLASS(cs->algo) != TEE_OPERATION_AE) + return TEE_ERROR_BAD_STATE; + + src_data = memtag_strip_tag_const(src_data); + dst_data = memtag_strip_tag(dst_data); + tag = memtag_strip_tag_const(tag); + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)src_data, src_len); + if (res != TEE_SUCCESS) + return res; + + if (!dst_len) { + dlen = 0; + } else { + res = get_user_u64_as_size_t(&dlen, dst_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)dst_data, dlen); + if (res != TEE_SUCCESS) + return res; + } + + if (dlen < src_len) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + /* Despite TEE Internal Core API up to v1.3, tag is [inbuf], not [in] */ + res = vm_check_access_rights(uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)tag, tag_len); + if (res != TEE_SUCCESS) + return res; + + enter_user_access(); + res = crypto_authenc_dec_final(cs->ctx, src_data, src_len, dst_data, + &dlen, tag, tag_len); + exit_user_access(); + +out: + if ((res == TEE_SUCCESS || res == TEE_ERROR_SHORT_BUFFER) && + dst_len != NULL) { + TEE_Result res2 = put_user_u64(dst_len, dlen); + + if (res2 != TEE_SUCCESS) + return res2; + } + + return res; +} + +static int pkcs1_get_salt_len(const TEE_Attribute *params, uint32_t num_params, + size_t default_len) +{ + size_t n; + + assert(default_len < INT_MAX); + + for (n = 0; n < num_params; n++) { + if (params[n].attributeID == TEE_ATTR_RSA_PSS_SALT_LENGTH) { + if (params[n].content.value.a < INT_MAX) + return params[n].content.value.a; + break; + } + } + /* + * If salt length isn't provided use the default value which is + * the length of the digest. + */ + return default_len; +} + +TEE_Result syscall_asymm_operate(unsigned long state, + const struct utee_attribute *usr_params, + size_t num_params, const void *src_data, size_t src_len, + void *dst_data, uint64_t *dst_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + struct tee_cryp_state *cs = NULL; + size_t dlen = 0; + struct tee_obj *o = NULL; + void *label = NULL; + size_t label_len = 0; + size_t n = 0; + int salt_len = 0; + TEE_Attribute *params = NULL; + size_t alloc_size = 0; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + src_data = memtag_strip_tag_const(src_data); + dst_data = memtag_strip_tag(dst_data); + + res = vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)src_data, src_len); + if (res != TEE_SUCCESS) + return res; + + res = get_user_u64_as_size_t(&dlen, dst_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)dst_data, dlen); + if (res != TEE_SUCCESS) + return res; + + if (MUL_OVERFLOW(sizeof(TEE_Attribute), num_params, &alloc_size)) + return TEE_ERROR_OVERFLOW; + + params = malloc(alloc_size); + if (!params) + return TEE_ERROR_OUT_OF_MEMORY; + res = copy_in_attrs(utc, usr_params, num_params, params); + if (res != TEE_SUCCESS) + goto out; + + res = tee_obj_get(utc, cs->key1, &o); + if (res != TEE_SUCCESS) + goto out; + if ((o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_GENERIC; + goto out; + } + + switch (cs->algo) { + case TEE_ALG_RSA_NOPAD: + if (cs->mode == TEE_MODE_ENCRYPT) { + enter_user_access(); + res = crypto_acipher_rsanopad_encrypt(o->attr, src_data, + src_len, dst_data, + &dlen); + exit_user_access(); + } else if (cs->mode == TEE_MODE_DECRYPT) { + enter_user_access(); + res = crypto_acipher_rsanopad_decrypt(o->attr, src_data, + src_len, dst_data, + &dlen); + exit_user_access(); + } else { + /* + * We will panic because "the mode is not compatible + * with the function" + */ + res = TEE_ERROR_GENERIC; + } + break; + + case TEE_ALG_SM2_PKE: + if (cs->mode == TEE_MODE_ENCRYPT) { + enter_user_access(); + res = crypto_acipher_sm2_pke_encrypt(o->attr, src_data, + src_len, dst_data, + &dlen); + exit_user_access(); + } else if (cs->mode == TEE_MODE_DECRYPT) { + enter_user_access(); + res = crypto_acipher_sm2_pke_decrypt(o->attr, src_data, + src_len, dst_data, + &dlen); + exit_user_access(); + } else { + res = TEE_ERROR_GENERIC; + } + break; + + case TEE_ALG_RSAES_PKCS1_V1_5: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_MD5: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512: + for (n = 0; n < num_params; n++) { + if (params[n].attributeID == TEE_ATTR_RSA_OAEP_LABEL) { + label = params[n].content.ref.buffer; + label_len = params[n].content.ref.length; + break; + } + /* + * If the optional TEE_ATTR_RSA_OAEP_MGF_HASH is + * provided for algorithm + * TEE_ALG_RSAES_PKCS1_OAEP_MGF1_x it must match + * the internal hash x since we don't support using + * a different hash for MGF1 yet. + */ + if (cs->algo != TEE_ALG_RSAES_PKCS1_V1_5 && + params[n].attributeID == + TEE_ATTR_RSA_OAEP_MGF_HASH) { + uint32_t hash = 0; + void *buf = params[n].content.ref.buffer; + + if (params[n].content.ref.length != + sizeof(hash)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = copy_from_user(&hash, buf, sizeof(hash)); + if (res) + goto out; + + if (hash != + TEE_INTERNAL_HASH_TO_ALGO(cs->algo)) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + } + } + + if (cs->mode == TEE_MODE_ENCRYPT) { + enter_user_access(); + res = crypto_acipher_rsaes_encrypt(cs->algo, o->attr, + label, label_len, + src_data, src_len, + dst_data, &dlen); + exit_user_access(); + } else if (cs->mode == TEE_MODE_DECRYPT) { + enter_user_access(); + res = crypto_acipher_rsaes_decrypt( + cs->algo, o->attr, label, label_len, + src_data, src_len, dst_data, &dlen); + exit_user_access(); + } else { + res = TEE_ERROR_BAD_PARAMETERS; + } + break; + +#if defined(CFG_CRYPTO_RSASSA_NA1) + case TEE_ALG_RSASSA_PKCS1_V1_5: +#endif + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + if (cs->mode != TEE_MODE_SIGN) { + res = TEE_ERROR_BAD_PARAMETERS; + break; + } + salt_len = pkcs1_get_salt_len(params, num_params, src_len); + enter_user_access(); + res = crypto_acipher_rsassa_sign(cs->algo, o->attr, salt_len, + src_data, src_len, dst_data, + &dlen); + exit_user_access(); + break; + + case TEE_ALG_DSA_SHA1: + case TEE_ALG_DSA_SHA224: + case TEE_ALG_DSA_SHA256: + enter_user_access(); + res = crypto_acipher_dsa_sign(cs->algo, o->attr, src_data, + src_len, dst_data, &dlen); + exit_user_access(); + break; + + case TEE_ALG_ED25519: + enter_user_access(); + res = tee_svc_obj_ed25519_sign(o->attr, src_data, src_len, + dst_data, &dlen, params, + num_params); + exit_user_access(); + break; + + case TEE_ALG_ECDSA_SHA1: + case TEE_ALG_ECDSA_SHA224: + case TEE_ALG_ECDSA_SHA256: + case TEE_ALG_ECDSA_SHA384: + case TEE_ALG_ECDSA_SHA512: + case TEE_ALG_SM2_DSA_SM3: + enter_user_access(); + res = crypto_acipher_ecc_sign(cs->algo, o->attr, src_data, + src_len, dst_data, &dlen); + exit_user_access(); + break; + default: + res = TEE_ERROR_BAD_PARAMETERS; + break; + } + +out: + free_wipe(params); + + if (res == TEE_SUCCESS || res == TEE_ERROR_SHORT_BUFFER) { + TEE_Result res2 = put_user_u64(dst_len, dlen); + + if (res2 != TEE_SUCCESS) + return res2; + } + + return res; +} + +TEE_Result syscall_asymm_verify(unsigned long state, + const struct utee_attribute *usr_params, + size_t num_params, const void *data, size_t data_len, + const void *sig, size_t sig_len) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + struct tee_cryp_state *cs = NULL; + TEE_Result res = TEE_SUCCESS; + TEE_Attribute *params = NULL; + struct tee_obj *o = NULL; + size_t hash_size = 0; + uint32_t hash_algo = 0; + int salt_len = 0; + size_t alloc_size = 0; + + res = tee_svc_cryp_get_state(sess, uref_to_vaddr(state), &cs); + if (res != TEE_SUCCESS) + return res; + + if (cs->mode != TEE_MODE_VERIFY) + return TEE_ERROR_BAD_PARAMETERS; + + data = memtag_strip_tag_const(data); + sig = memtag_strip_tag_const(sig); + + res = vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)data, data_len); + if (res != TEE_SUCCESS) + return res; + + res = vm_check_access_rights(&utc->uctx, + TEE_MEMORY_ACCESS_READ | + TEE_MEMORY_ACCESS_ANY_OWNER, + (uaddr_t)sig, sig_len); + if (res != TEE_SUCCESS) + return res; + + if (MUL_OVERFLOW(sizeof(TEE_Attribute), num_params, &alloc_size)) + return TEE_ERROR_OVERFLOW; + + params = malloc(alloc_size); + if (!params) + return TEE_ERROR_OUT_OF_MEMORY; + res = copy_in_attrs(utc, usr_params, num_params, params); + if (res != TEE_SUCCESS) + goto out; + + res = tee_obj_get(utc, cs->key1, &o); + if (res != TEE_SUCCESS) + goto out; + if ((o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + switch (TEE_ALG_GET_MAIN_ALG(cs->algo)) { + case TEE_MAIN_ALGO_RSA: + if (cs->algo != TEE_ALG_RSASSA_PKCS1_V1_5) { + hash_algo = TEE_DIGEST_HASH_TO_ALGO(cs->algo); + res = tee_alg_get_digest_size(hash_algo, &hash_size); + if (res != TEE_SUCCESS) + break; + if (data_len != hash_size) { + res = TEE_ERROR_BAD_PARAMETERS; + break; + } + salt_len = pkcs1_get_salt_len(params, num_params, + hash_size); + } + enter_user_access(); + res = crypto_acipher_rsassa_verify(cs->algo, o->attr, salt_len, + data, data_len, sig, + sig_len); + exit_user_access(); + break; + + case TEE_MAIN_ALGO_DSA: + hash_algo = TEE_DIGEST_HASH_TO_ALGO(cs->algo); + res = tee_alg_get_digest_size(hash_algo, &hash_size); + if (res != TEE_SUCCESS) + break; + + if (data_len != hash_size) { + struct dsa_public_key *key = o->attr; + + /* + * Depending on the DSA algorithm (NIST), the + * digital signature output size may be truncated + * to the size of a key pair (Q prime size). Q + * prime size must be less or equal than the hash + * output length of the hash algorithm involved. + * + * We're checking here in order to be able to + * return this particular error code, which will + * cause TEE_AsymmetricVerifyDigest() to panic as + * required by GP. crypto_acipher_dsa_verify() is + * implemented in the glue layer of the crypto + * library and it might be a bit harder to catch + * this particular case there or lead to duplicated + * code in different crypto glue layers. + * + * The GP spec says that we SHOULD panic if + * data_len != hash_size, but that would break a + * few of the DSA tests in xtest where the + * hash_size is larger than possible data_len. So + * the compromise is in case data_len != hash_size + * check that it's not smaller than what makes + * sense. + */ + if (data_len != crypto_bignum_num_bytes(key->q)) { + res = TEE_ERROR_BAD_PARAMETERS; + break; + } + } + enter_user_access(); + res = crypto_acipher_dsa_verify(cs->algo, o->attr, data, + data_len, sig, sig_len); + exit_user_access(); + break; + + case TEE_MAIN_ALGO_ED25519: + enter_user_access(); + res = tee_svc_obj_ed25519_verify(o->attr, data, + data_len, sig, sig_len, + params, num_params); + exit_user_access(); + break; + + case TEE_MAIN_ALGO_ECDSA: + case TEE_MAIN_ALGO_SM2_DSA_SM3: + enter_user_access(); + res = crypto_acipher_ecc_verify(cs->algo, o->attr, data, + data_len, sig, sig_len); + exit_user_access(); + break; + + default: + res = TEE_ERROR_NOT_SUPPORTED; + } + +out: + free_wipe(params); + return res; +} diff --git a/optee_os/core/tee/tee_svc_storage.c b/optee_os/core/tee/tee_svc_storage.c new file mode 100644 index 0000000..3adf62d --- /dev/null +++ b/optee_os/core/tee/tee_svc_storage.c @@ -0,0 +1,916 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, 2022 Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Header of GP formated secure storage files */ +struct tee_svc_storage_head { + uint32_t attr_size; + uint32_t objectSize; + uint32_t maxObjectSize; + uint32_t objectUsage; + uint32_t objectType; + uint32_t have_attrs; +}; + +struct tee_storage_enum { + TAILQ_ENTRY(tee_storage_enum) link; + struct tee_fs_dir *dir; + const struct tee_file_operations *fops; +}; + +static TEE_Result tee_svc_storage_get_enum(struct user_ta_ctx *utc, + vaddr_t enum_id, + struct tee_storage_enum **e_out) +{ + struct tee_storage_enum *e; + + TAILQ_FOREACH(e, &utc->storage_enums, link) { + if (enum_id == (vaddr_t)e) { + *e_out = e; + return TEE_SUCCESS; + } + } + return TEE_ERROR_BAD_PARAMETERS; +} + +static TEE_Result tee_svc_close_enum(struct user_ta_ctx *utc, + struct tee_storage_enum *e) +{ + if (e == NULL || utc == NULL) + return TEE_ERROR_BAD_PARAMETERS; + + TAILQ_REMOVE(&utc->storage_enums, e, link); + + if (e->fops) + e->fops->closedir(e->dir); + + e->dir = NULL; + e->fops = NULL; + + free(e); + + return TEE_SUCCESS; +} + +static void remove_corrupt_obj(struct user_ta_ctx *utc, struct tee_obj *o) +{ + o->pobj->fops->remove(o->pobj); + if (!(utc->ta_ctx.flags & TA_FLAG_DONT_CLOSE_HANDLE_ON_CORRUPT_OBJECT)) + tee_obj_close(utc, o); +} + +static TEE_Result tee_svc_storage_read_head(struct tee_obj *o) +{ + TEE_Result res = TEE_SUCCESS; + size_t bytes; + struct tee_svc_storage_head head; + const struct tee_file_operations *fops = o->pobj->fops; + void *attr = NULL; + size_t size; + size_t tmp = 0; + + assert(!o->fh); + res = fops->open(o->pobj, &size, &o->fh); + if (res != TEE_SUCCESS) + goto exit; + + /* read head */ + bytes = sizeof(struct tee_svc_storage_head); + res = fops->read(o->fh, 0, &head, NULL, &bytes); + if (res != TEE_SUCCESS) { + if (res == TEE_ERROR_CORRUPT_OBJECT) + EMSG("Head corrupt"); + goto exit; + } + + if (ADD_OVERFLOW(sizeof(head), head.attr_size, &tmp)) { + res = TEE_ERROR_OVERFLOW; + goto exit; + } + if (tmp > size) { + res = TEE_ERROR_CORRUPT_OBJECT; + goto exit; + } + + if (bytes != sizeof(struct tee_svc_storage_head)) { + res = TEE_ERROR_BAD_FORMAT; + goto exit; + } + + res = tee_obj_set_type(o, head.objectType, head.maxObjectSize); + if (res != TEE_SUCCESS) + goto exit; + + o->ds_pos = tmp; + + if (head.attr_size) { + attr = malloc(head.attr_size); + if (!attr) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto exit; + } + + /* read meta */ + bytes = head.attr_size; + res = fops->read(o->fh, sizeof(struct tee_svc_storage_head), + attr, NULL, &bytes); + if (res == TEE_ERROR_OUT_OF_MEMORY) + goto exit; + if (res != TEE_SUCCESS || bytes != head.attr_size) + res = TEE_ERROR_CORRUPT_OBJECT; + if (res) + goto exit; + } + + res = tee_obj_attr_from_binary(o, attr, head.attr_size); + if (res != TEE_SUCCESS) + goto exit; + + o->info.dataSize = size - sizeof(head) - head.attr_size; + o->info.objectSize = head.objectSize; + o->info.objectUsage = head.objectUsage; + o->info.objectType = head.objectType; + o->have_attrs = head.have_attrs; + +exit: + free(attr); + + return res; +} + +TEE_Result syscall_storage_obj_open(unsigned long storage_id, void *object_id, + size_t object_id_len, unsigned long flags, + uint32_t *obj) +{ + const unsigned long valid_flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE | + TEE_DATA_FLAG_ACCESS_WRITE_META | + TEE_DATA_FLAG_SHARE_READ | + TEE_DATA_FLAG_SHARE_WRITE; + const struct tee_file_operations *fops = + tee_svc_storage_file_ops(storage_id); + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + struct tee_pobj *po = NULL; + struct tee_obj *o = NULL; + void *oid_bbuf = NULL; + + if (flags & ~valid_flags) + return TEE_ERROR_BAD_PARAMETERS; + + if (!fops) { + res = TEE_ERROR_ITEM_NOT_FOUND; + goto exit; + } + + if (object_id_len > TEE_OBJECT_ID_MAX_LEN) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + res = bb_memdup_user_private(object_id, object_id_len, &oid_bbuf); + if (res) + goto exit; + + res = tee_pobj_get((void *)&sess->ctx->uuid, oid_bbuf, + object_id_len, flags, TEE_POBJ_USAGE_OPEN, fops, + &po); + bb_free(oid_bbuf, object_id_len); + if (res != TEE_SUCCESS) + goto err; + + o = tee_obj_alloc(); + if (o == NULL) { + tee_pobj_release(po); + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + o->info.handleFlags = TEE_HANDLE_FLAG_PERSISTENT | + TEE_HANDLE_FLAG_INITIALIZED | flags; + o->pobj = po; + tee_obj_add(utc, o); + + res = tee_svc_storage_read_head(o); + if (res != TEE_SUCCESS) { + if (res == TEE_ERROR_CORRUPT_OBJECT) { + EMSG("Object corrupt"); + goto err; + } + goto oclose; + } + + res = copy_kaddr_to_uref(obj, o); + if (res != TEE_SUCCESS) + goto oclose; + + goto exit; + +oclose: + tee_obj_close(utc, o); + o = NULL; + +err: + if (res == TEE_ERROR_NO_DATA || res == TEE_ERROR_BAD_FORMAT) + res = TEE_ERROR_CORRUPT_OBJECT; + if (res == TEE_ERROR_CORRUPT_OBJECT && o) + remove_corrupt_obj(utc, o); + +exit: + return res; +} + +static TEE_Result tee_svc_storage_init_file(struct tee_obj *o, bool overwrite, + struct tee_obj *attr_o, + void *data, uint32_t len) +{ + TEE_Result res = TEE_SUCCESS; + struct tee_svc_storage_head head = { }; + const struct tee_file_operations *fops = o->pobj->fops; + void *attr = NULL; + size_t attr_size = 0; + + if (attr_o) { + if (o != attr_o) { + res = tee_obj_set_type(o, attr_o->info.objectType, + attr_o->info.maxObjectSize); + if (res) + return res; + res = tee_obj_attr_copy_from(o, attr_o); + if (res) + return res; + o->have_attrs = attr_o->have_attrs; + o->info.objectUsage = attr_o->info.objectUsage; + o->info.objectSize = attr_o->info.objectSize; + } + res = tee_obj_attr_to_binary(o, NULL, &attr_size); + if (res) + return res; + if (attr_size) { + attr = malloc(attr_size); + if (!attr) + return TEE_ERROR_OUT_OF_MEMORY; + res = tee_obj_attr_to_binary(o, attr, &attr_size); + if (res != TEE_SUCCESS) + goto exit; + } + } else { + res = tee_obj_set_type(o, TEE_TYPE_DATA, 0); + if (res != TEE_SUCCESS) + goto exit; + } + + o->ds_pos = sizeof(struct tee_svc_storage_head) + attr_size; + + /* write head */ + head.attr_size = attr_size; + head.objectSize = o->info.objectSize; + head.maxObjectSize = o->info.maxObjectSize; + head.objectUsage = o->info.objectUsage; + head.objectType = o->info.objectType; + head.have_attrs = o->have_attrs; + + res = fops->create(o->pobj, overwrite, &head, sizeof(head), attr, + attr_size, NULL, data, len, &o->fh); + + if (res) + o->ds_pos = 0; + else + o->info.dataSize = len; +exit: + free(attr); + return res; +} + +TEE_Result syscall_storage_obj_create(unsigned long storage_id, void *object_id, + size_t object_id_len, unsigned long flags, + unsigned long attr, void *data, size_t len, + uint32_t *obj) +{ + const unsigned long valid_flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE | + TEE_DATA_FLAG_ACCESS_WRITE_META | + TEE_DATA_FLAG_SHARE_READ | + TEE_DATA_FLAG_SHARE_WRITE | + TEE_DATA_FLAG_OVERWRITE; + const struct tee_file_operations *fops = + tee_svc_storage_file_ops(storage_id); + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + struct tee_obj *attr_o = NULL; + TEE_Result res = TEE_SUCCESS; + struct tee_pobj *po = NULL; + struct tee_obj *o = NULL; + void *oid_bbuf = NULL; + + if (flags & ~valid_flags) + return TEE_ERROR_BAD_PARAMETERS; + + if (!fops) + return TEE_ERROR_ITEM_NOT_FOUND; + + if (object_id_len > TEE_OBJECT_ID_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + object_id = memtag_strip_tag(object_id); + data = memtag_strip_tag(data); + + /* Check presence of optional buffer */ + if (len && !data) + return TEE_ERROR_BAD_PARAMETERS; + + res = bb_memdup_user_private(object_id, object_id_len, &oid_bbuf); + if (res) + return res; + + res = tee_pobj_get((void *)&sess->ctx->uuid, oid_bbuf, + object_id_len, flags, TEE_POBJ_USAGE_CREATE, + fops, &po); + bb_free(oid_bbuf, object_id_len); + if (res != TEE_SUCCESS) + goto err; + + if (attr != TEE_HANDLE_NULL) { + res = tee_obj_get(utc, uref_to_vaddr(attr), &attr_o); + if (res != TEE_SUCCESS) + goto err; + /* The supplied handle must be one of an initialized object */ + if (!(attr_o->info.handleFlags & TEE_HANDLE_FLAG_INITIALIZED)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + } + + if (!obj && attr_o && + !(attr_o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT)) { + /* + * The caller expects the supplied attributes handle to be + * transformed into a persistent object. + */ + uint32_t saved_flags = attr_o->info.handleFlags; + + attr_o->info.handleFlags = TEE_HANDLE_FLAG_PERSISTENT | + TEE_HANDLE_FLAG_INITIALIZED | flags; + attr_o->pobj = po; + res = tee_svc_storage_init_file(attr_o, + flags & TEE_DATA_FLAG_OVERWRITE, + attr_o, data, len); + if (res) { + attr_o->info.handleFlags = saved_flags; + attr_o->pobj = NULL; + goto err; + } + } else { + o = tee_obj_alloc(); + if (!o) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + o->info.handleFlags = TEE_HANDLE_FLAG_PERSISTENT | + TEE_HANDLE_FLAG_INITIALIZED | flags; + o->pobj = po; + + res = tee_svc_storage_init_file(o, + flags & TEE_DATA_FLAG_OVERWRITE, + attr_o, data, len); + if (res != TEE_SUCCESS) + goto err; + + po = NULL; /* o owns it from now on */ + tee_obj_add(utc, o); + + if (obj) { + res = copy_kaddr_to_uref(obj, o); + if (res != TEE_SUCCESS) + goto oclose; + } + + tee_pobj_create_final(o->pobj); + + if (!obj) + tee_obj_close(utc, o); + } + + return TEE_SUCCESS; + +oclose: + tee_obj_close(utc, o); + return res; + +err: + if (res == TEE_ERROR_NO_DATA || res == TEE_ERROR_BAD_FORMAT) + res = TEE_ERROR_CORRUPT_OBJECT; + if (res == TEE_ERROR_CORRUPT_OBJECT && po) + fops->remove(po); + if (o) { + fops->close(&o->fh); + tee_obj_free(o); + } + if (po) + tee_pobj_release(po); + + return res; +} + +TEE_Result syscall_storage_obj_del(unsigned long obj) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + + res = tee_obj_get(utc, uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return res; + + if (!(o->info.handleFlags & TEE_DATA_FLAG_ACCESS_WRITE_META)) + return TEE_ERROR_ACCESS_CONFLICT; + + if (o->pobj == NULL || o->pobj->obj_id == NULL) + return TEE_ERROR_BAD_STATE; + + if (IS_ENABLED(CFG_NXP_SE05X)) { + /* Cryptographic layer house-keeping */ + res = crypto_storage_obj_del(o); + if (res) + return res; + } + + res = o->pobj->fops->remove(o->pobj); + tee_obj_close(utc, o); + + return res; +} + +TEE_Result syscall_storage_obj_rename(unsigned long obj, void *object_id, + size_t object_id_len) +{ + const struct tee_file_operations *fops = NULL; + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + struct tee_pobj *po = NULL; + struct tee_obj *o = NULL; + char *new_file = NULL; + char *old_file = NULL; + void *oid_bbuf = NULL; + + if (object_id_len > TEE_OBJECT_ID_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + res = tee_obj_get(utc, uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return res; + + if (!(o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT)) { + res = TEE_ERROR_BAD_STATE; + goto exit; + } + + if (!(o->info.handleFlags & TEE_DATA_FLAG_ACCESS_WRITE_META)) { + res = TEE_ERROR_BAD_STATE; + goto exit; + } + + if (o->pobj == NULL || o->pobj->obj_id == NULL) { + res = TEE_ERROR_BAD_STATE; + goto exit; + } + + res = bb_memdup_user_private(object_id, object_id_len, &oid_bbuf); + if (res) + goto exit; + + /* reserve dest name */ + fops = o->pobj->fops; + res = tee_pobj_get((void *)&sess->ctx->uuid, oid_bbuf, + object_id_len, TEE_DATA_FLAG_ACCESS_WRITE_META, + TEE_POBJ_USAGE_RENAME, fops, &po); + bb_free(oid_bbuf, object_id_len); + if (res != TEE_SUCCESS) + goto exit; + + /* move */ + res = fops->rename(o->pobj, po, false /* no overwrite */); + if (res) + goto exit; + + res = tee_pobj_rename(o->pobj, po->obj_id, po->obj_id_len); + +exit: + tee_pobj_release(po); + + free(new_file); + free(old_file); + + return res; +} + +TEE_Result syscall_storage_alloc_enum(uint32_t *obj_enum) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + struct tee_storage_enum *e = NULL; + + if (obj_enum == NULL) + return TEE_ERROR_BAD_PARAMETERS; + + e = malloc(sizeof(struct tee_storage_enum)); + if (e == NULL) + return TEE_ERROR_OUT_OF_MEMORY; + + e->dir = NULL; + e->fops = NULL; + TAILQ_INSERT_TAIL(&utc->storage_enums, e, link); + + return copy_kaddr_to_uref(obj_enum, e); +} + +TEE_Result syscall_storage_free_enum(unsigned long obj_enum) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + struct tee_storage_enum *e = NULL; + TEE_Result res = TEE_SUCCESS; + + res = tee_svc_storage_get_enum(utc, + uref_to_vaddr(obj_enum), &e); + if (res != TEE_SUCCESS) + return res; + + return tee_svc_close_enum(utc, e); +} + +TEE_Result syscall_storage_reset_enum(unsigned long obj_enum) +{ + struct ts_session *sess = ts_get_current_session(); + struct tee_storage_enum *e = NULL; + TEE_Result res = TEE_SUCCESS; + + res = tee_svc_storage_get_enum(to_user_ta_ctx(sess->ctx), + uref_to_vaddr(obj_enum), &e); + if (res != TEE_SUCCESS) + return res; + + if (e->fops) { + e->fops->closedir(e->dir); + e->fops = NULL; + e->dir = NULL; + } + assert(!e->dir); + + return TEE_SUCCESS; +} + +TEE_Result syscall_storage_start_enum(unsigned long obj_enum, + unsigned long storage_id) +{ + struct ts_session *sess = ts_get_current_session(); + struct tee_storage_enum *e = NULL; + TEE_Result res = TEE_SUCCESS; + const struct tee_file_operations *fops = + tee_svc_storage_file_ops(storage_id); + + res = tee_svc_storage_get_enum(to_user_ta_ctx(sess->ctx), + uref_to_vaddr(obj_enum), &e); + if (res != TEE_SUCCESS) + return res; + + if (e->dir) { + e->fops->closedir(e->dir); + e->dir = NULL; + } + + if (!fops) + return TEE_ERROR_ITEM_NOT_FOUND; + + e->fops = fops; + + return fops->opendir(&sess->ctx->uuid, &e->dir); +} + +TEE_Result syscall_storage_next_enum(unsigned long obj_enum, + struct utee_object_info *info, + void *obj_id, uint64_t *len) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + struct tee_storage_enum *e = NULL; + struct tee_fs_dirent *d = NULL; + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + uint64_t l = 0; + struct utee_object_info bbuf = { }; + + res = tee_svc_storage_get_enum(utc, uref_to_vaddr(obj_enum), &e); + if (res != TEE_SUCCESS) + goto exit; + + info = memtag_strip_tag(info); + obj_id = memtag_strip_tag(obj_id); + + /* check rights of the provided buffers */ + res = vm_check_access_rights(&utc->uctx, TEE_MEMORY_ACCESS_WRITE, + (uaddr_t)info, sizeof(*info)); + if (res != TEE_SUCCESS) + goto exit; + + res = vm_check_access_rights(&utc->uctx, TEE_MEMORY_ACCESS_WRITE, + (uaddr_t)obj_id, TEE_OBJECT_ID_MAX_LEN); + if (res != TEE_SUCCESS) + goto exit; + + if (!e->fops) { + res = TEE_ERROR_ITEM_NOT_FOUND; + goto exit; + } + + res = e->fops->readdir(e->dir, &d); + if (res != TEE_SUCCESS) + goto exit; + + o = tee_obj_alloc(); + if (o == NULL) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto exit; + } + + res = tee_pobj_get(&sess->ctx->uuid, d->oid, d->oidlen, 0, + TEE_POBJ_USAGE_ENUM, e->fops, &o->pobj); + if (res) + goto exit; + + o->info.handleFlags = o->pobj->flags | TEE_HANDLE_FLAG_PERSISTENT | + TEE_HANDLE_FLAG_INITIALIZED; + + res = tee_svc_storage_read_head(o); + if (res != TEE_SUCCESS) + goto exit; + + bbuf = (struct utee_object_info){ + .obj_type = o->info.objectType, + .obj_size = o->info.objectSize, + .max_obj_size = o->info.maxObjectSize, + .obj_usage = o->info.objectUsage, + .data_size = o->info.dataSize, + .data_pos = o->info.dataPosition, + .handle_flags = o->info.handleFlags, + }; + res = copy_to_user(info, &bbuf, sizeof(bbuf)); + if (res) + goto exit; + + res = copy_to_user(obj_id, o->pobj->obj_id, o->pobj->obj_id_len); + if (res) + goto exit; + + l = o->pobj->obj_id_len; + res = copy_to_user_private(len, &l, sizeof(*len)); + +exit: + if (o) { + if (o->pobj) { + o->pobj->fops->close(&o->fh); + tee_pobj_release(o->pobj); + } + tee_obj_free(o); + } + + return res; +} + +TEE_Result syscall_storage_obj_read(unsigned long obj, void *data, size_t len, + uint64_t *count) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + uint64_t u_count = 0; + size_t pos_tmp = 0; + size_t bytes = 0; + + res = tee_obj_get(utc, uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + goto exit; + + if (!(o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT)) { + res = TEE_ERROR_BAD_STATE; + goto exit; + } + + if (!(o->info.handleFlags & TEE_DATA_FLAG_ACCESS_READ)) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto exit; + } + + /* Guard o->info.dataPosition += bytes below from overflowing */ + if (ADD_OVERFLOW(o->info.dataPosition, len, &pos_tmp)) { + res = TEE_ERROR_OVERFLOW; + goto exit; + } + + data = memtag_strip_tag(data); + + bytes = len; + if (ADD_OVERFLOW(o->ds_pos, o->info.dataPosition, &pos_tmp)) { + res = TEE_ERROR_OVERFLOW; + goto exit; + } + res = o->pobj->fops->read(o->fh, pos_tmp, NULL, data, &bytes); + if (res != TEE_SUCCESS) { + if (res == TEE_ERROR_CORRUPT_OBJECT) { + EMSG("Object corrupt"); + remove_corrupt_obj(utc, o); + } + goto exit; + } + + o->info.dataPosition += bytes; + + u_count = bytes; + res = copy_to_user_private(count, &u_count, sizeof(*count)); +exit: + return res; +} + +TEE_Result syscall_storage_obj_write(unsigned long obj, void *data, size_t len) +{ + struct ts_session *sess = ts_get_current_session(); + struct user_ta_ctx *utc = to_user_ta_ctx(sess->ctx); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + size_t pos_tmp = 0; + + res = tee_obj_get(utc, uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + goto exit; + + if (!(o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT)) { + res = TEE_ERROR_BAD_STATE; + goto exit; + } + + if (!(o->info.handleFlags & TEE_DATA_FLAG_ACCESS_WRITE)) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto exit; + } + + /* Guard o->info.dataPosition += bytes below from overflowing */ + if (ADD_OVERFLOW(o->info.dataPosition, len, &pos_tmp)) { + res = TEE_ERROR_OVERFLOW; + goto exit; + } + + data = memtag_strip_tag(data); + + if (ADD_OVERFLOW(o->ds_pos, o->info.dataPosition, &pos_tmp)) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto exit; + } + res = o->pobj->fops->write(o->fh, pos_tmp, NULL, data, len); + if (res != TEE_SUCCESS) { + if (res == TEE_ERROR_CORRUPT_OBJECT) { + EMSG("Object corrupt"); + remove_corrupt_obj(utc, o); + } + goto exit; + } + + o->info.dataPosition += len; + if (o->info.dataPosition > o->info.dataSize) + o->info.dataSize = o->info.dataPosition; + +exit: + return res; +} + +TEE_Result syscall_storage_obj_trunc(unsigned long obj, size_t len) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + size_t off = 0; + size_t attr_size = 0; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + goto exit; + + if (!(o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT)) { + res = TEE_ERROR_BAD_STATE; + goto exit; + } + + if (!(o->info.handleFlags & TEE_DATA_FLAG_ACCESS_WRITE)) { + res = TEE_ERROR_ACCESS_CONFLICT; + goto exit; + } + + res = tee_obj_attr_to_binary(o, NULL, &attr_size); + if (res != TEE_SUCCESS) + goto exit; + + if (ADD_OVERFLOW(sizeof(struct tee_svc_storage_head), attr_size, + &off)) { + res = TEE_ERROR_OVERFLOW; + goto exit; + } + if (ADD_OVERFLOW(len, off, &off)) { + res = TEE_ERROR_OVERFLOW; + goto exit; + } + res = o->pobj->fops->truncate(o->fh, off); + switch (res) { + case TEE_SUCCESS: + o->info.dataSize = len; + break; + case TEE_ERROR_CORRUPT_OBJECT: + EMSG("Object corruption"); + remove_corrupt_obj(to_user_ta_ctx(sess->ctx), o); + break; + default: + res = TEE_ERROR_GENERIC; + break; + } + +exit: + return res; +} + +TEE_Result syscall_storage_obj_seek(unsigned long obj, int32_t offset, + unsigned long whence) +{ + struct ts_session *sess = ts_get_current_session(); + TEE_Result res = TEE_SUCCESS; + struct tee_obj *o = NULL; + tee_fs_off_t new_pos = 0; + + res = tee_obj_get(to_user_ta_ctx(sess->ctx), uref_to_vaddr(obj), &o); + if (res != TEE_SUCCESS) + return res; + + if (!(o->info.handleFlags & TEE_HANDLE_FLAG_PERSISTENT)) + return TEE_ERROR_BAD_STATE; + + switch (whence) { + case TEE_DATA_SEEK_SET: + new_pos = offset; + break; + case TEE_DATA_SEEK_CUR: + if (ADD_OVERFLOW(o->info.dataPosition, offset, &new_pos)) + return TEE_ERROR_OVERFLOW; + break; + case TEE_DATA_SEEK_END: + if (ADD_OVERFLOW(o->info.dataSize, offset, &new_pos)) + return TEE_ERROR_OVERFLOW; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + if (new_pos < 0) + new_pos = 0; + + if (new_pos > TEE_DATA_MAX_POSITION) { + EMSG("Position is beyond TEE_DATA_MAX_POSITION"); + return TEE_ERROR_BAD_PARAMETERS; + } + + o->info.dataPosition = new_pos; + + return TEE_SUCCESS; +} + +void tee_svc_storage_close_all_enum(struct user_ta_ctx *utc) +{ + struct tee_storage_enum_head *eh = &utc->storage_enums; + + /* disregard return value */ + while (!TAILQ_EMPTY(eh)) + tee_svc_close_enum(utc, TAILQ_FIRST(eh)); +} diff --git a/optee_os/core/tee/tee_ta_enc_manager.c b/optee_os/core/tee/tee_ta_enc_manager.c new file mode 100644 index 0000000..2bab308 --- /dev/null +++ b/optee_os/core/tee/tee_ta_enc_manager.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include + +TEE_Result tee_ta_decrypt_init(void **enc_ctx, struct shdr_encrypted_ta *ehdr, + size_t len) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t key[TEE_TA_ENC_KEY_SIZE] = {0}; + + res = crypto_authenc_alloc_ctx(enc_ctx, ehdr->enc_algo); + if (res != TEE_SUCCESS) + return res; + + res = tee_otp_get_ta_enc_key(ehdr->flags & SHDR_ENC_KEY_TYPE_MASK, + key, sizeof(key)); + if (res != TEE_SUCCESS) + goto out_init; + + res = crypto_authenc_init(*enc_ctx, TEE_MODE_DECRYPT, key, sizeof(key), + SHDR_ENC_GET_IV(ehdr), ehdr->iv_size, + ehdr->tag_size, 0, len); + +out_init: + if (res != TEE_SUCCESS) + crypto_authenc_free_ctx(*enc_ctx); + + memzero_explicit(key, sizeof(key)); + return res; +} + +TEE_Result tee_ta_decrypt_update(void *enc_ctx, uint8_t *dst, uint8_t *src, + size_t len) +{ + TEE_Result res = TEE_SUCCESS; + size_t dlen = len; + + res = crypto_authenc_update_payload(enc_ctx, TEE_MODE_DECRYPT, src, len, + dst, &dlen); + if (res != TEE_SUCCESS) + crypto_authenc_free_ctx(enc_ctx); + + return res; +} + +TEE_Result tee_ta_decrypt_final(void *enc_ctx, struct shdr_encrypted_ta *ehdr, + uint8_t *dst, uint8_t *src, size_t len) +{ + TEE_Result res = TEE_SUCCESS; + size_t dlen = len; + + res = crypto_authenc_dec_final(enc_ctx, src, len, dst, &dlen, + SHDR_ENC_GET_TAG(ehdr), ehdr->tag_size); + + crypto_authenc_free_ctx(enc_ctx); + + return res; +} diff --git a/optee_os/core/tee/tee_time_generic.c b/optee_os/core/tee/tee_time_generic.c new file mode 100644 index 0000000..303b044 --- /dev/null +++ b/optee_os/core/tee/tee_time_generic.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include + +struct tee_ta_time_offs { + TEE_UUID uuid; + TEE_Time offs; + bool positive; +}; + +static struct tee_ta_time_offs *tee_time_offs; +static size_t tee_time_num_offs; + +static TEE_Result tee_time_ta_get_offs(const TEE_UUID *uuid, + const TEE_Time **offs, bool *positive) +{ + size_t n; + + for (n = 0; n < tee_time_num_offs; n++) { + if (memcmp(uuid, &tee_time_offs[n].uuid, sizeof(TEE_UUID)) + == 0) { + *offs = &tee_time_offs[n].offs; + *positive = tee_time_offs[n].positive; + return TEE_SUCCESS; + } + } + return TEE_ERROR_TIME_NOT_SET; +} + +static TEE_Result tee_time_ta_set_offs(const TEE_UUID *uuid, + const TEE_Time *offs, bool positive) +{ + size_t n; + struct tee_ta_time_offs *o; + + for (n = 0; n < tee_time_num_offs; n++) { + if (memcmp(uuid, &tee_time_offs[n].uuid, sizeof(TEE_UUID)) + == 0) { + tee_time_offs[n].offs = *offs; + tee_time_offs[n].positive = positive; + return TEE_SUCCESS; + } + } + + n = tee_time_num_offs + 1; + o = realloc(tee_time_offs, n * sizeof(struct tee_ta_time_offs)); + if (!o) + return TEE_ERROR_OUT_OF_MEMORY; + tee_time_offs = o; + tee_time_offs[tee_time_num_offs].uuid = *uuid; + tee_time_offs[tee_time_num_offs].offs = *offs; + tee_time_offs[tee_time_num_offs].positive = positive; + tee_time_num_offs = n; + return TEE_SUCCESS; +} + +TEE_Result tee_time_get_ta_time(const TEE_UUID *uuid, TEE_Time *time) +{ + TEE_Result res; + const TEE_Time *offs; + bool positive; + TEE_Time t; + TEE_Time t2; + + res = tee_time_ta_get_offs(uuid, &offs, &positive); + if (res != TEE_SUCCESS) + return res; + + res = tee_time_get_sys_time(&t); + if (res != TEE_SUCCESS) + return res; + + if (positive) { + TEE_TIME_ADD(t, *offs, t2); + + /* Detect wrapping, the wrapped time should be returned. */ + if (TEE_TIME_LT(t2, t)) + res = TEE_ERROR_OVERFLOW; + } else { + TEE_TIME_SUB(t, *offs, t2); + + /* Detect wrapping, the wrapped time should be returned. */ + if (TEE_TIME_LE(t, t2)) + res = TEE_ERROR_OVERFLOW; + } + *time = t2; + + return res; +} + +TEE_Result tee_time_set_ta_time(const TEE_UUID *uuid, const TEE_Time *time) +{ + TEE_Result res; + TEE_Time offs; + TEE_Time t; + + /* Check that time is normalized. */ + if (time->millis >= TEE_TIME_MILLIS_BASE) + return TEE_ERROR_BAD_PARAMETERS; + + res = tee_time_get_sys_time(&t); + if (res != TEE_SUCCESS) + return res; + + if (TEE_TIME_LT(t, *time)) { + TEE_TIME_SUB(*time, t, offs); + return tee_time_ta_set_offs(uuid, &offs, true); + } else { + TEE_TIME_SUB(t, *time, offs); + return tee_time_ta_set_offs(uuid, &offs, false); + } +} + +void tee_time_busy_wait(uint32_t milliseconds_delay) +{ + TEE_Time curr; + TEE_Time delta; + TEE_Time end; + + if (tee_time_get_sys_time(&curr) != TEE_SUCCESS) + panic(); + delta.seconds = milliseconds_delay / 1000; + delta.millis = milliseconds_delay % 1000; + TEE_TIME_ADD(curr, delta, end); + + while (TEE_TIME_LT(curr, end)) + if (tee_time_get_sys_time(&curr) != TEE_SUCCESS) + panic(); +} diff --git a/optee_os/core/tee/uuid.c b/optee_os/core/tee/uuid.c new file mode 100644 index 0000000..0b24c8d --- /dev/null +++ b/optee_os/core/tee/uuid.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include + +void tee_uuid_to_octets(uint8_t *d, const TEE_UUID *s) +{ + d[0] = s->timeLow >> 24; + d[1] = s->timeLow >> 16; + d[2] = s->timeLow >> 8; + d[3] = s->timeLow; + d[4] = s->timeMid >> 8; + d[5] = s->timeMid; + d[6] = s->timeHiAndVersion >> 8; + d[7] = s->timeHiAndVersion; + memcpy(d + 8, s->clockSeqAndNode, sizeof(s->clockSeqAndNode)); +} + +void tee_uuid_from_octets(TEE_UUID *d, const uint8_t *s) +{ + d->timeLow = SHIFT_U32(s[0], 24) | SHIFT_U32(s[1], 16) | + SHIFT_U32(s[2], 8) | s[3]; + d->timeMid = SHIFT_U32(s[4], 8) | s[5]; + d->timeHiAndVersion = SHIFT_U32(s[6], 8) | s[7]; + memcpy(d->clockSeqAndNode, s + 8, sizeof(d->clockSeqAndNode)); +} diff --git a/optee_os/core/tests/ftmn_boot_tests.c b/optee_os/core/tests/ftmn_boot_tests.c new file mode 100644 index 0000000..3cd1236 --- /dev/null +++ b/optee_os/core/tests/ftmn_boot_tests.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +/* + * Simple straightforward tests. + */ +static TEE_Result simple_call_func_res; + +static TEE_Result __noinline simple_call_func1(void) +{ + TEE_Result res = simple_call_func_res; + + FTMN_CALLEE_DONE(res); + return res; +} + +static TEE_Result __noinline simple_call_memcmp(const void *s1, const void *s2, + size_t n) +{ + if (!FTMN_CALLEE_DONE_MEMCMP(memcmp, s1, s2, n)) + return TEE_SUCCESS; + return TEE_ERROR_GENERIC; +} + +static void __noinline simple_call(void) +{ + TEE_Result res = TEE_SUCCESS; + struct ftmn ftmn = { }; + static const char s1[] = "s1"; + + simple_call_func_res = TEE_SUCCESS; + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, simple_call_func1); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(1), res); + + simple_call_func_res = TEE_ERROR_GENERIC; + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, simple_call_func1); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(2, 1), res); + + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + simple_call_memcmp, s1, s1, sizeof(s1)); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(3, 2), res); +} + +/* + * Simulate calling with multiple unmitigated functions in the chain + * between checked callee and the caller. The result has always been set + * regardless of return value. + */ + +static TEE_Result __noinline two_level_call_memcmp2(const void *s1, + const void *s2, size_t n) +{ + if (!FTMN_CALLEE_DONE_MEMCMP(memcmp, s1, s2, n)) + return TEE_SUCCESS; + /* + * If FTMN_CALLEE_DONE_MEMCMP() returned non-zero the strings are + * different. Update with an error code we can understand. + */ + FTMN_CALLEE_UPDATE_NOT_ZERO(TEE_ERROR_GENERIC); + return TEE_ERROR_GENERIC; +} + +static TEE_Result __noinline two_level_call_memcmp1(const void *s1, + const void *s2, size_t n) +{ + return two_level_call_memcmp2(s1, s2, n); +} + +static TEE_Result __noinline two_level_call_memcmp(const void *s1, + const void *s2, size_t n) +{ + unsigned long func_hash = FTMN_FUNC_HASH("two_level_call_memcmp2"); + struct ftmn ftmn = { }; + TEE_Result res = TEE_SUCCESS; + + FTMN_PUSH_LINKED_CALL(&ftmn, func_hash); + res = two_level_call_memcmp1(s1, s2, n); + FTMN_SET_CHECK_RES_FROM_CALL(&ftmn, 0, res); + FTMN_POP_LINKED_CALL(&ftmn); + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR1, 0, res); + + return res; +} + +static void __noinline two_level_call(void) +{ + struct ftmn ftmn = { }; + TEE_Result res = TEE_SUCCESS; + static const char s1[] = "s1"; + static const char s2[] = "s2"; + + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + two_level_call_memcmp, s1, s1, sizeof(s1)); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(1), res); + + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + two_level_call_memcmp, s1, s2, sizeof(s1)); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(2, 1), res); +} + +/* + * Simulate chained calls in several levels. + * + * For instance ree_fs_ta_open() -> shdr_verify_signature() -> + * crypto_acipher_rsassa_verify() -> ... -> + * mbedtls_rsa_rsassa_pss_verify_ext() + */ + +static TEE_Result __noinline chained_call_memcmp2(const void *s1, + const void *s2, size_t n) +{ + if (!FTMN_CALLEE_DONE_MEMCMP(memcmp, s1, s2, n)) + return TEE_SUCCESS; + return TEE_ERROR_GENERIC; +} + +static TEE_Result __noinline chained_call_memcmp1(const void *s1, + const void *s2, size_t n) +{ + TEE_Result res = chained_call_memcmp2(s1, s2, n); + + /* + * If s1 and s2 has the same content but different pointers we're + * testing the case with an error detected after the linked leaf + * function has been called. + */ + if (!res && s1 != s2) + res = TEE_ERROR_BAD_STATE; + + return res; +} + +static TEE_Result __noinline chained_call_memcmp(const void *s1, + const void *s2, size_t n) +{ + struct ftmn ftmn = { }; + TEE_Result res = TEE_SUCCESS; + + FTMN_PUSH_LINKED_CALL(&ftmn, FTMN_FUNC_HASH("chained_call_memcmp2")); + + res = chained_call_memcmp1(s1, s2, n); + + if (!res) + FTMN_SET_CHECK_RES_FROM_CALL(&ftmn, FTMN_INCR0, res); + else + FTMN_SET_CHECK_RES(&ftmn, FTMN_INCR0, res); + FTMN_POP_LINKED_CALL(&ftmn); + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR0, FTMN_STEP_COUNT(1), res); + + return res; +} + +static void __noinline chained_calls(void) +{ + struct ftmn ftmn = { }; + static const char s[] = "s1s2s1"; + TEE_Result res = TEE_SUCCESS; + + /* Test a normal success case. */ + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, chained_call_memcmp, s, s, 2); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(1), res); + + /* Test the case where the leaf function detects an error. */ + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + chained_call_memcmp, s, s + 2, 2); + assert(res == TEE_ERROR_GENERIC); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(2, 1), + TEE_ERROR_GENERIC); + + /* + * Test the case where a function in the call chain detects an error + * after a the leaf function has returned success. + */ + FTMN_CALL_FUNC(res, &ftmn, FTMN_INCR0, + chained_call_memcmp, s, s + 4, 2); + assert(res == TEE_ERROR_BAD_STATE); + ftmn_expect_state(&ftmn, FTMN_INCR1, FTMN_STEP_COUNT(3, 2), + TEE_ERROR_BAD_STATE); +} + +#define CALL_TEST_FUNC(x) do { \ + DMSG("Calling " #x "()"); \ + x(); \ + DMSG("Return from " #x "()"); \ + } while (0) + +static TEE_Result ftmn_boot_tests(void) +{ + CALL_TEST_FUNC(simple_call); + CALL_TEST_FUNC(two_level_call); + CALL_TEST_FUNC(chained_calls); + + DMSG("*************************************************"); + DMSG("************** Tests complete *****************"); + DMSG("*************************************************"); + return TEE_SUCCESS; +} + +driver_init_late(ftmn_boot_tests); diff --git a/optee_os/core/tests/sub.mk b/optee_os/core/tests/sub.mk new file mode 100644 index 0000000..b0833d5 --- /dev/null +++ b/optee_os/core/tests/sub.mk @@ -0,0 +1 @@ +srcs-y += ftmn_boot_tests.c diff --git a/optee_os/keys/default.pem b/optee_os/keys/default.pem new file mode 100644 index 0000000..a8e5dcd --- /dev/null +++ b/optee_os/keys/default.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAploYrcCzzuMMkCt9QJusu1pJiEIKseCSwIaR5p7voOHxL5D2 +Zt3qFVrAKLSd31IMbqXGGUOsaajyX/5gAKXSOswSAfNbugxRHCz0sHe51nI1HA9z +zS8W+bqJhKCbv4tbf+iLi5/IceocbB8AoO7KPr8YjlebSGbri9RAUa6GoYb40Av6 +ti4lLgX/dgFG1z3rH3z83x6Ci16wyqPqFr4tblRQnLhB7xfHf2BUqvlwoPiYLoL5 +jF/7nIiOd7Aycpka4ARevx2q9/AFPLSV3U26OnGh+HKOHZjjE7rKsCe/h1lUJ/jo +7MaFN9I6leSlCZrQH6OGsJLYAh4zddsmvyf8NQIDAQABAoIBAEoooBfVcFMqpM8g +lBXZUIe1GFLGHj5t/v4U/7Xv5KO0FfE+jpoNE+2xJzj+S+LikOhCvq1Enff8eRCS +NfoT8kSD9QMpmhq2ovIMmIiS1yXihAqeWOF/vJ5XqjaV4tjhJ0k+Qfdco98knscU +ixMzYLXYOvo509/TAe60EHhZdWtMKCmHqvFBRDhYFF+HUxWnJa39G1u828ljv8uN +iRCXbEjoaWDwBJWRzDZJehrkQc2gQAkFpyePVqA3wm1tx1ZHEVwCqYOXOP49/LAc +rQfoWUpm4chjEHnZEb+dQxa3N8Qeq2lqgcqrOX8NIfpzWTWQ1JM+ksinQwMJ0BDV ++Lf1nAECgYEA2KBBtsj41ITez8iA0DpXovH1F9oU9aJ9JKopNos8zIZg012bE5Ib +kFII0LiBGKLUP2PepTsPDIOxibLQDojg1aBGHISZVp/y81B4Rk7Wwg206ZQ848dM +AhACIrzOtJLoVEZTPf7C5QrNMhViyf2c8vFBjfTuM7Ze6a0fsvfJ2ikCgYEAxJaO +XKnOgKoG1Fe9JVM01U+260BEybNUbbXQCnvMjTSZCsOEmST9mhXhC56lhCNqmynt +FxNuq/JMwtS3TRhrYCPPCLFly+JNTqUS1mLQ9ytVbaw6fp9mSxFUQE9d9obkGwJC +3uXEx+bc7YSGuwD34wikX0tB1zzGIXRH2JlB6y0CgYEAgFlDVjdxJwmqslXL7JQE +xKizXPR7oqYpm4WucYDB/zRHPjSh41CYQEQ6sOAArUYcl4pmqmN5oSKOGFX37ASx +oiEaKV9vMuIGIp+1A4NGvyc734FTKmo04Kv3d/mCR38bZINeYscFVou11EzTgXZg +fZjwDFnooH8XcUCAHZuu3zECgYEAt6pltSMcgE7Z9JdIas/TI5vHLQEtgzhdH+bm +SHUjRaWnekvy3VWW7deVGtMMlmi0V+US0iVgl5rXposv+gJfh0HNUrTSg2qZiWlm +MQ+/Zb2BoeZv/A8Wdfnpue+svP+Hq8gwC6mlcJI7RFk1uw8LiXxwfWuFRv9LqMPH +wBBnar0CgYAJF3ukzm7rnut19lzS4r1uEIuuPwk/DjEkkil7U9wE5fr+ds5rCtXj +9591DLHrtE/CBOWym2+tIDr7OnmfU+934524mtLWVZdf39/DFM9Fc1VaSualrj/1 +BOLhBb7MEZqN7hHcQP7+49YW72GgX3pm0Ov5ke33FzN3ubIwAS6T7A== +-----END RSA PRIVATE KEY----- diff --git a/optee_os/keys/default_ta.pem b/optee_os/keys/default_ta.pem new file mode 120000 index 0000000..08aee5e --- /dev/null +++ b/optee_os/keys/default_ta.pem @@ -0,0 +1 @@ +default.pem \ No newline at end of file diff --git a/optee_os/ldelf/dl.c b/optee_os/ldelf/dl.c new file mode 100644 index 0000000..6e737bf --- /dev/null +++ b/optee_os/ldelf/dl.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ +#include +#include + +#include "dl.h" +#include "ta_elf.h" + +TEE_Result dlopen_entry(struct dl_entry_arg *arg) +{ + TEE_UUID zero = { }; + + if (arg->dlopen.flags != (RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!memcmp(&arg->dlopen.uuid, &zero, sizeof(zero))) + return TEE_SUCCESS; + + return ta_elf_add_library(&arg->dlopen.uuid); +} + +TEE_Result dlsym_entry(struct dl_entry_arg *arg) +{ + struct ta_elf *elf = NULL; + TEE_UUID zero = { }; + + if (memcmp(&arg->dlsym.uuid, &zero, sizeof(zero))) { + elf = ta_elf_find_elf(&arg->dlsym.uuid); + if (!elf) + return TEE_ERROR_ITEM_NOT_FOUND; + } + + return ta_elf_resolve_sym(arg->dlsym.symbol, &arg->dlsym.val, NULL, + elf); +} + diff --git a/optee_os/ldelf/dl.h b/optee_os/ldelf/dl.h new file mode 100644 index 0000000..66828a5 --- /dev/null +++ b/optee_os/ldelf/dl.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef LDELF_DL_H +#define LDELF_DL_H + +#include +#include + +TEE_Result dlopen_entry(struct dl_entry_arg *arg); +TEE_Result dlsym_entry(struct dl_entry_arg *arg); + +#endif /*LDELF_DL_H*/ + diff --git a/optee_os/ldelf/ftrace.c b/optee_os/ldelf/ftrace.c new file mode 100644 index 0000000..bab60f7 --- /dev/null +++ b/optee_os/ldelf/ftrace.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ftrace.h" +#include "ta_elf.h" + +#define MIN_FTRACE_BUF_SIZE 1024 +#define MAX_HEADER_STRLEN 128 + +static struct ftrace_buf *fbuf; + +bool ftrace_init(struct ftrace_buf **fbuf_ptr) +{ + struct __ftrace_info *finfo = NULL; + struct ta_elf *elf = TAILQ_FIRST(&main_elf_queue); + TEE_Result res = TEE_SUCCESS; + vaddr_t val = 0; + int count = 0; + size_t fbuf_size = 0; + size_t pad = 0; + char *p = NULL; + char magic[] = { 'F', 'T', 'R', 'A', 'C', 'E', 0x00, 0x01 }; + + res = ta_elf_resolve_sym("__ftrace_info", &val, NULL, NULL); + if (res) + return false; + + finfo = (struct __ftrace_info *)val; + + assert(elf && elf->is_main); + + if (SUB_OVERFLOW(finfo->buf_end.ptr64, finfo->buf_start.ptr64, + &fbuf_size)) + return false; + + if (fbuf_size < MIN_FTRACE_BUF_SIZE) { + DMSG("ftrace buffer too small"); + return false; + } + + fbuf = (struct ftrace_buf *)(vaddr_t)finfo->buf_start.ptr64; + fbuf->head_off = sizeof(struct ftrace_buf); + p = (char *)fbuf + fbuf->head_off; + count = snprintk(p, MAX_HEADER_STRLEN, + "Function graph for TA: %pUl @ %lx\n", + (void *)&elf->uuid, elf->load_addr); + assert(count < MAX_HEADER_STRLEN); + p += count; + + fbuf->ret_func_ptr = finfo->ret_ptr.ptr64; + fbuf->ret_idx = 0; + fbuf->lr_idx = 0; + fbuf->suspend_time = 0; + fbuf->buf_off = fbuf->head_off + count; + /* For proper alignment of uint64_t values in the ftrace buffer */ + pad = 8 - (vaddr_t)p % 8; + if (pad == 8) + pad = 0; + while (pad--) { + *p++ = 0; + fbuf->buf_off++; + count++; + } + /* Delimiter for easier decoding */ + memcpy(p, magic, sizeof(magic)); + fbuf->buf_off += sizeof(magic); + count += sizeof(magic); + fbuf->curr_idx = 0; + fbuf->max_size = fbuf_size - sizeof(struct ftrace_buf) - count; + fbuf->syscall_trace_enabled = false; + fbuf->syscall_trace_suspended = false; + + *fbuf_ptr = fbuf; + + return true; +} + +void ftrace_copy_buf(void *pctx, void (*copy_func)(void *pctx, void *b, + size_t bl)) +{ + if (fbuf) { + struct ta_elf *elf = TAILQ_FIRST(&main_elf_queue); + char *hstart = (char *)fbuf + fbuf->head_off; + char *cstart = (char *)fbuf + fbuf->buf_off; + char *ccurr = cstart + fbuf->curr_idx * sizeof(uint64_t); + size_t csize = 0; + size_t dump_size = 0; + char *end = NULL; + + assert(elf && elf->is_main); + + if (fbuf->overflow) + csize = fbuf->max_size; + else + csize = fbuf->curr_idx * sizeof(uint64_t); + dump_size = fbuf->buf_off - fbuf->head_off + csize; + end = hstart + dump_size; + + /* Header */ + copy_func(pctx, hstart, fbuf->buf_off - fbuf->head_off); + if (fbuf->overflow) { + /* From current index to end of circular buffer */ + copy_func(pctx, ccurr, end - ccurr); + } + /* From start of circular buffer to current index */ + copy_func(pctx, cstart, ccurr - cstart); + } +} + +void ftrace_map_lr(uint64_t *lr) +{ + if (fbuf) { + if (*lr == fbuf->ret_func_ptr && + fbuf->lr_idx < fbuf->ret_idx) { + fbuf->lr_idx++; + *lr = fbuf->ret_stack[fbuf->ret_idx - fbuf->lr_idx]; + } + } +} diff --git a/optee_os/ldelf/ftrace.h b/optee_os/ldelf/ftrace.h new file mode 100644 index 0000000..83dcf4a --- /dev/null +++ b/optee_os/ldelf/ftrace.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#ifndef FTRACE_H +#define FTRACE_H + +#include +#include + +#ifdef CFG_FTRACE_SUPPORT +bool ftrace_init(struct ftrace_buf **fbuf_ptr); +void ftrace_copy_buf(void *pctx, void (*copy_func)(void *pctx, void *b, + size_t bl)); +void ftrace_map_lr(uint64_t *lr); +#else +static inline void ftrace_map_lr(uint64_t *lr __unused) +{ +} +#endif + +#endif /*FTRACE_H*/ + diff --git a/optee_os/ldelf/include/ldelf.h b/optee_os/ldelf/include/ldelf.h new file mode 100644 index 0000000..f638400 --- /dev/null +++ b/optee_os/ldelf/include/ldelf.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020-2023, Arm Limited + * Copyright 2022-2023 NXP + */ + +#ifndef __LDELF_H +#define __LDELF_H + +#ifndef __ASSEMBLER__ +#include +#include +#include + +/* Size of stack for TEE Core to allocate */ +#define LDELF_STACK_SIZE (4096 * 2) + +/* + * struct ldelf_arg - argument for ldelf + * @uuid: [in] UUID of TA to load + * @is_32bit: [out] 1 if a 32bit TA or 0 if a 64bit TA + * @flags: [out] Flags field of TA header + * @entry_func: [out] TA entry function + * @stack_ptr: [out] TA stack pointer + * @dump_entry: [out] Dump TA mappings and stack trace + * @ftrace_entry: [out] Dump TA mappings and ftrace buffer + * @fbuf: [out] ftrace buffer pointer + * @dl_entry: [out] Dynamic linking interface (for libdl) + */ +struct ldelf_arg { + TEE_UUID uuid; + uint32_t is_32bit; + uint32_t flags; + uint64_t entry_func; + uint64_t load_addr; + uint64_t stack_ptr; + uint64_t dump_entry; + uint64_t ftrace_entry; + uint64_t dl_entry; + struct ftrace_buf *fbuf; +}; + +#define DUMP_MAP_READ BIT(0) +#define DUMP_MAP_WRITE BIT(1) +#define DUMP_MAP_EXEC BIT(2) +#define DUMP_MAP_SECURE BIT(3) +#define DUMP_MAP_EPHEM BIT(4) +#define DUMP_MAP_LDELF BIT(7) + +/* + * struct dump_entry_arg - argument for ldelf_arg::dump_entry() + */ +struct dump_entry_arg { + union { + struct { + uint32_t regs[16]; + } arm32; + struct { + uint64_t fp; + uint64_t sp; + uint64_t pc; + } arm64; + struct { + unsigned long fp; + unsigned long sp; + unsigned long pc; + } rv; + }; + bool is_32bit; + size_t num_maps; + struct dump_map { + vaddr_t va; + paddr_t pa; + size_t sz; + uint32_t flags; + } maps[]; +}; + +/* + * struct dl_entry_arg - argument for ldelf_arg::dl_entry() + */ +struct dl_entry_arg { + uint32_t cmd; + TEE_Result ret; + union { + struct { + TEE_UUID uuid; /* in */ + uint32_t flags; /* in */ + } dlopen; + struct { + TEE_UUID uuid; /* in */ + vaddr_t val; /* out */ + char symbol[]; /* in */ + } dlsym; + }; +}; + +/* + * Values for dl_entry_arg::cmd + */ +#define LDELF_DL_ENTRY_DLOPEN 0 +#define LDELF_DL_ENTRY_DLSYM 1 + +/* + * Values for dl_entry_arg::dlopen::flags + */ +#define RTLD_NOW 2 +#define RTLD_GLOBAL 0x100 +#define RTLD_NODELETE 0x1000 + +#define LDELF_MAP_FLAG_SHAREABLE BIT32(0) +#define LDELF_MAP_FLAG_WRITEABLE BIT32(1) +#define LDELF_MAP_FLAG_EXECUTABLE BIT32(2) +#define LDELF_MAP_FLAG_BTI BIT32(3) + +#endif /*!__ASSEMBLER__*/ + +#define LDELF_RETURN 0 +#define LDELF_LOG 1 +#define LDELF_PANIC 2 +#define LDELF_MAP_ZI 3 +#define LDELF_UNMAP 4 +#define LDELF_OPEN_BIN 5 +#define LDELF_CLOSE_BIN 6 +#define LDELF_MAP_BIN 7 +#define LDELF_CP_FROM_BIN 8 +#define LDELF_SET_PROT 9 +#define LDELF_REMAP 10 +#define LDELF_GEN_RND_NUM 11 + +#define LDELF_SCN_MAX 11 + +/* + * ldelf is loaded into memory by TEE Core. BSS is initialized and a + * stack is allocated and supplied in SP register. A struct ldelf_arg + * is placed in the stack and a pointer to the struct is provided in + * r0/x0 on ARM and in a0/x10 on RISC-V. + * + * ldelf relocates itself to the address where it is loaded before the main + * C routine is called. + * + * In the main C routine the TA is loaded using the PTA System interface. + */ + +#endif /*__LDELF_H*/ diff --git a/optee_os/ldelf/include/ldelf_syscalls.h b/optee_os/ldelf/include/ldelf_syscalls.h new file mode 100644 index 0000000..785f163 --- /dev/null +++ b/optee_os/ldelf/include/ldelf_syscalls.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Arm Limited + */ + +#ifndef LDELF_SYSCALLS_H +#define LDELF_SYSCALLS_H + +#include +#include +#include + +void _ldelf_return(unsigned long ret) __noreturn; +void _ldelf_log(const void *buf, size_t len); +void _ldelf_panic(unsigned long code); +TEE_Result _ldelf_map_zi(vaddr_t *va, size_t num_bytes, size_t pad_begin, + size_t pad_end, unsigned long flags); +TEE_Result _ldelf_unmap(vaddr_t va, size_t num_bytes); +TEE_Result _ldelf_open_bin(const TEE_UUID *uuid, size_t uuid_size, + uint32_t *handle); +TEE_Result _ldelf_close_bin(unsigned long handle); +TEE_Result _ldelf_map_bin(vaddr_t *va, size_t num_bytes, unsigned long handle, + size_t offs, size_t pad_begin, size_t pad_end, + unsigned long flags); +TEE_Result _ldelf_cp_from_bin(void *dst, size_t offs, size_t num_bytes, + unsigned long handle); +TEE_Result _ldelf_set_prot(unsigned long va, size_t num_bytes, + unsigned long flags); +TEE_Result _ldelf_remap(unsigned long old_va, vaddr_t *new_va, size_t num_bytes, + size_t pad_begin, size_t pad_end); +TEE_Result _ldelf_gen_rnd_num(void *buf, size_t num_bytes); + +#endif /* LDELF_SYSCALLS_H */ diff --git a/optee_os/ldelf/ldelf.ld.S b/optee_os/ldelf/ldelf.ld.S new file mode 100644 index 0000000..1e34a19 --- /dev/null +++ b/optee_os/ldelf/ldelf.ld.S @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2016-2019, Linaro Limited + */ + +#ifdef ARM32 +OUTPUT_FORMAT("elf32-littlearm") +OUTPUT_ARCH(arm) +#endif +#ifdef ARM64 +OUTPUT_FORMAT("elf64-littleaarch64") +OUTPUT_ARCH(aarch64) +#endif +#ifdef RV64 +OUTPUT_ARCH(riscv) +#endif + +ENTRY(_ldelf_start) +SECTIONS { + .text : { + __text_start = .; + *(.text._ldelf_start) + *(.text .text.*) + *(.stub) + *(.glue_7) + *(.glue_7t) + *(.gnu.linkonce.t.*) + /* Workaround for an erratum in ARM's VFP11 coprocessor */ + *(.vfp11_veneer) + __text_end = .; + } + .note.gnu.property : { *(.note.gnu.property) } + .plt : { *(.plt) } + + .eh_frame : { *(.eh_frame) } + .rodata : { + *(.gnu.linkonce.r.*) + *(.rodata .rodata.*) + } + /* .ARM.exidx is sorted, so has to go in its own output section. */ + .ARM.exidx : { *(.ARM.exidx* .gnu.linkonce.armexidx.*) } + .ARM.extab : { *(.ARM.extab*) } + .ctors : { *(.ctors) } + .dtors : { *(.dtors) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .hash : { *(.hash) } + + /* + * Relocation sections may be aligned on 4 or 8 bytes. With ALIGN(8) + * we avoid any padding between __reloc_begin and the first relocation. + */ + . = ALIGN(8); + __reloc_begin = .; + .rel.got : { *(.rel.got) } + .rela.got : { *(.rela.got) } + .rel.text : { *(.rel.text) *(.rel.gnu.linkonce.t*) } + .rela.text : { *(.rela.text) *(.rela.gnu.linkonce.t*) } + .rel.data : { *(.rel.data) *(.rel.gnu.linkonce.d*) } + .rela.data : { *(.rela.data) *(.rela.gnu.linkonce.d*) } + .rel.rodata : { *(.rel.rodata) *(.rel.gnu.linkonce.r*) } + .rela.rodata : { *(.rela.rodata) *(.rela.gnu.linkonce.r*) } + .rel.dyn : { *(.rel.dyn) } + .rela.dyn : { *(.rela.dyn) } + .rel.ctors : { *(.rel.ctors) } + .rela.ctors : { *(.rela.ctors) } + .rel.dtors : { *(.rel.dtors) } + .rela.dtors : { *(.rela.dtors) } + .rel.init : { *(.rel.init) } + .rela.init : { *(.rela.init) } + .rel.fini : { *(.rel.fini) } + .rela.fini : { *(.rela.fini) } + .rel.bss : { *(.rel.bss) } + .rela.bss : { *(.rela.bss) } + .rel.plt : { *(.rel.plt) } + .rela.plt : { *(.rela.plt) } + __reloc_end = .; + + /* Page align to allow dropping execute bit for RW data */ + . = ALIGN(4096); + + .dynamic : { *(.dynamic) } + .got : { *(.got.plt) *(.got) } + + .data : { *(.data .data.* .gnu.linkonce.d.*) } + .bss : { *(.bss .bss.* .gnu.linkonce.b.* COMMON) } + + /DISCARD/ : { *(.interp) } +} diff --git a/optee_os/ldelf/ldelf.mk b/optee_os/ldelf/ldelf.mk new file mode 100644 index 0000000..aa7bb57 --- /dev/null +++ b/optee_os/ldelf/ldelf.mk @@ -0,0 +1,58 @@ +include mk/cleanvars.mk + +# Set current submodule (used for module specific flags compile result etc) +sm := ldelf +sm-$(sm) := y + +link-out-dir$(sm) := $(out-dir)/$(sm) + +cppflags$(sm) := $(core-platform-cppflags) +cflags$(sm) := $(core-platform-cflags) -fpie -fvisibility=hidden +aflags$(sm) := $(core-platform-aflags) + +# ldelf is compiled for the same arch or register width as core +ifeq ($(CFG_ARM64_core),y) +CFG_ARM64_$(sm) := y +endif +ifeq ($(CFG_ARM32_core),y) +CFG_ARM32_$(sm) := y +endif +ifeq ($(CFG_RV64_core),y) +CFG_RV64_$(sm) := y +endif +ifeq ($(CFG_RV32_core),y) +CFG_RV32_$(sm) := y +endif +arch-bits-$(sm) := $(arch-bits-core) + +cppflags$(sm) += -include $(conf-file) +cppflags$(sm) += -DTRACE_LEVEL=$(CFG_TEE_CORE_LOG_LEVEL) +cppflags$(sm) += -D__LDELF__ + +# Use same compiler as for core +CROSS_COMPILE_$(sm) := $(CROSS_COMPILE_core) +COMPILER_$(sm) := $(COMPILER_core) +include mk/$(COMPILER_$(sm)).mk + +base-prefix := $(sm)- + +libname = utils +libdir = lib/libutils +include mk/lib.mk + +libname = utee +libdir = lib/libutee +include mk/lib.mk + +libname = unw +libdir = lib/libunw +include mk/lib.mk + +base-prefix := + +subdirs = ldelf +include mk/subdir.mk + +include mk/compile.mk + +include ldelf/link.mk diff --git a/optee_os/ldelf/link.mk b/optee_os/ldelf/link.mk new file mode 100644 index 0000000..bd49551 --- /dev/null +++ b/optee_os/ldelf/link.mk @@ -0,0 +1,59 @@ +link-script$(sm) = ldelf/ldelf.ld.S +link-script-pp$(sm) = $(link-out-dir$(sm))/ldelf.lds +link-script-dep$(sm) = $(link-out-dir$(sm))/.ldelf.ld.d + +.PHONY: ldelf +ldelf: $(link-out-dir$(sm))/ldelf.dmp +ldelf: $(link-out-dir$(sm))/ldelf.elf +all: ldelf + +cleanfiles += $(link-out-dir$(sm))/ldelf.dmp +cleanfiles += $(link-out-dir$(sm))/ldelf.map +cleanfiles += $(link-out-dir$(sm))/ldelf.elf +cleanfiles += $(link-script-pp$(sm)) $(link-script-dep$(sm)) + +link-ldflags = -pie -static --gc-sections +link-ldflags += -T $(link-script-pp$(sm)) +link-ldflags += -Map=$(link-out-dir$(sm))/ldelf.map +link-ldflags += --sort-section=alignment +link-ldflags += -z max-page-size=4096 # OP-TEE always uses 4K alignment +ifeq ($(CFG_CORE_BTI),y) +link-ldflags += $(call ld-option,-z force-bti) --fatal-warnings +endif +ifeq ($(CFG_ARM32_$(sm)), y) +link-ldflags += $(call ld-option,--no-warn-execstack) +endif +link-ldflags += $(link-ldflags$(sm)) + +link-ldadd = $(addprefix -L,$(libdirs)) +link-ldadd += --start-group $(addprefix -l,$(libnames)) --end-group +ldargs-ldelf.elf := $(link-ldflags) $(objs) $(link-ldadd) $(libgcc$(sm)) + +link-script-cppflags-$(sm) := \ + $(filter-out $(CPPFLAGS_REMOVE) $(cppflags-remove), \ + $(nostdinc$(sm)) $(CPPFLAGS) \ + $(addprefix -I,$(incdirs$(sm)) $(link-out-dir$(sm))) \ + $(cppflags$(sm))) + +-include $(link-script-dep$(sm)) + +link-script-pp-makefiles$(sm) = $(filter-out %.d %.cmd,$(MAKEFILE_LIST)) + +define gen-link-t +$(link-script-pp$(sm)): $(link-script$(sm)) $(conf-file) \ + $(link-script-pp-makefiles$(sm)) + @$(cmd-echo-silent) ' CPP $$@' + $(q)mkdir -p $$(dir $$@) + $(q)$(CPP$(sm)) -P -MT $$@ -MD -MF $(link-script-dep$(sm)) \ + $(link-script-cppflags-$(sm)) $$< -o $$@ + +$(link-out-dir$(sm))/ldelf.elf: $(objs) $(libdeps) $(link-script-pp$(sm)) + @$(cmd-echo-silent) ' LD $$@' + $(q)$(LD$(sm)) $(ldargs-ldelf.elf) -o $$@ + +$(link-out-dir$(sm))/ldelf.dmp: $(link-out-dir$(sm))/ldelf.elf + @$(cmd-echo-silent) ' OBJDUMP $$@' + $(q)$(OBJDUMP$(sm)) -l -x -d $$< > $$@ +endef + +$(eval $(call gen-link-t)) diff --git a/optee_os/ldelf/main.c b/optee_os/ldelf/main.c new file mode 100644 index 0000000..032b328 --- /dev/null +++ b/optee_os/ldelf/main.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2022-2023, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dl.h" +#include "ftrace.h" +#include "sys.h" +#include "ta_elf.h" + +static size_t mpool_size = 4 * SMALL_PAGE_SIZE; +static vaddr_t mpool_base; + +static void __printf(2, 0) print_to_console(void *pctx __unused, + const char *fmt, va_list ap) +{ + trace_vprintf(NULL, 0, TRACE_ERROR, true, fmt, ap); +} + +static void __noreturn __maybe_unused dump_ta_state(struct dump_entry_arg *arg) +{ + struct ta_elf *elf = TAILQ_FIRST(&main_elf_queue); + + assert(elf && elf->is_main); + EMSG_RAW("Status of TA %pUl", (void *)&elf->uuid); +#if defined(ARM32) || defined(ARM64) + EMSG_RAW(" arch: %s", elf->is_32bit ? "arm" : "aarch64"); +#elif defined(RV32) || defined(RV64) + EMSG_RAW(" arch: %s", elf->is_32bit ? "riscv32" : "riscv64"); +#endif + + ta_elf_print_mappings(NULL, print_to_console, &main_elf_queue, + arg->num_maps, arg->maps, mpool_base); + +#if defined(ARM32) || defined(ARM64) + if (arg->is_32bit) + ta_elf_stack_trace_a32(arg->arm32.regs); + else + ta_elf_stack_trace_a64(arg->arm64.fp, arg->arm64.sp, + arg->arm64.pc); +#elif defined(RV32) || defined(RV64) + ta_elf_stack_trace_riscv(arg->rv.fp, arg->rv.pc); +#endif + + sys_return_cleanup(); +} + +#ifdef CFG_FTRACE_SUPPORT +struct print_buf_ctx { + char *buf; + size_t blen; + size_t ret; +}; + +static void __printf(2, 0) print_to_pbuf(void *pctx, const char *fmt, + va_list ap) +{ + struct print_buf_ctx *pbuf = pctx; + char *buf = NULL; + size_t blen = 0; + int ret = 0; + + if (pbuf->buf && pbuf->blen > pbuf->ret) { + buf = pbuf->buf + pbuf->ret; + blen = pbuf->blen - pbuf->ret; + } + + ret = vsnprintk(buf, blen, fmt, ap); + assert(ret >= 0); + + pbuf->ret += ret; +} + +static void copy_to_pbuf(void *pctx, void *b, size_t bl) +{ + struct print_buf_ctx *pbuf = pctx; + char *buf = NULL; + size_t blen = 0; + + if (pbuf->buf && pbuf->blen > pbuf->ret) { + buf = pbuf->buf + pbuf->ret; + blen = pbuf->blen - pbuf->ret; + memcpy(buf, b, MIN(blen, bl)); + } + + pbuf->ret += bl; + +} + +static void __noreturn ftrace_dump(void *buf, size_t *blen) +{ + struct print_buf_ctx pbuf = { .buf = buf, .blen = *blen }; + + ta_elf_print_mappings(&pbuf, print_to_pbuf, &main_elf_queue, + 0, NULL, mpool_base); + ftrace_copy_buf(&pbuf, copy_to_pbuf); + *blen = pbuf.ret; + sys_return_cleanup(); +} +#endif + +static void __noreturn dl_entry(struct dl_entry_arg *arg) +{ + switch (arg->cmd) { + case LDELF_DL_ENTRY_DLOPEN: + arg->ret = dlopen_entry(arg); + break; + case LDELF_DL_ENTRY_DLSYM: + arg->ret = dlsym_entry(arg); + break; + default: + arg->ret = TEE_ERROR_NOT_SUPPORTED; + } + + sys_return_cleanup(); +} + +/* + * ldelf()- Loads ELF into memory + * @arg: Argument passing to/from TEE Core + * + * Only called from assembly + */ +void __noreturn ldelf(struct ldelf_arg *arg); +void ldelf(struct ldelf_arg *arg) +{ + TEE_Result res = TEE_SUCCESS; + struct ta_elf *elf = NULL; + + DMSG("Loading TS %pUl", (void *)&arg->uuid); + res = sys_map_zi(mpool_size, 0, &mpool_base, 0, 0); + if (res) { + EMSG("sys_map_zi(%zu): result %"PRIx32, mpool_size, res); + panic(); + } + malloc_add_pool((void *)mpool_base, mpool_size); + + /* Load the main binary and get a list of dependencies, if any. */ + ta_elf_load_main(&arg->uuid, &arg->is_32bit, &arg->stack_ptr, + &arg->flags); + + /* + * Load binaries, ta_elf_load() may add external libraries to the + * list, so the loop will end when all the dependencies are + * satisfied. + */ + TAILQ_FOREACH(elf, &main_elf_queue, link) + ta_elf_load_dependency(elf, arg->is_32bit); + + TAILQ_FOREACH(elf, &main_elf_queue, link) { + ta_elf_relocate(elf); + ta_elf_finalize_mappings(elf); + } + + ta_elf_finalize_load_main(&arg->entry_func, &arg->load_addr); + + arg->ftrace_entry = 0; +#ifdef CFG_FTRACE_SUPPORT + if (ftrace_init(&arg->fbuf)) + arg->ftrace_entry = (vaddr_t)(void *)ftrace_dump; +#endif + + TAILQ_FOREACH(elf, &main_elf_queue, link) + DMSG("ELF (%pUl) at %#"PRIxVA, + (void *)&elf->uuid, elf->load_addr); + +#if TRACE_LEVEL >= TRACE_ERROR + arg->dump_entry = (vaddr_t)(void *)dump_ta_state; +#else + arg->dump_entry = 0; +#endif + arg->dl_entry = (vaddr_t)(void *)dl_entry; + + sys_return_cleanup(); +} diff --git a/optee_os/ldelf/pauth.c b/optee_os/ldelf/pauth.c new file mode 100644 index 0000000..ac70b44 --- /dev/null +++ b/optee_os/ldelf/pauth.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +#include +#include +#include "pauth.h" + +void pauth_strip_pac(uint64_t *lr) +{ + const uint64_t va_mask = GENMASK_64(CFG_LPAE_ADDR_SPACE_BITS - 1, 0); + + *lr = *lr & va_mask; +} diff --git a/optee_os/ldelf/pauth.h b/optee_os/ldelf/pauth.h new file mode 100644 index 0000000..02f9b07 --- /dev/null +++ b/optee_os/ldelf/pauth.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Linaro Limited + */ + +#ifndef PAUTH_H +#define PAUTH_H + +#include +#include + +void pauth_strip_pac(uint64_t *lr); + +#endif /*PAUTH_H*/ + diff --git a/optee_os/ldelf/start_a32.S b/optee_os/ldelf/start_a32.S new file mode 100644 index 0000000..70aa31e --- /dev/null +++ b/optee_os/ldelf/start_a32.S @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include +#include + +/* + * _start() - Entry of ldelf + * + * See include/ldelf.h for details on TEE Core interaction. + * + * void start(struct ldelf_arg *arg); + */ +FUNC _ldelf_start , : + /* + * First ldelf needs to be relocated. The binary is compiled to + * contain only a minimal number of R_ARM_RELATIVE relocations in + * read/write memory, leaving read-only and executeble memory + * untouched. + */ + adr r4, reloc_begin_rel + ldr r5, reloc_begin_rel + ldr r6, reloc_end_rel + add r5, r5, r4 + add r6, r6, r4 + cmp r5, r6 + beq 2f + + adr r4, _ldelf_start /* Get the load offset */ + + /* Loop over the relocations (Elf32_Rel) and process all entries */ +1: ldmia r5!, {r7-r8} /* r7 == r_offset, r8 = r_info */ + and r8, r8, #0xff + cmp r8, #R_ARM_RELATIVE + /* We're currently only supporting R_ARM_RELATIVE relocations */ + bne 3f + + /* Update the pointer at r_offset + load_offset */ + add r7, r7, r4 + ldr r8, [r7] + add r8, r8, r4 + str r8, [r7] + + cmp r5, r6 + blo 1b + +2: bl ldelf + mov r0, #0 + bl _ldelf_return +3: mov r0, #0 + bl _ldelf_panic +reloc_begin_rel: + .word __reloc_begin - reloc_begin_rel +reloc_end_rel: + .word __reloc_end - reloc_end_rel +END_FUNC _ldelf_start diff --git a/optee_os/ldelf/start_a64.S b/optee_os/ldelf/start_a64.S new file mode 100644 index 0000000..fae72da --- /dev/null +++ b/optee_os/ldelf/start_a64.S @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include +#include + +/* + * _start() - Entry of ldelf + * + * See include/ldelf.h for details on TEE Core interaction. + * + * void start(struct ldelf_arg *arg); + */ +FUNC _ldelf_start , : + /* + * First ldelf needs to be relocated. The binary is compiled to + * contain only a minimal number of R_AARCH64_RELATIVE relocations + * in read/write memory, leaving read-only and executeble memory + * untouched. + */ + adr x4, reloc_begin_rel + ldr w5, reloc_begin_rel + ldr w6, reloc_end_rel + add x5, x5, x4 + add x6, x6, x4 + cmp x5, x6 + beq 2f + + adr x4, _ldelf_start /* Get the load offset */ + + /* Loop over the relocations (Elf64_Rela) and process all entries */ +1: ldp x7, x8, [x5], #16 /* x7 = r_offset, x8 = r_info */ + ldr x9, [x5], #8 /* x9 = r_addend */ + and x8, x8, #0xffffffff + cmp x8, #R_AARCH64_RELATIVE + /* We're currently only supporting R_AARCH64_RELATIVE relocations */ + bne 3f + + /* + * Update the pointer at r_offset + load_offset with r_addend + + * load_offset. + */ + add x7, x7, x4 + add x9, x9, x4 + str x9, [x7] + + cmp x5, x6 + blo 1b + +2: bl ldelf + mov x0, #0 + bl _ldelf_return +3: mov x0, #0 + bl _ldelf_panic +reloc_begin_rel: + .word __reloc_begin - reloc_begin_rel +reloc_end_rel: + .word __reloc_end - reloc_end_rel +END_FUNC _ldelf_start + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/ldelf/start_rv64.S b/optee_os/ldelf/start_rv64.S new file mode 100644 index 0000000..42d36f5 --- /dev/null +++ b/optee_os/ldelf/start_rv64.S @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022 Beijing ESWIN Computing Technology Co., Ltd. + */ + +#include +#include + +/* + * _ldelf_start() - Entry of ldelf + * + * See include/ldelf.h for details on TEE Core interaction. + * + * void _ldelf_start(struct ldelf_arg *arg); + */ +FUNC _ldelf_start , : + /* + * First ldelf needs to be relocated. The binary is compiled to + * contain only a minimal number of R_RISCV_RELATIVE relocations in + * read/write memory, leaving read-only and executable memory + * untouched. + */ + lla a1, reloc_end_rel + lw a3, 0(a1) + lla a1, reloc_begin_rel + lw a2, 0(a1) + add a2, a2, a1 + add a3, a3, a1 + beq a2, a3, 2f + + lla a1, _ldelf_start /* Get the load offset */ + + /* Loop over the relocations (Elf64_Rela) and process all entries */ +1: + ld t1, 0(a2) /* t1 = r_offset */ + ld t2, 8(a2) /* t2 = r_info */ + ld t3, 16(a2) /* t3 = r_addend */ + addi a2, a2, 24 + and t2, t2, 0xff + addi t4, zero, R_RISCV_RELATIVE + bne t2, t4, 3f + + /* Update the pointer at r_offset + load offset */ + add t1, a1, t1 + ld t4, 0(t1) + add t4, t4, t3 + add t4, t4, a1 + sd t4, 0(t1) + ble a2, a3, 1b +2: + jal ldelf + addi a0, a0, 0 + jal _ldelf_return +3: + addi a0, a0, 0 + jal _ldelf_panic +reloc_begin_rel: + .word __reloc_begin - reloc_begin_rel +reloc_end_rel: + .word __reloc_end - reloc_end_rel +END_FUNC _ldelf_start diff --git a/optee_os/ldelf/sub.mk b/optee_os/ldelf/sub.mk new file mode 100644 index 0000000..40c1a0a --- /dev/null +++ b/optee_os/ldelf/sub.mk @@ -0,0 +1,15 @@ +global-incdirs-y += include +srcs-$(CFG_ARM32_$(sm)) += start_a32.S +srcs-$(CFG_ARM64_$(sm)) += start_a64.S +srcs-$(CFG_ARM32_$(sm)) += syscalls_a32.S +srcs-$(CFG_ARM64_$(sm)) += syscalls_a64.S +srcs-$(CFG_ARM64_$(sm)) += tlsdesc_rel_a64.S +srcs-$(CFG_RV64_$(sm)) += start_rv64.S +srcs-$(call cfg-one-enabled,CFG_RV32_$(sm) CFG_RV64_$(sm)) += syscalls_rv.S +srcs-y += dl.c +srcs-y += main.c +srcs-y += sys.c +srcs-y += ta_elf.c +srcs-y += ta_elf_rel.c +srcs-$(CFG_FTRACE_SUPPORT) += ftrace.c +srcs-$(CFG_TA_PAUTH) += pauth.c diff --git a/optee_os/ldelf/sys.c b/optee_os/ldelf/sys.c new file mode 100644 index 0000000..2017d93 --- /dev/null +++ b/optee_os/ldelf/sys.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include +#include + +#include "sys.h" + +int trace_level = TRACE_LEVEL; +const char trace_ext_prefix[] = "LD"; + +void __panic(const char *file __maybe_unused, const int line __maybe_unused, + const char *func __maybe_unused) +{ + if (!file && !func) + EMSG_RAW("Panic"); + else + EMSG_RAW("Panic at %s:%d %s%s%s", + file ? file : "?", file ? line : 0, + func ? "<" : "", func ? func : "", func ? ">" : ""); + + _ldelf_panic(1); + /*NOTREACHED*/ + while (true) + ; +} + +void sys_return_cleanup(void) +{ + _ldelf_return(0); + /*NOTREACHED*/ + while (true) + ; +} + +TEE_Result sys_map_zi(size_t num_bytes, uint32_t flags, vaddr_t *va, + size_t pad_begin, size_t pad_end) +{ + return _ldelf_map_zi(va, num_bytes, pad_begin, pad_end, flags); +} + +TEE_Result sys_unmap(vaddr_t va, size_t num_bytes) +{ + return _ldelf_unmap(va, num_bytes); +} + +TEE_Result sys_open_ta_bin(const TEE_UUID *uuid, uint32_t *handle) +{ + return _ldelf_open_bin(uuid, sizeof(TEE_UUID), handle); +} + +TEE_Result sys_close_ta_bin(uint32_t handle) +{ + return _ldelf_close_bin(handle); +} + +TEE_Result sys_map_ta_bin(vaddr_t *va, size_t num_bytes, uint32_t flags, + uint32_t handle, size_t offs, size_t pad_begin, + size_t pad_end) +{ + return _ldelf_map_bin(va, num_bytes, handle, offs, + pad_begin, pad_end, flags); +} + + +TEE_Result sys_copy_from_ta_bin(void *dst, size_t num_bytes, uint32_t handle, + size_t offs) +{ + return _ldelf_cp_from_bin(dst, offs, num_bytes, handle); +} + +TEE_Result sys_set_prot(vaddr_t va, size_t num_bytes, uint32_t flags) +{ + return _ldelf_set_prot(va, num_bytes, flags); +} + +TEE_Result sys_remap(vaddr_t old_va, vaddr_t *new_va, size_t num_bytes, + size_t pad_begin, size_t pad_end) +{ + return _ldelf_remap(old_va, new_va, num_bytes, pad_begin, pad_end); +} + +TEE_Result sys_gen_random_num(void *buf, size_t blen) +{ + return _ldelf_gen_rnd_num(buf, blen); +} diff --git a/optee_os/ldelf/sys.h b/optee_os/ldelf/sys.h new file mode 100644 index 0000000..7051d55 --- /dev/null +++ b/optee_os/ldelf/sys.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#ifndef SYS_H +#define SYS_H + +#include +#include +#include +#include +#include +#include + +#if defined(CFG_TEE_CORE_DEBUG) +#define panic() __panic(__FILE__, __LINE__, __func__) +#else +#define panic() __panic(NULL, 0, NULL) +#endif + +/* A small page is the smallest unit of memory that can be mapped */ +#define SMALL_PAGE_SHIFT 12 +#define SMALL_PAGE_MASK 0x00000fff +#define SMALL_PAGE_SIZE 0x00001000 + +void __noreturn __panic(const char *file, const int line, const char *func); +void __noreturn sys_return_cleanup(void); + +#define err(res, ...) \ + do { \ + trace_printf_helper(TRACE_ERROR, true, __VA_ARGS__); \ + _ldelf_return(res); \ + } while (0) + +TEE_Result sys_map_zi(size_t num_bytes, uint32_t flags, vaddr_t *va, + size_t pad_begin, size_t pad_end); +TEE_Result sys_unmap(vaddr_t va, size_t num_bytes); +TEE_Result sys_open_ta_bin(const TEE_UUID *uuid, uint32_t *handle); +TEE_Result sys_close_ta_bin(uint32_t handle); +TEE_Result sys_map_ta_bin(vaddr_t *va, size_t num_bytes, uint32_t flags, + uint32_t handle, size_t offs, size_t pad_begin, + size_t pad_end); +TEE_Result sys_copy_from_ta_bin(void *dst, size_t num_bytes, uint32_t handle, + size_t offs); +TEE_Result sys_set_prot(vaddr_t va, size_t num_bytes, uint32_t flags); +TEE_Result sys_remap(vaddr_t old_va, vaddr_t *new_va, size_t num_bytes, + size_t pad_begin, size_t pad_end); +TEE_Result sys_gen_random_num(void *buf, size_t blen); + +#endif /*SYS_H*/ diff --git a/optee_os/ldelf/syscalls_a32.S b/optee_os/ldelf/syscalls_a32.S new file mode 100644 index 0000000..332fa94 --- /dev/null +++ b/optee_os/ldelf/syscalls_a32.S @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include + + .section .text + .balign 4 + .code 32 + + .macro LDELF_SYSCALL name, scn, num_args + FUNC \name , : + + .if \num_args > 8 + .error "Too many arguments for syscall" + .endif + + push {r5-r7, lr} +UNWIND( .save {r5-r7, lr}) + mov r7, #(\scn) + + .if \num_args <= 4 + @ No arguments passed on stack + mov r6, #0 + .else + @ Tell number of arguments passed on the stack + mov r6, #(\num_args - 4) + @ Point just before the push (4 registers) above on the first argument + add r5, sp, #(4 * 4) + .endif + + svc #0 + pop {r5-r7, pc} + END_FUNC \name + .endm + + FUNC _ldelf_panic, : + push {r0-r11, lr} +UNWIND( .save {r0-r11, lr}) + mov lr, pc + push {lr} +UNWIND( .save {lr}) + mov r1, sp + bl __ldelf_panic + /* Not reached */ + END_FUNC _ldelf_panic + +#include "syscalls_asm.S" diff --git a/optee_os/ldelf/syscalls_a64.S b/optee_os/ldelf/syscalls_a64.S new file mode 100644 index 0000000..dab790c --- /dev/null +++ b/optee_os/ldelf/syscalls_a64.S @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2020, Arm Limited + */ + +#include + + .section .text + + .macro LDELF_SYSCALL name, scn, num_args + FUNC \name , : + + .if \num_args > 8 + .error "Too many arguments for syscall" + .endif + + mov x8, #(\scn) + svc #0 + ret + END_FUNC \name + .endm + + FUNC _ldelf_panic, : + stp x29, x30, [sp, #-16]! + mov x1, sp + bl __ldelf_panic + /* Not reached */ + END_FUNC _ldelf_panic + +#include "syscalls_asm.S" + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/ldelf/syscalls_asm.S b/optee_os/ldelf/syscalls_asm.S new file mode 100644 index 0000000..959ec6a --- /dev/null +++ b/optee_os/ldelf/syscalls_asm.S @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Arm Limited + */ + +#include + +/* macro function name syscall num num of args */ +LDELF_SYSCALL _ldelf_return, LDELF_RETURN, 1 +LDELF_SYSCALL _ldelf_log, LDELF_LOG, 2 +LDELF_SYSCALL __ldelf_panic, LDELF_PANIC, 1 +LDELF_SYSCALL _ldelf_map_zi, LDELF_MAP_ZI, 6 +LDELF_SYSCALL _ldelf_unmap, LDELF_UNMAP, 3 +LDELF_SYSCALL _ldelf_open_bin, LDELF_OPEN_BIN, 3 +LDELF_SYSCALL _ldelf_close_bin, LDELF_CLOSE_BIN, 1 +LDELF_SYSCALL _ldelf_map_bin, LDELF_MAP_BIN, 8 +LDELF_SYSCALL _ldelf_cp_from_bin, LDELF_CP_FROM_BIN, 4 +LDELF_SYSCALL _ldelf_set_prot, LDELF_SET_PROT, 4 +LDELF_SYSCALL _ldelf_remap, LDELF_REMAP, 7 +LDELF_SYSCALL _ldelf_gen_rnd_num, LDELF_GEN_RND_NUM, 2 diff --git a/optee_os/ldelf/syscalls_rv.S b/optee_os/ldelf/syscalls_rv.S new file mode 100644 index 0000000..ba6eb30 --- /dev/null +++ b/optee_os/ldelf/syscalls_rv.S @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 ESWIN Corp. or its affiliates. + * Authors: + * Liu Shiwei + */ + +#include + + .section .text + + .macro LDELF_SYSCALL name, scn, num_args +FUNC \name , : + + .if \num_args > 8 + .error "Too many arguments for syscall" + .endif + + li t0, \scn + li t1, \num_args + ecall + ret +END_FUNC \name + .endm + +FUNC _ldelf_panic, : + j __ldelf_panic + /* Not reached */ +END_FUNC _ldelf_panic + +#include "syscalls_asm.S" diff --git a/optee_os/ldelf/ta_elf.c b/optee_os/ldelf/ta_elf.c new file mode 100644 index 0000000..cf802ac --- /dev/null +++ b/optee_os/ldelf/ta_elf.c @@ -0,0 +1,2005 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020-2023, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sys.h" +#include "ta_elf.h" + +/* + * Layout of a 32-bit struct dl_phdr_info for a 64-bit ldelf to access a 32-bit + * TA + */ +struct dl_phdr_info32 { + uint32_t dlpi_addr; + uint32_t dlpi_name; + uint32_t dlpi_phdr; + uint16_t dlpi_phnum; + uint64_t dlpi_adds; + uint64_t dlpi_subs; + uint32_t dlpi_tls_modid; + uint32_t dlpi_tls_data; +}; + +static vaddr_t ta_stack; +static vaddr_t ta_stack_size; + +struct ta_elf_queue main_elf_queue = TAILQ_HEAD_INITIALIZER(main_elf_queue); + +/* + * Main application is always ID 1, shared libraries with TLS take IDs 2 and + * above + */ +static void assign_tls_mod_id(struct ta_elf *elf) +{ + static size_t last_tls_mod_id = 1; + + if (elf->is_main) + assert(last_tls_mod_id == 1); /* Main always comes first */ + elf->tls_mod_id = last_tls_mod_id++; +} + +static struct ta_elf *queue_elf_helper(const TEE_UUID *uuid) +{ + struct ta_elf *elf = calloc(1, sizeof(*elf)); + + if (!elf) + return NULL; + + TAILQ_INIT(&elf->segs); + + elf->uuid = *uuid; + TAILQ_INSERT_TAIL(&main_elf_queue, elf, link); + return elf; +} + +static struct ta_elf *queue_elf(const TEE_UUID *uuid) +{ + struct ta_elf *elf = ta_elf_find_elf(uuid); + + if (elf) + return NULL; + + elf = queue_elf_helper(uuid); + if (!elf) + err(TEE_ERROR_OUT_OF_MEMORY, "queue_elf_helper"); + + return elf; +} + +struct ta_elf *ta_elf_find_elf(const TEE_UUID *uuid) +{ + struct ta_elf *elf = NULL; + + TAILQ_FOREACH(elf, &main_elf_queue, link) + if (!memcmp(uuid, &elf->uuid, sizeof(*uuid))) + return elf; + + return NULL; +} + +#if defined(ARM32) || defined(ARM64) +static TEE_Result e32_parse_ehdr(struct ta_elf *elf, Elf32_Ehdr *ehdr) +{ + if (ehdr->e_ident[EI_VERSION] != EV_CURRENT || + ehdr->e_ident[EI_CLASS] != ELFCLASS32 || + ehdr->e_ident[EI_DATA] != ELFDATA2LSB || + ehdr->e_ident[EI_OSABI] != ELFOSABI_NONE || + ehdr->e_type != ET_DYN || ehdr->e_machine != EM_ARM || + (ehdr->e_flags & EF_ARM_ABIMASK) != EF_ARM_ABI_VERSION || +#ifndef CFG_WITH_VFP + (ehdr->e_flags & EF_ARM_ABI_FLOAT_HARD) || +#endif + ehdr->e_phentsize != sizeof(Elf32_Phdr) || + ehdr->e_shentsize != sizeof(Elf32_Shdr)) + return TEE_ERROR_BAD_FORMAT; + + elf->is_32bit = true; + elf->e_entry = ehdr->e_entry; + elf->e_phoff = ehdr->e_phoff; + elf->e_shoff = ehdr->e_shoff; + elf->e_phnum = ehdr->e_phnum; + elf->e_shnum = ehdr->e_shnum; + elf->e_phentsize = ehdr->e_phentsize; + elf->e_shentsize = ehdr->e_shentsize; + + return TEE_SUCCESS; +} + +#ifdef ARM64 +static TEE_Result e64_parse_ehdr(struct ta_elf *elf, Elf64_Ehdr *ehdr) +{ + if (ehdr->e_ident[EI_VERSION] != EV_CURRENT || + ehdr->e_ident[EI_CLASS] != ELFCLASS64 || + ehdr->e_ident[EI_DATA] != ELFDATA2LSB || + ehdr->e_ident[EI_OSABI] != ELFOSABI_NONE || + ehdr->e_type != ET_DYN || ehdr->e_machine != EM_AARCH64 || + ehdr->e_flags || ehdr->e_phentsize != sizeof(Elf64_Phdr) || + ehdr->e_shentsize != sizeof(Elf64_Shdr)) + return TEE_ERROR_BAD_FORMAT; + + + elf->is_32bit = false; + elf->e_entry = ehdr->e_entry; + elf->e_phoff = ehdr->e_phoff; + elf->e_shoff = ehdr->e_shoff; + elf->e_phnum = ehdr->e_phnum; + elf->e_shnum = ehdr->e_shnum; + elf->e_phentsize = ehdr->e_phentsize; + elf->e_shentsize = ehdr->e_shentsize; + + return TEE_SUCCESS; +} +#else /*ARM64*/ +static TEE_Result e64_parse_ehdr(struct ta_elf *elf __unused, + Elf64_Ehdr *ehdr __unused) +{ + return TEE_ERROR_NOT_SUPPORTED; +} +#endif /*ARM64*/ +#endif /* ARM32 || ARM64 */ + +#if defined(RV64) +static TEE_Result e32_parse_ehdr(struct ta_elf *elf __unused, + Elf32_Ehdr *ehdr __unused) +{ + return TEE_ERROR_BAD_FORMAT; +} + +static TEE_Result e64_parse_ehdr(struct ta_elf *elf, Elf64_Ehdr *ehdr) +{ + if (ehdr->e_ident[EI_VERSION] != EV_CURRENT || + ehdr->e_ident[EI_CLASS] != ELFCLASS64 || + ehdr->e_ident[EI_DATA] != ELFDATA2LSB || + ehdr->e_ident[EI_OSABI] != ELFOSABI_NONE || + ehdr->e_type != ET_DYN || ehdr->e_machine != EM_RISCV || + ehdr->e_phentsize != sizeof(Elf64_Phdr) || + ehdr->e_shentsize != sizeof(Elf64_Shdr)) + return TEE_ERROR_BAD_FORMAT; + + elf->is_32bit = false; + elf->e_entry = ehdr->e_entry; + elf->e_phoff = ehdr->e_phoff; + elf->e_shoff = ehdr->e_shoff; + elf->e_phnum = ehdr->e_phnum; + elf->e_shnum = ehdr->e_shnum; + elf->e_phentsize = ehdr->e_phentsize; + elf->e_shentsize = ehdr->e_shentsize; + + return TEE_SUCCESS; +} +#endif /* RV64 */ + +static void check_phdr_in_range(struct ta_elf *elf, unsigned int type, + vaddr_t addr, size_t memsz) +{ + vaddr_t max_addr = 0; + + if (ADD_OVERFLOW(addr, memsz, &max_addr)) + err(TEE_ERROR_BAD_FORMAT, "Program header %#x overflow", type); + + /* + * elf->load_addr and elf->max_addr are both using the + * final virtual addresses, while this program header is + * relative to 0. + */ + if (max_addr > elf->max_addr - elf->load_addr) + err(TEE_ERROR_BAD_FORMAT, "Program header %#x out of bounds", + type); +} + +static void read_dyn(struct ta_elf *elf, vaddr_t addr, + size_t idx, unsigned int *tag, size_t *val) +{ + if (elf->is_32bit) { + Elf32_Dyn *dyn = (Elf32_Dyn *)(addr + elf->load_addr); + + *tag = dyn[idx].d_tag; + *val = dyn[idx].d_un.d_val; + } else { + Elf64_Dyn *dyn = (Elf64_Dyn *)(addr + elf->load_addr); + + *tag = dyn[idx].d_tag; + *val = dyn[idx].d_un.d_val; + } +} + +static void check_range(struct ta_elf *elf, const char *name, const void *ptr, + size_t sz) +{ + size_t max_addr = 0; + + if ((vaddr_t)ptr < elf->load_addr) + err(TEE_ERROR_BAD_FORMAT, "%s %p out of range", name, ptr); + + if (ADD_OVERFLOW((vaddr_t)ptr, sz, &max_addr)) + err(TEE_ERROR_BAD_FORMAT, "%s range overflow", name); + + if (max_addr > elf->max_addr) + err(TEE_ERROR_BAD_FORMAT, + "%s %p..%#zx out of range", name, ptr, max_addr); +} + +static void check_hashtab(struct ta_elf *elf, void *ptr, size_t num_buckets, + size_t num_chains) +{ + /* + * Starting from 2 as the first two words are mandatory and hold + * num_buckets and num_chains. So this function is called twice, + * first to see that there's indeed room for num_buckets and + * num_chains and then to see that all of it fits. + * See http://www.sco.com/developers/gabi/latest/ch5.dynamic.html#hash + */ + size_t num_words = 2; + size_t sz = 0; + + if (!IS_ALIGNED_WITH_TYPE(ptr, uint32_t)) + err(TEE_ERROR_BAD_FORMAT, "Bad alignment of DT_HASH %p", ptr); + + if (ADD_OVERFLOW(num_words, num_buckets, &num_words) || + ADD_OVERFLOW(num_words, num_chains, &num_words) || + MUL_OVERFLOW(num_words, sizeof(uint32_t), &sz)) + err(TEE_ERROR_BAD_FORMAT, "DT_HASH overflow"); + + check_range(elf, "DT_HASH", ptr, sz); +} + +static void check_gnu_hashtab(struct ta_elf *elf, void *ptr) +{ + struct gnu_hashtab *h = ptr; + size_t num_words = 4; /* nbuckets, symoffset, bloom_size, bloom_shift */ + size_t bloom_words = 0; + size_t sz = 0; + + if (!IS_ALIGNED_WITH_TYPE(ptr, uint32_t)) + err(TEE_ERROR_BAD_FORMAT, "Bad alignment of DT_GNU_HASH %p", + ptr); + + if (elf->gnu_hashtab_size < sizeof(*h)) + err(TEE_ERROR_BAD_FORMAT, "DT_GNU_HASH too small"); + + /* Check validity of h->nbuckets and h->bloom_size */ + + if (elf->is_32bit) + bloom_words = h->bloom_size; + else + bloom_words = h->bloom_size * 2; + if (ADD_OVERFLOW(num_words, h->nbuckets, &num_words) || + ADD_OVERFLOW(num_words, bloom_words, &num_words) || + MUL_OVERFLOW(num_words, sizeof(uint32_t), &sz) || + sz > elf->gnu_hashtab_size) + err(TEE_ERROR_BAD_FORMAT, "DT_GNU_HASH overflow"); +} + +static void save_hashtab(struct ta_elf *elf) +{ + uint32_t *hashtab = NULL; + size_t n = 0; + + if (elf->is_32bit) { + Elf32_Shdr *shdr = elf->shdr; + + for (n = 0; n < elf->e_shnum; n++) { + void *addr = (void *)(vaddr_t)(shdr[n].sh_addr + + elf->load_addr); + + if (shdr[n].sh_type == SHT_HASH) { + elf->hashtab = addr; + } else if (shdr[n].sh_type == SHT_GNU_HASH) { + elf->gnu_hashtab = addr; + elf->gnu_hashtab_size = shdr[n].sh_size; + } + } + } else { + Elf64_Shdr *shdr = elf->shdr; + + for (n = 0; n < elf->e_shnum; n++) { + void *addr = (void *)(vaddr_t)(shdr[n].sh_addr + + elf->load_addr); + + if (shdr[n].sh_type == SHT_HASH) { + elf->hashtab = addr; + } else if (shdr[n].sh_type == SHT_GNU_HASH) { + elf->gnu_hashtab = addr; + elf->gnu_hashtab_size = shdr[n].sh_size; + } + } + } + + if (elf->hashtab) { + check_hashtab(elf, elf->hashtab, 0, 0); + hashtab = elf->hashtab; + check_hashtab(elf, elf->hashtab, hashtab[0], hashtab[1]); + } + if (elf->gnu_hashtab) + check_gnu_hashtab(elf, elf->gnu_hashtab); +} + +static void save_soname_from_segment(struct ta_elf *elf, unsigned int type, + vaddr_t addr, size_t memsz) +{ + size_t dyn_entsize = 0; + size_t num_dyns = 0; + size_t n = 0; + unsigned int tag = 0; + size_t val = 0; + char *str_tab = NULL; + + if (type != PT_DYNAMIC) + return; + + if (elf->is_32bit) + dyn_entsize = sizeof(Elf32_Dyn); + else + dyn_entsize = sizeof(Elf64_Dyn); + + assert(!(memsz % dyn_entsize)); + num_dyns = memsz / dyn_entsize; + + for (n = 0; n < num_dyns; n++) { + read_dyn(elf, addr, n, &tag, &val); + if (tag == DT_STRTAB) { + str_tab = (char *)(val + elf->load_addr); + break; + } + } + for (n = 0; n < num_dyns; n++) { + read_dyn(elf, addr, n, &tag, &val); + if (tag == DT_SONAME) { + elf->soname = str_tab + val; + break; + } + } +} + +static void save_soname(struct ta_elf *elf) +{ + size_t n = 0; + + if (elf->is_32bit) { + Elf32_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) + save_soname_from_segment(elf, phdr[n].p_type, + phdr[n].p_vaddr, + phdr[n].p_memsz); + } else { + Elf64_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) + save_soname_from_segment(elf, phdr[n].p_type, + phdr[n].p_vaddr, + phdr[n].p_memsz); + } +} + +static void e32_save_symtab(struct ta_elf *elf, size_t tab_idx) +{ + Elf32_Shdr *shdr = elf->shdr; + size_t str_idx = shdr[tab_idx].sh_link; + + elf->dynsymtab = (void *)(shdr[tab_idx].sh_addr + elf->load_addr); + if (!IS_ALIGNED_WITH_TYPE(elf->dynsymtab, Elf32_Sym)) + err(TEE_ERROR_BAD_FORMAT, "Bad alignment of dynsymtab %p", + elf->dynsymtab); + check_range(elf, "Dynsymtab", elf->dynsymtab, shdr[tab_idx].sh_size); + + if (shdr[tab_idx].sh_size % sizeof(Elf32_Sym)) + err(TEE_ERROR_BAD_FORMAT, + "Size of dynsymtab not an even multiple of Elf32_Sym"); + elf->num_dynsyms = shdr[tab_idx].sh_size / sizeof(Elf32_Sym); + + if (str_idx >= elf->e_shnum) + err(TEE_ERROR_BAD_FORMAT, "Dynstr section index out of range"); + elf->dynstr = (void *)(shdr[str_idx].sh_addr + elf->load_addr); + check_range(elf, "Dynstr", elf->dynstr, shdr[str_idx].sh_size); + + elf->dynstr_size = shdr[str_idx].sh_size; +} + +static void e64_save_symtab(struct ta_elf *elf, size_t tab_idx) +{ + Elf64_Shdr *shdr = elf->shdr; + size_t str_idx = shdr[tab_idx].sh_link; + + elf->dynsymtab = (void *)(vaddr_t)(shdr[tab_idx].sh_addr + + elf->load_addr); + + if (!IS_ALIGNED_WITH_TYPE(elf->dynsymtab, Elf64_Sym)) + err(TEE_ERROR_BAD_FORMAT, "Bad alignment of .dynsym/DYNSYM %p", + elf->dynsymtab); + check_range(elf, ".dynsym/DYNSYM", elf->dynsymtab, + shdr[tab_idx].sh_size); + + if (shdr[tab_idx].sh_size % sizeof(Elf64_Sym)) + err(TEE_ERROR_BAD_FORMAT, + "Size of .dynsym/DYNSYM not an even multiple of Elf64_Sym"); + elf->num_dynsyms = shdr[tab_idx].sh_size / sizeof(Elf64_Sym); + + if (str_idx >= elf->e_shnum) + err(TEE_ERROR_BAD_FORMAT, + ".dynstr/STRTAB section index out of range"); + elf->dynstr = (void *)(vaddr_t)(shdr[str_idx].sh_addr + elf->load_addr); + check_range(elf, ".dynstr/STRTAB", elf->dynstr, shdr[str_idx].sh_size); + + elf->dynstr_size = shdr[str_idx].sh_size; +} + +static void save_symtab(struct ta_elf *elf) +{ + size_t n = 0; + + if (elf->is_32bit) { + Elf32_Shdr *shdr = elf->shdr; + + for (n = 0; n < elf->e_shnum; n++) { + if (shdr[n].sh_type == SHT_DYNSYM) { + e32_save_symtab(elf, n); + break; + } + } + } else { + Elf64_Shdr *shdr = elf->shdr; + + for (n = 0; n < elf->e_shnum; n++) { + if (shdr[n].sh_type == SHT_DYNSYM) { + e64_save_symtab(elf, n); + break; + } + } + + } + + save_hashtab(elf); + save_soname(elf); +} + +static void init_elf(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + vaddr_t va = 0; + uint32_t flags = LDELF_MAP_FLAG_SHAREABLE; + size_t sz = 0; + + res = sys_open_ta_bin(&elf->uuid, &elf->handle); + if (res) + err(res, "sys_open_ta_bin(%pUl)", (void *)&elf->uuid); + + /* + * Map it read-only executable when we're loading a library where + * the ELF header is included in a load segment. + */ + if (!elf->is_main) + flags |= LDELF_MAP_FLAG_EXECUTABLE; + res = sys_map_ta_bin(&va, SMALL_PAGE_SIZE, flags, elf->handle, 0, 0, 0); + if (res) + err(res, "sys_map_ta_bin"); + elf->ehdr_addr = va; + if (!elf->is_main) { + elf->load_addr = va; + elf->max_addr = va + SMALL_PAGE_SIZE; + elf->max_offs = SMALL_PAGE_SIZE; + } + + if (!IS_ELF(*(Elf32_Ehdr *)va)) + err(TEE_ERROR_BAD_FORMAT, "TA is not an ELF"); + + res = e32_parse_ehdr(elf, (void *)va); + if (res == TEE_ERROR_BAD_FORMAT) + res = e64_parse_ehdr(elf, (void *)va); + if (res) + err(res, "Cannot parse ELF"); + + if (MUL_OVERFLOW(elf->e_phnum, elf->e_phentsize, &sz) || + ADD_OVERFLOW(sz, elf->e_phoff, &sz)) + err(TEE_ERROR_BAD_FORMAT, "Program headers size overflow"); + + if (sz > SMALL_PAGE_SIZE) + err(TEE_ERROR_NOT_SUPPORTED, "Cannot read program headers"); + + elf->phdr = (void *)(va + elf->e_phoff); +} + +static size_t roundup(size_t v) +{ + return ROUNDUP(v, SMALL_PAGE_SIZE); +} + +static size_t rounddown(size_t v) +{ + return ROUNDDOWN(v, SMALL_PAGE_SIZE); +} + +static void add_segment(struct ta_elf *elf, size_t offset, size_t vaddr, + size_t filesz, size_t memsz, size_t flags, size_t align) +{ + struct segment *seg = calloc(1, sizeof(*seg)); + + if (!seg) + err(TEE_ERROR_OUT_OF_MEMORY, "calloc"); + + if (memsz < filesz) + err(TEE_ERROR_BAD_FORMAT, "Memsz smaller than filesz"); + + seg->offset = offset; + seg->vaddr = vaddr; + seg->filesz = filesz; + seg->memsz = memsz; + seg->flags = flags; + seg->align = align; + + TAILQ_INSERT_TAIL(&elf->segs, seg, link); +} + +static void parse_load_segments(struct ta_elf *elf) +{ + size_t n = 0; + + if (elf->is_32bit) { + Elf32_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) + if (phdr[n].p_type == PT_LOAD) { + add_segment(elf, phdr[n].p_offset, + phdr[n].p_vaddr, phdr[n].p_filesz, + phdr[n].p_memsz, phdr[n].p_flags, + phdr[n].p_align); + } else if (phdr[n].p_type == PT_ARM_EXIDX) { + elf->exidx_start = phdr[n].p_vaddr; + elf->exidx_size = phdr[n].p_filesz; + } else if (phdr[n].p_type == PT_TLS) { + assign_tls_mod_id(elf); + } + } else { + Elf64_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) + if (phdr[n].p_type == PT_LOAD) { + add_segment(elf, phdr[n].p_offset, + phdr[n].p_vaddr, phdr[n].p_filesz, + phdr[n].p_memsz, phdr[n].p_flags, + phdr[n].p_align); + } else if (phdr[n].p_type == PT_TLS) { + elf->tls_start = phdr[n].p_vaddr; + elf->tls_filesz = phdr[n].p_filesz; + elf->tls_memsz = phdr[n].p_memsz; + } else if (IS_ENABLED(CFG_TA_BTI) && + phdr[n].p_type == PT_GNU_PROPERTY) { + elf->prop_start = phdr[n].p_vaddr; + elf->prop_align = phdr[n].p_align; + elf->prop_memsz = phdr[n].p_memsz; + } + } +} + +static void copy_remapped_to(struct ta_elf *elf, const struct segment *seg) +{ + uint8_t *dst = (void *)(seg->vaddr + elf->load_addr); + size_t n = 0; + size_t offs = seg->offset; + size_t num_bytes = seg->filesz; + + if (offs < elf->max_offs) { + n = MIN(elf->max_offs - offs, num_bytes); + memcpy(dst, (void *)(elf->max_addr + offs - elf->max_offs), n); + dst += n; + offs += n; + num_bytes -= n; + } + + if (num_bytes) { + TEE_Result res = sys_copy_from_ta_bin(dst, num_bytes, + elf->handle, offs); + + if (res) + err(res, "sys_copy_from_ta_bin"); + elf->max_offs += offs; + } +} + +static void adjust_segments(struct ta_elf *elf) +{ + struct segment *seg = NULL; + struct segment *prev_seg = NULL; + size_t prev_end_addr = 0; + size_t align = 0; + size_t mask = 0; + + /* Sanity check */ + TAILQ_FOREACH(seg, &elf->segs, link) { + size_t dummy __maybe_unused = 0; + + assert(seg->align >= SMALL_PAGE_SIZE); + assert(!ADD_OVERFLOW(seg->vaddr, seg->memsz, &dummy)); + assert(seg->filesz <= seg->memsz); + assert((seg->offset & SMALL_PAGE_MASK) == + (seg->vaddr & SMALL_PAGE_MASK)); + + prev_seg = TAILQ_PREV(seg, segment_head, link); + if (prev_seg) { + assert(seg->vaddr >= prev_seg->vaddr + prev_seg->memsz); + assert(seg->offset >= + prev_seg->offset + prev_seg->filesz); + } + if (!align) + align = seg->align; + assert(align == seg->align); + } + + mask = align - 1; + + seg = TAILQ_FIRST(&elf->segs); + if (seg) + seg = TAILQ_NEXT(seg, link); + while (seg) { + prev_seg = TAILQ_PREV(seg, segment_head, link); + prev_end_addr = prev_seg->vaddr + prev_seg->memsz; + + /* + * This segment may overlap with the last "page" in the + * previous segment in two different ways: + * 1. Virtual address (and offset) overlaps => + * Permissions needs to be merged. The offset must have + * the SMALL_PAGE_MASK bits set as vaddr and offset must + * add up with prevsion segment. + * + * 2. Only offset overlaps => + * The same page in the ELF is mapped at two different + * virtual addresses. As a limitation this segment must + * be mapped as writeable. + */ + + /* Case 1. */ + if (rounddown(seg->vaddr) < prev_end_addr) { + assert((seg->vaddr & mask) == (seg->offset & mask)); + assert(prev_seg->memsz == prev_seg->filesz); + + /* + * Merge the segments and their permissions. + * Note that the may be a small hole between the + * two sections. + */ + prev_seg->filesz = seg->vaddr + seg->filesz - + prev_seg->vaddr; + prev_seg->memsz = seg->vaddr + seg->memsz - + prev_seg->vaddr; + prev_seg->flags |= seg->flags; + + TAILQ_REMOVE(&elf->segs, seg, link); + free(seg); + seg = TAILQ_NEXT(prev_seg, link); + continue; + } + + /* Case 2. */ + if ((seg->offset & mask) && + rounddown(seg->offset) < + (prev_seg->offset + prev_seg->filesz)) { + + assert(seg->flags & PF_W); + seg->remapped_writeable = true; + } + + /* + * No overlap, but we may need to align address, offset and + * size. + */ + seg->filesz += seg->vaddr - rounddown(seg->vaddr); + seg->memsz += seg->vaddr - rounddown(seg->vaddr); + seg->vaddr = rounddown(seg->vaddr); + seg->offset = rounddown(seg->offset); + seg = TAILQ_NEXT(seg, link); + } + +} + +static void populate_segments_legacy(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + struct segment *seg = NULL; + vaddr_t va = 0; + + assert(elf->is_legacy); + TAILQ_FOREACH(seg, &elf->segs, link) { + struct segment *last_seg = TAILQ_LAST(&elf->segs, segment_head); + size_t pad_end = roundup(last_seg->vaddr + last_seg->memsz - + seg->vaddr - seg->memsz); + size_t num_bytes = roundup(seg->memsz); + + if (!elf->load_addr) + va = 0; + else + va = seg->vaddr + elf->load_addr; + + + if (!(seg->flags & PF_R)) + err(TEE_ERROR_NOT_SUPPORTED, + "Segment must be readable"); + + res = sys_map_zi(num_bytes, 0, &va, 0, pad_end); + if (res) + err(res, "sys_map_zi"); + res = sys_copy_from_ta_bin((void *)va, seg->filesz, + elf->handle, seg->offset); + if (res) + err(res, "sys_copy_from_ta_bin"); + + if (!elf->load_addr) + elf->load_addr = va; + elf->max_addr = va + num_bytes; + elf->max_offs = seg->offset + seg->filesz; + } +} + +static size_t get_pad_begin(void) +{ +#ifdef CFG_TA_ASLR + size_t min = CFG_TA_ASLR_MIN_OFFSET_PAGES; + size_t max = CFG_TA_ASLR_MAX_OFFSET_PAGES; + TEE_Result res = TEE_SUCCESS; + uint32_t rnd32 = 0; + size_t rnd = 0; + + COMPILE_TIME_ASSERT(CFG_TA_ASLR_MIN_OFFSET_PAGES < + CFG_TA_ASLR_MAX_OFFSET_PAGES); + if (max > min) { + res = sys_gen_random_num(&rnd32, sizeof(rnd32)); + if (res) { + DMSG("Random read failed: %#"PRIx32, res); + return min * SMALL_PAGE_SIZE; + } + rnd = rnd32 % (max - min); + } + + return (min + rnd) * SMALL_PAGE_SIZE; +#else /*!CFG_TA_ASLR*/ + return 0; +#endif /*!CFG_TA_ASLR*/ +} + +static void populate_segments(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + struct segment *seg = NULL; + vaddr_t va = 0; + size_t pad_begin = 0; + + assert(!elf->is_legacy); + TAILQ_FOREACH(seg, &elf->segs, link) { + struct segment *last_seg = TAILQ_LAST(&elf->segs, segment_head); + size_t pad_end = roundup(last_seg->vaddr + last_seg->memsz - + seg->vaddr - seg->memsz); + + if (seg->remapped_writeable) { + size_t num_bytes = roundup(seg->vaddr + seg->memsz) - + rounddown(seg->vaddr); + + assert(elf->load_addr); + va = rounddown(elf->load_addr + seg->vaddr); + assert(va >= elf->max_addr); + res = sys_map_zi(num_bytes, 0, &va, 0, pad_end); + if (res) + err(res, "sys_map_zi"); + + copy_remapped_to(elf, seg); + elf->max_addr = va + num_bytes; + } else { + uint32_t flags = 0; + size_t filesz = seg->filesz; + size_t memsz = seg->memsz; + size_t offset = seg->offset; + size_t vaddr = seg->vaddr; + + if (offset < elf->max_offs) { + /* + * We're in a load segment which overlaps + * with (or is covered by) the first page + * of a shared library. + */ + if (vaddr + filesz < SMALL_PAGE_SIZE) { + size_t num_bytes = 0; + + /* + * If this segment is completely + * covered, take next. + */ + if (vaddr + memsz <= SMALL_PAGE_SIZE) + continue; + + /* + * All data of the segment is + * loaded, but we need to zero + * extend it. + */ + va = elf->max_addr; + num_bytes = roundup(vaddr + memsz) - + roundup(vaddr) - + SMALL_PAGE_SIZE; + assert(num_bytes); + res = sys_map_zi(num_bytes, 0, &va, 0, + 0); + if (res) + err(res, "sys_map_zi"); + elf->max_addr = roundup(va + num_bytes); + continue; + } + + /* Partial overlap, remove the first page. */ + vaddr += SMALL_PAGE_SIZE; + filesz -= SMALL_PAGE_SIZE; + memsz -= SMALL_PAGE_SIZE; + offset += SMALL_PAGE_SIZE; + } + + if (!elf->load_addr) { + va = 0; + pad_begin = get_pad_begin(); + /* + * If mapping with pad_begin fails we'll + * retry without pad_begin, effectively + * disabling ASLR for the current ELF file. + */ + } else { + va = vaddr + elf->load_addr; + pad_begin = 0; + } + + if (seg->flags & PF_W) + flags |= LDELF_MAP_FLAG_WRITEABLE; + else + flags |= LDELF_MAP_FLAG_SHAREABLE; + if (seg->flags & PF_X) + flags |= LDELF_MAP_FLAG_EXECUTABLE; + if (!(seg->flags & PF_R)) + err(TEE_ERROR_NOT_SUPPORTED, + "Segment must be readable"); + if (flags & LDELF_MAP_FLAG_WRITEABLE) { + res = sys_map_zi(memsz, 0, &va, pad_begin, + pad_end); + if (pad_begin && res == TEE_ERROR_OUT_OF_MEMORY) + res = sys_map_zi(memsz, 0, &va, 0, + pad_end); + if (res) + err(res, "sys_map_zi"); + res = sys_copy_from_ta_bin((void *)va, filesz, + elf->handle, offset); + if (res) + err(res, "sys_copy_from_ta_bin"); + } else { + if (filesz != memsz) + err(TEE_ERROR_BAD_FORMAT, + "Filesz and memsz mismatch"); + res = sys_map_ta_bin(&va, filesz, flags, + elf->handle, offset, + pad_begin, pad_end); + if (pad_begin && res == TEE_ERROR_OUT_OF_MEMORY) + res = sys_map_ta_bin(&va, filesz, flags, + elf->handle, + offset, 0, + pad_end); + if (res) + err(res, "sys_map_ta_bin"); + } + + if (!elf->load_addr) + elf->load_addr = va; + elf->max_addr = roundup(va + memsz); + elf->max_offs += filesz; + } + } +} + +static void ta_elf_add_bti(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + struct segment *seg = NULL; + uint32_t flags = LDELF_MAP_FLAG_EXECUTABLE | LDELF_MAP_FLAG_BTI; + + TAILQ_FOREACH(seg, &elf->segs, link) { + vaddr_t va = elf->load_addr + seg->vaddr; + + if (seg->flags & PF_X) { + res = sys_set_prot(va, seg->memsz, flags); + if (res) + err(res, "sys_set_prot"); + } + } +} + +static void parse_property_segment(struct ta_elf *elf) +{ + char *desc = NULL; + size_t align = elf->prop_align; + size_t desc_offset = 0; + size_t prop_offset = 0; + vaddr_t va = 0; + Elf_Note *note = NULL; + char *name = NULL; + + if (!IS_ENABLED(CFG_TA_BTI) || !elf->prop_start) + return; + + check_phdr_in_range(elf, PT_GNU_PROPERTY, elf->prop_start, + elf->prop_memsz); + + va = elf->load_addr + elf->prop_start; + note = (void *)va; + name = (char *)(note + 1); + + if (elf->prop_memsz < sizeof(*note) + sizeof(ELF_NOTE_GNU)) + return; + + if (note->n_type != NT_GNU_PROPERTY_TYPE_0 || + note->n_namesz != sizeof(ELF_NOTE_GNU) || + memcmp(name, ELF_NOTE_GNU, sizeof(ELF_NOTE_GNU)) || + !IS_POWER_OF_TWO(align)) + return; + + desc_offset = ROUNDUP(sizeof(*note) + sizeof(ELF_NOTE_GNU), align); + + if (desc_offset > elf->prop_memsz || + ROUNDUP(desc_offset + note->n_descsz, align) > elf->prop_memsz) + return; + + desc = (char *)(va + desc_offset); + + do { + Elf_Prop *prop = (void *)(desc + prop_offset); + size_t data_offset = prop_offset + sizeof(*prop); + + if (note->n_descsz < data_offset) + return; + + data_offset = confine_array_index(data_offset, note->n_descsz); + + if (prop->pr_type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) { + uint32_t *pr_data = (void *)(desc + data_offset); + + if (note->n_descsz < (data_offset + sizeof(*pr_data)) && + prop->pr_datasz != sizeof(*pr_data)) + return; + + if (*pr_data & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) { + DMSG("BTI Feature present in note property"); + elf->bti_enabled = true; + } + } + + prop_offset += ROUNDUP(sizeof(*prop) + prop->pr_datasz, align); + } while (prop_offset < note->n_descsz); +} + +static void map_segments(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + + parse_load_segments(elf); + adjust_segments(elf); + if (TAILQ_FIRST(&elf->segs)->offset < SMALL_PAGE_SIZE) { + vaddr_t va = 0; + size_t sz = elf->max_addr - elf->load_addr; + struct segment *seg = TAILQ_LAST(&elf->segs, segment_head); + size_t pad_begin = get_pad_begin(); + + /* + * We're loading a library, if not other parts of the code + * need to be updated too. + */ + assert(!elf->is_main); + + /* + * Now that we know how much virtual memory is needed move + * the already mapped part to a location which can + * accommodate us. + */ + res = sys_remap(elf->load_addr, &va, sz, pad_begin, + roundup(seg->vaddr + seg->memsz)); + if (res == TEE_ERROR_OUT_OF_MEMORY) + res = sys_remap(elf->load_addr, &va, sz, 0, + roundup(seg->vaddr + seg->memsz)); + if (res) + err(res, "sys_remap"); + elf->ehdr_addr = va; + elf->load_addr = va; + elf->max_addr = va + sz; + elf->phdr = (void *)(va + elf->e_phoff); + } +} + +static void add_deps_from_segment(struct ta_elf *elf, unsigned int type, + vaddr_t addr, size_t memsz) +{ + size_t dyn_entsize = 0; + size_t num_dyns = 0; + size_t n = 0; + unsigned int tag = 0; + size_t val = 0; + TEE_UUID uuid = { }; + char *str_tab = NULL; + size_t str_tab_sz = 0; + + if (type != PT_DYNAMIC) + return; + + check_phdr_in_range(elf, type, addr, memsz); + + if (elf->is_32bit) + dyn_entsize = sizeof(Elf32_Dyn); + else + dyn_entsize = sizeof(Elf64_Dyn); + + assert(!(memsz % dyn_entsize)); + num_dyns = memsz / dyn_entsize; + + for (n = 0; n < num_dyns && !(str_tab && str_tab_sz); n++) { + read_dyn(elf, addr, n, &tag, &val); + if (tag == DT_STRTAB) + str_tab = (char *)(val + elf->load_addr); + else if (tag == DT_STRSZ) + str_tab_sz = val; + } + check_range(elf, ".dynstr/STRTAB", str_tab, str_tab_sz); + + for (n = 0; n < num_dyns; n++) { + read_dyn(elf, addr, n, &tag, &val); + if (tag != DT_NEEDED) + continue; + if (val >= str_tab_sz) + err(TEE_ERROR_BAD_FORMAT, + "Offset into .dynstr/STRTAB out of range"); + tee_uuid_from_str(&uuid, str_tab + val); + queue_elf(&uuid); + } +} + +static void add_dependencies(struct ta_elf *elf) +{ + size_t n = 0; + + if (elf->is_32bit) { + Elf32_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) + add_deps_from_segment(elf, phdr[n].p_type, + phdr[n].p_vaddr, phdr[n].p_memsz); + } else { + Elf64_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) + add_deps_from_segment(elf, phdr[n].p_type, + phdr[n].p_vaddr, phdr[n].p_memsz); + } +} + +static void copy_section_headers(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + size_t sz = 0; + size_t offs = 0; + + if (MUL_OVERFLOW(elf->e_shnum, elf->e_shentsize, &sz)) + err(TEE_ERROR_BAD_FORMAT, "Section headers size overflow"); + + elf->shdr = malloc(sz); + if (!elf->shdr) + err(TEE_ERROR_OUT_OF_MEMORY, "malloc"); + + /* + * We're assuming that section headers comes after the load segments, + * but if it's a very small dynamically linked library the section + * headers can still end up (partially?) in the first mapped page. + */ + if (elf->e_shoff < SMALL_PAGE_SIZE) { + assert(!elf->is_main); + offs = MIN(SMALL_PAGE_SIZE - elf->e_shoff, sz); + memcpy(elf->shdr, (void *)(elf->load_addr + elf->e_shoff), + offs); + } + + if (offs < sz) { + res = sys_copy_from_ta_bin((uint8_t *)elf->shdr + offs, + sz - offs, elf->handle, + elf->e_shoff + offs); + if (res) + err(res, "sys_copy_from_ta_bin"); + } +} + +static void close_handle(struct ta_elf *elf) +{ + TEE_Result res = sys_close_ta_bin(elf->handle); + + if (res) + err(res, "sys_close_ta_bin"); + elf->handle = -1; +} + +static void clean_elf_load_main(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + + /* + * Clean up from last attempt to load + */ + res = sys_unmap(elf->ehdr_addr, SMALL_PAGE_SIZE); + if (res) + err(res, "sys_unmap"); + + while (!TAILQ_EMPTY(&elf->segs)) { + struct segment *seg = TAILQ_FIRST(&elf->segs); + vaddr_t va = 0; + size_t num_bytes = 0; + + va = rounddown(elf->load_addr + seg->vaddr); + if (seg->remapped_writeable) + num_bytes = roundup(seg->vaddr + seg->memsz) - + rounddown(seg->vaddr); + else + num_bytes = seg->memsz; + + res = sys_unmap(va, num_bytes); + if (res) + err(res, "sys_unmap"); + + TAILQ_REMOVE(&elf->segs, seg, link); + free(seg); + } + + free(elf->shdr); + memset(&elf->is_32bit, 0, + (vaddr_t)&elf->uuid - (vaddr_t)&elf->is_32bit); + + TAILQ_INIT(&elf->segs); +} + +#ifdef ARM64 +/* + * Allocates an offset in the TA's Thread Control Block for the TLS segment of + * the @elf module. + */ +#define TCB_HEAD_SIZE (2 * sizeof(long)) +static void set_tls_offset(struct ta_elf *elf) +{ + static size_t next_offs = TCB_HEAD_SIZE; + + if (!elf->tls_start) + return; + + /* Module has a TLS segment */ + elf->tls_tcb_offs = next_offs; + next_offs += elf->tls_memsz; +} +#else +static void set_tls_offset(struct ta_elf *elf __unused) {} +#endif + +static void load_main(struct ta_elf *elf) +{ + vaddr_t va = 0; + + init_elf(elf); + map_segments(elf); + populate_segments(elf); + add_dependencies(elf); + copy_section_headers(elf); + save_symtab(elf); + close_handle(elf); + set_tls_offset(elf); + parse_property_segment(elf); + if (elf->bti_enabled) + ta_elf_add_bti(elf); + + if (!ta_elf_resolve_sym("ta_head", &va, NULL, elf)) + elf->head = (struct ta_head *)va; + else + elf->head = (struct ta_head *)elf->load_addr; + if (elf->head->depr_entry != UINT64_MAX) { + /* + * Legacy TAs sets their entry point in ta_head. For + * non-legacy TAs the entry point of the ELF is set instead + * and leaving the ta_head entry point set to UINT64_MAX to + * indicate that it's not used. + * + * NB, everything before the commit a73b5878c89d ("Replace + * ta_head.entry with elf entry") is considered legacy TAs + * for ldelf. + * + * Legacy TAs cannot be mapped with shared memory segments + * so restart the mapping if it turned out we're loading a + * legacy TA. + */ + + DMSG("Reloading TA %pUl as legacy TA", (void *)&elf->uuid); + clean_elf_load_main(elf); + elf->is_legacy = true; + init_elf(elf); + map_segments(elf); + populate_segments_legacy(elf); + add_dependencies(elf); + copy_section_headers(elf); + save_symtab(elf); + close_handle(elf); + elf->head = (struct ta_head *)elf->load_addr; + /* + * Check that the TA is still a legacy TA, if it isn't give + * up now since we're likely under attack. + */ + if (elf->head->depr_entry == UINT64_MAX) + err(TEE_ERROR_GENERIC, + "TA %pUl was changed on disk to non-legacy", + (void *)&elf->uuid); + } + +} + +void ta_elf_load_main(const TEE_UUID *uuid, uint32_t *is_32bit, uint64_t *sp, + uint32_t *ta_flags) +{ + struct ta_elf *elf = queue_elf(uuid); + vaddr_t va = 0; + TEE_Result res = TEE_SUCCESS; + + assert(elf); + elf->is_main = true; + + load_main(elf); + + *is_32bit = elf->is_32bit; + res = sys_map_zi(elf->head->stack_size, 0, &va, 0, 0); + if (res) + err(res, "sys_map_zi stack"); + + if (elf->head->flags & ~TA_FLAGS_MASK) + err(TEE_ERROR_BAD_FORMAT, "Invalid TA flags(s) %#"PRIx32, + elf->head->flags & ~TA_FLAGS_MASK); + + *ta_flags = elf->head->flags; + *sp = va + elf->head->stack_size; + ta_stack = va; + ta_stack_size = elf->head->stack_size; +} + +void ta_elf_finalize_load_main(uint64_t *entry, uint64_t *load_addr) +{ + struct ta_elf *elf = TAILQ_FIRST(&main_elf_queue); + TEE_Result res = TEE_SUCCESS; + + assert(elf->is_main); + + res = ta_elf_set_init_fini_info_compat(elf->is_32bit); + if (res) + err(res, "ta_elf_set_init_fini_info_compat"); + res = ta_elf_set_elf_phdr_info(elf->is_32bit); + if (res) + err(res, "ta_elf_set_elf_phdr_info"); + + if (elf->is_legacy) + *entry = elf->head->depr_entry; + else + *entry = elf->e_entry + elf->load_addr; + + *load_addr = elf->load_addr; +} + + +void ta_elf_load_dependency(struct ta_elf *elf, bool is_32bit) +{ + if (elf->is_main) + return; + + init_elf(elf); + if (elf->is_32bit != is_32bit) + err(TEE_ERROR_BAD_FORMAT, "ELF %pUl is %sbit (expected %sbit)", + (void *)&elf->uuid, elf->is_32bit ? "32" : "64", + is_32bit ? "32" : "64"); + + map_segments(elf); + populate_segments(elf); + add_dependencies(elf); + copy_section_headers(elf); + save_symtab(elf); + close_handle(elf); + set_tls_offset(elf); + parse_property_segment(elf); + if (elf->bti_enabled) + ta_elf_add_bti(elf); +} + +void ta_elf_finalize_mappings(struct ta_elf *elf) +{ + TEE_Result res = TEE_SUCCESS; + struct segment *seg = NULL; + + if (!elf->is_legacy) + return; + + TAILQ_FOREACH(seg, &elf->segs, link) { + vaddr_t va = elf->load_addr + seg->vaddr; + uint32_t flags = 0; + + if (seg->flags & PF_W) + flags |= LDELF_MAP_FLAG_WRITEABLE; + if (seg->flags & PF_X) + flags |= LDELF_MAP_FLAG_EXECUTABLE; + + res = sys_set_prot(va, seg->memsz, flags); + if (res) + err(res, "sys_set_prot"); + } +} + +static void __printf(3, 4) print_wrapper(void *pctx, print_func_t print_func, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + print_func(pctx, fmt, ap); + va_end(ap); +} + +static void print_seg(void *pctx, print_func_t print_func, + size_t idx __maybe_unused, int elf_idx __maybe_unused, + vaddr_t va __maybe_unused, paddr_t pa __maybe_unused, + size_t sz __maybe_unused, uint32_t flags) +{ + int rc __maybe_unused = 0; + int width __maybe_unused = 8; + char desc[14] __maybe_unused = ""; + char flags_str[] __maybe_unused = "----"; + + if (elf_idx > -1) { + rc = snprintf(desc, sizeof(desc), " [%d]", elf_idx); + assert(rc >= 0); + } else { + if (flags & DUMP_MAP_EPHEM) { + rc = snprintf(desc, sizeof(desc), " (param)"); + assert(rc >= 0); + } + if (flags & DUMP_MAP_LDELF) { + rc = snprintf(desc, sizeof(desc), " (ldelf)"); + assert(rc >= 0); + } + if (va == ta_stack) { + rc = snprintf(desc, sizeof(desc), " (stack)"); + assert(rc >= 0); + } + } + + if (flags & DUMP_MAP_READ) + flags_str[0] = 'r'; + if (flags & DUMP_MAP_WRITE) + flags_str[1] = 'w'; + if (flags & DUMP_MAP_EXEC) + flags_str[2] = 'x'; + if (flags & DUMP_MAP_SECURE) + flags_str[3] = 's'; + + print_wrapper(pctx, print_func, + "region %2zu: va 0x%0*"PRIxVA" pa 0x%0*"PRIxPA" size 0x%06zx flags %s%s\n", + idx, width, va, width, pa, sz, flags_str, desc); +} + +static bool get_next_in_order(struct ta_elf_queue *elf_queue, + struct ta_elf **elf, struct segment **seg, + size_t *elf_idx) +{ + struct ta_elf *e = NULL; + struct segment *s = NULL; + size_t idx = 0; + vaddr_t va = 0; + struct ta_elf *e2 = NULL; + size_t i2 = 0; + + assert(elf && seg && elf_idx); + e = *elf; + s = *seg; + assert((e == NULL && s == NULL) || (e != NULL && s != NULL)); + + if (s) { + s = TAILQ_NEXT(s, link); + if (s) { + *seg = s; + return true; + } + } + + if (e) + va = e->load_addr; + + /* Find the ELF with next load address */ + e = NULL; + TAILQ_FOREACH(e2, elf_queue, link) { + if (e2->load_addr > va) { + if (!e || e2->load_addr < e->load_addr) { + e = e2; + idx = i2; + } + } + i2++; + } + if (!e) + return false; + + *elf = e; + *seg = TAILQ_FIRST(&e->segs); + *elf_idx = idx; + return true; +} + +void ta_elf_print_mappings(void *pctx, print_func_t print_func, + struct ta_elf_queue *elf_queue, size_t num_maps, + struct dump_map *maps, vaddr_t mpool_base) +{ + struct segment *seg = NULL; + struct ta_elf *elf = NULL; + size_t elf_idx = 0; + size_t idx = 0; + size_t map_idx = 0; + + /* + * Loop over all segments and maps, printing virtual address in + * order. Segment has priority if the virtual address is present + * in both map and segment. + */ + get_next_in_order(elf_queue, &elf, &seg, &elf_idx); + while (true) { + vaddr_t va = -1; + size_t sz = 0; + uint32_t flags = DUMP_MAP_SECURE; + size_t offs = 0; + + if (seg) { + va = rounddown(seg->vaddr + elf->load_addr); + sz = roundup(seg->vaddr + seg->memsz) - + rounddown(seg->vaddr); + } + + while (map_idx < num_maps && maps[map_idx].va <= va) { + uint32_t f = 0; + + /* If there's a match, it should be the same map */ + if (maps[map_idx].va == va) { + /* + * In shared libraries the first page is + * mapped separately with the rest of that + * segment following back to back in a + * separate entry. + */ + if (map_idx + 1 < num_maps && + maps[map_idx].sz == SMALL_PAGE_SIZE) { + vaddr_t next_va = maps[map_idx].va + + maps[map_idx].sz; + size_t comb_sz = maps[map_idx].sz + + maps[map_idx + 1].sz; + + if (next_va == maps[map_idx + 1].va && + comb_sz == sz && + maps[map_idx].flags == + maps[map_idx + 1].flags) { + /* Skip this and next entry */ + map_idx += 2; + continue; + } + } + assert(maps[map_idx].sz == sz); + } else if (maps[map_idx].va < va) { + if (maps[map_idx].va == mpool_base) + f |= DUMP_MAP_LDELF; + print_seg(pctx, print_func, idx, -1, + maps[map_idx].va, maps[map_idx].pa, + maps[map_idx].sz, + maps[map_idx].flags | f); + idx++; + } + map_idx++; + } + + if (!seg) + break; + + offs = rounddown(seg->offset); + if (seg->flags & PF_R) + flags |= DUMP_MAP_READ; + if (seg->flags & PF_W) + flags |= DUMP_MAP_WRITE; + if (seg->flags & PF_X) + flags |= DUMP_MAP_EXEC; + + print_seg(pctx, print_func, idx, elf_idx, va, offs, sz, flags); + idx++; + + if (!get_next_in_order(elf_queue, &elf, &seg, &elf_idx)) + seg = NULL; + } + + elf_idx = 0; + TAILQ_FOREACH(elf, elf_queue, link) { + print_wrapper(pctx, print_func, + " [%zu] %pUl @ 0x%0*"PRIxVA"\n", + elf_idx, (void *)&elf->uuid, 8, elf->load_addr); + elf_idx++; + } +} + +#ifdef CFG_UNWIND + +#if defined(ARM32) || defined(ARM64) +/* Called by libunw */ +bool find_exidx(vaddr_t addr, vaddr_t *idx_start, vaddr_t *idx_end) +{ + struct segment *seg = NULL; + struct ta_elf *elf = NULL; + vaddr_t a = 0; + + TAILQ_FOREACH(elf, &main_elf_queue, link) { + if (addr < elf->load_addr) + continue; + a = addr - elf->load_addr; + TAILQ_FOREACH(seg, &elf->segs, link) { + if (a < seg->vaddr) + continue; + if (a - seg->vaddr < seg->filesz) { + *idx_start = elf->exidx_start + elf->load_addr; + *idx_end = elf->exidx_start + elf->load_addr + + elf->exidx_size; + return true; + } + } + } + + return false; +} + +void ta_elf_stack_trace_a32(uint32_t regs[16]) +{ + struct unwind_state_arm32 state = { }; + + memcpy(state.registers, regs, sizeof(state.registers)); + print_stack_arm32(&state, ta_stack, ta_stack_size); +} + +void ta_elf_stack_trace_a64(uint64_t fp, uint64_t sp, uint64_t pc) +{ + struct unwind_state_arm64 state = { .fp = fp, .sp = sp, .pc = pc }; + + print_stack_arm64(&state, ta_stack, ta_stack_size); +} +#elif defined(RV32) || defined(RV64) +void ta_elf_stack_trace_riscv(uint64_t fp, uint64_t pc) +{ + struct unwind_state_riscv state = { .fp = fp, .pc = pc }; + + print_stack_riscv(&state, ta_stack, ta_stack_size); +} +#endif + +#endif /* CFG_UNWIND */ + +TEE_Result ta_elf_add_library(const TEE_UUID *uuid) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct ta_elf *ta = TAILQ_FIRST(&main_elf_queue); + struct ta_elf *lib = ta_elf_find_elf(uuid); + struct ta_elf *elf = NULL; + + if (lib) + return TEE_SUCCESS; /* Already mapped */ + + lib = queue_elf_helper(uuid); + if (!lib) + return TEE_ERROR_OUT_OF_MEMORY; + + for (elf = lib; elf; elf = TAILQ_NEXT(elf, link)) + ta_elf_load_dependency(elf, ta->is_32bit); + + for (elf = lib; elf; elf = TAILQ_NEXT(elf, link)) { + ta_elf_relocate(elf); + ta_elf_finalize_mappings(elf); + } + + for (elf = lib; elf; elf = TAILQ_NEXT(elf, link)) + DMSG("ELF (%pUl) at %#"PRIxVA, + (void *)&elf->uuid, elf->load_addr); + + res = ta_elf_set_init_fini_info_compat(ta->is_32bit); + if (res) + return res; + + return ta_elf_set_elf_phdr_info(ta->is_32bit); +} + +/* Get address/size of .init_array and .fini_array from the dynamic segment */ +static void get_init_fini_array(struct ta_elf *elf, unsigned int type, + vaddr_t addr, size_t memsz, vaddr_t *init, + size_t *init_cnt, vaddr_t *fini, + size_t *fini_cnt) +{ + size_t addrsz = 0; + size_t dyn_entsize = 0; + size_t num_dyns = 0; + size_t n = 0; + unsigned int tag = 0; + size_t val = 0; + + assert(type == PT_DYNAMIC); + + check_phdr_in_range(elf, type, addr, memsz); + + if (elf->is_32bit) { + dyn_entsize = sizeof(Elf32_Dyn); + addrsz = 4; + } else { + dyn_entsize = sizeof(Elf64_Dyn); + addrsz = 8; + } + + assert(!(memsz % dyn_entsize)); + num_dyns = memsz / dyn_entsize; + + for (n = 0; n < num_dyns; n++) { + read_dyn(elf, addr, n, &tag, &val); + if (tag == DT_INIT_ARRAY) + *init = val + elf->load_addr; + else if (tag == DT_FINI_ARRAY) + *fini = val + elf->load_addr; + else if (tag == DT_INIT_ARRAYSZ) + *init_cnt = val / addrsz; + else if (tag == DT_FINI_ARRAYSZ) + *fini_cnt = val / addrsz; + } +} + +/* Get address/size of .init_array and .fini_array in @elf (if present) */ +static void elf_get_init_fini_array(struct ta_elf *elf, vaddr_t *init, + size_t *init_cnt, vaddr_t *fini, + size_t *fini_cnt) +{ + size_t n = 0; + + if (elf->is_32bit) { + Elf32_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) { + if (phdr[n].p_type == PT_DYNAMIC) { + get_init_fini_array(elf, phdr[n].p_type, + phdr[n].p_vaddr, + phdr[n].p_memsz, + init, init_cnt, fini, + fini_cnt); + return; + } + } + } else { + Elf64_Phdr *phdr = elf->phdr; + + for (n = 0; n < elf->e_phnum; n++) { + if (phdr[n].p_type == PT_DYNAMIC) { + get_init_fini_array(elf, phdr[n].p_type, + phdr[n].p_vaddr, + phdr[n].p_memsz, + init, init_cnt, fini, + fini_cnt); + return; + } + } + } +} + +/* + * Deprecated by __elf_phdr_info below. Kept for compatibility. + * + * Pointers to ELF initialization and finalization functions are extracted by + * ldelf and stored on the TA heap, then exported to the TA via the global + * symbol __init_fini_info. libutee in OP-TEE 3.9.0 uses this mechanism. + */ + +struct __init_fini { + uint32_t flags; + uint16_t init_size; + uint16_t fini_size; + + void (**init)(void); /* @init_size entries */ + void (**fini)(void); /* @fini_size entries */ +}; + +#define __IFS_VALID BIT(0) +#define __IFS_INIT_HAS_RUN BIT(1) +#define __IFS_FINI_HAS_RUN BIT(2) + +struct __init_fini_info { + uint32_t reserved; + uint16_t size; + uint16_t pad; + struct __init_fini *ifs; /* @size entries */ +}; + +/* 32-bit variants for a 64-bit ldelf to access a 32-bit TA */ + +struct __init_fini32 { + uint32_t flags; + uint16_t init_size; + uint16_t fini_size; + uint32_t init; + uint32_t fini; +}; + +struct __init_fini_info32 { + uint32_t reserved; + uint16_t size; + uint16_t pad; + uint32_t ifs; +}; + +static TEE_Result realloc_ifs(vaddr_t va, size_t cnt, bool is_32bit) +{ + struct __init_fini_info32 *info32 = (struct __init_fini_info32 *)va; + struct __init_fini_info *info = (struct __init_fini_info *)va; + struct __init_fini32 *ifs32 = NULL; + struct __init_fini *ifs = NULL; + size_t prev_cnt = 0; + void *ptr = NULL; + + if (is_32bit) { + ptr = (void *)(vaddr_t)info32->ifs; + ptr = realloc(ptr, cnt * sizeof(struct __init_fini32)); + if (!ptr) + return TEE_ERROR_OUT_OF_MEMORY; + ifs32 = ptr; + prev_cnt = info32->size; + if (cnt > prev_cnt) + memset(ifs32 + prev_cnt, 0, + (cnt - prev_cnt) * sizeof(*ifs32)); + info32->ifs = (uint32_t)(vaddr_t)ifs32; + info32->size = cnt; + } else { + ptr = realloc(info->ifs, cnt * sizeof(struct __init_fini)); + if (!ptr) + return TEE_ERROR_OUT_OF_MEMORY; + ifs = ptr; + prev_cnt = info->size; + if (cnt > prev_cnt) + memset(ifs + prev_cnt, 0, + (cnt - prev_cnt) * sizeof(*ifs)); + info->ifs = ifs; + info->size = cnt; + } + + return TEE_SUCCESS; +} + +static void fill_ifs(vaddr_t va, size_t idx, struct ta_elf *elf, bool is_32bit) +{ + struct __init_fini_info32 *info32 = (struct __init_fini_info32 *)va; + struct __init_fini_info *info = (struct __init_fini_info *)va; + struct __init_fini32 *ifs32 = NULL; + struct __init_fini *ifs = NULL; + size_t init_cnt = 0; + size_t fini_cnt = 0; + vaddr_t init = 0; + vaddr_t fini = 0; + + if (is_32bit) { + assert(idx < info32->size); + ifs32 = &((struct __init_fini32 *)(vaddr_t)info32->ifs)[idx]; + + if (ifs32->flags & __IFS_VALID) + return; + + elf_get_init_fini_array(elf, &init, &init_cnt, &fini, + &fini_cnt); + + ifs32->init = (uint32_t)init; + ifs32->init_size = init_cnt; + + ifs32->fini = (uint32_t)fini; + ifs32->fini_size = fini_cnt; + + ifs32->flags |= __IFS_VALID; + } else { + assert(idx < info->size); + ifs = &info->ifs[idx]; + + if (ifs->flags & __IFS_VALID) + return; + + elf_get_init_fini_array(elf, &init, &init_cnt, &fini, + &fini_cnt); + + ifs->init = (void (**)(void))init; + ifs->init_size = init_cnt; + + ifs->fini = (void (**)(void))fini; + ifs->fini_size = fini_cnt; + + ifs->flags |= __IFS_VALID; + } +} + +/* + * Set or update __init_fini_info in the TA with information from the ELF + * queue + */ +TEE_Result ta_elf_set_init_fini_info_compat(bool is_32bit) +{ + struct __init_fini_info *info = NULL; + TEE_Result res = TEE_SUCCESS; + struct ta_elf *elf = NULL; + vaddr_t info_va = 0; + size_t cnt = 0; + + res = ta_elf_resolve_sym("__init_fini_info", &info_va, NULL, NULL); + if (res) { + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + /* + * Not an error, only TAs linked against libutee from + * OP-TEE 3.9.0 have this symbol. + */ + return TEE_SUCCESS; + } + return res; + } + assert(info_va); + + info = (struct __init_fini_info *)info_va; + if (info->reserved) + return TEE_ERROR_NOT_SUPPORTED; + + TAILQ_FOREACH(elf, &main_elf_queue, link) + cnt++; + + /* Queue has at least one file (main) */ + assert(cnt); + + res = realloc_ifs(info_va, cnt, is_32bit); + if (res) + goto err; + + cnt = 0; + TAILQ_FOREACH(elf, &main_elf_queue, link) { + fill_ifs(info_va, cnt, elf, is_32bit); + cnt++; + } + + return TEE_SUCCESS; +err: + free(info); + return res; +} + +static TEE_Result realloc_elf_phdr_info(vaddr_t va, size_t cnt, bool is_32bit) +{ + struct __elf_phdr_info32 *info32 = (struct __elf_phdr_info32 *)va; + struct __elf_phdr_info *info = (struct __elf_phdr_info *)va; + struct dl_phdr_info32 *dlpi32 = NULL; + struct dl_phdr_info *dlpi = NULL; + size_t prev_cnt = 0; + void *ptr = NULL; + + if (is_32bit) { + ptr = (void *)(vaddr_t)info32->dlpi; + ptr = realloc(ptr, cnt * sizeof(*dlpi32)); + if (!ptr) + return TEE_ERROR_OUT_OF_MEMORY; + dlpi32 = ptr; + prev_cnt = info32->count; + if (cnt > prev_cnt) + memset(dlpi32 + prev_cnt, 0, + (cnt - prev_cnt) * sizeof(*dlpi32)); + info32->dlpi = (uint32_t)(vaddr_t)dlpi32; + info32->count = cnt; + } else { + ptr = realloc(info->dlpi, cnt * sizeof(*dlpi)); + if (!ptr) + return TEE_ERROR_OUT_OF_MEMORY; + dlpi = ptr; + prev_cnt = info->count; + if (cnt > prev_cnt) + memset(dlpi + prev_cnt, 0, + (cnt - prev_cnt) * sizeof(*dlpi)); + info->dlpi = dlpi; + info->count = cnt; + } + + return TEE_SUCCESS; +} + +static void fill_elf_phdr_info(vaddr_t va, size_t idx, struct ta_elf *elf, + bool is_32bit) +{ + struct __elf_phdr_info32 *info32 = (struct __elf_phdr_info32 *)va; + struct __elf_phdr_info *info = (struct __elf_phdr_info *)va; + struct dl_phdr_info32 *dlpi32 = NULL; + struct dl_phdr_info *dlpi = NULL; + + if (is_32bit) { + assert(idx < info32->count); + dlpi32 = (struct dl_phdr_info32 *)(vaddr_t)info32->dlpi + idx; + + dlpi32->dlpi_addr = elf->load_addr; + if (elf->soname) + dlpi32->dlpi_name = (vaddr_t)elf->soname; + else + dlpi32->dlpi_name = (vaddr_t)&info32->zero; + dlpi32->dlpi_phdr = (vaddr_t)elf->phdr; + dlpi32->dlpi_phnum = elf->e_phnum; + dlpi32->dlpi_adds = 1; /* No unloading on dlclose() currently */ + dlpi32->dlpi_subs = 0; /* No unloading on dlclose() currently */ + dlpi32->dlpi_tls_modid = elf->tls_mod_id; + dlpi32->dlpi_tls_data = elf->tls_start; + } else { + assert(idx < info->count); + dlpi = info->dlpi + idx; + + dlpi->dlpi_addr = elf->load_addr; + if (elf->soname) + dlpi->dlpi_name = elf->soname; + else + dlpi->dlpi_name = &info32->zero; + dlpi->dlpi_phdr = elf->phdr; + dlpi->dlpi_phnum = elf->e_phnum; + dlpi->dlpi_adds = 1; /* No unloading on dlclose() currently */ + dlpi->dlpi_subs = 0; /* No unloading on dlclose() currently */ + dlpi->dlpi_tls_modid = elf->tls_mod_id; + dlpi->dlpi_tls_data = (void *)elf->tls_start; + } +} + +/* Set or update __elf_hdr_info in the TA with information from the ELF queue */ +TEE_Result ta_elf_set_elf_phdr_info(bool is_32bit) +{ + struct __elf_phdr_info *info = NULL; + TEE_Result res = TEE_SUCCESS; + struct ta_elf *elf = NULL; + vaddr_t info_va = 0; + size_t cnt = 0; + + res = ta_elf_resolve_sym("__elf_phdr_info", &info_va, NULL, NULL); + if (res) { + if (res == TEE_ERROR_ITEM_NOT_FOUND) { + /* Older TA */ + return TEE_SUCCESS; + } + return res; + } + assert(info_va); + + info = (struct __elf_phdr_info *)info_va; + if (info->reserved) + return TEE_ERROR_NOT_SUPPORTED; + + TAILQ_FOREACH(elf, &main_elf_queue, link) + cnt++; + + res = realloc_elf_phdr_info(info_va, cnt, is_32bit); + if (res) + return res; + + cnt = 0; + TAILQ_FOREACH(elf, &main_elf_queue, link) { + fill_elf_phdr_info(info_va, cnt, elf, is_32bit); + cnt++; + } + + return TEE_SUCCESS; +} diff --git a/optee_os/ldelf/ta_elf.h b/optee_os/ldelf/ta_elf.h new file mode 100644 index 0000000..fdb6508 --- /dev/null +++ b/optee_os/ldelf/ta_elf.h @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2022-2023, Arm Limited + */ + +#ifndef TA_ELF_H +#define TA_ELF_H + +#include +#include +#include +#include +#include + +struct segment { + size_t offset; + size_t vaddr; + size_t filesz; + size_t memsz; + size_t flags; + size_t align; + bool remapped_writeable; + TAILQ_ENTRY(segment) link; +}; + +TAILQ_HEAD(segment_head, segment); + +struct ta_elf { + bool is_main; + bool is_32bit; /* Initialized from Elf32_Ehdr/Elf64_Ehdr */ + bool is_legacy; + bool bti_enabled; + + vaddr_t load_addr; + vaddr_t max_addr; + vaddr_t max_offs; + + vaddr_t ehdr_addr; + + /* Initialized from Elf32_Ehdr/Elf64_Ehdr */ + vaddr_t e_entry; + vaddr_t e_phoff; + vaddr_t e_shoff; + unsigned int e_phnum; + unsigned int e_shnum; + unsigned int e_phentsize; + unsigned int e_shentsize; + + void *phdr; + void *shdr; + /* + * dynsymtab and dynstr are used for external symbols, they may hold + * other symbols too. + */ + void *dynsymtab; + size_t num_dynsyms; + const char *dynstr; + size_t dynstr_size; + + /* DT_HASH hash table for faster resolution of external symbols */ + void *hashtab; + /* DT_GNU_HASH table as an alternative to DT_HASH */ + void *gnu_hashtab; + size_t gnu_hashtab_size; + + /* DT_SONAME */ + char *soname; + + struct segment_head segs; + + vaddr_t exidx_start; + size_t exidx_size; + + /* Thread Local Storage */ + + size_t tls_mod_id; + /* PT_TLS segment */ + vaddr_t tls_start; + size_t tls_filesz; /* Covers the .tdata section */ + size_t tls_memsz; /* Covers the .tdata and .tbss sections */ +#ifdef ARM64 + /* Offset of the copy of the TLS block in the TLS area of the TCB */ + size_t tls_tcb_offs; +#endif + + /* PT_GNU_PROPERTY segment */ + vaddr_t prop_start; + size_t prop_align; + size_t prop_memsz; + + uint32_t handle; + + struct ta_head *head; + + TEE_UUID uuid; + TAILQ_ENTRY(ta_elf) link; +}; + +TAILQ_HEAD(ta_elf_queue, ta_elf); + +/* Format of the DT_GNU_HASH entry in the ELF dynamic section */ +struct gnu_hashtab { + uint32_t nbuckets; + uint32_t symoffset; + uint32_t bloom_size; + uint32_t bloom_shift; + /* + * Followed by: + * + * uint{32,64}_t bloom[bloom_size]; + * uint32_t buckets[nbuckets]; + * uint32_t chain[]; + */ +}; + +typedef void (*print_func_t)(void *pctx, const char *fmt, va_list ap) + __printf(2, 0); + +extern struct ta_elf_queue main_elf_queue; +struct ta_elf *ta_elf_find_elf(const TEE_UUID *uuid); + +void ta_elf_load_main(const TEE_UUID *uuid, uint32_t *is_32bit, uint64_t *sp, + uint32_t *ta_flags); +void ta_elf_finalize_load_main(uint64_t *entry, uint64_t *load_addr); +void ta_elf_load_dependency(struct ta_elf *elf, bool is_32bit); +void ta_elf_relocate(struct ta_elf *elf); +void ta_elf_finalize_mappings(struct ta_elf *elf); + +void ta_elf_print_mappings(void *pctx, print_func_t print_func, + struct ta_elf_queue *elf_queue, size_t num_maps, + struct dump_map *maps, vaddr_t mpool_base); + +#ifdef CFG_UNWIND +void ta_elf_stack_trace_a32(uint32_t regs[16]); +void ta_elf_stack_trace_a64(uint64_t fp, uint64_t sp, uint64_t pc); +void ta_elf_stack_trace_riscv(uint64_t fp, uint64_t pc); +#else +static inline void ta_elf_stack_trace_a32(uint32_t regs[16] __unused) { } +static inline void ta_elf_stack_trace_a64(uint64_t fp __unused, + uint64_t sp __unused, + uint64_t pc __unused) { } +static inline void ta_elf_stack_trace_riscv(uint64_t fp __unused, + uint64_t pc __unused) { } +#endif /*CFG_UNWIND*/ + +TEE_Result ta_elf_resolve_sym(const char *name, vaddr_t *val, + struct ta_elf **found_elf, struct ta_elf *elf); +TEE_Result ta_elf_add_library(const TEE_UUID *uuid); +TEE_Result ta_elf_set_init_fini_info_compat(bool is_32bit); +TEE_Result ta_elf_set_elf_phdr_info(bool is_32bit); + +#endif /*TA_ELF_H*/ diff --git a/optee_os/ldelf/ta_elf_rel.c b/optee_os/ldelf/ta_elf_rel.c new file mode 100644 index 0000000..5d5b1eb --- /dev/null +++ b/optee_os/ldelf/ta_elf_rel.c @@ -0,0 +1,752 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sys.h" +#include "ta_elf.h" + +static uint32_t elf_hash(const char *name) +{ + const unsigned char *p = (const unsigned char *)name; + uint32_t h = 0; + uint32_t g = 0; + + while (*p) { + h = (h << 4) + *p++; + g = h & 0xf0000000; + if (g) + h ^= g >> 24; + h &= ~g; + } + return h; +} + +static uint32_t gnu_hash(const char *name) +{ + const unsigned char *p = (const unsigned char *)name; + uint32_t h = 5381; + + while (*p) + h = (h << 5) + h + *p++; + + return h; +} + +static bool sym_compare(struct ta_elf *elf, unsigned int st_bind, + unsigned int st_type, size_t st_shndx, + size_t st_name, size_t st_value, const char *name, + vaddr_t *val, bool weak_ok) +{ + bool bind_ok = false; + + if (!st_name) + return false; + if (st_name > elf->dynstr_size) + err(TEE_ERROR_BAD_FORMAT, "Symbol name out of range"); + if (strcmp(name, elf->dynstr + st_name)) + return false; + if (st_bind == STB_GLOBAL || (weak_ok && st_bind == STB_WEAK)) + bind_ok = true; + if (!bind_ok) + return false; + if (st_bind == STB_WEAK && st_shndx == SHN_UNDEF) { + if (val) + *val = 0; + return true; + } + if (st_shndx == SHN_UNDEF || st_shndx == SHN_XINDEX) + return false; + + switch (st_type) { + case STT_NOTYPE: + case STT_OBJECT: + case STT_FUNC: + if (st_value > (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, + "Symbol location out of range"); + if (val) + *val = st_value + elf->load_addr; + break; + case STT_TLS: + if (val) + *val = st_value; + break; + default: + err(TEE_ERROR_NOT_SUPPORTED, "Symbol type not supported"); + } + + return true; +} + +static bool check_found_sym(struct ta_elf *elf, const char *name, vaddr_t *val, + bool weak_ok, size_t n) +{ + Elf32_Sym *sym32 = NULL; + Elf64_Sym *sym64 = NULL; + unsigned int st_bind = 0; + unsigned int st_type = 0; + size_t st_shndx = 0; + size_t st_name = 0; + size_t st_value = 0; + + if (n >= elf->num_dynsyms) + err(TEE_ERROR_BAD_FORMAT, "Index out of range"); + + /* + * We're loading values from sym[] which later + * will be used to load something. + * => Spectre V1 pattern, need to cap the index + * against speculation. + */ + n = confine_array_index(n, elf->num_dynsyms); + + if (elf->is_32bit) { + sym32 = elf->dynsymtab; + st_bind = ELF32_ST_BIND(sym32[n].st_info); + st_type = ELF32_ST_TYPE(sym32[n].st_info); + st_shndx = sym32[n].st_shndx; + st_name = sym32[n].st_name; + st_value = sym32[n].st_value; + } else { + sym64 = elf->dynsymtab; + st_bind = ELF64_ST_BIND(sym64[n].st_info); + st_type = ELF64_ST_TYPE(sym64[n].st_info); + st_shndx = sym64[n].st_shndx; + st_name = sym64[n].st_name; + st_value = sym64[n].st_value; + } + + return sym_compare(elf, st_bind, st_type, st_shndx, st_name, st_value, + name, val, weak_ok); +} + +static TEE_Result resolve_sym_helper(const char *name, vaddr_t *val, + struct ta_elf *elf, bool weak_ok) +{ + uint32_t n = 0; + uint32_t hash = 0; + + if (elf->gnu_hashtab) { + struct gnu_hashtab *h = elf->gnu_hashtab; + uint32_t *end = (void *)((uint8_t *)elf->gnu_hashtab + + elf->gnu_hashtab_size); + uint32_t *bucket = NULL; + uint32_t *chain = NULL; + uint32_t hashval = 0; + + hash = gnu_hash(name); + + if (elf->is_32bit) { + uint32_t *bloom = (void *)(h + 1); + uint32_t word = bloom[(hash / 32) % h->bloom_size]; + uint32_t mask = BIT32(hash % 32) | + BIT32((hash >> h->bloom_shift) % 32); + + if ((word & mask) != mask) + return TEE_ERROR_ITEM_NOT_FOUND; + bucket = bloom + h->bloom_size; + } else { + uint64_t *bloom = (void *)(h + 1); + uint64_t word = bloom[(hash / 64) % h->bloom_size]; + uint64_t mask = BIT64(hash % 64) | + BIT64((hash >> h->bloom_shift) % 64); + + if ((word & mask) != mask) + return TEE_ERROR_ITEM_NOT_FOUND; + bucket = (uint32_t *)(bloom + h->bloom_size); + } + chain = bucket + h->nbuckets; + + n = bucket[hash % h->nbuckets]; + if (n < h->symoffset) + return TEE_ERROR_ITEM_NOT_FOUND; + + hash |= 1; + do { + size_t idx = n - h->symoffset; + + if (chain + idx > end) + return TEE_ERROR_ITEM_NOT_FOUND; + + hashval = chain[idx]; + + if ((hashval | 1) == hash && + check_found_sym(elf, name, val, weak_ok, n)) + return TEE_SUCCESS; + + n++; + } while (!(hashval & 1)); + } else if (elf->hashtab) { + /* + * Using uint32_t here for convenience because both Elf64_Word + * and Elf32_Word are 32-bit types + */ + uint32_t *hashtab = elf->hashtab; + uint32_t nbuckets = hashtab[0]; + uint32_t nchains = hashtab[1]; + uint32_t *bucket = &hashtab[2]; + uint32_t *chain = &bucket[nbuckets]; + + hash = elf_hash(name); + + for (n = bucket[hash % nbuckets]; n; n = chain[n]) { + if (n >= nchains) + err(TEE_ERROR_BAD_FORMAT, "Index out of range"); + if (check_found_sym(elf, name, val, weak_ok, n)) + return TEE_SUCCESS; + } + } + + return TEE_ERROR_ITEM_NOT_FOUND; +} + +/* + * Look for named symbol in @elf, or all modules if @elf == NULL. Global symbols + * are searched first, then weak ones. Last option, when at least one weak but + * undefined symbol exists, resolve to zero. Otherwise return + * TEE_ERROR_ITEM_NOT_FOUND. + * @val (if != 0) receives the symbol value + * @found_elf (if != 0) receives the module where the symbol is found + */ +TEE_Result ta_elf_resolve_sym(const char *name, vaddr_t *val, + struct ta_elf **found_elf, + struct ta_elf *elf) +{ + if (elf) { + /* Search global symbols */ + if (!resolve_sym_helper(name, val, elf, false /* !weak_ok */)) + goto success; + /* Search weak symbols */ + if (!resolve_sym_helper(name, val, elf, true /* weak_ok */)) + goto success; + } + + TAILQ_FOREACH(elf, &main_elf_queue, link) { + if (!resolve_sym_helper(name, val, elf, false /* !weak_ok */)) + goto success; + if (!resolve_sym_helper(name, val, elf, true /* weak_ok */)) + goto success; + } + + return TEE_ERROR_ITEM_NOT_FOUND; + +success: + if (found_elf) + *found_elf = elf; + return TEE_SUCCESS; +} + +static void e32_get_sym_name(const Elf32_Sym *sym_tab, size_t num_syms, + const char *str_tab, size_t str_tab_size, + Elf32_Rel *rel, const char **name, + bool *weak_undef) +{ + size_t sym_idx = 0; + size_t name_idx = 0; + + sym_idx = ELF32_R_SYM(rel->r_info); + if (sym_idx >= num_syms) + err(TEE_ERROR_BAD_FORMAT, "Symbol index out of range"); + sym_idx = confine_array_index(sym_idx, num_syms); + + name_idx = sym_tab[sym_idx].st_name; + if (name_idx >= str_tab_size) + err(TEE_ERROR_BAD_FORMAT, "Name index out of range"); + *name = str_tab + name_idx; + + if (!weak_undef) + return; + if (sym_tab[sym_idx].st_shndx == SHN_UNDEF && + ELF32_ST_BIND(sym_tab[sym_idx].st_info) == STB_WEAK) + *weak_undef = true; + else + *weak_undef = false; +} + +static void resolve_sym(const char *name, vaddr_t *val, struct ta_elf **mod, + bool err_if_not_found) +{ + TEE_Result res = ta_elf_resolve_sym(name, val, mod, NULL); + + if (res) { + if (err_if_not_found) + err(res, "Symbol %s not found", name); + else + *val = 0; + } +} + +static void e32_process_dyn_rel(const Elf32_Sym *sym_tab, size_t num_syms, + const char *str_tab, size_t str_tab_size, + Elf32_Rel *rel, Elf32_Addr *where) +{ + const char *name = NULL; + vaddr_t val = 0; + bool weak_undef = false; + + e32_get_sym_name(sym_tab, num_syms, str_tab, str_tab_size, rel, &name, + &weak_undef); + resolve_sym(name, &val, NULL, !weak_undef); + *where = val; +} + +static void e32_tls_get_module(const Elf32_Sym *sym_tab, size_t num_syms, + const char *str_tab, size_t str_tab_size, + Elf32_Rel *rel, struct ta_elf **mod) +{ + const char *name = NULL; + size_t sym_idx = 0; + + sym_idx = ELF32_R_SYM(rel->r_info); + if (sym_idx >= num_syms) + err(TEE_ERROR_BAD_FORMAT, "Symbol index out of range"); + sym_idx = confine_array_index(sym_idx, num_syms); + if (!sym_idx || sym_tab[sym_idx].st_shndx != SHN_UNDEF) { + /* No symbol, or symbol is defined in current module */ + return; + } + + e32_get_sym_name(sym_tab, num_syms, str_tab, str_tab_size, rel, &name, + NULL); + resolve_sym(name, NULL, mod, false); +} + +static void e32_tls_resolve(const Elf32_Sym *sym_tab, size_t num_syms, + const char *str_tab, size_t str_tab_size, + Elf32_Rel *rel, vaddr_t *val) +{ + const char *name = NULL; + + e32_get_sym_name(sym_tab, num_syms, str_tab, str_tab_size, rel, &name, + NULL); + resolve_sym(name, val, NULL, false); +} + +static void e32_relocate(struct ta_elf *elf, unsigned int rel_sidx) +{ + Elf32_Shdr *shdr = elf->shdr; + Elf32_Rel *rel = NULL; + Elf32_Rel *rel_end = NULL; + size_t sym_tab_idx = 0; + Elf32_Sym *sym_tab = NULL; + size_t num_syms = 0; + size_t sh_end = 0; + const char *str_tab = NULL; + size_t str_tab_size = 0; + + assert(shdr[rel_sidx].sh_type == SHT_REL); + + assert(shdr[rel_sidx].sh_entsize == sizeof(Elf32_Rel)); + + sym_tab_idx = shdr[rel_sidx].sh_link; + if (sym_tab_idx) { + size_t str_tab_idx = 0; + + if (sym_tab_idx >= elf->e_shnum) + err(TEE_ERROR_BAD_FORMAT, "SYMTAB index out of range"); + sym_tab_idx = confine_array_index(sym_tab_idx, elf->e_shnum); + + assert(shdr[sym_tab_idx].sh_entsize == sizeof(Elf32_Sym)); + + /* Check the address is inside ELF memory */ + if (ADD_OVERFLOW(shdr[sym_tab_idx].sh_addr, + shdr[sym_tab_idx].sh_size, &sh_end)) + err(TEE_ERROR_BAD_FORMAT, "Overflow"); + if (sh_end >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, "SYMTAB out of range"); + + sym_tab = (Elf32_Sym *)(elf->load_addr + + shdr[sym_tab_idx].sh_addr); + + num_syms = shdr[sym_tab_idx].sh_size / sizeof(Elf32_Sym); + + str_tab_idx = shdr[sym_tab_idx].sh_link; + if (str_tab_idx) { + if (str_tab_idx >= elf->e_shnum) + err(TEE_ERROR_BAD_FORMAT, + "STRTAB index out of range"); + str_tab_idx = confine_array_index(str_tab_idx, + elf->e_shnum); + + /* Check the address is inside ELF memory */ + if (ADD_OVERFLOW(shdr[str_tab_idx].sh_addr, + shdr[str_tab_idx].sh_size, &sh_end)) + err(TEE_ERROR_BAD_FORMAT, "Overflow"); + if (sh_end >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, + "STRTAB out of range"); + + str_tab = (const char *)(elf->load_addr + + shdr[str_tab_idx].sh_addr); + str_tab_size = shdr[str_tab_idx].sh_size; + } + } + + /* Check the address is inside TA memory */ + if (ADD_OVERFLOW(shdr[rel_sidx].sh_addr, + shdr[rel_sidx].sh_size, &sh_end)) + err(TEE_ERROR_BAD_FORMAT, "Overflow"); + if (sh_end >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, ".rel.*/REL out of range"); + rel = (Elf32_Rel *)(elf->load_addr + shdr[rel_sidx].sh_addr); + + rel_end = rel + shdr[rel_sidx].sh_size / sizeof(Elf32_Rel); + for (; rel < rel_end; rel++) { + struct ta_elf *mod = NULL; + Elf32_Addr *where = NULL; + size_t sym_idx = 0; + vaddr_t val = 0; + + /* Check the address is inside TA memory */ + if (rel->r_offset >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, + "Relocation offset out of range"); + where = (Elf32_Addr *)(elf->load_addr + rel->r_offset); + + switch (ELF32_R_TYPE(rel->r_info)) { + case R_ARM_NONE: + /* + * One would expect linker prevents such useless entry + * in the relocation table. We still handle this type + * here in case such entries exist. + */ + break; + case R_ARM_ABS32: + sym_idx = ELF32_R_SYM(rel->r_info); + if (sym_idx >= num_syms) + err(TEE_ERROR_BAD_FORMAT, + "Symbol index out of range"); + if (sym_tab[sym_idx].st_shndx == SHN_UNDEF) { + /* Symbol is external */ + e32_process_dyn_rel(sym_tab, num_syms, str_tab, + str_tab_size, rel, where); + } else { + *where += elf->load_addr + + sym_tab[sym_idx].st_value; + } + break; + case R_ARM_REL32: + sym_idx = ELF32_R_SYM(rel->r_info); + if (sym_idx >= num_syms) + err(TEE_ERROR_BAD_FORMAT, + "Symbol index out of range"); + *where += sym_tab[sym_idx].st_value - rel->r_offset; + break; + case R_ARM_RELATIVE: + *where += elf->load_addr; + break; + case R_ARM_GLOB_DAT: + case R_ARM_JUMP_SLOT: + if (!sym_tab) + err(TEE_ERROR_BAD_FORMAT, + "Missing symbol table"); + e32_process_dyn_rel(sym_tab, num_syms, str_tab, + str_tab_size, rel, where); + break; + case R_ARM_TLS_DTPMOD32: + if (!sym_tab) + err(TEE_ERROR_BAD_FORMAT, + "Missing symbol table"); + mod = elf; + e32_tls_get_module(sym_tab, num_syms, str_tab, + str_tab_size, rel, &mod); + *where = mod->tls_mod_id; + break; + case R_ARM_TLS_DTPOFF32: + if (!sym_tab) + err(TEE_ERROR_BAD_FORMAT, + "Missing symbol table"); + e32_tls_resolve(sym_tab, num_syms, str_tab, + str_tab_size, rel, &val); + *where = val; + break; + default: + err(TEE_ERROR_BAD_FORMAT, "Unknown relocation type %d", + ELF32_R_TYPE(rel->r_info)); + } + } +} + +#if defined(ARM64) || defined(RV64) +static void e64_get_sym_name(const Elf64_Sym *sym_tab, size_t num_syms, + const char *str_tab, size_t str_tab_size, + Elf64_Rela *rela, const char **name, + bool *weak_undef) +{ + size_t sym_idx = 0; + size_t name_idx = 0; + + sym_idx = ELF64_R_SYM(rela->r_info); + if (sym_idx >= num_syms) + err(TEE_ERROR_BAD_FORMAT, "Symbol index out of range"); + sym_idx = confine_array_index(sym_idx, num_syms); + + name_idx = sym_tab[sym_idx].st_name; + if (name_idx >= str_tab_size) + err(TEE_ERROR_BAD_FORMAT, "Name index out of range"); + *name = str_tab + name_idx; + + if (sym_tab[sym_idx].st_shndx == SHN_UNDEF && + ELF64_ST_BIND(sym_tab[sym_idx].st_info) == STB_WEAK) + *weak_undef = true; + else + *weak_undef = false; +} + +static void e64_process_dyn_rela(const Elf64_Sym *sym_tab, size_t num_syms, + const char *str_tab, size_t str_tab_size, + Elf64_Rela *rela, Elf64_Addr *where) +{ + const char *name = NULL; + uintptr_t val = 0; + bool weak_undef = false; + + e64_get_sym_name(sym_tab, num_syms, str_tab, str_tab_size, rela, &name, + &weak_undef); + resolve_sym(name, &val, NULL, !weak_undef); + *where = val; +} + +#ifdef ARM64 +static void e64_process_tls_tprel_rela(const Elf64_Sym *sym_tab, + size_t num_syms, const char *str_tab, + size_t str_tab_size, Elf64_Rela *rela, + Elf64_Addr *where, struct ta_elf *elf) +{ + struct ta_elf *mod = NULL; + bool weak_undef = false; + const char *name = NULL; + size_t sym_idx = 0; + vaddr_t symval = 0; + + sym_idx = ELF64_R_SYM(rela->r_info); + if (sym_idx) { + e64_get_sym_name(sym_tab, num_syms, str_tab, str_tab_size, rela, + &name, &weak_undef); + resolve_sym(name, &symval, &mod, !weak_undef); + } else { + mod = elf; + } + *where = symval + mod->tls_tcb_offs + rela->r_addend; +} + +struct tlsdesc { + long (*resolver)(struct tlsdesc *td); + long value; +}; + +/* Helper function written in assembly due to the calling convention */ +long tlsdesc_resolve(struct tlsdesc *td); + +static void e64_process_tlsdesc_rela(const Elf64_Sym *sym_tab, size_t num_syms, + const char *str_tab, size_t str_tab_size, + Elf64_Rela *rela, Elf64_Addr *where, + struct ta_elf *elf) +{ + /* + * @where points to a pair of 64-bit words in the GOT or PLT which is + * mapped to a struct tlsdesc: + * + * - resolver() must return the offset of the thread-local variable + * relative to TPIDR_EL0. + * - value is implementation-dependent. The TLS_TPREL handling code is + * re-used to get the desired offset so that tlsdesc_resolve() just + * needs to return this value. + * + * Both the TA and ldelf are AArch64 so it is OK to point to a function + * in ldelf. + */ + *where = (Elf64_Addr)tlsdesc_resolve; + e64_process_tls_tprel_rela(sym_tab, num_syms, str_tab, str_tab_size, + rela, where + 1, elf); +} +#endif /*ARM64*/ + +static void e64_relocate(struct ta_elf *elf, unsigned int rel_sidx) +{ + Elf64_Shdr *shdr = elf->shdr; + Elf64_Rela *rela = NULL; + Elf64_Rela *rela_end = NULL; + size_t sym_tab_idx = 0; + Elf64_Sym *sym_tab = NULL; + size_t num_syms = 0; + size_t sh_end = 0; + const char *str_tab = NULL; + size_t str_tab_size = 0; + + assert(shdr[rel_sidx].sh_type == SHT_RELA); + + assert(shdr[rel_sidx].sh_entsize == sizeof(Elf64_Rela)); + + sym_tab_idx = shdr[rel_sidx].sh_link; + if (sym_tab_idx) { + size_t str_tab_idx = 0; + + if (sym_tab_idx >= elf->e_shnum) + err(TEE_ERROR_BAD_FORMAT, "SYMTAB index out of range"); + sym_tab_idx = confine_array_index(sym_tab_idx, elf->e_shnum); + + assert(shdr[sym_tab_idx].sh_entsize == sizeof(Elf64_Sym)); + + /* Check the address is inside TA memory */ + if (ADD_OVERFLOW(shdr[sym_tab_idx].sh_addr, + shdr[sym_tab_idx].sh_size, &sh_end)) + err(TEE_ERROR_BAD_FORMAT, "Overflow"); + if (sh_end >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, "SYMTAB out of range"); + + sym_tab = (Elf64_Sym *)(elf->load_addr + + shdr[sym_tab_idx].sh_addr); + + num_syms = shdr[sym_tab_idx].sh_size / sizeof(Elf64_Sym); + + str_tab_idx = shdr[sym_tab_idx].sh_link; + if (str_tab_idx) { + if (str_tab_idx >= elf->e_shnum) + err(TEE_ERROR_BAD_FORMAT, + "STRTAB index out of range"); + str_tab_idx = confine_array_index(str_tab_idx, + elf->e_shnum); + + /* Check the address is inside ELF memory */ + if (ADD_OVERFLOW(shdr[str_tab_idx].sh_addr, + shdr[str_tab_idx].sh_size, &sh_end)) + err(TEE_ERROR_BAD_FORMAT, "Overflow"); + if (sh_end >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, + "STRTAB out of range"); + + str_tab = (const char *)(elf->load_addr + + shdr[str_tab_idx].sh_addr); + str_tab_size = shdr[str_tab_idx].sh_size; + } + } + + /* Check the address is inside TA memory */ + if (ADD_OVERFLOW(shdr[rel_sidx].sh_addr, + shdr[rel_sidx].sh_size, &sh_end)) + err(TEE_ERROR_BAD_FORMAT, "Overflow"); + if (sh_end >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, ".rel.*/REL out of range"); + rela = (Elf64_Rela *)(elf->load_addr + shdr[rel_sidx].sh_addr); + + rela_end = rela + shdr[rel_sidx].sh_size / sizeof(Elf64_Rela); + for (; rela < rela_end; rela++) { + Elf64_Addr *where = NULL; + size_t sym_idx = 0; + + /* Check the address is inside TA memory */ + if (rela->r_offset >= (elf->max_addr - elf->load_addr)) + err(TEE_ERROR_BAD_FORMAT, + "Relocation offset out of range"); + + where = (Elf64_Addr *)(elf->load_addr + rela->r_offset); + + switch (ELF64_R_TYPE(rela->r_info)) { +#ifdef ARM64 + case R_AARCH64_NONE: + /* + * One would expect linker prevents such useless entry + * in the relocation table. We still handle this type + * here in case such entries exist. + */ + break; + case R_AARCH64_ABS64: + sym_idx = ELF64_R_SYM(rela->r_info); + if (sym_idx >= num_syms) + err(TEE_ERROR_BAD_FORMAT, + "Symbol index out of range"); + sym_idx = confine_array_index(sym_idx, num_syms); + if (sym_tab[sym_idx].st_shndx == SHN_UNDEF) { + /* Symbol is external */ + e64_process_dyn_rela(sym_tab, num_syms, str_tab, + str_tab_size, rela, where); + } else { + *where = rela->r_addend + elf->load_addr + + sym_tab[sym_idx].st_value; + } + break; + case R_AARCH64_RELATIVE: + *where = rela->r_addend + elf->load_addr; + break; + case R_AARCH64_GLOB_DAT: + case R_AARCH64_JUMP_SLOT: + e64_process_dyn_rela(sym_tab, num_syms, str_tab, + str_tab_size, rela, where); + break; + case R_AARCH64_TLS_TPREL: + e64_process_tls_tprel_rela(sym_tab, num_syms, str_tab, + str_tab_size, rela, where, + elf); + break; + case R_AARCH64_TLSDESC: + e64_process_tlsdesc_rela(sym_tab, num_syms, str_tab, + str_tab_size, rela, where, + elf); + break; +#endif /*ARM64*/ +#ifdef RV64 + case R_RISCV_NONE: + /* + * One would expect linker prevents such useless entry + * in the relocation table. We still handle this type + * here in case such entries exist. + */ + break; + case R_RISCV_RELATIVE: + *where = rela->r_addend + elf->load_addr; + break; + case R_RISCV_64: + e64_process_dyn_rela(sym_tab, num_syms, str_tab, + str_tab_size, rela, where); + *where += rela->r_addend; + break; + case R_RISCV_JUMP_SLOT: + e64_process_dyn_rela(sym_tab, num_syms, str_tab, + str_tab_size, rela, where); + break; +#endif /*RV64*/ + default: + err(TEE_ERROR_BAD_FORMAT, "Unknown relocation type %zd", + ELF64_R_TYPE(rela->r_info)); + } + } +} +#else /*ARM64 || RV64*/ +static void __noreturn e64_relocate(struct ta_elf *elf __unused, + unsigned int rel_sidx __unused) +{ + err(TEE_ERROR_NOT_SUPPORTED, "arm64 not supported"); +} +#endif /*ARM64 || RV64*/ + +void ta_elf_relocate(struct ta_elf *elf) +{ + size_t n = 0; + + if (elf->is_32bit) { + Elf32_Shdr *shdr = elf->shdr; + + for (n = 0; n < elf->e_shnum; n++) + if (shdr[n].sh_type == SHT_REL) + e32_relocate(elf, n); + } else { + Elf64_Shdr *shdr = elf->shdr; + + for (n = 0; n < elf->e_shnum; n++) + if (shdr[n].sh_type == SHT_RELA) + e64_relocate(elf, n); + + } +} diff --git a/optee_os/ldelf/tlsdesc_rel_a64.S b/optee_os/ldelf/tlsdesc_rel_a64.S new file mode 100644 index 0000000..b187505 --- /dev/null +++ b/optee_os/ldelf/tlsdesc_rel_a64.S @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Huawei Technologies Co. Ltd. + */ + +#include + +/* + * long tlsdesc_resolve(struct tlsdesc *); + * + * Must preserve all registers except x0, x1 and the processor flags. + * See https://www.fsfla.org/~lxoliva/writeups/TLS/RFC-TLSDESC-ARM.txt section + * "Resolvers' Calling Convention". The document applies to 32-bit Arm but other + * sources mention similar constraints for other architectures. + */ +FUNC tlsdesc_resolve , : + ldr x0, [x0, #8] + ret +END_FUNC tlsdesc_resolve + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/lib/libdl/dlfcn.c b/optee_os/lib/libdl/dlfcn.c new file mode 100644 index 0000000..338c176 --- /dev/null +++ b/optee_os/lib/libdl/dlfcn.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Linaro limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static TEE_TASessionHandle sess = TEE_HANDLE_NULL; +static size_t hcount; + +static TEE_Result invoke_system_pta(uint32_t cmd_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const TEE_UUID core_uuid = PTA_SYSTEM_UUID; + TEE_Result res = TEE_ERROR_GENERIC; + + if (sess == TEE_HANDLE_NULL) { + res = TEE_OpenTASession(&core_uuid, TEE_TIMEOUT_INFINITE, + 0, NULL, &sess, NULL); + if (res) + return res; + } + return TEE_InvokeTACommand(sess, TEE_TIMEOUT_INFINITE, + cmd_id, param_types, params, NULL); +} + +struct dl_handle { + TEE_UUID uuid; +}; + +void *dlopen(const char *filename, int flags) +{ + TEE_Param params[TEE_NUM_PARAMS] = { }; + struct dl_handle *h = NULL; + uint32_t param_types = 0; + TEE_Result res = TEE_ERROR_GENERIC; + TEE_UUID uuid = { }; + + h = malloc(sizeof(*h)); + if (!h) + return NULL; + + if (filename) { + res = tee_uuid_from_str(&uuid, filename); + if (res) + goto err; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + params[0].memref.buffer = (void *)&uuid; + params[0].memref.size = sizeof(uuid); + params[1].value.a = flags; + + res = invoke_system_pta(PTA_SYSTEM_DLOPEN, param_types, params); + if (res) + goto err; + + __utee_tcb_init(); + __utee_call_elf_init_fn(); + } + + hcount++; + h->uuid = uuid; + return (void *)h; +err: + free(h); + return NULL; +} + +int dlclose(void *handle) +{ + free(handle); + hcount--; + if (!hcount && sess != TEE_HANDLE_NULL) { + TEE_CloseTASession(sess); + sess = TEE_HANDLE_NULL; + } + return 0; +} + +void *dlsym(void *handle, const char *symbol) +{ + TEE_Result res = TEE_ERROR_GENERIC; + TEE_Param params[TEE_NUM_PARAMS] = { }; + struct dl_handle *h = handle; + uint32_t param_types = 0; + void *ptr = NULL; + + if (!handle || !symbol) + return NULL; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE); + + params[0].memref.buffer = &h->uuid; + params[0].memref.size = sizeof(h->uuid); + params[1].memref.buffer = (void *)symbol; + params[1].memref.size = strlen(symbol) + 1; + + res = invoke_system_pta(PTA_SYSTEM_DLSYM, param_types, params); + if (!res) + ptr = (void *)(vaddr_t)reg_pair_to_64(params[2].value.a, + params[2].value.b); + + return ptr; +} + diff --git a/optee_os/lib/libdl/include/dlfcn.h b/optee_os/lib/libdl/include/dlfcn.h new file mode 100644 index 0000000..2110375 --- /dev/null +++ b/optee_os/lib/libdl/include/dlfcn.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019 Linaro limited + */ + +#ifndef _DLFCN_H_ +#define _DLFCN_H_ + +/* Relocations are performed when the object is loaded. */ +#define RTLD_NOW 2 +/* All symbols are available for relocation processing of other modules. */ +#define RTLD_GLOBAL 0x100 +/* Do not unload the shared object during dlclose(). */ +#define RTLD_NODELETE 0x1000 +/* Other flags are not supported */ + +/* Note: @flags must be (RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE) */ +void *dlopen(const char *filename, int flags); +int dlclose(void *handle); +void *dlsym(void *handle, const char *symbol); + +#endif /* _DLFCN_H_ */ diff --git a/optee_os/lib/libdl/sub.mk b/optee_os/lib/libdl/sub.mk new file mode 100644 index 0000000..ec94656 --- /dev/null +++ b/optee_os/lib/libdl/sub.mk @@ -0,0 +1,2 @@ +global-incdirs-y += include +srcs-y += dlfcn.c diff --git a/optee_os/lib/libmbedtls/core/aes.c b/optee_os/lib/libmbedtls/core/aes.c new file mode 100644 index 0000000..56ab50b --- /dev/null +++ b/optee_os/lib/libmbedtls/core/aes.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(MBEDTLS_AES_ALT) +void mbedtls_aes_init(mbedtls_aes_context *ctx) +{ + assert(ctx); + memset(ctx, 0, sizeof(*ctx)); +} + +void mbedtls_aes_free( mbedtls_aes_context *ctx ) +{ + if (ctx) + mbedtls_platform_zeroize(ctx, sizeof(*ctx)); +} + +int mbedtls_aes_setkey_enc(mbedtls_aes_context *ctx, const unsigned char *key, + unsigned int keybits) +{ + assert(ctx && key); + + if (keybits != 128 && keybits != 192 && keybits != 256) + return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH; + + if (crypto_accel_aes_expand_keys(key, keybits / 8, ctx->key, NULL, + sizeof(ctx->key), &ctx->round_count)) + return MBEDTLS_ERR_AES_BAD_INPUT_DATA; + + return 0; +} + +int mbedtls_aes_setkey_dec(mbedtls_aes_context *ctx, const unsigned char *key, + unsigned int keybits) +{ + uint32_t enc_key[sizeof(ctx->key)] = { 0 }; + + assert(ctx && key); + + if (keybits != 128 && keybits != 192 && keybits != 256) + return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH; + + if (crypto_accel_aes_expand_keys(key, keybits / 8, enc_key, ctx->key, + sizeof(ctx->key), &ctx->round_count)) + return MBEDTLS_ERR_AES_BAD_INPUT_DATA; + + return 0; +} +#endif /*MBEDTLS_AES_ALT*/ diff --git a/optee_os/lib/libmbedtls/core/aes_cbc.c b/optee_os/lib/libmbedtls/core/aes_cbc.c new file mode 100644 index 0000000..34a6f00 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/aes_cbc.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + * Copyright (C) 2021, Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" + +struct mbed_aes_cbc_ctx { + struct crypto_cipher_ctx ctx; + int mbed_mode; + mbedtls_aes_context aes_ctx; + unsigned char iv[TEE_AES_BLOCK_SIZE]; +}; + +static const struct crypto_cipher_ops mbed_aes_cbc_ops; + +static struct mbed_aes_cbc_ctx *to_aes_cbc_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_aes_cbc_ops); + + return container_of(ctx, struct mbed_aes_cbc_ctx, ctx); +} + +static TEE_Result mbed_aes_cbc_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv, size_t iv_len) +{ + struct mbed_aes_cbc_ctx *c = to_aes_cbc_ctx(ctx); + int mbed_res = 0; + + if (iv_len != sizeof(c->iv)) + return TEE_ERROR_BAD_PARAMETERS; + memcpy(c->iv, iv, sizeof(c->iv)); + + mbedtls_aes_init(&c->aes_ctx); + + if (mode == TEE_MODE_ENCRYPT) { + c->mbed_mode = MBEDTLS_AES_ENCRYPT; + mbed_res = mbedtls_aes_setkey_enc(&c->aes_ctx, key1, + key1_len * 8); + } else { + c->mbed_mode = MBEDTLS_AES_DECRYPT; + mbed_res = mbedtls_aes_setkey_dec(&c->aes_ctx, key1, + key1_len * 8); + } + + if (mbed_res) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_aes_cbc_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct mbed_aes_cbc_ctx *c = to_aes_cbc_ctx(ctx); + + if (mbedtls_aes_crypt_cbc(&c->aes_ctx, c->mbed_mode, len, c->iv, + data, dst)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static void mbed_aes_cbc_final(struct crypto_cipher_ctx *ctx) +{ + mbedtls_aes_free(&to_aes_cbc_ctx(ctx)->aes_ctx); +} + +static void mbed_aes_cbc_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_aes_cbc_ctx(ctx)); +} + +static void mbed_aes_cbc_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct mbed_aes_cbc_ctx *src = to_aes_cbc_ctx(src_ctx); + struct mbed_aes_cbc_ctx *dst = to_aes_cbc_ctx(dst_ctx); + + memcpy(dst->iv, src->iv, sizeof(dst->iv)); + dst->mbed_mode = src->mbed_mode; + mbed_copy_mbedtls_aes_context(&dst->aes_ctx, &src->aes_ctx); +} + +static const struct crypto_cipher_ops mbed_aes_cbc_ops = { + .init = mbed_aes_cbc_init, + .update = mbed_aes_cbc_update, + .final = mbed_aes_cbc_final, + .free_ctx = mbed_aes_cbc_free_ctx, + .copy_state = mbed_aes_cbc_copy_state, +}; + +TEE_Result crypto_aes_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct mbed_aes_cbc_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &mbed_aes_cbc_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} + +#if defined(MBEDTLS_AES_ALT) +int mbedtls_aes_crypt_cbc(mbedtls_aes_context *ctx, int mode, size_t length, + unsigned char iv[16], const unsigned char *input, + unsigned char *output) +{ + if (length % 16) + return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; + + if (mode == MBEDTLS_AES_ENCRYPT) + crypto_accel_aes_cbc_enc(output, input, ctx->key, + ctx->round_count, length / 16, iv); + else + crypto_accel_aes_cbc_dec(output, input, ctx->key, + ctx->round_count, length / 16, iv); + + return 0; +} +#endif /*MBEDTLS_AES_ALT*/ diff --git a/optee_os/lib/libmbedtls/core/aes_ctr.c b/optee_os/lib/libmbedtls/core/aes_ctr.c new file mode 100644 index 0000000..093d912 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/aes_ctr.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + * Copyright (C) 2021, Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" + +struct mbed_aes_ctr_ctx { + struct crypto_cipher_ctx ctx; + mbedtls_aes_context aes_ctx; + size_t nc_off; + unsigned char counter[TEE_AES_BLOCK_SIZE]; + unsigned char block[TEE_AES_BLOCK_SIZE]; +}; + +static const struct crypto_cipher_ops mbed_aes_ctr_ops; + +static struct mbed_aes_ctr_ctx *to_aes_ctr_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_aes_ctr_ops); + + return container_of(ctx, struct mbed_aes_ctr_ctx, ctx); +} + +static TEE_Result mbed_aes_ctr_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode __unused, + const uint8_t *key1, size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv, size_t iv_len) +{ + struct mbed_aes_ctr_ctx *c = to_aes_ctr_ctx(ctx); + + if (iv_len != sizeof(c->counter)) + return TEE_ERROR_BAD_PARAMETERS; + memcpy(c->counter, iv, sizeof(c->counter)); + + mbedtls_aes_init(&c->aes_ctx); + c->nc_off = 0; + + if (mbedtls_aes_setkey_enc(&c->aes_ctx, key1, key1_len * 8)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_aes_ctr_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct mbed_aes_ctr_ctx *c = to_aes_ctr_ctx(ctx); + + if (mbedtls_aes_crypt_ctr(&c->aes_ctx, len, &c->nc_off, c->counter, + c->block, data, dst)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static void mbed_aes_ctr_final(struct crypto_cipher_ctx *ctx) +{ + struct mbed_aes_ctr_ctx *c = to_aes_ctr_ctx(ctx); + + mbedtls_aes_free(&c->aes_ctx); + memset(c->block, 0, sizeof(c->block)); +} + +static void mbed_aes_ctr_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_aes_ctr_ctx(ctx)); +} + +static void mbed_aes_ctr_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct mbed_aes_ctr_ctx *src = to_aes_ctr_ctx(src_ctx); + struct mbed_aes_ctr_ctx *dst = to_aes_ctr_ctx(dst_ctx); + + memcpy(dst->counter, src->counter, sizeof(dst->counter)); + memcpy(dst->block, src->block, sizeof(dst->block)); + dst->nc_off = src->nc_off; + mbed_copy_mbedtls_aes_context(&dst->aes_ctx, &src->aes_ctx); +} + +static const struct crypto_cipher_ops mbed_aes_ctr_ops = { + .init = mbed_aes_ctr_init, + .update = mbed_aes_ctr_update, + .final = mbed_aes_ctr_final, + .free_ctx = mbed_aes_ctr_free_ctx, + .copy_state = mbed_aes_ctr_copy_state, +}; + +TEE_Result crypto_aes_ctr_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct mbed_aes_ctr_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &mbed_aes_ctr_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} + +#if defined(MBEDTLS_AES_ALT) +static void next_ctr(unsigned char stream_block[16], mbedtls_aes_context *ctx, + unsigned char nonce_counter[16]) +{ + const unsigned char zeroes[16] = { 0 }; + + crypto_accel_aes_ctr_be_enc(stream_block, zeroes, ctx->key, + ctx->round_count, 1, nonce_counter); +} + +int mbedtls_aes_crypt_ctr(mbedtls_aes_context *ctx, size_t length, + size_t *nc_off, unsigned char nonce_counter[16], + unsigned char stream_block[16], + const unsigned char *input, unsigned char *output) +{ + size_t offs = 0; + + if (*nc_off >= 16) + return MBEDTLS_ERR_AES_BAD_INPUT_DATA; + + /* + * If the stream_block is in use, continue until done or + * stream_block is consumed. + */ + while (*nc_off) { + output[offs] = stream_block[*nc_off] ^ input[offs]; + offs++; + *nc_off = (*nc_off + 1) % 16; + if (offs == length) + return 0; + } + + if ((length - offs) >= 16) { + size_t block_count = (length - offs) / 16; + + crypto_accel_aes_ctr_be_enc(output + offs, input + offs, + ctx->key, ctx->round_count, + block_count, nonce_counter); + offs += block_count * 16; + } + + while (offs < length) { + if (!*nc_off) + next_ctr(stream_block, ctx, nonce_counter); + output[offs] = stream_block[*nc_off] ^ input[offs]; + offs++; + *nc_off = (*nc_off + 1) % 16; + } + + return 0; +} +#endif /*MBEDTLS_AES_ALT*/ diff --git a/optee_os/lib/libmbedtls/core/aes_ecb.c b/optee_os/lib/libmbedtls/core/aes_ecb.c new file mode 100644 index 0000000..934befa --- /dev/null +++ b/optee_os/lib/libmbedtls/core/aes_ecb.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + * Copyright (C) 2021, Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" + +struct mbed_aes_ecb_ctx { + struct crypto_cipher_ctx ctx; + int mbed_mode; + mbedtls_aes_context aes_ctx; +}; + +static const struct crypto_cipher_ops mbed_aes_ecb_ops; + +static struct mbed_aes_ecb_ctx *to_aes_ecb_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_aes_ecb_ops); + + return container_of(ctx, struct mbed_aes_ecb_ctx, ctx); +} + +static TEE_Result mbed_aes_ecb_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct mbed_aes_ecb_ctx *c = to_aes_ecb_ctx(ctx); + int mbed_res = 0; + + mbedtls_aes_init(&c->aes_ctx); + + if (mode == TEE_MODE_ENCRYPT) { + c->mbed_mode = MBEDTLS_AES_ENCRYPT; + mbed_res = mbedtls_aes_setkey_enc(&c->aes_ctx, key1, + key1_len * 8); + } else { + c->mbed_mode = MBEDTLS_AES_DECRYPT; + mbed_res = mbedtls_aes_setkey_dec(&c->aes_ctx, key1, + key1_len * 8); + } + + if (mbed_res) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_aes_ecb_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct mbed_aes_ecb_ctx *c = to_aes_ecb_ctx(ctx); + size_t block_size = TEE_AES_BLOCK_SIZE; + size_t offs = 0; + + if (len % block_size) + return TEE_ERROR_BAD_PARAMETERS; + + for (offs = 0; offs < len; offs += block_size) { + if (mbedtls_aes_crypt_ecb(&c->aes_ctx, c->mbed_mode, + data + offs, dst + offs)) + return TEE_ERROR_BAD_STATE; + } + + return TEE_SUCCESS; +} + +static void mbed_aes_ecb_final(struct crypto_cipher_ctx *ctx) +{ + mbedtls_aes_free(&to_aes_ecb_ctx(ctx)->aes_ctx); +} + +static void mbed_aes_ecb_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_aes_ecb_ctx(ctx)); +} + +static void mbed_aes_ecb_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct mbed_aes_ecb_ctx *src = to_aes_ecb_ctx(src_ctx); + struct mbed_aes_ecb_ctx *dst = to_aes_ecb_ctx(dst_ctx); + + dst->mbed_mode = src->mbed_mode; + mbed_copy_mbedtls_aes_context(&dst->aes_ctx, &src->aes_ctx); +} + +static const struct crypto_cipher_ops mbed_aes_ecb_ops = { + .init = mbed_aes_ecb_init, + .update = mbed_aes_ecb_update, + .final = mbed_aes_ecb_final, + .free_ctx = mbed_aes_ecb_free_ctx, + .copy_state = mbed_aes_ecb_copy_state, +}; + +TEE_Result crypto_aes_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct mbed_aes_ecb_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &mbed_aes_ecb_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} + +#if defined(MBEDTLS_AES_ALT) +int mbedtls_aes_crypt_ecb(mbedtls_aes_context *ctx, int mode, + const unsigned char input[16], + unsigned char output[16]) + +{ + if (mode == MBEDTLS_AES_ENCRYPT) + crypto_accel_aes_ecb_enc(output, input, ctx->key, + ctx->round_count, 1); + else + crypto_accel_aes_ecb_dec(output, input, ctx->key, + ctx->round_count, 1); + + return 0; +} +#endif /*MBEDTLS_AES_ALT*/ diff --git a/optee_os/lib/libmbedtls/core/bignum.c b/optee_os/lib/libmbedtls/core/bignum.c new file mode 100644 index 0000000..dea30f6 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/bignum.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#define ciL (sizeof(mbedtls_mpi_uint)) /* chars in limb */ +#define biL (ciL << 3) /* bits in limb */ +#define BITS_TO_LIMBS(i) ((i) / biL + ((i) % biL != 0)) + +size_t crypto_bignum_num_bytes(struct bignum *a) +{ + assert(a != NULL); + return mbedtls_mpi_size((const mbedtls_mpi *)a); +} + +size_t crypto_bignum_num_bits(struct bignum *a) +{ + assert(a != NULL); + return mbedtls_mpi_bitlen((const mbedtls_mpi *)a); +} + +int32_t crypto_bignum_compare(struct bignum *a, struct bignum *b) +{ + int ret = 0; + + assert(a != NULL); + assert(b != NULL); + ret = mbedtls_mpi_cmp_mpi((const mbedtls_mpi *)a, + (const mbedtls_mpi *)b); + return CMP_TRILEAN(ret, 0); +} + +void crypto_bignum_bn2bin(const struct bignum *from, uint8_t *to) +{ + size_t len = 0; + + assert(from != NULL); + assert(to != NULL); + len = crypto_bignum_num_bytes((struct bignum *)from); + if (mbedtls_mpi_write_binary((mbedtls_mpi *)from, to, len)) + panic(); +} + +TEE_Result crypto_bignum_bin2bn(const uint8_t *from, size_t fromsize, + struct bignum *to) +{ + assert(from != NULL); + assert(to != NULL); + if (mbedtls_mpi_read_binary((mbedtls_mpi *)to, from, fromsize)) + return TEE_ERROR_BAD_PARAMETERS; + return TEE_SUCCESS; +} + +void crypto_bignum_copy(struct bignum *to, const struct bignum *from) +{ + assert(from != NULL); + assert(to != NULL); + if (mbedtls_mpi_copy((mbedtls_mpi *)to, (const mbedtls_mpi *)from)) + panic(); +} + +struct bignum *crypto_bignum_allocate(size_t size_bits) +{ + mbedtls_mpi *bn = NULL; + + if (size_bits > CFG_CORE_BIGNUM_MAX_BITS) + size_bits = CFG_CORE_BIGNUM_MAX_BITS; + + bn = calloc(1, sizeof(mbedtls_mpi)); + if (!bn) + return NULL; + mbedtls_mpi_init(bn); + if (mbedtls_mpi_grow(bn, BITS_TO_LIMBS(size_bits)) != 0) { + free(bn); + return NULL; + } + + return (struct bignum *)bn; +} + +void crypto_bignum_free(struct bignum **s) +{ + assert(s); + + mbedtls_mpi_free((mbedtls_mpi *)*s); + free(*s); + *s = NULL; +} + +void crypto_bignum_clear(struct bignum *s) +{ + mbedtls_mpi *bn = (mbedtls_mpi *)s; + + memset(bn->p, 0, mbedtls_mpi_size((const mbedtls_mpi *)bn)); +} diff --git a/optee_os/lib/libmbedtls/core/cmac.c b/optee_os/lib/libmbedtls/core/cmac.c new file mode 100644 index 0000000..53878ba --- /dev/null +++ b/optee_os/lib/libmbedtls/core/cmac.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + * Copyright (C) 2021, SumUp Services GmbH + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mbed_cmac_ctx { + struct crypto_mac_ctx mac_ctx; + mbedtls_cipher_context_t cipher_ctx; + mbedtls_cipher_id_t cipher_id; +}; + +static const struct crypto_mac_ops mbed_cmac_ops; + +static struct mbed_cmac_ctx *to_cmac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx); + assert(ctx->ops == &mbed_cmac_ops); + + return container_of(ctx, struct mbed_cmac_ctx, mac_ctx); +} + +static TEE_Result mbed_cmac_init(struct crypto_mac_ctx *ctx, + const uint8_t *key, size_t len) +{ + struct mbed_cmac_ctx *c = to_cmac_ctx(ctx); + const mbedtls_cipher_info_t *cipher_info = NULL; + + cipher_info = mbedtls_cipher_info_from_values(c->cipher_id, + len * 8, + MBEDTLS_MODE_ECB); + if (!cipher_info) + return TEE_ERROR_NOT_SUPPORTED; + + if (mbedtls_cipher_setup_info(&c->cipher_ctx, cipher_info)) + return TEE_ERROR_BAD_STATE; + + if (mbedtls_cipher_cmac_reset(&c->cipher_ctx)) + return TEE_ERROR_BAD_STATE; + + if (mbedtls_cipher_cmac_starts(&c->cipher_ctx, key, len * 8)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_cmac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + struct mbed_cmac_ctx *c = to_cmac_ctx(ctx); + + if (mbedtls_cipher_cmac_update(&c->cipher_ctx, data, len)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_cmac_final(struct crypto_mac_ctx *ctx, + uint8_t *digest, size_t len) +{ + struct mbed_cmac_ctx *c = to_cmac_ctx(ctx); + size_t block_size = TEE_AES_BLOCK_SIZE; + uint8_t block_digest[TEE_AES_BLOCK_SIZE] = { }; + uint8_t *tmp_digest = NULL; + + COMPILE_TIME_ASSERT(TEE_AES_BLOCK_SIZE >= TEE_DES_BLOCK_SIZE); + + if (len == 0) + return TEE_ERROR_BAD_PARAMETERS; + + if (c->cipher_id == MBEDTLS_CIPHER_ID_3DES) + block_size = TEE_DES_BLOCK_SIZE; + + if (len < block_size) + tmp_digest = block_digest; /* use a tempory buffer */ + else + tmp_digest = digest; + + if (mbedtls_cipher_cmac_finish(&c->cipher_ctx, tmp_digest)) + return TEE_ERROR_BAD_STATE; + + if (len < block_size) + memcpy(digest, tmp_digest, len); + + return TEE_SUCCESS; +} + +static void mbed_cmac_free_ctx(struct crypto_mac_ctx *ctx) +{ + struct mbed_cmac_ctx *c = to_cmac_ctx(ctx); + + mbedtls_cipher_free(&c->cipher_ctx); + free(c); +} + +static void mbed_cmac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct mbed_cmac_ctx *src = to_cmac_ctx(src_ctx); + struct mbed_cmac_ctx *dst = to_cmac_ctx(dst_ctx); + + if (mbedtls_cipher_clone(&dst->cipher_ctx, &src->cipher_ctx)) + panic(); +} + +static const struct crypto_mac_ops mbed_cmac_ops = { + .init = mbed_cmac_init, + .update = mbed_cmac_update, + .final = mbed_cmac_final, + .free_ctx = mbed_cmac_free_ctx, + .copy_state = mbed_cmac_copy_state, +}; + +static TEE_Result crypto_cmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret, + const mbedtls_cipher_id_t cipher_id, + int key_bitlen) +{ + int mbed_res = 0; + struct mbed_cmac_ctx *c = NULL; + const mbedtls_cipher_info_t *cipher_info = NULL; + + /* + * Use a default key length for getting 'cipher_info' to do the + * setup. The 'cipher_info' will need to be re-assigned with final + * key length obtained in mbed_cmac_init() above. + * + * This is safe since 'mbedtls_cipher_base_t' (used for cipher + * context) uses the same fixed allocation all key lengths. + */ + cipher_info = mbedtls_cipher_info_from_values(cipher_id, key_bitlen, + MBEDTLS_MODE_ECB); + if (!cipher_info) + return TEE_ERROR_NOT_SUPPORTED; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->cipher_id = cipher_id; + c->mac_ctx.ops = &mbed_cmac_ops; + + mbedtls_cipher_init(&c->cipher_ctx); + mbed_res = mbedtls_cipher_setup(&c->cipher_ctx, cipher_info); + if (mbed_res) { + free(c); + if (mbed_res == MBEDTLS_ERR_CIPHER_ALLOC_FAILED) + return TEE_ERROR_OUT_OF_MEMORY; + return TEE_ERROR_NOT_SUPPORTED; + } + mbed_res = mbedtls_cipher_cmac_setup(&c->cipher_ctx); + if (mbed_res) { + free(c); + return TEE_ERROR_NOT_SUPPORTED; + } + + *ctx_ret = &c->mac_ctx; + + return TEE_SUCCESS; +} + +TEE_Result crypto_des3_cmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret) +{ + return crypto_cmac_alloc_ctx(ctx_ret, MBEDTLS_CIPHER_ID_3DES, 192); +} + +TEE_Result crypto_aes_cmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret) +{ + return crypto_cmac_alloc_ctx(ctx_ret, MBEDTLS_CIPHER_ID_AES, 128); +} diff --git a/optee_os/lib/libmbedtls/core/des3_cbc.c b/optee_os/lib/libmbedtls/core/des3_cbc.c new file mode 100644 index 0000000..17dba17 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/des3_cbc.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mbed_des3_cbc_ctx { + struct crypto_cipher_ctx ctx; + int mbed_mode; + mbedtls_des3_context des3_ctx; + unsigned char iv[TEE_DES_BLOCK_SIZE]; +}; + +static const struct crypto_cipher_ops mbed_des3_cbc_ops; + +static struct mbed_des3_cbc_ctx *to_des3_cbc_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_des3_cbc_ops); + + return container_of(ctx, struct mbed_des3_cbc_ctx, ctx); +} + +static TEE_Result mbed_des3_cbc_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, + const uint8_t *key1, size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct mbed_des3_cbc_ctx *c = to_des3_cbc_ctx(ctx); + int mbed_res = 0; + + if (key1_len != MBEDTLS_DES_KEY_SIZE * 2 && + key1_len != MBEDTLS_DES_KEY_SIZE * 3) + return TEE_ERROR_BAD_PARAMETERS; + if (iv_len != sizeof(c->iv)) + return TEE_ERROR_BAD_PARAMETERS; + memcpy(c->iv, iv, sizeof(c->iv)); + + mbedtls_des3_init(&c->des3_ctx); + + if (mode == TEE_MODE_ENCRYPT) { + c->mbed_mode = MBEDTLS_DES_ENCRYPT; + if (key1_len == MBEDTLS_DES_KEY_SIZE * 3) + mbed_res = mbedtls_des3_set3key_enc(&c->des3_ctx, key1); + else + mbed_res = mbedtls_des3_set2key_enc(&c->des3_ctx, key1); + } else { + c->mbed_mode = MBEDTLS_DES_DECRYPT; + if (key1_len == MBEDTLS_DES_KEY_SIZE * 3) + mbed_res = mbedtls_des3_set3key_dec(&c->des3_ctx, key1); + else + mbed_res = mbedtls_des3_set2key_dec(&c->des3_ctx, key1); + } + + if (mbed_res) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_des3_cbc_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct mbed_des3_cbc_ctx *c = to_des3_cbc_ctx(ctx); + + if (mbedtls_des3_crypt_cbc(&c->des3_ctx, c->mbed_mode, len, c->iv, + data, dst)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static void mbed_des3_cbc_final(struct crypto_cipher_ctx *ctx) +{ + mbedtls_des3_free(&to_des3_cbc_ctx(ctx)->des3_ctx); +} + +static void mbed_des3_cbc_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_des3_cbc_ctx(ctx)); +} + +static void mbed_des3_cbc_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct mbed_des3_cbc_ctx *src = to_des3_cbc_ctx(src_ctx); + struct mbed_des3_cbc_ctx *dst = to_des3_cbc_ctx(dst_ctx); + + memcpy(dst->iv, src->iv, sizeof(dst->iv)); + dst->mbed_mode = src->mbed_mode; + dst->des3_ctx = src->des3_ctx; +} + +static const struct crypto_cipher_ops mbed_des3_cbc_ops = { + .init = mbed_des3_cbc_init, + .update = mbed_des3_cbc_update, + .final = mbed_des3_cbc_final, + .free_ctx = mbed_des3_cbc_free_ctx, + .copy_state = mbed_des3_cbc_copy_state, +}; + +TEE_Result crypto_des3_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct mbed_des3_cbc_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &mbed_des3_cbc_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/lib/libmbedtls/core/des3_ecb.c b/optee_os/lib/libmbedtls/core/des3_ecb.c new file mode 100644 index 0000000..56e3b8d --- /dev/null +++ b/optee_os/lib/libmbedtls/core/des3_ecb.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mbed_des3_ecb_ctx { + struct crypto_cipher_ctx ctx; + mbedtls_des3_context des3_ctx; +}; + +static const struct crypto_cipher_ops mbed_des3_ecb_ops; + +static struct mbed_des3_ecb_ctx *to_des3_ecb_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_des3_ecb_ops); + + return container_of(ctx, struct mbed_des3_ecb_ctx, ctx); +} + +static TEE_Result mbed_des3_ecb_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, + const uint8_t *key1, size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct mbed_des3_ecb_ctx *c = to_des3_ecb_ctx(ctx); + int mbed_res = 0; + + if (key1_len != MBEDTLS_DES_KEY_SIZE * 2 && + key1_len != MBEDTLS_DES_KEY_SIZE * 3) + return TEE_ERROR_BAD_PARAMETERS; + + mbedtls_des3_init(&c->des3_ctx); + + if (key1_len == MBEDTLS_DES_KEY_SIZE * 3) { + if (mode == TEE_MODE_ENCRYPT) + mbed_res = mbedtls_des3_set3key_enc(&c->des3_ctx, key1); + else + mbed_res = mbedtls_des3_set3key_dec(&c->des3_ctx, key1); + } else { + if (mode == TEE_MODE_ENCRYPT) + mbed_res = mbedtls_des3_set2key_enc(&c->des3_ctx, key1); + else + mbed_res = mbedtls_des3_set2key_dec(&c->des3_ctx, key1); + } + + if (mbed_res) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_des3_ecb_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct mbed_des3_ecb_ctx *c = to_des3_ecb_ctx(ctx); + size_t block_size = TEE_DES_BLOCK_SIZE; + size_t offs = 0; + + if (len % block_size) + return TEE_ERROR_BAD_PARAMETERS; + + for (offs = 0; offs < len; offs += block_size) { + if (mbedtls_des3_crypt_ecb(&c->des3_ctx, data + offs, + dst + offs)) + return TEE_ERROR_BAD_STATE; + } + + return TEE_SUCCESS; +} + +static void mbed_des3_ecb_final(struct crypto_cipher_ctx *ctx) +{ + mbedtls_des3_free(&to_des3_ecb_ctx(ctx)->des3_ctx); +} + +static void mbed_des3_ecb_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_des3_ecb_ctx(ctx)); +} + +static void mbed_des3_ecb_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct mbed_des3_ecb_ctx *src = to_des3_ecb_ctx(src_ctx); + struct mbed_des3_ecb_ctx *dst = to_des3_ecb_ctx(dst_ctx); + + dst->des3_ctx = src->des3_ctx; +} + +static const struct crypto_cipher_ops mbed_des3_ecb_ops = { + .init = mbed_des3_ecb_init, + .update = mbed_des3_ecb_update, + .final = mbed_des3_ecb_final, + .free_ctx = mbed_des3_ecb_free_ctx, + .copy_state = mbed_des3_ecb_copy_state, +}; + +TEE_Result crypto_des3_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct mbed_des3_ecb_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &mbed_des3_ecb_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/lib/libmbedtls/core/des_cbc.c b/optee_os/lib/libmbedtls/core/des_cbc.c new file mode 100644 index 0000000..a79b794 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/des_cbc.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mbed_des_cbc_ctx { + struct crypto_cipher_ctx ctx; + int mbed_mode; + mbedtls_des_context des_ctx; + unsigned char iv[TEE_DES_BLOCK_SIZE]; +}; + +static const struct crypto_cipher_ops mbed_des_cbc_ops; + +static struct mbed_des_cbc_ctx *to_des_cbc_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_des_cbc_ops); + + return container_of(ctx, struct mbed_des_cbc_ctx, ctx); +} + +static TEE_Result mbed_des_cbc_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct mbed_des_cbc_ctx *c = to_des_cbc_ctx(ctx); + int mbed_res = 0; + + if (key1_len != MBEDTLS_DES_KEY_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + if (iv_len != sizeof(c->iv)) + return TEE_ERROR_BAD_PARAMETERS; + memcpy(c->iv, iv, sizeof(c->iv)); + + mbedtls_des_init(&c->des_ctx); + + if (mode == TEE_MODE_ENCRYPT) { + c->mbed_mode = MBEDTLS_DES_ENCRYPT; + mbed_res = mbedtls_des_setkey_enc(&c->des_ctx, key1); + } else { + c->mbed_mode = MBEDTLS_DES_DECRYPT; + mbed_res = mbedtls_des_setkey_dec(&c->des_ctx, key1); + } + + if (mbed_res) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_des_cbc_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct mbed_des_cbc_ctx *c = to_des_cbc_ctx(ctx); + + if (mbedtls_des_crypt_cbc(&c->des_ctx, c->mbed_mode, len, c->iv, + data, dst)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static void mbed_des_cbc_final(struct crypto_cipher_ctx *ctx) +{ + mbedtls_des_free(&to_des_cbc_ctx(ctx)->des_ctx); +} + +static void mbed_des_cbc_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_des_cbc_ctx(ctx)); +} + +static void mbed_des_cbc_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct mbed_des_cbc_ctx *src = to_des_cbc_ctx(src_ctx); + struct mbed_des_cbc_ctx *dst = to_des_cbc_ctx(dst_ctx); + + memcpy(dst->iv, src->iv, sizeof(dst->iv)); + dst->mbed_mode = src->mbed_mode; + dst->des_ctx = src->des_ctx; +} + +static const struct crypto_cipher_ops mbed_des_cbc_ops = { + .init = mbed_des_cbc_init, + .update = mbed_des_cbc_update, + .final = mbed_des_cbc_final, + .free_ctx = mbed_des_cbc_free_ctx, + .copy_state = mbed_des_cbc_copy_state, +}; + +TEE_Result crypto_des_cbc_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct mbed_des_cbc_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &mbed_des_cbc_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/lib/libmbedtls/core/des_ecb.c b/optee_os/lib/libmbedtls/core/des_ecb.c new file mode 100644 index 0000000..38823a6 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/des_ecb.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mbed_des_ecb_ctx { + struct crypto_cipher_ctx ctx; + mbedtls_des_context des_ctx; +}; + +static const struct crypto_cipher_ops mbed_des_ecb_ops; + +static struct mbed_des_ecb_ctx *to_des_ecb_ctx(struct crypto_cipher_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_des_ecb_ops); + + return container_of(ctx, struct mbed_des_ecb_ctx, ctx); +} + +static TEE_Result mbed_des_ecb_init(struct crypto_cipher_ctx *ctx, + TEE_OperationMode mode, const uint8_t *key1, + size_t key1_len, + const uint8_t *key2 __unused, + size_t key2_len __unused, + const uint8_t *iv __unused, + size_t iv_len __unused) +{ + struct mbed_des_ecb_ctx *c = to_des_ecb_ctx(ctx); + int mbed_res = 0; + + if (key1_len != MBEDTLS_DES_KEY_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + mbedtls_des_init(&c->des_ctx); + + if (mode == TEE_MODE_ENCRYPT) + mbed_res = mbedtls_des_setkey_enc(&c->des_ctx, key1); + else + mbed_res = mbedtls_des_setkey_dec(&c->des_ctx, key1); + + if (mbed_res) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_des_ecb_update(struct crypto_cipher_ctx *ctx, + bool last_block __unused, + const uint8_t *data, size_t len, + uint8_t *dst) +{ + struct mbed_des_ecb_ctx *c = to_des_ecb_ctx(ctx); + size_t block_size = TEE_DES_BLOCK_SIZE; + size_t offs = 0; + + if (len % block_size) + return TEE_ERROR_BAD_PARAMETERS; + + for (offs = 0; offs < len; offs += block_size) { + if (mbedtls_des_crypt_ecb(&c->des_ctx, data + offs, dst + offs)) + return TEE_ERROR_BAD_STATE; + } + + return TEE_SUCCESS; +} + +static void mbed_des_ecb_final(struct crypto_cipher_ctx *ctx) +{ + mbedtls_des_free(&to_des_ecb_ctx(ctx)->des_ctx); +} + +static void mbed_des_ecb_free_ctx(struct crypto_cipher_ctx *ctx) +{ + free(to_des_ecb_ctx(ctx)); +} + +static void mbed_des_ecb_copy_state(struct crypto_cipher_ctx *dst_ctx, + struct crypto_cipher_ctx *src_ctx) +{ + struct mbed_des_ecb_ctx *src = to_des_ecb_ctx(src_ctx); + struct mbed_des_ecb_ctx *dst = to_des_ecb_ctx(dst_ctx); + + dst->des_ctx = src->des_ctx; +} + +static const struct crypto_cipher_ops mbed_des_ecb_ops = { + .init = mbed_des_ecb_init, + .update = mbed_des_ecb_update, + .final = mbed_des_ecb_final, + .free_ctx = mbed_des_ecb_free_ctx, + .copy_state = mbed_des_ecb_copy_state, +}; + +TEE_Result crypto_des_ecb_alloc_ctx(struct crypto_cipher_ctx **ctx_ret) +{ + struct mbed_des_ecb_ctx *c = NULL; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->ctx.ops = &mbed_des_ecb_ops; + *ctx_ret = &c->ctx; + + return TEE_SUCCESS; +} diff --git a/optee_os/lib/libmbedtls/core/dh.c b/optee_os/lib/libmbedtls/core/dh.c new file mode 100644 index 0000000..25c49cc --- /dev/null +++ b/optee_os/lib/libmbedtls/core/dh.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" + +TEE_Result crypto_acipher_alloc_dh_keypair(struct dh_keypair *s, + size_t key_size_bits) +{ + memset(s, 0, sizeof(*s)); + s->g = crypto_bignum_allocate(key_size_bits); + if (!s->g) + goto err; + s->p = crypto_bignum_allocate(key_size_bits); + if (!s->p) + goto err; + s->y = crypto_bignum_allocate(key_size_bits); + if (!s->y) + goto err; + s->x = crypto_bignum_allocate(key_size_bits); + if (!s->x) + goto err; + s->q = crypto_bignum_allocate(key_size_bits); + if (!s->q) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->g); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->y); + crypto_bignum_free(&s->x); + return TEE_ERROR_OUT_OF_MEMORY; +} + +TEE_Result crypto_acipher_gen_dh_key(struct dh_keypair *key, + struct bignum *q __unused, + size_t xbits, size_t key_size) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + mbedtls_dhm_context dhm; + unsigned char *buf = NULL; + size_t xbytes = 0; + size_t len = 0; + + memset(&dhm, 0, sizeof(dhm)); + mbedtls_dhm_init(&dhm); + + dhm.G = *(mbedtls_mpi *)key->g; + dhm.P = *(mbedtls_mpi *)key->p; + + len = mbedtls_dhm_get_len(&dhm); + if (key_size != 8 * len) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (xbits == 0) + xbytes = len; + else + xbytes = xbits / 8; + + buf = malloc(len); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + lmd_res = mbedtls_dhm_make_public(&dhm, (int)xbytes, buf, + len, mbd_rand, NULL); + if (lmd_res != 0) { + FMSG("mbedtls_dhm_make_public err, return is 0x%x", -lmd_res); + res = TEE_ERROR_BAD_PARAMETERS; + } else { + crypto_bignum_bin2bn(buf, xbytes, key->y); + crypto_bignum_copy(key->x, (void *)&dhm.X); + res = TEE_SUCCESS; + } +out: + free(buf); + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&dhm.G); + mbedtls_mpi_init(&dhm.P); + mbedtls_dhm_free(&dhm); + return res; +} + +TEE_Result crypto_acipher_dh_shared_secret(struct dh_keypair *private_key, + struct bignum *public_key, + struct bignum *secret) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + mbedtls_dhm_context dhm; + unsigned char *buf = NULL; + size_t olen = 0; + size_t len = 0; + + memset(&dhm, 0, sizeof(dhm)); + mbedtls_dhm_init(&dhm); + + dhm.G = *(mbedtls_mpi *)private_key->g; + dhm.P = *(mbedtls_mpi *)private_key->p; + dhm.GX = *(mbedtls_mpi *)private_key->y; + dhm.X = *(mbedtls_mpi *)private_key->x; + dhm.GY = *(mbedtls_mpi *)public_key; + + len = mbedtls_dhm_get_len(&dhm); + + buf = malloc(len); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + lmd_res = mbedtls_dhm_calc_secret(&dhm, buf, len, + &olen, mbd_rand, NULL); + if (lmd_res != 0) { + FMSG("mbedtls_dhm_calc_secret failed, ret is 0x%x", -lmd_res); + res = TEE_ERROR_BAD_PARAMETERS; + } else { + crypto_bignum_bin2bn(buf, olen, secret); + res = TEE_SUCCESS; + } +out: + free(buf); + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&dhm.G); + mbedtls_mpi_init(&dhm.P); + mbedtls_mpi_init(&dhm.GX); + mbedtls_mpi_init(&dhm.X); + mbedtls_mpi_init(&dhm.GY); + mbedtls_dhm_free(&dhm); + return res; +} diff --git a/optee_os/lib/libmbedtls/core/ecc.c b/optee_os/lib/libmbedtls/core/ecc.c new file mode 100644 index 0000000..39adf0e --- /dev/null +++ b/optee_os/lib/libmbedtls/core/ecc.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" +#include "sm2-dsa.h" +#include "sm2-pke.h" + +/* Translate mbedtls result to TEE result */ +static TEE_Result get_tee_result(int lmd_res) +{ + switch (lmd_res) { + case 0: + return TEE_SUCCESS; + case MBEDTLS_ERR_ECP_VERIFY_FAILED: + return TEE_ERROR_SIGNATURE_INVALID; + case MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL: + return TEE_ERROR_SHORT_BUFFER; + default: + return TEE_ERROR_BAD_STATE; + } +} + +static void ecc_free_public_key(struct ecc_public_key *s) +{ + if (!s) + return; + + crypto_bignum_free(&s->x); + crypto_bignum_free(&s->y); +} + +static TEE_Result ecc_get_keysize(uint32_t curve, uint32_t algo, + size_t *key_size_bytes, size_t *key_size_bits) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + *key_size_bits = 192; + *key_size_bytes = 24; + break; + case TEE_ECC_CURVE_NIST_P224: + *key_size_bits = 224; + *key_size_bytes = 28; + break; + case TEE_ECC_CURVE_NIST_P256: + *key_size_bits = 256; + *key_size_bytes = 32; + break; + case TEE_ECC_CURVE_NIST_P384: + *key_size_bits = 384; + *key_size_bytes = 48; + break; + case TEE_ECC_CURVE_NIST_P521: + *key_size_bits = 521; + *key_size_bytes = 66; + break; + case TEE_ECC_CURVE_SM2: + *key_size_bits = 256; + *key_size_bytes = 32; + if (algo != 0 && algo != TEE_ALG_SM2_DSA_SM3 && + algo != TEE_ALG_SM2_KEP && algo != TEE_ALG_SM2_PKE) + return TEE_ERROR_BAD_PARAMETERS; + break; + default: + *key_size_bits = 0; + *key_size_bytes = 0; + return TEE_ERROR_NOT_SUPPORTED; + } + + return TEE_SUCCESS; +} + +static mbedtls_ecp_group_id curve_to_group_id(uint32_t curve) +{ + switch (curve) { + case TEE_ECC_CURVE_NIST_P192: + return MBEDTLS_ECP_DP_SECP192R1; + case TEE_ECC_CURVE_NIST_P224: + return MBEDTLS_ECP_DP_SECP224R1; + case TEE_ECC_CURVE_NIST_P256: + return MBEDTLS_ECP_DP_SECP256R1; + case TEE_ECC_CURVE_NIST_P384: + return MBEDTLS_ECP_DP_SECP384R1; + case TEE_ECC_CURVE_NIST_P521: + return MBEDTLS_ECP_DP_SECP521R1; + case TEE_ECC_CURVE_SM2: + return MBEDTLS_ECP_DP_SM2; + default: + return MBEDTLS_ECP_DP_NONE; + } +} + +static TEE_Result ecc_generate_keypair(struct ecc_keypair *key, size_t key_size) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + mbedtls_ecdsa_context ecdsa; + mbedtls_ecp_group_id gid; + size_t key_size_bytes = 0; + size_t key_size_bits = 0; + + memset(&ecdsa, 0, sizeof(ecdsa)); + memset(&gid, 0, sizeof(gid)); + + res = ecc_get_keysize(key->curve, 0, &key_size_bytes, &key_size_bits); + if (res != TEE_SUCCESS) + return res; + + if (key_size != key_size_bits) + return TEE_ERROR_BAD_PARAMETERS; + + mbedtls_ecdsa_init(&ecdsa); + + /* Generate the ECC key */ + gid = curve_to_group_id(key->curve); + lmd_res = mbedtls_ecdsa_genkey(&ecdsa, gid, mbd_rand, NULL); + if (lmd_res != 0) { + res = TEE_ERROR_BAD_PARAMETERS; + FMSG("mbedtls_ecdsa_genkey failed."); + goto exit; + } + + /* check the size of the keys */ + if ((mbedtls_mpi_bitlen(&ecdsa.Q.X) > key_size_bits) || + (mbedtls_mpi_bitlen(&ecdsa.Q.Y) > key_size_bits) || + (mbedtls_mpi_bitlen(&ecdsa.d) > key_size_bits)) { + res = TEE_ERROR_BAD_PARAMETERS; + FMSG("Check the size of the keys failed."); + goto exit; + } + + /* check LMD is returning z==1 */ + if (mbedtls_mpi_bitlen(&ecdsa.Q.Z) != 1) { + res = TEE_ERROR_BAD_PARAMETERS; + FMSG("Check LMD failed."); + goto exit; + } + + /* Copy the key */ + crypto_bignum_copy(key->d, (void *)&ecdsa.d); + crypto_bignum_copy(key->x, (void *)&ecdsa.Q.X); + crypto_bignum_copy(key->y, (void *)&ecdsa.Q.Y); + + res = TEE_SUCCESS; +exit: + mbedtls_ecdsa_free(&ecdsa); /* Free the temporary key */ + return res; +} + +static TEE_Result ecc_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, uint8_t *sig, + size_t *sig_len) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + const mbedtls_pk_info_t *pk_info = NULL; + mbedtls_ecdsa_context ecdsa; + mbedtls_ecp_group_id gid; + size_t key_size_bytes = 0; + size_t key_size_bits = 0; + mbedtls_mpi r; + mbedtls_mpi s; + + memset(&ecdsa, 0, sizeof(ecdsa)); + memset(&gid, 0, sizeof(gid)); + memset(&r, 0, sizeof(r)); + memset(&s, 0, sizeof(s)); + + if (algo == 0) + return TEE_ERROR_BAD_PARAMETERS; + + mbedtls_mpi_init(&r); + mbedtls_mpi_init(&s); + + mbedtls_ecdsa_init(&ecdsa); + + gid = curve_to_group_id(key->curve); + lmd_res = mbedtls_ecp_group_load(&ecdsa.grp, gid); + if (lmd_res != 0) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + ecdsa.d = *(mbedtls_mpi *)key->d; + + res = ecc_get_keysize(key->curve, algo, &key_size_bytes, + &key_size_bits); + if (res != TEE_SUCCESS) + goto out; + + if (*sig_len < 2 * key_size_bytes) { + *sig_len = 2 * key_size_bytes; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + pk_info = mbedtls_pk_info_from_type(MBEDTLS_PK_ECDSA); + if (pk_info == NULL) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + lmd_res = mbedtls_ecdsa_sign(&ecdsa.grp, &r, &s, &ecdsa.d, msg, + msg_len, mbd_rand, NULL); + if (lmd_res == 0) { + *sig_len = 2 * key_size_bytes; + memset(sig, 0, *sig_len); + mbedtls_mpi_write_binary(&r, sig + *sig_len / 2 - + mbedtls_mpi_size(&r), + mbedtls_mpi_size(&r)); + + mbedtls_mpi_write_binary(&s, sig + *sig_len - + mbedtls_mpi_size(&s), + mbedtls_mpi_size(&s)); + res = TEE_SUCCESS; + } else { + FMSG("mbedtls_ecdsa_sign failed, returned 0x%x\n", -lmd_res); + res = TEE_ERROR_GENERIC; + } +out: + mbedtls_mpi_free(&r); + mbedtls_mpi_free(&s); + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&ecdsa.d); + mbedtls_ecdsa_free(&ecdsa); + return res; +} + +static TEE_Result ecc_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + mbedtls_ecdsa_context ecdsa; + mbedtls_ecp_group_id gid; + size_t key_size_bytes, key_size_bits = 0; + uint8_t one[1] = { 1 }; + mbedtls_mpi r; + mbedtls_mpi s; + + memset(&ecdsa, 0, sizeof(ecdsa)); + memset(&gid, 0, sizeof(gid)); + memset(&r, 0, sizeof(r)); + memset(&s, 0, sizeof(s)); + + if (algo == 0) + return TEE_ERROR_BAD_PARAMETERS; + + mbedtls_mpi_init(&r); + mbedtls_mpi_init(&s); + + mbedtls_ecdsa_init(&ecdsa); + + gid = curve_to_group_id(key->curve); + lmd_res = mbedtls_ecp_group_load(&ecdsa.grp, gid); + if (lmd_res != 0) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + ecdsa.Q.X = *(mbedtls_mpi *)key->x; + ecdsa.Q.Y = *(mbedtls_mpi *)key->y; + mbedtls_mpi_read_binary(&ecdsa.Q.Z, one, sizeof(one)); + + res = ecc_get_keysize(key->curve, algo, + &key_size_bytes, &key_size_bits); + if (res != TEE_SUCCESS) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* check keysize vs sig_len */ + if ((key_size_bytes * 2) != sig_len) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + mbedtls_mpi_read_binary(&r, sig, sig_len / 2); + mbedtls_mpi_read_binary(&s, sig + sig_len / 2, sig_len / 2); + + lmd_res = mbedtls_ecdsa_verify(&ecdsa.grp, msg, msg_len, &ecdsa.Q, + &r, &s); + if (lmd_res != 0) { + FMSG("mbedtls_ecdsa_verify failed, returned 0x%x", -lmd_res); + res = get_tee_result(lmd_res); + } +out: + mbedtls_mpi_free(&r); + mbedtls_mpi_free(&s); + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&ecdsa.Q.X); + mbedtls_mpi_init(&ecdsa.Q.Y); + mbedtls_ecdsa_free(&ecdsa); + return res; +} + +static TEE_Result ecc_shared_secret(struct ecc_keypair *private_key, + struct ecc_public_key *public_key, + void *secret, unsigned long *secret_len) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + uint8_t one[1] = { 1 }; + mbedtls_ecdh_context ecdh; + mbedtls_ecp_group_id gid; + size_t out_len = 0; + + memset(&ecdh, 0, sizeof(ecdh)); + memset(&gid, 0, sizeof(gid)); + mbedtls_ecdh_init(&ecdh); + gid = curve_to_group_id(private_key->curve); + lmd_res = mbedtls_ecdh_setup(&ecdh, gid); + if (lmd_res != 0) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + assert(ecdh.var == MBEDTLS_ECDH_VARIANT_MBEDTLS_2_0); + ecdh.ctx.mbed_ecdh.d = *(mbedtls_mpi *)private_key->d; + ecdh.ctx.mbed_ecdh.Qp.X = *(mbedtls_mpi *)public_key->x; + ecdh.ctx.mbed_ecdh.Qp.Y = *(mbedtls_mpi *)public_key->y; + mbedtls_mpi_read_binary(&ecdh.ctx.mbed_ecdh.Qp.Z, one, sizeof(one)); + + lmd_res = mbedtls_ecdh_calc_secret(&ecdh, &out_len, secret, + *secret_len, mbd_rand, NULL); + if (lmd_res != 0) { + res = get_tee_result(lmd_res); + goto out; + } + *secret_len = out_len; +out: + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&ecdh.ctx.mbed_ecdh.d); + mbedtls_mpi_init(&ecdh.ctx.mbed_ecdh.Qp.X); + mbedtls_mpi_init(&ecdh.ctx.mbed_ecdh.Qp.Y); + mbedtls_ecdh_free(&ecdh); + return res; +} + +static const struct crypto_ecc_keypair_ops ecc_keypair_ops = { + .generate = ecc_generate_keypair, + .sign = ecc_sign, + .shared_secret = ecc_shared_secret, +}; + +static const struct crypto_ecc_keypair_ops sm2_pke_keypair_ops = { + .generate = ecc_generate_keypair, + .decrypt = sm2_mbedtls_pke_decrypt, +}; + +static const struct crypto_ecc_keypair_ops sm2_kep_keypair_ops = { + .generate = ecc_generate_keypair, +}; + +static const struct crypto_ecc_keypair_ops sm2_dsa_keypair_ops = { + .generate = ecc_generate_keypair, + .sign = sm2_mbedtls_dsa_sign, +}; + +const struct crypto_ecc_keypair_ops * +crypto_asym_get_ecc_keypair_ops(uint32_t key_type) +{ + switch (key_type) { + case TEE_TYPE_ECDSA_KEYPAIR: + case TEE_TYPE_ECDH_KEYPAIR: + return &ecc_keypair_ops; + case TEE_TYPE_SM2_DSA_KEYPAIR: + if (!IS_ENABLED(CFG_CRYPTO_SM2_DSA)) + return NULL; + return &sm2_dsa_keypair_ops; + case TEE_TYPE_SM2_PKE_KEYPAIR: + if (!IS_ENABLED(CFG_CRYPTO_SM2_PKE)) + return NULL; + return &sm2_pke_keypair_ops; + case TEE_TYPE_SM2_KEP_KEYPAIR: + if (!IS_ENABLED(CFG_CRYPTO_SM2_KEP)) + return NULL; + return &sm2_kep_keypair_ops; + default: + return NULL; + } +} + +TEE_Result crypto_asym_alloc_ecc_keypair(struct ecc_keypair *s, + uint32_t key_type, + size_t key_size_bits) +{ + memset(s, 0, sizeof(*s)); + + switch (key_type) { + case TEE_TYPE_ECDSA_KEYPAIR: + case TEE_TYPE_ECDH_KEYPAIR: + s->ops = &ecc_keypair_ops; + break; + case TEE_TYPE_SM2_DSA_KEYPAIR: + if (!IS_ENABLED(CFG_CRYPTO_SM2_DSA)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_dsa_keypair_ops; + break; + case TEE_TYPE_SM2_PKE_KEYPAIR: + if (!IS_ENABLED(CFG_CRYPTO_SM2_PKE)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_pke_keypair_ops; + break; + case TEE_TYPE_SM2_KEP_KEYPAIR: + if (!IS_ENABLED(CFG_CRYPTO_SM2_KEP)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_kep_keypair_ops; + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + + s->d = crypto_bignum_allocate(key_size_bits); + if (!s->d) + goto err; + s->x = crypto_bignum_allocate(key_size_bits); + if (!s->x) + goto err; + s->y = crypto_bignum_allocate(key_size_bits); + if (!s->y) + goto err; + + return TEE_SUCCESS; + +err: + crypto_bignum_free(&s->d); + crypto_bignum_free(&s->x); + + return TEE_ERROR_OUT_OF_MEMORY; +} + +static const struct crypto_ecc_public_ops ecc_public_key_ops = { + .free = ecc_free_public_key, + .verify = ecc_verify, +}; + +static const struct crypto_ecc_public_ops sm2_pke_public_key_ops = { + .free = ecc_free_public_key, + .encrypt = sm2_mbedtls_pke_encrypt, +}; + +static const struct crypto_ecc_public_ops sm2_kep_public_key_ops = { + .free = ecc_free_public_key, +}; + +static const struct crypto_ecc_public_ops sm2_dsa_public_key_ops = { + .free = ecc_free_public_key, + .verify = sm2_mbedtls_dsa_verify, +}; + +const struct crypto_ecc_public_ops* +crypto_asym_get_ecc_public_ops(uint32_t key_type) +{ + switch (key_type) { + case TEE_TYPE_ECDSA_PUBLIC_KEY: + case TEE_TYPE_ECDH_PUBLIC_KEY: + return &ecc_public_key_ops; + case TEE_TYPE_SM2_DSA_PUBLIC_KEY: + if (!IS_ENABLED(CFG_CRYPTO_SM2_DSA)) + return NULL; + + return &sm2_dsa_public_key_ops; + case TEE_TYPE_SM2_PKE_PUBLIC_KEY: + if (!IS_ENABLED(CFG_CRYPTO_SM2_PKE)) + return NULL; + + return &sm2_pke_public_key_ops; + case TEE_TYPE_SM2_KEP_PUBLIC_KEY: + if (!IS_ENABLED(CFG_CRYPTO_SM2_KEP)) + return NULL; + return &sm2_kep_public_key_ops; + default: + return NULL; + } +} + +TEE_Result crypto_asym_alloc_ecc_public_key(struct ecc_public_key *s, + uint32_t key_type, + size_t key_size_bits) +{ + memset(s, 0, sizeof(*s)); + + switch (key_type) { + case TEE_TYPE_ECDSA_PUBLIC_KEY: + case TEE_TYPE_ECDH_PUBLIC_KEY: + s->ops = &ecc_public_key_ops; + break; + case TEE_TYPE_SM2_DSA_PUBLIC_KEY: + if (!IS_ENABLED(CFG_CRYPTO_SM2_DSA)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_dsa_public_key_ops; + break; + case TEE_TYPE_SM2_PKE_PUBLIC_KEY: + if (!IS_ENABLED(CFG_CRYPTO_SM2_PKE)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_pke_public_key_ops; + break; + case TEE_TYPE_SM2_KEP_PUBLIC_KEY: + if (!IS_ENABLED(CFG_CRYPTO_SM2_KEP)) + return TEE_ERROR_NOT_IMPLEMENTED; + + s->curve = TEE_ECC_CURVE_SM2; + s->ops = &sm2_kep_public_key_ops; + break; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } + + s->x = crypto_bignum_allocate(key_size_bits); + if (!s->x) + goto err; + s->y = crypto_bignum_allocate(key_size_bits); + if (!s->y) + goto err; + + return TEE_SUCCESS; + +err: + crypto_bignum_free(&s->x); + + return TEE_ERROR_OUT_OF_MEMORY; +} diff --git a/optee_os/lib/libmbedtls/core/hash.c b/optee_os/lib/libmbedtls/core/hash.c new file mode 100644 index 0000000..651a9bf --- /dev/null +++ b/optee_os/lib/libmbedtls/core/hash.c @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mbed_hash_ctx { + struct crypto_hash_ctx hash_ctx; + mbedtls_md_context_t md_ctx; +}; + +static const struct crypto_hash_ops mbed_hash_ops; + +static struct mbed_hash_ctx *to_hash_ctx(struct crypto_hash_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_hash_ops); + + return container_of(ctx, struct mbed_hash_ctx, hash_ctx); +} + +static TEE_Result mbed_hash_init(struct crypto_hash_ctx *ctx) +{ + if (mbedtls_md_starts(&to_hash_ctx(ctx)->md_ctx)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_hash_update(struct crypto_hash_ctx *ctx, + const uint8_t *data, size_t len) +{ + if (mbedtls_md_update(&to_hash_ctx(ctx)->md_ctx, data, len)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_hash_final(struct crypto_hash_ctx *ctx, uint8_t *digest, + size_t len) +{ + struct mbed_hash_ctx *hc = to_hash_ctx(ctx); + uint8_t block_digest[TEE_MAX_HASH_SIZE] = { 0 }; + uint8_t *tmp_digest = NULL; + size_t hash_size = 0; + + if (len == 0) + return TEE_ERROR_BAD_PARAMETERS; + + hash_size = mbedtls_md_get_size(mbedtls_md_info_from_ctx(&hc->md_ctx)); + if (hash_size > len) { + if (hash_size > sizeof(block_digest)) + return TEE_ERROR_BAD_STATE; + tmp_digest = block_digest; /* use a tempory buffer */ + } else { + tmp_digest = digest; + } + + if (mbedtls_md_finish(&hc->md_ctx, tmp_digest)) + return TEE_ERROR_BAD_STATE; + + if (hash_size > len) + memcpy(digest, tmp_digest, len); + + return TEE_SUCCESS; +} + +static void mbed_hash_free_ctx(struct crypto_hash_ctx *ctx) +{ + struct mbed_hash_ctx *hc = to_hash_ctx(ctx); + + mbedtls_md_free(&hc->md_ctx); + free(hc); +} + +static void mbed_hash_copy_state(struct crypto_hash_ctx *dst_ctx, + struct crypto_hash_ctx *src_ctx) +{ + struct mbed_hash_ctx *src = to_hash_ctx(src_ctx); + struct mbed_hash_ctx *dst = to_hash_ctx(dst_ctx); + + if (mbedtls_md_clone(&dst->md_ctx, &src->md_ctx)) + panic(); +} + +static const struct crypto_hash_ops mbed_hash_ops = { + .init = mbed_hash_init, + .update = mbed_hash_update, + .final = mbed_hash_final, + .free_ctx = mbed_hash_free_ctx, + .copy_state = mbed_hash_copy_state, +}; + +static TEE_Result mbed_hash_alloc_ctx(struct crypto_hash_ctx **ctx_ret, + mbedtls_md_type_t md_type) +{ + int mbed_res = 0; + struct mbed_hash_ctx *hc = NULL; + const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(md_type); + + if (!md_info) + return TEE_ERROR_NOT_SUPPORTED; + + hc = calloc(1, sizeof(*hc)); + if (!hc) + return TEE_ERROR_OUT_OF_MEMORY; + + hc->hash_ctx.ops = &mbed_hash_ops; + mbed_res = mbedtls_md_setup(&hc->md_ctx, md_info, 0); + if (mbed_res) { + free(hc); + if (mbed_res == MBEDTLS_ERR_MD_ALLOC_FAILED) + return TEE_ERROR_OUT_OF_MEMORY; + return TEE_ERROR_NOT_SUPPORTED; + } + + *ctx_ret = &hc->hash_ctx; + + return TEE_SUCCESS; +} + +#if defined(CFG_CRYPTO_MD5) +TEE_Result crypto_md5_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return mbed_hash_alloc_ctx(ctx, MBEDTLS_MD_MD5); +} +#endif + +#if defined(CFG_CRYPTO_SHA1) +TEE_Result crypto_sha1_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return mbed_hash_alloc_ctx(ctx, MBEDTLS_MD_SHA1); +} +#endif + +#if defined(CFG_CRYPTO_SHA224) +TEE_Result crypto_sha224_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return mbed_hash_alloc_ctx(ctx, MBEDTLS_MD_SHA224); +} +#endif + +#if defined(CFG_CRYPTO_SHA256) +TEE_Result crypto_sha256_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return mbed_hash_alloc_ctx(ctx, MBEDTLS_MD_SHA256); +} +#endif + +#if defined(CFG_CRYPTO_SHA384) +TEE_Result crypto_sha384_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return mbed_hash_alloc_ctx(ctx, MBEDTLS_MD_SHA384); +} +#endif + +#if defined(CFG_CRYPTO_SHA512) +TEE_Result crypto_sha512_alloc_ctx(struct crypto_hash_ctx **ctx) +{ + return mbed_hash_alloc_ctx(ctx, MBEDTLS_MD_SHA512); +} +#endif + +#if defined(CFG_CRYPTO_SHA256) +TEE_Result hash_sha256_check(const uint8_t *hash, const uint8_t *data, + size_t data_size) +{ + mbedtls_sha256_context hs; + uint8_t digest[TEE_SHA256_HASH_SIZE] = { 0 }; + + memset(&hs, 0, sizeof(hs)); + mbedtls_sha256_init(&hs); + mbedtls_sha256_starts(&hs, 0); + mbedtls_sha256_update(&hs, data, data_size); + mbedtls_sha256_finish(&hs, digest); + mbedtls_sha256_free(&hs); + + if (consttime_memcmp(digest, hash, sizeof(digest))) + return TEE_ERROR_SECURITY; + return TEE_SUCCESS; +} +#endif + +#if defined(MBEDTLS_SHA1_PROCESS_ALT) +int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, + const unsigned char data[64]) +{ + MBEDTLS_INTERNAL_VALIDATE_RET(ctx != NULL, + MBEDTLS_ERR_SHA1_BAD_INPUT_DATA); + MBEDTLS_INTERNAL_VALIDATE_RET((const unsigned char *)data != NULL, + MBEDTLS_ERR_SHA1_BAD_INPUT_DATA); + + crypto_accel_sha1_compress(ctx->state, data, 1); + + return 0; +} +#endif /*MBEDTLS_SHA1_PROCESS_ALT*/ + +#if defined(MBEDTLS_SHA256_PROCESS_ALT) +int mbedtls_internal_sha256_process(mbedtls_sha256_context *ctx, + const unsigned char data[64]) +{ + MBEDTLS_INTERNAL_VALIDATE_RET(ctx != NULL, + MBEDTLS_ERR_SHA256_BAD_INPUT_DATA); + MBEDTLS_INTERNAL_VALIDATE_RET((const unsigned char *)data != NULL, + MBEDTLS_ERR_SHA256_BAD_INPUT_DATA); + + crypto_accel_sha256_compress(ctx->state, data, 1); + + return 0; +} +#endif /*MBEDTLS_SHA256_PROCESS_ALT*/ + +#if defined(MBEDTLS_SHA512_PROCESS_ALT) +int mbedtls_internal_sha512_process(mbedtls_sha512_context *ctx, + const unsigned char data[64]) +{ + MBEDTLS_INTERNAL_VALIDATE_RET(ctx != NULL, + MBEDTLS_ERR_SHA512_BAD_INPUT_DATA); + MBEDTLS_INTERNAL_VALIDATE_RET((const unsigned char *)data != NULL, + MBEDTLS_ERR_SHA512_BAD_INPUT_DATA); + + crypto_accel_sha512_compress(ctx->state, data, 1); + + return 0; +} +#endif /*MBEDTLS_SHA512_PROCESS_ALT*/ diff --git a/optee_os/lib/libmbedtls/core/hmac.c b/optee_os/lib/libmbedtls/core/hmac.c new file mode 100644 index 0000000..0f93017 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/hmac.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mbed_hmac_ctx { + struct crypto_mac_ctx mac_ctx; + mbedtls_md_context_t md_ctx; +}; + +static const struct crypto_mac_ops mbed_hmac_ops; + +static struct mbed_hmac_ctx *to_hmac_ctx(struct crypto_mac_ctx *ctx) +{ + assert(ctx && ctx->ops == &mbed_hmac_ops); + + return container_of(ctx, struct mbed_hmac_ctx, mac_ctx); +} + +static TEE_Result mbed_hmac_init(struct crypto_mac_ctx *ctx, + const uint8_t *key, size_t len) +{ + if (mbedtls_md_hmac_starts(&to_hmac_ctx(ctx)->md_ctx, key, len)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_hmac_update(struct crypto_mac_ctx *ctx, + const uint8_t *data, size_t len) +{ + if (mbedtls_md_hmac_update(&to_hmac_ctx(ctx)->md_ctx, data, len)) + return TEE_ERROR_BAD_STATE; + + return TEE_SUCCESS; +} + +static TEE_Result mbed_hmac_final(struct crypto_mac_ctx *ctx, uint8_t *digest, + size_t len) +{ + struct mbed_hmac_ctx *c = to_hmac_ctx(ctx); + uint8_t block_digest[TEE_MAX_HASH_SIZE] = { 0 }; + uint8_t *tmp_digest = NULL; + size_t hmac_size = 0; + + if (len == 0) + return TEE_ERROR_BAD_PARAMETERS; + + hmac_size = mbedtls_md_get_size(mbedtls_md_info_from_ctx(&c->md_ctx)); + if (hmac_size > len) { + if (hmac_size > sizeof(block_digest)) + return TEE_ERROR_BAD_STATE; + tmp_digest = block_digest; /* use a tempory buffer */ + } else { + tmp_digest = digest; + } + + if (mbedtls_md_hmac_finish(&c->md_ctx, tmp_digest)) + return TEE_ERROR_BAD_STATE; + + if (hmac_size > len) + memcpy(digest, tmp_digest, len); + + return TEE_SUCCESS; +} + +static void mbed_hmac_free_ctx(struct crypto_mac_ctx *ctx) +{ + struct mbed_hmac_ctx *c = to_hmac_ctx(ctx); + + mbedtls_md_free(&c->md_ctx); + free(c); +} + +static void mbed_hmac_copy_state(struct crypto_mac_ctx *dst_ctx, + struct crypto_mac_ctx *src_ctx) +{ + struct mbed_hmac_ctx *src = to_hmac_ctx(src_ctx); + struct mbed_hmac_ctx *dst = to_hmac_ctx(dst_ctx); + + if (mbedtls_md_clone(&dst->md_ctx, &src->md_ctx)) + panic(); +} + +static const struct crypto_mac_ops mbed_hmac_ops = { + .init = mbed_hmac_init, + .update = mbed_hmac_update, + .final = mbed_hmac_final, + .free_ctx = mbed_hmac_free_ctx, + .copy_state = mbed_hmac_copy_state, +}; + +static TEE_Result mbed_hmac_alloc_ctx(struct crypto_mac_ctx **ctx_ret, + mbedtls_md_type_t md_type) +{ + int mbed_res = 0; + struct mbed_hmac_ctx *c = NULL; + const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(md_type); + + if (!md_info) + return TEE_ERROR_NOT_SUPPORTED; + + c = calloc(1, sizeof(*c)); + if (!c) + return TEE_ERROR_OUT_OF_MEMORY; + + c->mac_ctx.ops = &mbed_hmac_ops; + mbed_res = mbedtls_md_setup(&c->md_ctx, md_info, 1); + if (mbed_res) { + free(c); + if (mbed_res == MBEDTLS_ERR_MD_ALLOC_FAILED) + return TEE_ERROR_OUT_OF_MEMORY; + return TEE_ERROR_NOT_SUPPORTED; + } + + *ctx_ret = &c->mac_ctx; + + return TEE_SUCCESS; +} + +#if defined(CFG_CRYPTO_MD5) +TEE_Result crypto_hmac_md5_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return mbed_hmac_alloc_ctx(ctx, MBEDTLS_MD_MD5); +} +#endif + +#if defined(CFG_CRYPTO_SHA1) +TEE_Result crypto_hmac_sha1_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return mbed_hmac_alloc_ctx(ctx, MBEDTLS_MD_SHA1); +} +#endif + +#if defined(CFG_CRYPTO_SHA224) +TEE_Result crypto_hmac_sha224_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return mbed_hmac_alloc_ctx(ctx, MBEDTLS_MD_SHA224); +} +#endif + +#if defined(CFG_CRYPTO_SHA256) +TEE_Result crypto_hmac_sha256_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return mbed_hmac_alloc_ctx(ctx, MBEDTLS_MD_SHA256); +} +#endif + +#if defined(CFG_CRYPTO_SHA384) +TEE_Result crypto_hmac_sha384_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return mbed_hmac_alloc_ctx(ctx, MBEDTLS_MD_SHA384); +} +#endif + +#if defined(CFG_CRYPTO_SHA512) +TEE_Result crypto_hmac_sha512_alloc_ctx(struct crypto_mac_ctx **ctx) +{ + return mbed_hmac_alloc_ctx(ctx, MBEDTLS_MD_SHA512); +} +#endif diff --git a/optee_os/lib/libmbedtls/core/mbed_helpers.c b/optee_os/lib/libmbedtls/core/mbed_helpers.c new file mode 100644 index 0000000..49d322e --- /dev/null +++ b/optee_os/lib/libmbedtls/core/mbed_helpers.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" + +/* Generate random number 1 <= n < max */ +TEE_Result mbed_gen_random_upto(mbedtls_mpi *n, mbedtls_mpi *max) +{ + size_t sz = mbedtls_mpi_size(max); + bool found = false; + int mres = 0; + + do { + mres = mbedtls_mpi_fill_random(n, sz, mbd_rand, NULL); + if (mres) + return TEE_ERROR_BAD_STATE; + if (mbedtls_mpi_bitlen(n) != 0 && + mbedtls_mpi_cmp_mpi(n, max) == -1) + found = true; + } while (!found); + + return TEE_SUCCESS; +} + diff --git a/optee_os/lib/libmbedtls/core/mbed_helpers.h b/optee_os/lib/libmbedtls/core/mbed_helpers.h new file mode 100644 index 0000000..1562c53 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/mbed_helpers.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#ifndef MBED_HELPERS_H +#define MBED_HELPERS_H + +#include +#include +#include +#include +#include + +static inline int mbd_rand(void *rng_state __unused, unsigned char *output, + size_t len) +{ + if (crypto_rng_read(output, len)) + return MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED; + return 0; +} + +static inline void mbed_copy_mbedtls_aes_context(mbedtls_aes_context *dst, + mbedtls_aes_context *src) +{ + *dst = *src; +} + +TEE_Result mbed_gen_random_upto(mbedtls_mpi *n, mbedtls_mpi *max); +#endif /*MBED_HELPERS_H*/ diff --git a/optee_os/lib/libmbedtls/core/rsa.c b/optee_os/lib/libmbedtls/core/rsa.c new file mode 100644 index 0000000..89ebb0d --- /dev/null +++ b/optee_os/lib/libmbedtls/core/rsa.c @@ -0,0 +1,817 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" +#include "../mbedtls/library/pk_wrap.h" +#include "../mbedtls/library/rsa_alt_helpers.h" + +static TEE_Result get_tee_result(int lmd_res) +{ + switch (lmd_res) { + case 0: + return TEE_SUCCESS; + case MBEDTLS_ERR_RSA_PRIVATE_FAILED + + MBEDTLS_ERR_MPI_BAD_INPUT_DATA: + case MBEDTLS_ERR_RSA_BAD_INPUT_DATA: + case MBEDTLS_ERR_RSA_INVALID_PADDING: + case MBEDTLS_ERR_PK_TYPE_MISMATCH: + return TEE_ERROR_BAD_PARAMETERS; + case MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE: + return TEE_ERROR_SHORT_BUFFER; + default: + return TEE_ERROR_BAD_STATE; + } +} + +static uint32_t tee_algo_to_mbedtls_hash_algo(uint32_t algo) +{ + switch (algo) { +#if defined(CFG_CRYPTO_SHA1) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1: + case TEE_ALG_SHA1: + case TEE_ALG_DSA_SHA1: + case TEE_ALG_HMAC_SHA1: + return MBEDTLS_MD_SHA1; +#endif +#if defined(CFG_CRYPTO_MD5) + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_MD5: + case TEE_ALG_MD5: + case TEE_ALG_HMAC_MD5: + return MBEDTLS_MD_MD5; +#endif +#if defined(CFG_CRYPTO_SHA224) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224: + case TEE_ALG_SHA224: + case TEE_ALG_DSA_SHA224: + case TEE_ALG_HMAC_SHA224: + return MBEDTLS_MD_SHA224; +#endif +#if defined(CFG_CRYPTO_SHA256) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256: + case TEE_ALG_SHA256: + case TEE_ALG_DSA_SHA256: + case TEE_ALG_HMAC_SHA256: + return MBEDTLS_MD_SHA256; +#endif +#if defined(CFG_CRYPTO_SHA384) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384: + case TEE_ALG_SHA384: + case TEE_ALG_HMAC_SHA384: + return MBEDTLS_MD_SHA384; +#endif +#if defined(CFG_CRYPTO_SHA512) + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512: + case TEE_ALG_SHA512: + case TEE_ALG_HMAC_SHA512: + return MBEDTLS_MD_SHA512; +#endif + default: + return MBEDTLS_MD_NONE; + } +} + +static TEE_Result rsa_init_and_complete_from_key_pair(mbedtls_rsa_context *rsa, + struct rsa_keypair *key) +{ + int lmd_res = 0; + + mbedtls_rsa_init(rsa); + + rsa->E = *(mbedtls_mpi *)key->e; + rsa->N = *(mbedtls_mpi *)key->n; + rsa->D = *(mbedtls_mpi *)key->d; + rsa->len = mbedtls_mpi_size(&rsa->N); + + if (key->p && crypto_bignum_num_bytes(key->p)) { + rsa->P = *(mbedtls_mpi *)key->p; + rsa->Q = *(mbedtls_mpi *)key->q; + rsa->QP = *(mbedtls_mpi *)key->qp; + rsa->DP = *(mbedtls_mpi *)key->dp; + rsa->DQ = *(mbedtls_mpi *)key->dq; + } else { + mbedtls_mpi_init_mempool(&rsa->P); + mbedtls_mpi_init_mempool(&rsa->Q); + mbedtls_mpi_init_mempool(&rsa->QP); + mbedtls_mpi_init_mempool(&rsa->DP); + mbedtls_mpi_init_mempool(&rsa->DQ); + + lmd_res = mbedtls_rsa_deduce_primes(&rsa->N, &rsa->E, &rsa->D, + &rsa->P, &rsa->Q); + if (lmd_res) { + DMSG("mbedtls_rsa_deduce_primes() returned 0x%x", + -lmd_res); + goto err; + } + + lmd_res = mbedtls_rsa_deduce_crt(&rsa->P, &rsa->Q, &rsa->D, + &rsa->DP, &rsa->DQ, &rsa->QP); + if (lmd_res) { + DMSG("mbedtls_rsa_deduce_crt() returned 0x%x", + -lmd_res); + goto err; + } + } + + return TEE_SUCCESS; +err: + mbedtls_mpi_free(&rsa->P); + mbedtls_mpi_free(&rsa->Q); + mbedtls_mpi_free(&rsa->QP); + mbedtls_mpi_free(&rsa->DP); + mbedtls_mpi_free(&rsa->DQ); + + return get_tee_result(lmd_res); +} + +static void mbd_rsa_free(mbedtls_rsa_context *rsa, struct rsa_keypair *key) +{ + /* + * The mpi's in @rsa are initialized from @key, but the primes and + * CRT part are generated if @key doesn't have them. When freeing + * we should only free the generated mpi's, the ones copied are + * reset instead. + */ + mbedtls_mpi_init(&rsa->E); + mbedtls_mpi_init(&rsa->N); + mbedtls_mpi_init(&rsa->D); + if (key->p && crypto_bignum_num_bytes(key->p)) { + mbedtls_mpi_init(&rsa->P); + mbedtls_mpi_init(&rsa->Q); + mbedtls_mpi_init(&rsa->QP); + mbedtls_mpi_init(&rsa->DP); + mbedtls_mpi_init(&rsa->DQ); + } + mbedtls_rsa_free(rsa); +} + +TEE_Result crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *s, + size_t key_size_bits) +__weak __alias("sw_crypto_acipher_alloc_rsa_keypair"); + +TEE_Result sw_crypto_acipher_alloc_rsa_keypair(struct rsa_keypair *s, + size_t key_size_bits) +{ + memset(s, 0, sizeof(*s)); + s->e = crypto_bignum_allocate(key_size_bits); + if (!s->e) + goto err; + s->d = crypto_bignum_allocate(key_size_bits); + if (!s->d) + goto err; + s->n = crypto_bignum_allocate(key_size_bits); + if (!s->n) + goto err; + s->p = crypto_bignum_allocate(key_size_bits); + if (!s->p) + goto err; + s->q = crypto_bignum_allocate(key_size_bits); + if (!s->q) + goto err; + s->qp = crypto_bignum_allocate(key_size_bits); + if (!s->qp) + goto err; + s->dp = crypto_bignum_allocate(key_size_bits); + if (!s->dp) + goto err; + s->dq = crypto_bignum_allocate(key_size_bits); + if (!s->dq) + goto err; + + return TEE_SUCCESS; +err: + crypto_acipher_free_rsa_keypair(s); + return TEE_ERROR_OUT_OF_MEMORY; +} + +TEE_Result crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *s, + size_t key_size_bits) +__weak __alias("sw_crypto_acipher_alloc_rsa_public_key"); + +TEE_Result sw_crypto_acipher_alloc_rsa_public_key(struct rsa_public_key *s, + size_t key_size_bits) +{ + memset(s, 0, sizeof(*s)); + s->e = crypto_bignum_allocate(key_size_bits); + if (!s->e) + return TEE_ERROR_OUT_OF_MEMORY; + s->n = crypto_bignum_allocate(key_size_bits); + if (!s->n) + goto err; + return TEE_SUCCESS; +err: + crypto_bignum_free(&s->e); + return TEE_ERROR_OUT_OF_MEMORY; +} + +void crypto_acipher_free_rsa_public_key(struct rsa_public_key *s) +__weak __alias("sw_crypto_acipher_free_rsa_public_key"); + +void sw_crypto_acipher_free_rsa_public_key(struct rsa_public_key *s) +{ + if (!s) + return; + crypto_bignum_free(&s->n); + crypto_bignum_free(&s->e); +} + +void crypto_acipher_free_rsa_keypair(struct rsa_keypair *s) +__weak __alias("sw_crypto_acipher_free_rsa_keypair"); + +void sw_crypto_acipher_free_rsa_keypair(struct rsa_keypair *s) +{ + if (!s) + return; + crypto_bignum_free(&s->e); + crypto_bignum_free(&s->d); + crypto_bignum_free(&s->n); + crypto_bignum_free(&s->p); + crypto_bignum_free(&s->q); + crypto_bignum_free(&s->qp); + crypto_bignum_free(&s->dp); + crypto_bignum_free(&s->dq); +} + +TEE_Result crypto_acipher_gen_rsa_key(struct rsa_keypair *key, + size_t key_size) +__weak __alias("sw_crypto_acipher_gen_rsa_key"); + +TEE_Result sw_crypto_acipher_gen_rsa_key(struct rsa_keypair *key, + size_t key_size) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_rsa_context rsa; + mbedtls_ctr_drbg_context rngctx; + int lmd_res = 0; + uint32_t e = 0; + + mbedtls_ctr_drbg_init(&rngctx); + if (mbedtls_ctr_drbg_seed(&rngctx, mbd_rand, NULL, NULL, 0)) + return TEE_ERROR_BAD_STATE; + + memset(&rsa, 0, sizeof(rsa)); + mbedtls_rsa_init(&rsa); + + /* get the public exponent */ + mbedtls_mpi_write_binary((mbedtls_mpi *)key->e, + (unsigned char *)&e, sizeof(uint32_t)); + + e = TEE_U32_FROM_BIG_ENDIAN(e); + lmd_res = mbedtls_rsa_gen_key(&rsa, mbedtls_ctr_drbg_random, &rngctx, + key_size, (int)e); + mbedtls_ctr_drbg_free(&rngctx); + if (lmd_res != 0) { + res = get_tee_result(lmd_res); + } else if ((size_t)mbedtls_mpi_bitlen(&rsa.N) != key_size) { + res = TEE_ERROR_BAD_PARAMETERS; + } else { + /* Copy the key */ + crypto_bignum_copy(key->e, (void *)&rsa.E); + crypto_bignum_copy(key->d, (void *)&rsa.D); + crypto_bignum_copy(key->n, (void *)&rsa.N); + crypto_bignum_copy(key->p, (void *)&rsa.P); + + crypto_bignum_copy(key->q, (void *)&rsa.Q); + crypto_bignum_copy(key->qp, (void *)&rsa.QP); + crypto_bignum_copy(key->dp, (void *)&rsa.DP); + crypto_bignum_copy(key->dq, (void *)&rsa.DQ); + + res = TEE_SUCCESS; + } + + mbedtls_rsa_free(&rsa); + + return res; +} + +TEE_Result crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsanopad_encrypt"); + +TEE_Result sw_crypto_acipher_rsanopad_encrypt(struct rsa_public_key *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_rsa_context rsa; + int lmd_res = 0; + uint8_t *buf = NULL; + unsigned long blen = 0; + unsigned long offset = 0; + + memset(&rsa, 0, sizeof(rsa)); + mbedtls_rsa_init(&rsa); + + rsa.E = *(mbedtls_mpi *)key->e; + rsa.N = *(mbedtls_mpi *)key->n; + + rsa.len = crypto_bignum_num_bytes((void *)&rsa.N); + + blen = CFG_CORE_BIGNUM_MAX_BITS / 8; + buf = malloc(blen); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + memset(buf, 0, blen); + memcpy(buf + rsa.len - src_len, src, src_len); + + lmd_res = mbedtls_rsa_public(&rsa, buf, buf); + if (lmd_res != 0) { + FMSG("mbedtls_rsa_public() returned 0x%x", -lmd_res); + res = get_tee_result(lmd_res); + goto out; + } + + /* Remove the zero-padding (leave one zero if buff is all zeroes) */ + offset = 0; + while ((offset < rsa.len - 1) && (buf[offset] == 0)) + offset++; + + if (*dst_len < rsa.len - offset) { + *dst_len = rsa.len - offset; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + *dst_len = rsa.len - offset; + memcpy(dst, buf + offset, *dst_len); +out: + free(buf); + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&rsa.E); + mbedtls_mpi_init(&rsa.N); + mbedtls_rsa_free(&rsa); + + return res; +} + +TEE_Result crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsanopad_decrypt"); + +TEE_Result sw_crypto_acipher_rsanopad_decrypt(struct rsa_keypair *key, + const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_rsa_context rsa = { }; + int lmd_res = 0; + uint8_t *buf = NULL; + unsigned long blen = 0; + unsigned long offset = 0; + + res = rsa_init_and_complete_from_key_pair(&rsa, key); + if (res) + return res; + + blen = CFG_CORE_BIGNUM_MAX_BITS / 8; + buf = malloc(blen); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + memset(buf, 0, blen); + memcpy(buf + rsa.len - src_len, src, src_len); + + lmd_res = mbedtls_rsa_private(&rsa, mbd_rand, NULL, buf, buf); + if (lmd_res != 0) { + FMSG("mbedtls_rsa_private() returned 0x%x", -lmd_res); + res = get_tee_result(lmd_res); + goto out; + } + + /* Remove the zero-padding (leave one zero if buff is all zeroes) */ + offset = 0; + while ((offset < rsa.len - 1) && (buf[offset] == 0)) + offset++; + + if (*dst_len < rsa.len - offset) { + *dst_len = rsa.len - offset; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + *dst_len = rsa.len - offset; + memcpy(dst, (char *)buf + offset, *dst_len); +out: + if (buf) + free(buf); + mbd_rsa_free(&rsa, key); + return res; +} + +TEE_Result crypto_acipher_rsaes_decrypt(uint32_t algo, + struct rsa_keypair *key, + const uint8_t *label __unused, + size_t label_len __unused, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsaes_decrypt"); + +TEE_Result sw_crypto_acipher_rsaes_decrypt(uint32_t algo, + struct rsa_keypair *key, + const uint8_t *label __unused, + size_t label_len __unused, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + int lmd_padding = 0; + size_t blen = 0; + size_t mod_size = 0; + void *buf = NULL; + mbedtls_rsa_context rsa = { }; + const mbedtls_pk_info_t *pk_info = NULL; + uint32_t md_algo = MBEDTLS_MD_NONE; + + res = rsa_init_and_complete_from_key_pair(&rsa, key); + if (res) + return res; + + /* + * Use a temporary buffer since we don't know exactly how large + * the required size of the out buffer without doing a partial + * decrypt. We know the upper bound though. + */ + if (algo == TEE_ALG_RSAES_PKCS1_V1_5) { + mod_size = crypto_bignum_num_bytes(key->n); + blen = mod_size - 11; + lmd_padding = MBEDTLS_RSA_PKCS_V15; + } else { + /* Decoded message is always shorter than encrypted message */ + blen = src_len; + lmd_padding = MBEDTLS_RSA_PKCS_V21; + } + + buf = malloc(blen); + if (!buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + pk_info = mbedtls_pk_info_from_type(MBEDTLS_PK_RSA); + if (!pk_info) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + /* + * TEE_ALG_RSAES_PKCS1_V1_5 is invalid in hash. But its hash algo will + * not be used in rsa, so skip it here. + */ + if (algo != TEE_ALG_RSAES_PKCS1_V1_5) { + md_algo = tee_algo_to_mbedtls_hash_algo(algo); + if (md_algo == MBEDTLS_MD_NONE) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + } + + mbedtls_rsa_set_padding(&rsa, lmd_padding, md_algo); + + lmd_res = pk_info->decrypt_func(&rsa, src, src_len, buf, &blen, + blen, mbd_rand, NULL); + if (lmd_res != 0) { + FMSG("decrypt_func() returned 0x%x", -lmd_res); + res = get_tee_result(lmd_res); + goto out; + } + + if (*dst_len < blen) { + *dst_len = blen; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + res = TEE_SUCCESS; + *dst_len = blen; + memcpy(dst, buf, blen); +out: + if (buf) + free(buf); + mbd_rsa_free(&rsa, key); + return res; +} + +TEE_Result crypto_acipher_rsaes_encrypt(uint32_t algo, + struct rsa_public_key *key, + const uint8_t *label __unused, + size_t label_len __unused, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +__weak __alias("sw_crypto_acipher_rsaes_encrypt"); + +TEE_Result sw_crypto_acipher_rsaes_encrypt(uint32_t algo, + struct rsa_public_key *key, + const uint8_t *label __unused, + size_t label_len __unused, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + int lmd_padding = 0; + size_t mod_size = 0; + mbedtls_rsa_context rsa; + const mbedtls_pk_info_t *pk_info = NULL; + uint32_t md_algo = MBEDTLS_MD_NONE; + + memset(&rsa, 0, sizeof(rsa)); + mbedtls_rsa_init(&rsa); + + rsa.E = *(mbedtls_mpi *)key->e; + rsa.N = *(mbedtls_mpi *)key->n; + + mod_size = crypto_bignum_num_bytes(key->n); + if (*dst_len < mod_size) { + *dst_len = mod_size; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + *dst_len = mod_size; + rsa.len = mod_size; + + if (algo == TEE_ALG_RSAES_PKCS1_V1_5) + lmd_padding = MBEDTLS_RSA_PKCS_V15; + else + lmd_padding = MBEDTLS_RSA_PKCS_V21; + + pk_info = mbedtls_pk_info_from_type(MBEDTLS_PK_RSA); + if (!pk_info) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + /* + * TEE_ALG_RSAES_PKCS1_V1_5 is invalid in hash. But its hash algo will + * not be used in rsa, so skip it here. + */ + if (algo != TEE_ALG_RSAES_PKCS1_V1_5) { + md_algo = tee_algo_to_mbedtls_hash_algo(algo); + if (md_algo == MBEDTLS_MD_NONE) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + } + + mbedtls_rsa_set_padding(&rsa, lmd_padding, md_algo); + + lmd_res = pk_info->encrypt_func(&rsa, src, src_len, dst, dst_len, + *dst_len, mbd_rand, NULL); + if (lmd_res != 0) { + FMSG("encrypt_func() returned 0x%x", -lmd_res); + res = get_tee_result(lmd_res); + goto out; + } + res = TEE_SUCCESS; +out: + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&rsa.E); + mbedtls_mpi_init(&rsa.N); + mbedtls_rsa_free(&rsa); + return res; +} + +TEE_Result crypto_acipher_rsassa_sign(uint32_t algo, struct rsa_keypair *key, + int salt_len __unused, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +__weak __alias("sw_crypto_acipher_rsassa_sign"); + +TEE_Result sw_crypto_acipher_rsassa_sign(uint32_t algo, struct rsa_keypair *key, + int salt_len __unused, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + int lmd_padding = 0; + size_t mod_size = 0; + size_t hash_size = 0; + mbedtls_rsa_context rsa = { }; + const mbedtls_pk_info_t *pk_info = NULL; + uint32_t md_algo = 0; + + res = rsa_init_and_complete_from_key_pair(&rsa, key); + if (res) + return res; + + switch (algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + lmd_padding = MBEDTLS_RSA_PKCS_V15; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + lmd_padding = MBEDTLS_RSA_PKCS_V21; + break; + default: + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + res = tee_alg_get_digest_size(TEE_DIGEST_HASH_TO_ALGO(algo), + &hash_size); + if (res != TEE_SUCCESS) + goto err; + + if (msg_len != hash_size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + mod_size = crypto_bignum_num_bytes(key->n); + if (*sig_len < mod_size) { + *sig_len = mod_size; + res = TEE_ERROR_SHORT_BUFFER; + goto err; + } + rsa.len = mod_size; + + md_algo = tee_algo_to_mbedtls_hash_algo(algo); + if (md_algo == MBEDTLS_MD_NONE) { + res = TEE_ERROR_NOT_SUPPORTED; + goto err; + } + + pk_info = mbedtls_pk_info_from_type(MBEDTLS_PK_RSA); + if (!pk_info) { + res = TEE_ERROR_NOT_SUPPORTED; + goto err; + } + + mbedtls_rsa_set_padding(&rsa, lmd_padding, md_algo); + + lmd_res = pk_info->sign_func(&rsa, md_algo, msg, msg_len, sig, + *sig_len, sig_len, mbd_rand, NULL); + if (lmd_res != 0) { + FMSG("sign_func failed, returned 0x%x", -lmd_res); + res = get_tee_result(lmd_res); + goto err; + } + res = TEE_SUCCESS; +err: + mbd_rsa_free(&rsa, key); + return res; +} + +TEE_Result crypto_acipher_rsassa_verify(uint32_t algo, + struct rsa_public_key *key, + int salt_len __unused, + const uint8_t *msg, + size_t msg_len, const uint8_t *sig, + size_t sig_len) +__weak __alias("sw_crypto_acipher_rsassa_verify"); + +TEE_Result sw_crypto_acipher_rsassa_verify(uint32_t algo, + struct rsa_public_key *key, + int salt_len __unused, + const uint8_t *msg, + size_t msg_len, const uint8_t *sig, + size_t sig_len) +{ + TEE_Result res = TEE_SUCCESS; + int lmd_res = 0; + int lmd_padding = 0; + size_t hash_size = 0; + size_t bigint_size = 0; + mbedtls_rsa_context rsa; + const mbedtls_pk_info_t *pk_info = NULL; + uint32_t md_algo = 0; + struct ftmn ftmn = { }; + unsigned long arg_hash = 0; + + /* + * The caller expects to call crypto_acipher_rsassa_verify(), + * update the hash as needed. + */ + FTMN_CALLEE_SWAP_HASH(FTMN_FUNC_HASH("crypto_acipher_rsassa_verify")); + + memset(&rsa, 0, sizeof(rsa)); + mbedtls_rsa_init(&rsa); + + rsa.E = *(mbedtls_mpi *)key->e; + rsa.N = *(mbedtls_mpi *)key->n; + + res = tee_alg_get_digest_size(TEE_DIGEST_HASH_TO_ALGO(algo), + &hash_size); + if (res != TEE_SUCCESS) + goto err; + + if (msg_len != hash_size) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + bigint_size = crypto_bignum_num_bytes(key->n); + if (sig_len < bigint_size) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto err; + } + + rsa.len = bigint_size; + + switch (algo) { + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + arg_hash = FTMN_FUNC_HASH("mbedtls_rsa_rsassa_pkcs1_v15_verify"); + lmd_padding = MBEDTLS_RSA_PKCS_V15; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + arg_hash = FTMN_FUNC_HASH("mbedtls_rsa_rsassa_pss_verify_ext"); + lmd_padding = MBEDTLS_RSA_PKCS_V21; + break; + default: + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + + md_algo = tee_algo_to_mbedtls_hash_algo(algo); + if (md_algo == MBEDTLS_MD_NONE) { + res = TEE_ERROR_NOT_SUPPORTED; + goto err; + } + + pk_info = mbedtls_pk_info_from_type(MBEDTLS_PK_RSA); + if (!pk_info) { + res = TEE_ERROR_NOT_SUPPORTED; + goto err; + } + + mbedtls_rsa_set_padding(&rsa, lmd_padding, md_algo); + + FTMN_PUSH_LINKED_CALL(&ftmn, arg_hash); + lmd_res = pk_info->verify_func(&rsa, md_algo, msg, msg_len, + sig, sig_len); + if (!lmd_res) + FTMN_SET_CHECK_RES_FROM_CALL(&ftmn, FTMN_INCR0, lmd_res); + FTMN_POP_LINKED_CALL(&ftmn); + if (lmd_res != 0) { + FMSG("verify_func failed, returned 0x%x", -lmd_res); + res = TEE_ERROR_SIGNATURE_INVALID; + goto err; + } + res = TEE_SUCCESS; + goto out; + +err: + FTMN_SET_CHECK_RES_NOT_ZERO(&ftmn, FTMN_INCR0, res); +out: + FTMN_CALLEE_DONE_CHECK(&ftmn, FTMN_INCR0, FTMN_STEP_COUNT(1), res); + /* Reset mpi to skip freeing here, those mpis will be freed with key */ + mbedtls_mpi_init(&rsa.E); + mbedtls_mpi_init(&rsa.N); + mbedtls_rsa_free(&rsa); + return res; +} diff --git a/optee_os/lib/libmbedtls/core/sm2-dsa.c b/optee_os/lib/libmbedtls/core/sm2-dsa.c new file mode 100644 index 0000000..6e6e1d3 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/sm2-dsa.c @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2021 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" +#include "sm2-dsa.h" + +/* SM2 uses 256 bit unsigned integers in big endian format */ +#define SM2_INT_SIZE_BYTES 32 + +/* + * GM/T 0003.1‒2012 Part1 2 Section 6.1 + */ +TEE_Result sm2_mbedtls_dsa_sign(uint32_t algo __unused, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_ecp_group grp = { }; + mbedtls_ecp_point x1y1p = { }; + int mres = 0; + mbedtls_mpi k = { }; + mbedtls_mpi e = { }; + mbedtls_mpi r = { }; + mbedtls_mpi s = { }; + mbedtls_mpi tmp = { }; + + if (*sig_len < 2 * SM2_INT_SIZE_BYTES) { + *sig_len = 64; + return TEE_ERROR_SHORT_BUFFER; + } + + mbedtls_mpi_init(&k); + mbedtls_mpi_init(&e); + mbedtls_mpi_init(&r); + mbedtls_mpi_init(&s); + mbedtls_mpi_init(&tmp); + + mbedtls_ecp_point_init(&x1y1p); + + mbedtls_ecp_group_init(&grp); + mres = mbedtls_ecp_group_load(&grp, MBEDTLS_ECP_DP_SM2); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + + /* + * Steps A1 and A2 are the generation of the hash value e from user + * information (ZA) and the message to be signed (M). There are not done + * here since @msg is expected to be the hash value e already. + */ + + /* Step A3: generate random number 1 <= k < n */ + do { + res = mbed_gen_random_upto(&k, &grp.N); + if (res) + goto out; + + res = TEE_ERROR_BAD_STATE; + + /* Step A4: compute (x1, y1) = [k]G */ + + mres = mbedtls_ecp_mul(&grp, &x1y1p, &k, &grp.G, mbd_rand, + NULL); + if (mres) + goto out; + + /* Step A5: compute r = (e + x1) mod n */ + + mbedtls_mpi_read_binary(&e, (unsigned char *)msg, msg_len); + mres = mbedtls_mpi_add_mpi(&r, &e, &x1y1p.X); + if (mres) + goto out; + mres = mbedtls_mpi_mod_mpi(&r, &r, &grp.N); + if (mres) + goto out; + + /* Step A5 (continued): return to A3 if r = 0 or r + k = n */ + + mres = mbedtls_mpi_add_mpi(&tmp, &r, &k); + if (mres) + goto out; + } while (!mbedtls_mpi_cmp_int(&r, 0) || + !mbedtls_mpi_cmp_mpi(&tmp, &grp.N)); + + /* Step A6: compute s = ((1 + dA)^-1 * (k - r*dA)) mod n */ + + mres = mbedtls_mpi_add_int(&s, (mbedtls_mpi *)key->d, 1); + if (mres) + goto out; + mres = mbedtls_mpi_inv_mod(&s, &s, &grp.N); + if (mres) + goto out; + mres = mbedtls_mpi_mul_mpi(&tmp, &r, (mbedtls_mpi *)key->d); + if (mres) + goto out; + mres = mbedtls_mpi_mod_mpi(&tmp, &tmp, &grp.N); + if (mres) + goto out; + mres = mbedtls_mpi_sub_mpi(&tmp, &k, &tmp); + if (mres) + goto out; + mres = mbedtls_mpi_mul_mpi(&s, &s, &tmp); + if (mres) + goto out; + mres = mbedtls_mpi_mod_mpi(&s, &s, &grp.N); + if (mres) + goto out; + + /* Step A7: convert (r, s) to binary for output */ + + *sig_len = 2 * SM2_INT_SIZE_BYTES; + memset(sig, 0, *sig_len); + mres = mbedtls_mpi_write_binary(&r, sig, SM2_INT_SIZE_BYTES); + if (mres) + goto out; + mres = mbedtls_mpi_write_binary(&s, sig + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (mres) + goto out; + + res = TEE_SUCCESS; +out: + mbedtls_ecp_point_free(&x1y1p); + mbedtls_mpi_free(&k); + mbedtls_mpi_free(&e); + mbedtls_mpi_free(&r); + mbedtls_mpi_free(&s); + mbedtls_mpi_free(&tmp); + mbedtls_ecp_group_free(&grp); + return res; +} + +/* + * GM/T 0003.1‒2012 Part1 2 Section 7.1 + */ +TEE_Result sm2_mbedtls_dsa_verify(uint32_t algo __unused, + struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len) +{ + TEE_Result res = TEE_ERROR_BAD_STATE; + mbedtls_ecp_group grp = { }; + mbedtls_mpi rprime = { }; + mbedtls_mpi sprime = { }; + mbedtls_mpi t = { }; + mbedtls_mpi eprime = { }; + mbedtls_mpi R = { }; + mbedtls_ecp_point x1y1p = { }; + mbedtls_ecp_point PA = { }; + int mres = 0; + + if (sig_len != 64) + return TEE_ERROR_BAD_PARAMETERS; + + mbedtls_mpi_init(&rprime); + mbedtls_mpi_init(&sprime); + mbedtls_mpi_init(&t); + mbedtls_mpi_init(&eprime); + mbedtls_mpi_init(&R); + + mbedtls_ecp_point_init(&x1y1p); + mbedtls_ecp_point_init(&PA); + + mbedtls_ecp_group_init(&grp); + mres = mbedtls_ecp_group_load(&grp, MBEDTLS_ECP_DP_SM2); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + + mres = mbedtls_mpi_read_binary(&rprime, sig, 32); + if (mres) + goto out; + mres = mbedtls_mpi_read_binary(&sprime, sig + 32, 32); + if (mres) + goto out; + + /* Step B1: verify r' in [1, n - 1] */ + + if (mbedtls_mpi_cmp_int(&rprime, 1) < 0 || + mbedtls_mpi_cmp_mpi(&rprime, &grp.N) >= 0) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + /* Step B2: verify s' in [1, n - 1] */ + + if (mbedtls_mpi_cmp_int(&sprime, 1) < 0 || + mbedtls_mpi_cmp_mpi(&sprime, &grp.N) >= 0) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + /* + * Steps B3: M'bar = (ZA || M') and B4: e' = Hv(M'bar) are not done here + * because @msg is supposed to contain the hash value e' already. + */ + + /* Step B5: t = (r' + s') mod n and check t != 0 */ + + mres = mbedtls_mpi_add_mpi(&t, &rprime, &sprime); + if (mres) + goto out; + mres = mbedtls_mpi_mod_mpi(&t, &t, &grp.N); + if (mres) + goto out; + if (!mbedtls_mpi_cmp_int(&t, 0)) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + /* Step B6: (x1', y1') = [s']G + [t]PA */ + + mres = mbedtls_mpi_copy(&PA.X, (mbedtls_mpi *)key->x); + if (mres) + goto out; + mres = mbedtls_mpi_copy(&PA.Y, (mbedtls_mpi *)key->y); + if (mres) + goto out; + mres = mbedtls_mpi_lset(&PA.Z, 1); + if (mres) + goto out; + + mres = mbedtls_ecp_muladd(&grp, &x1y1p, &sprime, &grp.G, &t, &PA); + if (mres) + goto out; + + /* Step B7: compute R = (e' + x1') mod n and verify R == r' */ + + mres = mbedtls_mpi_read_binary(&eprime, msg, msg_len); + if (mres) + goto out; + mres = mbedtls_mpi_add_mpi(&R, &eprime, &x1y1p.X); + if (mres) + goto out; + mres = mbedtls_mpi_mod_mpi(&R, &R, &grp.N); + if (mres) + goto out; + if (mbedtls_mpi_cmp_mpi(&R, &rprime)) { + res = TEE_ERROR_SIGNATURE_INVALID; + goto out; + } + + res = TEE_SUCCESS; +out: + mbedtls_ecp_point_free(&x1y1p); + mbedtls_ecp_point_free(&PA); + mbedtls_mpi_free(&rprime); + mbedtls_mpi_free(&sprime); + mbedtls_mpi_free(&t); + mbedtls_mpi_free(&eprime); + mbedtls_mpi_free(&R); + mbedtls_ecp_group_free(&grp); + return res; +} diff --git a/optee_os/lib/libmbedtls/core/sm2-dsa.h b/optee_os/lib/libmbedtls/core/sm2-dsa.h new file mode 100644 index 0000000..a7b22ea --- /dev/null +++ b/optee_os/lib/libmbedtls/core/sm2-dsa.h @@ -0,0 +1,20 @@ + +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + */ + +#ifndef _SM2_DSA_H_ + +#include +#include +#include + +TEE_Result sm2_mbedtls_dsa_sign(uint32_t algo, struct ecc_keypair *key, + const uint8_t *msg, size_t msg_len, + uint8_t *sig, size_t *sig_len); + +TEE_Result sm2_mbedtls_dsa_verify(uint32_t algo, struct ecc_public_key *key, + const uint8_t *msg, size_t msg_len, + const uint8_t *sig, size_t sig_len); +#endif /* _SM2_DSA_H_ */ diff --git a/optee_os/lib/libmbedtls/core/sm2-kep.c b/optee_os/lib/libmbedtls/core/sm2-kep.c new file mode 100644 index 0000000..46f994e --- /dev/null +++ b/optee_os/lib/libmbedtls/core/sm2-kep.c @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020-21 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" + +/* SM2 uses 256 bit unsigned integers in big endian format */ +#define SM2_INT_SIZE_BYTES 32 + +/* The public x and y values extracted from a public or private ECC key */ +struct key_xy { + mbedtls_mpi *x; + mbedtls_mpi *y; +}; + +/* + * Compute a hash of a user's identity and public key + * For user A: ZA = SM3(ENTLA || IDA || a || b || xG || yG || xA || yA) + */ +static TEE_Result sm2_kep_compute_Z(const mbedtls_ecp_group *grp, uint8_t *Z, + size_t Zlen, const uint8_t *id, + size_t idlen, struct key_xy *key) +{ + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t ENTLEN[2] = { }; + uint8_t buf[SM2_INT_SIZE_BYTES] = { }; + void *ctx = NULL; + int mres = 0; + + if (Zlen < TEE_SM3_HASH_SIZE) + return TEE_ERROR_SHORT_BUFFER; + + /* + * ENTLEN is the length in bits if the user's distinguished identifier + * encoded over 16 bits in big endian format. + */ + ENTLEN[0] = (idlen * 8) >> 8; + ENTLEN[1] = idlen * 8; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + + res = crypto_hash_init(ctx); + if (res) + goto out; + + res = crypto_hash_update(ctx, ENTLEN, sizeof(ENTLEN)); + if (res) + goto out; + + res = crypto_hash_update(ctx, id, idlen); + if (res) + goto out; + + mres = mbedtls_mpi_write_binary(&grp->A, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mres = mbedtls_mpi_write_binary(&grp->B, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mres = mbedtls_mpi_write_binary(&grp->G.X, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mres = mbedtls_mpi_write_binary(&grp->G.Y, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mres = mbedtls_mpi_write_binary(key->x, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + mres = mbedtls_mpi_write_binary(key->y, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + res = crypto_hash_final(ctx, Z, TEE_SM3_HASH_SIZE); +out: + crypto_hash_free_ctx(ctx); + return res; +} + +/* + * Compute a verification value, to be checked against the value sent by the + * peer. + * On the initiator's side: + * S1 = SM3(0x02 || yU || SM3(xU || ZA || ZB || x1 || y1 || x2 || y2)) + * On the responder's side: + * S2 = SM3(0x03 || yV || SM3(xV || ZA || ZB || x1 || y1 || x2 || y2)) + */ +static TEE_Result sm2_kep_compute_S(uint8_t *S, size_t S_len, uint8_t flag, + mbedtls_ecp_point *UV, const uint8_t *ZAZB, + size_t ZAZB_len, + struct key_xy *initiator_eph_key, + struct key_xy *responder_eph_key) +{ + uint8_t hash[TEE_SM3_HASH_SIZE] = { }; + TEE_Result res = TEE_ERROR_GENERIC; + uint8_t buf[SM2_INT_SIZE_BYTES]; + void *ctx = NULL; + int mres = 0; + + if (S_len < TEE_SM3_HASH_SIZE) + return TEE_ERROR_SHORT_BUFFER; + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + + /* Compute the inner hash */ + + res = crypto_hash_init(ctx); + if (res) + goto out; + + /* xU or xV */ + mres = mbedtls_mpi_write_binary(&UV->X, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* ZA || ZB */ + res = crypto_hash_update(ctx, ZAZB, ZAZB_len); + if (res) + goto out; + + /* x1 */ + mres = mbedtls_mpi_write_binary(initiator_eph_key->x, buf, + SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* y1 */ + mres = mbedtls_mpi_write_binary(initiator_eph_key->y, buf, + SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* x2 */ + mres = mbedtls_mpi_write_binary(responder_eph_key->x, buf, + SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* y2 */ + mres = mbedtls_mpi_write_binary(responder_eph_key->y, buf, + SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + res = crypto_hash_final(ctx, hash, sizeof(hash)); + if (res) + goto out; + + /* Now compute S */ + + res = crypto_hash_init(ctx); + if (res) + goto out; + + /* 0x02 or 0x03 */ + res = crypto_hash_update(ctx, &flag, sizeof(flag)); + if (res) + goto out; + + /* yU or yV */ + mres = mbedtls_mpi_write_binary(&UV->Y, buf, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + res = crypto_hash_update(ctx, buf, sizeof(buf)); + if (res) + goto out; + + /* Inner SM3(...) */ + res = crypto_hash_update(ctx, hash, sizeof(hash)); + if (res) + goto out; + + res = crypto_hash_final(ctx, S, TEE_SM3_HASH_SIZE); + +out: + crypto_hash_free_ctx(ctx); + return res; + +} + +static void extract_xy_from_keypair(struct key_xy *xy, + const struct ecc_keypair *pair) +{ + xy->x = (mbedtls_mpi *)pair->x; + xy->y = (mbedtls_mpi *)pair->y; + /* Other fields are not used */ +} + +static void extract_xy_from_public_key(struct key_xy *xy, + const struct ecc_public_key *from) +{ + xy->x = (mbedtls_mpi *)from->x; + xy->y = (mbedtls_mpi *)from->y; +} + +/* + * GM/T 0003.1‒2012 Part 3 Section 6.1 + * Key exchange protocol + */ +TEE_Result crypto_acipher_sm2_kep_derive(struct ecc_keypair *my_key, + struct ecc_keypair *my_eph_key, + struct ecc_public_key *peer_key, + struct ecc_public_key *peer_eph_key, + struct sm2_kep_parms *p) +{ + /* + * Variable names and documented steps reflect the initator side (user A + * in the spec), but the other side is quite similar hence only one + * function. + */ + uint8_t xUyUZAZB[2 * SM2_INT_SIZE_BYTES + 2 * TEE_SM3_HASH_SIZE] = { }; + struct key_xy initiator_eph_key = { }; + struct key_xy responder_eph_key = { }; + struct key_xy initiator_key = { }; + struct key_xy responder_key = { }; + TEE_Result res = TEE_ERROR_BAD_STATE; + uint8_t tmp[SM2_INT_SIZE_BYTES] = { }; + mbedtls_ecp_group grp = { }; + mbedtls_ecp_point PB = { }; + mbedtls_ecp_point RB = { }; + mbedtls_ecp_point U = { }; + mbedtls_mpi x1bar = { }; + mbedtls_mpi x2bar = { }; + mbedtls_mpi tA = { }; + mbedtls_mpi h = { }; + mbedtls_mpi htA = { }; + mbedtls_mpi one = { }; + int mres = 0; + + if (p->is_initiator) { + extract_xy_from_keypair(&initiator_eph_key, my_eph_key); + extract_xy_from_public_key(&responder_eph_key, peer_eph_key); + extract_xy_from_keypair(&initiator_key, my_key); + extract_xy_from_public_key(&responder_key, peer_key); + } else { + extract_xy_from_public_key(&initiator_eph_key, peer_eph_key); + extract_xy_from_keypair(&responder_eph_key, my_eph_key); + extract_xy_from_public_key(&initiator_key, peer_key); + extract_xy_from_keypair(&responder_key, my_key); + } + + mbedtls_mpi_init(&x1bar); + mbedtls_mpi_init(&x2bar); + mbedtls_mpi_init(&tA); + mbedtls_mpi_init(&h); + mbedtls_mpi_init(&htA); + mbedtls_mpi_init(&one); + + mbedtls_ecp_point_init(&PB); + mbedtls_ecp_point_init(&RB); + mbedtls_ecp_point_init(&U); + + mbedtls_ecp_group_init(&grp); + mres = mbedtls_ecp_group_load(&grp, MBEDTLS_ECP_DP_SM2); + if (mres) + goto out; + + /* + * Steps A1-A3 are supposedly done already (generate ephemeral key, send + * it to peer). + * Step A4: (x1, y1) = RA; x1bar = 2^w + (x1 & (2^w - 1)) + */ + + mres = mbedtls_mpi_write_binary((mbedtls_mpi *)my_eph_key->x, tmp, + SM2_INT_SIZE_BYTES); + if (mres) + goto out; + tmp[SM2_INT_SIZE_BYTES / 2] |= 0x80; + mres = mbedtls_mpi_read_binary(&x1bar, tmp + SM2_INT_SIZE_BYTES / 2, + SM2_INT_SIZE_BYTES / 2); + if (mres) + goto out; + + /* Step A5: tA = (dA + x1bar * rA) mod n */ + + mres = mbedtls_mpi_mul_mpi(&tA, &x1bar, (mbedtls_mpi *)my_eph_key->d); + if (mres) + goto out; + mres = mbedtls_mpi_mod_mpi(&tA, &tA, &grp.N); + if (mres) + goto out; + mres = mbedtls_mpi_add_mpi(&tA, &tA, (mbedtls_mpi *)my_key->d); + if (mres) + goto out; + mres = mbedtls_mpi_mod_mpi(&tA, &tA, &grp.N); + if (mres) + goto out; + + /* Step A6: verify whether RB verifies the curve equation */ + + mbedtls_mpi_copy(&RB.X, (mbedtls_mpi *)peer_eph_key->x); + mbedtls_mpi_copy(&RB.Y, (mbedtls_mpi *)peer_eph_key->y); + mbedtls_mpi_lset(&RB.Z, 1); + mres = mbedtls_ecp_check_pubkey(&grp, &RB); + if (mres) + goto out; + + /* Step A6 (continued): (x2, y2) = RB; x2bar = 2^w + (x2 & (2^w - 1)) */ + + mres = mbedtls_mpi_write_binary((mbedtls_mpi *)peer_eph_key->x, tmp, + SM2_INT_SIZE_BYTES); + if (mres) + goto out; + tmp[SM2_INT_SIZE_BYTES / 2] |= 0x80; + mres = mbedtls_mpi_read_binary(&x2bar, tmp + SM2_INT_SIZE_BYTES / 2, + SM2_INT_SIZE_BYTES / 2); + if (mres) + goto out; + + /* Step A7: compute U = [h.tA](PB + [x2bar]RB) and check for infinity */ + + mres = mbedtls_mpi_copy(&PB.X, (mbedtls_mpi *)peer_key->x); + if (mres) + goto out; + mres = mbedtls_mpi_copy(&PB.Y, (mbedtls_mpi *)peer_key->y); + if (mres) + goto out; + mres = mbedtls_mpi_lset(&PB.Z, 1); + if (mres) + goto out; + mres = mbedtls_mpi_lset(&one, 1); + if (mres) + goto out; + + mres = mbedtls_ecp_muladd(&grp, &U, &one, &PB, &x2bar, &RB); + if (mres) + goto out; + + /* Note: the cofactor for SM2 is 1 so [h.tA] == tA */ + mres = mbedtls_ecp_mul(&grp, &U, &tA, &U, mbd_rand, NULL); + if (mres) + goto out; + + /* + * "Point is zero" is same as "point is at infinity". Returns 1 if + * point is zero, < 0 on error and 0 if point is non-zero. + */ + mres = mbedtls_ecp_is_zero(&U); + if (mres) + goto out; + + /* Step A8: compute KA = KDF(xU || yU || ZA || ZB, klen) */ + + /* xU */ + mres = mbedtls_mpi_write_binary(&U.X, xUyUZAZB, SM2_INT_SIZE_BYTES); + if (mres) + goto out; + + /* yU */ + mres = mbedtls_mpi_write_binary(&U.Y, xUyUZAZB + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (mres) + goto out; + + /* ZA */ + res = sm2_kep_compute_Z(&grp, xUyUZAZB + 2 * SM2_INT_SIZE_BYTES, + TEE_SM3_HASH_SIZE, p->initiator_id, + p->initiator_id_len, &initiator_key); + if (res) + goto out; + + /* ZB */ + res = sm2_kep_compute_Z(&grp, xUyUZAZB + 2 * SM2_INT_SIZE_BYTES + + TEE_SM3_HASH_SIZE, + TEE_SM3_HASH_SIZE, p->responder_id, + p->responder_id_len, &responder_key); + if (res) + goto out; + + res = sm2_kdf(xUyUZAZB, sizeof(xUyUZAZB), p->out, p->out_len); + if (res) + goto out; + + /* Step A9: compute S1 and check S1 == SB */ + + if (p->conf_in) { + uint8_t S1[TEE_SM3_HASH_SIZE] = { }; + uint8_t flag = p->is_initiator ? 0x02 : 0x03; + + if (p->conf_in_len < TEE_SM3_HASH_SIZE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + res = sm2_kep_compute_S(S1, sizeof(S1), flag, &U, + xUyUZAZB + 2 * SM2_INT_SIZE_BYTES, + 2 * SM2_INT_SIZE_BYTES, + &initiator_eph_key, &responder_eph_key); + if (res) + goto out; + + if (consttime_memcmp(S1, p->conf_in, sizeof(S1))) { + /* Verification failed */ + res = TEE_ERROR_BAD_STATE; + goto out; + } + } + + /* Step A10: compute SA */ + + if (p->conf_out) { + uint8_t flag = p->is_initiator ? 0x03 : 0x02; + + if (p->conf_out_len < TEE_SM3_HASH_SIZE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = sm2_kep_compute_S(p->conf_out, TEE_SM3_HASH_SIZE, flag, + &U, xUyUZAZB + 2 * SM2_INT_SIZE_BYTES, + 2 * SM2_INT_SIZE_BYTES, + &initiator_eph_key, &responder_eph_key); + } +out: + mbedtls_mpi_free(&x1bar); + mbedtls_mpi_free(&x2bar); + mbedtls_mpi_free(&tA); + mbedtls_mpi_free(&h); + mbedtls_mpi_free(&htA); + mbedtls_mpi_free(&one); + mbedtls_ecp_point_free(&PB); + mbedtls_ecp_point_free(&RB); + mbedtls_ecp_point_free(&U); + mbedtls_ecp_group_free(&grp); + return res; +} diff --git a/optee_os/lib/libmbedtls/core/sm2-pke.c b/optee_os/lib/libmbedtls/core/sm2-pke.c new file mode 100644 index 0000000..e1c3d23 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/sm2-pke.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019-2021 Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mbed_helpers.h" +#include "sm2-pke.h" + +/* SM2 uses 256 bit unsigned integers in big endian format */ +#define SM2_INT_SIZE_BYTES 32 + +static TEE_Result +sm2_uncompressed_bytes_to_point(const mbedtls_ecp_group *grp, + mbedtls_ecp_point *p, const uint8_t *x1y1, + size_t max_size, size_t *consumed) +{ + uint8_t *ptr = (uint8_t *)x1y1; + int mres = 0; + + if (max_size < (size_t)(2 * SM2_INT_SIZE_BYTES)) + return TEE_ERROR_BAD_PARAMETERS; + + mres = mbedtls_mpi_read_binary(&p->X, ptr, SM2_INT_SIZE_BYTES); + if (mres) + return TEE_ERROR_BAD_PARAMETERS; + + ptr += SM2_INT_SIZE_BYTES; + + mres = mbedtls_mpi_read_binary(&p->Y, ptr, SM2_INT_SIZE_BYTES); + if (mres) + return TEE_ERROR_BAD_PARAMETERS; + + mres = mbedtls_mpi_lset(&p->Z, 1); + if (mres) + return TEE_ERROR_BAD_PARAMETERS; + + mres = mbedtls_ecp_check_pubkey(grp, p); + if (mres) + return TEE_ERROR_BAD_PARAMETERS; + + *consumed = 2 * SM2_INT_SIZE_BYTES + 1; /* PC */ + + return TEE_SUCCESS; +} + +/* + * GM/T 0003.1‒2012 Part 1 Section 4.2.9 + * Conversion of a byte string @buf to a point @p. Makes sure @p is on the curve + * defined by domain parameters @dp. + * Note: only the uncompressed form is supported. Uncompressed and hybrid forms + * are TBD. + */ +static TEE_Result sm2_bytes_to_point(const mbedtls_ecp_group *grp, + mbedtls_ecp_point *p, const uint8_t *buf, + size_t max_size, size_t *consumed) +{ + uint8_t PC = 0; + + if (!max_size) + return TEE_ERROR_BAD_PARAMETERS; + + PC = buf[0]; + + switch (PC) { + case 0x02: + case 0x03: + /* Compressed form */ + return TEE_ERROR_NOT_SUPPORTED; + case 0x04: + /* Uncompressed form */ + return sm2_uncompressed_bytes_to_point(grp, p, buf + 1, + max_size - 1, consumed); + case 0x06: + case 0x07: + /* Hybrid form */ + return TEE_ERROR_NOT_SUPPORTED; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_ERROR_GENERIC; +} + +static bool is_zero(const uint8_t *buf, size_t size) +{ + uint8_t v = 0; + size_t i = 0; + + for (i = 0; i < size; i++) + v |= buf[i]; + + return !v; +} + +/* + * GM/T 0003.1‒2012 Part 4 Section 7.1 + * Decryption algorithm + */ +TEE_Result sm2_mbedtls_pke_decrypt(struct ecc_keypair *key, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t x2y2[64] = { }; + mbedtls_ecp_point C1 = { }; + size_t C1_len = 0; + mbedtls_ecp_point x2y2p = { }; + mbedtls_ecp_group grp = { }; + void *ctx = NULL; + int mres = 0; + uint8_t *t = NULL; + size_t C2_len = 0; + size_t i = 0; + size_t out_len = 0; + uint8_t *eom = NULL; + uint8_t u[TEE_SM3_HASH_SIZE] = { }; + + /* + * Input buffer src is (C1 || C2 || C3) + * - C1 represents a point (should be on the curve) + * - C2 is the encrypted message + * - C3 is a SM3 hash + */ + + mbedtls_ecp_point_init(&C1); + mbedtls_ecp_point_init(&x2y2p); + + mbedtls_ecp_group_init(&grp); + mres = mbedtls_ecp_group_load(&grp, MBEDTLS_ECP_DP_SM2); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + + /* Step B1: read and validate point C1 from encrypted message */ + + res = sm2_bytes_to_point(&grp, &C1, src, src_len, &C1_len); + if (res) + goto out; + + /* + * Step B2: S = [h]C1, the cofactor h is 1 for SM2 so S == C1. + * The fact that S is on the curve has already been checked in + * sm2_bytes_to_point(). + */ + + /* Step B3: (x2, y2) = [dB]C1 */ + + mres = mbedtls_ecp_mul(&grp, &x2y2p, (mbedtls_mpi *)key->d, &C1, + mbd_rand, NULL); + if (mres) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + if (mbedtls_mpi_size(&x2y2p.X) > SM2_INT_SIZE_BYTES || + mbedtls_mpi_size(&x2y2p.Y) > SM2_INT_SIZE_BYTES) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + mres = mbedtls_mpi_write_binary(&x2y2p.X, x2y2, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + mres = mbedtls_mpi_write_binary(&x2y2p.Y, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step B4: t = KDF(x2 || y2, klen) */ + + /* C = C1 || C2 || C3 */ + if (src_len <= C1_len + TEE_SM3_HASH_SIZE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + C2_len = src_len - C1_len - TEE_SM3_HASH_SIZE; + + t = calloc(1, C2_len); + if (!t) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = sm2_kdf(x2y2, sizeof(x2y2), t, C2_len); + if (res) + goto out; + + if (is_zero(t, C2_len)) { + res = TEE_ERROR_CIPHERTEXT_INVALID; + goto out; + } + + /* Step B5: get C2 from C and compute Mprime = C2 (+) t */ + + out_len = MIN(*dst_len, C2_len); + for (i = 0; i < out_len; i++) + dst[i] = src[C1_len + i] ^ t[i]; + *dst_len = out_len; + if (out_len < C2_len) { + eom = calloc(1, C2_len - out_len); + if (!eom) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + for (i = out_len; i < C2_len; i++) + eom[i - out_len] = src[C1_len + i] ^ t[i]; + } + + /* Step B6: compute u = Hash(x2 || M' || y2) and compare with C3 */ + + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, x2y2, SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_update(ctx, dst, out_len); + if (res) + goto out; + if (out_len < C2_len) { + res = crypto_hash_update(ctx, eom, C2_len - out_len); + if (res) + goto out; + } + res = crypto_hash_update(ctx, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_final(ctx, u, sizeof(u)); + if (res) + goto out; + + if (consttime_memcmp(u, src + C1_len + C2_len, TEE_SM3_HASH_SIZE)) { + res = TEE_ERROR_CIPHERTEXT_INVALID; + goto out; + } +out: + free(eom); + free(t); + crypto_hash_free_ctx(ctx); + mbedtls_ecp_point_free(&C1); + mbedtls_ecp_point_free(&x2y2p); + mbedtls_ecp_group_free(&grp); + return res; +} + +/* + * GM/T 0003.1‒2012 Part 1 Section 4.2.8 + * Conversion of point @p to a byte string @buf (uncompressed form). + */ +static TEE_Result sm2_point_to_bytes(uint8_t *buf, size_t *size, + const mbedtls_ecp_point *p) +{ + size_t xsize = mbedtls_mpi_size(&p->X); + size_t ysize = mbedtls_mpi_size(&p->Y); + size_t sz = 2 * SM2_INT_SIZE_BYTES + 1; + int mres = 0; + + if (xsize > SM2_INT_SIZE_BYTES || ysize > SM2_INT_SIZE_BYTES || + *size < sz) + return TEE_ERROR_BAD_STATE; + + memset(buf, 0, sz); + buf[0] = 0x04; /* Uncompressed form indicator */ + mres = mbedtls_mpi_write_binary(&p->X, buf + 1, SM2_INT_SIZE_BYTES); + if (mres) + return TEE_ERROR_BAD_STATE; + mres = mbedtls_mpi_write_binary(&p->Y, buf + 1 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (mres) + return TEE_ERROR_BAD_STATE; + + *size = sz; + + return TEE_SUCCESS; +} + +/* + * GM/T 0003.1‒2012 Part 4 Section 6.1 + * Encryption algorithm + */ +TEE_Result sm2_mbedtls_pke_encrypt(struct ecc_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_ecp_group grp = { }; + mbedtls_ecp_point x2y2p = { }; + mbedtls_ecp_point PB = { }; + mbedtls_ecp_point C1 = { }; + uint8_t x2y2[64] = { }; + uint8_t *t = NULL; + int mres = 0; + mbedtls_mpi k = { }; + size_t C1_len = 0; + void *ctx = NULL; + size_t i = 0; + + mbedtls_mpi_init(&k); + + mbedtls_ecp_point_init(&x2y2p); + mbedtls_ecp_point_init(&PB); + mbedtls_ecp_point_init(&C1); + + mbedtls_ecp_group_init(&grp); + mres = mbedtls_ecp_group_load(&grp, MBEDTLS_ECP_DP_SM2); + if (mres) { + res = TEE_ERROR_GENERIC; + goto out; + } + + /* Step A1: generate random number 1 <= k < n */ + + res = mbed_gen_random_upto(&k, &grp.N); + if (res) + goto out; + + /* Step A2: compute C1 = [k]G */ + + mres = mbedtls_ecp_mul(&grp, &C1, &k, &grp.G, mbd_rand, NULL); + if (mres) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* + * Step A3: compute S = [h]PB and check for infinity. + * The cofactor h is 1 for SM2 so S == PB, nothing to do. + */ + + /* Step A4: compute (x2, y2) = [k]PB */ + + mbedtls_mpi_copy(&PB.X, (mbedtls_mpi *)key->x); + mbedtls_mpi_copy(&PB.Y, (mbedtls_mpi *)key->y); + mbedtls_mpi_lset(&PB.Z, 1); + + mres = mbedtls_ecp_mul(&grp, &x2y2p, &k, &PB, mbd_rand, NULL); + if (mres) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + if (mbedtls_mpi_size(&x2y2p.X) > SM2_INT_SIZE_BYTES || + mbedtls_mpi_size(&x2y2p.Y) > SM2_INT_SIZE_BYTES) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + mres = mbedtls_mpi_write_binary(&x2y2p.X, x2y2, SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + mres = mbedtls_mpi_write_binary(&x2y2p.Y, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (mres) { + res = TEE_ERROR_BAD_STATE; + goto out; + } + + /* Step A5: compute t = KDF(x2 || y2, klen) */ + + t = calloc(1, src_len); + if (!t) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = sm2_kdf(x2y2, sizeof(x2y2), t, src_len); + if (res) + goto out; + + if (is_zero(t, src_len)) { + res = TEE_ERROR_CIPHERTEXT_INVALID; + goto out; + } + + /* + * Steps A6, A7, A8: + * Compute C2 = M (+) t + * Compute C3 = Hash(x2 || M || y2) + * Output C = C1 || C2 || C3 + */ + + /* C1 */ + C1_len = *dst_len; + res = sm2_point_to_bytes(dst, &C1_len, &C1); + if (res) + goto out; + + if (*dst_len < C1_len + src_len + TEE_SM3_HASH_SIZE) { + *dst_len = C1_len + src_len + TEE_SM3_HASH_SIZE; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + /* C2 */ + for (i = 0; i < src_len; i++) + dst[i + C1_len] = src[i] ^ t[i]; + + /* C3 */ + res = crypto_hash_alloc_ctx(&ctx, TEE_ALG_SM3); + if (res) + goto out; + res = crypto_hash_init(ctx); + if (res) + goto out; + res = crypto_hash_update(ctx, x2y2, SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_update(ctx, src, src_len); + if (res) + goto out; + res = crypto_hash_update(ctx, x2y2 + SM2_INT_SIZE_BYTES, + SM2_INT_SIZE_BYTES); + if (res) + goto out; + res = crypto_hash_final(ctx, dst + C1_len + src_len, TEE_SM3_HASH_SIZE); + if (res) + goto out; + + *dst_len = C1_len + src_len + TEE_SM3_HASH_SIZE; +out: + crypto_hash_free_ctx(ctx); + free(t); + mbedtls_ecp_point_free(&x2y2p); + mbedtls_ecp_point_free(&PB); + mbedtls_ecp_point_free(&C1); + mbedtls_ecp_group_free(&grp); + mbedtls_mpi_free(&k); + return res; +} diff --git a/optee_os/lib/libmbedtls/core/sm2-pke.h b/optee_os/lib/libmbedtls/core/sm2-pke.h new file mode 100644 index 0000000..7e8d013 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/sm2-pke.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + */ + +#ifndef _SM2_PKE_H_ + +#include +#include +#include + +TEE_Result sm2_mbedtls_pke_encrypt(struct ecc_public_key *key, + const uint8_t *src, size_t src_len, + uint8_t *dst, size_t *dst_len); + +TEE_Result sm2_mbedtls_pke_decrypt(struct ecc_keypair *key, const uint8_t *src, + size_t src_len, uint8_t *dst, + size_t *dst_len); +#endif /* _SM2_PKE_H_ */ diff --git a/optee_os/lib/libmbedtls/core/sub.mk b/optee_os/lib/libmbedtls/core/sub.mk new file mode 100644 index 0000000..2b6a517 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/sub.mk @@ -0,0 +1,33 @@ +cppflags-y += -DMBEDTLS_ALLOW_PRIVATE_ACCESS +srcs-y += mbed_helpers.c +srcs-y += tomcrypt.c +srcs-$(call cfg-one-enabled, CFG_CRYPTO_MD5 CFG_CRYPTO_SHA1 CFG_CRYPTO_SHA224 \ + CFG_CRYPTO_SHA256 CFG_CRYPTO_SHA384 \ + CFG_CRYPTO_SHA512) += hash.c + +ifeq ($(CFG_CRYPTO_AES),y) +srcs-y += aes.c +srcs-$(CFG_CRYPTO_ECB) += aes_ecb.c +srcs-$(CFG_CRYPTO_CBC) += aes_cbc.c +srcs-$(CFG_CRYPTO_CTR) += aes_ctr.c +endif +ifeq ($(CFG_CRYPTO_DES),y) +srcs-$(CFG_CRYPTO_ECB) += des_ecb.c +srcs-$(CFG_CRYPTO_ECB) += des3_ecb.c +srcs-$(CFG_CRYPTO_CBC) += des_cbc.c +srcs-$(CFG_CRYPTO_CBC) += des3_cbc.c +endif + +srcs-$(CFG_CRYPTO_HMAC) += hmac.c +srcs-$(CFG_CRYPTO_CMAC) += cmac.c + +ifneq ($(CFG_CRYPTO_DSA),y) +srcs-$(call cfg-one-enabled, CFG_CRYPTO_RSA CFG_CRYPTO_DH \ + CFG_CRYPTO_ECC) += bignum.c +endif +srcs-$(CFG_CRYPTO_RSA) += rsa.c +srcs-$(CFG_CRYPTO_DH) += dh.c +srcs-$(CFG_CRYPTO_ECC) += ecc.c +srcs-$(CFG_CRYPTO_SM2_DSA) += sm2-dsa.c +srcs-$(CFG_CRYPTO_SM2_KEP) += sm2-kep.c +srcs-$(CFG_CRYPTO_SM2_PKE) += sm2-pke.c diff --git a/optee_os/lib/libmbedtls/core/tomcrypt.c b/optee_os/lib/libmbedtls/core/tomcrypt.c new file mode 100644 index 0000000..049d9f3 --- /dev/null +++ b/optee_os/lib/libmbedtls/core/tomcrypt.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2018, ARM Limited + * Copyright (C) 2019, Linaro Limited + */ + +#include +#include + +TEE_Result crypto_init(void) +{ + tomcrypt_init(); + + return TEE_SUCCESS; +} diff --git a/optee_os/lib/libmbedtls/include/aes_alt.h b/optee_os/lib/libmbedtls/include/aes_alt.h new file mode 100644 index 0000000..88a7103 --- /dev/null +++ b/optee_os/lib/libmbedtls/include/aes_alt.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2020, Linaro Limited */ + +#ifndef __MBEDTLS_AES_ALT_H +#define __MBEDTLS_AES_ALT_H + +typedef struct mbedtls_aes_context { + uint32_t key[60]; + unsigned int round_count; +} mbedtls_aes_context; + +#endif /*__MBEDTLS_AES_ALT_H*/ diff --git a/optee_os/lib/libmbedtls/include/mbedtls_config_kernel.h b/optee_os/lib/libmbedtls/include/mbedtls_config_kernel.h new file mode 100644 index 0000000..c3924f4 --- /dev/null +++ b/optee_os/lib/libmbedtls/include/mbedtls_config_kernel.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2018, Linaro Limited */ +#ifndef __MBEDTLS_CONFIG_KERNEL_H +#define __MBEDTLS_CONFIG_KERNEL_H + +#ifdef ARM32 +#define MBEDTLS_HAVE_INT32 +#endif +#ifdef ARM64 +#define MBEDTLS_HAVE_INT64 +#endif +#define MBEDTLS_BIGNUM_C +#define MBEDTLS_GENPRIME + +/* Test if Mbedtls is the primary crypto lib */ +#ifdef CFG_CRYPTOLIB_NAME_mbedtls + +#if defined(CFG_CRYPTO_MD5) +#define MBEDTLS_MD5_C +#define MBEDTLS_MD_C +#endif + +#if defined(CFG_CRYPTO_SHA1) +#define MBEDTLS_SHA1_C +#define MBEDTLS_MD_C +#if defined(CFG_CORE_CRYPTO_SHA1_ACCEL) +#define MBEDTLS_SHA1_PROCESS_ALT +#endif +#endif + +#if defined(CFG_CRYPTO_SHA224) || defined(CFG_CRYPTO_SHA256) +#if defined(CFG_CRYPTO_SHA224) +#define MBEDTLS_SHA224_C +#endif +#if defined(CFG_CRYPTO_SHA256) +#define MBEDTLS_SHA256_C +#endif +#define MBEDTLS_MD_C +#if defined(CFG_CORE_CRYPTO_SHA256_ACCEL) +#define MBEDTLS_SHA256_PROCESS_ALT +#endif +#endif + +#if defined(CFG_CRYPTO_SHA384) || defined(CFG_CRYPTO_SHA512) +#if defined(CFG_CRYPTO_SHA384) +#define MBEDTLS_SHA384_C +#endif +#if defined(CFG_CRYPTO_SHA512) +#define MBEDTLS_SHA512_C +#endif +#define MBEDTLS_MD_C +#if defined(CFG_CORE_CRYPTO_SHA512_ACCEL) +#define MBEDTLS_SHA512_PROCESS_ALT +#endif +#endif + +#if defined(CFG_CRYPTO_HMAC) +#define MBEDTLS_MD_C +#endif + +#if defined(CFG_CRYPTO_AES) +#define MBEDTLS_AES_C +#define MBEDTLS_AES_ROM_TABLES +#if defined(CFG_CORE_CRYPTO_AES_ACCEL) +#define MBEDTLS_AES_ALT +#endif +#endif + +#if defined(CFG_CRYPTO_DES) +#define MBEDTLS_DES_C +#endif + +#if defined(CFG_CRYPTO_CBC) +#define MBEDTLS_CIPHER_MODE_CBC +#endif + +#if defined(CFG_CRYPTO_CTR) +#define MBEDTLS_CIPHER_MODE_CTR +#endif + +#if defined(CFG_CRYPTO_CMAC) +#define MBEDTLS_CMAC_C +#define MBEDTLS_CIPHER_C +#endif + +#if defined(CFG_CRYPTO_RSA) || defined(CFG_CRYPTO_DH) || defined(CFG_CRYPTO_ECC) +#define MBEDTLS_BIGNUM_C +#define MBEDTLS_OID_C +#define MBEDTLS_PKCS1_V15 +#define MBEDTLS_PKCS1_V21 +#define MBEDTLS_PK_C +#define MBEDTLS_GENPRIME +#define MBEDTLS_CTR_DRBG_C +#endif + +#if defined(CFG_CRYPTO_RSA) +#define MBEDTLS_RSA_C +#endif + +#if defined(CFG_CRYPTO_RSA) || defined(CFG_CRYPTO_ECC) +#define MBEDTLS_ASN1_PARSE_C +#define MBEDTLS_ASN1_WRITE_C +#endif + +#if defined(CFG_CRYPTO_DH) +#define MBEDTLS_DHM_C +#endif + +#if defined(CFG_CRYPTO_ECC) +#define MBEDTLS_ECP_DP_SECP192R1_ENABLED +#define MBEDTLS_ECP_DP_SECP224R1_ENABLED +#define MBEDTLS_ECP_DP_SECP256R1_ENABLED +#define MBEDTLS_ECP_DP_SECP384R1_ENABLED +#define MBEDTLS_ECP_DP_SECP521R1_ENABLED +#define MBEDTLS_ECP_DP_SECP192K1_ENABLED +#define MBEDTLS_ECP_DP_SECP224K1_ENABLED +#define MBEDTLS_ECP_DP_SECP256K1_ENABLED +#define MBEDTLS_ECP_DP_BP256R1_ENABLED +#define MBEDTLS_ECP_DP_BP384R1_ENABLED +#define MBEDTLS_ECP_DP_BP512R1_ENABLED +#define MBEDTLS_ECP_DP_CURVE25519_ENABLED +#define MBEDTLS_ECP_C +#define MBEDTLS_ECDSA_C +#define MBEDTLS_ECDH_C +#define MBEDTLS_ECDH_LEGACY_CONTEXT +#if defined(CFG_CRYPTO_DSA) || defined(CFG_CRYPTO_SM2_PKE) || \ + defined(CFG_CRYPTO_SM2_KEP) +#define MBEDTLS_ECP_DP_SM2_ENABLED +#endif +#endif + +#endif /*CFG_CRYPTOLIB_NAME_mbedtls*/ + +#include + +#endif /* __MBEDTLS_CONFIG_KERNEL_H */ diff --git a/optee_os/lib/libmbedtls/include/mbedtls_config_uta.h b/optee_os/lib/libmbedtls/include/mbedtls_config_uta.h new file mode 100644 index 0000000..83b454e --- /dev/null +++ b/optee_os/lib/libmbedtls/include/mbedtls_config_uta.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2018, Linaro Limited */ +#ifndef __MBEDTLS_CONFIG_UTA_H +#define __MBEDTLS_CONFIG_UTA_H + +/* + * When wrapping using TEE_BigInt to represent a mbedtls_mpi we can only + * use 32-bit arithmetics. + */ +#define MBEDTLS_HAVE_INT32 + +#define MBEDTLS_CIPHER_MODE_CBC +#define MBEDTLS_PKCS1_V15 + +#define MBEDTLS_CIPHER_C +#define MBEDTLS_DES_C +#define MBEDTLS_AES_C +#define MBEDTLS_NIST_KW_C +#define MBEDTLS_POLY1305_C +#define MBEDTLS_CHACHA20_C +#define MBEDTLS_CHACHAPOLY_C + +#define MBEDTLS_SHA1_C +#define MBEDTLS_SHA256_C +#define MBEDTLS_MD_C +#define MBEDTLS_MD5_C + +#define MBEDTLS_CTR_DRBG_C +#define MBEDTLS_ENTROPY_C +#define MBEDTLS_NO_PLATFORM_ENTROPY + +#define MBEDTLS_BIGNUM_C +#define MBEDTLS_GENPRIME +#define MBEDTLS_RSA_C +#define MBEDTLS_ECDH_C +#define MBEDTLS_ECDSA_C +#define MBEDTLS_ECP_C +#define MBEDTLS_ECP_DP_SECP192R1_ENABLED +#define MBEDTLS_ECP_DP_SECP224R1_ENABLED +#define MBEDTLS_ECP_DP_SECP256R1_ENABLED +#define MBEDTLS_ECP_DP_SECP384R1_ENABLED +#define MBEDTLS_ECP_DP_SECP521R1_ENABLED +#define MBEDTLS_ECP_DP_BP256R1_ENABLED +#define MBEDTLS_ECP_DP_BP384R1_ENABLED +#define MBEDTLS_ECP_DP_BP512R1_ENABLED +#define MBEDTLS_ECP_DP_SECP192K1_ENABLED +#define MBEDTLS_ECP_DP_SECP224K1_ENABLED +#define MBEDTLS_ECP_DP_SECP256K1_ENABLED + +#define MBEDTLS_PK_C +#define MBEDTLS_PK_PARSE_C +#define MBEDTLS_PK_WRITE_C +#define MBEDTLS_OID_C +#define MBEDTLS_ASN1_PARSE_C +#define MBEDTLS_ASN1_WRITE_C +#define MBEDTLS_X509_CRT_PARSE_C +#define MBEDTLS_X509_CRT_WRITE_C +#define MBEDTLS_X509_CSR_PARSE_C +#define MBEDTLS_X509_CSR_WRITE_C +#define MBEDTLS_X509_CREATE_C +#define MBEDTLS_X509_CHECK_KEY_USAGE +#define MBEDTLS_X509_USE_C +#define MBEDTLS_BASE64_C +#define MBEDTLS_CERTS_C +#define MBEDTLS_PEM_PARSE_C +#define MBEDTLS_PEM_WRITE_C + +#include + +#endif /* __MBEDTLS_CONFIG_UTA_H */ diff --git a/optee_os/lib/libmbedtls/sub.mk b/optee_os/lib/libmbedtls/sub.mk new file mode 100644 index 0000000..05951c0 --- /dev/null +++ b/optee_os/lib/libmbedtls/sub.mk @@ -0,0 +1,141 @@ +global-incdirs-y += include +global-incdirs-y += mbedtls/include + +# OBJS_CRYPTO from make file +SRCS_CRYPTO := +ifneq ($(sm),core) +SRCS_CRYPTO += aes.c +SRCS_CRYPTO += aesce.c +SRCS_CRYPTO += aesni.c +SRCS_CRYPTO += aria.c +SRCS_CRYPTO += asn1parse.c +SRCS_CRYPTO += asn1write.c +SRCS_CRYPTO += base64.c +SRCS_CRYPTO += bignum.c +SRCS_CRYPTO += bignum_core.c +SRCS_CRYPTO += bignum_mod_raw.c +SRCS_CRYPTO += camellia.c +SRCS_CRYPTO += ccm.c +SRCS_CRYPTO += chacha20.c +SRCS_CRYPTO += chachapoly.c +SRCS_CRYPTO += cipher.c +SRCS_CRYPTO += cipher_wrap.c +SRCS_CRYPTO += cmac.c +SRCS_CRYPTO += constant_time.c +SRCS_CRYPTO += ctr_drbg.c +SRCS_CRYPTO += des.c +SRCS_CRYPTO += dhm.c +SRCS_CRYPTO += ecdh.c +SRCS_CRYPTO += ecdsa.c +SRCS_CRYPTO += ecjpake.c +SRCS_CRYPTO += ecp.c +SRCS_CRYPTO += ecp_curves.c +SRCS_CRYPTO += entropy.c +SRCS_CRYPTO += entropy_poll.c +SRCS_CRYPTO += error.c +SRCS_CRYPTO += gcm.c +SRCS_CRYPTO += hash_info.c +SRCS_CRYPTO += hkdf.c +SRCS_CRYPTO += hmac_drbg.c +SRCS_CRYPTO += lmots.c +SRCS_CRYPTO += lms.c +SRCS_CRYPTO += md.c +SRCS_CRYPTO += md5.c +SRCS_CRYPTO += memory_buffer_alloc.c +SRCS_CRYPTO += nist_kw.c +SRCS_CRYPTO += oid.c +SRCS_CRYPTO += padlock.c +SRCS_CRYPTO += pem.c +SRCS_CRYPTO += pk.c +SRCS_CRYPTO += pk_wrap.c +SRCS_CRYPTO += pkcs7.c +SRCS_CRYPTO += pkcs12.c +SRCS_CRYPTO += pkcs5.c +SRCS_CRYPTO += pkparse.c +SRCS_CRYPTO += pkwrite.c +SRCS_CRYPTO += platform.c +SRCS_CRYPTO += platform_util.c +SRCS_CRYPTO += poly1305.c +SRCS_CRYPTO += ripemd160.c +SRCS_CRYPTO += rsa.c +SRCS_CRYPTO += rsa_alt_helpers.c +SRCS_CRYPTO += sha1.c +SRCS_CRYPTO += sha256.c +SRCS_CRYPTO += sha512.c +SRCS_CRYPTO += threading.c +SRCS_CRYPTO += timing.c +SRCS_CRYPTO += version.c +SRCS_CRYPTO += version_features.c +else +SRCS_CRYPTO += aes.c +SRCS_CRYPTO += aesni.c +SRCS_CRYPTO += asn1parse.c +SRCS_CRYPTO += asn1write.c +SRCS_CRYPTO += bignum.c +SRCS_CRYPTO += bignum_core.c +SRCS_CRYPTO += bignum_mod_raw.c +SRCS_CRYPTO += cipher.c +SRCS_CRYPTO += cipher_wrap.c +SRCS_CRYPTO += cmac.c +SRCS_CRYPTO += constant_time.c +SRCS_CRYPTO += ctr_drbg.c +SRCS_CRYPTO += des.c +SRCS_CRYPTO += dhm.c +SRCS_CRYPTO += ecdh.c +SRCS_CRYPTO += ecdsa.c +SRCS_CRYPTO += ecp.c +SRCS_CRYPTO += ecp_curves.c +SRCS_CRYPTO += hash_info.c +SRCS_CRYPTO += md.c +SRCS_CRYPTO += md5.c +SRCS_CRYPTO += oid.c +SRCS_CRYPTO += pk.c +SRCS_CRYPTO += pk_wrap.c +SRCS_CRYPTO += platform.c +SRCS_CRYPTO += platform_util.c +SRCS_CRYPTO += rsa.c +SRCS_CRYPTO += rsa_alt_helpers.c +SRCS_CRYPTO += sha1.c +SRCS_CRYPTO += sha256.c +SRCS_CRYPTO += sha512.c +endif + +# OBJS_X509 +SRCS_X509 := +SRCS_X509 += x509.c +SRCS_X509 += x509_create.c +SRCS_X509 += x509_crl.c +SRCS_X509 += x509_crt.c +SRCS_X509 += x509_csr.c +SRCS_X509 += x509write_crt.c +SRCS_X509 += x509write_csr.c + +# OBJS_TLS +SRCS_TLS := +SRCS_TLS += debug.c +SRCS_TLS += net_sockets.c +SRCS_TLS += ssl_cache.c +SRCS_TLS += ssl_client.c +SRCS_TLS += ssl_ciphersuites.c +SRCS_TLS += ssl_cookie.c +SRCS_TLS += ssl_debug_helpers_generated.c +SRCS_TLS += ssl_ticket.c +SRCS_TLS += ssl_tls.c +SRCS_TLS += ssl_tls12_client.c +SRCS_TLS += ssl_tls12_server.c +SRCS_TLS += ssl_tls13_client.c +SRCS_TLS += ssl_tls13_generic.c +SRCS_TLS += ssl_tls13_server.c + +srcs-y += $(addprefix mbedtls/library/, $(SRCS_CRYPTO)) +srcs-$(sm-$(ta-target)) += $(addprefix mbedtls/library/, $(SRCS_X509)) +srcs-$(sm-$(ta-target)) += $(addprefix mbedtls/library/, $(SRCS_TLS)) + +cflags-lib-y += -Wno-redundant-decls +cflags-lib-y += -Wno-switch-default +cflags-lib-y += -Wno-declaration-after-statement +cflags-lib-y += -Wno-unused-function + +ifeq ($(CFG_CRYPTOLIB_NAME_mbedtls),y) +subdirs-$(sm-core) += core +endif diff --git a/optee_os/lib/libunw/include/unw/unwind.h b/optee_os/lib/libunw/include/unw/unwind.h new file mode 100644 index 0000000..79d135b --- /dev/null +++ b/optee_os/lib/libunw/include/unw/unwind.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: (BSD-2-Clause AND MIT-CMU) */ +/*- + * Copyright (c) 2023 Andes Technology Corporation + * Copyright (c) 2015-2019, Linaro Limited + * Copyright (c) 2000, 2001 Ben Harris + * Copyright (c) 1996 Scott K. Stevens + * + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + * + * $FreeBSD$ + */ + +#ifndef UNW_UNWIND_H +#define UNW_UNWIND_H + +#include +#include + +/* The state of the unwind process (32-bit mode) */ +struct unwind_state_arm32 { + uint32_t registers[16]; + uint32_t start_pc; + vaddr_t insn; + unsigned int entries; + unsigned int byte; + uint16_t update_mask; +}; + +#ifdef CFG_UNWIND +/* + * Unwind a 32-bit stack. + * @stack, @stack_size: the bottom of the stack and its size, respectively. + * Returns false when there is nothing more to unwind. + */ +bool unwind_stack_arm32(struct unwind_state_arm32 *state, + vaddr_t stack, size_t stack_size); + +void print_stack_arm32(struct unwind_state_arm32 *state, + vaddr_t stack, size_t stack_size); +#else +static inline bool unwind_stack_arm32(struct unwind_state_arm32 *state __unused, + vaddr_t stack __unused, + size_t stack_size __unused) +{ + return false; +} + +static inline void print_stack_arm32(struct unwind_state_arm32 *state __unused, + vaddr_t stack __unused, + size_t stack_size __unused) +{ +} +#endif + +/* + * External helper function. Must be implemented by the caller of the 32-bit + * stack unwinding functions. + */ +bool find_exidx(vaddr_t addr, vaddr_t *idx_start, vaddr_t *idx_end); + +/* The state of the unwind process (64-bit mode) */ +struct unwind_state_arm64 { + uint64_t fp; + uint64_t sp; + uint64_t pc; +}; + +#if defined(ARM64) && defined(CFG_UNWIND) +/* + * Unwind a 64-bit stack. + * @stack, @stack_size: the bottom of the stack and its size, respectively. + * Returns false when there is nothing more to unwind. + */ +bool unwind_stack_arm64(struct unwind_state_arm64 *state, + vaddr_t stack, size_t stack_size); + +void print_stack_arm64(struct unwind_state_arm64 *state, + vaddr_t stack, size_t stack_size); +#else +static inline bool unwind_stack_arm64(struct unwind_state_arm64 *state __unused, + vaddr_t stack __unused, + size_t stack_size __unused) +{ + return false; +} + +static inline void print_stack_arm64(struct unwind_state_arm64 *state __unused, + vaddr_t stack __unused, + size_t stack_size __unused) +{ +} +#endif + +/* The state of the unwind process */ +struct unwind_state_riscv { + unsigned long fp; + unsigned long pc; +}; + +#if (defined(RV32) || defined(RV64)) && defined(CFG_UNWIND) +/* + * Unwind stack. + * @stack, @stack_size: the bottom of the stack and its size, respectively. + * Returns false when there is nothing more to unwind. + */ +bool unwind_stack_riscv(struct unwind_state_riscv *state, + vaddr_t stack, size_t stack_size); + +void print_stack_riscv(struct unwind_state_riscv *state, + vaddr_t stack, size_t stack_size); +#else +static inline bool unwind_stack_riscv(struct unwind_state_riscv *state __unused, + vaddr_t stack __unused, + size_t stack_size __unused) +{ + return false; +} + +static inline void print_stack_riscv(struct unwind_state_riscv *state __unused, + vaddr_t stack __unused, + size_t stack_size __unused) +{ +} +#endif + +/* + * External helper function optionally implemented by the caller of the 64-bit + * stack unwinding functions. + */ +void ftrace_map_lr(uint64_t *lr); + +/* Strip out PAuth tags from LR content if applicable */ +void pauth_strip_pac(uint64_t *lr); + +#endif /*UNW_UNWIND_H*/ diff --git a/optee_os/lib/libunw/sub.mk b/optee_os/lib/libunw/sub.mk new file mode 100644 index 0000000..e383801 --- /dev/null +++ b/optee_os/lib/libunw/sub.mk @@ -0,0 +1,8 @@ +global-incdirs-y += include +ifeq ($(CFG_UNWIND),y) +srcs-y += unwind_arm32.c +srcs-$(CFG_ARM64_$(sm)) += unwind_arm64.c +ifneq (,$(filter y,$(CFG_RV32_$(sm)) $(CFG_RV64_$(sm)))) +srcs-y += unwind_riscv.c +endif +endif diff --git a/optee_os/lib/libunw/unwind_arm32.c b/optee_os/lib/libunw/unwind_arm32.c new file mode 100644 index 0000000..18b3480 --- /dev/null +++ b/optee_os/lib/libunw/unwind_arm32.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2015-2019 Linaro Limited + * Copyright 2013-2014 Andrew Turner. + * Copyright 2013-2014 Ian Lepore. + * Copyright 2013-2014 Rui Paulo. + * Copyright 2013 Eitan Adler. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +/* The register names */ +#define FP 11 +#define SP 13 +#define LR 14 +#define PC 15 + +/* + * Definitions for the instruction interpreter. + * + * The ARM EABI specifies how to perform the frame unwinding in the + * Exception Handling ABI for the ARM Architecture document. To perform + * the unwind we need to know the initial frame pointer, stack pointer, + * link register and program counter. We then find the entry within the + * index table that points to the function the program counter is within. + * This gives us either a list of three instructions to process, a 31-bit + * relative offset to a table of instructions, or a value telling us + * we can't unwind any further. + * + * When we have the instructions to process we need to decode them + * following table 4 in section 9.3. This describes a collection of bit + * patterns to encode that steps to take to update the stack pointer and + * link register to the correct values at the start of the function. + */ + +/* A special case when we are unable to unwind past this function */ +#define EXIDX_CANTUNWIND 1 + +/* + * Entry types. + * These are the only entry types that have been seen in the kernel. + */ +#define ENTRY_MASK 0xff000000 +#define ENTRY_ARM_SU16 0x80000000 +#define ENTRY_ARM_LU16 0x81000000 + +/* Instruction masks. */ +#define INSN_VSP_MASK 0xc0 +#define INSN_VSP_SIZE_MASK 0x3f +#define INSN_STD_MASK 0xf0 +#define INSN_STD_DATA_MASK 0x0f +#define INSN_POP_TYPE_MASK 0x08 +#define INSN_POP_COUNT_MASK 0x07 +#define INSN_VSP_LARGE_INC_MASK 0xff + +/* Instruction definitions */ +#define INSN_VSP_INC 0x00 +#define INSN_VSP_DEC 0x40 +#define INSN_POP_MASKED 0x80 +#define INSN_VSP_REG 0x90 +#define INSN_POP_COUNT 0xa0 +#define INSN_FINISH 0xb0 +#define INSN_POP_REGS 0xb1 +#define INSN_VSP_LARGE_INC 0xb2 + +/* An item in the exception index table */ +struct unwind_idx { + uint32_t offset; + uint32_t insn; +}; + +static bool copy_in(void *dst, const void *src, size_t n) +{ + memcpy(dst, src, n); + return true; +} + +/* Expand a 31-bit signed value to a 32-bit signed value */ +static int32_t expand_prel31(uint32_t prel31) +{ + return prel31 | SHIFT_U32(prel31 & BIT32(30), 1); +} + +/* + * Perform a binary search of the index table to find the function + * with the largest address that does not exceed addr. + */ +static struct unwind_idx *find_index(uint32_t addr) +{ + vaddr_t idx_start = 0; + vaddr_t idx_end = 0; + unsigned int min = 0; + unsigned int mid = 0; + unsigned int max = 0; + struct unwind_idx *start = NULL; + struct unwind_idx *item = NULL; + int32_t prel31_addr = 0; + vaddr_t func_addr = 0; + + if (!find_exidx(addr, &idx_start, &idx_end)) + return NULL; + + start = (struct unwind_idx *)idx_start; + + min = 0; + max = (idx_end - idx_start) / sizeof(struct unwind_idx); + + while (min != max) { + mid = min + (max - min + 1) / 2; + + item = &start[mid]; + + prel31_addr = expand_prel31(item->offset); + func_addr = (vaddr_t)&item->offset + prel31_addr; + + if (func_addr <= addr) + min = mid; + else + max = mid - 1; + } + + return &start[min]; +} + +/* Reads the next byte from the instruction list */ +static bool unwind_exec_read_byte(struct unwind_state_arm32 *state, + uint32_t *ret_insn) +{ + uint32_t insn; + + if (!copy_in(&insn, (void *)state->insn, sizeof(insn))) + return false; + + /* Read the unwind instruction */ + *ret_insn = (insn >> (state->byte * 8)) & 0xff; + + /* Update the location of the next instruction */ + if (state->byte == 0) { + state->byte = 3; + state->insn += sizeof(uint32_t); + state->entries--; + } else { + state->byte--; + } + + return true; +} + +static bool pop_vsp(uint32_t *reg, vaddr_t *vsp, vaddr_t stack, + size_t stack_size) +{ + if (*vsp < stack) + return false; + if (*vsp + sizeof(*reg) > stack + stack_size) + return false; + + if (!copy_in(reg, (void *)*vsp, sizeof(*reg))) + return false; + (*vsp) += sizeof(*reg); + return true; +} + +/* Executes the next instruction on the list */ +static bool unwind_exec_insn(struct unwind_state_arm32 *state, vaddr_t stack, + size_t stack_size) +{ + uint32_t insn; + vaddr_t vsp = state->registers[SP]; + int update_vsp = 0; + + /* Read the next instruction */ + if (!unwind_exec_read_byte(state, &insn)) + return false; + + if ((insn & INSN_VSP_MASK) == INSN_VSP_INC) { + state->registers[SP] += ((insn & INSN_VSP_SIZE_MASK) << 2) + 4; + + } else if ((insn & INSN_VSP_MASK) == INSN_VSP_DEC) { + state->registers[SP] -= ((insn & INSN_VSP_SIZE_MASK) << 2) + 4; + + } else if ((insn & INSN_STD_MASK) == INSN_POP_MASKED) { + uint32_t mask; + unsigned int reg; + + /* Load the mask */ + if (!unwind_exec_read_byte(state, &mask)) + return false; + mask |= (insn & INSN_STD_DATA_MASK) << 8; + + /* We have a refuse to unwind instruction */ + if (mask == 0) + return false; + + /* Update SP */ + update_vsp = 1; + + /* Load the registers */ + for (reg = 4; mask && reg < 16; mask >>= 1, reg++) { + if (mask & 1) { + if (!pop_vsp(&state->registers[reg], &vsp, + stack, stack_size)) + return false; + state->update_mask |= 1 << reg; + + /* If we have updated SP kep its value */ + if (reg == SP) + update_vsp = 0; + } + } + + } else if ((insn & INSN_STD_MASK) == INSN_VSP_REG && + ((insn & INSN_STD_DATA_MASK) != 13) && + ((insn & INSN_STD_DATA_MASK) != 15)) { + /* sp = register */ + state->registers[SP] = + state->registers[insn & INSN_STD_DATA_MASK]; + + } else if ((insn & INSN_STD_MASK) == INSN_POP_COUNT) { + unsigned int count, reg; + + /* Read how many registers to load */ + count = insn & INSN_POP_COUNT_MASK; + + /* Update sp */ + update_vsp = 1; + + /* Pop the registers */ + for (reg = 4; reg <= 4 + count; reg++) { + if (!pop_vsp(&state->registers[reg], &vsp, + stack, stack_size)) + return false; + state->update_mask |= 1 << reg; + } + + /* Check if we are in the pop r14 version */ + if ((insn & INSN_POP_TYPE_MASK) != 0) { + if (!pop_vsp(&state->registers[14], &vsp, + stack, stack_size)) + return false; + } + + } else if (insn == INSN_FINISH) { + /* Stop processing */ + state->entries = 0; + + } else if (insn == INSN_POP_REGS) { + uint32_t mask; + unsigned int reg; + + if (!unwind_exec_read_byte(state, &mask)) + return false; + if (mask == 0 || (mask & 0xf0) != 0) + return false; + + /* Update SP */ + update_vsp = 1; + + /* Load the registers */ + for (reg = 0; mask && reg < 4; mask >>= 1, reg++) { + if (mask & 1) { + if (!pop_vsp(&state->registers[reg], &vsp, + stack, stack_size)) + return false; + state->update_mask |= 1 << reg; + } + } + + } else if ((insn & INSN_VSP_LARGE_INC_MASK) == INSN_VSP_LARGE_INC) { + uint32_t uleb128; + + /* Read the increment value */ + if (!unwind_exec_read_byte(state, &uleb128)) + return false; + + state->registers[SP] += 0x204 + (uleb128 << 2); + + } else { + /* We hit a new instruction that needs to be implemented */ + DMSG("Unhandled instruction %.2x", insn); + return false; + } + + if (update_vsp) + state->registers[SP] = vsp; + + return true; +} + +/* Performs the unwind of a function */ +static bool unwind_tab(struct unwind_state_arm32 *state, vaddr_t stack, + size_t stack_size) +{ + uint32_t entry; + uint32_t insn; + + /* Set PC to a known value */ + state->registers[PC] = 0; + + if (!copy_in(&insn, (void *)state->insn, sizeof(insn))) { + DMSG("Bad insn addr %p", (void *)state->insn); + return true; + } + + /* Read the personality */ + entry = insn & ENTRY_MASK; + + if (entry == ENTRY_ARM_SU16) { + state->byte = 2; + state->entries = 1; + } else if (entry == ENTRY_ARM_LU16) { + state->byte = 1; + state->entries = ((insn >> 16) & 0xFF) + 1; + } else { + DMSG("Unknown entry: %x", entry); + return true; + } + + while (state->entries > 0) { + if (!unwind_exec_insn(state, stack, stack_size)) + return true; + } + + /* + * The program counter was not updated, load it from the link register. + */ + if (state->registers[PC] == 0) { + state->registers[PC] = state->registers[LR]; + + /* + * If the program counter changed, flag it in the update mask. + */ + if (state->start_pc != state->registers[PC]) + state->update_mask |= 1 << PC; + } + + return false; +} + +bool unwind_stack_arm32(struct unwind_state_arm32 *state, + vaddr_t stack, size_t stack_size) +{ + struct unwind_idx *index; + bool finished; + + /* Reset the mask of updated registers */ + state->update_mask = 0; + + /* The pc value is correct and will be overwritten, save it */ + state->start_pc = state->registers[PC]; + + /* + * Find the item to run. Subtract 2 from PC to make sure that we're + * still inside the calling function in case a __no_return function + * (typically panic()) is called unconditionally and may cause LR and + * thus this PC to point into the next and entirely unrelated function. + */ + index = find_index(state->start_pc - 2); + if (!index) + return false; + + finished = false; + if (index->insn != EXIDX_CANTUNWIND) { + if (index->insn & (1U << 31)) { + /* The data is within the instruction */ + state->insn = (vaddr_t)&index->insn; + } else { + /* A prel31 offset to the unwind table */ + state->insn = (vaddr_t)&index->insn + + expand_prel31(index->insn); + } + + /* Run the unwind function */ + finished = unwind_tab(state, stack, stack_size); + } + + /* This is the top of the stack, finish */ + if (index->insn == EXIDX_CANTUNWIND) + finished = true; + + return !finished; +} + +void print_stack_arm32(struct unwind_state_arm32 *state, + vaddr_t stack, size_t stack_size) +{ + trace_printf_helper_raw(TRACE_ERROR, true, "Call stack:"); + do { + trace_printf_helper_raw(TRACE_ERROR, true, " 0x%08" PRIx32, + state->registers[PC]); + } while (unwind_stack_arm32(state, stack, stack_size)); +} diff --git a/optee_os/lib/libunw/unwind_arm64.c b/optee_os/lib/libunw/unwind_arm64.c new file mode 100644 index 0000000..b6e6cd1 --- /dev/null +++ b/optee_os/lib/libunw/unwind_arm64.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: BSD-2-Clause +/*- + * Copyright (c) 2015-2019 Linaro Limited + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +void __weak ftrace_map_lr(uint64_t *lr __unused) +{} + +void __weak pauth_strip_pac(uint64_t *lr __unused) +{} + +static bool copy_in_reg(uint64_t *reg, vaddr_t addr) +{ + memcpy(reg, (void *)addr, sizeof(*reg)); + return true; +} + +bool unwind_stack_arm64(struct unwind_state_arm64 *frame, + vaddr_t stack, size_t stack_size) +{ + vaddr_t fp = frame->fp; + + if (fp < stack) + return false; + if (fp + sizeof(uint64_t) * 3 > stack + stack_size) + return false; + + frame->sp = fp + 0x10; + /* FP to previous frame (X29) */ + if (!copy_in_reg(&frame->fp, fp)) + return false; + if (!frame->fp) + return false; + /* LR (X30) */ + if (!copy_in_reg(&frame->pc, fp + 8)) + return false; + + pauth_strip_pac(&frame->pc); + + ftrace_map_lr(&frame->pc); + + frame->pc -= 4; + + return true; +} + +void print_stack_arm64(struct unwind_state_arm64 *state, + vaddr_t stack, size_t stack_size) +{ + vaddr_t va = 0; + int width = 8; + + trace_printf_helper_raw(TRACE_ERROR, true, "Call stack:"); + + ftrace_map_lr(&state->pc); + do { + va = memtag_strip_tag_vaddr((void *)state->pc); + trace_printf_helper_raw(TRACE_ERROR, true, " 0x%0*"PRIxVA, + width, va); + } while (unwind_stack_arm64(state, stack, stack_size)); +} diff --git a/optee_os/lib/libunw/unwind_riscv.c b/optee_os/lib/libunw/unwind_riscv.c new file mode 100644 index 0000000..e777015 --- /dev/null +++ b/optee_os/lib/libunw/unwind_riscv.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-2-Clause +/*- + * Copyright (c) 2023 Andes Technology Corporation + * Copyright (c) 2015-2019 Linaro Limited + * Copyright (c) 2015 The FreeBSD Foundation + */ + +#include +#include +#include +#include +#include +#include + +void __weak ftrace_map_lr(uint64_t *lr __unused) +{ +} + +bool unwind_stack_riscv(struct unwind_state_riscv *frame, + vaddr_t stack, size_t stack_size) +{ + vaddr_t fp = frame->fp; + struct unwind_state_riscv *caller_state = NULL; + + if (fp < stack) + return false; + if (fp > stack + stack_size) + return false; + + /* + * | ..... | ^ unwind upwards + * | ..... | | + * +=============+ <--+ | +======= caller FP ==========+ + * | RA | | | + * +-------------+ | | + * | caller FP | ---|-+ ^ + * +-------------+ | caller stack frame + * | ..... | | v + * | ..... | | + * | ..... | | + * +=============+ | +== caller SP / trapped FP ==+ + * | RA | | + * +-------------+ | + * | caller FP | ---+ ^ + * +-------------+ trapped stack frame + * | ..... | v + * | ..... | + * | ..... | + * +=============+ +======== trapped SP ========+ + * | + * | grow downwards + * V + */ + + /* Get caller FP and RA */ + caller_state = (struct unwind_state_riscv *)fp - 1; + frame->fp = caller_state->fp; + frame->pc = caller_state->pc; + + ftrace_map_lr(&frame->pc); + + frame->pc -= 4; + + return true; +} + +void print_stack_riscv(struct unwind_state_riscv *state, + vaddr_t stack, size_t stack_size) +{ + int width = sizeof(unsigned long); + + trace_printf_helper_raw(TRACE_ERROR, true, "Call stack:"); + + ftrace_map_lr(&state->pc); + do { + trace_printf_helper_raw(TRACE_ERROR, true, " 0x%0*"PRIxVA, + width, state->pc); + } while (unwind_stack_riscv(state, stack, stack_size)); +} diff --git a/optee_os/lib/libutee/abort.c b/optee_os/lib/libutee/abort.c new file mode 100644 index 0000000..09ccd96 --- /dev/null +++ b/optee_os/lib/libutee/abort.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include +#include +#include +#include + +void abort(void) +{ + printf("Abort!\n"); + _utee_panic(0); + /* Not reached */ + while (1) + ; +} diff --git a/optee_os/lib/libutee/arch/arm/arm32_user_sysreg.txt b/optee_os/lib/libutee/arch/arm/arm32_user_sysreg.txt new file mode 100644 index 0000000..8905219 --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/arm32_user_sysreg.txt @@ -0,0 +1,14 @@ +# Format of file +# +# lines beginning with '@' will be printed as additional comments + +@ Based on register description in +@ ARM Architecture Reference Manual +@ ARMv7-A and ARMv7-R edition +@ Issue C.c +@ + +@ B8.2 Generic Timer registers summary +CNTFRQ c14 0 c0 0 RW Counter Frequency register +CNTPCT - 0 c14 - RO Physical Count register +CNTVCT - 1 c14 - RO Virtual Count register diff --git a/optee_os/lib/libutee/arch/arm/gprof/gmon.h b/optee_os/lib/libutee/arch/arm/gprof/gmon.h new file mode 100644 index 0000000..da5466b --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/gprof/gmon.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: (BSD-2-Clause AND BSD-3-Clause) */ +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This file is adapted from glibc' gmon/sys/gmon.h. + *- + * Copyright (c) 1982, 1986, 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * See gmon_out.h for gmon.out format. + */ + +#ifndef GMON_H +#define GMON_H + +#include +#include + +/* Exported by the TA linker script */ +extern uint8_t __text_start[]; +extern uint8_t __text_end[]; + +void __mcount_internal(unsigned long frompc, unsigned long selfpc); + + +/* + * Histogram counters are unsigned shorts (according to the kernel). + */ +#define HISTCOUNTER unsigned short + +/* + * Fraction of text space to allocate for histogram counters here, 1/2 + */ +#define HISTFRACTION 2 + +/* + * Fraction of text space to allocate for from hash buckets. + * The value of HASHFRACTION is based on the minimum number of bytes + * of separation between two subroutine call points in the object code. + * Given MIN_SUBR_SEPARATION bytes of separation the value of + * HASHFRACTION is calculated as: + * + * HASHFRACTION = MIN_SUBR_SEPARATION / (2 * sizeof(short) - 1); + * + * For example, on the VAX, the shortest two call sequence is: + * + * calls $0,(r0) + * calls $0,(r0) + * + * which is separated by only three bytes, thus HASHFRACTION is + * calculated as: + * + * HASHFRACTION = 3 / (2 * 2 - 1) = 1 + * + * Note that the division above rounds down, thus if MIN_SUBR_FRACTION + * is less than three, this algorithm will not work! + * + * In practice, however, call instructions are rarely at a minimal + * distance. Hence, we will define HASHFRACTION to be 2 across all + * architectures. This saves a reasonable amount of space for + * profiling data structures without (in practice) sacrificing + * any granularity. + */ +#define HASHFRACTION 2 + +/* + * Percent of text space to allocate for tostructs. + * This is a heuristic; we will fail with a warning when profiling programs + * with a very large number of very small functions, but that's + * normally OK. + * 2 is probably still a good value for normal programs. + * Profiling a test case with 64000 small functions will work if + * you raise this value to 3 and link statically (which bloats the + * text size, thus raising the number of arcs expected by the heuristic). + */ +#define ARCDENSITY 3 + +/* + * Always allocate at least this many tostructs. This + * hides the inadequacy of the ARCDENSITY heuristic, at least + * for small programs. + */ +#define MINARCS 50 + +/* + * The type used to represent indices into gmonparam.tos[]. + */ +#define ARCINDEX unsigned long + +/* + * Maximum number of arcs we want to allow. + * Used to be max representable value of ARCINDEX minus 2, but now + * that ARCINDEX is a long, that's too large; we don't really want + * to allow a 48 gigabyte table. + * The old value of 1<<16 wasn't high enough in practice for large C++ + * programs; will 1<<20 be adequate for long? FIXME + */ +#define MAXARCS (1 << 20) + +struct tostruct { + unsigned long selfpc; + long count; + ARCINDEX link; +}; + +/* + * A raw arc, with pointers to the calling site and the called site and a + * count. + */ +struct rawarc { + unsigned long raw_frompc; + unsigned long raw_selfpc; + long raw_count; +}; + +/* + * The profiling data structures are housed in this structure. + */ +struct gmonparam { + long int state; + unsigned short *kcount; + unsigned long kcountsize; + ARCINDEX *froms; + unsigned long fromssize; + struct tostruct *tos; + unsigned long tossize; + unsigned long tolimit; + unsigned long lowpc; + unsigned long highpc; + unsigned long textsize; + unsigned long hashfraction; + long log_hashfraction; + /* */ + uint32_t prof_rate; /* PC sampling frequency */ +}; + +/* + * Possible states of profiling. + */ +#define GMON_PROF_ON 0 +#define GMON_PROF_BUSY 1 +#define GMON_PROF_ERROR 2 +#define GMON_PROF_OFF 3 +#define GMON_PROF_OFF_EXITING 4 + +#endif /* GMON_H */ diff --git a/optee_os/lib/libutee/arch/arm/gprof/gmon_out.h b/optee_os/lib/libutee/arch/arm/gprof/gmon_out.h new file mode 100644 index 0000000..6faadaa --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/gprof/gmon_out.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * gmon.out file format + * + * This file is adapted from glibc's gmon/sys/gmon_out.h + * Although gmon/sys/gmon_out.h is covered by the LGPL v2.1 license or later + * as stated below, please note the following: + * (https://www.gnu.org/licenses/lgpl-3.0.en.html#section3) + * + * "3. Object Code Incorporating Material from Library Header Files. + * The object code form of an Application may incorporate material from a + * header file that is part of the Library. You may convey such object code + * under terms of your choice, provided that, if the incorporated material + * is not limited to numerical parameters, data structure layouts and + * accessors, or small macros, inline functions and templates (ten or fewer + * lines in length), you do both of the following: [...]" + * + * The code below is indeed limited to data structure layouts. + */ + +/* + * Copyright (C) 1996-2016 Free Software Foundation, Inc. + * This file is part of the GNU C Library. + * Contributed by David Mosberger . + * + * The GNU C Library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * The GNU C Library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with the GNU C Library; if not, see + * . + */ + +/* + * This file specifies the format of gmon.out files. It should have + * as few external dependencies as possible as it is going to be included + * in many different programs. That is, minimize the number of #include's. + * + * A gmon.out file consists of a header (defined by gmon_hdr) followed by + * a sequence of records. Each record starts with a one-byte tag + * identifying the type of records, followed by records specific data. + */ + +#ifndef GMON_OUT_H +#define GMON_OUT_H + +#define GMON_MAGIC "gmon" /* magic cookie */ +#define GMON_VERSION 1 /* version number */ + +/* + * Raw header as it appears on file (without padding). This header + * always comes first in gmon.out and is then followed by a series + * records defined below. + * Virtual addresses are stored as uintptr_t, gprof knows which size to expect + * because the executable file is provided. + */ +struct gmon_hdr { + char cookie[4]; + int32_t version; + char spare[3 * 4]; +} __packed; + +/* types of records in this file: */ +enum gmon_record_tag { + GMON_TAG_TIME_HIST = 0, + GMON_TAG_CG_ARC = 1, + GMON_TAG_BB_COUNT = 2 +}; + +struct gmon_hist_hdr { + uintptr_t low_pc; /* base pc address of sample buffer */ + uintptr_t high_pc; /* max pc address of sampled buffer */ + uint32_t hist_size; /* size of sample buffer */ + uint32_t prof_rate; /* profiling clock rate */ + char dimen[15]; /* phys. dim., usually "seconds" */ + char dimen_abbrev; /* usually 's' for "seconds" */ +} __packed; + +struct gmon_cg_arc_record { + uintptr_t from_pc; /* address within caller's body */ + uintptr_t self_pc; /* address within callee's body */ + int32_t count; /* number of arc traversals */ +} __packed; + +#endif /* GMON_OUT_H */ diff --git a/optee_os/lib/libutee/arch/arm/gprof/gprof.c b/optee_os/lib/libutee/arch/arm/gprof/gprof.c new file mode 100644 index 0000000..d35da96 --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/gprof/gprof.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: (BSD-2-Clause AND BSD-3-Clause) +/* + * Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Portions of this file are adapted from glibc: + * gmon/gmon.c + * gmon/mcount.c + * + *- + * Copyright (c) 1983, 1992, 1993, 2011 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gmon.h" +#include "gmon_out.h" +#include "gprof_pta.h" + +static void *gprof_buf; +static size_t gprof_buf_len; + +#if defined(ARM32) +#define MCOUNT_SYM __gnu_mcount_nc +#elif defined(ARM64) +#define MCOUNT_SYM _mcount +#endif + +static void dummy(void) {} +void (*MCOUNT_SYM)(void) __weak = dummy; + +static bool ta_instrumented(void) +{ + /* + * Return true if the mcount function is called somewhere (and therefore + * profiling should be initialized). + * Since gprof is not supported with shared libraries, checking if + * mcount is called is the same as checking if it is present in the + * TA binary, because the function would be eliminated at link time if + * not used. + */ + return dummy != MCOUNT_SYM; +} + +#undef MCOUNT_SYM + +static void *gprof_alloc(size_t len) +{ + assert(!gprof_buf); + gprof_buf = tee_map_zi(len, TEE_MEMORY_ACCESS_ANY_OWNER); + gprof_buf_len = len; + return gprof_buf; +} + +static struct gmonparam _gmonparam = { GMON_PROF_OFF }; + +static uint32_t _gprof_file_id; /* File id returned by tee-supplicant */ + +static int _gprof_s_scale; +#define SCALE_1_TO_1 0x10000L + +/* Adjust PC so that gprof can locate it in the TA ELF file */ +static unsigned long __noprof adjust_pc(unsigned long pc) +{ + return pc - (unsigned long)__text_start + sizeof(struct ta_head); +} + +void __utee_gprof_init(void) +{ + unsigned long lowpc; + unsigned long highpc; + struct gmonparam *p = &_gmonparam; + size_t bufsize; + TEE_Result res; + char *cp; + + if (!ta_instrumented()) + return; + + lowpc = adjust_pc((unsigned long)__text_start); + highpc = adjust_pc((unsigned long)__text_end); + + /* + * Round lowpc and highpc to multiples of the density we're using + * so the rest of the scaling (here and in gprof) stays in ints. + */ + p->lowpc = ROUNDDOWN(lowpc, HISTFRACTION * sizeof(HISTCOUNTER)); + p->highpc = ROUNDUP(highpc, HISTFRACTION * sizeof(HISTCOUNTER)); + p->textsize = p->highpc - p->lowpc; + p->kcountsize = ROUNDUP(p->textsize / HISTFRACTION, sizeof(*p->froms)); + p->hashfraction = HASHFRACTION; + p->log_hashfraction = -1; + /* + * The following test must be kept in sync with the corresponding + * test in __mcount_internal + */ + if ((HASHFRACTION & (HASHFRACTION - 1)) == 0) { + /* + * If HASHFRACTION is a power of two, mcount can use shifting + * instead of integer division. Precompute shift amount. + */ + p->log_hashfraction = __builtin_ffs(p->hashfraction * + sizeof(*p->froms)) - 1; + } + p->fromssize = p->textsize / HASHFRACTION; + p->tolimit = p->textsize * ARCDENSITY / 100; + if (p->tolimit < MINARCS) + p->tolimit = MINARCS; + else if (p->tolimit > MAXARCS) + p->tolimit = MAXARCS; + p->tossize = p->tolimit * sizeof(struct tostruct); + + bufsize = p->kcountsize + p->fromssize + p->tossize; + + IMSG("gprof: initializing"); + DMSG("TA text size: %zu, gprof buffer size: %zu", + __text_end - __text_start, bufsize); + + cp = gprof_alloc(bufsize); + if (!cp) { + EMSG("gprof: could not allocate profiling buffer"); + p->tos = NULL; + p->state = GMON_PROF_ERROR; + return; + } + + p->tos = (struct tostruct *)cp; + cp += p->tossize; + p->kcount = (HISTCOUNTER *)cp; + cp += p->kcountsize; + p->froms = (ARCINDEX *)cp; + + p->tos[0].link = 0; + + if (p->kcountsize < p->textsize) + _gprof_s_scale = ((float)p->kcountsize / p->textsize) * + SCALE_1_TO_1; + else + _gprof_s_scale = SCALE_1_TO_1; + + res = __pta_gprof_pc_sampling_start(p->kcount, p->kcountsize, + p->lowpc + + ((unsigned long)__text_start - + sizeof(struct ta_head)), + _gprof_s_scale); + if (res != TEE_SUCCESS) + EMSG("gprof: could not start PC sampling (0x%08x)", res); + + p->state = GMON_PROF_ON; +} + +static void _gprof_write_buf(void *buf, size_t size) +{ + TEE_Result res; + + res = __pta_gprof_send(buf, size, &_gprof_file_id); + if (res != TEE_SUCCESS) + EMSG("gprof: could not send gprof data (0x%08x)", res); +} + +static void _gprof_write_header(void) +{ + struct gmon_hdr ghdr; + size_t size = sizeof(struct gmon_hdr); + + memcpy(&ghdr.cookie[0], GMON_MAGIC, sizeof(ghdr.cookie)); + ghdr.version = GMON_VERSION; + memset(ghdr.spare, '\0', sizeof(ghdr.spare)); + + _gprof_write_buf(&ghdr, size); +} + +static void _gprof_write_hist(void) +{ + struct out_record { + uint8_t tag; + struct gmon_hist_hdr hist_hdr; + } __packed out = { + .tag = GMON_TAG_TIME_HIST, + .hist_hdr = { + .low_pc = _gmonparam.lowpc, + .high_pc = _gmonparam.highpc, + .hist_size = _gmonparam.kcountsize/sizeof(HISTCOUNTER), + .prof_rate = _gmonparam.prof_rate, + .dimen = "seconds", + .dimen_abbrev = 's', + } + }; + + _gprof_write_buf(&out, sizeof(out)); + _gprof_write_buf(_gmonparam.kcount, _gmonparam.kcountsize); +} + +static void _gprof_write_call_graph(void) +{ +#define NARCS_PER_WRITE 16 + struct out_record { + uint8_t tag; + uint8_t data[sizeof(struct gmon_cg_arc_record)]; + } out[NARCS_PER_WRITE]; + struct gmon_cg_arc_record arc; + ARCINDEX from_index, to_index; + unsigned long from_len; + unsigned long frompc; + int nfilled = 0; + + from_len = _gmonparam.fromssize / sizeof(*_gmonparam.froms); + + for (from_index = 0; from_index < from_len; ++from_index) { + + if (_gmonparam.froms[from_index] == 0) + continue; + + frompc = _gmonparam.lowpc; + frompc += (from_index * _gmonparam.hashfraction + * sizeof(*_gmonparam.froms)); + for (to_index = _gmonparam.froms[from_index]; + to_index != 0; + to_index = _gmonparam.tos[to_index].link) { + + arc.from_pc = frompc; + arc.self_pc = _gmonparam.tos[to_index].selfpc; + arc.count = _gmonparam.tos[to_index].count; + + out[nfilled].tag = GMON_TAG_CG_ARC; + memcpy(out[nfilled].data, &arc, sizeof(arc)); + + if (++nfilled == NARCS_PER_WRITE) { + _gprof_write_buf(out, sizeof(out)); + nfilled = 0; + } + } + } + if (nfilled > 0) + _gprof_write_buf(out, nfilled * sizeof(out[0])); +} + +/* Stop profiling and send profile data in gmon.out format to Normal World */ +void __utee_gprof_fini(void) +{ + TEE_Result res; + + if (_gmonparam.state != GMON_PROF_ON) + return; + + /* Stop call graph tracing */ + _gmonparam.state = GMON_PROF_OFF_EXITING; + + /* Stop TA sampling */ + res = __pta_gprof_pc_sampling_stop(&_gmonparam.prof_rate); + + _gprof_write_header(); + if (res == TEE_SUCCESS) + _gprof_write_hist(); + _gprof_write_call_graph(); + + __pta_gprof_fini(); + + if (gprof_buf) { + res = tee_unmap(gprof_buf, gprof_buf_len); + assert(!res); + gprof_buf = NULL; + } +} + +/* + * Called from the assembly stub (_mcount or __gnu_mcount_nc). + * + * __mcount_internal updates data structures that represent traversals of the + * program's call graph edges. frompc and selfpc are the return + * address and function address that represents the given call graph edge. + */ +void __noprof __mcount_internal(unsigned long frompc, unsigned long selfpc) +{ + ARCINDEX *frompcindex; + struct tostruct *top, *prevtop; + struct gmonparam *p; + ARCINDEX toindex; + int i; + + p = &_gmonparam; + + /* + * Check that we are profiling and that we aren't recursively invoked. + */ + if (p->state != GMON_PROF_ON) + return; + p->state = GMON_PROF_BUSY; + + frompc = adjust_pc(frompc); + selfpc = adjust_pc(selfpc); + + /* Check that frompcindex is a reasonable pc value. */ + frompc -= p->lowpc; + if (frompc > p->textsize) + goto done; + + /* Note: keep in sync. with the initialization function above */ + if ((HASHFRACTION & (HASHFRACTION - 1)) == 0) { + /* Avoid integer divide if possible */ + i = frompc >> p->log_hashfraction; + } else { + i = frompc / (p->hashfraction * sizeof(*p->froms)); + } + frompcindex = &p->froms[i]; + toindex = *frompcindex; + if (toindex == 0) { + /* First time traversing this arc */ + toindex = ++p->tos[0].link; + if (toindex >= p->tolimit) { + /* Halt further profiling */ + goto overflow; + } + + *frompcindex = toindex; + top = &p->tos[toindex]; + top->selfpc = selfpc; + top->count = 1; + top->link = 0; + goto done; + } + top = &p->tos[toindex]; + if (top->selfpc == selfpc) { + /* Arc at front of chain; usual case */ + top->count++; + goto done; + } + /* + * Have to go looking down chain for it. + * top points to what we are looking at, + * prevtop points to previous top. + * we know it is not at the head of the chain. + */ + for (;;) { + if (top->link == 0) { + /* + * top is end of the chain and none of the chain + * had top->selfpc == selfpc. + * so we allocate a new tostruct + * and link it to the head of the chain. + */ + toindex = ++p->tos[0].link; + if (toindex >= p->tolimit) + goto overflow; + + top = &p->tos[toindex]; + top->selfpc = selfpc; + top->count = 1; + top->link = *frompcindex; + *frompcindex = toindex; + goto done; + } + /* + * Otherwise, check the next arc on the chain. + */ + prevtop = top; + top = &p->tos[top->link]; + if (top->selfpc == selfpc) { + /* + * There it is. Increment its count, move it to the + * head of the chain. + */ + top->count++; + toindex = prevtop->link; + prevtop->link = top->link; + top->link = *frompcindex; + *frompcindex = toindex; + goto done; + } + } +done: + p->state = GMON_PROF_ON; + return; +overflow: + p->state = GMON_PROF_ERROR; +} diff --git a/optee_os/lib/libutee/arch/arm/gprof/gprof_pta.c b/optee_os/lib/libutee/arch/arm/gprof/gprof_pta.c new file mode 100644 index 0000000..f77d46b --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/gprof/gprof_pta.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include +#include +#include +#include "gprof_pta.h" + +static TEE_TASessionHandle sess = TEE_HANDLE_NULL; + +static TEE_Result invoke_gprof_pta(uint32_t cmd_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + static const TEE_UUID core_uuid = PTA_GPROF_UUID; + TEE_Result res; + + if (!sess) { + res = TEE_OpenTASession(&core_uuid, TEE_TIMEOUT_INFINITE, + 0, NULL, &sess, NULL); + if (res != TEE_SUCCESS) + return res; + } + res = TEE_InvokeTACommand(sess, TEE_TIMEOUT_INFINITE, cmd_id, + param_types, params, NULL); + return res; +} + +TEE_Result __pta_gprof_send(void *buf, size_t len, uint32_t *id) +{ + TEE_Param params[TEE_NUM_PARAMS]; + uint32_t param_types; + TEE_Result res; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INOUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + memset(params, 0, sizeof(params)); + params[0].value.a = *id; + params[1].memref.buffer = buf; + params[1].memref.size = len; + res = invoke_gprof_pta(PTA_GPROF_SEND, param_types, params); + if (res == TEE_SUCCESS) + *id = params[0].value.a; + return res; +} + +TEE_Result __pta_gprof_pc_sampling_start(void *buf, size_t len, size_t offset, + size_t scale) +{ + TEE_Param params[TEE_NUM_PARAMS]; + uint32_t param_types; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + memset(params, 0, sizeof(params)); + params[0].memref.buffer = buf; + params[0].memref.size = len; + params[1].value.a = offset; + params[1].value.b = scale; + return invoke_gprof_pta(PTA_GPROF_START_PC_SAMPLING, param_types, + params); +} + +TEE_Result __pta_gprof_pc_sampling_stop(uint32_t *rate) +{ + TEE_Param params[TEE_NUM_PARAMS]; + uint32_t param_types; + TEE_Result res; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + memset(params, 0, sizeof(params)); + res = invoke_gprof_pta(PTA_GPROF_STOP_PC_SAMPLING, param_types, + params); + if (res != TEE_SUCCESS) + return res; + if (rate) + *rate = params[0].value.a; + return res; +} + +void __pta_gprof_fini(void) +{ + if (sess) + TEE_CloseTASession(sess); +} diff --git a/optee_os/lib/libutee/arch/arm/gprof/gprof_pta.h b/optee_os/lib/libutee/arch/arm/gprof/gprof_pta.h new file mode 100644 index 0000000..6b7bcc3 --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/gprof/gprof_pta.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef __GPROF_PTA_H +#define __GPROF_PTA_H + +#include +#include +#include +#include + +TEE_Result __pta_gprof_send(void *buf, size_t len, uint32_t *id); +TEE_Result __pta_gprof_pc_sampling_start(void *buf, size_t len, size_t offset, + size_t scale); +TEE_Result __pta_gprof_pc_sampling_stop(uint32_t *rate); +void __pta_gprof_fini(void); +#endif /* __GPROF_PTA_H */ diff --git a/optee_os/lib/libutee/arch/arm/gprof/sub.mk b/optee_os/lib/libutee/arch/arm/gprof/sub.mk new file mode 100644 index 0000000..eacd788 --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/gprof/sub.mk @@ -0,0 +1,5 @@ +cppflags-y += -I$(sub-dir)/../../.. + +srcs-$(CFG_TA_GPROF_SUPPORT) += gprof.c +srcs-$(CFG_TA_GPROF_SUPPORT) += gprof_pta.c +cflags-remove-gprof.c-y += -Wcast-align diff --git a/optee_os/lib/libutee/arch/arm/sub.mk b/optee_os/lib/libutee/arch/arm/sub.mk new file mode 100644 index 0000000..7afd068 --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/sub.mk @@ -0,0 +1,9 @@ +cppflags-y += -I$(sub-dir)/../.. + +srcs-$(CFG_ARM32_$(sm)) += utee_syscalls_a32.S +srcs-$(CFG_ARM64_$(sm)) += utee_syscalls_a64.S + +ifneq ($(sm),ldelf) +subdirs-y += gprof +endif #$(sm-$(sm)-is-ld) + diff --git a/optee_os/lib/libutee/arch/arm/utee_syscalls_a32.S b/optee_os/lib/libutee/arch/arm/utee_syscalls_a32.S new file mode 100644 index 0000000..2dea83a --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/utee_syscalls_a32.S @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include + + .section .note.GNU-stack,"",%progbits + + .section .text + .balign 4 + .code 32 + + .macro UTEE_SYSCALL name, scn, num_args + FUNC \name , : + push {r5-r7,lr} +UNWIND( .save {r5-r7,lr}) +#if defined(CFG_SYSCALL_WRAPPERS_MCOUNT) && !defined(__LDELF__) + .if \scn != TEE_SCN_RETURN + mov ip, sp + push {r0-r4, fp, ip} + add fp, ip, #16 + push {lr} + bl __gnu_mcount_nc + pop {r0-r4, fp, ip} + mov sp, ip + .endif +#endif + mov r7, #(\scn) + .if \num_args > TEE_SVC_MAX_ARGS + .error "Too many arguments for syscall" + .endif + .if \num_args <= 4 + @ No arguments passed on stack + mov r6, #0 + .else + @ Tell number of arguments passed on the stack + mov r6, #(\num_args - 4) + @ Point just before the push (4 registers) above on the first argument + add r5, sp, #(4 * 4) + .endif + svc #0 + pop {r5-r7,pc} + END_FUNC \name + .endm + + FUNC _utee_panic, : + push {r0-r11, lr} +UNWIND( .save {r0-r11, lr}) + mov lr, pc + push {lr} +UNWIND( .save {lr}) + mov r1, sp + bl __utee_panic + /* Not reached */ + END_FUNC _utee_panic + +#include diff --git a/optee_os/lib/libutee/arch/arm/utee_syscalls_a64.S b/optee_os/lib/libutee/arch/arm/utee_syscalls_a64.S new file mode 100644 index 0000000..2e6e6c0 --- /dev/null +++ b/optee_os/lib/libutee/arch/arm/utee_syscalls_a64.S @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include + + .section .text + + .macro UTEE_SYSCALL name, scn, num_args + FUNC \name , : + + .if \num_args > TEE_SVC_MAX_ARGS || \num_args > 8 + .error "Too many arguments for syscall" + .endif +#if defined(CFG_SYSCALL_WRAPPERS_MCOUNT) && !defined(__LDELF__) + .if \scn != TEE_SCN_RETURN + stp x29, x30, [sp, #-80]! + mov x29, sp + stp x0, x1, [sp, #16] + stp x2, x3, [sp, #32] + stp x4, x5, [sp, #48] + stp x6, x7, [sp, #64] + mov x0, x30 + bl _mcount + ldp x0, x1, [sp, #16] + ldp x2, x3, [sp, #32] + ldp x4, x5, [sp, #48] + ldp x6, x7, [sp, #64] + ldp x29, x30, [sp], #80 + .endif +#endif + mov x8, #(\scn) + svc #0 + ret + END_FUNC \name + .endm + + FUNC _utee_panic, : + stp x29, x30, [sp, #-16]! + mov x1, sp + bl __utee_panic + /* Not reached */ + END_FUNC _utee_panic + +#include + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/lib/libutee/arch/riscv/sub.mk b/optee_os/lib/libutee/arch/riscv/sub.mk new file mode 100644 index 0000000..16f776a --- /dev/null +++ b/optee_os/lib/libutee/arch/riscv/sub.mk @@ -0,0 +1,3 @@ +cppflags-y += -I$(sub-dir)/../.. + +srcs-y += utee_syscalls_rv.S diff --git a/optee_os/lib/libutee/arch/riscv/utee_syscalls_rv.S b/optee_os/lib/libutee/arch/riscv/utee_syscalls_rv.S new file mode 100644 index 0000000..d45ec52 --- /dev/null +++ b/optee_os/lib/libutee/arch/riscv/utee_syscalls_rv.S @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright 2022 Beijing ESWIN Computing Technology Co., Ltd. + */ + +#include +#include + + .section .text + + .macro UTEE_SYSCALL name, scn, num_args + FUNC \name , : + + .if \num_args > TEE_SVC_MAX_ARGS || \num_args > 8 + .error "Too many arguments for syscall" + .endif + li t0, \scn + li t1, \num_args + ecall + ret + END_FUNC \name + .endm + + FUNC _utee_panic, : + /* The stack pointer is always kept 16-byte aligned */ + add sp, sp, -16 + /* Save return address and frame pointer to stack */ +#if defined(RV32) + sw s0, 8(sp) + sw ra, 12(sp) +#elif defined(RV64) + sd s0, 0(sp) + sd ra, 8(sp) +#endif + /* Assign a1 as stack pointer for scall_save_panic_stack() */ + mv a1, sp + /* Use tail call here because we will not return from it */ + tail __utee_panic + /* Not reached */ + END_FUNC _utee_panic + +#include diff --git a/optee_os/lib/libutee/assert.c b/optee_os/lib/libutee/assert.c new file mode 100644 index 0000000..022b202 --- /dev/null +++ b/optee_os/lib/libutee/assert.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include +#include +#include +#include +#include +#include + +void _assert_log(const char *expr __maybe_unused, + const char *file __maybe_unused, + const int line __maybe_unused, + const char *func __maybe_unused) +{ + EMSG_RAW("assertion '%s' failed at %s:%d in %s()", + expr, file, line, func); +} + +void __noreturn _assert_break(void) +{ + _utee_panic(TEE_ERROR_GENERIC); + /* Not reached */ + while (1) + ; +} diff --git a/optee_os/lib/libutee/base64.c b/optee_os/lib/libutee/base64.c new file mode 100644 index 0000000..a1eacec --- /dev/null +++ b/optee_os/lib/libutee/base64.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include "base64.h" + +static const char base64_table[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +size_t _base64_enc_len(size_t size) +{ + return 4 * ((size + 2) / 3) + 1; +} + +bool _base64_enc(const void *data, size_t dlen, char *buf, size_t *blen) +{ + size_t n = 0; + size_t boffs = 0; + const unsigned char *d = data; + + n = _base64_enc_len(dlen); + if (*blen < n) { + *blen = n; + return false; + } + + for (n = 0; n < dlen; n += 3) { + uint32_t igrp; + + igrp = d[n]; + igrp <<= 8; + + if ((n + 1) < dlen) + igrp |= d[n + 1]; + igrp <<= 8; + + if ((n + 2) < dlen) + igrp |= d[n + 2]; + + buf[boffs] = base64_table[(igrp >> 18) & 0x3f]; + buf[boffs + 1] = base64_table[(igrp >> 12) & 0x3f]; + if ((n + 1) < dlen) + buf[boffs + 2] = base64_table[(igrp >> 6) & 0x3f]; + else + buf[boffs + 2] = '='; + if ((n + 2) < dlen) + buf[boffs + 3] = base64_table[igrp & 0x3f]; + else + buf[boffs + 3] = '='; + + boffs += 4; + } + buf[boffs++] = '\0'; + + *blen = boffs; + return true; +} + +static bool get_idx(char ch, uint8_t *idx) +{ + size_t n = 0; + + for (n = 0; base64_table[n] != '\0'; n++) { + if (ch == base64_table[n]) { + *idx = n; + return true; + } + } + return false; +} + +bool _base64_dec(const char *data, size_t size, void *buf, size_t *blen) +{ + bool ret = false; + size_t n = 0; + uint8_t idx = 0; + uint8_t *b = buf; + size_t m = 0; + size_t s = 0; + uint8_t byte = 0; + + for (n = 0; n < size && data[n] != '\0'; n++) { + if (data[n] == '=') + break; /* Reached pad characters, we're done */ + + if (!get_idx(data[n], &idx)) + continue; + + switch (s) { + case 0: + byte = idx << 2; + s++; + break; + case 1: + if (b && m < *blen) + b[m] = byte | (idx >> 4); + m++; + byte = (idx & 0xf) << 4; + s++; + break; + case 2: + if (b && m < *blen) + b[m] = byte | (idx >> 2); + m++; + byte = (idx & 0x3) << 6; + s++; + break; + case 3: + if (b && m < *blen) + b[m] = byte | idx; + m++; + s = 0; + break; + default: + return false; /* "Can't happen" */ + } + } + + /* + * We don't detect if input was bad, but that's OK with the spec. + * We expect that each fully extracted byte is stored in output buffer. + */ + ret = (!m && !*blen) || (b && (m <= *blen)); + *blen = m; + + return ret; +} diff --git a/optee_os/lib/libutee/base64.h b/optee_os/lib/libutee/base64.h new file mode 100644 index 0000000..ed4ba36 --- /dev/null +++ b/optee_os/lib/libutee/base64.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef BASE64_H +#define BASE64_H + +#include +#include +#include + +bool _base64_enc(const void *data, size_t size, char *buf, size_t *blen); +bool _base64_dec(const char *data, size_t size, void *buf, size_t *blen); +size_t _base64_enc_len(size_t size); + +#endif /* BASE64_H */ diff --git a/optee_os/lib/libutee/include/__tee_ipsocket.h b/optee_os/lib/libutee/include/__tee_ipsocket.h new file mode 100644 index 0000000..c25437c --- /dev/null +++ b/optee_os/lib/libutee/include/__tee_ipsocket.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef ____TEE_IPSOCKET_H +#define ____TEE_IPSOCKET_H + +typedef enum TEE_ipSocket_ipVersion_e { + TEE_IP_VERSION_DC = 0, /* don’t care */ + TEE_IP_VERSION_4 = 1, + TEE_IP_VERSION_6 = 2 +} TEE_ipSocket_ipVersion; + +#endif /*____TEE_IPSOCKET_H*/ diff --git a/optee_os/lib/libutee/include/__tee_isocket_defines.h b/optee_os/lib/libutee/include/__tee_isocket_defines.h new file mode 100644 index 0000000..bfb42c7 --- /dev/null +++ b/optee_os/lib/libutee/include/__tee_isocket_defines.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ +#ifndef ____TEE_ISOCKET_DEFINES_H +#define ____TEE_ISOCKET_DEFINES_H + +#define TEE_ISOCKET_VERSION 0x01000000 + +#define TEE_ISOCKET_ERROR_PROTOCOL 0xF1007001 +#define TEE_ISOCKET_ERROR_REMOTE_CLOSED 0xF1007002 +#define TEE_ISOCKET_ERROR_TIMEOUT 0xF1007003 +#define TEE_ISOCKET_ERROR_OUT_OF_RESOURCES 0xF1007004 +#define TEE_ISOCKET_ERROR_LARGE_BUFFER 0xF1007005 +#define TEE_ISOCKET_WARNING_PROTOCOL 0xF1007006 +#define TEE_ISOCKET_ERROR_HOSTNAME 0xF1007007 + +#endif /*____TEE_ISOCKET_DEFINES_H*/ diff --git a/optee_os/lib/libutee/include/__tee_tcpsocket_defines.h b/optee_os/lib/libutee/include/__tee_tcpsocket_defines.h new file mode 100644 index 0000000..ddaf58d --- /dev/null +++ b/optee_os/lib/libutee/include/__tee_tcpsocket_defines.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef ____TEE_TCPSOCKET_DEFINES_H +#define ____TEE_TCPSOCKET_DEFINES_H + +/* Protocol identifier */ +#define TEE_ISOCKET_PROTOCOLID_TCP 0x65 + +/* Instance specific errors */ +#define TEE_ISOCKET_TCP_WARNING_UNKNOWN_OUT_OF_BAND 0xF1010002 + +#endif /*____TEE_TCPSOCKET_DEFINES_H*/ diff --git a/optee_os/lib/libutee/include/__tee_tcpsocket_defines_extensions.h b/optee_os/lib/libutee/include/__tee_tcpsocket_defines_extensions.h new file mode 100644 index 0000000..ca9ece0 --- /dev/null +++ b/optee_os/lib/libutee/include/__tee_tcpsocket_defines_extensions.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef ____TEE_TCPSOCKET_DEFINES_EXTENSIONS_H +#define ____TEE_TCPSOCKET_DEFINES_EXTENSIONS_H + +/* Instance and implementation specific ioctl functions */ +#define TEE_TCP_SET_RECVBUF 0x65f00000 +#define TEE_TCP_SET_SENDBUF 0x65f00001 + +#endif /*____TEE_TCPSOCKET_DEFINES_EXTENSIONS_H*/ diff --git a/optee_os/lib/libutee/include/__tee_udpsocket_defines.h b/optee_os/lib/libutee/include/__tee_udpsocket_defines.h new file mode 100644 index 0000000..ef0b4f2 --- /dev/null +++ b/optee_os/lib/libutee/include/__tee_udpsocket_defines.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef ____TEE_UDPSOCKET_DEFINES_H +#define ____TEE_UDPSOCKET_DEFINES_H + +/* Protocol identifier */ +#define TEE_ISOCKET_PROTOCOLID_UDP 0x66 + +/* Instance specific errors */ +#define TEE_ISOCKET_UDP_WARNING_UNKNOWN_OUT_OF_BAND 0xF1020002 + +/* Instance specific ioctl functions */ +#define TEE_UDP_CHANGEADDR 0x66000001 +#define TEE_UDP_CHANGEPORT 0x66000002 + +#endif /*____TEE_UDPSOCKET_DEFINES_H*/ diff --git a/optee_os/lib/libutee/include/arm64_user_sysreg.h b/optee_os/lib/libutee/include/arm64_user_sysreg.h new file mode 100644 index 0000000..3205b73 --- /dev/null +++ b/optee_os/lib/libutee/include/arm64_user_sysreg.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ +#ifndef ARM64_USER_SYSREG_H +#define ARM64_USER_SYSREG_H + +#include +#include + +/* + * Templates for register read/write functions based on mrs/msr + */ + +#define DEFINE_REG_READ_FUNC_(reg, type, asmreg) \ +static inline __noprof type read_##reg(void) \ +{ \ + uint64_t val64 = 0; \ + \ + asm volatile("mrs %0, " #asmreg : "=r" (val64)); \ + return val64; \ +} + +#define DEFINE_REG_WRITE_FUNC_(reg, type, asmreg) \ +static inline __noprof void write_##reg(type val) \ +{ \ + uint64_t val64 = val; \ + \ + asm volatile("msr " #asmreg ", %0" : : "r" (val64)); \ +} + +/* ARM Generic timer functions */ +DEFINE_REG_READ_FUNC_(cntfrq, uint32_t, cntfrq_el0) +DEFINE_REG_READ_FUNC_(cntpct, uint64_t, cntpct_el0) +DEFINE_REG_READ_FUNC_(cntvct, uint64_t, cntvct_el0) +DEFINE_REG_READ_FUNC_(tpidr_el0, uint64_t, tpidr_el0) +DEFINE_REG_WRITE_FUNC_(tpidr_el0, uint64_t, tpidr_el0) + +DEFINE_REG_READ_FUNC_(dczid_el0, uint64_t, dczid_el0) + +#endif /*ARM64_USER_SYSREG_H*/ diff --git a/optee_os/lib/libutee/include/arm_user_sysreg.h b/optee_os/lib/libutee/include/arm_user_sysreg.h new file mode 100644 index 0000000..974388b --- /dev/null +++ b/optee_os/lib/libutee/include/arm_user_sysreg.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ +#ifndef ARM_USER_SYSREG_H +#define ARM_USER_SYSREG_H + +#include + +#ifdef ARM32 +#include +#endif + +#ifdef ARM64 +#include +#endif + +#ifndef __ASSEMBLER__ +static inline __noprof void isb(void) +{ + asm volatile ("isb"); +} + +static inline __noprof uint64_t barrier_read_counter_timer(void) +{ + isb(); +#ifdef CFG_CORE_SEL2_SPMC + return read_cntvct(); +#else + return read_cntpct(); +#endif +} +#endif + +#endif /*ARM_USER_SYSREG_H*/ diff --git a/optee_os/lib/libutee/include/elf.h b/optee_os/lib/libutee/include/elf.h new file mode 100644 index 0000000..b616b79 --- /dev/null +++ b/optee_os/lib/libutee/include/elf.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ +#ifndef _ELF_H_ +#define _ELF_H_ + +#if defined(ARM32) || defined(RV32) + +#include + +typedef Elf32_Addr Elf_Addr; +typedef Elf32_Half Elf_Half; +typedef Elf32_Off Elf_Off; +typedef Elf32_Sword Elf_Sword; +typedef Elf32_Word Elf_Word; +typedef Elf32_Lword Elf_Lword; +typedef Elf32_Hashelt Elf_Hashelt; +typedef Elf32_Size Elf_Size; +typedef Elf32_Ssize Elf_Ssize; +typedef Elf32_Ehdr Elf_Ehdr; +typedef Elf32_Shdr Elf_Shdr; +typedef Elf32_Phdr Elf_Phdr; +typedef Elf32_Dyn Elf_Dyn; +typedef Elf32_Rel Elf_Rel; +typedef Elf32_Rela Elf_Rela; +typedef Elf32_Move Elf_Move; +typedef Elf32_Cap Elf_Cap; +typedef Elf32_Sym Elf_Sym; +typedef Elf32_Verdef Elf_Verdef; +typedef Elf32_Verdaux Elf_Verdaux; +typedef Elf32_Verneed Elf_Verneed; +typedef Elf32_Vernaux Elf_Vernaux; +typedef Elf32_Syminfo Elf_Syminfo; + +#elif defined(ARM64) || defined(RV64) + +#include + +typedef Elf64_Addr Elf_Addr; +typedef Elf64_Half Elf_Half; +typedef Elf64_Off Elf_Off; +typedef Elf64_Sword Elf_Sword; +typedef Elf64_Sxword Elf_Sxword; +typedef Elf64_Word Elf_Word; +typedef Elf64_Lword Elf_Lword; +typedef Elf64_Xword Elf_Xword; +typedef Elf64_Hashelt Elf_Hashelt; +typedef Elf64_Size Elf_Size; +typedef Elf64_Ssize Elf_Ssize; +typedef Elf64_Ehdr Elf_Ehdr; +typedef Elf64_Shdr Elf_Shdr; +typedef Elf64_Phdr Elf_Phdr; +typedef Elf64_Dyn Elf_Dyn; +typedef Elf64_Rel Elf_Rel; +typedef Elf64_Rela Elf_Rela; +typedef Elf64_Move Elf_Move; +typedef Elf64_Cap Elf_Cap; +typedef Elf64_Sym Elf_Sym; +typedef Elf64_Verdef Elf_Verdef; +typedef Elf64_Verdaux Elf_Verdaux; +typedef Elf64_Verneed Elf_Verneed; +typedef Elf64_Vernaux Elf_Vernaux; +typedef Elf64_Syminfo Elf_Syminfo; + +#else +#error Unknown architecture +#endif + +#endif /* _ELF_H_ */ diff --git a/optee_os/lib/libutee/include/elf32.h b/optee_os/lib/libutee/include/elf32.h new file mode 100644 index 0000000..de74f6f --- /dev/null +++ b/optee_os/lib/libutee/include/elf32.h @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/*- + * Copyright (c) 1996-1998 John D. Polstra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_ELF32_H_ +#define _SYS_ELF32_H_ 1 + +#include +#include + +/* + * ELF definitions common to all 32-bit architectures. + */ + +typedef uint32_t Elf32_Addr; +typedef uint16_t Elf32_Half; +typedef uint32_t Elf32_Off; +typedef int32_t Elf32_Sword; +typedef uint32_t Elf32_Word; +typedef uint64_t Elf32_Lword; + +typedef Elf32_Word Elf32_Hashelt; + +/* Non-standard class-dependent datatype used for abstraction. */ +typedef Elf32_Word Elf32_Size; +typedef Elf32_Sword Elf32_Ssize; + +/* + * ELF header. + */ + +typedef struct { + unsigned char e_ident[EI_NIDENT]; /* File identification. */ + Elf32_Half e_type; /* File type. */ + Elf32_Half e_machine; /* Machine architecture. */ + Elf32_Word e_version; /* ELF format version. */ + Elf32_Addr e_entry; /* Entry point. */ + Elf32_Off e_phoff; /* Program header file offset. */ + Elf32_Off e_shoff; /* Section header file offset. */ + Elf32_Word e_flags; /* Architecture-specific flags. */ + Elf32_Half e_ehsize; /* Size of ELF header in bytes. */ + Elf32_Half e_phentsize; /* Size of program header entry. */ + Elf32_Half e_phnum; /* Number of program header entries. */ + Elf32_Half e_shentsize; /* Size of section header entry. */ + Elf32_Half e_shnum; /* Number of section header entries. */ + Elf32_Half e_shstrndx; /* Section name strings section. */ +} Elf32_Ehdr; + +/* + * Section header. + */ + +typedef struct { + Elf32_Word sh_name; /* Section name (index into the + section header string table). */ + Elf32_Word sh_type; /* Section type. */ + Elf32_Word sh_flags; /* Section flags. */ + Elf32_Addr sh_addr; /* Address in memory image. */ + Elf32_Off sh_offset; /* Offset in file. */ + Elf32_Word sh_size; /* Size in bytes. */ + Elf32_Word sh_link; /* Index of a related section. */ + Elf32_Word sh_info; /* Depends on section type. */ + Elf32_Word sh_addralign; /* Alignment in bytes. */ + Elf32_Word sh_entsize; /* Size of each entry in section. */ +} Elf32_Shdr; + +/* + * Program header. + */ + +typedef struct { + Elf32_Word p_type; /* Entry type. */ + Elf32_Off p_offset; /* File offset of contents. */ + Elf32_Addr p_vaddr; /* Virtual address in memory image. */ + Elf32_Addr p_paddr; /* Physical address (not used). */ + Elf32_Word p_filesz; /* Size of contents in file. */ + Elf32_Word p_memsz; /* Size of contents in memory. */ + Elf32_Word p_flags; /* Access permission flags. */ + Elf32_Word p_align; /* Alignment in memory and file. */ +} Elf32_Phdr; + +/* + * Dynamic structure. The ".dynamic" section contains an array of them. + */ + +typedef struct { + Elf32_Sword d_tag; /* Entry type. */ + union { + Elf32_Word d_val; /* Integer value. */ + Elf32_Addr d_ptr; /* Address value. */ + } d_un; +} Elf32_Dyn; + +/* + * Relocation entries. + */ + +/* Relocations that don't need an addend field. */ +typedef struct { + Elf32_Addr r_offset; /* Location to be relocated. */ + Elf32_Word r_info; /* Relocation type and symbol index. */ +} Elf32_Rel; + +/* Relocations that need an addend field. */ +typedef struct { + Elf32_Addr r_offset; /* Location to be relocated. */ + Elf32_Word r_info; /* Relocation type and symbol index. */ + Elf32_Sword r_addend; /* Addend. */ +} Elf32_Rela; + +/* Macros for accessing the fields of r_info. */ +#define ELF32_R_SYM(info) ((info) >> 8) +#define ELF32_R_TYPE(info) ((unsigned char)(info)) + +/* Macro for constructing r_info from field values. */ +#define ELF32_R_INFO(sym, type) (((sym) << 8) + (unsigned char)(type)) + +/* + * Note entry header + */ +typedef Elf_Note Elf32_Nhdr; + +/* + * Move entry + */ +typedef struct { + Elf32_Lword m_value; /* symbol value */ + Elf32_Word m_info; /* size + index */ + Elf32_Word m_poffset; /* symbol offset */ + Elf32_Half m_repeat; /* repeat count */ + Elf32_Half m_stride; /* stride info */ +} Elf32_Move; + +/* + * The macros compose and decompose values for Move.r_info + * + * sym = ELF32_M_SYM(M.m_info) + * size = ELF32_M_SIZE(M.m_info) + * M.m_info = ELF32_M_INFO(sym, size) + */ +#define ELF32_M_SYM(info) ((info)>>8) +#define ELF32_M_SIZE(info) ((unsigned char)(info)) +#define ELF32_M_INFO(sym, size) (((sym)<<8)+(unsigned char)(size)) + +/* + * Hardware/Software capabilities entry + */ +typedef struct { + Elf32_Word c_tag; /* how to interpret value */ + union { + Elf32_Word c_val; + Elf32_Addr c_ptr; + } c_un; +} Elf32_Cap; + +/* + * Symbol table entries. + */ + +typedef struct { + Elf32_Word st_name; /* String table index of name. */ + Elf32_Addr st_value; /* Symbol value. */ + Elf32_Word st_size; /* Size of associated object. */ + unsigned char st_info; /* Type and binding information. */ + unsigned char st_other; /* Reserved (not used). */ + Elf32_Half st_shndx; /* Section index of symbol. */ +} Elf32_Sym; + +/* Macros for accessing the fields of st_info. */ +#define ELF32_ST_BIND(info) ((info) >> 4) +#define ELF32_ST_TYPE(info) ((info) & 0xf) + +/* Macro for constructing st_info from field values. */ +#define ELF32_ST_INFO(bind, type) (((bind) << 4) + ((type) & 0xf)) + +/* Macro for accessing the fields of st_other. */ +#define ELF32_ST_VISIBILITY(oth) ((oth) & 0x3) + +/* Structures used by Sun & GNU symbol versioning. */ +typedef struct +{ + Elf32_Half vd_version; + Elf32_Half vd_flags; + Elf32_Half vd_ndx; + Elf32_Half vd_cnt; + Elf32_Word vd_hash; + Elf32_Word vd_aux; + Elf32_Word vd_next; +} Elf32_Verdef; + +typedef struct +{ + Elf32_Word vda_name; + Elf32_Word vda_next; +} Elf32_Verdaux; + +typedef struct +{ + Elf32_Half vn_version; + Elf32_Half vn_cnt; + Elf32_Word vn_file; + Elf32_Word vn_aux; + Elf32_Word vn_next; +} Elf32_Verneed; + +typedef struct +{ + Elf32_Word vna_hash; + Elf32_Half vna_flags; + Elf32_Half vna_other; + Elf32_Word vna_name; + Elf32_Word vna_next; +} Elf32_Vernaux; + +typedef Elf32_Half Elf32_Versym; + +typedef struct { + Elf32_Half si_boundto; /* direct bindings - symbol bound to */ + Elf32_Half si_flags; /* per symbol flags */ +} Elf32_Syminfo; + +#endif /* !_SYS_ELF32_H_ */ diff --git a/optee_os/lib/libutee/include/elf64.h b/optee_os/lib/libutee/include/elf64.h new file mode 100644 index 0000000..060819b --- /dev/null +++ b/optee_os/lib/libutee/include/elf64.h @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/*- + * Copyright (c) 1996-1998 John D. Polstra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_ELF64_H_ +#define _SYS_ELF64_H_ 1 + +#include +#include + +/* + * ELF definitions common to all 64-bit architectures. + */ + +typedef uint64_t Elf64_Addr; +typedef uint16_t Elf64_Half; +typedef uint64_t Elf64_Off; +typedef int32_t Elf64_Sword; +typedef int64_t Elf64_Sxword; +typedef uint32_t Elf64_Word; +typedef uint64_t Elf64_Lword; +typedef uint64_t Elf64_Xword; + +/* + * Types of dynamic symbol hash table bucket and chain elements. + * + * This is inconsistent among 64 bit architectures, so a machine dependent + * typedef is required. + */ + +typedef Elf64_Word Elf64_Hashelt; + +/* Non-standard class-dependent datatype used for abstraction. */ +typedef Elf64_Xword Elf64_Size; +typedef Elf64_Sxword Elf64_Ssize; + +/* + * ELF header. + */ + +typedef struct { + unsigned char e_ident[EI_NIDENT]; /* File identification. */ + Elf64_Half e_type; /* File type. */ + Elf64_Half e_machine; /* Machine architecture. */ + Elf64_Word e_version; /* ELF format version. */ + Elf64_Addr e_entry; /* Entry point. */ + Elf64_Off e_phoff; /* Program header file offset. */ + Elf64_Off e_shoff; /* Section header file offset. */ + Elf64_Word e_flags; /* Architecture-specific flags. */ + Elf64_Half e_ehsize; /* Size of ELF header in bytes. */ + Elf64_Half e_phentsize; /* Size of program header entry. */ + Elf64_Half e_phnum; /* Number of program header entries. */ + Elf64_Half e_shentsize; /* Size of section header entry. */ + Elf64_Half e_shnum; /* Number of section header entries. */ + Elf64_Half e_shstrndx; /* Section name strings section. */ +} Elf64_Ehdr; + +/* + * Section header. + */ + +typedef struct { + Elf64_Word sh_name; /* Section name (index into the + section header string table). */ + Elf64_Word sh_type; /* Section type. */ + Elf64_Xword sh_flags; /* Section flags. */ + Elf64_Addr sh_addr; /* Address in memory image. */ + Elf64_Off sh_offset; /* Offset in file. */ + Elf64_Xword sh_size; /* Size in bytes. */ + Elf64_Word sh_link; /* Index of a related section. */ + Elf64_Word sh_info; /* Depends on section type. */ + Elf64_Xword sh_addralign; /* Alignment in bytes. */ + Elf64_Xword sh_entsize; /* Size of each entry in section. */ +} Elf64_Shdr; + +/* + * Program header. + */ + +typedef struct { + Elf64_Word p_type; /* Entry type. */ + Elf64_Word p_flags; /* Access permission flags. */ + Elf64_Off p_offset; /* File offset of contents. */ + Elf64_Addr p_vaddr; /* Virtual address in memory image. */ + Elf64_Addr p_paddr; /* Physical address (not used). */ + Elf64_Xword p_filesz; /* Size of contents in file. */ + Elf64_Xword p_memsz; /* Size of contents in memory. */ + Elf64_Xword p_align; /* Alignment in memory and file. */ +} Elf64_Phdr; + +/* + * Dynamic structure. The ".dynamic" section contains an array of them. + */ + +typedef struct { + Elf64_Sxword d_tag; /* Entry type. */ + union { + Elf64_Xword d_val; /* Integer value. */ + Elf64_Addr d_ptr; /* Address value. */ + } d_un; +} Elf64_Dyn; + +/* + * Relocation entries. + */ + +/* Relocations that don't need an addend field. */ +typedef struct { + Elf64_Addr r_offset; /* Location to be relocated. */ + Elf64_Xword r_info; /* Relocation type and symbol index. */ +} Elf64_Rel; + +/* Relocations that need an addend field. */ +typedef struct { + Elf64_Addr r_offset; /* Location to be relocated. */ + Elf64_Xword r_info; /* Relocation type and symbol index. */ + Elf64_Sxword r_addend; /* Addend. */ +} Elf64_Rela; + +/* Macros for accessing the fields of r_info. */ +#define ELF64_R_SYM(info) ((info) >> 32) +#define ELF64_R_TYPE(info) ((info) & 0xffffffffL) + +/* Macro for constructing r_info from field values. */ +#define ELF64_R_INFO(sym, type) (((sym) << 32) + ((type) & 0xffffffffL)) + +#define ELF64_R_TYPE_DATA(info) (((Elf64_Xword)(info)<<32)>>40) +#define ELF64_R_TYPE_ID(info) (((Elf64_Xword)(info)<<56)>>56) +#define ELF64_R_TYPE_INFO(data, type) \ + (((Elf64_Xword)(data)<<8)+(Elf64_Xword)(type)) + +/* + * Note entry header + */ +typedef Elf_Note Elf64_Nhdr; + +/* + * Move entry + */ +typedef struct { + Elf64_Lword m_value; /* symbol value */ + Elf64_Xword m_info; /* size + index */ + Elf64_Xword m_poffset; /* symbol offset */ + Elf64_Half m_repeat; /* repeat count */ + Elf64_Half m_stride; /* stride info */ +} Elf64_Move; + +#define ELF64_M_SYM(info) ((info)>>8) +#define ELF64_M_SIZE(info) ((unsigned char)(info)) +#define ELF64_M_INFO(sym, size) (((sym)<<8)+(unsigned char)(size)) + +/* + * Hardware/Software capabilities entry + */ +typedef struct { + Elf64_Xword c_tag; /* how to interpret value */ + union { + Elf64_Xword c_val; + Elf64_Addr c_ptr; + } c_un; +} Elf64_Cap; + +/* + * Symbol table entries. + */ + +typedef struct { + Elf64_Word st_name; /* String table index of name. */ + unsigned char st_info; /* Type and binding information. */ + unsigned char st_other; /* Reserved (not used). */ + Elf64_Half st_shndx; /* Section index of symbol. */ + Elf64_Addr st_value; /* Symbol value. */ + Elf64_Xword st_size; /* Size of associated object. */ +} Elf64_Sym; + +/* Macros for accessing the fields of st_info. */ +#define ELF64_ST_BIND(info) ((info) >> 4) +#define ELF64_ST_TYPE(info) ((info) & 0xf) + +/* Macro for constructing st_info from field values. */ +#define ELF64_ST_INFO(bind, type) (((bind) << 4) + ((type) & 0xf)) + +/* Macro for accessing the fields of st_other. */ +#define ELF64_ST_VISIBILITY(oth) ((oth) & 0x3) + +/* Structures used by Sun & GNU-style symbol versioning. */ +typedef struct { + Elf64_Half vd_version; + Elf64_Half vd_flags; + Elf64_Half vd_ndx; + Elf64_Half vd_cnt; + Elf64_Word vd_hash; + Elf64_Word vd_aux; + Elf64_Word vd_next; +} Elf64_Verdef; + +typedef struct { + Elf64_Word vda_name; + Elf64_Word vda_next; +} Elf64_Verdaux; + +typedef struct { + Elf64_Half vn_version; + Elf64_Half vn_cnt; + Elf64_Word vn_file; + Elf64_Word vn_aux; + Elf64_Word vn_next; +} Elf64_Verneed; + +typedef struct { + Elf64_Word vna_hash; + Elf64_Half vna_flags; + Elf64_Half vna_other; + Elf64_Word vna_name; + Elf64_Word vna_next; +} Elf64_Vernaux; + +typedef Elf64_Half Elf64_Versym; + +typedef struct { + Elf64_Half si_boundto; /* direct bindings - symbol bound to */ + Elf64_Half si_flags; /* per symbol flags */ +} Elf64_Syminfo; + +#endif /* !_SYS_ELF64_H_ */ diff --git a/optee_os/lib/libutee/include/elf_common.h b/optee_os/lib/libutee/include/elf_common.h new file mode 100644 index 0000000..0ac08fa --- /dev/null +++ b/optee_os/lib/libutee/include/elf_common.h @@ -0,0 +1,1058 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/*- + * Copyright (c) 2000, 2001, 2008, 2011, David E. O'Brien + * Copyright (c) 1998 John D. Polstra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_ELF_COMMON_H_ +#define _SYS_ELF_COMMON_H_ 1 + +#include + +/* + * ELF definitions that are independent of architecture or word size. + */ + +#ifndef __ASSEMBLER__ +/* + * Note header. The ".note" section contains an array of notes. Each + * begins with this header, aligned to a word boundary. Immediately + * following the note header is n_namesz bytes of name, padded to the + * next word boundary. Then comes n_descsz bytes of descriptor, again + * padded to a word boundary. The values of n_namesz and n_descsz do + * not include the padding. + */ + +typedef struct { + uint32_t n_namesz; /* Length of name. */ + uint32_t n_descsz; /* Length of descriptor. */ + uint32_t n_type; /* Type of this note. */ +} Elf_Note; + +/* + * The header for GNU-style hash sections. + */ + +typedef struct { + uint32_t gh_nbuckets; /* Number of hash buckets. */ + uint32_t gh_symndx; /* First visible symbol in .dynsym. */ + uint32_t gh_maskwords; /* #maskwords used in bloom filter. */ + uint32_t gh_shift2; /* Bloom filter shift count. */ +} Elf_GNU_Hash_Header; + +/* + * Program Property Array + */ +typedef struct { + uint32_t pr_type; + uint32_t pr_datasz; +} Elf_Prop; +#endif /*__ASSEMBLER__*/ + +/* Indexes into the e_ident array. Keep synced with + http://www.sco.com/developers/gabi/latest/ch4.eheader.html */ +#define EI_MAG0 0 /* Magic number, byte 0. */ +#define EI_MAG1 1 /* Magic number, byte 1. */ +#define EI_MAG2 2 /* Magic number, byte 2. */ +#define EI_MAG3 3 /* Magic number, byte 3. */ +#define EI_CLASS 4 /* Class of machine. */ +#define EI_DATA 5 /* Data format. */ +#define EI_VERSION 6 /* ELF format version. */ +#define EI_OSABI 7 /* Operating system / ABI identification */ +#define EI_ABIVERSION 8 /* ABI version */ +#define OLD_EI_BRAND 8 /* Start of architecture identification. */ +#define EI_PAD 9 /* Start of padding (per SVR4 ABI). */ +#define EI_NIDENT 16 /* Size of e_ident array. */ + +/* Values for the magic number bytes. */ +#define ELFMAG0 0x7f +#define ELFMAG1 'E' +#define ELFMAG2 'L' +#define ELFMAG3 'F' +#define ELFMAG "\177ELF" /* magic string */ +#define SELFMAG 4 /* magic string size */ + +/* Values for e_ident[EI_VERSION] and e_version. */ +#define EV_NONE 0 +#define EV_CURRENT 1 + +/* Values for e_ident[EI_CLASS]. */ +#define ELFCLASSNONE 0 /* Unknown class. */ +#define ELFCLASS32 1 /* 32-bit architecture. */ +#define ELFCLASS64 2 /* 64-bit architecture. */ + +/* Values for e_ident[EI_DATA]. */ +#define ELFDATANONE 0 /* Unknown data format. */ +#define ELFDATA2LSB 1 /* 2's complement little-endian. */ +#define ELFDATA2MSB 2 /* 2's complement big-endian. */ + +/* Values for e_ident[EI_OSABI]. */ +#define ELFOSABI_NONE 0 /* UNIX System V ABI */ +#define ELFOSABI_HPUX 1 /* HP-UX operating system */ +#define ELFOSABI_NETBSD 2 /* NetBSD */ +#define ELFOSABI_LINUX 3 /* GNU/Linux */ +#define ELFOSABI_HURD 4 /* GNU/Hurd */ +#define ELFOSABI_86OPEN 5 /* 86Open common IA32 ABI */ +#define ELFOSABI_SOLARIS 6 /* Solaris */ +#define ELFOSABI_AIX 7 /* AIX */ +#define ELFOSABI_IRIX 8 /* IRIX */ +#define ELFOSABI_FREEBSD 9 /* FreeBSD */ +#define ELFOSABI_TRU64 10 /* TRU64 UNIX */ +#define ELFOSABI_MODESTO 11 /* Novell Modesto */ +#define ELFOSABI_OPENBSD 12 /* OpenBSD */ +#define ELFOSABI_OPENVMS 13 /* Open VMS */ +#define ELFOSABI_NSK 14 /* HP Non-Stop Kernel */ +#define ELFOSABI_AROS 15 /* Amiga Research OS */ +#define ELFOSABI_ARM 97 /* ARM */ +#define ELFOSABI_STANDALONE 255 /* Standalone (embedded) application */ + +#define ELFOSABI_SYSV ELFOSABI_NONE /* symbol used in old spec */ +#define ELFOSABI_MONTEREY ELFOSABI_AIX /* Monterey */ + +/* e_ident */ +#define IS_ELF(ehdr) ((ehdr).e_ident[EI_MAG0] == ELFMAG0 && \ + (ehdr).e_ident[EI_MAG1] == ELFMAG1 && \ + (ehdr).e_ident[EI_MAG2] == ELFMAG2 && \ + (ehdr).e_ident[EI_MAG3] == ELFMAG3) + +/* Values for e_type. */ +#define ET_NONE 0 /* Unknown type. */ +#define ET_REL 1 /* Relocatable. */ +#define ET_EXEC 2 /* Executable. */ +#define ET_DYN 3 /* Shared object. */ +#define ET_CORE 4 /* Core file. */ +#define ET_LOOS 0xfe00 /* First operating system specific. */ +#define ET_HIOS 0xfeff /* Last operating system-specific. */ +#define ET_LOPROC 0xff00 /* First processor-specific. */ +#define ET_HIPROC 0xffff /* Last processor-specific. */ + +/* Values for e_machine. */ +#define EM_NONE 0 /* Unknown machine. */ +#define EM_M32 1 /* AT&T WE32100. */ +#define EM_SPARC 2 /* Sun SPARC. */ +#define EM_386 3 /* Intel i386. */ +#define EM_68K 4 /* Motorola 68000. */ +#define EM_88K 5 /* Motorola 88000. */ +#define EM_860 7 /* Intel i860. */ +#define EM_MIPS 8 /* MIPS R3000 Big-Endian only. */ +#define EM_S370 9 /* IBM System/370. */ +#define EM_MIPS_RS3_LE 10 /* MIPS R3000 Little-Endian. */ +#define EM_PARISC 15 /* HP PA-RISC. */ +#define EM_VPP500 17 /* Fujitsu VPP500. */ +#define EM_SPARC32PLUS 18 /* SPARC v8plus. */ +#define EM_960 19 /* Intel 80960. */ +#define EM_PPC 20 /* PowerPC 32-bit. */ +#define EM_PPC64 21 /* PowerPC 64-bit. */ +#define EM_S390 22 /* IBM System/390. */ +#define EM_V800 36 /* NEC V800. */ +#define EM_FR20 37 /* Fujitsu FR20. */ +#define EM_RH32 38 /* TRW RH-32. */ +#define EM_RCE 39 /* Motorola RCE. */ +#define EM_ARM 40 /* ARM. */ +#define EM_SH 42 /* Hitachi SH. */ +#define EM_SPARCV9 43 /* SPARC v9 64-bit. */ +#define EM_TRICORE 44 /* Siemens TriCore embedded processor. */ +#define EM_ARC 45 /* Argonaut RISC Core. */ +#define EM_H8_300 46 /* Hitachi H8/300. */ +#define EM_H8_300H 47 /* Hitachi H8/300H. */ +#define EM_H8S 48 /* Hitachi H8S. */ +#define EM_H8_500 49 /* Hitachi H8/500. */ +#define EM_IA_64 50 /* Intel IA-64 Processor. */ +#define EM_MIPS_X 51 /* Stanford MIPS-X. */ +#define EM_COLDFIRE 52 /* Motorola ColdFire. */ +#define EM_68HC12 53 /* Motorola M68HC12. */ +#define EM_MMA 54 /* Fujitsu MMA. */ +#define EM_PCP 55 /* Siemens PCP. */ +#define EM_NCPU 56 /* Sony nCPU. */ +#define EM_NDR1 57 /* Denso NDR1 microprocessor. */ +#define EM_STARCORE 58 /* Motorola Star*Core processor. */ +#define EM_ME16 59 /* Toyota ME16 processor. */ +#define EM_ST100 60 /* STMicroelectronics ST100 processor. */ +#define EM_TINYJ 61 /* Advanced Logic Corp. TinyJ processor. */ +#define EM_X86_64 62 /* Advanced Micro Devices x86-64 */ +#define EM_AMD64 EM_X86_64 /* Advanced Micro Devices x86-64 (compat) */ +#define EM_PDSP 63 /* Sony DSP Processor. */ +#define EM_FX66 66 /* Siemens FX66 microcontroller. */ +#define EM_ST9PLUS 67 /* STMicroelectronics ST9+ 8/16 + microcontroller. */ +#define EM_ST7 68 /* STmicroelectronics ST7 8-bit + microcontroller. */ +#define EM_68HC16 69 /* Motorola MC68HC16 microcontroller. */ +#define EM_68HC11 70 /* Motorola MC68HC11 microcontroller. */ +#define EM_68HC08 71 /* Motorola MC68HC08 microcontroller. */ +#define EM_68HC05 72 /* Motorola MC68HC05 microcontroller. */ +#define EM_SVX 73 /* Silicon Graphics SVx. */ +#define EM_ST19 74 /* STMicroelectronics ST19 8-bit mc. */ +#define EM_VAX 75 /* Digital VAX. */ +#define EM_CRIS 76 /* Axis Communications 32-bit embedded + processor. */ +#define EM_JAVELIN 77 /* Infineon Technologies 32-bit embedded + processor. */ +#define EM_FIREPATH 78 /* Element 14 64-bit DSP Processor. */ +#define EM_ZSP 79 /* LSI Logic 16-bit DSP Processor. */ +#define EM_MMIX 80 /* Donald Knuth's educational 64-bit proc. */ +#define EM_HUANY 81 /* Harvard University machine-independent + object files. */ +#define EM_PRISM 82 /* SiTera Prism. */ +#define EM_AVR 83 /* Atmel AVR 8-bit microcontroller. */ +#define EM_FR30 84 /* Fujitsu FR30. */ +#define EM_D10V 85 /* Mitsubishi D10V. */ +#define EM_D30V 86 /* Mitsubishi D30V. */ +#define EM_V850 87 /* NEC v850. */ +#define EM_M32R 88 /* Mitsubishi M32R. */ +#define EM_MN10300 89 /* Matsushita MN10300. */ +#define EM_MN10200 90 /* Matsushita MN10200. */ +#define EM_PJ 91 /* picoJava. */ +#define EM_OPENRISC 92 /* OpenRISC 32-bit embedded processor. */ +#define EM_ARC_A5 93 /* ARC Cores Tangent-A5. */ +#define EM_XTENSA 94 /* Tensilica Xtensa Architecture. */ +#define EM_VIDEOCORE 95 /* Alphamosaic VideoCore processor. */ +#define EM_TMM_GPP 96 /* Thompson Multimedia General Purpose + Processor. */ +#define EM_NS32K 97 /* National Semiconductor 32000 series. */ +#define EM_TPC 98 /* Tenor Network TPC processor. */ +#define EM_SNP1K 99 /* Trebia SNP 1000 processor. */ +#define EM_ST200 100 /* STMicroelectronics ST200 microcontroller. */ +#define EM_IP2K 101 /* Ubicom IP2xxx microcontroller family. */ +#define EM_MAX 102 /* MAX Processor. */ +#define EM_CR 103 /* National Semiconductor CompactRISC + microprocessor. */ +#define EM_F2MC16 104 /* Fujitsu F2MC16. */ +#define EM_MSP430 105 /* Texas Instruments embedded microcontroller + msp430. */ +#define EM_BLACKFIN 106 /* Analog Devices Blackfin (DSP) processor. */ +#define EM_SE_C33 107 /* S1C33 Family of Seiko Epson processors. */ +#define EM_SEP 108 /* Sharp embedded microprocessor. */ +#define EM_ARCA 109 /* Arca RISC Microprocessor. */ +#define EM_UNICORE 110 /* Microprocessor series from PKU-Unity Ltd. + and MPRC of Peking University */ +#define EM_AARCH64 183 /* AArch64 (64-bit ARM) */ +#define EM_RISCV 243 /* RISC-V */ + +/* Non-standard or deprecated. */ +#define EM_486 6 /* Intel i486. */ +#define EM_MIPS_RS4_BE 10 /* MIPS R4000 Big-Endian */ +#define EM_ALPHA_STD 41 /* Digital Alpha (standard value). */ +#define EM_ALPHA 0x9026 /* Alpha (written in the absence of an ABI) */ + +/* e_flags for EM_ARM */ +#define EF_ARM_ABI_VERSION 0x05000000 /* ABI version 5 */ +#define EF_ARM_ABIMASK 0xFF000000 +#define EF_ARM_BE8 0x00800000 +#define EF_ARM_ABI_FLOAT_HARD 0x00000400 /* ABI version 5 and later */ +#define EF_ARM_ABI_FLOAT_SOFT 0x00000200 /* ABI version 5 and later */ + +/* Special section indexes. */ +#define SHN_UNDEF 0 /* Undefined, missing, irrelevant. */ +#define SHN_LORESERVE 0xff00 /* First of reserved range. */ +#define SHN_LOPROC 0xff00 /* First processor-specific. */ +#define SHN_HIPROC 0xff1f /* Last processor-specific. */ +#define SHN_LOOS 0xff20 /* First operating system-specific. */ +#define SHN_HIOS 0xff3f /* Last operating system-specific. */ +#define SHN_ABS 0xfff1 /* Absolute values. */ +#define SHN_COMMON 0xfff2 /* Common data. */ +#define SHN_XINDEX 0xffff /* Escape -- index stored elsewhere. */ +#define SHN_HIRESERVE 0xffff /* Last of reserved range. */ + +/* sh_type */ +#define SHT_NULL 0 /* inactive */ +#define SHT_PROGBITS 1 /* program defined information */ +#define SHT_SYMTAB 2 /* symbol table section */ +#define SHT_STRTAB 3 /* string table section */ +#define SHT_RELA 4 /* relocation section with addends */ +#define SHT_HASH 5 /* symbol hash table section */ +#define SHT_DYNAMIC 6 /* dynamic section */ +#define SHT_NOTE 7 /* note section */ +#define SHT_NOBITS 8 /* no space section */ +#define SHT_REL 9 /* relocation section - no addends */ +#define SHT_SHLIB 10 /* reserved - purpose unknown */ +#define SHT_DYNSYM 11 /* dynamic symbol table section */ +#define SHT_INIT_ARRAY 14 /* Initialization function pointers. */ +#define SHT_FINI_ARRAY 15 /* Termination function pointers. */ +#define SHT_PREINIT_ARRAY 16 /* Pre-initialization function ptrs. */ +#define SHT_GROUP 17 /* Section group. */ +#define SHT_SYMTAB_SHNDX 18 /* Section indexes (see SHN_XINDEX). */ +#define SHT_LOOS 0x60000000 /* First of OS specific semantics */ +#define SHT_LOSUNW 0x6ffffff4 +#define SHT_SUNW_dof 0x6ffffff4 +#define SHT_SUNW_cap 0x6ffffff5 +#define SHT_SUNW_SIGNATURE 0x6ffffff6 +#define SHT_GNU_HASH 0x6ffffff6 +#define SHT_GNU_LIBLIST 0x6ffffff7 +#define SHT_SUNW_ANNOTATE 0x6ffffff7 +#define SHT_SUNW_DEBUGSTR 0x6ffffff8 +#define SHT_SUNW_DEBUG 0x6ffffff9 +#define SHT_SUNW_move 0x6ffffffa +#define SHT_SUNW_COMDAT 0x6ffffffb +#define SHT_SUNW_syminfo 0x6ffffffc +#define SHT_SUNW_verdef 0x6ffffffd +#define SHT_GNU_verdef 0x6ffffffd /* Symbol versions provided */ +#define SHT_SUNW_verneed 0x6ffffffe +#define SHT_GNU_verneed 0x6ffffffe /* Symbol versions required */ +#define SHT_SUNW_versym 0x6fffffff +#define SHT_GNU_versym 0x6fffffff /* Symbol version table */ +#define SHT_HISUNW 0x6fffffff +#define SHT_HIOS 0x6fffffff /* Last of OS specific semantics */ +#define SHT_LOPROC 0x70000000 /* reserved range for processor */ +#define SHT_AMD64_UNWIND 0x70000001 /* unwind information */ +#define SHT_ARM_EXIDX 0x70000001 /* Exception index table. */ +#define SHT_ARM_PREEMPTMAP 0x70000002 /* BPABI DLL dynamic linking + pre-emption map. */ +#define SHT_ARM_ATTRIBUTES 0x70000003 /* Object file compatibility + attributes. */ +#define SHT_ARM_DEBUGOVERLAY 0x70000004 /* See DBGOVL for details. */ +#define SHT_ARM_OVERLAYSECTION 0x70000005 /* See DBGOVL for details. */ +#define SHT_MIPS_REGINFO 0x70000006 +#define SHT_MIPS_OPTIONS 0x7000000d +#define SHT_MIPS_DWARF 0x7000001e /* MIPS gcc uses MIPS_DWARF */ +#define SHT_HIPROC 0x7fffffff /* specific section header types */ +#define SHT_LOUSER 0x80000000 /* reserved range for application */ +#define SHT_HIUSER 0xffffffff /* specific indexes */ + +/* Flags for sh_flags. */ +#define SHF_WRITE 0x1 /* Section contains writable data. */ +#define SHF_ALLOC 0x2 /* Section occupies memory. */ +#define SHF_EXECINSTR 0x4 /* Section contains instructions. */ +#define SHF_MERGE 0x10 /* Section may be merged. */ +#define SHF_STRINGS 0x20 /* Section contains strings. */ +#define SHF_INFO_LINK 0x40 /* sh_info holds section index. */ +#define SHF_LINK_ORDER 0x80 /* Special ordering requirements. */ +#define SHF_OS_NONCONFORMING 0x100 /* OS-specific processing required. */ +#define SHF_GROUP 0x200 /* Member of section group. */ +#define SHF_TLS 0x400 /* Section contains TLS data. */ +#define SHF_MASKOS 0x0ff00000 /* OS-specific semantics. */ +#define SHF_MASKPROC 0xf0000000 /* Processor-specific semantics. */ + +/* Values for p_type. */ +#define PT_NULL 0 /* Unused entry. */ +#define PT_LOAD 1 /* Loadable segment. */ +#define PT_DYNAMIC 2 /* Dynamic linking information segment. */ +#define PT_INTERP 3 /* Pathname of interpreter. */ +#define PT_NOTE 4 /* Auxiliary information. */ +#define PT_SHLIB 5 /* Reserved (not used). */ +#define PT_PHDR 6 /* Location of program header itself. */ +#define PT_TLS 7 /* Thread local storage segment */ +#define PT_LOOS 0x60000000 /* First OS-specific. */ +#define PT_SUNW_UNWIND 0x6464e550 /* amd64 UNWIND program header */ +#define PT_GNU_EH_FRAME 0x6474e550 +#define PT_GNU_STACK 0x6474e551 +#define PT_GNU_RELRO 0x6474e552 +#define PT_GNU_PROPERTY 0x6474e553 +#define PT_LOSUNW 0x6ffffffa +#define PT_SUNWBSS 0x6ffffffa /* Sun Specific segment */ +#define PT_SUNWSTACK 0x6ffffffb /* describes the stack segment */ +#define PT_SUNWDTRACE 0x6ffffffc /* private */ +#define PT_SUNWCAP 0x6ffffffd /* hard/soft capabilities segment */ +#define PT_HISUNW 0x6fffffff +#define PT_HIOS 0x6fffffff /* Last OS-specific. */ +#define PT_LOPROC 0x70000000 /* First processor-specific type. */ +#define PT_ARM_EXIDX 0x70000001 /* .ARM.exidx segment */ +#define PT_HIPROC 0x7fffffff /* Last processor-specific type. */ + +/* Values for p_flags. */ +#define PF_X 0x1 /* Executable. */ +#define PF_W 0x2 /* Writable. */ +#define PF_R 0x4 /* Readable. */ +#define PF_MASKOS 0x0ff00000 /* Operating system-specific. */ +#define PF_MASKPROC 0xf0000000 /* Processor-specific. */ + +/* Extended program header index. */ +#define PN_XNUM 0xffff + +/* Values for d_tag. */ +#define DT_NULL 0 /* Terminating entry. */ +#define DT_NEEDED 1 /* String table offset of a needed shared + library. */ +#define DT_PLTRELSZ 2 /* Total size in bytes of PLT relocations. */ +#define DT_PLTGOT 3 /* Processor-dependent address. */ +#define DT_HASH 4 /* Address of symbol hash table. */ +#define DT_STRTAB 5 /* Address of string table. */ +#define DT_SYMTAB 6 /* Address of symbol table. */ +#define DT_RELA 7 /* Address of ElfNN_Rela relocations. */ +#define DT_RELASZ 8 /* Total size of ElfNN_Rela relocations. */ +#define DT_RELAENT 9 /* Size of each ElfNN_Rela relocation entry. */ +#define DT_STRSZ 10 /* Size of string table. */ +#define DT_SYMENT 11 /* Size of each symbol table entry. */ +#define DT_INIT 12 /* Address of initialization function. */ +#define DT_FINI 13 /* Address of finalization function. */ +#define DT_SONAME 14 /* String table offset of shared object + name. */ +#define DT_RPATH 15 /* String table offset of library path. [sup] */ +#define DT_SYMBOLIC 16 /* Indicates "symbolic" linking. [sup] */ +#define DT_REL 17 /* Address of ElfNN_Rel relocations. */ +#define DT_RELSZ 18 /* Total size of ElfNN_Rel relocations. */ +#define DT_RELENT 19 /* Size of each ElfNN_Rel relocation. */ +#define DT_PLTREL 20 /* Type of relocation used for PLT. */ +#define DT_DEBUG 21 /* Reserved (not used). */ +#define DT_TEXTREL 22 /* Indicates there may be relocations in + non-writable segments. [sup] */ +#define DT_JMPREL 23 /* Address of PLT relocations. */ +#define DT_BIND_NOW 24 /* [sup] */ +#define DT_INIT_ARRAY 25 /* Address of the array of pointers to + initialization functions */ +#define DT_FINI_ARRAY 26 /* Address of the array of pointers to + termination functions */ +#define DT_INIT_ARRAYSZ 27 /* Size in bytes of the array of + initialization functions. */ +#define DT_FINI_ARRAYSZ 28 /* Size in bytes of the array of + termination functions. */ +#define DT_RUNPATH 29 /* String table offset of a null-terminated + library search path string. */ +#define DT_FLAGS 30 /* Object specific flag values. */ +#define DT_ENCODING 32 /* Values greater than or equal to DT_ENCODING + and less than DT_LOOS follow the rules for + the interpretation of the d_un union + as follows: even == 'd_ptr', odd == 'd_val' + or none */ +#define DT_PREINIT_ARRAY 32 /* Address of the array of pointers to + pre-initialization functions. */ +#define DT_PREINIT_ARRAYSZ 33 /* Size in bytes of the array of + pre-initialization functions. */ +#define DT_MAXPOSTAGS 34 /* number of positive tags */ +#define DT_LOOS 0x6000000d /* First OS-specific */ +#define DT_SUNW_AUXILIARY 0x6000000d /* symbol auxiliary name */ +#define DT_SUNW_RTLDINF 0x6000000e /* ld.so.1 info (private) */ +#define DT_SUNW_FILTER 0x6000000f /* symbol filter name */ +#define DT_SUNW_CAP 0x60000010 /* hardware/software */ +#define DT_HIOS 0x6ffff000 /* Last OS-specific */ + +/* + * DT_* entries which fall between DT_VALRNGHI & DT_VALRNGLO use the + * Dyn.d_un.d_val field of the Elf*_Dyn structure. + */ +#define DT_VALRNGLO 0x6ffffd00 +#define DT_CHECKSUM 0x6ffffdf8 /* elf checksum */ +#define DT_PLTPADSZ 0x6ffffdf9 /* pltpadding size */ +#define DT_MOVEENT 0x6ffffdfa /* move table entry size */ +#define DT_MOVESZ 0x6ffffdfb /* move table size */ +#define DT_FEATURE_1 0x6ffffdfc /* feature holder */ +#define DT_POSFLAG_1 0x6ffffdfd /* flags for DT_* entries, effecting */ + /* the following DT_* entry. */ + /* See DF_P1_* definitions */ +#define DT_SYMINSZ 0x6ffffdfe /* syminfo table size (in bytes) */ +#define DT_SYMINENT 0x6ffffdff /* syminfo entry size (in bytes) */ +#define DT_VALRNGHI 0x6ffffdff + +/* + * DT_* entries which fall between DT_ADDRRNGHI & DT_ADDRRNGLO use the + * Dyn.d_un.d_ptr field of the Elf*_Dyn structure. + * + * If any adjustment is made to the ELF object after it has been + * built, these entries will need to be adjusted. + */ +#define DT_ADDRRNGLO 0x6ffffe00 +#define DT_GNU_HASH 0x6ffffef5 /* GNU-style hash table */ +#define DT_CONFIG 0x6ffffefa /* configuration information */ +#define DT_DEPAUDIT 0x6ffffefb /* dependency auditing */ +#define DT_AUDIT 0x6ffffefc /* object auditing */ +#define DT_PLTPAD 0x6ffffefd /* pltpadding (sparcv9) */ +#define DT_MOVETAB 0x6ffffefe /* move table */ +#define DT_SYMINFO 0x6ffffeff /* syminfo table */ +#define DT_ADDRRNGHI 0x6ffffeff + +#define DT_VERSYM 0x6ffffff0 /* Address of versym section. */ +#define DT_RELACOUNT 0x6ffffff9 /* number of RELATIVE relocations */ +#define DT_RELCOUNT 0x6ffffffa /* number of RELATIVE relocations */ +#define DT_FLAGS_1 0x6ffffffb /* state flags - see DF_1_* defs */ +#define DT_VERDEF 0x6ffffffc /* Address of verdef section. */ +#define DT_VERDEFNUM 0x6ffffffd /* Number of elems in verdef section */ +#define DT_VERNEED 0x6ffffffe /* Address of verneed section. */ +#define DT_VERNEEDNUM 0x6fffffff /* Number of elems in verneed section */ + +#define DT_LOPROC 0x70000000 /* First processor-specific type. */ +#define DT_DEPRECATED_SPARC_REGISTER 0x7000001 +#define DT_AUXILIARY 0x7ffffffd /* shared library auxiliary name */ +#define DT_USED 0x7ffffffe /* ignored - same as needed */ +#define DT_FILTER 0x7fffffff /* shared library filter name */ +#define DT_HIPROC 0x7fffffff /* Last processor-specific type. */ + +/* Values for DT_FLAGS */ +#define DF_ORIGIN 0x0001 /* Indicates that the object being loaded may + make reference to the $ORIGIN substitution + string */ +#define DF_SYMBOLIC 0x0002 /* Indicates "symbolic" linking. */ +#define DF_TEXTREL 0x0004 /* Indicates there may be relocations in + non-writable segments. */ +#define DF_BIND_NOW 0x0008 /* Indicates that the dynamic linker should + process all relocations for the object + containing this entry before transferring + control to the program. */ +#define DF_STATIC_TLS 0x0010 /* Indicates that the shared object or + executable contains code using a static + thread-local storage scheme. */ + +/* Values for DT_FLAGS_1 */ +#define DF_1_BIND_NOW 0x00000001 /* Same as DF_BIND_NOW */ +#define DF_1_GLOBAL 0x00000002 /* Set the RTLD_GLOBAL for object */ +#define DF_1_NODELETE 0x00000008 /* Set the RTLD_NODELETE for object */ +#define DF_1_LOADFLTR 0x00000010 /* Immediate loading of filtees */ +#define DF_1_NOOPEN 0x00000040 /* Do not allow loading on dlopen() */ +#define DF_1_ORIGIN 0x00000080 /* Process $ORIGIN */ +#define DF_1_INTERPOSE 0x00000400 /* Interpose all objects but main */ +#define DF_1_NODEFLIB 0x00000800 /* Do not search default paths */ + +/* Note section names */ +#define ELF_NOTE_FREEBSD "FreeBSD" +#define ELF_NOTE_NETBSD "NetBSD" +#define ELF_NOTE_SOLARIS "SUNW Solaris" +#define ELF_NOTE_GNU "GNU" + +/* Values for n_type. Used in core files. */ +#define NT_PRSTATUS 1 /* Process status. */ +#define NT_FPREGSET 2 /* Floating point registers. */ +#define NT_PRPSINFO 3 /* Process state info. */ +#define NT_THRMISC 7 /* Thread miscellaneous info. */ +#define NT_PROCSTAT_PROC 8 /* Procstat proc data. */ +#define NT_PROCSTAT_FILES 9 /* Procstat files data. */ +#define NT_PROCSTAT_VMMAP 10 /* Procstat vmmap data. */ +#define NT_PROCSTAT_GROUPS 11 /* Procstat groups data. */ +#define NT_PROCSTAT_UMASK 12 /* Procstat umask data. */ +#define NT_PROCSTAT_RLIMIT 13 /* Procstat rlimit data. */ +#define NT_PROCSTAT_OSREL 14 /* Procstat osreldate data. */ +#define NT_PROCSTAT_PSSTRINGS 15 /* Procstat ps_strings data. */ +#define NT_PROCSTAT_AUXV 16 /* Procstat auxv data. */ + +/* GNU note types. */ +#define NT_GNU_ABI_TAG 1 +#define NT_GNU_HWCAP 2 +#define NT_GNU_BUILD_ID 3 +#define NT_GNU_GOLD_VERSION 4 +#define NT_GNU_PROPERTY_TYPE_0 5 + +#define GNU_PROPERTY_LOPROC 0xc0000000 +#define GNU_PROPERTY_HIPROC 0xdfffffff + +#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 + +#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 0x00000001 +#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 0x00000002 + +#define GNU_PROPERTY_X86_FEATURE_1_AND 0xc0000002 + +/* Symbol Binding - ELFNN_ST_BIND - st_info */ +#define STB_LOCAL 0 /* Local symbol */ +#define STB_GLOBAL 1 /* Global symbol */ +#define STB_WEAK 2 /* like global - lower precedence */ +#define STB_LOOS 10 /* Reserved range for operating system */ +#define STB_HIOS 12 /* specific semantics. */ +#define STB_LOPROC 13 /* reserved range for processor */ +#define STB_HIPROC 15 /* specific semantics. */ + +/* Symbol type - ELFNN_ST_TYPE - st_info */ +#define STT_NOTYPE 0 /* Unspecified type. */ +#define STT_OBJECT 1 /* Data object. */ +#define STT_FUNC 2 /* Function. */ +#define STT_SECTION 3 /* Section. */ +#define STT_FILE 4 /* Source file. */ +#define STT_COMMON 5 /* Uninitialized common block. */ +#define STT_TLS 6 /* TLS object. */ +#define STT_NUM 7 +#define STT_LOOS 10 /* Reserved range for operating system */ +#define STT_GNU_IFUNC 10 +#define STT_HIOS 12 /* specific semantics. */ +#define STT_LOPROC 13 /* reserved range for processor */ +#define STT_HIPROC 15 /* specific semantics. */ + +/* Symbol visibility - ELFNN_ST_VISIBILITY - st_other */ +#define STV_DEFAULT 0x0 /* Default visibility (see binding). */ +#define STV_INTERNAL 0x1 /* Special meaning in relocatable objects. */ +#define STV_HIDDEN 0x2 /* Not visible. */ +#define STV_PROTECTED 0x3 /* Visible but not preemptible. */ +#define STV_EXPORTED 0x4 +#define STV_SINGLETON 0x5 +#define STV_ELIMINATE 0x6 + +/* Special symbol table indexes. */ +#define STN_UNDEF 0 /* Undefined symbol index. */ + +/* Symbol versioning flags. */ +#define VER_DEF_CURRENT 1 +#define VER_DEF_IDX(x) VER_NDX(x) + +#define VER_FLG_BASE 0x01 +#define VER_FLG_WEAK 0x02 + +#define VER_NEED_CURRENT 1 +#define VER_NEED_WEAK (1u << 15) +#define VER_NEED_HIDDEN VER_NDX_HIDDEN +#define VER_NEED_IDX(x) VER_NDX(x) + +#define VER_NDX_LOCAL 0 +#define VER_NDX_GLOBAL 1 +#define VER_NDX_GIVEN 2 + +#define VER_NDX_HIDDEN (1u << 15) +#define VER_NDX(x) ((x) & ~(1u << 15)) + +#define CA_SUNW_NULL 0 +#define CA_SUNW_HW_1 1 /* first hardware capabilities entry */ +#define CA_SUNW_SF_1 2 /* first software capabilities entry */ + +/* + * Syminfo flag values + */ +#define SYMINFO_FLG_DIRECT 0x0001 /* symbol ref has direct association */ + /* to object containing defn. */ +#define SYMINFO_FLG_PASSTHRU 0x0002 /* ignored - see SYMINFO_FLG_FILTER */ +#define SYMINFO_FLG_COPY 0x0004 /* symbol is a copy-reloc */ +#define SYMINFO_FLG_LAZYLOAD 0x0008 /* object containing defn should be */ + /* lazily-loaded */ +#define SYMINFO_FLG_DIRECTBIND 0x0010 /* ref should be bound directly to */ + /* object containing defn. */ +#define SYMINFO_FLG_NOEXTDIRECT 0x0020 /* don't let an external reference */ + /* directly bind to this symbol */ +#define SYMINFO_FLG_FILTER 0x0002 /* symbol ref is associated to a */ +#define SYMINFO_FLG_AUXILIARY 0x0040 /* standard or auxiliary filter */ + +/* + * Syminfo.si_boundto values. + */ +#define SYMINFO_BT_SELF 0xffff /* symbol bound to self */ +#define SYMINFO_BT_PARENT 0xfffe /* symbol bound to parent */ +#define SYMINFO_BT_NONE 0xfffd /* no special symbol binding */ +#define SYMINFO_BT_EXTERN 0xfffc /* symbol defined as external */ +#define SYMINFO_BT_LOWRESERVE 0xff00 /* beginning of reserved entries */ + +/* + * Syminfo version values. + */ +#define SYMINFO_NONE 0 /* Syminfo version */ +#define SYMINFO_CURRENT 1 +#define SYMINFO_NUM 2 + +/* + * Relocation types. + * + * All machine architectures are defined here to allow tools on one to + * handle others. + */ + +#define R_386_NONE 0 /* No relocation. */ +#define R_386_32 1 /* Add symbol value. */ +#define R_386_PC32 2 /* Add PC-relative symbol value. */ +#define R_386_GOT32 3 /* Add PC-relative GOT offset. */ +#define R_386_PLT32 4 /* Add PC-relative PLT offset. */ +#define R_386_COPY 5 /* Copy data from shared object. */ +#define R_386_GLOB_DAT 6 /* Set GOT entry to data address. */ +#define R_386_JMP_SLOT 7 /* Set GOT entry to code address. */ +#define R_386_RELATIVE 8 /* Add load address of shared object. */ +#define R_386_GOTOFF 9 /* Add GOT-relative symbol address. */ +#define R_386_GOTPC 10 /* Add PC-relative GOT table address. */ +#define R_386_TLS_TPOFF 14 /* Negative offset in static TLS block */ +#define R_386_TLS_IE 15 /* Absolute address of GOT for -ve static TLS */ +#define R_386_TLS_GOTIE 16 /* GOT entry for negative static TLS block */ +#define R_386_TLS_LE 17 /* Negative offset relative to static TLS */ +#define R_386_TLS_GD 18 /* 32 bit offset to GOT (index,off) pair */ +#define R_386_TLS_LDM 19 /* 32 bit offset to GOT (index,zero) pair */ +#define R_386_TLS_GD_32 24 /* 32 bit offset to GOT (index,off) pair */ +#define R_386_TLS_GD_PUSH 25 /* pushl instruction for Sun ABI GD sequence */ +#define R_386_TLS_GD_CALL 26 /* call instruction for Sun ABI GD sequence */ +#define R_386_TLS_GD_POP 27 /* popl instruction for Sun ABI GD sequence */ +#define R_386_TLS_LDM_32 28 /* 32 bit offset to GOT (index,zero) pair */ +#define R_386_TLS_LDM_PUSH 29 /* pushl instruction for Sun ABI LD sequence */ +#define R_386_TLS_LDM_CALL 30 /* call instruction for Sun ABI LD sequence */ +#define R_386_TLS_LDM_POP 31 /* popl instruction for Sun ABI LD sequence */ +#define R_386_TLS_LDO_32 32 /* 32 bit offset from start of TLS block */ +#define R_386_TLS_IE_32 33 /* 32 bit offset to GOT static TLS offset entry */ +#define R_386_TLS_LE_32 34 /* 32 bit offset within static TLS block */ +#define R_386_TLS_DTPMOD32 35 /* GOT entry containing TLS index */ +#define R_386_TLS_DTPOFF32 36 /* GOT entry containing TLS offset */ +#define R_386_TLS_TPOFF32 37 /* GOT entry of -ve static TLS offset */ +#define R_386_IRELATIVE 42 /* PLT entry resolved indirectly at runtime */ + +#define R_RISCV_NONE 0 /* No relocation. */ +#define R_RISCV_64 2 /* 64-bit relocation. */ +#define R_RISCV_RELATIVE 3 /* Adjust a link address (A) to its + load address (B + A). */ +#define R_RISCV_JUMP_SLOT 5 /* Indicates the symbol associated + with a PLT entry. */ + +#define R_AARCH64_NONE 0 /* No relocation. */ +#define R_AARCH64_ABS64 257 +#define R_AARCH64_GLOB_DAT 1025 /* Set GOT entry to data address. */ +#define R_AARCH64_JUMP_SLOT 1026 /* Set GOT entry to code address. */ +#define R_AARCH64_RELATIVE 1027 /* Add load address of shared object. */ +#define R_AARCH64_TLS_TPREL 1030 /* Offset of the TLS block in the TCB */ +#define R_AARCH64_TLSDESC 1031 /* TLS descriptor to be filled */ + +#define R_ARM_NONE 0 /* No relocation. */ +#define R_ARM_PC24 1 +#define R_ARM_ABS32 2 +#define R_ARM_REL32 3 +#define R_ARM_PC13 4 +#define R_ARM_ABS16 5 +#define R_ARM_ABS12 6 +#define R_ARM_THM_ABS5 7 +#define R_ARM_ABS8 8 +#define R_ARM_SBREL32 9 +#define R_ARM_THM_PC22 10 +#define R_ARM_THM_PC8 11 +#define R_ARM_AMP_VCALL9 12 +#define R_ARM_SWI24 13 +#define R_ARM_THM_SWI8 14 +#define R_ARM_XPC25 15 +#define R_ARM_THM_XPC22 16 +/* TLS relocations */ +#define R_ARM_TLS_DTPMOD32 17 /* ID of module containing symbol */ +#define R_ARM_TLS_DTPOFF32 18 /* Offset in TLS block */ +#define R_ARM_TLS_TPOFF32 19 /* Offset in static TLS block */ +#define R_ARM_COPY 20 /* Copy data from shared object. */ +#define R_ARM_GLOB_DAT 21 /* Set GOT entry to data address. */ +#define R_ARM_JUMP_SLOT 22 /* Set GOT entry to code address. */ +#define R_ARM_RELATIVE 23 /* Add load address of shared object. */ +#define R_ARM_GOTOFF 24 /* Add GOT-relative symbol address. */ +#define R_ARM_GOTPC 25 /* Add PC-relative GOT table address. */ +#define R_ARM_GOT32 26 /* Add PC-relative GOT offset. */ +#define R_ARM_PLT32 27 /* Add PC-relative PLT offset. */ +#define R_ARM_GNU_VTENTRY 100 +#define R_ARM_GNU_VTINHERIT 101 +#define R_ARM_RSBREL32 250 +#define R_ARM_THM_RPC22 251 +#define R_ARM_RREL32 252 +#define R_ARM_RABS32 253 +#define R_ARM_RPC24 254 +#define R_ARM_RBASE 255 + +/* Name Value Field Calculation */ +#define R_IA_64_NONE 0 /* None */ +#define R_IA_64_IMM14 0x21 /* immediate14 S + A */ +#define R_IA_64_IMM22 0x22 /* immediate22 S + A */ +#define R_IA_64_IMM64 0x23 /* immediate64 S + A */ +#define R_IA_64_DIR32MSB 0x24 /* word32 MSB S + A */ +#define R_IA_64_DIR32LSB 0x25 /* word32 LSB S + A */ +#define R_IA_64_DIR64MSB 0x26 /* word64 MSB S + A */ +#define R_IA_64_DIR64LSB 0x27 /* word64 LSB S + A */ +#define R_IA_64_GPREL22 0x2a /* immediate22 @gprel(S + A) */ +#define R_IA_64_GPREL64I 0x2b /* immediate64 @gprel(S + A) */ +#define R_IA_64_GPREL32MSB 0x2c /* word32 MSB @gprel(S + A) */ +#define R_IA_64_GPREL32LSB 0x2d /* word32 LSB @gprel(S + A) */ +#define R_IA_64_GPREL64MSB 0x2e /* word64 MSB @gprel(S + A) */ +#define R_IA_64_GPREL64LSB 0x2f /* word64 LSB @gprel(S + A) */ +#define R_IA_64_LTOFF22 0x32 /* immediate22 @ltoff(S + A) */ +#define R_IA_64_LTOFF64I 0x33 /* immediate64 @ltoff(S + A) */ +#define R_IA_64_PLTOFF22 0x3a /* immediate22 @pltoff(S + A) */ +#define R_IA_64_PLTOFF64I 0x3b /* immediate64 @pltoff(S + A) */ +#define R_IA_64_PLTOFF64MSB 0x3e /* word64 MSB @pltoff(S + A) */ +#define R_IA_64_PLTOFF64LSB 0x3f /* word64 LSB @pltoff(S + A) */ +#define R_IA_64_FPTR64I 0x43 /* immediate64 @fptr(S + A) */ +#define R_IA_64_FPTR32MSB 0x44 /* word32 MSB @fptr(S + A) */ +#define R_IA_64_FPTR32LSB 0x45 /* word32 LSB @fptr(S + A) */ +#define R_IA_64_FPTR64MSB 0x46 /* word64 MSB @fptr(S + A) */ +#define R_IA_64_FPTR64LSB 0x47 /* word64 LSB @fptr(S + A) */ +#define R_IA_64_PCREL60B 0x48 /* immediate60 form1 S + A - P */ +#define R_IA_64_PCREL21B 0x49 /* immediate21 form1 S + A - P */ +#define R_IA_64_PCREL21M 0x4a /* immediate21 form2 S + A - P */ +#define R_IA_64_PCREL21F 0x4b /* immediate21 form3 S + A - P */ +#define R_IA_64_PCREL32MSB 0x4c /* word32 MSB S + A - P */ +#define R_IA_64_PCREL32LSB 0x4d /* word32 LSB S + A - P */ +#define R_IA_64_PCREL64MSB 0x4e /* word64 MSB S + A - P */ +#define R_IA_64_PCREL64LSB 0x4f /* word64 LSB S + A - P */ +#define R_IA_64_LTOFF_FPTR22 0x52 /* immediate22 @ltoff(@fptr(S + A)) */ +#define R_IA_64_LTOFF_FPTR64I 0x53 /* immediate64 @ltoff(@fptr(S + A)) */ +#define R_IA_64_LTOFF_FPTR32MSB 0x54 /* word32 MSB @ltoff(@fptr(S + A)) */ +#define R_IA_64_LTOFF_FPTR32LSB 0x55 /* word32 LSB @ltoff(@fptr(S + A)) */ +#define R_IA_64_LTOFF_FPTR64MSB 0x56 /* word64 MSB @ltoff(@fptr(S + A)) */ +#define R_IA_64_LTOFF_FPTR64LSB 0x57 /* word64 LSB @ltoff(@fptr(S + A)) */ +#define R_IA_64_SEGREL32MSB 0x5c /* word32 MSB @segrel(S + A) */ +#define R_IA_64_SEGREL32LSB 0x5d /* word32 LSB @segrel(S + A) */ +#define R_IA_64_SEGREL64MSB 0x5e /* word64 MSB @segrel(S + A) */ +#define R_IA_64_SEGREL64LSB 0x5f /* word64 LSB @segrel(S + A) */ +#define R_IA_64_SECREL32MSB 0x64 /* word32 MSB @secrel(S + A) */ +#define R_IA_64_SECREL32LSB 0x65 /* word32 LSB @secrel(S + A) */ +#define R_IA_64_SECREL64MSB 0x66 /* word64 MSB @secrel(S + A) */ +#define R_IA_64_SECREL64LSB 0x67 /* word64 LSB @secrel(S + A) */ +#define R_IA_64_REL32MSB 0x6c /* word32 MSB BD + A */ +#define R_IA_64_REL32LSB 0x6d /* word32 LSB BD + A */ +#define R_IA_64_REL64MSB 0x6e /* word64 MSB BD + A */ +#define R_IA_64_REL64LSB 0x6f /* word64 LSB BD + A */ +#define R_IA_64_LTV32MSB 0x74 /* word32 MSB S + A */ +#define R_IA_64_LTV32LSB 0x75 /* word32 LSB S + A */ +#define R_IA_64_LTV64MSB 0x76 /* word64 MSB S + A */ +#define R_IA_64_LTV64LSB 0x77 /* word64 LSB S + A */ +#define R_IA_64_PCREL21BI 0x79 /* immediate21 form1 S + A - P */ +#define R_IA_64_PCREL22 0x7a /* immediate22 S + A - P */ +#define R_IA_64_PCREL64I 0x7b /* immediate64 S + A - P */ +#define R_IA_64_IPLTMSB 0x80 /* function descriptor MSB special */ +#define R_IA_64_IPLTLSB 0x81 /* function descriptor LSB speciaal */ +#define R_IA_64_SUB 0x85 /* immediate64 A - S */ +#define R_IA_64_LTOFF22X 0x86 /* immediate22 special */ +#define R_IA_64_LDXMOV 0x87 /* immediate22 special */ +#define R_IA_64_TPREL14 0x91 /* imm14 @tprel(S + A) */ +#define R_IA_64_TPREL22 0x92 /* imm22 @tprel(S + A) */ +#define R_IA_64_TPREL64I 0x93 /* imm64 @tprel(S + A) */ +#define R_IA_64_TPREL64MSB 0x96 /* word64 MSB @tprel(S + A) */ +#define R_IA_64_TPREL64LSB 0x97 /* word64 LSB @tprel(S + A) */ +#define R_IA_64_LTOFF_TPREL22 0x9a /* imm22 @ltoff(@tprel(S+A)) */ +#define R_IA_64_DTPMOD64MSB 0xa6 /* word64 MSB @dtpmod(S + A) */ +#define R_IA_64_DTPMOD64LSB 0xa7 /* word64 LSB @dtpmod(S + A) */ +#define R_IA_64_LTOFF_DTPMOD22 0xaa /* imm22 @ltoff(@dtpmod(S+A)) */ +#define R_IA_64_DTPREL14 0xb1 /* imm14 @dtprel(S + A) */ +#define R_IA_64_DTPREL22 0xb2 /* imm22 @dtprel(S + A) */ +#define R_IA_64_DTPREL64I 0xb3 /* imm64 @dtprel(S + A) */ +#define R_IA_64_DTPREL32MSB 0xb4 /* word32 MSB @dtprel(S + A) */ +#define R_IA_64_DTPREL32LSB 0xb5 /* word32 LSB @dtprel(S + A) */ +#define R_IA_64_DTPREL64MSB 0xb6 /* word64 MSB @dtprel(S + A) */ +#define R_IA_64_DTPREL64LSB 0xb7 /* word64 LSB @dtprel(S + A) */ +#define R_IA_64_LTOFF_DTPREL22 0xba /* imm22 @ltoff(@dtprel(S+A)) */ + +#define R_MIPS_NONE 0 /* No reloc */ +#define R_MIPS_16 1 /* Direct 16 bit */ +#define R_MIPS_32 2 /* Direct 32 bit */ +#define R_MIPS_REL32 3 /* PC relative 32 bit */ +#define R_MIPS_26 4 /* Direct 26 bit shifted */ +#define R_MIPS_HI16 5 /* High 16 bit */ +#define R_MIPS_LO16 6 /* Low 16 bit */ +#define R_MIPS_GPREL16 7 /* GP relative 16 bit */ +#define R_MIPS_LITERAL 8 /* 16 bit literal entry */ +#define R_MIPS_GOT16 9 /* 16 bit GOT entry */ +#define R_MIPS_PC16 10 /* PC relative 16 bit */ +#define R_MIPS_CALL16 11 /* 16 bit GOT entry for function */ +#define R_MIPS_GPREL32 12 /* GP relative 32 bit */ +#define R_MIPS_64 18 /* Direct 64 bit */ +#define R_MIPS_GOTHI16 21 /* GOT HI 16 bit */ +#define R_MIPS_GOTLO16 22 /* GOT LO 16 bit */ +#define R_MIPS_CALLHI16 30 /* upper 16 bit GOT entry for function */ +#define R_MIPS_CALLLO16 31 /* lower 16 bit GOT entry for function */ + +#define R_PPC_NONE 0 /* No relocation. */ +#define R_PPC_ADDR32 1 +#define R_PPC_ADDR24 2 +#define R_PPC_ADDR16 3 +#define R_PPC_ADDR16_LO 4 +#define R_PPC_ADDR16_HI 5 +#define R_PPC_ADDR16_HA 6 +#define R_PPC_ADDR14 7 +#define R_PPC_ADDR14_BRTAKEN 8 +#define R_PPC_ADDR14_BRNTAKEN 9 +#define R_PPC_REL24 10 +#define R_PPC_REL14 11 +#define R_PPC_REL14_BRTAKEN 12 +#define R_PPC_REL14_BRNTAKEN 13 +#define R_PPC_GOT16 14 +#define R_PPC_GOT16_LO 15 +#define R_PPC_GOT16_HI 16 +#define R_PPC_GOT16_HA 17 +#define R_PPC_PLTREL24 18 +#define R_PPC_COPY 19 +#define R_PPC_GLOB_DAT 20 +#define R_PPC_JMP_SLOT 21 +#define R_PPC_RELATIVE 22 +#define R_PPC_LOCAL24PC 23 +#define R_PPC_UADDR32 24 +#define R_PPC_UADDR16 25 +#define R_PPC_REL32 26 +#define R_PPC_PLT32 27 +#define R_PPC_PLTREL32 28 +#define R_PPC_PLT16_LO 29 +#define R_PPC_PLT16_HI 30 +#define R_PPC_PLT16_HA 31 +#define R_PPC_SDAREL16 32 +#define R_PPC_SECTOFF 33 +#define R_PPC_SECTOFF_LO 34 +#define R_PPC_SECTOFF_HI 35 +#define R_PPC_SECTOFF_HA 36 + +/* + * 64-bit relocations + */ +#define R_PPC64_ADDR64 38 +#define R_PPC64_ADDR16_HIGHER 39 +#define R_PPC64_ADDR16_HIGHERA 40 +#define R_PPC64_ADDR16_HIGHEST 41 +#define R_PPC64_ADDR16_HIGHESTA 42 +#define R_PPC64_UADDR64 43 +#define R_PPC64_REL64 44 +#define R_PPC64_PLT64 45 +#define R_PPC64_PLTREL64 46 +#define R_PPC64_TOC16 47 +#define R_PPC64_TOC16_LO 48 +#define R_PPC64_TOC16_HI 49 +#define R_PPC64_TOC16_HA 50 +#define R_PPC64_TOC 51 +#define R_PPC64_DTPMOD64 68 +#define R_PPC64_TPREL64 73 +#define R_PPC64_DTPREL64 78 + +/* + * TLS relocations + */ +#define R_PPC_TLS 67 +#define R_PPC_DTPMOD32 68 +#define R_PPC_TPREL16 69 +#define R_PPC_TPREL16_LO 70 +#define R_PPC_TPREL16_HI 71 +#define R_PPC_TPREL16_HA 72 +#define R_PPC_TPREL32 73 +#define R_PPC_DTPREL16 74 +#define R_PPC_DTPREL16_LO 75 +#define R_PPC_DTPREL16_HI 76 +#define R_PPC_DTPREL16_HA 77 +#define R_PPC_DTPREL32 78 +#define R_PPC_GOT_TLSGD16 79 +#define R_PPC_GOT_TLSGD16_LO 80 +#define R_PPC_GOT_TLSGD16_HI 81 +#define R_PPC_GOT_TLSGD16_HA 82 +#define R_PPC_GOT_TLSLD16 83 +#define R_PPC_GOT_TLSLD16_LO 84 +#define R_PPC_GOT_TLSLD16_HI 85 +#define R_PPC_GOT_TLSLD16_HA 86 +#define R_PPC_GOT_TPREL16 87 +#define R_PPC_GOT_TPREL16_LO 88 +#define R_PPC_GOT_TPREL16_HI 89 +#define R_PPC_GOT_TPREL16_HA 90 + +/* + * The remaining relocs are from the Embedded ELF ABI, and are not in the + * SVR4 ELF ABI. + */ + +#define R_PPC_EMB_NADDR32 101 +#define R_PPC_EMB_NADDR16 102 +#define R_PPC_EMB_NADDR16_LO 103 +#define R_PPC_EMB_NADDR16_HI 104 +#define R_PPC_EMB_NADDR16_HA 105 +#define R_PPC_EMB_SDAI16 106 +#define R_PPC_EMB_SDA2I16 107 +#define R_PPC_EMB_SDA2REL 108 +#define R_PPC_EMB_SDA21 109 +#define R_PPC_EMB_MRKREF 110 +#define R_PPC_EMB_RELSEC16 111 +#define R_PPC_EMB_RELST_LO 112 +#define R_PPC_EMB_RELST_HI 113 +#define R_PPC_EMB_RELST_HA 114 +#define R_PPC_EMB_BIT_FLD 115 +#define R_PPC_EMB_RELSDA 116 + +#define R_SPARC_NONE 0 +#define R_SPARC_8 1 +#define R_SPARC_16 2 +#define R_SPARC_32 3 +#define R_SPARC_DISP8 4 +#define R_SPARC_DISP16 5 +#define R_SPARC_DISP32 6 +#define R_SPARC_WDISP30 7 +#define R_SPARC_WDISP22 8 +#define R_SPARC_HI22 9 +#define R_SPARC_22 10 +#define R_SPARC_13 11 +#define R_SPARC_LO10 12 +#define R_SPARC_GOT10 13 +#define R_SPARC_GOT13 14 +#define R_SPARC_GOT22 15 +#define R_SPARC_PC10 16 +#define R_SPARC_PC22 17 +#define R_SPARC_WPLT30 18 +#define R_SPARC_COPY 19 +#define R_SPARC_GLOB_DAT 20 +#define R_SPARC_JMP_SLOT 21 +#define R_SPARC_RELATIVE 22 +#define R_SPARC_UA32 23 +#define R_SPARC_PLT32 24 +#define R_SPARC_HIPLT22 25 +#define R_SPARC_LOPLT10 26 +#define R_SPARC_PCPLT32 27 +#define R_SPARC_PCPLT22 28 +#define R_SPARC_PCPLT10 29 +#define R_SPARC_10 30 +#define R_SPARC_11 31 +#define R_SPARC_64 32 +#define R_SPARC_OLO10 33 +#define R_SPARC_HH22 34 +#define R_SPARC_HM10 35 +#define R_SPARC_LM22 36 +#define R_SPARC_PC_HH22 37 +#define R_SPARC_PC_HM10 38 +#define R_SPARC_PC_LM22 39 +#define R_SPARC_WDISP16 40 +#define R_SPARC_WDISP19 41 +#define R_SPARC_GLOB_JMP 42 +#define R_SPARC_7 43 +#define R_SPARC_5 44 +#define R_SPARC_6 45 +#define R_SPARC_DISP64 46 +#define R_SPARC_PLT64 47 +#define R_SPARC_HIX22 48 +#define R_SPARC_LOX10 49 +#define R_SPARC_H44 50 +#define R_SPARC_M44 51 +#define R_SPARC_L44 52 +#define R_SPARC_REGISTER 53 +#define R_SPARC_UA64 54 +#define R_SPARC_UA16 55 +#define R_SPARC_TLS_GD_HI22 56 +#define R_SPARC_TLS_GD_LO10 57 +#define R_SPARC_TLS_GD_ADD 58 +#define R_SPARC_TLS_GD_CALL 59 +#define R_SPARC_TLS_LDM_HI22 60 +#define R_SPARC_TLS_LDM_LO10 61 +#define R_SPARC_TLS_LDM_ADD 62 +#define R_SPARC_TLS_LDM_CALL 63 +#define R_SPARC_TLS_LDO_HIX22 64 +#define R_SPARC_TLS_LDO_LOX10 65 +#define R_SPARC_TLS_LDO_ADD 66 +#define R_SPARC_TLS_IE_HI22 67 +#define R_SPARC_TLS_IE_LO10 68 +#define R_SPARC_TLS_IE_LD 69 +#define R_SPARC_TLS_IE_LDX 70 +#define R_SPARC_TLS_IE_ADD 71 +#define R_SPARC_TLS_LE_HIX22 72 +#define R_SPARC_TLS_LE_LOX10 73 +#define R_SPARC_TLS_DTPMOD32 74 +#define R_SPARC_TLS_DTPMOD64 75 +#define R_SPARC_TLS_DTPOFF32 76 +#define R_SPARC_TLS_DTPOFF64 77 +#define R_SPARC_TLS_TPOFF32 78 +#define R_SPARC_TLS_TPOFF64 79 + +#define R_X86_64_NONE 0 /* No relocation. */ +#define R_X86_64_64 1 /* Add 64 bit symbol value. */ +#define R_X86_64_PC32 2 /* PC-relative 32 bit signed sym value. */ +#define R_X86_64_GOT32 3 /* PC-relative 32 bit GOT offset. */ +#define R_X86_64_PLT32 4 /* PC-relative 32 bit PLT offset. */ +#define R_X86_64_COPY 5 /* Copy data from shared object. */ +#define R_X86_64_GLOB_DAT 6 /* Set GOT entry to data address. */ +#define R_X86_64_JMP_SLOT 7 /* Set GOT entry to code address. */ +#define R_X86_64_RELATIVE 8 /* Add load address of shared object. */ +#define R_X86_64_GOTPCREL 9 /* Add 32 bit signed pcrel offset to GOT. */ +#define R_X86_64_32 10 /* Add 32 bit zero extended symbol value */ +#define R_X86_64_32S 11 /* Add 32 bit sign extended symbol value */ +#define R_X86_64_16 12 /* Add 16 bit zero extended symbol value */ +#define R_X86_64_PC16 13 /* Add 16 bit signed extended pc relative symbol value */ +#define R_X86_64_8 14 /* Add 8 bit zero extended symbol value */ +#define R_X86_64_PC8 15 /* Add 8 bit signed extended pc relative symbol value */ +#define R_X86_64_DTPMOD64 16 /* ID of module containing symbol */ +#define R_X86_64_DTPOFF64 17 /* Offset in TLS block */ +#define R_X86_64_TPOFF64 18 /* Offset in static TLS block */ +#define R_X86_64_TLSGD 19 /* PC relative offset to GD GOT entry */ +#define R_X86_64_TLSLD 20 /* PC relative offset to LD GOT entry */ +#define R_X86_64_DTPOFF32 21 /* Offset in TLS block */ +#define R_X86_64_GOTTPOFF 22 /* PC relative offset to IE GOT entry */ +#define R_X86_64_TPOFF32 23 /* Offset in static TLS block */ +#define R_X86_64_IRELATIVE 37 + + +#endif /* !_SYS_ELF_COMMON_H_ */ diff --git a/optee_os/lib/libutee/include/k3/otp_keywriting_ta.h b/optee_os/lib/libutee/include/k3/otp_keywriting_ta.h new file mode 100644 index 0000000..85b5863 --- /dev/null +++ b/optee_os/lib/libutee/include/k3/otp_keywriting_ta.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Texas Instruments System Control Interface Driver + * + * Copyright (C) 2023 Texas Instruments Incorporated - https://www.ti.com/ + * Manorit Chawdhry + */ + +#ifndef __K3_OTP_KEYWRITING_TA_H__ +#define __K3_OTP_KEYWRITING_TA_H__ + +/* UUID of the trusted application */ +#define PTA_K3_OTP_KEYWRITING_UUID \ + { 0xacc50f40, 0x0613, 0x4abd, \ + { 0x8d, 0xfe, 0xa9, 0x64, 0xcb, 0x74, 0xeb, 0x69} } + +#define PTA_K3_OTP_KEYWRITING_NAME "pta_k3_otp.ta" + +/* + * TA_OTP_KEYWRITING_CMD_READ_MMR - Read an extended OTP bit + * param[0] (value/input) 32-bit MMR index + * param[1] (value/output) OTP value written in efuse + * param[2] unused + * param[3] unused + */ +#define TA_OTP_KEYWRITING_CMD_READ_MMR 0 + +/* + * TA_OTP_KEYWRITING_CMD_WRITE_ROW - Write into extended OTP row + * param[0] (value/input) Row index + * param[1].a (value/input) Value to be written + * param[1].b (value/input) Mask for the value + * param[2] unused + * param[3] unused + */ +#define TA_OTP_KEYWRITING_CMD_WRITE_ROW 1 + +/* + * TA_OTP_KEYWRITING_CMD_LOCK_ROW - Lock an extended OTP row + * param[0].a (value/input) Row index + * param[0].b (value/input) + * BIT(0) - soft_lock + * BIT(1) - hw_read_lock + * BIT(2) - hw_write_lock + * param[1] unused + * param[2] unused + * param[3] unused + */ +#define TA_OTP_KEYWRITING_CMD_LOCK_ROW 2 + +#define K3_OTP_KEYWRITING_SOFT_LOCK BIT(0) +#define K3_OTP_KEYWRITING_HW_READ_LOCK BIT(1) +#define K3_OTP_KEYWRITING_HW_WRITE_LOCK BIT(2) + +#endif /* __K3_OTP_KEYWRITING_TA_H__ */ diff --git a/optee_os/lib/libutee/include/link.h b/optee_os/lib/libutee/include/link.h new file mode 100644 index 0000000..88d4eec --- /dev/null +++ b/optee_os/lib/libutee/include/link.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2012 The Android Open Source Project + */ + +#ifndef _LINK_H_ +#define _LINK_H_ + +#include +#include + +struct dl_phdr_info { + Elf_Addr dlpi_addr; /* module relocation base */ + const char *dlpi_name; /* module name */ + const Elf_Phdr *dlpi_phdr; /* pointer to module's phdr */ + Elf_Half dlpi_phnum; /* number of entries in phdr */ + unsigned long long dlpi_adds; /* total # of loads */ + unsigned long long dlpi_subs; /* total # of unloads */ + size_t dlpi_tls_modid; + void *dlpi_tls_data; +}; + +int dl_iterate_phdr(int (*callback)(struct dl_phdr_info *info, size_t size, + void *data), + void *data); + +#endif /* _LINK_H_ */ diff --git a/optee_os/lib/libutee/include/pta_apdu.h b/optee_os/lib/libutee/include/pta_apdu.h new file mode 100644 index 0000000..8fed78f --- /dev/null +++ b/optee_os/lib/libutee/include/pta_apdu.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (C) 2021, Foundries Limited + * Author: Jorge Ramirez + */ + +#ifndef __PTA_APDU_H +#define __PTA_APDU_H + +#define PTA_APDU_UUID { 0x3f3eb880, 0x3639, 0x11ec, \ + { 0x9b, 0x9d, 0x0f, 0x3f, 0xc9, 0x46, 0x8f, 0x50 } } + +/* + * ADPU based communication with the Secure Element + * + * [in] value[0].a Use APDU TXRX hints: PTA_APDU_TXRX_CASE_* + * [in] memref[1].buffer APDU header. + * [in] memref[1].size APDU header length. + * [in] memref[2].buffer request (APDU raw frame). + * [in] memref[2].size request length. + * [out] memref[3].buffer response (APDU raw frame). + * [out] memref[3].size response length. + * + * Result: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_NOT_IMPLEMENTED - Invoke command not implemented + * TEE_ERROR_GENERIC - Invoke command failure + */ +#define PTA_CMD_TXRX_APDU_RAW_FRAME 0 + +/* + * Type identifier for the APDU message as described by Smart Card Standard + * ISO7816-4 about ADPU message bodies decoding convention: + * + * https://cardwerk.com/smart-card-standard-iso7816-4-section-5-basic-organizations/#chap5_3_2 + */ +#define PTA_APDU_TXRX_CASE_NO_HINT 0 +#define PTA_APDU_TXRX_CASE_1 1 +#define PTA_APDU_TXRX_CASE_2 2 +#define PTA_APDU_TXRX_CASE_2E 3 +#define PTA_APDU_TXRX_CASE_3 4 +#define PTA_APDU_TXRX_CASE_3E 5 +#define PTA_APDU_TXRX_CASE_4 6 +#define PTA_APDU_TXRX_CASE_4E 7 + +#endif /* __PTA_APDU_H */ diff --git a/optee_os/lib/libutee/include/pta_attestation.h b/optee_os/lib/libutee/include/pta_attestation.h new file mode 100644 index 0000000..76a1d3a --- /dev/null +++ b/optee_os/lib/libutee/include/pta_attestation.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2021, Huawei Technologies Co., Ltd + */ + +/* + * Provide remote attestation services + */ + +#ifndef __PTA_ATTESTATION_H +#define __PTA_ATTESTATION_H + +#define PTA_ATTESTATION_UUID { 0x39800861, 0x182a, 0x4720, \ + { 0x9b, 0x67, 0x2b, 0xcd, 0x62, 0x2b, 0xc0, 0xb5 } } + +/* + * Get the RSA public key that should be used to verify the values returned by + * other commands. + * + * [out] memref[0] Public key exponent in big endian order + * [out] memref[1] Modulus in big endian order + * [out] value[2] Signature algorithm used by other commands. + * Currently always + * TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256. + * + * Return codes: + * TEE_SUCCESS + * TEE_ERROR_GENERIC - Internal error + * TEE_ERROR_SHORT_BUFFER - One or both buffers are too small, required size + * is provided in memref[i].size + */ +#define PTA_ATTESTATION_GET_PUBKEY 0x0 + +/* + * Return the digest found in the header of a Trusted Application binary or a + * Trusted Shared library + * + * [in] memref[0] UUID of the TA or shared library + * [in] memref[1] Nonce (random non-NULL, non-empty buffer of any + * size to prevent replay attacks) + * [out] memref[2] Output buffer. Receives the signed digest. + * - The first 32 bytes are the digest itself (from + * the TA signed header: struct shdr::hash) + * - The following bytes are a signature: + * SIG(SHA256(Nonce | digest)) + * - The algorithm is + * TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256 with a salt + * length of 32. + * - The key pair is generated internally and stored + * in secure storage. The public key can be + * retrieved with command PTA_ATTESTATION_GET_PUBKEY + * (typically during device provisioning). + * Given that the sigature length is equal to the + * RSA modulus size in bytes, the output buffer size + * should be at least (digest size + modulus size) + * bytes. For example, for a 32-byte SHA256 digest and + * 2048 bit key (256 bytes) the minimum buffer size is + * 288 bytes. + * + * Return codes: + * TEE_SUCCESS + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_SHORT_BUFFER - Output buffer size less than required + */ +#define PTA_ATTESTATION_GET_TA_SHDR_DIGEST 0x1 + +/* + * Return a signed hash for a running user space TA, which must be the caller + * of this PTA. It is a runtime measurement of the memory pages that contain + * immutable data (code and read-only data). + * + * [in] memref[0] Nonce + * [out] memref[1] SHA256 hash of the TA memory followed by a + * signature. See PTA_ATTESTATION_GET_TA_HDR_DIGEST + * for a description of the signature. + * + * Return codes: + * TEE_SUCCESS + * TEE_ERROR_ACCESS_DENIED - Caller is not a user space TA + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_SHORT_BUFFER - Output buffer size less than required + */ +#define PTA_ATTESTATION_HASH_TA_MEMORY 0x2 + +/* + * Return a signed hash of the TEE OS (kernel) memory. It is a runtime + * measurement of the memory pages that contain immutable data (code and + * read-only data). + * + * [in] memref[0] Nonce + * [out] memref[1] SHA256 hash of the TEE memory followed by a + * signature. See PTA_ATTESTATION_GET_TA_HDR_DIGEST + * for a description of the signature. + * + * Return codes: + * TEE_SUCCESS + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_SHORT_BUFFER - Output buffer size less than required + */ +#define PTA_ATTESTATION_HASH_TEE_MEMORY 0x3 + +#endif /* __PTA_ATTESTATION_H */ diff --git a/optee_os/lib/libutee/include/pta_benchmark.h b/optee_os/lib/libutee/include/pta_benchmark.h new file mode 100644 index 0000000..2b6f157 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_benchmark.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __PTA_BENCHMARK_H +#define __PTA_BENCHMARK_H + +/* + * Interface to the benchmark pseudo-TA, which is used for registering + * timestamp buffers + */ + +#define BENCHMARK_UUID \ + { 0x0b9a63b0, 0xb4c6, 0x4c85, \ + { 0xa2, 0x84, 0xa2, 0x28, 0xef, 0x54, 0x7b, 0x4e } } + +/* + * Benchmark PTA supported commands + */ +#define BENCHMARK_CMD(id) (0xFA190000 | ((id) & 0xFFFF)) +#define BENCHMARK_CMD_ALLOCATE_BUF BENCHMARK_CMD(1) +#define BENCHMARK_CMD_GET_MEMREF BENCHMARK_CMD(2) +#define BENCHMARK_CMD_UNREGISTER BENCHMARK_CMD(3) + +#endif /* __PTA_BENCHMARK_H */ diff --git a/optee_os/lib/libutee/include/pta_device.h b/optee_os/lib/libutee/include/pta_device.h new file mode 100644 index 0000000..2cd2bcd --- /dev/null +++ b/optee_os/lib/libutee/include/pta_device.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Linaro Limited + */ + +/* + * Enumerate the pseudo TAs that have the TA_FLAG_DEVICE_ENUM flag enabled. + */ + +#ifndef __PTA_DEVICE_H +#define __PTA_DEVICE_H + +#define PTA_DEVICE_UUID { 0x7011a688, 0xddde, 0x4053, \ + { 0xa5, 0xa9, 0x7b, 0x3c, 0x4d, 0xdf, 0x13, 0xb8 } } + +/* + * Get device UUIDs + * + * [out] memref[0] Array of device UUIDs + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_SHORT_BUFFER - Output buffer size less than required + */ +#define PTA_CMD_GET_DEVICES 0x0 /* before tee-supplicant run */ +#define PTA_CMD_GET_DEVICES_SUPP 0x1 /* after tee-supplicant run */ + +#endif /* __PTA_DEVICE_H */ diff --git a/optee_os/lib/libutee/include/pta_gprof.h b/optee_os/lib/libutee/include/pta_gprof.h new file mode 100644 index 0000000..60e9c8f --- /dev/null +++ b/optee_os/lib/libutee/include/pta_gprof.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef __PTA_GPROF_H +#define __PTA_GPROF_H + +/* + * Interface to the gprof pseudo-TA, which is used by libutee to control TA + * profiling and forward data to tee-supplicant. + */ + +#define PTA_GPROF_UUID { 0x2f6e0d48, 0xc574, 0x426d, { \ + 0x82, 0x4e, 0x40, 0x19, 0x8c, 0xde, 0x5c, 0xac } } + +/* + * Send TA profiling data (gmon.out format) to tee-supplicant + * Data may be sent in several chunks: first set id to 0, then re-use the + * allocated value in subsequent calls. + * + * [in/out] value[0].a: id + * [in] memref[1]: profiling data + */ +#define PTA_GPROF_SEND 0 + +/* + * Start PC sampling of a user TA session + * + * [in/out] memref[0]: sampling buffer + * [in] value[1].a: offset: the lowest PC value in the TA + * [in] value[1].b: scale: histogram scaling factor + */ +#define PTA_GPROF_START_PC_SAMPLING 1 + +/* + * Stop PC sampling of a user TA session and retrieve data + * + * [out] value[0].a: sampling frequency + */ +#define PTA_GPROF_STOP_PC_SAMPLING 2 + +#endif /* __PTA_GPROF_H */ diff --git a/optee_os/lib/libutee/include/pta_imx_dek_blob.h b/optee_os/lib/libutee/include/pta_imx_dek_blob.h new file mode 100644 index 0000000..afea562 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_imx_dek_blob.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2019 NXP + * + * brief PTA DEK Blob interface identification. + */ +#ifndef __PTA_IMX_DEK_BLOB_H__ +#define __PTA_IMX_DEK_BLOB_H__ + +#define PTA_DEK_BLOB_UUID {0xef477737, 0x0db1, 0x4a9d, \ + {0x84, 0x37, 0xf2, 0xf5, 0x35, 0xc0, 0xbd, 0x92} } + +/* HAB DEK Blob encapsulation */ +#define PTA_IMX_DEK_BLOB_CMD_GENERATE 0 + +#endif /* __PTA_IMX_DEK_BLOB_H__ */ diff --git a/optee_os/lib/libutee/include/pta_imx_digprog.h b/optee_os/lib/libutee/include/pta_imx_digprog.h new file mode 100644 index 0000000..bb00400 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_imx_digprog.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + */ +#ifndef __PTA_IMX_DIGPROG_H__ +#define __PTA_IMX_DIGPROG_H__ + +#define PTA_DIGPROG_UUID { 0xbfdbe409, 0x3302, 0x4abf, \ + { 0x99, 0xbc, 0x5f, 0x62, 0x32, 0x26, 0xa7, 0x39 } } + +#endif /* __PTA_IMX_DIGPROG_H__ */ diff --git a/optee_os/lib/libutee/include/pta_imx_manufacturing_protection.h b/optee_os/lib/libutee/include/pta_imx_manufacturing_protection.h new file mode 100644 index 0000000..fe4c274 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_imx_manufacturing_protection.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2018-2019, 2023 NXP + */ +#ifndef PTA_IMX_MANUFACTURING_PROTECTION_H +#define PTA_IMX_MANUFACTURING_PROTECTION_H + +#define PTA_MANUFACT_PROTEC_UUID { 0x83268b7c, 0x85e3, 0x11e8, \ + { 0xad, 0xc0, 0xfa, 0x7a, 0xe0, 0x1b, 0xbe, 0xbc} } + +/* + * Sign the given message with the manufacturing protection private key + * + * [in] memref[0].buffer Message buffer + * [in] memref[0].size Message size + * [out] memref[1].buffer Signature buffer + * [out] memref[1].size Signature size + * [out] memref[2].buffer MPMR buffer + * [out] memref[2].size MPMR size + */ +#define PTA_IMX_MP_CMD_SIGNATURE_MPMR 0 + +/* + * Get the manufacturing protection public key + * + * [out] memref[0].buffer Public key buffer + * [out] memref[0].size Public key size + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_IMX_MP_CMD_GET_PUBLIC_KEY 1 + +#endif /* PTA_IMX_MANUFACTURING_PROTECTION_H */ diff --git a/optee_os/lib/libutee/include/pta_imx_ocotp.h b/optee_os/lib/libutee/include/pta_imx_ocotp.h new file mode 100644 index 0000000..d54e2e8 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_imx_ocotp.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2021 NXP + */ +#ifndef __PTA_IMX_OCOTP_H__ +#define __PTA_IMX_OCOTP_H__ + +#define PTA_OCOTP_UUID { 0x9abdf255, 0xd8fa, 0x40de, \ + { 0x8f, 0x60, 0x4d, 0x0b, 0x27, 0x92, 0x7b, 0x7d } } + +/** + * Read chip UID + * + * [out] memref[0].buffer Output buffer to store UID + * [out] memref[0].size Size of the UID (64 bits) + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input parameter + * TEE_ERROR_GENERIC - UID not available + */ +#define PTA_OCOTP_CMD_CHIP_UID 0 + +/* + * Read chip OTP fuse + * + * [in] params[0].value.a Fuse bank number + * [in] params[0].value.b Fuse word number + * [out] params[1].value.a Fuse value + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input parameter + * TEE_ERROR_BUSY - OCOTP not available + */ +#define PTA_OCOTP_CMD_READ_FUSE 1 +#endif /* __PTA_IMX_OCOTP_H__ */ diff --git a/optee_os/lib/libutee/include/pta_invoke_tests.h b/optee_os/lib/libutee/include/pta_invoke_tests.h new file mode 100644 index 0000000..79cd989 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_invoke_tests.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __PTA_INVOKE_TESTS_H +#define __PTA_INVOKE_TESTS_H + +#define PTA_INVOKE_TESTS_UUID \ + { 0xd96a5b40, 0xc3e5, 0x21e3, \ + { 0x87, 0x94, 0x10, 0x02, 0xa5, 0xd5, 0xc6, 0x1b } } + +/* Trace some hello string. Parameters are not used/checked. */ +#define PTA_INVOKE_TESTS_CMD_TRACE 0 + +/* + * Types of parameter drives the test sequences: + * - test on value parameters + * - test on SHM memory reference parameters + * - test on SDP memory reference parameters + */ +#define PTA_INVOKE_TESTS_CMD_PARAMS 1 + +/* Run some core internal tests. Parameters are not used/checked. */ +#define PTA_INVOKE_TESTS_CMD_SELF_TESTS 2 + +/* + * Secure data path: check that PTA can copy data from non-secure shared memory + * to SDP secure memory + * + * [in] memref[0] source (non-secure shared memory) + * [out] memref[1] destination (SDP secure memory) + */ +#define PTA_INVOKE_TESTS_CMD_COPY_NSEC_TO_SEC 3 + +/* + * Secure data path: check that PTA can read data from SDP secure memory and + * write it back. Data are processed so that client check the expected + * read/write sequence succeed. + * + * [in/out] memref[0] SDP secure buffer to read from and write to + */ +#define PTA_INVOKE_TESTS_CMD_READ_MODIFY_SEC 4 + +/* + * Secure data path: check that PTA can copy data from SDP secure memory to + * non-secure shared memory + * + * [in] memref[0] source (SDP secure memory) + * [out] memref[1] destination (non-secure shared memory) + */ +#define PTA_INVOKE_TESTS_CMD_COPY_SEC_TO_NSEC 5 + +/* + * Tests FS hash-tree corner cases in error handling + */ +#define PTA_INVOKE_TESTS_CMD_FS_HTREE 6 + +/* + * Tests mutex + * + * [in] value[0].a Test function PTA_MUTEX_TEST_* + * [in] value[0].b delay number + * [out] value[1].a before lock concurency + * [out] value[1].b during lock concurency + */ +#define PTA_MUTEX_TEST_WRITER 0 +#define PTA_MUTEX_TEST_READER 1 +#define PTA_INVOKE_TESTS_CMD_MUTEX 7 + +/* + * Tests lock dependency checking algorithm + */ +#define PTA_INVOKE_TESTS_CMD_LOCKDEP 8 + +/* + * These values should match the ones in + * optee_test/ta/aes_perf/include/ta_aes_perf.h + */ +#define PTA_INVOKE_TESTS_AES_ECB 0 +#define PTA_INVOKE_TESTS_AES_CBC 1 +#define PTA_INVOKE_TESTS_AES_CTR 2 +#define PTA_INVOKE_TESTS_AES_XTS 3 +#define PTA_INVOKE_TESTS_AES_GCM 4 + +/* + * AES performance tests + * + * [in] value[0].a Top 16 bits Decrypt, low 16 bits key size in bits + * [in] value[0].b AES mode, one of + * PTA_INVOKE_TESTS_AES_{ECB_NOPAD,CBC_NOPAD,CTR,XTS,GCM} + * [in] value[1].a repetition count + * [in] value[1].b unit size + * [in] memref[2] In buffer + * [in] memref[3] Out buffer + */ +#define PTA_INVOKE_TEST_CMD_AES_PERF 9 + +/* + * NULL memory reference parameter + * + * [in/out] memref[0] NULL memory reference of size zero + */ +#define PTA_INVOKE_TESTS_CMD_MEMREF_NULL 10 + +/* + * Retrieve results of the dt_driver framework internal test + */ +#define PTA_INVOKE_TESTS_CMD_DT_DRIVER_TESTS 11 + +#endif /*__PTA_INVOKE_TESTS_H*/ + diff --git a/optee_os/lib/libutee/include/pta_rng.h b/optee_os/lib/libutee/include/pta_rng.h new file mode 100644 index 0000000..5f9264c --- /dev/null +++ b/optee_os/lib/libutee/include/pta_rng.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2018, Linaro Limited + */ + +#ifndef __PTA_RNG_H +#define __PTA_RNG_H + +#define PTA_RNG_UUID { 0xab7a617c, 0xb8e7, 0x4d8f, \ + { 0x83, 0x01, 0xd0, 0x9b, 0x61, 0x03, 0x6b, 0x64 } } + +#define TEE_ERROR_HEALTH_TEST_FAIL 0x00000001 + +/* + * PTA_CMD_GET_ENTROPY - Get Entropy from RNG using Thermal Sensor + * + * [in/out] memref[0] - Entropy buffer memory reference + * param[1] unused + * param[2] unused + * param[3] unused + * + * Result: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_NOT_SUPPORTED - Requested entropy size greater than size of pool + * TEE_ERROR_HEALTH_TEST_FAIL - Continuous health testing failed + */ +#define PTA_CMD_GET_ENTROPY 0x0 + +/* + * PTA_CMD_GET_RNG_INFO - Get RNG information + * + * [out] value[0].a - RNG data-rate in bytes per second + * [out] value[0].b - Quality/Entropy per 1024 bit of data + * param[1] unused + * param[2] unused + * param[3] unused + * + * Result: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_CMD_GET_RNG_INFO 0x1 + +#endif /* __PTA_RNG_H */ diff --git a/optee_os/lib/libutee/include/pta_rtc.h b/optee_os/lib/libutee/include/pta_rtc.h new file mode 100644 index 0000000..ae14c2e --- /dev/null +++ b/optee_os/lib/libutee/include/pta_rtc.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022, Microchip + */ +#ifndef __PTA_RTC_H +#define __PTA_RTC_H + +#include + +#define PTA_RTC_UUID { 0xf389f8c8, 0x845f, 0x496c, \ + { 0x8b, 0xbe, 0xd6, 0x4b, 0xd2, 0x4c, 0x92, 0xfd } } + +#define PTA_RTC_INFO_VERSION 0x1 + +/* + * RTC provides set/get offset and thus command PTA_CMD_RTC_GET_OFFSET and + * PTA_CMD_RTC_SET_OFFSET might be called + */ +#define PTA_RTC_FEATURE_CORRECTION BIT(0) + +struct pta_rtc_time { + uint32_t tm_sec; + uint32_t tm_min; + uint32_t tm_hour; + uint32_t tm_mday; + uint32_t tm_mon; + uint32_t tm_year; + uint32_t tm_wday; +}; + +/* + * struct pta_rtc_info - RTC service information + * @version - 1st 64bit cell, version of the structure: PTA_RTC_INFO_VERSION + * @features - 64bit flag mask related to PTA_RTC_FEATURE_* + * @range_min - Minima time reference the RTC can be programmed to + * @range_max - Maxima time reference the RTC can reach + */ +struct pta_rtc_info { + uint64_t version; + uint64_t features; + struct pta_rtc_time range_min; + struct pta_rtc_time range_max; +}; + +/* + * PTA_CMD_RTC_GET_INFO - Get RTC information + * + * [out] memref[0] RTC buffer memory reference containing a struct + * pta_rtc_info + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_CMD_RTC_GET_INFO 0x0 + +/* + * PTA_CMD_RTC_GET_TIME - Get time from RTC + * + * [out] memref[0] RTC buffer memory reference containing a struct + * pta_rtc_time + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_CMD_RTC_GET_TIME 0x1 + +/* + * PTA_CMD_RTC_SET_TIME - Set time from RTC + * + * [in] memref[0] RTC buffer memory reference containing a struct + * pta_rtc_time to be used as RTC time + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_CMD_RTC_SET_TIME 0x2 + +/* + * PTA_CMD_RTC_GET_OFFSET - Get RTC offset + * + * [out] value[0].a RTC offset (signed 32bit value) + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_CMD_RTC_GET_OFFSET 0x3 + +/* + * PTA_CMD_RTC_SET_OFFSET - Set RTC offset + * + * [in] value[0].a RTC offset to be set (signed 32bit value) + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_CMD_RTC_SET_OFFSET 0x4 + +#endif /* __PTA_RTC_H */ diff --git a/optee_os/lib/libutee/include/pta_scmi_client.h b/optee_os/lib/libutee/include/pta_scmi_client.h new file mode 100644 index 0000000..3e7b931 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_scmi_client.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019-2021, Linaro Limited + */ +#ifndef PTA_SCMI_CLIENT_H +#define PTA_SCMI_CLIENT_H + +#define PTA_SCMI_UUID { 0xa8cfe406, 0xd4f5, 0x4a2e, \ + { 0x9f, 0x8d, 0xa2, 0x5d, 0xc7, 0x54, 0xc0, 0x99 } } + +#define PTA_SCMI_NAME "PTA-SCMI" + +/* + * PTA_SCMI_CMD_CAPABILITIES - Get channel capabilities + * + * [out] value[0].a: Capabilities bit mask (PTA_SCMI_CAPS_*) + * [out] value[0].b: Extended capabilities or 0 + */ +#define PTA_SCMI_CMD_CAPABILITIES 0 + +/* + * PTA_SCMI_CMD_PROCESS_SMT_CHANNEL - Process SCMI message in SMT buffer + * + * [in] value[0].a: Channel handle + * + * Shared memory used for SCMI message/response exhange is expected + * already identified and bound to channel handle in both SCMI agent + * and SCMI server (OP-TEE) parts. + * The memory uses SMT header to carry SCMI meta-data (protocol ID and + * protocol message ID). + */ +#define PTA_SCMI_CMD_PROCESS_SMT_CHANNEL 1 + +/* + * PTA_SCMI_CMD_PROCESS_SMT_CHANNEL_MESSAGE - Process SCMI message in + * SMT buffer pointed by memref parameters + * + * [in] value[0].a: Channel handle + * [in/out] memref[1]: Message/response buffer (SMT and SCMI payload) + * + * Shared memory used for SCMI message/response is a SMT buffer + * referenced by param[1]. It shall be 128 bytes large to fit response + * payload whatever message playload size. + * The memory uses SMT header to carry SCMI meta-data (protocol ID and + * protocol message ID). + */ +#define PTA_SCMI_CMD_PROCESS_SMT_CHANNEL_MESSAGE 2 + +/* + * PTA_SCMI_CMD_GET_CHANNEL_HANDLE - Get handle for an SCMI channel + * + * Get a handle for the SCMI channel. This handle value is to be passed + * as argument to some commands as PTA_SCMI_CMD_PROCESS_*. + * + * [in] value[0].a: Channel identifier or 0 if no assigned ID + * [in] value[0].b: Requested capabilities mask (PTA_SCMI_CAPS_*) + * [out] value[0].a: Returned channel handle + */ +#define PTA_SCMI_CMD_GET_CHANNEL_HANDLE 3 + +/* + * PTA_SCMI_CMD_PROCESS_MSG_CHANNEL - Process SCMI message in a MSG + * buffer pointed by memref parameters + * + * [in] value[0].a: Channel handle + * [in] memref[1]: Input message shared buffer + * [out] memref[2]: Output message shared buffer + */ +#define PTA_SCMI_CMD_PROCESS_MSG_CHANNEL 4 + +/* + * Capabilities + */ + +/* Channel supports shared memory using the SMT header protocol */ +#define PTA_SCMI_CAPS_SMT_HEADER BIT32(0) + +/* Channel supports shared memory using the MSG header protocol */ +#define PTA_SCMI_CAPS_MSG_HEADER BIT32(1) + +/* Mask of defined capabilities */ +#define PTA_SCMI_CAPS_MASK GENMASK_32(1, 0) + +#endif /* SCMI_PTA_SCMI_CLIENT_H */ diff --git a/optee_os/lib/libutee/include/pta_scp03.h b/optee_os/lib/libutee/include/pta_scp03.h new file mode 100644 index 0000000..393ca56 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_scp03.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (C) 2020, Foundries Limited + */ + +#ifndef __PTA_SCP03_H +#define __PTA_SCP03_H + +#define PTA_SCP03_UUID { 0xbe0e5821, 0xe718, 0x4f77, \ + { 0xab, 0x3e, 0x8e, 0x6c, 0x73, 0xa9, 0xc7, 0x35 } } + +/* + * Enable SCP03 support on the SE + * + * [in] value[0].a Use session keys PTA_SCP03_SESSION_* + * + * Result: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_NOT_IMPLEMENTED - Invoke command not implemented + * TEE_ERROR_GENERIC - Invoke command failure + */ +#define PTA_CMD_ENABLE_SCP03 0 + +/* Enable the session using the current keys in the Secure Element */ +#define PTA_SCP03_SESSION_CURRENT_KEYS 0 + +/* Enable the session after replacing the current keys in the Secure Element */ +#define PTA_SCP03_SESSION_ROTATE_KEYS 1 + +#endif /* __PTA_SCP03_H */ diff --git a/optee_os/lib/libutee/include/pta_secstor_ta_mgmt.h b/optee_os/lib/libutee/include/pta_secstor_ta_mgmt.h new file mode 100644 index 0000000..47c74b0 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_secstor_ta_mgmt.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017, Linaro Limited + */ + +#ifndef __PTA_SECSTOR_TA_MGMT_H +#define __PTA_SECSTOR_TA_MGMT_H + +/* + * Bootstrap (install initial) Trusted Application or Secure Domain into + * secure storage from a signed binary. + * + * [in] memref[0]: signed binary + */ +#define PTA_SECSTOR_TA_MGMT_BOOTSTRAP 0 + +#define PTA_SECSTOR_TA_MGMT_UUID { 0x6e256cba, 0xfc4d, 0x4941, { \ + 0xad, 0x09, 0x2c, 0xa1, 0x86, 0x03, 0x42, \ + 0xdd } } + +#endif /*__PTA_SECSTOR_TA_MGMT_H*/ diff --git a/optee_os/lib/libutee/include/pta_socket.h b/optee_os/lib/libutee/include/pta_socket.h new file mode 100644 index 0000000..ad5f7d7 --- /dev/null +++ b/optee_os/lib/libutee/include/pta_socket.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#ifndef __PTA_SOCKET +#define __PTA_SOCKET + +#define PTA_SOCKET_UUID { 0x3b996a7d, 0x2c2b, 0x4a49, { \ + 0xa8, 0x96, 0xe1, 0xfb, 0x57, 0x66, 0xd2, 0xf4 } } + +/* + * [in] value[0].a ip version TEE_IP_VERSION_* from tee_ipsocket.h + * [in] value[0].b server port number + * [in] memref[1] server address + * [in] value[2].a protocol, TEE_ISOCKET_PROTOCOLID_* + * [out] value[3].a socket handle + */ +#define PTA_SOCKET_OPEN 1 + +/* + * [in] value[0].a socket handle + */ +#define PTA_SOCKET_CLOSE 2 + +#define PTA_SOCKET_TIMEOUT_NONBLOCKING 0 +#define PTA_SOCKET_TIMEOUT_BLOCKING 0xffffffff + +/* + * [in] value[0].a socket handle + * [in] value[0].b timeout ms or TEE_TIMEOUT_INFINITE + * [in] memref[1] buffer to transmit + * [out] value[2].a number of transmitted bytes + */ +#define PTA_SOCKET_SEND 3 + +/* + * [in] value[0].a socket handle + * [in] value[0].b timeout ms or TEE_TIMEOUT_INFINITE + * [out] memref[1] buffer + */ +#define PTA_SOCKET_RECV 4 + +/* + * [in] value[0].a socket handle + * [in] value[0].b ioctl command + * [in/out] memref[1] buffer + */ +#define PTA_SOCKET_IOCTL 5 + +#endif /*__PTA_SOCKET*/ diff --git a/optee_os/lib/libutee/include/pta_stm32mp_bsec.h b/optee_os/lib/libutee/include/pta_stm32mp_bsec.h new file mode 100644 index 0000000..d16fadf --- /dev/null +++ b/optee_os/lib/libutee/include/pta_stm32mp_bsec.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022, STMicroelectronics - All Rights Reserved + */ + +#ifndef __PTA_STM32MP_BSEC_H +#define __PTA_STM32MP_BSEC_H + +#define PTA_BSEC_UUID { 0x94cf71ad, 0x80e6, 0x40b5, \ + { 0xa7, 0xc6, 0x3d, 0xc5, 0x01, 0xeb, 0x28, 0x03 } } + +/** + * Read OTP memory + * + * [in] value[0].a OTP start offset in byte + * [in] value[0].b Access type, see PTA_BSEC_TYPE_* + * [out] memref[1].buffer Output buffer to store read values + * [out] memref[1].size Size of OTP to be read + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_ACCESS_DENIED - OTP not accessible by caller + */ +#define PTA_BSEC_CMD_READ_OTP 0x0 + +/** + * Write OTP memory + * + * [in] value[0].a OTP start offset in byte + * [in] value[0].b Access type (0 : shadow, + * 1 : fuse, 2 : lock) + * [in] memref[1].buffer Input buffer to read values + * [in] memref[1].size Size of OTP to be written + * + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + * TEE_ERROR_ACCESS_DENIED - OTP not accessible by caller + */ +#define PTA_BSEC_CMD_WRITE_OTP 0x1 + +/** + * Get BSEC state + * Return the chip security level by reading the BSEC state + * + * [out] value[0].a One of PTA_BSEC_STATE_* + * Return codes: + * TEE_SUCCESS - Invoke command success + * TEE_ERROR_BAD_PARAMETERS - Incorrect input param + */ +#define PTA_BSEC_CMD_GET_STATE 0x3 + +enum stm32_bsec_pta_sec_state { + PTA_BSEC_STATE_SEC_OPEN = 0, + PTA_BSEC_STATE_SEC_CLOSE = 1, + PTA_BSEC_STATE_INVALID = 3 +}; + +/* + * Access types identifiers for PTA_BSEC_CMD_READ_OTP and + * PTA_BSEC_CMD_WRITE_OTP = value[in].b. + * + * PTA_BSEC_SHADOW_ACCESS Access OTP shadow memory + * PTA_BSEC_FUSE_ACCESS Access OTP fuse memory + * PTA_BSEC_LOCKS_ACCESS Access OTP locks. The locks value read/written + * in memref[1] 32bit words are related to bit flag + * masks PTA_BSEC_LOCK_*. + */ +#define PTA_BSEC_SHADOW_ACCESS 0 +#define PTA_BSEC_FUSE_ACCESS 1 +#define PTA_BSEC_LOCKS_ACCESS 2 + +/* + * PTA_BSEC_LOCK_* - Bit mask of OTP locks in memref[1] + * + * PTA_BSEC_LOCK_PERM Fuse programming permanent lock + * PTA_BSEC_LOCK_SHADOW_R Shadow programming (from fuse) lock + * PTA_BSEC_LOCK_SHADOW_W Shadow memory write lock + * PTA_BSEC_LOCK_SHADOW_P Fuse programming sticky lock + * PTA_BSEC_LOCK_ERROR Flag indicating an error in lock access + */ +#define PTA_BSEC_LOCK_PERM BIT(30) +#define PTA_BSEC_LOCK_SHADOW_R BIT(29) +#define PTA_BSEC_LOCK_SHADOW_W BIT(28) +#define PTA_BSEC_LOCK_SHADOW_P BIT(27) +#define PTA_BSEC_LOCK_ERROR BIT(26) + +#endif /* __PTA_STM32MP_BSEC_H */ diff --git a/optee_os/lib/libutee/include/pta_system.h b/optee_os/lib/libutee/include/pta_system.h new file mode 100644 index 0000000..a23f60b --- /dev/null +++ b/optee_os/lib/libutee/include/pta_system.h @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2019, Linaro Limited + * Copyright (c) 2020, Open Mobile Platform LLC + */ +#ifndef __PTA_SYSTEM_H +#define __PTA_SYSTEM_H + +#include + +/* + * Interface to the pseudo TA, which is provides misc. auxiliary services, + * extending existing GlobalPlatform Core API + */ + +#define PTA_SYSTEM_UUID { 0x3a2f8978, 0x5dc0, 0x11e8, { \ + 0x9c, 0x2d, 0xfa, 0x7a, 0xe0, 0x1b, 0xbe, 0xbc } } + +/* + * Having keys with too few bits impose a potential security risk, hence set a + * lower bound of 128 bits. + */ +#define TA_DERIVED_KEY_MIN_SIZE 16 + +/* Same value as max in huk_subkey_derive */ +#define TA_DERIVED_KEY_MAX_SIZE 32 + +#define TA_DERIVED_EXTRA_DATA_MAX_SIZE 1024 + +/* + * Add (re-seed) caller-provided entropy to the RNG pool. Keymaster + * implementations need to securely mix the provided entropy into their pool, + * which also must contain internally-generated entropy from a hardware random + * number generator. + * + * [in] memref[0]: entropy input data + */ +#define PTA_SYSTEM_ADD_RNG_ENTROPY 0 + +/* + * Derives a device and TA unique key. The caller can also provide extra data + * that will be mixed together with existing device unique properties. If no + * extra data is provided, then the derived key will only use device unique + * properties and caller TA UUID. + * + * [in] params[0].memref.buffer Buffer for extra data + * [in] params[0].memref.size Size of extra data (max 1024 bytes) + * [out] params[1].memref.buffer Buffer for the derived key + * [out] params[1].memref.size Size of the derived key (16 to 32 bytes) + */ +#define PTA_SYSTEM_DERIVE_TA_UNIQUE_KEY 1 + +/* Memory can be shared with other TAs */ +#define PTA_SYSTEM_MAP_FLAG_SHAREABLE BIT32(0) +/* Read/write memory */ +#define PTA_SYSTEM_MAP_FLAG_WRITEABLE BIT32(1) +/* Executable memory */ +#define PTA_SYSTEM_MAP_FLAG_EXECUTABLE BIT32(2) + +/* + * Map zero initialized memory + * + * [in] value[0].a: Number of bytes + * [in] value[0].b: Flags, 0 or PTA_SYSTEM_MAP_FLAG_SHAREABLE + * [out] value[1].a: Address upper 32-bits + * [out] value[1].b: Address lower 32-bits + * [in] value[2].a: Extra pad before memory range + * [in] value[2].b: Extra pad after memory range + */ +#define PTA_SYSTEM_MAP_ZI 2 + +/* + * Unmap memory + * + * [in] value[0].a: Number of bytes + * [in] value[0].b: Must be 0 + * [in] value[1].a: Address upper 32-bits + * [in] value[1].b: Address lower 32-bits + */ +#define PTA_SYSTEM_UNMAP 3 + +/* + * Find and opens an TA binary and return a handle + * + * [in] memref[0]: UUID of TA binary + * [out] value[1].a: Handle to TA binary + * [out] value[1].b: 0 + */ +#define PTA_SYSTEM_OPEN_TA_BINARY 4 + +/* + * Close an TA binary handle + * + * When a TA is done mapping new parts of an TA binary it closes the handle + * to free resources, established mappings remains. + * + * [in] value[1].a: Handle to TA binary + * [in] value[1].b: Must be 0 + * + * Returns TEE_SUCCESS if the TA binary was verified successfully. + */ +#define PTA_SYSTEM_CLOSE_TA_BINARY 5 + +/* + * Map segment of TA binary + * + * Different parts of an TA binary file needs different permissions. + * Read-write mapped parts are private to the TA, while read-only (which + * includes execute) mapped parts are shared with other TAs. This is + * transparent to the TA. If the supplied address in value[3] is 0 a + * suitable address is selected, else it will either be mapped at that + * address of an error is returned. + * + * [in] value[0].a: Handle to TA binary + * [in] value[0].b: Flags, PTA_SYSTEM_MAP_FLAG_* + * [in] value[1].a: Offset into TA binary, must be page aligned + * [in] value[1].b: Number of bytes, the last page will be zero + * extended if not page aligned + * [in/out] value[2].a: Address upper 32-bits + * [in/out] value[2].b: Address lower 32-bits + * [in] value[3].a: Extra pad before memory range + * [in] value[3].b: Extra pad after memory range + */ +#define PTA_SYSTEM_MAP_TA_BINARY 6 + +/* + * Copy a memory range from TA binary + * + * [in] value[0].a: Handle to TA binary + * [in] value[0].b: Offset into TA binary + * [out] memref[1]: Destination + */ +#define PTA_SYSTEM_COPY_FROM_TA_BINARY 7 + +/* + * Set memory protection + * + * [in] value[0].a: Number of bytes + * [in] value[0].b: Flags, PTA_SYSTEM_MAP_FLAG_* + * [in] value[1].a: Address upper 32-bits + * [in] value[1].b: Address lower 32-bits + */ +#define PTA_SYSTEM_SET_PROT 8 + +/* + * Remap a segment of a TA mapping + * + * Moves an already mapped segment of a TA to a new address. If the + * supplied new address is 0 a suitable address is selected, else it will + * either be mapped at that address or an error is returned. + * + * [in] value[0].a: Number of bytes, must match length rounded up to + * closest page of original mapping + * [in] value[0].b: Must be 0 + * [in] value[1].a: Old address upper 32-bits + * [in] value[1].b: Old address lower 32-bits + * [in/out] value[2].a: New address upper 32-bits + * [in/out] value[2].b: New address lower 32-bits + * [in] value[3].a: Extra pad before memory range + * [in] value[3].b: Extra pad after memory range + */ +#define PTA_SYSTEM_REMAP 9 + +/* + * Load a shared library + * + * [in] memref[0]: the UUID of the shared library (@filename) + * [in] value[1].a: @flags, must be (RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE) + * + * Used by: (libdl) dlopen(const char *filename, int flags) + */ +#define PTA_SYSTEM_DLOPEN 10 + +/* + * Resolve a symbol in a previously loaded shared library or in the whole TA + * + * [in] memref[0]: the UUID of the shared library, or the nil UUID to + * search the whole TA + * [in] memref[1]: symbol name (@symbol) + * [out] value[2]: address of the symbol or NULL + * + * Used by: (libdl) dlsym(void *handle, const char *symbol) + */ +#define PTA_SYSTEM_DLSYM 11 + +/* + * Retrieves a copy of the TPM Event log held in secure memory. + * + * [out] memref[0]: Pointer to the buffer where to store the event log. + */ +#define PTA_SYSTEM_GET_TPM_EVENT_LOG 12 + +/* + * Invoke a tee-supplicant's plugin + * + * [in] memref[0] uuid of the plugin (TEE_UUID) + * [in] value[1].a command for the plugin + * [in] value[1].b sub_command for the plugin + * [in/out] memref[2] additional data for the plugin + * [out] value[3].a output length of data + */ +#define PTA_SYSTEM_SUPP_PLUGIN_INVOKE 13 + +#endif /* __PTA_SYSTEM_H */ diff --git a/optee_os/lib/libutee/include/riscv_user_sysreg.h b/optee_os/lib/libutee/include/riscv_user_sysreg.h new file mode 100644 index 0000000..eed5d97 --- /dev/null +++ b/optee_os/lib/libutee/include/riscv_user_sysreg.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + */ +#ifndef RISCV_USER_SYSREG_H +#define RISCV_USER_SYSREG_H + +#include + +#define read_csr(csr) \ + ({ \ + register unsigned long v; \ + asm volatile ("csrr %0, " #csr : "=r"(v) : : "memory"); \ + v; \ + }) + +static inline __noprof uint64_t read_time(void) +{ + uint64_t time = 0; + uint32_t hi __maybe_unused = 0; + uint32_t lo __maybe_unused = 0; + +#ifdef RV32 + do { + hi = read_csr(timeh); + lo = read_csr(time); + } while (hi != read_csr(timeh)); + + time = SHIFT_U64(hi, 32) | lo; +#else /*RV64*/ + time = read_csr(time); +#endif /*RV32*/ + + return time; +} + +/* These barriers need to enforce ordering on both devices and memory. */ +static inline __noprof void mb(void) +{ + asm volatile ("fence" : : : "memory"); +} + +static inline __noprof uint64_t barrier_read_counter_timer(void) +{ + mb(); /* Get timer value after pending operations have completed */ + return read_time(); +} + +static inline __noprof uint32_t read_cntfrq(void) +{ + return CFG_RISCV_MTIME_RATE; +} + +#endif /* RISCV_USER_SYSREG_H */ diff --git a/optee_os/lib/libutee/include/rng_pta_client.h b/optee_os/lib/libutee/include/rng_pta_client.h new file mode 100644 index 0000000..6de3c64 --- /dev/null +++ b/optee_os/lib/libutee/include/rng_pta_client.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2018-2022, Linaro Limited + * + * This header file is deprecated and kept for backward compatibility. + */ + +#ifndef __RNG_PTA_CLIENT_H +#define __RNG_PTA_CLIENT_H + +#include + +#endif /* __RNG_PTA_CLIENT_H */ diff --git a/optee_os/lib/libutee/include/tee_api.h b/optee_os/lib/libutee/include/tee_api.h new file mode 100644 index 0000000..22b6b04 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_api.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_API_H +#define TEE_API_H + +#include + +#endif /* TEE_API_H */ diff --git a/optee_os/lib/libutee/include/tee_api_compat.h b/optee_os/lib/libutee/include/tee_api_compat.h new file mode 100644 index 0000000..166f1a7 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_api_compat.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Linaro Limited + */ + +#ifndef __TEE_API_COMPAT_H +#define __TEE_API_COMPAT_H + +/* + * This function will be called from TA_OpenSessionEntryPoint() in + * user_ta_header.c (if compiled with __OPTEE_CORE_API_COMPAT_1_1), the + * compatibility entry function is passed as a function pointer in @fp. + * This is needed since libutee is never compiled with + * __OPTEE_CORE_API_COMPAT_1_1, but we still need a way to call the + * compatibility function __GP11_TA_InvokeCommandEntryPoint(), but only + * when __OPTEE_CORE_API_COMPAT_1_1 is defined. + */ +TEE_Result __ta_open_sess(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS], + void **sess_ctx, + TEE_Result (*fp)(uint32_t, + __GP11_TEE_Param [TEE_NUM_PARAMS], + void **)); + +/* + * Same as for __ta_open_sess_func(), except that @fp now is a pointer to + * __GP11_TA_InvokeCommandEntryPoint(). + */ +TEE_Result __ta_invoke_cmd(void *sess_ctx, uint32_t cmd_id, uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS], + TEE_Result (*fp)(void *, uint32_t, uint32_t, + __GP11_TEE_Param [TEE_NUM_PARAMS])); + +#if __OPTEE_CORE_API_COMPAT_1_1 +/* Types */ +#define TEE_Attribute __GP11_TEE_Attribute +#define TEE_ObjectInfo __GP11_TEE_ObjectInfo +#define TEE_Param __GP11_TEE_Param + +/* Functions */ +#define TA_InvokeCommandEntryPoint __GP11_TA_InvokeCommandEntryPoint +#define TA_OpenSessionEntryPoint __GP11_TA_OpenSessionEntryPoint +#define TEE_AEDecryptFinal __GP11_TEE_AEDecryptFinal +#define TEE_AEEncryptFinal __GP11_TEE_AEEncryptFinal +#define TEE_AEInit __GP11_TEE_AEInit +#define TEE_AEUpdateAAD __GP11_TEE_AEUpdateAAD +#define TEE_AEUpdate __GP11_TEE_AEUpdate +#define TEE_AllocateTransientObject __GP11_TEE_AllocateTransientObject +#define TEE_AsymmetricDecrypt __GP11_TEE_AsymmetricDecrypt +#define TEE_AsymmetricEncrypt __GP11_TEE_AsymmetricEncrypt +#define TEE_AsymmetricSignDigest __GP11_TEE_AsymmetricSignDigest +#define TEE_AsymmetricVerifyDigest __GP11_TEE_AsymmetricVerifyDigest +#define TEE_BigIntConvertFromOctetString __GP11_TEE_BigIntConvertFromOctetString +#define TEE_BigIntConvertToOctetString __GP11_TEE_BigIntConvertToOctetString +#define TEE_BigIntFMMContextSizeInU32 __GP11_TEE_BigIntFMMContextSizeInU32 +#define TEE_BigIntFMMSizeInU32 __GP11_TEE_BigIntFMMSizeInU32 +#define TEE_BigIntInitFMMContext __GP11_TEE_BigIntInitFMMContext +#define TEE_BigIntInitFMM __GP11_TEE_BigIntInitFMM +#define TEE_BigIntShiftRight __GP11_TEE_BigIntShiftRight +#define TEE_CheckMemoryAccessRights __GP11_TEE_CheckMemoryAccessRights +#define TEE_CipherDoFinal __GP11_TEE_CipherDoFinal +#define TEE_CipherInit __GP11_TEE_CipherInit +#define TEE_CipherUpdate __GP11_TEE_CipherUpdate +#define TEE_CreatePersistentObject __GP11_TEE_CreatePersistentObject +#define TEE_DeriveKey __GP11_TEE_DeriveKey +#define TEE_DigestDoFinal __GP11_TEE_DigestDoFinal +#define TEE_DigestUpdate __GP11_TEE_DigestUpdate +#define TEE_FreeOperation __GP11_TEE_FreeOperation +#define TEE_GenerateKey __GP11_TEE_GenerateKey +#define TEE_GenerateRandom __GP11_TEE_GenerateRandom +#define TEE_GetNextPersistentObject __GP11_TEE_GetNextPersistentObject +#define TEE_GetObjectBufferAttribute __GP11_TEE_GetObjectBufferAttribute +#define TEE_GetObjectInfo1 __GP11_TEE_GetObjectInfo1 +#define TEE_GetObjectInfo __GP11_TEE_GetObjectInfo +#define TEE_GetOperationInfoMultiple __GP11_TEE_GetOperationInfoMultiple +#define TEE_GetPropertyAsBinaryBlock __GP11_TEE_GetPropertyAsBinaryBlock +#define TEE_GetPropertyAsString __GP11_TEE_GetPropertyAsString +#define TEE_GetPropertyName __GP11_TEE_GetPropertyName +#define TEE_InitRefAttribute __GP11_TEE_InitRefAttribute +#define TEE_InitValueAttribute __GP11_TEE_InitValueAttribute +#define TEE_InvokeTACommand __GP11_TEE_InvokeTACommand +#define TEE_MACCompareFinal __GP11_TEE_MACCompareFinal +#define TEE_MACComputeFinal __GP11_TEE_MACComputeFinal +#define TEE_MACInit __GP11_TEE_MACInit +#define TEE_MACUpdate __GP11_TEE_MACUpdate +#define TEE_Malloc __GP11_TEE_Malloc +#define TEE_MemCompare __GP11_TEE_MemCompare +#define TEE_MemFill __GP11_TEE_MemFill +#define TEE_MemMove __GP11_TEE_MemMove +#define TEE_OpenPersistentObject __GP11_TEE_OpenPersistentObject +#define TEE_OpenTASession __GP11_TEE_OpenTASession +#define TEE_PopulateTransientObject __GP11_TEE_PopulateTransientObject +#define TEE_ReadObjectData __GP11_TEE_ReadObjectData +#define TEE_Realloc __GP11_TEE_Realloc +#define TEE_RenamePersistentObject __GP11_TEE_RenamePersistentObject +#define TEE_SeekObjectData __GP11_TEE_SeekObjectData +#define TEE_SetOperationKey2 __GP11_TEE_SetOperationKey2 +#define TEE_SetOperationKey __GP11_TEE_SetOperationKey +#define TEE_TruncateObjectData __GP11_TEE_TruncateObjectData +#define TEE_WriteObjectData __GP11_TEE_WriteObjectData +#endif + +#endif /*__TEE_API_COMPAT_H*/ diff --git a/optee_os/lib/libutee/include/tee_api_defines.h b/optee_os/lib/libutee/include/tee_api_defines.h new file mode 100644 index 0000000..9e0979c --- /dev/null +++ b/optee_os/lib/libutee/include/tee_api_defines.h @@ -0,0 +1,697 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2022, Linaro Limited + */ + +/* Based on GP TEE Internal Core API Specification Version 1.3.1 */ + +#ifndef TEE_API_DEFINES_H +#define TEE_API_DEFINES_H + +#define TEE_CORE_API_MAJOR_VERSION 1U +#define TEE_CORE_API_MINOR_VERSION 3U +#define TEE_CORE_API_MAINTENANCE_VERSION 1U +#define TEE_CORE_API_VERSION \ + ((TEE_CORE_API_MAJOR_VERSION << 24) | \ + (TEE_CORE_API_MINOR_VERSION << 16) | \ + (TEE_CORE_API_MAINTENANCE_VERSION << 8)) +#define TEE_CORE_API_1_3_1 + +/* + * Below follows the GP defined way of letting a TA define that it wants an + * API compatible with version 1.1 or the latest. An alternative approach + * is to set __OPTEE_CORE_API_COMPAT_1_1, but that's an OP-TEE extension. + * + * The GP specs (>= 1.2) requires that only APIs defined in the indicated + * version SHALL be made available when using this mechanism. However, that + * is far beyond what ordinary standards requires as they permit + * extensions. With this, in OP-TEE, extensions and new API that doesn't + * interfere with the selected version of the standard will be permitted. + */ +#if defined(TEE_CORE_API_REQUIRED_MAINTENANCE_VERSION) && \ + !defined(TEE_CORE_API_REQUIRED_MINOR_VERSION) +#error "Required TEE_CORE_API_REQUIRED_MINOR_VERSION undefined" +#endif +#if defined(TEE_CORE_API_REQUIRED_MINOR_VERSION) && \ + !defined(TEE_CORE_API_REQUIRED_MAJOR_VERSION) +#error "Required TEE_CORE_API_REQUIRED_MAJOR_VERSION undefined" +#endif + +#if defined(TEE_CORE_API_REQUIRED_MAJOR_VERSION) +#if TEE_CORE_API_REQUIRED_MAJOR_VERSION != 1 && \ + TEE_CORE_API_REQUIRED_MAJOR_VERSION != 0 +#error "Required major version not supported" +#endif +#ifdef TEE_CORE_API_REQUIRED_MINOR_VERSION +#if TEE_CORE_API_REQUIRED_MINOR_VERSION == 1 +#define __OPTEE_CORE_API_COMPAT_1_1 1 +#else +#error "Required minor version not supported" +#endif +#if defined(TEE_CORE_API_REQUIRED_MAINTENANCE_VERSION) && \ + TEE_CORE_API_REQUIRED_MAINTENANCE_VERSION != 0 +#error "Required maintenance version not supported" +#endif +#endif +#endif + +/* + * For backwards compatibility with v1.1 as provided by up to OP-TEE + * version 3.19.0, define __OPTEE_CORE_API_COMPAT_1_1 to 1. + * + * Some versions of the GP specs have introduced backwards incompatible + * changes. For example the v1.0: + * + * TEE_Result TEE_DigestDoFinal(TEE_OperationHandle operation, + * const void *chunk, uint32_t chunkLen, + * void *hash, uint32_t *hashLen); + * + * Was changed in v1.1.1 to this: + * + * TEE_Result TEE_DigestDoFinal(TEE_OperationHandle operation, + * const void *chunk, size_t chunkLen, + * void *hash, size_t *hashLen); + * + * Note the type change for "hashLen", a source of problem especially on + * platforms where size_t is a 64-bit unsigned integer. + * + * As a way of allowing older TAs to be compiled with a newer version of + * the API we can turn off or hide different incompatible changes. New + * features which are not interfering with older versions of the API are + * not disabled. So by enabling __OPTEE_CORE_API_COMPAT_1_1 will not result + * in pure 1.1 API, it will be a hybrid API that should work with most TAs + * not yet updated to the new API. + * + * Backwards compatibility is provided by duplicating all functions that + * has changed since v1.1. The original function is given a "__GP11_" + * prefix and preprocessor macros are used to let a legacy TA use the old + * function instead. The same principle applies to defined types. + */ +#ifndef __OPTEE_CORE_API_COMPAT_1_1 +#define __OPTEE_CORE_API_COMPAT_1_1 0 +#endif + +#define TEE_HANDLE_NULL 0 + +#define TEE_TIMEOUT_INFINITE 0xFFFFFFFF + +/* API Error Codes */ +#define TEE_SUCCESS 0x00000000 +#define TEE_ERROR_CORRUPT_OBJECT 0xF0100001 +#define TEE_ERROR_CORRUPT_OBJECT_2 0xF0100002 +#define TEE_ERROR_STORAGE_NOT_AVAILABLE 0xF0100003 +#define TEE_ERROR_STORAGE_NOT_AVAILABLE_2 0xF0100004 +#define TEE_ERROR_UNSUPPORTED_VERSION 0xF0100005 +#define TEE_ERROR_CIPHERTEXT_INVALID 0xF0100006 +#define TEE_ERROR_GENERIC 0xFFFF0000 +#define TEE_ERROR_ACCESS_DENIED 0xFFFF0001 +#define TEE_ERROR_CANCEL 0xFFFF0002 +#define TEE_ERROR_ACCESS_CONFLICT 0xFFFF0003 +#define TEE_ERROR_EXCESS_DATA 0xFFFF0004 +#define TEE_ERROR_BAD_FORMAT 0xFFFF0005 +#define TEE_ERROR_BAD_PARAMETERS 0xFFFF0006 +#define TEE_ERROR_BAD_STATE 0xFFFF0007 +#define TEE_ERROR_ITEM_NOT_FOUND 0xFFFF0008 +#define TEE_ERROR_NOT_IMPLEMENTED 0xFFFF0009 +#define TEE_ERROR_NOT_SUPPORTED 0xFFFF000A +#define TEE_ERROR_NO_DATA 0xFFFF000B +#define TEE_ERROR_OUT_OF_MEMORY 0xFFFF000C +#define TEE_ERROR_BUSY 0xFFFF000D +#define TEE_ERROR_COMMUNICATION 0xFFFF000E +#define TEE_ERROR_SECURITY 0xFFFF000F +#define TEE_ERROR_SHORT_BUFFER 0xFFFF0010 +#define TEE_ERROR_EXTERNAL_CANCEL 0xFFFF0011 +#define TEE_ERROR_TIMEOUT 0xFFFF3001 +#define TEE_ERROR_OVERFLOW 0xFFFF300F +#define TEE_ERROR_TARGET_DEAD 0xFFFF3024 +#define TEE_ERROR_STORAGE_NO_SPACE 0xFFFF3041 +#define TEE_ERROR_MAC_INVALID 0xFFFF3071 +#define TEE_ERROR_SIGNATURE_INVALID 0xFFFF3072 +#define TEE_ERROR_TIME_NOT_SET 0xFFFF5000 +#define TEE_ERROR_TIME_NEEDS_RESET 0xFFFF5001 + +/* Parameter Type Constants */ +#define TEE_PARAM_TYPE_NONE 0 +#define TEE_PARAM_TYPE_VALUE_INPUT 1 +#define TEE_PARAM_TYPE_VALUE_OUTPUT 2 +#define TEE_PARAM_TYPE_VALUE_INOUT 3 +#define TEE_PARAM_TYPE_MEMREF_INPUT 5 +#define TEE_PARAM_TYPE_MEMREF_OUTPUT 6 +#define TEE_PARAM_TYPE_MEMREF_INOUT 7 + +/* Login Type Constants */ +#define TEE_LOGIN_PUBLIC 0x00000000 +#define TEE_LOGIN_USER 0x00000001 +#define TEE_LOGIN_GROUP 0x00000002 +#define TEE_LOGIN_APPLICATION 0x00000004 +#define TEE_LOGIN_APPLICATION_USER 0x00000005 +#define TEE_LOGIN_APPLICATION_GROUP 0x00000006 +#define TEE_LOGIN_TRUSTED_APP 0xF0000000 + +/* Origin Code Constants */ +#define TEE_ORIGIN_API 0x00000001 +#define TEE_ORIGIN_COMMS 0x00000002 +#define TEE_ORIGIN_TEE 0x00000003 +#define TEE_ORIGIN_TRUSTED_APP 0x00000004 + +/* Property Sets pseudo handles */ +#define TEE_PROPSET_TEE_IMPLEMENTATION (TEE_PropSetHandle)0xFFFFFFFD +#define TEE_PROPSET_CURRENT_CLIENT (TEE_PropSetHandle)0xFFFFFFFE +#define TEE_PROPSET_CURRENT_TA (TEE_PropSetHandle)0xFFFFFFFF + +/* Memory Access Rights Constants */ +#define TEE_MEMORY_ACCESS_READ 0x00000001 +#define TEE_MEMORY_ACCESS_WRITE 0x00000002 +#define TEE_MEMORY_ACCESS_ANY_OWNER 0x00000004 + +/* Memory Management Constant */ +#define TEE_MALLOC_FILL_ZERO 0x00000000 +#define TEE_MALLOC_NO_FILL 0x00000001 +#define TEE_MALLOC_NO_SHARE 0x00000002 + +/* TEE_Whence Constants */ +#define TEE_DATA_SEEK_SET 0x00000000 +#define TEE_DATA_SEEK_CUR 0x00000001 +#define TEE_DATA_SEEK_END 0x00000002 +#define TEE_WHENCE_ILLEGAL_VALUE 0x7FFFFFFF + +/* TEE_OperationMode Values */ +#define TEE_MODE_ENCRYPT 0x00000000 +#define TEE_MODE_DECRYPT 0x00000001 +#define TEE_MODE_SIGN 0x00000002 +#define TEE_MODE_VERIFY 0x00000003 +#define TEE_MODE_MAC 0x00000004 +#define TEE_MODE_DIGEST 0x00000005 +#define TEE_MODE_DERIVE 0x00000006 +#define TEE_MODE_ILLEGAL_VALUE 0x7FFFFFFF + +/* Other constants */ +#define TEE_STORAGE_PRIVATE 0x00000001 + +#define TEE_DATA_FLAG_ACCESS_READ 0x00000001 +#define TEE_DATA_FLAG_ACCESS_WRITE 0x00000002 +#define TEE_DATA_FLAG_ACCESS_WRITE_META 0x00000004 +#define TEE_DATA_FLAG_SHARE_READ 0x00000010 +#define TEE_DATA_FLAG_SHARE_WRITE 0x00000020 +#define TEE_DATA_FLAG_OVERWRITE 0x00000400 +#define TEE_DATA_MAX_POSITION 0xFFFFFFFF +#define TEE_OBJECT_ID_MAX_LEN 64 +#define TEE_USAGE_EXTRACTABLE 0x00000001 +#define TEE_USAGE_ENCRYPT 0x00000002 +#define TEE_USAGE_DECRYPT 0x00000004 +#define TEE_USAGE_MAC 0x00000008 +#define TEE_USAGE_SIGN 0x00000010 +#define TEE_USAGE_VERIFY 0x00000020 +#define TEE_USAGE_DERIVE 0x00000040 +#define TEE_HANDLE_FLAG_PERSISTENT 0x00010000 +#define TEE_HANDLE_FLAG_INITIALIZED 0x00020000 +#define TEE_HANDLE_FLAG_KEY_SET 0x00040000 +#define TEE_HANDLE_FLAG_EXPECT_TWO_KEYS 0x00080000 +#define TEE_HANDLE_FLAG_EXTRACTING 0x00100000 +#define TEE_OPERATION_CIPHER 1 +#define TEE_OPERATION_MAC 3 +#define TEE_OPERATION_AE 4 +#define TEE_OPERATION_DIGEST 5 +#define TEE_OPERATION_ASYMMETRIC_CIPHER 6 +#define TEE_OPERATION_ASYMMETRIC_SIGNATURE 7 +#define TEE_OPERATION_KEY_DERIVATION 8 +#define TEE_OPERATION_STATE_INITIAL 0x00000000 +#define TEE_OPERATION_STATE_ACTIVE 0x00000001 +#define TEE_OPERATION_STATE_EXTRACTING 0x00000002 + +/* Algorithm Identifiers */ +#define TEE_ALG_AES_ECB_NOPAD 0x10000010 +#define TEE_ALG_AES_CBC_NOPAD 0x10000110 +#define TEE_ALG_AES_CTR 0x10000210 +#define TEE_ALG_AES_CTS 0x10000310 +#define TEE_ALG_AES_XTS 0x10000410 +#define TEE_ALG_AES_CBC_MAC_NOPAD 0x30000110 +#define TEE_ALG_AES_CBC_MAC_PKCS5 0x30000510 +#define TEE_ALG_AES_CMAC 0x30000610 +#define TEE_ALG_AES_CCM 0x40000710 +#define TEE_ALG_AES_GCM 0x40000810 +#define TEE_ALG_DES_ECB_NOPAD 0x10000011 +#define TEE_ALG_DES_CBC_NOPAD 0x10000111 +#define TEE_ALG_DES_CBC_MAC_NOPAD 0x30000111 +#define TEE_ALG_DES_CBC_MAC_PKCS5 0x30000511 +#define TEE_ALG_DES3_ECB_NOPAD 0x10000013 +#define TEE_ALG_DES3_CBC_NOPAD 0x10000113 +#define TEE_ALG_DES3_CBC_MAC_NOPAD 0x30000113 +#define TEE_ALG_DES3_CBC_MAC_PKCS5 0x30000513 +#define TEE_ALG_SM4_ECB_NOPAD 0x10000014 +#define TEE_ALG_SM4_CBC_NOPAD 0x10000114 +#define TEE_ALG_SM4_CTR 0x10000214 +#define TEE_ALG_RSASSA_PKCS1_V1_5_MD5 0x70001830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA1 0x70002830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA224 0x70003830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA256 0x70004830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA384 0x70005830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA512 0x70006830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_MD5SHA1 0x7000F830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA3_224 0x70008830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA3_256 0x70009830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA3_384 0x7000A830 +#define TEE_ALG_RSASSA_PKCS1_V1_5_SHA3_512 0x7000B830 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1 0x70212930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224 0x70313930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256 0x70414930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384 0x70515930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512 0x70616930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA3_224 0x70818930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA3_256 0x70919930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA3_384 0x70A1A930 +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA3_512 0x70B1B930 +#define TEE_ALG_RSAES_PKCS1_V1_5 0x60000130 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1 0x60210230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224 0x60310230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256 0x60410230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384 0x60510230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512 0x60610230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA3_224 0x60810230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA3_256 0x60910230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA3_384 0x60A10230 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA3_512 0x60B10230 +#define TEE_ALG_RSA_NOPAD 0x60000030 +#define TEE_ALG_DSA_SHA1 0x70002131 +#define TEE_ALG_DSA_SHA224 0x70003131 +#define TEE_ALG_DSA_SHA256 0x70004131 +#define TEE_ALG_DSA_SHA3_224 0x70008131 +#define TEE_ALG_DSA_SHA3_256 0x70009131 +#define TEE_ALG_DSA_SHA3_384 0x7000A131 +#define TEE_ALG_DSA_SHA3_512 0x7000B131 +#define TEE_ALG_SM2_DSA_SM3 0x70006045 +#define TEE_ALG_DH_DERIVE_SHARED_SECRET 0x80000032 +#define TEE_ALG_SM2_KEP 0x60000045 +#define TEE_ALG_MD5 0x50000001 +#define TEE_ALG_SHA1 0x50000002 +#define TEE_ALG_SHA224 0x50000003 +#define TEE_ALG_SHA256 0x50000004 +#define TEE_ALG_SHA384 0x50000005 +#define TEE_ALG_SHA512 0x50000006 +#define TEE_ALG_SHA3_224 0x50000008 +#define TEE_ALG_SHA3_256 0x50000009 +#define TEE_ALG_SHA3_384 0x5000000A +#define TEE_ALG_SHA3_512 0x5000000B +#define TEE_ALG_MD5SHA1 0x5000000F +#define TEE_ALG_HMAC_MD5 0x30000001 +#define TEE_ALG_HMAC_SHA1 0x30000002 +#define TEE_ALG_HMAC_SHA224 0x30000003 +#define TEE_ALG_HMAC_SHA256 0x30000004 +#define TEE_ALG_HMAC_SHA384 0x30000005 +#define TEE_ALG_HMAC_SHA512 0x30000006 +#define TEE_ALG_HMAC_SM3 0x30000007 +#define TEE_ALG_HMAC_SHA3_224 0x30000008 +#define TEE_ALG_HMAC_SHA3_256 0x30000009 +#define TEE_ALG_HMAC_SHA3_384 0x3000000A +#define TEE_ALG_HMAC_SHA3_512 0x3000000B + +/* + * These are used in the OP-TEE ABI, due to an inconsistency in the v1.1 + * specification the wrong values we assumed and now we're stuck with those. + * + * In GP Internal Core API v1.1 + * "Table 6-12: Structure of Algorithm Identifier" + * indicates ECDSA have the algorithm "0x41" and ECDH "0x42" + * whereas + * "Table 6-11: List of Algorithm Identifiers" defines + * TEE_ALG_ECDSA_P192 as 0x70001042 + * + * We chose to define __OPTEE_TEE_ALG_ECDSA_P192 as 0x70001041 and so on + * to conform to table 6-12. + */ +#define __OPTEE_ALG_ECDSA_P192 0x70001041 +#define __OPTEE_ALG_ECDSA_P224 0x70002041 +#define __OPTEE_ALG_ECDSA_P256 0x70003041 +#define __OPTEE_ALG_ECDSA_P384 0x70004041 +#define __OPTEE_ALG_ECDSA_P521 0x70005041 +#define __OPTEE_ALG_ECDH_P192 0x80001042 +#define __OPTEE_ALG_ECDH_P224 0x80002042 +#define __OPTEE_ALG_ECDH_P256 0x80003042 +#define __OPTEE_ALG_ECDH_P384 0x80004042 +#define __OPTEE_ALG_ECDH_P521 0x80005042 + +/* TEE_ALG_ECDSA_P* and TEE_ALG_ECDH_P* are deprecated */ +#define TEE_ALG_ECDSA_P192 TEE_ALG_ECDSA_SHA1 +#define TEE_ALG_ECDSA_P224 TEE_ALG_ECDSA_SHA224 +#define TEE_ALG_ECDSA_P256 TEE_ALG_ECDSA_SHA256 +#define TEE_ALG_ECDSA_P384 TEE_ALG_ECDSA_SHA384 +#define TEE_ALG_ECDSA_P521 TEE_ALG_ECDSA_SHA512 +#define TEE_ALG_ECDH_P192 TEE_ALG_ECDH_DERIVE_SHARED_SECRET +#define TEE_ALG_ECDH_P224 TEE_ALG_ECDH_DERIVE_SHARED_SECRET +#define TEE_ALG_ECDH_P256 TEE_ALG_ECDH_DERIVE_SHARED_SECRET +#define TEE_ALG_ECDH_P384 TEE_ALG_ECDH_DERIVE_SHARED_SECRET +#define TEE_ALG_ECDH_P521 TEE_ALG_ECDH_DERIVE_SHARED_SECRET + +#define TEE_ALG_ECDH_DERIVE_SHARED_SECRET 0x80000042 +#define TEE_ALG_ECDSA_SHA1 0x70001042 +#define TEE_ALG_ECDSA_SHA224 0x70002042 +#define TEE_ALG_ECDSA_SHA256 0x70003042 +#define TEE_ALG_ECDSA_SHA384 0x70004042 +#define TEE_ALG_ECDSA_SHA512 0x70005042 +#define TEE_ALG_ECDSA_SHA3_224 0x70006042 +#define TEE_ALG_ECDSA_SHA3_256 0x70007042 +#define TEE_ALG_ECDSA_SHA3_384 0x70008042 +#define TEE_ALG_ECDSA_SHA3_512 0x70009042 + +#define TEE_ALG_ED25519 0x70006043 +#define TEE_ALG_ED448 0x70006044 +#define TEE_ALG_SM2_PKE 0x80000046 +#define TEE_ALG_HKDF 0x80000047 +#define TEE_ALG_SM3 0x50000007 +#define TEE_ALG_X25519 0x80000044 +#define TEE_ALG_X448 0x80000045 +#define TEE_ALG_SM4_ECB_PKCS5 0x10000015 +#define TEE_ALG_SM4_CBC_PKCS5 0x10000115 +#define TEE_ALG_ILLEGAL_VALUE 0xEFFFFFFF + +#define TEE_ALG_SHA3_224 0x50000008 +#define TEE_ALG_SHA3_256 0x50000009 +#define TEE_ALG_SHA3_384 0x5000000A +#define TEE_ALG_SHA3_512 0x5000000B +#define TEE_ALG_SHAKE128 0x50000101 +#define TEE_ALG_SHAKE256 0x50000102 + +/* Object Types */ + +#define TEE_TYPE_AES 0xA0000010 +#define TEE_TYPE_DES 0xA0000011 +#define TEE_TYPE_DES3 0xA0000013 +#define TEE_TYPE_SM4 0xA0000014 +#define TEE_TYPE_HMAC_MD5 0xA0000001 +#define TEE_TYPE_HMAC_SHA1 0xA0000002 +#define TEE_TYPE_HMAC_SHA224 0xA0000003 +#define TEE_TYPE_HMAC_SHA256 0xA0000004 +#define TEE_TYPE_HMAC_SHA384 0xA0000005 +#define TEE_TYPE_HMAC_SHA512 0xA0000006 +#define TEE_TYPE_HMAC_SM3 0xA0000007 +#define TEE_TYPE_HMAC_SHA3_224 0xA0000008 +#define TEE_TYPE_HMAC_SHA3_256 0xA0000009 +#define TEE_TYPE_HMAC_SHA3_384 0xA000000A +#define TEE_TYPE_HMAC_SHA3_512 0xA000000B +#define TEE_TYPE_RSA_PUBLIC_KEY 0xA0000030 +#define TEE_TYPE_RSA_KEYPAIR 0xA1000030 +#define TEE_TYPE_DSA_PUBLIC_KEY 0xA0000031 +#define TEE_TYPE_DSA_KEYPAIR 0xA1000031 +#define TEE_TYPE_DH_KEYPAIR 0xA1000032 +#define TEE_TYPE_ECDSA_PUBLIC_KEY 0xA0000041 +#define TEE_TYPE_ECDSA_KEYPAIR 0xA1000041 +#define TEE_TYPE_ECDH_PUBLIC_KEY 0xA0000042 +#define TEE_TYPE_ECDH_KEYPAIR 0xA1000042 +#define TEE_TYPE_ED25519_PUBLIC_KEY 0xA0000043 +#define TEE_TYPE_ED25519_KEYPAIR 0xA1000043 +#define TEE_TYPE_ED448_PUBLIC_KEY 0xA0000048 +#define TEE_TYPE_ED448_KEYPAIR 0xA1000048 +#define TEE_TYPE_X448_PUBLIC_KEY 0xA0000049 +#define TEE_TYPE_X448_KEYPAIR 0xA1000049 +#define TEE_TYPE_SM2_DSA_PUBLIC_KEY 0xA0000045 +#define TEE_TYPE_SM2_DSA_KEYPAIR 0xA1000045 +#define TEE_TYPE_SM2_KEP_PUBLIC_KEY 0xA0000046 +#define TEE_TYPE_SM2_KEP_KEYPAIR 0xA1000046 +#define TEE_TYPE_SM2_PKE_PUBLIC_KEY 0xA0000047 +#define TEE_TYPE_SM2_PKE_KEYPAIR 0xA1000047 +#define TEE_TYPE_HKDF 0xA000004A +#define TEE_TYPE_GENERIC_SECRET 0xA0000000 +#define TEE_TYPE_CORRUPTED_OBJECT 0xA00000BE +#define TEE_TYPE_DATA 0xA00000BF +#define TEE_TYPE_X25519_PUBLIC_KEY 0xA0000044 +#define TEE_TYPE_X25519_KEYPAIR 0xA1000044 +#define TEE_TYPE_ILLEGAL_VALUE 0xEFFFFFFF + +/* List of Object or Operation Attributes */ + +#define TEE_ATTR_SECRET_VALUE 0xC0000000 +#define TEE_ATTR_RSA_MODULUS 0xD0000130 +#define TEE_ATTR_RSA_PUBLIC_EXPONENT 0xD0000230 +#define TEE_ATTR_RSA_PRIVATE_EXPONENT 0xC0000330 +#define TEE_ATTR_RSA_PRIME1 0xC0000430 +#define TEE_ATTR_RSA_PRIME2 0xC0000530 +#define TEE_ATTR_RSA_EXPONENT1 0xC0000630 +#define TEE_ATTR_RSA_EXPONENT2 0xC0000730 +#define TEE_ATTR_RSA_COEFFICIENT 0xC0000830 +#define TEE_ATTR_DSA_PRIME 0xD0001031 +#define TEE_ATTR_DSA_SUBPRIME 0xD0001131 +#define TEE_ATTR_DSA_BASE 0xD0001231 +#define TEE_ATTR_DSA_PUBLIC_VALUE 0xD0000131 +#define TEE_ATTR_DSA_PRIVATE_VALUE 0xC0000231 +#define TEE_ATTR_DH_PRIME 0xD0001032 +#define TEE_ATTR_DH_SUBPRIME 0xD0001132 +#define TEE_ATTR_DH_BASE 0xD0001232 +#define TEE_ATTR_DH_X_BITS 0xF0001332 +#define TEE_ATTR_DH_PUBLIC_VALUE 0xD0000132 +#define TEE_ATTR_DH_PRIVATE_VALUE 0xC0000232 +#define TEE_ATTR_RSA_OAEP_LABEL 0xD0000930 +#define TEE_ATTR_RSA_OAEP_MGF_HASH 0xD0000931 +#define TEE_ATTR_RSA_PSS_SALT_LENGTH 0xF0000A30 +#define TEE_ATTR_ECC_PUBLIC_VALUE_X 0xD0000141 +#define TEE_ATTR_ECC_PUBLIC_VALUE_Y 0xD0000241 +#define TEE_ATTR_ECC_PRIVATE_VALUE 0xC0000341 +#define TEE_ATTR_ECC_CURVE 0xF0000441 +#define TEE_ATTR_SM2_ID_INITIATOR 0xD0000446 +#define TEE_ATTR_SM2_ID_RESPONDER 0xD0000546 +#define TEE_ATTR_SM2_KEP_USER 0xF0000646 +#define TEE_ATTR_SM2_KEP_CONFIRMATION_IN 0xD0000746 +#define TEE_ATTR_SM2_KEP_CONFIRMATION_OUT 0xD0000846 + +/* + * Commit 5b385b3f835d ("core: crypto: add support for SM2 KEP") defined by + * mistake the wrong values for these two. OP-TEE recognizes these two as + * alternative IDs in parallel with the correct official values when + * supplied as parameters when deriving a key using the TEE_ALG_SM2_KEP + * algorithm. + */ +#define __OPTEE_SM2_KEP_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_X 0xD0000946 +#define __OPTEE_SM2_KEP_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_Y 0xD0000A46 + +#define TEE_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_X 0xD0000146 +#define TEE_ATTR_ECC_EPHEMERAL_PUBLIC_VALUE_Y 0xD0000246 +#define TEE_ATTR_EDDSA_CTX 0xD0000643 +#define TEE_ATTR_ED25519_PUBLIC_VALUE 0xD0000743 +#define TEE_ATTR_ED25519_PRIVATE_VALUE 0xC0000843 +#define TEE_ATTR_X25519_PUBLIC_VALUE 0xD0000944 +#define TEE_ATTR_X25519_PRIVATE_VALUE 0xC0000A44 +#define TEE_ATTR_EDDSA_PREHASH 0xF0000004 +#define TEE_ATTR_X448_PUBLIC_VALUE 0xD0000A45 +#define TEE_ATTR_X448_PRIVATE_VALUE 0xC0000A46 +#define TEE_ATTR_HKDF_SALT 0xD0000946 +#define TEE_ATTR_HKDF_INFO 0xD0000A46 +#define TEE_ATTR_HKDF_HASH_ALGORITHM 0xF0000B46 +#define TEE_ATTR_KDF_KEY_SIZE 0xF0000C46 + +#define TEE_ATTR_FLAG_PUBLIC (1 << 28) +#define TEE_ATTR_FLAG_VALUE (1 << 29) +/* + * Deprecated, but kept for backwards compatibility + * + * Be careful with GPD TEE Internal API specification v1.0 where table 6-12 + * defines BIT [28] with the right meaning whereas sections 5.4.3 and 5.4.4 + * falsely describe a reversed bit flag value meaning. + */ +#define TEE_ATTR_BIT_PROTECTED TEE_ATTR_FLAG_PUBLIC +#define TEE_ATTR_BIT_VALUE TEE_ATTR_FLAG_VALUE + +/* List of Supported ECC Curves */ +#define TEE_CRYPTO_ELEMENT_NONE 0x00000000 +#define TEE_ECC_CURVE_NIST_P192 0x00000001 +#define TEE_ECC_CURVE_NIST_P224 0x00000002 +#define TEE_ECC_CURVE_NIST_P256 0x00000003 +#define TEE_ECC_CURVE_NIST_P384 0x00000004 +#define TEE_ECC_CURVE_NIST_P521 0x00000005 +#define TEE_ECC_CURVE_25519 0x00000300 +#define TEE_ECC_CURVE_SM2 0x00000400 + +/* Panicked Functions Identification */ +/* TA Interface */ +#define TEE_PANIC_ID_TA_CLOSESESSIONENTRYPOINT 0x00000101 +#define TEE_PANIC_ID_TA_CREATEENTRYPOINT 0x00000102 +#define TEE_PANIC_ID_TA_DESTROYENTRYPOINT 0x00000103 +#define TEE_PANIC_ID_TA_INVOKECOMMANDENTRYPOINT 0x00000104 +#define TEE_PANIC_ID_TA_OPENSESSIONENTRYPOINT 0x00000105 +/* Property Access */ +#define TEE_PANIC_ID_TEE_ALLOCATEPROPERTYENUMERATOR 0x00000201 +#define TEE_PANIC_ID_TEE_FREEPROPERTYENUMERATOR 0x00000202 +#define TEE_PANIC_ID_TEE_GETNEXTPROPERTY 0x00000203 +#define TEE_PANIC_ID_TEE_GETPROPERTYASBINARYBLOCK 0x00000204 +#define TEE_PANIC_ID_TEE_GETPROPERTYASBOOL 0x00000205 +#define TEE_PANIC_ID_TEE_GETPROPERTYASIDENTITY 0x00000206 +#define TEE_PANIC_ID_TEE_GETPROPERTYASSTRING 0x00000207 +#define TEE_PANIC_ID_TEE_GETPROPERTYASU32 0x00000208 +#define TEE_PANIC_ID_TEE_GETPROPERTYASUUID 0x00000209 +#define TEE_PANIC_ID_TEE_GETPROPERTYNAME 0x0000020A +#define TEE_PANIC_ID_TEE_RESETPROPERTYENUMERATOR 0x0000020B +#define TEE_PANIC_ID_TEE_STARTPROPERTYENUMERATOR 0x0000020C +/* Panic Function */ +#define TEE_PANIC_ID_TEE_PANIC 0x00000301 +/* Internal Client API */ +#define TEE_PANIC_ID_TEE_CLOSETASESSION 0x00000401 +#define TEE_PANIC_ID_TEE_INVOKETACOMMAND 0x00000402 +#define TEE_PANIC_ID_TEE_OPENTASESSION 0x00000403 +/* Cancellation */ +#define TEE_PANIC_ID_TEE_GETCANCELLATIONFLAG 0x00000501 +#define TEE_PANIC_ID_TEE_MASKCANCELLATION 0x00000502 +#define TEE_PANIC_ID_TEE_UNMASKCANCELLATION 0x00000503 +/* Memory Management */ +#define TEE_PANIC_ID_TEE_CHECKMEMORYACCESSRIGHTS 0x00000601 +#define TEE_PANIC_ID_TEE_FREE 0x00000602 +#define TEE_PANIC_ID_TEE_GETINSTANCEDATA 0x00000603 +#define TEE_PANIC_ID_TEE_MALLOC 0x00000604 +#define TEE_PANIC_ID_TEE_MEMCOMPARE 0x00000605 +#define TEE_PANIC_ID_TEE_MEMFILL 0x00000606 +#define TEE_PANIC_ID_TEE_MEMMOVE 0x00000607 +#define TEE_PANIC_ID_TEE_REALLOC 0x00000608 +#define TEE_PANIC_ID_TEE_SETINSTANCEDATA 0x00000609 +/* Generic Object */ +#define TEE_PANIC_ID_TEE_CLOSEOBJECT 0x00000701 +#define TEE_PANIC_ID_TEE_GETOBJECTBUFFERATTRIBUTE 0x00000702 +/* deprecated */ +#define TEE_PANIC_ID_TEE_GETOBJECTINFO 0x00000703 +#define TEE_PANIC_ID_TEE_GETOBJECTVALUEATTRIBUTE 0x00000704 +/* deprecated */ +#define TEE_PANIC_ID_TEE_RESTRICTOBJECTUSAGE 0x00000705 +#define TEE_PANIC_ID_TEE_GETOBJECTINFO1 0x00000706 +#define TEE_PANIC_ID_TEE_RESTRICTOBJECTUSAGE1 0x00000707 +/* Transient Object */ +#define TEE_PANIC_ID_TEE_ALLOCATETRANSIENTOBJECT 0x00000801 +/* deprecated */ +#define TEE_PANIC_ID_TEE_COPYOBJECTATTRIBUTES 0x00000802 +#define TEE_PANIC_ID_TEE_FREETRANSIENTOBJECT 0x00000803 +#define TEE_PANIC_ID_TEE_GENERATEKEY 0x00000804 +#define TEE_PANIC_ID_TEE_INITREFATTRIBUTE 0x00000805 +#define TEE_PANIC_ID_TEE_INITVALUEATTRIBUTE 0x00000806 +#define TEE_PANIC_ID_TEE_POPULATETRANSIENTOBJECT 0x00000807 +#define TEE_PANIC_ID_TEE_RESETTRANSIENTOBJECT 0x00000808 +#define TEE_PANIC_ID_TEE_COPYOBJECTATTRIBUTES1 0x00000809 +/* Persistent Object */ +/* deprecated */ +#define TEE_PANIC_ID_TEE_CLOSEANDDELETEPERSISTENTOBJECT 0x00000901 +#define TEE_PANIC_ID_TEE_CREATEPERSISTENTOBJECT 0x00000902 +#define TEE_PANIC_ID_TEE_OPENPERSISTENTOBJECT 0x00000903 +#define TEE_PANIC_ID_TEE_RENAMEPERSISTENTOBJECT 0x00000904 +#define TEE_PANIC_ID_TEE_CLOSEANDDELETEPERSISTENTOBJECT1 0x00000905 +/* Persistent Object Enumeration */ +#define TEE_PANIC_ID_TEE_ALLOCATEPERSISTENTOBJECTENUMERATOR 0x00000A01 +#define TEE_PANIC_ID_TEE_FREEPERSISTENTOBJECTENUMERATOR 0x00000A02 +#define TEE_PANIC_ID_TEE_GETNEXTPERSISTENTOBJECT 0x00000A03 +#define TEE_PANIC_ID_TEE_RESETPERSISTENTOBJECTENUMERATOR 0x00000A04 +#define TEE_PANIC_ID_TEE_STARTPERSISTENTOBJECTENUMERATOR 0x00000A05 +/* Data Stream Access */ +#define TEE_PANIC_ID_TEE_READOBJECTDATA 0x00000B01 +#define TEE_PANIC_ID_TEE_SEEKOBJECTDATA 0x00000B02 +#define TEE_PANIC_ID_TEE_TRUNCATEOBJECTDATA 0x00000B03 +#define TEE_PANIC_ID_TEE_WRITEOBJECTDATA 0x00000B04 +/* Generic Operation */ +#define TEE_PANIC_ID_TEE_ALLOCATEOPERATION 0x00000C01 +#define TEE_PANIC_ID_TEE_COPYOPERATION 0x00000C02 +#define TEE_PANIC_ID_TEE_FREEOPERATION 0x00000C03 +#define TEE_PANIC_ID_TEE_GETOPERATIONINFO 0x00000C04 +#define TEE_PANIC_ID_TEE_RESETOPERATION 0x00000C05 +#define TEE_PANIC_ID_TEE_SETOPERATIONKEY 0x00000C06 +#define TEE_PANIC_ID_TEE_SETOPERATIONKEY2 0x00000C07 +#define TEE_PANIC_ID_TEE_GETOPERATIONINFOMULTIPLE 0x00000C08 +/* Message Digest */ +#define TEE_PANIC_ID_TEE_DIGESTDOFINAL 0x00000D01 +#define TEE_PANIC_ID_TEE_DIGESTUPDATE 0x00000D02 +/* Symmetric Cipher */ +#define TEE_PANIC_ID_TEE_CIPHERDOFINAL 0x00000E01 +#define TEE_PANIC_ID_TEE_CIPHERINIT 0x00000E02 +#define TEE_PANIC_ID_TEE_CIPHERUPDATE 0x00000E03 +/* MAC */ +#define TEE_PANIC_ID_TEE_MACCOMPAREFINAL 0x00000F01 +#define TEE_PANIC_ID_TEE_MACCOMPUTEFINAL 0x00000F02 +#define TEE_PANIC_ID_TEE_MACINIT 0x00000F03 +#define TEE_PANIC_ID_TEE_MACUPDATE 0x00000F04 +/* Authenticated Encryption */ +#define TEE_PANIC_ID_TEE_AEDECRYPTFINAL 0x00001001 +#define TEE_PANIC_ID_TEE_AEENCRYPTFINAL 0x00001002 +#define TEE_PANIC_ID_TEE_AEINIT 0x00001003 +#define TEE_PANIC_ID_TEE_AEUPDATE 0x00001004 +#define TEE_PANIC_ID_TEE_AEUPDATEAAD 0x00001005 +/* Asymmetric */ +#define TEE_PANIC_ID_TEE_ASYMMETRICDECRYPT 0x00001101 +#define TEE_PANIC_ID_TEE_ASYMMETRICENCRYPT 0x00001102 +#define TEE_PANIC_ID_TEE_ASYMMETRICSIGNDIGEST 0x00001103 +#define TEE_PANIC_ID_TEE_ASYMMETRICVERIFYDIGEST 0x00001104 +/* Key Derivation */ +#define TEE_PANIC_ID_TEE_DERIVEKEY 0x00001201 +/* Random Data Generation */ +#define TEE_PANIC_ID_TEE_GENERATERANDOM 0x00001301 +/* Time */ +#define TEE_PANIC_ID_TEE_GETREETIME 0x00001401 +#define TEE_PANIC_ID_TEE_GETSYSTEMTIME 0x00001402 +#define TEE_PANIC_ID_TEE_GETTAPERSISTENTTIME 0x00001403 +#define TEE_PANIC_ID_TEE_SETTAPERSISTENTTIME 0x00001404 +#define TEE_PANIC_ID_TEE_WAIT 0x00001405 +/* Memory Allocation and Size of Objects */ +#define TEE_PANIC_ID_TEE_BIGINTFMMCONTEXTSIZEINU32 0x00001501 +#define TEE_PANIC_ID_TEE_BIGINTFMMSIZEINU32 0x00001502 +/* Initialization */ +#define TEE_PANIC_ID_TEE_BIGINTINIT 0x00001601 +#define TEE_PANIC_ID_TEE_BIGINTINITFMM 0x00001602 +#define TEE_PANIC_ID_TEE_BIGINTINITFMMCONTEXT 0x00001603 +/* Converter */ +#define TEE_PANIC_ID_TEE_BIGINTCONVERTFROMOCTETSTRING 0x00001701 +#define TEE_PANIC_ID_TEE_BIGINTCONVERTFROMS32 0x00001702 +#define TEE_PANIC_ID_TEE_BIGINTCONVERTTOOCTETSTRING 0x00001703 +#define TEE_PANIC_ID_TEE_BIGINTCONVERTTOS32 0x00001704 +/* Logical Operation */ +#define TEE_PANIC_ID_TEE_BIGINTCMP 0x00001801 +#define TEE_PANIC_ID_TEE_BIGINTCMPS32 0x00001802 +#define TEE_PANIC_ID_TEE_BIGINTGETBIT 0x00001803 +#define TEE_PANIC_ID_TEE_BIGINTGETBITCOUNT 0x00001804 +#define TEE_PANIC_ID_TEE_BIGINTSHIFTRIGHT 0x00001805 +/* Basic Arithmetic */ +#define TEE_PANIC_ID_TEE_BIGINTADD 0x00001901 +#define TEE_PANIC_ID_TEE_BIGINTDIV 0x00001902 +#define TEE_PANIC_ID_TEE_BIGINTMUL 0x00001903 +#define TEE_PANIC_ID_TEE_BIGINTNEG 0x00001904 +#define TEE_PANIC_ID_TEE_BIGINTSQUARE 0x00001905 +#define TEE_PANIC_ID_TEE_BIGINTSUB 0x00001906 +/* Modular Arithmetic */ +#define TEE_PANIC_ID_TEE_BIGINTADDMOD 0x00001A01 +#define TEE_PANIC_ID_TEE_BIGINTINVMOD 0x00001A02 +#define TEE_PANIC_ID_TEE_BIGINTMOD 0x00001A03 +#define TEE_PANIC_ID_TEE_BIGINTMULMOD 0x00001A04 +#define TEE_PANIC_ID_TEE_BIGINTSQUAREMOD 0x00001A05 +#define TEE_PANIC_ID_TEE_BIGINTSUBMOD 0x00001A06 +/* Other Arithmetic */ +#define TEE_PANIC_ID_TEE_BIGINTCOMPUTEEXTENDEDGCD 0x00001B01 +#define TEE_PANIC_ID_TEE_BIGINTISPROBABLEPRIME 0x00001B02 +#define TEE_PANIC_ID_TEE_BIGINTRELATIVEPRIME 0x00001B03 +/* Fast Modular Multiplication */ +#define TEE_PANIC_ID_TEE_BIGINTCOMPUTEFMM 0x00001C01 +#define TEE_PANIC_ID_TEE_BIGINTCONVERTFROMFMM 0x00001C02 +#define TEE_PANIC_ID_TEE_BIGINTCONVERTTOFMM 0x00001C03 + +/* + * The macro TEE_PARAM_TYPES can be used to construct a value that you can + * compare against an incoming paramTypes to check the type of all the + * parameters in one comparison, like in the following example: + * if (paramTypes != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + * TEE_PARAM_TYPE_MEMREF_OUPUT, + * TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE)) { + * return TEE_ERROR_BAD_PARAMETERS; + * } + */ +#define TEE_PARAM_TYPES(t0,t1,t2,t3) \ + ((t0) | ((t1) << 4) | ((t2) << 8) | ((t3) << 12)) + +/* + * The macro TEE_PARAM_TYPE_GET can be used to extract the type of a given + * parameter from paramTypes if you need more fine-grained type checking. + */ +#define TEE_PARAM_TYPE_GET(t, i) ((((uint32_t)t) >> ((i)*4)) & 0xF) + +/* + * The macro TEE_PARAM_TYPE_SET can be used to load the type of a given + * parameter from paramTypes without specifying all types (TEE_PARAM_TYPES) + */ +#define TEE_PARAM_TYPE_SET(t, i) (((uint32_t)(t) & 0xF) << ((i)*4)) + +/* Not specified in the standard */ +#define TEE_NUM_PARAMS 4 + +/* TEE Arithmetical APIs */ + +#define TEE_BigIntSizeInU32(n) ((((n)+31)/32)+2) + +#endif /* TEE_API_DEFINES_H */ diff --git a/optee_os/lib/libutee/include/tee_api_defines_extensions.h b/optee_os/lib/libutee/include/tee_api_defines_extensions.h new file mode 100644 index 0000000..e908b37 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_api_defines_extensions.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2021, Linaro Limited + * Copyright (c) 2021, SumUp Services GmbH + */ + +#ifndef TEE_API_DEFINES_EXTENSIONS_H +#define TEE_API_DEFINES_EXTENSIONS_H + +/* + * RSA signatures with MD5 hash + * Values prefixed with vendor ID bit31 with by TEE bitfields IDs + */ +#define TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5 0xF0111930 +#define TEE_ALG_RSAES_PKCS1_OAEP_MGF1_MD5 0xF0110230 + +/* + * API extended result codes as per TEE_Result IDs defined in GPD TEE + * Internal Core API specification v1.1: + * + * 0x70000000 - 0x7FFFFFFF: Reserved for implementation-specific return + * code providing non-error information + * 0x80000000 - 0x8FFFFFFF: Reserved for implementation-specific errors + * + * TEE_ERROR_DEFER_DRIVER_INIT - Device driver failed to initialize because + * the driver depends on a device not yet initialized. + */ +#define TEE_ERROR_DEFER_DRIVER_INIT 0x80000000 + +/* + * TEE_ERROR_NODE_DISABLED - Device driver failed to initialize because it is + * not allocated for TEE environment. + */ +#define TEE_ERROR_NODE_DISABLED 0x80000001 + +/* + * HMAC-based Extract-and-Expand Key Derivation Function (HKDF) + */ + +#define TEE_ALG_HKDF_MD5_DERIVE_KEY 0x800010C0 +#define TEE_ALG_HKDF_SHA1_DERIVE_KEY 0x800020C0 +#define TEE_ALG_HKDF_SHA224_DERIVE_KEY 0x800030C0 +#define TEE_ALG_HKDF_SHA256_DERIVE_KEY 0x800040C0 +#define TEE_ALG_HKDF_SHA384_DERIVE_KEY 0x800050C0 +#define TEE_ALG_HKDF_SHA512_DERIVE_KEY 0x800060C0 + +#define TEE_TYPE_HKDF_IKM 0xA10000C0 + +#define TEE_ATTR_HKDF_IKM 0xC00001C0 +/* + * There is a name clash with the official attributes TEE_ATTR_HKDF_SALT + * and TEE_ATTR_HKDF_INFO so define these alternative ID. + */ +#define __OPTEE_TEE_ATTR_HKDF_SALT 0xD00002C0 +#define __OPTEE_ATTR_HKDF_INFO 0xD00003C0 +#define TEE_ATTR_HKDF_OKM_LENGTH 0xF00004C0 + +/* + * Concatenation Key Derivation Function (Concat KDF) + * NIST SP 800-56A section 5.8.1 + */ + +#define TEE_ALG_CONCAT_KDF_SHA1_DERIVE_KEY 0x800020C1 +#define TEE_ALG_CONCAT_KDF_SHA224_DERIVE_KEY 0x800030C1 +#define TEE_ALG_CONCAT_KDF_SHA256_DERIVE_KEY 0x800040C1 +#define TEE_ALG_CONCAT_KDF_SHA384_DERIVE_KEY 0x800050C1 +#define TEE_ALG_CONCAT_KDF_SHA512_DERIVE_KEY 0x800060C1 + +#define TEE_TYPE_CONCAT_KDF_Z 0xA10000C1 + +#define TEE_ATTR_CONCAT_KDF_Z 0xC00001C1 +#define TEE_ATTR_CONCAT_KDF_OTHER_INFO 0xD00002C1 +#define TEE_ATTR_CONCAT_KDF_DKM_LENGTH 0xF00003C1 + +/* + * PKCS #5 v2.0 Key Derivation Function 2 (PBKDF2) + * RFC 2898 section 5.2 + * https://www.ietf.org/rfc/rfc2898.txt + */ + +#define TEE_ALG_PBKDF2_HMAC_SHA1_DERIVE_KEY 0x800020C2 + +#define TEE_TYPE_PBKDF2_PASSWORD 0xA10000C2 + +#define TEE_ATTR_PBKDF2_PASSWORD 0xC00001C2 +#define TEE_ATTR_PBKDF2_SALT 0xD00002C2 +#define TEE_ATTR_PBKDF2_ITERATION_COUNT 0xF00003C2 +#define TEE_ATTR_PBKDF2_DKM_LENGTH 0xF00004C2 + +/* + * PKCS#1 v1.5 RSASSA pre-hashed sign/verify + */ + +#define TEE_ALG_RSASSA_PKCS1_V1_5 0xF0000830 + +/* + * TDEA CMAC (NIST SP800-38B) + */ +#define TEE_ALG_DES3_CMAC 0xF0000613 + +/* + * SM4-XTS + */ +#define TEE_ALG_SM4_XTS 0xF0000414 + +/* + * Implementation-specific object storage constants + */ + +/* Storage is provided by the Rich Execution Environment (REE) */ +#define TEE_STORAGE_PRIVATE_REE 0x80000000 +/* Storage is the Replay Protected Memory Block partition of an eMMC device */ +#define TEE_STORAGE_PRIVATE_RPMB 0x80000100 +/* Was TEE_STORAGE_PRIVATE_SQL, which isn't supported any longer */ +#define TEE_STORAGE_PRIVATE_SQL_RESERVED 0x80000200 + +/* + * Extension of "Memory Access Rights Constants" + * #define TEE_MEMORY_ACCESS_READ 0x00000001 + * #define TEE_MEMORY_ACCESS_WRITE 0x00000002 + * #define TEE_MEMORY_ACCESS_ANY_OWNER 0x00000004 + * + * TEE_MEMORY_ACCESS_NONSECURE : if set TEE_CheckMemoryAccessRights() + * successfully returns only if target vmem range is mapped non-secure. + * + * TEE_MEMORY_ACCESS_SECURE : if set TEE_CheckMemoryAccessRights() + * successfully returns only if target vmem range is mapped secure. + + */ +#define TEE_MEMORY_ACCESS_NONSECURE 0x10000000 +#define TEE_MEMORY_ACCESS_SECURE 0x20000000 + +/* + * Implementation-specific login types + */ + +/* Private login method for REE kernel clients */ +#define TEE_LOGIN_REE_KERNEL 0x80000000 + +#endif /* TEE_API_DEFINES_EXTENSIONS_H */ diff --git a/optee_os/lib/libutee/include/tee_api_types.h b/optee_os/lib/libutee/include/tee_api_types.h new file mode 100644 index 0000000..b78acd4 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_api_types.h @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +/* Based on GP TEE Internal API Specification Version 0.11 */ +#ifndef TEE_API_TYPES_H +#define TEE_API_TYPES_H + +#include +#include +#include +#include +#include + +/* + * Common Definitions + */ + +typedef uint32_t TEE_Result; + +typedef struct { + uint32_t timeLow; + uint16_t timeMid; + uint16_t timeHiAndVersion; + uint8_t clockSeqAndNode[8]; +} TEE_UUID; + +/* + * The TEE_Identity structure defines the full identity of a Client: + * - login is one of the TEE_LOGIN_XXX constants + * - uuid contains the client UUID or Nil if not applicable + */ +typedef struct { + uint32_t login; + TEE_UUID uuid; +} TEE_Identity; + +/* + * This union describes one parameter passed by the Trusted Core Framework + * to the entry points TA_OpenSessionEntryPoint or + * TA_InvokeCommandEntryPoint or by the TA to the functions + * TEE_OpenTASession or TEE_InvokeTACommand. + * + * Which of the field value or memref to select is determined by the + * parameter type specified in the argument paramTypes passed to the entry + * point. +*/ +typedef union { + struct { + void *buffer; + size_t size; + } memref; + struct { + uint32_t a; + uint32_t b; + } value; +} TEE_Param; + +typedef union { + struct { + void *buffer; + uint32_t size; + } memref; + struct { + uint32_t a; + uint32_t b; + } value; +} __GP11_TEE_Param; + +/* + * The type of opaque handles on TA Session. These handles are returned by + * the function TEE_OpenTASession. + */ +typedef struct __TEE_TASessionHandle *TEE_TASessionHandle; + +/* + * The type of opaque handles on property sets or enumerators. These + * handles are either one of the pseudo handles TEE_PROPSET_XXX or are + * returned by the function TEE_AllocatePropertyEnumerator. +*/ +typedef struct __TEE_PropSetHandle *TEE_PropSetHandle; + +typedef struct __TEE_ObjectHandle *TEE_ObjectHandle; +typedef struct __TEE_ObjectEnumHandle *TEE_ObjectEnumHandle; +typedef struct __TEE_OperationHandle *TEE_OperationHandle; + +/* + * Storage Definitions + */ + +typedef uint32_t TEE_ObjectType; + +typedef struct { + uint32_t objectType; + uint32_t objectSize; + uint32_t maxObjectSize; + uint32_t objectUsage; + size_t dataSize; + size_t dataPosition; + uint32_t handleFlags; +} TEE_ObjectInfo; + +typedef struct { + uint32_t objectType; + __extension__ union { + uint32_t keySize; /* used in 1.1 spec */ + uint32_t objectSize; /* used in 1.1.1 spec */ + }; + __extension__ union { + uint32_t maxKeySize; /* used in 1.1 spec */ + uint32_t maxObjectSize; /* used in 1.1.1 spec */ + }; + uint32_t objectUsage; + uint32_t dataSize; + uint32_t dataPosition; + uint32_t handleFlags; +} __GP11_TEE_ObjectInfo; + +typedef uint32_t TEE_Whence; + +typedef struct { + uint32_t attributeID; + union { + struct { + void *buffer; + size_t length; + } ref; + struct { + uint32_t a, b; + } value; + } content; +} TEE_Attribute; + +typedef struct { + uint32_t attributeID; + union { + struct { + void *buffer; + uint32_t length; + } ref; + struct { + uint32_t a, b; + } value; + } content; +} __GP11_TEE_Attribute; + +/* Cryptographic Operations API */ + +typedef uint32_t TEE_OperationMode; + +typedef struct { + uint32_t algorithm; + uint32_t operationClass; + uint32_t mode; + uint32_t digestLength; + uint32_t maxKeySize; + uint32_t keySize; + uint32_t requiredKeyUsage; + uint32_t handleState; +} TEE_OperationInfo; + +typedef struct { + uint32_t keySize; + uint32_t requiredKeyUsage; +} TEE_OperationInfoKey; + +typedef struct { + uint32_t algorithm; + uint32_t operationClass; + uint32_t mode; + uint32_t digestLength; + uint32_t maxKeySize; + uint32_t handleState; + uint32_t operationState; + uint32_t numberOfKeys; + TEE_OperationInfoKey keyInformation[]; +} TEE_OperationInfoMultiple; + +/* Time & Date API */ + +typedef struct { + uint32_t seconds; + uint32_t millis; +} TEE_Time; + +/* TEE Arithmetical APIs */ + +typedef uint32_t TEE_BigInt; + +typedef uint32_t TEE_BigIntFMM; + +typedef uint32_t TEE_BigIntFMMContext __aligned(__alignof__(void *)); + +/* Tee Secure Element APIs */ + +typedef struct __TEE_SEServiceHandle *TEE_SEServiceHandle; +typedef struct __TEE_SEReaderHandle *TEE_SEReaderHandle; +typedef struct __TEE_SESessionHandle *TEE_SESessionHandle; +typedef struct __TEE_SEChannelHandle *TEE_SEChannelHandle; + +typedef struct { + bool sePresent; + bool teeOnly; + bool selectResponseEnable; +} TEE_SEReaderProperties; + +typedef struct { + uint8_t *buffer; + size_t bufferLen; +} TEE_SEAID; + +/* Other definitions */ +typedef uint32_t TEE_ErrorOrigin; +typedef void *TEE_Session; + +#define TEE_MEM_INPUT 0x00000001 +#define TEE_MEM_OUTPUT 0x00000002 + +#define TEE_MEMREF_0_USED 0x00000001 +#define TEE_MEMREF_1_USED 0x00000002 +#define TEE_MEMREF_2_USED 0x00000004 +#define TEE_MEMREF_3_USED 0x00000008 + +#define TEE_SE_READER_NAME_MAX 20 + +#endif /* TEE_API_TYPES_H */ diff --git a/optee_os/lib/libutee/include/tee_arith_internal.h b/optee_os/lib/libutee/include/tee_arith_internal.h new file mode 100644 index 0000000..fc5678e --- /dev/null +++ b/optee_os/lib/libutee/include/tee_arith_internal.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef GUARD_TEE_MATHAPI_H +#define GUARD_TEE_MATHAPI_H + +#include /* for size_t */ +#include /* for uint32_t and friends */ +#include /* for bool (!) */ + +/************************************************************* + * + * MACRO DEFINITIONS + * + *************************************************************/ + +/*------------------------------------------------------------ + * + * How functions are exported + * + */ +#define TEE_MATHAPI_EXPORT + +/* + * The modes for String Conversion + */ +#define TEE_STRING_MODE_HEX_UC MPA_STRING_MODE_HEX_UC +#define TEE_STRING_MODE_HEX_LC MPA_STRING_MODE_HEX_UC + +/*------------------------------------------------------------ + * + * Define IN, OUT, INBUF and OUTBUF to keep format from the spec. + * + */ +#define IN const +#define OUT +#define INOUT +#define INBUF const +#define OUTBUF + +/************************************************************* + * + * MEMORY ALLOCATION AND SIZE + * + *************************************************************/ + +/************************************************************* + * + * INITIALIZATION FUNCTIONS + * + *************************************************************/ + +/* + * !!! This function must be called before you do anything else !!! + * NOTE: Not part of the spec + */ +TEE_MATHAPI_EXPORT void _TEE_MathAPI_Init(void); + +#endif diff --git a/optee_os/lib/libutee/include/tee_internal_api.h b/optee_os/lib/libutee/include/tee_internal_api.h new file mode 100644 index 0000000..887b52f --- /dev/null +++ b/optee_os/lib/libutee/include/tee_internal_api.h @@ -0,0 +1,832 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +/* Based on GP TEE Internal API Specification Version 0.27 */ +#ifndef TEE_INTERNAL_API_H +#define TEE_INTERNAL_API_H + +#ifdef __TEE_API_COMPAT_H +#error " must not be included before " +#endif + +#include +#include +#include +#include +#include + +/* Property access functions */ + +TEE_Result TEE_GetPropertyAsString(TEE_PropSetHandle propsetOrEnumerator, + const char *name, char *valueBuffer, + size_t *valueBufferLen); +TEE_Result __GP11_TEE_GetPropertyAsString(TEE_PropSetHandle propsetOrEnumerator, + const char *name, char *valueBuffer, + uint32_t *valueBufferLen); + +TEE_Result TEE_GetPropertyAsBool(TEE_PropSetHandle propsetOrEnumerator, + const char *name, bool *value); + +TEE_Result TEE_GetPropertyAsU32(TEE_PropSetHandle propsetOrEnumerator, + const char *name, uint32_t *value); + +TEE_Result TEE_GetPropertyAsU64(TEE_PropSetHandle propsetOrEnumerator, + const char *name, uint64_t *value); + +TEE_Result TEE_GetPropertyAsBinaryBlock(TEE_PropSetHandle propsetOrEnumerator, + const char *name, void *valueBuffer, + size_t *valueBufferLen); +TEE_Result +__GP11_TEE_GetPropertyAsBinaryBlock(TEE_PropSetHandle propsetOrEnumerator, + const char *name, void *valueBuffer, + uint32_t *valueBufferLen); + +TEE_Result TEE_GetPropertyAsUUID(TEE_PropSetHandle propsetOrEnumerator, + const char *name, TEE_UUID *value); + +TEE_Result TEE_GetPropertyAsIdentity(TEE_PropSetHandle propsetOrEnumerator, + const char *name, TEE_Identity *value); + +TEE_Result TEE_AllocatePropertyEnumerator(TEE_PropSetHandle *enumerator); + +void TEE_FreePropertyEnumerator(TEE_PropSetHandle enumerator); + +void TEE_StartPropertyEnumerator(TEE_PropSetHandle enumerator, + TEE_PropSetHandle propSet); + +void TEE_ResetPropertyEnumerator(TEE_PropSetHandle enumerator); + +TEE_Result TEE_GetPropertyName(TEE_PropSetHandle enumerator, + void *nameBuffer, size_t *nameBufferLen); +TEE_Result __GP11_TEE_GetPropertyName(TEE_PropSetHandle enumerator, + void *nameBuffer, + uint32_t *nameBufferLen); + +TEE_Result TEE_GetNextProperty(TEE_PropSetHandle enumerator); + +/* System API - Misc */ + +void TEE_Panic(TEE_Result panicCode); + +/* System API - Internal Client API */ + +TEE_Result TEE_OpenTASession(const TEE_UUID *destination, + uint32_t cancellationRequestTimeout, + uint32_t paramTypes, + TEE_Param params[TEE_NUM_PARAMS], + TEE_TASessionHandle *session, + uint32_t *returnOrigin); +TEE_Result __GP11_TEE_OpenTASession(const TEE_UUID *destination, + uint32_t cancellationRequestTimeout, + uint32_t paramTypes, + __GP11_TEE_Param params[TEE_NUM_PARAMS], + TEE_TASessionHandle *session, + uint32_t *returnOrigin); + +void TEE_CloseTASession(TEE_TASessionHandle session); + +TEE_Result TEE_InvokeTACommand(TEE_TASessionHandle session, + uint32_t cancellationRequestTimeout, + uint32_t commandID, uint32_t paramTypes, + TEE_Param params[TEE_NUM_PARAMS], + uint32_t *returnOrigin); +TEE_Result __GP11_TEE_InvokeTACommand(TEE_TASessionHandle session, + uint32_t cancellationRequestTimeout, + uint32_t commandID, uint32_t paramTypes, + __GP11_TEE_Param params[TEE_NUM_PARAMS], + uint32_t *returnOrigin); + +/* System API - Cancellations */ + +bool TEE_GetCancellationFlag(void); + +bool TEE_UnmaskCancellation(void); + +bool TEE_MaskCancellation(void); + +/* System API - Memory Management */ + +TEE_Result TEE_CheckMemoryAccessRights(uint32_t accessFlags, void *buffer, + size_t size); +TEE_Result __GP11_TEE_CheckMemoryAccessRights(uint32_t accessFlags, + void *buffer, uint32_t size); + +void TEE_SetInstanceData(const void *instanceData); + +const void *TEE_GetInstanceData(void); + +void *TEE_Malloc(size_t size, uint32_t hint); +void *__GP11_TEE_Malloc(uint32_t size, uint32_t hint); + +void *TEE_Realloc(void *buffer, size_t newSize); +void *__GP11_TEE_Realloc(void *buffer, uint32_t newSize); + +void TEE_Free(void *buffer); + +void *TEE_MemMove(void *dest, const void *src, size_t size); +void *__GP11_TEE_MemMove(void *dest, const void *src, uint32_t size); + +/* + * Note: TEE_MemCompare() has a constant-time implementation (execution time + * does not depend on buffer content but only on buffer size). It is the main + * difference with memcmp(). + */ +int32_t TEE_MemCompare(const void *buffer1, const void *buffer2, size_t size); +int32_t __GP11_TEE_MemCompare(const void *buffer1, const void *buffer2, + uint32_t size); + +void TEE_MemFill(void *buff, uint32_t x, size_t size); +void __GP11_TEE_MemFill(void *buff, uint32_t x, uint32_t size); + +/* Data and Key Storage API - Generic Object Functions */ + +void TEE_GetObjectInfo(TEE_ObjectHandle object, TEE_ObjectInfo *objectInfo); +void __GP11_TEE_GetObjectInfo(TEE_ObjectHandle object, + __GP11_TEE_ObjectInfo *objectInfo); + +TEE_Result TEE_GetObjectInfo1(TEE_ObjectHandle object, + TEE_ObjectInfo *objectInfo); +TEE_Result __GP11_TEE_GetObjectInfo1(TEE_ObjectHandle object, + __GP11_TEE_ObjectInfo *objectInfo); + +void TEE_RestrictObjectUsage(TEE_ObjectHandle object, uint32_t objectUsage); +TEE_Result TEE_RestrictObjectUsage1(TEE_ObjectHandle object, + uint32_t objectUsage); + +TEE_Result TEE_GetObjectBufferAttribute(TEE_ObjectHandle object, + uint32_t attributeID, void *buffer, + size_t *size); +TEE_Result __GP11_TEE_GetObjectBufferAttribute(TEE_ObjectHandle object, + uint32_t attributeID, + void *buffer, uint32_t *size); + +TEE_Result TEE_GetObjectValueAttribute(TEE_ObjectHandle object, + uint32_t attributeID, uint32_t *a, + uint32_t *b); + +void TEE_CloseObject(TEE_ObjectHandle object); + +/* Data and Key Storage API - Transient Object Functions */ + +TEE_Result TEE_AllocateTransientObject(TEE_ObjectType objectType, + uint32_t maxObjectSize, + TEE_ObjectHandle *object); +TEE_Result __GP11_TEE_AllocateTransientObject(TEE_ObjectType objectType, + uint32_t maxKeySize, + TEE_ObjectHandle *object); + +void TEE_FreeTransientObject(TEE_ObjectHandle object); + +void TEE_ResetTransientObject(TEE_ObjectHandle object); + +TEE_Result TEE_PopulateTransientObject(TEE_ObjectHandle object, + const TEE_Attribute *attrs, + uint32_t attrCount); +TEE_Result __GP11_TEE_PopulateTransientObject(TEE_ObjectHandle object, + const __GP11_TEE_Attribute *attrs, + uint32_t attrCount); + +void TEE_InitRefAttribute(TEE_Attribute *attr, uint32_t attributeID, + const void *buffer, size_t length); +void __GP11_TEE_InitRefAttribute(__GP11_TEE_Attribute *attr, + uint32_t attributeID, + const void *buffer, uint32_t length); + +void TEE_InitValueAttribute(TEE_Attribute *attr, uint32_t attributeID, + uint32_t a, uint32_t b); +void __GP11_TEE_InitValueAttribute(__GP11_TEE_Attribute *attr, + uint32_t attributeID, + uint32_t a, uint32_t b); + +void TEE_CopyObjectAttributes(TEE_ObjectHandle destObject, + TEE_ObjectHandle srcObject); + +TEE_Result TEE_CopyObjectAttributes1(TEE_ObjectHandle destObject, + TEE_ObjectHandle srcObject); + +TEE_Result TEE_GenerateKey(TEE_ObjectHandle object, uint32_t keySize, + const TEE_Attribute *params, uint32_t paramCount); +TEE_Result __GP11_TEE_GenerateKey(TEE_ObjectHandle object, uint32_t keySize, + const __GP11_TEE_Attribute *params, + uint32_t paramCount); + +/* Data and Key Storage API - Persistent Object Functions */ + +TEE_Result TEE_OpenPersistentObject(uint32_t storageID, const void *objectID, + size_t objectIDLen, uint32_t flags, + TEE_ObjectHandle *object); +TEE_Result __GP11_TEE_OpenPersistentObject(uint32_t storageID, + const void *objectID, + uint32_t objectIDLen, uint32_t flags, + TEE_ObjectHandle *object); + +TEE_Result TEE_CreatePersistentObject(uint32_t storageID, const void *objectID, + size_t objectIDLen, uint32_t flags, + TEE_ObjectHandle attributes, + const void *initialData, + size_t initialDataLen, + TEE_ObjectHandle *object); +TEE_Result __GP11_TEE_CreatePersistentObject(uint32_t storageID, + const void *objectID, + uint32_t objectIDLen, + uint32_t flags, + TEE_ObjectHandle attributes, + const void *initialData, + uint32_t initialDataLen, + TEE_ObjectHandle *object); + +void TEE_CloseAndDeletePersistentObject(TEE_ObjectHandle object); + +TEE_Result TEE_CloseAndDeletePersistentObject1(TEE_ObjectHandle object); + +TEE_Result TEE_RenamePersistentObject(TEE_ObjectHandle object, + const void *newObjectID, + size_t newObjectIDLen); +TEE_Result __GP11_TEE_RenamePersistentObject(TEE_ObjectHandle object, + const void *newObjectID, + uint32_t newObjectIDLen); + +TEE_Result TEE_AllocatePersistentObjectEnumerator(TEE_ObjectEnumHandle * + objectEnumerator); + +void TEE_FreePersistentObjectEnumerator(TEE_ObjectEnumHandle objectEnumerator); + +void TEE_ResetPersistentObjectEnumerator(TEE_ObjectEnumHandle objectEnumerator); + +TEE_Result TEE_StartPersistentObjectEnumerator(TEE_ObjectEnumHandle + objectEnumerator, + uint32_t storageID); + +TEE_Result TEE_GetNextPersistentObject(TEE_ObjectEnumHandle objectEnumerator, + TEE_ObjectInfo *objectInfo, + void *objectID, size_t *objectIDLen); +TEE_Result +__GP11_TEE_GetNextPersistentObject(TEE_ObjectEnumHandle objectEnumerator, + __GP11_TEE_ObjectInfo *objectInfo, + void *objectID, uint32_t *objectIDLen); + +/* Data and Key Storage API - Data Stream Access Functions */ + +TEE_Result TEE_ReadObjectData(TEE_ObjectHandle object, void *buffer, + size_t size, size_t *count); +TEE_Result __GP11_TEE_ReadObjectData(TEE_ObjectHandle object, void *buffer, + uint32_t size, uint32_t *count); + +TEE_Result TEE_WriteObjectData(TEE_ObjectHandle object, const void *buffer, + size_t size); +TEE_Result __GP11_TEE_WriteObjectData(TEE_ObjectHandle object, + const void *buffer, uint32_t size); + +TEE_Result TEE_TruncateObjectData(TEE_ObjectHandle object, size_t size); +TEE_Result __GP11_TEE_TruncateObjectData(TEE_ObjectHandle object, + uint32_t size); + +TEE_Result TEE_SeekObjectData(TEE_ObjectHandle object, intmax_t offset, + TEE_Whence whence); +TEE_Result __GP11_TEE_SeekObjectData(TEE_ObjectHandle object, int32_t offset, + TEE_Whence whence); + +/* Cryptographic Operations API - Generic Operation Functions */ + +TEE_Result TEE_AllocateOperation(TEE_OperationHandle *operation, + uint32_t algorithm, uint32_t mode, + uint32_t maxKeySize); + +void TEE_FreeOperation(TEE_OperationHandle operation); +void __GP11_TEE_FreeOperation(TEE_OperationHandle operation); + +void TEE_GetOperationInfo(TEE_OperationHandle operation, + TEE_OperationInfo *operationInfo); + +TEE_Result +TEE_GetOperationInfoMultiple(TEE_OperationHandle operation, + TEE_OperationInfoMultiple *operationInfoMultiple, + size_t *operationSize); +TEE_Result +__GP11_TEE_GetOperationInfoMultiple(TEE_OperationHandle operation, + TEE_OperationInfoMultiple *info, + uint32_t *operationSize); + +void TEE_ResetOperation(TEE_OperationHandle operation); + +TEE_Result TEE_SetOperationKey(TEE_OperationHandle operation, + TEE_ObjectHandle key); +TEE_Result __GP11_TEE_SetOperationKey(TEE_OperationHandle operation, + TEE_ObjectHandle key); + +TEE_Result TEE_SetOperationKey2(TEE_OperationHandle operation, + TEE_ObjectHandle key1, TEE_ObjectHandle key2); +TEE_Result __GP11_TEE_SetOperationKey2(TEE_OperationHandle operation, + TEE_ObjectHandle key1, + TEE_ObjectHandle key2); + +void TEE_CopyOperation(TEE_OperationHandle dstOperation, + TEE_OperationHandle srcOperation); + +TEE_Result TEE_IsAlgorithmSupported(uint32_t algId, uint32_t element); + +/* Cryptographic Operations API - Message Digest Functions */ + +void TEE_DigestUpdate(TEE_OperationHandle operation, + const void *chunk, size_t chunkSize); +void __GP11_TEE_DigestUpdate(TEE_OperationHandle operation, + const void *chunk, uint32_t chunkSize); + +TEE_Result TEE_DigestDoFinal(TEE_OperationHandle operation, const void *chunk, + size_t chunkLen, void *hash, size_t *hashLen); +TEE_Result __GP11_TEE_DigestDoFinal(TEE_OperationHandle operation, + const void *chunk, uint32_t chunkLen, + void *hash, uint32_t *hashLen); + +TEE_Result TEE_DigestExtract(TEE_OperationHandle operation, void *hash, + size_t *hashLen); + +/* Cryptographic Operations API - Symmetric Cipher Functions */ + +void TEE_CipherInit(TEE_OperationHandle operation, const void *IV, + size_t IVLen); +void __GP11_TEE_CipherInit(TEE_OperationHandle operation, const void *IV, + uint32_t IVLen); + +TEE_Result TEE_CipherUpdate(TEE_OperationHandle operation, const void *srcData, + size_t srcLen, void *destData, size_t *destLen); +TEE_Result __GP11_TEE_CipherUpdate(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen); + +TEE_Result TEE_CipherDoFinal(TEE_OperationHandle operation, + const void *srcData, size_t srcLen, + void *destData, size_t *destLen); +TEE_Result __GP11_TEE_CipherDoFinal(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen); + +/* Cryptographic Operations API - MAC Functions */ + +void TEE_MACInit(TEE_OperationHandle operation, const void *IV, + size_t IVLen); +void __GP11_TEE_MACInit(TEE_OperationHandle operation, const void *IV, + uint32_t IVLen); + +void TEE_MACUpdate(TEE_OperationHandle operation, const void *chunk, + size_t chunkSize); +void __GP11_TEE_MACUpdate(TEE_OperationHandle operation, const void *chunk, + uint32_t chunkSize); + +TEE_Result TEE_MACComputeFinal(TEE_OperationHandle operation, + const void *message, size_t messageLen, + void *mac, size_t *macLen); +TEE_Result __GP11_TEE_MACComputeFinal(TEE_OperationHandle operation, + const void *message, uint32_t messageLen, + void *mac, uint32_t *macLen); + +TEE_Result TEE_MACCompareFinal(TEE_OperationHandle operation, + const void *message, size_t messageLen, + const void *mac, size_t macLen); +TEE_Result __GP11_TEE_MACCompareFinal(TEE_OperationHandle operation, + const void *message, uint32_t messageLen, + const void *mac, uint32_t macLen); + +/* Cryptographic Operations API - Authenticated Encryption Functions */ + +TEE_Result TEE_AEInit(TEE_OperationHandle operation, const void *nonce, + size_t nonceLen, uint32_t tagLen, size_t AADLen, + size_t payloadLen); +TEE_Result __GP11_TEE_AEInit(TEE_OperationHandle operation, const void *nonce, + uint32_t nonceLen, uint32_t tagLen, + uint32_t AADLen, uint32_t payloadLen); + +void TEE_AEUpdateAAD(TEE_OperationHandle operation, const void *AADdata, + size_t AADdataLen); +void __GP11_TEE_AEUpdateAAD(TEE_OperationHandle operation, const void *AADdata, + uint32_t AADdataLen); + +TEE_Result TEE_AEUpdate(TEE_OperationHandle operation, const void *srcData, + size_t srcLen, void *destData, size_t *destLen); +TEE_Result __GP11_TEE_AEUpdate(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen); + +TEE_Result TEE_AEEncryptFinal(TEE_OperationHandle operation, + const void *srcData, size_t srcLen, + void *destData, size_t *destLen, void *tag, + size_t *tagLen); +TEE_Result __GP11_TEE_AEEncryptFinal(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen, + void *tag, uint32_t *tagLen); + +TEE_Result TEE_AEDecryptFinal(TEE_OperationHandle operation, + const void *srcData, size_t srcLen, + void *destData, size_t *destLen, void *tag, + size_t tagLen); +TEE_Result __GP11_TEE_AEDecryptFinal(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen, + void *tag, uint32_t tagLen); + +/* Cryptographic Operations API - Asymmetric Functions */ + +TEE_Result TEE_AsymmetricEncrypt(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *srcData, + size_t srcLen, void *destData, + size_t *destLen); +TEE_Result __GP11_TEE_AsymmetricEncrypt(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen); + +TEE_Result TEE_AsymmetricDecrypt(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *srcData, + size_t srcLen, void *destData, + size_t *destLen); +TEE_Result __GP11_TEE_AsymmetricDecrypt(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen); + +TEE_Result TEE_AsymmetricSignDigest(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *digest, + size_t digestLen, void *signature, + size_t *signatureLen); +TEE_Result __GP11_TEE_AsymmetricSignDigest(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *digest, + uint32_t digestLen, void *signature, + uint32_t *signatureLen); + +TEE_Result TEE_AsymmetricVerifyDigest(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *digest, + size_t digestLen, const void *signature, + size_t signatureLen); +TEE_Result __GP11_TEE_AsymmetricVerifyDigest(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *digest, + uint32_t digestLen, + const void *signature, + uint32_t signatureLen); + +/* Cryptographic Operations API - Key Derivation Functions */ + +void TEE_DeriveKey(TEE_OperationHandle operation, + const TEE_Attribute *params, uint32_t paramCount, + TEE_ObjectHandle derivedKey); +void __GP11_TEE_DeriveKey(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, TEE_ObjectHandle derivedKey); + +/* Cryptographic Operations API - Random Number Generation Functions */ + +void TEE_GenerateRandom(void *randomBuffer, size_t randomBufferLen); +void __GP11_TEE_GenerateRandom(void *randomBuffer, uint32_t randomBufferLen); + +/* Date & Time API */ + +void TEE_GetSystemTime(TEE_Time *time); + +TEE_Result TEE_Wait(uint32_t timeout); + +TEE_Result TEE_GetTAPersistentTime(TEE_Time *time); + +TEE_Result TEE_SetTAPersistentTime(const TEE_Time *time); + +void TEE_GetREETime(TEE_Time *time); + +/* TEE Arithmetical API - Memory allocation and size of objects */ + +size_t TEE_BigIntFMMSizeInU32(size_t modulusSizeInBits); +uint32_t __GP11_TEE_BigIntFMMSizeInU32(uint32_t modulusSizeInBits); + +size_t TEE_BigIntFMMContextSizeInU32(size_t modulusSizeInBits); +uint32_t __GP11_TEE_BigIntFMMContextSizeInU32(uint32_t modulusSizeInBits); + +/* TEE Arithmetical API - Initialization functions */ + +void TEE_BigIntInit(TEE_BigInt *bigInt, size_t len); +void __GP11_TEE_BigIntInit(TEE_BigInt *bigInt, uint32_t len); + +void TEE_BigIntInitFMMContext(TEE_BigIntFMMContext *context, size_t len, + const TEE_BigInt *modulus); +void __GP11_TEE_BigIntInitFMMContext(TEE_BigIntFMMContext *context, + uint32_t len, const TEE_BigInt *modulus); + +TEE_Result TEE_BigIntInitFMMContext1(TEE_BigIntFMMContext *context, + size_t len, const TEE_BigInt *modulus); + +void TEE_BigIntInitFMM(TEE_BigIntFMM *bigIntFMM, size_t len); +void __GP11_TEE_BigIntInitFMM(TEE_BigIntFMM *bigIntFMM, uint32_t len); + +/* TEE Arithmetical API - Converter functions */ + +TEE_Result TEE_BigIntConvertFromOctetString(TEE_BigInt *dest, + const uint8_t *buffer, + size_t bufferLen, + int32_t sign); +TEE_Result __GP11_TEE_BigIntConvertFromOctetString(TEE_BigInt *dest, + const uint8_t *buffer, + uint32_t bufferLen, + int32_t sign); + +TEE_Result TEE_BigIntConvertToOctetString(uint8_t *buffer, size_t *bufferLen, + const TEE_BigInt *bigInt); +TEE_Result __GP11_TEE_BigIntConvertToOctetString(uint8_t *buffer, + uint32_t *bufferLen, + const TEE_BigInt *bigInt); + +void TEE_BigIntConvertFromS32(TEE_BigInt *dest, int32_t shortVal); + +TEE_Result TEE_BigIntConvertToS32(int32_t *dest, const TEE_BigInt *src); + +/* TEE Arithmetical API - Logical operations */ + +int32_t TEE_BigIntCmp(const TEE_BigInt *op1, const TEE_BigInt *op2); + +int32_t TEE_BigIntCmpS32(const TEE_BigInt *op, int32_t shortVal); + +void TEE_BigIntShiftRight(TEE_BigInt *dest, const TEE_BigInt *op, + size_t bits); +void __GP11_TEE_BigIntShiftRight(TEE_BigInt *dest, const TEE_BigInt *op, + uint32_t bits); + +bool TEE_BigIntGetBit(const TEE_BigInt *src, uint32_t bitIndex); + +uint32_t TEE_BigIntGetBitCount(const TEE_BigInt *src); + +TEE_Result TEE_BigIntSetBit(TEE_BigInt *op, uint32_t bitIndex, bool value); + +TEE_Result TEE_BigIntAssign(TEE_BigInt *dest, const TEE_BigInt *src); + +TEE_Result TEE_BigIntAbs(TEE_BigInt *dest, const TEE_BigInt *src); + +void TEE_BigIntAdd(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2); + +void TEE_BigIntSub(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2); + +void TEE_BigIntNeg(TEE_BigInt *dest, const TEE_BigInt *op); + +void TEE_BigIntMul(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2); + +void TEE_BigIntSquare(TEE_BigInt *dest, const TEE_BigInt *op); + +void TEE_BigIntDiv(TEE_BigInt *dest_q, TEE_BigInt *dest_r, + const TEE_BigInt *op1, const TEE_BigInt *op2); + +/* TEE Arithmetical API - Modular arithmetic operations */ + +void TEE_BigIntMod(TEE_BigInt *dest, const TEE_BigInt *op, + const TEE_BigInt *n); + +void TEE_BigIntAddMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n); + +void TEE_BigIntSubMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n); + +void TEE_BigIntMulMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n); + +void TEE_BigIntSquareMod(TEE_BigInt *dest, const TEE_BigInt *op, + const TEE_BigInt *n); + +void TEE_BigIntInvMod(TEE_BigInt *dest, const TEE_BigInt *op, + const TEE_BigInt *n); + +TEE_Result TEE_BigIntExpMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n, + const TEE_BigIntFMMContext *context); + +/* TEE Arithmetical API - Other arithmetic operations */ + +bool TEE_BigIntRelativePrime(const TEE_BigInt *op1, const TEE_BigInt *op2); + +void TEE_BigIntComputeExtendedGcd(TEE_BigInt *gcd, TEE_BigInt *u, + TEE_BigInt *v, const TEE_BigInt *op1, + const TEE_BigInt *op2); + +int32_t TEE_BigIntIsProbablePrime(const TEE_BigInt *op, + uint32_t confidenceLevel); + +/* TEE Arithmetical API - Fast modular multiplication operations */ + +void TEE_BigIntConvertToFMM(TEE_BigIntFMM *dest, const TEE_BigInt *src, + const TEE_BigInt *n, + const TEE_BigIntFMMContext *context); + +void TEE_BigIntConvertFromFMM(TEE_BigInt *dest, const TEE_BigIntFMM *src, + const TEE_BigInt *n, + const TEE_BigIntFMMContext *context); + +void TEE_BigIntFMMConvertToBigInt(TEE_BigInt *dest, const TEE_BigIntFMM *src, + const TEE_BigInt *n, + const TEE_BigIntFMMContext *context); + +void TEE_BigIntComputeFMM(TEE_BigIntFMM *dest, const TEE_BigIntFMM *op1, + const TEE_BigIntFMM *op2, const TEE_BigInt *n, + const TEE_BigIntFMMContext *context); + +#define TA_EXPORT + +/* + * TA Interface + * + * Each Trusted Application must provide the Implementation with a number + * of functions, collectively called the “TA interface”. These functions + * are the entry points called by the Trusted Core Framework to create the + * instance, notify the instance that a new client is connecting, notify + * the instance when the client invokes a command, etc. + * + * Trusted Application Entry Points: + */ + +/* + * The function TA_CreateEntryPoint is the Trusted Application's + * constructor, which the Framework calls when it creates a new instance of + * the Trusted Application. To register instance data, the implementation + * of this constructor can use either global variables or the function + * TEE_InstanceSetData. + * + * Return Value: + * - TEE_SUCCESS: if the instance is successfully created, the function + * must return TEE_SUCCESS. + * - Any other value: if any other code is returned the instance is not + * created, and no other entry points of this instance will be called. + * The Framework MUST reclaim all resources and dereference all objects + * related to the creation of the instance. + * + * If this entry point was called as a result of a client opening a + * session, the error code is returned to the client and the session is + * not opened. + */ +TEE_Result TA_EXPORT TA_CreateEntryPoint(void); + +/* + * The function TA_DestroyEntryPoint is the Trusted Application‟s + * destructor, which the Framework calls when the instance is being + * destroyed. + * + * When the function TA_DestroyEntryPoint is called, the Framework + * guarantees that no client session is currently open. Once the call to + * TA_DestroyEntryPoint has been completed, no other entry point of this + * instance will ever be called. + * + * Note that when this function is called, all resources opened by the + * instance are still available. It is only after the function returns that + * the Implementation MUST start automatically reclaiming resources left + * opened. + * + * Return Value: + * This function can return no success or error code. After this function + * returns the Implementation MUST consider the instance destroyed and + * reclaims all resources left open by the instance. + */ +void TA_EXPORT TA_DestroyEntryPoint(void); + +/* + * The Framework calls the function TA_OpenSessionEntryPoint when a client + * requests to open a session with the Trusted Application. The open + * session request may result in a new Trusted Application instance being + * created as defined in section 4.5. + * + * The client can specify parameters in an open operation which are passed + * to the Trusted Application instance in the arguments paramTypes and + * params. These arguments can also be used by the Trusted Application + * instance to transfer response data back to the client. See section 4.3.6 + * for a specification of how to handle the operation parameters. + * + * If this function returns TEE_SUCCESS, the client is connected to a + * Trusted Application instance and can invoke Trusted Application + * commands. When the client disconnects, the Framework will eventually + * call the TA_CloseSessionEntryPoint entry point. + * + * If the function returns any error, the Framework rejects the connection + * and returns the error code and the current content of the parameters the + * client. The return origin is then set to TEE_ORIGIN_TRUSTED_APP. + * + * The Trusted Application instance can register a session data pointer by + * setting *psessionContext. The value of this pointer is not interpreted + * by the Framework, and is simply passed back to other TA_ functions + * within this session. Note that *sessionContext may be set with a pointer + * to a memory allocated by the Trusted Application instance or with + * anything else, like an integer, a handle etc. The Framework will not + * automatically free *sessionContext when the session is closed; the + * Trusted Application instance is responsible for freeing memory if + * required. + * + * During the call to TA_OpenSessionEntryPoint the client may request to + * cancel the operation. See section 4.10 for more details on + * cancellations. If the call to TA_OpenSessionEntryPoint returns + * TEE_SUCCESS, the client must consider the session as successfully opened + * and explicitly close it if necessary. + * + * Parameters: + * - paramTypes: the types of the four parameters. + * - params: a pointer to an array of four parameters. + * - sessionContext: A pointer to a variable that can be filled by the + * Trusted Application instance with an opaque void* data pointer + * + * Return Value: + * - TEE_SUCCESS if the session is successfully opened. + * - Any other value if the session could not be open. + * o The error code may be one of the pre-defined codes, or may be a new + * error code defined by the Trusted Application implementation itself. + */ +TEE_Result TA_EXPORT TA_OpenSessionEntryPoint(uint32_t paramTypes, + TEE_Param params[TEE_NUM_PARAMS], + void **sessionContext); +TEE_Result TA_EXPORT +__GP11_TA_OpenSessionEntryPoint(uint32_t paramTypes, + __GP11_TEE_Param params[TEE_NUM_PARAMS], + void **sessionContext); + +/* + * The Framework calls this function to close a client session. During the + * call to this function the implementation can use any session functions. + * + * The Trusted Application implementation is responsible for freeing any + * resources consumed by the session being closed. Note that the Trusted + * Application cannot refuse to close a session, but can hold the closing + * until it returns from TA_CloseSessionEntryPoint. This is why this + * function cannot return an error code. + * + * Parameters: + * - sessionContext: The value of the void* opaque data pointer set by the + * Trusted Application in the function TA_OpenSessionEntryPoint for this + * session. + */ +void TA_EXPORT TA_CloseSessionEntryPoint(void *sessionContext); + +/* + * The Framework calls this function when the client invokes a command + * within the given session. + * + * The Trusted Application can access the parameters sent by the client + * through the paramTypes and params arguments. It can also use these + * arguments to transfer response data back to the client. + * + * During the call to TA_InvokeCommandEntryPoint the client may request to + * cancel the operation. + * + * A command is always invoked within the context of a client session. + * Thus, any session function can be called by the command implementation. + * + * Parameter: + * - sessionContext: The value of the void* opaque data pointer set by the + * Trusted Application in the function TA_OpenSessionEntryPoint. + * - commandID: A Trusted Application-specific code that identifies the + * command to be invoked. + * - paramTypes: the types of the four parameters. + * - params: a pointer to an array of four parameters. + * + * Return Value: + * - TEE_SUCCESS: if the command is successfully executed, the function + * must return this value. + * - Any other value: if the invocation of the command fails for any + * reason. + * o The error code may be one of the pre-defined codes, or may be a new + * error code defined by the Trusted Application implementation itself. + */ + +TEE_Result TA_EXPORT +TA_InvokeCommandEntryPoint(void *sessionContext, uint32_t commandID, + uint32_t paramTypes, + TEE_Param params[TEE_NUM_PARAMS]); +TEE_Result TA_EXPORT +__GP11_TA_InvokeCommandEntryPoint(void *sessionContext, uint32_t commandID, + uint32_t paramTypes, + __GP11_TEE_Param params[TEE_NUM_PARAMS]); + +/* + * Matching Client Functions <--> TA Functions + * + * TEE_OpenSession or TEE_OpenTASession: + * If a new Trusted Application instance is needed to handle the session, + * TA_CreateEntryPoint is called. + * Then, TA_OpenSessionEntryPoint is called. + * + * TEE_InvokeCommand or TEE_InvokeTACommand: + * TA_InvokeCommandEntryPoint is called. + * + * TEE_CloseSession or TEE_CloseTASession: + * TA_CloseSessionEntryPoint is called. + * For a multi-instance TA or for a single-instance, non keep-alive TA, if + * the session closed was the last session on the instance, then + * TA_DestroyEntryPoint is called. Otherwise, the instance is kept until + * the TEE shuts down. + */ + +#include + +#endif /*TEE_INTERNAL_API_H*/ diff --git a/optee_os/lib/libutee/include/tee_internal_api_extensions.h b/optee_os/lib/libutee/include/tee_internal_api_extensions.h new file mode 100644 index 0000000..78b0e39 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_internal_api_extensions.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#ifndef TEE_INTERNAL_API_EXTENSIONS_H +#define TEE_INTERNAL_API_EXTENSIONS_H + +/* trace support */ +#include +#include +#include +#include + +void tee_user_mem_mark_heap(void); +size_t tee_user_mem_check_heap(void); +/* Hint implementation defines */ +#define TEE_USER_MEM_HINT_NO_FILL_ZERO 0x80000000 + +/* + * Cache maintenance support (TA requires the CACHE_MAINTENANCE property) + * + * TEE_CacheClean() Write back to memory any dirty data cache lines. The line + * is marked as not dirty. The valid bit is unchanged. + * + * TEE_CacheFlush() Purges any valid data cache lines. Any dirty cache lines + * are first written back to memory, then the cache line is + * invalidated. + * + * TEE_CacheInvalidate() Invalidate any valid data cache lines. Any dirty line + * are not written back to memory. + */ +TEE_Result TEE_CacheClean(char *buf, size_t len); +TEE_Result TEE_CacheFlush(char *buf, size_t len); +TEE_Result TEE_CacheInvalidate(char *buf, size_t len); + +/* + * tee_map_zi() - Map zero initialized memory + * @len: Number of bytes + * @flags: 0 or TEE_MEMORY_ACCESS_ANY_OWNER to allow sharing with other TAs + * + * Returns valid pointer on success or NULL on error. + */ +void *tee_map_zi(size_t len, uint32_t flags); + +/* + * tee_unmap() - Unmap previously mapped memory + * @buf: Buffer + * @len: Number of bytes + * + * Note that supplied @buf and @len has to match exactly what has + * previously been returned by tee_map_zi(). + * + * Return TEE_SUCCESS on success or TEE_ERRROR_* on failure. + */ +TEE_Result tee_unmap(void *buf, size_t len); + +/* + * Convert a UUID string @s into a TEE_UUID @uuid + * Expected format for @s is: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + * 'x' being any hexadecimal digit (0-9a-fA-F) + */ +TEE_Result tee_uuid_from_str(TEE_UUID *uuid, const char *s); + +/* + * tee_invoke_supp_plugin() - invoke a tee-supplicant's plugin + * @uuid: uuid of the plugin + * @cmd: command for the plugin + * @sub_cmd: subcommand for the plugin + * @buf: data [for/from] the plugin [in/out] + * @len: length of the input buf + * @outlen: pointer to length of the output data (if they will be used) + * + * Return TEE_SUCCESS on success or TEE_ERRROR_* on failure. + */ +TEE_Result tee_invoke_supp_plugin(const TEE_UUID *uuid, uint32_t cmd, + uint32_t sub_cmd, void *buf, size_t len, + size_t *outlen); + +#endif diff --git a/optee_os/lib/libutee/include/tee_isocket.h b/optee_os/lib/libutee/include/tee_isocket.h new file mode 100644 index 0000000..d87c204 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_isocket.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef __TEE_ISOCKET_H +#define __TEE_ISOCKET_H + +#include +#include +#include <__tee_isocket_defines.h> + +typedef void *TEE_iSocketHandle; + +typedef const struct TEE_iSocket_s { + uint32_t TEE_iSocketVersion; + uint8_t protocolID; + TEE_Result (*open)(TEE_iSocketHandle *ctx, void *setup, + uint32_t *protocolError); + + TEE_Result (*close)(TEE_iSocketHandle ctx); + + TEE_Result (*send)(TEE_iSocketHandle ctx, const void *buf, + uint32_t *length, uint32_t timeout); + + TEE_Result (*recv)(TEE_iSocketHandle ctx, void *buf, uint32_t *length, + uint32_t timeout); + + uint32_t (*error)(TEE_iSocketHandle ctx); + + TEE_Result (*ioctl)(TEE_iSocketHandle ctx, uint32_t commandCode, + void *buf, uint32_t *length); +} TEE_iSocket; + +#endif /*__TEE_ISOCKET_H*/ diff --git a/optee_os/lib/libutee/include/tee_syscall_numbers.h b/optee_os/lib/libutee/include/tee_syscall_numbers.h new file mode 100644 index 0000000..82d429b --- /dev/null +++ b/optee_os/lib/libutee/include/tee_syscall_numbers.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef TEE_SYSCALL_NUMBERS_H +#define TEE_SYSCALL_NUMBERS_H + +#define TEE_SCN_RETURN 0 +#define TEE_SCN_LOG 1 +#define TEE_SCN_PANIC 2 +#define TEE_SCN_GET_PROPERTY 3 +#define TEE_SCN_GET_PROPERTY_NAME_TO_INDEX 4 +#define TEE_SCN_OPEN_TA_SESSION 5 +#define TEE_SCN_CLOSE_TA_SESSION 6 +#define TEE_SCN_INVOKE_TA_COMMAND 7 +#define TEE_SCN_CHECK_ACCESS_RIGHTS 8 +#define TEE_SCN_GET_CANCELLATION_FLAG 9 +#define TEE_SCN_UNMASK_CANCELLATION 10 +#define TEE_SCN_MASK_CANCELLATION 11 +#define TEE_SCN_WAIT 12 +#define TEE_SCN_GET_TIME 13 +#define TEE_SCN_SET_TA_TIME 14 +#define TEE_SCN_CRYP_STATE_ALLOC 15 +#define TEE_SCN_CRYP_STATE_COPY 16 +#define TEE_SCN_CRYP_STATE_FREE 17 +#define TEE_SCN_HASH_INIT 18 +#define TEE_SCN_HASH_UPDATE 19 +#define TEE_SCN_HASH_FINAL 20 +#define TEE_SCN_CIPHER_INIT 21 +#define TEE_SCN_CIPHER_UPDATE 22 +#define TEE_SCN_CIPHER_FINAL 23 +#define TEE_SCN_CRYP_OBJ_GET_INFO 24 +#define TEE_SCN_CRYP_OBJ_RESTRICT_USAGE 25 +#define TEE_SCN_CRYP_OBJ_GET_ATTR 26 +#define TEE_SCN_CRYP_OBJ_ALLOC 27 +#define TEE_SCN_CRYP_OBJ_CLOSE 28 +#define TEE_SCN_CRYP_OBJ_RESET 29 +#define TEE_SCN_CRYP_OBJ_POPULATE 30 +#define TEE_SCN_CRYP_OBJ_COPY 31 +#define TEE_SCN_CRYP_DERIVE_KEY 32 +#define TEE_SCN_CRYP_RANDOM_NUMBER_GENERATE 33 +#define TEE_SCN_AUTHENC_INIT 34 +#define TEE_SCN_AUTHENC_UPDATE_AAD 35 +#define TEE_SCN_AUTHENC_UPDATE_PAYLOAD 36 +#define TEE_SCN_AUTHENC_ENC_FINAL 37 +#define TEE_SCN_AUTHENC_DEC_FINAL 38 +#define TEE_SCN_ASYMM_OPERATE 39 +#define TEE_SCN_ASYMM_VERIFY 40 +#define TEE_SCN_STORAGE_OBJ_OPEN 41 +#define TEE_SCN_STORAGE_OBJ_CREATE 42 +#define TEE_SCN_STORAGE_OBJ_DEL 43 +#define TEE_SCN_STORAGE_OBJ_RENAME 44 +#define TEE_SCN_STORAGE_ENUM_ALLOC 45 +#define TEE_SCN_STORAGE_ENUM_FREE 46 +#define TEE_SCN_STORAGE_ENUM_RESET 47 +#define TEE_SCN_STORAGE_ENUM_START 48 +#define TEE_SCN_STORAGE_ENUM_NEXT 49 +#define TEE_SCN_STORAGE_OBJ_READ 50 +#define TEE_SCN_STORAGE_OBJ_WRITE 51 +#define TEE_SCN_STORAGE_OBJ_TRUNC 52 +#define TEE_SCN_STORAGE_OBJ_SEEK 53 +#define TEE_SCN_CRYP_OBJ_GENERATE_KEY 54 +/* Deprecated Secure Element API syscalls return TEE_ERROR_NOT_SUPPORTED */ +#define TEE_SCN_SE_SERVICE_OPEN__DEPRECATED 55 +#define TEE_SCN_SE_SERVICE_CLOSE__DEPRECATED 56 +#define TEE_SCN_SE_SERVICE_GET_READERS__DEPRECATED 57 +#define TEE_SCN_SE_READER_GET_PROP__DEPRECATED 58 +#define TEE_SCN_SE_READER_GET_NAME__DEPRECATED 59 +#define TEE_SCN_SE_READER_OPEN_SESSION__DEPRECATED 60 +#define TEE_SCN_SE_READER_CLOSE_SESSIONS__DEPRECATED 61 +#define TEE_SCN_SE_SESSION_IS_CLOSED__DEPRECATED 62 +#define TEE_SCN_SE_SESSION_GET_ATR__DEPRECATED 63 +#define TEE_SCN_SE_SESSION_OPEN_CHANNEL__DEPRECATED 64 +#define TEE_SCN_SE_SESSION_CLOSE__DEPRECATED 65 +#define TEE_SCN_SE_CHANNEL_SELECT_NEXT__DEPRECATED 66 +#define TEE_SCN_SE_CHANNEL_GET_SELECT_RESP__DEPRECATED 67 +#define TEE_SCN_SE_CHANNEL_TRANSMIT__DEPRECATED 68 +#define TEE_SCN_SE_CHANNEL_CLOSE__DEPRECATED 69 +/* End of deprecated Secure Element API syscalls */ +#define TEE_SCN_CACHE_OPERATION 70 + +#define TEE_SCN_MAX 70 + +/* Maximum number of allowed arguments for a syscall */ +#define TEE_SVC_MAX_ARGS 8 + +#endif /* TEE_SYSCALL_NUMBERS_H */ diff --git a/optee_os/lib/libutee/include/tee_ta_api.h b/optee_os/lib/libutee/include/tee_ta_api.h new file mode 100644 index 0000000..84d2084 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_ta_api.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +/* Based on GP TEE Internal API Specification Version 0.22 */ +#ifndef TEE_TA_API_H +#define TEE_TA_API_H + +#include + +#endif diff --git a/optee_os/lib/libutee/include/tee_tcpsocket.h b/optee_os/lib/libutee/include/tee_tcpsocket.h new file mode 100644 index 0000000..c80c243 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_tcpsocket.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef __TEE_TCPSOCKET_H +#define __TEE_TCPSOCKET_H + +#include +#include <__tee_ipsocket.h> +#include <__tee_tcpsocket_defines.h> + +typedef struct TEE_tcpSocket_Setup_s { + TEE_ipSocket_ipVersion ipVersion; + char *server_addr; + uint16_t server_port; +} TEE_tcpSocket_Setup; + +extern TEE_iSocket *const TEE_tcpSocket; + +#endif /*__TEE_TCPSOCKET_H*/ diff --git a/optee_os/lib/libutee/include/tee_udpsocket.h b/optee_os/lib/libutee/include/tee_udpsocket.h new file mode 100644 index 0000000..83563e7 --- /dev/null +++ b/optee_os/lib/libutee/include/tee_udpsocket.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef __TEE_UDPSOCKET_H +#define __TEE_UDPSOCKET_H + +#include +#include <__tee_ipsocket.h> +#include <__tee_udpsocket_defines.h> + +typedef struct TEE_udpSocket_Setup_s { + TEE_ipSocket_ipVersion ipVersion; + char *server_addr; + uint16_t server_port; +} TEE_udpSocket_Setup; + +extern TEE_iSocket *const TEE_udpSocket; + +#endif /*__TEE_UDPSOCKET_H*/ diff --git a/optee_os/lib/libutee/include/user_ta_header.h b/optee_os/lib/libutee/include/user_ta_header.h new file mode 100644 index 0000000..056b94c --- /dev/null +++ b/optee_os/lib/libutee/include/user_ta_header.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2018, Linaro Limited. + */ + +#ifndef USER_TA_HEADER_H +#define USER_TA_HEADER_H + +#include +#include + +#define TA_FLAG_USER_MODE 0 /* Deprecated, was BIT32(0) */ +#define TA_FLAG_EXEC_DDR 0 /* Deprecated, was BIT32(1) */ +#define TA_FLAG_SINGLE_INSTANCE BIT32(2) +#define TA_FLAG_MULTI_SESSION BIT32(3) +#define TA_FLAG_INSTANCE_KEEP_ALIVE BIT32(4) /* remains after last close */ +#define TA_FLAG_SECURE_DATA_PATH BIT32(5) /* accesses SDP memory */ +#define TA_FLAG_REMAP_SUPPORT 0 /* Deprecated, was BIT32(6) */ +#define TA_FLAG_CACHE_MAINTENANCE BIT32(7) /* use cache flush syscall */ + /* + * TA instance can execute multiple sessions concurrently + * (pseudo-TAs only). + */ +#define TA_FLAG_CONCURRENT BIT32(8) + /* + * Device enumeration is done in two stages by the normal world, first + * before the tee-supplicant has started and then once more when the + * tee-supplicant is started. The flags below control if the TA should + * be reported in the first or second or case. + */ +#define TA_FLAG_DEVICE_ENUM BIT32(9) /* without tee-supplicant */ +#define TA_FLAG_DEVICE_ENUM_SUPP BIT32(10) /* with tee-supplicant */ + /* See also "gpd.ta.doesNotCloseHandleOnCorruptObject" */ +#define TA_FLAG_DONT_CLOSE_HANDLE_ON_CORRUPT_OBJECT \ + BIT32(11) + +#define TA_FLAGS_MASK GENMASK_32(10, 0) + +struct ta_head { + TEE_UUID uuid; + uint32_t stack_size; + uint32_t flags; + uint64_t depr_entry; +}; + +#if defined(CFG_FTRACE_SUPPORT) +#define FTRACE_RETFUNC_DEPTH 50 +union compat_ptr { + uint64_t ptr64; + struct { + uint32_t lo; + uint32_t hi; + } ptr32; +}; + +struct __ftrace_info { + union compat_ptr buf_start; + union compat_ptr buf_end; + union compat_ptr ret_ptr; +}; + +struct ftrace_buf { + uint64_t ret_func_ptr; /* __ftrace_return pointer */ + uint64_t ret_stack[FTRACE_RETFUNC_DEPTH]; /* Return stack */ + uint32_t ret_idx; /* Return stack index */ + uint32_t lr_idx; /* lr index used for stack unwinding */ + uint64_t begin_time[FTRACE_RETFUNC_DEPTH]; /* Timestamp */ + uint64_t suspend_time; /* Suspend timestamp */ + uint32_t curr_idx; /* Current entry in the (circular) buffer */ + uint32_t max_size; /* Max allowed size of ftrace buffer */ + uint32_t head_off; /* Ftrace buffer header offset */ + uint32_t buf_off; /* Ftrace buffer offset */ + bool syscall_trace_enabled; /* Some syscalls are never traced */ + bool syscall_trace_suspended; /* By foreign interrupt or RPC */ + bool overflow; /* Circular buffer has wrapped */ +}; + +/* Defined by the linker script */ +extern struct ftrace_buf __ftrace_buf_start; +extern uint8_t __ftrace_buf_end[]; + +unsigned long ftrace_return(void); +void __ftrace_return(void); +#endif + +void __utee_call_elf_init_fn(void); +void __utee_call_elf_fini_fn(void); + +void __utee_tcb_init(void); + +/* + * Information about the ELF objects loaded by the application + */ + +struct __elf_phdr_info { + uint32_t reserved; + uint16_t count; + uint8_t reserved2; + char zero; + struct dl_phdr_info *dlpi; /* @count entries */ +}; + +/* 32-bit variant for a 64-bit ldelf to access a 32-bit TA */ +struct __elf_phdr_info32 { + uint32_t reserved; + uint16_t count; + uint8_t reserved2; + char zero; + uint32_t dlpi; +}; + +extern struct __elf_phdr_info __elf_phdr_info; + +#define TA_PROP_STR_SINGLE_INSTANCE "gpd.ta.singleInstance" +#define TA_PROP_STR_MULTI_SESSION "gpd.ta.multiSession" +#define TA_PROP_STR_KEEP_ALIVE "gpd.ta.instanceKeepAlive" +#define TA_PROP_STR_DATA_SIZE "gpd.ta.dataSize" +#define TA_PROP_STR_STACK_SIZE "gpd.ta.stackSize" +#define TA_PROP_STR_VERSION "gpd.ta.version" +#define TA_PROP_STR_DESCRIPTION "gpd.ta.description" +#define TA_PROP_STR_ENDIAN "gpd.ta.endian" +#define TA_PROP_STR_DOES_NOT_CLOSE_HANDLE_ON_CORRUPT_OBJECT \ + "gpd.ta.doesNotCloseHandleOnCorruptObject" + +enum user_ta_prop_type { + USER_TA_PROP_TYPE_BOOL, /* bool */ + USER_TA_PROP_TYPE_U32, /* uint32_t */ + USER_TA_PROP_TYPE_UUID, /* TEE_UUID */ + USER_TA_PROP_TYPE_IDENTITY, /* TEE_Identity */ + USER_TA_PROP_TYPE_STRING, /* zero terminated string of char */ + USER_TA_PROP_TYPE_BINARY_BLOCK, /* zero terminated base64 coded string */ + USER_TA_PROP_TYPE_U64, /* uint64_t */ + USER_TA_PROP_TYPE_INVALID, /* invalid value */ +}; + +struct user_ta_property { + const char *name; + enum user_ta_prop_type type; + const void *value; +}; + +extern const struct user_ta_property ta_props[]; +extern const size_t ta_num_props; + +extern uint8_t __ta_no_share_heap[]; +extern const size_t __ta_no_share_heap_size; +/* Needed by TEE_CheckMemoryAccessRights() */ +extern uint32_t ta_param_types; +extern TEE_Param ta_params[TEE_NUM_PARAMS]; +extern struct malloc_ctx *__ta_no_share_malloc_ctx; + +int tahead_get_trace_level(void); + +#endif /* USER_TA_HEADER_H */ diff --git a/optee_os/lib/libutee/include/utee_defines.h b/optee_os/lib/libutee/include/utee_defines.h new file mode 100644 index 0000000..8a3ccf3 --- /dev/null +++ b/optee_os/lib/libutee/include/utee_defines.h @@ -0,0 +1,356 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2021, SumUp Services GmbH + */ +#ifndef UTEE_DEFINES_H +#define UTEE_DEFINES_H + +#include +#include +#include +#include + +/* + * Copied from TEE Internal API specificaion v1.0 table 6-9 "Structure of + * Algorithm Identifier". + */ +#define TEE_MAIN_ALGO_MD5 0x01 +#define TEE_MAIN_ALGO_SHA1 0x02 +#define TEE_MAIN_ALGO_SHA224 0x03 +#define TEE_MAIN_ALGO_SHA256 0x04 +#define TEE_MAIN_ALGO_SHA384 0x05 +#define TEE_MAIN_ALGO_SHA512 0x06 +#define TEE_MAIN_ALGO_SM3 0x07 +#define TEE_MAIN_ALGO_SHA3_224 0x08 +#define TEE_MAIN_ALGO_SHA3_256 0x09 +#define TEE_MAIN_ALGO_SHA3_384 0x0A +#define TEE_MAIN_ALGO_SHA3_512 0x0B +#define TEE_MAIN_ALGO_AES 0x10 +#define TEE_MAIN_ALGO_DES 0x11 +#define TEE_MAIN_ALGO_DES2 0x12 +#define TEE_MAIN_ALGO_DES3 0x13 +#define TEE_MAIN_ALGO_SM4 0x14 /* Not in v1.2, extrapolated */ +#define TEE_MAIN_ALGO_RSA 0x30 +#define TEE_MAIN_ALGO_DSA 0x31 +#define TEE_MAIN_ALGO_DH 0x32 +#define TEE_MAIN_ALGO_ECDSA 0x41 +#define TEE_MAIN_ALGO_ECDH 0x42 +#define TEE_MAIN_ALGO_ED25519 0x43 +#define TEE_MAIN_ALGO_SM2_DSA_SM3 0x45 /* Not in v1.2 spec */ +#define TEE_MAIN_ALGO_SM2_KEP 0x46 /* Not in v1.2 spec */ +#define TEE_MAIN_ALGO_SM2_PKE 0x47 /* Not in v1.2 spec */ +#define TEE_MAIN_ALGO_HKDF 0xC0 /* OP-TEE extension */ +#define TEE_MAIN_ALGO_CONCAT_KDF 0xC1 /* OP-TEE extension */ +#define TEE_MAIN_ALGO_PBKDF2 0xC2 /* OP-TEE extension */ +#define TEE_MAIN_ALGO_X25519 0x44 /* Not in v1.2 spec */ +#define TEE_MAIN_ALGO_SHAKE128 0xC3 /* OP-TEE extension */ +#define TEE_MAIN_ALGO_SHAKE256 0xC4 /* OP-TEE extension */ +#define TEE_MAIN_ALGO_X448 0x49 + + +#define TEE_CHAIN_MODE_ECB_NOPAD 0x0 +#define TEE_CHAIN_MODE_CBC_NOPAD 0x1 +#define TEE_CHAIN_MODE_CTR 0x2 +#define TEE_CHAIN_MODE_CTS 0x3 +#define TEE_CHAIN_MODE_XTS 0x4 +#define TEE_CHAIN_MODE_CBC_MAC_PKCS5 0x5 +#define TEE_CHAIN_MODE_CMAC 0x6 +#define TEE_CHAIN_MODE_CCM 0x7 +#define TEE_CHAIN_MODE_GCM 0x8 +#define TEE_CHAIN_MODE_PKCS1_PSS_MGF1 0x9 /* ??? */ + + +static inline uint32_t __tee_alg_get_class(uint32_t algo) +{ + if (algo == TEE_ALG_SM2_PKE) + return TEE_OPERATION_ASYMMETRIC_CIPHER; + if (algo == TEE_ALG_SM2_KEP) + return TEE_OPERATION_KEY_DERIVATION; + if (algo == TEE_ALG_RSASSA_PKCS1_V1_5) + return TEE_OPERATION_ASYMMETRIC_SIGNATURE; + if (algo == TEE_ALG_DES3_CMAC) + return TEE_OPERATION_MAC; + if (algo == TEE_ALG_SM4_XTS) + return TEE_OPERATION_CIPHER; + if (algo == TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5) + return TEE_OPERATION_ASYMMETRIC_SIGNATURE; + if (algo == TEE_ALG_RSAES_PKCS1_OAEP_MGF1_MD5) + return TEE_OPERATION_ASYMMETRIC_CIPHER; + + return (algo >> 28) & 0xF; /* Bits [31:28] */ +} + +#define TEE_ALG_GET_CLASS(algo) __tee_alg_get_class(algo) + +static inline uint32_t __tee_alg_get_main_alg(uint32_t algo) +{ + switch (algo) { + case TEE_ALG_SM2_PKE: + return TEE_MAIN_ALGO_SM2_PKE; + case TEE_ALG_SM2_KEP: + return TEE_MAIN_ALGO_SM2_KEP; + case TEE_ALG_X25519: + return TEE_MAIN_ALGO_X25519; + case TEE_ALG_ED25519: + return TEE_MAIN_ALGO_ED25519; + case TEE_ALG_ECDSA_SHA1: + case TEE_ALG_ECDSA_SHA224: + case TEE_ALG_ECDSA_SHA256: + case TEE_ALG_ECDSA_SHA384: + case TEE_ALG_ECDSA_SHA512: + return TEE_MAIN_ALGO_ECDSA; + case TEE_ALG_HKDF: + return TEE_MAIN_ALGO_HKDF; + case TEE_ALG_SHAKE128: + return TEE_MAIN_ALGO_SHAKE128; + case TEE_ALG_SHAKE256: + return TEE_MAIN_ALGO_SHAKE256; + case TEE_ALG_X448: + return TEE_MAIN_ALGO_X448; + default: + break; + } + + return algo & 0xff; +} + +#define TEE_ALG_GET_MAIN_ALG(algo) __tee_alg_get_main_alg(algo) + + /* Bits [11:8] */ +#define TEE_ALG_GET_CHAIN_MODE(algo) (((algo) >> 8) & 0xF) + +/* + * Value not defined in the GP spec, and not used as bits 15-12 of any TEE_ALG* + * value. TEE_ALG_SM2_DSA_SM3 has value 0x6 for bits 15-12 which would yield the + * SHA512 digest if we were to apply the bit masks that were valid up to the TEE + * Internal Core API v1.1. + */ +#define __TEE_MAIN_HASH_SM3 0x7 + +static inline uint32_t __tee_alg_get_digest_hash(uint32_t algo) +{ + if (algo == TEE_ALG_SM2_DSA_SM3) + return __TEE_MAIN_HASH_SM3; + + /* Bits [15:12] */ + return (algo >> 12) & 0xF; +} + +#define TEE_ALG_GET_DIGEST_HASH(algo) __tee_alg_get_digest_hash(algo) + + /* Bits [23:20] */ +#define TEE_ALG_GET_INTERNAL_HASH(algo) (((algo) >> 20) & 0x7) + +static inline uint32_t __tee_alg_get_key_type(uint32_t algo, bool with_priv) +{ + uint32_t key_type = 0xA0000000 | TEE_ALG_GET_MAIN_ALG(algo); + + if (with_priv) + key_type |= 0x01000000; + + return key_type; +} + +#define TEE_ALG_GET_KEY_TYPE(algo, with_private_key) \ + __tee_alg_get_key_type(algo, with_private_key) + +static inline uint32_t __tee_alg_hash_algo(uint32_t main_hash) +{ + if (main_hash == __TEE_MAIN_HASH_SM3) + return TEE_ALG_SM3; + + return (TEE_OPERATION_DIGEST << 28) | main_hash; +} + + /* Return hash algorithm based on main hash */ +#define TEE_ALG_HASH_ALGO(main_hash) __tee_alg_hash_algo(main_hash) + + /* Extract internal hash and return hash algorithm */ +#define TEE_INTERNAL_HASH_TO_ALGO(algo) \ + TEE_ALG_HASH_ALGO(TEE_ALG_GET_INTERNAL_HASH(algo)) + + /* Extract digest hash and return hash algorithm */ +#define TEE_DIGEST_HASH_TO_ALGO(algo) \ + TEE_ALG_HASH_ALGO(TEE_ALG_GET_DIGEST_HASH(algo)) + +/* Return HMAC algorithm based on main hash */ +#define TEE_ALG_HMAC_ALGO(main_hash) \ + (TEE_OPERATION_MAC << 28 | (main_hash)) + +#define TEE_AES_BLOCK_SIZE 16UL +#define TEE_DES_BLOCK_SIZE 8UL +#define TEE_SM4_BLOCK_SIZE 16UL + +#define TEE_AES_MAX_KEY_SIZE 32UL + + /* SHA-512 */ +#ifndef TEE_MD5_HASH_SIZE +typedef enum { + TEE_MD5_HASH_SIZE = 16, + TEE_SHA1_HASH_SIZE = 20, + TEE_SHA224_HASH_SIZE = 28, + TEE_SHA256_HASH_SIZE = 32, + TEE_SM3_HASH_SIZE = 32, + TEE_SHA384_HASH_SIZE = 48, + TEE_SHA512_HASH_SIZE = 64, + TEE_MD5SHA1_HASH_SIZE = (TEE_MD5_HASH_SIZE + TEE_SHA1_HASH_SIZE), + TEE_MAX_HASH_SIZE = 64, +} t_hash_size; +#endif + +#define TEE_MAC_SIZE_AES_CBC_MAC_NOPAD +#define TEE_MAC_SIZE_AES_CBC_MAC_PKCS5 +#define TEE_MAC_SIZE_AES_CMAC +#define TEE_MAC_SIZE_DES_CBC_MAC_PKCS5 + +static inline size_t __tee_alg_get_digest_size(uint32_t algo) +{ + switch (algo) { + case TEE_ALG_MD5: + case TEE_ALG_HMAC_MD5: + return TEE_MD5_HASH_SIZE; + case TEE_ALG_SHA1: + case TEE_ALG_HMAC_SHA1: + case TEE_ALG_DSA_SHA1: + case TEE_ALG_ECDSA_SHA1: + return TEE_SHA1_HASH_SIZE; + case TEE_ALG_SHA224: + case TEE_ALG_SHA3_224: + case TEE_ALG_HMAC_SHA224: + case TEE_ALG_HMAC_SHA3_224: + case TEE_ALG_DSA_SHA224: + case TEE_ALG_ECDSA_SHA224: + return TEE_SHA224_HASH_SIZE; + case TEE_ALG_SHA256: + case TEE_ALG_SHA3_256: + case TEE_ALG_HMAC_SHA256: + case TEE_ALG_HMAC_SHA3_256: + case TEE_ALG_DSA_SHA256: + case TEE_ALG_ECDSA_SHA256: + return TEE_SHA256_HASH_SIZE; + case TEE_ALG_SHA384: + case TEE_ALG_SHA3_384: + case TEE_ALG_HMAC_SHA384: + case TEE_ALG_HMAC_SHA3_384: + case TEE_ALG_ECDSA_SHA384: + return TEE_SHA384_HASH_SIZE; + case TEE_ALG_SHA512: + case TEE_ALG_SHA3_512: + case TEE_ALG_HMAC_SHA512: + case TEE_ALG_HMAC_SHA3_512: + case TEE_ALG_ECDSA_SHA512: + return TEE_SHA512_HASH_SIZE; + case TEE_ALG_SM3: + case TEE_ALG_HMAC_SM3: + return TEE_SM3_HASH_SIZE; + case TEE_ALG_AES_CBC_MAC_NOPAD: + case TEE_ALG_AES_CBC_MAC_PKCS5: + case TEE_ALG_AES_CMAC: + return TEE_AES_BLOCK_SIZE; + case TEE_ALG_DES_CBC_MAC_NOPAD: + case TEE_ALG_DES_CBC_MAC_PKCS5: + case TEE_ALG_DES3_CBC_MAC_NOPAD: + case TEE_ALG_DES3_CBC_MAC_PKCS5: + case TEE_ALG_DES3_CMAC: + return TEE_DES_BLOCK_SIZE; + default: + return 0; + } +} + + /* Return algorithm digest size */ +#define TEE_ALG_GET_DIGEST_SIZE(algo) __tee_alg_get_digest_size(algo) + +/* + * Bit indicating that the attribute is a value attribute + * See TEE Internal API specificaion v1.0 table 6-12 "Partial Structure of + * Attribute Identifier" + */ + + +#ifdef __compiler_bswap64 +#define TEE_U64_BSWAP(x) __compiler_bswap64((x)) +#else +#define TEE_U64_BSWAP(x) ((uint64_t)( \ + (((uint64_t)(x) & UINT64_C(0xff00000000000000ULL)) >> 56) | \ + (((uint64_t)(x) & UINT64_C(0x00ff000000000000ULL)) >> 40) | \ + (((uint64_t)(x) & UINT64_C(0x0000ff0000000000ULL)) >> 24) | \ + (((uint64_t)(x) & UINT64_C(0x000000ff00000000ULL)) >> 8) | \ + (((uint64_t)(x) & UINT64_C(0x00000000ff000000ULL)) << 8) | \ + (((uint64_t)(x) & UINT64_C(0x0000000000ff0000ULL)) << 24) | \ + (((uint64_t)(x) & UINT64_C(0x000000000000ff00ULL)) << 40) | \ + (((uint64_t)(x) & UINT64_C(0x00000000000000ffULL)) << 56))) +#endif + +#ifdef __compiler_bswap32 +#define TEE_U32_BSWAP(x) __compiler_bswap32((x)) +#else +#define TEE_U32_BSWAP(x) ((uint32_t)( \ + (((uint32_t)(x) & UINT32_C(0xff000000)) >> 24) | \ + (((uint32_t)(x) & UINT32_C(0x00ff0000)) >> 8) | \ + (((uint32_t)(x) & UINT32_C(0x0000ff00)) << 8) | \ + (((uint32_t)(x) & UINT32_C(0x000000ff)) << 24))) +#endif + +#ifdef __compiler_bswap16 +#define TEE_U16_BSWAP(x) __compiler_bswap16((x)) +#else +#define TEE_U16_BSWAP(x) ((uint16_t)( \ + (((uint16_t)(x) & UINT16_C(0xff00)) >> 8) | \ + (((uint16_t)(x) & UINT16_C(0x00ff)) << 8))) +#endif + +/* If we we're on a big endian platform we'll have to update these */ +#define TEE_U64_FROM_BIG_ENDIAN(x) TEE_U64_BSWAP(x) +#define TEE_U32_FROM_BIG_ENDIAN(x) TEE_U32_BSWAP(x) +#define TEE_U16_FROM_BIG_ENDIAN(x) TEE_U16_BSWAP(x) +#define TEE_U64_TO_BIG_ENDIAN(x) TEE_U64_BSWAP(x) +#define TEE_U32_TO_BIG_ENDIAN(x) TEE_U32_BSWAP(x) +#define TEE_U16_TO_BIG_ENDIAN(x) TEE_U16_BSWAP(x) + +#define TEE_TIME_MILLIS_BASE 1000 + +#define TEE_TIME_LT(t1, t2) \ + (((t1).seconds == (t2).seconds) ? \ + ((t1).millis < (t2).millis) : \ + ((t1).seconds < (t2).seconds)) + +#define TEE_TIME_LE(t1, t2) \ + (((t1).seconds == (t2).seconds) ? \ + ((t1).millis <= (t2).millis) : \ + ((t1).seconds <= (t2).seconds)) + +#define TEE_TIME_ADD(t1, t2, dst) do { \ + (dst).seconds = (t1).seconds + (t2).seconds; \ + (dst).millis = (t1).millis + (t2).millis; \ + if ((dst).millis >= TEE_TIME_MILLIS_BASE) { \ + (dst).seconds++; \ + (dst).millis -= TEE_TIME_MILLIS_BASE; \ + } \ + } while (0) + +#define TEE_TIME_SUB(t1, t2, dst) do { \ + (dst).seconds = (t1).seconds - (t2).seconds; \ + if ((t1).millis < (t2).millis) { \ + (dst).seconds--; \ + (dst).millis = (t1).millis + TEE_TIME_MILLIS_BASE - (t2).millis;\ + } else { \ + (dst).millis = (t1).millis - (t2).millis; \ + } \ + } while (0) + +/* ------------------------------------------------------------ */ +/* OTP mapping */ +/* ------------------------------------------------------------ */ +#define HW_UNIQUE_KEY_WORD1 (8) +#define HW_UNIQUE_KEY_LENGTH (16) +#define HW_UNIQUE_KEY_WORD2 (HW_UNIQUE_KEY_WORD1 + 1) +#define HW_UNIQUE_KEY_WORD3 (HW_UNIQUE_KEY_WORD1 + 2) +#define HW_UNIQUE_KEY_WORD4 (HW_UNIQUE_KEY_WORD1 + 3) + +#define UTEE_SE_READER_PRESENT (1 << 0) +#define UTEE_SE_READER_TEE_ONLY (1 << 1) +#define UTEE_SE_READER_SELECT_RESPONE_ENABLE (1 << 2) + +#endif /* UTEE_DEFINES_H */ diff --git a/optee_os/lib/libutee/include/utee_syscalls.h b/optee_os/lib/libutee/include/utee_syscalls.h new file mode 100644 index 0000000..e83f803 --- /dev/null +++ b/optee_os/lib/libutee/include/utee_syscalls.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef UTEE_SYSCALLS_H +#define UTEE_SYSCALLS_H + +#include +#include +#include + +#include +#include +#include + +/* + * Arguments must use the native register width, unless it's a signed + * argument then it must be a 32-bit value instead to avoid problems with + * sign extension. To keep it simple, only use pointers, int32_t, unsigned + * long and size_t. Pointers may only point structures or types based on + * fixed width integer types. Only exception are buffers with opaque data. + * + * Return values should not use a fixed width larger than 32 bits, unsigned + * long and pointers are OK though. + * + * Members in structs on the other hand should only use fixed width integer + * types; uint32_t, uint64_t etc. To keep it simple, use uint64_t for all + * length fields. + */ + +void _utee_return(unsigned long ret) __noreturn; + +void _utee_log(const void *buf, size_t len); + +/* This is not __noreturn because AArch32 stack unwinding fails otherwise */ +void _utee_panic(unsigned long code); + +/* prop_set is TEE_PROPSET_xxx*/ +TEE_Result _utee_get_property(unsigned long prop_set, unsigned long index, + void *name, uint32_t *name_len, void *buf, + uint32_t *blen, uint32_t *prop_type); + +TEE_Result _utee_get_property_name_to_index(unsigned long prop_set, + const void *name, + unsigned long name_len, + uint32_t *index); + +/* sess has type TEE_TASessionHandle */ +TEE_Result _utee_open_ta_session(const TEE_UUID *dest, + unsigned long cancel_req_to, + struct utee_params *params, uint32_t *sess, + uint32_t *ret_orig); + +/* sess has type TEE_TASessionHandle */ +TEE_Result _utee_close_ta_session(unsigned long sess); + +/* sess has type TEE_TASessionHandle */ +TEE_Result _utee_invoke_ta_command(unsigned long sess, + unsigned long cancel_req_to, + unsigned long cmd_id, + struct utee_params *params, + uint32_t *ret_orig); + +TEE_Result _utee_check_access_rights(uint32_t flags, const void *buf, + size_t len); + +/* cancel has type bool */ +TEE_Result _utee_get_cancellation_flag(uint32_t *cancel); + +/* old_mask has type bool */ +TEE_Result _utee_unmask_cancellation(uint32_t *old_mask); + +/* old_mask has type bool */ +TEE_Result _utee_mask_cancellation(uint32_t *old_mask); + +TEE_Result _utee_wait(unsigned long timeout); + +/* cat has type enum _utee_time_category */ +TEE_Result _utee_get_time(unsigned long cat, TEE_Time *time); + +TEE_Result _utee_set_ta_time(const TEE_Time *time); + +TEE_Result _utee_cryp_state_alloc(unsigned long algo, unsigned long op_mode, + unsigned long key1, unsigned long key2, + uint32_t *state); +TEE_Result _utee_cryp_state_copy(unsigned long dst, unsigned long src); +TEE_Result _utee_cryp_state_free(unsigned long state); + +/* iv and iv_len are ignored for some algorithms */ +TEE_Result _utee_hash_init(unsigned long state, const void *iv, size_t iv_len); +TEE_Result _utee_hash_update(unsigned long state, const void *chunk, + size_t chunk_size); +TEE_Result _utee_hash_final(unsigned long state, const void *chunk, + size_t chunk_size, void *hash, uint64_t *hash_len); + +TEE_Result _utee_cipher_init(unsigned long state, const void *iv, + size_t iv_len); +TEE_Result _utee_cipher_update(unsigned long state, const void *src, + size_t src_len, void *dest, uint64_t *dest_len); +TEE_Result _utee_cipher_final(unsigned long state, const void *src, + size_t src_len, void *dest, uint64_t *dest_len); + +/* Generic Object Functions */ +TEE_Result _utee_cryp_obj_get_info(unsigned long obj, + struct utee_object_info *info); +TEE_Result _utee_cryp_obj_restrict_usage(unsigned long obj, + unsigned long usage); +TEE_Result _utee_cryp_obj_get_attr(unsigned long obj, unsigned long attr_id, + void *buffer, uint64_t *size); + +/* Transient Object Functions */ +/* type has type TEE_ObjectType */ +TEE_Result _utee_cryp_obj_alloc(unsigned long type, unsigned long max_size, + uint32_t *obj); +TEE_Result _utee_cryp_obj_close(unsigned long obj); +TEE_Result _utee_cryp_obj_reset(unsigned long obj); +TEE_Result _utee_cryp_obj_populate(unsigned long obj, + struct utee_attribute *attrs, + unsigned long attr_count); +TEE_Result _utee_cryp_obj_copy(unsigned long dst_obj, unsigned long src_obj); + +TEE_Result _utee_cryp_obj_generate_key(unsigned long obj, + unsigned long key_size, + const struct utee_attribute *params, + unsigned long param_count); + +TEE_Result _utee_cryp_derive_key(unsigned long state, + const struct utee_attribute *params, + unsigned long param_count, + unsigned long derived_key); + +TEE_Result _utee_cryp_random_number_generate(void *buf, size_t blen); + +TEE_Result _utee_authenc_init(unsigned long state, const void *nonce, + size_t nonce_len, size_t tag_len, size_t aad_len, + size_t payload_len); +TEE_Result _utee_authenc_update_aad(unsigned long state, const void *aad_data, + size_t aad_data_len); +TEE_Result _utee_authenc_update_payload(unsigned long state, + const void *src_data, size_t src_len, + void *dest_data, uint64_t *dest_len); +TEE_Result _utee_authenc_enc_final(unsigned long state, const void *src_data, + size_t src_len, void *dest_data, + uint64_t *dest_len, void *tag, + uint64_t *tag_len); +TEE_Result _utee_authenc_dec_final(unsigned long state, const void *src_data, + size_t src_len, void *dest_data, + uint64_t *dest_len, const void *tag, + size_t tag_len); + +TEE_Result _utee_asymm_operate(unsigned long state, + const struct utee_attribute *params, + unsigned long num_params, const void *src_data, + size_t src_len, void *dest_data, + uint64_t *dest_len); + +TEE_Result _utee_asymm_verify(unsigned long state, + const struct utee_attribute *params, + unsigned long num_params, const void *data, + size_t data_len, const void *sig, size_t sig_len); + +/* Persistant Object Functions */ +/* obj is of type TEE_ObjectHandle */ +TEE_Result _utee_storage_obj_open(unsigned long storage_id, + const void *object_id, size_t object_id_len, + unsigned long flags, uint32_t *obj); + +/* + * attr is of type TEE_ObjectHandle + * obj is of type TEE_ObjectHandle + */ +TEE_Result _utee_storage_obj_create(unsigned long storage_id, + const void *object_id, + size_t object_id_len, unsigned long flags, + unsigned long attr, const void *data, + size_t len, uint32_t *obj); + +/* obj is of type TEE_ObjectHandle */ +TEE_Result _utee_storage_obj_del(unsigned long obj); + +/* obj is of type TEE_ObjectHandle */ +TEE_Result _utee_storage_obj_rename(unsigned long obj, const void *new_obj_id, + size_t new_obj_id_len); + +/* Persistent Object Enumeration Functions */ +/* obj_enum is of type TEE_ObjectEnumHandle */ +TEE_Result _utee_storage_alloc_enum(uint32_t *obj_enum); + + +/* obj_enum is of type TEE_ObjectEnumHandle */ +TEE_Result _utee_storage_free_enum(unsigned long obj_enum); + +/* obj_enum is of type TEE_ObjectEnumHandle */ +TEE_Result _utee_storage_reset_enum(unsigned long obj_enum); + +/* obj_enum is of type TEE_ObjectEnumHandle */ +TEE_Result _utee_storage_start_enum(unsigned long obj_enum, + unsigned long storage_id); + +/* obj_enum is of type TEE_ObjectEnumHandle */ +TEE_Result _utee_storage_next_enum(unsigned long obj_enum, + struct utee_object_info *info, + void *obj_id, uint64_t *len); + +/* Data Stream Access Functions */ +/* obj is of type TEE_ObjectHandle */ +TEE_Result _utee_storage_obj_read(unsigned long obj, void *data, size_t len, + uint64_t *count); + +/* obj is of type TEE_ObjectHandle */ +TEE_Result _utee_storage_obj_write(unsigned long obj, const void *data, + size_t len); + +/* obj is of type TEE_ObjectHandle */ +TEE_Result _utee_storage_obj_trunc(unsigned long obj, size_t len); + +/* obj is of type TEE_ObjectHandle */ +/* whence is of type TEE_Whence */ +TEE_Result _utee_storage_obj_seek(unsigned long obj, int32_t offset, + unsigned long whence); + +/* op is of type enum _utee_cache_operation */ +TEE_Result _utee_cache_operation(void *va, size_t l, unsigned long op); + +TEE_Result _utee_gprof_send(void *buf, size_t size, uint32_t *id); + +#endif /* UTEE_SYSCALLS_H */ diff --git a/optee_os/lib/libutee/include/utee_syscalls_asm.S b/optee_os/lib/libutee/include/utee_syscalls_asm.S new file mode 100644 index 0000000..44bc60d --- /dev/null +++ b/optee_os/lib/libutee/include/utee_syscalls_asm.S @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + + + UTEE_SYSCALL _utee_return, TEE_SCN_RETURN, 1 + + UTEE_SYSCALL _utee_log, TEE_SCN_LOG, 2 + + UTEE_SYSCALL __utee_panic, TEE_SCN_PANIC, 2 + + UTEE_SYSCALL _utee_get_property, TEE_SCN_GET_PROPERTY, 7 + + UTEE_SYSCALL _utee_get_property_name_to_index, \ + TEE_SCN_GET_PROPERTY_NAME_TO_INDEX, 4 + + UTEE_SYSCALL _utee_open_ta_session, TEE_SCN_OPEN_TA_SESSION, 5 + + UTEE_SYSCALL _utee_close_ta_session, TEE_SCN_CLOSE_TA_SESSION, 1 + + UTEE_SYSCALL _utee_invoke_ta_command, TEE_SCN_INVOKE_TA_COMMAND, 5 + + UTEE_SYSCALL _utee_get_cancellation_flag, \ + TEE_SCN_GET_CANCELLATION_FLAG, 1 + + UTEE_SYSCALL _utee_check_access_rights, TEE_SCN_CHECK_ACCESS_RIGHTS, 3 + + UTEE_SYSCALL _utee_unmask_cancellation, TEE_SCN_UNMASK_CANCELLATION, 1 + + UTEE_SYSCALL _utee_mask_cancellation, TEE_SCN_MASK_CANCELLATION, 1 + + UTEE_SYSCALL _utee_wait, TEE_SCN_WAIT, 1 + + UTEE_SYSCALL _utee_get_time, TEE_SCN_GET_TIME, 2 + + UTEE_SYSCALL _utee_set_ta_time, TEE_SCN_SET_TA_TIME, 1 + + UTEE_SYSCALL _utee_cryp_state_alloc, TEE_SCN_CRYP_STATE_ALLOC, 5 + + UTEE_SYSCALL _utee_cryp_state_copy, TEE_SCN_CRYP_STATE_COPY, 2 + + UTEE_SYSCALL _utee_cryp_state_free, TEE_SCN_CRYP_STATE_FREE, 1 + + UTEE_SYSCALL _utee_hash_init, TEE_SCN_HASH_INIT, 3 + + UTEE_SYSCALL _utee_hash_update, TEE_SCN_HASH_UPDATE, 3 + + UTEE_SYSCALL _utee_hash_final, TEE_SCN_HASH_FINAL, 5 + + UTEE_SYSCALL _utee_cipher_init, TEE_SCN_CIPHER_INIT, 3 + + UTEE_SYSCALL _utee_cipher_update, TEE_SCN_CIPHER_UPDATE, 5 + + UTEE_SYSCALL _utee_cipher_final, TEE_SCN_CIPHER_FINAL, 5 + + UTEE_SYSCALL _utee_cryp_obj_get_info, TEE_SCN_CRYP_OBJ_GET_INFO, 2 + + UTEE_SYSCALL _utee_cryp_obj_restrict_usage, \ + TEE_SCN_CRYP_OBJ_RESTRICT_USAGE, 2 + + UTEE_SYSCALL _utee_cryp_obj_get_attr, TEE_SCN_CRYP_OBJ_GET_ATTR, 4 + + UTEE_SYSCALL _utee_cryp_obj_alloc, TEE_SCN_CRYP_OBJ_ALLOC, 3 + + UTEE_SYSCALL _utee_cryp_obj_close, TEE_SCN_CRYP_OBJ_CLOSE, 1 + + UTEE_SYSCALL _utee_cryp_obj_reset, TEE_SCN_CRYP_OBJ_RESET, 1 + + UTEE_SYSCALL _utee_cryp_obj_populate, TEE_SCN_CRYP_OBJ_POPULATE, 3 + + UTEE_SYSCALL _utee_cryp_obj_copy, TEE_SCN_CRYP_OBJ_COPY, 2 + + UTEE_SYSCALL _utee_cryp_derive_key, TEE_SCN_CRYP_DERIVE_KEY, 4 + + UTEE_SYSCALL _utee_cryp_random_number_generate, \ + TEE_SCN_CRYP_RANDOM_NUMBER_GENERATE, 2 + + UTEE_SYSCALL _utee_authenc_init, TEE_SCN_AUTHENC_INIT, 6 + + UTEE_SYSCALL _utee_authenc_update_aad, TEE_SCN_AUTHENC_UPDATE_AAD, 3 + + UTEE_SYSCALL _utee_authenc_update_payload, \ + TEE_SCN_AUTHENC_UPDATE_PAYLOAD, 5 + + UTEE_SYSCALL _utee_authenc_enc_final, TEE_SCN_AUTHENC_ENC_FINAL, 7 + + UTEE_SYSCALL _utee_authenc_dec_final, TEE_SCN_AUTHENC_DEC_FINAL, 7 + + UTEE_SYSCALL _utee_asymm_operate, TEE_SCN_ASYMM_OPERATE, 7 + + UTEE_SYSCALL _utee_asymm_verify, TEE_SCN_ASYMM_VERIFY, 7 + + UTEE_SYSCALL _utee_storage_obj_open, TEE_SCN_STORAGE_OBJ_OPEN, 5 + + UTEE_SYSCALL _utee_storage_obj_create, TEE_SCN_STORAGE_OBJ_CREATE, 8 + + UTEE_SYSCALL _utee_storage_obj_del, TEE_SCN_STORAGE_OBJ_DEL, 1 + + UTEE_SYSCALL _utee_storage_obj_rename, TEE_SCN_STORAGE_OBJ_RENAME, 3 + + UTEE_SYSCALL _utee_storage_alloc_enum, TEE_SCN_STORAGE_ENUM_ALLOC, 1 + + UTEE_SYSCALL _utee_storage_free_enum, TEE_SCN_STORAGE_ENUM_FREE, 1 + + UTEE_SYSCALL _utee_storage_reset_enum, TEE_SCN_STORAGE_ENUM_RESET, 1 + + UTEE_SYSCALL _utee_storage_start_enum, TEE_SCN_STORAGE_ENUM_START, 2 + + UTEE_SYSCALL _utee_storage_next_enum, TEE_SCN_STORAGE_ENUM_NEXT, 4 + + UTEE_SYSCALL _utee_storage_obj_read, TEE_SCN_STORAGE_OBJ_READ, 4 + + UTEE_SYSCALL _utee_storage_obj_write, TEE_SCN_STORAGE_OBJ_WRITE, 3 + + UTEE_SYSCALL _utee_storage_obj_trunc, TEE_SCN_STORAGE_OBJ_TRUNC, 2 + + UTEE_SYSCALL _utee_storage_obj_seek, TEE_SCN_STORAGE_OBJ_SEEK, 3 + + UTEE_SYSCALL _utee_cryp_obj_generate_key, \ + TEE_SCN_CRYP_OBJ_GENERATE_KEY, 4 + + UTEE_SYSCALL _utee_cache_operation, TEE_SCN_CACHE_OPERATION, 3 diff --git a/optee_os/lib/libutee/include/utee_types.h b/optee_os/lib/libutee/include/utee_types.h new file mode 100644 index 0000000..547c21a --- /dev/null +++ b/optee_os/lib/libutee/include/utee_types.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef UTEE_TYPES_H +#define UTEE_TYPES_H + +#include +#include + +enum utee_time_category { + UTEE_TIME_CAT_SYSTEM = 0, + UTEE_TIME_CAT_TA_PERSISTENT, + UTEE_TIME_CAT_REE +}; + +enum utee_entry_func { + UTEE_ENTRY_FUNC_OPEN_SESSION = 0, + UTEE_ENTRY_FUNC_CLOSE_SESSION, + UTEE_ENTRY_FUNC_INVOKE_COMMAND, + /* + * UTEE_ENTRY_FUNC_DUMP_MEMSTATS + * [out] value[0].a Byte size currently allocated + * [out] value[0].b Max bytes allocated since stats reset + * [out] value[1].a Pool byte size + * [out] value[1].b Number of failed allocation requests + * [out] value[2].a Biggest byte size which allocation failed + * [out] value[2].b Biggest byte size which allocation succeeded + */ + UTEE_ENTRY_FUNC_DUMP_MEMSTATS, +}; + +/* + * Cache operation types. + * Used when extensions TEE_CacheClean() / TEE_CacheFlush() / + * TEE_CacheInvalidate() are used + */ +enum utee_cache_operation { + TEE_CACHECLEAN = 0, + TEE_CACHEFLUSH, + TEE_CACHEINVALIDATE, +}; + +struct utee_params { + uint64_t types; + /* vals[n * 2] corresponds to either value.a or memref.buffer + * vals[n * 2 + ] corresponds to either value.b or memref.size + * when converting to/from struct tee_ta_param + */ + uint64_t vals[TEE_NUM_PARAMS * 2]; +}; + +struct utee_attribute { + uint64_t a; /* also serves as a pointer for references */ + uint64_t b; /* also serves as a length for references */ + uint32_t attribute_id; +}; + +struct utee_object_info { + uint32_t obj_type; + uint32_t obj_size; + uint32_t max_obj_size; + uint32_t obj_usage; + uint32_t data_size; + uint32_t data_pos; + uint32_t handle_flags; +}; + +#endif /* UTEE_TYPES_H */ diff --git a/optee_os/lib/libutee/sub.mk b/optee_os/lib/libutee/sub.mk new file mode 100644 index 0000000..1a3398e --- /dev/null +++ b/optee_os/lib/libutee/sub.mk @@ -0,0 +1,25 @@ +global-incdirs-y += include + +srcs-y += abort.c +srcs-y += assert.c +srcs-y += tee_uuid_from_str.c +srcs-y += trace_ext.c + +ifneq ($(sm),ldelf) +srcs-y += base64.c +srcs-y += tee_api.c +srcs-y += tee_api_arith_mpi.c +cppflags-tee_api_arith_mpi.c-y += -DMBEDTLS_ALLOW_PRIVATE_ACCESS +srcs-y += tee_api_objects.c +srcs-y += tee_api_operations.c +srcs-y += tee_api_panic.c +srcs-y += tee_api_property.c +srcs-y += tee_socket_pta.c +srcs-y += tee_system_pta.c +srcs-y += tee_tcpudp_socket.c +srcs-y += tcb.c +srcs-y += user_ta_entry.c +srcs-y += user_ta_entry_compat.c +endif #ifneq ($(sm),ldelf) + +subdirs-y += arch/$(ARCH) diff --git a/optee_os/lib/libutee/tcb.c b/optee_os/lib/libutee/tcb.c new file mode 100644 index 0000000..20d44e7 --- /dev/null +++ b/optee_os/lib/libutee/tcb.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ +/* + * Support for Thread-Local Storage (TLS) ABIs for ARMv7/Aarch32 and Aarch64. + * + * TAs are currently single-threaded, so the only benefit of implementing these + * ABIs is to support toolchains that need them even when the target program is + * single-threaded. Such as, the g++ compiler from the GCC toolchain targeting a + * "Posix thread" Linux runtime, which OP-TEE has been using for quite some time + * (arm-linux-gnueabihf-* and aarch64-linux-gnu-*). This allows building C++ TAs + * without having to build a specific toolchain with --disable-threads. + * + * This implementation is based on [1]. + * + * - "TLS data structures variant 1" (section 3): the AArch64 compiler uses the + * TPIDR_EL0 to access TLS data directly. This assumes a specific layout for + * the TCB, and (for shared objects) the use of R_AARCH64_TLS_TPREL + * relocations. + * - The "General Dynamic access model" (section 4.1): the ARMv7/Aarch32 + * compiler inserts calls to the __tls_get_addr() function which has to be + * implemented by the runtime library. The function takes a module ID and an + * offset parameter, which are provided thanks to R_ARM_TLS_DTPMOD32 and + * R_ARM_TLS_DTPOFF32 relocations. + * + * In addition, dl_iterate_phdr() is implemented here, because it is used by the + * g++ Aarch64 exception handling and it does use the TCB to provide TLS + * information to the caller. + * + * [1] "ELF Handling For Thread-Local Storage" + * https://www.akkadia.org/drepper/tls.pdf + */ + +#include +#include +#include +#include +#include +#include +#include "user_ta_header.h" + +/* DTV - Dynamic Thread Vector + * + * Maintains an array of pointers to TLS data for each module in the TCB. Each + * module that has a TLS segment has an entry (and consequently, some space in + * the tcb_head::tls buffer). The index is the "module ID". + * dtv[0].size is the number of elements in the vector + * dtv[1].tls points to TLS for the main executable (may be NULL) + * tls[2 .. (size-1)] are for shared libraries + */ +union dtv { + unsigned long size; + uint8_t *tls; +}; + +#define DTV_SIZE(size) (sizeof(union dtv) + (size)) + +/* Thread Control Block */ +struct tcb_head { + /* Two words are reserved as per the "TLS variant 1" ABI */ + union dtv *dtv; + unsigned long reserved; + /* + * The rest of the structure contains the TLS blocks for each ELF module + * having a PT_TLS segment. Each block is a copy of the .tdata section + * plus some zero-initialized space for .tbss. + */ + uint8_t tls[]; +}; + +/* + * Since TAs are single threaded, only one TCB is needed. This would need to + * change if multi-threading is introduced. + */ +static struct tcb_head *_tcb; +static size_t _tls_size; + +#define TCB_SIZE(tls_size) (sizeof(*_tcb) + (tls_size)) + +/* + * Initialize or update the TCB. + * Called on application initialization and when additional shared objects are + * loaded via dlopen(). + */ +void __utee_tcb_init(void) +{ + struct dl_phdr_info *dlpi = NULL; + const Elf_Phdr *phdr = NULL; + size_t total_size = 0; + size_t size = 0; + size_t i = 0; + size_t j = 0; + + /* Compute the size needed for all the TLS blocks */ + for (i = 0; i < __elf_phdr_info.count; i++) { + dlpi = __elf_phdr_info.dlpi + i; + for (j = 0; j < dlpi->dlpi_phnum; j++) { + phdr = dlpi->dlpi_phdr + j; + if (phdr->p_type == PT_TLS) { + total_size += phdr->p_memsz; + break; + } + } + } + + /* ELF modules currently cannot be unmapped */ + assert(total_size >= _tls_size); + + if (total_size == _tls_size) + return; + + /* (Re-)allocate the TCB */ + _tcb = realloc(_tcb, TCB_SIZE(total_size)); + if (!_tcb) { + EMSG("TCB allocation failed (%zu bytes)", TCB_SIZE(total_size)); + abort(); + } + + /* (Re-)allocate the DTV. + 1 since dtv[0] holds the size */ + size = DTV_SIZE((__elf_phdr_info.count + 1) * sizeof(union dtv)); + _tcb->dtv = realloc(_tcb->dtv, size); + if (!_tcb->dtv) { + EMSG("DTV allocation failed (%zu bytes)", size); + abort(); + } + + /* Copy TLS data to the TCB */ + size = 0; + for (i = 0; i < __elf_phdr_info.count; i++) { + dlpi = __elf_phdr_info.dlpi + i; + for (j = 0; j < dlpi->dlpi_phnum; j++) { + phdr = dlpi->dlpi_phdr + j; + if (phdr->p_type != PT_TLS) + continue; + if (size + phdr->p_memsz <= _tls_size) { + /* Already copied */ + break; + } + _tcb->dtv[i + 1].tls = _tcb->tls + size; + /* Copy .tdata */ + memcpy(_tcb->tls + size, + (void *)(dlpi->dlpi_addr + phdr->p_vaddr), + phdr->p_filesz); + /* Initialize .tbss */ + memset(_tcb->tls + size + phdr->p_filesz, 0, + phdr->p_memsz - phdr->p_filesz); + size += phdr->p_memsz; + } + } + _tcb->dtv[0].size = i; + + _tls_size = total_size; +#ifdef ARM64 + /* + * Aarch64 ABI requirement: the thread pointer shall point to the + * thread's TCB. ARMv7 and Aarch32 access the TCB via _tls_get_addr(). + */ + write_tpidr_el0((vaddr_t)_tcb); +#endif +} + +struct tls_index { + unsigned long module; + unsigned long offset; +}; + +void *__tls_get_addr(struct tls_index *ti); + +void *__tls_get_addr(struct tls_index *ti) +{ + return _tcb->dtv[ti->module].tls + ti->offset; +} + +int dl_iterate_phdr(int (*callback)(struct dl_phdr_info *, size_t, void *), + void *data) +{ + struct dl_phdr_info *dlpi = NULL; + size_t id = 0; + size_t i = 0; + int st = 0; + + /* + * dlpi_tls_data is thread-specific so if we were to support + * multi-threading, we would need one copy of struct dl_phdr_info per + * thread. Could be a pre-allocated area, or could be allocated on the + * heap. Doing the latter here so that it would at least work if/when we + * add thread support. Further optimization can always come later. + */ + dlpi = calloc(1, sizeof(*dlpi)); + if (!dlpi) { + EMSG("dl_phdr_info allocation failed"); + abort(); + } + + for (i = 0; i < __elf_phdr_info.count; i++) { + memcpy(dlpi, __elf_phdr_info.dlpi + i, sizeof(*dlpi)); + dlpi->dlpi_tls_data = NULL; + id = dlpi->dlpi_tls_modid; + if (id) + dlpi->dlpi_tls_data = _tcb->dtv[id].tls; + st = callback(dlpi, sizeof(*dlpi), data); + } + + free(dlpi); + return st; +} diff --git a/optee_os/lib/libutee/tee_api.c b/optee_os/lib/libutee/tee_api.c new file mode 100644 index 0000000..4c9b240 --- /dev/null +++ b/optee_os/lib/libutee/tee_api.c @@ -0,0 +1,813 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "tee_api_private.h" + +/* + * return a known non-NULL invalid pointer when the + * requested size is zero + */ +#define TEE_NULL_SIZED_VA ((void *)1) +#define TEE_NULL_SIZED_NO_SHARE_VA ((void *)2) + +/* + * Workaround build error in Teaclave TrustZone SDK + * + * These are supposed to be provided by ta/arch/arm/user_ta_header.c, but + * Teaclave TrustZone SDK seems to roll their own in Rust. + */ +uint8_t __ta_no_share_heap[0] __weak; +const size_t __ta_no_share_heap_size __weak; +struct malloc_ctx *__ta_no_share_malloc_ctx __weak; + +static const void *tee_api_instance_data; + +/* System API - Internal Client API */ + +static void copy_param(struct utee_params *up, uint32_t param_types, + const TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t n = 0; + uint64_t a = 0; + uint64_t b = 0; + + up->types = param_types; + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(up->types, n)) { + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + a = params[n].value.a; + b = params[n].value.b; + break; + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + case TEE_PARAM_TYPE_MEMREF_INPUT: + a = (vaddr_t)params[n].memref.buffer; + b = params[n].memref.size; + break; + default: + a = 0; + b = 0; + } + up->vals[n * 2] = a; + up->vals[n * 2 + 1] = b; + } +} + +static void copy_gp11_param(struct utee_params *up, uint32_t param_types, + const __GP11_TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t n = 0; + uint64_t a = 0; + uint64_t b = 0; + + up->types = param_types; + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(up->types, n)) { + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + a = params[n].value.a; + b = params[n].value.b; + break; + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + case TEE_PARAM_TYPE_MEMREF_INPUT: + a = (vaddr_t)params[n].memref.buffer; + b = params[n].memref.size; + break; + default: + a = 0; + b = 0; + } + up->vals[n * 2] = a; + up->vals[n * 2 + 1] = b; + } +} + +static TEE_Result map_tmp_param(struct utee_params *up, void **tmp_buf, + size_t *tmp_len, void *tmp_va[TEE_NUM_PARAMS]) +{ + size_t n = 0; + uint8_t *tb = NULL; + size_t tbl = 0; + size_t tmp_align = sizeof(vaddr_t) * 2; + bool is_tmp_mem[TEE_NUM_PARAMS] = { false }; + void *b = NULL; + size_t s = 0; + const uint32_t flags = TEE_MEMORY_ACCESS_READ; + + /* + * If a memory parameter points to TA private memory we need to + * allocate a temporary buffer to avoid exposing the memory + * directly to the called TA. + */ + + *tmp_buf = NULL; + *tmp_len = 0; + for (n = 0; n < TEE_NUM_PARAMS; n++) { + tmp_va[n] = NULL; + switch (TEE_PARAM_TYPE_GET(up->types, n)) { + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + b = (void *)(vaddr_t)up->vals[n * 2]; + s = up->vals[n * 2 + 1]; + /* + * We're only allocating temporary memory if the + * buffer is completely within TA memory. If it's + * NULL, empty, partially outside or completely + * outside TA memory there's nothing more we need + * to do here. If there's security/permissions + * problem we'll get an error in the + * invoke_command/open_session below. + */ + if (b && s && + !TEE_CheckMemoryAccessRights(flags, b, s)) { + is_tmp_mem[n] = true; + tbl += ROUNDUP(s, tmp_align); + } + break; + default: + break; + } + } + + if (tbl) { + tb = tee_map_zi(tbl, TEE_MEMORY_ACCESS_ANY_OWNER); + if (!tb) + return TEE_ERROR_OUT_OF_MEMORY; + *tmp_buf = tb; + *tmp_len = tbl; + } + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(up->types, n)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + case TEE_PARAM_TYPE_MEMREF_INPUT: + if (!is_tmp_mem[n]) + break; + s = up->vals[n * 2 + 1]; + b = (void *)(vaddr_t)up->vals[n * 2]; + tmp_va[n] = tb; + tb += ROUNDUP(s, tmp_align); + up->vals[n * 2] = (vaddr_t)tmp_va[n]; + if (TEE_PARAM_TYPE_GET(up->types, n) != + TEE_PARAM_TYPE_MEMREF_OUTPUT) + memcpy(tmp_va[n], b, s); + break; + default: + break; + } + } + + return TEE_SUCCESS; + +} + +static void update_out_param(TEE_Param params[TEE_NUM_PARAMS], + void *tmp_va[TEE_NUM_PARAMS], + const struct utee_params *up) +{ + size_t n; + uint32_t types = up->types; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uintptr_t a = up->vals[n * 2]; + uintptr_t b = up->vals[n * 2 + 1]; + + switch (TEE_PARAM_TYPE_GET(types, n)) { + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + params[n].value.a = a; + params[n].value.b = b; + break; + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + if (tmp_va[n]) + memcpy(params[n].memref.buffer, tmp_va[n], + MIN(b, params[n].memref.size)); + params[n].memref.size = b; + break; + default: + break; + } + } +} + +static void update_out_gp11_param(__GP11_TEE_Param params[TEE_NUM_PARAMS], + void *tmp_va[TEE_NUM_PARAMS], + const struct utee_params *up) +{ + size_t n = 0; + uint32_t types = up->types; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uintptr_t a = up->vals[n * 2]; + uintptr_t b = up->vals[n * 2 + 1]; + + switch (TEE_PARAM_TYPE_GET(types, n)) { + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + params[n].value.a = a; + params[n].value.b = b; + break; + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + if (tmp_va[n]) + memcpy(params[n].memref.buffer, tmp_va[n], + MIN(b, params[n].memref.size)); + params[n].memref.size = b; + break; + default: + break; + } + } +} + +static bool bufs_intersect(void *buf1, size_t sz1, void *buf2, size_t sz2) +{ + vaddr_t b1 = (vaddr_t)buf1; + vaddr_t b2 = (vaddr_t)buf2; + vaddr_t e1 = b1 + sz1 - 1; + vaddr_t e2 = b2 + sz2 - 1; + + if (!sz1 || !sz2) + return false; + + if (e1 < b2 || e2 < b1) + return false; + + return true; +} + +static TEE_Result check_mem_access_rights_params(uint32_t flags, void *buf, + size_t len) +{ + size_t n = 0; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uint32_t f = TEE_MEMORY_ACCESS_ANY_OWNER; + + switch (TEE_PARAM_TYPE_GET(ta_param_types, n)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + f |= TEE_MEMORY_ACCESS_WRITE; + fallthrough; + case TEE_PARAM_TYPE_MEMREF_INPUT: + f |= TEE_MEMORY_ACCESS_READ; + if (bufs_intersect(buf, len, + ta_params[n].memref.buffer, + ta_params[n].memref.size)) { + if ((flags & f) != flags) + return TEE_ERROR_ACCESS_DENIED; + } + break; + default: + break; + } + } + + return TEE_SUCCESS; +} + +static bool buf_overlaps_no_share_heap(void *buf, size_t size) +{ + struct malloc_ctx *ctx = __ta_no_share_malloc_ctx; + + return ctx && raw_malloc_buffer_overlaps_heap(ctx, buf, size); +} + +static void check_invoke_param(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t n = 0; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uint32_t f = TEE_MEMORY_ACCESS_ANY_OWNER; + void *buf = params[n].memref.buffer; + size_t size = params[n].memref.size; + + switch (TEE_PARAM_TYPE_GET(pt, n)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + f |= TEE_MEMORY_ACCESS_WRITE; + fallthrough; + case TEE_PARAM_TYPE_MEMREF_INPUT: + f |= TEE_MEMORY_ACCESS_READ; + if (check_mem_access_rights_params(f, buf, size)) + TEE_Panic(0); + if (buf_overlaps_no_share_heap(buf, size)) + TEE_Panic(0); + break; + default: + break; + } + } +} + +TEE_Result TEE_OpenTASession(const TEE_UUID *destination, + uint32_t cancellationRequestTimeout, + uint32_t paramTypes, + TEE_Param params[TEE_NUM_PARAMS], + TEE_TASessionHandle *session, + uint32_t *returnOrigin) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_params up = { }; + uint32_t s = 0; + void *tmp_buf = NULL; + size_t tmp_len = 0; + void *tmp_va[TEE_NUM_PARAMS] = { NULL }; + + if (paramTypes) { + __utee_check_inout_annotation(params, + sizeof(TEE_Param) * + TEE_NUM_PARAMS); + check_invoke_param(paramTypes, params); + } + __utee_check_out_annotation(session, sizeof(*session)); + + copy_param(&up, paramTypes, params); + res = map_tmp_param(&up, &tmp_buf, &tmp_len, tmp_va); + if (res) + goto out; + res = _utee_open_ta_session(destination, cancellationRequestTimeout, + &up, &s, returnOrigin); + update_out_param(params, tmp_va, &up); + if (tmp_buf) { + TEE_Result res2 = tee_unmap(tmp_buf, tmp_len); + + if (res2) + TEE_Panic(res2); + } + +out: + /* + * Specification says that *session must hold TEE_HANDLE_NULL is + * TEE_SUCCESS isn't returned. Set it here explicitly in case + * the syscall fails before out parameters has been updated. + */ + if (res != TEE_SUCCESS) + s = TEE_HANDLE_NULL; + + *session = (TEE_TASessionHandle)(uintptr_t)s; + return res; +} + +TEE_Result __GP11_TEE_OpenTASession(const TEE_UUID *destination, + uint32_t cancellationRequestTimeout, + uint32_t paramTypes, + __GP11_TEE_Param params[TEE_NUM_PARAMS], + TEE_TASessionHandle *session, + uint32_t *returnOrigin) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_params up = { }; + uint32_t s = 0; + void *tmp_buf = NULL; + size_t tmp_len = 0; + void *tmp_va[TEE_NUM_PARAMS] = { NULL }; + + if (paramTypes) + __utee_check_inout_annotation(params, + sizeof(__GP11_TEE_Param) * + TEE_NUM_PARAMS); + __utee_check_out_annotation(session, sizeof(*session)); + + copy_gp11_param(&up, paramTypes, params); + res = map_tmp_param(&up, &tmp_buf, &tmp_len, tmp_va); + if (res) + goto out; + res = _utee_open_ta_session(destination, cancellationRequestTimeout, + &up, &s, returnOrigin); + update_out_gp11_param(params, tmp_va, &up); + if (tmp_buf) { + TEE_Result res2 = tee_unmap(tmp_buf, tmp_len); + + if (res2) + TEE_Panic(res2); + } + +out: + /* + * Specification says that *session must hold TEE_HANDLE_NULL if + * TEE_SUCCESS isn't returned. Set it here explicitly in case + * the syscall fails before out parameters has been updated. + */ + if (res != TEE_SUCCESS) + s = TEE_HANDLE_NULL; + + *session = (TEE_TASessionHandle)(uintptr_t)s; + return res; +} + +void TEE_CloseTASession(TEE_TASessionHandle session) +{ + if (session != TEE_HANDLE_NULL) { + TEE_Result res = _utee_close_ta_session((uintptr_t)session); + + if (res != TEE_SUCCESS) + TEE_Panic(res); + } +} + +TEE_Result TEE_InvokeTACommand(TEE_TASessionHandle session, + uint32_t cancellationRequestTimeout, + uint32_t commandID, uint32_t paramTypes, + TEE_Param params[TEE_NUM_PARAMS], + uint32_t *returnOrigin) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t ret_origin = TEE_ORIGIN_TEE; + struct utee_params up = { }; + void *tmp_buf = NULL; + size_t tmp_len = 0; + void *tmp_va[TEE_NUM_PARAMS] = { NULL }; + + if (paramTypes) { + __utee_check_inout_annotation(params, + sizeof(TEE_Param) * + TEE_NUM_PARAMS); + check_invoke_param(paramTypes, params); + } + if (returnOrigin) + __utee_check_out_annotation(returnOrigin, + sizeof(*returnOrigin)); + + copy_param(&up, paramTypes, params); + res = map_tmp_param(&up, &tmp_buf, &tmp_len, tmp_va); + if (res) + goto out; + res = _utee_invoke_ta_command((uintptr_t)session, + cancellationRequestTimeout, + commandID, &up, &ret_origin); + update_out_param(params, tmp_va, &up); + if (tmp_buf) { + TEE_Result res2 = tee_unmap(tmp_buf, tmp_len); + + if (res2) + TEE_Panic(res2); + } + +out: + if (returnOrigin != NULL) + *returnOrigin = ret_origin; + + if (ret_origin == TEE_ORIGIN_TRUSTED_APP) + return res; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_OUT_OF_MEMORY && + res != TEE_ERROR_TARGET_DEAD) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_InvokeTACommand(TEE_TASessionHandle session, + uint32_t cancellationRequestTimeout, + uint32_t commandID, uint32_t paramTypes, + __GP11_TEE_Param params[TEE_NUM_PARAMS], + uint32_t *returnOrigin) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t ret_origin = TEE_ORIGIN_TEE; + struct utee_params up = { }; + void *tmp_buf = NULL; + size_t tmp_len = 0; + void *tmp_va[TEE_NUM_PARAMS] = { NULL }; + + if (paramTypes) + __utee_check_inout_annotation(params, + sizeof(__GP11_TEE_Param) * + TEE_NUM_PARAMS); + if (returnOrigin) + __utee_check_out_annotation(returnOrigin, + sizeof(*returnOrigin)); + + copy_gp11_param(&up, paramTypes, params); + res = map_tmp_param(&up, &tmp_buf, &tmp_len, tmp_va); + if (res) + goto out; + res = _utee_invoke_ta_command((uintptr_t)session, + cancellationRequestTimeout, + commandID, &up, &ret_origin); + update_out_gp11_param(params, tmp_va, &up); + if (tmp_buf) { + TEE_Result res2 = tee_unmap(tmp_buf, tmp_len); + + if (res2) + TEE_Panic(res2); + } + +out: + if (returnOrigin) + *returnOrigin = ret_origin; + + if (ret_origin == TEE_ORIGIN_TRUSTED_APP) + return res; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_OUT_OF_MEMORY && + res != TEE_ERROR_TARGET_DEAD) + TEE_Panic(res); + + return res; +} + +/* System API - Cancellations */ + +bool TEE_GetCancellationFlag(void) +{ + uint32_t c; + TEE_Result res = _utee_get_cancellation_flag(&c); + + if (res != TEE_SUCCESS) + c = 0; + return !!c; +} + +bool TEE_UnmaskCancellation(void) +{ + uint32_t old_mask; + TEE_Result res = _utee_unmask_cancellation(&old_mask); + + if (res != TEE_SUCCESS) + TEE_Panic(res); + return !!old_mask; +} + +bool TEE_MaskCancellation(void) +{ + uint32_t old_mask; + TEE_Result res = _utee_mask_cancellation(&old_mask); + + if (res != TEE_SUCCESS) + TEE_Panic(res); + return !!old_mask; +} + +/* System API - Memory Management */ + +TEE_Result TEE_CheckMemoryAccessRights(uint32_t accessFlags, void *buffer, + size_t size) +{ + uint32_t flags = accessFlags; + + if (!size) + return TEE_SUCCESS; + + /* + * Check access rights against memory mapping. If this check is + * OK the size can't cause an overflow when added with buffer. + */ + if (_utee_check_access_rights(accessFlags, buffer, size)) + return TEE_ERROR_ACCESS_DENIED; + + /* + * Check access rights against input parameters. + * + * Clear eventual extension flags like TEE_MEMORY_ACCESS_NONSECURE + * and TEE_MEMORY_ACCESS_SECURE. + */ + flags &= TEE_MEMORY_ACCESS_READ | TEE_MEMORY_ACCESS_WRITE | + TEE_MEMORY_ACCESS_ANY_OWNER; + if (check_mem_access_rights_params(flags, buffer, size)) + return TEE_ERROR_ACCESS_DENIED; + + if (malloc_buffer_overlaps_heap(buffer, size) && + !malloc_buffer_is_within_alloced(buffer, size)) + return TEE_ERROR_ACCESS_DENIED; + + return TEE_SUCCESS; +} + +TEE_Result __GP11_TEE_CheckMemoryAccessRights(uint32_t accessFlags, + void *buffer, uint32_t size) +{ + return TEE_CheckMemoryAccessRights(accessFlags, buffer, size); +} + +void TEE_SetInstanceData(const void *instanceData) +{ + tee_api_instance_data = instanceData; +} + +const void *TEE_GetInstanceData(void) +{ + return tee_api_instance_data; +} + +void *TEE_MemMove(void *dest, const void *src, size_t size) +{ + return memmove(dest, src, size); +} + +void *__GP11_TEE_MemMove(void *dest, const void *src, uint32_t size) +{ + return TEE_MemMove(dest, src, size); +} + +int32_t TEE_MemCompare(const void *buffer1, const void *buffer2, size_t size) +{ + return consttime_memcmp(buffer1, buffer2, size); +} + +int32_t __GP11_TEE_MemCompare(const void *buffer1, const void *buffer2, + uint32_t size) +{ + return TEE_MemCompare(buffer1, buffer2, size); +} + +void TEE_MemFill(void *buff, uint32_t x, size_t size) +{ + memset(buff, x, size); +} + +void __GP11_TEE_MemFill(void *buff, uint32_t x, uint32_t size) +{ + TEE_MemFill(buff, x, size); +} + +/* Date & Time API */ + +void TEE_GetSystemTime(TEE_Time *time) +{ + TEE_Result res = _utee_get_time(UTEE_TIME_CAT_SYSTEM, time); + + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +TEE_Result TEE_Wait(uint32_t timeout) +{ + TEE_Result res = _utee_wait(timeout); + + if (res != TEE_SUCCESS && res != TEE_ERROR_CANCEL) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_GetTAPersistentTime(TEE_Time *time) +{ + TEE_Result res; + + res = _utee_get_time(UTEE_TIME_CAT_TA_PERSISTENT, time); + + if (res != TEE_SUCCESS && res != TEE_ERROR_OVERFLOW) { + time->seconds = 0; + time->millis = 0; + } + + if (res != TEE_SUCCESS && + res != TEE_ERROR_TIME_NOT_SET && + res != TEE_ERROR_TIME_NEEDS_RESET && + res != TEE_ERROR_OVERFLOW && + res != TEE_ERROR_OUT_OF_MEMORY) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_SetTAPersistentTime(const TEE_Time *time) +{ + TEE_Result res; + + res = _utee_set_ta_time(time); + + if (res != TEE_SUCCESS && + res != TEE_ERROR_OUT_OF_MEMORY && + res != TEE_ERROR_STORAGE_NO_SPACE) + TEE_Panic(res); + + return res; +} + +void TEE_GetREETime(TEE_Time *time) +{ + TEE_Result res = _utee_get_time(UTEE_TIME_CAT_REE, time); + + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void *TEE_Malloc(size_t len, uint32_t hint) +{ + switch (hint) { + case TEE_MALLOC_FILL_ZERO: + if (!len) + return TEE_NULL_SIZED_VA; + return calloc(1, len); + + case TEE_MALLOC_NO_FILL: + TEE_Panic(0); + break; + + case TEE_MALLOC_NO_SHARE: + if (!len) + return TEE_NULL_SIZED_NO_SHARE_VA; + if (!__ta_no_share_malloc_ctx) + return NULL; + return raw_calloc(0, 0, 1, len, __ta_no_share_malloc_ctx); + + case TEE_MALLOC_NO_FILL | TEE_MALLOC_NO_SHARE: + if (!len) + return TEE_NULL_SIZED_NO_SHARE_VA; + if (!__ta_no_share_malloc_ctx) + return NULL; + return raw_malloc(0, 0, len, __ta_no_share_malloc_ctx); + + case TEE_USER_MEM_HINT_NO_FILL_ZERO: + if (!len) + return TEE_NULL_SIZED_VA; + return malloc(len); + + default: + break; + } + + EMSG("Invalid hint %#" PRIx32, hint); + + return NULL; +} + +void *__GP11_TEE_Malloc(uint32_t size, uint32_t hint) +{ + return TEE_Malloc(size, hint); +} + +static bool addr_is_in_no_share_heap(void *p) +{ + return buf_overlaps_no_share_heap(p, 1); +} + +void *TEE_Realloc(void *buffer, size_t newSize) +{ + if (!newSize) { + void *ret = NULL; + + if (addr_is_in_no_share_heap(buffer)) + ret = TEE_NULL_SIZED_NO_SHARE_VA; + else + ret = TEE_NULL_SIZED_VA; + + TEE_Free(buffer); + + return ret; + } + + if (buffer == TEE_NULL_SIZED_VA) + return calloc(1, newSize); + if (buffer == TEE_NULL_SIZED_NO_SHARE_VA) { + if (!__ta_no_share_malloc_ctx) + return NULL; + return raw_calloc(0, 0, 1, newSize, __ta_no_share_malloc_ctx); + } + + if (addr_is_in_no_share_heap(buffer)) + return raw_realloc(buffer, 0, 0, newSize, + __ta_no_share_malloc_ctx); + else + return realloc(buffer, newSize); +} + +void *__GP11_TEE_Realloc(void *buffer, uint32_t newSize) +{ + return TEE_Realloc(buffer, newSize); +} + +void TEE_Free(void *buffer) +{ + if (buffer != TEE_NULL_SIZED_VA && + buffer != TEE_NULL_SIZED_NO_SHARE_VA) { + if (addr_is_in_no_share_heap(buffer)) + raw_free(buffer, __ta_no_share_malloc_ctx, false); + else + free(buffer); + } +} + +/* Cache maintenance support (TA requires the CACHE_MAINTENANCE property) */ +TEE_Result TEE_CacheClean(char *buf, size_t len) +{ + return _utee_cache_operation(buf, len, TEE_CACHECLEAN); +} +TEE_Result TEE_CacheFlush(char *buf, size_t len) +{ + return _utee_cache_operation(buf, len, TEE_CACHEFLUSH); +} + +TEE_Result TEE_CacheInvalidate(char *buf, size_t len) +{ + return _utee_cache_operation(buf, len, TEE_CACHEINVALIDATE); +} diff --git a/optee_os/lib/libutee/tee_api_arith_mpi.c b/optee_os/lib/libutee/tee_api_arith_mpi.c new file mode 100644 index 0000000..5842f13 --- /dev/null +++ b/optee_os/lib/libutee/tee_api_arith_mpi.c @@ -0,0 +1,1017 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MPI_MEMPOOL_SIZE (14 * 1024) + +static void __noreturn api_panic(const char *func, int line, const char *msg) +{ + printf("Panic function %s, line %d: %s\n", func, line, msg); + TEE_Panic(0xB16127 /*BIGINT*/); + while (1) + ; /* Panic will crash the thread */ +} + +#define API_PANIC(x) api_panic(__func__, __LINE__, x) + +static void __noreturn mpi_panic(const char *func, int line, int rc) +{ + printf("Panic function %s, line %d, code %d\n", func, line, rc); + TEE_Panic(0xB16127 /*BIGINT*/); + while (1) + ; /* Panic will crash the thread */ +} + +#define MPI_CHECK(x) do { \ + int _rc = (x); \ + \ + if (_rc) \ + mpi_panic(__func__, __LINE__, _rc); \ + } while (0) + +void _TEE_MathAPI_Init(void) +{ + static uint8_t data[MPI_MEMPOOL_SIZE] __aligned(MEMPOOL_ALIGN); + + mbedtls_mpi_mempool = mempool_alloc_pool(data, sizeof(data), NULL); + if (!mbedtls_mpi_mempool) + API_PANIC("Failed to initialize memory pool"); +} + +struct bigint_hdr { + int32_t sign; + uint16_t alloc_size; + uint16_t nblimbs; +}; + +#define BIGINT_HDR_SIZE_IN_U32 2 + +static TEE_Result copy_mpi_to_bigint(mbedtls_mpi *mpi, TEE_BigInt *bigInt) +{ + struct bigint_hdr *hdr = (struct bigint_hdr *)bigInt; + size_t n = mpi->n; + + /* Trim of eventual insignificant zeroes */ + while (n && !mpi->p[n - 1]) + n--; + + if (hdr->alloc_size < n) + return TEE_ERROR_OVERFLOW; + + hdr->nblimbs = n; + hdr->sign = mpi->s; + memcpy(hdr + 1, mpi->p, mpi->n * sizeof(mbedtls_mpi_uint)); + + return TEE_SUCCESS; +} + +/* + * Initializes a MPI. + * + * A temporary MPI is allocated and if a bigInt is supplied the MPI is + * initialized with the value of the bigInt. + */ +static void get_mpi(mbedtls_mpi *mpi, const TEE_BigInt *bigInt) +{ + /* + * The way the GP spec is defining the bignums it's + * difficult/tricky to do it using 64-bit arithmetics given that + * we'd need 64-bit alignment of the data as well. + */ + COMPILE_TIME_ASSERT(sizeof(mbedtls_mpi_uint) == sizeof(uint32_t)); + + /* + * The struct bigint_hdr is the overhead added to the bigint and + * is required to take exactly 2 uint32_t. + */ + COMPILE_TIME_ASSERT(sizeof(struct bigint_hdr) == + sizeof(uint32_t) * BIGINT_HDR_SIZE_IN_U32); + + mbedtls_mpi_init_mempool(mpi); + + if (bigInt) { + const struct bigint_hdr *hdr = (struct bigint_hdr *)bigInt; + const mbedtls_mpi_uint *p = (const mbedtls_mpi_uint *)(hdr + 1); + size_t n = hdr->nblimbs; + + /* Trim of eventual insignificant zeroes */ + while (n && !p[n - 1]) + n--; + + MPI_CHECK(mbedtls_mpi_grow(mpi, n)); + mpi->s = hdr->sign; + memcpy(mpi->p, p, n * sizeof(mbedtls_mpi_uint)); + } +} + +void TEE_BigIntInit(TEE_BigInt *bigInt, size_t len) +{ + struct bigint_hdr *hdr = (struct bigint_hdr *)bigInt; + + static_assert(MBEDTLS_MPI_MAX_LIMBS + BIGINT_HDR_SIZE_IN_U32 >= + CFG_TA_BIGNUM_MAX_BITS / 32); + + memset(bigInt, 0, len * sizeof(uint32_t)); + hdr->sign = 1; + + /* "gpd.tee.arith.maxBigIntSize" is assigned CFG_TA_BIGNUM_MAX_BITS */ + if (len > CFG_TA_BIGNUM_MAX_BITS / 4) + API_PANIC("Too large bigint"); + hdr->alloc_size = len - BIGINT_HDR_SIZE_IN_U32; +} + +void __GP11_TEE_BigIntInit(TEE_BigInt *bigInt, uint32_t len) +{ + TEE_BigIntInit(bigInt, len); +} + +TEE_Result TEE_BigIntConvertFromOctetString(TEE_BigInt *dest, + const uint8_t *buffer, + size_t bufferLen, int32_t sign) +{ + TEE_Result res; + mbedtls_mpi mpi_dest; + + get_mpi(&mpi_dest, NULL); + + if (mbedtls_mpi_read_binary(&mpi_dest, buffer, bufferLen)) + res = TEE_ERROR_OVERFLOW; + else + res = TEE_SUCCESS; + + if (sign < 0) + mpi_dest.s = -1; + + if (!res) + res = copy_mpi_to_bigint(&mpi_dest, dest); + + mbedtls_mpi_free(&mpi_dest); + + return res; +} + +TEE_Result __GP11_TEE_BigIntConvertFromOctetString(TEE_BigInt *dest, + const uint8_t *buffer, + uint32_t bufferLen, + int32_t sign) +{ + return TEE_BigIntConvertFromOctetString(dest, buffer, bufferLen, sign); +} + +TEE_Result TEE_BigIntConvertToOctetString(uint8_t *buffer, size_t *bufferLen, + const TEE_BigInt *bigInt) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_mpi mpi; + size_t sz; + + get_mpi(&mpi, bigInt); + + sz = mbedtls_mpi_size(&mpi); + if (sz <= *bufferLen) + MPI_CHECK(mbedtls_mpi_write_binary(&mpi, buffer, sz)); + else + res = TEE_ERROR_SHORT_BUFFER; + + *bufferLen = sz; + + mbedtls_mpi_free(&mpi); + + return res; +} + +TEE_Result __GP11_TEE_BigIntConvertToOctetString(uint8_t *buffer, + uint32_t *bufferLen, + const TEE_BigInt *bigInt) +{ + TEE_Result res = TEE_SUCCESS; + size_t l = *bufferLen; + + res = TEE_BigIntConvertToOctetString(buffer, &l, bigInt); + *bufferLen = l; + return res; +} + +void TEE_BigIntConvertFromS32(TEE_BigInt *dest, int32_t shortVal) +{ + mbedtls_mpi mpi; + + get_mpi(&mpi, dest); + + MPI_CHECK(mbedtls_mpi_lset(&mpi, shortVal)); + + MPI_CHECK(copy_mpi_to_bigint(&mpi, dest)); + mbedtls_mpi_free(&mpi); +} + +TEE_Result TEE_BigIntConvertToS32(int32_t *dest, const TEE_BigInt *src) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_mpi mpi; + uint32_t v; + + get_mpi(&mpi, src); + + if (mbedtls_mpi_write_binary(&mpi, (void *)&v, sizeof(v))) { + res = TEE_ERROR_OVERFLOW; + goto out; + } + + if (mpi.s > 0) { + if (ADD_OVERFLOW(0, TEE_U32_FROM_BIG_ENDIAN(v), dest)) + res = TEE_ERROR_OVERFLOW; + } else { + if (SUB_OVERFLOW(0, TEE_U32_FROM_BIG_ENDIAN(v), dest)) + res = TEE_ERROR_OVERFLOW; + } + +out: + mbedtls_mpi_free(&mpi); + + return res; +} + +int32_t TEE_BigIntCmp(const TEE_BigInt *op1, const TEE_BigInt *op2) +{ + mbedtls_mpi mpi1; + mbedtls_mpi mpi2; + int32_t rc; + + get_mpi(&mpi1, op1); + get_mpi(&mpi2, op2); + + rc = mbedtls_mpi_cmp_mpi(&mpi1, &mpi2); + + mbedtls_mpi_free(&mpi1); + mbedtls_mpi_free(&mpi2); + + return rc; +} + +int32_t TEE_BigIntCmpS32(const TEE_BigInt *op, int32_t shortVal) +{ + mbedtls_mpi mpi; + int32_t rc; + + get_mpi(&mpi, op); + + rc = mbedtls_mpi_cmp_int(&mpi, shortVal); + + mbedtls_mpi_free(&mpi); + + return rc; +} + +void TEE_BigIntShiftRight(TEE_BigInt *dest, const TEE_BigInt *op, size_t bits) +{ + mbedtls_mpi mpi_dest; + mbedtls_mpi mpi_op; + + get_mpi(&mpi_dest, dest); + + if (dest == op) { + MPI_CHECK(mbedtls_mpi_shift_r(&mpi_dest, bits)); + goto out; + } + + get_mpi(&mpi_op, op); + + if (mbedtls_mpi_size(&mpi_dest) >= mbedtls_mpi_size(&mpi_op)) { + MPI_CHECK(mbedtls_mpi_copy(&mpi_dest, &mpi_op)); + MPI_CHECK(mbedtls_mpi_shift_r(&mpi_dest, bits)); + } else { + mbedtls_mpi mpi_t; + + get_mpi(&mpi_t, NULL); + + /* + * We're using a temporary buffer to avoid the corner case + * where destination is unexpectedly overflowed by up to + * @bits number of bits. + */ + MPI_CHECK(mbedtls_mpi_copy(&mpi_t, &mpi_op)); + MPI_CHECK(mbedtls_mpi_shift_r(&mpi_t, bits)); + MPI_CHECK(mbedtls_mpi_copy(&mpi_dest, &mpi_t)); + + mbedtls_mpi_free(&mpi_t); + } + + mbedtls_mpi_free(&mpi_op); + +out: + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest, dest)); + mbedtls_mpi_free(&mpi_dest); +} + +void __GP11_TEE_BigIntShiftRight(TEE_BigInt *dest, const TEE_BigInt *op, + uint32_t bits) +{ + TEE_BigIntShiftRight(dest, op, bits); +} + +bool TEE_BigIntGetBit(const TEE_BigInt *src, uint32_t bitIndex) +{ + bool rc; + mbedtls_mpi mpi; + + get_mpi(&mpi, src); + + rc = mbedtls_mpi_get_bit(&mpi, bitIndex); + + mbedtls_mpi_free(&mpi); + + return rc; +} + +uint32_t TEE_BigIntGetBitCount(const TEE_BigInt *src) +{ + uint32_t rc; + mbedtls_mpi mpi; + + get_mpi(&mpi, src); + + rc = mbedtls_mpi_bitlen(&mpi); + + mbedtls_mpi_free(&mpi); + + return rc; +} + +TEE_Result TEE_BigIntSetBit(TEE_BigInt *op, uint32_t bitIndex, bool value) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_mpi mpi = { }; + int rc = 0; + + get_mpi(&mpi, op); + + rc = mbedtls_mpi_set_bit(&mpi, bitIndex, value); + if (rc) + res = TEE_ERROR_OVERFLOW; + else + res = copy_mpi_to_bigint(&mpi, op); + + mbedtls_mpi_free(&mpi); + + return res; +} + +TEE_Result TEE_BigIntAssign(TEE_BigInt *dest, const TEE_BigInt *src) +{ + const struct bigint_hdr *src_hdr = (struct bigint_hdr *)src; + struct bigint_hdr *dst_hdr = (struct bigint_hdr *)dest; + + if (dst_hdr == src_hdr) + return TEE_SUCCESS; + + if (dst_hdr->alloc_size < src_hdr->nblimbs) + return TEE_ERROR_OVERFLOW; + + dst_hdr->nblimbs = src_hdr->nblimbs; + dst_hdr->sign = src_hdr->sign; + memcpy(dst_hdr + 1, src_hdr + 1, src_hdr->nblimbs * sizeof(uint32_t)); + + return TEE_SUCCESS; +} + +TEE_Result TEE_BigIntAbs(TEE_BigInt *dest, const TEE_BigInt *src) +{ + TEE_Result res = TEE_BigIntAssign(dest, src); + + if (!res) + ((struct bigint_hdr *)dest)->sign = 1; + + return res; +} + +static void bigint_binary(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, + int (*func)(mbedtls_mpi *X, const mbedtls_mpi *A, + const mbedtls_mpi *B)) +{ + mbedtls_mpi mpi_dest; + mbedtls_mpi mpi_op1; + mbedtls_mpi mpi_op2; + mbedtls_mpi *pop1 = &mpi_op1; + mbedtls_mpi *pop2 = &mpi_op2; + + get_mpi(&mpi_dest, dest); + + if (op1 == dest) + pop1 = &mpi_dest; + else + get_mpi(&mpi_op1, op1); + + if (op2 == dest) + pop2 = &mpi_dest; + else if (op2 == op1) + pop2 = pop1; + else + get_mpi(&mpi_op2, op2); + + MPI_CHECK(func(&mpi_dest, pop1, pop2)); + + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest, dest)); + mbedtls_mpi_free(&mpi_dest); + if (pop1 == &mpi_op1) + mbedtls_mpi_free(&mpi_op1); + if (pop2 == &mpi_op2) + mbedtls_mpi_free(&mpi_op2); +} + +static void bigint_binary_mod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n, + int (*func)(mbedtls_mpi *X, const mbedtls_mpi *A, + const mbedtls_mpi *B)) +{ + mbedtls_mpi mpi_dest; + mbedtls_mpi mpi_op1; + mbedtls_mpi mpi_op2; + mbedtls_mpi mpi_n; + mbedtls_mpi *pop1 = &mpi_op1; + mbedtls_mpi *pop2 = &mpi_op2; + mbedtls_mpi mpi_t; + + if (TEE_BigIntCmpS32(n, 2) < 0) + API_PANIC("Modulus is too short"); + + get_mpi(&mpi_dest, dest); + get_mpi(&mpi_n, n); + + if (op1 == dest) + pop1 = &mpi_dest; + else + get_mpi(&mpi_op1, op1); + + if (op2 == dest) + pop2 = &mpi_dest; + else if (op2 == op1) + pop2 = pop1; + else + get_mpi(&mpi_op2, op2); + + get_mpi(&mpi_t, NULL); + + MPI_CHECK(func(&mpi_t, pop1, pop2)); + MPI_CHECK(mbedtls_mpi_mod_mpi(&mpi_dest, &mpi_t, &mpi_n)); + + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest, dest)); + mbedtls_mpi_free(&mpi_dest); + if (pop1 == &mpi_op1) + mbedtls_mpi_free(&mpi_op1); + if (pop2 == &mpi_op2) + mbedtls_mpi_free(&mpi_op2); + mbedtls_mpi_free(&mpi_t); + mbedtls_mpi_free(&mpi_n); +} + +void TEE_BigIntAdd(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2) +{ + bigint_binary(dest, op1, op2, mbedtls_mpi_add_mpi); +} + +void TEE_BigIntSub(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2) +{ + bigint_binary(dest, op1, op2, mbedtls_mpi_sub_mpi); +} + +void TEE_BigIntNeg(TEE_BigInt *dest, const TEE_BigInt *src) +{ + mbedtls_mpi mpi_dest; + + get_mpi(&mpi_dest, dest); + + if (dest != src) { + mbedtls_mpi mpi_src; + + get_mpi(&mpi_src, src); + + MPI_CHECK(mbedtls_mpi_copy(&mpi_dest, &mpi_src)); + + mbedtls_mpi_free(&mpi_src); + } + + mpi_dest.s *= -1; + + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest, dest)); + mbedtls_mpi_free(&mpi_dest); +} + +void TEE_BigIntMul(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2) +{ + size_t bs1 = TEE_BigIntGetBitCount(op1); + size_t bs2 = TEE_BigIntGetBitCount(op2); + size_t s = TEE_BigIntSizeInU32(bs1) + TEE_BigIntSizeInU32(bs2); + TEE_BigInt zero[TEE_BigIntSizeInU32(1)] = { 0 }; + TEE_BigInt *tmp = NULL; + + tmp = mempool_alloc(mbedtls_mpi_mempool, sizeof(uint32_t) * s); + if (!tmp) + TEE_Panic(TEE_ERROR_OUT_OF_MEMORY); + + TEE_BigIntInit(tmp, s); + TEE_BigIntInit(zero, TEE_BigIntSizeInU32(1)); + + bigint_binary(tmp, op1, op2, mbedtls_mpi_mul_mpi); + + TEE_BigIntAdd(dest, tmp, zero); + + mempool_free(mbedtls_mpi_mempool, tmp); +} + +void TEE_BigIntSquare(TEE_BigInt *dest, const TEE_BigInt *op) +{ + TEE_BigIntMul(dest, op, op); +} + +void TEE_BigIntDiv(TEE_BigInt *dest_q, TEE_BigInt *dest_r, + const TEE_BigInt *op1, const TEE_BigInt *op2) +{ + mbedtls_mpi mpi_dest_q; + mbedtls_mpi mpi_dest_r; + mbedtls_mpi mpi_op1; + mbedtls_mpi mpi_op2; + mbedtls_mpi *pop1 = &mpi_op1; + mbedtls_mpi *pop2 = &mpi_op2; + + get_mpi(&mpi_dest_q, dest_q); + get_mpi(&mpi_dest_r, dest_r); + + if (op1 == dest_q) + pop1 = &mpi_dest_q; + else if (op1 == dest_r) + pop1 = &mpi_dest_r; + else + get_mpi(&mpi_op1, op1); + + if (op2 == dest_q) + pop2 = &mpi_dest_q; + else if (op2 == dest_r) + pop2 = &mpi_dest_r; + else if (op2 == op1) + pop2 = pop1; + else + get_mpi(&mpi_op2, op2); + + MPI_CHECK(mbedtls_mpi_div_mpi(&mpi_dest_q, &mpi_dest_r, pop1, pop2)); + + if (dest_q) + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest_q, dest_q)); + if (dest_r) + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest_r, dest_r)); + mbedtls_mpi_free(&mpi_dest_q); + mbedtls_mpi_free(&mpi_dest_r); + if (pop1 == &mpi_op1) + mbedtls_mpi_free(&mpi_op1); + if (pop2 == &mpi_op2) + mbedtls_mpi_free(&mpi_op2); +} + +void TEE_BigIntMod(TEE_BigInt *dest, const TEE_BigInt *op, const TEE_BigInt *n) +{ + if (TEE_BigIntCmpS32(n, 2) < 0) + API_PANIC("Modulus is too short"); + + bigint_binary(dest, op, n, mbedtls_mpi_mod_mpi); +} + +void TEE_BigIntAddMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n) +{ + bigint_binary_mod(dest, op1, op2, n, mbedtls_mpi_add_mpi); +} + +void TEE_BigIntSubMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n) +{ + bigint_binary_mod(dest, op1, op2, n, mbedtls_mpi_sub_mpi); +} + +void TEE_BigIntMulMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n) +{ + bigint_binary_mod(dest, op1, op2, n, mbedtls_mpi_mul_mpi); +} + +void TEE_BigIntSquareMod(TEE_BigInt *dest, const TEE_BigInt *op, + const TEE_BigInt *n) +{ + TEE_BigIntMulMod(dest, op, op, n); +} + +void TEE_BigIntInvMod(TEE_BigInt *dest, const TEE_BigInt *op, + const TEE_BigInt *n) +{ + mbedtls_mpi mpi_dest; + mbedtls_mpi mpi_op; + mbedtls_mpi mpi_n; + mbedtls_mpi *pop = &mpi_op; + + if (TEE_BigIntCmpS32(n, 2) < 0 || TEE_BigIntCmpS32(op, 0) == 0) + API_PANIC("too small modulus or trying to invert zero"); + + get_mpi(&mpi_dest, dest); + get_mpi(&mpi_n, n); + + if (op == dest) + pop = &mpi_dest; + else + get_mpi(&mpi_op, op); + + MPI_CHECK(mbedtls_mpi_inv_mod(&mpi_dest, pop, &mpi_n)); + + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest, dest)); + mbedtls_mpi_free(&mpi_dest); + mbedtls_mpi_free(&mpi_n); + if (pop == &mpi_op) + mbedtls_mpi_free(&mpi_op); +} + +bool TEE_BigIntRelativePrime(const TEE_BigInt *op1, const TEE_BigInt *op2) +{ + bool rc; + mbedtls_mpi mpi_op1; + mbedtls_mpi mpi_op2; + mbedtls_mpi *pop2 = &mpi_op2; + mbedtls_mpi gcd; + + get_mpi(&mpi_op1, op1); + + if (op2 == op1) + pop2 = &mpi_op1; + else + get_mpi(&mpi_op2, op2); + + get_mpi(&gcd, NULL); + + MPI_CHECK(mbedtls_mpi_gcd(&gcd, &mpi_op1, &mpi_op2)); + + rc = !mbedtls_mpi_cmp_int(&gcd, 1); + + mbedtls_mpi_free(&gcd); + mbedtls_mpi_free(&mpi_op1); + if (pop2 == &mpi_op2) + mbedtls_mpi_free(&mpi_op2); + + return rc; +} + +static bool mpi_is_odd(mbedtls_mpi *x) +{ + return mbedtls_mpi_get_bit(x, 0); +} + +static bool mpi_is_even(mbedtls_mpi *x) +{ + return !mpi_is_odd(x); +} + +TEE_Result TEE_BigIntExpMod(TEE_BigInt *dest, const TEE_BigInt *op1, + const TEE_BigInt *op2, const TEE_BigInt *n, + const TEE_BigIntFMMContext *context __unused) +{ + TEE_Result res = TEE_SUCCESS; + mbedtls_mpi mpi_dest = { }; + mbedtls_mpi mpi_op1 = { }; + mbedtls_mpi mpi_op2 = { }; + mbedtls_mpi mpi_n = { }; + mbedtls_mpi *pop1 = &mpi_op1; + mbedtls_mpi *pop2 = &mpi_op2; + + get_mpi(&mpi_dest, dest); + get_mpi(&mpi_n, n); + if (op1 == dest) + pop1 = &mpi_dest; + else + get_mpi(&mpi_op1, op1); + + if (op2 == dest) + pop2 = &mpi_dest; + else if (op2 == op1) + pop2 = pop1; + else + get_mpi(&mpi_op2, op2); + + if (mbedtls_mpi_cmp_int(&mpi_n, 2) <= 0) + API_PANIC("too small modulus"); + if (!mpi_is_odd(&mpi_n)) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + + MPI_CHECK(mbedtls_mpi_exp_mod(&mpi_dest, pop1, pop2, &mpi_n, NULL)); + MPI_CHECK(copy_mpi_to_bigint(&mpi_dest, dest)); +out: + mbedtls_mpi_free(&mpi_dest); + mbedtls_mpi_free(&mpi_n); + if (pop1 == &mpi_op1) + mbedtls_mpi_free(&mpi_op1); + if (pop2 == &mpi_op2) + mbedtls_mpi_free(&mpi_op2); + + return res; +} + +/* + * Based on libmpa implementation __mpa_egcd(), modified to work with MPI + * instead. + */ +static void mpi_egcd(mbedtls_mpi *gcd, mbedtls_mpi *a, mbedtls_mpi *b, + mbedtls_mpi *x_in, mbedtls_mpi *y_in) +{ + mbedtls_mpi_uint k; + mbedtls_mpi A; + mbedtls_mpi B; + mbedtls_mpi C; + mbedtls_mpi D; + mbedtls_mpi x; + mbedtls_mpi y; + mbedtls_mpi u; + + get_mpi(&A, NULL); + get_mpi(&B, NULL); + get_mpi(&C, NULL); + get_mpi(&D, NULL); + get_mpi(&x, NULL); + get_mpi(&y, NULL); + get_mpi(&u, NULL); + + /* have y < x from assumption */ + if (!mbedtls_mpi_cmp_int(y_in, 0)) { + MPI_CHECK(mbedtls_mpi_lset(a, 1)); + MPI_CHECK(mbedtls_mpi_lset(b, 0)); + MPI_CHECK(mbedtls_mpi_copy(gcd, x_in)); + goto out; + } + + MPI_CHECK(mbedtls_mpi_copy(&x, x_in)); + MPI_CHECK(mbedtls_mpi_copy(&y, y_in)); + + k = 0; + while (mpi_is_even(&x) && mpi_is_even(&y)) { + k++; + MPI_CHECK(mbedtls_mpi_shift_r(&x, 1)); + MPI_CHECK(mbedtls_mpi_shift_r(&y, 1)); + } + + MPI_CHECK(mbedtls_mpi_copy(&u, &x)); + MPI_CHECK(mbedtls_mpi_copy(gcd, &y)); + MPI_CHECK(mbedtls_mpi_lset(&A, 1)); + MPI_CHECK(mbedtls_mpi_lset(&B, 0)); + MPI_CHECK(mbedtls_mpi_lset(&C, 0)); + MPI_CHECK(mbedtls_mpi_lset(&D, 1)); + + while (mbedtls_mpi_cmp_int(&u, 0)) { + while (mpi_is_even(&u)) { + MPI_CHECK(mbedtls_mpi_shift_r(&u, 1)); + if (mpi_is_odd(&A) || mpi_is_odd(&B)) { + MPI_CHECK(mbedtls_mpi_add_mpi(&A, &A, &y)); + MPI_CHECK(mbedtls_mpi_sub_mpi(&B, &B, &x)); + } + MPI_CHECK(mbedtls_mpi_shift_r(&A, 1)); + MPI_CHECK(mbedtls_mpi_shift_r(&B, 1)); + } + + while (mpi_is_even(gcd)) { + MPI_CHECK(mbedtls_mpi_shift_r(gcd, 1)); + if (mpi_is_odd(&C) || mpi_is_odd(&D)) { + MPI_CHECK(mbedtls_mpi_add_mpi(&C, &C, &y)); + MPI_CHECK(mbedtls_mpi_sub_mpi(&D, &D, &x)); + } + MPI_CHECK(mbedtls_mpi_shift_r(&C, 1)); + MPI_CHECK(mbedtls_mpi_shift_r(&D, 1)); + + } + + if (mbedtls_mpi_cmp_mpi(&u, gcd) >= 0) { + MPI_CHECK(mbedtls_mpi_sub_mpi(&u, &u, gcd)); + MPI_CHECK(mbedtls_mpi_sub_mpi(&A, &A, &C)); + MPI_CHECK(mbedtls_mpi_sub_mpi(&B, &B, &D)); + } else { + MPI_CHECK(mbedtls_mpi_sub_mpi(gcd, gcd, &u)); + MPI_CHECK(mbedtls_mpi_sub_mpi(&C, &C, &A)); + MPI_CHECK(mbedtls_mpi_sub_mpi(&D, &D, &B)); + } + } + + MPI_CHECK(mbedtls_mpi_copy(a, &C)); + MPI_CHECK(mbedtls_mpi_copy(b, &D)); + MPI_CHECK(mbedtls_mpi_shift_l(gcd, k)); + +out: + mbedtls_mpi_free(&A); + mbedtls_mpi_free(&B); + mbedtls_mpi_free(&C); + mbedtls_mpi_free(&D); + mbedtls_mpi_free(&x); + mbedtls_mpi_free(&y); + mbedtls_mpi_free(&u); +} + +void TEE_BigIntComputeExtendedGcd(TEE_BigInt *gcd, TEE_BigInt *u, + TEE_BigInt *v, const TEE_BigInt *op1, + const TEE_BigInt *op2) +{ + mbedtls_mpi mpi_gcd_res; + mbedtls_mpi mpi_op1; + mbedtls_mpi mpi_op2; + mbedtls_mpi *pop2 = &mpi_op2; + + get_mpi(&mpi_gcd_res, gcd); + get_mpi(&mpi_op1, op1); + + if (op2 == op1) + pop2 = &mpi_op1; + else + get_mpi(&mpi_op2, op2); + + if (!u && !v) { + MPI_CHECK(mbedtls_mpi_gcd(&mpi_gcd_res, &mpi_op1, pop2)); + } else { + mbedtls_mpi mpi_u; + mbedtls_mpi mpi_v; + int8_t s1 = mpi_op1.s; + int8_t s2 = pop2->s; + int cmp; + + mpi_op1.s = 1; + pop2->s = 1; + + get_mpi(&mpi_u, u); + get_mpi(&mpi_v, v); + + cmp = mbedtls_mpi_cmp_abs(&mpi_op1, pop2); + if (cmp == 0) { + MPI_CHECK(mbedtls_mpi_copy(&mpi_gcd_res, &mpi_op1)); + MPI_CHECK(mbedtls_mpi_lset(&mpi_u, 1)); + MPI_CHECK(mbedtls_mpi_lset(&mpi_v, 0)); + } else if (cmp > 0) { + mpi_egcd(&mpi_gcd_res, &mpi_u, &mpi_v, &mpi_op1, pop2); + } else { + mpi_egcd(&mpi_gcd_res, &mpi_v, &mpi_u, pop2, &mpi_op1); + } + + mpi_u.s *= s1; + mpi_v.s *= s2; + + MPI_CHECK(copy_mpi_to_bigint(&mpi_u, u)); + MPI_CHECK(copy_mpi_to_bigint(&mpi_v, v)); + mbedtls_mpi_free(&mpi_u); + mbedtls_mpi_free(&mpi_v); + } + + MPI_CHECK(copy_mpi_to_bigint(&mpi_gcd_res, gcd)); + mbedtls_mpi_free(&mpi_gcd_res); + mbedtls_mpi_free(&mpi_op1); + if (pop2 == &mpi_op2) + mbedtls_mpi_free(&mpi_op2); +} + +static int rng_read(void *ignored __unused, unsigned char *buf, size_t blen) +{ + if (_utee_cryp_random_number_generate(buf, blen)) + return MBEDTLS_ERR_MPI_FILE_IO_ERROR; + return 0; +} + +int32_t TEE_BigIntIsProbablePrime(const TEE_BigInt *op, + uint32_t confidenceLevel) +{ + int rc; + mbedtls_mpi mpi_op; + + get_mpi(&mpi_op, op); + + rc = mbedtls_mpi_is_prime_ext(&mpi_op, MAX(confidenceLevel, 80U), + rng_read, NULL); + + mbedtls_mpi_free(&mpi_op); + + if (rc) + return 0; + + return 1; +} + +/* + * Not so fast FMM implementation based on the normal big int functions. + * + * Note that these functions (along with all the other functions in this + * file) only are used directly by the TA doing bigint arithmetics on its + * own. Performance of RSA operations in TEE Internal API are not affected + * by this. + */ +void TEE_BigIntInitFMM(TEE_BigIntFMM *bigIntFMM, size_t len) +{ + TEE_BigIntInit(bigIntFMM, len); +} + +void __GP11_TEE_BigIntInitFMM(TEE_BigIntFMM *bigIntFMM, uint32_t len) +{ + TEE_BigIntInitFMM(bigIntFMM, len); +} + +void TEE_BigIntInitFMMContext(TEE_BigIntFMMContext *context __unused, + size_t len __unused, + const TEE_BigInt *modulus __unused) +{ +} + +void __GP11_TEE_BigIntInitFMMContext(TEE_BigIntFMMContext *context, + uint32_t len, const TEE_BigInt *modulus) +{ + TEE_BigIntInitFMMContext(context, len, modulus); +} + +TEE_Result TEE_BigIntInitFMMContext1(TEE_BigIntFMMContext *context __unused, + size_t len __unused, + const TEE_BigInt *modulus __unused) +{ + return TEE_SUCCESS; +} + +size_t TEE_BigIntFMMSizeInU32(size_t modulusSizeInBits) +{ + return TEE_BigIntSizeInU32(modulusSizeInBits); +} + +uint32_t __GP11_TEE_BigIntFMMSizeInU32(uint32_t modulusSizeInBits) +{ + return TEE_BigIntFMMSizeInU32(modulusSizeInBits); +} + +size_t TEE_BigIntFMMContextSizeInU32(size_t modulusSizeInBits __unused) +{ + /* Return something larger than 0 to keep malloc() and friends happy */ + return 1; +} + +uint32_t __GP11_TEE_BigIntFMMContextSizeInU32(uint32_t modulusSizeInBits) +{ + return TEE_BigIntFMMContextSizeInU32(modulusSizeInBits); +} + +void TEE_BigIntConvertToFMM(TEE_BigIntFMM *dest, const TEE_BigInt *src, + const TEE_BigInt *n, + const TEE_BigIntFMMContext *context __unused) +{ + TEE_BigIntMod(dest, src, n); +} + +void TEE_BigIntConvertFromFMM(TEE_BigInt *dest, const TEE_BigIntFMM *src, + const TEE_BigInt *n __unused, + const TEE_BigIntFMMContext *context __unused) +{ + mbedtls_mpi mpi_dst; + mbedtls_mpi mpi_src; + + get_mpi(&mpi_dst, dest); + get_mpi(&mpi_src, src); + + MPI_CHECK(mbedtls_mpi_copy(&mpi_dst, &mpi_src)); + + MPI_CHECK(copy_mpi_to_bigint(&mpi_dst, dest)); + mbedtls_mpi_free(&mpi_dst); + mbedtls_mpi_free(&mpi_src); +} + +void TEE_BigIntComputeFMM(TEE_BigIntFMM *dest, const TEE_BigIntFMM *op1, + const TEE_BigIntFMM *op2, const TEE_BigInt *n, + const TEE_BigIntFMMContext *context __unused) +{ + mbedtls_mpi mpi_dst; + mbedtls_mpi mpi_op1; + mbedtls_mpi mpi_op2; + mbedtls_mpi mpi_n; + mbedtls_mpi mpi_t; + + get_mpi(&mpi_dst, dest); + get_mpi(&mpi_op1, op1); + get_mpi(&mpi_op2, op2); + get_mpi(&mpi_n, n); + get_mpi(&mpi_t, NULL); + + MPI_CHECK(mbedtls_mpi_mul_mpi(&mpi_t, &mpi_op1, &mpi_op2)); + MPI_CHECK(mbedtls_mpi_mod_mpi(&mpi_dst, &mpi_t, &mpi_n)); + + mbedtls_mpi_free(&mpi_t); + mbedtls_mpi_free(&mpi_n); + mbedtls_mpi_free(&mpi_op2); + mbedtls_mpi_free(&mpi_op1); + MPI_CHECK(copy_mpi_to_bigint(&mpi_dst, dest)); + mbedtls_mpi_free(&mpi_dst); +} diff --git a/optee_os/lib/libutee/tee_api_objects.c b/optee_os/lib/libutee/tee_api_objects.c new file mode 100644 index 0000000..2514bde --- /dev/null +++ b/optee_os/lib/libutee/tee_api_objects.c @@ -0,0 +1,1054 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include + +#include +#include +#include "tee_api_private.h" + +#define TEE_USAGE_DEFAULT 0xffffffff + +void __utee_from_attr(struct utee_attribute *ua, const TEE_Attribute *attrs, + uint32_t attr_count) +{ + size_t n; + + for (n = 0; n < attr_count; n++) { + ua[n].attribute_id = attrs[n].attributeID; + if (attrs[n].attributeID & TEE_ATTR_FLAG_VALUE) { + ua[n].a = attrs[n].content.value.a; + ua[n].b = attrs[n].content.value.b; + } else { + ua[n].a = (uintptr_t)attrs[n].content.ref.buffer; + ua[n].b = attrs[n].content.ref.length; + } + } +} + +void __utee_from_gp11_attr(struct utee_attribute *ua, + const __GP11_TEE_Attribute *attrs, + uint32_t attr_count) +{ + size_t n = 0; + + for (n = 0; n < attr_count; n++) { + ua[n].attribute_id = attrs[n].attributeID; + if (attrs[n].attributeID & TEE_ATTR_FLAG_VALUE) { + ua[n].a = attrs[n].content.value.a; + ua[n].b = attrs[n].content.value.b; + } else { + ua[n].a = (uintptr_t)attrs[n].content.ref.buffer; + ua[n].b = attrs[n].content.ref.length; + } + } +} + +/* Data and Key Storage API - Generic Object Functions */ +/* + * Use of this function is deprecated + * new code SHOULD use the TEE_GetObjectInfo1 function instead + * These functions will be removed at some future major revision of + * this specification + */ +void TEE_GetObjectInfo(TEE_ObjectHandle object, TEE_ObjectInfo *objectInfo) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + + if (res != TEE_SUCCESS) + TEE_Panic(res); + + if (info.obj_type == TEE_TYPE_CORRUPTED_OBJECT) { + objectInfo->objectSize = 0; + objectInfo->maxObjectSize = 0; + objectInfo->objectUsage = 0; + objectInfo->dataSize = 0; + objectInfo->dataPosition = 0; + objectInfo->handleFlags = 0; + } else { + objectInfo->objectType = info.obj_type; + objectInfo->objectSize = info.obj_size; + objectInfo->maxObjectSize = info.max_obj_size; + objectInfo->objectUsage = info.obj_usage; + objectInfo->dataSize = info.data_size; + objectInfo->dataPosition = info.data_pos; + objectInfo->handleFlags = info.handle_flags; + } +} + +void __GP11_TEE_GetObjectInfo(TEE_ObjectHandle object, + __GP11_TEE_ObjectInfo *objectInfo) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + + if (res != TEE_SUCCESS) + TEE_Panic(res); + + if (info.obj_type == TEE_TYPE_CORRUPTED_OBJECT) { + objectInfo->keySize = 0; + objectInfo->maxKeySize = 0; + objectInfo->objectUsage = 0; + objectInfo->dataSize = 0; + objectInfo->dataPosition = 0; + objectInfo->handleFlags = 0; + } else { + objectInfo->objectType = info.obj_type; + objectInfo->keySize = info.obj_size; + objectInfo->maxKeySize = info.max_obj_size; + objectInfo->objectUsage = info.obj_usage; + objectInfo->dataSize = info.data_size; + objectInfo->dataPosition = info.data_pos; + objectInfo->handleFlags = info.handle_flags; + } +} + +TEE_Result TEE_GetObjectInfo1(TEE_ObjectHandle object, + TEE_ObjectInfo *objectInfo) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + + if (res != TEE_SUCCESS && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + objectInfo->objectType = info.obj_type; + objectInfo->objectSize = info.obj_size; + objectInfo->maxObjectSize = info.max_obj_size; + objectInfo->objectUsage = info.obj_usage; + objectInfo->dataSize = info.data_size; + objectInfo->dataPosition = info.data_pos; + objectInfo->handleFlags = info.handle_flags; + + return res; +} + +TEE_Result __GP11_TEE_GetObjectInfo1(TEE_ObjectHandle object, + __GP11_TEE_ObjectInfo *objectInfo) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + + if (res != TEE_SUCCESS && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + objectInfo->objectType = info.obj_type; + objectInfo->keySize = info.obj_size; + objectInfo->maxKeySize = info.max_obj_size; + objectInfo->objectUsage = info.obj_usage; + objectInfo->dataSize = info.data_size; + objectInfo->dataPosition = info.data_pos; + objectInfo->handleFlags = info.handle_flags; + + return res; +} + +/* + * Use of this function is deprecated + * new code SHOULD use the TEE_RestrictObjectUsage1 function instead + * These functions will be removed at some future major revision of + * this specification + */ +void TEE_RestrictObjectUsage(TEE_ObjectHandle object, uint32_t objectUsage) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (info.obj_type == TEE_TYPE_CORRUPTED_OBJECT) + return; + + res = TEE_RestrictObjectUsage1(object, objectUsage); + + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +TEE_Result TEE_RestrictObjectUsage1(TEE_ObjectHandle object, uint32_t objectUsage) +{ + TEE_Result res; + + res = _utee_cryp_obj_restrict_usage((unsigned long)object, + objectUsage); + + if (res != TEE_SUCCESS && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_GetObjectBufferAttribute(TEE_ObjectHandle object, + uint32_t attributeID, void *buffer, + size_t *size) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + uint64_t sz = 0; + + __utee_check_inout_annotation(size, sizeof(*size)); + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (res != TEE_SUCCESS) + goto exit; + + /* This function only supports reference attributes */ + if ((attributeID & TEE_ATTR_FLAG_VALUE)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + sz = *size; + res = _utee_cryp_obj_get_attr((unsigned long)object, attributeID, + buffer, &sz); + *size = sz; + +exit: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_SHORT_BUFFER && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_GetObjectBufferAttribute(TEE_ObjectHandle object, + uint32_t attributeID, + void *buffer, uint32_t *size) +{ + TEE_Result res = TEE_SUCCESS; + size_t l = 0; + + __utee_check_inout_annotation(size, sizeof(*size)); + l = *size; + res = TEE_GetObjectBufferAttribute(object, attributeID, buffer, &l); + *size = l; + return res; +} + +TEE_Result TEE_GetObjectValueAttribute(TEE_ObjectHandle object, + uint32_t attributeID, uint32_t *a, + uint32_t *b) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + uint32_t buf[2]; + uint64_t size = sizeof(buf); + + if (a) + __utee_check_out_annotation(a, sizeof(*a)); + if (b) + __utee_check_out_annotation(b, sizeof(*b)); + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (res != TEE_SUCCESS) + goto exit; + + /* This function only supports value attributes */ + if (!(attributeID & TEE_ATTR_FLAG_VALUE)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto exit; + } + + res = _utee_cryp_obj_get_attr((unsigned long)object, attributeID, buf, + &size); + +exit: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + if (size != sizeof(buf)) + TEE_Panic(0); + + if (res == TEE_SUCCESS) { + if (a) + *a = buf[0]; + if (b) + *b = buf[1]; + } + + return res; +} + +void TEE_CloseObject(TEE_ObjectHandle object) +{ + TEE_Result res; + + if (object == TEE_HANDLE_NULL) + return; + + res = _utee_cryp_obj_close((unsigned long)object); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +/* Data and Key Storage API - Transient Object Functions */ + +TEE_Result TEE_AllocateTransientObject(TEE_ObjectType objectType, + uint32_t maxObjectSize, + TEE_ObjectHandle *object) +{ + if (objectType == TEE_TYPE_DATA) + return TEE_ERROR_NOT_SUPPORTED; + + return __GP11_TEE_AllocateTransientObject(objectType, maxObjectSize, + object); +} + +TEE_Result __GP11_TEE_AllocateTransientObject(TEE_ObjectType objectType, + uint32_t maxKeySize, + TEE_ObjectHandle *object) +{ + TEE_Result res; + uint32_t obj; + + __utee_check_out_annotation(object, sizeof(*object)); + + res = _utee_cryp_obj_alloc(objectType, maxKeySize, &obj); + + if (res != TEE_SUCCESS && + res != TEE_ERROR_OUT_OF_MEMORY && + res != TEE_ERROR_NOT_SUPPORTED) + TEE_Panic(res); + + if (res == TEE_SUCCESS) + *object = (TEE_ObjectHandle)(uintptr_t)obj; + + return res; +} + +void TEE_FreeTransientObject(TEE_ObjectHandle object) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + if (object == TEE_HANDLE_NULL) + return; + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + if ((info.handle_flags & TEE_HANDLE_FLAG_PERSISTENT) != 0) + TEE_Panic(0); + + res = _utee_cryp_obj_close((unsigned long)object); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void TEE_ResetTransientObject(TEE_ObjectHandle object) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + if (object == TEE_HANDLE_NULL) + return; + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + if ((info.handle_flags & TEE_HANDLE_FLAG_PERSISTENT) != 0) + TEE_Panic(0); + + res = _utee_cryp_obj_reset((unsigned long)object); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +TEE_Result TEE_PopulateTransientObject(TEE_ObjectHandle object, + const TEE_Attribute *attrs, + uint32_t attrCount) +{ + struct utee_attribute ua[attrCount]; + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + __utee_check_attr_in_annotation(attrs, attrCount); + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + /* Must be a transient object */ + if ((info.handle_flags & TEE_HANDLE_FLAG_PERSISTENT) != 0) + TEE_Panic(0); + + /* Must not be initialized already */ + if ((info.handle_flags & TEE_HANDLE_FLAG_INITIALIZED) != 0) + TEE_Panic(0); + + __utee_from_attr(ua, attrs, attrCount); + res = _utee_cryp_obj_populate((unsigned long)object, ua, attrCount); + if (res != TEE_SUCCESS && res != TEE_ERROR_BAD_PARAMETERS) + TEE_Panic(res); + return res; +} + +TEE_Result __GP11_TEE_PopulateTransientObject(TEE_ObjectHandle object, + const __GP11_TEE_Attribute *attrs, + uint32_t attrCount) +{ + struct utee_attribute ua[attrCount]; + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + __utee_check_gp11_attr_in_annotation(attrs, attrCount); + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + /* Must be a transient object */ + if ((info.handle_flags & TEE_HANDLE_FLAG_PERSISTENT) != 0) + TEE_Panic(0); + + /* Must not be initialized already */ + if ((info.handle_flags & TEE_HANDLE_FLAG_INITIALIZED) != 0) + TEE_Panic(0); + + __utee_from_gp11_attr(ua, attrs, attrCount); + res = _utee_cryp_obj_populate((unsigned long)object, ua, attrCount); + if (res != TEE_SUCCESS && res != TEE_ERROR_BAD_PARAMETERS) + TEE_Panic(res); + return res; +} + +void TEE_InitRefAttribute(TEE_Attribute *attr, uint32_t attributeID, + const void *buffer, size_t length) +{ + __utee_check_out_annotation(attr, sizeof(*attr)); + + if ((attributeID & TEE_ATTR_FLAG_VALUE) != 0) + TEE_Panic(0); + attr->attributeID = attributeID; + attr->content.ref.buffer = (void *)buffer; + attr->content.ref.length = length; +} + +void __GP11_TEE_InitRefAttribute(__GP11_TEE_Attribute *attr, + uint32_t attributeID, + const void *buffer, uint32_t length) +{ + __utee_check_out_annotation(attr, sizeof(*attr)); + + if ((attributeID & TEE_ATTR_FLAG_VALUE) != 0) + TEE_Panic(0); + attr->attributeID = attributeID; + attr->content.ref.buffer = (void *)buffer; + attr->content.ref.length = length; +} + +void TEE_InitValueAttribute(TEE_Attribute *attr, uint32_t attributeID, + uint32_t a, uint32_t b) +{ + __utee_check_out_annotation(attr, sizeof(*attr)); + + if ((attributeID & TEE_ATTR_FLAG_VALUE) == 0) + TEE_Panic(0); + attr->attributeID = attributeID; + attr->content.value.a = a; + attr->content.value.b = b; +} + +void __GP11_TEE_InitValueAttribute(__GP11_TEE_Attribute *attr, + uint32_t attributeID, + uint32_t a, uint32_t b) +{ + __utee_check_out_annotation(attr, sizeof(*attr)); + + if ((attributeID & TEE_ATTR_FLAG_VALUE) == 0) + TEE_Panic(0); + attr->attributeID = attributeID; + attr->content.value.a = a; + attr->content.value.b = b; +} + +/* + * Use of this function is deprecated + * new code SHOULD use the TEE_CopyObjectAttributes1 function instead + * These functions will be removed at some future major revision of + * this specification + */ +void TEE_CopyObjectAttributes(TEE_ObjectHandle destObject, + TEE_ObjectHandle srcObject) +{ + struct utee_object_info src_info = { }; + TEE_Result res = TEE_SUCCESS; + + res = _utee_cryp_obj_get_info((unsigned long)srcObject, &src_info); + if (src_info.obj_type == TEE_TYPE_CORRUPTED_OBJECT) + return; + + res = TEE_CopyObjectAttributes1(destObject, srcObject); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +TEE_Result TEE_CopyObjectAttributes1(TEE_ObjectHandle destObject, + TEE_ObjectHandle srcObject) +{ + struct utee_object_info dst_info = { }; + struct utee_object_info src_info = { }; + TEE_Result res = TEE_SUCCESS; + + res = _utee_cryp_obj_get_info((unsigned long)destObject, &dst_info); + if (res != TEE_SUCCESS) + goto exit; + + res = _utee_cryp_obj_get_info((unsigned long)srcObject, &src_info); + if (res != TEE_SUCCESS) + goto exit; + + if (!(src_info.handle_flags & TEE_HANDLE_FLAG_INITIALIZED)) + TEE_Panic(0); + + if ((dst_info.handle_flags & TEE_HANDLE_FLAG_PERSISTENT)) + TEE_Panic(0); + + if ((dst_info.handle_flags & TEE_HANDLE_FLAG_INITIALIZED)) + TEE_Panic(0); + + res = _utee_cryp_obj_copy((unsigned long)destObject, + (unsigned long)srcObject); + +exit: + if (res != TEE_SUCCESS && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_GenerateKey(TEE_ObjectHandle object, uint32_t keySize, + const TEE_Attribute *params, uint32_t paramCount) +{ + TEE_Result res; + struct utee_attribute ua[paramCount]; + + __utee_check_attr_in_annotation(params, paramCount); + + __utee_from_attr(ua, params, paramCount); + res = _utee_cryp_obj_generate_key((unsigned long)object, keySize, + ua, paramCount); + + if (res != TEE_SUCCESS && res != TEE_ERROR_BAD_PARAMETERS) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_GenerateKey(TEE_ObjectHandle object, uint32_t keySize, + const __GP11_TEE_Attribute *params, + uint32_t paramCount) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + + __utee_check_gp11_attr_in_annotation(params, paramCount); + + __utee_from_gp11_attr(ua, params, paramCount); + res = _utee_cryp_obj_generate_key((unsigned long)object, keySize, + ua, paramCount); + + if (res != TEE_SUCCESS && res != TEE_ERROR_BAD_PARAMETERS) + TEE_Panic(res); + + return res; +} + +/* Data and Key Storage API - Persistent Object Functions */ + +TEE_Result TEE_OpenPersistentObject(uint32_t storageID, const void *objectID, + size_t objectIDLen, uint32_t flags, + TEE_ObjectHandle *object) +{ + TEE_Result res; + uint32_t obj; + + __utee_check_out_annotation(object, sizeof(*object)); + + res = _utee_storage_obj_open(storageID, objectID, objectIDLen, flags, + &obj); + if (res == TEE_SUCCESS) + *object = (TEE_ObjectHandle)(uintptr_t)obj; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_ACCESS_CONFLICT && + res != TEE_ERROR_OUT_OF_MEMORY && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + if (res != TEE_SUCCESS) + *object = TEE_HANDLE_NULL; + + return res; +} + +TEE_Result __GP11_TEE_OpenPersistentObject(uint32_t storageID, + const void *objectID, + uint32_t objectIDLen, uint32_t flags, + TEE_ObjectHandle *object) +{ + return TEE_OpenPersistentObject(storageID, objectID, objectIDLen, + flags, object); +} + +TEE_Result TEE_CreatePersistentObject(uint32_t storageID, const void *objectID, + size_t objectIDLen, uint32_t flags, + TEE_ObjectHandle attributes, + const void *initialData, + size_t initialDataLen, + TEE_ObjectHandle *object) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t *obj_ptr = NULL; + uint32_t obj = 0; + + if (object) { + __utee_check_out_annotation(object, sizeof(*object)); + obj_ptr = &obj; + } + + res = _utee_storage_obj_create(storageID, objectID, objectIDLen, flags, + (unsigned long)attributes, initialData, + initialDataLen, obj_ptr); + + if (res == TEE_SUCCESS && object) + *object = (TEE_ObjectHandle)(uintptr_t)obj; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_ACCESS_CONFLICT && + res != TEE_ERROR_OUT_OF_MEMORY && + res != TEE_ERROR_STORAGE_NO_SPACE && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + if (res != TEE_SUCCESS && object) + *object = TEE_HANDLE_NULL; + + return res; +} + +TEE_Result __GP11_TEE_CreatePersistentObject(uint32_t storageID, + const void *objectID, + uint32_t objectIDLen, + uint32_t flags, + TEE_ObjectHandle attributes, + const void *initialData, + uint32_t initialDataLen, + TEE_ObjectHandle *object) +{ + __utee_check_out_annotation(object, sizeof(*object)); + + return TEE_CreatePersistentObject(storageID, objectID, objectIDLen, + flags, attributes, initialData, + initialDataLen, object); +} + +/* + * Use of this function is deprecated + * new code SHOULD use the TEE_CloseAndDeletePersistentObject1 function instead + * These functions will be removed at some future major revision of + * this specification + */ +void TEE_CloseAndDeletePersistentObject(TEE_ObjectHandle object) +{ + TEE_Result res; + + if (object == TEE_HANDLE_NULL) + return; + + res = TEE_CloseAndDeletePersistentObject1(object); + + if (res != TEE_SUCCESS) + TEE_Panic(0); +} + +TEE_Result TEE_CloseAndDeletePersistentObject1(TEE_ObjectHandle object) +{ + TEE_Result res; + + if (object == TEE_HANDLE_NULL) + return TEE_SUCCESS; + + res = _utee_storage_obj_del((unsigned long)object); + + if (res != TEE_SUCCESS && res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + + +TEE_Result TEE_RenamePersistentObject(TEE_ObjectHandle object, + const void *newObjectID, + size_t newObjectIDLen) +{ + TEE_Result res; + + if (object == TEE_HANDLE_NULL) { + res = TEE_ERROR_ITEM_NOT_FOUND; + goto out; + } + + res = _utee_storage_obj_rename((unsigned long)object, newObjectID, + newObjectIDLen); + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ACCESS_CONFLICT && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_RenamePersistentObject(TEE_ObjectHandle object, + const void *newObjectID, + uint32_t newObjectIDLen) +{ + return TEE_RenamePersistentObject(object, newObjectID, newObjectIDLen); +} + +TEE_Result TEE_AllocatePersistentObjectEnumerator(TEE_ObjectEnumHandle * + objectEnumerator) +{ + TEE_Result res; + uint32_t oe; + + __utee_check_out_annotation(objectEnumerator, + sizeof(*objectEnumerator)); + + res = _utee_storage_alloc_enum(&oe); + + if (res != TEE_SUCCESS) + oe = TEE_HANDLE_NULL; + + *objectEnumerator = (TEE_ObjectEnumHandle)(uintptr_t)oe; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ACCESS_CONFLICT) + TEE_Panic(res); + + return res; +} + +void TEE_FreePersistentObjectEnumerator(TEE_ObjectEnumHandle objectEnumerator) +{ + TEE_Result res; + + if (objectEnumerator == TEE_HANDLE_NULL) + return; + + res = _utee_storage_free_enum((unsigned long)objectEnumerator); + + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void TEE_ResetPersistentObjectEnumerator(TEE_ObjectEnumHandle objectEnumerator) +{ + TEE_Result res; + + if (objectEnumerator == TEE_HANDLE_NULL) + return; + + res = _utee_storage_reset_enum((unsigned long)objectEnumerator); + + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +TEE_Result TEE_StartPersistentObjectEnumerator(TEE_ObjectEnumHandle + objectEnumerator, + uint32_t storageID) +{ + TEE_Result res; + + res = _utee_storage_start_enum((unsigned long)objectEnumerator, + storageID); + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_GetNextPersistentObject(TEE_ObjectEnumHandle objectEnumerator, + TEE_ObjectInfo *objectInfo, + void *objectID, size_t *objectIDLen) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + uint64_t len = 0; + + if (objectInfo) + __utee_check_out_annotation(objectInfo, sizeof(*objectInfo)); + __utee_check_out_annotation(objectIDLen, sizeof(*objectIDLen)); + + if (!objectID) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + len = *objectIDLen; + res = _utee_storage_next_enum((unsigned long)objectEnumerator, + &info, objectID, &len); + if (objectInfo) { + objectInfo->objectType = info.obj_type; + objectInfo->objectSize = info.obj_size; + objectInfo->maxObjectSize = info.max_obj_size; + objectInfo->objectUsage = info.obj_usage; + objectInfo->dataSize = info.data_size; + objectInfo->dataPosition = info.data_pos; + objectInfo->handleFlags = info.handle_flags; + } + *objectIDLen = len; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result +__GP11_TEE_GetNextPersistentObject(TEE_ObjectEnumHandle objectEnumerator, + __GP11_TEE_ObjectInfo *objectInfo, + void *objectID, uint32_t *objectIDLen) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + uint64_t len = 0; + + if (objectInfo) + __utee_check_out_annotation(objectInfo, sizeof(*objectInfo)); + __utee_check_out_annotation(objectIDLen, sizeof(*objectIDLen)); + + if (!objectID) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + len = *objectIDLen; + res = _utee_storage_next_enum((unsigned long)objectEnumerator, + &info, objectID, &len); + if (objectInfo) { + objectInfo->objectType = info.obj_type; + objectInfo->keySize = info.obj_size; + objectInfo->maxKeySize = info.max_obj_size; + objectInfo->objectUsage = info.obj_usage; + objectInfo->dataSize = info.data_size; + objectInfo->dataPosition = info.data_pos; + objectInfo->handleFlags = info.handle_flags; + } + *objectIDLen = len; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +/* Data and Key Storage API - Data Stream Access Functions */ + +TEE_Result TEE_ReadObjectData(TEE_ObjectHandle object, void *buffer, + size_t size, size_t *count) +{ + TEE_Result res; + uint64_t cnt64; + + if (object == TEE_HANDLE_NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_out_annotation(count, sizeof(*count)); + + cnt64 = *count; + res = _utee_storage_obj_read((unsigned long)object, buffer, size, + &cnt64); + *count = cnt64; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_ReadObjectData(TEE_ObjectHandle object, void *buffer, + uint32_t size, uint32_t *count) +{ + TEE_Result res = TEE_SUCCESS; + size_t cnt = 0; + + __utee_check_out_annotation(count, sizeof(*count)); + cnt = *count; + res = TEE_ReadObjectData(object, buffer, size, &cnt); + *count = cnt; + return res; +} + +TEE_Result TEE_WriteObjectData(TEE_ObjectHandle object, const void *buffer, + size_t size) +{ + TEE_Result res; + + if (object == TEE_HANDLE_NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (size > TEE_DATA_MAX_POSITION) { + res = TEE_ERROR_OVERFLOW; + goto out; + } + + res = _utee_storage_obj_write((unsigned long)object, buffer, size); + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_STORAGE_NO_SPACE && + res != TEE_ERROR_OVERFLOW && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_WriteObjectData(TEE_ObjectHandle object, + const void *buffer, uint32_t size) +{ + return TEE_WriteObjectData(object, buffer, size); +} + +TEE_Result TEE_TruncateObjectData(TEE_ObjectHandle object, size_t size) +{ + TEE_Result res; + + if (object == TEE_HANDLE_NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = _utee_storage_obj_trunc((unsigned long)object, size); + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_STORAGE_NO_SPACE && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_TruncateObjectData(TEE_ObjectHandle object, + uint32_t size) +{ + return TEE_TruncateObjectData(object, size); +} + +TEE_Result TEE_SeekObjectData(TEE_ObjectHandle object, intmax_t offset, + TEE_Whence whence) +{ + struct utee_object_info info = { }; + TEE_Result res = TEE_SUCCESS; + + if (object == TEE_HANDLE_NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = _utee_cryp_obj_get_info((unsigned long)object, &info); + if (res != TEE_SUCCESS) + goto out; + + switch (whence) { + case TEE_DATA_SEEK_SET: + if (offset > 0 && (uint32_t)offset > TEE_DATA_MAX_POSITION) { + res = TEE_ERROR_OVERFLOW; + goto out; + } + break; + case TEE_DATA_SEEK_CUR: + if (offset > 0 && + ((uint32_t)offset + info.data_pos > TEE_DATA_MAX_POSITION || + (uint32_t)offset + info.data_pos < info.data_pos)) { + res = TEE_ERROR_OVERFLOW; + goto out; + } + break; + case TEE_DATA_SEEK_END: + if (offset > 0 && + ((uint32_t)offset + info.data_size > + TEE_DATA_MAX_POSITION || + (uint32_t)offset + info.data_size < info.data_size)) { + res = TEE_ERROR_OVERFLOW; + goto out; + } + break; + default: + res = TEE_ERROR_ITEM_NOT_FOUND; + goto out; + } + + res = _utee_storage_obj_seek((unsigned long)object, offset, whence); + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_OVERFLOW && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_SeekObjectData(TEE_ObjectHandle object, int32_t offset, + TEE_Whence whence) +{ + return TEE_SeekObjectData(object, offset, whence); +} diff --git a/optee_os/lib/libutee/tee_api_operations.c b/optee_os/lib/libutee/tee_api_operations.c new file mode 100644 index 0000000..f44350d --- /dev/null +++ b/optee_os/lib/libutee/tee_api_operations.c @@ -0,0 +1,2712 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2021, SumUp Services GmbH + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tee_api_private.h" + +struct __TEE_OperationHandle { + TEE_OperationInfo info; + TEE_ObjectHandle key1; + TEE_ObjectHandle key2; + uint32_t operationState;/* Operation state : INITIAL or ACTIVE */ + + /* + * buffer to collect complete blocks or to keep a complete digest + * for TEE_DigestExtract(). + */ + uint8_t *buffer; + bool buffer_two_blocks; /* True if two blocks need to be buffered */ + size_t block_size; /* Block size of cipher */ + size_t buffer_offs; /* Offset in buffer */ + uint32_t state; /* Handle to state in TEE Core */ +}; + +/* Cryptographic Operations API - Generic Operation Functions */ + +TEE_Result TEE_AllocateOperation(TEE_OperationHandle *operation, + uint32_t algorithm, uint32_t mode, + uint32_t maxKeySize) +{ + TEE_Result res; + TEE_OperationHandle op = TEE_HANDLE_NULL; + uint32_t handle_state = 0; + size_t block_size = 1; + uint32_t req_key_usage; + bool with_private_key = false; + bool buffer_two_blocks = false; + + if (!operation) + TEE_Panic(0); + + if (algorithm == TEE_ALG_AES_XTS || algorithm == TEE_ALG_SM2_KEP || + algorithm == TEE_ALG_SM4_XTS) + handle_state = TEE_HANDLE_FLAG_EXPECT_TWO_KEYS; + + /* Check algorithm max key size */ + switch (algorithm) { + case TEE_ALG_DSA_SHA1: + if (maxKeySize < 512) + return TEE_ERROR_NOT_SUPPORTED; + if (maxKeySize > 1024) + return TEE_ERROR_NOT_SUPPORTED; + if (maxKeySize % 64 != 0) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_DSA_SHA224: + if (maxKeySize != 2048) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_DSA_SHA256: + if (maxKeySize != 2048 && maxKeySize != 3072) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_ECDSA_SHA1: + case __OPTEE_ALG_ECDSA_P192: + case __OPTEE_ALG_ECDH_P192: + if (maxKeySize != 192) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_ECDSA_SHA224: + case __OPTEE_ALG_ECDSA_P224: + case __OPTEE_ALG_ECDH_P224: + if (maxKeySize != 224) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_ECDSA_SHA256: + case __OPTEE_ALG_ECDSA_P256: + case __OPTEE_ALG_ECDH_P256: + case TEE_ALG_SM2_PKE: + case TEE_ALG_SM2_DSA_SM3: + if (maxKeySize != 256) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_SM2_KEP: + /* Two 256-bit keys */ + if (maxKeySize != 512) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_ECDSA_SHA384: + case __OPTEE_ALG_ECDSA_P384: + case __OPTEE_ALG_ECDH_P384: + if (maxKeySize != 384) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_ECDSA_SHA512: + case __OPTEE_ALG_ECDSA_P521: + case __OPTEE_ALG_ECDH_P521: + if (maxKeySize != 521) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_ECDH_DERIVE_SHARED_SECRET: + if (maxKeySize > 521) + return TEE_ERROR_NOT_SUPPORTED; + break; + + case TEE_ALG_ED25519: + case TEE_ALG_X25519: + if (maxKeySize != 256) + return TEE_ERROR_NOT_SUPPORTED; + break; + default: + break; + } + + /* Check algorithm mode (and maxKeySize for digests) */ + switch (algorithm) { + case TEE_ALG_AES_CTS: + case TEE_ALG_AES_XTS: + case TEE_ALG_SM4_XTS: + buffer_two_blocks = true; + fallthrough; + case TEE_ALG_AES_ECB_NOPAD: + case TEE_ALG_AES_CBC_NOPAD: + case TEE_ALG_AES_CCM: + case TEE_ALG_DES_ECB_NOPAD: + case TEE_ALG_DES_CBC_NOPAD: + case TEE_ALG_DES3_ECB_NOPAD: + case TEE_ALG_DES3_CBC_NOPAD: + case TEE_ALG_SM4_ECB_NOPAD: + case TEE_ALG_SM4_CBC_NOPAD: + case TEE_ALG_SM4_CTR: + if (TEE_ALG_GET_MAIN_ALG(algorithm) == TEE_MAIN_ALGO_AES) + block_size = TEE_AES_BLOCK_SIZE; + else if (TEE_ALG_GET_MAIN_ALG(algorithm) == TEE_MAIN_ALGO_SM4) + block_size = TEE_SM4_BLOCK_SIZE; + else + block_size = TEE_DES_BLOCK_SIZE; + fallthrough; + case TEE_ALG_AES_CTR: + case TEE_ALG_AES_GCM: + if (mode == TEE_MODE_ENCRYPT) + req_key_usage = TEE_USAGE_ENCRYPT; + else if (mode == TEE_MODE_DECRYPT) + req_key_usage = TEE_USAGE_DECRYPT; + else + return TEE_ERROR_NOT_SUPPORTED; + break; + +#if defined(CFG_CRYPTO_RSASSA_NA1) + case TEE_ALG_RSASSA_PKCS1_V1_5: +#endif + case TEE_ALG_RSASSA_PKCS1_V1_5_MD5: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA1: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA224: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA384: + case TEE_ALG_RSASSA_PKCS1_V1_5_SHA512: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + case TEE_ALG_DSA_SHA1: + case TEE_ALG_DSA_SHA224: + case TEE_ALG_DSA_SHA256: + case TEE_ALG_ECDSA_SHA1: + case TEE_ALG_ECDSA_SHA224: + case TEE_ALG_ECDSA_SHA256: + case TEE_ALG_ECDSA_SHA384: + case TEE_ALG_ECDSA_SHA512: + case __OPTEE_ALG_ECDSA_P192: + case __OPTEE_ALG_ECDSA_P224: + case __OPTEE_ALG_ECDSA_P256: + case __OPTEE_ALG_ECDSA_P384: + case __OPTEE_ALG_ECDSA_P521: + case TEE_ALG_SM2_DSA_SM3: + case TEE_ALG_ED25519: + if (mode == TEE_MODE_SIGN) { + with_private_key = true; + req_key_usage = TEE_USAGE_SIGN; + } else if (mode == TEE_MODE_VERIFY) { + req_key_usage = TEE_USAGE_VERIFY; + } else { + return TEE_ERROR_NOT_SUPPORTED; + } + break; + + case TEE_ALG_RSAES_PKCS1_V1_5: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_MD5: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384: + case TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512: + case TEE_ALG_SM2_PKE: + if (mode == TEE_MODE_ENCRYPT) { + req_key_usage = TEE_USAGE_ENCRYPT; + } else if (mode == TEE_MODE_DECRYPT) { + with_private_key = true; + req_key_usage = TEE_USAGE_DECRYPT; + } else { + return TEE_ERROR_NOT_SUPPORTED; + } + break; + + case TEE_ALG_RSA_NOPAD: + if (mode == TEE_MODE_ENCRYPT) { + req_key_usage = TEE_USAGE_ENCRYPT | TEE_USAGE_VERIFY; + } else if (mode == TEE_MODE_DECRYPT) { + with_private_key = true; + req_key_usage = TEE_USAGE_DECRYPT | TEE_USAGE_SIGN; + } else { + return TEE_ERROR_NOT_SUPPORTED; + } + break; + + case TEE_ALG_DH_DERIVE_SHARED_SECRET: + case TEE_ALG_ECDH_DERIVE_SHARED_SECRET: + case __OPTEE_ALG_ECDH_P192: + case __OPTEE_ALG_ECDH_P224: + case __OPTEE_ALG_ECDH_P256: + case __OPTEE_ALG_ECDH_P384: + case __OPTEE_ALG_ECDH_P521: + case TEE_ALG_HKDF_MD5_DERIVE_KEY: + case TEE_ALG_HKDF_SHA1_DERIVE_KEY: + case TEE_ALG_HKDF_SHA224_DERIVE_KEY: + case TEE_ALG_HKDF_SHA256_DERIVE_KEY: + case TEE_ALG_HKDF_SHA384_DERIVE_KEY: + case TEE_ALG_HKDF_SHA512_DERIVE_KEY: + case TEE_ALG_CONCAT_KDF_SHA1_DERIVE_KEY: + case TEE_ALG_CONCAT_KDF_SHA224_DERIVE_KEY: + case TEE_ALG_CONCAT_KDF_SHA256_DERIVE_KEY: + case TEE_ALG_CONCAT_KDF_SHA384_DERIVE_KEY: + case TEE_ALG_CONCAT_KDF_SHA512_DERIVE_KEY: + case TEE_ALG_PBKDF2_HMAC_SHA1_DERIVE_KEY: + case TEE_ALG_SM2_KEP: + case TEE_ALG_X25519: + if (mode != TEE_MODE_DERIVE) + return TEE_ERROR_NOT_SUPPORTED; + with_private_key = true; + req_key_usage = TEE_USAGE_DERIVE; + break; + + case TEE_ALG_MD5: + case TEE_ALG_SHA1: + case TEE_ALG_SHA224: + case TEE_ALG_SHA256: + case TEE_ALG_SHA384: + case TEE_ALG_SHA512: + case TEE_ALG_SHA3_224: + case TEE_ALG_SHA3_256: + case TEE_ALG_SHA3_384: + case TEE_ALG_SHA3_512: + case TEE_ALG_SHAKE128: + case TEE_ALG_SHAKE256: + case TEE_ALG_SM3: + if (mode != TEE_MODE_DIGEST) + return TEE_ERROR_NOT_SUPPORTED; + if (maxKeySize) + return TEE_ERROR_NOT_SUPPORTED; + /* v1.1: flags always set for digest operations */ + handle_state |= TEE_HANDLE_FLAG_KEY_SET; + req_key_usage = 0; + break; + + case TEE_ALG_DES_CBC_MAC_NOPAD: + case TEE_ALG_AES_CBC_MAC_NOPAD: + case TEE_ALG_AES_CBC_MAC_PKCS5: + case TEE_ALG_AES_CMAC: + case TEE_ALG_DES_CBC_MAC_PKCS5: + case TEE_ALG_DES3_CBC_MAC_NOPAD: + case TEE_ALG_DES3_CBC_MAC_PKCS5: + case TEE_ALG_DES3_CMAC: + case TEE_ALG_HMAC_MD5: + case TEE_ALG_HMAC_SHA1: + case TEE_ALG_HMAC_SHA224: + case TEE_ALG_HMAC_SHA256: + case TEE_ALG_HMAC_SHA384: + case TEE_ALG_HMAC_SHA512: + case TEE_ALG_HMAC_SHA3_224: + case TEE_ALG_HMAC_SHA3_256: + case TEE_ALG_HMAC_SHA3_384: + case TEE_ALG_HMAC_SHA3_512: + case TEE_ALG_HMAC_SM3: + if (mode != TEE_MODE_MAC) + return TEE_ERROR_NOT_SUPPORTED; + req_key_usage = TEE_USAGE_MAC; + break; + + default: + return TEE_ERROR_NOT_SUPPORTED; + } + + op = TEE_Malloc(sizeof(*op), TEE_MALLOC_FILL_ZERO); + if (!op) + return TEE_ERROR_OUT_OF_MEMORY; + + op->info.algorithm = algorithm; + op->info.operationClass = TEE_ALG_GET_CLASS(algorithm); +#ifdef CFG_CRYPTO_RSASSA_NA1 + if (algorithm == TEE_ALG_RSASSA_PKCS1_V1_5) + op->info.operationClass = TEE_OPERATION_ASYMMETRIC_SIGNATURE; +#endif + op->info.mode = mode; + op->info.digestLength = TEE_ALG_GET_DIGEST_SIZE(algorithm); + op->info.maxKeySize = maxKeySize; + op->info.requiredKeyUsage = req_key_usage; + op->info.handleState = handle_state; + + /* + * Needed to buffer the digest if TEE_DigestExtract() doesn't + * retrieve the entire digest in one go. + */ + if (op->info.operationClass == TEE_OPERATION_DIGEST) + block_size = op->info.digestLength; + + if (block_size > 1) { + size_t buffer_size = block_size; + + if (buffer_two_blocks) + buffer_size *= 2; + + op->buffer = TEE_Malloc(buffer_size, + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (op->buffer == NULL) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + } + op->block_size = block_size; + op->buffer_two_blocks = buffer_two_blocks; + + if (TEE_ALG_GET_CLASS(algorithm) != TEE_OPERATION_DIGEST) { + uint32_t mks = maxKeySize; + TEE_ObjectType key_type = TEE_ALG_GET_KEY_TYPE(algorithm, + with_private_key); + + /* + * If two keys are expected the max key size is the sum of + * the size of both keys. + */ + if (op->info.handleState & TEE_HANDLE_FLAG_EXPECT_TWO_KEYS) + mks /= 2; + + res = TEE_AllocateTransientObject(key_type, mks, &op->key1); + if (res != TEE_SUCCESS) + goto out; + + if (op->info.handleState & TEE_HANDLE_FLAG_EXPECT_TWO_KEYS) { + res = TEE_AllocateTransientObject(key_type, mks, + &op->key2); + if (res != TEE_SUCCESS) + goto out; + } + } + + res = _utee_cryp_state_alloc(algorithm, mode, (unsigned long)op->key1, + (unsigned long)op->key2, &op->state); + if (res != TEE_SUCCESS) + goto out; + + /* + * Initialize digest operations + * Other multi-stage operations initialized w/ TEE_xxxInit functions + * Non-applicable on asymmetric operations + */ + if (TEE_ALG_GET_CLASS(algorithm) == TEE_OPERATION_DIGEST) { + res = _utee_hash_init(op->state, NULL, 0); + if (res != TEE_SUCCESS) + goto out; + /* v1.1: flags always set for digest operations */ + op->info.handleState |= TEE_HANDLE_FLAG_INITIALIZED; + } + + op->operationState = TEE_OPERATION_STATE_INITIAL; + + *operation = op; + +out: + if (res != TEE_SUCCESS) { + if (res != TEE_ERROR_OUT_OF_MEMORY && + res != TEE_ERROR_NOT_SUPPORTED) + TEE_Panic(res); + if (op) { + if (op->state) { + TEE_FreeOperation(op); + } else { + TEE_Free(op->buffer); + TEE_FreeTransientObject(op->key1); + TEE_FreeTransientObject(op->key2); + TEE_Free(op); + } + } + } + + return res; +} + +void TEE_FreeOperation(TEE_OperationHandle operation) +{ + TEE_Result res; + + if (operation == TEE_HANDLE_NULL) + return; + + /* + * Note that keys should not be freed here, since they are + * claimed by the operation they will be freed by + * utee_cryp_state_free(). + */ + res = _utee_cryp_state_free(operation->state); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + TEE_Free(operation->buffer); + TEE_Free(operation); +} + +void __GP11_TEE_FreeOperation(TEE_OperationHandle operation) +{ + if (operation == TEE_HANDLE_NULL) + TEE_Panic(0); + TEE_FreeOperation(operation); +} + +void TEE_GetOperationInfo(TEE_OperationHandle operation, + TEE_OperationInfo *operationInfo) +{ + if (operation == TEE_HANDLE_NULL) + TEE_Panic(0); + + __utee_check_out_annotation(operationInfo, sizeof(*operationInfo)); + + *operationInfo = operation->info; + if (operationInfo->handleState & TEE_HANDLE_FLAG_EXPECT_TWO_KEYS) { + operationInfo->keySize = 0; + operationInfo->requiredKeyUsage = 0; + } +} + +TEE_Result TEE_GetOperationInfoMultiple(TEE_OperationHandle op, + TEE_OperationInfoMultiple *op_info, + size_t *size) +{ + TEE_Result res = TEE_SUCCESS; + TEE_ObjectInfo kinfo = { }; + size_t max_key_count = 0; + bool two_keys = false; + + if (op == TEE_HANDLE_NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + __utee_check_outbuf_annotation(op_info, size); + + if (*size < sizeof(*op_info)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + max_key_count = (*size - sizeof(*op_info)) / + sizeof(TEE_OperationInfoKey); + + TEE_MemFill(op_info, 0, *size); + + /* Two keys flag (TEE_ALG_AES_XTS only) */ + two_keys = op->info.handleState & TEE_HANDLE_FLAG_EXPECT_TWO_KEYS; + + if (op->info.mode == TEE_MODE_DIGEST) { + op_info->numberOfKeys = 0; + } else if (!two_keys) { + if (max_key_count < 1) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + res = TEE_GetObjectInfo1(op->key1, &kinfo); + /* Key1 is not a valid handle, "can't happen". */ + if (res) + goto out; + + op_info->keyInformation[0].keySize = kinfo.objectSize; + op_info->keyInformation[0].requiredKeyUsage = + op->info.requiredKeyUsage; + op_info->numberOfKeys = 1; + } else { + if (max_key_count < 2) { + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + res = TEE_GetObjectInfo1(op->key1, &kinfo); + /* Key1 is not a valid handle, "can't happen". */ + if (res) + goto out; + + op_info->keyInformation[0].keySize = kinfo.objectSize; + op_info->keyInformation[0].requiredKeyUsage = + op->info.requiredKeyUsage; + + res = TEE_GetObjectInfo1(op->key2, &kinfo); + /* Key2 is not a valid handle, "can't happen". */ + if (res) + goto out; + + op_info->keyInformation[1].keySize = kinfo.objectSize; + op_info->keyInformation[1].requiredKeyUsage = + op->info.requiredKeyUsage; + + op_info->numberOfKeys = 2; + } + + op_info->algorithm = op->info.algorithm; + op_info->operationClass = op->info.operationClass; + op_info->mode = op->info.mode; + op_info->digestLength = op->info.digestLength; + op_info->maxKeySize = op->info.maxKeySize; + op_info->handleState = op->info.handleState; + op_info->operationState = op->operationState; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result +__GP11_TEE_GetOperationInfoMultiple(TEE_OperationHandle operation, + TEE_OperationInfoMultiple *info, + uint32_t *operationSize) +{ + TEE_Result res = TEE_SUCCESS; + size_t s = 0; + + __utee_check_gp11_outbuf_annotation(info, operationSize); + s = *operationSize; + res = TEE_GetOperationInfoMultiple(operation, info, &s); + *operationSize = s; + return res; +} + +static void reset_operation_state(TEE_OperationHandle op) +{ + op->operationState = TEE_OPERATION_STATE_INITIAL; + + if (op->info.operationClass == TEE_OPERATION_DIGEST) { + TEE_Result res = _utee_hash_init(op->state, NULL, 0); + + if (res != TEE_SUCCESS) + TEE_Panic(res); + op->info.handleState |= TEE_HANDLE_FLAG_INITIALIZED; + } else { + op->info.handleState &= ~TEE_HANDLE_FLAG_INITIALIZED; + } +} + +void TEE_ResetOperation(TEE_OperationHandle operation) +{ + if (operation == TEE_HANDLE_NULL) + TEE_Panic(0); + + if (!(operation->info.handleState & TEE_HANDLE_FLAG_KEY_SET)) + TEE_Panic(0); + + reset_operation_state(operation); +} + +TEE_Result TEE_SetOperationKey(TEE_OperationHandle operation, + TEE_ObjectHandle key) +{ + TEE_Result res; + uint32_t key_size = 0; + TEE_ObjectInfo key_info; + + if (operation == TEE_HANDLE_NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (key == TEE_HANDLE_NULL) { + /* Operation key cleared */ + TEE_ResetTransientObject(operation->key1); + operation->info.handleState &= ~TEE_HANDLE_FLAG_KEY_SET; + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) + reset_operation_state(operation); + return TEE_SUCCESS; + } + + /* No key for digest operation */ + if (operation->info.operationClass == TEE_OPERATION_DIGEST) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* Two keys flag not expected (TEE_ALG_AES_XTS excluded) */ + if ((operation->info.handleState & TEE_HANDLE_FLAG_EXPECT_TWO_KEYS) != + 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = TEE_GetObjectInfo1(key, &key_info); + /* Key is not a valid handle */ + if (res != TEE_SUCCESS) + goto out; + + /* Supplied key has to meet required usage */ + if ((key_info.objectUsage & operation->info.requiredKeyUsage) != + operation->info.requiredKeyUsage) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (operation->info.maxKeySize < key_info.objectSize) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + key_size = key_info.objectSize; + + TEE_ResetTransientObject(operation->key1); + operation->info.handleState &= ~TEE_HANDLE_FLAG_KEY_SET; + + res = TEE_CopyObjectAttributes1(operation->key1, key); + if (res != TEE_SUCCESS) + goto out; + + operation->info.handleState |= TEE_HANDLE_FLAG_KEY_SET; + + operation->info.keySize = key_size; + + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) + reset_operation_state(operation); + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_SetOperationKey(TEE_OperationHandle operation, + TEE_ObjectHandle key) +{ + if (operation == TEE_HANDLE_NULL || + operation->operationState != TEE_OPERATION_STATE_INITIAL) + TEE_Panic(0); + + return TEE_SetOperationKey(operation, key); +} + +static TEE_Result set_operation_key2(TEE_OperationHandle operation, + TEE_ObjectHandle key1, + TEE_ObjectHandle key2) +{ + TEE_Result res; + uint32_t key_size = 0; + TEE_ObjectInfo key_info1; + TEE_ObjectInfo key_info2; + + if (operation == TEE_HANDLE_NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * Key1/Key2 and/or are not initialized and + * Either both keys are NULL or both are not NULL + */ + if (!key1 && !key2) { + /* Clear the keys */ + TEE_ResetTransientObject(operation->key1); + TEE_ResetTransientObject(operation->key2); + operation->info.handleState &= ~TEE_HANDLE_FLAG_KEY_SET; + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) + reset_operation_state(operation); + return TEE_SUCCESS; + } else if (!key1 || !key2) { + /* Both keys are obviously not valid. */ + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* No key for digest operation */ + if (operation->info.operationClass == TEE_OPERATION_DIGEST) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* Two keys flag expected (TEE_ALG_AES_XTS and TEE_ALG_SM2_KEP only) */ + if ((operation->info.handleState & TEE_HANDLE_FLAG_EXPECT_TWO_KEYS) == + 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = TEE_GetObjectInfo1(key1, &key_info1); + /* Key1 is not a valid handle */ + if (res != TEE_SUCCESS) + goto out; + + /* Supplied key has to meet required usage */ + if ((key_info1.objectUsage & operation->info. + requiredKeyUsage) != operation->info.requiredKeyUsage) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = TEE_GetObjectInfo1(key2, &key_info2); + /* Key2 is not a valid handle */ + if (res != TEE_SUCCESS) { + if (res == TEE_ERROR_CORRUPT_OBJECT) + res = TEE_ERROR_CORRUPT_OBJECT_2; + goto out; + } + + /* Supplied key has to meet required usage */ + if ((key_info2.objectUsage & operation->info. + requiredKeyUsage) != operation->info.requiredKeyUsage) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * All the multi key algorithm currently supported requires the keys to + * be of equal size. + */ + if (key_info1.objectSize != key_info2.objectSize) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + + } + + if (operation->info.maxKeySize < key_info1.objectSize) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * Odd that only the size of one key should be reported while + * size of two key are used when allocating the operation. + */ + key_size = key_info1.objectSize; + + TEE_ResetTransientObject(operation->key1); + TEE_ResetTransientObject(operation->key2); + operation->info.handleState &= ~TEE_HANDLE_FLAG_KEY_SET; + + res = TEE_CopyObjectAttributes1(operation->key1, key1); + if (res != TEE_SUCCESS) + goto out; + res = TEE_CopyObjectAttributes1(operation->key2, key2); + if (res != TEE_SUCCESS) { + if (res == TEE_ERROR_CORRUPT_OBJECT) + res = TEE_ERROR_CORRUPT_OBJECT_2; + goto out; + } + + operation->info.handleState |= TEE_HANDLE_FLAG_KEY_SET; + + operation->info.keySize = key_size; + + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) + reset_operation_state(operation); +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_CORRUPT_OBJECT && + res != TEE_ERROR_CORRUPT_OBJECT_2 && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE && + res != TEE_ERROR_STORAGE_NOT_AVAILABLE_2) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_SetOperationKey2(TEE_OperationHandle operation, + TEE_ObjectHandle key1, TEE_ObjectHandle key2) +{ + if (operation != TEE_HANDLE_NULL && key1 && key1 == key2) + return TEE_ERROR_SECURITY; + + return set_operation_key2(operation, key1, key2); +} + +TEE_Result __GP11_TEE_SetOperationKey2(TEE_OperationHandle operation, + TEE_ObjectHandle key1, + TEE_ObjectHandle key2) +{ + if (operation == TEE_HANDLE_NULL || + operation->operationState != TEE_OPERATION_STATE_INITIAL) + TEE_Panic(0); + + return set_operation_key2(operation, key1, key2); +} + +void TEE_CopyOperation(TEE_OperationHandle dst_op, TEE_OperationHandle src_op) +{ + TEE_Result res; + + if (dst_op == TEE_HANDLE_NULL || src_op == TEE_HANDLE_NULL) + TEE_Panic(0); + if (dst_op->info.algorithm != src_op->info.algorithm) + TEE_Panic(0); + if (dst_op->info.mode != src_op->info.mode) + TEE_Panic(0); + if (src_op->info.operationClass != TEE_OPERATION_DIGEST) { + TEE_ObjectHandle key1 = TEE_HANDLE_NULL; + TEE_ObjectHandle key2 = TEE_HANDLE_NULL; + + if (src_op->info.handleState & TEE_HANDLE_FLAG_KEY_SET) { + key1 = src_op->key1; + key2 = src_op->key2; + } + + if ((src_op->info.handleState & + TEE_HANDLE_FLAG_EXPECT_TWO_KEYS) == 0) { + TEE_SetOperationKey(dst_op, key1); + } else { + TEE_SetOperationKey2(dst_op, key1, key2); + } + } + dst_op->info.handleState = src_op->info.handleState; + dst_op->info.keySize = src_op->info.keySize; + dst_op->info.digestLength = src_op->info.digestLength; + dst_op->operationState = src_op->operationState; + + if (dst_op->buffer_two_blocks != src_op->buffer_two_blocks || + dst_op->block_size != src_op->block_size) + TEE_Panic(0); + + if (dst_op->buffer != NULL) { + size_t sz = src_op->block_size; + + if (src_op->buffer == NULL) + TEE_Panic(0); + + if (src_op->buffer_two_blocks) + sz *= 2; + memcpy(dst_op->buffer, src_op->buffer, sz); + dst_op->buffer_offs = src_op->buffer_offs; + } else if (src_op->buffer != NULL) { + TEE_Panic(0); + } + + res = _utee_cryp_state_copy(dst_op->state, src_op->state); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +/* Cryptographic Operations API - Message Digest Functions */ + +static void init_hash_operation(TEE_OperationHandle operation, const void *IV, + uint32_t IVLen) +{ + TEE_Result res; + + /* + * Note : IV and IVLen are never used in current implementation + * This is why coherent values of IV and IVLen are not checked + */ + res = _utee_hash_init(operation->state, IV, IVLen); + if (res != TEE_SUCCESS) + TEE_Panic(res); + operation->buffer_offs = 0; + operation->info.handleState |= TEE_HANDLE_FLAG_INITIALIZED; +} + +void TEE_DigestUpdate(TEE_OperationHandle operation, + const void *chunk, size_t chunkSize) +{ + TEE_Result res = TEE_ERROR_GENERIC; + + if (operation == TEE_HANDLE_NULL || + operation->info.operationClass != TEE_OPERATION_DIGEST) + TEE_Panic(0); + + operation->operationState = TEE_OPERATION_STATE_ACTIVE; + + res = _utee_hash_update(operation->state, chunk, chunkSize); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void __GP11_TEE_DigestUpdate(TEE_OperationHandle operation, + const void *chunk, uint32_t chunkSize) +{ + return TEE_DigestUpdate(operation, chunk, chunkSize); +} + +TEE_Result TEE_DigestDoFinal(TEE_OperationHandle operation, const void *chunk, + size_t chunkLen, void *hash, size_t *hashLen) +{ + TEE_Result res = TEE_SUCCESS; + uint64_t hl = 0; + size_t len = 0; + + if ((operation == TEE_HANDLE_NULL) || + (!chunk && chunkLen) || + (operation->info.operationClass != TEE_OPERATION_DIGEST)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + if (operation->operationState == TEE_OPERATION_STATE_EXTRACTING && + chunkLen) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_inout_annotation(hashLen, sizeof(*hashLen)); + + if (operation->operationState == TEE_OPERATION_STATE_EXTRACTING && + operation->buffer) { + /* + * This is not an Extendable-Output Function and we have + * already started extracting + */ + len = MIN(operation->block_size - operation->buffer_offs, + *hashLen); + memcpy(hash, operation->buffer + operation->buffer_offs, len); + *hashLen = len; + } else { + hl = *hashLen; + res = _utee_hash_final(operation->state, chunk, chunkLen, hash, + &hl); + *hashLen = hl; + if (res) + goto out; + } + + /* Reset operation state */ + init_hash_operation(operation, NULL, 0); + + operation->operationState = TEE_OPERATION_STATE_INITIAL; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_DigestDoFinal(TEE_OperationHandle operation, + const void *chunk, uint32_t chunkLen, + void *hash, uint32_t *hashLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t l = 0; + + __utee_check_inout_annotation(hashLen, sizeof(*hashLen)); + l = *hashLen; + res = TEE_DigestDoFinal(operation, chunk, chunkLen, hash, &l); + *hashLen = l; + return res; +} + +TEE_Result TEE_DigestExtract(TEE_OperationHandle operation, void *hash, + size_t *hashLen) +{ + TEE_Result res = TEE_SUCCESS; + uint64_t hl = 0; + size_t len = 0; + + if (operation == TEE_HANDLE_NULL || + operation->info.operationClass != TEE_OPERATION_DIGEST) + TEE_Panic(0); + __utee_check_inout_annotation(hashLen, sizeof(*hashLen)); + + if (!operation->buffer) { + /* This is an Extendable-Output Function */ + operation->info.handleState |= TEE_HANDLE_FLAG_EXTRACTING; + operation->operationState = TEE_OPERATION_STATE_EXTRACTING; + hl = *hashLen; + res = _utee_hash_final(operation->state, NULL, 0, hash, &hl); + if (res) + TEE_Panic(0); + *hashLen = hl; + + return TEE_SUCCESS; + } + + if (operation->operationState != TEE_OPERATION_STATE_EXTRACTING) { + hl = operation->block_size; + res = _utee_hash_final(operation->state, NULL, 0, + operation->buffer, &hl); + if (res) + TEE_Panic(0); + if (hl != operation->block_size) + TEE_Panic(0); + assert(!operation->buffer_offs); + operation->info.handleState |= TEE_HANDLE_FLAG_EXTRACTING; + operation->operationState = TEE_OPERATION_STATE_EXTRACTING; + } + + len = MIN(operation->block_size - operation->buffer_offs, *hashLen); + memcpy(hash, operation->buffer + operation->buffer_offs, len); + *hashLen = len; + operation->buffer_offs += len; + + return TEE_SUCCESS; +} + +/* Cryptographic Operations API - Symmetric Cipher Functions */ + +void TEE_CipherInit(TEE_OperationHandle operation, const void *IV, + size_t IVLen) +{ + TEE_Result res; + + if (operation == TEE_HANDLE_NULL) + TEE_Panic(0); + + if (operation->info.operationClass != TEE_OPERATION_CIPHER) + TEE_Panic(0); + + if (!(operation->info.handleState & TEE_HANDLE_FLAG_KEY_SET) || + !(operation->key1)) + TEE_Panic(0); + + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) + TEE_ResetOperation(operation); + + if (IV && IVLen) { + if (operation->info.algorithm == TEE_ALG_AES_ECB_NOPAD || + operation->info.algorithm == TEE_ALG_DES_ECB_NOPAD || + operation->info.algorithm == TEE_ALG_DES3_ECB_NOPAD || + operation->info.algorithm == TEE_ALG_SM4_ECB_NOPAD) + TEE_Panic(0); + } + + operation->operationState = TEE_OPERATION_STATE_ACTIVE; + + res = _utee_cipher_init(operation->state, IV, IVLen); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + operation->buffer_offs = 0; + operation->info.handleState |= TEE_HANDLE_FLAG_INITIALIZED; +} + +void __GP11_TEE_CipherInit(TEE_OperationHandle operation, const void *IV, + uint32_t IVLen) +{ + return TEE_CipherInit(operation, IV, IVLen); +} + +static TEE_Result tee_buffer_update( + TEE_OperationHandle op, + TEE_Result(*update_func)(unsigned long state, const void *src, + size_t slen, void *dst, uint64_t *dlen), + const void *src_data, size_t src_len, + void *dest_data, uint64_t *dest_len) +{ + TEE_Result res; + const uint8_t *src = src_data; + size_t slen = src_len; + uint8_t *dst = dest_data; + size_t dlen = *dest_len; + size_t acc_dlen = 0; + uint64_t tmp_dlen; + size_t l; + size_t buffer_size; + size_t buffer_left; + + if (!src) { + if (slen) + TEE_Panic(0); + goto out; + } + + if (op->buffer_two_blocks) { + buffer_size = op->block_size * 2; + buffer_left = 1; + } else { + buffer_size = op->block_size; + buffer_left = 0; + } + + if (op->buffer_offs > 0) { + /* Fill up complete block */ + if (op->buffer_offs < op->block_size) + l = MIN(slen, op->block_size - op->buffer_offs); + else + l = MIN(slen, buffer_size - op->buffer_offs); + memcpy(op->buffer + op->buffer_offs, src, l); + op->buffer_offs += l; + src += l; + slen -= l; + if ((op->buffer_offs % op->block_size) != 0) + goto out; /* Nothing left to do */ + } + + /* If we can feed from buffer */ + if ((op->buffer_offs > 0) && + ((op->buffer_offs + slen) >= (buffer_size + buffer_left))) { + l = ROUNDUP(op->buffer_offs + slen - buffer_size, + op->block_size); + l = MIN(op->buffer_offs, l); + tmp_dlen = dlen; + res = update_func(op->state, op->buffer, l, dst, &tmp_dlen); + if (res != TEE_SUCCESS) + TEE_Panic(res); + dst += tmp_dlen; + dlen -= tmp_dlen; + acc_dlen += tmp_dlen; + op->buffer_offs -= l; + if (op->buffer_offs > 0) { + /* + * Slen is small enough to be contained in rest buffer. + */ + memcpy(op->buffer, op->buffer + l, buffer_size - l); + memcpy(op->buffer + op->buffer_offs, src, slen); + op->buffer_offs += slen; + goto out; /* Nothing left to do */ + } + } + + if (slen >= (buffer_size + buffer_left)) { + /* Buffer is empty, feed as much as possible from src */ + if (op->info.algorithm == TEE_ALG_AES_CTS) + l = ROUNDUP(slen - buffer_size, op->block_size); + else + l = ROUNDUP(slen - buffer_size + 1, op->block_size); + + tmp_dlen = dlen; + res = update_func(op->state, src, l, dst, &tmp_dlen); + if (res != TEE_SUCCESS) + TEE_Panic(res); + src += l; + slen -= l; + dst += tmp_dlen; + dlen -= tmp_dlen; + acc_dlen += tmp_dlen; + } + + /* Slen is small enough to be contained in buffer. */ + memcpy(op->buffer + op->buffer_offs, src, slen); + op->buffer_offs += slen; + +out: + *dest_len = acc_dlen; + return TEE_SUCCESS; +} + +TEE_Result TEE_CipherUpdate(TEE_OperationHandle operation, const void *srcData, + size_t srcLen, void *destData, size_t *destLen) +{ + TEE_Result res; + size_t req_dlen; + uint64_t dl; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + + if (operation->info.operationClass != TEE_OPERATION_CIPHER) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (operation->operationState != TEE_OPERATION_STATE_ACTIVE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (!srcData && !srcLen) { + *destLen = 0; + res = TEE_SUCCESS; + goto out; + } + + /* Calculate required dlen */ + if (operation->block_size > 1) { + req_dlen = ((operation->buffer_offs + srcLen) / + operation->block_size) * operation->block_size; + } else { + req_dlen = srcLen; + } + if (operation->buffer_two_blocks) { + if (req_dlen > operation->block_size * 2) + req_dlen -= operation->block_size * 2; + else + req_dlen = 0; + } + /* + * Check that required destLen is big enough before starting to feed + * data to the algorithm. Errors during feeding of data are fatal as we + * can't restore sync with this API. + */ + if (*destLen < req_dlen) { + *destLen = req_dlen; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + dl = *destLen; + if (operation->block_size > 1) { + res = tee_buffer_update(operation, _utee_cipher_update, srcData, + srcLen, destData, &dl); + } else { + if (srcLen > 0) { + res = _utee_cipher_update(operation->state, srcData, + srcLen, destData, &dl); + } else { + res = TEE_SUCCESS; + dl = 0; + } + } + *destLen = dl; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_CipherUpdate(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t dl = 0; + + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + dl = *destLen; + res = TEE_CipherUpdate(operation, srcData, srcLen, destData, &dl); + *destLen = dl; + return res; +} + +TEE_Result TEE_CipherDoFinal(TEE_OperationHandle operation, + const void *srcData, size_t srcLen, + void *destData, size_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *dst = destData; + size_t acc_dlen = 0; + uint64_t tmp_dlen = 0; + size_t req_dlen = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + if (destLen) + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + + if (operation->info.operationClass != TEE_OPERATION_CIPHER) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (operation->operationState != TEE_OPERATION_STATE_ACTIVE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * Check that the final block doesn't require padding for those + * algorithms that requires client to supply padding. + */ + if (operation->info.algorithm == TEE_ALG_AES_ECB_NOPAD || + operation->info.algorithm == TEE_ALG_AES_CBC_NOPAD || + operation->info.algorithm == TEE_ALG_DES_ECB_NOPAD || + operation->info.algorithm == TEE_ALG_DES_CBC_NOPAD || + operation->info.algorithm == TEE_ALG_DES3_ECB_NOPAD || + operation->info.algorithm == TEE_ALG_DES3_CBC_NOPAD || + operation->info.algorithm == TEE_ALG_SM4_ECB_NOPAD || + operation->info.algorithm == TEE_ALG_SM4_CBC_NOPAD) { + if (((operation->buffer_offs + srcLen) % operation->block_size) + != 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + } + + /* + * Check that required destLen is big enough before starting to feed + * data to the algorithm. Errors during feeding of data are fatal as we + * can't restore sync with this API. + */ + if (operation->block_size > 1) { + req_dlen = operation->buffer_offs + srcLen; + } else { + req_dlen = srcLen; + } + if (destLen) + tmp_dlen = *destLen; + if (tmp_dlen < req_dlen) { + if (destLen) + *destLen = req_dlen; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + if (operation->block_size > 1) { + if (srcLen) { + res = tee_buffer_update(operation, _utee_cipher_update, + srcData, srcLen, dst, + &tmp_dlen); + if (res != TEE_SUCCESS) + goto out; + + dst += tmp_dlen; + acc_dlen += tmp_dlen; + + tmp_dlen = *destLen - acc_dlen; + } + res = _utee_cipher_final(operation->state, operation->buffer, + operation->buffer_offs, dst, + &tmp_dlen); + } else { + res = _utee_cipher_final(operation->state, srcData, srcLen, dst, + &tmp_dlen); + } + if (res != TEE_SUCCESS) + goto out; + + acc_dlen += tmp_dlen; + if (destLen) + *destLen = acc_dlen; + + operation->info.handleState &= ~TEE_HANDLE_FLAG_INITIALIZED; + + operation->operationState = TEE_OPERATION_STATE_INITIAL; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_CipherDoFinal(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t dl = 0; + + if (destLen) { + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + dl = *destLen; + } + res = TEE_CipherDoFinal(operation, srcData, srcLen, destData, &dl); + if (destLen) + *destLen = dl; + return res; +} + +/* Cryptographic Operations API - MAC Functions */ + +void TEE_MACInit(TEE_OperationHandle operation, const void *IV, size_t IVLen) +{ + if (operation == TEE_HANDLE_NULL) + TEE_Panic(0); + + if (operation->info.operationClass != TEE_OPERATION_MAC) + TEE_Panic(0); + + if (!(operation->info.handleState & TEE_HANDLE_FLAG_KEY_SET) || + !(operation->key1)) + TEE_Panic(0); + + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) + TEE_ResetOperation(operation); + + operation->operationState = TEE_OPERATION_STATE_ACTIVE; + + init_hash_operation(operation, IV, IVLen); +} + +void __GP11_TEE_MACInit(TEE_OperationHandle operation, const void *IV, + uint32_t IVLen) +{ + return TEE_MACInit(operation, IV, IVLen); +} + +void TEE_MACUpdate(TEE_OperationHandle operation, const void *chunk, + size_t chunkSize) +{ + TEE_Result res; + + if (operation == TEE_HANDLE_NULL || (chunk == NULL && chunkSize != 0)) + TEE_Panic(0); + + if (operation->info.operationClass != TEE_OPERATION_MAC) + TEE_Panic(0); + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) + TEE_Panic(0); + + if (operation->operationState != TEE_OPERATION_STATE_ACTIVE) + TEE_Panic(0); + + res = _utee_hash_update(operation->state, chunk, chunkSize); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void __GP11_TEE_MACUpdate(TEE_OperationHandle operation, const void *chunk, + uint32_t chunkSize) +{ + return TEE_MACUpdate(operation, chunk, chunkSize); +} + +TEE_Result TEE_MACComputeFinal(TEE_OperationHandle operation, + const void *message, size_t messageLen, + void *mac, size_t *macLen) +{ + TEE_Result res; + uint64_t ml; + + if (operation == TEE_HANDLE_NULL || (!message && messageLen)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_inout_annotation(macLen, sizeof(*macLen)); + + if (operation->info.operationClass != TEE_OPERATION_MAC) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (operation->operationState != TEE_OPERATION_STATE_ACTIVE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + ml = *macLen; + res = _utee_hash_final(operation->state, message, messageLen, mac, &ml); + *macLen = ml; + if (res != TEE_SUCCESS) + goto out; + + operation->info.handleState &= ~TEE_HANDLE_FLAG_INITIALIZED; + + operation->operationState = TEE_OPERATION_STATE_INITIAL; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_MACComputeFinal(TEE_OperationHandle operation, + const void *message, uint32_t messageLen, + void *mac, uint32_t *macLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t ml = 0; + + __utee_check_inout_annotation(macLen, sizeof(*macLen)); + ml = *macLen; + res = TEE_MACComputeFinal(operation, message, messageLen, mac, &ml); + *macLen = ml; + return res; +} + +TEE_Result TEE_MACCompareFinal(TEE_OperationHandle operation, + const void *message, size_t messageLen, + const void *mac, size_t macLen) +{ + TEE_Result res; + uint8_t computed_mac[TEE_MAX_HASH_SIZE] = { 0 }; + size_t computed_mac_size = TEE_MAX_HASH_SIZE; + + if (operation->info.operationClass != TEE_OPERATION_MAC) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (operation->operationState != TEE_OPERATION_STATE_ACTIVE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = TEE_MACComputeFinal(operation, message, messageLen, computed_mac, + &computed_mac_size); + if (res != TEE_SUCCESS) + goto out; + + if (computed_mac_size != macLen) { + res = TEE_ERROR_MAC_INVALID; + goto out; + } + + if (consttime_memcmp(mac, computed_mac, computed_mac_size) != 0) { + res = TEE_ERROR_MAC_INVALID; + goto out; + } + + operation->operationState = TEE_OPERATION_STATE_INITIAL; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_MAC_INVALID) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_MACCompareFinal(TEE_OperationHandle operation, + const void *message, uint32_t messageLen, + const void *mac, uint32_t macLen) +{ + return TEE_MACCompareFinal(operation, message, messageLen, mac, macLen); +} + +/* Cryptographic Operations API - Authenticated Encryption Functions */ + +TEE_Result TEE_AEInit(TEE_OperationHandle operation, const void *nonce, + size_t nonceLen, uint32_t tagLen, size_t AADLen, + size_t payloadLen) +{ + TEE_Result res; + + if (operation == TEE_HANDLE_NULL || nonce == NULL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (operation->info.operationClass != TEE_OPERATION_AE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * AES-CCM tag len is specified by AES-CCM spec and handled in TEE Core + * in the implementation. But AES-GCM spec doesn't specify the tag len + * according to the same principle so we have to check here instead to + * be GP compliant. + */ + if (operation->info.algorithm == TEE_ALG_AES_GCM) { + /* + * From GP spec: For AES-GCM, can be 128, 120, 112, 104, or 96 + */ + if (tagLen < 96 || tagLen > 128 || (tagLen % 8 != 0)) { + res = TEE_ERROR_NOT_SUPPORTED; + goto out; + } + } + + res = _utee_authenc_init(operation->state, nonce, nonceLen, tagLen / 8, + AADLen, payloadLen); + if (res != TEE_SUCCESS) + goto out; + + operation->info.digestLength = tagLen / 8; + operation->buffer_offs = 0; + operation->info.handleState |= TEE_HANDLE_FLAG_INITIALIZED; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_NOT_SUPPORTED) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AEInit(TEE_OperationHandle operation, const void *nonce, + uint32_t nonceLen, uint32_t tagLen, + uint32_t AADLen, uint32_t payloadLen) +{ + return TEE_AEInit(operation, nonce, nonceLen, tagLen, AADLen, + payloadLen); +} + +void TEE_AEUpdateAAD(TEE_OperationHandle operation, const void *AADdata, + size_t AADdataLen) +{ + TEE_Result res = TEE_SUCCESS; + + if (operation == TEE_HANDLE_NULL || (!AADdata && AADdataLen)) + TEE_Panic(0); + + if (operation->info.operationClass != TEE_OPERATION_AE) + TEE_Panic(0); + + if (operation->operationState != TEE_OPERATION_STATE_INITIAL) + TEE_Panic(0); + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) + TEE_Panic(0); + + res = _utee_authenc_update_aad(operation->state, AADdata, AADdataLen); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void __GP11_TEE_AEUpdateAAD(TEE_OperationHandle operation, const void *AADdata, + uint32_t AADdataLen) +{ + TEE_Result res = TEE_SUCCESS; + + if (operation == TEE_HANDLE_NULL || + (AADdata == NULL && AADdataLen != 0)) + TEE_Panic(0); + + if (operation->info.operationClass != TEE_OPERATION_AE) + TEE_Panic(0); + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) + TEE_Panic(0); + + res = _utee_authenc_update_aad(operation->state, AADdata, AADdataLen); + + operation->operationState = TEE_OPERATION_STATE_ACTIVE; + + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +static TEE_Result ae_update_helper(TEE_OperationHandle operation, + const void *src, size_t slen, void *dst, + size_t *dlen) +{ + TEE_Result res = TEE_SUCCESS; + size_t req_dlen = 0; + uint64_t dl = 0; + + if (!src && !slen) { + *dlen = 0; + return TEE_SUCCESS; + } + + /* + * Check that required destLen is big enough before starting to feed + * data to the algorithm. Errors during feeding of data are fatal as we + * can't restore sync with this API. + */ + if (operation->block_size > 1) { + req_dlen = ROUNDDOWN(operation->buffer_offs + slen, + operation->block_size); + } else { + req_dlen = slen; + } + + dl = *dlen; + if (dl < req_dlen) { + *dlen = req_dlen; + return TEE_ERROR_SHORT_BUFFER; + } + + if (operation->block_size > 1) { + res = tee_buffer_update(operation, _utee_authenc_update_payload, + src, slen, dst, &dl); + } else { + if (slen > 0) { + res = _utee_authenc_update_payload(operation->state, + src, slen, dst, &dl); + } else { + dl = 0; + res = TEE_SUCCESS; + } + } + + if (!res) + *dlen = dl; + + return res; +} + +TEE_Result TEE_AEUpdate(TEE_OperationHandle operation, const void *srcData, + size_t srcLen, void *destData, size_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_outbuf_annotation(destData, destLen); + + if (operation->info.operationClass != TEE_OPERATION_AE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + res = ae_update_helper(operation, srcData, srcLen, destData, destLen); + if (res != TEE_ERROR_SHORT_BUFFER && srcLen) + operation->operationState = TEE_OPERATION_STATE_ACTIVE; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AEUpdate(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t dl = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_gp11_outbuf_annotation(destData, destLen); + + if (operation->info.operationClass != TEE_OPERATION_AE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + dl = *destLen; + res = ae_update_helper(operation, srcData, srcLen, destData, &dl); + *destLen = dl; + + if (res != TEE_SUCCESS) + goto out; + + operation->operationState = TEE_OPERATION_STATE_ACTIVE; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_AEEncryptFinal(TEE_OperationHandle operation, + const void *srcData, size_t srcLen, + void *destData, size_t *destLen, void *tag, + size_t *tagLen) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *dst = destData; + size_t acc_dlen = 0; + uint64_t tmp_dlen = 0; + size_t req_dlen = 0; + uint64_t tl = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + __utee_check_inout_annotation(tagLen, sizeof(*tagLen)); + + if (operation->info.operationClass != TEE_OPERATION_AE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * Check that required destLen is big enough before starting to feed + * data to the algorithm. Errors during feeding of data are fatal as we + * can't restore sync with this API. + * + * Need to check this before update_payload since sync would be lost if + * we return short buffer after that. + */ + res = TEE_ERROR_GENERIC; + + req_dlen = operation->buffer_offs + srcLen; + if (*destLen < req_dlen) { + *destLen = req_dlen; + res = TEE_ERROR_SHORT_BUFFER; + } + + if (*tagLen < operation->info.digestLength) { + *tagLen = operation->info.digestLength; + res = TEE_ERROR_SHORT_BUFFER; + } + + if (res == TEE_ERROR_SHORT_BUFFER) + goto out; + + tl = *tagLen; + tmp_dlen = *destLen - acc_dlen; + if (operation->block_size > 1) { + res = tee_buffer_update(operation, _utee_authenc_update_payload, + srcData, srcLen, dst, &tmp_dlen); + if (res != TEE_SUCCESS) + goto out; + + dst += tmp_dlen; + acc_dlen += tmp_dlen; + + tmp_dlen = *destLen - acc_dlen; + res = _utee_authenc_enc_final(operation->state, + operation->buffer, + operation->buffer_offs, dst, + &tmp_dlen, tag, &tl); + } else { + res = _utee_authenc_enc_final(operation->state, srcData, + srcLen, dst, &tmp_dlen, + tag, &tl); + } + *tagLen = tl; + if (res != TEE_SUCCESS) + goto out; + + acc_dlen += tmp_dlen; + *destLen = acc_dlen; + + operation->info.handleState &= ~TEE_HANDLE_FLAG_INITIALIZED; + + operation->operationState = TEE_OPERATION_STATE_INITIAL; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AEEncryptFinal(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen, + void *tag, uint32_t *tagLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t dl = 0; + size_t tl = 0; + + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + __utee_check_inout_annotation(tagLen, sizeof(*tagLen)); + dl = *destLen; + tl = *tagLen; + res = TEE_AEEncryptFinal(operation, srcData, srcLen, destData, &dl, + tag, &tl); + *destLen = dl; + *tagLen = tl; + return res; +} + +TEE_Result TEE_AEDecryptFinal(TEE_OperationHandle operation, + const void *srcData, size_t srcLen, + void *destData, size_t *destLen, void *tag, + size_t tagLen) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *dst = destData; + size_t acc_dlen = 0; + uint64_t tmp_dlen = 0; + size_t req_dlen = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + + if (operation->info.operationClass != TEE_OPERATION_AE) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if ((operation->info.handleState & TEE_HANDLE_FLAG_INITIALIZED) == 0) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + /* + * Check that required destLen is big enough before starting to feed + * data to the algorithm. Errors during feeding of data are fatal as we + * can't restore sync with this API. + */ + req_dlen = operation->buffer_offs + srcLen; + if (*destLen < req_dlen) { + *destLen = req_dlen; + res = TEE_ERROR_SHORT_BUFFER; + goto out; + } + + tmp_dlen = *destLen - acc_dlen; + if (operation->block_size > 1) { + res = tee_buffer_update(operation, _utee_authenc_update_payload, + srcData, srcLen, dst, &tmp_dlen); + if (res != TEE_SUCCESS) + goto out; + + dst += tmp_dlen; + acc_dlen += tmp_dlen; + + tmp_dlen = *destLen - acc_dlen; + res = _utee_authenc_dec_final(operation->state, + operation->buffer, + operation->buffer_offs, dst, + &tmp_dlen, tag, tagLen); + } else { + res = _utee_authenc_dec_final(operation->state, srcData, + srcLen, dst, &tmp_dlen, + tag, tagLen); + } + if (res != TEE_SUCCESS) + goto out; + + /* Supplied tagLen should match what we initiated with */ + if (tagLen != operation->info.digestLength) + res = TEE_ERROR_MAC_INVALID; + + acc_dlen += tmp_dlen; + *destLen = acc_dlen; + + operation->info.handleState &= ~TEE_HANDLE_FLAG_INITIALIZED; + + operation->operationState = TEE_OPERATION_STATE_INITIAL; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER && + res != TEE_ERROR_MAC_INVALID) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AEDecryptFinal(TEE_OperationHandle operation, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen, + void *tag, uint32_t tagLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t dl = 0; + + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + dl = *destLen; + res = TEE_AEDecryptFinal(operation, srcData, srcLen, destData, &dl, + tag, tagLen); + *destLen = dl; + return res; +} + +/* Cryptographic Operations API - Asymmetric Functions */ + +TEE_Result TEE_AsymmetricEncrypt(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *srcData, + size_t srcLen, void *destData, + size_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + uint64_t dl = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) + TEE_Panic(0); + + __utee_check_attr_in_annotation(params, paramCount); + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != TEE_OPERATION_ASYMMETRIC_CIPHER) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_ENCRYPT) + TEE_Panic(0); + + __utee_from_attr(ua, params, paramCount); + dl = *destLen; + res = _utee_asymm_operate(operation->state, ua, paramCount, srcData, + srcLen, destData, &dl); + *destLen = dl; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER && + res != TEE_ERROR_BAD_PARAMETERS && + res != TEE_ERROR_CIPHERTEXT_INVALID && + res != TEE_ERROR_NOT_SUPPORTED) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AsymmetricEncrypt(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + uint64_t dl = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) + TEE_Panic(0); + + __utee_check_gp11_attr_in_annotation(params, paramCount); + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != TEE_OPERATION_ASYMMETRIC_CIPHER) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_ENCRYPT) + TEE_Panic(0); + + __utee_from_gp11_attr(ua, params, paramCount); + dl = *destLen; + res = _utee_asymm_operate(operation->state, ua, paramCount, srcData, + srcLen, destData, &dl); + *destLen = dl; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER && + res != TEE_ERROR_BAD_PARAMETERS) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_AsymmetricDecrypt(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *srcData, + size_t srcLen, void *destData, + size_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + uint64_t dl = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) + TEE_Panic(0); + + __utee_check_attr_in_annotation(params, paramCount); + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != TEE_OPERATION_ASYMMETRIC_CIPHER) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_DECRYPT) + TEE_Panic(0); + + __utee_from_attr(ua, params, paramCount); + dl = *destLen; + res = _utee_asymm_operate(operation->state, ua, paramCount, srcData, + srcLen, destData, &dl); + *destLen = dl; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER && + res != TEE_ERROR_BAD_PARAMETERS && + res != TEE_ERROR_CIPHERTEXT_INVALID && + res != TEE_ERROR_NOT_SUPPORTED) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AsymmetricDecrypt(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *srcData, uint32_t srcLen, + void *destData, uint32_t *destLen) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + uint64_t dl = 0; + + if (operation == TEE_HANDLE_NULL || (!srcData && srcLen)) + TEE_Panic(0); + + __utee_check_gp11_attr_in_annotation(params, paramCount); + __utee_check_inout_annotation(destLen, sizeof(*destLen)); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != TEE_OPERATION_ASYMMETRIC_CIPHER) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_DECRYPT) + TEE_Panic(0); + + __utee_from_gp11_attr(ua, params, paramCount); + dl = *destLen; + res = _utee_asymm_operate(operation->state, ua, paramCount, srcData, + srcLen, destData, &dl); + *destLen = dl; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_SHORT_BUFFER && + res != TEE_ERROR_BAD_PARAMETERS) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_AsymmetricSignDigest(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *digest, + size_t digestLen, void *signature, + size_t *signatureLen) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + uint64_t sl = 0; + + if (operation == TEE_HANDLE_NULL || (!digest && digestLen)) + TEE_Panic(0); + + __utee_check_attr_in_annotation(params, paramCount); + __utee_check_inout_annotation(signatureLen, sizeof(*signatureLen)); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != + TEE_OPERATION_ASYMMETRIC_SIGNATURE) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_SIGN) + TEE_Panic(0); + + __utee_from_attr(ua, params, paramCount); + sl = *signatureLen; + res = _utee_asymm_operate(operation->state, ua, paramCount, digest, + digestLen, signature, &sl); + *signatureLen = sl; + + if (res != TEE_SUCCESS && res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AsymmetricSignDigest(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *digest, + uint32_t digestLen, void *signature, + uint32_t *signatureLen) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + uint64_t sl = 0; + + if (operation == TEE_HANDLE_NULL || (!digest && digestLen)) + TEE_Panic(0); + + __utee_check_gp11_attr_in_annotation(params, paramCount); + __utee_check_inout_annotation(signatureLen, sizeof(*signatureLen)); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != + TEE_OPERATION_ASYMMETRIC_SIGNATURE) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_SIGN) + TEE_Panic(0); + + __utee_from_gp11_attr(ua, params, paramCount); + sl = *signatureLen; + res = _utee_asymm_operate(operation->state, ua, paramCount, digest, + digestLen, signature, &sl); + *signatureLen = sl; + + if (res != TEE_SUCCESS && res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(res); + + return res; +} + +TEE_Result TEE_AsymmetricVerifyDigest(TEE_OperationHandle operation, + const TEE_Attribute *params, + uint32_t paramCount, const void *digest, + size_t digestLen, + const void *signature, + size_t signatureLen) +{ + TEE_Result res; + struct utee_attribute ua[paramCount]; + + if (operation == TEE_HANDLE_NULL || + (digest == NULL && digestLen != 0) || + (signature == NULL && signatureLen != 0)) + TEE_Panic(0); + + __utee_check_attr_in_annotation(params, paramCount); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != + TEE_OPERATION_ASYMMETRIC_SIGNATURE) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_VERIFY) + TEE_Panic(0); + + __utee_from_attr(ua, params, paramCount); + res = _utee_asymm_verify(operation->state, ua, paramCount, digest, + digestLen, signature, signatureLen); + + if (res != TEE_SUCCESS && res != TEE_ERROR_SIGNATURE_INVALID) + TEE_Panic(res); + + return res; +} + +TEE_Result __GP11_TEE_AsymmetricVerifyDigest(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, + const void *digest, + uint32_t digestLen, + const void *signature, + uint32_t signatureLen) +{ + TEE_Result res = TEE_SUCCESS; + struct utee_attribute ua[paramCount]; + + if (operation == TEE_HANDLE_NULL || (!digest && digestLen) || + (!signature && signatureLen)) + TEE_Panic(0); + + __utee_check_gp11_attr_in_annotation(params, paramCount); + + if (!operation->key1) + TEE_Panic(0); + if (operation->info.operationClass != + TEE_OPERATION_ASYMMETRIC_SIGNATURE) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_VERIFY) + TEE_Panic(0); + + __utee_from_gp11_attr(ua, params, paramCount); + res = _utee_asymm_verify(operation->state, ua, paramCount, digest, + digestLen, signature, signatureLen); + + if (res != TEE_SUCCESS && res != TEE_ERROR_SIGNATURE_INVALID) + TEE_Panic(res); + + return res; +} + +/* Cryptographic Operations API - Key Derivation Functions */ + +void TEE_DeriveKey(TEE_OperationHandle operation, + const TEE_Attribute *params, uint32_t paramCount, + TEE_ObjectHandle derivedKey) +{ + struct utee_attribute ua[paramCount]; + struct utee_object_info key_info = { }; + TEE_Result res = TEE_SUCCESS; + + if (operation == TEE_HANDLE_NULL || derivedKey == 0) + TEE_Panic(0); + + __utee_check_attr_in_annotation(params, paramCount); + + if (TEE_ALG_GET_CLASS(operation->info.algorithm) != + TEE_OPERATION_KEY_DERIVATION) + TEE_Panic(0); + + if (operation->info.operationClass != TEE_OPERATION_KEY_DERIVATION) + TEE_Panic(0); + if (!operation->key1) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_DERIVE) + TEE_Panic(0); + if ((operation->info.handleState & TEE_HANDLE_FLAG_KEY_SET) == 0) + TEE_Panic(0); + + res = _utee_cryp_obj_get_info((unsigned long)derivedKey, &key_info); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + if (key_info.obj_type != TEE_TYPE_GENERIC_SECRET) + TEE_Panic(0); + if ((key_info.handle_flags & TEE_HANDLE_FLAG_INITIALIZED) != 0) + TEE_Panic(0); + + __utee_from_attr(ua, params, paramCount); + res = _utee_cryp_derive_key(operation->state, ua, paramCount, + (unsigned long)derivedKey); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void __GP11_TEE_DeriveKey(TEE_OperationHandle operation, + const __GP11_TEE_Attribute *params, + uint32_t paramCount, TEE_ObjectHandle derivedKey) +{ + struct utee_attribute ua[paramCount]; + struct utee_object_info key_info = { }; + TEE_Result res = TEE_SUCCESS; + + if (operation == TEE_HANDLE_NULL || derivedKey == 0) + TEE_Panic(0); + + __utee_check_gp11_attr_in_annotation(params, paramCount); + + if (TEE_ALG_GET_CLASS(operation->info.algorithm) != + TEE_OPERATION_KEY_DERIVATION) + TEE_Panic(0); + + if (operation->info.operationClass != TEE_OPERATION_KEY_DERIVATION) + TEE_Panic(0); + if (!operation->key1) + TEE_Panic(0); + if (operation->info.mode != TEE_MODE_DERIVE) + TEE_Panic(0); + if ((operation->info.handleState & TEE_HANDLE_FLAG_KEY_SET) == 0) + TEE_Panic(0); + + res = _utee_cryp_obj_get_info((unsigned long)derivedKey, &key_info); + if (res != TEE_SUCCESS) + TEE_Panic(res); + + if (key_info.obj_type != TEE_TYPE_GENERIC_SECRET) + TEE_Panic(0); + if ((key_info.handle_flags & TEE_HANDLE_FLAG_INITIALIZED) != 0) + TEE_Panic(0); + + __utee_from_gp11_attr(ua, params, paramCount); + res = _utee_cryp_derive_key(operation->state, ua, paramCount, + (unsigned long)derivedKey); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +/* Cryptographic Operations API - Random Number Generation Functions */ + +void TEE_GenerateRandom(void *randomBuffer, size_t randomBufferLen) +{ + TEE_Result res; + + res = _utee_cryp_random_number_generate(randomBuffer, randomBufferLen); + if (res != TEE_SUCCESS) + TEE_Panic(res); +} + +void __GP11_TEE_GenerateRandom(void *randomBuffer, uint32_t randomBufferLen) +{ + TEE_GenerateRandom(randomBuffer, randomBufferLen); +} + +int rand(void) +{ + int rc; + + TEE_GenerateRandom(&rc, sizeof(rc)); + + /* + * RAND_MAX is the larges int, INT_MAX which is all bits but the + * highest bit set. + */ + return rc & RAND_MAX; +} + +TEE_Result TEE_IsAlgorithmSupported(uint32_t alg, uint32_t element) +{ + if (IS_ENABLED(CFG_CRYPTO_AES)) { + if (IS_ENABLED(CFG_CRYPTO_ECB)) { + if (alg == TEE_ALG_AES_ECB_NOPAD) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CBC)) { + if (alg == TEE_ALG_AES_CBC_NOPAD) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CTR)) { + if (alg == TEE_ALG_AES_CTR) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CTS)) { + if (alg == TEE_ALG_AES_CTS) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_XTS)) { + if (alg == TEE_ALG_AES_XTS) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CBC_MAC)) { + if (alg == TEE_ALG_AES_CBC_MAC_NOPAD || + alg == TEE_ALG_AES_CBC_MAC_PKCS5) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CMAC)) { + if (alg == TEE_ALG_AES_CMAC) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CCM)) { + if (alg == TEE_ALG_AES_CCM) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_GCM)) { + if (alg == TEE_ALG_AES_GCM) + goto check_element_none; + } + } + if (IS_ENABLED(CFG_CRYPTO_DES)) { + if (IS_ENABLED(CFG_CRYPTO_ECB)) { + if (alg == TEE_ALG_DES_ECB_NOPAD || + alg == TEE_ALG_DES3_ECB_NOPAD) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CBC)) { + if (alg == TEE_ALG_DES_CBC_NOPAD || + alg == TEE_ALG_DES3_CBC_NOPAD) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CBC_MAC)) { + if (alg == TEE_ALG_DES_CBC_MAC_NOPAD || + alg == TEE_ALG_DES_CBC_MAC_PKCS5 || + alg == TEE_ALG_DES3_CBC_MAC_NOPAD || + alg == TEE_ALG_DES3_CBC_MAC_PKCS5) + goto check_element_none; + } + } + if (IS_ENABLED(CFG_CRYPTO_MD5)) { + if (alg == TEE_ALG_MD5) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA1)) { + if (alg == TEE_ALG_SHA1) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA224)) { + if (alg == TEE_ALG_SHA224) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA256)) { + if (alg == TEE_ALG_SHA256) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA384)) { + if (alg == TEE_ALG_SHA384) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA512)) { + if (alg == TEE_ALG_SHA512) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_224)) { + if (alg == TEE_ALG_SHA3_224) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_256)) { + if (alg == TEE_ALG_SHA3_256) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_384)) { + if (alg == TEE_ALG_SHA3_384) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_512)) { + if (alg == TEE_ALG_SHA3_512) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_MD5) && IS_ENABLED(CFG_CRYPTO_SHA1)) { + if (alg == TEE_ALG_MD5SHA1) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_HMAC)) { + if (IS_ENABLED(CFG_CRYPTO_MD5)) { + if (alg == TEE_ALG_HMAC_MD5) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA1)) { + if (alg == TEE_ALG_HMAC_SHA1) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA224)) { + if (alg == TEE_ALG_HMAC_SHA224) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA256)) { + if (alg == TEE_ALG_HMAC_SHA256) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA384)) { + if (alg == TEE_ALG_HMAC_SHA384) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA512)) { + if (alg == TEE_ALG_HMAC_SHA512) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_224)) { + if (alg == TEE_ALG_HMAC_SHA3_224) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_256)) { + if (alg == TEE_ALG_HMAC_SHA3_256) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_384)) { + if (alg == TEE_ALG_HMAC_SHA3_384) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA3_512)) { + if (alg == TEE_ALG_HMAC_SHA3_512) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SM3)) { + if (alg == TEE_ALG_HMAC_SM3) + goto check_element_none; + } + } + if (IS_ENABLED(CFG_CRYPTO_SM3)) { + if (alg == TEE_ALG_SM3) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SM4)) { + if (IS_ENABLED(CFG_CRYPTO_ECB)) { + if (alg == TEE_ALG_SM4_ECB_NOPAD) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CBC)) { + if (alg == TEE_ALG_SM4_CBC_NOPAD) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_CTR)) { + if (alg == TEE_ALG_SM4_CTR) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_XTS)) { + if (alg == TEE_ALG_SM4_XTS) + goto check_element_none; + } + } + if (IS_ENABLED(CFG_CRYPTO_RSA)) { + if (IS_ENABLED(CFG_CRYPTO_MD5)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5_MD5 || + alg == TEE_ALG_RSASSA_PKCS1_PSS_MGF1_MD5 || + alg == TEE_ALG_RSAES_PKCS1_OAEP_MGF1_MD5) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA1)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5_SHA1 || + alg == TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1 || + alg == TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_MD5) && IS_ENABLED(CFG_CRYPTO_SHA1)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5_MD5SHA1) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA224)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5_SHA224 || + alg == TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224 || + alg == TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA256)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5_SHA256 || + alg == TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256 || + alg == TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA384)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5_SHA384 || + alg == TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384 || + alg == TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA512)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5_SHA512 || + alg == TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512 || + alg == TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_RSASSA_NA1)) { + if (alg == TEE_ALG_RSASSA_PKCS1_V1_5) + goto check_element_none; + } + if (alg == TEE_ALG_RSA_NOPAD) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_DSA)) { + if (IS_ENABLED(CFG_CRYPTO_SHA1)) { + if (alg == TEE_ALG_DSA_SHA1) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA224)) { + if (alg == TEE_ALG_DSA_SHA224) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_SHA256)) { + if (alg == TEE_ALG_DSA_SHA256) + goto check_element_none; + } + } + if (IS_ENABLED(CFG_CRYPTO_DH)) { + if (alg == TEE_ALG_DH_DERIVE_SHARED_SECRET) + goto check_element_none; + } + if (IS_ENABLED(CFG_CRYPTO_ECC)) { + if ((alg == __OPTEE_ALG_ECDH_P192 || + alg == __OPTEE_ALG_ECDSA_P192 || + alg == TEE_ALG_ECDH_DERIVE_SHARED_SECRET || + alg == TEE_ALG_ECDSA_SHA1) && + element == TEE_ECC_CURVE_NIST_P192) + return TEE_SUCCESS; + if ((alg == __OPTEE_ALG_ECDH_P224 || + alg == __OPTEE_ALG_ECDSA_P224 || + alg == TEE_ALG_ECDH_DERIVE_SHARED_SECRET || + alg == TEE_ALG_ECDSA_SHA224) && + element == TEE_ECC_CURVE_NIST_P224) + return TEE_SUCCESS; + if ((alg == __OPTEE_ALG_ECDH_P256 || + alg == __OPTEE_ALG_ECDSA_P256 || + alg == TEE_ALG_ECDH_DERIVE_SHARED_SECRET || + alg == TEE_ALG_ECDSA_SHA256) && + element == TEE_ECC_CURVE_NIST_P256) + return TEE_SUCCESS; + if ((alg == __OPTEE_ALG_ECDH_P384 || + alg == __OPTEE_ALG_ECDSA_P384 || + alg == TEE_ALG_ECDH_DERIVE_SHARED_SECRET || + alg == TEE_ALG_ECDSA_SHA384) && + element == TEE_ECC_CURVE_NIST_P384) + return TEE_SUCCESS; + if ((alg == __OPTEE_ALG_ECDH_P521 || + alg == __OPTEE_ALG_ECDSA_P521 || + alg == TEE_ALG_ECDH_DERIVE_SHARED_SECRET || + alg == TEE_ALG_ECDSA_SHA512) && + element == TEE_ECC_CURVE_NIST_P521) + return TEE_SUCCESS; + } + if (IS_ENABLED(CFG_CRYPTO_SM2_DSA)) { + if (alg == TEE_ALG_SM2_DSA_SM3 && element == TEE_ECC_CURVE_SM2) + return TEE_SUCCESS; + } + if (IS_ENABLED(CFG_CRYPTO_SM2_KEP)) { + if (alg == TEE_ALG_SM2_KEP && element == TEE_ECC_CURVE_SM2) + return TEE_SUCCESS; + } + if (IS_ENABLED(CFG_CRYPTO_SM2_PKE)) { + if (alg == TEE_ALG_SM2_PKE && element == TEE_ECC_CURVE_SM2) + return TEE_SUCCESS; + } + if (IS_ENABLED(CFG_CRYPTO_X25519)) { + if (alg == TEE_ALG_X25519 && element == TEE_ECC_CURVE_25519) + return TEE_SUCCESS; + } + if (IS_ENABLED(CFG_CRYPTO_ED25519)) { + if (alg == TEE_ALG_ED25519 && element == TEE_ECC_CURVE_25519) + return TEE_SUCCESS; + } + + return TEE_ERROR_NOT_SUPPORTED; +check_element_none: + if (element == TEE_CRYPTO_ELEMENT_NONE) + return TEE_SUCCESS; + return TEE_ERROR_NOT_SUPPORTED; +} diff --git a/optee_os/lib/libutee/tee_api_panic.c b/optee_os/lib/libutee/tee_api_panic.c new file mode 100644 index 0000000..35105bc --- /dev/null +++ b/optee_os/lib/libutee/tee_api_panic.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#include "tee_api_private.h" + +#define ACCESS_RW (TEE_MEMORY_ACCESS_READ | TEE_MEMORY_ACCESS_WRITE) +#define ACCESS_W_ANY (TEE_MEMORY_ACCESS_WRITE | TEE_MEMORY_ACCESS_ANY_OWNER) +#define ACCESS_R TEE_MEMORY_ACCESS_READ +#define ACCESS_W TEE_MEMORY_ACCESS_WRITE + +/* System API - Misc */ + +void TEE_Panic(TEE_Result panicCode) +{ + _utee_panic(panicCode); +#ifdef __COVERITY__ + __coverity_panic__(); +#endif +} + +static void check_res(const char *msg __maybe_unused, TEE_Result res) +{ + if (res) { + DMSG("%s: error %#"PRIx32, msg, res); + TEE_Panic(0); + } +} + +static TEE_Result check_access(uint32_t flags, void *buf, size_t len) +{ + if (!len) + return TEE_SUCCESS; + + if (!buf) + return TEE_ERROR_SECURITY; + + if (IS_ENABLED(CFG_TA_STRICT_ANNOTATION_CHECKS)) + return TEE_CheckMemoryAccessRights(flags, buf, len); + + return TEE_SUCCESS; +} + +void __utee_check_outbuf_annotation(void *buf, size_t *len) +{ + check_res("[outbuf] len", + check_access(ACCESS_RW, len, sizeof(*len))); + check_res("[outbuf] buf", + check_access(ACCESS_W_ANY, buf, *len)); +} + +void __utee_check_gp11_outbuf_annotation(void *buf, uint32_t *len) +{ + check_res("[outbuf] len", + check_access(ACCESS_RW, len, sizeof(*len))); + check_res("[outbuf] buf", + check_access(ACCESS_W_ANY, buf, *len)); +} + +void __utee_check_instring_annotation(const char *buf) +{ + check_res("[instring]", + check_access(ACCESS_R, (char *)buf, strlen(buf) + 1)); +} + +void __utee_check_outstring_annotation(char *buf, size_t *len) +{ + check_res("[outstring] len", + check_access(ACCESS_RW, len, sizeof(*len))); + check_res("[outstring] buf", + check_access(ACCESS_W_ANY, buf, *len)); +} + +void __utee_check_gp11_outstring_annotation(char *buf, uint32_t *len) +{ + check_res("[outstring] len", + check_access(ACCESS_RW, len, sizeof(*len))); + check_res("[outstring] buf", + check_access(ACCESS_W_ANY, buf, *len)); +} + +void __utee_check_out_annotation(void *buf, const size_t len) +{ + check_res("[out]", + check_access(ACCESS_W, buf, len)); +} + +void __utee_check_attr_in_annotation(const TEE_Attribute *attr, size_t count) +{ + check_res("[in] attr", + check_access(ACCESS_R, (void *)attr, sizeof(*attr) * count)); +} + +void __utee_check_gp11_attr_in_annotation(const __GP11_TEE_Attribute *attr, + size_t count) + +{ + check_res("[in] attr", + check_access(ACCESS_R, (void *)attr, sizeof(*attr) * count)); +} + +void __utee_check_inout_annotation(void *buf, const size_t len) +{ + check_res("[inout]", + check_access(ACCESS_RW, buf, len)); +} diff --git a/optee_os/lib/libutee/tee_api_private.h b/optee_os/lib/libutee/tee_api_private.h new file mode 100644 index 0000000..2370372 --- /dev/null +++ b/optee_os/lib/libutee/tee_api_private.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2020, Linaro Limited + */ +#ifndef TEE_API_PRIVATE +#define TEE_API_PRIVATE + +#include +#include + + +void __utee_from_attr(struct utee_attribute *ua, const TEE_Attribute *attrs, + uint32_t attr_count); +void __utee_from_gp11_attr(struct utee_attribute *ua, + const __GP11_TEE_Attribute *attrs, + uint32_t attr_count); + +TEE_Result __utee_entry(unsigned long func, unsigned long session_id, + struct utee_params *up, unsigned long cmd_id); + + +#if defined(CFG_TA_GPROF_SUPPORT) +void __utee_gprof_init(void); +void __utee_gprof_fini(void); +#else +static inline void __utee_gprof_init(void) {} +static inline void __utee_gprof_fini(void) {} +#endif + +/* + * The functions help checking that the pointers comply with the parameters + * annotation as described in the spec. Any descrepency results in a panic + * of the TA. + */ +void __utee_check_out_annotation(void *buf, const size_t len); +void __utee_check_inout_annotation(void *buf, const size_t len); +void __utee_check_attr_in_annotation(const TEE_Attribute *attr, size_t count); +void __utee_check_gp11_attr_in_annotation(const __GP11_TEE_Attribute *attr, + size_t count); +void __utee_check_outbuf_annotation(void *buf, size_t *len); +void __utee_check_gp11_outbuf_annotation(void *buf, uint32_t *len); +void __utee_check_instring_annotation(const char *buf); +void __utee_check_outstring_annotation(char *buf, size_t *len); +void __utee_check_gp11_outstring_annotation(char *buf, uint32_t *len); + +#endif /*TEE_API_PRIVATE*/ diff --git a/optee_os/lib/libutee/tee_api_property.c b/optee_os/lib/libutee/tee_api_property.c new file mode 100644 index 0000000..5f33e2c --- /dev/null +++ b/optee_os/lib/libutee/tee_api_property.c @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2017-2020, Linaro Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base64.h" +#include "string_ext.h" +#include "tee_api_private.h" + +#define PROP_STR_MAX 80 + +#define PROP_ENUMERATOR_NOT_STARTED 0xffffffff + +struct prop_enumerator { + uint32_t idx; /* current index */ + TEE_PropSetHandle prop_set; /* part of TEE_PROPSET_xxx */ +}; + +const struct user_ta_property tee_props[] = { + { + "gpd.tee.arith.maxBigIntSize", + USER_TA_PROP_TYPE_U32, + &(const uint32_t){CFG_TA_BIGNUM_MAX_BITS} + }, + { + "gpd.tee.sockets.version", + USER_TA_PROP_TYPE_U32, + &(const uint32_t){TEE_ISOCKET_VERSION} + }, + { + "gpd.tee.sockets.tcp.version", + USER_TA_PROP_TYPE_U32, + &(const uint32_t){TEE_ISOCKET_VERSION} + }, + { + "gpd.tee.internalCore.version", + USER_TA_PROP_TYPE_U32, + &(const uint32_t){TEE_CORE_API_VERSION} + }, +}; + +static TEE_Result propset_get(TEE_PropSetHandle h, + const struct user_ta_property **eps, + size_t *eps_len) +{ + if (h == TEE_PROPSET_CURRENT_TA) { + *eps = ta_props; + *eps_len = ta_num_props; + } else if (h == TEE_PROPSET_CURRENT_CLIENT) { + *eps = NULL; + *eps_len = 0; + } else if (h == TEE_PROPSET_TEE_IMPLEMENTATION) { + *eps = tee_props; + *eps_len = ARRAY_SIZE(tee_props); + } else { + return TEE_ERROR_ITEM_NOT_FOUND; + } + + return TEE_SUCCESS; +} + +static TEE_Result propget_get_ext_prop(const struct user_ta_property *ep, + enum user_ta_prop_type *type, + void *buf, uint32_t *len) +{ + size_t l; + + *type = ep->type; + switch (*type) { + case USER_TA_PROP_TYPE_BOOL: + l = sizeof(bool); + break; + case USER_TA_PROP_TYPE_U32: + l = sizeof(uint32_t); + break; + case USER_TA_PROP_TYPE_U64: + l = sizeof(uint64_t); + break; + case USER_TA_PROP_TYPE_UUID: + l = sizeof(TEE_UUID); + break; + case USER_TA_PROP_TYPE_IDENTITY: + l = sizeof(TEE_Identity); + break; + case USER_TA_PROP_TYPE_STRING: + /* take the leading 0 into account */ + l = strlen(ep->value) + 1; + break; + case USER_TA_PROP_TYPE_BINARY_BLOCK: + /* + * in case of TA property, a binary block is provided as a + * string, which is base64 encoded. We must first decode it, + * without taking into account the zero termination of the + * string + */ + l = *len; + if (!_base64_dec(ep->value, strlen(ep->value), buf, &l) && + l <= *len) + return TEE_ERROR_GENERIC; + if (*len < l) { + *len = l; + return TEE_ERROR_SHORT_BUFFER; + } + + *len = l; + return TEE_SUCCESS; + default: + return TEE_ERROR_GENERIC; + } + + if (*len < l) { + *len = l; + return TEE_ERROR_SHORT_BUFFER; + } + + *len = l; + memcpy(buf, ep->value, l); + return TEE_SUCCESS; +} + +static bool is_propset_pseudo_handle(TEE_PropSetHandle h) +{ + return h == TEE_PROPSET_CURRENT_TA || + h == TEE_PROPSET_CURRENT_CLIENT || + h == TEE_PROPSET_TEE_IMPLEMENTATION; +} + +static TEE_Result propget_get_property(TEE_PropSetHandle h, const char *name, + enum user_ta_prop_type *type, + void *buf, uint32_t *len) +{ + TEE_Result res; + const struct user_ta_property *eps; + size_t eps_len; + uint32_t prop_type; + uint32_t index; + + if (is_propset_pseudo_handle(h)) { + size_t n; + + res = propset_get(h, &eps, &eps_len); + if (res != TEE_SUCCESS) + return res; + + for (n = 0; n < eps_len; n++) { + if (!strcmp(name, eps[n].name)) + return propget_get_ext_prop(eps + n, type, + buf, len); + } + + /* get the index from the name */ + res = _utee_get_property_name_to_index((unsigned long)h, name, + strlen(name) + 1, + &index); + if (res != TEE_SUCCESS) + return res; + res = _utee_get_property((unsigned long)h, index, NULL, NULL, + buf, len, &prop_type); + } else { + struct prop_enumerator *pe = (struct prop_enumerator *)h; + uint32_t idx = pe->idx; + + if (idx == PROP_ENUMERATOR_NOT_STARTED) + return TEE_ERROR_ITEM_NOT_FOUND; + + res = propset_get(pe->prop_set, &eps, &eps_len); + if (res != TEE_SUCCESS) + return res; + + if (idx < eps_len) + return propget_get_ext_prop(eps + idx, type, buf, len); + idx -= eps_len; + + res = _utee_get_property((unsigned long)pe->prop_set, idx, + NULL, NULL, buf, len, &prop_type); + if (res == TEE_ERROR_ITEM_NOT_FOUND) + res = TEE_ERROR_BAD_PARAMETERS; + } + + *type = prop_type; + return res; +} + +TEE_Result TEE_GetPropertyAsString(TEE_PropSetHandle propsetOrEnumerator, + const char *name, char *value, + size_t *value_len) +{ + TEE_Result res = TEE_ERROR_GENERIC; + size_t l = 0; + enum user_ta_prop_type type = USER_TA_PROP_TYPE_INVALID; + void *tmp_buf = 0; + uint32_t tmp_len = 0; + uint32_t uint32_val = 0; + bool bool_val = false; + TEE_Identity *p_identity_val = NULL; + + if (is_propset_pseudo_handle(propsetOrEnumerator)) + __utee_check_instring_annotation(name); + __utee_check_outstring_annotation(value, value_len); + + tmp_len = *value_len; + if (tmp_len < sizeof(TEE_Identity)) + tmp_len = sizeof(TEE_Identity); + tmp_buf = TEE_Malloc(tmp_len, TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!tmp_buf) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto out; + } + + res = propget_get_property(propsetOrEnumerator, name, &type, + tmp_buf, &tmp_len); + if (res != TEE_SUCCESS) { + if (res == TEE_ERROR_SHORT_BUFFER) { + if (type == USER_TA_PROP_TYPE_BINARY_BLOCK) { + /* + * in this case, we must enlarge the buffer + * with the size of the of the base64 encoded + * see base64_enc() function + */ + tmp_len = _base64_enc_len(tmp_len); + } + *value_len = tmp_len; + } + goto out; + } + + switch (type) { + case USER_TA_PROP_TYPE_BOOL: + bool_val = *((bool *)tmp_buf); + l = strlcpy(value, (bool_val ? "true" : "false"), *value_len); + break; + + case USER_TA_PROP_TYPE_U32: + uint32_val = *((uint32_t *)tmp_buf); + l = snprintf(value, *value_len, "%u", uint32_val); + break; + + case USER_TA_PROP_TYPE_UUID: + l = snprintk(value, *value_len, "%pUl", tmp_buf); + break; + + case USER_TA_PROP_TYPE_IDENTITY: + p_identity_val = ((TEE_Identity *)tmp_buf); + l = snprintk(value, *value_len, "%u:%pUl", + p_identity_val->login, + (void *)(&(p_identity_val->uuid))); + break; + + case USER_TA_PROP_TYPE_STRING: + l = strlcpy(value, tmp_buf, *value_len); + break; + + case USER_TA_PROP_TYPE_BINARY_BLOCK: + l = *value_len; /* l includes the zero-termination */ + if (!_base64_enc(tmp_buf, tmp_len, value, &l) && + l <= *value_len) { + res = TEE_ERROR_GENERIC; + goto out; + } + l--; /* remove the zero-termination that is added later */ + break; + + default: + res = TEE_ERROR_BAD_FORMAT; + goto out; + } + + l++; /* include zero termination */ + + if (l > *value_len) + res = TEE_ERROR_SHORT_BUFFER; + *value_len = l; + +out: + if (tmp_buf) + TEE_Free(tmp_buf); + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(0); + + return res; +} + +TEE_Result __GP11_TEE_GetPropertyAsString(TEE_PropSetHandle propsetOrEnumerator, + const char *name, char *valueBuffer, + uint32_t *valueBufferLen) +{ + TEE_Result res = TEE_SUCCESS; + size_t l = 0; + + __utee_check_gp11_outstring_annotation(valueBuffer, valueBufferLen); + l = *valueBufferLen; + res = TEE_GetPropertyAsString(propsetOrEnumerator, name, valueBuffer, + &l); + *valueBufferLen = l; + return res; +} + +TEE_Result TEE_GetPropertyAsBool(TEE_PropSetHandle propsetOrEnumerator, + const char *name, bool *value) +{ + TEE_Result res; + enum user_ta_prop_type type; + uint32_t bool_len = sizeof(bool); + + if (is_propset_pseudo_handle(propsetOrEnumerator)) + __utee_check_instring_annotation(name); + __utee_check_out_annotation(value, sizeof(*value)); + + type = USER_TA_PROP_TYPE_BOOL; + res = propget_get_property(propsetOrEnumerator, name, &type, + value, &bool_len); + if (type != USER_TA_PROP_TYPE_BOOL) + res = TEE_ERROR_BAD_FORMAT; + if (res != TEE_SUCCESS) + goto out; + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_BAD_FORMAT) + TEE_Panic(0); + + return res; +} + +TEE_Result TEE_GetPropertyAsU32(TEE_PropSetHandle propsetOrEnumerator, + const char *name, uint32_t *value) +{ + TEE_Result res; + enum user_ta_prop_type type; + uint32_t uint32_len = sizeof(uint32_t); + + if (is_propset_pseudo_handle(propsetOrEnumerator)) + __utee_check_instring_annotation(name); + __utee_check_out_annotation(value, sizeof(*value)); + + type = USER_TA_PROP_TYPE_U32; + res = propget_get_property(propsetOrEnumerator, name, &type, + value, &uint32_len); + if (type != USER_TA_PROP_TYPE_U32) + res = TEE_ERROR_BAD_FORMAT; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_BAD_FORMAT) + TEE_Panic(0); + + return res; +} + +TEE_Result TEE_GetPropertyAsU64(TEE_PropSetHandle propsetOrEnumerator, + const char *name, uint64_t *value) +{ + TEE_Result res; + enum user_ta_prop_type type; + uint32_t uint64_len = sizeof(*value); + + if (is_propset_pseudo_handle(propsetOrEnumerator)) + __utee_check_instring_annotation(name); + __utee_check_out_annotation(value, sizeof(*value)); + + type = USER_TA_PROP_TYPE_U64; + res = propget_get_property(propsetOrEnumerator, name, &type, + value, &uint64_len); + if (type != USER_TA_PROP_TYPE_U64) + res = TEE_ERROR_BAD_FORMAT; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_BAD_FORMAT) + TEE_Panic(0); + + return res; +} + +TEE_Result +__GP11_TEE_GetPropertyAsBinaryBlock(TEE_PropSetHandle propsetOrEnumerator, + const char *name, void *value, + uint32_t *value_len) +{ + TEE_Result res = TEE_SUCCESS; + enum user_ta_prop_type type = USER_TA_PROP_TYPE_BOOL; + + if (is_propset_pseudo_handle(propsetOrEnumerator)) + __utee_check_instring_annotation(name); + __utee_check_gp11_outbuf_annotation(value, value_len); + + type = USER_TA_PROP_TYPE_BINARY_BLOCK; + res = propget_get_property(propsetOrEnumerator, name, &type, + value, value_len); + if (type != USER_TA_PROP_TYPE_BINARY_BLOCK) + res = TEE_ERROR_BAD_FORMAT; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_BAD_FORMAT && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(0); + + return res; +} + +TEE_Result TEE_GetPropertyAsBinaryBlock(TEE_PropSetHandle propsetOrEnumerator, + const char *name, void *value, + size_t *value_len) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t l = 0; + + __utee_check_outbuf_annotation(value, value_len); + l = *value_len; + res = __GP11_TEE_GetPropertyAsBinaryBlock(propsetOrEnumerator, name, + value, &l); + *value_len = l; + return res; +} + +TEE_Result TEE_GetPropertyAsUUID(TEE_PropSetHandle propsetOrEnumerator, + const char *name, TEE_UUID *value) +{ + TEE_Result res; + enum user_ta_prop_type type; + uint32_t uuid_len = sizeof(TEE_UUID); + + if (is_propset_pseudo_handle(propsetOrEnumerator)) + __utee_check_instring_annotation(name); + __utee_check_out_annotation(value, sizeof(*value)); + + type = USER_TA_PROP_TYPE_UUID; + res = propget_get_property(propsetOrEnumerator, name, &type, + value, &uuid_len); + if (type != USER_TA_PROP_TYPE_UUID) + res = TEE_ERROR_BAD_FORMAT; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_BAD_FORMAT) + TEE_Panic(0); + + return res; +} + +TEE_Result TEE_GetPropertyAsIdentity(TEE_PropSetHandle propsetOrEnumerator, + const char *name, TEE_Identity *value) +{ + TEE_Result res; + enum user_ta_prop_type type; + uint32_t identity_len = sizeof(TEE_Identity); + + if (is_propset_pseudo_handle(propsetOrEnumerator)) + __utee_check_instring_annotation(name); + __utee_check_out_annotation(value, sizeof(*value)); + + type = USER_TA_PROP_TYPE_IDENTITY; + res = propget_get_property(propsetOrEnumerator, name, &type, + value, &identity_len); + if (type != USER_TA_PROP_TYPE_IDENTITY) + res = TEE_ERROR_BAD_FORMAT; + + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_BAD_FORMAT) + TEE_Panic(0); + + return res; +} + +TEE_Result TEE_AllocatePropertyEnumerator(TEE_PropSetHandle *enumerator) +{ + TEE_Result res; + struct prop_enumerator *pe; + + __utee_check_out_annotation(enumerator, sizeof(*enumerator)); + + pe = TEE_Malloc(sizeof(struct prop_enumerator), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (pe == NULL) { + res = TEE_ERROR_OUT_OF_MEMORY; + goto err; + } + + *enumerator = (TEE_PropSetHandle) pe; + TEE_ResetPropertyEnumerator(*enumerator); + + goto out; + +err: + if (res == TEE_ERROR_OUT_OF_MEMORY) + return res; + TEE_Panic(0); +out: + return TEE_SUCCESS; +} + +void TEE_ResetPropertyEnumerator(TEE_PropSetHandle enumerator) +{ + struct prop_enumerator *pe = (struct prop_enumerator *)enumerator; + + pe->idx = PROP_ENUMERATOR_NOT_STARTED; +} + +void TEE_FreePropertyEnumerator(TEE_PropSetHandle enumerator) +{ + struct prop_enumerator *pe = (struct prop_enumerator *)enumerator; + + TEE_Free(pe); +} + +void TEE_StartPropertyEnumerator(TEE_PropSetHandle enumerator, + TEE_PropSetHandle propSet) +{ + struct prop_enumerator *pe = (struct prop_enumerator *)enumerator; + + if (!pe) + return; + + pe->idx = 0; + pe->prop_set = propSet; +} + +TEE_Result __GP11_TEE_GetPropertyName(TEE_PropSetHandle enumerator, + void *name, uint32_t *name_len) +{ + TEE_Result res; + struct prop_enumerator *pe = (struct prop_enumerator *)enumerator; + const struct user_ta_property *eps; + size_t eps_len; + const char *str; + size_t bufferlen; + + if (!pe) { + res = TEE_ERROR_BAD_PARAMETERS; + goto err; + } + __utee_check_gp11_outstring_annotation(name, name_len); + + bufferlen = *name_len; + res = propset_get(pe->prop_set, &eps, &eps_len); + if (res != TEE_SUCCESS) + goto err; + + if (pe->idx < eps_len) { + str = eps[pe->idx].name; + bufferlen = strlcpy(name, str, *name_len) + 1; + if (bufferlen > *name_len) + res = TEE_ERROR_SHORT_BUFFER; + *name_len = bufferlen; + } else { + res = _utee_get_property((unsigned long)pe->prop_set, + pe->idx - eps_len, name, name_len, + NULL, NULL, NULL); + if (res != TEE_SUCCESS) + goto err; + } + +err: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND && + res != TEE_ERROR_SHORT_BUFFER) + TEE_Panic(0); + return res; +} + +TEE_Result TEE_GetPropertyName(TEE_PropSetHandle enumerator, + void *nameBuffer, size_t *nameBufferLen) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t l = 0; + + __utee_check_outstring_annotation(nameBuffer, nameBufferLen); + l = *nameBufferLen; + res = __GP11_TEE_GetPropertyName(enumerator, nameBuffer, &l); + *nameBufferLen = l; + return res; +} + +TEE_Result TEE_GetNextProperty(TEE_PropSetHandle enumerator) +{ + TEE_Result res; + struct prop_enumerator *pe = (struct prop_enumerator *)enumerator; + uint32_t next_idx; + const struct user_ta_property *eps; + size_t eps_len; + + if (!pe) { + res = TEE_ERROR_BAD_PARAMETERS; + goto out; + } + + if (pe->idx == PROP_ENUMERATOR_NOT_STARTED) { + res = TEE_ERROR_ITEM_NOT_FOUND; + goto out; + } + + res = propset_get(pe->prop_set, &eps, &eps_len); + if (res != TEE_SUCCESS) + goto out; + + next_idx = pe->idx + 1; + pe->idx = next_idx; + if (next_idx < eps_len) + res = TEE_SUCCESS; + else + res = _utee_get_property((unsigned long)pe->prop_set, + next_idx - eps_len, NULL, NULL, NULL, + NULL, NULL); + +out: + if (res != TEE_SUCCESS && + res != TEE_ERROR_ITEM_NOT_FOUND) + TEE_Panic(0); + return res; +} diff --git a/optee_os/lib/libutee/tee_socket_private.h b/optee_os/lib/libutee/tee_socket_private.h new file mode 100644 index 0000000..28f48d9 --- /dev/null +++ b/optee_os/lib/libutee/tee_socket_private.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#ifndef __TEE_SOCKET_PRIVATE_H +#define __TEE_SOCKET_PRIVATE_H + +#include +#include <__tee_ipsocket.h> + +static inline uint8_t __tee_socket_ioctl_cmd_to_proto(uint32_t cmd_code) +{ + return cmd_code >> 24; +} + +TEE_Result __tee_socket_pta_open(TEE_ipSocket_ipVersion ip_vers, + const char *addr, uint16_t port, + uint32_t protocol, uint32_t *handle); + +TEE_Result __tee_socket_pta_close(uint32_t handle); + +TEE_Result __tee_socket_pta_send(uint32_t handle, const void *buf, + uint32_t *len, uint32_t timeout); + +TEE_Result __tee_socket_pta_recv(uint32_t handle, void *buf, uint32_t *len, + uint32_t timeout); + +TEE_Result __tee_socket_pta_ioctl(uint32_t handle, uint32_t command, void *buf, + uint32_t *len); + +#endif /*__TEE_SOCKET_PRIVATE_H*/ diff --git a/optee_os/lib/libutee/tee_socket_pta.c b/optee_os/lib/libutee/tee_socket_pta.c new file mode 100644 index 0000000..2104c15 --- /dev/null +++ b/optee_os/lib/libutee/tee_socket_pta.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#include +#include +#include +#include <__tee_tcpsocket_defines.h> +#include <__tee_udpsocket_defines.h> + +#include "tee_socket_private.h" + +static TEE_Result invoke_socket_pta(uint32_t cmd_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + static TEE_TASessionHandle sess = TEE_HANDLE_NULL; + static const TEE_UUID socket_uuid = PTA_SOCKET_UUID; + + if (sess == TEE_HANDLE_NULL) { + TEE_Result res = TEE_OpenTASession(&socket_uuid, + TEE_TIMEOUT_INFINITE, + 0, NULL, &sess, NULL); + + if (res != TEE_SUCCESS) + return res; + } + + return TEE_InvokeTACommand(sess, TEE_TIMEOUT_INFINITE, cmd_id, + param_types, params, NULL); +} + +TEE_Result __tee_socket_pta_open(TEE_ipSocket_ipVersion ip_vers, + const char *addr, uint16_t port, + uint32_t protocol, uint32_t *handle) +{ + TEE_Result res; + uint32_t param_types; + TEE_Param params[TEE_NUM_PARAMS]; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT); + memset(params, 0, sizeof(params)); + + switch (ip_vers) { + case TEE_IP_VERSION_DC: + case TEE_IP_VERSION_4: + case TEE_IP_VERSION_6: + params[0].value.a = ip_vers; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + params[0].value.b = port; + + if (!addr) + return TEE_ERROR_BAD_PARAMETERS; + params[1].memref.buffer = (void *)addr; + params[1].memref.size = strlen(addr) + 1; + + switch (protocol) { + case TEE_ISOCKET_PROTOCOLID_TCP: + case TEE_ISOCKET_PROTOCOLID_UDP: + params[2].value.a = protocol; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + res = invoke_socket_pta(PTA_SOCKET_OPEN, param_types, params); + if (res == TEE_SUCCESS) + *handle = params[3].value.a; + return res; +} + +TEE_Result __tee_socket_pta_close(uint32_t handle) +{ + uint32_t param_types; + TEE_Param params[TEE_NUM_PARAMS]; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + memset(params, 0, sizeof(params)); + + params[0].value.a = handle; + return invoke_socket_pta(PTA_SOCKET_CLOSE, param_types, params); +} + +TEE_Result __tee_socket_pta_send(uint32_t handle, const void *buf, + uint32_t *len, uint32_t timeout) +{ + TEE_Result res; + uint32_t param_types; + TEE_Param params[TEE_NUM_PARAMS]; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE); + memset(params, 0, sizeof(params)); + + params[0].value.a = handle; + params[0].value.b = timeout; + + params[1].memref.buffer = (void *)buf; + params[1].memref.size = *len; + + res = invoke_socket_pta(PTA_SOCKET_SEND, param_types, params); + *len = params[2].value.a; + return res; +} + +TEE_Result __tee_socket_pta_recv(uint32_t handle, void *buf, uint32_t *len, + uint32_t timeout) + +{ + TEE_Result res; + uint32_t param_types; + TEE_Param params[TEE_NUM_PARAMS]; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE); + memset(params, 0, sizeof(params)); + + params[0].value.a = handle; + params[0].value.b = timeout; + + params[1].memref.buffer = buf; + params[1].memref.size = *len; + + res = invoke_socket_pta(PTA_SOCKET_RECV, param_types, params); + *len = params[1].memref.size; + return res; +} + +TEE_Result __tee_socket_pta_ioctl(uint32_t handle, uint32_t command, void *buf, + uint32_t *len) +{ + TEE_Result res; + uint32_t param_types; + TEE_Param params[TEE_NUM_PARAMS]; + + param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, TEE_PARAM_TYPE_NONE); + memset(params, 0, sizeof(params)); + + params[0].value.a = handle; + params[0].value.b = command; + + params[1].memref.buffer = buf; + params[1].memref.size = *len; + + res = invoke_socket_pta(PTA_SOCKET_IOCTL, param_types, params); + *len = params[1].memref.size; + return res; +} diff --git a/optee_os/lib/libutee/tee_system_pta.c b/optee_os/lib/libutee/tee_system_pta.c new file mode 100644 index 0000000..6c35045 --- /dev/null +++ b/optee_os/lib/libutee/tee_system_pta.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2020, Open Mobile Platform LLC + */ + +#include +#include +#include +#include +#include +#include + +static TEE_Result invoke_system_pta(uint32_t cmd_id, uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + static TEE_TASessionHandle sess = TEE_HANDLE_NULL; + static const TEE_UUID uuid = PTA_SYSTEM_UUID; + + if (sess == TEE_HANDLE_NULL) { + TEE_Result res = TEE_OpenTASession(&uuid, TEE_TIMEOUT_INFINITE, + 0, NULL, &sess, NULL); + + if (res) + return res; + } + + return TEE_InvokeTACommand(sess, TEE_TIMEOUT_INFINITE, cmd_id, + param_types, params, NULL); +} + +void *tee_map_zi(size_t len, uint32_t flags) +{ + uint32_t param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INOUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param params[TEE_NUM_PARAMS] = { }; + TEE_Result res = TEE_SUCCESS; + + params[0].value.a = len; + if (params[0].value.a != len) + return NULL; + switch (flags) { + case 0: + break; + case TEE_MEMORY_ACCESS_ANY_OWNER: + params[0].value.b = PTA_SYSTEM_MAP_FLAG_SHAREABLE; + break; + default: + return NULL; + } + + res = invoke_system_pta(PTA_SYSTEM_MAP_ZI, param_types, params); + if (res) + return NULL; + + return (void *)(vaddr_t)reg_pair_to_64(params[1].value.a, + params[1].value.b); +} + +TEE_Result tee_unmap(void *buf, size_t len) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param params[TEE_NUM_PARAMS] = { }; + + params[0].value.a = len; + reg_pair_from_64((vaddr_t)buf, ¶ms[1].value.a, ¶ms[1].value.b); + + res = invoke_system_pta(PTA_SYSTEM_UNMAP, param_types, params); + if (res) + EMSG("Invoke PTA_SYSTEM_UNMAP: buf %p, len %#zx", buf, len); + + return res; +} + +TEE_Result tee_invoke_supp_plugin(const TEE_UUID *uuid, uint32_t cmd, + uint32_t sub_cmd, void *buf, size_t len, + size_t *outlen) +{ + TEE_Result res = TEE_SUCCESS; + uint32_t param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_VALUE_OUTPUT); + TEE_Param params[TEE_NUM_PARAMS] = { }; + + if (!uuid || (len && !buf) || (!len && buf)) + return TEE_ERROR_BAD_PARAMETERS; + + params[0].memref.buffer = (void *)uuid; + params[0].memref.size = sizeof(TEE_UUID); + params[1].value.a = cmd; + params[1].value.b = sub_cmd; + params[2].memref.buffer = buf; + params[2].memref.size = len; + + res = invoke_system_pta(PTA_SYSTEM_SUPP_PLUGIN_INVOKE, param_types, + params); + if (res) + EMSG("Invoke tee-supplicant's plugin failed: %#"PRIx32, res); + + if (outlen) + *outlen = params[3].value.a; + + return res; +} diff --git a/optee_os/lib/libutee/tee_tcpudp_socket.c b/optee_os/lib/libutee/tee_tcpudp_socket.c new file mode 100644 index 0000000..5117151 --- /dev/null +++ b/optee_os/lib/libutee/tee_tcpudp_socket.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2016-2017, Linaro Limited + */ + +#include +#include +#include +#include +#include <__tee_tcpsocket_defines_extensions.h> +#include + +#include "tee_socket_private.h" + +struct socket_ctx { + uint32_t handle; + uint32_t proto_error; +}; + +static TEE_Result tcp_open(TEE_iSocketHandle *ctx, void *setup, + uint32_t *proto_error) +{ + TEE_Result res; + struct socket_ctx *sock_ctx; + TEE_tcpSocket_Setup *tcp_setup = setup; + + if (!ctx || !setup || !proto_error) + TEE_Panic(0); + + *proto_error = TEE_SUCCESS; + + sock_ctx = TEE_Malloc(sizeof(*sock_ctx), TEE_MALLOC_FILL_ZERO); + if (!sock_ctx) + return TEE_ERROR_OUT_OF_MEMORY; + + res = __tee_socket_pta_open(tcp_setup->ipVersion, + tcp_setup->server_addr, + tcp_setup->server_port, + TEE_ISOCKET_PROTOCOLID_TCP, + &sock_ctx->handle); + if (res != TEE_SUCCESS) { + TEE_Free(sock_ctx); + sock_ctx = NULL; + } + *ctx = (TEE_iSocketHandle)sock_ctx; + + switch (res) { + case TEE_ISOCKET_ERROR_HOSTNAME: + *proto_error = res; + return TEE_ISOCKET_ERROR_PROTOCOL; + case TEE_ISOCKET_TCP_WARNING_UNKNOWN_OUT_OF_BAND: + *proto_error = res; + return TEE_ISOCKET_WARNING_PROTOCOL; + default: + return res; + } +} + +static TEE_Result udp_open(TEE_iSocketHandle *ctx, void *setup, + uint32_t *proto_error) +{ + TEE_Result res; + struct socket_ctx *sock_ctx; + TEE_udpSocket_Setup *udp_setup = setup; + + if (!ctx || !setup || !proto_error) + TEE_Panic(0); + + *proto_error = TEE_SUCCESS; + + sock_ctx = TEE_Malloc(sizeof(*sock_ctx), TEE_MALLOC_FILL_ZERO); + if (!sock_ctx) + return TEE_ERROR_OUT_OF_MEMORY; + + res = __tee_socket_pta_open(udp_setup->ipVersion, + udp_setup->server_addr, + udp_setup->server_port, + TEE_ISOCKET_PROTOCOLID_UDP, + &sock_ctx->handle); + if (res != TEE_SUCCESS) { + TEE_Free(sock_ctx); + sock_ctx = NULL; + } + *ctx = (TEE_iSocketHandle)sock_ctx; + + switch (res) { + case TEE_ISOCKET_ERROR_HOSTNAME: + *proto_error = res; + return TEE_ISOCKET_ERROR_PROTOCOL; + case TEE_ISOCKET_UDP_WARNING_UNKNOWN_OUT_OF_BAND: + *proto_error = res; + return TEE_ISOCKET_WARNING_PROTOCOL; + default: + return res; + } +} + +static TEE_Result sock_close(TEE_iSocketHandle ctx) +{ + TEE_Result res; + struct socket_ctx *sock_ctx = (struct socket_ctx *)ctx; + + if (ctx == TEE_HANDLE_NULL) + return TEE_SUCCESS; + + res = __tee_socket_pta_close(sock_ctx->handle); + TEE_Free(sock_ctx); + + return res; +} + +static TEE_Result sock_send(TEE_iSocketHandle ctx, const void *buf, + uint32_t *length, uint32_t timeout) +{ + TEE_Result res; + struct socket_ctx *sock_ctx = (struct socket_ctx *)ctx; + + if (ctx == TEE_HANDLE_NULL || !buf || !length) + TEE_Panic(0); + + res = __tee_socket_pta_send(sock_ctx->handle, buf, length, timeout); + sock_ctx->proto_error = res; + + return res; +} + +static TEE_Result sock_recv(TEE_iSocketHandle ctx, void *buf, uint32_t *length, + uint32_t timeout) +{ + TEE_Result res; + struct socket_ctx *sock_ctx = (struct socket_ctx *)ctx; + + if (ctx == TEE_HANDLE_NULL || !length || (!buf && *length)) + TEE_Panic(0); + + res = __tee_socket_pta_recv(sock_ctx->handle, buf, length, timeout); + sock_ctx->proto_error = res; + + return res; +} + +static uint32_t sock_error(TEE_iSocketHandle ctx) +{ + struct socket_ctx *sock_ctx = (struct socket_ctx *)ctx; + + if (ctx == TEE_HANDLE_NULL) + TEE_Panic(0); + + return sock_ctx->proto_error; +} + +static TEE_Result tcp_ioctl(TEE_iSocketHandle ctx, uint32_t commandCode, + void *buf, uint32_t *length) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct socket_ctx *sock_ctx = (struct socket_ctx *)ctx; + + if (ctx == TEE_HANDLE_NULL || !length || (!buf && *length)) + TEE_Panic(0); + + if (__tee_socket_ioctl_cmd_to_proto(commandCode) == 0) + return TEE_SUCCESS; + + switch (commandCode) { + case TEE_TCP_SET_RECVBUF: + case TEE_TCP_SET_SENDBUF: + res = __tee_socket_pta_ioctl(sock_ctx->handle, commandCode, + buf, length); + break; + default: + TEE_Panic(0); + } + + sock_ctx->proto_error = res; + + return res; +} + +static TEE_Result udp_ioctl(TEE_iSocketHandle ctx, uint32_t commandCode, + void *buf, uint32_t *length) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct socket_ctx *sock_ctx = (struct socket_ctx *)ctx; + + if (ctx == TEE_HANDLE_NULL || !length || (!buf && *length)) + TEE_Panic(0); + + if (__tee_socket_ioctl_cmd_to_proto(commandCode) == 0) + return TEE_SUCCESS; + + switch (commandCode) { + case TEE_UDP_CHANGEADDR: + case TEE_UDP_CHANGEPORT: + res = __tee_socket_pta_ioctl(sock_ctx->handle, commandCode, + buf, length); + break; + default: + TEE_Panic(0); + } + + sock_ctx->proto_error = res; + + return res; +} + + + +static TEE_iSocket tcp_socket_instance = { + .TEE_iSocketVersion = TEE_ISOCKET_VERSION, + .protocolID = TEE_ISOCKET_PROTOCOLID_TCP, + .open = tcp_open, + .close = sock_close, + .send = sock_send, + .recv = sock_recv, + .error = sock_error, + .ioctl = tcp_ioctl, +}; + +static TEE_iSocket udp_socket_instance = { + .TEE_iSocketVersion = TEE_ISOCKET_VERSION, + .protocolID = TEE_ISOCKET_PROTOCOLID_UDP, + .open = udp_open, + .close = sock_close, + .send = sock_send, + .recv = sock_recv, + .error = sock_error, + .ioctl = udp_ioctl, +}; + +TEE_iSocket *const TEE_tcpSocket = &tcp_socket_instance; +TEE_iSocket *const TEE_udpSocket = &udp_socket_instance; diff --git a/optee_os/lib/libutee/tee_uuid_from_str.c b/optee_os/lib/libutee/tee_uuid_from_str.c new file mode 100644 index 0000000..04dc564 --- /dev/null +++ b/optee_os/lib/libutee/tee_uuid_from_str.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include +#include +#include +#include +#include + +static int hex(char c) +{ + char lc = tolower(c); + + if (isdigit(lc)) + return lc - '0'; + if (isxdigit(lc)) + return lc - 'a' + 10; + return -1; +} + +static uint32_t parse_hex(const char *s, size_t nchars, uint32_t *res) +{ + uint32_t v = 0; + size_t n = 0; + int c = 0; + + for (n = 0; n < nchars; n++) { + c = hex(s[n]); + if (c == -1) { + *res = TEE_ERROR_BAD_FORMAT; + goto out; + } + v = (v << 4) + c; + } + *res = TEE_SUCCESS; +out: + return v; +} + +TEE_Result tee_uuid_from_str(TEE_UUID *uuid, const char *s) +{ + TEE_Result res = TEE_SUCCESS; + TEE_UUID u = { }; + const char *p = s; + size_t i = 0; + + if (!p || strnlen(p, 37) != 36) + return TEE_ERROR_BAD_FORMAT; + if (p[8] != '-' || p[13] != '-' || p[18] != '-' || p[23] != '-') + return TEE_ERROR_BAD_FORMAT; + + u.timeLow = parse_hex(p, 8, &res); + if (res) + goto out; + p += 9; + u.timeMid = parse_hex(p, 4, &res); + if (res) + goto out; + p += 5; + u.timeHiAndVersion = parse_hex(p, 4, &res); + if (res) + goto out; + p += 5; + for (i = 0; i < 8; i++) { + u.clockSeqAndNode[i] = parse_hex(p, 2, &res); + if (res) + goto out; + if (i == 1) + p += 3; + else + p += 2; + } + *uuid = u; +out: + return res; +} diff --git a/optee_os/lib/libutee/trace_ext.c b/optee_os/lib/libutee/trace_ext.c new file mode 100644 index 0000000..70c2c30 --- /dev/null +++ b/optee_os/lib/libutee/trace_ext.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef __LDELF__ +#include +#endif + +#if TRACE_LEVEL > 0 + +void trace_ext_puts(const char *str) +{ +#ifdef __LDELF__ + _ldelf_log(str, strlen(str)); +#else + _utee_log(str, strlen(str)); +#endif +} + +int trace_ext_get_thread_id(void) +{ + return -1; +} + +/* + * printf and puts - stdio printf support + * + * 'printf()' and 'puts()' traces have the 'info' trace level. + */ +int printf(const char *fmt, ...) +{ + char to_format[MAX_PRINT_SIZE]; + va_list ap; + int s; + + if (trace_get_level() < TRACE_PRINTF_LEVEL) + return 0; + + va_start(ap, fmt); + s = vsnprintf(to_format, sizeof(to_format), fmt, ap); + va_end(ap); + + if (s < 0) + return s; + + trace_ext_puts(to_format); + + return s; +} + +int puts(const char *str) +{ + if (trace_get_level() >= TRACE_PRINTF_LEVEL) { + trace_ext_puts(str); + trace_ext_puts("\n"); + } + return 1; +} + +int putchar(int c) +{ + char str[2] = { (char)c, '\0' }; + + if (trace_get_level() >= TRACE_PRINTF_LEVEL) + trace_ext_puts(str); + /* + * From the putchar() man page: + * "fputc(), putc() and putchar() return the character written as an + * unsigned char cast to an int or EOF on error." + */ + return (int)(unsigned char)c; +} + +#else + +void trace_ext_puts(const char *str __unused) +{ +} + +int printf(const char *fmt __unused, ...) +{ + return 0; +} + +int puts(const char *str __unused) +{ + return 0; +} + +int putchar(int c) +{ + return (int)(unsigned char)c; +} + +#endif diff --git a/optee_os/lib/libutee/user_ta_entry.c b/optee_os/lib/libutee/user_ta_entry.c new file mode 100644 index 0000000..2ac3e25 --- /dev/null +++ b/optee_os/lib/libutee/user_ta_entry.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2022, Linaro Limited. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tee_api_private.h" + +struct ta_session { + uint32_t session_id; + void *session_ctx; + TAILQ_ENTRY(ta_session) link; +}; + +static TAILQ_HEAD(ta_sessions, ta_session) ta_sessions = + TAILQ_HEAD_INITIALIZER(ta_sessions); + +static bool init_done; + +/* From user_ta_header.c, built within TA */ +extern uint8_t ta_heap[]; +extern const size_t ta_heap_size; +extern struct ta_head ta_head; + +uint32_t ta_param_types; +TEE_Param ta_params[TEE_NUM_PARAMS]; + +struct malloc_ctx *__ta_no_share_malloc_ctx; + +struct __elf_phdr_info __elf_phdr_info; + +struct phdr_info { + struct dl_phdr_info info; + TAILQ_ENTRY(phdr_info) link; +}; + +static TAILQ_HEAD(phdr_info_head, phdr_info) __phdr_info_head = + TAILQ_HEAD_INITIALIZER(__phdr_info_head); +/* + * Keep track of how many modules have been initialized so that subsequent + * dlopen() calls will not run the same initializers again + */ +static size_t _num_mod_init; + +static int _init_iterate_phdr_cb(struct dl_phdr_info *info, + size_t size __unused, void *data) +{ + struct phdr_info *qe = NULL; + size_t *count = data; + + qe = malloc(sizeof(*qe)); + if (!qe) { + EMSG("init/fini: out of memory"); + abort(); + } + qe->info = *info; + TAILQ_INSERT_TAIL(&__phdr_info_head, qe, link); + (*count)++; + return 0; +} + +static void _get_fn_array(struct dl_phdr_info *info, Elf_Sword tag_a, + Elf_Sword tag_s, void (***fn)(void), size_t *num_fn) +{ + const Elf_Phdr *phdr = NULL; + Elf_Dyn *dyn = NULL; + size_t num_dyn = 0; + size_t i = 0; + size_t j = 0; + + for (i = 0; i < info->dlpi_phnum; i++) { + phdr = info->dlpi_phdr + i; + if (phdr->p_type != PT_DYNAMIC) + continue; + num_dyn = phdr->p_memsz / sizeof(Elf_Dyn); + dyn = (Elf_Dyn *)(phdr->p_vaddr + info->dlpi_addr); + for (j = 0; j < num_dyn; j++) { + if (*fn && *num_fn) + break; + if (dyn->d_tag == DT_NULL) { + break; + } else if (dyn->d_tag == tag_a) { + *fn = (void (**)(void))(dyn->d_un.d_ptr + + info->dlpi_addr); + } else if (dyn->d_tag == tag_s) { + *num_fn = dyn->d_un.d_val / sizeof(Elf_Addr); + } + dyn++; + } + } +} + +void __utee_call_elf_init_fn(void) +{ + void (**fn)(void) = NULL; + size_t num_mod = 0; + size_t num_fn = 0; + size_t mod = 0; + size_t i = 0; + struct phdr_info *qe = NULL; + struct phdr_info *qe2 = NULL; + + dl_iterate_phdr(_init_iterate_phdr_cb, &num_mod); + + /* Reverse order: dependencies first */ + TAILQ_FOREACH_REVERSE(qe, &__phdr_info_head, phdr_info_head, link) { + if (mod == num_mod - _num_mod_init) + break; + _get_fn_array(&qe->info, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, &fn, + &num_fn); + for (i = 0; i < num_fn; i++) + fn[i](); + fn = NULL; + num_fn = 0; + mod++; + } + _num_mod_init += mod; + + TAILQ_FOREACH_SAFE(qe, &__phdr_info_head, link, qe2) { + TAILQ_REMOVE(&__phdr_info_head, qe, link); + free(qe); + } +} + +static int _fini_iterate_phdr_cb(struct dl_phdr_info *info, + size_t size __unused, void *data __unused) +{ + void (**fn)(void) = NULL; + size_t num_fn = 0; + size_t i = 0; + + _get_fn_array(info, DT_FINI_ARRAY, DT_FINI_ARRAYSZ, &fn, &num_fn); + + for (i = 1; i <= num_fn; i++) + fn[num_fn - i](); + + return 0; +} + +void __utee_call_elf_fini_fn(void) +{ + dl_iterate_phdr(_fini_iterate_phdr_cb, NULL); +} + +static unsigned int get_memtag_implementation(void) +{ + const char *s = "org.trustedfirmware.optee.cpu.feat_memtag_implemented"; + uint32_t v = 0; + + if (TEE_GetPropertyAsU32(TEE_PROPSET_TEE_IMPLEMENTATION, s, &v)) + return 0; + return v; +} + +static TEE_Result init_instance(void) +{ + trace_set_level(tahead_get_trace_level()); + __utee_gprof_init(); + malloc_add_pool(ta_heap, ta_heap_size); + if (__ta_no_share_heap_size) { + __ta_no_share_malloc_ctx = malloc(raw_malloc_get_ctx_size()); + if (__ta_no_share_malloc_ctx) { + raw_malloc_init_ctx(__ta_no_share_malloc_ctx); + raw_malloc_add_pool(__ta_no_share_malloc_ctx, + __ta_no_share_heap, + __ta_no_share_heap_size); + } + } + memtag_init_ops(get_memtag_implementation()); + _TEE_MathAPI_Init(); + __utee_tcb_init(); + __utee_call_elf_init_fn(); + return TA_CreateEntryPoint(); +} + +static void uninit_instance(void) +{ + __utee_gprof_fini(); + TA_DestroyEntryPoint(); + __utee_call_elf_fini_fn(); +} + +static void ta_header_save_params(uint32_t param_types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + ta_param_types = param_types; + + if (params) + memcpy(ta_params, params, sizeof(ta_params)); + else + memset(ta_params, 0, sizeof(ta_params)); +} + +static struct ta_session *ta_header_get_session(uint32_t session_id) +{ + struct ta_session *itr; + + TAILQ_FOREACH(itr, &ta_sessions, link) { + if (itr->session_id == session_id) + return itr; + } + return NULL; +} + +static TEE_Result ta_header_add_session(uint32_t session_id) +{ + struct ta_session *itr = ta_header_get_session(session_id); + TEE_Result res; + + if (itr) + return TEE_SUCCESS; + + if (!init_done) { + init_done = true; + res = init_instance(); + if (res) + return res; + } + + itr = TEE_Malloc(sizeof(struct ta_session), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!itr) + return TEE_ERROR_OUT_OF_MEMORY; + itr->session_id = session_id; + itr->session_ctx = 0; + TAILQ_INSERT_TAIL(&ta_sessions, itr, link); + + return TEE_SUCCESS; +} + +static void ta_header_remove_session(uint32_t session_id) +{ + struct ta_session *itr; + bool keep_alive; + + TAILQ_FOREACH(itr, &ta_sessions, link) { + if (itr->session_id == session_id) { + TAILQ_REMOVE(&ta_sessions, itr, link); + TEE_Free(itr); + + keep_alive = + (ta_head.flags & TA_FLAG_SINGLE_INSTANCE) && + (ta_head.flags & TA_FLAG_INSTANCE_KEEP_ALIVE); + if (TAILQ_EMPTY(&ta_sessions) && !keep_alive) + uninit_instance(); + + return; + } + } +} + +static void to_utee_params(struct utee_params *up, uint32_t param_types, + const TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t n = 0; + + up->types = param_types; + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(param_types, n)) { + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + up->vals[n * 2] = params[n].value.a; + up->vals[n * 2 + 1] = params[n].value.b; + break; + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + up->vals[n * 2] = (uintptr_t)params[n].memref.buffer; + up->vals[n * 2 + 1] = params[n].memref.size; + break; + default: + up->vals[n * 2] = 0; + up->vals[n * 2 + 1] = 0; + break; + } + } +} + +static void from_utee_params(TEE_Param params[TEE_NUM_PARAMS], + uint32_t *param_types, + const struct utee_params *up) +{ + size_t n; + uint32_t types = up->types; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + uintptr_t a = up->vals[n * 2]; + uintptr_t b = up->vals[n * 2 + 1]; + + switch (TEE_PARAM_TYPE_GET(types, n)) { + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + params[n].value.a = a; + params[n].value.b = b; + break; + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + params[n].memref.buffer = (void *)a; + params[n].memref.size = b; + break; + default: + break; + } + } + + if (param_types) + *param_types = types; +} + +static TEE_Result entry_open_session(unsigned long session_id, + struct utee_params *up) +{ + TEE_Result res; + struct ta_session *session; + uint32_t param_types; + TEE_Param params[TEE_NUM_PARAMS]; + + res = ta_header_add_session(session_id); + if (res != TEE_SUCCESS) + return res; + + session = ta_header_get_session(session_id); + if (!session) + return TEE_ERROR_BAD_STATE; + + from_utee_params(params, ¶m_types, up); + ta_header_save_params(param_types, params); + + res = TA_OpenSessionEntryPoint(param_types, params, + &session->session_ctx); + + to_utee_params(up, param_types, params); + + if (res != TEE_SUCCESS) + ta_header_remove_session(session_id); + return res; +} + +static TEE_Result entry_close_session(unsigned long session_id) +{ + struct ta_session *session = ta_header_get_session(session_id); + + if (!session) + return TEE_ERROR_BAD_STATE; + + TA_CloseSessionEntryPoint(session->session_ctx); + + ta_header_remove_session(session_id); + return TEE_SUCCESS; +} + +static TEE_Result entry_invoke_command(unsigned long session_id, + struct utee_params *up, unsigned long cmd_id) +{ + TEE_Result res; + uint32_t param_types; + TEE_Param params[TEE_NUM_PARAMS]; + struct ta_session *session = ta_header_get_session(session_id); + + if (!session) + return TEE_ERROR_BAD_STATE; + + from_utee_params(params, ¶m_types, up); + ta_header_save_params(param_types, params); + + res = TA_InvokeCommandEntryPoint(session->session_ctx, cmd_id, + param_types, params); + + to_utee_params(up, param_types, params); + return res; +} + +#if defined(CFG_TA_STATS) +static TEE_Result entry_dump_memstats(unsigned long session_id __unused, + struct utee_params *up) +{ + uint32_t param_types = 0; + TEE_Param params[TEE_NUM_PARAMS] = { }; + struct malloc_stats stats = { }; + + from_utee_params(params, ¶m_types, up); + ta_header_save_params(param_types, params); + + if (TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE) != param_types) + return TEE_ERROR_BAD_PARAMETERS; + + malloc_get_stats(&stats); + params[0].value.a = stats.allocated; + params[0].value.b = stats.max_allocated; + params[1].value.a = stats.size; + params[1].value.b = stats.num_alloc_fail; + params[2].value.a = stats.biggest_alloc_fail; + params[2].value.b = stats.biggest_alloc_fail_used; + to_utee_params(up, param_types, params); + + return TEE_SUCCESS; +} +#endif + +TEE_Result __utee_entry(unsigned long func, unsigned long session_id, + struct utee_params *up, unsigned long cmd_id) +{ + TEE_Result res; + + switch (func) { + case UTEE_ENTRY_FUNC_OPEN_SESSION: + res = entry_open_session(session_id, up); + break; + case UTEE_ENTRY_FUNC_CLOSE_SESSION: + res = entry_close_session(session_id); + break; + case UTEE_ENTRY_FUNC_INVOKE_COMMAND: + res = entry_invoke_command(session_id, up, cmd_id); + break; +#if defined(CFG_TA_STATS) + case UTEE_ENTRY_FUNC_DUMP_MEMSTATS: + res = entry_dump_memstats(session_id, up); + break; +#endif + default: + res = TEE_ERROR_NOT_SUPPORTED; + break; + } + ta_header_save_params(0, NULL); + + return res; +} diff --git a/optee_os/lib/libutee/user_ta_entry_compat.c b/optee_os/lib/libutee/user_ta_entry_compat.c new file mode 100644 index 0000000..f7b4798 --- /dev/null +++ b/optee_os/lib/libutee/user_ta_entry_compat.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2022, Linaro Limited. + */ +#include +#include + +static void to_gp11_param(uint32_t pt, const TEE_Param params[TEE_NUM_PARAMS], + __GP11_TEE_Param gp11_params[TEE_NUM_PARAMS]) +{ + size_t n = 0; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(pt, n)) { + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + gp11_params[n].value.a = params[n].value.a; + gp11_params[n].value.b = params[n].value.b; + break; + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + gp11_params[n].memref.buffer = params[n].memref.buffer; + gp11_params[n].memref.size = params[n].memref.size; + break; + default: + break; + } + } +} + +static void from_gp11_param(uint32_t pt, + const __GP11_TEE_Param gp11_params[TEE_NUM_PARAMS], + TEE_Param params[TEE_NUM_PARAMS]) +{ + size_t n = 0; + + for (n = 0; n < TEE_NUM_PARAMS; n++) { + switch (TEE_PARAM_TYPE_GET(pt, n)) { + case TEE_PARAM_TYPE_VALUE_INPUT: + case TEE_PARAM_TYPE_VALUE_OUTPUT: + case TEE_PARAM_TYPE_VALUE_INOUT: + params[n].value.a = gp11_params[n].value.a; + params[n].value.b = gp11_params[n].value.b; + break; + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + params[n].memref.buffer = gp11_params[n].memref.buffer; + params[n].memref.size = gp11_params[n].memref.size; + break; + default: + break; + } + } +} + +/* + * Legacy TAs will due to macros define __GP11_TA_OpenSessionEntryPoint() + * instead so call that function instead. + */ + +TEE_Result __ta_open_sess(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS], + void **sess_ctx, + TEE_Result (*fp)(uint32_t, + __GP11_TEE_Param [TEE_NUM_PARAMS], + void **)) +{ + __GP11_TEE_Param gp11_params[TEE_NUM_PARAMS] = { }; + TEE_Result res = TEE_SUCCESS; + + to_gp11_param(pt, params, gp11_params); + res = fp(pt, gp11_params, sess_ctx); + from_gp11_param(pt, gp11_params, params); + + return res; +} + +/* + * Legacy TAs will due to macros define __GP11_TA_InvokeCommandEntryPoint() + * instead so call that function instead. + */ + +TEE_Result __ta_invoke_cmd(void *sess_ctx, uint32_t cmd_id, uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS], + TEE_Result (*fp)(void *, uint32_t, uint32_t, + __GP11_TEE_Param [TEE_NUM_PARAMS])) +{ + __GP11_TEE_Param gp11_params[TEE_NUM_PARAMS] = { }; + TEE_Result res = TEE_SUCCESS; + + to_gp11_param(pt, params, gp11_params); + res = fp(sess_ctx, cmd_id, pt, gp11_params); + from_gp11_param(pt, gp11_params, params); + + return res; +} diff --git a/optee_os/lib/libutils/ext/arch/arm/aeabi_unwind.c b/optee_os/lib/libutils/ext/arch/arm/aeabi_unwind.c new file mode 100644 index 0000000..332ee45 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/aeabi_unwind.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017, Linaro Limited + */ + +void __aeabi_unwind_cpp_pr0(void); +void __aeabi_unwind_cpp_pr0(void) +{ +} + +void __aeabi_unwind_cpp_pr1(void); +void __aeabi_unwind_cpp_pr1(void) +{ +} + +void __aeabi_unwind_cpp_pr2(void); +void __aeabi_unwind_cpp_pr2(void) +{ +} + diff --git a/optee_os/lib/libutils/ext/arch/arm/atomic_a32.S b/optee_os/lib/libutils/ext/arch/arm/atomic_a32.S new file mode 100644 index 0000000..2be73ff --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/atomic_a32.S @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2016, Linaro Limited + */ + +#include + + .section .note.GNU-stack,"",%progbits + +/* uint32_t atomic_inc32(uint32_t *v); */ +FUNC atomic_inc32 , : + ldrex r1, [r0] + add r1, r1, #1 + strex r2, r1, [r0] + cmp r2, #0 + bne atomic_inc32 + mov r0, r1 + bx lr +END_FUNC atomic_inc32 + +/* uint32_t atomic_dec32(uint32_t *v); */ +FUNC atomic_dec32 , : + ldrex r1, [r0] + sub r1, r1, #1 + strex r2, r1, [r0] + cmp r2, #0 + bne atomic_dec32 + mov r0, r1 + bx lr +END_FUNC atomic_dec32 diff --git a/optee_os/lib/libutils/ext/arch/arm/atomic_a64.S b/optee_os/lib/libutils/ext/arch/arm/atomic_a64.S new file mode 100644 index 0000000..0b6ab8c --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/atomic_a64.S @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015-2016, Linaro Limited + */ + +#include + + +/* uint32_t atomic_inc32(uint32_t *v); */ +FUNC atomic_inc32 , : + ldaxr w1, [x0] + add w1, w1, #1 + stxr w2, w1, [x0] + cmp w2, #0 + bne atomic_inc32 + mov w0, w1 + ret +END_FUNC atomic_inc32 + +/* uint32_t atomic_dec32(uint32_t *v); */ +FUNC atomic_dec32 , : + ldaxr w1, [x0] + sub w1, w1, #1 + stxr w2, w1, [x0] + cmp w2, #0 + bne atomic_dec32 + mov w0, w1 + ret +END_FUNC atomic_dec32 + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/lib/libutils/ext/arch/arm/auxval.c b/optee_os/lib/libutils/ext/arch/arm/auxval.c new file mode 100644 index 0000000..98bca85 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/auxval.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, EPAM Systems + */ + +#include + +unsigned long int __getauxval (unsigned long int type); +unsigned long int __getauxval (unsigned long int type __unused) +{ + return 0; +} diff --git a/optee_os/lib/libutils/ext/arch/arm/mcount_a32.S b/optee_os/lib/libutils/ext/arch/arm/mcount_a32.S new file mode 100644 index 0000000..54dc3c0 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/mcount_a32.S @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include + +#if defined(CFG_TA_GPROF_SUPPORT) || defined(CFG_FTRACE_SUPPORT) + + .section .note.GNU-stack,"",%progbits + +/* + * Convert return address to call site address by subtracting the size of the + * mcount call instruction (blx __gnu_mcount_nc). + */ +.macro mcount_adj_pc rd, rn + bic \rd, \rn, #1 /* Clear thumb bit if present */ + sub \rd, \rd, #4 +.endm + +/* + * With the -pg option, GCC (4.4+) inserts a call to __gnu_mcount_nc into + * every function prologue. + * The caller of the instrumented function can be determined from the lr value + * stored on the top of the stack. The callee, i.e. the instrumented function + * itself, is determined from the current value of lr. Then we call: + * void __mcount_internal(void *frompc, void *selfpc); + */ +FUNC __gnu_mcount_nc, : +UNWIND( .cantunwind) + stmdb sp!, {r0-r3, lr} +#if defined(CFG_TA_GPROF_SUPPORT) && !defined(__KERNEL__) + ldr r0, [sp, #20] /* lr of instrumented func */ + mcount_adj_pc r0, r0 + mcount_adj_pc r1, lr /* instrumented func */ + bl __mcount_internal +#endif +#ifdef CFG_FTRACE_SUPPORT + /* Get instrumented function's pc value */ + ldr r0, [sp, #16] + mcount_adj_pc r0, r0 + /* Get instrumented function's lr address pointer */ + sub r1, fp, #4 + bl ftrace_enter +#endif + ldmia sp!, {r0-r3, ip, lr} + bx ip +END_FUNC __gnu_mcount_nc + +#ifdef CFG_FTRACE_SUPPORT +FUNC __ftrace_return, : + /* save return value regs */ + stmdb sp!, {r0-r3} + + /* get return address of parent func */ + bl ftrace_return + mov lr, r0 + + /* restore return value regs */ + ldmia sp!, {r0-r3} + bx lr +END_FUNC __ftrace_return +#endif + +#endif /* CFG_TA_GPROF_SUPPORT || CFG_FTRACE_SUPPORT */ diff --git a/optee_os/lib/libutils/ext/arch/arm/mcount_a64.S b/optee_os/lib/libutils/ext/arch/arm/mcount_a64.S new file mode 100644 index 0000000..7d94b08 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/mcount_a64.S @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016, Linaro Limited + */ + +#include + +#if defined(CFG_TA_GPROF_SUPPORT) || defined(CFG_FTRACE_SUPPORT) + +/* + * Convert return address to call site address by subtracting the size of one + * instruction. + */ +.macro adjust_pc rd, rn + sub \rd, \rn, #4 +.endm + +/* Get instrumented function's pc value */ +.macro get_pc reg + ldr \reg, [x29, #8] + sub \reg, \reg, #4 +.endm + +/* Get instrumented function's lr address pointer */ +.macro get_lr_addr reg + ldr \reg, [x29] + add \reg, \reg, #8 +.endm + +/* + * void _mcount(void *return_address) + * @return_address: return address to instrumented function + * + * With the -pg option, the compiler inserts a call to _mcount into + * every function prologue. + * x0 contains the value of lr (x30) before the call, that is the return + * address to the caller of the instrumented function. The callee, i.e. the + * instrumented function itself, is determined from the current value of x30. + * Then we call: + * void __mcount_internal(void *frompc, void *selfpc); + */ +FUNC _mcount, : + stp x29, x30, [sp, #-16]! + mov x29, sp +#if defined(CFG_TA_GPROF_SUPPORT) && !defined(__KERNEL__) + adjust_pc x0, x0 + adjust_pc x1, x30 + bl __mcount_internal +#endif +#ifdef CFG_FTRACE_SUPPORT + get_pc x0 + get_lr_addr x1 + bl ftrace_enter +#endif + ldp x29, x30, [sp], #16 + ret +END_FUNC _mcount + +#ifdef CFG_FTRACE_SUPPORT +FUNC __ftrace_return, : + /* Save return value regs */ + sub sp, sp, #64 + stp x0, x1, [sp] + stp x2, x3, [sp, #16] + stp x4, x5, [sp, #32] + stp x6, x7, [sp, #48] + + /* Get return address of parent func */ + bl ftrace_return + mov x30, x0 + + /* Restore return value regs */ + ldp x0, x1, [sp] + ldp x2, x3, [sp, #16] + ldp x4, x5, [sp, #32] + ldp x6, x7, [sp, #48] + add sp, sp, #64 + + ret +END_FUNC __ftrace_return +#endif + +#endif /* CFG_TA_GPROF_SUPPORT || CFG_FTRACE_SUPPORT */ + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/lib/libutils/ext/arch/arm/memtag.c b/optee_os/lib/libutils/ext/arch/arm/memtag.c new file mode 100644 index 0000000..8249599 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/memtag.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#if MEMTAG_IS_ENABLED + +/* This is for AArch64 only, MTE is only available in this mode */ + +static unsigned int dczid_block_size(void) +{ + return SHIFT_U32(4U, read_dczid_el0() & 0xf); +} + +static bool data_zero_prohibited(void) +{ +#ifdef __KERNEL__ + return false; +#else + return read_dczid_el0() & BIT(4); +#endif +} + +static void dc_gzva(uint64_t va) +{ + asm volatile ("dc gzva, %0" : : "r" (va)); +} + +static void dc_gva(uint64_t va) +{ + asm volatile ("dc gva, %0" : : "r" (va)); +} + +static vaddr_t stg_and_advance(vaddr_t va) +{ + asm volatile("stg %0, [%0], #16" : "+r"(va) : : "memory"); + return va; +} + +static void *insert_random_tag(void *addr) +{ + asm volatile("irg %0, %0" : "+r"(addr) : : ); + return addr; +} + +static void *load_tag(void *addr) +{ + asm volatile("ldg %0, [%0]" : "+r"(addr) : : ); + return addr; +} + +static void set_tags_dc_gva(vaddr_t va, size_t size, size_t dcsz) +{ + do { + dc_gva(va); + va += dcsz; + size -= dcsz; + } while (size); +} + +static void clear_mem_dc_gzva(vaddr_t va, size_t size, size_t dcsz) +{ + do { + dc_gzva(va); + va += dcsz; + size -= dcsz; + } while (size); +} + +static void *set_tags_helper(void *addr, size_t size) +{ + vaddr_t va = (vaddr_t)addr; + vaddr_t end = va + size; + + assert(!(va & MEMTAG_GRANULE_MASK)); + assert(!(size & MEMTAG_GRANULE_MASK)); + + while (va < end) + va = stg_and_advance(va); + + return addr; +} + +static void *set_tags_dc_helper(void *addr, size_t size) +{ + size_t dcsz = dczid_block_size(); + vaddr_t va = (vaddr_t)addr; + size_t mask = dcsz - 1; + size_t s = 0; + + if (va & mask) { + s = MIN(dcsz - (va & mask), size); + set_tags_helper((void *)va, s); + va += s; + size -= s; + } + s = size & ~mask; + if (s) { + set_tags_dc_gva(va, s, dcsz); + va += s; + size -= s; + } + if (size) + set_tags_helper((void *)va, size); + + return addr; +} + +static void *set_tags_dc(void *addr, size_t size, uint8_t tag) +{ + return set_tags_dc_helper(memtag_insert_tag(addr, tag), size); +} + +static void *set_random_tags_dc(void *addr, size_t size) +{ + return set_tags_dc_helper(insert_random_tag(addr), size); +} + +static void *set_tags(void *addr, size_t size, uint8_t tag) +{ + return set_tags_helper(memtag_insert_tag(addr, tag), size); +} + +static void *set_random_tags(void *addr, size_t size) +{ + return set_tags_helper(insert_random_tag(addr), size); +} + +static void clear_mem(void *va, size_t size) +{ + set_tags(va, size, 0); + memset(memtag_strip_tag(va), 0, size); +} + +static void clear_mem_dc(void *addr, size_t size) +{ + size_t dcsz = dczid_block_size(); + vaddr_t va = (vaddr_t)addr; + size_t mask = dcsz - 1; + size_t s = 0; + + if (va & mask) { + s = MIN(dcsz - (va & mask), size); + clear_mem((void *)va, s); + va += s; + size -= s; + } + s = size & ~mask; + if (s) { + clear_mem_dc_gzva(va, s, dcsz); + va += s; + size -= s; + } + if (size) + clear_mem((void *)va, size); +} + +static uint8_t read_tag(const void *addr) +{ + return memtag_get_tag(load_tag((void *)addr)); +} + +static const struct __memtag_ops ops_dc_enabled = { + .set_tags = set_tags_dc, + .set_random_tags = set_random_tags_dc, + .clear_mem = clear_mem_dc, + .read_tag = read_tag, +}; + +static const struct __memtag_ops ops_enabled = { + .set_tags = set_tags, + .set_random_tags = set_random_tags, + .clear_mem = clear_mem, + .read_tag = read_tag, +}; + +const struct __memtag_ops __memtag_ops_disabled = { + .set_tags = __memtag_disabled_set_tags, + .set_random_tags = __memtag_disabled_set_random_tags, + .clear_mem = __memtag_disabled_clear_mem, + .read_tag = __memtag_disabled_read_tag, +}; + +const struct __memtag_ops *__memtag_ops = &__memtag_ops_disabled; + +void memtag_init_ops(unsigned int memtag_impl) +{ + if (memtag_impl >= 2) { + /* + * Data zero is always available for S-EL1 if MTE is + * enabled, but for S-EL0 it may depend on configuration. + */ + if (data_zero_prohibited()) + __memtag_ops = &ops_enabled; + else + __memtag_ops = &ops_dc_enabled; + } else { + __memtag_ops = &__memtag_ops_disabled; + } +} +#endif diff --git a/optee_os/lib/libutils/ext/arch/arm/sub.mk b/optee_os/lib/libutils/ext/arch/arm/sub.mk new file mode 100644 index 0000000..6726bb0 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/arm/sub.mk @@ -0,0 +1,11 @@ +ifeq ($(CFG_UNWIND),y) +srcs-$(CFG_ARM32_$(sm)) += aeabi_unwind.c +endif +srcs-$(CFG_ARM32_$(sm)) += atomic_a32.S +srcs-$(CFG_ARM64_$(sm)) += atomic_a64.S +srcs-y += auxval.c +ifneq ($(sm),ldelf) # TA, core +srcs-$(CFG_ARM32_$(sm)) += mcount_a32.S +srcs-$(CFG_ARM64_$(sm)) += mcount_a64.S +endif +srcs-$(CFG_MEMTAG) += memtag.c diff --git a/optee_os/lib/libutils/ext/arch/riscv/atomic_rv.S b/optee_os/lib/libutils/ext/arch/riscv/atomic_rv.S new file mode 100644 index 0000000..cc63aee --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/riscv/atomic_rv.S @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright 2022-2023 NXP + */ + +#include + +/* uint32_t atomic_inc32(uint32_t *v); */ +FUNC atomic_inc32 , : + li a1, 1 + amoadd.w.aqrl a2, a1, (a0) + add a0, a1, a2 + ret +END_FUNC atomic_inc32 + +/* uint32_t atomic_dec32(uint32_t *v); */ +FUNC atomic_dec32 , : + li a1, -1 + amoadd.w.aqrl a2, a1, (a0) + add a0, a1, a2 + ret +END_FUNC atomic_dec32 diff --git a/optee_os/lib/libutils/ext/arch/riscv/mcount_rv.S b/optee_os/lib/libutils/ext/arch/riscv/mcount_rv.S new file mode 100644 index 0000000..a9f8db2 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/riscv/mcount_rv.S @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright (c) 2016, Linaro Limited + */ + +#include + +#if defined(CFG_FTRACE_SUPPORT) + +/* + * Convert return address to call site address by subtracting the size of one + * instruction. + */ +.macro adjust_pc rd, rn + addi \rd, \rn, -4 +.endm + +#ifdef RV32 + +/* Get instrumented function's pc value */ +.macro get_pc reg + LDR \reg, REGOFF(3)(sp) + addi \reg, \reg, -4 +.endm + +/* Get instrumented function's ra address pointer */ +.macro get_ra_addr reg + LDR \reg, REGOFF(2)(sp) + addi \reg, \reg, -4 +.endm + +#else /* RV64 */ + +/* Get instrumented function's pc value */ +.macro get_pc reg + LDR \reg, REGOFF(1)(sp) + addi \reg, \reg, -4 +.endm + +/* Get instrumented function's ra address pointer */ +.macro get_ra_addr reg + LDR \reg, REGOFF(0)(sp) + addi \reg, \reg, -8 +.endm + +#endif /* RV32 */ + +/* + * void _mcount(void *return_address) + * @return_address: return address to instrumented function + * + * With the -pg option, the compiler inserts a call to _mcount into + * every function prologue. + * a0 contains the value of ra before the call, that is the return + * address to the caller of the instrumented function. The callee, i.e. the + * instrumented function itself, is determined from the current value of ra. + * Then we call: + * void __mcount_internal(void *frompc, void *selfpc); + */ +FUNC _mcount, : + addi sp, sp, -16 + /* Save ra and s0(fp) onto stack */ +#ifdef RV32 + STR ra, REGOFF(3)(sp) + STR s0, REGOFF(2)(sp) +#else + STR ra, REGOFF(1)(sp) + STR s0, REGOFF(0)(sp) +#endif + /* Setup frame pointer */ + addi s0, sp, 16 +#ifdef CFG_FTRACE_SUPPORT + get_pc a0 + get_ra_addr a1 + call ftrace_enter +#endif + /* Restore ra and s0(fp) from stack */ +#ifdef RV32 + LDR s0, REGOFF(2)(sp) + LDR ra, REGOFF(3)(sp) +#else + LDR s0, REGOFF(0)(sp) + LDR ra, REGOFF(1)(sp) +#endif + addi sp, sp, 16 + ret +END_FUNC _mcount + +#ifdef CFG_FTRACE_SUPPORT +FUNC __ftrace_return, : + /* Save return value regs */ + addi sp, sp, -REGOFF(8) + STR a0, REGOFF(0)(sp) + STR a1, REGOFF(1)(sp) + STR a2, REGOFF(2)(sp) + STR a3, REGOFF(3)(sp) + STR a4, REGOFF(4)(sp) + STR a5, REGOFF(5)(sp) + STR a6, REGOFF(6)(sp) + STR a7, REGOFF(7)(sp) + + /* Get return address of parent func */ + call ftrace_return + mv ra, a0 + + /* Restore return value regs */ + LDR a0, REGOFF(0)(sp) + LDR a1, REGOFF(1)(sp) + LDR a2, REGOFF(2)(sp) + LDR a3, REGOFF(3)(sp) + LDR a4, REGOFF(4)(sp) + LDR a5, REGOFF(5)(sp) + LDR a6, REGOFF(6)(sp) + LDR a7, REGOFF(7)(sp) + addi sp, sp, REGOFF(8) + + ret +END_FUNC __ftrace_return +#endif + +#endif /* CFG_FTRACE_SUPPORT */ diff --git a/optee_os/lib/libutils/ext/arch/riscv/sub.mk b/optee_os/lib/libutils/ext/arch/riscv/sub.mk new file mode 100644 index 0000000..01a1d58 --- /dev/null +++ b/optee_os/lib/libutils/ext/arch/riscv/sub.mk @@ -0,0 +1,4 @@ +srcs-y+= atomic_rv.S +ifneq ($(sm),ldelf) # TA, core +srcs-y += mcount_rv.S +endif diff --git a/optee_os/lib/libutils/ext/consttime_memcmp.c b/optee_os/lib/libutils/ext/consttime_memcmp.c new file mode 100644 index 0000000..d0bba7e --- /dev/null +++ b/optee_os/lib/libutils/ext/consttime_memcmp.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2019 Linaro Limited + * Copyright (C) 2004-2007, 2011, 2012 Internet Systems Consortium, Inc. ("ISC") + * Copyright (C) 1999-2001, 2003 Internet Software Consortium. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + +/* Making a portable memcmp that has no internal branches and loops always + * once for every byte without early-out shortcut has a few challenges. + * + * Inspired by 'timingsafe_memcmp()' from the BSD system and + * https://github.com/libressl-portable/openbsd/blob/master/src/lib/libc/string/timingsafe_memcmp.c + * + * Sadly, that one is not portable C: It makes assumptions on the representation + * of negative integers and assumes sign-preserving right-shift of negative + * signed values. This is a rewrite from scratch that should not suffer from + * such issues. + * + * 2015-12-12, J. Perlinger (perlinger-at-ntp-dot-org) + */ +int consttime_memcmp(const void *p1, const void *p2, size_t nb) { + const unsigned char *ucp1 = p1; + const unsigned char *ucp2 = p2; + unsigned int isLT = 0u; + unsigned int isGT = 0u; + volatile unsigned int mask = (1u << CHAR_BIT); + + for (/*NOP*/; 0 != nb; --nb, ++ucp1, ++ucp2) { + isLT |= mask & + ((unsigned int)*ucp1 - (unsigned int)*ucp2); + isGT |= mask & + ((unsigned int)*ucp2 - (unsigned int)*ucp1); + mask &= ~(isLT | isGT); + } + return (int)(isGT >> CHAR_BIT) - (int)(isLT >> CHAR_BIT); +} diff --git a/optee_os/lib/libutils/ext/fault_mitigation.c b/optee_os/lib/libutils/ext/fault_mitigation.c new file mode 100644 index 0000000..9bffb4e --- /dev/null +++ b/optee_os/lib/libutils/ext/fault_mitigation.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, Linaro Limited + */ + +#include +#include + +#ifndef __KERNEL__ +struct ftmn_func_arg *__ftmn_global_func_arg; +#endif + +/* + * These functions can be implemented in assembly if needed. They would + * provide the same API but an implementation more resilient to fault + * injections. + * + * For now there is no need since it's enough with the single redundancy + * provided just by having these function implemented separately from where + * they are used. + */ + +unsigned long __weak ___ftmn_return_res(struct ftmn_check *check, + unsigned long steps, unsigned long res) +{ + if (check->steps != steps) + FTMN_PANIC(); + if ((check->res ^ FTMN_DEFAULT_HASH) != res) + FTMN_PANIC(); + return res; +} + +void __weak ___ftmn_expect_state(struct ftmn_check *check, enum ftmn_incr incr, + unsigned long steps, unsigned long res) +{ + if ((check->res ^ FTMN_DEFAULT_HASH) != res) + FTMN_PANIC(); + if (check->steps != steps) + FTMN_PANIC(); + check->steps += incr; +} + +void __weak ___ftmn_callee_done(struct ftmn_func_arg *arg, + unsigned long my_hash, + unsigned long res) +{ + arg->hash ^= my_hash; + arg->res = arg->hash ^ res; +} + +void __weak ___ftmn_callee_done_not_zero(struct ftmn_func_arg *arg, + unsigned long my_hash, + unsigned long res) +{ + if (res == 0) + FTMN_PANIC(); + arg->hash ^= my_hash; + arg->res = arg->hash ^ res; +} + +void __weak ___ftmn_callee_done_memcmp(struct ftmn_func_arg *arg, + unsigned long my_hash, int res, + ftmn_memcmp_t my_memcmp, + const void *p1, const void *p2, + size_t nb) +{ + int res2 = 0; + + if (!nb) + FTMN_PANIC(); + + res2 = my_memcmp(p1, p2, nb); + if (res2 != res) + FTMN_PANIC(); + + arg->hash ^= my_hash; + arg->res = arg->hash ^ res; +} + +void __weak ___ftmn_callee_done_check(struct ftmn_func_arg *arg, + unsigned long my_hash, + struct ftmn_check *check, + enum ftmn_incr incr, unsigned long steps, + unsigned long res) +{ + if ((check->res ^ FTMN_DEFAULT_HASH) != res) + FTMN_PANIC(); + if (check->steps != steps) + FTMN_PANIC(); + + check->steps += incr; + if (arg) { + arg->hash ^= my_hash; + arg->res = check->res ^ FTMN_DEFAULT_HASH ^ arg->hash; + } + +} + +void ___ftmn_callee_update_not_zero(struct ftmn_func_arg *arg, + unsigned long res) +{ + if (!res) + FTMN_PANIC(); + arg->res = arg->hash ^ res; +} + + +void __weak ___ftmn_copy_linked_call_res(struct ftmn_check *check, + enum ftmn_incr incr, + struct ftmn_func_arg *arg, + unsigned long res) +{ + if ((arg->res ^ arg->hash) != res) + FTMN_PANIC(); + check->res = res ^ FTMN_DEFAULT_HASH; + check->steps += incr; +} + +void __weak ___ftmn_set_check_res(struct ftmn_check *check, enum ftmn_incr incr, + unsigned long res) +{ + check->steps += incr; + check->res = res ^ FTMN_DEFAULT_HASH; +} + +void __weak ___ftmn_set_check_res_not_zero(struct ftmn_check *check, + enum ftmn_incr incr, + unsigned long res) +{ + if (!res) + FTMN_PANIC(); + check->steps += incr; + check->res = res ^ FTMN_DEFAULT_HASH; +} + +void __weak ___ftmn_set_check_res_memcmp(struct ftmn_check *check, + enum ftmn_incr incr, int res, + ftmn_memcmp_t my_memcmp, + const void *p1, const void *p2, + size_t nb) +{ + int res2 = 0; + + if (!nb) + FTMN_PANIC(); + + res2 = my_memcmp(p1, p2, nb); + if (res2 != res) + FTMN_PANIC(); + + check->steps += incr; + check->res = FTMN_DEFAULT_HASH ^ res; +} diff --git a/optee_os/lib/libutils/ext/ftrace/ftrace.c b/optee_os/lib/libutils/ext/ftrace/ftrace.c new file mode 100644 index 0000000..d7984e7 --- /dev/null +++ b/optee_os/lib/libutils/ext/ftrace/ftrace.c @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019, Linaro Limited + */ + +/* + * APIs defined in this file are required to use __noprof attribute to + * avoid any circular dependency during profiling. So this requirement + * prohibits these APIs to use standard library APIs as those can be + * profiled too. + */ + +#include +#include +#include +#if defined(__KERNEL__) +#if defined(ARM32) || defined(ARM64) +#include +#elif defined(RV32) || defined(RV64) +#include +#endif +#include +#include +#include +#include +#else +#if defined(ARM32) || defined(ARM64) +#include +#elif defined(RV32) || defined(RV64) +#include +#endif +#include +#include +#endif +#include "ftrace.h" + +static __noprof struct ftrace_buf *get_fbuf(void) +{ +#if defined(__KERNEL__) + short int ct = thread_get_id_may_fail(); + struct ts_session *s = NULL; + struct thread_specific_data *tsd = NULL; + + if (ct == -1) + return NULL; + + if (!(core_mmu_user_va_range_is_defined() && + core_mmu_user_mapping_is_active())) + return NULL; + + tsd = thread_get_tsd(); + s = TAILQ_FIRST(&tsd->sess_stack); + + if (!s || tsd->ctx != s->ctx) + return NULL; + + if (!is_ta_ctx(s->ctx) || to_ta_ctx(s->ctx)->panicked) + return NULL; + + if (s->fbuf && s->fbuf->syscall_trace_enabled && + !s->fbuf->syscall_trace_suspended) + return s->fbuf; + else + return NULL; +#else + return &__ftrace_buf_start; +#endif +} + +static void __noprof add_elem(struct ftrace_buf *fbuf, uint8_t level, + uint64_t val) +{ + uint64_t *elem = NULL; + size_t idx = fbuf->curr_idx; + + /* Make sure the topmost byte doesn't contain useful information */ + assert(!(val >> 56)); + + elem = (uint64_t *)((vaddr_t)fbuf + fbuf->buf_off) + idx; + *elem = SHIFT_U64(level, 56) | val; + + idx++; + if ((idx + 1) * sizeof(*elem) > fbuf->max_size) { + idx = 0; + fbuf->overflow = true; + } + + fbuf->curr_idx = idx; +} + +void __noprof ftrace_enter(unsigned long pc, unsigned long *lr) +{ + uint64_t now = barrier_read_counter_timer(); + struct ftrace_buf *fbuf = get_fbuf(); + + if (!fbuf || !fbuf->buf_off || !fbuf->max_size) + return; + + add_elem(fbuf, fbuf->ret_idx + 1, pc); + + if (fbuf->ret_idx < FTRACE_RETFUNC_DEPTH) { + fbuf->ret_stack[fbuf->ret_idx] = *lr; + fbuf->begin_time[fbuf->ret_idx] = now; + fbuf->ret_idx++; + } else { + /* + * This scenario isn't expected as function call depth + * shouldn't be more than FTRACE_RETFUNC_DEPTH. + */ +#if defined(__KERNEL__) + panic(); +#else + _utee_panic(0); +#endif + } + + *lr = (unsigned long)&__ftrace_return; +} + +unsigned long __noprof ftrace_return(void) +{ + uint64_t now = barrier_read_counter_timer(); + struct ftrace_buf *fbuf = get_fbuf(); + uint64_t start = 0; + uint64_t elapsed = 0; + + /* Check for valid return index */ + if (!fbuf || !fbuf->ret_idx || fbuf->ret_idx > FTRACE_RETFUNC_DEPTH) + return 0; + + fbuf->ret_idx--; + start = fbuf->begin_time[fbuf->ret_idx]; + elapsed = (now - start) * 1000000000 / read_cntfrq(); + add_elem(fbuf, 0, elapsed); + + return fbuf->ret_stack[fbuf->ret_idx]; +} + +#if !defined(__KERNEL__) +void __noprof ftrace_longjmp(unsigned int *ret_idx) +{ + while (__ftrace_buf_start.ret_idx > *ret_idx) + ftrace_return(); +} + +void __noprof ftrace_setjmp(unsigned int *ret_idx) +{ + *ret_idx = __ftrace_buf_start.ret_idx; +} +#endif diff --git a/optee_os/lib/libutils/ext/ftrace/ftrace.h b/optee_os/lib/libutils/ext/ftrace/ftrace.h new file mode 100644 index 0000000..f4dc04a --- /dev/null +++ b/optee_os/lib/libutils/ext/ftrace/ftrace.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Linaro Limited + */ + +#ifndef FTRACE_H +#define FTRACE_H + +void ftrace_enter(unsigned long pc, unsigned long *lr); + +#endif /* FTRACE_H */ diff --git a/optee_os/lib/libutils/ext/ftrace/sub.mk b/optee_os/lib/libutils/ext/ftrace/sub.mk new file mode 100644 index 0000000..a813617 --- /dev/null +++ b/optee_os/lib/libutils/ext/ftrace/sub.mk @@ -0,0 +1,5 @@ +cppflags-y += -I$(sub-dir)/../../.. + +ifneq ($(sm),ldelf) # TA, core +srcs-$(CFG_FTRACE_SUPPORT) += ftrace.c +endif diff --git a/optee_os/lib/libutils/ext/include/arm64_bti.S b/optee_os/lib/libutils/ext/include/arm64_bti.S new file mode 100644 index 0000000..63f42c7 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/arm64_bti.S @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2021, Linaro Limited + */ + +#include + +/* + * This macro emits a program property note section identifying + * architecture features which require special handling, mainly for + * use in assembly files in the libraries linked with TA's. + */ + +.macro emit_aarch64_feature_1_and, feat + .pushsection .note.gnu.property, "a" + .align 3 + .long 2f - 1f /* n_namesz */ + .long 6f - 3f /* n_desc_sz */ + .long NT_GNU_PROPERTY_TYPE_0 /* n_type */ +1: .string "GNU" /* name */ +2: + .align 3 +3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND /* pr_type */ + .long 5f - 4f /* pr_datasz */ +4: + /* + * This is described with an array of char in the Linux API + * spec but the text and all other usage (including binutils, + * clang and GCC) treat this as a 32 bit value so no swizzling + * is required for big endian. + */ + .long \feat /* property */ +5: + .align 3 +6: + .popsection +.endm diff --git a/optee_os/lib/libutils/ext/include/asm.S b/optee_os/lib/libutils/ext/include/asm.S new file mode 100644 index 0000000..ff1c4b8 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/asm.S @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2020, Linaro Limited + */ + +#if defined(__aarch64__) +#include +#elif defined(__riscv) +#include +#endif + +#if defined(__aarch64__) && ((defined(__KERNEL__) && defined(CFG_CORE_BTI)) || \ + (!defined(__KERNEL__) && defined(CFG_TA_BTI))) +#define BTI(...) __VA_ARGS__ +#else +#define BTI(...) +#endif + +#if defined(CFG_UNWIND) && defined(__arm__) +#define UNWIND(...) __VA_ARGS__ +#else +#define UNWIND(...) +#endif + + .macro FUNC name colon section=default align=4 _bti=default + .ifc \section\(),default + .section .text.\name + .else + .section \section , "ax" , %progbits + .endif + .global \name + .type \name , %function + .balign \align + \name \colon +UNWIND( .fnstart) + .ifc \_bti\(),default +BTI( bti c) + .endif + .endm + + .macro LOCAL_FUNC name colon section=default align=4 _bti=default + .ifc \section\(),default + .section .text.\name + .else + .section \section , "ax" , %progbits + .endif + .type \name , %function + .balign \align + \name \colon +UNWIND( .fnstart) + .ifc \_bti\(),default +BTI( bti c) + .endif + .endm + + .macro WEAK_FUNC name colon section=default align=4 _bti=default + .ifc \section\(),default + .section .text.\name + .else + .section \section , "ax" , %progbits + .endif + .weak \name + .type \name , %function + .balign \align + \name \colon +UNWIND( .fnstart) + .ifc \_bti\(),default +BTI( bti c) + .endif + .endm + + .macro END_FUNC name +UNWIND( .fnend) + .size \name , .-\name + .endm + + .macro DATA name colon + .global \name + .type \name , %object + \name \colon + .endm + + .macro LOCAL_DATA name colon + .type \name , %object + \name \colon + .endm + + .macro END_DATA name + .size \name , .-\name + .endm diff --git a/optee_os/lib/libutils/ext/include/atomic.h b/optee_os/lib/libutils/ext/include/atomic.h new file mode 100644 index 0000000..1c5d461 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/atomic.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2016-2019, Linaro Limited + */ + +#ifndef __ATOMIC_H +#define __ATOMIC_H + +#include +#include + +uint32_t atomic_inc32(volatile uint32_t *v); +uint32_t atomic_dec32(volatile uint32_t *v); + +static inline bool atomic_cas_uint(unsigned int *p, unsigned int *oval, + unsigned int nval) +{ + return __compiler_compare_and_swap(p, oval, nval); +} + +static inline bool atomic_cas_u32(uint32_t *p, uint32_t *oval, uint32_t nval) +{ + return __compiler_compare_and_swap(p, oval, nval); +} + +static inline int atomic_load_int(int *p) +{ + return __compiler_atomic_load(p); +} + +static inline short int atomic_load_short(short int *p) +{ + return __compiler_atomic_load(p); +} + +static inline unsigned int atomic_load_uint(unsigned int *p) +{ + return __compiler_atomic_load(p); +} + +static inline uint32_t atomic_load_u32(const uint32_t *p) +{ + return __compiler_atomic_load(p); +} + +static inline void atomic_store_int(int *p, int val) +{ + __compiler_atomic_store(p, val); +} + +static inline void atomic_store_short(short int *p, short int val) +{ + __compiler_atomic_store(p, val); +} + +static inline void atomic_store_uint(unsigned int *p, unsigned int val) +{ + __compiler_atomic_store(p, val); +} + +static inline void atomic_store_u32(uint32_t *p, uint32_t val) +{ + __compiler_atomic_store(p, val); +} + +#endif /*__ATOMIC_H*/ diff --git a/optee_os/lib/libutils/ext/include/bitstring.h b/optee_os/lib/libutils/ext/include/bitstring.h new file mode 100644 index 0000000..bebf092 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/bitstring.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/*- + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Paul Vixie. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_BITSTRING_H_ +#define _SYS_BITSTRING_H_ + +typedef unsigned char bitstr_t; + +/* internal macros */ + /* byte of the bitstring bit is in */ +#define _bit_byte(bit) \ + ((bit) >> 3) + + /* mask for the bit within its byte */ +#define _bit_mask(bit) \ + (1 << ((bit)&0x7)) + +/* external macros */ + /* bytes in a bitstring of nbits bits */ +#define bitstr_size(nbits) \ + (((nbits) + 7) >> 3) + + /* allocate a bitstring */ +#define bit_alloc(nbits) \ + (bitstr_t *)calloc((size_t)bitstr_size(nbits), sizeof(bitstr_t)) + + /* allocate a bitstring on the stack */ +#define bit_decl(name, nbits) \ + ((name)[bitstr_size(nbits)]) + + /* is bit N of bitstring name set? */ +#define bit_test(name, bit) \ + ((name)[_bit_byte(bit)] & _bit_mask(bit)) + + /* set bit N of bitstring name */ +#define bit_set(name, bit) \ + ((name)[_bit_byte(bit)] |= _bit_mask(bit)) + + /* clear bit N of bitstring name */ +#define bit_clear(name, bit) \ + ((name)[_bit_byte(bit)] &= ~_bit_mask(bit)) + + /* clear bits start ... stop in bitstring */ +#define bit_nclear(name, start, stop) do { \ + register bitstr_t *_name = (name); \ + register int _start = (start), _stop = (stop); \ + register int _startbyte = _bit_byte(_start); \ + register int _stopbyte = _bit_byte(_stop); \ + if (_startbyte == _stopbyte) { \ + _name[_startbyte] &= ((0xff >> (8 - (_start&0x7))) | \ + (0xff << ((_stop&0x7) + 1))); \ + } else { \ + _name[_startbyte] &= 0xff >> (8 - (_start&0x7)); \ + while (++_startbyte < _stopbyte) \ + _name[_startbyte] = 0; \ + _name[_stopbyte] &= 0xff << ((_stop&0x7) + 1); \ + } \ +} while (0) + + /* set bits start ... stop in bitstring */ +#define bit_nset(name, start, stop) do { \ + register bitstr_t *_name = (name); \ + register int _start = (start), _stop = (stop); \ + register int _startbyte = _bit_byte(_start); \ + register int _stopbyte = _bit_byte(_stop); \ + if (_startbyte == _stopbyte) { \ + _name[_startbyte] |= ((0xff << (_start&0x7)) & \ + (0xff >> (7 - (_stop&0x7)))); \ + } else { \ + _name[_startbyte] |= 0xff << ((_start)&0x7); \ + while (++_startbyte < _stopbyte) \ + _name[_startbyte] = 0xff; \ + _name[_stopbyte] |= 0xff >> (7 - (_stop&0x7)); \ + } \ +} while (0) + + /* find first bit clear in name */ +#define bit_ffc(name, nbits, value) do { \ + register bitstr_t *_name = (name); \ + register int _byte, _nbits = (nbits); \ + register int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ + if (_nbits > 0) \ + for (_byte = 0; _byte <= _stopbyte; ++_byte) \ + if (_name[_byte] != 0xff) { \ + bitstr_t _lb; \ + _value = _byte << 3; \ + for (_lb = _name[_byte]; (_lb&0x1); \ + ++_value, _lb >>= 1); \ + break; \ + } \ + if (_value >= nbits) \ + _value = -1; \ + *(value) = _value; \ +} while (0) + + /* find first bit set in name */ +#define bit_ffs_from(name, nbits, startbit, value) do { \ + bitstr_t *_name = (name); \ + int _byte = 0; \ + int _nbits = (nbits); \ + int _startbyte = _bit_byte(startbit); \ + int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ + bitstr_t _test_bit_mask = 0xff << ((startbit) % 8); \ + \ + if (_nbits > 0) { \ + for (_byte = _startbyte; _byte <= _stopbyte; ++_byte) { \ + if (_name[_byte] & _test_bit_mask) { \ + bitstr_t _lb = 0; \ + \ + _value = _byte << 3; \ + for (_lb = _name[_byte] & _test_bit_mask; \ + !(_lb&0x1); ++_value, _lb >>= 1) \ + ; \ + break; \ + } \ + _test_bit_mask = 0xff; \ + } \ + } \ + if (_value >= nbits) \ + _value = -1; \ + *(value) = _value; \ +} while (0) + +#define bit_ffs(name, nbits, value) \ + bit_ffs_from((name), (nbits), 0, (value)) + +#endif /* !_SYS_BITSTRING_H_ */ diff --git a/optee_os/lib/libutils/ext/include/compiler.h b/optee_os/lib/libutils/ext/include/compiler.h new file mode 100644 index 0000000..47e8838 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/compiler.h @@ -0,0 +1,287 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#ifndef COMPILER_H +#define COMPILER_H + +/* + * Macros that should be used instead of using __attribute__ directly to + * ease portability and make the code easier to read. + * + * Some of the defines below is known to sometimes cause conflicts when + * this file is included from xtest in normal world. It is assumed that + * the conflicting defines has the same meaning in that environment. + * Surrounding the troublesome defines with #ifndef should be enough. + */ +#define __deprecated __attribute__((deprecated)) +#ifndef __packed +#define __packed __attribute__((packed)) +#endif +#define __weak __attribute__((weak)) +#define __alias(x) __attribute__((alias(x))) +#ifndef __noreturn +#define __noreturn __attribute__((__noreturn__)) +#endif +#define __pure __attribute__((pure)) +#define __aligned(x) __attribute__((aligned(x))) +#define __printf(a, b) __attribute__((format(printf, a, b))) +#define __noinline __attribute__((noinline)) +#define __attr_const __attribute__((__const__)) +#ifndef __unused +#define __unused __attribute__((unused)) +#endif +#define __maybe_unused __attribute__((unused)) +#ifndef __used +#define __used __attribute__((__used__)) +#endif +#define __must_check __attribute__((warn_unused_result)) +#define __cold __attribute__((__cold__)) +#define __section(x) __attribute__((section(x))) +#define __data __section(".data") +#define __bss __section(".bss") +#ifdef __clang__ +#define __SECTION_FLAGS_RODATA +#else +/* + * Override sections flags/type generated by the C compiler to make sure they + * are: "a",%progbits (thus creating an allocatable, non-writeable, non- + * executable data section). + * The trailing COMMENT_CHAR comments out the flags generated by the compiler. + * This avoids a harmless warning with GCC. + */ +#if defined(__aarch64__) || defined(__arm__) +#define COMMENT_CHAR "//" +#else +#define COMMENT_CHAR "#" +#endif +#define __SECTION_FLAGS_RODATA ",\"a\",%progbits " COMMENT_CHAR +#endif +#define __rodata __section(".rodata" __SECTION_FLAGS_RODATA) +#define __rodata_dummy __section(".rodata.dummy" __SECTION_FLAGS_RODATA) +#define __rodata_unpaged(x) \ + __section(".rodata.__unpaged." x __SECTION_FLAGS_RODATA) +#ifdef CFG_CORE_ASLR +#define __relrodata_unpaged(x) __section(".data.rel.ro.__unpaged." x) +#else +#define __relrodata_unpaged(x) __rodata_unpaged(x) +#endif +#ifdef CFG_NS_VIRTUALIZATION +#define __nex_bss __section(".nex_bss") +#define __nex_data __section(".nex_data") +#else /* CFG_NS_VIRTUALIZATION */ +#define __nex_bss +#define __nex_data +#endif /* CFG_NS_VIRTUALIZATION */ +#define __noprof __attribute__((no_instrument_function)) +#define __nostackcheck __attribute__((no_instrument_function)) + +#define __compiler_bswap64(x) __builtin_bswap64((x)) +#define __compiler_bswap32(x) __builtin_bswap32((x)) +#define __compiler_bswap16(x) __builtin_bswap16((x)) + +#define __GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + \ + __GNUC_PATCHLEVEL__) + +#if __GCC_VERSION >= 50100 && !defined(__CHECKER__) +#define __HAVE_BUILTIN_OVERFLOW 1 +#endif + +#if __GCC_VERSION >= 90100 && !defined(__CHECKER__) +#define __HAVE_SINGLE_ARGUMENT_STATIC_ASSERT 1 +#endif + +#ifdef __HAVE_BUILTIN_OVERFLOW +#define __compiler_add_overflow(a, b, res) \ + __builtin_add_overflow((a), (b), (res)) + +#define __compiler_sub_overflow(a, b, res) \ + __builtin_sub_overflow((a), (b), (res)) + +#define __compiler_mul_overflow(a, b, res) \ + __builtin_mul_overflow((a), (b), (res)) +#else /*!__HAVE_BUILTIN_OVERFLOW*/ + +/* + * Copied/inspired from https://www.fefe.de/intof.html + */ + +#define __INTOF_ASSIGN(dest, src) (__extension__({ \ + typeof(src) __intof_x = (src); \ + typeof(dest) __intof_y = __intof_x; \ + (((uintmax_t)__intof_x == (uintmax_t)__intof_y) && \ + ((__intof_x < 1) == (__intof_y < 1)) ? \ + (void)((dest) = __intof_y) , 0 : 1); \ +})) + +#define __INTOF_ADD(c, a, b) (__extension__({ \ + typeof(a) __intofa_a = (a); \ + typeof(b) __intofa_b = (b); \ + intmax_t __intofa_a_signed = __intofa_a; \ + uintmax_t __intofa_a_unsigned = __intofa_a; \ + intmax_t __intofa_b_signed = __intofa_b; \ + uintmax_t __intofa_b_unsigned = __intofa_b; \ + \ + __intofa_b < 1 ? \ + __intofa_a < 1 ? \ + ((INTMAX_MIN - __intofa_b_signed <= \ + __intofa_a_signed)) ? \ + __INTOF_ASSIGN((c), __intofa_a_signed + \ + __intofa_b_signed) : 1 \ + : \ + ((__intofa_a_unsigned >= (uintmax_t)-__intofa_b) ? \ + __INTOF_ASSIGN((c), __intofa_a_unsigned + \ + __intofa_b_signed) \ + : \ + __INTOF_ASSIGN((c), \ + (intmax_t)(__intofa_a_unsigned + \ + __intofa_b_signed))) \ + : \ + __intofa_a < 1 ? \ + ((__intofa_b_unsigned >= (uintmax_t)-__intofa_a) ? \ + __INTOF_ASSIGN((c), __intofa_a_signed + \ + __intofa_b_unsigned) \ + : \ + __INTOF_ASSIGN((c), \ + (intmax_t)(__intofa_a_signed + \ + __intofa_b_unsigned))) \ + : \ + ((UINTMAX_MAX - __intofa_b_unsigned >= \ + __intofa_a_unsigned) ? \ + __INTOF_ASSIGN((c), __intofa_a_unsigned + \ + __intofa_b_unsigned) : 1); \ +})) + +#define __INTOF_SUB(c, a, b) (__extension__({ \ + typeof(a) __intofs_a = a; \ + typeof(b) __intofs_b = b; \ + intmax_t __intofs_a_signed = __intofs_a; \ + uintmax_t __intofs_a_unsigned = __intofs_a; \ + intmax_t __intofs_b_signed = __intofs_b; \ + uintmax_t __intofs_b_unsigned = __intofs_b; \ + \ + __intofs_b < 1 ? \ + __intofs_a < 1 ? \ + ((INTMAX_MAX + __intofs_b_signed >= \ + __intofs_a_signed) ? \ + __INTOF_ASSIGN((c), __intofs_a_signed - \ + __intofs_b_signed) : 1) \ + : \ + (((uintmax_t)(UINTMAX_MAX + __intofs_b_signed) >= \ + __intofs_a_unsigned) ? \ + __INTOF_ASSIGN((c), __intofs_a - \ + __intofs_b) : 1) \ + : \ + __intofs_a < 1 ? \ + (((intmax_t)(INTMAX_MIN + __intofs_b) <= \ + __intofs_a_signed) ? \ + __INTOF_ASSIGN((c), \ + (intmax_t)(__intofs_a_signed - \ + __intofs_b_unsigned)) : 1) \ + : \ + ((__intofs_b_unsigned <= __intofs_a_unsigned) ? \ + __INTOF_ASSIGN((c), __intofs_a_unsigned - \ + __intofs_b_unsigned) \ + : \ + __INTOF_ASSIGN((c), \ + (intmax_t)(__intofs_a_unsigned - \ + __intofs_b_unsigned))); \ +})) + +/* + * Dealing with detecting overflow in multiplication of integers. + * + * First step is to remove two corner cases with the minum signed integer + * which can't be represented as a positive integer + sign. + * Multiply with 0 or 1 can't overflow, no checking needed of the operation, + * only if it can be assigned to the result. + * + * After the corner cases are eliminated we convert the two factors to + * positive unsigned values, keeping track of the original in another + * variable which is used at the end to determine the sign of the product. + * + * The two terms (a and b) are divided into upper and lower half (x1 upper + * and x0 lower), so the product is: + * ((a1 << hshift) + a0) * ((b1 << hshift) + b0) + * which also is: + * ((a1 * b1) << (hshift * 2)) + (T1) + * ((a1 * b0 + a0 * b1) << hshift) + (T2) + * (a0 * b0) (T3) + * + * From this we can tell and (a1 * b1) has to be 0 or we'll overflow, that + * is, at least one of a1 or b1 has to be 0. Once this has been checked the + * addition: ((a1 * b0) << hshift) + ((a0 * b1) << hshift) + * isn't an addition as one of the terms will be 0. + * + * Since each factor in: (a0 * b0) + * only uses half the capicity of the underlaying type it can't overflow + * + * The addition of T2 and T3 can overflow so we use __INTOF_ADD() to + * perform that addition. If the addition succeeds without overflow the + * result is assigned the required sign and checked for overflow again. + */ + +#define __intof_mul_negate ((__intof_oa < 1) != (__intof_ob < 1)) +#define __intof_mul_hshift (sizeof(uintmax_t) * 8 / 2) +#define __intof_mul_hmask (UINTMAX_MAX >> __intof_mul_hshift) +#define __intof_mul_a0 ((uintmax_t)(__intof_a) >> __intof_mul_hshift) +#define __intof_mul_b0 ((uintmax_t)(__intof_b) >> __intof_mul_hshift) +#define __intof_mul_a1 ((uintmax_t)(__intof_a) & __intof_mul_hmask) +#define __intof_mul_b1 ((uintmax_t)(__intof_b) & __intof_mul_hmask) +#define __intof_mul_t (__intof_mul_a1 * __intof_mul_b0 + \ + __intof_mul_a0 * __intof_mul_b1) + +#define __INTOF_MUL(c, a, b) (__extension__({ \ + typeof(a) __intof_oa = (a); \ + typeof(a) __intof_a = __intof_oa < 1 ? -__intof_oa : __intof_oa; \ + typeof(b) __intof_ob = (b); \ + typeof(b) __intof_b = __intof_ob < 1 ? -__intof_ob : __intof_ob; \ + typeof(c) __intof_c; \ + \ + __intof_oa == 0 || __intof_ob == 0 || \ + __intof_oa == 1 || __intof_ob == 1 ? \ + __INTOF_ASSIGN((c), __intof_oa * __intof_ob) : \ + (__intof_mul_a0 && __intof_mul_b0) || \ + __intof_mul_t > __intof_mul_hmask ? 1 : \ + __INTOF_ADD((__intof_c), __intof_mul_t << __intof_mul_hshift, \ + __intof_mul_a1 * __intof_mul_b1) ? 1 : \ + __intof_mul_negate ? __INTOF_ASSIGN((c), -__intof_c) : \ + __INTOF_ASSIGN((c), __intof_c); \ +})) + +#define __compiler_add_overflow(a, b, res) __INTOF_ADD(*(res), (a), (b)) +#define __compiler_sub_overflow(a, b, res) __INTOF_SUB(*(res), (a), (b)) +#define __compiler_mul_overflow(a, b, res) __INTOF_MUL(*(res), (a), (b)) + +#endif /*!__HAVE_BUILTIN_OVERFLOW*/ + +#define __compiler_compare_and_swap(p, oval, nval) \ + __atomic_compare_exchange_n((p), (oval), (nval), true, \ + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED) \ + +#define __compiler_atomic_load(p) __atomic_load_n((p), __ATOMIC_RELAXED) +#define __compiler_atomic_store(p, val) \ + __atomic_store_n((p), (val), __ATOMIC_RELAXED) + +#define barrier() asm volatile ("" : : : "memory") + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#if __has_attribute(__fallthrough__) +#define fallthrough __attribute__((__fallthrough__)) +#else +#define fallthrough do {} while (0) /* fallthrough */ +#endif + +#ifndef __clang__ +#define __no_stackprot __attribute__((__optimize__ ("-fno-stack-protector"))) +#else +#define __no_stackprot +#endif + +#define __inhibit_loop_to_libcall \ + __attribute__ ((__optimize__ ("-fno-tree-loop-distribute-patterns"))) +#endif /*COMPILER_H*/ diff --git a/optee_os/lib/libutils/ext/include/config.h b/optee_os/lib/libutils/ext/include/config.h new file mode 100644 index 0000000..86fe92e --- /dev/null +++ b/optee_os/lib/libutils/ext/include/config.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright (c) 2017 Intel Corporation + * + */ +#ifndef __CONFIG_H_ +#define __CONFIG_H_ + +/** + * @brief Check for macro definition in compiler-visible expressions + * + * This trick was pioneered in Linux as the config_enabled() macro. + * The madness has the effect of taking a macro value that may be + * defined to "1" (e.g. CONFIG_MYFEATURE), or may not be defined at + * all and turning it into a literal expression that can be used at + * "runtime". That is, it works similarly to + * "defined(CONFIG_MYFEATURE)" does except that it is an expansion + * that can exist in a standard expression and be seen by the compiler + * and optimizer. Thus much ifdef usage can be replaced with cleaner + * expressions like: + * + * if (IS_ENABLED(CONFIG_MYFEATURE)) + * myfeature_enable(); + * + * INTERNAL + * First pass just to expand any existing macros, we need the macro + * value to be e.g. a literal "1" at expansion time in the next macro, + * not "(1)", etc... Standard recursive expansion does not work. + */ +#define IS_ENABLED(config_macro) Z_IS_ENABLED1(config_macro) + +/** + * @brief As IS_ENABLED() but to be used with _CFG_* internal switches + * to avoid triggering false positives with checkpatch. + */ +#define IS_ENABLED2(config_macro) IS_ENABLED(config_macro) + +/* Now stick on a "_XXXX" prefix, it will now be "_XXXX1" if config_macro + * is "1", or just "_XXXX" if it's undefined. + * ENABLED: Z_IS_ENABLED2(_XXXX1) + * DISABLED Z_IS_ENABLED2(_XXXX) + */ +#define Z_IS_ENABLED1(config_macro) Z_IS_ENABLED2(_XXXX##config_macro) + +/* Here's the core trick, we map "_XXXX1" to "_YYYY," (i.e. a string + * with a trailing comma), so it has the effect of making this a + * two-argument tuple to the preprocessor only in the case where the + * value is defined to "1" + * ENABLED: _YYYY, <--- note comma! + * DISABLED: _XXXX + */ +#define _XXXX1 _YYYY, + +/* Then we append an extra argument to fool the gcc preprocessor into + * accepting it as a varargs macro. + * arg1 arg2 arg3 + * ENABLED: Z_IS_ENABLED3(_YYYY, 1, 0) + * DISABLED Z_IS_ENABLED3(_XXXX 1, 0) + */ +#define Z_IS_ENABLED2(one_or_two_args) \ + Z_IS_ENABLED3(one_or_two_args true, false) + +/* And our second argument is thus now cooked to be 1 in the case + * where the value is defined to 1, and 0 if not: + */ +#define Z_IS_ENABLED3(ignore_this, val, ...) val + +#endif // __CONFIG_H_ diff --git a/optee_os/lib/libutils/ext/include/confine_array_index.h b/optee_os/lib/libutils/ext/include/confine_array_index.h new file mode 100644 index 0000000..cf831dc --- /dev/null +++ b/optee_os/lib/libutils/ext/include/confine_array_index.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright (c) 2020 Linaro Limited */ +// Copyright 2019 The Fuchsia Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/* + * Content of LICENSE file mentioned above: +Copyright 2019 The Fuchsia Authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef FBL_CONFINE_ARRAY_INDEX_H_ +#define FBL_CONFINE_ARRAY_INDEX_H_ + +#include + +// confine_array_index() bounds-checks and sanitizes an array index safely in the presence of +// speculative execution information leak bugs such as Spectre V1. confine_array_index() always +// returns a sanitized index, even in speculative-path execution. +// +// Callers need to combine confine_array_index with a conventional bounds check; the bounds +// check will return any necessary errors in the nonspeculative path, confine_array_index will +// confine indexes in the speculative path. +// +// Use: +// confine_array_index() returns |index|, if it is < size, or 0 if |index| is >= size. +// +// Example (may leak table1 contents): +// 1: int lookup3(size_t index) { +// 2: if (index >= table1_size) { +// 3: return -1; +// 4: } +// 5: size_t index2 = table1[index]; +// 6: return table2[index2]; +// 7: } +// +// Converted: +// +// 1: int lookup3(size_t index) { +// 2: if (index >= table1_size) { +// 3: return -1; +// 4: } +// 5: size_t safe_index = confine_array_index(index, table1_size); +// 6: size_t index2 = table1[safe_index]; +// 7: return table2[index2]; +// 8: } +#ifdef __aarch64__ +static inline size_t confine_array_index(size_t index, size_t size) { + size_t safe_index; + // Use a conditional select and a CSDB barrier to enforce validation of |index|. + // See "Cache Speculation Side-channels" whitepaper, section "Software Mitigation". + // "" The combination of both a conditional select/conditional move and the new barrier are + // sufficient to address this problem on ALL Arm implementations... "" + asm( + "cmp %1, %2\n" // %1 holds the unsanitized index + "csel %0, %1, xzr, lo\n" // Select index or zero based on carry (%1 within range) + "hint #20\n" // csdb + : "=r"(safe_index) + : "r"(index), "r"(size) + : "cc"); + return safe_index; +} +#endif +#ifdef __arm__ +static inline size_t confine_array_index(size_t index, size_t size) +{ + size_t ret_val = index; + + /* + * For the ARMv7/AArch32 case we're basing the select and barrier + * code on __load_no_speculate1() in as we + * lack the csel instruction. + */ + +#ifdef __thumb2__ + asm volatile ( + ".syntax unified\n" + "cmp %0, %1\n" + "it cs\n" +#ifdef __clang__ +#pragma clang diagnostic push + /* Avoid 'deprecated instruction in IT block [-Werror,-Winline-asm]' */ +#pragma clang diagnostic ignored "-Winline-asm" +#endif + "movcs %0, #0\n" +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + ".inst.n 0xf3af\t@ CSDB\n" + ".inst.n 0x8014\t@ CSDB" + : "+r" (ret_val) : "r" (size) : "cc"); +#else + asm volatile ( + ".syntax unified\n" + "cmp %0, %1\n" /* %0 holds the unsanitized index */ + "movcs %0, #0\n" + ".inst 0xe320f014\t@ CSDB" + : "+r" (ret_val) : "r" (size) : "cc"); +#endif + + return ret_val; +} +#endif /* __arm__ */ + +#ifdef __x86_64__ +static inline size_t confine_array_index(size_t index, size_t size) { + size_t safe_index = 0; + // Use a conditional move to enforce validation of |index|. + // The conditional move has a data dependency on the result of a comparison and cannot + // execute until the comparison is resolved. + // See "Software Techniques for Managing Speculation on AMD Processors", Mitigation V1-2. + // See "Analyzing potential bounds check bypass vulnerabilities", Revision 002, + // Section 5.2 Bounds clipping + __asm__( + "cmp %1, %2\n" + "cmova %1, %0\n" // Select between $0 and |index| + : "+r"(safe_index) + : "r"(index), "r"(size) + : "cc"); + return safe_index; +} +#endif + +#ifdef __riscv +static inline size_t confine_array_index(size_t index, size_t size) { + /* + * The naive C implementation without protection + * against side-channel attacks. + */ + if (index < size) + return index; + return 0; +} +#endif +#endif // FBL_CONFINE_ARRAY_INDEX_H_ diff --git a/optee_os/lib/libutils/ext/include/fault_mitigation.h b/optee_os/lib/libutils/ext/include/fault_mitigation.h new file mode 100644 index 0000000..06a6f7f --- /dev/null +++ b/optee_os/lib/libutils/ext/include/fault_mitigation.h @@ -0,0 +1,690 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Linaro Limited + */ +#ifndef __FAULT_MITIGATION_H +#define __FAULT_MITIGATION_H + +#include +#include +#include +#include + +#ifdef __KERNEL__ +#include +#include +#else +#include +#endif + +/* + * Fault migitigation helpers to make successful Hardware Fault Attacks + * harder to achieve. The paper [1] by Riscure gives background to the + * problem. + * + * These helpers aim to make it hard for a single glitch attack to succeed + * while the protected function or one of the ftmn_*() functions are + * executed. + * + * To have something to work with we assume that a single glitch may affect + * a few instructions in sequence to do nothing or to corrupt the content + * of a few registers. + * + * Using the terminology from [1] we are implementing the following patterns: + * 3 FAULT.VALUE.CHECK + * 5 FAULT.DECISION.CHECK + * 9 FAULT.FLOW.CONTROL + * + * Additionally are the following patterns also acknowledged with a few + * comments: + * 1. FAULT.CONSTANT.CODING + * Zero is normally a success code in OP-TEE so special functions are + * added to record anything but a zero result. + * 8. FAULT.NESTED.CHECK + * The linked calls performed by for instance FTMN_CALL_FUNC() addresses + * this by relying on the called function to update a state in + * struct ftmn_func_arg which is checked when the function has returned. + * 11. FAULT.PENALTY + * This is implicit since we're normally trying to protect things post + * boot and booting takes quite some time. + * + * [1] https://web.archive.org/web/20220616035354/https://www.riscure.com/uploads/2020/05/Riscure_Whitepaper_Fault_Mitigation_Patterns_final.pdf + */ + +#include +#include + +/* + * struct ftmn_check - track current checked state + * @steps: accumulated checkpoints + * @res: last stored result or return value + * + * While a function is executed it can update its state as a way of keeping + * track of important passages inside the function. When the function + * returns with for instance ftmn_return_res() it is checked that the + * accumulated state matches the expected state. + * + * @res is xored with FTMN_DEFAULT_HASH in order to retrieve the saved + * result or return value. + */ +struct ftmn_check { + unsigned long steps; + unsigned long res; +}; + +/* + * struct ftmn_func_arg - track a called function + * @hash: xor bitmask + * @res: stored result xored with @hash + * + * When the call of a function is tracked @hash is initialized to hash of + * caller xored with hash of called function. Before the called function + * updates @res it first xors @hash with its own hash, which is supposed to + * restore @hash to the hash of the calling function. This allows the + * calling function to confirm that the correct function has been called. + */ +struct ftmn_func_arg { + unsigned long hash; + unsigned long res; +}; + +/* + * struct ftmn - link a tracked call chain + * @check: local checked state + * @arg: argument for the next called tracked function + * @saved_arg: pointer to an optional argument passed to this function + * @arg_pp: cached return value from __ftmn_get_tsd_func_arg_pp() + * @my_hash: the hash of the calling function + * @called_hash:the hash of the called function + * + * In order to maintain the linked call chain of tracked functions the + * struct ftmn_func_arg passed to this function is saved in @saved_arg + * before updating the argument pointer with @arg. + */ +struct ftmn { + struct ftmn_check check; + struct ftmn_func_arg arg; + struct ftmn_func_arg *saved_arg; + struct ftmn_func_arg **arg_pp; + unsigned long my_hash; + unsigned long called_hash; +}; + +/* + * enum ftmn_incr - increase counter values + * + * Prime numbers to be used when increasing the accumulated state. + * Different increase counters can be used to keep apart different + * checkpoints. + */ +enum ftmn_incr { + FTMN_INCR0 = 7873, + FTMN_INCR1 = 7877, + FTMN_INCR2 = 7879, + FTMN_INCR3 = 7883, + FTMN_INCR4 = 7901, + FTMN_INCR5 = 7907, + FTMN_INCR_RESERVED = 7919, +}; + +typedef int (*ftmn_memcmp_t)(const void *p1, const void *p2, size_t nb); + +/* The default hash used when xoring the result in struct ftmn_check */ +#ifdef __ILP32__ +#define FTMN_DEFAULT_HASH 0x9c478bf6UL +#else +#define FTMN_DEFAULT_HASH 0xc478bf63e9500cb5UL +#endif + +/* + * FTMN_PANIC() - FTMN specific panic function + * + * This function is called whenever the FTMN function detects an + * inconsistency. An inconsistency is able to occur if the system is + * subject to a fault injection attack, in this case doing a panic() isn't + * an extreme measure. + */ +#ifdef __KERNEL__ +#define FTMN_PANIC() panic(); +#else +#define FTMN_PANIC() TEE_Panic(0); +#endif + +#define __FTMN_MAX_FUNC_NAME_LEN 256 + +#define __FTMN_FUNC_BYTE(f, o, l) ((o) < (l) ? (uint8_t)(f)[(o)] : 0) + +#define __FTMN_GET_FUNC_U64(f, o, l) \ + (SHIFT_U64(__FTMN_FUNC_BYTE((f), (o), (l)), 0) | \ + SHIFT_U64(__FTMN_FUNC_BYTE((f), (o) + 1, (l)), 8) | \ + SHIFT_U64(__FTMN_FUNC_BYTE((f), (o) + 2, (l)), 16) | \ + SHIFT_U64(__FTMN_FUNC_BYTE((f), (o) + 3, (l)), 24) | \ + SHIFT_U64(__FTMN_FUNC_BYTE((f), (o) + 4, (l)), 32) | \ + SHIFT_U64(__FTMN_FUNC_BYTE((f), (o) + 5, (l)), 40) | \ + SHIFT_U64(__FTMN_FUNC_BYTE((f), (o) + 6, (l)), 48) | \ + SHIFT_U64(__FTMN_FUNC_BYTE((f), (o) + 7, (l)), 56)) + +#define __FTMN_FUNC_HASH32(f, o, l) \ + (__FTMN_GET_FUNC_U64((f), (o), (l)) ^ \ + __FTMN_GET_FUNC_U64((f), (o) + 8, (l))) + +#define __FTMN_FUNC_HASH16(f, o, l) \ + (__FTMN_FUNC_HASH32((f), (o), (l)) ^ \ + __FTMN_FUNC_HASH32((f), (o) + __FTMN_MAX_FUNC_NAME_LEN / 16, (l))) + +#define __FTMN_FUNC_HASH8(f, o, l) \ + (__FTMN_FUNC_HASH16((f), (o), (l)) ^ \ + __FTMN_FUNC_HASH16((f), (o) + __FTMN_MAX_FUNC_NAME_LEN / 8, (l))) + +#define __FTMN_FUNC_HASH4(f, o, l) \ + (__FTMN_FUNC_HASH8((f), (o), (l)) ^ \ + __FTMN_FUNC_HASH8((f), (o) + __FTMN_MAX_FUNC_NAME_LEN / 4, (l))) + +#define __FTMN_FUNC_HASH2(f, l) \ + (__FTMN_FUNC_HASH4(f, 0, l) ^ \ + __FTMN_FUNC_HASH4(f, __FTMN_MAX_FUNC_NAME_LEN / 2, l)) + +#ifdef __ILP32__ +#define __FTMN_FUNC_HASH(f, l) \ + (unsigned long)(__FTMN_FUNC_HASH2((f), (l)) ^ \ + (__FTMN_FUNC_HASH2((f), (l)) >> 32)) +#else +#define __FTMN_FUNC_HASH(f, l) (unsigned long)__FTMN_FUNC_HASH2((f), (l)) +#endif + +#define __ftmn_step_count_1(c0) ((c0) * FTMN_INCR0) +#define __ftmn_step_count_2(c0, c1) \ + (__ftmn_step_count_1(c0) + (c1) * FTMN_INCR1) +#define __ftmn_step_count_3(c0, c1, c2) \ + (__ftmn_step_count_2(c0, c1) + (c2) * FTMN_INCR2) +#define __ftmn_step_count_4(c0, c1, c2, c3) \ + (__ftmn_step_count_3(c0, c1, c2) + (c3) * FTMN_INCR3) +#define __ftmn_step_count_5(c0, c1, c2, c3, c4) \ + (__ftmn_step_count_4(c0, c1, c2, c3) + (c4) * FTMN_INCR4) +#define __ftmn_step_count_6(c0, c1, c2, c3, c4, c5) \ + (__ftmn_step_count_5(c0, c1, c2, c3, c4) + (c5) * FTMN_INCR5) +#define ___ftmn_args_count(_0, _1, _2, _3, _4, _5, x, ...) x +#define __ftmn_args_count(...) \ + ___ftmn_args_count(__VA_ARGS__, 6, 5, 4, 3, 2, 1, 0) +#define ___ftmn_step_count(count, ...) __ftmn_step_count_ ## count(__VA_ARGS__) +#define __ftmn_step_count(count, ...) ___ftmn_step_count(count, __VA_ARGS__) + +unsigned long ___ftmn_return_res(struct ftmn_check *check, unsigned long steps, + unsigned long res); +void ___ftmn_expect_state(struct ftmn_check *check, enum ftmn_incr incr, + unsigned long steps, unsigned long res); + +void ___ftmn_callee_done(struct ftmn_func_arg *arg, unsigned long my_hash, + unsigned long res); +void ___ftmn_callee_done_not_zero(struct ftmn_func_arg *arg, + unsigned long my_hash, + unsigned long res); +void ___ftmn_callee_done_memcmp(struct ftmn_func_arg *arg, + unsigned long my_hash, int res, + ftmn_memcmp_t my_memcmp, + const void *p1, const void *p2, size_t nb); +void ___ftmn_callee_done_check(struct ftmn_func_arg *arg, unsigned long my_hash, + struct ftmn_check *check, enum ftmn_incr incr, + unsigned long steps, unsigned long res); + +void ___ftmn_callee_update_not_zero(struct ftmn_func_arg *arg, + unsigned long res); + +void ___ftmn_set_check_res(struct ftmn_check *check, enum ftmn_incr incr, + unsigned long res); +void ___ftmn_set_check_res_not_zero(struct ftmn_check *check, + enum ftmn_incr incr, + unsigned long res); +void ___ftmn_set_check_res_memcmp(struct ftmn_check *check, enum ftmn_incr incr, + int res, ftmn_memcmp_t my_memcmp, + const void *p1, const void *p2, size_t nb); + +void ___ftmn_copy_linked_call_res(struct ftmn_check *check, enum ftmn_incr incr, + struct ftmn_func_arg *arg, unsigned long res); + + +#ifndef __KERNEL__ +extern struct ftmn_func_arg *__ftmn_global_func_arg; +#endif + +static inline struct ftmn_func_arg **__ftmn_get_tsd_func_arg_pp(void) +{ +#if defined(CFG_FAULT_MITIGATION) && defined(__KERNEL__) + if (thread_get_id_may_fail() >= 0) + return &thread_get_tsd()->ftmn_arg; + else + return &thread_get_core_local()->ftmn_arg; +#elif defined(CFG_FAULT_MITIGATION) + return &__ftmn_global_func_arg; +#else + return NULL; +#endif +} + +static inline struct ftmn_func_arg *__ftmn_get_tsd_func_arg(void) +{ + struct ftmn_func_arg **pp = __ftmn_get_tsd_func_arg_pp(); + + if (!pp) + return NULL; + + return *pp; +} + +static inline void __ftmn_push_linked_call(struct ftmn *ftmn, + unsigned long my_hash, + unsigned long called_hash) +{ + struct ftmn_func_arg **arg_pp = __ftmn_get_tsd_func_arg_pp(); + + if (arg_pp) { + ftmn->arg_pp = arg_pp; + ftmn->my_hash = my_hash; + ftmn->called_hash = called_hash; + ftmn->saved_arg = *ftmn->arg_pp; + *ftmn->arg_pp = &ftmn->arg; + ftmn->arg.hash = my_hash; + } +} + +static inline void __ftmn_pop_linked_call(struct ftmn *ftmn) +{ + if (ftmn->arg_pp) + *ftmn->arg_pp = ftmn->saved_arg; +} + +static inline void __ftmn_copy_linked_call_res(struct ftmn *f, + enum ftmn_incr incr, + unsigned long res) +{ + if (f->arg_pp) { + assert(f->arg.hash == (f->my_hash ^ f->called_hash)); + assert(&f->arg == *f->arg_pp); + assert((f->arg.hash ^ f->arg.res) == res); + ___ftmn_copy_linked_call_res(&f->check, incr, &f->arg, res); + } +} + +static inline void __ftmn_calle_swap_hash(struct ftmn_func_arg *arg, + unsigned long my_old_hash, + unsigned long my_new_hash) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION) && arg) + arg->hash ^= my_old_hash ^ my_new_hash; +} + +static inline void __ftmn_callee_done(struct ftmn_func_arg *arg, + unsigned long my_hash, unsigned long res) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION) && arg) + ___ftmn_callee_done(arg, my_hash, res); +} + +static inline void __ftmn_callee_done_not_zero(struct ftmn_func_arg *arg, + unsigned long hash, + unsigned long res) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION) && arg) + ___ftmn_callee_done_not_zero(arg, hash, res); +} + +static inline int +__ftmn_callee_done_memcmp(struct ftmn_func_arg *arg, unsigned long hash, + ftmn_memcmp_t my_memcmp, + const void *p1, const void *p2, size_t nb) +{ + int res = my_memcmp(p1, p2, nb); + + if (IS_ENABLED(CFG_FAULT_MITIGATION) && arg) + ___ftmn_callee_done_memcmp(arg, hash, res, my_memcmp, + p1, p2, nb); + + return res; +} + +static inline void __ftmn_callee_done_check(struct ftmn *ftmn, + unsigned long my_hash, + enum ftmn_incr incr, + unsigned long steps, + unsigned long res) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION)) + ___ftmn_callee_done_check(__ftmn_get_tsd_func_arg(), my_hash, + &ftmn->check, incr, steps, res); +} + +static inline void __ftmn_callee_update_not_zero(struct ftmn_func_arg *arg, + unsigned long res) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION) && arg) + ___ftmn_callee_update_not_zero(arg, res); +} + +static inline void __ftmn_set_check_res(struct ftmn *ftmn, enum ftmn_incr incr, + unsigned long res) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION)) + ___ftmn_set_check_res(&ftmn->check, incr, res); +} + +static inline void __ftmn_set_check_res_not_zero(struct ftmn *ftmn, + enum ftmn_incr incr, + unsigned long res) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION)) + ___ftmn_set_check_res_not_zero(&ftmn->check, incr, res); +} + + + +/* + * FTMN_FUNC_HASH() - "hash" a function name + * + * Function names are "hashed" into an unsigned long. The "hashing" is done + * by xoring each 32/64 bit word of the function name producing a bit + * pattern that should be mostly unique for each function. Only the first + * 256 characters of the name are used when xoring as this is expected to + * be optimized to be calculated when compiling the source code in order to + * minimize the overhead. + */ +#define FTMN_FUNC_HASH(name) __FTMN_FUNC_HASH(name, sizeof(name)) + +/* + * FTMN_PUSH_LINKED_CALL() - push call into a linked call chain + * @ftmn: The local struct ftmn + * @called_func_hash: The hash of the called function + * + * Inserts a call into a linked call chain or starts a new call chain if + * the passed struct ftmn_func_arg pointer was NULL. + * + * Each FTMN_PUSH_LINKED_CALL() is supposed to be matched by a + * FTMN_POP_LINKED_CALL(). + */ +#define FTMN_PUSH_LINKED_CALL(ftmn, called_func_hash) \ + __ftmn_push_linked_call((ftmn), FTMN_FUNC_HASH(__func__), \ + (called_func_hash)) + +/* + * FTMN_SET_CHECK_RES_FROM_CALL() - copy the result from a linked call + * @ftmn: The struct ftmn used during the linked call + * @incr: Value to increase the checked state with + * @res: Returned result to be match against the saved/copied result + * + * This macro is called just after a checked linked function has returned. + * The return value from the function is copied from the struct ftmn_func_arg + * passed to the called function into the local checked state. The checked + * state is increased with @incr. @res is checked against the saved result + * of the called function. + */ +#define FTMN_SET_CHECK_RES_FROM_CALL(ftmn, incr, res) \ + __ftmn_copy_linked_call_res((ftmn), (incr), (res)) + +/* + * FTMN_POP_LINKED_CALL() - remove a call from a linked call chain + * @ftmn: The local struct ftmn + * + * Supposed to match a call to FTMN_PUSH_LINKED_CALL() + */ +#define FTMN_POP_LINKED_CALL(ftmn) __ftmn_pop_linked_call((ftmn)) + +/* + * FTMN_CALL_FUNC() - Do a linked call to a function + * @res: Variable to be assigned the result of the called function + * @ftmn: The local struct ftmn + * @incr: Value to increase the checked state with + * @func: Function to be called + * @...: Arguments to pass to @func + * + * This macro can be used to make a linked call to another function, the + * callee. This macro depends on the callee to always update the struct + * ftmn_func_arg (part of struct ftmn) even when returning an error. + * + * Note that in the cases where the callee may skip updating the struct + * ftmn_func_arg this macro cannot be used as + * FTMN_SET_CHECK_RES_FROM_CALL() would cause a panic due to mismatching + * return value and saved result. + */ +#define FTMN_CALL_FUNC(res, ftmn, incr, func, ...) \ + do { \ + FTMN_PUSH_LINKED_CALL((ftmn), FTMN_FUNC_HASH(#func)); \ + (res) = func(__VA_ARGS__); \ + FTMN_SET_CHECK_RES_FROM_CALL((ftmn), (incr), (res)); \ + FTMN_POP_LINKED_CALL((ftmn)); \ + } while (0) + +/* + * FTMN_CALLEE_DONE() - Record result of callee + * @res: Result or return value + * + * The passed result will be stored in the struct ftmn_func_arg struct + * supplied by the caller. This function must only be called once by the + * callee. + * + * Note that this function is somewhat dangerous as any passed value will + * be stored so if the value has been tampered with there is no additional + * redundant checks to rely on. + */ +#define FTMN_CALLEE_DONE(res) \ + __ftmn_callee_done(__ftmn_get_tsd_func_arg(), \ + FTMN_FUNC_HASH(__func__), (res)) +/* + * FTMN_CALLEE_DONE_NOT_ZERO() - Record non-zero result of callee + * @res: Result or return value + * + * The passed result will be stored in the struct ftmn_func_arg struct + * supplied by the caller. This function must only be called once by the + * callee. + * + * Note that this function is somewhat dangerous as any passed value will + * be stored so if the value has been tampered with there is no additional + * redundant checks to rely on. However, there are extra checks against + * unintentionally storing a zero which often is interpreted as a + * successful return value. + */ +#define FTMN_CALLEE_DONE_NOT_ZERO(res) \ + __ftmn_callee_done_not_zero(__ftmn_get_tsd_func_arg(), \ + FTMN_FUNC_HASH(__func__), (res)) + +/* + * FTMN_CALLEE_DONE_CHECK() - Record result of callee with checked state + * @ftmn: The local struct ftmn + * @incr: Value to increase the checked state with + * @exp_steps: Expected recorded checkpoints + * @res: Result or return value + * + * The passed result will be stored in the struct ftmn_func_arg struct + * supplied by the caller. This function must only be called once by the + * callee. + * + * @res is double checked against the value stored in local checked state. + * @exp_steps is checked against the locate checked state. The local + * checked state is increased by @incr. + */ +#define FTMN_CALLEE_DONE_CHECK(ftmn, incr, exp_steps, res) \ + __ftmn_callee_done_check((ftmn), FTMN_FUNC_HASH(__func__), \ + (incr), (exp_steps), (res)) + +/* + * FTMN_CALLEE_DONE_MEMCMP() - Record result of memcmp() in a callee + * @my_memcmp: Function pointer of custom memcmp() + * @p1: Pointer to first buffer + * @p2: Pointer to second buffer + * @nb: Number of bytes + * + * The result from the mem compare is saved in the local checked state. + * This function must only be called once by the callee. + */ +#define FTMN_CALLEE_DONE_MEMCMP(my_memcmp, p1, p2, nb) \ + __ftmn_callee_done_memcmp(__ftmn_get_tsd_func_arg(), \ + FTMN_FUNC_HASH(__func__), (my_memcmp), \ + (p1), (p2), (nb)) + +/* + * FTMN_CALLEE_UPDATE_NOT_ZERO() - Update the result of a callee with a + * non-zero value + * @res: Result or return value + * + * The passed result will be stored in the struct ftmn_func_arg struct + * supplied by the caller. This function can be called any number of times + * by the callee, provided that one of the FTMN_CALLEE_DONE_XXX() functions + * has been called first. + * + * Note that this function is somewhat dangerous as any passed value will + * be stored so if the value has been tampered with there is no additional + * redundant checks to rely on. However, there are extra checks against + * unintentionally storing a zero which often is interpreted as a + * successful return value. + */ +#define FTMN_CALLEE_UPDATE_NOT_ZERO(res) \ + __ftmn_callee_update_not_zero(__ftmn_get_tsd_func_arg(), res) + +/* + * FTMN_CALLEE_SWAP_HASH() - Remove old hash and add new hash + * @my_old_hash: The old hash to remove + * + * This macro replaces the old expected function hash with the hash of the + * current function. + * + * If a function is called using an alias the caller uses the hash of the + * alias not the real function name. This hash is recoded in the field + * "hash" in struct ftmn_func_arg which can be found with + * __ftmn_get_tsd_func_arg(). + * + * The FTMN_CALLE_* functions only work with the real function name so the + * old hash must be removed and replaced with the new for the calling + * function to be able to verify the result. + */ +#define FTMN_CALLEE_SWAP_HASH(my_old_hash) \ + __ftmn_calle_swap_hash(__ftmn_get_tsd_func_arg(), \ + (my_old_hash), FTMN_FUNC_HASH(__func__)) + +/* + * FTMN_SET_CHECK_RES() - Records a result in local checked state + * @ftmn: The local struct ftmn + * @incr: Value to increase the checked state with + * @res: Result or return value + * + * Note that this function is somewhat dangerous as any passed value will + * be stored so if the value has been tampered with there is no additional + * redundant checks to rely on. + */ +#define FTMN_SET_CHECK_RES(ftmn, incr, res) \ + __ftmn_set_check_res((ftmn), (incr), (res)) + +/* + * FTMN_SET_CHECK_RES_NOT_ZERO() - Records a non-zero result in local checked + * state + * @ftmn: The local struct ftmn + * @incr: Value to increase the checked state with + * @res: Result or return value + * + * Note that this function is somewhat dangerous as any passed value will + * be stored so if the value has been tampered with there is no additional + * redundant checks to rely on. However, there are extra checks against + * unintentionally storing a zero which often is interpreted as a + * successful return value. + */ +#define FTMN_SET_CHECK_RES_NOT_ZERO(ftmn, incr, res) \ + __ftmn_set_check_res_not_zero((ftmn), (incr), (res)) + +static inline int ftmn_set_check_res_memcmp(struct ftmn *ftmn, + enum ftmn_incr incr, + ftmn_memcmp_t my_memcmp, + const void *p1, const void *p2, + size_t nb) +{ + int res = my_memcmp(p1, p2, nb); + + if (IS_ENABLED(CFG_FAULT_MITIGATION)) + ___ftmn_set_check_res_memcmp(&ftmn->check, incr, res, + my_memcmp, p1, p2, nb); + + return res; +} + +/* + * FTMN_STEP_COUNT() - Calculate total step count + * + * Takes variable number of arguments, up to a total of 6. Where arg0 + * is the number of times the counter has been increased by FTMN_INCR0, + * arg1 FTMN_INCR1 and so on. + */ +#define FTMN_STEP_COUNT(...) \ + __ftmn_step_count(__ftmn_args_count(__VA_ARGS__), __VA_ARGS__) + +/* + * ftmn_checkpoint() - Add a checkpoint + * @ftmn: The local struct ftmn + * @incr: Value to increase the checked state with + * + * Adds a checkpoint by increasing the internal checked state. This + * can be checked at a later point in the calling function, for instance + * with ftmn_return_res(). + */ +static inline void ftmn_checkpoint(struct ftmn *ftmn, enum ftmn_incr incr) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION)) { + /* + * The purpose of the barriers is to prevent the compiler + * from optimizing this increase to some other location + * in the calling function. + */ + barrier(); + ftmn->check.steps += incr; + barrier(); + } +} + +/* + * ftmn_expect_state() - Check expected state + * @ftmn: The local struct ftmn + * @incr: Value to increase the checked state with + * @steps: Expected accumulated steps + * @res: Expected saved result or return value + * + * This is a more advanced version of ftmn_checkpoint() which before + * increasing the accumulated steps first checks the accumulated steps and + * saved result or return value. + */ +static inline void ftmn_expect_state(struct ftmn *ftmn, + enum ftmn_incr incr, unsigned long steps, + unsigned long res) +{ + if (IS_ENABLED(CFG_FAULT_MITIGATION)) { + assert((ftmn->check.res ^ FTMN_DEFAULT_HASH) == res); + assert(ftmn->check.steps == steps); + + ___ftmn_expect_state(&ftmn->check, incr, steps, res); + } +} + +/* + * ftmn_return_res() - Check and return result + * @ftmn: The local struct ftmn + * @steps: Expected accumulated steps + * @res: Expected saved result or return value + * + * Checks that the internal accumulated state matches the supplied @steps + * and that the saved result or return value matches the supplied one. + * + * Returns @res. + */ +static inline unsigned long ftmn_return_res(struct ftmn *ftmn, + unsigned long steps, + unsigned long res) +{ + /* + * We're expecting that the compiler does a tail call optimization + * allowing ___ftmn_return_res() to have full control over the + * returned value. Thus trying to reduce the window where the + * return value can be tampered with. + */ + if (IS_ENABLED(CFG_FAULT_MITIGATION)) { + assert((ftmn->check.res ^ FTMN_DEFAULT_HASH) == res); + assert(ftmn->check.steps == steps); + + return ___ftmn_return_res(&ftmn->check, steps, res); + } + return res; +} +#endif /*__FAULT_MITIGATION_H*/ diff --git a/optee_os/lib/libutils/ext/include/mempool.h b/optee_os/lib/libutils/ext/include/mempool.h new file mode 100644 index 0000000..4a73171 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/mempool.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ + +#ifndef __MEMPOOL_H +#define __MEMPOOL_H + +#include + +/* + * Memory pool for large temporary memory allocations that must not fail. + * With the first allocation from an unused (idle or free) pool the pool + * becomes reserved for that particular thread, until all allocations are + * freed again. In order to avoid dead-lock and ease code review it is good + * practise to free everything allocated by a certain function before + * returning. + */ + +/* + * struct mempool_item - internal struct to keep track of an item + */ +struct mempool_item { + size_t size; + ssize_t prev_item_offset; + ssize_t next_item_offset; +}; + +struct mempool; + +#define MEMPOOL_ALIGN __alignof__(long) + +#if defined(__KERNEL__) +/* + * System wide memory pool for large temporary memory allocation. + */ +extern struct mempool *mempool_default; +#endif + +/* + * mempool_alloc_pool() - Allocate a new memory pool + * @data: a block of memory to carve out items from, must + * have an alignment of MEMPOOL_ALIGN. + * @size: size fo the block of memory + * @release_mem: function to call when the pool has been emptied, + * ignored if NULL. + * returns a pointer to a valid pool on success or NULL on failure. + */ +struct mempool *mempool_alloc_pool(void *data, size_t size, + void (*release_mem)(void *ptr, size_t size)); + +/* + * mempool_alloc() - Allocate an item from a memory pool + * @pool: A memory pool created with mempool_alloc_pool() + * @size: Size in bytes of the item to allocate + * return a valid pointer on success or NULL on failure. + */ +void *mempool_alloc(struct mempool *pool, size_t size); + +/* + * mempool_calloc() - Allocate and zero initialize an array of elements from a + * memory pool + * @pool: A memory pool created with mempool_alloc_pool() + * @nmemb: Number of elements in the array + * @size: Size in bytes of each element in the array + * return a valid pointer on success or NULL on failure. + */ +void *mempool_calloc(struct mempool *pool, size_t nmemb, size_t size); + +/* + * mempool_free() - Frees a previously allocated item + * @pool: A memory pool create with mempool_alloc_pool() + * @ptr: A pointer to a previously allocated item + */ +void mempool_free(struct mempool *pool, void *ptr); + +#endif /*__MEMPOOL_H*/ diff --git a/optee_os/lib/libutils/ext/include/memtag.h b/optee_os/lib/libutils/ext/include/memtag.h new file mode 100644 index 0000000..d55d8c0 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/memtag.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Linaro Limited + */ + +#ifndef __MEMTAG_H +#define __MEMTAG_H + +#include +#include +#include +#include + +#if defined(CFG_MEMTAG) && defined(__aarch64__) +#define MEMTAG_IS_ENABLED 1 +#define MEMTAG_TAG_SHIFT 56 +#define MEMTAG_TAG_WIDTH 4 +#define MEMTAG_TAG_MASK (BIT(MEMTAG_TAG_WIDTH) - 1) + +#define MEMTAG_GRANULE_SIZE 16 +#else +#define MEMTAG_IS_ENABLED 0 +#define MEMTAG_GRANULE_SIZE 1 +#define MEMTAG_TAG_WIDTH 0 +#endif + +#define MEMTAG_GRANULE_MASK (MEMTAG_GRANULE_SIZE - 1) + +struct __memtag_ops { + void *(*set_tags)(void *addr, size_t size, uint8_t tag); + void *(*set_random_tags)(void *addr, size_t size); + void (*clear_mem)(void *addr, size_t size); + uint8_t (*read_tag)(const void *addr); +}; + +extern const struct __memtag_ops __memtag_ops_disabled; +extern const struct __memtag_ops *__memtag_ops; + +static inline void *__memtag_disabled_set_tags(void *addr, size_t size __unused, + uint8_t tag __unused) +{ + return addr; +} + +static inline void *__memtag_disabled_set_random_tags(void *addr, + size_t size __unused) +{ + return addr; +} + +static inline void __memtag_disabled_clear_mem(void *addr, size_t size) +{ + memset(addr, 0, size); +} + +static inline uint8_t __memtag_disabled_read_tag(const void *addr __unused) +{ + return 0; +} + +/* + * memtag_set_tags() - Tag a memory range + * @addr: Start of memory range + * @size: Size of memory range + * @tag: Tag to use + * + * The memory range is updated with the supplied tag. An eventual tag + * already present in the upper bits of the address in @addr is ignored. + * + * @addr and @size must be aligned/multiple of MEMTAG_GRANULE_SIZE. + * + * Returns an address with the new tag inserted to be used to access this + * memory area. + */ +static inline void *memtag_set_tags(void *addr, size_t size, uint8_t tag) +{ +#if MEMTAG_IS_ENABLED + return __memtag_ops->set_tags(addr, size, tag); +#else + return __memtag_disabled_set_tags(addr, size, tag); +#endif +} + +/* + * memtag_set_random_tags() - Tag a memory range with a random tag + * @addr: Start of memory range + * @size: Size of memory range + * + * The memory range is updated with a randomly generated tag. An eventual + * tag already present in the upper bits of the address in @addr is + * ignored. + * + * @addr and @size must be aligned/multiple of MEMTAG_GRANULE_SIZE. + * + * Returns an address with the new tag inserted to be used to access this + * memory area. + */ +static inline void *memtag_set_random_tags(void *addr, size_t size) +{ +#if MEMTAG_IS_ENABLED + return __memtag_ops->set_random_tags(addr, size); +#else + return __memtag_disabled_set_random_tags(addr, size); +#endif +} + +static inline void memtag_clear_mem(void *addr, size_t size) +{ +#if MEMTAG_IS_ENABLED + __memtag_ops->clear_mem(addr, size); +#else + __memtag_disabled_clear_mem(addr, size); +#endif +} + +/* + * memtag_strip_tag_vaddr() - Removes an eventual tag from an address + * @addr: Address to strip + * + * Returns a vaddr_t without an eventual tag. + */ +static inline vaddr_t memtag_strip_tag_vaddr(const void *addr) +{ + vaddr_t va = (vaddr_t)addr; + +#if MEMTAG_IS_ENABLED + va &= ~SHIFT_U64(MEMTAG_TAG_MASK, MEMTAG_TAG_SHIFT); +#endif + + return va; +} + +/* + * memtag_strip_tag_const() - Removes an eventual tag from an address + * @addr: Address to strip + * + * Returns the address without an eventual tag. + */ +static inline const void *memtag_strip_tag_const(const void *addr) +{ + return (const void *)memtag_strip_tag_vaddr(addr); +} + +/* + * memtag_strip_tag() - Removes an eventual tag from an address + * @addr: Address to strip + * + * Returns the address without an eventual tag. + */ +static inline void *memtag_strip_tag(void *addr) +{ + return (void *)memtag_strip_tag_vaddr(addr); +} + +/* + * memtag_insert_tag_vaddr() - Inserts a tag into an address + * @addr: Address to transform + * @tag: Tag to insert + * + * Returns the address with the new tag inserted. + */ +static inline vaddr_t memtag_insert_tag_vaddr(vaddr_t addr, + uint8_t tag __maybe_unused) +{ + vaddr_t va = memtag_strip_tag_vaddr((void *)addr); + +#if MEMTAG_IS_ENABLED + va |= SHIFT_U64(tag, MEMTAG_TAG_SHIFT); +#endif + + return va; +} + +/* + * memtag_insert_tag() - Inserts a tag into an address + * @addr: Address to transform + * @tag: Tag to insert + * + * Returns the address with the new tag inserted. + */ +static inline void *memtag_insert_tag(void *addr, uint8_t tag) +{ + return (void *)memtag_insert_tag_vaddr((vaddr_t)addr, tag); +} + +/* + * memtag_get_tag() - Extract a tag from an address + * @addr: Address with an eventual tag + * + * Returns the extracted tag. + */ +static inline uint8_t memtag_get_tag(const void *addr __maybe_unused) +{ +#if MEMTAG_IS_ENABLED + uint64_t va = (vaddr_t)addr; + + return (va >> MEMTAG_TAG_SHIFT) & MEMTAG_TAG_MASK; +#else + return 0; +#endif +} + +static inline uint8_t memtag_read_tag(const void *addr) +{ +#if MEMTAG_IS_ENABLED + return __memtag_ops->read_tag(addr); +#else + return __memtag_disabled_read_tag(addr); +#endif +} + +static inline void memtag_assert_tag(const void *addr __maybe_unused) +{ + assert(memtag_get_tag(addr) == memtag_read_tag(addr)); +} + +#if MEMTAG_IS_ENABLED +void memtag_init_ops(unsigned int memtag_impl); +#else +static inline void memtag_init_ops(unsigned int memtag_impl __unused) +{ +} +#endif + +static inline bool memtag_is_enabled(void) +{ +#if MEMTAG_IS_ENABLED + return __memtag_ops != &__memtag_ops_disabled; +#else + return false; +#endif +} + +#endif /*__MEMTAG_H*/ diff --git a/optee_os/lib/libutils/ext/include/printk.h b/optee_os/lib/libutils/ext/include/printk.h new file mode 100644 index 0000000..4f21b25 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/printk.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015 Linaro Limited + */ + +/* + * This file provides extensions to the standard snprintf() and vsnprintf() + * functions. These 'k' variants support additional formats. + */ + +#ifndef PRINTK_H +#define PRINTK_H + +#include +#include +#include + +int snprintk(char *str, size_t size, const char *fmt, ...) + __attribute__((__format__(__printf__, 3, 4))); +int vsnprintk(char *str, size_t size, const char *fmt, va_list ap) + __attribute__((__format__(__printf__, 3, 0))); + +int __vsnprintf(char *str, size_t size, const char *fmt, va_list ap, + bool ext) __attribute__((__format__(__printf__, 3, 0))); +int __vsprintf(char *bf, const char *fmt, va_list ap) + __attribute__((__format__(__printf__, 2, 0))); + +#endif /* PRINTK_H */ diff --git a/optee_os/lib/libutils/ext/include/riscv.S b/optee_os/lib/libutils/ext/include/riscv.S new file mode 100644 index 0000000..62d5b47 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/riscv.S @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + */ + +#if __riscv_xlen == 32 +#define STR sw +#define LDR lw +#define REGOFF(x) ((x) * 4) +#elif __riscv_xlen == 64 +#define STR sd +#define LDR ld +#define REGOFF(x) ((x) * 8) +#endif diff --git a/optee_os/lib/libutils/ext/include/speculation_barrier.h b/optee_os/lib/libutils/ext/include/speculation_barrier.h new file mode 100644 index 0000000..ef74e36 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/speculation_barrier.h @@ -0,0 +1,508 @@ +/* SPDX-License-Identifier: BSL-1.0 */ +/* Copyright (c) 2017 Arm Limited. All rights reserved. + + Boost Software License - Version 1.0 - August 17th, 2003 + + Permission is hereby granted, free of charge, to any person or organization + obtaining a copy of the software and accompanying documentation covered by + this license (the "Software") to use, reproduce, display, distribute, + execute, and transmit the Software, and to prepare derivative works of the + Software, and to permit third-parties to whom the Software is furnished to do + so, all subject to the following: + + The copyright notices in the Software and this entire statement, including + the above license grant, this restriction and the following disclaimer, must + be included in all copies of the Software, in whole or in part, and all + derivative works of the Software, unless such copies or derivative works are + solely in the form of machine-executable object code generated by a source + language processor. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT + SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR + ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. */ + + +#ifdef __HAVE_LOAD_NO_SPECULATE +#define load_no_speculate(__ptr, __low, __high) \ +(__extension__ ({ \ + __typeof__ ((__ptr)) __ptr_once = (__ptr); \ + __builtin_load_no_speculate (__ptr_once, __low, __high, \ + 0, __ptr_once); \ +})) + +#define load_no_speculate_fail(__ptr, __low, __high, __failval) \ +(__extension__ ({ \ + __typeof__ ((__ptr)) __ptr_once = (__ptr); \ + __builtin_load_no_speculate (__ptr_once, __low, __high, \ + __failval, __ptr_once); \ +})) + +#define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ + (__builtin_load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) + +#else + +#ifdef __GNUC__ +#define __UNUSED __attribute__((unused)) +#else +#define __UNUSED +#endif + +#ifdef __aarch64__ + +#define __load_no_speculate1(__ptr, __low, __high, __failval, \ + __cmpptr, __w, __sz) \ +(__extension__ ({ \ + __typeof__ (0 + (*(__ptr))) __nln_val; \ + /* This typecasting is required to ensure correct handling of upper \ + bits of failval, to ensure a clean return from the CSEL below. */ \ + __typeof__(*(__ptr)) __fv \ + = (__typeof__(*(__ptr)))(unsigned long long) (__failval); \ + /* If __high is explicitly NULL, we must not emit the \ + upper-bound comparison. We need to cast __high to an \ + unsigned long long before handing it to __builtin_constant_p to \ + ensure that clang/llvm correctly detects NULL as a constant if it \ + is defined as (void*) 0. */ \ + if (__builtin_constant_p ((unsigned long long)__high) \ + && __high == ((void *)0)) \ + { \ + __asm__ volatile ( \ + "cmp\t%[__c], %[__l]\n\t" \ + "bcc\t.ns%=\n\t" \ + "ldr" __sz "\t%" __w "[__v], %[__p]\n" \ + ".ns%=:\n\t" \ + "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cs\n\t" \ + "hint\t#0x14 // CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&r" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + /* The memory location from which we will load. */ \ + [__p] "m" (*(__ptr)), \ + /* The value to return if the condition check fails. */ \ + [__f] "rZ" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + else \ + { \ + __asm__ volatile ( \ + "cmp\t%[__c], %[__l]\n\t" \ + "ccmp\t%[__c], %[__h], 2, cs\n\t" \ + "bcs\t.ns%=\n\t" \ + "ldr" __sz "\t%" __w "[__v], %[__p]\n" \ + ".ns%=:\n\t" \ + "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cc\n\t" \ + "hint\t#0x14 // CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&r" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + [__h] "r" (__high), \ + /* The memory location from which we will load. */ \ + [__p] "m" (*(__ptr)), \ + /* The value to return if the condition check fails. */ \ + [__f] "rZ" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + (__typeof__ (*(__ptr))) __nln_val; \ +})) + +#define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr) \ +(__extension__ ({ \ + __typeof__ (0 + *(__ptr)) __nl_val; \ + \ + switch (sizeof(*(__ptr))) { \ + case 1: \ + __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ + __failval, __cmpptr, "w", "b"); \ + break; \ + case 2: \ + __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ + __failval, __cmpptr, "w", "h"); \ + break; \ + case 4: \ + __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ + __failval, __cmpptr, "w", ""); \ + break; \ + case 8: \ + __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ + __failval, __cmpptr, "x", ""); \ + break; \ + default: \ + { \ + char __static_assert_no_speculate_load_size_too_big \ + [sizeof (__nl_val) > 8 ? -1 : 1] __UNUSED; \ + break; \ + } \ + } \ + \ + (__typeof__ (*(__ptr))) __nl_val; \ +})) + +#define load_no_speculate(__ptr, __low, __high) \ +(__extension__ ({ \ + __typeof__ ((__ptr)) __ptr_once = (__ptr); \ + __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once); \ +})) + +#define load_no_speculate_fail(__ptr, __low, __high, __failval) \ +(__extension__ ({ \ + __typeof__ ((__ptr)) __ptr_once = (__ptr); \ + __load_no_speculate (__ptr_once, __low, __high, \ + __failval, __ptr_once); \ +})) + +#define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ + (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) + +/* AArch32 support for ARM and Thumb-2. Thumb-1 is not supported. */ +#elif defined (__ARM_32BIT_STATE) && (defined (__thumb2__) || !defined (__thumb__)) +#ifdef __thumb2__ +/* Thumb2 case. */ + +#define __load_no_speculate1(__ptr, __low, __high, __failval, \ + __cmpptr, __sz) \ +(__extension__ ({ \ + __typeof__ (0 + *(__ptr)) __nln_val; \ + __typeof__(*(__ptr)) __fv \ + = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ + /* If __high is explicitly NULL, we must not emit the \ + upper-bound comparison. We need to cast __high to an \ + unsigned long before handing it to __builtin_constant_p to \ + ensure that clang/llvm correctly detects NULL as a constant if it \ + is defined as (void*) 0. */ \ + if (__builtin_constant_p ((unsigned long)__high) \ + && __high == ((void *)0)) \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "bcc\t.ns%=\n\t" \ + "ldr" __sz "\t%[__v], %[__p]\n" \ + ".ns%=:\n\t" \ + "it\tcc\n\t" \ + "movcc\t%[__v], %[__f]\n\t" \ + ".inst.n 0xf3af\t@ CSDB\n\t" \ + ".inst.n 0x8014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&l" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + /* The memory location from which we will load. */ \ + [__p] "m" (*(__ptr)), \ + /* The value to return if the condition check fails. */ \ + [__f] "r" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + else \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "it\tcs\n\t" \ + "cmpcs\t%[__h], %[__c]\n\t" \ + "bls\t.ns%=\n\t" \ + "ldr" __sz "\t%[__v], %[__p]\n" \ + ".ns%=:\n\t" \ + "it\tls\n\t" \ + "movls\t%[__v], %[__f]\n\t" \ + ".inst.n 0xf3af\t@ CSDB\n\t" \ + ".inst.n 0x8014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&l" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + [__h] "r" (__high), \ + /* The memory location from which we will load. */ \ + [__p] "m" (*(__ptr)), \ + /* The value to return if the condition check fails. */ \ + [__f] "r" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + (__typeof__ (*(__ptr))) __nln_val; \ +})) \ + \ +/* Double-word version. */ +#define __load_no_speculate2(__ptr, __low, __high, __failval, \ + __cmpptr) \ +(__extension__ ({ \ + __typeof__ (0 + *(__ptr)) __nln_val; \ + __typeof__(*(__ptr)) __fv \ + = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ + /* If __high is explicitly NULL, we must not emit the \ + upper-bound comparison. We need to cast __high to an \ + unsigned long before handing it to __builtin_constant_p to \ + ensure that clang/llvm correctly detects NULL as a constant if it \ + is defined as (void*) 0. */ \ + if (__builtin_constant_p ((unsigned long)__high) \ + && __high == ((void *)0)) \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "bcc\t.ns%=\n\t" \ + "ldr\t%Q[__v], [%[__p]]\n\t" \ + "ldr\t%R[__v], [%[__p], #4]\n" \ + ".ns%=:\n\t" \ + "it\tcc\n\t" \ + "movcc\t%Q[__v], %Q[__f]\n\t" \ + "it\tcc\n\t" \ + "movcc\t%R[__v], %R[__f]\n\t" \ + ".inst.n 0xf3af\t@ CSDB\n\t" \ + ".inst.n 0x8014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&l" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + /* The memory location from which we will load. */ \ + [__p] "r" (__ptr), \ + /* The value to return if the condition check fails. */ \ + [__f] "r" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + else \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "it\tcs\n\t" \ + "cmpcs\t%[__h], %[__c]\n\t" \ + "bls\t.ns%=\n\t" \ + "ldr\t%Q[__v], [%[__p]]\n\t" \ + "ldr\t%R[__v], [%[__p], #4]\n" \ + ".ns%=:\n\t" \ + "it\tls\n\t" \ + "movls\t%Q[__v], %Q[__f]\n\t" \ + "it\tls\n\t" \ + "movls\t%R[__v], %R[__f]\n\t" \ + ".inst.n 0xf3af\t@ CSDB\n\t" \ + ".inst.n 0x8014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&l" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + [__h] "r" (__high), \ + /* The memory location from which we will load. */ \ + [__p] "r" (__ptr), \ + /* The value to return if the condition check fails. */ \ + [__f] "r" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + (__typeof__ (*(__ptr))) __nln_val; \ +})) + +#else +/* ARM case. */ + +#define __load_no_speculate1(__ptr, __low, __high, __failval, \ + __cmpptr, __sz) \ +(__extension__ ({ \ + __typeof__ (0 + *(__ptr)) __nln_val; \ + __typeof__(*(__ptr)) __fv \ + = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ + /* If __high is explicitly NULL, we must not emit the \ + upper-bound comparison. We need to cast __high to an \ + unsigned long before handing it to __builtin_constant_p to \ + ensure that clang/llvm correctly detects NULL as a constant if it \ + is defined as (void*) 0. */ \ + if (__builtin_constant_p ((unsigned long)__high) \ + && __high == ((void *)0)) \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "ldr" __sz "cs\t%[__v], %[__p]\n\t" \ + "movcc\t%[__v], %[__f]\n\t" \ + ".inst 0xe320f014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&r" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + /* The memory location from which we will load. */ \ + [__p] "m" (*(__ptr)), \ + /* The value to return if the condition check fails. */ \ + [__f] "rKI" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + else \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "cmpcs\t%[__h], %[__c]\n\t" \ + "ldr" __sz "hi\t%[__v], %[__p]\n\t" \ + "movls\t%[__v], %[__f]\n\t" \ + ".inst 0xe320f014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&r" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + [__h] "r" (__high), \ + /* The memory location from which we will load. */ \ + [__p] "m" (*(__ptr)), \ + /* The value to return if the condition check fails. */ \ + [__f] "rKI" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + (__typeof__ (*(__ptr))) __nln_val; \ +})) + +/* Double-word version. */ +#define __load_no_speculate2(__ptr, __low, __high, __failval, \ + __cmpptr) \ +(__extension__ ({ \ + __typeof__ (0 + *(__ptr)) __nln_val; \ + __typeof__(*(__ptr)) __fv \ + = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ + /* If __high is explicitly NULL, we must not emit the \ + upper-bound comparison. We need to cast __high to an \ + unsigned long before handing it to __builtin_constant_p to \ + ensure that clang/llvm correctly detects NULL as a constant if it \ + is defined as (void*) 0. */ \ + if (__builtin_constant_p ((unsigned long)__high) \ + && __high == ((void *)0)) \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "ldrcs\t%Q[__v], [%[__p]]\n\t" \ + "ldrcs\t%R[__v], [%[__p], #4]\n\t" \ + "movcc\t%Q[__v], %Q[__f]\n\t" \ + "movcc\t%R[__v], %R[__f]\n\t" \ + ".inst 0xe320f014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&r" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + /* The memory location from which we will load. */ \ + [__p] "r" (__ptr), \ + /* The value to return if the condition check fails. */ \ + [__f] "r" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + else \ + { \ + __asm__ volatile ( \ + ".syntax unified\n\t" \ + "cmp\t%[__c], %[__l]\n\t" \ + "cmpcs\t%[__h], %[__c]\n\t" \ + "ldrhi\t%Q[__v], [%[__p]]\n\t" \ + "ldrhi\t%R[__v], [%[__p], #4]\n\t" \ + "movls\t%Q[__v], %Q[__f]\n\t" \ + "movls\t%R[__v], %R[__f]\n\t" \ + ".inst 0xe320f014\t@ CSDB" \ + /* The value we have loaded, or failval if the condition check \ + fails. */ \ + : [__v] "=&r" (__nln_val) \ + /* The pointer we wish to use for comparisons, and the low and \ + high bounds to use in that comparison. Note that this need \ + not be the same as the pointer from which we will load. */ \ + : [__c] "r" (__cmpptr), [__l] "r" (__low), \ + [__h] "r" (__high), \ + /* The memory location from which we will load. */ \ + [__p] "r" (__ptr), \ + /* The value to return if the condition check fails. */ \ + [__f] "r" (__fv) \ + /* We always clobber the condition codes. */ \ + : "cc"); \ + } \ + (__typeof__ (*(__ptr))) __nln_val; \ +})) + +#endif // __thumb2__ + +/* Common to ARM and Thumb2. */ + +#define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr) \ +(__extension__ ({ \ + __typeof__ (0 + *(__ptr)) __nl_val; \ + \ + switch (sizeof(*(__ptr))) { \ + case 1: \ + __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ + __failval, __cmpptr, "b"); \ + break; \ + case 2: \ + __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ + __failval, __cmpptr, "h"); \ + break; \ + case 4: \ + __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ + __failval, __cmpptr, ""); \ + break; \ + case 8: \ + __nl_val = __load_no_speculate2 (__ptr, __low, __high, \ + __failval, __cmpptr); \ + break; \ + default: \ + { \ + char __static_assert_no_speculate_load_size_too_big \ + [sizeof (__nl_val) > 8 ? -1 : 1] __UNUSED; \ + break; \ + } \ + } \ + \ + (__typeof__ (*(__ptr))) __nl_val; \ +})) + +#define load_no_speculate(__ptr, __low, __high) \ +(__extension__ ({ \ + __typeof__ ((__ptr)) __ptr_once = (__ptr); \ + __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once); \ +})) + +#define load_no_speculate_fail(__ptr, __low, __high, __failval) \ +(__extension__ ({ \ + __typeof__ ((__ptr)) __ptr_once = (__ptr); \ + __load_no_speculate (__ptr_once, __low, __high, \ + __failval, __ptr_once); \ +})) + +#define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ + (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) + +#else +#error "No fallback provided for load_no_speculate" +#endif + +#endif diff --git a/optee_os/lib/libutils/ext/include/stdlib_ext.h b/optee_os/lib/libutils/ext/include/stdlib_ext.h new file mode 100644 index 0000000..26b5c1e --- /dev/null +++ b/optee_os/lib/libutils/ext/include/stdlib_ext.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +/* + * This file provides extensions to functions defined in + */ + +#ifndef __STDLIB_EXT_H +#define __STDLIB_EXT_H + +#include + +/* Overwrite buffer with a fixed pattern and free it. @ptr may be NULL. */ +void free_wipe(void *ptr); + +#endif /* __STDLIB_EXT_H */ diff --git a/optee_os/lib/libutils/ext/include/string_ext.h b/optee_os/lib/libutils/ext/include/string_ext.h new file mode 100644 index 0000000..0ee908e --- /dev/null +++ b/optee_os/lib/libutils/ext/include/string_ext.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +/* + * This file provides extensions for functions not defined in + */ + +#ifndef __STRING_EXT_H +#define __STRING_EXT_H + +#include +#include + +/* + * Copy src to string dst of siz size. At most siz-1 characters + * will be copied. Always NUL terminates (unless siz == 0). + * Returns strlen(src); if retval >= siz, truncation occurred. + */ +size_t strlcpy(char *dst, const char *src, size_t size); +size_t strlcat(char *dst, const char *src, size_t size); + +/* A constant-time version of memcmp() */ +int consttime_memcmp(const void *p1, const void *p2, size_t nb); + +/* Deprecated. For backward compatibility. */ +static inline int buf_compare_ct(const void *s1, const void *s2, size_t n) +{ + return consttime_memcmp(s1, s2, n); +} + +/* Variant of strdup() that uses nex_malloc() instead of malloc() */ +char *nex_strdup(const char *s); + +/* + * Like memset(s, 0, count) but prevents the compiler from optimizing the call + * away. Such "dead store elimination" optimizations typically occur when + * clearing a *local* variable that is not used after it is cleared; but + * link-time optimization (LTO) can also trigger code elimination in other + * circumstances. See "Dead Store Elimination (Still) Considered Harmful" [1] + * for details and examples (and note that the Cland compiler enables LTO by + * default!). + * + * [1] https://www.usenix.org/system/files/conference/usenixsecurity17/sec17-yang.pdf + * + * Practically speaking: + * + * - Use memzero_explicit() to *clear* (as opposed to initialize) *sensitive* + * data (such as keys, passwords, cryptographic state); + * - Otherwise, use memset(). + */ +void memzero_explicit(void *s, size_t count); + +#endif /* __STRING_EXT_H */ diff --git a/optee_os/lib/libutils/ext/include/trace.h b/optee_os/lib/libutils/ext/include/trace.h new file mode 100644 index 0000000..6f28bdb --- /dev/null +++ b/optee_os/lib/libutils/ext/include/trace.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TRACE_H +#define TRACE_H + +#include +#include +#include +#include +#include + +#define MAX_PRINT_SIZE 256 +#define MAX_FUNC_PRINT_SIZE 32 + +#ifndef TRACE_LEVEL +#define TRACE_LEVEL TRACE_MAX +#endif + +/* + * Symbols provided by the entity that uses this API. + */ +extern int trace_level; +extern const char trace_ext_prefix[]; +void trace_ext_puts(const char *str); +int trace_ext_get_thread_id(void); +int trace_ext_get_core_id(void); +void trace_set_level(int level); +int trace_get_level(void); +void plat_trace_ext_puts(const char *str); + +/* Internal functions used by the macros below */ +void trace_vprintf(const char *func, int line, int level, bool level_ok, + const char *fmt, va_list args) __printf(5, 0); +void trace_printf(const char *func, int line, int level, bool level_ok, + const char *fmt, ...) __printf(5, 6); + +#define trace_printf_helper(level, level_ok, ...) \ + trace_printf(__func__, __LINE__, (level), (level_ok), \ + __VA_ARGS__) + +/* Formatted trace tagged with level independent */ +#if (TRACE_LEVEL <= 0) +#define MSG(...) (void)0 +#else +#define MSG(...) trace_printf_helper(0, false, __VA_ARGS__) +#endif + +/* Formatted trace tagged with TRACE_ERROR level */ +#if (TRACE_LEVEL < TRACE_ERROR) +#define EMSG(...) (void)0 +#else +#define EMSG(...) trace_printf_helper(TRACE_ERROR, true, __VA_ARGS__) +#endif + +/* Formatted trace tagged with TRACE_INFO level */ +#if (TRACE_LEVEL < TRACE_INFO) +#define IMSG(...) (void)0 +#else +#define IMSG(...) trace_printf_helper(TRACE_INFO, true, __VA_ARGS__) +#endif + +/* Formatted trace tagged with TRACE_DEBUG level */ +#if (TRACE_LEVEL < TRACE_DEBUG) +#define DMSG(...) (void)0 +#else +#define DMSG(...) trace_printf_helper(TRACE_DEBUG, true, __VA_ARGS__) +#endif + +/* Formatted trace tagged with TRACE_FLOW level */ +#if (TRACE_LEVEL < TRACE_FLOW) +#define FMSG(...) (void)0 +#else +#define FMSG(...) trace_printf_helper(TRACE_FLOW, true, __VA_ARGS__) +#endif + +/* Formatted trace tagged with TRACE_FLOW level and prefix with '> ' */ +#define INMSG(...) FMSG("> " __VA_ARGS__) +/* Formatted trace tagged with TRACE_FLOW level and prefix with '< ' */ +#define OUTMSG(...) FMSG("< " __VA_ARGS__) +/* Formatted trace tagged with TRACE_FLOW level and prefix with '< ' and print + * an error message if r != 0 */ +#define OUTRMSG(r) \ + do { \ + OUTMSG("r=[%x]", r); \ + return r; \ + } while (0) + +void dhex_dump(const char *function, int line, int level, + const void *buf, int len); +#if (TRACE_LEVEL < TRACE_DEBUG) +#define DHEXDUMP(buf, len) (void)0 +#else +#define DHEXDUMP(buf, len) dhex_dump(__func__, __LINE__, TRACE_DEBUG, \ + buf, len) +#endif + + +/* Trace api without trace formatting */ + +#define trace_printf_helper_raw(level, level_ok, ...) \ + trace_printf(NULL, 0, (level), (level_ok), __VA_ARGS__) + +/* No formatted trace tagged with level independent */ +#if (TRACE_LEVEL <= 0) +#define MSG_RAW(...) (void)0 +#else +#define MSG_RAW(...) trace_printf_helper_raw(0, false, __VA_ARGS__) +#endif + +/* No formatted trace tagged with TRACE_ERROR level */ +#if (TRACE_LEVEL < TRACE_ERROR) +#define EMSG_RAW(...) (void)0 +#else +#define EMSG_RAW(...) trace_printf_helper_raw(TRACE_ERROR, true, __VA_ARGS__) +#endif + +/* No formatted trace tagged with TRACE_INFO level */ +#if (TRACE_LEVEL < TRACE_INFO) +#define IMSG_RAW(...) (void)0 +#else +#define IMSG_RAW(...) trace_printf_helper_raw(TRACE_INFO, true, __VA_ARGS__) +#endif + +/* No formatted trace tagged with TRACE_DEBUG level */ +#if (TRACE_LEVEL < TRACE_DEBUG) +#define DMSG_RAW(...) (void)0 +#else +#define DMSG_RAW(...) trace_printf_helper_raw(TRACE_DEBUG, true, __VA_ARGS__) +#endif + +/* No formatted trace tagged with TRACE_FLOW level */ +#if (TRACE_LEVEL < TRACE_FLOW) +#define FMSG_RAW(...) (void)0 +#else +#define FMSG_RAW(...) trace_printf_helper_raw(TRACE_FLOW, true, __VA_ARGS__) +#endif + +#if (TRACE_LEVEL <= 0) +#define SMSG(...) (void)0 +#else +/* + * Synchronised flushed trace, an Always message straight to HW trace IP. + * Current only supported inside OP-TEE kernel, will be just like an EMSG() + * in another context. + */ +#define SMSG(...) \ + trace_printf(__func__, __LINE__, TRACE_ERROR, true, __VA_ARGS__) + +#endif /* TRACE_LEVEL */ +#endif /* TRACE_H */ diff --git a/optee_os/lib/libutils/ext/include/trace_levels.h b/optee_os/lib/libutils/ext/include/trace_levels.h new file mode 100644 index 0000000..7fd5181 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/trace_levels.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef TRACE_LEVELS_H +#define TRACE_LEVELS_H + +/* + * Trace levels. + * + * ALWAYS is used when you always want a print to be seen, but it is not always + * an error. + * + * ERROR is used when some kind of error has happened, this is most likely the + * print you will use most of the time when you report some kind of error. + * + * INFO is used when you want to print some 'normal' text to the user. + * This is the default level. + * + * DEBUG is used to print extra information to enter deeply in the module. + * + * FLOW is used to print the execution flox, typically the in/out of functions. + * + */ + +#define TRACE_MIN 0 +#define TRACE_ERROR 1 +#define TRACE_INFO 2 +#define TRACE_DEBUG 3 +#define TRACE_FLOW 4 +#define TRACE_MAX TRACE_FLOW + +/* Trace level of the casual printf */ +#define TRACE_PRINTF_LEVEL TRACE_ERROR + +#endif /*TRACE_LEVELS_H*/ diff --git a/optee_os/lib/libutils/ext/include/types_ext.h b/optee_os/lib/libutils/ext/include/types_ext.h new file mode 100644 index 0000000..0bbade5 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/types_ext.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __TYPES_EXT_H +#define __TYPES_EXT_H + +#include +#include +#include +#include +#include + +typedef uintptr_t uaddr_t; +#define PRIxUA PRIxPTR + +typedef uintptr_t vaddr_t; +#define PRIxVA PRIxPTR + +#if defined(__ILP32__) && defined(CFG_CORE_LARGE_PHYS_ADDR) +typedef uint64_t paddr_t; +typedef uint64_t paddr_size_t; +#define PRIxPA PRIx64 +#define PRIxPASZ PRIx64 +#define __SIZEOF_PADDR__ 8 +#else +typedef uintptr_t paddr_t; +typedef uintptr_t paddr_size_t; +#define PRIxPA PRIxPTR +#define PRIxPASZ PRIxPTR +#define __SIZEOF_PADDR__ __SIZEOF_POINTER__ +#endif + +#define PRIxVA_WIDTH ((int)(sizeof(vaddr_t) * 2)) +#define PRIxPA_WIDTH ((int)(sizeof(paddr_t) * 2)) + +#endif /* __TYPES_EXT_H */ diff --git a/optee_os/lib/libutils/ext/include/util.h b/optee_os/lib/libutils/ext/include/util.h new file mode 100644 index 0000000..aedd508 --- /dev/null +++ b/optee_os/lib/libutils/ext/include/util.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef UTIL_H +#define UTIL_H + +#include +#include + +#define SIZE_4K UINTPTR_C(0x1000) +#define SIZE_1M UINTPTR_C(0x100000) +#define SIZE_2M UINTPTR_C(0x200000) +#define SIZE_4M UINTPTR_C(0x400000) +#define SIZE_8M UINTPTR_C(0x800000) +#define SIZE_2G UINTPTR_C(0x80000000) + +#ifndef MAX +#ifndef __ASSEMBLER__ +#define MAX(a, b) \ + (__extension__({ __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + _a > _b ? _a : _b; })) + +#define MIN(a, b) \ + (__extension__({ __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + _a < _b ? _a : _b; })) +#else +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif +#endif + +/* + * In some particular conditions MAX and MIN macros fail to + * build from C source file implmentation. In such case one + * need to use MAX_UNSAFE/MIN_UNSAFE instead. + */ +#define MAX_UNSAFE(a, b) (((a) > (b)) ? (a) : (b)) +#define MIN_UNSAFE(a, b) (((a) < (b)) ? (a) : (b)) + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + +#ifndef __ASSEMBLER__ +/* Round up the even multiple of size, size has to be a multiple of 2 */ +#define ROUNDUP(v, size) (((v) + ((__typeof__(v))(size) - 1)) & \ + ~((__typeof__(v))(size) - 1)) + +#define ROUNDUP_OVERFLOW(v, size, res) (__extension__({ \ + typeof(*(res)) __roundup_tmp = 0; \ + typeof(v) __roundup_mask = (typeof(v))(size) - 1; \ + \ + ADD_OVERFLOW((v), __roundup_mask, &__roundup_tmp) ? 1 : \ + ((void)(*(res) = __roundup_tmp & ~__roundup_mask), 0); \ +})) + +/* + * Rounds up to the nearest multiple of y and then divides by y. Safe + * against overflow, y has to be a multiple of 2. + * + * This macro is intended to be used to convert from "number of bytes" to + * "number of pages" or similar units. Example: + * num_pages = ROUNDUP_DIV(num_bytes, SMALL_PAGE_SIZE); + */ +#define ROUNDUP_DIV(x, y) (__extension__({ \ + typeof(x) __roundup_x = (x); \ + typeof(y) __roundup_mask = (typeof(x))(y) - 1; \ + \ + (__roundup_x / (y)) + (__roundup_x & __roundup_mask ? 1 : 0); \ +})) + +/* Round down the even multiple of size, size has to be a multiple of 2 */ +#define ROUNDDOWN(v, size) ((v) & ~((__typeof__(v))(size) - 1)) + +/* + * Round up the result of x / y to the nearest upper integer if result is not + * already an integer. + */ +#define DIV_ROUND_UP(x, y) (((x) + (y) - 1) / (y)) + +/* Unsigned integer division with nearest rounding variant */ +#define UDIV_ROUND_NEAREST(x, y) \ + (__extension__ ({ __typeof__(x) _x = (x); \ + __typeof__(y) _y = (y); \ + (_x + (_y / 2)) / _y; })) +#else +#define ROUNDUP(x, y) ((((x) + (y) - 1) / (y)) * (y)) +#define ROUNDDOWN(x, y) (((x) / (y)) * (y)) +#define UDIV_ROUND_NEAREST(x, y) (((x) + ((y) / 2)) / (y)) +#endif + +/* x has to be of an unsigned type */ +#define IS_POWER_OF_TWO(x) (((x) != 0) && (((x) & (~(x) + 1)) == (x))) + +#define IS_ALIGNED(x, a) (((x) & ((a) - 1)) == 0) +#define IS_ALIGNED_WITH_TYPE(x, type) \ + (__extension__({ \ + type __is_aligned_y; \ + IS_ALIGNED((uintptr_t)(x), __alignof__(__is_aligned_y)); \ + })) + +#define TO_STR(x) _TO_STR(x) +#define _TO_STR(x) #x + +#define CONCAT(x, y) _CONCAT(x, y) +#define _CONCAT(x, y) x##y + +#define container_of(ptr, type, member) \ + (__extension__({ \ + const typeof(((type *)0)->member) *__ptr = (ptr); \ + (type *)((unsigned long)(__ptr) - offsetof(type, member)); \ + })) + +#define MEMBER_SIZE(type, member) sizeof(((type *)0)->member) + +#ifdef __ASSEMBLER__ +#define BIT32(nr) (1 << (nr)) +#define BIT64(nr) (1 << (nr)) +#define SHIFT_U32(v, shift) ((v) << (shift)) +#define SHIFT_U64(v, shift) ((v) << (shift)) +#else +#define BIT32(nr) (UINT32_C(1) << (nr)) +#define BIT64(nr) (UINT64_C(1) << (nr)) +#define SHIFT_U32(v, shift) ((uint32_t)(v) << (shift)) +#define SHIFT_U64(v, shift) ((uint64_t)(v) << (shift)) +#endif +#define BIT(nr) BIT32(nr) + +/* + * Create a contiguous bitmask starting at bit position @l and ending at + * position @h. For example + * GENMASK_64(39, 21) gives us the 64bit vector 0x000000ffffe00000. + */ +#define GENMASK_32(h, l) \ + (((~UINT32_C(0)) << (l)) & (~UINT32_C(0) >> (32 - 1 - (h)))) + +#define GENMASK_64(h, l) \ + (((~UINT64_C(0)) << (l)) & (~UINT64_C(0) >> (64 - 1 - (h)))) + +/* + * Checking overflow for addition, subtraction and multiplication. Result + * of operation is stored in res which is a pointer to some kind of + * integer. + * + * The macros return true if an overflow occurred and *res is undefined. + */ +#define ADD_OVERFLOW(a, b, res) __compiler_add_overflow((a), (b), (res)) +#define SUB_OVERFLOW(a, b, res) __compiler_sub_overflow((a), (b), (res)) +#define MUL_OVERFLOW(a, b, res) __compiler_mul_overflow((a), (b), (res)) + +/* Return a signed +1, 0 or -1 value based on data comparison */ +#define CMP_TRILEAN(a, b) \ + (__extension__({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + \ + _a > _b ? 1 : _a < _b ? -1 : 0; \ + })) + +#ifndef __ASSEMBLER__ +static inline uint64_t reg_pair_to_64(uint32_t reg0, uint32_t reg1) +{ + return (uint64_t)reg0 << 32 | reg1; +} + +static inline uint32_t high32_from_64(uint64_t val) +{ + return val >> 32; +} + +static inline uint32_t low32_from_64(uint64_t val) +{ + return val; +} + +static inline void reg_pair_from_64(uint64_t val, uint32_t *reg0, + uint32_t *reg1) +{ + *reg0 = high32_from_64(val); + *reg1 = low32_from_64(val); +} + +/* Get and set bit fields */ +static inline uint32_t get_field_u32(uint32_t reg, uint32_t mask) +{ + return (reg & mask) / (mask & ~(mask - 1)); +} + +static inline uint32_t set_field_u32(uint32_t reg, uint32_t mask, uint32_t val) +{ + return (reg & ~mask) | (val * (mask & ~(mask - 1))); +} + +static inline uint64_t get_field_u64(uint64_t reg, uint64_t mask) +{ + return (reg & mask) / (mask & ~(mask - 1)); +} + +static inline uint64_t set_field_u64(uint64_t reg, uint64_t mask, uint64_t val) +{ + return (reg & ~mask) | (val * (mask & ~(mask - 1))); +} +#endif + +#endif /*UTIL_H*/ diff --git a/optee_os/lib/libutils/ext/mempool.c b/optee_os/lib/libutils/ext/mempool.c new file mode 100644 index 0000000..30508aa --- /dev/null +++ b/optee_os/lib/libutils/ext/mempool.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2018-2019, Linaro Limited + */ + + +#include +#include +#include +#include +#include +#include + +#if defined(__KERNEL__) +#include +#include +#endif + +/* + * Allocation of temporary memory buffers which are used in a stack like + * fashion. One exmaple is when a Big Number is needed for a temporary + * variable in a Big Number computation: Big Number operations (add,...), + * crypto algorithms (rsa, ecc,,...). + * + * The allocation algorithm takes memory buffers from a pool, + * characterized by (cf. struct mempool): + * - the total size (in bytes) of the pool + * - the offset of the last item allocated in the pool (struct + * mempool_item). This offset is -1 is nothing is allocated yet. + * + * Each item consists of (struct mempool_item) + * - the size of the item + * - the offsets, in the pool, of the previous and next items + * + * The allocation allocates an item for a given size. + * The allocation is performed in the pool after the last + * allocated items. This means: + * - the heap is never used. + * - there is no assumption on the size of the allocated memory buffers. Only + * the size of the pool will limit the allocation. + * - a constant time allocation and free as there is no list scan + * - but a potentially fragmented memory as the allocation does not take into + * account "holes" in the pool (allocation is performed after the last + * allocated variable). Indeed, this interface is supposed to be used + * with stack like allocations to avoid this issue. This means that + * allocated items: + * - should have a short life cycle + * - if an item A is allocated before another item B, then A should be + * released after B. + * So the potential fragmentation is mitigated. + */ + + +struct mempool { + size_t size; /* size of the memory pool, in bytes */ + vaddr_t data; + struct malloc_ctx *mctx; +#ifdef CFG_MEMPOOL_REPORT_LAST_OFFSET + size_t max_allocated; +#endif +#if defined(__KERNEL__) + void (*release_mem)(void *ptr, size_t size); + struct recursive_mutex mu; +#endif +}; + +#if defined(__KERNEL__) +struct mempool *mempool_default; +#endif + +static void init_mpool(struct mempool *pool) +{ + size_t sz = pool->size - raw_malloc_get_ctx_size(); + vaddr_t v = ROUNDDOWN(pool->data + sz, sizeof(long) * 2); + + /* + * v is the placed as close to the end of the data pool as possible + * where the struct malloc_ctx can be placed. This location is selected + * as an optimization for the pager case to get better data + * locality since raw_malloc() starts to allocate from the end of + * the supplied data pool. + */ + assert(v > pool->data); + pool->mctx = (struct malloc_ctx *)v; + raw_malloc_init_ctx(pool->mctx); + raw_malloc_add_pool(pool->mctx, (void *)pool->data, v - pool->data); +} + +static void get_pool(struct mempool *pool __maybe_unused) +{ +#if defined(__KERNEL__) + mutex_lock_recursive(&pool->mu); + if (!pool->mctx) + init_mpool(pool); + +#endif +} + +static void put_pool(struct mempool *pool __maybe_unused) +{ +#if defined(__KERNEL__) + if (mutex_get_recursive_lock_depth(&pool->mu) == 1) { + /* + * As the refcount is about to become 0 there should be no items + * left + */ + if (pool->release_mem) { + pool->mctx = NULL; + pool->release_mem((void *)pool->data, pool->size); + } + } + mutex_unlock_recursive(&pool->mu); +#endif +} + +struct mempool * +mempool_alloc_pool(void *data, size_t size, + void (*release_mem)(void *ptr, size_t size) __maybe_unused) +{ + struct mempool *pool = calloc(1, sizeof(*pool)); + + COMPILE_TIME_ASSERT(MEMPOOL_ALIGN >= __alignof__(struct mempool_item)); + assert(!((vaddr_t)data & (MEMPOOL_ALIGN - 1))); + + if (pool) { + pool->size = size; + pool->data = (vaddr_t)data; +#if defined(__KERNEL__) + pool->release_mem = release_mem; + mutex_init_recursive(&pool->mu); +#else + init_mpool(pool); +#endif + } + + return pool; +} + +void *mempool_alloc(struct mempool *pool, size_t size) +{ + void *p = NULL; + + get_pool(pool); + + p = raw_malloc(0, 0, size, pool->mctx); + if (p) { +#ifdef CFG_MEMPOOL_REPORT_LAST_OFFSET + struct malloc_stats stats = { }; + + raw_malloc_get_stats(pool->mctx, &stats); + if (stats.max_allocated > pool->max_allocated) { + pool->max_allocated = stats.max_allocated; + DMSG("Max memory usage increased to %zu", + pool->max_allocated); + } +#endif + return p; + } + + EMSG("Failed to allocate %zu bytes, please tune the pool size", size); + put_pool(pool); + return NULL; +} + +void *mempool_calloc(struct mempool *pool, size_t nmemb, size_t size) +{ + size_t sz; + void *p; + + if (MUL_OVERFLOW(nmemb, size, &sz)) + return NULL; + + p = mempool_alloc(pool, sz); + if (p) + memset(p, 0, sz); + + return p; +} + +void mempool_free(struct mempool *pool, void *ptr) +{ + if (ptr) { + raw_free(ptr, pool->mctx, false /*!wipe*/); + put_pool(pool); + } +} diff --git a/optee_os/lib/libutils/ext/memzero_explicit.c b/optee_os/lib/libutils/ext/memzero_explicit.c new file mode 100644 index 0000000..36e8e66 --- /dev/null +++ b/optee_os/lib/libutils/ext/memzero_explicit.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019 Linaro Limited + */ + +#include +#include + +/* + * This method prevents dead store elimination, which could happen in case + * link-time optimization (LTO) is used. + * See "Dead Store Elimination (Still) Considered Harmful" [1] section 3.3.3. + * + * [1] + * http://www.usenix.org/system/files/conference/usenixsecurity17/sec17-yang.pdf + */ +static volatile void * (*memset_func)(void *, int, size_t) = + (volatile void * (*)(void *, int, size_t))&memset; + +void memzero_explicit(void *s, size_t count) +{ + memset_func(s, 0, count); +} diff --git a/optee_os/lib/libutils/ext/nex_strdup.c b/optee_os/lib/libutils/ext/nex_strdup.c new file mode 100644 index 0000000..0710327 --- /dev/null +++ b/optee_os/lib/libutils/ext/nex_strdup.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018 EPAM Systems + */ +#include +#include +#include + +char *nex_strdup(const char *s) +{ + size_t l = strlen(s) + 1; + char *p = nex_malloc(l); + + if (p) + memcpy(p, s, l); + return p; +} diff --git a/optee_os/lib/libutils/ext/pthread_stubs.c b/optee_os/lib/libutils/ext/pthread_stubs.c new file mode 100644 index 0000000..4f5555d --- /dev/null +++ b/optee_os/lib/libutils/ext/pthread_stubs.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2023, Linaro Limited + */ + +#include + +#define pthread_mutex_t void + +int pthread_mutex_lock(pthread_mutex_t *mutex __unused); +int pthread_mutex_unlock(pthread_mutex_t *mutex __unused); + +int __weak pthread_mutex_lock(pthread_mutex_t *mutex __unused) +{ + return 0; +} + +int __weak pthread_mutex_unlock(pthread_mutex_t *mutex __unused) +{ + return 0; +} diff --git a/optee_os/lib/libutils/ext/snprintk.c b/optee_os/lib/libutils/ext/snprintk.c new file mode 100644 index 0000000..c8fa531 --- /dev/null +++ b/optee_os/lib/libutils/ext/snprintk.c @@ -0,0 +1,622 @@ +// SPDX-License-Identifier: (BSD-2-Clause AND BSD-3-Clause) +/* + * Imported from NetBSD 5.1 with modifications to make it a vsnprintf(3) + * function + */ + +/* $NetBSD: subr_prf.c,v 1.156 2014/08/15 11:05:35 apb Exp $ */ + +/*- + * Copyright (c) 1986, 1988, 1991, 1993 + * The Regents of the University of California. All rights reserved. + * (c) UNIX System Laboratories, Inc. + * All or some portions of this file are derived from material licensed + * to the University of California by American Telephone and Telegraph + * Co. or Unix System Laboratories, Inc. and are reproduced herein with + * the permission of UNIX System Laboratories, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)subr_prf.c 8.4 (Berkeley) 5/4/95 + * + * Copyright (c) 2015 Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include + +/* flags for kprintf */ +#define TOCONS 0x0001 /* to the console */ +#define TOTTY 0x0002 /* to the process' tty */ +#define TOLOG 0x0004 /* to the kernel message buffer */ +#define TOBUFONLY 0x0008 /* to the buffer (only) [for snprintk] */ +#define TODDB 0x0010 /* to ddb console */ +#define NOLOCK 0x1000 /* don't acquire a tty lock */ + +/* max size buffer kprintf needs to print a UUID */ +#define KPRINTF_BUFSIZE 37 + +/* + * The following macro is used to remove const cast-away warnings + * from gcc -Wcast-qual; it should be used with caution because it + * can hide valid errors; in particular most valid uses are in + * situations where the API requires it, not to cast away string + * constants. We don't use *intptr_t on purpose here and we are + * explicit about unsigned long so that we don't have additional + * dependencies. + */ +#define __UNCONST(a) ((void *)(unsigned long)(const void *)(a)) + +#define putchar(c, flags, tty) \ + do { (void)(c); (void)(flags); (void)(tty); } while(0) + +static int kprintf(const char *fmt0, int oflags, void *vp, char *sbuf, + va_list ap, bool ext); + +static const char hexdigits[] = "0123456789abcdef"; +static const char HEXDIGITS[] = "0123456789ABCDEF"; + +/* + * snprintk: print a message to a buffer. Same as snprintf but supports + * format extensions. + */ +int +snprintk(char *bf, size_t size, const char *fmt, ...) +{ + int retval; + va_list ap; + + va_start(ap, fmt); + retval = vsnprintk(bf, size, fmt, ap); + va_end(ap); + + return retval; +} + +/* + * vsnprintk: print a message to a buffer [already have va_list] + * Same as vsnprintf but supports format extensions. + */ +int +vsnprintk(char *bf, size_t size, const char *fmt, va_list ap) +{ + return __vsnprintf(bf, size, fmt, ap, true); +} + +int +__vsnprintf(char *bf, size_t size, const char *fmt, va_list ap, + bool ext) + +{ + int retval; + char *p; + + p = bf + size; + retval = kprintf(fmt, TOBUFONLY, &p, bf, ap, ext); + if (bf && size > 0) { + /* nul terminate */ + if (size <= (size_t)retval) + bf[size - 1] = '\0'; + else + bf[retval] = '\0'; + } + return retval; +} + +int __vsprintf(char *bf, const char *fmt, va_list ap) +{ + return kprintf(fmt, TOBUFONLY, NULL, bf, ap, false); +} + +/* + * kprintf: scaled down version of printf(3). + * + * this version based on vfprintf() from libc which was derived from + * software contributed to Berkeley by Chris Torek. + * + */ + +/* + * macros for converting digits to letters and vice versa + */ +#define to_digit(c) ((c) - '0') +#define is_digit(c) ((unsigned)to_digit(c) <= 9) +#define to_char(n) ((n) + '0') + +/* + * flags used during conversion. + */ +#define ALT 0x001 /* alternate form */ +#define HEXPREFIX 0x002 /* add 0x or 0X prefix */ +#define LADJUST 0x004 /* left adjustment */ +#define LONGDBL 0x008 /* long double; unimplemented */ +#define LONGINT 0x010 /* long integer */ +#define QUADINT 0x020 /* quad integer */ +#define SHORTINT 0x040 /* short integer */ +#define MAXINT 0x080 /* intmax_t */ +#define PTRINT 0x100 /* intptr_t */ +#define SIZEINT 0x200 /* size_t */ +#define ZEROPAD 0x400 /* zero (as opposed to blank) pad */ +#define FPT 0x800 /* Floating point number */ + + /* + * To extend shorts properly, we need both signed and unsigned + * argument extraction methods. + */ +#define SARG() \ + (flags&MAXINT ? va_arg(ap, intmax_t) : \ + flags&PTRINT ? va_arg(ap, intptr_t) : \ + flags&SIZEINT ? va_arg(ap, ssize_t) : /* XXX */ \ + flags&QUADINT ? va_arg(ap, int64_t) : \ + flags&LONGINT ? va_arg(ap, long) : \ + flags&SHORTINT ? (long)(short)va_arg(ap, int) : \ + (long)va_arg(ap, int)) +#define UARG() \ + (flags&MAXINT ? va_arg(ap, uintmax_t) : \ + flags&PTRINT ? va_arg(ap, uintptr_t) : \ + flags&SIZEINT ? va_arg(ap, size_t) : \ + flags&QUADINT ? va_arg(ap, uint64_t) : \ + flags&LONGINT ? va_arg(ap, unsigned long) : \ + flags&SHORTINT ? (unsigned long)(unsigned short)va_arg(ap, int) : \ + (unsigned long)va_arg(ap, unsigned int)) + +#define KPRINTF_PUTCHAR(C) { \ + if (oflags == TOBUFONLY) { \ + if (sbuf && ((vp == NULL) || (sbuf < tailp))) \ + *sbuf++ = (C); \ + } else { \ + putchar((C), oflags, vp); \ + } \ +} + +static int uuid2str(char *dst, size_t size, void *ptr) +{ + struct { + uint32_t lo; + uint16_t mid; + uint16_t hi_ver; + uint8_t seq_n[8]; + } *uuid = ptr; + + return snprintk(dst, size, + "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", + uuid->lo, uuid->mid, uuid->hi_ver, + uuid->seq_n[0], uuid->seq_n[1], + uuid->seq_n[2], uuid->seq_n[3], + uuid->seq_n[4], uuid->seq_n[5], + uuid->seq_n[6], uuid->seq_n[7]); +} + +/* + * Guts of kernel printf. Note, we already expect to be in a mutex! + */ +static int +kprintf(const char *fmt0, int oflags, void *vp, char *sbuf, va_list ap, + bool ext) +{ + const char *fmt; /* format string */ + int ch; /* character from fmt */ + int n; /* handy integer (short term usage) */ + char *cp; /* handy char pointer (short term usage) */ + int flags; /* flags as above */ + int ret; /* return value accumulator */ + int width; /* width from format (%8d), or 0 */ + int prec; /* precision from format (%.3d), or -1 */ + char sign; /* sign prefix (' ', '+', '-', or \0) */ + + uint64_t _uquad; /* integer arguments %[diouxX] */ + enum { OCT, DEC, HEX } base;/* base for [diouxX] conversion */ + int dprec; /* a copy of prec if [diouxX], 0 otherwise */ + int realsz; /* field size expanded by dprec */ + int size; /* size of converted field or string */ + const char *xdigs; /* digits for [xX] conversion */ + char bf[KPRINTF_BUFSIZE]; /* space for %c, %[diouxX], possibly %pUl */ + char *tailp; /* tail pointer for snprintk */ + + if (oflags == TOBUFONLY && (vp != NULL)) + tailp = *(char **)vp; + else + tailp = NULL; + + cp = NULL; /* XXX: shutup gcc */ + size = 0; /* XXX: shutup gcc */ + + fmt = fmt0; + ret = 0; + + xdigs = NULL; /* XXX: shut up gcc warning */ + + /* + * Scan the format for conversions (`%' character). + */ + for (;;) { + for (; *fmt != '%' && *fmt; fmt++) { + ret++; + KPRINTF_PUTCHAR(*fmt); + } + if (*fmt == 0) + goto done; + + fmt++; /* skip over '%' */ + + flags = 0; + dprec = 0; + width = 0; + prec = -1; + sign = '\0'; + +rflag: ch = *fmt++; +reswitch: switch (ch) { + case ' ': + /* + * ``If the space and + flags both appear, the space + * flag will be ignored.'' + * -- ANSI X3J11 + */ + if (!sign) + sign = ' '; + goto rflag; + case '#': + flags |= ALT; + goto rflag; + case '*': + /* + * ``A negative field width argument is taken as a + * - flag followed by a positive field width.'' + * -- ANSI X3J11 + * They don't exclude field widths read from args. + */ + if ((width = va_arg(ap, int)) >= 0) + goto rflag; + width = -width; + fallthrough; + case '-': + flags |= LADJUST; + goto rflag; + case '+': + sign = '+'; + goto rflag; + case '.': + if ((ch = *fmt++) == '*') { + n = va_arg(ap, int); + prec = n < 0 ? -1 : n; + goto rflag; + } + n = 0; + while (is_digit(ch)) { + n = 10 * n + to_digit(ch); + ch = *fmt++; + } + prec = n < 0 ? -1 : n; + goto reswitch; + case '0': + /* + * ``Note that 0 is taken as a flag, not as the + * beginning of a field width.'' + * -- ANSI X3J11 + */ + flags |= ZEROPAD; + goto rflag; + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + n = 0; + do { + n = 10 * n + to_digit(ch); + ch = *fmt++; + } while (is_digit(ch)); + width = n; + goto reswitch; + case 'h': + flags |= SHORTINT; + goto rflag; + case 'j': + flags |= MAXINT; + goto rflag; + case 'l': + if (*fmt == 'l') { + fmt++; + flags |= QUADINT; + } else { + flags |= LONGINT; + } + goto rflag; + case 'q': + flags |= QUADINT; + goto rflag; + case 't': + flags |= PTRINT; + goto rflag; + case 'z': + flags |= SIZEINT; + goto rflag; + case 'c': + *(cp = bf) = va_arg(ap, int); + size = 1; + sign = '\0'; + break; + case 'D': + flags |= LONGINT; + fallthrough; + case 'd': + case 'i': + _uquad = SARG(); + if ((int64_t)_uquad < 0) { + _uquad = -_uquad; + sign = '-'; + } + base = DEC; + goto number; + case 'n': + if (flags & MAXINT) + *va_arg(ap, intmax_t *) = ret; + else if (flags & PTRINT) + *va_arg(ap, intptr_t *) = ret; + else if (flags & SIZEINT) + *va_arg(ap, ssize_t *) = ret; + else if (flags & QUADINT) + *va_arg(ap, int64_t *) = ret; + else if (flags & LONGINT) + *va_arg(ap, long *) = ret; + else if (flags & SHORTINT) + *va_arg(ap, short *) = ret; + else + *va_arg(ap, int *) = ret; + continue; /* no output */ + case 'O': + flags |= LONGINT; + fallthrough; + case 'o': + _uquad = UARG(); + base = OCT; + goto nosign; + case 'p': + if (ext && *fmt == 'U' && *(fmt+1) == 'l') { + /* + * Non-standard format available in [v]snprintk + * only + */ + fmt += 2; + size = uuid2str(bf, sizeof(bf), + va_arg(ap, void *)); + cp = bf; + sign = '\0'; + break; + } + /* + * ``The argument shall be a pointer to void. The + * value of the pointer is converted to a sequence + * of printable characters, in an implementation- + * defined manner.'' + * -- ANSI X3J11 + */ + /* NOSTRICT */ + _uquad = (unsigned long)va_arg(ap, void *); + base = HEX; + xdigs = hexdigits; + flags |= HEXPREFIX; + ch = 'x'; + goto nosign; + case 's': + if ((cp = va_arg(ap, char *)) == NULL) + /*XXXUNCONST*/ + cp = __UNCONST("(null)"); + if (prec >= 0) { + /* + * can't use strlen; can only look for the + * NUL in the first `prec' characters, and + * strlen() will go further. + */ + char *p = memchr(cp, 0, prec); + + if (p != NULL) { + size = p - cp; + if (size > prec) + size = prec; + } else + size = prec; + } else + size = strlen(cp); + sign = '\0'; + break; + case 'U': + flags |= LONGINT; + fallthrough; + case 'u': + _uquad = UARG(); + base = DEC; + goto nosign; + case 'X': + xdigs = HEXDIGITS; + goto hex; + case 'x': + xdigs = hexdigits; +hex: _uquad = UARG(); + base = HEX; + /* leading 0x/X only if non-zero */ + if (flags & ALT && _uquad != 0) + flags |= HEXPREFIX; + + /* unsigned conversions */ +nosign: sign = '\0'; + /* + * ``... diouXx conversions ... if a precision is + * specified, the 0 flag will be ignored.'' + * -- ANSI X3J11 + */ +number: if ((dprec = prec) >= 0) + flags &= ~ZEROPAD; + + /* + * ``The result of converting a zero value with an + * explicit precision of zero is no characters.'' + * -- ANSI X3J11 + */ + cp = bf + KPRINTF_BUFSIZE; + if (_uquad != 0 || prec != 0) { + /* + * Unsigned mod is hard, and unsigned mod + * by a constant is easier than that by + * a variable; hence this switch. + */ + switch (base) { + case OCT: + do { + *--cp = to_char(_uquad & 7); + _uquad >>= 3; + } while (_uquad); + /* handle octal leading 0 */ + if (flags & ALT && *cp != '0') + *--cp = '0'; + break; + + case DEC: + /* many numbers are 1 digit */ + while (_uquad >= 10) { + *--cp = to_char(_uquad % 10); + _uquad /= 10; + } + *--cp = to_char(_uquad); + break; + + case HEX: + do { + *--cp = xdigs[_uquad & 15]; + _uquad >>= 4; + } while (_uquad); + break; + + default: + /*XXXUNCONST*/ + cp = __UNCONST("bug in kprintf: bad base"); + size = strlen(cp); + goto skipsize; + } + } + size = bf + KPRINTF_BUFSIZE - cp; + skipsize: + break; + default: /* "%?" prints ?, unless ? is NUL */ + if (ch == '\0') + goto done; + /* pretend it was %c with argument ch */ + cp = bf; + *cp = ch; + size = 1; + sign = '\0'; + break; + } + + /* + * All reasonable formats wind up here. At this point, `cp' + * points to a string which (if not flags&LADJUST) should be + * padded out to `width' places. If flags&ZEROPAD, it should + * first be prefixed by any sign or other prefix; otherwise, + * it should be blank padded before the prefix is emitted. + * After any left-hand padding and prefixing, emit zeroes + * required by a decimal [diouxX] precision, then print the + * string proper, then emit zeroes required by any leftover + * floating precision; finally, if LADJUST, pad with blanks. + * + * Compute actual size, so we know how much to pad. + * size excludes decimal prec; realsz includes it. + */ + realsz = dprec > size ? dprec : size; + if (sign) + realsz++; + else if (flags & HEXPREFIX) + realsz+= 2; + + /* adjust ret */ + ret += width > realsz ? width : realsz; + + /* right-adjusting blank padding */ + if ((flags & (LADJUST|ZEROPAD)) == 0) { + n = width - realsz; + while (n-- > 0) + KPRINTF_PUTCHAR(' '); + } + + /* prefix */ + if (sign) { + KPRINTF_PUTCHAR(sign); + } else if (flags & HEXPREFIX) { + KPRINTF_PUTCHAR('0'); + KPRINTF_PUTCHAR(ch); + } + + /* right-adjusting zero padding */ + if ((flags & (LADJUST|ZEROPAD)) == ZEROPAD) { + n = width - realsz; + while (n-- > 0) + KPRINTF_PUTCHAR('0'); + } + + /* leading zeroes from decimal precision */ + n = dprec - size; + while (n-- > 0) + KPRINTF_PUTCHAR('0'); + + /* the string or number proper */ + for (; size--; cp++) + KPRINTF_PUTCHAR(*cp); + /* left-adjusting padding (always blank) */ + if (flags & LADJUST) { + n = width - realsz; + while (n-- > 0) + KPRINTF_PUTCHAR(' '); + } + } + +done: + if ((oflags == TOBUFONLY) && (vp != NULL)) + *(char **)vp = sbuf; + return ret; +} diff --git a/optee_os/lib/libutils/ext/strlcat.c b/optee_os/lib/libutils/ext/strlcat.c new file mode 100644 index 0000000..3bb09df --- /dev/null +++ b/optee_os/lib/libutils/ext/strlcat.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* This file is copied from newlib-1.19 */ + +/* $OpenBSD: strlcat.c,v 1.8 2001/05/13 15:40:15 deraadt Exp $ */ + +/* + * Copyright (c) 1998 Todd C. Miller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#if defined(LIBC_SCCS) && !defined(lint) +static char *rcsid = + "$OpenBSD: strlcat.c,v 1.8 2001/05/13 15:40:15 deraadt Exp $"; +#endif /* LIBC_SCCS and not lint */ + +#include +#include +/* #include */ + +size_t strlen(const char *s); +/* + * Appends src to string dst of size siz (unlike strncat, siz is the + * full size of dst, not space left). At most siz-1 characters + * will be copied. Always NUL terminates (unless siz <= strlen(dst)). + * Returns strlen(src) + MIN(siz, strlen(initial dst)). + * If retval >= siz, truncation occurred. + */ +size_t strlcat(char *dst, const char *src, size_t siz) +{ + register char *d = dst; + register const char *s = src; + register size_t n = siz; + size_t dlen; + + /* Find the end of dst and adjust bytes left but don't go past end */ + while (n-- != 0 && *d != '\0') + d++; + dlen = d - dst; + n = siz - dlen; + + if (n == 0) + return dlen + strlen(s); + + while (*s != '\0') { + if (n != 1) { + *d++ = *s; + n--; + } + s++; + } + *d = '\0'; + + return dlen + (s - src); /* count does not include NUL */ +} diff --git a/optee_os/lib/libutils/ext/strlcpy.c b/optee_os/lib/libutils/ext/strlcpy.c new file mode 100644 index 0000000..5f3ef06 --- /dev/null +++ b/optee_os/lib/libutils/ext/strlcpy.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* This file is copied from newlib-1.19 */ + +/* $OpenBSD: strlcpy.c,v 1.5 2001/05/13 15:40:16 deraadt Exp $ */ + +/* + * Copyright (c) 1998 Todd C. Miller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#if defined(LIBC_SCCS) && !defined(lint) +static char *rcsid = + "$OpenBSD: strlcpy.c,v 1.5 2001/05/13 15:40:16 deraadt Exp $"; +#endif /* LIBC_SCCS and not lint */ + +#include +#include + +/* + * Copy src to string dst of size siz. At most siz-1 characters + * will be copied. Always NUL terminates (unless siz == 0). + * Returns strlen(src); if retval >= siz, truncation occurred. + */ +size_t strlcpy(char *dst, const char *src, size_t siz) +{ + register char *d = dst; + register const char *s = src; + register size_t n = siz; + + /* Copy as many bytes as will fit */ + if (n != 0 && --n != 0) { + do { + if ((*d++ = *s++) == 0) + break; + } while (--n != 0); + } + + /* Not enough room in dst, add NUL and traverse rest of src */ + if (n == 0) { + if (siz != 0) + *d = '\0'; /* NUL-terminate dst */ + while (*s++) + ; + } + + return s - src - 1; /* count does not include NUL */ +} diff --git a/optee_os/lib/libutils/ext/sub.mk b/optee_os/lib/libutils/ext/sub.mk new file mode 100644 index 0000000..3ab5afd --- /dev/null +++ b/optee_os/lib/libutils/ext/sub.mk @@ -0,0 +1,18 @@ +global-incdirs-y += include + +srcs-y += snprintk.c +srcs-y += strlcat.c +srcs-y += strlcpy.c +srcs-y += trace.c +srcs-y += mempool.c +srcs-y += nex_strdup.c +srcs-y += consttime_memcmp.c +srcs-y += memzero_explicit.c +srcs-y += fault_mitigation.c + +ifneq (,$(filter ta_%,$(sm))) +srcs-y += pthread_stubs.c +endif + +subdirs-y += arch/$(ARCH) +subdirs-y += ftrace diff --git a/optee_os/lib/libutils/ext/trace.c b/optee_os/lib/libutils/ext/trace.c new file mode 100644 index 0000000..33c099d --- /dev/null +++ b/optee_os/lib/libutils/ext/trace.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#if defined(__KERNEL__) +#include +#endif + +#include +#include +#include +#include +#include +#include + +#if (TRACE_LEVEL < TRACE_MIN) || (TRACE_LEVEL > TRACE_MAX) +#error "Invalid value of TRACE_LEVEL" +#endif + +#if (TRACE_LEVEL >= TRACE_ERROR) + +void trace_set_level(int level) +{ + if (((int)level >= TRACE_MIN) && (level <= TRACE_MAX)) + trace_level = level; + else + trace_level = TRACE_MAX; +} + +int trace_get_level(void) +{ + return trace_level; +} + +static char trace_level_to_string(int level, bool level_ok) +{ + /* + * U = Unused + * E = Error + * I = Information + * D = Debug + * F = Flow + */ + static const char lvl_strs[] = { 'U', 'E', 'I', 'D', 'F' }; + int l = 0; + + if (!level_ok) + return 'M'; + + if ((level >= TRACE_MIN) && (level <= TRACE_MAX)) + l = level; + + return lvl_strs[l]; +} + +static int print_thread_id(char *buf, size_t bs) +{ +#if CFG_NUM_THREADS > 100 + int num_thread_digits = 3; +#elif CFG_NUM_THREADS > 10 + int num_thread_digits = 2; +#else + int num_thread_digits = 1; +#endif + int thread_id = trace_ext_get_thread_id(); + + if (thread_id >= 0) + return snprintk(buf, bs, "%0*d ", num_thread_digits, thread_id); + else + return snprintk(buf, bs, "%*s ", num_thread_digits, ""); +} + +#if defined(__KERNEL__) +static int print_core_id(char *buf, size_t bs) +{ +#if CFG_TEE_CORE_NB_CORE > 100 + const int num_digits = 3; + const char qm[] = "???"; +#elif CFG_TEE_CORE_NB_CORE > 10 + const int num_digits = 2; + const char qm[] = "??"; +#else + const int num_digits = 1; + const char qm[] = "?"; +#endif + int core_id = trace_ext_get_core_id(); + + if (core_id >= 0) + return snprintk(buf, bs, "%0*u ", num_digits, core_id); + else + return snprintk(buf, bs, "%s ", qm); +} +#else /* defined(__KERNEL__) */ +static int print_core_id(char *buf __unused, size_t bs __unused) +{ + return 0; +} +#endif + +/* Format trace of user ta. Inline with kernel ta */ +void trace_printf(const char *function, int line, int level, bool level_ok, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + trace_vprintf(function, line, level, level_ok, fmt, ap); + va_end(ap); +} +void trace_vprintf(const char *function, int line, int level, bool level_ok, + const char *fmt, va_list ap) +{ + char buf[MAX_PRINT_SIZE]; + size_t boffs = 0; + int res; + + if (level_ok && level > trace_level) + return; + + /* Print the type of message */ + res = snprintk(buf, sizeof(buf), "%c/", + trace_level_to_string(level, level_ok)); + if (res < 0) + return; + boffs += res; + + /* Print the location, i.e., TEE core or TA */ + res = snprintk(buf + boffs, sizeof(buf) - boffs, "%s:", + trace_ext_prefix); + if (res < 0) + return; + boffs += res; + + if (level_ok && (BIT(level) & CFG_MSG_LONG_PREFIX_MASK)) { + /* Print the core ID if in atomic context */ + res = print_core_id(buf + boffs, sizeof(buf) - boffs); + if (res < 0) + return; + boffs += res; + + /* Print the Thread ID */ + res = print_thread_id(buf + boffs, sizeof(buf) - boffs); + if (res < 0) + return; + boffs += res; + + if (function) { + res = snprintk(buf + boffs, sizeof(buf) - boffs, "%s:%d ", + function, line); + if (res < 0) + return; + boffs += res; + } + } else { + /* Add space after location info */ + if (boffs >= sizeof(buf) - 1) + return; + buf[boffs++] = ' '; + buf[boffs] = 0; + } + + res = vsnprintk(buf + boffs, sizeof(buf) - boffs, fmt, ap); + if (res > 0) + boffs += res; + + if (boffs >= (sizeof(buf) - 1)) + boffs = sizeof(buf) - 2; + + buf[boffs] = '\n'; + while (boffs && buf[boffs] == '\n') + boffs--; + boffs++; + buf[boffs + 1] = '\0'; + + trace_ext_puts(buf); +} + +#else + +/* + * In case we have a zero or negative trace level when compiling optee_os, we + * have to add stubs to trace functions in case they are used with TA having a + * non-zero trace level + */ + +void trace_set_level(int level __unused) +{ +} + +int trace_get_level(void) +{ + return 0; +} + +void trace_printf(const char *function __unused, int line __unused, + int level __unused, bool level_ok __unused, + const char *fmt __unused, ...) +{ +} + +#endif + +#if (TRACE_LEVEL >= TRACE_DEBUG) +struct strbuf { + char buf[MAX_PRINT_SIZE]; + char *ptr; +}; + +static int __printf(2, 3) append(struct strbuf *sbuf, const char *fmt, ...) +{ + int left; + int len; + va_list ap; + + if (sbuf->ptr == NULL) + sbuf->ptr = sbuf->buf; + left = sizeof(sbuf->buf) - (sbuf->ptr - sbuf->buf); + va_start(ap, fmt); + len = vsnprintk(sbuf->ptr, left, fmt, ap); + va_end(ap); + if (len < 0) { + /* Format error */ + return 0; + } + if (len >= left) { + /* Output was truncated */ + return 0; + } + sbuf->ptr += MIN(left, len); + return 1; +} + +void dhex_dump(const char *function, int line, int level, + const void *buf, int len) +{ + int i; + int ok; + struct strbuf sbuf; + char *in = (char *)buf; + + if (level <= trace_level) { + sbuf.ptr = NULL; + for (i = 0; i < len; i++) { + if ((i % 16) == 0) { + ok = append(&sbuf, "%0*" PRIxVA " ", + PRIxVA_WIDTH, (vaddr_t)(in + i)); + if (!ok) + goto err; + } + ok = append(&sbuf, "%02x ", in[i]); + if (!ok) + goto err; + if ((i % 16) == 7) { + ok = append(&sbuf, " "); + if (!ok) + goto err; + } else if ((i % 16) == 15) { + trace_printf(function, line, level, true, "%s", + sbuf.buf); + sbuf.ptr = NULL; + } + } + if (sbuf.ptr) { + /* Buffer is not empty: flush it */ + trace_printf(function, line, level, true, "%s", + sbuf.buf); + + } + } + return; +err: + DMSG("Hex dump error"); +} +#else + +/* + * In case we have trace level less than debug when compiling optee_os, we have + * to add stubs to trace functions in case they are used with TA having a + * a higher trace level + */ + +void dhex_dump(const char *function __unused, int line __unused, + int level __unused, + const void *buf __unused, int len __unused) +{ +} + +#endif diff --git a/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_divmod.c b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_divmod.c new file mode 100644 index 0000000..6335b6b --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_divmod.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +/* + * Form ABI specifications: + * int __aeabi_idiv(int numerator, int denominator); + * unsigned __aeabi_uidiv(unsigned numerator, unsigned denominator); + * + * typedef struct { int quot; int rem; } idiv_return; + * typedef struct { unsigned quot; unsigned rem; } uidiv_return; + * + * __value_in_regs idiv_return __aeabi_idivmod(int numerator, + * int *denominator); + * __value_in_regs uidiv_return __aeabi_uidivmod(unsigned *numerator, + * unsigned denominator); + */ + +#include + +/* struct qr - stores qutient/remainder to handle divmod EABI interfaces. */ +struct qr { + unsigned q; /* computed quotient */ + unsigned r; /* computed remainder */ + unsigned q_n; /* specficies if quotient shall be negative */ + unsigned r_n; /* specficies if remainder shall be negative */ +}; + +static void uint_div_qr(unsigned numerator, unsigned denominator, + struct qr *qr); + +/* returns in R0 and R1 by tail calling an asm function */ +unsigned __aeabi_uidivmod(unsigned numerator, unsigned denominator); + +unsigned __aeabi_uidiv(unsigned numerator, unsigned denominator); + +/* returns in R0 and R1 by tail calling an asm function */ +signed __aeabi_idivmod(signed numerator, signed denominator); + +signed __aeabi_idiv(signed numerator, signed denominator); + +/* + * __ste_idivmod_ret_t __aeabi_idivmod(signed numerator, signed denominator) + * Numerator and Denominator are received in R0 and R1. + * Where __ste_idivmod_ret_t is returned in R0 and R1. + * + * __ste_uidivmod_ret_t __aeabi_uidivmod(unsigned numerator, + * unsigned denominator) + * Numerator and Denominator are received in R0 and R1. + * Where __ste_uidivmod_ret_t is returned in R0 and R1. + */ +#ifdef __GNUC__ +signed ret_idivmod_values(signed quotient, signed remainder); +unsigned ret_uidivmod_values(unsigned quotient, unsigned remainder); +#else +#error "Compiler not supported" +#endif + +static void division_qr(unsigned n, unsigned p, struct qr *qr) +{ + unsigned i = 1, q = 0; + if (p == 0) { + qr->r = 0xFFFFFFFF; /* division by 0 */ + return; + } + + while ((p >> 31) == 0) { + i = i << 1; /* count the max division steps */ + p = p << 1; /* increase p until it has maximum size*/ + } + + while (i > 0) { + q = q << 1; /* write bit in q at index (size-1) */ + if (n >= p) + { + n -= p; + q++; + } + p = p >> 1; /* decrease p */ + i = i >> 1; /* decrease remaining size in q */ + } + qr->r = n; + qr->q = q; +} + +static void uint_div_qr(unsigned numerator, unsigned denominator, struct qr *qr) +{ + + division_qr(numerator, denominator, qr); + + /* negate quotient and/or remainder according to requester */ + if (qr->q_n) + qr->q = -qr->q; + if (qr->r_n) + qr->r = -qr->r; +} + +unsigned __aeabi_uidiv(unsigned numerator, unsigned denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + uint_div_qr(numerator, denominator, &qr); + + return qr.q; +} + +unsigned __no_stackprot __aeabi_uidivmod(unsigned numerator, unsigned denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + uint_div_qr(numerator, denominator, &qr); + + return ret_uidivmod_values(qr.q, qr.r); +} + +signed __aeabi_idiv(signed numerator, signed denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + if (((numerator < 0) && (denominator > 0)) || + ((numerator > 0) && (denominator < 0))) + qr.q_n = 1; /* quotient shall be negate */ + if (numerator < 0) { + numerator = -numerator; + qr.r_n = 1; /* remainder shall be negate */ + } + if (denominator < 0) + denominator = -denominator; + + uint_div_qr(numerator, denominator, &qr); + + return qr.q; +} + +signed __no_stackprot __aeabi_idivmod(signed numerator, signed denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + if (((numerator < 0) && (denominator > 0)) || + ((numerator > 0) && (denominator < 0))) + qr.q_n = 1; /* quotient shall be negate */ + if (numerator < 0) { + numerator = -numerator; + qr.r_n = 1; /* remainder shall be negate */ + } + if (denominator < 0) + denominator = -denominator; + + uint_div_qr(numerator, denominator, &qr); + + return ret_idivmod_values(qr.q, qr.r); +} diff --git a/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_divmod_a32.S b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_divmod_a32.S new file mode 100644 index 0000000..37ae9ec --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_divmod_a32.S @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include + + .section .note.GNU-stack,"",%progbits + +/* + * signed ret_idivmod_values(signed quot, signed rem); + * return quotient and remaining the EABI way (regs r0,r1) + */ +FUNC ret_idivmod_values , : + bx lr +END_FUNC ret_idivmod_values + +/* + * unsigned ret_uidivmod_values(unsigned quot, unsigned rem); + * return quotient and remaining the EABI way (regs r0,r1) + */ +FUNC ret_uidivmod_values , : + bx lr +END_FUNC ret_uidivmod_values diff --git a/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_ldivmod.c b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_ldivmod.c new file mode 100644 index 0000000..043d296 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_ldivmod.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, STMicroelectronics International N.V. + */ + +/* struct lqr - stores qutient/remainder to handle divmod EABI interfaces. */ +struct lqr { + unsigned long long q; /* computed quotient */ + unsigned long long r; /* computed remainder */ + unsigned q_n; /* specficies if quotient shall be negative */ + unsigned r_n; /* specficies if remainder shall be negative */ +}; + +static void ul_div_qr(unsigned long long numerator, + unsigned long long denominator, struct lqr *qr); + + +static void division_lqr(unsigned long long n, unsigned long long p, + struct lqr *qr) +{ + unsigned long long i = 1, q = 0; + if (p == 0) { + qr->r = 0xFFFFFFFFFFFFFFFFULL; /* division by 0 */ + return; + } + + while ((p >> 63) == 0) { + i = i << 1; /* count the max division steps */ + p = p << 1; /* increase p until it has maximum size*/ + } + + while (i > 0) { + q = q << 1; /* write bit in q at index (size-1) */ + if (n >= p) { + n -= p; + q++; + } + p = p >> 1; /* decrease p */ + i = i >> 1; /* decrease remaining size in q */ + } + qr->r = n; + qr->q = q; +} + +static void ul_div_qr(unsigned long long numerator, + unsigned long long denominator, struct lqr *qr) +{ + + division_lqr(numerator, denominator, qr); + + /* negate quotient and/or remainder according to requester */ + if (qr->q_n) + qr->q = -qr->q; + if (qr->r_n) + qr->r = -qr->r; +} + +struct asm_ulqr { + unsigned long long v0; + unsigned long long v1; +}; + +/* called from assembly function __aeabi_uldivmod */ +void __ul_divmod(struct asm_ulqr *asm_ulqr); +void __ul_divmod(struct asm_ulqr *asm_ulqr) +{ + unsigned long long numerator = asm_ulqr->v0; + unsigned long long denominator = asm_ulqr->v1; + struct lqr qr = { .q_n = 0, .r_n = 0 }; + + ul_div_qr(numerator, denominator, &qr); + + asm_ulqr->v0 = qr.q; + asm_ulqr->v1 = qr.r; +} + +struct asm_lqr { + long long v0; + long long v1; +}; + +/* called from assembly function __aeabi_ldivmod */ +void __l_divmod(struct asm_lqr *asm_lqr); +void __l_divmod(struct asm_lqr *asm_lqr) +{ + long long numerator = asm_lqr->v0; + long long denominator = asm_lqr->v1; + struct lqr qr = { .q_n = 0, .r_n = 0 }; + + if (((numerator < 0) && (denominator > 0)) || + ((numerator > 0) && (denominator < 0))) + qr.q_n = 1; /* quotient shall be negate */ + if (numerator < 0) { + numerator = -numerator; + qr.r_n = 1; /* remainder shall be negate */ + } + if (denominator < 0) + denominator = -denominator; + + ul_div_qr(numerator, denominator, &qr); + + asm_lqr->v0 = qr.q; + asm_lqr->v1 = qr.r; +} diff --git a/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_ldivmod_a32.S b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_ldivmod_a32.S new file mode 100644 index 0000000..5c3353e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_ldivmod_a32.S @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include + + .section .note.GNU-stack,"",%progbits + +/* + * __value_in_regs lldiv_t __aeabi_ldivmod( long long n, long long d) + */ +FUNC __aeabi_ldivmod , : + push {ip, lr} +UNWIND( .save {ip, lr}) + push {r0-r3} +UNWIND( .save {r0-r3}) + mov r0, sp + bl __l_divmod + pop {r0-r3} + pop {ip, pc} +END_FUNC __aeabi_ldivmod + +/* + * __value_in_regs ulldiv_t __aeabi_uldivmod( + * unsigned long long n, unsigned long long d) + */ +FUNC __aeabi_uldivmod , : + push {ip, lr} +UNWIND( .save {ip, lr}) + push {r0-r3} +UNWIND( .save {r0-r3}) + mov r0, sp + bl __ul_divmod + pop {r0-r3} + pop {ip, pc} +END_FUNC __aeabi_uldivmod diff --git a/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_shift.c b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_shift.c new file mode 100644 index 0000000..cf68962 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_shift.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +union dword { + unsigned long long dw; + unsigned long w[2]; +}; + +long long __aeabi_llsl(long long a, int shift); +long long __aeabi_llsl(long long a, int shift) +{ + union dword dword = { .dw = a }; + unsigned long hi = dword.w[1]; + unsigned long lo = dword.w[0]; + + if (shift >= 32) { + hi = lo << (shift - 32); + lo = 0; + } else if (shift > 0) { + hi = (hi << shift) | (lo >> (32 - shift)); + lo = lo << shift; + } + + dword.w[1] = hi; + dword.w[0] = lo; + return dword.dw; +} + +long long __aeabi_llsr(long long a, int shift); +long long __aeabi_llsr(long long a, int shift) +{ + union dword dword = { .dw = a }; + unsigned long hi = dword.w[1]; + unsigned long lo = dword.w[0]; + + if (shift >= 32) { + lo = hi >> (shift - 32); + hi = 0; + } else if (shift > 0) { + lo = (lo >> shift) | (hi << (32 - shift)); + hi = hi >> shift; + } + + dword.w[1] = hi; + dword.w[0] = lo; + return dword.dw; +} diff --git a/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_softfloat.c b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_softfloat.c new file mode 100644 index 0000000..ef92ab8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/arm32_aeabi_softfloat.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include "platform.h" +#include + +/* + * On ARM32 EABI defines both a soft-float ABI and a hard-float ABI, + * hard-float is basically a super set of soft-float. Hard-float requires + * all the support routines provided for soft-float, but the compiler may + * choose to optimize to not use some of them. + * + * The AEABI functions uses soft-float calling convention even if the + * functions are compiled for hard-float. So where float and double would + * have been expected we use aeabi_float_t and aeabi_double_t respectively + * instead. + */ +typedef unsigned aeabi_float_t; +typedef unsigned long long aeabi_double_t; + +/* + * Helpers to convert between float32 and aeabi_float_t, and float64 and + * aeabi_double_t used by the AEABI functions below. + */ +static aeabi_float_t f32_to_f(float32_t val) +{ + union { + float32_t from; + aeabi_float_t to; + } res = { .from = val }; + + return res.to; +} + +static float32_t f32_from_f(aeabi_float_t val) +{ + union { + aeabi_float_t from; + float32_t to; + } res = { .from = val }; + + return res.to; +} + +static aeabi_double_t f64_to_d(float64_t val) +{ + union { + float64_t from; + aeabi_double_t to; + } res = { .from = val }; + + return res.to; +} + +static float64_t f64_from_d(aeabi_double_t val) +{ + union { + aeabi_double_t from; + float64_t to; + } res = { .from = val }; + + return res.to; +} + +/* + * From ARM Run-time ABI for ARM Architecture + * ARM IHI 0043D, current through ABI release 2.09 + * + * 4.1.2 The floating-point helper functions + */ + +/* + * Table 2, Standard aeabi_double_t precision floating-point arithmetic helper + * functions + */ + +aeabi_double_t __aeabi_dadd(aeabi_double_t a, aeabi_double_t b) +{ + return f64_to_d(f64_add(f64_from_d(a), f64_from_d(b))); +} + +aeabi_double_t __aeabi_ddiv(aeabi_double_t a, aeabi_double_t b) +{ + return f64_to_d(f64_div(f64_from_d(a), f64_from_d(b))); +} + +aeabi_double_t __aeabi_dmul(aeabi_double_t a, aeabi_double_t b) +{ + return f64_to_d(f64_mul(f64_from_d(a), f64_from_d(b))); +} + + +aeabi_double_t __aeabi_drsub(aeabi_double_t a, aeabi_double_t b) +{ + return f64_to_d(f64_sub(f64_from_d(b), f64_from_d(a))); +} + +aeabi_double_t __aeabi_dsub(aeabi_double_t a, aeabi_double_t b) +{ + return f64_to_d(f64_sub(f64_from_d(a), f64_from_d(b))); +} + +/* + * Table 3, double precision floating-point comparison helper functions + */ + +int __aeabi_dcmpeq(aeabi_double_t a, aeabi_double_t b) +{ + return f64_eq(f64_from_d(a), f64_from_d(b)); +} + +int __aeabi_dcmplt(aeabi_double_t a, aeabi_double_t b) +{ + return f64_lt(f64_from_d(a), f64_from_d(b)); +} + +int __aeabi_dcmple(aeabi_double_t a, aeabi_double_t b) +{ + return f64_le(f64_from_d(a), f64_from_d(b)); +} + +int __aeabi_dcmpge(aeabi_double_t a, aeabi_double_t b) +{ + return f64_le(f64_from_d(b), f64_from_d(a)); +} + +int __aeabi_dcmpgt(aeabi_double_t a, aeabi_double_t b) +{ + return f64_lt(f64_from_d(b), f64_from_d(a)); +} + +/* + * Table 4, Standard single precision floating-point arithmetic helper + * functions + */ + +aeabi_float_t __aeabi_fadd(aeabi_float_t a, aeabi_float_t b) +{ + return f32_to_f(f32_add(f32_from_f(a), f32_from_f(b))); +} + +aeabi_float_t __aeabi_fdiv(aeabi_float_t a, aeabi_float_t b) +{ + return f32_to_f(f32_div(f32_from_f(a), f32_from_f(b))); +} + +aeabi_float_t __aeabi_fmul(aeabi_float_t a, aeabi_float_t b) +{ + return f32_to_f(f32_mul(f32_from_f(a), f32_from_f(b))); +} + +aeabi_float_t __aeabi_frsub(aeabi_float_t a, aeabi_float_t b) +{ + return f32_to_f(f32_sub(f32_from_f(b), f32_from_f(a))); +} + +aeabi_float_t __aeabi_fsub(aeabi_float_t a, aeabi_float_t b) +{ + return f32_to_f(f32_sub(f32_from_f(a), f32_from_f(b))); +} + +/* + * Table 5, Standard single precision floating-point comparison helper + * functions + */ + +int __aeabi_fcmpeq(aeabi_float_t a, aeabi_float_t b) +{ + return f32_eq(f32_from_f(a), f32_from_f(b)); +} + +int __aeabi_fcmplt(aeabi_float_t a, aeabi_float_t b) +{ + return f32_lt(f32_from_f(a), f32_from_f(b)); +} + +int __aeabi_fcmple(aeabi_float_t a, aeabi_float_t b) +{ + return f32_le(f32_from_f(a), f32_from_f(b)); +} + +int __aeabi_fcmpge(aeabi_float_t a, aeabi_float_t b) +{ + return f32_le(f32_from_f(b), f32_from_f(a)); +} + +int __aeabi_fcmpgt(aeabi_float_t a, aeabi_float_t b) +{ + return f32_lt(f32_from_f(b), f32_from_f(a)); +} + +/* + * Table 6, Standard floating-point to integer conversions + */ + +int __aeabi_d2iz(aeabi_double_t a) +{ + return f64_to_i32_r_minMag(f64_from_d(a), false); +} + +unsigned __aeabi_d2uiz(aeabi_double_t a) +{ + return f64_to_ui32_r_minMag(f64_from_d(a), false); +} + +long long __aeabi_d2lz(aeabi_double_t a) +{ + return f64_to_i64_r_minMag(f64_from_d(a), false); +} + +unsigned long long __aeabi_d2ulz(aeabi_double_t a) +{ + return f64_to_ui64_r_minMag(f64_from_d(a), false); +} + +int __aeabi_f2iz(aeabi_float_t a) +{ + return f32_to_i32_r_minMag(f32_from_f(a), false); +} + +unsigned __aeabi_f2uiz(aeabi_float_t a) +{ + return f32_to_ui32_r_minMag(f32_from_f(a), false); +} + +long long __aeabi_f2lz(aeabi_float_t a) +{ + return f32_to_i64_r_minMag(f32_from_f(a), false); +} + +unsigned long long __aeabi_f2ulz(aeabi_float_t a) +{ + return f32_to_ui64_r_minMag(f32_from_f(a), false); +} + +/* + * Table 7, Standard conversions between floating types + */ + +aeabi_float_t __aeabi_d2f(aeabi_double_t a) +{ + return f32_to_f(f64_to_f32(f64_from_d(a))); +} + +aeabi_double_t __aeabi_f2d(aeabi_float_t a) +{ + return f64_to_d(f32_to_f64(f32_from_f(a))); +} + +/* + * Table 8, Standard integer to floating-point conversions + */ + +aeabi_double_t __aeabi_i2d(int a) +{ + return f64_to_d(i32_to_f64(a)); +} + +aeabi_double_t __aeabi_ui2d(unsigned a) +{ + return f64_to_d(ui32_to_f64(a)); +} + +aeabi_double_t __aeabi_l2d(long long a) +{ + return f64_to_d(i64_to_f64(a)); +} + +aeabi_double_t __aeabi_ul2d(unsigned long long a) +{ + return f64_to_d(ui64_to_f64(a)); +} + +aeabi_float_t __aeabi_i2f(int a) +{ + return f32_to_f(i32_to_f32(a)); +} + +aeabi_float_t __aeabi_ui2f(unsigned a) +{ + return f32_to_f(ui32_to_f32(a)); +} + +aeabi_float_t __aeabi_l2f(long long a) +{ + return f32_to_f(i64_to_f32(a)); +} + +aeabi_float_t __aeabi_ul2f(unsigned long long a) +{ + return f32_to_f(ui64_to_f32(a)); +} diff --git a/optee_os/lib/libutils/isoc/arch/arm/setjmp_a32.S b/optee_os/lib/libutils/isoc/arch/arm/setjmp_a32.S new file mode 100644 index 0000000..f8a0b70 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/setjmp_a32.S @@ -0,0 +1,281 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +/* This is a simple version of setjmp and longjmp. + + Nick Clifton, Cygnus Solutions, 13 June 1997. */ + +/* ANSI concatenation macros. */ +#define CONCAT(a, b) CONCAT2(a, b) +#define CONCAT2(a, b) a##b + +#ifndef __USER_LABEL_PREFIX__ +#error __USER_LABEL_PREFIX__ not defined +#endif + +#define SYM(x) CONCAT (__USER_LABEL_PREFIX__, x) + +#ifdef __ELF__ +#define TYPE(x) .type SYM(x),function +#define SIZE(x) .size SYM(x), . - SYM(x) +#else +#define TYPE(x) +#define SIZE(x) +#endif + + .section .note.GNU-stack,"",%progbits + +/* Arm/Thumb interworking support: + + The interworking scheme expects functions to use a BX instruction + to return control to their parent. Since we need this code to work + in both interworked and non-interworked environments as well as with + older processors which do not have the BX instruction we do the + following: + Test the return address. + If the bottom bit is clear perform an "old style" function exit. + (We know that we are in ARM mode and returning to an ARM mode caller). + Otherwise use the BX instruction to perform the function exit. + + We know that we will never attempt to perform the BX instruction on + an older processor, because that kind of processor will never be + interworked, and a return address with the bottom bit set will never + be generated. + + In addition, we do not actually assemble the BX instruction as this would + require us to tell the assembler that the processor is an ARM7TDMI and + it would store this information in the binary. We want this binary to be + able to be linked with binaries compiled for older processors however, so + we do not want such information stored there. + + If we are running using the APCS-26 convention however, then we never + test the bottom bit, because this is part of the processor status. + Instead we just do a normal return, since we know that we cannot be + returning to a Thumb caller - the Thumb does not support APCS-26. + + Function entry is much simpler. If we are compiling for the Thumb we + just switch into ARM mode and then drop through into the rest of the + function. The function exit code will take care of the restore to + Thumb mode. + + For Thumb-2 do everything in Thumb mode. */ + +#if defined(__ARM_ARCH_6M__) +/* ARMv6-M has to be implemented in Thumb mode. */ + +.thumb +.thumb_func + .globl SYM (setjmp) + TYPE (setjmp) +SYM (setjmp): + /* Save registers in jump buffer. */ + stmia r0!, {r4, r5, r6, r7} + mov r1, r8 + mov r2, r9 + mov r3, r10 + mov r4, fp + mov r5, sp + mov r6, lr + stmia r0!, {r1, r2, r3, r4, r5, r6} + sub r0, r0, #40 + /* Restore callee-saved low regs. */ + ldmia r0!, {r4, r5, r6, r7} + /* Return zero. */ + mov r0, #0 + bx lr + +.thumb_func + .globl SYM (longjmp) + TYPE (longjmp) +SYM (longjmp): + /* Restore High regs. */ + add r0, r0, #16 + ldmia r0!, {r2, r3, r4, r5, r6} + mov r8, r2 + mov r9, r3 + mov r10, r4 + mov fp, r5 + mov sp, r6 + ldmia r0!, {r3} /* lr */ + /* Restore low regs. */ + sub r0, r0, #40 + ldmia r0!, {r4, r5, r6, r7} + /* Return the result argument, or 1 if it is zero. */ + mov r0, r1 + bne 1f + mov r0, #1 +1: + bx r3 + +#else + +#ifdef __APCS_26__ +#define RET movs pc, lr +#elif defined(__thumb2__) +#define RET bx lr +#else +#define RET tst lr, #1; \ + moveq pc, lr ; \ +.word 0xe12fff1e /* bx lr */ +#endif + +#ifdef __thumb2__ +.macro COND where when + i\where \when +.endm +#else +.macro COND where when +.endm +#endif + +#if defined(__thumb2__) +.syntax unified +.macro MODE + .thumb + .thumb_func +.endm +.macro PROLOGUE name +.endm + +#elif defined(__thumb__) +#define MODE .thumb_func +.macro PROLOGUE name + .code 16 + bx pc + nop + .code 32 +SYM (.arm_start_of.\name): +.endm +#else /* Arm */ +#define MODE .code 32 +.macro PROLOGUE name +.endm +#endif + +.macro FUNC_START name + .text + .align 2 + MODE + .globl SYM (\name) + TYPE (\name) +SYM (\name): + PROLOGUE \name +.endm + +.macro FUNC_END name + RET + SIZE (\name) +.endm + +/* -------------------------------------------------------------------- + int setjmp (jmp_buf); + -------------------------------------------------------------------- */ + + FUNC_START setjmp + + /* Save all the callee-preserved registers into the jump buffer. */ +#ifdef __thumb2__ + mov ip, sp + stmea a1!, { v1-v7, fp, ip, lr } +#else + stmea a1!, { v1-v7, fp, ip} + str sp, [a1], #4 + str lr, [a1], #4 +#endif + +#if 0 /* Simulator does not cope with FP instructions yet. */ +#ifndef __SOFTFP__ + /* Save the floating point registers. */ + sfmea f4, 4, [a1] +#endif +#endif + +#ifdef CFG_FTRACE_SUPPORT + stmdb sp!, { lr } + /* + * As ftrace is supported in ARM mode only, so hardcode jmp_buf + * offset used to save ftrace return index. + */ + add a1, a1, #48 + bl ftrace_setjmp + ldmia sp!, { lr } +#endif + + /* When setting up the jump buffer return 0. */ + mov a1, #0 + + FUNC_END setjmp + +/* -------------------------------------------------------------------- + volatile void longjmp (jmp_buf, int); + -------------------------------------------------------------------- */ + + FUNC_START longjmp + + /* If we have stack extension code it ought to be handled here. */ + +#ifdef CFG_FTRACE_SUPPORT + stmdb sp!, { a1, a2, lr } + /* + * As ftrace is supported in ARM mode only, so hardcode jmp_buf + * offset used to restore ftrace return stack. + */ + add a1, a1, #92 + bl ftrace_longjmp + ldmia sp!, { a1, a2, lr } +#endif + + /* Restore the registers, retrieving the state when setjmp() was called. */ +#ifdef __thumb2__ + ldmfd a1!, { v1-v7, fp, ip, lr } + mov sp, ip +#else + ldmfd a1!, { v1-v7, fp, ip } + ldr sp, [a1], #4 + ldr lr, [a1], #4 +#endif + +#if 0 /* Simulator does not cope with FP instructions yet. */ +#ifndef __SOFTFP__ + /* Restore floating point registers as well. */ + lfmfd f4, 4, [a1] +#endif +#endif + /* Put the return value into the integer result register. + But if it is zero then return 1 instead. */ + movs a1, a2 +#ifdef __thumb2__ + it eq +#endif + moveq a1, #1 + + FUNC_END longjmp +#endif diff --git a/optee_os/lib/libutils/isoc/arch/arm/setjmp_a64.S b/optee_os/lib/libutils/isoc/arch/arm/setjmp_a64.S new file mode 100644 index 0000000..0b2e068 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/setjmp_a64.S @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + Copyright (c) 2011, 2012 ARM Ltd + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the company may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#define GPR_LAYOUT \ + REG_PAIR (x19, x20, 0); \ + REG_PAIR (x21, x22, 16); \ + REG_PAIR (x23, x24, 32); \ + REG_PAIR (x25, x26, 48); \ + REG_PAIR (x27, x28, 64); \ + REG_PAIR (x29, x30, 80); \ + REG_ONE (x16, 96) + +#define FPR_LAYOUT \ + REG_PAIR ( d8, d9, 112); \ + REG_PAIR (d10, d11, 128); \ + REG_PAIR (d12, d13, 144); \ + REG_PAIR (d14, d15, 160); + +// int setjmp (jmp_buf) + .global setjmp + .type setjmp, %function +setjmp: +BTI( bti c) + mov x16, sp +#define REG_PAIR(REG1, REG2, OFFS) stp REG1, REG2, [x0, OFFS] +#define REG_ONE(REG1, OFFS) str REG1, [x0, OFFS] + GPR_LAYOUT + FPR_LAYOUT +#ifdef CFG_FTRACE_SUPPORT + stp x29, x30, [sp, #-16]! + mov x29, sp + add x0, x0, #104 + bl ftrace_setjmp + ldp x29, x30, [sp], #16 +#endif +#undef REG_PAIR +#undef REG_ONE + mov w0, #0 + ret + .size setjmp, .-setjmp + +// void longjmp (jmp_buf, int) __attribute__ ((noreturn)) + .global longjmp + .type longjmp, %function +longjmp: +BTI( bti c) +#define REG_PAIR(REG1, REG2, OFFS) ldp REG1, REG2, [x0, OFFS] +#define REG_ONE(REG1, OFFS) ldr REG1, [x0, OFFS] +#ifdef CFG_FTRACE_SUPPORT + stp x0, x1, [sp, #-16]! + stp x29, x30, [sp, #-16]! + mov x29, sp + add x0, x0, #104 + bl ftrace_longjmp + ldp x29, x30, [sp], #16 + ldp x0, x1, [sp], #16 +#endif + GPR_LAYOUT + FPR_LAYOUT +#undef REG_PAIR +#undef REG_ONE + mov sp, x16 + cmp w1, #0 + cinc w0, w1, eq +/* + * clang has a bug and doesn't insert bti after setjmp + * causing BTI ecxception. Remove this when the bug is fixed. + * https://bugs.llvm.org/show_bug.cgi?id=49544 + */ +#if defined(__clang__) && defined(CFG_TA_BTI) + ret +#else + // use br not ret, as ret is guaranteed to mispredict + br x30 +#endif + .size longjmp, .-longjmp + +BTI(emit_aarch64_feature_1_and GNU_PROPERTY_AARCH64_FEATURE_1_BTI) diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/COPYING.txt b/optee_os/lib/libutils/isoc/arch/arm/softfloat/COPYING.txt new file mode 100644 index 0000000..869e6da --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/COPYING.txt @@ -0,0 +1,37 @@ + +License for Berkeley SoftFloat Release 3a + +John R. Hauser +2015 October 23 + +The following applies to the whole of SoftFloat Release 3a as well as to +each source file individually. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/README.html b/optee_os/lib/libutils/isoc/arch/arm/softfloat/README.html new file mode 100644 index 0000000..ec64db7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/README.html @@ -0,0 +1,49 @@ + + + + +Berkeley SoftFloat Package Overview + + + + +

Package Overview for Berkeley SoftFloat Release 3a

+ +

+John R. Hauser
+2015 October 23
+

+ +

+Berkeley SoftFloat is a software implementation of binary floating-point that +conforms to the IEEE Standard for Floating-Point Arithmetic. +SoftFloat is distributed in the form of C source code. +Building the SoftFloat sources generates a library file (typically +softfloat.a or libsoftfloat.a) containing the +floating-point subroutines. +

+ +

+The SoftFloat package is documented in the following files in the +doc subdirectory: +

+ + + + + + + + + + + + + +
SoftFloat.htmlDocumentation for using the SoftFloat functions.
SoftFloat-source.htmlDocumentation for building SoftFloat.
SoftFloat-history.html   History of the major changes to SoftFloat.
+
+Other files in the package comprise the source code for SoftFloat. +

+ + + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/README.txt b/optee_os/lib/libutils/isoc/arch/arm/softfloat/README.txt new file mode 100644 index 0000000..0011b11 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/README.txt @@ -0,0 +1,21 @@ + +Package Overview for Berkeley SoftFloat Release 3a + +John R. Hauser +2015 October 23 + +Berkeley SoftFloat is a software implementation of binary floating-point +that conforms to the IEEE Standard for Floating-Point Arithmetic. SoftFloat +is distributed in the form of C source code. Building the SoftFloat sources +generates a library file (typically "softfloat.a" or "libsoftfloat.a") +containing the floating-point subroutines. + +The SoftFloat package is documented in the following files in the "doc" +subdirectory: + + SoftFloat.html Documentation for using the SoftFloat functions. + SoftFloat-source.html Documentation for building SoftFloat. + SoftFloat-history.html History of the major changes to SoftFloat. + +Other files in the package comprise the source code for SoftFloat. + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/arm32_include/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/arm32_include/platform.h new file mode 100644 index 0000000..d8d39da --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/arm32_include/platform.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header template is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#define LITTLEENDIAN 1 +#define INLINE static inline +#define SOFTFLOAT_FAST_DIV64TO32 +#define SOFTFLOAT_FAST_INT64 diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-GCC/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-GCC/Makefile new file mode 100644 index 0000000..62d1e45 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-GCC/Makefile @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile is part of the SoftFloat IEEE Floating-Point Arithmetic +# Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +SOURCE_DIR = ../../source +SPECIALIZE_TYPE = 8086 + +SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +DELETE = rm -f +C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +COMPILE_C = \ + gcc -c -Werror-implicit-function-declaration $(SOFTFLOAT_OPTS) \ + $(C_INCLUDES) -O2 -o $@ +MAKELIB = ar crs $@ + +OBJ = .o +LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_compare96M$(OBJ) \ + s_compare128M$(OBJ) \ + s_shortShiftLeft64To96M$(OBJ) \ + s_shortShiftLeftM$(OBJ) \ + s_shiftLeftM$(OBJ) \ + s_shortShiftRightM$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJamM$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJamM$(OBJ) \ + s_shiftRightM$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_addM$(OBJ) \ + s_addCarryM$(OBJ) \ + s_addComplCarryM$(OBJ) \ + s_negXM$(OBJ) \ + s_sub1XM$(OBJ) \ + s_subM$(OBJ) \ + s_mul64To128M$(OBJ) \ + s_mul128MTo256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + s_remStepMBy32$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80MToCommonNaN$(OBJ) \ + s_commonNaNToExtF80M$(OBJ) \ + s_propagateNaNExtF80M$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128MToCommonNaN$(OBJ) \ + s_commonNaNToF128M$(OBJ) \ + s_propagateNaNF128M$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackMToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackMToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_tryPropagateNaNExtF80M$(OBJ) \ + s_invalidExtF80M$(OBJ) \ + s_normExtF80SigM$(OBJ) \ + s_roundPackMToExtF80M$(OBJ) \ + s_normRoundPackMToExtF80M$(OBJ) \ + s_addExtF80M$(OBJ) \ + s_compareNonnormExtF80M$(OBJ) \ + s_isNaNF128M$(OBJ) \ + s_tryPropagateNaNF128M$(OBJ) \ + s_invalidF128M$(OBJ) \ + s_shiftNormSigF128M$(OBJ) \ + s_roundPackMToF128M$(OBJ) \ + s_normRoundPackMToF128M$(OBJ) \ + s_addF128M$(OBJ) \ + s_mulAddF128M$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-GCC/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-GCC/platform.h new file mode 100644 index 0000000..6121db8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-GCC/platform.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define INLINE extern inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-SSE2-GCC/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-SSE2-GCC/Makefile new file mode 100644 index 0000000..93bc566 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-SSE2-GCC/Makefile @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile is part of the SoftFloat IEEE Floating-Point Arithmetic +# Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +SOURCE_DIR = ../../source +SPECIALIZE_TYPE = 8086-SSE + +SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +DELETE = rm -f +C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +COMPILE_C = \ + gcc -c -Werror-implicit-function-declaration $(SOFTFLOAT_OPTS) \ + $(C_INCLUDES) -O2 -o $@ +MAKELIB = ar crs $@ + +OBJ = .o +LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_compare96M$(OBJ) \ + s_compare128M$(OBJ) \ + s_shortShiftLeft64To96M$(OBJ) \ + s_shortShiftLeftM$(OBJ) \ + s_shiftLeftM$(OBJ) \ + s_shortShiftRightM$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJamM$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJamM$(OBJ) \ + s_shiftRightM$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_addM$(OBJ) \ + s_addCarryM$(OBJ) \ + s_addComplCarryM$(OBJ) \ + s_negXM$(OBJ) \ + s_sub1XM$(OBJ) \ + s_subM$(OBJ) \ + s_mul64To128M$(OBJ) \ + s_mul128MTo256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + s_remStepMBy32$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80MToCommonNaN$(OBJ) \ + s_commonNaNToExtF80M$(OBJ) \ + s_propagateNaNExtF80M$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128MToCommonNaN$(OBJ) \ + s_commonNaNToF128M$(OBJ) \ + s_propagateNaNF128M$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackMToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackMToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_tryPropagateNaNExtF80M$(OBJ) \ + s_invalidExtF80M$(OBJ) \ + s_normExtF80SigM$(OBJ) \ + s_roundPackMToExtF80M$(OBJ) \ + s_normRoundPackMToExtF80M$(OBJ) \ + s_addExtF80M$(OBJ) \ + s_compareNonnormExtF80M$(OBJ) \ + s_isNaNF128M$(OBJ) \ + s_tryPropagateNaNF128M$(OBJ) \ + s_invalidF128M$(OBJ) \ + s_shiftNormSigF128M$(OBJ) \ + s_roundPackMToF128M$(OBJ) \ + s_normRoundPackMToF128M$(OBJ) \ + s_addF128M$(OBJ) \ + s_mulAddF128M$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-SSE2-GCC/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-SSE2-GCC/platform.h new file mode 100644 index 0000000..6121db8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-386-SSE2-GCC/platform.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define INLINE extern inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-x86_64-GCC/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-x86_64-GCC/Makefile new file mode 100644 index 0000000..63f8c2f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-x86_64-GCC/Makefile @@ -0,0 +1,336 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile is part of the SoftFloat IEEE Floating-Point Arithmetic +# Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +SOURCE_DIR = ../../source +SPECIALIZE_TYPE = 8086-SSE + +SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +DELETE = rm -f +C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +COMPILE_C = \ + gcc -c -Werror-implicit-function-declaration -DSOFTFLOAT_FAST_INT64 \ + $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ +MAKELIB = ar crs $@ + +OBJ = .o +LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_eq128$(OBJ) \ + s_le128$(OBJ) \ + s_lt128$(OBJ) \ + s_shortShiftLeft128$(OBJ) \ + s_shortShiftRight128$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJam64Extra$(OBJ) \ + s_shortShiftRightJam128$(OBJ) \ + s_shortShiftRightJam128Extra$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJam64Extra$(OBJ) \ + s_shiftRightJam128$(OBJ) \ + s_shiftRightJam128Extra$(OBJ) \ + s_shiftRightJam256M$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_add128$(OBJ) \ + s_add256M$(OBJ) \ + s_sub128$(OBJ) \ + s_sub256M$(OBJ) \ + s_mul64ByShifted32To128$(OBJ) \ + s_mul64To128$(OBJ) \ + s_mul128By32$(OBJ) \ + s_mul128To256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80UIToCommonNaN$(OBJ) \ + s_commonNaNToExtF80UI$(OBJ) \ + s_propagateNaNExtF80UI$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128UIToCommonNaN$(OBJ) \ + s_commonNaNToF128UI$(OBJ) \ + s_propagateNaNF128UI$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_normSubnormalExtF80Sig$(OBJ) \ + s_roundPackToExtF80$(OBJ) \ + s_normRoundPackToExtF80$(OBJ) \ + s_addMagsExtF80$(OBJ) \ + s_subMagsExtF80$(OBJ) \ + s_normSubnormalF128Sig$(OBJ) \ + s_roundPackToF128$(OBJ) \ + s_normRoundPackToF128$(OBJ) \ + s_addMagsF128$(OBJ) \ + s_subMagsF128$(OBJ) \ + s_mulAddF128$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80_to_ui32$(OBJ) \ + extF80_to_ui64$(OBJ) \ + extF80_to_i32$(OBJ) \ + extF80_to_i64$(OBJ) \ + extF80_to_ui32_r_minMag$(OBJ) \ + extF80_to_ui64_r_minMag$(OBJ) \ + extF80_to_i32_r_minMag$(OBJ) \ + extF80_to_i64_r_minMag$(OBJ) \ + extF80_to_f32$(OBJ) \ + extF80_to_f64$(OBJ) \ + extF80_to_f128$(OBJ) \ + extF80_roundToInt$(OBJ) \ + extF80_add$(OBJ) \ + extF80_sub$(OBJ) \ + extF80_mul$(OBJ) \ + extF80_div$(OBJ) \ + extF80_rem$(OBJ) \ + extF80_sqrt$(OBJ) \ + extF80_eq$(OBJ) \ + extF80_le$(OBJ) \ + extF80_lt$(OBJ) \ + extF80_eq_signaling$(OBJ) \ + extF80_le_quiet$(OBJ) \ + extF80_lt_quiet$(OBJ) \ + extF80_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128_to_ui32$(OBJ) \ + f128_to_ui64$(OBJ) \ + f128_to_i32$(OBJ) \ + f128_to_i64$(OBJ) \ + f128_to_ui32_r_minMag$(OBJ) \ + f128_to_ui64_r_minMag$(OBJ) \ + f128_to_i32_r_minMag$(OBJ) \ + f128_to_i64_r_minMag$(OBJ) \ + f128_to_f32$(OBJ) \ + f128_to_extF80$(OBJ) \ + f128_to_f64$(OBJ) \ + f128_roundToInt$(OBJ) \ + f128_add$(OBJ) \ + f128_sub$(OBJ) \ + f128_mul$(OBJ) \ + f128_mulAdd$(OBJ) \ + f128_div$(OBJ) \ + f128_rem$(OBJ) \ + f128_sqrt$(OBJ) \ + f128_eq$(OBJ) \ + f128_le$(OBJ) \ + f128_lt$(OBJ) \ + f128_eq_signaling$(OBJ) \ + f128_le_quiet$(OBJ) \ + f128_lt_quiet$(OBJ) \ + f128_isSignalingNaN$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-x86_64-GCC/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-x86_64-GCC/platform.h new file mode 100644 index 0000000..6121db8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Linux-x86_64-GCC/platform.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define INLINE extern inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-MinGW/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-MinGW/Makefile new file mode 100644 index 0000000..62d1e45 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-MinGW/Makefile @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile is part of the SoftFloat IEEE Floating-Point Arithmetic +# Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +SOURCE_DIR = ../../source +SPECIALIZE_TYPE = 8086 + +SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +DELETE = rm -f +C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +COMPILE_C = \ + gcc -c -Werror-implicit-function-declaration $(SOFTFLOAT_OPTS) \ + $(C_INCLUDES) -O2 -o $@ +MAKELIB = ar crs $@ + +OBJ = .o +LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_compare96M$(OBJ) \ + s_compare128M$(OBJ) \ + s_shortShiftLeft64To96M$(OBJ) \ + s_shortShiftLeftM$(OBJ) \ + s_shiftLeftM$(OBJ) \ + s_shortShiftRightM$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJamM$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJamM$(OBJ) \ + s_shiftRightM$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_addM$(OBJ) \ + s_addCarryM$(OBJ) \ + s_addComplCarryM$(OBJ) \ + s_negXM$(OBJ) \ + s_sub1XM$(OBJ) \ + s_subM$(OBJ) \ + s_mul64To128M$(OBJ) \ + s_mul128MTo256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + s_remStepMBy32$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80MToCommonNaN$(OBJ) \ + s_commonNaNToExtF80M$(OBJ) \ + s_propagateNaNExtF80M$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128MToCommonNaN$(OBJ) \ + s_commonNaNToF128M$(OBJ) \ + s_propagateNaNF128M$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackMToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackMToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_tryPropagateNaNExtF80M$(OBJ) \ + s_invalidExtF80M$(OBJ) \ + s_normExtF80SigM$(OBJ) \ + s_roundPackMToExtF80M$(OBJ) \ + s_normRoundPackMToExtF80M$(OBJ) \ + s_addExtF80M$(OBJ) \ + s_compareNonnormExtF80M$(OBJ) \ + s_isNaNF128M$(OBJ) \ + s_tryPropagateNaNF128M$(OBJ) \ + s_invalidF128M$(OBJ) \ + s_shiftNormSigF128M$(OBJ) \ + s_roundPackMToF128M$(OBJ) \ + s_normRoundPackMToF128M$(OBJ) \ + s_addF128M$(OBJ) \ + s_mulAddF128M$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-MinGW/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-MinGW/platform.h new file mode 100644 index 0000000..6121db8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-MinGW/platform.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define INLINE extern inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-SSE2-MinGW/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-SSE2-MinGW/Makefile new file mode 100644 index 0000000..93bc566 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-SSE2-MinGW/Makefile @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile is part of the SoftFloat IEEE Floating-Point Arithmetic +# Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +SOURCE_DIR = ../../source +SPECIALIZE_TYPE = 8086-SSE + +SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +DELETE = rm -f +C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +COMPILE_C = \ + gcc -c -Werror-implicit-function-declaration $(SOFTFLOAT_OPTS) \ + $(C_INCLUDES) -O2 -o $@ +MAKELIB = ar crs $@ + +OBJ = .o +LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_compare96M$(OBJ) \ + s_compare128M$(OBJ) \ + s_shortShiftLeft64To96M$(OBJ) \ + s_shortShiftLeftM$(OBJ) \ + s_shiftLeftM$(OBJ) \ + s_shortShiftRightM$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJamM$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJamM$(OBJ) \ + s_shiftRightM$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_addM$(OBJ) \ + s_addCarryM$(OBJ) \ + s_addComplCarryM$(OBJ) \ + s_negXM$(OBJ) \ + s_sub1XM$(OBJ) \ + s_subM$(OBJ) \ + s_mul64To128M$(OBJ) \ + s_mul128MTo256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + s_remStepMBy32$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80MToCommonNaN$(OBJ) \ + s_commonNaNToExtF80M$(OBJ) \ + s_propagateNaNExtF80M$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128MToCommonNaN$(OBJ) \ + s_commonNaNToF128M$(OBJ) \ + s_propagateNaNF128M$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackMToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackMToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_tryPropagateNaNExtF80M$(OBJ) \ + s_invalidExtF80M$(OBJ) \ + s_normExtF80SigM$(OBJ) \ + s_roundPackMToExtF80M$(OBJ) \ + s_normRoundPackMToExtF80M$(OBJ) \ + s_addExtF80M$(OBJ) \ + s_compareNonnormExtF80M$(OBJ) \ + s_isNaNF128M$(OBJ) \ + s_tryPropagateNaNF128M$(OBJ) \ + s_invalidF128M$(OBJ) \ + s_shiftNormSigF128M$(OBJ) \ + s_roundPackMToF128M$(OBJ) \ + s_normRoundPackMToF128M$(OBJ) \ + s_addF128M$(OBJ) \ + s_mulAddF128M$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-SSE2-MinGW/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-SSE2-MinGW/platform.h new file mode 100644 index 0000000..6121db8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win32-SSE2-MinGW/platform.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define INLINE extern inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win64-MinGW-w64/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win64-MinGW-w64/Makefile new file mode 100644 index 0000000..ac16ed9 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win64-MinGW-w64/Makefile @@ -0,0 +1,336 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile is part of the SoftFloat IEEE Floating-Point Arithmetic +# Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +SOURCE_DIR = ../../source +SPECIALIZE_TYPE = 8086-SSE + +SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +DELETE = rm -f +C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +COMPILE_C = \ + x86_64-w64-mingw32-gcc -c -Werror-implicit-function-declaration \ + -DSOFTFLOAT_FAST_INT64 $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ +MAKELIB = x86_64-w64-mingw32-ar crs $@ + +OBJ = .o +LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_eq128$(OBJ) \ + s_le128$(OBJ) \ + s_lt128$(OBJ) \ + s_shortShiftLeft128$(OBJ) \ + s_shortShiftRight128$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJam64Extra$(OBJ) \ + s_shortShiftRightJam128$(OBJ) \ + s_shortShiftRightJam128Extra$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJam64Extra$(OBJ) \ + s_shiftRightJam128$(OBJ) \ + s_shiftRightJam128Extra$(OBJ) \ + s_shiftRightJam256M$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_add128$(OBJ) \ + s_add256M$(OBJ) \ + s_sub128$(OBJ) \ + s_sub256M$(OBJ) \ + s_mul64ByShifted32To128$(OBJ) \ + s_mul64To128$(OBJ) \ + s_mul128By32$(OBJ) \ + s_mul128To256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80UIToCommonNaN$(OBJ) \ + s_commonNaNToExtF80UI$(OBJ) \ + s_propagateNaNExtF80UI$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128UIToCommonNaN$(OBJ) \ + s_commonNaNToF128UI$(OBJ) \ + s_propagateNaNF128UI$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_normSubnormalExtF80Sig$(OBJ) \ + s_roundPackToExtF80$(OBJ) \ + s_normRoundPackToExtF80$(OBJ) \ + s_addMagsExtF80$(OBJ) \ + s_subMagsExtF80$(OBJ) \ + s_normSubnormalF128Sig$(OBJ) \ + s_roundPackToF128$(OBJ) \ + s_normRoundPackToF128$(OBJ) \ + s_addMagsF128$(OBJ) \ + s_subMagsF128$(OBJ) \ + s_mulAddF128$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80_to_ui32$(OBJ) \ + extF80_to_ui64$(OBJ) \ + extF80_to_i32$(OBJ) \ + extF80_to_i64$(OBJ) \ + extF80_to_ui32_r_minMag$(OBJ) \ + extF80_to_ui64_r_minMag$(OBJ) \ + extF80_to_i32_r_minMag$(OBJ) \ + extF80_to_i64_r_minMag$(OBJ) \ + extF80_to_f32$(OBJ) \ + extF80_to_f64$(OBJ) \ + extF80_to_f128$(OBJ) \ + extF80_roundToInt$(OBJ) \ + extF80_add$(OBJ) \ + extF80_sub$(OBJ) \ + extF80_mul$(OBJ) \ + extF80_div$(OBJ) \ + extF80_rem$(OBJ) \ + extF80_sqrt$(OBJ) \ + extF80_eq$(OBJ) \ + extF80_le$(OBJ) \ + extF80_lt$(OBJ) \ + extF80_eq_signaling$(OBJ) \ + extF80_le_quiet$(OBJ) \ + extF80_lt_quiet$(OBJ) \ + extF80_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128_to_ui32$(OBJ) \ + f128_to_ui64$(OBJ) \ + f128_to_i32$(OBJ) \ + f128_to_i64$(OBJ) \ + f128_to_ui32_r_minMag$(OBJ) \ + f128_to_ui64_r_minMag$(OBJ) \ + f128_to_i32_r_minMag$(OBJ) \ + f128_to_i64_r_minMag$(OBJ) \ + f128_to_f32$(OBJ) \ + f128_to_extF80$(OBJ) \ + f128_to_f64$(OBJ) \ + f128_roundToInt$(OBJ) \ + f128_add$(OBJ) \ + f128_sub$(OBJ) \ + f128_mul$(OBJ) \ + f128_mulAdd$(OBJ) \ + f128_div$(OBJ) \ + f128_rem$(OBJ) \ + f128_sqrt$(OBJ) \ + f128_eq$(OBJ) \ + f128_le$(OBJ) \ + f128_lt$(OBJ) \ + f128_eq_signaling$(OBJ) \ + f128_le_quiet$(OBJ) \ + f128_lt_quiet$(OBJ) \ + f128_isSignalingNaN$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win64-MinGW-w64/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win64-MinGW-w64/platform.h new file mode 100644 index 0000000..6121db8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/Win64-MinGW-w64/platform.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define INLINE extern inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-FAST_INT64/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-FAST_INT64/Makefile new file mode 100644 index 0000000..5be1ca0 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-FAST_INT64/Makefile @@ -0,0 +1,337 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile template is part of the SoftFloat IEEE Floating-Point +# Arithmetic Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +# Edit lines marked with `==>'. See "SoftFloat-source.html". + +==> SOURCE_DIR = ../../source +==> SPECIALIZE_TYPE = 8086 + +==> SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +==> DELETE = rm -f +==> C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +==> COMPILE_C = \ +==> cc -c -DSOFTFLOAT_FAST_INT64 $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ +==> MAKELIB = ar crs $@ + +==> OBJ = .o +==> LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_eq128$(OBJ) \ + s_le128$(OBJ) \ + s_lt128$(OBJ) \ + s_shortShiftLeft128$(OBJ) \ + s_shortShiftRight128$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJam64Extra$(OBJ) \ + s_shortShiftRightJam128$(OBJ) \ + s_shortShiftRightJam128Extra$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJam64Extra$(OBJ) \ + s_shiftRightJam128$(OBJ) \ + s_shiftRightJam128Extra$(OBJ) \ + s_shiftRightJam256M$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_add128$(OBJ) \ + s_add256M$(OBJ) \ + s_sub128$(OBJ) \ + s_sub256M$(OBJ) \ + s_mul64ByShifted32To128$(OBJ) \ + s_mul64To128$(OBJ) \ + s_mul128By32$(OBJ) \ + s_mul128To256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80UIToCommonNaN$(OBJ) \ + s_commonNaNToExtF80UI$(OBJ) \ + s_propagateNaNExtF80UI$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128UIToCommonNaN$(OBJ) \ + s_commonNaNToF128UI$(OBJ) \ + s_propagateNaNF128UI$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_normSubnormalExtF80Sig$(OBJ) \ + s_roundPackToExtF80$(OBJ) \ + s_normRoundPackToExtF80$(OBJ) \ + s_addMagsExtF80$(OBJ) \ + s_subMagsExtF80$(OBJ) \ + s_normSubnormalF128Sig$(OBJ) \ + s_roundPackToF128$(OBJ) \ + s_normRoundPackToF128$(OBJ) \ + s_addMagsF128$(OBJ) \ + s_subMagsF128$(OBJ) \ + s_mulAddF128$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80_to_ui32$(OBJ) \ + extF80_to_ui64$(OBJ) \ + extF80_to_i32$(OBJ) \ + extF80_to_i64$(OBJ) \ + extF80_to_ui32_r_minMag$(OBJ) \ + extF80_to_ui64_r_minMag$(OBJ) \ + extF80_to_i32_r_minMag$(OBJ) \ + extF80_to_i64_r_minMag$(OBJ) \ + extF80_to_f32$(OBJ) \ + extF80_to_f64$(OBJ) \ + extF80_to_f128$(OBJ) \ + extF80_roundToInt$(OBJ) \ + extF80_add$(OBJ) \ + extF80_sub$(OBJ) \ + extF80_mul$(OBJ) \ + extF80_div$(OBJ) \ + extF80_rem$(OBJ) \ + extF80_sqrt$(OBJ) \ + extF80_eq$(OBJ) \ + extF80_le$(OBJ) \ + extF80_lt$(OBJ) \ + extF80_eq_signaling$(OBJ) \ + extF80_le_quiet$(OBJ) \ + extF80_lt_quiet$(OBJ) \ + extF80_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128_to_ui32$(OBJ) \ + f128_to_ui64$(OBJ) \ + f128_to_i32$(OBJ) \ + f128_to_i64$(OBJ) \ + f128_to_ui32_r_minMag$(OBJ) \ + f128_to_ui64_r_minMag$(OBJ) \ + f128_to_i32_r_minMag$(OBJ) \ + f128_to_i64_r_minMag$(OBJ) \ + f128_to_f32$(OBJ) \ + f128_to_extF80$(OBJ) \ + f128_to_f64$(OBJ) \ + f128_roundToInt$(OBJ) \ + f128_add$(OBJ) \ + f128_sub$(OBJ) \ + f128_mul$(OBJ) \ + f128_mulAdd$(OBJ) \ + f128_div$(OBJ) \ + f128_rem$(OBJ) \ + f128_sqrt$(OBJ) \ + f128_eq$(OBJ) \ + f128_le$(OBJ) \ + f128_lt$(OBJ) \ + f128_eq_signaling$(OBJ) \ + f128_le_quiet$(OBJ) \ + f128_lt_quiet$(OBJ) \ + f128_isSignalingNaN$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-FAST_INT64/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-FAST_INT64/platform.h new file mode 100644 index 0000000..cea3332 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-FAST_INT64/platform.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header template is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +// Edit lines marked with `==>'. See "SoftFloat-source.html". + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +==> #define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +==> #define INLINE inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-not-FAST_INT64/Makefile b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-not-FAST_INT64/Makefile new file mode 100644 index 0000000..e827222 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-not-FAST_INT64/Makefile @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: BSD-3-Clause + +#============================================================================= +# +# This Makefile template is part of the SoftFloat IEEE Floating-Point +# Arithmetic Package, Release 3a, by John R. Hauser. +# +# Copyright 2011, 2012, 2013, 2014 The Regents of the University of +# California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the University nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= + +# Edit lines marked with `==>'. See "SoftFloat-source.html". + +==> SOURCE_DIR = ../../source +==> SPECIALIZE_TYPE = 8086 + +==> SOFTFLOAT_OPTS = -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV64TO32 + +==> DELETE = rm -f +==> C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +==> COMPILE_C = cc -c $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ +==> MAKELIB = ar crs $@ + +==> OBJ = .o +==> LIB = .a + +.PHONY: all +all: softfloat$(LIB) + +OBJS_PRIMITIVES = \ + s_compare96M$(OBJ) \ + s_compare128M$(OBJ) \ + s_shortShiftLeft64To96M$(OBJ) \ + s_shortShiftLeftM$(OBJ) \ + s_shiftLeftM$(OBJ) \ + s_shortShiftRightM$(OBJ) \ + s_shortShiftRightJam64$(OBJ) \ + s_shortShiftRightJamM$(OBJ) \ + s_shiftRightJam32$(OBJ) \ + s_shiftRightJam64$(OBJ) \ + s_shiftRightJamM$(OBJ) \ + s_shiftRightM$(OBJ) \ + s_countLeadingZeros8$(OBJ) \ + s_countLeadingZeros32$(OBJ) \ + s_countLeadingZeros64$(OBJ) \ + s_addM$(OBJ) \ + s_addCarryM$(OBJ) \ + s_addComplCarryM$(OBJ) \ + s_negXM$(OBJ) \ + s_sub1XM$(OBJ) \ + s_subM$(OBJ) \ + s_mul64To128M$(OBJ) \ + s_mul128MTo256M$(OBJ) \ + s_approxRecip32_1$(OBJ) \ + s_approxRecipSqrt32_1$(OBJ) \ + s_remStepMBy32$(OBJ) \ + +OBJS_SPECIALIZE = \ + softfloat_raiseFlags$(OBJ) \ + s_f32UIToCommonNaN$(OBJ) \ + s_commonNaNToF32UI$(OBJ) \ + s_propagateNaNF32UI$(OBJ) \ + s_f64UIToCommonNaN$(OBJ) \ + s_commonNaNToF64UI$(OBJ) \ + s_propagateNaNF64UI$(OBJ) \ + extF80M_isSignalingNaN$(OBJ) \ + s_extF80MToCommonNaN$(OBJ) \ + s_commonNaNToExtF80M$(OBJ) \ + s_propagateNaNExtF80M$(OBJ) \ + f128M_isSignalingNaN$(OBJ) \ + s_f128MToCommonNaN$(OBJ) \ + s_commonNaNToF128M$(OBJ) \ + s_propagateNaNF128M$(OBJ) \ + +OBJS_OTHERS = \ + s_roundPackToUI32$(OBJ) \ + s_roundPackMToUI64$(OBJ) \ + s_roundPackToI32$(OBJ) \ + s_roundPackMToI64$(OBJ) \ + s_normSubnormalF32Sig$(OBJ) \ + s_roundPackToF32$(OBJ) \ + s_normRoundPackToF32$(OBJ) \ + s_addMagsF32$(OBJ) \ + s_subMagsF32$(OBJ) \ + s_mulAddF32$(OBJ) \ + s_normSubnormalF64Sig$(OBJ) \ + s_roundPackToF64$(OBJ) \ + s_normRoundPackToF64$(OBJ) \ + s_addMagsF64$(OBJ) \ + s_subMagsF64$(OBJ) \ + s_mulAddF64$(OBJ) \ + s_tryPropagateNaNExtF80M$(OBJ) \ + s_invalidExtF80M$(OBJ) \ + s_normExtF80SigM$(OBJ) \ + s_roundPackMToExtF80M$(OBJ) \ + s_normRoundPackMToExtF80M$(OBJ) \ + s_addExtF80M$(OBJ) \ + s_compareNonnormExtF80M$(OBJ) \ + s_isNaNF128M$(OBJ) \ + s_tryPropagateNaNF128M$(OBJ) \ + s_invalidF128M$(OBJ) \ + s_shiftNormSigF128M$(OBJ) \ + s_roundPackMToF128M$(OBJ) \ + s_normRoundPackMToF128M$(OBJ) \ + s_addF128M$(OBJ) \ + s_mulAddF128M$(OBJ) \ + softfloat_state$(OBJ) \ + ui32_to_f32$(OBJ) \ + ui32_to_f64$(OBJ) \ + ui32_to_extF80M$(OBJ) \ + ui32_to_f128M$(OBJ) \ + ui64_to_f32$(OBJ) \ + ui64_to_f64$(OBJ) \ + ui64_to_extF80M$(OBJ) \ + ui64_to_f128M$(OBJ) \ + i32_to_f32$(OBJ) \ + i32_to_f64$(OBJ) \ + i32_to_extF80M$(OBJ) \ + i32_to_f128M$(OBJ) \ + i64_to_f32$(OBJ) \ + i64_to_f64$(OBJ) \ + i64_to_extF80M$(OBJ) \ + i64_to_f128M$(OBJ) \ + f32_to_ui32$(OBJ) \ + f32_to_ui64$(OBJ) \ + f32_to_i32$(OBJ) \ + f32_to_i64$(OBJ) \ + f32_to_ui32_r_minMag$(OBJ) \ + f32_to_ui64_r_minMag$(OBJ) \ + f32_to_i32_r_minMag$(OBJ) \ + f32_to_i64_r_minMag$(OBJ) \ + f32_to_f64$(OBJ) \ + f32_to_extF80M$(OBJ) \ + f32_to_f128M$(OBJ) \ + f32_roundToInt$(OBJ) \ + f32_add$(OBJ) \ + f32_sub$(OBJ) \ + f32_mul$(OBJ) \ + f32_mulAdd$(OBJ) \ + f32_div$(OBJ) \ + f32_rem$(OBJ) \ + f32_sqrt$(OBJ) \ + f32_eq$(OBJ) \ + f32_le$(OBJ) \ + f32_lt$(OBJ) \ + f32_eq_signaling$(OBJ) \ + f32_le_quiet$(OBJ) \ + f32_lt_quiet$(OBJ) \ + f32_isSignalingNaN$(OBJ) \ + f64_to_ui32$(OBJ) \ + f64_to_ui64$(OBJ) \ + f64_to_i32$(OBJ) \ + f64_to_i64$(OBJ) \ + f64_to_ui32_r_minMag$(OBJ) \ + f64_to_ui64_r_minMag$(OBJ) \ + f64_to_i32_r_minMag$(OBJ) \ + f64_to_i64_r_minMag$(OBJ) \ + f64_to_f32$(OBJ) \ + f64_to_extF80M$(OBJ) \ + f64_to_f128M$(OBJ) \ + f64_roundToInt$(OBJ) \ + f64_add$(OBJ) \ + f64_sub$(OBJ) \ + f64_mul$(OBJ) \ + f64_mulAdd$(OBJ) \ + f64_div$(OBJ) \ + f64_rem$(OBJ) \ + f64_sqrt$(OBJ) \ + f64_eq$(OBJ) \ + f64_le$(OBJ) \ + f64_lt$(OBJ) \ + f64_eq_signaling$(OBJ) \ + f64_le_quiet$(OBJ) \ + f64_lt_quiet$(OBJ) \ + f64_isSignalingNaN$(OBJ) \ + extF80M_to_ui32$(OBJ) \ + extF80M_to_ui64$(OBJ) \ + extF80M_to_i32$(OBJ) \ + extF80M_to_i64$(OBJ) \ + extF80M_to_ui32_r_minMag$(OBJ) \ + extF80M_to_ui64_r_minMag$(OBJ) \ + extF80M_to_i32_r_minMag$(OBJ) \ + extF80M_to_i64_r_minMag$(OBJ) \ + extF80M_to_f32$(OBJ) \ + extF80M_to_f64$(OBJ) \ + extF80M_to_f128M$(OBJ) \ + extF80M_roundToInt$(OBJ) \ + extF80M_add$(OBJ) \ + extF80M_sub$(OBJ) \ + extF80M_mul$(OBJ) \ + extF80M_div$(OBJ) \ + extF80M_rem$(OBJ) \ + extF80M_sqrt$(OBJ) \ + extF80M_eq$(OBJ) \ + extF80M_le$(OBJ) \ + extF80M_lt$(OBJ) \ + extF80M_eq_signaling$(OBJ) \ + extF80M_le_quiet$(OBJ) \ + extF80M_lt_quiet$(OBJ) \ + f128M_to_ui32$(OBJ) \ + f128M_to_ui64$(OBJ) \ + f128M_to_i32$(OBJ) \ + f128M_to_i64$(OBJ) \ + f128M_to_ui32_r_minMag$(OBJ) \ + f128M_to_ui64_r_minMag$(OBJ) \ + f128M_to_i32_r_minMag$(OBJ) \ + f128M_to_i64_r_minMag$(OBJ) \ + f128M_to_f32$(OBJ) \ + f128M_to_f64$(OBJ) \ + f128M_to_extF80M$(OBJ) \ + f128M_roundToInt$(OBJ) \ + f128M_add$(OBJ) \ + f128M_sub$(OBJ) \ + f128M_mul$(OBJ) \ + f128M_mulAdd$(OBJ) \ + f128M_div$(OBJ) \ + f128M_rem$(OBJ) \ + f128M_sqrt$(OBJ) \ + f128M_eq$(OBJ) \ + f128M_le$(OBJ) \ + f128M_lt$(OBJ) \ + f128M_eq_signaling$(OBJ) \ + f128M_le_quiet$(OBJ) \ + f128M_lt_quiet$(OBJ) \ + +OBJS_ALL = $(OBJS_PRIMITIVES) $(OBJS_SPECIALIZE) $(OBJS_OTHERS) + +$(OBJS_ALL): \ + platform.h $(SOURCE_DIR)/include/primitiveTypes.h \ + $(SOURCE_DIR)/include/primitives.h +$(OBJS_SPECIALIZE) $(OBJS_OTHERS): \ + $(SOURCE_DIR)/include/softfloat_types.h $(SOURCE_DIR)/include/internals.h \ + $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/specialize.h \ + $(SOURCE_DIR)/include/softfloat.h + +$(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$*.c + +$(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +softfloat$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + +.PHONY: clean +clean: + $(DELETE) $(OBJS_ALL) softfloat$(LIB) + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-not-FAST_INT64/platform.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-not-FAST_INT64/platform.h new file mode 100644 index 0000000..cea3332 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/build/template-not-FAST_INT64/platform.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header template is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +// Edit lines marked with `==>'. See "SoftFloat-source.html". + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +==> #define LITTLEENDIAN 1 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +==> #define INLINE inline + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat-history.html b/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat-history.html new file mode 100644 index 0000000..08cab39 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat-history.html @@ -0,0 +1,155 @@ + + + + +Berkeley SoftFloat History + + + + +

History of Berkeley SoftFloat, to Release 3a

+ +

+John R. Hauser
+2015 October 23
+

+ + +

Release 3a (2015 October)

+ +
    + +
  • +Replaced the license text supplied by the University of California, Berkeley. + +
+ + +

Release 3 (2015 February)

+ +
    + +
  • +Complete rewrite, funded by the University of California, Berkeley, and +consequently having a different use license than earlier releases. +Major changes included renaming most types and functions, upgrading some +algorithms, restructuring the source files, and making SoftFloat into a true +library. + +
  • +Added functions to convert between floating-point and unsigned integers, both +32-bit and 64-bit (uint32_t and +uint64_t). + +
  • +Added functions for fused multiply-add, for all supported floating-point +formats except 80-bit double-extended-precision. + +
  • +Added support for a fifth rounding mode, near_maxMag (round to +nearest, with ties to maximum magnitude, away from zero). + +
  • +Dropped the timesoftfloat program (now part of the Berkeley +TestFloat package). + +
+ + +

Release 2c (2015 January)

+ +
    + +
  • +Fixed mistakes affecting some 64-bit processors. + +
  • +Further improved the documentation and the wording for the legal restrictions +on using SoftFloat releases through 2c (not applicable to +Release 3 or later). + +
+ + +

Release 2b (2002 May)

+ +
    + +
  • +Made minor updates to the documentation, including improved wording for the +legal restrictions on using SoftFloat. + +
+ + +

Release 2a (1998 December)

+ +
    + +
  • +Added functions to convert between 64-bit integers +(int64) and all supported floating-point formats. + +
  • +Fixed a bug in all 64-bit-version square root functions except +float32_sqrt that caused the result sometimes to be off by +1 unit in the last place (1 ulp) from what it should +be. +(Bug discovered by Paul Donahue.) + +
  • +Improved the Makefiles. +
+ + +

Release 2 (1997 June)

+ +
    + +
  • +Created the 64-bit (bits64) version, adding the +floatx80 and float128 formats. + +
  • +Changed the source directory structure, splitting the sources into a +bits32 and a bits64 version. +Renamed environment.h to milieu.h to avoid confusion +with environment variables. + +
  • +Fixed a small error that caused float64_round_to_int often to +round the wrong way in nearest/even mode when the operand was between +220 and 221 and halfway between two integers. + +
+ + +

Release 1a (1996 July)

+ +
    + +
  • +Corrected a mistake that caused borderline underflow cases not to raise the +underflow flag when they should have. +(Problem reported by Doug Priest.) + +
  • +Added the float_detect_tininess variable to control whether +tininess is detected before or after rounding. + +
+ + +

Release 1 (1996 July)

+ +
    + +
  • +Original release, based on work done for the International Computer Science +Institute (ICSI) in Berkeley, California. + +
+ + + + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat-source.html b/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat-source.html new file mode 100644 index 0000000..dff77aa --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat-source.html @@ -0,0 +1,590 @@ + + + + +Berkeley SoftFloat Source Documentation + + + + +

Berkeley SoftFloat Release 3a: Source Documentation

+ +

+John R. Hauser
+2015 October 23
+

+ + +

Contents

+ +
+ +++ + + + + + + + + + + + + + + + + + + + +
1. Introduction
2. Limitations
3. Acknowledgments and License
4. SoftFloat Package Directory Structure
5. Issues for Porting SoftFloat to a New Target
5.1. Standard Headers <stdbool.h> and + <stdint.h>
5.2. Specializing Floating-Point Behavior
5.3. Macros for Build Options
5.4. Adapting a Template Target Directory
5.5. Target-Specific Optimization of Primitive Functions
6. Testing SoftFloat
7. Providing SoftFloat as a Common Library for Applications
8. Contact Information
+
+ + +

1. Introduction

+ +

+This document gives information needed for compiling and/or porting Berkeley +SoftFloat, a library of C functions implementing binary floating-point +conforming to the IEEE Standard for Floating-Point Arithmetic. +For basic documentation about SoftFloat refer to +SoftFloat.html. +

+ +

+The source code for SoftFloat is intended to be relatively machine-independent +and should be compilable with any ISO-Standard C compiler that also supports +64-bit integers. +SoftFloat has been successfully compiled with the GNU C Compiler +(gcc) for several platforms. +

+ +

+Release 3 of SoftFloat was a complete rewrite relative to +Release 2 or earlier. +Changes to the interface of SoftFloat functions are documented in +SoftFloat.html. +The current version of SoftFloat is Release 3a. +

+ + +

2. Limitations

+ +

+SoftFloat assumes the computer has an addressable byte size of either 8 or +16 bits. +(Nearly all computers in use today have 8-bit bytes.) +

+ +

+SoftFloat is written in C and is designed to work with other C code. +The C compiler used must conform at a minimum to the 1989 ANSI standard for the +C language (same as the 1990 ISO standard) and must in addition support basic +arithmetic on 64-bit integers. +Earlier releases of SoftFloat included implementations of 32-bit +single-precision and 64-bit double-precision floating-point that +did not require 64-bit integers, but this option is not supported +starting with Release 3. +Since 1999, ISO standards for C have mandated compiler support for +64-bit integers. +A compiler conforming to the 1999 C Standard or later is recommended but not +strictly required. +

+ +

+C Standard header files <stdbool.h> and +<stdint.h> are required for defining standard Boolean and +integer types. +If these headers are not supplied with the C compiler, minimal substitutes must +be provided. +SoftFloat’s dependence on these headers is detailed later in +section 5.1, Standard Headers <stdbool.h> and +<stdint.h>. +

+ + +

3. Acknowledgments and License

+ +

+The SoftFloat package was written by me, John R. Hauser. +Release 3 of SoftFloat was a completely new implementation +supplanting earlier releases. +The project to create Release 3 (and now 3a) was done +in the employ of the University of California, Berkeley, within the Department +of Electrical Engineering and Computer Sciences, first for the Parallel +Computing Laboratory (Par Lab) and then for the ASPIRE Lab. +The work was officially overseen by Prof. Krste Asanovic, with funding provided +by these sources: +

+ ++++ + + + + + + + + + +
Par Lab: +Microsoft (Award #024263), Intel (Award #024894), and U.C. Discovery +(Award #DIG07-10227), with additional support from Par Lab affiliates Nokia, +NVIDIA, Oracle, and Samsung. +
ASPIRE Lab: +DARPA PERFECT program (Award #HR0011-12-2-0016), with additional support from +ASPIRE industrial sponsor Intel and ASPIRE affiliates Google, Nokia, NVIDIA, +Oracle, and Samsung. +
+
+

+ +

+The following applies to the whole of SoftFloat Release 3a as well +as to each source file individually. +

+ +

+Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. +All rights reserved. +

+ +

+Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +

    + +
  1. +

    +Redistributions of source code must retain the above copyright notice, this +list of conditions, and the following disclaimer. +

    + +
  2. +

    +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions, and the following disclaimer in the documentation and/or +other materials provided with the distribution. +

    + +
  3. +

    +Neither the name of the University nor the names of its contributors may be +used to endorse or promote products derived from this software without specific +prior written permission. +

    + +
+

+ +

+THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS “AS IS”, +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. +IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +

+ + +

4. SoftFloat Package Directory Structure

+ +

+Because SoftFloat is targeted to multiple platforms, its source code is +slightly scattered between target-specific and target-independent directories +and files. +The supplied directory structure is as follows: +

+
+doc
+source
+    include
+    8086
+    8086-SSE
+build
+    template-FAST_INT64
+    template-not-FAST_INT64
+    Linux-386-GCC
+    Linux-386-SSE2-GCC
+    Linux-x86_64-GCC
+    Win32-MinGW
+    Win32-SSE2-MinGW
+    Win64-MinGW-w64
+
+
+The majority of the SoftFloat sources are provided in the source +directory. +The include subdirectory of source contains several +header files (unsurprisingly), while the 8086 and +8086-SSE subdirectories contain source files that +specialize the floating-point behavior to match the Intel x86 line of +processors. +The files in directory 8086 give floating-point behavior +consistent solely with Intel’s older, 8087-derived floating-point, while +those in 8086-SSE update the behavior of the +non-extended formats (float32_t, float64_t, and +float128_t) to mirror Intel’s more recent Streaming SIMD +Extensions (SSE) and other compatible extensions. +If other specializations are attempted, these would be expected to be other +subdirectories of source alongside 8086 and +8086-SSE. +Specialization is covered later, in section 5.2, Specializing +Floating-Point Behavior. +

+ +

+The build directory is intended to contain a subdirectory for each +target platform for which a build of the SoftFloat library may be created. +For each build target, the target’s subdirectory is where all derived +object files and the completed SoftFloat library (typically +softfloat.a or libsoftfloat.a) are created. +The two template subdirectories are not actual build targets but +contain sample files for creating new target directories. +(The meaning of FAST_INT64 will be explained later.) +

+ +

+Ignoring the template directories, the supplied target directories +are intended to follow a naming system of +<execution-environment>-<compiler>. +For the example targets, +<execution-environment> is +Linux-386, Linux-386-SSE2, +Linux-x86_64, Win32, +Win32-SSE2, or Win64, and +<compiler> is GCC, +MinGW, or MinGW-w64. +

+ +

+As supplied, each target directory contains two files: +

+
+Makefile
+platform.h
+
+
+The provided Makefile is written for GNU make. +A build of SoftFloat for the specific target is begun by executing the +make command with the target directory as the current directory. +A completely different build tool can be used if an appropriate +Makefile equivalent is created. +

+ +

+The platform.h header file exists to provide a location for +additional C declarations specific to the build target. +Every C source file of SoftFloat contains a #include for +platform.h. +In many cases, the contents of platform.h can be as simple as one +or two lines of code. +At the other extreme, to get maximal performance from SoftFloat, it may be +desirable to include in header platform.h (directly or via +#include) declarations for numerous target-specific optimizations. +Such possibilities are discussed in the next section, Issues for Porting +SoftFloat to a New Target. +If the target’s compiler or library has bugs or other shortcomings, +workarounds for these issues may also be possible with target-specific +declarations in platform.h, avoiding the need to modify the main +SoftFloat sources. +

+ + +

5. Issues for Porting SoftFloat to a New Target

+ +

5.1. Standard Headers <stdbool.h> and <stdint.h>

+ +

+The SoftFloat sources make use of standard headers +<stdbool.h> and <stdint.h>, which have +been part of the ISO C Standard Library since 1999. +With any recent compiler, these standard headers are likely to be supported, +even if the compiler does not claim complete conformance to the latest ISO C +Standard. +For older or nonstandard compilers, substitutes for +<stdbool.h> and <stdint.h> may need to be +created. +SoftFloat depends on these names from <stdbool.h>: +

+
+bool
+true
+false
+
+
+and on these names from <stdint.h>: +
+
+uint16_t
+uint32_t
+uint64_t
+int32_t
+int64_t
+UINT64_C
+INT64_C
+uint_least8_t
+uint_fast8_t
+uint_fast16_t
+uint_fast32_t
+uint_fast64_t
+int_fast8_t
+int_fast16_t
+int_fast32_t
+int_fast64_t
+
+
+

+ + +

5.2. Specializing Floating-Point Behavior

+ +

+The IEEE Floating-Point Standard allows for some flexibility in a conforming +implementation, particularly concerning NaNs. +The SoftFloat source directory is supplied with some +specialization subdirectories containing possible definitions for this +implementation-specific behavior. +For example, the 8086 and 8086-SSE +subdirectories have source files that specialize SoftFloat’s behavior to +match that of Intel’s x86 line of processors. +The files in a specialization subdirectory must determine: +

    +
  • +whether tininess for underflow is detected before or after rounding by default; +
  • +what (if anything) special happens when exceptions are raised; +
  • +how signaling NaNs are distinguished from quiet NaNs; +
  • +the default generated quiet NaNs; and +
  • +how NaNs are propagated from function inputs to output. +
+

+ +

+As provided, the build process for a target expects to involve exactly +one specialization directory that defines all of these +implementation-specific details for the target. +A specialization directory such as 8086 is expected to contain a +header file called specialize.h, together with whatever other +source files are needed to complete the specialization. +

+ +

+A new build target may use an existing specialization, such as the ones +provided by the 8086 and 8086-SSE +subdirectories. +If a build target needs a new specialization, different from any existing ones, +it is recommended that a new specialization subdirectory be created in the +source directory for this purpose. +The specialize.h header file from any of the provided +specialization subdirectories can be used as a model for what definitions are +needed. +

+ + +

5.3. Macros for Build Options

+ +

+The SoftFloat source files adapt the floating-point implementation according to +a few C preprocessor macros: +

+
+
LITTLEENDIAN +
+Must be defined for little-endian machines; must not be defined for big-endian +machines. +
SOFTFLOAT_FAST_INT64 +
+Can be defined to indicate that the build target’s implementation of +64-bit arithmetic is efficient. +For newer 64-bit processors, this macro should usually be defined. +For very small microprocessors whose buses and registers are 8-bit +or 16-bit in size, this macro should usually not be defined. +Whether this macro should be defined for a 32-bit processor may +depend on the target machine and the applications that will use SoftFloat. +
SOFTFLOAT_FAST_DIV64TO32 +
+Can be defined to indicate that the target’s division operator +in C (written as /) is reasonably efficient for +dividing a 64-bit unsigned integer by a 32-bit +unsigned integer. +Setting this macro may affect the performance of division, remainder, and +square root operations. +
INLINE_LEVEL +
+Can be defined to an integer to determine the degree of inlining requested of +the compiler. +Larger numbers request that more inlining be done. +If this macro is not defined or is defined to a value less than 1 +(zero or negative), no inlining is requested. +The maximum effective value is no higher than 5. +Defining this macro to a value greater than 5 is the same as defining it +to 5. +
INLINE +
+Specifies the sequence of tokens used to indicate that a C function should be +inlined. +If macro INLINE_LEVEL is defined with a value of 1 or higher, this +macro must be defined; otherwise, this macro is ignored and need not be +defined. +For some compilers, this macro can be defined as the single keyword +inline. +Historically, the gcc compiler has required that this macro be +defined to extern inline. +
+
+

+ +

+Following the usual custom for C, for the first three macros (all +except INLINE_LEVEL and INLINE), the content of any +definition is irrelevant; +what matters is a macro’s effect on #ifdef directives. +

+ +

+It is recommended that any definitions of macros LITTLEENDIAN and +INLINE be made in a build target’s platform.h +header file, because these macros are expected to be determined inflexibly by +the target machine and compiler. +The other three macros control optimization and might be better located in the +target’s Makefile (or its equivalent). +

+ + +

5.4. Adapting a Template Target Directory

+ +

+In the build directory, two template subdirectories +provide models for new target directories. +Two different templates exist because different functions are needed in the +SoftFloat library depending on whether macro SOFTFLOAT_FAST_INT64 +is defined. +If macro SOFTFLOAT_FAST_INT64 will be defined, +template-FAST_INT64 is the template to use; +otherwise, template-not-FAST_INT64 is the appropriate +template. +A new target directory can be created by copying the correct template directory +and editing the files inside. +To avoid confusion, it would be wise to refrain from editing the files within a +template directory directly. +

+ + +

5.5. Target-Specific Optimization of Primitive Functions

+ +

+Header file primitives.h (in directory +source/include) declares macros and functions for numerous +underlying arithmetic operations upon which many of SoftFloat’s +floating-point functions are ultimately built. +The SoftFloat sources include implementations of all of these functions/macros, +written as standard C code, so a complete and correct SoftFloat library can be +built using only the supplied code for all functions. +However, for many targets, SoftFloat’s performance can be improved by +substituting target-specific implementations of some of the functions/macros +declared in primitives.h. +

+ +

+For example, primitives.h declares a function called +softfloat_countLeadingZeros32 that takes an unsigned +32-bit integer as an argument and returns the maximal number of +the integer’s most-significant bits that are all zeros. +While the SoftFloat sources include an implementation of this function written +in standard C, many processors can perform this same function +directly in only one or two machine instructions. +An alternative, target-specific implementation that maps to those instructions +is likely to be more efficient than the generic C code from the SoftFloat +package. +

+ +

+A build target can replace the supplied version of any function or macro of +primitives.h by defining a macro with the same name in the +target’s platform.h header file. +For this purpose, it may be helpful for platform.h to +#include header file primitiveTypes.h, which defines +types used for arguments and results of functions declared in +primitives.h. +When a desired replacement implementation is a function, not a macro, it is +sufficient for platform.h to include the line +

+
+#define <function-name> <function-name>
+
+
+where <function-name> is the name of the +function. +This technically defines <function-name> as a +macro, but one that resolves to the same name, which may then be a function. +(A preprocessor conforming to the C Standard must limit recursive macro +expansion from being applied more than once.) +

+ + +

6. Testing SoftFloat

+ +

+SoftFloat can be tested using the testsoftfloat program by the +same author. +This program is part of the Berkeley TestFloat package available at the Web +page +http://www.jhauser.us/arithmetic/TestFloat.html. +The TestFloat package also has a program called timesoftfloat that +measures the speed of SoftFloat’s floating-point functions. +

+ + +

7. Providing SoftFloat as a Common Library for Applications

+ +

+Header file softfloat.h defines the SoftFloat interface as seen by +clients. +If the SoftFloat library will be made a common library for programs on a +particular system, the supplied softfloat.h has a couple of +deficiencies for this purpose: +

    +
  • +As supplied, softfloat.h depends on another header, +softfloat_types.h, that is not intended for public use but which +must also be visible to the programmer’s compiler. +
  • +More troubling, at the time softfloat.h is included in a C +source file, macro SOFTFLOAT_FAST_INT64 must be defined, or not +defined, consistent with whether this macro was defined when the SoftFloat +library was built. +
+In the situation that new programs may regularly #include header +file softfloat.h, it is recommended that a custom, self-contained +version of this header file be created that eliminates these issues. +

+ + +

8. Contact Information

+ +

+At the time of this writing, the most up-to-date information about SoftFloat +and the latest release can be found at the Web page +http://www.jhauser.us/arithmetic/SoftFloat.html. +

+ + + + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat.html b/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat.html new file mode 100644 index 0000000..19176dc --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/doc/SoftFloat.html @@ -0,0 +1,1453 @@ + + + + +Berkeley SoftFloat Library Interface + + + + +

Berkeley SoftFloat Release 3a: Library Interface

+ +

+John R. Hauser
+2015 October 23
+

+ + +

Contents

+ +
+ +++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
1. Introduction
2. Limitations
3. Acknowledgments and License
4. Types and Functions
4.1. Boolean and Integer Types
4.2. Floating-Point Types
4.3. Supported Floating-Point Functions
4.4. Non-canonical Representations in extFloat80_t
4.5. Conventions for Passing Arguments and Results
5. Reserved Names
6. Mode Variables
6.1. Rounding Mode
6.2. Underflow Detection
6.3. Rounding Precision for the 80-Bit Extended Format
7. Exceptions and Exception Flags
8. Function Details
8.1. Conversions from Integer to Floating-Point
8.2. Conversions from Floating-Point to Integer
8.3. Conversions Among Floating-Point Types
8.4. Basic Arithmetic Functions
8.5. Fused Multiply-Add Functions
8.6. Remainder Functions
8.7. Round-to-Integer Functions
8.8. Comparison Functions
8.9. Signaling NaN Test Functions
8.10. Raise-Exception Function
9. Changes from SoftFloat Release 2
9.1. Name Changes
9.2. Changes to Function Arguments
9.3. Added Capabilities
9.4. Better Compatibility with the C Language
9.5. New Organization as a Library
9.6. Optimization Gains (and Losses)
10. Future Directions
11. Contact Information
+
+ + +

1. Introduction

+ +

+Berkeley SoftFloat is a software implementation of binary floating-point that +conforms to the IEEE Standard for Floating-Point Arithmetic. +The current release supports four binary formats: 32-bit +single-precision, 64-bit double-precision, 80-bit +double-extended-precision, and 128-bit quadruple-precision. +The following functions are supported for each format: +

    +
  • +addition, subtraction, multiplication, division, and square root; +
  • +fused multiply-add as defined by the IEEE Standard, except for +80-bit double-extended-precision; +
  • +remainder as defined by the IEEE Standard; +
  • +round to integral value; +
  • +comparisons; +
  • +conversions to/from other supported formats; and +
  • +conversions to/from 32-bit and 64-bit integers, +signed and unsigned. +
+All operations required by the original 1985 version of the IEEE Floating-Point +Standard are implemented, except for conversions to and from decimal. +

+ +

+This document gives information about the types defined and the routines +implemented by SoftFloat. +It does not attempt to define or explain the IEEE Floating-Point Standard. +Information about the standard is available elsewhere. +

+ +

+The current version of SoftFloat is Release 3a. +The only difference between this version and the previous +Release 3 is the replacement of the license text supplied by the +University of California. +

+ +

+The functional interface of SoftFloat Release 3 and afterward +differs in many details from that of earlier releases. +For specifics of these differences, see section 9 below, +Changes from SoftFloat Release 2. +

+ + +

2. Limitations

+ +

+SoftFloat assumes the computer has an addressable byte size of 8 or +16 bits. +(Nearly all computers in use today have 8-bit bytes.) +

+ +

+SoftFloat is written in C and is designed to work with other C code. +The C compiler used must conform at a minimum to the 1989 ANSI standard for the +C language (same as the 1990 ISO standard) and must in addition support basic +arithmetic on 64-bit integers. +Earlier releases of SoftFloat included implementations of 32-bit +single-precision and 64-bit double-precision floating-point that +did not require 64-bit integers, but this option is not supported +starting with Release 3. +Since 1999, ISO standards for C have mandated compiler support for +64-bit integers. +A compiler conforming to the 1999 C Standard or later is recommended but not +strictly required. +

+ +

+Most operations not required by the original 1985 version of the IEEE +Floating-Point Standard but added in the 2008 version are not yet supported in +SoftFloat Release 3a. +

+ + +

3. Acknowledgments and License

+ +

+The SoftFloat package was written by me, John R. Hauser. +Release 3 of SoftFloat was a completely new implementation +supplanting earlier releases. +The project to create Release 3 (and now 3a) was done +in the employ of the University of California, Berkeley, within the Department +of Electrical Engineering and Computer Sciences, first for the Parallel +Computing Laboratory (Par Lab) and then for the ASPIRE Lab. +The work was officially overseen by Prof. Krste Asanovic, with funding provided +by these sources: +

+ ++++ + + + + + + + + + +
Par Lab: +Microsoft (Award #024263), Intel (Award #024894), and U.C. Discovery +(Award #DIG07-10227), with additional support from Par Lab affiliates Nokia, +NVIDIA, Oracle, and Samsung. +
ASPIRE Lab: +DARPA PERFECT program (Award #HR0011-12-2-0016), with additional support from +ASPIRE industrial sponsor Intel and ASPIRE affiliates Google, Nokia, NVIDIA, +Oracle, and Samsung. +
+
+

+ +

+The following applies to the whole of SoftFloat Release 3a as well +as to each source file individually. +

+ +

+Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. +All rights reserved. +

+ +

+Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +

    + +
  1. +

    +Redistributions of source code must retain the above copyright notice, this +list of conditions, and the following disclaimer. +

    + +
  2. +

    +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions, and the following disclaimer in the documentation and/or +other materials provided with the distribution. +

    + +
  3. +

    +Neither the name of the University nor the names of its contributors may be +used to endorse or promote products derived from this software without specific +prior written permission. +

    + +
+

+ +

+THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS “AS IS”, +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. +IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +

+ + +

4. Types and Functions

+ +

+The types and functions of SoftFloat are declared in header file +softfloat.h. +

+ +

4.1. Boolean and Integer Types

+ +

+Header file softfloat.h depends on standard headers +<stdbool.h> and <stdint.h> to define type +bool and several integer types. +These standard headers have been part of the ISO C Standard Library since 1999. +With any recent compiler, they are likely to be supported, even if the compiler +does not claim complete conformance to the ISO C Standard. +For older or nonstandard compilers, a port of SoftFloat may have substitutes +for these headers. +Header softfloat.h depends only on the name bool from +<stdbool.h> and on these type names from +<stdint.h>: +

+
+uint16_t
+uint32_t
+uint64_t
+int32_t
+int64_t
+uint_fast8_t
+uint_fast32_t
+uint_fast64_t
+
+
+

+ + +

4.2. Floating-Point Types

+ +

+The softfloat.h header defines four floating-point types: +

+ + + + + + + + + + + + + + + + + +
float32_t32-bit single-precision binary format
float64_t64-bit double-precision binary format
extFloat80_t   80-bit double-extended-precision binary format (old Intel or +Motorola format)
float128_t128-bit quadruple-precision binary format
+
+The non-extended types are each exactly the size specified: +32 bits for float32_t, 64 bits for +float64_t, and 128 bits for float128_t. +Aside from these size requirements, the definitions of all these types may +differ for different ports of SoftFloat to specific systems. +A given port of SoftFloat may or may not define some of the floating-point +types as aliases for the C standard types float, +double, and long double. +

+ +

+Header file softfloat.h also defines a structure, +struct extFloat80M, for the representation of +80-bit double-extended-precision floating-point values in memory. +This structure is the same size as type extFloat80_t and contains +at least these two fields (not necessarily in this order): +

+
+uint16_t signExp;
+uint64_t signif;
+
+
+Field signExp contains the sign and exponent of the floating-point +value, with the sign in the most significant bit (bit 15) and the +encoded exponent in the other 15 bits. +Field signif is the complete 64-bit significand of +the floating-point value. +(In the usual encoding for 80-bit extended floating-point, the +leading 1 bit of normalized numbers is not implicit but is stored +in the most significant bit of the significand.) +

+ +

4.3. Supported Floating-Point Functions

+ +

+SoftFloat implements these arithmetic operations for its floating-point types: +

    +
  • +conversions between any two floating-point formats; +
  • +for each floating-point format, conversions to and from signed and unsigned +32-bit and 64-bit integers; +
  • +for each format, the usual addition, subtraction, multiplication, division, and +square root operations; +
  • +for each format except extFloat80_t, the fused multiply-add +operation defined by the IEEE Standard; +
  • +for each format, the floating-point remainder operation defined by the IEEE +Standard; +
  • +for each format, a “round to integer” operation that rounds to the +nearest integer value in the same format; and +
  • +comparisons between two values in the same floating-point format. +
+

+ +

+The following operations required by the 2008 IEEE Floating-Point Standard are +not supported in SoftFloat Release 3a: +

    +
  • +nextUp, nextDown, minNum, maxNum, minNumMag, +maxNumMag, scaleB, and logB; +
  • +conversions between floating-point formats and decimal or hexadecimal character +sequences; +
  • +all “quiet-computation” operations (copy, negate, +abs, and copySign, which all involve only simple copying and/or +manipulation of the floating-point sign bit); and +
  • +all “non-computational” operations other than isSignaling +(which is supported). +
+

+ +

4.4. Non-canonical Representations in extFloat80_t

+ +

+Because the 80-bit double-extended-precision format, +extFloat80_t, stores an explicit leading significand bit, many +floating-point numbers are encodable in this type in equivalent normalized and +denormalized forms. +Zeros and values in the subnormal range have each only a single possible +encoding, for which the leading significand bit must be 0. +For other finite values (outside the subnormal range), a unique normalized +representation, with leading significand bit set to 1, always +exists, and is considered the canonical representation of the value. +Any equivalent denormalized representations (having leading significand bit +of 0) are non-canonical. +Similarly, the leading significand bit is expected to be 1 for +infinities and NaNs as well; +any infinity or NaN with a leading significand bit of 0 is again +considered non-canonical. +In short, for an extFloat80_t representation to be canonical, the +leading significand bit must be 1 unless it is required to +be 0 because the encoded value is zero or a subnormal. +

+ +

+Functions are not guaranteed to operate as expected when inputs of type +extFloat80_t are non-canonical. +Assuming all of a function’s extFloat80_t inputs (if any) +are canonical, function outputs of type extFloat80_t will always +be canonical. +

+ +

4.5. Conventions for Passing Arguments and Results

+ +

+Values that are at most 64 bits in size (i.e., not the +80-bit or 128-bit floating-point formats) are in all +cases passed as function arguments by value. +Likewise, when an output of a function is no more than 64 bits, it +is always returned directly as the function result. +Thus, for example, the SoftFloat function for adding two 64-bit +floating-point values has this simple signature: +

+float64_t f64_add( float64_t, float64_t ); +
+

+ +

+The story is more complex when function inputs and outputs are +80-bit and 128-bit floating-point. +For these types, SoftFloat always provides a function that passes these larger +values into or out of the function indirectly, via pointers. +For example, for adding two 128-bit floating-point values, +SoftFloat supplies this function: +

+void f128M_add( const float128_t *, const float128_t *, float128_t * ); +
+The first two arguments point to the values to be added, and the last argument +points to the location where the sum will be stored. +The M in the name f128M_add is mnemonic for the fact +that the 128-bit inputs and outputs are “in memory”, +pointed to by pointer arguments. +

+ +

+All ports of SoftFloat implement these pass-by-pointer functions for +types extFloat80_t and float128_t. +At the same time, SoftFloat ports may also implement alternate versions of +these same functions that pass extFloat80_t and +float128_t by value, like the smaller formats. +Thus, besides the function with name f128M_add shown above, a +SoftFloat port may also supply an equivalent function with this signature: +

+float128_t f128_add( float128_t, float128_t ); +
+

+ +

+As a general rule, on computers where the machine word size is +32 bits or smaller, only the pass-by-pointer versions of functions +(e.g., f128M_add) are provided for types extFloat80_t +and float128_t, because passing such large types directly can have +significant extra cost. +On computers where the word size is 64 bits or larger, both +function versions (f128M_add and f128_add) are +provided, because the cost of passing by value is then more reasonable. +Applications that must be portable accross both classes of computers must use +the pointer-based functions, as these are always implemented. +However, if it is known that SoftFloat includes the by-value functions for all +platforms of interest, programmers can use whichever version they prefer. +

+ + +

5. Reserved Names

+ +

+In addition to the variables and functions documented here, SoftFloat defines +some symbol names for its own private use. +These private names always begin with the prefix +‘softfloat_’. +When a program includes header softfloat.h or links with the +SoftFloat library, all names with prefix ‘softfloat_’ +are reserved for possible use by SoftFloat. +Applications that use SoftFloat should not define their own names with this +prefix, and should reference only such names as are documented. +

+ + +

6. Mode Variables

+ +

+The following variables control rounding mode, underflow detection, and the +80-bit extended format’s rounding precision: +

+softfloat_roundingMode
+softfloat_detectTininess
+extF80_roundingPrecision +
+These mode variables are covered in the next several subsections. +

+ +

6.1. Rounding Mode

+ +

+All five rounding modes defined by the 2008 IEEE Floating-Point Standard are +implemented for all operations that require rounding. +The rounding mode is selected by the global variable +

+uint_fast8_t softfloat_roundingMode; +
+This variable may be set to one of the values +
+ + + + + + + + + + + + + + + + + + + + + +
softfloat_round_near_evenround to nearest, with ties to even
softfloat_round_near_maxMag  round to nearest, with ties to maximum magnitude (away from zero)
softfloat_round_minMaground to minimum magnitude (toward zero)
softfloat_round_minround to minimum (down)
softfloat_round_maxround to maximum (up)
+
+Variable softfloat_roundingMode is initialized to +softfloat_round_near_even. +

+ +

6.2. Underflow Detection

+ +

+In the terminology of the IEEE Standard, SoftFloat can detect tininess for +underflow either before or after rounding. +The choice is made by the global variable +

+uint_fast8_t softfloat_detectTininess; +
+which can be set to either +
+softfloat_tininess_beforeRounding
+softfloat_tininess_afterRounding +
+Detecting tininess after rounding is better because it results in fewer +spurious underflow signals. +The other option is provided for compatibility with some systems. +Like most systems (and as required by the newer 2008 IEEE Standard), SoftFloat +always detects loss of accuracy for underflow as an inexact result. +

+ +

6.3. Rounding Precision for the 80-Bit Extended Format

+ +

+For extFloat80_t only, the rounding precision of the basic +arithmetic operations is controlled by the global variable +

+uint_fast8_t extF80_roundingPrecision; +
+The operations affected are: +
+extF80_add
+extF80_sub
+extF80_mul
+extF80_div
+extF80_sqrt +
+When extF80_roundingPrecision is set to its default value of 80, +these operations are rounded to the full precision of the 80-bit +double-extended-precision format, like occurs for other formats. +Setting extF80_roundingPrecision to 32 or to 64 causes the +operations listed to be rounded to 32-bit precision (equivalent to +float32_t) or to 64-bit precision (equivalent to +float64_t), respectively. +When rounding to reduced precision, additional bits in the result significand +beyond the rounding point are set to zero. +The consequences of setting extF80_roundingPrecision to a value +other than 32, 64, or 80 is not specified. +Operations other than the ones listed above are not affected by +extF80_roundingPrecision. +

+ + +

7. Exceptions and Exception Flags

+ +

+All five exception flags required by the IEEE Floating-Point Standard are +implemented. +Each flag is stored as a separate bit in the global variable +

+uint_fast8_t softfloat_exceptionFlags; +
+The positions of the exception flag bits within this variable are determined by +the bit masks +
+softfloat_flag_inexact
+softfloat_flag_underflow
+softfloat_flag_overflow
+softfloat_flag_infinite
+softfloat_flag_invalid +
+Variable softfloat_exceptionFlags is initialized to all zeros, +meaning no exceptions. +

+ +

+An individual exception flag can be cleared with the statement +

+softfloat_exceptionFlags &= ~softfloat_flag_<exception>; +
+where <exception> is the appropriate name. +To raise a floating-point exception, function softfloat_raise +should normally be used. +

+ +

+When SoftFloat detects an exception other than inexact, it calls +softfloat_raise. +The default version of this function simply raises the corresponding exception +flags. +Particular ports of SoftFloat may support alternate behavior, such as exception +traps, by modifying the default softfloat_raise. +A program may also supply its own softfloat_raise function to +override the one from the SoftFloat library. +

+ +

+Because inexact results occur frequently under most circumstances (and thus are +hardly exceptional), SoftFloat does not ordinarily call +softfloat_raise for inexact exceptions. +It does always raise the inexact exception flag as required. +

+ + +

8. Function Details

+ +

+In this section, <float> appears in function names as +a substitute for one of these abbreviations: +

+ + + + + + + + + + + + + + + + + + + + + + + + + +
f32indicates float32_t, passed by value
f64indicates float64_t, passed by value
extF80M   indicates extFloat80_t, passed indirectly via pointers
extF80indicates extFloat80_t, passed by value
f128Mindicates float128_t, passed indirectly via pointers
f128indicates float128_t, passed by value
+
+The circumstances under which values of floating-point types +extFloat80_t and float128_t may be passed either by +value or indirectly via pointers was discussed earlier in +section 4.5, Conventions for Passing Arguments and Results. +

+ +

8.1. Conversions from Integer to Floating-Point

+ +

+All conversions from a 32-bit or 64-bit integer, +signed or unsigned, to a floating-point format are supported. +Functions performing these conversions have these names: +

+ui32_to_<float>
+ui64_to_<float>
+i32_to_<float>
+i64_to_<float> +
+Conversions from 32-bit integers to 64-bit +double-precision and larger formats are always exact, and likewise conversions +from 64-bit integers to 80-bit +double-extended-precision and 128-bit quadruple-precision are also +always exact. +

+ +

+Each conversion function takes one input of the appropriate type and generates +one output. +The following illustrates the signatures of these functions in cases when the +floating-point result is passed either by value or via pointers: +

+
+float64_t i32_to_f64( int32_t a );
+
+
+void i32_to_f128M( int32_t a, float128_t *destPtr );
+
+
+

+ +

8.2. Conversions from Floating-Point to Integer

+ +

+Conversions from a floating-point format to a 32-bit or +64-bit integer, signed or unsigned, are supported with these +functions: +

+<float>_to_ui32
+<float>_to_ui64
+<float>_to_i32
+<float>_to_i64 +
+The functions have signatures as follows, depending on whether the +floating-point input is passed by value or via pointers: +
+
+int_fast32_t f64_to_i32( float64_t a, uint_fast8_t roundingMode, bool exact );
+
+
+int_fast32_t
+ f128M_to_i32( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact );
+
+
+The roundingMode argument specifies the rounding mode for +the conversion. +The variable that usually indicates rounding mode, +softfloat_roundingMode, is ignored. +Argument exact determines whether the inexact +exception flag is raised if the conversion is not exact. +If exact is true, the inexact flag may +be raised; +otherwise, it will not be, even if the conversion is inexact. +

+ +

+Conversions from floating-point to integer raise the invalid exception +if the source value cannot be rounded to a representable integer of the desired +size (32 or 64 bits). +In such a circumstance, if the floating-point input is a NaN or if the +conversion is to an unsigned integer type, the largest positive integer is +returned; +otherwise, the largest integer with the same sign as the input is returned. +The functions that convert to integer types never raise the overflow +exception. +

+ +

+Note that, when converting to an unsigned integer type, if the invalid +exception is raised because the input floating-point value would round to a +negative integer, the value returned is the maximum positive unsigned +integer. +Zero is not returned when the invalid exception is raised, even when +zero is the closest integer to the original floating-point value. +

+ +

+Because languages such as C require that conversions to integers +be rounded toward zero, the following functions are provided for improved speed +and convenience: +

+<float>_to_ui32_r_minMag
+<float>_to_ui64_r_minMag
+<float>_to_i32_r_minMag
+<float>_to_i64_r_minMag +
+These functions round only toward zero (to minimum magnitude). +The signatures for these functions are the same as above without the redundant +roundingMode argument: +
+
+int_fast32_t f64_to_i32_r_minMag( float64_t a, bool exact );
+
+
+int_fast32_t f128M_to_i32_r_minMag( const float128_t *aPtr, bool exact );
+
+
+

+ +

8.3. Conversions Among Floating-Point Types

+ +

+Conversions between floating-point formats are done by functions with these +names: +

+<float>_to_<float> +
+All combinations of source and result type are supported where the source and +result are different formats. +There are four different styles of signature for these functions, depending on +whether the input and the output floating-point values are passed by value or +via pointers: +
+
+float32_t f64_to_f32( float64_t a );
+
+
+float32_t f128M_to_f32( const float128_t *aPtr );
+
+
+void f32_to_f128M( float32_t a, float128_t *destPtr );
+
+
+void extF80M_to_f128M( const extFloat80_t *aPtr, float128_t *destPtr );
+
+
+

+ +

+Conversions from a smaller to a larger floating-point format are always exact +and so require no rounding. +

+ +

8.4. Basic Arithmetic Functions

+ +

+The following basic arithmetic functions are provided: +

+<float>_add
+<float>_sub
+<float>_mul
+<float>_div
+<float>_sqrt +
+Each floating-point operation takes two operands, except for sqrt +(square root) which takes only one. +The operands and result are all of the same floating-point format. +Signatures for these functions take the following forms: +
+
+float64_t f64_add( float64_t a, float64_t b );
+
+
+void
+ f128M_add(
+     const float128_t *aPtr, const float128_t *bPtr, float128_t *destPtr );
+
+
+float64_t f64_sqrt( float64_t a );
+
+
+void f128M_sqrt( const float128_t *aPtr, float128_t *destPtr );
+
+
+When floating-point values are passed indirectly through pointers, arguments +aPtr and bPtr point to the input +operands, and the last argument, destPtr, points to the +location where the result is stored. +

+ +

+Rounding of the 80-bit double-extended-precision +(extFloat80_t) functions is affected by variable +extF80_roundingPrecision, as explained earlier in +section 6.3, +Rounding Precision for the 80-Bit Extended Format. +

+ +

8.5. Fused Multiply-Add Functions

+ +

+The 2008 version of the IEEE Floating-Point Standard defines a fused +multiply-add operation that does a combined multiplication and addition +with only a single rounding. +SoftFloat implements fused multiply-add with functions +

+<float>_mulAdd +
+Unlike other operations, fused multiple-add is supported only for the +non-extended formats, float32_t, float64_t, and +float128_t. +No fused multiple-add function is currently provided for the +80-bit double-extended-precision type, extFloat80_t. +

+ +

+Depending on whether floating-point values are passed by value or via pointers, +the fused multiply-add functions have signatures of these forms: +

+
+float64_t f64_mulAdd( float64_t a, float64_t b, float64_t c );
+
+
+void
+ f128M_mulAdd(
+     const float128_t *aPtr,
+     const float128_t *bPtr,
+     const float128_t *cPtr,
+     float128_t *destPtr
+ );
+
+
+The functions compute +(a × b) + + c +with a single rounding. +When floating-point values are passed indirectly through pointers, arguments +aPtr, bPtr, and +cPtr point to operands a, +b, and c respectively, and +destPtr points to the location where the result is stored. +

+ +

+If one of the multiplication operands a and +b is infinite and the other is zero, these functions raise +the invalid exception even if operand c is a quiet NaN. +

+ +

8.6. Remainder Functions

+ +

+For each format, SoftFloat implements the remainder operation defined by the +IEEE Floating-Point Standard. +The remainder functions have names +

+<float>_rem +
+Each remainder operation takes two floating-point operands of the same format +and returns a result in the same format. +Depending on whether floating-point values are passed by value or via pointers, +the remainder functions have signatures of these forms: +
+
+float64_t f64_rem( float64_t a, float64_t b );
+
+
+void
+ f128M_rem(
+     const float128_t *aPtr, const float128_t *bPtr, float128_t *destPtr );
+
+
+When floating-point values are passed indirectly through pointers, arguments +aPtr and bPtr point to operands +a and b respectively, and +destPtr points to the location where the result is stored. +

+ +

+The IEEE Standard remainder operation computes the value +a + − n × b, +where n is the integer closest to +a ÷ b. +If a ÷ b is exactly +halfway between two integers, n is the even integer closest to +a ÷ b. +The IEEE Standard’s remainder operation is always exact and so requires +no rounding. +

+ +

+Depending on the relative magnitudes of the operands, the remainder +functions can take considerably longer to execute than the other SoftFloat +functions. +This is inherent in the remainder operation itself and is not a flaw in the +SoftFloat implementation. +

+ +

8.7. Round-to-Integer Functions

+ +

+For each format, SoftFloat implements the round-to-integer operation specified +by the IEEE Floating-Point Standard. +These functions are named +

+<float>_roundToInt +
+Each round-to-integer operation takes a single floating-point operand. +This operand is rounded to an integer according to a specified rounding mode, +and the resulting integer value is returned in the same floating-point format. +(Note that the result is not an integer type.) +

+ +

+The signatures of the round-to-integer functions are similar to those for +conversions to an integer type: +

+
+float64_t f64_roundToInt( float64_t a, uint_fast8_t roundingMode, bool exact );
+
+
+void
+ f128M_roundToInt(
+     const float128_t *aPtr,
+     uint_fast8_t roundingMode,
+     bool exact,
+     float128_t *destPtr
+ );
+
+
+The roundingMode argument specifies the rounding mode to +apply. +The variable that usually indicates rounding mode, +softfloat_roundingMode, is ignored. +Argument exact determines whether the inexact +exception flag is raised if the conversion is not exact. +If exact is true, the inexact flag may +be raised; +otherwise, it will not be, even if the conversion is inexact. +When floating-point values are passed indirectly through pointers, +aPtr points to the input operand and +destPtr points to the location where the result is stored. +

+ +

8.8. Comparison Functions

+ +

+For each format, the following floating-point comparison functions are +provided: +

+<float>_eq
+<float>_le
+<float>_lt +
+Each comparison takes two operands of the same type and returns a Boolean. +The abbreviation eq stands for “equal” (=); +le stands for “less than or equal” (≤); +and lt stands for “less than” (<). +Depending on whether the floating-point operands are passed by value or via +pointers, the comparison functions have signatures of these forms: +
+
+bool f64_eq( float64_t a, float64_t b );
+
+
+bool f128M_eq( const float128_t *aPtr, const float128_t *bPtr );
+
+
+

+ +

+The usual greater-than (>), greater-than-or-equal (≥), and not-equal +(≠) comparisons are easily obtained from the functions provided. +The not-equal function is just the logical complement of the equal function. +The greater-than-or-equal function is identical to the less-than-or-equal +function with the arguments in reverse order, and likewise the greater-than +function is identical to the less-than function with the arguments reversed. +

+ +

+The IEEE Floating-Point Standard specifies that the less-than-or-equal and +less-than comparisons by default raise the invalid exception if either +operand is any kind of NaN. +Equality comparisons, on the other hand, are defined by default to raise the +invalid exception only for signaling NaNs, not quiet NaNs. +For completeness, SoftFloat provides these complementary functions: +

+<float>_eq_signaling
+<float>_le_quiet
+<float>_lt_quiet +
+The signaling equality comparisons are identical to the default +equality comparisons except that the invalid exception is raised for any +NaN input, not just for signaling NaNs. +Similarly, the quiet comparison functions are identical to their +default counterparts except that the invalid exception is not raised for +quiet NaNs. +

+ +

8.9. Signaling NaN Test Functions

+ +

+Functions for testing whether a floating-point value is a signaling NaN are +provided with these names: +

+<float>_isSignalingNaN +
+The functions take one floating-point operand and return a Boolean indicating +whether the operand is a signaling NaN. +Accordingly, the functions have the forms +
+
+bool f64_isSignalingNaN( float64_t a );
+
+
+bool f128M_isSignalingNaN( const float128_t *aPtr );
+
+
+

+ +

8.10. Raise-Exception Function

+ +

+SoftFloat provides a single function for raising floating-point exceptions: +

+
+void softfloat_raise( uint_fast8_t exceptions );
+
+
+The exceptions argument is a mask indicating the set of +exceptions to raise. +(See earlier section 7, Exceptions and Exception Flags.) +In addition to setting the specified exception flags in variable +softfloat_exceptionFlags, the softfloat_raise +function may cause a trap or abort appropriate for the current system. +

+ + +

9. Changes from SoftFloat Release 2

+ +

+Apart from a change in the legal use license, Release 3 of +SoftFloat introduced numerous technical differences compared to earlier +releases. +

+ +

9.1. Name Changes

+ +

+The most obvious and pervasive difference compared to Release 2 +is that the names of most functions and variables have changed, even when the +behavior has not. +First, the floating-point types, the mode variables, the exception flags +variable, the function to raise exceptions, and various associated constants +have been renamed as follows: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
old name, Release 2:new name, Release 3:
float32float32_t
float64float64_t
floatx80extFloat80_t
float128float128_t
float_rounding_modesoftfloat_roundingMode
float_round_nearest_evensoftfloat_round_near_even
float_round_to_zerosoftfloat_round_minMag
float_round_downsoftfloat_round_min
float_round_upsoftfloat_round_max
float_detect_tininesssoftfloat_detectTininess
float_tininess_before_rounding    softfloat_tininess_beforeRounding
float_tininess_after_roundingsoftfloat_tininess_afterRounding
floatx80_rounding_precisionextF80_roundingPrecision
float_exception_flagssoftfloat_exceptionFlags
float_flag_inexactsoftfloat_flag_inexact
float_flag_underflowsoftfloat_flag_underflow
float_flag_overflowsoftfloat_flag_overflow
float_flag_divbyzerosoftfloat_flag_infinite
float_flag_invalidsoftfloat_flag_invalid
float_raisesoftfloat_raise
+
+

+ +

+Furthermore, Release 3 adopted the following new abbreviations for +function names: +

+ + + + + + + + + + + +
used in names in Release 2:    used in names in Release 3:
int32 i32
int64 i64
float32 f32
float64 f64
floatx80 extF80
float128 f128
+
+Thus, for example, the function to add two 32-bit floating-point +numbers, previously called float32_add in Release 2, +is now f32_add. +Lastly, there have been a few other changes to function names: +
+ + + + + + + + + + + + + + + + + + + + + +
used in names in Release 2:   used in names in Release 3:   relevant functions:
_round_to_zero_r_minMagconversions from floating-point to integer (section 8.2)
round_to_introundToIntround-to-integer functions (section 8.7)
is_signaling_nan    isSignalingNaNsignaling NaN test functions (section 8.9)
+
+

+ +

9.2. Changes to Function Arguments

+ +

+Besides simple name changes, some operations were given a different interface +in Release 3 than they had in Release 2: +

    + +
  • +

    +Since Release 3, integer arguments and results of functions have +standard types from header <stdint.h>, such as +uint32_t, whereas previously their types could be defined +differently for each port of SoftFloat, usually using traditional C types such +as unsigned int. +Likewise, functions in Release 3 and later pass Booleans as +standard type bool from <stdbool.h>, whereas +previously these were again passed as a port-specific type (usually +int). +

    + +
  • +

    +As explained earlier in section 4.5, Conventions for Passing +Arguments and Results, SoftFloat functions in Release 3 and +later may pass 80-bit and 128-bit floating-point +values through pointers, meaning that functions take pointer arguments and then +read or write floating-point values at the locations indicated by the pointers. +In Release 2, floating-point arguments and results were always +passed by value, regardless of their size. +

    + +
  • +

    +Functions that round to an integer have additional +roundingMode and exact arguments that +they did not have in Release 2. +Refer to sections 8.2 and 8.7 for descriptions of these functions +since Release 3. +For Release 2, the rounding mode, when needed, was taken from the +same global variable that affects the basic arithmetic operations (now called +softfloat_roundingMode but previously known as +float_rounding_mode). +Also, for Release 2, if the original floating-point input was not +an exact integer value, and if the invalid exception was not raised by +the function, the inexact exception was always raised. +Release 2 had no option to suppress raising inexact in this +case. +Applications using SoftFloat Release 3 or later can get the same +effect as Release 2 by passing variable +softfloat_roundingMode for argument +roundingMode and true for argument +exact. +

    + +
+

+ +

9.3. Added Capabilities

+ +

+With Release 3, some new features have been added that were not +present in Release 2: +

    + +
  • +

    +A port of SoftFloat can now define any of the floating-point types +float32_t, float64_t, extFloat80_t, and +float128_t as aliases for C’s standard floating-point types +float, double, and long +double, using either #define or typedef. +This potential convenience was not supported under Release 2. +

    + +

    +(Note, however, that there may be a performance cost to defining +SoftFloat’s floating-point types this way, depending on the platform and +the applications using SoftFloat. +Ports of SoftFloat may choose to forgo the convenience in favor of better +speed.) +

    + +

    +

  • +Functions have been added for converting between the floating-point types and +unsigned integers. +Release 2 supported only signed integers, not unsigned. +

    + +

    +

  • +A new, fifth rounding mode, softfloat_round_near_maxMag (round to +nearest, with ties to maximum magnitude, away from zero) is now supported for +all cases involving rounding. +

    + +

    +

  • +Fused multiply-add functions have been added for the non-extended formats, +float32_t, float64_t, and float128_t. +

    + +
+

+ +

9.4. Better Compatibility with the C Language

+ +

+Release 3 of SoftFloat was written to conform better to the ISO C +Standard’s rules for portability. +For example, older releases of SoftFloat employed type conversions in ways +that, while commonly practiced, are not fully defined by the C Standard. +Such problematic type conversions have generally been replaced by the use of +unions, the behavior around which is more strictly regulated these days. +

+ +

9.5. New Organization as a Library

+ +

+Starting with Release 3, SoftFloat now builds as a library. +Previously, SoftFloat compiled into a single, monolithic object file containing +all the SoftFloat functions, with the consequence that a program linking with +SoftFloat would get every SoftFloat function in its binary file even if only a +few functions were actually used. +With SoftFloat in the form of a library, a program that is linked by a standard +linker will include only those functions of SoftFloat that it needs and no +others. +

+ +

9.6. Optimization Gains (and Losses)

+ +

+Individual SoftFloat functions have been variously improved in +Release 3 compared to earlier releases. +In particular, better, faster algorithms have been deployed for the operations +of division, square root, and remainder. +For functions operating on the larger 80-bit and +128-bit formats, extFloat80_t and +float128_t, code size has also generally been reduced. +

+ +

+However, because Release 2 compiled all of SoftFloat together as a +single object file, compilers could make optimizations across function calls +when one SoftFloat function calls another. +Now that the functions of SoftFloat are compiled separately and only afterward +linked together into a program, there is not usually the same opportunity to +optimize across function calls. +Some loss of speed has been observed due to this change. +

+ + +

10. Future Directions

+ +

+The following improvements are anticipated for future releases of SoftFloat: +

    +
  • +support for the common 16-bit “half-precision” +floating-point format; +
  • +more functions from the 2008 version of the IEEE Floating-Point Standard; +
  • +consistent, defined behavior for non-canonical representations of extended +format extFloat80_t (discussed in section 4.4, +Non-canonical Representations in extFloat80_t). + +
+

+ + +

11. Contact Information

+ +

+At the time of this writing, the most up-to-date information about SoftFloat +and the latest release can be found at the Web page +http://www.jhauser.us/arithmetic/SoftFloat.html. +

+ + + + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/extF80M_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/extF80M_isSignalingNaN.c new file mode 100644 index 0000000..31bb286 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/extF80M_isSignalingNaN.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +bool extF80M_isSignalingNaN( const extFloat80_t *aPtr ) +{ + const struct extFloat80M *aSPtr; + uint64_t uiA0; + + aSPtr = (const struct extFloat80M *) aPtr; + if ( (aSPtr->signExp & 0x7FFF) != 0x7FFF ) return false; + uiA0 = aSPtr->signif; + return + ! (uiA0 & UINT64_C( 0x4000000000000000 )) + && (uiA0 & UINT64_C( 0x3FFFFFFFFFFFFFFF)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/f128M_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/f128M_isSignalingNaN.c new file mode 100644 index 0000000..d962049 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/f128M_isSignalingNaN.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "primitives.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +bool f128M_isSignalingNaN( const float128_t *aPtr ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + if ( (uiA96 & 0x7FFF8000) != 0x7FFF0000 ) return false; + return + ((uiA96 & 0x00007FFF) != 0) + || ((aWPtr[indexWord( 4, 2 )] | aWPtr[indexWord( 4, 1 )] + | aWPtr[indexWord( 4, 0 )]) + != 0); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToExtF80M.c new file mode 100644 index 0000000..4573e44 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToExtF80M.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and stores this NaN at the location pointed to by +| `zSPtr'. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToExtF80M( + const struct commonNaN *aPtr, struct extFloat80M *zSPtr ) +{ + + zSPtr->signExp = packToExtF80UI64( aPtr->sign, 0x7FFF ); + zSPtr->signif = UINT64_C( 0xC000000000000000 ) | aPtr->v64>>1; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToExtF80UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToExtF80UI.c new file mode 100644 index 0000000..6f6083d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToExtF80UI.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and returns the bit pattern of this value as an unsigned +| integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToExtF80UI( const struct commonNaN *aPtr ) +{ + struct uint128 uiZ; + + uiZ.v64 = (uint_fast16_t) aPtr->sign<<15 | 0x7FFF; + uiZ.v0 = UINT64_C( 0xC000000000000000 ) | aPtr->v64>>1; + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF128M.c new file mode 100644 index 0000000..50545c8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF128M.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and stores this NaN at the location pointed to by `zWPtr'. Argument +| `zWPtr' points to an array of four 32-bit elements that concatenate in the +| platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToF128M( const struct commonNaN *aPtr, uint32_t *zWPtr ) +{ + + softfloat_shortShiftRight128M( (const uint32_t *) &aPtr->v0, 16, zWPtr ); + zWPtr[indexWordHi( 4 )] |= (uint32_t) aPtr->sign<<31 | 0x7FFF8000; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF128UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF128UI.c new file mode 100644 index 0000000..22585f9 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF128UI.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToF128UI( const struct commonNaN *aPtr ) +{ + struct uint128 uiZ; + + uiZ = softfloat_shortShiftRight128( aPtr->v64, aPtr->v0, 16 ); + uiZ.v64 |= (uint_fast64_t) aPtr->sign<<63 | UINT64_C( 0x7FFF800000000000 ); + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF32UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF32UI.c new file mode 100644 index 0000000..568930d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF32UI.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 32-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast32_t softfloat_commonNaNToF32UI( const struct commonNaN *aPtr ) +{ + + return (uint_fast32_t) aPtr->sign<<31 | 0x7FC00000 | aPtr->v64>>41; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF64UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF64UI.c new file mode 100644 index 0000000..dfb7090 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_commonNaNToF64UI.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 64-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast64_t softfloat_commonNaNToF64UI( const struct commonNaN *aPtr ) +{ + + return + (uint_fast64_t) aPtr->sign<<63 | UINT64_C( 0x7FF8000000000000 ) + | aPtr->v64>>12; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_extF80MToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_extF80MToCommonNaN.c new file mode 100644 index 0000000..e2b448b --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_extF80MToCommonNaN.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the 80-bit extended floating-point value pointed to by `aSPtr' is +| a NaN, converts this NaN to the common NaN form, and stores the resulting +| common NaN at the location pointed to by `zPtr'. If the NaN is a signaling +| NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80MToCommonNaN( + const struct extFloat80M *aSPtr, struct commonNaN *zPtr ) +{ + + if ( extF80M_isSignalingNaN( (const extFloat80_t *) aSPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = signExtF80UI64( aSPtr->signExp ); + zPtr->v64 = aSPtr->signif<<1; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_extF80UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_extF80UIToCommonNaN.c new file mode 100644 index 0000000..e6476cf --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_extF80UIToCommonNaN.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of an 80-bit extended floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80UIToCommonNaN( + uint_fast16_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ) +{ + + if ( softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = uiA64>>15; + zPtr->v64 = uiA0<<1; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f128MToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f128MToCommonNaN.c new file mode 100644 index 0000000..5cb8577 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f128MToCommonNaN.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the 128-bit floating-point value pointed to by `aWPtr' is a NaN, +| converts this NaN to the common NaN form, and stores the resulting common +| NaN at the location pointed to by `zPtr'. If the NaN is a signaling NaN, +| the invalid exception is raised. Argument `aWPtr' points to an array of +| four 32-bit elements that concatenate in the platform's normal endian order +| to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_f128MToCommonNaN( const uint32_t *aWPtr, struct commonNaN *zPtr ) +{ + + if ( f128M_isSignalingNaN( (const float128_t *) aWPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = aWPtr[indexWordHi( 4 )]>>31; + softfloat_shortShiftLeft128M( aWPtr, 16, (uint32_t *) &zPtr->v0 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f128UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f128UIToCommonNaN.c new file mode 100644 index 0000000..556c349 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f128UIToCommonNaN.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of a 128-bit floating-point NaN, converts this NaN to +| the common NaN form, and stores the resulting common NaN at the location +| pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid exception +| is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_f128UIToCommonNaN( + uint_fast64_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ) +{ + struct uint128 NaNSig; + + if ( softfloat_isSigNaNF128UI( uiA64, uiA0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + NaNSig = softfloat_shortShiftLeft128( uiA64, uiA0, 16 ); + zPtr->sign = uiA64>>63; + zPtr->v64 = NaNSig.v64; + zPtr->v0 = NaNSig.v0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f32UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f32UIToCommonNaN.c new file mode 100644 index 0000000..6df9df4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f32UIToCommonNaN.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 32-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f32UIToCommonNaN( uint_fast32_t uiA, struct commonNaN *zPtr ) +{ + + if ( softfloat_isSigNaNF32UI( uiA ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = uiA>>31; + zPtr->v64 = (uint_fast64_t) uiA<<41; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f64UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f64UIToCommonNaN.c new file mode 100644 index 0000000..3312852 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_f64UIToCommonNaN.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 64-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f64UIToCommonNaN( uint_fast64_t uiA, struct commonNaN *zPtr ) +{ + + if ( softfloat_isSigNaNF64UI( uiA ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = uiA>>63; + zPtr->v64 = uiA<<12; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNExtF80M.c new file mode 100644 index 0000000..36babff --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNExtF80M.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 80-bit extended floating-point values +| pointed to by `aSPtr' and `bSPtr' is a NaN, stores the combined NaN result +| at the location pointed to by `zSPtr'. If either original floating-point +| value is a signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNExtF80M( + const struct extFloat80M *aSPtr, + const struct extFloat80M *bSPtr, + struct extFloat80M *zSPtr + ) +{ + bool isSigNaNA; + const struct extFloat80M *sPtr; + bool isSigNaNB; + uint_fast16_t uiB64; + uint64_t uiB0; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiMagA64, uiMagB64; + + isSigNaNA = extF80M_isSignalingNaN( (const extFloat80_t *) aSPtr ); + sPtr = aSPtr; + if ( ! bSPtr ) { + if ( isSigNaNA ) softfloat_raiseFlags( softfloat_flag_invalid ); + goto copy; + } + isSigNaNB = extF80M_isSignalingNaN( (const extFloat80_t *) bSPtr ); + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + uiB64 = bSPtr->signExp; + if ( isSigNaNB ) goto returnLargerUIMag; + uiB0 = bSPtr->signif; + if ( isNaNExtF80UI( uiB64, uiB0 ) ) goto copyB; + goto copy; + } else { + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) ) goto copy; + goto copyB; + } + } + uiB64 = bSPtr->signExp; + returnLargerUIMag: + uiA64 = aSPtr->signExp; + uiMagA64 = uiA64 & 0x7FFF; + uiMagB64 = uiB64 & 0x7FFF; + if ( uiMagA64 < uiMagB64 ) goto copyB; + if ( uiMagB64 < uiMagA64 ) goto copy; + uiA0 = aSPtr->signif; + uiB0 = bSPtr->signif; + if ( uiA0 < uiB0 ) goto copyB; + if ( uiB0 < uiA0 ) goto copy; + if ( uiA64 < uiB64 ) goto copy; + copyB: + sPtr = bSPtr; + copy: + zSPtr->signExp = sPtr->signExp; + zSPtr->signif = sPtr->signif | UINT64_C( 0xC000000000000000 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNExtF80UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNExtF80UI.c new file mode 100644 index 0000000..fe80d5f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNExtF80UI.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as an 80-bit extended floating-point value, and likewise interpreting +| the unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 80-bit extended floating-point value, and assuming at least on of these +| floating-point values is a NaN, returns the bit pattern of the combined NaN +| result. If either original floating-point value is a signaling NaN, the +| invalid exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNExtF80UI( + uint_fast16_t uiA64, + uint_fast64_t uiA0, + uint_fast16_t uiB64, + uint_fast64_t uiB0 + ) +{ + bool isSigNaNA, isSigNaNB; + uint_fast64_t uiNonsigA0, uiNonsigB0; + uint_fast16_t uiMagA64, uiMagB64; + struct uint128 uiZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + isSigNaNA = softfloat_isSigNaNExtF80UI( uiA64, uiA0 ); + isSigNaNB = softfloat_isSigNaNExtF80UI( uiB64, uiB0 ); + /*------------------------------------------------------------------------ + | Make NaNs non-signaling. + *------------------------------------------------------------------------*/ + uiNonsigA0 = uiA0 | UINT64_C( 0xC000000000000000 ); + uiNonsigB0 = uiB0 | UINT64_C( 0xC000000000000000 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + if ( isSigNaNB ) goto returnLargerMag; + if ( isNaNExtF80UI( uiB64, uiB0 ) ) goto returnB; + goto returnA; + } else { + if ( isNaNExtF80UI( uiA64, uiA0 ) ) goto returnA; + goto returnB; + } + } + returnLargerMag: + uiMagA64 = uiA64 & 0x7FFF; + uiMagB64 = uiB64 & 0x7FFF; + if ( uiMagA64 < uiMagB64 ) goto returnB; + if ( uiMagB64 < uiMagA64 ) goto returnA; + if ( uiNonsigA0 < uiNonsigB0 ) goto returnB; + if ( uiNonsigB0 < uiNonsigA0 ) goto returnA; + if ( uiA64 < uiB64 ) goto returnA; + returnB: + uiZ.v64 = uiB64; + uiZ.v0 = uiNonsigB0; + return uiZ; + returnA: + uiZ.v64 = uiA64; + uiZ.v0 = uiNonsigA0; + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF128M.c new file mode 100644 index 0000000..ec6936a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF128M.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 128-bit floating-point values pointed to by +| `aWPtr' and `bWPtr' is a NaN, stores the combined NaN result at the location +| pointed to by `zWPtr'. If either original floating-point value is a +| signaling NaN, the invalid exception is raised. Each of `aWPtr', `bWPtr', +| and `zWPtr' points to an array of four 32-bit elements that concatenate in +| the platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNF128M( + const uint32_t *aWPtr, const uint32_t *bWPtr, uint32_t *zWPtr ) +{ + bool isSigNaNA; + const uint32_t *ptr; + + ptr = aWPtr; + isSigNaNA = f128M_isSignalingNaN( (const float128_t *) aWPtr ); + if ( + isSigNaNA + || (bWPtr && f128M_isSignalingNaN( (const float128_t *) bWPtr )) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) goto copy; + } + if ( ! softfloat_isNaNF128M( aWPtr ) ) ptr = bWPtr; + copy: + zWPtr[indexWordHi( 4 )] = ptr[indexWordHi( 4 )] | 0x00008000; + zWPtr[indexWord( 4, 2 )] = ptr[indexWord( 4, 2 )]; + zWPtr[indexWord( 4, 1 )] = ptr[indexWord( 4, 1 )]; + zWPtr[indexWord( 4, 0 )] = ptr[indexWord( 4, 0 )]; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF128UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF128UI.c new file mode 100644 index 0000000..654250f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF128UI.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as a 128-bit floating-point value, and likewise interpreting the +| unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 128-bit floating-point value, and assuming at least on of these floating- +| point values is a NaN, returns the bit pattern of the combined NaN result. +| If either original floating-point value is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNF128UI( + uint_fast64_t uiA64, + uint_fast64_t uiA0, + uint_fast64_t uiB64, + uint_fast64_t uiB0 + ) +{ + bool isSigNaNA; + struct uint128 uiZ; + + isSigNaNA = softfloat_isSigNaNF128UI( uiA64, uiA0 ); + if ( isSigNaNA || softfloat_isSigNaNF128UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) goto returnNonsigA; + } + if ( isNaNF128UI( uiA64, uiA0 ) ) { + returnNonsigA: + uiZ.v64 = uiA64; + uiZ.v0 = uiA0; + } else { + uiZ.v64 = uiB64; + uiZ.v0 = uiB0; + } + uiZ.v64 |= UINT64_C( 0x0000800000000000 ); + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF32UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF32UI.c new file mode 100644 index 0000000..f1910c7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF32UI.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 32-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast32_t + softfloat_propagateNaNF32UI( uint_fast32_t uiA, uint_fast32_t uiB ) +{ + bool isSigNaNA; + + isSigNaNA = softfloat_isSigNaNF32UI( uiA ); + if ( isSigNaNA || softfloat_isSigNaNF32UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) return uiA | 0x00400000; + } + return (isNaNF32UI( uiA ) ? uiA : uiB) | 0x00400000; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF64UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF64UI.c new file mode 100644 index 0000000..40fc9dd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/s_propagateNaNF64UI.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 64-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast64_t + softfloat_propagateNaNF64UI( uint_fast64_t uiA, uint_fast64_t uiB ) +{ + bool isSigNaNA; + + isSigNaNA = softfloat_isSigNaNF64UI( uiA ); + if ( isSigNaNA || softfloat_isSigNaNF64UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) return uiA | UINT64_C( 0x0008000000000000 ); + } + return (isNaNF64UI( uiA ) ? uiA : uiB) | UINT64_C( 0x0008000000000000 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/softfloat_raiseFlags.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/softfloat_raiseFlags.c new file mode 100644 index 0000000..2b5aba8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/softfloat_raiseFlags.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include "platform.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Raises the exceptions specified by `flags'. Floating-point traps can be +| defined here if desired. It is currently not possible for such a trap +| to substitute a result value. If traps are not implemented, this routine +| should be simply `softfloat_exceptionFlags |= flags;'. +*----------------------------------------------------------------------------*/ +void softfloat_raiseFlags( uint_fast8_t flags ) +{ + + softfloat_exceptionFlags |= flags; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/specialize.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/specialize.h new file mode 100644 index 0000000..f8888e5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/specialize.h @@ -0,0 +1,319 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#ifndef specialize_h +#define specialize_h 1 + +#include +#include +#include "softfloat_types.h" + +/*---------------------------------------------------------------------------- +| Default value for `softfloat_detectTininess'. +*----------------------------------------------------------------------------*/ +#define init_detectTininess softfloat_tininess_afterRounding + +/*---------------------------------------------------------------------------- +| "Common NaN" structure, used to transfer NaN representations from one format +| to another. +*----------------------------------------------------------------------------*/ +struct commonNaN { + bool sign; +#ifdef LITTLEENDIAN + uint64_t v0, v64; +#else + uint64_t v64, v0; +#endif +}; + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 32-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF32UI 0xFFC00000 + +/*---------------------------------------------------------------------------- +| Returns true when 32-bit unsigned integer `uiA' has the bit pattern of a +| 32-bit floating-point signaling NaN. +| Note: This macro evaluates its argument more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNF32UI( uiA ) ((((uiA) & 0x7FC00000) == 0x7F800000) && ((uiA) & 0x003FFFFF)) + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 32-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f32UIToCommonNaN( uint_fast32_t uiA, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 32-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast32_t softfloat_commonNaNToF32UI( const struct commonNaN *aPtr ); + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 32-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast32_t + softfloat_propagateNaNF32UI( uint_fast32_t uiA, uint_fast32_t uiB ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 64-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF64UI UINT64_C( 0xFFF8000000000000 ) + +/*---------------------------------------------------------------------------- +| Returns true when 64-bit unsigned integer `uiA' has the bit pattern of a +| 64-bit floating-point signaling NaN. +| Note: This macro evaluates its argument more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNF64UI( uiA ) ((((uiA) & UINT64_C( 0x7FF8000000000000 )) == UINT64_C( 0x7FF0000000000000 )) && ((uiA) & UINT64_C( 0x0007FFFFFFFFFFFF ))) + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 64-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f64UIToCommonNaN( uint_fast64_t uiA, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 64-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast64_t softfloat_commonNaNToF64UI( const struct commonNaN *aPtr ); + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 64-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast64_t + softfloat_propagateNaNF64UI( uint_fast64_t uiA, uint_fast64_t uiB ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 80-bit extended floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNExtF80UI64 0xFFFF +#define defaultNaNExtF80UI0 UINT64_C( 0xC000000000000000 ) + +/*---------------------------------------------------------------------------- +| Returns true when the 80-bit unsigned integer formed from concatenating +| 16-bit `uiA64' and 64-bit `uiA0' has the bit pattern of an 80-bit extended +| floating-point signaling NaN. +| Note: This macro evaluates its arguments more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) ((((uiA64) & 0x7FFF) == 0x7FFF) && ! ((uiA0) & UINT64_C( 0x4000000000000000 )) && ((uiA0) & UINT64_C( 0x3FFFFFFFFFFFFFFF ))) + +#ifdef SOFTFLOAT_FAST_INT64 + +/*---------------------------------------------------------------------------- +| The following functions are needed only when `SOFTFLOAT_FAST_INT64' is +| defined. +*----------------------------------------------------------------------------*/ + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of an 80-bit extended floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80UIToCommonNaN( + uint_fast16_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and returns the bit pattern of this value as an unsigned +| integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToExtF80UI( const struct commonNaN *aPtr ); + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as an 80-bit extended floating-point value, and likewise interpreting +| the unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 80-bit extended floating-point value, and assuming at least on of these +| floating-point values is a NaN, returns the bit pattern of the combined NaN +| result. If either original floating-point value is a signaling NaN, the +| invalid exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNExtF80UI( + uint_fast16_t uiA64, + uint_fast64_t uiA0, + uint_fast16_t uiB64, + uint_fast64_t uiB0 + ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 128-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF128UI64 UINT64_C( 0xFFFF800000000000 ) +#define defaultNaNF128UI0 UINT64_C( 0 ) + +/*---------------------------------------------------------------------------- +| Returns true when the 128-bit unsigned integer formed from concatenating +| 64-bit `uiA64' and 64-bit `uiA0' has the bit pattern of a 128-bit floating- +| point signaling NaN. +| Note: This macro evaluates its arguments more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNF128UI( uiA64, uiA0 ) ((((uiA64) & UINT64_C( 0x7FFF800000000000 )) == UINT64_C( 0x7FFF000000000000 )) && ((uiA0) || ((uiA64) & UINT64_C( 0x00007FFFFFFFFFFF )))) + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of a 128-bit floating-point NaN, converts this NaN to +| the common NaN form, and stores the resulting common NaN at the location +| pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid exception +| is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_f128UIToCommonNaN( + uint_fast64_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToF128UI( const struct commonNaN * ); + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as a 128-bit floating-point value, and likewise interpreting the +| unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 128-bit floating-point value, and assuming at least on of these floating- +| point values is a NaN, returns the bit pattern of the combined NaN result. +| If either original floating-point value is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNF128UI( + uint_fast64_t uiA64, + uint_fast64_t uiA0, + uint_fast64_t uiB64, + uint_fast64_t uiB0 + ); + +#else + +/*---------------------------------------------------------------------------- +| The following functions are needed only when `SOFTFLOAT_FAST_INT64' is not +| defined. +*----------------------------------------------------------------------------*/ + +/*---------------------------------------------------------------------------- +| Assuming the 80-bit extended floating-point value pointed to by `aSPtr' is +| a NaN, converts this NaN to the common NaN form, and stores the resulting +| common NaN at the location pointed to by `zPtr'. If the NaN is a signaling +| NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80MToCommonNaN( + const struct extFloat80M *aSPtr, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and stores this NaN at the location pointed to by +| `zSPtr'. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToExtF80M( + const struct commonNaN *aPtr, struct extFloat80M *zSPtr ); + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 80-bit extended floating-point values +| pointed to by `aSPtr' and `bSPtr' is a NaN, stores the combined NaN result +| at the location pointed to by `zSPtr'. If either original floating-point +| value is a signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNExtF80M( + const struct extFloat80M *aSPtr, + const struct extFloat80M *bSPtr, + struct extFloat80M *zSPtr + ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 128-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF128UI96 0xFFFF8000 +#define defaultNaNF128UI64 0 +#define defaultNaNF128UI32 0 +#define defaultNaNF128UI0 0 + +/*---------------------------------------------------------------------------- +| Assuming the 128-bit floating-point value pointed to by `aWPtr' is a NaN, +| converts this NaN to the common NaN form, and stores the resulting common +| NaN at the location pointed to by `zPtr'. If the NaN is a signaling NaN, +| the invalid exception is raised. Argument `aWPtr' points to an array of +| four 32-bit elements that concatenate in the platform's normal endian order +| to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_f128MToCommonNaN( const uint32_t *aWPtr, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and stores this NaN at the location pointed to by `zWPtr'. Argument +| `zWPtr' points to an array of four 32-bit elements that concatenate in the +| platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToF128M( const struct commonNaN *aPtr, uint32_t *zWPtr ); + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 128-bit floating-point values pointed to by +| `aWPtr' and `bWPtr' is a NaN, stores the combined NaN result at the location +| pointed to by `zWPtr'. If either original floating-point value is a +| signaling NaN, the invalid exception is raised. Each of `aWPtr', `bWPtr', +| and `zWPtr' points to an array of four 32-bit elements that concatenate in +| the platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNF128M( + const uint32_t *aWPtr, const uint32_t *bWPtr, uint32_t *zWPtr ); + +#endif + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/sub.mk b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/sub.mk new file mode 100644 index 0000000..175c1c3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086-SSE/sub.mk @@ -0,0 +1,7 @@ +srcs-y += softfloat_raiseFlags.c +srcs-y += s_propagateNaNF32UI.c +srcs-y += s_f32UIToCommonNaN.c +srcs-y += s_commonNaNToF64UI.c +srcs-y += s_f64UIToCommonNaN.c +srcs-y += s_propagateNaNF64UI.c +srcs-y += s_commonNaNToF32UI.c diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/extF80M_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/extF80M_isSignalingNaN.c new file mode 100644 index 0000000..31bb286 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/extF80M_isSignalingNaN.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +bool extF80M_isSignalingNaN( const extFloat80_t *aPtr ) +{ + const struct extFloat80M *aSPtr; + uint64_t uiA0; + + aSPtr = (const struct extFloat80M *) aPtr; + if ( (aSPtr->signExp & 0x7FFF) != 0x7FFF ) return false; + uiA0 = aSPtr->signif; + return + ! (uiA0 & UINT64_C( 0x4000000000000000 )) + && (uiA0 & UINT64_C( 0x3FFFFFFFFFFFFFFF)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/f128M_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/f128M_isSignalingNaN.c new file mode 100644 index 0000000..d962049 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/f128M_isSignalingNaN.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "primitives.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +bool f128M_isSignalingNaN( const float128_t *aPtr ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + if ( (uiA96 & 0x7FFF8000) != 0x7FFF0000 ) return false; + return + ((uiA96 & 0x00007FFF) != 0) + || ((aWPtr[indexWord( 4, 2 )] | aWPtr[indexWord( 4, 1 )] + | aWPtr[indexWord( 4, 0 )]) + != 0); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToExtF80M.c new file mode 100644 index 0000000..4573e44 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToExtF80M.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and stores this NaN at the location pointed to by +| `zSPtr'. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToExtF80M( + const struct commonNaN *aPtr, struct extFloat80M *zSPtr ) +{ + + zSPtr->signExp = packToExtF80UI64( aPtr->sign, 0x7FFF ); + zSPtr->signif = UINT64_C( 0xC000000000000000 ) | aPtr->v64>>1; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToExtF80UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToExtF80UI.c new file mode 100644 index 0000000..6f6083d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToExtF80UI.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and returns the bit pattern of this value as an unsigned +| integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToExtF80UI( const struct commonNaN *aPtr ) +{ + struct uint128 uiZ; + + uiZ.v64 = (uint_fast16_t) aPtr->sign<<15 | 0x7FFF; + uiZ.v0 = UINT64_C( 0xC000000000000000 ) | aPtr->v64>>1; + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF128M.c new file mode 100644 index 0000000..50545c8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF128M.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and stores this NaN at the location pointed to by `zWPtr'. Argument +| `zWPtr' points to an array of four 32-bit elements that concatenate in the +| platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToF128M( const struct commonNaN *aPtr, uint32_t *zWPtr ) +{ + + softfloat_shortShiftRight128M( (const uint32_t *) &aPtr->v0, 16, zWPtr ); + zWPtr[indexWordHi( 4 )] |= (uint32_t) aPtr->sign<<31 | 0x7FFF8000; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF128UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF128UI.c new file mode 100644 index 0000000..22585f9 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF128UI.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToF128UI( const struct commonNaN *aPtr ) +{ + struct uint128 uiZ; + + uiZ = softfloat_shortShiftRight128( aPtr->v64, aPtr->v0, 16 ); + uiZ.v64 |= (uint_fast64_t) aPtr->sign<<63 | UINT64_C( 0x7FFF800000000000 ); + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF32UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF32UI.c new file mode 100644 index 0000000..568930d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF32UI.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 32-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast32_t softfloat_commonNaNToF32UI( const struct commonNaN *aPtr ) +{ + + return (uint_fast32_t) aPtr->sign<<31 | 0x7FC00000 | aPtr->v64>>41; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF64UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF64UI.c new file mode 100644 index 0000000..dfb7090 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_commonNaNToF64UI.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 64-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast64_t softfloat_commonNaNToF64UI( const struct commonNaN *aPtr ) +{ + + return + (uint_fast64_t) aPtr->sign<<63 | UINT64_C( 0x7FF8000000000000 ) + | aPtr->v64>>12; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_extF80MToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_extF80MToCommonNaN.c new file mode 100644 index 0000000..e2b448b --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_extF80MToCommonNaN.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the 80-bit extended floating-point value pointed to by `aSPtr' is +| a NaN, converts this NaN to the common NaN form, and stores the resulting +| common NaN at the location pointed to by `zPtr'. If the NaN is a signaling +| NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80MToCommonNaN( + const struct extFloat80M *aSPtr, struct commonNaN *zPtr ) +{ + + if ( extF80M_isSignalingNaN( (const extFloat80_t *) aSPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = signExtF80UI64( aSPtr->signExp ); + zPtr->v64 = aSPtr->signif<<1; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_extF80UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_extF80UIToCommonNaN.c new file mode 100644 index 0000000..e6476cf --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_extF80UIToCommonNaN.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of an 80-bit extended floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80UIToCommonNaN( + uint_fast16_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ) +{ + + if ( softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = uiA64>>15; + zPtr->v64 = uiA0<<1; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f128MToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f128MToCommonNaN.c new file mode 100644 index 0000000..5cb8577 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f128MToCommonNaN.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the 128-bit floating-point value pointed to by `aWPtr' is a NaN, +| converts this NaN to the common NaN form, and stores the resulting common +| NaN at the location pointed to by `zPtr'. If the NaN is a signaling NaN, +| the invalid exception is raised. Argument `aWPtr' points to an array of +| four 32-bit elements that concatenate in the platform's normal endian order +| to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_f128MToCommonNaN( const uint32_t *aWPtr, struct commonNaN *zPtr ) +{ + + if ( f128M_isSignalingNaN( (const float128_t *) aWPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = aWPtr[indexWordHi( 4 )]>>31; + softfloat_shortShiftLeft128M( aWPtr, 16, (uint32_t *) &zPtr->v0 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f128UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f128UIToCommonNaN.c new file mode 100644 index 0000000..556c349 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f128UIToCommonNaN.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of a 128-bit floating-point NaN, converts this NaN to +| the common NaN form, and stores the resulting common NaN at the location +| pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid exception +| is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_f128UIToCommonNaN( + uint_fast64_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ) +{ + struct uint128 NaNSig; + + if ( softfloat_isSigNaNF128UI( uiA64, uiA0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + NaNSig = softfloat_shortShiftLeft128( uiA64, uiA0, 16 ); + zPtr->sign = uiA64>>63; + zPtr->v64 = NaNSig.v64; + zPtr->v0 = NaNSig.v0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f32UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f32UIToCommonNaN.c new file mode 100644 index 0000000..6df9df4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f32UIToCommonNaN.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 32-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f32UIToCommonNaN( uint_fast32_t uiA, struct commonNaN *zPtr ) +{ + + if ( softfloat_isSigNaNF32UI( uiA ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = uiA>>31; + zPtr->v64 = (uint_fast64_t) uiA<<41; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f64UIToCommonNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f64UIToCommonNaN.c new file mode 100644 index 0000000..3312852 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_f64UIToCommonNaN.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 64-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f64UIToCommonNaN( uint_fast64_t uiA, struct commonNaN *zPtr ) +{ + + if ( softfloat_isSigNaNF64UI( uiA ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + zPtr->sign = uiA>>63; + zPtr->v64 = uiA<<12; + zPtr->v0 = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNExtF80M.c new file mode 100644 index 0000000..36babff --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNExtF80M.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 80-bit extended floating-point values +| pointed to by `aSPtr' and `bSPtr' is a NaN, stores the combined NaN result +| at the location pointed to by `zSPtr'. If either original floating-point +| value is a signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNExtF80M( + const struct extFloat80M *aSPtr, + const struct extFloat80M *bSPtr, + struct extFloat80M *zSPtr + ) +{ + bool isSigNaNA; + const struct extFloat80M *sPtr; + bool isSigNaNB; + uint_fast16_t uiB64; + uint64_t uiB0; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiMagA64, uiMagB64; + + isSigNaNA = extF80M_isSignalingNaN( (const extFloat80_t *) aSPtr ); + sPtr = aSPtr; + if ( ! bSPtr ) { + if ( isSigNaNA ) softfloat_raiseFlags( softfloat_flag_invalid ); + goto copy; + } + isSigNaNB = extF80M_isSignalingNaN( (const extFloat80_t *) bSPtr ); + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + uiB64 = bSPtr->signExp; + if ( isSigNaNB ) goto returnLargerUIMag; + uiB0 = bSPtr->signif; + if ( isNaNExtF80UI( uiB64, uiB0 ) ) goto copyB; + goto copy; + } else { + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) ) goto copy; + goto copyB; + } + } + uiB64 = bSPtr->signExp; + returnLargerUIMag: + uiA64 = aSPtr->signExp; + uiMagA64 = uiA64 & 0x7FFF; + uiMagB64 = uiB64 & 0x7FFF; + if ( uiMagA64 < uiMagB64 ) goto copyB; + if ( uiMagB64 < uiMagA64 ) goto copy; + uiA0 = aSPtr->signif; + uiB0 = bSPtr->signif; + if ( uiA0 < uiB0 ) goto copyB; + if ( uiB0 < uiA0 ) goto copy; + if ( uiA64 < uiB64 ) goto copy; + copyB: + sPtr = bSPtr; + copy: + zSPtr->signExp = sPtr->signExp; + zSPtr->signif = sPtr->signif | UINT64_C( 0xC000000000000000 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNExtF80UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNExtF80UI.c new file mode 100644 index 0000000..fe80d5f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNExtF80UI.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as an 80-bit extended floating-point value, and likewise interpreting +| the unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 80-bit extended floating-point value, and assuming at least on of these +| floating-point values is a NaN, returns the bit pattern of the combined NaN +| result. If either original floating-point value is a signaling NaN, the +| invalid exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNExtF80UI( + uint_fast16_t uiA64, + uint_fast64_t uiA0, + uint_fast16_t uiB64, + uint_fast64_t uiB0 + ) +{ + bool isSigNaNA, isSigNaNB; + uint_fast64_t uiNonsigA0, uiNonsigB0; + uint_fast16_t uiMagA64, uiMagB64; + struct uint128 uiZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + isSigNaNA = softfloat_isSigNaNExtF80UI( uiA64, uiA0 ); + isSigNaNB = softfloat_isSigNaNExtF80UI( uiB64, uiB0 ); + /*------------------------------------------------------------------------ + | Make NaNs non-signaling. + *------------------------------------------------------------------------*/ + uiNonsigA0 = uiA0 | UINT64_C( 0xC000000000000000 ); + uiNonsigB0 = uiB0 | UINT64_C( 0xC000000000000000 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + if ( isSigNaNB ) goto returnLargerMag; + if ( isNaNExtF80UI( uiB64, uiB0 ) ) goto returnB; + goto returnA; + } else { + if ( isNaNExtF80UI( uiA64, uiA0 ) ) goto returnA; + goto returnB; + } + } + returnLargerMag: + uiMagA64 = uiA64 & 0x7FFF; + uiMagB64 = uiB64 & 0x7FFF; + if ( uiMagA64 < uiMagB64 ) goto returnB; + if ( uiMagB64 < uiMagA64 ) goto returnA; + if ( uiNonsigA0 < uiNonsigB0 ) goto returnB; + if ( uiNonsigB0 < uiNonsigA0 ) goto returnA; + if ( uiA64 < uiB64 ) goto returnA; + returnB: + uiZ.v64 = uiB64; + uiZ.v0 = uiNonsigB0; + return uiZ; + returnA: + uiZ.v64 = uiA64; + uiZ.v0 = uiNonsigA0; + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF128M.c new file mode 100644 index 0000000..43ce632 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF128M.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 128-bit floating-point values pointed to by +| `aWPtr' and `bWPtr' is a NaN, stores the combined NaN result at the location +| pointed to by `zWPtr'. If either original floating-point value is a +| signaling NaN, the invalid exception is raised. Each of `aWPtr', `bWPtr', +| and `zWPtr' points to an array of four 32-bit elements that concatenate in +| the platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNF128M( + const uint32_t *aWPtr, const uint32_t *bWPtr, uint32_t *zWPtr ) +{ + bool isSigNaNA; + const uint32_t *ptr; + bool isSigNaNB; + uint32_t uiA96, uiB96, wordMagA, wordMagB; + + isSigNaNA = f128M_isSignalingNaN( (const float128_t *) aWPtr ); + ptr = aWPtr; + if ( ! bWPtr ) { + if ( isSigNaNA ) softfloat_raiseFlags( softfloat_flag_invalid ); + goto copy; + } + isSigNaNB = f128M_isSignalingNaN( (const float128_t *) bWPtr ); + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + if ( isSigNaNB ) goto returnLargerUIMag; + if ( softfloat_isNaNF128M( bWPtr ) ) goto copyB; + goto copy; + } else { + if ( softfloat_isNaNF128M( aWPtr ) ) goto copy; + goto copyB; + } + } + returnLargerUIMag: + uiA96 = aWPtr[indexWordHi( 4 )]; + uiB96 = bWPtr[indexWordHi( 4 )]; + wordMagA = uiA96 & 0x7FFFFFFF; + wordMagB = uiB96 & 0x7FFFFFFF; + if ( wordMagA < wordMagB ) goto copyB; + if ( wordMagB < wordMagA ) goto copy; + wordMagA = aWPtr[indexWord( 4, 2 )]; + wordMagB = bWPtr[indexWord( 4, 2 )]; + if ( wordMagA < wordMagB ) goto copyB; + if ( wordMagB < wordMagA ) goto copy; + wordMagA = aWPtr[indexWord( 4, 1 )]; + wordMagB = bWPtr[indexWord( 4, 1 )]; + if ( wordMagA < wordMagB ) goto copyB; + if ( wordMagB < wordMagA ) goto copy; + wordMagA = aWPtr[indexWord( 4, 0 )]; + wordMagB = bWPtr[indexWord( 4, 0 )]; + if ( wordMagA < wordMagB ) goto copyB; + if ( wordMagB < wordMagA ) goto copy; + if ( uiA96 < uiB96 ) goto copy; + copyB: + ptr = bWPtr; + copy: + zWPtr[indexWordHi( 4 )] = ptr[indexWordHi( 4 )] | 0x00008000; + zWPtr[indexWord( 4, 2 )] = ptr[indexWord( 4, 2 )]; + zWPtr[indexWord( 4, 1 )] = ptr[indexWord( 4, 1 )]; + zWPtr[indexWord( 4, 0 )] = ptr[indexWord( 4, 0 )]; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF128UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF128UI.c new file mode 100644 index 0000000..f7e0f76 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF128UI.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as a 128-bit floating-point value, and likewise interpreting the +| unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 128-bit floating-point value, and assuming at least on of these floating- +| point values is a NaN, returns the bit pattern of the combined NaN result. +| If either original floating-point value is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNF128UI( + uint_fast64_t uiA64, + uint_fast64_t uiA0, + uint_fast64_t uiB64, + uint_fast64_t uiB0 + ) +{ + bool isSigNaNA, isSigNaNB; + uint_fast64_t uiNonsigA64, uiNonsigB64, uiMagA64, uiMagB64; + struct uint128 uiZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + isSigNaNA = softfloat_isSigNaNF128UI( uiA64, uiA0 ); + isSigNaNB = softfloat_isSigNaNF128UI( uiB64, uiB0 ); + /*------------------------------------------------------------------------ + | Make NaNs non-signaling. + *------------------------------------------------------------------------*/ + uiNonsigA64 = uiA64 | UINT64_C( 0x0000800000000000 ); + uiNonsigB64 = uiB64 | UINT64_C( 0x0000800000000000 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + if ( isSigNaNB ) goto returnLargerMag; + if ( isNaNF128UI( uiB64, uiB0 ) ) goto returnB; + goto returnA; + } else { + if ( isNaNF128UI( uiA64, uiA0 ) ) goto returnA; + goto returnB; + } + } + returnLargerMag: + uiMagA64 = uiNonsigA64 & UINT64_C( 0x7FFFFFFFFFFFFFFF ); + uiMagB64 = uiNonsigB64 & UINT64_C( 0x7FFFFFFFFFFFFFFF ); + if ( uiMagA64 < uiMagB64 ) goto returnB; + if ( uiMagB64 < uiMagA64 ) goto returnA; + if ( uiA0 < uiB0 ) goto returnB; + if ( uiB0 < uiA0 ) goto returnA; + if ( uiNonsigA64 < uiNonsigB64 ) goto returnA; + returnB: + uiZ.v64 = uiNonsigB64; + uiZ.v0 = uiB0; + return uiZ; + returnA: + uiZ.v64 = uiNonsigA64; + uiZ.v0 = uiA0; + return uiZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF32UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF32UI.c new file mode 100644 index 0000000..22ed68e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF32UI.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 32-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast32_t + softfloat_propagateNaNF32UI( uint_fast32_t uiA, uint_fast32_t uiB ) +{ + bool isSigNaNA, isSigNaNB; + uint_fast32_t uiNonsigA, uiNonsigB, uiMagA, uiMagB; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + isSigNaNA = softfloat_isSigNaNF32UI( uiA ); + isSigNaNB = softfloat_isSigNaNF32UI( uiB ); + /*------------------------------------------------------------------------ + | Make NaNs non-signaling. + *------------------------------------------------------------------------*/ + uiNonsigA = uiA | 0x00400000; + uiNonsigB = uiB | 0x00400000; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + if ( isSigNaNB ) goto returnLargerMag; + return isNaNF32UI( uiB ) ? uiNonsigB : uiNonsigA; + } else { + return isNaNF32UI( uiA ) ? uiNonsigA : uiNonsigB; + } + } + returnLargerMag: + uiMagA = uiNonsigA & 0x7FFFFFFF; + uiMagB = uiNonsigB & 0x7FFFFFFF; + if ( uiMagA < uiMagB ) return uiNonsigB; + if ( uiMagB < uiMagA ) return uiNonsigA; + return (uiNonsigA < uiNonsigB) ? uiNonsigA : uiNonsigB; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF64UI.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF64UI.c new file mode 100644 index 0000000..bb76f56 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/s_propagateNaNF64UI.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 64-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast64_t + softfloat_propagateNaNF64UI( uint_fast64_t uiA, uint_fast64_t uiB ) +{ + bool isSigNaNA, isSigNaNB; + uint_fast64_t uiNonsigA, uiNonsigB, uiMagA, uiMagB; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + isSigNaNA = softfloat_isSigNaNF64UI( uiA ); + isSigNaNB = softfloat_isSigNaNF64UI( uiB ); + /*------------------------------------------------------------------------ + | Make NaNs non-signaling. + *------------------------------------------------------------------------*/ + uiNonsigA = uiA | UINT64_C( 0x0008000000000000 ); + uiNonsigB = uiB | UINT64_C( 0x0008000000000000 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isSigNaNA | isSigNaNB ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( isSigNaNA ) { + if ( isSigNaNB ) goto returnLargerMag; + return isNaNF64UI( uiB ) ? uiNonsigB : uiNonsigA; + } else { + return isNaNF64UI( uiA ) ? uiNonsigA : uiNonsigB; + } + } + returnLargerMag: + uiMagA = uiNonsigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ); + uiMagB = uiNonsigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ); + if ( uiMagA < uiMagB ) return uiNonsigB; + if ( uiMagB < uiMagA ) return uiNonsigA; + return (uiNonsigA < uiNonsigB) ? uiNonsigA : uiNonsigB; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/softfloat_raiseFlags.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/softfloat_raiseFlags.c new file mode 100644 index 0000000..2b5aba8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/softfloat_raiseFlags.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include "platform.h" +#include "softfloat.h" + +/*---------------------------------------------------------------------------- +| Raises the exceptions specified by `flags'. Floating-point traps can be +| defined here if desired. It is currently not possible for such a trap +| to substitute a result value. If traps are not implemented, this routine +| should be simply `softfloat_exceptionFlags |= flags;'. +*----------------------------------------------------------------------------*/ +void softfloat_raiseFlags( uint_fast8_t flags ) +{ + + softfloat_exceptionFlags |= flags; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/specialize.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/specialize.h new file mode 100644 index 0000000..8f4d6c8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/8086/specialize.h @@ -0,0 +1,319 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#ifndef specialize_h +#define specialize_h 1 + +#include +#include +#include "softfloat_types.h" + +/*---------------------------------------------------------------------------- +| Default value for `softfloat_detectTininess'. +*----------------------------------------------------------------------------*/ +#define init_detectTininess softfloat_tininess_afterRounding; + +/*---------------------------------------------------------------------------- +| "Common NaN" structure, used to transfer NaN representations from one format +| to another. +*----------------------------------------------------------------------------*/ +struct commonNaN { + bool sign; +#ifdef LITTLEENDIAN + uint64_t v0, v64; +#else + uint64_t v64, v0; +#endif +}; + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 32-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF32UI 0xFFC00000 + +/*---------------------------------------------------------------------------- +| Returns true when 32-bit unsigned integer `uiA' has the bit pattern of a +| 32-bit floating-point signaling NaN. +| Note: This macro evaluates its argument more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNF32UI( uiA ) ((((uiA) & 0x7FC00000) == 0x7F800000) && ((uiA) & 0x003FFFFF)) + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 32-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f32UIToCommonNaN( uint_fast32_t uiA, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 32-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast32_t softfloat_commonNaNToF32UI( const struct commonNaN *aPtr ); + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 32-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast32_t + softfloat_propagateNaNF32UI( uint_fast32_t uiA, uint_fast32_t uiB ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 64-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF64UI UINT64_C( 0xFFF8000000000000 ) + +/*---------------------------------------------------------------------------- +| Returns true when 64-bit unsigned integer `uiA' has the bit pattern of a +| 64-bit floating-point signaling NaN. +| Note: This macro evaluates its argument more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNF64UI( uiA ) ((((uiA) & UINT64_C( 0x7FF8000000000000 )) == UINT64_C( 0x7FF0000000000000 )) && ((uiA) & UINT64_C( 0x0007FFFFFFFFFFFF ))) + +/*---------------------------------------------------------------------------- +| Assuming `uiA' has the bit pattern of a 64-bit floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void softfloat_f64UIToCommonNaN( uint_fast64_t uiA, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 64-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +uint_fast64_t softfloat_commonNaNToF64UI( const struct commonNaN *aPtr ); + +/*---------------------------------------------------------------------------- +| Interpreting `uiA' and `uiB' as the bit patterns of two 64-bit floating- +| point values, at least one of which is a NaN, returns the bit pattern of +| the combined NaN result. If either `uiA' or `uiB' has the pattern of a +| signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +uint_fast64_t + softfloat_propagateNaNF64UI( uint_fast64_t uiA, uint_fast64_t uiB ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 80-bit extended floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNExtF80UI64 0xFFFF +#define defaultNaNExtF80UI0 UINT64_C( 0xC000000000000000 ) + +/*---------------------------------------------------------------------------- +| Returns true when the 80-bit unsigned integer formed from concatenating +| 16-bit `uiA64' and 64-bit `uiA0' has the bit pattern of an 80-bit extended +| floating-point signaling NaN. +| Note: This macro evaluates its arguments more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) ((((uiA64) & 0x7FFF) == 0x7FFF) && ! ((uiA0) & UINT64_C( 0x4000000000000000 )) && ((uiA0) & UINT64_C( 0x3FFFFFFFFFFFFFFF ))) + +#ifdef SOFTFLOAT_FAST_INT64 + +/*---------------------------------------------------------------------------- +| The following functions are needed only when `SOFTFLOAT_FAST_INT64' is +| defined. +*----------------------------------------------------------------------------*/ + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of an 80-bit extended floating-point NaN, converts +| this NaN to the common NaN form, and stores the resulting common NaN at the +| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80UIToCommonNaN( + uint_fast16_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and returns the bit pattern of this value as an unsigned +| integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToExtF80UI( const struct commonNaN *aPtr ); + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as an 80-bit extended floating-point value, and likewise interpreting +| the unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 80-bit extended floating-point value, and assuming at least on of these +| floating-point values is a NaN, returns the bit pattern of the combined NaN +| result. If either original floating-point value is a signaling NaN, the +| invalid exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNExtF80UI( + uint_fast16_t uiA64, + uint_fast64_t uiA0, + uint_fast16_t uiB64, + uint_fast64_t uiB0 + ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 128-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF128UI64 UINT64_C( 0xFFFF800000000000 ) +#define defaultNaNF128UI0 UINT64_C( 0 ) + +/*---------------------------------------------------------------------------- +| Returns true when the 128-bit unsigned integer formed from concatenating +| 64-bit `uiA64' and 64-bit `uiA0' has the bit pattern of a 128-bit floating- +| point signaling NaN. +| Note: This macro evaluates its arguments more than once. +*----------------------------------------------------------------------------*/ +#define softfloat_isSigNaNF128UI( uiA64, uiA0 ) ((((uiA64) & UINT64_C( 0x7FFF800000000000 )) == UINT64_C( 0x7FFF000000000000 )) && ((uiA0) || ((uiA64) & UINT64_C( 0x00007FFFFFFFFFFF )))) + +/*---------------------------------------------------------------------------- +| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0' +| has the bit pattern of a 128-bit floating-point NaN, converts this NaN to +| the common NaN form, and stores the resulting common NaN at the location +| pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid exception +| is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_f128UIToCommonNaN( + uint_fast64_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and returns the bit pattern of this value as an unsigned integer. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_commonNaNToF128UI( const struct commonNaN * ); + +/*---------------------------------------------------------------------------- +| Interpreting the unsigned integer formed from concatenating `uiA64' and +| `uiA0' as a 128-bit floating-point value, and likewise interpreting the +| unsigned integer formed from concatenating `uiB64' and `uiB0' as another +| 128-bit floating-point value, and assuming at least on of these floating- +| point values is a NaN, returns the bit pattern of the combined NaN result. +| If either original floating-point value is a signaling NaN, the invalid +| exception is raised. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_propagateNaNF128UI( + uint_fast64_t uiA64, + uint_fast64_t uiA0, + uint_fast64_t uiB64, + uint_fast64_t uiB0 + ); + +#else + +/*---------------------------------------------------------------------------- +| The following functions are needed only when `SOFTFLOAT_FAST_INT64' is not +| defined. +*----------------------------------------------------------------------------*/ + +/*---------------------------------------------------------------------------- +| Assuming the 80-bit extended floating-point value pointed to by `aSPtr' is +| a NaN, converts this NaN to the common NaN form, and stores the resulting +| common NaN at the location pointed to by `zPtr'. If the NaN is a signaling +| NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_extF80MToCommonNaN( + const struct extFloat80M *aSPtr, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into an 80-bit extended +| floating-point NaN, and stores this NaN at the location pointed to by +| `zSPtr'. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToExtF80M( + const struct commonNaN *aPtr, struct extFloat80M *zSPtr ); + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 80-bit extended floating-point values +| pointed to by `aSPtr' and `bSPtr' is a NaN, stores the combined NaN result +| at the location pointed to by `zSPtr'. If either original floating-point +| value is a signaling NaN, the invalid exception is raised. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNExtF80M( + const struct extFloat80M *aSPtr, + const struct extFloat80M *bSPtr, + struct extFloat80M *zSPtr + ); + +/*---------------------------------------------------------------------------- +| The bit pattern for a default generated 128-bit floating-point NaN. +*----------------------------------------------------------------------------*/ +#define defaultNaNF128UI96 0xFFFF8000 +#define defaultNaNF128UI64 0 +#define defaultNaNF128UI32 0 +#define defaultNaNF128UI0 0 + +/*---------------------------------------------------------------------------- +| Assuming the 128-bit floating-point value pointed to by `aWPtr' is a NaN, +| converts this NaN to the common NaN form, and stores the resulting common +| NaN at the location pointed to by `zPtr'. If the NaN is a signaling NaN, +| the invalid exception is raised. Argument `aWPtr' points to an array of +| four 32-bit elements that concatenate in the platform's normal endian order +| to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_f128MToCommonNaN( const uint32_t *aWPtr, struct commonNaN *zPtr ); + +/*---------------------------------------------------------------------------- +| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point +| NaN, and stores this NaN at the location pointed to by `zWPtr'. Argument +| `zWPtr' points to an array of four 32-bit elements that concatenate in the +| platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_commonNaNToF128M( const struct commonNaN *aPtr, uint32_t *zWPtr ); + +/*---------------------------------------------------------------------------- +| Assuming at least one of the two 128-bit floating-point values pointed to by +| `aWPtr' and `bWPtr' is a NaN, stores the combined NaN result at the location +| pointed to by `zWPtr'. If either original floating-point value is a +| signaling NaN, the invalid exception is raised. Each of `aWPtr', `bWPtr', +| and `zWPtr' points to an array of four 32-bit elements that concatenate in +| the platform's normal endian order to form a 128-bit floating-point value. +*----------------------------------------------------------------------------*/ +void + softfloat_propagateNaNF128M( + const uint32_t *aWPtr, const uint32_t *bWPtr, uint32_t *zWPtr ); + +#endif + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_add.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_add.c new file mode 100644 index 0000000..25f4e96 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_add.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + extF80M_add( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + extFloat80_t + (*magsFuncPtr)( + uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool ); +#endif + + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + signA = signExtF80UI64( uiA64 ); + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + signB = signExtF80UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + *zPtr = softfloat_addMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + *zPtr = softfloat_subMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_addMagsExtF80 : softfloat_subMagsExtF80; + *zPtr = (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + +#else + +void + extF80M_add( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + + softfloat_addExtF80M( + (const struct extFloat80M *) aPtr, + (const struct extFloat80M *) bPtr, + (struct extFloat80M *) zPtr, + false + ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_div.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_div.c new file mode 100644 index 0000000..ef600c5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_div.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + extF80M_div( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + + *zPtr = extF80_div( *aPtr, *bPtr ); + +} + +#else + +void + extF80M_div( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + struct extFloat80M *zSPtr; + uint_fast16_t uiA64; + int32_t expA; + uint_fast16_t uiB64; + int32_t expB; + bool signZ; + uint64_t sigA, x64; + int32_t expZ; + int shiftCount; + uint32_t y[3], recip32, sigB[3]; + int ix; + uint32_t q, qs[2]; + uint_fast16_t uiZ64; + uint64_t uiZ0; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + zSPtr = (struct extFloat80M *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + expA = expExtF80UI64( uiA64 ); + uiB64 = bSPtr->signExp; + expB = expExtF80UI64( uiB64 ); + signZ = signExtF80UI64( uiA64 ) ^ signExtF80UI64( uiB64 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNExtF80M( aSPtr, bSPtr, zSPtr ) ) return; + if ( expA == 0x7FFF ) { + if ( expB == 0x7FFF ) goto invalid; + goto infinity; + } + goto zero; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + sigA = aSPtr->signif; + x64 = bSPtr->signif; + if ( ! expB ) expB = 1; + if ( ! (x64 & UINT64_C( 0x8000000000000000 )) ) { + if ( ! x64 ) { + if ( ! sigA ) goto invalid; + softfloat_raiseFlags( softfloat_flag_infinite ); + goto infinity; + } + expB += softfloat_normExtF80SigM( &x64 ); + } + if ( ! expA ) expA = 1; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigA ) goto zero; + expA += softfloat_normExtF80SigM( &sigA ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA - expB + 0x3FFF; + shiftCount = 29; + if ( sigA < x64 ) { + --expZ; + shiftCount = 30; + } + softfloat_shortShiftLeft64To96M( sigA, shiftCount, y ); + recip32 = softfloat_approxRecip32_1( x64>>32 ); + sigB[indexWord( 3, 0 )] = (uint32_t) x64<<30; + x64 >>= 2; + sigB[indexWord( 3, 2 )] = x64>>32; + sigB[indexWord( 3, 1 )] = x64; + ix = 2; + for (;;) { + x64 = (uint64_t) y[indexWordHi( 3 )] * recip32; + q = (x64 + 0x80000000)>>32; + --ix; + if ( ix < 0 ) break; + softfloat_remStep96MBy32( y, 29, sigB, q, y ); + if ( y[indexWordHi( 3 )] & 0x80000000 ) { + --q; + softfloat_add96M( y, sigB, y ); + } + qs[ix] = q; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ((q + 1) & 0x3FFFFF) < 2 ) { + softfloat_remStep96MBy32( y, 29, sigB, q, y ); + if ( y[indexWordHi( 3 )] & 0x80000000 ) { + --q; + softfloat_add96M( y, sigB, y ); + } else if ( softfloat_compare96M( sigB, y ) <= 0 ) { + ++q; + softfloat_sub96M( y, sigB, y ); + } + if ( + y[indexWordLo( 3 )] || y[indexWord( 3, 1 )] || y[indexWord( 3, 2 )] + ) { + q |= 1; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + x64 = (uint64_t) q<<9; + y[indexWord( 3, 0 )] = x64; + x64 = ((uint64_t) qs[0]<<6) + (x64>>32); + y[indexWord( 3, 1 )] = x64; + y[indexWord( 3, 2 )] = (qs[1]<<3) + (x64>>32); + softfloat_roundPackMToExtF80M( + signZ, expZ, y, extF80_roundingPrecision, zSPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_invalidExtF80M( zSPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infinity: + uiZ64 = packToExtF80UI64( signZ, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ64 = packToExtF80UI64( signZ, 0 ); + uiZ0 = 0; + uiZ: + zSPtr->signExp = uiZ64; + zSPtr->signif = uiZ0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_eq.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_eq.c new file mode 100644 index 0000000..bbf828d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_eq.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool extF80M_eq( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + + return extF80_eq( *aPtr, *bPtr ); + +} + +#else + +bool extF80M_eq( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiB64; + uint64_t uiB0; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) + || softfloat_isSigNaNExtF80UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( uiA0 == uiB0 ) { + return (uiA64 == uiB64) || ! uiA0; + } else { + if ( ! ((uiA0 & uiB0) & UINT64_C( 0x8000000000000000 )) ) { + return ! softfloat_compareNonnormExtF80M( aSPtr, bSPtr ); + } + return false; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_eq_signaling.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_eq_signaling.c new file mode 100644 index 0000000..d8879f3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_eq_signaling.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool extF80M_eq_signaling( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + + return extF80_eq_signaling( *aPtr, *bPtr ); + +} + +#else + +bool extF80M_eq_signaling( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiB64; + uint64_t uiB0; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( uiA0 == uiB0 ) { + return (uiA64 == uiB64) || ! uiA0; + } else { + if ( ! ((uiA0 & uiB0) & UINT64_C( 0x8000000000000000 )) ) { + return ! softfloat_compareNonnormExtF80M( aSPtr, bSPtr ); + } + return false; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_le.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_le.c new file mode 100644 index 0000000..fff29ba --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_le.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool extF80M_le( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + + return extF80_le( *aPtr, *bPtr ); + +} + +#else + +bool extF80M_le( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiB64; + uint64_t uiB0; + bool signA, ltMags; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signA = signExtF80UI64( uiA64 ); + if ( (uiA64 ^ uiB64) & 0x8000 ) { + /*-------------------------------------------------------------------- + | Signs are different. + *--------------------------------------------------------------------*/ + return signA || ! (uiA0 | uiB0); + } else { + /*-------------------------------------------------------------------- + | Signs are the same. + *--------------------------------------------------------------------*/ + if ( ! ((uiA0 & uiB0) & UINT64_C( 0x8000000000000000 )) ) { + return (softfloat_compareNonnormExtF80M( aSPtr, bSPtr ) <= 0); + } + if ( uiA64 == uiB64 ) { + if ( uiA0 == uiB0 ) return true; + ltMags = (uiA0 < uiB0); + } else { + ltMags = (uiA64 < uiB64); + } + return signA ^ ltMags; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_le_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_le_quiet.c new file mode 100644 index 0000000..5661064 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_le_quiet.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool extF80M_le_quiet( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + + return extF80_le_quiet( *aPtr, *bPtr ); + +} + +#else + +bool extF80M_le_quiet( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiB64; + uint64_t uiB0; + bool signA, ltMags; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) + || softfloat_isSigNaNExtF80UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signA = signExtF80UI64( uiA64 ); + if ( (uiA64 ^ uiB64) & 0x8000 ) { + /*-------------------------------------------------------------------- + | Signs are different. + *--------------------------------------------------------------------*/ + return signA || ! (uiA0 | uiB0); + } else { + /*-------------------------------------------------------------------- + | Signs are the same. + *--------------------------------------------------------------------*/ + if ( ! ((uiA0 & uiB0) & UINT64_C( 0x8000000000000000 )) ) { + return (softfloat_compareNonnormExtF80M( aSPtr, bSPtr ) <= 0); + } + if ( uiA64 == uiB64 ) { + if ( uiA0 == uiB0 ) return true; + ltMags = (uiA0 < uiB0); + } else { + ltMags = (uiA64 < uiB64); + } + return signA ^ ltMags; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_lt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_lt.c new file mode 100644 index 0000000..d9d2293 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_lt.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool extF80M_lt( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + + return extF80_lt( *aPtr, *bPtr ); + +} + +#else + +bool extF80M_lt( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiB64; + uint64_t uiB0; + bool signA, ltMags; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signA = signExtF80UI64( uiA64 ); + if ( (uiA64 ^ uiB64) & 0x8000 ) { + /*-------------------------------------------------------------------- + | Signs are different. + *--------------------------------------------------------------------*/ + return signA && ((uiA0 | uiB0) != 0); + } else { + /*-------------------------------------------------------------------- + | Signs are the same. + *--------------------------------------------------------------------*/ + if ( ! ((uiA0 & uiB0) & UINT64_C( 0x8000000000000000 )) ) { + return (softfloat_compareNonnormExtF80M( aSPtr, bSPtr ) < 0); + } + if ( uiA64 == uiB64 ) { + if ( uiA0 == uiB0 ) return false; + ltMags = (uiA0 < uiB0); + } else { + ltMags = (uiA64 < uiB64); + } + return signA ^ ltMags; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_lt_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_lt_quiet.c new file mode 100644 index 0000000..a650b9a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_lt_quiet.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool extF80M_lt_quiet( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + + return extF80_lt_quiet( *aPtr, *bPtr ); + +} + +#else + +bool extF80M_lt_quiet( const extFloat80_t *aPtr, const extFloat80_t *bPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint64_t uiA0; + uint_fast16_t uiB64; + uint64_t uiB0; + bool signA, ltMags; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) + || softfloat_isSigNaNExtF80UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signA = signExtF80UI64( uiA64 ); + if ( (uiA64 ^ uiB64) & 0x8000 ) { + /*-------------------------------------------------------------------- + | Signs are different. + *--------------------------------------------------------------------*/ + return signA && ((uiA0 | uiB0) != 0); + } else { + /*-------------------------------------------------------------------- + | Signs are the same. + *--------------------------------------------------------------------*/ + if ( ! ((uiA0 & uiB0) & UINT64_C( 0x8000000000000000 )) ) { + return (softfloat_compareNonnormExtF80M( aSPtr, bSPtr ) < 0); + } + if ( uiA64 == uiB64 ) { + if ( uiA0 == uiB0 ) return false; + ltMags = (uiA0 < uiB0); + } else { + ltMags = (uiA64 < uiB64); + } + return signA ^ ltMags; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_mul.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_mul.c new file mode 100644 index 0000000..4edaf2c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_mul.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + extF80M_mul( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + + *zPtr = extF80_mul( *aPtr, *bPtr ); + +} + +#else + +void + extF80M_mul( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + struct extFloat80M *zSPtr; + uint_fast16_t uiA64; + int32_t expA; + uint_fast16_t uiB64; + int32_t expB; + bool signZ; + uint_fast16_t exp, uiZ64; + uint64_t uiZ0, sigA, sigB; + int32_t expZ; + uint32_t sigProd[4], *extSigZPtr; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + zSPtr = (struct extFloat80M *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + expA = expExtF80UI64( uiA64 ); + uiB64 = bSPtr->signExp; + expB = expExtF80UI64( uiB64 ); + signZ = signExtF80UI64( uiA64 ) ^ signExtF80UI64( uiB64 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNExtF80M( aSPtr, bSPtr, zSPtr ) ) return; + if ( + (! aSPtr->signif && (expA != 0x7FFF)) + || (! bSPtr->signif && (expB != 0x7FFF)) + ) { + softfloat_invalidExtF80M( zSPtr ); + return; + } + uiZ64 = packToExtF80UI64( signZ, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) expA = 1; + sigA = aSPtr->signif; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigA ) goto zero; + expA += softfloat_normExtF80SigM( &sigA ); + } + if ( ! expB ) expB = 1; + sigB = bSPtr->signif; + if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigB ) goto zero; + expB += softfloat_normExtF80SigM( &sigB ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x3FFE; + softfloat_mul64To128M( sigA, sigB, sigProd ); + if ( sigProd[indexWordLo( 4 )] ) sigProd[indexWord( 4, 1 )] |= 1; + extSigZPtr = &sigProd[indexMultiwordHi( 4, 3 )]; + if ( sigProd[indexWordHi( 4 )] < 0x80000000 ) { + --expZ; + softfloat_add96M( extSigZPtr, extSigZPtr, extSigZPtr ); + } + softfloat_roundPackMToExtF80M( + signZ, expZ, extSigZPtr, extF80_roundingPrecision, zSPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ64 = packToExtF80UI64( signZ, 0 ); + uiZ0 = 0; + uiZ: + zSPtr->signExp = uiZ64; + zSPtr->signif = uiZ0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_rem.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_rem.c new file mode 100644 index 0000000..c497141 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_rem.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + extF80M_rem( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + + *zPtr = extF80_rem( *aPtr, *bPtr ); + +} + +#else + +void + extF80M_rem( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + struct extFloat80M *zSPtr; + uint_fast16_t uiA64; + int32_t expA, expB; + uint64_t x64; + bool signRem; + uint64_t sigA; + int32_t expDiff; + uint32_t rem[3], x[3], sig32B, q, recip32, rem2[3], *remPtr, *altRemPtr; + uint32_t *newRemPtr, wordMeanRem; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + zSPtr = (struct extFloat80M *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + expA = expExtF80UI64( uiA64 ); + expB = expExtF80UI64( bSPtr->signExp ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNExtF80M( aSPtr, bSPtr, zSPtr ) ) return; + if ( expA == 0x7FFF ) goto invalid; + /*-------------------------------------------------------------------- + | If we get here, then argument b is an infinity and `expB' is 0x7FFF; + | Doubling `expB' is an easy way to ensure that `expDiff' later is + | less than -1, which will result in returning a canonicalized version + | of argument a. + *--------------------------------------------------------------------*/ + expB += expB; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) expB = 1; + x64 = bSPtr->signif; + if ( ! (x64 & UINT64_C( 0x8000000000000000 )) ) { + if ( ! x64 ) goto invalid; + expB += softfloat_normExtF80SigM( &x64 ); + } + signRem = signExtF80UI64( uiA64 ); + if ( ! expA ) expA = 1; + sigA = aSPtr->signif; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigA ) { + expA = 0; + goto copyA; + } + expA += softfloat_normExtF80SigM( &sigA ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expA - expB; + if ( expDiff < -1 ) goto copyA; + rem[indexWord( 3, 2 )] = sigA>>34; + rem[indexWord( 3, 1 )] = sigA>>2; + rem[indexWord( 3, 0 )] = (uint32_t) sigA<<30; + x[indexWord( 3, 0 )] = (uint32_t) x64<<30; + sig32B = x64>>32; + x64 >>= 2; + x[indexWord( 3, 2 )] = x64>>32; + x[indexWord( 3, 1 )] = x64; + if ( expDiff < 1 ) { + if ( expDiff ) { + --expB; + softfloat_add96M( x, x, x ); + q = 0; + } else { + q = (softfloat_compare96M( x, rem ) <= 0); + if ( q ) softfloat_sub96M( rem, x, rem ); + } + } else { + recip32 = softfloat_approxRecip32_1( sig32B ); + expDiff -= 30; + for (;;) { + x64 = (uint64_t) rem[indexWordHi( 3 )] * recip32; + if ( expDiff < 0 ) break; + q = (x64 + 0x80000000)>>32; + softfloat_remStep96MBy32( rem, 29, x, q, rem ); + if ( rem[indexWordHi( 3 )] & 0x80000000 ) { + softfloat_add96M( rem, x, rem ); + } + expDiff -= 29; + } + /*-------------------------------------------------------------------- + | (`expDiff' cannot be less than -29 here.) + *--------------------------------------------------------------------*/ + q = (uint32_t) (x64>>32)>>(~expDiff & 31); + softfloat_remStep96MBy32( rem, expDiff + 30, x, q, rem ); + if ( rem[indexWordHi( 3 )] & 0x80000000 ) { + remPtr = rem; + altRemPtr = rem2; + softfloat_add96M( remPtr, x, altRemPtr ); + goto selectRem; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + remPtr = rem; + altRemPtr = rem2; + do { + ++q; + newRemPtr = altRemPtr; + softfloat_sub96M( remPtr, x, newRemPtr ); + altRemPtr = remPtr; + remPtr = newRemPtr; + } while ( ! (remPtr[indexWordHi( 3 )] & 0x80000000) ); + selectRem: + softfloat_add96M( remPtr, altRemPtr, x ); + wordMeanRem = x[indexWordHi( 3 )]; + if ( + (wordMeanRem & 0x80000000) + || (! wordMeanRem && (q & 1) && ! x[indexWord( 3, 0 )] + && ! x[indexWord( 3, 1 )]) + ) { + remPtr = altRemPtr; + } + if ( remPtr[indexWordHi( 3 )] & 0x80000000 ) { + signRem = ! signRem; + softfloat_negX96M( remPtr ); + } + softfloat_normRoundPackMToExtF80M( signRem, expB + 2, remPtr, 80, zSPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_invalidExtF80M( zSPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + copyA: + if ( expA < 1 ) { + sigA >>= 1 - expA; + expA = 0; + } + zSPtr->signExp = packToExtF80UI64( signRem, expA ); + zSPtr->signif = sigA; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_roundToInt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_roundToInt.c new file mode 100644 index 0000000..b89af5e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_roundToInt.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + extF80M_roundToInt( + const extFloat80_t *aPtr, + uint_fast8_t roundingMode, + bool exact, + extFloat80_t *zPtr + ) +{ + + *zPtr = extF80_roundToInt( *aPtr, roundingMode, exact ); + +} + +#else + +void + extF80M_roundToInt( + const extFloat80_t *aPtr, + uint_fast8_t roundingMode, + bool exact, + extFloat80_t *zPtr + ) +{ + const struct extFloat80M *aSPtr; + struct extFloat80M *zSPtr; + uint_fast16_t uiA64, signUI64; + int32_t exp; + uint64_t sigA; + uint_fast16_t uiZ64; + uint64_t sigZ, lastBitMask, roundBitsMask; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + zSPtr = (struct extFloat80M *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + signUI64 = uiA64 & packToExtF80UI64( 1, 0 ); + exp = expExtF80UI64( uiA64 ); + sigA = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) && (exp != 0x7FFF) ) { + if ( ! sigA ) { + uiZ64 = signUI64; + sigZ = 0; + goto uiZ; + } + exp += softfloat_normExtF80SigM( &sigA ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp <= 0x3FFE ) { + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + switch ( roundingMode ) { + case softfloat_round_near_even: + if ( ! (sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) break; + case softfloat_round_near_maxMag: + if ( exp == 0x3FFE ) goto mag1; + break; + case softfloat_round_min: + if ( signUI64 ) goto mag1; + break; + case softfloat_round_max: + if ( ! signUI64 ) goto mag1; + break; + } + uiZ64 = signUI64; + sigZ = 0; + goto uiZ; + mag1: + uiZ64 = signUI64 | 0x3FFF; + sigZ = UINT64_C( 0x8000000000000000 ); + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x403E <= exp ) { + if ( exp == 0x7FFF ) { + if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + softfloat_propagateNaNExtF80M( aSPtr, 0, zSPtr ); + return; + } + sigZ = UINT64_C( 0x8000000000000000 ); + } else { + sigZ = sigA; + } + uiZ64 = signUI64 | exp; + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ64 = signUI64 | exp; + lastBitMask = (uint64_t) 1<<(0x403E - exp); + roundBitsMask = lastBitMask - 1; + sigZ = sigA; + if ( roundingMode == softfloat_round_near_maxMag ) { + sigZ += lastBitMask>>1; + } else if ( roundingMode == softfloat_round_near_even ) { + sigZ += lastBitMask>>1; + if ( ! (sigZ & roundBitsMask) ) sigZ &= ~lastBitMask; + } else if ( roundingMode != softfloat_round_minMag ) { + if ( (signUI64 != 0) ^ (roundingMode == softfloat_round_max) ) { + sigZ += roundBitsMask; + } + } + sigZ &= ~roundBitsMask; + if ( ! sigZ ) { + ++uiZ64; + sigZ = UINT64_C( 0x8000000000000000 ); + } + if ( exact && (sigZ != sigA) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + uiZ: + zSPtr->signExp = uiZ64; + zSPtr->signif = sigZ; + return; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_sqrt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_sqrt.c new file mode 100644 index 0000000..2be7f03 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_sqrt.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void extF80M_sqrt( const extFloat80_t *aPtr, extFloat80_t *zPtr ) +{ + + *zPtr = extF80_sqrt( *aPtr ); + +} + +#else + +void extF80M_sqrt( const extFloat80_t *aPtr, extFloat80_t *zPtr ) +{ + const struct extFloat80M *aSPtr; + struct extFloat80M *zSPtr; + uint_fast16_t uiA64, signUI64; + int32_t expA; + uint64_t rem64; + int32_t expZ; + uint32_t rem[4], sig32A, recipSqrt32, sig32Z, q; + uint64_t sig64Z, x64; + uint32_t term[4], extSigZ[3]; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + zSPtr = (struct extFloat80M *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + signUI64 = uiA64 & packToExtF80UI64( 1, 0 ); + expA = expExtF80UI64( uiA64 ); + rem64 = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( rem64 & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + softfloat_propagateNaNExtF80M( aSPtr, 0, zSPtr ); + return; + } + if ( signUI64 ) goto invalid; + rem64 = UINT64_C( 0x8000000000000000 ); + goto copyA; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) expA = 1; + if ( ! (rem64 & UINT64_C( 0x8000000000000000 )) ) { + if ( ! rem64 ) { + uiA64 = signUI64; + goto copyA; + } + expA += softfloat_normExtF80SigM( &rem64 ); + } + if ( signUI64 ) goto invalid; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = ((expA - 0x3FFF)>>1) + 0x3FFF; + expA &= 1; + softfloat_shortShiftLeft64To96M( + rem64, 30 - expA, &rem[indexMultiwordHi( 4, 3 )] ); + sig32A = rem64>>32; + recipSqrt32 = softfloat_approxRecipSqrt32_1( expA, sig32A ); + sig32Z = ((uint64_t) sig32A * recipSqrt32)>>32; + if ( expA ) sig32Z >>= 1; + rem64 = + ((uint64_t) rem[indexWord( 4, 3 )]<<32 | rem[indexWord( 4, 2 )]) + - (uint64_t) sig32Z * sig32Z; + rem[indexWord( 4, 3 )] = rem64>>32; + rem[indexWord( 4, 2 )] = rem64; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = ((uint32_t) (rem64>>2) * (uint64_t) recipSqrt32)>>32; + sig64Z = ((uint64_t) sig32Z<<32) + ((uint64_t) q<<3); + x64 = ((uint64_t) sig32Z<<32) + sig64Z; + term[indexWord( 3, 2 )] = 0; + term[indexWord( 3, 1 )] = x64>>32; + term[indexWord( 3, 0 )] = x64; + softfloat_remStep96MBy32( + &rem[indexMultiwordHi( 4, 3 )], + 29, + term, + q, + &rem[indexMultiwordHi( 4, 3 )] + ); + rem64 = (uint64_t) rem[indexWord( 4, 3 )]<<32 | rem[indexWord( 4, 2 )]; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = (((uint32_t) (rem64>>2) * (uint64_t) recipSqrt32)>>32) + 2; + x64 = (uint64_t) q<<7; + extSigZ[indexWord( 3, 0 )] = x64; + x64 = (sig64Z<<1) + (x64>>32); + extSigZ[indexWord( 3, 2 )] = x64>>32; + extSigZ[indexWord( 3, 1 )] = x64; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (q & 0xFFFFFF) <= 2 ) { + q &= ~(uint32_t) 0xFFFF; + extSigZ[indexWordLo( 3 )] = q<<7; + x64 = sig64Z + (q>>27); + term[indexWord( 4, 3 )] = 0; + term[indexWord( 4, 2 )] = x64>>32; + term[indexWord( 4, 1 )] = x64; + term[indexWord( 4, 0 )] = q<<5; + rem[indexWord( 4, 0 )] = 0; + softfloat_remStep128MBy32( rem, 28, term, q, rem ); + q = rem[indexWordHi( 4 )]; + if ( q & 0x80000000 ) { + softfloat_sub1X96M( extSigZ ); + } else { + if ( q || rem[indexWord( 4, 1 )] || rem[indexWord( 4, 2 )] ) { + extSigZ[indexWordLo( 3 )] |= 1; + } + } + } + softfloat_roundPackMToExtF80M( + 0, expZ, extSigZ, extF80_roundingPrecision, zSPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_invalidExtF80M( zSPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + copyA: + zSPtr->signExp = uiA64; + zSPtr->signif = rem64; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_sub.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_sub.c new file mode 100644 index 0000000..1688635 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_sub.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + extF80M_sub( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + const struct extFloat80M *aSPtr, *bSPtr; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + extFloat80_t + (*magsFuncPtr)( + uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool ); +#endif + + aSPtr = (const struct extFloat80M *) aPtr; + bSPtr = (const struct extFloat80M *) bPtr; + uiA64 = aSPtr->signExp; + uiA0 = aSPtr->signif; + signA = signExtF80UI64( uiA64 ); + uiB64 = bSPtr->signExp; + uiB0 = bSPtr->signif; + signB = signExtF80UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + *zPtr = softfloat_subMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + *zPtr = softfloat_addMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_subMagsExtF80 : softfloat_addMagsExtF80; + *zPtr = (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + +#else + +void + extF80M_sub( + const extFloat80_t *aPtr, const extFloat80_t *bPtr, extFloat80_t *zPtr ) +{ + + softfloat_addExtF80M( + (const struct extFloat80M *) aPtr, + (const struct extFloat80M *) bPtr, + (struct extFloat80M *) zPtr, + true + ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f128M.c new file mode 100644 index 0000000..ef6be5e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f128M.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void extF80M_to_f128M( const extFloat80_t *aPtr, float128_t *zPtr ) +{ + + *zPtr = extF80_to_f128( *aPtr ); + +} + +#else + +void extF80M_to_f128M( const extFloat80_t *aPtr, float128_t *zPtr ) +{ + const struct extFloat80M *aSPtr; + uint32_t *zWPtr; + uint_fast16_t uiA64; + bool sign; + int32_t exp; + uint64_t sig; + struct commonNaN commonNaN; + uint32_t uiZ96; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + zWPtr = (uint32_t *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zWPtr[indexWord( 4, 0 )] = 0; + if ( exp == 0x7FFF ) { + if ( sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + softfloat_extF80MToCommonNaN( aSPtr, &commonNaN ); + softfloat_commonNaNToF128M( &commonNaN, zWPtr ); + return; + } + uiZ96 = packToF128UI96( sign, 0x7FFF, 0 ); + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp ) --exp; + if ( ! (sig & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sig ) { + uiZ96 = packToF128UI96( sign, 0, 0 ); + goto uiZ; + } + exp += softfloat_normExtF80SigM( &sig ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zWPtr[indexWord( 4, 1 )] = (uint32_t) sig<<17; + sig >>= 15; + zWPtr[indexWord( 4, 2 )] = sig; + if ( exp < 0 ) { + zWPtr[indexWordHi( 4 )] = sig>>32; + softfloat_shiftRight96M( + &zWPtr[indexMultiwordHi( 4, 3 )], + -exp, + &zWPtr[indexMultiwordHi( 4, 3 )] + ); + exp = 0; + sig = (uint64_t) zWPtr[indexWordHi( 4 )]<<32; + } + zWPtr[indexWordHi( 4 )] = packToF128UI96( sign, exp, sig>>32 ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ: + zWPtr[indexWord( 4, 3 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f32.c new file mode 100644 index 0000000..e666d49 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f32.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +float32_t extF80M_to_f32( const extFloat80_t *aPtr ) +{ + + return extF80_to_f32( *aPtr ); + +} + +#else + +float32_t extF80M_to_f32( const extFloat80_t *aPtr ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + bool sign; + int32_t exp; + uint64_t sig; + struct commonNaN commonNaN; + uint32_t uiZ, sig32; + union ui32_f32 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp == 0x7FFF ) { + if ( sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + softfloat_extF80MToCommonNaN( aSPtr, &commonNaN ); + uiZ = softfloat_commonNaNToF32UI( &commonNaN ); + } else { + uiZ = packToF32UI( sign, 0xFF, 0 ); + } + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! (sig & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sig ) { + uiZ = packToF32UI( sign, 0, 0 ); + goto uiZ; + } + exp += softfloat_normExtF80SigM( &sig ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + sig32 = softfloat_shortShiftRightJam64( sig, 33 ); + exp -= 0x3F81; + if ( sizeof (int_fast16_t) < sizeof (int32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return softfloat_roundPackToF32( sign, exp, sig32 | 0x40000000 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f64.c new file mode 100644 index 0000000..2080d80 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_f64.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +float64_t extF80M_to_f64( const extFloat80_t *aPtr ) +{ + + return extF80_to_f64( *aPtr ); + +} + +#else + +float64_t extF80M_to_f64( const extFloat80_t *aPtr ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + bool sign; + int32_t exp; + uint64_t sig; + struct commonNaN commonNaN; + uint64_t uiZ; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp == 0x7FFF ) { + if ( sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + softfloat_extF80MToCommonNaN( aSPtr, &commonNaN ); + uiZ = softfloat_commonNaNToF64UI( &commonNaN ); + } else { + uiZ = packToF64UI( sign, 0x7FF, 0 ); + } + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! (sig & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sig ) { + uiZ = packToF64UI( sign, 0, 0 ); + goto uiZ; + } + exp += softfloat_normExtF80SigM( &sig ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + sig = softfloat_shortShiftRightJam64( sig, 1 ); + exp -= 0x3C01; + if ( sizeof (int_fast16_t) < sizeof (int32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return + softfloat_roundPackToF64( + sign, exp, sig | UINT64_C( 0x4000000000000000 ) ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i32.c new file mode 100644 index 0000000..223dccb --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i32.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast32_t + extF80M_to_i32( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return extF80_to_i32( *aPtr, roundingMode, exact ); + +} + +#else + +int_fast32_t + extF80M_to_i32( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + bool sign; + int32_t exp; + uint64_t sig; + int32_t shiftCount; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) sign = 0; + shiftCount = 0x4037 - exp; + if ( shiftCount <= 0 ) { + if ( sig>>32 ) goto invalid; + if ( -32 < shiftCount ) { + sig <<= -shiftCount; + } else { + if ( (uint32_t) sig ) goto invalid; + } + } else { + sig = softfloat_shiftRightJam64( sig, shiftCount ); + } + return softfloat_roundPackToI32( sign, sig, roundingMode, exact ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return sign ? -0x7FFFFFFF - 1 : 0x7FFFFFFF; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i32_r_minMag.c new file mode 100644 index 0000000..78127d5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i32_r_minMag.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast32_t extF80M_to_i32_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + + return extF80_to_i32_r_minMag( *aPtr, exact ); + +} + +#else + +int_fast32_t extF80M_to_i32_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + int32_t exp; + uint64_t sig; + int32_t shiftCount; + bool raiseInexact; + int32_t z; + uint64_t shiftedSig; + uint32_t absZ; + union { uint32_t ui; int32_t i; } u; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! sig && (exp != 0x7FFF) ) return 0; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + raiseInexact = exact; + z = 0; + } else { + raiseInexact = false; + if ( shiftCount < 0 ) { + if ( sig>>32 || (shiftCount <= -31) ) goto invalid; + shiftedSig = (uint64_t) (uint32_t) sig<<-shiftCount; + if ( shiftedSig>>32 ) goto invalid; + absZ = shiftedSig; + } else { + shiftedSig = sig; + if ( shiftCount ) shiftedSig >>= shiftCount; + if ( shiftedSig>>32 ) goto invalid; + absZ = shiftedSig; + if ( exact && shiftCount ) { + raiseInexact = ((uint64_t) absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast64_t + extF80M_to_i64( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return extF80_to_i64( *aPtr, roundingMode, exact ); + +} + +#else + +int_fast64_t + extF80M_to_i64( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + bool sign; + int32_t exp; + uint64_t sig; + int32_t shiftCount; + uint32_t extSig[3]; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + shiftCount = 0x403E - exp; + if ( shiftCount < 0 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return + ! sign + || ((exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ? INT64_C( 0x7FFFFFFFFFFFFFFF ) + : -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + extSig[indexWord( 3, 2 )] = sig>>32; + extSig[indexWord( 3, 1 )] = sig; + extSig[indexWord( 3, 0 )] = 0; + if ( shiftCount ) softfloat_shiftRightJam96M( extSig, shiftCount, extSig ); + return softfloat_roundPackMToI64( sign, extSig, roundingMode, exact ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i64_r_minMag.c new file mode 100644 index 0000000..c9ce172 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_i64_r_minMag.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast64_t extF80M_to_i64_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + + return extF80_to_i64_r_minMag( *aPtr, exact ); + +} + +#else + +int_fast64_t extF80M_to_i64_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + int32_t exp; + uint64_t sig; + int32_t shiftCount; + bool raiseInexact; + int64_t z; + uint64_t absZ; + union { uint64_t ui; int64_t i; } u; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! sig && (exp != 0x7FFF) ) return 0; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + raiseInexact = exact; + z = 0; + } else { + raiseInexact = false; + if ( shiftCount < 0 ) { + if ( shiftCount <= -63 ) goto invalid; + shiftCount = -shiftCount; + absZ = sig<>shiftCount != sig ) goto invalid; + } else { + absZ = sig; + if ( shiftCount ) absZ >>= shiftCount; + if ( exact && shiftCount ) { + raiseInexact = (absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast32_t + extF80M_to_ui32( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return extF80_to_ui32( *aPtr, roundingMode, exact ); + +} + +#else + +uint_fast32_t + extF80M_to_ui32( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + bool sign; + int32_t exp; + uint64_t sig; + int32_t shiftCount; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + shiftCount = 0x4037 - exp; + if ( shiftCount <= 0 ) { + if ( sig>>32 ) goto invalid; + if ( -32 < shiftCount ) { + sig <<= -shiftCount; + } else { + if ( (uint32_t) sig ) goto invalid; + } + } else { + sig = softfloat_shiftRightJam64( sig, shiftCount ); + } + return softfloat_roundPackToUI32( sign, sig, roundingMode, exact ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return 0xFFFFFFFF; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_ui32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_ui32_r_minMag.c new file mode 100644 index 0000000..6d0a6cd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_ui32_r_minMag.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast32_t extF80M_to_ui32_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + + return extF80_to_ui32_r_minMag( *aPtr, exact ); + +} + +#else + +uint_fast32_t extF80M_to_ui32_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + int32_t exp; + uint64_t sig; + int32_t shiftCount; + bool sign; + uint64_t shiftedSig; + uint32_t z; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! sig && (exp != 0x7FFF) ) return 0; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + return 0; + } + sign = signExtF80UI64( uiA64 ); + if ( shiftCount < 0 ) { + if ( sign || sig>>32 || (shiftCount <= -31) ) goto invalid; + shiftedSig = (uint64_t) (uint32_t) sig<<-shiftCount; + if ( shiftedSig>>32 ) goto invalid; + z = shiftedSig; + } else { + shiftedSig = sig; + if ( shiftCount ) shiftedSig >>= shiftCount; + if ( shiftedSig>>32 ) goto invalid; + z = shiftedSig; + if ( sign && z ) goto invalid; + if ( exact && shiftCount && ((uint64_t) z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast64_t + extF80M_to_ui64( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return extF80_to_ui64( *aPtr, roundingMode, exact ); + +} + +#else + +uint_fast64_t + extF80M_to_ui64( + const extFloat80_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + int32_t exp, shiftCount; + bool sign; + uint64_t sig; + uint32_t extSig[3]; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + exp = expExtF80UI64( uiA64 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + shiftCount = 0x403E - exp; + if ( shiftCount < 0 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + sign = signExtF80UI64( uiA64 ); + sig = aSPtr->signif; + extSig[indexWord( 3, 2 )] = sig>>32; + extSig[indexWord( 3, 1 )] = sig; + extSig[indexWord( 3, 0 )] = 0; + if ( shiftCount ) softfloat_shiftRightJam96M( extSig, shiftCount, extSig ); + return softfloat_roundPackMToUI64( sign, extSig, roundingMode, exact ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_ui64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_ui64_r_minMag.c new file mode 100644 index 0000000..1e3e673 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80M_to_ui64_r_minMag.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast64_t extF80M_to_ui64_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + + return extF80_to_ui64_r_minMag( *aPtr, exact ); + +} + +#else + +uint_fast64_t extF80M_to_ui64_r_minMag( const extFloat80_t *aPtr, bool exact ) +{ + const struct extFloat80M *aSPtr; + uint_fast16_t uiA64; + int32_t exp; + uint64_t sig; + int32_t shiftCount; + bool sign; + uint64_t z; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aSPtr = (const struct extFloat80M *) aPtr; + uiA64 = aSPtr->signExp; + exp = expExtF80UI64( uiA64 ); + sig = aSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! sig && (exp != 0x7FFF) ) return 0; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + return 0; + } + sign = signExtF80UI64( uiA64 ); + if ( shiftCount < 0 ) { + if ( sign || (shiftCount <= -63) ) goto invalid; + shiftCount = -shiftCount; + z = sig<>shiftCount != sig ) goto invalid; + } else { + z = sig; + if ( shiftCount ) z >>= shiftCount; + if ( sign && z ) goto invalid; + if ( exact && shiftCount && (z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +extFloat80_t extF80_add( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + extFloat80_t + (*magsFuncPtr)( + uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool ); +#endif + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + signA = signExtF80UI64( uiA64 ); + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + signB = signExtF80UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_addMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + return softfloat_subMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_addMagsExtF80 : softfloat_subMagsExtF80; + return (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_div.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_div.c new file mode 100644 index 0000000..45a7bb4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_div.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t extF80_div( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + int_fast32_t expA; + uint_fast64_t sigA; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signB; + int_fast32_t expB; + uint_fast64_t sigB; + bool signZ; + struct exp32_sig64 normExpSig; + int_fast32_t expZ; + struct uint128 rem; + uint_fast32_t recip32; + uint_fast64_t sigZ; + int ix; + uint_fast64_t q64; + uint_fast32_t q; + struct uint128 term; + uint_fast64_t sigZExtra; + struct uint128 uiZ; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + signA = signExtF80UI64( uiA64 ); + expA = expExtF80UI64( uiA64 ); + sigA = uiA0; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + signB = signExtF80UI64( uiB64 ); + expB = expExtF80UI64( uiB64 ); + sigB = uiB0; + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + if ( expB == 0x7FFF ) { + if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + goto invalid; + } + goto infinity; + } + if ( expB == 0x7FFF ) { + if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + goto zero; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) expB = 1; + if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigB ) { + if ( ! sigA ) goto invalid; + softfloat_raiseFlags( softfloat_flag_infinite ); + goto infinity; + } + normExpSig = softfloat_normSubnormalExtF80Sig( sigB ); + expB += normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) expA = 1; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigA ) goto zero; + normExpSig = softfloat_normSubnormalExtF80Sig( sigA ); + expA += normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA - expB + 0x3FFF; + if ( sigA < sigB ) { + --expZ; + rem = softfloat_shortShiftLeft128( 0, sigA, 32 ); + } else { + rem = softfloat_shortShiftLeft128( 0, sigA, 31 ); + } + recip32 = softfloat_approxRecip32_1( sigB>>32 ); + sigZ = 0; + ix = 2; + for (;;) { + q64 = (uint_fast64_t) (uint32_t) (rem.v64>>2) * recip32; + q = (q64 + 0x80000000)>>32; + --ix; + if ( ix < 0 ) break; + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + term = softfloat_mul64ByShifted32To128( sigB, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + --q; + rem = softfloat_add128( rem.v64, rem.v0, sigB>>32, sigB<<32 ); + } + sigZ = (sigZ<<29) + q; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ((q + 1) & 0x3FFFFF) < 2 ) { + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + term = softfloat_mul64ByShifted32To128( sigB, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + term = softfloat_shortShiftLeft128( 0, sigB, 32 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + --q; + rem = softfloat_add128( rem.v64, rem.v0, term.v64, term.v0 ); + } else if ( softfloat_le128( term.v64, term.v0, rem.v64, rem.v0 ) ) { + ++q; + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + } + if ( rem.v64 | rem.v0 ) q |= 1; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + sigZ = (sigZ<<6) + (q>>23); + sigZExtra = (uint64_t) ((uint_fast64_t) q<<41); + return + softfloat_roundPackToExtF80( + signZ, expZ, sigZ, sigZExtra, extF80_roundingPrecision ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNExtF80UI( uiA64, uiA0, uiB64, uiB0 ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ64 = defaultNaNExtF80UI64; + uiZ0 = defaultNaNExtF80UI0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infinity: + uiZ64 = packToExtF80UI64( signZ, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ64 = packToExtF80UI64( signZ, 0 ); + uiZ0 = 0; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_eq.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_eq.c new file mode 100644 index 0000000..fd66662 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_eq.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool extF80_eq( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) + || softfloat_isSigNaNExtF80UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + return + (uiA0 == uiB0) + && ((uiA64 == uiB64) || (! uiA0 && ! ((uiA64 | uiB64) & 0x7FFF))); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_eq_signaling.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_eq_signaling.c new file mode 100644 index 0000000..d684701 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_eq_signaling.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool extF80_eq_signaling( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + return + (uiA0 == uiB0) + && ((uiA64 == uiB64) || (! uiA0 && ! ((uiA64 | uiB64) & 0x7FFF))); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_isSignalingNaN.c new file mode 100644 index 0000000..f6cea98 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_isSignalingNaN.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool extF80_isSignalingNaN( extFloat80_t a ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + + uA.f = a; + return softfloat_isSigNaNExtF80UI( uA.s.signExp, uA.s.signif ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_le.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_le.c new file mode 100644 index 0000000..fc5ed46 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_le.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool extF80_le( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signExtF80UI64( uiA64 ); + signB = signExtF80UI64( uiB64 ); + return + (signA != signB) + ? signA || ! (((uiA64 | uiB64) & 0x7FFF) | uiA0 | uiB0) + : ((uiA64 == uiB64) && (uiA0 == uiB0)) + || (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_le_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_le_quiet.c new file mode 100644 index 0000000..d526ecb --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_le_quiet.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool extF80_le_quiet( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) + || softfloat_isSigNaNExtF80UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signExtF80UI64( uiA64 ); + signB = signExtF80UI64( uiB64 ); + return + (signA != signB) + ? signA || ! (((uiA64 | uiB64) & 0x7FFF) | uiA0 | uiB0) + : ((uiA64 == uiB64) && (uiA0 == uiB0)) + || (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_lt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_lt.c new file mode 100644 index 0000000..d9a1a8d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_lt.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool extF80_lt( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signExtF80UI64( uiA64 ); + signB = signExtF80UI64( uiB64 ); + return + (signA != signB) + ? signA && (((uiA64 | uiB64) & 0x7FFF) | uiA0 | uiB0) + : ((uiA64 != uiB64) || (uiA0 != uiB0)) + && (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_lt_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_lt_quiet.c new file mode 100644 index 0000000..bcaaf0c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_lt_quiet.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool extF80_lt_quiet( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) + || softfloat_isSigNaNExtF80UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signExtF80UI64( uiA64 ); + signB = signExtF80UI64( uiB64 ); + return + (signA != signB) + ? signA && (((uiA64 | uiB64) & 0x7FFF) | uiA0 | uiB0) + : ((uiA64 != uiB64) || (uiA0 != uiB0)) + && (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_mul.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_mul.c new file mode 100644 index 0000000..272352c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_mul.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t extF80_mul( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + int_fast32_t expA; + uint_fast64_t sigA; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signB; + int_fast32_t expB; + uint_fast64_t sigB; + bool signZ; + uint_fast64_t magBits; + struct exp32_sig64 normExpSig; + int_fast32_t expZ; + struct uint128 sig128Z, uiZ; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + signA = signExtF80UI64( uiA64 ); + expA = expExtF80UI64( uiA64 ); + sigA = uiA0; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + signB = signExtF80UI64( uiB64 ); + expB = expExtF80UI64( uiB64 ); + sigB = uiB0; + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( + (sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + || ((expB == 0x7FFF) && (sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ) { + goto propagateNaN; + } + magBits = expB | sigB; + goto infArg; + } + if ( expB == 0x7FFF ) { + if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + magBits = expA | sigA; + goto infArg; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) expA = 1; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigA ) goto zero; + normExpSig = softfloat_normSubnormalExtF80Sig( sigA ); + expA += normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) expB = 1; + if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigB ) goto zero; + normExpSig = softfloat_normSubnormalExtF80Sig( sigB ); + expB += normExpSig.exp; + sigB = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x3FFE; + sig128Z = softfloat_mul64To128( sigA, sigB ); + if ( sig128Z.v64 < UINT64_C( 0x8000000000000000 ) ) { + --expZ; + sig128Z = + softfloat_add128( + sig128Z.v64, sig128Z.v0, sig128Z.v64, sig128Z.v0 ); + } + return + softfloat_roundPackToExtF80( + signZ, expZ, sig128Z.v64, sig128Z.v0, extF80_roundingPrecision ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNExtF80UI( uiA64, uiA0, uiB64, uiB0 ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infArg: + if ( ! magBits ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ64 = defaultNaNExtF80UI64; + uiZ0 = defaultNaNExtF80UI0; + } else { + uiZ64 = packToExtF80UI64( signZ, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + } + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ64 = packToExtF80UI64( signZ, 0 ); + uiZ0 = 0; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_rem.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_rem.c new file mode 100644 index 0000000..9dfb6f4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_rem.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t extF80_rem( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + int_fast32_t expA; + uint_fast64_t sigA; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signB; + int_fast32_t expB; + uint_fast64_t sigB; + struct exp32_sig64 normExpSig; + int_fast32_t expDiff; + struct uint128 rem, shiftedSigB; + uint_fast32_t q, recip32; + uint_fast64_t q64; + struct uint128 term, altRem, meanRem; + bool signRem; + struct uint128 uiZ; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + signA = signExtF80UI64( uiA64 ); + expA = expExtF80UI64( uiA64 ); + sigA = uiA0; + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + signB = signExtF80UI64( uiB64 ); + expB = expExtF80UI64( uiB64 ); + sigB = uiB0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( + (sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + || ((expB == 0x7FFF) && (sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ) { + goto propagateNaN; + } + goto invalid; + } + if ( expB == 0x7FFF ) { + if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + /*-------------------------------------------------------------------- + | Argument b is an infinity. Doubling `expB' is an easy way to ensure + | that `expDiff' later is less than -1, which will result in returning + | a canonicalized version of argument a. + *--------------------------------------------------------------------*/ + expB += expB; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) expB = 1; + if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigB ) goto invalid; + normExpSig = softfloat_normSubnormalExtF80Sig( sigB ); + expB += normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) expA = 1; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigA ) { + expA = 0; + goto copyA; + } + normExpSig = softfloat_normSubnormalExtF80Sig( sigA ); + expA += normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expA - expB; + if ( expDiff < -1 ) goto copyA; + rem = softfloat_shortShiftLeft128( 0, sigA, 32 ); + shiftedSigB = softfloat_shortShiftLeft128( 0, sigB, 32 ); + if ( expDiff < 1 ) { + if ( expDiff ) { + --expB; + shiftedSigB = softfloat_shortShiftLeft128( 0, sigB, 33 ); + q = 0; + } else { + q = (sigB <= sigA); + if ( q ) { + rem = + softfloat_sub128( + rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 ); + } + } + } else { + recip32 = softfloat_approxRecip32_1( sigB>>32 ); + expDiff -= 30; + for (;;) { + q64 = (uint_fast64_t) (uint32_t) (rem.v64>>2) * recip32; + if ( expDiff < 0 ) break; + q = (q64 + 0x80000000)>>32; + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + term = softfloat_mul64ByShifted32To128( sigB, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + rem = + softfloat_add128( + rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 ); + } + expDiff -= 29; + } + /*-------------------------------------------------------------------- + | (`expDiff' cannot be less than -29 here.) + *--------------------------------------------------------------------*/ + q = (uint32_t) (q64>>32)>>(~expDiff & 31); + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, expDiff + 30 ); + term = softfloat_mul64ByShifted32To128( sigB, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + altRem = + softfloat_add128( + rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 ); + goto selectRem; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + do { + altRem = rem; + ++q; + rem = + softfloat_sub128( + rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 ); + } while ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) ); + selectRem: + meanRem = softfloat_add128( rem.v64, rem.v0, altRem.v64, altRem.v0 ); + if ( + (meanRem.v64 & UINT64_C( 0x8000000000000000 )) + || (! (meanRem.v64 | meanRem.v0) && (q & 1)) + ) { + rem = altRem; + } + signRem = signA; + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + signRem = ! signRem; + rem = softfloat_sub128( 0, 0, rem.v64, rem.v0 ); + } + return + softfloat_normRoundPackToExtF80( + signRem, expB + 32, rem.v64, rem.v0, 80 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNExtF80UI( uiA64, uiA0, uiB64, uiB0 ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ64 = defaultNaNExtF80UI64; + uiZ0 = defaultNaNExtF80UI0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + copyA: + if ( expA < 1 ) { + sigA >>= 1 - expA; + expA = 0; + } + uiZ64 = packToExtF80UI64( signA, expA ); + uiZ0 = sigA; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_roundToInt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_roundToInt.c new file mode 100644 index 0000000..0598e17 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_roundToInt.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t + extF80_roundToInt( extFloat80_t a, uint_fast8_t roundingMode, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64, signUI64; + int_fast32_t exp; + uint_fast64_t sigA; + uint_fast16_t uiZ64; + uint_fast64_t sigZ; + struct exp32_sig64 normExpSig; + struct uint128 uiZ; + uint_fast64_t lastBitMask, roundBitsMask; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.s.signExp; + signUI64 = uiA64 & packToExtF80UI64( 1, 0 ); + exp = expExtF80UI64( uiA64 ); + sigA = uA.s.signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) && (exp != 0x7FFF) ) { + if ( ! sigA ) { + uiZ64 = signUI64; + sigZ = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalExtF80Sig( sigA ); + exp += normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x403E <= exp ) { + if ( exp == 0x7FFF ) { + if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + uiZ = softfloat_propagateNaNExtF80UI( uiA64, sigA, 0, 0 ); + uiZ64 = uiZ.v64; + sigZ = uiZ.v0; + goto uiZ; + } + sigZ = UINT64_C( 0x8000000000000000 ); + } else { + sigZ = sigA; + } + uiZ64 = signUI64 | exp; + goto uiZ; + } + if ( exp <= 0x3FFE ) { + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + switch ( roundingMode ) { + case softfloat_round_near_even: + if ( ! (sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) break; + case softfloat_round_near_maxMag: + if ( exp == 0x3FFE ) goto mag1; + break; + case softfloat_round_min: + if ( signUI64 ) goto mag1; + break; + case softfloat_round_max: + if ( ! signUI64 ) goto mag1; + break; + } + uiZ64 = signUI64; + sigZ = 0; + goto uiZ; + mag1: + uiZ64 = signUI64 | 0x3FFF; + sigZ = UINT64_C( 0x8000000000000000 ); + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ64 = signUI64 | exp; + lastBitMask = (uint_fast64_t) 1<<(0x403E - exp); + roundBitsMask = lastBitMask - 1; + sigZ = sigA; + if ( roundingMode == softfloat_round_near_maxMag ) { + sigZ += lastBitMask>>1; + } else if ( roundingMode == softfloat_round_near_even ) { + sigZ += lastBitMask>>1; + if ( ! (sigZ & roundBitsMask) ) sigZ &= ~lastBitMask; + } else if ( roundingMode != softfloat_round_minMag ) { + if ( (signUI64 != 0) ^ (roundingMode == softfloat_round_max) ) { + sigZ += roundBitsMask; + } + } + sigZ &= ~roundBitsMask; + if ( ! sigZ ) { + ++uiZ64; + sigZ = UINT64_C( 0x8000000000000000 ); + } + if ( exact && (sigZ != sigA) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = sigZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_sqrt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_sqrt.c new file mode 100644 index 0000000..a53808a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_sqrt.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t extF80_sqrt( extFloat80_t a ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + int_fast32_t expA; + uint_fast64_t sigA; + struct uint128 uiZ; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + struct exp32_sig64 normExpSig; + int_fast32_t expZ; + uint_fast32_t sig32A, recipSqrt32, sig32Z; + struct uint128 rem; + uint_fast64_t q, sigZ, x64; + struct uint128 term; + uint_fast64_t sigZExtra; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + signA = signExtF80UI64( uiA64 ); + expA = expExtF80UI64( uiA64 ); + sigA = uiA0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + uiZ = softfloat_propagateNaNExtF80UI( uiA64, uiA0, 0, 0 ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + goto uiZ; + } + if ( ! signA ) return a; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( signA ) { + if ( ! sigA ) goto zero; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) expA = 1; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( ! sigA ) goto zero; + normExpSig = softfloat_normSubnormalExtF80Sig( sigA ); + expA += normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + | (`sig32Z' is guaranteed to be a lower bound on the square root of + | `sig32A', which makes `sig32Z' also a lower bound on the square root of + | `sigA'.) + *------------------------------------------------------------------------*/ + expZ = ((expA - 0x3FFF)>>1) + 0x3FFF; + expA &= 1; + sig32A = sigA>>32; + recipSqrt32 = softfloat_approxRecipSqrt32_1( expA, sig32A ); + sig32Z = ((uint_fast64_t) sig32A * recipSqrt32)>>32; + if ( expA ) { + sig32Z >>= 1; + rem = softfloat_shortShiftLeft128( 0, sigA, 61 ); + } else { + rem = softfloat_shortShiftLeft128( 0, sigA, 62 ); + } + rem.v64 -= (uint_fast64_t) sig32Z * sig32Z; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = ((uint_fast64_t) (uint32_t) (rem.v64>>2) * recipSqrt32)>>32; + sigZ = ((uint_fast64_t) sig32Z<<32) + (q<<3); + x64 = ((uint_fast64_t) sig32Z<<32) + sigZ; + term = softfloat_mul64ByShifted32To128( x64, q ); + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = (((uint_fast64_t) (uint32_t) (rem.v64>>2) * recipSqrt32)>>32) + 2; + x64 = sigZ; + sigZ = (sigZ<<1) + (q>>25); + sigZExtra = (uint64_t) (q<<39); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (q & 0xFFFFFF) <= 2 ) { + q &= ~(uint_fast64_t) 0xFFFF; + sigZExtra = (uint64_t) (q<<39); + term = softfloat_mul64ByShifted32To128( x64 + (q>>27), q ); + x64 = (uint_fast64_t) (uint32_t) (q<<5) * (uint32_t) q; + term = softfloat_add128( term.v64, term.v0, 0, x64 ); + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 28 ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + if ( ! sigZExtra ) --sigZ; + --sigZExtra; + } else { + if ( rem.v64 | rem.v0 ) sigZExtra |= 1; + } + } + return + softfloat_roundPackToExtF80( + 0, expZ, sigZ, sigZExtra, extF80_roundingPrecision ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ64 = defaultNaNExtF80UI64; + uiZ0 = defaultNaNExtF80UI0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ64 = packToExtF80UI64( signA, 0 ); + uiZ0 = 0; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_sub.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_sub.c new file mode 100644 index 0000000..1c348e8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_sub.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +extFloat80_t extF80_sub( extFloat80_t a, extFloat80_t b ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool signA; + union { struct extFloat80M s; extFloat80_t f; } uB; + uint_fast16_t uiB64; + uint_fast64_t uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + extFloat80_t + (*magsFuncPtr)( + uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool ); +#endif + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + signA = signExtF80UI64( uiA64 ); + uB.f = b; + uiB64 = uB.s.signExp; + uiB0 = uB.s.signif; + signB = signExtF80UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_subMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + return softfloat_addMagsExtF80( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_subMagsExtF80 : softfloat_addMagsExtF80; + return (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f128.c new file mode 100644 index 0000000..01c3cd2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f128.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t extF80_to_f128( extFloat80_t a ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + uint_fast16_t exp; + uint_fast64_t sig; + struct commonNaN commonNaN; + struct uint128 uiZ; + bool sign; + struct uint128 sig128; + union ui128_f128 uZ; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + exp = expExtF80UI64( uiA64 ); + sig = uiA0 & UINT64_C( 0x7FFFFFFFFFFFFFFF ); + if ( (exp == 0x7FFF) && sig ) { + softfloat_extF80UIToCommonNaN( uiA64, uiA0, &commonNaN ); + uiZ = softfloat_commonNaNToF128UI( &commonNaN ); + } else { + sign = signExtF80UI64( uiA64 ); + sig128 = softfloat_shortShiftLeft128( 0, sig, 49 ); + uiZ.v64 = packToF128UI64( sign, exp, sig128.v64 ); + uiZ.v0 = sig128.v0; + } + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f32.c new file mode 100644 index 0000000..56482c2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f32.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t extF80_to_f32( extFloat80_t a ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig; + struct commonNaN commonNaN; + uint_fast32_t uiZ, sig32; + union ui32_f32 uZ; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = uiA0; + if ( exp == 0x7FFF ) { + if ( sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + softfloat_extF80UIToCommonNaN( uiA64, uiA0, &commonNaN ); + uiZ = softfloat_commonNaNToF32UI( &commonNaN ); + } else { + uiZ = packToF32UI( sign, 0xFF, 0 ); + } + goto uiZ; + } + sig32 = softfloat_shortShiftRightJam64( sig, 33 ); + if ( ! (exp | sig32) ) { + uiZ = packToF32UI( sign, 0, 0 ); + goto uiZ; + } + exp -= 0x3F81; + if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return softfloat_roundPackToF32( sign, exp, sig32 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f64.c new file mode 100644 index 0000000..0d3d897 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_f64.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t extF80_to_f64( extFloat80_t a ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + uint_fast64_t uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig; + struct commonNaN commonNaN; + uint_fast64_t uiZ; + union ui64_f64 uZ; + + uA.f = a; + uiA64 = uA.s.signExp; + uiA0 = uA.s.signif; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = uiA0; + if ( ! (exp | sig) ) { + uiZ = packToF64UI( sign, 0, 0 ); + goto uiZ; + } + if ( exp == 0x7FFF ) { + if ( sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + softfloat_extF80UIToCommonNaN( uiA64, uiA0, &commonNaN ); + uiZ = softfloat_commonNaNToF64UI( &commonNaN ); + } else { + uiZ = packToF64UI( sign, 0x7FF, 0 ); + } + goto uiZ; + } + sig = softfloat_shortShiftRightJam64( sig, 1 ); + exp -= 0x3C01; + if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return softfloat_roundPackToF64( sign, exp, sig ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i32.c new file mode 100644 index 0000000..f698b98 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i32.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t + extF80_to_i32( extFloat80_t a, uint_fast8_t roundingMode, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + bool sign; + int_fast32_t exp; + uint_fast64_t sig; + int_fast32_t shiftCount; + + uA.f = a; + uiA64 = uA.s.signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = uA.s.signif; + if ( (exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) sign = 0; + shiftCount = 0x4037 - exp; + if ( shiftCount <= 0 ) shiftCount = 1; + sig = softfloat_shiftRightJam64( sig, shiftCount ); + return softfloat_roundPackToI32( sign, sig, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i32_r_minMag.c new file mode 100644 index 0000000..d66cfe9 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i32_r_minMag.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t extF80_to_i32_r_minMag( extFloat80_t a, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + int_fast32_t exp; + uint_fast64_t sig; + int_fast32_t shiftCount; + bool sign; + int_fast32_t absZ; + + uA.f = a; + uiA64 = uA.s.signExp; + exp = expExtF80UI64( uiA64 ); + sig = uA.s.signif; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + sign = signExtF80UI64( uiA64 ); + if ( shiftCount < 33 ) { + if ( + (uiA64 == packToExtF80UI64( 1, 0x401E )) + && (sig < UINT64_C( 0x8000000100000000 )) + ) { + if ( exact && (sig & UINT64_C( 0x00000000FFFFFFFF )) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + } else { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( + ! sign + || ((exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ) { + return 0x7FFFFFFF; + } + } + return -0x7FFFFFFF - 1; + } + absZ = sig>>shiftCount; + if ( + exact && ((uint_fast64_t) (uint_fast32_t) absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t + extF80_to_i64( extFloat80_t a, uint_fast8_t roundingMode, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + bool sign; + int_fast32_t exp; + uint_fast64_t sig; + int_fast32_t shiftCount; + uint_fast64_t sigExtra; + struct uint64_extra sig64Extra; + + uA.f = a; + uiA64 = uA.s.signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = uA.s.signif; + shiftCount = 0x403E - exp; + if ( shiftCount <= 0 ) { + if ( shiftCount ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return + ! sign + || ((exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ? INT64_C( 0x7FFFFFFFFFFFFFFF ) + : -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + sigExtra = 0; + } else { + sig64Extra = softfloat_shiftRightJam64Extra( sig, 0, shiftCount ); + sig = sig64Extra.v; + sigExtra = sig64Extra.extra; + } + return + softfloat_roundPackToI64( sign, sig, sigExtra, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i64_r_minMag.c new file mode 100644 index 0000000..9ed5455 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_i64_r_minMag.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t extF80_to_i64_r_minMag( extFloat80_t a, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + int_fast32_t exp; + uint_fast64_t sig; + int_fast32_t shiftCount; + bool sign; + int_fast64_t absZ; + + uA.f = a; + uiA64 = uA.s.signExp; + exp = expExtF80UI64( uiA64 ); + sig = uA.s.signif; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + sign = signExtF80UI64( uiA64 ); + if ( shiftCount <= 0 ) { + if ( + (uiA64 != packToExtF80UI64( 1, 0x403E )) + || (sig != UINT64_C( 0x8000000000000000 )) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( + ! sign + || ((exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ) { + return INT64_C( 0x7FFFFFFFFFFFFFFF ); + } + } + return -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + absZ = sig>>shiftCount; + if ( exact && (uint64_t) (sig<<(-shiftCount & 63)) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return sign ? -absZ : absZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui32.c new file mode 100644 index 0000000..516a6fd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui32.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t + extF80_to_ui32( extFloat80_t a, uint_fast8_t roundingMode, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + bool sign; + int_fast32_t exp; + uint_fast64_t sig; + int_fast32_t shiftCount; + + uA.f = a; + uiA64 = uA.s.signExp; + sign = signExtF80UI64( uiA64 ); + exp = expExtF80UI64( uiA64 ); + sig = uA.s.signif; + shiftCount = 0x4037 - exp; + if ( shiftCount <= 0 ) shiftCount = 1; + sig = softfloat_shiftRightJam64( sig, shiftCount ); + return softfloat_roundPackToUI32( sign, sig, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui32_r_minMag.c new file mode 100644 index 0000000..4a2c119 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui32_r_minMag.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t extF80_to_ui32_r_minMag( extFloat80_t a, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + int_fast32_t exp; + uint_fast64_t sig; + int_fast32_t shiftCount; + uint_fast32_t z; + + uA.f = a; + uiA64 = uA.s.signExp; + exp = expExtF80UI64( uiA64 ); + sig = uA.s.signif; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signExtF80UI64( uiA64 ) || (shiftCount < 32) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return 0xFFFFFFFF; + } + z = sig>>shiftCount; + if ( exact && ((uint_fast64_t) z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t + extF80_to_ui64( extFloat80_t a, uint_fast8_t roundingMode, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + int_fast32_t exp, shiftCount; + bool sign; + uint_fast64_t sig, sigExtra; + struct uint64_extra sig64Extra; + + uA.f = a; + uiA64 = uA.s.signExp; + exp = expExtF80UI64( uiA64 ); + shiftCount = 0x403E - exp; + if ( shiftCount < 0 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + sign = signExtF80UI64( uiA64 ); + sig = uA.s.signif; + sigExtra = 0; + if ( shiftCount ) { + sig64Extra = softfloat_shiftRightJam64Extra( sig, 0, shiftCount ); + sig = sig64Extra.v; + sigExtra = sig64Extra.extra; + } + return + softfloat_roundPackToUI64( sign, sig, sigExtra, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui64_r_minMag.c new file mode 100644 index 0000000..4d48b08 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/extF80_to_ui64_r_minMag.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t extF80_to_ui64_r_minMag( extFloat80_t a, bool exact ) +{ + union { struct extFloat80M s; extFloat80_t f; } uA; + uint_fast16_t uiA64; + int_fast32_t exp; + uint_fast64_t sig; + int_fast32_t shiftCount; + uint_fast64_t z; + + uA.f = a; + uiA64 = uA.s.signExp; + exp = expExtF80UI64( uiA64 ); + sig = uA.s.signif; + shiftCount = 0x403E - exp; + if ( 64 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signExtF80UI64( uiA64 ) || (shiftCount < 0) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + z = sig>>shiftCount; + if ( exact && (z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + f128M_add( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + const uint64_t *aWPtr, *bWPtr; + uint_fast64_t uiA64, uiA0; + bool signA; + uint_fast64_t uiB64, uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + float128_t + (*magsFuncPtr)( + uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool ); +#endif + + aWPtr = (const uint64_t *) aPtr; + bWPtr = (const uint64_t *) bPtr; + uiA64 = aWPtr[indexWord( 2, 1 )]; + uiA0 = aWPtr[indexWord( 2, 0 )]; + signA = signF128UI64( uiA64 ); + uiB64 = bWPtr[indexWord( 2, 1 )]; + uiB0 = bWPtr[indexWord( 2, 0 )]; + signB = signF128UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + *zPtr = softfloat_addMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + *zPtr = softfloat_subMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_addMagsF128 : softfloat_subMagsF128; + *zPtr = (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + +#else + +void + f128M_add( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + + softfloat_addF128M( + (const uint32_t *) aPtr, + (const uint32_t *) bPtr, + (uint32_t *) zPtr, + false + ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_div.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_div.c new file mode 100644 index 0000000..5c9135f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_div.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + f128M_div( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + + *zPtr = f128_div( *aPtr, *bPtr ); + +} + +#else + +void + f128M_div( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t *zWPtr, uiA96; + bool signA; + int32_t expA; + uint32_t uiB96; + bool signB; + int32_t expB; + bool signZ; + uint32_t y[5], sigB[4]; + int32_t expZ; + uint32_t recip32; + int ix; + uint64_t q64; + uint32_t q, qs[3], uiZ96; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + zWPtr = (uint32_t *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA96 = aWPtr[indexWordHi( 4 )]; + signA = signF128UI96( uiA96 ); + expA = expF128UI96( uiA96 ); + uiB96 = bWPtr[indexWordHi( 4 )]; + signB = signF128UI96( uiB96 ); + expB = expF128UI96( uiB96 ); + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNF128M( aWPtr, bWPtr, zWPtr ) ) return; + if ( expA == 0x7FFF ) { + if ( expB == 0x7FFF ) goto invalid; + goto infinity; + } + goto zero; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expA = softfloat_shiftNormSigF128M( aWPtr, 13, y ); + expB = softfloat_shiftNormSigF128M( bWPtr, 13, sigB ); + if ( expA == -128 ) { + if ( expB == -128 ) goto invalid; + goto zero; + } + if ( expB == -128 ) { + softfloat_raiseFlags( softfloat_flag_infinite ); + goto infinity; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA - expB + 0x3FFE; + if ( softfloat_compare128M( y, sigB ) < 0 ) { + --expZ; + softfloat_add128M( y, y, y ); + } + recip32 = + softfloat_approxRecip32_1( + ((uint64_t) sigB[indexWord( 4, 3 )]<<32 | sigB[indexWord( 4, 2 )]) + >>30 + ); + ix = 3; + for (;;) { + q64 = (uint64_t) y[indexWordHi( 4 )] * recip32; + q = (q64 + 0x80000000)>>32; + --ix; + if ( ix < 0 ) break; + softfloat_remStep128MBy32( y, 29, sigB, q, y ); + if ( y[indexWordHi( 4 )] & 0x80000000 ) { + --q; + softfloat_add128M( y, sigB, y ); + } + qs[ix] = q; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ((q + 1) & 7) < 2 ) { + softfloat_remStep128MBy32( y, 29, sigB, q, y ); + if ( y[indexWordHi( 4 )] & 0x80000000 ) { + --q; + softfloat_add128M( y, sigB, y ); + } else if ( softfloat_compare128M( sigB, y ) <= 0 ) { + ++q; + softfloat_sub128M( y, sigB, y ); + } + if ( + y[indexWordLo( 4 )] || y[indexWord( 4, 1 )] + || (y[indexWord( 4, 2 )] | y[indexWord( 4, 3 )]) + ) { + q |= 1; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q64 = (uint64_t) q<<28; + y[indexWord( 5, 0 )] = q64; + q64 = ((uint64_t) qs[0]<<25) + (q64>>32); + y[indexWord( 5, 1 )] = q64; + q64 = ((uint64_t) qs[1]<<22) + (q64>>32); + y[indexWord( 5, 2 )] = q64; + q64 = ((uint64_t) qs[2]<<19) + (q64>>32); + y[indexWord( 5, 3 )] = q64; + y[indexWord( 5, 4 )] = q64>>32; + softfloat_roundPackMToF128M( signZ, expZ, y, zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_invalidF128M( zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infinity: + uiZ96 = packToF128UI96( signZ, 0x7FFF, 0 ); + goto uiZ96; + zero: + uiZ96 = packToF128UI96( signZ, 0, 0 ); + uiZ96: + zWPtr[indexWordHi( 4 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_eq.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_eq.c new file mode 100644 index 0000000..6c9de8d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_eq.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool f128M_eq( const float128_t *aPtr, const float128_t *bPtr ) +{ + + return f128_eq( *aPtr, *bPtr ); + +} + +#else + +bool f128M_eq( const float128_t *aPtr, const float128_t *bPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t wordA, wordB, uiA96, uiB96; + bool possibleOppositeZeros; + uint32_t mashWord; + + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + wordA = aWPtr[indexWord( 4, 2 )]; + wordB = bWPtr[indexWord( 4, 2 )]; + if ( wordA != wordB ) goto false_checkSigNaNs; + uiA96 = aWPtr[indexWordHi( 4 )]; + uiB96 = bWPtr[indexWordHi( 4 )]; + possibleOppositeZeros = false; + if ( uiA96 != uiB96 ) { + possibleOppositeZeros = (((uiA96 | uiB96) & 0x7FFFFFFF) == 0); + if ( ! possibleOppositeZeros ) goto false_checkSigNaNs; + } + mashWord = wordA | wordB; + wordA = aWPtr[indexWord( 4, 1 )]; + wordB = bWPtr[indexWord( 4, 1 )]; + if ( wordA != wordB ) goto false_checkSigNaNs; + mashWord |= wordA | wordB; + wordA = aWPtr[indexWord( 4, 0 )]; + wordB = bWPtr[indexWord( 4, 0 )]; + if ( wordA != wordB ) goto false_checkSigNaNs; + if ( possibleOppositeZeros && ((mashWord | wordA | wordB) != 0) ) { + goto false_checkSigNaNs; + } + if ( ! softfloat_isNaNF128M( aWPtr ) && ! softfloat_isNaNF128M( bWPtr ) ) { + return true; + } + false_checkSigNaNs: + if ( + f128M_isSignalingNaN( (const float128_t *) aWPtr ) + || f128M_isSignalingNaN( (const float128_t *) bWPtr ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_eq_signaling.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_eq_signaling.c new file mode 100644 index 0000000..2f90147 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_eq_signaling.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool f128M_eq_signaling( const float128_t *aPtr, const float128_t *bPtr ) +{ + + return f128_eq_signaling( *aPtr, *bPtr ); + +} + +#else + +bool f128M_eq_signaling( const float128_t *aPtr, const float128_t *bPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t wordA, wordB, uiA96, uiB96; + bool possibleOppositeZeros; + uint32_t mashWord; + + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + if ( softfloat_isNaNF128M( aWPtr ) || softfloat_isNaNF128M( bWPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + wordA = aWPtr[indexWord( 4, 2 )]; + wordB = bWPtr[indexWord( 4, 2 )]; + if ( wordA != wordB ) return false; + uiA96 = aWPtr[indexWordHi( 4 )]; + uiB96 = bWPtr[indexWordHi( 4 )]; + possibleOppositeZeros = false; + if ( uiA96 != uiB96 ) { + possibleOppositeZeros = (((uiA96 | uiB96) & 0x7FFFFFFF) == 0); + if ( ! possibleOppositeZeros ) return false; + } + mashWord = wordA | wordB; + wordA = aWPtr[indexWord( 4, 1 )]; + wordB = bWPtr[indexWord( 4, 1 )]; + if ( wordA != wordB ) return false; + mashWord |= wordA | wordB; + wordA = aWPtr[indexWord( 4, 0 )]; + wordB = bWPtr[indexWord( 4, 0 )]; + return + (wordA == wordB) + && (! possibleOppositeZeros || ((mashWord | wordA | wordB) == 0)); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_le.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_le.c new file mode 100644 index 0000000..55fe9d5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_le.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool f128M_le( const float128_t *aPtr, const float128_t *bPtr ) +{ + + return f128_le( *aPtr, *bPtr ); + +} + +#else + +bool f128M_le( const float128_t *aPtr, const float128_t *bPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t uiA96, uiB96; + bool signA, signB; + uint32_t wordA, wordB; + + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + if ( softfloat_isNaNF128M( aWPtr ) || softfloat_isNaNF128M( bWPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + uiA96 = aWPtr[indexWordHi( 4 )]; + uiB96 = bWPtr[indexWordHi( 4 )]; + signA = signF128UI96( uiA96 ); + signB = signF128UI96( uiB96 ); + if ( signA != signB ) { + if ( signA ) return true; + if ( (uiA96 | uiB96) & 0x7FFFFFFF ) return false; + wordA = aWPtr[indexWord( 4, 2 )]; + wordB = bWPtr[indexWord( 4, 2 )]; + if ( wordA | wordB ) return false; + wordA = aWPtr[indexWord( 4, 1 )]; + wordB = bWPtr[indexWord( 4, 1 )]; + if ( wordA | wordB ) return false; + wordA = aWPtr[indexWord( 4, 0 )]; + wordB = bWPtr[indexWord( 4, 0 )]; + return ((wordA | wordB) == 0); + } + if ( signA ) { + aWPtr = (const uint32_t *) bPtr; + bWPtr = (const uint32_t *) aPtr; + } + return (softfloat_compare128M( aWPtr, bWPtr ) <= 0); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_le_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_le_quiet.c new file mode 100644 index 0000000..39baf75 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_le_quiet.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool f128M_le_quiet( const float128_t *aPtr, const float128_t *bPtr ) +{ + + return f128_le_quiet( *aPtr, *bPtr ); + +} + +#else + +bool f128M_le_quiet( const float128_t *aPtr, const float128_t *bPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t uiA96, uiB96; + bool signA, signB; + uint32_t wordA, wordB; + + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + if ( softfloat_isNaNF128M( aWPtr ) || softfloat_isNaNF128M( bWPtr ) ) { + if ( f128M_isSignalingNaN( aPtr ) || f128M_isSignalingNaN( bPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + uiA96 = aWPtr[indexWordHi( 4 )]; + uiB96 = bWPtr[indexWordHi( 4 )]; + signA = signF128UI96( uiA96 ); + signB = signF128UI96( uiB96 ); + if ( signA != signB ) { + if ( signA ) return true; + if ( (uiA96 | uiB96) & 0x7FFFFFFF ) return false; + wordA = aWPtr[indexWord( 4, 2 )]; + wordB = bWPtr[indexWord( 4, 2 )]; + if ( wordA | wordB ) return false; + wordA = aWPtr[indexWord( 4, 1 )]; + wordB = bWPtr[indexWord( 4, 1 )]; + if ( wordA | wordB ) return false; + wordA = aWPtr[indexWord( 4, 0 )]; + wordB = bWPtr[indexWord( 4, 0 )]; + return ((wordA | wordB) == 0); + } + if ( signA ) { + aWPtr = (const uint32_t *) bPtr; + bWPtr = (const uint32_t *) aPtr; + } + return (softfloat_compare128M( aWPtr, bWPtr ) <= 0); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_lt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_lt.c new file mode 100644 index 0000000..4102485 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_lt.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool f128M_lt( const float128_t *aPtr, const float128_t *bPtr ) +{ + + return f128_lt( *aPtr, *bPtr ); + +} + +#else + +bool f128M_lt( const float128_t *aPtr, const float128_t *bPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t uiA96, uiB96; + bool signA, signB; + uint32_t wordA, wordB; + + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + if ( softfloat_isNaNF128M( aWPtr ) || softfloat_isNaNF128M( bWPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + uiA96 = aWPtr[indexWordHi( 4 )]; + uiB96 = bWPtr[indexWordHi( 4 )]; + signA = signF128UI96( uiA96 ); + signB = signF128UI96( uiB96 ); + if ( signA != signB ) { + if ( signB ) return false; + if ( (uiA96 | uiB96) & 0x7FFFFFFF ) return true; + wordA = aWPtr[indexWord( 4, 2 )]; + wordB = bWPtr[indexWord( 4, 2 )]; + if ( wordA | wordB ) return true; + wordA = aWPtr[indexWord( 4, 1 )]; + wordB = bWPtr[indexWord( 4, 1 )]; + if ( wordA | wordB ) return true; + wordA = aWPtr[indexWord( 4, 0 )]; + wordB = bWPtr[indexWord( 4, 0 )]; + return ((wordA | wordB) != 0); + } + if ( signA ) { + aWPtr = (const uint32_t *) bPtr; + bWPtr = (const uint32_t *) aPtr; + } + return (softfloat_compare128M( aWPtr, bWPtr ) < 0); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_lt_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_lt_quiet.c new file mode 100644 index 0000000..cb2b114 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_lt_quiet.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +bool f128M_lt_quiet( const float128_t *aPtr, const float128_t *bPtr ) +{ + + return f128_lt_quiet( *aPtr, *bPtr ); + +} + +#else + +bool f128M_lt_quiet( const float128_t *aPtr, const float128_t *bPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t uiA96, uiB96; + bool signA, signB; + uint32_t wordA, wordB; + + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + if ( softfloat_isNaNF128M( aWPtr ) || softfloat_isNaNF128M( bWPtr ) ) { + if ( f128M_isSignalingNaN( aPtr ) || f128M_isSignalingNaN( bPtr ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + uiA96 = aWPtr[indexWordHi( 4 )]; + uiB96 = bWPtr[indexWordHi( 4 )]; + signA = signF128UI96( uiA96 ); + signB = signF128UI96( uiB96 ); + if ( signA != signB ) { + if ( signB ) return false; + if ( (uiA96 | uiB96) & 0x7FFFFFFF ) return true; + wordA = aWPtr[indexWord( 4, 2 )]; + wordB = bWPtr[indexWord( 4, 2 )]; + if ( wordA | wordB ) return true; + wordA = aWPtr[indexWord( 4, 1 )]; + wordB = bWPtr[indexWord( 4, 1 )]; + if ( wordA | wordB ) return true; + wordA = aWPtr[indexWord( 4, 0 )]; + wordB = bWPtr[indexWord( 4, 0 )]; + return ((wordA | wordB) != 0); + } + if ( signA ) { + aWPtr = (const uint32_t *) bPtr; + bWPtr = (const uint32_t *) aPtr; + } + return (softfloat_compare128M( aWPtr, bWPtr ) < 0); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_mul.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_mul.c new file mode 100644 index 0000000..4d99e42 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_mul.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + f128M_mul( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + + *zPtr = f128_mul( *aPtr, *bPtr ); + +} + +#else + +void + f128M_mul( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t *zWPtr; + uint32_t uiA96; + int32_t expA; + uint32_t uiB96; + int32_t expB; + bool signZ; + const uint32_t *ptr; + uint32_t uiZ96, sigA[4]; + uint_fast8_t shiftCount; + uint32_t sigB[4]; + int32_t expZ; + uint32_t sigProd[8], *extSigZPtr; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + zWPtr = (uint32_t *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA96 = aWPtr[indexWordHi( 4 )]; + expA = expF128UI96( uiA96 ); + uiB96 = bWPtr[indexWordHi( 4 )]; + expB = expF128UI96( uiB96 ); + signZ = signF128UI96( uiA96 ) ^ signF128UI96( uiB96 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNF128M( aWPtr, bWPtr, zWPtr ) ) return; + ptr = aWPtr; + if ( ! expA ) goto possiblyInvalid; + if ( ! expB ) { + ptr = bWPtr; + possiblyInvalid: + if ( + ! fracF128UI96( ptr[indexWordHi( 4 )] ) + && ! (ptr[indexWord( 4, 2 )] | ptr[indexWord( 4, 1 )] + | ptr[indexWord( 4, 0 )]) + ) { + softfloat_invalidF128M( zWPtr ); + return; + } + } + uiZ96 = packToF128UI96( signZ, 0x7FFF, 0 ); + goto uiZ96; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA ) { + sigA[indexWordHi( 4 )] = fracF128UI96( uiA96 ) | 0x00010000; + sigA[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + sigA[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + sigA[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + } else { + expA = softfloat_shiftNormSigF128M( aWPtr, 0, sigA ); + if ( expA == -128 ) goto zero; + } + if ( expB ) { + sigB[indexWordHi( 4 )] = fracF128UI96( uiB96 ) | 0x00010000; + sigB[indexWord( 4, 2 )] = bWPtr[indexWord( 4, 2 )]; + sigB[indexWord( 4, 1 )] = bWPtr[indexWord( 4, 1 )]; + sigB[indexWord( 4, 0 )] = bWPtr[indexWord( 4, 0 )]; + } else { + expB = softfloat_shiftNormSigF128M( bWPtr, 0, sigB ); + if ( expB == -128 ) goto zero; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x4000; + softfloat_mul128MTo256M( sigA, sigB, sigProd ); + if ( + sigProd[indexWord( 8, 2 )] + || (sigProd[indexWord( 8, 1 )] | sigProd[indexWord( 8, 0 )]) + ) { + sigProd[indexWord( 8, 3 )] |= 1; + } + extSigZPtr = &sigProd[indexMultiwordHi( 8, 5 )]; + shiftCount = 16; + if ( extSigZPtr[indexWordHi( 5 )] & 2 ) { + ++expZ; + shiftCount = 15; + } + softfloat_shortShiftLeft160M( extSigZPtr, shiftCount, extSigZPtr ); + softfloat_roundPackMToF128M( signZ, expZ, extSigZPtr, zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ96 = packToF128UI96( signZ, 0, 0 ); + uiZ96: + zWPtr[indexWordHi( 4 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_mulAdd.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_mulAdd.c new file mode 100644 index 0000000..193ed9d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_mulAdd.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + f128M_mulAdd( + const float128_t *aPtr, + const float128_t *bPtr, + const float128_t *cPtr, + float128_t *zPtr + ) +{ + const uint64_t *aWPtr, *bWPtr, *cWPtr; + uint_fast64_t uiA64, uiA0; + uint_fast64_t uiB64, uiB0; + uint_fast64_t uiC64, uiC0; + + aWPtr = (const uint64_t *) aPtr; + bWPtr = (const uint64_t *) bPtr; + cWPtr = (const uint64_t *) cPtr; + uiA64 = aWPtr[indexWord( 2, 1 )]; + uiA0 = aWPtr[indexWord( 2, 0 )]; + uiB64 = bWPtr[indexWord( 2, 1 )]; + uiB0 = bWPtr[indexWord( 2, 0 )]; + uiC64 = cWPtr[indexWord( 2, 1 )]; + uiC0 = cWPtr[indexWord( 2, 0 )]; + *zPtr = softfloat_mulAddF128( uiA64, uiA0, uiB64, uiB0, uiC64, uiC0, 0 ); + +} + +#else + +void + f128M_mulAdd( + const float128_t *aPtr, + const float128_t *bPtr, + const float128_t *cPtr, + float128_t *zPtr + ) +{ + + softfloat_mulAddF128M( + (const uint32_t *) aPtr, + (const uint32_t *) bPtr, + (const uint32_t *) cPtr, + (uint32_t *) zPtr, + 0 + ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_rem.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_rem.c new file mode 100644 index 0000000..52c038d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_rem.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + f128M_rem( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + + *zPtr = f128_rem( *aPtr, *bPtr ); + +} + +#else + +void + f128M_rem( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + const uint32_t *aWPtr, *bWPtr; + uint32_t *zWPtr, uiA96; + int32_t expA, expB; + uint32_t x[4], rem1[5], *remPtr; + bool signRem; + int32_t expDiff; + uint32_t q, recip32; + uint64_t q64; + uint32_t rem2[5], *altRemPtr, *newRemPtr, wordMeanRem; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aWPtr = (const uint32_t *) aPtr; + bWPtr = (const uint32_t *) bPtr; + zWPtr = (uint32_t *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA96 = aWPtr[indexWordHi( 4 )]; + expA = expF128UI96( uiA96 ); + expB = expF128UI96( bWPtr[indexWordHi( 4 )] ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNF128M( aWPtr, bWPtr, zWPtr ) ) return; + if ( expA == 0x7FFF ) goto invalid; + goto copyA; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA < expB - 1 ) goto copyA; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expB = softfloat_shiftNormSigF128M( bWPtr, 13, x ); + if ( expB == -128 ) goto invalid; + remPtr = &rem1[indexMultiwordLo( 5, 4 )]; + expA = softfloat_shiftNormSigF128M( aWPtr, 13, remPtr ); + if ( expA == -128 ) goto copyA; + signRem = signF128UI96( uiA96 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expA - expB; + if ( expDiff < 1 ) { + if ( expDiff < -1 ) goto copyA; + if ( expDiff ) { + --expB; + softfloat_add128M( x, x, x ); + q = 0; + } else { + q = (softfloat_compare128M( x, remPtr ) <= 0); + if ( q ) softfloat_sub128M( remPtr, x, remPtr ); + } + } else { + recip32 = + softfloat_approxRecip32_1( + ((uint64_t) x[indexWord( 4, 3 )]<<32 | x[indexWord( 4, 2 )]) + >>30 + ); + expDiff -= 30; + for (;;) { + q64 = (uint64_t) remPtr[indexWordHi( 4 )] * recip32; + if ( expDiff < 0 ) break; + q = (q64 + 0x80000000)>>32; + softfloat_remStep128MBy32( remPtr, 29, x, q, remPtr ); + if ( remPtr[indexWordHi( 4 )] & 0x80000000 ) { + softfloat_add128M( remPtr, x, remPtr ); + } + expDiff -= 29; + } + /*-------------------------------------------------------------------- + | (`expDiff' cannot be less than -29 here.) + *--------------------------------------------------------------------*/ + q = (uint32_t) (q64>>32)>>(~expDiff & 31); + softfloat_remStep128MBy32( remPtr, expDiff + 30, x, q, remPtr ); + if ( remPtr[indexWordHi( 4 )] & 0x80000000 ) { + altRemPtr = &rem2[indexMultiwordLo( 5, 4 )]; + softfloat_add128M( remPtr, x, altRemPtr ); + goto selectRem; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + altRemPtr = &rem2[indexMultiwordLo( 5, 4 )]; + do { + ++q; + newRemPtr = altRemPtr; + softfloat_sub128M( remPtr, x, newRemPtr ); + altRemPtr = remPtr; + remPtr = newRemPtr; + } while ( ! (remPtr[indexWordHi( 4 )] & 0x80000000) ); + selectRem: + softfloat_add128M( remPtr, altRemPtr, x ); + wordMeanRem = x[indexWordHi( 4 )]; + if ( + (wordMeanRem & 0x80000000) + || (! wordMeanRem && (q & 1) && ! x[indexWord( 4, 0 )] + && ! (x[indexWord( 4, 2 )] | x[indexWord( 4, 1 )])) + ) { + remPtr = altRemPtr; + } + if ( remPtr[indexWordHi( 4 )] & 0x80000000 ) { + signRem = ! signRem; + softfloat_negX128M( remPtr ); + } + remPtr -= indexMultiwordLo( 5, 4 ); + remPtr[indexWordHi( 5 )] = 0; + softfloat_normRoundPackMToF128M( signRem, expB + 18, remPtr, zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_invalidF128M( zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + copyA: + zWPtr[indexWordHi( 4 )] = uiA96; + zWPtr[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + zWPtr[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + zWPtr[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_roundToInt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_roundToInt.c new file mode 100644 index 0000000..170e070 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_roundToInt.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + f128M_roundToInt( + const float128_t *aPtr, + uint_fast8_t roundingMode, + bool exact, + float128_t *zPtr + ) +{ + + *zPtr = f128_roundToInt( *aPtr, roundingMode, exact ); + +} + +#else + +void + f128M_roundToInt( + const float128_t *aPtr, + uint_fast8_t roundingMode, + bool exact, + float128_t *zPtr + ) +{ + const uint32_t *aWPtr; + uint32_t *zWPtr; + uint32_t ui96; + int32_t exp; + uint32_t sigExtra; + bool sign; + uint_fast8_t bitPos; + bool roundNear; + unsigned int index, lastIndex; + bool extra; + uint32_t wordA, bit, wordZ; + uint_fast8_t carry; + uint32_t extrasMask; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aWPtr = (const uint32_t *) aPtr; + zWPtr = (uint32_t *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + ui96 = aWPtr[indexWordHi( 4 )]; + exp = expF128UI96( ui96 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp < 0x3FFF ) { + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + sigExtra = aWPtr[indexWord( 4, 2 )]; + if ( ! sigExtra ) { + sigExtra = aWPtr[indexWord( 4, 1 )] | aWPtr[indexWord( 4, 0 )]; + } + if ( ! sigExtra && ! (ui96 & 0x7FFFFFFF) ) goto ui96; + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + sign = signF128UI96( ui96 ); + switch ( roundingMode ) { + case softfloat_round_near_even: + if ( ! fracF128UI96( ui96 ) && ! sigExtra ) break; + case softfloat_round_near_maxMag: + if ( exp == 0x3FFE ) goto mag1; + break; + case softfloat_round_min: + if ( sign ) goto mag1; + break; + case softfloat_round_max: + if ( ! sign ) goto mag1; + break; + } + ui96 = packToF128UI96( sign, 0, 0 ); + goto ui96; + mag1: + ui96 = packToF128UI96( sign, 0x3FFF, 0 ); + goto ui96; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x406F <= exp ) { + if ( + (exp == 0x7FFF) + && (fracF128UI96( ui96 ) + || (aWPtr[indexWord( 4, 2 )] | aWPtr[indexWord( 4, 1 )] + | aWPtr[indexWord( 4, 0 )])) + ) { + softfloat_propagateNaNF128M( aWPtr, 0, zWPtr ); + return; + } + zWPtr[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + zWPtr[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + zWPtr[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + goto ui96; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + bitPos = 0x406F - exp; + roundNear = + (roundingMode == softfloat_round_near_maxMag) + || (roundingMode == softfloat_round_near_even); + bitPos -= roundNear; + index = indexWordLo( 4 ); + lastIndex = indexWordHi( 4 ); + extra = 0; + for (;;) { + wordA = aWPtr[index]; + if ( bitPos < 32 ) break; + if ( wordA ) extra = 1; + zWPtr[index] = 0; + index += wordIncr; + bitPos -= 32; + } + bit = (uint32_t) 1< +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void f128M_sqrt( const float128_t *aPtr, float128_t *zPtr ) +{ + + *zPtr = f128_sqrt( *aPtr ); + +} + +#else + +void f128M_sqrt( const float128_t *aPtr, float128_t *zPtr ) +{ + const uint32_t *aWPtr; + uint32_t *zWPtr; + uint32_t uiA96; + bool signA; + int32_t rawExpA; + uint32_t rem[6]; + int32_t expA, expZ; + uint64_t rem64; + uint32_t sig32A, recipSqrt32, sig32Z, qs[3], q; + uint64_t sig64Z, x64; + uint32_t term[5], y[5], rem32; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aWPtr = (const uint32_t *) aPtr; + zWPtr = (uint32_t *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA96 = aWPtr[indexWordHi( 4 )]; + signA = signF128UI96( uiA96 ); + rawExpA = expF128UI96( uiA96 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( rawExpA == 0x7FFF ) { + if ( + fracF128UI96( uiA96 ) + || (aWPtr[indexWord( 4, 2 )] | aWPtr[indexWord( 4, 1 )] + | aWPtr[indexWord( 4, 0 )]) + ) { + softfloat_propagateNaNF128M( aWPtr, 0, zWPtr ); + return; + } + if ( ! signA ) goto copyA; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expA = softfloat_shiftNormSigF128M( aWPtr, 13 - (rawExpA & 1), rem ); + if ( expA == -128 ) goto copyA; + if ( signA ) goto invalid; + /*------------------------------------------------------------------------ + | (`sig32Z' is guaranteed to be a lower bound on the square root of + | `sig32A', which makes `sig32Z' also a lower bound on the square root of + | `sigA'.) + *------------------------------------------------------------------------*/ + expZ = ((expA - 0x3FFF)>>1) + 0x3FFE; + expA &= 1; + rem64 = (uint64_t) rem[indexWord( 4, 3 )]<<32 | rem[indexWord( 4, 2 )]; + if ( expA ) { + if ( ! rawExpA ) { + softfloat_shortShiftRight128M( rem, 1, rem ); + rem64 >>= 1; + } + sig32A = rem64>>29; + } else { + sig32A = rem64>>30; + } + recipSqrt32 = softfloat_approxRecipSqrt32_1( expA, sig32A ); + sig32Z = ((uint64_t) sig32A * recipSqrt32)>>32; + if ( expA ) sig32Z >>= 1; + qs[2] = sig32Z; + rem64 -= (uint64_t) sig32Z * sig32Z; + rem[indexWord( 4, 3 )] = rem64>>32; + rem[indexWord( 4, 2 )] = rem64; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = ((uint32_t) (rem64>>2) * (uint64_t) recipSqrt32)>>32; + qs[1] = q; + sig64Z = ((uint64_t) sig32Z<<32) + ((uint64_t) q<<3); + x64 = ((uint64_t) sig32Z<<32) + sig64Z; + term[indexWord( 4, 3 )] = 0; + term[indexWord( 4, 2 )] = x64>>32; + term[indexWord( 4, 1 )] = x64; + term[indexWord( 4, 0 )] = 0; + softfloat_remStep128MBy32( rem, 29, term, q, y ); + rem64 = (uint64_t) y[indexWord( 4, 3 )]<<32 | y[indexWord( 4, 2 )]; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = ((uint32_t) (rem64>>2) * (uint64_t) recipSqrt32)>>32; + sig64Z <<= 1; + /*------------------------------------------------------------------------ + | (Repeating this loop is a rare occurrence.) + *------------------------------------------------------------------------*/ + for (;;) { + x64 = sig64Z + (q>>26); + term[indexWord( 4, 2 )] = x64>>32; + term[indexWord( 4, 1 )] = x64; + term[indexWord( 4, 0 )] = q<<6; + term[indexWord( 4, 3 )] = 0; + softfloat_remStep128MBy32( + y, 29, term, q, &rem[indexMultiwordHi( 6, 4 )] ); + rem32 = rem[indexWordHi( 6 )]; + if ( ! (rem32 & 0x80000000) ) break; + --q; + } + qs[0] = q; + rem64 = (uint64_t) rem32<<32 | rem[indexWord( 6, 4 )]; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = (((uint32_t) (rem64>>2) * (uint64_t) recipSqrt32)>>32) + 2; + x64 = (uint64_t) q<<27; + y[indexWord( 5, 0 )] = x64; + x64 = ((uint64_t) qs[0]<<24) + (x64>>32); + y[indexWord( 5, 1 )] = x64; + x64 = ((uint64_t) qs[1]<<21) + (x64>>32); + y[indexWord( 5, 2 )] = x64; + x64 = ((uint64_t) qs[2]<<18) + (x64>>32); + y[indexWord( 5, 3 )] = x64; + y[indexWord( 5, 4 )] = x64>>32; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (q & 0xF) <= 2 ) { + q &= ~3; + y[indexWordLo( 5 )] = q<<27; + term[indexWord( 5, 4 )] = 0; + term[indexWord( 5, 3 )] = 0; + term[indexWord( 5, 2 )] = 0; + term[indexWord( 5, 1 )] = q>>6; + term[indexWord( 5, 0 )] = q<<26; + softfloat_sub160M( y, term, term ); + rem[indexWord( 6, 1 )] = 0; + rem[indexWord( 6, 0 )] = 0; + softfloat_remStep160MBy32( + &rem[indexMultiwordLo( 6, 5 )], + 14, + term, + q, + &rem[indexMultiwordLo( 6, 5 )] + ); + rem32 = rem[indexWord( 6, 4 )]; + if ( rem32 & 0x80000000 ) { + softfloat_sub1X160M( y ); + } else { + if ( + rem32 || rem[indexWord( 6, 0 )] || rem[indexWord( 6, 1 )] + || (rem[indexWord( 6, 3 )] | rem[indexWord( 6, 2 )]) + ) { + y[indexWordLo( 5 )] |= 1; + } + } + } + softfloat_roundPackMToF128M( 0, expZ, y, zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_invalidF128M( zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + copyA: + zWPtr[indexWordHi( 4 )] = uiA96; + zWPtr[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + zWPtr[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + zWPtr[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_sub.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_sub.c new file mode 100644 index 0000000..c55f455 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_sub.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void + f128M_sub( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + const uint64_t *aWPtr, *bWPtr; + uint_fast64_t uiA64, uiA0; + bool signA; + uint_fast64_t uiB64, uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + float128_t + (*magsFuncPtr)( + uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool ); +#endif + + aWPtr = (const uint64_t *) aPtr; + bWPtr = (const uint64_t *) bPtr; + uiA64 = aWPtr[indexWord( 2, 1 )]; + uiA0 = aWPtr[indexWord( 2, 0 )]; + signA = signF128UI64( uiA64 ); + uiB64 = bWPtr[indexWord( 2, 1 )]; + uiB0 = bWPtr[indexWord( 2, 0 )]; + signB = signF128UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + *zPtr = softfloat_subMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + *zPtr = softfloat_addMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_subMagsF128 : softfloat_addMagsF128; + *zPtr = (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + +#else + +void + f128M_sub( const float128_t *aPtr, const float128_t *bPtr, float128_t *zPtr ) +{ + + softfloat_addF128M( + (const uint32_t *) aPtr, + (const uint32_t *) bPtr, + (uint32_t *) zPtr, + true + ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_extF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_extF80M.c new file mode 100644 index 0000000..ff2ae39 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_extF80M.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void f128M_to_extF80M( const float128_t *aPtr, extFloat80_t *zPtr ) +{ + + *zPtr = f128_to_extF80( *aPtr ); + +} + +#else + +void f128M_to_extF80M( const float128_t *aPtr, extFloat80_t *zPtr ) +{ + const uint32_t *aWPtr; + struct extFloat80M *zSPtr; + uint32_t uiA96; + bool sign; + int32_t exp; + struct commonNaN commonNaN; + uint32_t sig[4]; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + aWPtr = (const uint32_t *) aPtr; + zSPtr = (struct extFloat80M *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA96 = aWPtr[indexWordHi( 4 )]; + sign = signF128UI96( uiA96 ); + exp = expF128UI96( uiA96 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp == 0x7FFF ) { + if ( softfloat_isNaNF128M( aWPtr ) ) { + softfloat_f128MToCommonNaN( aWPtr, &commonNaN ); + softfloat_commonNaNToExtF80M( &commonNaN, zSPtr ); + return; + } + zSPtr->signExp = packToExtF80UI64( sign, 0x7FFF ); + zSPtr->signif = UINT64_C( 0x8000000000000000 ); + return; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + exp = softfloat_shiftNormSigF128M( aWPtr, 15, sig ); + if ( exp == -128 ) { + zSPtr->signExp = packToExtF80UI64( sign, 0 ); + zSPtr->signif = 0; + return; + } + if ( sig[indexWord( 4, 0 )] ) sig[indexWord( 4, 1 )] |= 1; + softfloat_roundPackMToExtF80M( + sign, exp, &sig[indexMultiwordHi( 4, 3 )], 80, zSPtr ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_f32.c new file mode 100644 index 0000000..a2c0e27 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_f32.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +float32_t f128M_to_f32( const float128_t *aPtr ) +{ + + return f128_to_f32( *aPtr ); + +} + +#else + +float32_t f128M_to_f32( const float128_t *aPtr ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + bool sign; + int32_t exp; + uint64_t sig64; + struct commonNaN commonNaN; + uint32_t uiZ, sig32; + union ui32_f32 uZ; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + sign = signF128UI96( uiA96 ); + exp = expF128UI96( uiA96 ); + sig64 = + (uint64_t) fracF128UI96( uiA96 )<<32 | aWPtr[indexWord( 4, 2 )] + | ((aWPtr[indexWord( 4, 1 )] | aWPtr[indexWord( 4, 0 )]) != 0); + if ( exp == 0x7FFF ) { + if ( sig64 ) { + softfloat_f128MToCommonNaN( aWPtr, &commonNaN ); + uiZ = softfloat_commonNaNToF32UI( &commonNaN ); + } else { + uiZ = packToF32UI( sign, 0xFF, 0 ); + } + goto uiZ; + } + sig32 = softfloat_shortShiftRightJam64( sig64, 18 ); + if ( ! (exp | sig32) ) { + uiZ = packToF32UI( sign, 0, 0 ); + goto uiZ; + } + exp -= 0x3F81; + if ( sizeof (int_fast16_t) < sizeof (int32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return softfloat_roundPackToF32( sign, exp, sig32 | 0x40000000 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_f64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_f64.c new file mode 100644 index 0000000..f5c6048 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_f64.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +float64_t f128M_to_f64( const float128_t *aPtr ) +{ + + return f128_to_f64( *aPtr ); + +} + +#else + +float64_t f128M_to_f64( const float128_t *aPtr ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + bool sign; + int32_t exp; + uint64_t sig64; + struct commonNaN commonNaN; + uint64_t uiZ; + uint32_t sig32; + union ui64_f64 uZ; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + sign = signF128UI96( uiA96 ); + exp = expF128UI96( uiA96 ); + sig64 = (uint64_t) fracF128UI96( uiA96 )<<32 | aWPtr[indexWord( 4, 2 )]; + if ( exp == 0x7FFF ) { + if ( sig64 || aWPtr[indexWord( 4, 1 )] | aWPtr[indexWord( 4, 0 )] ) { + softfloat_f128MToCommonNaN( aWPtr, &commonNaN ); + uiZ = softfloat_commonNaNToF64UI( &commonNaN ); + } else { + uiZ = packToF64UI( sign, 0x7FF, 0 ); + } + goto uiZ; + } + sig32 = aWPtr[indexWord( 4, 1 )]; + sig64 = sig64<<14 | sig32>>18; + if ( (sig32 & 0x0003FFFF) || aWPtr[indexWord( 4, 0 )] ) sig64 |= 1; + if ( ! exp && ! sig64 ) { + uiZ = packToF64UI( sign, 0, 0 ); + goto uiZ; + } + exp -= 0x3C01; + if ( sizeof (int_fast16_t) < sizeof (int32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return + softfloat_roundPackToF64( + sign, exp, sig64 | UINT64_C( 0x4000000000000000 ) ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i32.c new file mode 100644 index 0000000..bfea0d7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i32.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast32_t + f128M_to_i32( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return f128_to_i32( *aPtr, roundingMode, exact ); + +} + +#else + +int_fast32_t + f128M_to_i32( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + bool sign; + int32_t exp; + uint64_t sig64; + bool notZeroSig0; + int32_t shiftCount; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + sign = signF128UI96( uiA96 ); + exp = expF128UI96( uiA96 ); + sig64 = (uint64_t) fracF128UI96( uiA96 )<<32 | aWPtr[indexWord( 4, 2 )]; + notZeroSig0 = false; + if ( aWPtr[indexWord( 4, 1 )] | aWPtr[indexWord( 4, 0 )] ) { + sig64 |= 1; + notZeroSig0 = true; + } + if ( (exp == 0x7FFF) && (notZeroSig0 || sig64) ) sign = 0; + if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 ); + shiftCount = 0x4028 - exp; + if ( 0 < shiftCount ) { + sig64 = softfloat_shiftRightJam64( sig64, shiftCount ); + } + return softfloat_roundPackToI32( sign, sig64, roundingMode, exact ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i32_r_minMag.c new file mode 100644 index 0000000..93fe6f2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i32_r_minMag.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast32_t f128M_to_i32_r_minMag( const float128_t *aPtr, bool exact ) +{ + + return f128_to_i32_r_minMag( *aPtr, exact ); + +} + +#else + +int_fast32_t f128M_to_i32_r_minMag( const float128_t *aPtr, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + bool sign; + int32_t exp; + uint64_t sig64; + int32_t shiftCount; + uint32_t absZ, uiZ; + union { uint32_t ui; int32_t i; } uZ; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + sign = signF128UI96( uiA96 ); + exp = expF128UI96( uiA96 ); + sig64 = (uint64_t) fracF128UI96( uiA96 )<<32 | aWPtr[indexWord( 4, 2 )]; + if ( aWPtr[indexWord( 4, 1 )] | aWPtr[indexWord( 4, 0 )] ) sig64 |= 1; + if ( exp < 0x3FFF ) { + if ( exact && (exp | sig64) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( 0x401F <= exp ) goto invalid; + shiftCount = 0x402F - exp; + sig64 |= UINT64_C( 0x0001000000000000 ); + absZ = sig64>>shiftCount; + uiZ = sign ? -absZ : absZ; + if ( uiZ>>31 != sign ) goto invalid; + if ( exact && ((uint64_t) absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast64_t + f128M_to_i64( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return f128_to_i64( *aPtr, roundingMode, exact ); + +} + +#else + +int_fast64_t + f128M_to_i64( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + int32_t exp; + bool sign; + uint32_t sig96; + int32_t shiftCount; + uint32_t sig[4]; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + exp = expF128UI96( uiA96 ); + sign = signF128UI96( uiA96 ); + sig96 = fracF128UI96( uiA96 ); + shiftCount = 0x404F - exp; + if ( shiftCount < 17 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return + ! sign + || ((exp == 0x7FFF) + && (sig96 + || ( aWPtr[indexWord( 4, 2 )] + | aWPtr[indexWord( 4, 1 )] + | aWPtr[indexWord( 4, 0 )] + ))) + ? INT64_C( 0x7FFFFFFFFFFFFFFF ) + : -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + if ( exp ) sig96 |= 0x00010000; + sig[indexWord( 4, 3 )] = sig96; + sig[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + sig[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + sig[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + softfloat_shiftRightJam128M( sig, shiftCount, sig ); + return + softfloat_roundPackMToI64( + sign, sig + indexMultiwordLo( 4, 3 ), roundingMode, exact ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i64_r_minMag.c new file mode 100644 index 0000000..5b18bc3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_i64_r_minMag.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +int_fast64_t f128M_to_i64_r_minMag( const float128_t *aPtr, bool exact ) +{ + + return f128_to_i64_r_minMag( *aPtr, exact ); + +} + +#else + +int_fast64_t f128M_to_i64_r_minMag( const float128_t *aPtr, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + bool sign; + int32_t exp, shiftCount; + uint32_t sig96, sig[4]; + uint64_t uiZ; + union { uint64_t ui; int64_t i; } uZ; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + sign = signF128UI96( uiA96 ); + exp = expF128UI96( uiA96 ); + shiftCount = 0x403E - exp; + if ( shiftCount < 0 ) goto invalid; + if ( exact ) { + sig96 = fracF128UI96( uiA96 ); + if ( exp ) sig96 |= 0x00010000; + sig[indexWord( 4, 3 )] = sig96; + sig[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + sig[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + sig[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + softfloat_shiftRightJam128M( sig, shiftCount + 17, sig ); + uiZ = (uint64_t) sig[indexWord( 4, 2 )]<<32 | sig[indexWord( 4, 1 )]; + if ( uiZ>>63 && (! sign || (uiZ != UINT64_C( 0x8000000000000000 ))) ) { + goto invalid; + } + if ( sig[indexWordLo( 4 )] ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + } else { + if ( 64 <= shiftCount ) return 0; + uiZ = + (uint64_t) fracF128UI96( uiA96 )<<47 + | (uint64_t) aWPtr[indexWord( 4, 2 )]<<15 + | aWPtr[indexWord( 4, 1 )]>>17; + if ( shiftCount ) { + uiZ |= UINT64_C( 0x8000000000000000 ); + uiZ >>= shiftCount; + } else { + if ( uiZ || ! sign ) goto invalid; + uiZ |= UINT64_C( 0x8000000000000000 ); + } + } + if ( sign ) uiZ = -uiZ; + uZ.ui = uiZ; + return uZ.i; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return + sign && ! softfloat_isNaNF128M( aWPtr ) + ? -UINT64_C( 0x7FFFFFFFFFFFFFFF ) - 1 + : UINT64_C( 0x7FFFFFFFFFFFFFFF ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui32.c new file mode 100644 index 0000000..493b028 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui32.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast32_t + f128M_to_ui32( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return f128_to_ui32( *aPtr, roundingMode, exact ); + +} + +#else + +uint_fast32_t + f128M_to_ui32( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + int32_t exp; + uint64_t sig64; + int32_t shiftCount; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + exp = expF128UI96( uiA96 ); + sig64 = (uint64_t) fracF128UI96( uiA96 )<<32 | aWPtr[indexWord( 4, 2 )]; + if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 ); + if ( aWPtr[indexWord( 4, 1 )] | aWPtr[indexWord( 4, 0 )] ) sig64 |= 1; + shiftCount = 0x4028 - exp; + if ( 0 < shiftCount ) { + sig64 = softfloat_shiftRightJam64( sig64, shiftCount ); + } + return + softfloat_roundPackToUI32( + signF128UI96( uiA96 ), sig64, roundingMode, exact ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui32_r_minMag.c new file mode 100644 index 0000000..fde3930 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui32_r_minMag.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast32_t f128M_to_ui32_r_minMag( const float128_t *aPtr, bool exact ) +{ + + return f128_to_ui32_r_minMag( *aPtr, exact ); + +} + +#else + +uint_fast32_t f128M_to_ui32_r_minMag( const float128_t *aPtr, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + int32_t exp; + uint64_t sig64; + int32_t shiftCount; + uint32_t z; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + exp = expF128UI96( uiA96 ); + sig64 = (uint64_t) fracF128UI96( uiA96 )<<32 | aWPtr[indexWord( 4, 2 )]; + if ( aWPtr[indexWord( 4, 1 )] | aWPtr[indexWord( 4, 0 )] ) sig64 |= 1; + shiftCount = 0x402F - exp; + if ( 49 <= shiftCount ) { + if ( exact && (exp | sig64) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signF128UI96( uiA96 ) || (shiftCount < 17) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return 0xFFFFFFFF; + } + sig64 |= UINT64_C( 0x0001000000000000 ); + z = sig64>>shiftCount; + if ( exact && ((uint64_t) z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast64_t + f128M_to_ui64( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + + return f128_to_ui64( *aPtr, roundingMode, exact ); + +} + +#else + +uint_fast64_t + f128M_to_ui64( const float128_t *aPtr, uint_fast8_t roundingMode, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + int32_t exp, shiftCount; + uint32_t sig96, sig[4]; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + exp = expF128UI96( uiA96 ); + shiftCount = 0x404F - exp; + if ( shiftCount < 17 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + sig96 = fracF128UI96( uiA96 ); + if ( exp ) sig96 |= 0x00010000; + sig[indexWord( 4, 3 )] = sig96; + sig[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + sig[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + sig[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + softfloat_shiftRightJam128M( sig, shiftCount, sig ); + return + softfloat_roundPackMToUI64( + signF128UI96( uiA96 ), + sig + indexMultiwordLo( 4, 3 ), + roundingMode, + exact + ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui64_r_minMag.c new file mode 100644 index 0000000..8807431 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128M_to_ui64_r_minMag.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast64_t f128M_to_ui64_r_minMag( const float128_t *aPtr, bool exact ) +{ + + return f128_to_ui64_r_minMag( *aPtr, exact ); + +} + +#else + +uint_fast64_t f128M_to_ui64_r_minMag( const float128_t *aPtr, bool exact ) +{ + const uint32_t *aWPtr; + uint32_t uiA96; + int32_t exp, shiftCount; + uint32_t sig96, sig[4]; + uint64_t z; + + aWPtr = (const uint32_t *) aPtr; + uiA96 = aWPtr[indexWordHi( 4 )]; + exp = expF128UI96( uiA96 ); + shiftCount = 0x403E - exp; + if ( shiftCount < 0 ) goto invalid; + if ( exact ) { + sig96 = fracF128UI96( uiA96 ); + if ( exp ) sig96 |= 0x00010000; + sig[indexWord( 4, 3 )] = sig96; + sig[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + sig[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + sig[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + softfloat_shiftRightJam128M( sig, shiftCount + 17, sig ); + z = (uint64_t) sig[indexWord( 4, 2 )]<<32 | sig[indexWord( 4, 1 )]; + if ( signF128UI96( uiA96 ) && z ) goto invalid; + if ( sig[indexWordLo( 4 )] ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + } else { + if ( 64 <= shiftCount ) return 0; + if ( signF128UI96( uiA96 ) ) goto invalid; + z = UINT64_C( 0x8000000000000000 ) + | (uint64_t) fracF128UI96( uiA96 )<<47 + | (uint64_t) aWPtr[indexWord( 4, 2 )]<<15 + | aWPtr[indexWord( 4, 1 )]>>17; + z >>= shiftCount; + } + return z; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_add.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_add.c new file mode 100644 index 0000000..7151116 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_add.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t f128_add( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool signA; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + float128_t + (*magsFuncPtr)( + uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool ); +#endif + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + signA = signF128UI64( uiA64 ); + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + signB = signF128UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_addMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + return softfloat_subMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_addMagsF128 : softfloat_subMagsF128; + return (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_div.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_div.c new file mode 100644 index 0000000..f8581b5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_div.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t f128_div( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool signA; + int_fast32_t expA; + struct uint128 sigA; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signB; + int_fast32_t expB; + struct uint128 sigB; + bool signZ; + struct exp32_sig128 normExpSig; + int_fast32_t expZ; + struct uint128 rem; + uint_fast32_t recip32; + int ix; + uint_fast64_t q64; + uint_fast32_t q; + struct uint128 term; + uint_fast32_t qs[3]; + uint_fast64_t sigZExtra; + struct uint128 sigZ, uiZ; + union ui128_f128 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + signA = signF128UI64( uiA64 ); + expA = expF128UI64( uiA64 ); + sigA.v64 = fracF128UI64( uiA64 ); + sigA.v0 = uiA0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + signB = signF128UI64( uiB64 ); + expB = expF128UI64( uiB64 ); + sigB.v64 = fracF128UI64( uiB64 ); + sigB.v0 = uiB0; + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( sigA.v64 | sigA.v0 ) goto propagateNaN; + if ( expB == 0x7FFF ) { + if ( sigB.v64 | sigB.v0 ) goto propagateNaN; + goto invalid; + } + goto infinity; + } + if ( expB == 0x7FFF ) { + if ( sigB.v64 | sigB.v0 ) goto propagateNaN; + goto zero; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) { + if ( ! (sigB.v64 | sigB.v0) ) { + if ( ! (expA | sigA.v64 | sigA.v0) ) goto invalid; + softfloat_raiseFlags( softfloat_flag_infinite ); + goto infinity; + } + normExpSig = softfloat_normSubnormalF128Sig( sigB.v64, sigB.v0 ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) { + if ( ! (sigA.v64 | sigA.v0) ) goto zero; + normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA - expB + 0x3FFE; + sigA.v64 |= UINT64_C( 0x0001000000000000 ); + sigB.v64 |= UINT64_C( 0x0001000000000000 ); + rem = sigA; + if ( softfloat_lt128( sigA.v64, sigA.v0, sigB.v64, sigB.v0 ) ) { + --expZ; + rem = softfloat_add128( sigA.v64, sigA.v0, sigA.v64, sigA.v0 ); + } + recip32 = softfloat_approxRecip32_1( sigB.v64>>17 ); + ix = 3; + for (;;) { + q64 = (uint_fast64_t) (uint32_t) (rem.v64>>19) * recip32; + q = (q64 + 0x80000000)>>32; + --ix; + if ( ix < 0 ) break; + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + term = softfloat_mul128By32( sigB.v64, sigB.v0, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + --q; + rem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 ); + } + qs[ix] = q; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ((q + 1) & 7) < 2 ) { + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + term = softfloat_mul128By32( sigB.v64, sigB.v0, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + --q; + rem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 ); + } else if ( softfloat_le128( sigB.v64, sigB.v0, rem.v64, rem.v0 ) ) { + ++q; + rem = softfloat_sub128( rem.v64, rem.v0, sigB.v64, sigB.v0 ); + } + if ( rem.v64 | rem.v0 ) q |= 1; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + sigZExtra = (uint64_t) ((uint_fast64_t) q<<60); + term = softfloat_shortShiftLeft128( 0, qs[1], 54 ); + sigZ = + softfloat_add128( + (uint_fast64_t) qs[2]<<19, ((uint_fast64_t) qs[0]<<25) + (q>>4), + term.v64, term.v0 + ); + return + softfloat_roundPackToF128( signZ, expZ, sigZ.v64, sigZ.v0, sigZExtra ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, uiB64, uiB0 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ.v64 = defaultNaNF128UI64; + uiZ.v0 = defaultNaNF128UI0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infinity: + uiZ.v64 = packToF128UI64( signZ, 0x7FFF, 0 ); + goto uiZ0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ.v64 = packToF128UI64( signZ, 0, 0 ); + uiZ0: + uiZ.v0 = 0; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_eq.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_eq.c new file mode 100644 index 0000000..069f2a0 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_eq.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f128_eq( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNF128UI( uiA64, uiA0 ) + || softfloat_isSigNaNF128UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + return + (uiA0 == uiB0) + && ( (uiA64 == uiB64) + || (! uiA0 && ! ((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_eq_signaling.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_eq_signaling.c new file mode 100644 index 0000000..7c18ebd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_eq_signaling.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f128_eq_signaling( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + return + (uiA0 == uiB0) + && ( (uiA64 == uiB64) + || (! uiA0 && ! ((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_isSignalingNaN.c new file mode 100644 index 0000000..0faa58a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_isSignalingNaN.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f128_isSignalingNaN( float128_t a ) +{ + union ui128_f128 uA; + + uA.f = a; + return softfloat_isSigNaNF128UI( uA.ui.v64, uA.ui.v0 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_le.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_le.c new file mode 100644 index 0000000..115501f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_le.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f128_le( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signF128UI64( uiA64 ); + signB = signF128UI64( uiB64 ); + return + (signA != signB) + ? signA + || ! (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + | uiA0 | uiB0) + : ((uiA64 == uiB64) && (uiA0 == uiB0)) + || (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_le_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_le_quiet.c new file mode 100644 index 0000000..e026501 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_le_quiet.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f128_le_quiet( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNF128UI( uiA64, uiA0 ) + || softfloat_isSigNaNF128UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signF128UI64( uiA64 ); + signB = signF128UI64( uiB64 ); + return + (signA != signB) + ? signA + || ! (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + | uiA0 | uiB0) + : ((uiA64 == uiB64) && (uiA0 == uiB0)) + || (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_lt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_lt.c new file mode 100644 index 0000000..b35c762 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_lt.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f128_lt( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signF128UI64( uiA64 ); + signB = signF128UI64( uiB64 ); + return + (signA != signB) + ? signA + && (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + | uiA0 | uiB0) + : ((uiA64 != uiB64) || (uiA0 != uiB0)) + && (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_lt_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_lt_quiet.c new file mode 100644 index 0000000..90d1388 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_lt_quiet.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f128_lt_quiet( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signA, signB; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) { + if ( + softfloat_isSigNaNF128UI( uiA64, uiA0 ) + || softfloat_isSigNaNF128UI( uiB64, uiB0 ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signF128UI64( uiA64 ); + signB = signF128UI64( uiB64 ); + return + (signA != signB) + ? signA + && (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + | uiA0 | uiB0) + : ((uiA64 != uiB64) || (uiA0 != uiB0)) + && (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_mul.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_mul.c new file mode 100644 index 0000000..61c0e73 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_mul.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t f128_mul( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool signA; + int_fast32_t expA; + struct uint128 sigA; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signB; + int_fast32_t expB; + struct uint128 sigB; + bool signZ; + uint_fast64_t magBits; + struct exp32_sig128 normExpSig; + int_fast32_t expZ; + uint64_t sig256Z[4]; + uint_fast64_t sigZExtra; + struct uint128 sigZ; + struct uint128_extra sig128Extra; + struct uint128 uiZ; + union ui128_f128 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + signA = signF128UI64( uiA64 ); + expA = expF128UI64( uiA64 ); + sigA.v64 = fracF128UI64( uiA64 ); + sigA.v0 = uiA0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + signB = signF128UI64( uiB64 ); + expB = expF128UI64( uiB64 ); + sigB.v64 = fracF128UI64( uiB64 ); + sigB.v0 = uiB0; + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( + (sigA.v64 | sigA.v0) || ((expB == 0x7FFF) && (sigB.v64 | sigB.v0)) + ) { + goto propagateNaN; + } + magBits = expB | sigB.v64 | sigB.v0; + goto infArg; + } + if ( expB == 0x7FFF ) { + if ( sigB.v64 | sigB.v0 ) goto propagateNaN; + magBits = expA | sigA.v64 | sigA.v0; + goto infArg; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! (sigA.v64 | sigA.v0) ) goto zero; + normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) { + if ( ! (sigB.v64 | sigB.v0) ) goto zero; + normExpSig = softfloat_normSubnormalF128Sig( sigB.v64, sigB.v0 ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x4000; + sigA.v64 |= UINT64_C( 0x0001000000000000 ); + sigB = softfloat_shortShiftLeft128( sigB.v64, sigB.v0, 16 ); + softfloat_mul128To256M( sigA.v64, sigA.v0, sigB.v64, sigB.v0, sig256Z ); + sigZExtra = sig256Z[indexWord( 4, 1 )] | (sig256Z[indexWord( 4, 0 )] != 0); + sigZ = + softfloat_add128( + sig256Z[indexWord( 4, 3 )], sig256Z[indexWord( 4, 2 )], + sigA.v64, sigA.v0 + ); + if ( UINT64_C( 0x0002000000000000 ) <= sigZ.v64 ) { + ++expZ; + sig128Extra = + softfloat_shortShiftRightJam128Extra( + sigZ.v64, sigZ.v0, sigZExtra, 1 ); + sigZ = sig128Extra.v; + sigZExtra = sig128Extra.extra; + } + return + softfloat_roundPackToF128( signZ, expZ, sigZ.v64, sigZ.v0, sigZExtra ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, uiB64, uiB0 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infArg: + if ( ! magBits ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ.v64 = defaultNaNF128UI64; + uiZ.v0 = defaultNaNF128UI0; + goto uiZ; + } + uiZ.v64 = packToF128UI64( signZ, 0x7FFF, 0 ); + goto uiZ0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ.v64 = packToF128UI64( signZ, 0, 0 ); + uiZ0: + uiZ.v0 = 0; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_mulAdd.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_mulAdd.c new file mode 100644 index 0000000..c34cae2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_mulAdd.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t f128_mulAdd( float128_t a, float128_t b, float128_t c ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + union ui128_f128 uC; + uint_fast64_t uiC64, uiC0; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + uC.f = c; + uiC64 = uC.ui.v64; + uiC0 = uC.ui.v0; + return softfloat_mulAddF128( uiA64, uiA0, uiB64, uiB0, uiC64, uiC0, 0 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_rem.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_rem.c new file mode 100644 index 0000000..08b9f44 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_rem.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t f128_rem( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool signA; + int_fast32_t expA; + struct uint128 sigA; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signB; + int_fast32_t expB; + struct uint128 sigB; + struct exp32_sig128 normExpSig; + struct uint128 rem; + int_fast32_t expDiff; + uint_fast32_t q, recip32; + uint_fast64_t q64; + struct uint128 term, altRem, meanRem; + bool signRem; + struct uint128 uiZ; + union ui128_f128 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + signA = signF128UI64( uiA64 ); + expA = expF128UI64( uiA64 ); + sigA.v64 = fracF128UI64( uiA64 ); + sigA.v0 = uiA0; + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + signB = signF128UI64( uiB64 ); + expB = expF128UI64( uiB64 ); + sigB.v64 = fracF128UI64( uiB64 ); + sigB.v0 = uiB0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( + (sigA.v64 | sigA.v0) || ((expB == 0x7FFF) && (sigB.v64 | sigB.v0)) + ) { + goto propagateNaN; + } + goto invalid; + } + if ( expB == 0x7FFF ) { + if ( sigB.v64 | sigB.v0 ) goto propagateNaN; + return a; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) { + if ( ! (sigB.v64 | sigB.v0) ) goto invalid; + normExpSig = softfloat_normSubnormalF128Sig( sigB.v64, sigB.v0 ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) { + if ( ! (sigA.v64 | sigA.v0) ) return a; + normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + sigA.v64 |= UINT64_C( 0x0001000000000000 ); + sigB.v64 |= UINT64_C( 0x0001000000000000 ); + rem = sigA; + expDiff = expA - expB; + if ( expDiff < 1 ) { + if ( expDiff < -1 ) return a; + if ( expDiff ) { + --expB; + sigB = softfloat_add128( sigB.v64, sigB.v0, sigB.v64, sigB.v0 ); + q = 0; + } else { + q = softfloat_le128( sigB.v64, sigB.v0, rem.v64, rem.v0 ); + if ( q ) { + rem = softfloat_sub128( rem.v64, rem.v0, sigB.v64, sigB.v0 ); + } + } + } else { + recip32 = softfloat_approxRecip32_1( sigB.v64>>17 ); + expDiff -= 30; + for (;;) { + q64 = (uint_fast64_t) (uint32_t) (rem.v64>>19) * recip32; + if ( expDiff < 0 ) break; + q = (q64 + 0x80000000)>>32; + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + term = softfloat_mul128By32( sigB.v64, sigB.v0, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + rem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 ); + } + expDiff -= 29; + } + /*-------------------------------------------------------------------- + | (`expDiff' cannot be less than -29 here.) + *--------------------------------------------------------------------*/ + q = (uint32_t) (q64>>32)>>(~expDiff & 31); + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, expDiff + 30 ); + term = softfloat_mul128By32( sigB.v64, sigB.v0, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + altRem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 ); + goto selectRem; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + do { + altRem = rem; + ++q; + rem = softfloat_sub128( rem.v64, rem.v0, sigB.v64, sigB.v0 ); + } while ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) ); + selectRem: + meanRem = softfloat_add128( rem.v64, rem.v0, altRem.v64, altRem.v0 ); + if ( + (meanRem.v64 & UINT64_C( 0x8000000000000000 )) + || (! (meanRem.v64 | meanRem.v0) && (q & 1)) + ) { + rem = altRem; + } + signRem = signA; + if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) { + signRem = ! signRem; + rem = softfloat_sub128( 0, 0, rem.v64, rem.v0 ); + } + return softfloat_normRoundPackToF128( signRem, expB - 1, rem.v64, rem.v0 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, uiB64, uiB0 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ.v64 = defaultNaNF128UI64; + uiZ.v0 = defaultNaNF128UI0; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_roundToInt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_roundToInt.c new file mode 100644 index 0000000..392bddc --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_roundToInt.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t + f128_roundToInt( float128_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + int_fast32_t exp; + struct uint128 uiZ; + uint_fast64_t lastBitMask, roundBitsMask; + bool roundNearEven; + union ui128_f128 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + exp = expF128UI64( uiA64 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x402F <= exp ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( 0x406F <= exp ) { + if ( (exp == 0x7FFF) && (fracF128UI64( uiA64 ) | uiA0) ) { + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, 0, 0 ); + goto uiZ; + } + return a; + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + lastBitMask = (uint_fast64_t) 2<<(0x406E - exp); + roundBitsMask = lastBitMask - 1; + uiZ.v64 = uiA64; + uiZ.v0 = uiA0; + roundNearEven = (roundingMode == softfloat_round_near_even); + if ( roundNearEven || (roundingMode == softfloat_round_near_maxMag) ) { + if ( exp == 0x402F ) { + if ( UINT64_C( 0x8000000000000000 ) <= uiZ.v0 ) { + ++uiZ.v64; + if ( + roundNearEven + && (uiZ.v0 == UINT64_C( 0x8000000000000000 )) + ) { + uiZ.v64 &= ~1; + } + } + } else { + uiZ = softfloat_add128( uiZ.v64, uiZ.v0, 0, lastBitMask>>1 ); + if ( roundNearEven && ! (uiZ.v0 & roundBitsMask) ) { + uiZ.v0 &= ~lastBitMask; + } + } + } else if ( roundingMode != softfloat_round_minMag ) { + if ( + signF128UI64( uiZ.v64 ) ^ (roundingMode == softfloat_round_max) + ) { + uiZ = softfloat_add128( uiZ.v64, uiZ.v0, 0, roundBitsMask ); + } + } + uiZ.v0 &= ~roundBitsMask; + } else { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( exp < 0x3FFF ) { + if ( ! ((uiA64 & UINT64_C( 0x7FFFFFFFFFFFFFFF )) | uiA0) ) { + return a; + } + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + uiZ.v64 = uiA64 & packToF128UI64( 1, 0, 0 ); + uiZ.v0 = 0; + switch ( roundingMode ) { + case softfloat_round_near_even: + if ( ! (fracF128UI64( uiA64 ) | uiA0) ) break; + case softfloat_round_near_maxMag: + if ( exp == 0x3FFE ) uiZ.v64 |= packToF128UI64( 0, 0x3FFF, 0 ); + break; + case softfloat_round_min: + if ( uiZ.v64 ) uiZ.v64 = packToF128UI64( 1, 0x3FFF, 0 ); + break; + case softfloat_round_max: + if ( ! uiZ.v64 ) uiZ.v64 = packToF128UI64( 0, 0x3FFF, 0 ); + break; + } + goto uiZ; + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + uiZ.v64 = uiA64; + uiZ.v0 = 0; + lastBitMask = (uint_fast64_t) 1<<(0x402F - exp); + roundBitsMask = lastBitMask - 1; + if ( roundingMode == softfloat_round_near_maxMag ) { + uiZ.v64 += lastBitMask>>1; + } else if ( roundingMode == softfloat_round_near_even ) { + uiZ.v64 += lastBitMask>>1; + if ( ! ((uiZ.v64 & roundBitsMask) | uiA0) ) { + uiZ.v64 &= ~lastBitMask; + } + } else if ( roundingMode != softfloat_round_minMag ) { + if ( + signF128UI64( uiZ.v64 ) ^ (roundingMode == softfloat_round_max) + ) { + uiZ.v64 = (uiZ.v64 | (uiA0 != 0)) + roundBitsMask; + } + } + uiZ.v64 &= ~roundBitsMask; + } + if ( exact && ((uiZ.v64 != uiA64) || (uiZ.v0 != uiA0)) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_sqrt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_sqrt.c new file mode 100644 index 0000000..2eb072e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_sqrt.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t f128_sqrt( float128_t a ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool signA; + int_fast32_t expA; + struct uint128 sigA, uiZ; + struct exp32_sig128 normExpSig; + int_fast32_t expZ; + uint_fast32_t sig32A, recipSqrt32, sig32Z; + struct uint128 rem; + uint32_t qs[3]; + uint_fast32_t q; + uint_fast64_t x64, sig64Z; + struct uint128 term, y; + uint_fast64_t sigZExtra; + struct uint128 sigZ; + union ui128_f128 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + signA = signF128UI64( uiA64 ); + expA = expF128UI64( uiA64 ); + sigA.v64 = fracF128UI64( uiA64 ); + sigA.v0 = uiA0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( sigA.v64 | sigA.v0 ) { + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, 0, 0 ); + goto uiZ; + } + if ( ! signA ) return a; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( signA ) { + if ( ! (expA | sigA.v64 | sigA.v0) ) return a; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! (sigA.v64 | sigA.v0) ) return a; + normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + | (`sig32Z' is guaranteed to be a lower bound on the square root of + | `sig32A', which makes `sig32Z' also a lower bound on the square root of + | `sigA'.) + *------------------------------------------------------------------------*/ + expZ = ((expA - 0x3FFF)>>1) + 0x3FFE; + expA &= 1; + sigA.v64 |= UINT64_C( 0x0001000000000000 ); + sig32A = sigA.v64>>17; + recipSqrt32 = softfloat_approxRecipSqrt32_1( expA, sig32A ); + sig32Z = ((uint_fast64_t) sig32A * recipSqrt32)>>32; + if ( expA ) { + sig32Z >>= 1; + rem = softfloat_shortShiftLeft128( sigA.v64, sigA.v0, 12 ); + } else { + rem = softfloat_shortShiftLeft128( sigA.v64, sigA.v0, 13 ); + } + qs[2] = sig32Z; + rem.v64 -= (uint_fast64_t) sig32Z * sig32Z; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = ((uint_fast64_t) (uint32_t) (rem.v64>>2) * recipSqrt32)>>32; + qs[1] = q; + x64 = (uint_fast64_t) sig32Z<<32; + sig64Z = x64 + ((uint_fast64_t) q<<3); + x64 += sig64Z; + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + term = softfloat_mul64ByShifted32To128( x64, q ); + rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = ((uint_fast64_t) (uint32_t) (rem.v64>>2) * recipSqrt32)>>32; + y = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 ); + sig64Z <<= 1; + /*------------------------------------------------------------------------ + | (Repeating this loop is a rare occurrence.) + *------------------------------------------------------------------------*/ + for (;;) { + term = softfloat_shortShiftLeft128( 0, sig64Z, 32 ); + term = softfloat_add128( term.v64, term.v0, 0, (uint_fast64_t) q<<6 ); + term = softfloat_mul128By32( term.v64, term.v0, q ); + rem = softfloat_sub128( y.v64, y.v0, term.v64, term.v0 ); + if ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) ) break; + --q; + } + qs[0] = q; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + q = (((uint_fast64_t) (uint32_t) (rem.v64>>2) * recipSqrt32)>>32) + 2; + sigZExtra = (uint64_t) ((uint_fast64_t) q<<59); + term = softfloat_shortShiftLeft128( 0, qs[1], 53 ); + sigZ = + softfloat_add128( + (uint_fast64_t) qs[2]<<18, ((uint_fast64_t) qs[0]<<24) + (q>>5), + term.v64, term.v0 + ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (q & 0xF) <= 2 ) { + q &= ~3; + sigZExtra = (uint64_t) ((uint_fast64_t) q<<59); + y = softfloat_shortShiftLeft128( sigZ.v64, sigZ.v0, 6 ); + y.v0 |= sigZExtra>>58; + term = softfloat_sub128( y.v64, y.v0, 0, q ); + y = softfloat_mul64ByShifted32To128( term.v0, q ); + term = softfloat_mul64ByShifted32To128( term.v64, q ); + term = softfloat_add128( term.v64, term.v0, 0, y.v64 ); + rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 20 ); + term = softfloat_sub128( term.v64, term.v0, rem.v64, rem.v0 ); + /*-------------------------------------------------------------------- + | The concatenation of `term' and `y.v0' is now the negative remainder + | (3 words altogether). + *--------------------------------------------------------------------*/ + if ( term.v64 & UINT64_C( 0x8000000000000000 ) ) { + sigZExtra |= 1; + } else { + if ( term.v64 | term.v0 | y.v0 ) { + if ( sigZExtra ) { + --sigZExtra; + } else { + sigZ = softfloat_sub128( sigZ.v64, sigZ.v0, 0, 1 ); + sigZExtra = ~0; + } + } + } + } + return softfloat_roundPackToF128( 0, expZ, sigZ.v64, sigZ.v0, sigZExtra ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ.v64 = defaultNaNF128UI64; + uiZ.v0 = defaultNaNF128UI0; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_sub.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_sub.c new file mode 100644 index 0000000..0ec2461 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_sub.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t f128_sub( float128_t a, float128_t b ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool signA; + union ui128_f128 uB; + uint_fast64_t uiB64, uiB0; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + float128_t + (*magsFuncPtr)( + uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool ); +#endif + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + signA = signF128UI64( uiA64 ); + uB.f = b; + uiB64 = uB.ui.v64; + uiB0 = uB.ui.v0; + signB = signF128UI64( uiB64 ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_subMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } else { + return softfloat_addMagsF128( uiA64, uiA0, uiB64, uiB0, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_subMagsF128 : softfloat_addMagsF128; + return (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_extF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_extF80.c new file mode 100644 index 0000000..53a01e3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_extF80.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t f128_to_extF80( float128_t a ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig64, sig0; + struct commonNaN commonNaN; + struct uint128 uiZ; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + struct exp32_sig128 normExpSig; + struct uint128 sig128; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ); + sig0 = uiA0; + if ( exp == 0x7FFF ) { + if ( sig64 | sig0 ) { + softfloat_f128UIToCommonNaN( uiA64, uiA0, &commonNaN ); + uiZ = softfloat_commonNaNToExtF80UI( &commonNaN ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + } else { + uiZ64 = packToExtF80UI64( sign, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + } + goto uiZ; + } + if ( ! exp ) { + if ( ! (sig64 | sig0) ) { + uiZ64 = packToExtF80UI64( sign, 0 ); + uiZ0 = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalF128Sig( sig64, sig0 ); + exp = normExpSig.exp; + sig64 = normExpSig.sig.v64; + sig0 = normExpSig.sig.v0; + } else { + sig64 |= UINT64_C( 0x0001000000000000 ); + } + sig128 = softfloat_shortShiftLeft128( sig64, sig0, 15 ); + return softfloat_roundPackToExtF80( sign, exp, sig128.v64, sig128.v0, 80 ); + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_f32.c new file mode 100644 index 0000000..d700602 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_f32.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t f128_to_f32( float128_t a ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig64; + struct commonNaN commonNaN; + uint_fast32_t uiZ, sig32; + union ui32_f32 uZ; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ) | (uiA0 != 0); + if ( exp == 0x7FFF ) { + if ( sig64 ) { + softfloat_f128UIToCommonNaN( uiA64, uiA0, &commonNaN ); + uiZ = softfloat_commonNaNToF32UI( &commonNaN ); + } else { + uiZ = packToF32UI( sign, 0xFF, 0 ); + } + goto uiZ; + } + sig32 = softfloat_shortShiftRightJam64( sig64, 18 ); + if ( ! (exp | sig32) ) { + uiZ = packToF32UI( sign, 0, 0 ); + goto uiZ; + } + exp -= 0x3F81; + if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return softfloat_roundPackToF32( sign, exp, sig32 | 0x40000000 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_f64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_f64.c new file mode 100644 index 0000000..a592d8f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_f64.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t f128_to_f64( float128_t a ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig64, sig0; + struct commonNaN commonNaN; + uint_fast64_t uiZ; + struct uint128 sig128; + union ui64_f64 uZ; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ); + sig0 = uiA0; + if ( exp == 0x7FFF ) { + if ( sig64 | sig0 ) { + softfloat_f128UIToCommonNaN( uiA64, uiA0, &commonNaN ); + uiZ = softfloat_commonNaNToF64UI( &commonNaN ); + } else { + uiZ = packToF64UI( sign, 0x7FF, 0 ); + } + goto uiZ; + } + sig128 = softfloat_shortShiftLeft128( sig64, sig0, 14 ); + sig64 = sig128.v64 | (sig128.v0 != 0); + if ( ! (exp | sig64) ) { + uiZ = packToF64UI( sign, 0, 0 ); + goto uiZ; + } + exp -= 0x3C01; + if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) { + if ( exp < -0x1000 ) exp = -0x1000; + } + return + softfloat_roundPackToF64( + sign, exp, sig64 | UINT64_C( 0x4000000000000000 ) ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i32.c new file mode 100644 index 0000000..5ac4b08 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i32.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t f128_to_i32( float128_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig64, sig0; + int_fast32_t shiftCount; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ); + sig0 = uiA0; + if ( (exp == 0x7FFF) && (sig64 | sig0) ) sign = 0; + if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 ); + sig64 |= (sig0 != 0); + shiftCount = 0x4028 - exp; + if ( 0 < shiftCount ) { + sig64 = softfloat_shiftRightJam64( sig64, shiftCount ); + } + return softfloat_roundPackToI32( sign, sig64, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i32_r_minMag.c new file mode 100644 index 0000000..c03e822 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i32_r_minMag.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t f128_to_i32_r_minMag( float128_t a, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + int_fast32_t exp; + uint_fast64_t sig64; + int_fast32_t shiftCount; + bool sign; + int_fast32_t absZ; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ) | (uiA0 != 0); + shiftCount = 0x402F - exp; + if ( 49 <= shiftCount ) { + if ( exact && (exp | sig64) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + sign = signF128UI64( uiA64 ); + if ( shiftCount < 18 ) { + if ( + sign && (shiftCount == 17) + && (sig64 < UINT64_C( 0x0000000000020000 )) + ) { + if ( exact && sig64 ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + } else { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( ! sign || ((exp == 0x7FFF) && sig64) ) return 0x7FFFFFFF; + } + return -0x7FFFFFFF - 1; + } + sig64 |= UINT64_C( 0x0001000000000000 ); + absZ = sig64>>shiftCount; + if ( + exact && ((uint_fast64_t) (uint_fast32_t) absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t f128_to_i64( float128_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig64, sig0; + int_fast32_t shiftCount; + struct uint128 sig128; + struct uint64_extra sigExtra; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ); + sig0 = uiA0; + shiftCount = 0x402F - exp; + if ( shiftCount <= 0 ) { + if ( shiftCount < -15 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return + ! sign || ((exp == 0x7FFF) && (sig64 | sig0)) + ? INT64_C( 0x7FFFFFFFFFFFFFFF ) + : -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + sig64 |= UINT64_C( 0x0001000000000000 ); + if ( shiftCount ) { + sig128 = softfloat_shortShiftLeft128( sig64, sig0, -shiftCount ); + sig64 = sig128.v64; + sig0 = sig128.v0; + } + } else { + if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 ); + sigExtra = softfloat_shiftRightJam64Extra( sig64, sig0, shiftCount ); + sig64 = sigExtra.v; + sig0 = sigExtra.extra; + } + return softfloat_roundPackToI64( sign, sig64, sig0, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i64_r_minMag.c new file mode 100644 index 0000000..0bd8c9a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_i64_r_minMag.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t f128_to_i64_r_minMag( float128_t a, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig64, sig0; + int_fast32_t shiftCount; + int_fast16_t negShiftCount; + int_fast64_t absZ; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ); + sig0 = uiA0; + shiftCount = 0x402F - exp; + if ( shiftCount < 0 ) { + if ( shiftCount < -14 ) { + if ( + (uiA64 == UINT64_C( 0xC03E000000000000 )) + && (sig0 < UINT64_C( 0x0002000000000000 )) + ) { + if ( exact && sig0 ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + } else { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( ! sign || ((exp == 0x7FFF) && (sig64 | sig0)) ) { + return UINT64_C( 0x7FFFFFFFFFFFFFFF ); + } + } + return -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + sig64 |= UINT64_C( 0x0001000000000000 ); + negShiftCount = -shiftCount; + absZ = sig64<>(shiftCount & 63); + if ( exact && (uint64_t) (sig0<>shiftCount; + if ( exact && (sig0 || (absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t + f128_to_ui32( float128_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp; + uint_fast64_t sig64, sig0; + int_fast32_t shiftCount; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ); + sig0 = uiA0; + if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 ); + sig64 |= (sig0 != 0); + shiftCount = 0x4028 - exp; + if ( 0 < shiftCount ) { + sig64 = softfloat_shiftRightJam64( sig64, shiftCount ); + } + return softfloat_roundPackToUI32( sign, sig64, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_ui32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_ui32_r_minMag.c new file mode 100644 index 0000000..075cc7d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_ui32_r_minMag.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t f128_to_ui32_r_minMag( float128_t a, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + int_fast32_t exp; + uint_fast64_t sig64; + int_fast32_t shiftCount; + uint_fast32_t z; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + exp = expF128UI64( uiA64 ); + sig64 = fracF128UI64( uiA64 ) | (uiA0 != 0); + shiftCount = 0x402F - exp; + if ( 49 <= shiftCount ) { + if ( exact && (exp | sig64) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signF128UI64( uiA64 ) || (shiftCount < 17) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return 0xFFFFFFFF; + } + sig64 |= UINT64_C( 0x0001000000000000 ); + z = sig64>>shiftCount; + if ( exact && ((uint_fast64_t) z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t + f128_to_ui64( float128_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + bool sign; + int_fast32_t exp, shiftCount; + uint_fast64_t sig64, sig0; + struct uint128 sig128; + struct uint64_extra sigExtra; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + sign = signF128UI64( uiA64 ); + exp = expF128UI64( uiA64 ); + shiftCount = 0x402F - exp; + if ( shiftCount <= 0 ) { + if ( shiftCount < -15 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + sig64 = fracF128UI64( uiA64 ) | UINT64_C( 0x0001000000000000 ); + sig0 = uiA0; + if ( shiftCount ) { + sig128 = softfloat_shortShiftLeft128( sig64, sig0, -shiftCount ); + sig64 = sig128.v64; + sig0 = sig128.v0; + } + } else { + sig64 = fracF128UI64( uiA64 ); + sig0 = uiA0; + if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 ); + sigExtra = softfloat_shiftRightJam64Extra( sig64, sig0, shiftCount ); + sig64 = sigExtra.v; + sig0 = sigExtra.extra; + } + return softfloat_roundPackToUI64( sign, sig64, sig0, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_ui64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_ui64_r_minMag.c new file mode 100644 index 0000000..756ee12 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f128_to_ui64_r_minMag.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t f128_to_ui64_r_minMag( float128_t a, bool exact ) +{ + union ui128_f128 uA; + uint_fast64_t uiA64, uiA0; + int_fast32_t exp, shiftCount; + uint_fast64_t sig64, sig0; + int_fast16_t negShiftCount; + uint_fast64_t z; + + uA.f = a; + uiA64 = uA.ui.v64; + uiA0 = uA.ui.v0; + exp = expF128UI64( uiA64 ); + shiftCount = 0x402F - exp; + if ( shiftCount < 0 ) { + if ( signF128UI64( uiA64 ) || (shiftCount < -15) ) goto invalid; + sig64 = fracF128UI64( uiA64 ) | UINT64_C( 0x0001000000000000 ); + sig0 = uiA0; + negShiftCount = -shiftCount; + z = sig64<>(shiftCount & 63); + if ( exact && (uint64_t) (sig0<>shiftCount; + if ( exact && (sig0 || (z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t f32_add( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool signA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 1) + float32_t (*magsFuncPtr)( uint_fast32_t, uint_fast32_t, bool ); +#endif + + uA.f = a; + uiA = uA.ui; + signA = signF32UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF32UI( uiB ); +#if defined INLINE_LEVEL && (1 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_addMagsF32( uiA, uiB, signA ); + } else { + return softfloat_subMagsF32( uiA, uiB, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_addMagsF32 : softfloat_subMagsF32; + return (*magsFuncPtr)( uiA, uiB, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_div.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_div.c new file mode 100644 index 0000000..a8c10ea --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_div.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t f32_div( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool signA; + int_fast16_t expA; + uint_fast32_t sigA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signB; + int_fast16_t expB; + uint_fast32_t sigB; + bool signZ; + struct exp16_sig32 normExpSig; + int_fast16_t expZ; +#ifdef SOFTFLOAT_FAST_DIV64TO32 + uint_fast64_t sig64A; + uint_fast32_t sigZ; +#else + uint_fast32_t sigZ; + uint_fast64_t rem; +#endif + uint_fast32_t uiZ; + union ui32_f32 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + signA = signF32UI( uiA ); + expA = expF32UI( uiA ); + sigA = fracF32UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF32UI( uiB ); + expB = expF32UI( uiB ); + sigB = fracF32UI( uiB ); + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0xFF ) { + if ( sigA ) goto propagateNaN; + if ( expB == 0xFF ) { + if ( sigB ) goto propagateNaN; + goto invalid; + } + goto infinity; + } + if ( expB == 0xFF ) { + if ( sigB ) goto propagateNaN; + goto zero; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) { + if ( ! sigB ) { + if ( ! (expA | sigA) ) goto invalid; + softfloat_raiseFlags( softfloat_flag_infinite ); + goto infinity; + } + normExpSig = softfloat_normSubnormalF32Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) { + if ( ! sigA ) goto zero; + normExpSig = softfloat_normSubnormalF32Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA - expB + 0x7E; + sigA |= 0x00800000; + sigB |= 0x00800000; +#ifdef SOFTFLOAT_FAST_DIV64TO32 + if ( sigA < sigB ) { + --expZ; + sig64A = (uint_fast64_t) sigA<<31; + } else { + sig64A = (uint_fast64_t) sigA<<30; + } + sigZ = sig64A / sigB; + if ( ! (sigZ & 0x3F) ) sigZ |= ((uint_fast64_t) sigB * sigZ != sig64A); +#else + if ( sigA < sigB ) { + --expZ; + sigA <<= 8; + } else { + sigA <<= 7; + } + sigB <<= 8; + sigZ = ((uint_fast64_t) sigA * softfloat_approxRecip32_1( sigB ))>>32; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + sigZ += 2; + if ( (sigZ & 0x3F) < 2 ) { + sigZ &= ~3; +#ifdef SOFTFLOAT_FAST_INT64 + rem = ((uint_fast64_t) sigA<<31) - (uint_fast64_t) sigZ * sigB; +#else + rem = ((uint_fast64_t) sigA<<32) - (uint_fast64_t) (sigZ<<1) * sigB; +#endif + if ( rem & UINT64_C( 0x8000000000000000 ) ) { + sigZ -= 4; + } else { + if ( rem ) sigZ |= 1; + } + } +#endif + return softfloat_roundPackToF32( signZ, expZ, sigZ ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF32UI( uiA, uiB ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF32UI; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infinity: + uiZ = packToF32UI( signZ, 0xFF, 0 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ = packToF32UI( signZ, 0, 0 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_eq.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_eq.c new file mode 100644 index 0000000..d01f498 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_eq.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f32_eq( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + union ui32_f32 uB; + uint_fast32_t uiB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF32UI( uiA ) || isNaNF32UI( uiB ) ) { + if ( + softfloat_isSigNaNF32UI( uiA ) || softfloat_isSigNaNF32UI( uiB ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + return (uiA == uiB) || ! (uint32_t) ((uiA | uiB)<<1); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_eq_signaling.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_eq_signaling.c new file mode 100644 index 0000000..de0e7c1 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_eq_signaling.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f32_eq_signaling( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + union ui32_f32 uB; + uint_fast32_t uiB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF32UI( uiA ) || isNaNF32UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + return (uiA == uiB) || ! (uint32_t) ((uiA | uiB)<<1); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_isSignalingNaN.c new file mode 100644 index 0000000..44061a3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_isSignalingNaN.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f32_isSignalingNaN( float32_t a ) +{ + union ui32_f32 uA; + + uA.f = a; + return softfloat_isSigNaNF32UI( uA.ui ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_le.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_le.c new file mode 100644 index 0000000..3b018fc --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_le.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f32_le( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF32UI( uiA ) || isNaNF32UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signF32UI( uiA ); + signB = signF32UI( uiB ); + return + (signA != signB) ? signA || ! (uint32_t) ((uiA | uiB)<<1) + : (uiA == uiB) || (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_le_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_le_quiet.c new file mode 100644 index 0000000..dd6701e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_le_quiet.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f32_le_quiet( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF32UI( uiA ) || isNaNF32UI( uiB ) ) { + if ( + softfloat_isSigNaNF32UI( uiA ) || softfloat_isSigNaNF32UI( uiB ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signF32UI( uiA ); + signB = signF32UI( uiB ); + return + (signA != signB) ? signA || ! (uint32_t) ((uiA | uiB)<<1) + : (uiA == uiB) || (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_lt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_lt.c new file mode 100644 index 0000000..9bc4787 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_lt.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f32_lt( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF32UI( uiA ) || isNaNF32UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signF32UI( uiA ); + signB = signF32UI( uiB ); + return + (signA != signB) ? signA && ((uint32_t) ((uiA | uiB)<<1) != 0) + : (uiA != uiB) && (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_lt_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_lt_quiet.c new file mode 100644 index 0000000..f5dbe5d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_lt_quiet.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f32_lt_quiet( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF32UI( uiA ) || isNaNF32UI( uiB ) ) { + if ( + softfloat_isSigNaNF32UI( uiA ) || softfloat_isSigNaNF32UI( uiB ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signF32UI( uiA ); + signB = signF32UI( uiB ); + return + (signA != signB) ? signA && ((uint32_t) ((uiA | uiB)<<1) != 0) + : (uiA != uiB) && (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_mul.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_mul.c new file mode 100644 index 0000000..d14c43b --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_mul.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t f32_mul( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool signA; + int_fast16_t expA; + uint_fast32_t sigA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signB; + int_fast16_t expB; + uint_fast32_t sigB; + bool signZ; + uint_fast32_t magBits; + struct exp16_sig32 normExpSig; + int_fast16_t expZ; + uint_fast32_t sigZ, uiZ; + union ui32_f32 uZ; + + uA.f = a; + uiA = uA.ui; + signA = signF32UI( uiA ); + expA = expF32UI( uiA ); + sigA = fracF32UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF32UI( uiB ); + expB = expF32UI( uiB ); + sigB = fracF32UI( uiB ); + signZ = signA ^ signB; + if ( expA == 0xFF ) { + if ( sigA || ((expB == 0xFF) && sigB) ) goto propagateNaN; + magBits = expB | sigB; + goto infArg; + } + if ( expB == 0xFF ) { + if ( sigB ) goto propagateNaN; + magBits = expA | sigA; + goto infArg; + } + if ( ! expA ) { + if ( ! sigA ) goto zero; + normExpSig = softfloat_normSubnormalF32Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) { + if ( ! sigB ) goto zero; + normExpSig = softfloat_normSubnormalF32Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + expZ = expA + expB - 0x7F; + sigA = (sigA | 0x00800000)<<7; + sigB = (sigB | 0x00800000)<<8; + sigZ = softfloat_shortShiftRightJam64( (uint_fast64_t) sigA * sigB, 32 ); + if ( sigZ < 0x40000000 ) { + --expZ; + sigZ <<= 1; + } + return softfloat_roundPackToF32( signZ, expZ, sigZ ); + propagateNaN: + uiZ = softfloat_propagateNaNF32UI( uiA, uiB ); + goto uiZ; + infArg: + if ( ! magBits ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF32UI; + } else { + uiZ = packToF32UI( signZ, 0xFF, 0 ); + } + goto uiZ; + zero: + uiZ = packToF32UI( signZ, 0, 0 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_mulAdd.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_mulAdd.c new file mode 100644 index 0000000..825aee1 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_mulAdd.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t f32_mulAdd( float32_t a, float32_t b, float32_t c ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + union ui32_f32 uB; + uint_fast32_t uiB; + union ui32_f32 uC; + uint_fast32_t uiC; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + uC.f = c; + uiC = uC.ui; + return softfloat_mulAddF32( uiA, uiB, uiC, 0 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_rem.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_rem.c new file mode 100644 index 0000000..fc7c5fa --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_rem.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t f32_rem( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool signA; + int_fast16_t expA; + uint_fast32_t sigA; + union ui32_f32 uB; + uint_fast32_t uiB; + int_fast16_t expB; + uint_fast32_t sigB; + struct exp16_sig32 normExpSig; + uint32_t rem; + int_fast16_t expDiff; + uint32_t q, recip32, altRem, meanRem; + bool signRem; + uint_fast32_t uiZ; + union ui32_f32 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + signA = signF32UI( uiA ); + expA = expF32UI( uiA ); + sigA = fracF32UI( uiA ); + uB.f = b; + uiB = uB.ui; + expB = expF32UI( uiB ); + sigB = fracF32UI( uiB ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0xFF ) { + if ( sigA || ((expB == 0xFF) && sigB) ) goto propagateNaN; + goto invalid; + } + if ( expB == 0xFF ) { + if ( sigB ) goto propagateNaN; + return a; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) { + if ( ! sigB ) goto invalid; + normExpSig = softfloat_normSubnormalF32Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) { + if ( ! sigA ) return a; + normExpSig = softfloat_normSubnormalF32Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + rem = sigA | 0x00800000; + sigB |= 0x00800000; + expDiff = expA - expB; + if ( expDiff < 1 ) { + if ( expDiff < -1 ) return a; + sigB <<= 6; + if ( expDiff ) { + rem <<= 5; + q = 0; + } else { + rem <<= 6; + q = (sigB <= rem); + if ( q ) rem -= sigB; + } + } else { + recip32 = softfloat_approxRecip32_1( sigB<<8 ); + /*-------------------------------------------------------------------- + | Changing the shift of `rem' here requires also changing the initial + | subtraction from `expDiff'. + *--------------------------------------------------------------------*/ + rem <<= 7; + expDiff -= 31; + /*-------------------------------------------------------------------- + | The scale of `sigB' affects how many bits are obtained during each + | cycle of the loop. Currently this is 29 bits per loop iteration, + | which is believed to be the maximum possible. + *--------------------------------------------------------------------*/ + sigB <<= 6; + for (;;) { + q = (rem * (uint_fast64_t) recip32)>>32; + if ( expDiff < 0 ) break; + rem = -(q * (uint32_t) sigB); + expDiff -= 29; + } + /*-------------------------------------------------------------------- + | (`expDiff' cannot be less than -30 here.) + *--------------------------------------------------------------------*/ + q >>= ~expDiff & 31; + rem = (rem<<(expDiff + 30)) - q * (uint32_t) sigB; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + do { + altRem = rem; + ++q; + rem -= sigB; + } while ( ! (rem & 0x80000000) ); + meanRem = rem + altRem; + if ( (meanRem & 0x80000000) || (! meanRem && (q & 1)) ) rem = altRem; + signRem = signA; + if ( 0x80000000 <= rem ) { + signRem = ! signRem; + rem = -rem; + } + return softfloat_normRoundPackToF32( signRem, expB, rem ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF32UI( uiA, uiB ); + goto uiZ; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF32UI; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_roundToInt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_roundToInt.c new file mode 100644 index 0000000..3dc945d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_roundToInt.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t f32_roundToInt( float32_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + int_fast16_t exp; + uint_fast32_t uiZ, lastBitMask, roundBitsMask; + union ui32_f32 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + exp = expF32UI( uiA ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp <= 0x7E ) { + if ( ! (uint32_t) (uiA<<1) ) return a; + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + uiZ = uiA & packToF32UI( 1, 0, 0 ); + switch ( roundingMode ) { + case softfloat_round_near_even: + if ( ! fracF32UI( uiA ) ) break; + case softfloat_round_near_maxMag: + if ( exp == 0x7E ) uiZ |= packToF32UI( 0, 0x7F, 0 ); + break; + case softfloat_round_min: + if ( uiZ ) uiZ = packToF32UI( 1, 0x7F, 0 ); + break; + case softfloat_round_max: + if ( ! uiZ ) uiZ = packToF32UI( 0, 0x7F, 0 ); + break; + } + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x96 <= exp ) { + if ( (exp == 0xFF) && fracF32UI( uiA ) ) { + uiZ = softfloat_propagateNaNF32UI( uiA, 0 ); + goto uiZ; + } + return a; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ = uiA; + lastBitMask = (uint_fast32_t) 1<<(0x96 - exp); + roundBitsMask = lastBitMask - 1; + if ( roundingMode == softfloat_round_near_maxMag ) { + uiZ += lastBitMask>>1; + } else if ( roundingMode == softfloat_round_near_even ) { + uiZ += lastBitMask>>1; + if ( ! (uiZ & roundBitsMask) ) uiZ &= ~lastBitMask; + } else if ( roundingMode != softfloat_round_minMag ) { + if ( signF32UI( uiZ ) ^ (roundingMode == softfloat_round_max) ) { + uiZ += roundBitsMask; + } + } + uiZ &= ~roundBitsMask; + if ( exact && (uiZ != uiA) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_sqrt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_sqrt.c new file mode 100644 index 0000000..3526578 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_sqrt.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t f32_sqrt( float32_t a ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool signA; + int_fast16_t expA; + uint_fast32_t sigA, uiZ; + struct exp16_sig32 normExpSig; + int_fast16_t expZ; + uint_fast32_t sigZ, shiftedSigZ; + uint32_t negRem; + union ui32_f32 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + signA = signF32UI( uiA ); + expA = expF32UI( uiA ); + sigA = fracF32UI( uiA ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0xFF ) { + if ( sigA ) { + uiZ = softfloat_propagateNaNF32UI( uiA, 0 ); + goto uiZ; + } + if ( ! signA ) return a; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( signA ) { + if ( ! (expA | sigA) ) return a; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! sigA ) return a; + normExpSig = softfloat_normSubnormalF32Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = ((expA - 0x7F)>>1) + 0x7E; + expA &= 1; + sigA = (sigA | 0x00800000)<<8; + sigZ = + ((uint_fast64_t) sigA * softfloat_approxRecipSqrt32_1( expA, sigA )) + >>32; + if ( expA ) sigZ >>= 1; + sigZ += 2; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (sigZ & 0x3F) < 2 ) { + shiftedSigZ = sigZ>>2; + negRem = shiftedSigZ * shiftedSigZ; + sigZ = shiftedSigZ<<2; + if ( negRem & 0x80000000 ) { + sigZ |= 1; + } else { + if ( negRem ) --sigZ; + } + } + return softfloat_roundPackToF32( 0, expZ, sigZ ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF32UI; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_sub.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_sub.c new file mode 100644 index 0000000..d0dd638 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_sub.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t f32_sub( float32_t a, float32_t b ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool signA; + union ui32_f32 uB; + uint_fast32_t uiB; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 1) + float32_t (*magsFuncPtr)( uint_fast32_t, uint_fast32_t, bool ); +#endif + + uA.f = a; + uiA = uA.ui; + signA = signF32UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF32UI( uiB ); +#if defined INLINE_LEVEL && (1 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_subMagsF32( uiA, uiB, signA ); + } else { + return softfloat_addMagsF32( uiA, uiB, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_subMagsF32 : softfloat_addMagsF32; + return (*magsFuncPtr)( uiA, uiB, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_extF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_extF80.c new file mode 100644 index 0000000..a500fcd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_extF80.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t f32_to_extF80( float32_t a ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool sign; + int_fast16_t exp; + uint_fast32_t sig; + struct commonNaN commonNaN; + struct uint128 uiZ; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + struct exp16_sig32 normExpSig; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + if ( exp == 0xFF ) { + if ( sig ) { + softfloat_f32UIToCommonNaN( uiA, &commonNaN ); + uiZ = softfloat_commonNaNToExtF80UI( &commonNaN ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + } else { + uiZ64 = packToExtF80UI64( sign, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + } + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ64 = packToExtF80UI64( sign, 0 ); + uiZ0 = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalF32Sig( sig ); + exp = normExpSig.exp; + sig = normExpSig.sig; + } + uiZ64 = packToExtF80UI64( sign, exp + 0x3F80 ); + uiZ0 = (uint_fast64_t) (sig | 0x00800000)<<40; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_extF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_extF80M.c new file mode 100644 index 0000000..88327c1 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_extF80M.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void f32_to_extF80M( float32_t a, extFloat80_t *zPtr ) +{ + + *zPtr = f32_to_extF80( a ); + +} + +#else + +void f32_to_extF80M( float32_t a, extFloat80_t *zPtr ) +{ + struct extFloat80M *zSPtr; + union ui32_f32 uA; + uint32_t uiA; + bool sign; + int_fast16_t exp; + uint32_t sig; + struct commonNaN commonNaN; + uint_fast16_t uiZ64; + uint64_t uiZ0; + struct exp16_sig32 normExpSig; + + zSPtr = (struct extFloat80M *) zPtr; + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + if ( exp == 0xFF ) { + if ( sig ) { + softfloat_f32UIToCommonNaN( uiA, &commonNaN ); + softfloat_commonNaNToExtF80M( &commonNaN, zSPtr ); + return; + } + uiZ64 = packToExtF80UI64( sign, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ64 = packToExtF80UI64( sign, 0 ); + uiZ0 = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalF32Sig( sig ); + exp = normExpSig.exp; + sig = normExpSig.sig; + } + uiZ64 = packToExtF80UI64( sign, exp + 0x3F80 ); + uiZ0 = UINT64_C( 0x8000000000000000 ) | (uint64_t) sig<<40; + uiZ: + zSPtr->signExp = uiZ64; + zSPtr->signif = uiZ0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f128.c new file mode 100644 index 0000000..bcd29f6 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f128.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t f32_to_f128( float32_t a ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool sign; + int_fast16_t exp; + uint_fast32_t sig; + struct commonNaN commonNaN; + struct uint128 uiZ; + struct exp16_sig32 normExpSig; + union ui128_f128 uZ; + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + if ( exp == 0xFF ) { + if ( sig ) { + softfloat_f32UIToCommonNaN( uiA, &commonNaN ); + uiZ = softfloat_commonNaNToF128UI( &commonNaN ); + } else { + uiZ.v64 = packToF128UI64( sign, 0x7FFF, 0 ); + uiZ.v0 = 0; + } + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ.v64 = packToF128UI64( sign, 0, 0 ); + uiZ.v0 = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalF32Sig( sig ); + exp = normExpSig.exp - 1; + sig = normExpSig.sig; + } + uiZ.v64 = packToF128UI64( sign, exp + 0x3F80, (uint_fast64_t) sig<<25 ); + uiZ.v0 = 0; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f128M.c new file mode 100644 index 0000000..f9c2a44 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f128M.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void f32_to_f128M( float32_t a, float128_t *zPtr ) +{ + + *zPtr = f32_to_f128( a ); + +} + +#else + +void f32_to_f128M( float32_t a, float128_t *zPtr ) +{ + uint32_t *zWPtr; + union ui32_f32 uA; + uint32_t uiA; + bool sign; + int_fast16_t exp; + uint32_t sig, uiZ64; + struct commonNaN commonNaN; + uint32_t uiZ96; + struct exp16_sig32 normExpSig; + uint64_t sig64; + + zWPtr = (uint32_t *) zPtr; + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + uiZ64 = 0; + if ( exp == 0xFF ) { + if ( sig ) { + softfloat_f32UIToCommonNaN( uiA, &commonNaN ); + softfloat_commonNaNToF128M( &commonNaN, zWPtr ); + return; + } + uiZ96 = packToF128UI96( sign, 0x7FFF, 0 ); + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ96 = packToF128UI96( sign, 0, 0 ); + goto uiZ; + } + normExpSig = softfloat_normSubnormalF32Sig( sig ); + exp = normExpSig.exp - 1; + sig = normExpSig.sig; + } + sig64 = (uint64_t) sig<<25; + uiZ96 = packToF128UI96( sign, exp + 0x3F80, sig64>>32 ); + uiZ64 = sig64; + uiZ: + zWPtr[indexWord( 4, 3 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = uiZ64; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f64.c new file mode 100644 index 0000000..6e38abd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_f64.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t f32_to_f64( float32_t a ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool sign; + int_fast16_t exp; + uint_fast32_t sig; + struct commonNaN commonNaN; + uint_fast64_t uiZ; + struct exp16_sig32 normExpSig; + union ui64_f64 uZ; + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + if ( exp == 0xFF ) { + if ( sig ) { + softfloat_f32UIToCommonNaN( uiA, &commonNaN ); + uiZ = softfloat_commonNaNToF64UI( &commonNaN ); + } else { + uiZ = packToF64UI( sign, 0x7FF, 0 ); + } + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ = packToF64UI( sign, 0, 0 ); + goto uiZ; + } + normExpSig = softfloat_normSubnormalF32Sig( sig ); + exp = normExpSig.exp - 1; + sig = normExpSig.sig; + } + uiZ = packToF64UI( sign, exp + 0x380, (uint_fast64_t) sig<<29 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i32.c new file mode 100644 index 0000000..99bb045 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i32.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t f32_to_i32( float32_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool sign; + int_fast16_t exp; + uint_fast32_t sig; + uint_fast64_t sig64; + int_fast16_t shiftCount; + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + if ( (exp == 0xFF) && sig ) sign = 0; + if ( exp ) sig |= 0x00800000; + sig64 = (uint_fast64_t) sig<<32; + shiftCount = 0xAF - exp; + if ( 0 < shiftCount ) { + sig64 = softfloat_shiftRightJam64( sig64, shiftCount ); + } + return softfloat_roundPackToI32( sign, sig64, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i32_r_minMag.c new file mode 100644 index 0000000..20fa808 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i32_r_minMag.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t f32_to_i32_r_minMag( float32_t a, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + int_fast16_t exp; + uint_fast32_t sig; + int_fast16_t shiftCount; + bool sign; + int_fast32_t absZ; + + uA.f = a; + uiA = uA.ui; + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + shiftCount = 0x9E - exp; + if ( 32 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + sign = signF32UI( uiA ); + if ( shiftCount <= 0 ) { + if ( uiA != packToF32UI( 1, 0x9E, 0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( ! sign || ((exp == 0xFF) && sig) ) return 0x7FFFFFFF; + } + return -0x7FFFFFFF - 1; + } + sig = (sig | 0x00800000)<<8; + absZ = sig>>shiftCount; + if ( exact && ((uint_fast32_t) absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t f32_to_i64( float32_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool sign; + int_fast16_t exp; + uint_fast32_t sig; + int_fast16_t shiftCount; +#ifdef SOFTFLOAT_FAST_INT64 + uint_fast64_t sig64, extra; + struct uint64_extra sig64Extra; +#else + uint32_t extSig[3]; +#endif + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + shiftCount = 0xBE - exp; + if ( shiftCount < 0 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( ! sign || ((exp == 0xFF) && sig) ) { + return INT64_C( 0x7FFFFFFFFFFFFFFF ); + } + return -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + if ( exp ) sig |= 0x00800000; +#ifdef SOFTFLOAT_FAST_INT64 + sig64 = (uint_fast64_t) sig<<40; + extra = 0; + if ( shiftCount ) { + sig64Extra = softfloat_shiftRightJam64Extra( sig64, 0, shiftCount ); + sig64 = sig64Extra.v; + extra = sig64Extra.extra; + } + return softfloat_roundPackToI64( sign, sig64, extra, roundingMode, exact ); +#else + extSig[indexWord( 3, 2 )] = sig<<8; + extSig[indexWord( 3, 1 )] = 0; + extSig[indexWord( 3, 0 )] = 0; + if ( shiftCount ) softfloat_shiftRightJam96M( extSig, shiftCount, extSig ); + return softfloat_roundPackMToI64( sign, extSig, roundingMode, exact ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i64_r_minMag.c new file mode 100644 index 0000000..bbf5b53 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_i64_r_minMag.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t f32_to_i64_r_minMag( float32_t a, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + int_fast16_t exp; + uint_fast32_t sig; + int_fast16_t shiftCount; + bool sign; + uint_fast64_t sig64; + int_fast64_t absZ; + + uA.f = a; + uiA = uA.ui; + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + shiftCount = 0xBE - exp; + if ( 64 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + sign = signF32UI( uiA ); + if ( shiftCount <= 0 ) { + if ( uiA != packToF32UI( 1, 0xBE, 0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( ! sign || ((exp == 0xFF) && sig) ) { + return INT64_C( 0x7FFFFFFFFFFFFFFF ); + } + } + return -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + sig |= 0x00800000; + sig64 = (uint_fast64_t) sig<<40; + absZ = sig64>>shiftCount; + shiftCount = 40 - shiftCount; + if ( exact && (shiftCount < 0) && (uint32_t) (sig<<(shiftCount & 31)) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return sign ? -absZ : absZ; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui32.c new file mode 100644 index 0000000..a6267ff --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui32.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t f32_to_ui32( float32_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool sign; + int_fast16_t exp; + uint_fast32_t sig; + uint_fast64_t sig64; + int_fast16_t shiftCount; + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + if ( exp ) sig |= 0x00800000; + sig64 = (uint_fast64_t) sig<<32; + shiftCount = 0xAF - exp; + if ( 0 < shiftCount ) { + sig64 = softfloat_shiftRightJam64( sig64, shiftCount ); + } + return softfloat_roundPackToUI32( sign, sig64, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui32_r_minMag.c new file mode 100644 index 0000000..090ceff --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui32_r_minMag.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t f32_to_ui32_r_minMag( float32_t a, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + int_fast16_t exp; + uint_fast32_t sig; + int_fast16_t shiftCount; + uint_fast32_t z; + + uA.f = a; + uiA = uA.ui; + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + shiftCount = 0x9E - exp; + if ( 32 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signF32UI( uiA ) || (shiftCount < 0) ) goto invalid; + sig = (sig | 0x00800000)<<8; + z = sig>>shiftCount; + if ( exact && (z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +uint_fast64_t f32_to_ui64( float32_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + bool sign; + int_fast16_t exp; + uint_fast32_t sig; + int_fast16_t shiftCount; + uint_fast64_t sig64, extra; + struct uint64_extra sig64Extra; + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + shiftCount = 0xBE - exp; + if ( shiftCount < 0 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + if ( exp ) sig |= 0x00800000; + sig64 = (uint_fast64_t) sig<<40; + extra = 0; + if ( shiftCount ) { + sig64Extra = softfloat_shiftRightJam64Extra( sig64, 0, shiftCount ); + sig64 = sig64Extra.v; + extra = sig64Extra.extra; + } + return + softfloat_roundPackToUI64( sign, sig64, extra, roundingMode, exact ); + +} + +#else + +uint_fast64_t f32_to_ui64( float32_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui32_f32 uA; + uint32_t uiA; + bool sign; + int_fast16_t exp; + uint32_t sig; + int_fast16_t shiftCount; + uint32_t extSig[3]; + + uA.f = a; + uiA = uA.ui; + sign = signF32UI( uiA ); + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + shiftCount = 0xBE - exp; + if ( shiftCount < 0 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + if ( exp ) sig |= 0x00800000; + extSig[indexWord( 3, 2 )] = sig<<8; + extSig[indexWord( 3, 1 )] = 0; + extSig[indexWord( 3, 0 )] = 0; + if ( shiftCount ) softfloat_shiftRightJam96M( extSig, shiftCount, extSig ); + return softfloat_roundPackMToUI64( sign, extSig, roundingMode, exact ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui64_r_minMag.c new file mode 100644 index 0000000..7d08797 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f32_to_ui64_r_minMag.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t f32_to_ui64_r_minMag( float32_t a, bool exact ) +{ + union ui32_f32 uA; + uint_fast32_t uiA; + int_fast16_t exp; + uint_fast32_t sig; + int_fast16_t shiftCount; + uint_fast64_t sig64, z; + + uA.f = a; + uiA = uA.ui; + exp = expF32UI( uiA ); + sig = fracF32UI( uiA ); + shiftCount = 0xBE - exp; + if ( 64 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signF32UI( uiA ) || (shiftCount < 0) ) goto invalid; + sig |= 0x00800000; + sig64 = (uint_fast64_t) sig<<40; + z = sig64>>shiftCount; + shiftCount = 40 - shiftCount; + if ( exact && (shiftCount < 0) && (uint32_t) (sig<<(shiftCount & 31)) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return z; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_add.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_add.c new file mode 100644 index 0000000..29ff80a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_add.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t f64_add( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool signA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + float64_t (*magsFuncPtr)( uint_fast64_t, uint_fast64_t, bool ); +#endif + + uA.f = a; + uiA = uA.ui; + signA = signF64UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF64UI( uiB ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_addMagsF64( uiA, uiB, signA ); + } else { + return softfloat_subMagsF64( uiA, uiB, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_addMagsF64 : softfloat_subMagsF64; + return (*magsFuncPtr)( uiA, uiB, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_div.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_div.c new file mode 100644 index 0000000..a73947a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_div.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t f64_div( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool signA; + int_fast16_t expA; + uint_fast64_t sigA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signB; + int_fast16_t expB; + uint_fast64_t sigB; + bool signZ; + struct exp16_sig64 normExpSig; + int_fast16_t expZ; + uint32_t recip32, sig32Z, doubleTerm; + uint_fast64_t rem; + uint32_t q; + uint_fast64_t sigZ; + uint_fast64_t uiZ; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + signA = signF64UI( uiA ); + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF64UI( uiB ); + expB = expF64UI( uiB ); + sigB = fracF64UI( uiB ); + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FF ) { + if ( sigA ) goto propagateNaN; + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN; + goto invalid; + } + goto infinity; + } + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN; + goto zero; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) { + if ( ! sigB ) { + if ( ! (expA | sigA) ) goto invalid; + softfloat_raiseFlags( softfloat_flag_infinite ); + goto infinity; + } + normExpSig = softfloat_normSubnormalF64Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) { + if ( ! sigA ) goto zero; + normExpSig = softfloat_normSubnormalF64Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA - expB + 0x3FE; + sigA |= UINT64_C( 0x0010000000000000 ); + sigB |= UINT64_C( 0x0010000000000000 ); + if ( sigA < sigB ) { + --expZ; + sigA <<= 11; + } else { + sigA <<= 10; + } + sigB <<= 11; + recip32 = softfloat_approxRecip32_1( sigB>>32 ) - 2; + sig32Z = ((uint32_t) (sigA>>32) * (uint_fast64_t) recip32)>>32; + doubleTerm = sig32Z<<1; + rem = + ((sigA - (uint_fast64_t) doubleTerm * (uint32_t) (sigB>>32))<<28) + - (uint_fast64_t) doubleTerm * ((uint32_t) sigB>>4); + q = (((uint32_t) (rem>>32) * (uint_fast64_t) recip32)>>32) + 4; + sigZ = ((uint_fast64_t) sig32Z<<32) + ((uint_fast64_t) q<<4); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (sigZ & 0x1FF) < 4<<4 ) { + q &= ~7; + sigZ &= ~(uint_fast64_t) 0x7F; + doubleTerm = q<<1; + rem = + ((rem - (uint_fast64_t) doubleTerm * (uint32_t) (sigB>>32))<<28) + - (uint_fast64_t) doubleTerm * ((uint32_t) sigB>>4); + if ( rem & UINT64_C( 0x8000000000000000 ) ) { + sigZ -= 1<<7; + } else { + if ( rem ) sigZ |= 1; + } + } + return softfloat_roundPackToF64( signZ, expZ, sigZ ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF64UI( uiA, uiB ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF64UI; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infinity: + uiZ = packToF64UI( signZ, 0x7FF, 0 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ = packToF64UI( signZ, 0, 0 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_eq.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_eq.c new file mode 100644 index 0000000..901626d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_eq.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f64_eq( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + union ui64_f64 uB; + uint_fast64_t uiB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF64UI( uiA ) || isNaNF64UI( uiB ) ) { + if ( + softfloat_isSigNaNF64UI( uiA ) || softfloat_isSigNaNF64UI( uiB ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + return (uiA == uiB) || ! ((uiA | uiB) & UINT64_C( 0x7FFFFFFFFFFFFFFF )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_eq_signaling.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_eq_signaling.c new file mode 100644 index 0000000..c58c509 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_eq_signaling.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f64_eq_signaling( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + union ui64_f64 uB; + uint_fast64_t uiB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF64UI( uiA ) || isNaNF64UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + return (uiA == uiB) || ! ((uiA | uiB) & UINT64_C( 0x7FFFFFFFFFFFFFFF )); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_isSignalingNaN.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_isSignalingNaN.c new file mode 100644 index 0000000..32b2a94 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_isSignalingNaN.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f64_isSignalingNaN( float64_t a ) +{ + union ui64_f64 uA; + + uA.f = a; + return softfloat_isSigNaNF64UI( uA.ui ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_le.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_le.c new file mode 100644 index 0000000..03bcb59 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_le.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f64_le( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF64UI( uiA ) || isNaNF64UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signF64UI( uiA ); + signB = signF64UI( uiB ); + return + (signA != signB) + ? signA || ! ((uiA | uiB) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + : (uiA == uiB) || (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_le_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_le_quiet.c new file mode 100644 index 0000000..0601f81 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_le_quiet.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f64_le_quiet( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF64UI( uiA ) || isNaNF64UI( uiB ) ) { + if ( + softfloat_isSigNaNF64UI( uiA ) || softfloat_isSigNaNF64UI( uiB ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signF64UI( uiA ); + signB = signF64UI( uiB ); + return + (signA != signB) + ? signA || ! ((uiA | uiB) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + : (uiA == uiB) || (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_lt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_lt.c new file mode 100644 index 0000000..72ab2cc --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_lt.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +bool f64_lt( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF64UI( uiA ) || isNaNF64UI( uiB ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return false; + } + signA = signF64UI( uiA ); + signB = signF64UI( uiB ); + return + (signA != signB) + ? signA && ((uiA | uiB) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + : (uiA != uiB) && (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_lt_quiet.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_lt_quiet.c new file mode 100644 index 0000000..2a6addd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_lt_quiet.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +bool f64_lt_quiet( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signA, signB; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + if ( isNaNF64UI( uiA ) || isNaNF64UI( uiB ) ) { + if ( + softfloat_isSigNaNF64UI( uiA ) || softfloat_isSigNaNF64UI( uiB ) + ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + } + return false; + } + signA = signF64UI( uiA ); + signB = signF64UI( uiB ); + return + (signA != signB) + ? signA && ((uiA | uiB) & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + : (uiA != uiB) && (signA ^ (uiA < uiB)); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_mul.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_mul.c new file mode 100644 index 0000000..441158e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_mul.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t f64_mul( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool signA; + int_fast16_t expA; + uint_fast64_t sigA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signB; + int_fast16_t expB; + uint_fast64_t sigB; + bool signZ; + uint_fast64_t magBits; + struct exp16_sig64 normExpSig; + int_fast16_t expZ; +#ifdef SOFTFLOAT_FAST_INT64 + struct uint128 sig128Z; +#else + uint32_t sig128Z[4]; +#endif + uint_fast64_t sigZ, uiZ; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + signA = signF64UI( uiA ); + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF64UI( uiB ); + expB = expF64UI( uiB ); + sigB = fracF64UI( uiB ); + signZ = signA ^ signB; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FF ) { + if ( sigA || ((expB == 0x7FF) && sigB) ) goto propagateNaN; + magBits = expB | sigB; + goto infArg; + } + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN; + magBits = expA | sigA; + goto infArg; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! sigA ) goto zero; + normExpSig = softfloat_normSubnormalF64Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) { + if ( ! sigB ) goto zero; + normExpSig = softfloat_normSubnormalF64Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x3FF; + sigA = (sigA | UINT64_C( 0x0010000000000000 ))<<10; + sigB = (sigB | UINT64_C( 0x0010000000000000 ))<<11; +#ifdef SOFTFLOAT_FAST_INT64 + sig128Z = softfloat_mul64To128( sigA, sigB ); + sigZ = sig128Z.v64 | (sig128Z.v0 != 0); +#else + softfloat_mul64To128M( sigA, sigB, sig128Z ); + sigZ = + (uint64_t) sig128Z[indexWord( 4, 3 )]<<32 | sig128Z[indexWord( 4, 2 )]; + if ( sig128Z[indexWord( 4, 1 )] || sig128Z[indexWord( 4, 0 )] ) sigZ |= 1; +#endif + if ( sigZ < UINT64_C( 0x4000000000000000 ) ) { + --expZ; + sigZ <<= 1; + } + return softfloat_roundPackToF64( signZ, expZ, sigZ ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF64UI( uiA, uiB ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infArg: + if ( ! magBits ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF64UI; + } else { + uiZ = packToF64UI( signZ, 0x7FF, 0 ); + } + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zero: + uiZ = packToF64UI( signZ, 0, 0 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_mulAdd.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_mulAdd.c new file mode 100644 index 0000000..50ff155 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_mulAdd.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t f64_mulAdd( float64_t a, float64_t b, float64_t c ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + union ui64_f64 uB; + uint_fast64_t uiB; + union ui64_f64 uC; + uint_fast64_t uiC; + + uA.f = a; + uiA = uA.ui; + uB.f = b; + uiB = uB.ui; + uC.f = c; + uiC = uC.ui; + return softfloat_mulAddF64( uiA, uiB, uiC, 0 ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_rem.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_rem.c new file mode 100644 index 0000000..a87fdd8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_rem.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t f64_rem( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool signA; + int_fast16_t expA; + uint_fast64_t sigA; + union ui64_f64 uB; + uint_fast64_t uiB; + int_fast16_t expB; + uint_fast64_t sigB; + struct exp16_sig64 normExpSig; + uint64_t rem; + int_fast16_t expDiff; + uint32_t q, recip32; + uint_fast64_t q64; + uint64_t altRem, meanRem; + bool signRem; + uint_fast64_t uiZ; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + signA = signF64UI( uiA ); + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + uB.f = b; + uiB = uB.ui; + expB = expF64UI( uiB ); + sigB = fracF64UI( uiB ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FF ) { + if ( sigA || ((expB == 0x7FF) && sigB) ) goto propagateNaN; + goto invalid; + } + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN; + return a; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA < expB - 1 ) return a; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expB ) { + if ( ! sigB ) goto invalid; + normExpSig = softfloat_normSubnormalF64Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + if ( ! expA ) { + if ( ! sigA ) return a; + normExpSig = softfloat_normSubnormalF64Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + rem = sigA | UINT64_C( 0x0010000000000000 ); + sigB |= UINT64_C( 0x0010000000000000 ); + expDiff = expA - expB; + if ( expDiff < 1 ) { + if ( expDiff < -1 ) return a; + sigB <<= 9; + if ( expDiff ) { + rem <<= 8; + q = 0; + } else { + rem <<= 9; + q = (sigB <= rem); + if ( q ) rem -= sigB; + } + } else { + recip32 = softfloat_approxRecip32_1( sigB>>21 ); + /*-------------------------------------------------------------------- + | Changing the shift of `rem' here requires also changing the initial + | subtraction from `expDiff'. + *--------------------------------------------------------------------*/ + rem <<= 9; + expDiff -= 30; + /*-------------------------------------------------------------------- + | The scale of `sigB' affects how many bits are obtained during each + | cycle of the loop. Currently this is 29 bits per loop iteration, + | the maximum possible. + *--------------------------------------------------------------------*/ + sigB <<= 9; + for (;;) { + q64 = (uint32_t) (rem>>32) * (uint_fast64_t) recip32; + if ( expDiff < 0 ) break; + q = (q64 + 0x80000000)>>32; +#ifdef SOFTFLOAT_FAST_INT64 + rem <<= 29; +#else + rem = (uint_fast64_t) (uint32_t) (rem>>3)<<32; +#endif + rem -= q * (uint64_t) sigB; + if ( rem & UINT64_C( 0x8000000000000000 ) ) rem += sigB; + expDiff -= 29; + } + /*-------------------------------------------------------------------- + | (`expDiff' cannot be less than -29 here.) + *--------------------------------------------------------------------*/ + q = (uint32_t) (q64>>32)>>(~expDiff & 31); + rem = (rem<<(expDiff + 30)) - q * (uint64_t) sigB; + if ( rem & UINT64_C( 0x8000000000000000 ) ) { + altRem = rem + sigB; + goto selectRem; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + do { + altRem = rem; + ++q; + rem -= sigB; + } while ( ! (rem & UINT64_C( 0x8000000000000000 )) ); + selectRem: + meanRem = rem + altRem; + if ( + (meanRem & UINT64_C( 0x8000000000000000 )) || (! meanRem && (q & 1)) + ) { + rem = altRem; + } + signRem = signA; + if ( rem & UINT64_C( 0x8000000000000000 ) ) { + signRem = ! signRem; + rem = -rem; + } + return softfloat_normRoundPackToF64( signRem, expB, rem ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNF64UI( uiA, uiB ); + goto uiZ; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF64UI; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_roundToInt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_roundToInt.c new file mode 100644 index 0000000..7c02a08 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_roundToInt.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t f64_roundToInt( float64_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + int_fast16_t exp; + uint_fast64_t uiZ, lastBitMask, roundBitsMask; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + exp = expF64UI( uiA ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( exp <= 0x3FE ) { + if ( ! (uiA & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) return a; + if ( exact ) softfloat_exceptionFlags |= softfloat_flag_inexact; + uiZ = uiA & packToF64UI( 1, 0, 0 ); + switch ( roundingMode ) { + case softfloat_round_near_even: + if ( ! fracF64UI( uiA ) ) break; + case softfloat_round_near_maxMag: + if ( exp == 0x3FE ) uiZ |= packToF64UI( 0, 0x3FF, 0 ); + break; + case softfloat_round_min: + if ( uiZ ) uiZ = packToF64UI( 1, 0x3FF, 0 ); + break; + case softfloat_round_max: + if ( ! uiZ ) uiZ = packToF64UI( 0, 0x3FF, 0 ); + break; + } + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x433 <= exp ) { + if ( (exp == 0x7FF) && fracF64UI( uiA ) ) { + uiZ = softfloat_propagateNaNF64UI( uiA, 0 ); + goto uiZ; + } + return a; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ = uiA; + lastBitMask = (uint_fast64_t) 1<<(0x433 - exp); + roundBitsMask = lastBitMask - 1; + if ( roundingMode == softfloat_round_near_maxMag ) { + uiZ += lastBitMask>>1; + } else if ( roundingMode == softfloat_round_near_even ) { + uiZ += lastBitMask>>1; + if ( ! (uiZ & roundBitsMask) ) uiZ &= ~lastBitMask; + } else if ( roundingMode != softfloat_round_minMag ) { + if ( signF64UI( uiZ ) ^ (roundingMode == softfloat_round_max) ) { + uiZ += roundBitsMask; + } + } + uiZ &= ~roundBitsMask; + if ( exact && (uiZ != uiA) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_sqrt.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_sqrt.c new file mode 100644 index 0000000..f2ffe2b --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_sqrt.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t f64_sqrt( float64_t a ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool signA; + int_fast16_t expA; + uint_fast64_t sigA, uiZ; + struct exp16_sig64 normExpSig; + int_fast16_t expZ; + uint32_t sig32A, recipSqrt32, sig32Z; + uint_fast64_t rem; + uint32_t q; + uint_fast64_t sigZ, shiftedSigZ; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + signA = signF64UI( uiA ); + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FF ) { + if ( sigA ) { + uiZ = softfloat_propagateNaNF64UI( uiA, 0 ); + goto uiZ; + } + if ( ! signA ) return a; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( signA ) { + if ( ! (expA | sigA) ) return a; + goto invalid; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! sigA ) return a; + normExpSig = softfloat_normSubnormalF64Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + /*------------------------------------------------------------------------ + | (`sig32Z' is guaranteed to be a lower bound on the square root of + | `sig32A', which makes `sig32Z' also a lower bound on the square root of + | `sigA'.) + *------------------------------------------------------------------------*/ + expZ = ((expA - 0x3FF)>>1) + 0x3FE; + expA &= 1; + sigA |= UINT64_C( 0x0010000000000000 ); + sig32A = sigA>>21; + recipSqrt32 = softfloat_approxRecipSqrt32_1( expA, sig32A ); + sig32Z = ((uint_fast64_t) sig32A * recipSqrt32)>>32; + if ( expA ) { + sigA <<= 8; + sig32Z >>= 1; + } else { + sigA <<= 9; + } + rem = sigA - (uint_fast64_t) sig32Z * sig32Z; + q = ((uint32_t) (rem>>2) * (uint_fast64_t) recipSqrt32)>>32; + sigZ = ((uint_fast64_t) sig32Z<<32 | 1<<5) + ((uint_fast64_t) q<<3); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (sigZ & 0x1FF) < 1<<5 ) { + sigZ &= ~(uint_fast64_t) 0x3F; + shiftedSigZ = sigZ>>6; + rem = (sigA<<52) - shiftedSigZ * shiftedSigZ; + if ( rem & UINT64_C( 0x8000000000000000 ) ) { + --sigZ; + } else { + if ( rem ) sigZ |= 1; + } + } + return softfloat_roundPackToF64( 0, expZ, sigZ ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF64UI; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_sub.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_sub.c new file mode 100644 index 0000000..c5b65b6 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_sub.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t f64_sub( float64_t a, float64_t b ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool signA; + union ui64_f64 uB; + uint_fast64_t uiB; + bool signB; +#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2) + float64_t (*magsFuncPtr)( uint_fast64_t, uint_fast64_t, bool ); +#endif + + uA.f = a; + uiA = uA.ui; + signA = signF64UI( uiA ); + uB.f = b; + uiB = uB.ui; + signB = signF64UI( uiB ); +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) + if ( signA == signB ) { + return softfloat_subMagsF64( uiA, uiB, signA ); + } else { + return softfloat_addMagsF64( uiA, uiB, signA ); + } +#else + magsFuncPtr = + (signA == signB) ? softfloat_subMagsF64 : softfloat_addMagsF64; + return (*magsFuncPtr)( uiA, uiB, signA ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_extF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_extF80.c new file mode 100644 index 0000000..0aec3e3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_extF80.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t f64_to_extF80( float64_t a ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + struct commonNaN commonNaN; + struct uint128 uiZ; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + struct exp16_sig64 normExpSig; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( exp == 0x7FF ) { + if ( sig ) { + softfloat_f64UIToCommonNaN( uiA, &commonNaN ); + uiZ = softfloat_commonNaNToExtF80UI( &commonNaN ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + } else { + uiZ64 = packToExtF80UI64( sign, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + } + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ64 = packToExtF80UI64( sign, 0 ); + uiZ0 = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalF64Sig( sig ); + exp = normExpSig.exp; + sig = normExpSig.sig; + } + uiZ64 = packToExtF80UI64( sign, exp + 0x3C00 ); + uiZ0 = (sig | UINT64_C( 0x0010000000000000 ))<<11; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_extF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_extF80M.c new file mode 100644 index 0000000..d0fbb17 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_extF80M.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void f64_to_extF80M( float64_t a, extFloat80_t *zPtr ) +{ + + *zPtr = f64_to_extF80( a ); + +} + +#else + +void f64_to_extF80M( float64_t a, extFloat80_t *zPtr ) +{ + struct extFloat80M *zSPtr; + union ui64_f64 uA; + uint64_t uiA; + bool sign; + int_fast16_t exp; + uint64_t sig; + struct commonNaN commonNaN; + uint_fast16_t uiZ64; + uint64_t uiZ0; + struct exp16_sig64 normExpSig; + + zSPtr = (struct extFloat80M *) zPtr; + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( exp == 0x7FF ) { + if ( sig ) { + softfloat_f64UIToCommonNaN( uiA, &commonNaN ); + softfloat_commonNaNToExtF80M( &commonNaN, zSPtr ); + return; + } + uiZ64 = packToExtF80UI64( sign, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ64 = packToExtF80UI64( sign, 0 ); + uiZ0 = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalF64Sig( sig ); + exp = normExpSig.exp; + sig = normExpSig.sig; + } + uiZ64 = packToExtF80UI64( sign, exp + 0x3C00 ); + uiZ0 = UINT64_C( 0x8000000000000000 ) | sig<<11; + uiZ: + zSPtr->signExp = uiZ64; + zSPtr->signif = uiZ0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f128.c new file mode 100644 index 0000000..319c3c6 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f128.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t f64_to_f128( float64_t a ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + struct commonNaN commonNaN; + struct uint128 uiZ; + struct exp16_sig64 normExpSig; + struct uint128 sig128; + union ui128_f128 uZ; + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( exp == 0x7FF ) { + if ( sig ) { + softfloat_f64UIToCommonNaN( uiA, &commonNaN ); + uiZ = softfloat_commonNaNToF128UI( &commonNaN ); + } else { + uiZ.v64 = packToF128UI64( sign, 0x7FFF, 0 ); + uiZ.v0 = 0; + } + goto uiZ; + } + if ( ! exp ) { + if ( ! sig ) { + uiZ.v64 = packToF128UI64( sign, 0, 0 ); + uiZ.v0 = 0; + goto uiZ; + } + normExpSig = softfloat_normSubnormalF64Sig( sig ); + exp = normExpSig.exp - 1; + sig = normExpSig.sig; + } + sig128 = softfloat_shortShiftLeft128( 0, sig, 60 ); + uiZ.v64 = packToF128UI64( sign, exp + 0x3C00, sig128.v64 ); + uiZ.v0 = sig128.v0; + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f128M.c new file mode 100644 index 0000000..f3d59d6 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f128M.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void f64_to_f128M( float64_t a, float128_t *zPtr ) +{ + + *zPtr = f64_to_f128( a ); + +} + +#else + +void f64_to_f128M( float64_t a, float128_t *zPtr ) +{ + uint32_t *zWPtr; + union ui64_f64 uA; + uint64_t uiA; + bool sign; + int_fast16_t exp; + uint64_t sig; + struct commonNaN commonNaN; + uint32_t uiZ96; + struct exp16_sig64 normExpSig; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zWPtr = (uint32_t *) zPtr; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zWPtr[indexWord( 4, 0 )] = 0; + if ( exp == 0x7FF ) { + if ( sig ) { + softfloat_f64UIToCommonNaN( uiA, &commonNaN ); + softfloat_commonNaNToF128M( &commonNaN, zWPtr ); + return; + } + uiZ96 = packToF128UI96( sign, 0x7FFF, 0 ); + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! exp ) { + if ( ! sig ) { + uiZ96 = packToF128UI96( sign, 0, 0 ); + goto uiZ; + } + normExpSig = softfloat_normSubnormalF64Sig( sig ); + exp = normExpSig.exp - 1; + sig = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zWPtr[indexWord( 4, 1 )] = (uint32_t) sig<<28; + sig >>= 4; + zWPtr[indexWordHi( 4 )] = packToF128UI96( sign, exp + 0x3C00, sig>>32 ); + zWPtr[indexWord( 4, 2 )] = sig; + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiZ: + zWPtr[indexWord( 4, 3 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f32.c new file mode 100644 index 0000000..5697dc8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_f32.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t f64_to_f32( float64_t a ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + struct commonNaN commonNaN; + uint_fast32_t uiZ, sig32; + union ui32_f32 uZ; + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( exp == 0x7FF ) { + if ( sig ) { + softfloat_f64UIToCommonNaN( uiA, &commonNaN ); + uiZ = softfloat_commonNaNToF32UI( &commonNaN ); + } else { + uiZ = packToF32UI( sign, 0xFF, 0 ); + } + goto uiZ; + } + sig32 = softfloat_shortShiftRightJam64( sig, 22 ); + if ( ! (exp | sig32) ) { + uiZ = packToF32UI( sign, 0, 0 ); + goto uiZ; + } + return softfloat_roundPackToF32( sign, exp - 0x381, sig32 | 0x40000000 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i32.c new file mode 100644 index 0000000..8375bc8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i32.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t f64_to_i32( float64_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( (exp == 0x7FF) && sig ) sign = 0; + if ( exp ) sig |= UINT64_C( 0x0010000000000000 ); + shiftCount = 0x42C - exp; + if ( 0 < shiftCount ) sig = softfloat_shiftRightJam64( sig, shiftCount ); + return softfloat_roundPackToI32( sign, sig, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i32_r_minMag.c new file mode 100644 index 0000000..464b289 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i32_r_minMag.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t f64_to_i32_r_minMag( float64_t a, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; + bool sign; + int_fast32_t absZ; + + uA.f = a; + uiA = uA.ui; + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + shiftCount = 0x433 - exp; + if ( 53 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + sign = signF64UI( uiA ); + if ( shiftCount < 22 ) { + if ( + sign && (exp == 0x41E) && (sig < UINT64_C( 0x0000000000200000 )) + ) { + if ( exact && sig ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + } else { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( ! sign || ((exp == 0x7FF) && sig) ) return 0x7FFFFFFF; + } + return -0x7FFFFFFF - 1; + } + sig |= UINT64_C( 0x0010000000000000 ); + absZ = sig>>shiftCount; + if ( exact && ((uint_fast64_t) (uint_fast32_t) absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t f64_to_i64( float64_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; +#ifdef SOFTFLOAT_FAST_INT64 + struct uint64_extra sigExtra; +#else + uint32_t extSig[3]; +#endif + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( exp ) sig |= UINT64_C( 0x0010000000000000 ); + shiftCount = 0x433 - exp; +#ifdef SOFTFLOAT_FAST_INT64 + if ( shiftCount <= 0 ) { + if ( shiftCount < -11 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return + ! sign + || ((exp == 0x7FF) + && (sig != UINT64_C( 0x0010000000000000 ))) + ? INT64_C( 0x7FFFFFFFFFFFFFFF ) + : -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + sigExtra.v = sig<<-shiftCount; + sigExtra.extra = 0; + } else { + sigExtra = softfloat_shiftRightJam64Extra( sig, 0, shiftCount ); + } + return + softfloat_roundPackToI64( + sign, sigExtra.v, sigExtra.extra, roundingMode, exact ); +#else + extSig[indexWord( 3, 0 )] = 0; + if ( shiftCount <= 0 ) { + if ( shiftCount < -11 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return + ! sign + || ((exp == 0x7FF) + && (sig != UINT64_C( 0x0010000000000000 ))) + ? INT64_C( 0x7FFFFFFFFFFFFFFF ) + : -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + sig <<= -shiftCount; + extSig[indexWord( 3, 2 )] = sig>>32; + extSig[indexWord( 3, 1 )] = sig; + } else { + extSig[indexWord( 3, 2 )] = sig>>32; + extSig[indexWord( 3, 1 )] = sig; + softfloat_shiftRightJam96M( extSig, shiftCount, extSig ); + } + return softfloat_roundPackMToI64( sign, extSig, roundingMode, exact ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i64_r_minMag.c new file mode 100644 index 0000000..8d9b9f1 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_i64_r_minMag.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t f64_to_i64_r_minMag( float64_t a, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; + int_fast64_t absZ; + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + shiftCount = 0x433 - exp; + if ( shiftCount <= 0 ) { + if ( shiftCount < -10 ) { + if ( uiA != packToF64UI( 1, 0x43E, 0 ) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + if ( ! sign || ((exp == 0x7FF) && sig) ) { + return INT64_C( 0x7FFFFFFFFFFFFFFF ); + } + } + return -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1; + } + sig |= UINT64_C( 0x0010000000000000 ); + absZ = sig<<-shiftCount; + } else { + if ( 53 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + sig |= UINT64_C( 0x0010000000000000 ); + absZ = sig>>shiftCount; + if ( exact && (absZ< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t f64_to_ui32( float64_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( exp ) sig |= UINT64_C( 0x0010000000000000 ); + shiftCount = 0x42C - exp; + if ( 0 < shiftCount ) sig = softfloat_shiftRightJam64( sig, shiftCount ); + return softfloat_roundPackToUI32( sign, sig, roundingMode, exact ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_ui32_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_ui32_r_minMag.c new file mode 100644 index 0000000..298e453 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_ui32_r_minMag.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t f64_to_ui32_r_minMag( float64_t a, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; + uint_fast32_t z; + + uA.f = a; + uiA = uA.ui; + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + shiftCount = 0x433 - exp; + if ( 53 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signF64UI( uiA ) || (shiftCount < 21) ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return 0xFFFFFFFF; + } + sig |= UINT64_C( 0x0010000000000000 ); + z = sig>>shiftCount; + if ( exact && ((uint_fast64_t) z< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t f64_to_ui64( float64_t a, uint_fast8_t roundingMode, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + bool sign; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; +#ifdef SOFTFLOAT_FAST_INT64 + struct uint64_extra sigExtra; +#else + uint32_t extSig[3]; +#endif + + uA.f = a; + uiA = uA.ui; + sign = signF64UI( uiA ); + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + if ( exp ) sig |= UINT64_C( 0x0010000000000000 ); + shiftCount = 0x433 - exp; +#ifdef SOFTFLOAT_FAST_INT64 + if ( shiftCount <= 0 ) { + if ( shiftCount < -11 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + sigExtra.v = sig<<-shiftCount; + sigExtra.extra = 0; + } else { + sigExtra = softfloat_shiftRightJam64Extra( sig, 0, shiftCount ); + } + return + softfloat_roundPackToUI64( + sign, sigExtra.v, sigExtra.extra, roundingMode, exact ); +#else + extSig[indexWord( 3, 0 )] = 0; + if ( shiftCount <= 0 ) { + if ( shiftCount < -11 ) { + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + sig <<= -shiftCount; + extSig[indexWord( 3, 2 )] = sig>>32; + extSig[indexWord( 3, 1 )] = sig; + } else { + extSig[indexWord( 3, 2 )] = sig>>32; + extSig[indexWord( 3, 1 )] = sig; + softfloat_shiftRightJam96M( extSig, shiftCount, extSig ); + } + return softfloat_roundPackMToUI64( sign, extSig, roundingMode, exact ); +#endif + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_ui64_r_minMag.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_ui64_r_minMag.c new file mode 100644 index 0000000..9a7d43c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/f64_to_ui64_r_minMag.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t f64_to_ui64_r_minMag( float64_t a, bool exact ) +{ + union ui64_f64 uA; + uint_fast64_t uiA; + int_fast16_t exp; + uint_fast64_t sig; + int_fast16_t shiftCount; + uint_fast64_t z; + + uA.f = a; + uiA = uA.ui; + exp = expF64UI( uiA ); + sig = fracF64UI( uiA ); + shiftCount = 0x433 - exp; + if ( 53 <= shiftCount ) { + if ( exact && (exp | sig) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return 0; + } + if ( signF64UI( uiA ) ) goto invalid; + if ( shiftCount <= 0 ) { + if ( shiftCount < -11 ) goto invalid; + z = (sig | UINT64_C( 0x0010000000000000 ))<<-shiftCount; + } else { + sig |= UINT64_C( 0x0010000000000000 ); + z = sig>>shiftCount; + if ( exact && (uint64_t) (sig<<(-shiftCount & 63)) ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + } + return z; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_extF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_extF80.c new file mode 100644 index 0000000..dd7beab --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_extF80.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +extFloat80_t i32_to_extF80( int32_t a ) +{ + uint_fast16_t uiZ64; + uint_fast32_t absA; + bool sign; + int_fast8_t shiftCount; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + uiZ64 = 0; + absA = 0; + if ( a ) { + sign = (a < 0); + absA = sign ? -(uint_fast32_t) a : (uint_fast32_t) a; + shiftCount = softfloat_countLeadingZeros32( absA ); + uiZ64 = packToExtF80UI64( sign, 0x401E - shiftCount ); + absA <<= shiftCount; + } + uZ.s.signExp = uiZ64; + uZ.s.signif = (uint_fast64_t) absA<<32; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_extF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_extF80M.c new file mode 100644 index 0000000..f81a639 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_extF80M.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void i32_to_extF80M( int32_t a, extFloat80_t *zPtr ) +{ + + *zPtr = i32_to_extF80( a ); + +} + +#else + +void i32_to_extF80M( int32_t a, extFloat80_t *zPtr ) +{ + struct extFloat80M *zSPtr; + uint_fast16_t uiZ64; + uint64_t sigZ; + bool sign; + uint32_t absA; + int_fast8_t shiftCount; + + zSPtr = (struct extFloat80M *) zPtr; + uiZ64 = 0; + sigZ = 0; + if ( a ) { + sign = (a < 0); + absA = sign ? -(uint32_t) a : (uint32_t) a; + shiftCount = softfloat_countLeadingZeros32( absA ); + uiZ64 = packToExtF80UI64( sign, 0x401E - shiftCount ); + sigZ = (uint64_t) (absA<signExp = uiZ64; + zSPtr->signif = sigZ; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f128.c new file mode 100644 index 0000000..56a6256 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f128.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t i32_to_f128( int32_t a ) +{ + uint_fast64_t uiZ64; + bool sign; + uint_fast32_t absA; + int_fast8_t shiftCount; + union ui128_f128 uZ; + + uiZ64 = 0; + if ( a ) { + sign = (a < 0); + absA = sign ? -(uint_fast32_t) a : (uint_fast32_t) a; + shiftCount = softfloat_countLeadingZeros32( absA ) + 17; + uiZ64 = + packToF128UI64( + sign, 0x402E - shiftCount, (uint_fast64_t) absA< +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void i32_to_f128M( int32_t a, float128_t *zPtr ) +{ + + *zPtr = i32_to_f128( a ); + +} + +#else + +void i32_to_f128M( int32_t a, float128_t *zPtr ) +{ + uint32_t *zWPtr; + uint32_t uiZ96, uiZ64; + bool sign; + uint32_t absA; + int_fast8_t shiftCount; + uint64_t normAbsA; + + zWPtr = (uint32_t *) zPtr; + uiZ96 = 0; + uiZ64 = 0; + if ( a ) { + sign = (a < 0); + absA = sign ? -(uint32_t) a : (uint32_t) a; + shiftCount = softfloat_countLeadingZeros32( absA ) + 17; + normAbsA = (uint64_t) absA<>32 ); + uiZ64 = normAbsA; + } + zWPtr[indexWord( 4, 3 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = uiZ64; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f32.c new file mode 100644 index 0000000..25dd5ba --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f32.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t i32_to_f32( int32_t a ) +{ + bool sign; + union ui32_f32 uZ; + uint_fast32_t absA; + + sign = (a < 0); + if ( ! (a & 0x7FFFFFFF) ) { + uZ.ui = sign ? packToF32UI( 1, 0x9E, 0 ) : 0; + return uZ.f; + } + absA = sign ? -(uint_fast32_t) a : (uint_fast32_t) a; + return softfloat_normRoundPackToF32( sign, 0x9C, absA ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f64.c new file mode 100644 index 0000000..1976ef7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i32_to_f64.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t i32_to_f64( int32_t a ) +{ + uint_fast64_t uiZ; + bool sign; + uint_fast32_t absA; + int_fast8_t shiftCount; + union ui64_f64 uZ; + + if ( ! a ) { + uiZ = 0; + } else { + sign = (a < 0); + absA = sign ? -(uint_fast32_t) a : (uint_fast32_t) a; + shiftCount = softfloat_countLeadingZeros32( absA ) + 21; + uiZ = + packToF64UI( + sign, 0x432 - shiftCount, (uint_fast64_t) absA< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +extFloat80_t i64_to_extF80( int64_t a ) +{ + uint_fast16_t uiZ64; + uint_fast64_t absA; + bool sign; + int_fast8_t shiftCount; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + uiZ64 = 0; + absA = 0; + if ( a ) { + sign = (a < 0); + absA = sign ? -(uint_fast64_t) a : (uint_fast64_t) a; + shiftCount = softfloat_countLeadingZeros64( absA ); + uiZ64 = packToExtF80UI64( sign, 0x403E - shiftCount ); + absA <<= shiftCount; + } + uZ.s.signExp = uiZ64; + uZ.s.signif = absA; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_extF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_extF80M.c new file mode 100644 index 0000000..1eb1ae4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_extF80M.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void i64_to_extF80M( int64_t a, extFloat80_t *zPtr ) +{ + + *zPtr = i64_to_extF80( a ); + +} + +#else + +void i64_to_extF80M( int64_t a, extFloat80_t *zPtr ) +{ + struct extFloat80M *zSPtr; + uint_fast16_t uiZ64; + uint64_t sigZ; + bool sign; + uint64_t absA; + int_fast8_t shiftCount; + + zSPtr = (struct extFloat80M *) zPtr; + uiZ64 = 0; + sigZ = 0; + if ( a ) { + sign = (a < 0); + absA = sign ? -(uint64_t) a : (uint64_t) a; + shiftCount = softfloat_countLeadingZeros64( absA ); + uiZ64 = packToExtF80UI64( sign, 0x403E - shiftCount ); + sigZ = absA<signExp = uiZ64; + zSPtr->signif = sigZ; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f128.c new file mode 100644 index 0000000..058850c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f128.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t i64_to_f128( int64_t a ) +{ + uint_fast64_t uiZ64, uiZ0; + bool sign; + uint_fast64_t absA; + int_fast8_t shiftCount; + struct uint128 zSig; + union ui128_f128 uZ; + + if ( ! a ) { + uiZ64 = 0; + uiZ0 = 0; + } else { + sign = (a < 0); + absA = sign ? -(uint_fast64_t) a : (uint_fast64_t) a; + shiftCount = softfloat_countLeadingZeros64( absA ) + 49; + if ( 64 <= shiftCount ) { + zSig.v64 = absA<<(shiftCount - 64); + zSig.v0 = 0; + } else { + zSig = softfloat_shortShiftLeft128( 0, absA, shiftCount ); + } + uiZ64 = packToF128UI64( sign, 0x406E - shiftCount, zSig.v64 ); + uiZ0 = zSig.v0; + } + uZ.ui.v64 = uiZ64; + uZ.ui.v0 = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f128M.c new file mode 100644 index 0000000..15bba34 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f128M.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void i64_to_f128M( int64_t a, float128_t *zPtr ) +{ + + *zPtr = i64_to_f128( a ); + +} + +#else + +void i64_to_f128M( int64_t a, float128_t *zPtr ) +{ + uint32_t *zWPtr; + uint32_t uiZ96, uiZ64; + bool sign; + uint64_t absA; + uint_fast8_t shiftCount; + uint32_t *ptr; + + zWPtr = (uint32_t *) zPtr; + uiZ96 = 0; + uiZ64 = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + if ( a ) { + sign = (a < 0); + absA = sign ? -(uint64_t) a : (uint64_t) a; + shiftCount = softfloat_countLeadingZeros64( absA ) + 17; + if ( shiftCount < 32 ) { + ptr = zWPtr + indexMultiwordHi( 4, 3 ); + ptr[indexWord( 3, 2 )] = 0; + ptr[indexWord( 3, 1 )] = absA>>32; + ptr[indexWord( 3, 0 )] = absA; + softfloat_shortShiftLeft96M( ptr, shiftCount, ptr ); + ptr[indexWordHi( 3 )] = + packToF128UI96( + sign, 0x404E - shiftCount, ptr[indexWordHi( 3 )] ); + return; + } + absA <<= shiftCount - 32; + uiZ96 = packToF128UI96( sign, 0x404E - shiftCount, absA>>32 ); + uiZ64 = absA; + } + zWPtr[indexWord( 4, 3 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = uiZ64; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f32.c new file mode 100644 index 0000000..3bd36c1 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/i64_to_f32.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t i64_to_f32( int64_t a ) +{ + bool sign; + uint_fast64_t absA; + int_fast8_t shiftCount; + union ui32_f32 u; + uint_fast32_t sig; + + sign = (a < 0); + absA = sign ? -(uint_fast64_t) a : (uint_fast64_t) a; + shiftCount = softfloat_countLeadingZeros64( absA ) - 40; + if ( 0 <= shiftCount ) { + u.ui = + a ? packToF32UI( + sign, 0x95 - shiftCount, (uint_fast32_t) absA< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t i64_to_f64( int64_t a ) +{ + bool sign; + union ui64_f64 uZ; + uint_fast64_t absA; + + sign = (a < 0); + if ( ! (a & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) { + uZ.ui = sign ? packToF64UI( 1, 0x43E, 0 ) : 0; + return uZ.f; + } + absA = sign ? -(uint_fast64_t) a : (uint_fast64_t) a; + return softfloat_normRoundPackToF64( sign, 0x43C, absA ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/internals.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/internals.h new file mode 100644 index 0000000..36dbd9f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/internals.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#ifndef internals_h +#define internals_h 1 + +#include +#include +#include "primitives.h" +#include "softfloat_types.h" + +union ui32_f32 { uint32_t ui; float32_t f; }; +union ui64_f64 { uint64_t ui; float64_t f; }; + +#ifdef SOFTFLOAT_FAST_INT64 +union extF80M_extF80 { struct extFloat80M fM; extFloat80_t f; }; +union ui128_f128 { struct uint128 ui; float128_t f; }; +#endif + +enum { + softfloat_mulAdd_subC = 1, + softfloat_mulAdd_subProd = 2 +}; + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +uint_fast32_t + softfloat_roundPackToUI32( bool, uint_fast64_t, uint_fast8_t, bool ); + +#ifdef SOFTFLOAT_FAST_INT64 +uint_fast64_t + softfloat_roundPackToUI64( + bool, uint_fast64_t, uint_fast64_t, uint_fast8_t, bool ); +#else +uint_fast64_t + softfloat_roundPackMToUI64( bool, uint32_t *, uint_fast8_t, bool ); +#endif + +int_fast32_t + softfloat_roundPackToI32( bool, uint_fast64_t, uint_fast8_t, bool ); + +#ifdef SOFTFLOAT_FAST_INT64 +int_fast64_t + softfloat_roundPackToI64( + bool, uint_fast64_t, uint_fast64_t, uint_fast8_t, bool ); +#else +int_fast64_t softfloat_roundPackMToI64( bool, uint32_t *, uint_fast8_t, bool ); +#endif + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define signF32UI( a ) ((bool) ((uint32_t) (a)>>31)) +#define expF32UI( a ) ((int_fast16_t) ((a)>>23) & 0xFF) +#define fracF32UI( a ) ((a) & 0x007FFFFF) +#define packToF32UI( sign, exp, sig ) (((uint32_t) (sign)<<31) + ((uint32_t) (exp)<<23) + (sig)) + +#define isNaNF32UI( a ) ((((a) & 0x7F800000) == 0x7F800000) && ((a) & 0x007FFFFF)) + +struct exp16_sig32 { int_fast16_t exp; uint_fast32_t sig; }; +struct exp16_sig32 softfloat_normSubnormalF32Sig( uint_fast32_t ); + +float32_t softfloat_roundPackToF32( bool, int_fast16_t, uint_fast32_t ); +float32_t softfloat_normRoundPackToF32( bool, int_fast16_t, uint_fast32_t ); + +float32_t softfloat_addMagsF32( uint_fast32_t, uint_fast32_t, bool ); +float32_t softfloat_subMagsF32( uint_fast32_t, uint_fast32_t, bool ); +float32_t + softfloat_mulAddF32( + uint_fast32_t, uint_fast32_t, uint_fast32_t, uint_fast8_t ); + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define signF64UI( a ) ((bool) ((uint64_t) (a)>>63)) +#define expF64UI( a ) ((int_fast16_t) ((a)>>52) & 0x7FF) +#define fracF64UI( a ) ((a) & UINT64_C( 0x000FFFFFFFFFFFFF )) +#define packToF64UI( sign, exp, sig ) ((uint64_t) (((uint_fast64_t) (sign)<<63) + ((uint_fast64_t) (exp)<<52) + (sig))) + +#define isNaNF64UI( a ) ((((a) & UINT64_C( 0x7FF0000000000000 )) == UINT64_C( 0x7FF0000000000000 )) && ((a) & UINT64_C( 0x000FFFFFFFFFFFFF ))) + +struct exp16_sig64 { int_fast16_t exp; uint_fast64_t sig; }; +struct exp16_sig64 softfloat_normSubnormalF64Sig( uint_fast64_t ); + +float64_t softfloat_roundPackToF64( bool, int_fast16_t, uint_fast64_t ); +float64_t softfloat_normRoundPackToF64( bool, int_fast16_t, uint_fast64_t ); + +float64_t softfloat_addMagsF64( uint_fast64_t, uint_fast64_t, bool ); +float64_t softfloat_subMagsF64( uint_fast64_t, uint_fast64_t, bool ); +float64_t + softfloat_mulAddF64( + uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast8_t ); + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define signExtF80UI64( a64 ) ((bool) ((uint16_t) (a64)>>15)) +#define expExtF80UI64( a64 ) ((a64) & 0x7FFF) +#define packToExtF80UI64( sign, exp ) ((uint_fast16_t) (sign)<<15 | (exp)) + +#define isNaNExtF80UI( a64, a0 ) ((((a64) & 0x7FFF) == 0x7FFF) && ((a0) & UINT64_C( 0x7FFFFFFFFFFFFFFF ))) + +#ifdef SOFTFLOAT_FAST_INT64 + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ + +struct exp32_sig64 { int_fast32_t exp; uint64_t sig; }; +struct exp32_sig64 softfloat_normSubnormalExtF80Sig( uint_fast64_t ); + +extFloat80_t + softfloat_roundPackToExtF80( + bool, int_fast32_t, uint_fast64_t, uint_fast64_t, uint_fast8_t ); +extFloat80_t + softfloat_normRoundPackToExtF80( + bool, int_fast32_t, uint_fast64_t, uint_fast64_t, uint_fast8_t ); + +extFloat80_t + softfloat_addMagsExtF80( + uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool ); +extFloat80_t + softfloat_subMagsExtF80( + uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool ); + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define signF128UI64( a64 ) ((bool) ((uint64_t) (a64)>>63)) +#define expF128UI64( a64 ) ((int_fast32_t) ((a64)>>48) & 0x7FFF) +#define fracF128UI64( a64 ) ((a64) & UINT64_C( 0x0000FFFFFFFFFFFF )) +#define packToF128UI64( sign, exp, sig64 ) (((uint_fast64_t) (sign)<<63) + ((uint_fast64_t) (exp)<<48) + (sig64)) + +#define isNaNF128UI( a64, a0 ) ((((a64) & UINT64_C( 0x7FFF000000000000 )) == UINT64_C( 0x7FFF000000000000 )) && (a0 || ((a64) & UINT64_C( 0x0000FFFFFFFFFFFF )))) + +struct exp32_sig128 { int_fast32_t exp; struct uint128 sig; }; +struct exp32_sig128 + softfloat_normSubnormalF128Sig( uint_fast64_t, uint_fast64_t ); + +float128_t + softfloat_roundPackToF128( + bool, int_fast32_t, uint_fast64_t, uint_fast64_t, uint_fast64_t ); +float128_t + softfloat_normRoundPackToF128( + bool, int_fast32_t, uint_fast64_t, uint_fast64_t ); + +float128_t + softfloat_addMagsF128( + uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool ); +float128_t + softfloat_subMagsF128( + uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool ); +float128_t + softfloat_mulAddF128( + uint_fast64_t, + uint_fast64_t, + uint_fast64_t, + uint_fast64_t, + uint_fast64_t, + uint_fast64_t, + uint_fast8_t + ); + +#else + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ + +bool + softfloat_tryPropagateNaNExtF80M( + const struct extFloat80M *, + const struct extFloat80M *, + struct extFloat80M * + ); +void softfloat_invalidExtF80M( struct extFloat80M * ); + +int softfloat_normExtF80SigM( uint64_t * ); + +void + softfloat_roundPackMToExtF80M( + bool, int32_t, uint32_t *, uint_fast8_t, struct extFloat80M * ); +void + softfloat_normRoundPackMToExtF80M( + bool, int32_t, uint32_t *, uint_fast8_t, struct extFloat80M * ); + +void + softfloat_addExtF80M( + const struct extFloat80M *, + const struct extFloat80M *, + struct extFloat80M *, + bool + ); + +int + softfloat_compareNonnormExtF80M( + const struct extFloat80M *, const struct extFloat80M * ); + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +#define signF128UI96( a96 ) ((bool) ((uint32_t) (a96)>>31)) +#define expF128UI96( a96 ) ((int32_t) ((a96)>>16) & 0x7FFF) +#define fracF128UI96( a96 ) ((a96) & 0x0000FFFF) +#define packToF128UI96( sign, exp, sig96 ) (((uint32_t) (sign)<<31) + ((uint32_t) (exp)<<16) + (sig96)) + +bool softfloat_isNaNF128M( const uint32_t * ); + +bool + softfloat_tryPropagateNaNF128M( + const uint32_t *, const uint32_t *, uint32_t * ); +void softfloat_invalidF128M( uint32_t * ); + +int softfloat_shiftNormSigF128M( const uint32_t *, uint_fast8_t, uint32_t * ); + +void softfloat_roundPackMToF128M( bool, int32_t, uint32_t *, uint32_t * ); +void softfloat_normRoundPackMToF128M( bool, int32_t, uint32_t *, uint32_t * ); + +void + softfloat_addF128M( const uint32_t *, const uint32_t *, uint32_t *, bool ); +void + softfloat_mulAddF128M( + const uint32_t *, + const uint32_t *, + const uint32_t *, + uint32_t *, + uint_fast8_t + ); + +#endif + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/primitiveTypes.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/primitiveTypes.h new file mode 100644 index 0000000..50570c0 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/primitiveTypes.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#ifndef primitiveTypes_h +#define primitiveTypes_h 1 + +#include + +#ifdef SOFTFLOAT_FAST_INT64 + +#ifdef LITTLEENDIAN +struct uint128 { uint64_t v0, v64; }; +struct uint64_extra { uint64_t extra, v; }; +struct uint128_extra { uint64_t extra; struct uint128 v; }; +#else +struct uint128 { uint64_t v64, v0; }; +struct uint64_extra { uint64_t v, extra; }; +struct uint128_extra { struct uint128 v; uint64_t extra; }; +#endif + +#endif + +/*---------------------------------------------------------------------------- +| These macros are used to isolate the differences in word order between big- +| endian and little-endian platforms. +*----------------------------------------------------------------------------*/ +#ifdef LITTLEENDIAN +#define wordIncr 1 +#define indexWord( total, n ) (n) +#define indexWordHi( total ) ((total) - 1) +#define indexWordLo( total ) 0 +#define indexMultiword( total, m, n ) (n) +#define indexMultiwordHi( total, n ) ((total) - (n)) +#define indexMultiwordLo( total, n ) 0 +#define indexMultiwordHiBut( total, n ) (n) +#define indexMultiwordLoBut( total, n ) 0 +#define INIT_UINTM4( v3, v2, v1, v0 ) { v0, v1, v2, v3 } +#else +#define wordIncr -1 +#define indexWord( total, n ) ((total) - 1 - (n)) +#define indexWordHi( total ) 0 +#define indexWordLo( total ) ((total) - 1) +#define indexMultiword( total, m, n ) ((total) - 1 - (m)) +#define indexMultiwordHi( total, n ) 0 +#define indexMultiwordLo( total, n ) ((total) - (n)) +#define indexMultiwordHiBut( total, n ) 0 +#define indexMultiwordLoBut( total, n ) (n) +#define INIT_UINTM4( v3, v2, v1, v0 ) { v3, v2, v1, v0 } +#endif + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/primitives.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/primitives.h new file mode 100644 index 0000000..486c63c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/primitives.h @@ -0,0 +1,1136 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#ifndef primitives_h +#define primitives_h 1 + +#include +#include +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRightJam64 +/*---------------------------------------------------------------------------- +| Shifts `a' right by the number of bits given in `count', which must be in +| the range 1 to 63. If any nonzero bits are shifted off, they are "jammed" +| into the least-significant bit of the shifted value by setting the least- +| significant bit to 1. This shifted-and-jammed value is returned. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + uint64_t softfloat_shortShiftRightJam64( uint64_t a, uint_fast8_t count ) + { return a>>count | ((a & (((uint_fast64_t) 1<>count | ((uint32_t) (a<<(-count & 31)) != 0) + : (a != 0); +} +#else +uint32_t softfloat_shiftRightJam32( uint32_t a, uint_fast16_t count ); +#endif +#endif + +#ifndef softfloat_shiftRightJam64 +/*---------------------------------------------------------------------------- +| Shifts `a' right by the number of bits given in `count', which must not +| be zero. If any nonzero bits are shifted off, they are "jammed" into the +| least-significant bit of the shifted value by setting the least-significant +| bit to 1. This shifted-and-jammed value is returned. +| The value of `count' can be arbitrarily large. In particular, if `count' +| is greater than 64, the result will be either 0 or 1, depending on whether +| `a' is zero or nonzero. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL) +INLINE uint64_t softfloat_shiftRightJam64( uint64_t a, uint_fast32_t count ) +{ + return + (count < 63) ? a>>count | ((uint64_t) (a<<(-count & 63)) != 0) + : (a != 0); +} +#else +uint64_t softfloat_shiftRightJam64( uint64_t a, uint_fast32_t count ); +#endif +#endif + +/*---------------------------------------------------------------------------- +| A constant table that translates an 8-bit unsigned integer (the array index) +| into the number of leading 0 bits before the most-significant 1 of that +| integer. For integer zero (index 0), the corresponding table element is 8. +*----------------------------------------------------------------------------*/ +extern const uint_least8_t softfloat_countLeadingZeros8[256]; + +#ifndef softfloat_countLeadingZeros32 +/*---------------------------------------------------------------------------- +| Returns the number of leading 0 bits before the most-significant 1 bit of +| `a'. If `a' is zero, 32 is returned. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL) +INLINE uint_fast8_t softfloat_countLeadingZeros32( uint32_t a ) +{ + uint_fast8_t count = 0; + if ( a < 0x10000 ) { + count = 16; + a <<= 16; + } + if ( a < 0x1000000 ) { + count += 8; + a <<= 8; + } + count += softfloat_countLeadingZeros8[a>>24]; + return count; +} +#else +uint_fast8_t softfloat_countLeadingZeros32( uint32_t a ); +#endif +#endif + +#ifndef softfloat_countLeadingZeros64 +/*---------------------------------------------------------------------------- +| Returns the number of leading 0 bits before the most-significant 1 bit of +| `a'. If `a' is zero, 64 is returned. +*----------------------------------------------------------------------------*/ +uint_fast8_t softfloat_countLeadingZeros64( uint64_t a ); +#endif + +#ifndef softfloat_approxRecip32_1 +/*---------------------------------------------------------------------------- +| Returns an approximation to the reciprocal of the number represented by `a', +| where `a' is interpreted as an unsigned fixed-point number with one integer +| bit and 31 fraction bits. The `a' input must be "normalized", meaning that +| its most-significant bit (bit 31) must be 1. Thus, if A is the value of +| the fixed-point interpretation of `a', then 1 <= A < 2. The returned value +| is interpreted as a pure unsigned fraction, having no integer bits and 32 +| fraction bits. The approximation returned is never greater than the true +| reciprocal 1/A, and it differs from the true reciprocal by at most 2.006 ulp +| (units in the last place). +*----------------------------------------------------------------------------*/ +#ifdef SOFTFLOAT_FAST_DIV64TO32 +#define softfloat_approxRecip32_1( a ) ((uint32_t) (UINT64_C( 0x7FFFFFFFFFFFFFFF ) / (uint32_t) (a))) +#else +uint32_t softfloat_approxRecip32_1( uint32_t a ); +#endif +#endif + +#ifndef softfloat_approxRecipSqrt32_1 +/*---------------------------------------------------------------------------- +| Returns an approximation to the reciprocal of the square root of the number +| represented by `a', where `a' is interpreted as an unsigned fixed-point +| number either with one integer bit and 31 fraction bits or with two integer +| bits and 30 fraction bits. The format of `a' is determined by `oddExpA', +| which must be either 0 or 1. If `oddExpA' is 1, `a' is interpreted as +| having one integer bit, and if `oddExpA' is 0, `a' is interpreted as having +| two integer bits. The `a' input must be "normalized", meaning that its +| most-significant bit (bit 31) must be 1. Thus, if A is the value of the +| fixed-point interpretation of `a', it follows that 1 <= A < 2 when `oddExpA' +| is 1, and 2 <= A < 4 when `oddExpA' is 0. +| The returned value is interpreted as a pure unsigned fraction, having +| no integer bits and 32 fraction bits. The approximation returned is never +| greater than the true reciprocal 1/sqrt(A), and it differs from the true +| reciprocal by at most 2.06 ulp (units in the last place). The approximation +| returned is also always within the range 0.5 to 1; thus, the most- +| significant bit of the result is always set. +*----------------------------------------------------------------------------*/ +uint32_t softfloat_approxRecipSqrt32_1( unsigned int oddExpA, uint32_t a ); +#endif + +#ifdef SOFTFLOAT_FAST_INT64 + +/*---------------------------------------------------------------------------- +| The following functions are needed only when `SOFTFLOAT_FAST_INT64' is +| defined. +*----------------------------------------------------------------------------*/ + +#ifndef softfloat_eq128 +/*---------------------------------------------------------------------------- +| Returns true if the 128-bit unsigned integer formed by concatenating `a64' +| and `a0' is equal to the 128-bit unsigned integer formed by concatenating +| `b64' and `b0'. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (1 <= INLINE_LEVEL) +INLINE + bool softfloat_eq128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) + { return (a64 == b64) && (a0 == b0); } +#else +bool softfloat_eq128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ); +#endif +#endif + +#ifndef softfloat_le128 +/*---------------------------------------------------------------------------- +| Returns true if the 128-bit unsigned integer formed by concatenating `a64' +| and `a0' is less than or equal to the 128-bit unsigned integer formed by +| concatenating `b64' and `b0'. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + bool softfloat_le128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) + { return (a64 < b64) || ((a64 == b64) && (a0 <= b0)); } +#else +bool softfloat_le128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ); +#endif +#endif + +#ifndef softfloat_lt128 +/*---------------------------------------------------------------------------- +| Returns true if the 128-bit unsigned integer formed by concatenating `a64' +| and `a0' is less than the 128-bit unsigned integer formed by concatenating +| `b64' and `b0'. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + bool softfloat_lt128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) + { return (a64 < b64) || ((a64 == b64) && (a0 < b0)); } +#else +bool softfloat_lt128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ); +#endif +#endif + +#ifndef softfloat_shortShiftLeft128 +/*---------------------------------------------------------------------------- +| Shifts the 128 bits formed by concatenating `a64' and `a0' left by the +| number of bits given in `count', which must be in the range 1 to 63. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + struct uint128 + softfloat_shortShiftLeft128( uint64_t a64, uint64_t a0, uint_fast8_t count ) +{ + struct uint128 z; + z.v64 = a64<>(-count & 63); + z.v0 = a0<>count; + z.v0 = a64<<(-count & 63) | a0>>count; + return z; +} +#else +struct uint128 + softfloat_shortShiftRight128( uint64_t a64, uint64_t a0, uint_fast8_t count ); +#endif +#endif + +#ifndef softfloat_shortShiftRightJam64Extra +/*---------------------------------------------------------------------------- +| This function is the same as `softfloat_shiftRightJam64Extra' (below), +| except that `count' must be in the range 1 to 63. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + struct uint64_extra + softfloat_shortShiftRightJam64Extra( + uint64_t a, uint64_t extra, uint_fast8_t count ) +{ + struct uint64_extra z; + z.v = a>>count; + z.extra = a<<(-count & 63) | (extra != 0); + return z; +} +#else +struct uint64_extra + softfloat_shortShiftRightJam64Extra( + uint64_t a, uint64_t extra, uint_fast8_t count ); +#endif +#endif + +#ifndef softfloat_shortShiftRightJam128 +/*---------------------------------------------------------------------------- +| Shifts the 128 bits formed by concatenating `a64' and `a0' right by the +| number of bits given in `count', which must be in the range 1 to 63. If any +| nonzero bits are shifted off, they are "jammed" into the least-significant +| bit of the shifted value by setting the least-significant bit to 1. This +| shifted-and-jammed value is returned. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL) +INLINE + struct uint128 + softfloat_shortShiftRightJam128( + uint64_t a64, uint64_t a0, uint_fast8_t count ) +{ + uint_fast8_t negCount = -count; + struct uint128 z; + z.v64 = a64>>count; + z.v0 = + a64<<(negCount & 63) | a0>>count + | ((uint64_t) (a0<<(negCount & 63)) != 0); + return z; +} +#else +struct uint128 + softfloat_shortShiftRightJam128( + uint64_t a64, uint64_t a0, uint_fast8_t count ); +#endif +#endif + +#ifndef softfloat_shortShiftRightJam128Extra +/*---------------------------------------------------------------------------- +| This function is the same as `softfloat_shiftRightJam128Extra' (below), +| except that `count' must be in the range 1 to 63. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL) +INLINE + struct uint128_extra + softfloat_shortShiftRightJam128Extra( + uint64_t a64, uint64_t a0, uint64_t extra, uint_fast8_t count ) +{ + uint_fast8_t negCount = -count; + struct uint128_extra z; + z.v.v64 = a64>>count; + z.v.v0 = a64<<(negCount & 63) | a0>>count; + z.extra = a0<<(negCount & 63) | (extra != 0); + return z; +} +#else +struct uint128_extra + softfloat_shortShiftRightJam128Extra( + uint64_t a64, uint64_t a0, uint64_t extra, uint_fast8_t count ); +#endif +#endif + +#ifndef softfloat_shiftRightJam64Extra +/*---------------------------------------------------------------------------- +| Shifts the 128 bits formed by concatenating `a' and `extra' right by 64 +| _plus_ the number of bits given in `count', which must not be zero. This +| shifted value is at most 64 nonzero bits and is returned in the `v' field +| of the `struct uint64_extra' result. The 64-bit `extra' field of the result +| contains a value formed as follows from the bits that were shifted off: The +| _last_ bit shifted off is the most-significant bit of the `extra' field, and +| the other 63 bits of the `extra' field are all zero if and only if _all_but_ +| _the_last_ bits shifted off were all zero. +| (This function makes more sense if `a' and `extra' are considered to form +| an unsigned fixed-point number with binary point between `a' and `extra'. +| This fixed-point value is shifted right by the number of bits given in +| `count', and the integer part of this shifted value is returned in the `v' +| field of the result. The fractional part of the shifted value is modified +| as described above and returned in the `extra' field of the result.) +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (4 <= INLINE_LEVEL) +INLINE + struct uint64_extra + softfloat_shiftRightJam64Extra( + uint64_t a, uint64_t extra, uint_fast32_t count ) +{ + struct uint64_extra z; + if ( count < 64 ) { + z.v = a>>count; + z.extra = a<<(-count & 63); + } else { + z.v = 0; + z.extra = (count == 64) ? a : (a != 0); + } + z.extra |= (extra != 0); + return z; +} +#else +struct uint64_extra + softfloat_shiftRightJam64Extra( + uint64_t a, uint64_t extra, uint_fast32_t count ); +#endif +#endif + +#ifndef softfloat_shiftRightJam128 +/*---------------------------------------------------------------------------- +| Shifts the 128 bits formed by concatenating `a64' and `a0' right by the +| number of bits given in `count', which must not be zero. If any nonzero +| bits are shifted off, they are "jammed" into the least-significant bit of +| the shifted value by setting the least-significant bit to 1. This shifted- +| and-jammed value is returned. +| The value of `count' can be arbitrarily large. In particular, if `count' +| is greater than 128, the result will be either 0 or 1, depending on whether +| the original 128 bits are all zeros. +*----------------------------------------------------------------------------*/ +struct uint128 + softfloat_shiftRightJam128( uint64_t a64, uint64_t a0, uint_fast32_t count ); +#endif + +#ifndef softfloat_shiftRightJam128Extra +/*---------------------------------------------------------------------------- +| Shifts the 192 bits formed by concatenating `a64', `a0', and `extra' right +| by 64 _plus_ the number of bits given in `count', which must not be zero. +| This shifted value is at most 128 nonzero bits and is returned in the `v' +| field of the `struct uint128_extra' result. The 64-bit `extra' field of the +| result contains a value formed as follows from the bits that were shifted +| off: The _last_ bit shifted off is the most-significant bit of the `extra' +| field, and the other 63 bits of the `extra' field are all zero if and only +| if _all_but_the_last_ bits shifted off were all zero. +| (This function makes more sense if `a64', `a0', and `extra' are considered +| to form an unsigned fixed-point number with binary point between `a0' and +| `extra'. This fixed-point value is shifted right by the number of bits +| given in `count', and the integer part of this shifted value is returned +| in the `v' field of the result. The fractional part of the shifted value +| is modified as described above and returned in the `extra' field of the +| result.) +*----------------------------------------------------------------------------*/ +struct uint128_extra + softfloat_shiftRightJam128Extra( + uint64_t a64, uint64_t a0, uint64_t extra, uint_fast32_t count ); +#endif + +#ifndef softfloat_shiftRightJam256M +/*---------------------------------------------------------------------------- +| Shifts the 256-bit unsigned integer pointed to by `aPtr' right by the number +| of bits given in `count', which must not be zero. If any nonzero bits are +| shifted off, they are "jammed" into the least-significant bit of the shifted +| value by setting the least-significant bit to 1. This shifted-and-jammed +| value is stored at the location pointed to by `zPtr'. Each of `aPtr' and +| `zPtr' points to an array of four 64-bit elements that concatenate in the +| platform's normal endian order to form a 256-bit integer. +| The value of `count' can be arbitrarily large. In particular, if `count' +| is greater than 256, the stored result will be either 0 or 1, depending on +| whether the original 256 bits are all zeros. +*----------------------------------------------------------------------------*/ +void + softfloat_shiftRightJam256M( + const uint64_t *aPtr, uint_fast32_t count, uint64_t *zPtr ); +#endif + +#ifndef softfloat_add128 +/*---------------------------------------------------------------------------- +| Returns the sum of the 128-bit integer formed by concatenating `a64' and +| `a0' and the 128-bit integer formed by concatenating `b64' and `b0'. The +| addition is modulo 2^128, so any carry out is lost. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + struct uint128 + softfloat_add128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) +{ + struct uint128 z; + z.v0 = a0 + b0; + z.v64 = a64 + b64 + (z.v0 < a0); + return z; +} +#else +struct uint128 + softfloat_add128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ); +#endif +#endif + +#ifndef softfloat_add256M +/*---------------------------------------------------------------------------- +| Adds the two 256-bit integers pointed to by `aPtr' and `bPtr'. The addition +| is modulo 2^256, so any carry out is lost. The sum is stored at the +| location pointed to by `zPtr'. Each of `aPtr', `bPtr', and `zPtr' points to +| an array of four 64-bit elements that concatenate in the platform's normal +| endian order to form a 256-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_add256M( + const uint64_t *aPtr, const uint64_t *bPtr, uint64_t *zPtr ); +#endif + +#ifndef softfloat_sub128 +/*---------------------------------------------------------------------------- +| Returns the difference of the 128-bit integer formed by concatenating `a64' +| and `a0' and the 128-bit integer formed by concatenating `b64' and `b0'. +| The subtraction is modulo 2^128, so any borrow out (carry out) is lost. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + struct uint128 + softfloat_sub128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) +{ + struct uint128 z; + z.v0 = a0 - b0; + z.v64 = a64 - b64; + z.v64 -= (a0 < b0); + return z; +} +#else +struct uint128 + softfloat_sub128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ); +#endif +#endif + +#ifndef softfloat_sub256M +/*---------------------------------------------------------------------------- +| Subtracts the 256-bit integer pointed to by `bPtr' from the 256-bit integer +| pointed to by `aPtr'. The addition is modulo 2^256, so any borrow out +| (carry out) is lost. The difference is stored at the location pointed to +| by `zPtr'. Each of `aPtr', `bPtr', and `zPtr' points to an array of four +| 64-bit elements that concatenate in the platform's normal endian order to +| form a 256-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_sub256M( + const uint64_t *aPtr, const uint64_t *bPtr, uint64_t *zPtr ); +#endif + +#ifndef softfloat_mul64ByShifted32To128 +/*---------------------------------------------------------------------------- +| Returns the 128-bit product of `a', `b', and 2^32. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL) +INLINE struct uint128 softfloat_mul64ByShifted32To128( uint64_t a, uint32_t b ) +{ + uint_fast64_t mid; + struct uint128 z; + mid = (uint_fast64_t) (uint32_t) a * b; + z.v0 = mid<<32; + z.v64 = (uint_fast64_t) (uint32_t) (a>>32) * b + (mid>>32); + return z; +} +#else +struct uint128 softfloat_mul64ByShifted32To128( uint64_t a, uint32_t b ); +#endif +#endif + +#ifndef softfloat_mul64To128 +/*---------------------------------------------------------------------------- +| Returns the 128-bit product of `a' and `b'. +*----------------------------------------------------------------------------*/ +struct uint128 softfloat_mul64To128( uint64_t a, uint64_t b ); +#endif + +#ifndef softfloat_mul128By32 +/*---------------------------------------------------------------------------- +| Returns the product of the 128-bit integer formed by concatenating `a64' and +| `a0', multiplied by `b'. The multiplication is modulo 2^128; any overflow +| bits are discarded. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (4 <= INLINE_LEVEL) +INLINE + struct uint128 softfloat_mul128By32( uint64_t a64, uint64_t a0, uint32_t b ) +{ + struct uint128 z; + uint_fast64_t mid; + uint_fast32_t carry; + z.v0 = a0 * b; + mid = (uint_fast64_t) (uint32_t) (a0>>32) * b; + carry = (uint32_t) ((uint_fast32_t) (z.v0>>32) - (uint_fast32_t) mid); + z.v64 = a64 * b + (uint_fast32_t) ((mid + carry)>>32); + return z; +} +#else +struct uint128 softfloat_mul128By32( uint64_t a64, uint64_t a0, uint32_t b ); +#endif +#endif + +#ifndef softfloat_mul128To256M +/*---------------------------------------------------------------------------- +| Multiplies the 128-bit unsigned integer formed by concatenating `a64' and +| `a0' by the 128-bit unsigned integer formed by concatenating `b64' and +| `b0'. The 256-bit product is stored at the location pointed to by `zPtr'. +| Argument `zPtr' points to an array of four 64-bit elements that concatenate +| in the platform's normal endian order to form a 256-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_mul128To256M( + uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0, uint64_t *zPtr ); +#endif + +#else + +/*---------------------------------------------------------------------------- +| The following functions are needed only when `SOFTFLOAT_FAST_INT64' is not +| defined. +*----------------------------------------------------------------------------*/ + +#ifndef softfloat_compare96M +/*---------------------------------------------------------------------------- +| Compares the two 96-bit unsigned integers pointed to by `aPtr' and `bPtr'. +| Returns -1 if the first integer (A) is less than the second (B); returns 0 +| if the two integers are equal; and returns +1 if the first integer (A) +| is greater than the second (B). (The result is thus the signum of A - B.) +| Each of `aPtr' and `bPtr' points to an array of three 32-bit elements that +| concatenate in the platform's normal endian order to form a 96-bit integer. +*----------------------------------------------------------------------------*/ +int_fast8_t softfloat_compare96M( const uint32_t *aPtr, const uint32_t *bPtr ); +#endif + +#ifndef softfloat_compare128M +/*---------------------------------------------------------------------------- +| Compares the two 128-bit unsigned integers pointed to by `aPtr' and `bPtr'. +| Returns -1 if the first integer (A) is less than the second (B); returns 0 +| if the two integers are equal; and returns +1 if the first integer (A) +| is greater than the second (B). (The result is thus the signum of A - B.) +| Each of `aPtr' and `bPtr' points to an array of four 32-bit elements that +| concatenate in the platform's normal endian order to form a 128-bit integer. +*----------------------------------------------------------------------------*/ +int_fast8_t + softfloat_compare128M( const uint32_t *aPtr, const uint32_t *bPtr ); +#endif + +#ifndef softfloat_shortShiftLeft64To96M +/*---------------------------------------------------------------------------- +| Extends `a' to 96 bits and shifts the value left by the number of bits given +| in `count', which must be in the range 1 to 31. The result is stored at the +| location pointed to by `zPtr'. Argument `zPtr' points to an array of three +| 32-bit elements that concatenate in the platform's normal endian order to +| form a 96-bit integer. +*----------------------------------------------------------------------------*/ +#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL) +INLINE + void + softfloat_shortShiftLeft64To96M( + uint64_t a, uint_fast8_t count, uint32_t *zPtr ) +{ + zPtr[indexWord( 3, 0 )] = (uint32_t) a<>= 32 - count; + zPtr[indexWord( 3, 2 )] = a>>32; + zPtr[indexWord( 3, 1 )] = a; +} +#else +void + softfloat_shortShiftLeft64To96M( + uint64_t a, uint_fast8_t count, uint32_t *zPtr ); +#endif +#endif + +#ifndef softfloat_shortShiftLeftM +/*---------------------------------------------------------------------------- +| Shifts the N-bit unsigned integer pointed to by `aPtr' left by the number +| of bits given in `count', where N = `size_words' * 32. The value of `count' +| must be in the range 1 to 31. Any nonzero bits shifted off are lost. The +| shifted N-bit result is stored at the location pointed to by `zPtr'. Each +| of `aPtr' and `zPtr' points to a `size_words'-long array of 32-bit elements +| that concatenate in the platform's normal endian order to form an N-bit +| integer. +*----------------------------------------------------------------------------*/ +void + softfloat_shortShiftLeftM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint_fast8_t count, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_shortShiftLeft96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shortShiftLeftM' with +| `size_words' = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_shortShiftLeft96M( aPtr, count, zPtr ) softfloat_shortShiftLeftM( 3, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shortShiftLeft128M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shortShiftLeftM' with +| `size_words' = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_shortShiftLeft128M( aPtr, count, zPtr ) softfloat_shortShiftLeftM( 4, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shortShiftLeft160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shortShiftLeftM' with +| `size_words' = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_shortShiftLeft160M( aPtr, count, zPtr ) softfloat_shortShiftLeftM( 5, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shiftLeftM +/*---------------------------------------------------------------------------- +| Shifts the N-bit unsigned integer pointed to by `aPtr' left by the number +| of bits given in `count', where N = `size_words' * 32. The value of `count' +| must not be zero. Any nonzero bits shifted off are lost. The shifted +| N-bit result is stored at the location pointed to by `zPtr'. Each of `aPtr' +| and `zPtr' points to a `size_words'-long array of 32-bit elements that +| concatenate in the platform's normal endian order to form an N-bit integer. +| The value of `count' can be arbitrarily large. In particular, if `count' +| is greater than N, the stored result will be 0. +*----------------------------------------------------------------------------*/ +void + softfloat_shiftLeftM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint32_t count, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_shiftLeft96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shiftLeftM' with +| `size_words' = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_shiftLeft96M( aPtr, count, zPtr ) softfloat_shiftLeftM( 3, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shiftLeft128M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shiftLeftM' with +| `size_words' = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_shiftLeft128M( aPtr, count, zPtr ) softfloat_shiftLeftM( 4, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shiftLeft160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shiftLeftM' with +| `size_words' = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_shiftLeft160M( aPtr, count, zPtr ) softfloat_shiftLeftM( 5, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shortShiftRightM +/*---------------------------------------------------------------------------- +| Shifts the N-bit unsigned integer pointed to by `aPtr' right by the number +| of bits given in `count', where N = `size_words' * 32. The value of `count' +| must be in the range 1 to 31. Any nonzero bits shifted off are lost. The +| shifted N-bit result is stored at the location pointed to by `zPtr'. Each +| of `aPtr' and `zPtr' points to a `size_words'-long array of 32-bit elements +| that concatenate in the platform's normal endian order to form an N-bit +| integer. +*----------------------------------------------------------------------------*/ +void + softfloat_shortShiftRightM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint_fast8_t count, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_shortShiftRight128M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shortShiftRightM' with +| `size_words' = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_shortShiftRight128M( aPtr, count, zPtr ) softfloat_shortShiftRightM( 4, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shortShiftRight160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shortShiftRightM' with +| `size_words' = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_shortShiftRight160M( aPtr, count, zPtr ) softfloat_shortShiftRightM( 5, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shortShiftRightJamM +/*---------------------------------------------------------------------------- +| Shifts the N-bit unsigned integer pointed to by `aPtr' right by the number +| of bits given in `count', where N = `size_words' * 32. The value of `count' +| must be in the range 1 to 31. If any nonzero bits are shifted off, they are +| "jammed" into the least-significant bit of the shifted value by setting the +| least-significant bit to 1. This shifted-and-jammed N-bit result is stored +| at the location pointed to by `zPtr'. Each of `aPtr' and `zPtr' points +| to a `size_words'-long array of 32-bit elements that concatenate in the +| platform's normal endian order to form an N-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_shortShiftRightJamM( + uint_fast8_t, const uint32_t *, uint_fast8_t, uint32_t * ); +#endif + +#ifndef softfloat_shortShiftRightJam160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shortShiftRightJamM' with +| `size_words' = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_shortShiftRightJam160M( aPtr, count, zPtr ) softfloat_shortShiftRightJamM( 5, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shiftRightM +/*---------------------------------------------------------------------------- +| Shifts the N-bit unsigned integer pointed to by `aPtr' right by the number +| of bits given in `count', where N = `size_words' * 32. The value of `count' +| must not be zero. Any nonzero bits shifted off are lost. The shifted +| N-bit result is stored at the location pointed to by `zPtr'. Each of `aPtr' +| and `zPtr' points to a `size_words'-long array of 32-bit elements that +| concatenate in the platform's normal endian order to form an N-bit integer. +| The value of `count' can be arbitrarily large. In particular, if `count' +| is greater than N, the stored result will be 0. +*----------------------------------------------------------------------------*/ +void + softfloat_shiftRightM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint32_t count, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_shiftRight96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shiftRightM' with +| `size_words' = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_shiftRight96M( aPtr, count, zPtr ) softfloat_shiftRightM( 3, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shiftRightJamM +/*---------------------------------------------------------------------------- +| Shifts the N-bit unsigned integer pointed to by `aPtr' right by the number +| of bits given in `count', where N = `size_words' * 32. The value of `count' +| must not be zero. If any nonzero bits are shifted off, they are "jammed" +| into the least-significant bit of the shifted value by setting the least- +| significant bit to 1. This shifted-and-jammed N-bit result is stored +| at the location pointed to by `zPtr'. Each of `aPtr' and `zPtr' points +| to a `size_words'-long array of 32-bit elements that concatenate in the +| platform's normal endian order to form an N-bit integer. +| The value of `count' can be arbitrarily large. In particular, if `count' +| is greater than N, the stored result will be either 0 or 1, depending on +| whether the original N bits are all zeros. +*----------------------------------------------------------------------------*/ +void + softfloat_shiftRightJamM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint32_t count, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_shiftRightJam96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shiftRightJamM' with +| `size_words' = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_shiftRightJam96M( aPtr, count, zPtr ) softfloat_shiftRightJamM( 3, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shiftRightJam128M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shiftRightJamM' with +| `size_words' = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_shiftRightJam128M( aPtr, count, zPtr ) softfloat_shiftRightJamM( 4, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_shiftRightJam160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_shiftRightJamM' with +| `size_words' = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_shiftRightJam160M( aPtr, count, zPtr ) softfloat_shiftRightJamM( 5, aPtr, count, zPtr ) +#endif + +#ifndef softfloat_addM +/*---------------------------------------------------------------------------- +| Adds the two N-bit integers pointed to by `aPtr' and `bPtr', where N = +| `size_words' * 32. The addition is modulo 2^N, so any carry out is lost. +| The N-bit sum is stored at the location pointed to by `zPtr'. Each of +| `aPtr', `bPtr', and `zPtr' points to a `size_words'-long array of 32-bit +| elements that concatenate in the platform's normal endian order to form an +| N-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_addM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_add96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_addM' with `size_words' +| = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_add96M( aPtr, bPtr, zPtr ) softfloat_addM( 3, aPtr, bPtr, zPtr ) +#endif + +#ifndef softfloat_add128M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_addM' with `size_words' +| = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_add128M( aPtr, bPtr, zPtr ) softfloat_addM( 4, aPtr, bPtr, zPtr ) +#endif + +#ifndef softfloat_add160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_addM' with `size_words' +| = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_add160M( aPtr, bPtr, zPtr ) softfloat_addM( 5, aPtr, bPtr, zPtr ) +#endif + +#ifndef softfloat_addCarryM +/*---------------------------------------------------------------------------- +| Adds the two N-bit unsigned integers pointed to by `aPtr' and `bPtr', where +| N = `size_words' * 32, plus `carry', which must be either 0 or 1. The N-bit +| sum (modulo 2^N) is stored at the location pointed to by `zPtr', and any +| carry out is returned as the result. Each of `aPtr', `bPtr', and `zPtr' +| points to a `size_words'-long array of 32-bit elements that concatenate in +| the platform's normal endian order to form an N-bit integer. +*----------------------------------------------------------------------------*/ +uint_fast8_t + softfloat_addCarryM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint_fast8_t carry, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_addComplCarryM +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_addCarryM', except that +| the value of the unsigned integer pointed to by `bPtr' is bit-wise completed +| before the addition. +*----------------------------------------------------------------------------*/ +uint_fast8_t + softfloat_addComplCarryM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint_fast8_t carry, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_addComplCarry96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_addComplCarryM' with +| `size_words' = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_addComplCarry96M( aPtr, bPtr, carry, zPtr ) softfloat_addComplCarryM( 3, aPtr, bPtr, carry, zPtr ) +#endif + +#ifndef softfloat_negXM +/*---------------------------------------------------------------------------- +| Replaces the N-bit unsigned integer pointed to by `zPtr' by the +| 2s-complement of itself, where N = `size_words' * 32. Argument `zPtr' +| points to a `size_words'-long array of 32-bit elements that concatenate in +| the platform's normal endian order to form an N-bit integer. +*----------------------------------------------------------------------------*/ +void softfloat_negXM( uint_fast8_t size_words, uint32_t *zPtr ); +#endif + +#ifndef softfloat_negX96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_negXM' with `size_words' +| = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_negX96M( zPtr ) softfloat_negXM( 3, zPtr ) +#endif + +#ifndef softfloat_negX128M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_negXM' with `size_words' +| = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_negX128M( zPtr ) softfloat_negXM( 4, zPtr ) +#endif + +#ifndef softfloat_negX160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_negXM' with `size_words' +| = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_negX160M( zPtr ) softfloat_negXM( 5, zPtr ) +#endif + +#ifndef softfloat_negX256M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_negXM' with `size_words' +| = 8 (N = 256). +*----------------------------------------------------------------------------*/ +#define softfloat_negX256M( zPtr ) softfloat_negXM( 8, zPtr ) +#endif + +#ifndef softfloat_sub1XM +/*---------------------------------------------------------------------------- +| Subtracts 1 from the N-bit integer pointed to by `zPtr', where N = +| `size_words' * 32. The subtraction is modulo 2^N, so any borrow out (carry +| out) is lost. Argument `zPtr' points to a `size_words'-long array of 32-bit +| elements that concatenate in the platform's normal endian order to form an +| N-bit integer. +*----------------------------------------------------------------------------*/ +void softfloat_sub1XM( uint_fast8_t size_words, uint32_t *zPtr ); +#endif + +#ifndef softfloat_sub1X96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_sub1XM' with `size_words' +| = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_sub1X96M( zPtr ) softfloat_sub1XM( 3, zPtr ) +#endif + +#ifndef softfloat_sub1X160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_sub1XM' with `size_words' +| = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_sub1X160M( zPtr ) softfloat_sub1XM( 5, zPtr ) +#endif + +#ifndef softfloat_subM +/*---------------------------------------------------------------------------- +| Subtracts the two N-bit integers pointed to by `aPtr' and `bPtr', where N = +| `size_words' * 32. The subtraction is modulo 2^N, so any borrow out (carry +| out) is lost. The N-bit difference is stored at the location pointed to by +| `zPtr'. Each of `aPtr', `bPtr', and `zPtr' points to a `size_words'-long +| array of 32-bit elements that concatenate in the platform's normal endian +| order to form an N-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_subM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_sub96M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_subM' with `size_words' +| = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_sub96M( aPtr, bPtr, zPtr ) softfloat_subM( 3, aPtr, bPtr, zPtr ) +#endif + +#ifndef softfloat_sub128M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_subM' with `size_words' +| = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_sub128M( aPtr, bPtr, zPtr ) softfloat_subM( 4, aPtr, bPtr, zPtr ) +#endif + +#ifndef softfloat_sub160M +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_subM' with `size_words' +| = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_sub160M( aPtr, bPtr, zPtr ) softfloat_subM( 5, aPtr, bPtr, zPtr ) +#endif + +#ifndef softfloat_mul64To128M +/*---------------------------------------------------------------------------- +| Multiplies `a' and `b' and stores the 128-bit product at the location +| pointed to by `zPtr'. Argument `zPtr' points to an array of four 32-bit +| elements that concatenate in the platform's normal endian order to form a +| 128-bit integer. +*----------------------------------------------------------------------------*/ +void softfloat_mul64To128M( uint64_t a, uint64_t b, uint32_t *zPtr ); +#endif + +#ifndef softfloat_mul128MTo256M +/*---------------------------------------------------------------------------- +| Multiplies the two 128-bit unsigned integers pointed to by `aPtr' and +| `bPtr', and stores the 256-bit product at the location pointed to by `zPtr'. +| Each of `aPtr' and `bPtr' points to an array of four 32-bit elements that +| concatenate in the platform's normal endian order to form a 128-bit integer. +| Argument `zPtr' points to an array of eight 32-bit elements that concatenate +| to form a 256-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_mul128MTo256M( + const uint32_t *aPtr, const uint32_t *bPtr, uint32_t *zPtr ); +#endif + +#ifndef softfloat_remStepMBy32 +/*---------------------------------------------------------------------------- +| Performs a "remainder reduction step" as follows: Arguments `remPtr' and +| `bPtr' both point to N-bit unsigned integers, where N = `size_words' * 32. +| Defining R and B as the values of those integers, the expression (R<<`count') +| - B * q is computed modulo 2^N, and the N-bit result is stored at the +| location pointed to by `zPtr'. Each of `remPtr', `bPtr', and `zPtr' points +| to a `size_words'-long array of 32-bit elements that concatenate in the +| platform's normal endian order to form an N-bit integer. +*----------------------------------------------------------------------------*/ +void + softfloat_remStepMBy32( + uint_fast8_t size_words, + const uint32_t *remPtr, + uint_fast8_t count, + const uint32_t *bPtr, + uint32_t q, + uint32_t *zPtr + ); +#endif + +#ifndef softfloat_remStep96MBy32 +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_remStepMBy32' with +| `size_words' = 3 (N = 96). +*----------------------------------------------------------------------------*/ +#define softfloat_remStep96MBy32( remPtr, count, bPtr, q, zPtr ) softfloat_remStepMBy32( 3, remPtr, count, bPtr, q, zPtr ) +#endif + +#ifndef softfloat_remStep128MBy32 +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_remStepMBy32' with +| `size_words' = 4 (N = 128). +*----------------------------------------------------------------------------*/ +#define softfloat_remStep128MBy32( remPtr, count, bPtr, q, zPtr ) softfloat_remStepMBy32( 4, remPtr, count, bPtr, q, zPtr ) +#endif + +#ifndef softfloat_remStep160MBy32 +/*---------------------------------------------------------------------------- +| This function or macro is the same as `softfloat_remStepMBy32' with +| `size_words' = 5 (N = 160). +*----------------------------------------------------------------------------*/ +#define softfloat_remStep160MBy32( remPtr, count, bPtr, q, zPtr ) softfloat_remStepMBy32( 5, remPtr, count, bPtr, q, zPtr ) +#endif + +#endif + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/softfloat.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/softfloat.h new file mode 100644 index 0000000..2f3851f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/softfloat.h @@ -0,0 +1,323 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + + +/*============================================================================ +| Note: If SoftFloat is made available as a general library for programs to +| use, it is strongly recommended that a platform-specific version of this +| header, "softfloat.h", be created that folds in "softfloat_types.h" and that +| eliminates all dependencies on compile-time macros. +*============================================================================*/ + + +#ifndef softfloat_h +#define softfloat_h 1 + +#include +#include + +#include "softfloat_types.h" + +/*---------------------------------------------------------------------------- +| Software floating-point underflow tininess-detection mode. +*----------------------------------------------------------------------------*/ +extern uint_fast8_t softfloat_detectTininess; +enum { + softfloat_tininess_beforeRounding = 0, + softfloat_tininess_afterRounding = 1 +}; + +/*---------------------------------------------------------------------------- +| Software floating-point rounding mode. +*----------------------------------------------------------------------------*/ +extern uint_fast8_t softfloat_roundingMode; +enum { + softfloat_round_near_even = 0, + softfloat_round_minMag = 1, + softfloat_round_min = 2, + softfloat_round_max = 3, + softfloat_round_near_maxMag = 4 +}; + +/*---------------------------------------------------------------------------- +| Software floating-point exception flags. +*----------------------------------------------------------------------------*/ +extern uint_fast8_t softfloat_exceptionFlags; +enum { + softfloat_flag_inexact = 1, + softfloat_flag_underflow = 2, + softfloat_flag_overflow = 4, + softfloat_flag_infinite = 8, + softfloat_flag_invalid = 16 +}; + +/*---------------------------------------------------------------------------- +| Routine to raise any or all of the software floating-point exception flags. +*----------------------------------------------------------------------------*/ +void softfloat_raiseFlags( uint_fast8_t ); + +/*---------------------------------------------------------------------------- +| Integer-to-floating-point conversion routines. +*----------------------------------------------------------------------------*/ +float32_t ui32_to_f32( uint32_t ); +float64_t ui32_to_f64( uint32_t ); +#ifdef SOFTFLOAT_FAST_INT64 +extFloat80_t ui32_to_extF80( uint32_t ); +float128_t ui32_to_f128( uint32_t ); +#endif +void ui32_to_extF80M( uint32_t, extFloat80_t * ); +void ui32_to_f128M( uint32_t, float128_t * ); +float32_t ui64_to_f32( uint64_t ); +float64_t ui64_to_f64( uint64_t ); +#ifdef SOFTFLOAT_FAST_INT64 +extFloat80_t ui64_to_extF80( uint64_t ); +float128_t ui64_to_f128( uint64_t ); +#endif +void ui64_to_extF80M( uint64_t, extFloat80_t * ); +void ui64_to_f128M( uint64_t, float128_t * ); +float32_t i32_to_f32( int32_t ); +float64_t i32_to_f64( int32_t ); +#ifdef SOFTFLOAT_FAST_INT64 +extFloat80_t i32_to_extF80( int32_t ); +float128_t i32_to_f128( int32_t ); +#endif +void i32_to_extF80M( int32_t, extFloat80_t * ); +void i32_to_f128M( int32_t, float128_t * ); +float32_t i64_to_f32( int64_t ); +float64_t i64_to_f64( int64_t ); +#ifdef SOFTFLOAT_FAST_INT64 +extFloat80_t i64_to_extF80( int64_t ); +float128_t i64_to_f128( int64_t ); +#endif +void i64_to_extF80M( int64_t, extFloat80_t * ); +void i64_to_f128M( int64_t, float128_t * ); + +/*---------------------------------------------------------------------------- +| 32-bit (single-precision) floating-point operations. +*----------------------------------------------------------------------------*/ +uint_fast32_t f32_to_ui32( float32_t, uint_fast8_t, bool ); +uint_fast64_t f32_to_ui64( float32_t, uint_fast8_t, bool ); +int_fast32_t f32_to_i32( float32_t, uint_fast8_t, bool ); +int_fast64_t f32_to_i64( float32_t, uint_fast8_t, bool ); +uint_fast32_t f32_to_ui32_r_minMag( float32_t, bool ); +uint_fast64_t f32_to_ui64_r_minMag( float32_t, bool ); +int_fast32_t f32_to_i32_r_minMag( float32_t, bool ); +int_fast64_t f32_to_i64_r_minMag( float32_t, bool ); +float64_t f32_to_f64( float32_t ); +#ifdef SOFTFLOAT_FAST_INT64 +extFloat80_t f32_to_extF80( float32_t ); +float128_t f32_to_f128( float32_t ); +#endif +void f32_to_extF80M( float32_t, extFloat80_t * ); +void f32_to_f128M( float32_t, float128_t * ); +float32_t f32_roundToInt( float32_t, uint_fast8_t, bool ); +float32_t f32_add( float32_t, float32_t ); +float32_t f32_sub( float32_t, float32_t ); +float32_t f32_mul( float32_t, float32_t ); +float32_t f32_mulAdd( float32_t, float32_t, float32_t ); +float32_t f32_div( float32_t, float32_t ); +float32_t f32_rem( float32_t, float32_t ); +float32_t f32_sqrt( float32_t ); +bool f32_eq( float32_t, float32_t ); +bool f32_le( float32_t, float32_t ); +bool f32_lt( float32_t, float32_t ); +bool f32_eq_signaling( float32_t, float32_t ); +bool f32_le_quiet( float32_t, float32_t ); +bool f32_lt_quiet( float32_t, float32_t ); +bool f32_isSignalingNaN( float32_t ); + +/*---------------------------------------------------------------------------- +| 64-bit (double-precision) floating-point operations. +*----------------------------------------------------------------------------*/ +uint_fast32_t f64_to_ui32( float64_t, uint_fast8_t, bool ); +uint_fast64_t f64_to_ui64( float64_t, uint_fast8_t, bool ); +int_fast32_t f64_to_i32( float64_t, uint_fast8_t, bool ); +int_fast64_t f64_to_i64( float64_t, uint_fast8_t, bool ); +uint_fast32_t f64_to_ui32_r_minMag( float64_t, bool ); +uint_fast64_t f64_to_ui64_r_minMag( float64_t, bool ); +int_fast32_t f64_to_i32_r_minMag( float64_t, bool ); +int_fast64_t f64_to_i64_r_minMag( float64_t, bool ); +float32_t f64_to_f32( float64_t ); +#ifdef SOFTFLOAT_FAST_INT64 +extFloat80_t f64_to_extF80( float64_t ); +float128_t f64_to_f128( float64_t ); +#endif +void f64_to_extF80M( float64_t, extFloat80_t * ); +void f64_to_f128M( float64_t, float128_t * ); +float64_t f64_roundToInt( float64_t, uint_fast8_t, bool ); +float64_t f64_add( float64_t, float64_t ); +float64_t f64_sub( float64_t, float64_t ); +float64_t f64_mul( float64_t, float64_t ); +float64_t f64_mulAdd( float64_t, float64_t, float64_t ); +float64_t f64_div( float64_t, float64_t ); +float64_t f64_rem( float64_t, float64_t ); +float64_t f64_sqrt( float64_t ); +bool f64_eq( float64_t, float64_t ); +bool f64_le( float64_t, float64_t ); +bool f64_lt( float64_t, float64_t ); +bool f64_eq_signaling( float64_t, float64_t ); +bool f64_le_quiet( float64_t, float64_t ); +bool f64_lt_quiet( float64_t, float64_t ); +bool f64_isSignalingNaN( float64_t ); + +/*---------------------------------------------------------------------------- +| Rounding precision for 80-bit extended double-precision floating-point. +| Valid values are 32, 64, and 80. +*----------------------------------------------------------------------------*/ +extern uint_fast8_t extF80_roundingPrecision; + +/*---------------------------------------------------------------------------- +| 80-bit extended double-precision floating-point operations. +*----------------------------------------------------------------------------*/ +#ifdef SOFTFLOAT_FAST_INT64 +uint_fast32_t extF80_to_ui32( extFloat80_t, uint_fast8_t, bool ); +uint_fast64_t extF80_to_ui64( extFloat80_t, uint_fast8_t, bool ); +int_fast32_t extF80_to_i32( extFloat80_t, uint_fast8_t, bool ); +int_fast64_t extF80_to_i64( extFloat80_t, uint_fast8_t, bool ); +uint_fast32_t extF80_to_ui32_r_minMag( extFloat80_t, bool ); +uint_fast64_t extF80_to_ui64_r_minMag( extFloat80_t, bool ); +int_fast32_t extF80_to_i32_r_minMag( extFloat80_t, bool ); +int_fast64_t extF80_to_i64_r_minMag( extFloat80_t, bool ); +float32_t extF80_to_f32( extFloat80_t ); +float64_t extF80_to_f64( extFloat80_t ); +float128_t extF80_to_f128( extFloat80_t ); +extFloat80_t extF80_roundToInt( extFloat80_t, uint_fast8_t, bool ); +extFloat80_t extF80_add( extFloat80_t, extFloat80_t ); +extFloat80_t extF80_sub( extFloat80_t, extFloat80_t ); +extFloat80_t extF80_mul( extFloat80_t, extFloat80_t ); +extFloat80_t extF80_div( extFloat80_t, extFloat80_t ); +extFloat80_t extF80_rem( extFloat80_t, extFloat80_t ); +extFloat80_t extF80_sqrt( extFloat80_t ); +bool extF80_eq( extFloat80_t, extFloat80_t ); +bool extF80_le( extFloat80_t, extFloat80_t ); +bool extF80_lt( extFloat80_t, extFloat80_t ); +bool extF80_eq_signaling( extFloat80_t, extFloat80_t ); +bool extF80_le_quiet( extFloat80_t, extFloat80_t ); +bool extF80_lt_quiet( extFloat80_t, extFloat80_t ); +bool extF80_isSignalingNaN( extFloat80_t ); +#endif +uint_fast32_t extF80M_to_ui32( const extFloat80_t *, uint_fast8_t, bool ); +uint_fast64_t extF80M_to_ui64( const extFloat80_t *, uint_fast8_t, bool ); +int_fast32_t extF80M_to_i32( const extFloat80_t *, uint_fast8_t, bool ); +int_fast64_t extF80M_to_i64( const extFloat80_t *, uint_fast8_t, bool ); +uint_fast32_t extF80M_to_ui32_r_minMag( const extFloat80_t *, bool ); +uint_fast64_t extF80M_to_ui64_r_minMag( const extFloat80_t *, bool ); +int_fast32_t extF80M_to_i32_r_minMag( const extFloat80_t *, bool ); +int_fast64_t extF80M_to_i64_r_minMag( const extFloat80_t *, bool ); +float32_t extF80M_to_f32( const extFloat80_t * ); +float64_t extF80M_to_f64( const extFloat80_t * ); +void extF80M_to_f128M( const extFloat80_t *, float128_t * ); +void + extF80M_roundToInt( + const extFloat80_t *, uint_fast8_t, bool, extFloat80_t * ); +void extF80M_add( const extFloat80_t *, const extFloat80_t *, extFloat80_t * ); +void extF80M_sub( const extFloat80_t *, const extFloat80_t *, extFloat80_t * ); +void extF80M_mul( const extFloat80_t *, const extFloat80_t *, extFloat80_t * ); +void extF80M_div( const extFloat80_t *, const extFloat80_t *, extFloat80_t * ); +void extF80M_rem( const extFloat80_t *, const extFloat80_t *, extFloat80_t * ); +void extF80M_sqrt( const extFloat80_t *, extFloat80_t * ); +bool extF80M_eq( const extFloat80_t *, const extFloat80_t * ); +bool extF80M_le( const extFloat80_t *, const extFloat80_t * ); +bool extF80M_lt( const extFloat80_t *, const extFloat80_t * ); +bool extF80M_eq_signaling( const extFloat80_t *, const extFloat80_t * ); +bool extF80M_le_quiet( const extFloat80_t *, const extFloat80_t * ); +bool extF80M_lt_quiet( const extFloat80_t *, const extFloat80_t * ); +bool extF80M_isSignalingNaN( const extFloat80_t * ); + +/*---------------------------------------------------------------------------- +| 128-bit (quadruple-precision) floating-point operations. +*----------------------------------------------------------------------------*/ +#ifdef SOFTFLOAT_FAST_INT64 +uint_fast32_t f128_to_ui32( float128_t, uint_fast8_t, bool ); +uint_fast64_t f128_to_ui64( float128_t, uint_fast8_t, bool ); +int_fast32_t f128_to_i32( float128_t, uint_fast8_t, bool ); +int_fast64_t f128_to_i64( float128_t, uint_fast8_t, bool ); +uint_fast32_t f128_to_ui32_r_minMag( float128_t, bool ); +uint_fast64_t f128_to_ui64_r_minMag( float128_t, bool ); +int_fast32_t f128_to_i32_r_minMag( float128_t, bool ); +int_fast64_t f128_to_i64_r_minMag( float128_t, bool ); +float32_t f128_to_f32( float128_t ); +float64_t f128_to_f64( float128_t ); +extFloat80_t f128_to_extF80( float128_t ); +float128_t f128_roundToInt( float128_t, uint_fast8_t, bool ); +float128_t f128_add( float128_t, float128_t ); +float128_t f128_sub( float128_t, float128_t ); +float128_t f128_mul( float128_t, float128_t ); +float128_t f128_mulAdd( float128_t, float128_t, float128_t ); +float128_t f128_div( float128_t, float128_t ); +float128_t f128_rem( float128_t, float128_t ); +float128_t f128_sqrt( float128_t ); +bool f128_eq( float128_t, float128_t ); +bool f128_le( float128_t, float128_t ); +bool f128_lt( float128_t, float128_t ); +bool f128_eq_signaling( float128_t, float128_t ); +bool f128_le_quiet( float128_t, float128_t ); +bool f128_lt_quiet( float128_t, float128_t ); +bool f128_isSignalingNaN( float128_t ); +#endif +uint_fast32_t f128M_to_ui32( const float128_t *, uint_fast8_t, bool ); +uint_fast64_t f128M_to_ui64( const float128_t *, uint_fast8_t, bool ); +int_fast32_t f128M_to_i32( const float128_t *, uint_fast8_t, bool ); +int_fast64_t f128M_to_i64( const float128_t *, uint_fast8_t, bool ); +uint_fast32_t f128M_to_ui32_r_minMag( const float128_t *, bool ); +uint_fast64_t f128M_to_ui64_r_minMag( const float128_t *, bool ); +int_fast32_t f128M_to_i32_r_minMag( const float128_t *, bool ); +int_fast64_t f128M_to_i64_r_minMag( const float128_t *, bool ); +float32_t f128M_to_f32( const float128_t * ); +float64_t f128M_to_f64( const float128_t * ); +void f128M_to_extF80M( const float128_t *, extFloat80_t * ); +void f128M_roundToInt( const float128_t *, uint_fast8_t, bool, float128_t * ); +void f128M_add( const float128_t *, const float128_t *, float128_t * ); +void f128M_sub( const float128_t *, const float128_t *, float128_t * ); +void f128M_mul( const float128_t *, const float128_t *, float128_t * ); +void + f128M_mulAdd( + const float128_t *, const float128_t *, const float128_t *, float128_t * + ); +void f128M_div( const float128_t *, const float128_t *, float128_t * ); +void f128M_rem( const float128_t *, const float128_t *, float128_t * ); +void f128M_sqrt( const float128_t *, float128_t * ); +bool f128M_eq( const float128_t *, const float128_t * ); +bool f128M_le( const float128_t *, const float128_t * ); +bool f128M_lt( const float128_t *, const float128_t * ); +bool f128M_eq_signaling( const float128_t *, const float128_t * ); +bool f128M_le_quiet( const float128_t *, const float128_t * ); +bool f128M_lt_quiet( const float128_t *, const float128_t * ); +bool f128M_isSignalingNaN( const float128_t * ); + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/softfloat_types.h b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/softfloat_types.h new file mode 100644 index 0000000..ec918ae --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/include/softfloat_types.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/*============================================================================ + +This C header file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#ifndef softfloat_types_h +#define softfloat_types_h 1 + +#include + +/*---------------------------------------------------------------------------- +| Types used to pass 32-bit, 64-bit, and 128-bit floating-point arguments and +| results to/from functions. These types must be exactly 32 bits, 64 bits, +| and 128 bits in size, respectively. Where a platform has "native" support +| for IEEE-Standard floating-point formats, the types below may, if desired, +| be defined as aliases for the native types (typically `float' and `double', +| and possibly `long double'). +*----------------------------------------------------------------------------*/ +typedef struct { uint32_t v; } float32_t; +typedef struct { uint64_t v; } float64_t; +typedef struct { uint64_t v[2]; } float128_t; + +/*---------------------------------------------------------------------------- +| The format of an 80-bit extended floating-point number in memory. This +| structure must contain a 16-bit field named `signExp' and a 64-bit field +| named `signif'. +*----------------------------------------------------------------------------*/ +#ifdef LITTLEENDIAN +struct extFloat80M { uint64_t signif; uint16_t signExp; }; +#else +struct extFloat80M { uint16_t signExp; uint64_t signif; }; +#endif + +/*---------------------------------------------------------------------------- +| The type used to pass 80-bit extended floating-point arguments and +| results to/from functions. This type must have size identical to +| `struct extFloat80M'. Type `extFloat80_t' can be defined as an alias for +| `struct extFloat80M'. Alternatively, if a platform has "native" support +| for IEEE-Standard 80-bit extended floating-point, it may be possible, +| if desired, to define `extFloat80_t' as an alias for the native type +| (presumably either `long double' or a nonstandard compiler-intrinsic type). +| In that case, the `signif' and `signExp' fields of `struct extFloat80M' +| must align exactly with the locations in memory of the sign, exponent, and +| significand of the native type. +*----------------------------------------------------------------------------*/ +typedef struct extFloat80M extFloat80_t; + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_add128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_add128.c new file mode 100644 index 0000000..fdfee4e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_add128.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_add128 + +struct uint128 + softfloat_add128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) +{ + struct uint128 z; + + z.v0 = a0 + b0; + z.v64 = a64 + b64 + (z.v0 < a0); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_add256M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_add256M.c new file mode 100644 index 0000000..bc2f9ae --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_add256M.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_add256M + +void + softfloat_add256M( + const uint64_t *aPtr, const uint64_t *bPtr, uint64_t *zPtr ) +{ + unsigned int index; + uint_fast8_t carry; + uint64_t wordA, wordZ; + + index = indexWordLo( 4 ); + carry = 0; + for (;;) { + wordA = aPtr[index]; + wordZ = wordA + bPtr[index] + carry; + zPtr[index] = wordZ; + if ( index == indexWordHi( 4 ) ) break; + carry = carry ? (wordZ <= wordA) : (wordZ < wordA); + index += wordIncr; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addCarryM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addCarryM.c new file mode 100644 index 0000000..de24208 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addCarryM.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_addCarryM + +uint_fast8_t + softfloat_addCarryM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint_fast8_t carry, + uint32_t *zPtr + ) +{ + unsigned int index, lastIndex; + uint32_t wordA, wordZ; + + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + for (;;) { + wordA = aPtr[index]; + wordZ = wordA + bPtr[index] + carry; + zPtr[index] = wordZ; + carry = carry ? (wordZ <= wordA) : (wordZ < wordA); + if ( index == lastIndex ) break; + index += wordIncr; + } + return carry; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addComplCarryM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addComplCarryM.c new file mode 100644 index 0000000..9b17275 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addComplCarryM.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_addComplCarryM + +uint_fast8_t + softfloat_addComplCarryM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint_fast8_t carry, + uint32_t *zPtr + ) +{ + unsigned int index, lastIndex; + uint32_t wordA, wordZ; + + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + for (;;) { + wordA = aPtr[index]; + wordZ = wordA + ~bPtr[index] + carry; + zPtr[index] = wordZ; + carry = carry ? (wordZ <= wordA) : (wordZ < wordA); + if ( index == lastIndex ) break; + index += wordIncr; + } + return carry; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addExtF80M.c new file mode 100644 index 0000000..16c5944 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addExtF80M.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +void + softfloat_addExtF80M( + const struct extFloat80M *aSPtr, + const struct extFloat80M *bSPtr, + struct extFloat80M *zSPtr, + bool negateB + ) +{ + uint32_t uiA64; + int32_t expA; + uint32_t uiB64; + int32_t expB; + uint32_t uiZ64; + bool signZ, signB; + const struct extFloat80M *tempSPtr; + uint64_t sigZ, sigB; + void + (*roundPackRoutinePtr)( + bool, int32_t, uint32_t *, uint_fast8_t, struct extFloat80M * ); + int32_t expDiff; + uint32_t extSigX[3], sigZExtra; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + expA = expExtF80UI64( uiA64 ); + uiB64 = bSPtr->signExp; + expB = expExtF80UI64( uiB64 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNExtF80M( aSPtr, bSPtr, zSPtr ) ) return; + uiZ64 = uiA64; + if ( expB == 0x7FFF ) { + uiZ64 = uiB64 ^ packToExtF80UI64( negateB, 0 ); + if ( (expA == 0x7FFF) && (uiZ64 != uiA64) ) { + softfloat_invalidExtF80M( zSPtr ); + return; + } + } + zSPtr->signExp = uiZ64; + zSPtr->signif = UINT64_C( 0x8000000000000000 ); + return; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signZ = signExtF80UI64( uiA64 ); + signB = signExtF80UI64( uiB64 ) ^ negateB; + negateB = (signZ != signB); + if ( expA < expB ) { + signZ = signB; + expA = expB; + expB = expExtF80UI64( uiA64 ); + tempSPtr = aSPtr; + aSPtr = bSPtr; + bSPtr = tempSPtr; + } + if ( ! expB ) { + expB = 1; + if ( ! expA ) expA = 1; + } + sigZ = aSPtr->signif; + sigB = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + roundPackRoutinePtr = softfloat_roundPackMToExtF80M; + expDiff = expA - expB; + if ( expDiff ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + extSigX[indexWord( 3, 2 )] = sigB>>32; + extSigX[indexWord( 3, 1 )] = sigB; + extSigX[indexWord( 3, 0 )] = 0; + softfloat_shiftRightJam96M( extSigX, expDiff, extSigX ); + sigB = + (uint64_t) extSigX[indexWord( 3, 2 )]<<32 + | extSigX[indexWord( 3, 1 )]; + if ( negateB ) { + sigZ -= sigB; + sigZExtra = extSigX[indexWordLo( 3 )]; + if ( sigZExtra ) { + --sigZ; + sigZExtra = -sigZExtra; + } + if ( ! (sigZ & UINT64_C( 0x8000000000000000 )) ) { + if ( sigZ & UINT64_C( 0x4000000000000000 ) ) { + --expA; + sigZ = sigZ<<1 | sigZExtra>>31; + sigZExtra <<= 1; + } else { + roundPackRoutinePtr = softfloat_normRoundPackMToExtF80M; + } + } + } else { + sigZ += sigB; + if ( sigZ & UINT64_C( 0x8000000000000000 ) ) goto sigZ; + sigZExtra = (uint32_t) sigZ<<31 | (extSigX[indexWordLo( 3 )] != 0); + goto completeNormAfterAdd; + } + } else { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + sigZExtra = 0; + if ( negateB ) { + if ( sigZ < sigB ) { + signZ = ! signZ; + sigZ = sigB - sigZ; + } else { + sigZ -= sigB; + if ( ! sigZ ) { + signZ = (softfloat_roundingMode == softfloat_round_min); + zSPtr->signExp = packToExtF80UI64( signZ, 0 ); + zSPtr->signif = 0; + return; + } + } + roundPackRoutinePtr = softfloat_normRoundPackMToExtF80M; + } else { + sigZ += sigB; + if ( sigZ < sigB ) { + sigZExtra = (uint32_t) sigZ<<31; + completeNormAfterAdd: + ++expA; + sigZ = UINT64_C( 0x8000000000000000 ) | sigZ>>1; + } else { + if ( ! (sigZ & UINT64_C( 0x8000000000000000 )) ) { + roundPackRoutinePtr = softfloat_normRoundPackMToExtF80M; + } + } + } + } + extSigX[indexWord( 3, 0 )] = sigZExtra; + sigZ: + extSigX[indexWord( 3, 2 )] = sigZ>>32; + extSigX[indexWord( 3, 1 )] = sigZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + roundPack: + (*roundPackRoutinePtr)( + signZ, expA, extSigX, extF80_roundingPrecision, zSPtr ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addF128M.c new file mode 100644 index 0000000..73c4fb3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addF128M.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +void + softfloat_addF128M( + const uint32_t *aWPtr, + const uint32_t *bWPtr, + uint32_t *zWPtr, + bool negateB + ) +{ + uint32_t uiA96; + int32_t expA; + uint32_t uiB96; + int32_t expB; + uint32_t uiZ96; + bool signZ, signB; + const uint32_t *tempPtr; + uint32_t sig96A, sig96B; + int32_t expDiff; + uint_fast8_t + (*addCarryMRoutinePtr)( + uint_fast8_t, + const uint32_t *, + const uint32_t *, + uint_fast8_t, + uint32_t * + ); + uint32_t extSigZ[5], wordSigZ; + uint_fast8_t carry; + void (*roundPackRoutinePtr)( bool, int32_t, uint32_t *, uint32_t * ); + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA96 = aWPtr[indexWordHi( 4 )]; + expA = expF128UI96( uiA96 ); + uiB96 = bWPtr[indexWordHi( 4 )]; + expB = expF128UI96( uiB96 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNF128M( aWPtr, bWPtr, zWPtr ) ) return; + uiZ96 = uiA96; + if ( expB == 0x7FFF ) { + uiZ96 = uiB96 ^ packToF128UI96( negateB, 0, 0 ); + if ( (expA == 0x7FFF) && (uiZ96 != uiA96) ) { + softfloat_invalidF128M( zWPtr ); + return; + } + } + zWPtr[indexWordHi( 4 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + return; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signZ = signF128UI96( uiA96 ); + signB = signF128UI96( uiB96 ) ^ negateB; + negateB = (signZ != signB); + if ( (uint32_t) (uiA96<<1) < (uint32_t) (uiB96<<1) ) { + signZ = signB; + expA = expB; + expB = expF128UI96( uiA96 ); + tempPtr = aWPtr; + aWPtr = bWPtr; + bWPtr = tempPtr; + uiA96 = uiB96; + uiB96 = bWPtr[indexWordHi( 4 )]; + } + sig96A = fracF128UI96( uiA96 ); + sig96B = fracF128UI96( uiB96 ); + if ( expA ) { + --expA; + sig96A |= 0x00010000; + if ( expB ) { + --expB; + sig96B |= 0x00010000; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + addCarryMRoutinePtr = + negateB ? softfloat_addComplCarryM : softfloat_addCarryM; + expDiff = expA - expB; + if ( expDiff ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + extSigZ[indexWordHi( 5 )] = sig96B; + extSigZ[indexWord( 5, 3 )] = bWPtr[indexWord( 4, 2 )]; + extSigZ[indexWord( 5, 2 )] = bWPtr[indexWord( 4, 1 )]; + extSigZ[indexWord( 5, 1 )] = bWPtr[indexWord( 4, 0 )]; + extSigZ[indexWord( 5, 0 )] = 0; + softfloat_shiftRightJam160M( extSigZ, expDiff, extSigZ ); + sig96B = extSigZ[indexWordHi( 5 )]; + carry = 0; + if ( negateB ) { + sig96B = ~sig96B; + wordSigZ = extSigZ[indexWordLo( 5 )]; + extSigZ[indexWordLo( 5 )] = -wordSigZ; + carry = ! wordSigZ; + } + carry = + (*addCarryMRoutinePtr)( + 3, + &aWPtr[indexMultiwordLo( 4, 3 )], + &extSigZ[indexMultiword( 5, 3, 1 )], + carry, + &extSigZ[indexMultiword( 5, 3, 1 )] + ); + wordSigZ = sig96A + sig96B + carry; + } else { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + extSigZ[indexWordLo( 5 )] = 0; + carry = + (*addCarryMRoutinePtr)( + 3, + &aWPtr[indexMultiwordLo( 4, 3 )], + &bWPtr[indexMultiwordLo( 4, 3 )], + negateB, + &extSigZ[indexMultiword( 5, 3, 1 )] + ); + if ( negateB ) { + wordSigZ = sig96A + ~sig96B + carry; + if ( wordSigZ & 0x80000000 ) { + signZ = ! signZ; + carry = + softfloat_addComplCarry96M( + &bWPtr[indexMultiwordLo( 4, 3 )], + &aWPtr[indexMultiwordLo( 4, 3 )], + 1, + &extSigZ[indexMultiword( 5, 3, 1 )] + ); + wordSigZ = sig96B + ~sig96A + carry; + } else { + if ( + ! wordSigZ && ! extSigZ[indexWord( 5, 3 )] + && ! ( extSigZ[indexWord( 5, 2 )] + | extSigZ[indexWord( 5, 1 )] + | extSigZ[indexWord( 5, 0 )] + ) + ) { + signZ = (softfloat_roundingMode == softfloat_round_min); + zWPtr[indexWordHi( 4 )] = packToF128UI96( signZ, 0, 0 ); + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + return; + } + } + } else { + wordSigZ = sig96A + sig96B + carry; + } + } + extSigZ[indexWordHi( 5 )] = wordSigZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + roundPackRoutinePtr = softfloat_normRoundPackMToF128M; + if ( 0x00010000 <= wordSigZ ) { + if ( 0x00020000 <= wordSigZ ) { + ++expA; + softfloat_shortShiftRightJam160M( extSigZ, 1, extSigZ ); + } + roundPackRoutinePtr = softfloat_roundPackMToF128M; + } + (*roundPackRoutinePtr)( signZ, expA, extSigZ, zWPtr ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addM.c new file mode 100644 index 0000000..2309a0b --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addM.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_addM + +void + softfloat_addM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint32_t *zPtr + ) +{ + unsigned int index, lastIndex; + uint_fast8_t carry; + uint32_t wordA, wordZ; + + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + carry = 0; + for (;;) { + wordA = aPtr[index]; + wordZ = wordA + bPtr[index] + carry; + zPtr[index] = wordZ; + if ( index == lastIndex ) break; + carry = carry ? (wordZ <= wordA) : (wordZ < wordA); + index += wordIncr; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsExtF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsExtF80.c new file mode 100644 index 0000000..3fb4b57 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsExtF80.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t + softfloat_addMagsExtF80( + uint_fast16_t uiA64, + uint_fast64_t uiA0, + uint_fast16_t uiB64, + uint_fast64_t uiB0, + bool signZ + ) +{ + int_fast32_t expA; + uint_fast64_t sigA; + int_fast32_t expB; + uint_fast64_t sigB; + int_fast32_t expDiff; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0, sigZ, sigZExtra; + struct exp32_sig64 normExpSig; + int_fast32_t expZ; + struct uint64_extra sig64Extra; + struct uint128 uiZ; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expA = expExtF80UI64( uiA64 ); + sigA = uiA0; + expB = expExtF80UI64( uiB64 ); + sigB = uiB0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expA - expB; + if ( ! expDiff ) { + if ( expA == 0x7FFF ) { + if ( (sigA | sigB) & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + goto propagateNaN; + } + uiZ64 = uiA64; + uiZ0 = uiA0; + goto uiZ; + } + sigZ = sigA + sigB; + sigZExtra = 0; + if ( ! expA ) { + normExpSig = softfloat_normSubnormalExtF80Sig( sigZ ); + expZ = normExpSig.exp + 1; + sigZ = normExpSig.sig; + goto roundAndPack; + } + expZ = expA; + goto shiftRight1; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expDiff < 0 ) { + if ( expB == 0x7FFF ) { + if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + uiZ64 = packToExtF80UI64( signZ, 0x7FFF ); + uiZ0 = uiB0; + goto uiZ; + } + expZ = expB; + if ( ! expA ) { + ++expDiff; + sigZExtra = 0; + if ( ! expDiff ) goto newlyAligned; + } + sig64Extra = softfloat_shiftRightJam64Extra( sigA, 0, -expDiff ); + sigA = sig64Extra.v; + sigZExtra = sig64Extra.extra; + } else { + if ( expA == 0x7FFF ) { + if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + uiZ64 = uiA64; + uiZ0 = uiA0; + goto uiZ; + } + expZ = expA; + if ( ! expB ) { + --expDiff; + sigZExtra = 0; + if ( ! expDiff ) goto newlyAligned; + } + sig64Extra = softfloat_shiftRightJam64Extra( sigB, 0, expDiff ); + sigB = sig64Extra.v; + sigZExtra = sig64Extra.extra; + } + newlyAligned: + sigZ = sigA + sigB; + if ( sigZ & UINT64_C( 0x8000000000000000 ) ) goto roundAndPack; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + shiftRight1: + sig64Extra = softfloat_shortShiftRightJam64Extra( sigZ, sigZExtra, 1 ); + sigZ = sig64Extra.v | UINT64_C( 0x8000000000000000 ); + sigZExtra = sig64Extra.extra; + ++expZ; + roundAndPack: + return + softfloat_roundPackToExtF80( + signZ, expZ, sigZ, sigZExtra, extF80_roundingPrecision ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNExtF80UI( uiA64, uiA0, uiB64, uiB0 ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF128.c new file mode 100644 index 0000000..7ceca87 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF128.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" + +float128_t + softfloat_addMagsF128( + uint_fast64_t uiA64, + uint_fast64_t uiA0, + uint_fast64_t uiB64, + uint_fast64_t uiB0, + bool signZ + ) +{ + int_fast32_t expA; + struct uint128 sigA; + int_fast32_t expB; + struct uint128 sigB; + int_fast32_t expDiff; + struct uint128 uiZ, sigZ; + int_fast32_t expZ; + uint_fast64_t sigZExtra; + struct uint128_extra sig128Extra; + union ui128_f128 uZ; + + expA = expF128UI64( uiA64 ); + sigA.v64 = fracF128UI64( uiA64 ); + sigA.v0 = uiA0; + expB = expF128UI64( uiB64 ); + sigB.v64 = fracF128UI64( uiB64 ); + sigB.v0 = uiB0; + expDiff = expA - expB; + if ( ! expDiff ) { + if ( expA == 0x7FFF ) { + if ( sigA.v64 | sigA.v0 | sigB.v64 | sigB.v0 ) goto propagateNaN; + uiZ.v64 = uiA64; + uiZ.v0 = uiA0; + goto uiZ; + } + sigZ = softfloat_add128( sigA.v64, sigA.v0, sigB.v64, sigB.v0 ); + if ( ! expA ) { + uiZ.v64 = packToF128UI64( signZ, 0, sigZ.v64 ); + uiZ.v0 = sigZ.v0; + goto uiZ; + } + expZ = expA; + sigZ.v64 |= UINT64_C( 0x0002000000000000 ); + sigZExtra = 0; + goto shiftRight1; + } + if ( expDiff < 0 ) { + if ( expB == 0x7FFF ) { + if ( sigB.v64 | sigB.v0 ) goto propagateNaN; + uiZ.v64 = packToF128UI64( signZ, 0x7FFF, 0 ); + uiZ.v0 = 0; + goto uiZ; + } + expZ = expB; + if ( expA ) { + sigA.v64 |= UINT64_C( 0x0001000000000000 ); + } else { + ++expDiff; + sigZExtra = 0; + if ( ! expDiff ) goto newlyAligned; + } + sig128Extra = + softfloat_shiftRightJam128Extra( sigA.v64, sigA.v0, 0, -expDiff ); + sigA = sig128Extra.v; + sigZExtra = sig128Extra.extra; + } else { + if ( expA == 0x7FFF ) { + if ( sigA.v64 | sigA.v0 ) goto propagateNaN; + uiZ.v64 = uiA64; + uiZ.v0 = uiA0; + goto uiZ; + } + expZ = expA; + if ( expB ) { + sigB.v64 |= UINT64_C( 0x0001000000000000 ); + } else { + --expDiff; + sigZExtra = 0; + if ( ! expDiff ) goto newlyAligned; + } + sig128Extra = + softfloat_shiftRightJam128Extra( sigB.v64, sigB.v0, 0, expDiff ); + sigB = sig128Extra.v; + sigZExtra = sig128Extra.extra; + } + newlyAligned: + sigZ = + softfloat_add128( + sigA.v64 | UINT64_C( 0x0001000000000000 ), + sigA.v0, + sigB.v64, + sigB.v0 + ); + --expZ; + if ( sigZ.v64 < UINT64_C( 0x0002000000000000 ) ) goto roundAndPack; + ++expZ; + shiftRight1: + sig128Extra = + softfloat_shortShiftRightJam128Extra( + sigZ.v64, sigZ.v0, sigZExtra, 1 ); + sigZ = sig128Extra.v; + sigZExtra = sig128Extra.extra; + roundAndPack: + return + softfloat_roundPackToF128( signZ, expZ, sigZ.v64, sigZ.v0, sigZExtra ); + propagateNaN: + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, uiB64, uiB0 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF32.c new file mode 100644 index 0000000..c63ccd3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF32.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" + +float32_t + softfloat_addMagsF32( uint_fast32_t uiA, uint_fast32_t uiB, bool signZ ) +{ + int_fast16_t expA; + uint_fast32_t sigA; + int_fast16_t expB; + uint_fast32_t sigB; + int_fast16_t expDiff; + uint_fast32_t uiZ; + int_fast16_t expZ; + uint_fast32_t sigZ; + union ui32_f32 uZ; + + expA = expF32UI( uiA ); + sigA = fracF32UI( uiA ); + expB = expF32UI( uiB ); + sigB = fracF32UI( uiB ); + expDiff = expA - expB; + sigA <<= 6; + sigB <<= 6; + if ( ! expDiff ) { + if ( expA == 0xFF ) { + if ( sigA | sigB ) goto propagateNaN; + uiZ = uiA; + goto uiZ; + } + if ( ! expA ) { + uiZ = packToF32UI( signZ, 0, (uiA + uiB) & 0x7FFFFFFF ); + goto uiZ; + } + expZ = expA; + sigZ = 0x40000000 + sigA + sigB; + } else { + if ( expDiff < 0 ) { + if ( expB == 0xFF ) { + if ( sigB ) goto propagateNaN; + uiZ = packToF32UI( signZ, 0xFF, 0 ); + goto uiZ; + } + expZ = expB; + sigA += expA ? 0x20000000 : sigA; + sigA = softfloat_shiftRightJam32( sigA, -expDiff ); + } else { + if ( expA == 0xFF ) { + if ( sigA ) goto propagateNaN; + uiZ = uiA; + goto uiZ; + } + expZ = expA; + sigB += expB ? 0x20000000 : sigB; + sigB = softfloat_shiftRightJam32( sigB, expDiff ); + } + sigZ = 0x20000000 + sigA + sigB; + if ( sigZ < 0x40000000 ) { + --expZ; + sigZ <<= 1; + } + } + return softfloat_roundPackToF32( signZ, expZ, sigZ ); + propagateNaN: + uiZ = softfloat_propagateNaNF32UI( uiA, uiB ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF64.c new file mode 100644 index 0000000..f651f7e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_addMagsF64.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" + +float64_t + softfloat_addMagsF64( uint_fast64_t uiA, uint_fast64_t uiB, bool signZ ) +{ + int_fast16_t expA; + uint_fast64_t sigA; + int_fast16_t expB; + uint_fast64_t sigB; + int_fast16_t expDiff; + uint_fast64_t uiZ; + int_fast16_t expZ; + uint_fast64_t sigZ; + union ui64_f64 uZ; + + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + expB = expF64UI( uiB ); + sigB = fracF64UI( uiB ); + expDiff = expA - expB; + sigA <<= 9; + sigB <<= 9; + if ( ! expDiff ) { + if ( expA == 0x7FF ) { + if ( sigA | sigB ) goto propagateNaN; + uiZ = uiA; + goto uiZ; + } + if ( ! expA ) { + uiZ = + packToF64UI( + signZ, 0, (uiA + uiB) & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ); + goto uiZ; + } + expZ = expA; + sigZ = UINT64_C( 0x4000000000000000 ) + sigA + sigB; + } else { + if ( expDiff < 0 ) { + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN; + uiZ = packToF64UI( signZ, 0x7FF, 0 ); + goto uiZ; + } + expZ = expB; + sigA += expA ? UINT64_C( 0x2000000000000000 ) : sigA; + sigA = softfloat_shiftRightJam64( sigA, -expDiff ); + } else { + if ( expA == 0x7FF ) { + if ( sigA ) goto propagateNaN; + uiZ = uiA; + goto uiZ; + } + expZ = expA; + sigB += expB ? UINT64_C( 0x2000000000000000 ) : sigB; + sigB = softfloat_shiftRightJam64( sigB, expDiff ); + } + sigZ = UINT64_C( 0x2000000000000000 ) + sigA + sigB; + if ( sigZ < UINT64_C( 0x4000000000000000 ) ) { + --expZ; + sigZ <<= 1; + } + } + return softfloat_roundPackToF64( signZ, expZ, sigZ ); + propagateNaN: + uiZ = softfloat_propagateNaNF64UI( uiA, uiB ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_approxRecip32_1.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_approxRecip32_1.c new file mode 100644 index 0000000..94d5532 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_approxRecip32_1.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_approxRecip32_1 + +uint32_t softfloat_approxRecip32_1( uint32_t a ) +{ + int index; + uint16_t eps; + static const uint16_t k0s[] = { + 0xFFC4, 0xF0BE, 0xE363, 0xD76F, 0xCCAD, 0xC2F0, 0xBA16, 0xB201, + 0xAA97, 0xA3C6, 0x9D7A, 0x97A6, 0x923C, 0x8D32, 0x887E, 0x8417 + }; + static const uint16_t k1s[] = { + 0xF0F1, 0xD62C, 0xBFA1, 0xAC77, 0x9C0A, 0x8DDB, 0x8185, 0x76BA, + 0x6D3B, 0x64D4, 0x5D5C, 0x56B1, 0x50B6, 0x4B55, 0x4679, 0x4211 + }; + uint16_t r0; + uint32_t delta0; + uint_fast32_t r; + uint32_t sqrDelta0; + + index = a>>27 & 0xF; + eps = (uint16_t) (a>>11); + r0 = k0s[index] - ((k1s[index] * (uint_fast32_t) eps)>>20); + delta0 = ~(uint_fast32_t) ((r0 * (uint_fast64_t) a)>>7); + r = ((uint_fast32_t) r0<<16) + ((r0 * (uint_fast64_t) delta0)>>24); + sqrDelta0 = ((uint_fast64_t) delta0 * delta0)>>32; + r += ((uint32_t) r * (uint_fast64_t) sqrDelta0)>>48; + return r; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_approxRecipSqrt32_1.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_approxRecipSqrt32_1.c new file mode 100644 index 0000000..b5cb328 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_approxRecipSqrt32_1.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_approxRecipSqrt32_1 + +uint32_t softfloat_approxRecipSqrt32_1( unsigned int oddExpA, uint32_t a ) +{ + int index; + uint16_t eps; + static const uint16_t k0s[] = { + 0xB4C9, 0xFFAB, 0xAA7D, 0xF11C, 0xA1C5, 0xE4C7, 0x9A43, 0xDA29, + 0x93B5, 0xD0E5, 0x8DED, 0xC8B7, 0x88C6, 0xC16D, 0x8424, 0xBAE1 + }; + static const uint16_t k1s[] = { + 0xA5A5, 0xEA42, 0x8C21, 0xC62D, 0x788F, 0xAA7F, 0x6928, 0x94B6, + 0x5CC7, 0x8335, 0x52A6, 0x74E2, 0x4A3E, 0x68FE, 0x432B, 0x5EFD + }; + uint16_t r0; + uint_fast32_t ESqrR0; + uint32_t delta0; + uint_fast32_t r; + uint32_t sqrDelta0; + + index = (a>>27 & 0xE) + oddExpA; + eps = (uint16_t) (a>>12); + r0 = k0s[index] - ((k1s[index] * (uint_fast32_t) eps)>>20); + ESqrR0 = (uint_fast32_t) r0 * r0; + if ( ! oddExpA ) ESqrR0 <<= 1; + delta0 = ~(uint_fast32_t) (((uint32_t) ESqrR0 * (uint_fast64_t) a)>>23); + r = ((uint_fast32_t) r0<<16) + ((r0 * (uint_fast64_t) delta0)>>25); + sqrDelta0 = ((uint_fast64_t) delta0 * delta0)>>32; + r += ((uint32_t) ((r>>1) + (r>>3) - ((uint_fast32_t) r0<<14)) + * (uint_fast64_t) sqrDelta0) + >>48; + if ( ! (r & 0x80000000) ) r = 0x80000000; + return r; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compare128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compare128M.c new file mode 100644 index 0000000..ac10845 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compare128M.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_compare128M + +int_fast8_t softfloat_compare128M( const uint32_t *aPtr, const uint32_t *bPtr ) +{ + unsigned int index, lastIndex; + uint32_t wordA, wordB; + + index = indexWordHi( 4 ); + lastIndex = indexWordLo( 4 ); + for (;;) { + wordA = aPtr[index]; + wordB = bPtr[index]; + if ( wordA != wordB ) return (wordA < wordB) ? -1 : 1; + if ( index == lastIndex ) break; + index -= wordIncr; + } + return 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compare96M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compare96M.c new file mode 100644 index 0000000..061c1e5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compare96M.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_compare96M + +int_fast8_t softfloat_compare96M( const uint32_t *aPtr, const uint32_t *bPtr ) +{ + unsigned int index, lastIndex; + uint32_t wordA, wordB; + + index = indexWordHi( 3 ); + lastIndex = indexWordLo( 3 ); + for (;;) { + wordA = aPtr[index]; + wordB = bPtr[index]; + if ( wordA != wordB ) return (wordA < wordB) ? -1 : 1; + if ( index == lastIndex ) break; + index -= wordIncr; + } + return 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compareNonnormExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compareNonnormExtF80M.c new file mode 100644 index 0000000..1f6b154 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_compareNonnormExtF80M.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat_types.h" + +int + softfloat_compareNonnormExtF80M( + const struct extFloat80M *aSPtr, const struct extFloat80M *bSPtr ) +{ + uint_fast16_t uiA64, uiB64; + uint64_t sigA; + bool signB; + uint64_t sigB; + int32_t expA, expB; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA64 = aSPtr->signExp; + uiB64 = bSPtr->signExp; + sigA = aSPtr->signif; + signB = signExtF80UI64( uiB64 ); + sigB = bSPtr->signif; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( (uiA64 ^ uiB64) & 0x8000 ) { + if ( ! (sigA | sigB) ) return 0; + goto resultFromSignB; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expA = expExtF80UI64( uiA64 ); + expB = expExtF80UI64( uiB64 ); + if ( expA == 0x7FFF ) { + if (expB == 0x7FFF) return 0; + signB = ! signB; + goto resultFromSignB; + } + if ( expB == 0x7FFF ) { + goto resultFromSignB; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) expA = 1; + if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) { + if ( sigA ) { + expA += softfloat_normExtF80SigM( &sigA ); + } else { + expA = -128; + } + } + if ( ! expB ) expB = 1; + if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) { + if ( sigB ) { + expB += softfloat_normExtF80SigM( &sigB ); + } else { + expB = -128; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( signB ) { + if ( expA < expB ) return 1; + if ( (expB < expA) || (sigB < sigA) ) return -1; + } else { + if ( expB < expA ) return 1; + if ( (expA < expB) || (sigA < sigB) ) return -1; + } + return (sigA != sigB); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + resultFromSignB: + return signB ? 1 : -1; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros32.c new file mode 100644 index 0000000..75735b2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros32.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_countLeadingZeros32 + +#define softfloat_countLeadingZeros32 softfloat_countLeadingZeros32 +#include "primitives.h" + +uint_fast8_t softfloat_countLeadingZeros32( uint32_t a ) +{ + uint_fast8_t count; + + count = 0; + if ( a < 0x10000 ) { + count = 16; + a <<= 16; + } + if ( a < 0x1000000 ) { + count += 8; + a <<= 8; + } + count += softfloat_countLeadingZeros8[a>>24]; + return count; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros64.c new file mode 100644 index 0000000..fe28125 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros64.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_countLeadingZeros64 + +#define softfloat_countLeadingZeros64 softfloat_countLeadingZeros64 +#include "primitives.h" + +uint_fast8_t softfloat_countLeadingZeros64( uint64_t a ) +{ + uint_fast8_t count; + uint32_t a32; + + count = 0; + a32 = a>>32; + if ( ! a32 ) { + count = 32; + a32 = a; + } + /*------------------------------------------------------------------------ + | From here, result is current count + count leading zeros of `a32'. + *------------------------------------------------------------------------*/ + if ( a32 < 0x10000 ) { + count += 16; + a32 <<= 16; + } + if ( a32 < 0x1000000 ) { + count += 8; + a32 <<= 8; + } + count += softfloat_countLeadingZeros8[a32>>24]; + return count; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros8.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros8.c new file mode 100644 index 0000000..3dd7719 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_countLeadingZeros8.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" + +const uint_least8_t softfloat_countLeadingZeros8[256] = { + 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_eq128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_eq128.c new file mode 100644 index 0000000..ee8d18a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_eq128.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" + +#ifndef softfloat_eq128 + +bool softfloat_eq128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) +{ + + return (a64 == b64) && (a0 == b0); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_invalidExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_invalidExtF80M.c new file mode 100644 index 0000000..8c09b2a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_invalidExtF80M.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include "platform.h" +#include "specialize.h" +#include "softfloat.h" + +void softfloat_invalidExtF80M( struct extFloat80M *zSPtr ) +{ + + softfloat_raiseFlags( softfloat_flag_invalid ); + zSPtr->signExp = defaultNaNExtF80UI64; + zSPtr->signif = defaultNaNExtF80UI0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_invalidF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_invalidF128M.c new file mode 100644 index 0000000..5c0b8e7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_invalidF128M.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitives.h" +#include "specialize.h" +#include "softfloat.h" + +void softfloat_invalidF128M( uint32_t *zWPtr ) +{ + + softfloat_raiseFlags( softfloat_flag_invalid ); + zWPtr[indexWord( 4, 3 )] = defaultNaNF128UI96; + zWPtr[indexWord( 4, 2 )] = defaultNaNF128UI64; + zWPtr[indexWord( 4, 1 )] = defaultNaNF128UI32; + zWPtr[indexWord( 4, 0 )] = defaultNaNF128UI0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_isNaNF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_isNaNF128M.c new file mode 100644 index 0000000..d98a9bb --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_isNaNF128M.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "primitives.h" + +/*---------------------------------------------------------------------------- +*----------------------------------------------------------------------------*/ +bool softfloat_isNaNF128M( const uint32_t *aWPtr ) +{ + uint32_t uiA96; + + uiA96 = aWPtr[indexWordHi( 4 )]; + if ( (uiA96 & 0x7FFF0000) != 0x7FFF0000 ) return false; + return + ((uiA96 & 0x0000FFFF) != 0) + || ((aWPtr[indexWord( 4, 2 )] | aWPtr[indexWord( 4, 1 )] + | aWPtr[indexWord( 4, 0 )]) + != 0); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_le128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_le128.c new file mode 100644 index 0000000..56e1d25 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_le128.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" + +#ifndef softfloat_le128 + +bool softfloat_le128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) +{ + + return (a64 < b64) || ((a64 == b64) && (a0 <= b0)); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_lt128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_lt128.c new file mode 100644 index 0000000..d976e17 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_lt128.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" + +#ifndef softfloat_lt128 + +bool softfloat_lt128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) +{ + + return (a64 < b64) || ((a64 == b64) && (a0 < b0)); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128By32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128By32.c new file mode 100644 index 0000000..baf67d2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128By32.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_mul128By32 + +struct uint128 softfloat_mul128By32( uint64_t a64, uint64_t a0, uint32_t b ) +{ + struct uint128 z; + uint_fast64_t mid; + uint_fast32_t carry; + + z.v0 = a0 * b; + mid = (uint_fast64_t) (uint32_t) (a0>>32) * b; + carry = (uint32_t) ((uint_fast32_t) (z.v0>>32) - (uint_fast32_t) mid); + z.v64 = a64 * b + (uint_fast32_t) ((mid + carry)>>32); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128MTo256M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128MTo256M.c new file mode 100644 index 0000000..b42b208 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128MTo256M.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_mul128MTo256M + +void + softfloat_mul128MTo256M( + const uint32_t *aPtr, const uint32_t *bPtr, uint32_t *zPtr ) +{ + uint32_t *lastZPtr, wordB; + uint64_t dwordProd; + uint32_t wordZ; + uint_fast8_t carry; + + bPtr += indexWordLo( 4 ); + lastZPtr = zPtr + indexMultiwordHi( 8, 5 ); + zPtr += indexMultiwordLo( 8, 5 ); + wordB = *bPtr; + dwordProd = (uint64_t) aPtr[indexWord( 4, 0 )] * wordB; + zPtr[indexWord( 5, 0 )] = dwordProd; + dwordProd = (uint64_t) aPtr[indexWord( 4, 1 )] * wordB + (dwordProd>>32); + zPtr[indexWord( 5, 1 )] = dwordProd; + dwordProd = (uint64_t) aPtr[indexWord( 4, 2 )] * wordB + (dwordProd>>32); + zPtr[indexWord( 5, 2 )] = dwordProd; + dwordProd = (uint64_t) aPtr[indexWord( 4, 3 )] * wordB + (dwordProd>>32); + zPtr[indexWord( 5, 3 )] = dwordProd; + zPtr[indexWord( 5, 4 )] = dwordProd>>32; + do { + bPtr += wordIncr; + zPtr += wordIncr; + wordB = *bPtr; + dwordProd = (uint64_t) aPtr[indexWord( 4, 0 )] * wordB; + wordZ = zPtr[indexWord( 5, 0 )] + (uint32_t) dwordProd; + zPtr[indexWord( 5, 0 )] = wordZ; + carry = (wordZ < (uint32_t) dwordProd); + dwordProd = + (uint64_t) aPtr[indexWord( 4, 1 )] * wordB + (dwordProd>>32); + wordZ = zPtr[indexWord( 5, 1 )] + (uint32_t) dwordProd + carry; + zPtr[indexWord( 5, 1 )] = wordZ; + carry = + carry ? (wordZ <= (uint32_t) dwordProd) + : (wordZ < (uint32_t) dwordProd); + dwordProd = + (uint64_t) aPtr[indexWord( 4, 2 )] * wordB + (dwordProd>>32); + wordZ = zPtr[indexWord( 5, 2 )] + (uint32_t) dwordProd + carry; + zPtr[indexWord( 5, 2 )] = wordZ; + carry = + carry ? (wordZ <= (uint32_t) dwordProd) + : (wordZ < (uint32_t) dwordProd); + dwordProd = + (uint64_t) aPtr[indexWord( 4, 3 )] * wordB + (dwordProd>>32); + wordZ = zPtr[indexWord( 5, 3 )] + (uint32_t) dwordProd + carry; + zPtr[indexWord( 5, 3 )] = wordZ; + carry = + carry ? (wordZ <= (uint32_t) dwordProd) + : (wordZ < (uint32_t) dwordProd); + zPtr[indexWord( 5, 4 )] = (dwordProd>>32) + carry; + } while ( zPtr != lastZPtr ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128To256M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128To256M.c new file mode 100644 index 0000000..f210937 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul128To256M.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_mul128To256M + +#define softfloat_mul128To256M softfloat_mul128To256M +#include "primitives.h" + +void + softfloat_mul128To256M( + uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0, uint64_t *zPtr ) +{ + struct uint128 p0, p64, p128; + uint_fast64_t z64, z128, z192; + + p0 = softfloat_mul64To128( a0, b0 ); + zPtr[indexWord( 4, 0 )] = p0.v0; + p64 = softfloat_mul64To128( a64, b0 ); + z64 = p64.v0 + p0.v64; + z128 = p64.v64 + (z64 < p64.v0); + p128 = softfloat_mul64To128( a64, b64 ); + z128 += p128.v0; + z192 = p128.v64 + (z128 < p128.v0); + p64 = softfloat_mul64To128( a0, b64 ); + z64 += p64.v0; + zPtr[indexWord( 4, 1 )] = z64; + p64.v64 += (z64 < p64.v0); + z128 += p64.v64; + zPtr[indexWord( 4, 2 )] = z128; + zPtr[indexWord( 4, 3 )] = z192 + (z128 < p64.v64); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64ByShifted32To128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64ByShifted32To128.c new file mode 100644 index 0000000..a16a609 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64ByShifted32To128.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_mul64ByShifted32To128 + +struct uint128 softfloat_mul64ByShifted32To128( uint64_t a, uint32_t b ) +{ + uint_fast64_t mid; + struct uint128 z; + + mid = (uint_fast64_t) (uint32_t) a * b; + z.v0 = mid<<32; + z.v64 = (uint_fast64_t) (uint32_t) (a>>32) * b + (mid>>32); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64To128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64To128.c new file mode 100644 index 0000000..d05c0b8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64To128.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_mul64To128 + +struct uint128 softfloat_mul64To128( uint64_t a, uint64_t b ) +{ + uint32_t a32, a0, b32, b0; + struct uint128 z; + uint64_t mid1, mid; + + a32 = a>>32; + a0 = a; + b32 = b>>32; + b0 = b; + z.v0 = (uint_fast64_t) a0 * b0; + mid1 = (uint_fast64_t) a32 * b0; + mid = mid1 + (uint_fast64_t) a0 * b32; + z.v64 = (uint_fast64_t) a32 * b32; + z.v64 += (uint_fast64_t) (mid < mid1)<<32 | mid>>32; + mid <<= 32; + z.v0 += mid; + z.v64 += (z.v0 < mid); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64To128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64To128M.c new file mode 100644 index 0000000..0ee736a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mul64To128M.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_mul64To128M + +void softfloat_mul64To128M( uint64_t a, uint64_t b, uint32_t *zPtr ) +{ + uint32_t a32, a0, b32, b0; + uint64_t z0, mid1, z64, mid; + + a32 = a>>32; + a0 = a; + b32 = b>>32; + b0 = b; + z0 = (uint64_t) a0 * b0; + mid1 = (uint64_t) a32 * b0; + mid = mid1 + (uint64_t) a0 * b32; + z64 = (uint64_t) a32 * b32; + z64 += (uint64_t) (mid < mid1)<<32 | mid>>32; + mid <<= 32; + z0 += mid; + zPtr[indexWord( 4, 1 )] = z0>>32; + zPtr[indexWord( 4, 0 )] = z0; + z64 += (z0 < mid); + zPtr[indexWord( 4, 3 )] = z64>>32; + zPtr[indexWord( 4, 2 )] = z64; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF128.c new file mode 100644 index 0000000..c9b5bbe --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF128.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t + softfloat_mulAddF128( + uint_fast64_t uiA64, + uint_fast64_t uiA0, + uint_fast64_t uiB64, + uint_fast64_t uiB0, + uint_fast64_t uiC64, + uint_fast64_t uiC0, + uint_fast8_t op + ) +{ + bool signA; + int_fast32_t expA; + struct uint128 sigA; + bool signB; + int_fast32_t expB; + struct uint128 sigB; + bool signC; + int_fast32_t expC; + struct uint128 sigC; + bool signZ; + uint_fast64_t magBits; + struct uint128 uiZ; + struct exp32_sig128 normExpSig; + int_fast32_t expZ; + uint64_t sig256Z[4]; + struct uint128 sigZ; + int_fast32_t shiftCount, expDiff; + struct uint128 x128; + uint64_t sig256C[4]; + static uint64_t zero256[4] = INIT_UINTM4( 0, 0, 0, 0 ); + uint_fast64_t sigZExtra, sig256Z0; + union ui128_f128 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signA = signF128UI64( uiA64 ); + expA = expF128UI64( uiA64 ); + sigA.v64 = fracF128UI64( uiA64 ); + sigA.v0 = uiA0; + signB = signF128UI64( uiB64 ); + expB = expF128UI64( uiB64 ); + sigB.v64 = fracF128UI64( uiB64 ); + sigB.v0 = uiB0; + signC = signF128UI64( uiC64 ) ^ (op == softfloat_mulAdd_subC); + expC = expF128UI64( uiC64 ); + sigC.v64 = fracF128UI64( uiC64 ); + sigC.v0 = uiC0; + signZ = signA ^ signB ^ (op == softfloat_mulAdd_subProd); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FFF ) { + if ( + (sigA.v64 | sigA.v0) || ((expB == 0x7FFF) && (sigB.v64 | sigB.v0)) + ) { + goto propagateNaN_ABC; + } + magBits = expB | sigB.v64 | sigB.v0; + goto infProdArg; + } + if ( expB == 0x7FFF ) { + if ( sigB.v64 | sigB.v0 ) goto propagateNaN_ABC; + magBits = expA | sigA.v64 | sigA.v0; + goto infProdArg; + } + if ( expC == 0x7FFF ) { + if ( sigC.v64 | sigC.v0 ) { + uiZ.v64 = 0; + uiZ.v0 = 0; + goto propagateNaN_ZC; + } + uiZ.v64 = uiC64; + uiZ.v0 = uiC0; + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! (sigA.v64 | sigA.v0) ) goto zeroProd; + normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) { + if ( ! (sigB.v64 | sigB.v0) ) goto zeroProd; + normExpSig = softfloat_normSubnormalF128Sig( sigB.v64, sigB.v0 ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x3FFE; + sigA.v64 |= UINT64_C( 0x0001000000000000 ); + sigB.v64 |= UINT64_C( 0x0001000000000000 ); + sigA = softfloat_shortShiftLeft128( sigA.v64, sigA.v0, 8 ); + sigB = softfloat_shortShiftLeft128( sigB.v64, sigB.v0, 15 ); + softfloat_mul128To256M( sigA.v64, sigA.v0, sigB.v64, sigB.v0, sig256Z ); + sigZ.v64 = sig256Z[indexWord( 4, 3 )]; + sigZ.v0 = sig256Z[indexWord( 4, 2 )]; + shiftCount = 0; + if ( ! (sigZ.v64 & UINT64_C( 0x0100000000000000 )) ) { + --expZ; + shiftCount = -1; + } + if ( ! expC ) { + if ( ! (sigC.v64 | sigC.v0) ) { + shiftCount += 8; + goto sigZ; + } + normExpSig = softfloat_normSubnormalF128Sig( sigC.v64, sigC.v0 ); + expC = normExpSig.exp; + sigC = normExpSig.sig; + } + sigC.v64 |= UINT64_C( 0x0001000000000000 ); + sigC = softfloat_shortShiftLeft128( sigC.v64, sigC.v0, 8 ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expZ - expC; + if ( expDiff < 0 ) { + expZ = expC; + if ( (signZ == signC) || (expDiff < -1) ) { + shiftCount -= expDiff; + if ( shiftCount ) { + sigZ = + softfloat_shiftRightJam128( + sigZ.v64, sigZ.v0, shiftCount ); + } + } else { + if ( ! shiftCount ) { + x128 = + softfloat_shortShiftRight128( + sig256Z[indexWord( 4, 1 )], sig256Z[indexWord( 4, 0 )], + 1 + ); + sig256Z[indexWord( 4, 1 )] = (sigZ.v0<<63) | x128.v64; + sig256Z[indexWord( 4, 0 )] = x128.v0; + sigZ = softfloat_shortShiftRight128( sigZ.v64, sigZ.v0, 1 ); + sig256Z[indexWord( 4, 3 )] = sigZ.v64; + sig256Z[indexWord( 4, 2 )] = sigZ.v0; + } + } + } else { + if ( shiftCount ) softfloat_add256M( sig256Z, sig256Z, sig256Z ); + if ( ! expDiff ) { + sigZ.v64 = sig256Z[indexWord( 4, 3 )]; + sigZ.v0 = sig256Z[indexWord( 4, 2 )]; + } else { + sig256C[indexWord( 4, 3 )] = sigC.v64; + sig256C[indexWord( 4, 2 )] = sigC.v0; + sig256C[indexWord( 4, 1 )] = 0; + sig256C[indexWord( 4, 0 )] = 0; + softfloat_shiftRightJam256M( sig256C, expDiff, sig256C ); + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + shiftCount = 8; + if ( signZ == signC ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( expDiff <= 0 ) { + sigZ = softfloat_add128( sigC.v64, sigC.v0, sigZ.v64, sigZ.v0 ); + } else { + softfloat_add256M( sig256Z, sig256C, sig256Z ); + sigZ.v64 = sig256Z[indexWord( 4, 3 )]; + sigZ.v0 = sig256Z[indexWord( 4, 2 )]; + } + if ( sigZ.v64 & UINT64_C( 0x0200000000000000 ) ) { + ++expZ; + shiftCount = 9; + } + } else { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( expDiff < 0 ) { + signZ = signC; + if ( expDiff < -1 ) { + sigZ = + softfloat_sub128( sigC.v64, sigC.v0, sigZ.v64, sigZ.v0 ); + sigZExtra = + sig256Z[indexWord( 4, 1 )] | sig256Z[indexWord( 4, 0 )]; + if ( sigZExtra ) { + sigZ = softfloat_sub128( sigZ.v64, sigZ.v0, 0, 1 ); + } + if ( ! (sigZ.v64 & UINT64_C( 0x0100000000000000 )) ) { + --expZ; + shiftCount = 7; + } + goto shiftRightRoundPack; + } else { + sig256C[indexWord( 4, 3 )] = sigC.v64; + sig256C[indexWord( 4, 2 )] = sigC.v0; + sig256C[indexWord( 4, 1 )] = 0; + sig256C[indexWord( 4, 0 )] = 0; + softfloat_sub256M( sig256C, sig256Z, sig256Z ); + } + } else if ( ! expDiff ) { + sigZ = softfloat_sub128( sigZ.v64, sigZ.v0, sigC.v64, sigC.v0 ); + if ( + ! (sigZ.v64 | sigZ.v0) && ! sig256Z[indexWord( 4, 1 )] + && ! sig256Z[indexWord( 4, 0 )] + ) { + goto completeCancellation; + } + sig256Z[indexWord( 4, 3 )] = sigZ.v64; + sig256Z[indexWord( 4, 2 )] = sigZ.v0; + if ( sigZ.v64 & UINT64_C( 0x8000000000000000 ) ) { + signZ ^= 1; + softfloat_sub256M( zero256, sig256Z, sig256Z ); + } + } else { + softfloat_sub256M( sig256Z, sig256C, sig256Z ); + if ( 1 < expDiff ) { + sigZ.v64 = sig256Z[indexWord( 4, 3 )]; + sigZ.v0 = sig256Z[indexWord( 4, 2 )]; + if ( ! (sigZ.v64 & UINT64_C( 0x0100000000000000 )) ) { + --expZ; + shiftCount = 7; + } + goto sigZ; + } + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + sigZ.v64 = sig256Z[indexWord( 4, 3 )]; + sigZ.v0 = sig256Z[indexWord( 4, 2 )]; + sigZExtra = sig256Z[indexWord( 4, 1 )]; + sig256Z0 = sig256Z[indexWord( 4, 0 )]; + if ( sigZ.v64 ) { + if ( sig256Z0 ) sigZExtra |= 1; + } else { + expZ -= 64; + sigZ.v64 = sigZ.v0; + sigZ.v0 = sigZExtra; + sigZExtra = sig256Z0; + if ( ! sigZ.v64 ) { + expZ -= 64; + sigZ.v64 = sigZ.v0; + sigZ.v0 = sigZExtra; + sigZExtra = 0; + if ( ! sigZ.v64 ) { + expZ -= 64; + sigZ.v64 = sigZ.v0; + sigZ.v0 = 0; + } + } + } + shiftCount = softfloat_countLeadingZeros64( sigZ.v64 ); + expZ += 7 - shiftCount; + shiftCount = 15 - shiftCount; + if ( 0 < shiftCount ) goto shiftRightRoundPack; + if ( shiftCount ) { + shiftCount = -shiftCount; + sigZ = + softfloat_shortShiftLeft128( sigZ.v64, sigZ.v0, shiftCount ); + x128 = softfloat_shortShiftLeft128( 0, sigZExtra, shiftCount ); + sigZ.v0 |= x128.v64; + sigZExtra = x128.v0; + } + goto roundPack; + } + sigZ: + sigZExtra = sig256Z[indexWord( 4, 1 )] | sig256Z[indexWord( 4, 0 )]; + shiftRightRoundPack: + sigZExtra = (uint64_t) (sigZ.v0<<(64 - shiftCount)) | (sigZExtra != 0); + sigZ = softfloat_shortShiftRight128( sigZ.v64, sigZ.v0, shiftCount ); + roundPack: + return + softfloat_roundPackToF128( + signZ, expZ - 1, sigZ.v64, sigZ.v0, sigZExtra ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN_ABC: + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, uiB64, uiB0 ); + goto propagateNaN_ZC; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infProdArg: + if ( magBits ) { + uiZ.v64 = packToF128UI64( signZ, 0x7FFF, 0 ); + uiZ.v0 = 0; + if ( expC != 0x7FFF ) goto uiZ; + if ( sigC.v64 | sigC.v0 ) goto propagateNaN_ZC; + if ( signZ == signC ) goto uiZ; + } + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ.v64 = defaultNaNF128UI64; + uiZ.v0 = defaultNaNF128UI0; + propagateNaN_ZC: + uiZ = softfloat_propagateNaNF128UI( uiZ.v64, uiZ.v0, uiC64, uiC0 ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zeroProd: + uiZ.v64 = uiC64; + uiZ.v0 = uiC0; + if ( ! (expC | sigC.v64 | sigC.v0) && (signZ != signC) ) { + completeCancellation: + uiZ.v64 = + packToF128UI64( + softfloat_roundingMode == softfloat_round_min, 0, 0 ); + uiZ.v0 = 0; + } + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF128M.c new file mode 100644 index 0000000..8ab8888 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF128M.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +void + softfloat_mulAddF128M( + const uint32_t *aWPtr, + const uint32_t *bWPtr, + const uint32_t *cWPtr, + uint32_t *zWPtr, + uint_fast8_t op + ) +{ + uint32_t uiA96; + int32_t expA; + uint32_t uiB96; + int32_t expB; + uint32_t uiC96; + bool signC; + int32_t expC; + bool signProd, prodIsInfinite; + uint32_t *ptr, uiZ96, sigA[4]; + uint_fast8_t shiftCount; + uint32_t sigX[5]; + int32_t expProd; + uint32_t sigProd[8], wordSig; + bool doSub; + uint_fast8_t + (*addCarryMRoutinePtr)( + uint_fast8_t, + const uint32_t *, + const uint32_t *, + uint_fast8_t, + uint32_t * + ); + int32_t expDiff; + bool signZ; + int32_t expZ; + uint32_t *extSigPtr; + uint_fast8_t carry; + void (*roundPackRoutinePtr)( bool, int32_t, uint32_t *, uint32_t * ); + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + uiA96 = aWPtr[indexWordHi( 4 )]; + expA = expF128UI96( uiA96 ); + uiB96 = bWPtr[indexWordHi( 4 )]; + expB = expF128UI96( uiB96 ); + uiC96 = cWPtr[indexWordHi( 4 )]; + signC = signF128UI96( uiC96 ) ^ (op == softfloat_mulAdd_subC); + expC = expF128UI96( uiC96 ); + signProd = + signF128UI96( uiA96 ) ^ signF128UI96( uiB96 ) + ^ (op == softfloat_mulAdd_subProd); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + prodIsInfinite = false; + if ( (expA == 0x7FFF) || (expB == 0x7FFF) ) { + if ( softfloat_tryPropagateNaNF128M( aWPtr, bWPtr, zWPtr ) ) { + goto propagateNaN_ZC; + } + ptr = (uint32_t *) aWPtr; + if ( ! (uint32_t) (uiA96<<1) ) goto possibleInvalidProd; + if ( ! (uint32_t) (uiB96<<1) ) { + ptr = (uint32_t *) bWPtr; + possibleInvalidProd: + if ( + ! (ptr[indexWord( 4, 2 )] | ptr[indexWord( 4, 1 )] + | ptr[indexWord( 4, 0 )]) + ) { + goto invalid; + } + } + prodIsInfinite = true; + } + if ( expC == 0x7FFF ) { + if ( + fracF128UI96( uiC96 ) + || (cWPtr[indexWord( 4, 2 )] | cWPtr[indexWord( 4, 1 )] + | cWPtr[indexWord( 4, 0 )]) + ) { + zWPtr[indexWordHi( 4 )] = 0; + goto propagateNaN_ZC; + } + if ( prodIsInfinite && (signProd != signC) ) goto invalid; + goto copyC; + } + if ( prodIsInfinite ) { + uiZ96 = packToF128UI96( signProd, 0x7FFF, 0 ); + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA ) { + sigA[indexWordHi( 4 )] = fracF128UI96( uiA96 ) | 0x00010000; + sigA[indexWord( 4, 2 )] = aWPtr[indexWord( 4, 2 )]; + sigA[indexWord( 4, 1 )] = aWPtr[indexWord( 4, 1 )]; + sigA[indexWord( 4, 0 )] = aWPtr[indexWord( 4, 0 )]; + } else { + expA = softfloat_shiftNormSigF128M( aWPtr, 0, sigA ); + if ( expA == -128 ) goto zeroProd; + } + if ( expB ) { + sigX[indexWordHi( 4 )] = fracF128UI96( uiB96 ) | 0x00010000; + sigX[indexWord( 4, 2 )] = bWPtr[indexWord( 4, 2 )]; + sigX[indexWord( 4, 1 )] = bWPtr[indexWord( 4, 1 )]; + sigX[indexWord( 4, 0 )] = bWPtr[indexWord( 4, 0 )]; + } else { + expB = softfloat_shiftNormSigF128M( bWPtr, 0, sigX ); + if ( expB == -128 ) goto zeroProd; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expProd = expA + expB - 0x3FF0; + softfloat_mul128MTo256M( sigA, sigX, sigProd ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + wordSig = fracF128UI96( uiC96 ); + if ( expC ) { + --expC; + wordSig |= 0x00010000; + } + sigX[indexWordHi( 5 )] = wordSig; + sigX[indexWord( 5, 3 )] = cWPtr[indexWord( 4, 2 )]; + sigX[indexWord( 5, 2 )] = cWPtr[indexWord( 4, 1 )]; + sigX[indexWord( 5, 1 )] = cWPtr[indexWord( 4, 0 )]; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + doSub = (signProd != signC); + addCarryMRoutinePtr = + doSub ? softfloat_addComplCarryM : softfloat_addCarryM; + expDiff = expProd - expC; + if ( expDiff <= 0 ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + signZ = signC; + expZ = expC; + if ( + sigProd[indexWord( 8, 2 )] + || (sigProd[indexWord( 8, 1 )] | sigProd[indexWord( 8, 0 )]) + ) { + sigProd[indexWord( 8, 3 )] |= 1; + } + extSigPtr = &sigProd[indexMultiwordHi( 8, 5 )]; + if ( expDiff ) { + softfloat_shiftRightJam160M( extSigPtr, -expDiff, extSigPtr ); + } + carry = 0; + if ( doSub ) { + wordSig = extSigPtr[indexWordLo( 5 )]; + extSigPtr[indexWordLo( 5 )] = -wordSig; + carry = ! wordSig; + } + (*addCarryMRoutinePtr)( + 4, + &sigX[indexMultiwordHi( 5, 4 )], + extSigPtr + indexMultiwordHi( 5, 4 ), + carry, + extSigPtr + indexMultiwordHi( 5, 4 ) + ); + wordSig = extSigPtr[indexWordHi( 5 )]; + if ( ! expZ ) { + if ( wordSig & 0x80000000 ) { + signZ = ! signZ; + softfloat_negX160M( extSigPtr ); + wordSig = extSigPtr[indexWordHi( 5 )]; + } + goto checkCancellation; + } + if ( wordSig < 0x00010000 ) { + --expZ; + softfloat_add160M( extSigPtr, extSigPtr, extSigPtr ); + goto roundPack; + } + goto extSigReady_noCancellation; + } else { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + signZ = signProd; + expZ = expProd; + sigX[indexWordLo( 5 )] = 0; + expDiff -= 128; + if ( 0 <= expDiff ) { + /*---------------------------------------------------------------- + *----------------------------------------------------------------*/ + if ( expDiff ) softfloat_shiftRightJam160M( sigX, expDiff, sigX ); + wordSig = sigX[indexWordLo( 5 )]; + carry = 0; + if ( doSub ) { + carry = ! wordSig; + wordSig = -wordSig; + } + carry = + (*addCarryMRoutinePtr)( + 4, + &sigProd[indexMultiwordLo( 8, 4 )], + &sigX[indexMultiwordHi( 5, 4 )], + carry, + &sigProd[indexMultiwordLo( 8, 4 )] + ); + sigProd[indexWord( 8, 2 )] |= wordSig; + ptr = &sigProd[indexWord( 8, 4 )]; + } else { + /*---------------------------------------------------------------- + *----------------------------------------------------------------*/ + shiftCount = expDiff & 31; + if ( shiftCount ) { + softfloat_shortShiftRight160M( sigX, shiftCount, sigX ); + } + expDiff >>= 5; + extSigPtr = + &sigProd[indexMultiwordLo( 8, 5 )] - wordIncr + + expDiff * -wordIncr; + carry = + (*addCarryMRoutinePtr)( 5, extSigPtr, sigX, doSub, extSigPtr ); + if ( expDiff == -4 ) { + /*------------------------------------------------------------ + *------------------------------------------------------------*/ + wordSig = sigProd[indexWordHi( 8 )]; + if ( wordSig & 0x80000000 ) { + signZ = ! signZ; + softfloat_negX256M( sigProd ); + wordSig = sigProd[indexWordHi( 8 )]; + } + /*------------------------------------------------------------ + *------------------------------------------------------------*/ + if ( wordSig ) goto expProdBigger_noWordShift; + wordSig = sigProd[indexWord( 8, 6 )]; + if ( 0x00040000 <= wordSig ) goto expProdBigger_noWordShift; + expZ -= 32; + extSigPtr = &sigProd[indexMultiwordHi( 8, 5 )] - wordIncr; + for (;;) { + if ( wordSig ) break; + wordSig = extSigPtr[indexWord( 5, 3 )]; + if ( 0x00040000 <= wordSig ) break; + expZ -= 32; + extSigPtr -= wordIncr; + if ( extSigPtr == &sigProd[indexMultiwordLo( 8, 5 )] ) { + goto checkCancellation; + } + } + /*------------------------------------------------------------ + *------------------------------------------------------------*/ + ptr = extSigPtr + indexWordLo( 5 ); + do { + ptr -= wordIncr; + if ( *ptr ) { + extSigPtr[indexWordLo( 5 )] |= 1; + break; + } + } while ( ptr != &sigProd[indexWordLo( 8 )] ); + wordSig = extSigPtr[indexWordHi( 5 )]; + goto extSigReady; + } + ptr = extSigPtr + indexWordHi( 5 ) + wordIncr; + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( carry != doSub ) { + if ( doSub ) { + do { + wordSig = *ptr; + *ptr = wordSig - 1; + ptr += wordIncr; + } while ( ! wordSig ); + } else { + do { + wordSig = *ptr + 1; + *ptr = wordSig; + ptr += wordIncr; + } while ( ! wordSig ); + } + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + expProdBigger_noWordShift: + if ( + sigProd[indexWord( 8, 2 )] + || (sigProd[indexWord( 8, 1 )] | sigProd[indexWord( 8, 0 )]) + ) { + sigProd[indexWord( 8, 3 )] |= 1; + } + extSigPtr = &sigProd[indexMultiwordHi( 8, 5 )]; + wordSig = extSigPtr[indexWordHi( 5 )]; + } + extSigReady: + roundPackRoutinePtr = softfloat_normRoundPackMToF128M; + if ( wordSig < 0x00010000 ) goto doRoundPack; + extSigReady_noCancellation: + if ( 0x00020000 <= wordSig ) { + ++expZ; + softfloat_shortShiftRightJam160M( extSigPtr, 1, extSigPtr ); + } + roundPack: + roundPackRoutinePtr = softfloat_roundPackMToF128M; + doRoundPack: + (*roundPackRoutinePtr)( signZ, expZ, extSigPtr, zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + invalid: + softfloat_invalidF128M( zWPtr ); + propagateNaN_ZC: + softfloat_propagateNaNF128M( zWPtr, cWPtr, zWPtr ); + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zeroProd: + if ( + ! (uint32_t) (uiC96<<1) && (signProd != signC) + && ! cWPtr[indexWord( 4, 2 )] + && ! (cWPtr[indexWord( 4, 1 )] | cWPtr[indexWord( 4, 0 )]) + ) { + goto completeCancellation; + } + copyC: + zWPtr[indexWordHi( 4 )] = uiC96; + zWPtr[indexWord( 4, 2 )] = cWPtr[indexWord( 4, 2 )]; + zWPtr[indexWord( 4, 1 )] = cWPtr[indexWord( 4, 1 )]; + zWPtr[indexWord( 4, 0 )] = cWPtr[indexWord( 4, 0 )]; + return; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + checkCancellation: + if ( + wordSig + || (extSigPtr[indexWord( 5, 3 )] | extSigPtr[indexWord( 5, 2 )]) + || (extSigPtr[indexWord( 5, 1 )] | extSigPtr[indexWord( 5, 0 )]) + ) { + goto extSigReady; + } + completeCancellation: + uiZ96 = + packToF128UI96( + (softfloat_roundingMode == softfloat_round_min), 0, 0 ); + uiZ: + zWPtr[indexWordHi( 4 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF32.c new file mode 100644 index 0000000..fa9a95a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_mulAddF32.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t + softfloat_mulAddF32( + uint_fast32_t uiA, uint_fast32_t uiB, uint_fast32_t uiC, uint_fast8_t op ) +{ + bool signA; + int_fast16_t expA; + uint_fast32_t sigA; + bool signB; + int_fast16_t expB; + uint_fast32_t sigB; + bool signC; + int_fast16_t expC; + uint_fast32_t sigC; + bool signProd; + uint_fast32_t magBits, uiZ; + struct exp16_sig32 normExpSig; + int_fast16_t expProd; + uint_fast64_t sigProd; + bool signZ; + int_fast16_t expZ; + uint_fast32_t sigZ; + int_fast16_t expDiff; + uint_fast64_t sig64Z, sig64C; + int_fast8_t shiftCount; + union ui32_f32 uZ; + + signA = signF32UI( uiA ); + expA = expF32UI( uiA ); + sigA = fracF32UI( uiA ); + signB = signF32UI( uiB ); + expB = expF32UI( uiB ); + sigB = fracF32UI( uiB ); + signC = signF32UI( uiC ) ^ (op == softfloat_mulAdd_subC); + expC = expF32UI( uiC ); + sigC = fracF32UI( uiC ); + signProd = signA ^ signB ^ (op == softfloat_mulAdd_subProd); + if ( expA == 0xFF ) { + if ( sigA || ((expB == 0xFF) && sigB) ) goto propagateNaN_ABC; + magBits = expB | sigB; + goto infProdArg; + } + if ( expB == 0xFF ) { + if ( sigB ) goto propagateNaN_ABC; + magBits = expA | sigA; + goto infProdArg; + } + if ( expC == 0xFF ) { + if ( sigC ) { + uiZ = 0; + goto propagateNaN_ZC; + } + uiZ = uiC; + goto uiZ; + } + if ( ! expA ) { + if ( ! sigA ) goto zeroProd; + normExpSig = softfloat_normSubnormalF32Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) { + if ( ! sigB ) goto zeroProd; + normExpSig = softfloat_normSubnormalF32Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + expProd = expA + expB - 0x7E; + sigA = (sigA | 0x00800000)<<7; + sigB = (sigB | 0x00800000)<<7; + sigProd = (uint_fast64_t) sigA * sigB; + if ( sigProd < UINT64_C( 0x2000000000000000 ) ) { + --expProd; + sigProd <<= 1; + } + signZ = signProd; + if ( ! expC ) { + if ( ! sigC ) { + expZ = expProd - 1; + sigZ = softfloat_shortShiftRightJam64( sigProd, 31 ); + goto roundPack; + } + normExpSig = softfloat_normSubnormalF32Sig( sigC ); + expC = normExpSig.exp; + sigC = normExpSig.sig; + } + sigC = (sigC | 0x00800000)<<6; + expDiff = expProd - expC; + if ( signProd == signC ) { + if ( expDiff <= 0 ) { + expZ = expC; + sigZ = sigC + softfloat_shiftRightJam64( sigProd, 32 - expDiff ); + } else { + expZ = expProd; + sig64Z = + sigProd + + softfloat_shiftRightJam64( + (uint_fast64_t) sigC<<32, expDiff ); + sigZ = softfloat_shortShiftRightJam64( sig64Z, 32 ); + } + if ( sigZ < 0x40000000 ) { + --expZ; + sigZ <<= 1; + } + } else { + sig64C = (uint_fast64_t) sigC<<32; + if ( expDiff < 0 ) { + signZ = signC; + expZ = expC; + sig64Z = sig64C - softfloat_shiftRightJam64( sigProd, -expDiff ); + } else if ( ! expDiff ) { + expZ = expProd; + sig64Z = sigProd - sig64C; + if ( ! sig64Z ) goto completeCancellation; + if ( sig64Z & UINT64_C( 0x8000000000000000 ) ) { + signZ ^= 1; + sig64Z = -sig64Z; + } + } else { + expZ = expProd; + sig64Z = sigProd - softfloat_shiftRightJam64( sig64C, expDiff ); + } + shiftCount = softfloat_countLeadingZeros64( sig64Z ) - 1; + expZ -= shiftCount; + shiftCount -= 32; + if ( shiftCount < 0 ) { + sigZ = softfloat_shortShiftRightJam64( sig64Z, -shiftCount ); + } else { + sigZ = (uint_fast32_t) sig64Z< +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +float64_t + softfloat_mulAddF64( + uint_fast64_t uiA, uint_fast64_t uiB, uint_fast64_t uiC, uint_fast8_t op ) +{ + bool signA; + int_fast16_t expA; + uint_fast64_t sigA; + bool signB; + int_fast16_t expB; + uint_fast64_t sigB; + bool signC; + int_fast16_t expC; + uint_fast64_t sigC; + bool signZ; + uint_fast64_t magBits, uiZ; + struct exp16_sig64 normExpSig; + int_fast16_t expZ; + struct uint128 sig128Z; + uint_fast64_t sigZ; + int_fast16_t expDiff; + struct uint128 sig128C; + int_fast8_t shiftCount; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signA = signF64UI( uiA ); + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + signB = signF64UI( uiB ); + expB = expF64UI( uiB ); + sigB = fracF64UI( uiB ); + signC = signF64UI( uiC ) ^ (op == softfloat_mulAdd_subC); + expC = expF64UI( uiC ); + sigC = fracF64UI( uiC ); + signZ = signA ^ signB ^ (op == softfloat_mulAdd_subProd); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FF ) { + if ( sigA || ((expB == 0x7FF) && sigB) ) goto propagateNaN_ABC; + magBits = expB | sigB; + goto infProdArg; + } + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN_ABC; + magBits = expA | sigA; + goto infProdArg; + } + if ( expC == 0x7FF ) { + if ( sigC ) { + uiZ = 0; + goto propagateNaN_ZC; + } + uiZ = uiC; + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! sigA ) goto zeroProd; + normExpSig = softfloat_normSubnormalF64Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) { + if ( ! sigB ) goto zeroProd; + normExpSig = softfloat_normSubnormalF64Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x3FE; + sigA = (sigA | UINT64_C( 0x0010000000000000 ))<<10; + sigB = (sigB | UINT64_C( 0x0010000000000000 ))<<10; + sig128Z = softfloat_mul64To128( sigA, sigB ); + if ( sig128Z.v64 < UINT64_C( 0x2000000000000000 ) ) { + --expZ; + sig128Z = + softfloat_add128( + sig128Z.v64, sig128Z.v0, sig128Z.v64, sig128Z.v0 ); + } + if ( ! expC ) { + if ( ! sigC ) { + --expZ; + sigZ = sig128Z.v64<<1 | (sig128Z.v0 != 0); + goto roundPack; + } + normExpSig = softfloat_normSubnormalF64Sig( sigC ); + expC = normExpSig.exp; + sigC = normExpSig.sig; + } + sigC = (sigC | UINT64_C( 0x0010000000000000 ))<<9; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expZ - expC; + if ( expDiff < 0 ) { + expZ = expC; + if ( (signZ == signC) || (expDiff < -1) ) { + sig128Z.v64 = softfloat_shiftRightJam64( sig128Z.v64, -expDiff ); + } else { + sig128Z = + softfloat_shortShiftRightJam128( sig128Z.v64, sig128Z.v0, 1 ); + } + } else if ( expDiff ) { + sig128C = softfloat_shiftRightJam128( sigC, 0, expDiff ); + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( signZ == signC ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( expDiff <= 0 ) { + sigZ = (sigC + sig128Z.v64) | (sig128Z.v0 != 0); + } else { + sig128Z = + softfloat_add128( + sig128Z.v64, sig128Z.v0, sig128C.v64, sig128C.v0 ); + sigZ = sig128Z.v64 | (sig128Z.v0 != 0); + } + if ( sigZ < UINT64_C( 0x4000000000000000 ) ) { + --expZ; + sigZ <<= 1; + } + } else { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( expDiff < 0 ) { + signZ = signC; + sig128Z = softfloat_sub128( sigC, 0, sig128Z.v64, sig128Z.v0 ); + } else if ( ! expDiff ) { + sig128Z.v64 = sig128Z.v64 - sigC; + if ( ! (sig128Z.v64 | sig128Z.v0) ) goto completeCancellation; + if ( sig128Z.v64 & UINT64_C( 0x8000000000000000 ) ) { + signZ ^= 1; + sig128Z = softfloat_sub128( 0, 0, sig128Z.v64, sig128Z.v0 ); + } + } else { + sig128Z = + softfloat_sub128( + sig128Z.v64, sig128Z.v0, sig128C.v64, sig128C.v0 ); + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( ! sig128Z.v64 ) { + expZ -= 64; + sig128Z.v64 = sig128Z.v0; + sig128Z.v0 = 0; + } + shiftCount = softfloat_countLeadingZeros64( sig128Z.v64 ) - 1; + expZ -= shiftCount; + if ( shiftCount < 0 ) { + sigZ = softfloat_shortShiftRightJam64( sig128Z.v64, -shiftCount ); + } else { + sig128Z = + softfloat_shortShiftLeft128( + sig128Z.v64, sig128Z.v0, shiftCount ); + sigZ = sig128Z.v64; + } + sigZ |= (sig128Z.v0 != 0); + } + roundPack: + return softfloat_roundPackToF64( signZ, expZ, sigZ ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN_ABC: + uiZ = softfloat_propagateNaNF64UI( uiA, uiB ); + goto propagateNaN_ZC; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infProdArg: + if ( magBits ) { + uiZ = packToF64UI( signZ, 0x7FF, 0 ); + if ( expC != 0x7FF ) goto uiZ; + if ( sigC ) goto propagateNaN_ZC; + if ( signZ == signC ) goto uiZ; + } + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF64UI; + propagateNaN_ZC: + uiZ = softfloat_propagateNaNF64UI( uiZ, uiC ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zeroProd: + uiZ = uiC; + if ( ! (expC | sigC) && (signZ != signC) ) { + completeCancellation: + uiZ = + packToF64UI( softfloat_roundingMode == softfloat_round_min, 0, 0 ); + } + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + +#else + +float64_t + softfloat_mulAddF64( + uint_fast64_t uiA, uint_fast64_t uiB, uint_fast64_t uiC, uint_fast8_t op ) +{ + bool signA; + int_fast16_t expA; + uint64_t sigA; + bool signB; + int_fast16_t expB; + uint64_t sigB; + bool signC; + int_fast16_t expC; + uint64_t sigC; + bool signZ; + uint64_t magBits, uiZ; + struct exp16_sig64 normExpSig; + int_fast16_t expZ; + uint32_t sig128Z[4]; + uint64_t sigZ; + int_fast16_t shiftCount, expDiff; + uint32_t sig128C[4]; + union ui64_f64 uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + signA = signF64UI( uiA ); + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + signB = signF64UI( uiB ); + expB = expF64UI( uiB ); + sigB = fracF64UI( uiB ); + signC = signF64UI( uiC ) ^ (op == softfloat_mulAdd_subC); + expC = expF64UI( uiC ); + sigC = fracF64UI( uiC ); + signZ = signA ^ signB ^ (op == softfloat_mulAdd_subProd); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( expA == 0x7FF ) { + if ( sigA || ((expB == 0x7FF) && sigB) ) goto propagateNaN_ABC; + magBits = expB | sigB; + goto infProdArg; + } + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN_ABC; + magBits = expA | sigA; + goto infProdArg; + } + if ( expC == 0x7FF ) { + if ( sigC ) { + uiZ = 0; + goto propagateNaN_ZC; + } + uiZ = uiC; + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( ! expA ) { + if ( ! sigA ) goto zeroProd; + normExpSig = softfloat_normSubnormalF64Sig( sigA ); + expA = normExpSig.exp; + sigA = normExpSig.sig; + } + if ( ! expB ) { + if ( ! sigB ) goto zeroProd; + normExpSig = softfloat_normSubnormalF64Sig( sigB ); + expB = normExpSig.exp; + sigB = normExpSig.sig; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA + expB - 0x3FE; + sigA = (sigA | UINT64_C( 0x0010000000000000 ))<<10; + sigB = (sigB | UINT64_C( 0x0010000000000000 ))<<11; + softfloat_mul64To128M( sigA, sigB, sig128Z ); + sigZ = + (uint64_t) sig128Z[indexWord( 4, 3 )]<<32 | sig128Z[indexWord( 4, 2 )]; + shiftCount = 0; + if ( ! (sigZ & UINT64_C( 0x4000000000000000 )) ) { + --expZ; + shiftCount = -1; + } + if ( ! expC ) { + if ( ! sigC ) { + if ( shiftCount ) sigZ <<= 1; + goto sigZ; + } + normExpSig = softfloat_normSubnormalF64Sig( sigC ); + expC = normExpSig.exp; + sigC = normExpSig.sig; + } + sigC = (sigC | UINT64_C( 0x0010000000000000 ))<<10; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expZ - expC; + if ( expDiff < 0 ) { + expZ = expC; + if ( (signZ == signC) || (expDiff < -1) ) { + shiftCount -= expDiff; + if ( shiftCount) { + sigZ = softfloat_shiftRightJam64( sigZ, shiftCount ); + } + } else { + if ( ! shiftCount ) { + softfloat_shortShiftRight128M( sig128Z, 1, sig128Z ); + } + } + } else { + if ( shiftCount ) softfloat_add128M( sig128Z, sig128Z, sig128Z ); + if ( ! expDiff ) { + sigZ = + (uint64_t) sig128Z[indexWord( 4, 3 )]<<32 + | sig128Z[indexWord( 4, 2 )]; + } else { + sig128C[indexWord( 4, 3 )] = sigC>>32; + sig128C[indexWord( 4, 2 )] = sigC; + sig128C[indexWord( 4, 1 )] = 0; + sig128C[indexWord( 4, 0 )] = 0; + softfloat_shiftRightJam128M( sig128C, expDiff, sig128C ); + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( signZ == signC ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( expDiff <= 0 ) { + sigZ += sigC; + } else { + softfloat_add128M( sig128Z, sig128C, sig128Z ); + sigZ = + (uint64_t) sig128Z[indexWord( 4, 3 )]<<32 + | sig128Z[indexWord( 4, 2 )]; + } + if ( sigZ & UINT64_C( 0x8000000000000000 ) ) { + ++expZ; + sigZ = softfloat_shortShiftRightJam64( sigZ, 1 ); + } + } else { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( expDiff < 0 ) { + signZ = signC; + if ( expDiff < -1 ) { + sigZ = sigC - sigZ; + if ( + sig128Z[indexWord( 4, 1 )] || sig128Z[indexWord( 4, 0 )] + ) { + sigZ = (sigZ - 1) | 1; + } + if ( ! (sigZ & UINT64_C( 0x4000000000000000 )) ) { + --expZ; + sigZ <<= 1; + } + goto roundPack; + } else { + sig128C[indexWord( 4, 3 )] = sigC>>32; + sig128C[indexWord( 4, 2 )] = sigC; + sig128C[indexWord( 4, 1 )] = 0; + sig128C[indexWord( 4, 0 )] = 0; + softfloat_sub128M( sig128C, sig128Z, sig128Z ); + } + } else if ( ! expDiff ) { + sigZ -= sigC; + if ( + ! sigZ && ! sig128Z[indexWord( 4, 1 )] + && ! sig128Z[indexWord( 4, 0 )] + ) { + goto completeCancellation; + } + sig128Z[indexWord( 4, 3 )] = sigZ>>32; + sig128Z[indexWord( 4, 2 )] = sigZ; + if ( sigZ & UINT64_C( 0x8000000000000000 ) ) { + signZ ^= 1; + softfloat_negX128M( sig128Z ); + } + } else { + softfloat_sub128M( sig128Z, sig128C, sig128Z ); + if ( 1 < expDiff ) { + sigZ = + (uint64_t) sig128Z[indexWord( 4, 3 )]<<32 + | sig128Z[indexWord( 4, 2 )]; + if ( ! (sigZ & UINT64_C( 0x4000000000000000 )) ) { + --expZ; + sigZ <<= 1; + } + goto sigZ; + } + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + shiftCount = 0; + sigZ = + (uint64_t) sig128Z[indexWord( 4, 3 )]<<32 + | sig128Z[indexWord( 4, 2 )]; + if ( ! sigZ ) { + shiftCount = 64; + sigZ = + (uint64_t) sig128Z[indexWord( 4, 1 )]<<32 + | sig128Z[indexWord( 4, 0 )]; + } + shiftCount += softfloat_countLeadingZeros64( sigZ ) - 1; + if ( shiftCount ) { + expZ -= shiftCount; + softfloat_shiftLeft128M( sig128Z, shiftCount, sig128Z ); + sigZ = + (uint64_t) sig128Z[indexWord( 4, 3 )]<<32 + | sig128Z[indexWord( 4, 2 )]; + } + } + sigZ: + if ( sig128Z[indexWord( 4, 1 )] || sig128Z[indexWord( 4, 0 )] ) sigZ |= 1; + roundPack: + return softfloat_roundPackToF64( signZ, expZ - 1, sigZ ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN_ABC: + uiZ = softfloat_propagateNaNF64UI( uiA, uiB ); + goto propagateNaN_ZC; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + infProdArg: + if ( magBits ) { + uiZ = packToF64UI( signZ, 0x7FF, 0 ); + if ( expC != 0x7FF ) goto uiZ; + if ( sigC ) goto propagateNaN_ZC; + if ( signZ == signC ) goto uiZ; + } + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF64UI; + propagateNaN_ZC: + uiZ = softfloat_propagateNaNF64UI( uiZ, uiC ); + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + zeroProd: + uiZ = uiC; + if ( ! (expC | sigC) && (signZ != signC) ) { + completeCancellation: + uiZ = + packToF64UI( softfloat_roundingMode == softfloat_round_min, 0, 0 ); + } + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_negXM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_negXM.c new file mode 100644 index 0000000..8a01edc --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_negXM.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_negXM + +void softfloat_negXM( uint_fast8_t size_words, uint32_t *zPtr ) +{ + unsigned int index, lastIndex; + uint_fast8_t carry; + uint32_t word; + + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + carry = 1; + for (;;) { + word = ~zPtr[index] + carry; + zPtr[index] = word; + if ( index == lastIndex ) break; + index += wordIncr; + if ( word ) carry = 0; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normExtF80SigM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normExtF80SigM.c new file mode 100644 index 0000000..1547900 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normExtF80SigM.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" + +int softfloat_normExtF80SigM( uint64_t *sigPtr ) +{ + uint64_t sig; + int_fast8_t shiftCount; + + sig = *sigPtr; + shiftCount = softfloat_countLeadingZeros64( sig ); + *sigPtr = sig< +#include +#include "platform.h" +#include "internals.h" + +void + softfloat_normRoundPackMToExtF80M( + bool sign, + int32_t exp, + uint32_t *extSigPtr, + uint_fast8_t roundingPrecision, + struct extFloat80M *zSPtr + ) +{ + int_fast16_t shiftCount; + uint32_t wordSig; + + shiftCount = 0; + wordSig = extSigPtr[indexWord( 3, 2 )]; + if ( ! wordSig ) { + shiftCount = 32; + wordSig = extSigPtr[indexWord( 3, 1 )]; + if ( ! wordSig ) { + shiftCount = 64; + wordSig = extSigPtr[indexWord( 3, 0 )]; + if ( ! wordSig ) { + zSPtr->signExp = packToExtF80UI64( sign, 0 ); + zSPtr->signif = 0; + return; + } + } + } + shiftCount += softfloat_countLeadingZeros32( wordSig ); + if ( shiftCount ) { + exp -= shiftCount; + softfloat_shiftLeft96M( extSigPtr, shiftCount, extSigPtr ); + } + softfloat_roundPackMToExtF80M( + sign, exp, extSigPtr, roundingPrecision, zSPtr ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackMToF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackMToF128M.c new file mode 100644 index 0000000..24cd5fb --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackMToF128M.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" + +void + softfloat_normRoundPackMToF128M( + bool sign, int32_t exp, uint32_t *extSigPtr, uint32_t *zWPtr ) +{ + const uint32_t *ptr; + int_fast16_t shiftCount; + uint32_t wordSig; + + ptr = extSigPtr + indexWordHi( 5 ); + shiftCount = 0; + for (;;) { + wordSig = *ptr; + if ( wordSig ) break; + shiftCount += 32; + if ( 160 <= shiftCount ) { + zWPtr[indexWordHi( 4 )] = packToF128UI96( sign, 0, 0 ); + zWPtr[indexWord( 4, 2 )] = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + return; + } + ptr -= wordIncr; + } + shiftCount += softfloat_countLeadingZeros32( wordSig ) - 15; + if ( shiftCount ) { + exp -= shiftCount; + softfloat_shiftLeft160M( extSigPtr, shiftCount, extSigPtr ); + } + softfloat_roundPackMToF128M( sign, exp, extSigPtr, zWPtr ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToExtF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToExtF80.c new file mode 100644 index 0000000..e446942 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToExtF80.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" + +extFloat80_t + softfloat_normRoundPackToExtF80( + bool sign, + int_fast32_t exp, + uint_fast64_t sig, + uint_fast64_t sigExtra, + uint_fast8_t roundingPrecision + ) +{ + int_fast8_t shiftCount; + struct uint128 sig128; + + if ( ! sig ) { + exp -= 64; + sig = sigExtra; + sigExtra = 0; + } + shiftCount = softfloat_countLeadingZeros64( sig ); + exp -= shiftCount; + if ( shiftCount ) { + sig128 = softfloat_shortShiftLeft128( sig, sigExtra, shiftCount ); + sig = sig128.v64; + sigExtra = sig128.v0; + } + return + softfloat_roundPackToExtF80( + sign, exp, sig, sigExtra, roundingPrecision ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToF128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToF128.c new file mode 100644 index 0000000..fa293cd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToF128.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" + +float128_t + softfloat_normRoundPackToF128( + bool sign, int_fast32_t exp, uint_fast64_t sig64, uint_fast64_t sig0 ) +{ + int_fast8_t shiftCount; + struct uint128 sig128; + union ui128_f128 uZ; + uint_fast64_t sigExtra; + struct uint128_extra sig128Extra; + + if ( ! sig64 ) { + exp -= 64; + sig64 = sig0; + sig0 = 0; + } + shiftCount = softfloat_countLeadingZeros64( sig64 ) - 15; + exp -= shiftCount; + if ( 0 <= shiftCount ) { + if ( shiftCount ) { + sig128 = softfloat_shortShiftLeft128( sig64, sig0, shiftCount ); + sig64 = sig128.v64; + sig0 = sig128.v0; + } + if ( (uint32_t) exp < 0x7FFD ) { + uZ.ui.v64 = packToF128UI64( sign, sig64 | sig0 ? exp : 0, sig64 ); + uZ.ui.v0 = sig0; + return uZ.f; + } + sigExtra = 0; + } else { + sig128Extra = + softfloat_shortShiftRightJam128Extra( + sig64, sig0, 0, -shiftCount ); + sig64 = sig128Extra.v.v64; + sig0 = sig128Extra.v.v0; + sigExtra = sig128Extra.extra; + } + return softfloat_roundPackToF128( sign, exp, sig64, sig0, sigExtra ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToF32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToF32.c new file mode 100644 index 0000000..8d1d0df --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normRoundPackToF32.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" + +float32_t + softfloat_normRoundPackToF32( bool sign, int_fast16_t exp, uint_fast32_t sig ) +{ + int_fast8_t shiftCount; + union ui32_f32 uZ; + + shiftCount = softfloat_countLeadingZeros32( sig ) - 1; + exp -= shiftCount; + if ( (7 <= shiftCount) && ((uint16_t) exp < 0xFD) ) { + uZ.ui = packToF32UI( sign, sig ? exp : 0, sig<<(shiftCount - 7) ); + return uZ.f; + } else { + return softfloat_roundPackToF32( sign, exp, sig< +#include +#include "platform.h" +#include "internals.h" + +float64_t + softfloat_normRoundPackToF64( bool sign, int_fast16_t exp, uint_fast64_t sig ) +{ + int_fast8_t shiftCount; + union ui64_f64 uZ; + + shiftCount = softfloat_countLeadingZeros64( sig ) - 1; + exp -= shiftCount; + if ( (10 <= shiftCount) && ((uint16_t) exp < 0x7FD) ) { + uZ.ui = packToF64UI( sign, sig ? exp : 0, sig<<(shiftCount - 10) ); + return uZ.f; + } else { + return softfloat_roundPackToF64( sign, exp, sig< +#include "platform.h" +#include "internals.h" + +struct exp32_sig64 softfloat_normSubnormalExtF80Sig( uint_fast64_t sig ) +{ + int_fast8_t shiftCount; + struct exp32_sig64 z; + + shiftCount = softfloat_countLeadingZeros64( sig ); + z.exp = -shiftCount; + z.sig = sig< +#include "platform.h" +#include "internals.h" + +struct exp32_sig128 + softfloat_normSubnormalF128Sig( uint_fast64_t sig64, uint_fast64_t sig0 ) +{ + int_fast8_t shiftCount; + struct exp32_sig128 z; + + if ( ! sig64 ) { + shiftCount = softfloat_countLeadingZeros64( sig0 ) - 15; + z.exp = -63 - shiftCount; + if ( shiftCount < 0 ) { + z.sig.v64 = sig0>>-shiftCount; + z.sig.v0 = sig0<<(shiftCount & 63); + } else { + z.sig.v64 = sig0< +#include "platform.h" +#include "internals.h" + +int softfloat_normSubnormalF128SigM( uint32_t *sigPtr ) +{ + const uint32_t *ptr; + int_fast16_t shiftCount; + uint32_t wordSig; + + ptr = sigPtr + indexWordHi( 4 ); + shiftCount = 0; + for (;;) { + wordSig = *ptr; + if ( wordSig ) break; + shiftCount += 32; + if ( 128 <= shiftCount ) return 1; + ptr -= wordIncr; + } + shiftCount += softfloat_countLeadingZeros32( wordSig ) - 15; + if ( shiftCount ) softfloat_shiftLeft128M( sigPtr, shiftCount, sigPtr ); + return 1 - shiftCount; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normSubnormalF32Sig.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normSubnormalF32Sig.c new file mode 100644 index 0000000..ad69631 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_normSubnormalF32Sig.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" + +struct exp16_sig32 softfloat_normSubnormalF32Sig( uint_fast32_t sig ) +{ + int_fast8_t shiftCount; + struct exp16_sig32 z; + + shiftCount = softfloat_countLeadingZeros32( sig ) - 8; + z.exp = 1 - shiftCount; + z.sig = sig< +#include "platform.h" +#include "internals.h" + +struct exp16_sig64 softfloat_normSubnormalF64Sig( uint_fast64_t sig ) +{ + int_fast8_t shiftCount; + struct exp16_sig64 z; + + shiftCount = softfloat_countLeadingZeros64( sig ) - 11; + z.exp = 1 - shiftCount; + z.sig = sig< +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_remStepMBy32 + +void + softfloat_remStepMBy32( + uint_fast8_t size_words, + const uint32_t *remPtr, + uint_fast8_t count, + const uint32_t *bPtr, + uint32_t q, + uint32_t *zPtr + ) +{ + uint_fast8_t negCount; + unsigned int index, lastIndex; + uint64_t dwordProd; + uint32_t wordRem, wordShiftedRem, wordProd; + uint_fast8_t borrow; + + negCount = -count; + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + dwordProd = (uint64_t) bPtr[index] * q; + wordRem = remPtr[index]; + wordShiftedRem = wordRem<>(negCount & 31); + index += wordIncr; + dwordProd = (uint64_t) bPtr[index] * q + (dwordProd>>32); + wordRem = remPtr[index]; + wordShiftedRem |= wordRem< +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +void + softfloat_roundPackMToExtF80M( + bool sign, + int32_t exp, + uint32_t *extSigPtr, + uint_fast8_t roundingPrecision, + struct extFloat80M *zSPtr + ) +{ + uint_fast8_t roundingMode; + bool roundNearEven; + uint64_t sig, roundIncrement, roundMask, roundBits; + bool isTiny; + uint32_t sigExtra; + bool doIncrement; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + roundingMode = softfloat_roundingMode; + roundNearEven = (roundingMode == softfloat_round_near_even); + sig = + (uint64_t) extSigPtr[indexWord( 3, 2 )]<<32 + | extSigPtr[indexWord( 3, 1 )]; + if ( roundingPrecision == 80 ) goto precision80; + if ( roundingPrecision == 64 ) { + roundIncrement = UINT64_C( 0x0000000000000400 ); + roundMask = UINT64_C( 0x00000000000007FF ); + } else if ( roundingPrecision == 32 ) { + roundIncrement = UINT64_C( 0x0000008000000000 ); + roundMask = UINT64_C( 0x000000FFFFFFFFFF ); + } else { + goto precision80; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( extSigPtr[indexWordLo( 3 )] ) sig |= 1; + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + roundIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ? roundMask + : 0; + } + roundBits = sig & roundMask; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x7FFD <= (uint32_t) (exp - 1) ) { + if ( exp <= 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < 0) + || (sig <= (uint64_t) (sig + roundIncrement)); + sig = softfloat_shiftRightJam64( sig, 1 - exp ); + roundBits = sig & roundMask; + if ( roundBits ) { + if ( isTiny ) softfloat_raiseFlags( softfloat_flag_underflow ); + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + sig += roundIncrement; + exp = ((sig & UINT64_C( 0x8000000000000000 )) != 0); + roundIncrement = roundMask + 1; + if ( roundNearEven && (roundBits<<1 == roundIncrement) ) { + roundMask |= roundIncrement; + } + sig &= ~roundMask; + goto packReturn; + } + if ( + (0x7FFE < exp) + || ((exp == 0x7FFE) && ((uint64_t) (sig + roundIncrement) < sig)) + ) { + goto overflow; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( roundBits ) softfloat_exceptionFlags |= softfloat_flag_inexact; + sig += roundIncrement; + if ( sig < roundIncrement ) { + ++exp; + sig = UINT64_C( 0x8000000000000000 ); + } + roundIncrement = roundMask + 1; + if ( roundNearEven && (roundBits<<1 == roundIncrement) ) { + roundMask |= roundIncrement; + } + sig &= ~roundMask; + goto packReturn; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + precision80: + sigExtra = extSigPtr[indexWordLo( 3 )]; + doIncrement = (0x80000000 <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( 0x7FFD <= (uint32_t) (exp - 1) ) { + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( exp <= 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < 0) + || ! doIncrement + || (sig < UINT64_C( 0xFFFFFFFFFFFFFFFF )); + softfloat_shiftRightJam96M( extSigPtr, 1 - exp, extSigPtr ); + sigExtra = extSigPtr[indexWordLo( 3 )]; + if ( sigExtra ) { + if ( isTiny ) softfloat_raiseFlags( softfloat_flag_underflow ); + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + doIncrement = (0x80000000 <= sigExtra); + if ( + ! roundNearEven + && (roundingMode != softfloat_round_near_maxMag) + ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + exp = 0; + sig = + (uint64_t) extSigPtr[indexWord( 3, 2 )]<<32 + | extSigPtr[indexWord( 3, 1 )]; + if ( doIncrement ) { + ++sig; + sig &= ~(uint64_t) (! (sigExtra & 0x7FFFFFFF) & roundNearEven); + exp = ((sig & UINT64_C( 0x8000000000000000 )) != 0); + } + goto packReturn; + } + /*-------------------------------------------------------------------- + *--------------------------------------------------------------------*/ + if ( + (0x7FFE < exp) + || ((exp == 0x7FFE) && (sig == UINT64_C( 0xFFFFFFFFFFFFFFFF )) + && doIncrement) + ) { + roundMask = 0; + overflow: + softfloat_raiseFlags( + softfloat_flag_overflow | softfloat_flag_inexact ); + if ( + roundNearEven + || (roundingMode == softfloat_round_near_maxMag) + || (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ) { + exp = 0x7FFF; + sig = UINT64_C( 0x8000000000000000 ); + } else { + exp = 0x7FFE; + sig = ~roundMask; + } + goto packReturn; + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + if ( sigExtra ) softfloat_exceptionFlags |= softfloat_flag_inexact; + if ( doIncrement ) { + ++sig; + if ( ! sig ) { + ++exp; + sig = UINT64_C( 0x8000000000000000 ); + } else { + sig &= ~(uint64_t) (! (sigExtra & 0x7FFFFFFF) & roundNearEven); + } + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + packReturn: + zSPtr->signExp = packToExtF80UI64( sign, exp ); + zSPtr->signif = sig; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToF128M.c new file mode 100644 index 0000000..91b750f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToF128M.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +void + softfloat_roundPackMToF128M( + bool sign, int32_t exp, uint32_t *extSigPtr, uint32_t *zWPtr ) +{ + uint_fast8_t roundingMode; + bool roundNearEven; + uint32_t sigExtra; + bool doIncrement, isTiny; + static const uint32_t maxSig[4] = + INIT_UINTM4( 0x0001FFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF ); + uint32_t ui, uj; + + roundingMode = softfloat_roundingMode; + roundNearEven = (roundingMode == softfloat_round_near_even); + sigExtra = extSigPtr[indexWordLo( 5 )]; + doIncrement = (0x80000000 <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + if ( 0x7FFD <= (uint32_t) exp ) { + if ( exp < 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < -1) + || ! doIncrement + || (softfloat_compare128M( + extSigPtr + indexMultiwordHi( 5, 4 ), maxSig ) + < 0); + softfloat_shiftRightJam160M( extSigPtr, -exp, extSigPtr ); + exp = 0; + sigExtra = extSigPtr[indexWordLo( 5 )]; + if ( isTiny && sigExtra ) { + softfloat_raiseFlags( softfloat_flag_underflow ); + } + doIncrement = (0x80000000 <= sigExtra); + if ( + ! roundNearEven + && (roundingMode != softfloat_round_near_maxMag) + ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + } else if ( + (0x7FFD < exp) + || ((exp == 0x7FFD) && doIncrement + && (softfloat_compare128M( + extSigPtr + indexMultiwordHi( 5, 4 ), maxSig ) + == 0)) + ) { + softfloat_raiseFlags( + softfloat_flag_overflow | softfloat_flag_inexact ); + if ( + roundNearEven + || (roundingMode == softfloat_round_near_maxMag) + || (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ) { + ui = packToF128UI96( sign, 0x7FFF, 0 ); + uj = 0; + } else { + ui = packToF128UI96( sign, 0x7FFE, 0x0000FFFF ); + uj = 0xFFFFFFFF; + } + zWPtr[indexWordHi( 4 )] = ui; + zWPtr[indexWord( 4, 2 )] = uj; + zWPtr[indexWord( 4, 1 )] = uj; + zWPtr[indexWord( 4, 0 )] = uj; + return; + } + } + if ( sigExtra ) softfloat_exceptionFlags |= softfloat_flag_inexact; + uj = extSigPtr[indexWord( 5, 1 )]; + if ( doIncrement ) { + ++uj; + if ( uj ) { + if ( ! (sigExtra & 0x7FFFFFFF) && roundNearEven ) uj &= ~1; + zWPtr[indexWord( 4, 2 )] = extSigPtr[indexWord( 5, 3 )]; + zWPtr[indexWord( 4, 1 )] = extSigPtr[indexWord( 5, 2 )]; + zWPtr[indexWord( 4, 0 )] = uj; + ui = extSigPtr[indexWordHi( 5 )]; + } else { + zWPtr[indexWord( 4, 0 )] = uj; + ui = extSigPtr[indexWord( 5, 2 )] + 1; + zWPtr[indexWord( 4, 1 )] = ui; + uj = extSigPtr[indexWord( 5, 3 )]; + if ( ui ) { + zWPtr[indexWord( 4, 2 )] = uj; + ui = extSigPtr[indexWordHi( 5 )]; + } else { + ++uj; + zWPtr[indexWord( 4, 2 )] = uj; + ui = extSigPtr[indexWordHi( 5 )]; + if ( ! uj ) ++ui; + } + } + } else { + zWPtr[indexWord( 4, 0 )] = uj; + ui = extSigPtr[indexWord( 5, 2 )]; + zWPtr[indexWord( 4, 1 )] = ui; + uj |= ui; + ui = extSigPtr[indexWord( 5, 3 )]; + zWPtr[indexWord( 4, 2 )] = ui; + uj |= ui; + ui = extSigPtr[indexWordHi( 5 )]; + uj |= ui; + if ( ! uj ) exp = 0; + } + zWPtr[indexWordHi( 4 )] = packToF128UI96( sign, exp, ui ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToI64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToI64.c new file mode 100644 index 0000000..196b268 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToI64.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t + softfloat_roundPackMToI64( + bool sign, uint32_t *extSigPtr, uint_fast8_t roundingMode, bool exact ) +{ + bool roundNearEven; + uint32_t sigExtra; + bool doIncrement; + uint64_t sig; + union { uint64_t ui; int64_t i; } uZ; + int64_t z; + + roundNearEven = (roundingMode == softfloat_round_near_even); + sigExtra = extSigPtr[indexWordLo( 3 )]; + doIncrement = (0x80000000 <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + sig = + (uint64_t) extSigPtr[indexWord( 3, 2 )]<<32 + | extSigPtr[indexWord( 3, 1 )]; + if ( doIncrement ) { + ++sig; + if ( ! sig ) goto invalid; + if ( ! (sigExtra & 0x7FFFFFFF) && roundNearEven ) sig &= ~1; + } + uZ.ui = sign ? -sig : sig; + z = uZ.i; + if ( z && ((z < 0) ^ sign) ) goto invalid; + if ( exact && sigExtra ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return z; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return + sign ? -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1 + : INT64_C( 0x7FFFFFFFFFFFFFFF ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToUI64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToUI64.c new file mode 100644 index 0000000..033baee --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackMToUI64.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t + softfloat_roundPackMToUI64( + bool sign, uint32_t *extSigPtr, uint_fast8_t roundingMode, bool exact ) +{ + bool roundNearEven; + uint32_t sigExtra; + bool doIncrement; + uint64_t sig; + + roundNearEven = (roundingMode == softfloat_round_near_even); + sigExtra = extSigPtr[indexWordLo( 3 )]; + doIncrement = (0x80000000 <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + sig = + (uint64_t) extSigPtr[indexWord( 3, 2 )]<<32 + | extSigPtr[indexWord( 3, 1 )]; + if ( doIncrement ) { + ++sig; + if ( ! sig ) goto invalid; + if ( ! (sigExtra & 0x7FFFFFFF) && roundNearEven ) sig &= ~1; + } + if ( sign && sig ) goto invalid; + if ( exact && sigExtra ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return sig; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToExtF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToExtF80.c new file mode 100644 index 0000000..77c6c69 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToExtF80.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +extFloat80_t + softfloat_roundPackToExtF80( + bool sign, + int_fast32_t exp, + uint_fast64_t sig, + uint_fast64_t sigExtra, + uint_fast8_t roundingPrecision + ) +{ + uint_fast8_t roundingMode; + bool roundNearEven; + uint_fast64_t roundIncrement, roundMask, roundBits; + bool isTiny, doIncrement; + struct uint64_extra sig64Extra; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + roundingMode = softfloat_roundingMode; + roundNearEven = (roundingMode == softfloat_round_near_even); + if ( roundingPrecision == 80 ) goto precision80; + if ( roundingPrecision == 64 ) { + roundIncrement = UINT64_C( 0x0000000000000400 ); + roundMask = UINT64_C( 0x00000000000007FF ); + } else if ( roundingPrecision == 32 ) { + roundIncrement = UINT64_C( 0x0000008000000000 ); + roundMask = UINT64_C( 0x000000FFFFFFFFFF ); + } else { + goto precision80; + } + sig |= (sigExtra != 0); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + roundIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ? roundMask + : 0; + } + roundBits = sig & roundMask; + if ( 0x7FFD <= (uint32_t) (exp - 1) ) { + if ( exp <= 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < 0) + || (sig <= (uint64_t) (sig + roundIncrement)); + sig = softfloat_shiftRightJam64( sig, 1 - exp ); + roundBits = sig & roundMask; + if ( isTiny && roundBits ) { + softfloat_raiseFlags( softfloat_flag_underflow ); + } + if ( roundBits ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + sig += roundIncrement; + exp = ((sig & UINT64_C( 0x8000000000000000 )) != 0); + roundIncrement = roundMask + 1; + if ( roundNearEven && (roundBits<<1 == roundIncrement) ) { + roundMask |= roundIncrement; + } + sig &= ~roundMask; + goto packReturn; + } + if ( + (0x7FFE < exp) + || ((exp == 0x7FFE) && ((uint64_t) (sig + roundIncrement) < sig)) + ) { + goto overflow; + } + } + if ( roundBits ) softfloat_exceptionFlags |= softfloat_flag_inexact; + sig = (uint64_t) (sig + roundIncrement); + if ( sig < roundIncrement ) { + ++exp; + sig = UINT64_C( 0x8000000000000000 ); + } + roundIncrement = roundMask + 1; + if ( roundNearEven && (roundBits<<1 == roundIncrement) ) { + roundMask |= roundIncrement; + } + sig &= ~roundMask; + if ( ! sig ) exp = 0; + goto packReturn; + precision80: + doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + if ( 0x7FFD <= (uint32_t) (exp - 1) ) { + if ( exp <= 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < 0) + || ! doIncrement + || (sig < UINT64_C( 0xFFFFFFFFFFFFFFFF )); + sig64Extra = + softfloat_shiftRightJam64Extra( sig, sigExtra, 1 - exp ); + sig = sig64Extra.v; + sigExtra = sig64Extra.extra; + if ( isTiny && sigExtra ) { + softfloat_raiseFlags( softfloat_flag_underflow ); + } + if ( sigExtra ) softfloat_exceptionFlags |= softfloat_flag_inexact; + doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra); + if ( + ! roundNearEven + && (roundingMode != softfloat_round_near_maxMag) + ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + exp = 0; + if ( doIncrement ) { + ++sig; + sig &= + ~(uint_fast64_t) + (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + & roundNearEven); + exp = ((sig & UINT64_C( 0x8000000000000000 )) != 0); + } + goto packReturn; + } + if ( + (0x7FFE < exp) + || ((exp == 0x7FFE) && (sig == UINT64_C( 0xFFFFFFFFFFFFFFFF )) + && doIncrement) + ) { + roundMask = 0; + overflow: + softfloat_raiseFlags( + softfloat_flag_overflow | softfloat_flag_inexact ); + if ( + roundNearEven + || (roundingMode == softfloat_round_near_maxMag) + || (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ) { + exp = 0x7FFF; + sig = UINT64_C( 0x8000000000000000 ); + } else { + exp = 0x7FFE; + sig = ~roundMask; + } + goto packReturn; + } + } + if ( sigExtra ) softfloat_exceptionFlags |= softfloat_flag_inexact; + if ( doIncrement ) { + ++sig; + if ( ! sig ) { + ++exp; + sig = UINT64_C( 0x8000000000000000 ); + } else { + sig &= + ~(uint_fast64_t) + (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + & roundNearEven); + } + } else { + if ( ! sig ) exp = 0; + } + packReturn: + uZ.s.signExp = packToExtF80UI64( sign, exp ); + uZ.s.signif = sig; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF128.c new file mode 100644 index 0000000..3d76da2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF128.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t + softfloat_roundPackToF128( + bool sign, + int_fast32_t exp, + uint_fast64_t sig64, + uint_fast64_t sig0, + uint_fast64_t sigExtra + ) +{ + uint_fast8_t roundingMode; + bool roundNearEven, doIncrement, isTiny; + struct uint128_extra sig128Extra; + uint_fast64_t uiZ64, uiZ0; + struct uint128 sig128; + union ui128_f128 uZ; + + roundingMode = softfloat_roundingMode; + roundNearEven = (roundingMode == softfloat_round_near_even); + doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + if ( 0x7FFD <= (uint32_t) exp ) { + if ( exp < 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < -1) + || ! doIncrement + || softfloat_lt128( + sig64, + sig0, + UINT64_C( 0x0001FFFFFFFFFFFF ), + UINT64_C( 0xFFFFFFFFFFFFFFFF ) + ); + sig128Extra = + softfloat_shiftRightJam128Extra( sig64, sig0, sigExtra, -exp ); + sig64 = sig128Extra.v.v64; + sig0 = sig128Extra.v.v0; + sigExtra = sig128Extra.extra; + exp = 0; + if ( isTiny && sigExtra ) { + softfloat_raiseFlags( softfloat_flag_underflow ); + } + doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra); + if ( + ! roundNearEven + && (roundingMode != softfloat_round_near_maxMag) + ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + } else if ( + (0x7FFD < exp) + || ((exp == 0x7FFD) + && softfloat_eq128( + sig64, + sig0, + UINT64_C( 0x0001FFFFFFFFFFFF ), + UINT64_C( 0xFFFFFFFFFFFFFFFF ) + ) + && doIncrement) + ) { + softfloat_raiseFlags( + softfloat_flag_overflow | softfloat_flag_inexact ); + if ( + roundNearEven + || (roundingMode == softfloat_round_near_maxMag) + || (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ) { + uiZ64 = packToF128UI64( sign, 0x7FFF, 0 ); + uiZ0 = 0; + } else { + uiZ64 = + packToF128UI64( + sign, 0x7FFE, UINT64_C( 0x0000FFFFFFFFFFFF ) ); + uiZ0 = UINT64_C( 0xFFFFFFFFFFFFFFFF ); + } + goto uiZ; + } + } + if ( sigExtra ) softfloat_exceptionFlags |= softfloat_flag_inexact; + if ( doIncrement ) { + sig128 = softfloat_add128( sig64, sig0, 0, 1 ); + sig64 = sig128.v64; + sig0 = + sig128.v0 + & ~(uint64_t) + (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + & roundNearEven); + } else { + if ( ! (sig64 | sig0) ) exp = 0; + } + uiZ64 = packToF128UI64( sign, exp, sig64 ); + uiZ0 = sig0; + uiZ: + uZ.ui.v64 = uiZ64; + uZ.ui.v0 = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF32.c new file mode 100644 index 0000000..bc27915 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF32.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t + softfloat_roundPackToF32( bool sign, int_fast16_t exp, uint_fast32_t sig ) +{ + uint_fast8_t roundingMode; + bool roundNearEven; + uint_fast8_t roundIncrement, roundBits; + bool isTiny; + uint_fast32_t uiZ; + union ui32_f32 uZ; + + roundingMode = softfloat_roundingMode; + roundNearEven = (roundingMode == softfloat_round_near_even); + roundIncrement = 0x40; + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + roundIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ? 0x7F + : 0; + } + roundBits = sig & 0x7F; + if ( 0xFD <= (uint16_t) exp ) { + if ( exp < 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < -1) + || (sig + roundIncrement < 0x80000000); + sig = softfloat_shiftRightJam32( sig, -exp ); + exp = 0; + roundBits = sig & 0x7F; + if ( isTiny && roundBits ) { + softfloat_raiseFlags( softfloat_flag_underflow ); + } + } else if ( (0xFD < exp) || (0x80000000 <= sig + roundIncrement) ) { + softfloat_raiseFlags( + softfloat_flag_overflow | softfloat_flag_inexact ); + uiZ = packToF32UI( sign, 0xFF, 0 ) - ! roundIncrement; + goto uiZ; + } + } + if ( roundBits ) softfloat_exceptionFlags |= softfloat_flag_inexact; + sig = (sig + roundIncrement)>>7; + sig &= ~(uint_fast32_t) (! (roundBits ^ 0x40) & roundNearEven); + uiZ = packToF32UI( sign, sig ? exp : 0, sig ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF64.c new file mode 100644 index 0000000..b0d96ce --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToF64.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t + softfloat_roundPackToF64( bool sign, int_fast16_t exp, uint_fast64_t sig ) +{ + uint_fast8_t roundingMode; + bool roundNearEven; + uint_fast16_t roundIncrement, roundBits; + bool isTiny; + uint_fast64_t uiZ; + union ui64_f64 uZ; + + roundingMode = softfloat_roundingMode; + roundNearEven = (roundingMode == softfloat_round_near_even); + roundIncrement = 0x200; + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + roundIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ? 0x3FF + : 0; + } + roundBits = sig & 0x3FF; + if ( 0x7FD <= (uint16_t) exp ) { + if ( exp < 0 ) { + isTiny = + (softfloat_detectTininess + == softfloat_tininess_beforeRounding) + || (exp < -1) + || (sig + roundIncrement < UINT64_C( 0x8000000000000000 )); + sig = softfloat_shiftRightJam64( sig, -exp ); + exp = 0; + roundBits = sig & 0x3FF; + if ( isTiny && roundBits ) { + softfloat_raiseFlags( softfloat_flag_underflow ); + } + } else if ( + (0x7FD < exp) + || (UINT64_C( 0x8000000000000000 ) <= sig + roundIncrement) + ) { + softfloat_raiseFlags( + softfloat_flag_overflow | softfloat_flag_inexact ); + uiZ = packToF64UI( sign, 0x7FF, 0 ) - ! roundIncrement; + goto uiZ; + } + } + if ( roundBits ) softfloat_exceptionFlags |= softfloat_flag_inexact; + sig = (sig + roundIncrement)>>10; + sig &= ~(uint_fast64_t) (! (roundBits ^ 0x200) & roundNearEven); + uiZ = packToF64UI( sign, sig ? exp : 0, sig ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToI32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToI32.c new file mode 100644 index 0000000..9b23223 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToI32.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast32_t + softfloat_roundPackToI32( + bool sign, uint_fast64_t sig, uint_fast8_t roundingMode, bool exact ) +{ + bool roundNearEven; + uint_fast8_t roundIncrement, roundBits; + uint_fast32_t sig32; + union { uint32_t ui; int32_t i; } uZ; + int_fast32_t z; + + roundNearEven = (roundingMode == softfloat_round_near_even); + roundIncrement = 0x40; + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + roundIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ? 0x7F + : 0; + } + roundBits = sig & 0x7F; + sig += roundIncrement; + if ( sig & UINT64_C( 0xFFFFFF8000000000 ) ) goto invalid; + sig32 = sig>>7; + sig32 &= ~(uint_fast32_t) (! (roundBits ^ 0x40) & roundNearEven); + uZ.ui = sign ? -sig32 : sig32; + z = uZ.i; + if ( z && ((z < 0) ^ sign) ) goto invalid; + if ( exact && roundBits ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return z; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return sign ? -0x7FFFFFFF - 1 : 0x7FFFFFFF; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToI64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToI64.c new file mode 100644 index 0000000..52c2a35 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToI64.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +int_fast64_t + softfloat_roundPackToI64( + bool sign, + uint_fast64_t sig, + uint_fast64_t sigExtra, + uint_fast8_t roundingMode, + bool exact + ) +{ + bool roundNearEven, doIncrement; + union { uint64_t ui; int64_t i; } uZ; + int_fast64_t z; + + roundNearEven = (roundingMode == softfloat_round_near_even); + doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + if ( doIncrement ) { + ++sig; + if ( ! sig ) goto invalid; + sig &= + ~(uint_fast64_t) + (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + & roundNearEven); + } + uZ.ui = sign ? -sig : sig; + z = uZ.i; + if ( z && ((z < 0) ^ sign) ) goto invalid; + if ( exact && sigExtra ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return z; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return + sign ? -INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1 + : INT64_C( 0x7FFFFFFFFFFFFFFF ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToUI32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToUI32.c new file mode 100644 index 0000000..613a28e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToUI32.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast32_t + softfloat_roundPackToUI32( + bool sign, uint_fast64_t sig, uint_fast8_t roundingMode, bool exact ) +{ + bool roundNearEven; + uint_fast8_t roundIncrement, roundBits; + uint_fast32_t z; + + roundNearEven = (roundingMode == softfloat_round_near_even); + roundIncrement = 0x40; + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + roundIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + ? 0x7F + : 0; + } + roundBits = sig & 0x7F; + sig += roundIncrement; + if ( sig & UINT64_C( 0xFFFFFF8000000000 ) ) goto invalid; + z = sig>>7; + z &= ~(uint_fast32_t) (! (roundBits ^ 0x40) & roundNearEven); + if ( sign && z ) goto invalid; + if ( exact && roundBits ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return z; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return 0xFFFFFFFF; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToUI64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToUI64.c new file mode 100644 index 0000000..5a82b8c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_roundPackToUI64.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +uint_fast64_t + softfloat_roundPackToUI64( + bool sign, + uint_fast64_t sig, + uint_fast64_t sigExtra, + uint_fast8_t roundingMode, + bool exact + ) +{ + bool roundNearEven, doIncrement; + + roundNearEven = (roundingMode == softfloat_round_near_even); + doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra); + if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) { + doIncrement = + (roundingMode + == (sign ? softfloat_round_min : softfloat_round_max)) + && sigExtra; + } + if ( doIncrement ) { + ++sig; + if ( ! sig ) goto invalid; + sig &= + ~(uint_fast64_t) + (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF )) + & roundNearEven); + } + if ( sign && sig ) goto invalid; + if ( exact && sigExtra ) { + softfloat_exceptionFlags |= softfloat_flag_inexact; + } + return sig; + invalid: + softfloat_raiseFlags( softfloat_flag_invalid ); + return UINT64_C( 0xFFFFFFFFFFFFFFFF ); + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftLeftM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftLeftM.c new file mode 100644 index 0000000..f80f29c --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftLeftM.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_shiftLeftM + +#define softfloat_shiftLeftM softfloat_shiftLeftM +#include "primitives.h" + +void + softfloat_shiftLeftM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint32_t count, + uint32_t *zPtr + ) +{ + uint32_t wordCount; + uint_fast8_t innerCount; + uint32_t *destPtr; + uint_fast8_t i; + + wordCount = count>>5; + if ( wordCount < size_words ) { + aPtr += indexMultiwordLoBut( size_words, wordCount ); + innerCount = count & 31; + if ( innerCount ) { + softfloat_shortShiftLeftM( + size_words - wordCount, + aPtr, + innerCount, + zPtr + indexMultiwordHiBut( size_words, wordCount ) + ); + if ( ! wordCount ) return; + } else { + aPtr += indexWordHi( size_words - wordCount ); + destPtr = zPtr + indexWordHi( size_words ); + for ( i = size_words - wordCount; i; --i ) { + *destPtr = *aPtr; + aPtr -= wordIncr; + destPtr -= wordIncr; + } + } + zPtr += indexMultiwordLo( size_words, wordCount ); + } else { + wordCount = size_words; + } + do { + *zPtr++ = 0; + --wordCount; + } while ( wordCount ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftNormSigF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftNormSigF128M.c new file mode 100644 index 0000000..ea68d60 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftNormSigF128M.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" + +int + softfloat_shiftNormSigF128M( + const uint32_t *wPtr, uint_fast8_t shiftCount, uint32_t *sigPtr ) +{ + uint32_t wordSig; + int32_t exp; + uint32_t leadingBit; + + wordSig = wPtr[indexWordHi( 4 )]; + exp = expF128UI96( wordSig ); + if ( exp ) { + softfloat_shortShiftLeft128M( wPtr, shiftCount, sigPtr ); + leadingBit = 0x00010000< +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shiftRightJam128 + +struct uint128 + softfloat_shiftRightJam128( uint64_t a64, uint64_t a0, uint_fast32_t count ) +{ + int_fast32_t negCount; + struct uint128 z; + + if ( count < 64 ) { + negCount = -count; + z.v64 = a64>>count; + z.v0 = + a64<<(negCount & 63) | a0>>count + | ((uint64_t) (a0<<(negCount & 63)) != 0); + } else { + z.v64 = 0; + z.v0 = + (count < 127) + ? a64>>(count & 63) + | (((a64 & (((uint_fast64_t) 1<<(count & 63)) - 1)) | a0) + != 0) + : ((a64 | a0) != 0); + } + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam128Extra.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam128Extra.c new file mode 100644 index 0000000..b322924 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam128Extra.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shiftRightJam128Extra + +struct uint128_extra + softfloat_shiftRightJam128Extra( + uint64_t a64, uint64_t a0, uint64_t extra, uint_fast32_t count ) +{ + int_fast32_t negCount; + struct uint128_extra z; + + negCount = -count; + if ( count < 64 ) { + z.v.v64 = a64>>count; + z.v.v0 = a64<<(negCount & 63) | a0>>count; + z.extra = a0<<(negCount & 63); + } else { + z.v.v64 = 0; + if ( count == 64 ) { + z.v.v0 = a64; + z.extra = a0; + } else { + extra |= a0; + if ( count < 128 ) { + z.v.v0 = a64>>(count & 63); + z.extra = a64<<(negCount & 63); + } else { + z.v.v0 = 0; + z.extra = (count == 128) ? a64 : (a64 != 0); + } + } + } + z.extra |= (extra != 0); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam256M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam256M.c new file mode 100644 index 0000000..965fb33 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam256M.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shiftRightJam256M + +static + void + softfloat_shortShiftRightJamM( + uint_fast8_t size_words, + const uint64_t *aPtr, + uint_fast8_t count, + uint64_t *zPtr + ) +{ + uint_fast8_t negCount; + unsigned int index, lastIndex; + uint64_t partWordZ, wordA; + + negCount = -count; + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + wordA = aPtr[index]; + partWordZ = wordA>>count; + if ( partWordZ<>count; + } + zPtr[index] = partWordZ; + +} + +void + softfloat_shiftRightJam256M( + const uint64_t *aPtr, uint_fast32_t count, uint64_t *zPtr ) +{ + uint64_t wordJam; + uint_fast32_t wordCount; + uint64_t *ptr; + uint_fast8_t i, innerCount; + + wordJam = 0; + wordCount = count>>6; + if ( wordCount ) { + if ( 4 < wordCount ) wordCount = 4; + ptr = (uint64_t *) (aPtr + indexMultiwordLo( 4, wordCount )); + i = wordCount; + do { + wordJam = *ptr++; + if ( wordJam ) break; + --i; + } while ( i ); + ptr = zPtr; + } + if ( wordCount < 4 ) { + aPtr += indexMultiwordHiBut( 4, wordCount ); + innerCount = count & 63; + if ( innerCount ) { + softfloat_shortShiftRightJamM( + 4 - wordCount, + aPtr, + innerCount, + zPtr + indexMultiwordLoBut( 4, wordCount ) + ); + if ( ! wordCount ) goto wordJam; + } else { + aPtr += indexWordLo( 4 - wordCount ); + ptr = zPtr + indexWordLo( 4 ); + for ( i = 4 - wordCount; i; --i ) { + *ptr = *aPtr; + aPtr += wordIncr; + ptr += wordIncr; + } + } + ptr = zPtr + indexMultiwordHi( 4, wordCount ); + } + do { + *ptr++ = 0; + --wordCount; + } while ( wordCount ); + wordJam: + if ( wordJam ) zPtr[indexWordLo( 4 )] |= 1; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam32.c new file mode 100644 index 0000000..44c1f2f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam32.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_shiftRightJam32 + +uint32_t softfloat_shiftRightJam32( uint32_t a, uint_fast16_t count ) +{ + + return + (count < 31) ? a>>count | ((uint32_t) (a<<(-count & 31)) != 0) + : (a != 0); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam64.c new file mode 100644 index 0000000..9387887 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam64.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_shiftRightJam64 + +uint64_t softfloat_shiftRightJam64( uint64_t a, uint_fast32_t count ) +{ + + return + (count < 63) ? a>>count | ((uint64_t) (a<<(-count & 63)) != 0) + : (a != 0); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam64Extra.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam64Extra.c new file mode 100644 index 0000000..e656a3f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJam64Extra.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shiftRightJam64Extra + +struct uint64_extra + softfloat_shiftRightJam64Extra( + uint64_t a, uint64_t extra, uint_fast32_t count ) +{ + struct uint64_extra z; + + if ( count < 64 ) { + z.v = a>>count; + z.extra = a<<(-count & 63); + } else { + z.v = 0; + z.extra = (count == 64) ? a : (a != 0); + } + z.extra |= (extra != 0); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJamM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJamM.c new file mode 100644 index 0000000..ef394bd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightJamM.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_shiftRightJamM + +#define softfloat_shiftRightJamM softfloat_shiftRightJamM +#include "primitives.h" + +void + softfloat_shiftRightJamM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint32_t count, + uint32_t *zPtr + ) +{ + uint32_t wordJam, wordCount, *ptr; + uint_fast8_t i, innerCount; + + wordJam = 0; + wordCount = count>>5; + if ( wordCount ) { + if ( size_words < wordCount ) wordCount = size_words; + ptr = (uint32_t *) (aPtr + indexMultiwordLo( size_words, wordCount )); + i = wordCount; + do { + wordJam = *ptr++; + if ( wordJam ) break; + --i; + } while ( i ); + ptr = zPtr; + } + if ( wordCount < size_words ) { + aPtr += indexMultiwordHiBut( size_words, wordCount ); + innerCount = count & 31; + if ( innerCount ) { + softfloat_shortShiftRightJamM( + size_words - wordCount, + aPtr, + innerCount, + zPtr + indexMultiwordLoBut( size_words, wordCount ) + ); + if ( ! wordCount ) goto wordJam; + } else { + aPtr += indexWordLo( size_words - wordCount ); + ptr = zPtr + indexWordLo( size_words ); + for ( i = size_words - wordCount; i; --i ) { + *ptr = *aPtr; + aPtr += wordIncr; + ptr += wordIncr; + } + } + ptr = zPtr + indexMultiwordHi( size_words, wordCount ); + } + do { + *ptr++ = 0; + --wordCount; + } while ( wordCount ); + wordJam: + if ( wordJam ) zPtr[indexWordLo( size_words )] |= 1; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightM.c new file mode 100644 index 0000000..50b35e4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shiftRightM.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_shiftRightM + +#define softfloat_shiftRightM softfloat_shiftRightM +#include "primitives.h" + +void + softfloat_shiftRightM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint32_t count, + uint32_t *zPtr + ) +{ + uint32_t wordCount; + uint_fast8_t innerCount; + uint32_t *destPtr; + uint_fast8_t i; + + wordCount = count>>5; + if ( wordCount < size_words ) { + aPtr += indexMultiwordHiBut( size_words, wordCount ); + innerCount = count & 31; + if ( innerCount ) { + softfloat_shortShiftRightM( + size_words - wordCount, + aPtr, + innerCount, + zPtr + indexMultiwordLoBut( size_words, wordCount ) + ); + if ( ! wordCount ) return; + } else { + aPtr += indexWordLo( size_words - wordCount ); + destPtr = zPtr + indexWordLo( size_words ); + for ( i = size_words - wordCount; i; --i ) { + *destPtr = *aPtr; + aPtr += wordIncr; + destPtr += wordIncr; + } + } + zPtr += indexMultiwordHi( size_words, wordCount ); + } else { + wordCount = size_words; + } + do { + *zPtr++ = 0; + --wordCount; + } while ( wordCount ); + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftLeft128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftLeft128.c new file mode 100644 index 0000000..4af115d --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftLeft128.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftLeft128 + +struct uint128 + softfloat_shortShiftLeft128( uint64_t a64, uint64_t a0, uint_fast8_t count ) +{ + struct uint128 z; + + z.v64 = a64<>(-count & 63); + z.v0 = a0< +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftLeft64To96M + +void + softfloat_shortShiftLeft64To96M( + uint64_t a, uint_fast8_t count, uint32_t *zPtr ) +{ + + zPtr[indexWord( 3, 0 )] = (uint32_t) a<>= 32 - count; + zPtr[indexWord( 3, 2 )] = a>>32; + zPtr[indexWord( 3, 1 )] = a; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftLeftM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftLeftM.c new file mode 100644 index 0000000..01e97c7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftLeftM.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftLeftM + +void + softfloat_shortShiftLeftM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint_fast8_t count, + uint32_t *zPtr + ) +{ + uint_fast8_t negCount; + unsigned int index, lastIndex; + uint32_t partWordZ, wordA; + + negCount = -count; + index = indexWordHi( size_words ); + lastIndex = indexWordLo( size_words ); + partWordZ = aPtr[index]<>(negCount & 31); + index -= wordIncr; + partWordZ = wordA< +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRight128 + +struct uint128 + softfloat_shortShiftRight128( uint64_t a64, uint64_t a0, uint_fast8_t count ) +{ + struct uint128 z; + + z.v64 = a64>>count; + z.v0 = a64<<(-count & 63) | a0>>count; + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightExtendM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightExtendM.c new file mode 100644 index 0000000..38b6fcd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightExtendM.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRightExtendM + +void + softfloat_shortShiftRightExtendM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint_fast8_t count, + uint32_t *zPtr + ) +{ + uint_fast8_t negCount; + unsigned int indexA, lastIndexA; + uint32_t partWordZ, wordA; + + negCount = -count; + indexA = indexWordLo( size_words ); + lastIndexA = indexWordHi( size_words ); + zPtr += indexWordLo( size_words + 1 ); + partWordZ = 0; + for (;;) { + wordA = aPtr[indexA]; + *zPtr = wordA<<(negCount & 31) | partWordZ; + zPtr += wordIncr; + partWordZ = wordA>>count; + if ( indexA == lastIndexA ) break; + indexA += wordIncr; + } + *zPtr = partWordZ; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam128.c new file mode 100644 index 0000000..ea630e6 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam128.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRightJam128 + +struct uint128 + softfloat_shortShiftRightJam128( + uint64_t a64, uint64_t a0, uint_fast8_t count ) +{ + int_fast8_t negCount; + struct uint128 z; + + negCount = -count; + z.v64 = a64>>count; + z.v0 = + a64<<(negCount & 63) | a0>>count + | ((uint64_t) (a0<<(negCount & 63)) != 0); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam128Extra.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam128Extra.c new file mode 100644 index 0000000..0bdc6b6 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam128Extra.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRightJam128Extra + +struct uint128_extra + softfloat_shortShiftRightJam128Extra( + uint64_t a64, uint64_t a0, uint64_t extra, uint_fast8_t count ) +{ + uint_fast8_t negCount; + struct uint128_extra z; + + negCount = -count; + z.v.v64 = a64>>count; + z.v.v0 = a64<<(negCount & 63) | a0>>count; + z.extra = a0<<(negCount & 63) | (extra != 0); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam64.c new file mode 100644 index 0000000..8f6b042 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJam64.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" + +#ifndef softfloat_shortShiftRightJam64 + +uint64_t softfloat_shortShiftRightJam64( uint64_t a, uint_fast8_t count ) +{ + + return a>>count | ((a & (((uint_fast64_t) 1< +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRightJam64Extra + +struct uint64_extra + softfloat_shortShiftRightJam64Extra( + uint64_t a, uint64_t extra, uint_fast8_t count ) +{ + struct uint64_extra z; + + z.v = a>>count; + z.extra = a<<(-count & 63) | (extra != 0); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJamM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJamM.c new file mode 100644 index 0000000..96ff2ab --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightJamM.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRightJamM + +void + softfloat_shortShiftRightJamM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint_fast8_t count, + uint32_t *zPtr + ) +{ + uint_fast8_t negCount; + unsigned int index, lastIndex; + uint32_t partWordZ, wordA; + + negCount = -count; + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + wordA = aPtr[index]; + partWordZ = wordA>>count; + if ( partWordZ<>count; + } + zPtr[index] = partWordZ; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightM.c new file mode 100644 index 0000000..2563d28 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_shortShiftRightM.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_shortShiftRightM + +void + softfloat_shortShiftRightM( + uint_fast8_t size_words, + const uint32_t *aPtr, + uint_fast8_t count, + uint32_t *zPtr + ) +{ + uint_fast8_t negCount; + unsigned int index, lastIndex; + uint32_t partWordZ, wordA; + + negCount = -count; + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + partWordZ = aPtr[index]>>count; + while ( index != lastIndex ) { + wordA = aPtr[index + wordIncr]; + zPtr[index] = wordA<<(negCount & 31) | partWordZ; + index += wordIncr; + partWordZ = wordA>>count; + } + zPtr[index] = partWordZ; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub128.c new file mode 100644 index 0000000..fb6000b --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub128.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_sub128 + +struct uint128 + softfloat_sub128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 ) +{ + struct uint128 z; + + z.v0 = a0 - b0; + z.v64 = a64 - b64 - (a0 < b0); + return z; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub1XM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub1XM.c new file mode 100644 index 0000000..492a0b2 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub1XM.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_sub1XM + +void softfloat_sub1XM( uint_fast8_t size_words, uint32_t *zPtr ) +{ + unsigned int index, lastIndex; + uint32_t wordA; + + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + for (;;) { + wordA = zPtr[index]; + zPtr[index] = wordA - 1; + if ( wordA || (index == lastIndex) ) break; + index += wordIncr; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub256M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub256M.c new file mode 100644 index 0000000..06fe75e --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_sub256M.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_sub256M + +void + softfloat_sub256M( + const uint64_t *aPtr, const uint64_t *bPtr, uint64_t *zPtr ) +{ + unsigned int index; + uint_fast8_t borrow; + uint64_t wordA, wordB; + + index = indexWordLo( 4 ); + borrow = 0; + for (;;) { + wordA = aPtr[index]; + wordB = bPtr[index]; + zPtr[index] = wordA - wordB - borrow; + if ( index == indexWordHi( 4 ) ) break; + borrow = borrow ? (wordA <= wordB) : (wordA < wordB); + index += wordIncr; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subM.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subM.c new file mode 100644 index 0000000..3dfe355 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subM.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of +California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "primitiveTypes.h" + +#ifndef softfloat_subM + +void + softfloat_subM( + uint_fast8_t size_words, + const uint32_t *aPtr, + const uint32_t *bPtr, + uint32_t *zPtr + ) +{ + unsigned int index, lastIndex; + uint_fast8_t borrow; + uint32_t wordA, wordB; + + index = indexWordLo( size_words ); + lastIndex = indexWordHi( size_words ); + borrow = 0; + for (;;) { + wordA = aPtr[index]; + wordB = bPtr[index]; + zPtr[index] = wordA - wordB - borrow; + if ( index == lastIndex ) break; + borrow = borrow ? (wordA <= wordB) : (wordA < wordB); + index += wordIncr; + } + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsExtF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsExtF80.c new file mode 100644 index 0000000..bc0e612 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsExtF80.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +extFloat80_t + softfloat_subMagsExtF80( + uint_fast16_t uiA64, + uint_fast64_t uiA0, + uint_fast16_t uiB64, + uint_fast64_t uiB0, + bool signZ + ) +{ + int_fast32_t expA; + uint_fast64_t sigA; + int_fast32_t expB; + uint_fast64_t sigB; + int_fast32_t expDiff; + uint_fast16_t uiZ64; + uint_fast64_t uiZ0; + int_fast32_t expZ; + uint_fast64_t sigExtra; + struct uint128 sig128, uiZ; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expA = expExtF80UI64( uiA64 ); + sigA = uiA0; + expB = expExtF80UI64( uiB64 ); + sigB = uiB0; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expDiff = expA - expB; + if ( 0 < expDiff ) goto expABigger; + if ( expDiff < 0 ) goto expBBigger; + if ( expA == 0x7FFF ) { + if ( (sigA | sigB) & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) { + goto propagateNaN; + } + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ64 = defaultNaNExtF80UI64; + uiZ0 = defaultNaNExtF80UI0; + goto uiZ; + } + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expZ = expA; + if ( ! expZ ) expZ = 1; + sigExtra = 0; + if ( sigB < sigA ) goto aBigger; + if ( sigA < sigB ) goto bBigger; + uiZ64 = + packToExtF80UI64( (softfloat_roundingMode == softfloat_round_min), 0 ); + uiZ0 = 0; + goto uiZ; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expBBigger: + if ( expB == 0x7FFF ) { + if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + uiZ64 = packToExtF80UI64( signZ ^ 1, 0x7FFF ); + uiZ0 = UINT64_C( 0x8000000000000000 ); + goto uiZ; + } + if ( ! expA ) { + ++expDiff; + sigExtra = 0; + if ( ! expDiff ) goto newlyAlignedBBigger; + } + sig128 = softfloat_shiftRightJam128( sigA, 0, -expDiff ); + sigA = sig128.v64; + sigExtra = sig128.v0; + newlyAlignedBBigger: + expZ = expB; + bBigger: + signZ ^= 1; + sig128 = softfloat_sub128( sigB, 0, sigA, sigExtra ); + goto normRoundPack; + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + expABigger: + if ( expA == 0x7FFF ) { + if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN; + uiZ64 = uiA64; + uiZ0 = uiA0; + goto uiZ; + } + if ( ! expB ) { + --expDiff; + sigExtra = 0; + if ( ! expDiff ) goto newlyAlignedABigger; + } + sig128 = softfloat_shiftRightJam128( sigB, 0, expDiff ); + sigB = sig128.v64; + sigExtra = sig128.v0; + newlyAlignedABigger: + expZ = expA; + aBigger: + sig128 = softfloat_sub128( sigA, 0, sigB, sigExtra ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + normRoundPack: + return + softfloat_normRoundPackToExtF80( + signZ, expZ, sig128.v64, sig128.v0, extF80_roundingPrecision ); + /*------------------------------------------------------------------------ + *------------------------------------------------------------------------*/ + propagateNaN: + uiZ = softfloat_propagateNaNExtF80UI( uiA64, uiA0, uiB64, uiB0 ); + uiZ64 = uiZ.v64; + uiZ0 = uiZ.v0; + uiZ: + uZ.s.signExp = uiZ64; + uZ.s.signif = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF128.c new file mode 100644 index 0000000..92c13b7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF128.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float128_t + softfloat_subMagsF128( + uint_fast64_t uiA64, + uint_fast64_t uiA0, + uint_fast64_t uiB64, + uint_fast64_t uiB0, + bool signZ + ) +{ + int_fast32_t expA; + struct uint128 sigA; + int_fast32_t expB; + struct uint128 sigB, sigZ; + int_fast32_t expDiff, expZ; + struct uint128 uiZ; + union ui128_f128 uZ; + + expA = expF128UI64( uiA64 ); + sigA.v64 = fracF128UI64( uiA64 ); + sigA.v0 = uiA0; + expB = expF128UI64( uiB64 ); + sigB.v64 = fracF128UI64( uiB64 ); + sigB.v0 = uiB0; + sigA = softfloat_shortShiftLeft128( sigA.v64, sigA.v0, 4 ); + sigB = softfloat_shortShiftLeft128( sigB.v64, sigB.v0, 4 ); + expDiff = expA - expB; + if ( 0 < expDiff ) goto expABigger; + if ( expDiff < 0 ) goto expBBigger; + if ( expA == 0x7FFF ) { + if ( sigA.v64 | sigA.v0 | sigB.v64 | sigB.v0 ) goto propagateNaN; + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ.v64 = defaultNaNF128UI64; + uiZ.v0 = defaultNaNF128UI0; + goto uiZ; + } + expZ = expA; + if ( ! expZ ) expZ = 1; + if ( sigB.v64 < sigA.v64 ) goto aBigger; + if ( sigA.v64 < sigB.v64 ) goto bBigger; + if ( sigB.v0 < sigA.v0 ) goto aBigger; + if ( sigA.v0 < sigB.v0 ) goto bBigger; + uiZ.v64 = + packToF128UI64( + (softfloat_roundingMode == softfloat_round_min), 0, 0 ); + uiZ.v0 = 0; + goto uiZ; + expBBigger: + if ( expB == 0x7FFF ) { + if ( sigB.v64 | sigB.v0 ) goto propagateNaN; + uiZ.v64 = packToF128UI64( signZ ^ 1, 0x7FFF, 0 ); + uiZ.v0 = 0; + goto uiZ; + } + if ( expA ) { + sigA.v64 |= UINT64_C( 0x0010000000000000 ); + } else { + ++expDiff; + if ( ! expDiff ) goto newlyAlignedBBigger; + } + sigA = softfloat_shiftRightJam128( sigA.v64, sigA.v0, -expDiff ); + newlyAlignedBBigger: + expZ = expB; + sigB.v64 |= UINT64_C( 0x0010000000000000 ); + bBigger: + signZ ^= 1; + sigZ = softfloat_sub128( sigB.v64, sigB.v0, sigA.v64, sigA.v0 ); + goto normRoundPack; + expABigger: + if ( expA == 0x7FFF ) { + if ( sigA.v64 | sigA.v0 ) goto propagateNaN; + uiZ.v64 = uiA64; + uiZ.v0 = uiA0; + goto uiZ; + } + if ( expB ) { + sigB.v64 |= UINT64_C( 0x0010000000000000 ); + } else { + --expDiff; + if ( ! expDiff ) goto newlyAlignedABigger; + } + sigB = softfloat_shiftRightJam128( sigB.v64, sigB.v0, expDiff ); + newlyAlignedABigger: + expZ = expA; + sigA.v64 |= UINT64_C( 0x0010000000000000 ); + aBigger: + sigZ = softfloat_sub128( sigA.v64, sigA.v0, sigB.v64, sigB.v0 ); + normRoundPack: + return softfloat_normRoundPackToF128( signZ, expZ - 5, sigZ.v64, sigZ.v0 ); + propagateNaN: + uiZ = softfloat_propagateNaNF128UI( uiA64, uiA0, uiB64, uiB0 ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF32.c new file mode 100644 index 0000000..5a7effc --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF32.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float32_t + softfloat_subMagsF32( uint_fast32_t uiA, uint_fast32_t uiB, bool signZ ) +{ + int_fast16_t expA; + uint_fast32_t sigA; + int_fast16_t expB; + uint_fast32_t sigB; + int_fast16_t expDiff; + uint_fast32_t uiZ; + int_fast16_t expZ; + uint_fast32_t sigZ; + union ui32_f32 uZ; + + expA = expF32UI( uiA ); + sigA = fracF32UI( uiA ); + expB = expF32UI( uiB ); + sigB = fracF32UI( uiB ); + expDiff = expA - expB; + sigA <<= 7; + sigB <<= 7; + if ( 0 < expDiff ) goto expABigger; + if ( expDiff < 0 ) goto expBBigger; + if ( expA == 0xFF ) { + if ( sigA | sigB ) goto propagateNaN; + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF32UI; + goto uiZ; + } + if ( ! expA ) { + expA = 1; + expB = 1; + } + if ( sigB < sigA ) goto aBigger; + if ( sigA < sigB ) goto bBigger; + uiZ = packToF32UI( softfloat_roundingMode == softfloat_round_min, 0, 0 ); + goto uiZ; + expBBigger: + if ( expB == 0xFF ) { + if ( sigB ) goto propagateNaN; + uiZ = packToF32UI( signZ ^ 1, 0xFF, 0 ); + goto uiZ; + } + sigA += expA ? 0x40000000 : sigA; + sigA = softfloat_shiftRightJam32( sigA, -expDiff ); + sigB |= 0x40000000; + bBigger: + signZ ^= 1; + expZ = expB; + sigZ = sigB - sigA; + goto normRoundPack; + expABigger: + if ( expA == 0xFF ) { + if ( sigA ) goto propagateNaN; + uiZ = uiA; + goto uiZ; + } + sigB += expB ? 0x40000000 : sigB; + sigB = softfloat_shiftRightJam32( sigB, expDiff ); + sigA |= 0x40000000; + aBigger: + expZ = expA; + sigZ = sigA - sigB; + normRoundPack: + return softfloat_normRoundPackToF32( signZ, expZ - 1, sigZ ); + propagateNaN: + uiZ = softfloat_propagateNaNF32UI( uiA, uiB ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF64.c new file mode 100644 index 0000000..462ed55 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_subMagsF64.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +float64_t + softfloat_subMagsF64( uint_fast64_t uiA, uint_fast64_t uiB, bool signZ ) +{ + int_fast16_t expA; + uint_fast64_t sigA; + int_fast16_t expB; + uint_fast64_t sigB; + int_fast16_t expDiff; + uint_fast64_t uiZ; + int_fast16_t expZ; + uint_fast64_t sigZ; + union ui64_f64 uZ; + + expA = expF64UI( uiA ); + sigA = fracF64UI( uiA ); + expB = expF64UI( uiB ); + sigB = fracF64UI( uiB ); + expDiff = expA - expB; + sigA <<= 10; + sigB <<= 10; + if ( 0 < expDiff ) goto expABigger; + if ( expDiff < 0 ) goto expBBigger; + if ( expA == 0x7FF ) { + if ( sigA | sigB ) goto propagateNaN; + softfloat_raiseFlags( softfloat_flag_invalid ); + uiZ = defaultNaNF64UI; + goto uiZ; + } + if ( ! expA ) { + expA = 1; + expB = 1; + } + if ( sigB < sigA ) goto aBigger; + if ( sigA < sigB ) goto bBigger; + uiZ = packToF64UI( softfloat_roundingMode == softfloat_round_min, 0, 0 ); + goto uiZ; + expBBigger: + if ( expB == 0x7FF ) { + if ( sigB ) goto propagateNaN; + uiZ = packToF64UI( signZ ^ 1, 0x7FF, 0 ); + goto uiZ; + } + sigA += expA ? UINT64_C( 0x4000000000000000 ) : sigA; + sigA = softfloat_shiftRightJam64( sigA, -expDiff ); + sigB |= UINT64_C( 0x4000000000000000 ); + bBigger: + signZ ^= 1; + expZ = expB; + sigZ = sigB - sigA; + goto normRoundPack; + expABigger: + if ( expA == 0x7FF ) { + if ( sigA ) goto propagateNaN; + uiZ = uiA; + goto uiZ; + } + sigB += expB ? UINT64_C( 0x4000000000000000 ) : sigB; + sigB = softfloat_shiftRightJam64( sigB, expDiff ); + sigA |= UINT64_C( 0x4000000000000000 ); + aBigger: + expZ = expA; + sigZ = sigA - sigB; + normRoundPack: + return softfloat_normRoundPackToF64( signZ, expZ - 1, sigZ ); + propagateNaN: + uiZ = softfloat_propagateNaNF64UI( uiA, uiB ); + uiZ: + uZ.ui = uiZ; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_tryPropagateNaNExtF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_tryPropagateNaNExtF80M.c new file mode 100644 index 0000000..40ede0a --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_tryPropagateNaNExtF80M.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" + +bool + softfloat_tryPropagateNaNExtF80M( + const struct extFloat80M *aSPtr, + const struct extFloat80M *bSPtr, + struct extFloat80M *zSPtr + ) +{ + uint_fast16_t ui64; + uint64_t ui0; + + ui64 = aSPtr->signExp; + ui0 = aSPtr->signif; + if ( isNaNExtF80UI( ui64, ui0 ) ) goto propagateNaN; + ui64 = bSPtr->signExp; + ui0 = bSPtr->signif; + if ( isNaNExtF80UI( ui64, ui0 ) ) goto propagateNaN; + return false; + propagateNaN: + softfloat_propagateNaNExtF80M( aSPtr, bSPtr, zSPtr ); + return true; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_tryPropagateNaNF128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_tryPropagateNaNF128M.c new file mode 100644 index 0000000..17bba25 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/s_tryPropagateNaNF128M.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" + +bool + softfloat_tryPropagateNaNF128M( + const uint32_t *aWPtr, const uint32_t *bWPtr, uint32_t *zWPtr ) +{ + + if ( softfloat_isNaNF128M( aWPtr ) || softfloat_isNaNF128M( bWPtr ) ) { + softfloat_propagateNaNF128M( aWPtr, bWPtr, zWPtr ); + return true; + } + return false; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/softfloat_state.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/softfloat_state.c new file mode 100644 index 0000000..37b101f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/softfloat_state.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "specialize.h" +#include "softfloat.h" + +uint_fast8_t softfloat_roundingMode = softfloat_round_near_even; +uint_fast8_t softfloat_detectTininess = init_detectTininess; +uint_fast8_t softfloat_exceptionFlags = 0; + +uint_fast8_t extF80_roundingPrecision = 80; + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/sub.mk b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/sub.mk new file mode 100644 index 0000000..c1d29cb --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/sub.mk @@ -0,0 +1,65 @@ +incdirs-lib-y += include +incdirs-lib-y += 8086-SSE +subdirs-y += 8086-SSE + +cflags-y += -Wno-aggregate-return +cflags-y += -Wno-sign-compare +cflags-y += -Wno-missing-prototypes +cflags-y += -Wno-missing-declarations + +srcs-y += f32_add.c +srcs-y += f32_div.c +srcs-y += f32_eq.c +srcs-y += f32_le.c +srcs-y += f32_lt.c +srcs-y += f32_mul.c +srcs-y += f32_sub.c +srcs-y += f32_to_f64.c +srcs-y += f32_to_i32_r_minMag.c +srcs-y += f32_to_i64_r_minMag.c +srcs-y += f32_to_ui32_r_minMag.c +srcs-y += f32_to_ui64_r_minMag.c + +srcs-y += f64_add.c +srcs-y += f64_div.c +srcs-y += f64_eq.c +srcs-y += f64_le.c +srcs-y += f64_lt.c +srcs-y += f64_mul.c +srcs-y += f64_sub.c +srcs-y += f64_to_f32.c +srcs-y += f64_to_i32_r_minMag.c +srcs-y += f64_to_i64_r_minMag.c +srcs-y += f64_to_ui32_r_minMag.c +srcs-y += f64_to_ui64_r_minMag.c + +srcs-y += i32_to_f32.c +srcs-y += i32_to_f64.c +srcs-y += i64_to_f32.c +srcs-y += i64_to_f64.c +srcs-y += ui32_to_f32.c +srcs-y += ui32_to_f64.c +srcs-y += ui64_to_f32.c +srcs-y += ui64_to_f64.c + +srcs-y += s_subMagsF32.c +srcs-y += s_subMagsF64.c +srcs-y += s_addMagsF32.c +srcs-y += s_addMagsF64.c +srcs-y += s_normSubnormalF64Sig.c +srcs-y += s_normSubnormalF32Sig.c +srcs-y += s_roundPackToF32.c +srcs-y += s_roundPackToF64.c +srcs-y += s_shortShiftRightJam64.c +srcs-y += s_shiftRightJam32.c +srcs-y += s_normRoundPackToF32.c +srcs-y += s_normRoundPackToF64.c +srcs-y += s_shiftRightJam64.c + +srcs-y += s_countLeadingZeros8.c +srcs-y += s_countLeadingZeros32.c +srcs-y += s_countLeadingZeros64.c + +srcs-y += s_mul64To128.c + +srcs-y += softfloat_state.c diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_extF80.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_extF80.c new file mode 100644 index 0000000..0af7e31 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_extF80.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +extFloat80_t ui32_to_extF80( uint32_t a ) +{ + uint_fast16_t uiZ64; + int_fast8_t shiftCount; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + uiZ64 = 0; + if ( a ) { + shiftCount = softfloat_countLeadingZeros32( a ); + uiZ64 = 0x401E - shiftCount; + a <<= shiftCount; + } + uZ.s.signExp = uiZ64; + uZ.s.signif = (uint_fast64_t) a<<32; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_extF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_extF80M.c new file mode 100644 index 0000000..8957b2f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_extF80M.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void ui32_to_extF80M( uint32_t a, extFloat80_t *zPtr ) +{ + + *zPtr = ui32_to_extF80( a ); + +} + +#else + +void ui32_to_extF80M( uint32_t a, extFloat80_t *zPtr ) +{ + struct extFloat80M *zSPtr; + uint_fast16_t uiZ64; + uint64_t sigZ; + int_fast8_t shiftCount; + + zSPtr = (struct extFloat80M *) zPtr; + uiZ64 = 0; + sigZ = 0; + if ( a ) { + shiftCount = softfloat_countLeadingZeros32( a ); + uiZ64 = packToExtF80UI64( 0, 0x401E - shiftCount ); + sigZ = (uint64_t) (a<signExp = uiZ64; + zSPtr->signif = sigZ; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f128.c new file mode 100644 index 0000000..d7416ab --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f128.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t ui32_to_f128( uint32_t a ) +{ + uint_fast64_t uiZ64; + int_fast8_t shiftCount; + union ui128_f128 uZ; + + uiZ64 = 0; + if ( a ) { + shiftCount = softfloat_countLeadingZeros32( a ) + 17; + uiZ64 = + packToF128UI64( + 0, 0x402E - shiftCount, (uint_fast64_t) a< +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void ui32_to_f128M( uint32_t a, float128_t *zPtr ) +{ + + *zPtr = ui32_to_f128( a ); + +} + +#else + +void ui32_to_f128M( uint32_t a, float128_t *zPtr ) +{ + uint32_t *zWPtr, uiZ96, uiZ64; + int_fast8_t shiftCount; + uint64_t normA; + + zWPtr = (uint32_t *) zPtr; + uiZ96 = 0; + uiZ64 = 0; + if ( a ) { + shiftCount = softfloat_countLeadingZeros32( a ) + 17; + normA = (uint64_t) a<>32 ); + uiZ64 = normA; + } + zWPtr[indexWord( 4, 3 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = uiZ64; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f32.c new file mode 100644 index 0000000..0335b6f --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f32.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t ui32_to_f32( uint32_t a ) +{ + union ui32_f32 uZ; + + if ( ! a ) { + uZ.ui = 0; + return uZ.f; + } + if ( a & 0x80000000 ) { + return softfloat_roundPackToF32( 0, 0x9D, a>>1 | (a & 1) ); + } else { + return softfloat_normRoundPackToF32( 0, 0x9C, a ); + } + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f64.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f64.c new file mode 100644 index 0000000..47b0d86 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui32_to_f64.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t ui32_to_f64( uint32_t a ) +{ + uint_fast64_t uiZ; + int_fast8_t shiftCount; + union ui64_f64 uZ; + + if ( ! a ) { + uiZ = 0; + } else { + shiftCount = softfloat_countLeadingZeros32( a ) + 21; + uiZ = + packToF64UI( + 0, 0x432 - shiftCount, (uint_fast64_t) a< +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +extFloat80_t ui64_to_extF80( uint64_t a ) +{ + uint_fast16_t uiZ64; + int_fast8_t shiftCount; + union { struct extFloat80M s; extFloat80_t f; } uZ; + + uiZ64 = 0; + if ( a ) { + shiftCount = softfloat_countLeadingZeros64( a ); + uiZ64 = 0x403E - shiftCount; + a <<= shiftCount; + } + uZ.s.signExp = uiZ64; + uZ.s.signif = a; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_extF80M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_extF80M.c new file mode 100644 index 0000000..35566b0 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_extF80M.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void ui64_to_extF80M( uint64_t a, extFloat80_t *zPtr ) +{ + + *zPtr = ui64_to_extF80( a ); + +} + +#else + +void ui64_to_extF80M( uint64_t a, extFloat80_t *zPtr ) +{ + struct extFloat80M *zSPtr; + uint_fast16_t uiZ64; + uint64_t sigZ; + int_fast8_t shiftCount; + + zSPtr = (struct extFloat80M *) zPtr; + uiZ64 = 0; + sigZ = 0; + if ( a ) { + shiftCount = softfloat_countLeadingZeros64( a ); + uiZ64 = packToExtF80UI64( 0, 0x403E - shiftCount ); + sigZ = a<signExp = uiZ64; + zSPtr->signif = sigZ; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f128.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f128.c new file mode 100644 index 0000000..cfd6de4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f128.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float128_t ui64_to_f128( uint64_t a ) +{ + uint_fast64_t uiZ64, uiZ0; + int_fast8_t shiftCount; + struct uint128 zSig; + union ui128_f128 uZ; + + if ( ! a ) { + uiZ64 = 0; + uiZ0 = 0; + } else { + shiftCount = softfloat_countLeadingZeros64( a ) + 49; + if ( 64 <= shiftCount ) { + zSig.v64 = a<<(shiftCount - 64); + zSig.v0 = 0; + } else { + zSig = softfloat_shortShiftLeft128( 0, a, shiftCount ); + } + uiZ64 = packToF128UI64( 0, 0x406E - shiftCount, zSig.v64 ); + uiZ0 = zSig.v0; + } + uZ.ui.v64 = uiZ64; + uZ.ui.v0 = uiZ0; + return uZ.f; + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f128M.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f128M.c new file mode 100644 index 0000000..b4821cd --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f128M.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +#ifdef SOFTFLOAT_FAST_INT64 + +void ui64_to_f128M( uint64_t a, float128_t *zPtr ) +{ + + *zPtr = ui64_to_f128( a ); + +} + +#else + +void ui64_to_f128M( uint64_t a, float128_t *zPtr ) +{ + uint32_t *zWPtr, uiZ96, uiZ64; + uint_fast8_t shiftCount; + uint32_t *ptr; + + zWPtr = (uint32_t *) zPtr; + uiZ96 = 0; + uiZ64 = 0; + zWPtr[indexWord( 4, 1 )] = 0; + zWPtr[indexWord( 4, 0 )] = 0; + if ( a ) { + shiftCount = softfloat_countLeadingZeros64( a ) + 17; + if ( shiftCount < 32 ) { + ptr = zWPtr + indexMultiwordHi( 4, 3 ); + ptr[indexWord( 3, 2 )] = 0; + ptr[indexWord( 3, 1 )] = a>>32; + ptr[indexWord( 3, 0 )] = a; + softfloat_shortShiftLeft96M( ptr, shiftCount, ptr ); + ptr[indexWordHi( 3 )] = + packToF128UI96( + 0, 0x404E - shiftCount, ptr[indexWordHi( 3 )] ); + return; + } + a <<= shiftCount - 32; + uiZ96 = packToF128UI96( 0, 0x404E - shiftCount, a>>32 ); + uiZ64 = a; + } + zWPtr[indexWord( 4, 3 )] = uiZ96; + zWPtr[indexWord( 4, 2 )] = uiZ64; + +} + +#endif + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f32.c b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f32.c new file mode 100644 index 0000000..cb12496 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/source/ui64_to_f32.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/*============================================================================ + +This C source file is part of the SoftFloat IEEE Floating-Point Arithmetic +Package, Release 3a, by John R. Hauser. + +Copyright 2011, 2012, 2013, 2014 The Regents of the University of California. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================*/ + +#include +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float32_t ui64_to_f32( uint64_t a ) +{ + int_fast8_t shiftCount; + union ui32_f32 u; + uint_fast32_t sig; + + shiftCount = softfloat_countLeadingZeros64( a ) - 40; + if ( 0 <= shiftCount ) { + u.ui = + a ? packToF32UI( + 0, 0x95 - shiftCount, (uint_fast32_t) a< +#include "platform.h" +#include "internals.h" +#include "softfloat.h" + +float64_t ui64_to_f64( uint64_t a ) +{ + union ui64_f64 uZ; + + if ( ! a ) { + uZ.ui = 0; + return uZ.f; + } + if ( a & UINT64_C( 0x8000000000000000 ) ) { + return + softfloat_roundPackToF64( + 0, 0x43D, softfloat_shortShiftRightJam64( a, 1 ) ); + } else { + return softfloat_normRoundPackToF64( 0, 0x43C, a ); + } + +} + diff --git a/optee_os/lib/libutils/isoc/arch/arm/softfloat/sub.mk b/optee_os/lib/libutils/isoc/arch/arm/softfloat/sub.mk new file mode 100644 index 0000000..1175052 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/softfloat/sub.mk @@ -0,0 +1,3 @@ +incdirs-lib-y += arm32_include + +subdirs-y += source diff --git a/optee_os/lib/libutils/isoc/arch/arm/sub.mk b/optee_os/lib/libutils/isoc/arch/arm/sub.mk new file mode 100644 index 0000000..1e11502 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/arm/sub.mk @@ -0,0 +1,32 @@ +# These files implements the__aeabi functions we need instead of +# relying on libgcc or equivalent as we need implementations suitable +# for bare metal. +srcs-$(CFG_ARM32_$(sm)) += arm32_aeabi_divmod_a32.S +srcs-$(CFG_ARM32_$(sm)) += arm32_aeabi_divmod.c +srcs-$(CFG_ARM32_$(sm)) += arm32_aeabi_ldivmod_a32.S +srcs-$(CFG_ARM32_$(sm)) += arm32_aeabi_ldivmod.c +srcs-$(CFG_ARM32_$(sm)) += arm32_aeabi_shift.c + +ifeq ($(CFG_ULIBS_MCOUNT),y) +# We would not like to profile __aeabi functions as these provide +# internal implementations for "/ %" operations. Also, "/ %" operations +# could be used inside profiling code which could create an incorrect +# cyclic behaviour. +cflags-remove-arm32_aeabi_divmod.c-y += -pg +cflags-remove-arm32_aeabi_ldivmod.c-y += -pg +cflags-remove-arm32_aeabi_shift.c-y += -pg +endif + +srcs-$(CFG_ARM32_$(sm)) += setjmp_a32.S +srcs-$(CFG_ARM64_$(sm)) += setjmp_a64.S + +ifeq ($(CFG_TA_FLOAT_SUPPORT),y) +# Floating point is only supported for user TAs +ifneq ($(sm),core) +srcs-$(CFG_ARM32_$(sm)) += arm32_aeabi_softfloat.c +cflags-arm32_aeabi_softfloat.c-y += -Wno-aggregate-return +cflags-arm32_aeabi_softfloat.c-y += -Wno-missing-prototypes +cflags-arm32_aeabi_softfloat.c-y += -Wno-missing-declarations +subdirs-$(CFG_ARM32_$(sm)) += softfloat +endif +endif diff --git a/optee_os/lib/libutils/isoc/arch/riscv/setjmp_rv.S b/optee_os/lib/libutils/isoc/arch/riscv/setjmp_rv.S new file mode 100644 index 0000000..acb75c5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/riscv/setjmp_rv.S @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2023 Andes Technology Corporation + * Copyright 2022-2023 NXP + */ +#include + +/* int setjmp (jmp_buf) */ +FUNC setjmp , : + STR s0, REGOFF(0)(a0) + STR s1, REGOFF(1)(a0) + STR s2, REGOFF(2)(a0) + STR s3, REGOFF(3)(a0) + STR s4, REGOFF(4)(a0) + STR s5, REGOFF(5)(a0) + STR s6, REGOFF(6)(a0) + STR s7, REGOFF(7)(a0) + STR s8, REGOFF(8)(a0) + STR s9, REGOFF(9)(a0) + STR s10, REGOFF(10)(a0) + STR s11, REGOFF(11)(a0) + STR ra, REGOFF(12)(a0) + STR sp, REGOFF(13)(a0) +#ifdef CFG_FTRACE_SUPPORT + addi sp, sp, -16 +#ifdef RV32 + STR ra, REGOFF(3)(sp) + STR s0, REGOFF(2)(sp) +#else + STR ra, REGOFF(1)(sp) + STR s0, REGOFF(0)(sp) +#endif + addi s0, sp, 16 + addi a0, a0, REGOFF(14) + call ftrace_setjmp +#ifdef RV32 + LDR s0, REGOFF(2)(sp) + LDR ra, REGOFF(3)(sp) +#else + LDR s0, REGOFF(0)(sp) + LDR ra, REGOFF(1)(sp) +#endif + addi sp, sp, 16 +#endif + li a0, 0 + ret +END_FUNC setjmp + +/* void longjmp (jmp_buf, int) __attribute__ ((noreturn)) */ +FUNC longjmp , : +#ifdef CFG_FTRACE_SUPPORT + addi sp, sp, -REGOFF(4) + STR a0, REGOFF(0)(sp) + STR a1, REGOFF(1)(sp) + STR s0, REGOFF(2)(sp) + STR ra, REGOFF(3)(sp) + addi s0, sp, REGOFF(4) + addi a0, a0, REGOFF(14) + call ftrace_longjmp + LDR a0, REGOFF(0)(sp) + LDR a1, REGOFF(1)(sp) + LDR s0, REGOFF(2)(sp) + LDR ra, REGOFF(3)(sp) + addi sp, sp, REGOFF(4) +#endif + LDR s0, REGOFF(0)(a0) + LDR s1, REGOFF(1)(a0) + LDR s2, REGOFF(2)(a0) + LDR s3, REGOFF(3)(a0) + LDR s4, REGOFF(4)(a0) + LDR s5, REGOFF(5)(a0) + LDR s6, REGOFF(6)(a0) + LDR s7, REGOFF(7)(a0) + LDR s8, REGOFF(8)(a0) + LDR s9, REGOFF(9)(a0) + LDR s10, REGOFF(10)(a0) + LDR s11, REGOFF(11)(a0) + LDR ra, REGOFF(12)(a0) + LDR sp, REGOFF(13)(a0) + seqz a0, a1 + add a0, a0, a1 + ret +END_FUNC longjmp diff --git a/optee_os/lib/libutils/isoc/arch/riscv/sub.mk b/optee_os/lib/libutils/isoc/arch/riscv/sub.mk new file mode 100644 index 0000000..91a66e4 --- /dev/null +++ b/optee_os/lib/libutils/isoc/arch/riscv/sub.mk @@ -0,0 +1 @@ +srcs-y += setjmp_rv.S diff --git a/optee_os/lib/libutils/isoc/bget.c b/optee_os/lib/libutils/isoc/bget.c new file mode 100644 index 0000000..724763c --- /dev/null +++ b/optee_os/lib/libutils/isoc/bget.c @@ -0,0 +1,1764 @@ +/* + + B G E T + + Buffer allocator + + Designed and implemented in April of 1972 by John Walker, based on the + Case Algol OPRO$ algorithm implemented in 1966. + + Reimplemented in 1975 by John Walker for the Interdata 70. + Reimplemented in 1977 by John Walker for the Marinchip 9900. + Reimplemented in 1982 by Duff Kurland for the Intel 8080. + + Portable C version implemented in September of 1990 by an older, wiser + instance of the original implementor. + + Souped up and/or weighed down slightly shortly thereafter by Greg + Lutz. + + AMIX edition, including the new compaction call-back option, prepared + by John Walker in July of 1992. + + Bug in built-in test program fixed, ANSI compiler warnings eradicated, + buffer pool validator implemented, and guaranteed repeatable test + added by John Walker in October of 1995. + + This program is in the public domain. + + 1. This is the book of the generations of Adam. In the day that God + created man, in the likeness of God made he him; + 2. Male and female created he them; and blessed them, and called + their name Adam, in the day when they were created. + 3. And Adam lived an hundred and thirty years, and begat a son in + his own likeness, and after his image; and called his name Seth: + 4. And the days of Adam after he had begotten Seth were eight + hundred years: and he begat sons and daughters: + 5. And all the days that Adam lived were nine hundred and thirty + years: and he died. + 6. And Seth lived an hundred and five years, and begat Enos: + 7. And Seth lived after he begat Enos eight hundred and seven years, + and begat sons and daughters: + 8. And all the days of Seth were nine hundred and twelve years: and + he died. + 9. And Enos lived ninety years, and begat Cainan: + 10. And Enos lived after he begat Cainan eight hundred and fifteen + years, and begat sons and daughters: + 11. And all the days of Enos were nine hundred and five years: and + he died. + 12. And Cainan lived seventy years and begat Mahalaleel: + 13. And Cainan lived after he begat Mahalaleel eight hundred and + forty years, and begat sons and daughters: + 14. And all the days of Cainan were nine hundred and ten years: and + he died. + 15. And Mahalaleel lived sixty and five years, and begat Jared: + 16. And Mahalaleel lived after he begat Jared eight hundred and + thirty years, and begat sons and daughters: + 17. And all the days of Mahalaleel were eight hundred ninety and + five years: and he died. + 18. And Jared lived an hundred sixty and two years, and he begat + Enoch: + 19. And Jared lived after he begat Enoch eight hundred years, and + begat sons and daughters: + 20. And all the days of Jared were nine hundred sixty and two years: + and he died. + 21. And Enoch lived sixty and five years, and begat Methuselah: + 22. And Enoch walked with God after he begat Methuselah three + hundred years, and begat sons and daughters: + 23. And all the days of Enoch were three hundred sixty and five + years: + 24. And Enoch walked with God: and he was not; for God took him. + 25. And Methuselah lived an hundred eighty and seven years, and + begat Lamech. + 26. And Methuselah lived after he begat Lamech seven hundred eighty + and two years, and begat sons and daughters: + 27. And all the days of Methuselah were nine hundred sixty and nine + years: and he died. + 28. And Lamech lived an hundred eighty and two years, and begat a + son: + 29. And he called his name Noah, saying, This same shall comfort us + concerning our work and toil of our hands, because of the ground + which the LORD hath cursed. + 30. And Lamech lived after he begat Noah five hundred ninety and + five years, and begat sons and daughters: + 31. And all the days of Lamech were seven hundred seventy and seven + years: and he died. + 32. And Noah was five hundred years old: and Noah begat Shem, Ham, + and Japheth. + + And buffers begat buffers, and links begat links, and buffer pools + begat links to chains of buffer pools containing buffers, and lo the + buffers and links and pools of buffers and pools of links to chains of + pools of buffers were fruitful and they multiplied and the Operating + System looked down upon them and said that it was Good. + + + INTRODUCTION + ============ + + BGET is a comprehensive memory allocation package which is easily + configured to the needs of an application. BGET is efficient in + both the time needed to allocate and release buffers and in the + memory overhead required for buffer pool management. It + automatically consolidates contiguous space to minimise + fragmentation. BGET is configured by compile-time definitions, + Major options include: + + * A built-in test program to exercise BGET and + demonstrate how the various functions are used. + + * Allocation by either the "first fit" or "best fit" + method. + + * Wiping buffers at release time to catch code which + references previously released storage. + + * Built-in routines to dump individual buffers or the + entire buffer pool. + + * Retrieval of allocation and pool size statistics. + + * Quantisation of buffer sizes to a power of two to + satisfy hardware alignment constraints. + + * Automatic pool compaction, growth, and shrinkage by + means of call-backs to user defined functions. + + Applications of BGET can range from storage management in + ROM-based embedded programs to providing the framework upon which + a multitasking system incorporating garbage collection is + constructed. BGET incorporates extensive internal consistency + checking using the mechanism; all these checks can be + turned off by compiling with NDEBUG defined, yielding a version of + BGET with minimal size and maximum speed. + + The basic algorithm underlying BGET has withstood the test of + time; more than 25 years have passed since the first + implementation of this code. And yet, it is substantially more + efficient than the native allocation schemes of many operating + systems: the Macintosh and Microsoft Windows to name two, on which + programs have obtained substantial speed-ups by layering BGET as + an application level memory manager atop the underlying system's. + + BGET has been implemented on the largest mainframes and the lowest + of microprocessors. It has served as the core for multitasking + operating systems, multi-thread applications, embedded software in + data network switching processors, and a host of C programs. And + while it has accreted flexibility and additional options over the + years, it remains fast, memory efficient, portable, and easy to + integrate into your program. + + + BGET IMPLEMENTATION ASSUMPTIONS + =============================== + + BGET is written in as portable a dialect of C as possible. The + only fundamental assumption about the underlying hardware + architecture is that memory is allocated is a linear array which + can be addressed as a vector of C "char" objects. On segmented + address space architectures, this generally means that BGET should + be used to allocate storage within a single segment (although some + compilers simulate linear address spaces on segmented + architectures). On segmented architectures, then, BGET buffer + pools may not be larger than a segment, but since BGET allows any + number of separate buffer pools, there is no limit on the total + storage which can be managed, only on the largest individual + object which can be allocated. Machines with a linear address + architecture, such as the VAX, 680x0, Sparc, MIPS, or the Intel + 80386 and above in native mode, may use BGET without restriction. + + + GETTING STARTED WITH BGET + ========================= + + Although BGET can be configured in a multitude of fashions, there + are three basic ways of working with BGET. The functions + mentioned below are documented in the following section. Please + excuse the forward references which are made in the interest of + providing a roadmap to guide you to the BGET functions you're + likely to need. + + Embedded Applications + --------------------- + + Embedded applications typically have a fixed area of memory + dedicated to buffer allocation (often in a separate RAM address + space distinct from the ROM that contains the executable code). + To use BGET in such an environment, simply call bpool() with the + start address and length of the buffer pool area in RAM, then + allocate buffers with bget() and release them with brel(). + Embedded applications with very limited RAM but abundant CPU speed + may benefit by configuring BGET for BestFit allocation (which is + usually not worth it in other environments). + + Malloc() Emulation + ------------------ + + If the C library malloc() function is too slow, not present in + your development environment (for example, an a native Windows or + Macintosh program), or otherwise unsuitable, you can replace it + with BGET. Initially define a buffer pool of an appropriate size + with bpool()--usually obtained by making a call to the operating + system's low-level memory allocator. Then allocate buffers with + bget(), bgetz(), and bgetr() (the last two permit the allocation + of buffers initialised to zero and [inefficient] re-allocation of + existing buffers for compatibility with C library functions). + Release buffers by calling brel(). If a buffer allocation request + fails, obtain more storage from the underlying operating system, + add it to the buffer pool by another call to bpool(), and continue + execution. + + Automatic Storage Management + ---------------------------- + + You can use BGET as your application's native memory manager and + implement automatic storage pool expansion, contraction, and + optionally application-specific memory compaction by compiling + BGET with the BECtl variable defined, then calling bectl() and + supplying functions for storage compaction, acquisition, and + release, as well as a standard pool expansion increment. All of + these functions are optional (although it doesn't make much sense + to provide a release function without an acquisition function, + does it?). Once the call-back functions have been defined with + bectl(), you simply use bget() and brel() to allocate and release + storage as before. You can supply an initial buffer pool with + bpool() or rely on automatic allocation to acquire the entire + pool. When a call on bget() cannot be satisfied, BGET first + checks if a compaction function has been supplied. If so, it is + called (with the space required to satisfy the allocation request + and a sequence number to allow the compaction routine to be called + successively without looping). If the compaction function is able + to free any storage (it needn't know whether the storage it freed + was adequate) it should return a nonzero value, whereupon BGET + will retry the allocation request and, if it fails again, call the + compaction function again with the next-higher sequence number. + + If the compaction function returns zero, indicating failure to + free space, or no compaction function is defined, BGET next tests + whether a non-NULL allocation function was supplied to bectl(). + If so, that function is called with an argument indicating how + many bytes of additional space are required. This will be the + standard pool expansion increment supplied in the call to bectl() + unless the original bget() call requested a buffer larger than + this; buffers larger than the standard pool block can be managed + "off the books" by BGET in this mode. If the allocation function + succeeds in obtaining the storage, it returns a pointer to the new + block and BGET expands the buffer pool; if it fails, the + allocation request fails and returns NULL to the caller. If a + non-NULL release function is supplied, expansion blocks which + become totally empty are released to the global free pool by + passing their addresses to the release function. + + Equipped with appropriate allocation, release, and compaction + functions, BGET can be used as part of very sophisticated memory + management strategies, including garbage collection. (Note, + however, that BGET is *not* a garbage collector by itself, and + that developing such a system requires much additional logic and + careful design of the application's memory allocation strategy.) + + + BGET FUNCTION DESCRIPTIONS + ========================== + + Functions implemented in this file (some are enabled by certain of + the optional settings below): + + void bpool(void *buffer, bufsize len); + + Create a buffer pool of bytes, using the storage starting at + . You can call bpool() subsequently to contribute + additional storage to the overall buffer pool. + + void *bget(bufsize size); + + Allocate a buffer of bytes. The address of the buffer is + returned, or NULL if insufficient memory was available to allocate + the buffer. + + void *bgetz(bufsize size); + + Allocate a buffer of bytes and clear it to all zeroes. The + address of the buffer is returned, or NULL if insufficient memory + was available to allocate the buffer. + + void *bgetr(void *buffer, bufsize newsize); + + Reallocate a buffer previously allocated by bget(), changing its + size to and preserving all existing data. NULL is + returned if insufficient memory is available to reallocate the + buffer, in which case the original buffer remains intact. + + void brel(void *buf); + + Return the buffer , previously allocated by bget(), to the + free space pool. + + void bectl(int (*compact)(bufsize sizereq, int sequence), + void *(*acquire)(bufsize size), + void (*release)(void *buf), + bufsize pool_incr); + + Expansion control: specify functions through which the package may + compact storage (or take other appropriate action) when an + allocation request fails, and optionally automatically acquire + storage for expansion blocks when necessary, and release such + blocks when they become empty. If is non-NULL, whenever + a buffer allocation request fails, the function will be + called with arguments specifying the number of bytes (total buffer + size, including header overhead) required to satisfy the + allocation request, and a sequence number indicating the number of + consecutive calls on attempting to satisfy this + allocation request. The sequence number is 1 for the first call + on for a given allocation request, and increments on + subsequent calls, permitting the function to take + increasingly dire measures in an attempt to free up storage. If + the function returns a nonzero value, the allocation + attempt is re-tried. If returns 0 (as it must if it + isn't able to release any space or add storage to the buffer + pool), the allocation request fails, which can trigger automatic + pool expansion if the argument is non-NULL. At the time + the function is called, the state of the buffer + allocator is identical to that at the moment the allocation + request was made; consequently, the function may call + brel(), bpool(), bstats(), and/or directly manipulate the buffer + pool in any manner which would be valid were the application in + control. This does not, however, relieve the function + of the need to ensure that whatever actions it takes do not change + things underneath the application that made the allocation + request. For example, a function that released a buffer + in the process of being reallocated with bgetr() would lead to + disaster. Implementing a safe and effective mechanism + requires careful design of an application's memory architecture, + and cannot generally be easily retrofitted into existing code. + + If is non-NULL, that function will be called whenever an + allocation request fails. If the function succeeds in + allocating the requested space and returns a pointer to the new + area, allocation will proceed using the expanded buffer pool. If + cannot obtain the requested space, it should return NULL + and the entire allocation process will fail. + specifies the normal expansion block size. Providing an + function will cause subsequent bget() requests for buffers too + large to be managed in the linked-block scheme (in other words, + larger than minus the buffer overhead) to be satisfied + directly by calls to the function. Automatic release of + empty pool blocks will occur only if all pool blocks in the system + are the size given by . + + void bstats(bufsize *curalloc, bufsize *totfree, + bufsize *maxfree, long *nget, long *nrel); + + The amount of space currently allocated is stored into the + variable pointed to by . The total free space (sum of + all free blocks in the pool) is stored into the variable pointed + to by , and the size of the largest single block in the + free space pool is stored into the variable pointed to by + . The variables pointed to by and are + filled, respectively, with the number of successful (non-NULL + return) bget() calls and the number of brel() calls. + + void bstatse(bufsize *pool_incr, long *npool, + long *npget, long *nprel, + long *ndget, long *ndrel); + + Extended statistics: The expansion block size will be stored into + the variable pointed to by , or the negative thereof if + automatic expansion block releases are disabled. The number of + currently active pool blocks will be stored into the variable + pointed to by . The variables pointed to by and + will be filled with, respectively, the number of expansion + block acquisitions and releases which have occurred. The + variables pointed to by and will be filled with + the number of bget() and brel() calls, respectively, managed + through blocks directly allocated by the acquisition and release + functions. + + void bufdump(void *buf); + + The buffer pointed to by is dumped on standard output. + + void bpoold(void *pool, int dumpalloc, int dumpfree); + + All buffers in the buffer pool , previously initialised by a + call on bpool(), are listed in ascending memory address order. If + is nonzero, the contents of allocated buffers are + dumped; if is nonzero, the contents of free blocks are + dumped. + + int bpoolv(void *pool); + + The named buffer pool, previously initialised by a call on + bpool(), is validated for bad pointers, overwritten data, etc. If + compiled with NDEBUG not defined, any error generates an assertion + failure. Otherwise 1 is returned if the pool is valid, 0 if an + error is found. + + + BGET CONFIGURATION + ================== +*/ + +/* + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ST BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* #define BGET_ENABLE_ALL_OPTIONS */ +#ifdef BGET_ENABLE_OPTION +#define TestProg 20000 /* Generate built-in test program + if defined. The value specifies + how many buffer allocation attempts + the test program should make. */ + +#define SizeQuant 4 /* Buffer allocation size quantum: + all buffers allocated are a + multiple of this size. This + MUST be a power of two. */ + +#define BufDump 1 /* Define this symbol to enable the + bpoold() function which dumps the + buffers in a buffer pool. */ + +#define BufValid 1 /* Define this symbol to enable the + bpoolv() function for validating + a buffer pool. */ + +#define DumpData 1 /* Define this symbol to enable the + bufdump() function which allows + dumping the contents of an allocated + or free buffer. */ + +#define BufStats 1 /* Define this symbol to enable the + bstats() function which calculates + the total free space in the buffer + pool, the largest available + buffer, and the total space + currently allocated. */ + +#define FreeWipe 1 /* Wipe free buffers to a guaranteed + pattern of garbage to trip up + miscreants who attempt to use + pointers into released buffers. */ + +#define BestFit 1 /* Use a best fit algorithm when + searching for space for an + allocation request. This uses + memory more efficiently, but + allocation will be much slower. */ + +#define BECtl 1 /* Define this symbol to enable the + bectl() function for automatic + pool space control. */ +#endif + +#include +#include + +#ifdef lint +#define NDEBUG /* Exits in asserts confuse lint */ +/* LINTLIBRARY */ /* Don't complain about def, no ref */ +extern char *sprintf(); /* Sun includes don't define sprintf */ +#endif + +#include +#include + +#ifdef BufDump /* BufDump implies DumpData */ +#ifndef DumpData +#define DumpData 1 +#endif +#endif + +#ifdef DumpData +#include +#endif + +#ifdef __KERNEL__ +#ifdef CFG_CORE_BGET_BESTFIT +#define BestFit 1 +#endif +#endif + +/* Declare the interface, including the requested buffer size type, + bufsize. */ + +#include "bget.h" + +#define MemSize int /* Type for size arguments to memxxx() + functions such as memcmp(). */ + +/* Queue links */ + +struct qlinks { + struct bfhead *flink; /* Forward link */ + struct bfhead *blink; /* Backward link */ +}; + +/* Header in allocated and free buffers */ + +struct bhead { + bufsize prevfree; /* Relative link back to previous + free buffer in memory or 0 if + previous buffer is allocated. */ + bufsize bsize; /* Buffer size: positive if free, + negative if allocated. */ +}; +#define BH(p) ((struct bhead *) (p)) + +/* Header in directly allocated buffers (by acqfcn) */ + +struct bdhead { + bufsize tsize; /* Total size, including overhead */ + bufsize offs; /* Offset from allocated buffer */ + struct bhead bh; /* Common header */ +}; +#define BDH(p) ((struct bdhead *) (p)) + +/* Header in free buffers */ + +struct bfhead { + struct bhead bh; /* Common allocated/free header */ + struct qlinks ql; /* Links on free list */ +}; +#define BFH(p) ((struct bfhead *) (p)) + +/* Poolset definition */ +struct bpoolset { + struct bfhead freelist; +#ifdef BufStats + bufsize totalloc; /* Total space currently allocated */ + long numget; /* Number of bget() calls */ + long numrel; /* Number of brel() calls */ +#ifdef BECtl + long numpblk; /* Number of pool blocks */ + long numpget; /* Number of block gets and rels */ + long numprel; + long numdget; /* Number of direct gets and rels */ + long numdrel; +#endif /* BECtl */ +#endif /* BufStats */ + +#ifdef BECtl + /* Automatic expansion block management functions */ + + int (*compfcn) _((bufsize sizereq, int sequence)); + void *(*acqfcn) _((bufsize size)); + void (*relfcn) _((void *buf)); + + bufsize exp_incr; /* Expansion block size */ + bufsize pool_len; /* 0: no bpool calls have been made + -1: not all pool blocks are + the same size + >0: (common) block size for all + bpool calls made so far + */ +#endif +}; + +/* Minimum allocation quantum: */ + +#define QLSize (sizeof(struct qlinks)) +#define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize) + +#define V (void) /* To denote unwanted returned values */ + +/* End sentinel: value placed in bsize field of dummy block delimiting + end of pool block. The most negative number which will fit in a + bufsize, defined in a way that the compiler will accept. */ + +#define ESent ((bufsize) (-(((1L << (sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2)) + +static bufsize buf_get_pos(struct bfhead *bf, bufsize align, bufsize hdr_size, + bufsize size) +{ + unsigned long buf = 0; + bufsize pos = 0; + + if (bf->bh.bsize < size) + return -1; + + /* + * plus sizeof(struct bhead) and hdr_size since buf will follow just + * after a struct bhead and an eventual extra header. + */ + buf = (unsigned long)bf + bf->bh.bsize - size + sizeof(struct bhead) + + hdr_size; + buf &= ~(align - 1); + pos = buf - (unsigned long)bf - sizeof(struct bhead) - hdr_size; + + if (pos == 0) /* exact match */ + return pos; + if (pos >= SizeQ + sizeof(struct bhead)) /* room for an empty buffer */ + return pos; + + return -1; +} + +/* BGET -- Allocate a buffer. */ + +void *bget(requested_align, hdr_size, requested_size, poolset) + bufsize requested_align; + bufsize hdr_size; + bufsize requested_size; + struct bpoolset *poolset; +{ + bufsize align = requested_align; + bufsize size = requested_size; + bufsize pos; + struct bfhead *b; +#ifdef BestFit + struct bfhead *best; +#endif + void *buf; +#ifdef BECtl + int compactseq = 0; +#endif + + assert(size > 0); + COMPILE_TIME_ASSERT(BGET_HDR_QUANTUM == SizeQ); + + if (align < 0 || (align > 0 && !IS_POWER_OF_TWO((unsigned long)align))) + return NULL; + if (hdr_size % BGET_HDR_QUANTUM != 0) + return NULL; + + if (size < SizeQ) { /* Need at least room for the */ + size = SizeQ; /* queue links. */ + } + if (align < SizeQ) + align = SizeQ; +#ifdef SizeQuant +#if SizeQuant > 1 + if (ADD_OVERFLOW(size, SizeQuant - 1, &size)) + return NULL; + + size = ROUNDDOWN(size, SizeQuant); +#endif +#endif + + /* Add overhead in allocated buffer to size required. */ + if (ADD_OVERFLOW(size, sizeof(struct bhead), &size)) + return NULL; + if (ADD_OVERFLOW(size, hdr_size, &size)) + return NULL; + +#ifdef BECtl + /* If a compact function was provided in the call to bectl(), wrap + a loop around the allocation process to allow compaction to + intervene in case we don't find a suitable buffer in the chain. */ + + while (1) { +#endif + b = poolset->freelist.ql.flink; +#ifdef BestFit + best = &poolset->freelist; +#endif + + + /* Scan the free list searching for the first buffer big enough + to hold the requested size buffer. */ + +#ifdef BestFit + while (b != &poolset->freelist) { + assert(b->bh.prevfree == 0); + pos = buf_get_pos(b, align, hdr_size, size); + if (pos >= 0) { + if ((best == &poolset->freelist) || + (b->bh.bsize < best->bh.bsize)) { + best = b; + } + } + b = b->ql.flink; /* Link to next buffer */ + } + b = best; +#endif /* BestFit */ + + while (b != &poolset->freelist) { + pos = buf_get_pos(b, align, hdr_size, size); + if (pos >= 0) { + struct bhead *b_alloc = BH((char *)b + pos); + struct bhead *b_next = BH((char *)b + b->bh.bsize); + + assert(b_next->prevfree == b->bh.bsize); + + /* + * Zero the back pointer in the next buffer in memory + * to indicate that this buffer is allocated. + */ + b_next->prevfree = 0; + + assert(b->ql.blink->ql.flink == b); + assert(b->ql.flink->ql.blink == b); + + if (pos == 0) { + /* + * Need to allocate from the beginning of this free block. + * Unlink the block and mark it as allocated. + */ + b->ql.blink->ql.flink = b->ql.flink; + b->ql.flink->ql.blink = b->ql.blink; + + /* Negate size to mark buffer allocated. */ + b->bh.bsize = -b->bh.bsize; + } else { + /* + * Carve out the memory allocation from the end of this + * free block. Negative size to mark buffer allocated. + */ + b_alloc->bsize = -(b->bh.bsize - pos); + b_alloc->prevfree = pos; + b->bh.bsize = pos; + } + + assert(b_alloc->bsize < 0); + /* + * At this point is b_alloc pointing to the allocated + * buffer and b_next at the buffer following. b might be a + * free block or a used block now. + */ + if (-b_alloc->bsize - size > SizeQ + sizeof(struct bhead)) { + /* + * b_alloc has too much unused memory at the + * end we need to split the block and register that + * last part as free. + */ + b = BFH((char *)b_alloc + size); + b->bh.bsize = -b_alloc->bsize - size; + b->bh.prevfree = 0; + b_alloc->bsize += b->bh.bsize; + + assert(poolset->freelist.ql.blink->ql.flink == + &poolset->freelist); + assert(poolset->freelist.ql.flink->ql.blink == + &poolset->freelist); + b->ql.flink = &poolset->freelist; + b->ql.blink = poolset->freelist.ql.blink; + poolset->freelist.ql.blink = b; + b->ql.blink->ql.flink = b; + + assert(BH((char *)b + b->bh.bsize) == b_next); + b_next->prevfree = b->bh.bsize; + } + +#ifdef BufStats + poolset->totalloc -= b_alloc->bsize; + poolset->numget++; /* Increment number of bget() calls */ +#endif + buf = (char *)b_alloc + sizeof(struct bhead); + return buf; + } + b = b->ql.flink; /* Link to next buffer */ + } +#ifdef BECtl + + /* We failed to find a buffer. If there's a compact function + defined, notify it of the size requested. If it returns + TRUE, try the allocation again. */ + + if ((poolset->compfcn == NULL) || + (!(poolset->compfcn)(size, ++compactseq))) { + break; + } + } + + /* No buffer available with requested size free. */ + + /* Don't give up yet -- look in the reserve supply. */ + + if (poolset->acqfcn != NULL) { + if (size > exp_incr - sizeof(struct bfhead) - align) { + + /* Request is too large to fit in a single expansion + block. Try to satisy it by a direct buffer acquisition. */ + char *p; + + size += sizeof(struct bdhead) - sizeof(struct bhead); + if (align > QLSize) + size += align; + p = poolset->acqfcn(size); + if (p != NULL) { + struct bdhead *bdh; + + if (align <= QLSize) { + bdh = BDH(p); + buf = bdh + 1; + } else { + unsigned long tp = (unsigned long)p; + + tp += sizeof(*bdh) + hdr_size + align; + tp &= ~(align - 1); + tp -= hdr_size; + buf = (void *)tp; + bdh = BDH((char *)buf - sizeof(*bdh)); + } + + /* Mark the buffer special by setting the size field + of its header to zero. */ + bdh->bh.bsize = 0; + bdh->bh.prevfree = 0; + bdh->tsize = size; + bdh->offs = (unsigned long)bdh - (unsigned long)p; +#ifdef BufStats + poolset->totalloc += size; + poolset->numget++; /* Increment number of bget() calls */ + poolset->numdget++; /* Direct bget() call count */ +#endif + return buf; + } + + } else { + + /* Try to obtain a new expansion block */ + + void *newpool; + + if ((newpool = poolset->acqfcn((bufsize) exp_incr)) != NULL) { + bpool(newpool, exp_incr, poolset); + buf = bget(align, hdr_size, requested_size, pool); /* This can't, I say, can't + get into a loop. */ + return buf; + } + } + } + + /* Still no buffer available */ + +#endif /* BECtl */ + + return NULL; +} + +/* BGETZ -- Allocate a buffer and clear its contents to zero. We clear + the entire contents of the buffer to zero, not just the + region requested by the caller. */ + +void *bgetz(align, hdr_size, size, poolset) + bufsize align; + bufsize hdr_size; + bufsize size; + struct bpoolset *poolset; +{ + char *buf = (char *) bget(align, hdr_size, size, poolset); + + if (buf != NULL) { + struct bhead *b; + bufsize rsize; + + b = BH(buf - sizeof(struct bhead)); + rsize = -(b->bsize); + if (rsize == 0) { + struct bdhead *bd; + + bd = BDH(buf - sizeof(struct bdhead)); + rsize = bd->tsize - sizeof(struct bdhead) - bd->offs; + } else { + rsize -= sizeof(struct bhead); + } + assert(rsize >= size); + V memset_unchecked(buf, 0, (MemSize) rsize); + } + return ((void *) buf); +} + +/* BGETR -- Reallocate a buffer. This is a minimal implementation, + simply in terms of brel() and bget(). It could be + enhanced to allow the buffer to grow into adjacent free + blocks and to avoid moving data unnecessarily. */ + +void *bgetr(buf, align, hdr_size, size, poolset) + void *buf; + bufsize align; + bufsize hdr_size; + bufsize size; + struct bpoolset *poolset; +{ + void *nbuf; + bufsize osize; /* Old size of buffer */ + struct bhead *b; + + if ((nbuf = bget(align, hdr_size, size, poolset)) == NULL) { /* Acquire new buffer */ + return NULL; + } + if (buf == NULL) { + return nbuf; + } + b = BH(((char *) buf) - sizeof(struct bhead)); + osize = -b->bsize; +#ifdef BECtl + if (osize == 0) { + /* Buffer acquired directly through acqfcn. */ + struct bdhead *bd; + + bd = BDH(((char *) buf) - sizeof(struct bdhead)); + osize = bd->tsize - sizeof(struct bdhead) - bd->offs; + } else +#endif + osize -= sizeof(struct bhead); + assert(osize > 0); + V memcpy_unchecked((char *) nbuf, (char *) buf, /* Copy the data */ + (MemSize) ((size < osize) ? size : osize)); +#ifndef __KERNEL__ + /* User space reallocations are always zeroed */ + if (size > osize) + V memset_unchecked((char *) nbuf + osize, 0, size - osize); +#endif + brel(buf, poolset, false /* !wipe */); + return nbuf; +} + +/* BREL -- Release a buffer. */ + +void brel(buf, poolset, wipe) + void *buf; + struct bpoolset *poolset; + int wipe; +{ + struct bfhead *b, *bn; + char *wipe_start; + bufsize wipe_size; + + b = BFH(((char *) buf) - sizeof(struct bhead)); +#ifdef BufStats + poolset->numrel++; /* Increment number of brel() calls */ +#endif + assert(buf != NULL); + +#ifdef FreeWipe + wipe = true; +#endif +#ifdef BECtl + if (b->bh.bsize == 0) { /* Directly-acquired buffer? */ + struct bdhead *bdh; + + bdh = BDH(((char *) buf) - sizeof(struct bdhead)); + assert(b->bh.prevfree == 0); +#ifdef BufStats + poolset->totalloc -= bdh->tsize; + assert(poolset->totalloc >= 0); + poolset->numdrel++; /* Number of direct releases */ +#endif /* BufStats */ + if (wipe) { + V memset_unchecked((char *) buf, 0x55, + (MemSize) (bdh->tsize - + sizeof(struct bdhead))); + } + assert(poolset->relfcn != NULL); + poolset->relfcn((char *)buf - sizeof(struct bdhead) - bdh->offs); /* Release it directly. */ + return; + } +#endif /* BECtl */ + + /* Buffer size must be negative, indicating that the buffer is + allocated. */ + + if (b->bh.bsize >= 0) { + bn = NULL; + } + assert(b->bh.bsize < 0); + + /* Back pointer in next buffer must be zero, indicating the + same thing: */ + + assert(BH((char *) b - b->bh.bsize)->prevfree == 0); + +#ifdef BufStats + poolset->totalloc += b->bh.bsize; + assert(poolset->totalloc >= 0); +#endif + + /* If the back link is nonzero, the previous buffer is free. */ + + if (b->bh.prevfree != 0) { + + /* The previous buffer is free. Consolidate this buffer with it + by adding the length of this buffer to the previous free + buffer. Note that we subtract the size in the buffer being + released, since it's negative to indicate that the buffer is + allocated. */ + + register bufsize size = b->bh.bsize; + + /* Only wipe the current buffer, including bfhead. */ + wipe_start = (char *)b; + wipe_size = -size; + + /* Make the previous buffer the one we're working on. */ + assert(BH((char *) b - b->bh.prevfree)->bsize == b->bh.prevfree); + b = BFH(((char *) b) - b->bh.prevfree); + b->bh.bsize -= size; + } else { + + /* The previous buffer isn't allocated. Insert this buffer + on the free list as an isolated free block. */ + + assert(poolset->freelist.ql.blink->ql.flink == &poolset->freelist); + assert(poolset->freelist.ql.flink->ql.blink == &poolset->freelist); + b->ql.flink = &poolset->freelist; + b->ql.blink = poolset->freelist.ql.blink; + poolset->freelist.ql.blink = b; + b->ql.blink->ql.flink = b; + b->bh.bsize = -b->bh.bsize; + + wipe_start = (char *)b + sizeof(struct bfhead); + wipe_size = b->bh.bsize - sizeof(struct bfhead); + } + + /* Now we look at the next buffer in memory, located by advancing from + the start of this buffer by its size, to see if that buffer is + free. If it is, we combine this buffer with the next one in + memory, dechaining the second buffer from the free list. */ + + bn = BFH(((char *) b) + b->bh.bsize); + if (bn->bh.bsize > 0) { + + /* The buffer is free. Remove it from the free list and add + its size to that of our buffer. */ + + assert(BH((char *) bn + bn->bh.bsize)->prevfree == bn->bh.bsize); + assert(bn->ql.blink->ql.flink == bn); + assert(bn->ql.flink->ql.blink == bn); + bn->ql.blink->ql.flink = bn->ql.flink; + bn->ql.flink->ql.blink = bn->ql.blink; + b->bh.bsize += bn->bh.bsize; + + /* Finally, advance to the buffer that follows the newly + consolidated free block. We must set its backpointer to the + head of the consolidated free block. We know the next block + must be an allocated block because the process of recombination + guarantees that two free blocks will never be contiguous in + memory. */ + + bn = BFH(((char *) b) + b->bh.bsize); + /* Only bfhead of next buffer needs to be wiped */ + wipe_size += sizeof(struct bfhead); + } + if (wipe) { + V memset_unchecked(wipe_start, 0x55, wipe_size); + } + assert(bn->bh.bsize < 0); + + /* The next buffer is allocated. Set the backpointer in it to point + to this buffer; the previous free buffer in memory. */ + + bn->bh.prevfree = b->bh.bsize; + +#ifdef BECtl + + /* If a block-release function is defined, and this free buffer + constitutes the entire block, release it. Note that pool_len + is defined in such a way that the test will fail unless all + pool blocks are the same size. */ + + if (poolset->relfcn != NULL && + ((bufsize) b->bh.bsize) == (pool_len - sizeof(struct bhead))) { + + assert(b->bh.prevfree == 0); + assert(BH((char *) b + b->bh.bsize)->bsize == ESent); + assert(BH((char *) b + b->bh.bsize)->prevfree == b->bh.bsize); + /* Unlink the buffer from the free list */ + b->ql.blink->ql.flink = b->ql.flink; + b->ql.flink->ql.blink = b->ql.blink; + + poolset->relfcn(b); +#ifdef BufStats + poolset->numprel++; /* Nr of expansion block releases */ + poolset->numpblk--; /* Total number of blocks */ + assert(numpblk == numpget - numprel); +#endif /* BufStats */ + } +#endif /* BECtl */ +} + +#ifdef BECtl + +/* BECTL -- Establish automatic pool expansion control */ + +void bectl(compact, acquire, release, pool_incr, poolset) + int (*compact) _((bufsize sizereq, int sequence)); + void *(*acquire) _((bufsize size)); + void (*release) _((void *buf)); + bufsize pool_incr; + struct bpoolset *poolset; +{ + poolset->compfcn = compact; + poolset->acqfcn = acquire; + poolset->relfcn = release; + poolset->exp_incr = pool_incr; +} +#endif + +/* BPOOL -- Add a region of memory to the buffer pool. */ + +void bpool(buf, len, poolset) + void *buf; + bufsize len; + struct bpoolset *poolset; +{ + struct bfhead *b = BFH(buf); + struct bhead *bn; + +#ifdef SizeQuant + len &= ~(SizeQuant - 1); +#endif +#ifdef BECtl + if (poolset->pool_len == 0) { + pool_len = len; + } else if (len != poolset->pool_len) { + poolset->pool_len = -1; + } +#ifdef BufStats + poolset->numpget++; /* Number of block acquisitions */ + poolset->numpblk++; /* Number of blocks total */ + assert(poolset->numpblk == poolset->numpget - poolset->numprel); +#endif /* BufStats */ +#endif /* BECtl */ + + /* Since the block is initially occupied by a single free buffer, + it had better not be (much) larger than the largest buffer + whose size we can store in bhead.bsize. */ + + assert(len - sizeof(struct bhead) <= -((bufsize) ESent + 1)); + + /* Clear the backpointer at the start of the block to indicate that + there is no free block prior to this one. That blocks + recombination when the first block in memory is released. */ + + b->bh.prevfree = 0; + + /* Chain the new block to the free list. */ + + assert(poolset->freelist.ql.blink->ql.flink == &poolset->freelist); + assert(poolset->freelist.ql.flink->ql.blink == &poolset->freelist); + b->ql.flink = &poolset->freelist; + b->ql.blink = poolset->freelist.ql.blink; + poolset->freelist.ql.blink = b; + b->ql.blink->ql.flink = b; + + /* Create a dummy allocated buffer at the end of the pool. This dummy + buffer is seen when a buffer at the end of the pool is released and + blocks recombination of the last buffer with the dummy buffer at + the end. The length in the dummy buffer is set to the largest + negative number to denote the end of the pool for diagnostic + routines (this specific value is not counted on by the actual + allocation and release functions). */ + + len -= sizeof(struct bhead); + b->bh.bsize = (bufsize) len; +#ifdef FreeWipe + V memset_unchecked(((char *) b) + sizeof(struct bfhead), 0x55, + (MemSize) (len - sizeof(struct bfhead))); +#endif + bn = BH(((char *) b) + len); + bn->prevfree = (bufsize) len; + /* Definition of ESent assumes two's complement! */ + assert((~0) == -1); + bn->bsize = ESent; +} + +#ifdef BufStats + +/* BSTATS -- Return buffer allocation free space statistics. */ + +void bstats(curalloc, totfree, maxfree, nget, nrel, poolset) + bufsize *curalloc, *totfree, *maxfree; + long *nget, *nrel; + struct bpoolset *poolset; +{ + struct bfhead *b = poolset->freelist.ql.flink; + + *nget = poolset->numget; + *nrel = poolset->numrel; + *curalloc = poolset->totalloc; + *totfree = 0; + *maxfree = -1; + while (b != &poolset->freelist) { + assert(b->bh.bsize > 0); + *totfree += b->bh.bsize; + if (b->bh.bsize > *maxfree) { + *maxfree = b->bh.bsize; + } + b = b->ql.flink; /* Link to next buffer */ + } +} + +#ifdef BECtl + +/* BSTATSE -- Return extended statistics */ + +void bstatse(pool_incr, npool, npget, nprel, ndget, ndrel, poolset) + bufsize *pool_incr; + long *npool, *npget, *nprel, *ndget, *ndrel; + struct bpoolset *poolset; +{ + *pool_incr = (poolset->pool_len < 0) ? + -poolset->exp_incr : poolset->exp_incr; + *npool = poolset->numpblk; + *npget = poolset->numpget; + *nprel = poolset->numprel; + *ndget = poolset->numdget; + *ndrel = poolset->numdrel; +} +#endif /* BECtl */ +#endif /* BufStats */ + +#ifdef DumpData + +/* BUFDUMP -- Dump the data in a buffer. This is called with the user + data pointer, and backs up to the buffer header. It will + dump either a free block or an allocated one. */ + +void bufdump(buf) + void *buf; +{ + struct bfhead *b; + unsigned char *bdump; + bufsize bdlen; + + b = BFH(((char *) buf) - sizeof(struct bhead)); + assert(b->bh.bsize != 0); + if (b->bh.bsize < 0) { + bdump = (unsigned char *) buf; + bdlen = (-b->bh.bsize) - sizeof(struct bhead); + } else { + bdump = (unsigned char *) (((char *) b) + sizeof(struct bfhead)); + bdlen = b->bh.bsize - sizeof(struct bfhead); + } + + while (bdlen > 0) { + int i, dupes = 0; + bufsize l = bdlen; + char bhex[50], bascii[20]; + + if (l > 16) { + l = 16; + } + + for (i = 0; i < l; i++) { + V snprintf(bhex + i * 3, sizeof(bhex) - i * 3, "%02X ", + bdump[i]); + bascii[i] = isprint(bdump[i]) ? bdump[i] : ' '; + } + bascii[i] = 0; + V printf("%-48s %s\n", bhex, bascii); + bdump += l; + bdlen -= l; + while ((bdlen > 16) && (memcmp((char *) (bdump - 16), + (char *) bdump, 16) == 0)) { + dupes++; + bdump += 16; + bdlen -= 16; + } + if (dupes > 1) { + V printf( + " (%d lines [%d bytes] identical to above line skipped)\n", + dupes, dupes * 16); + } else if (dupes == 1) { + bdump -= 16; + bdlen += 16; + } + } +} +#endif + +#ifdef BufDump + +/* BPOOLD -- Dump a buffer pool. The buffer headers are always listed. + If DUMPALLOC is nonzero, the contents of allocated buffers + are dumped. If DUMPFREE is nonzero, free blocks are + dumped as well. If FreeWipe checking is enabled, free + blocks which have been clobbered will always be dumped. */ + +void bpoold(buf, dumpalloc, dumpfree) + void *buf; + int dumpalloc, dumpfree; +{ + struct bfhead *b = BFH(buf); + + while (b->bh.bsize != ESent) { + bufsize bs = b->bh.bsize; + + if (bs < 0) { + bs = -bs; + V printf("Allocated buffer: size %6ld bytes.\n", (long) bs); + if (dumpalloc) { + bufdump((void *) (((char *) b) + sizeof(struct bhead))); + } + } else { + char *lerr = ""; + + assert(bs > 0); + if ((b->ql.blink->ql.flink != b) || + (b->ql.flink->ql.blink != b)) { + lerr = " (Bad free list links)"; + } + V printf("Free block: size %6ld bytes.%s\n", + (long) bs, lerr); +#ifdef FreeWipe + lerr = ((char *) b) + sizeof(struct bfhead); + if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) || + (memcmp(lerr, lerr + 1, + (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) { + V printf( + "(Contents of above free block have been overstored.)\n"); + bufdump((void *) (((char *) b) + sizeof(struct bhead))); + } else +#endif + if (dumpfree) { + bufdump((void *) (((char *) b) + sizeof(struct bhead))); + } + } + b = BFH(((char *) b) + bs); + } +} +#endif /* BufDump */ + +#ifdef BufValid + +/* BPOOLV -- Validate a buffer pool. If NDEBUG isn't defined, + any error generates an assertion failure. */ + +int bpoolv(buf) + void *buf; +{ + struct bfhead *b = BFH(buf); + + while (b->bh.bsize != ESent) { + bufsize bs = b->bh.bsize; + + if (bs < 0) { + bs = -bs; + } else { + const char *lerr = ""; + + assert(bs > 0); + if (bs <= 0) { + return 0; + } + if ((b->ql.blink->ql.flink != b) || + (b->ql.flink->ql.blink != b)) { + V printf("Free block: size %6ld bytes. (Bad free list links)\n", + (long) bs); + assert(0); + return 0; + } +#ifdef FreeWipe + lerr = ((char *) b) + sizeof(struct bfhead); + if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) || + (memcmp(lerr, lerr + 1, + (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) { + V printf( + "(Contents of above free block have been overstored.)\n"); + bufdump((void *) (((char *) b) + sizeof(struct bhead))); + assert(0); + return 0; + } +#endif + } + b = BFH(((char *) b) + bs); + } + return 1; +} +#endif /* BufValid */ + + /***********************\ + * * + * Built-in test program * + * * + \***********************/ + +#if !defined(__KERNEL__) && !defined(__LDELF__) && defined(CFG_TA_BGET_TEST) + +#define TestProg 20000 + +#ifdef BECtl +#define PoolSize 300000 /* Test buffer pool size */ +#else +#define PoolSize 50000 /* Test buffer pool size */ +#endif +#define ExpIncr 32768 /* Test expansion block size */ +#define CompactTries 10 /* Maximum tries at compacting */ + +#define dumpAlloc 0 /* Dump allocated buffers ? */ +#define dumpFree 0 /* Dump free buffers ? */ + +static char *bchain = NULL; /* Our private buffer chain */ +static char *bp = NULL; /* Our initial buffer pool */ + +#ifdef UsingFloat +#include +#endif + +static unsigned long int next = 1; + +static void *(*mymalloc)(size_t size); +static void (*myfree)(void *ptr); + +static struct bpoolset mypoolset = { + .freelist = { + .bh = { 0, 0}, + .ql = { &mypoolset.freelist, &mypoolset.freelist}, + } +}; + +/* Return next random integer */ + +static int myrand(void) +{ + next = next * 1103515245L + 12345; + return (unsigned int) (next / 65536L) % 32768L; +} + +/* Set seed for random generator */ + +static void mysrand(unsigned int seed) +{ + next = seed; +} + +/* STATS -- Edit statistics returned by bstats() or bstatse(). */ + +static void stats(const char *when __maybe_unused, + struct bpoolset *poolset __maybe_unused) +{ +#ifdef BufStats + bufsize cural, totfree, maxfree; + long nget, nfree; +#endif +#ifdef BECtl + bufsize pincr; + long totblocks, npget, nprel, ndget, ndrel; +#endif + +#ifdef BufStats + bstats(&cural, &totfree, &maxfree, &nget, &nfree, poolset); + V printf( + "%s: %ld gets, %ld releases. %ld in use, %ld free, largest = %ld\n", + when, nget, nfree, (long) cural, (long) totfree, (long) maxfree); +#endif +#ifdef BECtl + bstatse(&pincr, &totblocks, &npget, &nprel, &ndget, &ndrel, poolset); + V printf( + " Blocks: size = %ld, %ld (%ld bytes) in use, %ld gets, %ld frees\n", + (long)pincr, totblocks, pincr * totblocks, npget, nprel); + V printf(" %ld direct gets, %ld direct frees\n", ndget, ndrel); +#endif /* BECtl */ +} + +#ifdef BECtl +static int protect = 0; /* Disable compaction during bgetr() */ + +/* BCOMPACT -- Compaction call-back function. */ + +static int bcompact(bsize, seq) + bufsize bsize; + int seq; +{ +#ifdef CompactTries + char *bc = bchain; + int i = myrand() & 0x3; + +#ifdef COMPACTRACE + V printf("Compaction requested. %ld bytes needed, sequence %d.\n", + (long) bsize, seq); +#endif + + if (protect || (seq > CompactTries)) { +#ifdef COMPACTRACE + V printf("Compaction gave up.\n"); +#endif + return 0; + } + + /* Based on a random cast, release a random buffer in the list + of allocated buffers. */ + + while (i > 0 && bc != NULL) { + bc = *((char **) bc); + i--; + } + if (bc != NULL) { + char *fb; + + fb = *((char **) bc); + if (fb != NULL) { + *((char **) bc) = *((char **) fb); + brel((void *) fb); + return 1; + } + } + +#ifdef COMPACTRACE + V printf("Compaction bailed out.\n"); +#endif +#endif /* CompactTries */ + return 0; +} + +/* BEXPAND -- Expand pool call-back function. */ + +static void *bexpand(size) + bufsize size; +{ + void *np = NULL; + bufsize cural, totfree, maxfree; + long nget, nfree; + + /* Don't expand beyond the total allocated size given by PoolSize. */ + + bstats(&cural, &totfree, &maxfree, &nget, &nfree); + + if (cural < PoolSize) { + np = (void *) mymalloc((unsigned) size); + } +#ifdef EXPTRACE + V printf("Expand pool by %ld -- %s.\n", (long) size, + np == NULL ? "failed" : "succeeded"); +#endif + return np; +} + +/* BSHRINK -- Shrink buffer pool call-back function. */ + +static void bshrink(buf) + void *buf; +{ + if (((char *) buf) == bp) { +#ifdef EXPTRACE + V printf("Initial pool released.\n"); +#endif + bp = NULL; + } +#ifdef EXPTRACE + V printf("Shrink pool.\n"); +#endif + myfree((char *) buf); +} + +#endif /* BECtl */ + +/* Restrict buffer requests to those large enough to contain our pointer and + small enough for the CPU architecture. */ + +static bufsize blimit(bufsize bs) +{ + if (bs < sizeof(char *)) { + bs = sizeof(char *); + } + + /* This is written out in this ugly fashion because the + cool expression in sizeof(int) that auto-configured + to any length int befuddled some compilers. */ + + if (sizeof(int) == 2) { + if (bs > 32767) { + bs = 32767; + } + } else { + if (bs > 200000) { + bs = 200000; + } + } + return bs; +} + +int bget_main_test(void *(*malloc_func)(size_t), void (*free_func)(void *)) +{ + int i; +#ifdef UsingFloat + double x; +#endif + + mymalloc = malloc_func; + myfree = free_func; + + /* Seed the random number generator. If Repeatable is defined, we + always use the same seed. Otherwise, we seed from the clock to + shake things up from run to run. */ + + mysrand(1234); + + /* Compute x such that pow(x, p) ranges between 1 and 4*ExpIncr as + p ranges from 0 to ExpIncr-1, with a concentration in the lower + numbers. */ + +#ifdef UsingFloat + x = 4.0 * ExpIncr; + x = log(x); + x = exp(log(4.0 * ExpIncr) / (ExpIncr - 1.0)); +#endif + +#ifdef BECtl + bectl(bcompact, bexpand, bshrink, (bufsize) ExpIncr, &mypoolset); + bp = mymalloc(ExpIncr); + assert(bp != NULL); + bpool((void *) bp, (bufsize) ExpIncr); +#else + bp = mymalloc(PoolSize); + assert(bp != NULL); + bpool((void *) bp, (bufsize) PoolSize, &mypoolset); +#endif + + stats("Create pool", &mypoolset); +#ifdef BufValid + V bpoolv((void *) bp); +#endif +#ifdef BufDump + bpoold((void *) bp, dumpAlloc, dumpFree); +#endif + + for (i = 0; i < TestProg; i++) { + char *cb; +#ifdef UsingFloat + bufsize bs = pow(x, (double) (myrand() & (ExpIncr - 1))); +#else + bufsize bs = (myrand() & (ExpIncr * 4 - 1)) / (1 << (myrand() & 0x7)); +#endif + bufsize align = 0; + bufsize hdr_size = 0; + + switch (rand() & 0x3) { + case 1: + align = 32; + break; + case 2: + align = 64; + break; + case 3: + align = 128; + break; + default: + break; + } + + hdr_size = (rand() & 0x3) * BGET_HDR_QUANTUM; + + assert(bs <= (((bufsize) 4) * ExpIncr)); + bs = blimit(bs); + if (myrand() & 0x400) { + cb = (char *) bgetz(align, hdr_size, bs, &mypoolset); + } else { + cb = (char *) bget(align, hdr_size, bs, &mypoolset); + } + if (cb == NULL) { +#ifdef EasyOut + break; +#else + char *bc = bchain; + + if (bc != NULL) { + char *fb; + + fb = *((char **) bc); + if (fb != NULL) { + *((char **) bc) = *((char **) fb); + brel((void *) fb, &mypoolset, true/*wipe*/); + } + } + continue; +#endif + } + assert(!align || !(((unsigned long)cb + hdr_size) & (align - 1))); + *((char **) cb) = (char *) bchain; + bchain = cb; + + /* Based on a random cast, release a random buffer in the list + of allocated buffers. */ + + if ((myrand() & 0x10) == 0) { + char *bc = bchain; + int j = myrand() & 0x3; + + while (j > 0 && bc != NULL) { + bc = *((char **) bc); + j--; + } + if (bc != NULL) { + char *fb; + + fb = *((char **) bc); + if (fb != NULL) { + *((char **) bc) = *((char **) fb); + brel((void *) fb, &mypoolset, true/*wipe*/); + } + } + } + + /* Based on a random cast, reallocate a random buffer in the list + to a random size */ + + if ((myrand() & 0x20) == 0) { + char *bc = bchain; + int j = myrand() & 0x3; + + while (j > 0 && bc != NULL) { + bc = *((char **) bc); + j--; + } + if (bc != NULL) { + char *fb; + + fb = *((char **) bc); + if (fb != NULL) { + char *newb; + +#ifdef UsingFloat + bs = pow(x, (double) (myrand() & (ExpIncr - 1))); +#else + bs = (rand() & (ExpIncr * 4 - 1)) / (1 << (rand() & 0x7)); +#endif + bs = blimit(bs); +#ifdef BECtl + protect = 1; /* Protect against compaction */ +#endif + newb = (char *) bgetr((void *) fb, align, hdr_size, bs, &mypoolset); +#ifdef BECtl + protect = 0; +#endif + if (newb != NULL) { + assert(!align || !(((unsigned long)newb + hdr_size) & + (align - 1))); + *((char **) bc) = newb; + } + } + } + } + } + stats("\nAfter allocation", &mypoolset); + if (bp != NULL) { +#ifdef BufValid + V bpoolv((void *) bp); +#endif +#ifdef BufDump + bpoold((void *) bp, dumpAlloc, dumpFree); +#endif + } + + while (bchain != NULL) { + char *buf = bchain; + + bchain = *((char **) buf); + brel((void *) buf, &mypoolset, true/*wipe*/); + } + stats("\nAfter release", &mypoolset); +#ifndef BECtl + if (bp != NULL) { +#ifdef BufValid + V bpoolv((void *) bp); +#endif +#ifdef BufDump + bpoold((void *) bp, dumpAlloc, dumpFree); +#endif + } +#endif + + return 0; +} +#endif diff --git a/optee_os/lib/libutils/isoc/bget.doc b/optee_os/lib/libutils/isoc/bget.doc new file mode 100644 index 0000000..aaa1bd9 --- /dev/null +++ b/optee_os/lib/libutils/isoc/bget.doc @@ -0,0 +1,338 @@ + + BGET -- Memory Allocator + ========================== + + by John Walker + http://www.fourmilab.ch/ + +BGET is a comprehensive memory allocation package which is easily +configured to the needs of an application. BGET is efficient in both +the time needed to allocate and release buffers and in the memory +overhead required for buffer pool management. It automatically +consolidates contiguous space to minimise fragmentation. BGET is +configured by compile-time definitions, Major options include: + + * A built-in test program to exercise BGET and + demonstrate how the various functions are used. + + * Allocation by either the "first fit" or "best fit" + method. + + * Wiping buffers at release time to catch code which + references previously released storage. + + * Built-in routines to dump individual buffers or the + entire buffer pool. + + * Retrieval of allocation and pool size statistics. + + * Quantisation of buffer sizes to a power of two to + satisfy hardware alignment constraints. + + * Automatic pool compaction, growth, and shrinkage by + means of call-backs to user defined functions. + +Applications of BGET can range from storage management in ROM-based +embedded programs to providing the framework upon which a multitasking +system incorporating garbage collection is constructed. BGET +incorporates extensive internal consistency checking using the + mechanism; all these checks can be turned off by compiling +with NDEBUG defined, yielding a version of BGET with minimal size and +maximum speed. + +The basic algorithm underlying BGET has withstood the test of time; more +than 25 years have passed since the first implementation of this code. +And yet, it is substantially more efficient than the native allocation +schemes of many operating systems: the Macintosh and Microsoft Windows +to name two, on which programs have obtained substantial speed-ups by +layering BGET as an application level memory manager atop the underlying +system's. + +BGET has been implemented on the largest mainframes and the lowest of +microprocessors. It has served as the core for multitasking operating +systems, multi-thread applications, embedded software in data network +switching processors, and a host of C programs. And while it has +accreted flexibility and additional options over the years, it remains +fast, memory efficient, portable, and easy to integrate into your +program. + + +BGET IMPLEMENTATION ASSUMPTIONS +=============================== + +BGET is written in as portable a dialect of C as possible. The only +fundamental assumption about the underlying hardware architecture is +that memory is allocated is a linear array which can be addressed as a +vector of C "char" objects. On segmented address space architectures, +this generally means that BGET should be used to allocate storage within +a single segment (although some compilers simulate linear address spaces +on segmented architectures). On segmented architectures, then, BGET +buffer pools may not be larger than a segment, but since BGET allows any +number of separate buffer pools, there is no limit on the total storage +which can be managed, only on the largest individual object which can be +allocated. Machines with a linear address architecture, such as the +VAX, 680x0, Sparc, MIPS, or the Intel 80386 and above in native mode, +may use BGET without restriction. + + +GETTING STARTED WITH BGET +========================= + +Although BGET can be configured in a multitude of fashions, there are +three basic ways of working with BGET. The functions mentioned below +are documented in the following section. Please excuse the forward +references which are made in the interest of providing a roadmap to +guide you to the BGET functions you're likely to need. + +Embedded Applications +--------------------- + +Embedded applications typically have a fixed area of memory dedicated to +buffer allocation (often in a separate RAM address space distinct from +the ROM that contains the executable code). To use BGET in such an +environment, simply call bpool() with the start address and length of +the buffer pool area in RAM, then allocate buffers with bget() and +release them with brel(). Embedded applications with very limited RAM +but abundant CPU speed may benefit by configuring BGET for BestFit +allocation (which is usually not worth it in other environments). + +Malloc() Emulation +------------------ + +If the C library malloc() function is too slow, not present in your +development environment (for example, an a native Windows or Macintosh +program), or otherwise unsuitable, you can replace it with BGET. +Initially define a buffer pool of an appropriate size with +bpool()--usually obtained by making a call to the operating system's +low-level memory allocator. Then allocate buffers with bget(), bgetz(), +and bgetr() (the last two permit the allocation of buffers initialised +to zero and [inefficient] re-allocation of existing buffers for +compatibility with C library functions). Release buffers by calling +brel(). If a buffer allocation request fails, obtain more storage from +the underlying operating system, add it to the buffer pool by another +call to bpool(), and continue execution. + +Automatic Storage Management +---------------------------- + +You can use BGET as your application's native memory manager and +implement automatic storage pool expansion, contraction, and optionally +application-specific memory compaction by compiling BGET with the BECtl +variable defined, then calling bectl() and supplying functions for +storage compaction, acquisition, and release, as well as a standard pool +expansion increment. All of these functions are optional (although it +doesn't make much sense to provide a release function without an +acquisition function, does it?). Once the call-back functions have been +defined with bectl(), you simply use bget() and brel() to allocate and +release storage as before. You can supply an initial buffer pool with +bpool() or rely on automatic allocation to acquire the entire pool. +When a call on bget() cannot be satisfied, BGET first checks if a +compaction function has been supplied. If so, it is called (with the +space required to satisfy the allocation request and a sequence number +to allow the compaction routine to be called successively without +looping). If the compaction function is able to free any storage (it +needn't know whether the storage it freed was adequate) it should return +a nonzero value, whereupon BGET will retry the allocation request and, +if it fails again, call the compaction function again with the +next-higher sequence number. + +If the compaction function returns zero, indicating failure to free +space, or no compaction function is defined, BGET next tests whether a +non-NULL allocation function was supplied to bectl(). If so, that +function is called with an argument indicating how many bytes of +additional space are required. This will be the standard pool expansion +increment supplied in the call to bectl() unless the original bget() +call requested a buffer larger than this; buffers larger than the +standard pool block can be managed "off the books" by BGET in this mode. +If the allocation function succeeds in obtaining the storage, it returns +a pointer to the new block and BGET expands the buffer pool; if it +fails, the allocation request fails and returns NULL to the caller. If +a non-NULL release function is supplied, expansion blocks which become +totally empty are released to the global free pool by passing their +addresses to the release function. + +Equipped with appropriate allocation, release, and compaction functions, +BGET can be used as part of very sophisticated memory management +strategies, including garbage collection. (Note, however, that BGET is +*not* a garbage collector by itself, and that developing such a system +requires much additional logic and careful design of the application's +memory allocation strategy.) + + +BGET FUNCTION DESCRIPTIONS +========================== + +Functions implemented by BGET (some are enabled by certain of the +optional settings below): + + void bpool(void *buffer, bufsize len); + +Create a buffer pool of bytes, using the storage starting at +. You can call bpool() subsequently to contribute additional +storage to the overall buffer pool. + + void *bget(bufsize size); + +Allocate a buffer of bytes. The address of the buffer is +returned, or NULL if insufficient memory was available to allocate the +buffer. + + void *bgetz(bufsize size); + +Allocate a buffer of bytes and clear it to all zeroes. The +address of the buffer is returned, or NULL if insufficient memory was +available to allocate the buffer. + + void *bgetr(void *buffer, bufsize newsize); + +Reallocate a buffer previously allocated by bget(), changing its size to + and preserving all existing data. NULL is returned if +insufficient memory is available to reallocate the buffer, in which case +the original buffer remains intact. + + void brel(void *buf); + +Return the buffer , previously allocated by bget(), to the free +space pool. + + void bectl(int (*compact)(bufsize sizereq, int sequence), + void *(*acquire)(bufsize size), + void (*release)(void *buf), + bufsize pool_incr); + +Expansion control: specify functions through which the package may +compact storage (or take other appropriate action) when an allocation +request fails, and optionally automatically acquire storage for +expansion blocks when necessary, and release such blocks when they +become empty. If is non-NULL, whenever a buffer allocation +request fails, the function will be called with arguments +specifying the number of bytes (total buffer size, including header +overhead) required to satisfy the allocation request, and a sequence +number indicating the number of consecutive calls on +attempting to satisfy this allocation request. The sequence number is 1 +for the first call on for a given allocation request, and +increments on subsequent calls, permitting the function to +take increasingly dire measures in an attempt to free up storage. If +the function returns a nonzero value, the allocation attempt +is re-tried. If returns 0 (as it must if it isn't able to +release any space or add storage to the buffer pool), the allocation +request fails, which can trigger automatic pool expansion if the + argument is non-NULL. At the time the function is +called, the state of the buffer allocator is identical to that at the +moment the allocation request was made; consequently, the +function may call brel(), bpool(), bstats(), and/or directly manipulate +the buffer pool in any manner which would be valid were the application +in control. This does not, however, relieve the function of +the need to ensure that whatever actions it takes do not change things +underneath the application that made the allocation request. For +example, a function that released a buffer in the process of +being reallocated with bgetr() would lead to disaster. Implementing a +safe and effective mechanism requires careful design of an +application's memory architecture, and cannot generally be easily +retrofitted into existing code. + +If is non-NULL, that function will be called whenever an +allocation request fails. If the function succeeds in +allocating the requested space and returns a pointer to the new area, +allocation will proceed using the expanded buffer pool. If +cannot obtain the requested space, it should return NULL and the entire +allocation process will fail. specifies the normal +expansion block size. Providing an function will cause +subsequent bget() requests for buffers too large to be managed in the +linked-block scheme (in other words, larger than minus the +buffer overhead) to be satisfied directly by calls to the +function. Automatic release of empty pool blocks will occur only if all +pool blocks in the system are the size given by . + + void bstats(bufsize *curalloc, bufsize *totfree, + bufsize *maxfree, long *nget, long *nrel); + +The amount of space currently allocated is stored into the variable +pointed to by . The total free space (sum of all free blocks +in the pool) is stored into the variable pointed to by , and +the size of the largest single block in the free space pool is stored +into the variable pointed to by . The variables pointed to by + and are filled, respectively, with the number of +successful (non-NULL return) bget() calls and the number of brel() +calls. + + void bstatse(bufsize *pool_incr, long *npool, + long *npget, long *nprel, + long *ndget, long *ndrel); + +Extended statistics: The expansion block size will be stored into the +variable pointed to by , or the negative thereof if automatic +expansion block releases are disabled. The number of currently active +pool blocks will be stored into the variable pointed to by . The +variables pointed to by and will be filled with, +respectively, the number of expansion block acquisitions and releases +which have occurred. The variables pointed to by and +will be filled with the number of bget() and brel() calls, respectively, +managed through blocks directly allocated by the acquisition and release +functions. + + void bufdump(void *buf); + +The buffer pointed to by is dumped on standard output. + + void bpoold(void *pool, int dumpalloc, int dumpfree); + +All buffers in the buffer pool , previously initialised by a call +on bpool(), are listed in ascending memory address order. If + is nonzero, the contents of allocated buffers are dumped; if + is nonzero, the contents of free blocks are dumped. + + int bpoolv(void *pool); + +The named buffer pool, previously initialised by a call on bpool(), is +validated for bad pointers, overwritten data, etc. If compiled with +NDEBUG not defined, any error generates an assertion failure. Otherwise 1 +is returned if the pool is valid, 0 if an error is found. + +BGET CONFIGURATION +================== + +#define TestProg 20000 /* Generate built-in test program + if defined. The value specifies + how many buffer allocation attempts + the test program should make. */ + +#define SizeQuant 4 /* Buffer allocation size quantum: + all buffers allocated are a + multiple of this size. This + MUST be a power of two. */ + +#define BufDump 1 /* Define this symbol to enable the + bpoold() function which dumps the + buffers in a buffer pool. */ + +#define BufValid 1 /* Define this symbol to enable the + bpoolv() function for validating + a buffer pool. */ + +#define DumpData 1 /* Define this symbol to enable the + bufdump() function which allows + dumping the contents of an allocated + or free buffer. */ + +#define BufStats 1 /* Define this symbol to enable the + bstats() function which calculates + the total free space in the buffer + pool, the largest available + buffer, and the total space + currently allocated. */ + +#define FreeWipe 1 /* Wipe free buffers to a guaranteed + pattern of garbage to trip up + miscreants who attempt to use + pointers into released buffers. */ + +#define BestFit 1 /* Use a best fit algorithm when + searching for space for an + allocation request. This uses + memory more efficiently, but + allocation will be much slower. */ + +#define BECtl 1 /* Define this symbol to enable the + bectl() function for automatic + pool space control. */ diff --git a/optee_os/lib/libutils/isoc/bget.h b/optee_os/lib/libutils/isoc/bget.h new file mode 100644 index 0000000..abef853 --- /dev/null +++ b/optee_os/lib/libutils/isoc/bget.h @@ -0,0 +1,55 @@ +/* + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ST BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + + Interface definitions for bget.c, the memory management package. + +*/ + + +#ifndef _ +#ifdef PROTOTYPES +#define _(x) x /* If compiler knows prototypes */ +#else +#define _(x) () /* It it doesn't */ +#endif /* PROTOTYPES */ +#endif + +#define BGET_HDR_QUANTUM (2 * sizeof(long)) + +typedef long bufsize; +struct bpoolset; + +void bpool _((void *buffer, bufsize len, struct bpoolset *poolset)); +void *bget _((bufsize align, bufsize hdr_size, bufsize size, struct bpoolset *poolset)); +void *bgetz _((bufsize align, bufsize hdr_size, bufsize size, struct bpoolset *poolset)); +void *bgetr _((void *buffer, bufsize align, bufsize hdr_size, bufsize newsize, + struct bpoolset *poolset)); +void brel _((void *buf, struct bpoolset *poolset, int wipe)); +void bectl _((int (*compact)(bufsize sizereq, int sequence), + void *(*acquire)(bufsize size), + void (*release)(void *buf), bufsize pool_incr, + struct bpoolset *poolset)); +void bstats _((bufsize *curalloc, bufsize *totfree, bufsize *maxfree, + long *nget, long *nrel, struct bpoolset *poolset)); +void bstatse _((bufsize *pool_incr, long *npool, long *npget, + long *nprel, long *ndget, long *ndrel, + struct bpoolset *poolset)); +void bufdump _((void *buf)); +void bpoold _((void *pool, int dumpalloc, int dumpfree)); +int bpoolv _((void *pool)); + +#if !defined(__KERNEL__) && !defined(__LDELF__) && defined(CFG_TA_BGET_TEST) +int bget_main_test(void *(*malloc_func)(size_t), void (*free_func)(void *)); +#endif diff --git a/optee_os/lib/libutils/isoc/bget_malloc.c b/optee_os/lib/libutils/isoc/bget_malloc.c new file mode 100644 index 0000000..15745bb --- /dev/null +++ b/optee_os/lib/libutils/isoc/bget_malloc.c @@ -0,0 +1,1140 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + * Copyright (c) 2022, Linaro Limited. + */ + +#define PROTOTYPES + +/* + * BGET CONFIGURATION + * ================== + */ +/* #define BGET_ENABLE_ALL_OPTIONS */ +#ifdef BGET_ENABLE_OPTION +#define TestProg 20000 /* Generate built-in test program + if defined. The value specifies + how many buffer allocation attempts + the test program should make. */ +#endif + + +#ifdef __LP64__ +#define SizeQuant 16 +#endif +#ifdef __ILP32__ +#define SizeQuant 8 +#endif + /* Buffer allocation size quantum: + all buffers allocated are a + multiple of this size. This + MUST be a power of two. */ + +#ifdef BGET_ENABLE_OPTION +#define BufDump 1 /* Define this symbol to enable the + bpoold() function which dumps the + buffers in a buffer pool. */ + +#define BufValid 1 /* Define this symbol to enable the + bpoolv() function for validating + a buffer pool. */ + +#define DumpData 1 /* Define this symbol to enable the + bufdump() function which allows + dumping the contents of an allocated + or free buffer. */ + +#define BufStats 1 /* Define this symbol to enable the + bstats() function which calculates + the total free space in the buffer + pool, the largest available + buffer, and the total space + currently allocated. */ + +#define FreeWipe 1 /* Wipe free buffers to a guaranteed + pattern of garbage to trip up + miscreants who attempt to use + pointers into released buffers. */ + +#define BestFit 1 /* Use a best fit algorithm when + searching for space for an + allocation request. This uses + memory more efficiently, but + allocation will be much slower. */ + +#define BECtl 1 /* Define this symbol to enable the + bectl() function for automatic + pool space control. */ +#endif + +#ifdef MEM_DEBUG +#undef NDEBUG +#define DumpData 1 +#define BufValid 1 +#define FreeWipe 1 +#endif + +#ifdef CFG_WITH_STATS +#define BufStats 1 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__KERNEL__) +/* Compiling for TEE Core */ +#include +#include +#include + +static void *memset_unchecked(void *s, int c, size_t n) +{ + return asan_memset_unchecked(s, c, n); +} + +static __maybe_unused void *memcpy_unchecked(void *dst, const void *src, + size_t n) +{ + return asan_memcpy_unchecked(dst, src, n); +} + +#else /*__KERNEL__*/ +/* Compiling for TA */ + +static void *memset_unchecked(void *s, int c, size_t n) +{ + return memset(s, c, n); +} + +static __maybe_unused void *memcpy_unchecked(void *dst, const void *src, + size_t n) +{ + return memcpy(dst, src, n); +} + +#endif /*__KERNEL__*/ + +#include "bget.c" /* this is ugly, but this is bget */ + +struct malloc_pool { + void *buf; + size_t len; +}; + +struct malloc_ctx { + struct bpoolset poolset; + struct malloc_pool *pool; + size_t pool_len; +#ifdef BufStats + struct malloc_stats mstats; +#endif +#ifdef __KERNEL__ + unsigned int spinlock; +#endif +}; + +#ifdef __KERNEL__ + +static uint32_t malloc_lock(struct malloc_ctx *ctx) +{ + return cpu_spin_lock_xsave(&ctx->spinlock); +} + +static void malloc_unlock(struct malloc_ctx *ctx, uint32_t exceptions) +{ + cpu_spin_unlock_xrestore(&ctx->spinlock, exceptions); +} + +#else /* __KERNEL__ */ + +static uint32_t malloc_lock(struct malloc_ctx *ctx __unused) +{ + return 0; +} + +static void malloc_unlock(struct malloc_ctx *ctx __unused, + uint32_t exceptions __unused) +{ +} + +#endif /* __KERNEL__ */ + +#define DEFINE_CTX(name) struct malloc_ctx name = \ + { .poolset = { .freelist = { {0, 0}, \ + {&name.poolset.freelist, \ + &name.poolset.freelist}}}} + +static DEFINE_CTX(malloc_ctx); + +#ifdef CFG_NS_VIRTUALIZATION +static __nex_data DEFINE_CTX(nex_malloc_ctx); +#endif + +static void print_oom(size_t req_size __maybe_unused, void *ctx __maybe_unused) +{ +#if defined(__KERNEL__) && defined(CFG_CORE_DUMP_OOM) + EMSG("Memory allocation failed: size %zu context %p", req_size, ctx); + print_kernel_stack(); +#endif +} + +/* Most of the stuff in this function is copied from bgetr() in bget.c */ +static __maybe_unused bufsize bget_buf_size(void *buf) +{ + bufsize osize; /* Old size of buffer */ + struct bhead *b; + + b = BH(((char *)buf) - sizeof(struct bhead)); + osize = -b->bsize; +#ifdef BECtl + if (osize == 0) { + /* Buffer acquired directly through acqfcn. */ + struct bdhead *bd; + + bd = BDH(((char *)buf) - sizeof(struct bdhead)); + osize = bd->tsize - sizeof(struct bdhead) - bd->offs; + } else +#endif + osize -= sizeof(struct bhead); + assert(osize > 0); + return osize; +} + +static void *maybe_tag_buf(uint8_t *buf, size_t hdr_size, size_t requested_size) +{ + if (!buf) + return NULL; + + COMPILE_TIME_ASSERT(MEMTAG_GRANULE_SIZE <= SizeQuant); + + if (MEMTAG_IS_ENABLED) { + size_t sz = ROUNDUP(requested_size, MEMTAG_GRANULE_SIZE); + + /* + * Allocated buffer can be larger than requested when + * allocating with memalign(), but we should never tag more + * than allocated. + */ + assert(bget_buf_size(buf) >= sz + hdr_size); + return memtag_set_random_tags(buf, sz + hdr_size); + } + +#if defined(__KERNEL__) + if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS)) + asan_tag_access(buf, buf + hdr_size + requested_size); +#endif + return buf; +} + +static void *maybe_untag_buf(void *buf) +{ + if (!buf) + return NULL; + + if (MEMTAG_IS_ENABLED) { + size_t sz = 0; + + memtag_assert_tag(buf); /* Trying to catch double free early */ + sz = bget_buf_size(memtag_strip_tag(buf)); + return memtag_set_tags(buf, sz, 0); + } + +#if defined(__KERNEL__) + if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS)) + asan_tag_heap_free(buf, (uint8_t *)buf + bget_buf_size(buf)); +#endif + return buf; +} + +static void *strip_tag(void *buf) +{ + if (MEMTAG_IS_ENABLED) + return memtag_strip_tag(buf); + return buf; +} + +static void tag_asan_free(void *buf __maybe_unused, size_t len __maybe_unused) +{ +#if defined(__KERNEL__) + asan_tag_heap_free(buf, (uint8_t *)buf + len); +#endif +} + +#ifdef BufStats + +static void *raw_malloc_return_hook(void *p, size_t hdr_size, + size_t requested_size, + struct malloc_ctx *ctx) +{ + if (ctx->poolset.totalloc > ctx->mstats.max_allocated) + ctx->mstats.max_allocated = ctx->poolset.totalloc; + + if (!p) { + ctx->mstats.num_alloc_fail++; + print_oom(requested_size, ctx); + if (requested_size > ctx->mstats.biggest_alloc_fail) { + ctx->mstats.biggest_alloc_fail = requested_size; + ctx->mstats.biggest_alloc_fail_used = + ctx->poolset.totalloc; + } + } + + return maybe_tag_buf(p, hdr_size, MAX(SizeQuant, requested_size)); +} + +static void gen_malloc_reset_stats(struct malloc_ctx *ctx) +{ + uint32_t exceptions = malloc_lock(ctx); + + ctx->mstats.max_allocated = 0; + ctx->mstats.num_alloc_fail = 0; + ctx->mstats.biggest_alloc_fail = 0; + ctx->mstats.biggest_alloc_fail_used = 0; + malloc_unlock(ctx, exceptions); +} + +void malloc_reset_stats(void) +{ + gen_malloc_reset_stats(&malloc_ctx); +} + +static void gen_malloc_get_stats(struct malloc_ctx *ctx, + struct malloc_stats *stats) +{ + uint32_t exceptions = malloc_lock(ctx); + + raw_malloc_get_stats(ctx, stats); + malloc_unlock(ctx, exceptions); +} + +void malloc_get_stats(struct malloc_stats *stats) +{ + gen_malloc_get_stats(&malloc_ctx, stats); +} + +#else /* BufStats */ + +static void *raw_malloc_return_hook(void *p, size_t hdr_size, + size_t requested_size, + struct malloc_ctx *ctx ) +{ + if (!p) + print_oom(requested_size, ctx); + + return maybe_tag_buf(p, hdr_size, MAX(SizeQuant, requested_size)); +} + +#endif /* BufStats */ + +#ifdef BufValid +static void raw_malloc_validate_pools(struct malloc_ctx *ctx) +{ + size_t n; + + for (n = 0; n < ctx->pool_len; n++) + bpoolv(ctx->pool[n].buf); +} +#else +static void raw_malloc_validate_pools(struct malloc_ctx *ctx __unused) +{ +} +#endif + +struct bpool_iterator { + struct bfhead *next_buf; + size_t pool_idx; +}; + +static void bpool_foreach_iterator_init(struct malloc_ctx *ctx, + struct bpool_iterator *iterator) +{ + iterator->pool_idx = 0; + iterator->next_buf = BFH(ctx->pool[0].buf); +} + +static bool bpool_foreach_pool(struct bpool_iterator *iterator, void **buf, + size_t *len, bool *isfree) +{ + struct bfhead *b = iterator->next_buf; + bufsize bs = b->bh.bsize; + + if (bs == ESent) + return false; + + if (bs < 0) { + /* Allocated buffer */ + bs = -bs; + + *isfree = false; + } else { + /* Free Buffer */ + *isfree = true; + + /* Assert that the free list links are intact */ + assert(b->ql.blink->ql.flink == b); + assert(b->ql.flink->ql.blink == b); + } + + *buf = (uint8_t *)b + sizeof(struct bhead); + *len = bs - sizeof(struct bhead); + + iterator->next_buf = BFH((uint8_t *)b + bs); + return true; +} + +static bool bpool_foreach(struct malloc_ctx *ctx, + struct bpool_iterator *iterator, void **buf) +{ + while (true) { + size_t len; + bool isfree; + + if (bpool_foreach_pool(iterator, buf, &len, &isfree)) { + if (isfree) + continue; + return true; + } + + if ((iterator->pool_idx + 1) >= ctx->pool_len) + return false; + + iterator->pool_idx++; + iterator->next_buf = BFH(ctx->pool[iterator->pool_idx].buf); + } +} + +/* Convenience macro for looping over all allocated buffers */ +#define BPOOL_FOREACH(ctx, iterator, bp) \ + for (bpool_foreach_iterator_init((ctx),(iterator)); \ + bpool_foreach((ctx),(iterator), (bp));) + +void *raw_memalign(size_t hdr_size, size_t ftr_size, size_t alignment, + size_t pl_size, struct malloc_ctx *ctx) +{ + void *ptr = NULL; + bufsize s; + + if (!alignment || !IS_POWER_OF_TWO(alignment)) + return NULL; + + raw_malloc_validate_pools(ctx); + + /* Compute total size, excluding the header */ + if (ADD_OVERFLOW(pl_size, ftr_size, &s)) + goto out; + + /* BGET doesn't like 0 sized allocations */ + if (!s) + s++; + + ptr = bget(alignment, hdr_size, s, &ctx->poolset); +out: + return raw_malloc_return_hook(ptr, hdr_size, pl_size, ctx); +} + +void *raw_malloc(size_t hdr_size, size_t ftr_size, size_t pl_size, + struct malloc_ctx *ctx) +{ + /* + * Note that we're feeding SizeQ as alignment, this is the smallest + * alignment that bget() can use. + */ + return raw_memalign(hdr_size, ftr_size, SizeQ, pl_size, ctx); +} + +void raw_free(void *ptr, struct malloc_ctx *ctx, bool wipe) +{ + raw_malloc_validate_pools(ctx); + + if (ptr) + brel(maybe_untag_buf(ptr), &ctx->poolset, wipe); +} + +void *raw_calloc(size_t hdr_size, size_t ftr_size, size_t pl_nmemb, + size_t pl_size, struct malloc_ctx *ctx) +{ + void *ptr = NULL; + bufsize s; + + raw_malloc_validate_pools(ctx); + + /* Compute total size, excluding hdr_size */ + if (MUL_OVERFLOW(pl_nmemb, pl_size, &s)) + goto out; + if (ADD_OVERFLOW(s, ftr_size, &s)) + goto out; + + /* BGET doesn't like 0 sized allocations */ + if (!s) + s++; + + ptr = bgetz(0, hdr_size, s, &ctx->poolset); +out: + return raw_malloc_return_hook(ptr, hdr_size, pl_nmemb * pl_size, ctx); +} + +void *raw_realloc(void *ptr, size_t hdr_size, size_t ftr_size, + size_t pl_size, struct malloc_ctx *ctx) +{ + void *p = NULL; + bufsize s; + + /* Compute total size */ + if (ADD_OVERFLOW(pl_size, hdr_size, &s)) + goto out; + if (ADD_OVERFLOW(s, ftr_size, &s)) + goto out; + + raw_malloc_validate_pools(ctx); + + /* BGET doesn't like 0 sized allocations */ + if (!s) + s++; + + p = bget(0, 0, s, &ctx->poolset); + + if (p && ptr) { + void *old_ptr = maybe_untag_buf(ptr); + bufsize old_sz = bget_buf_size(old_ptr); + + if (old_sz < s) { + memcpy_unchecked(p, old_ptr, old_sz); +#ifndef __KERNEL__ + /* User space reallocations are always zeroed */ + memset_unchecked((uint8_t *)p + old_sz, 0, s - old_sz); +#endif + } else { + memcpy_unchecked(p, old_ptr, s); + } + + brel(old_ptr, &ctx->poolset, false /*!wipe*/); + } +out: + return raw_malloc_return_hook(p, hdr_size, pl_size, ctx); +} + +#ifdef ENABLE_MDBG + +struct mdbg_hdr { + const char *fname; + uint16_t line; + uint32_t pl_size; + uint32_t magic; +#if defined(ARM64) + uint64_t pad; +#endif +}; + +#define MDBG_HEADER_MAGIC 0xadadadad +#define MDBG_FOOTER_MAGIC 0xecececec + +static size_t mdbg_get_ftr_size(size_t pl_size) +{ + size_t ftr_pad = ROUNDUP(pl_size, sizeof(uint32_t)) - pl_size; + + return ftr_pad + sizeof(uint32_t); +} + +static uint32_t *mdbg_get_footer(struct mdbg_hdr *hdr) +{ + uint32_t *footer; + + footer = (uint32_t *)((uint8_t *)(hdr + 1) + hdr->pl_size + + mdbg_get_ftr_size(hdr->pl_size)); + footer--; + return strip_tag(footer); +} + +static void mdbg_update_hdr(struct mdbg_hdr *hdr, const char *fname, + int lineno, size_t pl_size) +{ + uint32_t *footer; + + hdr->fname = fname; + hdr->line = lineno; + hdr->pl_size = pl_size; + hdr->magic = MDBG_HEADER_MAGIC; + + footer = mdbg_get_footer(hdr); + *footer = MDBG_FOOTER_MAGIC; +} + +static void *gen_mdbg_malloc(struct malloc_ctx *ctx, const char *fname, + int lineno, size_t size) +{ + struct mdbg_hdr *hdr; + uint32_t exceptions = malloc_lock(ctx); + + /* + * Check struct mdbg_hdr works with BGET_HDR_QUANTUM. + */ + COMPILE_TIME_ASSERT((sizeof(struct mdbg_hdr) % BGET_HDR_QUANTUM) == 0); + + hdr = raw_malloc(sizeof(struct mdbg_hdr), + mdbg_get_ftr_size(size), size, ctx); + if (hdr) { + mdbg_update_hdr(hdr, fname, lineno, size); + hdr++; + } + + malloc_unlock(ctx, exceptions); + return hdr; +} + +static void assert_header(struct mdbg_hdr *hdr __maybe_unused) +{ + assert(hdr->magic == MDBG_HEADER_MAGIC); + assert(*mdbg_get_footer(hdr) == MDBG_FOOTER_MAGIC); +} + +static void gen_mdbg_free(struct malloc_ctx *ctx, void *ptr, bool wipe) +{ + struct mdbg_hdr *hdr = ptr; + + if (hdr) { + hdr--; + assert_header(hdr); + hdr->magic = 0; + *mdbg_get_footer(hdr) = 0; + raw_free(hdr, ctx, wipe); + } +} + +static void free_helper(void *ptr, bool wipe) +{ + uint32_t exceptions = malloc_lock(&malloc_ctx); + + gen_mdbg_free(&malloc_ctx, ptr, wipe); + malloc_unlock(&malloc_ctx, exceptions); +} + +static void *gen_mdbg_calloc(struct malloc_ctx *ctx, const char *fname, int lineno, + size_t nmemb, size_t size) +{ + struct mdbg_hdr *hdr; + uint32_t exceptions = malloc_lock(ctx); + + hdr = raw_calloc(sizeof(struct mdbg_hdr), + mdbg_get_ftr_size(nmemb * size), nmemb, size, + ctx); + if (hdr) { + mdbg_update_hdr(hdr, fname, lineno, nmemb * size); + hdr++; + } + malloc_unlock(ctx, exceptions); + return hdr; +} + +static void *gen_mdbg_realloc_unlocked(struct malloc_ctx *ctx, const char *fname, + int lineno, void *ptr, size_t size) +{ + struct mdbg_hdr *hdr = ptr; + + if (hdr) { + hdr--; + assert_header(hdr); + } + hdr = raw_realloc(hdr, sizeof(struct mdbg_hdr), + mdbg_get_ftr_size(size), size, ctx); + if (hdr) { + mdbg_update_hdr(hdr, fname, lineno, size); + hdr++; + } + return hdr; +} + +static void *gen_mdbg_realloc(struct malloc_ctx *ctx, const char *fname, + int lineno, void *ptr, size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(ctx); + + p = gen_mdbg_realloc_unlocked(ctx, fname, lineno, ptr, size); + malloc_unlock(ctx, exceptions); + return p; +} + +#define realloc_unlocked(ctx, ptr, size) \ + gen_mdbg_realloc_unlocked(ctx, __FILE__, __LINE__, (ptr), (size)) + +static void *gen_mdbg_memalign(struct malloc_ctx *ctx, const char *fname, + int lineno, size_t alignment, size_t size) +{ + struct mdbg_hdr *hdr; + uint32_t exceptions = malloc_lock(ctx); + + hdr = raw_memalign(sizeof(struct mdbg_hdr), mdbg_get_ftr_size(size), + alignment, size, ctx); + if (hdr) { + mdbg_update_hdr(hdr, fname, lineno, size); + hdr++; + } + malloc_unlock(ctx, exceptions); + return hdr; +} + + +static void *get_payload_start_size(void *raw_buf, size_t *size) +{ + struct mdbg_hdr *hdr = raw_buf; + + assert(bget_buf_size(hdr) >= hdr->pl_size); + *size = hdr->pl_size; + return hdr + 1; +} + +static void gen_mdbg_check(struct malloc_ctx *ctx, int bufdump) +{ + struct bpool_iterator itr; + void *b; + uint32_t exceptions = malloc_lock(ctx); + + raw_malloc_validate_pools(ctx); + + BPOOL_FOREACH(ctx, &itr, &b) { + struct mdbg_hdr *hdr = (struct mdbg_hdr *)b; + + assert_header(hdr); + + if (bufdump > 0) { + const char *fname = hdr->fname; + + if (!fname) + fname = "unknown"; + + IMSG("buffer: %d bytes %s:%d\n", + hdr->pl_size, fname, hdr->line); + } + } + + malloc_unlock(ctx, exceptions); +} + +void *mdbg_malloc(const char *fname, int lineno, size_t size) +{ + return gen_mdbg_malloc(&malloc_ctx, fname, lineno, size); +} + +void *mdbg_calloc(const char *fname, int lineno, size_t nmemb, size_t size) +{ + return gen_mdbg_calloc(&malloc_ctx, fname, lineno, nmemb, size); +} + +void *mdbg_realloc(const char *fname, int lineno, void *ptr, size_t size) +{ + return gen_mdbg_realloc(&malloc_ctx, fname, lineno, ptr, size); +} + +void *mdbg_memalign(const char *fname, int lineno, size_t alignment, + size_t size) +{ + return gen_mdbg_memalign(&malloc_ctx, fname, lineno, alignment, size); +} + +#if __STDC_VERSION__ >= 201112L +void *mdbg_aligned_alloc(const char *fname, int lineno, size_t alignment, + size_t size) +{ + if (size % alignment) + return NULL; + + return gen_mdbg_memalign(&malloc_ctx, fname, lineno, alignment, size); +} +#endif /* __STDC_VERSION__ */ + +void mdbg_check(int bufdump) +{ + gen_mdbg_check(&malloc_ctx, bufdump); +} + +/* + * Since malloc debug is enabled, malloc() and friends are redirected by macros + * to mdbg_malloc() etc. + * We still want to export the standard entry points in case they are referenced + * by the application, either directly or via external libraries. + */ +#undef malloc +void *malloc(size_t size) +{ + return mdbg_malloc(__FILE__, __LINE__, size); +} + +#undef calloc +void *calloc(size_t nmemb, size_t size) +{ + return mdbg_calloc(__FILE__, __LINE__, nmemb, size); +} + +#undef realloc +void *realloc(void *ptr, size_t size) +{ + return mdbg_realloc(__FILE__, __LINE__, ptr, size); +} + +#else /* ENABLE_MDBG */ + +void *malloc(size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&malloc_ctx); + + p = raw_malloc(0, 0, size, &malloc_ctx); + malloc_unlock(&malloc_ctx, exceptions); + return p; +} + +static void free_helper(void *ptr, bool wipe) +{ + uint32_t exceptions = malloc_lock(&malloc_ctx); + + raw_free(ptr, &malloc_ctx, wipe); + malloc_unlock(&malloc_ctx, exceptions); +} + +void *calloc(size_t nmemb, size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&malloc_ctx); + + p = raw_calloc(0, 0, nmemb, size, &malloc_ctx); + malloc_unlock(&malloc_ctx, exceptions); + return p; +} + +static void *realloc_unlocked(struct malloc_ctx *ctx, void *ptr, + size_t size) +{ + return raw_realloc(ptr, 0, 0, size, ctx); +} + +void *realloc(void *ptr, size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&malloc_ctx); + + p = realloc_unlocked(&malloc_ctx, ptr, size); + malloc_unlock(&malloc_ctx, exceptions); + return p; +} + +void *memalign(size_t alignment, size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&malloc_ctx); + + p = raw_memalign(0, 0, alignment, size, &malloc_ctx); + malloc_unlock(&malloc_ctx, exceptions); + return p; +} + +#if __STDC_VERSION__ >= 201112L +void *aligned_alloc(size_t alignment, size_t size) +{ + if (size % alignment) + return NULL; + + return memalign(alignment, size); +} +#endif /* __STDC_VERSION__ */ + +static void *get_payload_start_size(void *ptr, size_t *size) +{ + *size = bget_buf_size(ptr); + return ptr; +} + +#endif + +void free(void *ptr) +{ + free_helper(ptr, false); +} + +void free_wipe(void *ptr) +{ + free_helper(ptr, true); +} + +static void gen_malloc_add_pool(struct malloc_ctx *ctx, void *buf, size_t len) +{ + uint32_t exceptions = malloc_lock(ctx); + + raw_malloc_add_pool(ctx, buf, len); + malloc_unlock(ctx, exceptions); +} + +static bool gen_malloc_buffer_is_within_alloced(struct malloc_ctx *ctx, + void *buf, size_t len) +{ + uint32_t exceptions = malloc_lock(ctx); + bool ret = false; + + ret = raw_malloc_buffer_is_within_alloced(ctx, buf, len); + malloc_unlock(ctx, exceptions); + + return ret; +} + +static bool gen_malloc_buffer_overlaps_heap(struct malloc_ctx *ctx, + void *buf, size_t len) +{ + bool ret = false; + uint32_t exceptions = malloc_lock(ctx); + + ret = raw_malloc_buffer_overlaps_heap(ctx, buf, len); + malloc_unlock(ctx, exceptions); + return ret; +} + +size_t raw_malloc_get_ctx_size(void) +{ + return sizeof(struct malloc_ctx); +} + +void raw_malloc_init_ctx(struct malloc_ctx *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); + ctx->poolset.freelist.ql.flink = &ctx->poolset.freelist; + ctx->poolset.freelist.ql.blink = &ctx->poolset.freelist; +} + +void raw_malloc_add_pool(struct malloc_ctx *ctx, void *buf, size_t len) +{ + const size_t min_len = sizeof(struct bhead) + sizeof(struct bfhead); + uintptr_t start = (uintptr_t)buf; + uintptr_t end = start + len; + void *p = NULL; + size_t l = 0; + + start = ROUNDUP(start, SizeQuant); + end = ROUNDDOWN(end, SizeQuant); + + if (start > end || (end - start) < min_len) { + DMSG("Skipping too small pool"); + return; + } + + /* First pool requires a bigger size */ + if (!ctx->pool_len && (end - start) < MALLOC_INITIAL_POOL_MIN_SIZE) { + DMSG("Skipping too small initial pool"); + return; + } + + tag_asan_free((void *)start, end - start); + bpool((void *)start, end - start, &ctx->poolset); + l = ctx->pool_len + 1; + p = realloc_unlocked(ctx, ctx->pool, sizeof(struct malloc_pool) * l); + assert(p); + ctx->pool = p; + ctx->pool[ctx->pool_len].buf = (void *)start; + ctx->pool[ctx->pool_len].len = end - start; +#ifdef BufStats + ctx->mstats.size += ctx->pool[ctx->pool_len].len; +#endif + ctx->pool_len = l; +} + +bool raw_malloc_buffer_overlaps_heap(struct malloc_ctx *ctx, + void *buf, size_t len) +{ + uintptr_t buf_start = (uintptr_t)strip_tag(buf); + uintptr_t buf_end = buf_start + len; + size_t n = 0; + + raw_malloc_validate_pools(ctx); + + for (n = 0; n < ctx->pool_len; n++) { + uintptr_t pool_start = (uintptr_t)strip_tag(ctx->pool[n].buf); + uintptr_t pool_end = pool_start + ctx->pool[n].len; + + if (buf_start > buf_end || pool_start > pool_end) + return true; /* Wrapping buffers, shouldn't happen */ + + if ((buf_start >= pool_start && buf_start < pool_end) || + (buf_end > pool_start && buf_end < pool_end)) + return true; + } + + return false; +} + +bool raw_malloc_buffer_is_within_alloced(struct malloc_ctx *ctx, + void *buf, size_t len) +{ + struct bpool_iterator itr = { }; + void *b = NULL; + uint8_t *start_buf = strip_tag(buf); + uint8_t *end_buf = start_buf + len; + + raw_malloc_validate_pools(ctx); + + /* Check for wrapping */ + if (start_buf > end_buf) + return false; + + BPOOL_FOREACH(ctx, &itr, &b) { + uint8_t *start_b = NULL; + uint8_t *end_b = NULL; + size_t s = 0; + + start_b = strip_tag(get_payload_start_size(b, &s)); + end_b = start_b + s; + if (start_buf >= start_b && end_buf <= end_b) + return true; + } + + return false; +} + +#ifdef CFG_WITH_STATS +void raw_malloc_get_stats(struct malloc_ctx *ctx, struct malloc_stats *stats) +{ + memcpy_unchecked(stats, &ctx->mstats, sizeof(*stats)); + stats->allocated = ctx->poolset.totalloc; +} +#endif + +void malloc_add_pool(void *buf, size_t len) +{ + gen_malloc_add_pool(&malloc_ctx, buf, len); +} + +bool malloc_buffer_is_within_alloced(void *buf, size_t len) +{ + return gen_malloc_buffer_is_within_alloced(&malloc_ctx, buf, len); +} + +bool malloc_buffer_overlaps_heap(void *buf, size_t len) +{ + return gen_malloc_buffer_overlaps_heap(&malloc_ctx, buf, len); +} + +#ifdef CFG_NS_VIRTUALIZATION + +#ifndef ENABLE_MDBG + +void *nex_malloc(size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&nex_malloc_ctx); + + p = raw_malloc(0, 0, size, &nex_malloc_ctx); + malloc_unlock(&nex_malloc_ctx, exceptions); + return p; +} + +void *nex_calloc(size_t nmemb, size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&nex_malloc_ctx); + + p = raw_calloc(0, 0, nmemb, size, &nex_malloc_ctx); + malloc_unlock(&nex_malloc_ctx, exceptions); + return p; +} + +void *nex_realloc(void *ptr, size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&nex_malloc_ctx); + + p = realloc_unlocked(&nex_malloc_ctx, ptr, size); + malloc_unlock(&nex_malloc_ctx, exceptions); + return p; +} + +void *nex_memalign(size_t alignment, size_t size) +{ + void *p; + uint32_t exceptions = malloc_lock(&nex_malloc_ctx); + + p = raw_memalign(0, 0, alignment, size, &nex_malloc_ctx); + malloc_unlock(&nex_malloc_ctx, exceptions); + return p; +} + +void nex_free(void *ptr) +{ + uint32_t exceptions = malloc_lock(&nex_malloc_ctx); + + raw_free(ptr, &nex_malloc_ctx, false /* !wipe */); + malloc_unlock(&nex_malloc_ctx, exceptions); +} + +#else /* ENABLE_MDBG */ + +void *nex_mdbg_malloc(const char *fname, int lineno, size_t size) +{ + return gen_mdbg_malloc(&nex_malloc_ctx, fname, lineno, size); +} + +void *nex_mdbg_calloc(const char *fname, int lineno, size_t nmemb, size_t size) +{ + return gen_mdbg_calloc(&nex_malloc_ctx, fname, lineno, nmemb, size); +} + +void *nex_mdbg_realloc(const char *fname, int lineno, void *ptr, size_t size) +{ + return gen_mdbg_realloc(&nex_malloc_ctx, fname, lineno, ptr, size); +} + +void *nex_mdbg_memalign(const char *fname, int lineno, size_t alignment, + size_t size) +{ + return gen_mdbg_memalign(&nex_malloc_ctx, fname, lineno, alignment, size); +} + +void nex_mdbg_check(int bufdump) +{ + gen_mdbg_check(&nex_malloc_ctx, bufdump); +} + +void nex_free(void *ptr) +{ + uint32_t exceptions = malloc_lock(&nex_malloc_ctx); + + gen_mdbg_free(&nex_malloc_ctx, ptr, false /* !wipe */); + malloc_unlock(&nex_malloc_ctx, exceptions); +} + +#endif /* ENABLE_MDBG */ + +void nex_malloc_add_pool(void *buf, size_t len) +{ + gen_malloc_add_pool(&nex_malloc_ctx, buf, len); +} + +bool nex_malloc_buffer_is_within_alloced(void *buf, size_t len) +{ + return gen_malloc_buffer_is_within_alloced(&nex_malloc_ctx, buf, len); +} + +bool nex_malloc_buffer_overlaps_heap(void *buf, size_t len) +{ + return gen_malloc_buffer_overlaps_heap(&nex_malloc_ctx, buf, len); +} + +#ifdef BufStats + +void nex_malloc_reset_stats(void) +{ + gen_malloc_reset_stats(&nex_malloc_ctx); +} + +void nex_malloc_get_stats(struct malloc_stats *stats) +{ + gen_malloc_get_stats(&nex_malloc_ctx, stats); +} + +#endif + +#endif diff --git a/optee_os/lib/libutils/isoc/fp.c b/optee_os/lib/libutils/isoc/fp.c new file mode 100644 index 0000000..ee7445b --- /dev/null +++ b/optee_os/lib/libutils/isoc/fp.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ + +#include + +struct _FILE { + char dummy; +}; + +static struct _FILE _fake_stdout; +static struct _FILE _fake_stderr; + +FILE *stdout = &_fake_stdout; +FILE *stderr = &_fake_stderr; diff --git a/optee_os/lib/libutils/isoc/fputc.c b/optee_os/lib/libutils/isoc/fputc.c new file mode 100644 index 0000000..3f154d7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/fputc.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ + +#include +#include + +int fputc(int c, FILE *stream) +{ + if (stream != stdout && stream != stderr) + abort(); + + return putchar(c); +} diff --git a/optee_os/lib/libutils/isoc/fputs.c b/optee_os/lib/libutils/isoc/fputs.c new file mode 100644 index 0000000..1c9cd87 --- /dev/null +++ b/optee_os/lib/libutils/isoc/fputs.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ + +#include +#include +#include + +int fputs(const char *s, FILE *stream) +{ + if (stream != stdout && stream != stderr) + abort(); + + trace_ext_puts(s); + return 0; +} diff --git a/optee_os/lib/libutils/isoc/fwrite.c b/optee_os/lib/libutils/isoc/fwrite.c new file mode 100644 index 0000000..0e8fee9 --- /dev/null +++ b/optee_os/lib/libutils/isoc/fwrite.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ + +#include +#include +#include + +size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream) +{ + int fd = 0; + + if (stream == stdout) + fd = 1; + else if (stream == stderr) + fd = 2; + else + abort(); + + return write(fd, ptr, size * nmemb); +} diff --git a/optee_os/lib/libutils/isoc/include/assert.h b/optee_os/lib/libutils/isoc/include/assert.h new file mode 100644 index 0000000..19738eb --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/assert.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __ASSERT_H +#define __ASSERT_H + +#include +#include + +void __noreturn _assert_break(void); +void _assert_log(const char *expr, const char *file, const int line, + const char *func); + +static inline void __noreturn _assert_trap(const char *expr_str, + const char *file, const int line, + const char *func) +{ + _assert_log(expr_str, file, line, func); + _assert_break(); +} + +static inline void _runtime_assert_trap(const char *expr_str, const char *file, + const int line, const char *func) +{ + volatile bool do_break = true; + + _assert_log(expr_str, file, line, func); + if (do_break) + _assert_break(); +} + +/* + * runtime_assert() behaves as assert() except that it doesn't tell the + * compiler it will never return. This can be used to avoid the warning: + * error: function might be candidate for attribute ‘noreturn’ + */ +#ifdef NDEBUG +#define assert(expr) ((void)0) +#define runtime_assert(expr) ((void)0) +#else +#define assert(expr) \ + ((expr) ? (void)0 : _assert_trap(#expr, __FILE__, __LINE__, __func__)) +#define runtime_assert(expr) \ + ((expr) ? (void)0 : \ + _runtime_assert_trap(#expr, __FILE__, __LINE__, __func__)) +#endif + +/* This macro is deprecated, please use static_assert instead */ +#define COMPILE_TIME_ASSERT(x) \ + do { \ + switch (0) { case 0: case ((x) ? 1: 0): default : break; } \ + } while (0) + +#endif + +#if !defined(__cplusplus) || (__cplusplus < 201103L) +#if defined(__HAVE_SINGLE_ARGUMENT_STATIC_ASSERT) +#define static_assert _Static_assert +#else +/* + * In gcc prior to 9.1 _Static_assert requires two arguments. To allow + * passing a single argument to static_assert() add a workaround with + * macros. + */ +#define ___args_count(_0, _1, x, ...) x +#define __args_count(...) ___args_count(__VA_ARGS__, 2, 1, 0) + +#define __static_assert_1(expr) _Static_assert(expr, "") +#define __static_assert_2(expr, msg) _Static_assert(expr, msg) +#define ___static_assert(count, ...) __static_assert_ ## count(__VA_ARGS__) +#define __static_assert(count, ...) ___static_assert(count, __VA_ARGS__) + +#define static_assert(...) \ + __static_assert(__args_count(__VA_ARGS__), __VA_ARGS__) +#endif +#endif /* __ASSERT_H */ diff --git a/optee_os/lib/libutils/isoc/include/ctype.h b/optee_os/lib/libutils/isoc/include/ctype.h new file mode 100644 index 0000000..bf3e588 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/ctype.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018, Linaro Limited + */ +#ifndef __CTYPE_H +#define __CTYPE_H + +int isalnum(int c); +int isalpha(int c); +int iscntrl(int c); +int isdigit(int c); +int isgraph(int c); +int islower(int c); +int isprint(int c); +int ispunct(int c); +int isspace(int c); +int isupper(int c); +int isxdigit(int c); +int tolower(int c); +int toupper(int c); + +#endif /*__CTYPE_H*/ diff --git a/optee_os/lib/libutils/isoc/include/inttypes.h b/optee_os/lib/libutils/isoc/include/inttypes.h new file mode 100644 index 0000000..5915a9d --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/inttypes.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __INTTYPES_H +#define __INTTYPES_H + +#include + +#ifdef __ILP32__ +#define __PRI64_PREFIX "ll" +#endif +#ifdef __LP64__ +#define __PRI64_PREFIX "l" +#endif +#define __PRIPTR_PREFIX "l" + +#define PRId8 "d" +#define PRId16 "d" +#define PRId32 "d" +#define PRId64 __PRI64_PREFIX "d" +#define PRIdPTR __PRIPTR_PREFIX "d" + +#define PRIi8 "i" +#define PRIi16 "i" +#define PRIi32 "i" +#define PRIi64 __PRI64_PREFIX "i" +#define PRIiPTR __PRIPTR_PREFIX "i" + +#define PRIo8 "o" +#define PRIo16 "o" +#define PRIo32 "o" +#define PRIo64 __PRI64_PREFIX "o" +#define PRIoPTR __PRIPTR_PREFIX "o" + +#define PRIu8 "u" +#define PRIu16 "u" +#define PRIu32 "u" +#define PRIu64 __PRI64_PREFIX "u" +#define PRIuPTR __PRIPTR_PREFIX "u" + +#define PRIx8 "x" +#define PRIx16 "x" +#define PRIx32 "x" +#define PRIx64 __PRI64_PREFIX "x" +#define PRIxPTR __PRIPTR_PREFIX "x" + +#define PRIX8 "X" +#define PRIX16 "X" +#define PRIX32 "X" +#define PRIX64 __PRI64_PREFIX "X" +#define PRIXPTR __PRIPTR_PREFIX "X" + +#endif /*__INTTYPES_H*/ diff --git a/optee_os/lib/libutils/isoc/include/limits.h b/optee_os/lib/libutils/isoc/include/limits.h new file mode 100644 index 0000000..4dbec54 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/limits.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __LIMITS_H +#define __LIMITS_H + +#define CHAR_BIT __CHAR_BIT__ + +#ifdef __CHAR_UNSIGNED__ +#define CHAR_MAX UCHAR_MAX +#define CHAR_MIN 0 +#else +#define CHAR_MAX SCHAR_MAX +#define CHAR_MIN SCHAR_MIN +#endif + +#define INT_MAX __INT_MAX__ +#define INT_MIN (-INT_MAX - 1) + +#define LONG_MAX __LONG_MAX__ +#define LONG_MIN (-LONG_MAX - 1L) + +#define LLONG_MAX __LONG_LONG_MAX__ +#define LLONG_MIN (-LLONG_MAX - 1LL) + +#define MB_LEN_MAX 1 + +#define SCHAR_MAX __SCHAR_MAX__ +#define SCHAR_MIN (-SCHAR_MAX - 1) + +#define SHRT_MAX __SHRT_MAX__ +#define SHRT_MIN (-SHRT_MAX - 1) + +#if __SCHAR_MAX__ == __INT_MAX__ +#define UCHAR_MAX (SCHAR_MAX * 2U + 1U) +#else +#define UCHAR_MAX (SCHAR_MAX * 2 + 1) +#endif + +#if __SHRT_MAX__ == __INT_MAX__ +#define USHRT_MAX (SHRT_MAX * 2U + 1U) +#else +#define USHRT_MAX (SHRT_MAX * 2 + 1) +#endif + +#define UINT_MAX (INT_MAX * 2U + 1U) + +#define ULONG_MAX (LONG_MAX * 2UL + 1UL) +#define ULLONG_MAX (LLONG_MAX * 2ULL + 1ULL) + +#endif /* __LIMITS_H */ diff --git a/optee_os/lib/libutils/isoc/include/malloc.h b/optee_os/lib/libutils/isoc/include/malloc.h new file mode 100644 index 0000000..d261955 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/malloc.h @@ -0,0 +1,180 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __MALLOC_H +#define __MALLOC_H + +#include +#include + +/* + * Due to bget implementation, the first memory pool registered shall have + * a min size. Choose 1kB which is reasonable. + */ +#define MALLOC_INITIAL_POOL_MIN_SIZE 1024 + +void *malloc(size_t size); +void *calloc(size_t nmemb, size_t size); +void *realloc(void *ptr, size_t size); +void *memalign(size_t alignment, size_t size); +void free(void *ptr); + +#if __STDC_VERSION__ >= 201112L +void *aligned_alloc(size_t alignment, size_t size); +#endif + +#ifdef ENABLE_MDBG + +void *mdbg_malloc(const char *fname, int lineno, size_t size); +void *mdbg_calloc(const char *fname, int lineno, size_t nmemb, size_t size); +void *mdbg_realloc(const char *fname, int lineno, void *ptr, size_t size); +void *mdbg_memalign(const char *fname, int lineno, size_t alignment, + size_t size); + +#if __STDC_VERSION__ >= 201112L +void *mdbg_aligned_alloc(const char *fname, int lineno, size_t alignment, + size_t size); +#endif + +void mdbg_check(int bufdump); + +#define malloc(size) mdbg_malloc(__FILE__, __LINE__, (size)) +#define calloc(nmemb, size) \ + mdbg_calloc(__FILE__, __LINE__, (nmemb), (size)) +#define realloc(ptr, size) \ + mdbg_realloc(__FILE__, __LINE__, (ptr), (size)) +#define memalign(alignment, size) \ + mdbg_memalign(__FILE__, __LINE__, (alignment), (size)) + +#if __STDC_VERSION__ >= 201112L +#define aligned_alloc(alignment, size) \ + mdbg_aligned_alloc(__FILE__, __LINE__, (alignment), (size)) +#endif /* __STDC_VERSION__ */ + +#else + +#define mdbg_check(x) do { } while (0) + +#endif + +/* + * Returns true if the supplied memory area is within a buffer + * previously allocated (and not freed yet). + * + * Used internally by TAs + */ +bool malloc_buffer_is_within_alloced(void *buf, size_t len); + +/* + * Returns true if the supplied memory area is overlapping the area used + * for heap. + * + * Used internally by TAs + */ +bool malloc_buffer_overlaps_heap(void *buf, size_t len); + +/* + * Adds a pool of memory to allocate from. + */ +void malloc_add_pool(void *buf, size_t len); + +#ifdef CFG_WITH_STATS +/* + * Get/reset allocation statistics + */ + +#define TEE_ALLOCATOR_DESC_LENGTH 32 +struct malloc_stats { + char desc[TEE_ALLOCATOR_DESC_LENGTH]; + uint32_t allocated; /* Bytes currently allocated */ + uint32_t max_allocated; /* Tracks max value of allocated */ + uint32_t size; /* Total size for this allocator */ + uint32_t num_alloc_fail; /* Number of failed alloc requests */ + uint32_t biggest_alloc_fail; /* Size of biggest failed alloc */ + uint32_t biggest_alloc_fail_used; /* Alloc bytes when above occurred */ +}; + +void malloc_get_stats(struct malloc_stats *stats); +void malloc_reset_stats(void); +#endif /* CFG_WITH_STATS */ + + +#ifdef CFG_NS_VIRTUALIZATION + +void nex_free(void *ptr); + +#ifdef ENABLE_MDBG + +void *nex_mdbg_malloc(const char *fname, int lineno, size_t size); +void *nex_mdbg_calloc(const char *fname, int lineno, size_t nmemb, size_t size); +void *nex_mdbg_realloc(const char *fname, int lineno, void *ptr, size_t size); +void *nex_mdbg_memalign(const char *fname, int lineno, size_t alignment, + size_t size); + +void nex_mdbg_check(int bufdump); + +#define nex_malloc(size) nex_mdbg_malloc(__FILE__, __LINE__, (size)) +#define nex_calloc(nmemb, size) \ + nex_mdbg_calloc(__FILE__, __LINE__, (nmemb), (size)) +#define nex_realloc(ptr, size) \ + nex_mdbg_realloc(__FILE__, __LINE__, (ptr), (size)) +#define nex_memalign(alignment, size) \ + nex_mdbg_memalign(__FILE__, __LINE__, (alignment), (size)) + +#else /* ENABLE_MDBG */ + +void *nex_malloc(size_t size); +void *nex_calloc(size_t nmemb, size_t size); +void *nex_realloc(void *ptr, size_t size); +void *nex_memalign(size_t alignment, size_t size); + +#define nex_mdbg_check(x) do { } while (0) + +#endif /* ENABLE_MDBG */ + +bool nex_malloc_buffer_is_within_alloced(void *buf, size_t len); +bool nex_malloc_buffer_overlaps_heap(void *buf, size_t len); +void nex_malloc_add_pool(void *buf, size_t len); + +#ifdef CFG_WITH_STATS +/* + * Get/reset allocation statistics + */ + +void nex_malloc_get_stats(struct malloc_stats *stats); +void nex_malloc_reset_stats(void); + +#endif /* CFG_WITH_STATS */ +#else /* CFG_NS_VIRTUALIZATION */ + +#define nex_free(ptr) free(ptr) +#define nex_malloc(size) malloc(size) +#define nex_calloc(nmemb, size) calloc(nmemb, size) +#define nex_realloc(ptr, size) realloc(ptr, size) +#define nex_memalign(alignment, size) memalign(alignment, size) + +#endif /* CFG_NS_VIRTUALIZATION */ + +struct malloc_ctx; +void *raw_memalign(size_t hdr_size, size_t ftr_size, size_t alignment, + size_t pl_size, struct malloc_ctx *ctx); +void *raw_malloc(size_t hdr_size, size_t ftr_size, size_t pl_size, + struct malloc_ctx *ctx); +void raw_free(void *ptr, struct malloc_ctx *ctx, bool wipe); +void *raw_calloc(size_t hdr_size, size_t ftr_size, size_t pl_nmemb, + size_t pl_size, struct malloc_ctx *ctx); +void *raw_realloc(void *ptr, size_t hdr_size, size_t ftr_size, + size_t pl_size, struct malloc_ctx *ctx); +size_t raw_malloc_get_ctx_size(void); +void raw_malloc_init_ctx(struct malloc_ctx *ctx); +void raw_malloc_add_pool(struct malloc_ctx *ctx, void *buf, size_t len); +bool raw_malloc_buffer_overlaps_heap(struct malloc_ctx *ctx, + void *buf, size_t len); +bool raw_malloc_buffer_is_within_alloced(struct malloc_ctx *ctx, + void *buf, size_t len); +#ifdef CFG_WITH_STATS +void raw_malloc_get_stats(struct malloc_ctx *ctx, struct malloc_stats *stats); +#endif + +#endif /* __MALLOC_H */ diff --git a/optee_os/lib/libutils/isoc/include/memory.h b/optee_os/lib/libutils/isoc/include/memory.h new file mode 100644 index 0000000..06ecdc7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/memory.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __MEMORY_H +#define __MEMORY_H +#include + +#endif /*__MEMORY_H*/ diff --git a/optee_os/lib/libutils/isoc/include/setjmp.h b/optee_os/lib/libutils/isoc/include/setjmp.h new file mode 100644 index 0000000..cb1e868 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/setjmp.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * Copyright (c) 2016, Linaro Limited + * Copyright 2022-2023 NXP + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SETJMP_H +#define __SETJMP_H + +#include + +#if defined(ARM32) +/* + * All callee preserved registers: + * v1 - v7, fp, ip, sp, lr, f4, f5, f6, f7 + * One additional 32-bit value used in case ftrace + * is enabled to restore ftrace return stack. + */ +#define _JBLEN 24 +#define _JBTYPE int +#endif + +#if defined(ARM64) +#define _JBLEN 22 +#define _JBTYPE long long +#endif + +#if defined(RV64) || defined(RV32) +/* + * Callee preserved registers: + * s0-s11, ra, sp + * One additional value used in case ftrace + * is enabled to restore ftrace return stack. + */ +#define _JBLEN 15 +#define _JBTYPE unsigned long +#endif + +#ifdef _JBLEN +typedef _JBTYPE jmp_buf[_JBLEN]; +#endif + +void __noreturn longjmp(jmp_buf env, int val); +int setjmp(jmp_buf env); + +#ifdef CFG_FTRACE_SUPPORT +void ftrace_longjmp(unsigned int *ret_idx); +void ftrace_setjmp(unsigned int *ret_idx); +#endif + +#endif /*__SETJMP_H*/ diff --git a/optee_os/lib/libutils/isoc/include/signal.h b/optee_os/lib/libutils/isoc/include/signal.h new file mode 100644 index 0000000..ee0bbdd --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/signal.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __SIGNAL_H +#define __SIGNAL_H + +#endif /*__SIGNAL_H*/ diff --git a/optee_os/lib/libutils/isoc/include/stdint.h b/optee_os/lib/libutils/isoc/include/stdint.h new file mode 100644 index 0000000..687fe7e --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/stdint.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +#include + +/* + * This file provides what C99 standard requires in + * 7.18 interger types + */ + +#ifndef __STDINT_H +#define __STDINT_H + +/* + * If compiler supplies neither __ILP32__ or __LP64__, try to figure it out + * here. + */ +#if !defined(__ILP32__) && !defined(__LP64__) +#if defined(__SIZEOF_INT__) && defined(__SIZEOF_POINTER__) && \ + defined(__SIZEOF_LONG__) +#if __SIZEOF_INT__ == 4 && __SIZEOF_POINTER__ == 4 && __SIZEOF_LONG__ == 4 +#define __ILP32__ 1 +#endif +#if __SIZEOF_INT__ == 4 && __SIZEOF_POINTER__ == 8 && __SIZEOF_LONG__ == 8 +#define __LP64__ 1 +#endif +#endif +#endif /* !defined(__ILP32__) && !defined(__LP64__) */ + +#if !defined(__ILP32__) && !defined(__LP64__) +#error Neither __ILP32__ nor __LP64__ is defined +#endif + +#ifndef __ASSEMBLER__ + +/* 7.18.1.1 Exact-width interger types */ +#ifndef __int8_t_defined +# define __int8_t_defined +typedef signed char int8_t; +typedef short int int16_t; +typedef int int32_t; +#ifdef __ILP32__ +__extension__ +typedef long long int int64_t; +#endif /*__ILP32__*/ +#ifdef __LP64__ +typedef long int int64_t; +#endif /*__LP64__*/ +#endif + +/* Unsigned. */ +typedef unsigned char uint8_t; +typedef unsigned short int uint16_t; +#ifndef __uint32_t_defined +typedef unsigned int uint32_t; +# define __uint32_t_defined +#endif +#ifdef __ILP32__ +__extension__ +typedef unsigned long long int uint64_t; +#endif /*__ILP32__*/ +#ifdef __LP64__ +typedef unsigned long int uint64_t; +#endif /*__LP64__*/ + +/* 7.18.1.2 Minimum-width integer types */ +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +/* 7.18.1.3 Fastest minimum-width integer types */ +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +/* 7.18.1.4 Integer types capable of holding object pointers */ +typedef long intptr_t; +typedef unsigned long uintptr_t; + +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + +#endif /*__ASSEMBLER__*/ + +/* + * 7.18.2 Limits of specified-width integer types + */ + +/* 7.18.2.1 Limits of exact-width interger types */ + +#define INT8_MIN (-0x7f-1) +#define INT16_MIN (-0x7fff-1) +#define INT32_MIN (-0x7fffffff-1) +#define INT64_MIN (-0x7fffffffffffffffL-1) + +#define INT8_MAX 0x7f +#define INT16_MAX 0x7fff +#define INT32_MAX 0x7fffffff +#define INT64_MAX 0x7fffffffffffffffL + +#define UINT8_MAX 0xff +#define UINT16_MAX 0xffff +#define UINT32_MAX 0xffffffffU +#define UINT64_MAX 0xffffffffffffffffUL + +/* 7.18.2.2 Limits of minimum-width integer types */ + +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST64_MIN INT64_MIN + +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MAX INT64_MAX + +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +/* 7.18.2.3 Limits of fastest minimum-width integer types */ + +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST64_MIN INT64_MIN + +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MAX INT64_MAX + +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +/* 7.18.2.4 Limits of integer types capable of holding object pointers */ + +#define INTPTR_MIN LONG_MIN +#define INTPTR_MAX LONG_MAX +#define UINTPTR_MAX ULONG_MAX + +/* 7.18.2.5 Limits of greatest-width integer types */ +#define INTMAX_MAX INT64_MAX +#define INTMAX_MIN INT64_MIN +#define UINTMAX_MAX UINT64_MAX + +/* 7.18.3 Limits of other integer types */ +#define SIZE_MAX ULONG_MAX + +/* + * 7.18.4 Macros for integer constants + */ + +#ifdef __ASSEMBLER__ +#define U(v) v +#define UL(v) v +#define ULL(v) v +#define L(v) v +#define LL(v) v +#else +#define U(v) v ## U +#define UL(v) v ## UL +#define ULL(v) v ## ULL +#define L(v) v ## L +#define LL(v) v ## LL +#endif + +/* 7.18.4.1 Macros for minimum-width integer constants */ + +#define INT8_C(v) v +#define UINT8_C(v) v +#define INT16_C(v) v +#define UINT16_C(v) v +#define INT32_C(v) v +#define UINT32_C(v) U(v) +#ifdef __ILP32__ +#define INT64_C(v) LL(v) +#define UINT64_C(v) ULL(v) +#endif +#ifdef __LP64__ +#define INT64_C(v) L(v) +#define UINT64_C(v) UL(v) +#endif + +#define UINTPTR_C(v) UL(v) + +/* 7.18.4.2 Macros for greatest-width integer constants */ + +#define INTMAX_C(v) INT64_C(v) +#define UINTMAX_C(v) UINT64_C(v) + +#endif /* __STDINT_H */ diff --git a/optee_os/lib/libutils/isoc/include/stdio.h b/optee_os/lib/libutils/isoc/include/stdio.h new file mode 100644 index 0000000..c73e6a8 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/stdio.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __STDIO_H +#define __STDIO_H + +#include +#include + +typedef struct _FILE FILE; + +int printf(const char *fmt, ...) + __attribute__ ((__format__ (__printf__, 1, 2))); +/* sprintf() is unsafe and should not be used. Prefer snprintf(). */ +int sprintf(char *str, const char *fmt, ...) + __attribute__ ((__format__ (__printf__, 2, 3))) + __attribute__ ((deprecated)); +int snprintf(char *str, size_t size, const char *fmt, ...) + __attribute__ ((__format__ (__printf__, 3, 4))); +int vsnprintf (char *str, size_t size, const char *fmt, va_list ap) + __attribute__ ((__format__ (__printf__, 3, 0))); +int __sprintf_chk(char *str, int flag, size_t slen, const char *fmt, ...) + __attribute__ ((__format__ (__printf__, 4, 5))); + +int puts(const char *str); +int putchar(int c); + +#ifndef __KERNEL__ + +extern FILE *stdout; +extern FILE *stderr; + +/* + * The functions below send their output synchronously to the secure console. + * They treat stdout and stderr the same, and will abort if stream is not one or + * the other. + */ + +int fputc(int c, FILE *stream); +int fputs(const char *s, FILE *stream); +size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); +#endif + +#endif /*__STDIO_H*/ diff --git a/optee_os/lib/libutils/isoc/include/stdlib.h b/optee_os/lib/libutils/isoc/include/stdlib.h new file mode 100644 index 0000000..78b192f --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/stdlib.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +/* + * This file provides what C99 standard requires for in + * 7.20.3 Memory management functions + */ + +#ifndef __STDLIB_H +#define __STDLIB_H + +#include +#include +#include + +void +qsort(void *aa, size_t n, size_t es, int (*cmp)(const void *, const void *)); + +void abort(void) __noreturn; + +int abs(int i); + +/* The largest number returned by rand() */ +#define RAND_MAX __INT_MAX__ +int rand(void); + +unsigned long _strtoul (const char *nptr, char **endptr, int base); +unsigned long strtoul (const char *s, char **ptr, int base); + +#endif /* __STDLIB_H */ diff --git a/optee_os/lib/libutils/isoc/include/string.h b/optee_os/lib/libutils/isoc/include/string.h new file mode 100644 index 0000000..c6dbbf3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/string.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ + +/* + * This file provides what C99 standard requires for + * for some functions + */ + +#ifndef __STRING_H +#define __STRING_H + +#include +#include + +void *memcpy(void *__restrict s1, const void *__restrict s2, size_t n); +void *memmove(void *s1, const void *s2, size_t n); +int memcmp(const void *s1, const void *s2, size_t n); +void *memset(void *s, int c, size_t n); + +int strcmp(const char *s1, const char *s2); +int strncmp(const char *s1, const char *s2, size_t n); +size_t strlen(const char *s); +size_t strnlen(const char *s, size_t n); +char *strdup(const char *s); +char *strndup(const char *s, size_t n); +char *strchr(const char *s, int c); +char *strstr(const char *big, const char *little); +char *strcpy(char *dest, const char *src); +char *strncpy(char *dest, const char *src, size_t n); +char *strrchr(const char *s, int i); +char *strtok_r(char *str, const char *delim, char **saveptr); + +void *memchr(const void *buf, int c, size_t length); + +#endif /* __STRING_H */ diff --git a/optee_os/lib/libutils/isoc/include/strings.h b/optee_os/lib/libutils/isoc/include/strings.h new file mode 100644 index 0000000..32917c5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/strings.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ +#ifndef __STRINGS_H +#define __STRINGS_H + +int bcmp(const void *s1, const void *s2, size_t n); + +#endif /*__STRINGS_H*/ diff --git a/optee_os/lib/libutils/isoc/include/sys/cdefs.h b/optee_os/lib/libutils/isoc/include/sys/cdefs.h new file mode 100644 index 0000000..be71531 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/sys/cdefs.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef SYS_CDEFS_H +#define SYS_CDEFS_H + +/* + * GCC 2.95 provides `__restrict' as an extension to C90 to support the + * C99-specific `restrict' type qualifier. We happen to use `__restrict' as + * a way to define the `restrict' type qualifier without disturbing older + * software that is unaware of C99 keywords. + */ +#if !(__GNUC__ == 2 && __GNUC_MINOR__ == 95) +#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901 || defined(lint) +#define __restrict +#else +#define __restrict restrict +#endif +#endif + +#define __always_inline __attribute__((always_inline)) inline + +#endif /*SYS_CDEFS_H*/ diff --git a/optee_os/lib/libutils/isoc/include/sys/queue.h b/optee_os/lib/libutils/isoc/include/sys/queue.h new file mode 100644 index 0000000..48c4bb5 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/sys/queue.h @@ -0,0 +1,721 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* $NetBSD: queue.h,v 1.49.6.1 2008/11/20 03:22:38 snj Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + */ + +#ifndef _SYS_QUEUE_H_ +#define _SYS_QUEUE_H_ + +/*#include */ + +/* + * This file defines five types of data structures: singly-linked lists, + * lists, simple queues, tail queues, and circular queues. + * + * A singly-linked list is headed by a single forward pointer. The + * elements are singly linked for minimum space and pointer manipulation + * overhead at the expense of O(n) removal for arbitrary elements. New + * elements can be added to the list after an existing element or at the + * head of the list. Elements being removed from the head of the list + * should use the explicit macro for this purpose for optimum + * efficiency. A singly-linked list may only be traversed in the forward + * direction. Singly-linked lists are ideal for applications with large + * datasets and few or no removals or for implementing a LIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may only be traversed in the forward direction. + * + * A simple queue is headed by a pair of pointers, one the head of the + * list and the other to the tail of the list. The elements are singly + * linked to save space, so elements can only be removed from the + * head of the list. New elements can be added to the list after + * an existing element, at the head of the list, or at the end of the + * list. A simple queue may only be traversed in the forward direction. + * + * A tail queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or + * after an existing element, at the head of the list, or at the end of + * the list. A tail queue may be traversed in either direction. + * + * A circle queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or after + * an existing element, at the head of the list, or at the end of the list. + * A circle queue may be traversed in either direction, but has a more + * complex end of list detection. + * + * For details on the use of these macros, see the queue(3) manual page. + */ + +/* + * List definitions. + */ +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define LIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +/* + * List functions. + */ +#if defined(_KERNEL) && defined(QUEUEDEBUG) +#define QUEUEDEBUG_LIST_INSERT_HEAD(head, elm, field) \ + if ((head)->lh_first && \ + (head)->lh_first->field.le_prev != &(head)->lh_first) \ + panic("LIST_INSERT_HEAD %p %s:%d", (head), __FILE__, __LINE__); +#define QUEUEDEBUG_LIST_OP(elm, field) \ + if ((elm)->field.le_next && \ + (elm)->field.le_next->field.le_prev != \ + &(elm)->field.le_next) \ + panic("LIST_* forw %p %s:%d", (elm), __FILE__, __LINE__);\ + if (*(elm)->field.le_prev != (elm)) \ + panic("LIST_* back %p %s:%d", (elm), __FILE__, __LINE__); +#define QUEUEDEBUG_LIST_POSTREMOVE(elm, field) \ + (elm)->field.le_next = (void *)1L; \ + (elm)->field.le_prev = (void *)1L; +#else +#define QUEUEDEBUG_LIST_INSERT_HEAD(head, elm, field) +#define QUEUEDEBUG_LIST_OP(elm, field) +#define QUEUEDEBUG_LIST_POSTREMOVE(elm, field) +#endif + +#define LIST_INIT(head) do { \ + (head)->lh_first = NULL; \ +} while (/* CONSTCOND */0) + +#define LIST_INSERT_AFTER(listelm, elm, field) do { \ + QUEUEDEBUG_LIST_OP((listelm), field) \ + if (((elm)->field.le_next = (listelm)->field.le_next) != NULL) \ + (listelm)->field.le_next->field.le_prev = \ + &(elm)->field.le_next; \ + (listelm)->field.le_next = (elm); \ + (elm)->field.le_prev = &(listelm)->field.le_next; \ +} while (/* CONSTCOND */0) + +#define LIST_INSERT_BEFORE(listelm, elm, field) do { \ + QUEUEDEBUG_LIST_OP((listelm), field) \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + (elm)->field.le_next = (listelm); \ + *(listelm)->field.le_prev = (elm); \ + (listelm)->field.le_prev = &(elm)->field.le_next; \ +} while (/* CONSTCOND */0) + +#define LIST_INSERT_HEAD(head, elm, field) do { \ + QUEUEDEBUG_LIST_INSERT_HEAD((head), (elm), field) \ + if (((elm)->field.le_next = (head)->lh_first) != NULL) \ + (head)->lh_first->field.le_prev = &(elm)->field.le_next;\ + (head)->lh_first = (elm); \ + (elm)->field.le_prev = &(head)->lh_first; \ +} while (/* CONSTCOND */0) + +#define LIST_REMOVE(elm, field) do { \ + QUEUEDEBUG_LIST_OP((elm), field) \ + if ((elm)->field.le_next != NULL) \ + (elm)->field.le_next->field.le_prev = \ + (elm)->field.le_prev; \ + *(elm)->field.le_prev = (elm)->field.le_next; \ + QUEUEDEBUG_LIST_POSTREMOVE((elm), field) \ +} while (/* CONSTCOND */0) + +#define LIST_FOREACH(var, head, field) \ + for ((var) = ((head)->lh_first); \ + (var); \ + (var) = ((var)->field.le_next)) + +/* + * List access methods. + */ +#define LIST_EMPTY(head) ((head)->lh_first == NULL) +#define LIST_FIRST(head) ((head)->lh_first) +#define LIST_NEXT(elm, field) ((elm)->field.le_next) + +#define LIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = LIST_FIRST((head)); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +/* + * Singly-linked List definitions. + */ +#define SLIST_HEAD(name, type) \ +struct name { \ + struct type *slh_first; /* first element */ \ +} + +#define SLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +/* + * Singly-linked List functions. + */ +#define SLIST_INIT(head) do { \ + (head)->slh_first = NULL; \ +} while (/* CONSTCOND */0) + +#define SLIST_INSERT_AFTER(slistelm, elm, field) do { \ + (elm)->field.sle_next = (slistelm)->field.sle_next; \ + (slistelm)->field.sle_next = (elm); \ +} while (/* CONSTCOND */0) + +#define SLIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.sle_next = (head)->slh_first; \ + (head)->slh_first = (elm); \ +} while (/* CONSTCOND */0) + +#define SLIST_REMOVE_HEAD(head, field) do { \ + (head)->slh_first = (head)->slh_first->field.sle_next; \ +} while (/* CONSTCOND */0) + +#define SLIST_REMOVE(head, elm, type, field) do { \ + if ((head)->slh_first == (elm)) { \ + SLIST_REMOVE_HEAD((head), field); \ + } \ + else { \ + struct type *curelm = (head)->slh_first; \ + while(curelm->field.sle_next != (elm)) \ + curelm = curelm->field.sle_next; \ + curelm->field.sle_next = \ + curelm->field.sle_next->field.sle_next; \ + } \ +} while (/* CONSTCOND */0) + +#define SLIST_REMOVE_AFTER(slistelm, field) do { \ + (slistelm)->field.sle_next = \ + SLIST_NEXT(SLIST_NEXT((slistelm), field), field); \ +} while (/* CONSTCOND */0) + +#define SLIST_FOREACH(var, head, field) \ + for((var) = (head)->slh_first; (var); (var) = (var)->field.sle_next) + +#define SLIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = SLIST_FIRST((head)); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +/* + * Singly-linked List access methods. + */ +#define SLIST_EMPTY(head) ((head)->slh_first == NULL) +#define SLIST_FIRST(head) ((head)->slh_first) +#define SLIST_NEXT(elm, field) ((elm)->field.sle_next) + +/* + * Singly-linked Tail queue declarations. + */ +#define STAILQ_HEAD(name, type) \ +struct name { \ + struct type *stqh_first; /* first element */ \ + struct type **stqh_last; /* addr of last next element */ \ +} + +#define STAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).stqh_first } + +#define STAILQ_ENTRY(type) \ +struct { \ + struct type *stqe_next; /* next element */ \ +} + +/* + * Singly-linked Tail queue functions. + */ +#define STAILQ_INIT(head) do { \ + (head)->stqh_first = NULL; \ + (head)->stqh_last = &(head)->stqh_first; \ +} while (/* CONSTCOND */0) + +#define STAILQ_INSERT_HEAD(head, elm, field) do { \ + if (((elm)->field.stqe_next = (head)->stqh_first) == NULL) \ + (head)->stqh_last = &(elm)->field.stqe_next; \ + (head)->stqh_first = (elm); \ +} while (/* CONSTCOND */0) + +#define STAILQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.stqe_next = NULL; \ + *(head)->stqh_last = (elm); \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (/* CONSTCOND */0) + +#define STAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ + if (((elm)->field.stqe_next = (listelm)->field.stqe_next) == NULL)\ + (head)->stqh_last = &(elm)->field.stqe_next; \ + (listelm)->field.stqe_next = (elm); \ +} while (/* CONSTCOND */0) + +#define STAILQ_REMOVE_HEAD(head, field) do { \ + if (((head)->stqh_first = (head)->stqh_first->field.stqe_next) == NULL) \ + (head)->stqh_last = &(head)->stqh_first; \ +} while (/* CONSTCOND */0) + +#define STAILQ_REMOVE(head, elm, type, field) do { \ + if ((head)->stqh_first == (elm)) { \ + STAILQ_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->stqh_first; \ + while (curelm->field.stqe_next != (elm)) \ + curelm = curelm->field.stqe_next; \ + if ((curelm->field.stqe_next = \ + curelm->field.stqe_next->field.stqe_next) == NULL) \ + (head)->stqh_last = &(curelm)->field.stqe_next; \ + } \ +} while (/* CONSTCOND */0) + +#define STAILQ_REMOVE_AFTER(head, elm, field) do { \ + if ((STAILQ_NEXT(elm, field) = \ + STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_FOREACH(var, head, field) \ + for ((var) = ((head)->stqh_first); \ + (var); \ + (var) = ((var)->field.stqe_next)) + +#define STAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = STAILQ_FIRST((head)); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_CONCAT(head1, head2) do { \ + if (!STAILQ_EMPTY((head2))) { \ + *(head1)->stqh_last = (head2)->stqh_first; \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_INIT((head2)); \ + } \ +} while (/* CONSTCOND */0) + +/* + * Singly-linked Tail queue access methods. + */ +#define STAILQ_EMPTY(head) ((head)->stqh_first == NULL) +#define STAILQ_FIRST(head) ((head)->stqh_first) +#define STAILQ_NEXT(elm, field) ((elm)->field.stqe_next) + +/* + * Simple queue definitions. + */ +#define SIMPLEQ_HEAD(name, type) \ +struct name { \ + struct type *sqh_first; /* first element */ \ + struct type **sqh_last; /* addr of last next element */ \ +} + +#define SIMPLEQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).sqh_first } + +#define SIMPLEQ_ENTRY(type) \ +struct { \ + struct type *sqe_next; /* next element */ \ +} + +/* + * Simple queue functions. + */ +#define SIMPLEQ_INIT(head) do { \ + (head)->sqh_first = NULL; \ + (head)->sqh_last = &(head)->sqh_first; \ +} while (/* CONSTCOND */0) + +#define SIMPLEQ_INSERT_HEAD(head, elm, field) do { \ + if (((elm)->field.sqe_next = (head)->sqh_first) == NULL) \ + (head)->sqh_last = &(elm)->field.sqe_next; \ + (head)->sqh_first = (elm); \ +} while (/* CONSTCOND */0) + +#define SIMPLEQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.sqe_next = NULL; \ + *(head)->sqh_last = (elm); \ + (head)->sqh_last = &(elm)->field.sqe_next; \ +} while (/* CONSTCOND */0) + +#define SIMPLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ + if (((elm)->field.sqe_next = (listelm)->field.sqe_next) == NULL)\ + (head)->sqh_last = &(elm)->field.sqe_next; \ + (listelm)->field.sqe_next = (elm); \ +} while (/* CONSTCOND */0) + +#define SIMPLEQ_REMOVE_HEAD(head, field) do { \ + if (((head)->sqh_first = (head)->sqh_first->field.sqe_next) == NULL) \ + (head)->sqh_last = &(head)->sqh_first; \ +} while (/* CONSTCOND */0) + +#define SIMPLEQ_REMOVE(head, elm, type, field) do { \ + if ((head)->sqh_first == (elm)) { \ + SIMPLEQ_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->sqh_first; \ + while (curelm->field.sqe_next != (elm)) \ + curelm = curelm->field.sqe_next; \ + if ((curelm->field.sqe_next = \ + curelm->field.sqe_next->field.sqe_next) == NULL) \ + (head)->sqh_last = &(curelm)->field.sqe_next; \ + } \ +} while (/* CONSTCOND */0) + +#define SIMPLEQ_FOREACH(var, head, field) \ + for ((var) = ((head)->sqh_first); \ + (var); \ + (var) = ((var)->field.sqe_next)) + +/* + * Simple queue access methods. + */ +#define SIMPLEQ_EMPTY(head) ((head)->sqh_first == NULL) +#define SIMPLEQ_FIRST(head) ((head)->sqh_first) +#define SIMPLEQ_NEXT(elm, field) ((elm)->field.sqe_next) + +/* + * Tail queue definitions. + */ +#define _TAILQ_HEAD(name, type, qual) \ +struct name { \ + qual type *tqh_first; /* first element */ \ + qual type *qual *tqh_last; /* addr of last next element */ \ +} +#define TAILQ_HEAD(name, type) _TAILQ_HEAD(name, struct type,) + +#define TAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).tqh_first } + +#define _TAILQ_ENTRY(type, qual) \ +struct { \ + qual type *tqe_next; /* next element */ \ + qual type *qual *tqe_prev; /* address of previous next element */\ +} +#define TAILQ_ENTRY(type) _TAILQ_ENTRY(struct type,) + +/* + * Tail queue functions. + */ +#if defined(_KERNEL) && defined(QUEUEDEBUG) +#define QUEUEDEBUG_TAILQ_INSERT_HEAD(head, elm, field) \ + if ((head)->tqh_first && \ + (head)->tqh_first->field.tqe_prev != &(head)->tqh_first) \ + panic("TAILQ_INSERT_HEAD %p %s:%d", (head), __FILE__, __LINE__); +#define QUEUEDEBUG_TAILQ_INSERT_TAIL(head, elm, field) \ + if (*(head)->tqh_last != NULL) \ + panic("TAILQ_INSERT_TAIL %p %s:%d", (head), __FILE__, __LINE__); +#define QUEUEDEBUG_TAILQ_OP(elm, field) \ + if ((elm)->field.tqe_next && \ + (elm)->field.tqe_next->field.tqe_prev != \ + &(elm)->field.tqe_next) \ + panic("TAILQ_* forw %p %s:%d", (elm), __FILE__, __LINE__);\ + if (*(elm)->field.tqe_prev != (elm)) \ + panic("TAILQ_* back %p %s:%d", (elm), __FILE__, __LINE__); +#define QUEUEDEBUG_TAILQ_PREREMOVE(head, elm, field) \ + if ((elm)->field.tqe_next == NULL && \ + (head)->tqh_last != &(elm)->field.tqe_next) \ + panic("TAILQ_PREREMOVE head %p elm %p %s:%d", \ + (head), (elm), __FILE__, __LINE__); +#define QUEUEDEBUG_TAILQ_POSTREMOVE(elm, field) \ + (elm)->field.tqe_next = (void *)1L; \ + (elm)->field.tqe_prev = (void *)1L; +#else +#define QUEUEDEBUG_TAILQ_INSERT_HEAD(head, elm, field) +#define QUEUEDEBUG_TAILQ_INSERT_TAIL(head, elm, field) +#define QUEUEDEBUG_TAILQ_OP(elm, field) +#define QUEUEDEBUG_TAILQ_PREREMOVE(head, elm, field) +#define QUEUEDEBUG_TAILQ_POSTREMOVE(elm, field) +#endif + +#define TAILQ_INIT(head) do { \ + (head)->tqh_first = NULL; \ + (head)->tqh_last = &(head)->tqh_first; \ +} while (/* CONSTCOND */0) + +#define TAILQ_INSERT_HEAD(head, elm, field) do { \ + QUEUEDEBUG_TAILQ_INSERT_HEAD((head), (elm), field) \ + if (((elm)->field.tqe_next = (head)->tqh_first) != NULL) \ + (head)->tqh_first->field.tqe_prev = \ + &(elm)->field.tqe_next; \ + else \ + (head)->tqh_last = &(elm)->field.tqe_next; \ + (head)->tqh_first = (elm); \ + (elm)->field.tqe_prev = &(head)->tqh_first; \ +} while (/* CONSTCOND */0) + +#define TAILQ_INSERT_TAIL(head, elm, field) do { \ + QUEUEDEBUG_TAILQ_INSERT_TAIL((head), (elm), field) \ + (elm)->field.tqe_next = NULL; \ + (elm)->field.tqe_prev = (head)->tqh_last; \ + *(head)->tqh_last = (elm); \ + (head)->tqh_last = &(elm)->field.tqe_next; \ +} while (/* CONSTCOND */0) + +#define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ + QUEUEDEBUG_TAILQ_OP((listelm), field) \ + if (((elm)->field.tqe_next = (listelm)->field.tqe_next) != NULL)\ + (elm)->field.tqe_next->field.tqe_prev = \ + &(elm)->field.tqe_next; \ + else \ + (head)->tqh_last = &(elm)->field.tqe_next; \ + (listelm)->field.tqe_next = (elm); \ + (elm)->field.tqe_prev = &(listelm)->field.tqe_next; \ +} while (/* CONSTCOND */0) + +#define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ + QUEUEDEBUG_TAILQ_OP((listelm), field) \ + (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ + (elm)->field.tqe_next = (listelm); \ + *(listelm)->field.tqe_prev = (elm); \ + (listelm)->field.tqe_prev = &(elm)->field.tqe_next; \ +} while (/* CONSTCOND */0) + +#define TAILQ_REMOVE(head, elm, field) do { \ + QUEUEDEBUG_TAILQ_PREREMOVE((head), (elm), field) \ + QUEUEDEBUG_TAILQ_OP((elm), field) \ + if (((elm)->field.tqe_next) != NULL) \ + (elm)->field.tqe_next->field.tqe_prev = \ + (elm)->field.tqe_prev; \ + else \ + (head)->tqh_last = (elm)->field.tqe_prev; \ + *(elm)->field.tqe_prev = (elm)->field.tqe_next; \ + QUEUEDEBUG_TAILQ_POSTREMOVE((elm), field); \ +} while (/* CONSTCOND */0) + +#define TAILQ_FOREACH(var, head, field) \ + for ((var) = ((head)->tqh_first); \ + (var); \ + (var) = ((var)->field.tqe_next)) + +#define TAILQ_FOREACH_SAFE(var, head, field, next) \ + for ((var) = ((head)->tqh_first); \ + (var) != NULL && ((next) = TAILQ_NEXT(var, field), 1); \ + (var) = (next)) + +#define TAILQ_FOREACH_REVERSE(var, head, headname, field) \ + for ((var) = (*(((struct headname *)((head)->tqh_last))->tqh_last)); \ + (var); \ + (var) = (*(((struct headname *)((var)->field.tqe_prev))->tqh_last))) + +#define TAILQ_FOREACH_REVERSE_SAFE(var, head, headname, field, prev) \ + for ((var) = TAILQ_LAST((head), headname); \ + (var) && ((prev) = TAILQ_PREV((var), headname, field), 1);\ + (var) = (prev)) + +#define TAILQ_CONCAT(head1, head2, field) do { \ + if (!TAILQ_EMPTY(head2)) { \ + *(head1)->tqh_last = (head2)->tqh_first; \ + (head2)->tqh_first->field.tqe_prev = (head1)->tqh_last; \ + (head1)->tqh_last = (head2)->tqh_last; \ + TAILQ_INIT((head2)); \ + } \ +} while (/* CONSTCOND */0) + +/* + * Tail queue access methods. + */ +#define TAILQ_EMPTY(head) ((head)->tqh_first == NULL) +#define TAILQ_FIRST(head) ((head)->tqh_first) +#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) + +#define TAILQ_LAST(head, headname) \ + (*(((struct headname *)((head)->tqh_last))->tqh_last)) +#define TAILQ_PREV(elm, headname, field) \ + (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) + +/* + * Circular queue definitions. + */ +#if defined(_KERNEL) && defined(QUEUEDEBUG) +#define QUEUEDEBUG_CIRCLEQ_HEAD(head, field) \ + if ((head)->cqh_first != (void *)(head) && \ + (head)->cqh_first->field.cqe_prev != (void *)(head)) \ + panic("CIRCLEQ head forw %p %s:%d", (head), \ + __FILE__, __LINE__); \ + if ((head)->cqh_last != (void *)(head) && \ + (head)->cqh_last->field.cqe_next != (void *)(head)) \ + panic("CIRCLEQ head back %p %s:%d", (head), \ + __FILE__, __LINE__); +#define QUEUEDEBUG_CIRCLEQ_ELM(head, elm, field) \ + if ((elm)->field.cqe_next == (void *)(head)) { \ + if ((head)->cqh_last != (elm)) \ + panic("CIRCLEQ elm last %p %s:%d", (elm), \ + __FILE__, __LINE__); \ + } else { \ + if ((elm)->field.cqe_next->field.cqe_prev != (elm)) \ + panic("CIRCLEQ elm forw %p %s:%d", (elm), \ + __FILE__, __LINE__); \ + } \ + if ((elm)->field.cqe_prev == (void *)(head)) { \ + if ((head)->cqh_first != (elm)) \ + panic("CIRCLEQ elm first %p %s:%d", (elm), \ + __FILE__, __LINE__); \ + } else { \ + if ((elm)->field.cqe_prev->field.cqe_next != (elm)) \ + panic("CIRCLEQ elm prev %p %s:%d", (elm), \ + __FILE__, __LINE__); \ + } +#define QUEUEDEBUG_CIRCLEQ_POSTREMOVE(elm, field) \ + (elm)->field.cqe_next = (void *)1L; \ + (elm)->field.cqe_prev = (void *)1L; +#else +#define QUEUEDEBUG_CIRCLEQ_HEAD(head, field) +#define QUEUEDEBUG_CIRCLEQ_ELM(head, elm, field) +#define QUEUEDEBUG_CIRCLEQ_POSTREMOVE(elm, field) +#endif + +#define CIRCLEQ_HEAD(name, type) \ +struct name { \ + struct type *cqh_first; /* first element */ \ + struct type *cqh_last; /* last element */ \ +} + +#define CIRCLEQ_HEAD_INITIALIZER(head) \ + { (void *)&head, (void *)&head } + +#define CIRCLEQ_ENTRY(type) \ +struct { \ + struct type *cqe_next; /* next element */ \ + struct type *cqe_prev; /* previous element */ \ +} + +/* + * Circular queue functions. + */ +#define CIRCLEQ_INIT(head) do { \ + (head)->cqh_first = (void *)(head); \ + (head)->cqh_last = (void *)(head); \ +} while (/* CONSTCOND */0) + +#define CIRCLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ + QUEUEDEBUG_CIRCLEQ_HEAD((head), field) \ + QUEUEDEBUG_CIRCLEQ_ELM((head), (listelm), field) \ + (elm)->field.cqe_next = (listelm)->field.cqe_next; \ + (elm)->field.cqe_prev = (listelm); \ + if ((listelm)->field.cqe_next == (void *)(head)) \ + (head)->cqh_last = (elm); \ + else \ + (listelm)->field.cqe_next->field.cqe_prev = (elm); \ + (listelm)->field.cqe_next = (elm); \ +} while (/* CONSTCOND */0) + +#define CIRCLEQ_INSERT_BEFORE(head, listelm, elm, field) do { \ + QUEUEDEBUG_CIRCLEQ_HEAD((head), field) \ + QUEUEDEBUG_CIRCLEQ_ELM((head), (listelm), field) \ + (elm)->field.cqe_next = (listelm); \ + (elm)->field.cqe_prev = (listelm)->field.cqe_prev; \ + if ((listelm)->field.cqe_prev == (void *)(head)) \ + (head)->cqh_first = (elm); \ + else \ + (listelm)->field.cqe_prev->field.cqe_next = (elm); \ + (listelm)->field.cqe_prev = (elm); \ +} while (/* CONSTCOND */0) + +#define CIRCLEQ_INSERT_HEAD(head, elm, field) do { \ + QUEUEDEBUG_CIRCLEQ_HEAD((head), field) \ + (elm)->field.cqe_next = (head)->cqh_first; \ + (elm)->field.cqe_prev = (void *)(head); \ + if ((head)->cqh_last == (void *)(head)) \ + (head)->cqh_last = (elm); \ + else \ + (head)->cqh_first->field.cqe_prev = (elm); \ + (head)->cqh_first = (elm); \ +} while (/* CONSTCOND */0) + +#define CIRCLEQ_INSERT_TAIL(head, elm, field) do { \ + QUEUEDEBUG_CIRCLEQ_HEAD((head), field) \ + (elm)->field.cqe_next = (void *)(head); \ + (elm)->field.cqe_prev = (head)->cqh_last; \ + if ((head)->cqh_first == (void *)(head)) \ + (head)->cqh_first = (elm); \ + else \ + (head)->cqh_last->field.cqe_next = (elm); \ + (head)->cqh_last = (elm); \ +} while (/* CONSTCOND */0) + +#define CIRCLEQ_REMOVE(head, elm, field) do { \ + QUEUEDEBUG_CIRCLEQ_HEAD((head), field) \ + QUEUEDEBUG_CIRCLEQ_ELM((head), (elm), field) \ + if ((elm)->field.cqe_next == (void *)(head)) \ + (head)->cqh_last = (elm)->field.cqe_prev; \ + else \ + (elm)->field.cqe_next->field.cqe_prev = \ + (elm)->field.cqe_prev; \ + if ((elm)->field.cqe_prev == (void *)(head)) \ + (head)->cqh_first = (elm)->field.cqe_next; \ + else \ + (elm)->field.cqe_prev->field.cqe_next = \ + (elm)->field.cqe_next; \ + QUEUEDEBUG_CIRCLEQ_POSTREMOVE((elm), field) \ +} while (/* CONSTCOND */0) + +#define CIRCLEQ_FOREACH(var, head, field) \ + for ((var) = ((head)->cqh_first); \ + (var) != (const void *)(head); \ + (var) = ((var)->field.cqe_next)) + +#define CIRCLEQ_FOREACH_REVERSE(var, head, field) \ + for ((var) = ((head)->cqh_last); \ + (var) != (const void *)(head); \ + (var) = ((var)->field.cqe_prev)) + +/* + * Circular queue access methods. + */ +#define CIRCLEQ_EMPTY(head) ((head)->cqh_first == (void *)(head)) +#define CIRCLEQ_FIRST(head) ((head)->cqh_first) +#define CIRCLEQ_LAST(head) ((head)->cqh_last) +#define CIRCLEQ_NEXT(elm, field) ((elm)->field.cqe_next) +#define CIRCLEQ_PREV(elm, field) ((elm)->field.cqe_prev) + +#define CIRCLEQ_LOOP_NEXT(head, elm, field) \ + (((elm)->field.cqe_next == (void *)(head)) \ + ? ((head)->cqh_first) \ + : (elm->field.cqe_next)) +#define CIRCLEQ_LOOP_PREV(head, elm, field) \ + (((elm)->field.cqe_prev == (void *)(head)) \ + ? ((head)->cqh_last) \ + : (elm->field.cqe_prev)) + +#endif /* !_SYS_QUEUE_H_ */ diff --git a/optee_os/lib/libutils/isoc/include/sys/types.h b/optee_os/lib/libutils/isoc/include/sys/types.h new file mode 100644 index 0000000..b03778e --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/sys/types.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef SYS_TYPES_H +#define SYS_TYPES_H + +#include + +#endif /*SYS_TYPES_H*/ diff --git a/optee_os/lib/libutils/isoc/include/time.h b/optee_os/lib/libutils/isoc/include/time.h new file mode 100644 index 0000000..9284159 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/time.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __TIME_H +#define __TIME_H + +#include +#include + +typedef int64_t time_t; + +#endif /*__TIME_H*/ diff --git a/optee_os/lib/libutils/isoc/include/unistd.h b/optee_os/lib/libutils/isoc/include/unistd.h new file mode 100644 index 0000000..4fe0316 --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/unistd.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __UNISTD_H +#define __UNISTD_H + +#include +#include + +#define __ssize_t_defined +typedef intptr_t ssize_t; + +/* @fd must be 1 or 2. Writes to the secure console. */ +ssize_t write(int fd, const void *buf, size_t count); + +#endif /* __UNISTD_H */ diff --git a/optee_os/lib/libutils/isoc/include/wchar.h b/optee_os/lib/libutils/isoc/include/wchar.h new file mode 100644 index 0000000..d473bdd --- /dev/null +++ b/optee_os/lib/libutils/isoc/include/wchar.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#ifndef __WCHAR_H +#define __WCHAR_H + +#endif /*__WCHAR_H*/ diff --git a/optee_os/lib/libutils/isoc/isalnum.c b/optee_os/lib/libutils/isoc/isalnum.c new file mode 100644 index 0000000..827572d --- /dev/null +++ b/optee_os/lib/libutils/isoc/isalnum.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, KAIST + */ +#include + +int isalnum(int c) +{ + return isalpha(c) || isdigit(c); +} diff --git a/optee_os/lib/libutils/isoc/isalpha.c b/optee_os/lib/libutils/isoc/isalpha.c new file mode 100644 index 0000000..340c084 --- /dev/null +++ b/optee_os/lib/libutils/isoc/isalpha.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ +#include + +int isalpha(int c) +{ + if (c >= 'A' && c <= 'Z') + return 1; + if (c >= 'a' && c <= 'z') + return 1; + return 0; +} diff --git a/optee_os/lib/libutils/isoc/iscntrl.c b/optee_os/lib/libutils/isoc/iscntrl.c new file mode 100644 index 0000000..93d2277 --- /dev/null +++ b/optee_os/lib/libutils/isoc/iscntrl.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, KAIST + */ +#include + +int iscntrl(int c) +{ + if (c < 0x20 || c == 0x7f) + return 1; + return 0; +} diff --git a/optee_os/lib/libutils/isoc/isdigit.c b/optee_os/lib/libutils/isoc/isdigit.c new file mode 100644 index 0000000..9bb6694 --- /dev/null +++ b/optee_os/lib/libutils/isoc/isdigit.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ +#include + +int isdigit(int c) +{ + if (c >= '0' && c <= '9') + return 1; + return 0; +} diff --git a/optee_os/lib/libutils/isoc/isgraph.c b/optee_os/lib/libutils/isoc/isgraph.c new file mode 100644 index 0000000..4e9de9f --- /dev/null +++ b/optee_os/lib/libutils/isoc/isgraph.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, KAIST + */ +#include + +int isgraph(int c) +{ + if (c >= 0x21 && c < 0x7f) + return 1; + return 0; +} diff --git a/optee_os/lib/libutils/isoc/islower.c b/optee_os/lib/libutils/isoc/islower.c new file mode 100644 index 0000000..fabba3d --- /dev/null +++ b/optee_os/lib/libutils/isoc/islower.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, KAIST + */ +#include + +int islower(int c) +{ + return (c >= 'a' && c <= 'z'); +} diff --git a/optee_os/lib/libutils/isoc/isprint.c b/optee_os/lib/libutils/isoc/isprint.c new file mode 100644 index 0000000..2735fd0 --- /dev/null +++ b/optee_os/lib/libutils/isoc/isprint.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, KAIST + */ +#include + +int isprint(int c) +{ + if (c >= 0x20 && c < 0x7f) + return 1; + return 0; +} diff --git a/optee_os/lib/libutils/isoc/ispunct.c b/optee_os/lib/libutils/isoc/ispunct.c new file mode 100644 index 0000000..44ddf34 --- /dev/null +++ b/optee_os/lib/libutils/isoc/ispunct.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, KAIST + */ +#include + +int ispunct(int c) +{ + return isgraph(c) && !isalnum(c); +} diff --git a/optee_os/lib/libutils/isoc/isspace.c b/optee_os/lib/libutils/isoc/isspace.c new file mode 100644 index 0000000..395a832 --- /dev/null +++ b/optee_os/lib/libutils/isoc/isspace.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ +#include + +int isspace(int c) +{ + return (c == ' '); +} diff --git a/optee_os/lib/libutils/isoc/isupper.c b/optee_os/lib/libutils/isoc/isupper.c new file mode 100644 index 0000000..385c5de --- /dev/null +++ b/optee_os/lib/libutils/isoc/isupper.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ +#include + +int isupper(int c) +{ + return (c >= 'A' && c <= 'Z'); +} diff --git a/optee_os/lib/libutils/isoc/isxdigit.c b/optee_os/lib/libutils/isoc/isxdigit.c new file mode 100644 index 0000000..d121d41 --- /dev/null +++ b/optee_os/lib/libutils/isoc/isxdigit.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018, Linaro Limited + */ +#include + +int isxdigit(int c) +{ + if (isdigit(c)) + return 1; + if (c >= 'A' && c <= 'F') + return 1; + if (c >= 'a' && c <= 'f') + return 1; + return 0; +} diff --git a/optee_os/lib/libutils/isoc/newlib/_ansi.h b/optee_os/lib/libutils/isoc/newlib/_ansi.h new file mode 100644 index 0000000..c1c0b0a --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/_ansi.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* Provide support for both ANSI and non-ANSI environments. */ + +/* Some ANSI environments are "broken" in the sense that __STDC__ cannot be + relied upon to have it's intended meaning. Therefore we must use our own + concoction: _HAVE_STDC. Always use _HAVE_STDC instead of __STDC__ in newlib + sources! + + To get a strict ANSI C environment, define macro __STRICT_ANSI__. This will + "comment out" the non-ANSI parts of the ANSI header files (non-ANSI header + files aren't affected). */ + +#ifndef _ANSIDECL_H_ +#define _ANSIDECL_H_ + +/*#include */ +/*#include */ + +/* + * First try to figure out whether we really are in an ANSI C environment. + * This probably needs some work. Perhaps sys/config.h can be + * prevailed upon to give us a clue. + */ + +#ifdef __STDC__ +#define _HAVE_STDC +#endif + +#ifdef _HAVE_STDC +#define _PTR void * +#define _AND , +#define _NOARGS void +#define _CONST const +#define _VOLATILE volatile +#define _SIGNED signed +#define _DOTS , ... +#define _VOID void +#ifdef __CYGWIN__ +#define _EXFUN(name, proto) __cdecl name proto +#define _EXPARM(name, proto) (* __cdecl name) proto +#else +#define _EXFUN(name, proto) name proto +#define _EXPARM(name, proto) (* name) proto +#endif +#define _DEFUN(name, arglist, args) name(args) +#define _DEFUN_VOID(name) name(_NOARGS) +#define _CAST_VOID (void) +#ifndef _LONG_DOUBLE +#define _LONG_DOUBLE long double +#endif +#ifndef _PARAMS +#define _PARAMS(paramlist) paramlist +#endif +#else +#define _PTR char * +#define _AND ; +#define _NOARGS +#define _CONST +#define _VOLATILE +#define _SIGNED +#define _DOTS +#define _VOID void +#define _EXFUN(name, proto) name() +#define _DEFUN(name, arglist, args) name arglist args; +#define _DEFUN_VOID(name) name() +#define _CAST_VOID +#define _LONG_DOUBLE double +#ifndef _PARAMS +#define _PARAMS(paramlist) () +#endif +#endif + +/* Support gcc's __attribute__ facility. */ + +#ifdef __GNUC__ +#define _ATTRIBUTE(attrs) __attribute__ (attrs) +#else +#define _ATTRIBUTE(attrs) +#endif + +/* ISO C++. */ + +#ifdef __cplusplus +#if !(defined(_BEGIN_STD_C) && defined(_END_STD_C)) +#ifdef _HAVE_STD_CXX +#define _BEGIN_STD_C namespace std { extern "C" { +#define _END_STD_C } } +#else +#define _BEGIN_STD_C extern "C" { +#define _END_STD_C } +#endif +#endif +#else +#define _BEGIN_STD_C +#define _END_STD_C +#endif + +#endif /* _ANSIDECL_H_ */ diff --git a/optee_os/lib/libutils/isoc/newlib/abs.c b/optee_os/lib/libutils/isoc/newlib/abs.c new file mode 100644 index 0000000..03179b3 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/abs.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION +<>---integer absolute value (magnitude) + +INDEX + abs + +ANSI_SYNOPSIS + #include + int abs(int <[i]>); + +TRAD_SYNOPSIS + #include + int abs(<[i]>) + int <[i]>; + +DESCRIPTION +<> returns +@tex +$|x|$, +@end tex +the absolute value of <[i]> (also called the magnitude +of <[i]>). That is, if <[i]> is negative, the result is the opposite +of <[i]>, but if <[i]> is nonnegative the result is <[i]>. + +The similar function <> uses and returns <> rather than <> values. + +RETURNS +The result is a nonnegative integer. + +PORTABILITY +<> is ANSI. + +No supporting OS subroutines are required. +*/ + +#include "_ansi.h" +#include + +int +_DEFUN (abs, (i), int i) +{ + return (i < 0) ? -i : i; +} diff --git a/optee_os/lib/libutils/isoc/newlib/bcmp.c b/optee_os/lib/libutils/isoc/newlib/bcmp.c new file mode 100644 index 0000000..893c3c7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/bcmp.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* +FUNCTION + <>---compare two memory areas + +INDEX + bcmp + +SYNOPSIS + #include + int bcmp(const void *<[s1]>, const void *<[s2]>, size_t <[n]>); + +DESCRIPTION + This function compares not more than <[n]> bytes of the + object pointed to by <[s1]> with the object pointed to by <[s2]>. + + This function is identical to <>. + +RETURNS + The function returns an integer greater than, equal to or + less than zero according to whether the object pointed to by + <[s1]> is greater than, equal to or less than the object + pointed to by <[s2]>. + +PORTABILITY +<> requires no supporting OS subroutines. + +QUICKREF + bcmp ansi pure +*/ + +#include +#include + +int +bcmp (const void *m1, + const void *m2, + size_t n) + +{ + return memcmp (m1, m2, n); +} diff --git a/optee_os/lib/libutils/isoc/newlib/memchr.c b/optee_os/lib/libutils/isoc/newlib/memchr.c new file mode 100644 index 0000000..81dd055 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/memchr.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---find character in memory + +INDEX + memchr + +ANSI_SYNOPSIS + #include + void *memchr(const void *<[src]>, int <[c]>, size_t <[length]>); + +TRAD_SYNOPSIS + #include + void *memchr(<[src]>, <[c]>, <[length]>) + void *<[src]>; + void *<[c]>; + size_t <[length]>; + +DESCRIPTION + This function searches memory starting at <<*<[src]>>> for the + character <[c]>. The search only ends with the first + occurrence of <[c]>, or after <[length]> characters; in + particular, <> does not terminate the search. + +RETURNS + If the character <[c]> is found within <[length]> characters + of <<*<[src]>>>, a pointer to the character is returned. If + <[c]> is not found, then <> is returned. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + memchr ansi pure +*/ + +#include "_ansi.h" +#include +#include + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X) ((long)X & (sizeof(long) - 1)) + +/* How many bytes are loaded each iteration of the word copy loop. */ +#define LBLOCKSIZE (sizeof(long)) + +/* Threshhold for punting to the bytewise iterator. */ +#define TOO_SMALL(LEN) ((LEN) < LBLOCKSIZE) + +#if LONG_MAX == 2147483647L +#define DETECTNULL(X) (((X) - 0x01010101L) & ~(X) & 0x80808080UL) +#else +#if LONG_MAX == 9223372036854775807L +/* Nonzero if X (a long int) contains a NULL byte. */ +#define DETECTNULL(X) (((X) - 0x0101010101010101L) & ~(X) & \ + 0x8080808080808080UL) +#else +#error long int is not a 32bit or 64bit type. +#endif +#endif + +#ifndef DETECTNULL +#error long int is not a 32bit or 64bit byte +#endif + +/* DETECTCHAR returns nonzero if (long)X contains the byte used + to fill (long)MASK. */ +#define DETECTCHAR(X, MASK) (DETECTNULL(X ^ MASK)) + +_PTR +_DEFUN(memchr, (src_void, c, length), _CONST _PTR src_void _AND int c + _AND size_t length) +{ + _CONST unsigned char *src = (_CONST unsigned char *)src_void; + unsigned char d = c; + +#if !defined(PREFER_SIZE_OVER_SPEED) && !defined(__OPTIMIZE_SIZE__) + unsigned long *asrc; + unsigned long mask; + int i; + + while (UNALIGNED(src)) { + if (!length--) + return NULL; + if (*src == d) + return (void *)src; + src++; + } + + if (!TOO_SMALL(length)) { + /* If we get this far, we know that length is large and src is + word-aligned. */ + /* The fast code reads the source one word at a time and only + performs the bytewise search on word-sized segments if they + contain the search character, which is detected by XORing + the word-sized segment with a word-sized block of the search + character and then detecting for the presence of NUL in the + result. */ + asrc = (unsigned long *)src; + mask = d << 8 | d; + mask = mask << 16 | mask; + for (i = 32; i < LBLOCKSIZE * 8; i <<= 1) + mask = (mask << i) | mask; + + while (length >= LBLOCKSIZE) { + if (DETECTCHAR(*asrc, mask)) + break; + length -= LBLOCKSIZE; + asrc++; + } + + /* If there are fewer than LBLOCKSIZE characters left, + then we resort to the bytewise loop. */ + + src = (unsigned char *)asrc; + } +#endif /* not PREFER_SIZE_OVER_SPEED */ + + while (length--) { + if (*src == d) + return (void *)src; + src++; + } + + return NULL; +} diff --git a/optee_os/lib/libutils/isoc/newlib/memcmp.c b/optee_os/lib/libutils/isoc/newlib/memcmp.c new file mode 100644 index 0000000..d4932ae --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/memcmp.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---compare two memory areas + +INDEX + memcmp + +ANSI_SYNOPSIS + #include + int memcmp(const void *<[s1]>, const void *<[s2]>, size_t <[n]>); + +TRAD_SYNOPSIS + #include + int memcmp(<[s1]>, <[s2]>, <[n]>) + void *<[s1]>; + void *<[s2]>; + size_t <[n]>; + +DESCRIPTION + This function compares not more than <[n]> characters of the + object pointed to by <[s1]> with the object pointed to by <[s2]>. + +RETURNS + The function returns an integer greater than, equal to or + less than zero according to whether the object pointed to by + <[s1]> is greater than, equal to or less than the object + pointed to by <[s2]>. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + memcmp ansi pure +*/ + +#include "_ansi.h" +#include + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X, Y) \ + (((long)X & (sizeof(long) - 1)) | ((long)Y & (sizeof(long) - 1))) + +/* How many bytes are copied each iteration of the word copy loop. */ +#define LBLOCKSIZE (sizeof(long)) + +/* Threshhold for punting to the byte copier. */ +#define TOO_SMALL(LEN) ((LEN) < LBLOCKSIZE) + +int +_DEFUN(memcmp, (m1, m2, n), _CONST _PTR m1 _AND _CONST _PTR m2 _AND size_t n) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + unsigned char *s1 = (unsigned char *)m1; + unsigned char *s2 = (unsigned char *)m2; + + while (n--) { + if (*s1 != *s2) + return *s1 - *s2; + s1++; + s2++; + } + return 0; +#else + unsigned char *s1 = (unsigned char *)m1; + unsigned char *s2 = (unsigned char *)m2; + unsigned long *a1; + unsigned long *a2; + + /* If the size is too small, or either pointer is unaligned, + then we punt to the byte compare loop. Hopefully this will + not turn up in inner loops. */ + if (!TOO_SMALL(n) && !UNALIGNED(s1, s2)) { + /* + * Otherwise, load and compare the blocks of memory one word at + * a time. + */ + a1 = (unsigned long *)s1; + a2 = (unsigned long *)s2; + while (n >= LBLOCKSIZE) { + if (*a1 != *a2) + break; + a1++; + a2++; + n -= LBLOCKSIZE; + } + + /* check m mod LBLOCKSIZE remaining characters */ + + s1 = (unsigned char *)a1; + s2 = (unsigned char *)a2; + } + + while (n--) { + if (*s1 != *s2) + return *s1 - *s2; + s1++; + s2++; + } + + return 0; +#endif /* not PREFER_SIZE_OVER_SPEED */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/memcpy.c b/optee_os/lib/libutils/isoc/newlib/memcpy.c new file mode 100644 index 0000000..7c4953a --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/memcpy.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---copy memory regions + +ANSI_SYNOPSIS + #include + void* memcpy(void *<[out]>, const void *<[in]>, size_t <[n]>); + +TRAD_SYNOPSIS + #include + void *memcpy(<[out]>, <[in]>, <[n]> + void *<[out]>; + void *<[in]>; + size_t <[n]>; + +DESCRIPTION + This function copies <[n]> bytes from the memory region + pointed to by <[in]> to the memory region pointed to by + <[out]>. + + If the regions overlap, the behavior is undefined. + +RETURNS + <> returns a pointer to the first byte of the <[out]> + region. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + memcpy ansi pure +*/ + +#include "_ansi.h" +#include + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X, Y) \ + (((long)X & (sizeof(long) - 1)) | ((long)Y & (sizeof(long) - 1))) + +/* How many bytes are copied each iteration of the 4X unrolled loop. */ +#define BIGBLOCKSIZE (sizeof(long) << 2) + +/* How many bytes are copied each iteration of the word copy loop. */ +#define LITTLEBLOCKSIZE (sizeof(long)) + +/* Threshhold for punting to the byte copier. */ +#define TOO_SMALL(LEN) ((LEN) < BIGBLOCKSIZE) + +_PTR +_DEFUN(memcpy, (dst0, src0, len0), _PTR dst0 _AND _CONST _PTR src0 _AND + size_t len0) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + char *dst = (char *)dst0; + char *src = (char *)src0; + + _PTR save = dst0; + + while (len0--) + *dst++ = *src++; + + return save; +#else + char *dst = dst0; + _CONST char *src = src0; + long *aligned_dst; + _CONST long *aligned_src; + + /* If the size is small, or either SRC or DST is unaligned, + then punt into the byte copy loop. This should be rare. */ + if (!TOO_SMALL(len0) && !UNALIGNED(src, dst)) { + aligned_dst = (long *)dst; + aligned_src = (long *)src; + + /* Copy 4X long words at a time if possible. */ + while (len0 >= BIGBLOCKSIZE) { + *aligned_dst++ = *aligned_src++; + *aligned_dst++ = *aligned_src++; + *aligned_dst++ = *aligned_src++; + *aligned_dst++ = *aligned_src++; + len0 -= BIGBLOCKSIZE; + } + + /* Copy one long word at a time if possible. */ + while (len0 >= LITTLEBLOCKSIZE) { + *aligned_dst++ = *aligned_src++; + len0 -= LITTLEBLOCKSIZE; + } + + /* Pick up any residual with a byte copier. */ + dst = (char *)aligned_dst; + src = (char *)aligned_src; + } + + while (len0--) + *dst++ = *src++; + + return dst0; +#endif /* not PREFER_SIZE_OVER_SPEED */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/memmove.c b/optee_os/lib/libutils/isoc/newlib/memmove.c new file mode 100644 index 0000000..c913e66 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/memmove.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---move possibly overlapping memory + +INDEX + memmove + +ANSI_SYNOPSIS + #include + void *memmove(void *<[dst]>, const void *<[src]>, size_t <[length]>); + +TRAD_SYNOPSIS + #include + void *memmove(<[dst]>, <[src]>, <[length]>) + void *<[dst]>; + void *<[src]>; + size_t <[length]>; + +DESCRIPTION + This function moves <[length]> characters from the block of + memory starting at <<*<[src]>>> to the memory starting at + <<*<[dst]>>>. <> reproduces the characters correctly + at <<*<[dst]>>> even if the two areas overlap. + +RETURNS + The function returns <[dst]> as passed. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + memmove ansi pure +*/ + +#include +#include "_ansi.h" +#include +#include + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X, Y) \ + (((long)X & (sizeof(long) - 1)) | ((long)Y & (sizeof(long) - 1))) + +/* How many bytes are copied each iteration of the 4X unrolled loop. */ +#define BIGBLOCKSIZE (sizeof(long) << 2) + +/* How many bytes are copied each iteration of the word copy loop. */ +#define LITTLEBLOCKSIZE (sizeof(long)) + +/* Threshhold for punting to the byte copier. */ +#define TOO_SMALL(LEN) ((LEN) < BIGBLOCKSIZE) + +/* SUPPRESS 20 */ +_PTR +_DEFUN(memmove, (dst_void, src_void, length), _PTR dst_void _AND _CONST _PTR + src_void _AND size_t length) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + char *dst = dst_void; + _CONST char *src = src_void; + + if (src < dst && dst < src + length) { + /* Have to copy backwards */ + src += length; + dst += length; + while (length--) + *--dst = *--src; + } else { + while (length--) + *dst++ = *src++; + } + + return dst_void; +#else + char *dst = dst_void; + _CONST char *src = src_void; + long *aligned_dst; + _CONST long *aligned_src; + + if (src < dst && dst < src + length) { + /* Destructive overlap...have to copy backwards */ + src += length; + dst += length; + while (length--) + *--dst = *--src; + } else { + /* + * Use optimizing algorithm for a non-destructive copy to + * closely match memcpy. If the size is small or either SRC or + * DST is unaligned, then punt into the byte copy loop. This + * should be rare. + */ + if (!TOO_SMALL(length) && !UNALIGNED(src, dst)) { + aligned_dst = (long *)dst; + aligned_src = (long *)src; + + /* Copy 4X long words at a time if possible. */ + while (length >= BIGBLOCKSIZE) { + *aligned_dst++ = *aligned_src++; + *aligned_dst++ = *aligned_src++; + *aligned_dst++ = *aligned_src++; + *aligned_dst++ = *aligned_src++; + length -= BIGBLOCKSIZE; + } + + /* Copy one long word at a time if possible. */ + while (length >= LITTLEBLOCKSIZE) { + *aligned_dst++ = *aligned_src++; + length -= LITTLEBLOCKSIZE; + } + + /* Pick up any residual with a byte copier. */ + dst = (char *)aligned_dst; + src = (char *)aligned_src; + } + + while (length--) + *dst++ = *src++; + } + + return dst_void; +#endif /* not PREFER_SIZE_OVER_SPEED */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/memset.c b/optee_os/lib/libutils/isoc/newlib/memset.c new file mode 100644 index 0000000..ade5142 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/memset.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---set an area of memory + +INDEX + memset + +ANSI_SYNOPSIS + #include + void *memset(void *<[dst]>, int <[c]>, size_t <[length]>); + +TRAD_SYNOPSIS + #include + void *memset(<[dst]>, <[c]>, <[length]>) + void *<[dst]>; + int <[c]>; + size_t <[length]>; + +DESCRIPTION + This function converts the argument <[c]> into an unsigned + char and fills the first <[length]> characters of the array + pointed to by <[dst]> to the value. + +RETURNS + <> returns the value of <[dst]>. + +PORTABILITY +<> is ANSI C. + + <> requires no supporting OS subroutines. + +QUICKREF + memset ansi pure +*/ + +#include "_ansi.h" +#include + +#define LBLOCKSIZE (sizeof(long)) +#define UNALIGNED(X) ((long)X & (LBLOCKSIZE - 1)) +#define TOO_SMALL(LEN) ((LEN) < LBLOCKSIZE) + +_PTR _DEFUN(memset, (m, c, n), _PTR m _AND int c _AND size_t n) +{ + char *s = (char *)m; + +#if !defined(PREFER_SIZE_OVER_SPEED) && !defined(__OPTIMIZE_SIZE__) + int i; + unsigned long buffer; + unsigned long *aligned_addr; + unsigned int d = c & 0xff; /* To avoid sign extension, copy C to an + unsigned variable. */ + + while (UNALIGNED(s)) { + if (n--) + *s++ = (char)c; + else + return m; + } + + if (!TOO_SMALL(n)) { + /* + * If we get this far, we know that n is large and s is + * word-aligned. + */ + aligned_addr = (unsigned long *)s; + + /* Store D into each char sized location in BUFFER so that + we can set large blocks quickly. */ + buffer = (d << 8) | d; + buffer |= (buffer << 16); + for (i = 32; i < LBLOCKSIZE * 8; i <<= 1) + buffer = (buffer << i) | buffer; + + /* Unroll the loop. */ + while (n >= LBLOCKSIZE * 4) { + *aligned_addr++ = buffer; + *aligned_addr++ = buffer; + *aligned_addr++ = buffer; + *aligned_addr++ = buffer; + n -= 4 * LBLOCKSIZE; + } + + while (n >= LBLOCKSIZE) { + *aligned_addr++ = buffer; + n -= LBLOCKSIZE; + } + /* Pick up the remainder with a bytewise loop. */ + s = (char *)aligned_addr; + } +#endif /* not PREFER_SIZE_OVER_SPEED */ + + while (n--) + *s++ = (char)c; + + return m; +} diff --git a/optee_os/lib/libutils/isoc/newlib/str-two-way.h b/optee_os/lib/libutils/isoc/newlib/str-two-way.h new file mode 100644 index 0000000..e0c7757 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/str-two-way.h @@ -0,0 +1,447 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* Byte-wise substring search, using the Two-Way algorithm. + * Copyright (C) 2008, 2010 Eric Blake + * Permission to use, copy, modify, and distribute this software + * is freely granted, provided that this notice is preserved. + */ + + +/* Before including this file, you need to include , and define: + RESULT_TYPE A macro that expands to the return type. + AVAILABLE(h, h_l, j, n_l) A macro that returns nonzero if there are + at least N_L bytes left starting at + H[J]. H is 'unsigned char *', H_L, J, + and N_L are 'size_t'; H_L is an + lvalue. For NUL-terminated searches, + H_L can be modified each iteration to + avoid having to compute the end of H + up front. + + For case-insensitivity, you may optionally define: + CMP_FUNC(p1, p2, l) A macro that returns 0 iff the first L + characters of P1 and P2 are equal. + CANON_ELEMENT(c) A macro that canonicalizes an element + right after it has been fetched from + one of the two strings. The argument + is an 'unsigned char'; the result must + be an 'unsigned char' as well. + + This file undefines the macros documented above, and defines + LONG_NEEDLE_THRESHOLD. +*/ + +#include +#include + +/* We use the Two-Way string matching algorithm, which guarantees + linear complexity with constant space. Additionally, for long + needles, we also use a bad character shift table similar to the + Boyer-Moore algorithm to achieve improved (potentially sub-linear) + performance. + + See http://www-igm.univ-mlv.fr/~lecroq/string/node26.html#SECTION00260 + and http://en.wikipedia.org/wiki/Boyer-Moore_string_search_algorithm +*/ + +/* Point at which computing a bad-byte shift table is likely to be + worthwhile. Small needles should not compute a table, since it + adds (1 << CHAR_BIT) + NEEDLE_LEN computations of preparation for a + speedup no greater than a factor of NEEDLE_LEN. The larger the + needle, the better the potential performance gain. On the other + hand, on non-POSIX systems with CHAR_BIT larger than eight, the + memory required for the table is prohibitive. */ +#if CHAR_BIT < 10 +# define LONG_NEEDLE_THRESHOLD 32U +#else +# define LONG_NEEDLE_THRESHOLD SIZE_MAX +#endif + +#define MAX(a, b) ((a < b) ? (b) : (a)) + +#ifndef CANON_ELEMENT +# define CANON_ELEMENT(c) c +#endif +#ifndef CMP_FUNC +# define CMP_FUNC memcmp +#endif + +/* Perform a critical factorization of NEEDLE, of length NEEDLE_LEN. + Return the index of the first byte in the right half, and set + *PERIOD to the global period of the right half. + + The global period of a string is the smallest index (possibly its + length) at which all remaining bytes in the string are repetitions + of the prefix (the last repetition may be a subset of the prefix). + + When NEEDLE is factored into two halves, a local period is the + length of the smallest word that shares a suffix with the left half + and shares a prefix with the right half. All factorizations of a + non-empty NEEDLE have a local period of at least 1 and no greater + than NEEDLE_LEN. + + A critical factorization has the property that the local period + equals the global period. All strings have at least one critical + factorization with the left half smaller than the global period. + + Given an ordered alphabet, a critical factorization can be computed + in linear time, with 2 * NEEDLE_LEN comparisons, by computing the + larger of two ordered maximal suffixes. The ordered maximal + suffixes are determined by lexicographic comparison of + periodicity. */ +static size_t +critical_factorization (const unsigned char *needle, size_t needle_len, + size_t *period) +{ + /* Index of last byte of left half, or SIZE_MAX. */ + size_t max_suffix, max_suffix_rev; + size_t j; /* Index into NEEDLE for current candidate suffix. */ + size_t k; /* Offset into current period. */ + size_t p; /* Intermediate period. */ + unsigned char a, b; /* Current comparison bytes. */ + + /* Invariants: + 0 <= j < NEEDLE_LEN - 1 + -1 <= max_suffix{,_rev} < j (treating SIZE_MAX as if it were signed) + min(max_suffix, max_suffix_rev) < global period of NEEDLE + 1 <= p <= global period of NEEDLE + p == global period of the substring NEEDLE[max_suffix{,_rev}+1...j] + 1 <= k <= p + */ + + /* Perform lexicographic search. */ + max_suffix = SIZE_MAX; + j = 0; + k = p = 1; + while (j + k < needle_len) + { + a = CANON_ELEMENT (needle[j + k]); + b = CANON_ELEMENT (needle[(size_t)(max_suffix + k)]); + if (a < b) + { + /* Suffix is smaller, period is entire prefix so far. */ + j += k; + k = 1; + p = j - max_suffix; + } + else if (a == b) + { + /* Advance through repetition of the current period. */ + if (k != p) + ++k; + else + { + j += p; + k = 1; + } + } + else /* b < a */ + { + /* Suffix is larger, start over from current location. */ + max_suffix = j++; + k = p = 1; + } + } + *period = p; + + /* Perform reverse lexicographic search. */ + max_suffix_rev = SIZE_MAX; + j = 0; + k = p = 1; + while (j + k < needle_len) + { + a = CANON_ELEMENT (needle[j + k]); + b = CANON_ELEMENT (needle[max_suffix_rev + k]); + if (b < a) + { + /* Suffix is smaller, period is entire prefix so far. */ + j += k; + k = 1; + p = j - max_suffix_rev; + } + else if (a == b) + { + /* Advance through repetition of the current period. */ + if (k != p) + ++k; + else + { + j += p; + k = 1; + } + } + else /* a < b */ + { + /* Suffix is larger, start over from current location. */ + max_suffix_rev = j++; + k = p = 1; + } + } + + /* Choose the longer suffix. Return the first byte of the right + half, rather than the last byte of the left half. */ + if (max_suffix_rev + 1 < max_suffix + 1) + return max_suffix + 1; + *period = p; + return max_suffix_rev + 1; +} + +/* Return the first location of non-empty NEEDLE within HAYSTACK, or + NULL. HAYSTACK_LEN is the minimum known length of HAYSTACK. This + method is optimized for NEEDLE_LEN < LONG_NEEDLE_THRESHOLD. + Performance is guaranteed to be linear, with an initialization cost + of 2 * NEEDLE_LEN comparisons. + + If AVAILABLE does not modify HAYSTACK_LEN (as in memmem), then at + most 2 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. + If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 * + HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. */ +static RETURN_TYPE +two_way_short_needle (const unsigned char *haystack, size_t haystack_len, + const unsigned char *needle, size_t needle_len) +{ + size_t i; /* Index into current byte of NEEDLE. */ + size_t j; /* Index into current window of HAYSTACK. */ + size_t period; /* The period of the right half of needle. */ + size_t suffix; /* The index of the right half of needle. */ + + /* Factor the needle into two halves, such that the left half is + smaller than the global period, and the right half is + periodic (with a period as large as NEEDLE_LEN - suffix). */ + suffix = critical_factorization (needle, needle_len, &period); + + /* Perform the search. Each iteration compares the right half + first. */ + if (CMP_FUNC (needle, needle + period, suffix) == 0) + { + /* Entire needle is periodic; a mismatch can only advance by the + period, so use memory to avoid rescanning known occurrences + of the period. */ + size_t memory = 0; + j = 0; + while (AVAILABLE (haystack, haystack_len, j, needle_len)) + { + /* Scan for matches in right half. */ + i = MAX (suffix, memory); + while (i < needle_len && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + ++i; + if (needle_len <= i) + { + /* Scan for matches in left half. */ + i = suffix - 1; + while (memory < i + 1 && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + --i; + if (i + 1 < memory + 1) + return (RETURN_TYPE) (haystack + j); + /* No match, so remember how many repetitions of period + on the right half were scanned. */ + j += period; + memory = needle_len - period; + } + else + { + j += i - suffix + 1; + memory = 0; + } + } + } + else + { + /* The two halves of needle are distinct; no extra memory is + required, and any mismatch results in a maximal shift. */ + period = MAX (suffix, needle_len - suffix) + 1; + j = 0; + while (AVAILABLE (haystack, haystack_len, j, needle_len)) + { + /* Scan for matches in right half. */ + i = suffix; + while (i < needle_len && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + ++i; + if (needle_len <= i) + { + /* Scan for matches in left half. */ + i = suffix - 1; + while (i != SIZE_MAX && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + --i; + if (i == SIZE_MAX) + return (RETURN_TYPE) (haystack + j); + j += period; + } + else + j += i - suffix + 1; + } + } + return NULL; +} + +/* Return the first location of non-empty NEEDLE within HAYSTACK, or + NULL. HAYSTACK_LEN is the minimum known length of HAYSTACK. This + method is optimized for LONG_NEEDLE_THRESHOLD <= NEEDLE_LEN. + Performance is guaranteed to be linear, with an initialization cost + of 3 * NEEDLE_LEN + (1 << CHAR_BIT) operations. + + If AVAILABLE does not modify HAYSTACK_LEN (as in memmem), then at + most 2 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching, + and sublinear performance O(HAYSTACK_LEN / NEEDLE_LEN) is possible. + If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 * + HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching, and + sublinear performance is not possible. */ +static RETURN_TYPE +two_way_long_needle (const unsigned char *haystack, size_t haystack_len, + const unsigned char *needle, size_t needle_len) +{ + size_t i; /* Index into current byte of NEEDLE. */ + size_t j; /* Index into current window of HAYSTACK. */ + size_t period; /* The period of the right half of needle. */ + size_t suffix; /* The index of the right half of needle. */ + size_t shift_table[1U << CHAR_BIT]; /* See below. */ + + /* Factor the needle into two halves, such that the left half is + smaller than the global period, and the right half is + periodic (with a period as large as NEEDLE_LEN - suffix). */ + suffix = critical_factorization (needle, needle_len, &period); + + /* Populate shift_table. For each possible byte value c, + shift_table[c] is the distance from the last occurrence of c to + the end of NEEDLE, or NEEDLE_LEN if c is absent from the NEEDLE. + shift_table[NEEDLE[NEEDLE_LEN - 1]] contains the only 0. */ + for (i = 0; i < 1U << CHAR_BIT; i++) + shift_table[i] = needle_len; + for (i = 0; i < needle_len; i++) + shift_table[CANON_ELEMENT (needle[i])] = needle_len - i - 1; + + /* Perform the search. Each iteration compares the right half + first. */ + if (CMP_FUNC (needle, needle + period, suffix) == 0) + { + /* Entire needle is periodic; a mismatch can only advance by the + period, so use memory to avoid rescanning known occurrences + of the period. */ + size_t memory = 0; + size_t shift; + j = 0; + while (AVAILABLE (haystack, haystack_len, j, needle_len)) + { + /* Check the last byte first; if it does not match, then + shift to the next possible match location. */ + shift = shift_table[CANON_ELEMENT (haystack[j + needle_len - 1])]; + if (0 < shift) + { + if (memory && shift < period) + { + /* Since needle is periodic, but the last period has + a byte out of place, there can be no match until + after the mismatch. */ + shift = needle_len - period; + } + memory = 0; + j += shift; + continue; + } + /* Scan for matches in right half. The last byte has + already been matched, by virtue of the shift table. */ + i = MAX (suffix, memory); + while (i < needle_len - 1 && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + ++i; + if (needle_len - 1 <= i) + { + /* Scan for matches in left half. */ + i = suffix - 1; + while (memory < i + 1 && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + --i; + if (i + 1 < memory + 1) + return (RETURN_TYPE) (haystack + j); + /* No match, so remember how many repetitions of period + on the right half were scanned. */ + j += period; + memory = needle_len - period; + } + else + { + j += i - suffix + 1; + memory = 0; + } + } + } + else + { + /* The two halves of needle are distinct; no extra memory is + required, and any mismatch results in a maximal shift. */ + size_t shift; + period = MAX (suffix, needle_len - suffix) + 1; + j = 0; + while (AVAILABLE (haystack, haystack_len, j, needle_len)) + { + /* Check the last byte first; if it does not match, then + shift to the next possible match location. */ + shift = shift_table[CANON_ELEMENT (haystack[j + needle_len - 1])]; + if (0 < shift) + { + j += shift; + continue; + } + /* Scan for matches in right half. The last byte has + already been matched, by virtue of the shift table. */ + i = suffix; + while (i < needle_len - 1 && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + ++i; + if (needle_len - 1 <= i) + { + /* Scan for matches in left half. */ + i = suffix - 1; + while (i != SIZE_MAX && (CANON_ELEMENT (needle[i]) + == CANON_ELEMENT (haystack[i + j]))) + --i; + if (i == SIZE_MAX) + return (RETURN_TYPE) (haystack + j); + j += period; + } + else + j += i - suffix + 1; + } + } + return NULL; +} + +#undef AVAILABLE +#undef CANON_ELEMENT +#undef CMP_FUNC +#undef MAX +#undef RETURN_TYPE diff --git a/optee_os/lib/libutils/isoc/newlib/strchr.c b/optee_os/lib/libutils/isoc/newlib/strchr.c new file mode 100644 index 0000000..1a1af15 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strchr.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---search for character in string + +INDEX + strchr + +ANSI_SYNOPSIS + #include + char * strchr(const char *<[string]>, int <[c]>); + +TRAD_SYNOPSIS + #include + char * strchr(<[string]>, <[c]>); + const char *<[string]>; + int <[c]>; + +DESCRIPTION + This function finds the first occurence of <[c]> (converted to + a char) in the string pointed to by <[string]> (including the + terminating null character). + +RETURNS + Returns a pointer to the located character, or a null pointer + if <[c]> does not occur in <[string]>. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strchr ansi pure +*/ + +#include +#include +#include "_ansi.h" + +/* Nonzero if X is not aligned on a "long" boundary. */ +#define UNALIGNED(X) ((long)X & (sizeof (long) - 1)) + +/* How many bytes are loaded each iteration of the word copy loop. */ +#define LBLOCKSIZE (sizeof (long)) + +#if LONG_MAX == 2147483647L +#define DETECTNULL(X) (((X) - 0x01010101L) & ~(X) & 0x80808080UL) +#else +#if LONG_MAX == 9223372036854775807L +/* Nonzero if X (a long int) contains a NULL byte. */ +#define DETECTNULL(X) (((X) - 0x0101010101010101L) & ~(X) & \ + 0x8080808080808080UL) +#else +#error long int is not a 32bit or 64bit type. +#endif +#endif + +/* DETECTCHAR returns nonzero if (long)X contains the byte used + to fill (long)MASK. */ +#define DETECTCHAR(X,MASK) (DETECTNULL(X ^ MASK)) + +char * +_DEFUN (strchr, (s1, i), + _CONST char *s1 _AND + int i) +{ + _CONST unsigned char *s = (_CONST unsigned char *)s1; + unsigned char c = i; + +#if !defined(PREFER_SIZE_OVER_SPEED) && !defined(__OPTIMIZE_SIZE__) + unsigned long mask,j; + unsigned long *aligned_addr; + + /* Special case for finding 0. */ + if (!c) + { + while (UNALIGNED (s)) + { + if (!*s) + return (char *) s; + s++; + } + /* Operate a word at a time. */ + aligned_addr = (unsigned long *) s; + while (!DETECTNULL (*aligned_addr)) + aligned_addr++; + /* Found the end of string. */ + s = (const unsigned char *) aligned_addr; + while (*s) + s++; + return (char *) s; + } + + /* All other bytes. Align the pointer, then search a long at a time. */ + while (UNALIGNED (s)) + { + if (!*s) + return NULL; + if (*s == c) + return (char *) s; + s++; + } + + mask = c; + for (j = 8; j < LBLOCKSIZE * 8; j <<= 1) + mask = (mask << j) | mask; + + aligned_addr = (unsigned long *) s; + while (!DETECTNULL (*aligned_addr) && !DETECTCHAR (*aligned_addr, mask)) + aligned_addr++; + + /* The block of bytes currently pointed to by aligned_addr + contains either a null or the target char, or both. We + catch it using the bytewise search. */ + + s = (unsigned char *) aligned_addr; + +#endif /* not PREFER_SIZE_OVER_SPEED */ + + while (*s && *s != c) + s++; + if (*s == c) + return (char *)s; + return NULL; +} diff --git a/optee_os/lib/libutils/isoc/newlib/strcmp.c b/optee_os/lib/libutils/isoc/newlib/strcmp.c new file mode 100644 index 0000000..2f36e27 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strcmp.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* This file is copied from newlib-1.19 */ + +/* +FUNCTION + <>---character string compare + +INDEX + strcmp + +ANSI_SYNOPSIS + #include + int strcmp(const char *<[a]>, const char *<[b]>); + +TRAD_SYNOPSIS + #include + int strcmp(<[a]>, <[b]>) + char *<[a]>; + char *<[b]>; + +DESCRIPTION + <> compares the string at <[a]> to + the string at <[b]>. + +RETURNS + If <<*<[a]>>> sorts lexicographically after <<*<[b]>>>, + <> returns a number greater than zero. If the two + strings match, <> returns zero. If <<*<[a]>>> + sorts lexicographically before <<*<[b]>>>, <> returns a + number less than zero. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strcmp ansi pure +*/ + +#include "_ansi.h" +#include +#include + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X, Y) \ + (((long)X & (sizeof(long) - 1)) | ((long)Y & (sizeof(long) - 1))) + +/* DETECTNULL returns nonzero if (long)X contains a NULL byte. */ +#if LONG_MAX == 2147483647L +#define DETECTNULL(X) (((X) - 0x01010101L) & ~(X) & 0x80808080UL) +#else +#if LONG_MAX == 9223372036854775807L +#define DETECTNULL(X) (((X) - 0x0101010101010101L) & ~(X) & \ + 0x8080808080808080UL) +#else +#error long int is not a 32bit or 64bit type. +#endif +#endif + +#ifndef DETECTNULL +#error long int is not a 32bit or 64bit byte +#endif + +int _DEFUN(strcmp, (s1, s2), _CONST char *s1 _AND _CONST char *s2) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + while (*s1 != '\0' && *s1 == *s2) { + s1++; + s2++; + } + + return (*(unsigned char *)s1) - (*(unsigned char *)s2); +#else + unsigned long *a1; + unsigned long *a2; + + /* If s1 or s2 are unaligned, then compare bytes. */ + if (!UNALIGNED(s1, s2)) { + /* + * If s1 and s2 are word-aligned, compare them a word at a time. + */ + a1 = (unsigned long *)s1; + a2 = (unsigned long *)s2; + while (*a1 == *a2) { + /* + * To get here, *a1 == *a2, thus if we find a null in + * *a1, then the strings must be equal, so return zero. + */ + if (DETECTNULL(*a1)) + return 0; + + a1++; + a2++; + } + + /* + * A difference was detected in last few bytes of s1, so search + * bytewise. + */ + s1 = (char *)a1; + s2 = (char *)a2; + } + + while (*s1 != '\0' && *s1 == *s2) { + s1++; + s2++; + } + return (*(unsigned char *)s1) - (*(unsigned char *)s2); +#endif /* not PREFER_SIZE_OVER_SPEED */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/strcpy.c b/optee_os/lib/libutils/isoc/newlib/strcpy.c new file mode 100644 index 0000000..9fc9ea1 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strcpy.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---copy string + +INDEX + strcpy + +ANSI_SYNOPSIS + #include + char *strcpy(char *<[dst]>, const char *<[src]>); + +TRAD_SYNOPSIS + #include + char *strcpy(<[dst]>, <[src]>) + char *<[dst]>; + char *<[src]>; + +DESCRIPTION + <> copies the string pointed to by <[src]> + (including the terminating null character) to the array + pointed to by <[dst]>. + +RETURNS + This function returns the initial value of <[dst]>. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strcpy ansi pure +*/ + +#include "_ansi.h" +#include +#include + +/*SUPPRESS 560*/ +/*SUPPRESS 530*/ + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X, Y) \ + (((long)X & (sizeof (long) - 1)) | ((long)Y & (sizeof (long) - 1))) + +#if LONG_MAX == 2147483647L +#define DETECTNULL(X) (((X) - 0x01010101) & ~(X) & 0x80808080) +#else +#if LONG_MAX == 9223372036854775807L +/* Nonzero if X (a long int) contains a NULL byte. */ +#define DETECTNULL(X) (((X) - 0x0101010101010101) & ~(X) & 0x8080808080808080) +#else +#error long int is not a 32bit or 64bit type. +#endif +#endif + +#ifndef DETECTNULL +#error long int is not a 32bit or 64bit byte +#endif + +char* +_DEFUN (strcpy, (dst0, src0), + char *dst0 _AND + _CONST char *src0) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + char *s = dst0; + + while (*dst0++ = *src0++) + ; + + return s; +#else + char *dst = dst0; + _CONST char *src = src0; + long *aligned_dst; + _CONST long *aligned_src; + + /* If SRC or DEST is unaligned, then copy bytes. */ + if (!UNALIGNED (src, dst)) + { + aligned_dst = (long*)dst; + aligned_src = (long*)src; + + /* SRC and DEST are both "long int" aligned, try to do "long int" + sized copies. */ + while (!DETECTNULL(*aligned_src)) + { + *aligned_dst++ = *aligned_src++; + } + + dst = (char*)aligned_dst; + src = (char*)aligned_src; + } + + while ((*dst++ = *src++)) + ; + return dst0; +#endif /* not PREFER_SIZE_OVER_SPEED */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/strlen.c b/optee_os/lib/libutils/isoc/newlib/strlen.c new file mode 100644 index 0000000..0a9d549 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strlen.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* This file is copied from newlib-1.19 */ + +/* +FUNCTION + <>---character string length + +INDEX + strlen + +ANSI_SYNOPSIS + #include + size_t strlen(const char *<[str]>); + +TRAD_SYNOPSIS + #include + size_t strlen(<[str]>) + char *<[src]>; + +DESCRIPTION + The <> function works out the length of the string + starting at <<*<[str]>>> by counting chararacters until it + reaches a <> character. + +RETURNS + <> returns the character count. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strlen ansi pure +*/ + +#include "_ansi.h" +#include +#include + +#define LBLOCKSIZE (sizeof(long)) +#define UNALIGNED(X) ((long)X & (LBLOCKSIZE - 1)) + +#if LONG_MAX == 2147483647L +#define DETECTNULL(X) (((X) - 0x01010101L) & ~(X) & 0x80808080UL) +#else +#if LONG_MAX == 9223372036854775807L +/* Nonzero if X (a long int) contains a NULL byte. */ +#define DETECTNULL(X) (((X) - 0x0101010101010101L) & ~(X) & \ + 0x8080808080808080UL) +#else +#error long int is not a 32bit or 64bit type. +#endif +#endif + +#ifndef DETECTNULL +#error long int is not a 32bit or 64bit byte +#endif + +size_t _DEFUN(strlen, (str), _CONST char *str) +{ + _CONST char *start = str; + +#if !defined(PREFER_SIZE_OVER_SPEED) && !defined(__OPTIMIZE_SIZE__) + unsigned long *aligned_addr; + + /* Align the pointer, so we can search a word at a time. */ + while (UNALIGNED(str)) { + if (!*str) + return str - start; + str++; + } + + /* If the string is word-aligned, we can check for the presence of + a null in each word-sized block. */ + aligned_addr = (unsigned long *)str; + while (!DETECTNULL(*aligned_addr)) + aligned_addr++; + + /* Once a null is detected, we check each byte in that block for a + precise position of the null. */ + str = (char *)aligned_addr; + +#endif /* not PREFER_SIZE_OVER_SPEED */ + + while (*str) + str++; + return str - start; +} diff --git a/optee_os/lib/libutils/isoc/newlib/strncmp.c b/optee_os/lib/libutils/isoc/newlib/strncmp.c new file mode 100644 index 0000000..3cf2b9c --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strncmp.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---character string compare + +INDEX + strncmp + +ANSI_SYNOPSIS + #include + int strncmp(const char *<[a]>, const char * <[b]>, size_t <[length]>); + +TRAD_SYNOPSIS + #include + int strncmp(<[a]>, <[b]>, <[length]>) + char *<[a]>; + char *<[b]>; + size_t <[length]> + +DESCRIPTION + <> compares up to <[length]> characters + from the string at <[a]> to the string at <[b]>. + +RETURNS + If <<*<[a]>>> sorts lexicographically after <<*<[b]>>>, + <> returns a number greater than zero. If the two + strings are equivalent, <> returns zero. If <<*<[a]>>> + sorts lexicographically before <<*<[b]>>>, <> returns a + number less than zero. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strncmp ansi pure +*/ + +#include "_ansi.h" +#include +#include + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X, Y) \ + (((long)X & (sizeof (long) - 1)) | ((long)Y & (sizeof (long) - 1))) + +/* DETECTNULL returns nonzero if (long)X contains a NULL byte. */ +#if LONG_MAX == 2147483647L +#define DETECTNULL(X) (((X) - 0x01010101L) & ~(X) & 0x80808080UL) +#else +#if LONG_MAX == 9223372036854775807L +#define DETECTNULL(X) (((X) - 0x0101010101010101L) & ~(X) & \ + 0x8080808080808080UL) +#else +#error long int is not a 32bit or 64bit type. +#endif +#endif + +#ifndef DETECTNULL +#error long int is not a 32bit or 64bit byte +#endif + +int +_DEFUN (strncmp, (s1, s2, n), + _CONST char *s1 _AND + _CONST char *s2 _AND + size_t n) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + if (n == 0) + return 0; + + while (n-- != 0 && *s1 == *s2) + { + if (n == 0 || *s1 == '\0') + break; + s1++; + s2++; + } + + return (*(unsigned char *) s1) - (*(unsigned char *) s2); +#else + unsigned long *a1; + unsigned long *a2; + + if (n == 0) + return 0; + + /* If s1 or s2 are unaligned, then compare bytes. */ + if (!UNALIGNED (s1, s2)) + { + /* If s1 and s2 are word-aligned, compare them a word at a time. */ + a1 = (unsigned long*)s1; + a2 = (unsigned long*)s2; + while (n >= sizeof (long) && *a1 == *a2) + { + n -= sizeof (long); + + /* If we've run out of bytes or hit a null, return zero + since we already know *a1 == *a2. */ + if (n == 0 || DETECTNULL (*a1)) + return 0; + + a1++; + a2++; + } + + /* A difference was detected in last few bytes of s1, so search bytewise */ + s1 = (char*)a1; + s2 = (char*)a2; + } + + while (n-- > 0 && *s1 == *s2) + { + /* If we've run out of bytes or hit a null, return zero + since we already know *s1 == *s2. */ + if (n == 0 || *s1 == '\0') + return 0; + s1++; + s2++; + } + return (*(unsigned char *) s1) - (*(unsigned char *) s2); +#endif /* not PREFER_SIZE_OVER_SPEED */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/strncpy.c b/optee_os/lib/libutils/isoc/newlib/strncpy.c new file mode 100644 index 0000000..2148434 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strncpy.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---counted copy string + +INDEX + strncpy + +ANSI_SYNOPSIS + #include + char *strncpy(char *restrict <[dst]>, const char *restrict <[src]>, + size_t <[length]>); + +TRAD_SYNOPSIS + #include + char *strncpy(<[dst]>, <[src]>, <[length]>) + char *<[dst]>; + char *<[src]>; + size_t <[length]>; + +DESCRIPTION + <> copies not more than <[length]> characters from the + the string pointed to by <[src]> (including the terminating + null character) to the array pointed to by <[dst]>. If the + string pointed to by <[src]> is shorter than <[length]> + characters, null characters are appended to the destination + array until a total of <[length]> characters have been + written. + +RETURNS + This function returns the initial value of <[dst]>. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strncpy ansi pure +*/ + +#include "_ansi.h" +#include +#include + +/*SUPPRESS 560*/ +/*SUPPRESS 530*/ + +/* Nonzero if either X or Y is not aligned on a "long" boundary. */ +#define UNALIGNED(X, Y) \ + (((long)X & (sizeof (long) - 1)) | ((long)Y & (sizeof (long) - 1))) + +#if LONG_MAX == 2147483647L +#define DETECTNULL(X) (((X) - 0x01010101) & ~(X) & 0x80808080) +#else +#if LONG_MAX == 9223372036854775807L +/* Nonzero if X (a long int) contains a NULL byte. */ +#define DETECTNULL(X) (((X) - 0x0101010101010101) & ~(X) & 0x8080808080808080) +#else +#error long int is not a 32bit or 64bit type. +#endif +#endif + +#ifndef DETECTNULL +#error long int is not a 32bit or 64bit byte +#endif + +#define TOO_SMALL(LEN) ((LEN) < sizeof (long)) + +char * +_DEFUN (strncpy, (dst0, src0), + char *__restrict dst0 _AND + _CONST char *__restrict src0 _AND + size_t count) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + char *dscan; + _CONST char *sscan; + + dscan = dst0; + sscan = src0; + while (count > 0) + { + --count; + if ((*dscan++ = *sscan++) == '\0') + break; + } + while (count-- > 0) + *dscan++ = '\0'; + + return dst0; +#else + char *dst = dst0; + _CONST char *src = src0; + long *aligned_dst; + _CONST long *aligned_src; + + /* If SRC and DEST is aligned and count large enough, then copy words. */ + if (!UNALIGNED (src, dst) && !TOO_SMALL (count)) + { + aligned_dst = (long*)dst; + aligned_src = (long*)src; + + /* SRC and DEST are both "long int" aligned, try to do "long int" + sized copies. */ + while (count >= sizeof (long int) && !DETECTNULL(*aligned_src)) + { + count -= sizeof (long int); + *aligned_dst++ = *aligned_src++; + } + + dst = (char*)aligned_dst; + src = (char*)aligned_src; + } + + while (count > 0) + { + --count; + if ((*dst++ = *src++) == '\0') + break; + } + + while (count-- > 0) + *dst++ = '\0'; + + return dst0; +#endif /* not PREFER_SIZE_OVER_SPEED */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/strnlen.c b/optee_os/lib/libutils/isoc/newlib/strnlen.c new file mode 100644 index 0000000..1927127 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strnlen.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---character string length + +INDEX + strnlen + +ANSI_SYNOPSIS + #include + size_t strnlen(const char *<[str]>, size_t <[n]>); + +TRAD_SYNOPSIS + #include + size_t strnlen(<[str]>, <[n]>) + char *<[src]>; + size_t <[n]>; + +DESCRIPTION + The <> function works out the length of the string + starting at <<*<[str]>>> by counting chararacters until it + reaches a NUL character or the maximum: <[n]> number of + characters have been inspected. + +RETURNS + <> returns the character count or <[n]>. + +PORTABILITY +<> is a GNU extension. + +<> requires no supporting OS subroutines. + +*/ + +#undef __STRICT_ANSI__ +#include "_ansi.h" +#include + +size_t +_DEFUN (strnlen, (str, n), + _CONST char *str _AND + size_t n) +{ + _CONST char *start = str; + + while (n-- > 0 && *str) + str++; + + return str - start; +} diff --git a/optee_os/lib/libutils/isoc/newlib/strrchr.c b/optee_os/lib/libutils/isoc/newlib/strrchr.c new file mode 100644 index 0000000..6e69acc --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strrchr.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---reverse search for character in string + +INDEX + strrchr + +ANSI_SYNOPSIS + #include + char * strrchr(const char *<[string]>, int <[c]>); + +TRAD_SYNOPSIS + #include + char * strrchr(<[string]>, <[c]>); + char *<[string]>; + int *<[c]>; + +DESCRIPTION + This function finds the last occurence of <[c]> (converted to + a char) in the string pointed to by <[string]> (including the + terminating null character). + +RETURNS + Returns a pointer to the located character, or a null pointer + if <[c]> does not occur in <[string]>. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strrchr ansi pure +*/ +#include "_ansi.h" +#include + +char * +_DEFUN (strrchr, (s, i), + _CONST char *s _AND + int i) +{ + _CONST char *last = NULL; + + if (i) + { + while ((s=strchr(s, i))) + { + last = s; + s++; + } + } + else + { + last = strchr(s, i); + } + + return (char *) last; +} diff --git a/optee_os/lib/libutils/isoc/newlib/strstr.c b/optee_os/lib/libutils/isoc/newlib/strstr.c new file mode 100644 index 0000000..349ae4f --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strstr.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---find string segment + +INDEX + strstr + +ANSI_SYNOPSIS + #include + char *strstr(const char *<[s1]>, const char *<[s2]>); + +TRAD_SYNOPSIS + #include + char *strstr(<[s1]>, <[s2]>) + char *<[s1]>; + char *<[s2]>; + +DESCRIPTION + Locates the first occurrence in the string pointed to by <[s1]> of + the sequence of characters in the string pointed to by <[s2]> + (excluding the terminating null character). + +RETURNS + Returns a pointer to the located string segment, or a null + pointer if the string <[s2]> is not found. If <[s2]> points to + a string with zero length, <[s1]> is returned. + +PORTABILITY +<> is ANSI C. + +<> requires no supporting OS subroutines. + +QUICKREF + strstr ansi pure +*/ + +#include "_ansi.h" +#include + +#if !defined(PREFER_SIZE_OVER_SPEED) && !defined(__OPTIMIZE_SIZE__) +# define RETURN_TYPE char * +# define AVAILABLE(h, h_l, j, n_l) \ + (!memchr ((h) + (h_l), '\0', (j) + (n_l) - (h_l)) \ + && ((h_l) = (j) + (n_l))) +# include "str-two-way.h" +#endif + +char * +_DEFUN (strstr, (searchee, lookfor), + _CONST char *searchee _AND + _CONST char *lookfor) +{ +#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__) + + /* Less code size, but quadratic performance in the worst case. */ + if (*searchee == 0) + { + if (*lookfor) + return (char *) NULL; + return (char *) searchee; + } + + while (*searchee) + { + size_t i; + i = 0; + + while (1) + { + if (lookfor[i] == 0) + { + return (char *) searchee; + } + + if (lookfor[i] != searchee[i]) + { + break; + } + i++; + } + searchee++; + } + + return (char *) NULL; + +#else /* compilation for speed */ + + /* Larger code size, but guaranteed linear performance. */ + const char *haystack = searchee; + const char *needle = lookfor; + size_t needle_len; /* Length of NEEDLE. */ + size_t haystack_len; /* Known minimum length of HAYSTACK. */ + int ok = 1; /* True if NEEDLE is prefix of HAYSTACK. */ + + /* Determine length of NEEDLE, and in the process, make sure + HAYSTACK is at least as long (no point processing all of a long + NEEDLE if HAYSTACK is too short). */ + while (*haystack && *needle) + ok &= *haystack++ == *needle++; + if (*needle) + return NULL; + if (ok) + return (char *) searchee; + + /* Reduce the size of haystack using strchr, since it has a smaller + linear coefficient than the Two-Way algorithm. */ + needle_len = needle - lookfor; + haystack = strchr (searchee + 1, *lookfor); + if (!haystack || needle_len == 1) + return (char *) haystack; + haystack_len = (haystack > searchee + needle_len ? 1 + : needle_len + searchee - haystack); + + /* Perform the search. */ + if (needle_len < LONG_NEEDLE_THRESHOLD) + return two_way_short_needle ((const unsigned char *) haystack, + haystack_len, + (const unsigned char *) lookfor, needle_len); + return two_way_long_needle ((const unsigned char *) haystack, haystack_len, + (const unsigned char *) lookfor, needle_len); +#endif /* compilation for speed */ +} diff --git a/optee_os/lib/libutils/isoc/newlib/strtok_r.c b/optee_os/lib/libutils/isoc/newlib/strtok_r.c new file mode 100644 index 0000000..084f794 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strtok_r.c @@ -0,0 +1,97 @@ +/* + * Copyright (c) 1988 Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +static char * +__strtok_r (register char *s, + register const char *delim, + char **lasts, + int skip_leading_delim) +{ + register char *spanp; + register int c, sc; + char *tok; + + + if (s == NULL && (s = *lasts) == NULL) + return (NULL); + + /* + * Skip (span) leading delimiters (s += strspn(s, delim), sort of). + */ +cont: + c = *s++; + for (spanp = (char *)delim; (sc = *spanp++) != 0;) { + if (c == sc) { + if (skip_leading_delim) { + goto cont; + } + else { + *lasts = s; + s[-1] = 0; + return (s - 1); + } + } + } + + if (c == 0) { /* no non-delimiter characters */ + *lasts = NULL; + return (NULL); + } + tok = s - 1; + + /* + * Scan token (scan for delimiters: s += strcspn(s, delim), sort of). + * Note that delim must have one NUL; we stop if we see that, too. + */ + for (;;) { + c = *s++; + spanp = (char *)delim; + do { + if ((sc = *spanp++) == c) { + if (c == 0) + s = NULL; + else + s[-1] = 0; + *lasts = s; + return (tok); + } + } while (sc != 0); + } + /* NOTREACHED */ +} + +char * +strtok_r (register char *__restrict s, + register const char *__restrict delim, + char **__restrict lasts) +{ + return __strtok_r (s, delim, lasts, 1); +} diff --git a/optee_os/lib/libutils/isoc/newlib/strtoul.c b/optee_os/lib/libutils/isoc/newlib/strtoul.c new file mode 100644 index 0000000..6ab4a32 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/strtoul.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 1994-2009 Red Hat, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* +FUNCTION + <>---string to unsigned long + +INDEX + strtoul +INDEX + _strtoul_r + +ANSI_SYNOPSIS + #include + unsigned long strtoul(const char *<[s]>, char **<[ptr]>, + int <[base]>); + + unsigned long _strtoul(const char *<[s]>, + char **<[ptr]>, int <[base]>); + +TRAD_SYNOPSIS + #include + unsigned long strtoul(<[s]>, <[ptr]>, <[base]>) + char *<[s]>; + char **<[ptr]>; + int <[base]>; + + unsigned long _strtoul(<[s]>, <[ptr]>, <[base]>) + char *<[s]>; + char **<[ptr]>; + int <[base]>; + +DESCRIPTION +The function <> converts the string <<*<[s]>>> to +an <>. First, it breaks down the string into three parts: +leading whitespace, which is ignored; a subject string consisting +of the digits meaningful in the radix specified by <[base]> +(for example, <<0>> through <<7>> if the value of <[base]> is 8); +and a trailing portion consisting of one or more unparseable characters, +which always includes the terminating null character. Then, it attempts +to convert the subject string into an unsigned long integer, and returns the +result. + +If the value of <[base]> is zero, the subject string is expected to look +like a normal C integer constant (save that no optional sign is permitted): +a possible <<0x>> indicating hexadecimal radix, and a number. +If <[base]> is between 2 and 36, the expected form of the subject is a +sequence of digits (which may include letters, depending on the +base) representing an integer in the radix specified by <[base]>. +The letters <>--<> (or <>--<>) are used as digits valued from +10 to 35. If <[base]> is 16, a leading <<0x>> is permitted. + +The subject sequence is the longest initial sequence of the input +string that has the expected form, starting with the first +non-whitespace character. If the string is empty or consists entirely +of whitespace, or if the first non-whitespace character is not a +permissible digit, the subject string is empty. + +If the subject string is acceptable, and the value of <[base]> is zero, +<> attempts to determine the radix from the input string. A +string with a leading <<0x>> is treated as a hexadecimal value; a string with +a leading <<0>> and no <> is treated as octal; all other strings are +treated as decimal. If <[base]> is between 2 and 36, it is used as the +conversion radix, as described above. Finally, a pointer to the first +character past the converted subject string is stored in <[ptr]>, if +<[ptr]> is not <>. + +If the subject string is empty (that is, if <<*>><[s]> does not start +with a substring in acceptable form), no conversion +is performed and the value of <[s]> is stored in <[ptr]> (if <[ptr]> is +not <>). + + + +RETURNS +<> returns the converted value, if any. If no conversion was +made, <<0>> is returned. + +<> returns <> if the magnitude of the converted +value is too large, and sets <> to <>. + +PORTABILITY +<> is ANSI. + +<> requires no supporting OS subroutines. +*/ + +/* + * Copyright (c) 1990 Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "_ansi.h" +#include +#include +#include + +/* + * Convert a string to an unsigned long integer. + * + * Ignores `locale' stuff. Assumes that the upper and lower case + * alphabets and digits are each contiguous. + */ +unsigned long +_DEFUN (_strtoul, (nptr, endptr, base), + _CONST char *nptr _AND + char **endptr _AND + int base) +{ + register const unsigned char *s = (const unsigned char *)nptr; + register unsigned long acc; + register int c; + register unsigned long cutoff; + register int neg = 0, any, cutlim; + + /* + * See strtol for comments as to the logic used. + */ + do { + c = *s++; + } while (isspace(c)); + if (c == '-') { + neg = 1; + c = *s++; + } else if (c == '+') + c = *s++; + if ((base == 0 || base == 16) && + c == '0' && (*s == 'x' || *s == 'X')) { + c = s[1]; + s += 2; + base = 16; + } + if (base == 0) + base = c == '0' ? 8 : 10; + cutoff = (unsigned long)ULONG_MAX / (unsigned long)base; + cutlim = (unsigned long)ULONG_MAX % (unsigned long)base; + for (acc = 0, any = 0;; c = *s++) { + if (isdigit(c)) + c -= '0'; + else if (isalpha(c)) + c -= isupper(c) ? 'A' - 10 : 'a' - 10; + else + break; + if (c >= base) + break; + if (any < 0 || acc > cutoff || (acc == cutoff && c > cutlim)) + any = -1; + else { + any = 1; + acc *= base; + acc += c; + } + } + if (any < 0) { + acc = ULONG_MAX; + } else if (neg) + acc = -acc; + if (endptr != 0) + *endptr = (char *) (any ? (char *)s - 1 : nptr); + return (acc); +} + +#ifndef _REENT_ONLY + +unsigned long +_DEFUN (strtoul, (s, ptr, base), + _CONST char *s _AND + char **ptr _AND + int base) +{ + return _strtoul (s, ptr, base); +} + +#endif diff --git a/optee_os/lib/libutils/isoc/newlib/sub.mk b/optee_os/lib/libutils/isoc/newlib/sub.mk new file mode 100644 index 0000000..20031c7 --- /dev/null +++ b/optee_os/lib/libutils/isoc/newlib/sub.mk @@ -0,0 +1,28 @@ +cflags-y += -Wno-sign-compare +cflags-y += -Wno-parentheses +cflags-remove-y += -Wcast-align + +srcs-y += abs.c +srcs-y += bcmp.c +srcs-y += memchr.c +srcs-y += memcmp.c +srcs-y += memcpy.c +ifeq (s,$(CFG_CC_OPT_LEVEL)) +cflags-memcpy.c-y += -O2 +endif +cflags-memcpy.c-y += $(call cc-option,-fno-tree-loop-distribute-patterns) +srcs-y += memmove.c +cflags-memmove.c-y += $(call cc-option,-fno-tree-loop-distribute-patterns) +srcs-y += memset.c +cflags-memset.c-y += $(call cc-option,-fno-tree-loop-distribute-patterns) +srcs-y += strchr.c +srcs-y += strcmp.c +srcs-y += strcpy.c +srcs-y += strlen.c +srcs-y += strncmp.c +srcs-y += strncpy.c +srcs-y += strnlen.c +srcs-y += strrchr.c +srcs-y += strstr.c +srcs-y += strtoul.c +srcs-y += strtok_r.c diff --git a/optee_os/lib/libutils/isoc/qsort.c b/optee_os/lib/libutils/isoc/qsort.c new file mode 100644 index 0000000..d2d8127 --- /dev/null +++ b/optee_os/lib/libutils/isoc/qsort.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* $OpenBSD: qsort.c,v 1.10 2005/08/08 08:05:37 espie Exp $ */ +/*- + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include +static __inline char + *med3(char *, char *, char *, int (*)(const void *, const void *)); +static __inline void swapfunc(char *, char *, int, int); +#define min(a, b) (a) < (b) ? a : b +/* + * Qsort routine from Bentley & McIlroy's "Engineering a Sort Function". + */ +#define swapcode(TYPE, parmi, parmj, n) { \ + long i = (n) / sizeof (TYPE); \ + TYPE *pi = (TYPE *) (parmi); \ + TYPE *pj = (TYPE *) (parmj); \ + do { \ + TYPE t = *pi; \ + *pi++ = *pj; \ + *pj++ = t; \ + } while (--i > 0); \ +} +#define SWAPINIT(a, es) swaptype = (uintptr_t)a % sizeof(long) || \ + es % sizeof(long) ? 2 : es == sizeof(long)? 0 : 1; +static __inline void +swapfunc(char *a, char *b, int n, int swaptype) +{ + if (swaptype <= 1) + swapcode(long, a, b, n) + else + swapcode(char, a, b, n) +} +#define swap(a, b) \ + if (swaptype == 0) { \ + long t = *(long *)(a); \ + *(long *)(a) = *(long *)(b); \ + *(long *)(b) = t; \ + } else \ + swapfunc(a, b, es, swaptype) +#define vecswap(a, b, n) if ((n) > 0) swapfunc(a, b, n, swaptype) +static __inline char * +med3(char *a, char *b, char *c, int (*cmp)(const void *, const void *)) +{ + return cmp(a, b) < 0 ? + (cmp(b, c) < 0 ? b : (cmp(a, c) < 0 ? c : a )) + :(cmp(b, c) > 0 ? b : (cmp(a, c) < 0 ? a : c )); +} +void +qsort(void *aa, size_t n, size_t es, int (*cmp)(const void *, const void *)) +{ + char *pa, *pb, *pc, *pd, *pl, *pm, *pn; + int d, r, swaptype, swap_cnt; + char *a = aa; + loop: SWAPINIT(a, es); + swap_cnt = 0; + if (n < 7) { + for (pm = (char *)a + es; pm < (char *) a + n * es; pm += es) + for (pl = pm; pl > (char *) a && cmp(pl - es, pl) > 0; + pl -= es) + swap(pl, pl - es); + return; + } + pm = (char *)a + (n / 2) * es; + if (n > 7) { + pl = (char *)a; + pn = (char *)a + (n - 1) * es; + if (n > 40) { + d = (n / 8) * es; + pl = med3(pl, pl + d, pl + 2 * d, cmp); + pm = med3(pm - d, pm, pm + d, cmp); + pn = med3(pn - 2 * d, pn - d, pn, cmp); + } + pm = med3(pl, pm, pn, cmp); + } + swap(a, pm); + pa = pb = (char *)a + es; + + pc = pd = (char *)a + (n - 1) * es; + for (;;) { + while (pb <= pc && (r = cmp(pb, a)) <= 0) { + if (r == 0) { + swap_cnt = 1; + swap(pa, pb); + pa += es; + } + pb += es; + } + while (pb <= pc && (r = cmp(pc, a)) >= 0) { + if (r == 0) { + swap_cnt = 1; + swap(pc, pd); + pd -= es; + } + pc -= es; + } + if (pb > pc) + break; + swap(pb, pc); + swap_cnt = 1; + pb += es; + pc -= es; + } + if (swap_cnt == 0) { /* Switch to insertion sort */ + for (pm = (char *) a + es; pm < (char *) a + n * es; pm += es) + for (pl = pm; pl > (char *) a && cmp(pl - es, pl) > 0; + pl -= es) + swap(pl, pl - es); + return; + } + pn = (char *)a + n * es; + r = min(pa - (char *)a, pb - pa); + vecswap(a, pb - r, r); + r = min(pd - pc, pn - pd - (int)es); + vecswap(pb, pn - r, r); + if ((r = pb - pa) > (int)es) + qsort(a, r / es, es, cmp); + if ((r = pd - pc) > (int)es) { + /* Iterate rather than recurse to save stack space */ + a = pn - r; + n = r / es; + goto loop; + } + /* qsort(pn - r, r / es, es, cmp);*/ +} diff --git a/optee_os/lib/libutils/isoc/snprintf.c b/optee_os/lib/libutils/isoc/snprintf.c new file mode 100644 index 0000000..f5cf74a --- /dev/null +++ b/optee_os/lib/libutils/isoc/snprintf.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ + +#include +#include + +int snprintf(char *bf, size_t size, const char *fmt, ...) +{ + int retval; + va_list ap; + + va_start(ap, fmt); + retval = __vsnprintf(bf, size, fmt, ap, false); + va_end(ap); + + return retval; +} + +int vsnprintf(char *bf, size_t size, const char *fmt, va_list ap) +{ + return __vsnprintf(bf, size, fmt, ap, false); +} diff --git a/optee_os/lib/libutils/isoc/sprintf.c b/optee_os/lib/libutils/isoc/sprintf.c new file mode 100644 index 0000000..d88fe41 --- /dev/null +++ b/optee_os/lib/libutils/isoc/sprintf.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ + +#include +#include +#include +#include + +int sprintf(char *str, const char *fmt, ...) +{ + int retval; + va_list ap; + + va_start(ap, fmt); + retval = __vsprintf(str, fmt, ap); + va_end(ap); + + return retval; +} + +int __sprintf_chk(char *str, int flag __unused, size_t slen, + const char *fmt, ...) +{ + int retval; + va_list ap; + + if (slen == 0) + abort(); + + va_start(ap, fmt); + retval = __vsnprintf(str, slen, fmt, ap, false); + va_end(ap); + + if (retval > 0 && (size_t)retval >= slen) + abort(); + + return retval; +} diff --git a/optee_os/lib/libutils/isoc/stack_check.c b/optee_os/lib/libutils/isoc/stack_check.c new file mode 100644 index 0000000..f800a2b --- /dev/null +++ b/optee_os/lib/libutils/isoc/stack_check.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include + +#if defined(__KERNEL__) +# include +# define PANIC() panic() +#elif defined(__LDELF__) +# include +# define PANIC() _ldelf_panic(2) +#else +# include +# define PANIC() _utee_panic(TEE_ERROR_OVERFLOW) +#endif + +void *__stack_chk_guard __nex_data = (void *)0x00000aff; + +void __attribute__((noreturn)) __stack_chk_fail(void); + +void __stack_chk_fail(void) +{ + EMSG_RAW("stack smashing detected"); + while (1) + PANIC(); +} + diff --git a/optee_os/lib/libutils/isoc/strdup.c b/optee_os/lib/libutils/isoc/strdup.c new file mode 100644 index 0000000..be7f59a --- /dev/null +++ b/optee_os/lib/libutils/isoc/strdup.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include + +char *strdup(const char *s) +{ + size_t l = strlen(s) + 1; + char *p = malloc(l); + + if (p) + memcpy(p, s, l); + return p; +} diff --git a/optee_os/lib/libutils/isoc/strndup.c b/optee_os/lib/libutils/isoc/strndup.c new file mode 100644 index 0000000..dd2934e --- /dev/null +++ b/optee_os/lib/libutils/isoc/strndup.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include + +char *strndup(const char *s, size_t n) +{ + size_t l = strnlen(s, n) + 1; + char *p = malloc(l); + + if (p) { + memcpy(p, s, l - 1); + p[l - 1] = '\0'; + } + return p; +} diff --git a/optee_os/lib/libutils/isoc/sub.mk b/optee_os/lib/libutils/isoc/sub.mk new file mode 100644 index 0000000..ef1ca5d --- /dev/null +++ b/optee_os/lib/libutils/isoc/sub.mk @@ -0,0 +1,40 @@ +global-incdirs-y += include + +srcs-y += bget_malloc.c +cflags-remove-bget_malloc.c-y += -Wold-style-definition -Wredundant-decls +cflags-bget_malloc.c-y += -Wno-sign-compare -Wno-cast-align +ifeq ($(sm),core) +cflags-remove-bget_malloc.c-y += $(cflags_kasan) +endif +srcs-y += isdigit.c +srcs-y += isxdigit.c +srcs-y += qsort.c +cflags-qsort.c-y += -Wno-inline +cflags-remove-qsort.c-y += -Wcast-align +srcs-y += sprintf.c +srcs-y += snprintf.c +srcs-y += stack_check.c +srcs-y += strdup.c +srcs-y += strndup.c +srcs-y += tolower.c +srcs-y += isalpha.c +srcs-y += isspace.c +srcs-y += isupper.c +srcs-y += isalnum.c +srcs-y += iscntrl.c +srcs-y += isgraph.c +srcs-y += islower.c +srcs-y += isprint.c +srcs-y += ispunct.c +srcs-y += toupper.c + +ifneq (,$(filter ta_%,$(sm))) +srcs-y += fp.c +srcs-y += fputc.c +srcs-y += fputs.c +srcs-y += fwrite.c +srcs-y += write.c +endif + +subdirs-y += newlib +subdirs-y += arch/$(ARCH) diff --git a/optee_os/lib/libutils/isoc/tolower.c b/optee_os/lib/libutils/isoc/tolower.c new file mode 100644 index 0000000..1700b9d --- /dev/null +++ b/optee_os/lib/libutils/isoc/tolower.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2015, Linaro Limited + */ +#include + +int tolower(int c) +{ + if (c >= 'A' && c <= 'Z') + return c - 'A' + 'a'; + return c; +} diff --git a/optee_os/lib/libutils/isoc/toupper.c b/optee_os/lib/libutils/isoc/toupper.c new file mode 100644 index 0000000..8cfa0d1 --- /dev/null +++ b/optee_os/lib/libutils/isoc/toupper.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019, KAIST + */ +#include + +int toupper(int c) +{ + if (c >= 'a' && c <= 'z') + return c - 'a' + 'A'; + return c; +} diff --git a/optee_os/lib/libutils/isoc/write.c b/optee_os/lib/libutils/isoc/write.c new file mode 100644 index 0000000..987de95 --- /dev/null +++ b/optee_os/lib/libutils/isoc/write.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2020, Huawei Technologies Co., Ltd + */ + +#include +#include +#include + +ssize_t write(int fd, const void *buf, size_t count) +{ + if (fd != 1 && fd != 2) + abort(); + + return printf("%*s", (int)count, (char *)buf); +} diff --git a/optee_os/lib/libutils/sub.mk b/optee_os/lib/libutils/sub.mk new file mode 100644 index 0000000..e49bd89 --- /dev/null +++ b/optee_os/lib/libutils/sub.mk @@ -0,0 +1,2 @@ +subdirs-$(CFG_LIBUTILS_WITH_ISOC) += isoc +subdirs-y += ext diff --git a/optee_os/mk/aosp_optee.mk b/optee_os/mk/aosp_optee.mk new file mode 100644 index 0000000..07bc67a --- /dev/null +++ b/optee_os/mk/aosp_optee.mk @@ -0,0 +1,148 @@ +########################################################## +## Common mk file used for Android to compile and ## +## integrate OP-TEE related components ## +## Following flags need to be defined in optee*.mk ## +## OPTEE_OS_DIR ## +## OPTEE_TA_TARGETS ## +## OPTEE_CFG_ARM64_CORE ## +## OPTEE_PLATFORM ## +## OPTEE_PLATFORM_FLAVOR ## +## OPTEE_EXTRA_FLAGS (optional) ## +## OPTEE_EXTRA_TA_FLAGS (optional) ## +## And BUILD_OPTEE_MK needs to be defined in optee*.mk ## +## to point to this file ## +## ## +## local_module needs to be defined before including ## +## this file to build TAs ## +## ## +########################################################## + +########################################################## +## define common variables, like TA_DEV_KIT_DIR ## +########################################################## +TOP_ROOT_ABS := $(realpath $(TOP)) + +PREBUILT_MAKE ?= prebuilts/build-tools/linux-x86/bin/make +# we need this check because the Pie build does not have +# this prebuilt make tool +ifneq (,$(wildcard $(PREBUILT_MAKE))) +# for master and versions which has prebuilt make +HOST_MAKE := $(PREBUILT_MAKE) + +# The AOSP build tool is not the regular make, +# that it adds -jN to $(MAKE), and that we should preserve +# the flag or we would lose parallel build +# The MAKE is redefined here in AOSP ckati: +# https://android.googlesource.com/platform/build/kati/+/master/main.cc#100 +ifneq (,$(filter -j%, $(MAKE))) +HOST_MAKE += $(filter -j%, $(MAKE)) +endif + +else +# For P and old versions which does not have prebuilt make, +# let's use MAKE as what we did before +HOST_MAKE := $(MAKE) +endif + + +# OPTEE_OUT_DIR could be exported explicitly +# if PRODUCT_OUT is not the default out directory in aosp workspace +OPTEE_OUT_DIR ?= $(PRODUCT_OUT)/optee +ABS_OPTEE_OUT_DIR ?= $(realpath $(PRODUCT_OUT))/optee +OPTEE_TA_OUT_DIR ?= $(OPTEE_OUT_DIR)/ta +ABS_OPTEE_TA_OUT_DIR ?= $(ABS_OPTEE_OUT_DIR)/ta +# Set so that OP-TEE clients can find the installed dev-kit, which +# depends on platform and its OP-TEE word-size. +OPTEE_OS_OUT_DIR ?= $(OPTEE_OUT_DIR)/arm-plat-$(OPTEE_PLATFORM) +ABS_OPTEE_OS_OUT_DIR := $(ABS_OPTEE_OUT_DIR)/arm-plat-$(OPTEE_PLATFORM) +ifneq ($(local_optee_ta_target),) +TA_TARGET := $(local_optee_ta_target) +else +TA_TARGET := $(OPTEE_TA_TARGETS) +endif +TA_DEV_KIT_DIR := $(ABS_OPTEE_OS_OUT_DIR)/export-$(TA_TARGET) + +CROSS_COMPILE64 ?= $(TOP_ROOT_ABS)/$(TARGET_TOOLS_PREFIX) +CROSS_COMPILE_LINE := CROSS_COMPILE64="$(CROSS_COMPILE64)" +ifneq ($(strip $($(combo_2nd_arch_prefix)TARGET_TOOLS_PREFIX)),) +CROSS_COMPILE32 ?= $(TOP_ROOT_ABS)/$($(combo_2nd_arch_prefix)TARGET_TOOLS_PREFIX) +CROSS_COMPILE_LINE += CROSS_COMPILE32="$(CROSS_COMPILE32)" +endif + +OPTEE_BIN := $(OPTEE_OS_OUT_DIR)/core/tee.bin + +$(OPTEE_BIN) : $(sort $(shell find -L $(OPTEE_OS_DIR))) + +########################################################### +## define making rules for $(OPTEE_BIN) target, and add ## +## condition check to make it only be defined once ## +## even though this mk file might be included multiple ## +## times. The process to generate $(OPTEE_BIN) file will ## +## generate the header files under ## +## $(TA_DEV_KIT_DIR)/host_include too. ## +## And the $(OPTEE_BIN) will be used as dependency for ## +## other projects ## +########################################################### +ifneq (true,$(BUILD_OPTEE_OS_DEFINED)) +BUILD_OPTEE_OS_DEFINED := true +$(OPTEE_BIN): + @echo "Start building optee_os..." + +$(HOST_MAKE) -C $(TOP_ROOT_ABS)/$(OPTEE_OS_DIR) \ + O=$(ABS_OPTEE_OS_OUT_DIR) \ + CFG_USER_TA_TARGETS=$(OPTEE_TA_TARGETS) \ + CFG_ARM64_core=$(OPTEE_CFG_ARM64_CORE) \ + PLATFORM=$(OPTEE_PLATFORM) \ + PLATFORM_FLAVOR=$(OPTEE_PLATFORM_FLAVOR) \ + $(CROSS_COMPILE_LINE) \ + $(OPTEE_EXTRA_FLAGS) + @echo "Finished building optee_os..." + +endif + +########################################################## +## Lines for building TAs automatically ## +## will only be included in Android.mk for TAs ## +## local_module: ## +## need to be defined before include for this ## +########################################################## +ifneq (false,$(INCLUDE_FOR_BUILD_TA)) +include $(CLEAR_VARS) + +define ta_class +$(if $(filter %.ta,$1),EXECUTABLES,STATIC_LIBRARIES) +endef + +LOCAL_MODULE := $(local_module) +LOCAL_PREBUILT_MODULE_FILE := $(OPTEE_TA_OUT_DIR)/$(LOCAL_MODULE) +LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR)/lib/optee_armtz +LOCAL_MODULE_CLASS := $(call ta_class,$(local_module)) +LOCAL_MODULE_TAGS := optional +LOCAL_REQUIRED_MODULES := $(local_module_deps) + +TA_TMP_DIR := $(addsuffix _$(TA_TARGET), $(subst /,_,$(LOCAL_PATH))) +TA_TMP_FILE := $(OPTEE_TA_OUT_DIR)/$(TA_TMP_DIR)/$(LOCAL_MODULE) +$(LOCAL_PREBUILT_MODULE_FILE): $(TA_TMP_FILE) + @mkdir -p $(dir $@) + cp -vf $< $@ + +TA_TMP_FILE_DEPS := +ifneq ($(local_module_deps),) +$(foreach dep, $(local_module_deps), $(eval TA_TMP_FILE_DEPS += $(call intermediates-dir-for,$(call ta_class,$(dep)),$(dep))/$(dep))) +endif +$(TA_TMP_FILE): $(TA_TMP_FILE_DEPS) +$(TA_TMP_FILE): PRIVATE_TA_SRC_DIR := $(LOCAL_PATH) +$(TA_TMP_FILE): PRIVATE_TA_TMP_FILE := $(TA_TMP_FILE) +$(TA_TMP_FILE): PRIVATE_TA_TMP_DIR := $(TA_TMP_DIR) +$(TA_TMP_FILE): PRIVATE_TA_DEV_KIT_DIR := $(TA_DEV_KIT_DIR) +$(TA_TMP_FILE): $(OPTEE_BIN) + @echo "Start building TA for $(PRIVATE_TA_SRC_DIR) $(PRIVATE_TA_TMP_FILE)..." + +$(HOST_MAKE) -C $(TOP_ROOT_ABS)/$(PRIVATE_TA_SRC_DIR) O=$(ABS_OPTEE_TA_OUT_DIR)/$(PRIVATE_TA_TMP_DIR) \ + TA_DEV_KIT_DIR=$(PRIVATE_TA_DEV_KIT_DIR) \ + $(CROSS_COMPILE_LINE) \ + $(OPTEE_EXTRA_TA_FLAGS) + @echo "Finished building TA for $(PRIVATE_TA_SRC_DIR) $(PRIVATE_TA_TMP_FILE)..." + +include $(BUILD_PREBUILT) +local_module_deps := +local_optee_ta_target := +endif diff --git a/optee_os/mk/cc-option.mk b/optee_os/mk/cc-option.mk new file mode 100644 index 0000000..715e8e4 --- /dev/null +++ b/optee_os/mk/cc-option.mk @@ -0,0 +1,17 @@ +_cc-option-supported = $(if $(shell $(CC$(sm)) -Werror $(1) -c -x c /dev/null -o /dev/null 2>/dev/null >/dev/null || echo "Not supported"),,1) +_cc-opt-cached-var-name = $(subst =,~,$(strip cached-cc-option-$(1)-$(subst $(empty) $(empty),,$(CC$(sm))))) +define _cc-option +$(eval _var_name := $(call _cc-opt-cached-var-name,$(1))) +$(eval $(_var_name) := $(if $(filter $(origin $(_var_name)),undefined),$(call _cc-option-supported,$(1)),$($(_var_name)))) +$(if $($(_var_name)),$(1),$(2)) +endef +cc-option = $(strip $(call _cc-option,$(1),$(2))) + +_ld-option-supported = $(if $(shell ($(LD$(sm)) -v $(1) 2>&1 || echo warning) | grep warning),,1) +_ld-opt-cached-var-name = $(subst =,~,$(subst $(empty) $(empty),,$(strip cached-ld-option-$(1)-$(LD$(sm))))) +define _ld-option +$(eval _var_name := $(call _ld-opt-cached-var-name,$(1))) +$(eval $(_var_name) := $(if $(filter $(origin $(_var_name)),undefined),$(call _ld-option-supported,$(1)),$($(_var_name)))) +$(if $($(_var_name)),$(1),$(2)) +endef +ld-option = $(strip $(call _ld-option,$(1),$(2))) diff --git a/optee_os/mk/checkconf.mk b/optee_os/mk/checkconf.mk new file mode 100644 index 0000000..449b1c2 --- /dev/null +++ b/optee_os/mk/checkconf.mk @@ -0,0 +1,183 @@ +# Generate/check/update a .h file to reflect the values of Makefile +# variables +# +# Example usage (by default, check-conf-h will consider all CFG_* +# and _CFG_* variables plus PLATFORM_*): +# +# path/to/conf.h: FORCE +# $(call check-conf-h) +# +# Or, to include only the variables with the given prefix(es): +# +# path/to/crypto_config.h: FORCE +# $(call check-conf-h,CFG_CRYPTO_ CRYPTO_) +define check-conf-h + $(q)set -e; \ + $(cmd-echo-silent) ' CHK $@'; \ + cnf='$(strip $(foreach var, \ + $(call cfg-vars-by-prefix,$1), \ + $(call cfg-make-define,$(var))))'; \ + guard="_`echo $@ | tr -- -/.+ _`_"; \ + mkdir -p $(dir $@); \ + echo "#ifndef $${guard}" >$@.tmp; \ + echo "#define $${guard}" >>$@.tmp; \ + echo -n "$${cnf}" | sed 's/_nl_ */\n/g' >>$@.tmp; \ + echo "#endif" >>$@.tmp; \ + $(call mv-if-changed,$@.tmp,$@) +endef + +define check-conf-cmake + $(q)set -e; \ + $(cmd-echo-silent) ' CHK $@'; \ + cnf='$(strip $(foreach var, \ + $(call cfg-vars-by-prefix,$1), \ + $(call cfg-cmake-set,$(var))))'; \ + mkdir -p $(dir $@); \ + echo "# auto-generated TEE configuration file" >$@.tmp; \ + echo "# TEE version ${TEE_IMPL_VERSION}" >>$@.tmp; \ + echo -n "$${cnf}" | sed 's/_nl_ */\n/g' >>$@.tmp; \ + $(call mv-if-changed,$@.tmp,$@) +endef + +define check-conf-mk + $(q)set -e; \ + $(cmd-echo-silent) ' CHK $@'; \ + cnf='$(strip $(foreach var, \ + $(call cfg-vars-by-prefix,CFG_), \ + $(strip $(var)=$($(var))_nl_)))'; \ + mkdir -p $(dir $@); \ + echo "# auto-generated TEE configuration file" >$@.tmp; \ + echo "# TEE version ${TEE_IMPL_VERSION}" >>$@.tmp; \ + echo "ARCH=${ARCH}" >>$@.tmp; \ + echo "PLATFORM=${PLATFORM}" >>$@.tmp; \ + echo "PLATFORM_FLAVOR=${PLATFORM_FLAVOR}" >>$@.tmp; \ + echo -n "$${cnf}" | sed 's/_nl_ */\n/g' >>$@.tmp; \ + $(call mv-if-changed,$@.tmp,$@) +endef + +define cfg-vars-by-prefix + $(strip $(if $(1),$(call _cfg-vars-by-prefix,$(1)), + $(call _cfg-vars-by-prefix,CFG_ _CFG_ PLATFORM_))) +endef + +define _cfg-vars-by-prefix + $(sort $(foreach prefix,$(1),$(filter $(prefix)%,$(.VARIABLES)))) +endef + +# Convert a makefile variable to a #define +# , n => +# y => 1 +# => +define cfg-make-define + $(strip $(if $(filter y,$($1)), + #define $1 1_nl_, + $(if $(filter xn x,x$($1)), + /* $1 is not set */_nl_, + #define $1 $($1)_nl_))) +endef + +# Convert a makefile variable to a cmake set statement +# , n => +# => +define cfg-cmake-set + $(strip $(if $(filter xn x,x$($1)), + # $1 is not set _nl_, + set($1 $($1))_nl_)) +endef + +# Returns 'y' if at least one variable is 'y', 'n' otherwise +# Example: +# FOO_OR_BAR := $(call cfg-one-enabled, FOO BAR) +cfg-one-enabled = $(if $(filter y, $(foreach var,$(1),$($(var)))),y,n) + +# Returns 'y' if all variables are 'y', 'n' otherwise +# Example: +# FOO_AND_BAR := $(call cfg-all-enabled, FOO BAR) +cfg-all-enabled = $(if $(strip $(1)),$(if $(call _cfg-all-enabled,$(1)),y,n),n) +_cfg-all-enabled = \ + $(strip \ + $(if $(1), \ + $(if $(filter y,$($(firstword $(1)))), \ + $(call _cfg-all-enabled,$(filter-out $(firstword $(1)),$(1))), \ + ), \ + y \ + ) \ + ) + +# Disable a configuration variable if some dependency is disabled +# Example: +# $(eval $(call cfg-depends-all,FOO,BAR BAZ)) +# Will set FOO to 'n' if it is initially 'y' and BAR or BAZ are not 'y' +cfg-depends-all = \ + $(strip \ + $(if $(filter y, $($(1))), \ + $(if $(filter y,$(call cfg-all-enabled,$(2))), \ + , \ + $(warning Warning: Disabling $(1) [requires $(strip $(2))]) \ + override $(1) := n \ + ) \ + ) \ + ) + +# Disable a configuration variable if all dependencies are disabled +# Example: +# $(eval $(call cfg-depends-one,FOO,BAR BAZ)) +# Will set FOO to 'n' if it is initially 'y' and both BAR and BAZ are not 'y' +cfg-depends-one = \ + $(strip \ + $(if $(filter y, $($(1))), \ + $(if $(filter y,$(call cfg-one-enabled,$(2))), \ + , \ + $(warning Warning: Disabling $(1) [requires (one of) $(strip $(2))]) \ + override $(1) := n \ + ) \ + ) \ + ) + + +# Enable all depend variables +# Example: +# $(eval $(call cfg-enable-all-depends,FOO,BAR BAZ)) +# Will enable BAR and BAZ if FOO is initially 'y' +cfg-enable-all-depends = \ + $(strip \ + $(if $(2), \ + $(if $(filter y, $($(1))), \ + $(if $(filter y,$($(firstword $(2)))), \ + , \ + $(warning Warning: Enabling $(firstword $(2)) [required by $(1)]) \ + $(eval override $(firstword $(2)) := y) \ + ) \ + $(call cfg-enable-all-depends,$(1),$(filter-out $(firstword $(2)),$(2))), \ + ) \ + , \ + ) \ + ) + +# Check if a configuration variable has an acceptable value +# Example: +# $(call cfg-check-value,FOO,foo bar) +# Will do nothing if $(CFG_FOO) is either foo or bar, and error out otherwise. +cfg-check-value = \ + $(if $(filter-out $(2),$(CFG_$(1))), \ + $(error CFG_$(1) is set to '$(CFG_$(1))', valid values are: $(2))) + +# Set a variable or error out if it was previously set to a different value +# The reason message (3rd parameter) is optional +# Example: +# $(call force,CFG_FOO,foo,required by CFG_BAR) +define force +$(eval $(call _force,$(strip $(1)),$(2),$(3))) +endef + +define _force +ifdef $(1) +ifneq ($($(1)),$(2)) +ifneq (,$(3)) +_reason := $$(_empty) [$(3)] +endif +$$(error $(1) is set to '$($(1))' (from $(origin $(1))) but its value must be '$(2)'$$(_reason)) +endif +endif +$(1) := $(2) +endef diff --git a/optee_os/mk/clang.mk b/optee_os/mk/clang.mk new file mode 100644 index 0000000..a045bee --- /dev/null +++ b/optee_os/mk/clang.mk @@ -0,0 +1,53 @@ +# CROSS_COMPILE_$(sm) is derived from CROSS_COMPILE/CROSS_COMPILE32/ +# CROSS_COMPILE64 and indicates the target that should be passed to clang. Path +# components are ignored, as well as any command before the compiler name (for +# instance "/some/path/ccache /other/path/arm-linux-gnueabihf-"). +# We try to extract any ccache command if present. +clang-target := $(patsubst %-,%,$(notdir $(lastword $(CROSS_COMPILE_$(sm))))) +ifeq ($(clang-target),aarch64-linux) +clang-target := aarch64-linux-gnu +endif +ccache-cmd := $(if $(findstring ccache,$(CROSS_COMPILE_$(sm))),$(firstword $(CROSS_COMPILE_$(sm))) ,) + +CC$(sm) := $(ccache-cmd)$(OPTEE_CLANG_COMPILER_PATH)clang --target=$(clang-target) +CXX$(sm) := false # Untested yet +# Due to the absence of clang-cpp in AOSP's prebuilt version of clang, +# use the equivalent command of 'clang -E' +CPP$(sm) := $(ccache-cmd)$(OPTEE_CLANG_COMPILER_PATH)clang --target=$(clang-target) -E +LD$(sm) := $(ccache-cmd)$(OPTEE_CLANG_COMPILER_PATH)ld.lld + +AR$(sm) := $(ccache-cmd)$(OPTEE_CLANG_COMPILER_PATH)llvm-ar +NM$(sm) := $(OPTEE_CLANG_COMPILER_PATH)llvm-nm +OBJCOPY$(sm) := $(OPTEE_CLANG_COMPILER_PATH)llvm-objcopy +OBJDUMP$(sm) := $(OPTEE_CLANG_COMPILER_PATH)llvm-objdump +READELF$(sm) := $(OPTEE_CLANG_COMPILER_PATH)llvm-readelf + +nostdinc$(sm) := -nostdinc -isystem $(shell $(CC$(sm)) \ + -print-file-name=include 2> /dev/null) + +comp-cflags-warns-clang := -Wno-language-extension-token \ + -Wno-gnu-zero-variadic-macro-arguments + +# Note, use the compiler runtime library (libclang_rt.builtins.*.a) instead of +# libgcc for clang +libgcc$(sm) := $(shell $(CC$(sm)) $(CFLAGS$(arch-bits-$(sm))) \ + -rtlib=compiler-rt -print-libgcc-file-name 2> /dev/null) + +# Core ASLR relies on the executable being ready to run from its preferred load +# address, because some symbols are used before the MMU is enabled and the +# relocations are applied. +ldflag-apply-dynamic-relocs := --apply-dynamic-relocs + +# Define these to something to discover accidental use +CC := false +CXX := false +CPP := false +LD := false +AR := false +NM := false +OBJCOPY := false +OBJDUMP := false +READELF := false +nostdinc := --bad-nostdinc-variable +libgcc := --bad-libgcc-variable + diff --git a/optee_os/mk/cleandirs.mk b/optee_os/mk/cleandirs.mk new file mode 100644 index 0000000..6c42b53 --- /dev/null +++ b/optee_os/mk/cleandirs.mk @@ -0,0 +1,42 @@ +# Cleaning directories generated during a previous build, +# a failed previous build or even no previous build. +# Track build directories through 'cleanfiles'. + +define _enum-parent-dirs +$(if $(1),$(1) $(if $(filter / ./,$(dir $(1))),,$(call enum-parent-dirs,$(dir $(1)))),) +endef + +define enum-parent-dirs +$(call _enum-parent-dirs,$(patsubst %/,%,$(1))) +endef + +define _reverse +$(if $(1),$(call _reverse,$(wordlist 2,$(words $(1)),$(1)))) $(firstword $(1)) +endef + +# Returns the list of all existing output directories up to $(O) including all +# intermediate levels, in depth first order so that rmdir can process them in +# order. May return an empty string. +# Example: if cleanfiles is "foo/a/file1 foo/b/c/d/file2" and O=foo, this will +# return "foo/b/c/d foo/b/c foo/b foo/a" (assuming all exist). +define cleandirs-for-rmdir +$(eval _O:=$(if $(O),$(O),.))$(wildcard $(addprefix $(_O)/,$(call _reverse, + $(sort $(foreach d,$(patsubst $(_O)/%,%,$(dir $(cleanfiles))), + $(call enum-parent-dirs,$(d))))))) +endef + +RMDIR := rmdir --ignore-fail-on-non-empty + +# Remove files with "rm -f". +# Split (possibly huge) file list into more manageable lines +# (200 files at a time), to minimize the odds of having: +# "/bin/bash: Argument list too long" +define do-rm-f + $(call _do-rm-f, $(wordlist 1, 200, $(1))) \ + $(eval _tail := $(wordlist 201, $(words $(1)), $(1))) + $(if $(_tail), $(call do-rm-f, $(_tail))) +endef + +define _do-rm-f + ${q}rm -f $1 +endef diff --git a/optee_os/mk/cleanvars.mk b/optee_os/mk/cleanvars.mk new file mode 100644 index 0000000..57ed631 --- /dev/null +++ b/optee_os/mk/cleanvars.mk @@ -0,0 +1,12 @@ +# Cleans all output variables from other make files to allow for a new start + +gen-srcs := +spec-srcs := +srcs := +objs := +libfiles := +libdirs := +libnames := +libdeps := +sm-$(sm) := +incdirs := diff --git a/optee_os/mk/compile.mk b/optee_os/mk/compile.mk new file mode 100644 index 0000000..b3d807b --- /dev/null +++ b/optee_os/mk/compile.mk @@ -0,0 +1,308 @@ +# Input +# +# The output from mk/sub.mk +# base-prefix +# conf-file [optional] if set, all objects will depend on $(conf-file) +# additional-compile-deps [optional] additional dependencies +# +# Output +# +# set objs +# update cleanfiles +# +# Generates explicit rules for all objs + +objs := + +# Disable all builtin rules +.SUFFIXES: + +comp-cflags$(sm) = -std=gnu11 +comp-aflags$(sm) = +comp-cppflags$(sm) = + +ifeq ($(CFG_WERROR),y) +comp-cflags$(sm) += -Werror +endif +comp-cflags$(sm) += -fdiagnostics-show-option + +comp-cflags-warns-high = \ + -Wall -Wcast-align \ + -Werror-implicit-function-declaration -Wextra -Wfloat-equal \ + -Wformat-nonliteral -Wformat-security -Wformat=2 -Winit-self \ + -Wmissing-declarations -Wmissing-format-attribute \ + -Wmissing-include-dirs -Wmissing-noreturn \ + -Wmissing-prototypes -Wnested-externs -Wpointer-arith \ + -Wshadow -Wstrict-prototypes -Wswitch-default \ + -Wwrite-strings \ + -Wno-missing-field-initializers -Wno-format-zero-length \ + -Wno-c2x-extensions +comp-cflags-warns-high += $(call cc-option,-Wpacked-not-aligned) +comp-cflags-warns-high += $(call cc-option,-Waddress-of-packed-member) +ifeq ($(CFG_WARN_DECL_AFTER_STATEMENT),y) +comp-cflags-warns-high += $(call cc-option,-Wdeclaration-after-statement) +endif +comp-cflags-warns-medium = \ + -Wredundant-decls +comp-cflags-warns-low = \ + -Wold-style-definition -Wstrict-aliasing=2 \ + -Wundef + +comp-cflags-warns-1:= $(comp-cflags-warns-high) +comp-cflags-warns-2:= $(comp-cflags-warns-1) $(comp-cflags-warns-medium) +comp-cflags-warns-3:= $(comp-cflags-warns-2) $(comp-cflags-warns-low) + +WARNS ?= 3 + +comp-cflags$(sm) += $(comp-cflags-warns-$(WARNS)) \ + $(comp-cflags-warns-$(COMPILER_$(sm))) + +CHECK ?= sparse + +.PHONY: FORCE +.PHONY: FORCE-GENSRC$(sm) +FORCE: +FORCE-GENSRC$(sm): + + +define process_srcs +objs += $2 +comp-dep-$2 := $$(dir $2).$$(notdir $2).d +comp-cmd-file-$2:= $$(dir $2).$$(notdir $2).cmd +comp-sm-$2 := $(sm) +comp-lib-$2 := $(libname)-$(sm) + +cleanfiles := $$(cleanfiles) $$(comp-dep-$2) $$(comp-cmd-file-$2) $2 + +ifeq ($$(filter %.c,$1),$1) +comp-q-$2 := CC # one trailing space +comp-compiler-$2 := $$(CC$(sm)) +comp-flags-$2 = $$(filter-out $$(CFLAGS_REMOVE) $$(cflags-remove) \ + $$(cflags-remove-$$(comp-sm-$2)) \ + $$(cflags-remove-$2), \ + $$(CFLAGS$$(arch-bits-$$(comp-sm-$2))) $$(CFLAGS_WARNS) \ + $$(comp-cflags$$(comp-sm-$2)) $$(cflags$$(comp-sm-$2)) \ + $$(cflags-lib$$(comp-lib-$2)) $$(cflags-$2)) +ifeq ($C,1) +check-cmd-$2 = $(CHECK) $$(comp-cppflags-$2) $$< +echo-check-$2 := $(cmd-echo-silent) +echo-check-cmd-$2 = $(cmd-echo) $$(subst \",\\\",$$(check-cmd-$2)) +endif + +else ifeq ($$(filter %.S,$1),$1) +comp-q-$2 := AS # one trailing space +comp-compiler-$2 := $$(CC$(sm)) +comp-flags-$2 = $$(filter-out $$(AFLAGS_REMOVE) $$(aflags-remove) \ + $$(aflags-remove-$$(comp-sm-$2)) \ + $$(aflags-remove-$2), \ + $$(AFLAGS) $$(comp-aflags$$(comp-sm-$2)) \ + $$(aflags$$(comp-sm-$2)) $$(aflags-$2)) + +else ifeq ($$(filter %.cpp,$1),$1) +comp-q-$2 := CXX +comp-compiler-$2 := $$(CXX$(sm)) +comp-flags-$2 = $$(filter-out $$(CXXFLAGS_REMOVE) $$(cxxflags-remove) \ + $$(cxxflags-remove-$$(comp-sm-$2)) \ + $$(cxxflags-remove-$2), \ + $$(CXXFLAGS) $$(comp-cxxflags$$(comp-sm-$2)) \ + $$(cxxflags$$(comp-sm-$2)) $$(cxxflags-$2)) + +else +$$(error "Don't know what to do with $1") +endif + +comp-cppflags-$2 = $$(filter-out $$(CPPFLAGS_REMOVE) $$(cppflags-remove) \ + $$(cppflags-remove-$$(comp-sm-$2)) \ + $$(cppflags-remove-$2), \ + $$(nostdinc$$(comp-sm-$2)) $$(CPPFLAGS) \ + $$(addprefix -I,$$(incdirs$$(comp-sm-$2))) \ + $$(addprefix -I,$$(incdirs-lib$$(comp-lib-$2))) \ + $$(addprefix -I,$$(incdirs-$2)) \ + $$(cppflags$$(comp-sm-$2)) \ + $$(cppflags-lib$$(comp-lib-$2)) $$(cppflags-$2)) \ + -D__FILE_ID__=$$(subst -,_,$$(subst /,_,$$(subst .,_,$1))) + +comp-flags-$2 += -MD -MF $$(comp-dep-$2) -MT $$@ +comp-flags-$2 += $$(comp-cppflags-$2) + +comp-cmd-$2 = $$(comp-compiler-$2) $$(comp-flags-$2) -c $$< -o $$@ +comp-objcpy-cmd-$2 = $$(OBJCOPY$(sm)) \ + --rename-section .rodata=.rodata.$1 \ + --rename-section .rodata.str1.1=.rodata.str1.1.$1 \ + $2 + +# Assign defaults if unassigned +echo-check-$2 ?= true +echo-check-cmd-$2 ?= true +check-cmd-$2 ?= true + +-include $$(comp-cmd-file-$2) +-include $$(comp-dep-$2) + + +$2: $1 FORCE-GENSRC$(sm) +# Check if any prerequisites are newer than the target and +# check if command line has changed + $$(if $$(strip $$(filter-out FORCE-GENSRC$(sm), $$?) \ + $$(filter-out $$(comp-cmd-$2), $$(old-cmd-$2)) \ + $$(filter-out $$(old-cmd-$2), $$(comp-cmd-$2))), \ + @set -e ;\ + mkdir -p $$(dir $2) ;\ + $$(echo-check-$2) ' CHECK $$<' ;\ + $$(echo-check-cmd-$2) ;\ + $$(check-cmd-$2) ;\ + $(cmd-echo-silent) ' $$(comp-q-$2) $$@' ;\ + $(cmd-echo) $$(subst \",\\\",$$(comp-cmd-$2)) ;\ + $$(comp-cmd-$2) ;\ + $(cmd-echo) $$(comp-objcpy-cmd-$2) ;\ + $$(comp-objcpy-cmd-$2) ;\ + echo "old-cmd-$2 := $$(subst \",\\\",$$(comp-cmd-$2))" > \ + $$(comp-cmd-file-$2) ;\ + ) + +endef + +$(foreach f, $(srcs), $(eval $(call \ + process_srcs,$(f),$(out-dir)/$(base-prefix)$$(basename $f).o))) + +# Handle generated source files, that is, files that are compiled from out-dir +$(foreach f, $(gen-srcs), $(eval $(call process_srcs,$(f),$$(basename $f).o))) + +# Handle specified source files, that is, files that have a specified path +# but where the object file should go into a specified out directory +$(foreach f, $(spec-srcs), $(eval $(call \ + process_srcs,$(f),$(spec-out-dir)/$$(notdir $$(basename $f)).o))) + +$(objs): $(conf-file) $(additional-compile-deps) + +define _gen-asm-defines-file +# c-filename in $1 +# h-filename in $2 +# s-filename in $3 +# Dependencies in $4 + +FORCE-GENSRC$(sm): $(2) + +comp-dep-$3 := $$(dir $3)$$(notdir $3).d +comp-cmd-file-$3:= $$(dir $3)$$(notdir $3).cmd +comp-sm-$3 := $(sm) + +cleanfiles := $$(cleanfiles) $$(comp-dep-$3) $$(comp-cmd-file-$3) $3 $2 + +comp-flags-$3 = $$(filter-out $$(CFLAGS_REMOVE) $$(cflags-remove) \ + $$(cflags-remove-$$(comp-sm-$3)) \ + $$(cflags-remove-$3), \ + $$(CFLAGS) $$(CFLAGS_WARNS) \ + $$(comp-cflags$$(comp-sm-$3)) $$(cflags$$(comp-sm-$3)) \ + $$(cflags-lib$$(comp-lib-$3)) $$(cflags-$3)) + +comp-cppflags-$3 = $$(filter-out $$(CPPFLAGS_REMOVE) $$(cppflags-remove) \ + $$(cppflags-remove-$$(comp-sm-$3)) \ + $$(cppflags-remove-$3), \ + $$(nostdinc$$(comp-sm-$3)) $$(CPPFLAGS) \ + $$(addprefix -I,$$(incdirs$$(comp-sm-$3))) \ + $$(addprefix -I,$$(incdirs-lib$$(comp-lib-$3))) \ + $$(addprefix -I,$$(incdirs-$3)) \ + $$(cppflags$$(comp-sm-$3)) \ + $$(cppflags-lib$$(comp-lib-$3)) $$(cppflags-$3)) + +comp-flags-$3 += -MD -MF $$(comp-dep-$3) -MT $$@ +comp-flags-$3 += $$(comp-cppflags-$3) + +comp-cmd-$3 = $$(CC$(sm)) $$(comp-flags-$3) -fverbose-asm -S $$< -o $$@ + + +-include $$(comp-cmd-file-$3) +-include $$(comp-dep-$3) + +$3: $1 $(conf-file) $(4) FORCE +# Check if any prerequisites are newer than the target and +# check if command line has changed + $$(if $$(strip $$(filter-out FORCE, $$?) \ + $$(filter-out $$(comp-cmd-$3), $$(old-cmd-$3)) \ + $$(filter-out $$(old-cmd-$3), $$(comp-cmd-$3))), \ + @set -e ;\ + mkdir -p $$(dir $2) $$(dir $3) ;\ + $(cmd-echo-silent) ' CC $$@'; \ + $(cmd-echo) $$(subst \",\\\",$$(comp-cmd-$3)) ;\ + $$(comp-cmd-$3) ;\ + echo "old-cmd-$3 := $$(subst \",\\\",$$(comp-cmd-$3))" > \ + $$(comp-cmd-file-$3) ;\ + ) + +guard-$2 := $$(subst -,_,$$(subst .,_,$$(subst /,_,$$(subst +,_,$2)))) + +$(2): $(3) + $(q)set -e; \ + $(cmd-echo-silent) ' CHK $$@'; \ + mkdir -p $$(dir $$@); \ + echo "#ifndef $$(guard-$2)" >$$@.tmp; \ + echo "#define $$(guard-$2)" >>$$@.tmp; \ + sed -ne 's|^.*==>\([^ ]*\) [\$$$$#]*\([-0-9]*\) \([^@/]*\).*|#define \1\t\2\t/* \3*/|p' \ + < $$< >>$$@.tmp; \ + echo "#endif" >>$$@.tmp; \ + $$(call mv-if-changed,$$@.tmp,$$@) + +endef + +define gen-asm-defines-file +$(call _gen-asm-defines-file,$1,$2,$(dir $2).$(notdir $(2:.h=.s)),$(asm-defines-$(notdir $(1))-deps)) +endef + +$(foreach f,$(asm-defines-files),$(eval $(call gen-asm-defines-file,$(f),$(out-dir)/$(sm)/include/generated/$(basename $(notdir $(f))).h))) + +# Device tree source file compilation +DTC := dtc +DTC_FLAGS += -I dts -O dtb +DTC_FLAGS += -Wno-unit_address_vs_reg + +define gen-dtb-file +# dts file path/name in $1 +# dtb file path/name in $2 + +dtb-basename-$2 := $$(basename $$(notdir $2)) +dtb-predts-$2 := $$(dir $2)$$(dtb-basename-$2).pre.dts +dtb-predep-$2 := $$(dir $2).$$(dtb-basename-$2).pre.dts.d +dtb-dep-$2 := $$(dir $2).$$(notdir $2).d +dtb-cmd-file-$2 := $$(dir $2).$$(notdir $2).cmd + +cleanfiles := $$(cleanfiles) $2 \ + $$(dtb-predts-$2) $$(dtb-predep-$2) \ + $$(dtb-dep-$2) $$(dtb-cmd-file-$2) + +dtb-cppflags-$2 := -Icore/include/ -x assembler-with-cpp -Ulinux -Uunix \ + -E -ffreestanding $$(CPPFLAGS) \ + -MD -MF $$(dtb-predep-$2) -MT $2 + +dtb-dtcflags-$2 := $$(DTC_FLAGS) -d $$(dtb-dep-$2) + +-include $$(dtb-dep-$2) +-include $$(dtb-predep-$2) +-include $$(dtb-cmd-file-$2) + +dtb-precmd-$2 = $$(CPP$(sm)) $$(dtb-cppflags-$2) -o $$(dtb-predts-$2) $$< +dtb-cmd-$2 = $$(DTC) $$(dtb-dtcflags-$2) -o $$@ $$(dtb-predts-$2) + +$2: $1 FORCE +# Check if any prerequisites are newer than the target and +# check if command line has changed + $$(if $$(strip $$(filter-out FORCE, $$?) \ + $$(filter-out $$(dtb-precmd-$2), $$(dtb-old-precmd-$2)) \ + $$(filter-out $$(dtb-old-precmd-$2), $$(dtb-precmd-$2)) \ + $$(filter-out $$(dtb-cmd-$2), $$(dtb-old-cmd-$2)) \ + $$(filter-out $$(dtb-old-cmd-$2), $$(dtb-cmd-$2))), \ + @set -e; \ + mkdir -p $$(dir $2); \ + $(cmd-echo-silent) ' CPP $$(dtb-predts-$2)'; \ + $$(dtb-precmd-$2); \ + $(cmd-echo-silent) ' DTC $$@'; \ + $$(dtb-cmd-$2); \ + echo "dtb-old-precmd-$2 := $$(subst \",\\\",$$(dtb-precmd-$2))" > \ + $$(dtb-cmd-file-$2) ;\ + echo "dtb-old-cmd-$2 := $$(subst \",\\\",$$(dtb-cmd-$2))" >> \ + $$(dtb-cmd-file-$2) ;\ + ) + +endef + +additional-compile-deps := diff --git a/optee_os/mk/config.mk b/optee_os/mk/config.mk new file mode 100644 index 0000000..a9a68e3 --- /dev/null +++ b/optee_os/mk/config.mk @@ -0,0 +1,1065 @@ +# Default configuration values for OP-TEE core (all platforms). +# +# Platform-specific overrides are in core/arch/arm32/plat-*/conf.mk. +# Some subsystem-specific defaults are not here but rather in */sub.mk. +# +# Configuration values may be assigned from multiple sources. +# From higher to lower priority: +# +# 1. Make arguments ('make CFG_FOO=bar...') +# 2. The file specified by $(CFG_OPTEE_CONFIG) (if defined) +# 3. The environment ('CFG_FOO=bar make...') +# 4. The platform-specific configuration file: core/arch/arm32/plat-*/conf.mk +# 5. This file +# 6. Subsystem-specific makefiles (*/sub.mk) +# +# Actual values used during the build are output to $(out-dir)/conf.mk +# (CFG_* variables only). + +# Cross-compiler prefix and suffix +CROSS_COMPILE ?= arm-linux-gnueabihf- +CROSS_COMPILE32 ?= $(CROSS_COMPILE) +CROSS_COMPILE64 ?= aarch64-linux-gnu- +COMPILER ?= gcc + +# For convenience +ifdef CFLAGS +CFLAGS32 ?= $(CFLAGS) +CFLAGS64 ?= $(CFLAGS) +endif + +# Compiler warning level. +# Supported values: undefined, 1, 2 and 3. 3 gives more warnings. +WARNS ?= 3 + +# Path to the Python interpreter used by the build system. +# This variable is set to the default python3 interpreter in the user's +# path. But build environments that require more explicit control can +# set the path to a specific interpreter through this variable. +PYTHON3 ?= python3 + +# Define DEBUG=1 to compile without optimization (forces -O0) +# DEBUG=1 +ifeq ($(DEBUG),1) +# For backwards compatibility +$(call force,CFG_CC_OPT_LEVEL,0) +$(call force,CFG_DEBUG_INFO,y) +endif + +# CFG_CC_OPT_LEVEL sets compiler optimization level passed with -O directive. +# Optimize for size by default, usually gives good performance too. +CFG_CC_OPT_LEVEL ?= s + +# Enabling CFG_DEBUG_INFO makes debug information embedded in core. +CFG_DEBUG_INFO ?= y + +# If y, enable debug features of the TEE core (assertions and lock checks +# are enabled, panic and assert messages are more verbose, data and prefetch +# aborts show a stack dump). When disabled, the NDEBUG directive is defined +# so assertions are disabled. +CFG_TEE_CORE_DEBUG ?= y + +# Log levels for the TEE core. Defines which core messages are displayed +# on the secure console. Disabling core log (level set to 0) also disables +# logs from the TAs. +# 0: none +# 1: error +# 2: error + info +# 3: error + info + debug +# 4: error + info + debug + flow +CFG_TEE_CORE_LOG_LEVEL ?= 2 + +# TA log level +# If user-mode library libutils.a is built with CFG_TEE_TA_LOG_LEVEL=0, +# TA tracing is disabled regardless of the value of CFG_TEE_TA_LOG_LEVEL +# when the TA is built. +CFG_TEE_TA_LOG_LEVEL ?= 1 + +# TA enablement +# When defined to "y", TA traces are output according to +# CFG_TEE_TA_LOG_LEVEL. Otherwise, they are not output at all +CFG_TEE_CORE_TA_TRACE ?= y + +# If y, enable the memory leak detection feature in the bget memory allocator. +# When this feature is enabled, calling mdbg_check(1) will print a list of all +# the currently allocated buffers and the location of the allocation (file and +# line number). +# Note: make sure the log level is high enough for the messages to show up on +# the secure console! For instance: +# - To debug user-mode (TA) allocations: build OP-TEE *and* the TA with: +# $ make CFG_TEE_TA_MALLOC_DEBUG=y CFG_TEE_TA_LOG_LEVEL=3 +# - To debug TEE core allocations: build OP-TEE with: +# $ make CFG_TEE_CORE_MALLOC_DEBUG=y CFG_TEE_CORE_LOG_LEVEL=3 +CFG_TEE_CORE_MALLOC_DEBUG ?= n +CFG_TEE_TA_MALLOC_DEBUG ?= n +# Prints an error message and dumps the stack on failed memory allocations +# using malloc() and friends. +CFG_CORE_DUMP_OOM ?= $(CFG_TEE_CORE_MALLOC_DEBUG) + +# Mask to select which messages are prefixed with long debugging information +# (severity, core ID, thread ID, component name, function name, line number) +# based on the message level. If BIT(level) is set, the long prefix is shown. +# Otherwise a short prefix is used (severity and component name only). +# Levels: 0=none 1=error 2=info 3=debug 4=flow +CFG_MSG_LONG_PREFIX_MASK ?= 0x1a + +# Number of threads +CFG_NUM_THREADS ?= 2 + +# API implementation version +CFG_TEE_API_VERSION ?= GPD-1.1-dev + +# Implementation description (implementation-dependent) +CFG_TEE_IMPL_DESCR ?= OPTEE + +# Should OPTEE_SMC_CALL_GET_OS_REVISION return a build identifier to Normal +# World? +CFG_OS_REV_REPORTS_GIT_SHA1 ?= y + +# The following values are not extracted from the "git describe" output because +# we might be outside of a Git environment, or the tree may have been cloned +# with limited depth not including any tag, so there is really no guarantee +# that TEE_IMPL_VERSION contains the major and minor revision numbers. +CFG_OPTEE_REVISION_MAJOR ?= 4 +CFG_OPTEE_REVISION_MINOR ?= 0 +CFG_OPTEE_REVISION_EXTRA ?= + +# Trusted OS implementation version +TEE_IMPL_VERSION ?= $(shell git describe --always --dirty=-dev 2>/dev/null || \ + echo Unknown_$(CFG_OPTEE_REVISION_MAJOR).$(CFG_OPTEE_REVISION_MINOR))$(CFG_OPTEE_REVISION_EXTRA) +ifeq ($(CFG_OS_REV_REPORTS_GIT_SHA1),y) +TEE_IMPL_GIT_SHA1 := 0x$(shell git rev-parse --short=8 HEAD 2>/dev/null || echo 0) +else +TEE_IMPL_GIT_SHA1 := 0x0 +endif + +# Trusted OS implementation manufacturer name +CFG_TEE_MANUFACTURER ?= LINARO + +# Trusted firmware version +CFG_TEE_FW_IMPL_VERSION ?= FW_IMPL_UNDEF + +# Trusted OS implementation manufacturer name +CFG_TEE_FW_MANUFACTURER ?= FW_MAN_UNDEF + +# Rich Execution Environment (REE) file system support: normal world OS +# provides the actual storage. +# This is the default FS when enabled (i.e., the one used when +# TEE_STORAGE_PRIVATE is passed to the trusted storage API) +CFG_REE_FS ?= y + +# RPMB file system support +CFG_RPMB_FS ?= n + +# Enable roll-back protection of REE file system using RPMB. +# Roll-back protection only works if CFG_RPMB_FS = y. +CFG_REE_FS_INTEGRITY_RPMB ?= $(CFG_RPMB_FS) +$(eval $(call cfg-depends-all,CFG_REE_FS_INTEGRITY_RPMB,CFG_RPMB_FS)) + +# Device identifier used when CFG_RPMB_FS = y. +# The exact meaning of this value is platform-dependent. On Linux, the +# tee-supplicant process will open /dev/mmcblkrpmb +CFG_RPMB_FS_DEV_ID ?= 0 + +# This config variable determines the number of entries read in from RPMB at +# once whenever a function traverses the RPMB FS. Increasing the default value +# has the following consequences: +# - More memory required on heap. A single FAT entry currently has a size of +# 256 bytes. +# - Potentially significant speed-ups for RPMB I/O. Depending on how many +# entries a function needs to traverse, the number of time-consuming RPMB +# read-in operations can be reduced. +# Chosing a proper value is both platform- (available memory) and use-case- +# dependent (potential number of FAT fs entries), so overwrite in platform +# config files +CFG_RPMB_FS_RD_ENTRIES ?= 8 + +# Enables caching of FAT FS entries when set to a value greater than zero. +# When enabled, the cache stores the first 'CFG_RPMB_FS_CACHE_ENTRIES' FAT FS +# entries. The cache is populated when FAT FS entries are initially read in. +# When traversing the FAT FS entries, we read from the cache instead of reading +# in the entries from RPMB storage. Consequently, when a FAT FS entry is +# written, the cache is updated. In scenarios where an estimate of the number +# of FAT FS entries can be made, the cache may be specifically tailored to +# store all entries. The caching can improve RPMB I/O at the cost +# of additional memory. +# Without caching, we temporarily require +# CFG_RPMB_FS_RD_ENTRIES*sizeof(struct rpmb_fat_entry) bytes of heap memory +# while traversing the FAT FS (e.g. in read_fat). +# For example 8*256 bytes = 2kB while in read_fat. +# With caching, we constantly require up to +# CFG_RPMB_FS_CACHE_ENTRIES*sizeof(struct rpmb_fat_entry) bytes of heap memory +# depending on how many elements are in the cache, and additional temporary +# CFG_RPMB_FS_RD_ENTRIES*sizeof(struct rpmb_fat_entry) bytes of heap memory +# in case the cache is too small to hold all elements when traversing. +CFG_RPMB_FS_CACHE_ENTRIES ?= 0 + +# Print RPMB data frames sent to and received from the RPMB device +CFG_RPMB_FS_DEBUG_DATA ?= n + +# Clear RPMB content at cold boot +CFG_RPMB_RESET_FAT ?= n + +# Use a hard coded RPMB key instead of deriving it from the platform HUK +CFG_RPMB_TESTKEY ?= n + +# Enables RPMB key programming by the TEE, in case the RPMB partition has not +# been configured yet. +# !!! Security warning !!! +# Do *NOT* enable this in product builds, as doing so would allow the TEE to +# leak the RPMB key. +# This option is useful in the following situations: +# - Testing +# - RPMB key provisioning in a controlled environment (factory setup) +CFG_RPMB_WRITE_KEY ?= n + +_CFG_WITH_SECURE_STORAGE := $(call cfg-one-enabled,CFG_REE_FS CFG_RPMB_FS) + +# Signing key for OP-TEE TA's +# When performing external HSM signing for TA's TA_SIGN_KEY can be set to dummy +# key and then set TA_PUBLIC_KEY to match public key from the HSM. +# TA_PUBLIC_KEY's public key will be embedded into OP-TEE OS. +TA_SIGN_KEY ?= keys/default_ta.pem +TA_PUBLIC_KEY ?= $(TA_SIGN_KEY) + +# Subkeys is a complement to the normal TA_SIGN_KEY where a subkey is used +# to verify a TA instead. To sign a TA using a previously prepared subkey +# two new options are added, TA_SUBKEY_ARGS and TA_SUBKEY_DEPS. It is +# typically used by assigning the following in the TA Makefile: +# BINARY = +# TA_SIGN_KEY = subkey.pem +# TA_SUBKEY_ARGS = --subkey subkey.bin --name subkey_ta +# TA_SUBKEY_DEPS = subkey.bin +# See the documentation for more details on subkeys. + +# Include lib/libutils/isoc in the build? Most platforms need this, but some +# may not because they obtain the isoc functions from elsewhere +CFG_LIBUTILS_WITH_ISOC ?= y + +# Enables floating point support for user TAs +# ARM32: EABI defines both a soft-float ABI and a hard-float ABI, +# hard-float is basically a super set of soft-float. Hard-float +# requires all the support routines provided for soft-float, but the +# compiler may choose to optimize to not use some of them and use +# the floating-point registers instead. +# ARM64: EABI doesn't define a soft-float ABI, everything is hard-float (or +# nothing with ` -mgeneral-regs-only`) +# With CFG_TA_FLOAT_SUPPORT enabled TA code is free use floating point types +CFG_TA_FLOAT_SUPPORT ?= y + +# Stack unwinding: print a stack dump to the console on core or TA abort, or +# when a TA panics. +# If CFG_UNWIND is enabled, both the kernel and user mode call stacks can be +# unwound (not paged TAs, however). +# Note that 32-bit ARM code needs unwind tables for this to work, so enabling +# this option will increase the size of the 32-bit TEE binary by a few KB. +# Similarly, TAs have to be compiled with -funwind-tables (default when the +# option is set) otherwise they can't be unwound. +# Warning: since the unwind sequence for user-mode (TA) code is implemented in +# the privileged layer of OP-TEE, enabling this feature will weaken the +# user/kernel isolation. Therefore it should be disabled in release builds. +ifeq ($(CFG_TEE_CORE_DEBUG),y) +CFG_UNWIND ?= y +endif + +# Enable support for dynamically loaded user TAs +CFG_WITH_USER_TA ?= y + +# Build user TAs included in this source tree +CFG_BUILD_IN_TREE_TA ?= y + +# Choosing the architecture(s) of user-mode libraries (used by TAs) +# +# Platforms may define a list of supported architectures for user-mode code +# by setting $(supported-ta-targets). Valid values are "ta_arm32", "ta_arm64", +# "ta_arm32 ta_arm64" and "ta_arm64 ta_arm32". +# $(supported-ta-targets) defaults to "ta_arm32" when the TEE core is 32-bits, +# and "ta_arm32 ta_arm64" when it is 64-bits (that is, when CFG_ARM64_core=y). +# The first entry in $(supported-ta-targets) has a special role, see +# CFG_USER_TA_TARGET_ below. +# +# CFG_USER_TA_TARGETS may be defined to restrict $(supported-ta-targets) or +# change the order of the values. +# +# The list of TA architectures is ultimately stored in $(ta-targets). + +# CFG_USER_TA_TARGET_ (for example, CFG_USER_TA_TARGET_avb), if +# defined, selects the unique TA architecture mode for building the in-tree TA +# . Can be either ta_arm32 or ta_arm64. +# By default, in-tree TAs are built using the first architecture specified in +# $(ta-targets). + +# Address Space Layout Randomization for user-mode Trusted Applications +# +# When this flag is enabled, the ELF loader will introduce a random offset +# when mapping the application in user space. ASLR makes the exploitation of +# memory corruption vulnerabilities more difficult. +CFG_TA_ASLR ?= y + +# How much ASLR may shift the base address (in pages). The base address is +# randomly shifted by an integer number of pages comprised between these two +# values. Bigger ranges are more secure because they make the addresses harder +# to guess at the expense of using more memory for the page tables. +CFG_TA_ASLR_MIN_OFFSET_PAGES ?= 0 +CFG_TA_ASLR_MAX_OFFSET_PAGES ?= 128 + +# Address Space Layout Randomization for TEE Core +# +# When this flag is enabled, the early init code will introduce a random +# offset when mapping TEE Core. ASLR makes the exploitation of memory +# corruption vulnerabilities more difficult. +CFG_CORE_ASLR ?= y + +# Stack Protection for TEE Core +# This flag enables the compiler stack protection mechanisms -fstack-protector. +# It will check the stack canary value before returning from a function to +# prevent buffer overflow attacks. Stack protector canary logic will be added +# for vulnerable functions that contain: +# - A character array larger than 8 bytes. +# - An 8-bit integer array larger than 8 bytes. +# - A call to alloca() with either a variable size or a constant size bigger +# than 8 bytes. +CFG_CORE_STACK_PROTECTOR ?= n +# This enable stack protector flag -fstack-protector-strong. Stack protector +# canary logic will be added for vulnerable functions that contain: +# - An array of any size and type. +# - A call to alloca(). +# - A local variable that has its address taken. +CFG_CORE_STACK_PROTECTOR_STRONG ?= y +# This enable stack protector flag -fstack-protector-all. Stack protector canary +# logic will be added to all functions regardless of their vulnerability. +CFG_CORE_STACK_PROTECTOR_ALL ?= n +# Stack Protection for TA +CFG_TA_STACK_PROTECTOR ?= n +CFG_TA_STACK_PROTECTOR_STRONG ?= y +CFG_TA_STACK_PROTECTOR_ALL ?= n + +_CFG_CORE_STACK_PROTECTOR := $(call cfg-one-enabled, CFG_CORE_STACK_PROTECTOR \ + CFG_CORE_STACK_PROTECTOR_STRONG \ + CFG_CORE_STACK_PROTECTOR_ALL) +_CFG_TA_STACK_PROTECTOR := $(call cfg-one-enabled, CFG_TA_STACK_PROTECTOR \ + CFG_TA_STACK_PROTECTOR_STRONG \ + CFG_TA_STACK_PROTECTOR_ALL) + +# Load user TAs from the REE filesystem via tee-supplicant +CFG_REE_FS_TA ?= y + +# Pre-authentication of TA binaries loaded from the REE filesystem +# +# - If CFG_REE_FS_TA_BUFFERED=y: load TA binary into a temporary buffer in the +# "Secure DDR" pool, check the signature, then process the file only if it is +# valid. +# - If disabled: hash the binaries as they are being processed and verify the +# signature as a last step. +CFG_REE_FS_TA_BUFFERED ?= n +$(eval $(call cfg-depends-all,CFG_REE_FS_TA_BUFFERED,CFG_REE_FS_TA)) + +# When CFG_REE_FS=y: +# Allow secure storage in the REE FS to be entirely deleted without causing +# anti-rollback errors. That is, rm /data/tee/dirf.db or rm -rf /data/tee (or +# whatever path is configured in tee-supplicant as CFG_TEE_FS_PARENT_PATH) +# can be used to reset the secure storage to a clean, empty state. +# Typically used for testing only since it weakens storage security. +CFG_REE_FS_ALLOW_RESET ?= n + +# Support for loading user TAs from a special section in the TEE binary. +# Such TAs are available even before tee-supplicant is available (hence their +# name), but note that many services exported to TAs may need tee-supplicant, +# so early use is limited to a subset of the TEE Internal Core API (crypto...) +# To use this feature, set EARLY_TA_PATHS to the paths to one or more TA ELF +# file(s). For example: +# $ make ... \ +# EARLY_TA_PATHS="path/to/8aaaf200-2450-11e4-abe2-0002a5d5c51b.stripped.elf \ +# path/to/cb3e5ba0-adf1-11e0-998b-0002a5d5c51b.stripped.elf" +# Typical build steps: +# $ make ta_dev_kit CFG_EARLY_TA=y # Create the dev kit (user mode libraries, +# # headers, makefiles), ready to build TAs. +# # CFG_EARLY_TA=y is optional, it prevents +# # later library recompilations. +# +# $ make EARLY_TA_PATHS= # Build OP-TEE and embbed the TA(s) +# +# Another option is CFG_IN_TREE_EARLY_TAS which is used to point at +# in-tree TAs. CFG_IN_TREE_EARLY_TAS is formatted as: +# / +# for instance avb/023f8f1a-292a-432b-8fc4-de8471358067 +ifneq ($(EARLY_TA_PATHS)$(CFG_IN_TREE_EARLY_TAS),) +$(call force,CFG_EARLY_TA,y) +else +CFG_EARLY_TA ?= n +endif + +ifeq ($(CFG_EARLY_TA),y) +$(call force,CFG_EMBEDDED_TS,y) +endif + +ifneq ($(SP_PATHS),) +$(call force,CFG_EMBEDDED_TS,y) +else +CFG_SECURE_PARTITION ?= n +endif + +ifeq ($(CFG_SECURE_PARTITION),y) +$(call force,CFG_EMBEDDED_TS,y) +endif + +ifeq ($(CFG_EMBEDDED_TS),y) +$(call force,CFG_ZLIB,y) +endif + +# By default the early TAs are compressed in the TEE binary, it is possible to +# not compress them with CFG_EARLY_TA_COMPRESS=n +CFG_EARLY_TA_COMPRESS ?= y + +# Enable paging, requires SRAM, can't be enabled by default +CFG_WITH_PAGER ?= n + +# Use the pager for user TAs +CFG_PAGED_USER_TA ?= $(CFG_WITH_PAGER) + +# If paging of user TAs, that is, R/W paging default to enable paging of +# TAG and IV in order to reduce heap usage. +CFG_CORE_PAGE_TAG_AND_IV ?= $(CFG_PAGED_USER_TA) + +# Runtime lock dependency checker: ensures that a proper locking hierarchy is +# used in the TEE core when acquiring and releasing mutexes. Any violation will +# cause a panic as soon as the invalid locking condition is detected. If +# CFG_UNWIND and CFG_LOCKDEP_RECORD_STACK are both enabled, the algorithm +# records the call stacks when locks are taken, and prints them when a +# potential deadlock is found. +# Expect a significant performance impact when enabling this. +CFG_LOCKDEP ?= n +CFG_LOCKDEP_RECORD_STACK ?= y + +# BestFit algorithm in bget reduces the fragmentation of the heap when running +# with the pager enabled or lockdep +CFG_CORE_BGET_BESTFIT ?= $(call cfg-one-enabled, CFG_WITH_PAGER CFG_LOCKDEP) + +# Enable support for detected undefined behavior in C +# Uses a lot of memory, can't be enabled by default +CFG_CORE_SANITIZE_UNDEFINED ?= n + +# Enable Kernel Address sanitizer, has a huge performance impact, uses a +# lot of memory and need platform specific adaptations, can't be enabled by +# default +CFG_CORE_SANITIZE_KADDRESS ?= n + +ifeq (y-y,$(CFG_CORE_SANITIZE_KADDRESS)-$(CFG_CORE_ASLR)) +$(error CFG_CORE_SANITIZE_KADDRESS and CFG_CORE_ASLR are not compatible) +endif + +# Add stack guards before/after stacks and periodically check them +CFG_WITH_STACK_CANARIES ?= y + +# Use compiler instrumentation to troubleshoot stack overflows. +# When enabled, most C functions check the stack pointer against the current +# stack limits on entry and panic immediately if it is out of range. +CFG_CORE_DEBUG_CHECK_STACKS ?= n + +# Use when the default stack allocations are not sufficient. +CFG_STACK_THREAD_EXTRA ?= 0 +CFG_STACK_TMP_EXTRA ?= 0 + +# Device Tree support +# +# When CFG_DT is enabled core embeds the FDT library (libfdt) allowing +# device tree blob (DTB) parsing from the core. +# +# When CFG_DT is enabled, the TEE _start function expects to find +# the address of a DTB in register X2/R2 provided by the early boot stage +# or value 0 if boot stage provides no DTB. +# +# When CFG_EXTERNAL_DT is enabled, the external device tree ABI is implemented +# and the external device tree is expected to be used/modified. Its value +# defaults to CFG_DT. +# +# When CFG_MAP_EXT_DT_SECURE is enabled the external device tree is expected to +# be in the secure memory. +# +# When CFG_EMBED_DTB is enabled, CFG_EMBED_DTB_SOURCE_FILE shall define the +# relative path of a DTS file located in core/arch/$(ARCH)/dts. +# The DTS file is compiled into a DTB file which content is embedded in a +# read-only section of the core. +ifneq ($(strip $(CFG_EMBED_DTB_SOURCE_FILE)),) +CFG_EMBED_DTB ?= y +endif +ifeq ($(filter y,$(CFG_EMBED_DTB) $(CFG_CORE_SEL1_SPMC) $(CFG_CORE_SEL2_SPMC) \ + $(CFG_CORE_EL3_SPMC)),y) +$(call force,CFG_DT,y) +endif +CFG_EMBED_DTB ?= n +CFG_DT ?= n +CFG_EXTERNAL_DT ?= $(CFG_DT) +CFG_MAP_EXT_DT_SECURE ?= n +ifeq ($(CFG_MAP_EXT_DT_SECURE),y) +$(call force,CFG_DT,y) +endif + +# Maximum size of the Device Tree Blob, has to be large enough to allow +# editing of the supplied DTB. +CFG_DTB_MAX_SIZE ?= 0x10000 + +# Maximum size of the init info data passed to Secure Partitions. +CFG_SP_INIT_INFO_MAX_SIZE ?= 0x1000 + +# Device Tree Overlay support. +# CFG_EXTERNAL_DTB_OVERLAY allows to append a DTB overlay into an existing +# external DTB. The overlay is created when no valid DTB overlay is found. +# CFG_GENERATE_DTB_OVERLAY allows to create a DTB overlay at external +# DTB location. +# External DTB location (physical address) is provided either by boot +# argument arg2 or from CFG_DT_ADDR if defined. +# A subsequent boot stage can then merge the generated overlay DTB into a main +# DTB using the standard fdt_overlay_apply() method. +CFG_EXTERNAL_DTB_OVERLAY ?= n +CFG_GENERATE_DTB_OVERLAY ?= n + +ifeq (y-y,$(CFG_EXTERNAL_DTB_OVERLAY)-$(CFG_GENERATE_DTB_OVERLAY)) +$(error CFG_EXTERNAL_DTB_OVERLAY and CFG_GENERATE_DTB_OVERLAY are exclusive) +endif +_CFG_USE_DTB_OVERLAY := $(call cfg-one-enabled,CFG_EXTERNAL_DTB_OVERLAY \ + CFG_GENERATE_DTB_OVERLAY) + +# All embedded tests are supposed to be disabled by default, this flag +# is used to control the default value of all other embedded tests +CFG_ENABLE_EMBEDDED_TESTS ?= n + +# Enable core self tests and related pseudo TAs +CFG_TEE_CORE_EMBED_INTERNAL_TESTS ?= $(CFG_ENABLE_EMBEDDED_TESTS) + +# Compiles bget_main_test() to be called from a test TA +CFG_TA_BGET_TEST ?= $(CFG_ENABLE_EMBEDDED_TESTS) + +# CFG_DT_DRIVER_EMBEDDED_TEST when enabled embedded DT driver probing tests. +# This also requires embedding a DTB with expected content. +# Default disable CFG_DRIVERS_CLK_EARLY_PROBE to probe clocks as other drivers. +# A probe deferral test mandates CFG_DRIVERS_DT_RECURSIVE_PROBE=n. +CFG_DT_DRIVER_EMBEDDED_TEST ?= n +ifeq ($(CFG_DT_DRIVER_EMBEDDED_TEST),y) +CFG_DRIVERS_CLK ?= y +CFG_DRIVERS_GPIO ?= y +CFG_DRIVERS_RSTCTRL ?= y +CFG_DRIVERS_CLK_EARLY_PROBE ?= n +$(call force,CFG_DRIVERS_DT_RECURSIVE_PROBE,n,Mandated by CFG_DT_DRIVER_EMBEDDED_TEST) +endif + +# CFG_DRIVERS_DT_RECURSIVE_PROBE when enabled forces a recursive subnode +# parsing in the embedded DTB for driver probing. The alternative is +# an exploration based on compatible drivers found. It is default disabled. +CFG_DRIVERS_DT_RECURSIVE_PROBE ?= n + +# This option enables OP-TEE to respond to SMP boot request: the Rich OS +# issues this to request OP-TEE to release secondaries cores out of reset, +# with specific core number and non-secure entry address. +CFG_BOOT_SECONDARY_REQUEST ?= n + +# Default heap size for Core, 64 kB +CFG_CORE_HEAP_SIZE ?= 65536 + +# Default size of nexus heap. 16 kB. Used only if CFG_NS_VIRTUALIZATION +# is enabled +CFG_CORE_NEX_HEAP_SIZE ?= 16384 + +# TA profiling. +# When this option is enabled, OP-TEE can execute Trusted Applications +# instrumented with GCC's -pg flag and will output profiling information +# in gmon.out format to /tmp/gmon-.out (path is defined in +# tee-supplicant) +# Note: this does not work well with shared libraries at the moment for a +# couple of reasons: +# 1. The profiling code assumes a unique executable section in the TA VA space. +# 2. The code used to detect at run time if the TA is intrumented assumes that +# the TA is linked statically. +CFG_TA_GPROF_SUPPORT ?= n + +# TA function tracing. +# When this option is enabled, OP-TEE can execute Trusted Applications +# instrumented with GCC's -pg flag and will output function tracing +# information for all functions compiled with -pg to +# /tmp/ftrace-.out (path is defined in tee-supplicant). +CFG_FTRACE_SUPPORT ?= n + +# Core syscall function tracing. +# When this option is enabled, OP-TEE core is instrumented with GCC's +# -pg flag and will output syscall function graph in user TA ftrace +# buffer +CFG_SYSCALL_FTRACE ?= n +$(call cfg-depends-all,CFG_SYSCALL_FTRACE,CFG_FTRACE_SUPPORT) + +# Enable to compile user TA libraries with profiling (-pg). +# Depends on CFG_TA_GPROF_SUPPORT or CFG_FTRACE_SUPPORT. +CFG_ULIBS_MCOUNT ?= n +# Profiling/tracing of syscall wrapper (utee_*) +CFG_SYSCALL_WRAPPERS_MCOUNT ?= $(CFG_ULIBS_MCOUNT) + +ifeq (y,$(filter y,$(CFG_ULIBS_MCOUNT) $(CFG_SYSCALL_WRAPPERS_MCOUNT))) +ifeq (,$(filter y,$(CFG_TA_GPROF_SUPPORT) $(CFG_FTRACE_SUPPORT))) +$(error Cannot instrument user libraries if user mode profiling is disabled) +endif +endif + +# Build libutee, libutils, libmbedtls as shared libraries. +# - Static libraries are still generated when this is enabled, but TAs will use +# the shared libraries unless explicitly linked with the -static flag. +# - Shared libraries are made of two files: for example, libutee is +# libutee.so and 527f1a47-b92c-4a74-95bd-72f19f4a6f74.ta. The '.so' file +# is a totally standard shared object, and should be used to link against. +# The '.ta' file is a signed version of the '.so' and should be installed +# in the same way as TAs so that they can be found at runtime. +CFG_ULIBS_SHARED ?= n + +ifeq (y-y,$(CFG_TA_GPROF_SUPPORT)-$(CFG_ULIBS_SHARED)) +$(error CFG_TA_GPROF_SUPPORT and CFG_ULIBS_SHARED are currently incompatible) +endif + +# CFG_GP_SOCKETS +# Enable Global Platform Sockets support +CFG_GP_SOCKETS ?= y + +# Enable Secure Data Path support in OP-TEE core (TA may be invoked with +# invocation parameters referring to specific secure memories). +CFG_SECURE_DATA_PATH ?= n + +# Enable storage for TAs in secure storage, depends on CFG_REE_FS=y +# TA binaries are stored encrypted in the REE FS and are protected by +# metadata in secure storage. +CFG_SECSTOR_TA ?= $(call cfg-all-enabled,CFG_REE_FS CFG_WITH_USER_TA) +$(eval $(call cfg-depends-all,CFG_SECSTOR_TA,CFG_REE_FS CFG_WITH_USER_TA)) + +# Enable the pseudo TA that managages TA storage in secure storage +CFG_SECSTOR_TA_MGMT_PTA ?= $(call cfg-all-enabled,CFG_SECSTOR_TA) +$(eval $(call cfg-depends-all,CFG_SECSTOR_TA_MGMT_PTA,CFG_SECSTOR_TA)) + +# Enable the pseudo TA for misc. auxilary services, extending existing +# GlobalPlatform TEE Internal Core API (for example, re-seeding RNG entropy +# pool etc...) +CFG_SYSTEM_PTA ?= $(CFG_WITH_USER_TA) +$(eval $(call cfg-depends-all,CFG_SYSTEM_PTA,CFG_WITH_USER_TA)) + +# Enable the pseudo TA for enumeration of TEE based devices for the normal +# world OS. +CFG_DEVICE_ENUM_PTA ?= y + +# The attestation pseudo TA provides an interface to request measurements of +# a TA or the TEE binary. +CFG_ATTESTATION_PTA ?= n +$(eval $(call cfg-depends-all,CFG_ATTESTATION_PTA,_CFG_WITH_SECURE_STORAGE)) + +# RSA key size (in bits) for the attestation PTA. Must be at least 528 given +# other algorithm parameters (RSA PSS with SHA-256 and 32-byte salt), but +# note that such a low value is not secure. +# See https://tools.ietf.org/html/rfc8017#section-8.1.1 and +# https://tools.ietf.org/html/rfc8017#section-9.1.1 +# emLen >= hlen + sLen + 2 = 32 + 32 + 2 = 66 +# emLen = ceil((modBits - 1) / 8) => emLen is the key size in bytes +CFG_ATTESTATION_PTA_KEY_SIZE ?= 3072 + +# Define the number of cores per cluster used in calculating core position. +# The cluster number is shifted by this value and added to the core ID, +# so its value represents log2(cores/cluster). +# Default is 2**(2) = 4 cores per cluster. +CFG_CORE_CLUSTER_SHIFT ?= 2 + +# Define the number of threads per core used in calculating processing +# element's position. The core number is shifted by this value and added to +# the thread ID, so its value represents log2(threads/core). +# Default is 2**(0) = 1 threads per core. +CFG_CORE_THREAD_SHIFT ?= 0 + +# Enable support for dynamic shared memory (shared memory anywhere in +# non-secure memory). +CFG_CORE_DYN_SHM ?= y + +# Enable support for reserved shared memory (shared memory in a carved out +# memory area). +CFG_CORE_RESERVED_SHM ?= y + +# Enables support for larger physical addresses, that is, it will define +# paddr_t as a 64-bit type. +CFG_CORE_LARGE_PHYS_ADDR ?= n + +# Define the maximum size, in bits, for big numbers in the Internal Core API +# Arithmetical functions. This does *not* influence the key size that may be +# manipulated through the Cryptographic API. +# Set this to a lower value to reduce the TA memory footprint. +CFG_TA_BIGNUM_MAX_BITS ?= 2048 + +# Define the maximum size, in bits, for big numbers in the TEE core (privileged +# layer). +# This value is an upper limit for the key size in any cryptographic algorithm +# implemented by the TEE core. +# Set this to a lower value to reduce the memory footprint. +CFG_CORE_BIGNUM_MAX_BITS ?= 4096 + +# Not used since libmpa was removed. Force the values to catch build scripts +# that would set = n. +$(call force,CFG_TA_MBEDTLS_MPI,y) +$(call force,CFG_TA_MBEDTLS,y) + +# Compile the TA library mbedTLS with self test functions, the functions +# need to be called to test anything +CFG_TA_MBEDTLS_SELF_TEST ?= y + +# By default use tomcrypt as the main crypto lib providing an implementation +# for the API in +# CFG_CRYPTOLIB_NAME is used as libname and +# CFG_CRYPTOLIB_DIR is used as libdir when compiling the library +# +# It's also possible to configure to use mbedtls instead of tomcrypt. +# Then the variables should be assigned as "CFG_CRYPTOLIB_NAME=mbedtls" and +# "CFG_CRYPTOLIB_DIR=lib/libmbedtls" respectively. +CFG_CRYPTOLIB_NAME ?= tomcrypt +CFG_CRYPTOLIB_DIR ?= core/lib/libtomcrypt + +# Not used since libmpa was removed. Force the value to catch build scripts +# that would set = n. +$(call force,CFG_CORE_MBEDTLS_MPI,y) + +# Enable virtualization support. OP-TEE will not work without compatible +# hypervisor if this option is enabled. +CFG_NS_VIRTUALIZATION ?= $(CFG_VIRTUALIZATION) +CFG_NS_VIRTUALIZATION ?= n + +ifeq ($(CFG_NS_VIRTUALIZATION),y) +$(call force,CFG_CORE_RODATA_NOEXEC,y) +$(call force,CFG_CORE_RWDATA_NOEXEC,y) + +# Default number of virtual guests +CFG_VIRT_GUEST_COUNT ?= 2 +endif + +# Enables backwards compatible derivation of RPMB and SSK keys +CFG_CORE_HUK_SUBKEY_COMPAT ?= y + +# Use SoC specific tee_otp_get_die_id() implementation for SSK key generation. +# This option depends on CFG_CORE_HUK_SUBKEY_COMPAT=y. +CFG_CORE_HUK_SUBKEY_COMPAT_USE_OTP_DIE_ID ?= n + +# Compress and encode conf.mk into the TEE core, and show the encoded string on +# boot (with severity TRACE_INFO). +CFG_SHOW_CONF_ON_BOOT ?= n + +# Enables support for passing a TPM Event Log stored in secure memory +# to a TA or FF-A SP, so a TPM Service could use it to extend any measurement +# taken before the service was up and running. +CFG_CORE_TPM_EVENT_LOG ?= n + +# When enabled, CFG_SCMI_MSG_DRIVERS embeds SCMI message drivers in the core. +# Refer to the supported SCMI features embedded upon CFG_SCMI_MSG_* +# +# CFG_SCMI_MSG_CLOCK embeds SCMI clock protocol support. +# CFG_SCMI_MSG_RESET_DOMAIN embeds SCMI reset domain protocol support. +# CFG_SCMI_MSG_SMT embeds a SMT header in shared device memory buffers +# CFG_SCMI_MSG_VOLTAGE_DOMAIN embeds SCMI voltage domain protocol support. +# CFG_SCMI_MSG_SMT_FASTCALL_ENTRY embeds fastcall SMC entry with SMT memory +# CFG_SCMI_MSG_SMT_INTERRUPT_ENTRY embeds interrupt entry with SMT memory +# CFG_SCMI_MSG_SMT_THREAD_ENTRY embeds threaded entry with SMT memory +# CFG_SCMI_MSG_SHM_MSG embeds a MSG header in cached shared memory buffer +CFG_SCMI_MSG_DRIVERS ?= n +ifeq ($(CFG_SCMI_MSG_DRIVERS),y) +CFG_SCMI_MSG_CLOCK ?= n +CFG_SCMI_MSG_RESET_DOMAIN ?= n +CFG_SCMI_MSG_SHM_MSG ?= n +CFG_SCMI_MSG_SMT ?= n +CFG_SCMI_MSG_SMT_FASTCALL_ENTRY ?= n +CFG_SCMI_MSG_SMT_INTERRUPT_ENTRY ?= n +CFG_SCMI_MSG_SMT_THREAD_ENTRY ?= n +CFG_SCMI_MSG_THREAD_ENTRY ?= n +CFG_SCMI_MSG_VOLTAGE_DOMAIN ?= n +$(eval $(call cfg-depends-all,CFG_SCMI_MSG_SMT_FASTCALL_ENTRY,CFG_SCMI_MSG_SMT)) +$(eval $(call cfg-depends-all,CFG_SCMI_MSG_SMT_INTERRUPT_ENTRY,CFG_SCMI_MSG_SMT)) +$(eval $(call cfg-depends-one,CFG_SCMI_MSG_SMT_THREAD_ENTRY,CFG_SCMI_MSG_SMT CFG_SCMI_MSG_SHM_MSG)) +ifeq ($(CFG_SCMI_MSG_SMT),y) +_CFG_SCMI_PTA_SMT_HEADER := y +endif +ifeq ($(CFG_SCMI_MSG_SHM_MSG),y) +_CFG_SCMI_PTA_MSG_HEADER := y +endif +endif + +# CFG_SCMI_SCPFW, when enabled, embeds the reference SCMI server implementation +# from SCP-firmware package as an built-in SCMI stack in core. This +# configuration mandates target product identifier is configured with +# CFG_SCMI_SCPFW_PRODUCT and the SCP-firmware source tree path with +# CFG_SCP_FIRMWARE. +CFG_SCMI_SCPFW ?= n + +ifeq ($(CFG_SCMI_SCPFW),y) +$(call force,CFG_SCMI_PTA,y,Required by CFG_SCMI_SCPFW) +ifeq (,$(CFG_SCMI_SCPFW_PRODUCT)) +$(error CFG_SCMI_SCPFW=y requires CFG_SCMI_SCPFW_PRODUCT configuration) +endif +ifeq (,$(wildcard $(CFG_SCP_FIRMWARE)/CMakeLists.txt)) +$(error CFG_SCMI_SCPFW=y requires CFG_SCP_FIRMWARE configuration) +endif +endif #CFG_SCMI_SCPFW + +ifeq ($(CFG_SCMI_MSG_DRIVERS)-$(CFG_SCMI_SCPFW),y-y) +$(error CFG_SCMI_MSG_DRIVERS=y and CFG_SCMI_SCPFW=y are mutually exclusive) +endif + +# When enabled, CFG_SCMI_MSG_USE_CLK embeds SCMI clocks registering services for +# the platform SCMI server and implements the platform plat_scmi_clock_*() +# functions. +CFG_SCMI_MSG_USE_CLK ?= n +$(eval $(call cfg-depends-all,CFG_SCMI_MSG_USE_CLK,CFG_DRIVERS_CLK CFG_SCMI_MSG_DRIVERS)) + +# Enable SCMI PTA interface for REE SCMI agents +CFG_SCMI_PTA ?= n +ifeq ($(CFG_SCMI_PTA),y) +_CFG_SCMI_PTA_SMT_HEADER ?= n +_CFG_SCMI_PTA_MSG_HEADER ?= n +endif + +ifneq ($(CFG_STMM_PATH),) +$(call force,CFG_WITH_STMM_SP,y) +else +CFG_WITH_STMM_SP ?= n +endif +ifeq ($(CFG_WITH_STMM_SP),y) +$(call force,CFG_ZLIB,y) +endif + +# When enabled checks that buffers passed to the GP Internal Core API +# comply with the rules added as annotations as part of the definition of +# the API. For example preventing buffers in non-secure shared memory when +# not allowed. +CFG_TA_STRICT_ANNOTATION_CHECKS ?= y + +# When enabled accepts the DES key sizes excluding parity bits as in +# the GP Internal API Specification v1.0 +CFG_COMPAT_GP10_DES ?= y + +# Defines a limit for many levels TAs may call each others. +CFG_CORE_MAX_SYSCALL_RECURSION ?= 4 + +# Pseudo-TA to export hardware RNG output to Normal World +# RNG characteristics are platform specific +CFG_HWRNG_PTA ?= n +ifeq ($(CFG_HWRNG_PTA),y) +# Output rate of hw_get_random_bytes() in bytes per second, 0: not rate-limited +CFG_HWRNG_RATE ?= 0 +# Quality/entropy of hw_get_random_bytes() per 1024 bits of output data, in bits +ifeq (,$(CFG_HWRNG_QUALITY)) +$(error CFG_HWRNG_QUALITY not defined) +endif +endif + +# CFG_PREALLOC_RPC_CACHE, when enabled, makes core to preallocate +# shared memory for each secure thread. When disabled, RPC shared +# memory is released once the secure thread has completed is execution. +ifeq ($(CFG_WITH_PAGER),y) +CFG_PREALLOC_RPC_CACHE ?= n +endif +CFG_PREALLOC_RPC_CACHE ?= y + +# When enabled, CFG_DRIVERS_CLK embeds a clock framework in OP-TEE core. +# This clock framework allows to describe clock tree and provides functions to +# get and configure the clocks. +# CFG_DRIVERS_CLK_DT embeds devicetree clock parsing support +# CFG_DRIVERS_CLK_FIXED add support for "fixed-clock" compatible clocks +# CFG_DRIVERS_CLK_EARLY_PROBE makes clocks probed at early_init initcall level. +CFG_DRIVERS_CLK ?= n +CFG_DRIVERS_CLK_DT ?= $(call cfg-all-enabled,CFG_DRIVERS_CLK CFG_DT) +CFG_DRIVERS_CLK_FIXED ?= $(CFG_DRIVERS_CLK_DT) +CFG_DRIVERS_CLK_EARLY_PROBE ?= $(CFG_DRIVERS_CLK_DT) + +$(eval $(call cfg-depends-all,CFG_DRIVERS_CLK_DT,CFG_DRIVERS_CLK CFG_DT)) +$(eval $(call cfg-depends-all,CFG_DRIVERS_CLK_FIXED,CFG_DRIVERS_CLK_DT)) + +# When enabled, CFG_DRIVERS_RSTCTRL embeds a reset controller framework in +# OP-TEE core to provide reset controls on subsystems of the devices. +CFG_DRIVERS_RSTCTRL ?= n + +# When enabled, CFG_DRIVERS_GPIO embeds a GPIO controller framework in +# OP-TEE core to provide GPIO support for drivers. +CFG_DRIVERS_GPIO ?= n + +# When enabled, CFG_DRIVERS_I2C provides I2C controller and devices support. +CFG_DRIVERS_I2C ?= n + +# When enabled, CFG_DRIVERS_PINCTRL embeds a pin muxing controller framework in +# OP-TEE core to provide drivers a way to apply pin muxing configurations based +# on device-tree. +CFG_DRIVERS_PINCTRL ?= n + +# When enabled, CFG_DRIVERS_REGULATOR embeds a voltage regulator framework in +# OP-TEE core to provide drivers a common regulator interface and describe +# the regulators dependencies using an embedded device tree. +# +# When enabled, CFG_REGULATOR_FIXED embeds a voltage regulator driver for +# DT compatible "regulator-fixed" devices. +CFG_DRIVERS_REGULATOR ?= n +CFG_REGULATOR_FIXED ?= n + +$(eval $(call cfg-enable-all-depends,CFG_REGULATOR_FIXED, \ + CFG_DRIVERS_REGULATOR CFG_DT)) + +# The purpose of this flag is to show a print when booting up the device that +# indicates whether the board runs a standard developer configuration or not. +# A developer configuration doesn't necessarily has to be secure. The intention +# is that the one making products based on OP-TEE should override this flag in +# plat-xxx/conf.mk for the platform they're basing their products on after +# they've finalized implementing stubbed functionality (see OP-TEE +# documentation/Porting guidelines) as well as vendor specific security +# configuration. +CFG_WARN_INSECURE ?= y + +# Enables warnings for declarations mixed with statements +CFG_WARN_DECL_AFTER_STATEMENT ?= y + +# Branch Target Identification (part of the ARMv8.5 Extensions) provides a +# mechanism to limit the set of locations to which computed branch instructions +# such as BR or BLR can jump. To make use of BTI in TEE core and ldelf on CPU's +# that support it, enable this option. A GCC toolchain built with +# --enable-standard-branch-protection is needed to use this option. +CFG_CORE_BTI ?= n + +$(eval $(call cfg-depends-all,CFG_CORE_BTI,CFG_ARM64_core)) + +# To make use of BTI in user space libraries and TA's on CPU's that support it, +# enable this option. +CFG_TA_BTI ?= $(CFG_CORE_BTI) + +$(eval $(call cfg-depends-all,CFG_TA_BTI,CFG_ARM64_core)) + +ifeq (y-y,$(CFG_NS_VIRTUALIZATION)-$(call cfg-one-enabled, CFG_TA_BTI CFG_CORE_BTI)) +$(error CFG_NS_VIRTUALIZATION and BTI are currently incompatible) +endif + +ifeq (y-y,$(CFG_PAGED_USER_TA)-$(CFG_TA_BTI)) +$(error CFG_PAGED_USER_TA and CFG_TA_BTI are currently incompatible) +endif + +# Memory Tagging Extension (part of the ARMv8.5 Extensions) implements lock +# and key access to memory. This is a hardware supported alternative to +# CFG_CORE_SANITIZE_KADDRESS which covers both S-EL1 and S-EL0. +CFG_MEMTAG ?= n + +$(eval $(call cfg-depends-all,CFG_MEMTAG,CFG_ARM64_core)) +ifeq (y-y,$(CFG_CORE_SANITIZE_KADDRESS)-$(CFG_MEMTAG)) +$(error CFG_CORE_SANITIZE_KADDRESS and CFG_MEMTAG are not compatible) +endif +ifeq (y-y,$(CFG_WITH_PAGER)-$(CFG_MEMTAG)) +$(error CFG_WITH_PAGER and CFG_MEMTAG are not compatible) +endif + +# Privileged Access Never (PAN, part of the ARMv8.1 Extensions) can be +# used to restrict accesses to unprivileged memory from privileged mode. +# For RISC-V architecture, CSR {m|s}status.SUM bit is used to implement PAN. +CFG_PAN ?= n + +$(eval $(call cfg-depends-one,CFG_PAN,CFG_ARM64_core CFG_RV64_core CFG_RV32_core)) + +# CFG_CORE_ASYNC_NOTIF is defined by the platform to enable support +# for sending asynchronous notifications to normal world. Note that an +# interrupt ID must be configurged by the platform too. Currently is only +# CFG_CORE_ASYNC_NOTIF_GIC_INTID defined. +CFG_CORE_ASYNC_NOTIF ?= n + +$(eval $(call cfg-enable-all-depends,CFG_MEMPOOL_REPORT_LAST_OFFSET, \ + CFG_WITH_STATS)) + +# Pointer Authentication (part of ARMv8.3 Extensions) provides instructions +# for signing and authenticating pointers against secret keys. These can +# be used to mitigate ROP (Return oriented programming) attacks. This is +# currently done by instructing the compiler to add paciasp/autiasp at the +# begging and end of functions to sign and verify ELR. +# +# The CFG_CORE_PAUTH enables these instructions for the core parts +# executing at EL1, with one secret key per thread and one secret key per +# physical CPU. +# +# The CFG_TA_PAUTH option enables these instructions for TA's at EL0. When +# this option is enabled, TEE core will initialize secret keys per TA. +CFG_CORE_PAUTH ?= n +CFG_TA_PAUTH ?= $(CFG_CORE_PAUTH) + +$(eval $(call cfg-depends-all,CFG_CORE_PAUTH,CFG_ARM64_core)) +$(eval $(call cfg-depends-all,CFG_TA_PAUTH,CFG_ARM64_core)) + +ifeq (y-y,$(CFG_NS_VIRTUALIZATION)-$(CFG_CORE_PAUTH)) +$(error CFG_NS_VIRTUALIZATION and CFG_CORE_PAUTH are currently incompatible) +endif +ifeq (y-y,$(CFG_NS_VIRTUALIZATION)-$(CFG_TA_PAUTH)) +$(error CFG_NS_VIRTUALIZATION and CFG_TA_PAUTH are currently incompatible) +endif + +ifeq (y-y,$(CFG_TA_GPROF_SUPPORT)-$(CFG_TA_PAUTH)) +$(error CFG_TA_GPROF_SUPPORT and CFG_TA_PAUTH are currently incompatible) +endif + +ifeq (y-y,$(CFG_FTRACE_SUPPORT)-$(CFG_TA_PAUTH)) +$(error CFG_FTRACE_SUPPORT and CFG_TA_PAUTH are currently incompatible) +endif + +# Enable support for generic watchdog registration +# This watchdog will then be usable by non-secure world through SMC calls. +CFG_WDT ?= n + +# Enable watchdog SMC handling compatible with arm-smc-wdt Linux driver +CFG_WDT_SM_HANDLER ?= n + +$(eval $(call cfg-enable-all-depends,CFG_WDT_SM_HANDLER,CFG_WDT)) + +# When CFG_WDT_SM_HANDLER=y, SMC function ID 0x82003D06 default implements +# arm-smc-wdt service. Platform can also override this ID with a platform +# specific SMC function ID to access arm-smc-wdt service thanks to +# optional config switch CFG_WDT_SM_HANDLER_ID. +CFG_WDT_SM_HANDLER_ID ?= 0x82003D06 + +# Allow using the udelay/mdelay function for platforms without ARM generic timer +# extension. When set to 'n', the plat_get_freq() function must be defined by +# the platform code +CFG_CORE_HAS_GENERIC_TIMER ?= y + +# Enable RTC API +CFG_DRIVERS_RTC ?= n + +# Enable PTA for RTC access from non-secure world +CFG_RTC_PTA ?= n + +# Enable the FF-A SPMC tests in xtests +CFG_SPMC_TESTS ?= n + +# Allocate the translation tables needed to map the S-EL0 application +# loaded +CFG_CORE_PREALLOC_EL0_TBLS ?= n +ifeq (y-y,$(CFG_CORE_PREALLOC_EL0_TBLS)-$(CFG_WITH_PAGER)) +$(error "CFG_WITH_PAGER can't support CFG_CORE_PREALLOC_EL0_TBLS") +endif + +# User TA runtime context dump. +# When this option is enabled, OP-TEE provides a debug method for +# developer to dump user TA's runtime context, including TA's heap stats. +# Developer can open a stats PTA session and then invoke command +# STATS_CMD_TA_STATS to get the context of loaded TAs. +CFG_TA_STATS ?= n + +# Enables best effort mitigations against fault injected when the hardware +# is tampered with. Details in lib/libutils/ext/include/fault_mitigation.h +CFG_FAULT_MITIGATION ?= y + +# Enables TEE Internal Core API v1.1 compatibility for in-tree TAs. Note +# that this doesn't affect libutee itself, it's only the TAs compiled with +# this set that are affected. Each out-of-tree must set this if to enable +# compatibility with version v1.1 as the value of this variable is not +# preserved in the TA dev-kit. +CFG_TA_OPTEE_CORE_API_COMPAT_1_1 ?= n + +# Change supported HMAC key size range, from 64 to 1024. +# This is needed to pass AOSP Keymaster VTS tests: +# Link to tests : https://android.googlesource.com/platform/hardware/interfaces/+/master/keymaster/3.0/vts/functional/keymaster_hidl_hal_test.cpp +# Module: VtsHalKeymasterV3_0TargetTest +# Testcases: - PerInstance/SigningOperationsTest# +# - PerInstance/NewKeyGenerationTest# +# - PerInstance/ImportKeyTest# +# - PerInstance/EncryptionOperationsTest# +# - PerInstance/AttestationTest# +# Note that this violates GP requirements of HMAC size range. +CFG_HMAC_64_1024_RANGE ?= n + +# Enable a hardware pbkdf2 function +# By default use standard pbkdf2 implementation +CFG_CRYPTO_HW_PBKDF2 ?= n +$(eval $(call cfg-depends-all,CFG_CRYPTO_HW_PBKDF2,CFG_CRYPTO_PBKDF2)) diff --git a/optee_os/mk/gcc.mk b/optee_os/mk/gcc.mk new file mode 100644 index 0000000..adc77a2 --- /dev/null +++ b/optee_os/mk/gcc.mk @@ -0,0 +1,36 @@ + +CC$(sm) := $(CROSS_COMPILE_$(sm))gcc +CXX$(sm) := $(CROSS_COMPILE_$(sm))g++ +CPP$(sm) := $(CROSS_COMPILE_$(sm))cpp +LD$(sm) := $(CROSS_COMPILE_$(sm))ld.bfd +AR$(sm) := $(CROSS_COMPILE_$(sm))ar +NM$(sm) := $(CROSS_COMPILE_$(sm))nm +OBJCOPY$(sm) := $(CROSS_COMPILE_$(sm))objcopy +OBJDUMP$(sm) := $(CROSS_COMPILE_$(sm))objdump +READELF$(sm) := $(CROSS_COMPILE_$(sm))readelf + +nostdinc$(sm) := -nostdinc -isystem $(shell $(CC$(sm)) \ + -print-file-name=include 2> /dev/null) + +# Get location of libgcc from gcc +libgcc$(sm) := $(shell $(CC$(sm)) $(CFLAGS$(arch-bits-$(sm))) \ + -print-libgcc-file-name 2> /dev/null) +libstdc++$(sm) := $(shell $(CXX$(sm)) $(CXXFLAGS$(arch-bits-$(sm))) $(comp-cxxflags$(sm)) \ + -print-file-name=libstdc++.a 2> /dev/null) +libgcc_eh$(sm) := $(shell $(CXX$(sm)) $(CXXFLAGS$(arch-bits-$(sm))) $(comp-cxxflags$(sm)) \ + -print-file-name=libgcc_eh.a 2> /dev/null) + +# Define these to something to discover accidental use +CC := false +CXX := false +CPP := false +LD := false +AR := false +NM := false +OBJCOPY := false +OBJDUMP := false +READELF := false +nostdinc := --bad-nostdinc-variable +libgcc := --bad-libgcc-variable + + diff --git a/optee_os/mk/lib.mk b/optee_os/mk/lib.mk new file mode 100644 index 0000000..029260d --- /dev/null +++ b/optee_os/mk/lib.mk @@ -0,0 +1,101 @@ +# Input +# +# libname the name of the lib +# libdir directory of lib which also is used as input to +# mk/subdir.mk +# conf-file [optional] if set, all objects will depend on $(conf-file) +# [if CFG_ULIBS_SHARED==y] +# libuuid the UUID of the shared lib +# libl other libraries this library depends on; used to generate the +# proper link arguments (-Lxxx -lyyy) and to add dependencies +# on the needed .so files +# [endif] +# +# Output +# +# updated cleanfiles and +# updated libfiles, libdirs, libnames and libdeps + + +subdirs = $(libdir) +include mk/subdir.mk +ifeq ($(filter $(sm), core ldelf),) # TA +ifeq ($(CFG_ULIBS_MCOUNT),y) +cflags-lib$(libname)-$(sm) += -pg +endif +endif +include mk/compile.mk + +lib-libfile := $(out-dir)/$(base-prefix)$(libdir)/lib$(libname).a +ifeq ($(CFG_ULIBS_SHARED),y) +lib-shlibfile := $(out-dir)/$(base-prefix)$(libdir)/lib$(libname).so +lib-shlibstrippedfile := $(out-dir)/$(base-prefix)$(libdir)/lib$(libname).stripped.so +lib-shlibtafile := $(out-dir)/$(base-prefix)$(libdir)/$(libuuid).ta +lib-libuuidln := $(out-dir)/$(base-prefix)$(libdir)/$(libuuid).elf +lib-shlibfile-$(libname)-$(sm) := $(lib-shlibfile) +lib-libdir-$(libname)-$(sm) := $(out-dir)/$(base-prefix)$(libdir) +lib-needed-so-files := $(foreach l,$(libl),$(lib-shlibfile-$(l)-$(sm))) +lib-Ll-args := $(foreach l,$(libl),-L$(lib-libdir-$(l)-$(sm)) -l$(l)) +endif +cleanfiles := $(lib-libfile) $(lib-shlibfile) $(lib-shlibstrippedfile) $(lib-shlibtafile) $(lib-libuuidln) $(cleanfiles) +libfiles := $(lib-libfile) $(lib-shlibfile) $(lib-shlibstrippedfile) $(lib-shlibtafile) $(lib-libuuidln) $(libfiles) +libdirs := $(out-dir)/$(base-prefix)$(libdir) $(libdirs) +libnames := $(libname) $(libnames) +libdeps := $(lib-libfile) $(libdeps) + +SIGN = scripts/sign_encrypt.py +TA_SIGN_KEY ?= keys/default_ta.pem + +define process-lib +ifeq ($(lib-use-ld), y) +$(lib-libfile): $(objs) + @echo ' LD $$@' + @mkdir -p $$(dir $$@) + $$(q)$$(LD$(sm)) $(lib-ldflags) -o $$@ $$^ +else +$(lib-libfile): $(objs) + @$(cmd-echo-silent) ' AR $$@' + @mkdir -p $$(dir $$@) + $$(q)rm -f $$@ && $$(AR$(sm)) rcs $$@ $$^ +endif +ifeq ($(CFG_ULIBS_SHARED),y) +ifeq ($(sm)-$(CFG_TA_BTI),ta_arm64-y) +lib-ldflags$(lib-shlibfile) += $$(call ld-option,-z force-bti) --fatal-warnings +endif +$(lib-shlibfile): $(objs) $(lib-needed-so-files) + @$(cmd-echo-silent) ' LD $$@' + @mkdir -p $$(dir $$@) + $$(q)$$(LD$(sm)) $(lib-ldflags) -shared -z max-page-size=4096 \ + $(call ld-option,-z separate-loadable-segments) \ + $$(lib-ldflags$(lib-shlibfile)) \ + --soname=$(libuuid) -o $$@ $$(filter-out %.so,$$^) $(lib-Ll-args) + +$(lib-shlibstrippedfile): $(lib-shlibfile) + @$(cmd-echo-silent) ' OBJCOPY $$@' + $$(q)$$(OBJCOPY$(sm)) --strip-unneeded $$< $$@ + +$(lib-shlibtafile): $(lib-shlibstrippedfile) $(TA_SIGN_KEY) + @$(cmd-echo-silent) ' SIGN $$@' + $$(q)$$(PYTHON3) $$(SIGN) --key $(TA_SIGN_KEY) --uuid $(libuuid) --in $$< --out $$@ + +$(lib-libuuidln): $(lib-shlibfile) + @$(cmd-echo-silent) ' LN $$@' + $$(q)ln -sf lib$(libname).so $$@ +endif +endef #process-lib + +$(eval $(call process-lib)) + +$(objs): $(conf-file) + +# Clean residues from processing +objs := +libname := +libuuid := +lib-use-ld := +lib-shlibfile := +lib-shlibstrippedfile := +lib-shlibtafile := +lib-libuuidln := +lib-needed-so-files := +libl := diff --git a/optee_os/mk/macros.mk b/optee_os/mk/macros.mk new file mode 100644 index 0000000..277ecf2 --- /dev/null +++ b/optee_os/mk/macros.mk @@ -0,0 +1,9 @@ +# Rename $1 to $2 only if file content differs. Otherwise just delete $1. +define mv-if-changed + if cmp -s $2 $1; then \ + rm -f $1; \ + else \ + $(cmd-echo-silent) ' UPD $2'; \ + mv $1 $2; \ + fi +endef diff --git a/optee_os/mk/subdir.mk b/optee_os/mk/subdir.mk new file mode 100644 index 0000000..d280c30 --- /dev/null +++ b/optee_os/mk/subdir.mk @@ -0,0 +1,183 @@ +# Input +# +# subdirs tells the subdirectories to descend +# +# Output +# +# set srcs gen-srcs +# set cflags-$(oname) cflags-remove-$(oname) +# cxxflags-$(oname) cxxflags-remove-$(oname) +# aflags-$(oname) aflags-remove-$(oname) +# cppflags-$(oname) cppflags-remove-$(oname) +# incdirs-$(oname) +# incdirs-lib$(libname)-$(sm) [if libname is defined] +# cppflags-lib$(libname)-$(sm) [if libname is defined] +# cflags-lib$(libname)-$(sm) [if libname is defined] +# cxxflags-lib$(libname)-$(sm) [if libname is defined] +# for each file found, oname is the name of the object file for corresponding +# source file + +srcs := +gen-srcs := +asm-defines-files := + +uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1))) + +define process-subdir-srcs-y +ifeq ($$(sub-dir),.) +srcs += $1 +oname := $(out-dir)/$(base-prefix)$(basename $1).o +else +ifneq ($(filter /%,$(1)),) +# $1 is an absolute path - start with "/" +srcs += $1 +oname := $(out-dir)/$(base-prefix)$(basename $1).o +else +srcs += $(sub-dir)/$1 +oname := $(out-dir)/$(base-prefix)$(basename $$(sub-dir)/$1).o +endif +endif +cflags-$$(oname) := $$(cflags-y) $$(cflags-$(1)-y) +cflags-remove-$$(oname) := $$(cflags-remove-y) \ + $$(cflags-remove-$(1)-y) +cxxflags-$$(oname) := $$(cxxflags-y) $$(cxxflags-$(1)-y) +cxxflags-remove-$$(oname) := $$(cxxflags-remove-y) \ + $$(cxxflags-remove-$(1)-y) +cppflags-$$(oname) := $$(cppflags-y) $$(cppflags-$(1)-y) +cppflags-remove-$$(oname) := $$(cppflags-remove-y) \ + $$(cppflags-remove-$(1)-y) +aflags-$$(oname) := $$(aflags-y) $$(aflags-$(1)-y) +aflags-remove-$$(oname) := $$(aflags-remove-y) \ + $$(aflags-remove-$(1)-y) +incdirs-$$(oname) := $$(thissubdir-incdirs) $$(addprefix $(sub-dir)/,$$(incdirs-$(1)-y)) +# Clear local filename specific variables to avoid accidental reuse +# in another subdirectory +cflags-$(1)-y := +cflags-remove-$(1)-y := +cflags-lib-y := +cxxflags-$(1)-y := +cxxflags-remove-$(1)-y := +cxxflags-lib-y := +cppflags-$(1)-y := +cppflags-remove-$(1)-y := +cppflags-lib-y := +aflags-$(1)-y := +aflags-remove-$(1)-y := +incdirs-$(1)-y := +fname := +oname := +endef #process-subdir-srcs-y + +define process-subdir-gensrcs-helper +# $1 gensrc-y element +# $2 full path and name of generated source file +# $3 full path and name of object file compiled from source file +# $4 full path to out directory +# $5 y if $2 must be generated before $(sm) starts building (e.g., .h file) + +gen-srcs += $2 +cleanfiles += $2 +oname := $3 + +FORCE-GENSRC$(sm): $(if $(filter y,$5),$2,) + +$$(addprefix $4,$$(produce-additional-$1)): $2 + +subdir-$2 := $$(sub-dir) +recipe-$2 := $$(recipe-$1) +$2: $$(depends-$1) + @$(cmd-echo-silent) ' GEN $2' + $(q)mkdir -p $4 + $(q)$$(recipe-$2) + +cflags-$$(oname) := $$(cflags-y) $$(cflags-$(1)-y) +cflags-remove-$$(oname) := $$(cflags-remove-y) \ + $$(cflags-remove-$(1)-y) +cxxflags-$$(oname) := $$(cxxflags-y) $$(cxxflags-$(1)-y) +cxxflags-remove-$$(oname) := $$(cxxflags-remove-y) \ + $$(cxxflags-remove-$(1)-y) +cppflags-$$(oname) := $$(cppflags-y) $$(cppflags-$(1)-y) +cppflags-remove-$$(oname) := $$(cppflags-remove-y) \ + $$(cppflags-remove-$(1)-y) +aflags-$$(oname) := $$(aflags-y) $$(aflags-$(1)-y) +aflags-remove-$$(oname) := $$(aflags-remove-y) \ + $$(aflags-remove-$(1)-y) +incdirs-$$(oname) := $$(thissubdir-incdirs) $$(addprefix $(sub-dir)/,$$(incdirs-$(1)-y)) +# Clear local filename specific variables to avoid accidental reuse +# in another subdirectory +cflags-$(1)-y := +cflags-remove-$(1)-y := +cflags-lib-y := +cxxflags-$(1)-y := +cxxflags-remove-$(1)-y := +cxxflags-lib-y := +cppflags-$(1)-y := +cppflags-remove-$(1)-y := +cppflags-lib-y := +aflags-$(1)-y := +aflags-remove-$(1)-y := +incdirs-$(1)-y := +fname := +oname := + +endef #process-subdir-gensrcs-helper + +define process-subdir-gensrcs-y +$$(eval $$(call process-subdir-gensrcs-helper,$1,$(sub-dir-out)/$$(produce-$1),$(sub-dir-out)/$(basename $(produce-$1)).o,$(sub-dir-out),$(force-gensrc-$1))) +endef #process-subdir-gensrcs-y + +define process-subdir-asm-defines-y +asm-defines-files += $(sub-dir)/$1 +endef #process-subdir-asm-defines-y + +define process-subdir +sub-dir := $1 +ifeq ($1,.) +sub-dir-out := $(patsubst %/,%,$(out-dir)/$(base-prefix)) +else +sub-dir-out := $(out-dir)/$(base-prefix)$1 +endif + +include $1/sub.mk +sub-subdirs := $$(addprefix $1/,$$(subdirs-y)) $$(subdirs_ext-y) +incdirs$(sm) := $(incdirs$(sm)) $$(addprefix $1/,$$(global-incdirs-y)) +thissubdir-incdirs := $(out-dir)/$(base-prefix)$1 $$(addprefix $1/,$$(incdirs-y)) $$(incdirs_ext-y) +ifneq ($$(libname),) +incdirs-lib$$(libname)-$$(sm) := $$(incdirs-lib$$(libname)-$$(sm)) $$(addprefix $1/,$$(incdirs-lib-y)) +cflags-lib$$(libname)-$$(sm) := $$(cflags-lib$$(libname)-$$(sm)) $$(cflags-lib-y) +cxxflags-lib$$(libname)-$$(sm) := $$(cxxflags-lib$$(libname)-$$(sm)) $$(cxxflags-lib-y) +cppflags-lib$$(libname)-$$(sm) := $$(cppflags-lib$$(libname)-$$(sm)) $$(cppflags-lib-y) +endif + +# Process files in current directory +$$(foreach g, $$(gensrcs-y), $$(eval $$(call process-subdir-gensrcs-y,$$(g)))) +$$(foreach s, $$(srcs-y), $$(eval $$(call process-subdir-srcs-y,$$(s)))) +$$(foreach a, $$(asm-defines-y), $$(eval $$(call process-subdir-asm-defines-y,$$(a)))) +# Clear flags used when processing current directory +srcs-y := +cflags-y := +cflags-lib-y := +cxxflags-y := +cxxflags-lib-y := +cppflags-y := +cppflags-lib-y := +aflags-y := +cflags-remove-y := +cxxflags-remove-y := +aflags-remove-y := +subdirs-y := +subdirs_ext-y := +global-incdirs-y := +incdirs-lib-y := +incdirs-y := +incdirs_ext-y := +gensrcs-y := +this-out-dir := +asm-defines-y := + +# Process subdirectories in current directory +$$(foreach sd, $$(call uniq,$$(sub-subdirs)), $$(eval $$(call process-subdir,$$(sd)))) +endef #process-subdir + +# Top subdirectories +$(foreach sd, $(call uniq,$(subdirs)), $(eval $(call process-subdir,$(sd)))) diff --git a/optee_os/scripts/arm32_sysreg.py b/optee_os/scripts/arm32_sysreg.py new file mode 100755 index 0000000..4b6f9cf --- /dev/null +++ b/optee_os/scripts/arm32_sysreg.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2018, Linaro Limited +# + + +import argparse +import sys +import re + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def my_err(line_number, msg): + eprint('Error: line:' + repr(line_number) + ' ' + msg) + sys.exit(1) + + +def gen_read64_macro(reg_name, opc1, crm, descr): + print('') + if len(descr): + print('\t# ' + descr) + print('\t.macro read_' + reg_name.lower() + ' reg0, reg1') + print('\tmrrc\tp15, ' + opc1 + ', \\reg0, \\reg1, ' + crm) + print('\t.endm') + + +def gen_write64_macro(reg_name, opc1, crm, descr): + print('') + if len(descr): + print('\t# ' + descr) + print('\t.macro write_' + reg_name.lower() + ' reg0, reg1') + print('\tmcrr\tp15, ' + opc1 + ', \\reg0, \\reg1, ' + crm) + print('\t.endm') + + +def gen_read32_macro(reg_name, crn, opc1, crm, opc2, descr): + print('') + if len(descr): + print('\t# ' + descr) + print('\t.macro read_' + reg_name.lower() + ' reg') + print('\tmrc p15, ' + opc1 + ', \\reg, ' + crn + ', ' + crm + ', ' + opc2) + print('\t.endm') + + +def gen_write32_macro(reg_name, crn, opc1, crm, opc2, descr): + print('') + if len(descr): + print('\t# ' + descr) + print('\t.macro write_' + reg_name.lower() + ' reg') + print('\tmcr p15, ' + opc1 + ', \\reg, ' + crn + ', ' + crm + ', ' + opc2) + print('\t.endm') + + +def gen_write32_dummy_macro(reg_name, crn, opc1, crm, opc2, descr): + print('') + if len(descr): + print('\t# ' + descr) + print('\t.macro write_' + reg_name.lower()) + print('\t# Register ignored') + print('\tmcr p15, ' + opc1 + ', r0, ' + crn + ', ' + crm + ', ' + opc2) + print('\t.endm') + + +def gen_read64_func(reg_name, opc1, crm, descr): + print('') + if len(descr): + print('/* ' + descr + ' */') + print('static inline __noprof uint64_t read_' + reg_name.lower() + + '(void)') + print('{') + print('\tuint64_t v;') + print('') + print('\tasm volatile ("mrrc p15, ' + opc1 + ', %Q0, %R0, ' + + crm + '"' + ' : "=r" (v));') + print('') + print('\treturn v;') + print('}') + + +def gen_write64_func(reg_name, opc1, crm, descr): + print('') + if len(descr): + print('/* ' + descr + ' */') + print('static inline __noprof void write_' + reg_name.lower() + + '(uint64_t v)') + print('{') + print('\tasm volatile ("mcrr p15, ' + opc1 + ', %Q0, %R0, ' + + crm + '"' + ' : : "r" (v));') + print('}') + + +def gen_read32_func(reg_name, crn, opc1, crm, opc2, descr): + print('') + if len(descr): + print('/* ' + descr + ' */') + print('static inline __noprof uint32_t read_' + reg_name.lower() + + '(void)') + print('{') + print('\tuint32_t v;') + print('') + print('\tasm volatile ("mrc p15, ' + opc1 + ', %0, ' + crn + ', ' + + crm + ', ' + opc2 + '"' + ' : "=r" (v));') + print('') + print('\treturn v;') + print('}') + + +def gen_write32_func(reg_name, crn, opc1, crm, opc2, descr): + print('') + if len(descr): + print('/* ' + descr + ' */') + print('static inline __noprof void write_' + reg_name.lower() + + '(uint32_t v)') + print('{') + print('\tasm volatile ("mcr p15, ' + opc1 + ', %0, ' + crn + ', ' + + crm + ', ' + opc2 + '"' + ' : : "r" (v));') + print('}') + + +def gen_write32_dummy_func(reg_name, crn, opc1, crm, opc2, descr): + print('') + if len(descr): + print('/* ' + descr + ' */') + print('static inline __noprof void write_' + reg_name.lower() + '(void)') + print('{') + print('\t/* Register ignored */') + print('\tasm volatile ("mcr p15, ' + opc1 + ', r0, ' + crn + ', ' + + crm + ', ' + opc2 + '");') + print('}') + + +def gen_file(line, line_number, s_file): + words = line.split() + if len(words) == 0: + return + + if len(re.findall('^ *#', line)): + return + + if len(re.findall('^ *@', line)): + comment = re.sub('^ *@', '', line) + comment = re.sub('^ *', '', comment) + comment = re.sub('[ \n]*$', '', comment) + if len(comment) == 0: + print('') + return + if s_file: + print('# ' + comment) + else: + print('/* ' + comment + ' */') + return + + reg_name = words[0] + crn = words[1] + opc1 = words[2] + crm = words[3] + opc2 = words[4] + access_type = words[5] + descr = " ".join(words[6:]) + + read_access = access_type == 'RO' or access_type == 'RW' + write_access = (access_type == 'WO' or access_type == 'RW' or + access_type == 'WOD') + dummy_access = access_type == 'WOD' + + if not read_access and not write_access: + my_err(line_number, 'bad Access Type "' + access_type + '"') + + if crn == '-': + if opc2 != '-': + my_err(line_number, 'bad opc2, expected -') + + if read_access: + if s_file: + gen_read64_macro(reg_name, opc1, crm, descr) + else: + gen_read64_func(reg_name, opc1, crm, descr) + + if s_file: + gen_write64_macro(reg_name, opc1, crm, descr) + else: + gen_write64_func(reg_name, opc1, crm, descr) + else: + if read_access: + if s_file: + gen_read32_macro(reg_name, crn, opc1, crm, opc2, descr) + else: + gen_read32_func(reg_name, crn, opc1, crm, opc2, descr) + + if write_access: + if dummy_access: + if s_file: + gen_write32_dummy_macro(reg_name, crn, opc1, crm, opc2, + descr) + else: + gen_write32_dummy_func(reg_name, crn, opc1, crm, opc2, + descr) + else: + if s_file: + gen_write32_macro(reg_name, crn, opc1, crm, opc2, descr) + else: + gen_write32_func(reg_name, crn, opc1, crm, opc2, descr) + + +def get_args(): + parser = argparse.ArgumentParser(description='Generates instructions to ' + 'access ARM32 system registers.') + + parser.add_argument('--s_file', action='store_true', + help='Generate an Assembly instead of a C file') + parser.add_argument('--guard', + help='Provide #ifdef in C file') + + return parser.parse_args() + + +def main(): + args = get_args() + + cmnt = 'Automatically generated, do not edit' + if args.s_file: + print('# ' + cmnt) + else: + print('/* ' + cmnt + ' */') + if args.guard is not None: + print('#ifndef ' + args.guard.upper().replace('.', '_')) + print('#define ' + args.guard.upper().replace('.', '_')) + print('#include ') + + line_number = 0 + for line in sys.stdin: + line_number = line_number + 1 + gen_file(line, line_number, args.s_file) + + if not args.s_file and args.guard is not None: + print('#endif /*' + args.guard.upper().replace('.', '_') + '*/') + + +if __name__ == '__main__': + main() diff --git a/optee_os/scripts/bin_to_c.py b/optee_os/scripts/bin_to_c.py new file mode 100755 index 0000000..8ac6d30 --- /dev/null +++ b/optee_os/scripts/bin_to_c.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2018, Linaro Limited +# + +import argparse +import array +import os +import re +import sys + + +def get_args(): + + parser = argparse.ArgumentParser(description='Converts a binary file ' + 'into C source file defining binary ' + 'data as a constant byte array.') + + parser.add_argument('--bin', required=True, + help='Path to the input binary file') + + parser.add_argument('--vname', required=True, + help='Variable name for the generated table in ' + 'the output C source file.') + + parser.add_argument('--out', required=True, + help='Path for the generated C file') + + parser.add_argument('--text', required=False, action='store_true', + help='Treat input as a text file') + + return parser.parse_args() + + +def main(): + + args = get_args() + + with open(args.bin, 'rb') as indata: + bytes = indata.read() + if args.text: + bytes += b'\0' + size = len(bytes) + + f = open(args.out, 'w') + f.write('/* Generated from ' + args.bin + ' by ' + + os.path.basename(__file__) + ' */\n\n') + f.write('#include \n') + f.write('#include \n') + if args.text: + f.write('__extension__ const char ' + args.vname + '[] = {\n') + else: + f.write('__extension__ const uint8_t ' + args.vname + '[] ' + + ' __aligned(__alignof__(uint64_t)) = {\n') + i = 0 + while i < size: + if i % 8 == 0: + f.write('\t\t') + if args.text and i != size - 1 and bytes[i] == b'\0': + print('Error: null byte encountered in text file') + sys.exit(1) + f.write(hex(bytes[i]) + ',') + i = i + 1 + if i % 8 == 0 or i == size: + f.write('\n') + else: + f.write(' ') + f.write('};\n') + f.close() + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/checkpatch.sh b/optee_os/scripts/checkpatch.sh new file mode 100755 index 0000000..c403dd2 --- /dev/null +++ b/optee_os/scripts/checkpatch.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +DIR="${BASH_SOURCE%/*}" + +# if no CHECKPATCH is explicitly given by the environment, try to +# locate checkpatch.pl: first take the one from the path, then check +# for a local copy of the linux headers, finally try sources downloaded +# with OP-TEE (for QEMU) +if [ -z "$CHECKPATCH" ]; then + CHECKPATCH=$(command -v checkpatch.pl) +fi +if [ -z "$CHECKPATCH" ]; then + CHECKPATCH=$(find /usr/src/linux-headers* -name checkpatch.pl -print -quit) +fi +if [ -z "$CHECKPATCH" ]; then + CHECKPATCH=$(find "$PWD/../linux" -name checkpatch.pl -print -quit) +fi + +source "$DIR/checkpatch_inc.sh" + +hash $CHECKPATCH 2>/dev/null || + { echo >&2 "Could not find checkpatch.pl, aborting"; exit 1; } + +help() { + cat <<-EOF +Usage: + checkpatch.sh [--working] + checkpatch.sh ... + checkpatch.sh ... + checkpatch.sh --diff + checkpatch.sh --cached + checkpatch.sh --help + +Args: + Any commit or any number of commits. + Any number of Git Revision Selections. (requires git v2.19) + https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection + +Options: + --working Check the working area [Default]. + --cached Check the staging area. + --diff Check the diff between commit1 and commit2. + --help Print this help message. + +Examples: + checkpatch.sh commit1 commit2 commit3 Check commit1, commit2, and commit3. + + checkpatch.sh HEAD~5 Check the commit 5 revisions before + the current HEAD. + + checkpatch.sh commit1..^commit2 Check each commit from commit1 to + commit2 inclusively. + (requires git v2.19) + + checkpatch.sh HEAD~5..HEAD~1 Check each commit from HEAD~5 to + HEAD~1 exclusively, aka not including + HEAD~1. (requires git v2.19) + + checkpatch.sh commit1...tags/tag1 Check each commit that exists + exclusively within the history of + only one of each given revision. + (requires git v2.19) + + checkpatch.sh HEAD~10-5 Check 5 commits moving forward in + history starting from HEAD~10. + (requires git v2.19) + + checkpatch.sh branch1 tags/tag1 Check the HEAD of branch1 and the + HEAD of tag1. (requires git v2.19) +EOF + exit 1 +} + +op=${1:---working} +case "$op" in + --cached) + echo "Checking staging area: " + checkstaging + ;; + --diff) + echo "Checking diff (diff $1...$2)" + checkdiff "$2" "$3" + ;; + --working) + echo "Checking working area: " + checkworking + ;; + --help|-h) + help + ;; + *) + echo "Checking commit(s):" + read -r MAJOR MINOR < <(git --version | awk -F '[. ]' '{print $3, $4}') + if (( MAJOR < 2 )) || (( MAJOR == 2 && MINOR < 19 )); then + for c in "$@"; do checkpatch "$c"; done + else + for arg in "$@"; do + # parse the argument into a git object or list of git objects + object="$(git rev-parse "${arg}")" || continue + # run checkpatch if the parsed argument represents a single commit hash + if git cat-file -e "${object}" 2>/dev/null; then + checkpatch "${object}" + else + # expand the object list and run checkpatch on each commit id + commits="$(echo "${object}" | git rev-list --stdin)" + for c in ${commits}; do checkpatch "$c"; done + fi + done + fi + ;; + +esac diff --git a/optee_os/scripts/checkpatch_inc.sh b/optee_os/scripts/checkpatch_inc.sh new file mode 100644 index 0000000..fb77d5c --- /dev/null +++ b/optee_os/scripts/checkpatch_inc.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +CHECKPATCH="${CHECKPATCH:-checkpatch.pl}" +CHECKPATCH_OPT="${CHECKPATCH_OPT:-}" +# checkpatch.pl will ignore the following paths +CHECKPATCH_IGNORE=$(echo \ + core/include/gen-asm-defines.h \ + core/lib/lib{fdt,tomcrypt} core/lib/zlib \ + lib/libutils lib/libmbedtls \ + lib/libutee/include/elf.h \ + lib/libutee/include/elf_common.h \ + core/arch/arm/include/arm{32,64}.h \ + core/arch/arm/plat-ti/api_monitor_index_a{9,15}.h \ + core/arch/arm/dts \ + ta/pkcs11/scripts/verify-helpers.sh \ + core/arch/riscv/include/encoding.h ) +_CP_EXCL=$(for p in $CHECKPATCH_IGNORE; do echo ":(exclude)$p" ; done) + +function _checkpatch() { + # Use --typedefsfile if supported by the checkpatch tool + typedefs_opt="--typedefsfile typedefs.checkpatch" + $CHECKPATCH --help 2>&1 | grep -q -- --typedefsfile || \ + typedefs_opt=""; + # Ignore NOT_UNIFIED_DIFF in case patch has no diff + # (e.g., all paths filtered out) + $CHECKPATCH $CHECKPATCH_OPT $typedefs_opt - +} + +function checkpatch() { + git show --oneline --no-patch $1 + # The first git 'format-patch' shows the commit message + # The second one produces the diff (might be empty if _CP_EXCL + # filters out all diffs) + (git format-patch $1^..$1 --stdout | sed -n '/^diff --git/q;p'; \ + git format-patch $1^..$1 --stdout -- $_CP_EXCL . | \ + sed -n '/^diff --git/,$p') | _checkpatch +} + +function checkstaging() { + git diff --cached -- . $_CP_EXCL | _checkpatch +} + +function checkworking() { + git diff -- . $_CP_EXCL | _checkpatch +} + +function checkdiff() { + git diff $1...$2 -- . $_CP_EXCL | _checkpatch +} + diff --git a/optee_os/scripts/derive_rpmb_key.py b/optee_os/scripts/derive_rpmb_key.py new file mode 100755 index 0000000..13f9a93 --- /dev/null +++ b/optee_os/scripts/derive_rpmb_key.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2023, Linaro Limited +# + +import sys + + +def hex_parse(str): + try: + h = bytes.fromhex(str) + except ValueError as e: + try: + # Try to pad with a '0' nibble in front + h = bytes.fromhex('0' + str) + print('Odd number of nibbles in hexadecimal string', + file=sys.stderr) + raise e + except ValueError: + raise e + return h + + +def get_args(): + import argparse + import textwrap + + parser = argparse.ArgumentParser( + allow_abbrev=False, + description='''Derive an RPMB key from the Hardware Unique Key used + by OP-TEE and the CID of the RPMB.''', + epilog='''Note that the derived key matches what the + __huk_subkey_derive() would produce. If huk_subkey_derive() + is overridden to call another function, please don't use + this script''') + + parser.add_argument('--quiet', action='store_true', default=False, + help='''Gives only the hexstring of the RPMB key as + output, intended for scripting''') + parser.add_argument('--testkey', action='store_true', default=False, + help='''Outputs the hardcoded test key''') + parser.add_argument('--huk', type=hex_parse, + help='''Hardware Unique Key (16 bytes), as returned + by the platform specific function + tee_otp_get_hw_unique_key() in OP-TEE''') + parser.add_argument('--cid', type=hex_parse, help='CID (16 bytes)') + parser.add_argument('--compat', action='store_true', default=False, + help='''Generates a backwards compatible key, + only to be used if OP-TEE is build with + CFG_CORE_HUK_SUBKEY_COMPAT=y''') + + return parser.parse_args() + + +def derive_key(huk, cid, compat): + import struct + from cryptography.hazmat.primitives import hashes, hmac + + # Prepare the CID and Clear the PRV (Product revision) and CRC (CRC7 + # checksum) fields as OP-TEE does. + data = bytearray(cid) + data[9] = 0 + data[15] = 0 + + # This is how __huk_subkey_derive() is implemented, if huk_subkey_derive() + # is overridden the key derived here may not match what OP-TEE is using + # + # HUK is as tee_otp_get_hw_unique_key() in OP-TEE returns it + h = hmac.HMAC(huk, hashes.SHA256()) + if not compat: + usage_word = struct.pack(' FTRACE\x00\x01 ... +# +# is an array of 64-bit integers. +# - When the topmost byte is 0, the entry indicates a function return and the +# remaining bytes are a duration in nanoseconds. +# - A non-zero value is a stack depth, indicating a function entry, and the +# remaining bytes are the function's address. + +import sys + + +line = "" +curr_depth = 0 + + +def usage(): + print(f"Usage: {sys.argv[0]} ftrace.out") + print("Converts a ftrace file to text. Output is written to stdout.") + sys.exit(0) + + +def format_time(ns): + if ns < 1000000: + us = ns / 1000 + return f"{us:7.3f} us" + else: + ms = ns / 1000000 + return f"{ms:7.3f} ms" + + +def display(depth, val): + global line, curr_depth + if depth != 0: + curr_depth = depth + if line != "": + line = line.replace("TIME", " " * 10) + " {" + print(line) + line = "" + line = f" TIME | {depth:3} | " + " " * depth + f"0x{val:016x}()" + else: + if line != "": + line = line.replace("TIME", format_time(val)) + print(line) + line = "" + else: + if curr_depth != 0: + curr_depth = curr_depth - 1 + print(" " + format_time(val) + f" | {curr_depth:3} | " + + " " * curr_depth + "}") + + +def main(): + if len(sys.argv) < 2: + usage() + with open(sys.argv[1], 'rb') as f: + s = f.read() + magic = s.find(b'FTRACE\x00\x01') + if magic == -1: + print("Magic not found", file=sys.stderr) + sys.exit(1) + print(s[:magic].rstrip(b'\x00').decode()) + s = s[magic + 8:] + for i in range(0, len(s), 8): + elem = int.from_bytes(s[i:i + 8], byteorder="little", signed=False) + depth = elem >> 56 + val = elem & 0xFFFFFFFFFFFFFF + display(depth, val) + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/gen_ld_sects.py b/optee_os/scripts/gen_ld_sects.py new file mode 100755 index 0000000..bc82dd8 --- /dev/null +++ b/optee_os/scripts/gen_ld_sects.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2017, Linaro Limited +# + +import sys +import re + + +def usage(): + print("Usage: {0}
[...]".format( + sys.argv[0])) + sys.exit(1) + + +def main(): + if len(sys.argv) < 2: + usage() + + in_shdr = False + section_headers = re.compile("Section Headers:") + key_to_flags = re.compile("Key to Flags:") + match_rule = re.compile(sys.argv[1]) + skip_sections = sys.argv[2:] + + for line in sys.stdin: + if section_headers.match(line): + in_shdr = True + continue + if key_to_flags.match(line): + in_shdr = False + continue + + if not in_shdr: + continue + + words = line.split() + + if len(words) < 3: + continue + + if words[0] == "[": + name_offs = 2 + else: + name_offs = 1 + + sect_name = words[name_offs] + sect_type = words[name_offs + 1] + + if sect_type != "PROGBITS": + continue + + if not match_rule.match(sect_name): + continue + + if sect_name in skip_sections: + continue + + print('\t*({0})'.format(sect_name)) + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/gen_ldelf_hex.py b/optee_os/scripts/gen_ldelf_hex.py new file mode 100755 index 0000000..a6123b2 --- /dev/null +++ b/optee_os/scripts/gen_ldelf_hex.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2019, Linaro Limited +# + +from __future__ import print_function +from __future__ import division + +import argparse +import sys +try: + from elftools.elf.elffile import ELFFile + from elftools.elf.sections import SymbolTableSection + from elftools.elf.constants import P_FLAGS +except ImportError: + print(""" +*** +Can't find elftools module. Probably it is not installed on your system. +You can install this module with + +$ apt install python3-pyelftools + +if you are using Ubuntu. Or try to search for "pyelftools" or "elftools" in +your package manager if you are using some other distribution. +*** +""") + raise + + +def round_up(n, m): + if n == 0: + return 0 + else: + return (((n - 1) // m) + 1) * m + + +def emit_load_segments(elffile, outf): + load_size = 0 + code_size = 0 + data_size = 0 + load_segments = [s for s in elffile.iter_segments() + if s['p_type'] == 'PT_LOAD'] + prev_segment = None + pad = 0 + pad_size = [] + w_found = False + n = 0 + # Check that load segments ordered by VA have the expected layout: + # read only first, then read-write. Compute padding at end of each segment, + # 0 if none is required. + for segment in load_segments: + if prev_segment: + pad = segment['p_vaddr'] - (prev_segment['p_vaddr'] + + prev_segment['p_filesz']) + else: + if segment['p_flags'] & P_FLAGS.PF_W: + print('Expected RO load segment(s) first') + sys.exit(1) + if segment['p_flags'] & P_FLAGS.PF_W: + if not w_found: + # End of RO segments, discard padding for the last one (it + # would just take up space in the generated C file) + pad = 0 + w_found = True + else: + if w_found: + print('RO load segment found after RW one(s) (m={})'.format(n)) + sys.exit(1) + if prev_segment: + if pad > 31: + # We expect segments to be tightly packed together for memory + # efficiency. 31 is an arbitrary, "sounds reasonable" value + # which might need to be adjusted -- who knows what the + # compiler/linker can do. + print('Warning: suspiciously large padding ({}) after load ' + 'segment {}, please check'.format(pad, n-1)) + pad_size.append(pad) + prev_segment = segment + n = n + 1 + pad_size.append(0) + n = 0 + # Compute code_size, data_size and load_size + for segment in load_segments: + sz = segment['p_filesz'] + pad_size[n] + if segment['p_flags'] & P_FLAGS.PF_W: + data_size += sz + else: + code_size += sz + load_size += sz + n = n + 1 + n = 0 + i = 0 + # Output data to C file + outf.write(b'const uint8_t ldelf_data[%d]' % round_up(load_size, 4096)) + outf.write(b' __aligned(4096) = {\n') + for segment in load_segments: + data = segment.data() + if pad_size[n]: + # Pad with zeros if needed + data += bytearray(pad_size[n]) + for j in range(len(data)): + if i % 8 == 0: + outf.write(b'\t') + outf.write(b'0x' + '{:02x}'.format(data[j]).encode('utf-8') + + b',') + i = i + 1 + if i % 8 == 0 or i == load_size: + outf.write(b'\n') + else: + outf.write(b' ') + n = n + 1 + outf.write(b'};\n') + + outf.write(b'const unsigned int ldelf_code_size = %d;\n' % code_size) + outf.write(b'const unsigned int ldelf_data_size = %d;\n' % data_size) + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument('--input', + required=True, type=argparse.FileType('rb'), + help='The input ldelf.elf') + + parser.add_argument('--output', + required=True, type=argparse.FileType('wb'), + help='The output ldelf_hex.c') + + return parser.parse_args() + + +def main(): + args = get_args() + inf = args.input + outf = args.output + + elffile = ELFFile(inf) + + outf.write(b'/* Automatically generated, do no edit */\n') + outf.write(b'#include \n') + outf.write(b'#include \n') + emit_load_segments(elffile, outf) + outf.write(b'const unsigned long ldelf_entry = %lu;\n' % + elffile.header['e_entry']) + + inf.close() + outf.close() + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/gen_stmm_hex.py b/optee_os/scripts/gen_stmm_hex.py new file mode 100755 index 0000000..ea16fdd --- /dev/null +++ b/optee_os/scripts/gen_stmm_hex.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2019, Linaro Limited +# + +import argparse +import sys +import zlib + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument('--input', + required=True, type=argparse.FileType('rb'), + help='The input StMM binary (BL32_AP_MM.fd)') + + parser.add_argument('--output', + required=True, type=argparse.FileType('w'), + help='The output stmm_hex.c') + + return parser.parse_args() + + +def main(): + args = get_args() + inf = args.input + outf = args.output + + bytes = inf.read() + uncompressed_size = len(bytes) + bytes = zlib.compress(bytes) + size = len(bytes) + + outf.write('/* Automatically generated, do no edit */\n') + outf.write('const unsigned char stmm_image[] = {\n') + i = 0 + while i < size: + if i % 8 == 0: + outf.write('\t') + outf.write('0x{:02x},'.format(bytes[i])) + i = i + 1 + if i % 8 == 0 or i == size: + outf.write('\n') + else: + outf.write(' ') + outf.write('};\n') + + outf.write('const unsigned int stmm_image_size = sizeof(stmm_image);\n') + outf.write('const unsigned int stmm_image_uncompressed_size = ' + '{:d};\n'.format(uncompressed_size)) + + inf.close() + outf.close() + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/gen_tee_bin.py b/optee_os/scripts/gen_tee_bin.py new file mode 100755 index 0000000..a4aa532 --- /dev/null +++ b/optee_os/scripts/gen_tee_bin.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2019, Linaro Limited +# + +from __future__ import print_function +from __future__ import division + +import argparse +import sys +import struct +import re +import hashlib +try: + from elftools.elf.elffile import ELFFile + from elftools.elf.constants import SH_FLAGS + from elftools.elf.enums import ENUM_RELOC_TYPE_ARM + from elftools.elf.enums import ENUM_RELOC_TYPE_AARCH64 + from elftools.elf.sections import SymbolTableSection + from elftools.elf.relocation import RelocationSection + +except ImportError: + print(""" +*** +Can't find elftools module. Probably it is not installed on your system. +You can install this module with + +$ apt install python3-pyelftools + +if you are using Ubuntu. Or try to search for "pyelftools" or "elftools" in +your package manager if you are using some other distribution. +*** +""") + raise + +small_page_size = 4 * 1024 +elffile_symbols = None +tee_pageable_bin = None +tee_pager_bin = None +tee_embdata_bin = None + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def round_up(n, m): + if n == 0: + return 0 + else: + return (((n - 1) // m) + 1) * m + + +def get_arch_id(elffile): + e_machine = elffile.header['e_machine'] + if e_machine == 'EM_ARM': + return 0 + if e_machine == 'EM_AARCH64': + return 1 + eprint('Unknown e_machine "%s"' % e_machine) + sys.exit(1) + + +def get_name(obj): + # Symbol or section .name might be a byte array or a string, we want a + # string + try: + name = obj.name.decode() + except (UnicodeDecodeError, AttributeError): + name = obj.name + return name + + +def get_symbol(elffile, name): + global elffile_symbols + global lsyms_def + if elffile_symbols is None: + elffile_symbols = dict() + lsyms_def = dict() + symbol_tables = [s for s in elffile.iter_sections() + if isinstance(s, SymbolTableSection)] + for section in symbol_tables: + for symbol in section.iter_symbols(): + symbol_name = get_name(symbol) + if symbol['st_info']['bind'] == 'STB_GLOBAL': + elffile_symbols[symbol_name] = symbol + elif symbol['st_info']['bind'] == 'STB_LOCAL': + if symbol_name not in elffile_symbols.keys(): + elffile_symbols[symbol_name] = symbol + if symbol_name not in lsyms_def.keys(): + lsyms_def[symbol_name] = 1 + else: + lsyms_def[symbol_name] += 1 + + if name in lsyms_def.keys() and lsyms_def[name] > 1: + eprint("Multiple definitions of local symbol %s" % name) + sys.exit(1) + if name not in elffile_symbols.keys(): + eprint("Cannot find symbol %s" % name) + sys.exit(1) + + return elffile_symbols[name] + + +def get_sections(elffile, pad_to, dump_names): + last_end = 0 + bin_data = bytearray() + + for section in elffile.iter_sections(): + section_name = get_name(section) + if (section['sh_type'] == 'SHT_NOBITS' or + not (section['sh_flags'] & SH_FLAGS.SHF_ALLOC) or + not dump_names.match(section_name)): + continue + + if last_end == 0: + bin_data = section.data() + else: + if section['sh_addr'] > last_end: + bin_data += bytearray(section['sh_addr'] - last_end) + bin_data += section.data() + + last_end = section['sh_addr'] + section['sh_size'] + + if pad_to > last_end: + bin_data += bytearray(pad_to - last_end) + last_end = pad_to + + return bin_data + + +def get_pageable_bin(elffile): + global tee_pageable_bin + if tee_pageable_bin is None: + pad_to = 0 + dump_names = re.compile(r'^\..*_(pageable|init)$') + tee_pageable_bin = get_sections(elffile, pad_to, dump_names) + return tee_pageable_bin + + +def get_pager_bin(elffile): + global tee_pager_bin + if tee_pager_bin is None: + pad_to = get_symbol(elffile, '__data_end')['st_value'] + dump_names = re.compile(r'^\.(text|nex_data|rodata|ctors|got|data|' + r'data\.rel\.ro|ARM\.exidx|ARM\.extab)$') + tee_pager_bin = get_sections(elffile, pad_to, dump_names) + + return tee_pager_bin + + +def get_reloc_bin(elffile): + if get_arch_id(elffile) == 0: + exp_rel_type = ENUM_RELOC_TYPE_ARM['R_ARM_RELATIVE'] + else: + exp_rel_type = ENUM_RELOC_TYPE_AARCH64['R_AARCH64_RELATIVE'] + + link_address = get_symbol(elffile, '__text_start')['st_value'] + + addrs = [] + for section in elffile.iter_sections(): + if not isinstance(section, RelocationSection): + continue + for rel in section.iter_relocations(): + if rel['r_info_type'] == 0: + continue + if rel['r_info_type'] != exp_rel_type: + eprint("Unexpected relocation type 0x%x" % + rel['r_info_type']) + sys.exit(1) + addrs.append(rel['r_offset'] - link_address) + + addrs.sort() + data = bytearray() + for a in addrs: + data += struct.pack('> 32 + init_load_addr_lo = init_load_addr & 0xffffffff + return init_load_addr_hi, init_load_addr_lo + + +def output_raw_bin(elffile, outf): + pager_bin = get_pager_bin(elffile) + pageable_bin = get_pageable_bin(elffile) + embdata_bin = get_embdata_bin(elffile) + init_bin_size = get_symbol(elffile, '__init_size')['st_value'] + + outf.write(pager_bin) + outf.write(pageable_bin[:init_bin_size]) + outf.write(embdata_bin) + outf.write(pageable_bin[init_bin_size:]) + + +def output_header_v1(elffile, outf): + arch_id = get_arch_id(elffile) + pager_bin = get_pager_bin(elffile) + pageable_bin = get_pageable_bin(elffile) + embdata_bin = get_embdata_bin(elffile) + init_load_addr = get_init_load_addr(elffile) + init_bin_size = get_symbol(elffile, '__init_size')['st_value'] + pager_bin_size = len(pager_bin) + paged_area_size = len(pageable_bin) + + init_mem_usage = (get_symbol(elffile, '__get_tee_init_end')['st_value'] - + get_symbol(elffile, '__text_start')['st_value'] + + len(embdata_bin)) + + init_size = (pager_bin_size + min(init_bin_size, paged_area_size) + + len(embdata_bin)) + paged_size = paged_area_size - min(init_bin_size, paged_area_size) + + magic = 0x4554504f # 'OPTE' + version = 1 + flags = 0 + outf.write(struct.pack('.*) ') +REVIEWED_RE = re.compile(r'^Reviewed-by: (?P.*>)') +ACKED_RE = re.compile(r'^Acked-by: (?P.*>)') +PATCH_START = re.compile(r'^From [0-9a-f]{40}') + + +def get_args(): + parser = argparse.ArgumentParser(description='Print the maintainers for ' + 'the given source files or directories; ' + 'or for the files modified by a patch or ' + 'a pull request. ' + '(With -m) Check if a patch or pull ' + 'request is properly Acked/Reviewed for ' + 'merging.') + parser.add_argument('-m', '--merge-check', action='store_true', + help='use Reviewed-by: and Acked-by: tags found in ' + 'patches to prevent display of information for all ' + 'the approved paths.') + parser.add_argument('-p', '--show-paths', action='store_true', + help='show all paths that are not approved.') + parser.add_argument('-s', '--strict', action='store_true', + help='stricter conditions for patch approval check: ' + 'subsystem "THE REST" is ignored for paths that ' + 'match some other subsystem.') + parser.add_argument('arg', nargs='*', help='file or patch') + parser.add_argument('-f', '--file', action='append', + help='treat following argument as a file path, not ' + 'a patch.') + parser.add_argument('-g', '--github-pr', action='append', type=int, + help='Github pull request ID. The script will ' + 'download the patchset from Github to a temporary ' + 'file and process it.') + parser.add_argument('-r', '--release-to', action='store_true', + help='show all the recipients to be used in release ' + 'announcement emails (i.e., maintainers, reviewers ' + 'and OP-TEE mailing list(s)) and exit.') + return parser.parse_args() + + +def check_cwd(): + cwd = os.getcwd() + parent = os.path.dirname(os.path.realpath(__file__)) + "/../" + if (os.path.realpath(cwd) != os.path.realpath(parent)): + print("Error: this script must be run from the top-level of the " + "optee_os tree") + exit(1) + + +# Parse MAINTAINERS and return a dictionary of subsystems such as: +# {'Subsystem name': {'R': ['foo', 'bar'], 'S': ['Maintained'], +# 'F': [ 'path1', 'path2' ]}, ...} +def parse_maintainers(): + subsystems = {} + check_cwd() + with open("MAINTAINERS", "r") as f: + start_found = False + ss = {} + name = '' + for line in f: + line = line.strip() + if not line: + continue + if not start_found: + if line.startswith("----------"): + start_found = True + continue + + if line[1] == ':': + letter = line[0] + if (not ss.get(letter)): + ss[letter] = [] + ss[letter].append(line[3:]) + else: + if name: + subsystems[name] = ss + name = line + ss = {} + if name: + subsystems[name] = ss + + return subsystems + + +# If @patchset is a patchset files and contains 2 patches or more, write +# individual patches to temporary files and return the paths. +# Otherwise return []. +def split_patchset(patchset): + psname = os.path.basename(patchset).replace('.', '_') + patchnum = 0 + of = None + ret = [] + f = None + try: + f = open(patchset, "r") + except OSError: + return [] + for line in f: + match = re.search(PATCH_START, line) + if match: + # New patch found: create new file + patchnum += 1 + prefix = "{}_{}_".format(patchnum, psname) + of = tempfile.NamedTemporaryFile(mode="w", prefix=prefix, + suffix=".patch", + delete=False) + ret.append(of.name) + if of: + of.write(line) + if len(ret) >= 2: + return ret + if len(ret) == 1: + os.remove(ret[0]) + return [] + + +# If @path is a patch file, returns the paths touched by the patch as well +# as the content of the review/ack tags +def get_paths_from_patch(patch): + paths = [] + approvers = [] + try: + with open(patch, "r") as f: + for line in f: + match = re.search(DIFF_GIT_RE, line) + if match: + p = match.group('path') + if p not in paths: + paths.append(p) + continue + match = re.search(REVIEWED_RE, line) + if match: + a = match.group('approver') + if a not in approvers: + approvers.append(a) + continue + match = re.search(ACKED_RE, line) + if match: + a = match.group('approver') + if a not in approvers: + approvers.append(a) + continue + except Exception: + pass + return (paths, approvers) + + +# Does @path match @pattern? +# @pattern has the syntax defined in the Linux MAINTAINERS file -- mostly a +# shell glob pattern, except that a trailing slash means a directory and +# everything below. Matching can easily be done by converting to a regexp. +def match_pattern(path, pattern): + # Append a trailing slash if path is an existing directory, so that it + # matches F: entries such as 'foo/bar/' + if not path.endswith('/') and os.path.isdir(path): + path += '/' + rep = "^" + pattern + rep = rep.replace('*', '[^/]+') + rep = rep.replace('?', '[^/]') + if rep.endswith('/'): + rep += '.*' + rep += '$' + return not not re.match(rep, path) + + +def get_subsystems_for_path(subsystems, path, strict): + found = {} + for key in subsystems: + def inner(): + excluded = subsystems[key].get('X') + if excluded: + for pattern in excluded: + if match_pattern(path, pattern): + return # next key + included = subsystems[key].get('F') + if not included: + return # next key + for pattern in included: + if match_pattern(path, pattern): + found[key] = subsystems[key] + inner() + if strict and len(found) > 1: + found.pop('THE REST', None) + return found + + +def get_ss_maintainers(subsys): + return subsys.get('M') or [] + + +def get_ss_reviewers(subsys): + return subsys.get('R') or [] + + +def get_ss_approvers(ss): + return get_ss_maintainers(ss) + get_ss_reviewers(ss) + + +def get_ss_lists(subsys): + return subsys.get('L') or [] + + +def approvers_have_approved(approved_by, approvers): + for n in approvers: + # Ignore anything after the email (Github ID...) + n = n.split('>', 1)[0] + for m in approved_by: + m = m.split('>', 1)[0] + if n == m: + return True + return False + + +def download(pr): + url = "https://github.com/OP-TEE/optee_os/pull/{}.patch".format(pr) + f = tempfile.NamedTemporaryFile(mode="wb", prefix="pr{}_".format(pr), + suffix=".patch", delete=False) + print("Downloading {}...".format(url), end='', flush=True) + f.write(urlopen(url).read()) + print(" Done.") + return f.name + + +def show_release_to(subsystems): + check_cwd() + with open("MAINTAINERS", "r") as f: + emails = sorted(set(re.findall(r'[RM]:\t(.*[\w]*<[\w\.-]+@[\w\.-]+>)', + f.read()))) + emails += get_ss_lists(subsystems["THE REST"]) + print(*emails, sep=', ') + + +def main(): + global args + + args = get_args() + + all_subsystems = parse_maintainers() + + if args.release_to: + show_release_to(all_subsystems) + return + + paths = [] + arglist = [] + downloads = [] + split_patches = [] + + for pr in args.github_pr or []: + downloads += [download(pr)] + + for arg in args.arg + downloads: + if os.path.exists(arg): + patches = split_patchset(arg) + if patches: + split_patches += patches + continue + arglist.append(arg) + + for arg in arglist + split_patches: + patch_paths = [] + approved_by = [] + if os.path.exists(arg): + # Try to parse as a patch + (patch_paths, approved_by) = get_paths_from_patch(arg) + if not patch_paths: + # Not a patch, consider the path itself + # as_posix() cleans the path a little bit (suppress leading ./ and + # duplicate slashes...) + patch_paths = [PurePath(arg).as_posix()] + for path in patch_paths: + approved = False + if args.merge_check: + ss_for_path = get_subsystems_for_path(all_subsystems, path, + args.strict) + for key in ss_for_path: + ss_approvers = get_ss_approvers(ss_for_path[key]) + if approvers_have_approved(approved_by, ss_approvers): + approved = True + if not approved: + paths += [path] + + for f in downloads + split_patches: + os.remove(f) + + if args.file: + paths += args.file + + if (args.show_paths): + print(paths) + + ss = {} + for path in paths: + ss.update(get_subsystems_for_path(all_subsystems, path, args.strict)) + for key in ss: + ss_name = key[:50] + (key[50:] and '...') + for name in ss[key].get('M') or []: + print("{} (maintainer:{})".format(name, ss_name)) + for name in ss[key].get('R') or []: + print("{} (reviewer:{})".format(name, ss_name)) + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/mem_usage.py b/optee_os/scripts/mem_usage.py new file mode 100755 index 0000000..e4c4c97 --- /dev/null +++ b/optee_os/scripts/mem_usage.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2014-2017, Linaro Limited +# +# SPDX-License-Identifier: BSD-3-Clause + +import argparse +import os +import subprocess +import sys + + +def get_args(): + parser = argparse.ArgumentParser(description='Shows the memory usage ' + 'of an OP-TEE based on ELF sections') + parser.add_argument('tee_elf', help='the OP-TEE ELF file (tee.elf)') + parser.add_argument('-a', '--all', action='store_true', + help=' same as -i -p -u -U') + parser.add_argument('-n', '--no-map', action='store_true', + help=' do not show the detailed section mappings and ' + 'RAM usage') + parser.add_argument('-i', '--init', action='store_true', + help='report the total size of the .*_init sections') + parser.add_argument('-p', '--paged', action='store_true', + help='report the total size of the .*_pageable ' + 'sections') + parser.add_argument('-u', '--unpaged', action='store_true', + help='report the total size of the unpaged sections, ' + 'that is, all sections but the ones in --init or ' + '--paged') + parser.add_argument('-U', '--unpaged-no-heap', action='store_true', + help='report the size of all unpaged sections ' + 'excluding heap space. Reflects the size of unpaged ' + 'code and data (.text, .rodata, .data, .bss, .nozi ' + 'and possibly unwind tables)') + parser.add_argument('-r', '--raw', action='store_true', + help='when processing -i, -p, -u, or -U, show only ' + 'the size (in decimal) and no other text') + return parser.parse_args() + + +def printf(format, *args): + sys.stdout.write(format % args) + + +def print_sect(name, addr, size, round_up=False, print_num_pages=False): + if args.no_map: + return + if size == 0: + size_kib = 0 + num_pages = 0 + else: + if round_up: + size_kib = (size - 1) / 1024 + 1 + else: + size_kib = size / 1024 + num_pages = (size - 1) / 4096 + 1 + + printf('%-16s %.8X - %.8X size %.8X %3d KiB', name, addr, addr + size, + size, size_kib) + if print_num_pages: + printf(' %d pages', num_pages) + printf('\n') + + +def print_pager_stat(name, size): + size_kib = size / 1024 + if args.raw: + printf('%d ', size) + else: + printf('%-36s size %.8X %3d KiB\n', name, size, size_kib) + + +def readelf_cmd(): + return os.getenv('CROSS_COMPILE', '') + 'readelf' + + +def main(): + global args + + in_shdr = False + sects = [] + init_size = 0 + paged_size = 0 + unpaged_size = 0 + unpaged_no_heap_size = 0 + + args = get_args() + env = os.environ.copy() + env['LC_ALL'] = 'C' + readelf = subprocess.Popen(str.split(readelf_cmd()) + ['-s', + args.tee_elf], + stdout=subprocess.PIPE, env=env, + universal_newlines=True) + for line in iter(readelf.stdout.readline, ''): + words = line.split() + if len(words) == 8 and words[7] == '_end_of_ram': + end_of_ram = int(words[1], 16) + break + readelf.terminate() + readelf = subprocess.Popen(str.split(readelf_cmd()) + ['-S', '-W', + args.tee_elf], + stdout=subprocess.PIPE, env=env, + universal_newlines=True) + for line in iter(readelf.stdout.readline, ''): + if 'Section Headers:' in line: + in_shdr = True + continue + if 'Key to Flags:' in line: + in_shdr = False + continue + if in_shdr: + words = line.split() + if words[0] == '[': + words.pop(0) + try: + (_, name, _, addr, offs, size, _, + flags) = words[:8] + except BaseException: + continue + if ('A' in flags): + sects.append({'name': name, 'addr': addr, + 'offs': offs, 'size': size}) + first_addr = None + for sect in sects: + if sect['addr'] != 0: + addr = sect['addr'] + if not first_addr: + first_addr = addr + if int(addr, 16) >= end_of_ram: + break + last_addr = addr + last_size = sect['size'] + + ram_usage = int(last_addr, 16) + int(last_size, 16) - int(first_addr, 16) + print_sect('RAM Usage', int(first_addr, 16), ram_usage, True, True) + + last_addr = 0 + last_size = 0 + for sect in sects: + name = sect['name'] + addr = int(sect['addr'], 16) + size = int(sect['size'], 16) + + if addr >= end_of_ram: + break + if last_addr != 0 and addr != last_addr + last_size: + print_sect('*hole*', last_addr + last_size, + addr - (last_addr + last_size)) + print_sect(name, addr, size) + if name.endswith('_init'): + init_size += size + elif name.endswith('_pageable'): + paged_size += size + else: + if not name.startswith('.heap'): + unpaged_no_heap_size += size + unpaged_size += size + last_addr = addr + last_size = size + + if args.all or args.init: + print_pager_stat('Init sections (.*_init)', init_size) + if args.all or args.paged: + print_pager_stat('Paged sections (.*_pageable)', paged_size) + if args.all or args.unpaged: + print_pager_stat('Unpaged sections ', unpaged_size) + if args.all or args.unpaged_no_heap: + print_pager_stat('Unpaged sections (heap excluded)', + unpaged_no_heap_size) + if (args.raw and (args.all or args.init or args.paged or + args.unpaged or args.unpaged_no_heap)): + printf('\n') + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/pem_to_pub_c.py b/optee_os/scripts/pem_to_pub_c.py new file mode 100755 index 0000000..dbdf2c4 --- /dev/null +++ b/optee_os/scripts/pem_to_pub_c.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2015, Linaro Limited + + +def get_args(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + '--prefix', required=True, + help='Prefix for the public key exponent and modulus in c file') + parser.add_argument( + '--out', required=True, + help='Name of c file for the public key') + parser.add_argument('--key', required=True, help='Name of key file') + + return parser.parse_args() + + +def main(): + import array + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + args = get_args() + + with open(args.key, 'rb') as f: + data = f.read() + + try: + key = serialization.load_pem_private_key(data, password=None, + backend=default_backend()) + key = key.public_key() + except ValueError: + key = serialization.load_pem_public_key(data, + backend=default_backend()) + + # Refuse public exponent with more than 32 bits. Otherwise the C + # compiler may simply truncate the value and proceed. + # This will lead to TAs seemingly having invalid signatures with a + # possible security issue for any e = k*2^32 + 1 (for any integer k). + if key.public_numbers().e > 0xffffffff: + raise ValueError( + 'Unsupported large public exponent detected. ' + + 'OP-TEE handles only public exponents up to 2^32 - 1.') + + with open(args.out, 'w') as f: + f.write("#include \n") + f.write("#include \n\n") + f.write("const uint32_t " + args.prefix + "_exponent = " + + str(key.public_numbers().e) + ";\n\n") + f.write("const uint8_t " + args.prefix + "_modulus[] = {\n") + i = 0 + nbuf = key.public_numbers().n.to_bytes(key.key_size >> 3, 'big') + for x in array.array("B", nbuf): + f.write("0x" + '{0:02x}'.format(x) + ",") + i = i + 1 + if i % 8 == 0: + f.write("\n") + else: + f.write(" ") + f.write("};\n") + f.write("const size_t " + args.prefix + "_modulus_size = sizeof(" + + args.prefix + "_modulus);\n") + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/print_tee_hash.py b/optee_os/scripts/print_tee_hash.py new file mode 100755 index 0000000..7b4aac1 --- /dev/null +++ b/optee_os/scripts/print_tee_hash.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2021, Huawei Technologies Co., Ltd +# + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from elftools.elf.elffile import ELFFile +from elftools.elf.sections import SymbolTableSection +import codecs +import sys + +verbose = False + + +def dump(buf): + print(codecs.encode(buf, 'hex').decode('utf-8')) + + +def resolve_symbol(elf, name): + for section in elf.iter_sections(): + if isinstance(section, SymbolTableSection): + for symbol in section.iter_symbols(): + if symbol.name == name: + return symbol.entry['st_value'] + raise RuntimeError(f'Symbol {name} not found') + + +def hash_range(h, elf, start, end): + global verbose + start_addr = resolve_symbol(elf, start) + end_addr = resolve_symbol(elf, end) + size = end_addr - start_addr + if verbose: + print(f'[{start}(0x{start_addr:x}), {end}(0x{end_addr:x})]: ' + f'{size} bytes') + for segment in elf.iter_segments(): + if (segment['p_type'] == 'PT_LOAD' and + segment['p_vaddr'] <= start_addr and + end_addr <= segment['p_vaddr'] + segment['p_filesz']): + begin_offs = start_addr - segment['p_vaddr'] + h.update(segment.data()[begin_offs:begin_offs + size]) + + +def hash_section(h, elf, name): + global verbose + s = elf.get_section_by_name(name) + if s is None: + return + d = s.data() + if verbose: + print(f'{name}: {len(d)} bytes') + h.update(d) + + +def main(): + global verbose + argc = len(sys.argv) + if argc != 2 and argc != 3: + print('Usage:', sys.argv[0], '') + return 1 + + if argc == 3 and sys.argv[1] == '-v': + verbose = True + + with open(sys.argv[argc - 1], 'rb') as f: + elf = ELFFile(f) + h = hashes.Hash(hashes.SHA256(), default_backend()) + hash_range(h, elf, '__text_start', '__text_data_start') + hash_range(h, elf, '__text_data_end', '__text_end') + hash_section(h, elf, '.text_init') + hash_section(h, elf, '.text_pageable') + hash_section(h, elf, '.rodata') + hash_section(h, elf, '.rodata_init') + hash_section(h, elf, '.rodata_pageable') + dump(h.finalize()) + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/sign_encrypt.py b/optee_os/scripts/sign_encrypt.py new file mode 100755 index 0000000..435d63b --- /dev/null +++ b/optee_os/scripts/sign_encrypt.py @@ -0,0 +1,968 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2015, 2017, 2019, Linaro Limited +# + +import sys +import math + + +sig_tee_alg = {'TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256': 0x70414930, + 'TEE_ALG_RSASSA_PKCS1_V1_5_SHA256': 0x70004830} + +enc_tee_alg = {'TEE_ALG_AES_GCM': 0x40000810} + +enc_key_type = {'SHDR_ENC_KEY_DEV_SPECIFIC': 0x0, + 'SHDR_ENC_KEY_CLASS_WIDE': 0x1} + +TEE_ATTR_RSA_MODULUS = 0xD0000130 +TEE_ATTR_RSA_PUBLIC_EXPONENT = 0xD0000230 + +SHDR_BOOTSTRAP_TA = 1 +SHDR_ENCRYPTED_TA = 2 +SHDR_SUBKEY = 3 +SHDR_MAGIC = 0x4f545348 +SHDR_SIZE = 20 +SK_HDR_SIZE = 20 +EHDR_SIZE = 12 +UUID_SIZE = 16 +# Use 12 bytes for nonce per recommendation +NONCE_SIZE = 12 +TAG_SIZE = 16 + + +def value_to_key(db, val): + for k, v in db.items(): + if v == val: + return k + + +def uuid_v5_sha512(namespace_bytes, name): + from cryptography.hazmat.primitives import hashes + from uuid import UUID + + h = hashes.Hash(hashes.SHA512()) + h.update(namespace_bytes + bytes(name, 'utf-8')) + digest = h.finalize() + return UUID(bytes=digest[:16], version=5) + + +def name_img_to_str(name_img): + return name_img.decode().split('\x00', 1)[0] + + +def uuid_parse(s): + from uuid import UUID + return UUID(s) + + +def int_parse(str): + return int(str, 0) + + +def get_args(): + import argparse + import textwrap + + class OnlyOne(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + a = self.dest + '_assigned' + if getattr(namespace, a, False): + raise argparse.ArgumentError(self, 'Can only be given once') + setattr(namespace, a, True) + setattr(namespace, self.dest, values) + + def arg_add_uuid(parser): + parser.add_argument( + '--uuid', required=True, type=uuid_parse, + help='String UUID of the TA') + + def arg_add_key(parser): + parser.add_argument( + '--key', required=True, help=''' + Name of signing and verification key file (PEM format) or an + Amazon Resource Name (arn:) of an AWS KMS asymmetric key. + At least public key for the commands digest, stitch, and + verify, else a private key''') + + def arg_add_enc_key(parser): + parser.add_argument( + '--enc-key', required=False, help='Encryption key string') + + def arg_add_enc_key_type(parser): + parser.add_argument( + '--enc-key-type', required=False, + default='SHDR_ENC_KEY_DEV_SPECIFIC', + choices=list(enc_key_type.keys()), help=''' + Encryption key type, + Defaults to SHDR_ENC_KEY_DEV_SPECIFIC.''') + + def arg_add_ta_version(parser): + parser.add_argument( + '--ta-version', required=False, type=int_parse, default=0, help=''' + TA version stored as a 32-bit unsigned integer and used for + rollback protection of TA install in the secure database. + Defaults to 0.''') + + def arg_add_sig(parser): + parser.add_argument( + '--sig', required=True, dest='sigf', + help='Name of signature input file, defaults to .sig') + + def arg_add_dig(parser): + parser.add_argument( + '--dig', required=True, dest='digf', + help='Name of digest output file, defaults to .dig') + + def arg_add_in(parser): + parser.add_argument( + '--in', required=False, dest='inf', help=''' + Name of application input file, defaults to + .stripped.elf''') + + def arg_add_out(parser): + parser.add_argument( + '--out', required=True, dest='outf', + help='Name of application output file, defaults to .ta') + + def arg_add_algo(parser): + parser.add_argument( + '--algo', required=False, choices=list(sig_tee_alg.keys()), + default='TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256', help=''' + The hash and signature algorithm. + Defaults to TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256.''') + + def arg_add_subkey(parser): + parser.add_argument( + '--subkey', action=OnlyOne, help='Name of subkey input file') + + def arg_add_name(parser): + parser.add_argument('--name', + help='Input name for subspace of a subkey') + + def arg_add_subkey_uuid_in(parser): + parser.add_argument( + '--in', required=True, dest='inf', + help='Name of subkey input file') + + def arg_add_max_depth(parser): + parser.add_argument( + '--max-depth', required=False, type=int_parse, help=''' + Max depth of subkeys below this subkey''') + + def arg_add_name_size(parser): + parser.add_argument( + '--name-size', required=True, type=int_parse, help=''' + Size of (unsigned) input name for subspace of a subkey. + Set to 0 to create an identity subkey (a subkey having + the same UUID as the next subkey or TA)''') + + def arg_add_subkey_version(parser): + parser.add_argument( + '--subkey-version', required=False, type=int_parse, default=0, + help='Subkey version used for rollback protection') + + def arg_add_subkey_in(parser): + parser.add_argument( + '--in', required=True, dest='inf', help=''' + Name of PEM file with the public key of the new subkey''') + + def arg_add_subkey_out(parser): + parser.add_argument( + '--out', required=True, dest='outf', + help='Name of subkey output file') + + def get_outf_default(parsed): + return str(parsed.uuid) + '.ta' + + def get_inf_default(parsed): + return str(parsed.uuid) + '.stripped.elf' + + def get_sigf_default(parsed): + return str(parsed.uuid) + '.sig' + + def get_digf_default(parsed): + return str(parsed.uuid) + '.dig' + + def assign_default_value(parsed, attr, func): + if hasattr(parsed, attr) and getattr(parsed, attr) is None: + setattr(parsed, attr, func(parsed)) + + parser = argparse.ArgumentParser( + description='Sign and encrypt (optional) a Trusted Application ' + + ' for OP-TEE.', + usage='%(prog)s ...', + epilog=' -h for detailed help') + subparsers = parser.add_subparsers( + title='valid commands, with possible aliases in ()', + dest='command', metavar='') + + parser_sign_enc = subparsers.add_parser( + 'sign-enc', prog=parser.prog + ' sign-enc', + help='Generate signed and optionally encrypted loadable TA image file') + parser_sign_enc.set_defaults(func=command_sign_enc) + arg_add_uuid(parser_sign_enc) + arg_add_ta_version(parser_sign_enc) + arg_add_in(parser_sign_enc) + arg_add_out(parser_sign_enc) + arg_add_key(parser_sign_enc) + arg_add_subkey(parser_sign_enc) + arg_add_name(parser_sign_enc) + arg_add_enc_key(parser_sign_enc) + arg_add_enc_key_type(parser_sign_enc) + arg_add_algo(parser_sign_enc) + + parser_digest = subparsers.add_parser( + 'digest', aliases=['generate-digest'], prog=parser.prog + ' digest', + formatter_class=argparse.RawDescriptionHelpFormatter, + help='Generate loadable TA binary image digest for offline signing', + epilog=textwrap.dedent('''\ + example offline signing command using OpenSSL for algorithm + TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + base64 -d .dig | \\ + openssl pkeyutl -sign -inkey .pem \\ + -pkeyopt digest:sha256 -pkeyopt rsa_padding_mode:pss \\ + -pkeyopt rsa_pss_saltlen:digest \\ + -pkeyopt rsa_mgf1_md:sha256 | \\ + base64 > .sig + + example offline signing command using OpenSSL for algorithm + TEE_ALG_RSASSA_PKCS1_V1_5_SHA256: + base64 -d .dig | \\ + openssl pkeyutl -sign -inkey .pem \\ + -pkeyopt digest:sha256 -pkeyopt rsa_padding_mode:pkcs1 | \\ + base64 > .sig + ''')) + parser_digest.set_defaults(func=command_digest) + arg_add_uuid(parser_digest) + arg_add_ta_version(parser_digest) + arg_add_in(parser_digest) + arg_add_key(parser_digest) + arg_add_enc_key(parser_digest) + arg_add_enc_key_type(parser_digest) + arg_add_algo(parser_digest) + arg_add_dig(parser_digest) + + parser_stitch = subparsers.add_parser( + 'stitch', aliases=['stitch-ta'], prog=parser.prog + ' stich', + help='Generate loadable signed and encrypted TA binary image file' + + ' from TA raw image and its signature') + parser_stitch.set_defaults(func=command_stitch) + arg_add_uuid(parser_stitch) + arg_add_ta_version(parser_stitch) + arg_add_in(parser_stitch) + arg_add_key(parser_stitch) + arg_add_out(parser_stitch) + arg_add_enc_key(parser_stitch) + arg_add_enc_key_type(parser_stitch) + arg_add_algo(parser_stitch) + arg_add_sig(parser_stitch) + + parser_verify = subparsers.add_parser( + 'verify', prog=parser.prog + ' verify', + help='Verify signed TA binary') + parser_verify.set_defaults(func=command_verify) + arg_add_uuid(parser_verify) + arg_add_in(parser_verify) + arg_add_key(parser_verify) + + parser_display = subparsers.add_parser( + 'display', prog=parser.prog + ' display', + help='Parses and displays a signed TA binary') + parser_display.set_defaults(func=command_display) + arg_add_in(parser_display) + + parser_subkey_uuid = subparsers.add_parser( + 'subkey-uuid', prog=parser.prog + ' subkey-uuid', + help='calculate the UUID of next TA or subkey') + parser_subkey_uuid.set_defaults(func=command_subkey_uuid) + arg_add_subkey_uuid_in(parser_subkey_uuid) + arg_add_name(parser_subkey_uuid) + + parser_sign_subkey = subparsers.add_parser( + 'sign-subkey', prog=parser.prog + ' sign-subkey', + help='Sign a subkey') + parser_sign_subkey.set_defaults(func=command_sign_subkey) + arg_add_name(parser_sign_subkey) + arg_add_subkey_in(parser_sign_subkey) + arg_add_uuid(parser_sign_subkey) + arg_add_key(parser_sign_subkey) + arg_add_subkey_out(parser_sign_subkey) + arg_add_max_depth(parser_sign_subkey) + arg_add_name_size(parser_sign_subkey) + arg_add_subkey(parser_sign_subkey) + arg_add_subkey_version(parser_sign_subkey) + arg_add_algo(parser_sign_subkey) + + argv = sys.argv[1:] + if (len(argv) > 0 and argv[0][0] == '-' and + argv[0] != '-h' and argv[0] != '--help'): + # The default sub-command is 'sign-enc' so add it to the parser + # if one is missing + argv = ['sign-enc'] + argv + + parsed = parser.parse_args(argv) + + if parsed.command is None: + parser.print_help() + sys.exit(1) + + # Set a few defaults if defined for the current command + assign_default_value(parsed, 'inf', get_inf_default) + assign_default_value(parsed, 'outf', get_outf_default) + assign_default_value(parsed, 'sigf', get_sigf_default) + assign_default_value(parsed, 'digf', get_digf_default) + + return parsed + + +def load_asymmetric_key_img(data): + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.serialization import ( + load_pem_private_key, load_pem_public_key) + + try: + return load_pem_private_key(data, password=None, + backend=default_backend()) + except ValueError: + return load_pem_public_key(data, backend=default_backend()) + + +def load_asymmetric_key(arg_key): + if arg_key.startswith('arn:'): + from sign_helper_kms import _RSAPrivateKeyInKMS + return _RSAPrivateKeyInKMS(arg_key) + else: + with open(arg_key, 'rb') as f: + return load_asymmetric_key_img(f.read()) + + +class BinaryImage: + def __init__(self, arg_inf, arg_key): + from cryptography.hazmat.primitives import hashes + + # Exactly what inf is holding isn't determined a this stage + if isinstance(arg_inf, str): + with open(arg_inf, 'rb') as f: + self.inf = f.read() + else: + self.inf = arg_inf + + if arg_key is None: + self.key = None + else: + if isinstance(arg_key, str): + self.key = load_asymmetric_key(arg_key) + else: + self.key = arg_key + self.sig_size = math.ceil(self.key.key_size / 8) + + self.chosen_hash = hashes.SHA256() + self.hash_size = self.chosen_hash.digest_size + + def __pack_img(self, img_type, sign_algo): + import struct + + self.sig_algo = sign_algo + self.img_type = img_type + self.shdr = struct.pack('= getattr(self, 'previous_max_depth')): + logger.error('Max depth of previous subkey is {} ' + .format(self.previous_max_depth) + + 'and the next value must be smaller') + sys.exit(1) + + def int_to_bytes(x: int) -> bytes: + return x.to_bytes((x.bit_length() + 8) // 8, 'big') + + n_bytes = int_to_bytes(subkey_pkey.public_numbers().n) + e_bytes = int_to_bytes(subkey_pkey.public_numbers().e) + attrs_end_offs = 16 + 5 * 4 + 2 * 3 * 4 + shdr_subkey = struct.pack(' 0: + # name_size is the previous subkey header + name_img = self.inf[offs:offs + name_size] + print(' next name: "{}"'.format(name_img_to_str(name_img))) + offs += name_size + print('Next header at offset: {} (0x{:x})' + .format(offs, offs)) + + shdr = self.inf[offs:offs + SHDR_SIZE] + [magic, img_type, img_size, algo_value, hash_size, + sig_size] = struct.unpack(' 0: + sk_image = BinaryImage(sk_image.next_inf, None) + sk_image.parse() + + if name is None: + name = '' + self.previous_max_depth = sk_image.max_depth + self.name_img = str.encode(name).ljust(sk_image.name_size, b'\0') + + def write(self, outf): + with open(outf, 'wb') as f: + if hasattr(self, 'subkey_img'): + f.write(self.subkey_img) + f.write(self.name_img) + f.write(self.shdr) + f.write(self.img_digest) + f.write(self.sig) + if hasattr(self, 'ta_uuid'): + f.write(self.ta_uuid) + f.write(self.ta_version) + if hasattr(self, 'ehdr'): + f.write(self.ehdr) + f.write(self.nonce) + f.write(self.tag) + f.write(self.ciphertext) + else: + f.write(self.img) + + +def load_ta_image(args): + ta_image = BinaryImage(args.inf, args.key) + + if args.enc_key: + ta_image.encrypt_ta(args.enc_key, args.enc_key_type, + args.algo, args.uuid, args.ta_version) + else: + ta_image.set_bootstrap_ta(args.algo, args.uuid, args.ta_version) + + return ta_image + + +def command_sign_enc(args): + ta_image = load_ta_image(args) + if args.subkey: + ta_image.add_subkey(args.subkey, args.name) + ta_image.sign() + ta_image.write(args.outf) + logger.info('Successfully signed application.') + + +def command_sign_subkey(args): + image = BinaryImage(args.inf, args.key) + if args.subkey: + image.add_subkey(args.subkey, args.name) + image.set_subkey(args.algo, args.name, args.uuid, args.subkey_version, + args.max_depth, args.name_size) + image.sign() + image.write(args.outf) + logger.info('Successfully signed subkey.') + + +def command_digest(args): + import base64 + + ta_image = load_ta_image(args) + with open(args.digf, 'wb+') as digfile: + digfile.write(base64.b64encode(ta_image.img_digest)) + + +def command_stitch(args): + ta_image = load_ta_image(args) + ta_image.add_signature(args.sigf) + ta_image.verify_signature() + ta_image.write(args.outf) + logger.info('Successfully applied signature.') + + +def command_verify(args): + import uuid + + image = BinaryImage(args.inf, args.key) + next_uuid = None + max_depth = -1 + while True: + image.parse() + if hasattr(image, 'subkey_hdr'): # Subkey + print('Subkey UUID: {}'.format(uuid.UUID(bytes=image.uuid))) + image.verify_signature() + image.verify_digest() + if next_uuid: + if uuid.UUID(bytes=image.uuid) != next_uuid: + raise Exception('UUID {} does not match {}' + .format(uuid.UUID(bytes=image.uuid), + next_uuid)) + if max_depth >= 0: + if image.max_depth < 0 or image.max_depth >= max_depth: + raise Exception('Invalid max_depth {} not less than {}' + .format(image.max_depth, max_depth)) + max_depth = image.max_depth + if len(image.next_inf) == 0: + logger.info('Subkey is correctly verified.') + return + if image.name_size > 0: + next_uuid = uuid_v5_sha512(image.uuid, + name_img_to_str(image.name_img)) + else: + next_uuid = image.uuid + image = BinaryImage(image.next_inf, image.subkey_key) + else: # TA + print('TA UUID: {}'.format(uuid.UUID(bytes=image.ta_uuid))) + if next_uuid: + if uuid.UUID(bytes=image.ta_uuid) != next_uuid: + raise Exception('UUID {} does not match {}' + .format(uuid.UUID(bytes=image.ta_uuid), + next_uuid)) + if hasattr(image, 'ciphertext'): + if args.enc_key is None: + logger.error('--enc_key needed to decrypt TA') + sys.exit(1) + image.decrypt_ta(args.enc_key) + image.verify_signature() + image.verify_digest() + image.verify_uuid(args.uuid) + logger.info('Trusted application is correctly verified.') + return + + +def command_display(args): + ta_image = BinaryImage(args.inf, None) + ta_image.display() + + +def command_subkey_uuid(args): + import uuid + + sk_image = BinaryImage(args.inf, None) + sk_image.parse() + if not hasattr(sk_image, 'next_inf'): + logger.error('Invalid subkey file') + sys.exit(1) + print('Subkey UUID: {}'.format(uuid.UUID(bytes=sk_image.uuid))) + while len(sk_image.next_inf) > 0: + sk_image = BinaryImage(sk_image.next_inf, None) + sk_image.parse() + print('Subkey UUID: {}'.format(uuid.UUID(bytes=sk_image.uuid))) + if args.name: + if len(args.name) > sk_image.name_size: + logger.error('Length of name ({}) '.format(len(args.name)) + + 'is larger than max name size ({})' + .format(sk_image.name_size)) + sys.exit(1) + print('Next subkey UUID: {}' + .format(uuid_v5_sha512(sk_image.uuid, args.name))) + else: + print('Next subkey UUID unchanged: {}' + .format(uuid.UUID(bytes=sk_image.uuid))) + + +def main(): + import logging + import os + + global logger + logging.basicConfig() + logger = logging.getLogger(os.path.basename(__file__)) + + args = get_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/sign_helper_kms.py b/optee_os/scripts/sign_helper_kms.py new file mode 100755 index 0000000..115795f --- /dev/null +++ b/optee_os/scripts/sign_helper_kms.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright Amazon.com Inc. or its affiliates +# +import typing + +import boto3 + +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ( + AsymmetricSignatureContext, + utils as asym_utils, +) +from cryptography.hazmat.primitives.asymmetric.padding import ( + AsymmetricPadding, + PKCS1v15, + PSS, +) +from cryptography.hazmat.primitives.asymmetric.rsa import ( + RSAPrivateKey, + RSAPrivateNumbers, + RSAPublicKey, +) + + +class _RSAPrivateKeyInKMS(RSAPrivateKey): + + def __init__(self, arn): + self.arn = arn + self.client = boto3.client('kms') + response = self.client.get_public_key(KeyId=self.arn) + + # Parse public key + self.public_key = serialization.load_der_public_key( + response['PublicKey']) + + @property + def key_size(self): + return self.public_key.key_size + + def public_key(self) -> RSAPublicKey: + return self.public_key + + def sign(self, data: bytes, padding: AsymmetricPadding, + algorithm: typing.Union[asym_utils.Prehashed, + hashes.HashAlgorithm] + ) -> bytes: + if isinstance(algorithm, asym_utils.Prehashed): + message_type = 'DIGEST' + else: + message_type = 'RAW' + + if isinstance(padding, PSS): + signing_alg = 'RSASSA_PSS_' + elif isinstance(padding, PKCS1v15): + signing_alg = 'RSASSA_PKCS1_V1_5_' + else: + raise TypeError("Unsupported padding") + + if (isinstance(algorithm._algorithm, hashes.SHA256) or + isinstance(algorithm, hashes.SHA256)): + signing_alg += 'SHA_256' + elif (isinstance(algorithm._algorithm, hashes.SHA384) or + isinstance(algorithm, hashes.SHA384)): + signing_alg += 'SHA_384' + elif (isinstance(algorithm._algorithm, hashes.SHA512) or + isinstance(algorithm, hashes.SHA512)): + signing_alg += 'SHA_512' + else: + raise TypeError("Unsupported hashing algorithm") + + response = self.client.sign( + KeyId=self.arn, Message=data, + MessageType=message_type, + SigningAlgorithm=signing_alg) + + return response['Signature'] + + # No need to implement these functions so we raise an exception + def signer( + self, padding: AsymmetricPadding, algorithm: hashes.HashAlgorithm + ) -> AsymmetricSignatureContext: + raise NotImplementedError + + def decrypt(self, ciphertext: bytes, padding: AsymmetricPadding) -> bytes: + raise NotImplementedError + + def private_numbers(self) -> RSAPrivateNumbers: + raise NotImplementedError + + def private_bytes( + self, + encoding: serialization.Encoding, + format: serialization.PrivateFormat, + encryption_algorithm: serialization.KeySerializationEncryption + ) -> bytes: + raise NotImplementedError diff --git a/optee_os/scripts/sign_rproc_fw.py b/optee_os/scripts/sign_rproc_fw.py new file mode 100755 index 0000000..7bff20b --- /dev/null +++ b/optee_os/scripts/sign_rproc_fw.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (C) 2023, STMicroelectronics +# + +try: + from elftools.elf.elffile import ELFFile + from elftools.elf.sections import SymbolTableSection + from elftools.elf.enums import ENUM_P_TYPE_ARM + from elftools.elf.enums import * +except ImportError: + print(""" +*** +ERROR: "pyelftools" python module is not installed or version < 0.25. +*** +""") + raise + +try: + from Cryptodome.Hash import SHA256 + from Cryptodome.Signature import pkcs1_15 + from Cryptodome.PublicKey import RSA + from Cryptodome.Signature import DSS + from Cryptodome.PublicKey import ECC +except ImportError: + print(""" + *** + ERROR: "pycryptodomex" python module should be installed. + *** + """) + raise + +import os +import sys +import struct +import logging +import binascii + +# Generated file structure: +# +# -----+-------------+ +# / | Magic | 32-bit word, magic value equal to +# / +-------------+ 0x3543A468 +# / +-------------+ +# / | version | 32-bit word, version of the format +# / +-------------+ +# +-----------+ +-------------+ +# | Header | | TLV size | 32-bit word, size of the TLV +# +-----------+ +-------------+ (aligned on 64-bit), in bytes. +# \ +-------------+ +# \ | sign size | 32-bit word, size of the signature +# \ +-------------+ (aligned on 64-bit), in bytes. +# \ +-------------+ +# \ | images size | 32-bit word, size of the images to +# -----+-------------+ load (aligned on 64-bit), in bytes. +# +# +-------------+ Information used to authenticate the +# | TLV | images and boot the remote processor, +# | | stored in Type-Length-Value format. +# +-------------+ 'Type' and 'Length' are 32-bit words. +# +# +-------------+ +# | Signature | Signature of the header and the TLV. +# +-------------+ +# +# +-------------+ +# | Firmware | +# | image 1 | +# +-------------+ +# ... +# +-------------+ +# | Firmware | +# | image n | +# +-------------+ + +# Generic type definitions +TLV_TYPES = { + 'SIGNTYPE': 0x00000001, # algorithm used for signature + 'HASHTYPE': 0x00000002, # algorithm used for computing segment's hash + 'NUM_IMG': 0x00000003, # number of images to load + 'IMGTYPE': 0x00000004, # array of type of images to load + 'IMGSIZE': 0x00000005, # array of the size of the images to load + 'HASHTABLE': 0x000000010, # segment hash table for authentication + 'PKEYINFO': 0x0000000011, # information to retrieve signature key +} + +GENERIC_TLV_TYPE_RANGE = range(0x00000000, 0x00010000) +PLATFORM_TLV_TYPE_RANGE = range(0x00010000, 0x00020000) + +HEADER_MAGIC = 0x3543A468 + +logging.basicConfig(stream=sys.stderr, level=logging.INFO) + +ENUM_HASH_TYPE = dict( + SHA256=1, +) + +ENUM_SIGNATURE_TYPE = dict( + RSA=1, + ECC=2, +) + +ENUM_BINARY_TYPE = dict( + ELF=1, +) + + +def dump_buffer(buf, step=16, name="", logger=logging.debug, indent=""): + logger("%s%s:" % (indent, name)) + for i in range(0, len(buf), step): + logger("%s " % (indent) + " ". + join(["%02X" % c for c in buf[i:i+step]])) + logger("\n") + + +class TLV(): + def __init__(self): + self.buf = bytearray() + self.tlvs = {} + + def add(self, kind, payload): + """ + Add a TLV record. Argument type is either the type scalar ID or a + matching string defined in TLV_TYPES. + """ + if isinstance(kind, int): + buf = struct.pack('II', kind, len(payload)) + else: + buf = struct.pack('II', TLV_TYPES[kind], len(payload)) + + # Ensure that each TLV is 64-bit aligned + align_64b = (len(payload) + len(buf)) % 8 + self.buf += buf + self.buf += payload + if align_64b: + self.buf += bytearray(8 - align_64b) + + def add_plat_tlv(self, cust_tlv): + # Get list of custom protected TLVs from the command-line + for tlv in cust_tlv: + type_id = int(tlv[0], 0) + + if type_id not in PLATFORM_TLV_TYPE_RANGE: + raise Exception('TLV %s not in range' % hex(type_id)) + + value = tlv[1] + if value.startswith('0x'): + int_val = int(value[2:], 16) + self.tlvs[type_id] = int_val.to_bytes(4, 'little') + else: + self.tlvs[type_id] = value.encode('utf-8') + + if self.tlvs is not None: + for type_id, value in self.tlvs.items(): + self.add(type_id, value) + + def get(self): + """ + Get a byte-array that concatenates all the TLV added. + """ + if len(self.buf) == 0: + return bytes() + return bytes(self.buf) + + +class RSA_Signature(object): + + def __init__(self, key): + self._hasher = SHA256.new() + self.signer = pkcs1_15.new(key) + + def hash_compute(self, segment): + self._hasher.update(segment) + + def sign(self): + return self.signer.sign(self._hasher) + + +class ECC_Signature(object): + + def __init__(self, key): + self._hasher = SHA256.new() + self.signer = DSS.new(key, 'fips-186-3') + + def hash_compute(self, segment): + self._hasher.update(segment) + + def sign(self): + return self.signer.sign(self._hasher) + + +Signature = { + 1: RSA_Signature, + 2: ECC_Signature, +} + + +class SegmentHashStruct: + pass + + +class SegmentHash(object): + ''' + Hash table based on Elf program segments + ''' + def __init__(self, img): + self._num_segments = img.num_segments() + self._pack_fmt = '<%dL' % 8 + self.img = img + self.hashProgTable = bytes() + self._offset = 0 + + def get_table(self): + ''' + Create a segment hash table containing for each segment: + - the segments header + - a hash of the segment + ''' + h = SHA256.new() + seg = SegmentHashStruct() + self.size = (h.digest_size + 32) * self._num_segments + logging.debug("hash section size %d" % self.size) + del h + self.buf = bytearray(self.size) + self._bufview_ = memoryview(self.buf) + + for i in range(self._num_segments): + h = SHA256.new() + segment = self.img.get_segment(i) + seg.header = self.img.get_segment(i).header + logging.debug("compute hash for segment offset %s" % seg.header) + h.update(segment.data()) + seg.hash = h.digest() + logging.debug("hash computed: %s" % seg.hash) + del h + struct.pack_into('0x[0-9a-f]+)') +# This gets the address from lines looking like this: +# E/TC:0 0x001044a8 +STACK_ADDR_RE = re.compile( + r'[UEIDFM]/(TC|LD):(\?*|[0-9]*) [0-9]* +(?P0x[0-9a-f]+)') +ABORT_ADDR_RE = re.compile(r'-abort at address (?P0x[0-9a-f]+)') +TA_PANIC_RE = re.compile(r'TA panicked with code (?P0x[0-9a-f]+)') +REGION_RE = re.compile(r'region +[0-9]+: va (?P0x[0-9a-f]+) ' + r'pa 0x[0-9a-f]+ size (?P0x[0-9a-f]+)' + r'( flags .{4} (\[(?P[0-9]+)\])?)?') +ELF_LIST_RE = re.compile(r'\[(?P[0-9]+)\] (?P[0-9a-f\-]+)' + r' @ (?P0x[0-9a-f\-]+)') +FUNC_GRAPH_RE = re.compile(r'Function graph') +GRAPH_ADDR_RE = re.compile(r'(?P0x[0-9a-f]+)') +GRAPH_RE = re.compile(r'}') + +epilog = ''' +This scripts reads an OP-TEE abort or panic message from stdin and adds debug +information to the output, such as ' at :' next to each +address in the call stack. Any message generated by OP-TEE and containing a +call stack can in principle be processed by this script. This currently +includes aborts and panics from the TEE core as well as from any TA. +The paths provided on the command line are used to locate the appropriate ELF +binary (tee.elf or Trusted Application). The GNU binutils (addr2line, objdump, +nm) are used to extract the debug info. If the CROSS_COMPILE environment +variable is set, it is used as a prefix to the binutils tools. That is, the +script will invoke $(CROSS_COMPILE)addr2line etc. If it is not set however, +the prefix will be determined automatically for each ELF file based on its +architecture. The resulting command is then expected to be found in the user's +PATH. + +OP-TEE abort and panic messages are sent to the secure console. They look like +the following: + + E/TC:0 User TA data-abort at address 0xffffdecd (alignment fault) + ... + E/TC:0 Call stack: + E/TC:0 0x4000549e + E/TC:0 0x40001f4b + E/TC:0 0x4000273f + E/TC:0 0x40005da7 + +Inspired by a script of the same name by the Chromium project. + +Sample usage: + + $ scripts/symbolize.py -d out/arm-plat-hikey/core -d ../optee_test/out/ta/* + + ^D + +Also, this script reads function graph generated for OP-TEE user TA from +/tmp/ftrace-.out file and resolves function addresses to corresponding +symbols. + +Sample usage: + + $ cat /tmp/ftrace-.out | scripts/symbolize.py -d .elf + + ^D +''' + +tee_result_names = { + '0xf0100001': 'TEE_ERROR_CORRUPT_OBJECT', + '0xf0100002': 'TEE_ERROR_CORRUPT_OBJECT_2', + '0xf0100003': 'TEE_ERROR_STORAGE_NOT_AVAILABLE', + '0xf0100004': 'TEE_ERROR_STORAGE_NOT_AVAILABLE_2', + '0xf0100006': 'TEE_ERROR_CIPHERTEXT_INVALID ', + '0xffff0000': 'TEE_ERROR_GENERIC', + '0xffff0001': 'TEE_ERROR_ACCESS_DENIED', + '0xffff0002': 'TEE_ERROR_CANCEL', + '0xffff0003': 'TEE_ERROR_ACCESS_CONFLICT', + '0xffff0004': 'TEE_ERROR_EXCESS_DATA', + '0xffff0005': 'TEE_ERROR_BAD_FORMAT', + '0xffff0006': 'TEE_ERROR_BAD_PARAMETERS', + '0xffff0007': 'TEE_ERROR_BAD_STATE', + '0xffff0008': 'TEE_ERROR_ITEM_NOT_FOUND', + '0xffff0009': 'TEE_ERROR_NOT_IMPLEMENTED', + '0xffff000a': 'TEE_ERROR_NOT_SUPPORTED', + '0xffff000b': 'TEE_ERROR_NO_DATA', + '0xffff000c': 'TEE_ERROR_OUT_OF_MEMORY', + '0xffff000d': 'TEE_ERROR_BUSY', + '0xffff000e': 'TEE_ERROR_COMMUNICATION', + '0xffff000f': 'TEE_ERROR_SECURITY', + '0xffff0010': 'TEE_ERROR_SHORT_BUFFER', + '0xffff0011': 'TEE_ERROR_EXTERNAL_CANCEL', + '0xffff300f': 'TEE_ERROR_OVERFLOW', + '0xffff3024': 'TEE_ERROR_TARGET_DEAD', + '0xffff3041': 'TEE_ERROR_STORAGE_NO_SPACE', + '0xffff3071': 'TEE_ERROR_MAC_INVALID', + '0xffff3072': 'TEE_ERROR_SIGNATURE_INVALID', + '0xffff5000': 'TEE_ERROR_TIME_NOT_SET', + '0xffff5001': 'TEE_ERROR_TIME_NEEDS_RESET', + } + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='Symbolizes OP-TEE abort dumps or function graphs', + epilog=epilog) + parser.add_argument('-d', '--dir', action='append', nargs='+', + help='Search for ELF file in DIR. tee.elf is needed ' + 'to decode a TEE Core or pseudo-TA abort, while ' + '.elf is required if a user-mode TA has ' + 'crashed. For convenience, ELF files may also be ' + 'given.') + parser.add_argument('-s', '--strip_path', nargs='?', + help='Strip STRIP_PATH from file paths (default: ' + 'current directory, use -s with no argument to show ' + 'full paths)', default=os.getcwd()) + + return parser.parse_args() + + +class Symbolizer(object): + def __init__(self, out, dirs, strip_path): + self._out = out + self._dirs = dirs + self._strip_path = strip_path + self._addr2line = None + self.reset() + + def my_Popen(self, cmd): + try: + return subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True, + bufsize=1) + except OSError as e: + if e.errno == errno.ENOENT: + print("*** Error:{}: command not found".format(cmd[0]), + file=sys.stderr) + sys.exit(1) + + def get_elf(self, elf_or_uuid): + if not elf_or_uuid.endswith('.elf'): + elf_or_uuid += '.elf' + for d in self._dirs: + if d.endswith(elf_or_uuid) and os.path.isfile(d): + return d + elf = glob.glob(d + '/' + elf_or_uuid) + if elf: + return elf[0] + + def set_arch(self, elf): + self._arch = os.getenv('CROSS_COMPILE') + if self._arch: + return + p = subprocess.Popen(['file', '-L', elf], stdout=subprocess.PIPE) + output = p.stdout.readlines() + p.terminate() + if b'ARM aarch64,' in output[0]: + self._arch = 'aarch64-linux-gnu-' + elif b'ARM,' in output[0]: + self._arch = 'arm-linux-gnueabihf-' + elif b'RISC-V,' in output[0]: + if b'32-bit' in output[0]: + self._arch = 'riscv32-unknown-linux-gnu-' + elif b'64-bit' in output[0]: + self._arch = 'riscv64-unknown-linux-gnu-' + + def arch_prefix(self, cmd, elf): + self.set_arch(elf) + if self._arch is None: + return '' + return self._arch + cmd + + def spawn_addr2line(self, elf_name): + if elf_name is None: + return + if self._addr2line_elf_name is elf_name: + return + if self._addr2line: + self._addr2line.terminate + self._addr2line = None + elf = self.get_elf(elf_name) + if not elf: + return + cmd = self.arch_prefix('addr2line', elf) + if not cmd: + return + self._addr2line = self.my_Popen([cmd, '-f', '-p', '-e', elf]) + self._addr2line_elf_name = elf_name + + # If addr falls into a region that maps a TA ELF file, return the load + # address of that file. + def elf_load_addr(self, addr): + if self._regions: + for r in self._regions: + r_addr = int(r[0], 16) + r_size = int(r[1], 16) + i_addr = int(addr, 16) + if (i_addr >= r_addr and i_addr < (r_addr + r_size)): + # Found region + elf_idx = r[2] + if elf_idx is not None: + return self._elfs[int(elf_idx)][1] + # In case address is not found in TA ELF file, fallback to tee.elf + # especially to symbolize mixed (user-space and kernel) addresses + # which is true when syscall ftrace is enabled along with TA + # ftrace. + return self._tee_load_addr + else: + # tee.elf + return self._tee_load_addr + + def elf_for_addr(self, addr): + l_addr = self.elf_load_addr(addr) + if l_addr == self._tee_load_addr: + return 'tee.elf' + for k in self._elfs: + e = self._elfs[k] + if int(e[1], 16) == int(l_addr, 16): + return e[0] + return None + + def subtract_load_addr(self, addr): + l_addr = self.elf_load_addr(addr) + if l_addr is None: + return None + if int(l_addr, 16) > int(addr, 16): + return '' + return '0x{:x}'.format(int(addr, 16) - int(l_addr, 16)) + + def resolve(self, addr): + reladdr = self.subtract_load_addr(addr) + self.spawn_addr2line(self.elf_for_addr(addr)) + if not reladdr or not self._addr2line: + return '???' + if self.elf_for_addr(addr) == 'tee.elf': + reladdr = '0x{:x}'.format(int(reladdr, 16) + + int(self.first_vma('tee.elf'), 16)) + try: + print(reladdr, file=self._addr2line.stdin) + ret = self._addr2line.stdout.readline().rstrip('\n') + except IOError: + ret = '!!!' + return ret + + def symbol_plus_offset(self, addr): + ret = '' + prevsize = 0 + reladdr = self.subtract_load_addr(addr) + elf_name = self.elf_for_addr(addr) + if elf_name is None: + return '' + elf = self.get_elf(elf_name) + if elf is None: + return '' + cmd = self.arch_prefix('nm', elf) + if not reladdr or not elf or not cmd: + return '' + ireladdr = int(reladdr, 16) + nm = self.my_Popen([cmd, '--numeric-sort', '--print-size', elf]) + for line in iter(nm.stdout.readline, ''): + try: + addr, size, _, name = line.split() + except ValueError: + # Size is missing + try: + addr, _, name = line.split() + size = '0' + except ValueError: + # E.g., undefined (external) symbols (line = "U symbol") + continue + iaddr = int(addr, 16) + isize = int(size, 16) + if iaddr == ireladdr: + ret = name + break + if iaddr < ireladdr and iaddr + isize >= ireladdr: + offs = ireladdr - iaddr + ret = name + '+' + str(offs) + break + if iaddr > ireladdr and prevsize == 0: + offs = iaddr + ireladdr + ret = prevname + '+' + str(offs) + break + prevsize = size + prevname = name + nm.terminate() + return ret + + def section_plus_offset(self, addr): + ret = '' + reladdr = self.subtract_load_addr(addr) + elf_name = self.elf_for_addr(addr) + if elf_name is None: + return '' + elf = self.get_elf(elf_name) + if elf is None: + return '' + cmd = self.arch_prefix('objdump', elf) + if not reladdr or not elf or not cmd: + return '' + iaddr = int(reladdr, 16) + objdump = self.my_Popen([cmd, '--section-headers', elf]) + for line in iter(objdump.stdout.readline, ''): + try: + idx, name, size, vma, lma, offs, algn = line.split() + except ValueError: + continue + ivma = int(vma, 16) + isize = int(size, 16) + if ivma == iaddr: + ret = name + break + if ivma < iaddr and ivma + isize >= iaddr: + offs = iaddr - ivma + ret = name + '+' + str(offs) + break + objdump.terminate() + return ret + + def process_abort(self, line): + ret = '' + match = re.search(ABORT_ADDR_RE, line) + addr = match.group('addr') + pre = match.start('addr') + post = match.end('addr') + sym = self.symbol_plus_offset(addr) + sec = self.section_plus_offset(addr) + if sym or sec: + ret += line[:pre] + ret += addr + if sym: + ret += ' ' + sym + if sec: + ret += ' ' + sec + ret += line[post:] + return ret + + # Return all ELF sections with the ALLOC flag + def read_sections(self, elf_name): + if elf_name is None: + return + if elf_name in self._sections: + return + elf = self.get_elf(elf_name) + if not elf: + return + cmd = self.arch_prefix('objdump', elf) + if not elf or not cmd: + return + self._sections[elf_name] = [] + objdump = self.my_Popen([cmd, '--section-headers', elf]) + for line in iter(objdump.stdout.readline, ''): + try: + _, name, size, vma, _, _, _ = line.split() + except ValueError: + if 'ALLOC' in line: + self._sections[elf_name].append([name, int(vma, 16), + int(size, 16)]) + + def first_vma(self, elf_name): + self.read_sections(elf_name) + return '0x{:x}'.format(self._sections[elf_name][0][1]) + + def overlaps(self, section, addr, size): + sec_addr = section[1] + sec_size = section[2] + if not size or not sec_size: + return False + return ((addr <= (sec_addr + sec_size - 1)) and + ((addr + size - 1) >= sec_addr)) + + def sections_in_region(self, addr, size, elf_idx): + ret = '' + addr = self.subtract_load_addr(addr) + if not addr: + return '' + iaddr = int(addr, 16) + isize = int(size, 16) + elf = self._elfs[int(elf_idx)][0] + if elf is None: + return '' + self.read_sections(elf) + if elf not in self._sections: + return '' + for s in self._sections[elf]: + if self.overlaps(s, iaddr, isize): + ret += ' ' + s[0] + return ret + + def reset(self): + self._call_stack_found = False + if self._addr2line: + self._addr2line.terminate() + self._addr2line = None + self._addr2line_elf_name = None + self._arch = None + self._saved_abort_line = '' + self._sections = {} # {elf_name: [[name, addr, size], ...], ...} + self._regions = [] # [[addr, size, elf_idx, saved line], ...] + self._elfs = {0: ["tee.elf", 0]} # {idx: [uuid, load_addr], ...} + self._tee_load_addr = '0x0' + self._func_graph_found = False + self._func_graph_skip_line = True + + def pretty_print_path(self, path): + if self._strip_path: + return re.sub(re.escape(self._strip_path) + '/*', '', path) + return path + + def write(self, line): + if self._call_stack_found: + match = re.search(STACK_ADDR_RE, line) + if match: + addr = match.group('addr') + pre = match.start('addr') + post = match.end('addr') + self._out.write(line[:pre]) + self._out.write(addr) + # The call stack contains return addresses (LR/ELR values). + # Heuristic: subtract 2 to obtain the call site of the function + # or the location of the exception. This value works for A64, + # A32 as well as Thumb. + pc = 0 + lr = int(addr, 16) + if lr: + pc = lr - 2 + res = self.resolve('0x{:x}'.format(pc)) + res = self.pretty_print_path(res) + self._out.write(' ' + res) + self._out.write(line[post:]) + return + else: + self.reset() + if self._func_graph_found: + match = re.search(GRAPH_ADDR_RE, line) + match_re = re.search(GRAPH_RE, line) + if match: + addr = match.group('addr') + pre = match.start('addr') + post = match.end('addr') + self._out.write(line[:pre]) + res = self.resolve(addr) + res_arr = re.split(' ', res) + self._out.write(res_arr[0]) + self._out.write(line[post:]) + self._func_graph_skip_line = False + return + elif match_re: + self._out.write(line) + return + elif self._func_graph_skip_line: + return + else: + self.reset() + match = re.search(REGION_RE, line) + if match: + # Region table: save info for later processing once + # we know which UUID corresponds to which ELF index + addr = match.group('addr') + size = match.group('size') + elf_idx = match.group('elf_idx') + self._regions.append([addr, size, elf_idx, line]) + return + match = re.search(ELF_LIST_RE, line) + if match: + # ELF list: save info for later. Region table and ELF list + # will be displayed when the call stack is reached + i = int(match.group('idx')) + self._elfs[i] = [match.group('uuid'), match.group('load_addr'), + line] + return + match = re.search(TA_PANIC_RE, line) + if match: + code = match.group('code') + if code in tee_result_names: + line = line.strip() + ' (' + tee_result_names[code] + ')\n' + self._out.write(line) + return + match = re.search(TEE_LOAD_ADDR_RE, line) + if match: + self._tee_load_addr = match.group('load_addr') + match = re.search(CALL_STACK_RE, line) + if match: + self._call_stack_found = True + if self._regions: + for r in self._regions: + r_addr = r[0] + r_size = r[1] + elf_idx = r[2] + saved_line = r[3] + if elf_idx is None: + self._out.write(saved_line) + else: + self._out.write(saved_line.strip() + + self.sections_in_region(r_addr, + r_size, + elf_idx) + + '\n') + if self._elfs: + for k in self._elfs: + e = self._elfs[k] + if (len(e) >= 3): + # TA executable or library + self._out.write(e[2].strip()) + elf = self.get_elf(e[0]) + if elf: + rpath = os.path.realpath(elf) + path = self.pretty_print_path(rpath) + self._out.write(' (' + path + ')') + self._out.write('\n') + # Here is a good place to resolve the abort address because we + # have all the information we need + if self._saved_abort_line: + self._out.write(self.process_abort(self._saved_abort_line)) + match = re.search(FUNC_GRAPH_RE, line) + if match: + self._func_graph_found = True + match = re.search(ABORT_ADDR_RE, line) + if match: + self.reset() + # At this point the arch and TA load address are unknown. + # Save the line so We can translate the abort address later. + self._saved_abort_line = line + self._out.write(line) + + def flush(self): + self._out.flush() + + +def main(): + args = get_args() + if args.dir: + # Flatten list in case -d is used several times *and* with multiple + # arguments + args.dirs = [item for sublist in args.dir for item in sublist] + else: + args.dirs = [] + symbolizer = Symbolizer(sys.stdout, args.dirs, args.strip_path) + + fd = sys.stdin.fileno() + isatty = os.isatty(fd) + if isatty: + old = termios.tcgetattr(fd) + new = termios.tcgetattr(fd) + new[3] = new[3] & ~termios.ECHO # lflags + try: + if isatty: + termios.tcsetattr(fd, termios.TCSADRAIN, new) + for line in sys.stdin: + symbolizer.write(line) + finally: + symbolizer.flush() + if isatty: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/tee_bin_parser.py b/optee_os/scripts/tee_bin_parser.py new file mode 100755 index 0000000..04bfb9f --- /dev/null +++ b/optee_os/scripts/tee_bin_parser.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2016, Linaro Limited +import struct +import argparse + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument('--input', required=False, dest='inf', + default='../out/arm/core/tee.bin', + help='The input tee.bin') + + return parser.parse_args() + + +def main(): + args = get_args() + + with open(args.inf, "rb") as f: + data = f.read(4) + magic = struct.unpack('\n\n') + f.write('#include \n\n') + f.write('const uint8_t ts_bin_' + ts_uuid.hex + '[] = {\n') + ts_size, ts_uncompressed_size = dump_bin(f, ts, args.compress) + f.write('};\n') + + if is_sp: + + f.write('#include \n\n') + f.write('const uint8_t fdt_bin_' + ts_uuid.hex + '[] = {\n') + dump_bin(f, args.manifest, False) + f.write('};\n') + f.write('SCATTERED_ARRAY_DEFINE_PG_ITEM(sp_images, struct \ + sp_image) = {\n') + f.write('\t.fdt = fdt_bin_' + ts_uuid.hex + ',\n') + + f.write('. image = {') + f.write('\t.flags = 0x{:04x},\n'.format(sp_get_flags(ts))) + else: + f.write('SCATTERED_ARRAY_DEFINE_PG_ITEM(early_tas, struct \ + embedded_ts) = {\n') + f.write('\t.flags = 0x{:04x},\n'.format(ta_get_flags(ts))) + f.write('\t.uuid = {\n') + f.write('\t\t.timeLow = 0x{:08x},\n'.format(ts_uuid.time_low)) + f.write('\t\t.timeMid = 0x{:04x},\n'.format(ts_uuid.time_mid)) + f.write('\t\t.timeHiAndVersion = ' + + '0x{:04x},\n'.format(ts_uuid.time_hi_version)) + f.write('\t\t.clockSeqAndNode = {\n') + csn = '{0:02x}{1:02x}{2:012x}'.format(ts_uuid.clock_seq_hi_variant, + ts_uuid.clock_seq_low, ts_uuid.node) + f.write('\t\t\t') + f.write(', '.join('0x' + csn[i:i + 2] for i in range(0, len(csn), 2))) + f.write('\n\t\t},\n\t},\n') + f.write('\t.size = sizeof(ts_bin_' + ts_uuid.hex + + '), /* {:d} */\n'.format(ts_size)) + f.write('\t.ts = ts_bin_' + ts_uuid.hex + ',\n') + if args.compress: + f.write('\t.uncompressed_size = ' + '{:d},\n'.format(ts_uncompressed_size)) + if is_sp: + f.write('}\n') + f.write('};\n') + f.close() + + +if __name__ == "__main__": + main() diff --git a/optee_os/scripts/update_changelog.py b/optee_os/scripts/update_changelog.py new file mode 100755 index 0000000..89e78ff --- /dev/null +++ b/optee_os/scripts/update_changelog.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2019, Linaro Limited +# + +from subprocess import Popen, PIPE +import argparse + + +def get_args(): + parser = argparse.ArgumentParser(description='Helper script that updates ' + 'the CHANGELOG.md file.\n' + 'Usage example:\n' + ' ./update_changelog.py ' + ' --changelog-file CHANGELOG.md' + ' --release-version 3.7.0' + ' --previous-release-version 3.6.0' + ' --release-date 2019-10-11') + + parser.add_argument('--changelog-file', action='store', required=False, + default='CHANGELOG.md', + help='Changelog file to be updated.') + + parser.add_argument('--release-date', action='store', required=True, + help='The release date (yyyy-mm-dd).') + + parser.add_argument('--release-version', action='store', required=True, + help='Release version (MAJOR.MINOR.PATCH).') + + parser.add_argument('--previous-release-version', action='store', + required=True, + help='Previous release version (MAJOR.MINOR.PATCH).') + + return parser.parse_args() + + +def prepend_write(filename, text): + with open(filename, 'r+') as f: + current_content = f.read() + f.seek(0, 0) + f.write(text + '\n' + current_content) + f.flush() + + +def get_previous_release_date(tag): + cmd = "git log -1 --date=format:%Y-%m-%d --format=format:%cd " \ + "{}".format(tag) + process = Popen(cmd.split(), stdout=PIPE) + (output, err) = process.communicate() + return output.decode("utf-8") + + +def main(): + global args + + args = get_args() + + gits = ["OP-TEE/optee_os", "OP-TEE/optee_client", "OP-TEE/optee_test", + "OP-TEE/build", "linaro-swg/optee_examples"] + + # Shorten name + clf = args.changelog_file + rv = args.release_version + prv = args.previous_release_version + rd = args.release_date + prd = get_previous_release_date(prv) + + # In some cases we need underscore in string + rvu = rv.replace('.', '_') + + text = "# OP-TEE - version {} ({})\n".format(rv, rd) + text += "\n" + text += "- Links to the release pages, commits and pull requests merged " \ + "into this release for:\n" + + for g in gits: + gu = g.replace('/', '_') + gu = gu.replace('-', '_') + text += " - {}: [release page][{}_release_{}], " \ + "[commits][{}_commits_{}] and [pull requests]" \ + "[{}_pr_{}]\n".format(g, gu, rvu, gu, rvu, gu, rvu) + + text += "\n" + + for g in gits: + gu = g.replace('/', '_') + gu = gu.replace('-', '_') + text += "\n[{}_release_{}]: https://github.com/{}/releases/tag/" \ + "{}\n".format(gu, rvu, g, rv) + text += "[{}_commits_{}]: https://github.com/{}/compare/" \ + "{}...{}\n".format(gu, rvu, g, prv, rv) + text += "[{}_pr_{}]: https://github.com/{}/pulls?q=is%3Apr+is%3A" \ + "merged+base%3Amaster+merged%3A{}..{}\n".format( + gu, rvu, g, prd, rd) + + prepend_write(args.changelog_file, text) + + +if __name__ == "__main__": + main() diff --git a/optee_os/ta/arch/arm/ta.ld.S b/optee_os/ta/arch/arm/ta.ld.S new file mode 100644 index 0000000..de0fa61 --- /dev/null +++ b/optee_os/ta/arch/arm/ta.ld.S @@ -0,0 +1,103 @@ +#ifdef ARM32 +OUTPUT_FORMAT("elf32-littlearm") +OUTPUT_ARCH(arm) +#define MCOUNT_SYM __gnu_mcount_nc +#endif +#ifdef ARM64 +OUTPUT_FORMAT("elf64-littleaarch64") +OUTPUT_ARCH(aarch64) +#define MCOUNT_SYM _mcount +#endif + +#ifndef CFG_FTRACE_BUF_SIZE +#define CFG_FTRACE_BUF_SIZE 2048 +#endif + +SECTIONS { + .text : { + __text_start = .; + *(.text .text.*) + *(.stub) + *(.glue_7) + *(.glue_7t) + *(.gnu.linkonce.t.*) + /* Workaround for an erratum in ARM's VFP11 coprocessor */ + *(.vfp11_veneer) + __text_end = .; + } + .note.gnu.property : { *(.note.gnu.property) } + .plt : { *(.plt) } + + .eh_frame_hdr : { + *(.eh_frame_hdr) + *(.eh_frame_entry .eh_frame_entry.*) + } + .eh_frame : { KEEP(*(.eh_frame)) *(.eh_frame.*) } + .gcc_except_table : { *(.gcc_except_table .gcc_except_table.*) } + .rodata : { + *(.gnu.linkonce.r.*) + *(.rodata .rodata.*) + } + .ARM.extab : { *(.ARM.extab* .gnu.linkonce.armextab.*) } + /* .ARM.exidx is sorted, so has to go in its own output section. */ + PROVIDE_HIDDEN(__exidx_start = .); + .ARM.exidx : { *(.ARM.exidx* .gnu.linkonce.armexidx.*) } + PROVIDE_HIDDEN(__exidx_end = .); + .ctors : { *(.ctors) } + .dtors : { *(.dtors) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .hash : { *(.hash) } + + /* Page align to allow dropping execute bit for RW data */ + . = ALIGN(4096); + + .dynamic : { *(.dynamic) } + .tdata : { *(.tdata .tdata.* .gnu.linkonce.td.*) } + .tbss : { *(.tbss .tbss.* .gnu.linkonce.tb.*) *(.tcommon) } + .got : { *(.got.plt) *(.got) } + .rel.text : { *(.rel.text) *(.rel.gnu.linkonce.t*) } + .rela.text : { *(.rela.text) *(.rela.gnu.linkonce.t*) } + .rel.data : { *(.rel.data) *(.rel.gnu.linkonce.d*) } + .rela.data : { *(.rela.data) *(.rela.gnu.linkonce.d*) } + .rel.tdata : { *(.rel.tdata .rel.tdata.* .rel.gnu.linkonce.td.*) } + .rela.tdata : { *(.rela.tdata .rela.tdata.* .rela.gnu.linkonce.td.*) } + .rel.tbss : { *(.rel.tbss .rel.tbss.* .rel.gnu.linkonce.tb.*) } + .rela.tbss : { *(.rela.tbss .rela.tbss.* .rela.gnu.linkonce.tb.*) } + .rel.rodata : { *(.rel.rodata) *(.rel.gnu.linkonce.r*) } + .rela.rodata : { *(.rela.rodata) *(.rela.gnu.linkonce.r*) } + .rel.dyn : { *(.rel.dyn) } + .rel.got : { *(.rel.got) } + .rela.got : { *(.rela.got) } + .rel.ctors : { *(.rel.ctors) } + .rela.ctors : { *(.rela.ctors) } + .rel.dtors : { *(.rel.dtors) } + .rela.dtors : { *(.rela.dtors) } + .rel.init : { *(.rel.init) } + .rela.init : { *(.rela.init) } + .rel.fini : { *(.rel.fini) } + .rela.fini : { *(.rela.fini) } + .rel.bss : { *(.rel.bss) } + .rela.bss : { *(.rela.bss) } + .rel.plt : { *(.rel.plt) } + .rela.plt : { *(.rela.plt) } + + .data : { *(.data .data.* .gnu.linkonce.d.*) } + .bss : { + *(.bss .bss.* .gnu.linkonce.b.* COMMON) + + /* + * TA tracing using ftrace + * Reserve some space for the ftrace buffer, only if the + * TA is instrumented (i.e., some files were built with -pg). + */ + . = ALIGN(8); + __ftrace_buf_start = .; + . += DEFINED(MCOUNT_SYM) ? + CFG_FTRACE_BUF_SIZE : 0; + __ftrace_buf_end = .; + } + + /DISCARD/ : { *(.interp) } +} + diff --git a/optee_os/ta/arch/arm/ta_entry_a32.S b/optee_os/ta/arch/arm/ta_entry_a32.S new file mode 100644 index 0000000..cd9a12f --- /dev/null +++ b/optee_os/ta/arch/arm/ta_entry_a32.S @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019, Linaro Limited + */ + +#include + + .section .note.GNU-stack,"",%progbits + +/* + * This function is the bottom of the user call stack. Mark it as such so that + * the unwinding code won't try to go further down. + * We need an assembly wrapper because Clang does not accept asm(".cantunwind") + * in a C function: + * + * user_ta_header.c:44:6: error: .fnstart must precede .cantunwind directive + * asm(".cantunwind"); + * ^ + */ +FUNC __ta_entry, : +UNWIND( .cantunwind) + bl __ta_entry_c +END_FUNC __ta_entry diff --git a/optee_os/ta/arch/riscv/ta.ld.S b/optee_os/ta/arch/riscv/ta.ld.S new file mode 100644 index 0000000..f807675 --- /dev/null +++ b/optee_os/ta/arch/riscv/ta.ld.S @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +#ifdef RV32 +OUTPUT_FORMAT("elf32-littleriscv") +OUTPUT_ARCH(riscv) +#endif +#ifdef RV64 +OUTPUT_FORMAT("elf64-littleriscv") +OUTPUT_ARCH(riscv) +#endif + +#define MCOUNT_SYM _mcount + +#ifndef CFG_FTRACE_BUF_SIZE +#define CFG_FTRACE_BUF_SIZE 2048 +#endif + +SECTIONS { + .ta_head : {*(.ta_head)} + .text : { + __text_start = .; + *(.text .text.*) + *(.stub) + *(.gnu.linkonce.t.*) + __text_end = .; + } + .note.gnu.property : { *(.note.gnu.property) } + .plt : { *(.plt) } + + .eh_frame_hdr : { + *(.eh_frame_hdr) + *(.eh_frame_entry .eh_frame_entry.*) + } + .eh_frame : { KEEP(*(.eh_frame)) *(.eh_frame.*) } + .gcc_except_table : { *(.gcc_except_table .gcc_except_table.*) } + .rodata : { + *(.gnu.linkonce.r.*) + *(.rodata .rodata.*) + } + .ctors : { *(.ctors) } + .dtors : { *(.dtors) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .hash : { *(.hash) } + + /* Page align to allow dropping execute bit for RW data */ + . = ALIGN(4096); + + .dynamic : { *(.dynamic) } + .tdata : { *(.tdata .tdata.* .gnu.linkonce.td.*) } + .tbss : { *(.tbss .tbss.* .gnu.linkonce.tb.*) *(.tcommon) } + .got : { *(.got.plt) *(.got) } + .rel.text : { *(.rel.text) *(.rel.gnu.linkonce.t*) } + .rela.text : { *(.rela.text) *(.rela.gnu.linkonce.t*) } + .rel.data : { *(.rel.data) *(.rel.gnu.linkonce.d*) } + .rela.data : { *(.rela.data) *(.rela.gnu.linkonce.d*) } + .rel.tdata : { *(.rel.tdata .rel.tdata.* .rel.gnu.linkonce.td.*) } + .rela.tdata : { *(.rela.tdata .rela.tdata.* .rela.gnu.linkonce.td.*) } + .rel.tbss : { *(.rel.tbss .rel.tbss.* .rel.gnu.linkonce.tb.*) } + .rela.tbss : { *(.rela.tbss .rela.tbss.* .rela.gnu.linkonce.tb.*) } + .rel.rodata : { *(.rel.rodata) *(.rel.gnu.linkonce.r*) } + .rela.rodata : { *(.rela.rodata) *(.rela.gnu.linkonce.r*) } + .rel.dyn : { *(.rel.dyn) } + .rel.got : { *(.rel.got) } + .rela.got : { *(.rela.got) } + .rel.ctors : { *(.rel.ctors) } + .rela.ctors : { *(.rela.ctors) } + .rel.dtors : { *(.rel.dtors) } + .rela.dtors : { *(.rela.dtors) } + .rel.init : { *(.rel.init) } + .rela.init : { *(.rela.init) } + .rel.fini : { *(.rel.fini) } + .rela.fini : { *(.rela.fini) } + .rel.bss : { *(.rel.bss) } + .rela.bss : { *(.rela.bss) } + .rel.plt : { *(.rel.plt) } + .rela.plt : { *(.rela.plt) } + + .data : { *(.data .data.* .gnu.linkonce.d.*) } + .bss : { + *(.bss .bss.* .gnu.linkonce.b.* COMMON) + + /* + * TA tracing using ftrace + * Reserve some space for the ftrace buffer, only if the + * TA is instrumented (i.e., some files were built with -pg). + */ + . = ALIGN(8); + __ftrace_buf_start = .; + . += DEFINED(MCOUNT_SYM) ? + CFG_FTRACE_BUF_SIZE : 0; + __ftrace_buf_end = .; + } + + /DISCARD/ : { *(.interp) } +} + diff --git a/optee_os/ta/avb/Makefile b/optee_os/ta/avb/Makefile new file mode 100644 index 0000000..7c8d2bd --- /dev/null +++ b/optee_os/ta/avb/Makefile @@ -0,0 +1,18 @@ +# The UUID for the Trusted Application +BINARY=023f8f1a-292a-432b-8fc4-de8471358067 + +ifdef TA_CROSS_COMPILE +CROSS_COMPILE ?= $(TA_CROSS_COMPILE) +endif +export CROSS_COMPILE + +CFG_TEE_TA_LOG_LEVEL ?= 2 +CPPFLAGS += -DCFG_TEE_TA_LOG_LEVEL=$(CFG_TEE_TA_LOG_LEVEL) + +-include $(TA_DEV_KIT_DIR)/mk/ta_dev_kit.mk + +ifeq ($(wildcard $(TA_DEV_KIT_DIR)/mk/ta_dev_kit.mk), ) +clean: + @echo 'Note: $$(TA_DEV_KIT_DIR)/mk/ta_dev_kit.mk not found, cannot clean TA' + @echo 'Note: TA_DEV_KIT_DIR=$(TA_DEV_KIT_DIR)' +endif diff --git a/optee_os/ta/avb/entry.c b/optee_os/ta/avb/entry.c new file mode 100644 index 0000000..a257c11 --- /dev/null +++ b/optee_os/ta/avb/entry.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* Copyright (c) 2018, Linaro Limited */ + +#include +#include +#include + +#include +#include + +#define DEFAULT_LOCK_STATE 0 + +static const uint32_t storageid = TEE_STORAGE_PRIVATE_RPMB; +static const char rb_obj_name[] = "rb_state"; +static const char *named_value_prefix = "named_value_"; + +static TEE_Result get_slot_offset(size_t slot, size_t *offset) +{ + if (slot >= TA_AVB_MAX_ROLLBACK_LOCATIONS) + return TEE_ERROR_BAD_PARAMETERS; + + *offset = sizeof(uint32_t) /* lock_state */ + slot * sizeof(uint64_t); + return TEE_SUCCESS; +} + +static TEE_Result create_rb_state(uint32_t lock_state, TEE_ObjectHandle *h) +{ + const uint32_t flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE | + TEE_DATA_FLAG_OVERWRITE; + + return TEE_CreatePersistentObject(storageid, rb_obj_name, + sizeof(rb_obj_name), flags, NULL, + &lock_state, sizeof(lock_state), h); +} + +static TEE_Result open_rb_state(uint32_t default_lock_state, + TEE_ObjectHandle *h) +{ + uint32_t flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE; + TEE_Result res; + + res = TEE_OpenPersistentObject(storageid, rb_obj_name, + sizeof(rb_obj_name), flags, h); + if (!res) + return TEE_SUCCESS; + + return create_rb_state(default_lock_state, h); +} + +static TEE_Result get_named_object_name(char *name_orig, + uint32_t name_orig_size, + char *name, uint32_t *name_size) +{ + size_t pref_len = strlen(named_value_prefix); + + if (name_orig_size + pref_len > + TEE_OBJECT_ID_MAX_LEN) + return TEE_ERROR_BAD_PARAMETERS; + + /* Start with prefix */ + TEE_MemMove(name, named_value_prefix, pref_len); + + /* Concatenate provided object name */ + TEE_MemMove(name + pref_len, name_orig, name_orig_size); + + *name_size = name_orig_size + pref_len; + + return TEE_SUCCESS; +} + +static TEE_Result read_rb_idx(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + size_t slot_offset; + uint64_t idx; + size_t count; + TEE_Result res; + TEE_ObjectHandle h; + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + res = get_slot_offset(params[0].value.a, &slot_offset); + if (res) + return res; + + res = open_rb_state(DEFAULT_LOCK_STATE, &h); + if (res) + return res; + + res = TEE_SeekObjectData(h, slot_offset, TEE_DATA_SEEK_SET); + if (res) + goto out; + + res = TEE_ReadObjectData(h, &idx, sizeof(idx), &count); + if (res) + goto out; + if (count != sizeof(idx)) { + idx = 0; /* Not yet written slots are reported as 0 */ + + if (count) { + /* + * Somehow the file didn't even hold a complete + * slot index entry. Write it as 0. + */ + res = TEE_SeekObjectData(h, slot_offset, + TEE_DATA_SEEK_SET); + if (res) + goto out; + res = TEE_WriteObjectData(h, &idx, sizeof(idx)); + if (res) + goto out; + } + } + + params[1].value.a = idx >> 32; + params[1].value.b = idx; +out: + TEE_CloseObject(h); + return res; +} + +static TEE_Result write_rb_idx(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + size_t slot_offset; + uint64_t widx; + uint64_t idx; + size_t count; + TEE_Result res; + TEE_ObjectHandle h; + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + res = get_slot_offset(params[0].value.a, &slot_offset); + if (res) + return res; + widx = ((uint64_t)params[1].value.a << 32) | params[1].value.b; + + res = open_rb_state(DEFAULT_LOCK_STATE, &h); + if (res) + return res; + + res = TEE_SeekObjectData(h, slot_offset, TEE_DATA_SEEK_SET); + if (res) + goto out; + + res = TEE_ReadObjectData(h, &idx, sizeof(idx), &count); + if (res) + goto out; + if (count != sizeof(idx)) + idx = 0; /* Not yet written slots are reported as 0 */ + + if (widx < idx) { + res = TEE_ERROR_SECURITY; + goto out; + } + + res = TEE_SeekObjectData(h, slot_offset, TEE_DATA_SEEK_SET); + if (res) + goto out; + + res = TEE_WriteObjectData(h, &widx, sizeof(widx)); +out: + TEE_CloseObject(h); + return res; +} + +static TEE_Result read_lock_state(uint32_t pt, TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + uint32_t lock_state; + size_t count; + TEE_Result res; + TEE_ObjectHandle h; + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + res = open_rb_state(DEFAULT_LOCK_STATE, &h); + if (res) + return res; + + res = TEE_ReadObjectData(h, &lock_state, sizeof(lock_state), &count); + if (res) + goto out; + if (count != sizeof(lock_state)) { + /* + * Client need write the lock state to recover, this can + * normally not happen. + */ + res = TEE_ERROR_CORRUPT_OBJECT; + goto out; + } + + params[0].value.a = lock_state; +out: + TEE_CloseObject(h); + return res; +} + +static TEE_Result write_lock_state(uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_VALUE_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + uint32_t wlock_state; + uint32_t lock_state; + size_t count; + TEE_Result res; + TEE_ObjectHandle h; + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + wlock_state = params[0].value.a; + + res = open_rb_state(wlock_state, &h); + if (res) + return res; + + res = TEE_ReadObjectData(h, &lock_state, sizeof(lock_state), &count); + if (res) + goto out; + if (count == sizeof(lock_state) && lock_state == wlock_state) + goto out; + + res = TEE_SeekObjectData(h, 0, TEE_DATA_SEEK_SET); + if (res) + goto out; + + res = TEE_WriteObjectData(h, &wlock_state, sizeof(wlock_state)); +out: + TEE_CloseObject(h); + return res; +} + +static TEE_Result write_persist_value(uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + const uint32_t flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE | + TEE_DATA_FLAG_OVERWRITE; + char name_full[TEE_OBJECT_ID_MAX_LEN] = { }; + TEE_ObjectHandle h = TEE_HANDLE_NULL; + TEE_Result res = TEE_SUCCESS; + uint32_t name_full_sz = 0; + uint32_t name_buf_sz = 0; + uint32_t value_sz = 0; + char *name_buf = NULL; + char *value = NULL; + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + name_buf = params[0].memref.buffer; + name_buf_sz = params[0].memref.size; + value_sz = params[1].memref.size; + value = TEE_Malloc(value_sz, 0); + if (!value) + return TEE_ERROR_OUT_OF_MEMORY; + + TEE_MemMove(value, params[1].memref.buffer, value_sz); + + res = get_named_object_name(name_buf, name_buf_sz, + name_full, &name_full_sz); + if (res) + goto out; + + res = TEE_CreatePersistentObject(storageid, name_full, + name_full_sz, + flags, NULL, value, + value_sz, &h); + if (res) + EMSG("Can't create named object value, res = 0x%x", res); + + TEE_CloseObject(h); +out: + TEE_Free(value); + + return res; +} + +static TEE_Result read_persist_value(uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + uint32_t flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE; + TEE_Result res = TEE_SUCCESS; + TEE_ObjectHandle h = TEE_HANDLE_NULL; + char name_full[TEE_OBJECT_ID_MAX_LEN]; + uint32_t name_full_sz = 0; + uint32_t name_buf_sz = 0; + char *name_buf = NULL; + uint32_t value_sz = 0; + char *value = NULL; + size_t count = 0; + + if (pt != exp_pt) + return TEE_ERROR_BAD_PARAMETERS; + + name_buf = params[0].memref.buffer; + name_buf_sz = params[0].memref.size; + value_sz = params[1].memref.size; + value = TEE_Malloc(value_sz, 0); + if (!value) + return TEE_ERROR_OUT_OF_MEMORY; + + res = get_named_object_name(name_buf, name_buf_sz, + name_full, &name_full_sz); + if (res) + goto out_free; + + res = TEE_OpenPersistentObject(storageid, name_full, + name_full_sz, flags, &h); + if (res) { + EMSG("Can't open named object value, res = 0x%x", res); + goto out_free; + } + + res = TEE_ReadObjectData(h, value, value_sz, &count); + if (res) { + EMSG("Can't read named object value, res = 0x%x", res); + goto out; + } + + TEE_MemMove(params[1].memref.buffer, value, + value_sz); + + params[1].memref.size = count; +out: + TEE_CloseObject(h); +out_free: + TEE_Free(value); + + return res; +} + +TEE_Result TA_CreateEntryPoint(void) +{ + return TEE_SUCCESS; +} + +void TA_DestroyEntryPoint(void) +{ +} + +TEE_Result TA_OpenSessionEntryPoint(uint32_t pt __unused, + TEE_Param params[4] __unused, + void **session __unused) +{ + return TEE_SUCCESS; +} + +void TA_CloseSessionEntryPoint(void *sess __unused) +{ +} + +TEE_Result TA_InvokeCommandEntryPoint(void *sess __unused, uint32_t cmd, + uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd) { + case TA_AVB_CMD_READ_ROLLBACK_INDEX: + return read_rb_idx(pt, params); + case TA_AVB_CMD_WRITE_ROLLBACK_INDEX: + return write_rb_idx(pt, params); + case TA_AVB_CMD_READ_LOCK_STATE: + return read_lock_state(pt, params); + case TA_AVB_CMD_WRITE_LOCK_STATE: + return write_lock_state(pt, params); + case TA_AVB_CMD_READ_PERSIST_VALUE: + return read_persist_value(pt, params); + case TA_AVB_CMD_WRITE_PERSIST_VALUE: + return write_persist_value(pt, params); + default: + EMSG("Command ID 0x%x is not supported", cmd); + return TEE_ERROR_NOT_SUPPORTED; + } +} diff --git a/optee_os/ta/avb/include/ta_avb.h b/optee_os/ta/avb/include/ta_avb.h new file mode 100644 index 0000000..45521f7 --- /dev/null +++ b/optee_os/ta/avb/include/ta_avb.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2018, Linaro Limited */ + +#ifndef __TA_AVB_H +#define __TA_AVB_H + +#define TA_AVB_UUID { 0x023f8f1a, 0x292a, 0x432b, \ + { 0x8f, 0xc4, 0xde, 0x84, 0x71, 0x35, 0x80, 0x67 } } + +#define TA_AVB_MAX_ROLLBACK_LOCATIONS 256 + +/* + * Gets the rollback index corresponding to the given rollback index slot. + * + * in params[0].value.a: rollback index slot + * out params[1].value.a: upper 32 bits of rollback index + * out params[1].value.b: lower 32 bits of rollback index + */ +#define TA_AVB_CMD_READ_ROLLBACK_INDEX 0 + +/* + * Updates the rollback index corresponding to the given rollback index slot. + * + * Will refuse to update a slot with a lower value. + * + * in params[0].value.a: rollback index slot + * in params[1].value.a: upper 32 bits of rollback index + * in params[1].value.b: lower 32 bits of rollback index + */ +#define TA_AVB_CMD_WRITE_ROLLBACK_INDEX 1 + +/* + * Gets the lock state of the device. + * + * out params[0].value.a: lock state + */ +#define TA_AVB_CMD_READ_LOCK_STATE 2 + +/* + * Sets the lock state of the device. + * + * If the lock state is changed all rollback slots will be reset to 0 + * + * in params[0].value.a: lock state + */ +#define TA_AVB_CMD_WRITE_LOCK_STATE 3 + +/* + * Reads a persistent value corresponding to the given name. + * + * in params[0].memref: persistent value name + * out params[1].memref: read persistent value buffer + */ +#define TA_AVB_CMD_READ_PERSIST_VALUE 4 + +/* + * Writes a persistent value corresponding to the given name. + * + * in params[0].memref: persistent value name + * in params[1].memref: persistent value buffer to write + */ +#define TA_AVB_CMD_WRITE_PERSIST_VALUE 5 + +#endif /*__TA_AVB_H*/ diff --git a/optee_os/ta/avb/sub.mk b/optee_os/ta/avb/sub.mk new file mode 100644 index 0000000..f1b35c0 --- /dev/null +++ b/optee_os/ta/avb/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += include +global-incdirs-y += . +srcs-y += entry.c diff --git a/optee_os/ta/avb/user_ta.mk b/optee_os/ta/avb/user_ta.mk new file mode 100644 index 0000000..c4cb2f7 --- /dev/null +++ b/optee_os/ta/avb/user_ta.mk @@ -0,0 +1,2 @@ +user-ta-uuid := 023f8f1a-292a-432b-8fc4-de8471358067 +user-ta-version := 0 diff --git a/optee_os/ta/avb/user_ta_header_defines.h b/optee_os/ta/avb/user_ta_header_defines.h new file mode 100644 index 0000000..21e49a7 --- /dev/null +++ b/optee_os/ta/avb/user_ta_header_defines.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Copyright (c) 2018, Linaro Limited */ + +#ifndef __USER_TA_HEADER_DEFINES_H +#define __USER_TA_HEADER_DEFINES_H + +#include + +#define TA_UUID TA_AVB_UUID + +#define TA_FLAGS (TA_FLAG_SINGLE_INSTANCE | \ + TA_FLAG_MULTI_SESSION) + +#define TA_STACK_SIZE (16 * 1024) +#define TA_DATA_SIZE (16 * 1024) + +#endif /*__USER_TA_HEADER_DEFINES_H*/ diff --git a/optee_os/ta/link.mk b/optee_os/ta/link.mk new file mode 100644 index 0000000..b00f220 --- /dev/null +++ b/optee_os/ta/link.mk @@ -0,0 +1,124 @@ +link-script$(sm) = $(ta-dev-kit-dir$(sm))/src/ta.ld.S +link-script-pp$(sm) = $(link-out-dir$(sm))/ta.lds +link-script-dep$(sm) = $(link-out-dir$(sm))/.ta.ld.d + +SIGN_ENC ?= $(PYTHON3) $(ta-dev-kit-dir$(sm))/scripts/sign_encrypt.py +TA_SIGN_KEY ?= $(ta-dev-kit-dir$(sm))/keys/default_ta.pem + +ifeq ($(CFG_ENCRYPT_TA),y) +# Default TA encryption key is a dummy key derived from default +# hardware unique key (an array of 16 zero bytes) to demonstrate +# usage of REE-FS TAs encryption feature. +# +# Note that a user of this TA encryption feature needs to provide +# encryption key and its handling corresponding to their security +# requirements. +TA_ENC_KEY ?= 'b64d239b1f3c7d3b06506229cd8ff7c8af2bb4db2168621ac62c84948468c4f4' +endif + +all: $(link-out-dir$(sm))/$(user-ta-uuid).dmp \ + $(link-out-dir$(sm))/$(user-ta-uuid).stripped.elf \ + $(link-out-dir$(sm))/$(user-ta-uuid).ta +cleanfiles += $(link-out-dir$(sm))/$(user-ta-uuid).elf +cleanfiles += $(link-out-dir$(sm))/$(user-ta-uuid).dmp +cleanfiles += $(link-out-dir$(sm))/$(user-ta-uuid).map +cleanfiles += $(link-out-dir$(sm))/$(user-ta-uuid).stripped.elf +cleanfiles += $(link-out-dir$(sm))/$(user-ta-uuid).ta +cleanfiles += $(link-script-pp$(sm)) $(link-script-dep$(sm)) + +link-ldflags = -e__ta_entry -pie +link-ldflags += -T $(link-script-pp$(sm)) +link-ldflags += -Map=$(link-out-dir$(sm))/$(user-ta-uuid).map +link-ldflags += --sort-section=alignment +link-ldflags += -z max-page-size=4096 # OP-TEE always uses 4K alignment +ifeq ($(sm)-$(CFG_TA_BTI),ta_arm64-y) +link-ldflags += $(call ld-option,-z force-bti) --fatal-warnings +endif +link-ldflags += --as-needed # Do not add dependency on unused shlib +link-ldflags += $(link-ldflags$(sm)) + +$(link-out-dir$(sm))/dyn_list: FORCE + @$(cmd-echo-silent) ' GEN $@' + $(q)mkdir -p $(dir $@) + $(q)echo "{" >$@.tmp + $(q)echo "__elf_phdr_info;" >>$@.tmp +ifeq ($(CFG_FTRACE_SUPPORT),y) + $(q)echo "__ftrace_info;" >>$@.tmp +endif + $(q)echo "trace_ext_prefix;" >>$@.tmp + $(q)echo "trace_level;" >>$@.tmp + $(q)echo "ta_head;" >>$@.tmp + $(q)echo "};" >>$@.tmp + $(q)$(call mv-if-changed,$@.tmp,$@) +link-ldflags += --dynamic-list $(link-out-dir$(sm))/dyn_list +dynlistdep = $(link-out-dir$(sm))/dyn_list +cleanfiles += $(link-out-dir$(sm))/dyn_list + +link-ldadd = $(user-ta-ldadd) $(addprefix -L,$(libdirs)) +link-ldadd += --start-group +link-ldadd += $(addprefix -l,$(libnames)) +ifneq (,$(filter %.cpp,$(srcs))) +link-ldflags += --eh-frame-hdr +link-ldadd += $(libstdc++$(sm)) $(libgcc_eh$(sm)) +endif +link-ldadd += --end-group + +link-ldadd-after-libgcc += $(addprefix -l,$(libnames-after-libgcc)) + +ldargs-$(user-ta-uuid).elf := $(link-ldflags) $(objs) $(link-ldadd) \ + $(libgcc$(sm)) $(link-ldadd-after-libgcc) + +link-script-cppflags-$(sm) := \ + $(filter-out $(CPPFLAGS_REMOVE) $(cppflags-remove), \ + $(nostdinc$(sm)) $(CPPFLAGS) \ + $(addprefix -I,$(incdirs$(sm)) $(link-out-dir$(sm))) \ + $(cppflags$(sm))) + +-include $(link-script-dep$(sm)) + +link-script-pp-makefiles$(sm) = $(filter-out %.d %.cmd,$(MAKEFILE_LIST)) + +define gen-link-t +$(link-script-pp$(sm)): $(link-script$(sm)) $(conf-file) $(link-script-pp-makefiles$(sm)) + @$(cmd-echo-silent) ' CPP $$@' + $(q)mkdir -p $$(dir $$@) + $(q)$(CPP$(sm)) -P -MT $$@ -MD -MF $(link-script-dep$(sm)) \ + $(link-script-cppflags-$(sm)) $$< -o $$@ + +$(link-out-dir$(sm))/$(user-ta-uuid).elf: $(objs) $(libdeps) \ + $(libdeps-after-libgcc) \ + $(link-script-pp$(sm)) \ + $(dynlistdep) \ + $(additional-link-deps) + @$(cmd-echo-silent) ' LD $$@' + $(q)$(LD$(sm)) $(ldargs-$(user-ta-uuid).elf) -o $$@ + +$(link-out-dir$(sm))/$(user-ta-uuid).dmp: \ + $(link-out-dir$(sm))/$(user-ta-uuid).elf + @$(cmd-echo-silent) ' OBJDUMP $$@' + $(q)$(OBJDUMP$(sm)) -l -x -d $$< > $$@ + +$(link-out-dir$(sm))/$(user-ta-uuid).stripped.elf: \ + $(link-out-dir$(sm))/$(user-ta-uuid).elf + @$(cmd-echo-silent) ' OBJCOPY $$@' + $(q)$(OBJCOPY$(sm)) --strip-unneeded $$< $$@ + +cmd-echo$(user-ta-uuid) := SIGN # +ifeq ($(CFG_ENCRYPT_TA),y) +crypt-args$(user-ta-uuid) := --enc-key $(TA_ENC_KEY) +cmd-echo$(user-ta-uuid) := SIGNENC +endif +$(link-out-dir$(sm))/$(user-ta-uuid).ta: \ + $(link-out-dir$(sm))/$(user-ta-uuid).stripped.elf \ + $(TA_SIGN_KEY) $(TA_SUBKEY_DEPS) \ + $(lastword $(SIGN_ENC)) + @$(cmd-echo-silent) ' $$(cmd-echo$(user-ta-uuid)) $$@' + $(q)$(SIGN_ENC) --key $(TA_SIGN_KEY) $(TA_SUBKEY_ARGS) \ + $$(crypt-args$(user-ta-uuid)) \ + --uuid $(user-ta-uuid) --ta-version $(user-ta-version) \ + --in $$< --out $$@ +endef + +$(eval $(call gen-link-t)) + +additional-link-deps := diff --git a/optee_os/ta/link_shlib.mk b/optee_os/ta/link_shlib.mk new file mode 100644 index 0000000..fdb6315 --- /dev/null +++ b/optee_os/ta/link_shlib.mk @@ -0,0 +1,54 @@ +ifeq (,$(shlibuuid)) +$(error SHLIBUUID not set) +endif +link-out-dir = $(out-dir) + +SIGN ?= $(TA_DEV_KIT_DIR)/scripts/sign_encrypt.py +TA_SIGN_KEY ?= $(TA_DEV_KIT_DIR)/keys/default_ta.pem + +all: $(link-out-dir)/$(shlibname).so $(link-out-dir)/$(shlibname).dmp \ + $(link-out-dir)/$(shlibname).stripped.so \ + $(link-out-dir)/$(shlibuuid).elf \ + $(link-out-dir)/$(shlibuuid).ta + +cleanfiles += $(link-out-dir)/$(shlibname).so +cleanfiles += $(link-out-dir)/$(shlibname).dmp +cleanfiles += $(link-out-dir)/$(shlibname).stripped.so +cleanfiles += $(link-out-dir)/$(shlibuuid).elf +cleanfiles += $(link-out-dir)/$(shlibuuid).ta + +shlink-ldflags = $(LDFLAGS) +shlink-ldflags += -shared -z max-page-size=4096 +shlink-ldflags += $(call ld-option,-z separate-loadable-segments) +ifeq ($(sm)-$(CFG_TA_BTI),ta_arm64-y) +shlink-ldflags += $(call ld-option,-z force-bti) --fatal-warnings +endif +shlink-ldflags += --as-needed # Do not add dependency on unused shlib + +shlink-ldadd = $(LDADD) +shlink-ldadd += $(addprefix -L,$(libdirs)) +shlink-ldadd += --start-group $(addprefix -l,$(libnames)) --end-group +ldargs-$(shlibname).so := $(shlink-ldflags) $(objs) $(shlink-ldadd) $(libgcc$(sm)) + + +$(link-out-dir)/$(shlibname).so: $(objs) $(libdeps) + @$(cmd-echo-silent) ' LD $@' + $(q)$(LD$(sm)) $(ldargs-$(shlibname).so) --soname=$(shlibuuid) -o $@ + +$(link-out-dir)/$(shlibname).dmp: $(link-out-dir)/$(shlibname).so + @$(cmd-echo-silent) ' OBJDUMP $@' + $(q)$(OBJDUMP$(sm)) -l -x -d $< > $@ + +$(link-out-dir)/$(shlibname).stripped.so: $(link-out-dir)/$(shlibname).so + @$(cmd-echo-silent) ' OBJCOPY $@' + $(q)$(OBJCOPY$(sm)) --strip-unneeded $< $@ + +$(link-out-dir)/$(shlibuuid).elf: $(link-out-dir)/$(shlibname).so + @$(cmd-echo-silent) ' LN $@' + $(q)ln -sf $( +#include + +#define PKCS11_TA_UUID { 0xfd02c9da, 0x306c, 0x48c7, \ + { 0xa4, 0x9c, 0xbb, 0xd8, 0x27, 0xae, 0x86, 0xee } } + +/* PKCS11 trusted application version information */ +#define PKCS11_TA_VERSION_MAJOR 0 +#define PKCS11_TA_VERSION_MINOR 1 +#define PKCS11_TA_VERSION_PATCH 0 + +/* Attribute specific values */ +#define PKCS11_CK_UNAVAILABLE_INFORMATION UINT32_C(0xFFFFFFFF) +#define PKCS11_UNDEFINED_ID UINT32_C(0xFFFFFFFF) +#define PKCS11_FALSE false +#define PKCS11_TRUE true + +/* + * Note on PKCS#11 TA commands ABI + * + * For evolution of the TA API and to not mess with the GPD TEE 4 parameters + * constraint, all the PKCS11 TA invocation commands use a subset of available + * the GPD TEE invocation parameter types. + * + * Param#0 is used for the so-called control arguments of the invoked command + * and for providing a PKCS#11 compliant status code for the request command. + * Param#0 is an in/out memory reference (aka memref[0]). The input buffer + * stores serialized arguments for the command. The output buffer store the + * 32bit TA return code for the command. As a consequence, param#0 shall + * always be an input/output memory reference of at least 32bit, more if + * the command expects more input arguments. + * + * When the TA returns with TEE_SUCCESS result, client shall always get the + * 32bit value stored in param#0 output buffer and use the value as TA + * return code for the invoked command. + * + * Param#1 can be used for input data arguments of the invoked command. + * It is unused or is an input memory reference, aka memref[1]. + * Evolution of the API may use memref[1] for output data as well. + * + * Param#2 is mostly used for output data arguments of the invoked command + * and for output handles generated from invoked commands. + * Few commands uses it for a secondary input data buffer argument. + * It is unused or is an input/output/in-out memory reference, aka memref[2]. + * + * Param#3 is currently unused and reserved for evolution of the API. + */ + +enum pkcs11_ta_cmd { + /* + * PKCS11_CMD_PING Ack TA presence and return version info + * + * [in] memref[0] = 32bit, unused, must be 0 + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = [ + * 32bit version major value, + * 32bit version minor value + * 32bit version patch value + * ] + */ + PKCS11_CMD_PING = 0, + + /* + * PKCS11_CMD_SLOT_LIST - Get the table of the valid slot IDs + * + * [in] memref[0] = 32bit, unused, must be 0 + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit array slot_ids[slot counts] + * + * The TA instance may represent several PKCS#11 slots and + * associated tokens. This commadn reports the IDs of embedded tokens. + * This command relates the PKCS#11 API function C_GetSlotList(). + */ + PKCS11_CMD_SLOT_LIST = 1, + + /* + * PKCS11_CMD_SLOT_INFO - Get cryptoki structured slot information + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_slot_info)info + * + * The TA instance may represent several PKCS#11 slots/tokens. + * This command relates the PKCS#11 API function C_GetSlotInfo(). + */ + PKCS11_CMD_SLOT_INFO = 2, + + /* + * PKCS11_CMD_TOKEN_INFO - Get cryptoki structured token information + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_token_info)info + * + * The TA instance may represent several PKCS#11 slots/tokens. + * This command relates the PKCS#11 API function C_GetTokenInfo(). + */ + PKCS11_CMD_TOKEN_INFO = 3, + + /* + * PKCS11_CMD_MECHANISM_IDS - Get list of the supported mechanisms + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit array mechanism IDs + * + * This command relates to the PKCS#11 API function + * C_GetMechanismList(). + */ + PKCS11_CMD_MECHANISM_IDS = 4, + + /* + * PKCS11_CMD_MECHANISM_INFO - Get information on a specific mechanism + * + * [in] memref[0] = [ + * 32bit slot ID, + * 32bit mechanism ID (PKCS11_CKM_*) + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_mechanism_info)info + * + * This command relates to the PKCS#11 API function + * C_GetMechanismInfo(). + */ + PKCS11_CMD_MECHANISM_INFO = 5, + + /* + * PKCS11_CMD_OPEN_SESSION - Open a session + * + * [in] memref[0] = [ + * 32bit slot ID, + * 32bit session flags, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit session handle + * + * This command relates to the PKCS#11 API function C_OpenSession(). + */ + PKCS11_CMD_OPEN_SESSION = 6, + + /* + * PKCS11_CMD_CLOSE_SESSION - Close an opened session + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_CloseSession(). + */ + PKCS11_CMD_CLOSE_SESSION = 7, + + /* + * PKCS11_CMD_CLOSE_ALL_SESSIONS - Close all client sessions on token + * + * [in] memref[0] = 32bit slot ID + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function + * C_CloseAllSessions(). + */ + PKCS11_CMD_CLOSE_ALL_SESSIONS = 8, + + /* + * PKCS11_CMD_SESSION_INFO - Get Cryptoki information on a session + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_session_info)info + * + * This command relates to the PKCS#11 API function C_GetSessionInfo(). + */ + PKCS11_CMD_SESSION_INFO = 9, + + /* + * PKCS11_CMD_INIT_TOKEN - Initialize PKCS#11 token + * + * [in] memref[0] = [ + * 32bit slot ID, + * 32bit PIN length, + * byte array label[32] + * byte array PIN[PIN length], + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_InitToken(). + */ + PKCS11_CMD_INIT_TOKEN = 10, + + /* + * PKCS11_CMD_INIT_PIN - Initialize user PIN + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit PIN byte size, + * byte array: PIN data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_InitPIN(). + */ + PKCS11_CMD_INIT_PIN = 11, + + /* + * PKCS11_CMD_SET_PIN - Change user PIN + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit old PIN byte size, + * 32bit new PIN byte size, + * byte array: PIN data, + * byte array: new PIN data, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_SetPIN(). + */ + PKCS11_CMD_SET_PIN = 12, + + /* + * PKCS11_CMD_LOGIN - Initialize user PIN + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit user identifier, enum pkcs11_user_type + * 32bit PIN byte size, + * byte array: PIN data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_Login(). + */ + PKCS11_CMD_LOGIN = 13, + + /* + * PKCS11_CMD_LOGOUT - Log out from token + * + * [in] memref[0] = [ + * 32bit session handle, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_Logout(). + */ + PKCS11_CMD_LOGOUT = 14, + + /* + * PKCS11_CMD_CREATE_OBJECT - Create a raw client assembled object in + * the session or token + * + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_CreateObject(). + */ + PKCS11_CMD_CREATE_OBJECT = 15, + + /* + * PKCS11_CMD_DESTROY_OBJECT - Destroy an object + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_DestroyObject(). + */ + PKCS11_CMD_DESTROY_OBJECT = 16, + + /* + * PKCS11_CMD_ENCRYPT_INIT - Initialize encryption processing + * PKCS11_CMD_DECRYPT_INIT - Initialize decryption processing + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle of the key, + * (struct pkcs11_attribute_head)mechanism + mecha params + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * These commands relate to the PKCS#11 API functions + * C_EncryptInit() and C_DecryptInit(). + */ + PKCS11_CMD_ENCRYPT_INIT = 17, + PKCS11_CMD_DECRYPT_INIT = 18, + + /* + * PKCS11_CMD_ENCRYPT_UPDATE - Update encryption processing + * PKCS11_CMD_DECRYPT_UPDATE - Update decryption processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = output processed data + * + * These commands relate to the PKCS#11 API functions + * C_EncryptUpdate() and C_DecryptUpdate(). + */ + PKCS11_CMD_ENCRYPT_UPDATE = 19, + PKCS11_CMD_DECRYPT_UPDATE = 20, + + /* + * PKCS11_CMD_ENCRYPT_FINAL - Finalize encryption processing + * PKCS11_CMD_DECRYPT_FINAL - Finalize decryption processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = output processed data + * + * These commands relate to the PKCS#11 API functions + * C_EncryptFinal() and C_DecryptFinal(). + */ + PKCS11_CMD_ENCRYPT_FINAL = 21, + PKCS11_CMD_DECRYPT_FINAL = 22, + + /* + * PKCS11_CMD_ENCRYPT_ONESHOT - Update and finalize encryption + * processing + * PKCS11_CMD_DECRYPT_ONESHOT - Update and finalize decryption + * processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = output processed data + * + * These commands relate to the PKCS#11 API functions C_Encrypt and + * C_Decrypt. + */ + PKCS11_CMD_ENCRYPT_ONESHOT = 23, + PKCS11_CMD_DECRYPT_ONESHOT = 24, + + /* + * PKCS11_CMD_SIGN_INIT - Initialize a signature computation + * processing + * PKCS11_CMD_VERIFY_INIT - Initialize a signature verification + * processing + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit key handle, + * (struct pkcs11_attribute_head)mechanism + + * mechanism params, + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * These commands relate to the PKCS#11 API functions C_SignInit() and + * C_VerifyInit(). + */ + PKCS11_CMD_SIGN_INIT = 25, + PKCS11_CMD_VERIFY_INIT = 26, + + /* + * PKCS11_CMD_SIGN_UPDATE - Update a signature computation processing + * PKCS11_CMD_VERIFY_UPDATE - Update a signature verification processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * + * These commands relate to the PKCS#11 API functions C_SignUpdate() and + * C_VerifyUpdate(). + */ + PKCS11_CMD_SIGN_UPDATE = 27, + PKCS11_CMD_VERIFY_UPDATE = 28, + + /* + * PKCS11_CMD_SIGN_FINAL - Finalize a signature computation processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = output signature + * + * This command relates to the PKCS#11 API functions C_SignFinal(). + */ + PKCS11_CMD_SIGN_FINAL = 29, + + /* + * PKCS11_CMD_VERIFY_FINAL - Finalize a signature verification + * processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[2] = input signature to be processed + * + * This command relates to the PKCS#11 API functions C_VerifyFinal(). + */ + PKCS11_CMD_VERIFY_FINAL = 30, + + /* + * PKCS11_CMD_SIGN_ONESHOT - Compute a signature + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = byte array: generated signature + * + * This command relates to the PKCS#11 API function C_Sign(). + */ + PKCS11_CMD_SIGN_ONESHOT = 31, + + /* + * PKCS11_CMD_VERIFY_ONESHOT - Compute and compare a signature + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [in] memref[2] = input signature to be processed + * + * This command relates to the PKCS#11 API function C_Verify(). + */ + PKCS11_CMD_VERIFY_ONESHOT = 32, + + /* + * PKCS11_CMD_GENERATE_KEY - Generate a symmetric key or domain + * parameters + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_GenerateKey(). + */ + PKCS11_CMD_GENERATE_KEY = 33, + + /* + * PKCS11_CMD_FIND_OBJECTS_INIT - Initialize an object search + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_FindOjectsInit(). + */ + PKCS11_CMD_FIND_OBJECTS_INIT = 34, + + /* + * PKCS11_CMD_FIND_OBJECTS - Get handles of matching objects + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit array object_handle_array[N] + * + * This command relates to the PKCS#11 API function C_FindOjects(). + * The size of object_handle_array depends on the size of the output + * buffer provided by the client. + */ + PKCS11_CMD_FIND_OBJECTS = 35, + + /* + * PKCS11_CMD_FIND_OBJECTS_FINAL - Finalize current objects search + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_FindOjectsFinal(). + */ + PKCS11_CMD_FIND_OBJECTS_FINAL = 36, + + /* + * PKCS11_CMD_GET_OBJECT_SIZE - Get byte size used by object in the TEE + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object_byte_size + * + * This command relates to the PKCS#11 API function C_GetObjectSize(). + */ + PKCS11_CMD_GET_OBJECT_SIZE = 37, + + /* + * PKCS11_CMD_GET_ATTRIBUTE_VALUE - Get the value of object attribute(s) + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = (struct pkcs11_object_head)attribs + attributes + * data + * + * This command relates to the PKCS#11 API function C_GetAttributeValue. + * Caller provides an attribute template as 3rd argument in memref[0] + * (referred here as attribs + attributes data). Upon successful + * completion, the TA returns the provided template filled with expected + * data through output argument memref[2] (referred here again as + * attribs + attributes data). + */ + PKCS11_CMD_GET_ATTRIBUTE_VALUE = 38, + + /* + * PKCS11_CMD_SET_ATTRIBUTE_VALUE - Set the value of object attribute(s) + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_SetAttributeValue. + * Caller provides an attribute template as 3rd argument in memref[0] + * (referred here as attribs + attributes data). + */ + PKCS11_CMD_SET_ATTRIBUTE_VALUE = 39, + + /* + * PKCS11_CMD_COPY_OBJECT - Copies an object, creating a new object for + * the copy. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit object handle, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_CopyObject(). + * Caller provides an attribute template as 3rd argument in memref[0] + * (referred here as attribs + attributes data). + */ + PKCS11_CMD_COPY_OBJECT = 40, + + /* + * PKCS11_CMD_SEED_RANDOM - Seed random data generator + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = byte array: seed material to feed into the RNG + * + * This command relates to the PKCS#11 API function C_SeedRandom(). + */ + PKCS11_CMD_SEED_RANDOM = 41, + + /* + * PKCS11_CMD_GENERATE_RANDOM - Generate random data + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = byte array: generated random + * + * This command relates to the PKCS#11 API function C_GenerateRandom(). + */ + PKCS11_CMD_GENERATE_RANDOM = 42, + + /* + * PKCS11_CMD_DERIVE_KEY - Derive a key from a parent key. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit parent key handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_DeriveKey(). + */ + PKCS11_CMD_DERIVE_KEY = 43, + + /* + * PKCS11_CMD_RELEASE_ACTIVE_PROCESSING - Release active processing + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit enum pkcs11_ta_cmd + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command is used to release active processing in case of + * Cryptoki API invocation error is detected in user space processing. + * Function derived from pkcs11_ta_cmd is used to verify that active + * processing matches. + */ + PKCS11_CMD_RELEASE_ACTIVE_PROCESSING = 44, + + /* + * PKCS11_CMD_DIGEST_INIT - Initialize a digest computation processing + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_attribute_head)mechanism + mecha params + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_DigestInit(). + */ + PKCS11_CMD_DIGEST_INIT = 45, + + /* + * PKCS11_CMD_DIGEST_KEY - Update digest with a key + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit key handle + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * + * This command relates to the PKCS#11 API function C_DigestKey(). + */ + PKCS11_CMD_DIGEST_KEY = 46, + + /* + * PKCS11_CMD_DIGEST_UPDATE - Update digest with data + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * + * This command relates to the PKCS#11 API function C_DigestUpdate(). + */ + PKCS11_CMD_DIGEST_UPDATE = 47, + + /* + * PKCS11_CMD_DIGEST_FINAL - Finalize a digest computation processing + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = output digest + * + * This command relates to the PKCS#11 API function C_DigestFinal(). + */ + PKCS11_CMD_DIGEST_FINAL = 48, + + /* + * PKCS11_CMD_DIGEST_ONESHOT - Compute a digest + * + * [in] memref[0] = 32bit session handle + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = input data to be processed + * [out] memref[2] = byte array: generated digest + * + * This command relates to the PKCS#11 API function C_Digest(). + */ + PKCS11_CMD_DIGEST_ONESHOT = 49, + + /* + * PKCS11_CMD_GENERATE_KEY_PAIR - Generate an asymmetric key pair + * + * [in] memref[0] = [ + * 32bit session handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)public key attribs + + * attributes data, + * (struct pkcs11_object_head)private key attribs + + * attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = [ + * 32bit public key object handle, + * 32bit private key object handle + * ] + * + * This command relates to the PKCS#11 API function + * C_GenerateKeyPair(). + */ + PKCS11_CMD_GENERATE_KEY_PAIR = 50, + + /* + * PKCS11_CMD_WRAP_KEY - Wraps a private or secret key. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit wrapping key handle, + * 32bit key handle, + * (struct pkcs11_attribute_head)mechanism + mecha params + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [out] memref[2] = wrapped key + * + * This command relates to the PKCS#11 API function C_WrapKey(). + */ + PKCS11_CMD_WRAP_KEY = 51, + + /* + * PKCS11_CMD_UNWRAP_KEY - Unwraps a wrapped key, creating a new + * private or secret key object. + * + * [in] memref[0] = [ + * 32bit session handle, + * 32bit unwrapping key handle, + * (struct pkcs11_attribute_head)mechanism + mecha params, + * (struct pkcs11_object_head)attribs + attributes data + * ] + * [out] memref[0] = 32bit return code, enum pkcs11_rc + * [in] memref[1] = wrapped key + * [out] memref[2] = 32bit object handle + * + * This command relates to the PKCS#11 API function C_UnwrapKey(). + */ + PKCS11_CMD_UNWRAP_KEY = 52, +}; + +/* + * Command return codes + * PKCS11_ relates CryptoKi client API CKR_ + */ +enum pkcs11_rc { + PKCS11_CKR_OK = 0, + PKCS11_CKR_CANCEL = 0x0001, + PKCS11_CKR_SLOT_ID_INVALID = 0x0003, + PKCS11_CKR_GENERAL_ERROR = 0x0005, + PKCS11_CKR_FUNCTION_FAILED = 0x0006, + PKCS11_CKR_ARGUMENTS_BAD = 0x0007, + PKCS11_CKR_ATTRIBUTE_READ_ONLY = 0x0010, + PKCS11_CKR_ATTRIBUTE_SENSITIVE = 0x0011, + PKCS11_CKR_ATTRIBUTE_TYPE_INVALID = 0x0012, + PKCS11_CKR_ATTRIBUTE_VALUE_INVALID = 0x0013, + PKCS11_CKR_ACTION_PROHIBITED = 0x001b, + PKCS11_CKR_DATA_INVALID = 0x0020, + PKCS11_CKR_DATA_LEN_RANGE = 0x0021, + PKCS11_CKR_DEVICE_ERROR = 0x0030, + PKCS11_CKR_DEVICE_MEMORY = 0x0031, + PKCS11_CKR_DEVICE_REMOVED = 0x0032, + PKCS11_CKR_ENCRYPTED_DATA_INVALID = 0x0040, + PKCS11_CKR_ENCRYPTED_DATA_LEN_RANGE = 0x0041, + PKCS11_CKR_KEY_HANDLE_INVALID = 0x0060, + PKCS11_CKR_KEY_SIZE_RANGE = 0x0062, + PKCS11_CKR_KEY_TYPE_INCONSISTENT = 0x0063, + PKCS11_CKR_KEY_INDIGESTIBLE = 0x0067, + PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED = 0x0068, + PKCS11_CKR_KEY_NOT_WRAPPABLE = 0x0069, + PKCS11_CKR_KEY_UNEXTRACTABLE = 0x006a, + PKCS11_CKR_MECHANISM_INVALID = 0x0070, + PKCS11_CKR_MECHANISM_PARAM_INVALID = 0x0071, + PKCS11_CKR_OBJECT_HANDLE_INVALID = 0x0082, + PKCS11_CKR_OPERATION_ACTIVE = 0x0090, + PKCS11_CKR_OPERATION_NOT_INITIALIZED = 0x0091, + PKCS11_CKR_PIN_INCORRECT = 0x00a0, + PKCS11_CKR_PIN_INVALID = 0x00a1, + PKCS11_CKR_PIN_LEN_RANGE = 0x00a2, + PKCS11_CKR_PIN_EXPIRED = 0x00a3, + PKCS11_CKR_PIN_LOCKED = 0x00a4, + PKCS11_CKR_SESSION_CLOSED = 0x00b0, + PKCS11_CKR_SESSION_COUNT = 0x00b1, + PKCS11_CKR_SESSION_HANDLE_INVALID = 0x00b3, + PKCS11_CKR_SESSION_PARALLEL_NOT_SUPPORTED = 0x00b4, + PKCS11_CKR_SESSION_READ_ONLY = 0x00b5, + PKCS11_CKR_SESSION_EXISTS = 0x00b6, + PKCS11_CKR_SESSION_READ_ONLY_EXISTS = 0x00b7, + PKCS11_CKR_SESSION_READ_WRITE_SO_EXISTS = 0x00b8, + PKCS11_CKR_SIGNATURE_INVALID = 0x00c0, + PKCS11_CKR_SIGNATURE_LEN_RANGE = 0x00c1, + PKCS11_CKR_TEMPLATE_INCOMPLETE = 0x00d0, + PKCS11_CKR_TEMPLATE_INCONSISTENT = 0x00d1, + PKCS11_CKR_TOKEN_NOT_PRESENT = 0x00e0, + PKCS11_CKR_TOKEN_NOT_RECOGNIZED = 0x00e1, + PKCS11_CKR_TOKEN_WRITE_PROTECTED = 0x00e2, + PKCS11_CKR_UNWRAPPING_KEY_HANDLE_INVALID = 0x00f0, + PKCS11_CKR_UNWRAPPING_KEY_SIZE_RANGE = 0x00f1, + PKCS11_CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT = 0x00f2, + PKCS11_CKR_USER_ALREADY_LOGGED_IN = 0x0100, + PKCS11_CKR_USER_NOT_LOGGED_IN = 0x0101, + PKCS11_CKR_USER_PIN_NOT_INITIALIZED = 0x0102, + PKCS11_CKR_USER_TYPE_INVALID = 0x0103, + PKCS11_CKR_USER_ANOTHER_ALREADY_LOGGED_IN = 0x0104, + PKCS11_CKR_USER_TOO_MANY_TYPES = 0x0105, + PKCS11_CKR_WRAPPED_KEY_INVALID = 0x0110, + PKCS11_CKR_WRAPPED_KEY_LEN_RANGE = 0x0112, + PKCS11_CKR_WRAPPING_KEY_HANDLE_INVALID = 0x0113, + PKCS11_CKR_WRAPPING_KEY_SIZE_RANGE = 0x0114, + PKCS11_CKR_WRAPPING_KEY_TYPE_INCONSISTENT = 0x0115, + PKCS11_CKR_RANDOM_SEED_NOT_SUPPORTED = 0x0120, + PKCS11_CKR_RANDOM_NO_RNG = 0x0121, + PKCS11_CKR_DOMAIN_PARAMS_INVALID = 0x0130, + PKCS11_CKR_CURVE_NOT_SUPPORTED = 0x0140, + PKCS11_CKR_BUFFER_TOO_SMALL = 0x0150, + PKCS11_CKR_SAVED_STATE_INVALID = 0x0160, + PKCS11_CKR_INFORMATION_SENSITIVE = 0x0170, + PKCS11_CKR_STATE_UNSAVEABLE = 0x0180, + PKCS11_CKR_PIN_TOO_WEAK = 0x01b8, + PKCS11_CKR_PUBLIC_KEY_INVALID = 0x01b9, + PKCS11_CKR_FUNCTION_REJECTED = 0x0200, + /* Vendor specific IDs not returned to client */ + PKCS11_RV_NOT_FOUND = 0x80000000, + PKCS11_RV_NOT_IMPLEMENTED = 0x80000001, +}; + +/* + * Arguments for PKCS11_CMD_SLOT_INFO + */ +#define PKCS11_SLOT_DESC_SIZE 64 +#define PKCS11_SLOT_MANUFACTURER_SIZE 32 +#define PKCS11_SLOT_VERSION_SIZE 2 + +struct pkcs11_slot_info { + uint8_t slot_description[PKCS11_SLOT_DESC_SIZE]; + uint8_t manufacturer_id[PKCS11_SLOT_MANUFACTURER_SIZE]; + uint32_t flags; + uint8_t hardware_version[PKCS11_SLOT_VERSION_SIZE]; + uint8_t firmware_version[PKCS11_SLOT_VERSION_SIZE]; +}; + +/* + * Values for pkcs11_slot_info::flags. + * PKCS11_CKFS_ reflects CryptoKi client API slot flags CKF_. + */ +#define PKCS11_CKFS_TOKEN_PRESENT (1U << 0) +#define PKCS11_CKFS_REMOVABLE_DEVICE (1U << 1) +#define PKCS11_CKFS_HW_SLOT (1U << 2) + +/* + * Arguments for PKCS11_CMD_TOKEN_INFO + */ +#define PKCS11_TOKEN_LABEL_SIZE 32 +#define PKCS11_TOKEN_MANUFACTURER_SIZE 32 +#define PKCS11_TOKEN_MODEL_SIZE 16 +#define PKCS11_TOKEN_SERIALNUM_SIZE 16 + +struct pkcs11_token_info { + uint8_t label[PKCS11_TOKEN_LABEL_SIZE]; + uint8_t manufacturer_id[PKCS11_TOKEN_MANUFACTURER_SIZE]; + uint8_t model[PKCS11_TOKEN_MODEL_SIZE]; + uint8_t serial_number[PKCS11_TOKEN_SERIALNUM_SIZE]; + uint32_t flags; + uint32_t max_session_count; + uint32_t session_count; + uint32_t max_rw_session_count; + uint32_t rw_session_count; + uint32_t max_pin_len; + uint32_t min_pin_len; + uint32_t total_public_memory; + uint32_t free_public_memory; + uint32_t total_private_memory; + uint32_t free_private_memory; + uint8_t hardware_version[2]; + uint8_t firmware_version[2]; + uint8_t utc_time[16]; +}; + +/* + * Values for pkcs11_token_info::flags. + * PKCS11_CKFT_ reflects CryptoKi client API token flags CKF_. + */ +#define PKCS11_CKFT_RNG (1U << 0) +#define PKCS11_CKFT_WRITE_PROTECTED (1U << 1) +#define PKCS11_CKFT_LOGIN_REQUIRED (1U << 2) +#define PKCS11_CKFT_USER_PIN_INITIALIZED (1U << 3) +#define PKCS11_CKFT_RESTORE_KEY_NOT_NEEDED (1U << 5) +#define PKCS11_CKFT_CLOCK_ON_TOKEN (1U << 6) +#define PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH (1U << 8) +#define PKCS11_CKFT_DUAL_CRYPTO_OPERATIONS (1U << 9) +#define PKCS11_CKFT_TOKEN_INITIALIZED (1U << 10) +#define PKCS11_CKFT_SECONDARY_AUTHENTICATION (1U << 11) +#define PKCS11_CKFT_USER_PIN_COUNT_LOW (1U << 16) +#define PKCS11_CKFT_USER_PIN_FINAL_TRY (1U << 17) +#define PKCS11_CKFT_USER_PIN_LOCKED (1U << 18) +#define PKCS11_CKFT_USER_PIN_TO_BE_CHANGED (1U << 19) +#define PKCS11_CKFT_SO_PIN_COUNT_LOW (1U << 20) +#define PKCS11_CKFT_SO_PIN_FINAL_TRY (1U << 21) +#define PKCS11_CKFT_SO_PIN_LOCKED (1U << 22) +#define PKCS11_CKFT_SO_PIN_TO_BE_CHANGED (1U << 23) +#define PKCS11_CKFT_ERROR_STATE (1U << 24) + +/* Values for user identity */ +enum pkcs11_user_type { + PKCS11_CKU_SO = 0x000, + PKCS11_CKU_USER = 0x001, + PKCS11_CKU_CONTEXT_SPECIFIC = 0x002, +}; + +/* + * TEE Identity based authentication for tokens + * + * When configuration CFG_PKCS11_TA_AUTH_TEE_IDENTITY is enabled TEE Identity + * based authentication scheme is enabled. + * + * Feature enablement per token basis is controlled by token flag: + * pkcs11_token_info->flags & PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH + * + * When calling C_InitToken() mode is determined based on SO PIN value. + * - If the PIN is empty (or NULL_PTR) then active client TEE Identity will be + * used as SO TEE Identity + * - If the PIN is given then normal PIN behavior is used + * + * Once TEE Identity based authentication is activated following operational + * changes happen: + * - PIN failure counters are disabled to prevent token authentication lockups + * - When C_Login() or so is performed actual PIN value is ignored and active + * client TEE Identity will be used + * + * After a successful call to C_InitToken(), one can switch authentication + * mode as user credentials have been cleared. After user credentials has been + * set authentication mode switching is protected. + * + * To switch the authentication mode from PIN to TEE Identity: + * - Make sure active TEE Identity is set for the TA connection + * - Login as SO so that PIN change affects SO + * - Call C_SetPIN() with empty PIN to capture current TEE Identity as SO + * credential + * - Optional: Successive call to C_SetPIN() can be used for change to other + * TEE Identity vs. current TA connection + * + * To switch the authentication mode from TEE Identity to PIN: + * - Make sure SO's TEE Identity is set for the TA connection + * - Login as SO so that PIN change affects SO + * - Call C_SetPIN() with any PIN that does not match TEE Identity PIN syntax + * - Optional: Successive call to C_SetPIN() can be used to change SO + * credential to any valid PIN if there was collision with TEE Identity PIN + * syntax + * + * Different types of TEE Identity authentication methods can be configured: + * - Configured with C_InitToken(), C_InitPIN() or by C_SetPIN() + * - PIN value follows below PIN syntax + * + * TEE Identity based authenticate PIN syntax: + * - PIN value: NULL_PTR or empty + * - Use active client TEE Identity + * - PIN value: public + * - TEE public login + * - PIN value: user: + * - TEE user login with client UUID matching user credentials + * - PIN value: group: + * - TEE group login with client UUID matching group credentials + */ + +/* Keywords for protected authenticated path PIN parser */ +#define PKCS11_AUTH_TEE_IDENTITY_PUBLIC "public" +#define PKCS11_AUTH_TEE_IDENTITY_USER "user:" +#define PKCS11_AUTH_TEE_IDENTITY_GROUP "group:" + +/* + * Values for 32bit session flags argument to PKCS11_CMD_OPEN_SESSION + * and pkcs11_session_info::flags. + * PKCS11_CKFSS_ reflects CryptoKi client API session flags CKF_. + */ +#define PKCS11_CKFSS_RW_SESSION (1U << 1) +#define PKCS11_CKFSS_SERIAL_SESSION (1U << 2) + +/* + * Arguments for PKCS11_CMD_SESSION_INFO + */ + +struct pkcs11_session_info { + uint32_t slot_id; + uint32_t state; + uint32_t flags; + uint32_t device_error; +}; + +/* Valid values for pkcs11_session_info::state */ +enum pkcs11_session_state { + PKCS11_CKS_RO_PUBLIC_SESSION = 0, + PKCS11_CKS_RO_USER_FUNCTIONS = 1, + PKCS11_CKS_RW_PUBLIC_SESSION = 2, + PKCS11_CKS_RW_USER_FUNCTIONS = 3, + PKCS11_CKS_RW_SO_FUNCTIONS = 4, +}; + +/* + * Arguments for PKCS11_CMD_MECHANISM_INFO + */ + +struct pkcs11_mechanism_info { + uint32_t min_key_size; + uint32_t max_key_size; + uint32_t flags; +}; + +/* + * Values for pkcs11_mechanism_info::flags. + * PKCS11_CKFM_ reflects CryptoKi client API mechanism flags CKF_. + */ +#define PKCS11_CKFM_HW (1U << 0) +#define PKCS11_CKFM_ENCRYPT (1U << 8) +#define PKCS11_CKFM_DECRYPT (1U << 9) +#define PKCS11_CKFM_DIGEST (1U << 10) +#define PKCS11_CKFM_SIGN (1U << 11) +#define PKCS11_CKFM_SIGN_RECOVER (1U << 12) +#define PKCS11_CKFM_VERIFY (1U << 13) +#define PKCS11_CKFM_VERIFY_RECOVER (1U << 14) +#define PKCS11_CKFM_GENERATE (1U << 15) +#define PKCS11_CKFM_GENERATE_KEY_PAIR (1U << 16) +#define PKCS11_CKFM_WRAP (1U << 17) +#define PKCS11_CKFM_UNWRAP (1U << 18) +#define PKCS11_CKFM_DERIVE (1U << 19) +#define PKCS11_CKFM_EC_F_P (1U << 20) +#define PKCS11_CKFM_EC_F_2M (1U << 21) +#define PKCS11_CKFM_EC_ECPARAMETERS (1U << 22) +#define PKCS11_CKFM_EC_NAMEDCURVE (1U << 23) +#define PKCS11_CKFM_EC_UNCOMPRESS (1U << 24) +#define PKCS11_CKFM_EC_COMPRESS (1U << 25) + +/* + * pkcs11_object_head - Header of object whose data are serialized in memory + * + * An object is made of several attributes. Attributes are stored one next to + * the other with byte alignment as a serialized byte array. The byte array + * of serialized attributes is prepended with the size of the attrs[] array + * in bytes and the number of attributes in the array, yielding the struct + * pkcs11_object_head. + * + * @attrs_size - byte size of whole byte array attrs[] + * @attrs_count - number of attribute items stored in attrs[] + * @attrs - then starts the attributes data + */ +struct pkcs11_object_head { + uint32_t attrs_size; + uint32_t attrs_count; + uint8_t attrs[]; +}; + +/* + * Attribute reference in the TA ABI. Each attribute starts with a header + * structure followed by the attribute value. The attribute byte size is + * defined in the attribute header. + * + * @id - the 32bit identifier of the attribute, see PKCS11_CKA_ + * @size - the 32bit value attribute byte size + * @data - then starts the attribute value + */ +struct pkcs11_attribute_head { + uint32_t id; + uint32_t size; + uint8_t data[]; +}; + +#define PKCS11_CKA_VENDOR_DEFINED 0x80000000UL + +/** + * The PKCS11_CKF_ARRAY_ATTRIBUTE flag identifies an attribute which + * consists of an array of values. + */ +#define PKCS11_CKF_ARRAY_ATTRIBUTE 0x40000000UL + +/* + * OP-TEE's vendor specific PKCS#11 attribute allocation + * + * bit 31 - PKCS11_CKA_VENDOR_DEFINED + * bit 30 - PKCS11_CKF_ARRAY_ATTRIBUTE - works like in normal attributes + * bit 24-29 - reserved in case PKCS#11 standard starts to use them + * bit 16-23 - allocated for OP-TEE attribute flags + * bit 0-15 - allocated for attribute identifier + */ + +/* OP-TEE attribute flags */ + +/** + * Flags mask for checking if OP-TEE attribute flags are set. + */ +#define PKCS11_CKA_OPTEE_FLAGS_MASK (PKCS11_CKA_VENDOR_DEFINED | \ + 0x00FF0000UL) + +/** + * PKCS11_CKA_OPTEE_FLAGS_HIDDEN defines attribute that will not be exported + * from PKCS11 TA to its client. From client application point of view the + * attribute does not exist. + */ +#define PKCS11_CKA_OPTEE_FLAGS_HIDDEN (PKCS11_CKA_VENDOR_DEFINED | \ + 0x00010000UL) + +/* + * Attribute identification IDs as of v2.40 excluding deprecated IDs. + * Valid values for struct pkcs11_attribute_head::id + * PKCS11_CKA_ reflects CryptoKi client API attribute IDs CKA_. + */ +enum pkcs11_attr_id { + PKCS11_CKA_CLASS = 0x0000, + PKCS11_CKA_TOKEN = 0x0001, + PKCS11_CKA_PRIVATE = 0x0002, + PKCS11_CKA_LABEL = 0x0003, + PKCS11_CKA_APPLICATION = 0x0010, + PKCS11_CKA_VALUE = 0x0011, + PKCS11_CKA_OBJECT_ID = 0x0012, + PKCS11_CKA_CERTIFICATE_TYPE = 0x0080, + PKCS11_CKA_ISSUER = 0x0081, + PKCS11_CKA_SERIAL_NUMBER = 0x0082, + PKCS11_CKA_AC_ISSUER = 0x0083, + PKCS11_CKA_OWNER = 0x0084, + PKCS11_CKA_ATTR_TYPES = 0x0085, + PKCS11_CKA_TRUSTED = 0x0086, + PKCS11_CKA_CERTIFICATE_CATEGORY = 0x0087, + PKCS11_CKA_JAVA_MIDP_SECURITY_DOMAIN = 0x0088, + PKCS11_CKA_URL = 0x0089, + PKCS11_CKA_HASH_OF_SUBJECT_PUBLIC_KEY = 0x008a, + PKCS11_CKA_HASH_OF_ISSUER_PUBLIC_KEY = 0x008b, + PKCS11_CKA_NAME_HASH_ALGORITHM = 0x008c, + PKCS11_CKA_CHECK_VALUE = 0x0090, + PKCS11_CKA_KEY_TYPE = 0x0100, + PKCS11_CKA_SUBJECT = 0x0101, + PKCS11_CKA_ID = 0x0102, + PKCS11_CKA_SENSITIVE = 0x0103, + PKCS11_CKA_ENCRYPT = 0x0104, + PKCS11_CKA_DECRYPT = 0x0105, + PKCS11_CKA_WRAP = 0x0106, + PKCS11_CKA_UNWRAP = 0x0107, + PKCS11_CKA_SIGN = 0x0108, + PKCS11_CKA_SIGN_RECOVER = 0x0109, + PKCS11_CKA_VERIFY = 0x010a, + PKCS11_CKA_VERIFY_RECOVER = 0x010b, + PKCS11_CKA_DERIVE = 0x010c, + PKCS11_CKA_START_DATE = 0x0110, + PKCS11_CKA_END_DATE = 0x0111, + PKCS11_CKA_MODULUS = 0x0120, + PKCS11_CKA_MODULUS_BITS = 0x0121, + PKCS11_CKA_PUBLIC_EXPONENT = 0x0122, + PKCS11_CKA_PRIVATE_EXPONENT = 0x0123, + PKCS11_CKA_PRIME_1 = 0x0124, + PKCS11_CKA_PRIME_2 = 0x0125, + PKCS11_CKA_EXPONENT_1 = 0x0126, + PKCS11_CKA_EXPONENT_2 = 0x0127, + PKCS11_CKA_COEFFICIENT = 0x0128, + PKCS11_CKA_PUBLIC_KEY_INFO = 0x0129, + PKCS11_CKA_PRIME = 0x0130, + PKCS11_CKA_SUBPRIME = 0x0131, + PKCS11_CKA_BASE = 0x0132, + PKCS11_CKA_PRIME_BITS = 0x0133, + PKCS11_CKA_SUBPRIME_BITS = 0x0134, + PKCS11_CKA_VALUE_BITS = 0x0160, + PKCS11_CKA_VALUE_LEN = 0x0161, + PKCS11_CKA_EXTRACTABLE = 0x0162, + PKCS11_CKA_LOCAL = 0x0163, + PKCS11_CKA_NEVER_EXTRACTABLE = 0x0164, + PKCS11_CKA_ALWAYS_SENSITIVE = 0x0165, + PKCS11_CKA_KEY_GEN_MECHANISM = 0x0166, + PKCS11_CKA_MODIFIABLE = 0x0170, + PKCS11_CKA_COPYABLE = 0x0171, + PKCS11_CKA_DESTROYABLE = 0x0172, + PKCS11_CKA_EC_PARAMS = 0x0180, + PKCS11_CKA_EC_POINT = 0x0181, + PKCS11_CKA_ALWAYS_AUTHENTICATE = 0x0202, + PKCS11_CKA_WRAP_WITH_TRUSTED = 0x0210, + PKCS11_CKA_WRAP_TEMPLATE = PKCS11_CKF_ARRAY_ATTRIBUTE | + 0x0211, + PKCS11_CKA_UNWRAP_TEMPLATE = PKCS11_CKF_ARRAY_ATTRIBUTE | + 0x0212, + PKCS11_CKA_DERIVE_TEMPLATE = PKCS11_CKF_ARRAY_ATTRIBUTE | + 0x0213, + PKCS11_CKA_OTP_FORMAT = 0x0220, + PKCS11_CKA_OTP_LENGTH = 0x0221, + PKCS11_CKA_OTP_TIME_INTERVAL = 0x0222, + PKCS11_CKA_OTP_USER_FRIENDLY_MODE = 0x0223, + PKCS11_CKA_OTP_CHALLENGE_REQUIREMENT = 0x0224, + PKCS11_CKA_OTP_TIME_REQUIREMENT = 0x0225, + PKCS11_CKA_OTP_COUNTER_REQUIREMENT = 0x0226, + PKCS11_CKA_OTP_PIN_REQUIREMENT = 0x0227, + PKCS11_CKA_OTP_COUNTER = 0x022e, + PKCS11_CKA_OTP_TIME = 0x022f, + PKCS11_CKA_OTP_USER_IDENTIFIER = 0x022a, + PKCS11_CKA_OTP_SERVICE_IDENTIFIER = 0x022b, + PKCS11_CKA_OTP_SERVICE_LOGO = 0x022c, + PKCS11_CKA_OTP_SERVICE_LOGO_TYPE = 0x022d, + PKCS11_CKA_GOSTR3410_PARAMS = 0x0250, + PKCS11_CKA_GOSTR3411_PARAMS = 0x0251, + PKCS11_CKA_GOST28147_PARAMS = 0x0252, + PKCS11_CKA_HW_FEATURE_TYPE = 0x0300, + PKCS11_CKA_RESET_ON_INIT = 0x0301, + PKCS11_CKA_HAS_RESET = 0x0302, + PKCS11_CKA_PIXEL_X = 0x0400, + PKCS11_CKA_PIXEL_Y = 0x0401, + PKCS11_CKA_RESOLUTION = 0x0402, + PKCS11_CKA_CHAR_ROWS = 0x0403, + PKCS11_CKA_CHAR_COLUMNS = 0x0404, + PKCS11_CKA_COLOR = 0x0405, + PKCS11_CKA_BITS_PER_PIXEL = 0x0406, + PKCS11_CKA_CHAR_SETS = 0x0480, + PKCS11_CKA_ENCODING_METHODS = 0x0481, + PKCS11_CKA_MIME_TYPES = 0x0482, + PKCS11_CKA_MECHANISM_TYPE = 0x0500, + PKCS11_CKA_REQUIRED_CMS_ATTRIBUTES = 0x0501, + PKCS11_CKA_DEFAULT_CMS_ATTRIBUTES = 0x0502, + PKCS11_CKA_SUPPORTED_CMS_ATTRIBUTES = 0x0503, + PKCS11_CKA_ALLOWED_MECHANISMS = PKCS11_CKF_ARRAY_ATTRIBUTE | + 0x0600, + + /* Vendor specific attributes */ + + /** + * TEE Internal API requires to have EC public key information + * available for private key operations. As EC private key object + * should not include CKA_EC_POINT include hidden one so that it does + * not need to be calculated on each operation. + */ + PKCS11_CKA_OPTEE_HIDDEN_EC_POINT = PKCS11_CKA_VENDOR_DEFINED | + PKCS11_CKA_OPTEE_FLAGS_HIDDEN | + 0x0000, + + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKA_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_CLASS + * PKCS11_CKO_ reflects CryptoKi client API object class IDs CKO_. + */ +enum pkcs11_class_id { + PKCS11_CKO_DATA = 0x000, + PKCS11_CKO_CERTIFICATE = 0x001, + PKCS11_CKO_PUBLIC_KEY = 0x002, + PKCS11_CKO_PRIVATE_KEY = 0x003, + PKCS11_CKO_SECRET_KEY = 0x004, + PKCS11_CKO_HW_FEATURE = 0x005, + PKCS11_CKO_DOMAIN_PARAMETERS = 0x006, + PKCS11_CKO_MECHANISM = 0x007, + PKCS11_CKO_OTP_KEY = 0x008, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKO_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_KEY_TYPE + * PKCS11_CKK_ reflects CryptoKi client API key type IDs CKK_. + * Note that this is only a subset of the PKCS#11 specification. + */ +enum pkcs11_key_type { + PKCS11_CKK_RSA = 0x000, + PKCS11_CKK_DSA = 0x001, + PKCS11_CKK_DH = 0x002, + PKCS11_CKK_EC = 0x003, + PKCS11_CKK_EDDSA = 0x004, + PKCS11_CKK_GENERIC_SECRET = 0x010, + PKCS11_CKK_AES = 0x01f, + PKCS11_CKK_MD5_HMAC = 0x027, + PKCS11_CKK_SHA_1_HMAC = 0x028, + PKCS11_CKK_SHA256_HMAC = 0x02b, + PKCS11_CKK_SHA384_HMAC = 0x02c, + PKCS11_CKK_SHA512_HMAC = 0x02d, + PKCS11_CKK_SHA224_HMAC = 0x02e, + PKCS11_CKK_EC_EDWARDS = 0x040, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKK_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_CERTIFICATE_TYPE + */ +enum pkcs11_certificate_type { + PKCS11_CKC_X_509 = 0x00000000UL, + PKCS11_CKC_X_509_ATTR_CERT = 0x00000001UL, + PKCS11_CKC_WTLS = 0x00000002UL, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKC_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for attribute PKCS11_CKA_CERTIFICATE_CATEGORY + */ +enum pkcs11_certificate_category { + PKCS11_CK_CERTIFICATE_CATEGORY_UNSPECIFIED = 0UL, + PKCS11_CK_CERTIFICATE_CATEGORY_TOKEN_USER = 1UL, + PKCS11_CK_CERTIFICATE_CATEGORY_AUTHORITY = 2UL, + PKCS11_CK_CERTIFICATE_CATEGORY_OTHER_ENTITY = 3UL, +}; + +/* + * Valid values for mechanism IDs + * PKCS11_CKM_ reflects CryptoKi client API mechanism IDs CKM_. + * Note that this will be extended as needed. + */ +enum pkcs11_mechanism_id { + PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN = 0x00000, + PKCS11_CKM_RSA_PKCS = 0x00001, + PKCS11_CKM_MD5_RSA_PKCS = 0x00005, + PKCS11_CKM_SHA1_RSA_PKCS = 0x00006, + PKCS11_CKM_RSA_PKCS_OAEP = 0x00009, + PKCS11_CKM_RSA_PKCS_PSS = 0x0000d, + PKCS11_CKM_SHA1_RSA_PKCS_PSS = 0x0000e, + PKCS11_CKM_SHA256_RSA_PKCS = 0x00040, + PKCS11_CKM_SHA384_RSA_PKCS = 0x00041, + PKCS11_CKM_SHA512_RSA_PKCS = 0x00042, + PKCS11_CKM_SHA256_RSA_PKCS_PSS = 0x00043, + PKCS11_CKM_SHA384_RSA_PKCS_PSS = 0x00044, + PKCS11_CKM_SHA512_RSA_PKCS_PSS = 0x00045, + PKCS11_CKM_SHA224_RSA_PKCS = 0x00046, + PKCS11_CKM_SHA224_RSA_PKCS_PSS = 0x00047, + PKCS11_CKM_MD5 = 0x00210, + PKCS11_CKM_MD5_HMAC = 0x00211, + PKCS11_CKM_MD5_HMAC_GENERAL = 0x00212, + PKCS11_CKM_SHA_1 = 0x00220, + PKCS11_CKM_SHA_1_HMAC = 0x00221, + PKCS11_CKM_SHA_1_HMAC_GENERAL = 0x00222, + PKCS11_CKM_SHA256 = 0x00250, + PKCS11_CKM_SHA256_HMAC = 0x00251, + PKCS11_CKM_SHA256_HMAC_GENERAL = 0x00252, + PKCS11_CKM_SHA224 = 0x00255, + PKCS11_CKM_SHA224_HMAC = 0x00256, + PKCS11_CKM_SHA224_HMAC_GENERAL = 0x00257, + PKCS11_CKM_SHA384 = 0x00260, + PKCS11_CKM_SHA384_HMAC = 0x00261, + PKCS11_CKM_SHA384_HMAC_GENERAL = 0x00262, + PKCS11_CKM_SHA512 = 0x00270, + PKCS11_CKM_SHA512_HMAC = 0x00271, + PKCS11_CKM_SHA512_HMAC_GENERAL = 0x00272, + PKCS11_CKM_GENERIC_SECRET_KEY_GEN = 0x00350, + PKCS11_CKM_EC_KEY_PAIR_GEN = 0x01040, + PKCS11_CKM_ECDSA = 0x01041, + PKCS11_CKM_ECDSA_SHA1 = 0x01042, + PKCS11_CKM_ECDSA_SHA224 = 0x01043, + PKCS11_CKM_ECDSA_SHA256 = 0x01044, + PKCS11_CKM_ECDSA_SHA384 = 0x01045, + PKCS11_CKM_ECDSA_SHA512 = 0x01046, + PKCS11_CKM_ECDH1_DERIVE = 0x01050, + PKCS11_CKM_RSA_AES_KEY_WRAP = 0x01054, + PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN = 0x01055, + PKCS11_CKM_EDDSA = 0x01057, + PKCS11_CKM_AES_KEY_GEN = 0x01080, + PKCS11_CKM_AES_ECB = 0x01081, + PKCS11_CKM_AES_CBC = 0x01082, + PKCS11_CKM_AES_CBC_PAD = 0x01085, + PKCS11_CKM_AES_CTR = 0x01086, + PKCS11_CKM_AES_CTS = 0x01089, + PKCS11_CKM_AES_CMAC = 0x0108a, + PKCS11_CKM_AES_CMAC_GENERAL = 0x0108b, + PKCS11_CKM_AES_ECB_ENCRYPT_DATA = 0x01104, + PKCS11_CKM_AES_CBC_ENCRYPT_DATA = 0x01105, + /* + * Vendor extensions below. + * PKCS11 added IDs for operation not related to a CK mechanism ID + */ + PKCS11_PROCESSING_IMPORT = 0x80000000, + PKCS11_CKM_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * PKCS11_CKD_ reflects CryptoKi client API key diff function IDs CKD_. + */ +enum pkcs11_keydiff_id { + PKCS11_CKD_NULL = 0x0001, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKD_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values MG function identifiers + * PKCS11_CKG_ reflects CryptoKi client API MG function IDs CKG_. + */ +enum pkcs11_mgf_id { + PKCS11_CKG_MGF1_SHA1 = 0x0001, + PKCS11_CKG_MGF1_SHA224 = 0x0005, + PKCS11_CKG_MGF1_SHA256 = 0x0002, + PKCS11_CKG_MGF1_SHA384 = 0x0003, + PKCS11_CKG_MGF1_SHA512 = 0x0004, + /* Vendor extension: reserved for undefined ID (~0U) */ + PKCS11_CKG_UNDEFINED_ID = PKCS11_UNDEFINED_ID, +}; + +/* + * Valid values for RSA PKCS/OAEP source type identifier + * PKCS11_CKZ_ reflects CryptoKi client API source type IDs CKZ_. + */ +#define PKCS11_CKZ_DATA_SPECIFIED 0x0001 + +#endif /*PKCS11_TA_H*/ diff --git a/optee_os/ta/pkcs11/scripts/dump_ec_curve_params.sh b/optee_os/ta/pkcs11/scripts/dump_ec_curve_params.sh new file mode 100755 index 0000000..61709db --- /dev/null +++ b/optee_os/ta/pkcs11/scripts/dump_ec_curve_params.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-2-Clause + +EC_CURVES="prime192v1 secp224r1 prime256v1 secp384r1 secp521r1" + +echo "/*" +echo " * DER encoded EC parameters generated with script:" +echo " * ta/pkcs11/scripts/dump_ec_params.sh" +echo " */" +echo "" + +for EC_CURVE in ${EC_CURVES} ; do + echo "static const uint8_t ${EC_CURVE}_name_der[] = {" + openssl ecparam -name ${EC_CURVE} -param_enc named_curve | \ + openssl asn1parse -noout -out /dev/stdout | \ + hexdump -v -e '/8 "\t"' -e '/1 "0x%02x, " ' -e '/8 "\n"' | \ + sed 's/0x ,//g' + echo "};" + echo "" +done + +for EC_CURVE in ${EC_CURVES} ; do + echo "static const uint8_t ${EC_CURVE}_oid_der[] = {" + openssl ecparam -name ${EC_CURVE} -param_enc explicit | \ + openssl asn1parse -noout -out /dev/stdout | \ + hexdump -v -e '/8 "\t"' -e '/1 "0x%02x, " ' -e '/8 "\n"' | \ + sed 's/0x ,//g' + echo "};" + echo "" +done diff --git a/optee_os/ta/pkcs11/scripts/verify-helpers.sh b/optee_os/ta/pkcs11/scripts/verify-helpers.sh new file mode 100755 index 0000000..32b8da2 --- /dev/null +++ b/optee_os/ta/pkcs11/scripts/verify-helpers.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-2-Clause + +SHOW_DETAILS=1 +ERRORS="" +basepath="$(dirname $(dirname $(readlink -f "$0")))" + +details () [[ ${SHOW_DETAILS} -ne 0 ]] + +verify_enum () { + PREFIX=$1 + SYMBOL_LIST="$(grep PKCS11\_$PREFIX\_ ${basepath}/include/pkcs11_ta.h | cut -f2 | cut '-d ' -f1)" + details && echo -e "\e[4m${PREFIX}\e[0m" + details && echo "" + details && echo -e "symbol\tuses\tstatus" + for symbol in $SYMBOL_LIST ; do + details && echo -n "$symbol" + COUNT="$(grep ${symbol} ${basepath}/src/* | wc -l)" + details && echo -n -e "\t${COUNT}\t" + PRESENT="$(grep PKCS11_ID.*\($symbol.*\) ${basepath}/src/pkcs11_helpers.c | wc -l)" + if [ ${PRESENT} -ne 0 ] ; then + details && echo -e "\e[32mOK\e[0m" + elif [ $COUNT -eq 0 ] ; then + details && echo -e "\e[33mmissing but unused\e[0m" + else + details && echo -e "\e[31mMISSING!\e[0m" + ERRORS+=" ${symbol}" + fi + done + details && echo "" +} + +verify_define () { + PREFIX=$1 + SYMBOL_LIST="$(grep PKCS11\_$PREFIX\_ ${basepath}/include/pkcs11_ta.h | grep ^#define | cut '-d ' -f2 | cut -f1)" + details && echo -e "\e[4m${PREFIX}\e[0m" + details && echo "" + details && echo -e "symbol\tuses\tstatus" + for symbol in $SYMBOL_LIST ; do + details && echo -n "$symbol" + COUNT="$(grep ${symbol} ${basepath}/src/* | wc -l)" + details && echo -n -e "\t${COUNT}\t" + PRESENT="$(grep PKCS11_ID.*\($symbol.*\) ${basepath}/src/pkcs11_helpers.c | wc -l)" + if [ ${PRESENT} -ne 0 ] ; then + details && echo -e "\e[32mOK\e[0m" + elif [ $COUNT -eq 0 ] ; then + details && echo -e "\e[33mmissing but unused\e[0m" + else + details && echo -e "\e[31mMISSING!\e[0m" + ERRORS+=" ${symbol}" + fi + done + details && echo "" +} + +usage() { + SCR=$(basename "$0") + echo "Usage: $SCR Verify that helpers are up to date" + echo " $SCR --quiet Only print errors" + echo " $SCR --help This help" + echo "" + echo "Verification checks that all PKCS11_* enums or defines from \ +include/pkcs11_ta.h are either present in src/pkcs11_helpers.c or not used at \ +all." + exit 1 +} + +while [[ $# -gt 0 ]]; do + arg="$1" + shift + + case $arg in + -q|--quiet) + SHOW_DETAILS=0 + ;; + -h|--help) + usage + ;; + *) + echo "error: invalid argument: ${arg}" 1>&2 + exit 1 + esac +done + +# check that symbols exists +verify_enum CKA +verify_define CKFS +verify_define CKFT +verify_define CKFSS +verify_enum CKS +verify_enum CKR +verify_enum CKO +verify_enum CKK + +if [ -n "${ERRORS}" ] ; then + SCR=$(basename "$0") + for symbol in $ERRORS ; do + echo "${SCR}: error: missing symbol ${symbol} in ${basepath}/src/pkcs11_helpers.c" 1>&2 + done + exit 1 +fi + +exit 0 diff --git a/optee_os/ta/pkcs11/src/attributes.c b/optee_os/ta/pkcs11/src/attributes.c new file mode 100644 index 0000000..1083cea --- /dev/null +++ b/optee_os/ta/pkcs11/src/attributes.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "pkcs11_helpers.h" +#include "serializer.h" + +enum pkcs11_rc init_attributes_head(struct obj_attrs **head) +{ + *head = TEE_Malloc(sizeof(**head), TEE_MALLOC_FILL_ZERO); + if (!*head) + return PKCS11_CKR_DEVICE_MEMORY; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc add_attribute(struct obj_attrs **head, uint32_t attribute, + void *data, size_t size) +{ + size_t buf_len = sizeof(struct obj_attrs) + (*head)->attrs_size; + char **bstart = (void *)head; + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint32_t data32 = 0; + + data32 = attribute; + rc = serialize(bstart, &buf_len, &data32, sizeof(uint32_t)); + if (rc) + return rc; + + data32 = size; + rc = serialize(bstart, &buf_len, &data32, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialize(bstart, &buf_len, data, size); + if (rc) + return rc; + + /* Alloced buffer is always well aligned */ + head = (void *)bstart; + (*head)->attrs_size += 2 * sizeof(uint32_t) + size; + (*head)->attrs_count++; + + return rc; +} + +static enum pkcs11_rc _remove_attribute(struct obj_attrs **head, + uint32_t attribute, bool empty) +{ + struct obj_attrs *h = *head; + char *cur = NULL; + char *end = NULL; + size_t next_off = 0; + + /* Let's find the target attribute */ + cur = (char *)h + sizeof(struct obj_attrs); + end = cur + h->attrs_size; + for (; cur < end; cur += next_off) { + struct pkcs11_attribute_head pkcs11_ref = { }; + + TEE_MemMove(&pkcs11_ref, cur, sizeof(pkcs11_ref)); + next_off = sizeof(pkcs11_ref) + pkcs11_ref.size; + + if (pkcs11_ref.id != attribute) + continue; + + if (empty && pkcs11_ref.size) + return PKCS11_CKR_FUNCTION_FAILED; + + TEE_MemMove(cur, cur + next_off, end - (cur + next_off)); + + h->attrs_count--; + h->attrs_size -= next_off; + end -= next_off; + next_off = 0; + + return PKCS11_CKR_OK; + } + + DMSG("Attribute %s (%#x) not found", id2str_attr(attribute), attribute); + return PKCS11_RV_NOT_FOUND; +} + +enum pkcs11_rc remove_empty_attribute(struct obj_attrs **head, + uint32_t attribute) +{ + return _remove_attribute(head, attribute, true /* empty */); +} + +void get_attribute_ptrs(struct obj_attrs *head, uint32_t attribute, + void **attr, uint32_t *attr_size, size_t *count) +{ + char *cur = (char *)head + sizeof(struct obj_attrs); + char *end = cur + head->attrs_size; + size_t next_off = 0; + size_t max_found = *count; + size_t found = 0; + void **attr_ptr = attr; + uint32_t *attr_size_ptr = attr_size; + + for (; cur < end; cur += next_off) { + /* Structure aligned copy of the pkcs11_ref in the object */ + struct pkcs11_attribute_head pkcs11_ref = { }; + + TEE_MemMove(&pkcs11_ref, cur, sizeof(pkcs11_ref)); + next_off = sizeof(pkcs11_ref) + pkcs11_ref.size; + + if (pkcs11_ref.id != attribute) + continue; + + found++; + + if (!max_found) + continue; /* only count matching attributes */ + + if (attr) { + if (pkcs11_ref.size) + *attr_ptr++ = cur + sizeof(pkcs11_ref); + else + *attr_ptr++ = NULL; + } + + if (attr_size) + *attr_size_ptr++ = pkcs11_ref.size; + + if (found == max_found) + break; + } + + /* Sanity */ + if (cur > end) { + DMSG("Exceeding serial object length"); + TEE_Panic(0); + } + + *count = found; +} + +enum pkcs11_rc get_attribute_ptr(struct obj_attrs *head, uint32_t attribute, + void **attr_ptr, uint32_t *attr_size) +{ + size_t count = 1; + + get_attribute_ptrs(head, attribute, attr_ptr, attr_size, &count); + + if (!count) + return PKCS11_RV_NOT_FOUND; + + if (count != 1) + return PKCS11_CKR_GENERAL_ERROR; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc get_attribute(struct obj_attrs *head, uint32_t attribute, + void *attr, uint32_t *attr_size) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + void *attr_ptr = NULL; + uint32_t size = 0; + + rc = get_attribute_ptr(head, attribute, &attr_ptr, &size); + if (rc) + return rc; + + if (attr_size && *attr_size < size) { + *attr_size = size; + /* This reuses buffer-to-small for any bad size matching */ + return PKCS11_CKR_BUFFER_TOO_SMALL; + } + + if (attr) + TEE_MemMove(attr, attr_ptr, size); + + if (attr_size) + *attr_size = size; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc set_attribute(struct obj_attrs **head, uint32_t attribute, + void *data, size_t size) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + + rc = _remove_attribute(head, attribute, false); + if (rc != PKCS11_CKR_OK && rc != PKCS11_RV_NOT_FOUND) + return rc; + + return add_attribute(head, attribute, data, size); +} + +enum pkcs11_rc modify_attributes_list(struct obj_attrs **dst, + struct obj_attrs *head) +{ + char *cur = (char *)head + sizeof(struct obj_attrs); + char *end = cur + head->attrs_size; + size_t len = 0; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + for (; cur < end; cur += len) { + struct pkcs11_attribute_head *cli_ref = (void *)cur; + /* Structure aligned copy of the pkcs11_ref in the object */ + struct pkcs11_attribute_head cli_head = { }; + + TEE_MemMove(&cli_head, cur, sizeof(cli_head)); + len = sizeof(cli_head) + cli_head.size; + + rc = set_attribute(dst, cli_head.id, + cli_head.size ? cli_ref->data : NULL, + cli_head.size); + if (rc) + return rc; + } + + return PKCS11_CKR_OK; +} + +bool get_bool(struct obj_attrs *head, uint32_t attribute) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint8_t bbool = 0; + uint32_t size = sizeof(bbool); + + rc = get_attribute(head, attribute, &bbool, &size); + + if (rc == PKCS11_RV_NOT_FOUND) + return false; + + assert(rc == PKCS11_CKR_OK); + return bbool; +} + +bool attributes_match_reference(struct obj_attrs *candidate, + struct obj_attrs *ref) +{ + size_t count = ref->attrs_count; + unsigned char *ref_attr = ref->attrs; + uint32_t rc = PKCS11_CKR_GENERAL_ERROR; + + if (!ref->attrs_count) { + DMSG("Empty reference match all"); + return true; + } + + for (count = 0; count < ref->attrs_count; count++) { + struct pkcs11_attribute_head pkcs11_ref = { }; + void *value = NULL; + uint32_t size = 0; + + TEE_MemMove(&pkcs11_ref, ref_attr, sizeof(pkcs11_ref)); + + /* Hidden attributes cannot be matched */ + if (attribute_is_hidden(&pkcs11_ref)) + return false; + + rc = get_attribute_ptr(candidate, pkcs11_ref.id, &value, &size); + + if (rc || !value || size != pkcs11_ref.size || + TEE_MemCompare(ref_attr + sizeof(pkcs11_ref), value, size)) + return false; + + ref_attr += sizeof(pkcs11_ref) + pkcs11_ref.size; + } + + return true; +} + +enum pkcs11_rc attributes_match_add_reference(struct obj_attrs **head, + struct obj_attrs *ref) +{ + size_t count = ref->attrs_count; + unsigned char *ref_attr = ref->attrs; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + if (!ref->attrs_count) + return PKCS11_CKR_OK; + + for (count = 0; count < ref->attrs_count; count++) { + struct pkcs11_attribute_head pkcs11_ref = { }; + void *value = NULL; + uint32_t size = 0; + + TEE_MemMove(&pkcs11_ref, ref_attr, sizeof(pkcs11_ref)); + + rc = get_attribute_ptr(*head, pkcs11_ref.id, &value, &size); + if (rc == PKCS11_RV_NOT_FOUND) { + rc = add_attribute(head, pkcs11_ref.id, + ref_attr + sizeof(pkcs11_ref), + pkcs11_ref.size); + if (rc) + return rc; + } else { + if (rc || !value || size != pkcs11_ref.size || + TEE_MemCompare(ref_attr + sizeof(pkcs11_ref), value, + size)) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + ref_attr += sizeof(pkcs11_ref) + pkcs11_ref.size; + } + + return PKCS11_CKR_OK; +} + +#if CFG_TEE_TA_LOG_LEVEL > 0 +/* + * Debug: dump CK attribute array to output trace + */ +#define ATTR_TRACE_FMT "%s attr %s / %s\t(0x%04"PRIx32" %"PRIu32"-byte" +#define ATTR_FMT_0BYTE ATTR_TRACE_FMT ")" +#define ATTR_FMT_1BYTE ATTR_TRACE_FMT ": %02x)" +#define ATTR_FMT_2BYTE ATTR_TRACE_FMT ": %02x %02x)" +#define ATTR_FMT_3BYTE ATTR_TRACE_FMT ": %02x %02x %02x)" +#define ATTR_FMT_4BYTE ATTR_TRACE_FMT ": %02x %02x %02x %02x)" +#define ATTR_FMT_ARRAY ATTR_TRACE_FMT ": %02x %02x %02x %02x ...)" + +static void __trace_attributes(char *prefix, void *src, void *end) +{ + size_t next_off = 0; + char *prefix2 = NULL; + size_t prefix_len = strlen(prefix); + char *cur = src; + + /* append 4 spaces to the prefix plus terminal '\0' */ + prefix2 = TEE_Malloc(prefix_len + 1 + 4, TEE_MALLOC_FILL_ZERO); + if (!prefix2) + return; + + TEE_MemMove(prefix2, prefix, prefix_len + 1); + TEE_MemFill(prefix2 + prefix_len, ' ', 4); + *(prefix2 + prefix_len + 4) = '\0'; + + for (; cur < (char *)end; cur += next_off) { + struct pkcs11_attribute_head pkcs11_ref = { }; + uint8_t data[4] = { 0 }; + + TEE_MemMove(&pkcs11_ref, cur, sizeof(pkcs11_ref)); + TEE_MemMove(&data[0], cur + sizeof(pkcs11_ref), + MIN(pkcs11_ref.size, sizeof(data))); + + next_off = sizeof(pkcs11_ref) + pkcs11_ref.size; + + switch (pkcs11_ref.size) { + case 0: + IMSG_RAW(ATTR_FMT_0BYTE, + prefix, id2str_attr(pkcs11_ref.id), "*", + pkcs11_ref.id, pkcs11_ref.size); + break; + case 1: + IMSG_RAW(ATTR_FMT_1BYTE, + prefix, id2str_attr(pkcs11_ref.id), + id2str_attr_value(pkcs11_ref.id, + pkcs11_ref.size, + cur + sizeof(pkcs11_ref)), + pkcs11_ref.id, pkcs11_ref.size, data[0]); + break; + case 2: + IMSG_RAW(ATTR_FMT_2BYTE, + prefix, id2str_attr(pkcs11_ref.id), + id2str_attr_value(pkcs11_ref.id, + pkcs11_ref.size, + cur + sizeof(pkcs11_ref)), + pkcs11_ref.id, pkcs11_ref.size, data[0], + data[1]); + break; + case 3: + IMSG_RAW(ATTR_FMT_3BYTE, + prefix, id2str_attr(pkcs11_ref.id), + id2str_attr_value(pkcs11_ref.id, + pkcs11_ref.size, + cur + sizeof(pkcs11_ref)), + pkcs11_ref.id, pkcs11_ref.size, + data[0], data[1], data[2]); + break; + case 4: + IMSG_RAW(ATTR_FMT_4BYTE, + prefix, id2str_attr(pkcs11_ref.id), + id2str_attr_value(pkcs11_ref.id, + pkcs11_ref.size, + cur + sizeof(pkcs11_ref)), + pkcs11_ref.id, pkcs11_ref.size, + data[0], data[1], data[2], data[3]); + break; + default: + IMSG_RAW(ATTR_FMT_ARRAY, + prefix, id2str_attr(pkcs11_ref.id), + id2str_attr_value(pkcs11_ref.id, + pkcs11_ref.size, + cur + sizeof(pkcs11_ref)), + pkcs11_ref.id, pkcs11_ref.size, + data[0], data[1], data[2], data[3]); + break; + } + + switch (pkcs11_ref.id) { + case PKCS11_CKA_WRAP_TEMPLATE: + case PKCS11_CKA_UNWRAP_TEMPLATE: + case PKCS11_CKA_DERIVE_TEMPLATE: + if (pkcs11_ref.size) + trace_attributes(prefix2, + cur + sizeof(pkcs11_ref)); + break; + default: + break; + } + } + + /* Sanity */ + if (cur != end) + EMSG("Warning: unexpected alignment in object attributes"); + + TEE_Free(prefix2); +} + +void trace_attributes(const char *prefix, void *ref) +{ + struct obj_attrs head = { }; + char *pre = NULL; + + TEE_MemMove(&head, ref, sizeof(head)); + + if (!head.attrs_count) + return; + + pre = TEE_Malloc(prefix ? strlen(prefix) + 2 : 2, TEE_MALLOC_FILL_ZERO); + if (!pre) { + EMSG("%s: out of memory", prefix); + return; + } + + if (prefix) + TEE_MemMove(pre, prefix, strlen(prefix)); + + IMSG_RAW("%s,--- (serial object) Attributes list --------", pre); + IMSG_RAW("%s| %"PRIu32" item(s) - %"PRIu32" bytes", + pre, head.attrs_count, head.attrs_size); + + pre[prefix ? strlen(prefix) : 0] = '|'; + __trace_attributes(pre, (char *)ref + sizeof(head), + (char *)ref + sizeof(head) + head.attrs_size); + + IMSG_RAW("%s`-----------------------", prefix ? prefix : ""); + + TEE_Free(pre); +} +#endif /*CFG_TEE_TA_LOG_LEVEL*/ diff --git a/optee_os/ta/pkcs11/src/attributes.h b/optee_os/ta/pkcs11/src/attributes.h new file mode 100644 index 0000000..2746ab6 --- /dev/null +++ b/optee_os/ta/pkcs11/src/attributes.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_ATTRIBUTES_H +#define PKCS11_TA_ATTRIBUTES_H + +#include +#include +#include +#include + +#include "pkcs11_helpers.h" + +/* + * Boolean property attributes (BPA): bit position in a 64 bit mask + * for boolean properties object can mandate as attribute, depending + * on the object. These attributes are often accessed and it is + * quicker to get them from a 64 bit field in the object instance + * rather than searching into the object attributes. + */ +#define PKCS11_BOOLPROPS_BASE 0 +#define PKCS11_BOOLPROPS_MAX_COUNT 64 + +enum boolprop_attr { + BPA_TOKEN = 0, + BPA_PRIVATE, + BPA_TRUSTED, + BPA_SENSITIVE, + BPA_ENCRYPT, + BPA_DECRYPT, + BPA_WRAP, + BPA_UNWRAP, + BPA_SIGN, + BPA_SIGN_RECOVER, + BPA_VERIFY, + BPA_VERIFY_RECOVER, + BPA_DERIVE, + BPA_EXTRACTABLE, + BPA_LOCAL, + BPA_NEVER_EXTRACTABLE, + BPA_ALWAYS_SENSITIVE, + BPA_MODIFIABLE, + BPA_COPYABLE, + BPA_DESTROYABLE, + BPA_ALWAYS_AUTHENTICATE, + BPA_WRAP_WITH_TRUSTED, +}; + +/* + * Header of a serialized memory object inside PKCS11 TA. + * + * @attrs_size: byte size of the serialized data + * @attrs_count: number of items in the blob + * @attrs: then starts the blob binary data + */ +struct obj_attrs { + uint32_t attrs_size; + uint32_t attrs_count; + uint8_t attrs[]; +}; + +/* + * init_attributes_head() - Allocate a reference for serialized attributes + * @head: *@head holds the retrieved pointer + * + * Retrieved pointer can be freed from a simple TEE_Free(reference). + * + * Return PKCS11_CKR_OK on success or a PKCS11 return code. + */ +enum pkcs11_rc init_attributes_head(struct obj_attrs **head); + +/* + * add_attribute() - Update serialized attributes to add an entry. + * + * @head: *@head points to serialized attributes, + * can be reallocated as attributes are added + * @attribute: Attribute ID to add + * @data: Opaque data of attribute + * @size: Size of data + * + * Return PKCS11_CKR_OK on success or a PKCS11 return code. + */ +enum pkcs11_rc add_attribute(struct obj_attrs **head, uint32_t attribute, + void *data, size_t size); + +/* + * Update serialized attributes to remove an empty entry. Can relocate the + * attribute list buffer. Only 1 instance of the entry is expected. + * + * Return PKCS11_CKR_OK on success or a PKCS11 return code. + */ +enum pkcs11_rc remove_empty_attribute(struct obj_attrs **head, uint32_t attrib); + +/* + * get_attribute_ptrs() - Get pointers to attributes with a given ID + * @head: Pointer to serialized attributes + * @attribute: Attribute ID to look for + * @attr: Array of pointers to the data inside @head + * @attr_size: Array of uint32_t holding the sizes of each value pointed to + * by @attr + * @count: Number of elements in the arrays above + * + * If *count == 0, count and return in *count the number of attributes matching + * the input attribute ID. + * + * If *count != 0, return the address and size of the attributes found, up to + * the occurrence number *count. attr and attr_size are expected large + * enough. attr is the output array of the values found. attr_size is the + * output array of the size of each value found. + * + * If attr_size != NULL, return in *attr_size attribute value size. + * If attr != NULL return in *attr the address of the attribute value. + */ +void get_attribute_ptrs(struct obj_attrs *head, uint32_t attribute, + void **attr, uint32_t *attr_size, size_t *count); + +/* + * get_attribute_ptrs() - Get pointer to the attribute of a given ID + * @head: Pointer to serialized attributes + * @attribute: Attribute ID + * @attr: *@attr holds the retrieved pointer to the attribute value + * @attr_size: Size of the attribute value + * + * If no matching attributes is found return PKCS11_RV_NOT_FOUND. + * If attr_size != NULL, return in *attr_size attribute value size. + * If attr != NULL, return in *attr the address of the attribute value. + * + * Return PKCS11_CKR_OK or PKCS11_RV_NOT_FOUND on success, or a PKCS11 return + * code. + */ +enum pkcs11_rc get_attribute_ptr(struct obj_attrs *head, uint32_t attribute, + void **attr_ptr, uint32_t *attr_size); + +/* + * get_attribute() - Copy out the attribute of a given ID + * @head: Pointer to serialized attributes + * @attribute: Attribute ID to look for + * @attr: holds the retrieved attribute value + * @attr_size: Size of the attribute value + * + * If attribute is not found, return PKCS11_RV_NOT_FOUND. + * + * If attr_size != NULL, check that attr has enough room for value (compare + * against *attr_size), copy attribute value to attr and finally return actual + * value size in *attr_size. + * + * If there is not enough room return PKCS11_CKR_BUFFER_TOO_SMALL with expected + * size in *attr_size. + * + * If attr is NULL and attr_size != NULL return expected buffer size in + * *attr_size. + * + * Return PKCS11_CKR_OK or PKCS11_RV_NOT_FOUND on success, or a PKCS11 return + * code. + */ +enum pkcs11_rc get_attribute(struct obj_attrs *head, uint32_t attribute, + void *attr, uint32_t *attr_size); + +/* + * set_attribute() - Set the attribute of a given ID with value + * @head: Pointer to serialized attributes where attribute is to be set, + * can be relocated as attributes are modified/added + * @attribute: Attribute ID to look for + * @data: Holds the attribute value to be set + * @size: Size of the attribute value + * + * Return PKCS11_CKR_OK on success or a PKCS11 return code. + */ +enum pkcs11_rc set_attribute(struct obj_attrs **head, uint32_t attribute, + void *data, size_t size); + +/* + * modify_attributes_list() - Modify the value of attributes in destination + * attribute list (serialized attributes) based on the value of attributes in + * the source attribute list + * @dst: Pointer to serialized attrbutes where attributes are to be + * modified, can be relocated as attributes are modified + * @head: Serialized attributes containing attributes which need to be + * modified in the destination attribute list + * + * Return PKCS11_CKR_OK on success + */ +enum pkcs11_rc modify_attributes_list(struct obj_attrs **dst, + struct obj_attrs *head); + +/* + * get_u32_attribute() - Copy out the 32-bit attribute value of a given ID + * @head: Pointer to serialized attributes + * @attribute: Attribute ID + * @attr: holds the retrieved 32-bit attribute value + * + * If attribute is not found, return PKCS11_RV_NOT_FOUND. + * If the retreived attribute doesn't have a 4 byte sized value + * PKCS11_CKR_GENERAL_ERROR is returned. + * + * Return PKCS11_CKR_OK or PKCS11_RV_NOT_FOUND on success, or a PKCS11 return + * code. + */ + +static inline enum pkcs11_rc get_u32_attribute(struct obj_attrs *head, + uint32_t attribute, + uint32_t *attr) +{ + uint32_t size = sizeof(uint32_t); + enum pkcs11_rc rc = get_attribute(head, attribute, attr, &size); + + if (!rc && size != sizeof(uint32_t)) + return PKCS11_CKR_GENERAL_ERROR; + + return rc; +} + +/* + * Return true if all attributes from the reference are found and match value + * in the candidate attribute list. + */ +bool attributes_match_reference(struct obj_attrs *ref, + struct obj_attrs *candidate); + +/* + * Check attributes from @ref are all found or added in @head + * + * Return PKCS11_CKR_OK on success, or a PKCS11 return code. + */ +enum pkcs11_rc attributes_match_add_reference(struct obj_attrs **head, + struct obj_attrs *ref); +/* + * get_class() - Get class ID of an object + * @head: Pointer to serialized attributes + * + * Returns the class ID of an object on succes or returns + * PKCS11_CKO_UNDEFINED_ID on error. + */ +static inline enum pkcs11_class_id get_class(struct obj_attrs *head) +{ + uint32_t class = 0; + uint32_t size = sizeof(class); + + if (get_attribute(head, PKCS11_CKA_CLASS, &class, &size)) + return PKCS11_CKO_UNDEFINED_ID; + + return class; +} + +/* + * get_key_type() - Get the key type of an object + * @head: Pointer to serialized attributes + * + * Returns the key type of an object on success or returns + * PKCS11_CKK_UNDEFINED_ID on error. + */ +static inline enum pkcs11_key_type get_key_type(struct obj_attrs *head) +{ + uint32_t type = 0; + uint32_t size = sizeof(type); + + if (get_attribute(head, PKCS11_CKA_KEY_TYPE, &type, &size)) + return PKCS11_CKK_UNDEFINED_ID; + + return type; +} + +/* + * get_certificate_type() - Get the certificate type of an object + * @head: Pointer to serialized attributes + * + * Returns the certificate type of an object on success or returns + * PKCS11_CKC_UNDEFINED_ID on error. + */ +static inline +enum pkcs11_certificate_type get_certificate_type(struct obj_attrs *head) +{ + uint32_t type = 0; + + if (get_u32_attribute(head, PKCS11_CKA_CERTIFICATE_TYPE, &type)) + return PKCS11_CKC_UNDEFINED_ID; + + return type; +} + +/* + * get_mechanism_type() - Get the mechanism type of an object + * @head: Pointer to serialized attributes + * + * Returns the mechanism type of an object on success or returns + * PKCS11_CKM_UNDEFINED_ID on error. + */ +static inline enum pkcs11_mechanism_id get_mechanism_type(struct obj_attrs *head) +{ + uint32_t type = 0; + uint32_t size = sizeof(type); + + if (get_attribute(head, PKCS11_CKA_MECHANISM_TYPE, &type, &size)) + return PKCS11_CKM_UNDEFINED_ID; + + return type; +} + +/* + * get_bool() - Get the bool value of an attribute + * @head: Pointer to serialized attributes + * @attribute: Attribute ID to look for + * + * May assert if attribute ID isn't of the boolean type. + * + * Returns the bool value of the supplied attribute ID on success if found + * else false. + */ +bool get_bool(struct obj_attrs *head, uint32_t attribute); + +#if CFG_TEE_TA_LOG_LEVEL > 0 +/* Debug: dump object attributes to IMSG() trace console */ +void trace_attributes(const char *prefix, void *ref); +#else +static inline void trace_attributes(const char *prefix __unused, + void *ref __unused) +{ +} +#endif /*CFG_TEE_TA_LOG_LEVEL*/ +#endif /*PKCS11_TA_ATTRIBUTES_H*/ diff --git a/optee_os/ta/pkcs11/src/entry.c b/optee_os/ta/pkcs11/src/entry.c new file mode 100644 index 0000000..0e36b91 --- /dev/null +++ b/optee_os/ta/pkcs11/src/entry.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "object.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" + +TEE_Result TA_CreateEntryPoint(void) +{ + return pkcs11_init(); +} + +void TA_DestroyEntryPoint(void) +{ + pkcs11_deinit(); +} + +TEE_Result TA_OpenSessionEntryPoint(uint32_t __unused param_types, + TEE_Param __unused params[4], + void **tee_session) +{ + struct pkcs11_client *client = register_client(); + + if (!client) + return TEE_ERROR_OUT_OF_MEMORY; + + *tee_session = client; + + return TEE_SUCCESS; +} + +void TA_CloseSessionEntryPoint(void *tee_session) +{ + struct pkcs11_client *client = tee_session2client(tee_session); + + unregister_client(client); +} + +/* + * Entry point for invocation command PKCS11_CMD_PING + * + * Return a PKCS11_CKR_* value which is also loaded into the output param#0 + */ +static enum pkcs11_rc entry_ping(uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *out = params + 2; + const uint32_t ver[] = { + PKCS11_TA_VERSION_MAJOR, + PKCS11_TA_VERSION_MINOR, + PKCS11_TA_VERSION_PATCH, + }; + + if (ptypes != exp_pt || + params[0].memref.size != TEE_PARAM0_SIZE_MIN || + out->memref.size != sizeof(ver)) + return PKCS11_CKR_ARGUMENTS_BAD; + + TEE_MemMove(out->memref.buffer, ver, sizeof(ver)); + + return PKCS11_CKR_OK; +} + +static bool __maybe_unused param_is_none(uint32_t ptypes, unsigned int index) +{ + return TEE_PARAM_TYPE_GET(ptypes, index) == + TEE_PARAM_TYPE_NONE; +} + +static bool __maybe_unused param_is_memref(uint32_t ptypes, unsigned int index) +{ + switch (TEE_PARAM_TYPE_GET(ptypes, index)) { + case TEE_PARAM_TYPE_MEMREF_INPUT: + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + return true; + default: + return false; + } +} + +static bool __maybe_unused param_is_input(uint32_t ptypes, unsigned int index) +{ + return TEE_PARAM_TYPE_GET(ptypes, index) == + TEE_PARAM_TYPE_MEMREF_INPUT; +} + +static bool __maybe_unused param_is_output(uint32_t ptypes, unsigned int index) +{ + return TEE_PARAM_TYPE_GET(ptypes, index) == + TEE_PARAM_TYPE_MEMREF_OUTPUT; +} + +/* + * Entry point for PKCS11 TA commands + * + * Param#0 (ctrl) is an output or an in/out buffer. Input data are serialized + * arguments for the invoked command while the output data is used to send + * back to the client a PKCS11 finer status ID than the GPD TEE result codes + * Client shall check the status ID from the parameter #0 output buffer together + * with the GPD TEE result code. + */ +TEE_Result TA_InvokeCommandEntryPoint(void *tee_session, uint32_t cmd, + uint32_t ptypes, + TEE_Param params[TEE_NUM_PARAMS]) +{ + struct pkcs11_client *client = tee_session2client(tee_session); + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + if (!client) + return TEE_ERROR_SECURITY; + + /* All command handlers will check only against 4 parameters */ + COMPILE_TIME_ASSERT(TEE_NUM_PARAMS == 4); + + /* + * Param#0 must be either an output or an inout memref as used to + * store the output return value for the invoked command. + */ + switch (TEE_PARAM_TYPE_GET(ptypes, 0)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + if (params[0].memref.size < sizeof(rc)) + return TEE_ERROR_BAD_PARAMETERS; + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + DMSG("%s p#0 %zu@%p, p#1 %s %zu@%p, p#2 %s %zu@%p", + id2str_ta_cmd(cmd), + params[0].memref.size, params[0].memref.buffer, + param_is_input(ptypes, 1) ? "in" : + param_is_output(ptypes, 1) ? "out" : "---", + param_is_memref(ptypes, 1) ? params[1].memref.size : 0, + param_is_memref(ptypes, 1) ? params[1].memref.buffer : NULL, + param_is_input(ptypes, 2) ? "in" : + param_is_output(ptypes, 2) ? "out" : "---", + param_is_memref(ptypes, 2) ? params[2].memref.size : 0, + param_is_memref(ptypes, 2) ? params[2].memref.buffer : NULL); + + switch (cmd) { + case PKCS11_CMD_PING: + rc = entry_ping(ptypes, params); + break; + + case PKCS11_CMD_SLOT_LIST: + rc = entry_ck_slot_list(ptypes, params); + break; + case PKCS11_CMD_SLOT_INFO: + rc = entry_ck_slot_info(ptypes, params); + break; + case PKCS11_CMD_TOKEN_INFO: + rc = entry_ck_token_info(ptypes, params); + break; + case PKCS11_CMD_MECHANISM_IDS: + rc = entry_ck_token_mecha_ids(ptypes, params); + break; + case PKCS11_CMD_MECHANISM_INFO: + rc = entry_ck_token_mecha_info(ptypes, params); + break; + + case PKCS11_CMD_OPEN_SESSION: + rc = entry_ck_open_session(client, ptypes, params); + break; + case PKCS11_CMD_CLOSE_SESSION: + rc = entry_ck_close_session(client, ptypes, params); + break; + case PKCS11_CMD_CLOSE_ALL_SESSIONS: + rc = entry_ck_close_all_sessions(client, ptypes, params); + break; + case PKCS11_CMD_SESSION_INFO: + rc = entry_ck_session_info(client, ptypes, params); + break; + + case PKCS11_CMD_INIT_TOKEN: + rc = entry_ck_token_initialize(ptypes, params); + break; + case PKCS11_CMD_INIT_PIN: + rc = entry_ck_init_pin(client, ptypes, params); + break; + case PKCS11_CMD_SET_PIN: + rc = entry_ck_set_pin(client, ptypes, params); + break; + case PKCS11_CMD_LOGIN: + rc = entry_ck_login(client, ptypes, params); + break; + case PKCS11_CMD_LOGOUT: + rc = entry_ck_logout(client, ptypes, params); + break; + + case PKCS11_CMD_CREATE_OBJECT: + rc = entry_create_object(client, ptypes, params); + break; + case PKCS11_CMD_DESTROY_OBJECT: + rc = entry_destroy_object(client, ptypes, params); + break; + + case PKCS11_CMD_ENCRYPT_INIT: + rc = entry_processing_init(client, ptypes, params, + PKCS11_FUNCTION_ENCRYPT); + break; + case PKCS11_CMD_DECRYPT_INIT: + rc = entry_processing_init(client, ptypes, params, + PKCS11_FUNCTION_DECRYPT); + break; + case PKCS11_CMD_ENCRYPT_UPDATE: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_ENCRYPT, + PKCS11_FUNC_STEP_UPDATE); + break; + case PKCS11_CMD_DECRYPT_UPDATE: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_DECRYPT, + PKCS11_FUNC_STEP_UPDATE); + break; + case PKCS11_CMD_ENCRYPT_ONESHOT: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_ENCRYPT, + PKCS11_FUNC_STEP_ONESHOT); + break; + case PKCS11_CMD_DECRYPT_ONESHOT: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_DECRYPT, + PKCS11_FUNC_STEP_ONESHOT); + break; + case PKCS11_CMD_ENCRYPT_FINAL: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_ENCRYPT, + PKCS11_FUNC_STEP_FINAL); + break; + case PKCS11_CMD_DECRYPT_FINAL: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_DECRYPT, + PKCS11_FUNC_STEP_FINAL); + break; + case PKCS11_CMD_SIGN_INIT: + rc = entry_processing_init(client, ptypes, params, + PKCS11_FUNCTION_SIGN); + break; + case PKCS11_CMD_VERIFY_INIT: + rc = entry_processing_init(client, ptypes, params, + PKCS11_FUNCTION_VERIFY); + break; + case PKCS11_CMD_SIGN_ONESHOT: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_SIGN, + PKCS11_FUNC_STEP_ONESHOT); + break; + case PKCS11_CMD_VERIFY_ONESHOT: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_VERIFY, + PKCS11_FUNC_STEP_ONESHOT); + break; + case PKCS11_CMD_SIGN_UPDATE: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_SIGN, + PKCS11_FUNC_STEP_UPDATE); + break; + case PKCS11_CMD_VERIFY_UPDATE: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_VERIFY, + PKCS11_FUNC_STEP_UPDATE); + break; + case PKCS11_CMD_SIGN_FINAL: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_SIGN, + PKCS11_FUNC_STEP_FINAL); + break; + case PKCS11_CMD_VERIFY_FINAL: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_VERIFY, + PKCS11_FUNC_STEP_FINAL); + break; + case PKCS11_CMD_GENERATE_KEY: + rc = entry_generate_secret(client, ptypes, params); + break; + case PKCS11_CMD_FIND_OBJECTS_INIT: + rc = entry_find_objects_init(client, ptypes, params); + break; + case PKCS11_CMD_FIND_OBJECTS: + rc = entry_find_objects(client, ptypes, params); + break; + case PKCS11_CMD_FIND_OBJECTS_FINAL: + rc = entry_find_objects_final(client, ptypes, params); + break; + case PKCS11_CMD_GET_ATTRIBUTE_VALUE: + rc = entry_get_attribute_value(client, ptypes, params); + break; + case PKCS11_CMD_GET_OBJECT_SIZE: + rc = entry_get_object_size(client, ptypes, params); + break; + case PKCS11_CMD_SET_ATTRIBUTE_VALUE: + rc = entry_set_attribute_value(client, ptypes, params); + break; + case PKCS11_CMD_COPY_OBJECT: + rc = entry_copy_object(client, ptypes, params); + break; + case PKCS11_CMD_SEED_RANDOM: + rc = entry_ck_seed_random(client, ptypes, params); + break; + case PKCS11_CMD_GENERATE_RANDOM: + rc = entry_ck_generate_random(client, ptypes, params); + break; + case PKCS11_CMD_DERIVE_KEY: + rc = entry_processing_key(client, ptypes, params, + PKCS11_FUNCTION_DERIVE); + break; + case PKCS11_CMD_RELEASE_ACTIVE_PROCESSING: + rc = entry_release_active_processing(client, ptypes, params); + break; + case PKCS11_CMD_DIGEST_INIT: + rc = entry_processing_init(client, ptypes, params, + PKCS11_FUNCTION_DIGEST); + break; + case PKCS11_CMD_DIGEST_UPDATE: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_DIGEST, + PKCS11_FUNC_STEP_UPDATE); + break; + case PKCS11_CMD_DIGEST_KEY: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_DIGEST, + PKCS11_FUNC_STEP_UPDATE_KEY); + break; + case PKCS11_CMD_DIGEST_ONESHOT: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_DIGEST, + PKCS11_FUNC_STEP_ONESHOT); + break; + case PKCS11_CMD_DIGEST_FINAL: + rc = entry_processing_step(client, ptypes, params, + PKCS11_FUNCTION_DIGEST, + PKCS11_FUNC_STEP_FINAL); + break; + case PKCS11_CMD_GENERATE_KEY_PAIR: + rc = entry_generate_key_pair(client, ptypes, params); + break; + case PKCS11_CMD_WRAP_KEY: + rc = entry_wrap_key(client, ptypes, params); + break; + case PKCS11_CMD_UNWRAP_KEY: + rc = entry_processing_key(client, ptypes, params, + PKCS11_FUNCTION_UNWRAP); + break; + default: + EMSG("Command %#"PRIx32" is not supported", cmd); + return TEE_ERROR_NOT_SUPPORTED; + } + + DMSG("%s rc %#"PRIx32"/%s", id2str_ta_cmd(cmd), rc, id2str_rc(rc)); + + TEE_MemMove(params[0].memref.buffer, &rc, sizeof(rc)); + params[0].memref.size = sizeof(rc); + + if (rc == PKCS11_CKR_BUFFER_TOO_SMALL) + return TEE_ERROR_SHORT_BUFFER; + else + return TEE_SUCCESS; +} diff --git a/optee_os/ta/pkcs11/src/handle.c b/optee_os/ta/pkcs11/src/handle.c new file mode 100644 index 0000000..d956874 --- /dev/null +++ b/optee_os/ta/pkcs11/src/handle.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014-2020, Linaro Limited + */ + +#include +#include +#include + +#include "handle.h" + +/* + * Define the initial capacity of the database. It should be a low number + * multiple of 2 since some databases a likely to only use a few handles. + * Since the algorithm is to doubles up when growing it shouldn't cause a + * noticeable overhead on large databases. + */ +#define HANDLE_DB_INITIAL_MAX_PTRS 4 + +/* Specific pointer ~0 denotes a still allocated but invalid handle */ +#define INVALID_HANDLE_PTR ((void *)~0) + +void handle_db_init(struct handle_db *db) +{ + TEE_MemFill(db, 0, sizeof(*db)); +} + +void handle_db_destroy(struct handle_db *db) +{ + if (db) { + TEE_Free(db->ptrs); + db->ptrs = NULL; + db->max_ptrs = 0; + } +} + +uint32_t handle_get(struct handle_db *db, void *ptr) +{ + uint32_t n = 0; + void *p = NULL; + uint32_t new_max_ptrs = 0; + + if (!db || !ptr || ptr == INVALID_HANDLE_PTR) + return 0; + + /* Try to find an empty location (index 0 is reserved as invalid) */ + for (n = 1; n < db->max_ptrs; n++) { + if (!db->ptrs[n]) { + db->ptrs[n] = ptr; + return n; + } + } + + /* No location available, grow the ptrs array */ + if (db->max_ptrs) + new_max_ptrs = db->max_ptrs * 2; + else + new_max_ptrs = HANDLE_DB_INITIAL_MAX_PTRS; + + p = TEE_Realloc(db->ptrs, new_max_ptrs * sizeof(void *)); + if (!p) + return 0; + db->ptrs = p; + TEE_MemFill(db->ptrs + db->max_ptrs, 0, + (new_max_ptrs - db->max_ptrs) * sizeof(void *)); + db->max_ptrs = new_max_ptrs; + + /* Since n stopped at db->max_ptrs there is an empty location there */ + db->ptrs[n] = ptr; + return n; +} + +static bool handle_is_valid(struct handle_db *db, uint32_t handle) +{ + return db && handle && handle < db->max_ptrs; +} + +void *handle_put(struct handle_db *db, uint32_t handle) +{ + void *p = NULL; + + if (!handle_is_valid(db, handle)) + return NULL; + + p = db->ptrs[handle]; + db->ptrs[handle] = NULL; + return p; +} + +void *handle_lookup(struct handle_db *db, uint32_t handle) +{ + if (!handle_is_valid(db, handle) || + db->ptrs[handle] == INVALID_HANDLE_PTR) + return NULL; + + return db->ptrs[handle]; +} + +void handle_invalidate(struct handle_db *db, uint32_t handle) +{ + if (handle_is_valid(db, handle)) { + if (!db->ptrs[handle]) + TEE_Panic(TEE_ERROR_GENERIC); + + db->ptrs[handle] = INVALID_HANDLE_PTR; + } +} + +uint32_t handle_lookup_handle(struct handle_db *db, void *ptr) +{ + uint32_t n = 0; + + if (ptr && ptr != INVALID_HANDLE_PTR) { + for (n = 1; n < db->max_ptrs; n++) + if (db->ptrs[n] == ptr) + return n; + } + + return 0; +} diff --git a/optee_os/ta/pkcs11/src/handle.h b/optee_os/ta/pkcs11/src/handle.h new file mode 100644 index 0000000..9ab9019 --- /dev/null +++ b/optee_os/ta/pkcs11/src/handle.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2014-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_HANDLE_H +#define PKCS11_TA_HANDLE_H + +#include + +struct handle_db { + void **ptrs; + uint32_t max_ptrs; +}; + +/* + * Initialize the handle database + */ +void handle_db_init(struct handle_db *db); + +/* + * Free all internal data structures of the database, but does not free + * the db pointer. The database is safe to reuse after it's destroyed, it + * just be empty again. + */ +void handle_db_destroy(struct handle_db *db); + +/* + * Allocate a new handle ID and assigns the supplied pointer to it, + * The function returns > 0 on success and 0 on failure. + */ +uint32_t handle_get(struct handle_db *db, void *ptr); + +/* + * Deallocate a handle. Returns the associated pointer of the handle + * if the handle was valid or NULL if it's invalid. + */ +void *handle_put(struct handle_db *db, uint32_t handle); + +/* + * Return the associated pointer of the handle if the handle is a valid + * handle. + * Returns NULL on failure. + */ +void *handle_lookup(struct handle_db *db, uint32_t handle); + +/* Return the handle associated to a pointer if found, else return 0 */ +uint32_t handle_lookup_handle(struct handle_db *db, void *ptr); + +/* Invalidate the reference referred by an allocated handle */ +void handle_invalidate(struct handle_db *db, uint32_t handle); + +#endif /*PKCS11_TA_HANDLE_H*/ diff --git a/optee_os/ta/pkcs11/src/object.c b/optee_os/ta/pkcs11/src/object.c new file mode 100644 index 0000000..053237b --- /dev/null +++ b/optee_os/ta/pkcs11/src/object.c @@ -0,0 +1,1248 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "handle.h" +#include "object.h" +#include "pkcs11_attributes.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "sanitize_object.h" +#include "serializer.h" + +/* + * Temporary list used to register allocated struct pkcs11_object instances + * so that destroy_object() can unconditionally remove the object from its + * list, being from an object destruction request or because object creation + * failed before being completed. Objects are moved to their target list at + * creation completion. + */ +LIST_HEAD(temp_obj_list, pkcs11_object) temporary_object_list = + LIST_HEAD_INITIALIZER(temp_obj_list); + +static struct ck_token *get_session_token(void *session); + +struct pkcs11_object *pkcs11_handle2object(uint32_t handle, + struct pkcs11_session *session) +{ + struct pkcs11_object *object = NULL; + + object = handle_lookup(get_object_handle_db(session), handle); + if (!object) + return NULL; + + /* + * If object is session only then no extra checks are needed as session + * objects has flat access control space + */ + if (!object->token) + return object; + + /* + * Only allow access to token object if session is associated with + * the token + */ + if (object->token != get_session_token(session)) + return NULL; + + return object; +} + +uint32_t pkcs11_object2handle(struct pkcs11_object *obj, + struct pkcs11_session *session) +{ + return handle_lookup_handle(get_object_handle_db(session), obj); +} + +/* Currently handle pkcs11 sessions and tokens */ + +static struct object_list *get_session_objects(void *session) +{ + /* Currently supporting only pkcs11 session */ + struct pkcs11_session *ck_session = session; + + return pkcs11_get_session_objects(ck_session); +} + +static struct ck_token *get_session_token(void *session) +{ + struct pkcs11_session *ck_session = session; + + return pkcs11_session2token(ck_session); +} + +/* Release resources of a non-persistent object */ +static void cleanup_volatile_obj_ref(struct pkcs11_object *obj) +{ + if (!obj) + return; + + LIST_REMOVE(obj, link); + + if (obj->key_handle != TEE_HANDLE_NULL) + TEE_FreeTransientObject(obj->key_handle); + + if (obj->attribs_hdl != TEE_HANDLE_NULL) + TEE_CloseObject(obj->attribs_hdl); + + TEE_Free(obj->attributes); + TEE_Free(obj->uuid); + TEE_Free(obj); +} + +/* Release resources of a persistent object including volatile resources */ +void cleanup_persistent_object(struct pkcs11_object *obj, + struct ck_token *token) +{ + TEE_Result res = TEE_SUCCESS; + + if (!obj) + return; + + /* Open handle with write properties to destroy the object */ + if (obj->attribs_hdl != TEE_HANDLE_NULL) + TEE_CloseObject(obj->attribs_hdl); + + res = TEE_OpenPersistentObject(TEE_STORAGE_PRIVATE, + obj->uuid, sizeof(TEE_UUID), + TEE_DATA_FLAG_ACCESS_WRITE_META, + &obj->attribs_hdl); + if (!res) + TEE_CloseAndDeletePersistentObject1(obj->attribs_hdl); + + obj->attribs_hdl = TEE_HANDLE_NULL; + destroy_object_uuid(token, obj); + + cleanup_volatile_obj_ref(obj); +} + +/* + * destroy_object - destroy an PKCS11 TA object + * + * @session - session requesting object destruction + * @obj - reference to the PKCS11 TA object + * @session_only - true if only session object shall be destroyed + */ +void destroy_object(struct pkcs11_session *session, struct pkcs11_object *obj, + bool session_only) +{ +#ifdef DEBUG + trace_attributes("[destroy]", obj->attributes); + if (obj->uuid) + MSG_RAW("[destroy] obj uuid %pUl", (void *)obj->uuid); +#endif + + if (session_only) { + /* Destroy object due to session closure */ + handle_put(get_object_handle_db(session), + pkcs11_object2handle(obj, session)); + cleanup_volatile_obj_ref(obj); + + return; + } + + /* Destroy target object (persistent or not) */ + if (get_bool(obj->attributes, PKCS11_CKA_TOKEN)) { + assert(obj->uuid); + /* Try twice otherwise panic! */ + if (unregister_persistent_object(session->token, obj->uuid) && + unregister_persistent_object(session->token, obj->uuid)) + TEE_Panic(0); + + handle_put(get_object_handle_db(session), + pkcs11_object2handle(obj, session)); + cleanup_persistent_object(obj, session->token); + + token_invalidate_object_handles(obj); + } else { + handle_put(get_object_handle_db(session), + pkcs11_object2handle(obj, session)); + cleanup_volatile_obj_ref(obj); + } +} + +static struct pkcs11_object *create_obj_instance(struct obj_attrs *head, + struct ck_token *token) +{ + struct pkcs11_object *obj = NULL; + + obj = TEE_Malloc(sizeof(struct pkcs11_object), TEE_MALLOC_FILL_ZERO); + if (!obj) + return NULL; + + obj->key_handle = TEE_HANDLE_NULL; + obj->attribs_hdl = TEE_HANDLE_NULL; + obj->attributes = head; + obj->token = token; + + return obj; +} + +struct pkcs11_object *create_token_object(struct obj_attrs *head, + TEE_UUID *uuid, + struct ck_token *token) +{ + struct pkcs11_object *obj = create_obj_instance(head, token); + + if (obj) + obj->uuid = uuid; + + return obj; +} + +/* + * create_object - create an PKCS11 TA object from its attributes and value + * + * @sess - session requesting object creation + * @head - reference to serialized attributes + * @out_handle - generated handle for the created object + */ +enum pkcs11_rc create_object(void *sess, struct obj_attrs *head, + uint32_t *out_handle) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct pkcs11_object *obj = NULL; + struct pkcs11_session *session = (struct pkcs11_session *)sess; + uint32_t obj_handle = 0; + +#ifdef DEBUG + trace_attributes("[create]", head); +#endif + + /* + * We do not check the key attributes. At this point, key attributes + * are expected consistent and reliable. + */ + + obj = create_obj_instance(head, NULL); + if (!obj) + return PKCS11_CKR_DEVICE_MEMORY; + + LIST_INSERT_HEAD(&temporary_object_list, obj, link); + + /* Create a handle for the object in the session database */ + obj_handle = handle_get(get_object_handle_db(session), obj); + if (!obj_handle) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto err; + } + + if (get_bool(obj->attributes, PKCS11_CKA_TOKEN)) { + TEE_Result res = TEE_SUCCESS; + + /* + * Get an ID for the persistent object + * Create the file + * Register the object in the persistent database + * (move the full sequence to persisent_db.c?) + */ + size_t size = sizeof(struct obj_attrs) + + obj->attributes->attrs_size; + uint32_t tee_obj_flags = TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE | + TEE_DATA_FLAG_ACCESS_WRITE_META; + + rc = create_object_uuid(get_session_token(session), obj); + if (rc) + goto err; + + res = TEE_CreatePersistentObject(TEE_STORAGE_PRIVATE, + obj->uuid, sizeof(TEE_UUID), + tee_obj_flags, + TEE_HANDLE_NULL, + obj->attributes, size, + &obj->attribs_hdl); + if (res) { + rc = tee2pkcs_error(res); + goto err; + } + + rc = register_persistent_object(get_session_token(session), + obj->uuid); + if (rc) + goto err; + + TEE_CloseObject(obj->attribs_hdl); + obj->attribs_hdl = TEE_HANDLE_NULL; + + /* Move object from temporary list to target token list */ + LIST_REMOVE(obj, link); + LIST_INSERT_HEAD(&session->token->object_list, obj, link); + } else { + /* Move object from temporary list to target session list */ + LIST_REMOVE(obj, link); + LIST_INSERT_HEAD(get_session_objects(session), obj, link); + } + + *out_handle = obj_handle; + + return PKCS11_CKR_OK; +err: + /* make sure that supplied "head" isn't freed */ + obj->attributes = NULL; + handle_put(get_object_handle_db(session), obj_handle); + if (get_bool(head, PKCS11_CKA_TOKEN)) + cleanup_persistent_object(obj, session->token); + else + cleanup_volatile_obj_ref(obj); + + return rc; +} + +enum pkcs11_rc entry_create_object(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + enum pkcs11_rc rc = PKCS11_CKR_OK; + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct obj_attrs *head = NULL; + struct pkcs11_object_head *template = NULL; + size_t template_size = 0; + uint32_t obj_handle = 0; + + /* + * Collect the arguments of the request + */ + + if (!client || ptypes != exp_pt || + out->memref.size != sizeof(obj_handle)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &template); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + template_size = sizeof(*template) + template->attrs_size; + + /* + * Prepare a clean initial state for the requested object attributes. + * Free temporary template once done. + */ + rc = create_attributes_from_template(&head, template, template_size, + NULL, PKCS11_FUNCTION_IMPORT, + PKCS11_PROCESSING_IMPORT, + PKCS11_CKO_UNDEFINED_ID); + TEE_Free(template); + template = NULL; + if (rc) + goto out; + + /* + * Check target object attributes match target processing + * Check target object attributes match token state + */ + rc = check_created_attrs_against_processing(PKCS11_PROCESSING_IMPORT, + head); + if (rc) + goto out; + + rc = check_created_attrs_against_token(session, head); + if (rc) + goto out; + + rc = check_access_attrs_against_token(session, head); + if (rc) + goto out; + + /* + * At this stage the object is almost created: all its attributes are + * referenced in @head, including the key value and are assumed + * reliable. Now need to register it and get a handle for it. + */ + rc = create_object(session, head, &obj_handle); + if (rc) + goto out; + + /* + * Now obj_handle (through the related struct pkcs11_object + * instance) owns the serialized buffer that holds the object + * attributes. We clear reference in head to NULL as the serializer + * object is now referred from obj_handle. This allows smooth pass + * through free at function exit. + */ + head = NULL; + + TEE_MemMove(out->memref.buffer, &obj_handle, sizeof(obj_handle)); + out->memref.size = sizeof(obj_handle); + + DMSG("PKCS11 session %"PRIu32": import object %#"PRIx32, + session->handle, obj_handle); + +out: + TEE_Free(template); + TEE_Free(head); + + return rc; +} + +enum pkcs11_rc entry_destroy_object(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + enum pkcs11_rc rc = PKCS11_CKR_OK; + TEE_Param *ctrl = params; + struct serialargs ctrlargs = { }; + uint32_t object_handle = 0; + struct pkcs11_session *session = NULL; + struct pkcs11_object *object = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get_u32(&ctrlargs, &object_handle); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + object = pkcs11_handle2object(object_handle, session); + if (!object) + return PKCS11_CKR_OBJECT_HANDLE_INVALID; + + /* Only session objects can be destroyed during a read-only session */ + if (get_bool(object->attributes, PKCS11_CKA_TOKEN) && + !pkcs11_session_is_read_write(session)) { + DMSG("Can't destroy persistent object"); + return PKCS11_CKR_SESSION_READ_ONLY; + } + + /* + * Only public objects can be destroyed unless normal user is logged in + */ + rc = check_access_attrs_against_token(session, object->attributes); + if (rc) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + + /* Objects with PKCS11_CKA_DESTROYABLE as false aren't destroyable */ + if (!get_bool(object->attributes, PKCS11_CKA_DESTROYABLE)) + return PKCS11_CKR_ACTION_PROHIBITED; + + destroy_object(session, object, false); + + DMSG("PKCS11 session %"PRIu32": destroy object %#"PRIx32, + session->handle, object_handle); + + return rc; +} + +static void release_find_obj_context(struct pkcs11_find_objects *find_ctx) +{ + if (!find_ctx) + return; + + TEE_Free(find_ctx->attributes); + TEE_Free(find_ctx->handles); + TEE_Free(find_ctx); +} + +static enum pkcs11_rc find_ctx_add(struct pkcs11_find_objects *find_ctx, + uint32_t handle) +{ + uint32_t *hdls = TEE_Realloc(find_ctx->handles, + (find_ctx->count + 1) * sizeof(*hdls)); + + if (!hdls) + return PKCS11_CKR_DEVICE_MEMORY; + + find_ctx->handles = hdls; + + *(find_ctx->handles + find_ctx->count) = handle; + find_ctx->count++; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_find_objects_init(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_session *sess = NULL; + struct pkcs11_object_head *template = NULL; + struct obj_attrs *req_attrs = NULL; + struct pkcs11_object *obj = NULL; + struct pkcs11_find_objects *find_ctx = NULL; + struct handle_db *object_db = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &template); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + /* Search objects only if no operation is on-going */ + if (session_is_active(session)) { + rc = PKCS11_CKR_OPERATION_ACTIVE; + goto out; + } + + if (session->find_ctx) { + EMSG("Active object search already in progress"); + rc = PKCS11_CKR_FUNCTION_FAILED; + goto out; + } + + rc = sanitize_client_object(&req_attrs, template, + sizeof(*template) + template->attrs_size, + PKCS11_UNDEFINED_ID, PKCS11_UNDEFINED_ID); + if (rc) + goto out; + + /* Must zero init the structure */ + find_ctx = TEE_Malloc(sizeof(*find_ctx), TEE_MALLOC_FILL_ZERO); + if (!find_ctx) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + TEE_Free(template); + template = NULL; + + switch (get_class(req_attrs)) { + case PKCS11_CKO_UNDEFINED_ID: + /* Unspecified class searches among data objects */ + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_PRIVATE_KEY: + case PKCS11_CKO_DATA: + case PKCS11_CKO_CERTIFICATE: + break; + default: + EMSG("Find object of class %s (%"PRIu32") is not supported", + id2str_class(get_class(req_attrs)), + get_class(req_attrs)); + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + /* + * Scan all objects (sessions and persistent ones) and set a list of + * candidates that match caller attributes. + */ + + /* Scan all session objects first */ + TAILQ_FOREACH(sess, get_session_list(session), link) { + LIST_FOREACH(obj, &sess->object_list, link) { + /* + * Skip all token objects as they could be from + * different token which the session does not have + * access + */ + if (obj->token) + continue; + + if (!attributes_match_reference(obj->attributes, + req_attrs)) + continue; + + rc = find_ctx_add(find_ctx, + pkcs11_object2handle(obj, session)); + if (rc) + goto out; + } + } + + object_db = get_object_handle_db(session); + + /* Scan token objects */ + LIST_FOREACH(obj, &session->token->object_list, link) { + uint32_t handle = 0; + bool new_load = false; + + if (!obj->attributes) { + rc = load_persistent_object_attributes(obj); + if (rc) { + rc = PKCS11_CKR_GENERAL_ERROR; + goto out; + } + + new_load = true; + } + + if (!obj->attributes || + check_access_attrs_against_token(session, + obj->attributes) || + !attributes_match_reference(obj->attributes, req_attrs)) { + if (new_load) + release_persistent_object_attributes(obj); + + continue; + } + + /* Resolve object handle for object */ + handle = pkcs11_object2handle(obj, session); + if (!handle) { + handle = handle_get(object_db, obj); + if (!handle) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + } + + rc = find_ctx_add(find_ctx, handle); + if (rc) + goto out; + } + + find_ctx->attributes = req_attrs; + req_attrs = NULL; + session->find_ctx = find_ctx; + find_ctx = NULL; + rc = PKCS11_CKR_OK; + +out: + TEE_Free(req_attrs); + TEE_Free(template); + release_find_obj_context(find_ctx); + + return rc; +} + +enum pkcs11_rc entry_find_objects(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_find_objects *ctx = NULL; + uint8_t *out_handles = NULL; + size_t out_count = 0; + size_t count = 0; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + out_count = out->memref.size / sizeof(uint32_t); + out_handles = out->memref.buffer; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + ctx = session->find_ctx; + + if (!ctx) + return PKCS11_CKR_OPERATION_NOT_INITIALIZED; + + for (count = 0; ctx->next < ctx->count && count < out_count; + ctx->next++, count++) + TEE_MemMove(out_handles + count * sizeof(uint32_t), + ctx->handles + ctx->next, sizeof(uint32_t)); + + /* Update output buffer according the number of handles provided */ + out->memref.size = count * sizeof(uint32_t); + + DMSG("PKCS11 session %"PRIu32": finding objects", session->handle); + + return PKCS11_CKR_OK; +} + +void release_session_find_obj_context(struct pkcs11_session *session) +{ + release_find_obj_context(session->find_ctx); + session->find_ctx = NULL; +} + +enum pkcs11_rc entry_find_objects_final(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!session->find_ctx) + return PKCS11_CKR_OPERATION_NOT_INITIALIZED; + + release_session_find_obj_context(session); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_get_attribute_value(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_object_head *template = NULL; + struct pkcs11_object *obj = NULL; + uint32_t object_handle = 0; + char *cur = NULL; + size_t len = 0; + char *end = NULL; + bool attr_sensitive = 0; + bool attr_type_invalid = 0; + bool buffer_too_small = 0; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &object_handle, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &template); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + obj = pkcs11_handle2object(object_handle, session); + if (!obj) { + rc = PKCS11_CKR_OBJECT_HANDLE_INVALID; + goto out; + } + + rc = check_access_attrs_against_token(session, obj->attributes); + if (rc) { + rc = PKCS11_CKR_OBJECT_HANDLE_INVALID; + goto out; + } + + /* Iterate over attributes and set their values */ + /* + * 1. If the specified attribute (i.e., the attribute specified by the + * type field) for the object cannot be revealed because the object is + * sensitive or unextractable, then the ulValueLen field in that triple + * is modified to hold the value PKCS11_CK_UNAVAILABLE_INFORMATION. + * + * 2. Otherwise, if the specified value for the object is invalid (the + * object does not possess such an attribute), then the ulValueLen field + * in that triple is modified to hold the value + * PKCS11_CK_UNAVAILABLE_INFORMATION. + * + * 3. Otherwise, if the pValue field has the value NULL_PTR, then the + * ulValueLen field is modified to hold the exact length of the + * specified attribute for the object. + * + * 4. Otherwise, if the length specified in ulValueLen is large enough + * to hold the value of the specified attribute for the object, then + * that attribute is copied into the buffer located at pValue, and the + * ulValueLen field is modified to hold the exact length of the + * attribute. + * + * 5. Otherwise, the ulValueLen field is modified to hold the value + * PKCS11_CK_UNAVAILABLE_INFORMATION. + */ + cur = (char *)template + sizeof(struct pkcs11_object_head); + end = cur + template->attrs_size; + + for (; cur < end; cur += len) { + struct pkcs11_attribute_head *cli_ref = (void *)cur; + struct pkcs11_attribute_head cli_head = { }; + void *data_ptr = NULL; + + /* Make copy of header so that is aligned properly. */ + TEE_MemMove(&cli_head, cli_ref, sizeof(cli_head)); + + len = sizeof(*cli_ref) + cli_head.size; + + /* Treat hidden attributes as missing attributes */ + if (attribute_is_hidden(&cli_head)) { + cli_head.size = PKCS11_CK_UNAVAILABLE_INFORMATION; + TEE_MemMove(&cli_ref->size, &cli_head.size, + sizeof(cli_head.size)); + attr_type_invalid = 1; + continue; + } + + /* We don't support getting value of indirect templates */ + if (pkcs11_attr_has_indirect_attributes(cli_head.id)) { + attr_type_invalid = 1; + continue; + } + + /* Check 1. */ + if (!attribute_is_exportable(&cli_head, obj)) { + cli_head.size = PKCS11_CK_UNAVAILABLE_INFORMATION; + TEE_MemMove(&cli_ref->size, &cli_head.size, + sizeof(cli_head.size)); + attr_sensitive = 1; + continue; + } + + /* Get real data pointer from template data */ + data_ptr = cli_head.size ? cli_ref->data : NULL; + + /* + * We assume that if size is 0, pValue was NULL, so we return + * the size of the required buffer for it (3., 4.) + */ + rc = get_attribute(obj->attributes, cli_head.id, data_ptr, + &cli_head.size); + /* Check 2. */ + switch (rc) { + case PKCS11_CKR_OK: + break; + case PKCS11_RV_NOT_FOUND: + cli_head.size = PKCS11_CK_UNAVAILABLE_INFORMATION; + attr_type_invalid = 1; + break; + case PKCS11_CKR_BUFFER_TOO_SMALL: + if (data_ptr) + buffer_too_small = 1; + break; + default: + rc = PKCS11_CKR_GENERAL_ERROR; + goto out; + } + + TEE_MemMove(&cli_ref->size, &cli_head.size, + sizeof(cli_head.size)); + } + + /* + * If case 1 applies to any of the requested attributes, then the call + * should return the value CKR_ATTRIBUTE_SENSITIVE. If case 2 applies to + * any of the requested attributes, then the call should return the + * value CKR_ATTRIBUTE_TYPE_INVALID. If case 5 applies to any of the + * requested attributes, then the call should return the value + * CKR_BUFFER_TOO_SMALL. As usual, if more than one of these error codes + * is applicable, Cryptoki may return any of them. Only if none of them + * applies to any of the requested attributes will CKR_OK be returned. + */ + + rc = PKCS11_CKR_OK; + if (attr_sensitive) + rc = PKCS11_CKR_ATTRIBUTE_SENSITIVE; + if (attr_type_invalid) + rc = PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + if (buffer_too_small) + rc = PKCS11_CKR_BUFFER_TOO_SMALL; + + /* Move updated template to out buffer */ + TEE_MemMove(out->memref.buffer, template, out->memref.size); + + DMSG("PKCS11 session %"PRIu32": get attributes %#"PRIx32, + session->handle, object_handle); + +out: + TEE_Free(template); + + return rc; +} + +enum pkcs11_rc entry_get_object_size(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + uint32_t object_handle = 0; + struct pkcs11_object *obj = NULL; + uint32_t obj_size = 0; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &object_handle, sizeof(uint32_t)); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + obj = pkcs11_handle2object(object_handle, session); + if (!obj) + return PKCS11_CKR_OBJECT_HANDLE_INVALID; + + rc = check_access_attrs_against_token(session, obj->attributes); + if (rc) + return PKCS11_CKR_OBJECT_HANDLE_INVALID; + + if (out->memref.size != sizeof(uint32_t)) + return PKCS11_CKR_ARGUMENTS_BAD; + + obj_size = ((struct obj_attrs *)obj->attributes)->attrs_size + + sizeof(struct obj_attrs); + TEE_MemMove(out->memref.buffer, &obj_size, sizeof(obj_size)); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_set_attribute_value(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_object_head *template = NULL; + size_t template_size = 0; + struct pkcs11_object *obj = NULL; + struct obj_attrs *head = NULL; + uint32_t object_handle = 0; + enum processing_func function = PKCS11_FUNCTION_MODIFY; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &object_handle, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &template); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + obj = pkcs11_handle2object(object_handle, session); + if (!obj) { + rc = PKCS11_CKR_OBJECT_HANDLE_INVALID; + goto out; + } + + /* Only session objects can be modified during a read-only session */ + if (object_is_token(obj->attributes) && + !pkcs11_session_is_read_write(session)) { + DMSG("Can't modify persistent object in a RO session"); + rc = PKCS11_CKR_SESSION_READ_ONLY; + goto out; + } + + /* + * Only public objects can be modified unless normal user is logged in + */ + rc = check_access_attrs_against_token(session, obj->attributes); + if (rc) { + rc = PKCS11_CKR_USER_NOT_LOGGED_IN; + goto out; + } + + /* Objects with PKCS11_CKA_MODIFIABLE as false aren't modifiable */ + if (!object_is_modifiable(obj->attributes)) { + rc = PKCS11_CKR_ACTION_PROHIBITED; + goto out; + } + + template_size = sizeof(*template) + template->attrs_size; + + /* + * Prepare a clean initial state (@head) for the template. Helps in + * removing any duplicates or inconsistent values from the + * template. + */ + rc = create_attributes_from_template(&head, template, template_size, + NULL, function, + PKCS11_CKM_UNDEFINED_ID, + PKCS11_CKO_UNDEFINED_ID); + if (rc) + goto out; + + /* Check the attributes in @head to see if they are modifiable */ + rc = check_attrs_against_modification(session, head, obj, function); + if (rc) + goto out; + + /* + * All checks complete. The attributes in @head have been checked and + * can now be used to set/modify the object attributes. + */ + rc = modify_attributes_list(&obj->attributes, head); + if (rc) + goto out; + + if (get_bool(obj->attributes, PKCS11_CKA_TOKEN)) { + rc = update_persistent_object_attributes(obj); + if (rc) + goto out; + } + + DMSG("PKCS11 session %"PRIu32": set attributes %#"PRIx32, + session->handle, object_handle); + +out: + TEE_Free(head); + TEE_Free(template); + return rc; +} + +enum pkcs11_rc entry_copy_object(struct pkcs11_client *client, uint32_t ptypes, + TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_object_head *template = NULL; + struct obj_attrs *head = NULL; + struct obj_attrs *head_new = NULL; + size_t template_size = 0; + struct pkcs11_object *obj = NULL; + uint32_t object_handle = 0; + uint32_t obj_handle = 0; + enum processing_func function = PKCS11_FUNCTION_COPY; + enum pkcs11_class_id class = PKCS11_CKO_UNDEFINED_ID; + + if (!client || ptypes != exp_pt || + out->memref.size != sizeof(obj_handle)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &object_handle, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &template); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + obj = pkcs11_handle2object(object_handle, session); + if (!obj) { + rc = PKCS11_CKR_OBJECT_HANDLE_INVALID; + goto out; + } + + /* Only session objects can be modified during a read-only session */ + if (object_is_token(obj->attributes) && + !pkcs11_session_is_read_write(session)) { + DMSG("Can't modify persistent object in a RO session"); + rc = PKCS11_CKR_SESSION_READ_ONLY; + goto out; + } + + /* + * Only public objects can be modified unless normal user is logged in + */ + rc = check_access_attrs_against_token(session, obj->attributes); + if (rc) { + rc = PKCS11_CKR_USER_NOT_LOGGED_IN; + goto out; + } + + /* Objects with PKCS11_CKA_COPYABLE as false can't be copied */ + if (!object_is_copyable(obj->attributes)) { + rc = PKCS11_CKR_ACTION_PROHIBITED; + goto out; + } + + template_size = sizeof(*template) + template->attrs_size; + + /* + * Prepare a clean initial state (@head) for the template. Helps in + * removing any duplicates or inconsistent values from the + * template. + */ + rc = create_attributes_from_template(&head, template, template_size, + NULL, function, + PKCS11_CKM_UNDEFINED_ID, + PKCS11_CKO_UNDEFINED_ID); + if (rc) + goto out; + + /* Check the attributes in @head to see if they are modifiable */ + rc = check_attrs_against_modification(session, head, obj, function); + if (rc) + goto out; + + class = get_class(obj->attributes); + + if (class == PKCS11_CKO_SECRET_KEY || + class == PKCS11_CKO_PRIVATE_KEY) { + /* + * If CKA_EXTRACTABLE attribute in passed template (@head) is + * modified to CKA_FALSE, CKA_NEVER_EXTRACTABLE should also + * change to CKA_FALSE in copied obj. So, add it to the + * passed template. + */ + uint8_t bbool = 0; + uint32_t size = sizeof(bbool); + + rc = get_attribute(head, PKCS11_CKA_EXTRACTABLE, &bbool, &size); + if (!rc && !bbool) { + rc = add_attribute(&head, PKCS11_CKA_NEVER_EXTRACTABLE, + &bbool, sizeof(uint8_t)); + if (rc) + goto out; + } + rc = PKCS11_CKR_OK; + } + + /* + * All checks have passed. Create a copy of the serialized buffer which + * holds the object attributes in @head_new for the new object + */ + template_size = sizeof(*obj->attributes) + obj->attributes->attrs_size; + head_new = TEE_Malloc(template_size, TEE_MALLOC_FILL_ZERO); + if (!head_new) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + TEE_MemMove(head_new, obj->attributes, template_size); + + /* + * Modify the copied attribute @head_new based on the template @head + * given by the callee + */ + rc = modify_attributes_list(&head_new, head); + if (rc) + goto out; + + /* + * At this stage the object is almost created: all its attributes are + * referenced in @head_new, including the key value and are assumed + * reliable. Now need to register it and get a handle for it. + */ + rc = create_object(session, head_new, &obj_handle); + if (rc) + goto out; + + /* + * Now obj_handle (through the related struct pkcs11_object + * instance) owns the serialized buffer that holds the object + * attributes. We clear reference in head to NULL as the serializer + * object is now referred from obj_handle. This allows smooth pass + * through free at function exit. + */ + head_new = NULL; + + TEE_MemMove(out->memref.buffer, &obj_handle, sizeof(obj_handle)); + out->memref.size = sizeof(obj_handle); + + DMSG("PKCS11 session %"PRIu32": copy object %#"PRIx32, + session->handle, obj_handle); + +out: + TEE_Free(head_new); + TEE_Free(head); + TEE_Free(template); + return rc; +} diff --git a/optee_os/ta/pkcs11/src/object.h b/optee_os/ta/pkcs11/src/object.h new file mode 100644 index 0000000..d1b791e --- /dev/null +++ b/optee_os/ta/pkcs11/src/object.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_OBJECT_H +#define PKCS11_TA_OBJECT_H + +#include +#include +#include + +struct ck_token; +struct obj_attrs; +struct pkcs11_client; +struct pkcs11_session; + +/* + * link: objects are referenced in a double-linked list + * attributes: pointer to the serialized object attributes + * key_handle: GPD TEE object handle if used in an operation + * key_type: GPD TEE key type (shortcut used for processing) + * token: associated token for the object + * uuid: object UUID in the persistent database if a persistent object, or NULL + * attribs_hdl: GPD TEE attributes handles if persistent object + */ +struct pkcs11_object { + LIST_ENTRY(pkcs11_object) link; + struct obj_attrs *attributes; + TEE_ObjectHandle key_handle; + uint32_t key_type; + struct ck_token *token; + TEE_UUID *uuid; + TEE_ObjectHandle attribs_hdl; +}; + +LIST_HEAD(object_list, pkcs11_object); + +struct pkcs11_object *pkcs11_handle2object(uint32_t client_handle, + struct pkcs11_session *session); + +uint32_t pkcs11_object2handle(struct pkcs11_object *obj, + struct pkcs11_session *session); + +struct pkcs11_object *create_token_object(struct obj_attrs *head, + TEE_UUID *uuid, + struct ck_token *token); + +enum pkcs11_rc create_object(void *session, struct obj_attrs *attributes, + uint32_t *handle); + +void cleanup_persistent_object(struct pkcs11_object *obj, + struct ck_token *token); + +void destroy_object(struct pkcs11_session *session, + struct pkcs11_object *object, bool session_object_only); + +/* + * Entry function called from the PKCS11 command parser + */ +enum pkcs11_rc entry_create_object(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_destroy_object(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_find_objects_init(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_find_objects(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_find_objects_final(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_get_attribute_value(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_get_object_size(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_set_attribute_value(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_copy_object(struct pkcs11_client *client, uint32_t ptypes, + TEE_Param *params); + +void release_session_find_obj_context(struct pkcs11_session *session); + +#endif /*PKCS11_TA_OBJECT_H*/ diff --git a/optee_os/ta/pkcs11/src/persistent_token.c b/optee_os/ta/pkcs11/src/persistent_token.c new file mode 100644 index 0000000..4efc742 --- /dev/null +++ b/optee_os/ta/pkcs11/src/persistent_token.c @@ -0,0 +1,714 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "pkcs11_token.h" +#include "pkcs11_helpers.h" + +#define PERSISTENT_OBJECT_ID_LEN 32 + +/* + * Token persistent objects + * + * The persistent objects are each identified by a UUID. + * The persistent object database stores the list of the UUIDs registered. For + * each it is expected that a file of ID "UUID" is stored in the TA secure + * storage. + */ +static TEE_Result get_db_file_name(struct ck_token *token, + char *name, size_t size) +{ + int n = snprintf(name, size, "token.db.%u", get_token_id(token)); + + if (n < 0 || (size_t)n >= size) + return TEE_ERROR_SECURITY; + else + return TEE_SUCCESS; +} + +static TEE_Result open_db_file(struct ck_token *token, + TEE_ObjectHandle *out_hdl) +{ + char file[PERSISTENT_OBJECT_ID_LEN] = { }; + TEE_Result res = TEE_ERROR_GENERIC; + + res = get_db_file_name(token, file, sizeof(file)); + if (res) + return res; + + return TEE_OpenPersistentObject(TEE_STORAGE_PRIVATE, file, sizeof(file), + TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE, + out_hdl); +} + +void update_persistent_db(struct ck_token *token) +{ + TEE_Result res = TEE_ERROR_GENERIC; + TEE_ObjectHandle db_hdl = TEE_HANDLE_NULL; + + res = open_db_file(token, &db_hdl); + if (res) { + EMSG("Failed to open token persistent db: %#"PRIx32, res); + TEE_Panic(0); + } + res = TEE_WriteObjectData(db_hdl, token->db_main, + sizeof(*token->db_main)); + if (res) { + EMSG("Failed to write to token persistent db: %#"PRIx32, res); + TEE_Panic(0); + } + + TEE_CloseObject(db_hdl); +} + +static enum pkcs11_rc do_hash(uint32_t user, const uint8_t *pin, + size_t pin_size, uint32_t salt, + uint8_t hash[TEE_MAX_HASH_SIZE]) +{ + TEE_Result res = TEE_SUCCESS; + TEE_OperationHandle oh = TEE_HANDLE_NULL; + size_t sz = TEE_MAX_HASH_SIZE; + + res = TEE_AllocateOperation(&oh, TEE_ALG_SHA256, TEE_MODE_DIGEST, 0); + if (res) + return tee2pkcs_error(res); + + TEE_DigestUpdate(oh, &user, sizeof(user)); + TEE_DigestUpdate(oh, &salt, sizeof(salt)); + res = TEE_DigestDoFinal(oh, pin, pin_size, hash, &sz); + TEE_FreeOperation(oh); + + if (res) + return PKCS11_CKR_GENERAL_ERROR; + + memset(hash + sz, 0, TEE_MAX_HASH_SIZE - sz); + return PKCS11_CKR_OK; +} + +enum pkcs11_rc hash_pin(enum pkcs11_user_type user, const uint8_t *pin, + size_t pin_size, uint32_t *salt, + uint8_t hash[TEE_MAX_HASH_SIZE]) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint32_t s = 0; + + TEE_GenerateRandom(&s, sizeof(s)); + if (!s) + s++; + + rc = do_hash(user, pin, pin_size, s, hash); + if (!rc) + *salt = s; + return rc; +} + +enum pkcs11_rc verify_pin(enum pkcs11_user_type user, const uint8_t *pin, + size_t pin_size, uint32_t salt, + const uint8_t hash[TEE_MAX_HASH_SIZE]) +{ + uint8_t tmp_hash[TEE_MAX_HASH_SIZE] = { 0 }; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + rc = do_hash(user, pin, pin_size, salt, tmp_hash); + if (rc) + return rc; + + if (buf_compare_ct(tmp_hash, hash, TEE_MAX_HASH_SIZE)) + rc = PKCS11_CKR_PIN_INCORRECT; + + return rc; +} + +#if defined(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) +enum pkcs11_rc setup_so_identity_auth_from_client(struct ck_token *token) +{ + TEE_Identity identity = { }; + TEE_Result res = TEE_SUCCESS; + + res = TEE_GetPropertyAsIdentity(TEE_PROPSET_CURRENT_CLIENT, + "gpd.client.identity", &identity); + if (res != TEE_SUCCESS) { + EMSG("TEE_GetPropertyAsIdentity: returned %#"PRIx32, res); + return PKCS11_CKR_PIN_INVALID; + } + + TEE_MemMove(&token->db_main->so_identity, &identity, sizeof(identity)); + token->db_main->flags |= PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH; + + token->db_main->so_pin_salt = 0; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc setup_identity_auth_from_pin(struct ck_token *token, + enum pkcs11_user_type user_type, + const uint8_t *pin, + size_t pin_size) +{ + TEE_Identity identity = { }; + TEE_Result res = TEE_SUCCESS; + uint32_t flags_clear = 0; + uint32_t flags_set = 0; + char *acl_string = NULL; + char *uuid_str = NULL; + + assert(token->db_main->flags & + PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH); + + if (!pin) { + /* Use client identity */ + res = TEE_GetPropertyAsIdentity(TEE_PROPSET_CURRENT_CLIENT, + "gpd.client.identity", + &identity); + if (res != TEE_SUCCESS) { + EMSG("TEE_GetPropertyAsIdentity: returned %#"PRIx32, + res); + return PKCS11_CKR_PIN_INVALID; + } + } else { + /* Parse PIN ACL string: : */ + acl_string = TEE_Malloc(pin_size + 1, TEE_MALLOC_FILL_ZERO); + if (!acl_string) + return PKCS11_CKR_DEVICE_MEMORY; + TEE_MemMove(acl_string, pin, pin_size); + + uuid_str = strstr(acl_string, ":"); + if (uuid_str) + uuid_str++; + if (strcmp(PKCS11_AUTH_TEE_IDENTITY_PUBLIC, acl_string) == 0) { + identity.login = TEE_LOGIN_PUBLIC; + } else if (strstr(acl_string, PKCS11_AUTH_TEE_IDENTITY_USER) == + acl_string) { + identity.login = TEE_LOGIN_USER; + } else if (strstr(acl_string, PKCS11_AUTH_TEE_IDENTITY_GROUP) == + acl_string) { + identity.login = TEE_LOGIN_GROUP; + } else { + EMSG("Invalid PIN ACL string - login"); + TEE_Free(acl_string); + return PKCS11_CKR_PIN_INVALID; + } + + if (identity.login != TEE_LOGIN_PUBLIC) { + if (!uuid_str) { + EMSG("Invalid PIN ACL string - colon"); + TEE_Free(acl_string); + return PKCS11_CKR_PIN_INVALID; + } + + res = tee_uuid_from_str(&identity.uuid, uuid_str); + if (res) { + EMSG("Invalid PIN ACL string - client id"); + TEE_Free(acl_string); + return PKCS11_CKR_PIN_INVALID; + } + } + + TEE_Free(acl_string); + } + + switch (user_type) { + case PKCS11_CKU_SO: + token->db_main->so_pin_count = 0; + token->db_main->so_pin_salt = 0; + flags_clear = PKCS11_CKFT_SO_PIN_COUNT_LOW | + PKCS11_CKFT_SO_PIN_FINAL_TRY | + PKCS11_CKFT_SO_PIN_LOCKED | + PKCS11_CKFT_SO_PIN_TO_BE_CHANGED; + + TEE_MemMove(&token->db_main->so_identity, &identity, + sizeof(identity)); + break; + case PKCS11_CKU_USER: + token->db_main->user_pin_count = 0; + token->db_main->user_pin_salt = 0; + flags_clear = PKCS11_CKFT_USER_PIN_COUNT_LOW | + PKCS11_CKFT_USER_PIN_FINAL_TRY | + PKCS11_CKFT_USER_PIN_LOCKED | + PKCS11_CKFT_USER_PIN_TO_BE_CHANGED; + flags_set = PKCS11_CKFT_USER_PIN_INITIALIZED; + + TEE_MemMove(&token->db_main->user_identity, &identity, + sizeof(identity)); + break; + default: + return PKCS11_CKR_FUNCTION_FAILED; + } + + token->db_main->flags &= ~flags_clear; + token->db_main->flags |= flags_set; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc verify_identity_auth(struct ck_token *token, + enum pkcs11_user_type user_type) +{ + TEE_Identity identity = { }; + TEE_Result res = TEE_SUCCESS; + + assert(token->db_main->flags & + PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH); + + res = TEE_GetPropertyAsIdentity(TEE_PROPSET_CURRENT_CLIENT, + "gpd.client.identity", &identity); + if (res != TEE_SUCCESS) { + EMSG("TEE_GetPropertyAsIdentity: returned %#"PRIx32, res); + return PKCS11_CKR_PIN_INVALID; + } + + if (user_type == PKCS11_CKU_SO) { + if (TEE_MemCompare(&token->db_main->so_identity, &identity, + sizeof(identity))) + return PKCS11_CKR_PIN_INCORRECT; + } else if (user_type == PKCS11_CKU_USER) { + if (TEE_MemCompare(&token->db_main->user_identity, &identity, + sizeof(identity))) + return PKCS11_CKR_PIN_INCORRECT; + } else { + return PKCS11_CKR_PIN_INCORRECT; + } + + return PKCS11_CKR_OK; +} +#endif /* CFG_PKCS11_TA_AUTH_TEE_IDENTITY */ + +/* + * Release resources relate to persistent database + */ +void close_persistent_db(struct ck_token *token __unused) +{ +} + +static int get_persistent_obj_idx(struct ck_token *token, TEE_UUID *uuid) +{ + size_t i = 0; + + if (!uuid) + return -1; + + for (i = 0; i < token->db_objs->count; i++) + if (!TEE_MemCompare(token->db_objs->uuids + i, + uuid, sizeof(TEE_UUID))) + return i; + + return -1; +} + +/* UUID for persistent object */ +enum pkcs11_rc create_object_uuid(struct ck_token *token, + struct pkcs11_object *obj) +{ + assert(!obj->uuid); + + obj->uuid = TEE_Malloc(sizeof(TEE_UUID), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!obj->uuid) + return PKCS11_CKR_DEVICE_MEMORY; + + obj->token = token; + + do { + TEE_GenerateRandom(obj->uuid, sizeof(TEE_UUID)); + } while (get_persistent_obj_idx(token, obj->uuid) >= 0); + + return PKCS11_CKR_OK; +} + +void destroy_object_uuid(struct ck_token *token __maybe_unused, + struct pkcs11_object *obj) +{ + assert(get_persistent_obj_idx(token, obj->uuid) < 0); + + TEE_Free(obj->uuid); + obj->uuid = NULL; +} + +enum pkcs11_rc get_persistent_objects_list(struct ck_token *token, + TEE_UUID *array, size_t *size) +{ + size_t out_size = *size; + + *size = token->db_objs->count * sizeof(TEE_UUID); + + if (out_size < *size) + return PKCS11_CKR_BUFFER_TOO_SMALL; + + if (array) + TEE_MemMove(array, token->db_objs->uuids, *size); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc unregister_persistent_object(struct ck_token *token, + TEE_UUID *uuid) +{ + TEE_ObjectHandle db_hdl = TEE_HANDLE_NULL; + struct token_persistent_objs *ptr = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + int count = 0; + int idx = 0; + + if (!uuid) + return PKCS11_CKR_OK; + + idx = get_persistent_obj_idx(token, uuid); + if (idx < 0) { + DMSG("Cannot unregister an invalid persistent object"); + return PKCS11_RV_NOT_FOUND; + } + + ptr = TEE_Malloc(sizeof(struct token_persistent_objs) + + ((token->db_objs->count - 1) * sizeof(TEE_UUID)), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!ptr) + return PKCS11_CKR_DEVICE_MEMORY; + + res = open_db_file(token, &db_hdl); + if (res) + goto out; + + res = TEE_SeekObjectData(db_hdl, sizeof(struct token_persistent_main), + TEE_DATA_SEEK_SET); + if (res) { + DMSG("Failed to read database"); + goto out; + } + + TEE_MemMove(ptr, token->db_objs, + sizeof(struct token_persistent_objs) + + idx * sizeof(TEE_UUID)); + + ptr->count--; + count = ptr->count - idx; + + TEE_MemMove(&ptr->uuids[idx], + &token->db_objs->uuids[idx + 1], + count * sizeof(TEE_UUID)); + + res = TEE_WriteObjectData(db_hdl, ptr, + sizeof(struct token_persistent_objs) + + ptr->count * sizeof(TEE_UUID)); + if (res) + DMSG("Failed to update database"); + TEE_Free(token->db_objs); + token->db_objs = ptr; + ptr = NULL; + +out: + TEE_CloseObject(db_hdl); + TEE_Free(ptr); + + return tee2pkcs_error(res); +} + +enum pkcs11_rc register_persistent_object(struct ck_token *token, + TEE_UUID *uuid) +{ + TEE_ObjectHandle db_hdl = TEE_HANDLE_NULL; + TEE_Result res = TEE_ERROR_GENERIC; + void *ptr = NULL; + size_t size = 0; + int count = 0; + + if (get_persistent_obj_idx(token, uuid) >= 0) + TEE_Panic(0); + + count = token->db_objs->count; + ptr = TEE_Realloc(token->db_objs, + sizeof(struct token_persistent_objs) + + ((count + 1) * sizeof(TEE_UUID))); + if (!ptr) + return PKCS11_CKR_DEVICE_MEMORY; + + token->db_objs = ptr; + TEE_MemMove(token->db_objs->uuids + count, uuid, sizeof(TEE_UUID)); + + size = sizeof(struct token_persistent_main) + + sizeof(struct token_persistent_objs) + + count * sizeof(TEE_UUID); + + res = open_db_file(token, &db_hdl); + if (res) + goto out; + + res = TEE_TruncateObjectData(db_hdl, size + sizeof(TEE_UUID)); + if (res) + goto out; + + res = TEE_SeekObjectData(db_hdl, sizeof(struct token_persistent_main), + TEE_DATA_SEEK_SET); + if (res) + goto out; + + token->db_objs->count++; + + res = TEE_WriteObjectData(db_hdl, token->db_objs, + sizeof(struct token_persistent_objs) + + token->db_objs->count * sizeof(TEE_UUID)); + if (res) + token->db_objs->count--; + +out: + TEE_CloseObject(db_hdl); + + return tee2pkcs_error(res); +} + +enum pkcs11_rc load_persistent_object_attributes(struct pkcs11_object *obj) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + TEE_Result res = TEE_ERROR_GENERIC; + TEE_ObjectHandle hdl = obj->attribs_hdl; + TEE_ObjectInfo info = { }; + struct obj_attrs *attr = NULL; + size_t read_bytes = 0; + + if (obj->attributes) + return PKCS11_CKR_OK; + + if (hdl == TEE_HANDLE_NULL) { + res = TEE_OpenPersistentObject(TEE_STORAGE_PRIVATE, + obj->uuid, sizeof(*obj->uuid), + TEE_DATA_FLAG_ACCESS_READ, &hdl); + if (res) { + EMSG("OpenPersistent failed %#"PRIx32, res); + return tee2pkcs_error(res); + } + } + + TEE_MemFill(&info, 0, sizeof(info)); + res = TEE_GetObjectInfo1(hdl, &info); + if (res) { + EMSG("GetObjectInfo failed %#"PRIx32, res); + rc = tee2pkcs_error(res); + goto out; + } + + attr = TEE_Malloc(info.dataSize, TEE_MALLOC_FILL_ZERO); + if (!attr) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + res = TEE_ReadObjectData(hdl, attr, info.dataSize, &read_bytes); + if (!res) { + res = TEE_SeekObjectData(hdl, 0, TEE_DATA_SEEK_SET); + if (res) + EMSG("Seek to 0 failed %#"PRIx32, res); + } + + if (res) { + rc = tee2pkcs_error(res); + EMSG("Read %zu bytes, failed %#"PRIx32, + read_bytes, res); + goto out; + } + if (read_bytes != info.dataSize) { + EMSG("Read %zu bytes, expected %zu", + read_bytes, info.dataSize); + rc = PKCS11_CKR_GENERAL_ERROR; + goto out; + } + + obj->attributes = attr; + attr = NULL; + + rc = PKCS11_CKR_OK; + +out: + TEE_Free(attr); + /* Close object only if it was open from this function */ + if (obj->attribs_hdl == TEE_HANDLE_NULL) + TEE_CloseObject(hdl); + + return rc; +} + +void release_persistent_object_attributes(struct pkcs11_object *obj) +{ + TEE_Free(obj->attributes); + obj->attributes = NULL; +} + +enum pkcs11_rc update_persistent_object_attributes(struct pkcs11_object *obj) +{ + TEE_Result res = TEE_ERROR_GENERIC; + TEE_ObjectHandle hdl = TEE_HANDLE_NULL; + uint32_t tee_obj_flags = TEE_DATA_FLAG_ACCESS_WRITE; + size_t size = 0; + + assert(obj && obj->attributes); + + res = TEE_OpenPersistentObject(TEE_STORAGE_PRIVATE, + obj->uuid, sizeof(*obj->uuid), + tee_obj_flags, &hdl); + if (res) { + EMSG("OpenPersistent failed %#"PRIx32, res); + return tee2pkcs_error(res); + } + + size = sizeof(struct obj_attrs) + obj->attributes->attrs_size; + + res = TEE_WriteObjectData(hdl, obj->attributes, size); + if (res) + goto out; + + res = TEE_TruncateObjectData(hdl, size); + +out: + TEE_CloseObject(hdl); + return tee2pkcs_error(res); +} + +/* + * Return the token instance, either initialized from reset or initialized + * from the token persistent state if found. + */ +struct ck_token *init_persistent_db(unsigned int token_id) +{ + struct ck_token *token = get_token(token_id); + TEE_Result res = TEE_ERROR_GENERIC; + TEE_ObjectHandle db_hdl = TEE_HANDLE_NULL; + /* Copy persistent database: main db and object db */ + struct token_persistent_main *db_main = NULL; + struct token_persistent_objs *db_objs = NULL; + void *ptr = NULL; + + if (!token) + return NULL; + + LIST_INIT(&token->object_list); + + db_main = TEE_Malloc(sizeof(*db_main), TEE_MALLOC_FILL_ZERO); + db_objs = TEE_Malloc(sizeof(*db_objs), TEE_MALLOC_FILL_ZERO); + if (!db_main || !db_objs) + goto error; + + res = open_db_file(token, &db_hdl); + + if (res == TEE_SUCCESS) { + size_t size = 0; + size_t idx = 0; + + IMSG("PKCS11 token %u: load db", token_id); + + size = sizeof(*db_main); + res = TEE_ReadObjectData(db_hdl, db_main, size, &size); + if (res || size != sizeof(*db_main)) + TEE_Panic(0); + + size = sizeof(*db_objs); + res = TEE_ReadObjectData(db_hdl, db_objs, size, &size); + if (res || size != sizeof(*db_objs)) + TEE_Panic(0); + + if (db_objs->count > 0) { + size += db_objs->count * sizeof(TEE_UUID); + ptr = TEE_Realloc(db_objs, size); + if (!ptr) + goto error; + + db_objs = ptr; + size -= sizeof(*db_objs); + res = TEE_ReadObjectData(db_hdl, db_objs->uuids, size, + &size); + if (res || size != (db_objs->count * sizeof(TEE_UUID))) + TEE_Panic(0); + } + + for (idx = 0; idx < db_objs->count; idx++) { + /* Create an empty object instance */ + struct pkcs11_object *obj = NULL; + TEE_UUID *uuid = NULL; + + uuid = TEE_Malloc(sizeof(TEE_UUID), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!uuid) + goto error; + + TEE_MemMove(uuid, &db_objs->uuids[idx], sizeof(*uuid)); + + obj = create_token_object(NULL, uuid, token); + if (!obj) + TEE_Panic(0); + + LIST_INSERT_HEAD(&token->object_list, obj, link); + } + + } else if (res == TEE_ERROR_ITEM_NOT_FOUND) { + char file[PERSISTENT_OBJECT_ID_LEN] = { }; + + IMSG("PKCS11 token %u: init db", token_id); + + TEE_MemFill(db_main, 0, sizeof(*db_main)); + TEE_MemFill(db_main->label, '*', sizeof(db_main->label)); + + db_main->flags = PKCS11_CKFT_SO_PIN_TO_BE_CHANGED | + PKCS11_CKFT_USER_PIN_TO_BE_CHANGED | + PKCS11_CKFT_RNG | + PKCS11_CKFT_LOGIN_REQUIRED; + + res = get_db_file_name(token, file, sizeof(file)); + if (res) + TEE_Panic(0); + + /* + * Object stores persistent state + persistent object + * references. + */ + res = TEE_CreatePersistentObject(TEE_STORAGE_PRIVATE, + file, sizeof(file), + TEE_DATA_FLAG_ACCESS_READ | + TEE_DATA_FLAG_ACCESS_WRITE, + TEE_HANDLE_NULL, + db_main, sizeof(*db_main), + &db_hdl); + if (res) { + EMSG("Failed to create db: %#"PRIx32, res); + goto error; + } + + res = TEE_TruncateObjectData(db_hdl, sizeof(*db_main) + + sizeof(*db_objs)); + if (res) + TEE_Panic(0); + + res = TEE_SeekObjectData(db_hdl, sizeof(*db_main), + TEE_DATA_SEEK_SET); + if (res) + TEE_Panic(0); + + db_objs->count = 0; + res = TEE_WriteObjectData(db_hdl, db_objs, sizeof(*db_objs)); + if (res) + TEE_Panic(0); + + } else { + goto error; + } + + token->db_main = db_main; + token->db_objs = db_objs; + TEE_CloseObject(db_hdl); + + return token; + +error: + TEE_Free(db_main); + TEE_Free(db_objs); + if (db_hdl != TEE_HANDLE_NULL) + TEE_CloseObject(db_hdl); + + return NULL; +} diff --git a/optee_os/ta/pkcs11/src/pkcs11_attributes.c b/optee_os/ta/pkcs11/src/pkcs11_attributes.c new file mode 100644 index 0000000..aef712e --- /dev/null +++ b/optee_os/ta/pkcs11/src/pkcs11_attributes.c @@ -0,0 +1,2829 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "handle.h" +#include "pkcs11_attributes.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" +#include "sanitize_object.h" +#include "serializer.h" +#include "token_capabilities.h" + +static uint32_t pkcs11_func2ckfm(enum processing_func function) +{ + switch (function) { + case PKCS11_FUNCTION_DIGEST: + return PKCS11_CKFM_DIGEST; + case PKCS11_FUNCTION_GENERATE: + return PKCS11_CKFM_GENERATE; + case PKCS11_FUNCTION_GENERATE_PAIR: + return PKCS11_CKFM_GENERATE_KEY_PAIR; + case PKCS11_FUNCTION_DERIVE: + return PKCS11_CKFM_DERIVE; + case PKCS11_FUNCTION_WRAP: + return PKCS11_CKFM_WRAP; + case PKCS11_FUNCTION_UNWRAP: + return PKCS11_CKFM_UNWRAP; + case PKCS11_FUNCTION_ENCRYPT: + return PKCS11_CKFM_ENCRYPT; + case PKCS11_FUNCTION_DECRYPT: + return PKCS11_CKFM_DECRYPT; + case PKCS11_FUNCTION_SIGN: + return PKCS11_CKFM_SIGN; + case PKCS11_FUNCTION_VERIFY: + return PKCS11_CKFM_VERIFY; + case PKCS11_FUNCTION_SIGN_RECOVER: + return PKCS11_CKFM_SIGN_RECOVER; + case PKCS11_FUNCTION_VERIFY_RECOVER: + return PKCS11_CKFM_VERIFY_RECOVER; + default: + return 0; + } +} + +enum pkcs11_rc +check_mechanism_against_processing(struct pkcs11_session *session, + enum pkcs11_mechanism_id mechanism_type, + enum processing_func function, + enum processing_step step) +{ + bool allowed = false; + + switch (step) { + case PKCS11_FUNC_STEP_INIT: + switch (function) { + case PKCS11_FUNCTION_IMPORT: + case PKCS11_FUNCTION_COPY: + case PKCS11_FUNCTION_MODIFY: + case PKCS11_FUNCTION_DESTROY: + return PKCS11_CKR_OK; + default: + break; + } + /* + * Check that the returned PKCS11_CKFM_* flag from + * pkcs11_func2ckfm() is among the ones from + * mechanism_supported_flags(). + */ + allowed = mechanism_supported_flags(mechanism_type) & + pkcs11_func2ckfm(function); + break; + + case PKCS11_FUNC_STEP_ONESHOT: + if (session->processing->always_authen && + !session->processing->relogged) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + + if (session->processing->step == PKCS11_FUNC_STEP_UPDATE || + session->processing->step == PKCS11_FUNC_STEP_FINAL) { + EMSG("Cannot perform one-shot on active processing"); + return PKCS11_CKR_OPERATION_ACTIVE; + } + + allowed = true; + break; + + case PKCS11_FUNC_STEP_UPDATE: + if (session->processing->always_authen && + !session->processing->relogged) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + + if (session->processing->step == PKCS11_FUNC_STEP_ONESHOT || + session->processing->step == PKCS11_FUNC_STEP_FINAL) { + EMSG("Cannot perform update on finalized processing"); + return PKCS11_CKR_OPERATION_ACTIVE; + } + + allowed = !mechanism_is_one_shot_only(mechanism_type); + break; + + case PKCS11_FUNC_STEP_UPDATE_KEY: + assert(function == PKCS11_FUNCTION_DIGEST); + + if (session->processing->always_authen && + !session->processing->relogged) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + + allowed = true; + break; + + case PKCS11_FUNC_STEP_FINAL: + if (session->processing->always_authen && + !session->processing->relogged) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + + if (session->processing->step == PKCS11_FUNC_STEP_ONESHOT) { + EMSG("Cannot perform final on oneshot processing"); + return PKCS11_CKR_OPERATION_ACTIVE; + } + return PKCS11_CKR_OK; + + default: + TEE_Panic(step); + break; + } + + if (!allowed) { + EMSG("Processing %#x/%s not permitted (%u/%u)", + (unsigned int)mechanism_type, id2str_proc(mechanism_type), + function, step); + return PKCS11_CKR_MECHANISM_INVALID; + } + + return PKCS11_CKR_OK; +} + +/* + * Object default boolean attributes as per PKCS#11 + */ +static uint8_t *pkcs11_object_default_boolprop(uint32_t attribute) +{ + static const uint8_t bool_true = 1; + static const uint8_t bool_false; + + switch (attribute) { + /* As per PKCS#11 default value */ + case PKCS11_CKA_MODIFIABLE: + case PKCS11_CKA_COPYABLE: + case PKCS11_CKA_DESTROYABLE: + return (uint8_t *)&bool_true; + case PKCS11_CKA_TOKEN: + case PKCS11_CKA_PRIVATE: + case PKCS11_CKA_WRAP_WITH_TRUSTED: + case PKCS11_CKA_ALWAYS_AUTHENTICATE: + case PKCS11_CKA_SENSITIVE: + return (uint8_t *)&bool_false; + /* Token specific default value */ + case PKCS11_CKA_SIGN: + case PKCS11_CKA_VERIFY: + case PKCS11_CKA_DERIVE: + case PKCS11_CKA_ENCRYPT: + case PKCS11_CKA_DECRYPT: + case PKCS11_CKA_SIGN_RECOVER: + case PKCS11_CKA_VERIFY_RECOVER: + case PKCS11_CKA_WRAP: + case PKCS11_CKA_UNWRAP: + case PKCS11_CKA_EXTRACTABLE: + case PKCS11_CKA_TRUSTED: + return (uint8_t *)&bool_false; + default: + DMSG("No default for boolprop attribute %#"PRIx32, attribute); + return NULL; + } +} + +/* + * Object expects several boolean attributes to be set to a default value + * or to a validate client configuration value. This function append the input + * attribute (id/size/value) in the serialized object. + */ +static enum pkcs11_rc pkcs11_import_object_boolprop(struct obj_attrs **out, + struct obj_attrs *templ, + uint32_t attribute) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint8_t bbool = 0; + uint32_t size = sizeof(uint8_t); + void *attr = NULL; + + rc = get_attribute(templ, attribute, &bbool, &size); + if (rc) { + if (rc != PKCS11_RV_NOT_FOUND) + return rc; + attr = pkcs11_object_default_boolprop(attribute); + if (!attr) + return PKCS11_CKR_TEMPLATE_INCOMPLETE; + } else { + attr = &bbool; + } + + /* Boolean attributes are 1byte in the ABI, no alignment issue */ + return add_attribute(out, attribute, attr, sizeof(uint8_t)); +} + +static enum pkcs11_rc set_mandatory_boolprops(struct obj_attrs **out, + struct obj_attrs *temp, + uint32_t const *bp, + size_t bp_count) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + size_t n = 0; + + for (n = 0; n < bp_count; n++) { + rc = pkcs11_import_object_boolprop(out, temp, bp[n]); + if (rc) + return rc; + } + + return rc; +} + +static enum pkcs11_rc set_mandatory_attributes(struct obj_attrs **out, + struct obj_attrs *temp, + uint32_t const *attrs, + size_t attrs_count) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + size_t n = 0; + + for (n = 0; n < attrs_count; n++) { + uint32_t size = 0; + void *value = NULL; + + if (get_attribute_ptr(temp, attrs[n], &value, &size)) + return PKCS11_CKR_TEMPLATE_INCOMPLETE; + + rc = add_attribute(out, attrs[n], value, size); + if (rc) + return rc; + } + + return rc; +} + +static enum pkcs11_rc get_default_value(enum pkcs11_attr_id id, void **value, + uint32_t *size) +{ + /* should have been taken care of already */ + assert(!pkcs11_attr_is_boolean(id)); + + /* All other attributes have an empty default value */ + *value = NULL; + *size = 0; + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc set_optional_attributes_with_def(struct obj_attrs **out, + struct obj_attrs *temp, + uint32_t const *attrs, + size_t attrs_count, + bool default_to_null) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + size_t n = 0; + + for (n = 0; n < attrs_count; n++) { + uint32_t size = 0; + void *value = NULL; + + rc = get_attribute_ptr(temp, attrs[n], &value, &size); + if (rc == PKCS11_RV_NOT_FOUND) { + if (default_to_null) { + rc = get_default_value(attrs[n], &value, &size); + } else { + rc = PKCS11_CKR_OK; + continue; + } + } + if (rc) + return rc; + + rc = add_attribute(out, attrs[n], value, size); + if (rc) + return rc; + } + + return rc; +} + +static enum pkcs11_rc set_attributes_opt_or_null(struct obj_attrs **out, + struct obj_attrs *temp, + uint32_t const *attrs, + size_t attrs_count) +{ + return set_optional_attributes_with_def(out, temp, attrs, attrs_count, + true /* defaults to empty */); +} + +static enum pkcs11_rc set_optional_attributes(struct obj_attrs **out, + struct obj_attrs *temp, + uint32_t const *attrs, + size_t attrs_count) +{ + return set_optional_attributes_with_def(out, temp, attrs, attrs_count, + false /* no default value */); +} + +/* + * Below are listed the mandated or optional expected attributes for + * PKCS#11 storage objects. + * + * Note: boolprops (mandated boolean attributes) PKCS11_CKA_ALWAYS_SENSITIVE, + * and PKCS11_CKA_NEVER_EXTRACTABLE are set by the token, not provided + * in the client template. + */ + +/* PKCS#11 specification for any object (session/token) of the storage */ +static const uint32_t any_object_boolprops[] = { + PKCS11_CKA_TOKEN, PKCS11_CKA_PRIVATE, + PKCS11_CKA_MODIFIABLE, PKCS11_CKA_COPYABLE, PKCS11_CKA_DESTROYABLE, +}; + +static const uint32_t any_object_opt_or_null[] = { + PKCS11_CKA_LABEL, +}; + +/* PKCS#11 specification for raw data object (+any_object_xxx) */ +const uint32_t raw_data_opt_or_null[] = { + PKCS11_CKA_OBJECT_ID, PKCS11_CKA_APPLICATION, PKCS11_CKA_VALUE, +}; + +/* PKCS#11 specification for certificate object (+pkcs11_any_object_xxx) */ +static const uint32_t pkcs11_certificate_mandated[] = { + PKCS11_CKA_CERTIFICATE_TYPE, +}; + +static const uint32_t pkcs11_certificate_boolprops[] = { + PKCS11_CKA_TRUSTED, +}; + +static const uint32_t pkcs11_certificate_optional[] = { + PKCS11_CKA_CERTIFICATE_CATEGORY, PKCS11_CKA_CHECK_VALUE, + PKCS11_CKA_START_DATE, PKCS11_CKA_END_DATE, PKCS11_CKA_PUBLIC_KEY_INFO, +}; + +/* + * PKCS#11 specification for X.509 certificate object (+pkcs11_certificate_xxx) + */ +static const uint32_t pkcs11_x509_certificate_mandated[] = { + PKCS11_CKA_SUBJECT, +}; + +static const uint32_t pkcs11_x509_certificate_optional[] = { + PKCS11_CKA_ID, PKCS11_CKA_ISSUER, PKCS11_CKA_SERIAL_NUMBER, + PKCS11_CKA_VALUE, PKCS11_CKA_URL, + PKCS11_CKA_HASH_OF_SUBJECT_PUBLIC_KEY, + PKCS11_CKA_HASH_OF_ISSUER_PUBLIC_KEY, + PKCS11_CKA_JAVA_MIDP_SECURITY_DOMAIN, PKCS11_CKA_NAME_HASH_ALGORITHM, +}; + +/* PKCS#11 specification for any key object (+any_object_xxx) */ +static const uint32_t any_key_boolprops[] = { + PKCS11_CKA_DERIVE, +}; + +static const uint32_t any_key_opt_or_null[] = { + PKCS11_CKA_ID, + PKCS11_CKA_START_DATE, PKCS11_CKA_END_DATE, +}; + +static const uint32_t any_key_optional[] = { + PKCS11_CKA_ALLOWED_MECHANISMS, +}; + +/* PKCS#11 specification for any symmetric key (+any_key_xxx) */ +static const uint32_t symm_key_boolprops[] = { + PKCS11_CKA_ENCRYPT, PKCS11_CKA_DECRYPT, + PKCS11_CKA_SIGN, PKCS11_CKA_VERIFY, + PKCS11_CKA_WRAP, PKCS11_CKA_UNWRAP, + PKCS11_CKA_SENSITIVE, PKCS11_CKA_EXTRACTABLE, + PKCS11_CKA_WRAP_WITH_TRUSTED, PKCS11_CKA_TRUSTED, +}; + +static const uint32_t symm_key_opt_or_null[] = { + PKCS11_CKA_WRAP_TEMPLATE, PKCS11_CKA_UNWRAP_TEMPLATE, + PKCS11_CKA_DERIVE_TEMPLATE, PKCS11_CKA_VALUE, +}; + +static const uint32_t symm_key_optional[] = { + PKCS11_CKA_VALUE_LEN, +}; + +/* PKCS#11 specification for any asymmetric public key (+any_key_xxx) */ +static const uint32_t public_key_boolprops[] = { + PKCS11_CKA_ENCRYPT, PKCS11_CKA_VERIFY, PKCS11_CKA_VERIFY_RECOVER, + PKCS11_CKA_WRAP, + PKCS11_CKA_TRUSTED, +}; + +static const uint32_t public_key_mandated[] = { +}; + +static const uint32_t public_key_opt_or_null[] = { + PKCS11_CKA_SUBJECT, PKCS11_CKA_WRAP_TEMPLATE, + PKCS11_CKA_PUBLIC_KEY_INFO, +}; + +/* PKCS#11 specification for any asymmetric private key (+any_key_xxx) */ +static const uint32_t private_key_boolprops[] = { + PKCS11_CKA_DECRYPT, PKCS11_CKA_SIGN, PKCS11_CKA_SIGN_RECOVER, + PKCS11_CKA_UNWRAP, + PKCS11_CKA_SENSITIVE, PKCS11_CKA_EXTRACTABLE, + PKCS11_CKA_WRAP_WITH_TRUSTED, PKCS11_CKA_ALWAYS_AUTHENTICATE, +}; + +static const uint32_t private_key_mandated[] = { +}; + +static const uint32_t private_key_opt_or_null[] = { + PKCS11_CKA_SUBJECT, PKCS11_CKA_UNWRAP_TEMPLATE, + PKCS11_CKA_PUBLIC_KEY_INFO, +}; + +/* PKCS#11 specification for any RSA key (+public/private_key_xxx) */ +static const uint32_t rsa_pub_key_gen_mand[] = { + PKCS11_CKA_MODULUS_BITS, +}; + +static const uint32_t rsa_pub_key_create_mand[] = { + PKCS11_CKA_MODULUS, PKCS11_CKA_PUBLIC_EXPONENT, +}; + +static const uint32_t rsa_pub_key_gen_opt_or_null[] = { + PKCS11_CKA_PUBLIC_EXPONENT, +}; + +static const uint32_t rsa_priv_key_opt_or_null[] = { + PKCS11_CKA_MODULUS, PKCS11_CKA_PUBLIC_EXPONENT, + PKCS11_CKA_PRIVATE_EXPONENT, + PKCS11_CKA_PRIME_1, PKCS11_CKA_PRIME_2, + PKCS11_CKA_EXPONENT_1, PKCS11_CKA_EXPONENT_2, PKCS11_CKA_COEFFICIENT, +}; + +/* PKCS#11 specification for any EC key (+public/private_key_xxx) */ +static const uint32_t ec_public_key_mandated[] = { + PKCS11_CKA_EC_PARAMS, +}; + +static const uint32_t ec_public_key_opt_or_null[] = { + PKCS11_CKA_EC_POINT, +}; + +static const uint32_t ec_private_key_mandated[] = { +}; + +static const uint32_t ec_private_key_opt_or_null[] = { + PKCS11_CKA_EC_PARAMS, + PKCS11_CKA_VALUE, +}; + +static const uint32_t eddsa_private_key_opt_or_null[] = { + PKCS11_CKA_EC_PARAMS, + PKCS11_CKA_VALUE, + PKCS11_CKA_EC_POINT, +}; + +static enum pkcs11_rc create_storage_attributes(struct obj_attrs **out, + struct obj_attrs *temp) +{ + enum pkcs11_class_id class = PKCS11_CKO_UNDEFINED_ID; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + rc = init_attributes_head(out); + if (rc) + return rc; + + /* Object class is mandatory */ + class = get_class(temp); + if (class == PKCS11_CKO_UNDEFINED_ID) { + EMSG("Class attribute not found"); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + rc = add_attribute(out, PKCS11_CKA_CLASS, &class, sizeof(uint32_t)); + if (rc) + return rc; + + rc = set_mandatory_boolprops(out, temp, any_object_boolprops, + ARRAY_SIZE(any_object_boolprops)); + if (rc) + return rc; + + return set_attributes_opt_or_null(out, temp, any_object_opt_or_null, + ARRAY_SIZE(any_object_opt_or_null)); +} + +static enum pkcs11_rc create_genkey_attributes(struct obj_attrs **out, + struct obj_attrs *temp) +{ + uint32_t type = PKCS11_CKO_UNDEFINED_ID; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + rc = create_storage_attributes(out, temp); + if (rc) + return rc; + + type = get_key_type(temp); + if (type == PKCS11_CKK_UNDEFINED_ID) { + EMSG("Key type attribute not found"); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + rc = add_attribute(out, PKCS11_CKA_KEY_TYPE, &type, sizeof(uint32_t)); + if (rc) + return rc; + + rc = set_mandatory_boolprops(out, temp, any_key_boolprops, + ARRAY_SIZE(any_key_boolprops)); + if (rc) + return rc; + + rc = set_attributes_opt_or_null(out, temp, any_key_opt_or_null, + ARRAY_SIZE(any_key_opt_or_null)); + if (rc) + return rc; + + return set_optional_attributes(out, temp, any_key_optional, + ARRAY_SIZE(any_key_optional)); + +} + +static enum pkcs11_rc create_symm_key_attributes(struct obj_attrs **out, + struct obj_attrs *temp) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(get_class(temp) == PKCS11_CKO_SECRET_KEY); + + rc = create_genkey_attributes(out, temp); + if (rc) + return rc; + + assert(get_class(*out) == PKCS11_CKO_SECRET_KEY); + + switch (get_key_type(*out)) { + case PKCS11_CKK_GENERIC_SECRET: + case PKCS11_CKK_AES: + case PKCS11_CKK_MD5_HMAC: + case PKCS11_CKK_SHA_1_HMAC: + case PKCS11_CKK_SHA256_HMAC: + case PKCS11_CKK_SHA384_HMAC: + case PKCS11_CKK_SHA512_HMAC: + case PKCS11_CKK_SHA224_HMAC: + break; + default: + EMSG("Invalid key type %#"PRIx32"/%s", + get_key_type(*out), id2str_key_type(get_key_type(*out))); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + rc = set_mandatory_boolprops(out, temp, symm_key_boolprops, + ARRAY_SIZE(symm_key_boolprops)); + if (rc) + return rc; + + rc = set_attributes_opt_or_null(out, temp, symm_key_opt_or_null, + ARRAY_SIZE(symm_key_opt_or_null)); + if (rc) + return rc; + + return set_optional_attributes(out, temp, symm_key_optional, + ARRAY_SIZE(symm_key_optional)); +} + +static enum pkcs11_rc create_data_attributes(struct obj_attrs **out, + struct obj_attrs *temp) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(get_class(temp) == PKCS11_CKO_DATA); + + rc = create_storage_attributes(out, temp); + if (rc) + return rc; + + assert(get_class(*out) == PKCS11_CKO_DATA); + + return set_attributes_opt_or_null(out, temp, raw_data_opt_or_null, + ARRAY_SIZE(raw_data_opt_or_null)); +} + +static enum pkcs11_rc create_certificate_attributes(struct obj_attrs **out, + struct obj_attrs *temp) +{ + uint32_t const *mandated = NULL; + uint32_t const *optional = NULL; + size_t mandated_count = 0; + size_t optional_count = 0; + void *attr_value = NULL; + uint32_t attr_size = 0; + uint32_t default_cert_category = + PKCS11_CK_CERTIFICATE_CATEGORY_UNSPECIFIED; + uint32_t default_name_hash_alg = PKCS11_CKM_SHA_1; + uint32_t cert_category = 0; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(get_class(temp) == PKCS11_CKO_CERTIFICATE); + + rc = create_storage_attributes(out, temp); + if (rc) + return rc; + + assert(get_class(*out) == PKCS11_CKO_CERTIFICATE); + + rc = set_mandatory_boolprops(out, temp, pkcs11_certificate_boolprops, + ARRAY_SIZE(pkcs11_certificate_boolprops)); + if (rc) + return rc; + + rc = set_mandatory_attributes(out, temp, pkcs11_certificate_mandated, + ARRAY_SIZE(pkcs11_certificate_mandated)); + if (rc) + return rc; + + rc = set_optional_attributes(out, temp, pkcs11_certificate_optional, + ARRAY_SIZE(pkcs11_certificate_optional)); + if (rc) + return rc; + + switch (get_certificate_type(*out)) { + case PKCS11_CKC_X_509: + mandated = pkcs11_x509_certificate_mandated; + optional = pkcs11_x509_certificate_optional; + mandated_count = ARRAY_SIZE(pkcs11_x509_certificate_mandated); + optional_count = ARRAY_SIZE(pkcs11_x509_certificate_optional); + break; + default: + EMSG("Invalid certificate type %#"PRIx32"/%s", + get_certificate_type(*out), + id2str_certificate_type(get_certificate_type(*out))); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + rc = set_mandatory_attributes(out, temp, mandated, mandated_count); + if (rc) + return rc; + + rc = set_optional_attributes(out, temp, optional, optional_count); + if (rc) + return rc; + + attr_size = 0; + rc = get_attribute_ptr(*out, PKCS11_CKA_CERTIFICATE_CATEGORY, + &attr_value, &attr_size); + if (rc == PKCS11_CKR_OK && attr_size == sizeof(cert_category)) { + /* Sanitize certificate category */ + TEE_MemMove(&cert_category, attr_value, sizeof(cert_category)); + + switch (cert_category) { + case PKCS11_CK_CERTIFICATE_CATEGORY_UNSPECIFIED: + case PKCS11_CK_CERTIFICATE_CATEGORY_TOKEN_USER: + case PKCS11_CK_CERTIFICATE_CATEGORY_AUTHORITY: + case PKCS11_CK_CERTIFICATE_CATEGORY_OTHER_ENTITY: + break; + default: + EMSG("Invalid certificate category %#"PRIx32, + cert_category); + + return PKCS11_CKR_ATTRIBUTE_VALUE_INVALID; + } + } else if (rc == PKCS11_RV_NOT_FOUND) { + /* Set default category when missing */ + rc = set_attribute(out, PKCS11_CKA_CERTIFICATE_CATEGORY, + &default_cert_category, + sizeof(default_cert_category)); + if (rc) + return rc; + } else { + /* All other cases are errors */ + EMSG("Invalid certificate category"); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + attr_size = 0; + rc = get_attribute_ptr(*out, PKCS11_CKA_NAME_HASH_ALGORITHM, NULL, + &attr_size); + if (rc == PKCS11_CKR_OK && attr_size == sizeof(uint32_t)) { + /* We accept any algorithm what caller wanted to specify */ + } else if (rc == PKCS11_RV_NOT_FOUND) { + /* Set default hash algorithm when missing */ + rc = set_attribute(out, PKCS11_CKA_NAME_HASH_ALGORITHM, + &default_name_hash_alg, + sizeof(default_name_hash_alg)); + if (rc) + return rc; + } else { + /* All other cases are errors */ + EMSG("Invalid name hash algorithm"); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + return rc; +} + +static enum pkcs11_rc create_pub_key_attributes(struct obj_attrs **out, + struct obj_attrs *temp, + enum processing_func function) +{ + uint32_t const *mandated = NULL; + uint32_t const *oon = NULL; + size_t mandated_count = 0; + size_t oon_count = 0; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(get_class(temp) == PKCS11_CKO_PUBLIC_KEY); + + rc = create_genkey_attributes(out, temp); + if (rc) + return rc; + + assert(get_class(*out) == PKCS11_CKO_PUBLIC_KEY); + + rc = set_mandatory_boolprops(out, temp, public_key_boolprops, + ARRAY_SIZE(public_key_boolprops)); + if (rc) + return rc; + + rc = set_mandatory_attributes(out, temp, public_key_mandated, + ARRAY_SIZE(public_key_mandated)); + if (rc) + return rc; + + rc = set_attributes_opt_or_null(out, temp, + public_key_opt_or_null, + ARRAY_SIZE(public_key_opt_or_null)); + if (rc) + return rc; + + switch (get_key_type(*out)) { + case PKCS11_CKK_RSA: + switch (function) { + case PKCS11_FUNCTION_GENERATE_PAIR: + mandated = rsa_pub_key_gen_mand; + oon = rsa_pub_key_gen_opt_or_null; + mandated_count = ARRAY_SIZE(rsa_pub_key_gen_mand); + oon_count = ARRAY_SIZE(rsa_pub_key_gen_opt_or_null); + break; + case PKCS11_FUNCTION_IMPORT: + mandated = rsa_pub_key_create_mand; + mandated_count = ARRAY_SIZE(rsa_pub_key_create_mand); + break; + default: + EMSG("Unsupported function %#"PRIx32"/%s", function, + id2str_function(function)); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + break; + case PKCS11_CKK_EC: + case PKCS11_CKK_EC_EDWARDS: + mandated = ec_public_key_mandated; + oon = ec_public_key_opt_or_null; + mandated_count = ARRAY_SIZE(ec_public_key_mandated); + oon_count = ARRAY_SIZE(ec_public_key_opt_or_null); + break; + default: + EMSG("Invalid key type %#"PRIx32"/%s", + get_key_type(*out), id2str_key_type(get_key_type(*out))); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + rc = set_mandatory_attributes(out, temp, mandated, mandated_count); + if (rc) + return rc; + + return set_attributes_opt_or_null(out, temp, oon, oon_count); +} + +static enum pkcs11_rc +create_pub_key_rsa_generated_attributes(struct obj_attrs **out, + struct obj_attrs *temp, + enum processing_func function) +{ + uint32_t key_bits = 0; + void *a_ptr = NULL; + uint32_t a_size = 0; + + if (function != PKCS11_FUNCTION_IMPORT) + return PKCS11_CKR_OK; + + /* Calculate CKA_MODULUS_BITS */ + + if (get_attribute_ptr(temp, PKCS11_CKA_MODULUS, + &a_ptr, &a_size) || !a_ptr) { + EMSG("No CKA_MODULUS attribute found in public key"); + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + } + + key_bits = a_size * 8; + + return add_attribute(out, PKCS11_CKA_MODULUS_BITS, &key_bits, + sizeof(key_bits)); +} + +static enum pkcs11_rc +create_pub_key_generated_attributes(struct obj_attrs **out, + struct obj_attrs *temp, + enum processing_func function) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + + switch (get_key_type(*out)) { + case PKCS11_CKK_RSA: + rc = create_pub_key_rsa_generated_attributes(out, temp, + function); + break; + default: + /* no-op */ + break; + } + + return rc; +} + +static enum pkcs11_rc create_priv_key_attributes(struct obj_attrs **out, + struct obj_attrs *temp) +{ + uint32_t const *mandated = NULL; + uint32_t const *oon = NULL; + size_t mandated_count = 0; + size_t oon_count = 0; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(get_class(temp) == PKCS11_CKO_PRIVATE_KEY); + + rc = create_genkey_attributes(out, temp); + if (rc) + return rc; + + assert(get_class(*out) == PKCS11_CKO_PRIVATE_KEY); + + rc = set_mandatory_boolprops(out, temp, private_key_boolprops, + ARRAY_SIZE(private_key_boolprops)); + if (rc) + return rc; + + rc = set_mandatory_attributes(out, temp, private_key_mandated, + ARRAY_SIZE(private_key_mandated)); + if (rc) + return rc; + + rc = set_attributes_opt_or_null(out, temp, private_key_opt_or_null, + ARRAY_SIZE(private_key_opt_or_null)); + if (rc) + return rc; + + switch (get_key_type(*out)) { + case PKCS11_CKK_RSA: + oon = rsa_priv_key_opt_or_null; + oon_count = ARRAY_SIZE(rsa_priv_key_opt_or_null); + break; + case PKCS11_CKK_EC: + mandated = ec_private_key_mandated; + oon = ec_private_key_opt_or_null; + mandated_count = ARRAY_SIZE(ec_private_key_mandated); + oon_count = ARRAY_SIZE(ec_private_key_opt_or_null); + break; + case PKCS11_CKK_EC_EDWARDS: + mandated = ec_private_key_mandated; + oon = eddsa_private_key_opt_or_null; + mandated_count = ARRAY_SIZE(ec_private_key_mandated); + oon_count = ARRAY_SIZE(eddsa_private_key_opt_or_null); + break; + default: + EMSG("Invalid key type %#"PRIx32"/%s", + get_key_type(*out), id2str_key_type(get_key_type(*out))); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + rc = set_mandatory_attributes(out, temp, mandated, mandated_count); + if (rc) + return rc; + + return set_attributes_opt_or_null(out, temp, oon, oon_count); +} + +static int mbd_rand(void *rng_state __unused, unsigned char *output, size_t len) +{ + TEE_GenerateRandom(output, len); + return 0; +} + +static enum pkcs11_rc +create_ec_priv_key_hidden_attributes(struct obj_attrs **out, + struct obj_attrs *temp, + enum processing_func function) +{ + struct mbedtls_ecp_keypair key_pair = { }; + mbedtls_ecp_group_id ec_curve = MBEDTLS_ECP_DP_NONE; + mbedtls_ecp_group key_pair_grp = { }; + mbedtls_ecp_point key_pair_Q = { }; + mbedtls_mpi key_pair_d = { }; + size_t buflen = 0; + uint8_t *buf = NULL; + size_t asnbuflen = 0; + uint8_t *asnbuf = NULL; + uint8_t *ptr = NULL; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + int tee_size = 0; + int tee_curve = 0; + void *a_ptr = NULL; + uint32_t a_size = 0; + int ret = 0; + + if (function != PKCS11_FUNCTION_IMPORT) + return PKCS11_CKR_OK; + + /* + * TEE internal API requires that for private key operations there + * needs to be also public key available. + * + * Generate hidden EC point from private key. + */ + + if (get_attribute_ptr(temp, PKCS11_CKA_EC_PARAMS, + &a_ptr, &a_size) || !a_ptr) { + EMSG("No EC_PARAMS attribute found in private key"); + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + } + + /* Just valdiate that curve is found */ + tee_size = ec_params2tee_keysize(a_ptr, a_size); + if (!tee_size) { + EMSG("Unsupported EC_PARAMS curve"); + return PKCS11_CKR_CURVE_NOT_SUPPORTED; + } + + tee_curve = ec_params2tee_curve(a_ptr, a_size); + + switch (tee_curve) { + case TEE_ECC_CURVE_NIST_P192: + ec_curve = MBEDTLS_ECP_DP_SECP192R1; + break; + case TEE_ECC_CURVE_NIST_P224: + ec_curve = MBEDTLS_ECP_DP_SECP224R1; + break; + case TEE_ECC_CURVE_NIST_P256: + ec_curve = MBEDTLS_ECP_DP_SECP256R1; + break; + case TEE_ECC_CURVE_NIST_P384: + ec_curve = MBEDTLS_ECP_DP_SECP384R1; + break; + case TEE_ECC_CURVE_NIST_P521: + ec_curve = MBEDTLS_ECP_DP_SECP521R1; + break; + default: + EMSG("Failed to map EC_PARAMS to supported curve"); + return PKCS11_CKR_CURVE_NOT_SUPPORTED; + } + + if (get_attribute_ptr(temp, PKCS11_CKA_VALUE, + &a_ptr, &a_size) || !a_ptr) { + EMSG("No VALUE attribute found in private key"); + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + } + + mbedtls_ecp_keypair_init(&key_pair); + mbedtls_ecp_group_init(&key_pair_grp); + mbedtls_mpi_init(&key_pair_d); + mbedtls_ecp_point_init(&key_pair_Q); + + ret = mbedtls_ecp_read_key(ec_curve, &key_pair, a_ptr, a_size); + if (ret) { + EMSG("Failed to parse CKA_VALUE"); + rc = PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + goto out; + } + + ret = mbedtls_ecp_export(&key_pair, &key_pair_grp, &key_pair_d, + &key_pair_Q); + if (ret) { + EMSG("Failed to export key"); + goto out; + } + + ret = mbedtls_ecp_mul(&key_pair_grp, &key_pair_Q, &key_pair_d, + &key_pair_grp.G, mbd_rand, NULL); + if (ret) { + EMSG("Failed to create public key"); + goto out; + } + + ret = mbedtls_ecp_check_privkey(&key_pair_grp, &key_pair_d); + if (ret) { + EMSG("Failed to verify private key"); + goto out; + } + + ret = mbedtls_ecp_check_pubkey(&key_pair_grp, &key_pair_Q); + if (ret) { + EMSG("Failed to verify public key"); + goto out; + } + + ret = mbedtls_ecp_point_write_binary(&key_pair_grp, &key_pair_Q, + MBEDTLS_ECP_PF_UNCOMPRESSED, + &buflen, NULL, 0); + if (ret != MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL) { + EMSG("Failed to determine size of binary public key"); + goto out; + } + + buf = TEE_Malloc(buflen, TEE_MALLOC_FILL_ZERO); + if (!buf) { + EMSG("Failed to allocate memory for public key"); + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + asnbuflen = 1 /* octet string */ + 5 /* length */ + buflen; + + asnbuf = TEE_Malloc(asnbuflen, TEE_MALLOC_FILL_ZERO); + if (!asnbuf) { + EMSG("Failed to allocate memory for public key"); + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + ret = mbedtls_ecp_point_write_binary(&key_pair_grp, &key_pair_Q, + MBEDTLS_ECP_PF_UNCOMPRESSED, + &buflen, buf, buflen); + if (ret) { + EMSG("Failed to write binary public key"); + goto out; + } + + /* Note: ASN.1 writing works backwards */ + ptr = asnbuf + asnbuflen; + + ret = mbedtls_asn1_write_octet_string(&ptr, asnbuf, buf, buflen); + if (ret < 0) { + EMSG("Failed to write asn1 public key"); + goto out; + } + + rc = add_attribute(out, PKCS11_CKA_OPTEE_HIDDEN_EC_POINT, ptr, + (size_t)ret); + +out: + TEE_Free(asnbuf); + TEE_Free(buf); + mbedtls_ecp_keypair_free(&key_pair); + mbedtls_ecp_group_free(&key_pair_grp); + mbedtls_mpi_free(&key_pair_d); + mbedtls_ecp_point_free(&key_pair_Q); + + return rc; +} + +static enum pkcs11_rc +create_priv_key_hidden_attributes(struct obj_attrs **out, + struct obj_attrs *temp, + enum processing_func function) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + + switch (get_key_type(*out)) { + case PKCS11_CKK_EC: + rc = create_ec_priv_key_hidden_attributes(out, temp, function); + break; + default: + /* no-op */ + break; + } + + return rc; +} + +static enum pkcs11_rc +sanitize_symm_key_attributes(struct obj_attrs **temp, + enum processing_func function) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint32_t a_size = 0; + + assert(get_class(*temp) == PKCS11_CKO_SECRET_KEY); + + rc = get_attribute_ptr(*temp, PKCS11_CKA_VALUE, NULL, &a_size); + + switch (get_key_type(*temp)) { + case PKCS11_CKK_GENERIC_SECRET: + case PKCS11_CKK_AES: + case PKCS11_CKK_MD5_HMAC: + case PKCS11_CKK_SHA_1_HMAC: + case PKCS11_CKK_SHA256_HMAC: + case PKCS11_CKK_SHA384_HMAC: + case PKCS11_CKK_SHA512_HMAC: + case PKCS11_CKK_SHA224_HMAC: + switch (function) { + case PKCS11_FUNCTION_IMPORT: + /* CKA_VALUE is a mandatory with C_CreateObject */ + if (rc || a_size == 0) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + if (get_attribute_ptr(*temp, PKCS11_CKA_VALUE_LEN, NULL, + NULL) != PKCS11_RV_NOT_FOUND) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + return add_attribute(temp, PKCS11_CKA_VALUE_LEN, + &a_size, sizeof(uint32_t)); + case PKCS11_FUNCTION_GENERATE: + if (rc != PKCS11_RV_NOT_FOUND) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + break; + default: + break; + } + break; + default: + EMSG("Invalid key type %#"PRIx32"/%s", + get_key_type(*temp), id2str_key_type(get_key_type(*temp))); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + return PKCS11_CKR_OK; +} + +/* + * Create an attribute list for a new object from a template and a parent + * object (optional) for an object generation function (generate, copy, + * derive...). + * + * PKCS#11 directives on the supplied template and expected return value: + * - template has an invalid attribute ID: ATTRIBUTE_TYPE_INVALID + * - template has an invalid value for an attribute: ATTRIBUTE_VALID_INVALID + * - template has value for a read-only attribute: ATTRIBUTE_READ_ONLY + * - template+default+parent => still miss an attribute: TEMPLATE_INCONSISTENT + * + * INFO on PKCS11_CMD_COPY_OBJECT: + * - parent PKCS11_CKA_COPYIABLE=false => return ACTION_PROHIBITED. + * - template can specify PKCS11_CKA_TOKEN, PKCS11_CKA_PRIVATE, + * PKCS11_CKA_MODIFIABLE, PKCS11_CKA_DESTROYABLE. + * - SENSITIVE can change from false to true, not from true to false. + * - LOCAL is the parent LOCAL + */ +enum pkcs11_rc +create_attributes_from_template(struct obj_attrs **out, void *template, + size_t template_size, + struct obj_attrs *parent, + enum processing_func function, + enum pkcs11_mechanism_id mecha, + enum pkcs11_class_id template_class) +{ + struct obj_attrs *temp = NULL; + struct obj_attrs *attrs = NULL; + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint8_t local = 0; + uint8_t always_sensitive = 0; + uint8_t never_extract = 0; + uint8_t extractable = 0; + uint32_t class = PKCS11_UNDEFINED_ID; + uint32_t type = PKCS11_UNDEFINED_ID; + uint32_t mechanism_id = PKCS11_CKM_UNDEFINED_ID; + struct obj_attrs *req_attrs = NULL; + uint32_t size = 0; + uint32_t indirect_template = PKCS11_CKA_UNDEFINED_ID; + +#ifdef DEBUG /* Sanity: check function argument */ + trace_attributes_from_api_head("template", template, template_size); + switch (function) { + case PKCS11_FUNCTION_GENERATE: + case PKCS11_FUNCTION_GENERATE_PAIR: + case PKCS11_FUNCTION_IMPORT: + case PKCS11_FUNCTION_MODIFY: + case PKCS11_FUNCTION_DERIVE: + case PKCS11_FUNCTION_UNWRAP: + case PKCS11_FUNCTION_COPY: + break; + default: + TEE_Panic(TEE_ERROR_NOT_SUPPORTED); + } +#endif + + /* + * For PKCS11_FUNCTION_GENERATE, find the class and type + * based on the mechanism. These will be passed as hint + * sanitize_client_object() and added in temp if not + * already present + */ + if (function == PKCS11_FUNCTION_GENERATE) { + switch (mecha) { + case PKCS11_CKM_GENERIC_SECRET_KEY_GEN: + class = PKCS11_CKO_SECRET_KEY; + type = PKCS11_CKK_GENERIC_SECRET; + break; + case PKCS11_CKM_AES_KEY_GEN: + class = PKCS11_CKO_SECRET_KEY; + type = PKCS11_CKK_AES; + break; + default: + TEE_Panic(TEE_ERROR_NOT_SUPPORTED); + } + } + + /* + * For PKCS11_FUNCTION_GENERATE_PAIR, find the class and type + * based on the mechanism. These will be passed as hint + * sanitize_client_object() and added in temp if not + * already present + */ + if (function == PKCS11_FUNCTION_GENERATE_PAIR) { + switch (mecha) { + case PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN: + class = template_class; + type = PKCS11_CKK_EDDSA; + break; + case PKCS11_CKM_EC_KEY_PAIR_GEN: + class = template_class; + type = PKCS11_CKK_EC; + break; + case PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN: + class = template_class; + type = PKCS11_CKK_RSA; + break; + default: + TEE_Panic(TEE_ERROR_NOT_SUPPORTED); + } + } + + /* + * Check and remove duplicates if any and create a new temporary + * template + */ + rc = sanitize_client_object(&temp, template, template_size, class, + type); + if (rc) + goto out; + + /* + * For function type modify and copy return the created template + * from here. Rest of the code below is for creating objects + * or generating keys. + */ + switch (function) { + case PKCS11_FUNCTION_MODIFY: + case PKCS11_FUNCTION_COPY: + *out = temp; + return rc; + case PKCS11_FUNCTION_DERIVE: + case PKCS11_FUNCTION_UNWRAP: + if (function == PKCS11_FUNCTION_UNWRAP) + indirect_template = PKCS11_CKA_UNWRAP_TEMPLATE; + else + indirect_template = PKCS11_CKA_DERIVE_TEMPLATE; + + rc = get_attribute_ptr(parent, indirect_template, + (void *)&req_attrs, &size); + if (rc == PKCS11_CKR_OK && size != 0) { + rc = attributes_match_add_reference(&temp, req_attrs); + if (rc) + goto out; + } + break; + default: + break; + } + + /* + * Check if class and type in temp are consistent with the mechanism + */ + switch (mecha) { + case PKCS11_CKM_GENERIC_SECRET_KEY_GEN: + if (get_class(temp) != PKCS11_CKO_SECRET_KEY || + get_key_type(temp) != PKCS11_CKK_GENERIC_SECRET) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + break; + case PKCS11_CKM_AES_KEY_GEN: + if (get_class(temp) != PKCS11_CKO_SECRET_KEY || + get_key_type(temp) != PKCS11_CKK_AES) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + break; + case PKCS11_CKM_EC_KEY_PAIR_GEN: + if ((get_class(temp) != PKCS11_CKO_PUBLIC_KEY && + get_class(temp) != PKCS11_CKO_PRIVATE_KEY) || + get_key_type(temp) != PKCS11_CKK_EC) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + break; + case PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN: + if ((get_class(temp) != PKCS11_CKO_PUBLIC_KEY && + get_class(temp) != PKCS11_CKO_PRIVATE_KEY) || + get_key_type(temp) != PKCS11_CKK_EC_EDWARDS) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + break; + case PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN: + if ((get_class(temp) != PKCS11_CKO_PUBLIC_KEY && + get_class(temp) != PKCS11_CKO_PRIVATE_KEY) || + get_key_type(temp) != PKCS11_CKK_RSA) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + break; + default: + break; + } + + if (!sanitize_consistent_class_and_type(temp)) { + EMSG("Inconsistent class/type"); + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + + /* + * TBD - Add a check to see if temp contains any attribute which + * is not consistent with the object class or type and return error. + * In current implementation such attributes are ignored and not + * added to final object while PKCS#11 specification expects a + * failure and an error code be returned. + */ + + switch (get_class(temp)) { + case PKCS11_CKO_DATA: + rc = create_data_attributes(&attrs, temp); + break; + case PKCS11_CKO_CERTIFICATE: + rc = create_certificate_attributes(&attrs, temp); + break; + case PKCS11_CKO_SECRET_KEY: + rc = sanitize_symm_key_attributes(&temp, function); + if (rc) + goto out; + rc = create_symm_key_attributes(&attrs, temp); + break; + case PKCS11_CKO_PUBLIC_KEY: + rc = create_pub_key_attributes(&attrs, temp, function); + if (rc) + goto out; + rc = create_pub_key_generated_attributes(&attrs, temp, + function); + break; + case PKCS11_CKO_PRIVATE_KEY: + rc = create_priv_key_attributes(&attrs, temp); + if (rc) + goto out; + rc = create_priv_key_hidden_attributes(&attrs, temp, function); + break; + default: + DMSG("Invalid object class %#"PRIx32"/%s", + get_class(temp), id2str_class(get_class(temp))); + + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + break; + } + if (rc) + goto out; + + if (get_attribute_ptr(temp, PKCS11_CKA_LOCAL, NULL, NULL) != + PKCS11_RV_NOT_FOUND) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + + if (get_attribute_ptr(temp, PKCS11_CKA_KEY_GEN_MECHANISM, NULL, NULL) != + PKCS11_RV_NOT_FOUND) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + + switch (function) { + case PKCS11_FUNCTION_GENERATE: + case PKCS11_FUNCTION_GENERATE_PAIR: + local = PKCS11_TRUE; + break; + case PKCS11_FUNCTION_IMPORT: + case PKCS11_FUNCTION_DERIVE: + case PKCS11_FUNCTION_UNWRAP: + default: + local = PKCS11_FALSE; + break; + } + rc = add_attribute(&attrs, PKCS11_CKA_LOCAL, &local, sizeof(local)); + if (rc) + goto out; + + switch (get_class(attrs)) { + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_PRIVATE_KEY: + case PKCS11_CKO_PUBLIC_KEY: + always_sensitive = PKCS11_FALSE; + never_extract = PKCS11_FALSE; + + switch (function) { + case PKCS11_FUNCTION_DERIVE: + always_sensitive = + get_bool(parent, PKCS11_CKA_ALWAYS_SENSITIVE) && + get_bool(attrs, PKCS11_CKA_SENSITIVE); + never_extract = + get_bool(parent, PKCS11_CKA_NEVER_EXTRACTABLE) && + !get_bool(attrs, PKCS11_CKA_EXTRACTABLE); + break; + case PKCS11_FUNCTION_UNWRAP: + always_sensitive = PKCS11_FALSE; + never_extract = PKCS11_FALSE; + extractable = PKCS11_TRUE; + + /* + * Check if template passed by user has CKA_EXTRACTABLE. + * If not, by default value of CKA_EXTRACTABLE is set as + * TRUE. + */ + if (get_attribute_ptr(temp, PKCS11_CKA_EXTRACTABLE, + NULL, + NULL) == PKCS11_RV_NOT_FOUND) { + rc = set_attribute(&attrs, + PKCS11_CKA_EXTRACTABLE, + &extractable, + sizeof(extractable)); + if (rc) + goto out; + } + break; + case PKCS11_FUNCTION_GENERATE: + case PKCS11_FUNCTION_GENERATE_PAIR: + always_sensitive = get_bool(attrs, + PKCS11_CKA_SENSITIVE); + never_extract = !get_bool(attrs, + PKCS11_CKA_EXTRACTABLE); + break; + default: + break; + } + + rc = add_attribute(&attrs, PKCS11_CKA_ALWAYS_SENSITIVE, + &always_sensitive, sizeof(always_sensitive)); + if (rc) + goto out; + + rc = add_attribute(&attrs, PKCS11_CKA_NEVER_EXTRACTABLE, + &never_extract, sizeof(never_extract)); + if (rc) + goto out; + + /* Keys mandate attribute PKCS11_CKA_KEY_GEN_MECHANISM */ + if (local) + mechanism_id = mecha; + else + mechanism_id = PKCS11_CK_UNAVAILABLE_INFORMATION; + + rc = add_attribute(&attrs, PKCS11_CKA_KEY_GEN_MECHANISM, + &mechanism_id, sizeof(mechanism_id)); + if (rc) + goto out; + break; + + default: + break; + } + + *out = attrs; + +#ifdef DEBUG + trace_attributes("object", attrs); +#endif + +out: + TEE_Free(temp); + if (rc) + TEE_Free(attrs); + + return rc; +} + +static enum pkcs11_rc check_attrs_misc_integrity(struct obj_attrs *head) +{ + if (get_bool(head, PKCS11_CKA_NEVER_EXTRACTABLE) && + get_bool(head, PKCS11_CKA_EXTRACTABLE)) { + DMSG("Never/Extractable attributes mismatch %d/%d", + get_bool(head, PKCS11_CKA_NEVER_EXTRACTABLE), + get_bool(head, PKCS11_CKA_EXTRACTABLE)); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + if (get_bool(head, PKCS11_CKA_ALWAYS_SENSITIVE) && + !get_bool(head, PKCS11_CKA_SENSITIVE)) { + DMSG("Sensitive/always attributes mismatch %d/%d", + get_bool(head, PKCS11_CKA_SENSITIVE), + get_bool(head, PKCS11_CKA_ALWAYS_SENSITIVE)); + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + return PKCS11_CKR_OK; +} + +bool object_is_private(struct obj_attrs *head) +{ + return get_bool(head, PKCS11_CKA_PRIVATE); +} + +bool object_is_token(struct obj_attrs *head) +{ + return get_bool(head, PKCS11_CKA_TOKEN); +} + +bool object_is_modifiable(struct obj_attrs *head) +{ + return get_bool(head, PKCS11_CKA_MODIFIABLE); +} + +bool object_is_copyable(struct obj_attrs *head) +{ + return get_bool(head, PKCS11_CKA_COPYABLE); +} + +/* + * Check access to object against authentication to token + */ +enum pkcs11_rc check_access_attrs_against_token(struct pkcs11_session *session, + struct obj_attrs *head) +{ + bool private = true; + + switch (get_class(head)) { + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_PRIVATE_KEY: + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_DATA: + case PKCS11_CKO_CERTIFICATE: + private = object_is_private(head); + break; + default: + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + + if (private && (pkcs11_session_is_public(session) || + pkcs11_session_is_so(session))) { + DMSG("Private object access from a public or SO session"); + + return PKCS11_CKR_USER_NOT_LOGGED_IN; + } + + return PKCS11_CKR_OK; +} + +/* + * Check the attributes of a to-be-created object matches the token state + */ +enum pkcs11_rc check_created_attrs_against_token(struct pkcs11_session *session, + struct obj_attrs *head) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + + rc = check_attrs_misc_integrity(head); + if (rc) + return rc; + + if (get_bool(head, PKCS11_CKA_TRUSTED) && + !pkcs11_session_is_so(session)) { + DMSG("Can't create trusted object"); + + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + + if (get_bool(head, PKCS11_CKA_TOKEN) && + !pkcs11_session_is_read_write(session)) { + DMSG("Can't create persistent object"); + + return PKCS11_CKR_SESSION_READ_ONLY; + } + + return PKCS11_CKR_OK; +} + +#define DMSG_BAD_BBOOL(attr, proc, head) \ + do { \ + uint32_t __maybe_unused _attr = (attr); \ + uint8_t __maybe_unused _bvalue = 0; \ + enum pkcs11_rc __maybe_unused _rc = PKCS11_CKR_OK; \ + \ + _rc = get_attribute((head), _attr, &_bvalue, NULL); \ + DMSG("%s issue for %s: %sfound, value %"PRIu8, \ + id2str_attr(_attr), id2str_proc((proc)), \ + _rc ? "not " : "", _bvalue); \ + } while (0) + +static bool __maybe_unused check_attr_bval(uint32_t proc_id __maybe_unused, + struct obj_attrs *head, + uint32_t attribute, bool val) +{ + uint8_t bbool = 0; + uint32_t sz = sizeof(bbool); + + if (!get_attribute(head, attribute, &bbool, &sz) && !!bbool == val) + return true; + + DMSG_BAD_BBOOL(attribute, proc_id, head); + return false; +} + +/* + * Check the attributes of a new secret match the processing/mechanism + * used to create it. + * + * @proc_id - PKCS11_CKM_xxx + * @head - head of the attributes of the to-be-created object. + */ +enum pkcs11_rc check_created_attrs_against_processing(uint32_t proc_id, + struct obj_attrs *head) +{ + /* + * Processings that do not create secrets are not expected to call + * this function which would panic. + */ + switch (proc_id) { + case PKCS11_PROCESSING_IMPORT: + case PKCS11_CKM_ECDH1_DERIVE: + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + case PKCS11_CKM_AES_ECB_ENCRYPT_DATA: + case PKCS11_CKM_AES_CBC_ENCRYPT_DATA: + case PKCS11_CKM_RSA_AES_KEY_WRAP: + assert(check_attr_bval(proc_id, head, PKCS11_CKA_LOCAL, false)); + break; + case PKCS11_CKM_GENERIC_SECRET_KEY_GEN: + case PKCS11_CKM_AES_KEY_GEN: + case PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN: + case PKCS11_CKM_EC_KEY_PAIR_GEN: + case PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN: + assert(check_attr_bval(proc_id, head, PKCS11_CKA_LOCAL, true)); + break; + default: + TEE_Panic(proc_id); + break; + } + + switch (proc_id) { + case PKCS11_CKM_GENERIC_SECRET_KEY_GEN: + assert(get_key_type(head) == PKCS11_CKK_GENERIC_SECRET); + break; + case PKCS11_CKM_AES_KEY_GEN: + assert(get_key_type(head) == PKCS11_CKK_AES); + break; + case PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN: + assert(get_key_type(head) == PKCS11_CKK_EC_EDWARDS); + break; + case PKCS11_CKM_EC_KEY_PAIR_GEN: + assert(get_key_type(head) == PKCS11_CKK_EC); + break; + case PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN: + assert(get_key_type(head) == PKCS11_CKK_RSA); + break; + case PKCS11_PROCESSING_IMPORT: + case PKCS11_CKM_ECDH1_DERIVE: + default: + break; + } + + return PKCS11_CKR_OK; +} + +/* Return min and max key size supported for a key_type in bytes */ +static void get_key_min_max_sizes(enum pkcs11_key_type key_type, + uint32_t *min_key_size, + uint32_t *max_key_size) +{ + enum pkcs11_mechanism_id mechanism = PKCS11_CKM_UNDEFINED_ID; + + switch (key_type) { + case PKCS11_CKK_GENERIC_SECRET: + mechanism = PKCS11_CKM_GENERIC_SECRET_KEY_GEN; + break; + case PKCS11_CKK_AES: + mechanism = PKCS11_CKM_AES_KEY_GEN; + break; + case PKCS11_CKK_MD5_HMAC: + mechanism = PKCS11_CKM_MD5_HMAC; + break; + case PKCS11_CKK_SHA_1_HMAC: + mechanism = PKCS11_CKM_SHA_1_HMAC; + break; + case PKCS11_CKK_SHA224_HMAC: + mechanism = PKCS11_CKM_SHA224_HMAC; + break; + case PKCS11_CKK_SHA256_HMAC: + mechanism = PKCS11_CKM_SHA256_HMAC; + break; + case PKCS11_CKK_SHA384_HMAC: + mechanism = PKCS11_CKM_SHA384_HMAC; + break; + case PKCS11_CKK_SHA512_HMAC: + mechanism = PKCS11_CKM_SHA512_HMAC; + break; + case PKCS11_CKK_EC: + mechanism = PKCS11_CKM_EC_KEY_PAIR_GEN; + break; + case PKCS11_CKK_EDDSA: + mechanism = PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN; + break; + case PKCS11_CKK_RSA: + mechanism = PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN; + break; + default: + TEE_Panic(key_type); + break; + } + + mechanism_supported_key_sizes_bytes(mechanism, min_key_size, + max_key_size); +} + +enum pkcs11_rc check_created_attrs(struct obj_attrs *key1, + struct obj_attrs *key2) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct obj_attrs *secret = NULL; + struct obj_attrs *private = NULL; + struct obj_attrs *public = NULL; + uint32_t max_key_size = 0; + uint32_t min_key_size = 0; + uint32_t key_length = 0; + + switch (get_class(key1)) { + case PKCS11_CKO_SECRET_KEY: + secret = key1; + break; + case PKCS11_CKO_PUBLIC_KEY: + public = key1; + break; + case PKCS11_CKO_PRIVATE_KEY: + private = key1; + break; + default: + return PKCS11_CKR_ATTRIBUTE_VALUE_INVALID; + } + + if (key2) { + switch (get_class(key2)) { + case PKCS11_CKO_PUBLIC_KEY: + public = key2; + if (private == key1) + break; + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + case PKCS11_CKO_PRIVATE_KEY: + private = key2; + if (public == key1) + break; + + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + default: + return PKCS11_CKR_ATTRIBUTE_VALUE_INVALID; + } + + if (get_key_type(private) != get_key_type(public)) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + if (secret) { + switch (get_key_type(secret)) { + case PKCS11_CKK_AES: + case PKCS11_CKK_GENERIC_SECRET: + case PKCS11_CKK_MD5_HMAC: + case PKCS11_CKK_SHA_1_HMAC: + case PKCS11_CKK_SHA224_HMAC: + case PKCS11_CKK_SHA256_HMAC: + case PKCS11_CKK_SHA384_HMAC: + case PKCS11_CKK_SHA512_HMAC: + break; + default: + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + /* Get key size */ + rc = get_u32_attribute(secret, PKCS11_CKA_VALUE_LEN, + &key_length); + if (rc) + return PKCS11_CKR_TEMPLATE_INCOMPLETE; + } + if (public) { + switch (get_key_type(public)) { + case PKCS11_CKK_RSA: + /* Get key size */ + rc = get_u32_attribute(public, PKCS11_CKA_MODULUS_BITS, + &key_length); + if (rc) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + key_length = ROUNDUP(key_length, 8) / 8; + break; + case PKCS11_CKK_EC: + case PKCS11_CKK_EC_EDWARDS: + break; + default: + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + } + if (private) { + switch (get_key_type(private)) { + case PKCS11_CKK_RSA: + case PKCS11_CKK_EC: + case PKCS11_CKK_EC_EDWARDS: + break; + default: + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + } + + /* + * Check key size for symmetric keys and RSA keys + * EC is bound to domains, no need to check here. + */ + switch (get_key_type(key1)) { + case PKCS11_CKK_EC: + case PKCS11_CKK_EC_EDWARDS: + return PKCS11_CKR_OK; + default: + break; + } + + get_key_min_max_sizes(get_key_type(key1), &min_key_size, &max_key_size); + if (key_length < min_key_size || key_length > max_key_size) { + EMSG("Length %"PRIu32" vs range [%"PRIu32" %"PRIu32"]", + key_length, min_key_size, max_key_size); + + return PKCS11_CKR_KEY_SIZE_RANGE; + } + + if (secret && get_key_type(secret) == PKCS11_CKK_AES) { + if (key_length != 16 && key_length != 24 && key_length != 32) + return PKCS11_CKR_KEY_SIZE_RANGE; + } + + return PKCS11_CKR_OK; +} + +/* Check processing ID against attribute ALLOWED_MECHANISMS if any */ +static bool parent_key_complies_allowed_processings(uint32_t proc_id, + struct obj_attrs *head) +{ + char *attr = NULL; + uint32_t size = 0; + uint32_t proc = 0; + size_t count = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + rc = get_attribute_ptr(head, PKCS11_CKA_ALLOWED_MECHANISMS, + (void *)&attr, &size); + if (rc == PKCS11_RV_NOT_FOUND) + return true; + if (rc) { + EMSG("unexpected attributes state"); + TEE_Panic(TEE_ERROR_BAD_STATE); + } + + for (count = size / sizeof(uint32_t); count; count--) { + TEE_MemMove(&proc, attr, sizeof(uint32_t)); + attr += sizeof(uint32_t); + + if (proc == proc_id) + return true; + } + + DMSG("can't find %s in allowed list", id2str_proc(proc_id)); + return false; +} + +static enum pkcs11_attr_id func_to_attr(enum processing_func func) +{ + switch (func) { + case PKCS11_FUNCTION_ENCRYPT: + return PKCS11_CKA_ENCRYPT; + case PKCS11_FUNCTION_DECRYPT: + return PKCS11_CKA_DECRYPT; + case PKCS11_FUNCTION_SIGN: + return PKCS11_CKA_SIGN; + case PKCS11_FUNCTION_VERIFY: + return PKCS11_CKA_VERIFY; + case PKCS11_FUNCTION_WRAP: + return PKCS11_CKA_WRAP; + case PKCS11_FUNCTION_UNWRAP: + return PKCS11_CKA_UNWRAP; + case PKCS11_FUNCTION_DERIVE: + return PKCS11_CKA_DERIVE; + default: + return PKCS11_CKA_UNDEFINED_ID; + } +} + +enum pkcs11_rc +check_parent_attrs_against_processing(enum pkcs11_mechanism_id proc_id, + enum processing_func function, + struct obj_attrs *head) +{ + enum pkcs11_class_id key_class = get_class(head); + enum pkcs11_key_type key_type = get_key_type(head); + enum pkcs11_attr_id attr = func_to_attr(function); + + if (!get_bool(head, attr)) { + DMSG("%s not permitted", id2str_attr(attr)); + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + + /* Check processing complies with parent key family */ + switch (proc_id) { + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + case PKCS11_CKM_AES_CTS: + case PKCS11_CKM_AES_CTR: + case PKCS11_CKM_AES_CMAC: + case PKCS11_CKM_AES_CMAC_GENERAL: + if (key_class == PKCS11_CKO_SECRET_KEY && + key_type == PKCS11_CKK_AES) + break; + + DMSG("%s invalid key %s/%s", id2str_proc(proc_id), + id2str_class(key_class), id2str_key_type(key_type)); + + if (function == PKCS11_FUNCTION_WRAP) + return PKCS11_CKR_WRAPPING_KEY_TYPE_INCONSISTENT; + else if (function == PKCS11_FUNCTION_UNWRAP) + return PKCS11_CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT; + else + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + + case PKCS11_CKM_AES_ECB_ENCRYPT_DATA: + case PKCS11_CKM_AES_CBC_ENCRYPT_DATA: + if (key_class != PKCS11_CKO_SECRET_KEY && + key_type != PKCS11_CKK_AES) + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + + if (get_bool(head, PKCS11_CKA_ENCRYPT)) { + /* + * Intentionally refuse to proceed despite + * PKCS#11 specifications v2.40 and v3.0 not expecting + * this behavior to avoid potential security issue + * where keys derived by these mechanisms can be + * revealed by doing data encryption using parent key. + */ + return PKCS11_CKR_FUNCTION_FAILED; + } + + break; + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA512_HMAC: + case PKCS11_CKM_MD5_HMAC_GENERAL: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + if (key_class != PKCS11_CKO_SECRET_KEY) + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + + if (key_type == PKCS11_CKK_GENERIC_SECRET) + break; + + switch (proc_id) { + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_MD5_HMAC_GENERAL: + if (key_type == PKCS11_CKK_MD5_HMAC) + break; + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + if (key_type == PKCS11_CKK_SHA_1_HMAC) + break; + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + if (key_type == PKCS11_CKK_SHA224_HMAC) + break; + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + if (key_type == PKCS11_CKK_SHA256_HMAC) + break; + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + if (key_type == PKCS11_CKK_SHA384_HMAC) + break; + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + case PKCS11_CKM_SHA512_HMAC: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + if (key_type == PKCS11_CKK_SHA512_HMAC) + break; + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + default: + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + break; + + case PKCS11_CKM_EDDSA: + if (key_type != PKCS11_CKK_EC_EDWARDS) { + EMSG("Invalid key %s for mechanism %s", + id2str_type(key_type, key_class), + id2str_proc(proc_id)); + return PKCS11_CKR_KEY_TYPE_INCONSISTENT; + } + if (key_class != PKCS11_CKO_PUBLIC_KEY && + key_class != PKCS11_CKO_PRIVATE_KEY) { + EMSG("Invalid key class for mechanism %s", + id2str_proc(proc_id)); + + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + break; + + case PKCS11_CKM_ECDSA: + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + case PKCS11_CKM_ECDH1_DERIVE: + if (key_type != PKCS11_CKK_EC) { + EMSG("Invalid key %s for mechanism %s", + id2str_type(key_type, key_class), + id2str_proc(proc_id)); + + return PKCS11_CKR_KEY_TYPE_INCONSISTENT; + } + if (key_class != PKCS11_CKO_PUBLIC_KEY && + key_class != PKCS11_CKO_PRIVATE_KEY) { + EMSG("Invalid key class for mechanism %s", + id2str_proc(proc_id)); + + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + break; + case PKCS11_CKM_RSA_PKCS: + case PKCS11_CKM_MD5_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS: + case PKCS11_CKM_SHA224_RSA_PKCS: + case PKCS11_CKM_SHA256_RSA_PKCS: + case PKCS11_CKM_SHA384_RSA_PKCS: + case PKCS11_CKM_SHA512_RSA_PKCS: + case PKCS11_CKM_RSA_AES_KEY_WRAP: + case PKCS11_CKM_RSA_PKCS_OAEP: + case PKCS11_CKM_RSA_PKCS_PSS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + if (key_type != PKCS11_CKK_RSA) { + EMSG("Invalid key %s for mechanism %s", + id2str_type(key_type, key_class), + id2str_proc(proc_id)); + + return PKCS11_CKR_KEY_TYPE_INCONSISTENT; + } + if (key_class != PKCS11_CKO_PUBLIC_KEY && + key_class != PKCS11_CKO_PRIVATE_KEY) { + EMSG("Invalid key class for mechanism %s", + id2str_proc(proc_id)); + + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + break; + default: + DMSG("Invalid processing %#"PRIx32"/%s", proc_id, + id2str_proc(proc_id)); + + return PKCS11_CKR_MECHANISM_INVALID; + } + + if (!parent_key_complies_allowed_processings(proc_id, head)) { + DMSG("Allowed mechanism failed"); + return PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED; + } + + return PKCS11_CKR_OK; +} + +bool attribute_is_exportable(struct pkcs11_attribute_head *req_attr, + struct pkcs11_object *obj) +{ + uint8_t boolval = 0; + uint32_t boolsize = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + enum pkcs11_class_id key_class = get_class(obj->attributes); + + if (attribute_is_hidden(req_attr)) + return false; + + if (key_class != PKCS11_CKO_SECRET_KEY && + key_class != PKCS11_CKO_PRIVATE_KEY) + return true; + + switch (req_attr->id) { + case PKCS11_CKA_PRIVATE_EXPONENT: + case PKCS11_CKA_PRIME_1: + case PKCS11_CKA_PRIME_2: + case PKCS11_CKA_EXPONENT_1: + case PKCS11_CKA_EXPONENT_2: + case PKCS11_CKA_COEFFICIENT: + case PKCS11_CKA_VALUE: + boolsize = sizeof(boolval); + rc = get_attribute(obj->attributes, PKCS11_CKA_EXTRACTABLE, + &boolval, &boolsize); + if (rc || boolval == PKCS11_FALSE) + return false; + + boolsize = sizeof(boolval); + rc = get_attribute(obj->attributes, PKCS11_CKA_SENSITIVE, + &boolval, &boolsize); + if (rc || boolval == PKCS11_TRUE) + return false; + break; + default: + break; + } + + return true; +} + +static bool attr_is_modifiable_any_key(struct pkcs11_attribute_head *attr) +{ + switch (attr->id) { + case PKCS11_CKA_ID: + case PKCS11_CKA_START_DATE: + case PKCS11_CKA_END_DATE: + case PKCS11_CKA_DERIVE: + return true; + default: + return false; + } +} + +static bool attr_is_modifiable_secret_key(struct pkcs11_attribute_head *attr, + struct pkcs11_session *session, + struct pkcs11_object *obj) +{ + switch (attr->id) { + case PKCS11_CKA_ENCRYPT: + case PKCS11_CKA_DECRYPT: + case PKCS11_CKA_SIGN: + case PKCS11_CKA_VERIFY: + case PKCS11_CKA_WRAP: + case PKCS11_CKA_UNWRAP: + return true; + /* Can't be modified once set to CK_FALSE - 12 in Table 10 */ + case PKCS11_CKA_EXTRACTABLE: + return get_bool(obj->attributes, attr->id); + /* Can't be modified once set to CK_TRUE - 11 in Table 10 */ + case PKCS11_CKA_SENSITIVE: + case PKCS11_CKA_WRAP_WITH_TRUSTED: + return !get_bool(obj->attributes, attr->id); + /* Change in CKA_TRUSTED can only be done by SO */ + case PKCS11_CKA_TRUSTED: + return pkcs11_session_is_so(session); + case PKCS11_CKA_NEVER_EXTRACTABLE: + case PKCS11_CKA_ALWAYS_SENSITIVE: + return false; + default: + return false; + } +} + +static bool attr_is_modifiable_public_key(struct pkcs11_attribute_head *attr, + struct pkcs11_session *session, + struct pkcs11_object *obj __unused) +{ + switch (attr->id) { + case PKCS11_CKA_SUBJECT: + case PKCS11_CKA_ENCRYPT: + case PKCS11_CKA_VERIFY: + case PKCS11_CKA_VERIFY_RECOVER: + case PKCS11_CKA_WRAP: + return true; + case PKCS11_CKA_TRUSTED: + /* Change in CKA_TRUSTED can only be done by SO */ + return pkcs11_session_is_so(session); + default: + return false; + } +} + +static bool attr_is_modifiable_private_key(struct pkcs11_attribute_head *attr, + struct pkcs11_session *sess __unused, + struct pkcs11_object *obj) +{ + switch (attr->id) { + case PKCS11_CKA_SUBJECT: + case PKCS11_CKA_DECRYPT: + case PKCS11_CKA_SIGN: + case PKCS11_CKA_SIGN_RECOVER: + case PKCS11_CKA_UNWRAP: + /* + * TBD: Revisit if we don't support PKCS11_CKA_PUBLIC_KEY_INFO + * Specification mentions that if this attribute is + * supplied as part of a template for C_CreateObject, C_CopyObject or + * C_SetAttributeValue for a private key, the token MUST verify + * correspondence between the private key data and the public key data + * as supplied in CKA_PUBLIC_KEY_INFO. This needs to be + * taken care of when this object type will be implemented + */ + case PKCS11_CKA_PUBLIC_KEY_INFO: + return true; + /* Can't be modified once set to CK_FALSE - 12 in Table 10 */ + case PKCS11_CKA_EXTRACTABLE: + return get_bool(obj->attributes, attr->id); + /* Can't be modified once set to CK_TRUE - 11 in Table 10 */ + case PKCS11_CKA_SENSITIVE: + case PKCS11_CKA_WRAP_WITH_TRUSTED: + return !get_bool(obj->attributes, attr->id); + case PKCS11_CKA_NEVER_EXTRACTABLE: + case PKCS11_CKA_ALWAYS_SENSITIVE: + return false; + default: + return false; + } +} + +static bool attr_is_modifiable_certificate(struct pkcs11_attribute_head *attr, + struct pkcs11_session *session, + struct pkcs11_object *obj) +{ + uint8_t boolval = 0; + uint32_t boolsize = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + /* Trusted certificates cannot be modified. */ + rc = get_attribute(obj->attributes, PKCS11_CKA_TRUSTED, + &boolval, &boolsize); + if (rc || boolval == PKCS11_TRUE) + return false; + + /* Common certificate attributes */ + switch (attr->id) { + case PKCS11_CKA_TRUSTED: + /* + * The CKA_TRUSTED attribute cannot be set to CK_TRUE by an + * application. It MUST be set by a token initialization + * application or by the token’s SO. + */ + return pkcs11_session_is_so(session); + case PKCS11_CKA_CERTIFICATE_TYPE: + case PKCS11_CKA_CERTIFICATE_CATEGORY: + return false; + default: + break; + } + + /* Certificate type specific attributes */ + switch (get_certificate_type(obj->attributes)) { + case PKCS11_CKC_X_509: + /* + * Only the CKA_ID, CKA_ISSUER, and CKA_SERIAL_NUMBER + * attributes may be modified after the object is created. + */ + switch (attr->id) { + case PKCS11_CKA_ID: + case PKCS11_CKA_ISSUER: + case PKCS11_CKA_SERIAL_NUMBER: + return true; + default: + break; + } + break; + default: + /* Unsupported certificate type */ + break; + } + + return false; +} + +static bool attribute_is_modifiable(struct pkcs11_session *session, + struct pkcs11_attribute_head *req_attr, + struct pkcs11_object *obj, + enum pkcs11_class_id class, + enum processing_func function) +{ + /* Check modifiable attributes common to any object */ + switch (req_attr->id) { + case PKCS11_CKA_LABEL: + return true; + case PKCS11_CKA_TOKEN: + case PKCS11_CKA_MODIFIABLE: + case PKCS11_CKA_DESTROYABLE: + case PKCS11_CKA_PRIVATE: + return function == PKCS11_FUNCTION_COPY; + case PKCS11_CKA_COPYABLE: + /* + * Specification mentions that if the attribute value is false + * it can't be set to true. Reading this we assume that it + * should be possible to modify this attribute even though this + * is not marked as modifiable in Table 10 if done in right + * direction i.e from TRUE -> FALSE. + */ + return get_bool(obj->attributes, req_attr->id); + default: + break; + } + + /* Attribute checking based on class type */ + switch (class) { + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_PRIVATE_KEY: + if (attr_is_modifiable_any_key(req_attr)) + return true; + if (class == PKCS11_CKO_SECRET_KEY && + attr_is_modifiable_secret_key(req_attr, session, obj)) + return true; + if (class == PKCS11_CKO_PUBLIC_KEY && + attr_is_modifiable_public_key(req_attr, session, obj)) + return true; + if (class == PKCS11_CKO_PRIVATE_KEY && + attr_is_modifiable_private_key(req_attr, session, obj)) + return true; + break; + case PKCS11_CKO_DATA: + /* None of the data object attributes are modifiable */ + return false; + case PKCS11_CKO_CERTIFICATE: + return attr_is_modifiable_certificate(req_attr, session, obj); + default: + break; + } + + return false; +} + +enum pkcs11_rc check_attrs_against_modification(struct pkcs11_session *session, + struct obj_attrs *head, + struct pkcs11_object *obj, + enum processing_func function) +{ + enum pkcs11_class_id class = PKCS11_CKO_UNDEFINED_ID; + char *cur = NULL; + char *end = NULL; + size_t len = 0; + + class = get_class(obj->attributes); + + cur = (char *)head + sizeof(struct obj_attrs); + end = cur + head->attrs_size; + + for (; cur < end; cur += len) { + /* Structure aligned copy of the pkcs11_ref in the object */ + struct pkcs11_attribute_head cli_ref = { }; + + TEE_MemMove(&cli_ref, cur, sizeof(cli_ref)); + len = sizeof(cli_ref) + cli_ref.size; + + /* Protect hidden attributes */ + if (attribute_is_hidden(&cli_ref)) + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + + /* + * Check 1 - Check if attribute belongs to the object + * The obj->attributes has all the attributes in + * it which are allowed for an object. + */ + if (get_attribute_ptr(obj->attributes, cli_ref.id, NULL, + NULL) == PKCS11_RV_NOT_FOUND) + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + + /* Check 2 - Is attribute modifiable */ + if (!attribute_is_modifiable(session, &cli_ref, obj, class, + function)) + return PKCS11_CKR_ATTRIBUTE_READ_ONLY; + + /* + * Checks for modification in PKCS11_CKA_TOKEN and + * PKCS11_CKA_PRIVATE are required for PKCS11_FUNCTION_COPY + * only, so skip them for PKCS11_FUNCTION_MODIFY. + */ + if (function == PKCS11_FUNCTION_MODIFY) + continue; + + /* + * An attempt to copy an object to a token will fail for + * RO session + */ + if (cli_ref.id == PKCS11_CKA_TOKEN && + get_bool(head, PKCS11_CKA_TOKEN)) { + if (!pkcs11_session_is_read_write(session)) { + DMSG("Can't copy to token in a RO session"); + return PKCS11_CKR_SESSION_READ_ONLY; + } + } + + if (cli_ref.id == PKCS11_CKA_PRIVATE) { + bool parent_priv = + get_bool(obj->attributes, cli_ref.id); + bool obj_priv = get_bool(head, cli_ref.id); + + /* + * If PKCS11_CKA_PRIVATE is being set to TRUE from + * FALSE, user has to be logged in + */ + if (!parent_priv && obj_priv) { + if ((pkcs11_session_is_public(session) || + pkcs11_session_is_so(session))) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + } + + /* + * Restriction added - Even for Copy, do not allow + * modification of CKA_PRIVATE from TRUE to FALSE + */ + if (parent_priv && !obj_priv) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + } + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc set_secret_key_data(struct obj_attrs **head, void *data, + size_t key_size) +{ + uint32_t size = sizeof(uint32_t); + uint32_t key_length = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + /* Get key size if present in template */ + rc = get_attribute(*head, PKCS11_CKA_VALUE_LEN, &key_length, &size); + if (rc && rc != PKCS11_RV_NOT_FOUND) + return rc; + + if (key_length) { + if (key_size < key_length) + return PKCS11_CKR_DATA_LEN_RANGE; + } else { + key_length = key_size; + rc = set_attribute(head, PKCS11_CKA_VALUE_LEN, &key_length, + sizeof(uint32_t)); + if (rc) + return rc; + } + + /* Now we can check the VALUE_LEN field */ + rc = check_created_attrs(*head, NULL); + if (rc) + return rc; + + /* Remove the default empty value attribute if found */ + rc = remove_empty_attribute(head, PKCS11_CKA_VALUE); + if (rc != PKCS11_CKR_OK && rc != PKCS11_RV_NOT_FOUND) + return PKCS11_CKR_GENERAL_ERROR; + + return add_attribute(head, PKCS11_CKA_VALUE, data, key_length); +} + +static enum pkcs11_rc set_private_key_data_rsa(struct obj_attrs **head, + void *data, + size_t key_size) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + int mbedtls_rc = 0; + uint32_t key_bits = 0; + uint32_t size = 0; + uint32_t buffer_size = 0; + void *buffer = NULL; + mbedtls_pk_context pk = { }; + mbedtls_rsa_context *rsa = NULL; + mbedtls_mpi n = { }; + mbedtls_mpi e = { }; + mbedtls_mpi d = { }; + mbedtls_mpi p = { }; + mbedtls_mpi q = { }; + + rc = get_u32_attribute(*head, PKCS11_CKA_MODULUS_BITS, &key_bits); + if (rc && rc != PKCS11_RV_NOT_FOUND) + return rc; + + if (remove_empty_attribute(head, PKCS11_CKA_MODULUS) || + remove_empty_attribute(head, PKCS11_CKA_PUBLIC_EXPONENT) || + remove_empty_attribute(head, PKCS11_CKA_PRIVATE_EXPONENT) || + remove_empty_attribute(head, PKCS11_CKA_PRIME_1) || + remove_empty_attribute(head, PKCS11_CKA_PRIME_2)) + return PKCS11_CKR_GENERAL_ERROR; + + mbedtls_pk_init(&pk); + mbedtls_mpi_init(&n); + mbedtls_mpi_init(&e); + mbedtls_mpi_init(&d); + mbedtls_mpi_init(&p); + mbedtls_mpi_init(&q); + + mbedtls_rc = mbedtls_pk_parse_key(&pk, data, key_size, + NULL, 0, mbd_rand, NULL); + if (mbedtls_rc) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + rsa = mbedtls_pk_rsa(pk); + mbedtls_rc = mbedtls_rsa_export(rsa, &n, &p, &q, &d, &e); + if (mbedtls_rc) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + if (key_bits && mbedtls_mpi_bitlen(&n) != key_bits) { + rc = PKCS11_CKR_WRAPPED_KEY_LEN_RANGE; + goto out; + } + + size = ROUNDUP_DIV(mbedtls_mpi_bitlen(&n), 8); + buffer_size = size; + buffer = TEE_Malloc(buffer_size, TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!buffer) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + mbedtls_rc = mbedtls_mpi_write_binary(&n, buffer, size); + if (mbedtls_rc) { + rc = PKCS11_CKR_WRAPPED_KEY_INVALID; + goto out; + } + + rc = add_attribute(head, PKCS11_CKA_MODULUS, buffer, size); + if (rc) + goto out; + + size = ROUNDUP_DIV(mbedtls_mpi_bitlen(&e), 8); + if (buffer_size < size) { + rc = PKCS11_CKR_WRAPPED_KEY_LEN_RANGE; + goto out; + } + + mbedtls_rc = mbedtls_mpi_write_binary(&e, buffer, size); + if (mbedtls_rc) { + rc = PKCS11_CKR_WRAPPED_KEY_INVALID; + goto out; + } + + rc = add_attribute(head, PKCS11_CKA_PUBLIC_EXPONENT, buffer, size); + if (rc) + goto out; + + size = ROUNDUP_DIV(mbedtls_mpi_bitlen(&d), 8); + if (buffer_size < size) { + rc = PKCS11_CKR_WRAPPED_KEY_LEN_RANGE; + goto out; + } + + mbedtls_rc = mbedtls_mpi_write_binary(&d, buffer, size); + if (mbedtls_rc) { + rc = PKCS11_CKR_WRAPPED_KEY_INVALID; + goto out; + } + + rc = add_attribute(head, PKCS11_CKA_PRIVATE_EXPONENT, buffer, size); + if (rc) + goto out; + + size = ROUNDUP_DIV(mbedtls_mpi_bitlen(&p), 8); + if (buffer_size < size) { + rc = PKCS11_CKR_WRAPPED_KEY_LEN_RANGE; + goto out; + } + + mbedtls_rc = mbedtls_mpi_write_binary(&p, buffer, size); + if (mbedtls_rc) { + rc = PKCS11_CKR_WRAPPED_KEY_INVALID; + goto out; + } + + rc = add_attribute(head, PKCS11_CKA_PRIME_1, buffer, size); + if (rc) + goto out; + + size = ROUNDUP_DIV(mbedtls_mpi_bitlen(&q), 8); + if (buffer_size < size) { + rc = PKCS11_CKR_WRAPPED_KEY_LEN_RANGE; + goto out; + } + + mbedtls_rc = mbedtls_mpi_write_binary(&q, buffer, size); + if (mbedtls_rc) { + rc = PKCS11_CKR_WRAPPED_KEY_INVALID; + goto out; + } + + rc = add_attribute(head, PKCS11_CKA_PRIME_2, buffer, size); + +out: + mbedtls_pk_free(&pk); + mbedtls_mpi_free(&n); + mbedtls_mpi_free(&e); + mbedtls_mpi_free(&d); + mbedtls_mpi_free(&p); + mbedtls_mpi_free(&q); + TEE_Free(buffer); + return rc; +} + +enum pkcs11_rc set_key_data(struct obj_attrs **head, void *data, + size_t key_size) +{ + switch (get_class(*head)) { + case PKCS11_CKO_SECRET_KEY: + return set_secret_key_data(head, data, key_size); + case PKCS11_CKO_PRIVATE_KEY: + if (get_key_type(*head) == PKCS11_CKK_RSA) + return set_private_key_data_rsa(head, data, key_size); + break; + default: + return PKCS11_CKR_GENERAL_ERROR; + } + + return PKCS11_CKR_GENERAL_ERROR; +} + +static enum pkcs11_rc alloc_copy_attribute_value(struct obj_attrs *head, + void **data, uint32_t *sz) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *buffer = NULL; + void *value = NULL; + + rc = get_attribute_ptr(head, PKCS11_CKA_VALUE, &value, sz); + if (rc) + return PKCS11_CKR_ARGUMENTS_BAD; + + buffer = TEE_Malloc(*sz, TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!buffer) + return PKCS11_CKR_DEVICE_MEMORY; + + TEE_MemMove(buffer, value, *sz); + *data = buffer; + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc +encode_rsa_private_key_der(struct obj_attrs *head, void **data, uint32_t *sz) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + int i = 0; + int mbedtls_rc = 0; + int start = 0; + int der_size = 0; + void *n = NULL; + void *p = NULL; + void *q = NULL; + void *d = NULL; + void *e = NULL; + uint32_t n_len = 0; + uint32_t p_len = 0; + uint32_t q_len = 0; + uint32_t d_len = 0; + uint32_t e_len = 0; + uint8_t *buffer = NULL; + mbedtls_pk_context pk = { }; + mbedtls_rsa_context *rsa = NULL; + const mbedtls_pk_info_t *pk_info = NULL; + + mbedtls_pk_init(&pk); + pk_info = mbedtls_pk_info_from_type(MBEDTLS_PK_RSA); + if (mbedtls_pk_setup(&pk, pk_info)) { + rc = PKCS11_CKR_GENERAL_ERROR; + goto out; + } + + rc = get_attribute_ptr(head, PKCS11_CKA_MODULUS, &n, &n_len); + if (rc) + goto out; + + rc = get_attribute_ptr(head, PKCS11_CKA_PRIME_1, &p, &p_len); + if (rc) + goto out; + + rc = get_attribute_ptr(head, PKCS11_CKA_PRIME_2, &q, &q_len); + if (rc) + goto out; + + rc = get_attribute_ptr(head, PKCS11_CKA_PRIVATE_EXPONENT, &d, &d_len); + if (rc) + goto out; + + rc = get_attribute_ptr(head, PKCS11_CKA_PUBLIC_EXPONENT, &e, &e_len); + if (rc) + goto out; + + rsa = mbedtls_pk_rsa(pk); + mbedtls_rc = mbedtls_rsa_import_raw(rsa, n, n_len, p, p_len, + q, q_len, d, d_len, e, e_len); + if (mbedtls_rc) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + if (mbedtls_rsa_complete(rsa)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + if (mbedtls_rsa_check_privkey(rsa)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + der_size = n_len * 8; + buffer = TEE_Malloc(der_size, TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!buffer) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + mbedtls_rc = mbedtls_pk_write_key_der(&pk, buffer, der_size); + if (mbedtls_rc < 0) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + start = der_size - mbedtls_rc; + for (i = 0; i < mbedtls_rc; i++) { + buffer[i] = buffer[i + start]; + buffer[i + start] = 0; + } + + *data = buffer; + *sz = mbedtls_rc; +out: + mbedtls_pk_free(&pk); + + if (rc) + TEE_Free(buffer); + + return rc; +} + +enum pkcs11_rc alloc_key_data_to_wrap(struct obj_attrs *head, void **data, + uint32_t *sz) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + switch (get_class(head)) { + case PKCS11_CKO_SECRET_KEY: + rc = alloc_copy_attribute_value(head, data, sz); + break; + case PKCS11_CKO_PRIVATE_KEY: + if (get_key_type(head) == PKCS11_CKK_RSA) + rc = encode_rsa_private_key_der(head, data, sz); + break; + default: + break; + } + + return rc; +} + +enum pkcs11_rc add_missing_attribute_id(struct obj_attrs **pub_head, + struct obj_attrs **priv_head) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *id1 = NULL; + uint32_t id1_size = 0; + void *id2 = NULL; + uint32_t id2_size = 0; + + assert(pub_head); + assert(priv_head); + + rc = get_attribute_ptr(*pub_head, PKCS11_CKA_ID, &id1, &id1_size); + if (rc) { + if (rc != PKCS11_RV_NOT_FOUND) + return rc; + id1 = NULL; + } else if (!id1_size) { + id1 = NULL; + } + + rc = get_attribute_ptr(*priv_head, PKCS11_CKA_ID, &id2, &id2_size); + if (rc) { + if (rc != PKCS11_RV_NOT_FOUND) + return rc; + id2 = NULL; + } else if (!id2_size) { + id2 = NULL; + } + + /* Both have value -- let them be what caller has specified them */ + if (id1 && id2) + return PKCS11_CKR_OK; + + /* Both are empty -- leave empty values */ + if (!id1 && !id2) + return PKCS11_CKR_OK; + + /* Cross copy CKA_ID value */ + if (id1) + return set_attribute(priv_head, PKCS11_CKA_ID, id1, id1_size); + else + return set_attribute(pub_head, PKCS11_CKA_ID, id2, id2_size); +} diff --git a/optee_os/ta/pkcs11/src/pkcs11_attributes.h b/optee_os/ta/pkcs11/src/pkcs11_attributes.h new file mode 100644 index 0000000..1834b43 --- /dev/null +++ b/optee_os/ta/pkcs11/src/pkcs11_attributes.h @@ -0,0 +1,218 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_PKCS11_ATTRIBUTES_H +#define PKCS11_TA_PKCS11_ATTRIBUTES_H + +#include +#include + +#include "serializer.h" + +struct obj_attrs; +struct pkcs11_object; +struct pkcs11_session; + +/* + * PKCS#11 directives on object attributes. + * Those with a '*' are optional, other must be defined, either by caller + * or by some known default value. + * + * [all] objects: class + * + * [stored] objects: persistent, need_authen, modifiable, copyable, + * destroyable, label*. + * + * [data] objects: [all], [stored], application_id*, object_id*, value. + * + * [key] objects: [all], [stored], type, id*, start_date/end_date*, + * derive, local, allowed_mechanisms*. + * + * [symm-key]: [key], sensitive, encrypt, decrypt, sign, verify, wrap, + * unwrap, extractable, wrap_with_trusted, trusted, + * wrap_template, unwrap_template, derive_template. + */ + +/* + * Utils to check compliance of attributes at various processing steps. + * Any processing operation is exclusively one of the following. + * + * Case 1: Create a secret from some local random value (C_CreateKey & friends) + * - client provides an attributes list template, PKCS11 TA completes with + * default attribute values. Object is created if attributes are + * consistent and comply token/session state. + * - PKCS11 sequence: + * - check/set token/session state + * - create an attribute list from client template and default values. + * - check new secret attributes complies requested mechanism. + * - check new secret attributes complies token/session state. + * - Generate the value for the secret. + * - Set some runtime attributes in the new secret. + * - Register the new secret and return a handle for it. + * + * Case 2: Create a secret from a client clear data (C_CreateObject) + * - client provides an attributes list template, PKCS11 TA completes with + * default attribute values. Object is created if attributes are + * consistent and comply token/session state. + * - check/set token/session state + * - create an attribute list from client template and default values. + * - check new secret attributes complies requested mechanism (raw-import). + * - check new secret attributes complies token/session state. + * - Set some runtime attributes in the new secret. + * - Register the new secret and return a handle for it. + + * Case 3: Use a secret for data processing + * - client provides a mechanism ID and the secret handle. + * - PKCS11 checks mechanism and secret comply, if mechanism and token/session + * state comply and last if secret and token/session state comply. + * - check/set token/session state + * - check secret's parent attributes complies requested processing. + * - check secret's parent attributes complies token/session state. + * - check new secret attributes complies secret's parent attributes. + * - check new secret attributes complies requested mechanism. + * - check new secret attributes complies token/session state. + * + * Case 4: Create a secret from a client template and a secret's parent + * (i.e derive a symmetric key) + * - client args: new-key template, mechanism ID, parent-key handle. + * - PKCS11 create a new-key attribute list based on template + default values + + * inheritance from the parent key attributes. + * - PKCS11 checks: + * - token/session state + * - parent-key vs mechanism + * - parent-key vs token/session state + * - parent-key vs new-key + * - new-key vs mechanism + * - new-key vs token/session state + * - then do processing + * - then finalize object creation + */ + +enum processing_func { + PKCS11_FUNCTION_DIGEST, + PKCS11_FUNCTION_GENERATE, + PKCS11_FUNCTION_GENERATE_PAIR, + PKCS11_FUNCTION_DERIVE, + PKCS11_FUNCTION_WRAP, + PKCS11_FUNCTION_UNWRAP, + PKCS11_FUNCTION_ENCRYPT, + PKCS11_FUNCTION_DECRYPT, + PKCS11_FUNCTION_SIGN, + PKCS11_FUNCTION_VERIFY, + PKCS11_FUNCTION_SIGN_RECOVER, + PKCS11_FUNCTION_VERIFY_RECOVER, + PKCS11_FUNCTION_IMPORT, + PKCS11_FUNCTION_COPY, + PKCS11_FUNCTION_MODIFY, + PKCS11_FUNCTION_DESTROY, + PKCS11_FUNCTION_UNKNOWN, +}; + +enum processing_step { + PKCS11_FUNC_STEP_INIT, + PKCS11_FUNC_STEP_ONESHOT, + PKCS11_FUNC_STEP_UPDATE, + PKCS11_FUNC_STEP_UPDATE_KEY, + PKCS11_FUNC_STEP_FINAL, +}; + +/* Create an attribute list for a new object */ +enum pkcs11_rc +create_attributes_from_template(struct obj_attrs **out, void *template, + size_t template_size, struct obj_attrs *parent, + enum processing_func func, + enum pkcs11_mechanism_id proc_mecha, + enum pkcs11_class_id template_class); + +/* + * The various checks to be performed before a processing: + * - create a new object in the current token state + * - use a parent object in the processing + * - use a mechanism with provided configuration + */ +enum pkcs11_rc check_created_attrs_against_token(struct pkcs11_session *session, + struct obj_attrs *head); + +enum pkcs11_rc check_created_attrs_against_processing(uint32_t proc_id, + struct obj_attrs *head); + +enum pkcs11_rc check_created_attrs(struct obj_attrs *key1, + struct obj_attrs *key2); + +/* + * Check the attributes of the parent secret (key) used in the processing + * do match the target processing. + * + * @proc_id - PKCS11_CKM_xxx + * @func - identifier of the processing function operated with @proc_id. + * @head - head of the attributes of parent object. + */ +enum pkcs11_rc +check_parent_attrs_against_processing(enum pkcs11_mechanism_id proc_id, + enum processing_func func, + struct obj_attrs *head); + +enum pkcs11_rc check_access_attrs_against_token(struct pkcs11_session *session, + struct obj_attrs *head); + +enum pkcs11_rc +check_mechanism_against_processing(struct pkcs11_session *session, + enum pkcs11_mechanism_id mechanism_type, + enum processing_func function, + enum processing_step step); + +static inline bool attribute_is_hidden(struct pkcs11_attribute_head *req_attr) +{ + return (req_attr->id & PKCS11_CKA_OPTEE_FLAGS_HIDDEN) == + PKCS11_CKA_OPTEE_FLAGS_HIDDEN; +} + +bool attribute_is_exportable(struct pkcs11_attribute_head *req_attr, + struct pkcs11_object *obj); + +bool object_is_private(struct obj_attrs *head); + +bool object_is_token(struct obj_attrs *head); + +bool object_is_modifiable(struct obj_attrs *head); + +bool object_is_copyable(struct obj_attrs *head); + +/* + * Check the attributes passed in template against the attributes which can be + * modified. These are the attributes marked with * 8,10,11 or 12 in Table 10 + * in PKCS #11 Cryptographic Token InterfaceBase Specification Version 2.40. + * Few attributes not with this marking but explicitly specified as modifiable + * in footnote of their tables are also considered to be modifiable + */ +enum pkcs11_rc check_attrs_against_modification(struct pkcs11_session *session, + struct obj_attrs *head, + struct pkcs11_object *obj, + enum processing_func function); + +enum pkcs11_rc set_key_data(struct obj_attrs **head, void *data, + size_t key_size); + +/* + * Get an allocated copy of key data to be wrapped from @head + * @head: Object attribute where to find key data to be wrapped + * @data: Output allocated and filled buffer upon success + * @sz: Key output data size in bytes upon success + * Return a pkcs11_rv compliant value + */ +enum pkcs11_rc alloc_key_data_to_wrap(struct obj_attrs *head, void **data, + uint32_t *sz); + +/* + * Adds CKA_ID attribute from paired object if missing. + * + * @pub_head - Public key object attributes + * @priv_head - Private key object attributes + * Return a PKCS11 return code + */ +enum pkcs11_rc add_missing_attribute_id(struct obj_attrs **pub_head, + struct obj_attrs **priv_head); + +#endif /*PKCS11_TA_PKCS11_ATTRIBUTES_H*/ diff --git a/optee_os/ta/pkcs11/src/pkcs11_helpers.c b/optee_os/ta/pkcs11/src/pkcs11_helpers.c new file mode 100644 index 0000000..364c7e6 --- /dev/null +++ b/optee_os/ta/pkcs11/src/pkcs11_helpers.c @@ -0,0 +1,844 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#include +#include +#include +#include + +#include "attributes.h" +#include "object.h" +#include "pkcs11_attributes.h" +#include "pkcs11_helpers.h" +#include "processing.h" + +static const char __maybe_unused unknown[] = ""; + +struct attr_size { + uint32_t id; + uint32_t size; +#if CFG_TEE_TA_LOG_LEVEL > 0 + const char *string; +#endif +}; + +#if CFG_TEE_TA_LOG_LEVEL > 0 +#define PKCS11_ID_SZ(_id, _sz) \ + { .id = (uint32_t)(_id), .size = (_sz), .string = #_id } +#else +#define PKCS11_ID_SZ(_id, _sz) \ + { .id = (uint32_t)(_id), .size = (_sz) } +#endif + +static const struct attr_size attr_ids[] = { + PKCS11_ID_SZ(PKCS11_CKA_CLASS, 4), + PKCS11_ID_SZ(PKCS11_CKA_KEY_TYPE, 4), + PKCS11_ID_SZ(PKCS11_CKA_VALUE, 0), + PKCS11_ID_SZ(PKCS11_CKA_VALUE_LEN, 4), + PKCS11_ID_SZ(PKCS11_CKA_KEY_GEN_MECHANISM, 4), + PKCS11_ID_SZ(PKCS11_CKA_LABEL, 0), + PKCS11_ID_SZ(PKCS11_CKA_CERTIFICATE_TYPE, 4), + PKCS11_ID_SZ(PKCS11_CKA_ISSUER, 0), + PKCS11_ID_SZ(PKCS11_CKA_SERIAL_NUMBER, 0), + PKCS11_ID_SZ(PKCS11_CKA_CERTIFICATE_CATEGORY, 4), + PKCS11_ID_SZ(PKCS11_CKA_URL, 0), + PKCS11_ID_SZ(PKCS11_CKA_HASH_OF_SUBJECT_PUBLIC_KEY, 0), + PKCS11_ID_SZ(PKCS11_CKA_HASH_OF_ISSUER_PUBLIC_KEY, 0), + PKCS11_ID_SZ(PKCS11_CKA_JAVA_MIDP_SECURITY_DOMAIN, 4), + PKCS11_ID_SZ(PKCS11_CKA_NAME_HASH_ALGORITHM, 4), + PKCS11_ID_SZ(PKCS11_CKA_CHECK_VALUE, 0), + PKCS11_ID_SZ(PKCS11_CKA_WRAP_TEMPLATE, 0), + PKCS11_ID_SZ(PKCS11_CKA_UNWRAP_TEMPLATE, 0), + PKCS11_ID_SZ(PKCS11_CKA_DERIVE_TEMPLATE, 0), + PKCS11_ID_SZ(PKCS11_CKA_START_DATE, 4), + PKCS11_ID_SZ(PKCS11_CKA_END_DATE, 4), + PKCS11_ID_SZ(PKCS11_CKA_OBJECT_ID, 0), + PKCS11_ID_SZ(PKCS11_CKA_APPLICATION, 0), + PKCS11_ID_SZ(PKCS11_CKA_MECHANISM_TYPE, 4), + PKCS11_ID_SZ(PKCS11_CKA_ID, 0), + PKCS11_ID_SZ(PKCS11_CKA_ALLOWED_MECHANISMS, 0), + PKCS11_ID_SZ(PKCS11_CKA_EC_POINT, 0), + PKCS11_ID_SZ(PKCS11_CKA_EC_PARAMS, 0), + PKCS11_ID_SZ(PKCS11_CKA_MODULUS, 0), + PKCS11_ID_SZ(PKCS11_CKA_MODULUS_BITS, 4), + PKCS11_ID_SZ(PKCS11_CKA_PUBLIC_EXPONENT, 0), + PKCS11_ID_SZ(PKCS11_CKA_PRIVATE_EXPONENT, 0), + PKCS11_ID_SZ(PKCS11_CKA_PRIME_1, 0), + PKCS11_ID_SZ(PKCS11_CKA_PRIME_2, 0), + PKCS11_ID_SZ(PKCS11_CKA_EXPONENT_1, 0), + PKCS11_ID_SZ(PKCS11_CKA_EXPONENT_2, 0), + PKCS11_ID_SZ(PKCS11_CKA_COEFFICIENT, 0), + PKCS11_ID_SZ(PKCS11_CKA_SUBJECT, 0), + PKCS11_ID_SZ(PKCS11_CKA_PUBLIC_KEY_INFO, 0), + PKCS11_ID_SZ(PKCS11_CKA_KEY_GEN_MECHANISM, 4), + /* Below are boolean attributes */ + PKCS11_ID_SZ(PKCS11_CKA_TOKEN, 1), + PKCS11_ID_SZ(PKCS11_CKA_PRIVATE, 1), + PKCS11_ID_SZ(PKCS11_CKA_TRUSTED, 1), + PKCS11_ID_SZ(PKCS11_CKA_SENSITIVE, 1), + PKCS11_ID_SZ(PKCS11_CKA_ENCRYPT, 1), + PKCS11_ID_SZ(PKCS11_CKA_DECRYPT, 1), + PKCS11_ID_SZ(PKCS11_CKA_WRAP, 1), + PKCS11_ID_SZ(PKCS11_CKA_UNWRAP, 1), + PKCS11_ID_SZ(PKCS11_CKA_SIGN, 1), + PKCS11_ID_SZ(PKCS11_CKA_SIGN_RECOVER, 1), + PKCS11_ID_SZ(PKCS11_CKA_VERIFY, 1), + PKCS11_ID_SZ(PKCS11_CKA_VERIFY_RECOVER, 1), + PKCS11_ID_SZ(PKCS11_CKA_DERIVE, 1), + PKCS11_ID_SZ(PKCS11_CKA_EXTRACTABLE, 1), + PKCS11_ID_SZ(PKCS11_CKA_LOCAL, 1), + PKCS11_ID_SZ(PKCS11_CKA_NEVER_EXTRACTABLE, 1), + PKCS11_ID_SZ(PKCS11_CKA_ALWAYS_SENSITIVE, 1), + PKCS11_ID_SZ(PKCS11_CKA_MODIFIABLE, 1), + PKCS11_ID_SZ(PKCS11_CKA_COPYABLE, 1), + PKCS11_ID_SZ(PKCS11_CKA_DESTROYABLE, 1), + PKCS11_ID_SZ(PKCS11_CKA_ALWAYS_AUTHENTICATE, 1), + PKCS11_ID_SZ(PKCS11_CKA_WRAP_WITH_TRUSTED, 1), + /* Specific PKCS11 TA internal attribute ID */ + PKCS11_ID_SZ(PKCS11_CKA_OPTEE_HIDDEN_EC_POINT, 0), + PKCS11_ID_SZ(PKCS11_CKA_UNDEFINED_ID, 0), +}; + +struct any_id { + uint32_t id; +#if CFG_TEE_TA_LOG_LEVEL > 0 + const char *string; +#endif +}; + +/* + * Macro PKCS11_ID() can be used to define cells in ID list arrays + * or ID/string conversion arrays. + */ +#if CFG_TEE_TA_LOG_LEVEL > 0 +#define PKCS11_ID(_id) { .id = _id, .string = #_id } +#else +#define PKCS11_ID(_id) { .id = _id } +#endif + +#define ID2STR(id, table, prefix) \ + id2str(id, table, ARRAY_SIZE(table), prefix) + +#if CFG_TEE_TA_LOG_LEVEL > 0 +/* Convert a PKCS11 ID into its label string */ +static const char *id2str(uint32_t id, const struct any_id *table, + size_t count, const char *prefix) +{ + size_t n = 0; + const char *str = NULL; + + for (n = 0; n < count; n++) { + if (id != table[n].id) + continue; + + str = table[n].string; + + /* Skip prefix provided matches found */ + if (prefix && !TEE_MemCompare(str, prefix, strlen(prefix))) + str += strlen(prefix); + + return str; + } + + return unknown; +} +#endif /* CFG_TEE_TA_LOG_LEVEL > 0 */ + +/* + * TA command IDs: used only as ID/string conversion for debug trace support + */ +static const struct any_id __maybe_unused string_ta_cmd[] = { + PKCS11_ID(PKCS11_CMD_PING), + PKCS11_ID(PKCS11_CMD_SLOT_LIST), + PKCS11_ID(PKCS11_CMD_SLOT_INFO), + PKCS11_ID(PKCS11_CMD_TOKEN_INFO), + PKCS11_ID(PKCS11_CMD_MECHANISM_IDS), + PKCS11_ID(PKCS11_CMD_MECHANISM_INFO), + PKCS11_ID(PKCS11_CMD_OPEN_SESSION), + PKCS11_ID(PKCS11_CMD_SESSION_INFO), + PKCS11_ID(PKCS11_CMD_CLOSE_SESSION), + PKCS11_ID(PKCS11_CMD_CLOSE_ALL_SESSIONS), + PKCS11_ID(PKCS11_CMD_INIT_TOKEN), + PKCS11_ID(PKCS11_CMD_INIT_PIN), + PKCS11_ID(PKCS11_CMD_SET_PIN), + PKCS11_ID(PKCS11_CMD_LOGIN), + PKCS11_ID(PKCS11_CMD_LOGOUT), + PKCS11_ID(PKCS11_CMD_CREATE_OBJECT), + PKCS11_ID(PKCS11_CMD_DESTROY_OBJECT), + PKCS11_ID(PKCS11_CMD_ENCRYPT_INIT), + PKCS11_ID(PKCS11_CMD_DECRYPT_INIT), + PKCS11_ID(PKCS11_CMD_ENCRYPT_UPDATE), + PKCS11_ID(PKCS11_CMD_DECRYPT_UPDATE), + PKCS11_ID(PKCS11_CMD_ENCRYPT_FINAL), + PKCS11_ID(PKCS11_CMD_DECRYPT_FINAL), + PKCS11_ID(PKCS11_CMD_ENCRYPT_ONESHOT), + PKCS11_ID(PKCS11_CMD_DECRYPT_ONESHOT), + PKCS11_ID(PKCS11_CMD_SIGN_INIT), + PKCS11_ID(PKCS11_CMD_VERIFY_INIT), + PKCS11_ID(PKCS11_CMD_SIGN_UPDATE), + PKCS11_ID(PKCS11_CMD_VERIFY_UPDATE), + PKCS11_ID(PKCS11_CMD_SIGN_FINAL), + PKCS11_ID(PKCS11_CMD_VERIFY_FINAL), + PKCS11_ID(PKCS11_CMD_SIGN_ONESHOT), + PKCS11_ID(PKCS11_CMD_VERIFY_ONESHOT), + PKCS11_ID(PKCS11_CMD_GENERATE_KEY), + PKCS11_ID(PKCS11_CMD_FIND_OBJECTS_INIT), + PKCS11_ID(PKCS11_CMD_FIND_OBJECTS), + PKCS11_ID(PKCS11_CMD_FIND_OBJECTS_FINAL), + PKCS11_ID(PKCS11_CMD_GET_OBJECT_SIZE), + PKCS11_ID(PKCS11_CMD_GET_ATTRIBUTE_VALUE), + PKCS11_ID(PKCS11_CMD_SET_ATTRIBUTE_VALUE), + PKCS11_ID(PKCS11_CMD_COPY_OBJECT), + PKCS11_ID(PKCS11_CMD_SEED_RANDOM), + PKCS11_ID(PKCS11_CMD_GENERATE_RANDOM), + PKCS11_ID(PKCS11_CMD_DERIVE_KEY), + PKCS11_ID(PKCS11_CMD_RELEASE_ACTIVE_PROCESSING), + PKCS11_ID(PKCS11_CMD_DIGEST_INIT), + PKCS11_ID(PKCS11_CMD_DIGEST_UPDATE), + PKCS11_ID(PKCS11_CMD_DIGEST_KEY), + PKCS11_ID(PKCS11_CMD_DIGEST_ONESHOT), + PKCS11_ID(PKCS11_CMD_DIGEST_FINAL), + PKCS11_ID(PKCS11_CMD_GENERATE_KEY_PAIR), + PKCS11_ID(PKCS11_CMD_WRAP_KEY), + PKCS11_ID(PKCS11_CMD_UNWRAP_KEY), +}; + +static const struct any_id __maybe_unused string_slot_flags[] = { + PKCS11_ID(PKCS11_CKFS_TOKEN_PRESENT), + PKCS11_ID(PKCS11_CKFS_REMOVABLE_DEVICE), + PKCS11_ID(PKCS11_CKFS_HW_SLOT), +}; + +static const struct any_id __maybe_unused string_token_flags[] = { + PKCS11_ID(PKCS11_CKFT_RNG), + PKCS11_ID(PKCS11_CKFT_WRITE_PROTECTED), + PKCS11_ID(PKCS11_CKFT_LOGIN_REQUIRED), + PKCS11_ID(PKCS11_CKFT_USER_PIN_INITIALIZED), + PKCS11_ID(PKCS11_CKFT_RESTORE_KEY_NOT_NEEDED), + PKCS11_ID(PKCS11_CKFT_CLOCK_ON_TOKEN), + PKCS11_ID(PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH), + PKCS11_ID(PKCS11_CKFT_DUAL_CRYPTO_OPERATIONS), + PKCS11_ID(PKCS11_CKFT_TOKEN_INITIALIZED), + PKCS11_ID(PKCS11_CKFT_USER_PIN_COUNT_LOW), + PKCS11_ID(PKCS11_CKFT_USER_PIN_FINAL_TRY), + PKCS11_ID(PKCS11_CKFT_USER_PIN_LOCKED), + PKCS11_ID(PKCS11_CKFT_USER_PIN_TO_BE_CHANGED), + PKCS11_ID(PKCS11_CKFT_SO_PIN_COUNT_LOW), + PKCS11_ID(PKCS11_CKFT_SO_PIN_FINAL_TRY), + PKCS11_ID(PKCS11_CKFT_SO_PIN_LOCKED), + PKCS11_ID(PKCS11_CKFT_SO_PIN_TO_BE_CHANGED), + PKCS11_ID(PKCS11_CKFT_ERROR_STATE), +}; + +static const struct any_id __maybe_unused string_session_flags[] = { + PKCS11_ID(PKCS11_CKFSS_RW_SESSION), + PKCS11_ID(PKCS11_CKFSS_SERIAL_SESSION), +}; + +static const struct any_id __maybe_unused string_session_state[] = { + PKCS11_ID(PKCS11_CKS_RO_PUBLIC_SESSION), + PKCS11_ID(PKCS11_CKS_RO_USER_FUNCTIONS), + PKCS11_ID(PKCS11_CKS_RW_PUBLIC_SESSION), + PKCS11_ID(PKCS11_CKS_RW_USER_FUNCTIONS), + PKCS11_ID(PKCS11_CKS_RW_SO_FUNCTIONS), +}; + +static const struct any_id __maybe_unused string_rc[] = { + PKCS11_ID(PKCS11_CKR_OK), + PKCS11_ID(PKCS11_CKR_SLOT_ID_INVALID), + PKCS11_ID(PKCS11_CKR_GENERAL_ERROR), + PKCS11_ID(PKCS11_CKR_FUNCTION_FAILED), + PKCS11_ID(PKCS11_CKR_ARGUMENTS_BAD), + PKCS11_ID(PKCS11_CKR_ATTRIBUTE_READ_ONLY), + PKCS11_ID(PKCS11_CKR_ATTRIBUTE_SENSITIVE), + PKCS11_ID(PKCS11_CKR_ATTRIBUTE_TYPE_INVALID), + PKCS11_ID(PKCS11_CKR_ATTRIBUTE_VALUE_INVALID), + PKCS11_ID(PKCS11_CKR_ACTION_PROHIBITED), + PKCS11_ID(PKCS11_CKR_DATA_LEN_RANGE), + PKCS11_ID(PKCS11_CKR_DEVICE_MEMORY), + PKCS11_ID(PKCS11_CKR_ENCRYPTED_DATA_INVALID), + PKCS11_ID(PKCS11_CKR_ENCRYPTED_DATA_LEN_RANGE), + PKCS11_ID(PKCS11_CKR_KEY_HANDLE_INVALID), + PKCS11_ID(PKCS11_CKR_KEY_SIZE_RANGE), + PKCS11_ID(PKCS11_CKR_KEY_TYPE_INCONSISTENT), + PKCS11_ID(PKCS11_CKR_KEY_INDIGESTIBLE), + PKCS11_ID(PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED), + PKCS11_ID(PKCS11_CKR_KEY_NOT_WRAPPABLE), + PKCS11_ID(PKCS11_CKR_KEY_UNEXTRACTABLE), + PKCS11_ID(PKCS11_CKR_MECHANISM_INVALID), + PKCS11_ID(PKCS11_CKR_MECHANISM_PARAM_INVALID), + PKCS11_ID(PKCS11_CKR_OBJECT_HANDLE_INVALID), + PKCS11_ID(PKCS11_CKR_OPERATION_ACTIVE), + PKCS11_ID(PKCS11_CKR_OPERATION_NOT_INITIALIZED), + PKCS11_ID(PKCS11_CKR_PIN_INCORRECT), + PKCS11_ID(PKCS11_CKR_PIN_INVALID), + PKCS11_ID(PKCS11_CKR_PIN_LEN_RANGE), + PKCS11_ID(PKCS11_CKR_PIN_EXPIRED), + PKCS11_ID(PKCS11_CKR_PIN_LOCKED), + PKCS11_ID(PKCS11_CKR_SESSION_HANDLE_INVALID), + PKCS11_ID(PKCS11_CKR_SESSION_PARALLEL_NOT_SUPPORTED), + PKCS11_ID(PKCS11_CKR_SESSION_READ_ONLY), + PKCS11_ID(PKCS11_CKR_SESSION_EXISTS), + PKCS11_ID(PKCS11_CKR_SESSION_READ_ONLY_EXISTS), + PKCS11_ID(PKCS11_CKR_SESSION_READ_WRITE_SO_EXISTS), + PKCS11_ID(PKCS11_CKR_SIGNATURE_INVALID), + PKCS11_ID(PKCS11_CKR_SIGNATURE_LEN_RANGE), + PKCS11_ID(PKCS11_CKR_TEMPLATE_INCOMPLETE), + PKCS11_ID(PKCS11_CKR_TEMPLATE_INCONSISTENT), + PKCS11_ID(PKCS11_CKR_TOKEN_NOT_PRESENT), + PKCS11_ID(PKCS11_CKR_TOKEN_NOT_RECOGNIZED), + PKCS11_ID(PKCS11_CKR_TOKEN_WRITE_PROTECTED), + PKCS11_ID(PKCS11_CKR_UNWRAPPING_KEY_HANDLE_INVALID), + PKCS11_ID(PKCS11_CKR_UNWRAPPING_KEY_SIZE_RANGE), + PKCS11_ID(PKCS11_CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT), + PKCS11_ID(PKCS11_CKR_USER_ALREADY_LOGGED_IN), + PKCS11_ID(PKCS11_CKR_USER_NOT_LOGGED_IN), + PKCS11_ID(PKCS11_CKR_USER_PIN_NOT_INITIALIZED), + PKCS11_ID(PKCS11_CKR_USER_TYPE_INVALID), + PKCS11_ID(PKCS11_CKR_USER_ANOTHER_ALREADY_LOGGED_IN), + PKCS11_ID(PKCS11_CKR_USER_TOO_MANY_TYPES), + PKCS11_ID(PKCS11_CKR_WRAPPED_KEY_INVALID), + PKCS11_ID(PKCS11_CKR_WRAPPED_KEY_LEN_RANGE), + PKCS11_ID(PKCS11_CKR_WRAPPING_KEY_HANDLE_INVALID), + PKCS11_ID(PKCS11_CKR_WRAPPING_KEY_SIZE_RANGE), + PKCS11_ID(PKCS11_CKR_WRAPPING_KEY_TYPE_INCONSISTENT), + PKCS11_ID(PKCS11_CKR_RANDOM_SEED_NOT_SUPPORTED), + PKCS11_ID(PKCS11_CKR_RANDOM_NO_RNG), + PKCS11_ID(PKCS11_CKR_DOMAIN_PARAMS_INVALID), + PKCS11_ID(PKCS11_CKR_CURVE_NOT_SUPPORTED), + PKCS11_ID(PKCS11_CKR_BUFFER_TOO_SMALL), + PKCS11_ID(PKCS11_CKR_PIN_TOO_WEAK), + PKCS11_ID(PKCS11_RV_NOT_FOUND), + PKCS11_ID(PKCS11_RV_NOT_IMPLEMENTED), +}; + +static const struct any_id __maybe_unused string_class[] = { + PKCS11_ID(PKCS11_CKO_SECRET_KEY), + PKCS11_ID(PKCS11_CKO_PUBLIC_KEY), + PKCS11_ID(PKCS11_CKO_PRIVATE_KEY), + PKCS11_ID(PKCS11_CKO_OTP_KEY), + PKCS11_ID(PKCS11_CKO_CERTIFICATE), + PKCS11_ID(PKCS11_CKO_DATA), + PKCS11_ID(PKCS11_CKO_DOMAIN_PARAMETERS), + PKCS11_ID(PKCS11_CKO_HW_FEATURE), + PKCS11_ID(PKCS11_CKO_MECHANISM), + PKCS11_ID(PKCS11_CKO_UNDEFINED_ID) +}; + +static const struct any_id __maybe_unused string_key_type[] = { + PKCS11_ID(PKCS11_CKK_AES), + PKCS11_ID(PKCS11_CKK_GENERIC_SECRET), + PKCS11_ID(PKCS11_CKK_MD5_HMAC), + PKCS11_ID(PKCS11_CKK_SHA_1_HMAC), + PKCS11_ID(PKCS11_CKK_SHA224_HMAC), + PKCS11_ID(PKCS11_CKK_SHA256_HMAC), + PKCS11_ID(PKCS11_CKK_SHA384_HMAC), + PKCS11_ID(PKCS11_CKK_SHA512_HMAC), + PKCS11_ID(PKCS11_CKK_EC), + PKCS11_ID(PKCS11_CKK_EC_EDWARDS), + PKCS11_ID(PKCS11_CKK_EDDSA), + PKCS11_ID(PKCS11_CKK_RSA), + PKCS11_ID(PKCS11_CKK_UNDEFINED_ID) +}; + +static const struct any_id __maybe_unused string_certificate_type[] = { + PKCS11_ID(PKCS11_CKC_X_509), + PKCS11_ID(PKCS11_CKC_X_509_ATTR_CERT), + PKCS11_ID(PKCS11_CKC_WTLS), + PKCS11_ID(PKCS11_CKC_UNDEFINED_ID) +}; + +/* + * Processing IDs not exported in the TA API. + * PKCS11_CKM_* mechanism IDs are looked up from mechanism_string_id(). + */ +static const struct any_id __maybe_unused string_internal_processing[] = { + PKCS11_ID(PKCS11_PROCESSING_IMPORT), +}; + +static const struct any_id __maybe_unused string_functions[] = { + PKCS11_ID(PKCS11_FUNCTION_DIGEST), + PKCS11_ID(PKCS11_FUNCTION_IMPORT), + PKCS11_ID(PKCS11_FUNCTION_ENCRYPT), + PKCS11_ID(PKCS11_FUNCTION_DECRYPT), + PKCS11_ID(PKCS11_FUNCTION_SIGN), + PKCS11_ID(PKCS11_FUNCTION_VERIFY), + PKCS11_ID(PKCS11_FUNCTION_DERIVE), + PKCS11_ID(PKCS11_FUNCTION_WRAP), + PKCS11_ID(PKCS11_FUNCTION_UNWRAP), +}; + +/* + * Conversion between PKCS11 TA and GPD TEE return codes + */ +enum pkcs11_rc tee2pkcs_error(TEE_Result res) +{ + switch (res) { + case TEE_SUCCESS: + return PKCS11_CKR_OK; + + case TEE_ERROR_BAD_PARAMETERS: + return PKCS11_CKR_ARGUMENTS_BAD; + + case TEE_ERROR_CIPHERTEXT_INVALID: + return PKCS11_CKR_ENCRYPTED_DATA_INVALID; + + case TEE_ERROR_OUT_OF_MEMORY: + return PKCS11_CKR_DEVICE_MEMORY; + + case TEE_ERROR_SHORT_BUFFER: + return PKCS11_CKR_BUFFER_TOO_SMALL; + + case TEE_ERROR_MAC_INVALID: + case TEE_ERROR_SIGNATURE_INVALID: + return PKCS11_CKR_SIGNATURE_INVALID; + + default: + return PKCS11_CKR_GENERAL_ERROR; + } +} + +/* + * Helper functions to analyse PKCS11 identifiers + */ + +/* Check attribute ID is known and size matches if fixed */ +bool valid_pkcs11_attribute_id(uint32_t id, uint32_t size) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(attr_ids); n++) + if (id == attr_ids[n].id) + return !attr_ids[n].size || size == attr_ids[n].size; + + return false; +} + +size_t pkcs11_attr_is_type(uint32_t attribute_id) +{ + enum pkcs11_attr_id id = attribute_id; + + switch (id) { + case PKCS11_CKA_KEY_TYPE: + case PKCS11_CKA_MECHANISM_TYPE: + case PKCS11_CKA_KEY_GEN_MECHANISM: + return sizeof(uint32_t); + default: + return 0; + } +} + +bool pkcs11_attr_has_indirect_attributes(uint32_t attribute_id) +{ + switch (attribute_id) { + case PKCS11_CKA_WRAP_TEMPLATE: + case PKCS11_CKA_UNWRAP_TEMPLATE: + case PKCS11_CKA_DERIVE_TEMPLATE: + return true; + default: + return false; + } +} + +bool pkcs11_class_has_type(uint32_t class) +{ + enum pkcs11_class_id class_id = class; + + switch (class_id) { + case PKCS11_CKO_CERTIFICATE: + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_PRIVATE_KEY: + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_MECHANISM: + case PKCS11_CKO_HW_FEATURE: + return true; + default: + return false; + } +} + +bool pkcs11_attr_class_is_key(uint32_t class) +{ + enum pkcs11_class_id class_id = class; + + switch (class_id) { + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_PRIVATE_KEY: + return true; + default: + return false; + } +} + +bool key_type_is_symm_key(uint32_t id) +{ + enum pkcs11_key_type key_type = id; + + switch (key_type) { + case PKCS11_CKK_AES: + case PKCS11_CKK_GENERIC_SECRET: + case PKCS11_CKK_MD5_HMAC: + case PKCS11_CKK_SHA_1_HMAC: + case PKCS11_CKK_SHA224_HMAC: + case PKCS11_CKK_SHA256_HMAC: + case PKCS11_CKK_SHA384_HMAC: + case PKCS11_CKK_SHA512_HMAC: + return true; + default: + return false; + } +} + +bool key_type_is_asymm_key(uint32_t id) +{ + enum pkcs11_key_type key_type = id; + + switch (key_type) { + case PKCS11_CKK_EC: + case PKCS11_CKK_EC_EDWARDS: + case PKCS11_CKK_RSA: + return true; + default: + return false; + } +} + +/* + * Returns shift position or -1 on error. + * Mainly used when PKCS11_SHEAD_WITH_BOOLPROPS is enabled + */ +int pkcs11_attr2boolprop_shift(uint32_t attr) +{ + static const uint32_t bpa[] = { + [BPA_TOKEN] = PKCS11_CKA_TOKEN, + [BPA_PRIVATE] = PKCS11_CKA_PRIVATE, + [BPA_TRUSTED] = PKCS11_CKA_TRUSTED, + [BPA_SENSITIVE] = PKCS11_CKA_SENSITIVE, + [BPA_ENCRYPT] = PKCS11_CKA_ENCRYPT, + [BPA_DECRYPT] = PKCS11_CKA_DECRYPT, + [BPA_WRAP] = PKCS11_CKA_WRAP, + [BPA_UNWRAP] = PKCS11_CKA_UNWRAP, + [BPA_SIGN] = PKCS11_CKA_SIGN, + [BPA_SIGN_RECOVER] = PKCS11_CKA_SIGN_RECOVER, + [BPA_VERIFY] = PKCS11_CKA_VERIFY, + [BPA_VERIFY_RECOVER] = PKCS11_CKA_VERIFY_RECOVER, + [BPA_DERIVE] = PKCS11_CKA_DERIVE, + [BPA_EXTRACTABLE] = PKCS11_CKA_EXTRACTABLE, + [BPA_LOCAL] = PKCS11_CKA_LOCAL, + [BPA_NEVER_EXTRACTABLE] = PKCS11_CKA_NEVER_EXTRACTABLE, + [BPA_ALWAYS_SENSITIVE] = PKCS11_CKA_ALWAYS_SENSITIVE, + [BPA_MODIFIABLE] = PKCS11_CKA_MODIFIABLE, + [BPA_COPYABLE] = PKCS11_CKA_COPYABLE, + [BPA_DESTROYABLE] = PKCS11_CKA_DESTROYABLE, + [BPA_ALWAYS_AUTHENTICATE] = PKCS11_CKA_ALWAYS_AUTHENTICATE, + [BPA_WRAP_WITH_TRUSTED] = PKCS11_CKA_WRAP_WITH_TRUSTED, + }; + size_t pos = 0; + + for (pos = 0; pos < ARRAY_SIZE(bpa); pos++) + if (bpa[pos] == attr) + return (int)pos; + + return -1; +} + +/* Initialize a TEE attribute for a target PKCS11 TA attribute in an object */ +bool pkcs2tee_load_attr(TEE_Attribute *tee_ref, uint32_t tee_id, + struct pkcs11_object *obj, + enum pkcs11_attr_id pkcs11_id) +{ + void *a_ptr = NULL; + uint8_t *der_ptr = NULL; + uint32_t a_size = 0; + uint32_t data32 = 0; + size_t hsize = 0; + size_t qsize = 0; + + switch (tee_id) { + case TEE_ATTR_ECC_PUBLIC_VALUE_X: + case TEE_ATTR_ECC_PUBLIC_VALUE_Y: + case TEE_ATTR_ECC_CURVE: + if (get_attribute_ptr(obj->attributes, PKCS11_CKA_EC_PARAMS, + &a_ptr, &a_size) || !a_ptr) { + EMSG("Missing EC_PARAMS attribute"); + return false; + } + + if (tee_id == TEE_ATTR_ECC_CURVE) { + data32 = ec_params2tee_curve(a_ptr, a_size); + TEE_InitValueAttribute(tee_ref, TEE_ATTR_ECC_CURVE, + data32, 0); + return true; + } + + data32 = (ec_params2tee_keysize(a_ptr, a_size) + 7) / 8; + + /* + * For private keys we need EC public key for TEE operations so + * first try to get hidden EC POINT and as backwards + * compatibility also check for CKA_EC_POINT. + * + * For public keys we only have CKA_EC_POINT but there is no + * harm to check for hidden one too. + */ + if (get_attribute_ptr(obj->attributes, + PKCS11_CKA_OPTEE_HIDDEN_EC_POINT, + &a_ptr, &a_size)) { + if (get_attribute_ptr(obj->attributes, + PKCS11_CKA_EC_POINT, + &a_ptr, &a_size)) { + /* + * Without EC public key we cannot proceed. + */ + EMSG("Missing EC_POINT attribute"); + return false; + } + } + + der_ptr = (uint8_t *)a_ptr; + + if (der_ptr[0] != 0x04) { + EMSG("Unsupported DER type"); + return false; + } + + if ((der_ptr[1] & 0x80) == 0) { + /* DER short definitive form up to 127 bytes */ + qsize = der_ptr[1] & 0x7F; + hsize = 2 /* der */ + 1 /* point compression */; + } else if (der_ptr[1] == 0x81) { + /* DER long definitive form up to 255 bytes */ + qsize = der_ptr[2]; + hsize = 3 /* der */ + 1 /* point compression */; + } else { + EMSG("Unsupported DER long form"); + return false; + } + + if (der_ptr[hsize - 1] != 0x04) { + EMSG("Unsupported EC_POINT compression"); + return false; + } + + if (a_size != (hsize - 1) + qsize) { + EMSG("Invalid EC_POINT attribute"); + return false; + } + + if (a_size != hsize + 2 * data32) { + EMSG("Invalid EC_POINT attribute"); + return false; + } + + if (tee_id == TEE_ATTR_ECC_PUBLIC_VALUE_X) + TEE_InitRefAttribute(tee_ref, tee_id, + der_ptr + hsize, data32); + else + TEE_InitRefAttribute(tee_ref, tee_id, + der_ptr + hsize + data32, + data32); + + return true; + + default: + break; + } + + if (get_attribute_ptr(obj->attributes, pkcs11_id, &a_ptr, &a_size)) + return false; + + TEE_InitRefAttribute(tee_ref, tee_id, a_ptr, a_size); + + return true; +} + +/* + * Initialize a TEE attribute with hash of a target PKCS11 TA attribute + * in an object. + */ +enum pkcs11_rc pkcs2tee_load_hashed_attr(TEE_Attribute *tee_ref, + uint32_t tee_id, + struct pkcs11_object *obj, + enum pkcs11_attr_id pkcs11_id, + uint32_t tee_algo, void *hash_ptr, + uint32_t *hash_size) +{ + TEE_OperationHandle handle = TEE_HANDLE_NULL; + void *a_ptr = NULL; + uint32_t a_size = 0; + enum pkcs11_rc rc = PKCS11_CKR_OK; + TEE_Result res = TEE_ERROR_GENERIC; + size_t tmp_sz = 0; + + rc = get_attribute_ptr(obj->attributes, pkcs11_id, &a_ptr, &a_size); + if (rc) + return rc; + + res = TEE_AllocateOperation(&handle, tee_algo, TEE_MODE_DIGEST, 0); + if (res) { + EMSG("TEE_AllocateOperation() failed %#"PRIx32, tee_algo); + return tee2pkcs_error(res); + } + + tmp_sz = *hash_size; + res = TEE_DigestDoFinal(handle, a_ptr, a_size, hash_ptr, &tmp_sz); + *hash_size = tmp_sz; + TEE_FreeOperation(handle); + if (res) { + EMSG("TEE_DigestDoFinal() failed %#"PRIx32, tee_algo); + return PKCS11_CKR_FUNCTION_FAILED; + } + + TEE_InitRefAttribute(tee_ref, tee_id, hash_ptr, *hash_size); + + return PKCS11_CKR_OK; +} + +/* Easy conversion between PKCS11 TA function of TEE crypto mode */ +void pkcs2tee_mode(uint32_t *tee_id, enum processing_func function) +{ + switch (function) { + case PKCS11_FUNCTION_ENCRYPT: + *tee_id = TEE_MODE_ENCRYPT; + break; + case PKCS11_FUNCTION_DECRYPT: + *tee_id = TEE_MODE_DECRYPT; + break; + case PKCS11_FUNCTION_SIGN: + *tee_id = TEE_MODE_SIGN; + break; + case PKCS11_FUNCTION_VERIFY: + *tee_id = TEE_MODE_VERIFY; + break; + case PKCS11_FUNCTION_DERIVE: + *tee_id = TEE_MODE_DERIVE; + break; + case PKCS11_FUNCTION_DIGEST: + *tee_id = TEE_MODE_DIGEST; + break; + default: + TEE_Panic(function); + } +} + +#if CFG_TEE_TA_LOG_LEVEL > 0 +const char *id2str_rc(uint32_t id) +{ + return ID2STR(id, string_rc, "PKCS11_CKR_"); +} + +const char *id2str_ta_cmd(uint32_t id) +{ + return ID2STR(id, string_ta_cmd, NULL); +} + +const char *id2str_slot_flag(uint32_t id) +{ + return ID2STR(id, string_slot_flags, "PKCS11_CKFS_"); +} + +const char *id2str_token_flag(uint32_t id) +{ + return ID2STR(id, string_token_flags, "PKCS11_CKFT_"); +} + +const char *id2str_session_flag(uint32_t id) +{ + return ID2STR(id, string_session_flags, "PKCS11_CKFSS_"); +} + +const char *id2str_session_state(uint32_t id) +{ + return ID2STR(id, string_session_state, "PKCS11_CKS_"); +} + +const char *id2str_attr(uint32_t id) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(attr_ids); n++) { + if (id == attr_ids[n].id) { + /* Skip PKCS11_CKA_ prefix */ + return attr_ids[n].string + strlen("PKCS11_CKA_"); + } + } + + return unknown; +} + +const char *id2str_class(uint32_t id) +{ + return ID2STR(id, string_class, "PKCS11_CKO_"); +} + +const char *id2str_type(uint32_t id, uint32_t class) +{ + enum pkcs11_class_id class_id = class; + enum pkcs11_key_type key_type = id; + + switch (class_id) { + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_PRIVATE_KEY: + return id2str_key_type(key_type); + default: + return unknown; + } +} + +const char *id2str_key_type(uint32_t id) +{ + return ID2STR(id, string_key_type, "PKCS11_CKK_"); +} + +const char *id2str_certificate_type(uint32_t id) +{ + return ID2STR(id, string_certificate_type, "PKCS11_CKC_"); +} + +const char *id2str_attr_value(uint32_t id, size_t size, void *value) +{ + static const char str_true[] = "TRUE"; + static const char str_false[] = "FALSE"; + static const char str_unknown[] = "*"; + uint32_t type = 0; + + if (pkcs11_attr2boolprop_shift(id) >= 0) + return *(uint8_t *)value ? str_true : str_false; + + if (size < sizeof(uint32_t)) + return str_unknown; + + TEE_MemMove(&type, value, sizeof(uint32_t)); + + switch (id) { + case PKCS11_CKA_CLASS: + return id2str_class(type); + case PKCS11_CKA_KEY_TYPE: + return id2str_key_type(type); + case PKCS11_CKA_MECHANISM_TYPE: + return id2str_mechanism(type); + default: + return str_unknown; + } +} + +const char *id2str_proc(uint32_t id) +{ + const char *str = ID2STR(id, string_internal_processing, + "PKCS11_PROCESSING_"); + + if (str != unknown) + return str; + + return id2str_mechanism(id); +} + +const char *id2str_function(uint32_t id) +{ + return ID2STR(id, string_functions, "PKCS11_FUNCTION_"); +} +#endif /*CFG_TEE_TA_LOG_LEVEL*/ diff --git a/optee_os/ta/pkcs11/src/pkcs11_helpers.h b/optee_os/ta/pkcs11/src/pkcs11_helpers.h new file mode 100644 index 0000000..41b9b57 --- /dev/null +++ b/optee_os/ta/pkcs11/src/pkcs11_helpers.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#ifndef PKCS11_HELPERS_H +#define PKCS11_HELPERS_H + +#include +#include +#include +#include + +#include +#include + +struct pkcs11_object; + +/* + * TEE invocation parameter#0 is an in/out buffer of at least 32bit + * to store the TA PKCS#11 compliant return value. + */ +#define TEE_PARAM0_SIZE_MIN sizeof(uint32_t) + +/* GPD TEE to PKCS11 status conversion */ +enum pkcs11_rc tee2pkcs_error(TEE_Result res); + +/* + * Return true if and only if attribute ID with companion attribute value + * size do match a valid attribute identifier. + * + * @attribute_id - Target PKCS11 attribute ID + * @size - Byte size of the attribute value, 0 if non-constant size + */ +bool valid_pkcs11_attribute_id(uint32_t attribute_id, uint32_t size); + +/* + * Return type attribute byte size if @attribute_id is the ID of a type + * attribute or 0 if not. + */ +size_t pkcs11_attr_is_type(uint32_t attribute_id); + +/* Return true if the attribute has indirect attributes */ +bool pkcs11_attr_has_indirect_attributes(uint32_t attribute_id); + +/* Return true if the object class is related to a type-in-class */ +bool pkcs11_class_has_type(uint32_t class_id); + +/* Return true if the object class relates to a key */ +bool pkcs11_attr_class_is_key(uint32_t class_id); + +/* Return true if the key type @key_type_id relates to a symmetric key */ +bool key_type_is_symm_key(uint32_t key_type_id); + +/* Return true if the key type @key_type_id relates to an asymmetric key */ +bool key_type_is_asymm_key(uint32_t key_type_id); + +/* Boolprop flag shift position if @attribute_id is boolean, else -1 */ +int pkcs11_attr2boolprop_shift(uint32_t attribute_id); + +/* Convert PKCS11 TA function ID into a TEE crypto operation mode */ +void pkcs2tee_mode(uint32_t *tee_id, enum processing_func function); + +/* Load TEE operation attributes from a PKCS11 object, return false on error */ +bool pkcs2tee_load_attr(TEE_Attribute *tee_ref, uint32_t tee_id, + struct pkcs11_object *obj, + enum pkcs11_attr_id pkcs11_id); + +/* Hash and load TEE operation attributes from a PKCS11 object */ +enum pkcs11_rc pkcs2tee_load_hashed_attr(TEE_Attribute *tee_ref, + uint32_t tee_id, + struct pkcs11_object *obj, + enum pkcs11_attr_id pkcs11_id, + uint32_t tee_algo, void *hash_ptr, + uint32_t *hash_size); + +/* Return true if attribute is a boolean, false otherwise */ +static inline bool pkcs11_attr_is_boolean(enum pkcs11_attr_id id) +{ + return pkcs11_attr2boolprop_shift(id) >= 0; +} + +#if CFG_TEE_TA_LOG_LEVEL > 0 +/* Id-to-string conversions only for trace support */ +const char *id2str_ta_cmd(uint32_t id); +const char *id2str_rc(uint32_t id); +const char *id2str_slot_flag(uint32_t id); +const char *id2str_token_flag(uint32_t id); +const char *id2str_session_flag(uint32_t id); +const char *id2str_session_state(uint32_t id); +const char *id2str_attr(uint32_t id); +const char *id2str_class(uint32_t id); +const char *id2str_type(uint32_t id, uint32_t class); +const char *id2str_key_type(uint32_t id); +const char *id2str_certificate_type(uint32_t id); +const char *id2str_attr_value(uint32_t id, size_t size, void *value); +const char *id2str_proc(uint32_t id); +const char *id2str_function(uint32_t id); + +static inline const char *id2str_mechanism(enum pkcs11_mechanism_id id) +{ + return mechanism_string_id(id); +} +#endif /* CFG_TEE_TA_LOG_LEVEL > 0 */ +#endif /*PKCS11_HELPERS_H*/ diff --git a/optee_os/ta/pkcs11/src/pkcs11_token.c b/optee_os/ta/pkcs11/src/pkcs11_token.c new file mode 100644 index 0000000..ab0fc29 --- /dev/null +++ b/optee_os/ta/pkcs11/src/pkcs11_token.c @@ -0,0 +1,1671 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "handle.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" +#include "serializer.h" +#include "token_capabilities.h" + +/* Number of tokens implemented by the TA. Token ID is the token index */ +#define TOKEN_COUNT CFG_PKCS11_TA_TOKEN_COUNT + +/* RNG chunk size used to split RNG generation to smaller sizes */ +#define RNG_CHUNK_SIZE 512U + +/* + * Structure tracking client applications + * + * @link - chained list of registered client applications + * @sessions - list of the PKCS11 sessions opened by the client application + * @object_handle_db - Database for object handles in name space of client + */ +struct pkcs11_client { + TAILQ_ENTRY(pkcs11_client) link; + struct session_list session_list; + struct handle_db session_handle_db; + struct handle_db object_handle_db; +}; + +/* Static allocation of tokens runtime instances (reset to 0 at load) */ +struct ck_token ck_token[TOKEN_COUNT]; + +static struct client_list pkcs11_client_list = + TAILQ_HEAD_INITIALIZER(pkcs11_client_list); + +static void close_ck_session(struct pkcs11_session *session); + +struct ck_token *get_token(unsigned int token_id) +{ + if (token_id < TOKEN_COUNT) + return &ck_token[confine_array_index(token_id, TOKEN_COUNT)]; + + return NULL; +} + +unsigned int get_token_id(struct ck_token *token) +{ + ptrdiff_t id = token - ck_token; + + assert(id >= 0 && id < TOKEN_COUNT); + return id; +} + +struct handle_db *get_object_handle_db(struct pkcs11_session *session) +{ + return &session->client->object_handle_db; +} + +struct session_list *get_session_list(struct pkcs11_session *session) +{ + return &session->client->session_list; +} + +struct pkcs11_client *tee_session2client(void *tee_session) +{ + struct pkcs11_client *client = NULL; + + TAILQ_FOREACH(client, &pkcs11_client_list, link) + if (client == tee_session) + break; + + return client; +} + +struct pkcs11_session *pkcs11_handle2session(uint32_t handle, + struct pkcs11_client *client) +{ + return handle_lookup(&client->session_handle_db, handle); +} + +void token_invalidate_object_handles(struct pkcs11_object *obj) +{ + struct pkcs11_client *client = NULL; + uint32_t handle = 0; + + TAILQ_FOREACH(client, &pkcs11_client_list, link) { + handle = handle_lookup_handle(&client->object_handle_db, obj); + if (handle) + handle_invalidate(&client->object_handle_db, handle); + } +} + +struct pkcs11_client *register_client(void) +{ + struct pkcs11_client *client = NULL; + + client = TEE_Malloc(sizeof(*client), TEE_MALLOC_FILL_ZERO); + if (!client) + return NULL; + + TAILQ_INSERT_HEAD(&pkcs11_client_list, client, link); + TAILQ_INIT(&client->session_list); + handle_db_init(&client->session_handle_db); + handle_db_init(&client->object_handle_db); + + return client; +} + +void unregister_client(struct pkcs11_client *client) +{ + struct pkcs11_session *session = NULL; + struct pkcs11_session *next = NULL; + + if (!client) { + EMSG("Invalid TEE session handle"); + return; + } + + TAILQ_FOREACH_SAFE(session, &client->session_list, link, next) + close_ck_session(session); + + TAILQ_REMOVE(&pkcs11_client_list, client, link); + handle_db_destroy(&client->object_handle_db); + handle_db_destroy(&client->session_handle_db); + TEE_Free(client); +} + +static TEE_Result pkcs11_token_init(unsigned int id) +{ + struct ck_token *token = init_persistent_db(id); + + if (!token) + return TEE_ERROR_SECURITY; + + if (token->state == PKCS11_TOKEN_RESET) { + /* As per PKCS#11 spec, token resets to read/write state */ + token->state = PKCS11_TOKEN_READ_WRITE; + token->session_count = 0; + token->rw_session_count = 0; + } + + return TEE_SUCCESS; +} + +TEE_Result pkcs11_init(void) +{ + unsigned int id = 0; + TEE_Result ret = TEE_ERROR_GENERIC; + + for (id = 0; id < TOKEN_COUNT; id++) { + ret = pkcs11_token_init(id); + if (ret) + break; + } + + return ret; +} + +void pkcs11_deinit(void) +{ + unsigned int id = 0; + + for (id = 0; id < TOKEN_COUNT; id++) + close_persistent_db(get_token(id)); +} + +/* + * Currently no support for dual operations. + */ +enum pkcs11_rc set_processing_state(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_object *obj1, + struct pkcs11_object *obj2) +{ + enum pkcs11_proc_state state = PKCS11_SESSION_READY; + struct active_processing *proc = NULL; + + if (session->processing) + return PKCS11_CKR_OPERATION_ACTIVE; + + switch (function) { + case PKCS11_FUNCTION_ENCRYPT: + state = PKCS11_SESSION_ENCRYPTING; + break; + case PKCS11_FUNCTION_DECRYPT: + state = PKCS11_SESSION_DECRYPTING; + break; + case PKCS11_FUNCTION_SIGN: + state = PKCS11_SESSION_SIGNING; + break; + case PKCS11_FUNCTION_VERIFY: + state = PKCS11_SESSION_VERIFYING; + break; + case PKCS11_FUNCTION_DIGEST: + state = PKCS11_SESSION_DIGESTING; + break; + case PKCS11_FUNCTION_DERIVE: + case PKCS11_FUNCTION_WRAP: + case PKCS11_FUNCTION_UNWRAP: + state = PKCS11_SESSION_BUSY; + break; + default: + TEE_Panic(function); + return -1; + } + + proc = TEE_Malloc(sizeof(*proc), TEE_MALLOC_FILL_ZERO); + if (!proc) + return PKCS11_CKR_DEVICE_MEMORY; + + /* Boolean are default to false and pointers to NULL */ + proc->state = state; + proc->step = PKCS11_FUNC_STEP_INIT; + proc->tee_op_handle = TEE_HANDLE_NULL; + proc->tee_hash_algo = 0; + proc->tee_hash_op_handle = TEE_HANDLE_NULL; + + if (obj1 && get_bool(obj1->attributes, PKCS11_CKA_ALWAYS_AUTHENTICATE)) + proc->always_authen = true; + + if (obj2 && get_bool(obj2->attributes, PKCS11_CKA_ALWAYS_AUTHENTICATE)) + proc->always_authen = true; + + session->processing = proc; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_slot_list(uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *out = params + 2; + uint32_t token_id = 0; + const size_t out_size = sizeof(token_id) * TOKEN_COUNT; + uint8_t *id = NULL; + + if (ptypes != exp_pt || + params[0].memref.size != TEE_PARAM0_SIZE_MIN) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (out->memref.size < out_size) { + out->memref.size = out_size; + + if (out->memref.buffer) + return PKCS11_CKR_BUFFER_TOO_SMALL; + else + return PKCS11_CKR_OK; + } + + for (token_id = 0, id = out->memref.buffer; token_id < TOKEN_COUNT; + token_id++, id += sizeof(token_id)) + TEE_MemMove(id, &token_id, sizeof(token_id)); + + out->memref.size = out_size; + + return PKCS11_CKR_OK; +} + +static void pad_str(uint8_t *str, size_t size) +{ + int n = strnlen((char *)str, size); + + TEE_MemFill(str + n, ' ', size - n); +} + +static void set_token_description(struct pkcs11_slot_info *info) +{ + char desc[sizeof(info->slot_description) + 1] = { 0 }; + TEE_UUID dev_id = { }; + TEE_Result res = TEE_ERROR_GENERIC; + int n = 0; + + res = TEE_GetPropertyAsUUID(TEE_PROPSET_TEE_IMPLEMENTATION, + "gpd.tee.deviceID", &dev_id); + if (res == TEE_SUCCESS) { + n = snprintk(desc, sizeof(desc), PKCS11_SLOT_DESCRIPTION + " - TEE UUID %pUl", (void *)&dev_id); + } else { + n = snprintf(desc, sizeof(desc), PKCS11_SLOT_DESCRIPTION + " - No TEE UUID"); + } + if (n < 0 || n >= (int)sizeof(desc)) + TEE_Panic(0); + + TEE_MemMove(info->slot_description, desc, n); + pad_str(info->slot_description, sizeof(info->slot_description)); +} + +enum pkcs11_rc entry_ck_slot_info(uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + uint32_t token_id = 0; + struct pkcs11_slot_info info = { + .slot_description = PKCS11_SLOT_DESCRIPTION, + .manufacturer_id = PKCS11_SLOT_MANUFACTURER, + .flags = PKCS11_CKFS_TOKEN_PRESENT, + .hardware_version = PKCS11_SLOT_HW_VERSION, + .firmware_version = PKCS11_SLOT_FW_VERSION, + }; + + COMPILE_TIME_ASSERT(sizeof(PKCS11_SLOT_DESCRIPTION) <= + sizeof(info.slot_description)); + COMPILE_TIME_ASSERT(sizeof(PKCS11_SLOT_MANUFACTURER) <= + sizeof(info.manufacturer_id)); + + if (ptypes != exp_pt || out->memref.size != sizeof(info)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get(&ctrlargs, &token_id, sizeof(token_id)); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!get_token(token_id)) + return PKCS11_CKR_SLOT_ID_INVALID; + + set_token_description(&info); + + pad_str(info.manufacturer_id, sizeof(info.manufacturer_id)); + + out->memref.size = sizeof(info); + TEE_MemMove(out->memref.buffer, &info, out->memref.size); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_token_info(uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + uint32_t token_id = 0; + struct ck_token *token = NULL; + struct pkcs11_token_info info = { + .manufacturer_id = PKCS11_TOKEN_MANUFACTURER, + .model = PKCS11_TOKEN_MODEL, + .max_session_count = UINT32_MAX, + .max_rw_session_count = UINT32_MAX, + .max_pin_len = PKCS11_TOKEN_PIN_SIZE_MAX, + .min_pin_len = PKCS11_TOKEN_PIN_SIZE_MIN, + .total_public_memory = UINT32_MAX, + .free_public_memory = UINT32_MAX, + .total_private_memory = UINT32_MAX, + .free_private_memory = UINT32_MAX, + .hardware_version = PKCS11_TOKEN_HW_VERSION, + .firmware_version = PKCS11_TOKEN_FW_VERSION, + }; + char sn[sizeof(info.serial_number) + 1] = { 0 }; + int n = 0; + + if (ptypes != exp_pt || out->memref.size != sizeof(info)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get(&ctrlargs, &token_id, sizeof(token_id)); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + token = get_token(token_id); + if (!token) + return PKCS11_CKR_SLOT_ID_INVALID; + + pad_str(info.manufacturer_id, sizeof(info.manufacturer_id)); + pad_str(info.model, sizeof(info.model)); + + n = snprintf(sn, sizeof(sn), "%0*"PRIu32, + (int)sizeof(info.serial_number), token_id); + if (n != (int)sizeof(info.serial_number)) + TEE_Panic(0); + + TEE_MemMove(info.serial_number, sn, sizeof(info.serial_number)); + pad_str(info.serial_number, sizeof(info.serial_number)); + + TEE_MemMove(info.label, token->db_main->label, sizeof(info.label)); + + info.flags = token->db_main->flags; + info.session_count = token->session_count; + info.rw_session_count = token->rw_session_count; + + TEE_MemMove(out->memref.buffer, &info, sizeof(info)); + + return PKCS11_CKR_OK; +} + +static void dmsg_print_supported_mechanism(unsigned int token_id __maybe_unused, + uint32_t *array __maybe_unused, + size_t count __maybe_unused) +{ + size_t __maybe_unused n = 0; + + if (TRACE_LEVEL < TRACE_DEBUG) + return; + + for (n = 0; n < count; n++) + DMSG("PKCS11 token %"PRIu32": mechanism 0x%04"PRIx32": %s", + token_id, array[n], id2str_mechanism(array[n])); +} + +enum pkcs11_rc entry_ck_token_mecha_ids(uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + uint32_t token_id = 0; + struct ck_token __maybe_unused *token = NULL; + size_t count = 0; + uint32_t *array = NULL; + + if (ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get(&ctrlargs, &token_id, sizeof(token_id)); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + token = get_token(token_id); + if (!token) + return PKCS11_CKR_SLOT_ID_INVALID; + + count = out->memref.size / sizeof(*array); + array = tee_malloc_mechanism_list(&count); + + if (out->memref.size < count * sizeof(*array)) { + assert(!array); + out->memref.size = count * sizeof(*array); + if (out->memref.buffer) + return PKCS11_CKR_BUFFER_TOO_SMALL; + else + return PKCS11_CKR_OK; + } + + if (!array) + return PKCS11_CKR_DEVICE_MEMORY; + + dmsg_print_supported_mechanism(token_id, array, count); + + out->memref.size = count * sizeof(*array); + TEE_MemMove(out->memref.buffer, array, out->memref.size); + + TEE_Free(array); + + return rc; +} + +enum pkcs11_rc entry_ck_token_mecha_info(uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + uint32_t token_id = 0; + uint32_t type = 0; + struct ck_token *token = NULL; + struct pkcs11_mechanism_info info = { }; + + if (ptypes != exp_pt || out->memref.size != sizeof(info)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get(&ctrlargs, &token_id, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &type, sizeof(uint32_t)); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + token = get_token(token_id); + if (!token) + return PKCS11_CKR_SLOT_ID_INVALID; + + if (!mechanism_is_valid(type)) + return PKCS11_CKR_MECHANISM_INVALID; + + info.flags = mechanism_supported_flags(type); + + pkcs11_mechanism_supported_key_sizes(type, &info.min_key_size, + &info.max_key_size); + + TEE_MemMove(out->memref.buffer, &info, sizeof(info)); + + DMSG("PKCS11 token %"PRIu32": mechanism 0x%"PRIx32" info", + token_id, type); + + return PKCS11_CKR_OK; +} + +/* Select the ReadOnly or ReadWrite state for session login state */ +static void set_session_state(struct pkcs11_client *client, + struct pkcs11_session *session, bool readonly) +{ + struct pkcs11_session *sess = NULL; + enum pkcs11_session_state state = PKCS11_CKS_RO_PUBLIC_SESSION; + + /* Default to public session if no session already registered */ + if (readonly) + state = PKCS11_CKS_RO_PUBLIC_SESSION; + else + state = PKCS11_CKS_RW_PUBLIC_SESSION; + + /* + * No need to check all client sessions, the first found in + * target token gives client login configuration. + */ + TAILQ_FOREACH(sess, &client->session_list, link) { + assert(sess != session); + + if (sess->token == session->token) { + switch (sess->state) { + case PKCS11_CKS_RW_PUBLIC_SESSION: + case PKCS11_CKS_RO_PUBLIC_SESSION: + if (readonly) + state = PKCS11_CKS_RO_PUBLIC_SESSION; + else + state = PKCS11_CKS_RW_PUBLIC_SESSION; + break; + case PKCS11_CKS_RO_USER_FUNCTIONS: + case PKCS11_CKS_RW_USER_FUNCTIONS: + if (readonly) + state = PKCS11_CKS_RO_USER_FUNCTIONS; + else + state = PKCS11_CKS_RW_USER_FUNCTIONS; + break; + case PKCS11_CKS_RW_SO_FUNCTIONS: + if (readonly) + TEE_Panic(0); + else + state = PKCS11_CKS_RW_SO_FUNCTIONS; + break; + default: + TEE_Panic(0); + } + break; + } + } + + session->state = state; +} + +enum pkcs11_rc entry_ck_open_session(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + uint32_t token_id = 0; + uint32_t flags = 0; + struct ck_token *token = NULL; + struct pkcs11_session *session = NULL; + bool readonly = false; + + if (!client || ptypes != exp_pt || + out->memref.size != sizeof(session->handle)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get(&ctrlargs, &token_id, sizeof(token_id)); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &flags, sizeof(flags)); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + token = get_token(token_id); + if (!token) + return PKCS11_CKR_SLOT_ID_INVALID; + + /* Sanitize session flags */ + if (!(flags & PKCS11_CKFSS_SERIAL_SESSION)) + return PKCS11_CKR_SESSION_PARALLEL_NOT_SUPPORTED; + + if (flags & ~(PKCS11_CKFSS_RW_SESSION | PKCS11_CKFSS_SERIAL_SESSION)) + return PKCS11_CKR_ARGUMENTS_BAD; + + readonly = !(flags & PKCS11_CKFSS_RW_SESSION); + + if (!readonly && token->state == PKCS11_TOKEN_READ_ONLY) + return PKCS11_CKR_TOKEN_WRITE_PROTECTED; + + if (readonly) { + /* Specifically reject read-only session under SO login */ + TAILQ_FOREACH(session, &client->session_list, link) + if (pkcs11_session_is_so(session)) + return PKCS11_CKR_SESSION_READ_WRITE_SO_EXISTS; + } + + session = TEE_Malloc(sizeof(*session), TEE_MALLOC_FILL_ZERO); + if (!session) + return PKCS11_CKR_DEVICE_MEMORY; + + session->handle = handle_get(&client->session_handle_db, session); + if (!session->handle) { + TEE_Free(session); + return PKCS11_CKR_DEVICE_MEMORY; + } + + session->token = token; + session->client = client; + + LIST_INIT(&session->object_list); + + set_session_state(client, session, readonly); + + TAILQ_INSERT_HEAD(&client->session_list, session, link); + + session->token->session_count++; + if (!readonly) + session->token->rw_session_count++; + + TEE_MemMove(out->memref.buffer, &session->handle, + sizeof(session->handle)); + + DMSG("Open PKCS11 session %"PRIu32, session->handle); + + return PKCS11_CKR_OK; +} + +static void close_ck_session(struct pkcs11_session *session) +{ + release_active_processing(session); + release_session_find_obj_context(session); + + /* Release all session objects */ + while (!LIST_EMPTY(&session->object_list)) + destroy_object(session, + LIST_FIRST(&session->object_list), true); + + TAILQ_REMOVE(&session->client->session_list, session, link); + handle_put(&session->client->session_handle_db, session->handle); + + session->token->session_count--; + if (pkcs11_session_is_read_write(session)) + session->token->rw_session_count--; + + DMSG("Close PKCS11 session %"PRIu32, session->handle); + + TEE_Free(session); +} + +enum pkcs11_rc entry_ck_close_session(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + close_ck_session(session); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_close_all_sessions(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + uint32_t token_id = 0; + struct ck_token *token = NULL; + struct pkcs11_session *session = NULL; + struct pkcs11_session *next = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get(&ctrlargs, &token_id, sizeof(uint32_t)); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + token = get_token(token_id); + if (!token) + return PKCS11_CKR_SLOT_ID_INVALID; + + DMSG("Close all sessions for PKCS11 token %"PRIu32, token_id); + + TAILQ_FOREACH_SAFE(session, &client->session_list, link, next) + if (session->token == token) + close_ck_session(session); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_session_info(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_session_info info = { + .flags = PKCS11_CKFSS_SERIAL_SESSION, + }; + + if (!client || ptypes != exp_pt || out->memref.size != sizeof(info)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + info.slot_id = get_token_id(session->token); + info.state = session->state; + if (pkcs11_session_is_read_write(session)) + info.flags |= PKCS11_CKFSS_RW_SESSION; + + TEE_MemMove(out->memref.buffer, &info, sizeof(info)); + + DMSG("Get find on PKCS11 session %"PRIu32, session->handle); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_token_initialize(uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + char label[PKCS11_TOKEN_LABEL_SIZE] = { 0 }; + struct pkcs11_client *client = NULL; + struct pkcs11_session *sess = NULL; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct ck_token *token = NULL; + TEE_Param *ctrl = params; + uint32_t token_id = 0; + uint32_t pin_size = 0; + void *pin = NULL; + struct pkcs11_object *obj = NULL; + + if (ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get(&ctrlargs, &token_id, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &pin_size, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &label, PKCS11_TOKEN_LABEL_SIZE); + if (rc) + return rc; + + rc = serialargs_get_ptr(&ctrlargs, &pin, pin_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + token = get_token(token_id); + if (!token) + return PKCS11_CKR_SLOT_ID_INVALID; + + if (token->db_main->flags & PKCS11_CKFT_SO_PIN_LOCKED) { + IMSG("Token %"PRIu32": SO PIN locked", token_id); + return PKCS11_CKR_PIN_LOCKED; + } + + /* Check there's no open session on this token */ + TAILQ_FOREACH(client, &pkcs11_client_list, link) + TAILQ_FOREACH(sess, &client->session_list, link) + if (sess->token == token) + return PKCS11_CKR_SESSION_EXISTS; + + /* Verify authentication if token is already initialized */ + if (token->db_main->flags & PKCS11_CKFT_TOKEN_INITIALIZED) { + unsigned int pin_count = 0; + + if (IS_ENABLED(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) && + (token->db_main->flags & + PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH)) { + /* Check TEE Identity based authentication if enabled */ + rc = verify_identity_auth(token, PKCS11_CKU_SO); + if (rc) + return rc; + + goto inited; + } + + /* Check PIN based authentication */ + rc = verify_pin(PKCS11_CKU_SO, pin, pin_size, + token->db_main->so_pin_salt, + token->db_main->so_pin_hash); + if (!rc) + goto inited; + + if (rc != PKCS11_CKR_PIN_INCORRECT) + return rc; + + token->db_main->flags |= PKCS11_CKFT_SO_PIN_COUNT_LOW; + token->db_main->so_pin_count++; + + pin_count = token->db_main->so_pin_count; + if (pin_count == PKCS11_TOKEN_SO_PIN_COUNT_MAX - 1) + token->db_main->flags |= PKCS11_CKFT_SO_PIN_FINAL_TRY; + if (pin_count == PKCS11_TOKEN_SO_PIN_COUNT_MAX) + token->db_main->flags |= PKCS11_CKFT_SO_PIN_LOCKED; + + update_persistent_db(token); + + return PKCS11_CKR_PIN_INCORRECT; + } + + /* Initialize SO's authentication */ + if (IS_ENABLED(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) && !pin) { + rc = setup_so_identity_auth_from_client(token); + if (rc) + return rc; + } else { + /* + * The spec doesn't permit returning + * PKCS11_CKR_PIN_LEN_RANGE for this function, take another + * error code. + */ + if (pin_size < PKCS11_TOKEN_PIN_SIZE_MIN || + pin_size > PKCS11_TOKEN_PIN_SIZE_MAX) + return PKCS11_CKR_ARGUMENTS_BAD; + + rc = hash_pin(PKCS11_CKU_SO, pin, pin_size, + &token->db_main->so_pin_salt, + token->db_main->so_pin_hash); + if (rc) + return rc; + } + +inited: + /* Make sure SO PIN counters are zeroed */ + token->db_main->flags &= ~(PKCS11_CKFT_SO_PIN_COUNT_LOW | + PKCS11_CKFT_SO_PIN_FINAL_TRY | + PKCS11_CKFT_SO_PIN_LOCKED | + PKCS11_CKFT_SO_PIN_TO_BE_CHANGED); + token->db_main->so_pin_count = 0; + + TEE_MemMove(token->db_main->label, label, PKCS11_TOKEN_LABEL_SIZE); + token->db_main->flags |= PKCS11_CKFT_TOKEN_INITIALIZED; + /* Reset user PIN */ + token->db_main->user_pin_salt = 0; + token->db_main->flags &= ~(PKCS11_CKFT_USER_PIN_INITIALIZED | + PKCS11_CKFT_USER_PIN_COUNT_LOW | + PKCS11_CKFT_USER_PIN_FINAL_TRY | + PKCS11_CKFT_USER_PIN_LOCKED | + PKCS11_CKFT_USER_PIN_TO_BE_CHANGED); + + update_persistent_db(token); + + /* Remove all persistent objects */ + while (!LIST_EMPTY(&token->object_list)) { + obj = LIST_FIRST(&token->object_list); + + /* Try twice otherwise panic! */ + if (unregister_persistent_object(token, obj->uuid) && + unregister_persistent_object(token, obj->uuid)) + TEE_Panic(0); + + cleanup_persistent_object(obj, token); + } + + IMSG("PKCS11 token %"PRIu32": initialized", token_id); + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc set_pin(struct pkcs11_session *session, + uint8_t *new_pin, size_t new_pin_size, + enum pkcs11_user_type user_type) +{ + struct ck_token *token = session->token; + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint32_t flags_clear = 0; + uint32_t flags_set = 0; + + if (token->db_main->flags & PKCS11_CKFT_WRITE_PROTECTED) + return PKCS11_CKR_TOKEN_WRITE_PROTECTED; + + if (!pkcs11_session_is_read_write(session)) + return PKCS11_CKR_SESSION_READ_ONLY; + + if (IS_ENABLED(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) && + (token->db_main->flags & + PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH)) { + rc = setup_identity_auth_from_pin(token, user_type, + new_pin, new_pin_size); + if (rc == PKCS11_CKR_OK) { + goto update_db; + } else if (rc == PKCS11_CKR_PIN_INVALID && + !(token->db_main->flags & + PKCS11_CKFT_USER_PIN_INITIALIZED)) { + /* + * PIN was not compatible with TEE Identity + * Authentication syntax so let's assume it might be a + * new SO PIN to switch the authentication mode. + */ + + /* Update mode flag if all PIN checks pass */ + flags_clear |= + PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH; + } else { + return rc; + } + } else if ((user_type == PKCS11_CKU_SO) && !new_pin && + IS_ENABLED(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) && + !(token->db_main->flags & + PKCS11_CKFT_USER_PIN_INITIALIZED)) { + /* + * Allow changing of token authentication mode before user pin + * has been initialized. + */ + + /* + * Set protected authentication path temporary until + * finalized. + */ + token->db_main->flags |= + PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH; + + /* + * Setup authentication from currently active TEE Client + * Identity. + */ + rc = setup_identity_auth_from_pin(token, PKCS11_CKU_SO, + NULL, 0); + if (rc) { + /* + * Failed to setup protected authentication path so + * clear the temporary flag. + */ + token->db_main->flags &= + ~PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH; + return rc; + } + + goto update_db; + } + + if (new_pin_size < PKCS11_TOKEN_PIN_SIZE_MIN || + new_pin_size > PKCS11_TOKEN_PIN_SIZE_MAX) + return PKCS11_CKR_PIN_LEN_RANGE; + + switch (user_type) { + case PKCS11_CKU_SO: + rc = hash_pin(user_type, new_pin, new_pin_size, + &token->db_main->so_pin_salt, + token->db_main->so_pin_hash); + if (rc) + return rc; + token->db_main->so_pin_count = 0; + flags_clear |= PKCS11_CKFT_SO_PIN_COUNT_LOW | + PKCS11_CKFT_SO_PIN_FINAL_TRY | + PKCS11_CKFT_SO_PIN_LOCKED | + PKCS11_CKFT_SO_PIN_TO_BE_CHANGED; + break; + case PKCS11_CKU_USER: + rc = hash_pin(user_type, new_pin, new_pin_size, + &token->db_main->user_pin_salt, + token->db_main->user_pin_hash); + if (rc) + return rc; + token->db_main->user_pin_count = 0; + flags_clear |= PKCS11_CKFT_USER_PIN_COUNT_LOW | + PKCS11_CKFT_USER_PIN_FINAL_TRY | + PKCS11_CKFT_USER_PIN_LOCKED | + PKCS11_CKFT_USER_PIN_TO_BE_CHANGED; + flags_set |= PKCS11_CKFT_USER_PIN_INITIALIZED; + break; + default: + return PKCS11_CKR_FUNCTION_FAILED; + } + +update_db: + token->db_main->flags &= ~flags_clear; + token->db_main->flags |= flags_set; + + update_persistent_db(token); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_init_pin(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + struct pkcs11_session *session = NULL; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + TEE_Param *ctrl = params; + uint32_t pin_size = 0; + void *pin = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &pin_size, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get_ptr(&ctrlargs, &pin, pin_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!pkcs11_session_is_so(session)) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + + assert(session->token->db_main->flags & PKCS11_CKFT_TOKEN_INITIALIZED); + + IMSG("PKCS11 session %"PRIu32": init PIN", session->handle); + + return set_pin(session, pin, pin_size, PKCS11_CKU_USER); +} + +static enum pkcs11_rc check_so_pin(struct pkcs11_session *session, + uint8_t *pin, size_t pin_size) +{ + struct ck_token *token = session->token; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(token->db_main->flags & PKCS11_CKFT_TOKEN_INITIALIZED); + + if (IS_ENABLED(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) && + token->db_main->flags & PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH) + return verify_identity_auth(token, PKCS11_CKU_SO); + + if (token->db_main->flags & PKCS11_CKFT_SO_PIN_LOCKED) + return PKCS11_CKR_PIN_LOCKED; + + rc = verify_pin(PKCS11_CKU_SO, pin, pin_size, + token->db_main->so_pin_salt, + token->db_main->so_pin_hash); + if (rc) { + unsigned int pin_count = 0; + + if (rc != PKCS11_CKR_PIN_INCORRECT) + return rc; + + token->db_main->flags |= PKCS11_CKFT_SO_PIN_COUNT_LOW; + token->db_main->so_pin_count++; + + pin_count = token->db_main->so_pin_count; + if (pin_count == PKCS11_TOKEN_SO_PIN_COUNT_MAX - 1) + token->db_main->flags |= PKCS11_CKFT_SO_PIN_FINAL_TRY; + if (pin_count == PKCS11_TOKEN_SO_PIN_COUNT_MAX) + token->db_main->flags |= PKCS11_CKFT_SO_PIN_LOCKED; + + update_persistent_db(token); + + if (token->db_main->flags & PKCS11_CKFT_SO_PIN_LOCKED) + return PKCS11_CKR_PIN_LOCKED; + + return PKCS11_CKR_PIN_INCORRECT; + } + + if (token->db_main->so_pin_count) { + token->db_main->so_pin_count = 0; + + update_persistent_db(token); + } + + if (token->db_main->flags & (PKCS11_CKFT_SO_PIN_COUNT_LOW | + PKCS11_CKFT_SO_PIN_FINAL_TRY)) { + token->db_main->flags &= ~(PKCS11_CKFT_SO_PIN_COUNT_LOW | + PKCS11_CKFT_SO_PIN_FINAL_TRY); + + update_persistent_db(token); + } + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc check_user_pin(struct pkcs11_session *session, + uint8_t *pin, size_t pin_size) +{ + struct ck_token *token = session->token; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + if (IS_ENABLED(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) && + token->db_main->flags & PKCS11_CKFT_PROTECTED_AUTHENTICATION_PATH) + return verify_identity_auth(token, PKCS11_CKU_USER); + + if (!token->db_main->user_pin_salt) + return PKCS11_CKR_USER_PIN_NOT_INITIALIZED; + + if (token->db_main->flags & PKCS11_CKFT_USER_PIN_LOCKED) + return PKCS11_CKR_PIN_LOCKED; + + rc = verify_pin(PKCS11_CKU_USER, pin, pin_size, + token->db_main->user_pin_salt, + token->db_main->user_pin_hash); + if (rc) { + unsigned int pin_count = 0; + + if (rc != PKCS11_CKR_PIN_INCORRECT) + return rc; + + token->db_main->flags |= PKCS11_CKFT_USER_PIN_COUNT_LOW; + token->db_main->user_pin_count++; + + pin_count = token->db_main->user_pin_count; + if (pin_count == PKCS11_TOKEN_USER_PIN_COUNT_MAX - 1) + token->db_main->flags |= PKCS11_CKFT_USER_PIN_FINAL_TRY; + if (pin_count == PKCS11_TOKEN_USER_PIN_COUNT_MAX) + token->db_main->flags |= PKCS11_CKFT_USER_PIN_LOCKED; + + update_persistent_db(token); + + if (token->db_main->flags & PKCS11_CKFT_USER_PIN_LOCKED) + return PKCS11_CKR_PIN_LOCKED; + + return PKCS11_CKR_PIN_INCORRECT; + } + + if (token->db_main->user_pin_count) { + token->db_main->user_pin_count = 0; + + update_persistent_db(token); + } + + if (token->db_main->flags & (PKCS11_CKFT_USER_PIN_COUNT_LOW | + PKCS11_CKFT_USER_PIN_FINAL_TRY)) { + token->db_main->flags &= ~(PKCS11_CKFT_USER_PIN_COUNT_LOW | + PKCS11_CKFT_USER_PIN_FINAL_TRY); + + update_persistent_db(token); + } + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_set_pin(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + struct pkcs11_session *session = NULL; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + uint32_t old_pin_size = 0; + TEE_Param *ctrl = params; + uint32_t pin_size = 0; + void *old_pin = NULL; + void *pin = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &old_pin_size, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &pin_size, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get_ptr(&ctrlargs, &old_pin, old_pin_size); + if (rc) + return rc; + + rc = serialargs_get_ptr(&ctrlargs, &pin, pin_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!pkcs11_session_is_read_write(session)) + return PKCS11_CKR_SESSION_READ_ONLY; + + if (pkcs11_session_is_so(session)) { + if (!(session->token->db_main->flags & + PKCS11_CKFT_TOKEN_INITIALIZED)) + return PKCS11_CKR_GENERAL_ERROR; + + rc = check_so_pin(session, old_pin, old_pin_size); + if (rc) + return rc; + + IMSG("PKCS11 session %"PRIu32": set PIN", session->handle); + + return set_pin(session, pin, pin_size, PKCS11_CKU_SO); + } + + if (!(session->token->db_main->flags & + PKCS11_CKFT_USER_PIN_INITIALIZED)) + return PKCS11_CKR_GENERAL_ERROR; + + rc = check_user_pin(session, old_pin, old_pin_size); + if (rc) + return rc; + + IMSG("PKCS11 session %"PRIu32": set PIN", session->handle); + + return set_pin(session, pin, pin_size, PKCS11_CKU_USER); +} + +static void session_login_user(struct pkcs11_session *session) +{ + struct pkcs11_client *client = session->client; + struct pkcs11_session *sess = NULL; + + TAILQ_FOREACH(sess, &client->session_list, link) { + if (sess->token != session->token) + continue; + + if (pkcs11_session_is_read_write(sess)) + sess->state = PKCS11_CKS_RW_USER_FUNCTIONS; + else + sess->state = PKCS11_CKS_RO_USER_FUNCTIONS; + } +} + +static void session_login_so(struct pkcs11_session *session) +{ + struct pkcs11_client *client = session->client; + struct pkcs11_session *sess = NULL; + + TAILQ_FOREACH(sess, &client->session_list, link) { + if (sess->token != session->token) + continue; + + if (pkcs11_session_is_read_write(sess)) + sess->state = PKCS11_CKS_RW_SO_FUNCTIONS; + else + TEE_Panic(0); + } +} + +static void session_logout(struct pkcs11_session *session) +{ + struct pkcs11_client *client = session->client; + struct pkcs11_session *sess = NULL; + + TAILQ_FOREACH(sess, &client->session_list, link) { + struct pkcs11_object *obj = NULL; + struct pkcs11_object *tobj = NULL; + uint32_t handle = 0; + + if (sess->token != session->token) + continue; + + release_active_processing(session); + + /* Destroy private session objects */ + LIST_FOREACH_SAFE(obj, &sess->object_list, link, tobj) { + if (object_is_private(obj->attributes)) + destroy_object(sess, obj, true); + } + + /* + * Remove handle of token private objects from + * sessions object_handle_db + */ + LIST_FOREACH(obj, &session->token->object_list, link) { + handle = pkcs11_object2handle(obj, session); + + if (handle && object_is_private(obj->attributes)) + handle_put(get_object_handle_db(sess), handle); + } + + release_session_find_obj_context(session); + + if (pkcs11_session_is_read_write(sess)) + sess->state = PKCS11_CKS_RW_PUBLIC_SESSION; + else + sess->state = PKCS11_CKS_RO_PUBLIC_SESSION; + } +} + +enum pkcs11_rc entry_ck_login(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + struct pkcs11_session *session = NULL; + struct pkcs11_session *sess = NULL; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + TEE_Param *ctrl = params; + uint32_t user_type = 0; + uint32_t pin_size = 0; + void *pin = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &user_type, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &pin_size, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get_ptr(&ctrlargs, &pin, pin_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + switch (user_type) { + case PKCS11_CKU_SO: + if (pkcs11_session_is_so(session)) + return PKCS11_CKR_USER_ALREADY_LOGGED_IN; + + if (pkcs11_session_is_user(session)) + return PKCS11_CKR_USER_ANOTHER_ALREADY_LOGGED_IN; + + TAILQ_FOREACH(sess, &client->session_list, link) + if (sess->token == session->token && + !pkcs11_session_is_read_write(sess)) + return PKCS11_CKR_SESSION_READ_ONLY_EXISTS; + + /* + * This is the point where we could check if another client + * has another user or SO logged in. + * + * The spec says: + * CKR_USER_TOO_MANY_TYPES: An attempt was made to have + * more distinct users simultaneously logged into the token + * than the token and/or library permits. For example, if + * some application has an open SO session, and another + * application attempts to log the normal user into a + * session, the attempt may return this error. It is not + * required to, however. Only if the simultaneous distinct + * users cannot be supported does C_Login have to return + * this value. Note that this error code generalizes to + * true multi-user tokens. + * + * So it's permitted to have another user or SO logged in + * from another client. + */ + + rc = check_so_pin(session, pin, pin_size); + if (!rc) + session_login_so(session); + + break; + + case PKCS11_CKU_USER: + if (pkcs11_session_is_so(session)) + return PKCS11_CKR_USER_ANOTHER_ALREADY_LOGGED_IN; + + if (pkcs11_session_is_user(session)) + return PKCS11_CKR_USER_ALREADY_LOGGED_IN; + + /* + * This is the point where we could check if another client + * has another user or SO logged in. + * See comment on CKR_USER_TOO_MANY_TYPES above. + */ + + rc = check_user_pin(session, pin, pin_size); + if (!rc) + session_login_user(session); + + break; + + case PKCS11_CKU_CONTEXT_SPECIFIC: + return PKCS11_CKR_OPERATION_NOT_INITIALIZED; + + default: + return PKCS11_CKR_USER_TYPE_INVALID; + } + + if (!rc) + IMSG("PKCS11 session %"PRIu32": login", session->handle); + + return rc; +} + +enum pkcs11_rc entry_ck_logout(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + struct pkcs11_session *session = NULL; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + TEE_Param *ctrl = params; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (pkcs11_session_is_public(session)) + return PKCS11_CKR_USER_NOT_LOGGED_IN; + + session_logout(session); + + IMSG("PKCS11 session %"PRIu32": logout", session->handle); + + return PKCS11_CKR_OK; +} + +static TEE_Result seed_rng_pool(void *seed, size_t length) +{ + static const TEE_UUID system_uuid = PTA_SYSTEM_UUID; + uint32_t param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param params[TEE_NUM_PARAMS] = { }; + TEE_TASessionHandle sess = TEE_HANDLE_NULL; + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t ret_orig = 0; + + params[0].memref.buffer = seed; + params[0].memref.size = (uint32_t)length; + + res = TEE_OpenTASession(&system_uuid, TEE_TIMEOUT_INFINITE, 0, NULL, + &sess, &ret_orig); + if (res != TEE_SUCCESS) { + EMSG("Can't open session to system PTA"); + return res; + } + + res = TEE_InvokeTACommand(sess, TEE_TIMEOUT_INFINITE, + PTA_SYSTEM_ADD_RNG_ENTROPY, + param_types, params, &ret_orig); + if (res != TEE_SUCCESS) + EMSG("Can't invoke system PTA"); + + TEE_CloseTASession(sess); + return res; +} + +enum pkcs11_rc entry_ck_seed_random(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *in = params + 1; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + TEE_Result res = TEE_SUCCESS; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (in->memref.size && !in->memref.buffer) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!in->memref.size) + return PKCS11_CKR_OK; + + res = seed_rng_pool(in->memref.buffer, in->memref.size); + if (res != TEE_SUCCESS) + return PKCS11_CKR_FUNCTION_FAILED; + + DMSG("PKCS11 session %"PRIu32": seed random", session->handle); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_ck_generate_random(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + void *buffer = NULL; + size_t buffer_size = 0; + uint8_t *data = NULL; + size_t left = 0; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (out->memref.size && !out->memref.buffer) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!out->memref.size) + return PKCS11_CKR_OK; + + buffer_size = MIN(out->memref.size, RNG_CHUNK_SIZE); + buffer = TEE_Malloc(buffer_size, TEE_MALLOC_FILL_ZERO); + if (!buffer) + return PKCS11_CKR_DEVICE_MEMORY; + + data = out->memref.buffer; + left = out->memref.size; + + while (left) { + size_t count = MIN(left, buffer_size); + + TEE_GenerateRandom(buffer, count); + TEE_MemMove(data, buffer, count); + + data += count; + left -= count; + } + + DMSG("PKCS11 session %"PRIu32": generate random", session->handle); + + TEE_Free(buffer); + + return PKCS11_CKR_OK; +} diff --git a/optee_os/ta/pkcs11/src/pkcs11_token.h b/optee_os/ta/pkcs11/src/pkcs11_token.h new file mode 100644 index 0000000..062f1ff --- /dev/null +++ b/optee_os/ta/pkcs11/src/pkcs11_token.h @@ -0,0 +1,356 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ +#ifndef PKCS11_TA_PKCS11_TOKEN_H +#define PKCS11_TA_PKCS11_TOKEN_H + +#include +#include +#include +#include + +#include "handle.h" +#include "object.h" +#include "pkcs11_attributes.h" + +/* Hard coded description */ +#define PKCS11_SLOT_DESCRIPTION "OP-TEE PKCS11 TA" +#define PKCS11_SLOT_MANUFACTURER "Linaro" +#define PKCS11_SLOT_HW_VERSION { 0, 0 } +#define PKCS11_SLOT_FW_VERSION { PKCS11_TA_VERSION_MAJOR, \ + PKCS11_TA_VERSION_MINOR } + +#define PKCS11_TOKEN_LABEL "OP-TEE PKCS#11 TA token" +#define PKCS11_TOKEN_MANUFACTURER PKCS11_SLOT_MANUFACTURER +#define PKCS11_TOKEN_MODEL "OP-TEE TA" +#define PKCS11_TOKEN_HW_VERSION PKCS11_SLOT_HW_VERSION +#define PKCS11_TOKEN_FW_VERSION PKCS11_SLOT_FW_VERSION + +enum pkcs11_token_state { + PKCS11_TOKEN_RESET = 0, + PKCS11_TOKEN_READ_WRITE, + PKCS11_TOKEN_READ_ONLY, +}; + +TAILQ_HEAD(client_list, pkcs11_client); +TAILQ_HEAD(session_list, pkcs11_session); + +struct pkcs11_client; + +#define PKCS11_MAX_USERS 2 +#define PKCS11_TOKEN_PIN_SIZE_MAX 128 +#define PKCS11_TOKEN_PIN_SIZE_MIN 4 +#define PKCS11_TOKEN_SO_PIN_COUNT_MAX 7 +#define PKCS11_TOKEN_USER_PIN_COUNT_MAX 7 + +/* + * Persistent state of the token + * + * @version - currently unused... + * @label - pkcs11 formatted token label, set by client + * @flags - pkcs11 token flags + * @so_pin_count - counter on security officer login failure + * @so_pin_salt - stores salt in hash of SO PIN, 0 if not set + * @so_pin_hash - stores hash of SO PIN + * @user_pin_count - counter on user login failure + * @user_pin_salt - stores salt in hash of user PIN, 0 if not set + * @user_pin_hash - stores hash of user PIN + */ +struct token_persistent_main { + uint32_t version; + uint8_t label[PKCS11_TOKEN_LABEL_SIZE]; + uint32_t flags; + uint32_t so_pin_count; + uint32_t so_pin_salt; + union { + uint8_t so_pin_hash[TEE_MAX_HASH_SIZE]; + TEE_Identity so_identity; + }; + uint32_t user_pin_count; + uint32_t user_pin_salt; + union { + uint8_t user_pin_hash[TEE_MAX_HASH_SIZE]; + TEE_Identity user_identity; + }; +}; + +/* + * Persistent objects in the token + * + * @count - number of objects stored in the token + * @uuids - array of object references/UUIDs (@count items) + */ +struct token_persistent_objs { + uint32_t count; + TEE_UUID uuids[]; +}; + +/* + * Runtime state of the token, complies with pkcs11 + * + * @state - Pkcs11 login is public, user, SO or custom + * @session_count - Counter for opened Pkcs11 sessions + * @rw_session_count - Count for opened Pkcs11 read/write sessions + * @object_list - List of the objects owned by the token + * @db_main - Volatile copy of the persistent main database + * @db_objs - Volatile copy of the persistent object database + */ +struct ck_token { + enum pkcs11_token_state state; + uint32_t session_count; + uint32_t rw_session_count; + struct object_list object_list; + /* Copy in RAM of the persistent database */ + struct token_persistent_main *db_main; + struct token_persistent_objs *db_objs; +}; + +/* + * A session can enter a processing state (encrypt, decrypt, digest, ...) + * only from the initialized state. A session must return the initialized + * state (from a processing finalization request) before entering another + * processing state. + */ +enum pkcs11_proc_state { + PKCS11_SESSION_READY = 0, /* No active processing */ + PKCS11_SESSION_ENCRYPTING, + PKCS11_SESSION_DECRYPTING, + PKCS11_SESSION_DIGESTING, + PKCS11_SESSION_DIGESTING_ENCRYPTING, /* case C_DigestEncryptUpdate */ + PKCS11_SESSION_DECRYPTING_DIGESTING, /* case C_DecryptDigestUpdate */ + PKCS11_SESSION_SIGNING, + PKCS11_SESSION_SIGNING_ENCRYPTING, /* case C_SignEncryptUpdate */ + PKCS11_SESSION_VERIFYING, + PKCS11_SESSION_DECRYPTING_VERIFYING, /* case C_DecryptVerifyUpdate */ + PKCS11_SESSION_SIGNING_RECOVER, + PKCS11_SESSION_VERIFYING_RECOVER, + PKCS11_SESSION_BUSY, +}; + +/* + * Context of the active processing in the session + * + * @state - ongoing active processing function or ready state + * @mecha_type - mechanism type of the active processing + * @always_authen - true if user need to login before each use + * @relogged - true once client logged since last operation update + * @op_step - last active operation step - update, final or one-shot + * @tee_op_handle - handle on active crypto operation or TEE_HANDLE_NULL + * @tee_hash_algo - hash algorithm identifier. + * @tee_hash_op_handle - handle on active hashing crypto operation or + * TEE_HANDLE_NULL + * @extra_ctx - context for the active processing + */ +struct active_processing { + enum pkcs11_proc_state state; + uint32_t mecha_type; + enum processing_step step; + bool always_authen; + bool relogged; + TEE_OperationHandle tee_op_handle; + uint32_t tee_hash_algo; + TEE_OperationHandle tee_hash_op_handle; + void *extra_ctx; +}; + +/* + * Pkcs11 objects search context + * + * @attributes - matching attributes list searched (null if no search) + * @count - number of matching handle found + * @handles - array of handle of matching objects + * @next - index of the next object handle to return to C_FindObject + */ +struct pkcs11_find_objects { + void *attributes; + size_t count; + uint32_t *handles; + size_t next; +}; + +/* + * Structure tracking the PKCS#11 sessions + * + * @link - List of the session belonging to a client + * @client - Client the session belongs to + * @token - Token this session belongs to + * @handle - Identifier of the session published to the client + * @object_list - Entry of the session objects list + * @state - R/W SO, R/W user, RO user, R/W public, RO public. + * @processing - Reference to initialized processing context if any + * @find_ctx - Reference to active search context (null if no active search) + */ +struct pkcs11_session { + TAILQ_ENTRY(pkcs11_session) link; + struct pkcs11_client *client; + struct ck_token *token; + enum pkcs11_mechanism_id handle; + struct object_list object_list; + enum pkcs11_session_state state; + struct active_processing *processing; + struct pkcs11_find_objects *find_ctx; +}; + +/* Initialize static token instance(s) from default/persistent database */ +TEE_Result pkcs11_init(void); +void pkcs11_deinit(void); + +/* Speculation safe lookup of token instance from token identifier */ +struct ck_token *get_token(unsigned int token_id); + +/* Return token identified from token instance address */ +unsigned int get_token_id(struct ck_token *token); + +/* Return client's (shared) object handle database associated with session */ +struct handle_db *get_object_handle_db(struct pkcs11_session *session); + +/* Access to persistent database */ +struct ck_token *init_persistent_db(unsigned int token_id); +void update_persistent_db(struct ck_token *token); +void close_persistent_db(struct ck_token *token); + +/* Load and release persistent object attributes in memory */ +enum pkcs11_rc load_persistent_object_attributes(struct pkcs11_object *obj); +void release_persistent_object_attributes(struct pkcs11_object *obj); +enum pkcs11_rc update_persistent_object_attributes(struct pkcs11_object *obj); + +enum pkcs11_rc hash_pin(enum pkcs11_user_type user, const uint8_t *pin, + size_t pin_size, uint32_t *salt, + uint8_t hash[TEE_MAX_HASH_SIZE]); +enum pkcs11_rc verify_pin(enum pkcs11_user_type user, const uint8_t *pin, + size_t pin_size, uint32_t salt, + const uint8_t hash[TEE_MAX_HASH_SIZE]); + +#if defined(CFG_PKCS11_TA_AUTH_TEE_IDENTITY) +enum pkcs11_rc setup_so_identity_auth_from_client(struct ck_token *token); +enum pkcs11_rc setup_identity_auth_from_pin(struct ck_token *token, + enum pkcs11_user_type user_type, + const uint8_t *pin, + size_t pin_size); +enum pkcs11_rc verify_identity_auth(struct ck_token *token, + enum pkcs11_user_type user_type); +#else +static inline enum pkcs11_rc +setup_so_identity_auth_from_client(struct ck_token *token __unused) +{ + return PKCS11_CKR_PIN_INVALID; +} + +static inline enum pkcs11_rc +setup_identity_auth_from_pin(struct ck_token *token __unused, + enum pkcs11_user_type user_type __unused, + const uint8_t *pin __unused, + size_t pin_size __unused) +{ + return PKCS11_CKR_PIN_INVALID; +} + +static inline enum pkcs11_rc +verify_identity_auth(struct ck_token *token __unused, + enum pkcs11_user_type user_type __unused) +{ + return PKCS11_CKR_PIN_INCORRECT; +} +#endif /* CFG_PKCS11_TA_AUTH_TEE_IDENTITY */ + +/* Token persistent objects */ +enum pkcs11_rc create_object_uuid(struct ck_token *token, + struct pkcs11_object *obj); +void destroy_object_uuid(struct ck_token *token, struct pkcs11_object *obj); +enum pkcs11_rc unregister_persistent_object(struct ck_token *token, + TEE_UUID *uuid); +enum pkcs11_rc register_persistent_object(struct ck_token *token, + TEE_UUID *uuid); +enum pkcs11_rc get_persistent_objects_list(struct ck_token *token, + TEE_UUID *array, size_t *size); + +/* + * Pkcs11 session support + */ +struct session_list *get_session_list(struct pkcs11_session *session); +struct pkcs11_client *tee_session2client(void *tee_session); +struct pkcs11_client *register_client(void); +void unregister_client(struct pkcs11_client *client); + +struct pkcs11_session *pkcs11_handle2session(uint32_t handle, + struct pkcs11_client *client); + +static inline bool session_is_active(struct pkcs11_session *session) +{ + return session->processing; +} + +enum pkcs11_rc set_processing_state(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_object *obj1, + struct pkcs11_object *obj2); + +static inline bool pkcs11_session_is_read_write(struct pkcs11_session *session) +{ + return session->state == PKCS11_CKS_RW_PUBLIC_SESSION || + session->state == PKCS11_CKS_RW_USER_FUNCTIONS || + session->state == PKCS11_CKS_RW_SO_FUNCTIONS; +} + +static inline bool pkcs11_session_is_public(struct pkcs11_session *session) +{ + return session->state == PKCS11_CKS_RO_PUBLIC_SESSION || + session->state == PKCS11_CKS_RW_PUBLIC_SESSION; +} + +static inline bool pkcs11_session_is_user(struct pkcs11_session *session) +{ + return session->state == PKCS11_CKS_RO_USER_FUNCTIONS || + session->state == PKCS11_CKS_RW_USER_FUNCTIONS; +} + +static inline bool pkcs11_session_is_so(struct pkcs11_session *session) +{ + return session->state == PKCS11_CKS_RW_SO_FUNCTIONS; +} + +static inline +struct object_list *pkcs11_get_session_objects(struct pkcs11_session *session) +{ + return &session->object_list; +} + +static inline +struct ck_token *pkcs11_session2token(struct pkcs11_session *session) +{ + return session->token; +} + +/* Invalidate any handle referring the object since the object no more exists */ +void token_invalidate_object_handles(struct pkcs11_object *obj); + +/* Entry point for the TA commands */ +enum pkcs11_rc entry_ck_slot_list(uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_slot_info(uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_token_info(uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_token_mecha_ids(uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_token_mecha_info(uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_open_session(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_close_session(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_close_all_sessions(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_session_info(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_token_initialize(uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_init_pin(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_set_pin(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_login(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_logout(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_seed_random(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); +enum pkcs11_rc entry_ck_generate_random(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +#endif /*PKCS11_TA_PKCS11_TOKEN_H*/ diff --git a/optee_os/ta/pkcs11/src/processing.c b/optee_os/ta/pkcs11/src/processing.c new file mode 100644 index 0000000..6be3482 --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing.c @@ -0,0 +1,1287 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "object.h" +#include "pkcs11_attributes.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" +#include "serializer.h" + +static enum pkcs11_rc get_ready_session(struct pkcs11_session *session) +{ + if (session_is_active(session)) + return PKCS11_CKR_OPERATION_ACTIVE; + + return PKCS11_CKR_OK; +} + +static enum processing_func func_for_cmd(enum pkcs11_ta_cmd cmd) +{ + switch (cmd) { + case PKCS11_CMD_ENCRYPT_UPDATE: + case PKCS11_CMD_ENCRYPT_ONESHOT: + case PKCS11_CMD_ENCRYPT_FINAL: + return PKCS11_FUNCTION_ENCRYPT; + case PKCS11_CMD_DECRYPT_UPDATE: + case PKCS11_CMD_DECRYPT_ONESHOT: + case PKCS11_CMD_DECRYPT_FINAL: + return PKCS11_FUNCTION_DECRYPT; + case PKCS11_CMD_SIGN_ONESHOT: + case PKCS11_CMD_SIGN_UPDATE: + case PKCS11_CMD_SIGN_FINAL: + return PKCS11_FUNCTION_SIGN; + case PKCS11_CMD_VERIFY_ONESHOT: + case PKCS11_CMD_VERIFY_UPDATE: + case PKCS11_CMD_VERIFY_FINAL: + return PKCS11_FUNCTION_VERIFY; + case PKCS11_CMD_DIGEST_UPDATE: + case PKCS11_CMD_DIGEST_KEY: + case PKCS11_CMD_DIGEST_ONESHOT: + case PKCS11_CMD_DIGEST_FINAL: + return PKCS11_FUNCTION_DIGEST; + default: + return PKCS11_FUNCTION_UNKNOWN; + } +} + +static bool func_matches_state(enum processing_func function, + enum pkcs11_proc_state state) +{ + switch (function) { + case PKCS11_FUNCTION_ENCRYPT: + return state == PKCS11_SESSION_ENCRYPTING || + state == PKCS11_SESSION_DIGESTING_ENCRYPTING || + state == PKCS11_SESSION_SIGNING_ENCRYPTING; + case PKCS11_FUNCTION_DECRYPT: + return state == PKCS11_SESSION_DECRYPTING || + state == PKCS11_SESSION_DECRYPTING_DIGESTING || + state == PKCS11_SESSION_DECRYPTING_VERIFYING; + case PKCS11_FUNCTION_DIGEST: + return state == PKCS11_SESSION_DIGESTING || + state == PKCS11_SESSION_DIGESTING_ENCRYPTING; + case PKCS11_FUNCTION_SIGN: + return state == PKCS11_SESSION_SIGNING || + state == PKCS11_SESSION_SIGNING_ENCRYPTING; + case PKCS11_FUNCTION_VERIFY: + return state == PKCS11_SESSION_VERIFYING || + state == PKCS11_SESSION_DECRYPTING_VERIFYING; + case PKCS11_FUNCTION_SIGN_RECOVER: + return state == PKCS11_SESSION_SIGNING_RECOVER; + case PKCS11_FUNCTION_VERIFY_RECOVER: + return state == PKCS11_SESSION_SIGNING_RECOVER; + default: + TEE_Panic(function); + return false; + } +} + +static enum pkcs11_rc get_active_session(struct pkcs11_session *session, + enum processing_func function) +{ + enum pkcs11_rc rc = PKCS11_CKR_OPERATION_NOT_INITIALIZED; + + if (session->processing && + func_matches_state(function, session->processing->state)) + rc = PKCS11_CKR_OK; + + return rc; +} + +void release_active_processing(struct pkcs11_session *session) +{ + if (!session->processing) + return; + + if (session->processing->tee_hash_op_handle != TEE_HANDLE_NULL) { + TEE_FreeOperation(session->processing->tee_hash_op_handle); + session->processing->tee_hash_op_handle = TEE_HANDLE_NULL; + session->processing->tee_hash_algo = 0; + } + + if (session->processing->tee_op_handle != TEE_HANDLE_NULL) { + TEE_FreeOperation(session->processing->tee_op_handle); + session->processing->tee_op_handle = TEE_HANDLE_NULL; + } + + TEE_Free(session->processing->extra_ctx); + + TEE_Free(session->processing); + session->processing = NULL; +} + +size_t get_object_key_bit_size(struct pkcs11_object *obj) +{ + void *a_ptr = NULL; + uint32_t a_size = 0; + struct obj_attrs *attrs = obj->attributes; + + switch (get_key_type(attrs)) { + case PKCS11_CKK_AES: + case PKCS11_CKK_GENERIC_SECRET: + case PKCS11_CKK_MD5_HMAC: + case PKCS11_CKK_SHA_1_HMAC: + case PKCS11_CKK_SHA224_HMAC: + case PKCS11_CKK_SHA256_HMAC: + case PKCS11_CKK_SHA384_HMAC: + case PKCS11_CKK_SHA512_HMAC: + if (get_attribute_ptr(attrs, PKCS11_CKA_VALUE, NULL, &a_size)) + return 0; + + return a_size * 8; + case PKCS11_CKK_RSA: + if (get_attribute_ptr(attrs, PKCS11_CKA_MODULUS, NULL, &a_size)) + return 0; + + return a_size * 8; + case PKCS11_CKK_EC: + if (get_attribute_ptr(attrs, PKCS11_CKA_EC_PARAMS, + &a_ptr, &a_size) || !a_ptr) + return 0; + + return ec_params2tee_keysize(a_ptr, a_size); + case PKCS11_CKK_EC_EDWARDS: + if (get_attribute_ptr(attrs, PKCS11_CKA_EC_POINT, NULL, + &a_size)) + return 0; + + return a_size * 8; + default: + TEE_Panic(0); + return 0; + } +} + +static enum pkcs11_rc generate_random_key_value(struct obj_attrs **head) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *data = NULL; + uint32_t data_size = 0; + uint32_t value_len = 0; + void *value = NULL; + + if (!*head) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + rc = get_attribute_ptr(*head, PKCS11_CKA_VALUE_LEN, &data, &data_size); + if (rc || data_size != sizeof(uint32_t)) { + DMSG("%s", rc ? "No attribute value_len found" : + "Invalid size for attribute VALUE_LEN"); + + return PKCS11_CKR_ATTRIBUTE_VALUE_INVALID; + } + TEE_MemMove(&value_len, data, data_size); + + /* Remove the default empty value attribute if found */ + rc = remove_empty_attribute(head, PKCS11_CKA_VALUE); + if (rc != PKCS11_CKR_OK && rc != PKCS11_RV_NOT_FOUND) + return PKCS11_CKR_GENERAL_ERROR; + + value = TEE_Malloc(value_len, TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!value) + return PKCS11_CKR_DEVICE_MEMORY; + + TEE_GenerateRandom(value, value_len); + + rc = add_attribute(head, PKCS11_CKA_VALUE, value, value_len); + + TEE_Free(value); + + return rc; +} + +enum pkcs11_rc entry_generate_secret(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_attribute_head *proc_params = NULL; + struct obj_attrs *head = NULL; + struct pkcs11_object_head *template = NULL; + size_t template_size = 0; + uint32_t obj_handle = 0; + + if (!client || ptypes != exp_pt || + out->memref.size != sizeof(obj_handle)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_alloc_get_one_attribute(&ctrlargs, &proc_params); + if (rc) + goto out; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &template); + if (rc) + goto out; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + rc = get_ready_session(session); + if (rc) + goto out; + + template_size = sizeof(*template) + template->attrs_size; + + rc = check_mechanism_against_processing(session, proc_params->id, + PKCS11_FUNCTION_GENERATE, + PKCS11_FUNC_STEP_INIT); + if (rc) { + DMSG("Invalid mechanism %#"PRIx32": %#x", proc_params->id, rc); + goto out; + } + + /* + * Prepare a clean initial state for the requested object attributes. + * Free temporary template once done. + */ + rc = create_attributes_from_template(&head, template, template_size, + NULL, PKCS11_FUNCTION_GENERATE, + proc_params->id, + PKCS11_CKO_UNDEFINED_ID); + if (rc) + goto out; + + TEE_Free(template); + template = NULL; + + rc = check_created_attrs(head, NULL); + if (rc) + goto out; + + rc = check_created_attrs_against_processing(proc_params->id, head); + if (rc) + goto out; + + rc = check_created_attrs_against_token(session, head); + if (rc) + goto out; + + /* + * Execute target processing and add value as attribute + * PKCS11_CKA_VALUE. Symm key generation: depends on target + * processing to be used. + */ + switch (proc_params->id) { + case PKCS11_CKM_GENERIC_SECRET_KEY_GEN: + case PKCS11_CKM_AES_KEY_GEN: + /* Generate random of size specified by attribute VALUE_LEN */ + rc = generate_random_key_value(&head); + if (rc) + goto out; + break; + + default: + rc = PKCS11_CKR_MECHANISM_INVALID; + goto out; + } + + TEE_Free(proc_params); + proc_params = NULL; + + /* + * Object is ready, register it and return a handle. + */ + rc = create_object(session, head, &obj_handle); + if (rc) + goto out; + + /* + * Now obj_handle (through the related struct pkcs11_object instance) + * owns the serialized buffer that holds the object attributes. + * We reset head to NULL as it is no more the buffer owner and would + * be freed at function out. + */ + head = NULL; + + TEE_MemMove(out->memref.buffer, &obj_handle, sizeof(obj_handle)); + out->memref.size = sizeof(obj_handle); + + DMSG("PKCS11 session %"PRIu32": generate secret %#"PRIx32, + session->handle, obj_handle); + +out: + TEE_Free(proc_params); + TEE_Free(template); + TEE_Free(head); + + return rc; +} + +enum pkcs11_rc alloc_get_tee_attribute_data(TEE_ObjectHandle tee_obj, + uint32_t attribute, + void **data, size_t *size) +{ + TEE_Result res = TEE_ERROR_GENERIC; + void *ptr = NULL; + size_t sz = 0; + + res = TEE_GetObjectBufferAttribute(tee_obj, attribute, NULL, &sz); + if (res != TEE_ERROR_SHORT_BUFFER) + return PKCS11_CKR_FUNCTION_FAILED; + + ptr = TEE_Malloc(sz, TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!ptr) + return PKCS11_CKR_DEVICE_MEMORY; + + res = TEE_GetObjectBufferAttribute(tee_obj, attribute, ptr, &sz); + if (res) { + TEE_Free(ptr); + } else { + *data = ptr; + *size = sz; + } + + return tee2pkcs_error(res); +} + +enum pkcs11_rc tee2pkcs_add_attribute(struct obj_attrs **head, + uint32_t pkcs11_id, + TEE_ObjectHandle tee_obj, + uint32_t tee_id) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *a_ptr = NULL; + size_t a_size = 0; + + rc = alloc_get_tee_attribute_data(tee_obj, tee_id, &a_ptr, &a_size); + if (rc) + goto out; + + rc = add_attribute(head, pkcs11_id, a_ptr, a_size); + + TEE_Free(a_ptr); + +out: + if (rc) + EMSG("Failed TEE attribute %#"PRIx32" for %#"PRIx32"/%s", + tee_id, pkcs11_id, id2str_attr(pkcs11_id)); + return rc; +} + +enum pkcs11_rc entry_generate_key_pair(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_attribute_head *proc_params = NULL; + struct obj_attrs *pub_head = NULL; + struct obj_attrs *priv_head = NULL; + struct pkcs11_object_head *pub_template = NULL; + struct pkcs11_object_head *priv_template = NULL; + struct pkcs11_object *object = NULL; + size_t pub_template_size = 0; + size_t priv_template_size = 0; + uint32_t pubkey_handle = 0; + uint32_t privkey_handle = 0; + uint32_t *hdl_ptr = NULL; + size_t out_ref_size = sizeof(pubkey_handle) + sizeof(privkey_handle); + + if (!client || ptypes != exp_pt || out->memref.size != out_ref_size) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_alloc_get_one_attribute(&ctrlargs, &proc_params); + if (rc) + goto out; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &pub_template); + if (rc) + goto out; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &priv_template); + if (rc) + goto out; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + rc = get_ready_session(session); + if (rc) + goto out; + + rc = check_mechanism_against_processing(session, proc_params->id, + PKCS11_FUNCTION_GENERATE_PAIR, + PKCS11_FUNC_STEP_INIT); + if (rc) + goto out; + + pub_template_size = sizeof(*pub_template) + pub_template->attrs_size; + + rc = create_attributes_from_template(&pub_head, pub_template, + pub_template_size, NULL, + PKCS11_FUNCTION_GENERATE_PAIR, + proc_params->id, + PKCS11_CKO_PUBLIC_KEY); + if (rc) + goto out; + + TEE_Free(pub_template); + pub_template = NULL; + + priv_template_size = sizeof(*priv_template) + + priv_template->attrs_size; + + rc = create_attributes_from_template(&priv_head, priv_template, + priv_template_size, NULL, + PKCS11_FUNCTION_GENERATE_PAIR, + proc_params->id, + PKCS11_CKO_PRIVATE_KEY); + if (rc) + goto out; + + TEE_Free(priv_template); + priv_template = NULL; + + /* Generate CKA_ID for keys if not specified by the templates */ + rc = add_missing_attribute_id(&pub_head, &priv_head); + if (rc) + goto out; + + /* Check created object against processing and token state */ + rc = check_created_attrs(pub_head, priv_head); + if (rc) + goto out; + + rc = check_created_attrs_against_processing(proc_params->id, pub_head); + if (rc) + goto out; + + rc = check_created_attrs_against_processing(proc_params->id, + priv_head); + if (rc) + goto out; + + rc = check_created_attrs_against_token(session, pub_head); + if (rc) + goto out; + + rc = check_access_attrs_against_token(session, pub_head); + if (rc) + goto out; + + rc = check_created_attrs_against_token(session, priv_head); + if (rc) + goto out; + + rc = check_access_attrs_against_token(session, priv_head); + if (rc) + goto out; + + /* Generate key pair */ + switch (proc_params->id) { + case PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN: + rc = generate_eddsa_keys(proc_params, &pub_head, &priv_head); + break; + case PKCS11_CKM_EC_KEY_PAIR_GEN: + rc = generate_ec_keys(proc_params, &pub_head, &priv_head); + break; + case PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN: + rc = generate_rsa_keys(proc_params, &pub_head, &priv_head); + break; + default: + rc = PKCS11_CKR_MECHANISM_INVALID; + break; + } + if (rc) + goto out; + + TEE_Free(proc_params); + proc_params = NULL; + + /* + * Object is ready, register it and return a handle. + */ + rc = create_object(session, pub_head, &pubkey_handle); + if (rc) + goto out; + + /* + * Now obj_handle (through the related struct pkcs11_object instance) + * owns the serialized buffer that holds the object attributes. + * We reset local pub_head to NULL to mark that ownership has been + * transferred. + */ + pub_head = NULL; + + rc = create_object(session, priv_head, &privkey_handle); + if (rc) + goto out; + + /* Ownership has been transferred so mark it with NULL */ + priv_head = NULL; + + hdl_ptr = (uint32_t *)out->memref.buffer; + + TEE_MemMove(hdl_ptr, &pubkey_handle, sizeof(pubkey_handle)); + TEE_MemMove(hdl_ptr + 1, &privkey_handle, sizeof(privkey_handle)); + + DMSG("PKCS11 session %"PRIu32": create key pair %#"PRIx32"/%#"PRIx32, + session->handle, privkey_handle, pubkey_handle); + + pubkey_handle = 0; + privkey_handle = 0; +out: + if (pubkey_handle) { + object = pkcs11_handle2object(pubkey_handle, session); + if (!object) + TEE_Panic(0); + destroy_object(session, object, false); + } + TEE_Free(priv_head); + TEE_Free(pub_head); + TEE_Free(priv_template); + TEE_Free(pub_template); + TEE_Free(proc_params); + + return rc; +} + +/* + * entry_processing_init - Generic entry for initializing a processing + * + * @client = client reference + * @ptype = Invocation parameter types + * @params = Invocation parameters reference + * @function - encrypt, decrypt, sign, verify, digest, ... + */ +enum pkcs11_rc entry_processing_init(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params, + enum processing_func function) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_attribute_head *proc_params = NULL; + uint32_t key_handle = 0; + struct pkcs11_object *obj = NULL; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (function != PKCS11_FUNCTION_DIGEST) { + rc = serialargs_get(&ctrlargs, &key_handle, sizeof(uint32_t)); + if (rc) + return rc; + } + + rc = serialargs_alloc_get_one_attribute(&ctrlargs, &proc_params); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out_free; + } + + rc = get_ready_session(session); + if (rc) + goto out_free; + + if (function != PKCS11_FUNCTION_DIGEST) { + obj = pkcs11_handle2object(key_handle, session); + if (!obj) { + rc = PKCS11_CKR_KEY_HANDLE_INVALID; + goto out_free; + } + } + + rc = set_processing_state(session, function, obj, NULL); + if (rc) + goto out; + + rc = check_mechanism_against_processing(session, proc_params->id, + function, + PKCS11_FUNC_STEP_INIT); + if (rc) + goto out; + + if (obj) { + rc = check_parent_attrs_against_processing(proc_params->id, + function, + obj->attributes); + if (rc) + goto out; + + rc = check_access_attrs_against_token(session, + obj->attributes); + if (rc) + goto out; + } + + if (processing_is_tee_symm(proc_params->id)) + rc = init_symm_operation(session, function, proc_params, obj); + else if (processing_is_tee_asymm(proc_params->id)) + rc = init_asymm_operation(session, function, proc_params, obj); + else if (processing_is_tee_digest(proc_params->id)) + rc = init_digest_operation(session, proc_params); + else + rc = PKCS11_CKR_MECHANISM_INVALID; + + if (rc == PKCS11_CKR_OK) { + DMSG("PKCS11 session %"PRIu32": init processing %s %s", + session->handle, id2str_proc(proc_params->id), + id2str_function(function)); + } + +out: + if (rc) + release_active_processing(session); +out_free: + TEE_Free(proc_params); + + return rc; +} + +/* + * entry_processing_step - Generic entry on active processing + * + * @client = client reference + * @ptype = Invocation parameter types + * @params = Invocation parameters reference + * @function - encrypt, decrypt, sign, verify, digest, ... + * @step - update, oneshot, final + */ +enum pkcs11_rc entry_processing_step(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params, + enum processing_func function, + enum processing_step step) +{ + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + enum pkcs11_mechanism_id mecha_type = PKCS11_CKM_UNDEFINED_ID; + uint32_t key_handle = 0; + struct pkcs11_object *obj = NULL; + + if (!client || + TEE_PARAM_TYPE_GET(ptypes, 0) != TEE_PARAM_TYPE_MEMREF_INOUT) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + if (step == PKCS11_FUNC_STEP_UPDATE_KEY) { + assert(function == PKCS11_FUNCTION_DIGEST); + + rc = serialargs_get(&ctrlargs, &key_handle, sizeof(uint32_t)); + if (rc) + return rc; + } + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + rc = get_active_session(session, function); + if (rc) + return rc; + + if (step == PKCS11_FUNC_STEP_UPDATE_KEY) { + assert(function == PKCS11_FUNCTION_DIGEST); + + obj = pkcs11_handle2object(key_handle, session); + if (!obj) { + rc = PKCS11_CKR_KEY_HANDLE_INVALID; + goto out; + } + + rc = check_access_attrs_against_token(session, + obj->attributes); + if (rc) { + rc = PKCS11_CKR_KEY_HANDLE_INVALID; + goto out; + } + } + + mecha_type = session->processing->mecha_type; + rc = check_mechanism_against_processing(session, mecha_type, + function, step); + if (rc) + goto out; + + if (processing_is_tee_symm(mecha_type)) + rc = step_symm_operation(session, function, step, + ptypes, params); + else if (processing_is_tee_asymm(mecha_type)) + rc = step_asymm_operation(session, function, step, + ptypes, params); + else if (processing_is_tee_digest(mecha_type)) + rc = step_digest_operation(session, step, obj, ptypes, params); + else + rc = PKCS11_CKR_MECHANISM_INVALID; + + if (rc == PKCS11_CKR_OK && (step == PKCS11_FUNC_STEP_UPDATE || + step == PKCS11_FUNC_STEP_UPDATE_KEY)) { + session->processing->step = PKCS11_FUNC_STEP_UPDATE; + DMSG("PKCS11 session%"PRIu32": processing %s %s", + session->handle, id2str_proc(mecha_type), + id2str_function(function)); + } + + if (rc == PKCS11_CKR_BUFFER_TOO_SMALL && + step == PKCS11_FUNC_STEP_ONESHOT) + session->processing->step = PKCS11_FUNC_STEP_ONESHOT; + + if (rc == PKCS11_CKR_BUFFER_TOO_SMALL && step == PKCS11_FUNC_STEP_FINAL) + session->processing->step = PKCS11_FUNC_STEP_FINAL; + +out: + switch (step) { + case PKCS11_FUNC_STEP_UPDATE: + case PKCS11_FUNC_STEP_UPDATE_KEY: + if (rc != PKCS11_CKR_OK && rc != PKCS11_CKR_BUFFER_TOO_SMALL) + release_active_processing(session); + break; + default: + /* ONESHOT and FINAL terminates processing on success */ + if (rc != PKCS11_CKR_BUFFER_TOO_SMALL) + release_active_processing(session); + break; + } + + return rc; +} + +enum pkcs11_rc entry_processing_key(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params, + enum processing_func function) +{ + TEE_Param *ctrl = params; + TEE_Param *out = params + 2; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_attribute_head *proc_params = NULL; + struct pkcs11_object_head *template = NULL; + uint32_t parent_handle = 0; + uint32_t obj_handle = 0; + struct pkcs11_object *parent = NULL; + struct obj_attrs *head = NULL; + size_t template_size = 0; + void *in_buf = NULL; + uint32_t in_size = 0; + void *out_buf = NULL; + uint32_t out_size = 0; + enum processing_func operation = PKCS11_FUNCTION_UNKNOWN; + + if (!client || + TEE_PARAM_TYPE_GET(ptypes, 0) != TEE_PARAM_TYPE_MEMREF_INOUT || + TEE_PARAM_TYPE_GET(ptypes, 2) != TEE_PARAM_TYPE_MEMREF_OUTPUT || + out->memref.size != sizeof(obj_handle) || + TEE_PARAM_TYPE_GET(ptypes, 3) != TEE_PARAM_TYPE_NONE) + return PKCS11_CKR_ARGUMENTS_BAD; + + switch (function) { + case PKCS11_FUNCTION_UNWRAP: + if (TEE_PARAM_TYPE_GET(ptypes, 1) != + TEE_PARAM_TYPE_MEMREF_INPUT) + return PKCS11_CKR_ARGUMENTS_BAD; + + in_buf = params[1].memref.buffer; + in_size = params[1].memref.size; + if (in_size && !in_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + + /* + * Some unwrap mechanisms require encryption to be + * performed on the data passed in proc_params by parent + * key. Hence set operation as PKCS11_FUNCTION_DECRYPT + * to be used with init_symm_operation() + */ + operation = PKCS11_FUNCTION_DECRYPT; + break; + case PKCS11_FUNCTION_DERIVE: + if (TEE_PARAM_TYPE_GET(ptypes, 1) != TEE_PARAM_TYPE_NONE) + return PKCS11_CKR_ARGUMENTS_BAD; + + /* + * Some derivation mechanism require encryption to be + * performed on the data passed in proc_params by parent + * key. Hence set operation as PKCS11_FUNCTION_ENCRYPT + * to be used with init_symm_operation() + */ + operation = PKCS11_FUNCTION_ENCRYPT; + break; + default: + return PKCS11_CKR_ARGUMENTS_BAD; + } + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &parent_handle, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_alloc_get_one_attribute(&ctrlargs, &proc_params); + if (rc) + return rc; + + rc = serialargs_alloc_get_attributes(&ctrlargs, &template); + if (rc) + goto out_free; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out_free; + } + + /* Return error if processing already active */ + rc = get_ready_session(session); + if (rc) + goto out_free; + + /* Check parent handle */ + parent = pkcs11_handle2object(parent_handle, session); + if (!parent) { + rc = PKCS11_CKR_KEY_HANDLE_INVALID; + goto out_free; + } + + /* Check if mechanism can be used for derivation function */ + rc = check_mechanism_against_processing(session, proc_params->id, + function, + PKCS11_FUNC_STEP_INIT); + if (rc) + goto out_free; + + /* Set the processing state to active */ + rc = set_processing_state(session, function, parent, NULL); + if (rc) + goto out_free; + + /* + * Check if base/parent key has CKA_DERIVE set and its key type is + * compatible with the mechanism passed + */ + rc = check_parent_attrs_against_processing(proc_params->id, function, + parent->attributes); + if (rc) { + /* + * CKR_KEY_FUNCTION_NOT_PERMITTED is not in the list of errors + * specified with C_Derive/Unwrap() in the specification. So + * return the next most appropriate error. + */ + if (rc == PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED) { + if (function == PKCS11_FUNCTION_UNWRAP) + rc = + PKCS11_CKR_UNWRAPPING_KEY_TYPE_INCONSISTENT; + else + rc = PKCS11_CKR_KEY_TYPE_INCONSISTENT; + } + goto out; + } + + /* Check access of base/parent key */ + rc = check_access_attrs_against_token(session, parent->attributes); + if (rc) + goto out; + + template_size = sizeof(*template) + template->attrs_size; + /* + * Prepare a clean initial state for the requested object attributes + * using base/parent key attributes. Free temporary template once done. + */ + rc = create_attributes_from_template(&head, template, template_size, + parent->attributes, + function, + proc_params->id, + PKCS11_CKO_UNDEFINED_ID); + if (rc) + goto out; + + TEE_Free(template); + template = NULL; + + /* check_created_attrs() is called later once key size is known */ + + rc = check_created_attrs_against_processing(proc_params->id, head); + if (rc) + goto out; + + rc = check_created_attrs_against_token(session, head); + if (rc) + goto out; + + rc = check_access_attrs_against_token(session, head); + if (rc) + goto out; + + if (processing_is_tee_symm(proc_params->id)) { + rc = init_symm_operation(session, operation, proc_params, + parent); + if (rc) + goto out; + + switch (function) { + case PKCS11_FUNCTION_DERIVE: + rc = derive_key_by_symm_enc(session, &out_buf, + &out_size); + break; + case PKCS11_FUNCTION_UNWRAP: + rc = unwrap_key_by_symm(session, in_buf, in_size, + &out_buf, &out_size); + break; + default: + TEE_Panic(function); + } + if (rc) + goto out; + + } else if (processing_is_tee_asymm(proc_params->id)) { + switch (function) { + case PKCS11_FUNCTION_DERIVE: + rc = init_asymm_operation(session, function, + proc_params, parent); + if (rc) + goto out; + + rc = do_asymm_derivation(session, proc_params, &head); + if (!rc) + goto done; + break; + case PKCS11_FUNCTION_UNWRAP: + rc = init_asymm_operation(session, operation, + proc_params, parent); + if (rc) + goto out; + + rc = unwrap_key_by_asymm(session, in_buf, in_size, + &out_buf, &out_size); + break; + default: + TEE_Panic(function); + } + + if (rc) + goto out; + } else { + rc = PKCS11_CKR_MECHANISM_INVALID; + goto out; + } + + rc = set_key_data(&head, out_buf, out_size); + if (rc) + goto out; + +done: + TEE_Free(out_buf); + out_buf = NULL; + + TEE_Free(proc_params); + proc_params = NULL; + + /* + * Object is ready, register it and return a handle. + */ + rc = create_object(session, head, &obj_handle); + if (rc) + goto out; + + /* + * Now obj_handle (through the related struct pkcs11_object instance) + * owns the serialized buffer that holds the object attributes. + * We reset head to NULL as it is no more the buffer owner and would + * be freed at function out. + */ + head = NULL; + + TEE_MemMove(out->memref.buffer, &obj_handle, sizeof(obj_handle)); + out->memref.size = sizeof(obj_handle); + + DMSG("PKCS11 session %"PRIu32": derive secret %#"PRIx32, + session->handle, obj_handle); + +out: + release_active_processing(session); +out_free: + TEE_Free(proc_params); + TEE_Free(template); + TEE_Free(head); + TEE_Free(out_buf); + + return rc; +} + +enum pkcs11_rc entry_release_active_processing(struct pkcs11_client *client, + uint32_t ptypes, + TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + enum processing_func function = PKCS11_FUNCTION_UNKNOWN; + uint32_t cmd = 0; + + if (!client || ptypes != exp_pt) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get_u32(&ctrlargs, &cmd); + + if (serialargs_remaining_bytes(&ctrlargs)) + return PKCS11_CKR_ARGUMENTS_BAD; + + function = func_for_cmd(cmd); + if (function == PKCS11_FUNCTION_UNKNOWN) + return PKCS11_CKR_ARGUMENTS_BAD; + + rc = get_active_session(session, function); + if (rc) + return rc; + + release_active_processing(session); + + DMSG("PKCS11 session %"PRIu32": release processing", session->handle); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc entry_wrap_key(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params) +{ + const uint32_t exp_pt = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INOUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE); + TEE_Param *ctrl = params; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs ctrlargs = { }; + struct pkcs11_session *session = NULL; + struct pkcs11_attribute_head *proc_params = NULL; + struct pkcs11_object *wrapping_key = NULL; + struct pkcs11_object *key = NULL; + void *req_attrs = NULL; + uint32_t wrapping_key_handle = 0; + uint32_t key_handle = 0; + uint32_t size = 0; + void *key_data = NULL; + uint32_t key_sz = 0; + void *out_buf = params[2].memref.buffer; + uint32_t out_size = params[2].memref.size; + const enum processing_func function = PKCS11_FUNCTION_WRAP; + + if (!client || ptypes != exp_pt || + (out_size && !out_buf)) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&ctrlargs, ctrl->memref.buffer, ctrl->memref.size); + + rc = serialargs_get_session_from_handle(&ctrlargs, client, &session); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &wrapping_key_handle, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get(&ctrlargs, &key_handle, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_alloc_get_one_attribute(&ctrlargs, &proc_params); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&ctrlargs)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out_free; + } + + rc = get_ready_session(session); + if (rc) + goto out_free; + + wrapping_key = pkcs11_handle2object(wrapping_key_handle, session); + if (!wrapping_key) { + rc = PKCS11_CKR_WRAPPING_KEY_HANDLE_INVALID; + goto out_free; + } + + key = pkcs11_handle2object(key_handle, session); + if (!key) { + rc = PKCS11_CKR_KEY_HANDLE_INVALID; + goto out_free; + } + + /* + * The wrapping key and key to be wrapped shouldn't be same. + * PKCS#11 spec doesn't explicitly state that but logically this isn't + * a use case and also acts as an attack vector, so explicitly + * disallow this. + */ + if (key == wrapping_key) { + rc = PKCS11_CKR_WRAPPING_KEY_HANDLE_INVALID; + goto out_free; + } + + rc = set_processing_state(session, function, wrapping_key, NULL); + if (rc) + goto out_free; + + /* Check if mechanism can be used for wrapping function */ + rc = check_mechanism_against_processing(session, proc_params->id, + function, + PKCS11_FUNC_STEP_INIT); + if (rc) + goto out; + + /* + * Check if wrapping key has CKA_WRAP set and its key type is + * compatible with the mechanism passed + */ + rc = check_parent_attrs_against_processing(proc_params->id, function, + wrapping_key->attributes); + if (rc) { + /* + * CKR_KEY_FUNCTION_NOT_PERMITTED is not in the list of errors + * specified with C_Wrap() in the specification. So + * return the next most appropriate error. + */ + if (rc == PKCS11_CKR_KEY_FUNCTION_NOT_PERMITTED) + rc = PKCS11_CKR_WRAPPING_KEY_TYPE_INCONSISTENT; + + goto out; + } + + /* Check access of wrapping key */ + rc = check_access_attrs_against_token(session, + wrapping_key->attributes); + if (rc) + goto out; + + switch (get_class(key->attributes)) { + case PKCS11_CKO_SECRET_KEY: + case PKCS11_CKO_PRIVATE_KEY: + break; + default: + rc = PKCS11_CKR_KEY_NOT_WRAPPABLE; + goto out; + } + + /* Check if key to be wrapped is extractable */ + if (!get_bool(key->attributes, PKCS11_CKA_EXTRACTABLE)) { + DMSG("Extractable property is false"); + rc = PKCS11_CKR_KEY_UNEXTRACTABLE; + goto out; + } + + if (get_bool(key->attributes, PKCS11_CKA_WRAP_WITH_TRUSTED) && + !get_bool(wrapping_key->attributes, PKCS11_CKA_TRUSTED)) { + DMSG("Wrap with trusted not satisfied"); + rc = PKCS11_CKR_KEY_NOT_WRAPPABLE; + goto out; + } + + rc = check_access_attrs_against_token(session, key->attributes); + if (rc) + goto out; + + rc = get_attribute_ptr(wrapping_key->attributes, + PKCS11_CKA_WRAP_TEMPLATE, &req_attrs, &size); + if (rc == PKCS11_CKR_OK && size != 0) { + if (!attributes_match_reference(key->attributes, req_attrs)) { + rc = PKCS11_CKR_KEY_HANDLE_INVALID; + goto out; + } + } + + rc = alloc_key_data_to_wrap(key->attributes, &key_data, &key_sz); + if (rc) + goto out; + + if (processing_is_tee_symm(proc_params->id)) { + rc = init_symm_operation(session, PKCS11_FUNCTION_ENCRYPT, + proc_params, wrapping_key); + if (rc) + goto out; + + rc = wrap_data_by_symm_enc(session, key_data, key_sz, out_buf, + &out_size); + } else { + rc = init_asymm_operation(session, PKCS11_FUNCTION_ENCRYPT, + proc_params, wrapping_key); + if (rc) + goto out; + + rc = wrap_data_by_asymm_enc(session, key_data, key_sz, out_buf, + &out_size); + } + + if (rc == PKCS11_CKR_OK || rc == PKCS11_CKR_BUFFER_TOO_SMALL) + params[2].memref.size = out_size; + +out: + release_active_processing(session); +out_free: + TEE_Free(key_data); + TEE_Free(proc_params); + return rc; +} diff --git a/optee_os/ta/pkcs11/src/processing.h b/optee_os/ta/pkcs11/src/processing.h new file mode 100644 index 0000000..9f84732 --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing.h @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_PROCESSING_H +#define PKCS11_TA_PROCESSING_H + +#include +#include +#include + +struct pkcs11_client; +struct pkcs11_session; +struct pkcs11_object; +struct active_processing; + +/** + * RSA PSS processing context + * + * @hash_alg: Hash algorithm mechanism + * @mgf_type: Mask generator function + * @salt_len: Length of the salt in bytes + */ +struct rsa_pss_processing_ctx { + enum pkcs11_mechanism_id hash_alg; + enum pkcs11_mgf_id mgf_type; + uint32_t salt_len; +}; + +/** + * RSA OAEP processing context + * + * @hash_alg: Hash algorithm mechanism + * @mgf_type: Mask generator function + * @source_type: Type of source. + * @source_data_len: Length of the source data. + * @source_data: Source data. + */ +struct rsa_oaep_processing_ctx { + enum pkcs11_mechanism_id hash_alg; + enum pkcs11_mgf_id mgf_type; + uint32_t source_type; + uint32_t source_data_len; + uint8_t source_data[]; +}; + +/** + * RSA AES key wrap processing context + * + * @hash_alg: Hash algorithm mechanism + * @mgf_type: Mask generator function + * @aes_key_bits: Length of AES key in bits + * @source_type: Type of source. + * @source_data_len: Length of the source data. + * @source_data: Source data. + */ +struct rsa_aes_key_wrap_processing_ctx { + enum pkcs11_mechanism_id hash_alg; + enum pkcs11_mgf_id mgf_type; + uint32_t aes_key_bits; + uint32_t source_type; + uint32_t source_data_len; + uint8_t source_data[]; +}; + +/** + * EDDSA processing context + * + * @flag: Prehash flag + * @ctx_len: Length of the context data + * @ctx: Context data + */ +struct eddsa_processing_ctx { + uint32_t flag; + uint32_t ctx_len; + uint8_t ctx[]; +}; + +/* + * Entry points from PKCS11 TA invocation commands + */ + +enum pkcs11_rc entry_generate_secret(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_generate_key_pair(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc entry_processing_init(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params, + enum processing_func function); + +enum pkcs11_rc entry_processing_step(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params, + enum processing_func function, + enum processing_step step); + +enum pkcs11_rc entry_processing_key(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params, + enum processing_func function); + +enum pkcs11_rc entry_release_active_processing(struct pkcs11_client *client, + uint32_t ptypes, + TEE_Param *params); + +enum pkcs11_rc entry_wrap_key(struct pkcs11_client *client, + uint32_t ptypes, TEE_Param *params); + +/* + * Util + */ +size_t get_object_key_bit_size(struct pkcs11_object *obj); + +void release_active_processing(struct pkcs11_session *session); + +enum pkcs11_rc alloc_get_tee_attribute_data(TEE_ObjectHandle tee_obj, + uint32_t attribute, + void **data, size_t *size); + +enum pkcs11_rc tee2pkcs_add_attribute(struct obj_attrs **head, + uint32_t pkcs11_id, + TEE_ObjectHandle tee_obj, + uint32_t tee_id); + +/* Asymmetric key operations util */ +bool processing_is_tee_asymm(uint32_t proc_id); + +enum pkcs11_rc init_asymm_operation(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj); + +enum pkcs11_rc step_asymm_operation(struct pkcs11_session *session, + enum processing_func function, + enum processing_step step, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc wrap_data_by_asymm_enc(struct pkcs11_session *session, + void *data, uint32_t data_sz, + void *out_buf, uint32_t *out_sz); + +enum pkcs11_rc unwrap_key_by_asymm(struct pkcs11_session *session, void *data, + uint32_t data_sz, void **out_buf, + uint32_t *out_sz); + +/* + * Symmetric crypto algorithm specific functions + */ +bool processing_is_tee_symm(enum pkcs11_mechanism_id proc_id); + +enum pkcs11_rc init_symm_operation(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *key); + +enum pkcs11_rc step_symm_operation(struct pkcs11_session *session, + enum processing_func function, + enum processing_step step, + uint32_t ptypes, TEE_Param *params); + +enum pkcs11_rc tee_init_ctr_operation(struct active_processing *processing, + void *proc_params, size_t params_size); + +enum pkcs11_rc derive_key_by_symm_enc(struct pkcs11_session *session, + void **out_buf, uint32_t *out_sz); + +enum pkcs11_rc wrap_data_by_symm_enc(struct pkcs11_session *session, + void *data, uint32_t data_sz, + void *out_buf, uint32_t *out_sz); + +enum pkcs11_rc unwrap_key_by_symm(struct pkcs11_session *session, void *data, + uint32_t data_sz, void **out_buf, + uint32_t *out_sz); + +/* Digest specific functions */ +bool processing_is_tee_digest(enum pkcs11_mechanism_id mecha_id); + +enum pkcs11_rc +init_digest_operation(struct pkcs11_session *session, + struct pkcs11_attribute_head *proc_params); + +enum pkcs11_rc step_digest_operation(struct pkcs11_session *session, + enum processing_step step, + struct pkcs11_object *obj, + uint32_t ptypes, TEE_Param *params); + +/* + * Elliptic curve crypto algorithm specific functions + */ +enum pkcs11_rc load_tee_ec_key_attrs(TEE_Attribute **tee_attrs, + size_t *tee_count, + struct pkcs11_object *obj); + +enum pkcs11_rc load_tee_eddsa_key_attrs(TEE_Attribute **tee_attrs, + size_t *tee_count, + struct pkcs11_object *obj); + +size_t ec_params2tee_keysize(void *attr, size_t size); + +uint32_t ec_params2tee_curve(void *attr, size_t size); + +enum pkcs11_rc pkcs2tee_algo_ecdsa(uint32_t *tee_id, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj); + +enum pkcs11_rc generate_ec_keys(struct pkcs11_attribute_head *proc_params, + struct obj_attrs **pub_head, + struct obj_attrs **priv_head); + +enum pkcs11_rc generate_eddsa_keys(struct pkcs11_attribute_head *proc_params, + struct obj_attrs **pub_head, + struct obj_attrs **priv_head); + +size_t ecdsa_get_input_max_byte_size(TEE_OperationHandle op); + +/* + * RSA crypto algorithm specific functions + */ +enum pkcs11_rc load_tee_rsa_key_attrs(TEE_Attribute **tee_attrs, + size_t *tee_count, + struct pkcs11_object *obj); + +enum pkcs11_rc +pkcs2tee_proc_params_rsa_pss(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params); + +enum pkcs11_rc pkcs2tee_validate_rsa_pss(struct active_processing *proc, + struct pkcs11_object *obj); + +enum pkcs11_rc pkcs2tee_algo_rsa_pss(uint32_t *tee_id, + struct pkcs11_attribute_head *params); + +enum pkcs11_rc +pkcs2tee_proc_params_rsa_oaep(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params); + +enum pkcs11_rc +pkcs2tee_proc_params_rsa_aes_wrap(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params); + +enum pkcs11_rc +pkcs2tee_proc_params_eddsa(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params); + +enum pkcs11_rc pkcs2tee_algo_rsa_oaep(uint32_t *tee_id, uint32_t *tee_hash_id, + struct pkcs11_attribute_head *params); + +enum pkcs11_rc +pkcs2tee_algo_rsa_aes_wrap(uint32_t *tee_id, uint32_t *tee_hash_id, + struct pkcs11_attribute_head *params); + +enum pkcs11_rc generate_rsa_keys(struct pkcs11_attribute_head *proc_params, + struct obj_attrs **pub_head, + struct obj_attrs **priv_head); + +size_t rsa_get_input_max_byte_size(TEE_OperationHandle op); + +enum pkcs11_rc do_asymm_derivation(struct pkcs11_session *session, + struct pkcs11_attribute_head *proc_params, + struct obj_attrs **head); + +enum pkcs11_rc pkcs2tee_param_ecdh(struct pkcs11_attribute_head *proc_params, + void **pub_data, size_t *pub_size); + +enum pkcs11_rc pkcs2tee_algo_ecdh(uint32_t *tee_id, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj); + +#endif /*PKCS11_TA_PROCESSING_H*/ diff --git a/optee_os/ta/pkcs11/src/processing_aes.c b/optee_os/ta/pkcs11/src/processing_aes.c new file mode 100644 index 0000000..f60ca4c --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing_aes.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" +#include "serializer.h" + +enum pkcs11_rc tee_init_ctr_operation(struct active_processing *processing, + void *proc_params, size_t params_size) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_OK; + /* CTR parameters */ + uint32_t incr_counter = 0; + void *counter_bits = NULL; + + if (!proc_params) + return PKCS11_CKR_ARGUMENTS_BAD; + + serialargs_init(&args, proc_params, params_size); + + rc = serialargs_get(&args, &incr_counter, sizeof(uint32_t)); + if (rc) + return rc; + + rc = serialargs_get_ptr(&args, &counter_bits, 16); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (incr_counter != 1) { + DMSG("Supports only 1 bit increment counter: %"PRIu32, + incr_counter); + + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + } + + TEE_CipherInit(processing->tee_op_handle, counter_bits, 16); + + return PKCS11_CKR_OK; +} diff --git a/optee_os/ta/pkcs11/src/processing_asymm.c b/optee_os/ta/pkcs11/src/processing_asymm.c new file mode 100644 index 0000000..30348c0 --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing_asymm.c @@ -0,0 +1,1112 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" +#include "serializer.h" + +bool processing_is_tee_asymm(uint32_t proc_id) +{ + switch (proc_id) { + /* RSA flavors */ + case PKCS11_CKM_RSA_AES_KEY_WRAP: + case PKCS11_CKM_RSA_PKCS: + case PKCS11_CKM_RSA_PKCS_OAEP: + case PKCS11_CKM_RSA_PKCS_PSS: + case PKCS11_CKM_MD5_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS: + case PKCS11_CKM_SHA224_RSA_PKCS: + case PKCS11_CKM_SHA256_RSA_PKCS: + case PKCS11_CKM_SHA384_RSA_PKCS: + case PKCS11_CKM_SHA512_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + /* EC flavors */ + case PKCS11_CKM_EDDSA: + case PKCS11_CKM_ECDSA: + case PKCS11_CKM_ECDH1_DERIVE: + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + return true; + default: + return false; + } +} + +static enum pkcs11_rc +pkcs2tee_algorithm(uint32_t *tee_id, uint32_t *tee_hash_id, + enum processing_func function __unused, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj) +{ + static const struct { + enum pkcs11_mechanism_id mech_id; + uint32_t tee_id; + uint32_t tee_hash_id; + } pkcs2tee_algo[] = { + /* RSA flavors */ + { PKCS11_CKM_RSA_AES_KEY_WRAP, 1, 0 }, + { PKCS11_CKM_RSA_PKCS, TEE_ALG_RSAES_PKCS1_V1_5, 0 }, + { PKCS11_CKM_RSA_PKCS_OAEP, 1, 0 }, + { PKCS11_CKM_RSA_PKCS_PSS, 1, 0 }, + { PKCS11_CKM_MD5_RSA_PKCS, TEE_ALG_RSASSA_PKCS1_V1_5_MD5, + TEE_ALG_MD5 }, + { PKCS11_CKM_SHA1_RSA_PKCS, TEE_ALG_RSASSA_PKCS1_V1_5_SHA1, + TEE_ALG_SHA1 }, + { PKCS11_CKM_SHA224_RSA_PKCS, TEE_ALG_RSASSA_PKCS1_V1_5_SHA224, + TEE_ALG_SHA224 }, + { PKCS11_CKM_SHA256_RSA_PKCS, TEE_ALG_RSASSA_PKCS1_V1_5_SHA256, + TEE_ALG_SHA256 }, + { PKCS11_CKM_SHA384_RSA_PKCS, TEE_ALG_RSASSA_PKCS1_V1_5_SHA384, + TEE_ALG_SHA384 }, + { PKCS11_CKM_SHA512_RSA_PKCS, TEE_ALG_RSASSA_PKCS1_V1_5_SHA512, + TEE_ALG_SHA512 }, + { PKCS11_CKM_SHA1_RSA_PKCS_PSS, + TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1, TEE_ALG_SHA1 }, + { PKCS11_CKM_SHA224_RSA_PKCS_PSS, + TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224, TEE_ALG_SHA224 }, + { PKCS11_CKM_SHA256_RSA_PKCS_PSS, + TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256, TEE_ALG_SHA256 }, + { PKCS11_CKM_SHA384_RSA_PKCS_PSS, + TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384, TEE_ALG_SHA384 }, + { PKCS11_CKM_SHA512_RSA_PKCS_PSS, + TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512, TEE_ALG_SHA512 }, + /* EC flavors (Must find key size from the object) */ + { PKCS11_CKM_ECDSA, 1, 0 }, + { PKCS11_CKM_ECDSA_SHA1, 1, TEE_ALG_SHA1 }, + { PKCS11_CKM_ECDSA_SHA224, 1, TEE_ALG_SHA224 }, + { PKCS11_CKM_ECDSA_SHA256, 1, TEE_ALG_SHA256 }, + { PKCS11_CKM_ECDSA_SHA384, 1, TEE_ALG_SHA384 }, + { PKCS11_CKM_ECDSA_SHA512, 1, TEE_ALG_SHA512 }, + { PKCS11_CKM_ECDH1_DERIVE, 1, 0 }, + { PKCS11_CKM_EDDSA, TEE_ALG_ED25519, 0 }, + }; + size_t n = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + for (n = 0; n < ARRAY_SIZE(pkcs2tee_algo); n++) { + if (pkcs2tee_algo[n].mech_id == proc_params->id) { + *tee_id = pkcs2tee_algo[n].tee_id; + *tee_hash_id = pkcs2tee_algo[n].tee_hash_id; + break; + } + } + + if (n == ARRAY_SIZE(pkcs2tee_algo)) + return PKCS11_RV_NOT_IMPLEMENTED; + + switch (proc_params->id) { + case PKCS11_CKM_RSA_PKCS_PSS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + rc = pkcs2tee_algo_rsa_pss(tee_id, proc_params); + break; + case PKCS11_CKM_RSA_PKCS_OAEP: + rc = pkcs2tee_algo_rsa_oaep(tee_id, tee_hash_id, proc_params); + break; + case PKCS11_CKM_RSA_AES_KEY_WRAP: + rc = pkcs2tee_algo_rsa_aes_wrap(tee_id, tee_hash_id, + proc_params); + break; + case PKCS11_CKM_ECDSA: + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + rc = pkcs2tee_algo_ecdsa(tee_id, proc_params, obj); + break; + case PKCS11_CKM_ECDH1_DERIVE: + rc = pkcs2tee_algo_ecdh(tee_id, proc_params, obj); + break; + default: + rc = PKCS11_CKR_OK; + break; + } + + /* + * PKCS#11 uses single mechanism CKM_RSA_PKCS for both ciphering and + * authentication whereas GPD TEE expects TEE_ALG_RSAES_PKCS1_V1_5 for + * ciphering and TEE_ALG_RSASSA_PKCS1_V1_5 for authentication. + */ + if (*tee_id == TEE_ALG_RSAES_PKCS1_V1_5 && + (function == PKCS11_FUNCTION_SIGN || + function == PKCS11_FUNCTION_VERIFY)) + *tee_id = TEE_ALG_RSASSA_PKCS1_V1_5; + + return rc; +} + +static enum pkcs11_rc pkcs2tee_key_type(uint32_t *tee_type, + struct pkcs11_object *obj, + enum processing_func function) +{ + enum pkcs11_class_id class = get_class(obj->attributes); + enum pkcs11_key_type type = get_key_type(obj->attributes); + + switch (class) { + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_PRIVATE_KEY: + break; + default: + TEE_Panic(class); + break; + } + + switch (type) { + case PKCS11_CKK_EC: + if (class == PKCS11_CKO_PRIVATE_KEY) { + if (function == PKCS11_FUNCTION_DERIVE) + *tee_type = TEE_TYPE_ECDH_KEYPAIR; + else + *tee_type = TEE_TYPE_ECDSA_KEYPAIR; + } else { + if (function == PKCS11_FUNCTION_DERIVE) + *tee_type = TEE_TYPE_ECDH_PUBLIC_KEY; + else + *tee_type = TEE_TYPE_ECDSA_PUBLIC_KEY; + } + break; + case PKCS11_CKK_RSA: + if (class == PKCS11_CKO_PRIVATE_KEY) + *tee_type = TEE_TYPE_RSA_KEYPAIR; + else + *tee_type = TEE_TYPE_RSA_PUBLIC_KEY; + break; + case PKCS11_CKK_EC_EDWARDS: + if (class == PKCS11_CKO_PRIVATE_KEY) + *tee_type = TEE_TYPE_ED25519_KEYPAIR; + else + *tee_type = TEE_TYPE_ED25519_PUBLIC_KEY; + break; + default: + TEE_Panic(type); + break; + } + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc +allocate_tee_operation(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_attribute_head *params, + struct pkcs11_object *obj) +{ + uint32_t size = (uint32_t)get_object_key_bit_size(obj); + uint32_t algo = 0; + uint32_t hash_algo = 0; + uint32_t mode = 0; + uint32_t hash_mode = 0; + TEE_Result res = TEE_ERROR_GENERIC; + struct active_processing *processing = session->processing; + + assert(processing->tee_op_handle == TEE_HANDLE_NULL); + assert(processing->tee_hash_op_handle == TEE_HANDLE_NULL); + + if (pkcs2tee_algorithm(&algo, &hash_algo, function, params, obj)) + return PKCS11_CKR_FUNCTION_FAILED; + + pkcs2tee_mode(&mode, function); + + if (hash_algo) { + pkcs2tee_mode(&hash_mode, PKCS11_FUNCTION_DIGEST); + + res = TEE_AllocateOperation(&processing->tee_hash_op_handle, + hash_algo, hash_mode, 0); + if (res) { + EMSG("TEE_AllocateOp. failed %#"PRIx32" %#"PRIx32, + hash_algo, hash_mode); + + if (res == TEE_ERROR_NOT_SUPPORTED) + return PKCS11_CKR_MECHANISM_INVALID; + return tee2pkcs_error(res); + } + processing->tee_hash_algo = hash_algo; + } + + res = TEE_AllocateOperation(&processing->tee_op_handle, + algo, mode, size); + if (res) + EMSG("TEE_AllocateOp. failed %#"PRIx32" %#"PRIx32" %#"PRIx32, + algo, mode, size); + + if (res == TEE_ERROR_NOT_SUPPORTED) + return PKCS11_CKR_MECHANISM_INVALID; + + if (res != TEE_SUCCESS && + processing->tee_hash_op_handle != TEE_HANDLE_NULL) { + TEE_FreeOperation(session->processing->tee_hash_op_handle); + processing->tee_hash_op_handle = TEE_HANDLE_NULL; + processing->tee_hash_algo = 0; + } + + return tee2pkcs_error(res); +} + +static enum pkcs11_rc load_tee_key(struct pkcs11_session *session, + struct pkcs11_object *obj, + enum processing_func function) +{ + TEE_Attribute *tee_attrs = NULL; + size_t tee_attrs_count = 0; + size_t object_size = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + TEE_Result res = TEE_ERROR_GENERIC; + enum pkcs11_class_id __maybe_unused class = get_class(obj->attributes); + enum pkcs11_key_type type = get_key_type(obj->attributes); + + assert(class == PKCS11_CKO_PUBLIC_KEY || + class == PKCS11_CKO_PRIVATE_KEY); + + if (obj->key_handle != TEE_HANDLE_NULL) { + switch (type) { + case PKCS11_CKK_RSA: + /* RSA loaded keys can be reused */ + assert((obj->key_type == TEE_TYPE_RSA_PUBLIC_KEY && + class == PKCS11_CKO_PUBLIC_KEY) || + (obj->key_type == TEE_TYPE_RSA_KEYPAIR && + class == PKCS11_CKO_PRIVATE_KEY)); + goto key_ready; + case PKCS11_CKK_EC: + /* Reuse EC TEE key only if already DSA or DH */ + switch (obj->key_type) { + case TEE_TYPE_ECDSA_PUBLIC_KEY: + case TEE_TYPE_ECDSA_KEYPAIR: + if (function != PKCS11_FUNCTION_DERIVE) + goto key_ready; + break; + case TEE_TYPE_ECDH_PUBLIC_KEY: + case TEE_TYPE_ECDH_KEYPAIR: + if (function == PKCS11_FUNCTION_DERIVE) + goto key_ready; + break; + default: + assert(0); + break; + } + break; + default: + assert(0); + break; + } + + TEE_CloseObject(obj->key_handle); + obj->key_handle = TEE_HANDLE_NULL; + } + + rc = pkcs2tee_key_type(&obj->key_type, obj, function); + if (rc) + return rc; + + object_size = get_object_key_bit_size(obj); + if (!object_size) + return PKCS11_CKR_GENERAL_ERROR; + + switch (type) { + case PKCS11_CKK_RSA: + rc = load_tee_rsa_key_attrs(&tee_attrs, &tee_attrs_count, obj); + break; + case PKCS11_CKK_EC: + rc = load_tee_ec_key_attrs(&tee_attrs, &tee_attrs_count, obj); + break; + case PKCS11_CKK_EC_EDWARDS: + rc = load_tee_eddsa_key_attrs(&tee_attrs, &tee_attrs_count, + obj); + break; + default: + break; + } + if (rc) + return rc; + + res = TEE_AllocateTransientObject(obj->key_type, object_size, + &obj->key_handle); + if (res) { + DMSG("TEE_AllocateTransientObject failed, %#"PRIx32, res); + + return tee2pkcs_error(res); + } + + res = TEE_PopulateTransientObject(obj->key_handle, + tee_attrs, tee_attrs_count); + + TEE_Free(tee_attrs); + + if (res) { + DMSG("TEE_PopulateTransientObject failed, %#"PRIx32, res); + + goto error; + } + +key_ready: + res = TEE_SetOperationKey(session->processing->tee_op_handle, + obj->key_handle); + if (res) { + DMSG("TEE_SetOperationKey failed, %#"PRIx32, res); + + goto error; + } + + return PKCS11_CKR_OK; + +error: + TEE_FreeTransientObject(obj->key_handle); + obj->key_handle = TEE_HANDLE_NULL; + return tee2pkcs_error(res); +} + +static enum pkcs11_rc +init_tee_operation(struct pkcs11_session *session, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct active_processing *proc = session->processing; + + switch (proc_params->id) { + case PKCS11_CKM_RSA_PKCS_PSS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + rc = pkcs2tee_proc_params_rsa_pss(proc, proc_params); + if (rc) + break; + + rc = pkcs2tee_validate_rsa_pss(proc, obj); + break; + case PKCS11_CKM_RSA_PKCS_OAEP: + rc = pkcs2tee_proc_params_rsa_oaep(proc, proc_params); + break; + case PKCS11_CKM_EDDSA: + rc = pkcs2tee_proc_params_eddsa(proc, proc_params); + break; + case PKCS11_CKM_RSA_AES_KEY_WRAP: + rc = pkcs2tee_proc_params_rsa_aes_wrap(proc, proc_params); + break; + default: + break; + } + + return rc; +} + +enum pkcs11_rc init_asymm_operation(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + assert(processing_is_tee_asymm(proc_params->id)); + + rc = allocate_tee_operation(session, function, proc_params, obj); + if (rc) + return rc; + + rc = load_tee_key(session, obj, function); + if (rc) + return rc; + + rc = init_tee_operation(session, proc_params, obj); + if (!rc) + session->processing->mecha_type = proc_params->id; + + return rc; +} + +/* + * step_sym_step - step (update/oneshot/final) on a symmetric crypto operation + * + * @session - current session + * @function - processing function (encrypt, decrypt, sign, ...) + * @step - step ID in the processing (oneshot, update, final) + * @ptypes - invocation parameter types + * @params - invocation parameter references + */ +enum pkcs11_rc step_asymm_operation(struct pkcs11_session *session, + enum processing_func function, + enum processing_step step, + uint32_t ptypes, TEE_Param *params) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + TEE_Result res = TEE_ERROR_GENERIC; + void *in_buf = NULL; + void *in2_buf = NULL; + void *out_buf = NULL; + void *hash_buf = NULL; + uint32_t in_size = 0; + uint32_t in2_size = 0; + size_t out_size = 0; + size_t hash_size = 0; + TEE_Attribute *tee_attrs = NULL; + size_t tee_attrs_count = 0; + bool output_data = false; + struct active_processing *proc = session->processing; + struct rsa_aes_key_wrap_processing_ctx *rsa_aes_ctx = NULL; + struct rsa_oaep_processing_ctx *rsa_oaep_ctx = NULL; + struct rsa_pss_processing_ctx *rsa_pss_ctx = NULL; + struct eddsa_processing_ctx *eddsa_ctx = NULL; + size_t sz = 0; + + if (TEE_PARAM_TYPE_GET(ptypes, 1) == TEE_PARAM_TYPE_MEMREF_INPUT) { + in_buf = params[1].memref.buffer; + in_size = params[1].memref.size; + if (in_size && !in_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 2) == TEE_PARAM_TYPE_MEMREF_INPUT) { + in2_buf = params[2].memref.buffer; + in2_size = params[2].memref.size; + if (in2_size && !in2_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 2) == TEE_PARAM_TYPE_MEMREF_OUTPUT) { + out_buf = params[2].memref.buffer; + out_size = params[2].memref.size; + if (out_size && !out_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 3) != TEE_PARAM_TYPE_NONE) + return PKCS11_CKR_ARGUMENTS_BAD; + + switch (step) { + case PKCS11_FUNC_STEP_ONESHOT: + case PKCS11_FUNC_STEP_UPDATE: + case PKCS11_FUNC_STEP_FINAL: + break; + default: + return PKCS11_CKR_GENERAL_ERROR; + } + + /* TEE attribute(s) required by the operation */ + switch (proc->mecha_type) { + case PKCS11_CKM_RSA_PKCS_PSS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + tee_attrs = TEE_Malloc(sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!tee_attrs) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + rsa_pss_ctx = proc->extra_ctx; + + TEE_InitValueAttribute(&tee_attrs[tee_attrs_count], + TEE_ATTR_RSA_PSS_SALT_LENGTH, + rsa_pss_ctx->salt_len, 0); + tee_attrs_count++; + break; + case PKCS11_CKM_EDDSA: + eddsa_ctx = proc->extra_ctx; + + tee_attrs = TEE_Malloc(2 * sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!tee_attrs) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + if (eddsa_ctx->flag) { + TEE_InitValueAttribute(&tee_attrs[tee_attrs_count], + TEE_ATTR_EDDSA_PREHASH, 0, 0); + tee_attrs_count++; + } + + if (eddsa_ctx->ctx_len > 0) { + TEE_InitRefAttribute(&tee_attrs[tee_attrs_count], + TEE_ATTR_EDDSA_CTX, eddsa_ctx->ctx, + eddsa_ctx->ctx_len); + tee_attrs_count++; + } + break; + case PKCS11_CKM_RSA_PKCS_OAEP: + rsa_oaep_ctx = proc->extra_ctx; + + if (!rsa_oaep_ctx->source_data_len) + break; + + tee_attrs = TEE_Malloc(sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!tee_attrs) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + TEE_InitRefAttribute(&tee_attrs[tee_attrs_count], + TEE_ATTR_RSA_OAEP_LABEL, + rsa_oaep_ctx->source_data, + rsa_oaep_ctx->source_data_len); + tee_attrs_count++; + break; + case PKCS11_CKM_RSA_AES_KEY_WRAP: + rsa_aes_ctx = proc->extra_ctx; + + if (!rsa_aes_ctx->source_data_len) + break; + + tee_attrs = TEE_Malloc(sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!tee_attrs) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto out; + } + + TEE_InitRefAttribute(&tee_attrs[tee_attrs_count], + TEE_ATTR_RSA_OAEP_LABEL, + rsa_aes_ctx->source_data, + rsa_aes_ctx->source_data_len); + tee_attrs_count++; + break; + default: + break; + } + + /* + * Handle multi stage update step for mechas needing hash + * calculation + */ + if (step == PKCS11_FUNC_STEP_UPDATE) { + switch (proc->mecha_type) { + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + case PKCS11_CKM_MD5_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS: + case PKCS11_CKM_SHA224_RSA_PKCS: + case PKCS11_CKM_SHA256_RSA_PKCS: + case PKCS11_CKM_SHA384_RSA_PKCS: + case PKCS11_CKM_SHA512_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + assert(proc->tee_hash_op_handle != TEE_HANDLE_NULL); + + TEE_DigestUpdate(proc->tee_hash_op_handle, in_buf, + in_size); + rc = PKCS11_CKR_OK; + break; + default: + /* + * Other mechanism do not expect multi stage + * operation + */ + rc = PKCS11_CKR_GENERAL_ERROR; + break; + } + + goto out; + } + + /* + * Handle multi stage one shot and final steps for mechas needing hash + * calculation + */ + switch (proc->mecha_type) { + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + case PKCS11_CKM_MD5_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS: + case PKCS11_CKM_SHA224_RSA_PKCS: + case PKCS11_CKM_SHA256_RSA_PKCS: + case PKCS11_CKM_SHA384_RSA_PKCS: + case PKCS11_CKM_SHA512_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + assert(proc->tee_hash_op_handle != TEE_HANDLE_NULL); + + hash_size = TEE_ALG_GET_DIGEST_SIZE(proc->tee_hash_algo); + hash_buf = TEE_Malloc(hash_size, 0); + if (!hash_buf) + return PKCS11_CKR_DEVICE_MEMORY; + + res = TEE_DigestDoFinal(proc->tee_hash_op_handle, + in_buf, in_size, hash_buf, + &hash_size); + + rc = tee2pkcs_error(res); + if (rc != PKCS11_CKR_OK) + goto out; + + break; + default: + break; + } + + /* + * Finalize either provided hash or calculated hash with signing + * operation + */ + + /* First determine amount of bytes for signing operation */ + switch (proc->mecha_type) { + case PKCS11_CKM_ECDSA: + sz = ecdsa_get_input_max_byte_size(proc->tee_op_handle); + if (!in_size || !sz) { + rc = PKCS11_CKR_FUNCTION_FAILED; + goto out; + } + + /* + * Note 3) Input the entire raw digest. Internally, this will + * be truncated to the appropriate number of bits. + */ + if (in_size > sz) + in_size = sz; + + if (function == PKCS11_FUNCTION_VERIFY && in2_size != 2 * sz) { + rc = PKCS11_CKR_SIGNATURE_LEN_RANGE; + goto out; + } + break; + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + /* Get key size in bytes */ + sz = ecdsa_get_input_max_byte_size(proc->tee_op_handle); + if (!sz) { + rc = PKCS11_CKR_FUNCTION_FAILED; + goto out; + } + + if (function == PKCS11_FUNCTION_VERIFY && + in2_size != 2 * sz) { + rc = PKCS11_CKR_SIGNATURE_LEN_RANGE; + goto out; + } + break; + case PKCS11_CKM_RSA_PKCS: + case PKCS11_CKM_MD5_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS: + case PKCS11_CKM_SHA224_RSA_PKCS: + case PKCS11_CKM_SHA256_RSA_PKCS: + case PKCS11_CKM_SHA384_RSA_PKCS: + case PKCS11_CKM_SHA512_RSA_PKCS: + case PKCS11_CKM_RSA_PKCS_PSS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + /* Get key size in bytes */ + sz = rsa_get_input_max_byte_size(proc->tee_op_handle); + if (!sz) { + rc = PKCS11_CKR_FUNCTION_FAILED; + goto out; + } + + if (function == PKCS11_FUNCTION_VERIFY && in2_size != sz) { + rc = PKCS11_CKR_SIGNATURE_LEN_RANGE; + goto out; + } + break; + default: + break; + } + + /* Next perform actual signing operation */ + switch (proc->mecha_type) { + case PKCS11_CKM_ECDSA: + case PKCS11_CKM_EDDSA: + case PKCS11_CKM_RSA_PKCS: + case PKCS11_CKM_RSA_PKCS_OAEP: + case PKCS11_CKM_RSA_PKCS_PSS: + /* For operations using provided input data */ + switch (function) { + case PKCS11_FUNCTION_ENCRYPT: + res = TEE_AsymmetricEncrypt(proc->tee_op_handle, + tee_attrs, tee_attrs_count, + in_buf, in_size, + out_buf, &out_size); + output_data = true; + rc = tee2pkcs_error(res); + if (rc == PKCS11_CKR_ARGUMENTS_BAD) + rc = PKCS11_CKR_DATA_LEN_RANGE; + break; + + case PKCS11_FUNCTION_DECRYPT: + res = TEE_AsymmetricDecrypt(proc->tee_op_handle, + tee_attrs, tee_attrs_count, + in_buf, in_size, + out_buf, &out_size); + output_data = true; + rc = tee2pkcs_error(res); + if (rc == PKCS11_CKR_ARGUMENTS_BAD) + rc = PKCS11_CKR_ENCRYPTED_DATA_LEN_RANGE; + break; + + case PKCS11_FUNCTION_SIGN: + res = TEE_AsymmetricSignDigest(proc->tee_op_handle, + tee_attrs, + tee_attrs_count, + in_buf, in_size, + out_buf, &out_size); + output_data = true; + rc = tee2pkcs_error(res); + break; + + case PKCS11_FUNCTION_VERIFY: + res = TEE_AsymmetricVerifyDigest(proc->tee_op_handle, + tee_attrs, + tee_attrs_count, + in_buf, in_size, + in2_buf, in2_size); + rc = tee2pkcs_error(res); + break; + + default: + TEE_Panic(function); + break; + } + break; + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + case PKCS11_CKM_MD5_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS: + case PKCS11_CKM_SHA224_RSA_PKCS: + case PKCS11_CKM_SHA256_RSA_PKCS: + case PKCS11_CKM_SHA384_RSA_PKCS: + case PKCS11_CKM_SHA512_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + /* For operations having hash operation use calculated hash */ + switch (function) { + case PKCS11_FUNCTION_SIGN: + res = TEE_AsymmetricSignDigest(proc->tee_op_handle, + tee_attrs, + tee_attrs_count, + hash_buf, hash_size, + out_buf, &out_size); + output_data = true; + rc = tee2pkcs_error(res); + break; + + case PKCS11_FUNCTION_VERIFY: + res = TEE_AsymmetricVerifyDigest(proc->tee_op_handle, + tee_attrs, + tee_attrs_count, + hash_buf, hash_size, + in2_buf, in2_size); + rc = tee2pkcs_error(res); + break; + + default: + TEE_Panic(function); + break; + } + break; + default: + TEE_Panic(proc->mecha_type); + break; + } + +out: + if (output_data && + (rc == PKCS11_CKR_OK || rc == PKCS11_CKR_BUFFER_TOO_SMALL)) { + switch (TEE_PARAM_TYPE_GET(ptypes, 2)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + params[2].memref.size = out_size; + break; + default: + rc = PKCS11_CKR_GENERAL_ERROR; + break; + } + } + + TEE_Free(hash_buf); + TEE_Free(tee_attrs); + + return rc; +} + +enum pkcs11_rc do_asymm_derivation(struct pkcs11_session *session, + struct pkcs11_attribute_head *proc_params, + struct obj_attrs **head) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + TEE_ObjectHandle out_handle = TEE_HANDLE_NULL; + TEE_Result res = TEE_ERROR_GENERIC; + TEE_Attribute tee_attrs[2] = { }; + size_t tee_attrs_count = 0; + uint32_t key_byte_size = 0; + uint32_t key_bit_size = 0; + void *a_ptr = NULL; + size_t a_size = 0; + + /* Remove default attribute set at template sanitization */ + if (remove_empty_attribute(head, PKCS11_CKA_VALUE)) + return PKCS11_CKR_FUNCTION_FAILED; + + rc = get_u32_attribute(*head, PKCS11_CKA_VALUE_LEN, &key_bit_size); + if (rc) + return rc; + + key_bit_size *= 8; + key_byte_size = (key_bit_size + 7) / 8; + + res = TEE_AllocateTransientObject(TEE_TYPE_GENERIC_SECRET, + key_byte_size * 8, &out_handle); + if (res) { + DMSG("TEE_AllocateTransientObject failed, %#"PRIx32, res); + return tee2pkcs_error(res); + } + + switch (proc_params->id) { + case PKCS11_CKM_ECDH1_DERIVE: + rc = pkcs2tee_param_ecdh(proc_params, &a_ptr, &a_size); + if (rc) + goto out; + + TEE_InitRefAttribute(&tee_attrs[tee_attrs_count], + TEE_ATTR_ECC_PUBLIC_VALUE_X, + a_ptr, a_size / 2); + tee_attrs_count++; + TEE_InitRefAttribute(&tee_attrs[tee_attrs_count], + TEE_ATTR_ECC_PUBLIC_VALUE_Y, + (char *)a_ptr + a_size / 2, + a_size / 2); + tee_attrs_count++; + break; + default: + TEE_Panic(proc_params->id); + break; + } + + TEE_DeriveKey(session->processing->tee_op_handle, &tee_attrs[0], + tee_attrs_count, out_handle); + + rc = alloc_get_tee_attribute_data(out_handle, TEE_ATTR_SECRET_VALUE, + &a_ptr, &a_size); + if (rc) + goto out; + + if (a_size * 8 < key_bit_size) + rc = PKCS11_CKR_KEY_SIZE_RANGE; + else + rc = add_attribute(head, PKCS11_CKA_VALUE, a_ptr, + key_byte_size); + TEE_Free(a_ptr); +out: + release_active_processing(session); + TEE_FreeTransientObject(out_handle); + + return rc; +} + +static enum pkcs11_rc wrap_rsa_aes_key(struct active_processing *proc, + void *data, uint32_t data_sz, + void *out_buf, uint32_t *out_sz) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + TEE_Result res = TEE_ERROR_GENERIC; + int mbedtls_rc = 0; + struct rsa_aes_key_wrap_processing_ctx *ctx = proc->extra_ctx; + mbedtls_nist_kw_context kw_ctx = { }; + uint8_t aes_key_value[32] = { }; + uint32_t aes_key_size = ctx->aes_key_bits / 8; + size_t aes_wrapped_size = *out_sz; + uint32_t expected_size = 0; + size_t target_key_size = 0; + const size_t kw_semiblock_len = 8; + + if (ctx->aes_key_bits != 128 && + ctx->aes_key_bits != 192 && + ctx->aes_key_bits != 256) + return PKCS11_CKR_ARGUMENTS_BAD; + + mbedtls_nist_kw_init(&kw_ctx); + TEE_GenerateRandom(aes_key_value, aes_key_size); + res = TEE_AsymmetricEncrypt(proc->tee_op_handle, + NULL, 0, + aes_key_value, aes_key_size, + out_buf, &aes_wrapped_size); + expected_size = aes_wrapped_size + data_sz + kw_semiblock_len; + if (res) { + if (res == TEE_ERROR_SHORT_BUFFER) + *out_sz = expected_size; + + rc = tee2pkcs_error(res); + goto out; + } + + if (*out_sz < expected_size) { + rc = PKCS11_CKR_BUFFER_TOO_SMALL; + *out_sz = expected_size; + goto out; + } + + mbedtls_rc = mbedtls_nist_kw_setkey(&kw_ctx, MBEDTLS_CIPHER_ID_AES, + aes_key_value, ctx->aes_key_bits, + true); + if (mbedtls_rc) { + if (mbedtls_rc == MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA) + rc = PKCS11_CKR_KEY_SIZE_RANGE; + else + rc = PKCS11_CKR_FUNCTION_FAILED; + + goto out; + } + + mbedtls_rc = mbedtls_nist_kw_wrap(&kw_ctx, MBEDTLS_KW_MODE_KWP, + data, data_sz, + (uint8_t *)out_buf + aes_wrapped_size, + &target_key_size, + *out_sz - aes_wrapped_size); + if (mbedtls_rc) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto out; + } + + assert(*out_sz >= target_key_size + aes_wrapped_size); + *out_sz = target_key_size + aes_wrapped_size; + +out: + mbedtls_nist_kw_free(&kw_ctx); + TEE_MemFill(aes_key_value, 0, aes_key_size); + return rc; +} + +static enum pkcs11_rc unwrap_rsa_aes_key(struct active_processing *proc, + void *data, uint32_t data_sz, + void **out_buf, uint32_t *out_sz) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + int mbedtls_rc = 0; + TEE_Result res = TEE_ERROR_GENERIC; + TEE_OperationInfo info = { }; + struct rsa_aes_key_wrap_processing_ctx *ctx = proc->extra_ctx; + mbedtls_nist_kw_context kw_ctx = { }; + uint8_t aes_key_value[32] = { }; + size_t aes_key_size = ctx->aes_key_bits / 8; + uint32_t wrapped_key_size = 0; + uint32_t rsa_key_size = 0; + size_t target_key_size = 0; + + if (ctx->aes_key_bits != 128 && + ctx->aes_key_bits != 192 && + ctx->aes_key_bits != 256) + return PKCS11_CKR_ARGUMENTS_BAD; + + TEE_GetOperationInfo(proc->tee_op_handle, &info); + rsa_key_size = info.keySize / 8; + wrapped_key_size = data_sz - rsa_key_size; + target_key_size = wrapped_key_size - 8; + + *out_buf = TEE_Malloc(target_key_size, TEE_MALLOC_FILL_ZERO); + if (!*out_buf) + return PKCS11_CKR_DEVICE_MEMORY; + + mbedtls_nist_kw_init(&kw_ctx); + res = TEE_AsymmetricDecrypt(proc->tee_op_handle, + NULL, 0, + data, rsa_key_size, + aes_key_value, &aes_key_size); + if (res) { + rc = tee2pkcs_error(res); + goto out; + } + + mbedtls_rc = mbedtls_nist_kw_setkey(&kw_ctx, MBEDTLS_CIPHER_ID_AES, + aes_key_value, ctx->aes_key_bits, + false); + if (mbedtls_rc) { + rc = PKCS11_CKR_WRAPPED_KEY_INVALID; + goto out; + } + + mbedtls_rc = mbedtls_nist_kw_unwrap(&kw_ctx, MBEDTLS_KW_MODE_KWP, + (uint8_t *)data + rsa_key_size, + wrapped_key_size, *out_buf, + &target_key_size, target_key_size); + if (mbedtls_rc) { + rc = PKCS11_CKR_WRAPPED_KEY_INVALID; + goto out; + } + + *out_sz = target_key_size; +out: + TEE_MemFill(aes_key_value, 0, aes_key_size); + mbedtls_nist_kw_free(&kw_ctx); + return rc; +} + +enum pkcs11_rc wrap_data_by_asymm_enc(struct pkcs11_session *session, + void *data, uint32_t data_sz, + void *out_buf, uint32_t *out_sz) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct active_processing *proc = session->processing; + + switch (proc->mecha_type) { + case PKCS11_CKM_RSA_AES_KEY_WRAP: + rc = wrap_rsa_aes_key(proc, data, data_sz, out_buf, out_sz); + break; + default: + return PKCS11_CKR_MECHANISM_INVALID; + } + + return rc; +} + +enum pkcs11_rc unwrap_key_by_asymm(struct pkcs11_session *session, + void *data, uint32_t data_sz, + void **out_buf, uint32_t *out_sz) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct active_processing *proc = session->processing; + + switch (proc->mecha_type) { + case PKCS11_CKM_RSA_AES_KEY_WRAP: + rc = unwrap_rsa_aes_key(proc, data, data_sz, out_buf, out_sz); + break; + default: + return PKCS11_CKR_MECHANISM_INVALID; + } + + return rc; +} diff --git a/optee_os/ta/pkcs11/src/processing_digest.c b/optee_os/ta/pkcs11/src/processing_digest.c new file mode 100644 index 0000000..c82f489 --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing_digest.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021, Vaisala Oyj + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "object.h" +#include "pkcs11_attributes.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" +#include "serializer.h" + +bool processing_is_tee_digest(enum pkcs11_mechanism_id mecha_id) +{ + switch (mecha_id) { + case PKCS11_CKM_MD5: + case PKCS11_CKM_SHA_1: + case PKCS11_CKM_SHA224: + case PKCS11_CKM_SHA256: + case PKCS11_CKM_SHA384: + case PKCS11_CKM_SHA512: + return true; + default: + return false; + } +} + +static enum pkcs11_rc +pkcs2tee_algorithm(uint32_t *tee_id, struct pkcs11_attribute_head *proc_params) +{ + static const struct { + enum pkcs11_mechanism_id mech_id; + uint32_t tee_id; + } pkcs2tee_algo[] = { + { PKCS11_CKM_MD5, TEE_ALG_MD5 }, + { PKCS11_CKM_SHA_1, TEE_ALG_SHA1 }, + { PKCS11_CKM_SHA224, TEE_ALG_SHA224 }, + { PKCS11_CKM_SHA256, TEE_ALG_SHA256 }, + { PKCS11_CKM_SHA384, TEE_ALG_SHA384 }, + { PKCS11_CKM_SHA512, TEE_ALG_SHA512 }, + }; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(pkcs2tee_algo); n++) { + if (proc_params->id == pkcs2tee_algo[n].mech_id) { + *tee_id = pkcs2tee_algo[n].tee_id; + return PKCS11_CKR_OK; + } + } + + return PKCS11_RV_NOT_IMPLEMENTED; +} + +static enum pkcs11_rc +allocate_tee_operation(struct pkcs11_session *session, + struct pkcs11_attribute_head *params) +{ + uint32_t algo = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + assert(session->processing->tee_op_handle == TEE_HANDLE_NULL); + + if (pkcs2tee_algorithm(&algo, params)) + return PKCS11_CKR_FUNCTION_FAILED; + + res = TEE_AllocateOperation(&session->processing->tee_op_handle, + algo, TEE_MODE_DIGEST, 0); + if (res) + EMSG("TEE_AllocateOp. failed %#"PRIx32, algo); + + if (res == TEE_ERROR_NOT_SUPPORTED) + return PKCS11_CKR_MECHANISM_INVALID; + + return tee2pkcs_error(res); +} + +enum pkcs11_rc init_digest_operation(struct pkcs11_session *session, + struct pkcs11_attribute_head *proc_params) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + assert(processing_is_tee_digest(proc_params->id)); + + rc = allocate_tee_operation(session, proc_params); + if (!rc) + session->processing->mecha_type = proc_params->id; + + return rc; +} + +/* + * step_digest_operation - processing digest operation step + * + * @session - current session + * @step - step ID in the processing (oneshot, update, final) + * @obj - PKCS#11 object for key based operations + * @ptype - invocation parameter types + * @params - invocation parameter references + */ +enum pkcs11_rc step_digest_operation(struct pkcs11_session *session, + enum processing_step step, + struct pkcs11_object *obj, + uint32_t ptypes, TEE_Param *params) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + TEE_Result res = TEE_ERROR_GENERIC; + void *in_buf = NULL; + size_t in_size = 0; + void *out_buf = NULL; + size_t out_size = 0; + void *secret_value = NULL; + uint32_t secret_value_size = 0; + enum pkcs11_key_type key_type = PKCS11_CKK_UNDEFINED_ID; + struct active_processing *proc = session->processing; + + if (TEE_PARAM_TYPE_GET(ptypes, 1) == TEE_PARAM_TYPE_MEMREF_INPUT) { + in_buf = params[1].memref.buffer; + in_size = params[1].memref.size; + if (in_size && !in_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 2) == TEE_PARAM_TYPE_MEMREF_OUTPUT) { + out_buf = params[2].memref.buffer; + out_size = params[2].memref.size; + if (out_size && !out_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 3) != TEE_PARAM_TYPE_NONE) + return PKCS11_CKR_ARGUMENTS_BAD; + + switch (step) { + case PKCS11_FUNC_STEP_ONESHOT: + case PKCS11_FUNC_STEP_UPDATE: + case PKCS11_FUNC_STEP_UPDATE_KEY: + case PKCS11_FUNC_STEP_FINAL: + break; + default: + TEE_Panic(step); + break; + } + + assert(proc->tee_op_handle != TEE_HANDLE_NULL); + + assert(processing_is_tee_digest(proc->mecha_type)); + + /* + * Feed active operation with data + */ + + switch (step) { + case PKCS11_FUNC_STEP_UPDATE_KEY: + assert(obj); + + if (!IS_ENABLED(CFG_PKCS11_TA_ALLOW_DIGEST_KEY)) + return PKCS11_CKR_KEY_INDIGESTIBLE; + + if (get_class(obj->attributes) != PKCS11_CKO_SECRET_KEY) + return PKCS11_CKR_KEY_INDIGESTIBLE; + + key_type = get_key_type(obj->attributes); + + if (key_type != PKCS11_CKK_GENERIC_SECRET && + key_type != PKCS11_CKK_AES) + return PKCS11_CKR_KEY_INDIGESTIBLE; + + rc = get_attribute_ptr(obj->attributes, PKCS11_CKA_VALUE, + &secret_value, &secret_value_size); + assert(!rc && secret_value && secret_value_size); + + TEE_DigestUpdate(proc->tee_op_handle, secret_value, + secret_value_size); + return PKCS11_CKR_OK; + + case PKCS11_FUNC_STEP_UPDATE: + if (!in_buf || !in_size) + return PKCS11_CKR_OK; + + TEE_DigestUpdate(proc->tee_op_handle, in_buf, in_size); + return PKCS11_CKR_OK; + + case PKCS11_FUNC_STEP_ONESHOT: + if (!out_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + + goto do_final; + + case PKCS11_FUNC_STEP_FINAL: + if (in_buf || !out_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + + goto do_final; + + default: + TEE_Panic(step); + break; + } + +do_final: + res = TEE_DigestDoFinal(proc->tee_op_handle, + in_buf, in_size, out_buf, + &out_size); + rc = tee2pkcs_error(res); + + if (rc == PKCS11_CKR_OK || rc == PKCS11_CKR_BUFFER_TOO_SMALL) + params[2].memref.size = out_size; + + return rc; +} diff --git a/optee_os/ta/pkcs11/src/processing_ec.c b/optee_os/ta/pkcs11/src/processing_ec.c new file mode 100644 index 0000000..adf9b5e --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing_ec.c @@ -0,0 +1,965 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "object.h" +#include "pkcs11_token.h" +#include "processing.h" + +/* + * DER encoded EC parameters generated with script: + * ta/pkcs11/scripts/dump_ec_curve_params.sh + */ + +static const uint8_t prime192v1_name_der[] = { + 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, + 0x01, 0x01, +}; + +static const uint8_t secp224r1_name_der[] = { + 0x06, 0x05, 0x2b, 0x81, 0x04, 0x00, 0x21, +}; + +static const uint8_t prime256v1_name_der[] = { + 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, + 0x01, 0x07, +}; + +static const uint8_t secp384r1_name_der[] = { + 0x06, 0x05, 0x2b, 0x81, 0x04, 0x00, 0x22, +}; + +static const uint8_t secp521r1_name_der[] = { + 0x06, 0x05, 0x2b, 0x81, 0x04, 0x00, 0x23, +}; + +static const uint8_t prime192v1_oid_der[] = { + 0x30, 0x81, 0xc7, 0x02, 0x01, 0x01, 0x30, 0x24, + 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x01, + 0x01, 0x02, 0x19, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x30, 0x4b, 0x04, 0x18, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, + 0x04, 0x18, 0x64, 0x21, 0x05, 0x19, 0xe5, 0x9c, + 0x80, 0xe7, 0x0f, 0xa7, 0xe9, 0xab, 0x72, 0x24, + 0x30, 0x49, 0xfe, 0xb8, 0xde, 0xec, 0xc1, 0x46, + 0xb9, 0xb1, 0x03, 0x15, 0x00, 0x30, 0x45, 0xae, + 0x6f, 0xc8, 0x42, 0x2f, 0x64, 0xed, 0x57, 0x95, + 0x28, 0xd3, 0x81, 0x20, 0xea, 0xe1, 0x21, 0x96, + 0xd5, 0x04, 0x31, 0x04, 0x18, 0x8d, 0xa8, 0x0e, + 0xb0, 0x30, 0x90, 0xf6, 0x7c, 0xbf, 0x20, 0xeb, + 0x43, 0xa1, 0x88, 0x00, 0xf4, 0xff, 0x0a, 0xfd, + 0x82, 0xff, 0x10, 0x12, 0x07, 0x19, 0x2b, 0x95, + 0xff, 0xc8, 0xda, 0x78, 0x63, 0x10, 0x11, 0xed, + 0x6b, 0x24, 0xcd, 0xd5, 0x73, 0xf9, 0x77, 0xa1, + 0x1e, 0x79, 0x48, 0x11, 0x02, 0x19, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x99, 0xde, 0xf8, 0x36, 0x14, + 0x6b, 0xc9, 0xb1, 0xb4, 0xd2, 0x28, 0x31, 0x02, + 0x01, 0x01, +}; + +static const uint8_t secp224r1_oid_der[] = { + 0x30, 0x81, 0xdf, 0x02, 0x01, 0x01, 0x30, 0x28, + 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x01, + 0x01, 0x02, 0x1d, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x30, 0x53, 0x04, 0x1c, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, + 0x04, 0x1c, 0xb4, 0x05, 0x0a, 0x85, 0x0c, 0x04, + 0xb3, 0xab, 0xf5, 0x41, 0x32, 0x56, 0x50, 0x44, + 0xb0, 0xb7, 0xd7, 0xbf, 0xd8, 0xba, 0x27, 0x0b, + 0x39, 0x43, 0x23, 0x55, 0xff, 0xb4, 0x03, 0x15, + 0x00, 0xbd, 0x71, 0x34, 0x47, 0x99, 0xd5, 0xc7, + 0xfc, 0xdc, 0x45, 0xb5, 0x9f, 0xa3, 0xb9, 0xab, + 0x8f, 0x6a, 0x94, 0x8b, 0xc5, 0x04, 0x39, 0x04, + 0xb7, 0x0e, 0x0c, 0xbd, 0x6b, 0xb4, 0xbf, 0x7f, + 0x32, 0x13, 0x90, 0xb9, 0x4a, 0x03, 0xc1, 0xd3, + 0x56, 0xc2, 0x11, 0x22, 0x34, 0x32, 0x80, 0xd6, + 0x11, 0x5c, 0x1d, 0x21, 0xbd, 0x37, 0x63, 0x88, + 0xb5, 0xf7, 0x23, 0xfb, 0x4c, 0x22, 0xdf, 0xe6, + 0xcd, 0x43, 0x75, 0xa0, 0x5a, 0x07, 0x47, 0x64, + 0x44, 0xd5, 0x81, 0x99, 0x85, 0x00, 0x7e, 0x34, + 0x02, 0x1d, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x16, 0xa2, 0xe0, 0xb8, 0xf0, 0x3e, 0x13, + 0xdd, 0x29, 0x45, 0x5c, 0x5c, 0x2a, 0x3d, 0x02, + 0x01, 0x01, +}; + +static const uint8_t prime256v1_oid_der[] = { + 0x30, 0x81, 0xf7, 0x02, 0x01, 0x01, 0x30, 0x2c, + 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x01, + 0x01, 0x02, 0x21, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x30, 0x5b, 0x04, 0x20, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, + 0x04, 0x20, 0x5a, 0xc6, 0x35, 0xd8, 0xaa, 0x3a, + 0x93, 0xe7, 0xb3, 0xeb, 0xbd, 0x55, 0x76, 0x98, + 0x86, 0xbc, 0x65, 0x1d, 0x06, 0xb0, 0xcc, 0x53, + 0xb0, 0xf6, 0x3b, 0xce, 0x3c, 0x3e, 0x27, 0xd2, + 0x60, 0x4b, 0x03, 0x15, 0x00, 0xc4, 0x9d, 0x36, + 0x08, 0x86, 0xe7, 0x04, 0x93, 0x6a, 0x66, 0x78, + 0xe1, 0x13, 0x9d, 0x26, 0xb7, 0x81, 0x9f, 0x7e, + 0x90, 0x04, 0x41, 0x04, 0x6b, 0x17, 0xd1, 0xf2, + 0xe1, 0x2c, 0x42, 0x47, 0xf8, 0xbc, 0xe6, 0xe5, + 0x63, 0xa4, 0x40, 0xf2, 0x77, 0x03, 0x7d, 0x81, + 0x2d, 0xeb, 0x33, 0xa0, 0xf4, 0xa1, 0x39, 0x45, + 0xd8, 0x98, 0xc2, 0x96, 0x4f, 0xe3, 0x42, 0xe2, + 0xfe, 0x1a, 0x7f, 0x9b, 0x8e, 0xe7, 0xeb, 0x4a, + 0x7c, 0x0f, 0x9e, 0x16, 0x2b, 0xce, 0x33, 0x57, + 0x6b, 0x31, 0x5e, 0xce, 0xcb, 0xb6, 0x40, 0x68, + 0x37, 0xbf, 0x51, 0xf5, 0x02, 0x21, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbc, + 0xe6, 0xfa, 0xad, 0xa7, 0x17, 0x9e, 0x84, 0xf3, + 0xb9, 0xca, 0xc2, 0xfc, 0x63, 0x25, 0x51, 0x02, + 0x01, 0x01, +}; + +static const uint8_t secp384r1_oid_der[] = { + 0x30, 0x82, 0x01, 0x57, 0x02, 0x01, 0x01, 0x30, + 0x3c, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, + 0x01, 0x01, 0x02, 0x31, 0x00, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x30, 0x7b, 0x04, + 0x30, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfe, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, + 0xfc, 0x04, 0x30, 0xb3, 0x31, 0x2f, 0xa7, 0xe2, + 0x3e, 0xe7, 0xe4, 0x98, 0x8e, 0x05, 0x6b, 0xe3, + 0xf8, 0x2d, 0x19, 0x18, 0x1d, 0x9c, 0x6e, 0xfe, + 0x81, 0x41, 0x12, 0x03, 0x14, 0x08, 0x8f, 0x50, + 0x13, 0x87, 0x5a, 0xc6, 0x56, 0x39, 0x8d, 0x8a, + 0x2e, 0xd1, 0x9d, 0x2a, 0x85, 0xc8, 0xed, 0xd3, + 0xec, 0x2a, 0xef, 0x03, 0x15, 0x00, 0xa3, 0x35, + 0x92, 0x6a, 0xa3, 0x19, 0xa2, 0x7a, 0x1d, 0x00, + 0x89, 0x6a, 0x67, 0x73, 0xa4, 0x82, 0x7a, 0xcd, + 0xac, 0x73, 0x04, 0x61, 0x04, 0xaa, 0x87, 0xca, + 0x22, 0xbe, 0x8b, 0x05, 0x37, 0x8e, 0xb1, 0xc7, + 0x1e, 0xf3, 0x20, 0xad, 0x74, 0x6e, 0x1d, 0x3b, + 0x62, 0x8b, 0xa7, 0x9b, 0x98, 0x59, 0xf7, 0x41, + 0xe0, 0x82, 0x54, 0x2a, 0x38, 0x55, 0x02, 0xf2, + 0x5d, 0xbf, 0x55, 0x29, 0x6c, 0x3a, 0x54, 0x5e, + 0x38, 0x72, 0x76, 0x0a, 0xb7, 0x36, 0x17, 0xde, + 0x4a, 0x96, 0x26, 0x2c, 0x6f, 0x5d, 0x9e, 0x98, + 0xbf, 0x92, 0x92, 0xdc, 0x29, 0xf8, 0xf4, 0x1d, + 0xbd, 0x28, 0x9a, 0x14, 0x7c, 0xe9, 0xda, 0x31, + 0x13, 0xb5, 0xf0, 0xb8, 0xc0, 0x0a, 0x60, 0xb1, + 0xce, 0x1d, 0x7e, 0x81, 0x9d, 0x7a, 0x43, 0x1d, + 0x7c, 0x90, 0xea, 0x0e, 0x5f, 0x02, 0x31, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc7, 0x63, 0x4d, 0x81, 0xf4, 0x37, 0x2d, 0xdf, + 0x58, 0x1a, 0x0d, 0xb2, 0x48, 0xb0, 0xa7, 0x7a, + 0xec, 0xec, 0x19, 0x6a, 0xcc, 0xc5, 0x29, 0x73, + 0x02, 0x01, 0x01, +}; + +static const uint8_t secp521r1_oid_der[] = { + 0x30, 0x82, 0x01, 0xc3, 0x02, 0x01, 0x01, 0x30, + 0x4d, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, + 0x01, 0x01, 0x02, 0x42, 0x01, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x30, 0x81, + 0x9f, 0x04, 0x42, 0x01, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xfc, 0x04, 0x42, 0x00, + 0x51, 0x95, 0x3e, 0xb9, 0x61, 0x8e, 0x1c, 0x9a, + 0x1f, 0x92, 0x9a, 0x21, 0xa0, 0xb6, 0x85, 0x40, + 0xee, 0xa2, 0xda, 0x72, 0x5b, 0x99, 0xb3, 0x15, + 0xf3, 0xb8, 0xb4, 0x89, 0x91, 0x8e, 0xf1, 0x09, + 0xe1, 0x56, 0x19, 0x39, 0x51, 0xec, 0x7e, 0x93, + 0x7b, 0x16, 0x52, 0xc0, 0xbd, 0x3b, 0xb1, 0xbf, + 0x07, 0x35, 0x73, 0xdf, 0x88, 0x3d, 0x2c, 0x34, + 0xf1, 0xef, 0x45, 0x1f, 0xd4, 0x6b, 0x50, 0x3f, + 0x00, 0x03, 0x15, 0x00, 0xd0, 0x9e, 0x88, 0x00, + 0x29, 0x1c, 0xb8, 0x53, 0x96, 0xcc, 0x67, 0x17, + 0x39, 0x32, 0x84, 0xaa, 0xa0, 0xda, 0x64, 0xba, + 0x04, 0x81, 0x85, 0x04, 0x00, 0xc6, 0x85, 0x8e, + 0x06, 0xb7, 0x04, 0x04, 0xe9, 0xcd, 0x9e, 0x3e, + 0xcb, 0x66, 0x23, 0x95, 0xb4, 0x42, 0x9c, 0x64, + 0x81, 0x39, 0x05, 0x3f, 0xb5, 0x21, 0xf8, 0x28, + 0xaf, 0x60, 0x6b, 0x4d, 0x3d, 0xba, 0xa1, 0x4b, + 0x5e, 0x77, 0xef, 0xe7, 0x59, 0x28, 0xfe, 0x1d, + 0xc1, 0x27, 0xa2, 0xff, 0xa8, 0xde, 0x33, 0x48, + 0xb3, 0xc1, 0x85, 0x6a, 0x42, 0x9b, 0xf9, 0x7e, + 0x7e, 0x31, 0xc2, 0xe5, 0xbd, 0x66, 0x01, 0x18, + 0x39, 0x29, 0x6a, 0x78, 0x9a, 0x3b, 0xc0, 0x04, + 0x5c, 0x8a, 0x5f, 0xb4, 0x2c, 0x7d, 0x1b, 0xd9, + 0x98, 0xf5, 0x44, 0x49, 0x57, 0x9b, 0x44, 0x68, + 0x17, 0xaf, 0xbd, 0x17, 0x27, 0x3e, 0x66, 0x2c, + 0x97, 0xee, 0x72, 0x99, 0x5e, 0xf4, 0x26, 0x40, + 0xc5, 0x50, 0xb9, 0x01, 0x3f, 0xad, 0x07, 0x61, + 0x35, 0x3c, 0x70, 0x86, 0xa2, 0x72, 0xc2, 0x40, + 0x88, 0xbe, 0x94, 0x76, 0x9f, 0xd1, 0x66, 0x50, + 0x02, 0x42, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xfa, 0x51, 0x86, 0x87, 0x83, + 0xbf, 0x2f, 0x96, 0x6b, 0x7f, 0xcc, 0x01, 0x48, + 0xf7, 0x09, 0xa5, 0xd0, 0x3b, 0xb5, 0xc9, 0xb8, + 0x89, 0x9c, 0x47, 0xae, 0xbb, 0x6f, 0xb7, 0x1e, + 0x91, 0x38, 0x64, 0x09, 0x02, 0x01, 0x01, +}; + +/* + * Edwards curves may be specified in two flavours: + * - as a PrintableString 'edwards25519' or 'edwards448' + * - as an OID, DER encoded ASN.1 Object + */ + +static const uint8_t ed25519_name_der[] = { + 0x13, 0x0c, 'e', 'd', 'w', 'a', 'r', 'd', 's', + '2', '5', '5', '1', '9', +}; + +static const uint8_t ed25519_oid_der[] = { + 0x06, 0x09, 0x2b, 0x06, 0x01, 0x04, 0x01, 0xda, + 0x47, 0x0f, 0x01, +}; + +struct supported_ecc_curve { + const uint8_t *oid_der; + size_t oid_size; + const uint8_t *name_der; + size_t name_size; + size_t key_size; + uint32_t tee_id; + const char *label; + size_t label_size; +}; + +#define ECC_CURVE(_tee_id, _key_size, _label) \ + { \ + .tee_id = (_tee_id), \ + .key_size = (_key_size), \ + .oid_der = _label ## _oid_der, \ + .oid_size = sizeof(_label ## _oid_der), \ + .name_der = _label ## _name_der, \ + .name_size = sizeof(_label ## _name_der), \ + .label = #_label, \ + .label_size = sizeof(#_label) - 1, \ + } + +static const struct supported_ecc_curve ec_curve_param[] = { + ECC_CURVE(TEE_ECC_CURVE_NIST_P192, 192, prime192v1), + ECC_CURVE(TEE_ECC_CURVE_NIST_P224, 224, secp224r1), + ECC_CURVE(TEE_ECC_CURVE_NIST_P256, 256, prime256v1), + ECC_CURVE(TEE_ECC_CURVE_NIST_P384, 384, secp384r1), + ECC_CURVE(TEE_ECC_CURVE_NIST_P521, 521, secp521r1), + ECC_CURVE(TEE_ECC_CURVE_25519, 256, ed25519), +}; + +static const struct supported_ecc_curve *get_curve(void *attr, size_t size) +{ + size_t idx = 0; + + /* Weak: not a real DER parser: try by params then by named curve */ + for (idx = 0; idx < ARRAY_SIZE(ec_curve_param); idx++) { + const struct supported_ecc_curve *curve = ec_curve_param + idx; + + if (size == curve->oid_size && + !TEE_MemCompare(attr, curve->oid_der, curve->oid_size)) + return curve; + + if (size == curve->name_size && + !TEE_MemCompare(attr, curve->name_der, curve->name_size)) + return curve; + } + + return NULL; +} + +size_t ec_params2tee_keysize(void *ec_params, size_t size) +{ + const struct supported_ecc_curve *curve = get_curve(ec_params, size); + + if (!curve) + return 0; + + return curve->key_size; +} + +/* + * This function intentionally panics if the curve is not found. + * Use ec_params2tee_keysize() to check the curve is supported by + * the internal core API. + */ +uint32_t ec_params2tee_curve(void *ec_params, size_t size) +{ + const struct supported_ecc_curve *curve = get_curve(ec_params, size); + + assert(curve); + + return curve->tee_id; +} + +enum pkcs11_rc load_tee_ec_key_attrs(TEE_Attribute **tee_attrs, + size_t *tee_count, + struct pkcs11_object *obj) +{ + TEE_Attribute *attrs = NULL; + size_t count = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + assert(get_key_type(obj->attributes) == PKCS11_CKK_EC); + + switch (get_class(obj->attributes)) { + case PKCS11_CKO_PUBLIC_KEY: + attrs = TEE_Malloc(3 * sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!attrs) + return PKCS11_CKR_DEVICE_MEMORY; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_ECC_CURVE, + obj, PKCS11_CKA_EC_PARAMS)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ECC_PUBLIC_VALUE_X, + obj, PKCS11_CKA_EC_POINT)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ECC_PUBLIC_VALUE_Y, + obj, PKCS11_CKA_EC_POINT)) + count++; + + if (count == 3) + rc = PKCS11_CKR_OK; + + break; + + case PKCS11_CKO_PRIVATE_KEY: + attrs = TEE_Malloc(4 * sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!attrs) + return PKCS11_CKR_DEVICE_MEMORY; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_ECC_CURVE, + obj, PKCS11_CKA_EC_PARAMS)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ECC_PRIVATE_VALUE, + obj, PKCS11_CKA_VALUE)) + count++; + + /* + * Standard does not have CKA_EC_POINT for EC private keys + * but that is required by TEE internal API. First try to get + * hidden EC POINT and for backwards compatibility then try to + * get CKA_EC_POINT value. + */ + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ECC_PUBLIC_VALUE_X, + obj, PKCS11_CKA_OPTEE_HIDDEN_EC_POINT)) + count++; + else if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ECC_PUBLIC_VALUE_X, + obj, PKCS11_CKA_EC_POINT)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ECC_PUBLIC_VALUE_Y, + obj, PKCS11_CKA_OPTEE_HIDDEN_EC_POINT)) + count++; + else if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ECC_PUBLIC_VALUE_Y, + obj, PKCS11_CKA_EC_POINT)) + count++; + + if (count == 4) + rc = PKCS11_CKR_OK; + + break; + + default: + assert(0); + break; + } + + if (rc == PKCS11_CKR_OK) { + *tee_attrs = attrs; + *tee_count = count; + } else { + TEE_Free(attrs); + } + + return rc; +} + +enum pkcs11_rc pkcs2tee_algo_ecdsa(uint32_t *tee_id, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj) +{ + switch (proc_params->id) { + case PKCS11_CKM_ECDSA: + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + break; + default: + return PKCS11_CKR_GENERAL_ERROR; + } + + /* + * TODO: Fixing this in a way to support also other EC curves would + * require OP-TEE to be updated for newer version of GlobalPlatform API + */ + switch (get_object_key_bit_size(obj)) { + case 192: + *tee_id = TEE_ALG_ECDSA_P192; + break; + case 224: + *tee_id = TEE_ALG_ECDSA_P224; + break; + case 256: + *tee_id = TEE_ALG_ECDSA_P256; + break; + case 384: + *tee_id = TEE_ALG_ECDSA_P384; + break; + case 521: + *tee_id = TEE_ALG_ECDSA_P521; + break; + default: + TEE_Panic(0); + break; + } + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc tee2pkcs_ec_attributes(struct obj_attrs **pub_head, + struct obj_attrs **priv_head, + TEE_ObjectHandle tee_obj, + size_t tee_size) +{ + void *x_ptr = NULL; + void *y_ptr = NULL; + uint8_t *ecpoint = NULL; + size_t x_size = 0; + size_t y_size = 0; + size_t psize = 0; + size_t qsize = 0; + size_t dersize = 0; + size_t poffset = 0; + size_t hsize = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_VALUE, + tee_obj, TEE_ATTR_ECC_PRIVATE_VALUE); + if (rc) + goto out; + + rc = alloc_get_tee_attribute_data(tee_obj, TEE_ATTR_ECC_PUBLIC_VALUE_X, + &x_ptr, &x_size); + if (rc) + goto out; + + rc = alloc_get_tee_attribute_data(tee_obj, TEE_ATTR_ECC_PUBLIC_VALUE_Y, + &y_ptr, &y_size); + if (rc) + goto x_cleanup; + + psize = (tee_size + 7) / 8; + if (x_size > psize || y_size > psize) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto y_cleanup; + } + + qsize = 1 + 2 * psize; + if (qsize < 0x80) { + /* DER short definitive form up to 127 bytes */ + dersize = qsize + 2; + hsize = 2 /* der */ + 1 /* point compression */; + } else if (qsize < 0x100) { + /* DER long definitive form up to 255 bytes */ + dersize = qsize + 3; + hsize = 3 /* der */ + 1 /* point compression */; + } else { + EMSG("Too long DER value"); + rc = PKCS11_CKR_MECHANISM_PARAM_INVALID; + goto y_cleanup; + } + + ecpoint = TEE_Malloc(dersize, TEE_MALLOC_FILL_ZERO); + if (!ecpoint) { + rc = PKCS11_CKR_DEVICE_MEMORY; + goto y_cleanup; + } + + if (qsize < 0x80) { + /* DER encoding */ + ecpoint[0] = 0x04; + ecpoint[1] = qsize & 0x7f; + + /* Only UNCOMPRESSED ECPOINT is currently supported */ + ecpoint[2] = 0x04; + } else if (qsize < 0x100) { + /* DER encoding */ + ecpoint[0] = 0x04; + ecpoint[1] = 0x80 | 0x01; /* long form, one size octet */ + ecpoint[2] = qsize & 0xFF; + + /* Only UNCOMPRESSED ECPOINT is currently supported */ + ecpoint[3] = 0x04; + } + + poffset = 0; + if (x_size < psize) + poffset = psize - x_size; + TEE_MemMove(ecpoint + hsize + poffset, x_ptr, x_size); + + poffset = 0; + if (y_size < psize) + poffset = psize - y_size; + TEE_MemMove(ecpoint + hsize + psize + poffset, y_ptr, y_size); + + /* + * Add PKCS11_CKA_OPTEE_HIDDEN_EC_POINT to private key object and + * standard PKCS11_CKA_EC_POINT to public key objects as + * TEE_PopulateTransientObject requires public x/y values + * for TEE_TYPE_ECDSA_KEYPAIR. + */ + rc = add_attribute(priv_head, PKCS11_CKA_OPTEE_HIDDEN_EC_POINT, + ecpoint, dersize); + if (rc) + goto ecpoint_cleanup; + + rc = add_attribute(pub_head, PKCS11_CKA_EC_POINT, ecpoint, dersize); + +ecpoint_cleanup: + TEE_Free(ecpoint); +y_cleanup: + TEE_Free(y_ptr); +x_cleanup: + TEE_Free(x_ptr); +out: + return rc; +} + +enum pkcs11_rc generate_ec_keys(struct pkcs11_attribute_head *proc_params, + struct obj_attrs **pub_head, + struct obj_attrs **priv_head) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *a_ptr = NULL; + uint32_t a_size = 0; + uint32_t tee_size = 0; + uint32_t tee_curve = 0; + TEE_ObjectHandle tee_obj = TEE_HANDLE_NULL; + TEE_Attribute tee_key_attr[1] = { }; + TEE_Result res = TEE_ERROR_GENERIC; + + if (!proc_params || !*pub_head || !*priv_head) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + if (remove_empty_attribute(pub_head, PKCS11_CKA_EC_POINT) || + remove_empty_attribute(priv_head, PKCS11_CKA_VALUE) || + remove_empty_attribute(priv_head, PKCS11_CKA_EC_PARAMS)) { + EMSG("Unexpected attribute(s) found"); + trace_attributes("public-key", *pub_head); + trace_attributes("private-key", *priv_head); + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + if (get_attribute_ptr(*pub_head, PKCS11_CKA_EC_PARAMS, + &a_ptr, &a_size) || !a_ptr) { + EMSG("No EC_PARAMS attribute found in public key"); + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + } + + tee_size = ec_params2tee_keysize(a_ptr, a_size); + if (!tee_size) + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + + tee_curve = ec_params2tee_curve(a_ptr, a_size); + + TEE_InitValueAttribute(tee_key_attr, TEE_ATTR_ECC_CURVE, tee_curve, 0); + + /* Create an ECDSA TEE key: will match PKCS11 ECDSA and ECDH */ + res = TEE_AllocateTransientObject(TEE_TYPE_ECDSA_KEYPAIR, tee_size, + &tee_obj); + if (res) { + EMSG("Transient alloc failed with %#"PRIx32, res); + return tee2pkcs_error(res); + } + + res = TEE_RestrictObjectUsage1(tee_obj, TEE_USAGE_EXTRACTABLE); + if (res) { + rc = tee2pkcs_error(res); + goto out; + } + + res = TEE_GenerateKey(tee_obj, tee_size, tee_key_attr, 1); + if (res) { + rc = tee2pkcs_error(res); + goto out; + } + + /* Private key needs the same EC_PARAMS as used by the public key */ + rc = add_attribute(priv_head, PKCS11_CKA_EC_PARAMS, a_ptr, a_size); + if (rc) + goto out; + + rc = tee2pkcs_ec_attributes(pub_head, priv_head, tee_obj, tee_size); + +out: + if (tee_obj != TEE_HANDLE_NULL) + TEE_CloseObject(tee_obj); + + return rc; +} + +enum pkcs11_rc load_tee_eddsa_key_attrs(TEE_Attribute **tee_attrs, + size_t *tee_count, + struct pkcs11_object *obj) +{ + TEE_Attribute *attrs = NULL; + size_t count = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + + assert(get_key_type(obj->attributes) == PKCS11_CKK_EC_EDWARDS); + + switch (get_class(obj->attributes)) { + case PKCS11_CKO_PUBLIC_KEY: + attrs = TEE_Malloc(sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!attrs) + return PKCS11_CKR_DEVICE_MEMORY; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ED25519_PUBLIC_VALUE, + obj, PKCS11_CKA_EC_POINT)) + count++; + + if (count == 1) + rc = PKCS11_CKR_OK; + + break; + + case PKCS11_CKO_PRIVATE_KEY: + attrs = TEE_Malloc(2 * sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!attrs) + return PKCS11_CKR_DEVICE_MEMORY; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ED25519_PRIVATE_VALUE, + obj, PKCS11_CKA_VALUE)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_ED25519_PUBLIC_VALUE, + obj, PKCS11_CKA_EC_POINT)) + count++; + + if (count == 2) + rc = PKCS11_CKR_OK; + + break; + + default: + assert(0); + break; + } + + if (rc == PKCS11_CKR_OK) { + *tee_attrs = attrs; + *tee_count = count; + } else { + TEE_Free(attrs); + } + + return rc; +} + +enum pkcs11_rc generate_eddsa_keys(struct pkcs11_attribute_head *proc_params, + struct obj_attrs **pub_head, + struct obj_attrs **priv_head) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *a_ptr = NULL; + uint32_t a_size = 0; + uint32_t tee_size = 0; + TEE_ObjectHandle tee_obj = TEE_HANDLE_NULL; + TEE_Result res = TEE_ERROR_GENERIC; + + if (!proc_params || !*pub_head || !*priv_head) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + if (remove_empty_attribute(pub_head, PKCS11_CKA_EC_POINT) || + remove_empty_attribute(priv_head, PKCS11_CKA_VALUE) || + remove_empty_attribute(priv_head, PKCS11_CKA_EC_POINT) || + remove_empty_attribute(priv_head, PKCS11_CKA_EC_PARAMS)) { + EMSG("Unexpected attribute(s) found"); + trace_attributes("public-key", *pub_head); + trace_attributes("private-key", *priv_head); + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + if (get_attribute_ptr(*pub_head, PKCS11_CKA_EC_PARAMS, + &a_ptr, &a_size) || !a_ptr) { + EMSG("No EC_PARAMS attribute found in public key"); + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + } + + tee_size = ec_params2tee_keysize(a_ptr, a_size); + if (!tee_size) + return PKCS11_CKR_ATTRIBUTE_TYPE_INVALID; + + res = TEE_AllocateTransientObject(TEE_TYPE_ED25519_KEYPAIR, tee_size, + &tee_obj); + if (res) { + EMSG("Transient alloc failed with %#"PRIx32, res); + return tee2pkcs_error(res); + } + + res = TEE_RestrictObjectUsage1(tee_obj, TEE_USAGE_EXTRACTABLE); + if (res) { + rc = tee2pkcs_error(res); + goto out; + } + + res = TEE_GenerateKey(tee_obj, tee_size, NULL, 0); + if (res) { + rc = tee2pkcs_error(res); + goto out; + } + + /* Private key needs the same EC_PARAMS as used by the public key */ + rc = add_attribute(priv_head, PKCS11_CKA_EC_PARAMS, a_ptr, a_size); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_VALUE, + tee_obj, TEE_ATTR_ED25519_PRIVATE_VALUE); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_EC_POINT, + tee_obj, TEE_ATTR_ED25519_PUBLIC_VALUE); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(pub_head, PKCS11_CKA_EC_POINT, + tee_obj, TEE_ATTR_ED25519_PUBLIC_VALUE); + +out: + if (tee_obj != TEE_HANDLE_NULL) + TEE_CloseObject(tee_obj); + + return rc; +} + +enum pkcs11_rc +pkcs2tee_proc_params_eddsa(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + uint32_t ctx_len = 0; + uint32_t flag = 0; + void *ctx_data = NULL; + struct serialargs args = { }; + struct eddsa_processing_ctx *ctx = NULL; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &flag); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &ctx_len); + if (rc) + return rc; + + rc = serialargs_get_ptr(&args, &ctx_data, ctx_len); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + proc->extra_ctx = TEE_Malloc(sizeof(*ctx) + ctx_len, + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!proc->extra_ctx) + return PKCS11_CKR_DEVICE_MEMORY; + + ctx = proc->extra_ctx; + ctx->ctx_len = ctx_len; + ctx->flag = flag; + TEE_MemMove(ctx->ctx, ctx_data, ctx_len); + + return PKCS11_CKR_OK; +} + +size_t ecdsa_get_input_max_byte_size(TEE_OperationHandle op) +{ + TEE_OperationInfo info = { }; + + TEE_GetOperationInfo(op, &info); + + switch (info.algorithm) { + case TEE_ALG_ECDSA_P192: + return 24; + case TEE_ALG_ECDSA_P224: + return 28; + case TEE_ALG_ECDSA_P256: + return 32; + case TEE_ALG_ECDSA_P384: + return 48; + case TEE_ALG_ECDSA_P521: + return 66; + default: + DMSG("Unexpected ECDSA algorithm %#"PRIx32, info.algorithm); + return 0; + } +} + +enum pkcs11_rc pkcs2tee_param_ecdh(struct pkcs11_attribute_head *proc_params, + void **pub_data, size_t *pub_size) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs args = { }; + uint32_t word = 0; + uint8_t byte = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + /* Skip KDF */ + rc = serialargs_get_u32(&args, &word); + if (rc) + return rc; + + /* Shared data size, shall be 0 */ + rc = serialargs_get_u32(&args, &word); + if (rc || word) + return rc; + + /* Public data size and content */ + rc = serialargs_get_u32(&args, &word); + if (rc || !word) + return rc; + + *pub_size = word; + + rc = serialargs_get(&args, &byte, sizeof(uint8_t)); + if (rc) + return rc; + + if (byte != 0x02 && byte != 0x03 && byte != 0x04) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (byte != 0x04) { + EMSG("DER compressed public key format not yet supported"); + return PKCS11_CKR_ARGUMENTS_BAD; + } + + *pub_size -= sizeof(uint8_t); + + if (*pub_size >= 0x80) { + EMSG("DER long definitive form not yet supported"); + return PKCS11_CKR_ARGUMENTS_BAD; + } + + rc = serialargs_get_ptr(&args, pub_data, *pub_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc pkcs2tee_algo_ecdh(uint32_t *tee_id, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct serialargs args = { }; + uint32_t kdf = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &kdf); + if (rc) + return rc; + + /* Remaining arguments are extracted by pkcs2tee_param_ecdh */ + if (kdf != PKCS11_CKD_NULL) { + DMSG("Only support CKD_NULL key derivation for ECDH"); + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + } + + switch (get_object_key_bit_size(obj)) { + case 192: + *tee_id = TEE_ALG_ECDH_P192; + break; + case 224: + *tee_id = TEE_ALG_ECDH_P224; + break; + case 256: + *tee_id = TEE_ALG_ECDH_P256; + break; + case 384: + *tee_id = TEE_ALG_ECDH_P384; + break; + case 521: + *tee_id = TEE_ALG_ECDH_P521; + break; + default: + TEE_Panic(0); + break; + } + + return PKCS11_CKR_OK; +} diff --git a/optee_os/ta/pkcs11/src/processing_rsa.c b/optee_os/ta/pkcs11/src/processing_rsa.c new file mode 100644 index 0000000..94eabbf --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing_rsa.c @@ -0,0 +1,777 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2018-2021, Linaro Limited + */ + +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "object.h" +#include "pkcs11_token.h" +#include "processing.h" + +enum pkcs11_rc +pkcs2tee_proc_params_rsa_pss(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct rsa_pss_processing_ctx *ctx = NULL; + uint32_t hash = 0; + uint32_t mgf = 0; + uint32_t salt_len = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &hash); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &mgf); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &salt_len); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + proc->extra_ctx = TEE_Malloc(sizeof(struct rsa_pss_processing_ctx), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!proc->extra_ctx) + return PKCS11_CKR_DEVICE_MEMORY; + + ctx = proc->extra_ctx; + + ctx->hash_alg = hash; + ctx->mgf_type = mgf; + ctx->salt_len = salt_len; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc pkcs2tee_validate_rsa_pss(struct active_processing *proc, + struct pkcs11_object *obj) +{ + struct rsa_pss_processing_ctx *rsa_pss_ctx = NULL; + size_t modulus_size = 0; + size_t hash_size = 0; + uint32_t k = 0; + + rsa_pss_ctx = proc->extra_ctx; + assert(rsa_pss_ctx); + + switch (rsa_pss_ctx->hash_alg) { + case PKCS11_CKM_SHA_1: + hash_size = TEE_ALG_GET_DIGEST_SIZE(TEE_ALG_SHA1); + break; + case PKCS11_CKM_SHA224: + hash_size = TEE_ALG_GET_DIGEST_SIZE(TEE_ALG_SHA224); + break; + case PKCS11_CKM_SHA256: + hash_size = TEE_ALG_GET_DIGEST_SIZE(TEE_ALG_SHA256); + break; + case PKCS11_CKM_SHA384: + hash_size = TEE_ALG_GET_DIGEST_SIZE(TEE_ALG_SHA384); + break; + case PKCS11_CKM_SHA512: + hash_size = TEE_ALG_GET_DIGEST_SIZE(TEE_ALG_SHA512); + break; + default: + assert(0); + break; + } + + modulus_size = get_object_key_bit_size(obj); + + /** + * The sLen field must be less than or equal to k*-2-hLen where + * hLen is the length in bytes of the hash value. k* is the + * length in bytes of the RSA modulus, except if the length in + * bits of the RSA modulus is one more than a multiple of 8, in + * which case k* is one less than the length in bytes of the + * RSA modulus. + */ + if ((modulus_size % 8) == 1) + k = modulus_size / 8; + else + k = ROUNDUP(modulus_size, 8) / 8; + + if (rsa_pss_ctx->salt_len > (k - 2 - hash_size)) + return PKCS11_CKR_KEY_SIZE_RANGE; + + return PKCS11_CKR_OK; +} + +/* + * Check or set TEE algorithm identifier upon PKCS11 mechanism parameters + * @tee_id: Input and/or output TEE algorithm identifier + * @proc_params: PKCS11 processing parameters + */ +enum pkcs11_rc pkcs2tee_algo_rsa_pss(uint32_t *tee_id, + struct pkcs11_attribute_head *proc_params) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + uint32_t hash = 0; + uint32_t mgf = 0; + uint32_t salt_len = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &hash); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &mgf); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &salt_len); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (proc_params->id == PKCS11_CKM_RSA_PKCS_PSS) { + if (hash == PKCS11_CKM_SHA_1 && mgf == PKCS11_CKG_MGF1_SHA1) { + *tee_id = TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1; + return PKCS11_CKR_OK; + } + if (hash == PKCS11_CKM_SHA224 && + mgf == PKCS11_CKG_MGF1_SHA224) { + *tee_id = TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224; + return PKCS11_CKR_OK; + } + if (hash == PKCS11_CKM_SHA256 && + mgf == PKCS11_CKG_MGF1_SHA256) { + *tee_id = TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256; + return PKCS11_CKR_OK; + } + if (hash == PKCS11_CKM_SHA384 && + mgf == PKCS11_CKG_MGF1_SHA384) { + *tee_id = TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384; + return PKCS11_CKR_OK; + } + if (hash == PKCS11_CKM_SHA512 && + mgf == PKCS11_CKG_MGF1_SHA512) { + *tee_id = TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512; + return PKCS11_CKR_OK; + } + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + } + + switch (*tee_id) { + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA1: + if (hash != PKCS11_CKM_SHA_1 || mgf != PKCS11_CKG_MGF1_SHA1) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA224: + if (hash != PKCS11_CKM_SHA224 || mgf != PKCS11_CKG_MGF1_SHA224) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA256: + if (hash != PKCS11_CKM_SHA256 || mgf != PKCS11_CKG_MGF1_SHA256) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA384: + if (hash != PKCS11_CKM_SHA384 || mgf != PKCS11_CKG_MGF1_SHA384) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + break; + case TEE_ALG_RSASSA_PKCS1_PSS_MGF1_SHA512: + if (hash != PKCS11_CKM_SHA512 || mgf != PKCS11_CKG_MGF1_SHA512) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + break; + default: + return PKCS11_CKR_GENERAL_ERROR; + } + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc +pkcs2tee_proc_params_rsa_oaep(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct rsa_oaep_processing_ctx *ctx = NULL; + uint32_t hash = 0; + uint32_t mgf = 0; + uint32_t source_type = 0; + void *source_data = NULL; + uint32_t source_size = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &hash); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &mgf); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_type); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_size); + if (rc) + return rc; + + rc = serialargs_get_ptr(&args, &source_data, source_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + proc->extra_ctx = TEE_Malloc(sizeof(struct rsa_oaep_processing_ctx) + + source_size, + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!proc->extra_ctx) + return PKCS11_CKR_DEVICE_MEMORY; + + ctx = proc->extra_ctx; + + ctx->hash_alg = hash; + ctx->mgf_type = mgf; + ctx->source_type = source_type; + ctx->source_data_len = source_size; + TEE_MemMove(ctx->source_data, source_data, source_size); + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc +pkcs2tee_proc_params_rsa_aes_wrap(struct active_processing *proc, + struct pkcs11_attribute_head *proc_params) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + struct rsa_aes_key_wrap_processing_ctx *ctx = NULL; + uint32_t aes_key_bits = 0; + uint32_t hash = 0; + uint32_t mgf = 0; + uint32_t source_type = 0; + void *source_data = NULL; + uint32_t source_size = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &aes_key_bits); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &hash); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &mgf); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_type); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_size); + if (rc) + return rc; + + rc = serialargs_get_ptr(&args, &source_data, source_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + proc->extra_ctx = + TEE_Malloc(sizeof(struct rsa_aes_key_wrap_processing_ctx) + + source_size, + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!proc->extra_ctx) + return PKCS11_CKR_DEVICE_MEMORY; + + ctx = proc->extra_ctx; + + ctx->aes_key_bits = aes_key_bits; + ctx->hash_alg = hash; + ctx->mgf_type = mgf; + ctx->source_type = source_type; + ctx->source_data_len = source_size; + TEE_MemMove(ctx->source_data, source_data, source_size); + + return PKCS11_CKR_OK; +} + +/* + * Set TEE RSA OAEP algorithm identifier upon PKCS11 mechanism parameters + * @tee_id: output TEE RSA OAEP algorithm identifier + * @tee_hash_id: output TEE hash algorithm identifier + * @proc_params: PKCS11 processing parameters + */ +enum pkcs11_rc +pkcs2tee_algo_rsa_oaep(uint32_t *tee_id, uint32_t *tee_hash_id, + struct pkcs11_attribute_head *proc_params) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + uint32_t hash = 0; + uint32_t mgf = 0; + uint32_t source_type = 0; + void *source_data = NULL; + uint32_t source_size = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &hash); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &mgf); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_type); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_size); + if (rc) + return rc; + + rc = serialargs_get_ptr(&args, &source_data, source_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (source_type != PKCS11_CKZ_DATA_SPECIFIED) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + + switch (proc_params->id) { + case PKCS11_CKM_RSA_PKCS_OAEP: + switch (hash) { + case PKCS11_CKM_SHA_1: + if (mgf != PKCS11_CKG_MGF1_SHA1) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1; + *tee_hash_id = TEE_ALG_SHA1; + break; + case PKCS11_CKM_SHA224: + if (mgf != PKCS11_CKG_MGF1_SHA224) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224; + *tee_hash_id = TEE_ALG_SHA224; + break; + case PKCS11_CKM_SHA256: + if (mgf != PKCS11_CKG_MGF1_SHA256) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256; + *tee_hash_id = TEE_ALG_SHA256; + break; + case PKCS11_CKM_SHA384: + if (mgf != PKCS11_CKG_MGF1_SHA384) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384; + *tee_hash_id = TEE_ALG_SHA384; + break; + case PKCS11_CKM_SHA512: + if (mgf != PKCS11_CKG_MGF1_SHA512) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512; + *tee_hash_id = TEE_ALG_SHA512; + break; + default: + EMSG("Unexpected %#"PRIx32"/%s", hash, + id2str_proc(hash)); + + return PKCS11_CKR_GENERAL_ERROR; + } + break; + default: + EMSG("Unexpected mechanism %#"PRIx32"/%s", proc_params->id, + id2str_proc(proc_params->id)); + + return PKCS11_CKR_GENERAL_ERROR; + } + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc +pkcs2tee_algo_rsa_aes_wrap(uint32_t *tee_id, uint32_t *tee_hash_id, + struct pkcs11_attribute_head *proc_params) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + uint32_t aes_key_bits = 0; + uint32_t hash = 0; + uint32_t mgf = 0; + uint32_t source_type = 0; + void *source_data = NULL; + uint32_t source_size = 0; + + serialargs_init(&args, proc_params->data, proc_params->size); + + rc = serialargs_get_u32(&args, &aes_key_bits); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &hash); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &mgf); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_type); + if (rc) + return rc; + + rc = serialargs_get_u32(&args, &source_size); + if (rc) + return rc; + + rc = serialargs_get_ptr(&args, &source_data, source_size); + if (rc) + return rc; + + if (serialargs_remaining_bytes(&args)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (source_type != PKCS11_CKZ_DATA_SPECIFIED) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + + switch (proc_params->id) { + case PKCS11_CKM_RSA_AES_KEY_WRAP: + switch (hash) { + case PKCS11_CKM_SHA_1: + if (mgf != PKCS11_CKG_MGF1_SHA1) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA1; + *tee_hash_id = TEE_ALG_SHA1; + break; + case PKCS11_CKM_SHA224: + if (mgf != PKCS11_CKG_MGF1_SHA224) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA224; + *tee_hash_id = TEE_ALG_SHA224; + break; + case PKCS11_CKM_SHA256: + if (mgf != PKCS11_CKG_MGF1_SHA256) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA256; + *tee_hash_id = TEE_ALG_SHA256; + break; + case PKCS11_CKM_SHA384: + if (mgf != PKCS11_CKG_MGF1_SHA384) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA384; + *tee_hash_id = TEE_ALG_SHA384; + break; + case PKCS11_CKM_SHA512: + if (mgf != PKCS11_CKG_MGF1_SHA512) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + *tee_id = TEE_ALG_RSAES_PKCS1_OAEP_MGF1_SHA512; + *tee_hash_id = TEE_ALG_SHA512; + break; + default: + EMSG("Unexpected %#"PRIx32"/%s", hash, + id2str_proc(hash)); + + return PKCS11_CKR_GENERAL_ERROR; + } + break; + default: + EMSG("Unexpected mechanism %#"PRIx32"/%s", proc_params->id, + id2str_proc(proc_params->id)); + + return PKCS11_CKR_GENERAL_ERROR; + } + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc load_tee_rsa_key_attrs(TEE_Attribute **tee_attrs, + size_t *tee_count, + struct pkcs11_object *obj) +{ + TEE_Attribute *attrs = NULL; + size_t count = 0; + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *a_ptr = NULL; + + assert(get_key_type(obj->attributes) == PKCS11_CKK_RSA); + + switch (get_class(obj->attributes)) { + case PKCS11_CKO_PUBLIC_KEY: + attrs = TEE_Malloc(2 * sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!attrs) + return PKCS11_CKR_DEVICE_MEMORY; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_RSA_MODULUS, + obj, PKCS11_CKA_MODULUS)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_RSA_PUBLIC_EXPONENT, obj, + PKCS11_CKA_PUBLIC_EXPONENT)) + count++; + + if (count == 2) + rc = PKCS11_CKR_OK; + + break; + + case PKCS11_CKO_PRIVATE_KEY: + attrs = TEE_Malloc(8 * sizeof(TEE_Attribute), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + if (!attrs) + return PKCS11_CKR_DEVICE_MEMORY; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_RSA_MODULUS, + obj, PKCS11_CKA_MODULUS)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_RSA_PUBLIC_EXPONENT, obj, + PKCS11_CKA_PUBLIC_EXPONENT)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], + TEE_ATTR_RSA_PRIVATE_EXPONENT, obj, + PKCS11_CKA_PRIVATE_EXPONENT)) + count++; + + if (count != 3) + break; + + /* If pre-computed values are present load those */ + rc = get_attribute_ptr(obj->attributes, PKCS11_CKA_PRIME_1, + &a_ptr, NULL); + if (rc != PKCS11_CKR_OK && rc != PKCS11_RV_NOT_FOUND) + break; + if (rc == PKCS11_RV_NOT_FOUND || !a_ptr) { + rc = PKCS11_CKR_OK; + break; + } + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_RSA_PRIME1, obj, + PKCS11_CKA_PRIME_1)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_RSA_PRIME2, obj, + PKCS11_CKA_PRIME_2)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_RSA_EXPONENT1, + obj, PKCS11_CKA_EXPONENT_1)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_RSA_EXPONENT2, + obj, PKCS11_CKA_EXPONENT_2)) + count++; + + if (pkcs2tee_load_attr(&attrs[count], TEE_ATTR_RSA_COEFFICIENT, + obj, PKCS11_CKA_COEFFICIENT)) + count++; + + if (count == 8) + rc = PKCS11_CKR_OK; + + break; + + default: + assert(0); + break; + } + + if (rc == PKCS11_CKR_OK) { + *tee_attrs = attrs; + *tee_count = count; + } else { + TEE_Free(attrs); + } + + return rc; +} + +static enum pkcs11_rc tee2pkcs_rsa_attributes(struct obj_attrs **pub_head, + struct obj_attrs **priv_head, + TEE_ObjectHandle tee_obj) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *a_ptr = NULL; + + rc = tee2pkcs_add_attribute(pub_head, PKCS11_CKA_MODULUS, tee_obj, + TEE_ATTR_RSA_MODULUS); + if (rc) + goto out; + + rc = get_attribute_ptr(*pub_head, PKCS11_CKA_PUBLIC_EXPONENT, &a_ptr, + NULL); + if (rc != PKCS11_CKR_OK && rc != PKCS11_RV_NOT_FOUND) + goto out; + + if (rc == PKCS11_CKR_OK && !a_ptr) { + rc = remove_empty_attribute(pub_head, + PKCS11_CKA_PUBLIC_EXPONENT); + if (rc) + goto out; + rc = PKCS11_RV_NOT_FOUND; + } + + if (rc == PKCS11_RV_NOT_FOUND) { + rc = tee2pkcs_add_attribute(pub_head, + PKCS11_CKA_PUBLIC_EXPONENT, + tee_obj, + TEE_ATTR_RSA_PUBLIC_EXPONENT); + if (rc) + goto out; + } + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_MODULUS, tee_obj, + TEE_ATTR_RSA_MODULUS); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_PUBLIC_EXPONENT, + tee_obj, TEE_ATTR_RSA_PUBLIC_EXPONENT); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_PRIVATE_EXPONENT, + tee_obj, TEE_ATTR_RSA_PRIVATE_EXPONENT); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_PRIME_1, tee_obj, + TEE_ATTR_RSA_PRIME1); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_PRIME_2, tee_obj, + TEE_ATTR_RSA_PRIME2); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_EXPONENT_1, tee_obj, + TEE_ATTR_RSA_EXPONENT1); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_EXPONENT_2, tee_obj, + TEE_ATTR_RSA_EXPONENT2); + if (rc) + goto out; + + rc = tee2pkcs_add_attribute(priv_head, PKCS11_CKA_COEFFICIENT, tee_obj, + TEE_ATTR_RSA_COEFFICIENT); +out: + return rc; +} + +enum pkcs11_rc generate_rsa_keys(struct pkcs11_attribute_head *proc_params, + struct obj_attrs **pub_head, + struct obj_attrs **priv_head) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + void *a_ptr = NULL; + uint32_t a_size = 0; + TEE_ObjectHandle tee_obj = TEE_HANDLE_NULL; + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t modulus_bits = 0; + TEE_Attribute tee_attrs[1] = { }; + uint32_t tee_count = 0; + + if (!proc_params || !*pub_head || !*priv_head) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + rc = get_attribute_ptr(*pub_head, PKCS11_CKA_MODULUS_BITS, &a_ptr, + &a_size); + if (rc != PKCS11_CKR_OK || a_size != sizeof(uint32_t)) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + TEE_MemMove(&modulus_bits, a_ptr, sizeof(uint32_t)); + + rc = get_attribute_ptr(*pub_head, PKCS11_CKA_PUBLIC_EXPONENT, &a_ptr, + &a_size); + if (rc != PKCS11_CKR_OK && rc != PKCS11_RV_NOT_FOUND) + return rc; + + if (rc == PKCS11_CKR_OK && a_ptr) { + TEE_InitRefAttribute(&tee_attrs[tee_count], + TEE_ATTR_RSA_PUBLIC_EXPONENT, + a_ptr, a_size); + tee_count++; + } + + if (remove_empty_attribute(priv_head, PKCS11_CKA_MODULUS) || + remove_empty_attribute(priv_head, PKCS11_CKA_PUBLIC_EXPONENT) || + remove_empty_attribute(priv_head, PKCS11_CKA_PRIVATE_EXPONENT) || + remove_empty_attribute(priv_head, PKCS11_CKA_PRIME_1) || + remove_empty_attribute(priv_head, PKCS11_CKA_PRIME_2) || + remove_empty_attribute(priv_head, PKCS11_CKA_EXPONENT_1) || + remove_empty_attribute(priv_head, PKCS11_CKA_EXPONENT_2) || + remove_empty_attribute(priv_head, PKCS11_CKA_COEFFICIENT)) { + EMSG("Unexpected attribute(s) found"); + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto out; + } + + /* Create an RSA TEE key */ + res = TEE_AllocateTransientObject(TEE_TYPE_RSA_KEYPAIR, modulus_bits, + &tee_obj); + if (res) { + DMSG("TEE_AllocateTransientObject failed %#"PRIx32, res); + + rc = tee2pkcs_error(res); + goto out; + } + + res = TEE_RestrictObjectUsage1(tee_obj, TEE_USAGE_EXTRACTABLE); + if (res) { + DMSG("TEE_RestrictObjectUsage1 failed %#"PRIx32, res); + + rc = tee2pkcs_error(res); + goto out; + } + + res = TEE_GenerateKey(tee_obj, modulus_bits, tee_attrs, tee_count); + if (res) { + DMSG("TEE_GenerateKey failed %#"PRIx32, res); + + rc = tee2pkcs_error(res); + goto out; + } + + rc = tee2pkcs_rsa_attributes(pub_head, priv_head, tee_obj); + +out: + if (tee_obj != TEE_HANDLE_NULL) + TEE_CloseObject(tee_obj); + + return rc; +} + +size_t rsa_get_input_max_byte_size(TEE_OperationHandle op) +{ + TEE_OperationInfo info = { }; + + TEE_GetOperationInfo(op, &info); + + return info.maxKeySize / 8; +} diff --git a/optee_os/ta/pkcs11/src/processing_symm.c b/optee_os/ta/pkcs11/src/processing_symm.c new file mode 100644 index 0000000..7363150 --- /dev/null +++ b/optee_os/ta/pkcs11/src/processing_symm.c @@ -0,0 +1,1145 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "object.h" +#include "pkcs11_attributes.h" +#include "pkcs11_helpers.h" +#include "pkcs11_token.h" +#include "processing.h" +#include "serializer.h" + +struct input_data_ref { + size_t size; + void *data; +}; + +bool processing_is_tee_symm(enum pkcs11_mechanism_id proc_id) +{ + switch (proc_id) { + /* Authentication */ + case PKCS11_CKM_AES_CMAC: + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA512_HMAC: + case PKCS11_CKM_AES_CMAC_GENERAL: + case PKCS11_CKM_MD5_HMAC_GENERAL: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + /* Ciphering */ + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + case PKCS11_CKM_AES_CTS: + case PKCS11_CKM_AES_CTR: + case PKCS11_CKM_AES_ECB_ENCRYPT_DATA: + case PKCS11_CKM_AES_CBC_ENCRYPT_DATA: + return true; + default: + return false; + } +} + +static enum pkcs11_rc +pkcs2tee_algorithm(uint32_t *tee_id, struct pkcs11_attribute_head *proc_params) +{ + static const struct { + enum pkcs11_mechanism_id mech_id; + uint32_t tee_id; + } pkcs2tee_algo[] = { + /* AES flavors */ + { PKCS11_CKM_AES_ECB, TEE_ALG_AES_ECB_NOPAD }, + { PKCS11_CKM_AES_CBC, TEE_ALG_AES_CBC_NOPAD }, + { PKCS11_CKM_AES_ECB_ENCRYPT_DATA, TEE_ALG_AES_ECB_NOPAD }, + { PKCS11_CKM_AES_CBC_ENCRYPT_DATA, TEE_ALG_AES_CBC_NOPAD }, + { PKCS11_CKM_AES_CTR, TEE_ALG_AES_CTR }, + { PKCS11_CKM_AES_CTS, TEE_ALG_AES_CTS }, + { PKCS11_CKM_AES_CMAC, TEE_ALG_AES_CMAC }, + { PKCS11_CKM_AES_CMAC_GENERAL, TEE_ALG_AES_CMAC }, + /* HMAC flavors */ + { PKCS11_CKM_MD5_HMAC, TEE_ALG_HMAC_MD5 }, + { PKCS11_CKM_SHA_1_HMAC, TEE_ALG_HMAC_SHA1 }, + { PKCS11_CKM_SHA224_HMAC, TEE_ALG_HMAC_SHA224 }, + { PKCS11_CKM_SHA256_HMAC, TEE_ALG_HMAC_SHA256 }, + { PKCS11_CKM_SHA384_HMAC, TEE_ALG_HMAC_SHA384 }, + { PKCS11_CKM_SHA512_HMAC, TEE_ALG_HMAC_SHA512 }, + { PKCS11_CKM_MD5_HMAC_GENERAL, TEE_ALG_HMAC_MD5 }, + { PKCS11_CKM_SHA_1_HMAC_GENERAL, TEE_ALG_HMAC_SHA1 }, + { PKCS11_CKM_SHA224_HMAC_GENERAL, TEE_ALG_HMAC_SHA224 }, + { PKCS11_CKM_SHA256_HMAC_GENERAL, TEE_ALG_HMAC_SHA256 }, + { PKCS11_CKM_SHA384_HMAC_GENERAL, TEE_ALG_HMAC_SHA384 }, + { PKCS11_CKM_SHA512_HMAC_GENERAL, TEE_ALG_HMAC_SHA512 }, + }; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(pkcs2tee_algo); n++) { + if (proc_params->id == pkcs2tee_algo[n].mech_id) { + *tee_id = pkcs2tee_algo[n].tee_id; + return PKCS11_CKR_OK; + } + } + + return PKCS11_RV_NOT_IMPLEMENTED; +} + +static enum pkcs11_rc pkcs2tee_key_type(uint32_t *tee_type, + struct pkcs11_object *obj) +{ + static const struct { + enum pkcs11_key_type key_type; + uint32_t tee_id; + } pkcs2tee_key_type[] = { + { PKCS11_CKK_AES, TEE_TYPE_AES }, + { PKCS11_CKK_GENERIC_SECRET, TEE_TYPE_GENERIC_SECRET }, + { PKCS11_CKK_MD5_HMAC, TEE_TYPE_HMAC_MD5 }, + { PKCS11_CKK_SHA_1_HMAC, TEE_TYPE_HMAC_SHA1 }, + { PKCS11_CKK_SHA224_HMAC, TEE_TYPE_HMAC_SHA224 }, + { PKCS11_CKK_SHA256_HMAC, TEE_TYPE_HMAC_SHA256 }, + { PKCS11_CKK_SHA384_HMAC, TEE_TYPE_HMAC_SHA384 }, + { PKCS11_CKK_SHA512_HMAC, TEE_TYPE_HMAC_SHA512 }, + }; + size_t n = 0; + enum pkcs11_key_type key_type = get_key_type(obj->attributes); + + assert(get_class(obj->attributes) == PKCS11_CKO_SECRET_KEY); + + for (n = 0; n < ARRAY_SIZE(pkcs2tee_key_type); n++) { + if (pkcs2tee_key_type[n].key_type == key_type) { + *tee_type = pkcs2tee_key_type[n].tee_id; + return PKCS11_CKR_OK; + } + } + + return PKCS11_RV_NOT_FOUND; +} + +static enum pkcs11_rc pkcsmech2tee_key_type(uint32_t *tee_type, + enum pkcs11_mechanism_id mech_id) +{ + static const struct { + enum pkcs11_mechanism_id mech; + uint32_t tee_id; + } pkcs2tee_key_type[] = { + { PKCS11_CKM_MD5_HMAC, TEE_TYPE_HMAC_MD5 }, + { PKCS11_CKM_SHA_1_HMAC, TEE_TYPE_HMAC_SHA1 }, + { PKCS11_CKM_SHA224_HMAC, TEE_TYPE_HMAC_SHA224 }, + { PKCS11_CKM_SHA256_HMAC, TEE_TYPE_HMAC_SHA256 }, + { PKCS11_CKM_SHA384_HMAC, TEE_TYPE_HMAC_SHA384 }, + { PKCS11_CKM_SHA512_HMAC, TEE_TYPE_HMAC_SHA512 }, + { PKCS11_CKM_MD5_HMAC_GENERAL, TEE_TYPE_HMAC_MD5 }, + { PKCS11_CKM_SHA_1_HMAC_GENERAL, TEE_TYPE_HMAC_SHA1 }, + { PKCS11_CKM_SHA224_HMAC_GENERAL, TEE_TYPE_HMAC_SHA224 }, + { PKCS11_CKM_SHA256_HMAC_GENERAL, TEE_TYPE_HMAC_SHA256 }, + { PKCS11_CKM_SHA384_HMAC_GENERAL, TEE_TYPE_HMAC_SHA384 }, + { PKCS11_CKM_SHA512_HMAC_GENERAL, TEE_TYPE_HMAC_SHA512 }, + }; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(pkcs2tee_key_type); n++) { + if (pkcs2tee_key_type[n].mech == mech_id) { + *tee_type = pkcs2tee_key_type[n].tee_id; + return PKCS11_CKR_OK; + } + } + + return PKCS11_RV_NOT_FOUND; +} + +static enum pkcs11_rc hmac_to_tee_hash(uint32_t *algo, + enum pkcs11_mechanism_id mech_id) +{ + static const struct { + enum pkcs11_mechanism_id mech; + uint32_t tee_id; + } hmac_hash[] = { + { PKCS11_CKM_MD5_HMAC, TEE_ALG_MD5 }, + { PKCS11_CKM_SHA_1_HMAC, TEE_ALG_SHA1 }, + { PKCS11_CKM_SHA224_HMAC, TEE_ALG_SHA224 }, + { PKCS11_CKM_SHA256_HMAC, TEE_ALG_SHA256 }, + { PKCS11_CKM_SHA384_HMAC, TEE_ALG_SHA384 }, + { PKCS11_CKM_SHA512_HMAC, TEE_ALG_SHA512 }, + { PKCS11_CKM_MD5_HMAC_GENERAL, TEE_ALG_MD5 }, + { PKCS11_CKM_SHA_1_HMAC_GENERAL, TEE_ALG_SHA1 }, + { PKCS11_CKM_SHA224_HMAC_GENERAL, TEE_ALG_SHA224 }, + { PKCS11_CKM_SHA256_HMAC_GENERAL, TEE_ALG_SHA256 }, + { PKCS11_CKM_SHA384_HMAC_GENERAL, TEE_ALG_SHA384 }, + { PKCS11_CKM_SHA512_HMAC_GENERAL, TEE_ALG_SHA512 }, + }; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(hmac_hash); n++) { + if (hmac_hash[n].mech == mech_id) { + *algo = hmac_hash[n].tee_id; + return PKCS11_CKR_OK; + } + } + + return PKCS11_RV_NOT_FOUND; +} + +static enum pkcs11_rc +allocate_tee_operation(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_attribute_head *params, + struct pkcs11_object *obj) +{ + uint32_t size = (uint32_t)get_object_key_bit_size(obj); + uint32_t key_size = size / 8; + uint32_t algo = 0; + uint32_t mode = 0; + uint32_t max_key_size = 0; + uint32_t min_key_size = 0; + TEE_Result res = TEE_ERROR_GENERIC; + + assert(session->processing->tee_op_handle == TEE_HANDLE_NULL); + + if (pkcs2tee_algorithm(&algo, params)) + return PKCS11_CKR_FUNCTION_FAILED; + + /* Sign/Verify with AES or generic key relate to TEE MAC operation */ + switch (params->id) { + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA512_HMAC: + case PKCS11_CKM_MD5_HMAC_GENERAL: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + mechanism_supported_key_sizes_bytes(params->id, &min_key_size, + &max_key_size); + if (key_size < min_key_size) + return PKCS11_CKR_KEY_SIZE_RANGE; + + /* + * If size of generic key is greater than the size + * supported by TEE API, this is not considered an + * error. When loading TEE key, we will hash the key + * to generate the appropriate key for HMAC operation. + * This key size will not be greater than the + * max_key_size. So we can use max_key_size for + * TEE_AllocateOperation(). + */ + if (key_size > max_key_size) + size = max_key_size * 8; + + mode = TEE_MODE_MAC; + break; + case PKCS11_CKM_AES_CMAC: + case PKCS11_CKM_AES_CMAC_GENERAL: + mode = TEE_MODE_MAC; + break; + default: + pkcs2tee_mode(&mode, function); + break; + } + + res = TEE_AllocateOperation(&session->processing->tee_op_handle, + algo, mode, size); + if (res) + EMSG("TEE_AllocateOp. failed %#"PRIx32" %#"PRIx32" %#"PRIx32, + algo, mode, size); + + if (res == TEE_ERROR_NOT_SUPPORTED) + return PKCS11_CKR_MECHANISM_INVALID; + + return tee2pkcs_error(res); +} + +static enum pkcs11_rc hash_secret_helper(enum pkcs11_mechanism_id mech_id, + struct pkcs11_object *obj, + TEE_Attribute *tee_attr, + void **ctx, + size_t *object_size_bits) +{ + uint32_t algo = 0; + void *hash_ptr = NULL; + uint32_t hash_size = 0; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + rc = hmac_to_tee_hash(&algo, mech_id); + if (rc) + return rc; + + hash_size = TEE_ALG_GET_DIGEST_SIZE(algo); + hash_ptr = TEE_Malloc(hash_size, 0); + if (!hash_ptr) + return PKCS11_CKR_DEVICE_MEMORY; + + rc = pkcs2tee_load_hashed_attr(tee_attr, TEE_ATTR_SECRET_VALUE, obj, + PKCS11_CKA_VALUE, algo, hash_ptr, + &hash_size); + if (rc) { + EMSG("No secret/hash error"); + TEE_Free(hash_ptr); + return rc; + } + + *ctx = hash_ptr; + + *object_size_bits = hash_size * 8; + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc load_tee_key(struct pkcs11_session *session, + struct pkcs11_object *obj, + struct pkcs11_attribute_head *proc_params) +{ + TEE_Attribute tee_attr = { }; + size_t object_size = 0; + uint32_t tee_key_type = 0; + enum pkcs11_key_type key_type = 0; + enum pkcs11_rc rc = PKCS11_CKR_OK; + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t max_key_size = 0; + uint32_t min_key_size = 0; + + if (obj->key_handle != TEE_HANDLE_NULL) { + /* Key was already loaded and fits current need */ + goto key_ready; + } + + object_size = get_object_key_bit_size(obj); + if (!object_size) + return PKCS11_CKR_GENERAL_ERROR; + + switch (proc_params->id) { + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA512_HMAC: + case PKCS11_CKM_MD5_HMAC_GENERAL: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + key_type = get_key_type(obj->attributes); + /* + * If Object Key type is PKCS11_CKK_GENERIC_SECRET, + * determine the tee_key_type using the + * mechanism instead of object key_type. + */ + if (key_type == PKCS11_CKK_GENERIC_SECRET) + rc = pkcsmech2tee_key_type(&tee_key_type, + proc_params->id); + else + rc = pkcs2tee_key_type(&tee_key_type, obj); + + if (rc) + return rc; + + mechanism_supported_key_sizes_bytes(proc_params->id, + &min_key_size, + &max_key_size); + + if ((object_size / 8) > max_key_size) { + rc = hash_secret_helper(proc_params->id, obj, &tee_attr, + &session->processing->extra_ctx, + &object_size); + if (rc) + return rc; + } else { + if (!pkcs2tee_load_attr(&tee_attr, + TEE_ATTR_SECRET_VALUE, + obj, + PKCS11_CKA_VALUE)) { + EMSG("No secret found"); + return PKCS11_CKR_FUNCTION_FAILED; + } + } + break; + + default: + rc = pkcs2tee_key_type(&tee_key_type, obj); + if (rc) + return rc; + + if (!pkcs2tee_load_attr(&tee_attr, TEE_ATTR_SECRET_VALUE, + obj, PKCS11_CKA_VALUE)) { + EMSG("No secret found"); + return PKCS11_CKR_FUNCTION_FAILED; + } + break; + } + + res = TEE_AllocateTransientObject(tee_key_type, object_size, + &obj->key_handle); + if (res) { + DMSG("TEE_AllocateTransientObject failed, %#"PRIx32, res); + return tee2pkcs_error(res); + } + + res = TEE_PopulateTransientObject(obj->key_handle, &tee_attr, 1); + if (res) { + DMSG("TEE_PopulateTransientObject failed, %#"PRIx32, res); + goto error; + } + +key_ready: + res = TEE_SetOperationKey(session->processing->tee_op_handle, + obj->key_handle); + if (res) { + DMSG("TEE_SetOperationKey failed, %#"PRIx32, res); + goto error; + } + + return PKCS11_CKR_OK; + +error: + TEE_FreeTransientObject(obj->key_handle); + obj->key_handle = TEE_HANDLE_NULL; + + return tee2pkcs_error(res); +} + +static enum pkcs11_rc +tee_init_derive_symm(struct active_processing *processing, + struct pkcs11_attribute_head *proc_params) +{ + struct serialargs args = { }; + enum pkcs11_rc rc = PKCS11_CKR_OK; + struct input_data_ref *param = NULL; + void *iv = NULL; + + if (!proc_params) + return PKCS11_CKR_ARGUMENTS_BAD; + + param = TEE_Malloc(sizeof(struct input_data_ref), TEE_MALLOC_FILL_ZERO); + if (!param) + return PKCS11_CKR_DEVICE_MEMORY; + + serialargs_init(&args, proc_params->data, proc_params->size); + + switch (proc_params->id) { + case PKCS11_CKM_AES_CBC_ENCRYPT_DATA: + rc = serialargs_get_ptr(&args, &iv, 16); + if (rc) + goto err; + break; + default: + break; + } + + rc = serialargs_get(&args, ¶m->size, sizeof(uint32_t)); + if (rc) + goto err; + + rc = serialargs_get_ptr(&args, ¶m->data, param->size); + if (rc) + goto err; + + if (serialargs_remaining_bytes(&args)) { + rc = PKCS11_CKR_ARGUMENTS_BAD; + goto err; + } + + processing->extra_ctx = param; + + switch (proc_params->id) { + case PKCS11_CKM_AES_ECB_ENCRYPT_DATA: + if (param->size % TEE_AES_BLOCK_SIZE) { + rc = PKCS11_CKR_DATA_LEN_RANGE; + goto err; + } + TEE_CipherInit(processing->tee_op_handle, NULL, 0); + break; + case PKCS11_CKM_AES_CBC_ENCRYPT_DATA: + if (param->size % TEE_AES_BLOCK_SIZE) { + rc = PKCS11_CKR_DATA_LEN_RANGE; + goto err; + } + TEE_CipherInit(processing->tee_op_handle, iv, 16); + break; + default: + TEE_Panic(proc_params->id); + break; + } + + return PKCS11_CKR_OK; + +err: + processing->extra_ctx = NULL; + TEE_Free(param); + return rc; +} + +static enum pkcs11_rc +input_hmac_len_is_valid(struct pkcs11_attribute_head *proc_params, + uint32_t hmac_len) +{ + uint32_t sign_sz = 0; + + switch (proc_params->id) { + case PKCS11_CKM_MD5_HMAC_GENERAL: + sign_sz = TEE_MD5_HASH_SIZE; + break; + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + sign_sz = TEE_SHA1_HASH_SIZE; + break; + case PKCS11_CKM_SHA224_HMAC_GENERAL: + sign_sz = TEE_SHA224_HASH_SIZE; + break; + case PKCS11_CKM_SHA256_HMAC_GENERAL: + sign_sz = TEE_SHA256_HASH_SIZE; + break; + case PKCS11_CKM_SHA384_HMAC_GENERAL: + sign_sz = TEE_SHA384_HASH_SIZE; + break; + case PKCS11_CKM_SHA512_HMAC_GENERAL: + sign_sz = TEE_SHA512_HASH_SIZE; + break; + case PKCS11_CKM_AES_CMAC_GENERAL: + sign_sz = TEE_AES_BLOCK_SIZE; + break; + default: + return PKCS11_CKR_MECHANISM_INVALID; + } + + if (!hmac_len || hmac_len > sign_sz) + return PKCS11_CKR_SIGNATURE_LEN_RANGE; + + return PKCS11_CKR_OK; +} + +static enum pkcs11_rc +init_tee_operation(struct pkcs11_session *session, + struct pkcs11_attribute_head *proc_params) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + uint32_t *pkcs11_data = NULL; + + switch (proc_params->id) { + case PKCS11_CKM_AES_CMAC: + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA512_HMAC: + if (proc_params->size) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + + TEE_MACInit(session->processing->tee_op_handle, NULL, 0); + rc = PKCS11_CKR_OK; + break; + case PKCS11_CKM_AES_CMAC_GENERAL: + case PKCS11_CKM_MD5_HMAC_GENERAL: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + if (proc_params->size != sizeof(uint32_t)) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + + pkcs11_data = TEE_Malloc(sizeof(uint32_t), + TEE_MALLOC_FILL_ZERO); + if (!pkcs11_data) + return PKCS11_CKR_DEVICE_MEMORY; + + TEE_MemMove(pkcs11_data, proc_params->data, sizeof(uint32_t)); + + rc = input_hmac_len_is_valid(proc_params, *pkcs11_data); + if (rc) { + TEE_Free(pkcs11_data); + return rc; + } + + session->processing->extra_ctx = (void *)pkcs11_data; + + TEE_MACInit(session->processing->tee_op_handle, NULL, 0); + rc = PKCS11_CKR_OK; + break; + case PKCS11_CKM_AES_ECB: + if (proc_params->size) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + + TEE_CipherInit(session->processing->tee_op_handle, NULL, 0); + rc = PKCS11_CKR_OK; + break; + case PKCS11_CKM_AES_CBC: + case PKCS11_CKM_AES_CTS: + if (proc_params->size != 16) + return PKCS11_CKR_MECHANISM_PARAM_INVALID; + + TEE_CipherInit(session->processing->tee_op_handle, + proc_params->data, 16); + rc = PKCS11_CKR_OK; + break; + case PKCS11_CKM_AES_CTR: + rc = tee_init_ctr_operation(session->processing, + proc_params->data, + proc_params->size); + break; + case PKCS11_CKM_AES_ECB_ENCRYPT_DATA: + case PKCS11_CKM_AES_CBC_ENCRYPT_DATA: + rc = tee_init_derive_symm(session->processing, proc_params); + break; + default: + TEE_Panic(proc_params->id); + break; + } + + return rc; +} + +enum pkcs11_rc init_symm_operation(struct pkcs11_session *session, + enum processing_func function, + struct pkcs11_attribute_head *proc_params, + struct pkcs11_object *obj) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(processing_is_tee_symm(proc_params->id)); + + rc = allocate_tee_operation(session, function, proc_params, obj); + if (rc) + return rc; + + rc = load_tee_key(session, obj, proc_params); + if (rc) + return rc; + + rc = init_tee_operation(session, proc_params); + if (!rc) + session->processing->mecha_type = proc_params->id; + + return rc; +} + +/* Validate input buffer size as per PKCS#11 constraints */ +static enum pkcs11_rc input_data_size_is_valid(struct active_processing *proc, + enum processing_func function, + size_t in_size) +{ + switch (proc->mecha_type) { + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + if (function == PKCS11_FUNCTION_ENCRYPT && + in_size % TEE_AES_BLOCK_SIZE) + return PKCS11_CKR_DATA_LEN_RANGE; + if (function == PKCS11_FUNCTION_DECRYPT && + in_size % TEE_AES_BLOCK_SIZE) + return PKCS11_CKR_ENCRYPTED_DATA_LEN_RANGE; + break; + case PKCS11_CKM_AES_CTS: + if (function == PKCS11_FUNCTION_ENCRYPT && + in_size < TEE_AES_BLOCK_SIZE) + return PKCS11_CKR_DATA_LEN_RANGE; + if (function == PKCS11_FUNCTION_DECRYPT && + in_size < TEE_AES_BLOCK_SIZE) + return PKCS11_CKR_ENCRYPTED_DATA_LEN_RANGE; + break; + default: + break; + } + + return PKCS11_CKR_OK; +} + +/* Validate input buffer size as per PKCS#11 constraints */ +static enum pkcs11_rc input_sign_size_is_valid(struct active_processing *proc, + size_t in_size) +{ + size_t sign_sz = 0; + + switch (proc->mecha_type) { + case PKCS11_CKM_MD5_HMAC: + sign_sz = TEE_MD5_HASH_SIZE; + break; + case PKCS11_CKM_SHA_1_HMAC: + sign_sz = TEE_SHA1_HASH_SIZE; + break; + case PKCS11_CKM_SHA224_HMAC: + sign_sz = TEE_SHA224_HASH_SIZE; + break; + case PKCS11_CKM_SHA256_HMAC: + sign_sz = TEE_SHA256_HASH_SIZE; + break; + case PKCS11_CKM_SHA384_HMAC: + sign_sz = TEE_SHA384_HASH_SIZE; + break; + case PKCS11_CKM_SHA512_HMAC: + sign_sz = TEE_SHA512_HASH_SIZE; + break; + case PKCS11_CKM_AES_CMAC: + sign_sz = TEE_AES_BLOCK_SIZE; + break; + default: + return PKCS11_CKR_GENERAL_ERROR; + } + + if (in_size != sign_sz) + return PKCS11_CKR_SIGNATURE_LEN_RANGE; + + return PKCS11_CKR_OK; +} + +/* + * step_sym_cipher - processing symmetric (and related) cipher operation step + * + * @session - current session + * @function - processing function (encrypt, decrypt, sign, ...) + * @step - step ID in the processing (oneshot, update, final) + * @ptype - invocation parameter types + * @params - invocation parameter references + */ +enum pkcs11_rc step_symm_operation(struct pkcs11_session *session, + enum processing_func function, + enum processing_step step, + uint32_t ptypes, TEE_Param *params) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + TEE_Result res = TEE_ERROR_GENERIC; + void *in_buf = NULL; + size_t in_size = 0; + void *out_buf = NULL; + size_t out_size = 0; + void *in2_buf = NULL; + uint32_t in2_size = 0; + bool output_data = false; + struct active_processing *proc = session->processing; + uint32_t hmac_len = 0; + uint8_t computed_mac[TEE_MAX_HASH_SIZE] = { 0 }; + size_t computed_mac_size = TEE_MAX_HASH_SIZE; + + if (TEE_PARAM_TYPE_GET(ptypes, 1) == TEE_PARAM_TYPE_MEMREF_INPUT) { + in_buf = params[1].memref.buffer; + in_size = params[1].memref.size; + if (in_size && !in_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 2) == TEE_PARAM_TYPE_MEMREF_INPUT) { + in2_buf = params[2].memref.buffer; + in2_size = params[2].memref.size; + if (in2_size && !in2_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 2) == TEE_PARAM_TYPE_MEMREF_OUTPUT) { + out_buf = params[2].memref.buffer; + out_size = params[2].memref.size; + if (out_size && !out_buf) + return PKCS11_CKR_ARGUMENTS_BAD; + } + if (TEE_PARAM_TYPE_GET(ptypes, 3) != TEE_PARAM_TYPE_NONE) + return PKCS11_CKR_ARGUMENTS_BAD; + + switch (step) { + case PKCS11_FUNC_STEP_ONESHOT: + case PKCS11_FUNC_STEP_UPDATE: + case PKCS11_FUNC_STEP_FINAL: + break; + default: + return PKCS11_CKR_GENERAL_ERROR; + } + + if (step != PKCS11_FUNC_STEP_FINAL) { + rc = input_data_size_is_valid(proc, function, in_size); + if (rc) + return rc; + } + + /* + * Feed active operation with data + */ + switch (proc->mecha_type) { + case PKCS11_CKM_AES_CMAC: + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA512_HMAC: + case PKCS11_CKM_AES_CMAC_GENERAL: + case PKCS11_CKM_MD5_HMAC_GENERAL: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + if (step == PKCS11_FUNC_STEP_FINAL || + step == PKCS11_FUNC_STEP_ONESHOT) + break; + + if (!in_buf) { + DMSG("No input data"); + return PKCS11_CKR_ARGUMENTS_BAD; + } + + switch (function) { + case PKCS11_FUNCTION_SIGN: + case PKCS11_FUNCTION_VERIFY: + TEE_MACUpdate(proc->tee_op_handle, in_buf, in_size); + rc = PKCS11_CKR_OK; + break; + default: + TEE_Panic(function); + break; + } + break; + + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + case PKCS11_CKM_AES_CTS: + case PKCS11_CKM_AES_CTR: + if (step == PKCS11_FUNC_STEP_FINAL || + step == PKCS11_FUNC_STEP_ONESHOT) + break; + + if (!in_buf) { + EMSG("No input data"); + return PKCS11_CKR_ARGUMENTS_BAD; + } + + switch (function) { + case PKCS11_FUNCTION_ENCRYPT: + case PKCS11_FUNCTION_DECRYPT: + res = TEE_CipherUpdate(proc->tee_op_handle, + in_buf, in_size, + out_buf, &out_size); + output_data = true; + rc = tee2pkcs_error(res); + break; + default: + TEE_Panic(function); + break; + } + break; + + default: + TEE_Panic(proc->mecha_type); + break; + } + + if (step == PKCS11_FUNC_STEP_UPDATE) + goto out; + + /* + * Finalize (PKCS11_FUNC_STEP_ONESHOT/_FINAL) operation + */ + switch (session->processing->mecha_type) { + case PKCS11_CKM_AES_CMAC: + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA512_HMAC: + switch (function) { + case PKCS11_FUNCTION_SIGN: + res = TEE_MACComputeFinal(proc->tee_op_handle, + in_buf, in_size, out_buf, + &out_size); + output_data = true; + rc = tee2pkcs_error(res); + break; + case PKCS11_FUNCTION_VERIFY: + rc = input_sign_size_is_valid(proc, in2_size); + if (rc) + return rc; + res = TEE_MACCompareFinal(proc->tee_op_handle, + in_buf, in_size, in2_buf, + in2_size); + rc = tee2pkcs_error(res); + break; + default: + TEE_Panic(function); + break; + } + + break; + + case PKCS11_CKM_AES_CMAC_GENERAL: + case PKCS11_CKM_MD5_HMAC_GENERAL: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + assert(session->processing->extra_ctx); + hmac_len = *(uint32_t *)session->processing->extra_ctx; + + switch (function) { + case PKCS11_FUNCTION_SIGN: + if (out_size < hmac_len) { + /* inform client of required size */ + out_size = hmac_len; + output_data = true; + rc = PKCS11_CKR_BUFFER_TOO_SMALL; + goto out; + } + + res = TEE_MACComputeFinal(proc->tee_op_handle, + in_buf, in_size, + computed_mac, + &computed_mac_size); + if (res == TEE_SUCCESS) { + /* truncate to hmac_len */ + TEE_MemMove(out_buf, computed_mac, hmac_len); + output_data = true; + } + + /* inform client of required size */ + out_size = hmac_len; + rc = tee2pkcs_error(res); + break; + case PKCS11_FUNCTION_VERIFY: + /* must compute full MAC before comparing partial */ + res = TEE_MACComputeFinal(proc->tee_op_handle, in_buf, + in_size, computed_mac, + &computed_mac_size); + + if (!in2_size || in2_size > computed_mac_size) { + EMSG("Invalid signature size: %"PRIu32, + in2_size); + return PKCS11_CKR_SIGNATURE_LEN_RANGE; + } + + if (res == TEE_SUCCESS) { + /* + * Only the first in2_size bytes of the + * signature to be verified is passed in from + * caller + */ + if (TEE_MemCompare(in2_buf, computed_mac, + in2_size)) { + res = TEE_ERROR_MAC_INVALID; + } + } + + rc = tee2pkcs_error(res); + break; + default: + TEE_Panic(function); + break; + } + + break; + + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + case PKCS11_CKM_AES_CTS: + case PKCS11_CKM_AES_CTR: + if (step == PKCS11_FUNC_STEP_ONESHOT && !in_buf) { + EMSG("No input data"); + return PKCS11_CKR_ARGUMENTS_BAD; + } + + switch (function) { + case PKCS11_FUNCTION_ENCRYPT: + case PKCS11_FUNCTION_DECRYPT: + res = TEE_CipherDoFinal(proc->tee_op_handle, + in_buf, in_size, + out_buf, &out_size); + output_data = true; + rc = tee2pkcs_error(res); + break; + default: + TEE_Panic(function); + break; + } + break; + default: + TEE_Panic(proc->mecha_type); + break; + } + +out: + if (output_data && + (rc == PKCS11_CKR_OK || rc == PKCS11_CKR_BUFFER_TOO_SMALL)) { + switch (TEE_PARAM_TYPE_GET(ptypes, 2)) { + case TEE_PARAM_TYPE_MEMREF_OUTPUT: + case TEE_PARAM_TYPE_MEMREF_INOUT: + params[2].memref.size = out_size; + break; + default: + rc = PKCS11_CKR_ARGUMENTS_BAD; + break; + } + } + + return rc; +} + +enum pkcs11_rc derive_key_by_symm_enc(struct pkcs11_session *session, + void **out_buf, uint32_t *out_size) +{ + enum pkcs11_rc rc = PKCS11_CKR_GENERAL_ERROR; + TEE_Result res = TEE_ERROR_GENERIC; + struct active_processing *proc = session->processing; + struct input_data_ref *input = proc->extra_ctx; + void *in_buf = NULL; + void *dest_buf = NULL; + uint32_t in_size = 0; + size_t tmp_sz = 0; + + switch (proc->mecha_type) { + case PKCS11_CKM_AES_ECB_ENCRYPT_DATA: + case PKCS11_CKM_AES_CBC_ENCRYPT_DATA: + if (!proc->extra_ctx) + return PKCS11_CKR_ARGUMENTS_BAD; + + in_buf = input->data; + in_size = input->size; + + *out_size = in_size; + dest_buf = TEE_Malloc(*out_size, 0); + if (!dest_buf) + return PKCS11_CKR_DEVICE_MEMORY; + + tmp_sz = *out_size; + res = TEE_CipherDoFinal(proc->tee_op_handle, in_buf, in_size, + dest_buf, &tmp_sz); + *out_size = tmp_sz; + rc = tee2pkcs_error(res); + if (rc) { + TEE_Free(dest_buf); + return rc; + } + + *out_buf = dest_buf; + break; + default: + return PKCS11_CKR_MECHANISM_INVALID; + } + + return rc; +} + +enum pkcs11_rc wrap_data_by_symm_enc(struct pkcs11_session *session, + void *data, uint32_t data_sz, + void *out_buf, uint32_t *out_sz) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct active_processing *proc = session->processing; + void *in_buf = NULL; + uint32_t align = 0; + uint32_t in_sz = data_sz; + size_t tmp_sz = *out_sz; + uint8_t *tmp_buf = out_buf; + + switch (proc->mecha_type) { + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + align = data_sz % TEE_AES_BLOCK_SIZE; + if (align) + in_sz = data_sz + (TEE_AES_BLOCK_SIZE - align); + + if (*out_sz < in_sz) { + *out_sz = in_sz; + return PKCS11_CKR_BUFFER_TOO_SMALL; + } + + if (align) { + if (data_sz > TEE_AES_BLOCK_SIZE) { + in_sz = data_sz - align; + res = TEE_CipherUpdate(proc->tee_op_handle, + data, in_sz, tmp_buf, + &tmp_sz); + if (res) { + assert(res != TEE_ERROR_SHORT_BUFFER); + return tee2pkcs_error(res); + } + tmp_buf += tmp_sz; + tmp_sz = *out_sz - tmp_sz; + } else { + in_sz = 0; + } + + in_buf = TEE_Malloc(TEE_AES_BLOCK_SIZE, + TEE_MALLOC_FILL_ZERO); + if (!in_buf) + return PKCS11_CKR_DEVICE_MEMORY; + + TEE_MemMove(in_buf, (uint8_t *)data + in_sz, align); + in_sz = TEE_AES_BLOCK_SIZE; + } else { + in_buf = data; + in_sz = data_sz; + } + + res = TEE_CipherDoFinal(proc->tee_op_handle, in_buf, in_sz, + tmp_buf, &tmp_sz); + if (res == TEE_SUCCESS || res == TEE_ERROR_SHORT_BUFFER) { + *out_sz = tmp_sz; + if (align) + *out_sz += tmp_buf - (uint8_t *)out_buf; + } + + if (align) + TEE_Free(in_buf); + + return tee2pkcs_error(res); + default: + return PKCS11_CKR_MECHANISM_INVALID; + } + + return PKCS11_CKR_GENERAL_ERROR; +} + +enum pkcs11_rc unwrap_key_by_symm(struct pkcs11_session *session, void *data, + uint32_t data_sz, void **out_buf, + uint32_t *out_sz) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct active_processing *proc = session->processing; + size_t tmp_sz = 0; + + if (input_data_size_is_valid(proc, PKCS11_FUNCTION_DECRYPT, data_sz)) + return PKCS11_CKR_WRAPPED_KEY_LEN_RANGE; + + switch (proc->mecha_type) { + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + *out_sz = 0; + res = TEE_CipherDoFinal(proc->tee_op_handle, data, data_sz, + NULL, &tmp_sz); + *out_sz = tmp_sz; + if (res != TEE_ERROR_SHORT_BUFFER) { + DMSG("TEE_CipherDoFinal() issue: %#"PRIx32, res); + return PKCS11_CKR_GENERAL_ERROR; + } + + *out_buf = TEE_Malloc(*out_sz, TEE_MALLOC_FILL_ZERO); + if (!*out_buf) + return PKCS11_CKR_DEVICE_MEMORY; + + res = TEE_CipherDoFinal(proc->tee_op_handle, data, data_sz, + *out_buf, &tmp_sz); + *out_sz = tmp_sz; + if (tee2pkcs_error(res)) { + TEE_Free(*out_buf); + *out_buf = NULL; + return PKCS11_CKR_WRAPPED_KEY_INVALID; + } + break; + default: + return PKCS11_CKR_MECHANISM_INVALID; + } + + return PKCS11_CKR_OK; +} diff --git a/optee_os/ta/pkcs11/src/sanitize_object.c b/optee_os/ta/pkcs11/src/sanitize_object.c new file mode 100644 index 0000000..5fdde55 --- /dev/null +++ b/optee_os/ta/pkcs11/src/sanitize_object.c @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "attributes.h" +#include "pkcs11_helpers.h" +#include "sanitize_object.h" +#include "serializer.h" +#include "token_capabilities.h" + +/* + * Functions to generate a serialized object. + * References are pointers to struct serializer. + */ + +bool sanitize_consistent_class_and_type(struct obj_attrs *attrs) +{ + switch (get_class(attrs)) { + case PKCS11_CKO_DATA: + case PKCS11_CKO_CERTIFICATE: + return true; + case PKCS11_CKO_SECRET_KEY: + return key_type_is_symm_key(get_key_type(attrs)); + case PKCS11_CKO_MECHANISM: + return mechanism_is_valid(get_mechanism_type(attrs)); + case PKCS11_CKO_PUBLIC_KEY: + case PKCS11_CKO_PRIVATE_KEY: + return key_type_is_asymm_key(get_key_type(attrs)); + case PKCS11_CKO_OTP_KEY: + case PKCS11_CKO_DOMAIN_PARAMETERS: + case PKCS11_CKO_HW_FEATURE: + default: + return false; + } + + return false; +} + +static enum pkcs11_rc read_attr_advance(void *buf, size_t blen, size_t *pos, + struct pkcs11_attribute_head *attr, + void **data) +{ + uint8_t *b = buf; + size_t data_pos = 0; + size_t next_pos = 0; + + if (ADD_OVERFLOW(*pos, sizeof(*attr), &data_pos) || data_pos > blen) + return PKCS11_CKR_FUNCTION_FAILED; + TEE_MemMove(attr, b + *pos, sizeof(*attr)); + + if (ADD_OVERFLOW(data_pos, attr->size, &next_pos) || next_pos > blen) + return PKCS11_CKR_FUNCTION_FAILED; + + *data = b + data_pos; + *pos = next_pos; + + return PKCS11_CKR_OK; +} + +/* Sanitize class/type in a client attribute list */ +static enum pkcs11_rc sanitize_class_and_type(struct obj_attrs **dst, void *src, + size_t src_size, + uint32_t class_hint, + uint32_t type_hint) +{ + uint32_t class_found = PKCS11_CKO_UNDEFINED_ID; + size_t pos = sizeof(struct pkcs11_object_head); + struct pkcs11_attribute_head cli_ref = { }; + uint32_t type_found = PKCS11_UNDEFINED_ID; + enum pkcs11_rc rc = PKCS11_CKR_OK; + void *data = NULL; + + while (pos != src_size) { + rc = read_attr_advance(src, src_size, &pos, &cli_ref, &data); + if (rc) + goto err; + + if (cli_ref.id == PKCS11_CKA_CLASS) { + uint32_t class = 0; + + if (cli_ref.size != sizeof(class)) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto err; + } + + TEE_MemMove(&class, data, sizeof(class)); + + if (class_found != PKCS11_CKO_UNDEFINED_ID && + class_found != class) { + EMSG("Conflicting class value"); + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto err; + } + + class_found = class; + continue; + } + + /* The attribute is a type-in-class */ + if (pkcs11_attr_is_type(cli_ref.id)) { + uint32_t type = 0; + + if (cli_ref.size != sizeof(type)) { + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto err; + } + + TEE_MemMove(&type, data, sizeof(type)); + + if (type_found != PKCS11_CKK_UNDEFINED_ID && + type_found != type) { + EMSG("Conflicting type-in-class value"); + rc = PKCS11_CKR_TEMPLATE_INCONSISTENT; + goto err; + } + + type_found = type; + } + } + + if (class_found != PKCS11_CKO_UNDEFINED_ID) { + rc = add_attribute(dst, PKCS11_CKA_CLASS, + &class_found, sizeof(class_found)); + if (rc) + return rc; + } else { + if (class_hint != PKCS11_CKO_UNDEFINED_ID) { + rc = add_attribute(dst, PKCS11_CKA_CLASS, + &class_hint, sizeof(class_hint)); + if (rc) + return rc; + } + } + + if (type_found != PKCS11_UNDEFINED_ID) { + rc = add_attribute(dst, PKCS11_CKA_KEY_TYPE, + &type_found, sizeof(type_found)); + if (rc) + return rc; + } else { + if (type_hint != PKCS11_UNDEFINED_ID) { + rc = add_attribute(dst, PKCS11_CKA_KEY_TYPE, + &type_hint, sizeof(type_hint)); + if (rc) + return rc; + } + } + + return PKCS11_CKR_OK; + +err: + trace_attributes_from_api_head("bad-template", src, src_size); + + return rc; +} + +static enum pkcs11_rc sanitize_boolprops(struct obj_attrs **dst, void *src, + size_t src_size) +{ + bitstr_t bit_decl(seen_attrs, PKCS11_BOOLPROPS_MAX_COUNT) = { 0 }; + bitstr_t bit_decl(boolprops, PKCS11_BOOLPROPS_MAX_COUNT) = { 0 }; + size_t pos = sizeof(struct pkcs11_object_head); + struct pkcs11_attribute_head cli_ref = { }; + enum pkcs11_rc rc = PKCS11_CKR_OK; + bool value = false; + void *data = NULL; + int idx = 0; + + /* + * We're keeping track of seen boolean attributes in the bitstring + * seen_attrs. The bitstring boolprops holds the recorded value + * once seen_attrs has been updated. + */ + + while (pos != src_size) { + rc = read_attr_advance(src, src_size, &pos, &cli_ref, &data); + if (rc) + return rc; + + idx = pkcs11_attr2boolprop_shift(cli_ref.id); + if (idx < 0) + continue; /* skipping non-boolean attributes */ + + if (idx >= PKCS11_BOOLPROPS_MAX_COUNT || + cli_ref.size != sizeof(uint8_t)) + return PKCS11_CKR_FUNCTION_FAILED; + + value = *(uint8_t *)data; + + /* + * If this attribute has already been seen, check that it + * still holds the same value as last time. + */ + if (bit_test(seen_attrs, idx) && + value != (bool)bit_test(boolprops, idx)) + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + + if (value) + bit_set(boolprops, idx); + + if (!bit_test(seen_attrs, idx)) { + uint8_t pkcs11_bool = value; + + rc = add_attribute(dst, cli_ref.id, &pkcs11_bool, + sizeof(pkcs11_bool)); + if (rc) + return rc; + } + bit_set(seen_attrs, idx); + } + + return PKCS11_CKR_OK; +} + +static uint32_t sanitize_indirect_attr(struct obj_attrs **dst, + struct pkcs11_attribute_head *cli_ref, + char *data) +{ + struct obj_attrs *obj2 = NULL; + enum pkcs11_rc rc = PKCS11_CKR_OK; + + assert(pkcs11_attr_has_indirect_attributes(cli_ref->id)); + + /* Build a new serial object while sanitizing the attributes list */ + rc = sanitize_client_object(&obj2, data, cli_ref->size, + PKCS11_CKO_UNDEFINED_ID, + PKCS11_UNDEFINED_ID); + if (rc) + goto out; + + rc = add_attribute(dst, cli_ref->id, obj2, + sizeof(*obj2) + obj2->attrs_size); +out: + TEE_Free(obj2); + return rc; +} + +enum pkcs11_rc sanitize_client_object(struct obj_attrs **dst, void *src, + size_t size, uint32_t class_hint, + uint32_t type_hint) +{ + struct pkcs11_attribute_head cli_ref = { }; + struct pkcs11_object_head head = { }; + enum pkcs11_rc rc = PKCS11_CKR_OK; + size_t pos = sizeof(head); + size_t sz_from_hdr = 0; + void *data = NULL; + + if (size < sizeof(head)) + return PKCS11_CKR_ARGUMENTS_BAD; + + TEE_MemMove(&head, src, sizeof(head)); + + if (ADD_OVERFLOW(sizeof(head), head.attrs_size, &sz_from_hdr) || + size < sz_from_hdr) + return PKCS11_CKR_ARGUMENTS_BAD; + + rc = init_attributes_head(dst); + if (rc) + return rc; + + rc = sanitize_class_and_type(dst, src, sz_from_hdr, class_hint, + type_hint); + if (rc) + return rc; + + rc = sanitize_boolprops(dst, src, sz_from_hdr); + if (rc) + return rc; + + while (pos != sz_from_hdr) { + rc = read_attr_advance(src, sz_from_hdr, &pos, &cli_ref, &data); + if (rc) + return rc; + + if (cli_ref.id == PKCS11_CKA_CLASS || + pkcs11_attr_is_type(cli_ref.id) || + pkcs11_attr_is_boolean(cli_ref.id)) + continue; + + if (pkcs11_attr_has_indirect_attributes(cli_ref.id)) { + rc = sanitize_indirect_attr(dst, &cli_ref, data); + if (rc) + return rc; + + continue; + } + + if (!valid_pkcs11_attribute_id(cli_ref.id, cli_ref.size)) { + EMSG("Invalid attribute id %#"PRIx32, cli_ref.id); + return PKCS11_CKR_TEMPLATE_INCONSISTENT; + } + + rc = add_attribute(dst, cli_ref.id, data, cli_ref.size); + if (rc) + return rc; + } + + return rc; +} + +/* + * Debug: dump object attribute array to output trace + */ + +static void __trace_attributes(char *prefix, void *src, void *end) +{ + size_t next = 0; + char *prefix2 = NULL; + size_t prefix_len = strlen(prefix); + char *cur = src; + + /* append 4 spaces to the prefix plus terminal '\0' */ + prefix2 = TEE_Malloc(prefix_len + 1 + 4, TEE_MALLOC_FILL_ZERO); + if (!prefix2) + return; + + TEE_MemMove(prefix2, prefix, prefix_len + 1); + TEE_MemFill(prefix2 + prefix_len, ' ', 4); + *(prefix2 + prefix_len + 4) = '\0'; + + for (; cur < (char *)end; cur += next) { + struct pkcs11_attribute_head pkcs11_ref; + uint8_t data[4] = { 0 }; + uint32_t data_u32 = 0; + char *start = NULL; + + TEE_MemMove(&pkcs11_ref, cur, sizeof(pkcs11_ref)); + TEE_MemMove(&data[0], cur + sizeof(pkcs11_ref), + MIN(pkcs11_ref.size, sizeof(data))); + TEE_MemMove(&data_u32, cur + sizeof(pkcs11_ref), + sizeof(data_u32)); + + next = sizeof(pkcs11_ref) + pkcs11_ref.size; + + DMSG_RAW("%s Attr %s / %s (%#04"PRIx32" %"PRIu32"-byte)", + prefix, id2str_attr(pkcs11_ref.id), + id2str_attr_value(pkcs11_ref.id, pkcs11_ref.size, + cur + sizeof(pkcs11_ref)), + pkcs11_ref.id, pkcs11_ref.size); + + switch (pkcs11_ref.size) { + case 0: + break; + case 1: + DMSG_RAW("%s Attr byte value: %02x", prefix, data[0]); + break; + case 2: + DMSG_RAW("%s Attr byte value: %02x %02x", + prefix, data[0], data[1]); + break; + case 3: + DMSG_RAW("%s Attr byte value: %02x %02x %02x", + prefix, data[0], data[1], data[2]); + break; + case 4: + DMSG_RAW("%s Attr byte value: %02x %02x %02x %02x", + prefix, data[0], data[1], data[2], data[3]); + break; + default: + DMSG_RAW("%s Attr byte value: %02x %02x %02x %02x ...", + prefix, data[0], data[1], data[2], data[3]); + break; + } + + switch (pkcs11_ref.id) { + case PKCS11_CKA_WRAP_TEMPLATE: + case PKCS11_CKA_UNWRAP_TEMPLATE: + case PKCS11_CKA_DERIVE_TEMPLATE: + start = cur + sizeof(pkcs11_ref); + trace_attributes_from_api_head(prefix2, start, + (char *)end - start); + break; + default: + break; + } + } + + /* Sanity */ + if (cur != (char *)end) + EMSG("Warning: unexpected alignment issue"); + + TEE_Free(prefix2); +} + +void trace_attributes_from_api_head(const char *prefix, void *ref, size_t size) +{ + struct pkcs11_object_head head = { }; + char *pre = NULL; + size_t offset = 0; + + TEE_MemMove(&head, ref, sizeof(head)); + + if (size > sizeof(head) + head.attrs_size) { + EMSG("template overflows client buffer (%zu/%zu)", + size, sizeof(head) + head.attrs_size); + return; + } + + pre = TEE_Malloc(prefix ? strlen(prefix) + 2 : 2, TEE_MALLOC_FILL_ZERO); + if (!pre) { + EMSG("%s: out of memory", prefix); + return; + } + if (prefix) + TEE_MemMove(pre, prefix, strlen(prefix)); + + DMSG_RAW("%s,--- (serial object) Attributes list --------", pre); + DMSG_RAW("%s| %"PRIu32" item(s) - %"PRIu32" bytes", + pre, head.attrs_count, head.attrs_size); + + offset = sizeof(head); + pre[prefix ? strlen(prefix) : 0] = '|'; + __trace_attributes(pre, (char *)ref + offset, + (char *)ref + offset + head.attrs_size); + + DMSG_RAW("%s`-----------------------", prefix ? prefix : ""); + + TEE_Free(pre); +} diff --git a/optee_os/ta/pkcs11/src/sanitize_object.h b/optee_os/ta/pkcs11/src/sanitize_object.h new file mode 100644 index 0000000..6b170a2 --- /dev/null +++ b/optee_os/ta/pkcs11/src/sanitize_object.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_SANITIZE_OBJECT_H +#define PKCS11_TA_SANITIZE_OBJECT_H + +#include "serializer.h" + +/* + * sanitize_consistent_class_and_type - Check object type matches object class + * + * @attrs - object attributes + * Return true if class/type matches, else return false + */ +bool sanitize_consistent_class_and_type(struct obj_attrs *attrs); + +/** + * sanitize_client_object - Setup a serializer from a serialized object + * + * @dst - output structure tracking the generated serial object + * @head - pointer to the formatted serialized object (its head) + * @size - byte size of the serialized binary blob + * @class_hint - Hint for class to be added to template if not presnet + * in serialized object. + * @type_hint - Hint for type to be added to template if not presnet + * in serialized object. + * + * This function copies an attribute list from a client API attribute head + * into a PKCS11 TA internal attribute structure. It generates a serialized + * attribute list with a consistent format and identified attribute IDs. + * + * @head points to a blob starting with a pkcs11 attribute header. + * @head may point to an unaligned address. + * This function allocates, fills and returns a serialized attribute list + * into a serializer container. + */ +enum pkcs11_rc sanitize_client_object(struct obj_attrs **dst, void *head, + size_t size, uint32_t class_hint, + uint32_t type_hint); + +/* Debug: dump attribute content as debug traces */ +void trace_attributes_from_api_head(const char *prefix, void *ref, size_t size); + +#endif /*PKCS11_TA_SANITIZE_OBJECT_H*/ diff --git a/optee_os/ta/pkcs11/src/serializer.c b/optee_os/ta/pkcs11/src/serializer.c new file mode 100644 index 0000000..dac930a --- /dev/null +++ b/optee_os/ta/pkcs11/src/serializer.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "pkcs11_token.h" +#include "serializer.h" + +/* + * Util routines for serializes unformatted arguments in a client memref + */ +void serialargs_init(struct serialargs *args, void *in, size_t size) +{ + args->start = in; + args->next = in; + args->size = size; +} + +enum pkcs11_rc serialargs_get(struct serialargs *args, void *out, size_t size) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + void *src = NULL; + + rc = serialargs_get_ptr(args, &src, size); + if (!rc) + TEE_MemMove(out, src, size); + + return rc; +} + +static enum pkcs11_rc alloc_and_get(struct serialargs *args, char *orig_next, + const void *buf0, size_t buf0_sz, + void **out, size_t size) +{ + enum pkcs11_rc rc = PKCS11_CKR_OK; + uint8_t *ptr = NULL; + void *src = NULL; + size_t sz = 0; + + if (ADD_OVERFLOW(buf0_sz, size, &sz)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!sz) { + *out = NULL; + return PKCS11_CKR_OK; + } + + rc = serialargs_get_ptr(args, &src, size); + if (rc) + return rc; + + ptr = TEE_Malloc(sz, TEE_MALLOC_FILL_ZERO); + if (!ptr) { + args->next = orig_next; + return PKCS11_CKR_DEVICE_MEMORY; + } + + TEE_MemMove(ptr, buf0, buf0_sz); + TEE_MemMove(ptr + buf0_sz, src, size); + + *out = ptr; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc serialargs_alloc_and_get(struct serialargs *args, + void **out, size_t size) +{ + return alloc_and_get(args, args->next, NULL, 0, out, size); +} + +enum pkcs11_rc serialargs_get_ptr(struct serialargs *args, void **out, + size_t size) +{ + void *ptr = args->next; + vaddr_t next_end = 0; + + if (ADD_OVERFLOW((vaddr_t)args->next, size, &next_end)) + return PKCS11_CKR_ARGUMENTS_BAD; + + if (!size) { + *out = NULL; + return PKCS11_CKR_OK; + } + + if ((char *)next_end > args->start + args->size) { + EMSG("arg too short: full %zd, remain %zd, expect %zd", + args->size, args->size - (args->next - args->start), size); + return PKCS11_CKR_ARGUMENTS_BAD; + } + + args->next += size; + *out = ptr; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc +serialargs_alloc_get_one_attribute(struct serialargs *args, + struct pkcs11_attribute_head **out) +{ + struct pkcs11_attribute_head head = { }; + enum pkcs11_rc rc = PKCS11_CKR_OK; + char *orig_next = args->next; + void *p = NULL; + + rc = serialargs_get(args, &head, sizeof(head)); + if (rc) + return rc; + + rc = alloc_and_get(args, orig_next, &head, sizeof(head), &p, head.size); + if (rc) + return rc; + + *out = p; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc serialargs_alloc_get_attributes(struct serialargs *args, + struct pkcs11_object_head **out) +{ + struct pkcs11_object_head attr = { }; + enum pkcs11_rc rc = PKCS11_CKR_OK; + char *orig_next = args->next; + void *p = NULL; + + rc = serialargs_get(args, &attr, sizeof(attr)); + if (rc) + return rc; + + rc = alloc_and_get(args, orig_next, &attr, sizeof(attr), &p, + attr.attrs_size); + if (rc) + return rc; + + *out = p; + + return PKCS11_CKR_OK; +} + +bool serialargs_remaining_bytes(struct serialargs *args) +{ + return args->next < args->start + args->size; +} + +enum pkcs11_rc serialargs_get_session_from_handle(struct serialargs *args, + struct pkcs11_client *client, + struct pkcs11_session **sess) +{ + uint32_t rv = PKCS11_CKR_GENERAL_ERROR; + uint32_t session_handle = 0; + struct pkcs11_session *session = NULL; + + rv = serialargs_get(args, &session_handle, sizeof(uint32_t)); + if (rv) + return rv; + + session = pkcs11_handle2session(session_handle, client); + if (!session) + return PKCS11_CKR_SESSION_HANDLE_INVALID; + + *sess = session; + + return PKCS11_CKR_OK; +} + +enum pkcs11_rc serialize(char **bstart, size_t *blen, void *data, size_t len) +{ + char *buf = NULL; + size_t nlen = 0; + + if (ADD_OVERFLOW(*blen, len, &nlen)) + return PKCS11_CKR_ARGUMENTS_BAD; + + buf = TEE_Realloc(*bstart, nlen); + if (!buf) + return PKCS11_CKR_DEVICE_MEMORY; + + TEE_MemMove(buf + *blen, data, len); + + *blen = nlen; + *bstart = buf; + + return PKCS11_CKR_OK; +} + diff --git a/optee_os/ta/pkcs11/src/serializer.h b/optee_os/ta/pkcs11/src/serializer.h new file mode 100644 index 0000000..3e6fe09 --- /dev/null +++ b/optee_os/ta/pkcs11/src/serializer.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef PKCS11_TA_SERIALIZER_H +#define PKCS11_TA_SERIALIZER_H + +#include +#include +#include + +struct pkcs11_client; +struct pkcs11_session; + +/* + * Util routines for serializes unformated arguments in a client memref + */ +struct serialargs { + char *start; + char *next; + size_t size; +}; + +struct pkcs11_client; +struct pkcs11_session; + +/* + * serialargs_init() - Initialize with a new input buffer + * @args: serializing state + * @in: input buffer + * @size: size of the input buffer + */ +void serialargs_init(struct serialargs *args, void *in, size_t size); + +/* + * serialargs_get() - copy out a chunk of data and advance + * @args: serializing state + * @out: output buffer + * @sz: number of bytes to copy to output buffer + * + * Returns PKCS11_CKR_OK on success or PKCS11_CKR_ARGUMENTS_BAD on failure. + */ +enum pkcs11_rc serialargs_get(struct serialargs *args, void *out, size_t sz); + +/* + * serialargs_get_u32() - copy out a uint32_t and advance + * @args: serializing state + * @out: output buffer + * + * Returns PKCS11_CKR_OK on success or PKCS11_CKR_ARGUMENTS_BAD on failure. + */ +static inline enum pkcs11_rc serialargs_get_u32(struct serialargs *args, + uint32_t *out) +{ + return serialargs_get(args, out, sizeof(*out)); +} + +/* + * serialargs_get_ptr() - get a pointer to a chunk of data and advance + * @args: serializing state + * @out: Pointer to the data retrieved in *@out + * @size: Number of bytes to advance + * + * Returns PKCS11_CKR_OK on success or PKCS11_CKR_ARGUMENTS_BAD on failure. + */ +enum pkcs11_rc serialargs_get_ptr(struct serialargs *args, void **out, + size_t size); + +/* + * serialargs_alloc_get_one_attribute() - allocate and extract one attribute + * @args: serializing state + * @out: Pointer to the allocated and extracted attribute in *@out + * + * Returns PKCS11_CKR_OK on success or an error code from enum pkcs11_rc on + * failure. + */ +enum pkcs11_rc +serialargs_alloc_get_one_attribute(struct serialargs *args, + struct pkcs11_attribute_head **out); + +/* + * serialargs_alloc_get_attributes() - allocate and extract an object + * @args: serializing state + * @out: Pointer to the allocated and extracted object in *@out + * + * Returns PKCS11_CKR_OK on success or an error code from enum pkcs11_rc on + * failure. + */ +enum pkcs11_rc serialargs_alloc_get_attributes(struct serialargs *args, + struct pkcs11_object_head **out); + +/* + * serialargs_alloc_and_get() - allocate and extract data + * @args: serializing state + * @out: Pointer to the allocated and extracted data in *@out + * @size: Number of bytes to extract + * + * Returns PKCS11_CKR_OK on success or an error code from enum pkcs11_rc on + * failure. + */ +enum pkcs11_rc serialargs_alloc_and_get(struct serialargs *args, + void **out, size_t size); + +/* + * serialargs_remaining_bytes() - check for remaining bytes + * @args: serializing state + * + * Returns true if there are remaining bytes in @args or false if all bytes + * are consumed. + */ +bool serialargs_remaining_bytes(struct serialargs *args); + +/* + * serialargs_get_session_from_handle() - extract and verify session + * @args: serializing state + * @client: client state + * @sess: The retrieved session handle is available in *@sess + * + * Returns PKCS11_CKR_OK on success or an error code from enum pkcs11_rc on + * failure. + */ +enum pkcs11_rc serialargs_get_session_from_handle(struct serialargs *args, + struct pkcs11_client *client, + struct pkcs11_session **sess); + +/* + * serialize() - append data into a serialized buffer + * @bstart: points to start of a buffer or NULL, *@bstart is updated + * with the new buffer if changed + * @blen: size of the *@bstart buffer, updated when data is added + * @data: data to appen to the buffer + * @len: size of the @data buffer + * + * Returns PKCS11_CKR_OK on success or an error code from enum pkcs11_rc on + * failure. + */ +enum pkcs11_rc serialize(char **bstart, size_t *blen, void *data, size_t len); + +#endif /*PKCS11_TA_SERIALIZER_H*/ diff --git a/optee_os/ta/pkcs11/src/sub.mk b/optee_os/ta/pkcs11/src/sub.mk new file mode 100644 index 0000000..5dfbc2b --- /dev/null +++ b/optee_os/ta/pkcs11/src/sub.mk @@ -0,0 +1,18 @@ +srcs-y += attributes.c +srcs-y += entry.c +srcs-y += handle.c +srcs-y += object.c +srcs-y += persistent_token.c +srcs-y += pkcs11_attributes.c +srcs-y += pkcs11_helpers.c +srcs-y += pkcs11_token.c +srcs-y += processing.c +srcs-y += processing_aes.c +srcs-y += processing_asymm.c +srcs-y += processing_digest.c +srcs-y += processing_ec.c +srcs-y += processing_rsa.c +srcs-y += processing_symm.c +srcs-y += sanitize_object.c +srcs-y += serializer.c +srcs-y += token_capabilities.c diff --git a/optee_os/ta/pkcs11/src/token_capabilities.c b/optee_os/ta/pkcs11/src/token_capabilities.c new file mode 100644 index 0000000..e76c1df --- /dev/null +++ b/optee_os/ta/pkcs11/src/token_capabilities.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include + +#include "pkcs11_helpers.h" +#include "token_capabilities.h" + +#define ALLOWED_PKCS11_CKFM \ + (PKCS11_CKFM_ENCRYPT | PKCS11_CKFM_DECRYPT | \ + PKCS11_CKFM_DERIVE | PKCS11_CKFM_DIGEST | \ + PKCS11_CKFM_SIGN | PKCS11_CKFM_SIGN_RECOVER | \ + PKCS11_CKFM_VERIFY | PKCS11_CKFM_VERIFY_RECOVER | \ + PKCS11_CKFM_GENERATE | PKCS11_CKFM_GENERATE_KEY_PAIR | \ + PKCS11_CKFM_WRAP | PKCS11_CKFM_UNWRAP) + +/* + * Definition of supported processings for a PKCS#11 mechanisms + * @id: Mechanism ID + * @flags: Valid PKCS11_CKFM_* for a mechanism as per PKCS#11 + * @one_shot: true of mechanism can be used for a one-short processing + * @string: Helper string of the mechanism ID for debug purpose + */ +struct pkcs11_mechachism_modes { + uint32_t id; + uint32_t flags; + bool one_shot; +#if CFG_TEE_TA_LOG_LEVEL > 0 + const char *string; +#endif +}; + +#if CFG_TEE_TA_LOG_LEVEL > 0 +#define MECHANISM(_label, _flags, _single_part) \ + { \ + .id = _label, \ + .one_shot = (_single_part), \ + .flags = (_flags), \ + .string = #_label, \ + } +#else +#define MECHANISM(_label, _flags, _single_part) \ + { \ + .id = _label, \ + .one_shot = (_single_part), \ + .flags = (_flags), \ + } +#endif + +#define SINGLE_PART_ONLY true +#define ANY_PART false + +#define CKFM_CIPHER (PKCS11_CKFM_ENCRYPT | PKCS11_CKFM_DECRYPT) +#define CKFM_WRAP_UNWRAP (PKCS11_CKFM_WRAP | PKCS11_CKFM_UNWRAP) +#define CKFM_CIPHER_WRAP (CKFM_CIPHER | CKFM_WRAP_UNWRAP) +#define CKFM_CIPHER_WRAP_DERIVE (CKFM_CIPHER_WRAP | PKCS11_CKFM_DERIVE) +#define CKFM_AUTH_NO_RECOVER (PKCS11_CKFM_SIGN | PKCS11_CKFM_VERIFY) +#define CKFM_AUTH_WITH_RECOVER (PKCS11_CKFM_SIGN_RECOVER | \ + PKCS11_CKFM_VERIFY_RECOVER) + +/* PKCS#11 specificies permitted operation for each mechanism */ +static const struct pkcs11_mechachism_modes pkcs11_modes[] = { + /* AES */ + MECHANISM(PKCS11_CKM_AES_ECB, CKFM_CIPHER_WRAP, ANY_PART), + MECHANISM(PKCS11_CKM_AES_CBC, CKFM_CIPHER_WRAP, ANY_PART), + MECHANISM(PKCS11_CKM_AES_CBC_PAD, CKFM_CIPHER_WRAP, ANY_PART), + MECHANISM(PKCS11_CKM_AES_CTS, CKFM_CIPHER_WRAP, ANY_PART), + MECHANISM(PKCS11_CKM_AES_CTR, CKFM_CIPHER_WRAP, ANY_PART), + MECHANISM(PKCS11_CKM_AES_CMAC, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_AES_CMAC_GENERAL, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_AES_ECB_ENCRYPT_DATA, PKCS11_CKFM_DERIVE, + ANY_PART), + MECHANISM(PKCS11_CKM_AES_CBC_ENCRYPT_DATA, PKCS11_CKFM_DERIVE, + ANY_PART), + MECHANISM(PKCS11_CKM_AES_KEY_GEN, PKCS11_CKFM_GENERATE, ANY_PART), + MECHANISM(PKCS11_CKM_GENERIC_SECRET_KEY_GEN, PKCS11_CKFM_GENERATE, + ANY_PART), + /* Digest */ + MECHANISM(PKCS11_CKM_MD5, PKCS11_CKFM_DIGEST, ANY_PART), + MECHANISM(PKCS11_CKM_SHA_1, PKCS11_CKFM_DIGEST, ANY_PART), + MECHANISM(PKCS11_CKM_SHA224, PKCS11_CKFM_DIGEST, ANY_PART), + MECHANISM(PKCS11_CKM_SHA256, PKCS11_CKFM_DIGEST, ANY_PART), + MECHANISM(PKCS11_CKM_SHA384, PKCS11_CKFM_DIGEST, ANY_PART), + MECHANISM(PKCS11_CKM_SHA512, PKCS11_CKFM_DIGEST, ANY_PART), + /* HMAC */ + MECHANISM(PKCS11_CKM_MD5_HMAC, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA_1_HMAC, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA224_HMAC, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA256_HMAC, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA384_HMAC, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA512_HMAC, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_MD5_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA_1_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER, + ANY_PART), + MECHANISM(PKCS11_CKM_SHA224_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER, + ANY_PART), + MECHANISM(PKCS11_CKM_SHA256_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER, + ANY_PART), + MECHANISM(PKCS11_CKM_SHA384_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER, + ANY_PART), + MECHANISM(PKCS11_CKM_SHA512_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER, + ANY_PART), + /* EC */ + MECHANISM(PKCS11_CKM_EC_KEY_PAIR_GEN, PKCS11_CKFM_GENERATE_KEY_PAIR, + ANY_PART), + MECHANISM(PKCS11_CKM_ECDH1_DERIVE, PKCS11_CKFM_DERIVE, + ANY_PART), + MECHANISM(PKCS11_CKM_ECDSA, CKFM_AUTH_NO_RECOVER, SINGLE_PART_ONLY), + MECHANISM(PKCS11_CKM_ECDSA_SHA1, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_ECDSA_SHA224, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_ECDSA_SHA256, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_ECDSA_SHA384, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_ECDSA_SHA512, CKFM_AUTH_NO_RECOVER, ANY_PART), + /* EDDSA */ + MECHANISM(PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN, + PKCS11_CKFM_GENERATE_KEY_PAIR, ANY_PART), + MECHANISM(PKCS11_CKM_EDDSA, CKFM_AUTH_NO_RECOVER, ANY_PART), + /* RSA */ + MECHANISM(PKCS11_CKM_RSA_AES_KEY_WRAP, CKFM_CIPHER_WRAP, + SINGLE_PART_ONLY), + MECHANISM(PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN, + PKCS11_CKFM_GENERATE_KEY_PAIR, ANY_PART), + MECHANISM(PKCS11_CKM_RSA_PKCS, CKFM_CIPHER_WRAP | CKFM_AUTH_NO_RECOVER | + CKFM_AUTH_WITH_RECOVER, SINGLE_PART_ONLY), + MECHANISM(PKCS11_CKM_RSA_PKCS_OAEP, CKFM_CIPHER_WRAP, + SINGLE_PART_ONLY), + MECHANISM(PKCS11_CKM_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER, + SINGLE_PART_ONLY), + MECHANISM(PKCS11_CKM_MD5_RSA_PKCS, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA1_RSA_PKCS, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA1_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA256_RSA_PKCS, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA384_RSA_PKCS, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA512_RSA_PKCS, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA256_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER, + ANY_PART), + MECHANISM(PKCS11_CKM_SHA384_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER, + ANY_PART), + MECHANISM(PKCS11_CKM_SHA512_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER, + ANY_PART), + MECHANISM(PKCS11_CKM_SHA224_RSA_PKCS, CKFM_AUTH_NO_RECOVER, ANY_PART), + MECHANISM(PKCS11_CKM_SHA224_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER, + ANY_PART), +}; + +#if CFG_TEE_TA_LOG_LEVEL > 0 +const char *mechanism_string_id(enum pkcs11_mechanism_id id) +{ + const size_t offset = sizeof("PKCS11_CKM_") - 1; + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(pkcs11_modes); n++) + if (pkcs11_modes[n].id == id) + return pkcs11_modes[n].string + offset; + + return "Unknown ID"; +} +#endif /*CFG_TEE_TA_LOG_LEVEL*/ + +/* + * Return true if @id is a valid mechanism ID + */ +bool mechanism_is_valid(enum pkcs11_mechanism_id id) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(pkcs11_modes); n++) + if (id == pkcs11_modes[n].id) + return true; + + return false; +} + +/* + * Return true if mechanism ID is valid and flags matches PKCS#11 compliancy + */ +bool __maybe_unused mechanism_flags_complies_pkcs11(uint32_t mechanism_type, + uint32_t flags) +{ + size_t n = 0; + + assert((flags & ~ALLOWED_PKCS11_CKFM) == 0); + + for (n = 0; n < ARRAY_SIZE(pkcs11_modes); n++) { + if (pkcs11_modes[n].id == mechanism_type) { + if (flags & ~pkcs11_modes[n].flags) + EMSG("%s flags: 0x%"PRIx32" vs 0x%"PRIx32, + id2str_mechanism(mechanism_type), + flags, pkcs11_modes[n].flags); + + return (flags & ~pkcs11_modes[n].flags) == 0; + } + } + + /* Mechanism ID unexpectedly not found */ + return false; +} + +bool mechanism_is_one_shot_only(uint32_t mechanism_type) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(pkcs11_modes); n++) + if (pkcs11_modes[n].id == mechanism_type) + return pkcs11_modes[n].one_shot; + + /* Mechanism ID unexpectedly not found */ + TEE_Panic(PKCS11_RV_NOT_FOUND); + /* Dummy return to keep compiler happy */ + return false; +} + +/* + * Field single_part_only is unused from array token_mechanism[], hence + * simply use ANY_PART for all mechanism there. + */ +#define TA_MECHANISM(_label, _flags) MECHANISM((_label), (_flags), ANY_PART) + +/* + * Arrays that centralizes the IDs and processing flags for mechanisms + * supported by each embedded token. + */ +const struct pkcs11_mechachism_modes token_mechanism[] = { + TA_MECHANISM(PKCS11_CKM_AES_ECB, CKFM_CIPHER_WRAP), + TA_MECHANISM(PKCS11_CKM_AES_CBC, CKFM_CIPHER_WRAP), + TA_MECHANISM(PKCS11_CKM_AES_CTR, CKFM_CIPHER), + TA_MECHANISM(PKCS11_CKM_AES_CTS, CKFM_CIPHER), + TA_MECHANISM(PKCS11_CKM_AES_CMAC, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_AES_CMAC_GENERAL, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_AES_ECB_ENCRYPT_DATA, PKCS11_CKFM_DERIVE), + TA_MECHANISM(PKCS11_CKM_AES_CBC_ENCRYPT_DATA, PKCS11_CKFM_DERIVE), + TA_MECHANISM(PKCS11_CKM_ECDH1_DERIVE, PKCS11_CKFM_DERIVE), + TA_MECHANISM(PKCS11_CKM_AES_KEY_GEN, PKCS11_CKFM_GENERATE), + TA_MECHANISM(PKCS11_CKM_GENERIC_SECRET_KEY_GEN, PKCS11_CKFM_GENERATE), + TA_MECHANISM(PKCS11_CKM_MD5, PKCS11_CKFM_DIGEST), + TA_MECHANISM(PKCS11_CKM_SHA_1, PKCS11_CKFM_DIGEST), + TA_MECHANISM(PKCS11_CKM_SHA224, PKCS11_CKFM_DIGEST), + TA_MECHANISM(PKCS11_CKM_SHA256, PKCS11_CKFM_DIGEST), + TA_MECHANISM(PKCS11_CKM_SHA384, PKCS11_CKFM_DIGEST), + TA_MECHANISM(PKCS11_CKM_SHA512, PKCS11_CKFM_DIGEST), + TA_MECHANISM(PKCS11_CKM_MD5_HMAC, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA_1_HMAC, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA224_HMAC, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA256_HMAC, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA384_HMAC, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA512_HMAC, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_MD5_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA_1_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA224_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA256_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA384_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA512_HMAC_GENERAL, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_EC_KEY_PAIR_GEN, + PKCS11_CKFM_GENERATE_KEY_PAIR), + TA_MECHANISM(PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN, + PKCS11_CKFM_GENERATE_KEY_PAIR), + TA_MECHANISM(PKCS11_CKM_ECDSA, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_ECDSA_SHA1, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_ECDSA_SHA224, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_ECDSA_SHA256, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_ECDSA_SHA384, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_ECDSA_SHA512, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_EDDSA, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_RSA_AES_KEY_WRAP, CKFM_CIPHER_WRAP), + TA_MECHANISM(PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN, + PKCS11_CKFM_GENERATE_KEY_PAIR), + TA_MECHANISM(PKCS11_CKM_RSA_PKCS, CKFM_CIPHER | CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_MD5_RSA_PKCS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA1_RSA_PKCS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_RSA_PKCS_OAEP, CKFM_CIPHER), + TA_MECHANISM(PKCS11_CKM_SHA1_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA256_RSA_PKCS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA384_RSA_PKCS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA512_RSA_PKCS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA256_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA384_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA512_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA224_RSA_PKCS, CKFM_AUTH_NO_RECOVER), + TA_MECHANISM(PKCS11_CKM_SHA224_RSA_PKCS_PSS, CKFM_AUTH_NO_RECOVER), +}; + +/* + * tee_malloc_mechanism_array - Allocate and fill array of supported mechanisms + * @count: [in] [out] Pointer to number of mechanism IDs in client resource + * Return allocated array of the supported mechanism IDs + * + * Allocates array with 32bit cells mechanism IDs for the supported ones only + * if *@count covers number mechanism IDs exposed. + */ +uint32_t *tee_malloc_mechanism_list(size_t *out_count) +{ + size_t n = 0; + size_t count = 0; + uint32_t *array = NULL; + + for (n = 0; n < ARRAY_SIZE(token_mechanism); n++) + if (token_mechanism[n].flags) + count++; + + if (*out_count >= count) + array = TEE_Malloc(count * sizeof(*array), + TEE_USER_MEM_HINT_NO_FILL_ZERO); + + *out_count = count; + + if (!array) + return NULL; + + for (n = 0; n < ARRAY_SIZE(token_mechanism); n++) { + if (token_mechanism[n].flags) { + count--; + array[count] = token_mechanism[n].id; + } + } + assert(!count); + + return array; +} + +uint32_t mechanism_supported_flags(enum pkcs11_mechanism_id id) +{ + size_t n = 0; + + for (n = 0; n < ARRAY_SIZE(token_mechanism); n++) { + if (id == token_mechanism[n].id) { + uint32_t flags = token_mechanism[n].flags; + + assert(mechanism_flags_complies_pkcs11(id, flags)); + return flags; + } + } + + return 0; +} + +void pkcs11_mechanism_supported_key_sizes(uint32_t proc_id, + uint32_t *min_key_size, + uint32_t *max_key_size) +{ + switch (proc_id) { + case PKCS11_CKM_GENERIC_SECRET_KEY_GEN: + /* This mechanism expects the keysize to be returned in bits */ + *min_key_size = 1; /* in bits */ + *max_key_size = 4096; /* in bits */ + break; + case PKCS11_CKM_MD5_HMAC: + case PKCS11_CKM_MD5_HMAC_GENERAL: + *min_key_size = 8; + *max_key_size = 64; + break; + case PKCS11_CKM_SHA_1_HMAC: + case PKCS11_CKM_SHA_1_HMAC_GENERAL: + *min_key_size = 10; + *max_key_size = 64; + break; + case PKCS11_CKM_SHA224_HMAC: + case PKCS11_CKM_SHA224_HMAC_GENERAL: + *min_key_size = 14; + *max_key_size = 64; + break; + case PKCS11_CKM_SHA256_HMAC: + case PKCS11_CKM_SHA256_HMAC_GENERAL: + *min_key_size = 24; + *max_key_size = 128; + break; + case PKCS11_CKM_SHA384_HMAC: + case PKCS11_CKM_SHA384_HMAC_GENERAL: + *min_key_size = 32; + *max_key_size = 128; + break; + case PKCS11_CKM_SHA512_HMAC: + case PKCS11_CKM_SHA512_HMAC_GENERAL: + *min_key_size = 32; + *max_key_size = 128; + break; + case PKCS11_CKM_AES_KEY_GEN: + case PKCS11_CKM_AES_ECB: + case PKCS11_CKM_AES_CBC: + case PKCS11_CKM_AES_CBC_PAD: + case PKCS11_CKM_AES_CTR: + case PKCS11_CKM_AES_CTS: + case PKCS11_CKM_AES_CMAC: + case PKCS11_CKM_AES_CMAC_GENERAL: + *min_key_size = 16; + *max_key_size = 32; + break; + case PKCS11_CKM_EC_KEY_PAIR_GEN: + case PKCS11_CKM_ECDSA: + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + case PKCS11_CKM_ECDH1_DERIVE: + *min_key_size = 160; /* in bits */ + *max_key_size = 521; /* in bits */ + break; + case PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN: + case PKCS11_CKM_EDDSA: + *min_key_size = 256; /* in bits */ + *max_key_size = 448; /* in bits */ + break; + case PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN: + case PKCS11_CKM_RSA_PKCS: + case PKCS11_CKM_MD5_RSA_PKCS: + case PKCS11_CKM_SHA1_RSA_PKCS: + case PKCS11_CKM_RSA_PKCS_OAEP: + case PKCS11_CKM_SHA1_RSA_PKCS_PSS: + case PKCS11_CKM_SHA256_RSA_PKCS: + case PKCS11_CKM_SHA384_RSA_PKCS: + case PKCS11_CKM_SHA512_RSA_PKCS: + case PKCS11_CKM_SHA256_RSA_PKCS_PSS: + case PKCS11_CKM_SHA384_RSA_PKCS_PSS: + case PKCS11_CKM_SHA512_RSA_PKCS_PSS: + case PKCS11_CKM_SHA224_RSA_PKCS: + case PKCS11_CKM_SHA224_RSA_PKCS_PSS: + *min_key_size = 256; /* in bits */ + *max_key_size = 4096; /* in bits */ + break; + default: + *min_key_size = 0; + *max_key_size = 0; + break; + } +} + +void mechanism_supported_key_sizes_bytes(uint32_t proc_id, + uint32_t *min_key_size, + uint32_t *max_key_size) +{ + pkcs11_mechanism_supported_key_sizes(proc_id, min_key_size, + max_key_size); + + switch (proc_id) { + case PKCS11_CKM_GENERIC_SECRET_KEY_GEN: + case PKCS11_CKM_EC_EDWARDS_KEY_PAIR_GEN: + case PKCS11_CKM_EC_KEY_PAIR_GEN: + case PKCS11_CKM_ECDSA: + case PKCS11_CKM_EDDSA: + case PKCS11_CKM_ECDSA_SHA1: + case PKCS11_CKM_ECDSA_SHA224: + case PKCS11_CKM_ECDSA_SHA256: + case PKCS11_CKM_ECDSA_SHA384: + case PKCS11_CKM_ECDSA_SHA512: + case PKCS11_CKM_RSA_PKCS_KEY_PAIR_GEN: + /* Size is in bits -> convert to bytes and ceil */ + *min_key_size = ROUNDUP(*min_key_size, 8) / 8; + *max_key_size = ROUNDUP(*max_key_size, 8) / 8; + break; + default: + /* Size is already in bytes */ + break; + } +} diff --git a/optee_os/ta/pkcs11/src/token_capabilities.h b/optee_os/ta/pkcs11/src/token_capabilities.h new file mode 100644 index 0000000..7614006 --- /dev/null +++ b/optee_os/ta/pkcs11/src/token_capabilities.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2017-2020, Linaro Limited + */ + +#ifndef TOKEN_CAPABILITIES_H +#define TOKEN_CAPABILITIES_H + +#include +#include +#include + +bool mechanism_flags_complies_pkcs11(uint32_t mechanism_type, uint32_t flags); + +bool mechanism_is_one_shot_only(uint32_t mechanism_type); + +bool mechanism_is_valid(enum pkcs11_mechanism_id id); + +#if CFG_TEE_TA_LOG_LEVEL > 0 +const char *mechanism_string_id(enum pkcs11_mechanism_id id); +#endif + +uint32_t *tee_malloc_mechanism_list(size_t *out_count); + +uint32_t mechanism_supported_flags(enum pkcs11_mechanism_id id); + +void pkcs11_mechanism_supported_key_sizes(uint32_t proc_id, + uint32_t *min_key_size, + uint32_t *max_key_size); + +void mechanism_supported_key_sizes_bytes(uint32_t proc_id, + uint32_t *min_key_size, + uint32_t *max_key_size); + +static inline bool mechanism_is_supported(enum pkcs11_mechanism_id id) +{ + return mechanism_supported_flags(id) != 0; +} + +#endif /*TOKEN_CAPABILITIES_H*/ diff --git a/optee_os/ta/pkcs11/src/user_ta_header_defines.h b/optee_os/ta/pkcs11/src/user_ta_header_defines.h new file mode 100644 index 0000000..4d468ca --- /dev/null +++ b/optee_os/ta/pkcs11/src/user_ta_header_defines.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2018-2019, Linaro Limited + */ + +#ifndef USER_TA_HEADER_DEFINES_H +#define USER_TA_HEADER_DEFINES_H + +#include + +#define TA_UUID PKCS11_TA_UUID + +#define TA_FLAGS (TA_FLAG_SINGLE_INSTANCE | \ + TA_FLAG_MULTI_SESSION | \ + TA_FLAG_INSTANCE_KEEP_ALIVE) + +#define TA_STACK_SIZE (4 * 1024) + +#define TA_DATA_SIZE CFG_PKCS11_TA_HEAP_SIZE + +#define TA_DESCRIPTION "PKCS#11 trusted application" +#define TA_VERSION TO_STR(PKCS11_TA_VERSION_MAJOR) "." \ + TO_STR(PKCS11_TA_VERSION_MINOR) "." \ + TO_STR(PKCS11_TA_VERSION_PATCH) + +#endif /*USER_TA_HEADER_DEFINES_H*/ diff --git a/optee_os/ta/pkcs11/sub.mk b/optee_os/ta/pkcs11/sub.mk new file mode 100644 index 0000000..30dd13c --- /dev/null +++ b/optee_os/ta/pkcs11/sub.mk @@ -0,0 +1,15 @@ +# Enable PKCS#11 TA's C_DigestKey support +CFG_PKCS11_TA_ALLOW_DIGEST_KEY ?= y + +# Enable PKCS#11 TA's TEE Identity based authentication support +CFG_PKCS11_TA_AUTH_TEE_IDENTITY ?= y + +# PKCS#11 TA heap size can be customized if 32kB is not enough +CFG_PKCS11_TA_HEAP_SIZE ?= (32 * 1024) + +# Defines the number of PKCS11 token implemented by the PKCS11 TA +CFG_PKCS11_TA_TOKEN_COUNT ?= 3 + +global-incdirs-y += include +global-incdirs-y += src +subdirs-y += src diff --git a/optee_os/ta/pkcs11/user_ta.mk b/optee_os/ta/pkcs11/user_ta.mk new file mode 100644 index 0000000..dda50cf --- /dev/null +++ b/optee_os/ta/pkcs11/user_ta.mk @@ -0,0 +1,8 @@ +user-ta-uuid := fd02c9da-306c-48c7-a49c-bbd827ae86ee + +all: pkcs11-ta-verify-helpers + +.PHONY: pkcs11-ta-verify-helpers +pkcs11-ta-verify-helpers: + @$(cmd-echo-silent) ' CHK ' $@ + ${q}ta/pkcs11/scripts/verify-helpers.sh --quiet diff --git a/optee_os/ta/ta.mk b/optee_os/ta/ta.mk new file mode 100644 index 0000000..09c20bf --- /dev/null +++ b/optee_os/ta/ta.mk @@ -0,0 +1,217 @@ +include mk/cleanvars.mk + +# Set current submodule (used for module specific flags compile result etc) +sm := $(ta-target) +sm-$(sm) := y + +# Setup compiler for this sub module +COMPILER_$(sm) ?= $(COMPILER) +include mk/$(COMPILER_$(sm)).mk + +# +# Config flags from mk/config.mk +# + +ta-stackp-cflags-$(CFG_TA_STACK_PROTECTOR) := -fstack-protector +ta-stackp-cflags-$(CFG_TA_STACK_PROTECTOR_STRONG) := -fstack-protector-strong +ta-stackp-cflags-$(CFG_TA_STACK_PROTECTOR_ALL) := -fstack-protector-all +$(sm)-platform-cflags += $(ta-stackp-cflags-y) + +ifeq ($(CFG_TA_MBEDTLS_SELF_TEST),y) +$(sm)-platform-cppflags += -DMBEDTLS_SELF_TEST +endif + +ifeq ($(CFG_TEE_TA_MALLOC_DEBUG),y) +# Build malloc debug code into libutils: (mdbg_malloc(), mdbg_free(), +# mdbg_check(), etc.). +$(sm)-platform-cppflags += -DENABLE_MDBG=1 +endif + +# Config variables to be explicitly exported to the dev kit conf.mk +ta-mk-file-export-vars-$(sm) += CFG_TA_FLOAT_SUPPORT +ta-mk-file-export-vars-$(sm) += CFG_CACHE_API +ta-mk-file-export-vars-$(sm) += CFG_SECURE_DATA_PATH +ta-mk-file-export-vars-$(sm) += CFG_TA_MBEDTLS_SELF_TEST +ta-mk-file-export-vars-$(sm) += CFG_TA_MBEDTLS +ta-mk-file-export-vars-$(sm) += CFG_TA_MBEDTLS_MPI +ta-mk-file-export-vars-$(sm) += CFG_SYSTEM_PTA +ta-mk-file-export-vars-$(sm) += CFG_FTRACE_SUPPORT +ta-mk-file-export-vars-$(sm) += CFG_UNWIND +ta-mk-file-export-vars-$(sm) += CFG_TA_MCOUNT +ta-mk-file-export-vars-$(sm) += CFG_TA_BTI +ta-mk-file-export-vars-$(sm) += CFG_TA_PAUTH +ta-mk-file-export-vars-$(sm) += CFG_CORE_TPM_EVENT_LOG +ta-mk-file-export-add-$(sm) += CFG_TEE_TA_LOG_LEVEL ?= $(CFG_TEE_TA_LOG_LEVEL)_nl_ +ta-mk-file-export-vars-$(sm) += CFG_TA_BGET_TEST +ta-mk-file-export-vars-$(sm) += CFG_ATTESTATION_PTA +ta-mk-file-export-vars-$(sm) += CFG_MEMTAG + +# Expand platform flags here as $(sm) will change if we have several TA +# targets. Platform flags should not change after inclusion of ta/ta.mk. +cppflags$(sm) := $(platform-cppflags) $($(sm)-platform-cppflags) +cflags$(sm) := $(platform-cflags) $($(sm)-platform-cflags) +aflags$(sm) := $(platform-aflags) $($(sm)-platform-aflags) + +# Changes to cppflags$(sm) will only affect how TA dev kit libraries are +# compiled, these flags are not propagated to the TA +cppflags$(sm) += -include $(conf-file) +cppflags$(sm) += -DTRACE_LEVEL=$(CFG_TEE_TA_LOG_LEVEL) + +ifeq ($(ta-target),ta_arm32) +arm32-user-sysreg-txt = lib/libutee/arch/arm/arm32_user_sysreg.txt +arm32-user-sysregs-$(arm32-user-sysreg-txt)-h := arm32_user_sysreg.h +arm32-user-sysregs += $(arm32-user-sysreg-txt) + +arm32-user-sysregs-out := $(out-dir)/include/generated + +define process-arm32-user-sysreg +FORCE-GENSRC$(sm): $$(arm32-user-sysregs-out)/$$(arm32-user-sysregs-$(1)-h) +cleanfiles := $$(cleanfiles) \ + $$(arm32-user-sysregs-out)/$$(arm32-user-sysregs-$(1)-h) + +$$(arm32-user-sysregs-out)/$$(arm32-user-sysregs-$(1)-h): \ + $(1) scripts/arm32_sysreg.py + @$(cmd-echo-silent) ' GEN $$@' + $(q)mkdir -p $$(dir $$@) + $(q)$(PYTHON3) scripts/arm32_sysreg.py --guard __$$(arm32-user-sysregs-$(1)-h) \ + < $$< > $$@ + +endef #process-arm32-user-sysreg + +$(foreach sr, $(arm32-user-sysregs), \ + $(eval $(call process-arm32-user-sysreg,$(sr)))) + +cppflags$(sm) += -I$(arm32-user-sysregs-out) +endif + +base-prefix := $(sm)- + +libname = utils +libdir = lib/libutils +libuuid = 71855bba-6055-4293-a63f-b0963a737360 +include mk/lib.mk + +libname = mbedtls +libdir = lib/libmbedtls +libuuid = 87bb6ae8-4b1d-49fe-9986-2b966132c309 +libl = utils +include mk/lib.mk +ta-mk-file-export-vars-$(sm) += CFG_TA_MBEDTLS + +libname = utee +libdir = lib/libutee +libuuid = 4b3d937e-d57e-418b-8673-1c04f2420226 +libl = mbedtls utils +include mk/lib.mk + +libname = dl +libdir = lib/libdl +libuuid = be807bbd-81e1-4dc4-bd99-3d363f240ece +libl = utee utils +include mk/lib.mk + +base-prefix := + +incdirs-host := $(filter-out lib/libutils%, $(incdirs$(sm))) +incfiles-extra-host := lib/libutils/ext/include/compiler.h +incfiles-extra-host += lib/libutils/ext/include/util.h +incfiles-extra-host += lib/libutils/ext/include/types_ext.h +incfiles-extra-host += $(conf-file) +incfiles-extra-host += $(conf-mk-file) +incfiles-extra-host += $(conf-cmake-file) +incfiles-extra-host += core/include/tee/tee_fs_key_manager.h +incfiles-extra-host += core/include/tee/fs_htree.h +incfiles-extra-host += core/include/signed_hdr.h +ifeq ($(ta-target),ta_arm32) +incfiles-extra-host += $(out-dir)/include/generated/arm32_user_sysreg.h +endif +# +# Copy lib files and exported headers from each lib +# + +define copy-file +$2/$$(notdir $1): $1 + @set -e; \ + mkdir -p $$(dir $$@) ; \ + $(cmd-echo-silent) ' INSTALL $$@' ; \ + cp -P $$< $$@ + +cleanfiles += $2/$$(notdir $1) +ta_dev_kit: $2/$$(notdir $1) +ta_dev_kit-files += $2/$$(notdir $1) +ta_dev_kit-files-$3 += $2/$$(notdir $1) +endef + +# Copy the .a files +$(foreach f, $(libfiles), \ + $(eval $(call copy-file, $(f), $(out-dir)/export-$(sm)/lib,lib))) + +# Copy .mk files +ta-mkfiles = mk/compile.mk mk/subdir.mk mk/gcc.mk mk/clang.mk mk/cleandirs.mk \ + mk/cc-option.mk mk/macros.mk \ + ta/link.mk ta/link_shlib.mk \ + ta/mk/ta_dev_kit.mk + +$(foreach f, $(ta-mkfiles), \ + $(eval $(call copy-file, $(f), $(out-dir)/export-$(sm)/mk))) + +# Copy the .h files for TAs +define copy-incdir +sf := $(subst $1/, , $(shell find $1 -name "*.[hS]")) +$$(foreach h, $$(sf), $$(eval $$(call copy-file, $1/$$(h), \ + $$(patsubst %/,%,$$(subst /./,/,$2/$$(dir $$(h)))),$3))) +endef +$(foreach d, $(incdirs$(sm)), \ + $(eval $(call copy-incdir,$(d),$(out-dir)/export-$(sm)/include,include))) + +# Copy the .h files needed by host +$(foreach d, $(incdirs-host), \ + $(eval $(call copy-incdir, $(d), $(out-dir)/export-$(sm)/host_include))) +$(foreach f, $(incfiles-extra-host), \ + $(eval $(call copy-file, $(f), $(out-dir)/export-$(sm)/host_include))) + +# Copy the src files +ta-srcfiles = ta/user_ta_header.c ta/arch/$(ARCH)/ta.ld.S +ifeq ($(ta-target),ta_arm32) +ta-srcfiles += ta/arch/$(ARCH)/ta_entry_a32.S +endif +$(foreach f, $(ta-srcfiles), \ + $(eval $(call copy-file, $(f), $(out-dir)/export-$(sm)/src))) + +# Copy keys +ta-keys := $(TA_SIGN_KEY) +# default_ta.pem is a symlink to default.pem, for backwards compatibility. +# If default_ta.pem is used, copy both files. +ifeq ($(TA_SIGN_KEY),keys/default_ta.pem) +ta-keys += keys/default.pem +endif + +$(foreach f, $(ta-keys), \ + $(eval $(call copy-file, $(f), $(out-dir)/export-$(sm)/keys))) + +# Copy the scripts +ta-scripts = scripts/sign_encrypt.py scripts/symbolize.py scripts/sign_rproc_fw.py +$(foreach f, $(ta-scripts), \ + $(eval $(call copy-file, $(f), $(out-dir)/export-$(sm)/scripts))) + +# Create config file +conf-mk-file-export := $(out-dir)/export-$(sm)/mk/conf.mk +sm-$(conf-mk-file-export) := $(sm) +define mk-file-export +.PHONY: $(conf-mk-file-export) +$(conf-mk-file-export): + @$$(cmd-echo-silent) ' CHK ' $$@ + $(q)mkdir -p $$(dir $$@) + $(q)echo sm := $$(sm-$(conf-mk-file-export)) > $$@.tmp + $(q)echo sm-$$(sm-$(conf-mk-file-export)) := y >> $$@.tmp + $(q)($$(foreach v, $$(ta-mk-file-export-vars-$$(sm-$(conf-mk-file-export))), \ + $$(if $$($$(v)),echo $$(v) := $$($$(v));,))) >> $$@.tmp + $(q)echo '$$(ta-mk-file-export-add-$$(sm-$(conf-mk-file-export)))' | sed 's/_nl_ */\n/g' >> $$@.tmp + $(q)$(call mv-if-changed,$$@.tmp,$$@) +endef +$(eval $(mk-file-export)) + +cleanfiles := $(cleanfiles) $(conf-mk-file-export) +ta_dev_kit: $(conf-mk-file-export) + +all: ta_dev_kit diff --git a/optee_os/ta/trusted_keys/entry.c b/optee_os/ta/trusted_keys/entry.c new file mode 100644 index 0000000..a806e72 --- /dev/null +++ b/optee_os/ta/trusted_keys/entry.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (C) 2019-2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define IV_SIZE 16 +#define TAG_SIZE 16 +#define MAX_BUF_SIZE 512 + +/* + * Acronym: + * + * TK - Trusted Key + */ + +struct tk_blob_hdr { + uint8_t reserved; + uint8_t iv[IV_SIZE]; + uint8_t tag[TAG_SIZE]; + uint8_t enc_key[]; +}; + +static TEE_Result get_random(uint32_t types, TEE_Param params[TEE_NUM_PARAMS]) +{ + uint8_t *rng_buf = NULL; + + DMSG("Invoked TA_CMD_GET_RANDOM"); + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + if (!params[0].memref.buffer || !params[0].memref.size) + return TEE_ERROR_BAD_PARAMETERS; + + rng_buf = TEE_Malloc(params[0].memref.size, TEE_MALLOC_FILL_ZERO); + if (!rng_buf) + return TEE_ERROR_OUT_OF_MEMORY; + + TEE_GenerateRandom(rng_buf, params[0].memref.size); + memcpy(params[0].memref.buffer, rng_buf, params[0].memref.size); + memzero_explicit(rng_buf, params[0].memref.size); + + TEE_Free(rng_buf); + + return TEE_SUCCESS; +} + +static TEE_Result derive_unique_key(uint8_t *key, uint16_t key_size, + uint8_t *extra, uint16_t extra_size) +{ + TEE_TASessionHandle sess = TEE_HANDLE_NULL; + TEE_Param params[TEE_NUM_PARAMS] = { }; + TEE_Result res = TEE_ERROR_GENERIC; + uint32_t ret_orig = 0; + uint32_t param_types = TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE); + + res = TEE_OpenTASession(&(const TEE_UUID)PTA_SYSTEM_UUID, + TEE_TIMEOUT_INFINITE, 0, NULL, &sess, + &ret_orig); + if (res) + return res; + + if (extra && extra_size) { + params[0].memref.buffer = extra; + params[0].memref.size = extra_size; + } + + params[1].memref.buffer = key; + params[1].memref.size = key_size; + + res = TEE_InvokeTACommand(sess, TEE_TIMEOUT_INFINITE, + PTA_SYSTEM_DERIVE_TA_UNIQUE_KEY, + param_types, params, &ret_orig); + + TEE_CloseTASession(sess); + + return res; +} + +static TEE_Result huk_ae_encrypt(TEE_OperationHandle crypto_op, uint8_t *in, + size_t in_sz, uint8_t *out, size_t *out_sz) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct tk_blob_hdr *hdr = (struct tk_blob_hdr *)out; + uint8_t iv[IV_SIZE] = { 0 }; + size_t enc_key_len = in_sz; + size_t tag_len = TAG_SIZE; + + hdr->reserved = 0; + TEE_GenerateRandom(iv, IV_SIZE); + memcpy(hdr->iv, iv, IV_SIZE); + + res = TEE_AEInit(crypto_op, hdr->iv, IV_SIZE, TAG_SIZE * 8, 0, 0); + if (res) + return res; + + res = TEE_AEEncryptFinal(crypto_op, in, in_sz, hdr->enc_key, + &enc_key_len, hdr->tag, &tag_len); + if (res || tag_len != TAG_SIZE) + return TEE_ERROR_SECURITY; + + if (ADD_OVERFLOW(enc_key_len, sizeof(*hdr), out_sz)) + return TEE_ERROR_SECURITY; + + return res; +} + +static TEE_Result huk_ae_decrypt(TEE_OperationHandle crypto_op, uint8_t *in, + size_t in_sz, uint8_t *out, size_t *out_sz) +{ + TEE_Result res = TEE_ERROR_GENERIC; + struct tk_blob_hdr *hdr = (struct tk_blob_hdr *)in; + uint8_t tag[TAG_SIZE] = { 0 }; + size_t enc_key_len = 0; + + if (SUB_OVERFLOW(in_sz, sizeof(*hdr), &enc_key_len)) + return TEE_ERROR_SECURITY; + + res = TEE_AEInit(crypto_op, hdr->iv, IV_SIZE, TAG_SIZE * 8, 0, 0); + if (res) + return res; + + memcpy(tag, hdr->tag, TAG_SIZE); + res = TEE_AEDecryptFinal(crypto_op, hdr->enc_key, enc_key_len, out, + out_sz, tag, TAG_SIZE); + if (res) + res = TEE_ERROR_SECURITY; + + return res; +} + +static TEE_Result huk_crypt(TEE_OperationMode mode, uint8_t *in, size_t in_sz, + uint8_t *out, size_t *out_sz) +{ + TEE_Result res = TEE_ERROR_GENERIC; + TEE_OperationHandle crypto_op = TEE_HANDLE_NULL; + TEE_ObjectHandle hkey = TEE_HANDLE_NULL; + uint8_t huk_key[TA_DERIVED_KEY_MAX_SIZE] = { }; + TEE_Attribute attr = { }; + + res = TEE_AllocateOperation(&crypto_op, TEE_ALG_AES_GCM, mode, + sizeof(huk_key) * 8); + if (res) + return res; + + res = derive_unique_key(huk_key, sizeof(huk_key), NULL, 0); + if (res) { + EMSG("derive_unique_key failed: returned %#"PRIx32, res); + goto out_op; + } + + res = TEE_AllocateTransientObject(TEE_TYPE_AES, sizeof(huk_key) * 8, + &hkey); + if (res) + goto out_op; + + attr.attributeID = TEE_ATTR_SECRET_VALUE; + attr.content.ref.buffer = huk_key; + attr.content.ref.length = sizeof(huk_key); + + res = TEE_PopulateTransientObject(hkey, &attr, 1); + if (res) + goto out_key; + + res = TEE_SetOperationKey(crypto_op, hkey); + if (res) + goto out_key; + + if (mode == TEE_MODE_ENCRYPT) { + res = huk_ae_encrypt(crypto_op, in, in_sz, out, out_sz); + if (res) + EMSG("huk_AE_encrypt failed: returned %#"PRIx32, res); + } else if (mode == TEE_MODE_DECRYPT) { + res = huk_ae_decrypt(crypto_op, in, in_sz, out, out_sz); + if (res) + EMSG("huk_AE_decrypt failed: returned %#"PRIx32, res); + } else { + TEE_Panic(0); + } + +out_key: + TEE_FreeTransientObject(hkey); +out_op: + TEE_FreeOperation(crypto_op); + memzero_explicit(huk_key, sizeof(huk_key)); + return res; +} + +static TEE_Result seal_trusted_key(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *in = NULL; + size_t in_sz = 0; + uint8_t *out = NULL; + size_t out_sz = 0; + + DMSG("Invoked TA_CMD_SEAL"); + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + in = params[0].memref.buffer; + in_sz = params[0].memref.size; + out = params[1].memref.buffer; + out_sz = params[1].memref.size; + + if (!in || !in_sz || in_sz > MAX_BUF_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + if ((!out && out_sz) || + (out && !IS_ALIGNED_WITH_TYPE(out, struct tk_blob_hdr)) || + out_sz > MAX_BUF_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + if ((in_sz + sizeof(struct tk_blob_hdr)) > out_sz) { + params[1].memref.size = in_sz + sizeof(struct tk_blob_hdr); + return TEE_ERROR_SHORT_BUFFER; + } + + res = huk_crypt(TEE_MODE_ENCRYPT, in, in_sz, out, &out_sz); + if (res == TEE_SUCCESS) { + assert(out_sz == in_sz + sizeof(struct tk_blob_hdr)); + params[1].memref.size = out_sz; + } + + return res; +} + +static TEE_Result unseal_trusted_key(uint32_t types, + TEE_Param params[TEE_NUM_PARAMS]) +{ + TEE_Result res = TEE_SUCCESS; + uint8_t *in = NULL; + size_t in_sz = 0; + uint8_t *out = NULL; + size_t out_sz = 0; + + DMSG("Invoked TA_CMD_UNSEAL"); + + if (types != TEE_PARAM_TYPES(TEE_PARAM_TYPE_MEMREF_INPUT, + TEE_PARAM_TYPE_MEMREF_OUTPUT, + TEE_PARAM_TYPE_NONE, + TEE_PARAM_TYPE_NONE)) + return TEE_ERROR_BAD_PARAMETERS; + + in = params[0].memref.buffer; + in_sz = params[0].memref.size; + out = params[1].memref.buffer; + out_sz = params[1].memref.size; + + if (!in || !IS_ALIGNED_WITH_TYPE(in, struct tk_blob_hdr) || + in_sz <= sizeof(struct tk_blob_hdr) || in_sz > MAX_BUF_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + if ((!out && out_sz) || out_sz > MAX_BUF_SIZE) + return TEE_ERROR_BAD_PARAMETERS; + + if (in_sz > (out_sz + sizeof(struct tk_blob_hdr))) { + params[1].memref.size = in_sz - sizeof(struct tk_blob_hdr); + return TEE_ERROR_SHORT_BUFFER; + } + + res = huk_crypt(TEE_MODE_DECRYPT, in, in_sz, out, &out_sz); + if (res == TEE_SUCCESS) { + assert(out_sz == in_sz - sizeof(struct tk_blob_hdr)); + params[1].memref.size = out_sz; + } + + return res; +} + +TEE_Result TA_CreateEntryPoint(void) +{ + return TEE_SUCCESS; +} + +void TA_DestroyEntryPoint(void) +{ +} + +TEE_Result TA_OpenSessionEntryPoint(uint32_t pt __unused, + TEE_Param params[TEE_NUM_PARAMS] __unused, + void **session __unused) +{ + TEE_Result res = TEE_ERROR_GENERIC; + TEE_PropSetHandle h = TEE_HANDLE_NULL; + TEE_Identity id = { }; + + res = TEE_AllocatePropertyEnumerator(&h); + if (res) + goto out; + + TEE_StartPropertyEnumerator(h, TEE_PROPSET_CURRENT_CLIENT); + + res = TEE_GetPropertyAsIdentity(h, NULL, &id); + if (res) + goto out; + + if (id.login != TEE_LOGIN_REE_KERNEL) + res = TEE_ERROR_ACCESS_DENIED; + +out: + if (h) + TEE_FreePropertyEnumerator(h); + return res; +} + +void TA_CloseSessionEntryPoint(void *sess __unused) +{ +} + +TEE_Result TA_InvokeCommandEntryPoint(void *sess __unused, uint32_t cmd, + uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + switch (cmd) { + case TA_CMD_GET_RANDOM: + return get_random(pt, params); + case TA_CMD_SEAL: + return seal_trusted_key(pt, params); + case TA_CMD_UNSEAL: + return unseal_trusted_key(pt, params); + default: + EMSG("Command ID %#"PRIx32" is not supported", cmd); + return TEE_ERROR_NOT_SUPPORTED; + } +} diff --git a/optee_os/ta/trusted_keys/include/trusted_keys.h b/optee_os/ta/trusted_keys/include/trusted_keys.h new file mode 100644 index 0000000..1571b37 --- /dev/null +++ b/optee_os/ta/trusted_keys/include/trusted_keys.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019-2020, Linaro Limited + */ + +#ifndef TRUSTED_KEYS_H +#define TRUSTED_KEYS_H + +#define TRUSTED_KEYS_UUID { 0xf04a0fe7, 0x1f5d, 0x4b9b, \ + { 0xab, 0xf7, 0x61, 0x9b, 0x85, 0xb4, 0xce, 0x8c } } + +/* + * Get random data for symmetric key + * + * [out] memref[0] Random data + */ +#define TA_CMD_GET_RANDOM 0x0 + +/* + * Seal trusted key using hardware unique key + * + * [in] memref[0] Plain key + * [out] memref[1] Sealed key datablob + */ +#define TA_CMD_SEAL 0x1 + +/* + * Unseal trusted key using hardware unique key + * + * [in] memref[0] Sealed key datablob + * [out] memref[1] Plain key + */ +#define TA_CMD_UNSEAL 0x2 + +#endif /* TRUSTED_KEYS_H */ diff --git a/optee_os/ta/trusted_keys/sub.mk b/optee_os/ta/trusted_keys/sub.mk new file mode 100644 index 0000000..f1b35c0 --- /dev/null +++ b/optee_os/ta/trusted_keys/sub.mk @@ -0,0 +1,3 @@ +global-incdirs-y += include +global-incdirs-y += . +srcs-y += entry.c diff --git a/optee_os/ta/trusted_keys/user_ta.mk b/optee_os/ta/trusted_keys/user_ta.mk new file mode 100644 index 0000000..bcb59c0 --- /dev/null +++ b/optee_os/ta/trusted_keys/user_ta.mk @@ -0,0 +1 @@ +user-ta-uuid := f04a0fe7-1f5d-4b9b-abf7-619b85b4ce8c diff --git a/optee_os/ta/trusted_keys/user_ta_header_defines.h b/optee_os/ta/trusted_keys/user_ta_header_defines.h new file mode 100644 index 0000000..2fc6689 --- /dev/null +++ b/optee_os/ta/trusted_keys/user_ta_header_defines.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2020, Linaro Limited + */ + +#ifndef USER_TA_HEADER_DEFINES_H +#define USER_TA_HEADER_DEFINES_H + +#include + +#define TA_UUID TRUSTED_KEYS_UUID + +#define TA_FLAGS (TA_FLAG_SINGLE_INSTANCE | \ + TA_FLAG_MULTI_SESSION | \ + TA_FLAG_DEVICE_ENUM) + +#define TA_STACK_SIZE (4 * 1024) +#define TA_DATA_SIZE (16 * 1024) + +#endif /*USER_TA_HEADER_DEFINES_H*/ diff --git a/optee_os/ta/user_ta_header.c b/optee_os/ta/user_ta_header.c new file mode 100644 index 0000000..d602f92 --- /dev/null +++ b/optee_os/ta/user_ta_header.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2014, STMicroelectronics International N.V. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern void *__stack_chk_guard; + +int trace_level = TRACE_LEVEL; + +const char trace_ext_prefix[] = "TA"; + +#ifndef TA_VERSION +#define TA_VERSION "Undefined version" +#endif + +#ifndef TA_DESCRIPTION +#define TA_DESCRIPTION "Undefined description" +#endif + +/* exprted to user_ta_header.c, built within TA */ +struct utee_params; + +#ifdef ARM32 +#define _C_FUNCTION(name) name##_c +#else +#define _C_FUNCTION(name) name +#endif /* ARM32 */ + +/* From libutee */ +TEE_Result __utee_entry(unsigned long func, unsigned long session_id, + struct utee_params *up, unsigned long cmd_id); + +void __noreturn _C_FUNCTION(__ta_entry)(unsigned long func, + unsigned long session_id, + struct utee_params *up, + unsigned long cmd_id); + +void __noreturn _C_FUNCTION(__ta_entry)(unsigned long func, + unsigned long session_id, + struct utee_params *up, + unsigned long cmd_id) +{ + static bool stack_canary_inited; + TEE_Result res = TEE_ERROR_GENERIC; + + if (IS_ENABLED(_CFG_TA_STACK_PROTECTOR) && !stack_canary_inited) { + uintptr_t canary = 0; + + res = _utee_cryp_random_number_generate(&canary, + sizeof(canary)); + if (res != TEE_SUCCESS) + _utee_return(res); + + /* Leave null byte in canary to prevent string base exploit */ + canary &= ~0xffUL; + + __stack_chk_guard = (void *)canary; + stack_canary_inited = true; + } + + res = __utee_entry(func, session_id, up, cmd_id); + +#if defined(CFG_FTRACE_SUPPORT) + /* + * __ta_entry is the first TA API called from TEE core. As it being + * __noreturn API, we need to call ftrace_return in this API just + * before _utee_return syscall to get proper ftrace call graph. + */ + ftrace_return(); +#endif + + _utee_return(res); +} + +/* + * According to GP Internal API, TA_STACK_SIZE corresponds to the stack + * size used by the TA code itself and does not include stack space + * possibly used by the Trusted Core Framework. + * Hence, stack_size which is the size of the stack to use, + * must be enlarged + * It has been set to 2048 to include trace framework and invoke commands + */ +#define TA_FRAMEWORK_STACK_SIZE 2048 + +const struct ta_head ta_head __section(".ta_head") = { + /* UUID, unique to each TA */ + .uuid = TA_UUID, + /* + * According to GP Internal API, TA_FRAMEWORK_STACK_SIZE corresponds to + * the stack size used by the TA code itself and does not include stack + * space possibly used by the Trusted Core Framework. + * Hence, stack_size which is the size of the stack to use, + * must be enlarged + */ + .stack_size = TA_STACK_SIZE + TA_FRAMEWORK_STACK_SIZE, + .flags = TA_FLAGS, + /* + * The TA entry doesn't go via this field any longer, to be able to + * reliably check that an old TA isn't loaded set this field to a + * fixed value. + */ + .depr_entry = UINT64_MAX, +}; + +/* Keeping the heap in bss */ +#if TA_DATA_SIZE < MALLOC_INITIAL_POOL_MIN_SIZE +#error TA_DATA_SIZE too small +#endif + +uint8_t ta_heap[TA_DATA_SIZE]; +const size_t ta_heap_size = sizeof(ta_heap); + +#ifndef TA_NO_SHARE_DATA_SIZE +#define TA_NO_SHARE_DATA_SIZE 0 +#endif +#if TA_NO_SHARE_DATA_SIZE && \ + TA_NO_SHARE_DATA_SIZE < MALLOC_INITIAL_POOL_MIN_SIZE +#error TA_NO_SHARE_DATA_SIZE too small +#endif + +uint8_t __ta_no_share_heap[TA_NO_SHARE_DATA_SIZE]; +const size_t __ta_no_share_heap_size = sizeof(__ta_no_share_heap); + +const struct user_ta_property ta_props[] = { + {TA_PROP_STR_SINGLE_INSTANCE, USER_TA_PROP_TYPE_BOOL, + &(const bool){(TA_FLAGS & TA_FLAG_SINGLE_INSTANCE) != 0}}, + + {TA_PROP_STR_MULTI_SESSION, USER_TA_PROP_TYPE_BOOL, + &(const bool){(TA_FLAGS & TA_FLAG_MULTI_SESSION) != 0}}, + + {TA_PROP_STR_KEEP_ALIVE, USER_TA_PROP_TYPE_BOOL, + &(const bool){(TA_FLAGS & TA_FLAG_INSTANCE_KEEP_ALIVE) != 0}}, + + {TA_PROP_STR_DATA_SIZE, USER_TA_PROP_TYPE_U32, + &(const uint32_t){TA_DATA_SIZE}}, + + {TA_PROP_STR_STACK_SIZE, USER_TA_PROP_TYPE_U32, + &(const uint32_t){TA_STACK_SIZE}}, + + {TA_PROP_STR_VERSION, USER_TA_PROP_TYPE_STRING, + TA_VERSION}, + + {TA_PROP_STR_DESCRIPTION, USER_TA_PROP_TYPE_STRING, + TA_DESCRIPTION}, + + /* Only little-endian supported */ + {TA_PROP_STR_ENDIAN, USER_TA_PROP_TYPE_U32, &(const uint32_t){0}}, + + {TA_PROP_STR_DOES_NOT_CLOSE_HANDLE_ON_CORRUPT_OBJECT, + USER_TA_PROP_TYPE_BOOL, + &(const bool){TA_FLAGS & TA_FLAG_DONT_CLOSE_HANDLE_ON_CORRUPT_OBJECT}}, + +/* + * Extended propietary properties, name of properties must not begin with + * "gpd." + */ +#ifdef TA_CURRENT_TA_EXT_PROPERTIES + TA_CURRENT_TA_EXT_PROPERTIES +#endif +}; + +const size_t ta_num_props = sizeof(ta_props) / sizeof(ta_props[0]); + +#ifdef CFG_FTRACE_SUPPORT +struct __ftrace_info __ftrace_info = { +#ifdef __ILP32__ + .buf_start.ptr32 = { .lo = (uint32_t)&__ftrace_buf_start }, + .buf_end.ptr32 = { .lo = (uint32_t)__ftrace_buf_end }, + .ret_ptr.ptr32 = { .lo = (uint32_t)&__ftrace_return }, +#else + .buf_start.ptr64 = (uint64_t)&__ftrace_buf_start, + .buf_end.ptr64 = (uint64_t)__ftrace_buf_end, + .ret_ptr.ptr64 = (uint64_t)&__ftrace_return, +#endif +}; +#endif + +int tahead_get_trace_level(void) +{ + /* + * Store trace level in TA head structure, as ta_head.prop_tracelevel + */ + return TRACE_LEVEL; +} + +#if __OPTEE_CORE_API_COMPAT_1_1 +#undef TA_OpenSessionEntryPoint +#undef TA_InvokeCommandEntryPoint +#undef TEE_Param +TEE_Result TA_OpenSessionEntryPoint(uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS], + void **sess_ctx) +{ + return __ta_open_sess(pt, params, sess_ctx, + __GP11_TA_OpenSessionEntryPoint); +} + +TEE_Result TA_InvokeCommandEntryPoint(void *sess_ctx, uint32_t cmd_id, + uint32_t pt, + TEE_Param params[TEE_NUM_PARAMS]) +{ + return __ta_invoke_cmd(sess_ctx, cmd_id, pt, params, + __GP11_TA_InvokeCommandEntryPoint); +} +#endif diff --git a/optee_os/typedefs.checkpatch b/optee_os/typedefs.checkpatch new file mode 100644 index 0000000..2c96568 --- /dev/null +++ b/optee_os/typedefs.checkpatch @@ -0,0 +1,41 @@ +# Note: please keep the entries in this file sorted in reverse alphabetical +# order (sort -r) +TEE_Whence +TEE_UUID +TEE_Time +TEE_TASessionHandle +TEE_Session +TEE_SESessionHandle +TEE_SEServiceHandle +TEE_SEReaderProperties +TEE_SEReaderHandle +TEE_SEChannelHandle +TEE_SEAID +TEE_Result +TEE_PropSetHandle +TEE_Param +TEE_OperationMode +TEE_OperationInfoMultiple +TEE_OperationInfoKey +TEE_OperationInfo +TEE_OperationHandle +TEE_ObjectType +TEE_ObjectInfo +TEE_ObjectHandle +TEE_ObjectEnumHandle +TEE_Identity +TEE_ErrorOrigin +TEE_BigIntFMMContext +TEE_BigIntFMM +TEE_BigInt +TEE_Attribute +TAILQ_ENTRY\(.*\) +STAILQ_HEAD\(.*\) +STAILQ_ENTRY\(.*\) +SLIST_HEAD\(.*\) +SLIST_ENTRY\(.*\) +mbedtls_mpi_uint +mbedtls_mpi +Elf_Phdr +Elf_Half +Elf_Addr diff --git a/wasm-micro-runtime/.clang-format b/wasm-micro-runtime/.clang-format new file mode 100644 index 0000000..0d945dd --- /dev/null +++ b/wasm-micro-runtime/.clang-format @@ -0,0 +1,61 @@ +# using [clang-formt-12 options](https://releases.llvm.org/12.0.0/tools/clang/docs/ClangFormatStyleOptions.html) +RawStringFormats: + - Language: Cpp + Delimiters: + - c + - C + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + - h + - hpp + CanonicalDelimiter: '' + BasedOnStyle: Mozilla + +Language: Cpp +BasedOnStyle: Mozilla +# 6.1 +IndentWidth: 4 +ContinuationIndentWidth: 4 +# 6.2 +TabWidth: 4 +UseTab: Never +# 6.3 +ColumnLimit: 80 +# 6.9 +AlignAfterOpenBracket: Align +BinPackArguments: true +BinPackParameters: true +# 6.10 +BreakBeforeBraces: Custom +BraceWrapping: + AfterCaseLabel: true + AfterClass: true + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: false + SplitEmptyNamespace: true +# 6.27 +BreakBeforeBinaryOperators: NonAssignment + +# additional +AlignEscapedNewlines: Left +AllowAllParametersOfDeclarationOnNextLine: false +AllowAllArgumentsOnNextLine: false +PointerAlignment: Right +SpaceAroundPointerQualifiers: After +SortIncludes: false diff --git a/wasm-micro-runtime/.clang-tidy b/wasm-micro-runtime/.clang-tidy new file mode 100644 index 0000000..67cd167 --- /dev/null +++ b/wasm-micro-runtime/.clang-tidy @@ -0,0 +1,16 @@ +# refer to https://clang.llvm.org/extra/clang-tidy/checks/list.html + +Checks: '-*, readability-identifier-naming, clang-analyzer-core.*,' +WarningsAsErrors: '-*' +HeaderFilterRegex: '' +FormatStyle: file +InheritParentConfig: false +AnalyzeTemporaryDtors: false +User: wamr +CheckOptions: + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.MacroDefinitionCase + value: UPPER_CASE diff --git a/wasm-micro-runtime/.devcontainer/Dockerfile b/wasm-micro-runtime/.devcontainer/Dockerfile new file mode 100644 index 0000000..1587a7d --- /dev/null +++ b/wasm-micro-runtime/.devcontainer/Dockerfile @@ -0,0 +1,150 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/cpp/.devcontainer/base.Dockerfile +# [Choice] Debian / Ubuntu version (use Debian 12/11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-12, debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT=debian-12 +FROM mcr.microsoft.com/vscode/devcontainers/cpp:${VARIANT} + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Asian/Shanghai + +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y apt-transport-https apt-utils build-essential \ + ca-certificates ccache cmake curl g++-multilib git gnupg \ + libgcc-12-dev lib32gcc-12-dev lsb-release \ + ninja-build ocaml ocamlbuild \ + software-properties-common tree tzdata \ + unzip valgrind vim wget zip --no-install-recommends \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +# +# binaryen +ARG BINARYEN_VER=114 +WORKDIR /opt +RUN wget -c --progress=dot:giga https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VER}/binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz \ + && tar xf binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz \ + && ln -sf /opt/binaryen-version_${BINARYEN_VER} /opt/binaryen \ + && rm binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz + +# +# CMAKE (https://apt.kitware.com/) +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=DL3008 +ARG CMAKE_VER=3.27.0 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-Linux-x86_64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && mkdir /opt/cmake-${CMAKE_VER} \ + && /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake-${CMAKE_VER} \ + && rm /tmp/cmake-install.sh \ + && ln -s /opt/cmake-${CMAKE_VER}/bin/* /usr/local/bin + +# +# install emsdk +WORKDIR /opt +RUN git clone https://github.com/emscripten-core/emsdk.git + +ARG EMSDK_VER=3.1.43 +WORKDIR /opt/emsdk +RUN git pull \ + && ./emsdk install ${EMSDK_VER} \ + && ./emsdk activate ${EMSDK_VER} \ + && echo "source /opt/emsdk/emsdk_env.sh" >> /root/.bashrc + +# +# install wasi-sdk +ARG WASI_SDK_VER=20 +RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VER}/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -P /opt \ + && tar xf /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -C /opt \ + && ln -sf /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk \ + && rm /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz + +# +#install wabt +ARG WABT_VER=1.0.33 +RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/wabt-${WABT_VER}-ubuntu.tar.gz -P /opt \ + && tar xf /opt/wabt-${WABT_VER}-ubuntu.tar.gz -C /opt \ + && ln -sf /opt/wabt-${WABT_VER} /opt/wabt \ + && rm /opt/wabt-${WABT_VER}-ubuntu.tar.gz + +# +# install bazelisk +ARG BAZELISK_VER=1.17.0 +RUN mkdir /opt/bazelisk \ + && wget -c --progress=dot:giga https://github.com/bazelbuild/bazelisk/releases/download/v${BAZELISK_VER}/bazelisk-linux-amd64 -P /opt/bazelisk \ + && chmod a+x /opt/bazelisk/bazelisk-linux-amd64 \ + && ln -fs /opt/bazelisk/bazelisk-linux-amd64 /opt/bazelisk/bazel + +# +# install clang+llvm +ARG LLVM_VER=16 +RUN apt-get purge -y clang-14 llvm-14 && apt-get autoremove -y +WORKDIR /etc/apt/apt.conf.d +RUN touch 99verfiy-peer.conf \ + && echo "Acquire { https::Verify-Peer false }" > 99verfiy-peer.conf + +WORKDIR /tmp +#RUN wget --progress=dot:giga https://apt.llvm.org/llvm.sh \ +# && chmod a+x ./llvm.sh \ +# && ./llvm.sh ${LLVM_VER} all + +# Workaround due to https://github.com/llvm/llvm-project/issues/62475 +# hadolint ignore=DL3008 +RUN set -ex \ + && echo "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${LLVM_VER} main" > /etc/apt/sources.list.d/apt.llvm.org.list \ + && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \ + && apt-get update \ + && apt-get install -y \ + clang-${LLVM_VER} lldb-${LLVM_VER} lld-${LLVM_VER} clangd-${LLVM_VER} clang-tidy-${LLVM_VER} clang-format-${LLVM_VER} clang-tools-${LLVM_VER} \ + llvm-${LLVM_VER}-dev lld-${LLVM_VER} lldb-${LLVM_VER} llvm-${LLVM_VER}-tools libomp-${LLVM_VER}-dev libc++-${LLVM_VER}-dev libc++abi-${LLVM_VER}-dev \ + libclang-common-${LLVM_VER}-dev libclang-${LLVM_VER}-dev libclang-cpp${LLVM_VER}-dev libunwind-${LLVM_VER}-dev \ + libclang-rt-${LLVM_VER}-dev libpolly-${LLVM_VER}-dev --no-install-recommends \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +# +# [Optional] + +# +# Install pip +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install -y --reinstall python3-venv python3-pip --no-install-recommends \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +# +# Install required python packages +# hadolint ignore=DL3013 +RUN python3 -m pip install --no-cache-dir --break-system-packages --upgrade pip \ + && pip3 install --no-cache-dir --break-system-packages black nose pycparser pylint + +# +# Install github-cli. It doens't work as a feature of devcontainer.json +ARG GH_CLI_VER=2.32.0 +WORKDIR /tmp +RUN wget -q https://github.com/cli/cli/releases/download/v${GH_CLI_VER}/gh_${GH_CLI_VER}_linux_amd64.deb \ + && dpkg -i gh_${GH_CLI_VER}_linux_amd64.deb + +# +# Install NodeJS +RUN wget -qO- https://deb.nodesource.com/setup_20.x | bash - +# hadolint ignore=DL3008 +RUN apt-get install -y nodejs --no-install-recommends + +# set path +ENV PATH="/opt/bazelisk:/usr/lib/llvm-${LLVM_VER}/bin:${PATH}" +ENV CC=/usr/lib/llvm-${LLVM_VER}/bin/clang CXX=/usr/lib/llvm-${LLVM_VER}/bin/clang++ +RUN printf "%s\n" "PS1='\n[ \u@wamr-dev-docker \W ]\n$ '" >> /root/.bashrc \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/* + +# set workdir when container run +VOLUME /workspaces +WORKDIR /workspaces diff --git a/wasm-micro-runtime/.devcontainer/devcontainer.json b/wasm-micro-runtime/.devcontainer/devcontainer.json new file mode 100644 index 0000000..5feb175 --- /dev/null +++ b/wasm-micro-runtime/.devcontainer/devcontainer.json @@ -0,0 +1,50 @@ +// Copyright (C) 2019 Intel Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/cpp +{ + "name": "WAMR-Dev", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-12, debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 + // Use Debian 12, Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon + "args": { + "BINARYEN_VER": "114", + "BAZELISK_VER": "1.17.0", + "CMAKE_VER": "3.27.0", + "EMSDK_VER": "3.1.43", + "GH_CLI_VER": "2.32.0", + "LLVM_VER": "16", + "VARIANT": "debian-12", + "WASI_SDK_VER": "20", + "WABT_VER": "1.0.33" + } + }, + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined" + ], + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": {}, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "dtsvet.vscode-wasm", + "llvm-vs-code-extensions.vscode-clangd", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-vscode.cmake-tools" + ] + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "curl https://sh.rustup.rs -sSf | bash -s -- -y", + // Comment out this line to run as root instead. + "remoteUser": "vscode" +} \ No newline at end of file diff --git a/wasm-micro-runtime/.github/ISSUE_TEMPLATE/blank_issue.md b/wasm-micro-runtime/.github/ISSUE_TEMPLATE/blank_issue.md new file mode 100644 index 0000000..57febe7 --- /dev/null +++ b/wasm-micro-runtime/.github/ISSUE_TEMPLATE/blank_issue.md @@ -0,0 +1,5 @@ +--- +name: Blank Issue +about: Create a blank issue. +title: '' +--- diff --git a/wasm-micro-runtime/.github/ISSUE_TEMPLATE/improvement.md b/wasm-micro-runtime/.github/ISSUE_TEMPLATE/improvement.md new file mode 100644 index 0000000..ffdf090 --- /dev/null +++ b/wasm-micro-runtime/.github/ISSUE_TEMPLATE/improvement.md @@ -0,0 +1,28 @@ +--- +name: Improvement +about: A feature request or code improvement. +title: '' +labels: '' +assignees: '' +--- + +Thanks for filing a feature request! Please fill out the TODOs below. + +#### Feature + +TODO: Brief description of the feature/improvement you'd like to see in WAMR + +#### Benefit + +TODO: What is the value of adding this in WAMR? What problems does it solve? + +#### Implementation + +TODO: Do you have an implementation plan, and/or ideas for data structures or +algorithms to use? + +#### Alternatives + +TODO: What are the alternative implementation approaches or alternative ways to +solve the problem that this feature would solve? How do these alternatives +compare to this proposal? diff --git a/wasm-micro-runtime/.github/ISSUE_TEMPLATE/report_bug.md b/wasm-micro-runtime/.github/ISSUE_TEMPLATE/report_bug.md new file mode 100644 index 0000000..d3058c9 --- /dev/null +++ b/wasm-micro-runtime/.github/ISSUE_TEMPLATE/report_bug.md @@ -0,0 +1,36 @@ +--- +name: WAMR bug or defect report +about: Report a bug or defect in WAMR +title: '' +--- + +Thanks for filing a bug or defect report! Please fill out the TODOs below. + +### Subject of the issue + +Describe the bug or defect here. + +### Test case + +Upload the related wasm file, wast file or the source files if you can. + +### Your environment + +* Host OS +* WAMR version, platform, cpu architecture, running mode, etc. + +### Steps to reproduce + +Tell us how to reproduce this bug or defect. + +### Expected behavior + +Tell us what should happen + +### Actual behavior + +Tell us what happens instead + +### Extra Info + +Anything else you'd like to add? diff --git a/wasm-micro-runtime/.github/dependabot.yml b/wasm-micro-runtime/.github/dependabot.yml new file mode 100644 index 0000000..0676cf7 --- /dev/null +++ b/wasm-micro-runtime/.github/dependabot.yml @@ -0,0 +1,35 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +version: 2 +updates: + +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + +- package-ecosystem: "docker" + directory: "/.devcontainer" + schedule: + interval: "weekly" + +- package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: "weekly" + +- package-ecosystem: "pip" + directory: "/build-scripts" + schedule: + interval: "weekly" + +- package-ecosystem: "pip" + directory: "/language-bindings/python/wasm-c-api" + schedule: + interval: "weekly" + +- package-ecosystem: "pip" + directory: "/language-bindings/python/wamr-api" + schedule: + interval: "weekly" diff --git a/wasm-micro-runtime/.github/scripts/codeql_buildscript.sh b/wasm-micro-runtime/.github/scripts/codeql_buildscript.sh new file mode 100755 index 0000000..4bcabfe --- /dev/null +++ b/wasm-micro-runtime/.github/scripts/codeql_buildscript.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +sudo apt update + +sudo apt install -y build-essential cmake g++-multilib libgcc-11-dev lib32gcc-11-dev ccache ninja-build ccache + +WAMR_DIR=${PWD} + +# TODO: use pre-built llvm binary to build wamrc to +# avoid static code analysing for llvm +: ' +# build wamrc +cd ${WAMR_DIR}/wamr-compiler +./build_llvm.sh +rm -fr build && mkdir build && cd build +cmake .. +make -j +if [[ $? != 0 ]]; then + echo "Failed to build wamrc!" + exit 1; +fi +' + +# build iwasm with default features enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with default features enabled!" + exit 1; +fi + +# build iwasm with default features enabled on x86_32 +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DWAMR_BUILD_TARGET=X86_32 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with default features enabled on x86_32!" + exit 1; +fi + +# build iwasm with classic interpreter enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_INTERP=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with classic interpreter enabled!" + exit 1; +fi + +# build iwasm with extra features enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug \ + -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \ + -DWAMR_BUILD_MULTI_MODULE=1 -DWAMR_BUILD_SIMD=1 \ + -DWAMR_BUILD_TAIL_CALL=1 -DWAMR_BUILD_REF_TYPES=1 \ + -DWAMR_BUILD_CUSTOM_NAME_SECTION=1 -DWAMR_BUILD_MEMORY_PROFILING=1 \ + -DWAMR_BUILD_PERF_PROFILING=1 -DWAMR_BUILD_DUMP_CALL_STACK=1 \ + -DWAMR_BUILD_LOAD_CUSTOM_SECTION=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build wamrc iwasm with extra features enabled!" + exit 1; +fi + +# build iwasm with global heap pool enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug \ + -DWAMR_BUILD_ALLOC_WITH_USER_DATA=1 \ + -DWAMR_DISABLE_STACK_HW_BOUND_CHECK=1 \ + -DWAMR_BUILD_GLOBAL_HEAP_POOL=1 \ + -DWAMR_BUILD_GLOBAL_HEAP_SIZE=131072 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with global heap pool enabled!" + exit 1; +fi + +# build iwasm with wasi-threads enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIB_WASI_THREADS=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with wasi-threads enabled!" + exit 1; +fi + +# build iwasm with GC enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_GC=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with GC enabled!" + exit 1; +fi + +# build iwasm with exception handling enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_EXCE_HANDLING=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with exception handling enabled!" + exit 1; +fi + +# build iwasm with memory64 enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MEMORY64=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with memory64 enabled!" + exit 1; +fi + +# build iwasm with hardware boundary check disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_HW_BOUND_CHECK=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with hardware boundary check disabled!" + exit 1; +fi + +# build iwasm with quick AOT entry disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_QUICK_AOT_ENTRY=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with quick AOT entry disabled!" + exit 1; +fi + +# build iwasm with wakeup of blocking operations disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_WAKEUP_BLOCKING_OP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with wakeup of blocking operations disabled!" + exit 1; +fi + +# build iwasm with module instance context disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MODULE_INST_CONTEXT=0 \ + -DWAMR_BUILD_LIBC_BUILTIN=0 -DWAMR_BUILD_LIBC_WASI=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with module instance context disabled!" + exit 1; +fi + +# build iwasm with libc-uvwasi enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIBC_UVWASI=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with libc-uvwasi enabled!" + exit 1; +fi + +# build iwasm with fast jit lazy mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with fast jit lazy mode enabled!" + exit 1; +fi + +# build iwasm with fast jit eager mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with fast jit eager mode enabled!" + exit 1; +fi + +# TODO: use pre-built llvm binary to build llvm-jit and multi-tier-jit +: ' +# build iwasm with llvm jit lazy mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build llvm jit lazy mode enabled!" + exit 1; +fi + +# build iwasm with llvm jit eager mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build llvm jit eager mode enabled!" + exit 1; +fi + +# build iwasm with multi-tier jit enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \ + -DWAMR_BUILD_FAST_JIT_DUMP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with multi-tier jit enabled!" + exit 1; +fi +' + +# build iwasm with wasm mini-loader enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MINI_LOADER=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build with wasm mini-loader enabled!" + exit 1; +fi + +# build iwasm with source debugging enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_DEBUG_INTERP=1 -DWAMR_BUILD_DEBUG_AOT=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with source debugging enabled!" + exit 1; +fi + +# build iwasm with AOT static PGO enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_STATIC_PGO=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with AOT static PGO enabled!" + exit 1; +fi + +# build iwasm with configurable bounds checks enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_CONFIGUABLE_BOUNDS_CHECKS=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with configurable bounds checks enabled!" + exit 1; +fi + +# build iwasm with linux perf support enabled +cd ${WAMR_DIR}/product-mini/platforms/linux/ +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LINUX_PERF=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with linux perf support enabled!" + exit 1; +fi diff --git a/wasm-micro-runtime/.github/scripts/codeql_fail_on_error.py b/wasm-micro-runtime/.github/scripts/codeql_fail_on_error.py new file mode 100755 index 0000000..f150c38 --- /dev/null +++ b/wasm-micro-runtime/.github/scripts/codeql_fail_on_error.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import json +import sys +import os +import requests + + +def fetch_dismissed_alerts(repo_name, github_token): + headers = { + "Authorization": f"token {github_token}", + "Accept": "application/vnd.github.v3+json", + } + url = ( + f"https://api.github.com/repos/{repo_name}/code-scanning/alerts?state=dismissed" + ) + response = requests.get(url, headers=headers) + return response.json() # This assumes a successful API call + + +def parse_location(location): + path = location.get("physicalLocation", {}).get("artifactLocation", {}).get("uri") + start_line = location.get("physicalLocation", {}).get("region", {}).get("startLine") + column_range = ( + location.get("physicalLocation", {}).get("region", {}).get("startColumn"), + location.get("physicalLocation", {}).get("region", {}).get("endColumn"), + ) + return (path, start_line, column_range) + + +def is_dismissed(rule_id, path, start_line, column_range, dismissed_alerts): + for alert in dismissed_alerts: + alert_rule_id = alert.get("rule", {}).get("id") + alert_path = alert.get("location", {}).get("path") + alert_start_line = alert.get("location", {}).get("start_line") + alert_column_range = ( + alert.get("location", {}).get("start_column"), + alert.get("location", {}).get("end_column"), + ) + + if ( + rule_id == alert_rule_id + and path == alert_path + and start_line == alert_start_line + and column_range == alert_column_range + ): + return True + return False + + +# Return whether SARIF file contains error-level results +def codeql_sarif_contain_error(filename, dismissed_alerts): + has_error = False + + with open(filename, "r") as f: + s = json.load(f) + + for run in s.get("runs", []): + rules_metadata = run["tool"]["driver"]["rules"] + if not rules_metadata: + rules_metadata = run["tool"]["extensions"][0]["rules"] + + for res in run.get("results", []): + if "ruleIndex" in res: + rule_index = res["ruleIndex"] + elif "rule" in res and "index" in res["rule"]: + rule_index = res["rule"]["index"] + else: + continue + + # check whether it's dismissed before + rule_id = res["ruleId"] + path, start_line, column_range = parse_location(res["locations"][0]) + # the source code is from dependencies + if "_deps" in path: + continue + if is_dismissed(rule_id, path, start_line, column_range, dismissed_alerts): + print( + f"====== Finding a dismissed entry: {rule_id} at {path}:{start_line} is dismissed.======" + ) + print(res) + continue + + try: + rule_level = rules_metadata[rule_index]["defaultConfiguration"]["level"] + except IndexError as e: + print(e, rule_index, len(rules_metadata)) + else: + if rule_level == "error": + # very likely to be an actual error + if rules_metadata[rule_index]["properties"].get("precision") in [ + "high", + "very-high", + ]: + # the security severity is above medium(Common Vulnerability Scoring System (CVSS) >= 4.0) + if "security-severity" in rules_metadata[rule_index][ + "properties" + ] and ( + float( + rules_metadata[rule_index]["properties"][ + "security-severity" + ] + ) + > 4.0 + ): + print("====== Finding a likely error. ======") + print(res) + has_error = True + + return has_error + + +if __name__ == "__main__": + GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") + GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") + dismissed_alerts = fetch_dismissed_alerts(GITHUB_REPOSITORY, GITHUB_TOKEN) + + if codeql_sarif_contain_error(sys.argv[1], dismissed_alerts): + sys.exit(1) diff --git a/wasm-micro-runtime/.github/scripts/extract_from_release_notes.py b/wasm-micro-runtime/.github/scripts/extract_from_release_notes.py new file mode 100644 index 0000000..3802d92 --- /dev/null +++ b/wasm-micro-runtime/.github/scripts/extract_from_release_notes.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +""" +Extract the latest release notes content from RELEASE_NOTES.md +""" + +import argparse +import os +import sys +import traceback + + +def latest_content(release_notes_path): + """ + can't change the format of the original content + """ + content = "" + start_extract = False + with open(release_notes_path, encoding="utf-8") as f: + for line in f: + if line.startswith("## "): + if start_extract: + break + + start_extract = True + continue + + # hit a separated line + if line.startswith("---"): + break + + content += line + + content += os.linesep + return content + + +def main(): + """ + GO!GO!!GO!!! + """ + parser = argparse.ArgumentParser(description="run the sample and examine outputs") + parser.add_argument("release_notes_path", type=str) + args = parser.parse_args() + + ret = 1 + try: + print(latest_content(args.release_notes_path)) + ret = 0 + except AssertionError: + traceback.print_exc() + return ret + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wasm-micro-runtime/.github/scripts/fetch_and_compare_version.py b/wasm-micro-runtime/.github/scripts/fetch_and_compare_version.py new file mode 100644 index 0000000..ad9e53a --- /dev/null +++ b/wasm-micro-runtime/.github/scripts/fetch_and_compare_version.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import re +import shlex +import subprocess +import sys + + +def fetch_version_from_code(): + """ + search the semantic version definition in core/version.h + """ + major, minor, patch = "", "", "" + with open("core/version.h", encoding="utf-8") as f: + for line in f: + if "WAMR_VERSION" not in line: + continue + + major_match = re.search(r"WAMR_VERSION_MAJOR (\d+)", line) + if major_match is not None: + major = major_match.groups()[0] + continue + + minor_match = re.search(r"WAMR_VERSION_MINOR (\d+)", line) + if minor_match is not None: + minor = minor_match.groups()[0] + continue + + patch_match = re.search(r"WAMR_VERSION_PATCH (\d+)", line) + if patch_match is not None: + patch = patch_match.groups()[0] + + if len(major) == 0 or len(minor) == 0 or len(patch) == 0: + raise Exception( + "can't find the semantic version definition likes WAMR_VERSION_*" + ) + return f"WAMR-{major}.{minor}.{patch}" + + +def fetch_latest_git_tag(): + """ + Get the most recent tag from the HEAD, + if it's main branch, it should be the latest release tag. + if it's release/x.x.x branch, it should be the latest release tag of the branch. + """ + list_tag_cmd = "git describe --tags --abbrev=0 HEAD" + p = subprocess.run(shlex.split(list_tag_cmd), capture_output=True, check=True) + + all_tags = p.stdout.decode().strip() + latest_tag = all_tags.split("\n")[-1] + return latest_tag + + +def match_version_pattern(v): + pattern = r"WAMR-\d+\.\d+\.\d+" + m = re.match(pattern, v) + return m is not None + + +def split_version_string(v): + """ + return the semantic version as an integer list + """ + pattern = r"WAMR-(\d+)\.(\d+)\.(\d+)" + m = re.match(pattern, v) + return [int(x) for x in m.groups()] + + +def compare_version_string(v1, v2): + """ + return value: + - 1. if v1 > v2 + - -1. if v1 < v2 + - 0. if v1 == v2 + """ + if not match_version_pattern(v1): + raise Exception(f"{v1} doesn't match the version pattern") + + if not match_version_pattern(v2): + raise Exception(f"{v2} doesn't match the version pattern") + + v1_sem_ver = split_version_string(v1) + v2_sem_ver = split_version_string(v2) + + return 0 if v1_sem_ver == v2_sem_ver else (1 if v1_sem_ver > v2_sem_ver else -1) + + +def is_major_or_minor_changed(v1, v2): + """ + return true if change either major of v2 or minor of v2 + return false or else + """ + if not match_version_pattern(v1): + raise Exception(f"{v1} doesn't match the version pattern") + + if not match_version_pattern(v2): + raise Exception(f"{v2} doesn't match the version pattern") + + v1_major, v1_minor, _ = split_version_string(v1) + v2_major, v2_minor, _ = split_version_string(v2) + + return v2_major != v1_major or v2_minor != v1_minor + + +def next_version(): + definition = fetch_version_from_code() + tag = fetch_latest_git_tag() + + new_version = "" + minor_changed = False + if compare_version_string(definition, tag) == 1: + new_version = definition.split("-")[-1] + + if is_major_or_minor_changed(tag, definition): + minor_changed = True + + return new_version, "major_minor_change" if minor_changed else "patch_change" + + +if __name__ == "__main__": + print(f"{next_version()[0]},{next_version()[1]}") + sys.exit(0) diff --git a/wasm-micro-runtime/.github/scripts/reuse_latest_release_binaries.py b/wasm-micro-runtime/.github/scripts/reuse_latest_release_binaries.py new file mode 100644 index 0000000..0fe2d97 --- /dev/null +++ b/wasm-micro-runtime/.github/scripts/reuse_latest_release_binaries.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import argparse +import json +import os +import shlex +import subprocess +import sys +from urllib.error import HTTPError, URLError +import urllib.request + + +def get_last_commit(target_path, cwd): + last_commit_cmd = f"git log -n 1 --pretty=format:%H -- {target_path}" + p = subprocess.run( + shlex.split(last_commit_cmd), capture_output=True, check=True, cwd=cwd + ) + return p.stdout.decode().strip() + + +def fetch_git_tags(): + list_tag_cmd = ( + 'git tag --list WAMR-*.*.* --sort=committerdate --format="%(refname:short)"' + ) + p = subprocess.run(shlex.split(list_tag_cmd), capture_output=True, check=True) + + all_tags = p.stdout.decode().strip() + return all_tags.split("\n") + + +def download_binaries(binary_name_stem, cwd): + """ + 1. find the latest release name + 2. form assets download url: + """ + try: + all_tags = fetch_git_tags() + # *release_process.yml* will create a tag and release at first + second_last_tag = all_tags[-2] + latest_tag = all_tags[-1] + + latest_url = "https://api.github.com/repos/bytecodealliance/wasm-micro-runtime/releases/latest" + print(f"::notice::query the latest release with {latest_url}...") + with urllib.request.urlopen(latest_url) as response: + body = response.read() + + release_name = json.loads(body)["name"] + + # WAMR-X.Y.Z -> X.Y.Z + second_last_sem_ver = second_last_tag[5:] + latest_sem_ver = latest_tag[5:] + assert latest_sem_ver in binary_name_stem + name_stem_in_release = binary_name_stem.replace( + latest_sem_ver, second_last_sem_ver + ) + + # download and rename + for file_ext in (".zip", ".tar.gz"): + assets_url = f"https://github.com/bytecodealliance/wasm-micro-runtime/releases/download/{release_name}/{name_stem_in_release}{file_ext}" + local_path = f"{binary_name_stem}{file_ext}" + print(f"::notice::download from {assets_url} and save as {local_path}...") + urllib.request.urlretrieve(assets_url, local_path) + return True + except HTTPError as error: + print(error.status, error.reason) + except URLError as error: + print(error.reason) + except TimeoutError: + print("Request timeout") + + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Reuse binaries of the latest release if no more modification on the_path since last_commit" + ) + parser.add_argument("working_directory", type=str) + parser.add_argument("--binary_name_stem", type=str) + parser.add_argument("--last_commit", type=str) + parser.add_argument("--the_path", type=str) + args = parser.parse_args() + + last_commit = get_last_commit(args.the_path, args.working_directory) + if last_commit == args.last_commit: + return download_binaries(args.binary_name_stem, args.working_directory) + else: + return False + + +if __name__ == "__main__": + # use output to indicate results + # echo "result=${result}" >> "$GITHUB_OUTPUT" + with open(os.environ.get("GITHUB_OUTPUT"), 'a') as output_file: + output_file.write("result=hit\n" if main() else "result=not-hit\n") + + # always return 0 + sys.exit(0) diff --git a/wasm-micro-runtime/.github/workflows/build_docker_images.yml b/wasm-micro-runtime/.github/workflows/build_docker_images.yml new file mode 100644 index 0000000..d5bf682 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/build_docker_images.yml @@ -0,0 +1,89 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: Create and publish Docker images + +on: + workflow_call: + inputs: + upload_url: + description: upload binary assets to the URL of release + type: string + required: true + ver_num: + description: a semantic version number. + type: string + required: true + +jobs: + build-and-push-images: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build and save Docker image(wasm-debug-server:${{ inputs.ver_num }}) to tar file + run: | + docker build -t wasm-debug-server:${{ inputs.ver_num }} . + docker save -o wasm-debug-server.tar wasm-debug-server:${{ inputs.ver_num }} + working-directory: test-tools/wamr-ide/WASM-Debug-Server/Docker + + - name: compress the tar file + run: | + tar czf wasm-debug-server-${{ inputs.ver_num }}.tar.gz wasm-debug-server.tar + zip wasm-debug-server-${{ inputs.ver_num }}.zip wasm-debug-server.tar + working-directory: test-tools/wamr-ide/WASM-Debug-Server/Docker + + - name: upload release tar.gz + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: test-tools/wamr-ide/WASM-Debug-Server/Docker/wasm-debug-server-${{ inputs.ver_num }}.tar.gz + asset_name: wasm-debug-server-${{ inputs.ver_num }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: test-tools/wamr-ide/WASM-Debug-Server/Docker/wasm-debug-server-${{ inputs.ver_num }}.zip + asset_name: wasm-debug-server-${{ inputs.ver_num }}.zip + asset_content_type: application/zip + + - name: Build and save Docker image(wasm-toolchain:${{ inputs.ver_num }}) to tar file + run: | + docker build -t wasm-toolchain:${{ inputs.ver_num }} . + docker save -o wasm-toolchain.tar wasm-toolchain:${{ inputs.ver_num }} + working-directory: test-tools/wamr-ide/WASM-Toolchain/Docker + + - name: compress the tar file + run: | + tar czf wasm-toolchain-${{ inputs.ver_num }}.tar.gz wasm-toolchain.tar + zip wasm-toolchain-${{ inputs.ver_num }}.zip wasm-toolchain.tar + working-directory: test-tools/wamr-ide/WASM-Toolchain/Docker + + - name: upload release tar.gz + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: test-tools/wamr-ide/WASM-Toolchain/Docker/wasm-toolchain-${{ inputs.ver_num }}.tar.gz + asset_name: wasm-toolchain-${{ inputs.ver_num }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: test-tools/wamr-ide/WASM-Toolchain/Docker/wasm-toolchain-${{ inputs.ver_num }}.zip + asset_name: wasm-toolchain-${{ inputs.ver_num }}.zip + asset_content_type: application/zip + diff --git a/wasm-micro-runtime/.github/workflows/build_iwasm_release.yml b/wasm-micro-runtime/.github/workflows/build_iwasm_release.yml new file mode 100644 index 0000000..86e50ed --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/build_iwasm_release.yml @@ -0,0 +1,108 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: build iwasm release + +on: + workflow_call: + inputs: + arch: + description: arch of the release + type: string + required: false + default: x86_64 + cwd: + description: workfing directory + type: string + required: true + llvm_cache_key: + description: the cache key of llvm libraries + type: string + required: true + runner: + description: OS of compilation + type: string + required: true + upload_url: + description: a semantic version number. it is required when `release` is true. + type: string + required: false + ver_num: + description: a semantic version number. it is required when `release` is true. + type: string + required: false + +jobs: + build: + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v4 + + - name: get cached LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ inputs.llvm_cache_key }} + fail-on-cache-miss: true + + - name: generate iwasm binary release + run: | + cmake -S . -B build \ + -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \ + -DWAMR_BUILD_CUSTOM_NAME_SECTION=0 \ + -DWAMR_BUILD_DEBUG_INTERP=0 \ + -DWAMR_BUILD_DEBUG_AOT=0 \ + -DWAMR_BUILD_DUMP_CALL_STACK=0 \ + -DWAMR_BUILD_LIBC_UVWASI=0 \ + -DWAMR_BUILD_LIBC_EMCC=0 \ + -DWAMR_BUILD_LIB_RATS=0 \ + -DWAMR_BUILD_LOAD_CUSTOM_SECTION=0 \ + -DWAMR_BUILD_MEMORY_PROFILING=0 \ + -DWAMR_BUILD_MINI_LOADER=0 \ + -DWAMR_BUILD_MULTI_MODULE=0 \ + -DWAMR_BUILD_PERF_PROFILING=0 \ + -DWAMR_BUILD_SPEC_TEST=0 \ + -DWAMR_BUILD_BULK_MEMORY=1 \ + -DWAMR_BUILD_LIB_PTHREAD=1 \ + -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \ + -DWAMR_BUILD_LIB_WASI_THREADS=1 \ + -DWAMR_BUILD_LIBC_BUILTIN=1 \ + -DWAMR_BUILD_LIBC_WASI=1 \ + -DWAMR_BUILD_REF_TYPES=1 \ + -DWAMR_BUILD_SIMD=1 \ + -DWAMR_BUILD_SHARED_MEMORY=1 \ + -DWAMR_BUILD_TAIL_CALL=1 \ + -DWAMR_BUILD_THREAD_MGR=1 + cmake --build build --config Release --parallel 4 + working-directory: ${{ inputs.cwd }} + + - name: compress the binary + run: | + tar czf iwasm-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz iwasm + zip iwasm-${{ inputs.ver_num }}-${{ inputs.runner }}.zip iwasm + working-directory: ${{ inputs.cwd }}/build + + - name: upload release tar.gz + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ${{ inputs.cwd }}/build/iwasm-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz + asset_name: iwasm-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ${{ inputs.cwd }}/build/iwasm-${{ inputs.ver_num }}-${{ inputs.runner }}.zip + asset_name: iwasm-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.zip + asset_content_type: application/zip diff --git a/wasm-micro-runtime/.github/workflows/build_llvm_libraries.yml b/wasm-micro-runtime/.github/workflows/build_llvm_libraries.yml new file mode 100644 index 0000000..67eaf61 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/build_llvm_libraries.yml @@ -0,0 +1,105 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: Reusable workflow-build_llvm_libraries + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: true + type: string + container_image: + required: false + type: string + outputs: + cache_key: + description: "A cached key of LLVM libraries" + value: ${{ jobs.build_llvm_libraries.outputs.key}} + +jobs: + build_llvm_libraries: + runs-on: ${{ inputs.os }} + # Using given container image if it is specified. + # Otherwise, it will be ignored by the runner. + container: + image: ${{ inputs.container_image }} + outputs: + key: ${{ steps.create_lib_cache_key.outputs.key}} + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: install dependencies for non macos-14 + if: inputs.os != 'macos-14' + run: /usr/bin/env python3 -m pip install -r requirements.txt + working-directory: build-scripts + + - name: install dependencies for macos-14 + if: inputs.os == 'macos-14' + run: /usr/bin/env python3 -m pip install -r requirements.txt --break-system-packages + working-directory: build-scripts + + - name: retrive the last commit ID + id: get_last_commit + run: echo "last_commit=$(GH_TOKEN=${{ secrets.GITHUB_TOKEN }} /usr/bin/env python3 ./build_llvm.py --llvm-ver)" >> $GITHUB_OUTPUT + working-directory: build-scripts + + # Bump the prefix number to evict all previous caches and + # enforce a clean build, in the unlikely case that some + # weird build error occur and llvm/build becomes a potential + # suspect. + - name: form the cache key of libraries + id: create_lib_cache_key + run: echo "key=0-llvm-libraries-${{ inputs.os }}-${{ inputs.arch }}-${{ steps.get_last_commit.outputs.last_commit }}" >> $GITHUB_OUTPUT + + - name: Cache LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ steps.create_lib_cache_key.outputs.key}} + + - uses: actions/cache@v4 + with: + path: ~/.ccache + key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }} + restore-keys: | + 0-ccache-${{ inputs.os }} + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && inputs.os == 'ubuntu-20.04' + + - uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }} + restore-keys: | + 0-ccache-${{ inputs.os }} + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && inputs.os == 'ubuntu-22.04' + + # Don't install dependencies if the cache is hit or running in docker container + - run: sudo apt install -y ccache ninja-build + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'ubuntu') && inputs.container_image == '' + + - uses: actions/cache@v4 + with: + path: ~/Library/Caches/ccache + key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }} + restore-keys: | + 0-ccache-${{ inputs.os }} + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'macos') + + - run: brew install ccache ninja + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'macos') + + - name: Build LLVM libraries + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: /usr/bin/env python3 ./build_llvm.py --arch ${{ inputs.arch }} + working-directory: build-scripts diff --git a/wasm-micro-runtime/.github/workflows/build_wamr_lldb.yml b/wasm-micro-runtime/.github/workflows/build_wamr_lldb.yml new file mode 100644 index 0000000..03474c5 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/build_wamr_lldb.yml @@ -0,0 +1,254 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: build wamr lldb + +on: + workflow_call: + inputs: + arch: + description: arch of the release + type: string + required: false + default: x86_64 + runner: + description: OS of compilation + type: string + required: true + upload_url: + description: upload binary assets to the URL of release + type: string + required: true + ver_num: + description: a semantic version number + type: string + required: true + wasi_sdk_url: + description: download WASI_SDK from this URL + type: string + required: false + default: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz" + +jobs: + try_reuse: + uses: ./.github/workflows/reuse_latest_release_binaries.yml + with: + binary_name_stem: "wamr-lldb-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}" + last_commit: "ea63ba4bd010c2285623ad4acc0262a4d63bcfea" + the_path: "./build-scripts/lldb_wasm.patch" + upload_url: ${{ inputs.upload_url }} + + build: + needs: try_reuse + if: needs.try_reuse.outputs.result != 'hit' + runs-on: ${{ inputs.runner }} + + env: + PYTHON_VERSION: '3.10' + PYTHON_UBUNTU_STANDALONE_BUILD: https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.10.11+20230507-x86_64-unknown-linux-gnu-install_only.tar.gz + PYTHON_MACOS_STANDALONE_BUILD: https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.10.11+20230507-x86_64-apple-darwin-install_only.tar.gz + steps: + - uses: actions/checkout@v4 + + - name: download and install wasi-sdk + run: | + cd /opt + basename=$(basename ${{ inputs.wasi_sdk_url }}) + sudo wget --progress=dot:giga ${{ inputs.wasi_sdk_url }} + sudo tar -xzf ${basename} + sudo rm ${basename} + sudo mv wasi-sdk-* wasi-sdk + + - name: Cache build + id: lldb_build_cache + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm-project/build/bin + ./core/deps/llvm-project/build/include + ./core/deps/llvm-project/build/lib + ./core/deps/llvm-project/build/libexec + ./core/deps/llvm-project/build/share + ./core/deps/llvm-project/lldb/tools/ + ./core/deps/llvm-project/wamr-lldb/ + key: ${{inputs.arch}}-${{ inputs.runner }}-lldb_build + + - name: setup xcode macos + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'macos') + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: latest-stable + + # Remove xCode command line tools, to prevent duplicate symbol compilation failures + - name: install utils macos + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'macos') + run: | + brew install swig cmake ninja libedit + sudo rm -rf /Library/Developer/CommandLineTools + + - name: install utils ubuntu + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'ubuntu') + run: sudo apt update && sudo apt-get install -y lld ninja-build + + # `git clone` takes ~7m + - name: download llvm + if: steps.lldb_build_cache.outputs.cache-hit != 'true' + run: | + wget https://github.com/llvm/llvm-project/archive/1f27fe6128769f00197925c3b8f6abb9d0e5cd2e.zip + unzip -q 1f27fe6128769f00197925c3b8f6abb9d0e5cd2e.zip + mv llvm-project-1f27fe6128769f00197925c3b8f6abb9d0e5cd2e llvm-project + working-directory: core/deps/ + + - name: apply wamr patch + if: steps.lldb_build_cache.outputs.cache-hit != 'true' + run: | + git init + git config user.email "action@github.com" + git config user.name "github action" + git apply ../../../build-scripts/lldb_wasm.patch + working-directory: core/deps/llvm-project + + - name: get stand-alone python ubuntu + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'ubuntu') + run: | + wget ${{ env.PYTHON_UBUNTU_STANDALONE_BUILD }} -O python.tar.gz + tar -xvf python.tar.gz + working-directory: core/deps + + - name: get stand-alone python macos + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'macos') + run: | + wget ${{ env.PYTHON_MACOS_STANDALONE_BUILD }} -O python.tar.gz + tar -xvf python.tar.gz + working-directory: core/deps + + - name: build lldb ubuntu + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'ubuntu') + run: | + echo "start to build lldb..." + mkdir -p wamr-lldb + cmake -S ./llvm -B build \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX=../wamr-lldb \ + -DCMAKE_BUILD_TYPE:STRING="Release" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DLLVM_ENABLE_PROJECTS="clang;lldb" \ + -DLLVM_TARGETS_TO_BUILD:STRING="X86;WebAssembly" \ + -DLLVM_BUILD_BENCHMARKS:BOOL=OFF \ + -DLLVM_BUILD_DOCS:BOOL=OFF \ + -DLLVM_BUILD_EXAMPLES:BOOL=OFF \ + -DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF \ + -DLLVM_BUILD_TESTS:BOOL=OFF \ + -DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF \ + -DLLVM_INCLUDE_DOCS:BOOL=OFF \ + -DLLVM_INCLUDE_EXAMPLES:BOOL=OFF \ + -DLLVM_INCLUDE_TESTS:BOOL=OFF \ + -DLLVM_ENABLE_BINDINGS:BOOL=OFF \ + -DLLVM_ENABLE_LIBXML2:BOOL=ON \ + -DLLVM_ENABLE_LLD:BOOL=ON \ + -DLLDB_ENABLE_PYTHON:BOOL=ON \ + -DLLDB_EMBED_PYTHON_HOME=ON \ + -DLLDB_PYTHON_HOME=.. \ + -DLLDB_PYTHON_RELATIVE_PATH=lib/lldb-python \ + -DPython3_EXECUTABLE="$(pwd)/../python/bin/python${{ env.PYTHON_VERSION }}" + cmake --build build --target lldb install --parallel $(nproc) + working-directory: core/deps/llvm-project + + - name: validate lldb ubuntu + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'ubuntu') + run: | + echo "start to validate lldb..." + mkdir -p wamr-debug + cmake -S product-mini/platforms/linux -B wamr-debug -DWAMR_BUILD_DEBUG_INTERP=1 + cmake --build wamr-debug --parallel $(nproc) + export LD_LIBRARY_PATH=$(pwd)/core/deps/python/lib:${LD_LIBRARY_PATH} + python3 ci/validate_lldb.py --port 1239 --lldb core/deps/wamr-lldb/bin/lldb --wamr wamr-debug/iwasm --verbose + working-directory: . + + - name: build lldb macos + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'macos') + run: | + echo "start to build lldb..." + mkdir -p wamr-lldb + cmake -S ./llvm -B build \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX=../wamr-lldb \ + -DCMAKE_BUILD_TYPE:STRING="Release" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DLLVM_ENABLE_PROJECTS="clang;lldb" \ + -DLLVM_TARGETS_TO_BUILD:STRING="X86;WebAssembly" \ + -DLLVM_BUILD_BENCHMARKS:BOOL=OFF \ + -DLLVM_BUILD_DOCS:BOOL=OFF \ + -DLLVM_BUILD_EXAMPLES:BOOL=OFF \ + -DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF \ + -DLLVM_BUILD_TESTS:BOOL=OFF \ + -DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF \ + -DLLVM_INCLUDE_DOCS:BOOL=OFF \ + -DLLVM_INCLUDE_EXAMPLES:BOOL=OFF \ + -DLLVM_INCLUDE_TESTS:BOOL=OFF \ + -DLLVM_ENABLE_BINDINGS:BOOL=OFF \ + -DLLVM_ENABLE_LIBXML2:BOOL=ON \ + -DLLDB_BUILD_FRAMEWORK:BOOL=OFF \ + -DLLDB_ENABLE_PYTHON:BOOL=ON \ + -DLLDB_EMBED_PYTHON_HOME=ON \ + -DLLDB_PYTHON_HOME=.. \ + -DLLDB_PYTHON_RELATIVE_PATH=lib/lldb-python \ + -DPython3_EXECUTABLE="$(pwd)/../python/bin/python${{ env.PYTHON_VERSION }}" + cmake --build build --target lldb install --parallel $(nproc) + working-directory: core/deps/llvm-project + + - name: pack a distribution + if: steps.lldb_build_cache.outputs.cache-hit != 'true' + run: | + mkdir -p wamr-lldb/bin + mkdir -p wamr-lldb/lib + cp build/bin/lldb* wamr-lldb/bin + cp lldb/tools/lldb-vscode/package.json wamr-lldb + cp -r lldb/tools/lldb-vscode/syntaxes/ wamr-lldb + working-directory: core/deps/llvm-project + + - name: pack ubuntu specific libraries + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'ubuntu') + run: | + cp build/lib/liblldb*.so wamr-lldb/lib + cp build/lib/liblldb*.so.* wamr-lldb/lib + cp -R build/lib/lldb-python wamr-lldb/lib + cp -R ../python/lib/python* wamr-lldb/lib + cp ../python/lib/libpython${{ env.PYTHON_VERSION }}.so.1.0 wamr-lldb/lib + working-directory: core/deps/llvm-project + + - name: pack macos specific libraries + if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'macos') + run: | + cp build/lib/liblldb*.dylib wamr-lldb/lib + cp -R build/lib/lldb-python wamr-lldb/lib + cp -R ../python/lib/python* wamr-lldb/lib + cp ../python/lib/libpython*.dylib wamr-lldb/lib + install_name_tool -change /install/lib/libpython${{ env.PYTHON_VERSION }}.dylib @rpath/libpython${{ env.PYTHON_VERSION }}.dylib wamr-lldb/lib/liblldb.*.dylib + # Patch path of python library -> https://github.com/indygreg/python-build-standalone/blob/85923ca3911784e6978b85d56e06e9ae75cb2dc4/docs/quirks.rst?plain=1#L412-L446 + working-directory: core/deps/llvm-project + + - name: compress the binary + run: | + tar czf wamr-lldb-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz wamr-lldb + zip -r wamr-lldb-${{ inputs.ver_num }}-${{ inputs.runner }}.zip wamr-lldb + working-directory: core/deps/llvm-project + + - name: upload release tar.gz + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: core/deps/llvm-project/wamr-lldb-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz + asset_name: wamr-lldb-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: core/deps/llvm-project/wamr-lldb-${{ inputs.ver_num }}-${{ inputs.runner }}.zip + asset_name: wamr-lldb-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.zip + asset_content_type: application/zip diff --git a/wasm-micro-runtime/.github/workflows/build_wamr_sdk.yml b/wasm-micro-runtime/.github/workflows/build_wamr_sdk.yml new file mode 100644 index 0000000..519bf96 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/build_wamr_sdk.yml @@ -0,0 +1,103 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: build wamr-sdk + +on: + workflow_call: + inputs: + arch: + description: arch of the release + type: string + required: false + default: x86_64 + config_file: + description: warm-sdk config file path + type: string + required: true + runner: + description: OS of compilation + type: string + required: true + upload_url: + description: upload binary assets to the URL of release + type: string + required: true + ver_num: + description: a semantic version number + type: string + required: true + wasi_sdk_url: + description: download WASI_SDK from this URL + type: string + required: true + wamr_app_framework_url: + description: download WAMR app framework to get wamr_sdk + type: string + required: true + +jobs: + build: + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v4 + + - name: download wamr-app-framework + run: | + git clone ${{ inputs.wamr_app_framework_url }} + cd wamr-app-framework + git submodule init + git submodule update + working-directory: wamr-sdk + + - name: download and install wasi-sdk + run: | + cd /opt + basename=$(basename ${{ inputs.wasi_sdk_url }}) + sudo wget --progress=dot:giga ${{ inputs.wasi_sdk_url }} + sudo tar -xzf ${basename} + sudo rm ${basename} + sudo mv wasi-sdk-* wasi-sdk + + - name: download dependencies + run: | + cd ./wamr-app-framework/deps + ./download.sh + working-directory: wamr-sdk + + - name: generate wamr-sdk release + run: | + cd ./wamr-app-framework/wamr-sdk + ./build_sdk.sh -n wamr-sdk -x $(pwd)/${{ inputs.config_file }} + working-directory: wamr-sdk + + - name: compress the binary + run: | + cd wamr-app-framework/wamr-sdk/out + tar czf wamr-sdk-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz wamr-sdk + zip -r wamr-sdk-${{ inputs.ver_num }}-${{ inputs.runner }}.zip wamr-sdk + working-directory: wamr-sdk + + - name: upload release tar.gz + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: wamr-sdk/wamr-app-framework/wamr-sdk/out/wamr-sdk-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz + asset_name: wamr-sdk-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: wamr-sdk/wamr-app-framework/wamr-sdk/out/wamr-sdk-${{ inputs.ver_num }}-${{ inputs.runner }}.zip + asset_name: wamr-sdk-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.zip + asset_content_type: application/zip + + - name: delete wamr-app-framework + run: | + rm -rf wamr-app-framework + working-directory: wamr-sdk diff --git a/wasm-micro-runtime/.github/workflows/build_wamr_vscode_ext.yml b/wasm-micro-runtime/.github/workflows/build_wamr_vscode_ext.yml new file mode 100644 index 0000000..322ba1c --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/build_wamr_vscode_ext.yml @@ -0,0 +1,73 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: build wamr-ide vscode extension + +on: + workflow_call: + inputs: + upload_url: + description: upload binary assets to the URL of release + type: string + required: true + ver_num: + description: a semantic version number. + type: string + required: true + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 16.x + uses: actions/setup-node@v4 + with: + node-version: 16.x + + - name: set vscode extension to correct version + run: | + npm install -g json + json -I -f package.json -e "this.version=\"${{ inputs.ver_num }}\"" + working-directory: test-tools/wamr-ide/VSCode-Extension + + - name: generate wamr ide vscode extension + run: | + npm install -g vsce + rm -rf node_modules + npm install + vsce package + working-directory: test-tools/wamr-ide/VSCode-Extension + + - name: publish wamr ide vscode extension to the vsce marketplace + if: ${{ github.repository == 'bytecodealliance/wasm-micro-runtime' }} + run: | + vsce publish -p ${{ secrets.TOKEN }} + working-directory: test-tools/wamr-ide/VSCode-Extension + + - name: compress the vscode extension + run: | + mv wamride-*.vsix wamr-ide.vsix + tar czf wamr-ide-${{ inputs.ver_num }}.tar.gz wamr-ide.vsix + zip wamr-ide-${{ inputs.ver_num }}.zip wamr-ide.vsix + working-directory: test-tools/wamr-ide/VSCode-Extension + + - name: upload release tar.gz + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: test-tools/wamr-ide/VSCode-Extension/wamr-ide-${{ inputs.ver_num }}.tar.gz + asset_name: wamr-ide-${{ inputs.ver_num }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: test-tools/wamr-ide/VSCode-Extension/wamr-ide-${{ inputs.ver_num }}.zip + asset_name: wamr-ide-${{ inputs.ver_num }}.zip + asset_content_type: application/zip diff --git a/wasm-micro-runtime/.github/workflows/build_wamrc.yml b/wasm-micro-runtime/.github/workflows/build_wamrc.yml new file mode 100644 index 0000000..7c4dab0 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/build_wamrc.yml @@ -0,0 +1,86 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: build wamrc + +on: + workflow_call: + inputs: + arch: + description: arch of the release + type: string + required: false + default: x86_64 + llvm_cache_key: + description: the cache key of llvm libraries + type: string + required: true + release: + description: it is a part of the release process + type: boolean + required: true + runner: + description: OS of compilation + type: string + required: true + upload_url: + description: a semantic version number. it is required when `release` is true. + type: string + required: false + ver_num: + description: a semantic version number. it is required when `release` is true. + type: string + required: false + +jobs: + build: + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v4 + + - name: get cached LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ inputs.llvm_cache_key }} + fail-on-cache-miss: true + + - name: generate wamrc binary release + run: | + cmake -S . -B build + cmake --build build --config Release --parallel 4 + working-directory: wamr-compiler + + - name: compress the binary + if: inputs.release + run: | + tar czf wamrc-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz wamrc + zip wamrc-${{ inputs.ver_num }}-${{ inputs.runner }}.zip wamrc + working-directory: wamr-compiler/build + + - name: upload release tar.gz + if: inputs.release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: wamr-compiler/build/wamrc-${{ inputs.ver_num }}-${{ inputs.runner }}.tar.gz + asset_name: wamrc-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + if: inputs.release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: wamr-compiler/build/wamrc-${{ inputs.ver_num }}-${{ inputs.runner }}.zip + asset_name: wamrc-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}.zip + asset_content_type: application/zip diff --git a/wasm-micro-runtime/.github/workflows/codeql.yml b/wasm-micro-runtime/.github/workflows/codeql.yml new file mode 100644 index 0000000..d4e7d05 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/codeql.yml @@ -0,0 +1,117 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +name: "CodeQL" + +on: + #pull_request: + # types: + # - opened + # branches: '*' + #push: + # branches: [ "main" ] + # midnight UTC + schedule: + - cron: '0 0 * * *' + # allow to be triggered manually + workflow_dispatch: + +jobs: + analyze: + if: github.repository == 'bytecodealliance/wasm-micro-runtime' + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-20.04' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'cpp' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: recursive + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + queries: security-and-quality + + # Command-line programs to run using the OS shell. + # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + - run: | + ./.github/scripts/codeql_buildscript.sh + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" + upload: false + id: step1 + + # Filter out rules with low severity or high false positve rate + # Also filter out warnings in third-party code + - name: Filter out unwanted errors and warnings + uses: advanced-security/filter-sarif@v1 + with: + patterns: | + -**:cpp/path-injection + -**:cpp/world-writable-file-creation + -**:cpp/poorly-documented-function + -**:cpp/potentially-dangerous-function + -**:cpp/use-of-goto + -**:cpp/integer-multiplication-cast-to-long + -**:cpp/comparison-with-wider-type + -**:cpp/leap-year/* + -**:cpp/ambiguously-signed-bit-field + -**:cpp/suspicious-pointer-scaling + -**:cpp/suspicious-pointer-scaling-void + -**:cpp/unsigned-comparison-zero + -**/cmake*/Modules/** + input: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif + output: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif + + - name: Upload CodeQL results to code scanning + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: ${{ steps.step1.outputs.sarif-output }} + category: "/language:${{matrix.language}}" + + - name: Upload CodeQL results as an artifact + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: codeql-results + path: ${{ steps.step1.outputs.sarif-output }} + retention-days: 10 + + - name: Fail if an error is found + run: | + ./.github/scripts/codeql_fail_on_error.py \ + ${{ steps.step1.outputs.sarif-output }}/cpp.sarif + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/wasm-micro-runtime/.github/workflows/coding_guidelines.yml b/wasm-micro-runtime/.github/workflows/coding_guidelines.yml new file mode 100644 index 0000000..b0aa0a2 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/coding_guidelines.yml @@ -0,0 +1,28 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: Coding Guidelines + +on: + # will be triggered on PR events + pull_request: + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + compliance_job: + runs-on: ubuntu-20.04 + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # github.event.pull_request.base.label = ${{github.repository}}/${{github.base_ref}} + - name: Run Coding Guidelines Checks + run: /usr/bin/env python3 ./ci/coding_guidelines_check.py --commits ${{ github.event.pull_request.base.sha }}..HEAD diff --git a/wasm-micro-runtime/.github/workflows/compilation_on_android_ubuntu.yml b/wasm-micro-runtime/.github/workflows/compilation_on_android_ubuntu.yml new file mode 100644 index 0000000..6b2a1a1 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/compilation_on_android_ubuntu.yml @@ -0,0 +1,804 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: compilation on android, ubuntu-22.04 + +on: + # will be triggered on PR events + pull_request: + types: + - opened + - synchronize + paths: + - ".github/workflows/build_llvm_libraries.yml" + - ".github/workflows/compilation_on_android_ubuntu.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + - "test-tools/wamr-ide/**" + # will be triggered on push events + push: + branches: + - main + - "dev/**" + paths: + - ".github/workflows/build_llvm_libraries.yml" + - ".github/workflows/compilation_on_android_ubuntu.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + - "test-tools/wamr-ide/**" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # For BUILD + AOT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + FAST_INTERP_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + FAST_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + LLVM_LAZY_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" + MULTI_TIER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + # For Spec Test + DEFAULT_TEST_OPTIONS: "-s spec -b -P" + MULTI_MODULES_TEST_OPTIONS: "-s spec -b -M -P" + SIMD_TEST_OPTIONS: "-s spec -b -S -P" + THREADS_TEST_OPTIONS: "-s spec -b -p -P" + X86_32_TARGET_TEST_OPTIONS: "-m x86_32 -P" + WASI_TEST_OPTIONS: "-s wasi_certification -w" + WAMR_COMPILER_TEST_OPTIONS: "-s wamr_compiler -S -b -P" + GC_TEST_OPTIONS: "-s spec -G -b -P" + MEMORY64_TEST_OPTIONS: "-s spec -W -b -P" + +jobs: + build_llvm_libraries_on_ubuntu_2204: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-22.04" + arch: "X86" + + build_wamrc: + needs: + [build_llvm_libraries_on_ubuntu_2204] + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + # since jobs.id can't contain the dot character + # it is hard to use `format` to assemble the cache key + - name: Get LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Build wamrc + run: | + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + + build_iwasm: + needs: + [build_llvm_libraries_on_ubuntu_2204] + runs-on: ${{ matrix.os }} + strategy: + matrix: + make_options_run_mode: [ + # Running mode + $AOT_BUILD_OPTIONS, + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $FAST_JIT_BUILD_OPTIONS, + $LLVM_LAZY_JIT_BUILD_OPTIONS, + $LLVM_EAGER_JIT_BUILD_OPTIONS, + $MULTI_TIER_JIT_BUILD_OPTIONS, + ] + make_options_feature: [ + # Features + "-DWAMR_BUILD_CUSTOM_NAME_SECTION=1", + "-DWAMR_BUILD_DEBUG_AOT=1", + "-DWAMR_BUILD_DEBUG_INTERP=1", + "-DWAMR_BUILD_DUMP_CALL_STACK=1", + "-DWAMR_BUILD_LIB_PTHREAD=1", + "-DWAMR_BUILD_LIB_WASI_THREADS=1", + "-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1", + "-DWAMR_BUILD_MINI_LOADER=1", + "-DWAMR_BUILD_MEMORY_PROFILING=1", + "-DWAMR_BUILD_MULTI_MODULE=1", + "-DWAMR_BUILD_PERF_PROFILING=1", + "-DWAMR_BUILD_REF_TYPES=1", + "-DWAMR_BUILD_SIMD=1", + "-DWAMR_BUILD_TAIL_CALL=1", + "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_MEMORY64=1", + ] + os: [ubuntu-22.04] + platform: [android, linux] + exclude: + # uncompatiable feature and platform + # uncompatiable mode and feature + # MULTI_MODULE only on INTERP mode and AOT mode + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + # SIMD only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + # DEBUG_INTERP only on CLASSIC INTERP mode + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + # DEBUG_AOT only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # TODO: DEBUG_AOT on JIT + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # MINI_LOADER only on INTERP mode + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + # Memory64 only on CLASSIC INTERP mode, and only on 64-bit platform + - make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + platform: android + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + # Fast-JIT and Multi-Tier-JIT mode don't support android + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + platform: android + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + platform: android + # LLVM JIT pre-built binary wasn't compiled by Android NDK + # and isn't available for android + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + platform: android + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + platform: android + include: + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + # only download llvm cache when needed + - name: Get LLVM libraries + id: retrieve_llvm_libs + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Build iwasm for linux + if: matrix.platform == 'linux' + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/${{ matrix.platform }} + + - name: Build iwasm for android + if: matrix.platform == 'android' + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} \ + -DWAMR_BUILD_TARGET=X86_64 + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/${{ matrix.platform }} + + - name: Build and run unit tests + run: | + mkdir build-unittests && cd build-unittests + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} + cmake --build . --config Release --parallel 4 + ctest + working-directory: tests/unit + + build_samples_wasm_c_api: + needs: + [ + build_iwasm, + build_llvm_libraries_on_ubuntu_2204, + build_wamrc, + ] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + make_options: [ + # Running mode + $AOT_BUILD_OPTIONS, + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $FAST_JIT_BUILD_OPTIONS, + $LLVM_LAZY_JIT_BUILD_OPTIONS, + $LLVM_EAGER_JIT_BUILD_OPTIONS, + $MULTI_TIER_JIT_BUILD_OPTIONS, + ] + os: [ubuntu-22.04] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", + ] + include: + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Get LLVM libraries + id: retrieve_llvm_libs + if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: download and install wabt + run: | + cd /opt + sudo wget ${{ matrix.wabt_release }} + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt + + - name: Build wamrc + if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) + run: | + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + + - name: Build Sample [wasm-c-api] + run: | + VERBOSE=1 + cmake -S . -B build ${{ matrix.make_options }} + cmake --build build --config Debug --parallel 4 + ctest --test-dir build --output-on-failure + working-directory: samples/wasm-c-api + + build_samples_others: + needs: + [ + build_iwasm, + build_llvm_libraries_on_ubuntu_2204, + build_wamrc, + ] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", + ] + include: + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: download and install wasi-sdk + run: | + cd /opt + sudo wget ${{ matrix.wasi_sdk_release }} + sudo tar -xzf wasi-sdk-*.tar.gz + sudo ln -sf wasi-sdk-20.0 wasi-sdk + + - name: download and install wabt + run: | + cd /opt + sudo wget ${{ matrix.wabt_release }} + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo ln -sf wabt-1.0.31 wabt + - name: Get LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + - name: Build wamrc + run: | + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + - name: Build Sample [basic] + run: | + cd samples/basic + ./build.sh + ./run.sh + + - name: Build Sample [file] + run: | + cd samples/file + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./src/iwasm -f wasm-app/file.wasm -d . + + - name: Build Sample [multi-thread] + run: | + cd samples/multi-thread + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./iwasm wasm-apps/test.wasm + + - name: Build Sample [multi-module] + run: | + cd samples/multi-module + mkdir build && cd build + cmake .. -DWAMR_BUILD_AOT=1 + cmake --build . --config Debug --parallel 4 + ./multi_module mC.wasm + ./multi_module mC.aot + + - name: Build Sample [spawn-thread] + run: | + cd samples/spawn-thread + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./spawn_thread + + - name: Build Sample [ref-types] + run: | + cd samples/ref-types + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./hello + + - name: Build Sample [wasi-threads] + run: | + cd samples/wasi-threads + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./iwasm wasm-apps/no_pthread.wasm + + - name: Build Sample [shared-module] + run: | + cd samples/shared-module + ./build.sh + ./run.sh + + - name: Build Sample [terminate] + run: | + cd samples/terminate + ./build.sh + ./run.sh + + - name: Build Sample [debug-tools] + run: | + cd samples/debug-tools + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./iwasm wasm-apps/trap.wasm | grep "#" > call_stack.txt + ./iwasm wasm-apps/trap.aot | grep "#" > call_stack_aot.txt + bash -x ../symbolicate.sh + + test: + needs: + [ + build_iwasm, + build_llvm_libraries_on_ubuntu_2204, + build_wamrc, + ] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04] + running_mode: + [ + "classic-interp", + "fast-interp", + "jit", + "aot", + "fast-jit", + "multi-tier-jit", + ] + test_option: + [ + $DEFAULT_TEST_OPTIONS, + $MULTI_MODULES_TEST_OPTIONS, + $SIMD_TEST_OPTIONS, + $THREADS_TEST_OPTIONS, + $WASI_TEST_OPTIONS, + $GC_TEST_OPTIONS, + $MEMORY64_TEST_OPTIONS, + ] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz", + ] + include: + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + ubuntu_version: "22.04" + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + running_mode: aot + test_option: $WAMR_COMPILER_TEST_OPTIONS + exclude: + # uncompatiable modes and features + # classic-interp and fast-interp don't support simd + - running_mode: "classic-interp" + test_option: $SIMD_TEST_OPTIONS + - running_mode: "fast-interp" + test_option: $SIMD_TEST_OPTIONS + # llvm jit doesn't support multi module + - running_mode: "jit" + test_option: $MULTI_MODULES_TEST_OPTIONS + # fast-jit doesn't support multi module, simd + - running_mode: "fast-jit" + test_option: $MULTI_MODULES_TEST_OPTIONS + - running_mode: "fast-jit" + test_option: $SIMD_TEST_OPTIONS + # multi-tier-jit doesn't support multi module, simd + - running_mode: "multi-tier-jit" + test_option: $MULTI_MODULES_TEST_OPTIONS + - running_mode: "multi-tier-jit" + test_option: $SIMD_TEST_OPTIONS + # fast-jit and multi-tier-jit don't support GC + - running_mode: "fast-jit" + test_option: $GC_TEST_OPTIONS + - running_mode: "multi-tier-jit" + test_option: $GC_TEST_OPTIONS + # aot, fast-interp, fast-jit, llvm-jit, multi-tier-jit don't support Memory64 + - running_mode: "aot" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "fast-interp" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "fast-jit" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "jit" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "multi-tier-jit" + test_option: $MEMORY64_TEST_OPTIONS + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Set-up OCaml + uses: ocaml/setup-ocaml@v2 + if: matrix.test_option == '$GC_TEST_OPTIONS' || matrix.test_option == '$MEMORY64_TEST_OPTIONS' + with: + ocaml-compiler: 4.13 + + - name: Set-up Ocamlbuild + if: matrix.test_option == '$GC_TEST_OPTIONS' || matrix.test_option == '$MEMORY64_TEST_OPTIONS' + run: opam install ocamlbuild dune menhir + + - name: download and install wasi-sdk + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: | + cd /opt + sudo wget ${{ matrix.wasi_sdk_release }} + sudo tar -xzf wasi-sdk-*.tar.gz + sudo mv wasi-sdk-20.0 wasi-sdk + + # It is a temporary solution until new wasi-sdk that includes bug fixes is released + - name: build wasi-libc from source + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: | + git clone https://github.com/WebAssembly/wasi-libc + cd wasi-libc + make -j AR=/opt/wasi-sdk/bin/llvm-ar NM=/opt/wasi-sdk/bin/llvm-nm CC=/opt/wasi-sdk/bin/clang THREAD_MODEL=posix + echo "SYSROOT_PATH=$PWD/sysroot" >> $GITHUB_ENV + + - name: set env variable(if llvm are used) + if: matrix.running_mode == 'aot' || matrix.running_mode == 'jit' || matrix.running_mode == 'multi-tier-jit' + run: echo "USE_LLVM=true" >> $GITHUB_ENV + + - name: set env variable(if x86_32 test needed) + if: > + ((matrix.test_option == '$DEFAULT_TEST_OPTIONS' || matrix.test_option == '$THREADS_TEST_OPTIONS' + || matrix.test_option == '$WASI_TEST_OPTIONS' || matrix.test_option == '$GC_TEST_OPTIONS') + && matrix.running_mode != 'fast-jit' && matrix.running_mode != 'jit' && matrix.running_mode != 'multi-tier-jit') + run: echo "TEST_ON_X86_32=true" >> $GITHUB_ENV + + #only download llvm libraries in jit and aot mode + - name: Get LLVM libraries + if: env.USE_LLVM == 'true' + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: env.USE_LLVM == 'true' && steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: install jq JSON processor + if: matrix.running_mode == 'aot' && matrix.test_option == '$WASI_TEST_OPTIONS' + run: sudo apt-get update && sudo apt install -y jq + + - name: Build WASI thread tests + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: bash build.sh --sysroot "$SYSROOT_PATH" + working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/ + + - name: build socket api tests + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: bash build.sh + working-directory: ./core/iwasm/libraries/lib-socket/test/ + + - name: run tests + timeout-minutes: 30 + if: matrix.test_option != '$GC_TEST_OPTIONS' && matrix.test_option != '$MEMORY64_TEST_OPTIONS' + run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} + working-directory: ./tests/wamr-test-suites + + - name: run gc or memory64 tests + timeout-minutes: 20 + if: matrix.test_option == '$GC_TEST_OPTIONS' || matrix.test_option == '$MEMORY64_TEST_OPTIONS' + run: | + eval $(opam env) + ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} + working-directory: ./tests/wamr-test-suites + + #only install x32 support libraries when to run x86_32 cases + - name: install x32 support libraries + if: env.TEST_ON_X86_32 == 'true' + run: + # Add another apt repository as some packages cannot + # be downloaded with the github default repository + sudo curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc && + sudo apt-add-repository https://packages.microsoft.com/ubuntu/${{ matrix.ubuntu_version }}/prod && + sudo apt-get update && + sudo apt install -y g++-multilib lib32gcc-9-dev + + - name: run tests x86_32 + timeout-minutes: 30 + if: env.TEST_ON_X86_32 == 'true' && matrix.test_option != '$GC_TEST_OPTIONS' + run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} + working-directory: ./tests/wamr-test-suites + + - name: run gc tests x86_32 + timeout-minutes: 20 + if: env.TEST_ON_X86_32 == 'true' && matrix.test_option == '$GC_TEST_OPTIONS' + run: | + eval $(opam env) + ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} + working-directory: ./tests/wamr-test-suites + + test-wamr-ide: + needs: + [ + build_iwasm + ] + runs-on: ubuntu-22.04 + env: + PYTHON_VERSION: '3.10' + PYTHON_UBUNTU_STANDALONE_BUILD: https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.10.11+20230507-x86_64-unknown-linux-gnu-install_only.tar.gz + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: install dependencies + run: | + rustup target add wasm32-wasi + sudo apt update && sudo apt-get install -y lld ninja-build + npm install + working-directory: test-tools/wamr-ide/VSCode-Extension + + - name: code style check + run: | + npm install --save-dev prettier + npm run prettier-format-check + working-directory: test-tools/wamr-ide/VSCode-Extension + + - name: build iwasm with source debugging feature + run: | + mkdir build + cd build + cmake .. -DWAMR_BUILD_DEBUG_INTERP=1 + make + working-directory: product-mini/platforms/linux + + - name: Cache LLDB + id: cache-lldb + uses: actions/cache@v4 + env: + cache-name: cache-lldb-vscode + with: + path: test-tools/wamr-ide/VSCode-Extension/resource/debug/linux + key: ${{ env.cache-name }}-${{ hashFiles('build-scripts/lldb_wasm.patch') }}-${{ env.PYTHON_UBUNTU_STANDALONE_BUILD }} + + - if: ${{ steps.cache-lldb.outputs.cache-hit != 'true' }} + name: get stand-alone python ubuntu + run: | + wget ${{ env.PYTHON_UBUNTU_STANDALONE_BUILD }} -O python.tar.gz + tar -xvf python.tar.gz + working-directory: core/deps + + - if: ${{ steps.cache-lldb.outputs.cache-hit != 'true' }} + name: download llvm + run: | + wget https://github.com/llvm/llvm-project/archive/1f27fe6128769f00197925c3b8f6abb9d0e5cd2e.zip + unzip -q 1f27fe6128769f00197925c3b8f6abb9d0e5cd2e.zip + mv llvm-project-1f27fe6128769f00197925c3b8f6abb9d0e5cd2e llvm-project + working-directory: core/deps + + - if: ${{ steps.cache-lldb.outputs.cache-hit != 'true' }} + name: apply wamr patch + run: | + git init + git config user.email "action@github.com" + git config user.name "github action" + git apply ../../../build-scripts/lldb_wasm.patch + working-directory: core/deps/llvm-project + + - if: ${{ steps.cache-lldb.outputs.cache-hit != 'true' }} + name: build lldb ubuntu + run: | + echo "start to build lldb..." + mkdir -p wamr-lldb + cmake -S ./llvm -B build \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX=../wamr-lldb \ + -DCMAKE_BUILD_TYPE:STRING="Release" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DLLVM_ENABLE_PROJECTS="clang;lldb" \ + -DLLVM_TARGETS_TO_BUILD:STRING="X86;WebAssembly" \ + -DLLVM_BUILD_BENCHMARKS:BOOL=OFF \ + -DLLVM_BUILD_DOCS:BOOL=OFF \ + -DLLVM_BUILD_EXAMPLES:BOOL=OFF \ + -DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF \ + -DLLVM_BUILD_TESTS:BOOL=OFF \ + -DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF \ + -DLLVM_INCLUDE_DOCS:BOOL=OFF \ + -DLLVM_INCLUDE_EXAMPLES:BOOL=OFF \ + -DLLVM_INCLUDE_TESTS:BOOL=OFF \ + -DLLVM_ENABLE_BINDINGS:BOOL=OFF \ + -DLLVM_ENABLE_LIBXML2:BOOL=ON \ + -DLLVM_ENABLE_LLD:BOOL=ON \ + -DLLDB_ENABLE_PYTHON:BOOL=ON \ + -DLLDB_EMBED_PYTHON_HOME=ON \ + -DLLDB_PYTHON_HOME=.. \ + -DLLDB_PYTHON_RELATIVE_PATH=lib/lldb-python \ + -DPython3_EXECUTABLE="$(pwd)/../python/bin/python${{ env.PYTHON_VERSION }}" + cmake --build build --target lldb install --parallel $(nproc) + working-directory: core/deps/llvm-project + + - if: ${{ steps.cache-lldb.outputs.cache-hit != 'true' }} + name: copy lldb to extension folder + run: | + mkdir -p bin + mkdir -p lib + cp ../../../../../../core/deps/llvm-project/lldb/tools/lldb-vscode/package.json ./ + cp -r ../../../../../../core/deps/llvm-project/lldb/tools/lldb-vscode/syntaxes/ ./ + cp ../../../../../../core/deps/llvm-project/build/bin/lldb* bin + cp ../../../../../../core/deps/llvm-project/build/lib/liblldb*.so lib + cp ../../../../../../core/deps/llvm-project/build/lib/liblldb*.so.* lib + cp -R ../../../../../../core/deps/llvm-project/build/lib/lldb-python lib + cp -R ../../../../../../core/deps/python/lib/python* lib + cp ../../../../../../core/deps/python/lib/libpython${{ env.PYTHON_VERSION }}.so.1.0 lib + working-directory: test-tools/wamr-ide/VSCode-Extension/resource/debug/linux + + - name: run tests + timeout-minutes: 5 + run: xvfb-run npm run test + working-directory: test-tools/wamr-ide/VSCode-Extension diff --git a/wasm-micro-runtime/.github/workflows/compilation_on_macos.yml b/wasm-micro-runtime/.github/workflows/compilation_on_macos.yml new file mode 100644 index 0000000..ec09432 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/compilation_on_macos.yml @@ -0,0 +1,381 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: compilation on macos + +on: + # will be triggered on PR events + pull_request: + types: + - opened + - synchronize + paths: + - ".github/workflows/build_llvm_libraries.yml" + - ".github/workflows/compilation_on_macos.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # will be triggered on push events + push: + branches: + - main + - "dev/**" + paths: + - ".github/workflows/build_llvm_libraries.yml" + - ".github/workflows/compilation_on_macos.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + LLVM_LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" + +jobs: + build_llvm_libraries_on_intel_macos: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "macos-latest" + arch: "X86" + build_llvm_libraries_on_arm_macos: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "macos-14" + arch: "AArch64 ARM" + + build_wamrc: + needs: [build_llvm_libraries_on_intel_macos] + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-latest + llvm_cache_key: ${{ needs.build_llvm_libraries_on_intel_macos.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Get LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Build wamrc + run: | + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + + build_iwasm: + needs: [build_llvm_libraries_on_intel_macos] + runs-on: ${{ matrix.os }} + strategy: + matrix: + make_options_run_mode: [ + # Running mode + $AOT_BUILD_OPTIONS, + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $LLVM_LAZY_JIT_BUILD_OPTIONS, + $LLVM_EAGER_JIT_BUILD_OPTIONS, + ] + make_options_feature: [ + # Features + "-DWAMR_BUILD_CUSTOM_NAME_SECTION=1", + # doesn't support + #"-DWAMR_BUILD_DEBUG_AOT=1", + "-DWAMR_BUILD_DEBUG_INTERP=1", + "-DWAMR_BUILD_DUMP_CALL_STACK=1", + "-DWAMR_BUILD_LIB_PTHREAD=1", + "-DWAMR_BUILD_LIB_WASI_THREADS=1", + "-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1", + "-DWAMR_BUILD_MINI_LOADER=1", + "-DWAMR_BUILD_MEMORY_PROFILING=1", + "-DWAMR_BUILD_MULTI_MODULE=1", + "-DWAMR_BUILD_PERF_PROFILING=1", + "-DWAMR_BUILD_REF_TYPES=1", + "-DWAMR_BUILD_SIMD=1", + "-DWAMR_BUILD_TAIL_CALL=1", + "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + ] + os: [macos-latest] + platform: [darwin] + exclude: + # uncompatiable feature and platform + # uncompatiable mode and feature + # MULTI_MODULE only on INTERP mode and AOT mode + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + # SIMD only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + # DEBUG_INTERP only on CLASSIC INTERP mode + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + # DEBUG_AOT only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # TODO: DEBUG_AOT on JIT + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # MINI_LOADER only on INTERP mode + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + include: + - os: macos-latest + llvm_cache_key: ${{ needs.build_llvm_libraries_on_intel_macos.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + # only download llvm cache when needed + - name: Get LLVM libraries + id: retrieve_llvm_libs + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Build iwasm + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/${{ matrix.platform }} + + build_samples_wasm_c_api: + needs: [build_iwasm] + runs-on: ${{ matrix.os }} + strategy: + matrix: + make_options: [ + # Running modes supported + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + # Running modes unsupported + #$LLVM_LAZY_JIT_BUILD_OPTIONS, + #$LLVM_EAGER_JIT_BUILD_OPTIONS, + #$AOT_BUILD_OPTIONS, + ] + os: [macos-latest] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-macos.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-macos-12.tar.gz", + ] + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: download and install wabt + run: | + cd /opt + sudo wget ${{ matrix.wabt_release }} + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt + + - name: Build Sample [wasm-c-api] + run: | + cmake -S . -B build ${{ matrix.make_options }} + cmake --build build --config Release --parallel 4 + ctest --test-dir build --output-on-failure + working-directory: samples/wasm-c-api + + build_samples_others: + needs: [build_iwasm, build_wamrc, build_llvm_libraries_on_intel_macos, build_llvm_libraries_on_arm_macos] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, macos-14] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-macos.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-macos-12.tar.gz", + ] + include: + - os: macos-latest + llvm_cache_key: ${{ needs.build_llvm_libraries_on_intel_macos.outputs.cache_key }} + - os: macos-14 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_arm_macos.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: download and install wasi-sdk + run: | + cd /opt + sudo wget ${{ matrix.wasi_sdk_release }} + sudo tar -xzf wasi-sdk-*.tar.gz + sudo ln -sf wasi-sdk-20.0 wasi-sdk + + - name: download and install wabt + run: | + cd /opt + sudo wget ${{ matrix.wabt_release }} + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo ln -sf wabt-1.0.31 wabt + + - name: Build Sample [basic] + run: | + cd samples/basic + ./build.sh + ./run.sh + + - name: Build Sample [file] + run: | + cd samples/file + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./src/iwasm -f wasm-app/file.wasm -d . + + - name: Build Sample [multi-thread] + run: | + cd samples/multi-thread + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./iwasm wasm-apps/test.wasm + + - name: Build Sample [multi-module] + run: | + cd samples/multi-module + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./multi_module mC.wasm + + - name: Build Sample [spawn-thread] + run: | + cd samples/spawn-thread + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./spawn_thread + + - name: Build Sample [ref-types] + run: | + cd samples/ref-types + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./hello + + - name: Get LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Build wamrc + run: | + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + + - name: Build Sample [wasi-threads] + run: | + cd samples/wasi-threads + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./iwasm wasm-apps/no_pthread.wasm + + ../../../wamr-compiler/build/wamrc --size-level=0 --enable-multi-thread -o wasm-apps/no_pthread.aot wasm-apps/no_pthread.wasm + ./iwasm wasm-apps/no_pthread.aot + + - name: Build Sample [shared-module] + run: | + cd samples/shared-module + ./build.sh + ./run.sh + + - name: Build Sample [terminate] + run: | + cd samples/terminate + ./build.sh + ./run.sh + + - name: Build Sample [debug-tools] + run: | + cd samples/debug-tools + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./iwasm wasm-apps/trap.wasm | grep "#" > call_stack.txt + ./iwasm wasm-apps/trap.aot | grep "#" > call_stack_aot.txt + bash -x ../symbolicate.sh diff --git a/wasm-micro-runtime/.github/workflows/compilation_on_nuttx.yml b/wasm-micro-runtime/.github/workflows/compilation_on_nuttx.yml new file mode 100644 index 0000000..98cbb24 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/compilation_on_nuttx.yml @@ -0,0 +1,116 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: compilation on nuttx + +on: + # will be triggered on PR events + pull_request: + types: + - opened + - synchronize + paths: + - ".github/workflows/compilation_on_nuttx.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # will be triggered on push events + push: + branches: + - main + - "dev/**" + paths: + - ".github/workflows/compilation_on_nuttx.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + WASI_SDK_PATH: "/opt/wasi-sdk" + +jobs: + build_iwasm_on_nuttx: + runs-on: ubuntu-latest + container: + image: ghcr.io/apache/nuttx/apache-nuttx-ci-linux@sha256:d9261eacf6c6ebe656c571757751c803e8f04c3ae9b820320a5ea5dd57b7205a + + strategy: + matrix: + nuttx_board_config: [ + # x64 + "boards/sim/sim/sim/configs/nsh", + # cortex-m0 + "boards/arm/rp2040/raspberrypi-pico/configs/nsh", + # cortex-m7 + "boards/arm/stm32h7/nucleo-h743zi/configs/nsh", + # riscv32imc + "boards/risc-v/espressif/esp32c3-generic/configs/nsh", + # riscv32gc + "boards/risc-v/qemu-rv/rv-virt/configs/nsh", + # riscv64gc + "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", + # arm64 + "boards/arm64/qemu/qemu-armv8a/configs/nsh", + ] + wamr_config_option: [ + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_FAST=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_FAST=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_WASI=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_FAST=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_CLASSIC=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_CLASSIC=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_WASI=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_CLASSIC=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_FAST=y\\n", + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_CLASSIC=y\\n", + ] + + steps: + - name: Checkout NuttX + uses: actions/checkout@v4 + with: + repository: apache/incubator-nuttx + ref: releases/12.4 + path: nuttx + + - name: Checkout NuttX Apps + uses: actions/checkout@v4 + with: + repository: apache/incubator-nuttx-apps + ref: releases/12.4 + path: apps + + - name: Checkout WAMR + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + path: apps/interpreters/wamr/wamr + + - name: Enable WAMR for NuttX + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_EOL_IS_LF=y\n${{ matrix.wamr_config_option }}' + find nuttx/boards/sim -name defconfig | xargs sed -i '$a\CONFIG_LIBM=y\n' + + - name: Build + run: | + cd nuttx + tools/configure.sh ${{ matrix.nuttx_board_config }} + make -j$(nproc) EXTRAFLAGS=-Werror diff --git a/wasm-micro-runtime/.github/workflows/compilation_on_sgx.yml b/wasm-micro-runtime/.github/workflows/compilation_on_sgx.yml new file mode 100644 index 0000000..030c765 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/compilation_on_sgx.yml @@ -0,0 +1,327 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: compilation on SGX + +on: + # will be triggered on PR events + pull_request: + types: + - opened + - synchronize + paths: + - ".github/workflows/build_llvm_libraries.yml" + - ".github/workflows/compilation_on_sgx.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # will be triggered on push events + push: + branches: + - main + - "dev/**" + paths: + - ".github/workflows/build_llvm_libraries.yml" + - ".github/workflows/compilation_on_sgx.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # ref types enabled in wamrc by default, so we need to enable it for iwasm in AOT mode + AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0 -DWAMR_BUILD_REF_TYPES=1" + CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + FAST_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=1" + LLVM_LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" + +jobs: + build_llvm_libraries: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-20.04" + arch: "X86" + + build_iwasm: + runs-on: ${{ matrix.os }} + strategy: + matrix: + make_options_run_mode: [ + # Running modes supported + $AOT_BUILD_OPTIONS, + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $FAST_JIT_BUILD_OPTIONS, + # Running modes unsupported + #$LLVM_LAZY_JIT_BUILD_OPTIONS, + #$LLVM_EAGER_JIT_BUILD_OPTIONS, + ] + make_options_feature: [ + # Features + "-DWAMR_BUILD_CUSTOM_NAME_SECTION=1", + # doesn't support + # "-DWAMR_BUILD_DEBUG_AOT=1", + # "-DWAMR_BUILD_DEBUG_INTERP=1", + "-DWAMR_BUILD_DUMP_CALL_STACK=1", + "-DWAMR_BUILD_LIB_PTHREAD=1", + "-DWAMR_BUILD_LIB_WASI_THREADS=1", + "-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1", + "-DWAMR_BUILD_MINI_LOADER=1", + "-DWAMR_BUILD_MEMORY_PROFILING=1", + "-DWAMR_BUILD_MULTI_MODULE=1", + "-DWAMR_BUILD_PERF_PROFILING=1", + "-DWAMR_BUILD_REF_TYPES=1", + # doesn't support + # "-DWAMR_BUILD_SIMD=1", + "-DWAMR_BUILD_TAIL_CALL=1", + "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_SGX_IPFS=1", + ] + os: [ubuntu-20.04] + platform: [linux-sgx] + exclude: + # uncompatiable mode and feature + # MINI_LOADER only on INTERP mode + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + steps: + - name: install SGX SDK and necessary libraries + run: | + mkdir -p /opt/intel + cd /opt/intel + wget https://download.01.org/intel-sgx/sgx-linux/2.15/distro/ubuntu20.04-server/sgx_linux_x64_sdk_2.15.100.3.bin + chmod +x sgx_linux_x64_sdk_2.15.100.3.bin + echo 'yes' | ./sgx_linux_x64_sdk_2.15.100.3.bin + echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list + wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - + sudo apt update + sudo apt install -y libsgx-launch libsgx-urts + source /opt/intel/sgxsdk/environment + + - name: checkout + uses: actions/checkout@v4 + + - name: Build iwasm + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} + cmake --build . --config Release --parallel 4 + cd ../enclave-sample + make + working-directory: product-mini/platforms/${{ matrix.platform }} + + run_samples_file: + needs: [build_iwasm, build_llvm_libraries] + runs-on: ${{ matrix.os }} + strategy: + matrix: + iwasm_make_options_run_mode: [ + # Running modes supported + $AOT_BUILD_OPTIONS, + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $FAST_JIT_BUILD_OPTIONS, + # Running modes unsupported + #$LLVM_LAZY_JIT_BUILD_OPTIONS, + #$LLVM_EAGER_JIT_BUILD_OPTIONS, + ] + os: [ubuntu-20.04] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", + ] + iwasm_make_options_feature: [ + # Features to be tested: IPFS + "-DWAMR_BUILD_SGX_IPFS=1", + ] + platform: [linux-sgx] + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries.outputs.cache_key }} + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: download and install wasi-sdk + run: | + cd /opt + sudo wget ${{ matrix.wasi_sdk_release }} + sudo tar -xzf wasi-sdk-*.tar.gz + sudo mv wasi-sdk-19.0 wasi-sdk + + - name: download and install wabt + run: | + cd /opt + sudo wget ${{ matrix.wabt_release }} + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt + + - name: build wasi-libc (needed for wasi-threads) + run: | + mkdir wasi-libc + cd wasi-libc + git init + # "Fix a_store operation in atomic.h" commit on main branch + git fetch https://github.com/WebAssembly/wasi-libc \ + 1dfe5c302d1c5ab621f7abf04620fae92700fd22 + git checkout FETCH_HEAD + make \ + AR=/opt/wasi-sdk/bin/llvm-ar \ + NM=/opt/wasi-sdk/bin/llvm-nm \ + CC=/opt/wasi-sdk/bin/clang \ + THREAD_MODEL=posix + working-directory: core/deps + + - name: install SGX SDK and necessary libraries + run: | + mkdir -p /opt/intel + cd /opt/intel + wget https://download.01.org/intel-sgx/sgx-linux/2.15/distro/ubuntu20.04-server/sgx_linux_x64_sdk_2.15.100.3.bin + chmod +x sgx_linux_x64_sdk_2.15.100.3.bin + echo 'yes' | ./sgx_linux_x64_sdk_2.15.100.3.bin + echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list + wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - + sudo apt update + sudo apt install -y libsgx-launch libsgx-urts + + - name: Build iwasm for testing samples + run: | + mkdir build && cd build + cmake .. ${{ matrix.iwasm_make_options_run_mode }} ${{ matrix.iwasm_make_options_feature }} + cmake --build . --config Release --parallel 4 + cd ../enclave-sample + make + working-directory: product-mini/platforms/${{ matrix.platform }} + + - name: Get LLVM libraries + if: matrix.iwasm_make_options_run_mode == '$AOT_BUILD_OPTIONS' + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + fail-on-cache-miss: true + + - name: Build wamrc only for testing samples in aot mode + if: matrix.iwasm_make_options_run_mode == '$AOT_BUILD_OPTIONS' + run: | + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + cp wamrc `pwd`/../../product-mini/platforms/${{ matrix.platform }}/enclave-sample + working-directory: wamr-compiler + + - name: Build Sample [file] + run: | + cd samples/file + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + cp wasm-app/file.wasm `pwd`/../../../product-mini/platforms/${{ matrix.platform }}/enclave-sample + + - name: Test Sample [file] in non-aot mode + if: matrix.iwasm_make_options_run_mode != '$AOT_BUILD_OPTIONS' + run: | + source /opt/intel/sgxsdk/environment + ./iwasm --dir=. file.wasm + working-directory: product-mini/platforms/${{ matrix.platform }}/enclave-sample + + - name: Test Sample [file] in aot mode + if: matrix.iwasm_make_options_run_mode == '$AOT_BUILD_OPTIONS' + run: | + source /opt/intel/sgxsdk/environment + ./wamrc -sgx -o file.aot file.wasm + ./iwasm --dir=. file.aot + working-directory: product-mini/platforms/${{ matrix.platform }}/enclave-sample + + spec_test_default: + needs: [build_iwasm, build_llvm_libraries] + runs-on: ubuntu-20.04 + strategy: + matrix: + running_mode: ["classic-interp", "fast-interp", "aot", "fast-jit"] + test_option: ["-x -p -s spec -b -P", "-x -p -s spec -S -b -P", "-x -p -s spec -X -b -P"] + llvm_cache_key: ["${{ needs.build_llvm_libraries.outputs.cache_key }}"] + exclude: + # classic-interp, fast-interp and fast-jit don't support simd + - running_mode: "classic-interp" + test_option: "-x -p -s spec -S -b -P" + - running_mode: "fast-interp" + test_option: "-x -p -s spec -S -b -P" + - running_mode: "fast-jit" + test_option: "-x -p -s spec -S -b -P" + # classic-interp, fast-interp and fast jit don't support XIP + - running_mode: "classic-interp" + test_option: "-x -p -s spec -X -b -P" + - running_mode: "fast-interp" + test_option: "-x -p -s spec -X -b -P" + - running_mode: "fast-jit" + test_option: "-x -p -s spec -X -b -P" + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Get LLVM libraries + if: matrix.running_mode == 'aot' + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: matrix.running_mode == 'aot' && steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: install SGX SDK and necessary libraries + run: | + mkdir -p /opt/intel + cd /opt/intel + wget https://download.01.org/intel-sgx/sgx-linux/2.15/distro/ubuntu20.04-server/sgx_linux_x64_sdk_2.15.100.3.bin + chmod +x sgx_linux_x64_sdk_2.15.100.3.bin + echo 'yes' | ./sgx_linux_x64_sdk_2.15.100.3.bin + echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list + wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - + sudo apt update + sudo apt install -y libsgx-launch libsgx-urts + + - name: run spec tests + run: | + source /opt/intel/sgxsdk/environment + ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} + working-directory: ./tests/wamr-test-suites diff --git a/wasm-micro-runtime/.github/workflows/compilation_on_windows.yml b/wasm-micro-runtime/.github/workflows/compilation_on_windows.yml new file mode 100644 index 0000000..8c5db4f --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/compilation_on_windows.yml @@ -0,0 +1,134 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: compilation on windows-latest + +on: + # will be triggered on PR events + pull_request: + types: + - opened + - synchronize + paths: + - ".github/workflows/compilation_on_windows.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # will be triggered on push events + push: + branches: + - main + - "dev/**" + paths: + - ".github/workflows/compilation_on_windows.yml" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # allow to be triggered manually + workflow_dispatch: + +env: + # For Spec Test + DEFAULT_TEST_OPTIONS: "-s spec -b" + MULTI_MODULES_TEST_OPTIONS: "-s spec -b -M" + THREADS_TEST_OPTIONS: "-s spec -b -p" + WASI_TEST_OPTIONS: "-s wasi_certification -w" + WASI_TEST_FILTER: ${{ github.workspace }}/product-mini/platforms/windows/wasi_filtered_tests.json + # Used when building the WASI socket and thread tests + CC: ${{ github.workspace }}/wasi-sdk/bin/clang + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: windows-latest + strategy: + matrix: + build_options: [ + "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_INTERP=0", + "-DWAMR_BUILD_AOT=0", + "-DWAMR_BUILD_TAIL_CALL=1", + "-DWAMR_BUILD_CUSTOM_NAME_SECTION=1", + "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_REF_TYPES=1", + "-DWAMR_BUILD_SIMD=1", + "-DWAMR_BUILD_DEBUG_INTERP=1", + "-DWAMR_BUILD_LIB_PTHREAD=1", + "-DWAMR_BUILD_LIB_WASI_THREADS=1", + "-DWAMR_BUILD_LIBC_UVWASI=0 -DWAMR_BUILD_LIBC_WASI=1" + ] + steps: + - uses: actions/checkout@v4 + + - name: clone uvwasi library + if: ${{ !contains(matrix.build_options, '-DWAMR_BUILD_LIBC_UVWASI=0') }} + run: | + cd core/deps + git clone https://github.com/nodejs/uvwasi.git + - name: Build iwasm + run: | + cd product-mini/platforms/windows + mkdir build && cd build + cmake .. ${{ matrix.build_options }} + cmake --build . --config Release --parallel 4 + + test: + runs-on: windows-latest + needs: [build] + strategy: + fail-fast: false + matrix: + running_mode: + [ + "classic-interp", + "fast-interp", + ] + test_option: + [ + $DEFAULT_TEST_OPTIONS, + $MULTI_MODULES_TEST_OPTIONS, + $THREADS_TEST_OPTIONS, + $WASI_TEST_OPTIONS, + ] + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: download and install wasi-sdk + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: | + curl "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0.m-mingw.tar.gz" -o wasi-sdk.tar.gz -L + mkdir wasi-sdk + tar -xzf wasi-sdk.tar.gz -C wasi-sdk --strip-components 1 + + - name: build socket api tests + shell: bash + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: ./build.sh + working-directory: ./core/iwasm/libraries/lib-socket/test/ + + - name: Build WASI thread tests + shell: bash + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: ./build.sh + working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/ + + - name: run tests + shell: bash + timeout-minutes: 20 + run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} + working-directory: ./tests/wamr-test-suites diff --git a/wasm-micro-runtime/.github/workflows/create_tag.yml b/wasm-micro-runtime/.github/workflows/create_tag.yml new file mode 100644 index 0000000..5480592 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/create_tag.yml @@ -0,0 +1,82 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: create a tag + +on: + workflow_call: + outputs: + minor_version: + description: "the new version is a minor version or a major version" + value: ${{ jobs.create_tag.outputs.minor_version}} + new_ver: + description: "the new version" + value: ${{ jobs.create_tag.outputs.new_ver}} + new_tag: + description: "the new tag just created" + value: ${{ jobs.create_tag.outputs.new_tag}} + +jobs: + create_tag: + runs-on: ubuntu-latest + outputs: + minor_version: ${{ steps.preparation.outputs.minor_version }} + new_ver: ${{ steps.preparation.outputs.new_ver }} + new_tag: ${{ steps.preparation.outputs.new_tag }} + + steps: + - uses: actions/checkout@v4 + # Full git history is needed to get a proper list of commits and tags + with: + fetch-depth: 0 + + - name: prepare + id: preparation + run: | + # show latest 3 versions on the branch that create release + # Set the initial commit to the head of the branch + commit="HEAD" + # + # Loop to get the three most recent tags + for i in {1..3} + do + # Get the most recent tag reachable from the current commit + tag=$(git describe --tags --abbrev=0 $commit) + + # Print the tag + echo "$tag" + + # Move to the commit before the found tag to find the next tag in the next iteration + commit=$(git rev-list -n 1 $tag^) + done + # compare latest git tag and semantic version definition + result=$(python3 ./.github/scripts/fetch_and_compare_version.py) + echo "script result is ${result}" + # + # return in a form like "WAMR-X.Y.Z,major_minor_change" or ",patch_change" + new_ver=$(echo "${result}" | awk -F',' '{print $1}') + diff_versioning=$(echo "${result}" | awk -F',' '{print $2}') + echo "next version is ${new_ver}, it ${diff_versioning}" + # + # set output + if [[ ${diff_versioning} == 'major_minor_change' ]];then + echo "minor_version=true" >> "$GITHUB_OUTPUT" + else + echo "minor_version=false" >> "$GITHUB_OUTPUT" + fi + # + # + if [[ -z ${new_ver} ]]; then + echo "::error::please indicate the right semantic version in core/version.h" + echo "new_ver=''" >> "$GITHUB_OUTPUT" + echo "new_tag=''" >> "$GITHUB_OUTPUT" + exit 1 + else + echo "new_ver=${new_ver}" >> "$GITHUB_OUTPUT" + echo "new_tag=WAMR-${new_ver}" >> "$GITHUB_OUTPUT" + fi + + - name: push tag + if: steps.preparation.outputs.new_tag != '' + run: | + git tag ${{ steps.preparation.outputs.new_tag }} + git push origin --force --tags diff --git a/wasm-micro-runtime/.github/workflows/hadolint_dockerfiles.yml b/wasm-micro-runtime/.github/workflows/hadolint_dockerfiles.yml new file mode 100644 index 0000000..c540649 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/hadolint_dockerfiles.yml @@ -0,0 +1,47 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: hadolint dockerfiles + +on: + # will be triggered on PR events + pull_request: + types: + - opened + - synchronize + paths: + - "**/Dockerfile*" + - ".github/workflows/hadolint_dockerfiles.yml" + push: + branches: + - main + - "dev/**" + paths: + - "**/Dockerfile*" + - ".github/workflows/hadolint_dockerfiles.yml" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-hadolint-on-dockerfiles: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # on default, hadolint will fail on warnings and errors + - name: Run hadolint on dockerfiles + run: | + docker pull hadolint/hadolint:latest-debian + find . -name "*Dockerfile*" | while read dockerfile; do + echo "run hadolint on $dockerfile:" + docker run --rm -i hadolint/hadolint:latest-debian hadolint - <"$dockerfile" + echo "successful" + done \ No newline at end of file diff --git a/wasm-micro-runtime/.github/workflows/nightly_run.yml b/wasm-micro-runtime/.github/workflows/nightly_run.yml new file mode 100644 index 0000000..4b62d11 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/nightly_run.yml @@ -0,0 +1,729 @@ +# Copyright (C) 2023 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: nightly_run + +on: + pull_request: + types: + - opened + - synchronize + # running nightly pipeline if you're changing it + # stress tests are run only in nightly at the moment, so running them in they are changed + paths: + - ".github/workflows/nightly_run.yml" + - "core/iwasm/libraries/lib-wasi-threads/stress-test/**" + + # midnight UTC + schedule: + - cron: "0 0 * * *" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # For BUILD + AOT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + FAST_INTERP_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + FAST_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + LLVM_LAZY_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" + MULTI_TIER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + # For Spec Test + DEFAULT_TEST_OPTIONS: "-s spec -b -P" + MULTI_MODULES_TEST_OPTIONS: "-s spec -b -M -P" + SIMD_TEST_OPTIONS: "-s spec -b -S -P" + THREADS_TEST_OPTIONS: "-s spec -b -p -P" + X86_32_TARGET_TEST_OPTIONS: "-m x86_32 -P" + WASI_TEST_OPTIONS: "-s wasi_certification -w" + +jobs: + build_llvm_libraries_on_ubuntu_2004: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-20.04" + arch: "X86" + build_llvm_libraries_on_ubuntu_2204: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-22.04" + arch: "X86" + + build_wamrc: + needs: + [ + build_llvm_libraries_on_ubuntu_2004, + ] + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + # since jobs.id can't contain the dot character + # it is hard to use `format` to assemble the cache key + - name: Get LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Build wamrc + run: | + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + + build_iwasm: + needs: + [ + build_llvm_libraries_on_ubuntu_2004, + ] + runs-on: ${{ matrix.os }} + strategy: + matrix: + make_options_run_mode: [ + # Running mode + $AOT_BUILD_OPTIONS, + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $FAST_JIT_BUILD_OPTIONS, + $LLVM_LAZY_JIT_BUILD_OPTIONS, + $LLVM_EAGER_JIT_BUILD_OPTIONS, + $MULTI_TIER_JIT_BUILD_OPTIONS, + ] + make_options_feature: [ + # Features + "-DWAMR_BUILD_CUSTOM_NAME_SECTION=1", + "-DWAMR_BUILD_DEBUG_AOT=1", + "-DWAMR_BUILD_DEBUG_INTERP=1", + "-DWAMR_BUILD_DUMP_CALL_STACK=1", + "-DWAMR_BUILD_LIB_PTHREAD=1", + "-DWAMR_BUILD_LIB_WASI_THREADS=1", + "-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1", + "-DWAMR_BUILD_MINI_LOADER=1", + "-DWAMR_BUILD_MEMORY_PROFILING=1", + "-DWAMR_BUILD_MULTI_MODULE=1", + "-DWAMR_BUILD_PERF_PROFILING=1", + "-DWAMR_BUILD_REF_TYPES=1", + "-DWAMR_BUILD_SIMD=1", + "-DWAMR_BUILD_TAIL_CALL=1", + "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_MEMORY64=1", + ] + os: [ubuntu-20.04] + platform: [android, linux] + exclude: + # uncompatiable feature and platform + # uncompatiable mode and feature + # MULTI_MODULE only on INTERP mode and AOT mode + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + # SIMD only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + # DEBUG_INTERP only on CLASSIC INTERP mode + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + # DEBUG_AOT only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # TODO: DEBUG_AOT on JIT + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # MINI_LOADER only on INTERP mode + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + # Memory64 only on CLASSIC INTERP mode, and only on 64-bit platform + - make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + platform: android + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + # Fast-JIT and Multi-Tier-JIT mode don't support android + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + platform: android + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + platform: android + # LLVM JIT pre-built binary wasn't compiled by Android NDK + # and isn't available for android + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + platform: android + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + platform: android + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + + steps: + - name: checkout + uses: actions/checkout@v4 + + # only download llvm cache when needed + - name: Get LLVM libraries + id: retrieve_llvm_libs + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Build iwasm for linux + if: matrix.platform == 'linux' + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/${{ matrix.platform }} + + - name: Build iwasm for android + if: matrix.platform == 'android' + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} \ + -DWAMR_BUILD_TARGET=X86_64 + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/${{ matrix.platform }} + + build_iwasm_linux_gcc4_8: + runs-on: ubuntu-latest + container: + image: ubuntu:14.04 + strategy: + matrix: + make_options_run_mode: [ + # Running mode + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $FAST_JIT_BUILD_OPTIONS, + ] + make_options_feature: [ + # Features + "-DWAMR_BUILD_CUSTOM_NAME_SECTION=1", + "-DWAMR_BUILD_DEBUG_AOT=1", + "-DWAMR_BUILD_DEBUG_INTERP=1", + "-DWAMR_BUILD_DUMP_CALL_STACK=1", + "-DWAMR_BUILD_LIB_PTHREAD=1", + "-DWAMR_BUILD_LIB_WASI_THREADS=1", + "-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1", + "-DWAMR_BUILD_MINI_LOADER=1", + "-DWAMR_BUILD_MEMORY_PROFILING=1", + "-DWAMR_BUILD_MULTI_MODULE=1", + "-DWAMR_BUILD_PERF_PROFILING=1", + "-DWAMR_BUILD_REF_TYPES=1", + "-DWAMR_BUILD_SIMD=1", + "-DWAMR_BUILD_TAIL_CALL=1", + "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_MEMORY64=1", + ] + exclude: + # uncompatiable feature and platform + # uncompatiable mode and feature + # MULTI_MODULE only on INTERP mode and AOT mode + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" + # SIMD only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_SIMD=1" + # DEBUG_INTERP only on CLASSIC INTERP mode + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + # DEBUG_AOT only on JIT/AOT mode + - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # TODO: DEBUG_AOT on JIT + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + # MINI_LOADER only on INTERP mode + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + # Memory64 only on CLASSIC INTERP mode + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Install dependencies + uses: nick-fields/retry@v2 + with: + timeout_minutes: 10 + max_attempts: 3 + command: apt update && apt install -y make g++-4.8 gcc-4.8 wget git + on_retry_command: sudo rm -r /var/lib/apt/lists/* + + - name: Install cmake + run: | + wget https://github.com/Kitware/CMake/releases/download/v3.26.1/cmake-3.26.1-linux-x86_64.tar.gz -O cmake.tar.gz + tar xzf cmake.tar.gz + cp cmake-3.26.1-linux-x86_64/bin/cmake /usr/local/bin + cp -r cmake-3.26.1-linux-x86_64/share/cmake-3.26/ /usr/local/share/ + - name: Build iwasm + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} -DCMAKE_C_COMPILER=gcc-4.8 -DCMAKE_CXX_COMPILER=g++-4.8 + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/linux + + build_samples_wasm_c_api: + needs: + [ + build_iwasm, + build_llvm_libraries_on_ubuntu_2004, + build_wamrc, + ] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + sanitizer: ["", "ubsan", "asan"] + make_options: [ + # Running mode + $AOT_BUILD_OPTIONS, + $CLASSIC_INTERP_BUILD_OPTIONS, + $FAST_INTERP_BUILD_OPTIONS, + $FAST_JIT_BUILD_OPTIONS, + $LLVM_LAZY_JIT_BUILD_OPTIONS, + $LLVM_EAGER_JIT_BUILD_OPTIONS, + $MULTI_TIER_JIT_BUILD_OPTIONS, + ] + os: [ubuntu-20.04] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", + ] + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + exclude: + - make_options: $MULTI_TIER_JIT_BUILD_OPTIONS + sanitizer: asan + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Get LLVM libraries + id: retrieve_llvm_libs + if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: download and install wabt + run: | + cd /opt + sudo wget ${{ matrix.wabt_release }} + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt + - name: Build wamrc + if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) + run: | + mkdir build && cd build + cmake -D WAMR_BUILD_SANITIZER="${{matrix.sanitizer}}" .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + + - name: Build Sample [wasm-c-api] + run: | + VERBOSE=1 + cmake -S . -B build ${{ matrix.make_options }} \ + -D WAMR_BUILD_SANITIZER="${{matrix.sanitizer}}" \ + -D WAMR_BUILD_QUICK_AOT_ENTRY=0 + cmake --build build --config Release --parallel 4 + ctest --test-dir build --output-on-failure + working-directory: samples/wasm-c-api + + build_samples_others: + needs: + [ + build_iwasm, + build_llvm_libraries_on_ubuntu_2004, + build_wamrc, + ] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", + ] + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: download and install wasi-sdk + run: | + cd /opt + sudo wget ${{ matrix.wasi_sdk_release }} + sudo tar -xzf wasi-sdk-*.tar.gz + sudo ln -sf wasi-sdk-20.0 wasi-sdk + - name: download and install wabt + run: | + cd /opt + sudo wget ${{ matrix.wabt_release }} + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo ln -sf wabt-1.0.31 wabt + + - name: Get LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Build wamrc + run: | + mkdir build && cd build + cmake -D WAMR_BUILD_SANITIZER="${{matrix.sanitizer}}" .. + cmake --build . --config Release --parallel 4 + working-directory: wamr-compiler + + - name: Build Sample [basic] + run: | + cd samples/basic + ./build.sh + ./run.sh + - name: Build Sample [file] + run: | + cd samples/file + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + ./src/iwasm -f wasm-app/file.wasm -d . + - name: Build Sample [multi-thread] + run: | + cd samples/multi-thread + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + ./iwasm wasm-apps/test.wasm + - name: Build Sample [multi-module] + run: | + cd samples/multi-module + mkdir build && cd build + cmake .. -DWAMR_BUILD_AOT=1 + cmake --build . --config Release --parallel 4 + ./multi_module mC.wasm + ./multi_module mC.aot + - name: Build Sample [spawn-thread] + run: | + cd samples/spawn-thread + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + ./spawn_thread + - name: Build Sample [ref-types] + run: | + cd samples/ref-types + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + ./hello + + - name: Build Sample [wasi-threads] + run: | + cd samples/wasi-threads + mkdir build && cd build + cmake .. + cmake --build . --config Release --parallel 4 + ./iwasm wasm-apps/no_pthread.wasm + + - name: Build Sample [shared-module] + run: | + cd samples/shared-module + ./build.sh + ./run.sh + + - name: Build Sample [terminate] + run: | + cd samples/terminate + ./build.sh + ./run.sh + test: + needs: + [ + build_iwasm, + build_llvm_libraries_on_ubuntu_2004, + build_llvm_libraries_on_ubuntu_2204, + build_wamrc, + ] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, ubuntu-22.04] + sanitizer: ["", "ubsan", "asan", "tsan"] + running_mode: + [ + "classic-interp", + "fast-interp", + "jit", + "aot", + "fast-jit", + "multi-tier-jit", + ] + test_option: + [ + $DEFAULT_TEST_OPTIONS, + $MULTI_MODULES_TEST_OPTIONS, + $SIMD_TEST_OPTIONS, + $THREADS_TEST_OPTIONS, + $WASI_TEST_OPTIONS, + ] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz", + ] + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + ubuntu_version: "20.04" + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + ubuntu_version: "22.04" + + exclude: + # uncompatiable modes and features + - os: ubuntu-20.04 + sanitizer: tsan + # asan works only for aot now + - running_mode: "classic-interp" + sanitizer: asan + - running_mode: "fast-interp" + sanitizer: asan + - running_mode: "jit" + sanitizer: asan + - running_mode: "fast-jit" + sanitizer: asan + - running_mode: "multi-tier-jit" + sanitizer: asan + - running_mode: "classic-interp" + sanitizer: tsan + - running_mode: "jit" + sanitizer: tsan + - running_mode: "fast-jit" + sanitizer: tsan + - running_mode: "multi-tier-jit" + sanitizer: tsan + # classic-interp and fast-interp don't support simd + - running_mode: "classic-interp" + test_option: $SIMD_TEST_OPTIONS + - running_mode: "fast-interp" + test_option: $SIMD_TEST_OPTIONS + # llvm jit doesn't support multi module + - running_mode: "jit" + test_option: $MULTI_MODULES_TEST_OPTIONS + # fast-jit doesn't support multi module, simd + - running_mode: "fast-jit" + test_option: $MULTI_MODULES_TEST_OPTIONS + - running_mode: "fast-jit" + test_option: $SIMD_TEST_OPTIONS + # multi-tier-jit doesn't support multi module, simd + - running_mode: "multi-tier-jit" + test_option: $MULTI_MODULES_TEST_OPTIONS + - running_mode: "multi-tier-jit" + test_option: $SIMD_TEST_OPTIONS + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: download and install wasi-sdk + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: | + cd /opt + sudo wget ${{ matrix.wasi_sdk_release }} + sudo tar -xzf wasi-sdk-*.tar.gz + sudo mv wasi-sdk-20.0 wasi-sdk + + # It is a temporary solution until new wasi-sdk that includes bug fixes is released + - name: build wasi-libc from source + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: | + git clone https://github.com/WebAssembly/wasi-libc + cd wasi-libc + make -j AR=/opt/wasi-sdk/bin/llvm-ar NM=/opt/wasi-sdk/bin/llvm-nm CC=/opt/wasi-sdk/bin/clang THREAD_MODEL=posix + echo "SYSROOT_PATH=$PWD/sysroot" >> $GITHUB_ENV + + - name: set env variable(if llvm are used) + if: matrix.running_mode == 'aot' || matrix.running_mode == 'jit' || matrix.running_mode == 'multi-tier-jit' + run: echo "USE_LLVM=true" >> $GITHUB_ENV + + - name: set env variable(if x86_32 test needed) + if: > + (matrix.test_option == '$DEFAULT_TEST_OPTIONS' || matrix.test_option == '$THREADS_TEST_OPTIONS' + || matrix.test_option == '$WASI_TEST_OPTIONS') + && matrix.running_mode != 'fast-jit' && matrix.running_mode != 'jit' && matrix.running_mode != 'multi-tier-jit' + run: echo "TEST_ON_X86_32=true" >> $GITHUB_ENV + + - name: set additional tsan options + run: | + echo "TSAN_OPTIONS=suppressions=$PWD/tsan_suppressions.txt" >> $GITHUB_ENV + sudo sysctl vm.mmap_rnd_bits=28 + working-directory: tests/wamr-test-suites + + #only download llvm libraries in jit and aot mode + - name: Get LLVM libraries + if: env.USE_LLVM == 'true' + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: env.USE_LLVM == 'true' && steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: install jq JSON processor + if: matrix.running_mode == 'aot' && matrix.test_option == '$WASI_TEST_OPTIONS' + run: sudo apt-get update && sudo apt install -y jq + + - name: Build WASI thread tests + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: bash build.sh --sysroot "$SYSROOT_PATH" + working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/ + + - name: Build WASI thread stress tests + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: bash build.sh --sysroot "$SYSROOT_PATH" + working-directory: ./core/iwasm/libraries/lib-wasi-threads/stress-test/ + + - name: build socket api tests + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: bash build.sh + working-directory: ./core/iwasm/libraries/lib-socket/test/ + + - name: run tests + timeout-minutes: 40 + run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} -T "${{ matrix.sanitizer }}" + working-directory: ./tests/wamr-test-suites + + #only install x32 support libraries when to run x86_32 cases + - name: install x32 support libraries + if: env.TEST_ON_X86_32 == 'true' + run: + # Add another apt repository as some packages cannot + # be downloaded with the github default repository + sudo curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc && + sudo apt-add-repository https://packages.microsoft.com/ubuntu/${{ matrix.ubuntu_version }}/prod && + sudo apt-get update && + sudo apt install -y g++-multilib lib32gcc-9-dev + + - name: run tests x86_32 + timeout-minutes: 40 + if: env.TEST_ON_X86_32 == 'true' + run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} + working-directory: ./tests/wamr-test-suites diff --git a/wasm-micro-runtime/.github/workflows/release_process.yml b/wasm-micro-runtime/.github/workflows/release_process.yml new file mode 100644 index 0000000..de62867 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/release_process.yml @@ -0,0 +1,218 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: the binary release processes + +on: + workflow_dispatch: + inputs: + require_confirmation: + description: "If the process requires a confirmation" + type: boolean + required: false + default: false + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + create_tag: + uses: ./.github/workflows/create_tag.yml + + create_release: + needs: [create_tag] + runs-on: ubuntu-latest + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + steps: + - uses: actions/checkout@v4 + + - name: prepare the release note + run: | + extract_result="$(python3 ./.github/scripts/extract_from_release_notes.py RELEASE_NOTES.md)" + echo "RELEASE_NOTE<> $GITHUB_ENV + echo "${extract_result}" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: create a release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ needs.create_tag.outputs.new_tag }} + release_name: ${{ needs.create_tag.outputs.new_tag }} + prerelease: ${{ inputs.require_confirmation || needs.create_tag.outputs.minor_version }} + draft: false + body: ${{ env.RELEASE_NOTE }} + + # + # LLVM_LIBRARIES + build_llvm_libraries_on_ubuntu_2004: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-20.04" + arch: "AArch64 ARM Mips RISCV X86" + + build_llvm_libraries_on_ubuntu_2204: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-22.04" + arch: "AArch64 ARM Mips RISCV X86" + + build_llvm_libraries_on_macos: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "macos-latest" + arch: "AArch64 ARM Mips RISCV X86" + + # + # WAMRC + release_wamrc_on_ubuntu_2004: + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2004] + uses: ./.github/workflows/build_wamrc.yml + with: + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + release: true + runner: ubuntu-20.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + + release_wamrc_on_ubuntu_2204: + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2204 ] + uses: ./.github/workflows/build_wamrc.yml + with: + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + release: true + runner: ubuntu-22.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver }} + + release_wamrc_on_ubuntu_macos: + needs: [create_tag, create_release, build_llvm_libraries_on_macos] + uses: ./.github/workflows/build_wamrc.yml + with: + llvm_cache_key: ${{ needs.build_llvm_libraries_on_macos.outputs.cache_key }} + release: true + runner: macos-latest + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver }} + + # + # IWASM + release_iwasm_on_ubuntu_2004: + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2004] + uses: ./.github/workflows/build_iwasm_release.yml + with: + cwd: product-mini/platforms/linux + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + runner: ubuntu-20.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + + release_iwasm_on_ubuntu_2204: + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2204] + uses: ./.github/workflows/build_iwasm_release.yml + with: + cwd: product-mini/platforms/linux + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + runner: ubuntu-22.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + + release_iwasm_on_macos: + needs: [create_tag, create_release, build_llvm_libraries_on_macos] + uses: ./.github/workflows/build_iwasm_release.yml + with: + cwd: product-mini/platforms/darwin + llvm_cache_key: ${{ needs.build_llvm_libraries_on_macos.outputs.cache_key }} + runner: macos-latest + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + + # + # WAMR_SDK + release_wamr_sdk_on_ubuntu_2004: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_sdk.yml + with: + config_file: wamr_config_ubuntu_release.cmake + runner: ubuntu-20.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz + wamr_app_framework_url: https://github.com/bytecodealliance/wamr-app-framework.git + + release_wamr_sdk_on_ubuntu_2204: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_sdk.yml + with: + config_file: wamr_config_ubuntu_release.cmake + runner: ubuntu-22.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz + wamr_app_framework_url: https://github.com/bytecodealliance/wamr-app-framework.git + + release_wamr_sdk_on_macos: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_sdk.yml + with: + config_file: wamr_config_macos_release.cmake + runner: macos-latest + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz + wamr_app_framework_url: https://github.com/bytecodealliance/wamr-app-framework.git + + # + # vscode extension cross-platform + release_wamr_ide_vscode_ext: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_vscode_ext.yml + secrets: inherit + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver }} + + # + # vscode extension docker images package + release_wamr_ide_docker_images_package: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_docker_images.yml + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver }} + + # + # WAMR_LLDB + release_wamr_lldb_on_ubuntu_2004: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_lldb.yml + with: + runner: ubuntu-20.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + + release_wamr_lldb_on_ubuntu_2204: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_lldb.yml + with: + runner: ubuntu-22.04 + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} + + release_wamr_lldb_on_macos_universal: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_lldb.yml + with: + runner: macos-latest + arch: universal + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver}} diff --git a/wasm-micro-runtime/.github/workflows/reuse_latest_release_binaries.yml b/wasm-micro-runtime/.github/workflows/reuse_latest_release_binaries.yml new file mode 100644 index 0000000..c9832ce --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/reuse_latest_release_binaries.yml @@ -0,0 +1,68 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: reuse binaries of the latest release if no more modification on the_path since last_commit + +on: + workflow_call: + inputs: + binary_name_stem: + type: string + required: true + last_commit: + type: string + required: true + the_path: + type: string + required: true + upload_url: + description: upload binary assets to the URL of release + type: string + required: true + outputs: + result: + value: ${{ jobs.build.outputs.result }} + +jobs: + reuse: + runs-on: ubuntu-latest + outputs: + result: ${{ steps.try_reuse.outputs.result }} + steps: + - uses: actions/checkout@v4 + # Full git history is needed to get a proper list of commits and tags + with: + fetch-depth: 0 + + - name: try to reuse binaries + id: try_reuse + run: | + echo '::echo::on' + python3 ./.github/scripts/reuse_latest_release_binaries.py \ + --binary_name_stem ${{ inputs.binary_name_stem }} \ + --last_commit ${{ inputs.last_commit }} \ + --the_path ${{ inputs.the_path }} . + ls -lh . + + - run: echo ${{ steps.try_reuse.outputs.result }} + + - name: upload release tar.gz + if: steps.try_reuse.outputs.result == 'hit' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ${{ inputs.binary_name_stem }}.tar.gz + asset_name: ${{ inputs.binary_name_stem }}.tar.gz + asset_content_type: application/x-gzip + + - name: upload release zip + if: steps.try_reuse.outputs.result == 'hit' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ${{ inputs.binary_name_stem }}.zip + asset_name: ${{ inputs.binary_name_stem }}.zip + asset_content_type: application/zip diff --git a/wasm-micro-runtime/.github/workflows/spec_test_on_nuttx.yml b/wasm-micro-runtime/.github/workflows/spec_test_on_nuttx.yml new file mode 100644 index 0000000..1fa3140 --- /dev/null +++ b/wasm-micro-runtime/.github/workflows/spec_test_on_nuttx.yml @@ -0,0 +1,208 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: spec test on nuttx + +on: + pull_request: + types: + - opened + - synchronize + paths: + - ".github/workflows/spec_test_on_nuttx.yml" + - "core/**" + - "!core/deps/**" + - "product-mini/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + - "wamr-sdk/**" + schedule: + - cron: '0 0 * * *' + + workflow_dispatch: + +env: + LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" + WASI_SDK_PATH: "/opt/wasi-sdk" + WAMR_COMMON_OPTION: + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_STACKSIZE=327680\\nCONFIG_INTERPRETERS_WAMR_LOG=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\\nCONFIG_INTERPRETERS_WAMR_REF_TYPES=y\\nCONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST=y\\nCONFIG_INTERPRETERS_WAMR_SHARED_MEMORY=y\\nCONFIG_INTERPRETERS_WAMR_BULK_MEMORY=y\\nCONFIG_EOL_IS_LF=y\\nCONFIG_ARM_SEMIHOSTING_HOSTFS=y\\nCONFIG_ARM_SEMIHOSTING_HOSTFS_CACHE_COHERENCE=y\\nCONFIG_RISCV_SEMIHOSTING_HOSTFS=y\\nCONFIG_FS_HOSTFS=y\\nCONFIG_LIBC_FLOATINGPOINT=y\\n" + +jobs: + build_llvm_libraries: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-22.04" + arch: "ARM RISCV AArch64" + container_image: ghcr.io/no1wudi/nuttx/apache-nuttx-ci-linux@sha256:8c4e00b607d4d6d66ba8f51c4544819a616eac69d3a2ac669e2af2150e2eb0f9 + + spec_test_on_qemu: + runs-on: ubuntu-latest + needs: [build_llvm_libraries] + container: + image: ghcr.io/no1wudi/nuttx/apache-nuttx-ci-linux@sha256:8c4e00b607d4d6d66ba8f51c4544819a616eac69d3a2ac669e2af2150e2eb0f9 + strategy: + matrix: + target_config: [ + # { + # config: "boards/arm64/qemu/qemu-armv8a/configs/nsh", + # target: "aarch64_vfp", + # fpu_type: "fp" + # }, + # { + # config: "boards/arm/imx6/sabre-6quad/configs/nsh", + # target: "thumbv7", + # fpu_type: "none" + # }, + { + config: "boards/arm/imx6/sabre-6quad/configs/nsh", + target: "thumbv7_vfp", + fpu_type: "dp" + }, + { + config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", + target: "riscv32", + fpu_type: "none" + }, + # { + # config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", + # target: "riscv32_ilp32d", + # fpu_type: "dp" + # }, + { + config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", + target: "riscv64", + fpu_type: "none" + }, + ] + + wamr_test_option: [ + { + mode: "-t aot", + option: "CONFIG_INTERPRETERS_WAMR_AOT=y\\n" + }, + # { + # mode: "-t aot -X", + # option: "CONFIG_INTERPRETERS_WAMR_AOT=y\\n" + # }, + # { + # mode: "-t classic-interp", + # option: "CONFIG_INTERPRETERS_WAMR_CLASSIC=y\\n" + # }, + # { + # mode: "-t fast-interp", + # option: "CONFIG_INTERPRETERS_WAMR_FAST=y\\n" + # }, + ] + + wamr_feature_option: + # Empty option for default + - { option: "", mode: "" } + - { option: "CONFIG_INTERPRETERS_WAMR_GC=y\\nCONFIG_INTERPRETERS_WAMR_AOT_STACK_FRAME=y\\n", mode: "-G" } + + exclude: + # XIP is not fully supported yet on RISCV64, some relocations can not be resolved + - target_config: { config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64" } + wamr_test_option: { mode: "-t aot -X" } + + steps: + - name: Checkout NuttX + uses: actions/checkout@v4 + with: + repository: apache/incubator-nuttx + ref: releases/12.4 + path: nuttx + + - name: Checkout NuttX Apps + uses: actions/checkout@v4 + with: + repository: apache/incubator-nuttx-apps + ref: releases/12.4 + path: apps + + - name: Checkout WAMR + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + path: apps/interpreters/wamr/wamr + + - name: Get LLVM libraries + if: contains(matrix.wamr_test_option.mode, 'aot') + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ needs.build_llvm_libraries.outputs.cache_key }} + + - name: Quit if cache miss + if: contains(matrix.wamr_test_option.mode, 'aot') && steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Copy LLVM + if: contains(matrix.wamr_test_option.mode, 'aot') + run: cp -r core/deps/llvm apps/interpreters/wamr/wamr/core/deps/llvm + + # Inject the config option to NuttX + # TODO: Merge this into NuttX once GC is generally available + - name: Modify Kconfig + run: | + echo "\n" >> apps/interpreters/wamr/Kconfig + echo "config INTERPRETERS_WAMR_GC" >> apps/interpreters/wamr/Kconfig + echo "\tbool \"Enable GC\"" >> apps/interpreters/wamr/Kconfig + echo "\tdefault n" >> apps/interpreters/wamr/Kconfig + echo "\n" >> apps/interpreters/wamr/Kconfig + echo "config INTERPRETERS_WAMR_AOT_STACK_FRAME" >> apps/interpreters/wamr/Kconfig + echo "\tbool \"Enable AOT stack frame\"" >> apps/interpreters/wamr/Kconfig + echo "\tdefault n" >> apps/interpreters/wamr/Kconfig + echo "\n" >> apps/interpreters/wamr/Kconfig + echo "config INTERPRETERS_WAMR_TAIL_CALL" >> apps/interpreters/wamr/Kconfig + echo "\tbool \"Enable Tail Call\"" >> apps/interpreters/wamr/Kconfig + echo "\tdefault y" >> apps/interpreters/wamr/Kconfig + + - name: Enable WAMR for NuttX + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\${{ env.WAMR_COMMON_OPTION }}' + + - name: Enable WAMR Interpreter for NuttX + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\${{ matrix.wamr_test_option.option }}' + + - name: Enable WAMR Feature for NuttX + if: matrix.wamr_feature_option.option != '' + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\${{ matrix.wamr_feature_option.option }}' + + - name: Disable FPU for NuttX + if: matrix.target_config.fpu_type == 'none' + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_FPU is not set\n' + + - name: Disable DPFPU for NuttX + if: matrix.target_config.fpu_type == 'fp' + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_DPFPU is not set\n' + + - name: Build wamrc + if: contains(matrix.wamr_test_option.mode, 'aot') + working-directory: apps/interpreters/wamr/wamr/wamr-compiler + run: | + cmake -Bbuild . + cmake --build build + + - name: Build + id: build_firmware + run: | + cd nuttx + tools/configure.sh ${{ matrix.target_config.config }} + make -j$(nproc) + echo "firmware=$PWD/nuttx" >> $GITHUB_OUTPUT + + - name: Test + run: | + cd apps/interpreters/wamr/wamr/tests/wamr-test-suites + ./test_wamr.sh -s spec ${{ matrix.wamr_test_option.mode }} -m ${{ matrix.target_config.target }} -b -Q -P -F ${{ steps.build_firmware.outputs.firmware }} ${{ matrix.wamr_feature_option.mode}} diff --git a/wasm-micro-runtime/.gitignore b/wasm-micro-runtime/.gitignore new file mode 100644 index 0000000..b85dd39 --- /dev/null +++ b/wasm-micro-runtime/.gitignore @@ -0,0 +1,39 @@ +.cache +.clangd +.vs +.vscode +.venv +/.idea +**/cmake-build-*/ +**/*build/ +*.obj +*.a +*.so +.clangd +.DS_Store + +core/deps/** +core/shared/mem-alloc/tlsf +core/iwasm/libraries/lib-wasi-threads/test/*.wasm +core/iwasm/libraries/lib-socket/test/*.wasm + +product-mini/app-samples/hello-world/test.wasm +product-mini/platforms/linux-sgx/enclave-sample/App/ +product-mini/platforms/linux-sgx/enclave-sample/Enclave/ +product-mini/platforms/linux-sgx/enclave-sample/iwasm + +build_out +tests/wamr-test-suites/workspace + +!/test-tools/wamr-ide/VSCode-Extension/.vscode + +samples/socket-api/wasm-src/inc/pthread.h + +**/__pycache__ + +tests/benchmarks/coremark/coremark* + +samples/workload/include/** +!samples/workload/include/.gitkeep + +# core/iwasm/libraries/wasi-threads \ No newline at end of file diff --git a/wasm-micro-runtime/ATTRIBUTIONS.md b/wasm-micro-runtime/ATTRIBUTIONS.md new file mode 100644 index 0000000..2c83d66 --- /dev/null +++ b/wasm-micro-runtime/ATTRIBUTIONS.md @@ -0,0 +1,105 @@ +WebAssembly Micro Runtime Attributions +====================================== + +WAMR project reused some components from other open source project: +- **cJson**: in the repository [wamr-app-framework](https://github.com/bytecodealliance/wamr-app-framework/), used in the host_tool for remotely managing wasm applications +- **contiki-ng**: for the coap protocol implementation +- **freebsd libm**: used in core/shared/platform/alios/bh_math.c +- **LVGL**: in the repository [wamr-app-framework](https://github.com/bytecodealliance/wamr-app-framework/), for the gui samples and wrapped the wasm graphic layer +- **llvm**: for the AOT/JIT compilation +- **wasm-c-api**: to implement the C-APIs of wasm. using headers and sameples +- **wasmtime**: for the wasi libc implementation +- **zephyr**: for several platform specific examples +- **WebAssembly debugging patch for LLDB**: for extending the ability of LLDB to support wasm debugging +- **libuv**: for the WASI Libc with uvwasi implementation +- **uvwasi**: for the WASI Libc with uvwasi implementation +- **asmjit**: for the Fast JIT x86-64 codegen implementation +- **zydis**: for the Fast JIT x86-64 codegen implementation +- **NuttX ELF headers**: used in core/iwasm/aot/debug/elf_parser.c +- **Dhrystone**: for the test benchmakr dhrystone + +The WAMR fast interpreter is a clean room development. We would acknowledge the inspirations by [WASM3](https://github.com/wasm3/wasm3) open source project for the approach of pre-calculated oprand stack location. + +| third party components | version number | latest release | vendor pages | CVE details | +| --- | --- | --- | --- | --- | +| cjson | 1.7.16 | 1.7.16 | https://github.com/DaveGamble/cJSON | https://www.cvedetails.com/vendor/19164/Cjson-Project.html | +| contiki-ng (er-coap) | unspecified | 3.0 | https://github.com/contiki-os/contiki | https://www.cvedetails.com/vendor/16528/Contiki-os.html | +| freebsd libm | unspecified | 13.0 | https://www.freebsd.org/ | https://www.cvedetails.com/vendor/6/Freebsd.html | +| LVGL | 6.0.1 | 7.11.0 | https://lvgl.io/ | | +| llvm | 11.0.1 | 12.0.0 | https://llvm.org | https://www.cvedetails.com/vendor/13260/Llvm.html | +| wasm-c-api | ac9b509f4df86e40e56e9b01f3f49afab0100037 | c9d31284651b975f05ac27cee0bab1377560b87e | https://github.com/WebAssembly/wasm-c-api | | +| wasmtime | unspecified | v0.26.0 | https://github.com/bytecodealliance/wasmtime | | +| zephyr | unspecified | v2.5.0 | https://www.zephyrproject.org/ | https://www.cvedetails.com/vendor/19255/Zephyrproject.html | +| WebAssembly debugging patch for LLDB | unspecified | unspecified | https://reviews.llvm.org/D78801 | | +| libuv | v1.46.0 | v1.46.0 | https://github.com/libuv/libuv | https://www.cvedetails.com/vendor/15402/Libuv-Project.html | +| uvwasi | unspecified | v0.0.12 | https://github.com/nodejs/uvwasi | | +| asmjit | unspecified | unspecified | https://github.com/asmjit/asmjit | | +| zydis | unspecified | e14a07895136182a5b53e181eec3b1c6e0b434de | https://github.com/zyantific/zydis | | +| NuttX ELF headers | 72313301e23f9c2de969fb64b9a0f67bb4c284df | 10.3.0 | https://github.com/apache/incubator-nuttx | | +| Dhrystone | 2.1 | 2.1 | https://fossies.org/linux/privat/old/ | | + +## Licenses + +### cJson + +[LICENSE](https://github.com/bytecodealliance/wamr-app-framework/blob/main/test-tools/host-tool/external/cJSON/LICENSE) + +### contiki-ng + +[LICENSE](./core/shared/coap/er-coap/LICENSE.md) + +### freebsd libm + +[COPYRIGHT](./core/shared/platform/common/math/COPYRIGHT) + +### LVGL + +[LICENSE](https://github.com/bytecodealliance/wamr-app-framework/blob/main/samples/littlevgl/LICENCE.txt) + +[LICENSE](https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-framework/wgl/app/wa-inc/lvgl/LICENCE.txt) + +### llvm + +[LICENSE](./LICENCE.txt) + +### wasm-c-api + +[LICENSE](./samples/wasm-c-api/src/LICENSE) + +### wasmtime + +[LICENSE](./core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/LICENSE) + +[LICENSE](./core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/LICENSE) + +[LICENSE](./core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/include/LICENSE) + +### zephyr + +[LICENSE](https://github.com/bytecodealliance/wamr-app-framework/blob/main/samples/gui/wasm-runtime-wgl/src/platform/zephyr/LICENSE) + +### libuv + +[LICENSE](./core/iwasm/libraries/libc-uvwasi/LICENSE_LIBUV) + +### uvwasi + +[LICENSE](./core/iwasm/libraries/libc-uvwasi/LICENSE_UVWASI) + +### asmjit + +[LICENSE](./core/iwasm/fast-jit/cg/LICENSE_ASMJIT) + +### zydis + +[LICENSE](./core/iwasm/fast-jit/cg/LICENSE_ZYDIS) + +### NuttX ELF headers + +[LICENSE](./core/iwasm/aot/debug/LICENSE_NUTTX) + +[NOTICE](./core/iwasm/aot/debug/NOTICE_NUTTX) + +### Dhrystone + +[LICENSE](./tests/benchmarks/dhrystone/LICENSE) diff --git a/wasm-micro-runtime/CMakeLists.txt b/wasm-micro-runtime/CMakeLists.txt new file mode 100644 index 0000000..0ffba05 --- /dev/null +++ b/wasm-micro-runtime/CMakeLists.txt @@ -0,0 +1,169 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.0) + +if(ESP_PLATFORM) + include (${COMPONENT_DIR}/build-scripts/esp-idf/wamr/CMakeLists.txt) + return() +endif() + +project (iwasm) + +set (CMAKE_VERBOSE_MAKEFILE OFF) + +if (NOT DEFINED WAMR_BUILD_PLATFORM) + string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +set (CMAKE_C_STANDARD 99) + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_JIT) + # Disable Fast JIT by default + set (WAMR_BUILD_FAST_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Disable multiple modules by default + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS) + # Disable wasi threads library by default + set (WAMR_BUILD_LIB_WASI_THREADS 0) +endif () + +if (NOT DEFINED WAMR_BUILD_MINI_LOADER) + # Disable wasm mini loader by default + set (WAMR_BUILD_MINI_LOADER 0) +endif () + +if (NOT DEFINED WAMR_BUILD_SIMD) + # Enable SIMD by default + set (WAMR_BUILD_SIMD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_REF_TYPES) + # Disable reference types by default + set (WAMR_BUILD_REF_TYPES 0) +endif () + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow -Wno-unused-parameter") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") + +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") + +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () +endif () + +# The following flags are to enhance security, but it may impact performance, +# we disable them by default. +#if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftrapv -D_FORTIFY_SOURCE=2") +#endif () +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-strong --param ssp-buffer-size=4") +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack,-z,relro,-z,now") + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +# STATIC LIBRARY +add_library(iwasm_static STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +set_target_properties (iwasm_static PROPERTIES OUTPUT_NAME vmlib) +target_include_directories(iwasm_static INTERFACE ${WAMR_ROOT_DIR}/core/iwasm/include) +target_link_libraries (iwasm_static INTERFACE ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) +if (WAMR_BUILD_WASM_CACHE EQUAL 1) + target_link_libraries(iwasm_static INTERFACE boringssl_crypto) +endif () + +install (TARGETS iwasm_static ARCHIVE DESTINATION lib) + +# SHARED LIBRARY +add_library (iwasm_shared SHARED ${WAMR_RUNTIME_LIB_SOURCE}) +set_target_properties (iwasm_shared PROPERTIES OUTPUT_NAME iwasm) +target_include_directories(iwasm_shared INTERFACE ${WAMR_ROOT_DIR}/core/iwasm/include) +target_link_libraries (iwasm_shared INTERFACE ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) +if (WAMR_BUILD_WASM_CACHE EQUAL 1) + target_link_libraries(iwasm_shared INTERFACE boringssl_crypto) +endif () + +if (MINGW) + target_link_libraries (iwasm_shared INTERFACE -lWs2_32 -lwsock32) +endif () + +install (TARGETS iwasm_shared LIBRARY DESTINATION lib) + +# HEADERS +install (FILES + ${WAMR_ROOT_DIR}/core/iwasm/include/wasm_c_api.h + ${WAMR_ROOT_DIR}/core/iwasm/include/wasm_export.h + ${WAMR_ROOT_DIR}/core/iwasm/include/lib_export.h + DESTINATION include) diff --git a/wasm-micro-runtime/CODE_OF_CONDUCT.md b/wasm-micro-runtime/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..5c5ebdd --- /dev/null +++ b/wasm-micro-runtime/CODE_OF_CONDUCT.md @@ -0,0 +1,49 @@ +# Contributor Covenant Code of Conduct + +*Note*: this Code of Conduct pertains to individuals' behavior. Please also see the [Organizational Code of Conduct][OCoC]. + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the Bytecode Alliance CoC team at [report@bytecodealliance.org](mailto:report@bytecodealliance.org). The CoC team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The CoC team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the Bytecode Alliance's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[OCoC]: ORG_CODE_OF_CONDUCT.md +[homepage]: https://www.contributor-covenant.org +[version]: https://www.contributor-covenant.org/version/1/4/ diff --git a/wasm-micro-runtime/CONTRIBUTING.md b/wasm-micro-runtime/CONTRIBUTING.md new file mode 100644 index 0000000..0e04101 --- /dev/null +++ b/wasm-micro-runtime/CONTRIBUTING.md @@ -0,0 +1,39 @@ +Contributing to WAMR +===================== +As an open-source project, we welcome and encourage the community to submit patches directly to the project. In our collaborative open source environment, standards and methods for submitting changes help reduce the chaos that can result from an active development community. +We want to make contributing to this project as easy and transparent as possible, whether it's: +- Reporting a bug +- the current state of the code +- Submitting a fix +- Proposing new features + +License +======= +WAMR uses the same license as LLVM: the `Apache 2.0 license` with the LLVM +exception. See the LICENSE file for details. This license allows you to freely +use, modify, distribute and sell your own products based on WAMR. +Any contributions you make will be under the same license. + +Code changes +=================== +We Use Github Flow, So All Code Changes Happen Through Pull Requests. Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests: + +- If you've added code that should be tested, add tests. Ensure the test suite passes. +- Avoid use macros for different platforms. Use seperate folder of source files to host diffeent platform logic. +- Put macro definitions inside share_lib/include/config.h if you have to use macro. +- Make sure your code lints and compliant to our coding style. +- Extend the application library is highly welcome. + +Coding Style +=============================== +Please use [K&R](https://en.wikipedia.org/wiki/Indentation_style#K.26R) coding style, such as 4 spaces for indentation rather than tabs etc. +We suggest using VS Code like IDE or stable coding format tools, like clang-format, to make your code compliant to the customized format(in .clang-format). + +Report bugs +=================== +We use GitHub issues to track public bugs. Report a bug by [open a new issue](https://github.com/intel/wasm-micro-runtime/issues/new). + +Code of Conduct +=============== + +WAMR is a [Bytecode Alliance](https://bytecodealliance.org/) project, and follows the Bytecode Alliance's [Code of Conduct](CODE_OF_CONDUCT.md) and [Organizational Code of Conduct](ORG_CODE_OF_CONDUCT.md). diff --git a/wasm-micro-runtime/LICENSE b/wasm-micro-runtime/LICENSE new file mode 100644 index 0000000..c6bd7e0 --- /dev/null +++ b/wasm-micro-runtime/LICENSE @@ -0,0 +1,219 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/wasm-micro-runtime/ORG_CODE_OF_CONDUCT.md b/wasm-micro-runtime/ORG_CODE_OF_CONDUCT.md new file mode 100644 index 0000000..e05e40c --- /dev/null +++ b/wasm-micro-runtime/ORG_CODE_OF_CONDUCT.md @@ -0,0 +1,139 @@ +# Bytecode Alliance Organizational Code of Conduct (OCoC) + +*Note*: this Code of Conduct pertains to organizations' behavior. Please also see the [Individual Code of Conduct](CODE_OF_CONDUCT.md). + +## Preamble + +The Bytecode Alliance (BA) welcomes involvement from organizations, +including commercial organizations. This document is an +*organizational* code of conduct, intended particularly to provide +guidance to commercial organizations. It is distinct from the +[Individual Code of Conduct (ICoC)](CODE_OF_CONDUCT.md), and does not +replace the ICoC. This OCoC applies to any group of people acting in +concert as a BA member or as a participant in BA activities, whether +or not that group is formally incorporated in some jurisdiction. + +The code of conduct described below is not a set of rigid rules, and +we did not write it to encompass every conceivable scenario that might +arise. For example, it is theoretically possible there would be times +when asserting patents is in the best interest of the BA community as +a whole. In such instances, consult with the BA, strive for +consensus, and interpret these rules with an intent that is generous +to the community the BA serves. + +While we may revise these guidelines from time to time based on +real-world experience, overall they are based on a simple principle: + +*Bytecode Alliance members should observe the distinction between + public community functions and private functions — especially + commercial ones — and should ensure that the latter support, or at + least do not harm, the former.* + +## Guidelines + + * **Do not cause confusion about Wasm standards or interoperability.** + + Having an interoperable WebAssembly core is a high priority for + the BA, and members should strive to preserve that core. It is fine + to develop additional non-standard features or APIs, but they + should always be clearly distinguished from the core interoperable + Wasm. + + Treat the WebAssembly name and any BA-associated names with + respect, and follow BA trademark and branding guidelines. If you + distribute a customized version of software originally produced by + the BA, or if you build a product or service using BA-derived + software, use names that clearly distinguish your work from the + original. (You should still provide proper attribution to the + original, of course, wherever such attribution would normally be + given.) + + Further, do not use the WebAssembly name or BA-associated names in + other public namespaces in ways that could cause confusion, e.g., + in company names, names of commercial service offerings, domain + names, publicly-visible social media accounts or online service + accounts, etc. It may sometimes be reasonable, however, to + register such a name in a new namespace and then immediately donate + control of that account to the BA, because that would help the project + maintain its identity. + + * **Do not restrict contributors.** If your company requires + employees or contractors to sign non-compete agreements, those + agreements must not prevent people from participating in the BA or + contributing to related projects. + + This does not mean that all non-compete agreements are incompatible + with this code of conduct. For example, a company may restrict an + employee's ability to solicit the company's customers. However, an + agreement must not block any form of technical or social + participation in BA activities, including but not limited to the + implementation of particular features. + + The accumulation of experience and expertise in individual persons, + who are ultimately free to direct their energy and attention as + they decide, is one of the most important drivers of progress in + open source projects. A company that limits this freedom may hinder + the success of the BA's efforts. + + * **Do not use patents as offensive weapons.** If any BA participant + prevents the adoption or development of BA technologies by + asserting its patents, that undermines the purpose of the + coalition. The collaboration fostered by the BA cannot include + members who act to undermine its work. + + * **Practice responsible disclosure** for security vulnerabilities. + Use designated, non-public reporting channels to disclose technical + vulnerabilities, and give the project a reasonable period to + respond, remediate, and patch. + + Vulnerability reporters may patch their company's own offerings, as + long as that patching does not significantly delay the reporting of + the vulnerability. Vulnerability information should never be used + for unilateral commercial advantage. Vendors may legitimately + compete on the speed and reliability with which they deploy + security fixes, but withholding vulnerability information damages + everyone in the long run by risking harm to the BA project's + reputation and to the security of all users. + + * **Respect the letter and spirit of open source practice.** While + there is not space to list here all possible aspects of standard + open source practice, some examples will help show what we mean: + + * Abide by all applicable open source license terms. Do not engage + in copyright violation or misattribution of any kind. + + * Do not claim others' ideas or designs as your own. + + * When others engage in publicly visible work (e.g., an upcoming + demo that is coordinated in a public issue tracker), do not + unilaterally announce early releases or early demonstrations of + that work ahead of their schedule in order to secure private + advantage (such as marketplace advantage) for yourself. + + The BA reserves the right to determine what constitutes good open + source practices and to take action as it deems appropriate to + encourage, and if necessary enforce, such practices. + +## Enforcement + +Instances of organizational behavior in violation of the OCoC may +be reported by contacting the Bytecode Alliance CoC team at +[report@bytecodealliance.org](mailto:report@bytecodealliance.org). The +CoC team will review and investigate all complaints, and will respond +in a way that it deems appropriate to the circumstances. The CoC team +is obligated to maintain confidentiality with regard to the reporter of +an incident. Further details of specific enforcement policies may be +posted separately. + +When the BA deems an organization in violation of this OCoC, the BA +will, at its sole discretion, determine what action to take. The BA +will decide what type, degree, and duration of corrective action is +needed, if any, before a violating organization can be considered for +membership (if it was not already a member) or can have its membership +reinstated (if it was a member and the BA canceled its membership due +to the violation). + +In practice, the BA's first approach will be to start a conversation, +with punitive enforcement used only as a last resort. Violations +often turn out to be unintentional and swiftly correctable with all +parties acting in good faith. diff --git a/wasm-micro-runtime/README.md b/wasm-micro-runtime/README.md new file mode 100644 index 0000000..d09e826 --- /dev/null +++ b/wasm-micro-runtime/README.md @@ -0,0 +1,114 @@ +# WebAssembly Micro Runtime + + +**A [Bytecode Alliance][BA] project** + +[BA]: https://bytecodealliance.org/ + +**[Guide](https://wamr.gitbook.io/)**  **[Website](https://bytecodealliance.github.io/wamr.dev)**  **[Chat](https://bytecodealliance.zulipchat.com/#narrow/stream/290350-wamr)** + +[Build WAMR](./doc/build_wamr.md) | [Build AOT Compiler](./wamr-compiler/README.md) | [Embed WAMR](./doc/embed_wamr.md) | [Export Native API](./doc/export_native_api.md) | [Build Wasm Apps](./doc/build_wasm_app.md) | [Samples](./samples/README.md) + +WebAssembly Micro Runtime (WAMR) is a lightweight standalone WebAssembly (Wasm) runtime with small footprint, high performance and highly configurable features for applications cross from embedded, IoT, edge to Trusted Execution Environment (TEE), smart contract, cloud native and so on. It includes a few parts as below: +- [**VMcore**](./core/iwasm/): A set of runtime libraries for loading and running Wasm modules. It supports rich running modes including interpreter, Ahead-of-Time compilation(AoT) and Just-in-Time compilation (JIT). WAMR supports two JIT tiers - Fast JIT, LLVM JIT, and dynamic tier-up from Fast JIT to LLVM JIT. +- [**iwasm**](./product-mini/): The executable binary built with WAMR VMcore which supports WASI and command line interface. +- [**wamrc**](./wamr-compiler/): The AOT compiler to compile Wasm file into AOT file +- Useful components and tools for building real solutions with WAMR vmcore: + - [App-framework](https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-framework/README.md): A framework for supporting APIs for the Wasm applications + - [App-manager](https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-mgr/README.md): A framework for dynamical loading the Wasm module remotely + - [WAMR-IDE](./test-tools/wamr-ide): An experimental VSCode extension for developping WebAssembly applications with C/C++ + + +### Key features +- Full compliant to the W3C Wasm MVP +- Small runtime binary size (~85K for interpreter and ~50K for AOT) and low memory usage +- Near to native speed by AOT and JIT +- Self-implemented AOT module loader to enable AOT working on Linux, Windows, MacOS, Android, SGX and MCU systems +- Choices of Wasm application libc support: the built-in libc subset for the embedded environment or [WASI](https://github.com/WebAssembly/WASI) for the standard libc +- [The simple C APIs to embed WAMR into host environment](./doc/embed_wamr.md), see [how to integrate WAMR](./doc/embed_wamr.md) and the [API list](./core/iwasm/include/wasm_export.h) +- [The mechanism to export native APIs to Wasm applications](./doc/export_native_api.md), see [how to register native APIs](./doc/export_native_api.md) +- [Multiple modules as dependencies](./doc/multi_module.md), ref to [document](./doc/multi_module.md) and [sample](samples/multi-module) +- [Multi-thread, pthread APIs and thread management](./doc/pthread_library.md), ref to [document](./doc/pthread_library.md) and [sample](samples/multi-thread) +- [wasi-threads](./doc/pthread_impls.md#wasi-threads-new), ref to [document](./doc/pthread_impls.md#wasi-threads-new) and [sample](samples/wasi-threads) +- [Linux SGX (Intel Software Guard Extension) support](./doc/linux_sgx.md), ref to [document](./doc/linux_sgx.md) +- [Source debugging support](./doc/source_debugging.md), ref to [document](./doc/source_debugging.md) +- [XIP (Execution In Place) support](./doc/xip.md), ref to [document](./doc/xip.md) +- [Berkeley/Posix Socket support](./doc/socket_api.md), ref to [document](./doc/socket_api.md) and [sample](./samples/socket-api) +- [Multi-tier JIT](./product-mini#linux) and [Running mode control](https://bytecodealliance.github.io/wamr.dev/blog/introduction-to-wamr-running-modes/) +- Language bindings: [Go](./language-bindings/go/README.md), [Python](./language-bindings/python/README.md), [Rust](./language-bindings/rust/README.md) + +### Wasm post-MVP features +- [wasm-c-api](https://github.com/WebAssembly/wasm-c-api), ref to [document](doc/wasm_c_api.md) and [sample](samples/wasm-c-api) +- [128-bit SIMD](https://github.com/WebAssembly/simd), ref to [samples/workload](samples/workload) +- [Reference Types](https://github.com/WebAssembly/reference-types), ref to [document](doc/ref_types.md) and [sample](samples/ref-types) +- [Bulk memory operations](https://github.com/WebAssembly/bulk-memory-operations), [Shared memory](https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#shared-linear-memory), [Memory64](https://github.com/WebAssembly/memory64) +- [Tail-call](https://github.com/WebAssembly/tail-call), [Garbage Collection](https://github.com/WebAssembly/gc), [Exception Handling](https://github.com/WebAssembly/exception-handling) + +### Supported architectures and platforms +The WAMR VMcore supports the following architectures: +- X86-64, X86-32 +- ARM, THUMB (ARMV7 Cortex-M7 and Cortex-A15 are tested) +- AArch64 (Cortex-A57 and Cortex-A53 are tested) +- RISCV64, RISCV32 (RISC-V LP64 and RISC-V LP64D are tested) +- XTENSA, MIPS, ARC + +The following platforms are supported, click each link below for how to build iwasm on that platform. Refer to [WAMR porting guide](./doc/port_wamr.md) for how to port WAMR to a new platform. +- [Linux](./product-mini/README.md#linux), [Linux SGX (Intel Software Guard Extension)](./doc/linux_sgx.md), [MacOS](./product-mini/README.md#macos), [Android](./product-mini/README.md#android), [Windows](./product-mini/README.md#windows), [Windows (MinGW)](./product-mini/README.md#mingw) +- [Zephyr](./product-mini/README.md#zephyr), [AliOS-Things](./product-mini/README.md#alios-things), [VxWorks](./product-mini/README.md#vxworks), [NuttX](./product-mini/README.md#nuttx), [RT-Thread](./product-mini/README.md#RT-Thread), [ESP-IDF](./product-mini/README.md#esp-idf) + + +## Getting started +- [Build VM core](./doc/build_wamr.md) and [Build wamrc AOT compiler](./wamr-compiler/README.md) +- [Build iwasm (mini product)](./product-mini/README.md): [Linux](./product-mini/README.md#linux), [SGX](./doc/linux_sgx.md), [MacOS](./product-mini/README.md#macos) and [Windows](./product-mini/README.md#windows) +- [Embed into C/C++](./doc/embed_wamr.md), [Embed into Python](./language-bindings/python), [Embed into Go](./language-bindings/go), [Embed in Rust](./language-bindings/rust) +- [Register native APIs for Wasm applications](./doc/export_native_api.md) +- [Build wamrc AOT compiler](./wamr-compiler/README.md) +- [Build Wasm applications](./doc/build_wasm_app.md) +- [Port WAMR to a new platform](./doc/port_wamr.md) +- [VS Code development container](./doc/devcontainer.md) +- [Samples](./samples) and [Benchmarks](./tests/benchmarks) +- [End-user APIs documentation](https://bytecodealliance.github.io/wamr.dev/apis/) + + +### Performance and memory +- [Blog: The WAMR memory model](https://bytecodealliance.github.io/wamr.dev/blog/the-wamr-memory-model/) +- [Blog: Understand WAMR heaps](https://bytecodealliance.github.io/wamr.dev/blog/understand-the-wamr-heaps/) and [stacks](https://bytecodealliance.github.io/wamr.dev/blog/understand-the-wamr-stacks/) +- [Blog: Introduction to WAMR running modes](https://bytecodealliance.github.io/wamr.dev/blog/introduction-to-wamr-running-modes/) +- [Memory usage tuning](./doc/memory_tune.md): the memory model and how to tune the memory usage +- [Memory usage profiling](./doc/build_wamr.md#enable-memory-profiling-experiment): how to profile the memory usage +- [Performance tuning](./doc/perf_tune.md): how to tune the performance +- [Benchmarks](./tests/benchmarks): checkout these links for how to run the benchmarks: [PolyBench](./tests/benchmarks/polybench), [CoreMark](./tests/benchmarks/coremark), [Sightglass](./tests/benchmarks/sightglass), [JetStream2](./tests/benchmarks/jetstream) +- [Performance and footprint data](https://github.com/bytecodealliance/wasm-micro-runtime/wiki/Performance): the performance and footprint data + + +Project Technical Steering Committee +==================================== +The [WAMR PTSC Charter](./TSC_Charter.md) governs the operations of the project TSC. +The current TSC members: +- [dongsheng28849455](https://github.com/dongsheng28849455) - **Dongsheng Yan**, +- [loganek](https://github.com/loganek) - **Marcin Kolny**, +- [lum1n0us](https://github.com/lum1n0us) - **Liang He**， +- [no1wudi](https://github.com/no1wudi) **Qi Huang**, +- [qinxk-inter](https://github.com/qinxk-inter) - **Xiaokang Qin**， +- [ttrenner ](https://github.com/ttrenner) - **Trenner, Thomas**， +- [wei-tang](https://github.com/wei-tang) - **Wei Tang**， +- [wenyongh](https://github.com/wenyongh) - **Wenyong Huang**， +- [woodsmc](https://github.com/woodsmc) - **Woods, Chris**， +- [xujuntwt95329](https://github.com/xujuntwt95329) - **Jun Xu**， +- [xwang98](https://github.com/xwang98) - **Xin Wang**， (chair) +- [yamt](https://github.com/yamt) - **Takashi Yamamoto**, + + +License +======= +WAMR uses the same license as LLVM: the `Apache 2.0 license` with the LLVM +exception. See the LICENSE file for details. This license allows you to freely +use, modify, distribute and sell your own products based on WAMR. +Any contributions you make will be under the same license. + +# More resources +- [Who use WAMR?](https://github.com/bytecodealliance/wasm-micro-runtime/wiki) +- [WAMR Blogs](https://bytecodealliance.github.io/wamr.dev/blog/) +- [Community news and events](https://bytecodealliance.github.io/wamr.dev/events/) +- [WAMR TSC meetings](https://github.com/bytecodealliance/wasm-micro-runtime/wiki/TSC-meeting-notes) + diff --git a/wasm-micro-runtime/RELEASE_NOTES.md b/wasm-micro-runtime/RELEASE_NOTES.md new file mode 100644 index 0000000..0721fc6 --- /dev/null +++ b/wasm-micro-runtime/RELEASE_NOTES.md @@ -0,0 +1,952 @@ +## WAMR-2.0.0 + +### Breaking Changes +- The AOT ABI was changed after GC and memory64 features were introduced: + - Implement GC feature for interpreter, AOT and LLVM-JIT (#3125) + - Implement memory64 for classic interpreter (#3266) + - Always allocate linear memory using mmap (#3052) + - Refactor APIs and data structures as preliminary work for Memory64 (#3209) +- Remove unused argument in wasm_runtime_lookup_function (#3218) +- Separate app-manager and app-framework from WAMR (#3129) + +### New Features +- Implement GC feature for interpreter, AOT and LLVM-JIT (#3125) +- Implement memory64 for classic interpreter (#3266) +- Add wasi_ephemeral_nn module support (#3241) + +### Bug Fixes +- EH: Fix broken stack usage calculation (#3121) +- Fix loader check_wasi_abi_compatibility (#3126) +- Fix possible integer overflow in loader target block check (#3133) +- Fix locel.set in polymorphic stack (#3135) +- Fix threads opcodes' boundary check in classic-interp and fast-interp (#3136) +- fast-interp: Fix copy_stack_top_i64 overlap issue (#3146) +- Fix a ubsan complaint "applying zero offset to null pointer" (#3160) +- fast-interp: Fix GC opcode ref.as_non_null (#3156) +- Fix llvm jit push funcref/externref result type issue (#3169) +- Fix wasm loader handling opcode br_table (#3176) +- Fix ref.func opcode check when GC is enabled (#3181) +- lldb_function_to_function_dbi: Fix a null dereference (#3189) +- Fix compilation errors on MinGW (#3217) +- Fix compilation errors on esp-idf platform (#3224) +- Fix aot relocation symbols not found on windows 32-bit (#3231) +- posix_file.c: Correct the dirfd argument that passes to fstatat (#3244) +- Fix compilation errors on zephyr platform (#3255) +- Fix dynamic offset not updated in op_br for block with ret type (#3269) +- aot debug: Fix a NULL dereference (#3274) +- thread mgr: Free aux stack only when it was allocated (#3282) +- interp: Restore context from prev_frame after tail calling a native function (#3283) +- Sync simd opcode definitions spec (#3290) +- Fix posix_fadvise error handling (#3323) +- Fix windows relocation string parsing issue (#3333) + +### Enhancements +- Zero the memory mapped from os_mmap in NuttX (#3132) +- Use logger for runtime error/debug prints (#3097) +- aot_compile_op_call: Stop setting calling convention explicitly (#3140) +- aot compiler: Place precheck wrapper before the corresponding wrapped function (#3141) +- Fix null pointer access in fast-interp when configurable soft bound check is enabled (#3150) +- Clarify how to verify SGX evidence without an Intel SGX-enabled platform (#3158) +- zephyr: Use zephyr sys_cache instead of CMSIS (#3162) +- VSCode IDE enhancement and readme update (#3172) +- Add vprintf override for android and esp-idf (#3174) +- zephyr: Include math only with minimal libc (#3177) +- zephyr: Implement Alloc_With_System_Allocator (#3179) +- Use indirect call in pre-checker function to avoid relocation in XIP mode (#3142) +- Implement the remaining Windows filesystem functions (#3166) +- Fix LLVM assertion failure and update CONTRIBUTING.md (#3197) +- Allow overriding max memory on module instantiation (#3198) +- Get location info from function indexes in addr2line script (#3206) +- Demangle function names in stack trace when using addr2line script (#3211) +- Refactor APIs and data structures as preliminary work for Memory64 (#3209) +- Allow converting the zero wasm address to native (#3215) +- Small refactor on WASMModuleInstance and fix Go/Python language bindings (#3227) +- Add esp32c6 support (#3234) +- Make android platform's cmake flags configurable (#3239) +- Go binding: Change C.long to C.int64_t when call wasm_runtime_set_wasi_args_ex (#3235) +- Implement apis to set and get the name of a wasm module (#3254) +- Append '\0' to every name string in aot name section (#3249) +- Add cmake flag to control aot intrinsics (#3261) +- Add lock and ref_count for runtime init (#3263) +- nuttx: Migrate NuttX CMake build for WAMR (#3256) +- LLVM 19: Switch to debug records (#3272) +- aot debug: Process lldb_function_to_function_dbi only for C (#3278) +- Fix warnings/issues reported in Windows and by CodeQL/Coverity (#3275) +- Enhance wasm loading with LoadArgs and support module names (#3265) +- Add wamr to esp-idf components registry (#3287) +- zephyr: Add missing pthread library functions (#3291) +- Add more checks in wasm loader (#3300) +- Log warning if growing table failed (#3310) +- Enhance GC subtyping checks (#3317) +- User defined memory allocator for different purposes (#3316) +- Add a comment on WASM_STACK_GUARD_SIZE (#3332) +- Allow executing malloc/free from native in memory64 mode (#3315) +- Add functions to expose module import/export info (#3330) + +### Others +- Add ARM MacOS to the CI (#3120) +- Download jetstream src from github instead of browserbench.org (#3196) +- Update document to add wamr-rust-sdk introduction (#3204) +- Fix nightly run tsan ASLR issue (#3233) +- Add CodeQL Workflow for Code Security Analysis (#2812) +- Add issue templates (#3248) +- Fix CI error when install packages for macos-14 (#3270) +- Update document for GC, exception handling and memory64 features (#3284) +- Update release CI (#3295) +- Add native-stack-overflow sample (#3321) + +--- + +## WAMR-1.3.2 + +### Breaking Changes + +### New Features +- Implement Exception Handling for classic interpreter (#3096) + - Use `cmake -DWAMR_BUILD_EXCE_HANDLING=1/0` option to enable/disable + the feature, and by default it is disabled + - It is still in highly experimental stage + +### Bug Fixes +- Fix build errors when initializing wasm_val_t values with macros (#3007) +- fix(wasm-c-api): Do not clone stack frames if there's no trap (#3008) +- classic-interp: Handle SIMD opcode when JIT is enabled (#3046) +- fast-interp: Fix dynamic offset error issue in else branch (#3058) +- wasm_cluster_destroy_spawned_exec_env: Avoid "invalid exec env" trap (#3068) +- thread-mgr: Fix locking problems around aux stack allocation (#3073) +- cosmopolitan: Update compiler and update platform_internal.h (#3079) +- wasi: Apply wasm_runtime_begin_blocking_op to poll as well (#3080) +- Fix memory/table segment checks in memory.init/table.init (#3081) +- perf profiling: Adjust the calculation of execution time (#3089) +- aot: Fix LLVMSetTailCallKind check (#3099) +- fast-interp: Fix stack recovery for else branch (#3100) +- fast-interp: Fix frame_offset pop order (#3101) +- Fix AOT compilation on MacOS (#3102) +- fast-interp: Fix block with parameter in polymorphic stack issue (#3112) +- Fix read and validation of misc/simd/atomic sub opcodes (#3115) + +### Enhancements +- Clear compilation warning and dead code (#3002) +- aot debug: Try to use a bit more appropriate file names (#3000) +- Increase default app thread stack size (#3010) +- Rename rwlock_init to avoid conflict (#3016) +- nuttx: Use larger alignment for os_mmap and comment why (#3017) +- Allow using mmap for shared memory if hw bound check is disabled (#3029) +- Don't redefine D_INO if already defined (#3036) +- Enhancements on wasm function execution time statistic (#2985) +- wamr-compiler: Fix non-x86{_64} host builds (#3037) +- Disable quick aot entry for interp and fast-jit (#3039) +- nuttx: Add option to enable quick aot entry (#3040) +- Set CONFIG_HAS_CAP_ENTER to support posix file api for freertos (#3041) +- Revert "Enable MAP_32BIT for macOS (#2992)" (#3032) +- Enable quick aot entry when hw bound check is disabled (#3044) +- Do not inherit WASM_SUSPEND_FLAG_BLOCKING from the parent thread (#3051) +- wasm_runtime_begin_blocking_op: A comment about usage expectation (#3056) +- Check arguments before calling bh_hash_map_find (#3055) +- Fix aot large model (--size-level=0) with LLVM 18 (#3057) +- Add flag to control Winsocket initialization (#3060) +- nuttx: If STACK_GUARD_SIZE is not set, leave it to config.h (#2927) +- Enhance setting write gs base with cmake variable (#3066) +- aot_reloc_x86_64.c: Suggest to try --size-level=0 as well (#3067) +- Fix some issues reported by CodeQL (#3064) +- Remove a lot of "unused parameter" warnings (#3075) +- Forward log and log level to custom bh_log callback (#3070) +- Fix inconsistent code style in aot_loader.c (#3082) +- freertos: Thread exit more common (#3094) +- Fix windows build error and compilation warnings (#3095) + +### Others +- Fix nightly-run CI failure (#3014) +- Build samples in debug mode (#3019) +- Remove deprecated tests in language-bindings python (#3018) +- Avoid unused thread_id warning and recompile multi-module sample (#3033) +- samples/terminate: Add a sample to demonstrate wasm_runtime_terminate (#3043) +- Bump NuttX version to 12.4.x in CI (#3047) +- perf_tune.md: Add refine the calling processes between host and wasm (#3065) +- build_wamr.md: Update the document (#3074) +- Fix download link for wasi-sdk (#3077) +- README.md: Fix typo tunning to tuning (#3078) +- Update outdated reference link in multi_module.md (#3092) +- Add comments to suppress warning from clang-tidy (#3088) +- CI: Update version of checkout to suppress warnings (#3093) +- test_wamr.sh: Allow using test script on different platforms (#3098) + +--- + +## WAMR-1.3.1 + +### Breaking Changes +- In multi-threading, when an exception was thrown in wasm_func_call(), + the trap returned contains the stack frames of the thread where the + exception occurs, but not the stack frames of the main thread. +- Disable emitting custom name section to AOT file with + `wamrc --enable-dump-call-stack` option, instead, use + `wamrc --emit-custom-sections=name` to emit it and make it clear. + +### New Features +- Enable AOT linux perf support (#2930) + +### Bug Fixes +- Corrects Zephyr include files for current versions of Zephyr (#2881) +- Fix possible dead lock in wasm_cluster_spawn_exec_env (#2882) +- Handle ambiguous fstflags on fd_filestat_set_times (#2892) +- Fix memory size not updating after growing in interpreter (#2898) +- fixed(freertos): Fix crash when wasm app call pthread_exit(NULL) (#2970) +- fast-jit: Fix const shift and const i64 compare issues (#2969) +- Fix ref.is_null processing in fast-interp loader (#2971) +- simd-128: The input lanes of integer-to-integer narrowing ops should be interpreted as signed (#2850) +- Fix ref.func function declared check in wasm loader (#2972) +- Fix fast-interp polymorphic stack processing (#2974) +- Fix potential recursive lock in pthread_create_wrapper (#2980) +- Fix build failure on esp-idf platform (#2991) +- Return stack frames of crashed thread when using wasm-c-api (#2908) +- Fix compilation error on iOS due to macOS-specific API (#2995) +- Fix a bug when emit the custom name section to aot file (#2987) +- Fix linux-sgx build error when libc-wasi is disabled (#2997) + +### Enhancements +- fix command-reactor: Look for _initialize only if _start not found (#2891) +- Refactor reloc symbols for riscv (#2894) +- Avoid memory import failure when wasi-threads is enabled (#2893) +- interpreter: Simplify memory.grow a bit (#2899) +- Avoid reporting timestamp if custom logger is used (#2905) +- Expose API to set log level in embedder (#2907) +- Add a script to translate jitted function names in flamegraph (#2906) +- Refine wasm-c-api wasm_func_call (#2922) +- Add VectorCombine pass for JIT and AOT (#2923) +- Enable wasm_runtime_terminate for single-threading (#2924) +- nuttx: Add CONFIG_INTERPRETERS_WAMR_DEBUG_AOT (#2929) +- Allow to control built-in libraries for wamrc from command line options (#2928) +- Fix a bug that appends '_precheck' to aot_func (#2936) +- freertos: Add os_cond_broadcast for pthread wrapper (#2937) +- Append .aot to .wasm as a custom section named "aot" (#2933) +- fix(sgx-ra): Fix building when enclave is built without librats ahead (#2968) +- Refine LLVM JIT function call process (#2925) +- Refine AOT function call process (#2940) +- Allow to set segue flags for wasm-c-api JIT (#2926) +- freertos: Minor changes for freertos libc_wasi build adaption (#2973) +- freertos: Change ssp_config.h due to clock_nanosleep() not supported in freertos (#2979) +- aot compiler: Some updates for LLVM 18 (#2981) +- Enable MAP_32BIT for macOS (#2992) +- Register quick call entries to speedup the aot/jit func call process (#2978) +- Refine AOT/JIT code call wasm-c-api import process (#2982) + +### Others +- compilation_on_nuttx.yml: Use docker image to simplify env setup (#2878) +- samples/spawn-thread: Disable libc and pthread (#2883) +- Add arm64 to nuttx compilation test (#2886) +- samples/spawn-thread: Tweak to expose a bug (#2888) +- Fix typo in CI config and suppress STORE_U8 in TSAN (#2802) +- Using docker image for nuttx spectest (#2887) +- doc: Separate source_debugging.md into two files (#2932) +- doc/build_wasm_app.md: Add a note about aot abi compatibility (#2993) + +--- + +## WAMR-1.3.0 + +### Breaking Changes +- Abstract POSIX filesystem functions (#2585) + - Change API wasm_runtime_set_wasi_args_ex's arguments + `int stdinfd/stdoutfd/stderrfd` to `int64_t stdinfd/stdoutfd/stderrfd` +- core/iwasm: Support mapped file system access on non-libuv WASI (#2628) + - Enable mapping host directories to guest directories by parsing + the `map_dir_list` argument in API `wasm_runtime_init_wasi` for libc-wasi +- Support muti-module for AOT mode (#2482) + - Add argument `package_type_t module_type` for module_reader callback +- Generate jitdump to support linux perf for LLVM JIT (#2788) + - Add a field `bool linux_perf_support` in RuntimeInitArgs +- Remove provision of unnecessary fd rights (#2579) +- libc-wasi: Conditionally support SYNC flags (#2581) + +### New Features +- Support muti-module for AOT mode (#2482) +- Implement libc-wasi for Windows platform (#2740) +- Implement module instance context APIs (#2436) +- Implement async termination of blocking thread (#2516) +- Generate jitdump to support linux perf for LLVM JIT (#2788) +- Add Cosmopolitan Libc Platform (#2598) + +### Bug Fixes +- sgx-ra: Disable the building of samples (#2507) +- Handle a return from wasi _start function correctly (#2529) +- fd_object_release: Preserve errno (#2535) +- Fix build error with ancient GCC (4.8) (#2553) +- Fix compiling error for RT-Thread (#2569) +- Fix potential unaligned store issue when extra return value is v128 (#2583) +- Fix loader push_pop_frame_ref_offset (#2590) +- Fix compilation error on Android platform (#2594) +- Ignore handling SIG_DFL/SIG_IGN for previous sig action (#2589) +- Fix nightly run sanitizer error in Fast JIT (#2601) +- Check ValueKind before extracting a constant int value (#2595) +- Patch implementations of vfbinop(min,max,pmin,pax) (#2584) +- Improve stack trace dump and fix coding guideline CI (#2599) +- aot_resolve_stack_sizes: Disable the size check for now (#2608) +- Remove module instance from hashmap in wasi_nn_destroy (#2613) +- Fix label index out-of-range references in op_br_table_cache (#2615) +- Fix compilation of shift opcodes on x86_64 and i386 architectures (#2619) +- Fix potential issue in aot compiler when translating block opcodes (#2622) +- Use another default pipeline when opt-level is 0 (#2624) +- Fix AOT shift operations for indirect constants (#2627) +- Fix fast-interp "pre-compiled label offset out of range" issue (#2659) +- Revert "Strip static and shared libraries of iwasm to reduce the binary size (#2431)" (#2669) +- Fix windows compilation on C++20 (#2670) +- Fix fast-jit f32/f64 truncate to i32/i64 (#2671) +- Fix use getrandom on cosmopolitan libc (#2674) +- Fix repeatedly initialize shared memory data and protect the memory's fields (#2673) +- Minor fixes for Go bindings (#2676) +- Fix issues reported by Coverity (#2681) +- Add more buffer boundary checks in wasm loader (#2734) +- Grab cluster->lock when modifying exec_env->module_inst (#2685) +- Fix CMSIS import with Zephyr 3.4+ (#2744) +- Fix log messages in Zephyr example (#2761) +- Fix fast-jit callnative translation (#2765) +- aot compiler: Disable musttail for thumb (#2771) +- Fix data/elem drop (#2747) +- Fix formatting in aot_dump_perf_profiling (#2796) +- Fix formatting in wasm_dump_perf_profiling (#2799) +- Fix memory.init opcode issue in fast-interp (#2798) +- aot compiler: Fix handle next reachable if block (#2793) +- Fix configurable bounds checks typo (#2809) +- Attestation: Free JSON from the Wasm module heap (#2803) +- Update Zephyr support to v3.5.0 and make instructions generic to boards (#2805) +- Return error when shutdown() fails (#2801) +- iwasm: Print help when meeting unknown cmd options (#2824) +- Fix fast-jit accessing shared memory's fields issue (#2841) +- Fix wasm loader handle op_br_table and op_drop (#2864) +- Fix block with type issue in fast interp (#2866) +- Fix float argument handling for riscv32 ilp32d (#2871) +- Portably handle fd_advise on directory fd (#2875) +- Fix sample basic intToStr was called with wrong length (#2876) + +### Enhancements +- Implement strict validation of thread IDs according to the specification (#2521) +- Stop abusing shared memory lock to protect exception (#2509) +- Implement os_usleep for posix (#2517) +- set_exception_visitor: Remove the special case for wasi proc exit (#2525) +- Revert "Return error when exception was raised after main thread finishes" (#2524) +- libc-wasi: Remove unused code (#2528) +- Add callback to handle memory.grow failures (#2522) +- Add context to enlarge memory error callback (#2546) +- Add ARM aeabi symbol for clearing memory content in a specific range (#2531) +- Unifdef -U WASMTIME_SSP_STATIC_CURFDS (#2533) +- Fix typo for IP address buffer (#2532) +- Add an API to terminate instance (#2538) +- Add user to enlarge memory error callback (#2546) +- runtest.py: Show accurate case amount in summary (#2549) +- Allow using custom signal handler from non-main thread (#2551) +- Return __WASI_EINVAL from fd_prestat_dir_name (#2580) +- Support AOT compiler with LLVM 17 (#2567) +- Add support for closing/renumbering preopen fds (#2578) +- Enable AOT usage on M1 mac (#2618) +- core/iwasm: Support mapped file system access on non-libuv WASI (#2628) +- Enable MASM automatically in runtime_lib.cmake (#2634) +- Abstract POSIX filesystem functions (#2585) +- Implement wasi clock_time/clock_res get (#2637) +- Fix several typo/warning/unused-code issues (#2655) +- Partial windows filesystem implementation (#2657) +- Apply no_sanitize_address for clang compiler in several places (#2663) +- Refactor clock functions to use WASI types (#2666) +- Refine lock/unlock shared memory (#2682) +- Fix several AOT compiler issues (#2697) +- Fix AOT compiler simd shift opcodes (#2715) +- Fix invalid use of jit_reg_is_const_val in fast-jit (#2718) +- Use user defined malloc/free functions for user defined memory allocator (#2717) +- Move WASI types into separate header (#2724) +- Provide default vprintf on UWP (#2725) +- Fix typo in Zephyr simple example (#2738) +- Fix switch-case fallthrough compilation warning (#2753) +- Add eabihf ABI support and set vendor-sys of bare-metal targets (#2745) +- Return uint32 from WASI functions (#2749) +- Add compilation flag to enable/disable heap corruption check (#2766) +- Extend os_mmap to support map file from fd (#2763) +- Fix printing ref.extern addresses in wasm_application.c (#2774) +- Remove unused JitBitmap (#2775) +- Use next generation crypto API on Windows (#2769) +- More precise help info of enabled targets for wamrc (#2783) +- Refine atomic operation flags in bh_atomic.h (#2780) +- Fix comment in WAMR_MEM_DUAL_BUS_MIRROR (#2791) +- Fix return type in wasm_loader_get_custom_section (#2794) +- Add support for custom sections in nuttx (#2795) +- Change is_shared_memory type from bool to uint8 (#2800) +- Fix typos in zephyr platform struct descriptions (#2818) +- Access linear memory size atomically (#2834) +- Output warning and quit if import/export name contains '\00' (#2806) +- Use wasm_config_t to pass private configuration to wasm_engine_new (#2837) +- core/iwasm/interpreter/wasm_loader.c: remove an extra validation (#2845) +- Don't add "+d" to riscv cpu features if already given (#2855) +- Fix compilation warnings on Windows (#2868) + +### Others +- Add mutex stress test (#2472) +- Add unit tests for the tid allocator (#2519) +- Add support for running tests on apple M1 macs (#2554) +- export_native_api.md: Add a note about thread termination (#2572) +- test_wamr.sh: Print a bit more meaningful message (#2574) +- run_wasi_tests.sh: Provide stdin by ourselves (#2576) +- Fix a few issues in "run_wasi_tests.sh: provide stdin by ourselves" (#2582) +- Fix compile error of tsf benchmark (#2588) +- test_wamr.sh: Bump wasi-testsuite version (#2568) +- samples/inst-context-threads: Add a brief explanation (#2592) +- doc/memory_tune.md: "remove malloc" hack is not relevant to wasi-threads (#2603) +- Refactor stress tests to make them runnable in reactor mode (#2614) +- Run rust tests from wasi-testsuite (#2484) +- spec-test-script: Fix NaN comparision between v128 values (#2605) +- CI: Enable testing AOT multi-module feature (#2621) +- Vote for nomination of Woods, Chris and Trenner, Thomas as TSC members (#2638) +- Add tsan for fast interp and aot (#2679) +- Enable WASI tests on Windows CI (#2699) +- docs: Fix typo in export native APIs doc (#2750) +- Update RISC-V compilers in Nuttx compilation CI and spec test CI (#2756) +- Enable more LLVM backends for the release wamrc binary (#2778) +- Disable FPU in NuttX spec test (#2781) +- Fix broken links in app-mgr README.md (#2786) +- Fix build error of libsodium benchmark (#2792) +- Fix wamr-test-suites script for macos (#2819) +- Run spec test for classic/fast-interp in NuttX CI (#2817) +- test_wamr.sh: Don't bother to build shared library (#2844) +- doc/build_wamr.md: Fix links to RISC-V named ABIs (#2852) +- Fix typos of CIDR in docs and help text (#2851) +- Enable spectest on riscv64 (#2843) +- Update FPU configuration in spec_test_on_nuttx.yml (#2856) + +--- + +## WAMR-1.2.3 + +### Breaking Changes +- Increase default native stack size (#2332) + +### New Features +- Implement the segue optimization for LLVM AOT/JIT (#2230) +- Implement AOT static PGO (#2243) +- Enable static PGO for Linux SGX (#2270) +- Add Rust Formatters to Debugger (Vector, Map etc.) (#2219) + +### Bug Fixes +- The Python language-binding needs python>=3.9 (#2228) +- aot_compile_op_call: Remove a wrong optimization (#2233) +- Fix typo in samples/ref-types (#2236) +- Update thread proposal ignore cases (#2246) +- Disable writting GS register on linux-sgx platform (#2255) +- Fix compile error of wamrc with llvm-13/llvm-14 (#2261) +- aot/jit: Set module layout (#2260) +- Fix build error with LLVM 16 (#2259) +- spec-test-script: Disable conversions.wast on i386 (#2269) +- Fix a heap corruption bug in ems realloc (#2279) +- Fix fast-interp issue of LAST_OP_OUTPUT_I32/64 check (#2295) +- Fix wamrc build issues with LLVM 13 and LLVM 16 (#2313) +- aot: Move stack_sizes table to a dedicated section (#2317) +- product-mini/platforms/linux: Mark vmlib POSITION_INDEPENDENT_CODE (#2323) +- aot: Avoid possible relocations around "stack_sizes" for XIP mode (#2322) +- Avoid switch lowering to lookup tables for XIP (#2339) +- Fix typo in zephyr's Dockerfile.old (#2354) +- Fix typo (dwarf) in the codebase (#2367) +- Implement suspend flags as atomic variable (#2361) +- Fix llvm jit failed to lookup aot_stack_sizes symbol issue (#2384) +- Fix some check issues on table operations (#2392) +- Fix ExpandMemoryOpPass doesn't work properly (#2399) +- Fix non-builtin BH_ATOMIC_32_FETCH_OR and BH_ATOMIC_32_FETCH_AND (#2400) +- Fix wasi-sockets tests (#2389) +- Fix result arity check on select_t opcode (#2406) +- Re-organize intrinsics in aot_reloc_riscv.c to fix some FPU issues (#2414) +- Fix lib-pthread issues (#2410) +- Fix typo in test_wamr.sh (#2421) +- Fix memory sharing (#2415) +- wasm_export.h: Fix struct wasm_val_t (#2435) +- Fix typos in wamrc print_help() (#2442) +- iwasm: Fix native lib cleanup after error occurs (#2443) +- Correct --heap-size option in messages (#2458) +- wasm_instantiate: Fix a potential integer overflow issue (#2459) +- Fix windows link error and clear windows warnings (#2463) +- aot: Disable musttail for mips (#2457) +- Fix opcode overwrite issue in fast interp (#2476) +- wamrc: Fix windows relocation to `aot_func_internal#n` (#2474) +- Fix windows AOT hw bound check (#2475) +- Fix typo in aot_emit_aot_file.c (#2478) + +### Enhancements +- A few changes related to WAMRC_LLC_COMPILER (#2218) +- Enhance linux-sgx CI (#2102) +- Add asan and ubsan to WAMR CI (#2161) +- Update doc on WAMR_DISABLE_HW_BOUND_CHECK 32-bit (#2262) +- wamrc: Add an incompatibility note in the help message (#2276) +- Add cmake variable to disable writing gs register (#2284) +- Make hmu_tree_node 4 byte aligned to reduce compiler warning (#2268) +- Appease unused warning on min_uint64 (#2277) +- Fix format warning by PRIu32 in [wasm|aot] dump call stack (#2251) +- Fix a compile warning due to missing include (#2293) +- Fix dockerfile linter warnings (#2291) +- Enable windows x86-32 AOT relocations (#2285) +- wamr-ide: Add vscode extension tests (#2292) +- AOT/JIT native stack bound check improvement (#2244) +- Add retries to flaky step in nightly run CI (#2306) +- Use system libuv if available (#1861) +- wasi-nn: Simplify cmake and headers' location (#2308) +- wasi-nn: Improve tests paths for local dev (#2309) +- aot: Implement a few more relocation types for riscv (#2318) +- wasi-nn: Add support of wasi-nn as shared lib (#2310) +- Add a few more assertions on structures to which aot abi is sensitive (#2326) +- Fix sanitizer errors in posix socket (#2331) +- Add "--xip" option for wamrc (#2336) +- Add "--enable-llvm-passes=" option to wamrc (#2335) +- Make memory access boundary check behavior configurable (#2289) +- Migrate ExpandMemoryOpPass to llvm new pass manager (#2334) +- Allow defining hints without exact socket type or address family (#2337) +- wamrc: Warn on text relocations for XIP (#2340) +- Add scripts to validate lldb source debugger (#2150) +- Add docker file to fix Zephy ESP32 linking issue (#2314) +- Add "--native-lib=" option to wamrc (#2342) +- Fix unused warnings on disable_bounds_checks (#2347) +- Add "--enable-builtin-intrinsics=" option to wamrc (#2341) +- nuttx: Add a kconfig for wasi-threads (#2343) +- iwasm: Disable app heap by default if wasi is enabled (#2346) +- Fix some static scan issues (#2362) +- Bring up WAMR on esp32-s3 device (#2348) +- ESP-IDF platform supports to load AOT to PSRAM and run it (#2385) +- Add hadolint CI for Dockerfile linting (#2387) +- Move generic parts of wasm_suspend_flags.h to bh_atomic.h (#2393) +- bh_atomic.h: Add comments (#2398) +- bh_atomic.h: Add BH_ATOMIC_32_FETCH_ADD/BH_ATOMIC_32_FETCH_SUB (#2408) +- Update libuv version to v1.46.0 (#2405) +- Remove a few unused functions (#2409) +- Add initial stress test (#2364) +- Move wasm_runtime_destroy_wasi and wasi_nn_destroy calls together (#2418) +- embed_wamr.md: Improvements about threads (#2420) +- Add runtime inited checks in Enclave command handlings to improve security (#2416) +- Add some relocation symbols for xtensa target (#2422) +- Remove unnecessary and extra zero length check in mem functions' macro (#2428) +- Introduce WASMModuleInstanceExtraCommon (#2429) +- Strip static and shared libraries of iwasm to reduce the binary size (#2431) +- Auto-check wrgsbase in cmake script (#2437) +- iwasm: call native lib init/deinit if exists (#2439) +- wasi-nn: Support uint8 quantized networks (#2433) +- Implement `wasm_externref_objdel` and `wasm_externref_set_cleanup` (#2455) +- wasi-nn: Improve TPU support (#2447) +- wamr-python: Enable debugging WASM and grant dir access (#2449) +- Build wasi-libc from source in WAMR CI (#2465) +- wamrc: More friendly to print help info (#2451) +- Add another wamr test (#2411) +- Fix issues reported by Coverity and clear windows warnings (#2467) +- Clone the input binary during wasm_module_validate (#2483) + +### Others +- Nuttx CI: Ignore the expired certificate for riscv gcc toolchain (#2222) +- core/iwasm/compilation: constify a bit (#2223) +- Bump requests from 2.28.2 to 2.31.0 in /build-scripts (#2229) +- dwarf_extractor: Constify a bit (#2278) +- AOTFuncContext: Remove a stale comment (#2283) +- Add performance tunning document (#2286) +- Reduce CI jobs number (#2296) +- CI: Update used node version to 16 (#2303) +- Update Docker image for latest version of external libraries & tools (#2374) +- Upgrade cJSON version to v1.7.16 (#2404) +- Upgrade XNNPACK workload (#2394) +- Build more benchmarks in workload XNNPACK (#2417) +- Upgrade SGX-RA integration for 0.1.2 and Ubuntu 20.04 (#2454) +- Add sample pre-commit hook (#2470) + +--- + +## WAMR-1.2.2 + +### Breaking Changes + +### New Features +- Implement Fast JIT multi-threading feature (#2134) + +### Bug Fixes +- Update request.ts wasm_response_send signature (#2122) +- Fix ems allocator unaligned memory access on riscv64 (#2140) +- libc_wasi_wrapper.c: Fix min func issue for size_t < 8 bytes on some platforms (#2152) +- Fix three multi-threading and wasm-c-api-imports issues (#2173) +- Fix build polybench benchmark error with wasi-sdk-19.0 (#2187) +- Fix wamr-ide debugger ignoring launch config (#2155) + +### Enhancements +- Add test for validating linear memory size updates (#2078) +- Update Zephyr docs to remove unsupported west subcommand (#2128) +- Update messages/comments to refer the new place of the version definition (#2133) +- build_wamr_lldb.yml: sync lldb build options between ubuntu and macos (#2132) +- build_wamr_vscode_ext.yml: vsce publish only on the official repo (#2130) +- VSCode-Extension: Download lldb built for ubuntu 20.04 (#2139) +- Avoid re-installing if Tensorflow is already installed for WASI-NN (#2148) +- wamrc: Add --stack-usage option (#2158) +- Fix URL in language-bindings/python/README.md (#2166) +- Fix URL in embed_wamr.md (#2165) +- Fix URL in README.md (#2168) +- Return error when exception was raised after main thread finishes (#2169) +- wasi-nn: Add external delegation to support several NPU/GPU (#2162) +- Update document for iwasm/wamrc dependent packages (#2183) +- Use a manual flag to disable clock_nanosleep on the unsupported platforms (#2176) +- Fix compile warnings on windows platform (#2208) + +### Others +- CI: Add ubsan checks to samples/wasm-c-api (#2147) +- CI: More precise trigger paths for github actions (#2157) + +--- + +## WAMR-1.2.1 + +### Breaking Changes + +### New Features + +### Bug Fixes +- libc-wasi/posix.c: Fix POLL{RD,WR}NORM in uClibc (#2069) +- Fix bh_assert for 64-bit platforms (#2071) +- wamr-ide: Modify Dockerfile to update base image version and fix build issue (#2068) +- Fix module_malloc/module_free issues (#2072) +- Fix use after free when dumping call stack (#2084) +- Fix compilation errors of workload xnnpack and meshoptimizer (#2081) +- Fix typo in Fast JIT's BUILD_COND_BR Macro (#2092) +- Fix sanitizer pointer overflow warning when perform pointer arithmetic (#2098) +- Update sample workload tensorflow (#2101) +- Fix ref.func forward-declared function check (#2099) +- Fix interpreter read linear memory size for multi-threading (#2088) + +### Enhancements +- Limit the minimal size of bh_hashmap (#2073) +- Bump tensorflow to 2.11.1 in /core/iwasm/libraries/wasi-nn/test (#2061) +- Bump tensorflow to 2.11.1 in install_tensorflow.sh (#2076) +- Add support for universal binaries on OSX (#2060) +- Update documents (#2100) + +### Others +- spectest/nuttx: Increase stack size of iwasm task (#2082) +- ci: Refactor windows build definition (#2087) +- ci: Enable WASI threads in CI (#2086) +- Use wasi-sdk-20 to build wasi-threads cases in CI (#2095) + +--- + +## WAMR-1.2.0 + +### Breaking Changes + + +### New Features +- Implement two-level Multi-tier JIT engine: tier-up from Fast JIT to LLVM JIT to get quick cold startup and better performance +- Enable running mode control for runtime, wasm module instance and iwasm +- Implement wasi-threads feature +- Upgrade toolkits: upgrade to llvm-15.0, wasi-sdk-19.0, emsdk-3.1.28 and so on +- Port WAMR to the FreeBSD platform +- Refactor wasi-nn to simplify the support for multiple frameworks +- wasi-nn: Enable GPU support +- wasi-nn: Support multiple TFLite models +- Add WAMR API bindings in Python +- Add libsodium benchmark + +### Bug Fixes +- Fix wasm-c-api import func link issue in wasm_instance_new +- Fix watchpoint segfault when using debug interp without server +- libc-wasi: Fix spurious poll timeout +- Fix typo verify_module in aot_compiler.c +- Fix failure about preopen of reactor modules +- Fix equal check in AOT XIP float cmp intrinsic +- Fix issue of resolving func name in custom name section +- Fix go language binding build on macos arm64 +- Prevent undefined behavior from c_api_func_imports == NULL +- Fix potential block issue in source debugger +- SGX IPFS: Fix a segfault and support seeking beyond the end of files while using SEEK_CUR/SEEK_END +- Fix undef error about WAMR_BUILD_MEMORY_PROFILING +- Fix jit memory overwritten after instance deinstantiate +- Fix stack alignment issue on ia32 +- Fix explicit casts and types in espidf_socket.c +- Fix potential integer overflow issue in wasm-c-api +- Fix libc-wasi build failure when using clang +- Fix wamrapi python binding for darwin +- Fix getting port issue in posix os_socket_bind +- Fix key error in build_llvm.py +- nuttx: Add missing pthread.h header +- Fix os_socket_addr_resolve() for IPv6 +- Enhance/Fix sample socket-api and workload +- Fix fast-jit build error +- Fix dead lock in source debugger +- fix debugger: Set termination flags also when in debug mode + +### Enhancements +- Add WAMR-IDE vscode extension to the Visual Studio Marketplace +- Refine Windows thread waiting list operations +- Improve wasm-c-api instantiation-time linking +- Enable platform support for esp-idf v5.0.1 +- Readme refactoring +- Add architecture diagram for wasm function +- Add architecture document for wasm export +- Add architecture diagram for wasm globals and classic-interp stack frame +- Use boringssl instead of openssl to implement wasm cache loading +- Implement i32.rem_s and i32.rem_u intrinsic +- Perfect the codebase for wamr-ide +- Remove unnecessary ret value control when spec test is enabled +- Use float version library routine for XIP aot_intrinsic_xxx APIs +- Register missing symbols for f32 to 64 bit integer conversion +- Report error in instantiation when meeting unlinked import globals +- Add more types and APIs for attr_container +- Simplify fcmp intrinsic logic for AOT/XIP +- Add some missing macros for int literals in wamr-sdk libc-builtin-sysroot stdint.h +- nuttx: Mock socket APIs if NET is disabled +- Main thread spread exception when thread-mgr is enabled +- Implement opcode atomic.wait and atomic.notify for Fast JIT +- Add docker images auto check and setup support for WAMR-IDE +- Make memory profiling show native stack usage +- Enable gcc-4.8 compilation +- Enable specifying out-of-source platform configuration cmake file +- Add gh api call for fetching llvm version (#1942) Fixes +- Don't terminate other threads when create thread failed +- Modify poll_oneoff in libc-wasi to make it interruptible +- Expose wasm_runtime_call_indirect +- Make a workaround for EGO when fstat returns NOT_SUPPORT +- Re-org calling post instantiation functions +- Enable custom llvm build flags +- support SSH for git clone llvm +- Support dump call stack on exception and dump call stack on nuttx +- Update document for source debugging +- Document some info about estimating memory usage +- Document the summary of two pthread implementations +- Refine aot compiler check suspend_flags and fix issue of multi-tier jit + +### Others +- Enable XIP in CI daily test +- Integrate wasi test suite to wamr-test-suites and CI +- Add CI for wasi-threads tests +- Update CIs and documents to make naming of generated binaries consist +- Enable CI wasi test suite for x86-32 classic/fast interpreter +- CI: Enable libc-wasi compilation test on NuttX +- CI: Enable Multi-tier JIT by default for released iwasm binary +- Enable CI build for gcc 4.8 on linux + +--- + +## WAMR-1.1.2 + +### Breaking Changes +- Remove the LLVM MCJIT mode, replace it with LLVM ORC JIT eager mode +- Add option to pass user data to the allocator functions of RuntimeInitArgs +- Change how iwasm returns: + - return 1 if an exception was thrown, else + - return the wasi exit code if the wasm app is a wasi app, else + - keep the same behavior as before +- Enable bulk memory by default + +### New Features +- Add control for the native stack check with hardware trap +- Add memory watchpoint support to debugger +- Add wasm_module_obtain() to clone wasm_module_t +- Implement Fast JIT dump call stack and perf profiling +- esp-idf: Add socket support for esp-idf platform + +### Bug Fixes +- Fix XIP issue caused by rem_s on RISC-V +- Fix XIP issues of fp to int cast and int rem/div +- Fix missing float cmp for XIP +- Correct the arch name for armv7a on NuttX +- Fix issue of restoring wasm operand stack +- Fix issue of thumb relocation R_ARM_THM_MOVT_ABS +- Fix fast jit issue of translating opcode i32.rem_s/i64.rem_s +- Fix interp/fast-jit float min/max issues +- Fix missing intrinsics for risc-v which were reported by spec test +- wasm-c-api: Fix init/destroy thread env multiple times issue +- Fix wasm-c-api import func link issue in wasm_instance_new +- Fix sample ref-types/wasm-c-api build error with wat2wasm low version +- Fix zephyr sample build errors +- Fix source debugger error handling: continue executing when detached +- Fix scenario where the timeout for atomic wait is set to negative number +- Fix link cxx object file error when building wamrc for docker image +- Fix XIP issue of handling 64-bit const in 32-bit target + +### Enhancements +- Refactor the layout of interpreter and AOT module instance +- Refactor LLVM JIT: remove mcjit and legacy pass manager, upgrade to ORCv2 JIT +- Refine Fast JIT call indirect and call native process +- Refine Fast JIT accessing memory/table instance and global data +- Refine AOT exception check when function return +- Enable source debugger reconnection +- Add wasm_runtime_get_wasi_exit_code +- linux-sgx: Use non-destructive modes for opening files using SGX IPFS +- Add wasm_runtime_unregister_natives +- Implement invokeNative asm code for MinGW +- Add wamr Blog link and Gitbook link to readme +- Remove unnecessary app heap memory clean operations to reduce process RSS +- Normalize how the global heap pool is configured across iwasm apps +- Refine the stack frame size check in interpreter +- Enlarge the default wasm operand stack size to 64KB +- Use cmake POSITION_INDEPENDENT_CODE instead of hardcoding -pie -fPIE +- Implement R_ARM_THM_MOVT_[ABS|REPL] for thumb +- Suppress the warnings when building with GCC11 +- samples/native-lib: Add a bit more complicated example +- Add mutex initializer for wasm-c-api engine operations +- XIP adaptation for xtensa platform +- Update libuv version number +- Remove an improper assumption when creating wasm_trap +- Avoid initialize LLVM repeatedly +- linux-sgx: Improve the remote attestation +- linux-sgx: Improve the documentation of SGX-RA sample +- linux-sgx: Allow to open files with arbitrary paths in the sandbox using IPFS +- Avoid raising exception when debugging with VSCode +- wamr-test-suites: Update runtest.py to support python3 +- Enable Nuttx spec test option and register aot symbols +- Use wabt binary instead of building from source in spec test +- nuttx: Enable ref types by Kconfig +- Update xtensa LLVM version to 15.x +- Add bh_print_proc_mem() to dump memory info of current process +- Create trap for error message when wasm_instance_new fails +- wamr-test-suites: Add support for ARM/RISCV by QEMU +- Enable to compile WAMR on platforms that don't support IPV6 +- Fix warnings in the posix socket implementation +- Update document for MacOS compilation +- Install patched LLDB on vscode extension activation +- Add ARM aeabi memcpy/memmove/memset symbols for AOT bulk memory ops +- Enable wasm cache loading in wasm-c-api + +### Others +- Add CIs to release new version and publish binary files +- Add more compilation groups of fast jit into CI +- Enable spec test on nuttx and daily run it + +--- + +## WAMR-1.1.1 + +- Implement Linux SGX socket API getpeername, recvfrom and sendto +- Implement Linux SGX POSIX calls based on getsockname and set/getbool +- Integrate WASI-NN into WAMR: support TensorFlow/CPU/F32 in the first stage +- Add timeout send/recv and multicast client/server socket examples +- Support cross building and linking LLVM shared libs for wamrc +- Add darwin support for app_framework +- Add ios support for product-mini +- Update export_native_api.md: Relax the "ground rule" +- wasm_export.h: Add comments on wasm_runtime_register_natives +- Remove unused wasm_runtime_is_module_registered +- samples/multi-module: Examine module registration a bit +- samples/native-lib: Fix exec_env type +- Fix Linux SGX directional OCALL parameter for getsockname +- Fix threads issue to enable running threads spec proposal test cases +- Fix the "register native with iwasm" stuff for macOS +- Fix issues in assemblyscript lib +- Wrap wasi_socket_ext api with extern "C" to fix link failure with cxx project +- Fix invalid size of memory allocated in wasi init +- posix_thread.c: Avoid sem_getvalue deprecation warning on macOS + +--- + +## WAMR-1.1.0 + +- Extend support for Socket API: + - Implement IPv6 (along with IPv4) for all the socket-related operations + - Enable resolving host name IP address by adding a host call to WASI + - Implement a security feature for controlling what domains are allowed to be resolved + - Allow configuring socket options by adding host calls to WASI for setting and reading the options + - Enable connection-less communication between hosts by adding host calls to WASI for sending + - data directly to a given address and receiving messages from a specific address + - Fix verification of the address in the address pool + - Add more samples and update the documents + - Implement SGX IPFS as POSIX backend for file interaction for linux-sgx +- Integrates the Intel SGX feature called Intel Protection File System Library (IPFS) into the runtime + to create, operate and delete files inside the enclave, while guaranteeing the confidentiality and + integrity of the data persisted +- Make libc-builtin buffered printf be a common feature +- Enable passing through arguments for build_llvm.sh +- Update \_\_wasi_sock_accept signature to match wasi_snapshot_preview1 +- Enable build wasi_socket_ext.c with both clang and clang++ +- Add check for code section size, fix interpreter float operations +- Prevent an already detached thread from being detached again for thread manager +- Fix several issues related to AOT debug and update source_debugging.md +- Fix Windows/MSVC build issues and compile warnings +- Fix wasm loader: function sub local count can be 0 +- Fix crash in dumping call stack when the AOT file doesn't contain custom name section +- Fix Dockerfile lint errors and suppress hadolint warnings for pinning versions part +- Fix Fast JIT issues reported by instrument test +- Fix link error for ESP-IDF 4.4.2 +- Fix syntax errors and undefined names in Python code +- Fix issues reported by Coverity +- Fix Go binding build error +- Fix a wrongly named parameter and enhance the docs in bh_hashmap.h + +--- + +## WAMR-1.0.0 + +- Implement Python language binding +- Implement Go language binding +- Implement Fast JIT engine +- Implement hw bound check for interpreter and Fast JIT +- Enable the semantic version mechanism for WAMR +- Implement POSIX semaphore support for linux platform +- Implement SGX getrandom/getentropy without ocall +- Enable remote attestation by librats in SGX mode +- Upgrade WAMR-IDE and source debugging +- Support print exception info in source debugger +- Support emit specified custom sections into AoT file +- Refactor spec test script and CI workflows +- Support integrate 3rd-party toolchains into wamrc +- Enable dump call stack to a buffer +- Enable aot compiler with llvm-14/15 +- Don't suppress prev signal handler in hw bound check +- Remove unnecessary memset after mmap +- Refine wasm\*runtime_call_wasm_a/v +- Enable app management and thread support for esp32 arch +- Enable libc-wasi support for esp-idf arch +- Implement xtensa XIP +- Enable memory leak check +- Introduce basic CI for nuttx +- Update documents +- Fix module_realloc with NULL ptr issue +- Fix a typo of macro in wasm_application.c +- nuttx: add CONFIG_INTERPRETERS_WAMR_PERF_PROFILING +- aot_reloc_xtensa.c: define \_\_packed if not available +- Fix bh_vector extend_vector not locked issue +- Enable build libc-wasi for nuttx +- Fix typo in embed_wamr.md +- Fix drop opcode issue in fast interpreter +- Fix typos in wasm_mini_loader.c +- Fix issues reported by Coverity and Klocwork +- Add missing aot relocation symbols for xtensa target +- Add arc compiler-rt functions and reloc type for mwdt +- Fix get invokeNative float ret value issue with clang compiler +- Make robust on choosing target assumption for X86_32 support +- Fix an issue of wasm_cluster_spread_custom_data when called before exec +- Fix socket api verification of addresses in the address pool +- Add API wasm_runtime_set_module_inst +- Set noexecstack CXX link flags for wamrc +- Add import subtyping validation +- Fix libc-wasi/uvwasi poll/environ_get issues +- Add missing symbol for aot_reloc_arc.c +- Add a dev docker container for WAMR repo +- Fix dump call stack issue in interpreter +- Fix windows thread data issue and enhance windows os_mmap +- Support custom stack guard size +- Implement i64.div and i64.rem intrinsics +- Let iwasm return non-zero value when running failed +- Reserve one pointer size for fast-interp code_compiled_size +- Enable libc-wasi support for esp-idf +- Expose wasm_runtime_get_exec_env_singleton to the API users +- Normalize wasm types to refine interpreter call_indirect +- Remove unused wasm_runtime_create_exec_env_and_call_wasm +- Fix linear memory page count issues +- debug: Retire wasm_debug\*(get|set)\_engine_active mechanism +- wasm_application.c: Do not start debug instance automatically +- Fix typo in simd_conversions.c +- nuttx: Add CONFIG_INTERPRETERS_WAMR_DEBUG_INTERP +- Add a new API to get free memory in memory pool +- Fix multi-module and some other issues +- Fix build issue of the meshoptimizer workload +- Fix build error on alios platform + +--- + +## WAMR-X.Y.Z + +### Breaking Changes + +### New Features + +### Bug Fixes + +### Enhancements + +### Others + +--- + + diff --git a/wasm-micro-runtime/SConscript b/wasm-micro-runtime/SConscript new file mode 100644 index 0000000..7db2741 --- /dev/null +++ b/wasm-micro-runtime/SConscript @@ -0,0 +1,23 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +# for module compiling +import os + +from building import * + +objs = [] +cwd = GetCurrentDir() +list = os.listdir(cwd) + +if GetDepend(['PKG_USING_WAMR']): + wamr_entry_sconscript = os.path.join(cwd, "product-mini", "platforms", "rt-thread", 'SConscript') + wamr_runlib_sconscript = os.path.join(cwd, "build-scripts", 'SConscript') + + objs = objs + SConscript(wamr_entry_sconscript) + objs = objs + SConscript(wamr_runlib_sconscript) + +Return('objs') diff --git a/wasm-micro-runtime/SECURITY.md b/wasm-micro-runtime/SECURITY.md new file mode 100644 index 0000000..fa3398c --- /dev/null +++ b/wasm-micro-runtime/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +Please refer to the [Bytecode Alliance security policy](https://bytecodealliance.org/security) for details on how to report security issues in WebAssembly Micro Runtime, our disclosure policy, and how to receive notifications about security issues. diff --git a/wasm-micro-runtime/TSC_Charter.md b/wasm-micro-runtime/TSC_Charter.md new file mode 100644 index 0000000..56c4a02 --- /dev/null +++ b/wasm-micro-runtime/TSC_Charter.md @@ -0,0 +1,168 @@ +# Project Technical Steering Committee (PTSC) Charter + +## Section 1. Guiding Principle + +The WebAssembly Micro Runtime (WAMR) project is part of the +Bytecode Alliance (BA) which operates transparently, openly, +collaboratively, and ethically. Project proposals, timelines, and status +must not merely be open, but also easily visible to outsiders. + +## Section 2. Project Governance under Bytecode Alliance + +Technical leadership for the WAMR projects within the Bytecode Alliance +is delegated to the projects through the project charter. Though the BA TSC +will not interfere with day-to-day discussions, votes or meetings of the PTSC, +the BA TSC may request additional amendments to the PTSC charter when +there is misalignment between the project charter and the BA mission and values. + + + +The PTSC structure described in this document may be overhauled as part of +establishing a BA TSC in order to adhere to constraints or requirements that +TSC will impose on project-level governance. + +## Section 3. Establishment of the PTSC + +PTSC memberships are not time-limited. There is no maximum size of the PTSC. +The size is expected to vary in order to ensure adequate coverage of important +areas of expertise, balanced with the ability to make decisions efficiently. +The PTSC must have at least four members. + +There is no specific set of requirements or qualifications for PTSC +membership beyond these rules. The PTSC may add additional members to the +PTSC by a standard PTSC motion and vote. A PTSC member may be removed from the +PTSC by voluntary resignation, by a standard PTSC motion, or in accordance to the +participation rules described below. + +Changes to PTSC membership should be posted in the agenda, and may be suggested +as any other agenda item. + +The PTSC may, at its discretion, invite any number of non-voting observers to +participate in the public portion of PTSC discussions and meetings. + +The PTSC shall meet regularly using tools that enable participation by the +community (e.g. weekly on a Zulip channel, or through any other +appropriate means selected by the PTSC ). The meeting shall be directed by +the PTSC Chairperson. Responsibility for directing individual meetings may be +delegated by the PTSC Chairperson to any other PTSC member. Minutes or an +appropriate recording shall be taken and made available to the community +through accessible public postings. + +PTSC members are expected to regularly participate in PTSC activities. + +In the case where an individual PTSC member -- within any three month period -- +attends fewer than 25% of the regularly scheduled meetings, does not +participate in PTSC discussions, *and* does not participate in PTSC votes, the +member shall be automatically removed from the PTSC. The member may be invited +to continue attending PTSC meetings as an observer. + +## Section 4. Responsibilities of the PTSC + +Subject to such policies as may be set by the BA TSC, the WAMR PTSC is +responsible for all technical development within the WAMR project, +including: + +* Setting release dates. +* Release quality standards. +* Technical direction. +* Project governance and process. +* GitHub repository hosting. +* Conduct guidelines. +* Maintaining the list of additional Collaborators. +* Development process and any coding standards. +* Mediating technical conflicts between Collaborators or Foundation +projects. + +The PTSC will define WAMR project’s release vehicles. + +## Section 5. WAMR Project Operations + +The PTSC will establish and maintain a development process for the WAMR +project. The development process will establish guidelines +for how the developers and community will operate. It will, for example, +establish appropriate timelines for PTSC review (e.g. agenda items must be +published at least a certain number of hours in advance of a PTSC +meeting). + +The PTSC and entire technical community will follow any processes as may +be specified by the Bytecode Alliance Board relating to the intake and license compliance +review of contributions, including the Bytecode Alliance IP Policy. + +## Section 6. Elections + +Leadership roles in the WAMR project will be peer elected +representatives of the community. + +For election of persons (such as the PTSC Chairperson), a multiple-candidate +method should be used, such as: + +* [Condorcet][] or +* [Single Transferable Vote][] + +Multiple-candidate methods may be reduced to simple election by plurality +when there are only two candidates for one position to be filled. No +election is required if there is only one candidate and no objections to +the candidate's election. Elections shall be done within the projects by +the Collaborators active in the project. + +The PTSC will elect from amongst voting PTSC members a PTSC Chairperson to +work on building an agenda for PTSC meetings. The PTSC shall hold annual + +elections to select a PTSC Chairperson; there are no limits on the number +of terms a PTSC Chairperson may serve. + +## Section 7. Voting + +For internal project decisions, Collaborators shall operate under Lazy +Consensus. The PTSC shall establish appropriate guidelines for +implementing Lazy Consensus (e.g. expected notification and review time +periods) within the development process. + +The PTSC follows a [Consensus Seeking][] decision making model. When an agenda +item has appeared to reach a consensus the moderator will ask "Does anyone +object?" as a final call for dissent from the consensus. + +If an agenda item cannot reach a consensus a PTSC member can call for +either a closing vote or a vote to table the issue to the next meeting. +The call for a vote must be seconded by a majority of the PTSC or else the +discussion will continue. + +For all votes, a simple majority of all PTSC members for, or against, the issue +wins. A PTSC member may choose to participate in any vote through abstention. + +## Section 8. Project Roles + +The WAMR git repository is maintained by the PTSC and +additional Collaborators who are added by the PTSC on an ongoing basis. + +Individuals making significant and valuable contributions, +“Contributor(s)”, are made Collaborators and given commit-access to the +project. These individuals are identified by the PTSC and their addition +as Collaborators is discussed during a PTSC meeting. Modifications of the +contents of the git repository are made on a collaborative basis as defined in +the development process. + +Collaborators may opt to elevate significant or controversial +modifications, or modifications that have not found consensus to the PTSC +for discussion by assigning the `tsc-agenda` tag to a pull request or +issue. The PTSC should serve as the final arbiter where required. The PTSC +will maintain and publish a list of current Collaborators, as +well as a development process guide for Collaborators and Contributors +looking to participate in the development effort. + +## Section 9. Definitions + +* **Contributors**: contribute code or other artifacts, but do not have +the right to commit to the code base. Contributors work with the +project’s Collaborators to have code committed to the code base. A +Contributor may be promoted to a Collaborator by the PTSC. Contributors should +rarely be encumbered by the PTSC. + +* **Project**: a technical collaboration effort, e.g. a subsystem, that +is organized through the project creation process and approved by the +PTSC. + +[Consensus Seeking]: https://en.wikipedia.org/wiki/Consensus-seeking_decision-making +[Condorcet]: https://en.wikipedia.org/wiki/Condorcet_method +[Single Transferable Vote]: https://en.wikipedia.org/wiki/Single_transferable_vote + diff --git a/wasm-micro-runtime/build-scripts/SConscript b/wasm-micro-runtime/build-scripts/SConscript new file mode 100644 index 0000000..648373b --- /dev/null +++ b/wasm-micro-runtime/build-scripts/SConscript @@ -0,0 +1,51 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import os +from building import * + +cwd = GetCurrentDir() +objs = [] + +WAMR_ROOT_DIR = os.path.join(cwd, "..") +SHARED_DIR = os.path.join(WAMR_ROOT_DIR, 'core', 'shared') +IWASM_DIR = os.path.join(WAMR_ROOT_DIR, 'core', 'iwasm') +DEPS_DIR = os.path.join(WAMR_ROOT_DIR, 'core', 'deps') + +if GetDepend(['WAMR_BUILD_INTERP']): + script_path = os.path.join(IWASM_DIR, 'interpreter', 'SConscript') + objs += SConscript(script_path) + +if GetDepend(['WAMR_BUILD_AOT']): + script_path = os.path.join(IWASM_DIR, 'aot', 'SConscript') + objs += SConscript(script_path) + if GetDepend(['WAMR_BUILD_JIT']): + script_path = os.path.join(IWASM_DIR, 'compilation', 'SConscript') + objs += SConscript(script_path) + +if GetDepend(['WAMR_BUILD_LIBC_BUILTIN']): + objs += SConscript(os.path.join(IWASM_DIR, 'libraries', 'libc-builtin', 'SConscript')) + +if GetDepend(['WAMR_BUILD_LIBC_WASI']): + objs += SConscript(os.path.join(IWASM_DIR, 'libraries', 'libc-wasi', 'SConscript')) + +if GetDepend(['WAMR_BUILD_LIB_PTHREAD']): + objs += SConscript(os.path.join(IWASM_DIR, 'libraries', 'libc-pthread', 'SConscript')) + +if GetDepend(['WAMR_BUILD_THREAD_MGR']): + objs += SConscript(os.path.join(IWASM_DIR, 'libraries', 'thread-mgr', 'SConscript')) + +if GetDepend(['WAMR_BUILD_LIBC_EMCC']): + objs += SConscript(os.path.join(IWASM_DIR, 'libraries', 'libc-emmc', 'SConscript')) + +objs += SConscript(os.path.join(cwd, 'SConscript_config')); + +objs += SConscript(os.path.join(SHARED_DIR, 'platform', 'rt-thread', 'SConscript')) +objs += SConscript(os.path.join(SHARED_DIR, 'mem-alloc', 'SConscript')) +objs += SConscript(os.path.join(IWASM_DIR, 'common', 'SConscript')) +objs += SConscript(os.path.join(SHARED_DIR, 'utils', 'SConscript')) + +Return('objs') diff --git a/wasm-micro-runtime/build-scripts/SConscript_config b/wasm-micro-runtime/build-scripts/SConscript_config new file mode 100644 index 0000000..2401f3a --- /dev/null +++ b/wasm-micro-runtime/build-scripts/SConscript_config @@ -0,0 +1,118 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import os +import re + +from building import * + +Import('rtconfig') + +src = [] +objs = [] +cwd = GetCurrentDir() + +IWASM_INC_DIR = os.path.join(cwd, '..', 'core', 'iwasm', 'include') + +CPPPATH = [IWASM_INC_DIR] + +if rtconfig.BUILD == 'debug': + CPPDEFINES = ['BH_DEBUG=1'] +else: + CPPDEFINES = ['BH_DEBUG=0'] + +if rtconfig.ARCH == 'arm': + if re.match('^cortex-m.*', rtconfig.CPU): + print('[WAMR] using thumbv4t') + CPPDEFINES += ['BUILD_TARGET_THUMB'] + CPPDEFINES += ['RTT_WAMR_BUILD_TARGET_THUMB'] + elif re.match('^cortex-a.*', rtconfig.CPU): + print('[WAMR] using armv7') + CPPDEFINES += ['BUILD_TARGET_ARM'] + CPPDEFINES += ['RTT_WAMR_BUILD_TARGET_ARMV7'] + elif re.match('^cortex-r.*', rtconfig.CPU): + print('[WAMR] using armv7') + CPPDEFINES += ['BUILD_TARGET_ARM'] + CPPDEFINES += ['RTT_WAMR_BUILD_TARGET_ARMV7'] + elif rtconfig.CPU == 'armv6': + print('[WAMR] using armv6') + CPPDEFINES += ['BUILD_TARGET_ARM'] + CPPDEFINES += ['RTT_WAMR_BUILD_TARGET_ARMV6'] + elif re.match('^arm9*', rtconfig.CPU): + print('[WAMR] using armv4') + CPPDEFINES += ['BUILD_TARGET_ARM'] + CPPDEFINES += ['RTT_WAMR_BUILD_TARGET_ARMV4'] +elif rtconfig.ARCH == 'ia32': + CPPDEFINES += ['BUILD_TARGET_X86_32', 'RTT_WAMR_BUILD_TARGET_X86_32'] +else: + print("[WAMR] unknown arch", rtconfig.ARCH) + +if GetDepend(['WAMR_BUILD_INTERP']): + CPPDEFINES += ['WASM_ENABLE_INTERP=1'] + if GetDepend(['WAMR_BUILD_FAST_INTERP']): + CPPDEFINES += ['WASM_ENABLE_FAST_INTERP=1'] + print("[WAMR] fast interpreter was enabled") + else: + CPPDEFINES += ['WASM_ENABLE_FAST_INTERP=0'] + print("[WAMR] fast interpreter was disabled") +else: + CPPDEFINES += ['WASM_ENABLE_INTERP=0'] + +CPPDEFINES += ['WASM_ENABLE_JIT=0'] + +if GetDepend(['WAMR_BUILD_MULTI_MODULE']): + CPPDEFINES += ['WASM_ENABLE_MULTI_MODULE=1'] +else: + CPPDEFINES += ['WASM_ENABLE_MULTI_MODULE=0'] + +if GetDepend(['WAMR_BUILD_SPEC_TEST']): + CPPDEFINES += ['WASM_ENABLE_SPEC_TEST=1'] + print("[WAMR] spec test compatible mode was enabled") + +if GetDepend(['WAMR_BUILD_BULK_MEMORY']): + CPPDEFINES += ['WASM_ENABLE_BULK_MEMORY=1'] + print("[WAMR] Bulk memory feature was enabled") +else: + CPPDEFINES += ['WASM_ENABLE_BULK_MEMORY=0'] + +if GetDepend(['WAMR_BUILD_SHARED_MEMORY']): + CPPDEFINES += ['WASM_ENABLE_SHARED_MEMORY=1'] + print("[WAMR] Shared memory enabled") +else: + CPPDEFINES += ['WASM_ENABLE_SHARED_MEMORY=0'] + +if GetDepend(['WAMR_BUILD_MINI_LOADER']): + CPPDEFINES += ['WASM_ENABLE_MINI_LOADER=1'] + print("[WAMR] mini loader enabled") +else: + CPPDEFINES += ['WASM_ENABLE_MINI_LOADER=0'] + +if GetDepend(['WAMR_DISABLE_HW_BOUND_CHECK']): + CPPDEFINES += ['WASM_DISABLE_HW_BOUND_CHECK=1'] + CPPDEFINES += ['WASM_DISABLE_STACK_HW_BOUND_CHECK=1'] + print("[WAMR] Hardware boundary check disabled") + +if GetDepend(['WAMR_BUILD_SIMD']): + CPPDEFINES += ['WASM_ENABLE_SIMD=1'] + print('[WAMR] SIMD enabled') + +if GetDepend(['WAMR_BUILD_MEMORY_PROFILING']): + CPPDEFINES += ['WASM_ENABLE_MEMORY_PROFILING=1'] + print('[WAMR] Memory profiling enabled') + +if GetDepend(['WAMR_BUILD_CUSTOM_NAME_SECTION']): + CPPDEFINES += ['WASM_ENABLE_CUSTOM_NAME_SECTION=1'] + print('[WAMR] Custom name section enabled') + +if GetDepend(['WAMR_BUILD_TAIL_CALL']): + CPPDEFINES += ['WASM_ENABLE_TAIL_CALL=1'] + print('[WAMR] Tail call enabledd') + +LIBS = ['m'] + +group = DefineGroup('wamr', src, depend = ['PKG_USING_WAMR'], CPPPATH = CPPPATH, CPPDEFINES = CPPDEFINES, LIBS = LIBS) + +Return('group') diff --git a/wasm-micro-runtime/build-scripts/build_llvm.py b/wasm-micro-runtime/build-scripts/build_llvm.py new file mode 100755 index 0000000..e5036e5 --- /dev/null +++ b/wasm-micro-runtime/build-scripts/build_llvm.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import argparse +import os +import pathlib +import requests +import shlex +import shutil +import subprocess +import sysconfig +import sys + + +def clone_llvm(dst_dir, llvm_repo, llvm_branch): + """ + any error will raise CallProcessError + """ + llvm_dir = dst_dir.joinpath("llvm").resolve() + + if not llvm_dir.exists(): + GIT_CLONE_CMD = f"git clone --depth 1 --branch {llvm_branch} {llvm_repo} llvm" + print(GIT_CLONE_CMD) + subprocess.check_output(shlex.split(GIT_CLONE_CMD), cwd=dst_dir) + + return llvm_dir + + +def query_llvm_version(llvm_info): + github_token = os.environ['GH_TOKEN'] + owner_project = llvm_info['repo'].replace("https://github.com/", "").replace(".git", "") + url = f"https://api.github.com/repos/{owner_project}/commits/{llvm_info['branch']}" + headers = { + 'Authorization': f"Bearer {github_token}" + } + + try: + response = requests.request("GET", url, headers=headers, data={}) + response.raise_for_status() + except requests.exceptions.HTTPError as error: + print (error) # for debugging purpose + return None + + response = response.json() + return response['sha'] + + +def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_flags=''): + LLVM_COMPILE_OPTIONS = [ + '-DCMAKE_BUILD_TYPE:STRING="Release"', + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + "-DLLVM_APPEND_VC_REV:BOOL=ON", + "-DLLVM_BUILD_EXAMPLES:BOOL=OFF", + "-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF", + "-DLLVM_ENABLE_BINDINGS:BOOL=OFF", + "-DLLVM_ENABLE_IDE:BOOL=OFF", + "-DLLVM_ENABLE_LIBEDIT=OFF", + "-DLLVM_ENABLE_TERMINFO:BOOL=OFF", + "-DLLVM_ENABLE_ZLIB:BOOL=ON", + "-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF", + "-DLLVM_INCLUDE_DOCS:BOOL=OFF", + "-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF", + "-DLLVM_INCLUDE_UTILS:BOOL=OFF", + "-DLLVM_INCLUDE_TESTS:BOOL=OFF", + "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON", + ] + + # ccache is not available on Windows + if not "windows" == platform: + LLVM_COMPILE_OPTIONS.append("-DLLVM_CCACHE_BUILD:BOOL=ON") + # perf support is available on Linux only + if "linux" == platform: + LLVM_COMPILE_OPTIONS.append("-DLLVM_USE_PERF:BOOL=ON") + + # use clang/clang++/lld. but macos doesn't support lld + if not sys.platform.startswith("darwin") and use_clang: + if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"): + os.environ["CC"] = "clang" + os.environ["CXX"] = "clang++" + LLVM_COMPILE_OPTIONS.append('-DLLVM_USE_LINKER:STRING="lld"') + print("Use the clang toolchain") + else: + print("Can not find clang, clang++ and lld, keep using the gcc toolchain") + else: + print("Use the gcc toolchain") + + LLVM_EXTRA_COMPILE_OPTIONS = { + "arc": [ + '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="ARC"', + "-DLLVM_ENABLE_LIBICUUC:BOOL=OFF", + "-DLLVM_ENABLE_LIBICUDATA:BOOL=OFF", + ], + "xtensa": [ + '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="Xtensa"', + ], + "windows": [ + "-DCMAKE_INSTALL_PREFIX=LLVM-install", + ], + "default": [], + } + + LLVM_TARGETS_TO_BUILD = [ + '-DLLVM_TARGETS_TO_BUILD:STRING="' + ";".join(backends) + '"' + if backends + else '-DLLVM_TARGETS_TO_BUILD:STRING="AArch64;ARM;Mips;RISCV;X86"' + ] + + LLVM_PROJECTS_TO_BUILD = [ + '-DLLVM_ENABLE_PROJECTS:STRING="' + ";".join(projects) + '"' if projects else "" + ] + + # lldb project requires libxml2 + LLVM_LIBXML2_OPTION = [ + "-DLLVM_ENABLE_LIBXML2:BOOL=" + ("ON" if "lldb" in projects else "OFF") + ] + + # enabling LLVM_INCLUDE_TOOLS will increase ~300M to the final package + LLVM_INCLUDE_TOOLS_OPTION = [ + "-DLLVM_INCLUDE_TOOLS:BOOL=ON" if projects else "-DLLVM_INCLUDE_TOOLS:BOOL=OFF" + ] + + if not llvm_dir.exists(): + raise Exception(f"{llvm_dir} doesn't exist") + + build_dir = llvm_dir.joinpath( + "win32build" if "windows" == platform else "build" + ).resolve() + build_dir.mkdir(exist_ok=True) + + lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve() + if lib_llvm_core_library.exists(): + print( + f"It has already been fully compiled. If want to a re-build, please remove {build_dir} manually and try again" + ) + return None + + compile_options = " ".join( + LLVM_COMPILE_OPTIONS + + LLVM_LIBXML2_OPTION + + LLVM_EXTRA_COMPILE_OPTIONS.get( + platform, LLVM_EXTRA_COMPILE_OPTIONS["default"] + ) + + LLVM_TARGETS_TO_BUILD + + LLVM_PROJECTS_TO_BUILD + + LLVM_INCLUDE_TOOLS_OPTION + ) + + CONFIG_CMD = f"cmake {compile_options} {extra_flags} ../llvm" + if "windows" == platform: + if "mingw" in sysconfig.get_platform().lower(): + CONFIG_CMD += " -G'Unix Makefiles'" + else: + CONFIG_CMD += " -A x64" + else: + CONFIG_CMD += " -G'Ninja'" + subprocess.check_call(shlex.split(CONFIG_CMD), cwd=build_dir) + + BUILD_CMD = "cmake --build . --target package" + ( + " --config Release" if "windows" == platform else "" + ) + subprocess.check_call(shlex.split(BUILD_CMD), cwd=build_dir) + + return build_dir + + +def repackage_llvm(llvm_dir): + build_dir = llvm_dir.joinpath("./build").resolve() + + packs = [f for f in build_dir.glob("LLVM-*.tar.gz")] + if len(packs) > 1: + raise Exception("Find more than one LLVM-*.tar.gz") + + if not packs: + return + + llvm_package = packs[0].name + # mv build/LLVM-*.gz . + shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir)) + # rm -r build + shutil.rmtree(str(build_dir)) + # mkdir build + build_dir.mkdir() + # tar xf ./LLVM-*.tar.gz --strip-components=1 --directory=build + CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}" + subprocess.check_call(shlex.split(CMD), cwd=llvm_dir) + # rm ./LLVM-1*.gz + os.remove(llvm_dir.joinpath(llvm_package).resolve()) + + +def main(): + parser = argparse.ArgumentParser(description="build necessary LLVM libraries") + parser.add_argument( + "--platform", + type=str, + choices=["android", "arc", "darwin", "linux", "windows", "xtensa"], + help="identify current platform", + ) + parser.add_argument( + "--arch", + nargs="+", + type=str, + choices=[ + "AArch64", + "ARC", + "ARM", + "Mips", + "RISCV", + "WebAssembly", + "X86", + "Xtensa", + ], + help="identify LLVM supported backends, separate by space, like '--arch ARM Mips X86'", + ) + parser.add_argument( + "--project", + nargs="+", + type=str, + default="", + choices=["clang", "lldb"], + help="identify extra LLVM projects, separate by space, like '--project clang lldb'", + ) + parser.add_argument( + "--llvm-ver", + action="store_true", + help="return the version info of generated llvm libraries", + ) + parser.add_argument( + "--use-clang", + action="store_true", + help="use clang instead of gcc", + ) + parser.add_argument( + "--extra-cmake-flags", + type=str, + default="", + help="custom extra cmake flags", + ) + options = parser.parse_args() + + # if the "platform" is not identified in the command line option, + # detect it + if not options.platform: + if sys.platform.startswith("win32") or sys.platform.startswith("msys"): + platform = "windows" + elif sys.platform.startswith("darwin"): + platform = "darwin" + else: + platform = "linux" + else: + platform = options.platform + + llvm_repo_and_branch = { + "arc": { + "repo": "https://github.com/llvm/llvm-project.git", + "repo_ssh": "git@github.com:llvm/llvm-project.git", + "branch": "release/15.x", + }, + "xtensa": { + "repo": "https://github.com/espressif/llvm-project.git", + "repo_ssh": "git@github.com:espressif/llvm-project.git", + "branch": "xtensa_release_15.x", + }, + "default": { + "repo": "https://github.com/llvm/llvm-project.git", + "repo_ssh": "git@github.com:llvm/llvm-project.git", + "branch": "release/15.x", + }, + } + + # retrieve the real file + current_file = pathlib.Path(__file__) + if current_file.is_symlink(): + current_file = pathlib.Path(os.readlink(current_file)) + + current_dir = current_file.parent.resolve() + deps_dir = current_dir.joinpath("../core/deps").resolve() + + try: + llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"]) + + if options.llvm_ver: + commit_hash = query_llvm_version(llvm_info) + print(commit_hash) + return commit_hash is not None + + repo_addr = llvm_info["repo"] + if os.environ.get('USE_GIT_SSH') == "true": + repo_addr = llvm_info["repo_ssh"] + else: + print("To use ssh for git clone, run: export USE_GIT_SSH=true") + + llvm_dir = clone_llvm(deps_dir, repo_addr, llvm_info["branch"]) + if ( + build_llvm( + llvm_dir, platform, options.arch, options.project, options.use_clang, + options.extra_cmake_flags + ) + is not None + ): + repackage_llvm(llvm_dir) + + return True + except subprocess.CalledProcessError: + return False + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/wasm-micro-runtime/build-scripts/config_common.cmake b/wasm-micro-runtime/build-scripts/config_common.cmake new file mode 100644 index 0000000..c3a957d --- /dev/null +++ b/wasm-micro-runtime/build-scripts/config_common.cmake @@ -0,0 +1,564 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +string(TOUPPER ${WAMR_BUILD_TARGET} WAMR_BUILD_TARGET) + +# Add definitions for the build target +if (WAMR_BUILD_TARGET STREQUAL "X86_64") + add_definitions(-DBUILD_TARGET_X86_64) +elseif (WAMR_BUILD_TARGET STREQUAL "AMD_64") + add_definitions(-DBUILD_TARGET_AMD_64) +elseif (WAMR_BUILD_TARGET STREQUAL "X86_32") + add_definitions(-DBUILD_TARGET_X86_32) +elseif (WAMR_BUILD_TARGET MATCHES "ARM.*") + if (WAMR_BUILD_TARGET MATCHES "(ARM.*)_VFP") + add_definitions(-DBUILD_TARGET_ARM_VFP) + add_definitions(-DBUILD_TARGET="${CMAKE_MATCH_1}") + else () + add_definitions(-DBUILD_TARGET_ARM) + add_definitions(-DBUILD_TARGET="${WAMR_BUILD_TARGET}") + endif () +elseif (WAMR_BUILD_TARGET MATCHES "THUMB.*") + if (WAMR_BUILD_TARGET MATCHES "(THUMB.*)_VFP") + add_definitions(-DBUILD_TARGET_THUMB_VFP) + add_definitions(-DBUILD_TARGET="${CMAKE_MATCH_1}") + else () + add_definitions(-DBUILD_TARGET_THUMB) + add_definitions(-DBUILD_TARGET="${WAMR_BUILD_TARGET}") + endif () +elseif (WAMR_BUILD_TARGET MATCHES "AARCH64.*") + add_definitions(-DBUILD_TARGET_AARCH64) + add_definitions(-DBUILD_TARGET="${WAMR_BUILD_TARGET}") +elseif (WAMR_BUILD_TARGET STREQUAL "MIPS") + add_definitions(-DBUILD_TARGET_MIPS) +elseif (WAMR_BUILD_TARGET STREQUAL "XTENSA") + add_definitions(-DBUILD_TARGET_XTENSA) +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV64" OR WAMR_BUILD_TARGET STREQUAL "RISCV64_LP64D") + add_definitions(-DBUILD_TARGET_RISCV64_LP64D) +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV64_LP64") + add_definitions(-DBUILD_TARGET_RISCV64_LP64) +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV32" OR WAMR_BUILD_TARGET STREQUAL "RISCV32_ILP32D") + add_definitions(-DBUILD_TARGET_RISCV32_ILP32D) +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV32_ILP32") + add_definitions(-DBUILD_TARGET_RISCV32_ILP32) +elseif (WAMR_BUILD_TARGET STREQUAL "ARC") + add_definitions(-DBUILD_TARGET_ARC) +else () + message (FATAL_ERROR "-- WAMR build target isn't set") +endif () + +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + add_definitions(-DBH_DEBUG=1) +endif () + +if (CMAKE_SIZEOF_VOID_P EQUAL 8) + if (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64" + OR WAMR_BUILD_TARGET MATCHES "AARCH64.*" OR WAMR_BUILD_TARGET MATCHES "RISCV64.*") + if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + # Add -fPIC flag if build as 64-bit + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "${CMAKE_SHARED_LIBRARY_LINK_C_FLAGS} -fPIC") + set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS} -fPIC") + endif () + else () + include(CheckCCompilerFlag) + Check_C_Compiler_Flag(-m32 M32_OK) + if (M32_OK OR WAMR_BUILD_TARGET STREQUAL "X86_32") + add_definitions (-m32) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32") + endif () + endif () +endif () + +if (WAMR_BUILD_TARGET MATCHES "ARM.*") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -marm") +elseif (WAMR_BUILD_TARGET MATCHES "THUMB.*") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mthumb") + set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -Wa,-mthumb") +endif () + +if (NOT WAMR_BUILD_INTERP EQUAL 1) +if (NOT WAMR_BUILD_AOT EQUAL 1) + message (FATAL_ERROR "-- WAMR Interpreter and AOT must be enabled at least one") +endif () +endif () + +if (WAMR_BUILD_FAST_JIT EQUAL 1) + if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0) + # Enable Lazy JIT by default + set (WAMR_BUILD_LAZY_JIT 1) + endif () +endif () + +if (WAMR_BUILD_JIT EQUAL 1) + if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0) + # Enable Lazy JIT by default + set (WAMR_BUILD_LAZY_JIT 1) + endif () + if (NOT DEFINED LLVM_DIR) + set (LLVM_SRC_ROOT "${WAMR_ROOT_DIR}/core/deps/llvm") + set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/build") + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/win32build") + endif () + if (NOT EXISTS "${LLVM_BUILD_ROOT}") + message (FATAL_ERROR "Cannot find LLVM dir: ${LLVM_BUILD_ROOT}") + endif () + set (CMAKE_PREFIX_PATH "${LLVM_BUILD_ROOT};${CMAKE_PREFIX_PATH}") + set (LLVM_DIR ${LLVM_BUILD_ROOT}/lib/cmake/llvm) + endif () + find_package(LLVM REQUIRED CONFIG) + include_directories(${LLVM_INCLUDE_DIRS}) + add_definitions(${LLVM_DEFINITIONS}) + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + + # Disable -Wredundant-move when building LLVM JIT + include(CheckCXXCompilerFlag) + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + check_cxx_compiler_flag("-Wredundant-move" CXX_SUPPORTS_REDUNDANT_MOVE_FLAG) + if (CXX_SUPPORTS_REDUNDANT_MOVE_FLAG) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-redundant-move") + endif () + # Enable exporting symbols after llvm-17, or LLVM JIT may run failed + # with `llvm_orc_registerEHFrameSectionWrapper` symbol not found error + if (${LLVM_PACKAGE_VERSION} VERSION_GREATER_EQUAL "17.0.0") + set (CMAKE_ENABLE_EXPORTS 1) + endif () + endif () +else () + unset (LLVM_AVAILABLE_LIBS) +endif () + +# Sanitizers + +if (NOT DEFINED WAMR_BUILD_SANITIZER) + set(WAMR_BUILD_SANITIZER $ENV{WAMR_BUILD_SANITIZER}) +endif () + +if (NOT DEFINED WAMR_BUILD_SANITIZER) + set(WAMR_BUILD_SANITIZER "") +elseif (WAMR_BUILD_SANITIZER STREQUAL "ubsan") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=alignment" ) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") +elseif (WAMR_BUILD_SANITIZER STREQUAL "asan") + if (NOT WAMR_BUILD_JIT EQUAL 1) + set (ASAN_OPTIONS "verbosity=2 debug=true ") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer -fsanitize=address -fno-sanitize-recover=all" ) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + endif() +elseif (WAMR_BUILD_SANITIZER STREQUAL "tsan") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" ) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") +elseif (NOT (WAMR_BUILD_SANITIZER STREQUAL "") ) + message(SEND_ERROR "Unsupported sanitizer: ${WAMR_BUILD_SANITIZER}") +endif() + +if (WAMR_BUILD_LINUX_PERF EQUAL 1) + if (NOT WAMR_BUILD_JIT AND NOT WAMR_BUILD_AOT) + message(WARNING "only support perf in aot and llvm-jit") + set(WAMR_BUILD_LINUX_PERF 0) + endif () +endif () + +######################################## + +message ("-- Build Configurations:") +message (" Build as target ${WAMR_BUILD_TARGET}") +message (" CMAKE_BUILD_TYPE " ${CMAKE_BUILD_TYPE}) +if (WAMR_BUILD_INTERP EQUAL 1) + message (" WAMR Interpreter enabled") +else () + message (" WAMR Interpreter disabled") +endif () +if (WAMR_BUILD_AOT EQUAL 1) + message (" WAMR AOT enabled") +else () + message (" WAMR AOT disabled") +endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1) + if (WAMR_BUILD_LAZY_JIT EQUAL 1) + add_definitions("-DWASM_ENABLE_LAZY_JIT=1") + message (" WAMR Fast JIT enabled with Lazy Compilation") + else () + message (" WAMR Fast JIT enabled with Eager Compilation") + endif () +else () + message (" WAMR Fast JIT disabled") +endif () +if (WAMR_BUILD_JIT EQUAL 1) + add_definitions("-DWASM_ENABLE_JIT=1") + if (WAMR_BUILD_LAZY_JIT EQUAL 1) + add_definitions("-DWASM_ENABLE_LAZY_JIT=1") + message (" WAMR LLVM ORC JIT enabled with Lazy Compilation") + else () + message (" WAMR LLVM ORC JIT enabled with Eager Compilation") + endif () +else () + message (" WAMR LLVM ORC JIT disabled") +endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1 AND WAMR_BUILD_JIT EQUAL 1 + AND WAMR_BUILD_LAZY_JIT EQUAL 1) + message (" Multi-tier JIT enabled") +endif () +if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) + message (" Libc builtin enabled") +else () + message (" Libc builtin disabled") +endif () +if (WAMR_BUILD_LIBC_UVWASI EQUAL 1) + message (" Libc WASI enabled with uvwasi implementation") +elseif (WAMR_BUILD_LIBC_WASI EQUAL 1) + message (" Libc WASI enabled") +else () + message (" Libc WASI disabled") +endif () +if ((WAMR_BUILD_FAST_INTERP EQUAL 1) AND (WAMR_BUILD_INTERP EQUAL 1)) + add_definitions (-DWASM_ENABLE_FAST_INTERP=1) + message (" Fast interpreter enabled") +else () + add_definitions (-DWASM_ENABLE_FAST_INTERP=0) + message (" Fast interpreter disabled") +endif () +if (WAMR_BUILD_MULTI_MODULE EQUAL 1) + add_definitions (-DWASM_ENABLE_MULTI_MODULE=1) + message (" Multiple modules enabled") +else () + add_definitions (-DWASM_ENABLE_MULTI_MODULE=0) + message (" Multiple modules disabled") +endif () +if (WAMR_BUILD_SPEC_TEST EQUAL 1) + add_definitions (-DWASM_ENABLE_SPEC_TEST=1) + message (" spec test compatible mode is on") +endif () +if (NOT DEFINED WAMR_BUILD_BULK_MEMORY) + # Enable bulk memory by default + set (WAMR_BUILD_BULK_MEMORY 1) +endif () +if (WAMR_BUILD_BULK_MEMORY EQUAL 1) + add_definitions (-DWASM_ENABLE_BULK_MEMORY=1) + message (" Bulk memory feature enabled") +else () + add_definitions (-DWASM_ENABLE_BULK_MEMORY=0) + message (" Bulk memory feature disabled") +endif () +if (WAMR_BUILD_SHARED_MEMORY EQUAL 1) + add_definitions (-DWASM_ENABLE_SHARED_MEMORY=1) + message (" Shared memory enabled") +else () + add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0) +endif () +if (WAMR_BUILD_MEMORY64 EQUAL 1) + # if native is 32-bit or cross-compiled to 32-bit + if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*") + message (FATAL_ERROR "-- Memory64 is only available on the 64-bit platform/target") + endif() + add_definitions (-DWASM_ENABLE_MEMORY64=1) + set (WAMR_DISABLE_HW_BOUND_CHECK 1) + message (" Memory64 memory enabled") +endif () +if (WAMR_BUILD_THREAD_MGR EQUAL 1) + message (" Thread manager enabled") +endif () +if (WAMR_BUILD_LIB_PTHREAD EQUAL 1) + message (" Lib pthread enabled") +endif () +if (WAMR_BUILD_LIB_PTHREAD_SEMAPHORE EQUAL 1) + message (" Lib pthread semaphore enabled") +endif () +if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) + message (" Lib wasi-threads enabled") +endif () +if (WAMR_BUILD_LIBC_EMCC EQUAL 1) + message (" Libc emcc enabled") +endif () +if (WAMR_BUILD_LIB_RATS EQUAL 1) + message (" Lib rats enabled") +endif() +if (WAMR_BUILD_MINI_LOADER EQUAL 1) + add_definitions (-DWASM_ENABLE_MINI_LOADER=1) + message (" WASM mini loader enabled") +else () + add_definitions (-DWASM_ENABLE_MINI_LOADER=0) +endif () +if (WAMR_DISABLE_HW_BOUND_CHECK EQUAL 1) + add_definitions (-DWASM_DISABLE_HW_BOUND_CHECK=1) + add_definitions (-DWASM_DISABLE_STACK_HW_BOUND_CHECK=1) + message (" Hardware boundary check disabled") +else () + add_definitions (-DWASM_DISABLE_HW_BOUND_CHECK=0) + if (WAMR_DISABLE_STACK_HW_BOUND_CHECK EQUAL 1) + add_definitions (-DWASM_DISABLE_STACK_HW_BOUND_CHECK=1) + message (" Hardware boundary check for native stack disabled") + else () + add_definitions (-DWASM_DISABLE_STACK_HW_BOUND_CHECK=0) + endif () +endif () +if (WAMR_DISABLE_WAKEUP_BLOCKING_OP EQUAL 1) + add_definitions (-DWASM_DISABLE_WAKEUP_BLOCKING_OP=1) + message (" Wakeup of blocking operations disabled") +else () + add_definitions (-DWASM_DISABLE_WAKEUP_BLOCKING_OP=0) + message (" Wakeup of blocking operations enabled") +endif () +if (WAMR_BUILD_SIMD EQUAL 1) + if (NOT WAMR_BUILD_TARGET MATCHES "RISCV64.*") + add_definitions (-DWASM_ENABLE_SIMD=1) + message (" SIMD enabled") + else () + message (" SIMD disabled due to not supported on target RISCV64") + endif () +endif () +if (WAMR_BUILD_AOT_STACK_FRAME EQUAL 1) + add_definitions (-DWASM_ENABLE_AOT_STACK_FRAME=1) + message (" AOT stack frame enabled") +endif () +if (WAMR_BUILD_MEMORY_PROFILING EQUAL 1) + add_definitions (-DWASM_ENABLE_MEMORY_PROFILING=1) + message (" Memory profiling enabled") +endif () +if (WAMR_BUILD_PERF_PROFILING EQUAL 1) + add_definitions (-DWASM_ENABLE_PERF_PROFILING=1) + message (" Performance profiling enabled") +endif () +if (DEFINED WAMR_APP_THREAD_STACK_SIZE_MAX) + add_definitions (-DAPP_THREAD_STACK_SIZE_MAX=${WAMR_APP_THREAD_STACK_SIZE_MAX}) +endif () +if (WAMR_BUILD_CUSTOM_NAME_SECTION EQUAL 1) + add_definitions (-DWASM_ENABLE_CUSTOM_NAME_SECTION=1) + message (" Custom name section enabled") +endif () +if (WAMR_BUILD_DUMP_CALL_STACK EQUAL 1) + add_definitions (-DWASM_ENABLE_DUMP_CALL_STACK=1) + message (" Dump call stack enabled") +endif () +if (WAMR_BUILD_TAIL_CALL EQUAL 1) + add_definitions (-DWASM_ENABLE_TAIL_CALL=1) + message (" Tail call enabled") +endif () +if (WAMR_BUILD_REF_TYPES EQUAL 1) + add_definitions (-DWASM_ENABLE_REF_TYPES=1) + message (" Reference types enabled") +else () + message (" Reference types disabled") +endif () +if (WAMR_BUILD_GC EQUAL 1) + message (" GC enabled") + if (WAMR_TEST_GC EQUAL 1) + message(" GC testing enabled") + endif() +endif () +if (WAMR_BUILD_GC EQUAL 1 AND WAMR_BUILD_GC_PERF_PROFILING EQUAL 1) + add_definitions (-DWASM_ENABLE_GC_PERF_PROFILING=1) + message (" GC performance profiling enabled") +else () + message (" GC performance profiling disabled") +endif () +if (WAMR_BUILD_STRINGREF EQUAL 1) + message (" Stringref enabled") + if (NOT DEFINED WAMR_STRINGREF_IMPL_SOURCE) + message (" Using WAMR builtin implementation for stringref") + else () + message (" Using custom implementation for stringref") + endif() +endif () +if (WAMR_BUILD_PERF_PROFILING EQUAL 1 OR + WAMR_BUILD_DUMP_CALL_STACK EQUAL 1 OR + WAMR_BUILD_GC EQUAL 1) + # Enable AOT/JIT stack frame when perf-profiling, dump-call-stack + # or GC is enabled + if (WAMR_BUILD_AOT EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1) + add_definitions (-DWASM_ENABLE_AOT_STACK_FRAME=1) + endif () +endif () +if (WAMR_BUILD_EXCE_HANDLING EQUAL 1) + add_definitions (-DWASM_ENABLE_EXCE_HANDLING=1) + add_definitions (-DWASM_ENABLE_TAGS=1) + message (" Exception Handling enabled") +endif () +if (DEFINED WAMR_BH_VPRINTF) + add_definitions (-DBH_VPRINTF=${WAMR_BH_VPRINTF}) +endif () +if (DEFINED WAMR_BH_LOG) + add_definitions (-DBH_LOG=${WAMR_BH_LOG}) +endif () +if (WAMR_DISABLE_APP_ENTRY EQUAL 1) + message (" WAMR application entry functions excluded") +endif () +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + message (" Debug Interpreter enabled") +endif () +if (WAMR_BUILD_DEBUG_AOT EQUAL 1) + message (" Debug AOT enabled") +endif () +if (WAMR_BUILD_LOAD_CUSTOM_SECTION EQUAL 1) + add_definitions (-DWASM_ENABLE_LOAD_CUSTOM_SECTION=1) + message (" Load custom section enabled") +endif () +if (WAMR_BUILD_GLOBAL_HEAP_POOL EQUAL 1) + add_definitions(-DWASM_ENABLE_GLOBAL_HEAP_POOL=1) + message (" Global heap pool enabled") +endif () +if (WAMR_BUILD_GLOBAL_HEAP_SIZE GREATER 0) + add_definitions (-DWASM_GLOBAL_HEAP_SIZE=${WAMR_BUILD_GLOBAL_HEAP_SIZE}) + message (" Custom global heap size: " ${WAMR_BUILD_GLOBAL_HEAP_SIZE}) +else () + # Spec test requires more heap pool size + if (WAMR_BUILD_SPEC_TEST EQUAL 1) + if (WAMR_BUILD_PLATFORM STREQUAL "linux-sgx") + math(EXPR WAMR_BUILD_GLOBAL_HEAP_SIZE "100 * 1024 * 1024") + else () + math(EXPR WAMR_BUILD_GLOBAL_HEAP_SIZE "300 * 1024 * 1024") + endif () + add_definitions (-DWASM_GLOBAL_HEAP_SIZE=${WAMR_BUILD_GLOBAL_HEAP_SIZE}) + else () + # By default, the global heap size is of 10 MB + math(EXPR WAMR_BUILD_GLOBAL_HEAP_SIZE "10 * 1024 * 1024") + add_definitions (-DWASM_GLOBAL_HEAP_SIZE=${WAMR_BUILD_GLOBAL_HEAP_SIZE}) + endif () +endif () +if (WAMR_BUILD_STACK_GUARD_SIZE GREATER 0) + add_definitions (-DWASM_STACK_GUARD_SIZE=${WAMR_BUILD_STACK_GUARD_SIZE}) + message (" Custom stack guard size: " ${WAMR_BUILD_STACK_GUARD_SIZE}) +endif () +if (WAMR_BUILD_SGX_IPFS EQUAL 1) + add_definitions (-DWASM_ENABLE_SGX_IPFS=1) + message (" SGX IPFS enabled") +endif () +if (WAMR_BUILD_WASI_NN EQUAL 1) + message (" WASI-NN enabled") + add_definitions (-DWASM_ENABLE_WASI_NN=1) + if (WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) + message (" WASI-NN: GPU enabled") + add_definitions (-DWASM_ENABLE_WASI_NN_GPU=1) + endif () + if (WAMR_BUILD_WASI_NN_ENABLE_EXTERNAL_DELEGATE EQUAL 1) + message (" WASI-NN: External Delegation enabled") + add_definitions (-DWASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE=1) + endif () + if (DEFINED WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH) + add_definitions (-DWASM_WASI_NN_EXTERNAL_DELEGATE_PATH="${WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH}") + endif () + if (WAMR_BUILD_WASI_EPHEMERAL_NN EQUAL 1) + message (" WASI-NN: WASI-Ephemeral-NN enabled") + add_definitions (-DWASM_ENABLE_WASI_EPHEMERAL_NN=1) + endif() +endif () +if (WAMR_BUILD_ALLOC_WITH_USER_DATA EQUAL 1) + add_definitions(-DWASM_MEM_ALLOC_WITH_USER_DATA=1) +endif() +if (WAMR_BUILD_WASM_CACHE EQUAL 1) + add_definitions (-DWASM_ENABLE_WASM_CACHE=1) + message (" Wasm files cache enabled") +endif () +if (WAMR_BUILD_MODULE_INST_CONTEXT EQUAL 1) + add_definitions (-DWASM_ENABLE_MODULE_INST_CONTEXT=1) + message (" Module instance context enabled") +endif () +if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1) + add_definitions (-DWASM_ENABLE_GC_VERIFY=1) + message (" GC heap verification enabled") +endif () +if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + add_definitions (-DCOLLECT_CODE_COVERAGE) + message (" Collect code coverage enabled") +endif () +if (WAMR_BUILD_STATIC_PGO EQUAL 1) + add_definitions (-DWASM_ENABLE_STATIC_PGO=1) + message (" AOT static PGO enabled") +endif () +if (WAMR_BUILD_TARGET STREQUAL "X86_64" + AND WAMR_BUILD_PLATFORM STREQUAL "linux") + if (WAMR_DISABLE_WRITE_GS_BASE EQUAL 1) + # disabled by user + set (DISABLE_WRITE_GS_BASE 1) + elseif (WAMR_DISABLE_WRITE_GS_BASE EQUAL 0) + # enabled by user + set (DISABLE_WRITE_GS_BASE 0) + elseif (CMAKE_CROSSCOMPILING) + # disabled in cross compilation environment + set (DISABLE_WRITE_GS_BASE 1) + else () + # auto-detected by the compiler + set (TEST_WRGSBASE_SOURCE "${CMAKE_BINARY_DIR}/test_wrgsbase.c") + file (WRITE "${TEST_WRGSBASE_SOURCE}" " + #include + #include + int main() { + uint64_t value; + asm volatile (\"wrgsbase %0\" : : \"r\"(value)); + printf(\"WRGSBASE instruction is available.\\n\"); + return 0; + }") + # Try to compile and run the test program + try_run (TEST_WRGSBASE_RESULT + TEST_WRGSBASE_COMPILED + ${CMAKE_BINARY_DIR}/test_wrgsbase + SOURCES ${TEST_WRGSBASE_SOURCE} + CMAKE_FLAGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + ) + #message("${TEST_WRGSBASE_COMPILED}, ${TEST_WRGSBASE_RESULT}") + if (TEST_WRGSBASE_RESULT EQUAL 0) + set (DISABLE_WRITE_GS_BASE 0) + else () + set (DISABLE_WRITE_GS_BASE 1) + endif () + endif () + if (DISABLE_WRITE_GS_BASE EQUAL 1) + add_definitions (-DWASM_DISABLE_WRITE_GS_BASE=1) + message (" Write linear memory base addr to x86 GS register disabled") + else () + add_definitions (-DWASM_DISABLE_WRITE_GS_BASE=0) + message (" Write linear memory base addr to x86 GS register enabled") + endif () +endif () +if (WAMR_CONFIGUABLE_BOUNDS_CHECKS EQUAL 1) + add_definitions (-DWASM_CONFIGURABLE_BOUNDS_CHECKS=1) + message (" Configurable bounds checks enabled") +endif () +if (WAMR_BUILD_LINUX_PERF EQUAL 1) + add_definitions (-DWASM_ENABLE_LINUX_PERF=1) + message (" Linux perf support enabled") +endif () +if (WAMR_BUILD_AOT EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1) + if (NOT DEFINED WAMR_BUILD_QUICK_AOT_ENTRY) + # Enable quick aot/jit entries by default + set (WAMR_BUILD_QUICK_AOT_ENTRY 1) + endif () + if (WAMR_BUILD_QUICK_AOT_ENTRY EQUAL 1) + add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=1) + message (" Quick AOT/JIT entries enabled") + else () + add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=0) + message (" Quick AOT/JIT entries disabled") + endif () +else () + # Disable quick aot/jit entries for interp and fast-jit + add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=0) +endif () +if (WAMR_BUILD_AOT EQUAL 1) + if (NOT DEFINED WAMR_BUILD_AOT_INTRINSICS) + # Enable aot intrinsics by default + set (WAMR_BUILD_AOT_INTRINSICS 1) + endif () + if (WAMR_BUILD_AOT_INTRINSICS EQUAL 1) + add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=1) + message (" AOT intrinsics enabled") + else () + add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=0) + message (" AOT intrinsics disabled") + endif () +else () + # Disable aot intrinsics for interp, fast-jit and llvm-jit + add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=0) +endif () +if (WAMR_BUILD_ALLOC_WITH_USAGE EQUAL 1) + add_definitions(-DWASM_MEM_ALLOC_WITH_USAGE=1) +endif() +if (NOT WAMR_BUILD_SANITIZER STREQUAL "") + message (" Sanitizer ${WAMR_BUILD_SANITIZER} enabled") +endif () diff --git a/wasm-micro-runtime/build-scripts/esp-idf/README.md b/wasm-micro-runtime/build-scripts/esp-idf/README.md new file mode 100644 index 0000000..6bec45d --- /dev/null +++ b/wasm-micro-runtime/build-scripts/esp-idf/README.md @@ -0,0 +1,31 @@ +# wasm-micro-runtime as ESP-IDF component + +You can build an ESP-IDF project with wasm-micro-runtime as a component: + +- Make sure you have the ESP-IDF properly installed and setup +- In particular have the following paths set: + - `WAMR_PATH` to point to your wasm-micro-runtime repository + - `IDF_PATH` to point to your ESP-IDF + - `source $IDF_PATH/export.sh` +- Create a new project, e.g.: `idf.py create-project wamr-hello` +- In the newly created project folder edit the `CMakeList.txt`: + + ``` + cmake_minimum_required(VERSION 3.5) + + include($ENV{IDF_PATH}/tools/cmake/project.cmake) + + set (COMPONENTS ${IDF_TARGET} main freertos esptool_py wamr) + + list(APPEND EXTRA_COMPONENT_DIRS "$ENV{WAMR_PATH}/build-scripts/esp-idf") + + project(wamr-hello) + ``` +- Develop your project in it's `main` component folder. + +You can find an example [here](../../product-mini/platforms/esp-idf). + +- Set target platform: `idf.py set-target esp32c3` +- Build: `idf.py build` +- Flash: `idf.py flash` +- Check the output: `idf.py monitor` \ No newline at end of file diff --git a/wasm-micro-runtime/build-scripts/esp-idf/wamr/CMakeLists.txt b/wasm-micro-runtime/build-scripts/esp-idf/wamr/CMakeLists.txt new file mode 100644 index 0000000..47ccb05 --- /dev/null +++ b/wasm-micro-runtime/build-scripts/esp-idf/wamr/CMakeLists.txt @@ -0,0 +1,57 @@ +# Copyright (C) 2021 Intel Corporation and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Set WAMR's build options +if("${IDF_TARGET}" STREQUAL "esp32c3" OR "${IDF_TARGET}" STREQUAL "esp32c6") + set(WAMR_BUILD_TARGET "RISCV32") +else() + set(WAMR_BUILD_TARGET "XTENSA") +endif() + +set(WAMR_BUILD_PLATFORM "esp-idf") + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_APP_FRAMEWORK) + set (WAMR_BUILD_APP_FRAMEWORK 0) +endif () + +if (NOT CMAKE_BUILD_EARLY_EXPANSION) + if (WAMR_BUILD_TARGET STREQUAL "XTENSA") + idf_build_set_property(COMPILE_DEFINITIONS "-DBUILD_TARGET_XTENSA=1" APPEND) + endif () + if (WAMR_BUILD_INTERP) + idf_build_set_property(COMPILE_DEFINITIONS "-DWASM_ENABLE_INTERP=1" APPEND) + endif () + if (WAMR_BUILD_AOT) + idf_build_set_property(COMPILE_DEFINITIONS "-DWASM_ENABLE_AOT=1" APPEND) + endif () + + set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../..) + include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +endif() + +idf_component_register(SRCS ${WAMR_RUNTIME_LIB_SOURCE} ${PLATFORM_SHARED_SOURCE} + INCLUDE_DIRS ${IWASM_DIR}/include ${UTILS_SHARED_DIR} ${PLATFORM_SHARED_DIR} ${PLATFORM_SHARED_DIR}/../include + REQUIRES pthread lwip esp_timer +) + + diff --git a/wasm-micro-runtime/build-scripts/involve_boringssl.cmake b/wasm-micro-runtime/build-scripts/involve_boringssl.cmake new file mode 100644 index 0000000..a0e069b --- /dev/null +++ b/wasm-micro-runtime/build-scripts/involve_boringssl.cmake @@ -0,0 +1,41 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +message(STATUS "involving boringssl...") + +include(ExternalProject) + +ExternalProject_Add(boringssl + PREFIX external/boringssl + # follow envoy, which tracks BoringSSL, which tracks Chromium + # https://github.com/envoyproxy/envoy/blob/main/bazel/repository_locations.bzl#L112 + # chromium-105.0.5195.37 (linux/beta) + URL https://github.com/google/boringssl/archive/098695591f3a2665fccef83a3732ecfc99acdcdd.tar.gz + URL_HASH SHA256=e141448cf6f686b6e9695f6b6459293fd602c8d51efe118a83106752cf7e1280 + DOWNLOAD_EXTRACT_TIMESTAMP NEW + # SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/../external/boringssl + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/src/boringssl-build/libssl.a + ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/ + && ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/src/boringssl-build/libcrypto.a + ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/ + && ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/src/boringssl/src/include/openssl + ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/openssl +) + +add_library(boringssl_ssl STATIC IMPORTED GLOBAL) +set_target_properties( + boringssl_ssl + PROPERTIES + IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/libssl.a + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/ +) +add_dependencies(boringssl_ssl boringssl) + +add_library(boringssl_crypto STATIC IMPORTED GLOBAL) +set_target_properties( + boringssl_crypto + PROPERTIES + IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/libcrypto.a + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR}/external/boringssl/ +) +add_dependencies(boringssl_crypto boringssl) \ No newline at end of file diff --git a/wasm-micro-runtime/build-scripts/lldb_wasm.patch b/wasm-micro-runtime/build-scripts/lldb_wasm.patch new file mode 100644 index 0000000..83221ef --- /dev/null +++ b/wasm-micro-runtime/build-scripts/lldb_wasm.patch @@ -0,0 +1,5906 @@ +diff --git a/lldb/bindings/CMakeLists.txt b/lldb/bindings/CMakeLists.txt +index 9759b069fdc4..25b427f8bcf2 100644 +--- a/lldb/bindings/CMakeLists.txt ++++ b/lldb/bindings/CMakeLists.txt +@@ -26,8 +26,6 @@ set(SWIG_COMMON_FLAGS + -features autodoc + -I${LLDB_SOURCE_DIR}/include + -I${CMAKE_CURRENT_SOURCE_DIR} +- -D__STDC_LIMIT_MACROS +- -D__STDC_CONSTANT_MACROS + ${DARWIN_EXTRAS} + ) + +diff --git a/lldb/bindings/interfaces.swig b/lldb/bindings/interfaces.swig +index c9a6d0f06056..021c7683d170 100644 +--- a/lldb/bindings/interfaces.swig ++++ b/lldb/bindings/interfaces.swig +@@ -1,8 +1,5 @@ + /* Various liblldb typedefs that SWIG needs to know about. */ + #define __extension__ /* Undefine GCC keyword to make Swig happy when processing glibc's stdint.h. */ +-/* The ISO C99 standard specifies that in C++ implementations limit macros such +- as INT32_MAX should only be defined if __STDC_LIMIT_MACROS is. */ +-#define __STDC_LIMIT_MACROS + %include "stdint.i" + + %include "lldb/lldb-defines.h" +diff --git a/lldb/bindings/python/python-typemaps.swig b/lldb/bindings/python/python-typemaps.swig +index b1ace4ff3b1e..5f8f4aa678c4 100644 +--- a/lldb/bindings/python/python-typemaps.swig ++++ b/lldb/bindings/python/python-typemaps.swig +@@ -439,7 +439,7 @@ bool SetNumberFromPyObject(double &number, PyObject *obj) { + + %typemap(out) lldb::FileSP { + $result = nullptr; +- lldb::FileSP &sp = $1; ++ const lldb::FileSP &sp = $1; + if (sp) { + PythonFile pyfile = unwrapOrSetPythonException(PythonFile::FromFile(*sp)); + if (!pyfile.IsValid()) +diff --git a/lldb/include/lldb/Breakpoint/Breakpoint.h b/lldb/include/lldb/Breakpoint/Breakpoint.h +index f2e2a0d22784..426d1129bd10 100644 +--- a/lldb/include/lldb/Breakpoint/Breakpoint.h ++++ b/lldb/include/lldb/Breakpoint/Breakpoint.h +@@ -9,6 +9,7 @@ + #ifndef LLDB_BREAKPOINT_BREAKPOINT_H + #define LLDB_BREAKPOINT_BREAKPOINT_H + ++#include + #include + #include + #include +diff --git a/lldb/include/lldb/Core/Module.h b/lldb/include/lldb/Core/Module.h +index dd7100c4616c..97d70daadbdc 100644 +--- a/lldb/include/lldb/Core/Module.h ++++ b/lldb/include/lldb/Core/Module.h +@@ -41,6 +41,7 @@ + + namespace lldb_private { + class CompilerDeclContext; ++class DWARFEvaluatorFactory; + class Function; + class Log; + class ObjectFile; +@@ -859,6 +860,8 @@ public: + /// Update the ArchSpec to a more specific variant. + bool MergeArchitecture(const ArchSpec &arch_spec); + ++ DWARFEvaluatorFactory *GetDWARFExpressionEvaluatorFactory(); ++ + /// \class LookupInfo Module.h "lldb/Core/Module.h" + /// A class that encapsulates name lookup information. + /// +@@ -985,6 +988,8 @@ protected: + m_first_file_changed_log : 1; /// See if the module was modified after it + /// was initially opened. + ++ std::unique_ptr m_dwarf_evaluator_factory; ++ + /// Resolve a file or load virtual address. + /// + /// Tries to resolve \a vm_addr as a file address (if \a +diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h +index be91929c62e1..8d876fc1fa2f 100644 +--- a/lldb/include/lldb/Core/PluginManager.h ++++ b/lldb/include/lldb/Core/PluginManager.h +@@ -508,6 +508,17 @@ public: + static bool CreateSettingForStructuredDataPlugin( + Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp, + ConstString description, bool is_global_property); ++ ++ // DWARFEvaluatorFactory ++ static bool ++ RegisterPlugin(ConstString name, const char *description, ++ DWARFEvaluatorFactoryCreateInstance create_callback); ++ ++ static bool ++ UnregisterPlugin(DWARFEvaluatorFactoryCreateInstance create_callback); ++ ++ static DWARFEvaluatorFactoryCreateInstance ++ GetDWARFEvaluatorFactoryCreateCallbackAtIndex(uint32_t idx); + }; + + } // namespace lldb_private +diff --git a/lldb/include/lldb/Expression/DWARFEvaluator.h b/lldb/include/lldb/Expression/DWARFEvaluator.h +new file mode 100644 +index 000000000000..6811cbeae3d3 +--- /dev/null ++++ b/lldb/include/lldb/Expression/DWARFEvaluator.h +@@ -0,0 +1,110 @@ ++//===-- DWARFEvaluator.h ----------------------------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLDB_EXPRESSION_DWARFEVALUATOR_H ++#define LLDB_EXPRESSION_DWARFEVALUATOR_H ++ ++#include "lldb/lldb-private.h" ++#include ++ ++namespace lldb_private { ++ ++class DWARFExpression; ++ ++/// \class DWARFEvaluator DWARFEvaluator.h ++/// "lldb/Expression/DWARFEvaluator.h" Evaluates DWARF opcodes. ++/// ++class DWARFEvaluator { ++public: ++ /// Crates a DWARF location expression evaluator ++ /// ++ /// \param[in] dwarf_expression ++ /// The DWARF expression to evaluate. ++ /// ++ /// \param[in] exe_ctx ++ /// The execution context in which to evaluate the location ++ /// expression. The location expression may access the target's ++ /// memory, especially if it comes from the expression parser. ++ /// ++ /// \param[in] reg_ctx ++ /// An optional parameter which provides a RegisterContext for use ++ /// when evaluating the expression (i.e. for fetching register values). ++ /// Normally this will come from the ExecutionContext's StackFrame but ++ /// in the case where an expression needs to be evaluated while building ++ /// the stack frame list, this short-cut is available. ++ /// ++ /// \param[in] initial_value_ptr ++ /// A value to put on top of the interpreter stack before evaluating ++ /// the expression, if the expression is parametrized. Can be NULL. ++ /// ++ /// \param[in] object_address_ptr ++ /// ++ DWARFEvaluator(const DWARFExpression &dwarf_expression, ++ ExecutionContext *exe_ctx, RegisterContext *reg_ctx, ++ const Value *initial_value_ptr, ++ const Value *object_address_ptr); ++ ++ /// DWARFEvaluator protocol. ++ /// \{ ++ ++ /// Evaluate the DWARF location expression ++ /// ++ /// \param[in] result ++ /// A value into which the result of evaluating the expression is ++ /// to be placed. ++ /// ++ /// \param[in] error_ptr ++ /// If non-NULL, used to report errors in expression evaluation. ++ /// ++ /// \return ++ /// True on success; false otherwise. If error_ptr is non-NULL, ++ /// details of the failure are provided through it. ++ virtual bool Evaluate(Value &result, Status *error_ptr); ++ ++ /// Evaluate the DWARF location expression with the opcodes specified. ++ /// ++ /// \param[in] opcodes ++ /// The DWARF opcodes to evaluate. ++ /// ++ /// \param[in] result ++ /// A value into which the result of evaluating the expression is ++ /// to be placed. ++ /// ++ /// \param[in] error_ptr ++ /// If non-NULL, used to report errors in expression evaluation. ++ /// ++ /// \return ++ /// True on success; false otherwise. If error_ptr is non-NULL, ++ /// details of the failure are provided through it. ++ virtual bool Evaluate(const DataExtractor &opcodes, Value &result, ++ Status *error_ptr); ++ ++ /// Evaluates a specific DWARF opcode in the context of a DWARF expression ++ virtual bool Evaluate(const uint8_t op, Process *process, StackFrame *frame, ++ std::vector &stack, const DataExtractor &opcodes, ++ lldb::offset_t &offset, Value &pieces, ++ uint64_t &op_piece_offset, Log *log, Status *error_ptr); ++ ++ /// \} ++ ++protected: ++ const DWARFExpression &m_dwarf_expression; ++ ExecutionContext *m_exe_ctx; ++ RegisterContext *m_reg_ctx; ++ const Value *m_initial_value_ptr; ++ const Value *m_object_address_ptr; ++ ++private: ++ DWARFEvaluator(const DWARFEvaluator &); ++ const DWARFEvaluator &operator=(const DWARFEvaluator &) = delete; ++ ++}; ++ ++} // namespace lldb_private ++ ++#endif // LLDB_EXPRESSION_DWARFEVALUATOR_H +diff --git a/lldb/include/lldb/Expression/DWARFEvaluatorFactory.h b/lldb/include/lldb/Expression/DWARFEvaluatorFactory.h +new file mode 100644 +index 000000000000..f3b496c580e4 +--- /dev/null ++++ b/lldb/include/lldb/Expression/DWARFEvaluatorFactory.h +@@ -0,0 +1,56 @@ ++//===-- DWARFEvaluatorFactory.h ---------------------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLDB_EXPRESSION_DWARFEVALUATORFACTORY_H ++#define LLDB_EXPRESSION_DWARFEVALUATORFACTORY_H ++ ++#include "lldb/Core/PluginInterface.h" ++#include "lldb/Utility/ConstString.h" ++#include "lldb/lldb-private.h" ++ ++class DWARFUnit; ++ ++namespace lldb_private { ++ ++class DWARFEvaluator; ++class DWARFExpression; ++ ++/// \class DWARFEvaluatorFactory DWARFEvaluatorFactory.h ++/// "lldb/Expression/DWARFEvaluatorFactory.h" Factory class that allows the ++/// registration of platform-specific DWARF expression evaluators, used to ++/// handle platform-specific DWARF opcodes. ++class DWARFEvaluatorFactory : public PluginInterface { ++public: ++ static std::unique_ptr FindPlugin(Module *module); ++ ++ /// PluginInterface protocol. ++ /// \{ ++ ConstString GetPluginName() override; ++ ++ uint32_t GetPluginVersion() override { return 1; } ++ /// \} ++ ++ DWARFEvaluatorFactory() {} ++ ++ /// DWARFEvaluatorFactory protocol. ++ /// \{ ++ virtual std::unique_ptr ++ CreateDWARFEvaluator(const DWARFExpression &dwarf_expression, ++ ExecutionContext *exe_ctx, RegisterContext *reg_ctx, ++ const Value *initial_value_ptr, ++ const Value *object_address_ptr); ++ /// \} ++ ++private: ++ DWARFEvaluatorFactory(const DWARFEvaluatorFactory &); ++ const DWARFEvaluatorFactory &operator=(const DWARFEvaluatorFactory &) = delete; ++}; ++ ++} // namespace lldb_private ++ ++#endif // LLDB_EXPRESSION_DWARFEVALUATORFACTORY_H +diff --git a/lldb/include/lldb/Expression/DWARFExpression.h b/lldb/include/lldb/Expression/DWARFExpression.h +index 1490ac2d614a..35c741d4e6ba 100644 +--- a/lldb/include/lldb/Expression/DWARFExpression.h ++++ b/lldb/include/lldb/Expression/DWARFExpression.h +@@ -120,6 +120,10 @@ public: + + void SetModule(const lldb::ModuleSP &module) { m_module_wp = module; } + ++ lldb::ModuleSP GetModule() const { return m_module_wp.lock(); } ++ ++ const DWARFUnit *GetDWARFCompileUnit() const { return m_dwarf_cu; } ++ + bool ContainsThreadLocalStorage() const; + + bool LinkThreadLocalStorage( +@@ -140,7 +144,7 @@ public: + lldb::addr_t func_file_addr); + + /// Return the call-frame-info style register kind +- int GetRegisterKind(); ++ lldb::RegisterKind GetRegisterKind() const; + + /// Set the call-frame-info style register kind + /// +@@ -219,6 +223,9 @@ public: + + bool MatchesOperand(StackFrame &frame, const Instruction::Operand &op); + ++ static lldb::addr_t ReadAddressFromDebugAddrSection(const DWARFUnit *dwarf_cu, ++ uint32_t index); ++ + llvm::Optional + GetLocationExpression(lldb::addr_t load_function_start, + lldb::addr_t addr) const; +diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h +index aaa2470d2931..c15f2db52fbc 100644 +--- a/lldb/include/lldb/Target/Process.h ++++ b/lldb/include/lldb/Target/Process.h +@@ -1434,7 +1434,7 @@ public: + /// vm_addr, \a buf, and \a size updated appropriately. Zero is + /// returned in the case of an error. + virtual size_t ReadMemory(lldb::addr_t vm_addr, void *buf, size_t size, +- Status &error); ++ Status &error, ExecutionContext *exe_ctx = nullptr); + + /// Read of memory from a process. + /// +diff --git a/lldb/include/lldb/Target/ProcessTrace.h b/lldb/include/lldb/Target/ProcessTrace.h +index 7b9d6b13dd6f..9525fc9750fd 100644 +--- a/lldb/include/lldb/Target/ProcessTrace.h ++++ b/lldb/include/lldb/Target/ProcessTrace.h +@@ -59,7 +59,7 @@ public: + bool WarnBeforeDetach() const override { return false; } + + size_t ReadMemory(lldb::addr_t addr, void *buf, size_t size, +- Status &error) override; ++ Status &error, ExecutionContext *exe_ctx = nullptr) override; + + size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, + Status &error) override; +diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h +index ad5298151e4a..5a3c0b27a738 100644 +--- a/lldb/include/lldb/lldb-forward.h ++++ b/lldb/include/lldb/lldb-forward.h +@@ -74,6 +74,7 @@ class Disassembler; + class DumpValueObjectOptions; + class DynamicCheckerFunctions; + class DynamicLoader; ++class DWARFEvaluatorFactory; + class Editline; + class EmulateInstruction; + class Environment; +diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h +index 2ed083ec8ae9..f4d500d198e8 100644 +--- a/lldb/include/lldb/lldb-private-interfaces.h ++++ b/lldb/include/lldb/lldb-private-interfaces.h +@@ -113,6 +113,8 @@ typedef lldb::REPLSP (*REPLCreateInstance)(Status &error, + const char *repl_options); + typedef int (*ComparisonFunction)(const void *, const void *); + typedef void (*DebuggerInitializeCallback)(Debugger &debugger); ++typedef DWARFEvaluatorFactory *(*DWARFEvaluatorFactoryCreateInstance)( ++ Module *module); + /// Trace + /// \{ + typedef llvm::Expected (*TraceCreateInstanceForSessionFile)( +diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp +index 19c97be15066..1647f93ec4f3 100644 +--- a/lldb/source/Core/Module.cpp ++++ b/lldb/source/Core/Module.cpp +@@ -16,6 +16,7 @@ + #include "lldb/Core/ModuleSpec.h" + #include "lldb/Core/SearchFilter.h" + #include "lldb/Core/Section.h" ++#include "lldb/Expression/DWARFEvaluatorFactory.h" + #include "lldb/Host/FileSystem.h" + #include "lldb/Host/Host.h" + #include "lldb/Host/HostInfo.h" +@@ -1659,3 +1660,9 @@ bool Module::GetIsDynamicLinkEditor() { + + return false; + } ++ ++DWARFEvaluatorFactory *Module::GetDWARFExpressionEvaluatorFactory() { ++ if (!m_dwarf_evaluator_factory) ++ m_dwarf_evaluator_factory = DWARFEvaluatorFactory::FindPlugin(this); ++ return m_dwarf_evaluator_factory.get(); ++} +diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp +index fcaa868b083e..59a404d4a7e1 100644 +--- a/lldb/source/Core/PluginManager.cpp ++++ b/lldb/source/Core/PluginManager.cpp +@@ -1597,3 +1597,32 @@ bool PluginManager::CreateSettingForStructuredDataPlugin( + ConstString("Settings for structured data plug-ins"), properties_sp, + description, is_global_property); + } ++ ++#pragma mark DWARFEvaluator ++ ++typedef PluginInstance ++ DWARFEvaluatorFactoryInstance; ++typedef PluginInstances ++ DWARFEvaluatorFactoryInstances; ++ ++static DWARFEvaluatorFactoryInstances &GetDWARFEvaluatorFactoryInstances() { ++ static DWARFEvaluatorFactoryInstances g_instances; ++ return g_instances; ++} ++ ++bool PluginManager::RegisterPlugin( ++ ConstString name, const char *description, ++ DWARFEvaluatorFactoryCreateInstance create_callback) { ++ return GetDWARFEvaluatorFactoryInstances().RegisterPlugin(name, description, ++ create_callback); ++} ++ ++bool PluginManager::UnregisterPlugin( ++ DWARFEvaluatorFactoryCreateInstance create_callback) { ++ return GetDWARFEvaluatorFactoryInstances().UnregisterPlugin(create_callback); ++} ++ ++DWARFEvaluatorFactoryCreateInstance ++PluginManager::GetDWARFEvaluatorFactoryCreateCallbackAtIndex(uint32_t idx) { ++ return GetDWARFEvaluatorFactoryInstances().GetCallbackAtIndex(idx); ++} +diff --git a/lldb/source/Core/Value.cpp b/lldb/source/Core/Value.cpp +index fb57c0fedf04..f92d6a54de94 100644 +--- a/lldb/source/Core/Value.cpp ++++ b/lldb/source/Core/Value.cpp +@@ -538,7 +538,7 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data, + + if (process) { + const size_t bytes_read = +- process->ReadMemory(address, dst, byte_size, error); ++ process->ReadMemory(address, dst, byte_size, error, exe_ctx); + if (bytes_read != byte_size) + error.SetErrorStringWithFormat( + "read memory from 0x%" PRIx64 " failed (%u of %u bytes read)", +diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp +index 9c1ba99da1d0..b15b214b2a2f 100644 +--- a/lldb/source/Core/ValueObject.cpp ++++ b/lldb/source/Core/ValueObject.cpp +@@ -735,7 +735,7 @@ size_t ValueObject::GetPointeeData(DataExtractor &data, uint32_t item_idx, + if (process) { + heap_buf_ptr->SetByteSize(bytes); + size_t bytes_read = process->ReadMemory( +- addr + offset, heap_buf_ptr->GetBytes(), bytes, error); ++ addr + offset, heap_buf_ptr->GetBytes(), bytes, error, &exe_ctx); + if (error.Success() || bytes_read > 0) { + data.SetData(data_sp); + return bytes_read; +diff --git a/lldb/source/Expression/CMakeLists.txt b/lldb/source/Expression/CMakeLists.txt +index bf94361dd6c1..4e76d547aeaf 100644 +--- a/lldb/source/Expression/CMakeLists.txt ++++ b/lldb/source/Expression/CMakeLists.txt +@@ -1,5 +1,7 @@ + add_lldb_library(lldbExpression + DiagnosticManager.cpp ++ DWARFEvaluator.cpp ++ DWARFEvaluatorFactory.cpp + DWARFExpression.cpp + Expression.cpp + ExpressionVariable.cpp +diff --git a/lldb/source/Expression/DWARFEvaluator.cpp b/lldb/source/Expression/DWARFEvaluator.cpp +new file mode 100644 +index 000000000000..06107e136197 +--- /dev/null ++++ b/lldb/source/Expression/DWARFEvaluator.cpp +@@ -0,0 +1,1952 @@ ++//===-- DWARFEvaluator.cpp ------------ -----------------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#include "lldb/Expression/DWARFEvaluator.h" ++#include "lldb/Expression/DWARFExpression.h" ++ ++#include "lldb/Core/Module.h" ++#include "lldb/Core/Value.h" ++#include "lldb/Core/dwarf.h" ++ ++#include "lldb/Utility/Log.h" ++#include "lldb/Utility/RegisterValue.h" ++ ++#include "lldb/Target/Process.h" ++#include "lldb/Target/RegisterContext.h" ++#include "lldb/Target/StackFrame.h" ++ ++#include "Plugins/SymbolFile/DWARF/DWARFUnit.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++ ++DWARFEvaluator::DWARFEvaluator(const DWARFExpression &dwarf_expression, ++ ExecutionContext *exe_ctx, ++ RegisterContext *reg_ctx, ++ const Value *initial_value_ptr, ++ const Value *object_address_ptr) ++ : m_dwarf_expression(dwarf_expression), m_exe_ctx(exe_ctx), ++ m_reg_ctx(reg_ctx), m_initial_value_ptr(initial_value_ptr), ++ m_object_address_ptr(object_address_ptr) {} ++ ++static bool ReadRegisterValueAsScalar(RegisterContext *reg_ctx, ++ lldb::RegisterKind reg_kind, ++ uint32_t reg_num, Status *error_ptr, ++ Value &value) { ++ if (reg_ctx == nullptr) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat("No register context in frame.\n"); ++ } else { ++ uint32_t native_reg = ++ reg_ctx->ConvertRegisterKindToRegisterNumber(reg_kind, reg_num); ++ if (native_reg == LLDB_INVALID_REGNUM) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat("Unable to convert register " ++ "kind=%u reg_num=%u to a native " ++ "register number.\n", ++ reg_kind, reg_num); ++ } else { ++ const RegisterInfo *reg_info = ++ reg_ctx->GetRegisterInfoAtIndex(native_reg); ++ RegisterValue reg_value; ++ if (reg_ctx->ReadRegister(reg_info, reg_value)) { ++ if (reg_value.GetScalarValue(value.GetScalar())) { ++ value.SetValueType(Value::ValueType::Scalar); ++ value.SetContext(Value::ContextType::RegisterInfo, ++ const_cast(reg_info)); ++ if (error_ptr) ++ error_ptr->Clear(); ++ return true; ++ } else { ++ // If we get this error, then we need to implement a value buffer in ++ // the dwarf expression evaluation function... ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "register %s can't be converted to a scalar value", ++ reg_info->name); ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat("register %s is not available", ++ reg_info->name); ++ } ++ } ++ } ++ return false; ++} ++ ++static bool Evaluate_DW_OP_entry_value(std::vector &stack, ++ ExecutionContext *exe_ctx, ++ RegisterContext *reg_ctx, ++ const DataExtractor &opcodes, ++ lldb::offset_t &opcode_offset, ++ Status *error_ptr, Log *log) { ++ // DW_OP_entry_value(sub-expr) describes the location a variable had upon ++ // function entry: this variable location is presumed to be optimized out at ++ // the current PC value. The caller of the function may have call site ++ // information that describes an alternate location for the variable (e.g. a ++ // constant literal, or a spilled stack value) in the parent frame. ++ // ++ // Example (this is pseudo-code & pseudo-DWARF, but hopefully illustrative): ++ // ++ // void child(int &sink, int x) { ++ // ... ++ // /* "x" gets optimized out. */ ++ // ++ // /* The location of "x" here is: DW_OP_entry_value($reg2). */ ++ // ++sink; ++ // } ++ // ++ // void parent() { ++ // int sink; ++ // ++ // /* ++ // * The callsite information emitted here is: ++ // * ++ // * DW_TAG_call_site ++ // * DW_AT_return_pc ... (for "child(sink, 123);") ++ // * DW_TAG_call_site_parameter (for "sink") ++ // * DW_AT_location ($reg1) ++ // * DW_AT_call_value ($SP - 8) ++ // * DW_TAG_call_site_parameter (for "x") ++ // * DW_AT_location ($reg2) ++ // * DW_AT_call_value ($literal 123) ++ // * ++ // * DW_TAG_call_site ++ // * DW_AT_return_pc ... (for "child(sink, 456);") ++ // * ... ++ // */ ++ // child(sink, 123); ++ // child(sink, 456); ++ // } ++ // ++ // When the program stops at "++sink" within `child`, the debugger determines ++ // the call site by analyzing the return address. Once the call site is found, ++ // the debugger determines which parameter is referenced by DW_OP_entry_value ++ // and evaluates the corresponding location for that parameter in `parent`. ++ ++ // 1. Find the function which pushed the current frame onto the stack. ++ if ((!exe_ctx || !exe_ctx->HasTargetScope()) || !reg_ctx) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: no exe/reg context"); ++ return false; ++ } ++ ++ StackFrame *current_frame = exe_ctx->GetFramePtr(); ++ Thread *thread = exe_ctx->GetThreadPtr(); ++ if (!current_frame || !thread) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: no current frame/thread"); ++ return false; ++ } ++ ++ Target &target = exe_ctx->GetTargetRef(); ++ StackFrameSP parent_frame = nullptr; ++ addr_t return_pc = LLDB_INVALID_ADDRESS; ++ uint32_t current_frame_idx = current_frame->GetFrameIndex(); ++ uint32_t num_frames = thread->GetStackFrameCount(); ++ for (uint32_t parent_frame_idx = current_frame_idx + 1; ++ parent_frame_idx < num_frames; ++parent_frame_idx) { ++ parent_frame = thread->GetStackFrameAtIndex(parent_frame_idx); ++ // Require a valid sequence of frames. ++ if (!parent_frame) ++ break; ++ ++ // Record the first valid return address, even if this is an inlined frame, ++ // in order to look up the associated call edge in the first non-inlined ++ // parent frame. ++ if (return_pc == LLDB_INVALID_ADDRESS) { ++ return_pc = parent_frame->GetFrameCodeAddress().GetLoadAddress(&target); ++ LLDB_LOG(log, ++ "Evaluate_DW_OP_entry_value: immediate ancestor with pc = {0:x}", ++ return_pc); ++ } ++ ++ // If we've found an inlined frame, skip it (these have no call site ++ // parameters). ++ if (parent_frame->IsInlined()) ++ continue; ++ ++ // We've found the first non-inlined parent frame. ++ break; ++ } ++ if (!parent_frame || !parent_frame->GetRegisterContext()) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: no parent frame with reg ctx"); ++ return false; ++ } ++ ++ Function *parent_func = ++ parent_frame->GetSymbolContext(eSymbolContextFunction).function; ++ if (!parent_func) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: no parent function"); ++ return false; ++ } ++ ++ // 2. Find the call edge in the parent function responsible for creating the ++ // current activation. ++ Function *current_func = ++ current_frame->GetSymbolContext(eSymbolContextFunction).function; ++ if (!current_func) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: no current function"); ++ return false; ++ } ++ ++ CallEdge *call_edge = nullptr; ++ ModuleList &modlist = target.GetImages(); ++ ExecutionContext parent_exe_ctx = *exe_ctx; ++ parent_exe_ctx.SetFrameSP(parent_frame); ++ if (!parent_frame->IsArtificial()) { ++ // If the parent frame is not artificial, the current activation may be ++ // produced by an ambiguous tail call. In this case, refuse to proceed. ++ call_edge = parent_func->GetCallEdgeForReturnAddress(return_pc, target); ++ if (!call_edge) { ++ LLDB_LOG(log, ++ "Evaluate_DW_OP_entry_value: no call edge for retn-pc = {0:x} " ++ "in parent frame {1}", ++ return_pc, parent_func->GetName()); ++ return false; ++ } ++ Function *callee_func = call_edge->GetCallee(modlist, parent_exe_ctx); ++ if (callee_func != current_func) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: ambiguous call sequence, " ++ "can't find real parent frame"); ++ return false; ++ } ++ } else { ++ // The StackFrameList solver machinery has deduced that an unambiguous tail ++ // call sequence that produced the current activation. The first edge in ++ // the parent that points to the current function must be valid. ++ for (auto &edge : parent_func->GetTailCallingEdges()) { ++ if (edge->GetCallee(modlist, parent_exe_ctx) == current_func) { ++ call_edge = edge.get(); ++ break; ++ } ++ } ++ } ++ if (!call_edge) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: no unambiguous edge from parent " ++ "to current function"); ++ return false; ++ } ++ ++ // 3. Attempt to locate the DW_OP_entry_value expression in the set of ++ // available call site parameters. If found, evaluate the corresponding ++ // parameter in the context of the parent frame. ++ const uint32_t subexpr_len = opcodes.GetULEB128(&opcode_offset); ++ const void *subexpr_data = opcodes.GetData(&opcode_offset, subexpr_len); ++ if (!subexpr_data) { ++ LLDB_LOG(log, "Evaluate_DW_OP_entry_value: subexpr could not be read"); ++ return false; ++ } ++ ++ const CallSiteParameter *matched_param = nullptr; ++ for (const CallSiteParameter ¶m : call_edge->GetCallSiteParameters()) { ++ DataExtractor param_subexpr_extractor; ++ if (!param.LocationInCallee.GetExpressionData(param_subexpr_extractor)) ++ continue; ++ lldb::offset_t param_subexpr_offset = 0; ++ const void *param_subexpr_data = ++ param_subexpr_extractor.GetData(¶m_subexpr_offset, subexpr_len); ++ if (!param_subexpr_data || ++ param_subexpr_extractor.BytesLeft(param_subexpr_offset) != 0) ++ continue; ++ ++ // At this point, the DW_OP_entry_value sub-expression and the callee-side ++ // expression in the call site parameter are known to have the same length. ++ // Check whether they are equal. ++ // ++ // Note that an equality check is sufficient: the contents of the ++ // DW_OP_entry_value subexpression are only used to identify the right call ++ // site parameter in the parent, and do not require any special handling. ++ if (memcmp(subexpr_data, param_subexpr_data, subexpr_len) == 0) { ++ matched_param = ¶m; ++ break; ++ } ++ } ++ if (!matched_param) { ++ LLDB_LOG(log, ++ "Evaluate_DW_OP_entry_value: no matching call site param found"); ++ return false; ++ } ++ ++ // TODO: Add support for DW_OP_push_object_address within a DW_OP_entry_value ++ // subexpresion whenever llvm does. ++ Value result; ++ const DWARFExpression ¶m_expr = matched_param->LocationInCaller; ++ if (!param_expr.Evaluate(&parent_exe_ctx, ++ parent_frame->GetRegisterContext().get(), ++ /*loclist_base_addr=*/LLDB_INVALID_ADDRESS, ++ /*initial_value_ptr=*/nullptr, ++ /*object_address_ptr=*/nullptr, result, error_ptr)) { ++ LLDB_LOG(log, ++ "Evaluate_DW_OP_entry_value: call site param evaluation failed"); ++ return false; ++ } ++ ++ stack.push_back(result); ++ return true; ++} ++ ++bool DWARFEvaluator::Evaluate(Value &result, Status *error_ptr) { ++ DataExtractor opcodes; ++ if (!m_dwarf_expression.GetExpressionData(opcodes)) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "no location, value may have been optimized out"); ++ return false; ++ } ++ return Evaluate(opcodes, result, error_ptr); ++} ++ ++bool DWARFEvaluator::Evaluate(const DataExtractor &opcodes, Value &result, ++ Status *error_ptr) { ++ if (opcodes.GetByteSize() == 0) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "no location, value may have been optimized out"); ++ return false; ++ } ++ std::vector stack; ++ ++ Process *process = nullptr; ++ StackFrame *frame = nullptr; ++ ++ if (m_exe_ctx) { ++ process = m_exe_ctx->GetProcessPtr(); ++ frame = m_exe_ctx->GetFramePtr(); ++ } ++ if (m_reg_ctx == nullptr && frame) ++ m_reg_ctx = frame->GetRegisterContext().get(); ++ ++ if (m_initial_value_ptr) ++ stack.push_back(*m_initial_value_ptr); ++ ++ lldb::offset_t offset = 0; ++ ++ /// Insertion point for evaluating multi-piece expression. ++ uint64_t op_piece_offset = 0; ++ Value pieces; // Used for DW_OP_piece ++ ++ Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS)); ++ ++ uint8_t _opcode = 0; ++ ++ while (opcodes.ValidOffset(offset)) { ++ const lldb::offset_t op_offset = offset; ++ const uint8_t op = opcodes.GetU8(&offset); ++ _opcode = op; ++ ++ if (log && log->GetVerbose()) { ++ size_t count = stack.size(); ++ LLDB_LOGF(log, "Stack before operation has %" PRIu64 " values:", ++ (uint64_t)count); ++ for (size_t i = 0; i < count; ++i) { ++ StreamString new_value; ++ new_value.Printf("[%" PRIu64 "]", (uint64_t)i); ++ stack[i].Dump(&new_value); ++ LLDB_LOGF(log, " %s", new_value.GetData()); ++ } ++ LLDB_LOGF(log, "0x%8.8" PRIx64 ": %s", op_offset, ++ DW_OP_value_to_name(op)); ++ } ++ ++ if (!Evaluate(op, process, frame, stack, opcodes, offset, pieces, ++ op_piece_offset, log, error_ptr)) ++ return false; ++ } ++ ++ if (stack.empty()) { ++ // Nothing on the stack, check if we created a piece value from DW_OP_piece ++ // or DW_OP_bit_piece opcodes ++ if (pieces.GetBuffer().GetByteSize()) ++ result = pieces; ++ else { ++ if (error_ptr) ++ error_ptr->SetErrorString("Stack empty after evaluation."); ++ return false; ++ } ++ } else { ++ if (log && log->GetVerbose()) { ++ size_t count = stack.size(); ++ LLDB_LOGF(log, "Stack after operation has %" PRIu64 " values:", ++ (uint64_t)count); ++ for (size_t i = 0; i < count; ++i) { ++ StreamString new_value; ++ new_value.Printf("[%" PRIu64 "]", (uint64_t)i); ++ stack[i].Dump(&new_value); ++ LLDB_LOGF(log, " %s", new_value.GetData()); ++ } ++ } ++ result = stack.back(); ++ } ++ return true; // Return true on success ++} ++ ++bool DWARFEvaluator::Evaluate(const uint8_t op, Process *process, ++ StackFrame *frame, std::vector &stack, ++ const DataExtractor &opcodes, ++ lldb::offset_t &offset, Value &pieces, ++ uint64_t &op_piece_offset, Log *log, ++ Status *error_ptr) { ++ Value tmp; ++ uint32_t reg_num; ++ ++ lldb::ModuleSP module_sp = m_dwarf_expression.GetModule(); ++ const DWARFUnit *dwarf_cu = m_dwarf_expression.GetDWARFCompileUnit(); ++ const lldb::RegisterKind reg_kind = m_dwarf_expression.GetRegisterKind(); ++ ++ switch (op) { ++ // The DW_OP_addr operation has a single operand that encodes a machine ++ // address and whose size is the size of an address on the target machine. ++ case DW_OP_addr: ++ stack.push_back(Scalar(opcodes.GetAddress(&offset))); ++ stack.back().SetValueType(Value::ValueType::FileAddress); ++ // Convert the file address to a load address, so subsequent ++ // DWARF operators can operate on it. ++ if (frame) ++ stack.back().ConvertToLoadAddress(module_sp.get(), ++ frame->CalculateTarget().get()); ++ break; ++ ++ // The DW_OP_addr_sect_offset4 is used for any location expressions in ++ // shared libraries that have a location like: ++ // DW_OP_addr(0x1000) ++ // If this address resides in a shared library, then this virtual address ++ // won't make sense when it is evaluated in the context of a running ++ // process where shared libraries have been slid. To account for this, this ++ // new address type where we can store the section pointer and a 4 byte ++ // offset. ++ // case DW_OP_addr_sect_offset4: ++ // { ++ // result_type = eResultTypeFileAddress; ++ // lldb::Section *sect = (lldb::Section ++ // *)opcodes.GetMaxU64(&offset, sizeof(void *)); ++ // lldb::addr_t sect_offset = opcodes.GetU32(&offset); ++ // ++ // Address so_addr (sect, sect_offset); ++ // lldb::addr_t load_addr = so_addr.GetLoadAddress(); ++ // if (load_addr != LLDB_INVALID_ADDRESS) ++ // { ++ // // We successfully resolve a file address to a load ++ // // address. ++ // stack.push_back(load_addr); ++ // break; ++ // } ++ // else ++ // { ++ // // We were able ++ // if (error_ptr) ++ // error_ptr->SetErrorStringWithFormat ("Section %s in ++ // %s is not currently loaded.\n", ++ // sect->GetName().AsCString(), ++ // sect->GetModule()->GetFileSpec().GetFilename().AsCString()); ++ // return false; ++ // } ++ // } ++ // break; ++ ++ // OPCODE: DW_OP_deref ++ // OPERANDS: none ++ // DESCRIPTION: Pops the top stack entry and treats it as an address. ++ // The value retrieved from that address is pushed. The size of the data ++ // retrieved from the dereferenced address is the size of an address on the ++ // target machine. ++ case DW_OP_deref: { ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Expression stack empty for DW_OP_deref."); ++ return false; ++ } ++ Value::ValueType value_type = stack.back().GetValueType(); ++ switch (value_type) { ++ case Value::ValueType::HostAddress: { ++ void *src = (void *)stack.back().GetScalar().ULongLong(); ++ intptr_t ptr; ++ ::memcpy(&ptr, src, sizeof(void *)); ++ stack.back().GetScalar() = ptr; ++ stack.back().ClearContext(); ++ } break; ++ case Value::ValueType::FileAddress: { ++ auto file_addr = stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); ++ if (!module_sp) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "need module to resolve file address for DW_OP_deref"); ++ return false; ++ } ++ Address so_addr; ++ if (!module_sp->ResolveFileAddress(file_addr, so_addr)) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "failed to resolve file address in module"); ++ return false; ++ } ++ addr_t load_Addr = so_addr.GetLoadAddress(m_exe_ctx->GetTargetPtr()); ++ if (load_Addr == LLDB_INVALID_ADDRESS) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat("failed to resolve load address"); ++ return false; ++ } ++ stack.back().GetScalar() = load_Addr; ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ // Fall through to load address code below... ++ } ++ LLVM_FALLTHROUGH; ++ case Value::ValueType::LoadAddress: ++ if (m_exe_ctx) { ++ if (process) { ++ lldb::addr_t pointer_addr = ++ stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); ++ Status error; ++ lldb::addr_t pointer_value = ++ process->ReadPointerFromMemory(pointer_addr, error); ++ if (pointer_value != LLDB_INVALID_ADDRESS) { ++ stack.back().GetScalar() = pointer_value; ++ stack.back().ClearContext(); ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "Failed to dereference pointer from 0x%" PRIx64 ++ " for DW_OP_deref: %s\n", ++ pointer_addr, error.AsCString()); ++ return false; ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "NULL process for DW_OP_deref.\n"); ++ return false; ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "NULL execution context for DW_OP_deref.\n"); ++ return false; ++ } ++ break; ++ ++ default: ++ break; ++ } ++ ++ } break; ++ ++ // OPCODE: DW_OP_deref_size ++ // OPERANDS: 1 ++ // 1 - uint8_t that specifies the size of the data to dereference. ++ // DESCRIPTION: Behaves like the DW_OP_deref operation: it pops the top ++ // stack entry and treats it as an address. The value retrieved from that ++ // address is pushed. In the DW_OP_deref_size operation, however, the size ++ // in bytes of the data retrieved from the dereferenced address is ++ // specified by the single operand. This operand is a 1-byte unsigned ++ // integral constant whose value may not be larger than the size of an ++ // address on the target machine. The data retrieved is zero extended to ++ // the size of an address on the target machine before being pushed on the ++ // expression stack. ++ case DW_OP_deref_size: { ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack empty for DW_OP_deref_size."); ++ return false; ++ } ++ uint8_t size = opcodes.GetU8(&offset); ++ Value::ValueType value_type = stack.back().GetValueType(); ++ switch (value_type) { ++ case Value::ValueType::HostAddress: { ++ void *src = (void *)stack.back().GetScalar().ULongLong(); ++ intptr_t ptr; ++ ::memcpy(&ptr, src, sizeof(void *)); ++ // I can't decide whether the size operand should apply to the bytes in ++ // their ++ // lldb-host endianness or the target endianness.. I doubt this'll ever ++ // come up but I'll opt for assuming big endian regardless. ++ switch (size) { ++ case 1: ++ ptr = ptr & 0xff; ++ break; ++ case 2: ++ ptr = ptr & 0xffff; ++ break; ++ case 3: ++ ptr = ptr & 0xffffff; ++ break; ++ case 4: ++ ptr = ptr & 0xffffffff; ++ break; ++ // the casts are added to work around the case where intptr_t is a 32 ++ // bit quantity; ++ // presumably we won't hit the 5..7 cases if (void*) is 32-bits in this ++ // program. ++ case 5: ++ ptr = (intptr_t)ptr & 0xffffffffffULL; ++ break; ++ case 6: ++ ptr = (intptr_t)ptr & 0xffffffffffffULL; ++ break; ++ case 7: ++ ptr = (intptr_t)ptr & 0xffffffffffffffULL; ++ break; ++ default: ++ break; ++ } ++ stack.back().GetScalar() = ptr; ++ stack.back().ClearContext(); ++ } break; ++ case Value::ValueType::LoadAddress: ++ if (m_exe_ctx) { ++ if (process) { ++ lldb::addr_t pointer_addr = ++ stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); ++ uint8_t addr_bytes[sizeof(lldb::addr_t)]; ++ Status error; ++ if (process->ReadMemory(pointer_addr, &addr_bytes, size, error) == ++ size) { ++ DataExtractor addr_data(addr_bytes, sizeof(addr_bytes), ++ process->GetByteOrder(), size); ++ lldb::offset_t addr_data_offset = 0; ++ switch (size) { ++ case 1: ++ stack.back().GetScalar() = addr_data.GetU8(&addr_data_offset); ++ break; ++ case 2: ++ stack.back().GetScalar() = addr_data.GetU16(&addr_data_offset); ++ break; ++ case 4: ++ stack.back().GetScalar() = addr_data.GetU32(&addr_data_offset); ++ break; ++ case 8: ++ stack.back().GetScalar() = addr_data.GetU64(&addr_data_offset); ++ break; ++ default: ++ stack.back().GetScalar() = ++ addr_data.GetAddress(&addr_data_offset); ++ } ++ stack.back().ClearContext(); ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "Failed to dereference pointer from 0x%" PRIx64 ++ " for DW_OP_deref: %s\n", ++ pointer_addr, error.AsCString()); ++ return false; ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "NULL process for DW_OP_deref.\n"); ++ return false; ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "NULL execution context for DW_OP_deref.\n"); ++ return false; ++ } ++ break; ++ ++ default: ++ break; ++ } ++ ++ } break; ++ ++ // OPCODE: DW_OP_xderef_size ++ // OPERANDS: 1 ++ // 1 - uint8_t that specifies the size of the data to dereference. ++ // DESCRIPTION: Behaves like the DW_OP_xderef operation: the entry at ++ // the top of the stack is treated as an address. The second stack entry is ++ // treated as an "address space identifier" for those architectures that ++ // support multiple address spaces. The top two stack elements are popped, ++ // a data item is retrieved through an implementation-defined address ++ // calculation and pushed as the new stack top. In the DW_OP_xderef_size ++ // operation, however, the size in bytes of the data retrieved from the ++ // dereferenced address is specified by the single operand. This operand is ++ // a 1-byte unsigned integral constant whose value may not be larger than ++ // the size of an address on the target machine. The data retrieved is zero ++ // extended to the size of an address on the target machine before being ++ // pushed on the expression stack. ++ case DW_OP_xderef_size: ++ if (error_ptr) ++ error_ptr->SetErrorString("Unimplemented opcode: DW_OP_xderef_size."); ++ return false; ++ // OPCODE: DW_OP_xderef ++ // OPERANDS: none ++ // DESCRIPTION: Provides an extended dereference mechanism. The entry at ++ // the top of the stack is treated as an address. The second stack entry is ++ // treated as an "address space identifier" for those architectures that ++ // support multiple address spaces. The top two stack elements are popped, ++ // a data item is retrieved through an implementation-defined address ++ // calculation and pushed as the new stack top. The size of the data ++ // retrieved from the dereferenced address is the size of an address on the ++ // target machine. ++ case DW_OP_xderef: ++ if (error_ptr) ++ error_ptr->SetErrorString("Unimplemented opcode: DW_OP_xderef."); ++ return false; ++ ++ // All DW_OP_constXXX opcodes have a single operand as noted below: ++ // ++ // Opcode Operand 1 ++ // DW_OP_const1u 1-byte unsigned integer constant DW_OP_const1s ++ // 1-byte signed integer constant DW_OP_const2u 2-byte unsigned integer ++ // constant DW_OP_const2s 2-byte signed integer constant DW_OP_const4u ++ // 4-byte unsigned integer constant DW_OP_const4s 4-byte signed integer ++ // constant DW_OP_const8u 8-byte unsigned integer constant DW_OP_const8s ++ // 8-byte signed integer constant DW_OP_constu unsigned LEB128 integer ++ // constant DW_OP_consts signed LEB128 integer constant ++ case DW_OP_const1u: ++ stack.push_back(Scalar((uint8_t)opcodes.GetU8(&offset))); ++ break; ++ case DW_OP_const1s: ++ stack.push_back(Scalar((int8_t)opcodes.GetU8(&offset))); ++ break; ++ case DW_OP_const2u: ++ stack.push_back(Scalar((uint16_t)opcodes.GetU16(&offset))); ++ break; ++ case DW_OP_const2s: ++ stack.push_back(Scalar((int16_t)opcodes.GetU16(&offset))); ++ break; ++ case DW_OP_const4u: ++ stack.push_back(Scalar((uint32_t)opcodes.GetU32(&offset))); ++ break; ++ case DW_OP_const4s: ++ stack.push_back(Scalar((int32_t)opcodes.GetU32(&offset))); ++ break; ++ case DW_OP_const8u: ++ stack.push_back(Scalar((uint64_t)opcodes.GetU64(&offset))); ++ break; ++ case DW_OP_const8s: ++ stack.push_back(Scalar((int64_t)opcodes.GetU64(&offset))); ++ break; ++ case DW_OP_constu: ++ stack.push_back(Scalar(opcodes.GetULEB128(&offset))); ++ break; ++ case DW_OP_consts: ++ stack.push_back(Scalar(opcodes.GetSLEB128(&offset))); ++ break; ++ ++ // OPCODE: DW_OP_dup ++ // OPERANDS: none ++ // DESCRIPTION: duplicates the value at the top of the stack ++ case DW_OP_dup: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Expression stack empty for DW_OP_dup."); ++ return false; ++ } else ++ stack.push_back(stack.back()); ++ break; ++ ++ // OPCODE: DW_OP_drop ++ // OPERANDS: none ++ // DESCRIPTION: pops the value at the top of the stack ++ case DW_OP_drop: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Expression stack empty for DW_OP_drop."); ++ return false; ++ } else ++ stack.pop_back(); ++ break; ++ ++ // OPCODE: DW_OP_over ++ // OPERANDS: none ++ // DESCRIPTION: Duplicates the entry currently second in the stack at ++ // the top of the stack. ++ case DW_OP_over: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_over."); ++ return false; ++ } else ++ stack.push_back(stack[stack.size() - 2]); ++ break; ++ ++ // OPCODE: DW_OP_pick ++ // OPERANDS: uint8_t index into the current stack ++ // DESCRIPTION: The stack entry with the specified index (0 through 255, ++ // inclusive) is pushed on the stack ++ case DW_OP_pick: { ++ uint8_t pick_idx = opcodes.GetU8(&offset); ++ if (pick_idx < stack.size()) ++ stack.push_back(stack[stack.size() - 1 - pick_idx]); ++ else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "Index %u out of range for DW_OP_pick.\n", pick_idx); ++ return false; ++ } ++ } break; ++ ++ // OPCODE: DW_OP_swap ++ // OPERANDS: none ++ // DESCRIPTION: swaps the top two stack entries. The entry at the top ++ // of the stack becomes the second stack entry, and the second entry ++ // becomes the top of the stack ++ case DW_OP_swap: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_swap."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.back() = stack[stack.size() - 2]; ++ stack[stack.size() - 2] = tmp; ++ } ++ break; ++ ++ // OPCODE: DW_OP_rot ++ // OPERANDS: none ++ // DESCRIPTION: Rotates the first three stack entries. The entry at ++ // the top of the stack becomes the third stack entry, the second entry ++ // becomes the top of the stack, and the third entry becomes the second ++ // entry. ++ case DW_OP_rot: ++ if (stack.size() < 3) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 3 items for DW_OP_rot."); ++ return false; ++ } else { ++ size_t last_idx = stack.size() - 1; ++ Value old_top = stack[last_idx]; ++ stack[last_idx] = stack[last_idx - 1]; ++ stack[last_idx - 1] = stack[last_idx - 2]; ++ stack[last_idx - 2] = old_top; ++ } ++ break; ++ ++ // OPCODE: DW_OP_abs ++ // OPERANDS: none ++ // DESCRIPTION: pops the top stack entry, interprets it as a signed ++ // value and pushes its absolute value. If the absolute value can not be ++ // represented, the result is undefined. ++ case DW_OP_abs: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_abs."); ++ return false; ++ } else if (!stack.back().ResolveValue(m_exe_ctx).AbsoluteValue()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Failed to take the absolute value of the first stack item."); ++ return false; ++ } ++ break; ++ ++ // OPCODE: DW_OP_and ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, performs a bitwise and ++ // operation on the two, and pushes the result. ++ case DW_OP_and: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_and."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) & tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_div ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, divides the former second ++ // entry by the former top of the stack using signed division, and pushes ++ // the result. ++ case DW_OP_div: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_div."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ if (tmp.ResolveValue(m_exe_ctx).IsZero()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Divide by zero."); ++ return false; ++ } else { ++ stack.pop_back(); ++ stack.back() = ++ stack.back().ResolveValue(m_exe_ctx) / tmp.ResolveValue(m_exe_ctx); ++ if (!stack.back().ResolveValue(m_exe_ctx).IsValid()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Divide failed."); ++ return false; ++ } ++ } ++ } ++ break; ++ ++ // OPCODE: DW_OP_minus ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, subtracts the former top ++ // of the stack from the former second entry, and pushes the result. ++ case DW_OP_minus: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_minus."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) - tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_mod ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values and pushes the result of ++ // the calculation: former second stack entry modulo the former top of the ++ // stack. ++ case DW_OP_mod: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_mod."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) % tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_mul ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack entries, multiplies them ++ // together, and pushes the result. ++ case DW_OP_mul: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_mul."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) * tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_neg ++ // OPERANDS: none ++ // DESCRIPTION: pops the top stack entry, and pushes its negation. ++ case DW_OP_neg: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_neg."); ++ return false; ++ } else { ++ if (!stack.back().ResolveValue(m_exe_ctx).UnaryNegate()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Unary negate failed."); ++ return false; ++ } ++ } ++ break; ++ ++ // OPCODE: DW_OP_not ++ // OPERANDS: none ++ // DESCRIPTION: pops the top stack entry, and pushes its bitwise ++ // complement ++ case DW_OP_not: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_not."); ++ return false; ++ } else { ++ if (!stack.back().ResolveValue(m_exe_ctx).OnesComplement()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Logical NOT failed."); ++ return false; ++ } ++ } ++ break; ++ ++ // OPCODE: DW_OP_or ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack entries, performs a bitwise or ++ // operation on the two, and pushes the result. ++ case DW_OP_or: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_or."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) | tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_plus ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack entries, adds them together, and ++ // pushes the result. ++ case DW_OP_plus: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_plus."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().GetScalar() += tmp.GetScalar(); ++ } ++ break; ++ ++ // OPCODE: DW_OP_plus_uconst ++ // OPERANDS: none ++ // DESCRIPTION: pops the top stack entry, adds it to the unsigned LEB128 ++ // constant operand and pushes the result. ++ case DW_OP_plus_uconst: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_plus_uconst."); ++ return false; ++ } else { ++ const uint64_t uconst_value = opcodes.GetULEB128(&offset); ++ // Implicit conversion from a UINT to a Scalar... ++ stack.back().GetScalar() += uconst_value; ++ if (!stack.back().GetScalar().IsValid()) { ++ if (error_ptr) ++ error_ptr->SetErrorString("DW_OP_plus_uconst failed."); ++ return false; ++ } ++ } ++ break; ++ ++ // OPCODE: DW_OP_shl ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack entries, shifts the former ++ // second entry left by the number of bits specified by the former top of ++ // the stack, and pushes the result. ++ case DW_OP_shl: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_shl."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) <<= tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_shr ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack entries, shifts the former second ++ // entry right logically (filling with zero bits) by the number of bits ++ // specified by the former top of the stack, and pushes the result. ++ case DW_OP_shr: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_shr."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ if (!stack.back().ResolveValue(m_exe_ctx).ShiftRightLogical( ++ tmp.ResolveValue(m_exe_ctx))) { ++ if (error_ptr) ++ error_ptr->SetErrorString("DW_OP_shr failed."); ++ return false; ++ } ++ } ++ break; ++ ++ // OPCODE: DW_OP_shra ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack entries, shifts the former second ++ // entry right arithmetically (divide the magnitude by 2, keep the same ++ // sign for the result) by the number of bits specified by the former top ++ // of the stack, and pushes the result. ++ case DW_OP_shra: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_shra."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) >>= tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_xor ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack entries, performs the bitwise ++ // exclusive-or operation on the two, and pushes the result. ++ case DW_OP_xor: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_xor."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) ^ tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_skip ++ // OPERANDS: int16_t ++ // DESCRIPTION: An unconditional branch. Its single operand is a 2-byte ++ // signed integer constant. The 2-byte constant is the number of bytes of ++ // the DWARF expression to skip forward or backward from the current ++ // operation, beginning after the 2-byte constant. ++ case DW_OP_skip: { ++ int16_t skip_offset = (int16_t)opcodes.GetU16(&offset); ++ lldb::offset_t new_offset = offset + skip_offset; ++ if (opcodes.ValidOffset(new_offset)) ++ offset = new_offset; ++ else { ++ if (error_ptr) ++ error_ptr->SetErrorString("Invalid opcode offset in DW_OP_skip."); ++ return false; ++ } ++ } break; ++ ++ // OPCODE: DW_OP_bra ++ // OPERANDS: int16_t ++ // DESCRIPTION: A conditional branch. Its single operand is a 2-byte ++ // signed integer constant. This operation pops the top of stack. If the ++ // value popped is not the constant 0, the 2-byte constant operand is the ++ // number of bytes of the DWARF expression to skip forward or backward from ++ // the current operation, beginning after the 2-byte constant. ++ case DW_OP_bra: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_bra."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ int16_t bra_offset = (int16_t)opcodes.GetU16(&offset); ++ Scalar zero(0); ++ if (tmp.ResolveValue(m_exe_ctx) != zero) { ++ lldb::offset_t new_offset = offset + bra_offset; ++ if (opcodes.ValidOffset(new_offset)) ++ offset = new_offset; ++ else { ++ if (error_ptr) ++ error_ptr->SetErrorString("Invalid opcode offset in DW_OP_bra."); ++ return false; ++ } ++ } ++ } ++ break; ++ ++ // OPCODE: DW_OP_eq ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, compares using the ++ // equals (==) operator. ++ // STACK RESULT: push the constant value 1 onto the stack if the result ++ // of the operation is true or the constant value 0 if the result of the ++ // operation is false. ++ case DW_OP_eq: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_eq."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) == tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_ge ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, compares using the ++ // greater than or equal to (>=) operator. ++ // STACK RESULT: push the constant value 1 onto the stack if the result ++ // of the operation is true or the constant value 0 if the result of the ++ // operation is false. ++ case DW_OP_ge: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_ge."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) >= tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_gt ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, compares using the ++ // greater than (>) operator. ++ // STACK RESULT: push the constant value 1 onto the stack if the result ++ // of the operation is true or the constant value 0 if the result of the ++ // operation is false. ++ case DW_OP_gt: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_gt."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) > tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_le ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, compares using the ++ // less than or equal to (<=) operator. ++ // STACK RESULT: push the constant value 1 onto the stack if the result ++ // of the operation is true or the constant value 0 if the result of the ++ // operation is false. ++ case DW_OP_le: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_le."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) <= tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_lt ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, compares using the ++ // less than (<) operator. ++ // STACK RESULT: push the constant value 1 onto the stack if the result ++ // of the operation is true or the constant value 0 if the result of the ++ // operation is false. ++ case DW_OP_lt: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_lt."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) < tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_ne ++ // OPERANDS: none ++ // DESCRIPTION: pops the top two stack values, compares using the ++ // not equal (!=) operator. ++ // STACK RESULT: push the constant value 1 onto the stack if the result ++ // of the operation is true or the constant value 0 if the result of the ++ // operation is false. ++ case DW_OP_ne: ++ if (stack.size() < 2) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 2 items for DW_OP_ne."); ++ return false; ++ } else { ++ tmp = stack.back(); ++ stack.pop_back(); ++ stack.back().ResolveValue(m_exe_ctx) = ++ stack.back().ResolveValue(m_exe_ctx) != tmp.ResolveValue(m_exe_ctx); ++ } ++ break; ++ ++ // OPCODE: DW_OP_litn ++ // OPERANDS: none ++ // DESCRIPTION: encode the unsigned literal values from 0 through 31. ++ // STACK RESULT: push the unsigned literal constant value onto the top ++ // of the stack. ++ case DW_OP_lit0: ++ case DW_OP_lit1: ++ case DW_OP_lit2: ++ case DW_OP_lit3: ++ case DW_OP_lit4: ++ case DW_OP_lit5: ++ case DW_OP_lit6: ++ case DW_OP_lit7: ++ case DW_OP_lit8: ++ case DW_OP_lit9: ++ case DW_OP_lit10: ++ case DW_OP_lit11: ++ case DW_OP_lit12: ++ case DW_OP_lit13: ++ case DW_OP_lit14: ++ case DW_OP_lit15: ++ case DW_OP_lit16: ++ case DW_OP_lit17: ++ case DW_OP_lit18: ++ case DW_OP_lit19: ++ case DW_OP_lit20: ++ case DW_OP_lit21: ++ case DW_OP_lit22: ++ case DW_OP_lit23: ++ case DW_OP_lit24: ++ case DW_OP_lit25: ++ case DW_OP_lit26: ++ case DW_OP_lit27: ++ case DW_OP_lit28: ++ case DW_OP_lit29: ++ case DW_OP_lit30: ++ case DW_OP_lit31: ++ stack.push_back(Scalar((uint64_t)(op - DW_OP_lit0))); ++ break; ++ ++ // OPCODE: DW_OP_regN ++ // OPERANDS: none ++ // DESCRIPTION: Push the value in register n on the top of the stack. ++ case DW_OP_reg0: ++ case DW_OP_reg1: ++ case DW_OP_reg2: ++ case DW_OP_reg3: ++ case DW_OP_reg4: ++ case DW_OP_reg5: ++ case DW_OP_reg6: ++ case DW_OP_reg7: ++ case DW_OP_reg8: ++ case DW_OP_reg9: ++ case DW_OP_reg10: ++ case DW_OP_reg11: ++ case DW_OP_reg12: ++ case DW_OP_reg13: ++ case DW_OP_reg14: ++ case DW_OP_reg15: ++ case DW_OP_reg16: ++ case DW_OP_reg17: ++ case DW_OP_reg18: ++ case DW_OP_reg19: ++ case DW_OP_reg20: ++ case DW_OP_reg21: ++ case DW_OP_reg22: ++ case DW_OP_reg23: ++ case DW_OP_reg24: ++ case DW_OP_reg25: ++ case DW_OP_reg26: ++ case DW_OP_reg27: ++ case DW_OP_reg28: ++ case DW_OP_reg29: ++ case DW_OP_reg30: ++ case DW_OP_reg31: { ++ reg_num = op - DW_OP_reg0; ++ ++ if (ReadRegisterValueAsScalar(m_reg_ctx, reg_kind, reg_num, error_ptr, tmp)) ++ stack.push_back(tmp); ++ else ++ return false; ++ } break; ++ // OPCODE: DW_OP_regx ++ // OPERANDS: ++ // ULEB128 literal operand that encodes the register. ++ // DESCRIPTION: Push the value in register on the top of the stack. ++ case DW_OP_regx: { ++ reg_num = opcodes.GetULEB128(&offset); ++ if (ReadRegisterValueAsScalar(m_reg_ctx, reg_kind, reg_num, error_ptr, tmp)) ++ stack.push_back(tmp); ++ else ++ return false; ++ } break; ++ ++ // OPCODE: DW_OP_bregN ++ // OPERANDS: ++ // SLEB128 offset from register N ++ // DESCRIPTION: Value is in memory at the address specified by register ++ // N plus an offset. ++ case DW_OP_breg0: ++ case DW_OP_breg1: ++ case DW_OP_breg2: ++ case DW_OP_breg3: ++ case DW_OP_breg4: ++ case DW_OP_breg5: ++ case DW_OP_breg6: ++ case DW_OP_breg7: ++ case DW_OP_breg8: ++ case DW_OP_breg9: ++ case DW_OP_breg10: ++ case DW_OP_breg11: ++ case DW_OP_breg12: ++ case DW_OP_breg13: ++ case DW_OP_breg14: ++ case DW_OP_breg15: ++ case DW_OP_breg16: ++ case DW_OP_breg17: ++ case DW_OP_breg18: ++ case DW_OP_breg19: ++ case DW_OP_breg20: ++ case DW_OP_breg21: ++ case DW_OP_breg22: ++ case DW_OP_breg23: ++ case DW_OP_breg24: ++ case DW_OP_breg25: ++ case DW_OP_breg26: ++ case DW_OP_breg27: ++ case DW_OP_breg28: ++ case DW_OP_breg29: ++ case DW_OP_breg30: ++ case DW_OP_breg31: { ++ reg_num = op - DW_OP_breg0; ++ ++ if (ReadRegisterValueAsScalar(m_reg_ctx, reg_kind, reg_num, error_ptr, ++ tmp)) { ++ int64_t breg_offset = opcodes.GetSLEB128(&offset); ++ tmp.ResolveValue(m_exe_ctx) += (uint64_t)breg_offset; ++ tmp.ClearContext(); ++ stack.push_back(tmp); ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ } else ++ return false; ++ } break; ++ // OPCODE: DW_OP_bregx ++ // OPERANDS: 2 ++ // ULEB128 literal operand that encodes the register. ++ // SLEB128 offset from register N ++ // DESCRIPTION: Value is in memory at the address specified by register ++ // N plus an offset. ++ case DW_OP_bregx: { ++ reg_num = opcodes.GetULEB128(&offset); ++ ++ if (ReadRegisterValueAsScalar(m_reg_ctx, reg_kind, reg_num, error_ptr, ++ tmp)) { ++ int64_t breg_offset = opcodes.GetSLEB128(&offset); ++ tmp.ResolveValue(m_exe_ctx) += (uint64_t)breg_offset; ++ tmp.ClearContext(); ++ stack.push_back(tmp); ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ } else ++ return false; ++ } break; ++ ++ case DW_OP_fbreg: ++ if (m_exe_ctx) { ++ if (frame) { ++ Scalar value; ++ if (frame->GetFrameBaseValue(value, error_ptr)) { ++ int64_t fbreg_offset = opcodes.GetSLEB128(&offset); ++ value += fbreg_offset; ++ stack.push_back(value); ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ } else ++ return false; ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Invalid stack frame in context for DW_OP_fbreg opcode."); ++ return false; ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "NULL execution context for DW_OP_fbreg.\n"); ++ return false; ++ } ++ ++ break; ++ ++ // OPCODE: DW_OP_nop ++ // OPERANDS: none ++ // DESCRIPTION: A place holder. It has no effect on the location stack ++ // or any of its values. ++ case DW_OP_nop: ++ break; ++ ++ // OPCODE: DW_OP_piece ++ // OPERANDS: 1 ++ // ULEB128: byte size of the piece ++ // DESCRIPTION: The operand describes the size in bytes of the piece of ++ // the object referenced by the DWARF expression whose result is at the top ++ // of the stack. If the piece is located in a register, but does not occupy ++ // the entire register, the placement of the piece within that register is ++ // defined by the ABI. ++ // ++ // Many compilers store a single variable in sets of registers, or store a ++ // variable partially in memory and partially in registers. DW_OP_piece ++ // provides a way of describing how large a part of a variable a particular ++ // DWARF expression refers to. ++ case DW_OP_piece: { ++ const uint64_t piece_byte_size = opcodes.GetULEB128(&offset); ++ ++ if (piece_byte_size > 0) { ++ Value curr_piece; ++ ++ if (stack.empty()) { ++ // In a multi-piece expression, this means that the current piece is ++ // not available. Fill with zeros for now by resizing the data and ++ // appending it ++ curr_piece.ResizeData(piece_byte_size); ++ // Note that "0" is not a correct value for the unknown bits. ++ // It would be better to also return a mask of valid bits together ++ // with the expression result, so the debugger can print missing ++ // members as "" or something. ++ ::memset(curr_piece.GetBuffer().GetBytes(), 0, piece_byte_size); ++ pieces.AppendDataToHostBuffer(curr_piece); ++ } else { ++ Status error; ++ // Extract the current piece into "curr_piece" ++ Value curr_piece_source_value(stack.back()); ++ stack.pop_back(); ++ ++ const Value::ValueType curr_piece_source_value_type = ++ curr_piece_source_value.GetValueType(); ++ switch (curr_piece_source_value_type) { ++ case Value::ValueType::LoadAddress: ++ if (process) { ++ if (curr_piece.ResizeData(piece_byte_size) == piece_byte_size) { ++ lldb::addr_t load_addr = ++ curr_piece_source_value.GetScalar().ULongLong( ++ LLDB_INVALID_ADDRESS); ++ if (process->ReadMemory( ++ load_addr, curr_piece.GetBuffer().GetBytes(), ++ piece_byte_size, error) != piece_byte_size) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "failed to read memory DW_OP_piece(%" PRIu64 ++ ") from 0x%" PRIx64, ++ piece_byte_size, load_addr); ++ return false; ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "failed to resize the piece memory buffer for " ++ "DW_OP_piece(%" PRIu64 ")", ++ piece_byte_size); ++ return false; ++ } ++ } ++ break; ++ ++ case Value::ValueType::FileAddress: ++ case Value::ValueType::HostAddress: ++ if (error_ptr) { ++ lldb::addr_t addr = curr_piece_source_value.GetScalar().ULongLong( ++ LLDB_INVALID_ADDRESS); ++ error_ptr->SetErrorStringWithFormat( ++ "failed to read memory DW_OP_piece(%" PRIu64 ++ ") from %s address 0x%" PRIx64, ++ piece_byte_size, ++ curr_piece_source_value.GetValueType() == ++ Value::ValueType::FileAddress ++ ? "file" ++ : "host", ++ addr); ++ } ++ return false; ++ ++ case Value::ValueType::Scalar: { ++ uint32_t bit_size = piece_byte_size * 8; ++ uint32_t bit_offset = 0; ++ Scalar &scalar = curr_piece_source_value.GetScalar(); ++ if (!scalar.ExtractBitfield(bit_size, bit_offset)) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "unable to extract %" PRIu64 " bytes from a %" PRIu64 ++ " byte scalar value.", ++ piece_byte_size, ++ (uint64_t)curr_piece_source_value.GetScalar().GetByteSize()); ++ return false; ++ } ++ // Create curr_piece with bit_size. By default Scalar ++ // grows to the nearest host integer type. ++ llvm::APInt fail_value(1, 0, false); ++ llvm::APInt ap_int = scalar.UInt128(fail_value); ++ assert(ap_int.getBitWidth() >= bit_size); ++ llvm::ArrayRef buf{ap_int.getRawData(), ++ ap_int.getNumWords()}; ++ curr_piece.GetScalar() = Scalar(llvm::APInt(bit_size, buf)); ++ } break; ++ } ++ ++ // Check if this is the first piece? ++ if (op_piece_offset == 0) { ++ // This is the first piece, we should push it back onto the stack ++ // so subsequent pieces will be able to access this piece and add ++ // to it. ++ if (pieces.AppendDataToHostBuffer(curr_piece) == 0) { ++ if (error_ptr) ++ error_ptr->SetErrorString("failed to append piece data"); ++ return false; ++ } ++ } else { ++ // If this is the second or later piece there should be a value on ++ // the stack. ++ if (pieces.GetBuffer().GetByteSize() != op_piece_offset) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "DW_OP_piece for offset %" PRIu64 ++ " but top of stack is of size %" PRIu64, ++ op_piece_offset, pieces.GetBuffer().GetByteSize()); ++ return false; ++ } ++ ++ if (pieces.AppendDataToHostBuffer(curr_piece) == 0) { ++ if (error_ptr) ++ error_ptr->SetErrorString("failed to append piece data"); ++ return false; ++ } ++ } ++ } ++ op_piece_offset += piece_byte_size; ++ } ++ } break; ++ ++ case DW_OP_bit_piece: // 0x9d ULEB128 bit size, ULEB128 bit offset (DWARF3); ++ if (stack.size() < 1) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_bit_piece."); ++ return false; ++ } else { ++ const uint64_t piece_bit_size = opcodes.GetULEB128(&offset); ++ const uint64_t piece_bit_offset = opcodes.GetULEB128(&offset); ++ switch (stack.back().GetValueType()) { ++ case Value::ValueType::Scalar: { ++ if (!stack.back().GetScalar().ExtractBitfield(piece_bit_size, ++ piece_bit_offset)) { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "unable to extract %" PRIu64 " bit value with %" PRIu64 ++ " bit offset from a %" PRIu64 " bit scalar value.", ++ piece_bit_size, piece_bit_offset, ++ (uint64_t)(stack.back().GetScalar().GetByteSize() * 8)); ++ return false; ++ } ++ } break; ++ ++ case Value::ValueType::FileAddress: ++ case Value::ValueType::LoadAddress: ++ case Value::ValueType::HostAddress: ++ if (error_ptr) { ++ error_ptr->SetErrorStringWithFormat( ++ "unable to extract DW_OP_bit_piece(bit_size = %" PRIu64 ++ ", bit_offset = %" PRIu64 ") from an address value.", ++ piece_bit_size, piece_bit_offset); ++ } ++ return false; ++ } ++ } ++ break; ++ ++ // OPCODE: DW_OP_push_object_address ++ // OPERANDS: none ++ // DESCRIPTION: Pushes the address of the object currently being ++ // evaluated as part of evaluation of a user presented expression. This ++ // object may correspond to an independent variable described by its own ++ // DIE or it may be a component of an array, structure, or class whose ++ // address has been dynamically determined by an earlier step during user ++ // expression evaluation. ++ case DW_OP_push_object_address: ++ if (m_object_address_ptr) ++ stack.push_back(*m_object_address_ptr); ++ else { ++ if (error_ptr) ++ error_ptr->SetErrorString("DW_OP_push_object_address used without " ++ "specifying an object address"); ++ return false; ++ } ++ break; ++ ++ // OPCODE: DW_OP_call2 ++ // OPERANDS: ++ // uint16_t compile unit relative offset of a DIE ++ // DESCRIPTION: Performs subroutine calls during evaluation ++ // of a DWARF expression. The operand is the 2-byte unsigned offset of a ++ // debugging information entry in the current compilation unit. ++ // ++ // Operand interpretation is exactly like that for DW_FORM_ref2. ++ // ++ // This operation transfers control of DWARF expression evaluation to the ++ // DW_AT_location attribute of the referenced DIE. If there is no such ++ // attribute, then there is no effect. Execution of the DWARF expression of ++ // a DW_AT_location attribute may add to and/or remove from values on the ++ // stack. Execution returns to the point following the call when the end of ++ // the attribute is reached. Values on the stack at the time of the call ++ // may be used as parameters by the called expression and values left on ++ // the stack by the called expression may be used as return values by prior ++ // agreement between the calling and called expressions. ++ case DW_OP_call2: ++ if (error_ptr) ++ error_ptr->SetErrorString("Unimplemented opcode DW_OP_call2."); ++ return false; ++ // OPCODE: DW_OP_call4 ++ // OPERANDS: 1 ++ // uint32_t compile unit relative offset of a DIE ++ // DESCRIPTION: Performs a subroutine call during evaluation of a DWARF ++ // expression. For DW_OP_call4, the operand is a 4-byte unsigned offset of ++ // a debugging information entry in the current compilation unit. ++ // ++ // Operand interpretation DW_OP_call4 is exactly like that for ++ // DW_FORM_ref4. ++ // ++ // This operation transfers control of DWARF expression evaluation to the ++ // DW_AT_location attribute of the referenced DIE. If there is no such ++ // attribute, then there is no effect. Execution of the DWARF expression of ++ // a DW_AT_location attribute may add to and/or remove from values on the ++ // stack. Execution returns to the point following the call when the end of ++ // the attribute is reached. Values on the stack at the time of the call ++ // may be used as parameters by the called expression and values left on ++ // the stack by the called expression may be used as return values by prior ++ // agreement between the calling and called expressions. ++ case DW_OP_call4: ++ if (error_ptr) ++ error_ptr->SetErrorString("Unimplemented opcode DW_OP_call4."); ++ return false; ++ ++ // OPCODE: DW_OP_stack_value ++ // OPERANDS: None ++ // DESCRIPTION: Specifies that the object does not exist in memory but ++ // rather is a constant value. The value from the top of the stack is the ++ // value to be used. This is the actual object value and not the location. ++ case DW_OP_stack_value: ++ if (stack.empty()) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_stack_value."); ++ return false; ++ } ++ stack.back().SetValueType(Value::ValueType::Scalar); ++ break; ++ ++ // OPCODE: DW_OP_convert ++ // OPERANDS: 1 ++ // A ULEB128 that is either a DIE offset of a ++ // DW_TAG_base_type or 0 for the generic (pointer-sized) type. ++ // ++ // DESCRIPTION: Pop the top stack element, convert it to a ++ // different type, and push the result. ++ case DW_OP_convert: { ++ if (stack.size() < 1) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Expression stack needs at least 1 item for DW_OP_convert."); ++ return false; ++ } ++ const uint64_t die_offset = opcodes.GetULEB128(&offset); ++ uint64_t bit_size; ++ bool sign; ++ if (die_offset == 0) { ++ // The generic type has the size of an address on the target ++ // machine and an unspecified signedness. Scalar has no ++ // "unspecified signedness", so we use unsigned types. ++ if (!module_sp) { ++ if (error_ptr) ++ error_ptr->SetErrorString("No module"); ++ return false; ++ } ++ sign = false; ++ bit_size = module_sp->GetArchitecture().GetAddressByteSize() * 8; ++ if (!bit_size) { ++ if (error_ptr) ++ error_ptr->SetErrorString("unspecified architecture"); ++ return false; ++ } ++ } else { ++ // Retrieve the type DIE that the value is being converted to. ++ // FIXME: the constness has annoying ripple effects. ++ DWARFDIE die = const_cast(dwarf_cu)->GetDIE(die_offset); ++ if (!die) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Cannot resolve DW_OP_convert type DIE"); ++ return false; ++ } ++ uint64_t encoding = ++ die.GetAttributeValueAsUnsigned(DW_AT_encoding, DW_ATE_hi_user); ++ bit_size = die.GetAttributeValueAsUnsigned(DW_AT_byte_size, 0) * 8; ++ if (!bit_size) ++ bit_size = die.GetAttributeValueAsUnsigned(DW_AT_bit_size, 0); ++ if (!bit_size) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Unsupported type size in DW_OP_convert"); ++ return false; ++ } ++ switch (encoding) { ++ case DW_ATE_signed: ++ case DW_ATE_signed_char: ++ sign = true; ++ break; ++ case DW_ATE_unsigned: ++ case DW_ATE_unsigned_char: ++ sign = false; ++ break; ++ default: ++ if (error_ptr) ++ error_ptr->SetErrorString("Unsupported encoding in DW_OP_convert"); ++ return false; ++ } ++ } ++ Scalar &top = stack.back().ResolveValue(m_exe_ctx); ++ top.TruncOrExtendTo(bit_size, sign); ++ break; ++ } ++ ++ // OPCODE: DW_OP_call_frame_cfa ++ // OPERANDS: None ++ // DESCRIPTION: Specifies a DWARF expression that pushes the value of ++ // the canonical frame address consistent with the call frame information ++ // located in .debug_frame (or in the FDEs of the eh_frame section). ++ case DW_OP_call_frame_cfa: ++ if (frame) { ++ // Note that we don't have to parse FDEs because this DWARF expression ++ // is commonly evaluated with a valid stack frame. ++ StackID id = frame->GetStackID(); ++ addr_t cfa = id.GetCallFrameAddress(); ++ if (cfa != LLDB_INVALID_ADDRESS) { ++ stack.push_back(Scalar(cfa)); ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ } else if (error_ptr) ++ error_ptr->SetErrorString("Stack frame does not include a canonical " ++ "frame address for DW_OP_call_frame_cfa " ++ "opcode."); ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorString("Invalid stack frame in context for " ++ "DW_OP_call_frame_cfa opcode."); ++ return false; ++ } ++ break; ++ ++ // OPCODE: DW_OP_form_tls_address (or the old pre-DWARFv3 vendor extension ++ // opcode, DW_OP_GNU_push_tls_address) ++ // OPERANDS: none ++ // DESCRIPTION: Pops a TLS offset from the stack, converts it to ++ // an address in the current thread's thread-local storage block, and ++ // pushes it on the stack. ++ case DW_OP_form_tls_address: ++ case DW_OP_GNU_push_tls_address: { ++ if (stack.size() < 1) { ++ if (error_ptr) { ++ if (op == DW_OP_form_tls_address) ++ error_ptr->SetErrorString( ++ "DW_OP_form_tls_address needs an argument."); ++ else ++ error_ptr->SetErrorString( ++ "DW_OP_GNU_push_tls_address needs an argument."); ++ } ++ return false; ++ } ++ ++ if (!m_exe_ctx || !module_sp) { ++ if (error_ptr) ++ error_ptr->SetErrorString("No context to evaluate TLS within."); ++ return false; ++ } ++ ++ Thread *thread = m_exe_ctx->GetThreadPtr(); ++ if (!thread) { ++ if (error_ptr) ++ error_ptr->SetErrorString("No thread to evaluate TLS within."); ++ return false; ++ } ++ ++ // Lookup the TLS block address for this thread and module. ++ const addr_t tls_file_addr = ++ stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); ++ const addr_t tls_load_addr = ++ thread->GetThreadLocalData(module_sp, tls_file_addr); ++ ++ if (tls_load_addr == LLDB_INVALID_ADDRESS) { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "No TLS data currently exists for this thread."); ++ return false; ++ } ++ ++ stack.back().GetScalar() = tls_load_addr; ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ } break; ++ ++ // OPCODE: DW_OP_addrx (DW_OP_GNU_addr_index is the legacy name.) ++ // OPERANDS: 1 ++ // ULEB128: index to the .debug_addr section ++ // DESCRIPTION: Pushes an address to the stack from the .debug_addr ++ // section with the base address specified by the DW_AT_addr_base attribute ++ // and the 0 based index is the ULEB128 encoded index. ++ case DW_OP_addrx: ++ case DW_OP_GNU_addr_index: { ++ if (!dwarf_cu) { ++ if (error_ptr) ++ error_ptr->SetErrorString("DW_OP_GNU_addr_index found without a " ++ "compile unit being specified"); ++ return false; ++ } ++ uint64_t index = opcodes.GetULEB128(&offset); ++ lldb::addr_t value = ++ DWARFExpression::ReadAddressFromDebugAddrSection(dwarf_cu, index); ++ stack.push_back(Scalar(value)); ++ stack.back().SetValueType(Value::ValueType::FileAddress); ++ } break; ++ ++ // OPCODE: DW_OP_GNU_const_index ++ // OPERANDS: 1 ++ // ULEB128: index to the .debug_addr section ++ // DESCRIPTION: Pushes an constant with the size of a machine address to ++ // the stack from the .debug_addr section with the base address specified ++ // by the DW_AT_addr_base attribute and the 0 based index is the ULEB128 ++ // encoded index. ++ case DW_OP_GNU_const_index: { ++ if (!dwarf_cu) { ++ if (error_ptr) ++ error_ptr->SetErrorString("DW_OP_GNU_const_index found without a " ++ "compile unit being specified"); ++ return false; ++ } ++ uint64_t index = opcodes.GetULEB128(&offset); ++ lldb::addr_t value = ++ DWARFExpression::ReadAddressFromDebugAddrSection(dwarf_cu, index); ++ stack.push_back(Scalar(value)); ++ } break; ++ ++ case DW_OP_entry_value: { ++ if (!Evaluate_DW_OP_entry_value(stack, m_exe_ctx, m_reg_ctx, opcodes, ++ offset, error_ptr, log)) { ++ LLDB_ERRORF(error_ptr, "Could not evaluate %s.", DW_OP_value_to_name(op)); ++ return false; ++ } ++ break; ++ } ++ ++ default: ++ LLDB_LOGF(log, "Unhandled opcode %s in DWARFExpression.", ++ DW_OP_value_to_name(op)); ++ break; ++ } ++ ++ return true; ++} +diff --git a/lldb/source/Expression/DWARFEvaluatorFactory.cpp b/lldb/source/Expression/DWARFEvaluatorFactory.cpp +new file mode 100644 +index 000000000000..c0612641204a +--- /dev/null ++++ b/lldb/source/Expression/DWARFEvaluatorFactory.cpp +@@ -0,0 +1,57 @@ ++//===-- DWARFEvaluatorFactory.cpp -----------------------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#include "lldb/Expression/DWARFEvaluatorFactory.h" ++#include "lldb/Expression/DWARFEvaluator.h" ++ ++#include "lldb/Core/PluginManager.h" ++#include "lldb/Core/Value.h" ++#include "lldb/Target/RegisterContext.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++ ++// PluginInterface protocol ++lldb_private::ConstString DWARFEvaluatorFactory::GetPluginName() { ++ static ConstString g_name("vendor-default"); ++ return g_name; ++} ++ ++// FindPlugin ++// ++// Platforms can register a callback to use when creating DWARF expression ++// evaluators to allow handling platform-specific DWARF codes. ++std::unique_ptr ++DWARFEvaluatorFactory::FindPlugin(Module *module) { ++ std::unique_ptr instance_up; ++ DWARFEvaluatorFactoryCreateInstance create_callback; ++ ++ for (size_t idx = 0; ++ (create_callback = ++ PluginManager::GetDWARFEvaluatorFactoryCreateCallbackAtIndex( ++ idx)) != nullptr; ++ ++idx) { ++ instance_up.reset(create_callback(module)); ++ ++ if (instance_up) { ++ return instance_up; ++ } ++ } ++ ++ instance_up.reset(new DWARFEvaluatorFactory()); ++ return instance_up; ++} ++ ++std::unique_ptr DWARFEvaluatorFactory::CreateDWARFEvaluator( ++ const DWARFExpression &dwarf_expression, ExecutionContext *exe_ctx, ++ RegisterContext *reg_ctx, const Value *initial_value_ptr, ++ const Value *object_address_ptr) { ++ return std::make_unique(dwarf_expression, exe_ctx, reg_ctx, ++ initial_value_ptr, ++ object_address_ptr); ++} +diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp +index a10546c1deae..4d13e4642af3 100644 +--- a/lldb/source/Expression/DWARFExpression.cpp ++++ b/lldb/source/Expression/DWARFExpression.cpp +@@ -15,6 +15,8 @@ + #include "lldb/Core/Module.h" + #include "lldb/Core/Value.h" + #include "lldb/Core/dwarf.h" ++#include "lldb/Expression/DWARFEvaluator.h" ++#include "lldb/Expression/DWARFEvaluatorFactory.h" + #include "lldb/Utility/DataEncoder.h" + #include "lldb/Utility/Log.h" + #include "lldb/Utility/RegisterValue.h" +@@ -41,8 +43,8 @@ + using namespace lldb; + using namespace lldb_private; + +-static lldb::addr_t +-ReadAddressFromDebugAddrSection(const DWARFUnit *dwarf_cu, ++lldb::addr_t ++DWARFExpression::ReadAddressFromDebugAddrSection(const DWARFUnit *dwarf_cu, + uint32_t index) { + uint32_t index_size = dwarf_cu->GetAddressByteSize(); + dw_offset_t addr_base = dwarf_cu->GetAddrBase(); +@@ -96,7 +98,7 @@ void DWARFExpression::SetLocationListAddresses(addr_t cu_file_addr, + m_loclist_addresses = LoclistAddresses{cu_file_addr, func_file_addr}; + } + +-int DWARFExpression::GetRegisterKind() { return m_reg_kind; } ++RegisterKind DWARFExpression::GetRegisterKind() const { return m_reg_kind; } + + void DWARFExpression::SetRegisterKind(RegisterKind reg_kind) { + m_reg_kind = reg_kind; +@@ -150,52 +152,6 @@ void DWARFExpression::GetDescription(Stream *s, lldb::DescriptionLevel level, + } + } + +-static bool ReadRegisterValueAsScalar(RegisterContext *reg_ctx, +- lldb::RegisterKind reg_kind, +- uint32_t reg_num, Status *error_ptr, +- Value &value) { +- if (reg_ctx == nullptr) { +- if (error_ptr) +- error_ptr->SetErrorString("No register context in frame.\n"); +- } else { +- uint32_t native_reg = +- reg_ctx->ConvertRegisterKindToRegisterNumber(reg_kind, reg_num); +- if (native_reg == LLDB_INVALID_REGNUM) { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat("Unable to convert register " +- "kind=%u reg_num=%u to a native " +- "register number.\n", +- reg_kind, reg_num); +- } else { +- const RegisterInfo *reg_info = +- reg_ctx->GetRegisterInfoAtIndex(native_reg); +- RegisterValue reg_value; +- if (reg_ctx->ReadRegister(reg_info, reg_value)) { +- if (reg_value.GetScalarValue(value.GetScalar())) { +- value.SetValueType(Value::ValueType::Scalar); +- value.SetContext(Value::ContextType::RegisterInfo, +- const_cast(reg_info)); +- if (error_ptr) +- error_ptr->Clear(); +- return true; +- } else { +- // If we get this error, then we need to implement a value buffer in +- // the dwarf expression evaluation function... +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "register %s can't be converted to a scalar value", +- reg_info->name); +- } +- } else { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat("register %s is not available", +- reg_info->name); +- } +- } +- } +- return false; +-} +- + /// Return the length in bytes of the set of operands for \p op. No guarantees + /// are made on the state of \p data after this call. + static offset_t GetOpcodeDataSize(const DataExtractor &data, +@@ -955,1719 +911,17 @@ bool DWARFExpression::Evaluate( + const Value *initial_value_ptr, const Value *object_address_ptr, + Value &result, Status *error_ptr) { + +- if (opcodes.GetByteSize() == 0) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "no location, value may have been optimized out"); +- return false; +- } +- std::vector stack; +- +- Process *process = nullptr; +- StackFrame *frame = nullptr; +- +- if (exe_ctx) { +- process = exe_ctx->GetProcessPtr(); +- frame = exe_ctx->GetFramePtr(); +- } +- if (reg_ctx == nullptr && frame) +- reg_ctx = frame->GetRegisterContext().get(); +- +- if (initial_value_ptr) +- stack.push_back(*initial_value_ptr); +- +- lldb::offset_t offset = 0; +- Value tmp; +- uint32_t reg_num; +- +- /// Insertion point for evaluating multi-piece expression. +- uint64_t op_piece_offset = 0; +- Value pieces; // Used for DW_OP_piece +- +- Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS)); +- // A generic type is "an integral type that has the size of an address and an +- // unspecified signedness". For now, just use the signedness of the operand. +- // TODO: Implement a real typed stack, and store the genericness of the value +- // there. +- auto to_generic = [&](auto v) { +- bool is_signed = std::is_signed::value; +- return Scalar(llvm::APSInt( +- llvm::APInt(8 * opcodes.GetAddressByteSize(), v, is_signed), +- !is_signed)); +- }; +- +- // The default kind is a memory location. This is updated by any +- // operation that changes this, such as DW_OP_stack_value, and reset +- // by composition operations like DW_OP_piece. +- LocationDescriptionKind dwarf4_location_description_kind = Memory; +- +- while (opcodes.ValidOffset(offset)) { +- const lldb::offset_t op_offset = offset; +- const uint8_t op = opcodes.GetU8(&offset); +- +- if (log && log->GetVerbose()) { +- size_t count = stack.size(); +- LLDB_LOGF(log, "Stack before operation has %" PRIu64 " values:", +- (uint64_t)count); +- for (size_t i = 0; i < count; ++i) { +- StreamString new_value; +- new_value.Printf("[%" PRIu64 "]", (uint64_t)i); +- stack[i].Dump(&new_value); +- LLDB_LOGF(log, " %s", new_value.GetData()); +- } +- LLDB_LOGF(log, "0x%8.8" PRIx64 ": %s", op_offset, +- DW_OP_value_to_name(op)); +- } +- +- switch (op) { +- // The DW_OP_addr operation has a single operand that encodes a machine +- // address and whose size is the size of an address on the target machine. +- case DW_OP_addr: +- stack.push_back(Scalar(opcodes.GetAddress(&offset))); +- stack.back().SetValueType(Value::ValueType::FileAddress); +- // Convert the file address to a load address, so subsequent +- // DWARF operators can operate on it. +- if (frame) +- stack.back().ConvertToLoadAddress(module_sp.get(), +- frame->CalculateTarget().get()); +- break; +- +- // The DW_OP_addr_sect_offset4 is used for any location expressions in +- // shared libraries that have a location like: +- // DW_OP_addr(0x1000) +- // If this address resides in a shared library, then this virtual address +- // won't make sense when it is evaluated in the context of a running +- // process where shared libraries have been slid. To account for this, this +- // new address type where we can store the section pointer and a 4 byte +- // offset. +- // case DW_OP_addr_sect_offset4: +- // { +- // result_type = eResultTypeFileAddress; +- // lldb::Section *sect = (lldb::Section +- // *)opcodes.GetMaxU64(&offset, sizeof(void *)); +- // lldb::addr_t sect_offset = opcodes.GetU32(&offset); +- // +- // Address so_addr (sect, sect_offset); +- // lldb::addr_t load_addr = so_addr.GetLoadAddress(); +- // if (load_addr != LLDB_INVALID_ADDRESS) +- // { +- // // We successfully resolve a file address to a load +- // // address. +- // stack.push_back(load_addr); +- // break; +- // } +- // else +- // { +- // // We were able +- // if (error_ptr) +- // error_ptr->SetErrorStringWithFormat ("Section %s in +- // %s is not currently loaded.\n", +- // sect->GetName().AsCString(), +- // sect->GetModule()->GetFileSpec().GetFilename().AsCString()); +- // return false; +- // } +- // } +- // break; +- +- // OPCODE: DW_OP_deref +- // OPERANDS: none +- // DESCRIPTION: Pops the top stack entry and treats it as an address. +- // The value retrieved from that address is pushed. The size of the data +- // retrieved from the dereferenced address is the size of an address on the +- // target machine. +- case DW_OP_deref: { +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString("Expression stack empty for DW_OP_deref."); +- return false; +- } +- Value::ValueType value_type = stack.back().GetValueType(); +- switch (value_type) { +- case Value::ValueType::HostAddress: { +- void *src = (void *)stack.back().GetScalar().ULongLong(); +- intptr_t ptr; +- ::memcpy(&ptr, src, sizeof(void *)); +- stack.back().GetScalar() = ptr; +- stack.back().ClearContext(); +- } break; +- case Value::ValueType::FileAddress: { +- auto file_addr = stack.back().GetScalar().ULongLong( +- LLDB_INVALID_ADDRESS); +- if (!module_sp) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "need module to resolve file address for DW_OP_deref"); +- return false; +- } +- Address so_addr; +- if (!module_sp->ResolveFileAddress(file_addr, so_addr)) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "failed to resolve file address in module"); +- return false; +- } +- addr_t load_Addr = so_addr.GetLoadAddress(exe_ctx->GetTargetPtr()); +- if (load_Addr == LLDB_INVALID_ADDRESS) { +- if (error_ptr) +- error_ptr->SetErrorString("failed to resolve load address"); +- return false; +- } +- stack.back().GetScalar() = load_Addr; +- // Fall through to load address promotion code below. +- } LLVM_FALLTHROUGH; +- case Value::ValueType::Scalar: +- // Promote Scalar to LoadAddress and fall through. +- stack.back().SetValueType(Value::ValueType::LoadAddress); +- LLVM_FALLTHROUGH; +- case Value::ValueType::LoadAddress: +- if (exe_ctx) { +- if (process) { +- lldb::addr_t pointer_addr = +- stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); +- Status error; +- lldb::addr_t pointer_value = +- process->ReadPointerFromMemory(pointer_addr, error); +- if (pointer_value != LLDB_INVALID_ADDRESS) { +- if (ABISP abi_sp = process->GetABI()) +- pointer_value = abi_sp->FixCodeAddress(pointer_value); +- stack.back().GetScalar() = pointer_value; +- stack.back().ClearContext(); +- } else { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "Failed to dereference pointer from 0x%" PRIx64 +- " for DW_OP_deref: %s\n", +- pointer_addr, error.AsCString()); +- return false; +- } +- } else { +- if (error_ptr) +- error_ptr->SetErrorString("NULL process for DW_OP_deref.\n"); +- return false; +- } +- } else { +- if (error_ptr) +- error_ptr->SetErrorString( +- "NULL execution context for DW_OP_deref.\n"); +- return false; +- } +- break; +- +- case Value::ValueType::Invalid: +- if (error_ptr) +- error_ptr->SetErrorString("Invalid value type for DW_OP_deref.\n"); +- return false; +- } +- +- } break; +- +- // OPCODE: DW_OP_deref_size +- // OPERANDS: 1 +- // 1 - uint8_t that specifies the size of the data to dereference. +- // DESCRIPTION: Behaves like the DW_OP_deref operation: it pops the top +- // stack entry and treats it as an address. The value retrieved from that +- // address is pushed. In the DW_OP_deref_size operation, however, the size +- // in bytes of the data retrieved from the dereferenced address is +- // specified by the single operand. This operand is a 1-byte unsigned +- // integral constant whose value may not be larger than the size of an +- // address on the target machine. The data retrieved is zero extended to +- // the size of an address on the target machine before being pushed on the +- // expression stack. +- case DW_OP_deref_size: { +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack empty for DW_OP_deref_size."); +- return false; +- } +- uint8_t size = opcodes.GetU8(&offset); +- Value::ValueType value_type = stack.back().GetValueType(); +- switch (value_type) { +- case Value::ValueType::HostAddress: { +- void *src = (void *)stack.back().GetScalar().ULongLong(); +- intptr_t ptr; +- ::memcpy(&ptr, src, sizeof(void *)); +- // I can't decide whether the size operand should apply to the bytes in +- // their +- // lldb-host endianness or the target endianness.. I doubt this'll ever +- // come up but I'll opt for assuming big endian regardless. +- switch (size) { +- case 1: +- ptr = ptr & 0xff; +- break; +- case 2: +- ptr = ptr & 0xffff; +- break; +- case 3: +- ptr = ptr & 0xffffff; +- break; +- case 4: +- ptr = ptr & 0xffffffff; +- break; +- // the casts are added to work around the case where intptr_t is a 32 +- // bit quantity; +- // presumably we won't hit the 5..7 cases if (void*) is 32-bits in this +- // program. +- case 5: +- ptr = (intptr_t)ptr & 0xffffffffffULL; +- break; +- case 6: +- ptr = (intptr_t)ptr & 0xffffffffffffULL; +- break; +- case 7: +- ptr = (intptr_t)ptr & 0xffffffffffffffULL; +- break; +- default: +- break; +- } +- stack.back().GetScalar() = ptr; +- stack.back().ClearContext(); +- } break; +- case Value::ValueType::Scalar: +- case Value::ValueType::LoadAddress: +- if (exe_ctx) { +- if (process) { +- lldb::addr_t pointer_addr = +- stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); +- uint8_t addr_bytes[sizeof(lldb::addr_t)]; +- Status error; +- if (process->ReadMemory(pointer_addr, &addr_bytes, size, error) == +- size) { +- DataExtractor addr_data(addr_bytes, sizeof(addr_bytes), +- process->GetByteOrder(), size); +- lldb::offset_t addr_data_offset = 0; +- switch (size) { +- case 1: +- stack.back().GetScalar() = addr_data.GetU8(&addr_data_offset); +- break; +- case 2: +- stack.back().GetScalar() = addr_data.GetU16(&addr_data_offset); +- break; +- case 4: +- stack.back().GetScalar() = addr_data.GetU32(&addr_data_offset); +- break; +- case 8: +- stack.back().GetScalar() = addr_data.GetU64(&addr_data_offset); +- break; +- default: +- stack.back().GetScalar() = +- addr_data.GetAddress(&addr_data_offset); +- } +- stack.back().ClearContext(); +- } else { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "Failed to dereference pointer from 0x%" PRIx64 +- " for DW_OP_deref: %s\n", +- pointer_addr, error.AsCString()); +- return false; +- } +- } else { +- if (error_ptr) +- error_ptr->SetErrorString("NULL process for DW_OP_deref_size.\n"); +- return false; +- } +- } else { +- if (error_ptr) +- error_ptr->SetErrorString( +- "NULL execution context for DW_OP_deref_size.\n"); +- return false; +- } +- break; +- +- case Value::ValueType::FileAddress: +- case Value::ValueType::Invalid: +- if (error_ptr) +- error_ptr->SetErrorString("Invalid value for DW_OP_deref_size.\n"); +- return false; +- } +- +- } break; +- +- // OPCODE: DW_OP_xderef_size +- // OPERANDS: 1 +- // 1 - uint8_t that specifies the size of the data to dereference. +- // DESCRIPTION: Behaves like the DW_OP_xderef operation: the entry at +- // the top of the stack is treated as an address. The second stack entry is +- // treated as an "address space identifier" for those architectures that +- // support multiple address spaces. The top two stack elements are popped, +- // a data item is retrieved through an implementation-defined address +- // calculation and pushed as the new stack top. In the DW_OP_xderef_size +- // operation, however, the size in bytes of the data retrieved from the +- // dereferenced address is specified by the single operand. This operand is +- // a 1-byte unsigned integral constant whose value may not be larger than +- // the size of an address on the target machine. The data retrieved is zero +- // extended to the size of an address on the target machine before being +- // pushed on the expression stack. +- case DW_OP_xderef_size: +- if (error_ptr) +- error_ptr->SetErrorString("Unimplemented opcode: DW_OP_xderef_size."); +- return false; +- // OPCODE: DW_OP_xderef +- // OPERANDS: none +- // DESCRIPTION: Provides an extended dereference mechanism. The entry at +- // the top of the stack is treated as an address. The second stack entry is +- // treated as an "address space identifier" for those architectures that +- // support multiple address spaces. The top two stack elements are popped, +- // a data item is retrieved through an implementation-defined address +- // calculation and pushed as the new stack top. The size of the data +- // retrieved from the dereferenced address is the size of an address on the +- // target machine. +- case DW_OP_xderef: +- if (error_ptr) +- error_ptr->SetErrorString("Unimplemented opcode: DW_OP_xderef."); +- return false; +- +- // All DW_OP_constXXX opcodes have a single operand as noted below: +- // +- // Opcode Operand 1 +- // DW_OP_const1u 1-byte unsigned integer constant +- // DW_OP_const1s 1-byte signed integer constant +- // DW_OP_const2u 2-byte unsigned integer constant +- // DW_OP_const2s 2-byte signed integer constant +- // DW_OP_const4u 4-byte unsigned integer constant +- // DW_OP_const4s 4-byte signed integer constant +- // DW_OP_const8u 8-byte unsigned integer constant +- // DW_OP_const8s 8-byte signed integer constant +- // DW_OP_constu unsigned LEB128 integer constant +- // DW_OP_consts signed LEB128 integer constant +- case DW_OP_const1u: +- stack.push_back(to_generic(opcodes.GetU8(&offset))); +- break; +- case DW_OP_const1s: +- stack.push_back(to_generic((int8_t)opcodes.GetU8(&offset))); +- break; +- case DW_OP_const2u: +- stack.push_back(to_generic(opcodes.GetU16(&offset))); +- break; +- case DW_OP_const2s: +- stack.push_back(to_generic((int16_t)opcodes.GetU16(&offset))); +- break; +- case DW_OP_const4u: +- stack.push_back(to_generic(opcodes.GetU32(&offset))); +- break; +- case DW_OP_const4s: +- stack.push_back(to_generic((int32_t)opcodes.GetU32(&offset))); +- break; +- case DW_OP_const8u: +- stack.push_back(to_generic(opcodes.GetU64(&offset))); +- break; +- case DW_OP_const8s: +- stack.push_back(to_generic((int64_t)opcodes.GetU64(&offset))); +- break; +- // These should also use to_generic, but we can't do that due to a +- // producer-side bug in llvm. See llvm.org/pr48087. +- case DW_OP_constu: +- stack.push_back(Scalar(opcodes.GetULEB128(&offset))); +- break; +- case DW_OP_consts: +- stack.push_back(Scalar(opcodes.GetSLEB128(&offset))); +- break; +- +- // OPCODE: DW_OP_dup +- // OPERANDS: none +- // DESCRIPTION: duplicates the value at the top of the stack +- case DW_OP_dup: +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString("Expression stack empty for DW_OP_dup."); +- return false; +- } else +- stack.push_back(stack.back()); +- break; +- +- // OPCODE: DW_OP_drop +- // OPERANDS: none +- // DESCRIPTION: pops the value at the top of the stack +- case DW_OP_drop: +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString("Expression stack empty for DW_OP_drop."); +- return false; +- } else +- stack.pop_back(); +- break; +- +- // OPCODE: DW_OP_over +- // OPERANDS: none +- // DESCRIPTION: Duplicates the entry currently second in the stack at +- // the top of the stack. +- case DW_OP_over: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_over."); +- return false; +- } else +- stack.push_back(stack[stack.size() - 2]); +- break; +- +- // OPCODE: DW_OP_pick +- // OPERANDS: uint8_t index into the current stack +- // DESCRIPTION: The stack entry with the specified index (0 through 255, +- // inclusive) is pushed on the stack +- case DW_OP_pick: { +- uint8_t pick_idx = opcodes.GetU8(&offset); +- if (pick_idx < stack.size()) +- stack.push_back(stack[stack.size() - 1 - pick_idx]); +- else { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "Index %u out of range for DW_OP_pick.\n", pick_idx); +- return false; +- } +- } break; +- +- // OPCODE: DW_OP_swap +- // OPERANDS: none +- // DESCRIPTION: swaps the top two stack entries. The entry at the top +- // of the stack becomes the second stack entry, and the second entry +- // becomes the top of the stack +- case DW_OP_swap: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_swap."); +- return false; +- } else { +- tmp = stack.back(); +- stack.back() = stack[stack.size() - 2]; +- stack[stack.size() - 2] = tmp; +- } +- break; +- +- // OPCODE: DW_OP_rot +- // OPERANDS: none +- // DESCRIPTION: Rotates the first three stack entries. The entry at +- // the top of the stack becomes the third stack entry, the second entry +- // becomes the top of the stack, and the third entry becomes the second +- // entry. +- case DW_OP_rot: +- if (stack.size() < 3) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 3 items for DW_OP_rot."); +- return false; +- } else { +- size_t last_idx = stack.size() - 1; +- Value old_top = stack[last_idx]; +- stack[last_idx] = stack[last_idx - 1]; +- stack[last_idx - 1] = stack[last_idx - 2]; +- stack[last_idx - 2] = old_top; +- } +- break; +- +- // OPCODE: DW_OP_abs +- // OPERANDS: none +- // DESCRIPTION: pops the top stack entry, interprets it as a signed +- // value and pushes its absolute value. If the absolute value can not be +- // represented, the result is undefined. +- case DW_OP_abs: +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_abs."); +- return false; +- } else if (!stack.back().ResolveValue(exe_ctx).AbsoluteValue()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Failed to take the absolute value of the first stack item."); +- return false; +- } +- break; +- +- // OPCODE: DW_OP_and +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, performs a bitwise and +- // operation on the two, and pushes the result. +- case DW_OP_and: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_and."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) & tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_div +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, divides the former second +- // entry by the former top of the stack using signed division, and pushes +- // the result. +- case DW_OP_div: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_div."); +- return false; +- } else { +- tmp = stack.back(); +- if (tmp.ResolveValue(exe_ctx).IsZero()) { +- if (error_ptr) +- error_ptr->SetErrorString("Divide by zero."); +- return false; +- } else { +- stack.pop_back(); +- stack.back() = +- stack.back().ResolveValue(exe_ctx) / tmp.ResolveValue(exe_ctx); +- if (!stack.back().ResolveValue(exe_ctx).IsValid()) { +- if (error_ptr) +- error_ptr->SetErrorString("Divide failed."); +- return false; +- } +- } +- } +- break; +- +- // OPCODE: DW_OP_minus +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, subtracts the former top +- // of the stack from the former second entry, and pushes the result. +- case DW_OP_minus: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_minus."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) - tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_mod +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values and pushes the result of +- // the calculation: former second stack entry modulo the former top of the +- // stack. +- case DW_OP_mod: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_mod."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) % tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_mul +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack entries, multiplies them +- // together, and pushes the result. +- case DW_OP_mul: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_mul."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) * tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_neg +- // OPERANDS: none +- // DESCRIPTION: pops the top stack entry, and pushes its negation. +- case DW_OP_neg: +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_neg."); +- return false; +- } else { +- if (!stack.back().ResolveValue(exe_ctx).UnaryNegate()) { +- if (error_ptr) +- error_ptr->SetErrorString("Unary negate failed."); +- return false; +- } +- } +- break; +- +- // OPCODE: DW_OP_not +- // OPERANDS: none +- // DESCRIPTION: pops the top stack entry, and pushes its bitwise +- // complement +- case DW_OP_not: +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_not."); +- return false; +- } else { +- if (!stack.back().ResolveValue(exe_ctx).OnesComplement()) { +- if (error_ptr) +- error_ptr->SetErrorString("Logical NOT failed."); +- return false; +- } +- } +- break; +- +- // OPCODE: DW_OP_or +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack entries, performs a bitwise or +- // operation on the two, and pushes the result. +- case DW_OP_or: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_or."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) | tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_plus +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack entries, adds them together, and +- // pushes the result. +- case DW_OP_plus: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_plus."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().GetScalar() += tmp.GetScalar(); +- } +- break; +- +- // OPCODE: DW_OP_plus_uconst +- // OPERANDS: none +- // DESCRIPTION: pops the top stack entry, adds it to the unsigned LEB128 +- // constant operand and pushes the result. +- case DW_OP_plus_uconst: +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_plus_uconst."); +- return false; +- } else { +- const uint64_t uconst_value = opcodes.GetULEB128(&offset); +- // Implicit conversion from a UINT to a Scalar... +- stack.back().GetScalar() += uconst_value; +- if (!stack.back().GetScalar().IsValid()) { +- if (error_ptr) +- error_ptr->SetErrorString("DW_OP_plus_uconst failed."); +- return false; +- } +- } +- break; +- +- // OPCODE: DW_OP_shl +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack entries, shifts the former +- // second entry left by the number of bits specified by the former top of +- // the stack, and pushes the result. +- case DW_OP_shl: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_shl."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) <<= tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_shr +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack entries, shifts the former second +- // entry right logically (filling with zero bits) by the number of bits +- // specified by the former top of the stack, and pushes the result. +- case DW_OP_shr: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_shr."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- if (!stack.back().ResolveValue(exe_ctx).ShiftRightLogical( +- tmp.ResolveValue(exe_ctx))) { +- if (error_ptr) +- error_ptr->SetErrorString("DW_OP_shr failed."); +- return false; +- } +- } +- break; +- +- // OPCODE: DW_OP_shra +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack entries, shifts the former second +- // entry right arithmetically (divide the magnitude by 2, keep the same +- // sign for the result) by the number of bits specified by the former top +- // of the stack, and pushes the result. +- case DW_OP_shra: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_shra."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) >>= tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_xor +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack entries, performs the bitwise +- // exclusive-or operation on the two, and pushes the result. +- case DW_OP_xor: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_xor."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) ^ tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_skip +- // OPERANDS: int16_t +- // DESCRIPTION: An unconditional branch. Its single operand is a 2-byte +- // signed integer constant. The 2-byte constant is the number of bytes of +- // the DWARF expression to skip forward or backward from the current +- // operation, beginning after the 2-byte constant. +- case DW_OP_skip: { +- int16_t skip_offset = (int16_t)opcodes.GetU16(&offset); +- lldb::offset_t new_offset = offset + skip_offset; +- if (opcodes.ValidOffset(new_offset)) +- offset = new_offset; +- else { +- if (error_ptr) +- error_ptr->SetErrorString("Invalid opcode offset in DW_OP_skip."); +- return false; +- } +- } break; +- +- // OPCODE: DW_OP_bra +- // OPERANDS: int16_t +- // DESCRIPTION: A conditional branch. Its single operand is a 2-byte +- // signed integer constant. This operation pops the top of stack. If the +- // value popped is not the constant 0, the 2-byte constant operand is the +- // number of bytes of the DWARF expression to skip forward or backward from +- // the current operation, beginning after the 2-byte constant. +- case DW_OP_bra: +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_bra."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- int16_t bra_offset = (int16_t)opcodes.GetU16(&offset); +- Scalar zero(0); +- if (tmp.ResolveValue(exe_ctx) != zero) { +- lldb::offset_t new_offset = offset + bra_offset; +- if (opcodes.ValidOffset(new_offset)) +- offset = new_offset; +- else { +- if (error_ptr) +- error_ptr->SetErrorString("Invalid opcode offset in DW_OP_bra."); +- return false; +- } +- } +- } +- break; +- +- // OPCODE: DW_OP_eq +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, compares using the +- // equals (==) operator. +- // STACK RESULT: push the constant value 1 onto the stack if the result +- // of the operation is true or the constant value 0 if the result of the +- // operation is false. +- case DW_OP_eq: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_eq."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) == tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_ge +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, compares using the +- // greater than or equal to (>=) operator. +- // STACK RESULT: push the constant value 1 onto the stack if the result +- // of the operation is true or the constant value 0 if the result of the +- // operation is false. +- case DW_OP_ge: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_ge."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) >= tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_gt +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, compares using the +- // greater than (>) operator. +- // STACK RESULT: push the constant value 1 onto the stack if the result +- // of the operation is true or the constant value 0 if the result of the +- // operation is false. +- case DW_OP_gt: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_gt."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) > tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_le +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, compares using the +- // less than or equal to (<=) operator. +- // STACK RESULT: push the constant value 1 onto the stack if the result +- // of the operation is true or the constant value 0 if the result of the +- // operation is false. +- case DW_OP_le: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_le."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) <= tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_lt +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, compares using the +- // less than (<) operator. +- // STACK RESULT: push the constant value 1 onto the stack if the result +- // of the operation is true or the constant value 0 if the result of the +- // operation is false. +- case DW_OP_lt: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_lt."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) < tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_ne +- // OPERANDS: none +- // DESCRIPTION: pops the top two stack values, compares using the +- // not equal (!=) operator. +- // STACK RESULT: push the constant value 1 onto the stack if the result +- // of the operation is true or the constant value 0 if the result of the +- // operation is false. +- case DW_OP_ne: +- if (stack.size() < 2) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 2 items for DW_OP_ne."); +- return false; +- } else { +- tmp = stack.back(); +- stack.pop_back(); +- stack.back().ResolveValue(exe_ctx) = +- stack.back().ResolveValue(exe_ctx) != tmp.ResolveValue(exe_ctx); +- } +- break; +- +- // OPCODE: DW_OP_litn +- // OPERANDS: none +- // DESCRIPTION: encode the unsigned literal values from 0 through 31. +- // STACK RESULT: push the unsigned literal constant value onto the top +- // of the stack. +- case DW_OP_lit0: +- case DW_OP_lit1: +- case DW_OP_lit2: +- case DW_OP_lit3: +- case DW_OP_lit4: +- case DW_OP_lit5: +- case DW_OP_lit6: +- case DW_OP_lit7: +- case DW_OP_lit8: +- case DW_OP_lit9: +- case DW_OP_lit10: +- case DW_OP_lit11: +- case DW_OP_lit12: +- case DW_OP_lit13: +- case DW_OP_lit14: +- case DW_OP_lit15: +- case DW_OP_lit16: +- case DW_OP_lit17: +- case DW_OP_lit18: +- case DW_OP_lit19: +- case DW_OP_lit20: +- case DW_OP_lit21: +- case DW_OP_lit22: +- case DW_OP_lit23: +- case DW_OP_lit24: +- case DW_OP_lit25: +- case DW_OP_lit26: +- case DW_OP_lit27: +- case DW_OP_lit28: +- case DW_OP_lit29: +- case DW_OP_lit30: +- case DW_OP_lit31: +- stack.push_back(to_generic(op - DW_OP_lit0)); +- break; +- +- // OPCODE: DW_OP_regN +- // OPERANDS: none +- // DESCRIPTION: Push the value in register n on the top of the stack. +- case DW_OP_reg0: +- case DW_OP_reg1: +- case DW_OP_reg2: +- case DW_OP_reg3: +- case DW_OP_reg4: +- case DW_OP_reg5: +- case DW_OP_reg6: +- case DW_OP_reg7: +- case DW_OP_reg8: +- case DW_OP_reg9: +- case DW_OP_reg10: +- case DW_OP_reg11: +- case DW_OP_reg12: +- case DW_OP_reg13: +- case DW_OP_reg14: +- case DW_OP_reg15: +- case DW_OP_reg16: +- case DW_OP_reg17: +- case DW_OP_reg18: +- case DW_OP_reg19: +- case DW_OP_reg20: +- case DW_OP_reg21: +- case DW_OP_reg22: +- case DW_OP_reg23: +- case DW_OP_reg24: +- case DW_OP_reg25: +- case DW_OP_reg26: +- case DW_OP_reg27: +- case DW_OP_reg28: +- case DW_OP_reg29: +- case DW_OP_reg30: +- case DW_OP_reg31: { +- dwarf4_location_description_kind = Register; +- reg_num = op - DW_OP_reg0; +- +- if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, error_ptr, tmp)) +- stack.push_back(tmp); +- else +- return false; +- } break; +- // OPCODE: DW_OP_regx +- // OPERANDS: +- // ULEB128 literal operand that encodes the register. +- // DESCRIPTION: Push the value in register on the top of the stack. +- case DW_OP_regx: { +- dwarf4_location_description_kind = Register; +- reg_num = opcodes.GetULEB128(&offset); +- if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, error_ptr, tmp)) +- stack.push_back(tmp); +- else +- return false; +- } break; +- +- // OPCODE: DW_OP_bregN +- // OPERANDS: +- // SLEB128 offset from register N +- // DESCRIPTION: Value is in memory at the address specified by register +- // N plus an offset. +- case DW_OP_breg0: +- case DW_OP_breg1: +- case DW_OP_breg2: +- case DW_OP_breg3: +- case DW_OP_breg4: +- case DW_OP_breg5: +- case DW_OP_breg6: +- case DW_OP_breg7: +- case DW_OP_breg8: +- case DW_OP_breg9: +- case DW_OP_breg10: +- case DW_OP_breg11: +- case DW_OP_breg12: +- case DW_OP_breg13: +- case DW_OP_breg14: +- case DW_OP_breg15: +- case DW_OP_breg16: +- case DW_OP_breg17: +- case DW_OP_breg18: +- case DW_OP_breg19: +- case DW_OP_breg20: +- case DW_OP_breg21: +- case DW_OP_breg22: +- case DW_OP_breg23: +- case DW_OP_breg24: +- case DW_OP_breg25: +- case DW_OP_breg26: +- case DW_OP_breg27: +- case DW_OP_breg28: +- case DW_OP_breg29: +- case DW_OP_breg30: +- case DW_OP_breg31: { +- reg_num = op - DW_OP_breg0; +- +- if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, error_ptr, +- tmp)) { +- int64_t breg_offset = opcodes.GetSLEB128(&offset); +- tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset; +- tmp.ClearContext(); +- stack.push_back(tmp); +- stack.back().SetValueType(Value::ValueType::LoadAddress); +- } else +- return false; +- } break; +- // OPCODE: DW_OP_bregx +- // OPERANDS: 2 +- // ULEB128 literal operand that encodes the register. +- // SLEB128 offset from register N +- // DESCRIPTION: Value is in memory at the address specified by register +- // N plus an offset. +- case DW_OP_bregx: { +- reg_num = opcodes.GetULEB128(&offset); +- +- if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, error_ptr, +- tmp)) { +- int64_t breg_offset = opcodes.GetSLEB128(&offset); +- tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset; +- tmp.ClearContext(); +- stack.push_back(tmp); +- stack.back().SetValueType(Value::ValueType::LoadAddress); +- } else +- return false; +- } break; +- +- case DW_OP_fbreg: +- if (exe_ctx) { +- if (frame) { +- Scalar value; +- if (frame->GetFrameBaseValue(value, error_ptr)) { +- int64_t fbreg_offset = opcodes.GetSLEB128(&offset); +- value += fbreg_offset; +- stack.push_back(value); +- stack.back().SetValueType(Value::ValueType::LoadAddress); +- } else +- return false; +- } else { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Invalid stack frame in context for DW_OP_fbreg opcode."); +- return false; +- } +- } else { +- if (error_ptr) +- error_ptr->SetErrorString( +- "NULL execution context for DW_OP_fbreg.\n"); +- return false; +- } +- +- break; +- +- // OPCODE: DW_OP_nop +- // OPERANDS: none +- // DESCRIPTION: A place holder. It has no effect on the location stack +- // or any of its values. +- case DW_OP_nop: +- break; +- +- // OPCODE: DW_OP_piece +- // OPERANDS: 1 +- // ULEB128: byte size of the piece +- // DESCRIPTION: The operand describes the size in bytes of the piece of +- // the object referenced by the DWARF expression whose result is at the top +- // of the stack. If the piece is located in a register, but does not occupy +- // the entire register, the placement of the piece within that register is +- // defined by the ABI. +- // +- // Many compilers store a single variable in sets of registers, or store a +- // variable partially in memory and partially in registers. DW_OP_piece +- // provides a way of describing how large a part of a variable a particular +- // DWARF expression refers to. +- case DW_OP_piece: { +- LocationDescriptionKind piece_locdesc = dwarf4_location_description_kind; +- // Reset for the next piece. +- dwarf4_location_description_kind = Memory; +- +- const uint64_t piece_byte_size = opcodes.GetULEB128(&offset); +- +- if (piece_byte_size > 0) { +- Value curr_piece; +- +- if (stack.empty()) { +- UpdateValueTypeFromLocationDescription( +- log, dwarf_cu, LocationDescriptionKind::Empty); +- // In a multi-piece expression, this means that the current piece is +- // not available. Fill with zeros for now by resizing the data and +- // appending it +- curr_piece.ResizeData(piece_byte_size); +- // Note that "0" is not a correct value for the unknown bits. +- // It would be better to also return a mask of valid bits together +- // with the expression result, so the debugger can print missing +- // members as "" or something. +- ::memset(curr_piece.GetBuffer().GetBytes(), 0, piece_byte_size); +- pieces.AppendDataToHostBuffer(curr_piece); +- } else { +- Status error; +- // Extract the current piece into "curr_piece" +- Value curr_piece_source_value(stack.back()); +- stack.pop_back(); +- UpdateValueTypeFromLocationDescription(log, dwarf_cu, piece_locdesc, +- &curr_piece_source_value); +- +- const Value::ValueType curr_piece_source_value_type = +- curr_piece_source_value.GetValueType(); +- switch (curr_piece_source_value_type) { +- case Value::ValueType::Invalid: +- return false; +- case Value::ValueType::LoadAddress: +- if (process) { +- if (curr_piece.ResizeData(piece_byte_size) == piece_byte_size) { +- lldb::addr_t load_addr = +- curr_piece_source_value.GetScalar().ULongLong( +- LLDB_INVALID_ADDRESS); +- if (process->ReadMemory( +- load_addr, curr_piece.GetBuffer().GetBytes(), +- piece_byte_size, error) != piece_byte_size) { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "failed to read memory DW_OP_piece(%" PRIu64 +- ") from 0x%" PRIx64, +- piece_byte_size, load_addr); +- return false; +- } +- } else { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "failed to resize the piece memory buffer for " +- "DW_OP_piece(%" PRIu64 ")", +- piece_byte_size); +- return false; +- } +- } +- break; +- +- case Value::ValueType::FileAddress: +- case Value::ValueType::HostAddress: +- if (error_ptr) { +- lldb::addr_t addr = curr_piece_source_value.GetScalar().ULongLong( +- LLDB_INVALID_ADDRESS); +- error_ptr->SetErrorStringWithFormat( +- "failed to read memory DW_OP_piece(%" PRIu64 +- ") from %s address 0x%" PRIx64, +- piece_byte_size, curr_piece_source_value.GetValueType() == +- Value::ValueType::FileAddress +- ? "file" +- : "host", +- addr); +- } +- return false; +- +- case Value::ValueType::Scalar: { +- uint32_t bit_size = piece_byte_size * 8; +- uint32_t bit_offset = 0; +- Scalar &scalar = curr_piece_source_value.GetScalar(); +- if (!scalar.ExtractBitfield( +- bit_size, bit_offset)) { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "unable to extract %" PRIu64 " bytes from a %" PRIu64 +- " byte scalar value.", +- piece_byte_size, +- (uint64_t)curr_piece_source_value.GetScalar() +- .GetByteSize()); +- return false; +- } +- // Create curr_piece with bit_size. By default Scalar +- // grows to the nearest host integer type. +- llvm::APInt fail_value(1, 0, false); +- llvm::APInt ap_int = scalar.UInt128(fail_value); +- assert(ap_int.getBitWidth() >= bit_size); +- llvm::ArrayRef buf{ap_int.getRawData(), +- ap_int.getNumWords()}; +- curr_piece.GetScalar() = Scalar(llvm::APInt(bit_size, buf)); +- } break; +- } +- +- // Check if this is the first piece? +- if (op_piece_offset == 0) { +- // This is the first piece, we should push it back onto the stack +- // so subsequent pieces will be able to access this piece and add +- // to it. +- if (pieces.AppendDataToHostBuffer(curr_piece) == 0) { +- if (error_ptr) +- error_ptr->SetErrorString("failed to append piece data"); +- return false; +- } +- } else { +- // If this is the second or later piece there should be a value on +- // the stack. +- if (pieces.GetBuffer().GetByteSize() != op_piece_offset) { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "DW_OP_piece for offset %" PRIu64 +- " but top of stack is of size %" PRIu64, +- op_piece_offset, pieces.GetBuffer().GetByteSize()); +- return false; +- } +- +- if (pieces.AppendDataToHostBuffer(curr_piece) == 0) { +- if (error_ptr) +- error_ptr->SetErrorString("failed to append piece data"); +- return false; +- } +- } +- } +- op_piece_offset += piece_byte_size; +- } +- } break; +- +- case DW_OP_bit_piece: // 0x9d ULEB128 bit size, ULEB128 bit offset (DWARF3); +- if (stack.size() < 1) { +- UpdateValueTypeFromLocationDescription(log, dwarf_cu, +- LocationDescriptionKind::Empty); +- // Reset for the next piece. +- dwarf4_location_description_kind = Memory; +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_bit_piece."); +- return false; +- } else { +- UpdateValueTypeFromLocationDescription( +- log, dwarf_cu, dwarf4_location_description_kind, &stack.back()); +- // Reset for the next piece. +- dwarf4_location_description_kind = Memory; +- const uint64_t piece_bit_size = opcodes.GetULEB128(&offset); +- const uint64_t piece_bit_offset = opcodes.GetULEB128(&offset); +- switch (stack.back().GetValueType()) { +- case Value::ValueType::Invalid: +- return false; +- case Value::ValueType::Scalar: { +- if (!stack.back().GetScalar().ExtractBitfield(piece_bit_size, +- piece_bit_offset)) { +- if (error_ptr) +- error_ptr->SetErrorStringWithFormat( +- "unable to extract %" PRIu64 " bit value with %" PRIu64 +- " bit offset from a %" PRIu64 " bit scalar value.", +- piece_bit_size, piece_bit_offset, +- (uint64_t)(stack.back().GetScalar().GetByteSize() * 8)); +- return false; +- } +- } break; +- +- case Value::ValueType::FileAddress: +- case Value::ValueType::LoadAddress: +- case Value::ValueType::HostAddress: +- if (error_ptr) { +- error_ptr->SetErrorStringWithFormat( +- "unable to extract DW_OP_bit_piece(bit_size = %" PRIu64 +- ", bit_offset = %" PRIu64 ") from an address value.", +- piece_bit_size, piece_bit_offset); +- } +- return false; +- } +- } +- break; +- +- // OPCODE: DW_OP_implicit_value +- // OPERANDS: 2 +- // ULEB128 size of the value block in bytes +- // uint8_t* block bytes encoding value in target's memory +- // representation +- // DESCRIPTION: Value is immediately stored in block in the debug info with +- // the memory representation of the target. +- case DW_OP_implicit_value: { +- dwarf4_location_description_kind = Implicit; +- +- const uint32_t len = opcodes.GetULEB128(&offset); +- const void *data = opcodes.GetData(&offset, len); +- +- if (!data) { +- LLDB_LOG(log, "Evaluate_DW_OP_implicit_value: could not be read data"); +- LLDB_ERRORF(error_ptr, "Could not evaluate %s.", +- DW_OP_value_to_name(op)); +- return false; +- } +- +- Value result(data, len); +- stack.push_back(result); +- break; +- } +- +- case DW_OP_implicit_pointer: { +- dwarf4_location_description_kind = Implicit; +- LLDB_ERRORF(error_ptr, "Could not evaluate %s.", DW_OP_value_to_name(op)); +- return false; +- } +- +- // OPCODE: DW_OP_push_object_address +- // OPERANDS: none +- // DESCRIPTION: Pushes the address of the object currently being +- // evaluated as part of evaluation of a user presented expression. This +- // object may correspond to an independent variable described by its own +- // DIE or it may be a component of an array, structure, or class whose +- // address has been dynamically determined by an earlier step during user +- // expression evaluation. +- case DW_OP_push_object_address: +- if (object_address_ptr) +- stack.push_back(*object_address_ptr); +- else { +- if (error_ptr) +- error_ptr->SetErrorString("DW_OP_push_object_address used without " +- "specifying an object address"); +- return false; +- } +- break; +- +- // OPCODE: DW_OP_call2 +- // OPERANDS: +- // uint16_t compile unit relative offset of a DIE +- // DESCRIPTION: Performs subroutine calls during evaluation +- // of a DWARF expression. The operand is the 2-byte unsigned offset of a +- // debugging information entry in the current compilation unit. +- // +- // Operand interpretation is exactly like that for DW_FORM_ref2. +- // +- // This operation transfers control of DWARF expression evaluation to the +- // DW_AT_location attribute of the referenced DIE. If there is no such +- // attribute, then there is no effect. Execution of the DWARF expression of +- // a DW_AT_location attribute may add to and/or remove from values on the +- // stack. Execution returns to the point following the call when the end of +- // the attribute is reached. Values on the stack at the time of the call +- // may be used as parameters by the called expression and values left on +- // the stack by the called expression may be used as return values by prior +- // agreement between the calling and called expressions. +- case DW_OP_call2: +- if (error_ptr) +- error_ptr->SetErrorString("Unimplemented opcode DW_OP_call2."); +- return false; +- // OPCODE: DW_OP_call4 +- // OPERANDS: 1 +- // uint32_t compile unit relative offset of a DIE +- // DESCRIPTION: Performs a subroutine call during evaluation of a DWARF +- // expression. For DW_OP_call4, the operand is a 4-byte unsigned offset of +- // a debugging information entry in the current compilation unit. +- // +- // Operand interpretation DW_OP_call4 is exactly like that for +- // DW_FORM_ref4. +- // +- // This operation transfers control of DWARF expression evaluation to the +- // DW_AT_location attribute of the referenced DIE. If there is no such +- // attribute, then there is no effect. Execution of the DWARF expression of +- // a DW_AT_location attribute may add to and/or remove from values on the +- // stack. Execution returns to the point following the call when the end of +- // the attribute is reached. Values on the stack at the time of the call +- // may be used as parameters by the called expression and values left on +- // the stack by the called expression may be used as return values by prior +- // agreement between the calling and called expressions. +- case DW_OP_call4: +- if (error_ptr) +- error_ptr->SetErrorString("Unimplemented opcode DW_OP_call4."); +- return false; +- +- // OPCODE: DW_OP_stack_value +- // OPERANDS: None +- // DESCRIPTION: Specifies that the object does not exist in memory but +- // rather is a constant value. The value from the top of the stack is the +- // value to be used. This is the actual object value and not the location. +- case DW_OP_stack_value: +- dwarf4_location_description_kind = Implicit; +- if (stack.empty()) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_stack_value."); +- return false; +- } +- stack.back().SetValueType(Value::ValueType::Scalar); +- break; +- +- // OPCODE: DW_OP_convert +- // OPERANDS: 1 +- // A ULEB128 that is either a DIE offset of a +- // DW_TAG_base_type or 0 for the generic (pointer-sized) type. +- // +- // DESCRIPTION: Pop the top stack element, convert it to a +- // different type, and push the result. +- case DW_OP_convert: { +- if (stack.size() < 1) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "Expression stack needs at least 1 item for DW_OP_convert."); +- return false; +- } +- const uint64_t die_offset = opcodes.GetULEB128(&offset); +- uint64_t bit_size; +- bool sign; +- if (die_offset == 0) { +- // The generic type has the size of an address on the target +- // machine and an unspecified signedness. Scalar has no +- // "unspecified signedness", so we use unsigned types. +- if (!module_sp) { +- if (error_ptr) +- error_ptr->SetErrorString("No module"); +- return false; +- } +- sign = false; +- bit_size = module_sp->GetArchitecture().GetAddressByteSize() * 8; +- if (!bit_size) { +- if (error_ptr) +- error_ptr->SetErrorString("unspecified architecture"); +- return false; +- } +- } else { +- // Retrieve the type DIE that the value is being converted to. +- // FIXME: the constness has annoying ripple effects. +- DWARFDIE die = const_cast(dwarf_cu)->GetDIE(die_offset); +- if (!die) { +- if (error_ptr) +- error_ptr->SetErrorString("Cannot resolve DW_OP_convert type DIE"); +- return false; +- } +- uint64_t encoding = +- die.GetAttributeValueAsUnsigned(DW_AT_encoding, DW_ATE_hi_user); +- bit_size = die.GetAttributeValueAsUnsigned(DW_AT_byte_size, 0) * 8; +- if (!bit_size) +- bit_size = die.GetAttributeValueAsUnsigned(DW_AT_bit_size, 0); +- if (!bit_size) { +- if (error_ptr) +- error_ptr->SetErrorString("Unsupported type size in DW_OP_convert"); +- return false; +- } +- switch (encoding) { +- case DW_ATE_signed: +- case DW_ATE_signed_char: +- sign = true; +- break; +- case DW_ATE_unsigned: +- case DW_ATE_unsigned_char: +- sign = false; +- break; +- default: +- if (error_ptr) +- error_ptr->SetErrorString("Unsupported encoding in DW_OP_convert"); +- return false; +- } +- } +- Scalar &top = stack.back().ResolveValue(exe_ctx); +- top.TruncOrExtendTo(bit_size, sign); +- break; +- } +- +- // OPCODE: DW_OP_call_frame_cfa +- // OPERANDS: None +- // DESCRIPTION: Specifies a DWARF expression that pushes the value of +- // the canonical frame address consistent with the call frame information +- // located in .debug_frame (or in the FDEs of the eh_frame section). +- case DW_OP_call_frame_cfa: +- if (frame) { +- // Note that we don't have to parse FDEs because this DWARF expression +- // is commonly evaluated with a valid stack frame. +- StackID id = frame->GetStackID(); +- addr_t cfa = id.GetCallFrameAddress(); +- if (cfa != LLDB_INVALID_ADDRESS) { +- stack.push_back(Scalar(cfa)); +- stack.back().SetValueType(Value::ValueType::LoadAddress); +- } else if (error_ptr) +- error_ptr->SetErrorString("Stack frame does not include a canonical " +- "frame address for DW_OP_call_frame_cfa " +- "opcode."); +- } else { +- if (error_ptr) +- error_ptr->SetErrorString("Invalid stack frame in context for " +- "DW_OP_call_frame_cfa opcode."); +- return false; +- } +- break; +- +- // OPCODE: DW_OP_form_tls_address (or the old pre-DWARFv3 vendor extension +- // opcode, DW_OP_GNU_push_tls_address) +- // OPERANDS: none +- // DESCRIPTION: Pops a TLS offset from the stack, converts it to +- // an address in the current thread's thread-local storage block, and +- // pushes it on the stack. +- case DW_OP_form_tls_address: +- case DW_OP_GNU_push_tls_address: { +- if (stack.size() < 1) { +- if (error_ptr) { +- if (op == DW_OP_form_tls_address) +- error_ptr->SetErrorString( +- "DW_OP_form_tls_address needs an argument."); +- else +- error_ptr->SetErrorString( +- "DW_OP_GNU_push_tls_address needs an argument."); +- } +- return false; +- } +- +- if (!exe_ctx || !module_sp) { +- if (error_ptr) +- error_ptr->SetErrorString("No context to evaluate TLS within."); +- return false; +- } +- +- Thread *thread = exe_ctx->GetThreadPtr(); +- if (!thread) { +- if (error_ptr) +- error_ptr->SetErrorString("No thread to evaluate TLS within."); +- return false; +- } +- +- // Lookup the TLS block address for this thread and module. +- const addr_t tls_file_addr = +- stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); +- const addr_t tls_load_addr = +- thread->GetThreadLocalData(module_sp, tls_file_addr); +- +- if (tls_load_addr == LLDB_INVALID_ADDRESS) { +- if (error_ptr) +- error_ptr->SetErrorString( +- "No TLS data currently exists for this thread."); +- return false; +- } +- +- stack.back().GetScalar() = tls_load_addr; +- stack.back().SetValueType(Value::ValueType::LoadAddress); +- } break; +- +- // OPCODE: DW_OP_addrx (DW_OP_GNU_addr_index is the legacy name.) +- // OPERANDS: 1 +- // ULEB128: index to the .debug_addr section +- // DESCRIPTION: Pushes an address to the stack from the .debug_addr +- // section with the base address specified by the DW_AT_addr_base attribute +- // and the 0 based index is the ULEB128 encoded index. +- case DW_OP_addrx: +- case DW_OP_GNU_addr_index: { +- if (!dwarf_cu) { +- if (error_ptr) +- error_ptr->SetErrorString("DW_OP_GNU_addr_index found without a " +- "compile unit being specified"); +- return false; +- } +- uint64_t index = opcodes.GetULEB128(&offset); +- lldb::addr_t value = ReadAddressFromDebugAddrSection(dwarf_cu, index); +- stack.push_back(Scalar(value)); +- stack.back().SetValueType(Value::ValueType::FileAddress); +- } break; +- +- // OPCODE: DW_OP_GNU_const_index +- // OPERANDS: 1 +- // ULEB128: index to the .debug_addr section +- // DESCRIPTION: Pushes an constant with the size of a machine address to +- // the stack from the .debug_addr section with the base address specified +- // by the DW_AT_addr_base attribute and the 0 based index is the ULEB128 +- // encoded index. +- case DW_OP_GNU_const_index: { +- if (!dwarf_cu) { +- if (error_ptr) +- error_ptr->SetErrorString("DW_OP_GNU_const_index found without a " +- "compile unit being specified"); +- return false; +- } +- uint64_t index = opcodes.GetULEB128(&offset); +- lldb::addr_t value = ReadAddressFromDebugAddrSection(dwarf_cu, index); +- stack.push_back(Scalar(value)); +- } break; +- +- case DW_OP_GNU_entry_value: +- case DW_OP_entry_value: { +- if (!Evaluate_DW_OP_entry_value(stack, exe_ctx, reg_ctx, opcodes, offset, +- error_ptr, log)) { +- LLDB_ERRORF(error_ptr, "Could not evaluate %s.", +- DW_OP_value_to_name(op)); +- return false; +- } +- break; +- } +- +- default: +- if (error_ptr) +- error_ptr->SetErrorStringWithFormatv( +- "Unhandled opcode {0} in DWARFExpression", LocationAtom(op)); +- return false; +- } +- } +- +- if (stack.empty()) { +- // Nothing on the stack, check if we created a piece value from DW_OP_piece +- // or DW_OP_bit_piece opcodes +- if (pieces.GetBuffer().GetByteSize()) { +- result = pieces; +- return true; +- } +- if (error_ptr) +- error_ptr->SetErrorString("Stack empty after evaluation."); +- return false; +- } +- +- UpdateValueTypeFromLocationDescription( +- log, dwarf_cu, dwarf4_location_description_kind, &stack.back()); +- +- if (log && log->GetVerbose()) { +- size_t count = stack.size(); +- LLDB_LOGF(log, +- "Stack after operation has %" PRIu64 " values:", (uint64_t)count); +- for (size_t i = 0; i < count; ++i) { +- StreamString new_value; +- new_value.Printf("[%" PRIu64 "]", (uint64_t)i); +- stack[i].Dump(&new_value); +- LLDB_LOGF(log, " %s", new_value.GetData()); +- } +- } +- result = stack.back(); +- return true; // Return true on success ++ DWARFExpression expr(module_sp, opcodes, dwarf_cu); ++ expr.SetRegisterKind(reg_kind); ++ ++ // Use the DWARF expression evaluator registered for this module (or ++ // DWARFEvaluator by default). ++ DWARFEvaluatorFactory *evaluator_factory = ++ module_sp->GetDWARFExpressionEvaluatorFactory(); ++ std::unique_ptr evaluator = ++ evaluator_factory->CreateDWARFEvaluator( ++ expr, exe_ctx, reg_ctx, initial_value_ptr, object_address_ptr); ++ return evaluator->Evaluate(result, error_ptr); + } + + static DataExtractor ToDataExtractor(const llvm::DWARFLocationExpression &loc, +diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp +index 00e9ccb762c3..2137a1ac8324 100644 +--- a/lldb/source/Interpreter/CommandInterpreter.cpp ++++ b/lldb/source/Interpreter/CommandInterpreter.cpp +@@ -759,6 +759,24 @@ void CommandInterpreter::LoadCommandDictionary() { + } + } + ++ std::unique_ptr connect_wasm_cmd_up( ++ new CommandObjectRegexCommand( ++ *this, "wasm", ++ "Connect to a WebAssembly process via remote GDB server. " ++ "If no host is specifed, localhost is assumed.", ++ "wasm [:]", 2, 0, false)); ++ if (connect_wasm_cmd_up) { ++ if (connect_wasm_cmd_up->AddRegexCommand( ++ "^([^:]+|\\[[0-9a-fA-F:]+.*\\]):([0-9]+)$", ++ "process connect --plugin wasm connect://%1:%2") && ++ connect_wasm_cmd_up->AddRegexCommand( ++ "^([[:digit:]]+)$", ++ "process connect --plugin wasm connect://localhost:%1")) { ++ CommandObjectSP command_sp(connect_wasm_cmd_up.release()); ++ m_command_dict[std::string(command_sp->GetCommandName())] = command_sp; ++ } ++ } ++ + std::unique_ptr connect_kdp_remote_cmd_up( + new CommandObjectRegexCommand( + *this, "kdp-remote", +diff --git a/lldb/source/Plugins/CMakeLists.txt b/lldb/source/Plugins/CMakeLists.txt +index 9181a4e47675..2be6ec3657c0 100644 +--- a/lldb/source/Plugins/CMakeLists.txt ++++ b/lldb/source/Plugins/CMakeLists.txt +@@ -2,6 +2,7 @@ add_subdirectory(ABI) + add_subdirectory(Architecture) + add_subdirectory(Disassembler) + add_subdirectory(DynamicLoader) ++add_subdirectory(DWARFEvaluator) + add_subdirectory(ExpressionParser) + add_subdirectory(Instruction) + add_subdirectory(InstrumentationRuntime) +@@ -32,6 +33,7 @@ set(LLDB_ENUM_PLUGINS "") + # FIXME: ProcessWindowsCommon needs to be initialized after all other process + # plugins but before ProcessGDBRemote. + set(LLDB_PROCESS_WINDOWS_PLUGIN "") ++set(LLDB_PROCESS_WASM_PLUGIN "") + set(LLDB_PROCESS_GDB_PLUGIN "") + + foreach(p ${LLDB_ALL_PLUGINS}) +@@ -43,6 +45,8 @@ foreach(p ${LLDB_ALL_PLUGINS}) + set(LLDB_PROCESS_WINDOWS_PLUGIN "LLDB_PLUGIN(${pStripped})\n") + elseif(${pStripped} STREQUAL "ProcessGDBRemote") + set(LLDB_PROCESS_GDB_PLUGIN "LLDB_PLUGIN(${pStripped})\n") ++ elseif(${pStripped} STREQUAL "ProcessWasm") ++ set(LLDB_PROCESS_WASM_PLUGIN "LLDB_PLUGIN(${pStripped})\n") + else() + set(LLDB_ENUM_PLUGINS "${LLDB_ENUM_PLUGINS}LLDB_PLUGIN(${pStripped})\n") + endif() +diff --git a/lldb/source/Plugins/DWARFEvaluator/CMakeLists.txt b/lldb/source/Plugins/DWARFEvaluator/CMakeLists.txt +new file mode 100644 +index 000000000000..73fad41e1a72 +--- /dev/null ++++ b/lldb/source/Plugins/DWARFEvaluator/CMakeLists.txt +@@ -0,0 +1 @@ ++add_subdirectory(wasm) +diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/CMakeLists.txt b/lldb/source/Plugins/DWARFEvaluator/wasm/CMakeLists.txt +new file mode 100644 +index 000000000000..e50b1bef7e69 +--- /dev/null ++++ b/lldb/source/Plugins/DWARFEvaluator/wasm/CMakeLists.txt +@@ -0,0 +1,10 @@ ++add_lldb_library(lldbPluginWasmDWARFEvaluatorFactory PLUGIN ++ WasmDWARFEvaluator.cpp ++ WasmDWARFEvaluatorFactory.cpp ++ ++ LINK_LIBS ++ lldbCore ++ lldbHost ++ lldbSymbol ++ lldbPluginObjectFileWasm ++ ) +diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.cpp b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.cpp +new file mode 100644 +index 000000000000..fdda1991d19f +--- /dev/null ++++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.cpp +@@ -0,0 +1,126 @@ ++//===-- WasmDWARFEvaluator.cpp --------------------------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#include "WasmDWARFEvaluator.h" ++ ++#include "Plugins/ObjectFile/wasm/ObjectFileWasm.h" ++#include "Plugins/Process/wasm/ProcessWasm.h" ++#include "lldb/Core/Module.h" ++#include "lldb/Core/PluginManager.h" ++#include "lldb/Core/Value.h" ++#include "lldb/Core/dwarf.h" ++#include "lldb/Expression/DWARFExpression.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++using namespace lldb_private::wasm; ++ ++bool WasmDWARFEvaluator::Evaluate(const uint8_t op, Process *process, ++ StackFrame *frame, std::vector &stack, ++ const DataExtractor &opcodes, ++ lldb::offset_t &offset, Value &pieces, ++ uint64_t &op_piece_offset, Log *log, ++ Status *error_ptr) { ++ lldb::ModuleSP module_sp = m_dwarf_expression.GetModule(); ++ ++ switch (op) { ++ case DW_OP_WASM_location: { ++ if (frame) { ++ const llvm::Triple::ArchType machine = ++ frame->CalculateTarget()->GetArchitecture().GetMachine(); ++ if (machine != llvm::Triple::wasm32) { ++ if (error_ptr) ++ error_ptr->SetErrorString("Invalid target architecture for " ++ "DW_OP_WASM_location opcode."); ++ return false; ++ } ++ ++ ProcessWasm *wasm_process = ++ static_cast(frame->CalculateProcess().get()); ++ int frame_index = frame->GetConcreteFrameIndex(); ++ uint64_t wasm_op = opcodes.GetULEB128(&offset); ++ uint64_t index = opcodes.GetULEB128(&offset); ++ uint8_t buf[16]; ++ size_t size = 0; ++ switch (wasm_op) { ++ case 0: // Local ++ if (!wasm_process->GetWasmLocal(frame_index, index, buf, 16, size)) { ++ return false; ++ } ++ break; ++ case 1: // Global ++ if (!wasm_process->GetWasmGlobal(frame_index, index, buf, 16, size)) { ++ return false; ++ } ++ break; ++ case 2: // Operand Stack ++ if (!wasm_process->GetWasmStackValue(frame_index, index, buf, 16, ++ size)) { ++ return false; ++ } ++ break; ++ default: ++ return false; ++ } ++ ++ if (size == sizeof(uint32_t)) { ++ uint32_t value; ++ memcpy(&value, buf, size); ++ stack.push_back(Scalar(value)); ++ } else if (size == sizeof(uint64_t)) { ++ uint64_t value; ++ memcpy(&value, buf, size); ++ stack.push_back(Scalar(value)); ++ } else ++ return false; ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorString("Invalid stack frame in context for " ++ "DW_OP_WASM_location opcode."); ++ return false; ++ } ++ } break; ++ ++ case DW_OP_addr: { ++ /// {addr} is an offset in the module Data section. ++ lldb::addr_t addr = opcodes.GetAddress(&offset); ++ stack.push_back(Scalar(addr)); ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ } break; ++ ++ case DW_OP_fbreg: ++ if (m_exe_ctx) { ++ if (frame) { ++ Scalar value; ++ if (frame->GetFrameBaseValue(value, error_ptr)) { ++ // The value is an address in the Wasm Memory space. ++ int64_t fbreg_offset = opcodes.GetSLEB128(&offset); ++ stack.push_back(Scalar(value.ULong() + fbreg_offset)); ++ stack.back().SetValueType(Value::ValueType::LoadAddress); ++ } else ++ return false; ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorString( ++ "Invalid stack frame in context for DW_OP_fbreg opcode."); ++ return false; ++ } ++ } else { ++ if (error_ptr) ++ error_ptr->SetErrorStringWithFormat( ++ "NULL execution context for DW_OP_fbreg.\n"); ++ return false; ++ } ++ break; ++ ++ default: ++ return DWARFEvaluator::Evaluate(op, process, frame, stack, opcodes, offset, ++ pieces, op_piece_offset, log, error_ptr); ++ } ++ return true; ++} +diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.h b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.h +new file mode 100644 +index 000000000000..a01159064a39 +--- /dev/null ++++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.h +@@ -0,0 +1,47 @@ ++//===-- WasmDWARFEvaluator.h ------------------------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATOR_H ++#define LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATOR_H ++ ++#include "lldb/Expression/DWARFEvaluator.h" ++#include "lldb/lldb-private.h" ++ ++namespace lldb_private { ++namespace wasm { ++ ++/// \class WasmDWARFEvaluator evaluates DWARF expressions in the context of a ++/// WebAssembly process. ++/// ++class WasmDWARFEvaluator : public DWARFEvaluator { ++public: ++ WasmDWARFEvaluator(const DWARFExpression &dwarf_expression, ++ ExecutionContext *exe_ctx, RegisterContext *reg_ctx, ++ const Value *initial_value_ptr, ++ const Value *object_address_ptr) ++ : DWARFEvaluator(dwarf_expression, exe_ctx, reg_ctx, initial_value_ptr, ++ object_address_ptr) {} ++ ++ /// DWARFEvaluator protocol. ++ /// \{ ++ bool Evaluate(const uint8_t op, Process *process, StackFrame *frame, ++ std::vector &stack, const DataExtractor &opcodes, ++ lldb::offset_t &offset, Value &pieces, ++ uint64_t &op_piece_offset, Log *log, ++ Status *error_ptr) override; ++ /// \} ++ ++private: ++ WasmDWARFEvaluator(const WasmDWARFEvaluator &); ++ const WasmDWARFEvaluator &operator=(const WasmDWARFEvaluator &) = delete; ++}; ++ ++} // namespace wasm ++} // namespace lldb_private ++ ++#endif // LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATOR_H +diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.cpp b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.cpp +new file mode 100644 +index 000000000000..d43e96a34d37 +--- /dev/null ++++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.cpp +@@ -0,0 +1,64 @@ ++//===-- WasmDWARFEvaluatorFactory.cpp -------------------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#include "WasmDWARFEvaluatorFactory.h" ++#include "WasmDWARFEvaluator.h" ++ ++#include "Plugins/ObjectFile/wasm/ObjectFileWasm.h" ++#include "lldb/Core/Module.h" ++#include "lldb/Core/PluginManager.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++using namespace lldb_private::wasm; ++ ++LLDB_PLUGIN_DEFINE(WasmDWARFEvaluatorFactory) ++ ++void WasmDWARFEvaluatorFactory::Initialize() { ++ PluginManager::RegisterPlugin(GetPluginNameStatic(), ++ GetPluginDescriptionStatic(), CreateInstance); ++} ++ ++void WasmDWARFEvaluatorFactory::Terminate() { ++ PluginManager::UnregisterPlugin(CreateInstance); ++} ++ ++lldb_private::ConstString WasmDWARFEvaluatorFactory::GetPluginNameStatic() { ++ static ConstString g_name("WASM"); ++ return g_name; ++} ++ ++const char *WasmDWARFEvaluatorFactory::GetPluginDescriptionStatic() { ++ return "DWARF expression evaluator factory for WASM."; ++} ++ ++// CreateInstance ++// ++// Platforms can register a callback to use when creating DWARF expression ++// evaluators to allow handling platform-specific DWARF codes. ++DWARFEvaluatorFactory * ++WasmDWARFEvaluatorFactory::CreateInstance(Module *module) { ++ if (!module) ++ return nullptr; ++ ++ ObjectFileWasm *obj_file = ++ llvm::dyn_cast_or_null(module->GetObjectFile()); ++ if (!obj_file) ++ return nullptr; ++ ++ return new WasmDWARFEvaluatorFactory(); ++} ++ ++std::unique_ptr WasmDWARFEvaluatorFactory::CreateDWARFEvaluator( ++ const DWARFExpression &dwarf_expression, ExecutionContext *exe_ctx, ++ RegisterContext *reg_ctx, const Value *initial_value_ptr, ++ const Value *object_address_ptr) { ++ return std::make_unique(dwarf_expression, exe_ctx, ++ reg_ctx, initial_value_ptr, ++ object_address_ptr); ++} +diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.h b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.h +new file mode 100644 +index 000000000000..8a946592a09a +--- /dev/null ++++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.h +@@ -0,0 +1,55 @@ ++//===-- WasmDWARFEvaluatorFactory.h -----------------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATORFACTORY_H ++#define LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATORFACTORY_H ++ ++#include "lldb/Expression/DWARFEvaluatorFactory.h" ++ ++namespace lldb_private { ++namespace wasm { ++ ++/// \class WasmDWARFEvaluatorFactory creates DWARF evaluators specialized to ++/// manage DWARF-specific opcodes. ++class WasmDWARFEvaluatorFactory : public DWARFEvaluatorFactory { ++public: ++ static void Initialize(); ++ static void Terminate(); ++ static lldb_private::ConstString GetPluginNameStatic(); ++ static const char *GetPluginDescriptionStatic(); ++ ++ static lldb_private::DWARFEvaluatorFactory *CreateInstance(Module *module); ++ ++ /// PluginInterface protocol. ++ /// \{ ++ lldb_private::ConstString GetPluginName() override { ++ return GetPluginNameStatic(); ++ } ++ uint32_t GetPluginVersion() override { return 1; } ++ /// \} ++ ++ WasmDWARFEvaluatorFactory() {} ++ ++ /// DWARFEvaluatorFactory protocol. ++ /// \{ ++ std::unique_ptr ++ CreateDWARFEvaluator(const DWARFExpression &dwarf_expression, ++ ExecutionContext *exe_ctx, RegisterContext *reg_ctx, ++ const Value *initial_value_ptr, ++ const Value *object_address_ptr) override; ++ /// \} ++ ++private: ++ WasmDWARFEvaluatorFactory(const WasmDWARFEvaluatorFactory &); ++ const WasmDWARFEvaluatorFactory &operator=(const WasmDWARFEvaluatorFactory &) = delete; ++}; ++ ++} // namespace wasm ++} // namespace lldb_private ++ ++#endif // LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATORFACTORY_H +diff --git a/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp b/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp +index ae7e011eaa52..24ea75d1971c 100644 +--- a/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp ++++ b/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp +@@ -62,6 +62,15 @@ void DynamicLoaderWasmDYLD::DidAttach() { + // Ask the process for the list of loaded WebAssembly modules. + auto error = m_process->LoadModules(); + LLDB_LOG_ERROR(log, std::move(error), "Couldn't load modules: {0}"); ++ ++ // TODO: multi-modules support ? ++ Target &target = m_process->GetTarget(); ++ const ModuleList &modules = target.GetImages(); ++ ModuleSP module_sp(modules.GetModuleAtIndex(0)); ++ // module_sp is nullptr if without libxml2 ++ if(module_sp) { ++ module_sp->PreloadSymbols(); ++ } + } + + ThreadPlanSP DynamicLoaderWasmDYLD::GetStepThroughTrampolinePlan(Thread &thread, +diff --git a/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp b/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp +index 5272da9ab33a..abc5523bfd70 100644 +--- a/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp ++++ b/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp +@@ -23,6 +23,7 @@ + #include "llvm/BinaryFormat/Wasm.h" + #include "llvm/Support/Endian.h" + #include "llvm/Support/Format.h" ++#include "Plugins/Process/wasm/ProcessWasm.h" + + using namespace lldb; + using namespace lldb_private; +@@ -341,6 +342,8 @@ void ObjectFileWasm::CreateSections(SectionList &unified_section_list) { + 0, // Alignment of the section + 0, // Flags for this section. + 1)); // Number of host bytes per target byte ++ if (section_type == eSectionTypeCode) ++ section_sp->SetPermissions(ePermissionsReadable|ePermissionsExecutable); + m_sections_up->AddSection(section_sp); + unified_section_list.AddSection(section_sp); + } +@@ -367,6 +370,7 @@ bool ObjectFileWasm::SetLoadAddress(Target &target, lldb::addr_t load_address, + assert(m_memory_addr == LLDB_INVALID_ADDRESS || + m_memory_addr == load_address); + ++ lldb::addr_t adjust_addr; + ModuleSP module_sp = GetModule(); + if (!module_sp) + return false; +@@ -381,8 +385,9 @@ bool ObjectFileWasm::SetLoadAddress(Target &target, lldb::addr_t load_address, + const size_t num_sections = section_list->GetSize(); + for (size_t sect_idx = 0; sect_idx < num_sections; ++sect_idx) { + SectionSP section_sp(section_list->GetSectionAtIndex(sect_idx)); ++ adjust_addr = load_address; + if (target.SetSectionLoadAddress( +- section_sp, load_address | section_sp->GetFileOffset())) { ++ section_sp, adjust_addr | section_sp->GetFileOffset())) { + ++num_loaded_sections; + } + } +diff --git a/lldb/source/Plugins/Platform/CMakeLists.txt b/lldb/source/Plugins/Platform/CMakeLists.txt +index 5f284e517dca..6084cbc9378d 100644 +--- a/lldb/source/Plugins/Platform/CMakeLists.txt ++++ b/lldb/source/Plugins/Platform/CMakeLists.txt +@@ -15,3 +15,4 @@ + add_subdirectory(POSIX) + add_subdirectory(gdb-server) + add_subdirectory(Android) ++add_subdirectory(wasm-remote) +diff --git a/lldb/source/Plugins/Platform/wasm-remote/CMakeLists.txt b/lldb/source/Plugins/Platform/wasm-remote/CMakeLists.txt +new file mode 100644 +index 000000000000..4a65765a5659 +--- /dev/null ++++ b/lldb/source/Plugins/Platform/wasm-remote/CMakeLists.txt +@@ -0,0 +1,10 @@ ++add_lldb_library(lldbPluginPlatformWasm PLUGIN ++ PlatformRemoteWasmServer.cpp ++ ++ LINK_LIBS ++ lldbBreakpoint ++ lldbCore ++ lldbHost ++ lldbTarget ++ lldbPluginProcessUtility ++ ) +diff --git a/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.cpp b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.cpp +new file mode 100644 +index 000000000000..f26d11f00e5c +--- /dev/null ++++ b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.cpp +@@ -0,0 +1,139 @@ ++#include "PlatformRemoteWasmServer.h" ++#include "lldb/Host/Config.h" ++ ++#include "lldb/Breakpoint/BreakpointLocation.h" ++#include "lldb/Core/Debugger.h" ++#include "lldb/Core/Module.h" ++#include "lldb/Core/ModuleList.h" ++#include "lldb/Core/ModuleSpec.h" ++#include "lldb/Core/PluginManager.h" ++#include "lldb/Core/StreamFile.h" ++#include "lldb/Host/ConnectionFileDescriptor.h" ++#include "lldb/Host/Host.h" ++#include "lldb/Host/HostInfo.h" ++#include "lldb/Host/PosixApi.h" ++#include "lldb/Target/Process.h" ++#include "lldb/Target/Target.h" ++#include "lldb/Utility/FileSpec.h" ++#include "lldb/Utility/Log.h" ++#include "lldb/Utility/ProcessInfo.h" ++#include "lldb/Utility/Status.h" ++#include "lldb/Utility/StreamString.h" ++#include "lldb/Utility/UriParser.h" ++ ++#include "Plugins/Process/Utility/GDBRemoteSignals.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++using namespace lldb_private::platform_wasm_server; ++ ++LLDB_PLUGIN_DEFINE_ADV(PlatformRemoteWASMServer, PlatformWasm) ++ ++static bool g_initialized = false; ++ ++void PlatformRemoteWASMServer::Initialize() { ++ Platform::Initialize(); ++ ++ if (!g_initialized) { ++ g_initialized = true; ++ PluginManager::RegisterPlugin( ++ PlatformRemoteWASMServer::GetPluginNameStatic(), ++ PlatformRemoteWASMServer::GetDescriptionStatic(), ++ PlatformRemoteWASMServer::CreateInstance); ++ } ++} ++ ++void PlatformRemoteWASMServer::Terminate() { ++ if (g_initialized) { ++ g_initialized = false; ++ PluginManager::UnregisterPlugin(PlatformRemoteWASMServer::CreateInstance); ++ } ++ ++ Platform::Terminate(); ++} ++ ++PlatformSP PlatformRemoteWASMServer::CreateInstance(bool force, ++ const ArchSpec *arch) { ++ Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_PLATFORM)); ++ if (log) { ++ const char *arch_name; ++ if (arch && arch->GetArchitectureName()) ++ arch_name = arch->GetArchitectureName(); ++ else ++ arch_name = ""; ++ ++ const char *triple_cstr = ++ arch ? arch->GetTriple().getTriple().c_str() : ""; ++ ++ LLDB_LOGF(log, "PlatformRemoteWASMServer::%s(force=%s, arch={%s,%s})", ++ __FUNCTION__, force ? "true" : "false", arch_name, triple_cstr); ++ } ++ ++ bool create = force; ++ if (!create && arch && arch->IsValid()) { ++ const llvm::Triple &triple = arch->GetTriple(); ++ if (arch->GetMachine() == llvm::Triple::wasm32 && ++ triple.getOS() == llvm::Triple::WASI) { ++ create = true; ++ } ++ } ++ ++ if (create) { ++ if (log) ++ LLDB_LOGF(log, "PlatformRemoteWASMServer::%s() creating platform", ++ __FUNCTION__); ++ return PlatformSP(new PlatformRemoteWASMServer()); ++ } ++ ++ if (log) ++ LLDB_LOGF(log, ++ "PlatformRemoteWASMServer::%s() aborting creation of platform", ++ __FUNCTION__); ++ return PlatformSP(); ++} ++ ++ConstString PlatformRemoteWASMServer::GetPluginNameStatic() { ++ static ConstString g_name("remote-wasm-server"); ++ return g_name; ++} ++ ++ConstString PlatformRemoteWASMServer::GetPluginName() { ++ return GetPluginNameStatic(); ++} ++ ++const char *PlatformRemoteWASMServer::GetDescriptionStatic() { ++ return "A platform that uses the GDB remote protocol as the communication " ++ "transport for Wasm Runtime"; ++} ++ ++size_t PlatformRemoteWASMServer::ConnectToWaitingProcesses(Debugger &debugger, ++ Status &error) { ++ std::vector connection_urls; ++ GetPendingGdbServerList(connection_urls); ++ ++ for (size_t i = 0; i < connection_urls.size(); ++i) { ++ ConnectProcess(connection_urls[i].c_str(), "wasm", debugger, nullptr, error); ++ if (error.Fail()) ++ return i; // We already connected to i process succsessfully ++ } ++ return connection_urls.size(); ++} ++ ++bool PlatformRemoteWASMServer::GetSupportedArchitectureAtIndex(uint32_t idx, ++ ArchSpec &arch) { ++ ArchSpec remote_arch = m_gdb_client.GetSystemArchitecture(); ++ if (idx == 0) { ++ arch = remote_arch; ++ return arch.IsValid(); ++ } else if (idx == 1 && remote_arch.IsValid() && ++ remote_arch.GetTriple().getOS() == llvm::Triple::WASI) { ++ return arch.IsValid(); ++ } ++ return false; ++} ++ ++/// Default Constructor ++PlatformRemoteWASMServer::PlatformRemoteWASMServer() ++ : PlatformRemoteGDBServer() ++ { ++ } +\ No newline at end of file +diff --git a/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.h b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.h +new file mode 100644 +index 000000000000..f306a79d3f4f +--- /dev/null ++++ b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.h +@@ -0,0 +1,37 @@ ++#ifndef LLDB_SOURCE_PLUGINS_PLATFORM_GDB_SERVER_PLATFORMREMOTEWASMSERVER_H ++#define LLDB_SOURCE_PLUGINS_PLATFORM_GDB_SERVER_PLATFORMREMOTEWASMSERVER_H ++ ++#include "Plugins/Platform/gdb-server/PlatformRemoteGDBServer.h" ++#include "lldb/Target/Platform.h" ++ ++namespace lldb_private { ++namespace platform_wasm_server { ++ ++class PlatformRemoteWASMServer : public lldb_private::platform_gdb_server::PlatformRemoteGDBServer{ ++ ++public: ++ static void Initialize(); ++ ++ static void Terminate(); ++ ++ static lldb::PlatformSP CreateInstance(bool force, const ArchSpec *arch); ++ ++ static ConstString GetPluginNameStatic(); ++ ++ static const char *GetDescriptionStatic(); ++ ++ size_t ConnectToWaitingProcesses(lldb_private::Debugger &debugger, ++ lldb_private::Status &error) override; ++ ++ bool GetSupportedArchitectureAtIndex(uint32_t idx, ArchSpec &arch) override; ++ ++ ConstString GetPluginName() override; ++ ++ PlatformRemoteWASMServer(); ++ ++}; ++ ++} // namespace platform_wasm_server ++} // namespace lldb_private ++ ++#endif +\ No newline at end of file +diff --git a/lldb/source/Plugins/Plugins.def.in b/lldb/source/Plugins/Plugins.def.in +index bf54598fb2f3..b0bd7b9965fe 100644 +--- a/lldb/source/Plugins/Plugins.def.in ++++ b/lldb/source/Plugins/Plugins.def.in +@@ -31,6 +31,7 @@ + + @LLDB_ENUM_PLUGINS@ + @LLDB_PROCESS_WINDOWS_PLUGIN@ ++@LLDB_PROCESS_WASM_PLUGIN@ + @LLDB_PROCESS_GDB_PLUGIN@ + + #undef LLDB_PLUGIN +diff --git a/lldb/source/Plugins/Process/CMakeLists.txt b/lldb/source/Plugins/Process/CMakeLists.txt +index bea5bac9eb21..7a0855e02ca2 100644 +--- a/lldb/source/Plugins/Process/CMakeLists.txt ++++ b/lldb/source/Plugins/Process/CMakeLists.txt +@@ -18,3 +18,4 @@ add_subdirectory(Utility) + add_subdirectory(elf-core) + add_subdirectory(mach-core) + add_subdirectory(minidump) ++add_subdirectory(wasm) +diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp +index 12bc7390c729..707ab85e5615 100644 +--- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp ++++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp +@@ -285,7 +285,7 @@ bool ProcessElfCore::IsAlive() { return true; } + + // Process Memory + size_t ProcessElfCore::ReadMemory(lldb::addr_t addr, void *buf, size_t size, +- Status &error) { ++ Status &error, ExecutionContext *exe_ctx) { + // Don't allow the caching that lldb_private::Process::ReadMemory does since + // in core files we have it all cached our our core file anyway. + return DoReadMemory(addr, buf, size, error); +diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h +index d8e3cc9ae3e1..f0bf9c4d3b00 100644 +--- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h ++++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h +@@ -84,7 +84,8 @@ public: + + // Process Memory + size_t ReadMemory(lldb::addr_t addr, void *buf, size_t size, +- lldb_private::Status &error) override; ++ lldb_private::Status &error, ++ lldb_private::ExecutionContext *exe_ctx = nullptr) override; + + size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, + lldb_private::Status &error) override; +diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +index 6914b37348ea..bb8a056049f3 100644 +--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp ++++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +@@ -334,6 +334,11 @@ ConstString ProcessGDBRemote::GetPluginName() { return GetPluginNameStatic(); } + + uint32_t ProcessGDBRemote::GetPluginVersion() { return 1; } + ++std::shared_ptr ++ProcessGDBRemote::CreateThread(lldb::tid_t tid) { ++ return std::make_shared(*this, tid); ++} ++ + bool ProcessGDBRemote::ParsePythonTargetDefinition( + const FileSpec &target_definition_fspec) { + ScriptInterpreter *interpreter = +@@ -1626,7 +1631,7 @@ bool ProcessGDBRemote::DoUpdateThreadList(ThreadList &old_thread_list, + ThreadSP thread_sp( + old_thread_list_copy.RemoveThreadByProtocolID(tid, false)); + if (!thread_sp) { +- thread_sp = std::make_shared(*this, tid); ++ thread_sp = CreateThread(tid); + LLDB_LOGV(log, "Making new thread: {0} for thread ID: {1:x}.", + thread_sp.get(), thread_sp->GetID()); + } else { +@@ -1742,7 +1747,7 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo( + + if (!thread_sp) { + // Create the thread if we need to +- thread_sp = std::make_shared(*this, tid); ++ thread_sp = CreateThread(tid); + m_thread_list_real.AddThread(thread_sp); + } + } +diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h +index fe04cdddd0f5..e4a14c64579a 100644 +--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h ++++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h +@@ -237,6 +237,8 @@ protected: + + bool SupportsMemoryTagging() override; + ++ virtual std::shared_ptr CreateThread(lldb::tid_t tid); ++ + /// Broadcaster event bits definitions. + enum { + eBroadcastBitAsyncContinue = (1 << 0), +diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp +index 84548edb5caa..0ae6f7e4a177 100644 +--- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp ++++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp +@@ -596,7 +596,7 @@ bool ProcessMachCore::WarnBeforeDetach() const { return false; } + + // Process Memory + size_t ProcessMachCore::ReadMemory(addr_t addr, void *buf, size_t size, +- Status &error) { ++ Status &error, ExecutionContext *exe_ctx) { + // Don't allow the caching that lldb_private::Process::ReadMemory does since + // in core files we have it all cached our our core file anyway. + return DoReadMemory(addr, buf, size, error); +diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h +index db77e96f1072..1c930896c743 100644 +--- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h ++++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h +@@ -65,7 +65,8 @@ public: + + // Process Memory + size_t ReadMemory(lldb::addr_t addr, void *buf, size_t size, +- lldb_private::Status &error) override; ++ lldb_private::Status &error, ++ lldb_private::ExecutionContext *exe_ctx = nullptr) override; + + size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, + lldb_private::Status &error) override; +diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp +index 385557422758..d8bb21581086 100644 +--- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp ++++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp +@@ -374,7 +374,7 @@ bool ProcessMinidump::IsAlive() { return true; } + bool ProcessMinidump::WarnBeforeDetach() const { return false; } + + size_t ProcessMinidump::ReadMemory(lldb::addr_t addr, void *buf, size_t size, +- Status &error) { ++ Status &error, ExecutionContext *exe_ctx) { + // Don't allow the caching that lldb_private::Process::ReadMemory does since + // we have it all cached in our dump file anyway. + return DoReadMemory(addr, buf, size, error); +diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.h b/lldb/source/Plugins/Process/minidump/ProcessMinidump.h +index 27b0da0047a5..e94ecab430c1 100644 +--- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.h ++++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.h +@@ -69,8 +69,8 @@ public: + + bool WarnBeforeDetach() const override; + +- size_t ReadMemory(lldb::addr_t addr, void *buf, size_t size, +- Status &error) override; ++ size_t ReadMemory(lldb::addr_t addr, void *buf, size_t size, Status &error, ++ ExecutionContext *exe_ctx = nullptr) override; + + size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, + Status &error) override; +diff --git a/lldb/source/Plugins/Process/wasm/CMakeLists.txt b/lldb/source/Plugins/Process/wasm/CMakeLists.txt +new file mode 100644 +index 000000000000..61efb933fa62 +--- /dev/null ++++ b/lldb/source/Plugins/Process/wasm/CMakeLists.txt +@@ -0,0 +1,12 @@ ++ ++add_lldb_library(lldbPluginProcessWasm PLUGIN ++ ProcessWasm.cpp ++ ThreadWasm.cpp ++ UnwindWasm.cpp ++ ++ LINK_LIBS ++ lldbCore ++ ${LLDB_PLUGINS} ++ LINK_COMPONENTS ++ Support ++ ) +diff --git a/lldb/source/Plugins/Process/wasm/ProcessWasm.cpp b/lldb/source/Plugins/Process/wasm/ProcessWasm.cpp +new file mode 100644 +index 000000000000..9c0fc7b7f270 +--- /dev/null ++++ b/lldb/source/Plugins/Process/wasm/ProcessWasm.cpp +@@ -0,0 +1,261 @@ ++//===-- ProcessWasm.cpp ---------------------------------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#include "ProcessWasm.h" ++#include "ThreadWasm.h" ++#include "lldb/Core/Module.h" ++#include "lldb/Core/PluginManager.h" ++#include "lldb/Utility/DataBufferHeap.h" ++ ++#include "lldb/Target/UnixSignals.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++using namespace lldb_private::process_gdb_remote; ++using namespace lldb_private::wasm; ++ ++LLDB_PLUGIN_DEFINE(ProcessWasm) ++ ++// ProcessGDBRemote constructor ++ProcessWasm::ProcessWasm(lldb::TargetSP target_sp, ListenerSP listener_sp) ++ : ProcessGDBRemote(target_sp, listener_sp) { ++ /* always use linux signals for wasm process */ ++ m_unix_signals_sp = UnixSignals::Create(ArchSpec{"wasm32-Ant-wasi-wasm"}); ++} ++ ++void ProcessWasm::Initialize() { ++ static llvm::once_flag g_once_flag; ++ ++ llvm::call_once(g_once_flag, []() { ++ PluginManager::RegisterPlugin(GetPluginNameStatic(), ++ GetPluginDescriptionStatic(), CreateInstance, ++ DebuggerInitialize); ++ }); ++} ++ ++void ProcessWasm::DebuggerInitialize(Debugger &debugger) { ++ ProcessGDBRemote::DebuggerInitialize(debugger); ++} ++ ++// PluginInterface ++ConstString ProcessWasm::GetPluginName() { return GetPluginNameStatic(); } ++ ++uint32_t ProcessWasm::GetPluginVersion() { return 1; } ++ ++ConstString ProcessWasm::GetPluginNameStatic() { ++ static ConstString g_name("wasm"); ++ return g_name; ++} ++ ++const char *ProcessWasm::GetPluginDescriptionStatic() { ++ return "GDB Remote protocol based WebAssembly debugging plug-in."; ++} ++ ++void ProcessWasm::Terminate() { ++ PluginManager::UnregisterPlugin(ProcessWasm::CreateInstance); ++} ++ ++lldb::ProcessSP ProcessWasm::CreateInstance(lldb::TargetSP target_sp, ++ ListenerSP listener_sp, ++ const FileSpec *crash_file_path, ++ bool can_connect) { ++ lldb::ProcessSP process_sp; ++ if (crash_file_path == nullptr) ++ process_sp = std::make_shared(target_sp, listener_sp); ++ return process_sp; ++} ++ ++bool ProcessWasm::CanDebug(lldb::TargetSP target_sp, ++ bool plugin_specified_by_name) { ++ if (plugin_specified_by_name) ++ return true; ++ ++ Module *exe_module = target_sp->GetExecutableModulePointer(); ++ if (exe_module) { ++ ObjectFile *exe_objfile = exe_module->GetObjectFile(); ++ return exe_objfile->GetArchitecture().GetMachine() == llvm::Triple::wasm32; ++ } ++ // However, if there is no wasm module, we return false, otherwise, ++ // we might use ProcessWasm to attach gdb remote. ++ return false; ++} ++ ++ ++ ++std::shared_ptr ProcessWasm::CreateThread(lldb::tid_t tid) { ++ return std::make_shared(*this, tid); ++} ++ ++size_t ProcessWasm::ReadMemory(lldb::addr_t vm_addr, void *buf, size_t size, ++ Status &error, ExecutionContext *exe_ctx) { ++ wasm_addr_t wasm_addr(vm_addr); ++ size_t nread = 0; ++ ++ switch (wasm_addr.GetType()) { ++ case WasmAddressType::Memory: ++ case WasmAddressType::Object: ++ return ProcessGDBRemote::ReadMemory(vm_addr, buf, size, error); ++ case WasmAddressType::Invalid: ++ default: ++ error.SetErrorStringWithFormat( ++ "Wasm read failed for invalid address 0x%" PRIx64, vm_addr); ++ return 0; ++ } ++} ++ ++size_t ProcessWasm::WasmReadMemory(uint32_t wasm_module_id, lldb::addr_t addr, ++ void *buf, size_t buffer_size) { ++ char packet[64]; ++ int packet_len = ++ ::snprintf(packet, sizeof(packet), "qWasmMem:%d;%" PRIx64 ";%" PRIx64, ++ wasm_module_id, static_cast(addr), ++ static_cast(buffer_size)); ++ assert(packet_len + 1 < (int)sizeof(packet)); ++ UNUSED_IF_ASSERT_DISABLED(packet_len); ++ StringExtractorGDBRemote response; ++ if (m_gdb_comm.SendPacketAndWaitForResponse(packet, response, GetInterruptTimeout()) == ++ GDBRemoteCommunication::PacketResult::Success) { ++ if (response.IsNormalResponse()) { ++ return response.GetHexBytes(llvm::MutableArrayRef( ++ static_cast(buf), buffer_size), ++ '\xdd'); ++ } ++ } ++ return 0; ++} ++ ++size_t ProcessWasm::WasmReadData(uint32_t wasm_module_id, lldb::addr_t addr, ++ void *buf, size_t buffer_size) { ++ char packet[64]; ++ int packet_len = ++ ::snprintf(packet, sizeof(packet), "qWasmData:%d;%" PRIx64 ";%" PRIx64, ++ wasm_module_id, static_cast(addr), ++ static_cast(buffer_size)); ++ assert(packet_len + 1 < (int)sizeof(packet)); ++ UNUSED_IF_ASSERT_DISABLED(packet_len); ++ StringExtractorGDBRemote response; ++ if (m_gdb_comm.SendPacketAndWaitForResponse(packet, response, GetInterruptTimeout()) == ++ GDBRemoteCommunication::PacketResult::Success) { ++ if (response.IsNormalResponse()) { ++ return response.GetHexBytes(llvm::MutableArrayRef( ++ static_cast(buf), buffer_size), ++ '\xdd'); ++ } ++ } ++ return 0; ++} ++ ++bool ProcessWasm::GetWasmLocal(int frame_index, int index, void *buf, ++ size_t buffer_size, size_t &size) { ++ StreamString packet; ++ packet.Printf("qWasmLocal:"); ++ packet.Printf("%d;%d", frame_index, index); ++ StringExtractorGDBRemote response; ++ if (m_gdb_comm.SendPacketAndWaitForResponse(packet.GetString(), response) != ++ GDBRemoteCommunication::PacketResult::Success) { ++ return false; ++ } ++ ++ if (!response.IsNormalResponse()) { ++ return false; ++ } ++ ++ DataBufferSP buffer_sp( ++ new DataBufferHeap(response.GetStringRef().size() / 2, 0)); ++ response.GetHexBytes(buffer_sp->GetData(), '\xcc'); ++ size = buffer_sp->GetByteSize(); ++ if (size <= buffer_size) { ++ memcpy(buf, buffer_sp->GetBytes(), size); ++ return true; ++ } ++ ++ return false; ++} ++ ++bool ProcessWasm::GetWasmGlobal(int frame_index, int index, void *buf, ++ size_t buffer_size, size_t &size) { ++ StreamString packet; ++ packet.PutCString("qWasmGlobal:"); ++ packet.Printf("%d;%d", frame_index, index); ++ StringExtractorGDBRemote response; ++ if (m_gdb_comm.SendPacketAndWaitForResponse(packet.GetString(), response) != ++ GDBRemoteCommunication::PacketResult::Success) { ++ return false; ++ } ++ ++ if (!response.IsNormalResponse()) { ++ return false; ++ } ++ ++ DataBufferSP buffer_sp( ++ new DataBufferHeap(response.GetStringRef().size() / 2, 0)); ++ response.GetHexBytes(buffer_sp->GetData(), '\xcc'); ++ size = buffer_sp->GetByteSize(); ++ if (size <= buffer_size) { ++ memcpy(buf, buffer_sp->GetBytes(), size); ++ return true; ++ } ++ ++ return false; ++} ++ ++bool ProcessWasm::GetWasmStackValue(int frame_index, int index, void *buf, ++ size_t buffer_size, size_t &size) { ++ StreamString packet; ++ packet.PutCString("qWasmStackValue:"); ++ packet.Printf("%d;%d", frame_index, index); ++ StringExtractorGDBRemote response; ++ if (m_gdb_comm.SendPacketAndWaitForResponse(packet.GetString(), response) != ++ GDBRemoteCommunication::PacketResult::Success) { ++ return false; ++ } ++ ++ if (!response.IsNormalResponse()) { ++ return false; ++ } ++ ++ DataBufferSP buffer_sp( ++ new DataBufferHeap(response.GetStringRef().size() / 2, 0)); ++ response.GetHexBytes(buffer_sp->GetData(), '\xcc'); ++ size = buffer_sp->GetByteSize(); ++ if (size <= buffer_size) { ++ memcpy(buf, buffer_sp->GetBytes(), size); ++ return true; ++ } ++ ++ return false; ++} ++ ++bool ProcessWasm::GetWasmCallStack(lldb::tid_t tid, ++ std::vector &call_stack_pcs) { ++ call_stack_pcs.clear(); ++ StreamString packet; ++ packet.Printf("qWasmCallStack:"); ++ packet.Printf("%llx", tid); ++ StringExtractorGDBRemote response; ++ if (m_gdb_comm.SendPacketAndWaitForResponse(packet.GetString(), response) != ++ GDBRemoteCommunication::PacketResult::Success) { ++ return false; ++ } ++ ++ if (!response.IsNormalResponse()) { ++ return false; ++ } ++ ++ addr_t buf[1024 / sizeof(addr_t)]; ++ size_t bytes = response.GetHexBytes( ++ llvm::MutableArrayRef((uint8_t *)buf, sizeof(buf)), '\xdd'); ++ if (bytes == 0) { ++ return false; ++ } ++ ++ for (size_t i = 0; i < bytes / sizeof(addr_t); i++) { ++ call_stack_pcs.push_back(buf[i]); ++ } ++ return true; ++} +diff --git a/lldb/source/Plugins/Process/wasm/ProcessWasm.h b/lldb/source/Plugins/Process/wasm/ProcessWasm.h +new file mode 100644 +index 000000000000..d3aece7a6554 +--- /dev/null ++++ b/lldb/source/Plugins/Process/wasm/ProcessWasm.h +@@ -0,0 +1,128 @@ ++//===-- ProcessWasm.h -------------------------------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLDB_SOURCE_PLUGINS_PROCESS_WASM_PROCESSWASM_H ++#define LLDB_SOURCE_PLUGINS_PROCESS_WASM_PROCESSWASM_H ++ ++#include "Plugins/Process/gdb-remote/ProcessGDBRemote.h" ++#include "lldb/Target/RegisterContext.h" ++ ++namespace lldb_private { ++namespace wasm { ++ ++// Each WebAssembly module has separated address spaces for Code and Memory. ++// A WebAssembly module also has a Data section which, when the module is ++// loaded, gets mapped into a region in the module Memory. ++// For the purpose of debugging, we can represent all these separated 32-bit ++// address spaces with a single virtual 64-bit address space. ++// ++// Struct wasm_addr_t provides this encoding using bitfields ++// ++enum WasmAddressType { ++ Memory = 0x00, ++ Object = 0x01, ++ Invalid = 0x03 ++}; ++struct wasm_addr_t { ++ uint64_t offset : 32; ++ uint64_t module_id : 30; ++ uint64_t type : 2; ++ ++ wasm_addr_t(lldb::addr_t addr) ++ : type(addr >> 62), module_id((addr & 0x00ffffff00000000) >> 32), ++ offset(addr & 0x00000000ffffffff) {} ++ ++ wasm_addr_t(WasmAddressType type_, uint32_t module_id_, uint32_t offset_) ++ : type(type_), module_id(module_id_), offset(offset_) {} ++ ++ WasmAddressType GetType() { return static_cast(type); } ++ operator lldb::addr_t() { return *(uint64_t *)this; } ++}; ++ ++/// ProcessWasm provides the access to the Wasm program state ++/// retrieved from the Wasm engine. ++class ProcessWasm : public process_gdb_remote::ProcessGDBRemote { ++public: ++ ProcessWasm(lldb::TargetSP target_sp, lldb::ListenerSP listener_sp); ++ ~ProcessWasm() override = default; ++ ++ static lldb::ProcessSP CreateInstance(lldb::TargetSP target_sp, ++ lldb::ListenerSP listener_sp, ++ const FileSpec *crash_file_path, ++ bool can_connect); ++ ++ static void Initialize(); ++ static void DebuggerInitialize(Debugger &debugger); ++ static void Terminate(); ++ static ConstString GetPluginNameStatic(); ++ static const char *GetPluginDescriptionStatic(); ++ ++ /// PluginInterface protocol. ++ /// \{ ++ ConstString GetPluginName() override; ++ uint32_t GetPluginVersion() override; ++ /// \} ++ ++ /// Process protocol. ++ /// \{ ++ size_t ReadMemory(lldb::addr_t vm_addr, void *buf, size_t size, Status &error, ++ ExecutionContext *exe_ctx = nullptr) override; ++ /// \} ++ ++ /// Query the value of a WebAssembly local variable from the WebAssembly ++ /// remote process. ++ bool GetWasmLocal(int frame_index, int index, void *buf, size_t buffer_size, ++ size_t &size); ++ ++ /// Query the value of a WebAssembly global variable from the WebAssembly ++ /// remote process. ++ bool GetWasmGlobal(int frame_index, int index, void *buf, size_t buffer_size, ++ size_t &size); ++ ++ /// Query the value of an item in the WebAssembly operand stack from the ++ /// WebAssembly remote process. ++ bool GetWasmStackValue(int frame_index, int index, void *buf, ++ size_t buffer_size, size_t &size); ++ ++ /// Read from the WebAssembly Memory space. ++ size_t WasmReadMemory(uint32_t wasm_module_id, lldb::addr_t addr, void *buf, ++ size_t buffer_size); ++ ++ /// Read from the WebAssembly Data space. ++ size_t WasmReadData(uint32_t wasm_module_id, lldb::addr_t addr, void *buf, ++ size_t buffer_size); ++ ++ /// Retrieve the current call stack from the WebAssembly remote process. ++ bool GetWasmCallStack(lldb::tid_t tid, ++ std::vector &call_stack_pcs); ++ ++ // Check if a given Process ++ bool CanDebug(lldb::TargetSP target_sp, ++ bool plugin_specified_by_name) override; ++ ++protected: ++ /// ProcessGDBRemote protocol. ++ /// \{ ++ std::shared_ptr ++ CreateThread(lldb::tid_t tid) override; ++ /// \} ++ ++private: ++ friend class UnwindWasm; ++ process_gdb_remote::GDBRemoteDynamicRegisterInfoSP &GetRegisterInfo() { ++ return m_register_info_sp; ++ } ++ ++ ProcessWasm(const ProcessWasm &); ++ const ProcessWasm &operator=(const ProcessWasm &) = delete; ++}; ++ ++} // namespace wasm ++} // namespace lldb_private ++ ++#endif // LLDB_SOURCE_PLUGINS_PROCESS_WASM_PROCESSWASM_H +diff --git a/lldb/source/Plugins/Process/wasm/ThreadWasm.cpp b/lldb/source/Plugins/Process/wasm/ThreadWasm.cpp +new file mode 100644 +index 000000000000..fa02073e7a52 +--- /dev/null ++++ b/lldb/source/Plugins/Process/wasm/ThreadWasm.cpp +@@ -0,0 +1,35 @@ ++//===-- ThreadWasm.cpp ----------------------------------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#include "ThreadWasm.h" ++ ++#include "ProcessWasm.h" ++#include "UnwindWasm.h" ++#include "lldb/Target/Target.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++using namespace lldb_private::wasm; ++ ++Unwind &ThreadWasm::GetUnwinder() { ++ if (!m_unwinder_up) { ++ assert(CalculateTarget()->GetArchitecture().GetMachine() == ++ llvm::Triple::wasm32); ++ m_unwinder_up.reset(new wasm::UnwindWasm(*this)); ++ } ++ return *m_unwinder_up; ++} ++ ++bool ThreadWasm::GetWasmCallStack(std::vector &call_stack_pcs) { ++ ProcessSP process_sp(GetProcess()); ++ if (process_sp) { ++ ProcessWasm *wasm_process = static_cast(process_sp.get()); ++ return wasm_process->GetWasmCallStack(GetID(), call_stack_pcs); ++ } ++ return false; ++} +diff --git a/lldb/source/Plugins/Process/wasm/ThreadWasm.h b/lldb/source/Plugins/Process/wasm/ThreadWasm.h +new file mode 100644 +index 000000000000..0a33c07de994 +--- /dev/null ++++ b/lldb/source/Plugins/Process/wasm/ThreadWasm.h +@@ -0,0 +1,41 @@ ++//===-- ThreadWasm.h --------------------------------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLDB_SOURCE_PLUGINS_PROCESS_WASM_THREADWASM_H ++#define LLDB_SOURCE_PLUGINS_PROCESS_WASM_THREADWASM_H ++ ++#include "Plugins/Process/gdb-remote/ThreadGDBRemote.h" ++ ++namespace lldb_private { ++namespace wasm { ++ ++/// ProcessWasm provides the access to the Wasm program state ++/// retrieved from the Wasm engine. ++class ThreadWasm : public process_gdb_remote::ThreadGDBRemote { ++public: ++ ThreadWasm(Process &process, lldb::tid_t tid) ++ : process_gdb_remote::ThreadGDBRemote(process, tid) {} ++ ~ThreadWasm() override = default; ++ ++ /// Retrieve the current call stack from the WebAssembly remote process. ++ bool GetWasmCallStack(std::vector &call_stack_pcs); ++ ++protected: ++ /// Thread protocol. ++ /// \{ ++ Unwind &GetUnwinder() override; ++ /// \} ++ ++ ThreadWasm(const ThreadWasm &); ++ const ThreadWasm &operator=(const ThreadWasm &) = delete; ++}; ++ ++} // namespace wasm ++} // namespace lldb_private ++ ++#endif // LLDB_SOURCE_PLUGINS_PROCESS_WASM_THREADWASM_H +diff --git a/lldb/source/Plugins/Process/wasm/UnwindWasm.cpp b/lldb/source/Plugins/Process/wasm/UnwindWasm.cpp +new file mode 100644 +index 000000000000..1a195cb9361a +--- /dev/null ++++ b/lldb/source/Plugins/Process/wasm/UnwindWasm.cpp +@@ -0,0 +1,74 @@ ++//===-- UnwindWasm.cpp ----------------------------------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#include "UnwindWasm.h" ++#include "Plugins/Process/gdb-remote/ThreadGDBRemote.h" ++#include "Plugins/Process/wasm/ProcessWasm.h" ++#include "Plugins/Process/wasm/ThreadWasm.h" ++ ++using namespace lldb; ++using namespace lldb_private; ++using namespace process_gdb_remote; ++using namespace wasm; ++ ++class WasmGDBRemoteRegisterContext : public GDBRemoteRegisterContext { ++public: ++ WasmGDBRemoteRegisterContext(ThreadGDBRemote &thread, ++ uint32_t concrete_frame_idx, ++ GDBRemoteDynamicRegisterInfoSP ®_info_sp, ++ uint64_t pc) ++ : GDBRemoteRegisterContext(thread, concrete_frame_idx, reg_info_sp, false, ++ false) { ++ PrivateSetRegisterValue(0, pc); ++ } ++}; ++ ++lldb::RegisterContextSP ++UnwindWasm::DoCreateRegisterContextForFrame(lldb_private::StackFrame *frame) { ++ if (m_frames.size() <= frame->GetFrameIndex()) { ++ return lldb::RegisterContextSP(); ++ } ++ ++ ThreadSP thread = frame->GetThread(); ++ ThreadGDBRemote *gdb_thread = static_cast(thread.get()); ++ ProcessWasm *wasm_process = ++ static_cast(thread->GetProcess().get()); ++ std::shared_ptr reg_ctx_sp = ++ std::make_shared( ++ *gdb_thread, frame->GetConcreteFrameIndex(), ++ wasm_process->GetRegisterInfo(), m_frames[frame->GetFrameIndex()]); ++ return reg_ctx_sp; ++} ++ ++uint32_t UnwindWasm::DoGetFrameCount() { ++ if (!m_unwind_complete) { ++ m_unwind_complete = true; ++ m_frames.clear(); ++ ++ ThreadWasm &wasm_thread = static_cast(GetThread()); ++ if (!wasm_thread.GetWasmCallStack(m_frames)) ++ m_frames.clear(); ++ } ++ return m_frames.size(); ++} ++ ++bool UnwindWasm::DoGetFrameInfoAtIndex(uint32_t frame_idx, lldb::addr_t &cfa, ++ lldb::addr_t &pc, ++ bool &behaves_like_zeroth_frame) { ++ if (m_frames.size() == 0) { ++ DoGetFrameCount(); ++ } ++ ++ if (frame_idx < m_frames.size()) { ++ behaves_like_zeroth_frame = (frame_idx == 0); ++ cfa = 0; ++ pc = m_frames[frame_idx]; ++ return true; ++ } ++ return false; ++} +\ No newline at end of file +diff --git a/lldb/source/Plugins/Process/wasm/UnwindWasm.h b/lldb/source/Plugins/Process/wasm/UnwindWasm.h +new file mode 100644 +index 000000000000..9bd1dac9a98a +--- /dev/null ++++ b/lldb/source/Plugins/Process/wasm/UnwindWasm.h +@@ -0,0 +1,55 @@ ++//===-- UnwindWasm.h --------------------------------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef lldb_UnwindWasm_h_ ++#define lldb_UnwindWasm_h_ ++ ++#include "lldb/Target/RegisterContext.h" ++#include "lldb/Target/Unwind.h" ++#include ++ ++namespace lldb_private { ++namespace wasm { ++ ++/// UnwindWasm manages stack unwinding for a WebAssembly process. ++class UnwindWasm : public lldb_private::Unwind { ++public: ++ UnwindWasm(lldb_private::Thread &thread) ++ : Unwind(thread), m_frames(), m_unwind_complete(false) {} ++ ~UnwindWasm() override = default; ++ ++protected: ++ /// Unwind protocol. ++ /// \{ ++ void DoClear() override { ++ m_frames.clear(); ++ m_unwind_complete = false; ++ } ++ ++ uint32_t DoGetFrameCount() override; ++ ++ bool DoGetFrameInfoAtIndex(uint32_t frame_idx, lldb::addr_t &cfa, ++ lldb::addr_t &pc, ++ bool &behaves_like_zeroth_frame) override; ++ ++ lldb::RegisterContextSP ++ DoCreateRegisterContextForFrame(lldb_private::StackFrame *frame) override; ++ /// \} ++ ++private: ++ std::vector m_frames; ++ bool m_unwind_complete; ++ ++ UnwindWasm(const UnwindWasm &); ++ const UnwindWasm &operator=(const UnwindWasm &) = delete; ++}; ++ ++} // namespace wasm ++} // namespace lldb_private ++ ++#endif // lldb_UnwindWasm_h_ +diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +index ccaf31317d75..c3ef5aebd46d 100644 +--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp ++++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +@@ -3212,8 +3212,13 @@ VariableSP SymbolFileDWARF::ParseVariableDIE(const SymbolContext &sc, + GetDWARFDeclContext(die).GetQualifiedNameAsConstString().GetCString(); + } + +- if (tag == DW_TAG_formal_parameter) ++ if (tag == DW_TAG_formal_parameter) { + scope = eValueTypeVariableArgument; ++ // For Wasm dwarft, pamameter may don't have location attr, ++ // so set module here ++ if (!location.GetModule()) ++ location.SetModule(module); ++ } + else { + // DWARF doesn't specify if a DW_TAG_variable is a local, global + // or static variable, so we have to do a little digging: +diff --git a/lldb/source/Target/PathMappingList.cpp b/lldb/source/Target/PathMappingList.cpp +index b660c310ef31..cd76421cec18 100644 +--- a/lldb/source/Target/PathMappingList.cpp ++++ b/lldb/source/Target/PathMappingList.cpp +@@ -218,7 +218,12 @@ bool PathMappingList::ReverseRemapPath(const FileSpec &file, FileSpec &fixed) co + } + + llvm::Optional PathMappingList::FindFile(const FileSpec &orig_spec) const { +- if (auto remapped = RemapPath(orig_spec.GetPath(), /*only_if_exists=*/true)) ++ // We must normalize the orig_spec again using the host's path style, ++ // otherwise there will be mismatch between the host and remote platform ++ // if they use different path styles. ++ if (auto remapped = RemapPath( ++ NormalizePath(ConstString(orig_spec.GetCString())).GetStringRef(), ++ /*only_if_exists=*/true)) + return remapped; + + return {}; +diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp +index a77ecddfbab6..e257f93508f6 100644 +--- a/lldb/source/Target/Platform.cpp ++++ b/lldb/source/Target/Platform.cpp +@@ -1970,6 +1970,12 @@ size_t Platform::GetSoftwareBreakpointTrapOpcode(Target &target, + trap_opcode_size = sizeof(g_i386_opcode); + } break; + ++ case llvm::Triple::wasm32: { ++ static const uint8_t g_wasm_opcode[] = {0x00}; // unreachable ++ trap_opcode = g_wasm_opcode; ++ trap_opcode_size = sizeof(g_wasm_opcode); ++ } break; ++ + default: + return 0; + } +diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp +index 8ecc66b592ea..f148987915de 100644 +--- a/lldb/source/Target/Process.cpp ++++ b/lldb/source/Target/Process.cpp +@@ -1892,7 +1892,8 @@ Status Process::DisableSoftwareBreakpoint(BreakpointSite *bp_site) { + // code + //#define VERIFY_MEMORY_READS + +-size_t Process::ReadMemory(addr_t addr, void *buf, size_t size, Status &error) { ++size_t Process::ReadMemory(addr_t addr, void *buf, size_t size, Status &error, ++ ExecutionContext *exe_ctx) { + error.Clear(); + if (!GetDisableMemoryCache()) { + #if defined(VERIFY_MEMORY_READS) +diff --git a/lldb/source/Target/ProcessTrace.cpp b/lldb/source/Target/ProcessTrace.cpp +index c878a2ac4eb9..ad5945b0ad1f 100644 +--- a/lldb/source/Target/ProcessTrace.cpp ++++ b/lldb/source/Target/ProcessTrace.cpp +@@ -88,7 +88,7 @@ void ProcessTrace::RefreshStateAfterStop() {} + Status ProcessTrace::DoDestroy() { return Status(); } + + size_t ProcessTrace::ReadMemory(addr_t addr, void *buf, size_t size, +- Status &error) { ++ Status &error, ExecutionContext *exe_ctx) { + // Don't allow the caching that lldb_private::Process::ReadMemory does since + // we have it all cached in the trace files. + return DoReadMemory(addr, buf, size, error); +diff --git a/lldb/source/Target/ThreadPlanStepRange.cpp b/lldb/source/Target/ThreadPlanStepRange.cpp +index 896e647bbb52..f76307016102 100644 +--- a/lldb/source/Target/ThreadPlanStepRange.cpp ++++ b/lldb/source/Target/ThreadPlanStepRange.cpp +@@ -334,7 +334,10 @@ bool ThreadPlanStepRange::SetNextBranchBreakpoint() { + // If we didn't find a branch, run to the end of the range. + if (branch_index == UINT32_MAX) { + uint32_t last_index = instructions->GetSize() - 1; +- if (last_index - pc_index > 1) { ++ /* This line causes the "step over was treated as step in" issue, we ++ * modify it as a workaround */ ++ /* The origin line is: if (last_index - pc_index > 1) { */ ++ if (last_index - pc_index >= 1) { + InstructionSP last_inst = + instructions->GetInstructionAtIndex(last_index); + size_t last_inst_size = last_inst->GetOpcode().GetByteSize(); +diff --git a/lldb/source/Target/UnixSignals.cpp b/lldb/source/Target/UnixSignals.cpp +index 4ec2e25c7e3b..24c88fe9ae4f 100644 +--- a/lldb/source/Target/UnixSignals.cpp ++++ b/lldb/source/Target/UnixSignals.cpp +@@ -46,6 +46,8 @@ lldb::UnixSignalsSP UnixSignals::Create(const ArchSpec &arch) { + return std::make_shared(); + case llvm::Triple::NetBSD: + return std::make_shared(); ++ case llvm::Triple::WASI: ++ return std::make_shared(); + default: + return std::make_shared(); + } +diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h +index 4310ba9ce9e0..297b3387999d 100644 +--- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h ++++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h +@@ -13,6 +13,7 @@ + #ifndef LLVM_EXECUTIONENGINE_ORC_ORCRPCEXECUTORPROCESSCONTROL_H + #define LLVM_EXECUTIONENGINE_ORC_ORCRPCEXECUTORPROCESSCONTROL_H + ++#include "llvm/ExecutionEngine/Orc/Core.h" + #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" + #include "llvm/ExecutionEngine/Orc/Shared/RPCUtils.h" + #include "llvm/ExecutionEngine/Orc/Shared/RawByteChannel.h" +diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h +index 753b1998c40c..27370c62dd6e 100644 +--- a/llvm/include/llvm/Support/MathExtras.h ++++ b/llvm/include/llvm/Support/MathExtras.h +@@ -16,6 +16,7 @@ + #include "llvm/Support/Compiler.h" + #include + #include ++#include + #include + #include + #include diff --git a/wasm-micro-runtime/build-scripts/requirements.txt b/wasm-micro-runtime/build-scripts/requirements.txt new file mode 100644 index 0000000..077c95d --- /dev/null +++ b/wasm-micro-runtime/build-scripts/requirements.txt @@ -0,0 +1 @@ +requests==2.31.0 \ No newline at end of file diff --git a/wasm-micro-runtime/build-scripts/runtime_lib.cmake b/wasm-micro-runtime/build-scripts/runtime_lib.cmake new file mode 100644 index 0000000..3ab0cff --- /dev/null +++ b/wasm-micro-runtime/build-scripts/runtime_lib.cmake @@ -0,0 +1,198 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +if (NOT DEFINED WAMR_ROOT_DIR) + set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../) +endif () +if (NOT DEFINED SHARED_DIR) + set (SHARED_DIR ${WAMR_ROOT_DIR}/core/shared) +endif () +if (NOT DEFINED IWASM_DIR) + set (IWASM_DIR ${WAMR_ROOT_DIR}/core/iwasm) +endif () +if (NOT DEFINED DEPS_DIR) + set (DEPS_DIR ${WAMR_ROOT_DIR}/core/deps) +endif () +if (NOT DEFINED SHARED_PLATFORM_CONFIG) + # CMake file for platform configuration. The PLATFORM_SHARED_SOURCE varable + # should point to a list of platform-specfic source files to compile. + set (SHARED_PLATFORM_CONFIG ${SHARED_DIR}/platform/${WAMR_BUILD_PLATFORM}/shared_platform.cmake) +endif () + +if (DEFINED EXTRA_SDK_INCLUDE_PATH) + message(STATUS, "EXTRA_SDK_INCLUDE_PATH = ${EXTRA_SDK_INCLUDE_PATH} ") + include_directories ( + ${EXTRA_SDK_INCLUDE_PATH} + ) +endif () + +# Set default options + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +################ optional according to settings ################ +if (WAMR_BUILD_FAST_JIT EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1) + # Enable classic interpreter if Fast JIT or LLVM JIT is enabled + set (WAMR_BUILD_INTERP 1) + set (WAMR_BUILD_FAST_INTERP 0) +endif () + +if (WAMR_BUILD_INTERP EQUAL 1) + include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) +endif () + +if (WAMR_BUILD_FAST_JIT EQUAL 1) + include (${IWASM_DIR}/fast-jit/iwasm_fast_jit.cmake) +endif () + +if (WAMR_BUILD_JIT EQUAL 1) + # Enable AOT if LLVM JIT is enabled + set (WAMR_BUILD_AOT 1) + include (${IWASM_DIR}/compilation/iwasm_compl.cmake) +endif () + +if (WAMR_BUILD_AOT EQUAL 1) + include (${IWASM_DIR}/aot/iwasm_aot.cmake) +endif () + +if (WAMR_BUILD_STRINGREF EQUAL 1) + set (WAMR_BUILD_GC 1) +endif () + +if (WAMR_BUILD_GC EQUAL 1) + include (${IWASM_DIR}/common/gc/iwasm_gc.cmake) + # Enable the dependent feature if GC is enabled + set (WAMR_BUILD_REF_TYPES 1) +endif () + +if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) + include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) +endif () + +if (WAMR_BUILD_LIBC_UVWASI EQUAL 1) + include (${IWASM_DIR}/libraries/libc-uvwasi/libc_uvwasi.cmake) + set (WAMR_BUILD_MODULE_INST_CONTEXT 1) +elseif (WAMR_BUILD_LIBC_WASI EQUAL 1) + include (${IWASM_DIR}/libraries/libc-wasi/libc_wasi.cmake) + set (WAMR_BUILD_MODULE_INST_CONTEXT 1) +endif () + +if (WAMR_BUILD_LIB_PTHREAD_SEMAPHORE EQUAL 1) + # Enable the dependent feature if lib pthread semaphore is enabled + set (WAMR_BUILD_LIB_PTHREAD 1) +endif () + +if (WAMR_BUILD_WASI_NN EQUAL 1) + include (${IWASM_DIR}/libraries/wasi-nn/cmake/wasi_nn.cmake) +endif () + +if (WAMR_BUILD_LIB_PTHREAD EQUAL 1) + include (${IWASM_DIR}/libraries/lib-pthread/lib_pthread.cmake) + # Enable the dependent feature if lib pthread is enabled + set (WAMR_BUILD_THREAD_MGR 1) + set (WAMR_BUILD_BULK_MEMORY 1) + set (WAMR_BUILD_SHARED_MEMORY 1) +endif () + +if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) + include (${IWASM_DIR}/libraries/lib-wasi-threads/lib_wasi_threads.cmake) + # Enable the dependent feature if lib wasi threads is enabled + set (WAMR_BUILD_THREAD_MGR 1) + set (WAMR_BUILD_BULK_MEMORY 1) + set (WAMR_BUILD_SHARED_MEMORY 1) +endif () + +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + set (WAMR_BUILD_THREAD_MGR 1) + include (${IWASM_DIR}/libraries/debug-engine/debug_engine.cmake) + + if (WAMR_BUILD_FAST_INTERP EQUAL 1) + set (WAMR_BUILD_FAST_INTERP 0) + message(STATUS + "Debugger doesn't work with fast interpreter, switch to classic interpreter") + endif () +endif () + +if (WAMR_BUILD_THREAD_MGR EQUAL 1) + include (${IWASM_DIR}/libraries/thread-mgr/thread_mgr.cmake) +endif () + +if (WAMR_BUILD_LIBC_EMCC EQUAL 1) + include (${IWASM_DIR}/libraries/libc-emcc/libc_emcc.cmake) +endif () + +if (WAMR_BUILD_LIB_RATS EQUAL 1) + include (${IWASM_DIR}/libraries/lib-rats/lib_rats.cmake) +endif () + +if (WAMR_BUILD_WASM_CACHE EQUAL 1) + include (${WAMR_ROOT_DIR}/build-scripts/involve_boringssl.cmake) +endif () + +####################### Common sources ####################### +if (NOT MSVC) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -ffunction-sections -fdata-sections \ + -Wall -Wno-unused-parameter -Wno-pedantic") +endif () + +# include the build config template file +include (${CMAKE_CURRENT_LIST_DIR}/config_common.cmake) + +include_directories (${IWASM_DIR}/include) + +file (GLOB header + ${IWASM_DIR}/include/*.h +) +LIST (APPEND RUNTIME_LIB_HEADER_LIST ${header}) + +if (WAMR_BUILD_PLATFORM STREQUAL "windows") + enable_language (ASM_MASM) +else() + enable_language (ASM) +endif() + +include (${SHARED_PLATFORM_CONFIG}) +include (${SHARED_DIR}/mem-alloc/mem_alloc.cmake) +include (${IWASM_DIR}/common/iwasm_common.cmake) +include (${SHARED_DIR}/utils/shared_utils.cmake) + + +set (source_all + ${PLATFORM_SHARED_SOURCE} + ${MEM_ALLOC_SHARED_SOURCE} + ${UTILS_SHARED_SOURCE} + ${LIBC_BUILTIN_SOURCE} + ${LIBC_WASI_SOURCE} + ${WASI_NN_SOURCES} + ${IWASM_COMMON_SOURCE} + ${IWASM_INTERP_SOURCE} + ${IWASM_AOT_SOURCE} + ${IWASM_COMPL_SOURCE} + ${IWASM_FAST_JIT_SOURCE} + ${IWASM_GC_SOURCE} + ${LIB_WASI_THREADS_SOURCE} + ${LIB_PTHREAD_SOURCE} + ${THREAD_MGR_SOURCE} + ${LIBC_EMCC_SOURCE} + ${LIB_RATS_SOURCE} + ${DEBUG_ENGINE_SOURCE} +) + +set (WAMR_RUNTIME_LIB_SOURCE ${source_all}) diff --git a/wasm-micro-runtime/ci/build_wamr.sh b/wasm-micro-runtime/ci/build_wamr.sh new file mode 100755 index 0000000..2ab42f2 --- /dev/null +++ b/wasm-micro-runtime/ci/build_wamr.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +readonly CURRENT_PATH=$(dirname "$(realpath "$0")") +readonly ROOT=$(realpath "${CURRENT_PATH}/..") +readonly VARIANT=$(lsb_release -c | awk '{print $2}') + +docker build \ + --memory=4G --cpu-quota=50000 \ + -t wamr_dev_${VARIANT}:0.1 -f "${ROOT}"/.devcontainer/Dockerfile "${ROOT}"/.devcontainer \ + && docker run --rm -it \ + --cap-add=SYS_PTRACE \ + --cpus=".5" \ + --memory=4G \ + --mount type=bind,src="${ROOT}",dst=/workspace \ + --name wamr_build_env \ + --security-opt=seccomp=unconfined \ + wamr_dev_${VARIANT}:0.1 \ + /bin/bash -c "\ + pwd \ + && pushd product-mini/platforms/linux \ + && rm -rf build \ + && mkdir build \ + && pushd build \ + && cmake .. \ + && make \ + && popd \ + && popd \ + && echo 'Copying the binary ...' \ + && rm -rf build_out \ + && mkdir build_out \ + && cp product-mini/platforms/linux/build/iwasm build_out/iwasm" diff --git a/wasm-micro-runtime/ci/coding_guidelines_check.py b/wasm-micro-runtime/ci/coding_guidelines_check.py new file mode 100644 index 0000000..924a242 --- /dev/null +++ b/wasm-micro-runtime/ci/coding_guidelines_check.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +import argparse +import re +import pathlib +import re +import shlex +import shutil +import subprocess +import sys +import unittest + +CLANG_FORMAT_CMD = "clang-format-12" +GIT_CLANG_FORMAT_CMD = "git-clang-format-12" + +# glob style patterns +EXCLUDE_PATHS = [ + "**/.git/*", + "**/.github/*", + "**/.vscode/*", + "**/build/*", + "**/build-scripts/*", + "**/ci/*", + "**/core/deps/*", + "**/doc/*", + "**/samples/wasm-c-api/src/*.*", + "**/samples/workload/*", + "**/test-tools/wasi-sdk/*", + "**/tests/wamr-test-suites/workspace/*", +] + +C_SUFFIXES = [".c", ".cpp", ".h"] +INVALID_DIR_NAME_SEGMENT = r"([a-zA-Z0-9]+\_[a-zA-Z0-9]+)" +INVALID_FILE_NAME_SEGMENT = r"([a-zA-Z0-9]+\-[a-zA-Z0-9]+)" + + +def locate_command(command: str) -> bool: + if not shutil.which(command): + print(f"Command '{command}'' not found") + return False + + return True + + +def is_excluded(path: str) -> bool: + path = pathlib.Path(path).resolve() + for exclude_path in EXCLUDE_PATHS: + if path.match(exclude_path): + return True + return False + + +def pre_flight_check(root: pathlib) -> bool: + def check_aspell(root): + return True + + def check_clang_foramt(root: pathlib) -> bool: + if not locate_command(CLANG_FORMAT_CMD): + return False + + # Quick syntax check for .clang-format + try: + subprocess.check_output( + shlex.split(f"{CLANG_FORMAT_CMD} --dump-config"), cwd=root + ) + except subprocess.CalledProcessError: + print(f"Might have a typo in .clang-format") + return False + return True + + def check_git_clang_format() -> bool: + return locate_command(GIT_CLANG_FORMAT_CMD) + + return check_aspell(root) and check_clang_foramt(root) and check_git_clang_format() + + +def run_clang_format(file_path: pathlib, root: pathlib) -> bool: + try: + subprocess.check_call( + shlex.split( + f"{CLANG_FORMAT_CMD} --style=file --Werror --dry-run {file_path}" + ), + cwd=root, + ) + return True + except subprocess.CalledProcessError: + print(f"{file_path} failed the check of {CLANG_FORMAT_CMD}") + return False + + +def run_clang_format_diff(root: pathlib, commits: str) -> bool: + """ + Use `clang-format-12` or `git-clang-format-12` to check code format of + the PR, with a commit range specified. It is required to format the + code before committing the PR, or it might fail to pass the CI check: + + 1. Install clang-format-12.0.0 + Normally we can install it by `sudo apt-get install clang-format-12`, + or download the `clang+llvm-12.0.0-xxx-tar.xz` package from + https://github.com/llvm/llvm-project/releases/tag/llvmorg-12.0.0 + and install it + + 2. Format the C/C++ source file + ``` shell + cd path/to/wamr/root + clang-format-12 --style file -i path/to/file + ``` + + The code wrapped by `/* clang-format off */` and `/* clang-format on */` + will not be formatted, you shall use them when the formatted code is not + readable or friendly: + + ``` cc + /* clang-format off */ + code snippets + /* clang-format on */ + ``` + + """ + try: + before, after = commits.split("..") + after = after if after else "HEAD" + COMMAND = ( + f"{GIT_CLANG_FORMAT_CMD} -v --binary " + f"{shutil.which(CLANG_FORMAT_CMD)} --style file " + f"--extensions c,cpp,h --diff {before} {after}" + ) + + p = subprocess.Popen( + shlex.split(COMMAND), + stdout=subprocess.PIPE, + stderr=None, + stdin=None, + universal_newlines=True, + ) + + stdout, _ = p.communicate() + if not stdout.startswith("diff --git"): + return True + + diff_content = stdout.split("\n") + found = False + for summary in [x for x in diff_content if x.startswith("diff --git")]: + # b/path/to/file -> path/to/file + with_invalid_format = re.split("\s+", summary)[-1][2:] + if not is_excluded(with_invalid_format): + print(f"--- {with_invalid_format} failed on code style checking.") + found = True + else: + return not found + except subprocess.subprocess.CalledProcessError: + return False + + +def run_aspell(file_path: pathlib, root: pathlib) -> bool: + return True + + +def check_dir_name(path: pathlib, root: pathlib) -> bool: + m = re.search(INVALID_DIR_NAME_SEGMENT, str(path.relative_to(root).parent)) + if m: + print(f"--- found a character '_' in {m.groups()} in {path}") + + return not m + + +def check_file_name(path: pathlib) -> bool: + m = re.search(INVALID_FILE_NAME_SEGMENT, path.stem) + if m: + print(f"--- found a character '-' in {m.groups()} in {path}") + + return not m + + +def parse_commits_range(root: pathlib, commits: str) -> list: + GIT_LOG_CMD = f"git log --pretty='%H' {commits}" + try: + ret = subprocess.check_output( + shlex.split(GIT_LOG_CMD), cwd=root, universal_newlines=True + ) + return [x for x in ret.split("\n") if x] + except subprocess.CalledProcessError: + print(f"can not parse any commit from the range {commits}") + return [] + + +def analysis_new_item_name(root: pathlib, commit: str) -> bool: + """ + For any file name in the repo, it is required to use '_' to replace '-'. + + For any directory name in the repo, it is required to use '-' to replace '_'. + """ + GIT_SHOW_CMD = f"git show --oneline --name-status --diff-filter A {commit}" + try: + invalid_items = True + output = subprocess.check_output( + shlex.split(GIT_SHOW_CMD), cwd=root, universal_newlines=True + ) + if not output: + return True + + NEW_FILE_PATTERN = "^A\s+(\S+)" + for line_no, line in enumerate(output.split("\n")): + # bypass the first line, usually it is the commit description + if line_no == 0: + continue + + if not line: + continue + + match = re.match(NEW_FILE_PATTERN, line) + if not match: + continue + + new_item = match.group(1) + new_item = pathlib.Path(new_item).resolve() + + if new_item.is_file(): + if not check_file_name(new_item): + invalid_items = False + continue + + new_item = new_item.parent + + if not check_dir_name(new_item, root): + invalid_items = False + continue + else: + return invalid_items + + except subprocess.CalledProcessError: + return False + + +def process_entire_pr(root: pathlib, commits: str) -> bool: + if not commits: + print("Please provide a commits range") + return False + + commit_list = parse_commits_range(root, commits) + if not commit_list: + print(f"Quit since there is no commit to check with") + return True + + print(f"there are {len(commit_list)} commits in the PR") + + found = False + if not analysis_new_item_name(root, commits): + print(f"{analysis_new_item_name.__doc__}") + found = True + + if not run_clang_format_diff(root, commits): + print(f"{run_clang_format_diff.__doc__}") + found = True + + return not found + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check if change meets all coding guideline requirements" + ) + parser.add_argument( + "-c", "--commits", default=None, help="Commit range in the form: a..b" + ) + options = parser.parse_args() + + wamr_root = pathlib.Path(__file__).parent.joinpath("..").resolve() + + if not pre_flight_check(wamr_root): + return False + + return process_entire_pr(wamr_root, options.commits) + + +# run with python3 -m unitest ci/coding_guidelines_check.py +class TestCheck(unittest.TestCase): + def test_check_dir_name_failed(self): + root = pathlib.Path("/root/Workspace/") + new_file_path = root.joinpath("core/shared/platform/esp_idf/espid_memmap.c") + self.assertFalse(check_dir_name(new_file_path, root)) + + def test_check_dir_name_pass(self): + root = pathlib.Path("/root/Workspace/") + new_file_path = root.joinpath("core/shared/platform/esp-idf/espid_memmap.c") + self.assertTrue(check_dir_name(new_file_path, root)) + + def test_check_file_name_failed(self): + new_file_path = pathlib.Path( + "/root/Workspace/core/shared/platform/esp-idf/espid-memmap.c" + ) + self.assertFalse(check_file_name(new_file_path)) + + def test_check_file_name_pass(self): + new_file_path = pathlib.Path( + "/root/Workspace/core/shared/platform/esp-idf/espid_memmap.c" + ) + self.assertTrue(check_file_name(new_file_path)) + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/wasm-micro-runtime/ci/pre_commit_hook_sample b/wasm-micro-runtime/ci/pre_commit_hook_sample new file mode 100755 index 0000000..682e789 --- /dev/null +++ b/wasm-micro-runtime/ci/pre_commit_hook_sample @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This is a sample of pre-commit hook that can be used to make your code fit the WAMR CI code style requirements. +# You need to have clang-format-12 installed to use this hook. +# To add this pre-commit hook, copy it to /.git/hooks/pre-commit +# (you don't need any extensions here) + +# Function to check if a file has a C or C++ extension + +is_c_or_cpp_file() { + file="$1" + if [[ "$filename" =~ \.(h|c|cpp)$ ]]; then + return 0 + else + return 1 + fi +} + +# Loop through staged files and apply command "abc" to C and C++ files +for staged_file in $(git diff --cached --name-only); do + if is_c_or_cpp_file "$staged_file"; then + clang-format-12 -Werror --style file --dry-run "$staged_file" 2>/dev/null + if [ $? -ne 0 ]; then + echo "Issues are found in $staged_file. Applying the fix" + clang-format-12 --style file -i "$staged_file" + fi + git add "$staged_file" # Add the modified file back to staging + fi +done diff --git a/wasm-micro-runtime/ci/setup.sh b/wasm-micro-runtime/ci/setup.sh new file mode 100755 index 0000000..ebdf73a --- /dev/null +++ b/wasm-micro-runtime/ci/setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This script executes some commands to make your onboarding with WAMR easier. +# For example, setting pre-commit hook that will make your code complaint with the +# code style requirements checked in WAMR CI + +echo "Copy the pre-commit hook to your hooks folder" +cp pre_commit_hook_sample ../.git/hooks/pre-commit + +# Feel free to propose your commands to this script to make developing WAMR easier + +echo "Setup is done" diff --git a/wasm-micro-runtime/ci/validate_lldb.py b/wasm-micro-runtime/ci/validate_lldb.py new file mode 100755 index 0000000..1b431ad --- /dev/null +++ b/wasm-micro-runtime/ci/validate_lldb.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import argparse +import time +from pathlib import Path +import subprocess, shlex + +SCRIPT_DIR = Path(__file__).parent.resolve() +REPO_ROOT_DIR = SCRIPT_DIR.parent +SAMPLE_CODE_FILE = REPO_ROOT_DIR / 'product-mini/app-samples/hello-world/main.c' +WASM_OUT_FILE = SCRIPT_DIR / 'out.wasm' + +parser = argparse.ArgumentParser( + description="Validate the customized lldb with sample code" +) +parser.add_argument( + "-l", "--lldb", dest='lldb', default='lldb', help="path to lldb executable" +) +parser.add_argument( + "-w", "--wamr", dest='wamr', default='iwasm', help="path to iwasm executable" +) +parser.add_argument( + "-p", "--port", dest='port', default='1234', help="debug server listen port" +) +parser.add_argument( + "-v", "--verbose", dest='verbose', action='store_true', default=False, help="display lldb stdout" +) + +options = parser.parse_args() + +lldb_command_epilogue = '-o q' + +test_cases = { + 'run_to_exit': '-o c', + 'func_breakpoint': '-o "b main" -o c -o c', + 'line_breakpoint': '-o "b main.c:12" -o c -o c', + 'break_on_unknown_func': '-o "b not_a_func" -o c', + 'watch_point': '-o "b main" -o c -o "watchpoint set variable buf" -o c -o "fr v buf" -o c', +} + +# Step1: Build wasm module with debug information +build_cmd = f'/opt/wasi-sdk/bin/clang -g -O0 -o {WASM_OUT_FILE} {SAMPLE_CODE_FILE}' +try: + print(f'building wasm module ...', end='', flush=True) + subprocess.check_call(shlex.split(build_cmd)) + print(f'\t OK') +except subprocess.CalledProcessError: + print("Failed to build wasm module with debug information") + exit(1) + +def print_process_output(p): + try: + outs, errs = p.communicate(timeout=2) + print("stdout:") + print(outs) + print("stderr:") + print(errs) + except subprocess.TimeoutExpired: + print("Failed to get process output") + +# Step2: Launch WAMR in debug mode and validate lldb commands + +iteration = 0 +for case, cmd in test_cases.items(): + lldb_command_prologue = f'{options.lldb} -o "process connect -p wasm connect://127.0.0.1:{int(options.port) + iteration}"' + wamr_cmd = f'{options.wamr} -g=127.0.0.1:{int(options.port) + iteration} {WASM_OUT_FILE}' + iteration += 1 + + has_error = False + print(f'validating case [{case}] ...', end='', flush=True) + lldb_cmd = f'{lldb_command_prologue} {cmd} {lldb_command_epilogue}' + + wamr_process = subprocess.Popen(shlex.split( + wamr_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + + time.sleep(0.1) + if (wamr_process.poll() != None): + print("\nWAMR doesn't wait for lldb connection") + print_process_output(wamr_process) + exit(1) + + lldb_process = subprocess.Popen(shlex.split( + lldb_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + + if (options.verbose): + while (lldb_process.poll() is None): + print(lldb_process.stdout.read(), end='', flush=True) + + try: + if (lldb_process.wait(5) != 0): + print(f"\nFailed to validate case [{case}]") + print_process_output(lldb_process) + has_error = True + + if wamr_process.wait(2) != 0: + print("\nWAMR process doesn't exit normally") + print_process_output(wamr_process) + has_error = True + + except subprocess.TimeoutExpired: + print(f"\nFailed to validate case [{case}]") + print("wamr output:") + print_process_output(wamr_process) + print("lldb output:") + print_process_output(lldb_process) + has_error = True + finally: + if (lldb_process.poll() == None): + print(f'\nterminating lldb process [{lldb_process.pid}]') + lldb_process.kill() + if (wamr_process.poll() == None): + print(f'terminating wamr process [{wamr_process.pid}]') + wamr_process.kill() + + if (has_error): + exit(1) + + print(f'\t OK') + + # wait 100ms to ensure the socket is closed + time.sleep(0.1) + +print('Validate lldb success') +exit(0) diff --git a/wasm-micro-runtime/core/config.h b/wasm-micro-runtime/core/config.h new file mode 100644 index 0000000..c341a9a --- /dev/null +++ b/wasm-micro-runtime/core/config.h @@ -0,0 +1,629 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _CONFIG_H_ +#define _CONFIG_H_ + +/* clang-format off */ +#if !defined(BUILD_TARGET_X86_64) \ + && !defined(BUILD_TARGET_AMD_64) \ + && !defined(BUILD_TARGET_AARCH64) \ + && !defined(BUILD_TARGET_X86_32) \ + && !defined(BUILD_TARGET_ARM) \ + && !defined(BUILD_TARGET_ARM_VFP) \ + && !defined(BUILD_TARGET_THUMB) \ + && !defined(BUILD_TARGET_THUMB_VFP) \ + && !defined(BUILD_TARGET_MIPS) \ + && !defined(BUILD_TARGET_XTENSA) \ + && !defined(BUILD_TARGET_RISCV64_LP64D) \ + && !defined(BUILD_TARGET_RISCV64_LP64) \ + && !defined(BUILD_TARGET_RISCV32_ILP32D) \ + && !defined(BUILD_TARGET_RISCV32_ILP32) \ + && !defined(BUILD_TARGET_ARC) +/* clang-format on */ +#if defined(__x86_64__) || defined(__x86_64) +#define BUILD_TARGET_X86_64 +#elif defined(__amd64__) || defined(__amd64) +#define BUILD_TARGET_AMD_64 +#elif defined(__aarch64__) +#define BUILD_TARGET_AARCH64 +#elif defined(__i386__) || defined(__i386) || defined(i386) +#define BUILD_TARGET_X86_32 +#elif defined(__thumb__) +#define BUILD_TARGET_THUMB +#define BUILD_TARGET "THUMBV4T" +#elif defined(__arm__) +#define BUILD_TARGET_ARM +#define BUILD_TARGET "ARMV4T" +#elif defined(__mips__) || defined(__mips) || defined(mips) +#define BUILD_TARGET_MIPS +#elif defined(__XTENSA__) +#define BUILD_TARGET_XTENSA +#elif defined(__riscv) && (__riscv_xlen == 64) +#define BUILD_TARGET_RISCV64_LP64D +#elif defined(__riscv) && (__riscv_xlen == 32) +#define BUILD_TARGET_RISCV32_ILP32D +#elif defined(__arc__) +#define BUILD_TARGET_ARC +#else +#error "Build target isn't set" +#endif +#endif + +#ifndef BH_DEBUG +#define BH_DEBUG 0 +#endif + +#define MEM_ALLOCATOR_EMS 0 +#define MEM_ALLOCATOR_TLSF 1 + +/* Default memory allocator */ +#define DEFAULT_MEM_ALLOCATOR MEM_ALLOCATOR_EMS + +#ifndef WASM_ENABLE_INTERP +#define WASM_ENABLE_INTERP 0 +#endif + +#ifndef WASM_ENABLE_AOT +#define WASM_ENABLE_AOT 0 +#endif + +#ifndef WASM_ENABLE_WORD_ALIGN_READ +#define WASM_ENABLE_WORD_ALIGN_READ 0 +#endif + +#define AOT_MAGIC_NUMBER 0x746f6100 +#define AOT_CURRENT_VERSION 3 + +#ifndef WASM_ENABLE_JIT +#define WASM_ENABLE_JIT 0 +#endif + +#ifndef WASM_ENABLE_LAZY_JIT +#define WASM_ENABLE_LAZY_JIT 0 +#endif + +#ifndef WASM_ORC_JIT_BACKEND_THREAD_NUM +/* The number of backend threads created by runtime */ +#define WASM_ORC_JIT_BACKEND_THREAD_NUM 4 +#endif + +#if WASM_ORC_JIT_BACKEND_THREAD_NUM < 1 +#error "WASM_ORC_JIT_BACKEND_THREAD_NUM must be greater than 0" +#endif + +#ifndef WASM_ORC_JIT_COMPILE_THREAD_NUM +/* The number of compilation threads created by LLVM JIT */ +#define WASM_ORC_JIT_COMPILE_THREAD_NUM 4 +#endif + +#if WASM_ORC_JIT_COMPILE_THREAD_NUM < 1 +#error "WASM_ORC_JIT_COMPILE_THREAD_NUM must be greater than 0" +#endif + +#if (WASM_ENABLE_AOT == 0) && (WASM_ENABLE_JIT != 0) +/* LLVM JIT can only be enabled when AOT is enabled */ +#undef WASM_ENABLE_JIT +#define WASM_ENABLE_JIT 0 + +#undef WASM_ENABLE_LAZY_JIT +#define WASM_ENABLE_LAZY_JIT 0 +#endif + +#ifndef WASM_ENABLE_FAST_JIT +#define WASM_ENABLE_FAST_JIT 0 +#endif + +#ifndef WASM_ENABLE_FAST_JIT_DUMP +#define WASM_ENABLE_FAST_JIT_DUMP 0 +#endif + +#ifndef FAST_JIT_DEFAULT_CODE_CACHE_SIZE +#define FAST_JIT_DEFAULT_CODE_CACHE_SIZE 10 * 1024 * 1024 +#endif + +#ifndef WASM_ENABLE_WAMR_COMPILER +#define WASM_ENABLE_WAMR_COMPILER 0 +#endif + +#ifndef WASM_ENABLE_LIBC_BUILTIN +#define WASM_ENABLE_LIBC_BUILTIN 0 +#endif + +#ifndef WASM_ENABLE_LIBC_WASI +#define WASM_ENABLE_LIBC_WASI 0 +#endif + +#ifndef WASM_ENABLE_UVWASI +#define WASM_ENABLE_UVWASI 0 +#endif + +#ifndef WASM_ENABLE_WASI_NN +#define WASM_ENABLE_WASI_NN 0 +#endif + +#ifndef WASM_ENABLE_WASI_NN_GPU +#define WASM_ENABLE_WASI_NN_GPU 0 +#endif + +#ifndef WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE +#define WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE 0 +#endif + +#ifndef WASM_ENABLE_WASI_EPHEMERAL_NN +#define WASM_ENABLE_WASI_EPHEMERAL_NN 0 +#endif + +/* Default disable libc emcc */ +#ifndef WASM_ENABLE_LIBC_EMCC +#define WASM_ENABLE_LIBC_EMCC 0 +#endif + +#ifndef WASM_ENABLE_LIB_RATS +#define WASM_ENABLE_LIB_RATS 0 +#endif + +#ifndef WASM_ENABLE_LIB_PTHREAD +#define WASM_ENABLE_LIB_PTHREAD 0 +#endif + +#ifndef WASM_ENABLE_LIB_PTHREAD_SEMAPHORE +#define WASM_ENABLE_LIB_PTHREAD_SEMAPHORE 0 +#endif + +#ifndef WASM_ENABLE_LIB_WASI_THREADS +#define WASM_ENABLE_LIB_WASI_THREADS 0 +#endif + +#ifndef WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION +#define WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION WASM_ENABLE_LIB_WASI_THREADS +#elif WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0 \ + && WASM_ENABLE_LIB_WASI_THREADS == 1 +#error "Heap aux stack allocation must be enabled for WASI threads" +#endif + +#ifndef WASM_ENABLE_BASE_LIB +#define WASM_ENABLE_BASE_LIB 0 +#endif + +#ifndef WASM_ENABLE_APP_FRAMEWORK +#define WASM_ENABLE_APP_FRAMEWORK 0 +#endif + +#ifndef WASM_HAVE_MREMAP +#define WASM_HAVE_MREMAP 0 +#endif + +/* Bulk memory operation */ +#ifndef WASM_ENABLE_BULK_MEMORY +#define WASM_ENABLE_BULK_MEMORY 0 +#endif + +/* Shared memory */ +#ifndef WASM_ENABLE_SHARED_MEMORY +#define WASM_ENABLE_SHARED_MEMORY 0 +#endif + +/* Thread manager */ +#ifndef WASM_ENABLE_THREAD_MGR +#define WASM_ENABLE_THREAD_MGR 0 +#endif + +/* Source debugging */ +#ifndef WASM_ENABLE_DEBUG_INTERP +#define WASM_ENABLE_DEBUG_INTERP 0 +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 +#ifndef DEBUG_EXECUTION_MEMORY_SIZE +/* 0x85000 is the size required by lldb, if this is changed to a smaller value, + * then the debugger will not be able to evaluate user expressions, other + * functionality such as breakpoint and stepping are not influenced by this */ +#define DEBUG_EXECUTION_MEMORY_SIZE 0x85000 +#endif +#endif /* end of WASM_ENABLE_DEBUG_INTERP != 0 */ + +#ifndef WASM_ENABLE_DEBUG_AOT +#define WASM_ENABLE_DEBUG_AOT 0 +#endif + +/* Custom sections */ +#ifndef WASM_ENABLE_LOAD_CUSTOM_SECTION +#define WASM_ENABLE_LOAD_CUSTOM_SECTION 0 +#endif + +/* WASM log system */ +#ifndef WASM_ENABLE_LOG +#define WASM_ENABLE_LOG 1 +#endif + +/* When this flag is set, WAMR will not automatically + * initialize sockets on Windows platforms. The host + * application is responsible for calling WSAStartup() + * before executing WAMR code that uses sockets, and + * calling WSACleanup() after. + * This flag passes control of socket initialization from + * WAMR to the host application. */ +#ifndef WASM_ENABLE_HOST_SOCKET_INIT +#define WASM_ENABLE_HOST_SOCKET_INIT 0 +#endif + +#ifndef WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS +#if defined(BUILD_TARGET_X86_32) || defined(BUILD_TARGET_X86_64) \ + || defined(BUILD_TARGET_AARCH64) +#define WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS 1 +#else +#define WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS 0 +#endif +#endif + +/* WASM Interpreter labels-as-values feature */ +#ifndef WASM_ENABLE_LABELS_AS_VALUES +#ifdef __GNUC__ +#define WASM_ENABLE_LABELS_AS_VALUES 1 +#else +#define WASM_ENABLE_LABELS_AS_VALUES 0 +#endif +#endif + +/* Enable fast interpreter or not */ +#ifndef WASM_ENABLE_FAST_INTERP +#define WASM_ENABLE_FAST_INTERP 0 +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 +#define WASM_DEBUG_PREPROCESSOR 0 +#endif + +/* Enable opcode counter or not */ +#ifndef WASM_ENABLE_OPCODE_COUNTER +#define WASM_ENABLE_OPCODE_COUNTER 0 +#endif + +/* Support a module with dependency, other modules */ +#ifndef WASM_ENABLE_MULTI_MODULE +#define WASM_ENABLE_MULTI_MODULE 0 +#endif + +/* Enable wasm mini loader or not */ +#ifndef WASM_ENABLE_MINI_LOADER +#define WASM_ENABLE_MINI_LOADER 0 +#endif + +/* Disable boundary check with hardware trap or not, + * enable it by default if it is supported */ +#ifndef WASM_DISABLE_HW_BOUND_CHECK +#define WASM_DISABLE_HW_BOUND_CHECK 0 +#endif + +/* Disable native stack access boundary check with hardware + * trap or not, enable it by default if it is supported */ +#ifndef WASM_DISABLE_STACK_HW_BOUND_CHECK +#define WASM_DISABLE_STACK_HW_BOUND_CHECK 0 +#endif + +/* Disable SIMD unless it is manualy enabled somewhere */ +#ifndef WASM_ENABLE_SIMD +#define WASM_ENABLE_SIMD 0 +#endif + +/* GC performance profiling */ +#ifndef WASM_ENABLE_GC_PERF_PROFILING +#define WASM_ENABLE_GC_PERF_PROFILING 0 +#endif + +/* Memory profiling */ +#ifndef WASM_ENABLE_MEMORY_PROFILING +#define WASM_ENABLE_MEMORY_PROFILING 0 +#endif + +/* Memory tracing */ +#ifndef WASM_ENABLE_MEMORY_TRACING +#define WASM_ENABLE_MEMORY_TRACING 0 +#endif + +/* Performance profiling */ +#ifndef WASM_ENABLE_PERF_PROFILING +#define WASM_ENABLE_PERF_PROFILING 0 +#endif + +/* Dump call stack */ +#ifndef WASM_ENABLE_DUMP_CALL_STACK +#define WASM_ENABLE_DUMP_CALL_STACK 0 +#endif + +/* AOT stack frame */ +#ifndef WASM_ENABLE_AOT_STACK_FRAME +#define WASM_ENABLE_AOT_STACK_FRAME 0 +#endif + +/* Heap verification */ +#ifndef BH_ENABLE_GC_VERIFY +#define BH_ENABLE_GC_VERIFY 0 +#endif + +/* Heap corruption check, enabled by default */ +#ifndef BH_ENABLE_GC_CORRUPTION_CHECK +#define BH_ENABLE_GC_CORRUPTION_CHECK 1 +#endif + +/* Enable global heap pool if heap verification is enabled */ +#if BH_ENABLE_GC_VERIFY != 0 +#define WASM_ENABLE_GLOBAL_HEAP_POOL 1 +#endif + +/* Global heap pool */ +#ifndef WASM_ENABLE_GLOBAL_HEAP_POOL +#define WASM_ENABLE_GLOBAL_HEAP_POOL 0 +#endif + +#ifndef WASM_ENABLE_SPEC_TEST +#define WASM_ENABLE_SPEC_TEST 0 +#endif + +/* Global heap pool size in bytes */ +#ifndef WASM_GLOBAL_HEAP_SIZE +#define WASM_GLOBAL_HEAP_SIZE (10 * 1024 * 1024) +#endif + +/* Max app number of all modules */ +#define MAX_APP_INSTALLATIONS 3 + +/* Default timer number in one app */ +#define DEFAULT_TIMERS_PER_APP 20 + +/* Max timer number in one app */ +#define MAX_TIMERS_PER_APP 30 + +/* Max connection number in one app */ +#define MAX_CONNECTION_PER_APP 20 + +/* Max resource registration number in one app */ +#define RESOURCE_REGISTRATION_NUM_MAX 16 + +/* Max length of resource/event url */ +#define RESOUCE_EVENT_URL_LEN_MAX 256 + +/* Default length of queue */ +#define DEFAULT_QUEUE_LENGTH 50 + +/* Default watchdog interval in ms */ +#define DEFAULT_WATCHDOG_INTERVAL (3 * 60 * 1000) + +/* The max percentage of global heap that app memory space can grow */ +#define APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT 1 / 3 + +/* Default min/max heap size of each app */ +#ifndef APP_HEAP_SIZE_DEFAULT +#define APP_HEAP_SIZE_DEFAULT (8 * 1024) +#endif +#define APP_HEAP_SIZE_MIN (256) +#define APP_HEAP_SIZE_MAX (512 * 1024 * 1024) + +/* Default min/max gc heap size of each app */ +#ifndef GC_HEAP_SIZE_DEFAULT +#define GC_HEAP_SIZE_DEFAULT (128 * 1024) +#endif +#define GC_HEAP_SIZE_MIN (4 * 1024) +#define GC_HEAP_SIZE_MAX (1024 * 1024 * 1024) + +/* Default wasm stack size of each app */ +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +#define DEFAULT_WASM_STACK_SIZE (16 * 1024) +#else +#define DEFAULT_WASM_STACK_SIZE (12 * 1024) +#endif +/* Min auxiliary stack size of each wasm thread */ +#define WASM_THREAD_AUX_STACK_SIZE_MIN (256) + +/* Default/min native stack size of each app thread */ +#if !(defined(APP_THREAD_STACK_SIZE_DEFAULT) \ + && defined(APP_THREAD_STACK_SIZE_MIN)) +#if defined(BH_PLATFORM_ZEPHYR) || defined(BH_PLATFORM_ALIOS_THINGS) \ + || defined(BH_PLATFORM_ESP_IDF) || defined(BH_PLATFORM_OPENRTOS) +#define APP_THREAD_STACK_SIZE_DEFAULT (6 * 1024) +#define APP_THREAD_STACK_SIZE_MIN (4 * 1024) +#elif defined(PTHREAD_STACK_DEFAULT) && defined(PTHREAD_STACK_MIN) +#define APP_THREAD_STACK_SIZE_DEFAULT PTHREAD_STACK_DEFAULT +#define APP_THREAD_STACK_SIZE_MIN PTHREAD_STACK_MIN +#elif WASM_ENABLE_UVWASI != 0 +/* UVWASI requires larger native stack */ +#define APP_THREAD_STACK_SIZE_DEFAULT (64 * 1024) +#define APP_THREAD_STACK_SIZE_MIN (48 * 1024) +#else +#define APP_THREAD_STACK_SIZE_DEFAULT (128 * 1024) +#define APP_THREAD_STACK_SIZE_MIN (24 * 1024) +#endif +#endif /* end of !(defined(APP_THREAD_STACK_SIZE_DEFAULT) \ + && defined(APP_THREAD_STACK_SIZE_MIN)) */ + +/* Max native stack size of each app thread */ +#if !defined(APP_THREAD_STACK_SIZE_MAX) +#define APP_THREAD_STACK_SIZE_MAX (8 * 1024 * 1024) +#endif + +/* Reserved bytes to the native thread stack boundary, throw native + * stack overflow exception if the guard boudary is reached + * + * WASM_STACK_GUARD_SIZE needs to be large enough for: + * + * - native functions + * + * w/o hw bound check, the overhead (aot_call_function etc) + the native + * function itself. as of writing this, the former is about 1000 bytes + * on macOS amd64. + * + * with hw bound check, theoretically, only needs to cover the logic to + * set up the jmp_buf stack. + * + * - aot runtime functions + * eg. aot_enlarge_memory. + * + * - w/o hw bound check, the intepreter loop + * + * the classic interpreter wasm_interp_call_func_bytecode alone + * seems to consume about 2600 bytes stack. + * (with the default configuration for macOS/amd64) + * + * libc snprintf (used by eg. wasm_runtime_set_exception) consumes about + * 1600 bytes stack on macOS/amd64, about 2000 bytes on Ubuntu amd64 20.04. + * + * - stack check wrapper functions generated by the aot compiler + * (--stack-bounds-checks=1) + * + * wamrc issues a warning + * "precheck functions themselves consume relatively large amount of stack" + * when it detects wrapper functions requiring more than 1KB. + * + * Note: on platforms with lazy function binding, don't forget to consider + * the symbol resolution overhead on the first call. For example, + * on Ubuntu amd64 20.04, it seems to consume about 1500 bytes. + */ +#ifndef WASM_STACK_GUARD_SIZE +#if WASM_ENABLE_UVWASI != 0 +/* UVWASI requires larger native stack */ +#define WASM_STACK_GUARD_SIZE (4096 * 6) +#else +#define WASM_STACK_GUARD_SIZE (1024) +#endif +#endif + +/* Guard page count for stack overflow check with hardware trap */ +#ifndef STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT +#define STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT 3 +#endif + +/* Default wasm block address cache size and conflict list size */ +#ifndef BLOCK_ADDR_CACHE_SIZE +#define BLOCK_ADDR_CACHE_SIZE 64 +#endif +#define BLOCK_ADDR_CONFLICT_SIZE 2 + +/* Default max thread num per cluster. Can be overwrite by + wasm_runtime_set_max_thread_num */ +#define CLUSTER_MAX_THREAD_NUM 4 + +#ifndef WASM_ENABLE_TAIL_CALL +#define WASM_ENABLE_TAIL_CALL 0 +#endif + +#ifndef WASM_ENABLE_CUSTOM_NAME_SECTION +#define WASM_ENABLE_CUSTOM_NAME_SECTION 0 +#endif + +#ifndef WASM_ENABLE_REF_TYPES +#define WASM_ENABLE_REF_TYPES 0 +#endif + +#ifndef WASM_ENABLE_GC +#define WASM_ENABLE_GC 0 +#endif + +#ifndef WASM_CONST_EXPR_STACK_SIZE +#if WASM_ENABLE_GC != 0 +#define WASM_CONST_EXPR_STACK_SIZE 8 +#else +#define WASM_CONST_EXPR_STACK_SIZE 4 +#endif +#endif + +#ifndef WASM_ENABLE_STRINGREF +#define WASM_ENABLE_STRINGREF 0 +#endif + +#ifndef GC_REFTYPE_MAP_SIZE_DEFAULT +#define GC_REFTYPE_MAP_SIZE_DEFAULT 64 +#endif + +#ifndef GC_RTTOBJ_MAP_SIZE_DEFAULT +#define GC_RTTOBJ_MAP_SIZE_DEFAULT 64 +#endif + +#ifndef WASM_ENABLE_EXCE_HANDLING +#define WASM_ENABLE_EXCE_HANDLING 0 +#endif + +#ifndef WASM_ENABLE_TAGS +#define WASM_ENABLE_TAGS 0 +#endif + +#ifndef WASM_ENABLE_SGX_IPFS +#define WASM_ENABLE_SGX_IPFS 0 +#endif + +#ifndef WASM_MEM_ALLOC_WITH_USER_DATA +#define WASM_MEM_ALLOC_WITH_USER_DATA 0 +#endif + +#ifndef WASM_ENABLE_WASM_CACHE +#define WASM_ENABLE_WASM_CACHE 0 +#endif + +#ifndef WASM_ENABLE_STATIC_PGO +#define WASM_ENABLE_STATIC_PGO 0 +#endif + +/* Disable writing linear memory base address to GS segment register, + by default only in linux x86-64, linear memory base addr is written + to GS segment register before calling wasm/aot function. */ +#ifndef WASM_DISABLE_WRITE_GS_BASE +#define WASM_DISABLE_WRITE_GS_BASE 0 +#endif + +/* Configurable bounds checks */ +#ifndef WASM_CONFIGURABLE_BOUNDS_CHECKS +#define WASM_CONFIGURABLE_BOUNDS_CHECKS 0 +#endif + +/* Some chip cannot support external ram with rwx attr at the same time, + it has to map it into 2 spaces of idbus and dbus, code in dbus can be + read/written and read/executed in ibus. so there are 2 steps to execute + the code, first, copy & do relocation in dbus space, and second execute + it in ibus space, since in the 2 spaces the contents are the same, + so we call it bus mirror. + */ +#ifndef WASM_MEM_DUAL_BUS_MIRROR +#define WASM_MEM_DUAL_BUS_MIRROR 0 +#endif + +/* The max number of module instance contexts. */ +#ifndef WASM_MAX_INSTANCE_CONTEXTS +#define WASM_MAX_INSTANCE_CONTEXTS 8 +#endif + +/* linux perf support */ +#ifndef WASM_ENABLE_LINUX_PERF +#define WASM_ENABLE_LINUX_PERF 0 +#endif + +/* Support registering quick AOT/JIT function entries of some func types + to speed up the calling process of invoking the AOT/JIT functions of + these types from the host embedder */ +#ifndef WASM_ENABLE_QUICK_AOT_ENTRY +#define WASM_ENABLE_QUICK_AOT_ENTRY 1 +#endif + +/* Support AOT intrinsic functions which can be called from the AOT code + when `--disable-llvm-intrinsics` flag or + `--enable-builtin-intrinsics=` is used by wamrc to + generate the AOT file */ +#ifndef WASM_ENABLE_AOT_INTRINSICS +#define WASM_ENABLE_AOT_INTRINSICS 1 +#endif + +/* Disable memory64 by default */ +#ifndef WASM_ENABLE_MEMORY64 +#define WASM_ENABLE_MEMORY64 0 +#endif + +#ifndef WASM_TABLE_MAX_SIZE +#define WASM_TABLE_MAX_SIZE 1024 +#endif + +#ifndef WASM_MEM_ALLOC_WITH_USAGE +#define WASM_MEM_ALLOC_WITH_USAGE 0 +#endif + +#endif /* end of _CONFIG_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/README.md b/wasm-micro-runtime/core/iwasm/README.md new file mode 100644 index 0000000..65ab78a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/README.md @@ -0,0 +1,14 @@ +# vmcore architecture +- [WAMR memory model overview](https://bytecodealliance.github.io/wamr.dev/blog/the-wamr-memory-model/) + +## Wasm function +- [Wasm function architecture](./doc/wasm_function.MD) + +## Exports +- [Wasm export architecture](./doc/wasm_exports.MD) + +## globals +- [Wasm globals architecture](./doc/wasm_globals.MD) + +## classic interpreter +- [classic interpreter](./doc/classic_interpreter.MD) \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/aot/SConscript b/wasm-micro-runtime/core/iwasm/aot/SConscript new file mode 100644 index 0000000..790f284 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/SConscript @@ -0,0 +1,33 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * +import re +Import('rtconfig') + +cwd = GetCurrentDir() + +src = Split(''' +aot_loader.c +aot_runtime.c +aot_intrinsic.c +''') + +if rtconfig.ARCH == 'arm': + if re.match('^cortex-m.*', rtconfig.CPU): + src += ['arch/aot_reloc_thumb.c'] + elif re.match('^cortex-a.*', rtconfig.CPU): + src += ['arch/aot_reloc_arm.c'] +elif rtconfig.ARCH == 'ia32': + src += ['arch/aot_reloc_x86_32.c'] + +CPPPATH = [cwd, cwd + '/../include'] + +CPPDEFINES = ['WASM_ENABLE_AOT=1'] + +group = DefineGroup('iwasm_aot', src, depend = [''], CPPPATH = CPPPATH, CPPDEFINES = CPPDEFINES) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_intrinsic.c b/wasm-micro-runtime/core/iwasm/aot/aot_intrinsic.c new file mode 100644 index 0000000..7b455cb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_intrinsic.c @@ -0,0 +1,894 @@ +/* + * Copyright (C) 2021 XiaoMi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_intrinsic.h" + +float32 +aot_intrinsic_fadd_f32(float32 a, float32 b) +{ + return a + b; +} + +float64 +aot_intrinsic_fadd_f64(float64 a, float64 b) +{ + return a + b; +} + +float32 +aot_intrinsic_fsub_f32(float32 a, float32 b) +{ + return a - b; +} + +float64 +aot_intrinsic_fsub_f64(float64 a, float64 b) +{ + return a - b; +} + +float32 +aot_intrinsic_fmul_f32(float32 a, float32 b) +{ + return a * b; +} + +float64 +aot_intrinsic_fmul_f64(float64 a, float64 b) +{ + return a * b; +} + +float32 +aot_intrinsic_fdiv_f32(float32 a, float32 b) +{ + return a / b; +} + +float64 +aot_intrinsic_fdiv_f64(float64 a, float64 b) +{ + return a / b; +} + +float32 +aot_intrinsic_fabs_f32(float32 a) +{ + return fabsf(a); +} + +float64 +aot_intrinsic_fabs_f64(float64 a) +{ + return fabs(a); +} + +float32 +aot_intrinsic_ceil_f32(float32 a) +{ + return ceilf(a); +} + +float64 +aot_intrinsic_ceil_f64(float64 a) +{ + return ceil(a); +} + +float32 +aot_intrinsic_floor_f32(float32 a) +{ + return floorf(a); +} + +float64 +aot_intrinsic_floor_f64(float64 a) +{ + return floor(a); +} + +float32 +aot_intrinsic_trunc_f32(float32 a) +{ + return truncf(a); +} + +float64 +aot_intrinsic_trunc_f64(float64 a) +{ + return trunc(a); +} + +float32 +aot_intrinsic_rint_f32(float32 a) +{ + return rintf(a); +} + +float64 +aot_intrinsic_rint_f64(float64 a) +{ + return rint(a); +} + +float32 +aot_intrinsic_sqrt_f32(float32 a) +{ + return sqrtf(a); +} + +float64 +aot_intrinsic_sqrt_f64(float64 a) +{ + return sqrt(a); +} + +float32 +aot_intrinsic_copysign_f32(float32 a, float32 b) +{ + return signbit(b) ? -fabsf(a) : fabsf(a); +} + +float64 +aot_intrinsic_copysign_f64(float64 a, float64 b) +{ + return signbit(b) ? -fabs(a) : fabs(a); +} + +float32 +aot_intrinsic_fmin_f32(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +float64 +aot_intrinsic_fmin_f64(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +float32 +aot_intrinsic_fmax_f32(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +float64 +aot_intrinsic_fmax_f64(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +uint32 +aot_intrinsic_clz_i32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 0x80000000)) { + num++; + type <<= 1; + } + return num; +} + +uint32 +aot_intrinsic_clz_i64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 0x8000000000000000LL)) { + num++; + type <<= 1; + } + return num; +} + +uint32 +aot_intrinsic_ctz_i32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +uint32 +aot_intrinsic_ctz_i64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +uint32 +aot_intrinsic_popcnt_i32(uint32 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +uint32 +aot_intrinsic_popcnt_i64(uint64 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +float32 +aot_intrinsic_i32_to_f32(int32 i) +{ + return (float32)i; +} + +float32 +aot_intrinsic_u32_to_f32(uint32 u) +{ + return (float32)u; +} + +float64 +aot_intrinsic_i32_to_f64(int32 i) +{ + return (float64)i; +} + +float64 +aot_intrinsic_u32_to_f64(uint32 u) +{ + return (float64)u; +} + +float32 +aot_intrinsic_i64_to_f32(int64 i) +{ + return (float32)i; +} + +float32 +aot_intrinsic_u64_to_f32(uint64 u) +{ + return (float32)u; +} + +float64 +aot_intrinsic_i64_to_f64(int64 i) +{ + return (float64)i; +} + +float64 +aot_intrinsic_u64_to_f64(uint64 u) +{ + return (float64)u; +} + +int32 +aot_intrinsic_f32_to_i32(float32 f) +{ + return (int32)f; +} + +uint32 +aot_intrinsic_f32_to_u32(float32 f) +{ + return (uint32)f; +} + +int64 +aot_intrinsic_f32_to_i64(float32 f) +{ + return (int64)f; +} + +uint64 +aot_intrinsic_f32_to_u64(float32 f) +{ + return (uint64)f; +} + +int32 +aot_intrinsic_f64_to_i32(float64 f) +{ + return (int32)f; +} + +uint32 +aot_intrinsic_f64_to_u32(float64 f) +{ + return (uint32)f; +} + +int64 +aot_intrinsic_f64_to_i64(float64 f) +{ + return (int64)f; +} + +uint64 +aot_intrinsic_f64_to_u64(float64 f) +{ + return (uint64)f; +} + +float64 +aot_intrinsic_f32_to_f64(float32 f) +{ + return (float64)f; +} + +float32 +aot_intrinsic_f64_to_f32(float64 f) +{ + return (float32)f; +} + +int32 +aot_intrinsic_f32_cmp(AOTFloatCond cond, float32 lhs, float32 rhs) +{ + switch (cond) { + case FLOAT_EQ: + return lhs == rhs ? 1 : 0; + + case FLOAT_LT: + return lhs < rhs ? 1 : 0; + + case FLOAT_GT: + return lhs > rhs ? 1 : 0; + + case FLOAT_LE: + return lhs <= rhs ? 1 : 0; + + case FLOAT_GE: + return lhs >= rhs ? 1 : 0; + + case FLOAT_NE: + return (isnan(lhs) || isnan(rhs) || lhs != rhs) ? 1 : 0; + + case FLOAT_UNO: + return (isnan(lhs) || isnan(rhs)) ? 1 : 0; + + default: + break; + } + return 0; +} + +int32 +aot_intrinsic_f64_cmp(AOTFloatCond cond, float64 lhs, float64 rhs) +{ + switch (cond) { + case FLOAT_EQ: + return lhs == rhs ? 1 : 0; + + case FLOAT_LT: + return lhs < rhs ? 1 : 0; + + case FLOAT_GT: + return lhs > rhs ? 1 : 0; + + case FLOAT_LE: + return lhs <= rhs ? 1 : 0; + + case FLOAT_GE: + return lhs >= rhs ? 1 : 0; + + case FLOAT_NE: + return (isnan(lhs) || isnan(rhs) || lhs != rhs) ? 1 : 0; + + case FLOAT_UNO: + return (isnan(lhs) || isnan(rhs)) ? 1 : 0; + + default: + break; + } + return 0; +} + +int64 +aot_intrinsic_i64_div_s(int64 l, int64 r) +{ + return l / r; +} + +int32 +aot_intrinsic_i32_div_s(int32 l, int32 r) +{ + return l / r; +} + +uint32 +aot_intrinsic_i32_div_u(uint32 l, uint32 r) +{ + return l / r; +} + +int32 +aot_intrinsic_i32_rem_s(int32 l, int32 r) +{ + return l % r; +} + +uint32 +aot_intrinsic_i32_rem_u(uint32 l, uint32 r) +{ + return l % r; +} + +uint64 +aot_intrinsic_i64_div_u(uint64 l, uint64 r) +{ + return l / r; +} + +int64 +aot_intrinsic_i64_rem_s(int64 l, int64 r) +{ + return l % r; +} + +uint64 +aot_intrinsic_i64_rem_u(uint64 l, uint64 r) +{ + return l % r; +} + +uint64 +aot_intrinsic_i64_bit_or(uint64 l, uint64 r) +{ + return l | r; +} + +uint64 +aot_intrinsic_i64_bit_and(uint64 l, uint64 r) +{ + return l & r; +} + +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + +typedef struct { + const char *llvm_intrinsic; + const char *native_intrinsic; + uint64 flag; +} aot_intrinsic; + +/* clang-format off */ +static const aot_intrinsic g_intrinsic_mapping[] = { + { "llvm.experimental.constrained.fadd.f32", "aot_intrinsic_fadd_f32", AOT_INTRINSIC_FLAG_F32_FADD }, + { "llvm.experimental.constrained.fadd.f64", "aot_intrinsic_fadd_f64", AOT_INTRINSIC_FLAG_F64_FADD }, + { "llvm.experimental.constrained.fsub.f32", "aot_intrinsic_fsub_f32", AOT_INTRINSIC_FLAG_F32_FSUB }, + { "llvm.experimental.constrained.fsub.f64", "aot_intrinsic_fsub_f64", AOT_INTRINSIC_FLAG_F64_FSUB }, + { "llvm.experimental.constrained.fmul.f32", "aot_intrinsic_fmul_f32", AOT_INTRINSIC_FLAG_F32_FMUL }, + { "llvm.experimental.constrained.fmul.f64", "aot_intrinsic_fmul_f64", AOT_INTRINSIC_FLAG_F64_FMUL }, + { "llvm.experimental.constrained.fdiv.f32", "aot_intrinsic_fdiv_f32", AOT_INTRINSIC_FLAG_F32_FDIV }, + { "llvm.experimental.constrained.fdiv.f64", "aot_intrinsic_fdiv_f64", AOT_INTRINSIC_FLAG_F64_FDIV }, + { "llvm.fabs.f32", "aot_intrinsic_fabs_f32", AOT_INTRINSIC_FLAG_F32_FABS }, + { "llvm.fabs.f64", "aot_intrinsic_fabs_f64", AOT_INTRINSIC_FLAG_F64_FABS }, + { "llvm.ceil.f32", "aot_intrinsic_ceil_f32", AOT_INTRINSIC_FLAG_F32_CEIL }, + { "llvm.ceil.f64", "aot_intrinsic_ceil_f64", AOT_INTRINSIC_FLAG_F64_CEIL }, + { "llvm.floor.f32", "aot_intrinsic_floor_f32", AOT_INTRINSIC_FLAG_F32_FLOOR }, + { "llvm.floor.f64", "aot_intrinsic_floor_f64", AOT_INTRINSIC_FLAG_F64_FLOOR }, + { "llvm.trunc.f32", "aot_intrinsic_trunc_f32", AOT_INTRINSIC_FLAG_F32_TRUNC }, + { "llvm.trunc.f64", "aot_intrinsic_trunc_f64", AOT_INTRINSIC_FLAG_F64_TRUNC }, + { "llvm.rint.f32", "aot_intrinsic_rint_f32", AOT_INTRINSIC_FLAG_F32_RINT }, + { "llvm.rint.f64", "aot_intrinsic_rint_f64", AOT_INTRINSIC_FLAG_F64_RINT }, + { "llvm.sqrt.f32", "aot_intrinsic_sqrt_f32", AOT_INTRINSIC_FLAG_F32_SQRT }, + { "llvm.sqrt.f64", "aot_intrinsic_sqrt_f64", AOT_INTRINSIC_FLAG_F64_SQRT }, + { "llvm.copysign.f32", "aot_intrinsic_copysign_f32", AOT_INTRINSIC_FLAG_F32_COPYSIGN }, + { "llvm.copysign.f64", "aot_intrinsic_copysign_f64", AOT_INTRINSIC_FLAG_F64_COPYSIGN }, + { "llvm.minnum.f32", "aot_intrinsic_fmin_f32", AOT_INTRINSIC_FLAG_F32_MIN }, + { "llvm.minnum.f64", "aot_intrinsic_fmin_f64", AOT_INTRINSIC_FLAG_F64_MIN }, + { "llvm.maxnum.f32", "aot_intrinsic_fmax_f32", AOT_INTRINSIC_FLAG_F32_MAX }, + { "llvm.maxnum.f64", "aot_intrinsic_fmax_f64", AOT_INTRINSIC_FLAG_F64_MAX }, + { "llvm.ctlz.i32", "aot_intrinsic_clz_i32", AOT_INTRINSIC_FLAG_I32_CLZ }, + { "llvm.ctlz.i64", "aot_intrinsic_clz_i64", AOT_INTRINSIC_FLAG_I64_CLZ }, + { "llvm.cttz.i32", "aot_intrinsic_ctz_i32", AOT_INTRINSIC_FLAG_I32_CTZ }, + { "llvm.cttz.i64", "aot_intrinsic_ctz_i64", AOT_INTRINSIC_FLAG_I64_CTZ }, + { "llvm.ctpop.i32", "aot_intrinsic_popcnt_i32", AOT_INTRINSIC_FLAG_I32_POPCNT }, + { "llvm.ctpop.i64", "aot_intrinsic_popcnt_i64", AOT_INTRINSIC_FLAG_I64_POPCNT }, + { "f64_convert_i32_s", "aot_intrinsic_i32_to_f64", AOT_INTRINSIC_FLAG_I32_TO_F64 }, + { "f64_convert_i32_u", "aot_intrinsic_u32_to_f64", AOT_INTRINSIC_FLAG_U32_TO_F64 }, + { "f32_convert_i32_s", "aot_intrinsic_i32_to_f32", AOT_INTRINSIC_FLAG_I32_TO_F32 }, + { "f32_convert_i32_u", "aot_intrinsic_u32_to_f32", AOT_INTRINSIC_FLAG_U32_TO_F32 }, + { "f64_convert_i64_s", "aot_intrinsic_i64_to_f64", AOT_INTRINSIC_FLAG_I32_TO_F64 }, + { "f64_convert_i64_u", "aot_intrinsic_u64_to_f64", AOT_INTRINSIC_FLAG_U64_TO_F64 }, + { "f32_convert_i64_s", "aot_intrinsic_i64_to_f32", AOT_INTRINSIC_FLAG_I64_TO_F32 }, + { "f32_convert_i64_u", "aot_intrinsic_u64_to_f32", AOT_INTRINSIC_FLAG_U64_TO_F32 }, + { "i32_trunc_f32_u", "aot_intrinsic_f32_to_u32", AOT_INTRINSIC_FLAG_F32_TO_U32 }, + { "i32_trunc_f32_s", "aot_intrinsic_f32_to_i32", AOT_INTRINSIC_FLAG_F32_TO_I32 }, + { "i32_trunc_f64_u", "aot_intrinsic_f64_to_u32", AOT_INTRINSIC_FLAG_F64_TO_U32 }, + { "i32_trunc_f64_s", "aot_intrinsic_f64_to_i32", AOT_INTRINSIC_FLAG_F64_TO_I32 }, + { "i64_trunc_f64_u", "aot_intrinsic_f64_to_u64", AOT_INTRINSIC_FLAG_F64_TO_U64 }, + { "i64_trunc_f32_s", "aot_intrinsic_f32_to_i64", AOT_INTRINSIC_FLAG_F32_TO_I64 }, + { "i64_trunc_f32_u", "aot_intrinsic_f32_to_u64", AOT_INTRINSIC_FLAG_F32_TO_U64 }, + { "i64_trunc_f64_s", "aot_intrinsic_f64_to_i64", AOT_INTRINSIC_FLAG_F64_TO_I64 }, + { "f32_demote_f64", "aot_intrinsic_f64_to_f32", AOT_INTRINSIC_FLAG_F64_TO_F32 }, + { "f64_promote_f32", "aot_intrinsic_f32_to_f64", AOT_INTRINSIC_FLAG_F32_TO_F64 }, + { "f32_cmp", "aot_intrinsic_f32_cmp", AOT_INTRINSIC_FLAG_F32_CMP }, + { "f64_cmp", "aot_intrinsic_f64_cmp", AOT_INTRINSIC_FLAG_F64_CMP }, + { "i32.const", NULL, AOT_INTRINSIC_FLAG_I32_CONST }, + { "i64.const", NULL, AOT_INTRINSIC_FLAG_I64_CONST }, + { "f32.const", NULL, AOT_INTRINSIC_FLAG_F32_CONST }, + { "f64.const", NULL, AOT_INTRINSIC_FLAG_F64_CONST }, + { "i64.div_s", "aot_intrinsic_i64_div_s", AOT_INTRINSIC_FLAG_I64_DIV_S}, + { "i32.div_s", "aot_intrinsic_i32_div_s", AOT_INTRINSIC_FLAG_I32_DIV_S}, + { "i32.div_u", "aot_intrinsic_i32_div_u", AOT_INTRINSIC_FLAG_I32_DIV_U}, + { "i32.rem_s", "aot_intrinsic_i32_rem_s", AOT_INTRINSIC_FLAG_I32_REM_S}, + { "i32.rem_u", "aot_intrinsic_i32_rem_u", AOT_INTRINSIC_FLAG_I32_REM_U}, + { "i64.div_u", "aot_intrinsic_i64_div_u", AOT_INTRINSIC_FLAG_I64_DIV_U}, + { "i64.rem_s", "aot_intrinsic_i64_rem_s", AOT_INTRINSIC_FLAG_I64_REM_S}, + { "i64.rem_u", "aot_intrinsic_i64_rem_u", AOT_INTRINSIC_FLAG_I64_REM_U}, + { "i64.or", "aot_intrinsic_i64_bit_or", AOT_INTRINSIC_FLAG_I64_BIT_OR}, + { "i64.and", "aot_intrinsic_i64_bit_and", AOT_INTRINSIC_FLAG_I64_BIT_AND}, +}; +/* clang-format on */ + +static const uint32 g_intrinsic_count = + sizeof(g_intrinsic_mapping) / sizeof(aot_intrinsic); + +const char * +aot_intrinsic_get_symbol(const char *llvm_intrinsic) +{ + uint32 cnt; + for (cnt = 0; cnt < g_intrinsic_count; cnt++) { + if (!strcmp(llvm_intrinsic, g_intrinsic_mapping[cnt].llvm_intrinsic)) { + return g_intrinsic_mapping[cnt].native_intrinsic; + } + } + return NULL; +} + +static void +add_intrinsic_capability(AOTCompContext *comp_ctx, uint64 flag) +{ + uint64 group = AOT_INTRINSIC_GET_GROUP_FROM_FLAG(flag); + if (group < sizeof(comp_ctx->flags) / sizeof(uint64)) { + comp_ctx->flags[group] |= flag; + } + else { + bh_log(BH_LOG_LEVEL_WARNING, __FILE__, __LINE__, + "intrinsic exceeds max limit."); + } +} + +static void +add_i64_common_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_DIV_S); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_DIV_U); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_REM_S); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_REM_U); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_BIT_OR); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_BIT_AND); +} + +static void +add_i32_common_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_DIV_S); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_DIV_U); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_REM_S); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_REM_U); +} + +static void +add_f32_common_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_FABS); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_FADD); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_FSUB); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_FMUL); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_FDIV); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_SQRT); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_CMP); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_MIN); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_MAX); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_CEIL); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_FLOOR); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TRUNC); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_RINT); +} + +static void +add_f64_common_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_FABS); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_FADD); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_FSUB); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_FMUL); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_MIN); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_MAX); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_CEIL); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_FLOOR); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TRUNC); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_RINT); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_FDIV); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_SQRT); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_CMP); +} + +static void +add_f32xi32_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_I32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_U32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_TO_F32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U32_TO_F32); +} + +static void +add_f64xi32_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_I32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_U32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_TO_F64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U32_TO_F64); +} + +static void +add_f32xi64_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_I64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_U64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_TO_F32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U64_TO_F32); +} + +static void +add_f64xi64_intrinsics(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_I64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_U64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_TO_F64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U64_TO_F64); +} + +static void +add_common_float_integer_convertion(AOTCompContext *comp_ctx) +{ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_TO_F32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U32_TO_F32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_TO_F64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U32_TO_F64); + + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_TO_F32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U64_TO_F32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_TO_F64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_U64_TO_F64); + + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_I32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_U32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_I64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_U64); + + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_I32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_U32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_I64); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_U64); + + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_TO_F32); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_TO_F64); +} + +bool +aot_intrinsic_check_capability(const AOTCompContext *comp_ctx, + const char *llvm_intrinsic) +{ + uint32 cnt; + uint64 flag; + uint64 group; + + for (cnt = 0; cnt < g_intrinsic_count; cnt++) { + if (!strcmp(llvm_intrinsic, g_intrinsic_mapping[cnt].llvm_intrinsic)) { + flag = g_intrinsic_mapping[cnt].flag; + group = AOT_INTRINSIC_GET_GROUP_FROM_FLAG(flag); + flag &= AOT_INTRINSIC_FLAG_MASK; + if (group < sizeof(comp_ctx->flags) / sizeof(uint64)) { + if (comp_ctx->flags[group] & flag) { + return true; + } + } + else { + bh_log(BH_LOG_LEVEL_WARNING, __FILE__, __LINE__, + "intrinsic exceeds max limit."); + } + } + } + return false; +} + +void +aot_intrinsic_fill_capability_flags(AOTCompContext *comp_ctx) +{ + uint32 i; + + memset(comp_ctx->flags, 0, sizeof(comp_ctx->flags)); + + /* Intrinsics from command line have highest priority */ + + if (comp_ctx->builtin_intrinsics) { + + /* Handle 'all' group */ + if (strstr(comp_ctx->builtin_intrinsics, "all")) { + for (i = 0; i < g_intrinsic_count; i++) { + add_intrinsic_capability(comp_ctx, g_intrinsic_mapping[i].flag); + } + return; + } + + /* Handle 'i32.common' group */ + if (strstr(comp_ctx->builtin_intrinsics, "i32.common")) { + add_i32_common_intrinsics(comp_ctx); + } + + /* Handle 'i64.common' group */ + if (strstr(comp_ctx->builtin_intrinsics, "i64.common")) { + add_i64_common_intrinsics(comp_ctx); + } + + /* Handle 'fp.common' group */ + if (strstr(comp_ctx->builtin_intrinsics, "fp.common")) { + add_f32_common_intrinsics(comp_ctx); + add_f64_common_intrinsics(comp_ctx); + } + + /* Handle 'f32.common' group */ + if (strstr(comp_ctx->builtin_intrinsics, "f32.common")) { + add_f32_common_intrinsics(comp_ctx); + } + + /* Handle 'f64.common' group */ + if (strstr(comp_ctx->builtin_intrinsics, "f64.common")) { + add_f64_common_intrinsics(comp_ctx); + } + + /* Handle 'f32xi32' group */ + if (strstr(comp_ctx->builtin_intrinsics, "f32xi32")) { + add_f32xi32_intrinsics(comp_ctx); + } + + /* Handle 'f64xi32' group */ + if (strstr(comp_ctx->builtin_intrinsics, "f64xi32")) { + add_f64xi32_intrinsics(comp_ctx); + } + + /* Handle 'f32xi64' group */ + if (strstr(comp_ctx->builtin_intrinsics, "f32xi64")) { + add_f32xi64_intrinsics(comp_ctx); + } + + /* Handle 'f64xi64' group */ + if (strstr(comp_ctx->builtin_intrinsics, "f64xi64")) { + add_f64xi64_intrinsics(comp_ctx); + } + + /* Handle 'fpxint' group */ + if (strstr(comp_ctx->builtin_intrinsics, "fpxint")) { + add_f32xi32_intrinsics(comp_ctx); + add_f64xi32_intrinsics(comp_ctx); + add_f32xi64_intrinsics(comp_ctx); + add_f64xi64_intrinsics(comp_ctx); + } + + /* Handle 'constop' group */ + if (strstr(comp_ctx->builtin_intrinsics, "constop")) { + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_CONST); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_CONST); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_CONST); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_CONST); + } + + /* Handle 'fp.common' group */ + if (strstr(comp_ctx->builtin_intrinsics, "fp.common")) { + add_f32_common_intrinsics(comp_ctx); + add_f64_common_intrinsics(comp_ctx); + } + + /* Handle other single items */ + for (i = 0; i < g_intrinsic_count; i++) { + if (strstr(comp_ctx->builtin_intrinsics, + g_intrinsic_mapping[i].llvm_intrinsic)) { + add_intrinsic_capability(comp_ctx, g_intrinsic_mapping[i].flag); + } + } + + return; + } + + if (!comp_ctx->target_cpu) + return; + + if (!strncmp(comp_ctx->target_arch, "thumb", 5)) { + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_CONST); + add_i32_common_intrinsics(comp_ctx); + if (!strcmp(comp_ctx->target_cpu, "cortex-m7")) { + } + else if (!strcmp(comp_ctx->target_cpu, "cortex-m4")) { + add_f64_common_intrinsics(comp_ctx); + } + else { + add_f32_common_intrinsics(comp_ctx); + add_f64_common_intrinsics(comp_ctx); + add_i64_common_intrinsics(comp_ctx); + add_common_float_integer_convertion(comp_ctx); + } + } + else if (!strncmp(comp_ctx->target_arch, "riscv", 5)) { + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_CONST); + /* + * Note: Use builtin intrinsics since hardware float operation + * will cause rodata relocation + */ + add_f32_common_intrinsics(comp_ctx); + add_f64_common_intrinsics(comp_ctx); + add_common_float_integer_convertion(comp_ctx); + if (!strncmp(comp_ctx->target_arch, "riscv32", 7)) { + add_i64_common_intrinsics(comp_ctx); + } + } + else if (!strncmp(comp_ctx->target_arch, "xtensa", 6)) { + /* + * Note: Use builtin intrinsics since hardware float operation + * will cause rodata relocation + */ + add_f32_common_intrinsics(comp_ctx); + add_i32_common_intrinsics(comp_ctx); + add_f64_common_intrinsics(comp_ctx); + add_i64_common_intrinsics(comp_ctx); + add_common_float_integer_convertion(comp_ctx); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_CONST); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_CONST); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I32_CONST); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_I64_CONST); + } + else { + /* + * Use constant value table by default + */ + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F32_CONST); + add_intrinsic_capability(comp_ctx, AOT_INTRINSIC_FLAG_F64_CONST); + } +} + +#endif /* WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 */ diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_intrinsic.h b/wasm-micro-runtime/core/iwasm/aot/aot_intrinsic.h new file mode 100644 index 0000000..6a456ef --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_intrinsic.h @@ -0,0 +1,306 @@ +/* + * Copyright (C) 2021 XiaoMi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_INTRINSIC_H +#define _AOT_INTRINSIC_H + +#include "aot_runtime.h" +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 +#include "aot_llvm.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define AOT_INTRINSIC_GROUPS 2 + +/* Use uint64 as flag container: + * - The upper 16 bits are the intrinsic group number + * - The lower 48 bits are the intrinsic capability mask + */ + +#define AOT_INTRINSIC_FLAG(group, number) \ + ((((uint64)(group & 0xffffLL)) << 48) | ((uint64)1 << number)) + +#define AOT_INTRINSIC_FLAG_MASK (0x0000ffffffffffffLL) + +#define AOT_INTRINSIC_GET_GROUP_FROM_FLAG(flag) \ + ((((uint64)flag) >> 48) & 0xffffLL) + +/* clang-format off */ +#define AOT_INTRINSIC_FLAG_F32_FADD AOT_INTRINSIC_FLAG(0, 0) +#define AOT_INTRINSIC_FLAG_F32_FSUB AOT_INTRINSIC_FLAG(0, 1) +#define AOT_INTRINSIC_FLAG_F32_FMUL AOT_INTRINSIC_FLAG(0, 2) +#define AOT_INTRINSIC_FLAG_F32_FDIV AOT_INTRINSIC_FLAG(0, 3) +#define AOT_INTRINSIC_FLAG_F32_FABS AOT_INTRINSIC_FLAG(0, 4) +#define AOT_INTRINSIC_FLAG_F32_CEIL AOT_INTRINSIC_FLAG(0, 5) +#define AOT_INTRINSIC_FLAG_F32_FLOOR AOT_INTRINSIC_FLAG(0, 6) +#define AOT_INTRINSIC_FLAG_F32_TRUNC AOT_INTRINSIC_FLAG(0, 7) +#define AOT_INTRINSIC_FLAG_F32_RINT AOT_INTRINSIC_FLAG(0, 8) +#define AOT_INTRINSIC_FLAG_F32_SQRT AOT_INTRINSIC_FLAG(0, 9) +#define AOT_INTRINSIC_FLAG_F32_COPYSIGN AOT_INTRINSIC_FLAG(0, 10) +#define AOT_INTRINSIC_FLAG_F32_MIN AOT_INTRINSIC_FLAG(0, 11) +#define AOT_INTRINSIC_FLAG_F32_MAX AOT_INTRINSIC_FLAG(0, 12) +#define AOT_INTRINSIC_FLAG_I32_CLZ AOT_INTRINSIC_FLAG(0, 13) +#define AOT_INTRINSIC_FLAG_I32_CTZ AOT_INTRINSIC_FLAG(0, 14) +#define AOT_INTRINSIC_FLAG_I32_POPCNT AOT_INTRINSIC_FLAG(0, 15) +#define AOT_INTRINSIC_FLAG_I32_TO_F32 AOT_INTRINSIC_FLAG(0, 16) +#define AOT_INTRINSIC_FLAG_U32_TO_F32 AOT_INTRINSIC_FLAG(0, 17) +#define AOT_INTRINSIC_FLAG_I32_TO_F64 AOT_INTRINSIC_FLAG(0, 18) +#define AOT_INTRINSIC_FLAG_U32_TO_F64 AOT_INTRINSIC_FLAG(0, 19) +#define AOT_INTRINSIC_FLAG_F32_TO_I32 AOT_INTRINSIC_FLAG(0, 20) +#define AOT_INTRINSIC_FLAG_F32_TO_U32 AOT_INTRINSIC_FLAG(0, 21) +#define AOT_INTRINSIC_FLAG_F32_TO_I64 AOT_INTRINSIC_FLAG(0, 22) +#define AOT_INTRINSIC_FLAG_F32_TO_U64 AOT_INTRINSIC_FLAG(0, 23) +#define AOT_INTRINSIC_FLAG_F32_TO_F64 AOT_INTRINSIC_FLAG(0, 24) +#define AOT_INTRINSIC_FLAG_F32_CMP AOT_INTRINSIC_FLAG(0, 25) +#define AOT_INTRINSIC_FLAG_F32_CONST AOT_INTRINSIC_FLAG(0, 26) +#define AOT_INTRINSIC_FLAG_I32_CONST AOT_INTRINSIC_FLAG(0, 27) +#define AOT_INTRINSIC_FLAG_I32_DIV_U AOT_INTRINSIC_FLAG(0, 28) +#define AOT_INTRINSIC_FLAG_I32_REM_S AOT_INTRINSIC_FLAG(0, 29) +#define AOT_INTRINSIC_FLAG_I32_REM_U AOT_INTRINSIC_FLAG(0, 30) +#define AOT_INTRINSIC_FLAG_I32_DIV_S AOT_INTRINSIC_FLAG(0, 31) + +#define AOT_INTRINSIC_FLAG_F64_FADD AOT_INTRINSIC_FLAG(1, 0) +#define AOT_INTRINSIC_FLAG_F64_FSUB AOT_INTRINSIC_FLAG(1, 1) +#define AOT_INTRINSIC_FLAG_F64_FMUL AOT_INTRINSIC_FLAG(1, 2) +#define AOT_INTRINSIC_FLAG_F64_FDIV AOT_INTRINSIC_FLAG(1, 3) +#define AOT_INTRINSIC_FLAG_F64_FABS AOT_INTRINSIC_FLAG(1, 4) +#define AOT_INTRINSIC_FLAG_F64_CEIL AOT_INTRINSIC_FLAG(1, 5) +#define AOT_INTRINSIC_FLAG_F64_FLOOR AOT_INTRINSIC_FLAG(1, 6) +#define AOT_INTRINSIC_FLAG_F64_TRUNC AOT_INTRINSIC_FLAG(1, 7) +#define AOT_INTRINSIC_FLAG_F64_RINT AOT_INTRINSIC_FLAG(1, 8) +#define AOT_INTRINSIC_FLAG_F64_SQRT AOT_INTRINSIC_FLAG(1, 9) +#define AOT_INTRINSIC_FLAG_F64_COPYSIGN AOT_INTRINSIC_FLAG(1, 10) +#define AOT_INTRINSIC_FLAG_F64_MIN AOT_INTRINSIC_FLAG(1, 11) +#define AOT_INTRINSIC_FLAG_F64_MAX AOT_INTRINSIC_FLAG(1, 12) +#define AOT_INTRINSIC_FLAG_I64_CLZ AOT_INTRINSIC_FLAG(1, 13) +#define AOT_INTRINSIC_FLAG_I64_CTZ AOT_INTRINSIC_FLAG(1, 14) +#define AOT_INTRINSIC_FLAG_I64_POPCNT AOT_INTRINSIC_FLAG(1, 15) +#define AOT_INTRINSIC_FLAG_I64_TO_F32 AOT_INTRINSIC_FLAG(1, 16) +#define AOT_INTRINSIC_FLAG_U64_TO_F32 AOT_INTRINSIC_FLAG(1, 17) +#define AOT_INTRINSIC_FLAG_I64_TO_F64 AOT_INTRINSIC_FLAG(1, 18) +#define AOT_INTRINSIC_FLAG_U64_TO_F64 AOT_INTRINSIC_FLAG(1, 19) +#define AOT_INTRINSIC_FLAG_F64_TO_I32 AOT_INTRINSIC_FLAG(1, 20) +#define AOT_INTRINSIC_FLAG_F64_TO_U32 AOT_INTRINSIC_FLAG(1, 21) +#define AOT_INTRINSIC_FLAG_F64_TO_I64 AOT_INTRINSIC_FLAG(1, 22) +#define AOT_INTRINSIC_FLAG_F64_TO_U64 AOT_INTRINSIC_FLAG(1, 23) +#define AOT_INTRINSIC_FLAG_F64_TO_F32 AOT_INTRINSIC_FLAG(1, 24) +#define AOT_INTRINSIC_FLAG_F64_CMP AOT_INTRINSIC_FLAG(1, 25) +#define AOT_INTRINSIC_FLAG_F64_CONST AOT_INTRINSIC_FLAG(1, 26) +#define AOT_INTRINSIC_FLAG_I64_CONST AOT_INTRINSIC_FLAG(1, 27) +#define AOT_INTRINSIC_FLAG_I64_DIV_S AOT_INTRINSIC_FLAG(1, 28) +#define AOT_INTRINSIC_FLAG_I64_DIV_U AOT_INTRINSIC_FLAG(1, 29) +#define AOT_INTRINSIC_FLAG_I64_REM_S AOT_INTRINSIC_FLAG(1, 30) +#define AOT_INTRINSIC_FLAG_I64_REM_U AOT_INTRINSIC_FLAG(1, 31) +#define AOT_INTRINSIC_FLAG_I64_BIT_OR AOT_INTRINSIC_FLAG(1, 32) +#define AOT_INTRINSIC_FLAG_I64_BIT_AND AOT_INTRINSIC_FLAG(1, 33) + +/* clang-format on */ + +float32 +aot_intrinsic_fadd_f32(float32 a, float32 b); + +float64 +aot_intrinsic_fadd_f64(float64 a, float64 b); + +float32 +aot_intrinsic_fsub_f32(float32 a, float32 b); + +float64 +aot_intrinsic_fsub_f64(float64 a, float64 b); + +float32 +aot_intrinsic_fmul_f32(float32 a, float32 b); + +float64 +aot_intrinsic_fmul_f64(float64 a, float64 b); + +float32 +aot_intrinsic_fdiv_f32(float32 a, float32 b); + +float64 +aot_intrinsic_fdiv_f64(float64 a, float64 b); + +float32 +aot_intrinsic_fabs_f32(float32 a); + +float64 +aot_intrinsic_fabs_f64(float64 a); + +float32 +aot_intrinsic_ceil_f32(float32 a); + +float64 +aot_intrinsic_ceil_f64(float64 a); + +float32 +aot_intrinsic_floor_f32(float32 a); + +float64 +aot_intrinsic_floor_f64(float64 a); + +float32 +aot_intrinsic_trunc_f32(float32 a); + +float64 +aot_intrinsic_trunc_f64(float64 a); + +float32 +aot_intrinsic_rint_f32(float32 a); + +float64 +aot_intrinsic_rint_f64(float64 a); + +float32 +aot_intrinsic_sqrt_f32(float32 a); + +float64 +aot_intrinsic_sqrt_f64(float64 a); + +float32 +aot_intrinsic_copysign_f32(float32 a, float32 b); + +float64 +aot_intrinsic_copysign_f64(float64 a, float64 b); + +float32 +aot_intrinsic_fmin_f32(float32 a, float32 b); + +float64 +aot_intrinsic_fmin_f64(float64 a, float64 b); + +float32 +aot_intrinsic_fmax_f32(float32 a, float32 b); + +float64 +aot_intrinsic_fmax_f64(float64 a, float64 b); + +uint32 +aot_intrinsic_clz_i32(uint32 type); + +uint32 +aot_intrinsic_clz_i64(uint64 type); + +uint32 +aot_intrinsic_ctz_i32(uint32 type); + +uint32 +aot_intrinsic_ctz_i64(uint64 type); + +uint32 +aot_intrinsic_popcnt_i32(uint32 u); + +uint32 +aot_intrinsic_popcnt_i64(uint64 u); + +float32 +aot_intrinsic_i32_to_f32(int32 i); + +float32 +aot_intrinsic_u32_to_f32(uint32 u); + +float64 +aot_intrinsic_i32_to_f64(int32 i); + +float64 +aot_intrinsic_u32_to_f64(uint32 u); + +float32 +aot_intrinsic_i64_to_f32(int64 i); + +float32 +aot_intrinsic_u64_to_f32(uint64 u); + +float64 +aot_intrinsic_i64_to_f64(int64 i); + +float64 +aot_intrinsic_u64_to_f64(uint64 u); + +int32 +aot_intrinsic_f32_to_i32(float32 f); + +uint32 +aot_intrinsic_f32_to_u32(float32 f); + +int64 +aot_intrinsic_f32_to_i64(float32 f); + +uint64 +aot_intrinsic_f32_to_u64(float32 f); + +int32 +aot_intrinsic_f64_to_i32(float64 f); + +uint32 +aot_intrinsic_f64_to_u32(float64 f); + +int64 +aot_intrinsic_f64_to_i64(float64 f); + +uint64 +aot_intrinsic_f64_to_u64(float64 f); + +float64 +aot_intrinsic_f32_to_f64(float32 f); + +float32 +aot_intrinsic_f64_to_f32(float64 f); + +int32 +aot_intrinsic_f32_cmp(AOTFloatCond cond, float32 lhs, float32 rhs); + +int32 +aot_intrinsic_f64_cmp(AOTFloatCond cond, float64 lhs, float64 rhs); + +int64 +aot_intrinsic_i64_div_s(int64 l, int64 r); + +int32 +aot_intrinsic_i32_div_s(int32 l, int32 r); + +uint32 +aot_intrinsic_i32_div_u(uint32 l, uint32 r); + +int32 +aot_intrinsic_i32_rem_s(int32 l, int32 r); + +uint32 +aot_intrinsic_i32_rem_u(uint32 l, uint32 r); + +uint64 +aot_intrinsic_i64_div_u(uint64 l, uint64 r); + +int64 +aot_intrinsic_i64_rem_s(int64 l, int64 r); + +uint64 +aot_intrinsic_i64_rem_u(uint64 l, uint64 r); + +uint64 +aot_intrinsic_i64_bit_or(uint64 l, uint64 r); + +uint64 +aot_intrinsic_i64_bit_and(uint64 l, uint64 r); + +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 +const char * +aot_intrinsic_get_symbol(const char *llvm_intrinsic); + +bool +aot_intrinsic_check_capability(const AOTCompContext *comp_ctx, + const char *llvm_intrinsic); + +void +aot_intrinsic_fill_capability_flags(AOTCompContext *comp_ctx); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _AOT_INTRINSIC_H */ diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_loader.c b/wasm-micro-runtime/core/iwasm/aot/aot_loader.c new file mode 100644 index 0000000..7db0db7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_loader.c @@ -0,0 +1,4380 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_runtime.h" +#include "bh_common.h" +#include "bh_log.h" +#include "aot_reloc.h" +#include "../common/wasm_runtime_common.h" +#include "../common/wasm_native.h" +#include "../compilation/aot.h" + +#if WASM_ENABLE_DEBUG_AOT != 0 +#include "debug/elf_parser.h" +#include "debug/jit_debug.h" +#endif + +#if WASM_ENABLE_LINUX_PERF != 0 +#include "aot_perf_map.h" +#endif + +#define YMM_PLT_PREFIX "__ymm@" +#define XMM_PLT_PREFIX "__xmm@" +#define REAL_PLT_PREFIX "__real@" + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, "AOT module load failed: %s", + string); + } +} + +static void +set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + snprintf(error_buf, error_buf_size, "AOT module load failed: %s", buf); + } +} + +#define exchange_uint8(p_data) (void)0 + +static void +exchange_uint16(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 1); + *(p_data + 1) = value; +} + +static void +exchange_uint32(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 3); + *(p_data + 3) = value; + + value = *(p_data + 1); + *(p_data + 1) = *(p_data + 2); + *(p_data + 2) = value; +} + +static void +exchange_uint64(uint8 *p_data) +{ + uint32 value; + + value = *(uint32 *)p_data; + *(uint32 *)p_data = *(uint32 *)(p_data + 4); + *(uint32 *)(p_data + 4) = value; + exchange_uint32(p_data); + exchange_uint32(p_data + 4); +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +static bool +check_buf(const uint8 *buf, const uint8 *buf_end, uint32 length, + char *error_buf, uint32 error_buf_size) +{ + if ((uintptr_t)buf + length < (uintptr_t)buf + || (uintptr_t)buf + length > (uintptr_t)buf_end) { + set_error_buf(error_buf, error_buf_size, "unexpect end"); + return false; + } + return true; +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if (!check_buf(buf, buf_end, length, error_buf, error_buf_size)) { \ + goto fail; \ + } \ + } while (0) + +static uint8 * +align_ptr(const uint8 *p, uint32 b) +{ + uintptr_t v = (uintptr_t)p; + uintptr_t m = b - 1; + return (uint8 *)((v + m) & ~m); +} + +static inline uint64 +GET_U64_FROM_ADDR(uint32 *addr) +{ + union { + uint64 val; + uint32 parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} + +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + +static inline uint8 +GET_U8_FROM_ADDR(const uint8 *p) +{ + uint8 res = 0; + bh_assert(p); + + const uint8 *p_aligned = align_ptr(p, 4); + p_aligned = (p_aligned > p) ? p_aligned - 4 : p_aligned; + + uint32 buf32 = *(const uint32 *)p_aligned; + const uint8 *pbuf = (const uint8 *)&buf32; + + res = *(uint8 *)(pbuf + (p - p_aligned)); + + return res; +} + +static inline uint16 +GET_U16_FROM_ADDR(const uint8 *p) +{ + uint16 res = 0; + bh_assert(p); + + const uint8 *p_aligned = align_ptr(p, 4); + p_aligned = (p_aligned > p) ? p_aligned - 4 : p_aligned; + + uint32 buf32 = *(const uint32 *)p_aligned; + const uint8 *pbuf = (const uint8 *)&buf32; + + res = *(uint16 *)(pbuf + (p - p_aligned)); + + return res; +} + +#define TEMPLATE_READ(p, p_end, res, type) \ + do { \ + if (sizeof(type) != sizeof(uint64)) \ + p = (uint8 *)align_ptr(p, sizeof(type)); \ + else \ + /* align 4 bytes if type is uint64 */ \ + p = (uint8 *)align_ptr(p, sizeof(uint32)); \ + CHECK_BUF(p, p_end, sizeof(type)); \ + if (sizeof(type) == sizeof(uint8)) \ + res = GET_U8_FROM_ADDR(p); \ + else if (sizeof(type) == sizeof(uint16)) \ + res = GET_U16_FROM_ADDR(p); \ + else if (sizeof(type) == sizeof(uint32)) \ + res = *(type *)p; \ + else \ + res = (type)GET_U64_FROM_ADDR((uint32 *)p); \ + if (!is_little_endian()) \ + exchange_##type((uint8 *)&res); \ + p += sizeof(type); \ + } while (0) + +#define read_byte_array(p, p_end, addr, len) \ + do { \ + CHECK_BUF(p, p_end, len); \ + bh_memcpy_wa(addr, len, p, len); \ + p += len; \ + } while (0) + +#define read_string(p, p_end, str) \ + do { \ + if (!(str = load_string((uint8 **)&p, p_end, module, \ + is_load_from_file_buf, true, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#else /* else of (WASM_ENABLE_WORD_ALIGN_READ != 0) */ + +#define TEMPLATE_READ(p, p_end, res, type) \ + do { \ + if (sizeof(type) != sizeof(uint64)) \ + p = (uint8 *)align_ptr(p, sizeof(type)); \ + else \ + /* align 4 bytes if type is uint64 */ \ + p = (uint8 *)align_ptr(p, sizeof(uint32)); \ + CHECK_BUF(p, p_end, sizeof(type)); \ + if (sizeof(type) != sizeof(uint64)) \ + res = *(type *)p; \ + else \ + res = (type)GET_U64_FROM_ADDR((uint32 *)p); \ + if (!is_little_endian()) \ + exchange_##type((uint8 *)&res); \ + p += sizeof(type); \ + } while (0) + +/* NOLINTBEGIN, disable lint for this region with clang-tidy */ + +#define read_byte_array(p, p_end, addr, len) \ + do { \ + CHECK_BUF(p, p_end, len); \ + bh_memcpy_s(addr, len, p, len); \ + p += len; \ + } while (0) + +#define read_string(p, p_end, str) \ + do { \ + if (!(str = load_string((uint8 **)&p, p_end, module, \ + is_load_from_file_buf, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#endif /* end of (WASM_ENABLE_WORD_ALIGN_READ != 0) */ + +#define read_uint8(p, p_end, res) TEMPLATE_READ(p, p_end, res, uint8) +#define read_uint16(p, p_end, res) TEMPLATE_READ(p, p_end, res, uint16) +#define read_uint32(p, p_end, res) TEMPLATE_READ(p, p_end, res, uint32) +#define read_uint64(p, p_end, res) TEMPLATE_READ(p, p_end, res, uint64) + +/* NOLINTEND */ + +/* Legal values for bin_type */ +#define BIN_TYPE_ELF32L 0 /* 32-bit little endian */ +#define BIN_TYPE_ELF32B 1 /* 32-bit big endian */ +#define BIN_TYPE_ELF64L 2 /* 64-bit little endian */ +#define BIN_TYPE_ELF64B 3 /* 64-bit big endian */ +#define BIN_TYPE_COFF32 4 /* 32-bit little endian */ +#define BIN_TYPE_COFF64 6 /* 64-bit little endian */ + +/* Legal values for e_type (object file type). */ +#define E_TYPE_NONE 0 /* No file type */ +#define E_TYPE_REL 1 /* Relocatable file */ +#define E_TYPE_EXEC 2 /* Executable file */ +#define E_TYPE_DYN 3 /* Shared object file */ +#define E_TYPE_XIP 4 /* eXecute In Place file */ + +/* Legal values for e_machine (architecture). */ +#define E_MACHINE_386 3 /* Intel 80386 */ +#define E_MACHINE_MIPS 8 /* MIPS R3000 big-endian */ +#define E_MACHINE_MIPS_RS3_LE 10 /* MIPS R3000 little-endian */ +#define E_MACHINE_ARM 40 /* ARM/Thumb */ +#define E_MACHINE_AARCH64 183 /* AArch64 */ +#define E_MACHINE_ARC 45 /* Argonaut RISC Core */ +#define E_MACHINE_IA_64 50 /* Intel Merced */ +#define E_MACHINE_MIPS_X 51 /* Stanford MIPS-X */ +#define E_MACHINE_X86_64 62 /* AMD x86-64 architecture */ +#define E_MACHINE_ARC_COMPACT 93 /* ARC International ARCompact */ +#define E_MACHINE_ARC_COMPACT2 195 /* Synopsys ARCompact V2 */ +#define E_MACHINE_XTENSA 94 /* Tensilica Xtensa Architecture */ +#define E_MACHINE_RISCV 243 /* RISC-V 32/64 */ +#define E_MACHINE_WIN_I386 0x14c /* Windows i386 architecture */ +#define E_MACHINE_WIN_X86_64 0x8664 /* Windows x86-64 architecture */ + +/* Legal values for e_version */ +#define E_VERSION_CURRENT 1 /* Current version */ + +static void * +loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +static char * +load_string(uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + bool is_load_from_file_buf, +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + bool is_vram_word_align, +#endif + char *error_buf, uint32 error_buf_size) +{ + uint8 *p = *p_buf; + const uint8 *p_end = buf_end; + char *str; + uint16 str_len; + + read_uint16(p, p_end, str_len); + CHECK_BUF(p, p_end, str_len); + + if (str_len == 0) { + str = ""; + } +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + else if (is_vram_word_align) { + if (!(str = aot_const_str_set_insert((uint8 *)p, str_len, module, + is_vram_word_align, error_buf, + error_buf_size))) { + goto fail; + } + } +#endif + else if (is_load_from_file_buf) { + /* The string is always terminated with '\0', use it directly. + * In this case, the file buffer can be reffered to after loading. + */ + bh_assert(p[str_len - 1] == '\0'); + str = (char *)p; + } + else { + /* Load from sections, the file buffer cannot be reffered to + after loading, we must create another string and insert it + into const string set */ + bh_assert(p[str_len - 1] == '\0'); + if (!(str = aot_const_str_set_insert((uint8 *)p, str_len, module, +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + is_vram_word_align, +#endif + error_buf, error_buf_size))) { + goto fail; + } + } + p += str_len; + + *p_buf = p; + return str; +fail: + return NULL; +} + +static bool +get_aot_file_target(AOTTargetInfo *target_info, char *target_buf, + uint32 target_buf_size, char *error_buf, + uint32 error_buf_size) +{ + char *machine_type = NULL; + switch (target_info->e_machine) { + case E_MACHINE_X86_64: + case E_MACHINE_WIN_X86_64: + machine_type = "x86_64"; + break; + case E_MACHINE_386: + case E_MACHINE_WIN_I386: + machine_type = "i386"; + break; + case E_MACHINE_ARM: + case E_MACHINE_AARCH64: + machine_type = target_info->arch; + break; + case E_MACHINE_MIPS: + machine_type = "mips"; + break; + case E_MACHINE_XTENSA: + machine_type = "xtensa"; + break; + case E_MACHINE_RISCV: + machine_type = "riscv"; + break; + case E_MACHINE_ARC_COMPACT: + case E_MACHINE_ARC_COMPACT2: + machine_type = "arc"; + break; + default: + set_error_buf_v(error_buf, error_buf_size, + "unknown machine type %d", target_info->e_machine); + return false; + } + if (strncmp(target_info->arch, machine_type, strlen(machine_type))) { + set_error_buf_v( + error_buf, error_buf_size, + "machine type (%s) isn't consistent with target type (%s)", + machine_type, target_info->arch); + return false; + } + snprintf(target_buf, target_buf_size, "%s", target_info->arch); + return true; +} + +static bool +check_machine_info(AOTTargetInfo *target_info, char *error_buf, + uint32 error_buf_size) +{ + char target_expected[32], target_got[32]; + + get_current_target(target_expected, sizeof(target_expected)); + + if (!get_aot_file_target(target_info, target_got, sizeof(target_got), + error_buf, error_buf_size)) + return false; + + if (strncmp(target_expected, target_got, strlen(target_expected))) { + set_error_buf_v(error_buf, error_buf_size, + "invalid target type, expected %s but got %s", + target_expected, target_got); + return false; + } + + return true; +} + +static bool +check_feature_flags(char *error_buf, uint32 error_buf_size, + uint64 feature_flags) +{ +#if WASM_ENABLE_SIMD == 0 + if (feature_flags & WASM_FEATURE_SIMD_128BIT) { + set_error_buf(error_buf, error_buf_size, + "SIMD is not enabled in this build"); + return false; + } +#endif + +#if WASM_ENABLE_BULK_MEMORY == 0 + if (feature_flags & WASM_FEATURE_BULK_MEMORY) { + set_error_buf(error_buf, error_buf_size, + "bulk memory is not enabled in this build"); + return false; + } +#endif + +#if WASM_ENABLE_THREAD_MGR == 0 + if (feature_flags & WASM_FEATURE_MULTI_THREAD) { + set_error_buf(error_buf, error_buf_size, + "thread is not enabled in this build"); + return false; + } +#endif + +#if WASM_ENABLE_REF_TYPES == 0 + if (feature_flags & WASM_FEATURE_REF_TYPES) { + set_error_buf(error_buf, error_buf_size, + "reference types is not enabled in this build"); + return false; + } +#endif + +#if WASM_ENABLE_GC == 0 + if (feature_flags & WASM_FEATURE_GARBAGE_COLLECTION) { + set_error_buf(error_buf, error_buf_size, + "garbage collection is not enabled in this build"); + return false; + } +#endif + + return true; +} + +#if WASM_ENABLE_GC != 0 +static WASMRefType * +reftype_set_insert(HashMap *ref_type_set, const WASMRefType *ref_type, + char *error_buf, uint32 error_buf_size) +{ + WASMRefType *ret = wasm_reftype_set_insert(ref_type_set, ref_type); + + if (!ret) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + } + return ret; +} +#endif + +static bool +load_target_info_section(const uint8 *buf, const uint8 *buf_end, + AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + AOTTargetInfo target_info; + const uint8 *p = buf, *p_end = buf_end; + bool is_target_little_endian, is_target_64_bit; + + read_uint16(p, p_end, target_info.bin_type); + read_uint16(p, p_end, target_info.abi_type); + read_uint16(p, p_end, target_info.e_type); + read_uint16(p, p_end, target_info.e_machine); + read_uint32(p, p_end, target_info.e_version); + read_uint32(p, p_end, target_info.e_flags); + read_uint64(p, p_end, target_info.feature_flags); + read_uint64(p, p_end, target_info.reserved); + read_byte_array(p, p_end, target_info.arch, sizeof(target_info.arch)); + + if (p != buf_end) { + set_error_buf(error_buf, error_buf_size, "invalid section size"); + return false; + } + + /* Check target endian type */ + is_target_little_endian = target_info.bin_type & 1 ? false : true; + if (is_little_endian() != is_target_little_endian) { + set_error_buf_v(error_buf, error_buf_size, + "invalid target endian type, expected %s but got %s", + is_little_endian() ? "little endian" : "big endian", + is_target_little_endian ? "little endian" + : "big endian"); + return false; + } + + /* Check target bit width */ + is_target_64_bit = target_info.bin_type & 2 ? true : false; + if ((sizeof(void *) == 8 ? true : false) != is_target_64_bit) { + set_error_buf_v(error_buf, error_buf_size, + "invalid target bit width, expected %s but got %s", + sizeof(void *) == 8 ? "64-bit" : "32-bit", + is_target_64_bit ? "64-bit" : "32-bit"); + return false; + } + + /* Check target elf file type */ + if (target_info.e_type != E_TYPE_REL && target_info.e_type != E_TYPE_XIP) { + set_error_buf(error_buf, error_buf_size, + "invalid object file type, " + "expected relocatable or XIP file type but got others"); + return false; + } + + /* Check machine info */ + if (!check_machine_info(&target_info, error_buf, error_buf_size)) { + return false; + } + + if (target_info.e_version != E_VERSION_CURRENT) { + set_error_buf(error_buf, error_buf_size, "invalid elf file version"); + return false; + } + + /* Finally, check feature flags */ + return check_feature_flags(error_buf, error_buf_size, + target_info.feature_flags); +fail: + return false; +} + +static void * +get_native_symbol_by_name(const char *name) +{ + void *func = NULL; + uint32 symnum = 0; + SymbolMap *sym = NULL; + + sym = get_target_symbol_map(&symnum); + + while (symnum--) { + if (strcmp(sym->symbol_name, name) == 0) { + func = sym->symbol_addr; + break; + } + sym++; + } + + return func; +} + +static bool +str2uint32(const char *buf, uint32 *p_res); + +static bool +str2uint64(const char *buf, uint64 *p_res); + +#if WASM_ENABLE_MULTI_MODULE != 0 +static void * +aot_loader_resolve_function(const char *module_name, const char *function_name, + const AOTFuncType *expected_function_type, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + void *function = NULL; + AOTExport *export = NULL; + AOTModule *module = NULL; + AOTFuncType *target_function_type = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_AoT) { + LOG_DEBUG("can not find a module named %s for function %s", module_name, + function_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (AOTModule *)module_reg; + export = loader_find_export(module_reg, module_name, function_name, + EXPORT_KIND_FUNC, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve function type and function */ + if (export->index < module->import_func_count) { + target_function_type = module->import_funcs[export->index].func_type; + function = module->import_funcs[export->index].func_ptr_linked; + } + else { + target_function_type = + (AOTFuncType *)module + ->types[module->func_type_indexes[export->index + - module->import_func_count]]; + function = + (module->func_ptrs[export->index - module->import_func_count]); + } + /* check function type */ + if (!wasm_type_equal((WASMType *)expected_function_type, + (WASMType *)target_function_type, module->types, + module->type_count)) { + LOG_DEBUG("%s.%s failed the type check", module_name, function_name); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + return function; +} +#endif /* end of WASM_ENABLE_MULTI_MODULE */ + +static bool +load_native_symbol_section(const uint8 *buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 cnt; + int32 i; + const char *symbol; + + read_uint32(p, p_end, cnt); + + if (cnt > 0) { + module->native_symbol_list = wasm_runtime_malloc(cnt * sizeof(void *)); + if (module->native_symbol_list == NULL) { + set_error_buf(error_buf, error_buf_size, + "malloc native symbol list failed"); + goto fail; + } + + for (i = cnt - 1; i >= 0; i--) { + read_string(p, p_end, symbol); + if (!strncmp(symbol, "f32#", 4) || !strncmp(symbol, "i32#", 4)) { + uint32 u32; + /* Resolve the raw int bits of f32 const */ + if (!str2uint32(symbol + 4, &u32)) { + set_error_buf_v(error_buf, error_buf_size, + "resolve symbol %s failed", symbol); + goto fail; + } + *(uint32 *)(&module->native_symbol_list[i]) = u32; + } + else if (!strncmp(symbol, "f64#", 4) + || !strncmp(symbol, "i64#", 4)) { + uint64 u64; + /* Resolve the raw int bits of f64 const */ + if (!str2uint64(symbol + 4, &u64)) { + set_error_buf_v(error_buf, error_buf_size, + "resolve symbol %s failed", symbol); + goto fail; + } + *(uint64 *)(&module->native_symbol_list[i]) = u64; + } + else if (!strncmp(symbol, "__ignore", 8)) { + /* Padding bytes to make f64 on 8-byte aligned address, + or it is the second 32-bit slot in 32-bit system */ + continue; + } + else { + module->native_symbol_list[i] = + get_native_symbol_by_name(symbol); + if (module->native_symbol_list[i] == NULL) { + set_error_buf_v(error_buf, error_buf_size, + "missing native symbol: %s", symbol); + goto fail; + } + } + } + } + + return true; +fail: + return false; +} + +static bool +load_name_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + const uint8 *p = buf, *p_end = buf_end; + uint32 *aux_func_indexes; + const char **aux_func_names; + uint32 name_type, subsection_size; + uint32 previous_name_type = 0; + uint32 num_func_name; + uint32 func_index; + uint32 previous_func_index = ~0U; + uint32 name_index; + int i = 0; + uint32 name_len; + uint64 size; + + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + read_uint32(p, p_end, name_len); + + if (name_len != 4 || p + name_len > p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + if (memcmp(p, "name", 4) != 0) { + set_error_buf(error_buf, error_buf_size, "invalid custom name section"); + return false; + } + p += name_len; + + while (p < p_end) { + read_uint32(p, p_end, name_type); + if (i != 0) { + if (name_type == previous_name_type) { + set_error_buf(error_buf, error_buf_size, + "duplicate sub-section"); + return false; + } + if (name_type < previous_name_type) { + set_error_buf(error_buf, error_buf_size, + "out-of-order sub-section"); + return false; + } + } + previous_name_type = name_type; + read_uint32(p, p_end, subsection_size); + CHECK_BUF(p, p_end, subsection_size); + switch (name_type) { + case SUB_SECTION_TYPE_FUNC: + if (subsection_size) { + read_uint32(p, p_end, num_func_name); + if (num_func_name + > module->import_func_count + module->func_count) { + set_error_buf(error_buf, error_buf_size, + "function name count out of bounds"); + return false; + } + module->aux_func_name_count = num_func_name; + + /* Allocate memory */ + size = sizeof(uint32) * (uint64)module->aux_func_name_count; + if (!(aux_func_indexes = module->aux_func_indexes = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + size = + sizeof(char **) * (uint64)module->aux_func_name_count; + if (!(aux_func_names = module->aux_func_names = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + for (name_index = 0; name_index < num_func_name; + name_index++) { + read_uint32(p, p_end, func_index); + if (name_index != 0 + && func_index == previous_func_index) { + set_error_buf(error_buf, error_buf_size, + "duplicate function name"); + return false; + } + if (name_index != 0 + && func_index < previous_func_index) { + set_error_buf(error_buf, error_buf_size, + "out-of-order function index "); + return false; + } + if (func_index + >= module->import_func_count + module->func_count) { + set_error_buf(error_buf, error_buf_size, + "function index out of bounds"); + return false; + } + previous_func_index = func_index; + *(aux_func_indexes + name_index) = func_index; + read_string(p, p_end, *(aux_func_names + name_index)); +#if 0 + LOG_DEBUG("func_index %u -> aux_func_name = %s\n", + func_index, *(aux_func_names + name_index)); +#endif + } + } + break; + case SUB_SECTION_TYPE_MODULE: /* TODO: Parse for module subsection + */ + case SUB_SECTION_TYPE_LOCAL: /* TODO: Parse for local subsection */ + default: + p = p + subsection_size; + break; + } + i++; + } + + return true; +fail: + return false; +#else + return true; +#endif /* WASM_ENABLE_CUSTOM_NAME_SECTION != 0 */ +} + +#if WASM_ENABLE_STRINGREF != 0 +static bool +load_string_literal_section(const uint8 *buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 reserved = 0, string_count = 0, i; + uint64 size; + + read_uint32(p, p_end, reserved); + if (reserved != 0) { + set_error_buf(error_buf, error_buf_size, + "invalid reserved slot in string literal count"); + goto fail; + } + + read_uint32(p, p_end, string_count); + if (string_count == 0) { + set_error_buf(error_buf, error_buf_size, + "invalid string literal count"); + goto fail; + } + module->string_literal_count = string_count; + + size = (uint64)sizeof(char *) * string_count; + if (!(module->string_literal_ptrs = + loader_malloc(size, error_buf, error_buf_size))) { + goto fail; + } + + size = (uint64)sizeof(uint32) * string_count; + if (!(module->string_literal_lengths = + loader_malloc(size, error_buf, error_buf_size))) { + goto fail; + } + + for (i = 0; i < string_count; i++) { + read_uint32(p, p_end, module->string_literal_lengths[i]); + } + + for (i = 0; i < string_count; i++) { + module->string_literal_ptrs[i] = p; + p += module->string_literal_lengths[i]; + } + + return true; + +fail: + return false; +} +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + +static bool +load_custom_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 sub_section_type; + + read_uint32(p, p_end, sub_section_type); + buf = p; + + switch (sub_section_type) { + case AOT_CUSTOM_SECTION_NATIVE_SYMBOL: + if (!load_native_symbol_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + goto fail; + break; + case AOT_CUSTOM_SECTION_NAME: + if (!load_name_section(buf, buf_end, module, is_load_from_file_buf, + error_buf, error_buf_size)) + LOG_VERBOSE("Load name section failed."); + else + LOG_VERBOSE("Load name section success."); + break; +#if WASM_ENABLE_STRINGREF != 0 + case AOT_CUSTOM_SECTION_STRING_LITERAL: + if (!load_string_literal_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + goto fail; + break; +#endif +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + case AOT_CUSTOM_SECTION_RAW: + { + const char *section_name; + WASMCustomSection *section; + + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + goto fail; + } + + read_string(p, p_end, section_name); + + section = loader_malloc(sizeof(WASMCustomSection), error_buf, + error_buf_size); + if (!section) { + goto fail; + } + + section->name_addr = (char *)section_name; + section->name_len = (uint32)strlen(section_name); + section->content_addr = (uint8 *)p; + section->content_len = (uint32)(p_end - p); + + section->next = module->custom_section_list; + module->custom_section_list = section; + LOG_VERBOSE("Load custom section [%s] success.", section_name); + break; + } +#endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 */ + default: + break; + } + + return true; +fail: + return false; +} + +static void +destroy_import_memories(AOTImportMemory *import_memories) +{ + wasm_runtime_free(import_memories); +} + +static void +destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count) +{ + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) + wasm_runtime_free(data_list[i]); + wasm_runtime_free(data_list); +} + +static bool +load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + InitializerExpression *expr, char *error_buf, + uint32 error_buf_size); + +static bool +load_mem_init_data_list(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTMemInitData **data_list; + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTMemInitData *) * (uint64)module->mem_init_data_count; + if (!(module->mem_init_data_list = data_list = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each memory data segment */ + for (i = 0; i < module->mem_init_data_count; i++) { + uint32 byte_count; + uint32 is_passive; + uint32 memory_index; + InitializerExpression init_value; + + read_uint32(buf, buf_end, is_passive); + read_uint32(buf, buf_end, memory_index); + if (!load_init_expr(&buf, buf_end, module, &init_value, error_buf, + error_buf_size)) { + return false; + } + read_uint32(buf, buf_end, byte_count); + size = offsetof(AOTMemInitData, bytes) + (uint64)byte_count; + if (!(data_list[i] = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + /* is_passive and memory_index is only used in bulk memory mode */ + data_list[i]->is_passive = (bool)is_passive; + data_list[i]->memory_index = memory_index; +#endif + data_list[i]->offset.init_expr_type = init_value.init_expr_type; + data_list[i]->offset.u = init_value.u; + data_list[i]->byte_count = byte_count; + read_byte_array(buf, buf_end, data_list[i]->bytes, + data_list[i]->byte_count); + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_memory_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + uint64 total_size; + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->import_memory_count); + /* We don't support import_memory_count > 0 currently */ + bh_assert(module->import_memory_count == 0); + + read_uint32(buf, buf_end, module->memory_count); + total_size = sizeof(AOTMemory) * (uint64)module->memory_count; + if (!(module->memories = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module->memory_count; i++) { + read_uint32(buf, buf_end, module->memories[i].memory_flags); + read_uint32(buf, buf_end, module->memories[i].num_bytes_per_page); + read_uint32(buf, buf_end, module->memories[i].mem_init_page_count); + read_uint32(buf, buf_end, module->memories[i].mem_max_page_count); + } + + read_uint32(buf, buf_end, module->mem_init_data_count); + + /* load memory init data list */ + if (module->mem_init_data_count > 0 + && !load_mem_init_data_list(&buf, buf_end, module, error_buf, + error_buf_size)) + return false; + + *p_buf = buf; + return true; +fail: + return false; +} + +#if WASM_ENABLE_GC != 0 +static void +destroy_init_expr(InitializerExpression *expr) +{ + if (expr->init_expr_type == INIT_EXPR_TYPE_STRUCT_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + wasm_runtime_free(expr->u.data); + } +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static void +destroy_import_tables(AOTImportTable *import_tables) +{ + wasm_runtime_free(import_tables); +} + +static void +destroy_tables(AOTTable *tables) +{ + wasm_runtime_free(tables); +} + +static void +destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count) +{ + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) { +#if WASM_ENABLE_GC != 0 + uint32 j; + for (j = 0; j < data_list[i]->value_count; j++) { + destroy_init_expr(&data_list[i]->init_values[j]); + } +#endif + wasm_runtime_free(data_list[i]); + } + wasm_runtime_free(data_list); +} + +static bool +load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + InitializerExpression *expr, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + uint32 init_expr_type = 0; + uint64 *i64x2 = NULL; + bool free_if_fail = false; + + buf = (uint8 *)align_ptr(buf, 4); + + read_uint32(buf, buf_end, init_expr_type); + + switch (init_expr_type) { + case INIT_EXPR_NONE: + break; + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_F32_CONST: + read_uint32(buf, buf_end, expr->u.i32); + break; + case INIT_EXPR_TYPE_I64_CONST: + case INIT_EXPR_TYPE_F64_CONST: + read_uint64(buf, buf_end, expr->u.i64); + break; + case INIT_EXPR_TYPE_V128_CONST: + i64x2 = (uint64 *)expr->u.v128.i64x2; + CHECK_BUF(buf, buf_end, sizeof(uint64) * 2); + wasm_runtime_read_v128(buf, &i64x2[0], &i64x2[1]); + buf += sizeof(uint64) * 2; + break; + case INIT_EXPR_TYPE_GET_GLOBAL: + read_uint32(buf, buf_end, expr->u.global_index); + break; + /* INIT_EXPR_TYPE_FUNCREF_CONST can be used when + both reference types and GC are disabled */ + case INIT_EXPR_TYPE_FUNCREF_CONST: + read_uint32(buf, buf_end, expr->u.ref_index); + break; +#if WASM_ENABLE_GC != 0 || WASM_ENABLE_REF_TYPES != 0 + case INIT_EXPR_TYPE_REFNULL_CONST: + read_uint32(buf, buf_end, expr->u.ref_index); + break; +#endif /* end of WASM_ENABLE_GC != 0 || WASM_ENABLE_REF_TYPES != 0 */ +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_I31_NEW: + read_uint32(buf, buf_end, expr->u.i32); + break; + case INIT_EXPR_TYPE_STRUCT_NEW: + { + uint64 size; + uint32 type_idx, field_count; + AOTStructType *struct_type = NULL; + WASMStructNewInitValues *init_values = NULL; + + read_uint32(buf, buf_end, type_idx); + read_uint32(buf, buf_end, field_count); + + size = offsetof(WASMStructNewInitValues, fields) + + sizeof(WASMValue) * (uint64)field_count; + if (!(init_values = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + free_if_fail = true; + init_values->count = field_count; + expr->u.data = init_values; + + if (type_idx >= module->type_count) { + set_error_buf(error_buf, error_buf_size, + "unknown struct type."); + goto fail; + } + + struct_type = (AOTStructType *)module->types[type_idx]; + + if (struct_type->field_count != field_count) { + set_error_buf(error_buf, error_buf_size, + "invalid field count."); + goto fail; + } + + if (field_count > 0) { + uint32 i; + + for (i = 0; i < field_count; i++) { + uint32 field_size = + wasm_value_type_size(struct_type->fields[i].field_type); + if (field_size <= sizeof(uint32)) + read_uint32(buf, buf_end, init_values->fields[i].u32); + else if (field_size == sizeof(uint64)) + read_uint64(buf, buf_end, init_values->fields[i].u64); + else if (field_size == sizeof(uint64) * 2) + read_byte_array(buf, buf_end, &init_values->fields[i], + field_size); + else { + bh_assert(0); + } + } + } + + break; + } + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + read_uint32(buf, buf_end, expr->u.type_index); + break; + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + uint32 array_elem_type; + uint32 type_idx, length; + WASMArrayNewInitValues *init_values = NULL; + + /* Note: at this time the aot types haven't been loaded */ + read_uint32(buf, buf_end, array_elem_type); + read_uint32(buf, buf_end, type_idx); + read_uint32(buf, buf_end, length); + + if (init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { + expr->u.array_new_default.type_index = type_idx; + expr->u.array_new_default.length = length; + } + else { + uint32 i, elem_size, elem_data_count; + uint64 size = offsetof(WASMArrayNewInitValues, elem_data) + + sizeof(WASMValue) * (uint64)length; + if (!(init_values = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + free_if_fail = true; + expr->u.data = init_values; + + init_values->type_idx = type_idx; + init_values->length = length; + + elem_data_count = + (init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) ? length + : 1; + elem_size = wasm_value_type_size((uint8)array_elem_type); + + for (i = 0; i < elem_data_count; i++) { + if (elem_size <= sizeof(uint32)) + read_uint32(buf, buf_end, + init_values->elem_data[i].u32); + else if (elem_size == sizeof(uint64)) + read_uint64(buf, buf_end, + init_values->elem_data[i].u64); + else if (elem_size == sizeof(uint64) * 2) + read_byte_array(buf, buf_end, + &init_values->elem_data[i], elem_size); + else { + bh_assert(0); + } + } + } + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + set_error_buf(error_buf, error_buf_size, "invalid init expr type."); + return false; + } + + expr->init_expr_type = (uint8)init_expr_type; + + *p_buf = buf; + return true; +fail: +#if WASM_ENABLE_GC != 0 + if (free_if_fail) { + wasm_runtime_free(expr->u.data); + } +#else + (void)free_if_fail; +#endif + return false; +} + +static bool +load_import_table_list(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTImportTable *import_table; +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; +#endif + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTImportTable) * (uint64)module->import_table_count; + if (!(module->import_tables = import_table = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* keep sync with aot_emit_table_info() aot_emit_aot_file */ + for (i = 0; i < module->import_table_count; i++, import_table++) { + read_uint8(buf, buf_end, import_table->elem_type); + read_uint8(buf, buf_end, import_table->table_flags); + read_uint8(buf, buf_end, import_table->possible_grow); +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(import_table->elem_type)) { + read_uint8(buf, buf_end, ref_type.ref_ht_common.nullable); + } +#endif + read_uint32(buf, buf_end, import_table->table_init_size); + read_uint32(buf, buf_end, import_table->table_max_size); +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(import_table->elem_type)) { + read_uint32(buf, buf_end, ref_type.ref_ht_common.heap_type); + + ref_type.ref_type = import_table->elem_type; + /* TODO: check ref_type */ + if (!(import_table->elem_ref_type = wasm_reftype_set_insert( + module->ref_type_set, &ref_type))) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + return false; + } + } +#endif + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_table_list(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTTable *table; +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; +#endif + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTTable) * (uint64)module->table_count; + if (!(module->tables = table = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each table data segment */ + for (i = 0; i < module->table_count; i++, table++) { + read_uint8(buf, buf_end, table->elem_type); + read_uint8(buf, buf_end, table->table_flags); + read_uint8(buf, buf_end, table->possible_grow); +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(table->elem_type)) { + read_uint8(buf, buf_end, ref_type.ref_ht_common.nullable); + } +#endif + read_uint32(buf, buf_end, table->table_init_size); + read_uint32(buf, buf_end, table->table_max_size); +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(table->elem_type)) { + read_uint32(buf, buf_end, ref_type.ref_ht_common.heap_type); + + ref_type.ref_type = table->elem_type; + /* TODO: check ref_type */ + if (!(table->elem_ref_type = wasm_reftype_set_insert( + module->ref_type_set, &ref_type))) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + return false; + } + } + if (!load_init_expr(&buf, buf_end, module, &table->init_expr, error_buf, + error_buf_size)) + return false; + + if (table->init_expr.init_expr_type >= INIT_EXPR_TYPE_STRUCT_NEW + && table->init_expr.init_expr_type + <= INIT_EXPR_TYPE_EXTERN_CONVERT_ANY) { + set_error_buf(error_buf, error_buf_size, + "unsupported initializer expression for table"); + return false; + } +#endif + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTTableInitData **data_list; +#if WASM_ENABLE_GC != 0 + WASMRefType reftype; +#endif + uint64 size; + uint32 i, j; + + /* Allocate memory */ + size = sizeof(AOTTableInitData *) * (uint64)module->table_init_data_count; + if (!(module->table_init_data_list = data_list = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each table data segment */ + for (i = 0; i < module->table_init_data_count; i++) { + uint32 mode, elem_type; + uint32 table_index, init_expr_type, value_count; + uint64 init_expr_value, size1; + + read_uint32(buf, buf_end, mode); + read_uint32(buf, buf_end, elem_type); + read_uint32(buf, buf_end, table_index); + read_uint32(buf, buf_end, init_expr_type); + read_uint64(buf, buf_end, init_expr_value); +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(elem_type)) { + uint16 ref_type, nullable; + read_uint16(buf, buf_end, ref_type); + if (elem_type != ref_type) { + set_error_buf(error_buf, error_buf_size, "invalid elem type"); + return false; + } + reftype.ref_ht_common.ref_type = (uint8)ref_type; + read_uint16(buf, buf_end, nullable); + if (nullable != 0 && nullable != 1) { + set_error_buf(error_buf, error_buf_size, + "invalid nullable value"); + return false; + } + reftype.ref_ht_common.nullable = (uint8)nullable; + read_uint32(buf, buf_end, reftype.ref_ht_common.heap_type); + } + else +#endif + { + /* Skip 8 byte for ref type info */ + buf += 8; + } + + read_uint32(buf, buf_end, value_count); + + size1 = sizeof(InitializerExpression) * (uint64)value_count; + size = offsetof(AOTTableInitData, init_values) + size1; + if (!(data_list[i] = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + data_list[i]->mode = mode; + data_list[i]->elem_type = elem_type; + data_list[i]->table_index = table_index; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(elem_type)) { + if (!(data_list[i]->elem_ref_type = + reftype_set_insert(module->ref_type_set, &reftype, + error_buf, error_buf_size))) { + goto fail; + } + } +#endif + data_list[i]->offset.init_expr_type = (uint8)init_expr_type; + data_list[i]->offset.u.i64 = (int64)init_expr_value; + data_list[i]->value_count = value_count; + for (j = 0; j < data_list[i]->value_count; j++) { + if (!load_init_expr(&buf, buf_end, module, + &data_list[i]->init_values[j], error_buf, + error_buf_size)) + return false; + } + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_table_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->import_table_count); + if (module->import_table_count > 0 + && !load_import_table_list(&buf, buf_end, module, error_buf, + error_buf_size)) + return false; + + read_uint32(buf, buf_end, module->table_count); + if (module->table_count > 0 + && !load_table_list(&buf, buf_end, module, error_buf, error_buf_size)) + return false; + + read_uint32(buf, buf_end, module->table_init_data_count); + + /* load table init data list */ + if (module->table_init_data_count > 0 + && !load_table_init_data_list(&buf, buf_end, module, error_buf, + error_buf_size)) + return false; + + *p_buf = buf; + return true; +fail: + return false; +} + +static void +destroy_type(AOTType *type) +{ +#if WASM_ENABLE_GC != 0 + if (type->ref_count > 1) { + /* The type is referenced by other types + of current aot module */ + type->ref_count--; + return; + } + + if (type->type_flag == WASM_TYPE_FUNC) { + AOTFuncType *func_type = (AOTFuncType *)type; + if (func_type->ref_type_maps != NULL) { + bh_assert(func_type->ref_type_map_count > 0); + wasm_runtime_free(func_type->ref_type_maps); + } + } + else if (type->type_flag == WASM_TYPE_STRUCT) { + AOTStructType *struct_type = (AOTStructType *)type; + if (struct_type->ref_type_maps != NULL) { + bh_assert(struct_type->ref_type_map_count > 0); + wasm_runtime_free(struct_type->ref_type_maps); + } + } +#endif + wasm_runtime_free(type); +} + +static void +destroy_types(AOTType **types, uint32 count) +{ + uint32 i; + for (i = 0; i < count; i++) { + if (types[i]) { + destroy_type(types[i]); + } + } + wasm_runtime_free(types); +} + +#if WASM_ENABLE_GC != 0 +static void +init_base_type(AOTType *base_type, uint32 type_idx, uint16 type_flag, + bool is_sub_final, uint32 parent_type_idx, uint16 rec_count, + uint16 rec_idx) +{ + base_type->type_flag = type_flag; + base_type->ref_count = 1; + base_type->is_sub_final = is_sub_final; + base_type->parent_type_idx = parent_type_idx; + base_type->rec_count = rec_count; + base_type->rec_idx = rec_idx; + base_type->rec_begin_type_idx = type_idx - rec_idx; +} + +static bool +load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTType **types; + uint64 size; + uint32 i, j; + uint32 type_flag, param_cell_num, ret_cell_num; + uint16 param_count, result_count, ref_type_map_count, rec_count, rec_idx; + bool is_equivalence_type, is_sub_final; + uint32 parent_type_idx; + WASMRefType ref_type; + + /* Allocate memory */ + size = sizeof(AOTFuncType *) * (uint64)module->type_count; + if (!(types = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + module->types = types; + + /* Create each type */ + for (i = 0; i < module->type_count; i++) { + buf = align_ptr(buf, 4); + + /* Read base type info */ + read_uint16(buf, buf_end, type_flag); + + read_uint8(buf, buf_end, is_equivalence_type); + /* If there is an equivalence type, re-use it */ + if (is_equivalence_type) { + uint8 u8; + /* padding */ + read_uint8(buf, buf_end, u8); + (void)u8; + + read_uint32(buf, buf_end, j); + if (module->types[j]->ref_count == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "wasm type's ref count too large"); + goto fail; + } + module->types[j]->ref_count++; + module->types[i] = module->types[j]; + continue; + } + + read_uint8(buf, buf_end, is_sub_final); + read_uint32(buf, buf_end, parent_type_idx); + read_uint16(buf, buf_end, rec_count); + read_uint16(buf, buf_end, rec_idx); + + if (type_flag == WASM_TYPE_FUNC) { + AOTFuncType *func_type; + + /* Read param count */ + read_uint16(buf, buf_end, param_count); + /* Read result count */ + read_uint16(buf, buf_end, result_count); + /* Read ref_type_map_count */ + read_uint16(buf, buf_end, ref_type_map_count); + + func_type = + loader_malloc(sizeof(AOTFuncType) + param_count + result_count, + error_buf, error_buf_size); + + if (!func_type) { + goto fail; + } + + types[i] = (AOTType *)func_type; + + init_base_type((AOTType *)func_type, i, type_flag, is_sub_final, + parent_type_idx, rec_count, rec_idx); + func_type->param_count = param_count; + func_type->result_count = result_count; + + /* Read types of params */ + read_byte_array(buf, buf_end, func_type->types, + func_type->param_count + func_type->result_count); + + func_type->ref_type_map_count = ref_type_map_count; + + param_cell_num = wasm_get_cell_num(func_type->types, param_count); + ret_cell_num = + wasm_get_cell_num(func_type->types + param_count, result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + goto fail; + } + + func_type->param_cell_num = param_cell_num; + func_type->ret_cell_num = ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + func_type->quick_aot_entry = + wasm_native_lookup_quick_aot_entry(func_type); +#endif + + LOG_VERBOSE("type %u: func, param count: %d, result count: %d, " + "ref type map count: %d", + i, param_count, result_count, ref_type_map_count); + + /* If ref_type_map is not empty, read ref_type_map */ + if (ref_type_map_count > 0) { + bh_assert(func_type->ref_type_map_count + <= func_type->param_count + func_type->result_count); + + /* align to 4 since param_count + result_count may be odd */ + buf = align_ptr(buf, 4); + + if (!(func_type->ref_type_maps = + loader_malloc(sizeof(WASMRefTypeMap) + * func_type->ref_type_map_count, + error_buf, error_buf_size))) { + goto fail; + } + + for (j = 0; j < func_type->ref_type_map_count; j++) { + read_uint16(buf, buf_end, + func_type->ref_type_maps[j].index); + read_uint8(buf, buf_end, ref_type.ref_ht_common.ref_type); + read_uint8(buf, buf_end, ref_type.ref_ht_common.nullable); + read_uint32(buf, buf_end, ref_type.ref_ht_common.heap_type); + /* TODO: check ref_type */ + if (!(func_type->ref_type_maps[j].ref_type = + wasm_reftype_set_insert(module->ref_type_set, + &ref_type))) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + goto fail; + } + } + + func_type->result_ref_type_maps = func_type->ref_type_maps; + for (j = 0; j < func_type->param_count; j++) { + if (wasm_is_type_multi_byte_type(func_type->types[j])) + func_type->result_ref_type_maps++; + } + } + } + else if (type_flag == WASM_TYPE_STRUCT) { + AOTStructType *struct_type; + const uint8 *buf_org; + uint16 *reference_table; + uint16 field_count, ref_field_count = 0; + uint32 offset; + + read_uint16(buf, buf_end, field_count); + read_uint16(buf, buf_end, ref_type_map_count); + + buf_org = buf; + + /* Traverse first time to get ref_field_count */ + for (j = 0; j < field_count; j++) { + uint8 field_flags, field_type; + + read_uint8(buf, buf_end, field_flags); + read_uint8(buf, buf_end, field_type); + if (wasm_is_type_reftype(field_type)) + ref_field_count++; + + (void)field_flags; + } + + struct_type = loader_malloc( + sizeof(AOTStructType) + + sizeof(WASMStructFieldType) * (uint64)field_count + + sizeof(uint16) * (uint64)(ref_field_count + 1), + error_buf, error_buf_size); + if (!struct_type) { + goto fail; + } + + offset = (uint32)offsetof(WASMStructObject, field_data); + types[i] = (AOTType *)struct_type; + + init_base_type((AOTType *)struct_type, i, type_flag, is_sub_final, + parent_type_idx, rec_count, rec_idx); + struct_type->field_count = field_count; + struct_type->ref_type_map_count = ref_type_map_count; + + struct_type->reference_table = reference_table = + (uint16 *)((uint8 *)struct_type + + offsetof(AOTStructType, fields) + + sizeof(WASMStructFieldType) * field_count); + *reference_table++ = ref_field_count; + + LOG_VERBOSE( + "type %u: struct, field count: %d, ref type map count: %d", i, + field_count, ref_type_map_count); + + buf = buf_org; + + /* Traverse again to read each field */ + for (j = 0; j < field_count; j++) { + uint8 field_type, field_size; + + read_uint8(buf, buf_end, struct_type->fields[j].field_flags); + read_uint8(buf, buf_end, field_type); + struct_type->fields[j].field_type = field_type; + struct_type->fields[j].field_size = field_size = + (uint8)wasm_reftype_size(field_type); +#if !(defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_X86_32)) + if (field_size == 2) + offset = align_uint(offset, 2); + else if (field_size >= 4) /* field size is 4 or 8 */ + offset = align_uint(offset, 4); +#endif + struct_type->fields[j].field_offset = offset; + if (wasm_is_type_reftype(field_type)) + *reference_table++ = offset; + offset += field_size; + LOG_VERBOSE(" field: %d, flags: %d, type: %d", j, + struct_type->fields[j].field_flags, + struct_type->fields[j].field_type); + } + + struct_type->total_size = offset; + buf = align_ptr(buf, 4); + + /* If ref_type_map is not empty, read ref_type_map */ + if (ref_type_map_count > 0) { + + bh_assert(struct_type->ref_type_map_count <= field_count); + + if (!(struct_type->ref_type_maps = + loader_malloc(sizeof(WASMRefTypeMap) + * struct_type->ref_type_map_count, + error_buf, error_buf_size))) { + goto fail; + } + + for (j = 0; j < struct_type->ref_type_map_count; j++) { + read_uint16(buf, buf_end, + struct_type->ref_type_maps[j].index); + read_uint8(buf, buf_end, ref_type.ref_ht_common.ref_type); + read_uint8(buf, buf_end, ref_type.ref_ht_common.nullable); + read_uint32(buf, buf_end, ref_type.ref_ht_common.heap_type); + /* TODO: check ref_type */ + if (!(struct_type->ref_type_maps[j].ref_type = + wasm_reftype_set_insert(module->ref_type_set, + &ref_type))) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + goto fail; + } + } + } + } + else if (type_flag == WASM_TYPE_ARRAY) { + AOTArrayType *array_type; + + array_type = + loader_malloc(sizeof(AOTArrayType), error_buf, error_buf_size); + + if (!array_type) { + goto fail; + } + + types[i] = (AOTType *)array_type; + + init_base_type((AOTType *)array_type, i, type_flag, is_sub_final, + parent_type_idx, rec_count, rec_idx); + read_uint16(buf, buf_end, array_type->elem_flags); + read_uint8(buf, buf_end, array_type->elem_type); + if (wasm_is_type_multi_byte_type(array_type->elem_type)) { + read_uint8(buf, buf_end, ref_type.ref_ht_common.nullable); + read_uint32(buf, buf_end, ref_type.ref_ht_common.heap_type); + ref_type.ref_type = array_type->elem_type; + /* TODO: check ref_type */ + if (!(array_type->elem_ref_type = wasm_reftype_set_insert( + module->ref_type_set, &ref_type))) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + goto fail; + } + } + + LOG_VERBOSE("type %u: array", i); + } + else { + set_error_buf_v(error_buf, error_buf_size, + "invalid type flag: %" PRIu32, type_flag); + goto fail; + } + + if ((rec_count == 0) || (rec_idx == rec_count - 1)) { + if (rec_count == 0) { + bh_assert(rec_idx == 0); + } + for (j = i - rec_idx; j <= i; j++) { + AOTType *cur_type = module->types[j]; + parent_type_idx = cur_type->parent_type_idx; + if (parent_type_idx != (uint32)-1) { /* has parent */ + AOTType *parent_type = module->types[parent_type_idx]; + + module->types[j]->parent_type = parent_type; + module->types[j]->root_type = parent_type->root_type; + if (parent_type->inherit_depth == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "parent type's inherit depth too large"); + goto fail; + } + module->types[j]->inherit_depth = + parent_type->inherit_depth + 1; + } + else { + module->types[j]->parent_type = NULL; + module->types[j]->root_type = module->types[j]; + module->types[j]->inherit_depth = 0; + } + } + + for (j = i - rec_idx; j <= i; j++) { + AOTType *cur_type = module->types[j]; + parent_type_idx = cur_type->parent_type_idx; + if (parent_type_idx != (uint32)-1) { /* has parent */ + AOTType *parent_type = module->types[parent_type_idx]; + /* subtyping has been checked during compilation */ + bh_assert(wasm_type_is_subtype_of( + module->types[j], parent_type, module->types, i + 1)); + (void)parent_type; + } + } + } + } + + if (module->type_count) { + if (!(module->rtt_types = loader_malloc((uint64)sizeof(WASMRttType *) + * module->type_count, + error_buf, error_buf_size))) { + goto fail; + } + } + + *p_buf = buf; + return true; + +fail: + /* Destroy all types */ + destroy_types(types, module->type_count); + module->types = NULL; + return false; +} +#else +static bool +load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTFuncType **func_types; + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTFuncType *) * (uint64)module->type_count; + if (!(func_types = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + module->types = (AOTType **)func_types; + + /* Create each function type */ + for (i = 0; i < module->type_count; i++) { + uint32 type_flag; + uint32 param_count, result_count; + uint32 param_cell_num, ret_cell_num; + uint64 size1; + + buf = align_ptr(buf, 4); + read_uint16(buf, buf_end, type_flag); + read_uint16(buf, buf_end, param_count); + read_uint16(buf, buf_end, result_count); + + size1 = (uint64)param_count + (uint64)result_count; + size = offsetof(AOTFuncType, types) + size1; + if (!(func_types[i] = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + func_types[i]->param_count = (uint16)param_count; + func_types[i]->result_count = (uint16)result_count; + read_byte_array(buf, buf_end, func_types[i]->types, (uint32)size1); + + param_cell_num = wasm_get_cell_num(func_types[i]->types, param_count); + ret_cell_num = + wasm_get_cell_num(func_types[i]->types + param_count, result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + return false; + } + + func_types[i]->param_cell_num = (uint16)param_cell_num; + func_types[i]->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + func_types[i]->quick_aot_entry = + wasm_native_lookup_quick_aot_entry(func_types[i]); +#endif + } + + *p_buf = buf; + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static bool +load_type_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->type_count); + + /* load function type */ + if (module->type_count > 0 + && !load_types(&buf, buf_end, module, error_buf, error_buf_size)) + return false; + + *p_buf = buf; + return true; +fail: + return false; +} + +static void +destroy_import_globals(AOTImportGlobal *import_globals) +{ + wasm_runtime_free(import_globals); +} + +static bool +load_import_globals(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTImportGlobal *import_globals; + uint64 size; + uint32 i, data_offset = 0; +#if WASM_ENABLE_LIBC_BUILTIN != 0 + WASMGlobalImport tmp_global; +#endif + + /* Allocate memory */ + size = sizeof(AOTImportGlobal) * (uint64)module->import_global_count; + if (!(module->import_globals = import_globals = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each import global */ + for (i = 0; i < module->import_global_count; i++) { + buf = (uint8 *)align_ptr(buf, 2); + read_uint8(buf, buf_end, import_globals[i].type); + read_uint8(buf, buf_end, import_globals[i].is_mutable); + read_string(buf, buf_end, import_globals[i].module_name); + read_string(buf, buf_end, import_globals[i].global_name); + +#if WASM_ENABLE_LIBC_BUILTIN != 0 + if (wasm_native_lookup_libc_builtin_global( + import_globals[i].module_name, import_globals[i].global_name, + &tmp_global)) { + if (tmp_global.type != import_globals[i].type + || tmp_global.is_mutable != import_globals[i].is_mutable) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + import_globals[i].global_data_linked = + tmp_global.global_data_linked; + import_globals[i].is_linked = true; + } +#else + import_globals[i].is_linked = false; +#endif + + import_globals[i].size = wasm_value_type_size(import_globals[i].type); + import_globals[i].data_offset = data_offset; + data_offset += import_globals[i].size; + module->global_data_size += import_globals[i].size; + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_import_global_info(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->import_global_count); + + /* load import globals */ + if (module->import_global_count > 0 + && !load_import_globals(&buf, buf_end, module, is_load_from_file_buf, + error_buf, error_buf_size)) + return false; + + *p_buf = buf; + return true; +fail: + return false; +} + +static void +destroy_globals(AOTGlobal *globals) +{ + wasm_runtime_free(globals); +} + +static bool +load_globals(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTGlobal *globals; + uint64 size; + uint32 i, data_offset = 0; + AOTImportGlobal *last_import_global; + + /* Allocate memory */ + size = sizeof(AOTGlobal) * (uint64)module->global_count; + if (!(module->globals = globals = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + if (module->import_global_count > 0) { + last_import_global = + &module->import_globals[module->import_global_count - 1]; + data_offset = + last_import_global->data_offset + last_import_global->size; + } + + /* Create each global */ + for (i = 0; i < module->global_count; i++) { + read_uint8(buf, buf_end, globals[i].type); + read_uint8(buf, buf_end, globals[i].is_mutable); + + buf = align_ptr(buf, 4); + + if (!load_init_expr(&buf, buf_end, module, &globals[i].init_expr, + error_buf, error_buf_size)) + return false; + + globals[i].size = wasm_value_type_size(globals[i].type); + globals[i].data_offset = data_offset; + data_offset += globals[i].size; + module->global_data_size += globals[i].size; + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_global_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->global_count); + + /* load globals */ + if (module->global_count > 0 + && !load_globals(&buf, buf_end, module, error_buf, error_buf_size)) + return false; + + *p_buf = buf; + return true; +fail: + return false; +} + +static void +destroy_import_funcs(AOTImportFunc *import_funcs) +{ + wasm_runtime_free(import_funcs); +} + +static bool +load_import_funcs(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + char *module_name, *field_name; + const uint8 *buf = *p_buf; + AOTImportFunc *import_funcs; + uint64 size; + uint32 i; +#if WASM_ENABLE_MULTI_MODULE != 0 + AOTModule *sub_module = NULL; + AOTFunc *linked_func = NULL; + AOTFuncType *declare_func_type = NULL; +#endif + + /* Allocate memory */ + size = sizeof(AOTImportFunc) * (uint64)module->import_func_count; + if (!(module->import_funcs = import_funcs = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each import func */ + for (i = 0; i < module->import_func_count; i++) { + read_uint16(buf, buf_end, import_funcs[i].func_type_index); + if (import_funcs[i].func_type_index >= module->type_count) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + return false; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + declare_func_type = + (AOTFuncType *)module->types[import_funcs[i].func_type_index]; + read_string(buf, buf_end, module_name); + read_string(buf, buf_end, field_name); + + import_funcs[i].module_name = module_name; + import_funcs[i].func_name = field_name; + linked_func = wasm_native_resolve_symbol( + module_name, field_name, declare_func_type, + &import_funcs[i].signature, &import_funcs[i].attachment, + &import_funcs[i].call_conv_raw); + if (!linked_func) { + if (!wasm_runtime_is_built_in_module(module_name)) { + sub_module = (AOTModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)module, module_name, error_buf, + error_buf_size); + if (!sub_module) { + return false; + } + } + linked_func = aot_loader_resolve_function( + module_name, field_name, declare_func_type, error_buf, + error_buf_size); + } + import_funcs[i].func_ptr_linked = linked_func; + import_funcs[i].func_type = declare_func_type; + +#else + import_funcs[i].func_type = + (AOTFuncType *)module->types[import_funcs[i].func_type_index]; + read_string(buf, buf_end, import_funcs[i].module_name); + read_string(buf, buf_end, import_funcs[i].func_name); + module_name = import_funcs[i].module_name; + field_name = import_funcs[i].func_name; + import_funcs[i].func_ptr_linked = wasm_native_resolve_symbol( + module_name, field_name, import_funcs[i].func_type, + &import_funcs[i].signature, &import_funcs[i].attachment, + &import_funcs[i].call_conv_raw); +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 + if (!strcmp(import_funcs[i].module_name, "wasi_unstable") + || !strcmp(import_funcs[i].module_name, "wasi_snapshot_preview1")) + module->import_wasi_api = true; +#endif + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_import_func_info(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->import_func_count); + + /* load import funcs */ + if (module->import_func_count > 0 + && !load_import_funcs(&buf, buf_end, module, is_load_from_file_buf, + error_buf, error_buf_size)) + return false; + + *p_buf = buf; + return true; +fail: + return false; +} + +static void +destroy_object_data_sections(AOTObjectDataSection *data_sections, + uint32 data_section_count) +{ + uint32 i; + AOTObjectDataSection *data_section = data_sections; + for (i = 0; i < data_section_count; i++, data_section++) + if (data_section->data) { +#if WASM_ENABLE_STATIC_PGO != 0 + if (!strncmp(data_section->name, "__llvm_prf_data", 15)) { + LLVMProfileData *data = (LLVMProfileData *)data_section->data; + if (data->values) { + uint32 num_value_sites = + data->num_value_sites[0] + data->num_value_sites[1]; + uint32 j; + for (j = 0; j < num_value_sites; j++) { + ValueProfNode *node = data->values[j], *node_next; + while (node) { + node_next = node->next; + wasm_runtime_free(node); + node = node_next; + } + } + wasm_runtime_free(data->values); + } + } +#endif + os_munmap(data_section->data, data_section->size); + } + wasm_runtime_free(data_sections); +} + +static bool +load_object_data_sections(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTObjectDataSection *data_sections; + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTObjectDataSection) * (uint64)module->data_section_count; + if (!(module->data_sections = data_sections = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each data section */ + for (i = 0; i < module->data_section_count; i++) { + int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + /* aot code and data in x86_64 must be in range 0 to 2G due to + relocation for R_X86_64_32/32S/PC32 */ + int map_flags = MMAP_MAP_32BIT; +#else + int map_flags = MMAP_MAP_NONE; +#endif + + read_string(buf, buf_end, data_sections[i].name); + read_uint32(buf, buf_end, data_sections[i].size); + + /* Allocate memory for data */ + if (data_sections[i].size > 0 + && !(data_sections[i].data = + os_mmap(NULL, data_sections[i].size, map_prot, map_flags, + os_get_invalid_handle()))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return false; + } +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +#if !defined(BH_PLATFORM_LINUX_SGX) && !defined(BH_PLATFORM_WINDOWS) \ + && !defined(BH_PLATFORM_DARWIN) + /* address must be in the first 2 Gigabytes of + the process address space */ + bh_assert((uintptr_t)data_sections[i].data < INT32_MAX); +#endif +#endif + + read_byte_array(buf, buf_end, data_sections[i].data, + data_sections[i].size); + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_object_data_sections_info(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->data_section_count); + + /* load object data sections */ + if (module->data_section_count > 0 + && !load_object_data_sections(&buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_init_data_section(const uint8 *buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + + if (!load_memory_info(&p, p_end, module, error_buf, error_buf_size) + || !load_table_info(&p, p_end, module, error_buf, error_buf_size) + || !load_type_info(&p, p_end, module, error_buf, error_buf_size) + || !load_import_global_info(&p, p_end, module, is_load_from_file_buf, + error_buf, error_buf_size) + || !load_global_info(&p, p_end, module, error_buf, error_buf_size) + || !load_import_func_info(&p, p_end, module, is_load_from_file_buf, + error_buf, error_buf_size)) + return false; + + /* load function count and start function index */ + read_uint32(p, p_end, module->func_count); + read_uint32(p, p_end, module->start_func_index); + + /* check start function index */ + if (module->start_func_index != (uint32)-1 + && (module->start_func_index + >= module->import_func_count + module->func_count)) { + set_error_buf(error_buf, error_buf_size, + "invalid start function index"); + return false; + } + + read_uint32(p, p_end, module->aux_data_end_global_index); + read_uint64(p, p_end, module->aux_data_end); + read_uint32(p, p_end, module->aux_heap_base_global_index); + read_uint64(p, p_end, module->aux_heap_base); + read_uint32(p, p_end, module->aux_stack_top_global_index); + read_uint64(p, p_end, module->aux_stack_bottom); + read_uint32(p, p_end, module->aux_stack_size); + + if (module->aux_data_end >= MAX_LINEAR_MEMORY_SIZE + || module->aux_heap_base >= MAX_LINEAR_MEMORY_SIZE + || module->aux_stack_bottom >= MAX_LINEAR_MEMORY_SIZE) { + set_error_buf( + error_buf, error_buf_size, + "invalid range of aux_date_end/aux_heap_base/aux_stack_bottom"); + return false; + } + + if (!load_object_data_sections_info(&p, p_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, + "invalid init data section size"); + return false; + } + + return true; +fail: + return false; +} + +static bool +load_text_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint8 *plt_base; + + if (module->func_count > 0 && buf_end == buf) { + set_error_buf(error_buf, error_buf_size, "invalid code size"); + return false; + } + + /* The layout is: literal size + literal + code (with plt table) */ + read_uint32(buf, buf_end, module->literal_size); + + /* literal data is at beginning of the text section */ + module->literal = (uint8 *)buf; + module->code = (void *)(buf + module->literal_size); + module->code_size = (uint32)(buf_end - (uint8 *)module->code); + +#if WASM_ENABLE_DEBUG_AOT != 0 + module->elf_size = module->code_size; + + if (is_ELF(module->code)) { + /* Now code points to an ELF object, we pull it down to .text section */ + uint64 offset; + uint64 size; + char *code_buf = module->code; + module->elf_hdr = code_buf; + if (!get_text_section(code_buf, &offset, &size)) { + set_error_buf(error_buf, error_buf_size, + "get text section of ELF failed"); + return false; + } + module->code = code_buf + offset; + module->code_size -= (uint32)offset; + } +#endif + + if ((module->code_size > 0) && !module->is_indirect_mode) { + plt_base = (uint8 *)buf_end - get_plt_table_size(); + init_plt_table(plt_base); + } + return true; +fail: + return false; +} + +static bool +load_function_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 i; + uint64 size, text_offset; + uint32 func_count = module->func_count; + +#if defined(BUILD_TARGET_XTENSA) + /* + * For Xtensa XIP, real func_count is doubled, including aot_func and + * aot_func_internal, so need to multipy func_count by 2 here. + */ + if (module->is_indirect_mode) { + func_count *= 2; + } +#endif + + size = sizeof(void *) * (uint64)func_count; + if (size > 0 + && !(module->func_ptrs = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < func_count; i++) { + if (sizeof(void *) == 8) { + read_uint64(p, p_end, text_offset); + } + else { + uint32 text_offset32; + read_uint32(p, p_end, text_offset32); + text_offset = text_offset32; + } + if (text_offset >= module->code_size) { + set_error_buf(error_buf, error_buf_size, + "invalid function code offset"); + return false; + } + module->func_ptrs[i] = (uint8 *)module->code + text_offset; +#if defined(BUILD_TARGET_THUMB) || defined(BUILD_TARGET_THUMB_VFP) + /* bits[0] of thumb function address must be 1 */ + module->func_ptrs[i] = (void *)((uintptr_t)module->func_ptrs[i] | 1); +#endif + } + + /* Set start function when function pointers are resolved */ + if (module->start_func_index != (uint32)-1) { + if (module->start_func_index >= module->import_func_count) + module->start_function = + module->func_ptrs[module->start_func_index + - module->import_func_count]; + else + /* TODO: fix start function can be import function issue */ + module->start_function = NULL; + } + else { + module->start_function = NULL; + } + + size = sizeof(uint32) * (uint64)func_count; + if (size > 0 + && !(module->func_type_indexes = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < func_count; i++) { + read_uint32(p, p_end, module->func_type_indexes[i]); + if (module->func_type_indexes[i] >= module->type_count) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + return false; + } + } + + size = sizeof(uint32) * (uint64)module->func_count; + + if (size > 0) { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + if (!(module->max_local_cell_nums = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module->func_count; i++) { + read_uint32(p, p_end, module->max_local_cell_nums[i]); + } + + if (!(module->max_stack_cell_nums = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module->func_count; i++) { + read_uint32(p, p_end, module->max_stack_cell_nums[i]); + } +#else + /* Ignore max_local_cell_num and max_stack_cell_num of each function */ + CHECK_BUF(p, p_end, ((uint32)size * 2)); + p += (uint32)size * 2; +#endif + } + +#if WASM_ENABLE_GC != 0 + /* Local(params and locals) ref flags for all import and non-imported + * functions. The flags indicate whether each cell in the AOTFrame local + * area is a GC reference. */ + size = sizeof(LocalRefFlag) + * (uint64)(module->import_func_count + module->func_count); + if (size > 0) { + if (!(module->func_local_ref_flags = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module->import_func_count + module->func_count; i++) { + uint32 local_ref_flag_cell_num; + + buf = (uint8 *)align_ptr(buf, sizeof(uint32)); + read_uint32( + p, p_end, + module->func_local_ref_flags[i].local_ref_flag_cell_num); + + local_ref_flag_cell_num = + module->func_local_ref_flags[i].local_ref_flag_cell_num; + size = sizeof(uint8) * (uint64)local_ref_flag_cell_num; + if (size > 0) { + if (!(module->func_local_ref_flags[i].local_ref_flags = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + read_byte_array(p, p_end, + module->func_local_ref_flags[i].local_ref_flags, + local_ref_flag_cell_num); + } + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + if (p != buf_end) { + set_error_buf(error_buf, error_buf_size, + "invalid function section size"); + return false; + } + + return true; +fail: + return false; +} + +static void +destroy_exports(AOTExport *exports) +{ + wasm_runtime_free(exports); +} + +static bool +load_exports(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, + bool is_load_from_file_buf, char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTExport *exports; + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTExport) * (uint64)module->export_count; + if (!(module->exports = exports = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each export */ + for (i = 0; i < module->export_count; i++) { + read_uint32(buf, buf_end, exports[i].index); + read_uint8(buf, buf_end, exports[i].kind); + read_string(buf, buf_end, exports[i].name); +#if 0 /* TODO: check kind and index */ + if (export_funcs[i].index >= + module->func_count + module->import_func_count) { + set_error_buf(error_buf, error_buf_size, + "function index is out of range"); + return false; + } +#endif + } + + *p_buf = buf; + return true; +fail: + return false; +} + +static bool +load_export_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + + /* load export functions */ + read_uint32(p, p_end, module->export_count); + if (module->export_count > 0 + && !load_exports(&p, p_end, module, is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "invalid export section size"); + return false; + } + + return true; +fail: + return false; +} + +static void * +get_data_section_addr(AOTModule *module, const char *section_name, + uint32 *p_data_size) +{ + uint32 i; + AOTObjectDataSection *data_section = module->data_sections; + + for (i = 0; i < module->data_section_count; i++, data_section++) { + if (!strcmp(data_section->name, section_name)) { + if (p_data_size) + *p_data_size = data_section->size; + return data_section->data; + } + } + + return NULL; +} + +const void * +aot_get_data_section_addr(AOTModule *module, const char *section_name, + uint32 *p_data_size) +{ + return get_data_section_addr(module, section_name, p_data_size); +} + +static void * +resolve_target_sym(const char *symbol, int32 *p_index) +{ + uint32 i, num = 0; + SymbolMap *target_sym_map; + + if (!(target_sym_map = get_target_symbol_map(&num))) + return NULL; + + for (i = 0; i < num; i++) { + if (!strcmp(target_sym_map[i].symbol_name, symbol) +#if defined(_WIN32) || defined(_WIN32_) + /* In Win32, the symbol name of function added by + LLVMAddFunction() is prefixed by '_', ignore it */ + || (strlen(symbol) > 1 && symbol[0] == '_' + && !strcmp(target_sym_map[i].symbol_name, symbol + 1)) +#endif + ) { + *p_index = (int32)i; + return target_sym_map[i].symbol_addr; + } + } + return NULL; +} + +static bool +is_literal_relocation(const char *reloc_sec_name) +{ + return !strcmp(reloc_sec_name, ".rela.literal"); +} + +static bool +str2uint32(const char *buf, uint32 *p_res) +{ + uint32 res = 0, val; + const char *buf_end = buf + 8; + char ch; + + while (buf < buf_end) { + ch = *buf++; + if (ch >= '0' && ch <= '9') + val = ch - '0'; + else if (ch >= 'a' && ch <= 'f') + val = ch - 'a' + 0xA; + else if (ch >= 'A' && ch <= 'F') + val = ch - 'A' + 0xA; + else + return false; + res = (res << 4) | val; + } + *p_res = res; + return true; +} + +static bool +str2uint64(const char *buf, uint64 *p_res) +{ + uint64 res = 0, val; + const char *buf_end = buf + 16; + char ch; + + while (buf < buf_end) { + ch = *buf++; + if (ch >= '0' && ch <= '9') + val = ch - '0'; + else if (ch >= 'a' && ch <= 'f') + val = ch - 'a' + 0xA; + else if (ch >= 'A' && ch <= 'F') + val = ch - 'A' + 0xA; + else + return false; + res = (res << 4) | val; + } + *p_res = res; + return true; +} + +#define R_X86_64_GOTPCREL 9 /* 32 bit signed PC relative offset to GOT */ + +static bool +is_text_section(const char *section_name) +{ + return !strcmp(section_name, ".text") || !strcmp(section_name, ".ltext"); +} + +static bool +do_text_relocation(AOTModule *module, AOTRelocationGroup *group, + char *error_buf, uint32 error_buf_size) +{ + bool is_literal = is_literal_relocation(group->section_name); + uint8 *aot_text = is_literal ? module->literal : module->code; + uint32 aot_text_size = + is_literal ? module->literal_size : module->code_size; + uint32 i, func_index, symbol_len; +#if defined(BH_PLATFORM_WINDOWS) + uint32 ymm_plt_index = 0, xmm_plt_index = 0; + uint32 real_plt_index = 0, float_plt_index = 0, j; +#endif + char symbol_buf[128] = { 0 }, *symbol, *p; + void *symbol_addr; + AOTRelocation *relocation = group->relocations; + + if (group->relocation_count > 0 && !aot_text) { + set_error_buf(error_buf, error_buf_size, + "invalid text relocation count"); + return false; + } + + for (i = 0; i < group->relocation_count; i++, relocation++) { + int32 symbol_index = -1; + symbol_len = (uint32)strlen(relocation->symbol_name); + if (symbol_len + 1 <= sizeof(symbol_buf)) + symbol = symbol_buf; + else { + if (!(symbol = loader_malloc(symbol_len + 1, error_buf, + error_buf_size))) { + return false; + } + } + bh_memcpy_s(symbol, symbol_len, relocation->symbol_name, symbol_len); + symbol[symbol_len] = '\0'; + +#if WASM_ENABLE_STATIC_PGO != 0 + if (!strcmp(symbol, "__llvm_profile_runtime") + || !strcmp(symbol, "__llvm_profile_register_function") + || !strcmp(symbol, "__llvm_profile_register_names_function")) { + continue; + } +#endif + + if (!strncmp(symbol, AOT_FUNC_PREFIX, strlen(AOT_FUNC_PREFIX))) { + p = symbol + strlen(AOT_FUNC_PREFIX); + if (*p == '\0' + || (func_index = (uint32)atoi(p)) > module->func_count) { + set_error_buf_v(error_buf, error_buf_size, + "invalid import symbol %s", symbol); + goto check_symbol_fail; + } +#if (defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)) \ + && !defined(BH_PLATFORM_WINDOWS) + if (relocation->relocation_type == R_X86_64_GOTPCREL) { + GOTItem *got_item = module->got_item_list; + uint32 got_item_idx = 0; + + while (got_item) { + if (got_item->func_idx == func_index) + break; + got_item_idx++; + got_item = got_item->next; + } + /* Calculate `GOT + G` */ + symbol_addr = module->got_func_ptrs + got_item_idx; + } + else + symbol_addr = module->func_ptrs[func_index]; +#else + symbol_addr = module->func_ptrs[func_index]; +#endif + } +#if defined(BH_PLATFORM_WINDOWS) && defined(BUILD_TARGET_X86_32) + /* AOT function name starts with '_' in windows x86-32 */ + else if (!strncmp(symbol, "_" AOT_FUNC_PREFIX, + strlen("_" AOT_FUNC_PREFIX))) { + p = symbol + strlen("_" AOT_FUNC_PREFIX); + if (*p == '\0' + || (func_index = (uint32)atoi(p)) > module->func_count) { + set_error_buf_v(error_buf, error_buf_size, "invalid symbol %s", + symbol); + goto check_symbol_fail; + } + symbol_addr = module->func_ptrs[func_index]; + } + else if (!strncmp(symbol, "_" AOT_FUNC_INTERNAL_PREFIX, + strlen("_" AOT_FUNC_INTERNAL_PREFIX))) { + p = symbol + strlen("_" AOT_FUNC_INTERNAL_PREFIX); + if (*p == '\0' + || (func_index = (uint32)atoi(p)) > module->func_count) { + set_error_buf_v(error_buf, error_buf_size, "invalid symbol %s", + symbol); + goto check_symbol_fail; + } + symbol_addr = module->func_ptrs[func_index]; + } +#endif + else if (is_text_section(symbol)) { + symbol_addr = module->code; + } + else if (!strcmp(symbol, ".data") || !strcmp(symbol, ".sdata") + || !strcmp(symbol, ".rdata") + || !strcmp(symbol, ".rodata") + /* ".rodata.cst4/8/16/.." */ + || !strncmp(symbol, ".rodata.cst", strlen(".rodata.cst")) + /* ".rodata.strn.m" */ + || !strncmp(symbol, ".rodata.str", strlen(".rodata.str")) + || !strcmp(symbol, AOT_STACK_SIZES_SECTION_NAME) +#if WASM_ENABLE_STATIC_PGO != 0 + || !strncmp(symbol, "__llvm_prf_cnts", 15) + || !strncmp(symbol, "__llvm_prf_data", 15) + || !strncmp(symbol, "__llvm_prf_names", 16) +#endif + ) { + symbol_addr = get_data_section_addr(module, symbol, NULL); + if (!symbol_addr) { + set_error_buf_v(error_buf, error_buf_size, + "invalid data section (%s)", symbol); + goto check_symbol_fail; + } + } + else if (!strcmp(symbol, ".literal")) { + symbol_addr = module->literal; + } +#if defined(BH_PLATFORM_WINDOWS) + /* Relocation for symbols which start with "__ymm@", "__xmm@" or + "__real@" and end with the ymm value, xmm value or real value. + In Windows PE file, the data is stored in some individual ".rdata" + sections. We simply create extra plt data, parse the values from + the symbols and stored them into the extra plt data. */ + else if (!strcmp(group->section_name, ".text") + && !strncmp(symbol, YMM_PLT_PREFIX, strlen(YMM_PLT_PREFIX)) + && strlen(symbol) == strlen(YMM_PLT_PREFIX) + 64) { + char ymm_buf[17] = { 0 }; + + symbol_addr = module->extra_plt_data + ymm_plt_index * 32; + for (j = 0; j < 4; j++) { + bh_memcpy_s(ymm_buf, sizeof(ymm_buf), + symbol + strlen(YMM_PLT_PREFIX) + 48 - 16 * j, 16); + if (!str2uint64(ymm_buf, + (uint64 *)((uint8 *)symbol_addr + 8 * j))) { + set_error_buf_v(error_buf, error_buf_size, + "resolve symbol %s failed", symbol); + goto check_symbol_fail; + } + } + ymm_plt_index++; + } + else if (!strcmp(group->section_name, ".text") + && !strncmp(symbol, XMM_PLT_PREFIX, strlen(XMM_PLT_PREFIX)) + && strlen(symbol) == strlen(XMM_PLT_PREFIX) + 32) { + char xmm_buf[17] = { 0 }; + + symbol_addr = module->extra_plt_data + module->ymm_plt_count * 32 + + xmm_plt_index * 16; + for (j = 0; j < 2; j++) { + bh_memcpy_s(xmm_buf, sizeof(xmm_buf), + symbol + strlen(XMM_PLT_PREFIX) + 16 - 16 * j, 16); + if (!str2uint64(xmm_buf, + (uint64 *)((uint8 *)symbol_addr + 8 * j))) { + set_error_buf_v(error_buf, error_buf_size, + "resolve symbol %s failed", symbol); + goto check_symbol_fail; + } + } + xmm_plt_index++; + } + else if (!strcmp(group->section_name, ".text") + && !strncmp(symbol, REAL_PLT_PREFIX, strlen(REAL_PLT_PREFIX)) + && strlen(symbol) == strlen(REAL_PLT_PREFIX) + 16) { + char real_buf[17] = { 0 }; + + symbol_addr = module->extra_plt_data + module->ymm_plt_count * 32 + + module->xmm_plt_count * 16 + real_plt_index * 8; + bh_memcpy_s(real_buf, sizeof(real_buf), + symbol + strlen(REAL_PLT_PREFIX), 16); + if (!str2uint64(real_buf, (uint64 *)symbol_addr)) { + set_error_buf_v(error_buf, error_buf_size, + "resolve symbol %s failed", symbol); + goto check_symbol_fail; + } + real_plt_index++; + } + else if (!strcmp(group->section_name, ".text") + && !strncmp(symbol, REAL_PLT_PREFIX, strlen(REAL_PLT_PREFIX)) + && strlen(symbol) == strlen(REAL_PLT_PREFIX) + 8) { + char float_buf[9] = { 0 }; + + symbol_addr = module->extra_plt_data + module->ymm_plt_count * 32 + + module->xmm_plt_count * 16 + + module->real_plt_count * 8 + float_plt_index * 4; + bh_memcpy_s(float_buf, sizeof(float_buf), + symbol + strlen(REAL_PLT_PREFIX), 8); + if (!str2uint32(float_buf, (uint32 *)symbol_addr)) { + set_error_buf_v(error_buf, error_buf_size, + "resolve symbol %s failed", symbol); + goto check_symbol_fail; + } + float_plt_index++; + } +#endif /* end of defined(BH_PLATFORM_WINDOWS) */ + else if (!(symbol_addr = resolve_target_sym(symbol, &symbol_index))) { + set_error_buf_v(error_buf, error_buf_size, + "resolve symbol %s failed", symbol); + goto check_symbol_fail; + } + + if (symbol != symbol_buf) + wasm_runtime_free(symbol); + + if (!apply_relocation( + module, aot_text, aot_text_size, relocation->relocation_offset, + relocation->relocation_addend, relocation->relocation_type, + symbol_addr, symbol_index, error_buf, error_buf_size)) + return false; + } + + return true; + +check_symbol_fail: + if (symbol != symbol_buf) + wasm_runtime_free(symbol); + return false; +} + +static bool +do_data_relocation(AOTModule *module, AOTRelocationGroup *group, + char *error_buf, uint32 error_buf_size) + +{ + uint8 *data_addr; + uint32 data_size = 0, i; + AOTRelocation *relocation = group->relocations; + void *symbol_addr; + char *symbol, *data_section_name; + + if (!strncmp(group->section_name, ".rela.", 6)) { + data_section_name = group->section_name + strlen(".rela"); + } + else if (!strncmp(group->section_name, ".rel.", 5)) { + data_section_name = group->section_name + strlen(".rel"); + } + else if (!strcmp(group->section_name, ".rdata")) { + data_section_name = group->section_name; + } +#if WASM_ENABLE_STATIC_PGO != 0 + else if (!strncmp(group->section_name, ".rel__llvm_prf_data", 19)) { + data_section_name = group->section_name + strlen(".rel"); + } + else if (!strncmp(group->section_name, ".rela__llvm_prf_data", 20)) { + data_section_name = group->section_name + strlen(".rela"); + } +#endif + else { + set_error_buf(error_buf, error_buf_size, + "invalid data relocation section name"); + return false; + } + + data_addr = get_data_section_addr(module, data_section_name, &data_size); + + if (group->relocation_count > 0 && !data_addr) { + set_error_buf(error_buf, error_buf_size, + "invalid data relocation count"); + return false; + } + + for (i = 0; i < group->relocation_count; i++, relocation++) { + symbol = relocation->symbol_name; + if (is_text_section(symbol)) { + symbol_addr = module->code; + } +#if WASM_ENABLE_STATIC_PGO != 0 + else if (!strncmp(symbol, AOT_FUNC_PREFIX, strlen(AOT_FUNC_PREFIX))) { + char *p = symbol + strlen(AOT_FUNC_PREFIX); + uint32 func_index; + if (*p == '\0' + || (func_index = (uint32)atoi(p)) > module->func_count) { + set_error_buf_v(error_buf, error_buf_size, + "invalid relocation symbol %s", symbol); + return false; + } + symbol_addr = module->func_ptrs[func_index]; + } + else if (!strcmp(symbol, "__llvm_prf_cnts")) { + uint32 j; + for (j = 0; j < module->data_section_count; j++) { + if (!strncmp(module->data_sections[j].name, symbol, 15)) { + bh_assert(relocation->relocation_addend + sizeof(uint64) + <= module->data_sections[j].size); + symbol_addr = module->data_sections[j].data; + break; + } + } + if (j == module->data_section_count) { + set_error_buf_v(error_buf, error_buf_size, + "invalid relocation symbol %s", symbol); + return false; + } + } + else if (!strncmp(symbol, "__llvm_prf_cnts", 15)) { + uint32 j; + for (j = 0; j < module->data_section_count; j++) { + if (!strcmp(module->data_sections[j].name, symbol)) { + symbol_addr = module->data_sections[j].data; + break; + } + } + if (j == module->data_section_count) { + set_error_buf_v(error_buf, error_buf_size, + "invalid relocation symbol %s", symbol); + return false; + } + } +#endif /* end of WASM_ENABLE_STATIC_PGO != 0 */ + else { + set_error_buf_v(error_buf, error_buf_size, + "invalid relocation symbol %s", symbol); + return false; + } + + if (!apply_relocation( + module, data_addr, data_size, relocation->relocation_offset, + relocation->relocation_addend, relocation->relocation_type, + symbol_addr, -1, error_buf, error_buf_size)) + return false; + } + + return true; +} + +static bool +validate_symbol_table(uint8 *buf, uint8 *buf_end, uint32 *offsets, uint32 count, + char *error_buf, uint32 error_buf_size) +{ + uint32 i, str_len_addr = 0; + uint16 str_len; + + for (i = 0; i < count; i++) { + if (offsets[i] != str_len_addr) + return false; + + read_uint16(buf, buf_end, str_len); + str_len_addr += (uint32)sizeof(uint16) + str_len; + str_len_addr = align_uint(str_len_addr, 2); + buf += str_len; + buf = (uint8 *)align_ptr(buf, 2); + } + + if (buf == buf_end) + return true; +fail: + return false; +} + +static bool +load_relocation_section(const uint8 *buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + AOTRelocationGroup *groups = NULL, *group; + uint32 symbol_count = 0; + uint32 group_count = 0, i, j, got_item_count = 0; + uint64 size; + uint32 *symbol_offsets, total_string_len; + uint8 *symbol_buf, *symbol_buf_end; + int map_prot, map_flags; + bool ret = false; + char **symbols = NULL; + + read_uint32(buf, buf_end, symbol_count); + + symbol_offsets = (uint32 *)buf; + for (i = 0; i < symbol_count; i++) { + CHECK_BUF(buf, buf_end, sizeof(uint32)); + buf += sizeof(uint32); + } + + read_uint32(buf, buf_end, total_string_len); + symbol_buf = (uint8 *)buf; + symbol_buf_end = symbol_buf + total_string_len; + + if (!validate_symbol_table(symbol_buf, symbol_buf_end, symbol_offsets, + symbol_count, error_buf, error_buf_size)) { + set_error_buf(error_buf, error_buf_size, + "validate symbol table failed"); + goto fail; + } + + if (symbol_count > 0) { + symbols = loader_malloc((uint64)sizeof(*symbols) * symbol_count, + error_buf, error_buf_size); + if (symbols == NULL) { + goto fail; + } + } + +#if defined(BH_PLATFORM_WINDOWS) + buf = symbol_buf_end; + read_uint32(buf, buf_end, group_count); + + for (i = 0; i < group_count; i++) { + uint32 name_index, relocation_count; + uint16 group_name_len; + uint8 *group_name; + + /* section name address is 4 bytes aligned. */ + buf = (uint8 *)align_ptr(buf, sizeof(uint32)); + read_uint32(buf, buf_end, name_index); + + if (name_index >= symbol_count) { + set_error_buf(error_buf, error_buf_size, + "symbol index out of range"); + goto fail; + } + + group_name = symbol_buf + symbol_offsets[name_index]; + group_name_len = *(uint16 *)group_name; + group_name += sizeof(uint16); + + read_uint32(buf, buf_end, relocation_count); + + for (j = 0; j < relocation_count; j++) { + AOTRelocation relocation = { 0 }; + char group_name_buf[128] = { 0 }; + char symbol_name_buf[128] = { 0 }; + uint32 symbol_index, offset32; + int32 addend32; + uint16 symbol_name_len; + uint8 *symbol_name; + + if (sizeof(void *) == 8) { + read_uint64(buf, buf_end, relocation.relocation_offset); + read_uint64(buf, buf_end, relocation.relocation_addend); + } + else { + read_uint32(buf, buf_end, offset32); + relocation.relocation_offset = (uint64)offset32; + read_uint32(buf, buf_end, addend32); + relocation.relocation_addend = (int64)addend32; + } + read_uint32(buf, buf_end, relocation.relocation_type); + read_uint32(buf, buf_end, symbol_index); + + if (symbol_index >= symbol_count) { + set_error_buf(error_buf, error_buf_size, + "symbol index out of range"); + goto fail; + } + + symbol_name = symbol_buf + symbol_offsets[symbol_index]; + symbol_name_len = *(uint16 *)symbol_name; + symbol_name += sizeof(uint16); + + bh_memcpy_s(group_name_buf, (uint32)sizeof(group_name_buf), + group_name, group_name_len); + bh_memcpy_s(symbol_name_buf, (uint32)sizeof(symbol_name_buf), + symbol_name, symbol_name_len); + + /* aot compiler emits string with '\0' since 2.0.0 */ + if (group_name_len == strlen(".text") + 1 + && !strncmp(group_name, ".text", strlen(".text"))) { + /* aot compiler emits string with '\0' since 2.0.0 */ + if (symbol_name_len == strlen(YMM_PLT_PREFIX) + 64 + 1 + && !strncmp(symbol_name, YMM_PLT_PREFIX, + strlen(YMM_PLT_PREFIX))) { + module->ymm_plt_count++; + } + else if (symbol_name_len == strlen(XMM_PLT_PREFIX) + 32 + 1 + && !strncmp(symbol_name, XMM_PLT_PREFIX, + strlen(XMM_PLT_PREFIX))) { + module->xmm_plt_count++; + } + else if (symbol_name_len == strlen(REAL_PLT_PREFIX) + 16 + 1 + && !strncmp(symbol_name, REAL_PLT_PREFIX, + strlen(REAL_PLT_PREFIX))) { + module->real_plt_count++; + } + else if (symbol_name_len >= strlen(REAL_PLT_PREFIX) + 8 + 1 + && !strncmp(symbol_name, REAL_PLT_PREFIX, + strlen(REAL_PLT_PREFIX))) { + module->float_plt_count++; + } + } + } + } + + /* Allocate memory for extra plt data */ + size = sizeof(uint64) * 4 * module->ymm_plt_count + + sizeof(uint64) * 2 * module->xmm_plt_count + + sizeof(uint64) * module->real_plt_count + + sizeof(uint32) * module->float_plt_count; + if (size > 0) { + map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC; + /* aot code and data in x86_64 must be in range 0 to 2G due to + relocation for R_X86_64_32/32S/PC32 */ + map_flags = MMAP_MAP_32BIT; + + if (size > UINT32_MAX + || !(module->extra_plt_data = + os_mmap(NULL, (uint32)size, map_prot, map_flags, + os_get_invalid_handle()))) { + set_error_buf(error_buf, error_buf_size, "mmap memory failed"); + goto fail; + } + module->extra_plt_data_size = (uint32)size; + } +#endif /* end of defined(BH_PLATFORM_WINDOWS) */ + +#if (defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)) \ + && !defined(BH_PLATFORM_WINDOWS) + buf = symbol_buf_end; + read_uint32(buf, buf_end, group_count); + + /* Resolve the relocations of type R_X86_64_GOTPCREL */ + for (i = 0; i < group_count; i++) { + uint32 name_index, relocation_count; + uint16 group_name_len; + uint8 *group_name; + + /* section name address is 4 bytes aligned. */ + buf = (uint8 *)align_ptr(buf, sizeof(uint32)); + read_uint32(buf, buf_end, name_index); + + if (name_index >= symbol_count) { + set_error_buf(error_buf, error_buf_size, + "symbol index out of range"); + goto fail; + } + + group_name = symbol_buf + symbol_offsets[name_index]; + group_name_len = *(uint16 *)group_name; + group_name += sizeof(uint16); + + read_uint32(buf, buf_end, relocation_count); + + for (j = 0; j < relocation_count; j++) { + AOTRelocation relocation = { 0 }; + char group_name_buf[128] = { 0 }; + char symbol_name_buf[128] = { 0 }; + uint32 symbol_index; + uint16 symbol_name_len; + uint8 *symbol_name; + + /* relocation offset and addend */ + buf += sizeof(void *) * 2; + + read_uint32(buf, buf_end, relocation.relocation_type); + read_uint32(buf, buf_end, symbol_index); + + if (symbol_index >= symbol_count) { + set_error_buf(error_buf, error_buf_size, + "symbol index out of range"); + goto fail; + } + + symbol_name = symbol_buf + symbol_offsets[symbol_index]; + symbol_name_len = *(uint16 *)symbol_name; + symbol_name += sizeof(uint16); + + bh_memcpy_s(group_name_buf, (uint32)sizeof(group_name_buf), + group_name, group_name_len); + bh_memcpy_s(symbol_name_buf, (uint32)sizeof(symbol_name_buf), + symbol_name, symbol_name_len); + + if (relocation.relocation_type == R_X86_64_GOTPCREL + && !strncmp(symbol_name_buf, AOT_FUNC_PREFIX, + strlen(AOT_FUNC_PREFIX))) { + uint32 func_idx = + atoi(symbol_name_buf + strlen(AOT_FUNC_PREFIX)); + GOTItem *got_item = module->got_item_list; + + if (func_idx >= module->func_count) { + set_error_buf(error_buf, error_buf_size, + "func index out of range"); + goto fail; + } + + while (got_item) { + if (got_item->func_idx == func_idx) + break; + got_item = got_item->next; + } + + if (!got_item) { + /* Create the got item and append to the list */ + got_item = wasm_runtime_malloc(sizeof(GOTItem)); + if (!got_item) { + set_error_buf(error_buf, error_buf_size, + "allocate memory failed"); + goto fail; + } + + got_item->func_idx = func_idx; + got_item->next = NULL; + if (!module->got_item_list) { + module->got_item_list = module->got_item_list_end = + got_item; + } + else { + module->got_item_list_end->next = got_item; + module->got_item_list_end = got_item; + } + + got_item_count++; + } + } + } + } + + if (got_item_count) { + GOTItem *got_item = module->got_item_list; + uint32 got_item_idx = 0; + + map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE; + /* aot code and data in x86_64 must be in range 0 to 2G due to + relocation for R_X86_64_32/32S/PC32 */ + map_flags = MMAP_MAP_32BIT; + + /* Create the GOT for func_ptrs, note that it is different from + the .got section of a dynamic object file */ + size = (uint64)sizeof(void *) * got_item_count; + if (size > UINT32_MAX + || !(module->got_func_ptrs = + os_mmap(NULL, (uint32)size, map_prot, map_flags, + os_get_invalid_handle()))) { + set_error_buf(error_buf, error_buf_size, "mmap memory failed"); + goto fail; + } + + while (got_item) { + module->got_func_ptrs[got_item_idx++] = + module->func_ptrs[got_item->func_idx]; + got_item = got_item->next; + } + + module->got_item_count = got_item_count; + } +#else + (void)got_item_count; +#endif /* (defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)) && \ + !defined(BH_PLATFORM_WINDOWS) */ + + buf = symbol_buf_end; + read_uint32(buf, buf_end, group_count); + + /* Allocate memory for relocation groups */ + size = sizeof(AOTRelocationGroup) * (uint64)group_count; + if (size > 0 + && !(groups = loader_malloc(size, error_buf, error_buf_size))) { + goto fail; + } + + /* Load each relocation group */ + for (i = 0, group = groups; i < group_count; i++, group++) { + AOTRelocation *relocation; + uint32 name_index; + + /* section name address is 4 bytes aligned. */ + buf = (uint8 *)align_ptr(buf, sizeof(uint32)); + read_uint32(buf, buf_end, name_index); + + if (name_index >= symbol_count) { + set_error_buf(error_buf, error_buf_size, + "symbol index out of range"); + goto fail; + } + + if (symbols[name_index] == NULL) { + uint8 *name_addr = symbol_buf + symbol_offsets[name_index]; + + read_string(name_addr, buf_end, symbols[name_index]); + } + group->section_name = symbols[name_index]; + + read_uint32(buf, buf_end, group->relocation_count); + + /* Allocate memory for relocations */ + size = sizeof(AOTRelocation) * (uint64)group->relocation_count; + if (!(group->relocations = relocation = + loader_malloc(size, error_buf, error_buf_size))) { + ret = false; + goto fail; + } + + /* Load each relocation */ + for (j = 0; j < group->relocation_count; j++, relocation++) { + uint32 symbol_index; + + if (sizeof(void *) == 8) { + read_uint64(buf, buf_end, relocation->relocation_offset); + read_uint64(buf, buf_end, relocation->relocation_addend); + } + else { + uint32 offset32, addend32; + read_uint32(buf, buf_end, offset32); + relocation->relocation_offset = (uint64)offset32; + read_uint32(buf, buf_end, addend32); + relocation->relocation_addend = (uint64)addend32; + } + read_uint32(buf, buf_end, relocation->relocation_type); + read_uint32(buf, buf_end, symbol_index); + + if (symbol_index >= symbol_count) { + set_error_buf(error_buf, error_buf_size, + "symbol index out of range"); + goto fail; + } + + if (symbols[symbol_index] == NULL) { + uint8 *symbol_addr = symbol_buf + symbol_offsets[symbol_index]; + + read_string(symbol_addr, buf_end, symbols[symbol_index]); + } + relocation->symbol_name = symbols[symbol_index]; + } + + if (!strcmp(group->section_name, ".rel.text") + || !strcmp(group->section_name, ".rela.text") + || !strcmp(group->section_name, ".rel.ltext") + || !strcmp(group->section_name, ".rela.ltext") + || !strcmp(group->section_name, ".rela.literal") +#ifdef BH_PLATFORM_WINDOWS + || !strcmp(group->section_name, ".text") +#endif + ) { +#if !defined(BH_PLATFORM_LINUX) && !defined(BH_PLATFORM_LINUX_SGX) \ + && !defined(BH_PLATFORM_DARWIN) && !defined(BH_PLATFORM_WINDOWS) + if (module->is_indirect_mode) { + set_error_buf(error_buf, error_buf_size, + "cannot apply relocation to text section " + "for aot file generated with " + "\"--enable-indirect-mode\" flag"); + goto fail; + } +#endif + if (!do_text_relocation(module, group, error_buf, error_buf_size)) + goto fail; + } + else { + if (!do_data_relocation(module, group, error_buf, error_buf_size)) + goto fail; + } + } + + /* Set read only for AOT code and some data sections */ + map_prot = MMAP_PROT_READ | MMAP_PROT_EXEC; + + if (module->code) { + /* The layout is: literal size + literal + code (with plt table) */ + uint8 *mmap_addr = module->literal - sizeof(uint32); + uint32 total_size = + sizeof(uint32) + module->literal_size + module->code_size; + os_mprotect(mmap_addr, total_size, map_prot); + } + + map_prot = MMAP_PROT_READ; + +#if defined(BH_PLATFORM_WINDOWS) + if (module->extra_plt_data) { + os_mprotect(module->extra_plt_data, module->extra_plt_data_size, + map_prot); + } +#endif + + for (i = 0; i < module->data_section_count; i++) { + AOTObjectDataSection *data_section = module->data_sections + i; + if (!strcmp(data_section->name, ".rdata") + || !strcmp(data_section->name, ".rodata") + /* ".rodata.cst4/8/16/.." */ + || !strncmp(data_section->name, ".rodata.cst", + strlen(".rodata.cst")) + /* ".rodata.strn.m" */ + || !strncmp(data_section->name, ".rodata.str", + strlen(".rodata.str"))) { + os_mprotect(data_section->data, data_section->size, map_prot); + } + } + + ret = true; + +fail: + if (symbols) { + wasm_runtime_free(symbols); + } + if (groups) { + for (i = 0, group = groups; i < group_count; i++, group++) + if (group->relocations) + wasm_runtime_free(group->relocations); + wasm_runtime_free(groups); + } + + (void)map_flags; + return ret; +} + +static bool +load_from_sections(AOTModule *module, AOTSection *sections, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + AOTSection *section = sections; + const uint8 *buf, *buf_end; + uint32 last_section_type = (uint32)-1, section_type; + uint32 i, func_index, func_type_index; + AOTFuncType *func_type; + AOTExport *exports; + + while (section) { + buf = section->section_body; + buf_end = buf + section->section_body_size; + /* Check sections */ + section_type = (uint32)section->section_type; + if ((last_section_type == (uint32)-1 + && section_type != AOT_SECTION_TYPE_TARGET_INFO) + || (last_section_type != (uint32)-1 + && (section_type != last_section_type + 1 + && section_type != AOT_SECTION_TYPE_CUSTOM))) { + set_error_buf(error_buf, error_buf_size, "invalid section order"); + return false; + } + last_section_type = section_type; + switch (section_type) { + case AOT_SECTION_TYPE_TARGET_INFO: + if (!load_target_info_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case AOT_SECTION_TYPE_INIT_DATA: + if (!load_init_data_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case AOT_SECTION_TYPE_TEXT: + if (!load_text_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case AOT_SECTION_TYPE_FUNCTION: + if (!load_function_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case AOT_SECTION_TYPE_EXPORT: + if (!load_export_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case AOT_SECTION_TYPE_RELOCATION: + if (!load_relocation_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case AOT_SECTION_TYPE_CUSTOM: + if (!load_custom_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + default: + set_error_buf(error_buf, error_buf_size, + "invalid aot section type"); + return false; + } + + section = section->next; + } + + if (last_section_type != AOT_SECTION_TYPE_RELOCATION + && last_section_type != AOT_SECTION_TYPE_CUSTOM) { + set_error_buf(error_buf, error_buf_size, "section missing"); + return false; + } + + /* Resolve malloc and free function */ + module->malloc_func_index = (uint32)-1; + module->free_func_index = (uint32)-1; + module->retain_func_index = (uint32)-1; + + exports = module->exports; + for (i = 0; i < module->export_count; i++) { + if (exports[i].kind == EXPORT_KIND_FUNC + && exports[i].index >= module->import_func_count) { + if (!strcmp(exports[i].name, "malloc")) { + func_index = exports[i].index - module->import_func_count; + func_type_index = module->func_type_indexes[func_index]; + func_type = (AOTFuncType *)module->types[func_type_index]; + if (func_type->param_count == 1 && func_type->result_count == 1 + && func_type->types[0] == VALUE_TYPE_I32 + && func_type->types[1] == VALUE_TYPE_I32) { + bh_assert(module->malloc_func_index == (uint32)-1); + module->malloc_func_index = func_index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + exports[i].name, exports[i].index); + } + } + else if (!strcmp(exports[i].name, "__new")) { + func_index = exports[i].index - module->import_func_count; + func_type_index = module->func_type_indexes[func_index]; + func_type = (AOTFuncType *)module->types[func_type_index]; + if (func_type->param_count == 2 && func_type->result_count == 1 + && func_type->types[0] == VALUE_TYPE_I32 + && func_type->types[1] == VALUE_TYPE_I32 + && func_type->types[2] == VALUE_TYPE_I32) { + uint32 j; + WASMExport *export_tmp; + + bh_assert(module->malloc_func_index == (uint32)-1); + module->malloc_func_index = func_index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + exports[i].name, exports[i].index); + + /* resolve retain function. + If not find, reset malloc function index */ + export_tmp = module->exports; + for (j = 0; j < module->export_count; j++, export_tmp++) { + if ((export_tmp->kind == EXPORT_KIND_FUNC) + && (!strcmp(export_tmp->name, "__retain") + || !strcmp(export_tmp->name, "__pin"))) { + func_index = + export_tmp->index - module->import_func_count; + func_type_index = + module->func_type_indexes[func_index]; + func_type = + (AOTFuncType *)module->types[func_type_index]; + if (func_type->param_count == 1 + && func_type->result_count == 1 + && func_type->types[0] == VALUE_TYPE_I32 + && func_type->types[1] == VALUE_TYPE_I32) { + bh_assert(module->retain_func_index + == (uint32)-1); + module->retain_func_index = export_tmp->index; + LOG_VERBOSE("Found retain function, name: %s, " + "index: %u", + export_tmp->name, + export_tmp->index); + break; + } + } + } + if (j == module->export_count) { + module->malloc_func_index = (uint32)-1; + LOG_VERBOSE("Can't find retain function," + "reset malloc function index to -1"); + } + } + } + else if ((!strcmp(exports[i].name, "free")) + || (!strcmp(exports[i].name, "__release")) + || (!strcmp(exports[i].name, "__unpin"))) { + func_index = exports[i].index - module->import_func_count; + func_type_index = module->func_type_indexes[func_index]; + func_type = (AOTFuncType *)module->types[func_type_index]; + if (func_type->param_count == 1 && func_type->result_count == 0 + && func_type->types[0] == VALUE_TYPE_I32) { + bh_assert(module->free_func_index == (uint32)-1); + module->free_func_index = func_index; + LOG_VERBOSE("Found free function, name: %s, index: %u", + exports[i].name, exports[i].index); + } + } + } + } + + /* Flush data cache before executing AOT code, + * otherwise unpredictable behavior can occur. */ + os_dcache_flush(); + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module); +#endif + +#if WASM_ENABLE_DEBUG_AOT != 0 + if (!jit_code_entry_create(module->elf_hdr, module->elf_size)) { + set_error_buf(error_buf, error_buf_size, + "create jit code entry failed"); + return false; + } +#endif + return true; +} + +static AOTModule * +create_module(char *name, char *error_buf, uint32 error_buf_size) +{ + AOTModule *module = + loader_malloc(sizeof(AOTModule), error_buf, error_buf_size); + bh_list_status ret; + + if (!module) { + return NULL; + } + + module->module_type = Wasm_Module_AoT; + + module->name = name; + +#if WASM_ENABLE_MULTI_MODULE != 0 + module->import_module_list = &module->import_module_list_head; + ret = bh_list_init(module->import_module_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + (void)ret; + +#if WASM_ENABLE_GC != 0 + if (!(module->ref_type_set = + wasm_reftype_set_create(GC_REFTYPE_MAP_SIZE_DEFAULT))) { + set_error_buf(error_buf, error_buf_size, "create reftype map failed"); + goto fail1; + } + + if (os_mutex_init(&module->rtt_type_lock)) { + set_error_buf(error_buf, error_buf_size, "init rtt type lock failed"); + goto fail2; + } +#endif + + return module; +#if WASM_ENABLE_GC != 0 +fail2: + bh_hash_map_destroy(module->ref_type_set); +fail1: +#endif + wasm_runtime_free(module); + return NULL; +} + +AOTModule * +aot_load_from_sections(AOTSection *section_list, char *error_buf, + uint32 error_buf_size) +{ + AOTModule *module = create_module("", error_buf, error_buf_size); + + if (!module) + return NULL; + + if (!load_from_sections(module, section_list, false, error_buf, + error_buf_size)) { + aot_unload(module); + return NULL; + } + + LOG_VERBOSE("Load module from sections success.\n"); + return module; +} + +static void +destroy_sections(AOTSection *section_list, bool destroy_aot_text) +{ + AOTSection *section = section_list, *next; + while (section) { + next = section->next; + if (destroy_aot_text && section->section_type == AOT_SECTION_TYPE_TEXT + && section->section_body) + os_munmap((uint8 *)section->section_body, + section->section_body_size); + wasm_runtime_free(section); + section = next; + } +} + +static bool +resolve_execute_mode(const uint8 *buf, uint32 size, bool *p_mode, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf + size; + uint32 section_type; + uint32 section_size = 0; + uint16 e_type = 0; + + p += 8; + while (p < p_end) { + read_uint32(p, p_end, section_type); + if (section_type <= AOT_SECTION_TYPE_SIGANATURE) { + read_uint32(p, p_end, section_size); + CHECK_BUF(p, p_end, section_size); + if (section_type == AOT_SECTION_TYPE_TARGET_INFO) { + p += 4; + read_uint16(p, p_end, e_type); + if (e_type == E_TYPE_XIP) { + *p_mode = true; + } + else { + *p_mode = false; + } + break; + } + } + else { /* section_type > AOT_SECTION_TYPE_SIGANATURE */ + set_error_buf(error_buf, error_buf_size, + "resolve execute mode failed"); + break; + } + p += section_size; + } + return true; +fail: + return false; +} + +static bool +create_sections(AOTModule *module, const uint8 *buf, uint32 size, + AOTSection **p_section_list, char *error_buf, + uint32 error_buf_size) +{ + AOTSection *section_list = NULL, *section_list_end = NULL, *section; + const uint8 *p = buf, *p_end = buf + size; + bool destroy_aot_text = false; + bool is_indirect_mode = false; + uint32 section_type; + uint32 section_size; + uint64 total_size; + uint8 *aot_text; +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + uint8 *mirrored_text; +#endif + + if (!resolve_execute_mode(buf, size, &is_indirect_mode, error_buf, + error_buf_size)) { + goto fail; + } + + module->is_indirect_mode = is_indirect_mode; + + p += 8; + while (p < p_end) { + read_uint32(p, p_end, section_type); + if (section_type < AOT_SECTION_TYPE_SIGANATURE + || section_type == AOT_SECTION_TYPE_CUSTOM) { + read_uint32(p, p_end, section_size); + CHECK_BUF(p, p_end, section_size); + + if (!(section = loader_malloc(sizeof(AOTSection), error_buf, + error_buf_size))) { + goto fail; + } + + memset(section, 0, sizeof(AOTSection)); + section->section_type = (int32)section_type; + section->section_body = (uint8 *)p; + section->section_body_size = section_size; + + if (section_type == AOT_SECTION_TYPE_TEXT) { + if ((section_size > 0) && !module->is_indirect_mode) { + int map_prot = + MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + /* aot code and data in x86_64 must be in range 0 to 2G due + to relocation for R_X86_64_32/32S/PC32 */ + int map_flags = MMAP_MAP_32BIT; +#else + int map_flags = MMAP_MAP_NONE; +#endif + total_size = + (uint64)section_size + aot_get_plt_table_size(); + total_size = (total_size + 3) & ~((uint64)3); + if (total_size >= UINT32_MAX + || !(aot_text = + os_mmap(NULL, (uint32)total_size, map_prot, + map_flags, os_get_invalid_handle()))) { + wasm_runtime_free(section); + set_error_buf(error_buf, error_buf_size, + "mmap memory failed"); + goto fail; + } +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +#if !defined(BH_PLATFORM_LINUX_SGX) && !defined(BH_PLATFORM_WINDOWS) \ + && !defined(BH_PLATFORM_DARWIN) + /* address must be in the first 2 Gigabytes of + the process address space */ + bh_assert((uintptr_t)aot_text < INT32_MAX); +#endif +#endif + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + mirrored_text = os_get_dbus_mirror(aot_text); + bh_assert(mirrored_text != NULL); + bh_memcpy_s(mirrored_text, (uint32)total_size, + section->section_body, (uint32)section_size); + os_dcache_flush(); +#else + bh_memcpy_s(aot_text, (uint32)total_size, + section->section_body, (uint32)section_size); +#endif + section->section_body = aot_text; + destroy_aot_text = true; + + if ((uint32)total_size > section->section_body_size) { + memset(aot_text + (uint32)section_size, 0, + (uint32)total_size - section_size); + section->section_body_size = (uint32)total_size; + } + } + } + + if (!section_list) + section_list = section_list_end = section; + else { + section_list_end->next = section; + section_list_end = section; + } + + p += section_size; + } + else { + set_error_buf(error_buf, error_buf_size, "invalid section id"); + goto fail; + } + } + + if (!section_list) { + set_error_buf(error_buf, error_buf_size, "create section list failed"); + return false; + } + + *p_section_list = section_list; + return true; +fail: + if (section_list) + destroy_sections(section_list, destroy_aot_text); + return false; +} + +static bool +load(const uint8 *buf, uint32 size, AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf_end = buf + size; + const uint8 *p = buf, *p_end = buf_end; + uint32 magic_number, version; + AOTSection *section_list = NULL; + bool ret; + + read_uint32(p, p_end, magic_number); + if (magic_number != AOT_MAGIC_NUMBER) { + set_error_buf(error_buf, error_buf_size, "magic header not detected"); + return false; + } + + read_uint32(p, p_end, version); + if (version != AOT_CURRENT_VERSION) { + set_error_buf(error_buf, error_buf_size, "unknown binary version"); + return false; + } + + if (!create_sections(module, buf, size, §ion_list, error_buf, + error_buf_size)) + return false; + + ret = load_from_sections(module, section_list, true, error_buf, + error_buf_size); + if (!ret) { + /* If load_from_sections() fails, then aot text is destroyed + in destroy_sections() */ + destroy_sections(section_list, module->is_indirect_mode ? false : true); + /* aot_unload() won't destroy aot text again */ + module->code = NULL; + } + else { + /* If load_from_sections() succeeds, then aot text is set to + module->code and will be destroyed in aot_unload() */ + destroy_sections(section_list, false); + } + +#if 0 + { + uint32 i; + for (i = 0; i < module->func_count; i++) { + LOG_VERBOSE("AOT func %u, addr: %p\n", i, module->func_ptrs[i]); + } + } +#endif + +#if WASM_ENABLE_LINUX_PERF != 0 + if (wasm_runtime_get_linux_perf()) + if (!aot_create_perf_map(module, error_buf, error_buf_size)) + goto fail; +#endif + + return ret; +fail: + return false; +} + +AOTModule * +aot_load_from_aot_file(const uint8 *buf, uint32 size, const LoadArgs *args, + char *error_buf, uint32 error_buf_size) +{ + AOTModule *module = create_module(args->name, error_buf, error_buf_size); + + if (!module) + return NULL; + + os_thread_jit_write_protect_np(false); /* Make memory writable */ + if (!load(buf, size, module, error_buf, error_buf_size)) { + aot_unload(module); + return NULL; + } + os_thread_jit_write_protect_np(true); /* Make memory executable */ + os_icache_flush(module->code, module->code_size); + + LOG_VERBOSE("Load module success.\n"); + return module; +} + +void +aot_unload(AOTModule *module) +{ + if (module->import_memories) + destroy_import_memories(module->import_memories); + + if (module->memories) + wasm_runtime_free(module->memories); + + if (module->mem_init_data_list) + destroy_mem_init_data_list(module->mem_init_data_list, + module->mem_init_data_count); + + if (module->native_symbol_list) + wasm_runtime_free(module->native_symbol_list); + + if (module->import_tables) + destroy_import_tables(module->import_tables); + + if (module->tables) { +#if WASM_ENABLE_GC != 0 + uint32 i; + for (i = 0; i < module->table_count; i++) { + destroy_init_expr(&module->tables[i].init_expr); + } +#endif + destroy_tables(module->tables); + } + + if (module->table_init_data_list) + destroy_table_init_data_list(module->table_init_data_list, + module->table_init_data_count); + + if (module->types) + destroy_types(module->types, module->type_count); + + if (module->import_globals) + destroy_import_globals(module->import_globals); + + if (module->globals) { +#if WASM_ENABLE_GC != 0 + uint32 i; + for (i = 0; i < module->global_count; i++) { + destroy_init_expr(&module->globals[i].init_expr); + } +#endif + destroy_globals(module->globals); + } + + if (module->import_funcs) + destroy_import_funcs(module->import_funcs); + + if (module->exports) + destroy_exports(module->exports); + + if (module->func_type_indexes) + wasm_runtime_free(module->func_type_indexes); + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + if (module->max_local_cell_nums) + wasm_runtime_free(module->max_local_cell_nums); + if (module->max_stack_cell_nums) + wasm_runtime_free(module->max_stack_cell_nums); +#endif + +#if WASM_ENABLE_GC != 0 + if (module->func_local_ref_flags) { + uint32 i; + for (i = 0; i < module->import_func_count + module->func_count; i++) { + if (module->func_local_ref_flags[i].local_ref_flags) { + wasm_runtime_free( + module->func_local_ref_flags[i].local_ref_flags); + } + } + + wasm_runtime_free(module->func_local_ref_flags); + } +#endif + + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + + if (module->const_str_set) + bh_hash_map_destroy(module->const_str_set); +#if WASM_ENABLE_MULTI_MODULE != 0 + /* just release the sub module list */ + if (module->import_module_list) { + WASMRegisteredModule *node = + bh_list_first_elem(module->import_module_list); + while (node) { + WASMRegisteredModule *next = bh_list_elem_next(node); + bh_list_remove(module->import_module_list, node); + wasm_runtime_free(node); + node = next; + } + } +#endif + + if (module->code && !module->is_indirect_mode) { + /* The layout is: literal size + literal + code (with plt table) */ + uint8 *mmap_addr = module->literal - sizeof(uint32); + uint32 total_size = + sizeof(uint32) + module->literal_size + module->code_size; + os_munmap(mmap_addr, total_size); + } + +#if defined(BH_PLATFORM_WINDOWS) + if (module->extra_plt_data) { + os_munmap(module->extra_plt_data, module->extra_plt_data_size); + } +#endif + +#if (defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)) \ + && !defined(BH_PLATFORM_WINDOWS) + { + GOTItem *got_item = module->got_item_list, *got_item_next; + + if (module->got_func_ptrs) { + os_munmap(module->got_func_ptrs, + sizeof(void *) * module->got_item_count); + } + while (got_item) { + got_item_next = got_item->next; + wasm_runtime_free(got_item); + got_item = got_item_next; + } + } +#endif + + if (module->data_sections) + destroy_object_data_sections(module->data_sections, + module->data_section_count); + +#if WASM_ENABLE_DEBUG_AOT != 0 + jit_code_entry_destroy(module->elf_hdr); +#endif + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + if (module->aux_func_indexes) { + wasm_runtime_free(module->aux_func_indexes); + } + if (module->aux_func_names) { + wasm_runtime_free((void *)module->aux_func_names); + } +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + wasm_runtime_destroy_custom_sections(module->custom_section_list); +#endif + +#if WASM_ENABLE_GC != 0 + if (module->ref_type_set) { + bh_hash_map_destroy(module->ref_type_set); + } + os_mutex_destroy(&module->rtt_type_lock); + if (module->rtt_types) { + uint32 i; + for (i = 0; i < module->type_count; i++) { + if (module->rtt_types[i]) + wasm_runtime_free(module->rtt_types[i]); + } + wasm_runtime_free(module->rtt_types); + } +#if WASM_ENABLE_STRINGREF != 0 + { + uint32 i; + for (i = 0; i < WASM_TYPE_STRINGVIEWITER - WASM_TYPE_STRINGREF + 1; + i++) { + if (module->stringref_rtts[i]) + wasm_runtime_free(module->stringref_rtts[i]); + } + + if (module->string_literal_lengths) { + wasm_runtime_free(module->string_literal_lengths); + } + + if (module->string_literal_ptrs) { + wasm_runtime_free((void *)module->string_literal_ptrs); + } + } +#endif +#endif + + wasm_runtime_free(module); +} + +uint32 +aot_get_plt_table_size() +{ + return get_plt_table_size(); +} + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +const uint8 * +aot_get_custom_section(const AOTModule *module, const char *name, uint32 *len) +{ + WASMCustomSection *section = module->custom_section_list; + + while (section) { + if (strcmp(section->name_addr, name) == 0) { + if (len) { + *len = section->content_len; + } + return section->content_addr; + } + + section = section->next; + } + + return NULL; +} +#endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION */ + +#if WASM_ENABLE_STATIC_PGO != 0 +void +aot_exchange_uint16(uint8 *p_data) +{ + return exchange_uint16(p_data); +} + +void +aot_exchange_uint32(uint8 *p_data) +{ + return exchange_uint32(p_data); +} + +void +aot_exchange_uint64(uint8 *p_data) +{ + return exchange_uint64(p_data); +} +#endif diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_perf_map.c b/wasm-micro-runtime/core/iwasm/aot/aot_perf_map.c new file mode 100644 index 0000000..22700dc --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_perf_map.c @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_perf_map.h" +#include "bh_log.h" +#include "bh_platform.h" + +#if WASM_ENABLE_LINUX_PERF != 0 +struct func_info { + uint32 idx; + void *ptr; +}; + +static uint32 +get_func_size(const AOTModule *module, struct func_info *sorted_func_ptrs, + uint32 idx) +{ + uint32 func_sz; + + if (idx == module->func_count - 1) + func_sz = (uintptr_t)module->code + module->code_size + - (uintptr_t)(sorted_func_ptrs[idx].ptr); + else + func_sz = (uintptr_t)(sorted_func_ptrs[idx + 1].ptr) + - (uintptr_t)(sorted_func_ptrs[idx].ptr); + + return func_sz; +} + +static int +compare_func_ptrs(const void *f1, const void *f2) +{ + return (intptr_t)((struct func_info *)f1)->ptr + - (intptr_t)((struct func_info *)f2)->ptr; +} + +static struct func_info * +sort_func_ptrs(const AOTModule *module, char *error_buf, uint32 error_buf_size) +{ + uint64 content_len; + struct func_info *sorted_func_ptrs; + unsigned i; + + content_len = (uint64)sizeof(struct func_info) * module->func_count; + sorted_func_ptrs = wasm_runtime_malloc(content_len); + if (!sorted_func_ptrs) { + snprintf(error_buf, error_buf_size, + "allocate memory failed when creating perf map"); + return NULL; + } + + for (i = 0; i < module->func_count; i++) { + sorted_func_ptrs[i].idx = i; + sorted_func_ptrs[i].ptr = module->func_ptrs[i]; + } + + qsort(sorted_func_ptrs, module->func_count, sizeof(struct func_info), + compare_func_ptrs); + + return sorted_func_ptrs; +} + +bool +aot_create_perf_map(const AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + struct func_info *sorted_func_ptrs = NULL; + char perf_map_path[64] = { 0 }; + char perf_map_info[128] = { 0 }; + FILE *perf_map = NULL; + uint32 i; + pid_t pid = getpid(); + bool ret = false; + + sorted_func_ptrs = sort_func_ptrs(module, error_buf, error_buf_size); + if (!sorted_func_ptrs) + goto quit; + + snprintf(perf_map_path, sizeof(perf_map_path) - 1, "/tmp/perf-%d.map", pid); + perf_map = fopen(perf_map_path, "a"); + if (!perf_map) { + LOG_WARNING("warning: can't create /tmp/perf-%d.map, because %s", pid, + strerror(errno)); + goto quit; + } + + const char *module_name = aot_get_module_name((AOTModule *)module); + for (i = 0; i < module->func_count; i++) { + memset(perf_map_info, 0, 128); + if (strlen(module_name) > 0) + snprintf(perf_map_info, 128, "%lx %x [%s]#aot_func#%u\n", + (uintptr_t)sorted_func_ptrs[i].ptr, + get_func_size(module, sorted_func_ptrs, i), module_name, + sorted_func_ptrs[i].idx); + else + snprintf(perf_map_info, 128, "%lx %x aot_func#%u\n", + (uintptr_t)sorted_func_ptrs[i].ptr, + get_func_size(module, sorted_func_ptrs, i), + sorted_func_ptrs[i].idx); + + /* fwrite() is thread safe */ + fwrite(perf_map_info, 1, strlen(perf_map_info), perf_map); + } + + LOG_VERBOSE("write map information from %s into /tmp/perf-%d.map", + module_name, pid); + ret = true; + +quit: + if (sorted_func_ptrs) + wasm_runtime_free(sorted_func_ptrs); + + if (perf_map) + fclose(perf_map); + + return ret; +} +#endif /* WASM_ENABLE_LINUX_PERF != 0 */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_perf_map.h b/wasm-micro-runtime/core/iwasm/aot/aot_perf_map.h new file mode 100644 index 0000000..3e6583c --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_perf_map.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_PERF_MAP_H_ +#define _AOT_PERF_MAP_H_ + +#include "aot_runtime.h" + +bool +aot_create_perf_map(const AOTModule *module, char *error_buf, + uint32 error_buf_size); + +#endif /* _AOT_PERF_MAP_H_ */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_reloc.h b/wasm-micro-runtime/core/iwasm/aot/aot_reloc.h new file mode 100644 index 0000000..8ead3cd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_reloc.h @@ -0,0 +1,249 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_RELOC_H_ +#define _AOT_RELOC_H_ + +#include "aot_runtime.h" +#include "aot_intrinsic.h" + +#if WASM_ENABLE_STRINGREF != 0 +#include "string_object.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + const char *symbol_name; + void *symbol_addr; +} SymbolMap; + +/* clang-format off */ +#define REG_SYM(symbol) { #symbol, (void *)symbol } + +#if WASM_ENABLE_BULK_MEMORY != 0 +#define REG_BULK_MEMORY_SYM() \ + REG_SYM(aot_memory_init), \ + REG_SYM(aot_data_drop), +#else +#define REG_BULK_MEMORY_SYM() +#endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "wasm_shared_memory.h" +#define REG_ATOMIC_WAIT_SYM() \ + REG_SYM(wasm_runtime_atomic_wait), \ + REG_SYM(wasm_runtime_atomic_notify), +#else +#define REG_ATOMIC_WAIT_SYM() +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +#define REG_REF_TYPES_SYM() \ + REG_SYM(aot_drop_table_seg), \ + REG_SYM(aot_table_init), \ + REG_SYM(aot_table_copy), \ + REG_SYM(aot_table_fill), \ + REG_SYM(aot_table_grow), +#else +#define REG_REF_TYPES_SYM() +#endif + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 +#define REG_AOT_TRACE_SYM() \ + REG_SYM(aot_alloc_frame), \ + REG_SYM(aot_free_frame), \ + REG_SYM(aot_frame_update_profile_info), +#else +#define REG_AOT_TRACE_SYM() +#endif + +#if WASM_ENABLE_AOT_INTRINSICS != 0 +#define REG_INTRINSIC_SYM() \ + REG_SYM(aot_intrinsic_fabs_f32), \ + REG_SYM(aot_intrinsic_fabs_f64), \ + REG_SYM(aot_intrinsic_floor_f32), \ + REG_SYM(aot_intrinsic_floor_f64), \ + REG_SYM(aot_intrinsic_ceil_f32), \ + REG_SYM(aot_intrinsic_ceil_f64), \ + REG_SYM(aot_intrinsic_trunc_f32), \ + REG_SYM(aot_intrinsic_trunc_f64), \ + REG_SYM(aot_intrinsic_rint_f32), \ + REG_SYM(aot_intrinsic_rint_f64), \ + REG_SYM(aot_intrinsic_sqrt_f32), \ + REG_SYM(aot_intrinsic_sqrt_f64), \ + REG_SYM(aot_intrinsic_copysign_f32), \ + REG_SYM(aot_intrinsic_copysign_f64), \ + REG_SYM(aot_intrinsic_fadd_f32), \ + REG_SYM(aot_intrinsic_fadd_f64), \ + REG_SYM(aot_intrinsic_fsub_f32), \ + REG_SYM(aot_intrinsic_fsub_f64), \ + REG_SYM(aot_intrinsic_fmul_f32), \ + REG_SYM(aot_intrinsic_fmul_f64), \ + REG_SYM(aot_intrinsic_fdiv_f32), \ + REG_SYM(aot_intrinsic_fdiv_f64), \ + REG_SYM(aot_intrinsic_fmin_f32), \ + REG_SYM(aot_intrinsic_fmin_f64), \ + REG_SYM(aot_intrinsic_fmax_f32), \ + REG_SYM(aot_intrinsic_fmax_f64), \ + REG_SYM(aot_intrinsic_clz_i32), \ + REG_SYM(aot_intrinsic_clz_i64), \ + REG_SYM(aot_intrinsic_ctz_i32), \ + REG_SYM(aot_intrinsic_ctz_i64), \ + REG_SYM(aot_intrinsic_popcnt_i32), \ + REG_SYM(aot_intrinsic_popcnt_i64), \ + REG_SYM(aot_intrinsic_i32_to_f32), \ + REG_SYM(aot_intrinsic_u32_to_f32), \ + REG_SYM(aot_intrinsic_i32_to_f64), \ + REG_SYM(aot_intrinsic_u32_to_f64), \ + REG_SYM(aot_intrinsic_i64_to_f32), \ + REG_SYM(aot_intrinsic_u64_to_f32), \ + REG_SYM(aot_intrinsic_i64_to_f64), \ + REG_SYM(aot_intrinsic_u64_to_f64), \ + REG_SYM(aot_intrinsic_f64_to_f32), \ + REG_SYM(aot_intrinsic_f32_to_i32), \ + REG_SYM(aot_intrinsic_f32_to_u32), \ + REG_SYM(aot_intrinsic_f32_to_i64), \ + REG_SYM(aot_intrinsic_f32_to_u64), \ + REG_SYM(aot_intrinsic_f64_to_i32), \ + REG_SYM(aot_intrinsic_f64_to_u32), \ + REG_SYM(aot_intrinsic_f64_to_i64), \ + REG_SYM(aot_intrinsic_f64_to_u64), \ + REG_SYM(aot_intrinsic_f32_to_f64), \ + REG_SYM(aot_intrinsic_f32_cmp), \ + REG_SYM(aot_intrinsic_f64_cmp), \ + REG_SYM(aot_intrinsic_i64_div_s), \ + REG_SYM(aot_intrinsic_i64_div_u), \ + REG_SYM(aot_intrinsic_i64_rem_s), \ + REG_SYM(aot_intrinsic_i64_rem_u), \ + REG_SYM(aot_intrinsic_i64_bit_or), \ + REG_SYM(aot_intrinsic_i64_bit_and), \ + REG_SYM(aot_intrinsic_i32_div_s), \ + REG_SYM(aot_intrinsic_i32_div_u), \ + REG_SYM(aot_intrinsic_i32_rem_s), \ + REG_SYM(aot_intrinsic_i32_rem_u), +#else +#define REG_INTRINSIC_SYM() +#endif + +#if WASM_ENABLE_STATIC_PGO != 0 +#define REG_LLVM_PGO_SYM() \ + { "__llvm_profile_instrument_target", llvm_profile_instrument_target }, \ + { "__llvm_profile_instrument_memop", llvm_profile_instrument_memop }, +#else +#define REG_LLVM_PGO_SYM() +#endif + +#if WASM_ENABLE_GC != 0 +#define REG_GC_SYM() \ + REG_SYM(aot_array_init_with_data), \ + REG_SYM(aot_create_func_obj), \ + REG_SYM(aot_obj_is_instance_of), \ + REG_SYM(aot_func_type_is_super_of), \ + REG_SYM(aot_rtt_type_new), \ + REG_SYM(wasm_array_obj_copy), \ + REG_SYM(wasm_array_obj_new), \ + REG_SYM(wasm_externref_obj_to_internal_obj), \ + REG_SYM(wasm_internal_obj_to_externref_obj), \ + REG_SYM(wasm_obj_is_type_of), \ + REG_SYM(wasm_struct_obj_new), +#else +#define REG_GC_SYM() +#endif + +#if WASM_ENABLE_STRINGREF != 0 +#define REG_STRINGREF_SYM() \ + REG_SYM(wasm_stringref_obj_new), \ + REG_SYM(wasm_stringview_wtf8_obj_new), \ + REG_SYM(wasm_stringview_wtf16_obj_new), \ + REG_SYM(wasm_stringview_iter_obj_new), \ + REG_SYM(wasm_string_destroy), \ + REG_SYM(wasm_string_new_const), \ + REG_SYM(wasm_string_new_with_encoding), \ + REG_SYM(wasm_string_measure), \ + REG_SYM(wasm_string_wtf16_get_length), \ + REG_SYM(wasm_string_encode), \ + REG_SYM(wasm_string_concat), \ + REG_SYM(wasm_string_eq), \ + REG_SYM(wasm_string_is_usv_sequence), \ + REG_SYM(wasm_string_create_view), \ + REG_SYM(wasm_string_advance), \ + REG_SYM(wasm_string_slice), \ + REG_SYM(wasm_string_get_wtf16_codeunit),\ + REG_SYM(wasm_string_next_codepoint), \ + REG_SYM(wasm_string_rewind), \ + REG_SYM(wasm_string_dump), +#else +#define REG_STRINGREF_SYM() +#endif + +#define REG_COMMON_SYMBOLS \ + REG_SYM(aot_set_exception_with_id), \ + REG_SYM(aot_invoke_native), \ + REG_SYM(aot_call_indirect), \ + REG_SYM(aot_enlarge_memory), \ + REG_SYM(aot_set_exception), \ + REG_SYM(aot_check_app_addr_and_convert),\ + REG_SYM(wasm_runtime_quick_invoke_c_api_native),\ + { "memset", (void*)aot_memset }, \ + { "memmove", (void*)aot_memmove }, \ + { "memcpy", (void*)aot_memmove }, \ + { "sqrt", (void*)aot_sqrt }, \ + { "sqrtf", (void*)aot_sqrtf }, \ + REG_SYM(fmin), \ + REG_SYM(fminf), \ + REG_SYM(fmax), \ + REG_SYM(fmaxf), \ + REG_SYM(ceil), \ + REG_SYM(ceilf), \ + REG_SYM(floor), \ + REG_SYM(floorf), \ + REG_SYM(trunc), \ + REG_SYM(truncf), \ + REG_SYM(rint), \ + REG_SYM(rintf), \ + REG_BULK_MEMORY_SYM() \ + REG_ATOMIC_WAIT_SYM() \ + REG_REF_TYPES_SYM() \ + REG_AOT_TRACE_SYM() \ + REG_INTRINSIC_SYM() \ + REG_LLVM_PGO_SYM() \ + REG_GC_SYM() \ + REG_STRINGREF_SYM() \ + +#define CHECK_RELOC_OFFSET(data_size) do { \ + if (!check_reloc_offset(target_section_size, \ + reloc_offset, data_size, \ + error_buf, error_buf_size)) \ + return false; \ + } while (0) + +SymbolMap * +get_target_symbol_map(uint32 *sym_num); + +uint32 +get_plt_table_size(); + +void +init_plt_table(uint8 *plt); + +void +get_current_target(char *target_buf, uint32 target_buf_size); + +bool +apply_relocation(AOTModule *module, + uint8 *target_section_addr, uint32 target_section_size, + uint64 reloc_offset, int64 reloc_addend, + uint32 reloc_type, void *symbol_addr, int32 symbol_index, + char *error_buf, uint32 error_buf_size); +/* clang-format off */ + +#ifdef __cplusplus +} +#endif + +#endif /* end of _AOT_RELOC_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_runtime.c b/wasm-micro-runtime/core/iwasm/aot/aot_runtime.c new file mode 100644 index 0000000..2ec0017 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_runtime.c @@ -0,0 +1,4768 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_runtime.h" +#include "bh_log.h" +#include "mem_alloc.h" +#include "../common/wasm_runtime_common.h" +#include "../common/wasm_memory.h" +#include "../interpreter/wasm_runtime.h" +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif + +/* + * Note: These offsets need to match the values hardcoded in + * AoT compilation code: aot_create_func_context, check_suspend_flags. + */ + +bh_static_assert(offsetof(WASMExecEnv, cur_frame) == 1 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, module_inst) == 2 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, argv_buf) == 3 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, native_stack_boundary) + == 4 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, suspend_flags) == 5 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, aux_stack_boundary) + == 6 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, aux_stack_bottom) + == 7 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, native_symbol) == 8 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, native_stack_top_min) + == 9 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, wasm_stack.top_boundary) + == 10 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, wasm_stack.top) + == 11 * sizeof(uintptr_t)); +bh_static_assert(offsetof(WASMExecEnv, wasm_stack.bottom) + == 12 * sizeof(uintptr_t)); + +bh_static_assert(offsetof(AOTModuleInstance, memories) == 1 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, func_ptrs) == 5 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, func_type_indexes) + == 6 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, cur_exception) + == 13 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, c_api_func_imports) + == 13 * sizeof(uint64) + 128 + 7 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, global_table_data) + == 13 * sizeof(uint64) + 128 + 14 * sizeof(uint64)); + +bh_static_assert(sizeof(AOTMemoryInstance) == 120); +bh_static_assert(offsetof(AOTTableInstance, elems) == 24); + +bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0); + +bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); + +bh_static_assert(sizeof(wasm_val_t) == 16); +bh_static_assert(offsetof(wasm_val_t, of) == 8); + +bh_static_assert(offsetof(AOTFrame, prev_frame) == sizeof(uintptr_t) * 0); +bh_static_assert(offsetof(AOTFrame, func_index) == sizeof(uintptr_t) * 1); +bh_static_assert(offsetof(AOTFrame, time_started) == sizeof(uintptr_t) * 2); +bh_static_assert(offsetof(AOTFrame, func_perf_prof_info) + == sizeof(uintptr_t) * 3); +bh_static_assert(offsetof(AOTFrame, ip_offset) == sizeof(uintptr_t) * 4); +bh_static_assert(offsetof(AOTFrame, sp) == sizeof(uintptr_t) * 5); +bh_static_assert(offsetof(AOTFrame, frame_ref) == sizeof(uintptr_t) * 6); +bh_static_assert(offsetof(AOTFrame, lp) == sizeof(uintptr_t) * 7); + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, "AOT module instantiate failed: %s", + string); + } +} + +static void +set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + snprintf(error_buf, error_buf_size, "AOT module instantiate failed: %s", + buf); + } +} + +static void * +runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +static bool +check_global_init_expr(const AOTModule *module, uint32 global_index, + char *error_buf, uint32 error_buf_size) +{ + if (global_index >= module->import_global_count + module->global_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown global %d", + global_index); + return false; + } + +#if WASM_ENABLE_GC == 0 + /** + * Currently, constant expressions occurring as initializers of + * globals are further constrained in that contained global.get + * instructions are only allowed to refer to imported globals. + * + * And initializer expression cannot reference a mutable global. + */ + if (global_index >= module->import_global_count + || module->import_globals->is_mutable) { + set_error_buf(error_buf, error_buf_size, + "constant expression required"); + return false; + } +#else + if (global_index >= module->import_global_count + module->global_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown global %u", + global_index); + return false; + } + if (global_index < module->import_global_count + && module->import_globals[global_index].is_mutable) { + set_error_buf(error_buf, error_buf_size, + "constant expression required"); + return false; + } +#endif + + return true; +} + +static void +init_global_data(uint8 *global_data, uint8 type, WASMValue *initial_value) +{ + switch (type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + *(int32 *)global_data = initial_value->i32; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + bh_memcpy_s(global_data, sizeof(int64), &initial_value->i64, + sizeof(int64)); + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + bh_memcpy_s(global_data, sizeof(V128), &initial_value->v128, + sizeof(V128)); + break; +#endif + default: +#if WASM_ENABLE_GC != 0 + if ((type >= (uint8)REF_TYPE_ARRAYREF + && type <= (uint8)REF_TYPE_NULLFUNCREF) + || (type >= (uint8)REF_TYPE_HT_NULLABLE + && type <= (uint8)REF_TYPE_HT_NON_NULLABLE) +#if WASM_ENABLE_STRINGREF != 0 + || (type >= (uint8)REF_TYPE_STRINGVIEWWTF8 + && type <= (uint8)REF_TYPE_STRINGREF) + || (type >= (uint8)REF_TYPE_STRINGVIEWITER + && type <= (uint8)REF_TYPE_STRINGVIEWWTF16) +#endif + ) { + bh_memcpy_s(global_data, sizeof(wasm_obj_t), + &initial_value->gc_obj, sizeof(wasm_obj_t)); + break; + } +#endif /* end of WASM_ENABLE_GC */ + bh_assert(0); + } +} + +#if WASM_ENABLE_GC != 0 +static bool +assign_table_init_value(AOTModuleInstance *module_inst, AOTModule *module, + InitializerExpression *init_expr, void *addr, + char *error_buf, uint32 error_buf_size) +{ + uint8 flag = init_expr->init_expr_type; + + bh_assert(flag >= INIT_EXPR_TYPE_GET_GLOBAL + && flag <= INIT_EXPR_TYPE_EXTERN_CONVERT_ANY); + + switch (flag) { + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr(module, init_expr->u.global_index, + error_buf, error_buf_size)) { + return false; + } + if (init_expr->u.global_index < module->import_global_count) { + PUT_REF_TO_ADDR( + addr, module->import_globals[init_expr->u.global_index] + .global_data_linked.gc_obj); + } + else { + uint32 global_idx = + init_expr->u.global_index - module->import_global_count; + return assign_table_init_value( + module_inst, module, &module->globals[global_idx].init_expr, + addr, error_buf, error_buf_size); + } + break; + } + case INIT_EXPR_TYPE_REFNULL_CONST: + { + WASMObjectRef gc_obj = NULL_REF; + PUT_REF_TO_ADDR(addr, gc_obj); + break; + } + case INIT_EXPR_TYPE_FUNCREF_CONST: + { + WASMFuncObjectRef func_obj = NULL; + uint32 func_idx = init_expr->u.u32; + + if (func_idx != UINT32_MAX) { + if (!(func_obj = + aot_create_func_obj(module_inst, func_idx, false, + error_buf, error_buf_size))) { + return false; + } + } + + PUT_REF_TO_ADDR(addr, func_obj); + break; + } + case INIT_EXPR_TYPE_I31_NEW: + { + WASMI31ObjectRef i31_obj = wasm_i31_obj_new(init_expr->u.i32); + PUT_REF_TO_ADDR(addr, i31_obj); + break; + } + case INIT_EXPR_TYPE_STRUCT_NEW: + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + { + WASMRttType *rtt_type; + WASMStructObjectRef struct_obj; + WASMStructType *struct_type; + WASMStructNewInitValues *init_values = NULL; + uint32 type_idx; + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + init_values = (WASMStructNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + } + else { + type_idx = init_expr->u.type_index; + } + + struct_type = (WASMStructType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + return false; + } + + if (!(struct_obj = wasm_struct_obj_new_internal( + ((AOTModuleInstanceExtra *)module_inst->e) + ->common.gc_heap_handle, + rtt_type))) { + set_error_buf(error_buf, error_buf_size, + "create struct object failed"); + return false; + } + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + uint32 field_idx; + + bh_assert(init_values->count == struct_type->field_count); + + for (field_idx = 0; field_idx < init_values->count; + field_idx++) { + wasm_struct_obj_set_field(struct_obj, field_idx, + &init_values->fields[field_idx]); + } + } + + PUT_REF_TO_ADDR(addr, struct_obj); + break; + } + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + WASMRttType *rtt_type; + WASMArrayObjectRef array_obj; + WASMArrayType *array_type; + WASMArrayNewInitValues *init_values = NULL; + WASMValue *arr_init_val = NULL, empty_val = { 0 }; + uint32 type_idx, len; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { + type_idx = init_expr->u.array_new_default.type_index; + len = init_expr->u.array_new_default.length; + arr_init_val = &empty_val; + } + else { + init_values = (WASMArrayNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + len = init_values->length; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW) { + arr_init_val = init_values->elem_data; + } + } + + array_type = (WASMArrayType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + return false; + } + + if (!(array_obj = wasm_array_obj_new_internal( + ((AOTModuleInstanceExtra *)module_inst->e) + ->common.gc_heap_handle, + rtt_type, len, arr_init_val))) { + set_error_buf(error_buf, error_buf_size, + "create array object failed"); + return false; + } + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + uint32 elem_idx; + + bh_assert(init_values); + + for (elem_idx = 0; elem_idx < len; elem_idx++) { + wasm_array_obj_set_elem(array_obj, elem_idx, + &init_values->elem_data[elem_idx]); + } + } + + PUT_REF_TO_ADDR(addr, array_obj); + break; + } + default: + set_error_buf(error_buf, error_buf_size, "invalid init expr type."); + return false; + } + + return true; +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static bool +global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + InitializerExpression *init_expr; + uint8 *p = module_inst->global_data; + AOTImportGlobal *import_global = module->import_globals; + AOTGlobal *global = module->globals; + + /* Initialize import global data */ + for (i = 0; i < module->import_global_count; i++, import_global++) { + bh_assert(import_global->data_offset + == (uint32)(p - module_inst->global_data)); + init_global_data(p, import_global->type, + &import_global->global_data_linked); + p += import_global->size; + } + + /* Initialize defined global data */ + for (i = 0; i < module->global_count; i++, global++) { + uint8 flag; + bh_assert(global->data_offset + == (uint32)(p - module_inst->global_data)); + init_expr = &global->init_expr; + flag = init_expr->init_expr_type; + switch (flag) { + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr(module, init_expr->u.global_index, + error_buf, error_buf_size)) { + return false; + } +#if WASM_ENABLE_GC == 0 + init_global_data( + p, global->type, + &module->import_globals[init_expr->u.global_index] + .global_data_linked); +#else + if (init_expr->u.global_index < module->import_global_count) { + init_global_data( + p, global->type, + &module->import_globals[init_expr->u.global_index] + .global_data_linked); + } + else { + uint32 global_idx = + init_expr->u.global_index - module->import_global_count; + init_global_data(p, global->type, + &module->globals[global_idx].init_expr.u); + } +#endif + break; + } +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case INIT_EXPR_TYPE_REFNULL_CONST: + { + *(uint32 *)p = NULL_REF; + break; + } +#elif WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_REFNULL_CONST: + { + WASMObjectRef gc_obj = NULL_REF; + PUT_REF_TO_ADDR(p, gc_obj); + break; + } +#endif +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_FUNCREF_CONST: + { + WASMFuncObjectRef func_obj = NULL; + uint32 func_idx = init_expr->u.u32; + + if (func_idx != UINT32_MAX) { + if (!(func_obj = + aot_create_func_obj(module_inst, func_idx, false, + error_buf, error_buf_size))) { + return false; + } + } + + PUT_REF_TO_ADDR(p, func_obj); + break; + } + case INIT_EXPR_TYPE_I31_NEW: + { + WASMI31ObjectRef i31_obj = wasm_i31_obj_new(init_expr->u.i32); + PUT_REF_TO_ADDR(p, i31_obj); + break; + } + case INIT_EXPR_TYPE_STRUCT_NEW: + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + { + WASMRttType *rtt_type; + WASMStructObjectRef struct_obj; + WASMStructType *struct_type; + WASMStructNewInitValues *init_values = NULL; + uint32 type_idx; + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + init_values = (WASMStructNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + } + else { + type_idx = init_expr->u.type_index; + } + + struct_type = (WASMStructType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + return false; + } + + if (!(struct_obj = wasm_struct_obj_new_internal( + ((AOTModuleInstanceExtra *)module_inst->e) + ->common.gc_heap_handle, + rtt_type))) { + set_error_buf(error_buf, error_buf_size, + "create struct object failed"); + return false; + } + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + uint32 field_idx; + + bh_assert(init_values->count == struct_type->field_count); + + for (field_idx = 0; field_idx < init_values->count; + field_idx++) { + wasm_struct_obj_set_field( + struct_obj, field_idx, + &init_values->fields[field_idx]); + } + } + + PUT_REF_TO_ADDR(p, struct_obj); + break; + } + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + WASMRttType *rtt_type; + WASMArrayObjectRef array_obj; + WASMArrayType *array_type; + WASMArrayNewInitValues *init_values = NULL; + WASMValue *arr_init_val = NULL, empty_val = { 0 }; + uint32 type_idx, len; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { + type_idx = init_expr->u.array_new_default.type_index; + len = init_expr->u.array_new_default.length; + arr_init_val = &empty_val; + } + else { + init_values = (WASMArrayNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + len = init_values->length; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW) { + arr_init_val = init_values->elem_data; + } + } + + array_type = (WASMArrayType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + return false; + } + + if (!(array_obj = wasm_array_obj_new_internal( + ((AOTModuleInstanceExtra *)module_inst->e) + ->common.gc_heap_handle, + rtt_type, len, arr_init_val))) { + set_error_buf(error_buf, error_buf_size, + "create array object failed"); + return false; + } + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + uint32 elem_idx; + + bh_assert(init_values); + + for (elem_idx = 0; elem_idx < len; elem_idx++) { + wasm_array_obj_set_elem( + array_obj, elem_idx, + &init_values->elem_data[elem_idx]); + } + } + + PUT_REF_TO_ADDR(p, array_obj); + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + { + init_global_data(p, global->type, &init_expr->u); + break; + } + } + p += global->size; + } + + bh_assert(module_inst->global_data_size + == (uint32)(p - module_inst->global_data)); + return true; +} + +static bool +tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, + AOTTableInstance *first_tbl_inst, char *error_buf, + uint32 error_buf_size) +{ + uint32 i, global_index, global_data_offset, base_offset, length; + uint64 total_size; + AOTTableInitData *table_seg; + AOTTableInstance *tbl_inst = first_tbl_inst; + + total_size = (uint64)sizeof(AOTTableInstance *) * module_inst->table_count; + if (total_size > 0 + && !(module_inst->tables = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* + * treat import table like a local one until we enable module linking + * in AOT mode + */ + for (i = 0; i != module_inst->table_count; ++i) { + if (i < module->import_table_count) { + AOTImportTable *import_table = module->import_tables + i; + tbl_inst->cur_size = import_table->table_init_size; + tbl_inst->max_size = + aot_get_imp_tbl_data_slots(import_table, false); +#if WASM_ENABLE_GC != 0 + tbl_inst->elem_type = module->tables[i].elem_type; + tbl_inst->elem_ref_type.elem_ref_type = + module->tables[i].elem_ref_type; +#endif + } + else { + AOTTable *table = module->tables + (i - module->import_table_count); + tbl_inst->cur_size = table->table_init_size; + tbl_inst->max_size = aot_get_tbl_data_slots(table, false); +#if WASM_ENABLE_GC != 0 + tbl_inst->elem_type = module->tables[i].elem_type; + tbl_inst->elem_ref_type.elem_ref_type = + module->tables[i].elem_ref_type; +#endif + } + + /* Set all elements to -1 or NULL_REF to mark them as uninitialized + * elements */ +#if WASM_ENABLE_GC == 0 + memset(tbl_inst->elems, 0xff, + sizeof(table_elem_type_t) * tbl_inst->max_size); +#else + memset(tbl_inst->elems, 0x00, + sizeof(table_elem_type_t) * tbl_inst->max_size); +#endif + + module_inst->tables[i] = tbl_inst; + tbl_inst = (AOTTableInstance *)((uint8 *)tbl_inst + + offsetof(AOTTableInstance, elems) + + sizeof(table_elem_type_t) + * tbl_inst->max_size); + } + + /* fill table with element segment content */ + for (i = 0; i < module->table_init_data_count; i++) { +#if WASM_ENABLE_GC == 0 + uint32 j; +#endif + table_seg = module->table_init_data_list[i]; + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + if (!wasm_elem_is_active(table_seg->mode)) + continue; +#endif + + bh_assert(table_seg->table_index < module_inst->table_count); + + tbl_inst = module_inst->tables[table_seg->table_index]; + bh_assert(tbl_inst); + +#if WASM_ENABLE_REF_TYPES != 0 + bh_assert( + table_seg->offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST + || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL + || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST + || table_seg->offset.init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST); +#else + bh_assert(table_seg->offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST + || table_seg->offset.init_expr_type + == INIT_EXPR_TYPE_GET_GLOBAL); +#endif + + /* Resolve table data base offset */ + if (table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { + global_index = table_seg->offset.u.global_index; + + if (!check_global_init_expr(module, global_index, error_buf, + error_buf_size)) { + return false; + } + + if (global_index < module->import_global_count) + global_data_offset = + module->import_globals[global_index].data_offset; + else + global_data_offset = + module->globals[global_index - module->import_global_count] + .data_offset; + + base_offset = + *(uint32 *)(module_inst->global_data + global_data_offset); + } + else + base_offset = (uint32)table_seg->offset.u.i32; + + /* Copy table data */ + /* base_offset only since length might negative */ + if (base_offset > tbl_inst->cur_size) { +#if WASM_ENABLE_REF_TYPES != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); +#endif + return false; + } + + /* base_offset + length(could be zero) */ + length = table_seg->value_count; + if (base_offset + length > tbl_inst->cur_size) { +#if WASM_ENABLE_REF_TYPES != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); +#endif + return false; + } + + /** + * Check function index in the current module inst for now. + * will check the linked table inst owner in future + */ +#if WASM_ENABLE_GC == 0 + for (j = 0; j < length; j++) { + tbl_inst->elems[base_offset + j] = + table_seg->init_values[j].u.ref_index; + } +#endif + } + + return true; +} + +static void +memories_deinstantiate(AOTModuleInstance *module_inst) +{ + uint32 i; + AOTMemoryInstance *memory_inst; + + for (i = 0; i < module_inst->memory_count; i++) { + memory_inst = module_inst->memories[i]; + if (memory_inst) { +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (shared_memory_is_shared(memory_inst)) { + uint32 ref_count = shared_memory_dec_reference(memory_inst); + /* if the reference count is not zero, + don't free the memory */ + if (ref_count > 0) + continue; + } +#endif + if (memory_inst->heap_handle) { + mem_allocator_destroy(memory_inst->heap_handle); + wasm_runtime_free(memory_inst->heap_handle); + } + + if (memory_inst->memory_data) { + wasm_deallocate_linear_memory(memory_inst); + } + } + } + wasm_runtime_free(module_inst->memories); +} + +static AOTMemoryInstance * +memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, + AOTModule *module, AOTMemoryInstance *memory_inst, + AOTMemory *memory, uint32 memory_idx, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + void *heap_handle; + uint32 num_bytes_per_page = memory->num_bytes_per_page; + uint32 init_page_count = memory->mem_init_page_count; + uint32 max_page_count = + wasm_runtime_get_max_mem(max_memory_pages, memory->mem_init_page_count, + memory->mem_max_page_count); + uint32 inc_page_count, global_idx; + uint32 bytes_of_last_page, bytes_to_page_end; + uint64 aux_heap_base, + heap_offset = (uint64)num_bytes_per_page * init_page_count; + uint64 memory_data_size, max_memory_data_size; + uint8 *p = NULL, *global_addr; + + bool is_shared_memory = false; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared_memory = memory->memory_flags & 0x02 ? true : false; + /* Shared memory */ + if (is_shared_memory && parent != NULL) { + AOTMemoryInstance *shared_memory_instance; + bh_assert(memory_idx == 0); + bh_assert(parent->memory_count > memory_idx); + shared_memory_instance = parent->memories[memory_idx]; + shared_memory_inc_reference(shared_memory_instance); + return shared_memory_instance; + } +#endif + + if (heap_size > 0 && module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1) { + /* Disable app heap, use malloc/free function exported + by wasm app to allocate/free memory instead */ + heap_size = 0; + } + + /* If initial memory is the largest size allowed, disallowing insert host + * managed heap */ + if (heap_size > 0 && heap_offset == MAX_LINEAR_MEMORY_SIZE) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + + if (init_page_count == max_page_count && init_page_count == 1) { + /* If only one page and at most one page, we just append + the app heap to the end of linear memory, enlarge the + num_bytes_per_page, and don't change the page count */ + heap_offset = num_bytes_per_page; + num_bytes_per_page += heap_size; + if (num_bytes_per_page < heap_size) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + } + else if (heap_size > 0) { + if (init_page_count == max_page_count && init_page_count == 0) { + /* If the memory data size is always 0, we resize it to + one page for app heap */ + num_bytes_per_page = heap_size; + heap_offset = 0; + inc_page_count = 1; + } + else if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base + < (uint64)num_bytes_per_page * init_page_count) { + /* Insert app heap before __heap_base */ + aux_heap_base = module->aux_heap_base; + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + inc_page_count = + (heap_size - bytes_to_page_end + num_bytes_per_page - 1) + / num_bytes_per_page; + heap_offset = aux_heap_base; + aux_heap_base += heap_size; + + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + if (bytes_to_page_end < 1 * BH_KB) { + aux_heap_base += 1 * BH_KB; + inc_page_count++; + } + + /* Adjust __heap_base global value */ + global_idx = module->aux_heap_base_global_index + - module->import_global_count; + global_addr = module_inst->global_data + + module->globals[global_idx].data_offset; + *(uint32 *)global_addr = (uint32)aux_heap_base; + LOG_VERBOSE("Reset __heap_base global to %" PRIu64, aux_heap_base); + } + else { + /* Insert app heap before new page */ + inc_page_count = + (heap_size + num_bytes_per_page - 1) / num_bytes_per_page; + heap_offset = (uint64)num_bytes_per_page * init_page_count; + heap_size = (uint64)num_bytes_per_page * inc_page_count; + if (heap_size > 0) + heap_size -= 1 * BH_KB; + } + init_page_count += inc_page_count; + max_page_count += inc_page_count; + if (init_page_count > DEFAULT_MAX_PAGES) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + if (max_page_count > DEFAULT_MAX_PAGES) + max_page_count = DEFAULT_MAX_PAGES; + } + + LOG_VERBOSE("Memory instantiate:"); + LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", + num_bytes_per_page, init_page_count, max_page_count); + LOG_VERBOSE(" data offset: %" PRIu64 ", stack size: %d", + module->aux_data_end, module->aux_stack_size); + LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %d\n", heap_offset, + heap_size); + + max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; + bh_assert(max_memory_data_size <= MAX_LINEAR_MEMORY_SIZE); + (void)max_memory_data_size; + + /* TODO: memory64 uses is_memory64 flag */ + if (wasm_allocate_linear_memory(&p, is_shared_memory, false, + num_bytes_per_page, init_page_count, + max_page_count, &memory_data_size) + != BHT_OK) { + set_error_buf(error_buf, error_buf_size, + "allocate linear memory failed"); + return NULL; + } + + memory_inst->module_type = Wasm_Module_AoT; + memory_inst->num_bytes_per_page = num_bytes_per_page; + memory_inst->cur_page_count = init_page_count; + memory_inst->max_page_count = max_page_count; + memory_inst->memory_data_size = memory_data_size; + + /* Init memory info */ + memory_inst->memory_data = p; + memory_inst->memory_data_end = p + memory_data_size; + + /* Initialize heap info */ + memory_inst->heap_data = p + heap_offset; + memory_inst->heap_data_end = p + heap_offset + heap_size; + if (heap_size > 0) { + uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); + + if (!(heap_handle = runtime_malloc((uint64)heap_struct_size, error_buf, + error_buf_size))) { + goto fail1; + } + + memory_inst->heap_handle = heap_handle; + + if (!mem_allocator_create_with_struct_and_pool( + heap_handle, heap_struct_size, memory_inst->heap_data, + heap_size)) { + set_error_buf(error_buf, error_buf_size, "init app heap failed"); + goto fail2; + } + } + + if (memory_data_size > 0) { + wasm_runtime_set_mem_bound_check_bytes(memory_inst, memory_data_size); + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + memory_inst->is_shared_memory = 1; + memory_inst->ref_count = 1; + } +#endif + + return memory_inst; + +fail2: + if (heap_size > 0) + wasm_runtime_free(memory_inst->heap_handle); +fail1: + wasm_deallocate_linear_memory(memory_inst); + + return NULL; +} + +static AOTMemoryInstance * +aot_get_default_memory(AOTModuleInstance *module_inst) +{ + if (module_inst->memories) + return module_inst->memories[0]; + else + return NULL; +} + +static bool +memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, + AOTModule *module, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + uint32 global_index, global_data_offset, base_offset, length; + uint32 i, memory_count = module->memory_count; + AOTMemoryInstance *memories, *memory_inst; + AOTMemInitData *data_seg; + uint64 total_size; + + module_inst->memory_count = memory_count; + total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count; + if (!(module_inst->memories = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + memories = module_inst->global_table_data.memory_instances; + for (i = 0; i < memory_count; i++, memories++) { + memory_inst = memory_instantiate( + module_inst, parent, module, memories, &module->memories[i], i, + heap_size, max_memory_pages, error_buf, error_buf_size); + if (!memory_inst) { + return false; + } + + module_inst->memories[i] = memory_inst; + } + + /* Get default memory instance */ + memory_inst = aot_get_default_memory(module_inst); + if (!memory_inst) { + /* Ignore setting memory init data if no memory inst is created */ + return true; + } + + for (i = 0; i < module->mem_init_data_count; i++) { + data_seg = module->mem_init_data_list[i]; +#if WASM_ENABLE_BULK_MEMORY != 0 + if (data_seg->is_passive) + continue; +#endif + if (parent != NULL) + /* Ignore setting memory init data if the memory has been + initialized */ + continue; + + bh_assert(data_seg->offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST + || data_seg->offset.init_expr_type + == INIT_EXPR_TYPE_GET_GLOBAL); + + /* Resolve memory data base offset */ + if (data_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { + global_index = data_seg->offset.u.global_index; + + if (!check_global_init_expr(module, global_index, error_buf, + error_buf_size)) { + return false; + } + + if (global_index < module->import_global_count) + global_data_offset = + module->import_globals[global_index].data_offset; + else + global_data_offset = + module->globals[global_index - module->import_global_count] + .data_offset; + + base_offset = + *(uint32 *)(module_inst->global_data + global_data_offset); + } + else { + base_offset = (uint32)data_seg->offset.u.i32; + } + + /* Copy memory data */ + bh_assert(memory_inst->memory_data + || memory_inst->memory_data_size == 0); + + /* Check memory data */ + /* check offset since length might negative */ + if (base_offset > memory_inst->memory_data_size) { + LOG_DEBUG("base_offset(%d) > memory_data_size(%" PRIu64 ")", + base_offset, memory_inst->memory_data_size); +#if WASM_ENABLE_REF_TYPES != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds memory access"); +#else + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); +#endif + return false; + } + + /* check offset + length(could be zero) */ + length = data_seg->byte_count; + if (base_offset + length > memory_inst->memory_data_size) { + LOG_DEBUG("base_offset(%d) + length(%d) > memory_data_size(%" PRIu64 + ")", + base_offset, length, memory_inst->memory_data_size); +#if WASM_ENABLE_REF_TYPES != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds memory access"); +#else + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); +#endif + return false; + } + + if (memory_inst->memory_data) { + bh_memcpy_s((uint8 *)memory_inst->memory_data + base_offset, + (uint32)memory_inst->memory_data_size - base_offset, + data_seg->bytes, length); + } + } + + return true; +} + +static bool +init_func_ptrs(AOTModuleInstance *module_inst, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + void **func_ptrs; + uint32 func_count = module->func_count; +#if defined(BUILD_TARGET_XTENSA) + /* + * For Xtensa XIP, real func_count is doubled, including aot_func and + * aot_func_internal, so need to multipy func_count by 2 here. + */ + if (module->is_indirect_mode) { + func_count *= 2; + } +#endif + + uint64 total_size = + ((uint64)module->import_func_count + func_count) * sizeof(void *); + + if (module->import_func_count + func_count == 0) + return true; + + /* Allocate memory */ + if (!(module_inst->func_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Set import function pointers */ + func_ptrs = (void **)module_inst->func_ptrs; + for (i = 0; i < module->import_func_count; i++, func_ptrs++) { + *func_ptrs = (void *)module->import_funcs[i].func_ptr_linked; + if (!*func_ptrs) { + const char *module_name = module->import_funcs[i].module_name; + const char *field_name = module->import_funcs[i].func_name; + LOG_WARNING("warning: failed to link import function (%s, %s)", + module_name, field_name); + } + } + + /* Set defined function pointers */ + bh_memcpy_s(func_ptrs, sizeof(void *) * func_count, module->func_ptrs, + sizeof(void *) * func_count); + return true; +} + +static bool +init_func_type_indexes(AOTModuleInstance *module_inst, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + uint32 *func_type_index; + uint32 func_count = module->func_count; +#if defined(BUILD_TARGET_XTENSA) + /* + * For Xtensa XIP, real func_count is doubled, including aot_func and + * aot_func_internal, so need to multipy func_count by 2 here. + */ + if (module->is_indirect_mode) { + func_count *= 2; + } +#endif + + uint64 total_size = + ((uint64)module->import_func_count + func_count) * sizeof(uint32); + + if (module->import_func_count + func_count == 0) + return true; + + /* Allocate memory */ + if (!(module_inst->func_type_indexes = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Set import function type indexes */ + func_type_index = module_inst->func_type_indexes; + for (i = 0; i < module->import_func_count; i++, func_type_index++) + *func_type_index = module->import_funcs[i].func_type_index; + + bh_memcpy_s(func_type_index, sizeof(uint32) * func_count, + module->func_type_indexes, sizeof(uint32) * func_count); + return true; +} + +static bool +create_export_funcs(AOTModuleInstance *module_inst, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + AOTExport *exports = module->exports; + AOTFunctionInstance *export_func; + uint64 size; + uint32 i, func_index, ftype_index; + + if (module_inst->export_func_count > 0) { + /* Allocate memory */ + size = sizeof(AOTFunctionInstance) + * (uint64)module_inst->export_func_count; + if (!(export_func = runtime_malloc(size, error_buf, error_buf_size))) { + return false; + } + module_inst->export_functions = (void *)export_func; + + for (i = 0; i < module->export_count; i++) { + if (exports[i].kind == EXPORT_KIND_FUNC) { + export_func->func_name = exports[i].name; + export_func->func_index = exports[i].index; + if (export_func->func_index < module->import_func_count) { + export_func->is_import_func = true; + export_func->u.func_import = + &module->import_funcs[export_func->func_index]; + } + else { + export_func->is_import_func = false; + func_index = + export_func->func_index - module->import_func_count; + ftype_index = module->func_type_indexes[func_index]; + export_func->u.func.func_type = + (AOTFuncType *)module->types[ftype_index]; + export_func->u.func.func_ptr = + module->func_ptrs[func_index]; + } + export_func++; + } + } + } + + return true; +} + +static bool +create_exports(AOTModuleInstance *module_inst, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + AOTExport *exports = module->exports; + uint32 i; + + for (i = 0; i < module->export_count; i++) { + switch (exports[i].kind) { + case EXPORT_KIND_FUNC: + module_inst->export_func_count++; + break; + case EXPORT_KIND_GLOBAL: + module_inst->export_global_count++; + break; + case EXPORT_KIND_TABLE: + module_inst->export_table_count++; + break; + case EXPORT_KIND_MEMORY: + module_inst->export_memory_count++; + break; + default: + return false; + } + } + + return create_export_funcs(module_inst, module, error_buf, error_buf_size); +} + +static AOTFunctionInstance * +lookup_post_instantiate_func(AOTModuleInstance *module_inst, + const char *func_name) +{ + AOTFunctionInstance *func; + AOTFuncType *func_type; + + if (!(func = aot_lookup_function(module_inst, func_name))) + /* Not found */ + return NULL; + + func_type = func->u.func.func_type; + if (!(func_type->param_count == 0 && func_type->result_count == 0)) + /* Not a valid function type, ignore it */ + return NULL; + + return func; +} + +static bool +execute_post_instantiate_functions(AOTModuleInstance *module_inst, + bool is_sub_inst, WASMExecEnv *exec_env_main) +{ + AOTModule *module = (AOTModule *)module_inst->module; + AOTFunctionInstance *initialize_func = NULL; + AOTFunctionInstance *post_inst_func = NULL; + AOTFunctionInstance *call_ctors_func = NULL; + WASMModuleInstanceCommon *module_inst_main = NULL; +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env = NULL, *exec_env_created = NULL; + bool ret = false; + +#if WASM_ENABLE_LIBC_WASI != 0 + /* + * WASI reactor instances may assume that _initialize will be called by + * the environment at most once, and that none of their other exports + * are accessed before that call. + */ + if (!is_sub_inst && module->import_wasi_api) { + initialize_func = + lookup_post_instantiate_func(module_inst, "_initialize"); + } +#endif + + /* Execute possible "__post_instantiate" function if wasm app is + compiled by emsdk's early version */ + if (!is_sub_inst) { + post_inst_func = + lookup_post_instantiate_func(module_inst, "__post_instantiate"); + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + /* Only execute the memory init function for main instance since + the data segments will be dropped once initialized */ + if (!is_sub_inst +#if WASM_ENABLE_LIBC_WASI != 0 + && !module->import_wasi_api +#endif + ) { + call_ctors_func = + lookup_post_instantiate_func(module_inst, "__wasm_call_ctors"); + } +#endif + + if (!module->start_function && !initialize_func && !post_inst_func + && !call_ctors_func) { + /* No post instantiation functions to call */ + return true; + } + + if (is_sub_inst) { + bh_assert(exec_env_main); +#ifdef OS_ENABLE_HW_BOUND_CHECK + bh_assert(exec_env_tls == exec_env_main); + (void)exec_env_tls; +#endif + exec_env = exec_env_main; + + /* Temporarily replace parent exec_env's module inst to current + module inst to avoid checking failure when calling the + wasm functions, and ensure that the exec_env's module inst + is the correct one. */ + module_inst_main = exec_env_main->module_inst; + wasm_exec_env_set_module_inst(exec_env, + (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + aot_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_main = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + +#if defined(os_writegsbase) + { + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + if (memory_inst) + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_inst->memory_data); + } +#endif + + /* Execute start function for both main insance and sub instance */ + if (module->start_function) { + AOTFunctionInstance start_func = { 0 }; + uint32 func_type_idx; + + start_func.func_name = ""; + start_func.func_index = module->start_func_index; + start_func.is_import_func = false; + func_type_idx = module->func_type_indexes[module->start_func_index + - module->import_func_count]; + start_func.u.func.func_type = + (AOTFuncType *)module->types[func_type_idx]; + start_func.u.func.func_ptr = module->start_function; + if (!aot_call_function(exec_env, &start_func, 0, NULL)) { + goto fail; + } + } + + if (initialize_func + && !aot_call_function(exec_env, initialize_func, 0, NULL)) { + goto fail; + } + + if (post_inst_func + && !aot_call_function(exec_env, post_inst_func, 0, NULL)) { + goto fail; + } + + if (call_ctors_func + && !aot_call_function(exec_env, call_ctors_func, 0, NULL)) { + goto fail; + } + + ret = true; + +fail: + if (is_sub_inst) { + /* Restore the parent exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env_main, module_inst_main); + } + else { + if (module_inst_main) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_main); + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + } + + return ret; +} + +static bool +check_linked_symbol(AOTModule *module, char *error_buf, uint32 error_buf_size) +{ + uint32 i; + + /* init_func_ptrs() will go through import functions */ + + for (i = 0; i < module->import_global_count; i++) { + AOTImportGlobal *global = module->import_globals + i; + if (!global->is_linked) { + set_error_buf_v(error_buf, error_buf_size, + "failed to link import global (%s, %s)", + global->module_name, global->global_name); + return false; + } + } + + return true; +} + +AOTModuleInstance * +aot_instantiate(AOTModule *module, AOTModuleInstance *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, uint32 error_buf_size) +{ + AOTModuleInstance *module_inst; +#if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_REF_TYPES != 0 + WASMModuleInstanceExtraCommon *common; +#endif + const uint32 module_inst_struct_size = + offsetof(AOTModuleInstance, global_table_data.bytes); + const uint64 module_inst_mem_inst_size = + (uint64)module->memory_count * sizeof(AOTMemoryInstance); + uint64 total_size, table_size = 0; + uint8 *p; + uint32 i, extra_info_offset; + const bool is_sub_inst = parent != NULL; +#if WASM_ENABLE_MULTI_MODULE != 0 + bool ret = false; +#endif + + /* Check heap size */ + heap_size = align_uint(heap_size, 8); + if (heap_size > APP_HEAP_SIZE_MAX) + heap_size = APP_HEAP_SIZE_MAX; + + total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size + + module->global_data_size; + + /* + * calculate size of table data + */ + for (i = 0; i != module->import_table_count; ++i) { + table_size += offsetof(AOTTableInstance, elems); + table_size += (uint64)sizeof(table_elem_type_t) + * (uint64)aot_get_imp_tbl_data_slots( + module->import_tables + i, false); + } + + for (i = 0; i != module->table_count; ++i) { + table_size += offsetof(AOTTableInstance, elems); + table_size += + (uint64)sizeof(table_elem_type_t) + * (uint64)aot_get_tbl_data_slots(module->tables + i, false); + } + total_size += table_size; + + /* The offset of AOTModuleInstanceExtra, make it 8-byte aligned */ + total_size = (total_size + 7LL) & ~7LL; + extra_info_offset = (uint32)total_size; + total_size += sizeof(AOTModuleInstanceExtra); + + /* Allocate module instance, global data, table data and heap data */ + if (!(module_inst = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + module_inst->module_type = Wasm_Module_AoT; + module_inst->module = (void *)module; + module_inst->e = + (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset); + +#if WASM_ENABLE_GC != 0 + /* Initialize gc heap first since it may be used when initializing + globals and others */ + if (!is_sub_inst) { + uint32 gc_heap_size = wasm_runtime_get_gc_heap_size_default(); + AOTModuleInstanceExtra *extra = + (AOTModuleInstanceExtra *)module_inst->e; + + if (gc_heap_size < GC_HEAP_SIZE_MIN) + gc_heap_size = GC_HEAP_SIZE_MIN; + if (gc_heap_size > GC_HEAP_SIZE_MAX) + gc_heap_size = GC_HEAP_SIZE_MAX; + + extra->common.gc_heap_pool = + runtime_malloc(gc_heap_size, error_buf, error_buf_size); + if (!extra->common.gc_heap_pool) + goto fail; + + extra->common.gc_heap_handle = + mem_allocator_create(extra->common.gc_heap_pool, gc_heap_size); + if (!extra->common.gc_heap_handle) + goto fail; + } +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + ((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list = + &((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list_head; + ret = wasm_runtime_sub_module_instantiate( + (WASMModuleCommon *)module, (WASMModuleInstanceCommon *)module_inst, + stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); + if (!ret) { + LOG_DEBUG("build a sub module list failed"); + goto fail; + } +#endif + + /* Initialize function type indexes before initializing global info, + module_inst->func_type_indexes may be used in the latter */ + if (!init_func_type_indexes(module_inst, module, error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_REF_TYPES != 0 + common = &((AOTModuleInstanceExtra *)module_inst->e)->common; +#endif +#if WASM_ENABLE_BULK_MEMORY != 0 + if (module->mem_init_data_count > 0) { + common->data_dropped = bh_bitmap_new(0, module->mem_init_data_count); + if (common->data_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + for (i = 0; i < module->mem_init_data_count; i++) { + if (!module->mem_init_data_list[i]->is_passive) + bh_bitmap_set_bit(common->data_dropped, i); + } + } +#endif +#if WASM_ENABLE_REF_TYPES != 0 + if (module->table_init_data_count > 0) { + common->elem_dropped = bh_bitmap_new(0, module->table_init_data_count); + if (common->elem_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + for (i = 0; i < module->table_init_data_count; i++) { + if (wasm_elem_is_active(module->table_init_data_list[i]->mode)) + bh_bitmap_set_bit(common->elem_dropped, i); + } + } +#endif + + /* Initialize global info */ + p = (uint8 *)module_inst + module_inst_struct_size + + module_inst_mem_inst_size; + module_inst->global_data = p; + module_inst->global_data_size = module->global_data_size; + if (!global_instantiate(module_inst, module, error_buf, error_buf_size)) + goto fail; + + /* Initialize table info */ + p += module->global_data_size; + module_inst->table_count = module->table_count + module->import_table_count; + if (!tables_instantiate(module_inst, module, (AOTTableInstance *)p, + error_buf, error_buf_size)) + goto fail; + + /* Initialize memory space */ + if (!memories_instantiate(module_inst, parent, module, heap_size, + max_memory_pages, error_buf, error_buf_size)) + goto fail; + + /* Initialize function pointers */ + if (!init_func_ptrs(module_inst, module, error_buf, error_buf_size)) + goto fail; + + if (!check_linked_symbol(module, error_buf, error_buf_size)) + goto fail; + + if (!create_exports(module_inst, module, error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_LIBC_WASI != 0 + if (!is_sub_inst) { + if (!wasm_runtime_init_wasi( + (WASMModuleInstanceCommon *)module_inst, + module->wasi_args.dir_list, module->wasi_args.dir_count, + module->wasi_args.map_dir_list, module->wasi_args.map_dir_count, + module->wasi_args.env, module->wasi_args.env_count, + module->wasi_args.addr_pool, module->wasi_args.addr_count, + module->wasi_args.ns_lookup_pool, + module->wasi_args.ns_lookup_count, module->wasi_args.argv, + module->wasi_args.argc, module->wasi_args.stdio[0], + module->wasi_args.stdio[1], module->wasi_args.stdio[2], + error_buf, error_buf_size)) + goto fail; + } +#endif + + /* Initialize the thread related data */ + if (stack_size == 0) + stack_size = DEFAULT_WASM_STACK_SIZE; +#if WASM_ENABLE_SPEC_TEST != 0 +#if WASM_ENABLE_TAIL_CALL == 0 + if (stack_size < 128 * 1024) + stack_size = 128 * 1024; +#else + /* Some tail-call cases require large operand stack */ + if (stack_size < 10 * 1024 * 1024) + stack_size = 10 * 1024 * 1024; +#endif +#endif + module_inst->default_wasm_stack_size = stack_size; + + ((AOTModuleInstanceExtra *)module_inst->e)->stack_sizes = + aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL); + +#if WASM_ENABLE_PERF_PROFILING != 0 + total_size = (uint64)sizeof(AOTFuncPerfProfInfo) + * (module->import_func_count + module->func_count); + if (!(module_inst->func_perf_profilings = + runtime_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } +#endif + +#if WASM_ENABLE_GC != 0 + for (i = 0; i < module_inst->table_count; i++) { + uint32 j; + AOTTable *table; + AOTTableInstance *table_inst; + table_elem_type_t *table_data; + + table = &module->tables[i]; + bh_assert(table); + + if (table->init_expr.init_expr_type == INIT_EXPR_NONE) { + continue; + } + + table_inst = module_inst->tables[i]; + bh_assert(table_inst); + + table_data = table_inst->elems; + bh_assert(table_data); + + for (j = 0; j < table_inst->cur_size; j++) { + if (!assign_table_init_value(module_inst, module, &table->init_expr, + table_data + j, error_buf, + error_buf_size)) { + goto fail; + } + } + } + + /* Initialize the table data with table init data */ + for (i = 0; + module_inst->table_count > 0 && i < module->table_init_data_count; + i++) { + + AOTTableInitData *table_init_data = module->table_init_data_list[i]; + AOTTableInstance *table; + table_elem_type_t *table_data; + uint8 tbl_elem_type; + uint32 tbl_init_size, tbl_max_size, j; + WASMRefType *tbl_elem_ref_type; + + bh_assert(table_init_data); + + bh_assert(table_init_data->table_index < module_inst->table_count); + table = module_inst->tables[table_init_data->table_index]; + bh_assert(table); + + table_data = table->elems; + bh_assert(table_data); + + wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, + table_init_data->table_index, &tbl_elem_type, &tbl_elem_ref_type, + &tbl_init_size, &tbl_max_size); + + if (!wasm_elem_is_declarative(table_init_data->mode) + && !wasm_reftype_is_subtype_of( + table_init_data->elem_type, table_init_data->elem_ref_type, + table->elem_type, table->elem_ref_type.elem_ref_type, + module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); + goto fail; + } + + (void)tbl_init_size; + (void)tbl_max_size; + + if (!wasm_elem_is_active(table_init_data->mode)) { + continue; + } + + bh_assert(table_init_data->offset.init_expr_type + == INIT_EXPR_TYPE_I32_CONST + || table_init_data->offset.init_expr_type + == INIT_EXPR_TYPE_GET_GLOBAL + || table_init_data->offset.init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST + || table_init_data->offset.init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST); + + /* init vec(funcidx) or vec(expr) */ + if (table_init_data->offset.init_expr_type + == INIT_EXPR_TYPE_GET_GLOBAL) { + uint32 data_offset; + if (!check_global_init_expr(module, + table_init_data->offset.u.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + if (table_init_data->offset.u.global_index + < module->import_global_count) { + data_offset = + module + ->import_globals[table_init_data->offset.u.global_index] + .data_offset; + } + else { + data_offset = + module + ->globals[table_init_data->offset.u.global_index + - module->import_global_count] + .data_offset; + } + + table_init_data->offset.u.i32 = + *(uint32 *)(module_inst->global_data + data_offset); + } + + /* check offset since length might negative */ + if ((uint32)table_init_data->offset.u.i32 > table->cur_size) { + LOG_DEBUG("base_offset(%d) > table->cur_size(%d)", + table_init_data->offset.u.i32, table->cur_size); + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); + goto fail; + } + + if ((uint32)table_init_data->offset.u.i32 + table_init_data->value_count + > table->cur_size) { + LOG_DEBUG("base_offset(%d) + length(%d) > table->cur_size(%d)", + table_init_data->offset.u.i32, + table_init_data->value_count, table->cur_size); + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); + goto fail; + } + + for (j = 0; j < module->table_init_data_list[i]->value_count; j++) { + if (!assign_table_init_value( + module_inst, module, &table_init_data->init_values[j], + table_data + table_init_data->offset.u.i32 + j, error_buf, + error_buf_size)) { + goto fail; + } + } + } +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (!(module_inst->frames = + runtime_malloc(sizeof(Vector), error_buf, error_buf_size))) { + goto fail; + } +#endif + + if (!execute_post_instantiate_functions(module_inst, is_sub_inst, + exec_env_main)) { + set_error_buf(error_buf, error_buf_size, module_inst->cur_exception); + goto fail; + } + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_module_inst_mem_consumption( + (WASMModuleInstanceCommon *)module_inst); +#endif + + return module_inst; + +fail: + aot_deinstantiate(module_inst, is_sub_inst); + return NULL; +} + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +static void +destroy_c_api_frames(Vector *frames) +{ + WASMCApiFrame frame = { 0 }; + uint32 i, total_frames, ret; + + total_frames = (uint32)bh_vector_size(frames); + + for (i = 0; i < total_frames; i++) { + ret = bh_vector_get(frames, i, &frame); + bh_assert(ret); + + if (frame.lp) + wasm_runtime_free(frame.lp); + } + + ret = bh_vector_destroy(frames); + bh_assert(ret); + (void)ret; +} +#endif + +void +aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) +{ + WASMModuleInstanceExtraCommon *common = + &((AOTModuleInstanceExtra *)module_inst->e)->common; + if (module_inst->exec_env_singleton) { + /* wasm_exec_env_destroy will call + wasm_cluster_wait_for_all_except_self to wait for other + threads, so as to destroy their exec_envs and module + instances first, and avoid accessing the shared resources + of current module instance after it is deinstantiated. */ + wasm_exec_env_destroy((WASMExecEnv *)module_inst->exec_env_singleton); + } + +#if WASM_ENABLE_PERF_PROFILING != 0 + if (module_inst->func_perf_profilings) + wasm_runtime_free(module_inst->func_perf_profilings); +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (module_inst->frames) { + destroy_c_api_frames(module_inst->frames); + wasm_runtime_free(module_inst->frames); + module_inst->frames = NULL; + } +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + wasm_runtime_sub_module_deinstantiate( + (WASMModuleInstanceCommon *)module_inst); +#endif + + if (module_inst->tables) + wasm_runtime_free(module_inst->tables); + + if (module_inst->memories) + memories_deinstantiate(module_inst); + + if (module_inst->export_functions) + wasm_runtime_free(module_inst->export_functions); + + if (module_inst->func_ptrs) + wasm_runtime_free(module_inst->func_ptrs); + + if (module_inst->func_type_indexes) + wasm_runtime_free(module_inst->func_type_indexes); + + if (module_inst->c_api_func_imports) + wasm_runtime_free(module_inst->c_api_func_imports); + +#if WASM_ENABLE_GC != 0 + if (!is_sub_inst) { + AOTModuleInstanceExtra *extra = + (AOTModuleInstanceExtra *)module_inst->e; + if (extra->common.gc_heap_handle) + mem_allocator_destroy(extra->common.gc_heap_handle); + if (extra->common.gc_heap_pool) + wasm_runtime_free(extra->common.gc_heap_pool); + } +#endif + + if (!is_sub_inst) { +#if WASM_ENABLE_WASI_NN != 0 + wasi_nn_destroy(module_inst); +#endif + wasm_native_call_context_dtors((WASMModuleInstanceCommon *)module_inst); + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap_delete(common->data_dropped); +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap_delete(common->elem_dropped); +#endif + + wasm_runtime_free(module_inst); +} + +AOTFunctionInstance * +aot_lookup_function(const AOTModuleInstance *module_inst, const char *name) +{ + uint32 i; + AOTFunctionInstance *export_funcs = + (AOTFunctionInstance *)module_inst->export_functions; + + for (i = 0; i < module_inst->export_func_count; i++) + if (!strcmp(export_funcs[i].func_name, name)) + return &export_funcs[i]; + return NULL; +} + +#ifdef OS_ENABLE_HW_BOUND_CHECK +static bool +invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, + const char *signature, void *attachment, + uint32 *argv, uint32 argc, uint32 *argv_ret) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); + WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; +#ifdef BH_PLATFORM_WINDOWS + int result; + bool has_exception; + char exception[EXCEPTION_BUF_LEN]; +#endif + bool ret; + + /* Check native stack overflow firstly to ensure we have enough + native stack to run the following codes before actually calling + the aot function in invokeNative function. */ + RECORD_STACK_USAGE(exec_env, (uint8 *)&module_inst); + if ((uint8 *)&module_inst < exec_env->native_stack_boundary + + page_size * (guard_page_count + 1)) { + aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW); + return false; + } + + if (!exec_env_tls) { + if (!os_thread_signal_inited()) { + aot_set_exception(module_inst, "thread signal env not inited"); + return false; + } + + /* Set thread handle and stack boundary if they haven't been set */ + wasm_exec_env_set_thread_info(exec_env); + + wasm_runtime_set_exec_env_tls(exec_env); + } + else { + if (exec_env_tls != exec_env) { + aot_set_exception(module_inst, "invalid exec env"); + return false; + } + } + + wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); + + if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick call if the quick aot entry is registered */ + if (!signature && func_type->quick_aot_entry) { + void (*invoke_native)(void *func_ptr, void *exec_env, uint32 *argv, + uint32 *argv_ret) = + func_type->quick_aot_entry; + invoke_native(func_ptr, exec_env, argv, argv_ret); + ret = !aot_copy_exception(module_inst, NULL); + } + else +#endif + { + ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv_ret); + } +#ifdef BH_PLATFORM_WINDOWS + has_exception = aot_copy_exception(module_inst, exception); + if (has_exception && strstr(exception, "native stack overflow")) { + /* After a stack overflow, the stack was left + in a damaged state, let the CRT repair it */ + result = _resetstkoflw(); + bh_assert(result != 0); + } +#endif + } + else { + /* Exception has been set in signal handler before calling longjmp */ + ret = false; + } + + jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env); + bh_assert(&jmpbuf_node == jmpbuf_node_pop); + if (!exec_env->jmpbuf_stack_top) { + wasm_runtime_set_exec_env_tls(NULL); + } + if (!ret) { + os_sigreturn(); + os_signal_unmask(); + } + (void)jmpbuf_node_pop; + return ret; +} +#define invoke_native_internal invoke_native_with_hw_bound_check /* NOLINT */ +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +static inline bool +invoke_native_internal(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret) +{ +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick call if the quick aot entry is registered */ + if (!signature && func_type->quick_aot_entry) { + AOTModuleInstance *module_inst = + (AOTModuleInstance *)exec_env->module_inst; + void (*invoke_native)(void *func_ptr, void *exec_env, uint32 *argv, + uint32 *argv_ret) = func_type->quick_aot_entry; + invoke_native(func_ptr, exec_env, argv, argv_ret); + return !aot_copy_exception(module_inst, NULL); + } +#endif + return wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv_ret); +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + +#ifdef AOT_STACK_FRAME_DEBUG +typedef void (*stack_frame_callback_t)(struct WASMExecEnv *exec_env); +static stack_frame_callback_t aot_stack_frame_callback; + +/* set the callback, only for debug purpose */ +void +aot_set_stack_frame_callback(stack_frame_callback_t callback) +{ + aot_stack_frame_callback = callback; +} +#endif + +bool +aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, + unsigned argc, uint32 argv[]) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + AOTFuncType *func_type = function->is_import_func + ? function->u.func_import->func_type + : function->u.func.func_type; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret; + void *func_ptr = function->is_import_func + ? function->u.func_import->func_ptr_linked + : function->u.func.func_ptr; +#if WASM_ENABLE_MULTI_MODULE != 0 + bh_list *sub_module_list_node = NULL; + const char *sub_inst_name = NULL; + const char *func_name = function->u.func_import->module_name; + if (function->is_import_func) { + sub_module_list_node = + ((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list; + sub_module_list_node = bh_list_first_elem(sub_module_list_node); + while (sub_module_list_node) { + sub_inst_name = + ((AOTSubModInstNode *)sub_module_list_node)->module_name; + if (strcmp(sub_inst_name, func_name) == 0) { + exec_env = wasm_runtime_get_exec_env_singleton( + (WASMModuleInstanceCommon *)((AOTSubModInstNode *) + sub_module_list_node) + ->module_inst); + module_inst = (AOTModuleInstance *)exec_env->module_inst; + break; + } + sub_module_list_node = bh_list_elem_next(sub_module_list_node); + } + if (exec_env == NULL) { + wasm_runtime_set_exception((WASMModuleInstanceCommon *)module_inst, + "create singleton exec_env failed"); + return false; + } + } +#endif + + if (argc < func_type->param_cell_num) { + char buf[108]; + snprintf(buf, sizeof(buf), + "invalid argument count %u, must be no smaller than %u", argc, + func_type->param_cell_num); + aot_set_exception(module_inst, buf); + return false; + } + argc = func_type->param_cell_num; + +#if defined(os_writegsbase) + { + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + if (memory_inst) + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_inst->memory_data); + } +#endif + + /* func pointer was looked up previously */ + bh_assert(func_ptr != NULL); + +#ifndef OS_ENABLE_HW_BOUND_CHECK + /* Set thread handle and stack boundary */ + wasm_exec_env_set_thread_info(exec_env); +#else + /* Set thread info in invoke_native_with_hw_bound_check when + hw bound check is enabled */ +#endif + + /* Set exec env, so it can be later retrieved from instance */ + module_inst->cur_exec_env = exec_env; + + if (ext_ret_count > 0) { + uint32 cell_num = 0, i; + uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; + uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL; + uint32 *argv_ret = argv; + uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); + uint64 size; +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + struct WASMInterpFrame *prev_frame = exec_env->cur_frame; +#endif + + /* Allocate memory all arguments */ + size = + sizeof(uint32) * (uint64)argc /* original arguments */ + + sizeof(void *) + * (uint64)ext_ret_count /* extra result values' addr */ + + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */ + if (size > sizeof(argv1_buf) + && !(argv1 = runtime_malloc(size, module_inst->cur_exception, + sizeof(module_inst->cur_exception)))) { + aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY); + return false; + } + + /* Copy original arguments */ + bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc); + + /* Get the extra result value's address */ + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + + /* Append each extra result value's address to original arguments */ + for (i = 0; i < ext_ret_count; i++) { + *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) = + (uintptr_t)(ext_rets + cell_num); + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + if (!aot_alloc_frame(exec_env, function->func_index)) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return false; + } +#endif + + ret = invoke_native_internal(exec_env, function->u.func.func_ptr, + func_type, NULL, NULL, argv1, argc, argv); + + if (!ret) { +#ifdef AOT_STACK_FRAME_DEBUG + if (aot_stack_frame_callback) { + aot_stack_frame_callback(exec_env); + } +#endif +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (aot_create_call_stack(exec_env)) { + aot_dump_call_stack(exec_env, true, NULL, 0); + } +#endif + } + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + /* Free all frames allocated, note that some frames + may be allocated in AOT code and havent' been + freed if exception occured */ + while (exec_env->cur_frame != prev_frame) + aot_free_frame(exec_env); +#endif + if (!ret) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return ret; + } + + /* Get extra result values */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + argv_ret++; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + argv_ret += 2; + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + argv_ret += 4; + break; +#endif + default: + bh_assert(0); + break; + } + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets, + sizeof(uint32) * cell_num); + + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return true; + } + else { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + struct WASMInterpFrame *prev_frame = exec_env->cur_frame; + + if (!aot_alloc_frame(exec_env, function->func_index)) { + return false; + } +#endif + + ret = invoke_native_internal(exec_env, func_ptr, func_type, NULL, NULL, + argv, argc, argv); + + if (aot_copy_exception(module_inst, NULL)) { +#ifdef AOT_STACK_FRAME_DEBUG + if (aot_stack_frame_callback) { + aot_stack_frame_callback(exec_env); + } +#endif +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (aot_create_call_stack(exec_env)) { + aot_dump_call_stack(exec_env, true, NULL, 0); + } +#endif + } + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + /* Free all frames allocated, note that some frames + may be allocated in AOT code and havent' been + freed if exception occured */ + while (exec_env->cur_frame != prev_frame) + aot_free_frame(exec_env); +#endif + + return ret && !aot_copy_exception(module_inst, NULL) ? true : false; + } +} + +void +aot_set_exception(AOTModuleInstance *module_inst, const char *exception) +{ + wasm_set_exception(module_inst, exception); +} + +void +aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id) +{ + if (id != EXCE_ALREADY_THROWN) + wasm_set_exception_with_id(module_inst, id); +#ifdef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_access_exce_check_guard_page(); +#endif +} + +const char * +aot_get_exception(AOTModuleInstance *module_inst) +{ + return wasm_get_exception(module_inst); +} + +bool +aot_copy_exception(AOTModuleInstance *module_inst, char *exception_buf) +{ + /* The field offsets of cur_exception in AOTModuleInstance and + WASMModuleInstance are the same */ + return wasm_copy_exception(module_inst, exception_buf); +} + +static bool +execute_malloc_function(AOTModuleInstance *module_inst, WASMExecEnv *exec_env, + AOTFunctionInstance *malloc_func, + AOTFunctionInstance *retain_func, uint32 size, + uint32 *p_result) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env_created = NULL; + WASMModuleInstanceCommon *module_inst_old = NULL; + uint32 argv[2], argc; + bool ret; + + argv[0] = size; + argc = 1; + if (retain_func) { + argv[1] = 0; + argc = 2; + } + + if (exec_env) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls) { + bh_assert(exec_env_tls == exec_env); + } +#endif + bh_assert(exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_old = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + ret = aot_call_function(exec_env, malloc_func, argc, argv); + + if (retain_func && ret) + ret = aot_call_function(exec_env, retain_func, 1, argv); + + if (module_inst_old) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_old); + + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + + if (ret) + *p_result = argv[0]; + return ret; +} + +static bool +execute_free_function(AOTModuleInstance *module_inst, WASMExecEnv *exec_env, + AOTFunctionInstance *free_func, uint32 offset) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env_created = NULL; + WASMModuleInstanceCommon *module_inst_old = NULL; + uint32 argv[2]; + bool ret; + + argv[0] = offset; + + if (exec_env) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls) { + bh_assert(exec_env_tls == exec_env); + } +#endif + bh_assert(exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_old = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + ret = aot_call_function(exec_env, free_func, 1, argv); + + if (module_inst_old) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_old); + + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + + return ret; +} + +uint64 +aot_module_malloc_internal(AOTModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr) +{ + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + AOTModule *module = (AOTModule *)module_inst->module; + uint8 *addr = NULL; + uint32 offset = 0; + + /* TODO: Memory64 size check based on memory idx type */ + bh_assert(size <= UINT32_MAX); + + if (!memory_inst) { + aot_set_exception(module_inst, "uninitialized memory"); + return 0; + } + + if (memory_inst->heap_handle) { + addr = mem_allocator_malloc(memory_inst->heap_handle, (uint32)size); + } + else if (module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1) { + AOTFunctionInstance *malloc_func, *retain_func = NULL; + char *malloc_func_name; + + if (module->retain_func_index != (uint32)-1) { + malloc_func_name = "__new"; + retain_func = aot_lookup_function(module_inst, "__retain"); + if (!retain_func) + retain_func = aot_lookup_function(module_inst, "__pin"); + bh_assert(retain_func); + } + else { + malloc_func_name = "malloc"; + } + malloc_func = aot_lookup_function(module_inst, malloc_func_name); + + if (!malloc_func + || !execute_malloc_function(module_inst, exec_env, malloc_func, + retain_func, (uint32)size, &offset)) { + return 0; + } + addr = offset ? (uint8 *)memory_inst->memory_data + offset : NULL; + } + + if (!addr) { + if (memory_inst->heap_handle + && mem_allocator_is_heap_corrupted(memory_inst->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + aot_set_exception(module_inst, "app heap corrupted"); + } + else { + LOG_WARNING("warning: allocate %" PRIu64 " bytes memory failed", + size); + } + return 0; + } + if (p_native_addr) + *p_native_addr = addr; + return (uint64)(addr - memory_inst->memory_data); +} + +uint64 +aot_module_realloc_internal(AOTModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr, uint64 size, + void **p_native_addr) +{ + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + uint8 *addr = NULL; + + /* TODO: Memory64 ptr and size check based on memory idx type */ + bh_assert(ptr <= UINT32_MAX); + bh_assert(size <= UINT32_MAX); + + if (!memory_inst) { + aot_set_exception(module_inst, "uninitialized memory"); + return 0; + } + + if (memory_inst->heap_handle) { + addr = mem_allocator_realloc( + memory_inst->heap_handle, + (uint32)ptr ? memory_inst->memory_data + (uint32)ptr : NULL, + (uint32)size); + } + + /* Only support realloc in WAMR's app heap */ + (void)exec_env; + + if (!addr) { + if (memory_inst->heap_handle + && mem_allocator_is_heap_corrupted(memory_inst->heap_handle)) { + aot_set_exception(module_inst, "app heap corrupted"); + } + else { + aot_set_exception(module_inst, "out of memory"); + } + return 0; + } + + if (p_native_addr) + *p_native_addr = addr; + return (uint64)(addr - memory_inst->memory_data); +} + +void +aot_module_free_internal(AOTModuleInstance *module_inst, WASMExecEnv *exec_env, + uint64 ptr) +{ + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + AOTModule *module = (AOTModule *)module_inst->module; + + if (!memory_inst) { + return; + } + + /* TODO: Memory64 ptr and size check based on memory idx type */ + bh_assert(ptr <= UINT32_MAX); + + if (ptr) { + uint8 *addr = memory_inst->memory_data + (uint32)ptr; + uint8 *memory_data_end; + + /* memory->memory_data_end may be changed in memory grow */ + SHARED_MEMORY_LOCK(memory_inst); + memory_data_end = memory_inst->memory_data_end; + SHARED_MEMORY_UNLOCK(memory_inst); + + if (memory_inst->heap_handle && memory_inst->heap_data < addr + && addr < memory_inst->heap_data_end) { + mem_allocator_free(memory_inst->heap_handle, addr); + } + else if (module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1 + && memory_inst->memory_data <= addr + && addr < memory_data_end) { + AOTFunctionInstance *free_func; + char *free_func_name; + + if (module->retain_func_index != (uint32)-1) { + free_func_name = "__release"; + } + else { + free_func_name = "free"; + } + free_func = aot_lookup_function(module_inst, free_func_name); + if (!free_func && module->retain_func_index != (uint32)-1) + free_func = aot_lookup_function(module_inst, "__unpin"); + + if (free_func) + execute_free_function(module_inst, exec_env, free_func, + (uint32)ptr); + } + } +} + +uint64 +aot_module_malloc(AOTModuleInstance *module_inst, uint64 size, + void **p_native_addr) +{ + return aot_module_malloc_internal(module_inst, NULL, size, p_native_addr); +} + +uint64 +aot_module_realloc(AOTModuleInstance *module_inst, uint64 ptr, uint64 size, + void **p_native_addr) +{ + return aot_module_realloc_internal(module_inst, NULL, ptr, size, + p_native_addr); +} + +void +aot_module_free(AOTModuleInstance *module_inst, uint64 ptr) +{ + aot_module_free_internal(module_inst, NULL, ptr); +} + +uint64 +aot_module_dup_data(AOTModuleInstance *module_inst, const char *src, + uint64 size) +{ + char *buffer; + uint64 buffer_offset; + + /* TODO: Memory64 size check based on memory idx type */ + bh_assert(size <= UINT32_MAX); + + buffer_offset = aot_module_malloc(module_inst, size, (void **)&buffer); + + if (buffer_offset != 0) { + buffer = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, buffer_offset); + bh_memcpy_s(buffer, (uint32)size, src, (uint32)size); + } + return buffer_offset; +} + +bool +aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) +{ + return wasm_enlarge_memory(module_inst, inc_page_count); +} + +bool +aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv) +{ + AOTModuleInstance *module_inst = + (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env); + AOTModule *aot_module = (AOTModule *)module_inst->module; + CApiFuncImport *c_api_func_import = + module_inst->c_api_func_imports + ? module_inst->c_api_func_imports + func_idx + : NULL; + uint32 *func_type_indexes = module_inst->func_type_indexes; + uint32 func_type_idx = func_type_indexes[func_idx]; + AOTFuncType *func_type = (AOTFuncType *)aot_module->types[func_type_idx]; + void **func_ptrs = module_inst->func_ptrs; + void *func_ptr = func_ptrs[func_idx]; + AOTImportFunc *import_func; + const char *signature; + void *attachment; + char buf[96]; + bool ret = false; +#if WASM_ENABLE_MULTI_MODULE != 0 + bh_list *sub_module_list_node = NULL; + const char *sub_inst_name = NULL; +#endif + bh_assert(func_idx < aot_module->import_func_count); + + import_func = aot_module->import_funcs + func_idx; + if (import_func->call_conv_wasm_c_api) + func_ptr = + c_api_func_import ? c_api_func_import->func_ptr_linked : NULL; + + if (!func_ptr) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + import_func->module_name, import_func->func_name); + aot_set_exception(module_inst, buf); + goto fail; + } + + attachment = import_func->attachment; + if (import_func->call_conv_wasm_c_api) { + ret = wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc, + argv, c_api_func_import->with_env_arg, c_api_func_import->env_arg); + } + else if (!import_func->call_conv_raw) { + signature = import_func->signature; +#if WASM_ENABLE_MULTI_MODULE != 0 + sub_module_list_node = + ((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list; + sub_module_list_node = bh_list_first_elem(sub_module_list_node); + while (sub_module_list_node) { + sub_inst_name = + ((AOTSubModInstNode *)sub_module_list_node)->module_name; + if (strcmp(sub_inst_name, import_func->module_name) == 0) { + exec_env = wasm_runtime_get_exec_env_singleton( + (WASMModuleInstanceCommon *)((AOTSubModInstNode *) + sub_module_list_node) + ->module_inst); + break; + } + sub_module_list_node = bh_list_elem_next(sub_module_list_node); + } + if (exec_env == NULL) { + wasm_runtime_set_exception((WASMModuleInstanceCommon *)module_inst, + "create singleton exec_env failed"); + goto fail; + } +#endif + ret = + wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv); + } + else { + signature = import_func->signature; + ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); + } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + +bool +aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, + uint32 argc, uint32 *argv) +{ + AOTModuleInstance *module_inst = + (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env); + AOTModule *aot_module = (AOTModule *)module_inst->module; + uint32 *func_type_indexes = module_inst->func_type_indexes; + AOTTableInstance *tbl_inst; + AOTFuncType *func_type; + void **func_ptrs = module_inst->func_ptrs, *func_ptr; + uint32 func_type_idx, func_idx, ext_ret_count; + table_elem_type_t tbl_elem_val = NULL_REF; + AOTImportFunc *import_func; + const char *signature = NULL; + void *attachment = NULL; + char buf[96]; + bool ret; + + /* this function is called from native code, so exec_env->handle and + exec_env->native_stack_boundary must have been set, we don't set + it again */ + + RECORD_STACK_USAGE(exec_env, (uint8 *)&module_inst); + if ((uint8 *)&module_inst < exec_env->native_stack_boundary) { + aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW); + goto fail; + } + + tbl_inst = module_inst->tables[tbl_idx]; + bh_assert(tbl_inst); + + if (table_elem_idx >= tbl_inst->cur_size) { + aot_set_exception_with_id(module_inst, EXCE_UNDEFINED_ELEMENT); + goto fail; + } + + tbl_elem_val = ((table_elem_type_t *)tbl_inst->elems)[table_elem_idx]; + if (tbl_elem_val == NULL_REF) { + aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT); + goto fail; + } + +#if WASM_ENABLE_GC == 0 + func_idx = (uint32)tbl_elem_val; +#else + func_idx = + wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val); +#endif + + func_type_idx = func_type_indexes[func_idx]; + func_type = (AOTFuncType *)aot_module->types[func_type_idx]; + + if (func_idx >= aot_module->import_func_count) { + /* func pointer was looked up previously */ + bh_assert(func_ptrs[func_idx] != NULL); + } + + if (!(func_ptr = func_ptrs[func_idx])) { + bh_assert(func_idx < aot_module->import_func_count); + import_func = aot_module->import_funcs + func_idx; + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + import_func->module_name, import_func->func_name); + aot_set_exception(module_inst, buf); + goto fail; + } + + if (func_idx < aot_module->import_func_count) { + /* Call native function */ + import_func = aot_module->import_funcs + func_idx; + signature = import_func->signature; + if (import_func->call_conv_raw) { + attachment = import_func->attachment; + ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, + argc, argv); + if (!ret) + goto fail; + + return true; + } + } + + ext_ret_count = + func_type->result_count > 1 ? func_type->result_count - 1 : 0; + if (ext_ret_count > 0) { + uint32 argv1_buf[32], *argv1 = argv1_buf; + uint32 *ext_rets = NULL, *argv_ret = argv; + uint32 cell_num = 0, i; + uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; + uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); + uint64 size; + + /* Allocate memory all arguments */ + size = + sizeof(uint32) * (uint64)argc /* original arguments */ + + sizeof(void *) + * (uint64)ext_ret_count /* extra result values' addr */ + + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */ + if (size > sizeof(argv1_buf) + && !(argv1 = runtime_malloc(size, module_inst->cur_exception, + sizeof(module_inst->cur_exception)))) { + aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY); + goto fail; + } + + /* Copy original arguments */ + bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc); + + /* Get the extra result value's address */ + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + + /* Append each extra result value's address to original arguments */ + for (i = 0; i < ext_ret_count; i++) { + *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) = + (uintptr_t)(ext_rets + cell_num); + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + + ret = invoke_native_internal(exec_env, func_ptr, func_type, signature, + attachment, argv1, argc, argv); + if (!ret) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + goto fail; + } + + /* Get extra result values */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + argv_ret++; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + argv_ret += 2; + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + argv_ret += 4; + break; +#endif + default: + bh_assert(0); + break; + } + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets, + sizeof(uint32) * cell_num); + + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + + return true; + } + else { + ret = invoke_native_internal(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv); + if (!ret) + goto fail; + + return true; + } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_access_exce_check_guard_page(); +#endif + return false; +} + +bool +aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr) +{ + bool ret; + + ret = wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, + app_buf_size, p_native_addr); + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + + return ret; +} + +void * +aot_memmove(void *dest, const void *src, size_t n) +{ + return memmove(dest, src, n); +} + +void * +aot_memset(void *s, int c, size_t n) +{ + return memset(s, c, n); +} + +double +aot_sqrt(double x) +{ + return sqrt(x); +} + +float +aot_sqrtf(float x) +{ + return sqrtf(x); +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, + uint32 len, uint32 dst) +{ + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + AOTModule *aot_module; + uint8 *data; + uint8 *maddr; + uint64 seg_len; + + if (bh_bitmap_get_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.data_dropped, + seg_index)) { + seg_len = 0; + data = NULL; + } + else { + aot_module = (AOTModule *)module_inst->module; + seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; + data = aot_module->mem_init_data_list[seg_index]->bytes; + } + + if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, + (uint64)dst, (uint64)len)) + return false; + + if ((uint64)offset + (uint64)len > seg_len) { + aot_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + maddr = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, (uint64)dst); + + SHARED_MEMORY_LOCK(memory_inst); + bh_memcpy_s(maddr, (uint32)(memory_inst->memory_data_size - dst), + data + offset, len); + SHARED_MEMORY_UNLOCK(memory_inst); + return true; +} + +bool +aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index) +{ + bh_bitmap_set_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.data_dropped, + seg_index); + /* Currently we can't free the dropped data segment + as the mem_init_data_count is a continuous array */ + return true; +} +#endif /* WASM_ENABLE_BULK_MEMORY */ + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +aot_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, uint32 size) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + AOTModule *module = (AOTModule *)module_inst->module; + + uint32 stack_top_idx = module->aux_stack_top_global_index; + uint64 data_end = module->aux_data_end; + uint64 stack_bottom = module->aux_stack_bottom; + bool is_stack_before_data = stack_bottom < data_end ? true : false; + + /* Check the aux stack space, currently we don't allocate space in heap */ + if ((is_stack_before_data && (size > start_offset)) + || ((!is_stack_before_data) && (start_offset - data_end < size))) + return false; + + if (stack_top_idx != (uint32)-1) { + /* The aux stack top is a wasm global, + set the initial value for the global */ + uint32 global_offset = module->globals[stack_top_idx].data_offset; + uint8 *global_addr = module_inst->global_data + global_offset; + /* TODO: Memory64 the type i32/i64 depends on memory idx type*/ + *(int32 *)global_addr = (uint32)start_offset; + + /* The aux stack boundary is a constant value, + set the value to exec_env */ + exec_env->aux_stack_boundary = (uintptr_t)start_offset - size; + exec_env->aux_stack_bottom = (uintptr_t)start_offset; + return true; + } + + return false; +} + +bool +aot_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, uint32 *size) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + AOTModule *module = (AOTModule *)module_inst->module; + + /* The aux stack information is resolved in loader + and store in module */ + uint64 stack_bottom = module->aux_stack_bottom; + uint32 total_aux_stack_size = module->aux_stack_size; + + if (stack_bottom != 0 && total_aux_stack_size != 0) { + if (start_offset) + *start_offset = stack_bottom; + if (size) + *size = total_aux_stack_size; + return true; + } + return false; +} +#endif + +#if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0) +static void +const_string_node_size_cb(void *key, void *value, void *p_const_string_size) +{ + uint32 const_string_size = 0; + const_string_size += bh_hash_map_get_elem_struct_size(); + const_string_size += strlen((const char *)value) + 1; + *(uint32 *)p_const_string_size += const_string_size; +} + +void +aot_get_module_mem_consumption(const AOTModule *module, + WASMModuleMemConsumption *mem_conspn) +{ + uint32 i, size; + + memset(mem_conspn, 0, sizeof(*mem_conspn)); + + mem_conspn->module_struct_size = sizeof(AOTModule); + + mem_conspn->types_size = sizeof(AOTFuncType *) * module->type_count; + for (i = 0; i < module->type_count; i++) { + AOTFuncType *type = (AOTFuncType *)module->types[i]; + size = offsetof(AOTFuncType, types) + + sizeof(uint8) * (type->param_count + type->result_count); + mem_conspn->types_size += size; + } + + mem_conspn->imports_size = + sizeof(AOTImportMemory) * module->import_memory_count + + sizeof(AOTImportTable) * module->import_table_count + + sizeof(AOTImportGlobal) * module->import_global_count + + sizeof(AOTImportFunc) * module->import_func_count; + + /* func_ptrs and func_type_indexes */ + mem_conspn->functions_size = + (sizeof(void *) + sizeof(uint32)) * module->func_count; + + mem_conspn->tables_size = sizeof(AOTTable) * module->table_count; + + mem_conspn->memories_size = sizeof(AOTMemory) * module->memory_count; + mem_conspn->globals_size = sizeof(AOTGlobal) * module->global_count; + mem_conspn->exports_size = sizeof(AOTExport) * module->export_count; + + mem_conspn->table_segs_size = + sizeof(AOTTableInitData *) * module->table_init_data_count; + for (i = 0; i < module->table_init_data_count; i++) { + AOTTableInitData *init_data = module->table_init_data_list[i]; + size = offsetof(AOTTableInitData, init_values) + + sizeof(InitializerExpression) * init_data->value_count; + mem_conspn->table_segs_size += size; + } + + mem_conspn->data_segs_size = + sizeof(AOTMemInitData *) * module->mem_init_data_count; + for (i = 0; i < module->mem_init_data_count; i++) { + mem_conspn->data_segs_size += sizeof(AOTMemInitData); + } + + if (module->const_str_set) { + uint32 const_string_size = 0; + + mem_conspn->const_strs_size = + bh_hash_map_get_struct_size(module->const_str_set); + + bh_hash_map_traverse(module->const_str_set, const_string_node_size_cb, + (void *)&const_string_size); + mem_conspn->const_strs_size += const_string_size; + } + + /* code size + literal size + object data section size */ + mem_conspn->aot_code_size = + module->code_size + module->literal_size + + sizeof(AOTObjectDataSection) * module->data_section_count; + for (i = 0; i < module->data_section_count; i++) { + AOTObjectDataSection *obj_data = module->data_sections + i; + mem_conspn->aot_code_size += sizeof(uint8) * obj_data->size; + } + + mem_conspn->total_size += mem_conspn->module_struct_size; + mem_conspn->total_size += mem_conspn->types_size; + mem_conspn->total_size += mem_conspn->imports_size; + mem_conspn->total_size += mem_conspn->functions_size; + mem_conspn->total_size += mem_conspn->tables_size; + mem_conspn->total_size += mem_conspn->memories_size; + mem_conspn->total_size += mem_conspn->globals_size; + mem_conspn->total_size += mem_conspn->exports_size; + mem_conspn->total_size += mem_conspn->table_segs_size; + mem_conspn->total_size += mem_conspn->data_segs_size; + mem_conspn->total_size += mem_conspn->const_strs_size; + mem_conspn->total_size += mem_conspn->aot_code_size; +} + +void +aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, + WASMModuleInstMemConsumption *mem_conspn) +{ + AOTTableInstance *tbl_inst; + uint32 i; + + memset(mem_conspn, 0, sizeof(*mem_conspn)); + + mem_conspn->module_inst_struct_size = sizeof(AOTModuleInstance); + + mem_conspn->memories_size = + sizeof(void *) * module_inst->memory_count + + sizeof(AOTMemoryInstance) * module_inst->memory_count; + for (i = 0; i < module_inst->memory_count; i++) { + AOTMemoryInstance *mem_inst = module_inst->memories[i]; + mem_conspn->memories_size += + mem_inst->num_bytes_per_page * mem_inst->cur_page_count; + mem_conspn->app_heap_size = + mem_inst->heap_data_end - mem_inst->heap_data; + /* size of app heap structure */ + mem_conspn->memories_size += mem_allocator_get_heap_struct_size(); + } + + mem_conspn->tables_size += + sizeof(AOTTableInstance *) * module_inst->table_count; + for (i = 0; i < module_inst->table_count; i++) { + tbl_inst = module_inst->tables[i]; + mem_conspn->tables_size += offsetof(AOTTableInstance, elems); + mem_conspn->tables_size += sizeof(uint32) * tbl_inst->max_size; + } + + /* func_ptrs and func_type_indexes */ + mem_conspn->functions_size = + (sizeof(void *) + sizeof(uint32)) + * (((AOTModule *)module_inst->module)->import_func_count + + ((AOTModule *)module_inst->module)->func_count); + + mem_conspn->globals_size = module_inst->global_data_size; + + mem_conspn->exports_size = + sizeof(AOTFunctionInstance) * (uint64)module_inst->export_func_count; + + mem_conspn->total_size += mem_conspn->module_inst_struct_size; + mem_conspn->total_size += mem_conspn->memories_size; + mem_conspn->total_size += mem_conspn->functions_size; + mem_conspn->total_size += mem_conspn->tables_size; + mem_conspn->total_size += mem_conspn->globals_size; + mem_conspn->total_size += mem_conspn->exports_size; +} +#endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \ + || (WASM_ENABLE_MEMORY_TRACING != 0) */ + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +void +aot_drop_table_seg(AOTModuleInstance *module_inst, uint32 tbl_seg_idx) +{ + bh_bitmap_set_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.elem_dropped, + tbl_seg_idx); +} + +void +aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + AOTTableInstance *tbl_inst; + AOTTableInitData *tbl_seg; + const AOTModule *module = (AOTModule *)module_inst->module; + table_elem_type_t *table_elems; + InitializerExpression *tbl_seg_init_values = NULL, *init_values; + uint32 i, tbl_seg_len = 0; +#if WASM_ENABLE_GC != 0 + void *func_obj; +#endif + + tbl_inst = module_inst->tables[tbl_idx]; + bh_assert(tbl_inst); + + tbl_seg = module->table_init_data_list[tbl_seg_idx]; + bh_assert(tbl_seg); + + if (!bh_bitmap_get_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.elem_dropped, + tbl_seg_idx)) { + /* table segment isn't dropped */ + tbl_seg_init_values = tbl_seg->init_values; + tbl_seg_len = tbl_seg->value_count; + } + + if (offset_len_out_of_bounds(src_offset, length, tbl_seg_len) + || offset_len_out_of_bounds(dst_offset, length, tbl_inst->cur_size)) { + aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + if (!length) { + return; + } + + table_elems = tbl_inst->elems + dst_offset; + init_values = tbl_seg_init_values + src_offset; + + for (i = 0; i < length; i++) { +#if WASM_ENABLE_GC != 0 + /* UINT32_MAX indicates that it is a null ref */ + if (init_values[i].u.ref_index != UINT32_MAX) { + if (!(func_obj = aot_create_func_obj(module_inst, + init_values[i].u.ref_index, + true, NULL, 0))) { + aot_set_exception_with_id(module_inst, EXCE_NULL_FUNC_OBJ); + return; + } + table_elems[i] = func_obj; + } + else { + table_elems[i] = NULL_REF; + } +#else + table_elems[i] = init_values[i].u.ref_index; +#endif + } +} + +void +aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + AOTTableInstance *src_tbl_inst, *dst_tbl_inst; + + src_tbl_inst = module_inst->tables[src_tbl_idx]; + bh_assert(src_tbl_inst); + + dst_tbl_inst = module_inst->tables[dst_tbl_idx]; + bh_assert(dst_tbl_inst); + + if (offset_len_out_of_bounds(dst_offset, length, dst_tbl_inst->cur_size) + || offset_len_out_of_bounds(src_offset, length, + src_tbl_inst->cur_size)) { + aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + /* if src_offset >= dst_offset, copy from front to back */ + /* if src_offset < dst_offset, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(AOTTableInstance, elems) + + dst_offset * sizeof(table_elem_type_t), + (dst_tbl_inst->cur_size - dst_offset) + * sizeof(table_elem_type_t), + (uint8 *)src_tbl_inst + offsetof(AOTTableInstance, elems) + + src_offset * sizeof(table_elem_type_t), + length * sizeof(table_elem_type_t)); +} + +void +aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length, + table_elem_type_t val, uint32 data_offset) +{ + AOTTableInstance *tbl_inst; + + tbl_inst = module_inst->tables[tbl_idx]; + bh_assert(tbl_inst); + + if (offset_len_out_of_bounds(data_offset, length, tbl_inst->cur_size)) { + aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + for (; length != 0; data_offset++, length--) { + tbl_inst->elems[data_offset] = val; + } +} + +uint32 +aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 inc_size, + table_elem_type_t init_val) +{ + AOTTableInstance *tbl_inst; + uint32 i, orig_size, total_size; + + tbl_inst = module_inst->tables[tbl_idx]; + if (!tbl_inst) { + return (uint32)-1; + } + + orig_size = tbl_inst->cur_size; + + if (!inc_size) { + return orig_size; + } + + if (tbl_inst->cur_size > UINT32_MAX - inc_size) { +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of integer overflow", + tbl_inst->cur_size, inc_size); +#endif + return (uint32)-1; + } + + total_size = tbl_inst->cur_size + inc_size; + if (total_size > tbl_inst->max_size) { +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of over max size", + tbl_inst->cur_size, inc_size); +#endif + return (uint32)-1; + } + + /* fill in */ + for (i = 0; i < inc_size; ++i) { + tbl_inst->elems[tbl_inst->cur_size + i] = init_val; + } + + tbl_inst->cur_size = total_size; + return orig_size; +} +#endif /* WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 +static const char * +lookup_func_name(const char **func_names, uint32 *func_indexes, + uint32 func_index_count, uint32 func_index) +{ + int64 low = 0, mid; + int64 high = func_index_count - 1; + + if (!func_names || !func_indexes || func_index_count == 0) + return NULL; + + while (low <= high) { + mid = (low + high) / 2; + if (func_index == func_indexes[mid]) { + return func_names[mid]; + } + else if (func_index < func_indexes[mid]) + high = mid - 1; + else + low = mid + 1; + } + + return NULL; +} +#endif /* WASM_ENABLE_CUSTOM_NAME_SECTION != 0 */ + +static const char * +get_func_name_from_index(const AOTModuleInstance *module_inst, + uint32 func_index) +{ + const char *func_name = NULL; + AOTModule *module = (AOTModule *)module_inst->module; + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + if ((func_name = + lookup_func_name(module->aux_func_names, module->aux_func_indexes, + module->aux_func_name_count, func_index))) { + return func_name; + } +#endif + + if (func_index < module->import_func_count) { + func_name = module->import_funcs[func_index].func_name; + } + else { + uint32 i; + + for (i = 0; i < module->export_count; i++) { + AOTExport export = module->exports[i]; + if (export.index == func_index && export.kind == EXPORT_KIND_FUNC) { + func_name = export.name; + break; + } + } + } + + return func_name; +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 || \ + WASM_ENABLE_PERF_PROFILING != 0 */ + +#if WASM_ENABLE_GC == 0 +bool +aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; +#if WASM_ENABLE_PERF_PROFILING != 0 + AOTFuncPerfProfInfo *func_perf_prof = + module_inst->func_perf_profilings + func_index; +#endif + AOTFrame *cur_frame, *frame; + uint32 size = (uint32)offsetof(AOTFrame, lp); + + cur_frame = (AOTFrame *)exec_env->cur_frame; + if (!cur_frame) + frame = (AOTFrame *)exec_env->wasm_stack.bottom; + else + frame = (AOTFrame *)((uint8 *)cur_frame + size); + + if ((uint8 *)frame + size > exec_env->wasm_stack.top_boundary) { + aot_set_exception(module_inst, "wasm operand stack overflow"); + return false; + } + + frame->func_index = func_index; + /* No need to initialize ip, it will be committed in jitted code + when needed */ + /* frame->ip = NULL; */ + frame->prev_frame = (AOTFrame *)exec_env->cur_frame; + +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = (uintptr_t)os_time_thread_cputime_us(); + frame->func_perf_prof_info = func_perf_prof; +#endif +#if WASM_ENABLE_MEMORY_PROFILING != 0 + { + uint32 wasm_stack_used = + (uint8 *)frame + size - exec_env->wasm_stack.bottom; + if (wasm_stack_used > exec_env->max_wasm_stack_used) + exec_env->max_wasm_stack_used = wasm_stack_used; + } +#endif + + exec_env->cur_frame = (struct WASMInterpFrame *)frame; + + return true; +} + +static inline void +aot_free_frame_internal(WASMExecEnv *exec_env) +{ + AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame; + AOTFrame *prev_frame = cur_frame->prev_frame; + +#if WASM_ENABLE_PERF_PROFILING != 0 + uint64 time_elapsed = + (uintptr_t)os_time_thread_cputime_us() - cur_frame->time_started; + + cur_frame->func_perf_prof_info->total_exec_time += time_elapsed; + cur_frame->func_perf_prof_info->total_exec_cnt++; + + /* parent function */ + if (prev_frame) + prev_frame->func_perf_prof_info->children_exec_time += time_elapsed; +#endif + + exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame; +} + +void +aot_free_frame(WASMExecEnv *exec_env) +{ + aot_free_frame_internal(exec_env); +} + +#else /* else of WASM_ENABLE_GC == 0 */ + +bool +aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + AOTModule *module = (AOTModule *)module_inst->module; +#if WASM_ENABLE_PERF_PROFILING != 0 + AOTFuncPerfProfInfo *func_perf_prof = + module_inst->func_perf_profilings + func_index; +#endif + AOTFrame *frame; + uint32 max_local_cell_num, max_stack_cell_num, all_cell_num; + uint32 aot_func_idx, frame_size; + + if (func_index >= module->import_func_count) { + aot_func_idx = func_index - module->import_func_count; + max_local_cell_num = module->max_local_cell_nums[aot_func_idx]; + max_stack_cell_num = module->max_stack_cell_nums[aot_func_idx]; + } + else { + AOTFuncType *func_type = module->import_funcs[func_index].func_type; + max_local_cell_num = + func_type->param_cell_num > 2 ? func_type->param_cell_num : 2; + max_stack_cell_num = 0; + } + + all_cell_num = max_local_cell_num + max_stack_cell_num; +#if WASM_ENABLE_GC == 0 + frame_size = (uint32)offsetof(AOTFrame, lp) + all_cell_num * 4; +#else + frame_size = + (uint32)offsetof(AOTFrame, lp) + align_uint(all_cell_num * 5, 4); +#endif + frame = wasm_exec_env_alloc_wasm_frame(exec_env, frame_size); + + if (!frame) { + aot_set_exception(module_inst, "wasm operand stack overflow"); + return false; + } + +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = (uintptr_t)os_time_thread_cputime_us(); + frame->func_perf_prof_info = func_perf_prof; +#endif + +#if WASM_ENABLE_GC != 0 + frame->sp = frame->lp + max_local_cell_num; + frame->frame_ref = (uint8 *)(frame->sp + max_stack_cell_num); +#endif + + frame->prev_frame = (AOTFrame *)exec_env->cur_frame; + exec_env->cur_frame = (struct WASMInterpFrame *)frame; + + frame->func_index = func_index; + return true; +} + +static inline void +aot_free_frame_internal(WASMExecEnv *exec_env) +{ + AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame; + AOTFrame *prev_frame = cur_frame->prev_frame; + +#if WASM_ENABLE_PERF_PROFILING != 0 + uint64 time_elapsed = + (uintptr_t)os_time_thread_cputime_us() - cur_frame->time_started; + + cur_frame->func_perf_prof_info->total_exec_time += time_elapsed; + cur_frame->func_perf_prof_info->total_exec_cnt++; + + /* parent function */ + if (prev_frame) + prev_frame->func_perf_prof_info->children_exec_time += time_elapsed; +#endif + + wasm_exec_env_free_wasm_frame(exec_env, cur_frame); + exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame; +} + +void +aot_free_frame(WASMExecEnv *exec_env) +{ + aot_free_frame_internal(exec_env); +} + +#endif /* end of WASM_ENABLE_GC == 0 */ + +void +aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame; + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + AOTFuncPerfProfInfo *func_perf_prof = + module_inst->func_perf_profilings + cur_frame->func_index; + + if (alloc_frame) { + cur_frame->time_started = (uintptr_t)os_time_thread_cputime_us(); + cur_frame->func_perf_prof_info = func_perf_prof; + } + else { + AOTFrame *prev_frame = cur_frame->prev_frame; + uint64 time_elapsed = + (uintptr_t)os_time_thread_cputime_us() - cur_frame->time_started; + + cur_frame->func_perf_prof_info->total_exec_time += time_elapsed; + cur_frame->func_perf_prof_info->total_exec_cnt++; + + /* parent function */ + if (prev_frame) + prev_frame->func_perf_prof_info->children_exec_time += time_elapsed; + } +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + if (alloc_frame) { +#if WASM_ENABLE_GC == 0 + uint32 wasm_stack_used = (uint8 *)exec_env->cur_frame + + (uint32)offsetof(AOTFrame, lp) + - exec_env->wasm_stack.bottom; +#else + uint32 wasm_stack_used = + exec_env->wasm_stack.top - exec_env->wasm_stack.bottom; +#endif + if (wasm_stack_used > exec_env->max_wasm_stack_used) + exec_env->max_wasm_stack_used = wasm_stack_used; + } +#endif +} +#endif /* end of WASM_ENABLE_AOT_STACK_FRAME != 0 */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +bool +aot_create_call_stack(struct WASMExecEnv *exec_env) +{ + AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame, + *first_frame = cur_frame; + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + AOTModule *module = (AOTModule *)module_inst->module; + uint32 n = 0; + + while (cur_frame) { + cur_frame = cur_frame->prev_frame; + n++; + } + + /* release previous stack frames and create new ones */ + destroy_c_api_frames(module_inst->frames); + if (!bh_vector_init(module_inst->frames, n, sizeof(WASMCApiFrame), false)) { + return false; + } + + cur_frame = first_frame; + while (cur_frame) { + WASMCApiFrame frame = { 0 }; + uint32 max_local_cell_num, max_stack_cell_num; + uint32 all_cell_num, lp_size; + + frame.instance = module_inst; + frame.module_offset = 0; + frame.func_index = (uint32)cur_frame->func_index; + frame.func_offset = (uint32)cur_frame->ip_offset; + frame.func_name_wp = get_func_name_from_index( + module_inst, (uint32)cur_frame->func_index); + + if (cur_frame->func_index >= module->import_func_count) { + uint32 aot_func_idx = + (uint32)(cur_frame->func_index - module->import_func_count); + max_local_cell_num = module->max_local_cell_nums[aot_func_idx]; + max_stack_cell_num = module->max_stack_cell_nums[aot_func_idx]; + } + else { + AOTFuncType *func_type = + module->import_funcs[cur_frame->func_index].func_type; + max_local_cell_num = + func_type->param_cell_num > 2 ? func_type->param_cell_num : 2; + max_stack_cell_num = 0; + } + + all_cell_num = max_local_cell_num + max_stack_cell_num; +#if WASM_ENABLE_GC == 0 + lp_size = all_cell_num * 4; +#else + lp_size = align_uint(all_cell_num * 5, 4); +#endif + if (lp_size > 0) { + if (!(frame.lp = wasm_runtime_malloc(lp_size))) { + destroy_c_api_frames(module_inst->frames); + return false; + } + bh_memcpy_s(frame.lp, lp_size, cur_frame->lp, lp_size); + +#if WASM_ENABLE_GC != 0 + uint32 local_ref_flags_cell_num = + module->func_local_ref_flags[frame.func_index] + .local_ref_flag_cell_num; + uint8 *local_ref_flags = + module->func_local_ref_flags[frame.func_index].local_ref_flags; + frame.sp = frame.lp + (cur_frame->sp - cur_frame->lp); + frame.frame_ref = (uint8 *)frame.lp + + (cur_frame->frame_ref - (uint8 *)cur_frame->lp); + /* copy local ref flags from AOT module */ + bh_memcpy_s(frame.frame_ref, local_ref_flags_cell_num, + local_ref_flags, lp_size); +#endif + } + + if (!bh_vector_append(module_inst->frames, &frame)) { + if (frame.lp) + wasm_runtime_free(frame.lp); + destroy_c_api_frames(module_inst->frames); + return false; + } + + cur_frame = cur_frame->prev_frame; + } + + return true; +} + +#define PRINT_OR_DUMP() \ + do { \ + total_len += \ + wasm_runtime_dump_line_buf_impl(line_buf, print, &buf, &len); \ + if ((!print) && buf && (len == 0)) { \ + exception_unlock(module_inst); \ + return total_len; \ + } \ + } while (0) + +uint32 +aot_dump_call_stack(WASMExecEnv *exec_env, bool print, char *buf, uint32 len) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + uint32 n = 0, total_len = 0, total_frames; + /* reserve 256 bytes for line buffer, any line longer than 256 bytes + * will be truncated */ + char line_buf[256]; + + if (!module_inst->frames) { + return 0; + } + + total_frames = (uint32)bh_vector_size(module_inst->frames); + if (total_frames == 0) { + return 0; + } + + exception_lock(module_inst); + snprintf(line_buf, sizeof(line_buf), "\n"); + PRINT_OR_DUMP(); + + while (n < total_frames) { + WASMCApiFrame frame = { 0 }; + uint32 line_length, i; + + if (!bh_vector_get(module_inst->frames, n, &frame)) { + exception_unlock(module_inst); + return 0; + } + + /* function name not exported, print number instead */ + if (frame.func_name_wp == NULL) { + line_length = snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 ": 0x%04x - $f%" PRIu32 "\n", + n, frame.func_offset, frame.func_index); + } + else { + line_length = snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 ": 0x%04x - %s\n", n, + frame.func_offset, frame.func_name_wp); + } + + if (line_length >= sizeof(line_buf)) { + uint32 line_buffer_len = sizeof(line_buf); + /* If line too long, ensure the last character is '\n' */ + for (i = line_buffer_len - 5; i < line_buffer_len - 2; i++) { + line_buf[i] = '.'; + } + line_buf[line_buffer_len - 2] = '\n'; + } + + PRINT_OR_DUMP(); + + n++; + } + snprintf(line_buf, sizeof(line_buf), "\n"); + PRINT_OR_DUMP(); + exception_unlock(module_inst); + + return total_len + 1; +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 */ + +#if WASM_ENABLE_PERF_PROFILING != 0 +void +aot_dump_perf_profiling(const AOTModuleInstance *module_inst) +{ + AOTFuncPerfProfInfo *perf_prof = + (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings; + AOTModule *module = (AOTModule *)module_inst->module; + uint32 total_func_count = module->import_func_count + module->func_count, i; + const char *func_name; + + os_printf("Performance profiler data:\n"); + for (i = 0; i < total_func_count; i++, perf_prof++) { + if (perf_prof->total_exec_cnt == 0) + continue; + + func_name = get_func_name_from_index(module_inst, i); + + if (func_name) + os_printf( + " func %s, execution time: %.3f ms, execution count: %" PRIu32 + " times, children execution time: %.3f ms\n", + func_name, perf_prof->total_exec_time / 1000.0f, + perf_prof->total_exec_cnt, + perf_prof->children_exec_time / 1000.0f); + else + os_printf(" func %" PRIu32 + ", execution time: %.3f ms, execution count: %" PRIu32 + " times, children execution time: %.3f ms\n", + i, perf_prof->total_exec_time / 1000.0f, + perf_prof->total_exec_cnt, + perf_prof->children_exec_time / 1000.0f); + } +} + +double +aot_summarize_wasm_execute_time(const AOTModuleInstance *inst) +{ + double ret = 0; + + AOTModule *module = (AOTModule *)inst->module; + uint32 total_func_count = module->import_func_count + module->func_count, i; + + for (i = 0; i < total_func_count; i++) { + AOTFuncPerfProfInfo *perf_prof = + (AOTFuncPerfProfInfo *)inst->func_perf_profilings + i; + ret += (perf_prof->total_exec_time - perf_prof->children_exec_time) + / 1000.0f; + } + + return ret; +} + +double +aot_get_wasm_func_exec_time(const AOTModuleInstance *inst, + const char *func_name) +{ + AOTModule *module = (AOTModule *)inst->module; + uint32 total_func_count = module->import_func_count + module->func_count, i; + + for (i = 0; i < total_func_count; i++) { + const char *name_in_wasm = get_func_name_from_index(inst, i); + if (name_in_wasm && strcmp(func_name, name_in_wasm) == 0) { + AOTFuncPerfProfInfo *perf_prof = + (AOTFuncPerfProfInfo *)inst->func_perf_profilings + i; + return (perf_prof->total_exec_time - perf_prof->children_exec_time) + / 1000.0f; + } + } + + return -1.0; +} +#endif /* end of WASM_ENABLE_PERF_PROFILING != 0 */ + +#if WASM_ENABLE_STATIC_PGO != 0 + +/* indirect call target */ +#define IPVK_IndirectCallTarget 0 +/* memory intrinsic functions size */ +#define IPVK_MemOPSize 1 +#define IPVK_First IPVK_IndirectCallTarget +#define IPVK_Last IPVK_MemOPSize + +#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24 +#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 + +static int hasNonDefaultValsPerSite = 0; +static uint32 VPMaxNumValsPerSite = INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE; + +static bool +cmpxchg_ptr(void **ptr, void *old_val, void *new_val) +{ +#if defined(os_atomic_cmpxchg) + return os_atomic_cmpxchg(ptr, &old_val, new_val); +#else + /* TODO: add lock when thread-manager is enabled */ + void *read = *ptr; + if (read == old_val) { + *ptr = new_val; + return true; + } + return false; +#endif +} + +static int +allocateValueProfileCounters(LLVMProfileData *Data) +{ + ValueProfNode **Mem; + uint64 NumVSites = 0, total_size; + uint32 VKI; + + /* When dynamic allocation is enabled, allow tracking the max number of + values allowed. */ + if (!hasNonDefaultValsPerSite) + VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE; + + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + NumVSites += Data->num_value_sites[VKI]; + + /* If NumVSites = 0, calloc is allowed to return a non-null pointer. */ + bh_assert(NumVSites > 0 && "NumVSites can't be zero"); + + total_size = (uint64)sizeof(ValueProfNode *) * NumVSites; + if (total_size > UINT32_MAX + || !(Mem = (ValueProfNode **)wasm_runtime_malloc((uint32)total_size))) { + return 0; + } + memset(Mem, 0, (uint32)total_size); + + if (!cmpxchg_ptr((void **)&Data->values, NULL, Mem)) { + wasm_runtime_free(Mem); + return 0; + } + return 1; +} + +static ValueProfNode * +allocateOneNode(void) +{ + ValueProfNode *Node; + + Node = wasm_runtime_malloc((uint32)sizeof(ValueProfNode)); + if (Node) + memset(Node, 0, sizeof(ValueProfNode)); + return Node; +} + +static void +instrumentTargetValueImpl(uint64 TargetValue, void *Data, uint32 CounterIndex, + uint64 CountValue) +{ + ValueProfNode **ValueCounters; + ValueProfNode *PrevVNode = NULL, *MinCountVNode = NULL, *CurVNode; + LLVMProfileData *PData = (LLVMProfileData *)Data; + uint64 MinCount = UINT64_MAX; + uint8 VDataCount = 0; + bool success = false; + + if (!PData) + return; + if (!CountValue) + return; + if (!PData->values) { + if (!allocateValueProfileCounters(PData)) + return; + } + + ValueCounters = (ValueProfNode **)PData->values; + CurVNode = ValueCounters[CounterIndex]; + + while (CurVNode) { + if (TargetValue == CurVNode->value) { + CurVNode->count += CountValue; + return; + } + if (CurVNode->count < MinCount) { + MinCount = CurVNode->count; + MinCountVNode = CurVNode; + } + PrevVNode = CurVNode; + CurVNode = CurVNode->next; + ++VDataCount; + } + + if (VDataCount >= VPMaxNumValsPerSite) { + if (MinCountVNode->count <= CountValue) { + CurVNode = MinCountVNode; + CurVNode->value = TargetValue; + CurVNode->count = CountValue; + } + else + MinCountVNode->count -= CountValue; + + return; + } + + CurVNode = allocateOneNode(); + if (!CurVNode) + return; + CurVNode->value = TargetValue; + CurVNode->count += CountValue; + + if (!ValueCounters[CounterIndex]) { + success = + cmpxchg_ptr((void **)&ValueCounters[CounterIndex], NULL, CurVNode); + } + else if (PrevVNode && !PrevVNode->next) { + success = cmpxchg_ptr((void **)&PrevVNode->next, 0, CurVNode); + } + + if (!success) { + wasm_runtime_free(CurVNode); + } +} + +void +llvm_profile_instrument_target(uint64 target_value, void *data, + uint32 counter_idx) +{ + instrumentTargetValueImpl(target_value, data, counter_idx, 1); +} + +static inline uint32 +popcount64(uint64 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static inline uint32 +clz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 0x8000000000000000LL)) { + num++; + type <<= 1; + } + return num; +} + +/* Map an (observed) memop size value to the representative value of its range. + For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ +static uint64 +InstrProfGetRangeRepValue(uint64 Value) +{ + if (Value <= 8) + /* The first ranges are individually tracked. Use the value as is. */ + return Value; + else if (Value >= 513) + /* The last range is mapped to its lowest value. */ + return 513; + else if (popcount64(Value) == 1) + /* If it's a power of two, use it as is. */ + return Value; + else + /* Otherwise, take to the previous power of two + 1. */ + return (((uint64)1) << (64 - clz64(Value) - 1)) + 1; +} + +void +llvm_profile_instrument_memop(uint64 target_value, void *data, + uint32 counter_idx) +{ + uint64 rep_value = InstrProfGetRangeRepValue(target_value); + instrumentTargetValueImpl(rep_value, data, counter_idx, 1); +} + +static uint32 +get_pgo_prof_data_size(AOTModuleInstance *module_inst, uint32 *p_num_prof_data, + uint32 *p_num_prof_counters, uint32 *p_padding_size, + uint32 *p_prof_counters_size, uint32 *p_prof_names_size, + uint32 *p_value_counters_size, uint8 **p_prof_names) +{ + AOTModule *module = (AOTModule *)module_inst->module; + LLVMProfileData *prof_data; + uint8 *prof_names = NULL; + uint32 num_prof_data = 0, num_prof_counters = 0, padding_size, i; + uint32 prof_counters_size = 0, prof_names_size = 0; + uint32 total_size, total_size_wo_value_counters; + + for (i = 0; i < module->data_section_count; i++) { + if (!strncmp(module->data_sections[i].name, "__llvm_prf_data", 15)) { + bh_assert(module->data_sections[i].size == sizeof(LLVMProfileData)); + num_prof_data++; + prof_data = (LLVMProfileData *)module->data_sections[i].data; + num_prof_counters += prof_data->num_counters; + } + else if (!strncmp(module->data_sections[i].name, "__llvm_prf_cnts", + 15)) { + prof_counters_size += module->data_sections[i].size; + } + else if (!strncmp(module->data_sections[i].name, "__llvm_prf_names", + 16)) { + prof_names_size = module->data_sections[i].size; + prof_names = module->data_sections[i].data; + } + } + + if (prof_counters_size != num_prof_counters * sizeof(uint64)) + return 0; + + total_size = sizeof(LLVMProfileRawHeader) + + num_prof_data * sizeof(LLVMProfileData_64) + + prof_counters_size + prof_names_size; + padding_size = sizeof(uint64) - (prof_names_size % sizeof(uint64)); + if (padding_size != sizeof(uint64)) + total_size += padding_size; + + /* Total size excluding value counters */ + total_size_wo_value_counters = total_size; + + for (i = 0; i < module->data_section_count; i++) { + if (!strncmp(module->data_sections[i].name, "__llvm_prf_data", 15)) { + uint32 j, k, num_value_sites, num_value_nodes; + ValueProfNode **values, *value_node; + + prof_data = (LLVMProfileData *)module->data_sections[i].data; + values = prof_data->values; + + if (prof_data->num_value_sites[0] > 0 + || prof_data->num_value_sites[1] > 0) { + /* TotalSize (uint32) and NumValueKinds (uint32) */ + total_size += 8; + for (j = 0; j < 2; j++) { + if ((num_value_sites = prof_data->num_value_sites[j]) > 0) { + /* ValueKind (uint32) and NumValueSites (uint32) */ + total_size += 8; + /* (Value + Counter) group counts of each value site, + each count is one byte */ + total_size += align_uint(num_value_sites, 8); + + if (values) { + for (k = 0; k < num_value_sites; k++) { + num_value_nodes = 0; + value_node = *values; + while (value_node) { + num_value_nodes++; + value_node = value_node->next; + } + if (num_value_nodes) { + /* (Value + Counter) groups */ + total_size += num_value_nodes * 8 * 2; + } + values++; + } + } + } + } + } + } + } + + if (p_num_prof_data) + *p_num_prof_data = num_prof_data; + if (p_num_prof_counters) + *p_num_prof_counters = num_prof_counters; + if (p_padding_size) + *p_padding_size = padding_size; + if (p_prof_counters_size) + *p_prof_counters_size = prof_counters_size; + if (p_prof_names_size) + *p_prof_names_size = prof_names_size; + if (p_value_counters_size) + *p_value_counters_size = total_size - total_size_wo_value_counters; + if (p_prof_names) + *p_prof_names = prof_names; + + return total_size; +} + +uint32 +aot_get_pgo_prof_data_size(AOTModuleInstance *module_inst) +{ + return get_pgo_prof_data_size(module_inst, NULL, NULL, NULL, NULL, NULL, + NULL, NULL); +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +uint32 +aot_dump_pgo_prof_data_to_buf(AOTModuleInstance *module_inst, char *buf, + uint32 len) +{ + AOTModule *module = (AOTModule *)module_inst->module; + LLVMProfileRawHeader prof_header = { 0 }; + LLVMProfileData *prof_data; + uint8 *prof_names = NULL; + uint32 num_prof_data = 0, num_prof_counters = 0, padding_size, i; + uint32 prof_counters_size = 0, prof_names_size = 0; + uint32 value_counters_size = 0, value_counters_size_backup = 0; + uint32 total_size, size; + int64 counters_delta, offset_counters; + + total_size = get_pgo_prof_data_size(module_inst, &num_prof_data, + &num_prof_counters, &padding_size, + &prof_counters_size, &prof_names_size, + &value_counters_size, &prof_names); + if (len < total_size) + return 0; + + value_counters_size_backup = value_counters_size; + value_counters_size = 0; + + prof_header.counters_delta = counters_delta = + sizeof(LLVMProfileData_64) * num_prof_data; + offset_counters = 0; + for (i = 0; i < module->data_section_count; i++) { + if (!strncmp(module->data_sections[i].name, "__llvm_prf_data", 15)) { + prof_data = (LLVMProfileData *)module->data_sections[i].data; + prof_data->offset_counters = counters_delta + offset_counters; + offset_counters += prof_data->num_counters * sizeof(uint64); + counters_delta -= sizeof(LLVMProfileData_64); + } + } + + prof_header.magic = 0xFF6C70726F667281LL; + /* Version 8 */ + prof_header.version = 0x0000000000000008LL; + /* with VARIANT_MASK_IR_PROF (IR Instrumentation) */ + prof_header.version |= 0x1ULL << 56; + /* with VARIANT_MASK_MEMPROF (Memory Profile) */ + prof_header.version |= 0x1ULL << 62; + prof_header.num_prof_data = num_prof_data; + prof_header.num_prof_counters = num_prof_counters; + prof_header.names_size = prof_names_size; + prof_header.value_kind_last = 1; + + if (!is_little_endian()) { + aot_exchange_uint64((uint8 *)&prof_header.magic); + aot_exchange_uint64((uint8 *)&prof_header.version); + aot_exchange_uint64((uint8 *)&prof_header.num_prof_data); + aot_exchange_uint64((uint8 *)&prof_header.num_prof_counters); + aot_exchange_uint64((uint8 *)&prof_header.names_size); + aot_exchange_uint64((uint8 *)&prof_header.counters_delta); + aot_exchange_uint64((uint8 *)&prof_header.value_kind_last); + } + + size = sizeof(LLVMProfileRawHeader); + bh_memcpy_s(buf, size, &prof_header, size); + buf += size; + + for (i = 0; i < module->data_section_count; i++) { + if (!strncmp(module->data_sections[i].name, "__llvm_prf_data", 15)) { + LLVMProfileData_64 *prof_data_64 = (LLVMProfileData_64 *)buf; + + /* Convert LLVMProfileData to LLVMProfileData_64, the pointer width + in the output file is alawys 8 bytes */ + prof_data = (LLVMProfileData *)module->data_sections[i].data; + prof_data_64->func_md5 = prof_data->func_md5; + prof_data_64->func_hash = prof_data->func_hash; + prof_data_64->offset_counters = prof_data->offset_counters; + prof_data_64->func_ptr = prof_data->func_ptr; + prof_data_64->values = (uint64)(uintptr_t)prof_data->values; + prof_data_64->num_counters = prof_data->num_counters; + prof_data_64->num_value_sites[0] = prof_data->num_value_sites[0]; + prof_data_64->num_value_sites[1] = prof_data->num_value_sites[1]; + + if (!is_little_endian()) { + aot_exchange_uint64((uint8 *)&prof_data_64->func_hash); + aot_exchange_uint64((uint8 *)&prof_data_64->offset_counters); + aot_exchange_uint64((uint8 *)&prof_data_64->offset_counters); + aot_exchange_uint64((uint8 *)&prof_data_64->func_ptr); + aot_exchange_uint64((uint8 *)&prof_data_64->values); + aot_exchange_uint32((uint8 *)&prof_data_64->num_counters); + aot_exchange_uint16((uint8 *)&prof_data_64->num_value_sites[0]); + aot_exchange_uint16((uint8 *)&prof_data_64->num_value_sites[1]); + } + buf += sizeof(LLVMProfileData_64); + } + } + + for (i = 0; i < module->data_section_count; i++) { + if (!strncmp(module->data_sections[i].name, "__llvm_prf_cnts", 15)) { + size = module->data_sections[i].size; + bh_memcpy_s(buf, size, module->data_sections[i].data, size); + buf += size; + } + } + + if (prof_names && prof_names_size > 0) { + size = prof_names_size; + bh_memcpy_s(buf, size, prof_names, size); + buf += size; + padding_size = sizeof(uint64) - (prof_names_size % sizeof(uint64)); + if (padding_size != sizeof(uint64)) { + char padding_buf[8] = { 0 }; + bh_memcpy_s(buf, padding_size, padding_buf, padding_size); + buf += padding_size; + } + } + + for (i = 0; i < module->data_section_count; i++) { + if (!strncmp(module->data_sections[i].name, "__llvm_prf_data", 15)) { + uint32 j, k, num_value_sites, num_value_nodes; + ValueProfNode **values, **values_tmp, *value_node; + + prof_data = (LLVMProfileData *)module->data_sections[i].data; + values = values_tmp = prof_data->values; + + if (prof_data->num_value_sites[0] > 0 + || prof_data->num_value_sites[1] > 0) { + uint32 *buf_total_size = (uint32 *)buf; + + buf += 4; /* emit TotalSize later */ + *(uint32 *)buf = (prof_data->num_value_sites[0] > 0 + && prof_data->num_value_sites[1] > 0) + ? 2 + : 1; + if (!is_little_endian()) + aot_exchange_uint32((uint8 *)buf); + buf += 4; + + for (j = 0; j < 2; j++) { + if ((num_value_sites = prof_data->num_value_sites[j]) > 0) { + /* ValueKind */ + *(uint32 *)buf = j; + if (!is_little_endian()) + aot_exchange_uint32((uint8 *)buf); + buf += 4; + /* NumValueSites */ + *(uint32 *)buf = num_value_sites; + if (!is_little_endian()) + aot_exchange_uint32((uint8 *)buf); + buf += 4; + + for (k = 0; k < num_value_sites; k++) { + num_value_nodes = 0; + if (values_tmp) { + value_node = *values_tmp; + while (value_node) { + num_value_nodes++; + value_node = value_node->next; + } + values_tmp++; + } + bh_assert(num_value_nodes < 255); + *(uint8 *)buf++ = (uint8)num_value_nodes; + } + if (num_value_sites % 8) { + buf += 8 - (num_value_sites % 8); + } + + for (k = 0; k < num_value_sites; k++) { + if (values) { + value_node = *values; + while (value_node) { + *(uint64 *)buf = value_node->value; + if (!is_little_endian()) + aot_exchange_uint64((uint8 *)buf); + buf += 8; + *(uint64 *)buf = value_node->count; + if (!is_little_endian()) + aot_exchange_uint64((uint8 *)buf); + buf += 8; + value_node = value_node->next; + } + values++; + } + } + } + } + + /* TotalSize */ + *(uint32 *)buf_total_size = + (uint8 *)buf - (uint8 *)buf_total_size; + if (!is_little_endian()) + aot_exchange_uint64((uint8 *)buf_total_size); + value_counters_size += (uint8 *)buf - (uint8 *)buf_total_size; + } + } + } + + bh_assert(value_counters_size == value_counters_size_backup); + (void)value_counters_size_backup; + + return total_size; +} +#endif /* end of WASM_ENABLE_STATIC_PGO != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +aot_create_func_obj(AOTModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, uint32 error_buf_size) +{ + AOTModule *module = (AOTModule *)module_inst->module; + WASMRttTypeRef rtt_type; + WASMFuncObjectRef func_obj; + AOTFuncType *func_type; + uint32 type_idx; + + if (throw_exce) { + error_buf = module_inst->cur_exception; + error_buf_size = sizeof(module_inst->cur_exception); + } + + if (func_idx >= module->import_func_count + module->func_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown function %d", + func_idx); + return NULL; + } + + type_idx = module_inst->func_type_indexes[func_idx]; + func_type = (AOTFuncType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new((AOTType *)func_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, "create rtt object failed"); + return NULL; + } + + if (!(func_obj = wasm_func_obj_new_internal( + ((AOTModuleInstanceExtra *)module_inst->e)->common.gc_heap_handle, + rtt_type, func_idx))) { + set_error_buf(error_buf, error_buf_size, "create func object failed"); + return NULL; + } + + return func_obj; +} + +bool +aot_obj_is_instance_of(AOTModuleInstance *module_inst, WASMObjectRef gc_obj, + uint32 type_index) +{ + AOTModule *aot_module = (AOTModule *)module_inst->module; + AOTType **types = aot_module->types; + uint32 type_count = aot_module->type_count; + + return wasm_obj_is_instance_of(gc_obj, type_index, types, type_count); +} + +bool +aot_func_type_is_super_of(AOTModuleInstance *module_inst, uint32 type_idx1, + uint32 type_idx2) +{ + AOTModule *aot_module = (AOTModule *)module_inst->module; + AOTType **types = aot_module->types; + + if (type_idx1 == type_idx2) + return true; + + bh_assert(types[type_idx1]->type_flag == WASM_TYPE_FUNC); + bh_assert(types[type_idx2]->type_flag == WASM_TYPE_FUNC); + return wasm_func_type_is_super_of((WASMFuncType *)types[type_idx1], + (WASMFuncType *)types[type_idx2]); +} + +WASMRttTypeRef +aot_rtt_type_new(AOTModuleInstance *module_inst, uint32 type_index) +{ + AOTModule *aot_module = (AOTModule *)module_inst->module; + AOTType *defined_type = aot_module->types[type_index]; + WASMRttType **rtt_types = aot_module->rtt_types; + uint32 rtt_type_count = aot_module->type_count; + korp_mutex *rtt_type_lock = &aot_module->rtt_type_lock; + + return wasm_rtt_type_new(defined_type, type_index, rtt_types, + rtt_type_count, rtt_type_lock); +} + +bool +aot_array_init_with_data(AOTModuleInstance *module_inst, uint32 seg_index, + uint32 data_seg_offset, WASMArrayObjectRef array_obj, + uint32 elem_size, uint32 array_len) +{ + AOTModule *aot_module; + uint8 *data = NULL; + uint8 *array_elem_base; + uint64 seg_len = 0; + uint64 total_size = (int64)elem_size * array_len; + + aot_module = (AOTModule *)module_inst->module; + seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; + data = aot_module->mem_init_data_list[seg_index]->bytes; + + if (data_seg_offset >= seg_len || total_size > seg_len - data_seg_offset) { + aot_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + array_elem_base = (uint8 *)wasm_array_obj_first_elem_addr(array_obj); + bh_memcpy_s(array_elem_base, (uint32)total_size, data + data_seg_offset, + (uint32)total_size); + + return true; +} + +static bool +aot_global_traverse_gc_rootset(AOTModuleInstance *module_inst, void *heap) +{ + AOTModule *module = (AOTModule *)module_inst->module; + uint8 *global_data = module_inst->global_data; + AOTImportGlobal *import_global = module->import_globals; + AOTGlobal *global = module->globals; + WASMObjectRef gc_obj; + uint32 i; + + for (i = 0; i < module->import_global_count; i++, import_global++) { + if (wasm_is_type_reftype(import_global->type)) { + gc_obj = GET_REF_FROM_ADDR((uint32 *)global_data); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + global_data += import_global->size; + } + + for (i = 0; i < module->global_count; i++, global++) { + if (wasm_is_type_reftype(global->type)) { + gc_obj = GET_REF_FROM_ADDR((uint32 *)global_data); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + global_data += global->size; + } + + return true; +} + +static bool +aot_table_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap) +{ + AOTTableInstance **tables = (AOTTableInstance **)module_inst->tables; + AOTTableInstance *table; + uint32 table_count = module_inst->table_count, i, j; + WASMObjectRef gc_obj, *table_elems; + + for (i = 0; i < table_count; i++) { + table = tables[i]; + table_elems = (WASMObjectRef *)table->elems; + for (j = 0; j < table->cur_size; j++) { + gc_obj = table_elems[j]; + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + } + + return true; +} + +static bool +local_object_refs_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMLocalObjectRef *r; + WASMObjectRef gc_obj; + + for (r = exec_env->cur_local_object_ref; r; r = r->prev) { + gc_obj = r->val; + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + return true; +} + +static bool +aot_frame_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + AOTFrame *frame; + AOTModule *module; + LocalRefFlag frame_local_flags; + WASMObjectRef gc_obj; + uint32 i, local_ref_flag_cell_num; + + module = (AOTModule *)wasm_exec_env_get_module(exec_env); + frame = (AOTFrame *)wasm_exec_env_get_cur_frame(exec_env); + for (; frame; frame = frame->prev_frame) { + /* local ref flags */ + frame_local_flags = module->func_local_ref_flags[frame->func_index]; + local_ref_flag_cell_num = frame_local_flags.local_ref_flag_cell_num; + for (i = 0; i < local_ref_flag_cell_num; i++) { + if (frame_local_flags.local_ref_flags[i]) { + gc_obj = GET_REF_FROM_ADDR(frame->lp + i); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) { + return false; + } + } +#if UINTPTR_MAX == UINT64_MAX + bh_assert(frame_local_flags.local_ref_flags[i + 1]); + i++; +#endif + } + } + + /* stack ref flags */ + uint8 *frame_ref = frame->frame_ref; + for (i = local_ref_flag_cell_num; i < (uint32)(frame->sp - frame->lp); + i++) { + if (frame_ref[i]) { + gc_obj = GET_REF_FROM_ADDR(frame->lp + i); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) { + return false; + } + } +#if UINTPTR_MAX == UINT64_MAX + bh_assert(frame_ref[i + 1]); + i++; +#endif + } + } + } + return true; +} + +bool +aot_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; + bool ret; + + ret = aot_global_traverse_gc_rootset(module_inst, heap); + if (!ret) + return ret; + + ret = aot_table_traverse_gc_rootset(module_inst, heap); + if (!ret) + return ret; + + ret = local_object_refs_traverse_gc_rootset(exec_env, heap); + if (!ret) + return ret; + + ret = aot_frame_traverse_gc_rootset(exec_env, heap); + if (!ret) + return ret; + + return true; +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +char * +aot_const_str_set_insert(const uint8 *str, int32 len, AOTModule *module, +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + bool is_vram_word_align, +#endif + char *error_buf, uint32 error_buf_size) +{ + HashMap *set = module->const_str_set; + char *c_str, *value; + + /* Create const string set if it isn't created */ + if (!set + && !(set = module->const_str_set = bh_hash_map_create( + 32, false, (HashFunc)wasm_string_hash, + (KeyEqualFunc)wasm_string_equal, NULL, wasm_runtime_free))) { + set_error_buf(error_buf, error_buf_size, + "create const string set failed"); + return NULL; + } + + /* Lookup const string set, use the string if found */ + if (!(c_str = runtime_malloc((uint32)len, error_buf, error_buf_size))) { + return NULL; + } +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + if (is_vram_word_align) { + bh_memcpy_wa(c_str, (uint32)len, str, (uint32)len); + } + else +#endif + { + bh_memcpy_s(c_str, len, str, (uint32)len); + } + + if ((value = bh_hash_map_find(set, c_str))) { + wasm_runtime_free(c_str); + return value; + } + + if (!bh_hash_map_insert(set, c_str, c_str)) { + set_error_buf(error_buf, error_buf_size, + "insert string to hash map failed"); + wasm_runtime_free(c_str); + return NULL; + } + + return c_str; +} + +bool +aot_set_module_name(AOTModule *module, const char *name, char *error_buf, + uint32_t error_buf_size) +{ + if (!name) + return false; + + module->name = aot_const_str_set_insert((const uint8 *)name, + (uint32)(strlen(name) + 1), module, +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + false, +#endif + error_buf, error_buf_size); + return module->name != NULL; +} + +const char * +aot_get_module_name(AOTModule *module) +{ + return module->name; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/aot_runtime.h b/wasm-micro-runtime/core/iwasm/aot/aot_runtime.h new file mode 100644 index 0000000..63f8c87 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/aot_runtime.h @@ -0,0 +1,790 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_RUNTIME_H_ +#define _AOT_RUNTIME_H_ + +#include "bh_platform.h" +#include "../common/wasm_runtime_common.h" +#include "../interpreter/wasm_runtime.h" +#include "../compilation/aot.h" +#if WASM_ENABLE_GC != 0 +#include "gc_export.h" +#endif + +#if WASM_ENABLE_WASI_NN != 0 +#include "../libraries/wasi-nn/src/wasi_nn_private.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Wasm feature supported, mainly used by AOTTargetInfo now */ +#define WASM_FEATURE_SIMD_128BIT (1 << 0) +#define WASM_FEATURE_BULK_MEMORY (1 << 1) +#define WASM_FEATURE_MULTI_THREAD (1 << 2) +#define WASM_FEATURE_REF_TYPES (1 << 3) +#define WASM_FEATURE_GARBAGE_COLLECTION (1 << 4) +#define WASM_FEATURE_EXCEPTION_HANDLING (1 << 5) +#define WASM_FEATURE_MEMORY64 (1 << 6) +#define WASM_FEATURE_MULTI_MEMORY (1 << 7) +#define WASM_FEATURE_DYNAMIC_LINKING (1 << 8) +#define WASM_FEATURE_COMPONENT_MODEL (1 << 9) +#define WASM_FEATURE_RELAXED_SIMD (1 << 10) +#define WASM_FEATURE_FLEXIBLE_VECTORS (1 << 11) + +typedef enum AOTSectionType { + AOT_SECTION_TYPE_TARGET_INFO = 0, + AOT_SECTION_TYPE_INIT_DATA = 1, + AOT_SECTION_TYPE_TEXT = 2, + AOT_SECTION_TYPE_FUNCTION = 3, + AOT_SECTION_TYPE_EXPORT = 4, + AOT_SECTION_TYPE_RELOCATION = 5, + AOT_SECTION_TYPE_SIGANATURE = 6, + AOT_SECTION_TYPE_CUSTOM = 100, +} AOTSectionType; + +typedef enum AOTCustomSectionType { + AOT_CUSTOM_SECTION_RAW = 0, + AOT_CUSTOM_SECTION_NATIVE_SYMBOL = 1, + AOT_CUSTOM_SECTION_ACCESS_CONTROL = 2, + AOT_CUSTOM_SECTION_NAME = 3, + AOT_CUSTOM_SECTION_STRING_LITERAL = 4, +} AOTCustomSectionType; + +typedef struct AOTObjectDataSection { + char *name; + uint8 *data; + uint32 size; +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + bool is_name_allocated; + bool is_data_allocated; +#endif +} AOTObjectDataSection; + +/* Relocation info */ +typedef struct AOTRelocation { + uint64 relocation_offset; + int64 relocation_addend; + uint32 relocation_type; + char *symbol_name; + /* index in the symbol offset field */ + uint32 symbol_index; +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + bool is_symbol_name_allocated; +#endif +} AOTRelocation; + +/* Relocation Group */ +typedef struct AOTRelocationGroup { + char *section_name; + /* index in the symbol offset field */ + uint32 name_index; + uint32 relocation_count; + AOTRelocation *relocations; +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + bool is_section_name_allocated; +#endif +} AOTRelocationGroup; + +/* AOT function instance */ +typedef struct AOTFunctionInstance { + char *func_name; + uint32 func_index; + bool is_import_func; + union { + struct { + AOTFuncType *func_type; + /* function pointer linked */ + void *func_ptr; + } func; + AOTImportFunc *func_import; + } u; +} AOTFunctionInstance; + +typedef struct AOTModuleInstanceExtra { + DefPointer(const uint32 *, stack_sizes); + WASMModuleInstanceExtraCommon common; +#if WASM_ENABLE_MULTI_MODULE != 0 + bh_list sub_module_inst_list_head; + bh_list *sub_module_inst_list; +#endif +} AOTModuleInstanceExtra; + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +typedef struct GOTItem { + uint32 func_idx; + struct GOTItem *next; +} GOTItem, *GOTItemList; +#endif + +#if WASM_ENABLE_GC != 0 +typedef struct LocalRefFlag { + uint32 local_ref_flag_cell_num; + uint8 *local_ref_flags; +} LocalRefFlag; +#endif + +typedef struct AOTModule { + uint32 module_type; + + /* import memories */ + uint32 import_memory_count; + AOTImportMemory *import_memories; + + /* memory info */ + uint32 memory_count; + AOTMemory *memories; + + /* init data */ + uint32 mem_init_data_count; + AOTMemInitData **mem_init_data_list; + + /* native symbol */ + void **native_symbol_list; + + /* import tables */ + uint32 import_table_count; + AOTImportTable *import_tables; + + /* tables */ + uint32 table_count; + AOTTable *tables; + + /* table init data info */ + uint32 table_init_data_count; + AOTTableInitData **table_init_data_list; + + /* type info */ + uint32 type_count; + AOTType **types; + + /* import global variable info */ + uint32 import_global_count; + AOTImportGlobal *import_globals; + + /* global variable info */ + uint32 global_count; + AOTGlobal *globals; + + /* total global variable size */ + uint32 global_data_size; + + /* import function info */ + uint32 import_func_count; + AOTImportFunc *import_funcs; + + /* function info */ + uint32 func_count; + /* func pointers of AOTed (un-imported) functions */ + void **func_ptrs; + /* func type indexes of AOTed (un-imported) functions */ + uint32 *func_type_indexes; +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + /* max local cell nums of AOTed (un-imported) functions */ + uint32 *max_local_cell_nums; + /* max stack cell nums of AOTed (un-imported) functions */ + uint32 *max_stack_cell_nums; +#endif + +#if WASM_ENABLE_GC != 0 + /* params + locals ref flags of (both import and AOTed) functions */ + LocalRefFlag *func_local_ref_flags; +#endif + + /* export info */ + uint32 export_count; + AOTExport *exports; + + /* start function index, -1 denotes no start function */ + uint32 start_func_index; + /* start function, point to AOTed function */ + void *start_function; + + uint32 malloc_func_index; + uint32 free_func_index; + uint32 retain_func_index; + + /* AOTed code */ + void *code; + uint32 code_size; + + /* literal for AOTed code */ + uint8 *literal; + uint32 literal_size; + +#if defined(BH_PLATFORM_WINDOWS) + /* extra plt data area for __ymm, __xmm and __real constants + in Windows platform */ + uint8 *extra_plt_data; + uint32 extra_plt_data_size; + uint32 ymm_plt_count; + uint32 xmm_plt_count; + uint32 real_plt_count; + uint32 float_plt_count; +#endif + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + uint32 got_item_count; + GOTItemList got_item_list; + GOTItemList got_item_list_end; + void **got_func_ptrs; +#endif + + /* data sections in AOT object file, including .data, .rodata + and .rodata.cstN. */ + AOTObjectDataSection *data_sections; + uint32 data_section_count; + + /* constant string set */ + HashMap *const_str_set; + + /* the index of auxiliary __data_end global, + -1 means unexported */ + uint32 aux_data_end_global_index; + /* auxiliary __data_end exported by wasm app */ + uint64 aux_data_end; + + /* the index of auxiliary __heap_base global, + -1 means unexported */ + uint32 aux_heap_base_global_index; + /* auxiliary __heap_base exported by wasm app */ + uint64 aux_heap_base; + + /* the index of auxiliary stack top global, + -1 means unexported */ + uint32 aux_stack_top_global_index; + /* auxiliary stack bottom resolved */ + uint64 aux_stack_bottom; + /* auxiliary stack size resolved */ + uint32 aux_stack_size; + + /* is indirect mode or not */ + bool is_indirect_mode; + +#if WASM_ENABLE_LIBC_WASI != 0 + WASIArguments wasi_args; + bool import_wasi_api; +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* TODO: add mutex for mutli-thread? */ + bh_list import_module_list_head; + bh_list *import_module_list; +#endif + +#if WASM_ENABLE_GC != 0 + /* Ref types hash set */ + HashMap *ref_type_set; + struct WASMRttType **rtt_types; + korp_mutex rtt_type_lock; +#if WASM_ENABLE_STRINGREF != 0 + /* special rtts for stringref types + - stringref + - stringview_wtf8 + - stringview_wtf16 + - stringview_iter + */ + struct WASMRttType *stringref_rtts[4]; + uint32 string_literal_count; + uint32 *string_literal_lengths; + const uint8 **string_literal_ptrs; +#endif +#endif + +#if WASM_ENABLE_DEBUG_AOT != 0 + void *elf_hdr; + uint32 elf_size; +#endif +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + const char **aux_func_names; + uint32 *aux_func_indexes; + uint32 aux_func_name_count; +#endif +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + WASMCustomSection *custom_section_list; +#endif + + /* user defined name */ + char *name; +} AOTModule; + +#define AOTMemoryInstance WASMMemoryInstance +#define AOTTableInstance WASMTableInstance +#define AOTModuleInstance WASMModuleInstance + +#if WASM_ENABLE_MULTI_MODULE != 0 +#define AOTSubModInstNode WASMSubModInstNode +#endif + +/* Target info, read from ELF header of object file */ +typedef struct AOTTargetInfo { + /* Binary type, elf32l/elf32b/elf64l/elf64b */ + uint16 bin_type; + /* ABI type */ + uint16 abi_type; + /* Object file type */ + uint16 e_type; + /* Architecture */ + uint16 e_machine; + /* Object file version */ + uint32 e_version; + /* Processor-specific flags */ + uint32 e_flags; + /* Specify wasm features supported */ + uint64 feature_flags; + /* Reserved */ + uint64 reserved; + /* Arch name */ + char arch[16]; +} AOTTargetInfo; + +typedef struct AOTFuncPerfProfInfo { + /* total execution time */ + uint64 total_exec_time; + /* total execution count */ + uint32 total_exec_cnt; + /* children execution time */ + uint64 children_exec_time; +} AOTFuncPerfProfInfo; + +/* AOT auxiliary call stack */ +typedef struct AOTFrame { + /* The frame of the caller which is calling current function */ + struct AOTFrame *prev_frame; + + /* The non-imported function index of current function */ + uintptr_t func_index; + + /* Used when performance profiling is enabled */ + uintptr_t time_started; + + /* Used when performance profiling is enabled */ + AOTFuncPerfProfInfo *func_perf_prof_info; + + /* Instruction pointer: offset to the bytecode array */ + uintptr_t ip_offset; + + /* Operand stack top pointer of the current frame */ + uint32 *sp; + + /* Frame ref flags (GC only) */ + uint8 *frame_ref; + + /** + * Frame data, the layout is: + * local area: parameters and local variables + * stack area: wasm operand stack + * frame ref flags (GC only): + * whether each cell in local and stack area is a GC obj + * currently local's ref flags are stored in AOTModule, + * here we only reserve the padding bytes + */ + uint32 lp[1]; +} AOTFrame; + +#if WASM_ENABLE_STATIC_PGO != 0 +typedef struct LLVMProfileRawHeader { + uint64 magic; + uint64 version; + uint64 binary_ids_size; + uint64 num_prof_data; + uint64 padding_bytes_before_counters; + uint64 num_prof_counters; + uint64 padding_bytes_after_counters; + uint64 names_size; + uint64 counters_delta; + uint64 names_delta; + uint64 value_kind_last; +} LLVMProfileRawHeader; + +typedef struct ValueProfNode { + uint64 value; + uint64 count; + struct ValueProfNode *next; +} ValueProfNode; + +/* The profiling data of data sections created by aot compiler and + used when profiling, the width of pointer can be 8 bytes (64-bit) + or 4 bytes (32-bit) */ +typedef struct LLVMProfileData { + uint64 func_md5; + uint64 func_hash; + uint64 offset_counters; + uintptr_t func_ptr; + ValueProfNode **values; + uint32 num_counters; + uint16 num_value_sites[2]; +} LLVMProfileData; + +/* The profiling data for writting to the output file, the width of + pointer is 8 bytes suppose we always use wamrc and llvm-profdata + with 64-bit mode */ +typedef struct LLVMProfileData_64 { + uint64 func_md5; + uint64 func_hash; + uint64 offset_counters; + uint64 func_ptr; + uint64 values; + uint32 num_counters; + uint16 num_value_sites[2]; +} LLVMProfileData_64; +#endif /* end of WASM_ENABLE_STATIC_PGO != 0 */ + +/** + * Load a AOT module from aot file buffer + * @param buf the byte buffer which contains the AOT file data + * @param size the size of the buffer + * @param error_buf output of the error info + * @param error_buf_size the size of the error string + * + * @return return AOT module loaded, NULL if failed + */ +AOTModule * +aot_load_from_aot_file(const uint8 *buf, uint32 size, const LoadArgs *args, + char *error_buf, uint32 error_buf_size); + +/** + * Load a AOT module from a specified AOT section list. + * + * @param section_list the section list which contains each section data + * @param error_buf output of the error info + * @param error_buf_size the size of the error string + * + * @return return AOT module loaded, NULL if failed + */ +AOTModule * +aot_load_from_sections(AOTSection *section_list, char *error_buf, + uint32 error_buf_size); + +/** + * Unload a AOT module. + * + * @param module the module to be unloaded + */ +void +aot_unload(AOTModule *module); + +/** + * Instantiate a AOT module. + * + * @param module the AOT module to instantiate + * @param parent the parent module instance + * @param heap_size the default heap size of the module instance, a heap will + * be created besides the app memory space. Both wasm app and native + * function can allocate memory from the heap. If heap_size is 0, the + * default heap size will be used. + * @param error_buf buffer to output the error info if failed + * @param error_buf_size the size of the error buffer + * + * @return return the instantiated AOT module instance, NULL if failed + */ +AOTModuleInstance * +aot_instantiate(AOTModule *module, AOTModuleInstance *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size); + +/** + * Deinstantiate a AOT module instance, destroy the resources. + * + * @param module_inst the AOT module instance to destroy + * @param is_sub_inst the flag of sub instance + */ +void +aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst); + +/** + * Lookup an exported function in the AOT module instance. + * + * @param module_inst the module instance + * @param name the name of the function + * + * @return the function instance found + */ +AOTFunctionInstance * +aot_lookup_function(const AOTModuleInstance *module_inst, const char *name); + +/** + * Call the given AOT function of a AOT module instance with + * arguments. + * + * @param exec_env the execution environment + * @param function the function to be called + * @param argc the number of arguments + * @param argv the arguments. If the function method has return value, + * the first (or first two in case 64-bit return value) element of + * argv stores the return value of the called AOT function after this + * function returns. + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call aot_get_exception to get exception info. + */ +bool +aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, + unsigned argc, uint32 argv[]); + +/** + * Set AOT module instance exception with exception string + * + * @param module the AOT module instance + * + * @param exception current exception string + */ +void +aot_set_exception(AOTModuleInstance *module_inst, const char *exception); + +void +aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id); + +/** + * Get exception info of the AOT module instance. + * + * @param module_inst the AOT module instance + * + * @return the exception string + */ +const char * +aot_get_exception(AOTModuleInstance *module_inst); + +/** + * @brief Copy exception in buffer passed as parameter. Thread-safe version of + * `aot_get_exception()` + * @note Buffer size must be no smaller than EXCEPTION_BUF_LEN + * @return true if exception found, false otherwise + */ +bool +aot_copy_exception(AOTModuleInstance *module_inst, char *exception_buf); + +uint64 +aot_module_malloc_internal(AOTModuleInstance *module_inst, WASMExecEnv *env, + uint64 size, void **p_native_addr); + +uint64 +aot_module_realloc_internal(AOTModuleInstance *module_inst, WASMExecEnv *env, + uint64 ptr, uint64 size, void **p_native_addr); + +void +aot_module_free_internal(AOTModuleInstance *module_inst, WASMExecEnv *env, + uint64 ptr); + +uint64 +aot_module_malloc(AOTModuleInstance *module_inst, uint64 size, + void **p_native_addr); + +uint64 +aot_module_realloc(AOTModuleInstance *module_inst, uint64 ptr, uint64 size, + void **p_native_addr); + +void +aot_module_free(AOTModuleInstance *module_inst, uint64 ptr); + +uint64 +aot_module_dup_data(AOTModuleInstance *module_inst, const char *src, + uint64 size); + +bool +aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count); + +/** + * Invoke native function from aot code + */ +bool +aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv); + +bool +aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, + uint32 argc, uint32 *argv); + +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr); + +uint32 +aot_get_plt_table_size(); + +void * +aot_memmove(void *dest, const void *src, size_t n); + +void * +aot_memset(void *s, int c, size_t n); + +double +aot_sqrt(double x); + +float +aot_sqrtf(float x); + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, + uint32 len, uint32 dst); + +bool +aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index); +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +aot_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, uint32 size); + +bool +aot_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, uint32 *size); +#endif + +void +aot_get_module_mem_consumption(const AOTModule *module, + WASMModuleMemConsumption *mem_conspn); + +void +aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, + WASMModuleInstMemConsumption *mem_conspn); + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +void +aot_drop_table_seg(AOTModuleInstance *module_inst, uint32 tbl_seg_idx); + +void +aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length, + table_elem_type_t val, uint32 data_offset); + +uint32 +aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_entries, table_elem_type_t init_val); +#endif + +bool +aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); + +void +aot_free_frame(WASMExecEnv *exec_env); + +void +aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame); + +bool +aot_create_call_stack(struct WASMExecEnv *exec_env); + +/** + * @brief Dump wasm call stack or get the size + * + * @param exec_env the execution environment + * @param print whether to print to stdout or not + * @param buf buffer to store the dumped content + * @param len length of the buffer + * + * @return when print is true, return the bytes printed out to stdout; when + * print is false and buf is NULL, return the size required to store the + * callstack content; when print is false and buf is not NULL, return the size + * dumped to the buffer, 0 means error and data in buf may be invalid + */ +uint32 +aot_dump_call_stack(WASMExecEnv *exec_env, bool print, char *buf, uint32 len); + +void +aot_dump_perf_profiling(const AOTModuleInstance *module_inst); + +double +aot_summarize_wasm_execute_time(const AOTModuleInstance *inst); + +double +aot_get_wasm_func_exec_time(const AOTModuleInstance *inst, + const char *func_name); + +const uint8 * +aot_get_custom_section(const AOTModule *module, const char *name, uint32 *len); + +const void * +aot_get_data_section_addr(AOTModule *module, const char *section_name, + uint32 *p_data_size); + +#if WASM_ENABLE_STATIC_PGO != 0 +void +llvm_profile_instrument_target(uint64 target_value, void *data, + uint32 counter_idx); + +void +llvm_profile_instrument_memop(uint64 target_value, void *data, + uint32 counter_idx); + +uint32 +aot_get_pgo_prof_data_size(AOTModuleInstance *module_inst); + +uint32 +aot_dump_pgo_prof_data_to_buf(AOTModuleInstance *module_inst, char *buf, + uint32 len); + +void +aot_exchange_uint16(uint8 *p_data); + +void +aot_exchange_uint32(uint8 *p_data); + +void +aot_exchange_uint64(uint8 *p_data); +#endif /* end of WASM_ENABLE_STATIC_PGO != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +aot_create_func_obj(AOTModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, uint32 error_buf_size); + +bool +aot_obj_is_instance_of(AOTModuleInstance *module_inst, WASMObjectRef gc_obj, + uint32 type_index); + +/* Whether func type1 is one of super types of func type2 */ +bool +aot_func_type_is_super_of(AOTModuleInstance *module_inst, uint32 type_idx1, + uint32 type_idx2); + +WASMRttTypeRef +aot_rtt_type_new(AOTModuleInstance *module_inst, uint32 type_index); + +bool +aot_array_init_with_data(AOTModuleInstance *module_inst, uint32 seg_index, + uint32 data_seg_offset, WASMArrayObjectRef array_obj, + uint32 elem_size, uint32 array_len); + +bool +aot_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap); +#endif /* end of WASM_ENABLE_GC != 0 */ + +char * +aot_const_str_set_insert(const uint8 *str, int32 len, AOTModule *module, +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + bool is_vram_word_align, +#endif + char *error_buf, uint32 error_buf_size); + +bool +aot_set_module_name(AOTModule *module, const char *name, char *error_buf, + uint32_t error_buf_size); + +const char * +aot_get_module_name(AOTModule *module); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_RUNTIME_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_aarch64.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_aarch64.c new file mode 100644 index 0000000..b4bb602 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_aarch64.c @@ -0,0 +1,410 @@ +/* + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#define R_AARCH64_MOVW_UABS_G0 263 +#define R_AARCH64_MOVW_UABS_G0_NC 264 +#define R_AARCH64_MOVW_UABS_G1 265 +#define R_AARCH64_MOVW_UABS_G1_NC 266 +#define R_AARCH64_MOVW_UABS_G2 267 +#define R_AARCH64_MOVW_UABS_G2_NC 268 +#define R_AARCH64_MOVW_UABS_G3 269 + +#define R_AARCH64_MOVW_SABS_G0 270 +#define R_AARCH64_MOVW_SABS_G1 271 +#define R_AARCH64_MOVW_SABS_G2 272 + +#define R_AARCH64_ADR_PREL_LO19 273 +#define R_AARCH64_ADR_PREL_LO21 274 +#define R_AARCH64_ADR_PREL_PG_HI21 275 +#define R_AARCH64_ADR_PREL_PG_HI21_NC 276 + +#define R_AARCH64_ADD_ABS_LO12_NC 277 + +#define R_AARCH64_LDST8_ABS_LO12_NC 278 +#define R_AARCH64_LDST16_ABS_LO12_NC 284 +#define R_AARCH64_LDST32_ABS_LO12_NC 285 +#define R_AARCH64_LDST64_ABS_LO12_NC 286 +#define R_AARCH64_LDST128_ABS_LO12_NC 299 + +#define R_AARCH64_JUMP26 282 +#define R_AARCH64_CALL26 283 + +/* clang-format off */ +static SymbolMap target_sym_map[] = { + REG_COMMON_SYMBOLS +}; +/* clang-format on */ + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +#if (defined(__APPLE__) || defined(__MACH__)) && defined(__arm64__) +#define BUILD_TARGET_AARCH64_DEFAULT "arm64" +#else +#define BUILD_TARGET_AARCH64_DEFAULT "aarch64v8" +#endif + +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + const char *s = BUILD_TARGET; + size_t s_size = sizeof(BUILD_TARGET); + char *d = target_buf; + + /* Set to "aarch64v8" by default if sub version isn't specified */ + if (strcmp(s, "AARCH64") == 0) { + s = BUILD_TARGET_AARCH64_DEFAULT; + s_size = sizeof(BUILD_TARGET_AARCH64_DEFAULT); + } + if (target_buf_size < s_size) { + s_size = target_buf_size; + } + while (--s_size) { + if (*s >= 'A' && *s <= 'Z') + *d++ = *s++ + 'a' - 'A'; + else + *d++ = *s++; + } + /* Ensure the string is null byte ('\0') terminated */ + *d = '\0'; +} +#undef BUILD_TARGET_AARCH64_DEFAULT + +static uint32 +get_plt_item_size() +{ + /* 6*4 bytes instructions and 8 bytes symbol address */ + return 32; +} + +void +init_plt_table(uint8 *plt) +{ + uint32 i, num = sizeof(target_sym_map) / sizeof(SymbolMap); + for (i = 0; i < num; i++) { + uint32 *p = (uint32 *)plt; + *p++ = 0xf81f0ffe; /* str x30, [sp, #-16]! */ + *p++ = 0x100000be; /* adr x30, #20; symbol addr is PC + 5 instructions + below */ + *p++ = 0xf94003de; /* ldr x30, [x30] */ + *p++ = 0xd63f03c0; /* blr x30 */ + *p++ = 0xf84107fe; /* ldr x30, [sp], #16 */ + *p++ = 0xd61f03c0; /* br x30 */ + /* symbol addr */ + *(uint64 *)p = (uint64)(uintptr_t)target_sym_map[i].symbol_addr; + p += 2; + plt += get_plt_item_size(); + } +} + +uint32 +get_plt_table_size() +{ + return get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); +} + +#define SIGN_EXTEND_TO_INT64(val, bits, val_ext) \ + do { \ + int64 m = (int64)((uint64)1 << (bits - 1)); \ + val_ext = ((int64)val ^ m) - m; \ + } while (0) + +#define Page(expr) ((expr) & ~0xFFF) + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { + case R_AARCH64_CALL26: + case R_AARCH64_JUMP26: + { + void *S, *P = (void *)(target_section_addr + reloc_offset); + int64 X, A, initial_addend; + int32 insn, imm26; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + insn = *(int32 *)P; + imm26 = insn & 0x3FFFFFF; + SIGN_EXTEND_TO_INT64(imm26 << 2, 28, initial_addend); + A = initial_addend; + A += (int64)reloc_addend; + + if (symbol_index < 0) { + /* Symbol address itself is an AOT function. + * Apply relocation with the symbol directly. + * Suppose the symbol address is in +-128MB relative + * to the relocation address. + */ + S = symbol_addr; + } + else { + uint8 *plt; + if (reloc_addend > 0) { + set_error_buf( + error_buf, error_buf_size, + "AOT module load failed: relocate to plt table " + "with reloc addend larger than 0 is unsupported."); + return false; + } + /* Symbol address is not an AOT function, + * but a function of runtime or native. Its address is + * beyond of the +-128MB space. Apply relocation with + * the PLT which branch to the target symbol address. + */ + S = plt = (uint8 *)module->code + module->code_size + - get_plt_table_size() + + get_plt_item_size() * symbol_index; + } + + /* S + A - P */ + X = (int64)S + A - (int64)P; + + /* Check overflow: +-128MB */ + if (X > (128 * BH_MB) || X < (-128 * BH_MB)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "target address out of range."); + return false; + } + + /* write the imm26 back to instruction */ + *(int32 *)P = (insn & 0xFC000000) | ((int32)((X >> 2) & 0x3FFFFFF)); + break; + } + + case R_AARCH64_MOVW_UABS_G0: + case R_AARCH64_MOVW_UABS_G0_NC: + case R_AARCH64_MOVW_UABS_G1: + case R_AARCH64_MOVW_UABS_G1_NC: + case R_AARCH64_MOVW_UABS_G2: + case R_AARCH64_MOVW_UABS_G2_NC: + case R_AARCH64_MOVW_UABS_G3: + { + void *S = symbol_addr, + *P = (void *)(target_section_addr + reloc_offset); + int64 X, A, initial_addend; + int32 insn, imm16; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + insn = *(int32 *)P; + imm16 = (insn >> 5) & 0xFFFF; + + SIGN_EXTEND_TO_INT64(imm16, 16, initial_addend); + A = initial_addend; + A += (int64)reloc_addend; + + /* S + A */ + X = (int64)S + A; + + /* No need to check overflow for this relocation type */ + switch (reloc_type) { + case R_AARCH64_MOVW_UABS_G0: + if (X < 0 || X >= (1LL << 16)) + goto overflow_check_fail; + break; + case R_AARCH64_MOVW_UABS_G1: + if (X < 0 || X >= (1LL << 32)) + goto overflow_check_fail; + break; + case R_AARCH64_MOVW_UABS_G2: + if (X < 0 || X >= (1LL << 48)) + goto overflow_check_fail; + break; + default: + break; + } + + /* write the imm16 back to bits[5:20] of instruction */ + switch (reloc_type) { + case R_AARCH64_MOVW_UABS_G0: + case R_AARCH64_MOVW_UABS_G0_NC: + *(int32 *)P = + (insn & 0xFFE0001F) | ((int32)((X & 0xFFFF) << 5)); + break; + case R_AARCH64_MOVW_UABS_G1: + case R_AARCH64_MOVW_UABS_G1_NC: + *(int32 *)P = (insn & 0xFFE0001F) + | ((int32)(((X >> 16) & 0xFFFF) << 5)); + break; + case R_AARCH64_MOVW_UABS_G2: + case R_AARCH64_MOVW_UABS_G2_NC: + *(int32 *)P = (insn & 0xFFE0001F) + | ((int32)(((X >> 32) & 0xFFFF) << 5)); + break; + case R_AARCH64_MOVW_UABS_G3: + *(int32 *)P = (insn & 0xFFE0001F) + | ((int32)(((X >> 48) & 0xFFFF) << 5)); + break; + default: + bh_assert(0); + break; + } + break; + } + + case R_AARCH64_ADR_PREL_PG_HI21: + case R_AARCH64_ADR_PREL_PG_HI21_NC: + { + void *S = symbol_addr, + *P = (void *)(target_section_addr + reloc_offset); + int64 X, A, initial_addend; + int32 insn, immhi19, immlo2, imm21; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + insn = *(int32 *)P; + immhi19 = (insn >> 5) & 0x7FFFF; + immlo2 = (insn >> 29) & 0x3; + imm21 = (immhi19 << 2) | immlo2; + + SIGN_EXTEND_TO_INT64(imm21 << 12, 33, initial_addend); + A = initial_addend; + A += (int64)reloc_addend; + + /* Page(S+A) - Page(P) */ + X = Page((int64)S + A) - Page((int64)P); + + /* Check overflow: +-4GB */ + if (reloc_type == R_AARCH64_ADR_PREL_PG_HI21 + && (X > ((int64)4 * BH_GB) || X < ((int64)-4 * BH_GB))) + goto overflow_check_fail; + + /* write the imm21 back to instruction */ + immhi19 = (int32)(((X >> 12) >> 2) & 0x7FFFF); + immlo2 = (int32)((X >> 12) & 0x3); + *(int32 *)P = (insn & 0x9F00001F) | (immlo2 << 29) | (immhi19 << 5); + + break; + } + + case R_AARCH64_ADD_ABS_LO12_NC: + { + void *S = symbol_addr, + *P = (void *)(target_section_addr + reloc_offset); + int64 X, A, initial_addend; + int32 insn, imm12; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + insn = *(int32 *)P; + imm12 = (insn >> 10) & 0xFFF; + + SIGN_EXTEND_TO_INT64(imm12, 12, initial_addend); + A = initial_addend; + A += (int64)reloc_addend; + + /* S + A */ + X = (int64)S + A; + + /* No need to check overflow for this relocation type */ + + /* write the imm12 back to instruction */ + *(int32 *)P = (insn & 0xFFC003FF) | ((int32)((X & 0xFFF) << 10)); + break; + } + + case R_AARCH64_LDST8_ABS_LO12_NC: + case R_AARCH64_LDST16_ABS_LO12_NC: + case R_AARCH64_LDST32_ABS_LO12_NC: + case R_AARCH64_LDST64_ABS_LO12_NC: + case R_AARCH64_LDST128_ABS_LO12_NC: + { + void *S = symbol_addr, + *P = (void *)(target_section_addr + reloc_offset); + int64 X, A, initial_addend; + int32 insn, imm12; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + insn = *(int32 *)P; + imm12 = (insn >> 10) & 0xFFF; + + SIGN_EXTEND_TO_INT64(imm12, 12, initial_addend); + A = initial_addend; + A += (int64)reloc_addend; + + /* S + A */ + X = (int64)S + A; + + /* No need to check overflow for this relocation type */ + + /* write the imm12 back to instruction */ + switch (reloc_type) { + case R_AARCH64_LDST8_ABS_LO12_NC: + *(int32 *)P = + (insn & 0xFFC003FF) | ((int32)((X & 0xFFF) << 10)); + break; + case R_AARCH64_LDST16_ABS_LO12_NC: + *(int32 *)P = (insn & 0xFFC003FF) + | ((int32)(((X & 0xFFF) >> 1) << 10)); + break; + case R_AARCH64_LDST32_ABS_LO12_NC: + *(int32 *)P = (insn & 0xFFC003FF) + | ((int32)(((X & 0xFFF) >> 2) << 10)); + break; + case R_AARCH64_LDST64_ABS_LO12_NC: + *(int32 *)P = (insn & 0xFFC003FF) + | ((int32)(((X & 0xFFF) >> 3) << 10)); + break; + case R_AARCH64_LDST128_ABS_LO12_NC: + *(int32 *)P = (insn & 0xFFC003FF) + | ((int32)(((X & 0xFFF) >> 4) << 10)); + break; + default: + bh_assert(0); + break; + } + break; + } + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %d.", + reloc_type); + return false; + } + + return true; + +overflow_check_fail: + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "target address out of range."); + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_arc.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_arc.c new file mode 100644 index 0000000..3329629 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_arc.c @@ -0,0 +1,419 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#define R_ARC_S21H_PCREL 14 +#define R_ARC_S21W_PCREL 15 +#define R_ARC_S25H_PCREL 16 +#define R_ARC_S25W_PCREL 17 +#define R_ARC_32 4 +#define R_ARC_32_ME 27 + +/* clang-format off */ +#ifndef __CCAC__ +void __st_r13_to_r15(); +void __st_r13_to_r16(); +void __st_r13_to_r17(); +void __st_r13_to_r18(); +void __st_r13_to_r19(); +void __st_r13_to_r20(); +void __st_r13_to_r21(); +void __st_r13_to_r22(); +void __st_r13_to_r23(); +void __st_r13_to_r24(); +void __st_r13_to_r25(); +void __ld_r13_to_r15(); +void __ld_r13_to_r16(); +void __ld_r13_to_r17(); +void __ld_r13_to_r18(); +void __ld_r13_to_r19(); +void __ld_r13_to_r20(); +void __ld_r13_to_r21(); +void __ld_r13_to_r22(); +void __ld_r13_to_r23(); +void __ld_r13_to_r24(); +void __ld_r13_to_r25(); +void __adddf3(); +void __addsf3(); +void __divdf3(); +void __divdi3(); +void __divsf3(); +void __divsi3(); +void __extendsfdf2(); +void __fixdfsi(); +void __floatsidf(); +void __floatsisf(); +void __muldf3(); +void __mulsf3(); +void __subdf3(); +void __subsf3(); +void __truncdfsf2(); +void __floatunsisf(); +void __fixunsdfsi(); +void __floatdisf(); +void __floatdidf(); +void __fixdfdi(); +void __ltsf2(); +void __gesf2(); +void __eqdf2(); +void __nedf2(); +void __ltsf2(); +void __nesf2(); +void __unordsf2(); +void __fixunssfsi(); +#else +void __ac_push_13_to_13(); +void __ac_push_13_to_14(); +void __ac_push_13_to_15(); +void __ac_push_13_to_16(); +void __ac_push_13_to_17(); +void __ac_push_13_to_18(); +void __ac_push_13_to_19(); +void __ac_push_13_to_20(); +void __ac_push_13_to_21(); +void __ac_push_13_to_22(); +void __ac_push_13_to_23(); +void __ac_push_13_to_24(); +void __ac_push_13_to_25(); +void __ac_push_13_to_26(); +void __ac_push_none(); +void __ac_pop_13_to_26(); +void __ac_pop_13_to_26v(); +void __ac_pop_13_to_25(); +void __ac_pop_13_to_25v(); +void __ac_pop_13_to_24(); +void __ac_pop_13_to_24v(); +void __ac_pop_13_to_23(); +void __ac_pop_13_to_23v(); +void __ac_pop_13_to_22(); +void __ac_pop_13_to_22v(); +void __ac_pop_13_to_21(); +void __ac_pop_13_to_21v(); +void __ac_pop_13_to_20(); +void __ac_pop_13_to_20v(); +void __ac_pop_13_to_19(); +void __ac_pop_13_to_19v(); +void __ac_pop_13_to_18(); +void __ac_pop_13_to_18v(); +void __ac_pop_13_to_17(); +void __ac_pop_13_to_17v(); +void __ac_pop_13_to_16(); +void __ac_pop_13_to_16v(); +void __ac_pop_13_to_15(); +void __ac_pop_13_to_15v(); +void __ac_pop_13_to_14(); +void __ac_pop_13_to_14v(); +void __ac_pop_13_to_13(); +void __ac_pop_13_to_13v(); +void __ac_pop_none(); +void __ac_pop_nonev(); +void __eqdf2(); +void __nedf2(); +void __ltsf2(); +void __nesf2(); +void __gesf2(); +void __gtsf2(); +void __unordsf2(); +void __truncdfhf2(); +void __truncsfhf2(); +#endif /* end of __CCAC__ */ + +void __ledf2(); +void __ltdf2(); +void __gedf2(); +void __gtdf2(); +void __eqsf2(); +void __lesf2(); +void __unorddf2(); +/* clang-format on */ + +static SymbolMap target_sym_map[] = { + /* clang-format off */ + REG_COMMON_SYMBOLS +#ifndef __CCAC__ + REG_SYM(__st_r13_to_r15), + REG_SYM(__st_r13_to_r16), + REG_SYM(__st_r13_to_r17), + REG_SYM(__st_r13_to_r18), + REG_SYM(__st_r13_to_r19), + REG_SYM(__st_r13_to_r20), + REG_SYM(__st_r13_to_r21), + REG_SYM(__st_r13_to_r22), + REG_SYM(__st_r13_to_r23), + REG_SYM(__st_r13_to_r24), + REG_SYM(__st_r13_to_r25), + REG_SYM(__ld_r13_to_r15), + REG_SYM(__ld_r13_to_r16), + REG_SYM(__ld_r13_to_r17), + REG_SYM(__ld_r13_to_r18), + REG_SYM(__ld_r13_to_r19), + REG_SYM(__ld_r13_to_r20), + REG_SYM(__ld_r13_to_r21), + REG_SYM(__ld_r13_to_r22), + REG_SYM(__ld_r13_to_r23), + REG_SYM(__ld_r13_to_r24), + REG_SYM(__ld_r13_to_r25), + REG_SYM(__adddf3), + REG_SYM(__addsf3), + REG_SYM(__divdf3), + REG_SYM(__divdi3), + REG_SYM(__divsf3), + REG_SYM(__divsi3), + REG_SYM(__extendsfdf2), + REG_SYM(__fixdfsi), + REG_SYM(__floatsidf), + REG_SYM(__floatsisf), + REG_SYM(__muldf3), + REG_SYM(__mulsf3), + REG_SYM(__subdf3), + REG_SYM(__subsf3), + REG_SYM(__truncdfsf2), + REG_SYM(__floatunsisf), + REG_SYM(__fixunsdfsi), + REG_SYM(__floatdisf), + REG_SYM(__floatdidf), + REG_SYM(__fixdfdi), + REG_SYM(__ltsf2), + REG_SYM(__gesf2), + REG_SYM(__eqdf2), + REG_SYM(__nedf2), + REG_SYM(__ltsf2), + REG_SYM(__nesf2), + REG_SYM(__unordsf2), + REG_SYM(__fixunssfsi), +#else + REG_SYM(__ac_push_13_to_13), + REG_SYM(__ac_push_13_to_14), + REG_SYM(__ac_push_13_to_15), + REG_SYM(__ac_push_13_to_16), + REG_SYM(__ac_push_13_to_17), + REG_SYM(__ac_push_13_to_18), + REG_SYM(__ac_push_13_to_19), + REG_SYM(__ac_push_13_to_20), + REG_SYM(__ac_push_13_to_21), + REG_SYM(__ac_push_13_to_22), + REG_SYM(__ac_push_13_to_23), + REG_SYM(__ac_push_13_to_24), + REG_SYM(__ac_push_13_to_25), + REG_SYM(__ac_push_13_to_26), + REG_SYM(__ac_push_none), + REG_SYM(__ac_pop_13_to_26), + REG_SYM(__ac_pop_13_to_26v), + REG_SYM(__ac_pop_13_to_25), + REG_SYM(__ac_pop_13_to_25v), + REG_SYM(__ac_pop_13_to_24), + REG_SYM(__ac_pop_13_to_24v), + REG_SYM(__ac_pop_13_to_23), + REG_SYM(__ac_pop_13_to_23v), + REG_SYM(__ac_pop_13_to_22), + REG_SYM(__ac_pop_13_to_22v), + REG_SYM(__ac_pop_13_to_21), + REG_SYM(__ac_pop_13_to_21v), + REG_SYM(__ac_pop_13_to_20), + REG_SYM(__ac_pop_13_to_20v), + REG_SYM(__ac_pop_13_to_19), + REG_SYM(__ac_pop_13_to_19v), + REG_SYM(__ac_pop_13_to_18), + REG_SYM(__ac_pop_13_to_18v), + REG_SYM(__ac_pop_13_to_17), + REG_SYM(__ac_pop_13_to_17v), + REG_SYM(__ac_pop_13_to_16), + REG_SYM(__ac_pop_13_to_16v), + REG_SYM(__ac_pop_13_to_15), + REG_SYM(__ac_pop_13_to_15v), + REG_SYM(__ac_pop_13_to_14), + REG_SYM(__ac_pop_13_to_14v), + REG_SYM(__ac_pop_13_to_13), + REG_SYM(__ac_pop_13_to_13v), + REG_SYM(__ac_pop_none), + REG_SYM(__ac_pop_nonev), + REG_SYM(__eqdf2), + REG_SYM(__nedf2), + REG_SYM(__ltsf2), + REG_SYM(__nesf2), + REG_SYM(__gesf2), + REG_SYM(__gtsf2), + REG_SYM(__unordsf2), + REG_SYM(__truncdfhf2), + REG_SYM(__truncsfhf2), +#endif /* end of __CCAC__ */ + + REG_SYM(__ledf2), + REG_SYM(__ltdf2), + REG_SYM(__gedf2), + REG_SYM(__gtdf2), + REG_SYM(__eqsf2), + REG_SYM(__lesf2), + REG_SYM(__unorddf2), + /* clang-format on */ +}; + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + snprintf(target_buf, target_buf_size, "arc"); +} + +uint32 +get_plt_table_size() +{ + return 0; +} + +void +init_plt_table(uint8 *plt) +{ + (void)plt; +} + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +static uint32 +middle_endian_convert(uint32 insn) +{ + return ((insn & 0xFFFF0000) >> 16) | ((insn & 0x0000FFFF) << 16); +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { + case R_ARC_S25H_PCREL: + { + uint32 insn = LOAD_I32(target_section_addr + reloc_offset); + int32 addend, value; + uintptr_t S, P; + intptr_t A; + + CHECK_RELOC_OFFSET(sizeof(void *)); + + /* Convert from middle endian */ + insn = middle_endian_convert(insn); + + addend = ((insn << 28) >> 28) << 10; + /* Extract the next 10 bits from Position 6 to 15 in insn */ + addend |= ((insn << 16) >> 22); + addend = addend << 10; + /* Extract the remaining 10 bits from Position 17 to 26 in insn */ + addend |= ((insn << 5) >> 22); + /* Fill in 1 bits to get the 25 bit Offset Value */ + addend = addend << 1; + + /* (S + A) - P */ + S = (uintptr_t)(uint8 *)symbol_addr; + A = (intptr_t)reloc_addend; + P = (uintptr_t)(target_section_addr + reloc_offset); + P &= (uintptr_t)~1; + value = (int32)(S + A + addend - P); + + insn = insn & 0xf8010030; + insn |= ((((value >> 1) & 0x3ff) << 17) + | (((value >> 1) & 0xffc00) >> 3) + | (((value >> 1) & 0xf00000) >> 19)); + + /* Convert to middle endian */ + insn = middle_endian_convert(insn); + + STORE_U32(target_section_addr + reloc_offset, insn); + break; + } + case R_ARC_S25W_PCREL: + { + uint32 insn = LOAD_I32(target_section_addr + reloc_offset); + int32 addend, value; + uintptr_t S, P; + intptr_t A; + + CHECK_RELOC_OFFSET(sizeof(void *)); + + /* Convert from middle endian */ + insn = middle_endian_convert(insn); + + addend = ((insn << 28) >> 28) << 10; + /* Extract the next 10 bits from Position 6 to 15 in insn */ + addend |= ((insn << 16) >> 22); + addend = addend << 9; + /* Extract the remaining 9 bits from Position 18 to 26 in insn */ + addend |= ((insn << 5) >> 23); + /* Fill in 2 bits to get the 25 bit Offset Value */ + addend = addend << 2; + + /* (S + A) - P */ + S = (uintptr_t)(uint8 *)symbol_addr; + A = (intptr_t)reloc_addend; + P = (uintptr_t)(target_section_addr + reloc_offset); + P &= (uintptr_t)~3; + value = (int32)(S + A + addend - P); + + insn = insn & 0xf8030030; + insn |= ((((value >> 2) & 0x1ff) << 18) + | (((value >> 2) & 0x7fe00) >> 3) + | (((value >> 2) & 0x780000) >> 19)); + + /* Convert to middle endian */ + insn = middle_endian_convert(insn); + + STORE_U32(target_section_addr + reloc_offset, insn); + break; + } + case R_ARC_32: + case R_ARC_32_ME: + { + uint32 insn; + + CHECK_RELOC_OFFSET(sizeof(void *)); + + /* (S + A) */ + insn = (uint32)((uintptr_t)symbol_addr + (intptr_t)reloc_addend); + + if (reloc_type == R_ARC_32_ME) + /* Convert to middle endian */ + insn = middle_endian_convert(insn); + + STORE_U32(target_section_addr + reloc_offset, insn); + break; + } + default: + { + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %d.", + reloc_type); + return false; + } + } + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_arm.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_arm.c new file mode 100644 index 0000000..5d79e8a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_arm.c @@ -0,0 +1,378 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#define R_ARM_CALL 28 /* PC relative 24 bit (BL, BLX). */ +#define R_ARM_JMP24 29 /* PC relative 24 bit (B/BL). */ +#define R_ARM_ABS32 2 /* Direct 32 bit */ +#define R_ARM_MOVW_ABS_NC 43 +#define R_ARM_MOVT_ABS 44 + +/* clang-format off */ +void __adddf3(); +void __addsf3(); +void __aeabi_d2iz(); +void __aeabi_d2lz(); +void __aeabi_d2ulz(); +void __aeabi_dadd(); +void __aeabi_dcmpge(); +void __aeabi_dcmple(); +void __aeabi_dcmplt(); +void __aeabi_dcmpun(); +void __aeabi_ddiv(); +void __aeabi_f2d(); +void __aeabi_f2iz(); +void __aeabi_f2lz(); +void __aeabi_f2ulz(); +void __aeabi_fcmpge(); +void __aeabi_fcmple(); +void __aeabi_fcmplt(); +void __aeabi_fcmpun(); +void __aeabi_i2d(); +void __aeabi_idiv(); +void __aeabi_idivmod(); +void __aeabi_l2d(); +void __aeabi_l2f(); +void __aeabi_ldivmod(); +void __aeabi_memcpy(); +void __aeabi_memmove(); +void __aeabi_memset(); +void __aeabi_memclr(); +void __aeabi_uidiv(); +void __aeabi_uidivmod(); +void __aeabi_ul2d(); +void __aeabi_ul2f(); +void __aeabi_uldivmod(); +void __clzsi2(); +void __divdf3(); +void __divdi3(); +void __divsf3(); +void __divsi3(); +void __eqdf2(); +void __extendsfdf2(); +void __fixdfdi(); +void __fixdfsi(); +void __fixsfdi(); +void __fixsfsi(); +void __fixunsdfdi(); +void __fixunsdfsi(); +void __fixunssfdi(); +void __floatdidf(); +void __floatdisf(); +void __floatsidf(); +void __floatsisf(); +void __floatundidf(); +void __floatundisf(); +void __floatunsidf(); +void __floatunsisf(); +void __gedf2(); +void __gesf2(); +void __gtdf2(); +void __gtsf2(); +void __ledf2(); +void __lesf2(); +void __ltdf2(); +void __ltsf2(); +void __moddi3(); +void __modsi3(); +void __muldf3(); +void __mulsf3(); +void __nedf2(); +void __nesf2(); +void __subdf3(); +void __subsf3(); +void __truncdfsf2(); +void __udivdi3(); +void __udivmoddi4(); +void __udivsi3(); +void __umoddi3(); +void __umodsi3(); +void __unorddf2(); +void __unordsf2(); +/* clang-format on */ + +static SymbolMap target_sym_map[] = { + /* clang-format off */ + REG_COMMON_SYMBOLS + /* compiler-rt symbols that come from compiler(e.g. gcc) */ + REG_SYM(__adddf3), + REG_SYM(__addsf3), + /* clang-format on */ + REG_SYM(__aeabi_d2iz), + REG_SYM(__aeabi_d2lz), + REG_SYM(__aeabi_d2ulz), + REG_SYM(__aeabi_dadd), + REG_SYM(__aeabi_dcmpge), + REG_SYM(__aeabi_dcmple), + REG_SYM(__aeabi_dcmplt), + REG_SYM(__aeabi_dcmpun), + REG_SYM(__aeabi_ddiv), + REG_SYM(__aeabi_f2d), + REG_SYM(__aeabi_f2iz), + REG_SYM(__aeabi_f2lz), + REG_SYM(__aeabi_f2ulz), + REG_SYM(__aeabi_fcmpge), + REG_SYM(__aeabi_fcmple), + REG_SYM(__aeabi_fcmplt), + REG_SYM(__aeabi_fcmpun), + REG_SYM(__aeabi_i2d), + REG_SYM(__aeabi_idiv), + REG_SYM(__aeabi_idivmod), + REG_SYM(__aeabi_l2d), + REG_SYM(__aeabi_l2f), + REG_SYM(__aeabi_ldivmod), + REG_SYM(__aeabi_memcpy), + REG_SYM(__aeabi_memmove), + REG_SYM(__aeabi_memset), + REG_SYM(__aeabi_memclr), + REG_SYM(__aeabi_uidiv), + REG_SYM(__aeabi_uidivmod), + REG_SYM(__aeabi_ul2d), + REG_SYM(__aeabi_ul2f), + REG_SYM(__aeabi_uldivmod), + REG_SYM(__clzsi2), + REG_SYM(__divdf3), + REG_SYM(__divdi3), + REG_SYM(__divsf3), + REG_SYM(__divsi3), + REG_SYM(__eqdf2), + REG_SYM(__extendsfdf2), + REG_SYM(__fixdfdi), + REG_SYM(__fixdfsi), + REG_SYM(__fixsfdi), + REG_SYM(__fixsfsi), + REG_SYM(__fixunsdfdi), + REG_SYM(__fixunsdfsi), + REG_SYM(__fixunssfdi), + REG_SYM(__floatdidf), + REG_SYM(__floatdisf), + REG_SYM(__floatsidf), + REG_SYM(__floatsisf), + REG_SYM(__floatundidf), + REG_SYM(__floatundisf), + REG_SYM(__floatunsidf), + REG_SYM(__floatunsisf), + REG_SYM(__gedf2), + REG_SYM(__gesf2), + REG_SYM(__gtdf2), + REG_SYM(__gtsf2), + REG_SYM(__ledf2), + REG_SYM(__lesf2), + REG_SYM(__ltdf2), + REG_SYM(__ltsf2), + REG_SYM(__moddi3), + REG_SYM(__modsi3), + REG_SYM(__muldf3), + REG_SYM(__muldf3), + REG_SYM(__mulsf3), + REG_SYM(__nedf2), + REG_SYM(__nesf2), + REG_SYM(__subdf3), + REG_SYM(__subsf3), + REG_SYM(__truncdfsf2), + REG_SYM(__udivdi3), + REG_SYM(__udivmoddi4), + REG_SYM(__udivsi3), + REG_SYM(__umoddi3), + REG_SYM(__umodsi3), + REG_SYM(__unorddf2), + REG_SYM(__unordsf2), +}; + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +#define BUILD_TARGET_ARM_DEFAULT "armv4" +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + const char *s = BUILD_TARGET; + size_t s_size = sizeof(BUILD_TARGET); + char *d = target_buf; + + /* Set to "armv4" by default if sub version isn't specified */ + if (strcmp(s, "ARM") == 0) { + s = BUILD_TARGET_ARM_DEFAULT; + s_size = sizeof(BUILD_TARGET_ARM_DEFAULT); + } + if (target_buf_size < s_size) { + s_size = target_buf_size; + } + while (--s_size) { + if (*s >= 'A' && *s <= 'Z') + *d++ = *s++ + 'a' - 'A'; + else + *d++ = *s++; + } + /* Ensure the string is null byte ('\0') terminated */ + *d = '\0'; +} +#undef BUILD_TARGET_ARM_DEFAULT + +uint32 +get_plt_item_size() +{ + /* 8 bytes instructions and 4 bytes symbol address */ + return 12; +} + +uint32 +get_plt_table_size() +{ + return get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); +} + +void +init_plt_table(uint8 *plt) +{ + uint32 i, num = sizeof(target_sym_map) / sizeof(SymbolMap); + for (i = 0; i < num; i++) { + uint32 *p = (uint32 *)plt; + /* ldr pc, [pc] */ + *p++ = 0xe59ff000; + /* nop */ + *p++ = 0xe1a00000; + /* symbol addr */ + *p++ = (uint32)(uintptr_t)target_sym_map[i].symbol_addr; + plt += get_plt_item_size(); + } +} + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { + case R_ARM_CALL: + case R_ARM_JMP24: + { + intptr_t result; + int32 RESULT_MASK = 0x03FFFFFE; + int32 insn = *(int32 *)(target_section_addr + reloc_offset); + /* Initial addend: sign_extend(insn[23:0] << 2) */ + int32 initial_addend = + ((insn & 0xFFFFFF) << 2) | ((insn & 0x800000) ? 0xFC000000 : 0); + + CHECK_RELOC_OFFSET(sizeof(int32)); + + if (symbol_index < 0) { + /* Symbol address itself is an AOT function. + * Apply relocation with the symbol directly. + * Suppose the symbol address is in +-32MB relative + * to the relocation address. + */ + /* operation: ((S + A) | T) - P where S is symbol address and T + * is 0 */ + result = + (intptr_t)((uintptr_t)symbol_addr + (intptr_t)reloc_addend + - (uintptr_t)(target_section_addr + + reloc_offset)); + } + else { + if (reloc_addend > 0) { + set_error_buf( + error_buf, error_buf_size, + "AOT module load failed: relocate to plt table " + "with reloc addend larger than 0 is unsupported."); + return false; + } + + /* Symbol address is not an AOT function, + * but a function of runtime or native. Its address is + * beyond of the +-32MB space. Apply relocation with + * the PLT which branch to the target symbol address. + */ + /* operation: ((S + A) | T) - P where S is PLT address and T is + * 0 */ + uint8 *plt = (uint8 *)module->code + module->code_size + - get_plt_table_size() + + get_plt_item_size() * symbol_index; + result = (intptr_t)((uintptr_t)plt + (intptr_t)reloc_addend + - (uintptr_t)(target_section_addr + + reloc_offset)); + } + + result += initial_addend; + + /* Check overflow: +-32MB */ + if (result > (32 * BH_MB) || result < (-32 * BH_MB)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "target address out of range."); + return false; + } + + *(int32 *)(target_section_addr + reloc_offset) = + (int32)((insn & 0xff000000) + | (((int32)result & RESULT_MASK) >> 2)); + break; + } + case R_ARM_ABS32: + { + intptr_t initial_addend; + /* (S + A) | T where T is 0 */ + CHECK_RELOC_OFFSET(sizeof(void *)); + initial_addend = + *(intptr_t *)(target_section_addr + (uint32)reloc_offset); + *(uintptr_t *)(target_section_addr + reloc_offset) = + (uintptr_t)symbol_addr + initial_addend + + (intptr_t)reloc_addend; + break; + } + case R_ARM_MOVW_ABS_NC: + case R_ARM_MOVT_ABS: + { + uintptr_t *loc; + uintptr_t addr; + CHECK_RELOC_OFFSET(sizeof(void *)); + loc = (uintptr_t *)(target_section_addr + (uint32)reloc_offset); + addr = (uintptr_t)symbol_addr + (intptr_t)reloc_addend; + if (reloc_type == R_ARM_MOVT_ABS) { + addr >>= 16; + } + *loc = ((*loc) & 0xfff0f000) | ((addr << 4) & 0x000f0000) + | (addr & 0x00000fff); + break; + } + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %d.", + reloc_type); + return false; + } + + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_mips.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_mips.c new file mode 100644 index 0000000..f9f06a0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_mips.c @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#define R_MIPS_32 2 /* Direct 32 bit */ +#define R_MIPS_26 4 /* Direct 26 bit shifted */ + +/* clang-format off */ +static SymbolMap target_sym_map[] = { + REG_COMMON_SYMBOLS +}; +/* clang-format on */ + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + snprintf(target_buf, target_buf_size, "mips"); +} + +static uint32 +get_plt_item_size() +{ + return 0; +} + +void +init_plt_table(uint8 *plt) +{ + (void)plt; +} + +uint32 +get_plt_table_size() +{ + return get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { + /* TODO: implement relocation for mips */ + case R_MIPS_26: + case R_MIPS_32: + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %d.", + reloc_type); + return false; + } + + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_riscv.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_riscv.c new file mode 100644 index 0000000..31830f7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_riscv.c @@ -0,0 +1,537 @@ +/* + * Copyright (C) 2021 XiaoMi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#define R_RISCV_32 1 +#define R_RISCV_64 2 +#define R_RISCV_CALL 18 +#define R_RISCV_CALL_PLT 19 +#define R_RISCV_PCREL_HI20 23 +#define R_RISCV_PCREL_LO12_I 24 +#define R_RISCV_PCREL_LO12_S 25 +#define R_RISCV_HI20 26 +#define R_RISCV_LO12_I 27 +#define R_RISCV_LO12_S 28 + +#define RV_OPCODE_SW 0x23 + +#undef NEED_SOFT_FP +#undef NEED_SOFT_DP +#undef NEED_SOFT_I32_MUL +#undef NEED_SOFT_I32_DIV +#undef NEED_SOFT_I64_MUL +#undef NEED_SOFT_I64_DIV + +#ifdef __riscv_flen +#if __riscv_flen == 32 +#define NEED_SOFT_DP +#endif +#else +#define NEED_SOFT_FP +#define NEED_SOFT_DP +#endif + +#ifndef __riscv_mul +#define NEED_SOFT_I32_MUL +#define NEED_SOFT_I64_MUL +#elif __riscv_xlen == 32 +#define NEED_SOFT_I64_MUL +#endif + +#ifndef __riscv_div +#define NEED_SOFT_I32_DIV +#define NEED_SOFT_I64_DIV +#elif __riscv_xlen == 32 +#define NEED_SOFT_I64_DIV +#endif + +/* clang-format off */ +void __adddf3(); +void __addsf3(); +void __divdf3(); +void __divdi3(); +void __divsf3(); +void __divsi3(); +void __eqdf2(); +void __eqsf2(); +void __extendsfdf2(); +void __fixdfdi(); +void __fixdfsi(); +void __fixsfdi(); +void __fixsfsi(); +void __fixunsdfdi(); +void __fixunsdfsi(); +void __fixunssfdi(); +void __fixunssfsi(); +void __floatdidf(); +void __floatdisf(); +void __floatsidf(); +void __floatsisf(); +void __floatundidf(); +void __floatundisf(); +void __floatunsidf(); +void __floatunsisf(); +void __gedf2(); +void __gesf2(); +void __gtdf2(); +void __gtsf2(); +void __ledf2(); +void __lesf2(); +void __ltdf2(); +void __ltsf2(); +void __moddi3(); +void __modsi3(); +void __muldf3(); +void __muldi3(); +void __mulsf3(); +void __mulsi3(); +void __nedf2(); +void __negdf2(); +void __negsf2(); +void __nesf2(); +void __subdf3(); +void __subsf3(); +void __truncdfsf2(); +void __udivdi3(); +void __udivsi3(); +void __umoddi3(); +void __umodsi3(); +void __unorddf2(); +void __unordsf2(); +/* clang-format on */ + +static SymbolMap target_sym_map[] = { + /* clang-format off */ + REG_COMMON_SYMBOLS +#ifdef NEED_SOFT_FP + REG_SYM(__addsf3), + REG_SYM(__divsf3), + REG_SYM(__eqsf2), + REG_SYM(__fixsfdi), + REG_SYM(__fixunssfdi), + REG_SYM(__fixunssfsi), + REG_SYM(__floatsidf), + REG_SYM(__gesf2), + REG_SYM(__gtsf2), + REG_SYM(__lesf2), + REG_SYM(__mulsf3), + REG_SYM(__negsf2), + REG_SYM(__nesf2), + REG_SYM(__subsf3), + REG_SYM(__unordsf2), +#elif __riscv_xlen == 32 + /* rv32f, support FP instruction but need soft routines + * to convert float and long long + */ + REG_SYM(__floatundisf), +#endif +#ifdef NEED_SOFT_DP + REG_SYM(__adddf3), + REG_SYM(__divdf3), + REG_SYM(__eqdf2), + REG_SYM(__extendsfdf2), + REG_SYM(__fixdfdi), + REG_SYM(__fixunsdfdi), + REG_SYM(__fixunsdfsi), + REG_SYM(__floatdidf), + REG_SYM(__floatsidf), + REG_SYM(__floatundidf), + REG_SYM(__floatunsidf), + REG_SYM(__gedf2), + REG_SYM(__gtdf2), + REG_SYM(__ledf2), + REG_SYM(__muldf3), + REG_SYM(__nedf2), + REG_SYM(__negdf2), + REG_SYM(__subdf3), + REG_SYM(__truncdfsf2), + REG_SYM(__unorddf2), +#elif __riscv_xlen == 32 + /* rv32d, support DP instruction but need soft routines + * to convert double and long long + */ + REG_SYM(__fixdfdi), + REG_SYM(__floatundidf), +#endif +#ifdef NEED_SOFT_I32_MUL + REG_SYM(__mulsi3), +#endif +#ifdef NEED_SOFT_I32_DIV + REG_SYM(__divsi3), + REG_SYM(__modsi3), + REG_SYM(__udivsi3), + REG_SYM(__umodsi3), +#endif +#ifdef NEED_SOFT_I64_MUL + REG_SYM(__muldi3), +#endif +#ifdef NEED_SOFT_I64_DIV + REG_SYM(__divdi3), + REG_SYM(__moddi3), + REG_SYM(__udivdi3), + REG_SYM(__umoddi3), +#endif + /* clang-format on */ +}; + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + snprintf(target_buf, target_buf_size, "riscv"); +} + +uint32 +get_plt_item_size() +{ +#if __riscv_xlen == 64 + /* auipc + ld + jalr + nop + addr */ + return 20; +#else + return 0; +#endif +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +/* Get a val from given address */ +static uint32 +rv_get_val(uint16 *addr) +{ + uint32 ret; + ret = *addr | (*(addr + 1)) << 16; + return ret; +} + +/* Set a val to given address */ +static void +rv_set_val(uint16 *addr, uint32 val) +{ + *addr = (val & 0xffff); + *(addr + 1) = (val >> 16); + +#ifdef __riscv_zifencei + __asm__ volatile("fence.i"); +#else + __asm__ volatile("fence"); +#endif +} + +/* Add a val to given address */ +static void +rv_add_val(uint16 *addr, uint32 val) +{ + uint32 cur = rv_get_val(addr); + rv_set_val(addr, cur + val); +} + +/** + * Get imm_hi and imm_lo from given integer + * + * @param imm given integer, signed 32bit + * @param imm_hi signed 20bit + * @param imm_lo signed 12bit + * + */ +static void +rv_calc_imm(int32 imm, int32 *imm_hi, int32 *imm_lo) +{ + int32 lo; + int32 hi = imm / 4096; + int32 r = imm % 4096; + + if (2047 < r) { + hi++; + } + else if (r < -2048) { + hi--; + } + + lo = imm - (hi * 4096); + + *imm_lo = lo; + *imm_hi = hi; +} + +uint32 +get_plt_table_size() +{ + return get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); +} + +void +init_plt_table(uint8 *plt) +{ +#if __riscv_xlen == 64 + uint32 i, num = sizeof(target_sym_map) / sizeof(SymbolMap); + uint8 *p; + + for (i = 0; i < num; i++) { + p = plt; + /* auipc t1, 0 */ + *(uint16 *)p = 0x0317; + p += 2; + *(uint16 *)p = 0x0000; + p += 2; + /* ld t1, 8(t1) */ + *(uint16 *)p = 0x3303; + p += 2; + *(uint16 *)p = 0x00C3; + p += 2; + /* jr t1 */ + *(uint16 *)p = 0x8302; + p += 2; + /* nop */ + *(uint16 *)p = 0x0001; + p += 2; + bh_memcpy_s(p, 8, &target_sym_map[i].symbol_addr, 8); + p += 8; + plt += get_plt_item_size(); + } +#endif +} + +typedef struct RelocTypeStrMap { + uint32 reloc_type; + char *reloc_str; +} RelocTypeStrMap; + +#define RELOC_TYPE_MAP(reloc_type) \ + { \ + reloc_type, #reloc_type \ + } + +static RelocTypeStrMap reloc_type_str_maps[] = { + RELOC_TYPE_MAP(R_RISCV_32), RELOC_TYPE_MAP(R_RISCV_64), + RELOC_TYPE_MAP(R_RISCV_CALL), RELOC_TYPE_MAP(R_RISCV_CALL_PLT), + RELOC_TYPE_MAP(R_RISCV_PCREL_HI20), RELOC_TYPE_MAP(R_RISCV_PCREL_LO12_I), + RELOC_TYPE_MAP(R_RISCV_PCREL_LO12_S), RELOC_TYPE_MAP(R_RISCV_HI20), + RELOC_TYPE_MAP(R_RISCV_LO12_I), RELOC_TYPE_MAP(R_RISCV_LO12_S), +}; + +static const char * +reloc_type_to_str(uint32 reloc_type) +{ + uint32 i; + + for (i = 0; i < sizeof(reloc_type_str_maps) / sizeof(RelocTypeStrMap); + i++) { + if (reloc_type_str_maps[i].reloc_type == reloc_type) + return reloc_type_str_maps[i].reloc_str; + } + + return "Unknown_Reloc_Type"; +} + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + int32 val, imm_hi, imm_lo, insn; + uint8 *addr = target_section_addr + reloc_offset; + char buf[128]; + + switch (reloc_type) { + case R_RISCV_32: + { + uint32 val_32 = + (uint32)((uintptr_t)symbol_addr + (intptr_t)reloc_addend); + + CHECK_RELOC_OFFSET(sizeof(uint32)); + if (val_32 != ((uintptr_t)symbol_addr + (intptr_t)reloc_addend)) { + goto fail_addr_out_of_range; + } + + rv_set_val((uint16 *)addr, val_32); + break; + } + +#if __riscv_xlen == 64 + case R_RISCV_64: + { + uint64 val_64 = + (uint64)((intptr_t)symbol_addr + (intptr_t)reloc_addend); + + CHECK_RELOC_OFFSET(sizeof(uint64)); + if (val_64 + != (uint64)((intptr_t)symbol_addr + (intptr_t)reloc_addend)) { + goto fail_addr_out_of_range; + } + + bh_memcpy_s(addr, 8, &val_64, 8); +#ifdef __riscv_zifencei + __asm__ volatile("fence.i"); +#else + __asm__ volatile("fence"); +#endif + break; + } +#endif + + case R_RISCV_CALL: + case R_RISCV_CALL_PLT: + case R_RISCV_PCREL_HI20: /* S + A - P */ + { + val = (int32)(intptr_t)((uint8 *)symbol_addr + reloc_addend - addr); + + CHECK_RELOC_OFFSET(sizeof(uint32)); + if (val != (intptr_t)((uint8 *)symbol_addr + reloc_addend - addr)) { + if (symbol_index >= 0) { + /* Call runtime function by plt code */ + symbol_addr = (uint8 *)module->code + module->code_size + - get_plt_table_size() + + get_plt_item_size() * symbol_index; + val = (int32)(intptr_t)((uint8 *)symbol_addr - addr); + } + } + + if (val != (intptr_t)((uint8 *)symbol_addr + reloc_addend - addr)) { + goto fail_addr_out_of_range; + } + + rv_calc_imm(val, &imm_hi, &imm_lo); + + rv_add_val((uint16 *)addr, (imm_hi << 12)); + if ((rv_get_val((uint16 *)(addr + 4)) & 0x7f) == RV_OPCODE_SW) { + /* Adjust imm for SW : S-type */ + val = (((int32)imm_lo >> 5) << 25) + + (((int32)imm_lo & 0x1f) << 7); + + rv_add_val((uint16 *)(addr + 4), val); + } + else { + /* Adjust imm for MV(ADDI)/JALR : I-type */ + rv_add_val((uint16 *)(addr + 4), ((int32)imm_lo << 20)); + } + break; + } + + case R_RISCV_HI20: /* S + A */ + { + val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend); + + CHECK_RELOC_OFFSET(sizeof(uint32)); + if (val != ((intptr_t)symbol_addr + (intptr_t)reloc_addend)) { + goto fail_addr_out_of_range; + } + + insn = rv_get_val((uint16 *)addr); + rv_calc_imm(val, &imm_hi, &imm_lo); + insn = (insn & 0x00000fff) | (imm_hi << 12); + rv_set_val((uint16 *)addr, insn); + break; + } + + case R_RISCV_PCREL_LO12_I: /* S - P */ + case R_RISCV_PCREL_LO12_S: /* S - P */ + { + /* Already handled in R_RISCV_PCREL_HI20, it should be skipped for + * most cases. But it is still needed for some special cases, e.g. + * ``` + * label: + * auipc t0, %pcrel_hi(symbol) # R_RISCV_PCREL_HI20 (symbol) + * lui t1, 1 + * lw t2, t0, %pcrel_lo(label) # R_RISCV_PCREL_LO12_I (label) + * add t2, t2, t1 + * sw t2, t0, %pcrel_lo(label) # R_RISCV_PCREL_LO12_S (label) + * ``` + * In this case, the R_RISCV_PCREL_LO12_I/S relocation should be + * handled after R_RISCV_PCREL_HI20 relocation. + * + * So, if the R_RISCV_PCREL_LO12_I/S relocation is not followed by + * R_RISCV_PCREL_HI20 relocation, it should be handled here but + * not implemented yet. + */ + + if ((uintptr_t)addr - (uintptr_t)symbol_addr + - (uintptr_t)reloc_addend + != 4) { + goto fail_addr_out_of_range; + } + break; + } + + case R_RISCV_LO12_I: /* S + A */ + { + + val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend); + + CHECK_RELOC_OFFSET(sizeof(uint32)); + + if (val != (intptr_t)symbol_addr + (intptr_t)reloc_addend) { + goto fail_addr_out_of_range; + } + + addr = target_section_addr + reloc_offset; + insn = rv_get_val((uint16 *)addr); + rv_calc_imm(val, &imm_hi, &imm_lo); + insn = (insn & 0x000fffff) | (imm_lo << 20); + rv_set_val((uint16 *)addr, insn); + break; + } + + case R_RISCV_LO12_S: + { + val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend); + + CHECK_RELOC_OFFSET(sizeof(uint32)); + if (val != ((intptr_t)symbol_addr + (intptr_t)reloc_addend)) { + goto fail_addr_out_of_range; + } + + addr = target_section_addr + reloc_offset; + rv_calc_imm(val, &imm_hi, &imm_lo); + val = (((int32)imm_lo >> 5) << 25) + (((int32)imm_lo & 0x1f) << 7); + rv_add_val((uint16 *)addr, val); + break; + } + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %" PRIu32 ".", + reloc_type); + return false; + } + + return true; + +fail_addr_out_of_range: + snprintf(buf, sizeof(buf), + "AOT module load failed: " + "relocation truncated to fit %s failed.", + reloc_type_to_str(reloc_type)); + set_error_buf(error_buf, error_buf_size, buf); + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_thumb.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_thumb.c new file mode 100644 index 0000000..2661486 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_thumb.c @@ -0,0 +1,431 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#define R_ARM_ABS32 2 /* Direct 32 bit */ +#define R_ARM_THM_CALL 10 /* PC relative (Thumb BL and ARMv5 Thumb BLX). */ +#define R_ARM_THM_JMP24 30 /* B.W */ +#define R_ARM_THM_MOVW_ABS_NC 47 +#define R_ARM_THM_MOVT_ABS 48 +#define R_ARM_THM_MOVW_PREL_NC 49 +#define R_ARM_THM_MOVT_PREL 50 + +/* clang-format off */ +void __adddf3(); +void __addsf3(); +void __aeabi_d2iz(); +void __aeabi_d2lz(); +void __aeabi_d2uiz(); +void __aeabi_d2ulz(); +void __aeabi_dadd(); +void __aeabi_dcmpge(); +void __aeabi_dcmpgt(); +void __aeabi_dcmple(); +void __aeabi_dcmplt(); +void __aeabi_dcmpun(); +void __aeabi_ddiv(); +void __aeabi_f2d(); +void __aeabi_f2iz(); +void __aeabi_f2lz(); +void __aeabi_f2ulz(); +void __aeabi_fcmpge(); +void __aeabi_fcmple(); +void __aeabi_fcmplt(); +void __aeabi_fcmpun(); +void __aeabi_i2d(); +void __aeabi_idiv(); +void __aeabi_idivmod(); +void __aeabi_l2d(); +void __aeabi_l2f(); +void __aeabi_ldivmod(); +void __aeabi_ui2d(); +void __aeabi_uidiv(); +void __aeabi_uidivmod(); +void __aeabi_ul2d(); +void __aeabi_ul2f(); +void __aeabi_uldivmod(); +void __ashldi3(); +void __clzsi2(); +void __divdf3(); +void __divdi3(); +void __divsi3(); +void __eqdf2(); +void __eqsf2(); +void __extendsfdf2(); +void __fixdfdi(); +void __fixdfsi(); +void __fixsfdi(); +void __fixunsdfdi(); +void __fixunsdfsi(); +void __fixunssfdi(); +void __floatdidf(); +void __floatdisf(); +void __floatsidf(); +void __floatsisf(); +void __floatundidf(); +void __floatundisf(); +void __floatunsidf(); +void __floatunsisf(); +void __gedf2(); +void __gesf2(); +void __gtdf2(); +void __gtsf2(); +void __ledf2(); +void __lesf2(); +void __lshrdi3(); +void __ltdf2(); +void __ltsf2(); +void __moddi3(); +void __modsi3(); +void __muldf3(); +void __muldi3(); +void __mulsf3(); +void __nedf2(); +void __nesf2(); +void __subdf3(); +void __subsf3(); +void __truncdfsf2(); +void __udivdi3(); +void __udivmoddi4(); +void __udivsi3(); +void __umoddi3(); +void __umodsi3(); +void __unorddf2(); +void __unordsf2(); +/* clang-format on */ + +static SymbolMap target_sym_map[] = { + /* clang-format off */ + REG_COMMON_SYMBOLS + /* compiler-rt symbols that come from compiler(e.g. gcc) */ +#if __ARM_ARCH != 6 + REG_SYM(__adddf3), + REG_SYM(__addsf3), + REG_SYM(__divdf3), + REG_SYM(__extendsfdf2), + REG_SYM(__fixdfsi), + REG_SYM(__floatsidf), + REG_SYM(__floatsisf), + REG_SYM(__floatunsidf), + REG_SYM(__floatunsisf), + REG_SYM(__muldf3), + REG_SYM(__mulsf3), + REG_SYM(__subdf3), + REG_SYM(__subsf3), + REG_SYM(__truncdfsf2), + REG_SYM(__unorddf2), + REG_SYM(__unordsf2), +#endif + /* clang-format on */ + REG_SYM(__aeabi_d2iz), + REG_SYM(__aeabi_d2lz), + REG_SYM(__aeabi_d2uiz), + REG_SYM(__aeabi_d2ulz), + REG_SYM(__aeabi_dadd), + REG_SYM(__aeabi_dcmpge), + REG_SYM(__aeabi_dcmpgt), + REG_SYM(__aeabi_dcmple), + REG_SYM(__aeabi_dcmplt), + REG_SYM(__aeabi_dcmpun), + REG_SYM(__aeabi_ddiv), + REG_SYM(__aeabi_f2d), + REG_SYM(__aeabi_f2iz), + REG_SYM(__aeabi_f2lz), + REG_SYM(__aeabi_f2ulz), + REG_SYM(__aeabi_fcmpge), + REG_SYM(__aeabi_fcmple), + REG_SYM(__aeabi_fcmplt), + REG_SYM(__aeabi_fcmpun), + REG_SYM(__aeabi_i2d), + REG_SYM(__aeabi_idiv), + REG_SYM(__aeabi_idivmod), + REG_SYM(__aeabi_l2d), + REG_SYM(__aeabi_l2f), + REG_SYM(__aeabi_ldivmod), + REG_SYM(__aeabi_ui2d), + REG_SYM(__aeabi_uidiv), + REG_SYM(__aeabi_uidivmod), + REG_SYM(__aeabi_ul2d), + REG_SYM(__aeabi_ul2f), + REG_SYM(__aeabi_uldivmod), + REG_SYM(__ashldi3), + REG_SYM(__clzsi2), + REG_SYM(__divdi3), + REG_SYM(__divsi3), + REG_SYM(__eqdf2), + REG_SYM(__eqsf2), + REG_SYM(__fixdfdi), + REG_SYM(__fixsfdi), + REG_SYM(__fixunsdfdi), + REG_SYM(__fixunsdfsi), + REG_SYM(__fixunssfdi), + REG_SYM(__floatdidf), + REG_SYM(__floatdisf), + REG_SYM(__floatundidf), + REG_SYM(__floatundisf), + REG_SYM(__gedf2), + REG_SYM(__gesf2), + REG_SYM(__gtdf2), + REG_SYM(__gtsf2), + REG_SYM(__ledf2), + REG_SYM(__lesf2), + REG_SYM(__lshrdi3), + REG_SYM(__ltdf2), + REG_SYM(__ltsf2), + REG_SYM(__moddi3), + REG_SYM(__modsi3), + REG_SYM(__muldi3), + REG_SYM(__nedf2), + REG_SYM(__nesf2), + REG_SYM(__udivdi3), + REG_SYM(__udivmoddi4), + REG_SYM(__udivsi3), + REG_SYM(__umoddi3), + REG_SYM(__umodsi3), +}; + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +#define BUILD_TARGET_THUMB_V4T "thumbv4t" +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + const char *s = BUILD_TARGET; + size_t s_size = sizeof(BUILD_TARGET); + char *d = target_buf; + + /* Set to "thumbv4t" by default if sub version isn't specified */ + if (strcmp(s, "THUMB") == 0) { + s = BUILD_TARGET_THUMB_V4T; + s_size = sizeof(BUILD_TARGET_THUMB_V4T); + } + if (target_buf_size < s_size) { + s_size = target_buf_size; + } + while (--s_size) { + if (*s >= 'A' && *s <= 'Z') + *d++ = *s++ + 'a' - 'A'; + else + *d++ = *s++; + } + /* Ensure the string is null byte ('\0') terminated */ + *d = '\0'; +} +#undef BUILD_TARGET_THUMB_V4T + +uint32 +get_plt_item_size() +{ + /* 16 bytes instructions and 4 bytes symbol address */ + return 20; +} + +uint32 +get_plt_table_size() +{ + return get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); +} + +void +init_plt_table(uint8 *plt) +{ + uint32 i, num = sizeof(target_sym_map) / sizeof(SymbolMap); + for (i = 0; i < num; i++) { + uint16 *p = (uint16 *)plt; + /* nop */ + *p++ = 0xbf00; + /* push {r4} */ + *p++ = 0xb410; + /* add r4, pc, #8 */ + *p++ = 0xa402; + /* ldr r4, [r4, #0] */ + *p++ = 0x6824; + /* mov ip, r4 */ + *p++ = 0x46a4; + /* pop {r4} */ + *p++ = 0xbc10; + /* mov pc, ip */ + *p++ = 0x46e7; + /* nop */ + *p++ = 0xbf00; + /* symbol addr */ + *(uint32 *)p = (uint32)(uintptr_t)target_sym_map[i].symbol_addr; + plt += get_plt_item_size(); + } +} + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { + case R_ARM_THM_CALL: + case R_ARM_THM_JMP24: + { + int32 RESULT_MASK = 0x01FFFFFE; + int32 result, result_masked; + int16 *reloc_addr; + int32 initial_addend_0, initial_addend_1, initial_addend; + bool sign; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + reloc_addr = (int16 *)(target_section_addr + reloc_offset); + initial_addend_0 = (*reloc_addr) & 0x7FF; + initial_addend_1 = (*(reloc_addr + 1)) & 0x7FF; + sign = (initial_addend_0 & 0x400) ? true : false; + initial_addend = (initial_addend_0 << 12) | (initial_addend_1 << 1) + | (sign ? 0xFF800000 : 0); + + if (symbol_index < 0) { + /* Symbol address itself is an AOT function. + * Apply relocation with the symbol directly. + * Suppose the symbol address is in +-4MB relative + * to the relocation address. + */ + /* operation: ((S + A) | T) - P where S is symbol address + and T is 1 */ + result = + (int32)(((intptr_t)((uintptr_t)symbol_addr + + (intptr_t)reloc_addend) + | 1) + - (intptr_t)(target_section_addr + reloc_offset)); + } + else { + if (reloc_addend > 0) { + set_error_buf( + error_buf, error_buf_size, + "AOT module load failed: relocate to plt table " + "with reloc addend larger than 0 is unsupported."); + return false; + } + + /* Symbol address is not an AOT function, + * but a function of runtime or native. Its address is + * beyond of the +-4MB space. Apply relocation with + * the PLT which branch to the target symbol address. + */ + /* operation: ((S + A) | T) - P where S is PLT address + and T is 1 */ + uint8 *plt = (uint8 *)module->code + module->code_size + - get_plt_table_size() + + get_plt_item_size() * symbol_index + 1; + result = + (int32)(((intptr_t)plt | 1) + - (intptr_t)(target_section_addr + reloc_offset)); + } + + result += initial_addend; + + /* Check overflow: +-4MB */ + if (result > (4 * BH_MB) || result < (-4 * BH_MB)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "target address out of range."); + return false; + } + + result_masked = (int32)result & RESULT_MASK; + initial_addend_0 = (result_masked >> 12) & 0x7FF; + initial_addend_1 = (result_masked >> 1) & 0x7FF; + + *reloc_addr = (*reloc_addr & ~0x7FF) | initial_addend_0; + *(reloc_addr + 1) = (*(reloc_addr + 1) & ~0x7FF) | initial_addend_1; + break; + } + case R_ARM_ABS32: + { + intptr_t initial_addend; + /* (S + A) | T where T is 0 */ + CHECK_RELOC_OFFSET(sizeof(void *)); + initial_addend = + *(intptr_t *)(target_section_addr + (uint32)reloc_offset); + *(uintptr_t *)(target_section_addr + reloc_offset) = + (uintptr_t)symbol_addr + initial_addend + + (intptr_t)reloc_addend; + break; + } + case R_ARM_THM_MOVW_ABS_NC: + case R_ARM_THM_MOVT_ABS: + case R_ARM_THM_MOVW_PREL_NC: + case R_ARM_THM_MOVT_PREL: + { + uint16 upper = *(uint16 *)(target_section_addr + reloc_offset); + uint16 lower = *(uint16 *)(target_section_addr + reloc_offset + 2); + int32 offset; + + /* + * MOVT/MOVW instructions encoding in Thumb-2: + * + * i = upper[10] + * imm4 = upper[3:0] + * imm3 = lower[14:12] + * imm8 = lower[7:0] + * + * imm16 = imm4:i:imm3:imm8 + */ + + offset = ((upper & 0x000f) << 12) | ((upper & 0x0400) << 1) + | ((lower & 0x7000) >> 4) | (lower & 0x00ff); + offset = (offset ^ 0x8000) - 0x8000; + + offset += (symbol_addr + reloc_addend); + + if (reloc_type == R_ARM_THM_MOVT_PREL + || reloc_type == R_ARM_THM_MOVW_PREL_NC) + offset -= (int32)(target_section_addr + reloc_offset); + if (reloc_type == R_ARM_THM_MOVT_ABS + || reloc_type == R_ARM_THM_MOVT_PREL) + offset >>= 16; + + upper = (uint16)((upper & 0xfbf0) | ((offset & 0xf000) >> 12) + | ((offset & 0x0800) >> 1)); + lower = (uint16)((lower & 0x8f00) | ((offset & 0x0700) << 4) + | (offset & 0x00ff)); + + *(uint16 *)(target_section_addr + reloc_offset) = upper; + *(uint16 *)(target_section_addr + reloc_offset + 2) = lower; + break; + } + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %" PRId32 ".", + reloc_type); + return false; + } + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_x86_32.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_x86_32.c new file mode 100644 index 0000000..0a423c3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_x86_32.c @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +/* clang-format off */ +#if !defined(BH_PLATFORM_WINDOWS) +#define R_386_32 1 /* Direct 32 bit */ +#define R_386_PC32 2 /* PC relative 32 bit */ +#define R_386_PLT32 4 /* 32-bit address ProcedureLinkageTable */ +#define R_386_TLS_GD_32 24 /* Direct 32 bit for general dynamic + thread local data */ +#else +#define IMAGE_REL_I386_DIR32 6 /* The target's 32-bit VA */ +#define IMAGE_REL_I386_REL32 20 /* The 32-bit relative displacement + to the target */ +#endif +/* clang-format on */ + +#if !defined(_WIN32) && !defined(_WIN32_) +/* clang-format off */ +void __divdi3(); +void __udivdi3(); +void __moddi3(); +void __umoddi3(); +/* clang-format on */ +#else +#pragma function(floor) +#pragma function(ceil) + +static int64 +__divdi3(int64 a, int64 b) +{ + return a / b; +} + +static uint64 +__udivdi3(uint64 a, uint64 b) +{ + return a / b; +} + +static int64 +__moddi3(int64 a, int64 b) +{ + return a % b; +} + +static uint64 +__umoddi3(uint64 a, uint64 b) +{ + return a % b; +} +#endif + +static uint64 +__aulldiv(uint64 a, uint64 b) +{ + return a / b; +} + +/* clang-format off */ +static SymbolMap target_sym_map[] = { + REG_COMMON_SYMBOLS + /* compiler-rt symbols that come from compiler(e.g. gcc) */ + REG_SYM(__divdi3), + REG_SYM(__udivdi3), + REG_SYM(__moddi3), + REG_SYM(__umoddi3), + REG_SYM(__aulldiv) +}; +/* clang-format on */ + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + snprintf(target_buf, target_buf_size, "i386"); +} + +uint32 +get_plt_table_size() +{ + return 0; +} + +void +init_plt_table(uint8 *plt) +{ + (void)plt; +} + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { +#if !defined(BH_PLATFORM_WINDOWS) + case R_386_32: +#if WASM_ENABLE_STATIC_PGO != 0 + case R_386_TLS_GD_32: +#endif +#else + case IMAGE_REL_I386_DIR32: +#endif + { + intptr_t value; + + CHECK_RELOC_OFFSET(sizeof(void *)); + value = *(intptr_t *)(target_section_addr + (uint32)reloc_offset); + *(uintptr_t *)(target_section_addr + reloc_offset) = + (uintptr_t)symbol_addr + (intptr_t)reloc_addend + + value; /* S + A */ + break; + } + +#if !defined(BH_PLATFORM_WINDOWS) + /* + * Handle R_386_PLT32 like R_386_PC32 since it should be able to reach + * any 32 bit address + */ + case R_386_PLT32: + case R_386_PC32: +#else + case IMAGE_REL_I386_REL32: +#endif + { + int32 value; + + CHECK_RELOC_OFFSET(sizeof(void *)); + value = *(int32 *)(target_section_addr + (uint32)reloc_offset); + *(uint32 *)(target_section_addr + (uint32)reloc_offset) = + (uint32)((uintptr_t)symbol_addr + (intptr_t)reloc_addend + - (uintptr_t)(target_section_addr + + (uint32)reloc_offset) +#if defined(BH_PLATFORM_WINDOWS) + - sizeof(int32) +#endif + + value); /* S + A - P */ + break; + } + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %d.", + reloc_type); + return false; + } + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_x86_64.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_x86_64.c new file mode 100644 index 0000000..d1f5cb5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_x86_64.c @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#if !defined(BH_PLATFORM_WINDOWS) +#define R_X86_64_64 1 /* Direct 64 bit */ +#define R_X86_64_PC32 2 /* PC relative 32 bit signed */ +#define R_X86_64_PLT32 4 /* 32 bit PLT address */ +#define R_X86_64_GOTPCREL 9 /* 32 bit signed PC relative offset to GOT */ +#define R_X86_64_32 10 /* Direct 32 bit zero extended */ +#define R_X86_64_32S 11 /* Direct 32 bit sign extended */ +#define R_X86_64_PC64 24 /* PC relative 64 bit */ +#else +#ifndef IMAGE_REL_AMD64_ADDR64 +#define IMAGE_REL_AMD64_ADDR64 1 /* The 64-bit VA of the relocation target */ +#define IMAGE_REL_AMD64_ADDR32 2 /* The 32-bit VA of the relocation target */ +/* clang-format off */ +#define IMAGE_REL_AMD64_REL32 4 /* The 32-bit relative address from + the byte following the relocation*/ +/* clang-format on */ +#endif +#endif + +#if defined(BH_PLATFORM_WINDOWS) +#pragma function(floor) +#pragma function(ceil) +#pragma function(floorf) +#pragma function(ceilf) +#endif + +/* clang-format off */ +static SymbolMap target_sym_map[] = { + REG_COMMON_SYMBOLS +}; +/* clang-format on */ + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + snprintf(target_buf, target_buf_size, "x86_64"); +} + +static uint32 +get_plt_item_size() +{ + /* size of mov instruction and jmp instruction */ + return 12; +} + +uint32 +get_plt_table_size() +{ + uint32 size = + get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); + return size; +} + +void +init_plt_table(uint8 *plt) +{ + uint32 i, num = sizeof(target_sym_map) / sizeof(SymbolMap); + uint8 *p; + + for (i = 0; i < num; i++) { + p = plt; + /* mov symbol_addr, rax */ + *p++ = 0x48; + *p++ = 0xB8; + *(uint64 *)p = (uint64)(uintptr_t)target_sym_map[i].symbol_addr; + p += sizeof(uint64); + /* jmp rax */ + *p++ = 0xFF; + *p++ = 0xE0; + plt += get_plt_item_size(); + } +} + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { +#if !defined(BH_PLATFORM_WINDOWS) + case R_X86_64_64: +#else + case IMAGE_REL_AMD64_ADDR64: +#endif + { + intptr_t value; + + CHECK_RELOC_OFFSET(sizeof(void *)); + value = *(intptr_t *)(target_section_addr + (uint32)reloc_offset); + *(uintptr_t *)(target_section_addr + reloc_offset) = + (uintptr_t)symbol_addr + reloc_addend + value; /* S + A */ + break; + } +#if defined(BH_PLATFORM_WINDOWS) + case IMAGE_REL_AMD64_ADDR32: + { + int32 value; + uintptr_t target_addr; + + CHECK_RELOC_OFFSET(sizeof(void *)); + value = *(int32 *)(target_section_addr + (uint32)reloc_offset); + target_addr = (uintptr_t)symbol_addr + reloc_addend + value; + if ((int32)target_addr != target_addr) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "relocation truncated to fit " + "IMAGE_REL_AMD64_ADDR32 failed. " + "Try using wamrc with --size-level=1 option."); + return false; + } + + *(int32 *)(target_section_addr + reloc_offset) = (int32)target_addr; + break; + } +#endif +#if !defined(BH_PLATFORM_WINDOWS) + case R_X86_64_PC32: + case R_X86_64_GOTPCREL: /* GOT + G has been calculated as symbol_addr */ + { + intptr_t target_addr = (intptr_t) /* S + A - P */ + ((uintptr_t)symbol_addr + reloc_addend + - (uintptr_t)(target_section_addr + reloc_offset)); + + CHECK_RELOC_OFFSET(sizeof(int32)); + if ((int32)target_addr != target_addr) { + set_error_buf( + error_buf, error_buf_size, + "AOT module load failed: " + "relocation truncated to fit R_X86_64_PC32 failed. " + "Try using wamrc with --size-level=1 or 0 option."); + return false; + } + + *(int32 *)(target_section_addr + reloc_offset) = (int32)target_addr; + break; + } + case R_X86_64_PC64: + { + intptr_t target_addr = (intptr_t) /* S + A - P */ + ((uintptr_t)symbol_addr + reloc_addend + - (uintptr_t)(target_section_addr + reloc_offset)); + + CHECK_RELOC_OFFSET(sizeof(int64)); + *(int64 *)(target_section_addr + reloc_offset) = (int64)target_addr; + break; + } + case R_X86_64_32: + case R_X86_64_32S: + { + char buf[128]; + uintptr_t target_addr = /* S + A */ + (uintptr_t)symbol_addr + reloc_addend; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + if ((reloc_type == R_X86_64_32 + && (uint32)target_addr != (uint64)target_addr) + || (reloc_type == R_X86_64_32S + && (int32)target_addr != (int64)target_addr)) { + snprintf(buf, sizeof(buf), + "AOT module load failed: " + "relocation truncated to fit %s failed. " + "Try using wamrc with --size-level=1 or 0 option.", + reloc_type == R_X86_64_32 ? "R_X86_64_32" + : "R_X86_64_32S"); + set_error_buf(error_buf, error_buf_size, buf); + return false; + } + + *(int32 *)(target_section_addr + reloc_offset) = (int32)target_addr; + break; + } +#endif +#if !defined(BH_PLATFORM_WINDOWS) + case R_X86_64_PLT32: +#else + case IMAGE_REL_AMD64_REL32: +#endif + { + uint8 *plt; + intptr_t target_addr = 0; + + CHECK_RELOC_OFFSET(sizeof(int32)); + + if (symbol_index >= 0) { + plt = (uint8 *)module->code + module->code_size + - get_plt_table_size() + + get_plt_item_size() * symbol_index; + target_addr = (intptr_t) /* L + A - P */ + ((uintptr_t)plt + reloc_addend + - (uintptr_t)(target_section_addr + reloc_offset)); + } + else { + target_addr = (intptr_t) /* S + A - P */ + ((uintptr_t)symbol_addr + reloc_addend + - (uintptr_t)(target_section_addr + reloc_offset)); + } + +#if defined(BH_PLATFORM_WINDOWS) + target_addr -= sizeof(int32); +#endif + if ((int32)target_addr != target_addr) { + set_error_buf( + error_buf, error_buf_size, + "AOT module load failed: " + "relocation truncated to fit " +#if !defined(BH_PLATFORM_WINDOWS) + "R_X86_64_PLT32 failed. " +#else + "IMAGE_REL_AMD64_32 failed." +#endif + "Try using wamrc with --size-level=1 or 0 option."); + return false; + } + *(int32 *)(target_section_addr + reloc_offset) = (int32)target_addr; + break; + } + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %d.", + reloc_type); + return false; + } + + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_xtensa.c b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_xtensa.c new file mode 100644 index 0000000..a29c9f2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/arch/aot_reloc_xtensa.c @@ -0,0 +1,320 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_reloc.h" + +#define R_XTENSA_32 1 /* Direct 32 bit */ +#define R_XTENSA_SLOT0_OP 20 /* PC relative */ + +/* clang-format off */ +/* for soft-float */ +void __floatsidf(); +void __divdf3(); +void __ltdf2(); + +/* for mul32 */ +void __mulsi3(); +void __muldi3(); + +void __modsi3(); + +void __divdi3(); + +void __udivdi3(); +void __unorddf2(); +void __adddf3(); +void __eqdf2(); +void __muldf3(); +void __gedf2(); +void __ledf2(); +void __fixunsdfsi(); +void __floatunsidf(); +void __subdf3(); +void __nedf2(); +void __fixdfsi(); +void __moddi3(); +void __extendsfdf2(); +void __truncdfsf2(); +void __gtdf2(); +void __umoddi3(); +void __floatdidf(); +void __divsf3(); +void __fixdfdi(); +void __floatundidf(); +void __fixsfdi(); +void __fixunssfdi(); +void __fixunsdfdi(); +void __floatdisf(); +void __floatundisf(); + + +static SymbolMap target_sym_map[] = { + REG_COMMON_SYMBOLS + + /* API's for soft-float */ + /* TODO: only register these symbols when Floating-Point Coprocessor + * Option is not enabled */ + REG_SYM(__floatsidf), + REG_SYM(__divdf3), + REG_SYM(__ltdf2), + + /* API's for 32-bit integer multiply */ + /* TODO: only register these symbols when 32-bit Integer Multiply Option + * is not enabled */ + REG_SYM(__mulsi3), + REG_SYM(__muldi3), + + REG_SYM(__modsi3), + REG_SYM(__divdi3), + + REG_SYM(__udivdi3), + REG_SYM(__unorddf2), + REG_SYM(__adddf3), + REG_SYM(__eqdf2), + REG_SYM(__muldf3), + REG_SYM(__gedf2), + REG_SYM(__ledf2), + REG_SYM(__fixunsdfsi), + REG_SYM(__floatunsidf), + REG_SYM(__subdf3), + REG_SYM(__nedf2), + REG_SYM(__fixdfsi), + REG_SYM(__moddi3), + REG_SYM(__extendsfdf2), + REG_SYM(__truncdfsf2), + REG_SYM(__gtdf2), + REG_SYM(__umoddi3), + REG_SYM(__floatdidf), + REG_SYM(__divsf3), + REG_SYM(__fixdfdi), + REG_SYM(__floatundidf), + REG_SYM(__fixsfdi), + REG_SYM(__fixunssfdi), + REG_SYM(__fixunsdfdi), + REG_SYM(__floatdisf), + REG_SYM(__floatundisf), +}; +/* clang-format on */ + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +SymbolMap * +get_target_symbol_map(uint32 *sym_num) +{ + *sym_num = sizeof(target_sym_map) / sizeof(SymbolMap); + return target_sym_map; +} + +void +get_current_target(char *target_buf, uint32 target_buf_size) +{ + snprintf(target_buf, target_buf_size, "xtensa"); +} + +static uint32 +get_plt_item_size() +{ + return 0; +} + +void +init_plt_table(uint8 *plt) +{ + (void)plt; +} + +uint32 +get_plt_table_size() +{ + return get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); +} + +static bool +check_reloc_offset(uint32 target_section_size, uint64 reloc_offset, + uint32 reloc_data_size, char *error_buf, + uint32 error_buf_size) +{ + if (!(reloc_offset < (uint64)target_section_size + && reloc_offset + reloc_data_size <= (uint64)target_section_size)) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: invalid relocation offset."); + return false; + } + return true; +} + +/* + * CPU like esp32 can read and write data through the instruction bus, but only + * in a word aligned manner; non-word-aligned access will cause a CPU exception. + * This function uses a world aligned manner to write 16bit value to instruction + * addreess. + */ +static void +put_imm16_to_addr(int16 imm16, int16 *addr) +{ + int8 bytes[8]; + int32 *addr_aligned1, *addr_aligned2; + + addr_aligned1 = (int32 *)((intptr_t)addr & ~3); + + if ((intptr_t)addr % 4 != 3) { + *(int32 *)bytes = *addr_aligned1; + *(int16 *)(bytes + ((intptr_t)addr % 4)) = imm16; + *addr_aligned1 = *(int32 *)bytes; + } + else { + addr_aligned2 = (int32 *)(((intptr_t)addr + 3) & ~3); + *(int32 *)bytes = *addr_aligned1; + *(int32 *)(bytes + 4) = *addr_aligned2; + *(int16 *)(bytes + 3) = imm16; + memcpy(addr_aligned1, bytes, 8); + } +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +#if !defined(__packed) +/* + * Note: This version check is a bit relaxed. + * The __packed__ attribute has been there since gcc 2 era. + */ +#if __GNUC__ >= 3 +#define __packed __attribute__((__packed__)) +#endif +#endif + +typedef union { + struct l32r_le { + int8 other; + int16 imm16; + } __packed l; + + struct l32r_be { + int16 imm16; + int8 other; + } __packed b; +} l32r_insn_t; + +bool +apply_relocation(AOTModule *module, uint8 *target_section_addr, + uint32 target_section_size, uint64 reloc_offset, + int64 reloc_addend, uint32 reloc_type, void *symbol_addr, + int32 symbol_index, char *error_buf, uint32 error_buf_size) +{ + switch (reloc_type) { + case R_XTENSA_32: + { + uint8 *insn_addr = target_section_addr + reloc_offset; +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + insn_addr = os_get_dbus_mirror((void *)insn_addr); + bh_assert(insn_addr != NULL); +#endif + int32 initial_addend; + /* (S + A) */ + if ((intptr_t)insn_addr & 3) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "instruction address unaligned."); + return false; + } + CHECK_RELOC_OFFSET(4); + initial_addend = *(int32 *)insn_addr; + *(uintptr_t *)insn_addr = (uintptr_t)symbol_addr + initial_addend + + (intptr_t)reloc_addend; + break; + } + + case R_XTENSA_SLOT0_OP: + { + uint8 *insn_addr = target_section_addr + reloc_offset; + /* Currently only l32r instruction generates R_XTENSA_SLOT0_OP + * relocation */ + l32r_insn_t *l32r_insn = (l32r_insn_t *)insn_addr; + uint8 *reloc_addr; + int32 relative_offset /*, initial_addend */; + int16 imm16; + + CHECK_RELOC_OFFSET(3); /* size of l32r instruction */ + + /* + imm16 = is_little_endian() ? + l32r_insn->l.imm16 : l32r_insn->b.imm16; + initial_addend = (int32)imm16 << 2; + */ + + reloc_addr = + (uint8 *)((uintptr_t)symbol_addr + (intptr_t)reloc_addend); + + if ((intptr_t)reloc_addr & 3) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "relocation address unaligned."); + return false; + } + + relative_offset = + (int32)((intptr_t)reloc_addr + - (((intptr_t)insn_addr + 3) & ~(intptr_t)3)); + /* relative_offset += initial_addend; */ + + /* check relative offset boundary */ + if (relative_offset < -256 * BH_KB || relative_offset > -4) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "target address out of range.\n" + "Try using `wamrc --size-level=0` to generate " + ".literal island."); + return false; + } + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + insn_addr = os_get_dbus_mirror((void *)insn_addr); + bh_assert(insn_addr != NULL); + l32r_insn = (l32r_insn_t *)insn_addr; +#endif + imm16 = (int16)(relative_offset >> 2); + + /* write back the imm16 to the l32r instruction */ + + /* GCC >= 9 complains if we have a pointer that could be + * unaligned. This can happen because the struct is packed. + * These pragma are to suppress the warnings because the + * function put_imm16_to_addr already handles unaligned + * pointers correctly. */ +#if __GNUC__ >= 9 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Waddress-of-packed-member" +#endif + if (is_little_endian()) + put_imm16_to_addr(imm16, &l32r_insn->l.imm16); + else + put_imm16_to_addr(imm16, &l32r_insn->b.imm16); +#if __GNUC__ >= 9 +#pragma GCC diagnostic pop +#endif + break; + } + + default: + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, + "Load relocation section failed: " + "invalid relocation type %d.", + (int)reloc_type); + return false; + } + + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/LICENSE_NUTTX b/wasm-micro-runtime/core/iwasm/aot/debug/LICENSE_NUTTX new file mode 100644 index 0000000..43fa6ab --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/LICENSE_NUTTX @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/NOTICE_NUTTX b/wasm-micro-runtime/core/iwasm/aot/debug/NOTICE_NUTTX new file mode 100644 index 0000000..17227cb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/NOTICE_NUTTX @@ -0,0 +1,5 @@ +Apache NuttX +Copyright 2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/elf.h b/wasm-micro-runtime/core/iwasm/aot/debug/elf.h new file mode 100644 index 0000000..9bdad65 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/elf.h @@ -0,0 +1,368 @@ +/**************************************************************************** + * include/elf.h + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __INCLUDE_ELF_H +#define __INCLUDE_ELF_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#define EI_NIDENT 16 /* Size of e_ident[] */ + +/* NOTE: elf64.h and elf32.h refer EI_NIDENT defined above */ + +#include "elf64.h" +#include "elf32.h" + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/* Values for Elf_Ehdr::e_type */ + +#define ET_NONE 0 /* No file type */ +#define ET_REL 1 /* Relocatable file */ +#define ET_EXEC 2 /* Executable file */ +#define ET_DYN 3 /* Shared object file */ +#define ET_CORE 4 /* Core file */ +#define ET_LOPROC 0xff00 /* Processor-specific */ +#define ET_HIPROC 0xffff /* Processor-specific */ + +/* Values for Elf_Ehdr::e_machine (most of this were not included in the + * original SCO document but have been gleaned from elsewhere). + */ + +#define EM_NONE 0 /* No machine */ +#define EM_M32 1 /* AT&T WE 32100 */ +#define EM_SPARC 2 /* SPARC */ +#define EM_386 3 /* Intel 80386 */ +#define EM_68K 4 /* Motorola 68000 */ +#define EM_88K 5 /* Motorola 88000 */ +#define EM_486 6 /* Intel 486+ */ +#define EM_860 7 /* Intel 80860 */ +#define EM_MIPS 8 /* MIPS R3000 Big-Endian */ +#define EM_MIPS_RS4_BE 10 /* MIPS R4000 Big-Endian */ +#define EM_PARISC 15 /* HPPA */ +#define EM_SPARC32PLUS 18 /* Sun's "v8plus" */ +#define EM_PPC 20 /* PowerPC */ +#define EM_PPC64 21 /* PowerPC64 */ +#define EM_ARM 40 /* ARM */ +#define EM_SH 42 /* SuperH */ +#define EM_SPARCV9 43 /* SPARC v9 64-bit */ +#define EM_H8_300 46 +#define EM_IA_64 50 /* HP/Intel IA-64 */ +#define EM_X86_64 62 /* AMD x86-64 */ +#define EM_S390 22 /* IBM S/390 */ +#define EM_CRIS 76 /* Axis Communications 32-bit embedded processor */ +#define EM_V850 87 /* NEC v850 */ +#define EM_M32R 88 /* Renesas M32R */ +#define EM_XTENSA 94 /* Tensilica Xtensa */ +#define EM_RISCV 243 /* RISC-V */ +#define EM_ALPHA 0x9026 +#define EM_CYGNUS_V850 0x9080 +#define EM_CYGNUS_M32R 0x9041 +#define EM_S390_OLD 0xa390 +#define EM_FRV 0x5441 + +/* Values for Elf_Ehdr::e_version */ + +#define EV_NONE 0 /* Invalid version */ +#define EV_CURRENT 1 /* The current version */ + +/* Table 2. Ehe ELF identifier */ + +#define EI_MAG0 0 /* File identification */ +#define EI_MAG1 1 +#define EI_MAG2 2 +#define EI_MAG3 3 +#define EI_CLASS 4 /* File class */ +#define EI_DATA 5 /* Data encoding */ +#define EI_VERSION 6 /* File version */ +#define EI_OSABI 7 /* OS ABI */ +#define EI_PAD 8 /* Start of padding bytes */ + +/* EI_NIDENT is defined in "Included Files" section */ + +#define EI_MAGIC_SIZE 4 +#define EI_MAGIC \ + { \ + 0x7f, 'E', 'L', 'F' \ + } + +#define ELFMAG0 0x7f /* EI_MAG */ +#define ELFMAG1 'E' +#define ELFMAG2 'L' +#define ELFMAG3 'F' +#define ELFMAG "\177ELF" + +/* Table 3. Values for EI_CLASS */ + +#define ELFCLASSNONE 0 /* Invalid class */ +#define ELFCLASS32 1 /* 32-bit objects */ +#define ELFCLASS64 2 /* 64-bit objects */ + +/* Table 4. Values for EI_DATA */ + +#define ELFDATANONE 0 /* Invalid data encoding */ +#define ELFDATA2LSB \ + 1 /* Least significant byte occupying the lowest address \ + */ +#define ELFDATA2MSB 2 /* Most significant byte occupying the lowest address */ + +/* Table 6. Values for EI_OSABI */ + +#define ELFOSABI_NONE 0 /* UNIX System V ABI */ +#define ELFOSABI_SYSV 0 /* Alias. */ +#define ELFOSABI_HPUX 1 /* HP-UX */ +#define ELFOSABI_NETBSD 2 /* NetBSD. */ +#define ELFOSABI_GNU 3 /* Object uses GNU ELF extensions. */ +#define ELFOSABI_LINUX ELFOSABI_GNU +/* Compatibility alias. */ +#define ELFOSABI_SOLARIS 6 /* Sun Solaris. */ +#define ELFOSABI_AIX 7 /* IBM AIX. */ +#define ELFOSABI_IRIX 8 /* SGI Irix. */ +#define ELFOSABI_FREEBSD 9 /* FreeBSD. */ +#define ELFOSABI_TRU64 10 /* Compaq TRU64 UNIX. */ +#define ELFOSABI_MODESTO 11 /* Novell Modesto. */ +#define ELFOSABI_OPENBSD 12 /* OpenBSD. */ +#define ELFOSABI_ARM_AEABI 64 /* ARM EABI */ +#define ELFOSABI_ARM 97 /* ARM */ +#define ELFOSABI_STANDALONE 255 /* Standalone (embedded) application */ + +#ifndef ELF_OSABI +#define ELF_OSABI ELFOSABI_NONE +#endif + +/* Table 7: Special Section Indexes */ + +#define SHN_UNDEF 0 +#define SHN_LOPROC 0xff00 +#define SHN_HIPROC 0xff1f +#define SHN_ABS 0xfff1 +#define SHN_COMMON 0xfff2 + +/* Figure 4-9: Section Types, sh_type */ + +#define SHT_NULL 0 +#define SHT_PROGBITS 1 +#define SHT_SYMTAB 2 +#define SHT_STRTAB 3 +#define SHT_RELA 4 +#define SHT_HASH 5 +#define SHT_DYNAMIC 6 +#define SHT_NOTE 7 +#define SHT_NOBITS 8 +#define SHT_REL 9 +#define SHT_SHLIB 10 +#define SHT_DYNSYM 11 +#define SHT_LOPROC 0x70000000 +#define SHT_HIPROC 0x7fffffff +#define SHT_LOUSER 0x80000000 +#define SHT_HIUSER 0xffffffff + +/* Figure 4-11: Section Attribute Flags, sh_flags */ + +#define SHF_WRITE 1 +#define SHF_ALLOC 2 +#define SHF_EXECINSTR 4 +#define SHF_MASKPROC 0xf0000000 + +/* Figure 4-16: Symbol Binding, ELF_ST_BIND */ + +#define STB_LOCAL 0 +#define STB_GLOBAL 1 +#define STB_WEAK 2 +#define STB_LOPROC 13 +#define STB_HIPROC 15 + +/* Figure 4-17: Symbol Types, ELF_ST_TYPE */ + +#define STT_NOTYPE 0 +#define STT_OBJECT 1 +#define STT_FUNC 2 +#define STT_SECTION 3 +#define STT_FILE 4 +#define STT_LOPROC 13 +#define STT_HIPROC 15 + +/* Figure 5-2: Segment Types, p_type */ + +#define PT_NULL 0 +#define PT_LOAD 1 +#define PT_DYNAMIC 2 +#define PT_INTERP 3 +#define PT_NOTE 4 +#define PT_SHLIB 5 +#define PT_PHDR 6 +#define PT_LOPROC 0x70000000 +#define PT_HIPROC 0x7fffffff + +/* Figure 5-3: Segment Flag Bits, p_flags */ + +#define PF_X 1 /* Execute */ +#define PF_W 2 /* Write */ +#define PF_R 4 /* Read */ +#define PF_MASKPROC 0xf0000000 /* Unspecified */ + +/* Figure 5-10: Dynamic Array Tags, d_tag */ + +#define DT_NULL 0 /* d_un=ignored */ +#define DT_NEEDED 1 /* d_un=d_val */ +#define DT_PLTRELSZ 2 /* d_un=d_val */ +#define DT_PLTGOT 3 /* d_un=d_ptr */ +#define DT_HASH 4 /* d_un=d_ptr */ +#define DT_STRTAB 5 /* d_un=d_ptr */ +#define DT_SYMTAB 6 /* d_un=d_ptr */ +#define DT_RELA 7 /* d_un=d_ptr */ +#define DT_RELASZ 8 /* d_un=d_val */ +#define DT_RELAENT 9 /* d_un=d_val */ +#define DT_STRSZ 10 /* d_un=d_val */ +#define DT_SYMENT 11 /* d_un=d_val */ +#define DT_INIT 12 /* d_un=d_ptr */ +#define DT_FINI 13 /* d_un=d_ptr */ +#define DT_SONAME 14 /* d_un=d_val */ +#define DT_RPATH 15 /* d_un=d_val */ +#define DT_SYMBOLIC 16 /* d_un=ignored */ +#define DT_REL 17 /* d_un=d_ptr */ +#define DT_RELSZ 18 /* d_un=d_val */ +#define DT_RELENT 19 /* d_un=d_val */ +#define DT_PLTREL 20 /* d_un=d_val */ +#define DT_DEBUG 21 /* d_un=d_ptr */ +#define DT_TEXTREL 22 /* d_un=ignored */ +#define DT_JMPREL 23 /* d_un=d_ptr */ +#define DT_BINDNOW 24 /* d_un=ignored */ +#define DT_LOPROC 0x70000000 /* d_un=unspecified */ +#define DT_HIPROC 0x7fffffff /* d_un= unspecified */ + +/* Legal values for note segment descriptor types for core files. */ + +#define NT_PRSTATUS 1 /* Contains copy of prstatus struct */ +#define NT_PRFPREG 2 /* Contains copy of fpregset struct. */ +#define NT_FPREGSET 2 /* Contains copy of fpregset struct */ +#define NT_PRPSINFO 3 /* Contains copy of prpsinfo struct */ +#define NT_PRXREG 4 /* Contains copy of prxregset struct */ +#define NT_TASKSTRUCT 4 /* Contains copy of task structure */ +#define NT_PLATFORM 5 /* String from sysinfo(SI_PLATFORM) */ +#define NT_AUXV 6 /* Contains copy of auxv array */ +#define NT_GWINDOWS 7 /* Contains copy of gwindows struct */ +#define NT_ASRS 8 /* Contains copy of asrset struct */ +#define NT_PSTATUS 10 /* Contains copy of pstatus struct */ +#define NT_PSINFO 13 /* Contains copy of psinfo struct */ +#define NT_PRCRED 14 /* Contains copy of prcred struct */ +#define NT_UTSNAME 15 /* Contains copy of utsname struct */ +#define NT_LWPSTATUS 16 /* Contains copy of lwpstatus struct */ +#define NT_LWPSINFO 17 /* Contains copy of lwpinfo struct */ +#define NT_PRFPXREG 20 /* Contains copy of fprxregset struct */ +#define NT_SIGINFO 0x53494749 +/* Contains copy of siginfo_t, + * size might increase + */ +#define NT_FILE 0x46494c45 +/* Contains information about mapped + * files + */ +#define NT_PRXFPREG 0x46e62b7f +/* Contains copy of user_fxsr_struct */ +#define NT_PPC_VMX 0x100 /* PowerPC Altivec/VMX registers */ +#define NT_PPC_SPE 0x101 /* PowerPC SPE/EVR registers */ +#define NT_PPC_VSX 0x102 /* PowerPC VSX registers */ +#define NT_PPC_TAR 0x103 /* Target Address Register */ +#define NT_PPC_PPR 0x104 /* Program Priority Register */ +#define NT_PPC_DSCR 0x105 /* Data Stream Control Register */ +#define NT_PPC_EBB 0x106 /* Event Based Branch Registers */ +#define NT_PPC_PMU 0x107 /* Performance Monitor Registers */ +#define NT_PPC_TM_CGPR 0x108 /* TM checkpointed GPR Registers */ +#define NT_PPC_TM_CFPR 0x109 /* TM checkpointed FPR Registers */ +#define NT_PPC_TM_CVMX 0x10a /* TM checkpointed VMX Registers */ +#define NT_PPC_TM_CVSX 0x10b /* TM checkpointed VSX Registers */ +#define NT_PPC_TM_SPR 0x10c /* TM Special Purpose Registers */ +#define NT_PPC_TM_CTAR \ + 0x10d /* TM checkpointed Target Address \ + * Register \ + */ +#define NT_PPC_TM_CPPR \ + 0x10e /* TM checkpointed Program Priority \ + * Register \ + */ +#define NT_PPC_TM_CDSCR \ + 0x10f /* TM checkpointed Data Stream Control \ + * Register \ + */ +#define NT_PPC_PKEY \ + 0x110 /* Memory Protection Keys \ + * registers. \ + */ +#define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */ +#define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */ +#define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */ +#define NT_S390_HIGH_GPRS 0x300 /* s390 upper register halves */ +#define NT_S390_TIMER 0x301 /* s390 timer register */ +#define NT_S390_TODCMP 0x302 /* s390 TOD clock comparator register */ +#define NT_S390_TODPREG 0x303 /* s390 TOD programmable register */ +#define NT_S390_CTRS 0x304 /* s390 control registers */ +#define NT_S390_PREFIX 0x305 /* s390 prefix register */ +#define NT_S390_LAST_BREAK 0x306 /* s390 breaking event address */ +#define NT_S390_SYSTEM_CALL 0x307 /* s390 system call restart data */ +#define NT_S390_TDB 0x308 /* s390 transaction diagnostic block */ +#define NT_S390_VXRS_LOW \ + 0x309 /* s390 vector registers 0-15 \ + * upper half. \ + */ +#define NT_S390_VXRS_HIGH 0x30a /* s390 vector registers 16-31. */ +#define NT_S390_GS_CB 0x30b /* s390 guarded storage registers. */ +#define NT_S390_GS_BC \ + 0x30c /* s390 guarded storage \ + * broadcast control block. \ + */ +#define NT_S390_RI_CB 0x30d /* s390 runtime instrumentation. */ +#define NT_ARM_VFP 0x400 /* ARM VFP/NEON registers */ +#define NT_ARM_TLS 0x401 /* ARM TLS register */ +#define NT_ARM_HW_BREAK 0x402 /* ARM hardware breakpoint registers */ +#define NT_ARM_HW_WATCH 0x403 /* ARM hardware watchpoint registers */ +#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */ +#define NT_ARM_SVE \ + 0x405 /* ARM Scalable Vector Extension \ + * registers \ + */ +#define NT_ARM_PAC_MASK \ + 0x406 /* ARM pointer authentication \ + * code masks. \ + */ +#define NT_ARM_PACA_KEYS \ + 0x407 /* ARM pointer authentication \ + * address keys. \ + */ +#define NT_ARM_PACG_KEYS \ + 0x408 /* ARM pointer authentication \ + * generic key. \ + */ +#define NT_VMCOREDD 0x700 /* Vmcore Device Dump Note. */ +#define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers. */ +#define NT_MIPS_FP_MODE 0x801 /* MIPS floating-point mode. */ +#define NT_MIPS_MSA 0x802 /* MIPS SIMD registers. */ + +/* Legal values for the note segment descriptor types for object files. */ + +#define NT_VERSION 1 /* Contains a version string. */ + +#endif /* __INCLUDE_ELF_H */ diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/elf32.h b/wasm-micro-runtime/core/iwasm/aot/debug/elf32.h new file mode 100644 index 0000000..b4b2794 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/elf32.h @@ -0,0 +1,161 @@ +/**************************************************************************** + * include/elf32.h + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __INCLUDE_ELF32_H +#define __INCLUDE_ELF32_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#define ELF32_ST_BIND(i) ((i) >> 4) +#define ELF32_ST_TYPE(i) ((i)&0xf) +#define ELF32_ST_INFO(b, t) (((b) << 4) | ((t)&0xf)) + +/* Definitions for Elf32_Rel*::r_info */ + +#define ELF32_R_SYM(i) ((i) >> 8) +#define ELF32_R_TYPE(i) ((i)&0xff) +#define ELF32_R_INFO(s, t) (((s) << 8) | ((t)&0xff)) + +#if 0 +#define ELF_R_SYM(i) ELF32_R_SYM(i) +#endif + +/**************************************************************************** + * Public Type Definitions + ****************************************************************************/ + +/* Figure 4.2: 32-Bit Data Types */ + +typedef uint32_t Elf32_Addr; /* Unsigned program address */ +typedef uint16_t Elf32_Half; /* Unsigned medium integer */ +typedef uint32_t Elf32_Off; /* Unsigned file offset */ +typedef int32_t Elf32_Sword; /* Signed large integer */ +typedef uint32_t Elf32_Word; /* Unsigned large integer */ + +/* Figure 4-3: ELF Header */ + +typedef struct { + unsigned char e_ident[EI_NIDENT]; + Elf32_Half e_type; + Elf32_Half e_machine; + Elf32_Word e_version; + Elf32_Addr e_entry; + Elf32_Off e_phoff; + Elf32_Off e_shoff; + Elf32_Word e_flags; + Elf32_Half e_ehsize; + Elf32_Half e_phentsize; + Elf32_Half e_phnum; + Elf32_Half e_shentsize; + Elf32_Half e_shnum; + Elf32_Half e_shstrndx; +} Elf32_Ehdr; + +/* Figure 4-8: Section Header */ + +typedef struct { + Elf32_Word sh_name; + Elf32_Word sh_type; + Elf32_Word sh_flags; + Elf32_Addr sh_addr; + Elf32_Off sh_offset; + Elf32_Word sh_size; + Elf32_Word sh_link; + Elf32_Word sh_info; + Elf32_Word sh_addralign; + Elf32_Word sh_entsize; +} Elf32_Shdr; + +/* Figure 4-15: Symbol Table Entry */ + +typedef struct { + Elf32_Word st_name; + Elf32_Addr st_value; + Elf32_Word st_size; + unsigned char st_info; + unsigned char st_other; + Elf32_Half st_shndx; +} Elf32_Sym; + +/* Figure 4-19: Relocation Entries */ + +typedef struct { + Elf32_Addr r_offset; + Elf32_Word r_info; +} Elf32_Rel; + +typedef struct { + Elf32_Addr r_offset; + Elf32_Word r_info; + Elf32_Sword r_addend; +} Elf32_Rela; + +/* Figure 5-1: Program Header */ + +typedef struct { + Elf32_Word p_type; + Elf32_Off p_offset; + Elf32_Addr p_vaddr; + Elf32_Addr p_paddr; + Elf32_Word p_filesz; + Elf32_Word p_memsz; + Elf32_Word p_flags; + Elf32_Word p_align; +} Elf32_Phdr; + +/* Figure 5-7: Note Information */ + +typedef struct { + Elf32_Word n_namesz; /* Length of the note's name. */ + Elf32_Word n_descsz; /* Length of the note's descriptor. */ + Elf32_Word n_type; /* Type of the note. */ +} Elf32_Nhdr; + +/* Figure 5-9: Dynamic Structure */ + +typedef struct { + Elf32_Sword d_tag; + union { + Elf32_Word d_val; + Elf32_Addr d_ptr; + } d_un; +} Elf32_Dyn; + +#if 0 +typedef Elf32_Addr Elf_Addr; +typedef Elf32_Ehdr Elf_Ehdr; +typedef Elf32_Rel Elf_Rel; +typedef Elf32_Rela Elf_Rela; +typedef Elf32_Nhdr Elf_Nhdr; +typedef Elf32_Phdr Elf_Phdr; +typedef Elf32_Sym Elf_Sym; +typedef Elf32_Shdr Elf_Shdr; +typedef Elf32_Word Elf_Word; +#endif + +#endif /* __INCLUDE_ELF32_H */ diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/elf64.h b/wasm-micro-runtime/core/iwasm/aot/debug/elf64.h new file mode 100644 index 0000000..499c737 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/elf64.h @@ -0,0 +1,161 @@ +/**************************************************************************** + * include/elf64.h + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __INCLUDE_ELF64_H +#define __INCLUDE_ELF64_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/* See ELF-64 Object File Format: Version 1.5 Draft 2 */ + +/* Definitions for Elf64_Rel*::r_info */ + +#define ELF64_R_SYM(i) ((i) >> 32) +#define ELF64_R_TYPE(i) ((i)&0xffffffffL) +#define ELF64_R_INFO(s, t) (((s) << 32) + ((t)&0xffffffffL)) + +#if 0 +#define ELF_R_SYM(i) ELF64_R_SYM(i) +#endif + +/**************************************************************************** + * Public Type Definitions + ****************************************************************************/ + +/* Table 1: ELF-64 Data Types */ + +typedef uint64_t Elf64_Addr; /* Unsigned program address */ +typedef uint64_t Elf64_Off; /* Unsigned file offset */ +typedef uint16_t Elf64_Half; /* Unsigned medium integer */ +typedef uint32_t Elf64_Word; /* Unsigned long integer */ +typedef int32_t Elf64_Sword; /* Signed integer */ +typedef uint64_t Elf64_Xword; /* Unsigned long integer */ +typedef int64_t Elf64_Sxword; /* Signed large integer */ + +/* Figure 2: ELF-64 Header */ + +typedef struct { + unsigned char e_ident[EI_NIDENT]; /* ELF identification */ + Elf64_Half e_type; /* Object file type */ + Elf64_Half e_machine; /* Machine type */ + Elf64_Word e_version; /* Object file version */ + Elf64_Addr e_entry; /* Entry point address */ + Elf64_Off e_phoff; /* Program header offset */ + Elf64_Off e_shoff; /* Section header offset */ + Elf64_Word e_flags; /* Processor-specific flags */ + Elf64_Half e_ehsize; /* ELF header size */ + Elf64_Half e_phentsize; /* Size of program header entry */ + Elf64_Half e_phnum; /* Number of program header entry */ + Elf64_Half e_shentsize; /* Size of section header entry */ + Elf64_Half e_shnum; /* Number of section header entries */ + Elf64_Half e_shstrndx; /* Section name string table index */ +} Elf64_Ehdr; + +/* Figure 3: ELF-64 Section Header */ + +typedef struct { + Elf64_Word sh_name; /* Section name */ + Elf64_Word sh_type; /* Section type */ + Elf64_Xword sh_flags; /* Section attributes */ + Elf64_Addr sh_addr; /* Virtual address in memory */ + Elf64_Off sh_offset; /* Offset in file */ + Elf64_Xword sh_size; /* Size of section */ + Elf64_Word sh_link; /* Link to other section */ + Elf64_Word sh_info; /* Miscellaneous information */ + Elf64_Xword sh_addralign; /* Address alignment boundary */ + Elf64_Xword sh_entsize; /* Size of entries, if section has table */ +} Elf64_Shdr; + +/* Figure 4: ELF-64 Symbol Table Entry */ + +typedef struct { + Elf64_Word st_name; /* Symbol name */ + unsigned char st_info; /* Type and Binding attributes */ + unsigned char st_other; /* Reserved */ + Elf64_Half st_shndx; /* Section table index */ + Elf64_Addr st_value; /* Symbol value */ + Elf64_Xword st_size; /* Size of object (e.g., common) */ +} Elf64_Sym; + +/* Figure 5: ELF-64 Relocation Entries */ + +typedef struct { + Elf64_Addr r_offset; /* Address of reference */ + Elf64_Xword r_info; /* Symbol index and type of relocation */ +} Elf64_Rel; + +typedef struct { + Elf64_Addr r_offset; /* Address of reference */ + Elf64_Xword r_info; /* Symbol index and type of relocation */ + Elf64_Sxword r_addend; /* Constant part of expression */ +} Elf64_Rela; + +/* Figure 6: ELF-64 Program Header Table Entry */ + +typedef struct { + Elf64_Word p_type; /* Type of segment */ + Elf64_Word p_flags; /* Segment attributes */ + Elf64_Off p_offset; /* Offset in file */ + Elf64_Addr p_vaddr; /* Virtual address in memory */ + Elf64_Addr p_paddr; /* Reserved */ + Elf64_Word p_filesz; /* Size of segment in file */ + Elf64_Word p_memsz; /* Size of segment in memory */ + Elf64_Word p_align; /* Alignment of segment */ +} Elf64_Phdr; + +/* Figure 7. Format of a Note Section */ + +typedef struct { + Elf64_Word n_namesz; /* Length of the note's name. */ + Elf64_Word n_descsz; /* Length of the note's descriptor. */ + Elf64_Word n_type; /* Type of the note. */ +} Elf64_Nhdr; + +/* Figure 8: Dynamic Table Structure */ + +typedef struct { + Elf64_Sxword d_tag; + union { + Elf64_Xword d_val; + Elf64_Addr d_ptr; + } d_un; +} Elf64_Dyn; + +#if 0 +typedef Elf64_Addr Elf_Addr; +typedef Elf64_Ehdr Elf_Ehdr; +typedef Elf64_Rel Elf_Rel; +typedef Elf64_Rela Elf_Rela; +typedef Elf64_Nhdr Elf_Nhdr; +typedef Elf64_Phdr Elf_Phdr; +typedef Elf64_Sym Elf_Sym; +typedef Elf64_Shdr Elf_Shdr; +typedef Elf64_Word Elf_Word; +#endif + +#endif /* __INCLUDE_ELF64_H */ diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/elf_parser.c b/wasm-micro-runtime/core/iwasm/aot/debug/elf_parser.c new file mode 100644 index 0000000..657f953 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/elf_parser.c @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "elf.h" + +#include "aot_runtime.h" +#include "bh_log.h" +#include "elf_parser.h" + +bool +is_ELF(void *buf) +{ + Elf32_Ehdr *eh = (Elf32_Ehdr *)buf; + if (!strncmp((char *)eh->e_ident, "\177ELF", 4)) { + LOG_VERBOSE("the buffer is ELF entry!"); + return true; + } + LOG_VERBOSE("the buffer is not ELF entry!"); + return false; +} + +static bool +is64Bit(Elf32_Ehdr *eh) +{ + if (eh->e_ident[EI_CLASS] == ELFCLASS64) + return true; + else + return false; +} + +static bool +is32Bit(Elf32_Ehdr *eh) +{ + if (eh->e_ident[EI_CLASS] == ELFCLASS32) + return true; + else + return false; +} + +bool +is_ELF64(void *buf) +{ + Elf64_Ehdr *eh = (Elf64_Ehdr *)buf; + if (!strncmp((char *)eh->e_ident, "\177ELF", 4)) { + LOG_VERBOSE("the buffer is ELF entry!"); + return true; + } + LOG_VERBOSE("the buffer is not ELF entry!"); + return false; +} + +static void +read_section_header_table(Elf32_Ehdr *eh, Elf32_Shdr *sh_table[]) +{ + uint32_t i; + char *buf = (char *)eh; + buf += eh->e_shoff; + LOG_VERBOSE("str index = %d count=%d", eh->e_shstrndx, eh->e_shnum); + for (i = 0; i < eh->e_shnum; i++) { + sh_table[i] = (Elf32_Shdr *)buf; + buf += eh->e_shentsize; + } +} + +static void +read_section_header_table64(Elf64_Ehdr *eh, Elf64_Shdr *sh_table[]) +{ + uint32_t i; + char *buf = (char *)eh; + buf += eh->e_shoff; + + for (i = 0; i < eh->e_shnum; i++) { + sh_table[i] = (Elf64_Shdr *)buf; + buf += eh->e_shentsize; + } +} + +static char * +get_section(Elf32_Ehdr *eh, Elf32_Shdr *section_header) +{ + char *buf = (char *)eh; + return buf + section_header->sh_offset; +} + +static char * +get_section64(Elf64_Ehdr *eh, Elf64_Shdr *section_header) +{ + char *buf = (char *)eh; + return buf + section_header->sh_offset; +} + +static bool +is_text_section(const char *section_name) +{ + return !strcmp(section_name, ".text") || !strcmp(section_name, ".ltext"); +} + +bool +get_text_section(void *buf, uint64_t *offset, uint64_t *size) +{ + bool ret = false; + uint32 i; + char *sh_str; + + /* Assumption: Only one of .text or .ltext is non-empty. */ + if (is64Bit(buf)) { + Elf64_Ehdr *eh = (Elf64_Ehdr *)buf; + Elf64_Shdr **sh_table = + wasm_runtime_malloc(eh->e_shnum * sizeof(Elf64_Shdr *)); + if (sh_table) { + read_section_header_table64(eh, sh_table); + sh_str = get_section64(eh, sh_table[eh->e_shstrndx]); + for (i = 0; i < eh->e_shnum; i++) { + if (is_text_section(sh_str + sh_table[i]->sh_name)) { + *offset = sh_table[i]->sh_offset; + *size = sh_table[i]->sh_size; + sh_table[i]->sh_addr = + (Elf64_Addr)(uintptr_t)((char *)buf + + sh_table[i]->sh_offset); + ret = true; + if (*size > 0) { + break; + } + } + } + wasm_runtime_free(sh_table); + } + } + else if (is32Bit(buf)) { + Elf32_Ehdr *eh = (Elf32_Ehdr *)buf; + Elf32_Shdr **sh_table = + wasm_runtime_malloc(eh->e_shnum * sizeof(Elf32_Shdr *)); + if (sh_table) { + read_section_header_table(eh, sh_table); + sh_str = get_section(eh, sh_table[eh->e_shstrndx]); + for (i = 0; i < eh->e_shnum; i++) { + if (is_text_section(sh_str + sh_table[i]->sh_name)) { + *offset = sh_table[i]->sh_offset; + *size = sh_table[i]->sh_size; + sh_table[i]->sh_addr = + (Elf32_Addr)(uintptr_t)((char *)buf + + sh_table[i]->sh_offset); + ret = true; + if (*size > 0) { + break; + } + } + } + wasm_runtime_free(sh_table); + } + } + + return ret; +} diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/elf_parser.h b/wasm-micro-runtime/core/iwasm/aot/debug/elf_parser.h new file mode 100644 index 0000000..887d82f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/elf_parser.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _ELF_PARSER_H_ +#define _ELF_PARSER_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +bool +is_ELF(void *buf); + +bool +is_ELF64(void *buf); + +bool +get_text_section(void *buf, uint64_t *offset, uint64_t *size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/jit_debug.c b/wasm-micro-runtime/core/iwasm/aot/debug/jit_debug.c new file mode 100644 index 0000000..4b0e46f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/jit_debug.c @@ -0,0 +1,254 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_log.h" +#include "bh_platform.h" +#include "../../interpreter/wasm_runtime.h" + +#include +#include +#include +#include +#include +#include +#include + +/* This must be kept in sync with gdb/gdb/jit.h */ +#ifdef __cplusplus +extern "C" { +#endif + +/* clang-format off */ +typedef enum JITAction { + JIT_NOACTION = 0, + JIT_REGISTER_FN, + JIT_UNREGISTER_FN +} JITAction; +/* clang-format on */ + +typedef struct JITCodeEntry { + struct JITCodeEntry *next_; + struct JITCodeEntry *prev_; + const uint8 *symfile_addr_; + uint64 symfile_size_; +} JITCodeEntry; + +typedef struct JITDescriptor { + uint32 version_; + uint32 action_flag_; + JITCodeEntry *relevant_entry_; + JITCodeEntry *first_entry_; +} JITDescriptor; + +/* LLVM has already define this */ +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) +/** + * GDB will place breakpoint into this function. + * To prevent GCC from inlining or removing it we place noinline attribute + * and inline assembler statement inside. + */ +void __attribute__((noinline)) __jit_debug_register_code(); + +void __attribute__((noinline)) __jit_debug_register_code() +{ + int x; + *(char *)&x = '\0'; +} + +/** + * GDB will inspect contents of this descriptor. + * Static initialization is necessary to prevent GDB from seeing + * uninitialized descriptor. + */ + +JITDescriptor __jit_debug_descriptor = { 1, JIT_NOACTION, NULL, NULL }; +#else +extern void +__jit_debug_register_code(); +extern JITDescriptor __jit_debug_descriptor; +#endif + +/** + * Call __jit_debug_register_code indirectly via global variable. + * This gives the debugger an easy way to inject custom code to + * handle the events. + */ +void (*__jit_debug_register_code_ptr)() = __jit_debug_register_code; + +#ifdef __cplusplus +} +#endif + +typedef struct WASMJITDebugEngine { + korp_mutex jit_entry_lock; + bh_list jit_entry_list; +} WASMJITDebugEngine; + +typedef struct WASMJITEntryNode { + struct WASMJITEntryNode *next; + JITCodeEntry *entry; +} WASMJITEntryNode; + +static WASMJITDebugEngine *jit_debug_engine; + +static JITCodeEntry * +CreateJITCodeEntryInternal(const uint8 *symfile_addr, uint64 symfile_size) +{ + JITCodeEntry *entry; + + os_mutex_lock(&jit_debug_engine->jit_entry_lock); + + if (!(entry = wasm_runtime_malloc(sizeof(JITCodeEntry)))) { + LOG_ERROR("WASM JIT Debug Engine error: failed to allocate memory"); + os_mutex_unlock(&jit_debug_engine->jit_entry_lock); + return NULL; + } + entry->symfile_addr_ = symfile_addr; + entry->symfile_size_ = symfile_size; + entry->prev_ = NULL; + + entry->next_ = __jit_debug_descriptor.first_entry_; + if (entry->next_ != NULL) { + entry->next_->prev_ = entry; + } + __jit_debug_descriptor.first_entry_ = entry; + __jit_debug_descriptor.relevant_entry_ = entry; + + __jit_debug_descriptor.action_flag_ = JIT_REGISTER_FN; + + (*__jit_debug_register_code_ptr)(); + + os_mutex_unlock(&jit_debug_engine->jit_entry_lock); + return entry; +} + +static void +DestroyJITCodeEntryInternal(JITCodeEntry *entry) +{ + os_mutex_lock(&jit_debug_engine->jit_entry_lock); + + if (entry->prev_ != NULL) { + entry->prev_->next_ = entry->next_; + } + else { + __jit_debug_descriptor.first_entry_ = entry->next_; + } + + if (entry->next_ != NULL) { + entry->next_->prev_ = entry->prev_; + } + + __jit_debug_descriptor.relevant_entry_ = entry; + __jit_debug_descriptor.action_flag_ = JIT_UNREGISTER_FN; + (*__jit_debug_register_code_ptr)(); + + wasm_runtime_free(entry); + + os_mutex_unlock(&jit_debug_engine->jit_entry_lock); +} + +bool +jit_debug_engine_init() +{ + if (jit_debug_engine) { + return true; + } + + if (!(jit_debug_engine = wasm_runtime_malloc(sizeof(WASMJITDebugEngine)))) { + LOG_ERROR("WASM JIT Debug Engine error: failed to allocate memory"); + return false; + } + memset(jit_debug_engine, 0, sizeof(WASMJITDebugEngine)); + + if (os_mutex_init(&jit_debug_engine->jit_entry_lock) != 0) { + wasm_runtime_free(jit_debug_engine); + jit_debug_engine = NULL; + return false; + } + + bh_list_init(&jit_debug_engine->jit_entry_list); + return true; +} + +void +jit_debug_engine_destroy() +{ + if (jit_debug_engine) { + WASMJITEntryNode *node, *node_next; + + /* Destroy all nodes */ + node = bh_list_first_elem(&jit_debug_engine->jit_entry_list); + while (node) { + node_next = bh_list_elem_next(node); + DestroyJITCodeEntryInternal(node->entry); + bh_list_remove(&jit_debug_engine->jit_entry_list, node); + wasm_runtime_free(node); + node = node_next; + } + + /* Destroy JIT Debug Engine */ + os_mutex_destroy(&jit_debug_engine->jit_entry_lock); + wasm_runtime_free(jit_debug_engine); + jit_debug_engine = NULL; + } +} + +bool +jit_code_entry_create(const uint8 *symfile_addr, uint64 symfile_size) +{ + JITCodeEntry *entry; + WASMJITEntryNode *node; + + if (!(node = wasm_runtime_malloc(sizeof(WASMJITEntryNode)))) { + LOG_ERROR("WASM JIT Debug Engine error: failed to allocate memory"); + return false; + } + + entry = CreateJITCodeEntryInternal(symfile_addr, symfile_size); + + if (!entry) { + wasm_runtime_free(node); + return false; + } + + node->entry = entry; + os_mutex_lock(&jit_debug_engine->jit_entry_lock); + bh_list_insert(&jit_debug_engine->jit_entry_list, node); + os_mutex_unlock(&jit_debug_engine->jit_entry_lock); + return true; +} + +void +jit_code_entry_destroy(const uint8 *symfile_addr) +{ + WASMJITEntryNode *node; + + node = bh_list_first_elem(&jit_debug_engine->jit_entry_list); + while (node) { + WASMJITEntryNode *next_node = bh_list_elem_next(node); + if (node->entry->symfile_addr_ == symfile_addr) { + DestroyJITCodeEntryInternal(node->entry); + os_mutex_lock(&jit_debug_engine->jit_entry_lock); + bh_list_remove(&jit_debug_engine->jit_entry_list, node); + os_mutex_unlock(&jit_debug_engine->jit_entry_lock); + wasm_runtime_free(node); + } + node = next_node; + } +} diff --git a/wasm-micro-runtime/core/iwasm/aot/debug/jit_debug.h b/wasm-micro-runtime/core/iwasm/aot/debug/jit_debug.h new file mode 100644 index 0000000..5e3e365 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/debug/jit_debug.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_DEBUG_H_ +#define _JIT_DEBUG_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_debug_engine_init(); + +void +jit_debug_engine_destroy(); + +bool +jit_code_entry_create(const uint8 *symfile_addr, uint64 symfile_size); + +void +jit_code_entry_destroy(const uint8 *symfile_addr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/aot/iwasm_aot.cmake b/wasm-micro-runtime/core/iwasm/aot/iwasm_aot.cmake new file mode 100644 index 0000000..efff88d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/aot/iwasm_aot.cmake @@ -0,0 +1,84 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (IWASM_AOT_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_AOT=1) + +include_directories (${IWASM_AOT_DIR}) + +file (GLOB c_source_all ${IWASM_AOT_DIR}/*.c) + +if (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_x86_64.c) +elseif (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_x86_32.c) +elseif (WAMR_BUILD_TARGET MATCHES "AARCH64.*") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_aarch64.c) +elseif (WAMR_BUILD_TARGET MATCHES "ARM.*") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_arm.c) +elseif (WAMR_BUILD_TARGET MATCHES "THUMB.*") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_thumb.c) +elseif (WAMR_BUILD_TARGET STREQUAL "MIPS") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_mips.c) +elseif (WAMR_BUILD_TARGET STREQUAL "XTENSA") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_xtensa.c) +elseif (WAMR_BUILD_TARGET MATCHES "RISCV*") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_riscv.c) +elseif (WAMR_BUILD_TARGET STREQUAL "ARC") + set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_arc.c) +else () + message (FATAL_ERROR "Build target isn't set") +endif () + +if (WAMR_BUILD_DEBUG_AOT EQUAL 1) + add_definitions(-DWASM_ENABLE_DEBUG_AOT=1) + file(GLOB debug_source ${IWASM_AOT_DIR}/debug/*.c) +endif() + +if ((WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + AND (WAMR_BUILD_PLATFORM STREQUAL "windows") + AND (NOT WAMR_DISABLE_HW_BOUND_CHECK EQUAL 1)) + include(FetchContent) + + FetchContent_Declare( + zycore + GIT_REPOSITORY https://github.com/zyantific/zycore-c.git + ) + FetchContent_GetProperties(zycore) + if (NOT zycore_POPULATED) + message ("-- Fetching zycore ..") + FetchContent_Populate(zycore) + include_directories("${zycore_SOURCE_DIR}/include") + include_directories("${zycore_BINARY_DIR}") + add_definitions(-DZYCORE_STATIC_BUILD=1) + add_subdirectory(${zycore_SOURCE_DIR} ${zycore_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zycore ${zycore_SOURCE_DIR}/src/*.c) + endif () + + FetchContent_Declare( + zydis + GIT_REPOSITORY https://github.com/zyantific/zydis.git + GIT_TAG e14a07895136182a5b53e181eec3b1c6e0b434de + ) + FetchContent_GetProperties(zydis) + if (NOT zydis_POPULATED) + message ("-- Fetching zydis ..") + FetchContent_Populate(zydis) + option(ZYDIS_FEATURE_ENCODER "" OFF) + option(ZYDIS_BUILD_TOOLS "" OFF) + option(ZYDIS_BUILD_EXAMPLES "" OFF) + option(ZYDIS_BUILD_MAN "" OFF) + option(ZYDIS_BUILD_DOXYGEN "" OFF) + include_directories("${zydis_BINARY_DIR}") + include_directories("${zydis_SOURCE_DIR}/include") + include_directories("${zydis_SOURCE_DIR}/src") + add_definitions(-DZYDIS_STATIC_BUILD=1) + add_subdirectory(${zydis_SOURCE_DIR} ${zydis_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zydis ${zydis_SOURCE_DIR}/src/*.c) + endif () +endif () + + +set (IWASM_AOT_SOURCE ${c_source_all} ${arch_source} ${debug_source} + ${c_source_zycore} ${c_source_zydis}) diff --git a/wasm-micro-runtime/core/iwasm/common/SConscript b/wasm-micro-runtime/core/iwasm/common/SConscript new file mode 100644 index 0000000..0a55f24 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/SConscript @@ -0,0 +1,28 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * +import re + +Import('rtconfig') + +cwd = GetCurrentDir() + +src = Glob('*.c') + +if rtconfig.ARCH == 'arm': + if re.match('^cortex-m.*', rtconfig.CPU): + src += ['arch/invokeNative_thumb.s'] + elif re.match('^cortex-a.*', rtconfig.CPU): + src += ['arch/invokeNative_arm.s'] +elif rtconfig.ARCH == 'ia32': + src += ['arch/invokeNative_ia32.s'] + +CPPPATH = [cwd, cwd + '/../include'] + +group = DefineGroup('iwasm_common', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_aarch64.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_aarch64.s new file mode 100644 index 0000000..ea5cbcb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_aarch64.s @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * + * x0 function ptr + * x1 argv + * x2 nstacks + */ + + sub sp, sp, #0x30 + stp x19, x20, [sp, #0x20] /* save the registers */ + stp x21, x22, [sp, #0x10] + stp x23, x24, [sp, #0x0] + + mov x19, x0 /* x19 = function ptr */ + mov x20, x1 /* x20 = argv */ + mov x21, x2 /* x21 = nstacks */ + mov x22, sp /* save the sp before call function */ + + /* Fill in float-point registers */ + ldp d0, d1, [x20], #16 /* d0 = argv[0], d1 = argv[1] */ + ldp d2, d3, [x20], #16 /* d2 = argv[2], d3 = argv[3] */ + ldp d4, d5, [x20], #16 /* d4 = argv[4], d5 = argv[5] */ + ldp d6, d7, [x20], #16 /* d6 = argv[6], d7 = argv[7] */ + + /* Fill integer registers */ + ldp x0, x1, [x20], #16 /* x0 = argv[8] = exec_env, x1 = argv[9] */ + ldp x2, x3, [x20], #16 /* x2 = argv[10], x3 = argv[11] */ + ldp x4, x5, [x20], #16 /* x4 = argv[12], x5 = argv[13] */ + ldp x6, x7, [x20], #16 /* x6 = argv[14], x7 = argv[15] */ + + /* Now x20 points to stack args */ + + /* Directly call the fucntion if no args in stack */ + cmp x21, #0 + beq call_func + + /* Fill all stack args: reserve stack space and fill one by one */ + mov x23, sp + bic sp, x23, #15 /* Ensure stack is 16 bytes aligned */ + lsl x23, x21, #3 /* x23 = nstacks * 8 */ + add x23, x23, #15 /* x23 = (x23 + 15) & ~15 */ + bic x23, x23, #15 + sub sp, sp, x23 /* reserved stack space for stack arguments */ + mov x23, sp + +loop_stack_args: /* copy stack arguments to stack */ + cmp x21, #0 + beq call_func + ldr x24, [x20], #8 + str x24, [x23], #8 + sub x21, x21, #1 + b loop_stack_args + +call_func: + mov x20, x30 /* save x30(lr) */ + blr x19 + mov sp, x22 /* restore sp which is saved before calling fuction*/ + +return: + mov x30, x20 /* restore x30(lr) */ + ldp x19, x20, [sp, #0x20] /* restore the registers in stack */ + ldp x21, x22, [sp, #0x10] + ldp x23, x24, [sp, #0x0] + add sp, sp, #0x30 /* restore sp */ + ret + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_aarch64_simd.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_aarch64_simd.s new file mode 100644 index 0000000..a6ccc15 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_aarch64_simd.s @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2020 Intel Corporation Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * + * x0 function ptr + * x1 argv + * x2 nstacks + */ + + sub sp, sp, #0x30 + stp x19, x20, [sp, #0x20] /* save the registers */ + stp x21, x22, [sp, #0x10] + stp x23, x24, [sp, #0x0] + + mov x19, x0 /* x19 = function ptr */ + mov x20, x1 /* x20 = argv */ + mov x21, x2 /* x21 = nstacks */ + mov x22, sp /* save the sp before call function */ + + /* Fill in float-point registers */ + ld1 {v0.2D, v1.2D, v2.2D, v3.2D}, [x20], #64 /* v0 = argv[0], v1 = argv[1], v2 = argv[2], v3 = argv[3]*/ + ld1 {v4.2D, v5.2D, v6.2D, v7.2D}, [x20], #64 /* v4 = argv[4], v5 = argv[5], v6 = argv[6], v7 = argv[7]*/ + + /* Fill inteter registers */ + ldp x0, x1, [x20], #16 /* x0 = argv[8] = exec_env, x1 = argv[9] */ + ldp x2, x3, [x20], #16 /* x2 = argv[10], x3 = argv[11] */ + ldp x4, x5, [x20], #16 /* x4 = argv[12], x5 = argv[13] */ + ldp x6, x7, [x20], #16 /* x6 = argv[14], x7 = argv[15] */ + + /* Now x20 points to stack args */ + + /* Directly call the fucntion if no args in stack */ + cmp x21, #0 + beq call_func + + /* Fill all stack args: reserve stack space and fill one by one */ + mov x23, sp + bic sp, x23, #15 /* Ensure stack is 16 bytes aligned */ + lsl x23, x21, #3 /* x23 = nstacks * 8 */ + add x23, x23, #15 /* x23 = (x23 + 15) & ~15 */ + bic x23, x23, #15 + sub sp, sp, x23 /* reserved stack space for stack arguments */ + mov x23, sp + +loop_stack_args: /* copy stack arguments to stack */ + cmp x21, #0 + beq call_func + ldr x24, [x20], #8 + str x24, [x23], #8 + sub x21, x21, #1 + b loop_stack_args + +call_func: + mov x20, x30 /* save x30(lr) */ + blr x19 + mov sp, x22 /* restore sp which is saved before calling fuction*/ + +return: + mov x30, x20 /* restore x30(lr) */ + ldp x19, x20, [sp, #0x20] /* restore the registers in stack */ + ldp x21, x22, [sp, #0x10] + ldp x23, x24, [sp, #0x0] + add sp, sp, #0x30 /* restore sp */ + ret + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arc.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arc.s new file mode 100644 index 0000000..e448eea --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arc.s @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * r0: function ptr + * r1: argv + * r2: nstacks + * ARC ABI: + * r0-r7: function arguments, caller-saved + * r8-r12: temp registers, caller-saved + */ + + push_s blink /* push return addr */ + st.aw fp, [sp, -4] /* push fp */ + mov fp, sp /* fp = sp */ + + mov r8, r0 /* r8 = func_ptr */ + mov r9, r1 /* r9 = argv */ + mov r10, r2 /* r10 = nstacks */ + + ld r0, [r9, 0] /* r0 = argv[0] */ + ld r1, [r9, 4] /* r1 = argv[1] */ + ld r2, [r9, 8] /* r2 = argv[2] */ + ld r3, [r9, 12] /* r3 = argv[3] */ + ld r4, [r9, 16] /* r4 = argv[4] */ + ld r5, [r9, 20] /* r5 = argv[5] */ + ld r6, [r9, 24] /* r6 = argv[6] */ + ld r7, [r9, 28] /* r7 = argv[7] */ + + add r9, r9, 32 /* r9 = stack_args */ + breq r10, 0, call_func /* if (r10 == 0) goto call_func */ + + asl r11, r10, 2 /* r11 = nstacks * 4 */ + sub sp, sp, r11 /* sp = sp - nstacks * 4 */ + and sp, sp, ~7 /* make sp 8-byte aligned */ + mov r11, sp /* r11 = sp */ + +loop_stack_args: + breq r10, 0, call_func /* if (r10 == 0) goto call_func */ + ld r12, [r9] /* r12 = stack_args[i] */ + st r12, [r11] /* stack[i] = r12 */ + add r9, r9, 4 /* r9 = r9 + 4 */ + add r11, r11, 4 /* r11 = r11 + 4 */ + sub r10, r10, 1 /* r10 = r10 + 1 */ + j loop_stack_args + +call_func: + jl [r8] /* call function */ + + mov sp, fp /* sp = fp */ + ld.ab fp, [sp, 4] /* pop fp */ + pop_s blink /* pop return addr */ + j_s [blink] /* ret */ + nop_s + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arm.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arm.s new file mode 100644 index 0000000..bfe8e3b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arm.s @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * + * r0 function ptr + * r1 argv + * r2 argc + */ + + stmfd sp!, {r4, r5, r6, r7, lr} + sub sp, sp, #4 /* make sp 8 byte aligned */ + mov ip, r0 /* ip = function ptr */ + mov r4, r1 /* r4 = argv */ + mov r5, r2 /* r5 = argc */ + + cmp r5, #1 /* at least one argument required: exec_env */ + blt return + + mov r6, #0 /* increased stack size */ + + ldr r0, [r4], #4 /* r0 = argv[0] = exec_env */ + cmp r5, #1 + beq call_func + + ldr r1, [r4], #4 /* r1 = argv[1] */ + cmp r5, #2 + beq call_func + + ldr r2, [r4], #4 /* r2 = argv[2] */ + cmp r5, #3 + beq call_func + + ldr r3, [r4], #4 /* r3 = argv[3] */ + cmp r5, #4 + beq call_func + + sub r5, r5, #4 /* argc -= 4, now we have r0 ~ r3 */ + + /* Ensure address is 8 byte aligned */ + mov r6, r5, lsl#2 /* r6 = argc * 4 */ + add r6, r6, #7 /* r6 = (r6 + 7) & ~7 */ + bic r6, r6, #7 + sub sp, sp, r6 /* reserved stack space for left arguments */ + mov r7, sp + +loop_args: /* copy left arguments to stack */ + cmp r5, #0 + beq call_func + ldr lr, [r4], #4 + str lr, [r7], #4 + sub r5, r5, #1 + b loop_args + +call_func: + blx ip + add sp, sp, r6 /* restore sp */ + +return: + add sp, sp, #4 + ldmfd sp!, {r4, r5, r6, r7, lr} + bx lr diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arm_vfp.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arm_vfp.s new file mode 100644 index 0000000..78a4bab --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_arm_vfp.s @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * + * r0 function ptr + * r1 argv + * r2 nstacks + */ + + stmfd sp!, {r4, r5, r6, r7, lr} + sub sp, sp, #4 /* make sp 8 byte aligned */ + mov ip, r0 /* ip = function ptr */ + mov r4, r1 /* r4 = argv */ + mov r5, r2 /* r5 = nstacks */ + mov r6, sp + + /* Fill all int args */ + ldr r0, [r4], #4 /* r0 = *(int*)&argv[0] = exec_env */ + ldr r1, [r4], #4 /* r1 = *(int*)&argv[1] */ + ldr r2, [r4], #4 /* r2 = *(int*)&argv[2] */ + ldr r3, [r4], #4 /* r3 = *(int*)&argv[3] */ + + /* Fill all float/double args to 16 single-precision registers, s0-s15, */ + /* which may also be accessed as 8 double-precision registers, d0-d7 (with */ + /* d0 overlapping s0, s1; d1 overlapping s2, s3; etc). */ + vldr s0, [r4, #0] /* s0 = *(float*)&argv[4] */ + vldr s1, [r4, #4] + vldr s2, [r4, #8] + vldr s3, [r4, #12] + vldr s4, [r4, #16] + vldr s5, [r4, #20] + vldr s6, [r4, #24] + vldr s7, [r4, #28] + vldr s8, [r4, #32] + vldr s9, [r4, #36] + vldr s10, [r4, #40] + vldr s11, [r4, #44] + vldr s12, [r4, #48] + vldr s13, [r4, #52] + vldr s14, [r4, #56] + vldr s15, [r4, #60] + /* Directly call the fucntion if no args in stack */ + cmp r5, #0 + beq call_func + + + /* Fill all stack args: reserve stack space and fill one by one */ + add r4, r4, #64 /* r4 points to stack args */ + bic sp, sp, #7 /* Ensure stack is 8 byte aligned */ + mov r7, r5, lsl#2 /* r7 = nstacks * 4 */ + add r7, r7, #7 /* r7 = (r7 + 7) & ~7 */ + bic r7, r7, #7 + sub sp, sp, r7 /* reserved stack space for stack arguments */ + mov r7, sp + +loop_stack_args: /* copy stack arguments to stack */ + cmp r5, #0 + beq call_func + ldr lr, [r4], #4 /* Note: caller should insure int64 and */ + str lr, [r7], #4 /* double are placed in 8 bytes aligned address */ + sub r5, r5, #1 + b loop_stack_args + +call_func: + blx ip + mov sp, r6 /* restore sp */ + +return: + add sp, sp, #4 /* make sp 8 byte aligned */ + ldmfd sp!, {r4, r5, r6, r7, lr} + bx lr + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64.asm b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64.asm new file mode 100644 index 0000000..df81153 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64.asm @@ -0,0 +1,62 @@ +; +; Copyright (C) 2019 Intel Corporation. All rights reserved. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; + +_TEXT SEGMENT + ; rcx func_ptr + ; rdx argv + ; r8 n_stacks + +invokeNative PROC + push rbp + mov rbp, rsp + + mov r10, rcx ; func_ptr + mov rax, rdx ; argv + mov rcx, r8 ; n_stacks + +; fill all fp args + movsd xmm0, qword ptr [rax + 0] + movsd xmm1, qword ptr [rax + 8] + movsd xmm2, qword ptr [rax + 16] + movsd xmm3, qword ptr [rax + 24] + +; check for stack args + cmp rcx, 0 + jz cycle_end + + mov rdx, rsp + and rdx, 15 + jz no_abort + int 3 +no_abort: + mov rdx, rcx + and rdx, 1 + shl rdx, 3 + sub rsp, rdx + +; store stack args + lea r9, qword ptr [rax + rcx * 8 + 56] + sub r9, rsp ; offset +cycle: + push qword ptr [rsp + r9] + loop cycle + +cycle_end: + mov rcx, [rax + 32] + mov rdx, [rax + 40] + mov r8, [rax + 48] + mov r9, [rax + 56] + + sub rsp, 32 ; shadow space + + call r10 + leave + ret + +invokeNative ENDP + +_TEXT ENDS + +END diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64.s new file mode 100644 index 0000000..739e84e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64.s @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN +.globl invokeNative + .type invokeNative, @function +invokeNative: +#else +.globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + /* rdi - function ptr */ + /* rsi - argv */ + /* rdx - n_stacks */ + + push %rbp + mov %rsp, %rbp + + mov %rdx, %r10 + mov %rsp, %r11 /* Check that stack is aligned on */ + and $8, %r11 /* 16 bytes. This code may be removed */ + je check_stack_succ /* when we are sure that compiler always */ + int3 /* calls us with aligned stack */ +check_stack_succ: + mov %r10, %r11 /* Align stack on 16 bytes before pushing */ + and $1, %r11 /* stack arguments in case we have an odd */ + shl $3, %r11 /* number of stack arguments */ + sub %r11, %rsp + /* store memory args */ + movq %rdi, %r11 /* func ptr */ + movq %r10, %rcx /* counter */ + lea 64+48-8(%rsi,%rcx,8), %r10 + sub %rsp, %r10 + cmpq $0, %rcx + je push_args_end +push_args: + push 0(%rsp,%r10) + loop push_args +push_args_end: + /* fill all fp args */ + movq 0x00(%rsi), %xmm0 + movq 0x08(%rsi), %xmm1 + movq 0x10(%rsi), %xmm2 + movq 0x18(%rsi), %xmm3 + movq 0x20(%rsi), %xmm4 + movq 0x28(%rsi), %xmm5 + movq 0x30(%rsi), %xmm6 + movq 0x38(%rsi), %xmm7 + + /* fill all int args */ + movq 0x40(%rsi), %rdi + movq 0x50(%rsi), %rdx + movq 0x58(%rsi), %rcx + movq 0x60(%rsi), %r8 + movq 0x68(%rsi), %r9 + movq 0x48(%rsi), %rsi + + call *%r11 + leave + ret + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64_simd.asm b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64_simd.asm new file mode 100644 index 0000000..084a0f6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64_simd.asm @@ -0,0 +1,62 @@ +; +; Copyright (C) 2019 Intel Corporation. All rights reserved. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; + +_TEXT SEGMENT + ; rcx func_ptr + ; rdx argv + ; r8 n_stacks + +invokeNative PROC + push rbp + mov rbp, rsp + + mov r10, rcx ; func_ptr + mov rax, rdx ; argv + mov rcx, r8 ; n_stacks + +; fill all fp args + movdqu xmm0, xmmword ptr [rax + 0] + movdqu xmm1, xmmword ptr [rax + 16] + movdqu xmm2, xmmword ptr [rax + 32] + movdqu xmm3, xmmword ptr [rax + 48] + +; check for stack args + cmp rcx, 0 + jz cycle_end + + mov rdx, rsp + and rdx, 15 + jz no_abort + int 3 +no_abort: + mov rdx, rcx + and rdx, 1 + shl rdx, 3 + sub rsp, rdx + +; store stack args + lea r9, qword ptr [rax + rcx * 8 + 88] + sub r9, rsp ; offset +cycle: + push qword ptr [rsp + r9] + loop cycle + +cycle_end: + mov rcx, [rax + 64] + mov rdx, [rax + 72] + mov r8, [rax + 80] + mov r9, [rax + 88] + + sub rsp, 32 ; shadow space + + call r10 + leave + ret + +invokeNative ENDP + +_TEXT ENDS + +END diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64_simd.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64_simd.s new file mode 100644 index 0000000..0043ac9 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_em64_simd.s @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN +.globl invokeNative + .type invokeNative, @function +invokeNative: +#else +.globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + /* rdi - function ptr */ + /* rsi - argv */ + /* rdx - n_stacks */ + + push %rbp + mov %rsp, %rbp + + mov %rdx, %r10 + mov %rsp, %r11 /* Check that stack is aligned on */ + and $8, %r11 /* 16 bytes. This code may be removed */ + je check_stack_succ /* when we are sure that compiler always */ + int3 /* calls us with aligned stack */ +check_stack_succ: + mov %r10, %r11 /* Align stack on 16 bytes before pushing */ + and $1, %r11 /* stack arguments in case we have an odd */ + shl $3, %r11 /* number of stack arguments */ + sub %r11, %rsp + /* store memory args */ + movq %rdi, %r11 /* func ptr */ + movq %r10, %rcx /* counter */ + lea 128+48-8(%rsi,%rcx,8), %r10 + sub %rsp, %r10 + cmpq $0, %rcx + je push_args_end +push_args: + push 0(%rsp,%r10) + loop push_args +push_args_end: + /* fill all fp args */ + movdqu 0x00(%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + + /* fill all int args */ + movq 0x80(%rsi), %rdi + movq 0x90(%rsi), %rdx + movq 0x98(%rsi), %rcx + movq 0xa0(%rsi), %r8 + movq 0xa8(%rsi), %r9 + movq 0x88(%rsi), %rsi + + call *%r11 + leave + ret + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_general.c b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_general.c new file mode 100644 index 0000000..4799c9f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_general.c @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../wasm_runtime_common.h" +#include "../wasm_exec_env.h" + +void +invokeNative(void (*native_code)(), uint32 argv[], uint32 argc) +{ + bh_assert(argc >= sizeof(WASMExecEnv *) / sizeof(uint32)); + + switch (argc) { + case 0: + native_code(); + break; + case 1: + native_code(argv[0]); + break; + case 2: + native_code(argv[0], argv[1]); + break; + case 3: + native_code(argv[0], argv[1], argv[2]); + break; + case 4: + native_code(argv[0], argv[1], argv[2], argv[3]); + break; + case 5: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4]); + break; + case 6: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]); + break; + case 7: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6]); + break; + case 8: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7]); + break; + case 9: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8]); + break; + case 10: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9]); + break; + case 11: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10]); + break; + case 12: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]); + break; + case 13: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12]); + break; + case 14: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12], argv[13]); + break; + case 15: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12], argv[13], argv[14]); + break; + case 16: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12], argv[13], argv[14], argv[15]); + break; + case 17: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12], argv[13], argv[14], argv[15], argv[16]); + break; + case 18: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12], argv[13], argv[14], argv[15], argv[16], + argv[17]); + break; + case 19: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12], argv[13], argv[14], argv[15], argv[16], + argv[17], argv[18]); + break; + case 20: + native_code(argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], + argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], + argv[12], argv[13], argv[14], argv[15], argv[16], + argv[17], argv[18], argv[19]); + break; + default: + { + /* FIXME: If this happen, add more cases. */ + WASMExecEnv *exec_env = *(WASMExecEnv **)argv; + WASMModuleInstanceCommon *module_inst = exec_env->module_inst; + wasm_runtime_set_exception( + module_inst, + "the argument number of native function exceeds maximum"); + return; + } + } +} diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_ia32.asm b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_ia32.asm new file mode 100644 index 0000000..c52c8d6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_ia32.asm @@ -0,0 +1,27 @@ +; +; Copyright (C) 2019 Intel Corporation. All rights reserved. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; + + .386 + .model flat + .code +_invokeNative PROC + push ebp + mov ebp,esp + mov ecx, [ebp+16] ; ecx = argc */ + mov edx, [ebp+12] ; edx = argv */ + test ecx, ecx + jz skip_push_args ; if ecx == 0, skip pushing arguments */ + lea edx, [edx+ecx*4-4] ; edx = edx + ecx * 4 - 4 */ + sub edx,esp ; edx = edx - esp */ +loop_push: + push [esp+edx] + loop loop_push ; loop ecx counts */ +skip_push_args: + mov edx, [ebp+8] ; edx = func_ptr */ + call edx + leave + ret +_invokeNative ENDP +END \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_ia32.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_ia32.s new file mode 100644 index 0000000..de1c1a5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_ia32.s @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN +.globl invokeNative + .type invokeNative, @function +invokeNative: +#else +.globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + push %ebp + movl %esp, %ebp + movl 16(%ebp), %ecx /* ecx = argc */ + leal 2(%ecx), %edx /* edx = ecx + 2 (count return address and saved ebp) */ + andl $3, %edx /* edx = edx % 4 */ + jz stack_aligned /* if edx == 0, stack is already 16 bytes aligned */ + leal -16(%esp, %edx, 4), %esp /* esp = esp - 16 + edx * 4 */ +stack_aligned: + test %ecx, %ecx + jz skip_push_args /* if ecx == 0, skip pushing arguments */ + movl 12(%ebp), %edx /* edx = argv */ + leal -4(%edx,%ecx,4), %edx /* edx = edx + ecx * 4 - 4 */ + subl %esp, %edx /* edx = edx - esp */ +1: + push 0(%esp,%edx) + loop 1b /* loop ecx counts */ +skip_push_args: + movl 8(%ebp), %edx /* edx = func_ptr */ + call *%edx + leave + ret + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mingw_x64.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mingw_x64.s new file mode 100644 index 0000000..cefaa28 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mingw_x64.s @@ -0,0 +1,57 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +.text +.align 2 +.globl invokeNative +invokeNative: + + # %rcx func_ptr + # %rdx argv + # %r8 n_stacks + + push %rbp + mov %rsp, %rbp + + mov %rcx, %r10 # func_ptr + mov %rdx, %rax # argv + mov %r8, %rcx # n_stacks + + # fill all fp args + movsd 0(%rax), %xmm0 + movsd 8(%rax), %xmm1 + movsd 16(%rax), %xmm2 + movsd 24(%rax), %xmm3 + + # check for stack args + cmp $0, %rcx + jz cycle_end + + mov %rsp, %rdx + and $15, %rdx + jz no_abort + int $3 +no_abort: + mov %rcx, %rdx + and $1, %rdx + shl $3, %rdx + sub %rdx, %rsp + + # store stack args + lea 56(%rax, %rcx, 8), %r9 + sub %rsp, %r9 # offset +cycle: + push (%rsp, %r9) + loop cycle + +cycle_end: + mov 32(%rax), %rcx + mov 40(%rax), %rdx + mov 48(%rax), %r8 + mov 56(%rax), %r9 + + sub $32, %rsp # shadow space + + call *%r10 + leave + ret diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mingw_x64_simd.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mingw_x64_simd.s new file mode 100644 index 0000000..48ae524 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mingw_x64_simd.s @@ -0,0 +1,57 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +.text +.align 2 +.globl invokeNative +invokeNative: + + # %rcx func_ptr + # %rdx argv + # %r8 n_stacks + + push %rbp + mov %rsp, %rbp + + mov %rcx, %r10 # func_ptr + mov %rdx, %rax # argv + mov %r8, %rcx # n_stacks + + # fill all fp args + movdqu 0(%rax), %xmm0 + movdqu 16(%rax), %xmm1 + movdqu 32(%rax), %xmm2 + movdqu 48(%rax), %xmm3 + + # check for stack args + cmp $0, %rcx + jz cycle_end + + mov %rsp, %rdx + and $15, %rdx + jz no_abort + int $3 +no_abort: + mov %rcx, %rdx + and $1, %rdx + shl $3, %rdx + sub %rdx, %rsp + + # store stack args + lea 88(%rax, %rcx, 8), %r9 + sub %rsp, %r9 # offset +cycle: + push (%rsp, %r9) + loop cycle + +cycle_end: + mov 64(%rax), %rcx + mov 72(%rax), %rdx + mov 80(%rax), %r8 + mov 88(%rax), %r9 + + sub $32, %rsp # shadow space + + call *%r10 + leave + ret diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mips.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mips.s new file mode 100644 index 0000000..645f4f2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_mips.s @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + + .text + .align 2 + .globl invokeNative + .ent invokeNative + .type invokeNative, @function + +/** + * On function entry parameters: + * $4 = func_ptr + * $5 = args + * $6 = arg_num + */ + +invokeNative: + .frame $fp, 8, $0 + .mask 0x00000000, 0 + .fmask 0x00000000, 0 + + /* Fixed part of frame */ + subu $sp, 8 + + /* save registers */ + sw $31, 4($sp) + sw $fp, 0($sp) + + /* set frame pointer to bottom of fixed frame */ + move $fp, $sp + + /* allocate enough stack space */ + sll $11, $6, 2 /* $11 == arg_num * 4 */ + subu $sp, $11 + + /* make 8-byte aligned */ + and $sp, ~7 + + move $9, $sp + move $25, $4 /* $25 = func_ptr */ + +push_args: + beq $6, 0, done /* arg_num == 0 ? */ + lw $8, 0($5) /* $8 = *args */ + sw $8, 0($9) /* store $8 to stack */ + addu $5, 4 /* args++ */ + addu $9, 4 /* sp++ */ + subu $6, 1 /* arg_num-- */ + j push_args + +done: + lw $4, 0($sp) /* Load $4..$7 from stack */ + lw $5, 4($sp) + lw $6, 8($sp) + lw $7, 12($sp) + ldc1 $f12, 0($sp) /* Load $f12, $f13, $f14, $f15 */ + ldc1 $f14, 8($sp) + + jalr $25 /* call function */ + + nop + + /* restore saved registers */ + move $sp, $fp + lw $31, 4($sp) + lw $fp, 0($sp) + + /* pop frame */ + addu $sp, $sp, 8 + + j $31 + .end invokeNative diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_osx_universal.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_osx_universal.s new file mode 100644 index 0000000..e2ca654 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_osx_universal.s @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#if defined(__aarch64__) +#if WASM_ENABLE_SIMD == 0 +#include "invokeNative_aarch64.s" +#else +#include "invokeNative_aarch64_simd.s" +#endif +#else +#if WASM_ENABLE_SIMD == 0 +#include "invokeNative_em64.s" +#else +#include "invokeNative_em64_simd.s" +#endif +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_riscv.S b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_riscv.S new file mode 100644 index 0000000..0908f73 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_riscv.S @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* + * The float abi macros used bellow are from risc-v c api: + * https://github.com/riscv/riscv-c-api-doc/blob/master/riscv-c-api.md + * + */ + +#if defined(__riscv_float_abi_soft) +#define RV_FPREG_SIZE 0 +#elif defined(__riscv_float_abi_single) +#define RV_OP_LOADFPREG flw +#define RV_OP_STROEFPREG fsw +#define RV_FPREG_SIZE 4 +#elif defined(__riscv_float_abi_double) +#define RV_OP_LOADFPREG fld +#define RV_OP_STROEFPREG fsd +#define RV_FPREG_SIZE 8 +#endif + +#if __riscv_xlen == 32 +#define RV_OP_LOADREG lw +#define RV_OP_STOREREG sw +#define RV_REG_SIZE 4 +#define RV_REG_SHIFT 2 +#define RV_FP_OFFSET (8 * RV_REG_SIZE) +#define RV_INT_OFFSET 0 +#else +#define RV_OP_LOADREG ld +#define RV_OP_STOREREG sd +#define RV_REG_SIZE 8 +#define RV_REG_SHIFT 3 +#define RV_FP_OFFSET 0 +#define RV_INT_OFFSET (8 * RV_FPREG_SIZE) +#endif + + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * + * a0 function ptr + * a1 argv + * a2 nstacks + */ + +/* + * sp (stack pointer) + * |- sd/sw to store 64/32-bit values from register to memory + * |- ld/lw to load from stack to register + * fp/s0 (frame pointer) + * a0-a7 (8 integer arguments) + * |- sd/sw to store + * |- ld/lw to load + * fa0-a7 (8 float arguments) + * |- fsd/fsw to store + * |- fld/fsw to load + * t0-t6 (temporaries regisgers) + * |- caller saved + */ + + /* reserve space on stack to save return address and frame pointer */ + addi sp, sp, - 2 * RV_REG_SIZE + RV_OP_STOREREG fp, 0 * RV_REG_SIZE(sp) /* save frame pointer */ + RV_OP_STOREREG ra, 1 * RV_REG_SIZE(sp) /* save return address */ + + mv fp, sp /* set frame pointer to bottom of fixed frame */ + + /* save function ptr, argv & nstacks */ + mv t0, a0 /* t0 = function ptr */ + mv t1, a1 /* t1 = argv array address */ + mv t2, a2 /* t2 = nstack */ + +#ifndef __riscv_float_abi_soft + /* fill in fa0-7 float-registers*/ + RV_OP_LOADFPREG fa0, RV_FP_OFFSET + 0 * RV_FPREG_SIZE(t1) /* fa0 */ + RV_OP_LOADFPREG fa1, RV_FP_OFFSET + 1 * RV_FPREG_SIZE(t1) /* fa1 */ + RV_OP_LOADFPREG fa2, RV_FP_OFFSET + 2 * RV_FPREG_SIZE(t1) /* fa2 */ + RV_OP_LOADFPREG fa3, RV_FP_OFFSET + 3 * RV_FPREG_SIZE(t1) /* fa3 */ + RV_OP_LOADFPREG fa4, RV_FP_OFFSET + 4 * RV_FPREG_SIZE(t1) /* fa4 */ + RV_OP_LOADFPREG fa5, RV_FP_OFFSET + 5 * RV_FPREG_SIZE(t1) /* fa5 */ + RV_OP_LOADFPREG fa6, RV_FP_OFFSET + 6 * RV_FPREG_SIZE(t1) /* fa6 */ + RV_OP_LOADFPREG fa7, RV_FP_OFFSET + 7 * RV_FPREG_SIZE(t1) /* fa7 */ +#endif + + /* fill in a0-7 integer-registers*/ + RV_OP_LOADREG a0, RV_INT_OFFSET + 0 * RV_REG_SIZE(t1) /* a0 */ + RV_OP_LOADREG a1, RV_INT_OFFSET + 1 * RV_REG_SIZE(t1) /* a1 */ + RV_OP_LOADREG a2, RV_INT_OFFSET + 2 * RV_REG_SIZE(t1) /* a2 */ + RV_OP_LOADREG a3, RV_INT_OFFSET + 3 * RV_REG_SIZE(t1) /* a3 */ + RV_OP_LOADREG a4, RV_INT_OFFSET + 4 * RV_REG_SIZE(t1) /* a4 */ + RV_OP_LOADREG a5, RV_INT_OFFSET + 5 * RV_REG_SIZE(t1) /* a5 */ + RV_OP_LOADREG a6, RV_INT_OFFSET + 6 * RV_REG_SIZE(t1) /* a6 */ + RV_OP_LOADREG a7, RV_INT_OFFSET + 7 * RV_REG_SIZE(t1) /* a7 */ + + /* t1 points to stack args */ + + /* RV_FPREG_SIZE is zero when __riscv_float_abi_soft defined */ + addi t1, t1, RV_REG_SIZE * 8 + RV_FPREG_SIZE * 8 + + /* directly call the function if no args in stack, + x0 always holds 0 */ + beq t2, x0, call_func + + /* reserve enough stack space for function arguments */ + sll t3, t2, RV_REG_SHIFT /* shift left 3 bits. t3 = n_stacks * 8 */ + sub sp, sp, t3 + + /* make 16-byte aligned */ + li t3, 15 + not t3, t3 + and sp, sp, t3 + + /* save sp in t4 register */ + mv t4, sp + + /* copy left arguments from caller stack to own frame stack */ +loop_stack_args: + beq t2, x0, call_func + RV_OP_LOADREG t5, 0(t1) /* load stack argument, t5 = argv[i] */ + RV_OP_STOREREG t5, 0(t4) /* store t5 to reseved stack, sp[j] = t5 */ + addi t1, t1, RV_REG_SIZE /* move to next stack argument */ + addi t4, t4, RV_REG_SIZE /* move to next stack pointer */ + addi t2, t2, -1 /* decrease t2 every loop, nstacks = nstacks -1 */ + j loop_stack_args + +call_func: + jalr t0 + + /* restore registers pushed in stack or saved in another register */ +return: + mv sp, fp /* restore sp saved in fp before function call */ + RV_OP_LOADREG fp, 0 * RV_REG_SIZE(sp) /* load previous frame poniter to fp register */ + RV_OP_LOADREG ra, 1 * RV_REG_SIZE(sp) /* load previous return address to ra register */ + addi sp, sp, 2 * RV_REG_SIZE /* pop frame, restore sp */ + jr ra diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_thumb.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_thumb.s new file mode 100644 index 0000000..3669fe7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_thumb.s @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * + * r0 function ptr + * r1 argv + * r2 argc + */ + + push {r4, r5, r6, r7} + push {lr} + sub sp, sp, #4 /* make sp 8 byte aligned */ + mov ip, r0 /* ip = function ptr */ + mov r4, r1 /* r4 = argv */ + mov r5, r2 /* r5 = argc */ + + cmp r5, #1 /* at least one argument required: exec_env */ + blt return + + mov r6, #0 /* increased stack size */ + + ldr r0, [r4] /* r0 = argv[0] = exec_env */ + add r4, r4, #4 /* r4 += 4 */ + cmp r5, #1 + beq call_func + + ldr r1, [r4] /* r1 = argv[1] */ + add r4, r4, #4 + cmp r5, #2 + beq call_func + + ldr r2, [r4] /* r2 = argv[2] */ + add r4, r4, #4 + cmp r5, #3 + beq call_func + + ldr r3, [r4] /* r3 = argv[3] */ + add r4, r4, #4 + cmp r5, #4 + beq call_func + + sub r5, r5, #4 /* argc -= 4, now we have r0 ~ r3 */ + + /* Ensure address is 8 byte aligned */ + lsl r6, r5, #2 /* r6 = argc * 4 */ + mov r7, #7 + add r6, r6, r7 /* r6 = (r6 + 7) & ~7 */ + bic r6, r6, r7 + add r6, r6, #4 /* +4 because odd(5) registers are in stack */ + mov r7, sp + sub r7, r7, r6 /* reserved stack space for left arguments */ + mov sp, r7 + + mov lr, r2 /* save r2 */ +loop_args: /* copy left arguments to stack */ + cmp r5, #0 + beq call_func1 + ldr r2, [r4] + add r4, r4, #4 + str r2, [r7] + add r7, r7, #4 + sub r5, r5, #1 + b loop_args + +call_func1: + mov r2, lr /* restore r2 */ + +call_func: + blx ip + add sp, sp, r6 /* restore sp */ + +return: + add sp, sp, #4 /* make sp 8 byte aligned */ + pop {r3} + pop {r4, r5, r6, r7} + mov lr, r3 + bx lr diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_thumb_vfp.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_thumb_vfp.s new file mode 100644 index 0000000..218cd91 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_thumb_vfp.s @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 +#ifndef BH_PLATFORM_DARWIN + .globl invokeNative + .type invokeNative, function +invokeNative: +#else + .globl _invokeNative +_invokeNative: +#endif /* end of BH_PLATFORM_DARWIN */ + +/* + * Arguments passed in: + * + * r0 function ptr + * r1 argv + * r2 nstacks + */ + + push {r4, r5, r6, r7} + push {lr} + sub sp, sp, #4 /* make sp 8 byte aligned */ + mov ip, r0 /* ip = function ptr */ + mov r4, r1 /* r4 = argv */ + mov r5, r2 /* r5 = nstacks */ + mov r7, sp + + /* Fill all int args */ + ldr r0, [r4, #0] /* r0 = *(int*)&argv[0] = exec_env */ + ldr r1, [r4, #4] /* r1 = *(int*)&argv[1] */ + ldr r2, [r4, #8] /* r2 = *(int*)&argv[2] */ + ldr r3, [r4, #12] /* r3 = *(int*)&argv[3] */ + add r4, r4, #16 /* r4 points to float args */ + + /* Fill all float/double args to 16 single-precision registers, s0-s15, */ + /* which may also be accessed as 8 double-precision registers, d0-d7 (with */ + /* d0 overlapping s0, s1; d1 overlapping s2, s3; etc). */ + vldr s0, [r4, #0] /* s0 = *(float*)&argv[4] */ + vldr s1, [r4, #4] + vldr s2, [r4, #8] + vldr s3, [r4, #12] + vldr s4, [r4, #16] + vldr s5, [r4, #20] + vldr s6, [r4, #24] + vldr s7, [r4, #28] + vldr s8, [r4, #32] + vldr s9, [r4, #36] + vldr s10, [r4, #40] + vldr s11, [r4, #44] + vldr s12, [r4, #48] + vldr s13, [r4, #52] + vldr s14, [r4, #56] + vldr s15, [r4, #60] + /* Directly call the fucntion if no args in stack */ + cmp r5, #0 + beq call_func + + mov lr, r2 /* save r2 */ + + /* Fill all stack args: reserve stack space and fill ony by one */ + add r4, r4, #64 /* r4 points to stack args */ + mov r6, sp + mov r7, #7 + bic r6, r6, r7 /* Ensure stack is 8 byte aligned */ + lsl r2, r5, #2 /* r2 = nstacks * 4 */ + add r2, r2, #7 /* r2 = (r2 + 7) & ~7 */ + bic r2, r2, r7 + sub r6, r6, r2 /* reserved stack space for stack arguments */ + mov r7, sp + mov sp, r6 + +loop_stack_args: /* copy stack arguments to stack */ + cmp r5, #0 + beq call_func1 + ldr r2, [r4] /* Note: caller should insure int64 and */ + add r4, r4, #4 /* double are placed in 8 bytes aligned address */ + str r2, [r6] + add r6, r6, #4 + + sub r5, r5, #1 + b loop_stack_args + +call_func1: + mov r2, lr /* restore r2 */ + +call_func: + blx ip + mov sp, r7 /* restore sp */ + +return: + add sp, sp, #4 /* make sp 8 byte aligned */ + pop {r3} + pop {r4, r5, r6, r7} + mov lr, r3 + bx lr + diff --git a/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_xtensa.s b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_xtensa.s new file mode 100644 index 0000000..ce03f12 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/arch/invokeNative_xtensa.s @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + .text + .align 2 + .global invokeNative + .type invokeNative,function + +/* + * Arguments passed in: + * + * a2 function pntr + * a3 argv + * a4 argc + */ + +invokeNative: + entry a1, 256 + + blti a4, 1, return /* at least one argument required: exec_env */ + + /* register a10 ~ a15 are used to pass first 6 arguments */ + + l32i.n a10, a3, 0 + beqi a4, 1, call_func + + l32i.n a11, a3, 4 + beqi a4, 2, call_func + + l32i.n a12, a3, 8 + beqi a4, 3, call_func + + l32i.n a13, a3, 12 + beqi a4, 4, call_func + + l32i.n a14, a3, 16 + beqi a4, 5, call_func + + l32i.n a15, a3, 20 + beqi a4, 6, call_func + + /* left arguments are passed through stack */ + + addi a4, a4, -6 + addi a3, a3, 24 /* move argv pointer */ + mov.n a6, a1 /* store stack pointer */ + addi a7, a1, 256 /* stack boundary */ + +loop_args: + beqi a4, 0, call_func + bge a6, a7, call_func /* reach stack boundary */ + + l32i.n a5, a3, 0 /* load argument to a5 */ + s32i.n a5, a6, 0 /* push data to stack */ + + addi a4, a4, -1 /* decrease argc */ + addi a3, a3, 4 /* move argv pointer */ + addi a6, a6, 4 /* move stack pointer */ + + j loop_args + +call_func: + mov.n a8, a2 + callx8 a8 + + /* the result returned from callee is stored in a2 + mov the result to a10 so the caller of this function + can receive the value */ + mov.n a2, a10 + mov.n a3, a11 + +return: + retw.n diff --git a/wasm-micro-runtime/core/iwasm/common/gc/gc_common.c b/wasm-micro-runtime/core/iwasm/common/gc/gc_common.c new file mode 100644 index 0000000..99fca86 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/gc_common.c @@ -0,0 +1,1006 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../wasm_runtime_common.h" +#include "gc_export.h" +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif + +static bool +wasm_ref_type_normalize(wasm_ref_type_t *ref_type) +{ + wasm_value_type_t value_type = ref_type->value_type; + int32 heap_type = ref_type->heap_type; + + if (!((value_type >= VALUE_TYPE_I16 && value_type <= VALUE_TYPE_I32) + || ((value_type >= (uint8)REF_TYPE_ARRAYREF + && value_type <= (uint8)REF_TYPE_NULLFUNCREF) + || (value_type >= (uint8)REF_TYPE_HT_NULLABLE + && value_type <= (uint8)REF_TYPE_HT_NON_NULLABLE) +#if WASM_ENABLE_STRINGREF != 0 + || (value_type >= (uint8)REF_TYPE_STRINGVIEWWTF8 + && value_type <= (uint8)REF_TYPE_STRINGREF) + || (value_type >= (uint8)REF_TYPE_STRINGVIEWITER + && value_type <= (uint8)REF_TYPE_STRINGVIEWWTF16) +#endif + ))) { + return false; + } + if (value_type == VALUE_TYPE_HT_NULLABLE_REF + || value_type == VALUE_TYPE_HT_NON_NULLABLE_REF) { + if (heap_type < 0 && !wasm_is_valid_heap_type(heap_type)) { + return false; + } + } + + if (value_type != REF_TYPE_HT_NULLABLE) { + ref_type->nullable = false; + } + else { + if (wasm_is_valid_heap_type(heap_type)) { + ref_type->value_type = +#if WASM_ENABLE_STRINGREF != 0 + (uint8)(REF_TYPE_STRINGVIEWITER + heap_type + - HEAP_TYPE_STRINGVIEWITER); +#else + (uint8)(REF_TYPE_ARRAYREF + heap_type - HEAP_TYPE_ARRAY); +#endif + ref_type->nullable = false; + ref_type->heap_type = 0; + } + else { + ref_type->nullable = true; + } + } + + return true; +} + +uint32 +wasm_get_defined_type_count(WASMModuleCommon *const module) +{ + uint32 type_count = 0; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + type_count = wasm_module->type_count; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + type_count = aot_module->type_count; + } +#endif + + return type_count; +} + +WASMType * +wasm_get_defined_type(WASMModuleCommon *const module, uint32 index) +{ + WASMType *type = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + bh_assert(index < wasm_module->type_count); + type = wasm_module->types[index]; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + bh_assert(index < aot_module->type_count); + type = aot_module->types[index]; + } +#endif + + return type; +} + +WASMType * +wasm_obj_get_defined_type(const WASMObjectRef obj) +{ + if ((!wasm_obj_is_struct_obj(obj)) && (!wasm_obj_is_array_obj(obj)) + && (!wasm_obj_is_func_obj(obj))) { + bh_assert(false); + } + + return ((WASMRttTypeRef)(obj->header))->defined_type; +} + +int32 +wasm_obj_get_defined_type_idx(WASMModuleCommon *const module, + const WASMObjectRef obj) +{ + WASMType *type = wasm_obj_get_defined_type(obj); + uint32 i, type_idx = (uint32)-1; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + uint32 type_count = wasm_module->type_count; + + for (i = 0; i < type_count; i++) { + if (wasm_module->types[i] == type) { + type_idx = i; + break; + } + } + bh_assert(type_idx < type_count); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + uint32 type_count = aot_module->type_count; + + for (i = 0; i < type_count; i++) { + if (aot_module->types[i] == type) { + type_idx = i; + break; + } + } + bh_assert(type_idx < type_count); + } +#endif + + return type_idx; +} + +bool +wasm_defined_type_is_func_type(WASMType *const def_type) +{ + return wasm_type_is_func_type(def_type); +} + +bool +wasm_defined_type_is_struct_type(WASMType *const def_type) +{ + return wasm_type_is_struct_type(def_type); +} + +bool +wasm_defined_type_is_array_type(WASMType *const def_type) +{ + return wasm_type_is_array_type(def_type); +} + +uint32 +wasm_func_type_get_param_count(WASMFuncType *const func_type) +{ + return func_type->param_count; +} + +wasm_ref_type_t +wasm_func_type_get_param_type(WASMFuncType *const func_type, uint32 param_idx) +{ + wasm_ref_type_t ref_type = { 0 }; + + bh_assert(param_idx < func_type->param_count); + + ref_type.value_type = func_type->types[param_idx]; + + if (wasm_is_type_multi_byte_type(func_type->types[param_idx])) { + WASMRefType *param_ref_type = wasm_reftype_map_find( + func_type->ref_type_maps, func_type->ref_type_map_count, param_idx); + bh_assert(param_ref_type); + ref_type.nullable = param_ref_type->ref_ht_common.nullable; + ref_type.heap_type = param_ref_type->ref_ht_common.heap_type; + } + + return ref_type; +} + +uint32 +wasm_func_type_get_result_count(WASMFuncType *const func_type) +{ + return (uint32)func_type->result_count; +} + +wasm_ref_type_t +wasm_func_type_get_result_type(WASMFuncType *const func_type, uint32 result_idx) +{ + wasm_ref_type_t ref_type = { 0 }; + uint32 result_idx_with_param; + + result_idx_with_param = func_type->param_count + result_idx; + bh_assert(result_idx < func_type->result_count); + + ref_type.value_type = func_type->types[result_idx_with_param]; + + if (wasm_is_type_multi_byte_type(func_type->types[result_idx_with_param])) { + WASMRefType *result_ref_type = wasm_reftype_map_find( + func_type->ref_type_maps, func_type->ref_type_map_count, + result_idx_with_param); + bh_assert(result_ref_type); + ref_type.nullable = result_ref_type->ref_ht_common.nullable; + ref_type.heap_type = result_ref_type->ref_ht_common.heap_type; + } + + return ref_type; +} + +uint32 +wasm_struct_type_get_field_count(WASMStructType *const struct_type) +{ + bh_assert(struct_type->base_type.type_flag == WASM_TYPE_STRUCT); + return struct_type->field_count; +} + +wasm_ref_type_t +wasm_struct_type_get_field_type(WASMStructType *const struct_type, + uint32 field_idx, bool *p_is_mutable) +{ + wasm_ref_type_t ref_type = { 0 }; + WASMStructFieldType field; + + bh_assert(struct_type->base_type.type_flag == WASM_TYPE_STRUCT); + bh_assert(field_idx < struct_type->field_count); + + field = struct_type->fields[field_idx]; + ref_type.value_type = field.field_type; + + if (wasm_is_type_multi_byte_type(field.field_type)) { + WASMRefType *field_ref_type = + wasm_reftype_map_find(struct_type->ref_type_maps, + struct_type->ref_type_map_count, field_idx); + bh_assert(field_ref_type); + ref_type.nullable = field_ref_type->ref_ht_common.nullable; + ref_type.heap_type = field_ref_type->ref_ht_common.heap_type; + } + + if (p_is_mutable) { + *p_is_mutable = field.field_flags & 1; + } + + return ref_type; +} + +wasm_ref_type_t +wasm_array_type_get_elem_type(WASMArrayType *const array_type, + bool *p_is_mutable) +{ + wasm_ref_type_t ref_type = { 0 }; + + ref_type.value_type = array_type->elem_type; + + if (wasm_is_type_multi_byte_type(array_type->elem_type)) { + WASMRefType *elem_ref_type = array_type->elem_ref_type; + ref_type.nullable = elem_ref_type->ref_ht_common.nullable; + ref_type.heap_type = elem_ref_type->ref_ht_common.heap_type; + } + + if (p_is_mutable) { + *p_is_mutable = array_type->elem_flags & 1; + } + + return ref_type; +} + +bool +wasm_defined_type_equal(WASMType *const def_type1, WASMType *const def_type2, + WASMModuleCommon *const module) +{ + WASMTypePtr *types = NULL; + uint32 type_count = 0; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + types = wasm_module->types; + type_count = wasm_module->type_count; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + types = aot_module->types; + type_count = aot_module->type_count; + } +#endif + + bh_assert(types); + + return wasm_type_equal(def_type1, def_type2, types, type_count); +} + +bool +wasm_defined_type_is_subtype_of(WASMType *const def_type1, + WASMType *const def_type2, + WASMModuleCommon *const module) +{ + WASMTypePtr *types = NULL; + uint32 type_count = 0; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + types = wasm_module->types; + type_count = wasm_module->type_count; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + types = aot_module->types; + type_count = aot_module->type_count; + } +#endif + + bh_assert(types); + + return wasm_type_is_subtype_of(def_type1, def_type2, types, type_count); +} + +void +wasm_ref_type_set_type_idx(wasm_ref_type_t *ref_type, bool nullable, + int32 type_idx) +{ + bh_assert(type_idx >= 0); + ref_type->value_type = + nullable ? VALUE_TYPE_HT_NULLABLE_REF : VALUE_TYPE_HT_NON_NULLABLE_REF; + ref_type->nullable = nullable; + ref_type->heap_type = type_idx; +} + +void +wasm_ref_type_set_heap_type(wasm_ref_type_t *ref_type, bool nullable, + int32 heap_type) +{ + bool ret; + + bh_assert(heap_type <= HEAP_TYPE_FUNC && heap_type >= HEAP_TYPE_NONE); + ref_type->value_type = + nullable ? VALUE_TYPE_HT_NULLABLE_REF : VALUE_TYPE_HT_NON_NULLABLE_REF; + ref_type->nullable = nullable; + ref_type->heap_type = heap_type; + ret = wasm_ref_type_normalize(ref_type); + bh_assert(ret); + (void)ret; +} + +bool +wasm_ref_type_equal(const wasm_ref_type_t *ref_type1, + const wasm_ref_type_t *ref_type2, + WASMModuleCommon *const module) +{ + wasm_ref_type_t ref_type1_norm = { 0 }; + wasm_ref_type_t ref_type2_norm = { 0 }; + uint32 type_count = 0; + WASMTypePtr *types = NULL; + uint8 type1; + uint8 type2; + + bh_memcpy_s(&ref_type1_norm, (uint32)sizeof(wasm_ref_type_t), ref_type1, + (uint32)sizeof(wasm_ref_type_t)); + bh_memcpy_s(&ref_type2_norm, (uint32)sizeof(wasm_ref_type_t), ref_type2, + (uint32)sizeof(wasm_ref_type_t)); + if (!wasm_ref_type_normalize(&ref_type1_norm)) { + return false; + } + if (!wasm_ref_type_normalize(&ref_type2_norm)) { + return false; + } + type1 = ref_type1_norm.value_type; + type2 = ref_type2_norm.value_type; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + types = ((WASMModule *)module)->types; + type_count = wasm_get_defined_type_count(module); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + types = ((AOTModule *)module)->types; + type_count = wasm_get_defined_type_count(module); + } +#endif + + return wasm_reftype_equal(type1, (WASMRefType *)&ref_type1_norm, type2, + (WASMRefType *)&ref_type2_norm, types, + type_count); +} + +bool +wasm_ref_type_is_subtype_of(const wasm_ref_type_t *ref_type1, + const wasm_ref_type_t *ref_type2, + WASMModuleCommon *const module) +{ + wasm_ref_type_t ref_type1_norm = { 0 }; + wasm_ref_type_t ref_type2_norm = { 0 }; + uint8 type1; + uint8 type2; + WASMTypePtr *types = NULL; + uint32 type_count = 0; + + bh_memcpy_s(&ref_type1_norm, (uint32)sizeof(wasm_ref_type_t), ref_type1, + (uint32)sizeof(wasm_ref_type_t)); + bh_memcpy_s(&ref_type2_norm, (uint32)sizeof(wasm_ref_type_t), ref_type2, + (uint32)sizeof(wasm_ref_type_t)); + if (!wasm_ref_type_normalize(&ref_type1_norm)) { + return false; + } + if (!wasm_ref_type_normalize(&ref_type2_norm)) { + return false; + } + type1 = ref_type1_norm.value_type; + type2 = ref_type2_norm.value_type; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + types = ((WASMModule *)module)->types; + type_count = wasm_get_defined_type_count(module); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + types = ((AOTModule *)module)->types; + type_count = wasm_get_defined_type_count(module); + } +#endif + + bh_assert(types); + + return wasm_reftype_is_subtype_of(type1, (WASMRefType *)&ref_type1_norm, + type2, (WASMRefType *)&ref_type2_norm, + types, type_count); +} + +WASMStructObjectRef +wasm_struct_obj_new_with_typeidx(WASMExecEnv *exec_env, uint32 type_idx) +{ + WASMStructObjectRef struct_obj; + WASMModuleInstanceCommon *module_inst = + wasm_runtime_get_module_inst(exec_env); + WASMType *type = NULL; + WASMRttTypeRef rtt_type = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModule *module = ((WASMModuleInstance *)module_inst)->module; + + bh_assert(type_idx < module->type_count); + type = module->types[type_idx]; + bh_assert(wasm_defined_type_is_struct_type(type)); + rtt_type = + wasm_rtt_type_new(type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)module_inst)->module; + + bh_assert(type_idx < module->type_count); + type = module->types[type_idx]; + bh_assert(wasm_defined_type_is_struct_type(type)); + rtt_type = + wasm_rtt_type_new(type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock); + } +#endif + + if (!rtt_type) { + return NULL; + } + struct_obj = wasm_struct_obj_new(exec_env, rtt_type); + + return struct_obj; +} + +WASMStructObjectRef +wasm_struct_obj_new_with_type(WASMExecEnv *exec_env, WASMStructType *type) +{ + WASMStructObjectRef struct_obj; + WASMModuleInstanceCommon *module_inst = + wasm_runtime_get_module_inst(exec_env); + WASMRttTypeRef rtt_type = NULL; + uint32 i = 0; + uint32 type_count = 0; + + bh_assert(type->base_type.type_flag == WASM_TYPE_STRUCT); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModule *module = ((WASMModuleInstance *)module_inst)->module; + + type_count = module->type_count; + + for (i = 0; i < type_count; i++) { + if (module->types[i] == (WASMType *)type) { + break; + } + } + bh_assert(i < type_count); + rtt_type = + wasm_rtt_type_new((WASMType *)type, i, module->rtt_types, + module->type_count, &module->rtt_type_lock); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)module_inst)->module; + + type_count = module->type_count; + + for (i = 0; i < type_count; i++) { + if (module->types[i] == (AOTType *)type) { + break; + } + } + bh_assert(i < type_count); + rtt_type = + wasm_rtt_type_new((AOTType *)type, i, module->rtt_types, + module->type_count, &module->rtt_type_lock); + } +#endif + + if (!rtt_type) { + return NULL; + } + struct_obj = wasm_struct_obj_new(exec_env, rtt_type); + + return struct_obj; +} + +WASMArrayObjectRef +wasm_array_obj_new_with_typeidx(WASMExecEnv *exec_env, uint32 type_idx, + uint32 length, wasm_value_t *init_value) +{ + WASMArrayObjectRef array_obj; + WASMModuleCommon *module = wasm_exec_env_get_module(exec_env); + WASMType *defined_type = wasm_get_defined_type(module, type_idx); + WASMRttTypeRef rtt_type = NULL; + + bh_assert(wasm_defined_type_is_array_type(defined_type)); + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + rtt_type = wasm_rtt_type_new( + defined_type, type_idx, wasm_module->rtt_types, + wasm_module->type_count, &wasm_module->rtt_type_lock); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + rtt_type = wasm_rtt_type_new( + defined_type, type_idx, aot_module->rtt_types, + aot_module->type_count, &aot_module->rtt_type_lock); + } +#endif + + if (!rtt_type) { + return NULL; + } + array_obj = wasm_array_obj_new(exec_env, rtt_type, length, init_value); + + return array_obj; +} + +WASMArrayObjectRef +wasm_array_obj_new_with_type(WASMExecEnv *exec_env, WASMArrayType *type, + uint32 length, wasm_value_t *init_value) +{ + WASMArrayObjectRef array_obj; + uint32 i, type_count, type_idx = 0; + WASMModuleCommon *module = wasm_exec_env_get_module(exec_env); + + bh_assert(type->base_type.type_flag == WASM_TYPE_ARRAY); + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + type_count = wasm_module->type_count; + for (i = 0; i < type_count; i++) { + if (wasm_module->types[i] == (WASMType *)type) { + break; + } + } + bh_assert(i < wasm_module->type_count); + + type_idx = i; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + type_count = aot_module->type_count; + for (i = 0; i < type_count; i++) { + if (aot_module->types[i] == (AOTType *)type) { + break; + } + } + bh_assert(i < aot_module->type_count); + + type_idx = i; + } +#endif + + array_obj = + wasm_array_obj_new_with_typeidx(exec_env, type_idx, length, init_value); + + return array_obj; +} + +WASMFuncObjectRef +wasm_func_obj_new_with_typeidx(WASMExecEnv *exec_env, uint32 type_idx, + uint32 func_idx_bound) +{ + WASMFuncObjectRef func_obj; + WASMRttTypeRef rtt_type = NULL; + WASMModuleCommon *module = wasm_exec_env_get_module(exec_env); + WASMType *defined_type = wasm_get_defined_type(module, type_idx); + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + rtt_type = wasm_rtt_type_new( + defined_type, type_idx, wasm_module->rtt_types, + wasm_module->type_count, &wasm_module->rtt_type_lock); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + rtt_type = wasm_rtt_type_new( + defined_type, type_idx, aot_module->rtt_types, + aot_module->type_count, &aot_module->rtt_type_lock); + } +#endif + + if (!rtt_type) { + return NULL; + } + func_obj = wasm_func_obj_new(exec_env, rtt_type, func_idx_bound); + + return func_obj; +} + +WASMFuncObjectRef +wasm_func_obj_new_with_type(WASMExecEnv *exec_env, WASMFuncType *type, + uint32 func_idx_bound) +{ + WASMFuncObjectRef func_obj; + uint32 i, type_count, type_idx = 0; + WASMModuleCommon *module = wasm_exec_env_get_module(exec_env); + + bh_assert(type->base_type.type_flag == WASM_TYPE_FUNC); + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + type_count = wasm_module->type_count; + for (i = 0; i < type_count; i++) { + if (wasm_module->types[i] == (WASMType *)type) { + break; + } + } + bh_assert(i < wasm_module->type_count); + + type_idx = i; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + type_count = aot_module->type_count; + for (i = 0; i < type_count; i++) { + if (aot_module->types[i] == (AOTType *)type) { + break; + } + } + bh_assert(i < aot_module->type_count); + + type_idx = i; + } +#endif + + func_obj = + wasm_func_obj_new_with_typeidx(exec_env, type_idx, func_idx_bound); + + return func_obj; +} + +bool +wasm_runtime_call_func_ref(WASMExecEnv *exec_env, + const WASMFuncObjectRef func_obj, uint32 argc, + uint32 argv[]) +{ + WASMFunctionInstanceCommon *func_inst = NULL; + uint32 func_idx = wasm_func_obj_get_func_idx_bound(func_obj); +#if WASM_ENABLE_AOT != 0 + AOTFunctionInstance aot_func_inst = { 0 }; +#endif + +#if WASM_ENABLE_INTERP != 0 + if (exec_env->module_inst->module_type == Wasm_Module_Bytecode) { + WASMFunctionInstance *wasm_func_inst; + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + + bh_assert(func_idx < module_inst->module->import_function_count + + module_inst->module->function_count); + wasm_func_inst = module_inst->e->functions + func_idx; + func_inst = (WASMFunctionInstanceCommon *)wasm_func_inst; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (exec_env->module_inst->module_type == Wasm_Module_AoT) { + uint32 func_type_idx; + AOTModuleInstance *module_inst = + (AOTModuleInstance *)exec_env->module_inst; + AOTModule *module = (AOTModule *)module_inst->module; + (void)module_inst; + + bh_assert(func_idx < module->import_func_count + module->func_count); + + aot_func_inst.func_name = ""; + aot_func_inst.func_index = func_idx; + aot_func_inst.is_import_func = false; + func_type_idx = + module->func_type_indexes[func_idx - module->import_func_count]; + aot_func_inst.u.func.func_type = + (AOTFuncType *)module->types[func_type_idx]; + aot_func_inst.u.func.func_ptr = + module->func_ptrs[func_idx - module->import_func_count]; + + func_inst = (WASMFunctionInstanceCommon *)(&aot_func_inst); + } +#endif + + bh_assert(func_inst); + return wasm_runtime_call_wasm(exec_env, func_inst, argc, argv); +} + +bool +wasm_runtime_call_func_ref_a(WASMExecEnv *exec_env, + const WASMFuncObjectRef func_obj, + uint32 num_results, wasm_val_t results[], + uint32 num_args, wasm_val_t *args) +{ + /* TODO */ + return false; +} + +bool +wasm_runtime_call_func_ref_v(wasm_exec_env_t exec_env, + const WASMFuncObjectRef func_obj, + uint32 num_results, wasm_val_t results[], + uint32 num_args, ...) +{ + /* TODO */ + return false; +} + +bool +wasm_obj_is_instance_of_defined_type(WASMObjectRef obj, WASMType *defined_type, + WASMModuleCommon *const module) +{ + WASMType **types = NULL; + uint32 type_count = 0; + uint32 type_idx = 0; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + type_count = wasm_module->type_count; + types = wasm_module->types; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + type_count = aot_module->type_count; + types = (WASMType **)aot_module->types; + } +#endif + + for (type_idx = 0; type_idx < type_count; type_idx++) { + if (types[type_idx] == defined_type) { + break; + } + } + bh_assert(type_idx < type_count); + + return wasm_obj_is_instance_of(obj, type_idx, types, type_count); +} + +bool +wasm_obj_is_instance_of_type_idx(WASMObjectRef obj, uint32 type_idx, + WASMModuleCommon *const module) +{ + WASMType **types = NULL; + uint32 type_count = 0; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + + types = wasm_module->types; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + types = (WASMType **)aot_module->types; + } +#endif + + bh_assert(types); + + return wasm_obj_is_instance_of(obj, type_idx, types, type_count); +} + +bool +wasm_obj_is_instance_of_ref_type(const WASMObjectRef obj, + const wasm_ref_type_t *ref_type) +{ + int32 heap_type = ref_type->heap_type; + return wasm_obj_is_type_of(obj, heap_type); +} + +void +wasm_runtime_push_local_obj_ref(WASMExecEnv *exec_env, WASMLocalObjectRef *ref) +{ + ref->val = NULL; + ref->prev = exec_env->cur_local_object_ref; + exec_env->cur_local_object_ref = ref; +} + +WASMLocalObjectRef * +wasm_runtime_pop_local_obj_ref(WASMExecEnv *exec_env) +{ + WASMLocalObjectRef *local_ref = exec_env->cur_local_object_ref; + exec_env->cur_local_object_ref = exec_env->cur_local_object_ref->prev; + return local_ref; +} + +void +wasm_runtime_pop_local_obj_refs(WASMExecEnv *exec_env, uint32 n) +{ + bh_assert(n > 0); + + do { + exec_env->cur_local_object_ref = exec_env->cur_local_object_ref->prev; + } while (--n > 0); +} + +WASMLocalObjectRef * +wasm_runtime_get_cur_local_obj_ref(WASMExecEnv *exec_env) +{ + WASMLocalObjectRef *local_ref = exec_env->cur_local_object_ref; + + bh_assert(local_ref); + return local_ref; +} + +void +wasm_runtime_gc_prepare(WASMExecEnv *exec_env) +{ +#if 0 + /* TODO: implement wasm_runtime_gc_prepare for multi-thread */ + exec_env->is_gc_reclaiming = false; + wasm_thread_suspend_all(); + exec_env->is_gc_reclaim = 1; + exec_env->requesting_suspend = 0; +#endif +} + +void +wasm_runtime_gc_finalize(WASMExecEnv *exec_env) +{ +#if 0 + /* TODO: implement wasm_runtime_gc_finalize for multi-thread */ + wasm_thread_resume_all(); + exec_env->doing_gc_reclaim = 0; +#endif +} + +bool +wasm_runtime_get_wasm_object_ref_list(WASMObjectRef obj, + bool *p_is_compact_mode, + uint32 *p_ref_num, uint16 **p_ref_list, + uint32 *p_ref_start_offset) +{ + return wasm_object_get_ref_list(obj, p_is_compact_mode, p_ref_num, + p_ref_list, p_ref_start_offset); +} + +bool +wasm_runtime_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ +#if WASM_ENABLE_INTERP != 0 + if (exec_env->module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_traverse_gc_rootset(exec_env, heap); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (exec_env->module_inst->module_type == Wasm_Module_AoT) { + return aot_traverse_gc_rootset(exec_env, heap); + } +#endif + return false; +} + +void +wasm_runtime_set_gc_heap_handle(WASMModuleInstanceCommon *module_inst, + void *gc_heap_handle) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + ((WASMModuleInstance *)module_inst)->e->common.gc_heap_handle = + gc_heap_handle; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + e->common.gc_heap_handle = gc_heap_handle; + } +#endif +} + +void * +wasm_runtime_get_gc_heap_handle(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return ((WASMModuleInstance *)module_inst)->e->common.gc_heap_handle; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + return e->common.gc_heap_handle; + } +#endif + return NULL; +} + +bool +wasm_runtime_get_wasm_object_extra_info_flag(WASMObjectRef obj) +{ + return obj->header & WASM_OBJ_EXTRA_INFO_FLAG; +} + +void +wasm_runtime_set_wasm_object_extra_info_flag(WASMObjectRef obj, bool set) +{ + if (set) { + obj->header |= WASM_OBJ_EXTRA_INFO_FLAG; + } + else { + obj->header &= ~WASM_OBJ_EXTRA_INFO_FLAG; + } +} diff --git a/wasm-micro-runtime/core/iwasm/common/gc/gc_object.c b/wasm-micro-runtime/core/iwasm/common/gc/gc_object.c new file mode 100644 index 0000000..333effc --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/gc_object.c @@ -0,0 +1,1081 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "gc_object.h" +#include "mem_alloc.h" +#include "../wasm_runtime_common.h" +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif +#if WASM_ENABLE_STRINGREF != 0 +#include "string_object.h" +#endif + +WASMRttTypeRef +wasm_rtt_type_new(WASMType *defined_type, uint32 defined_type_idx, + WASMRttType **rtt_types, uint32 rtt_type_count, + korp_mutex *rtt_type_lock) +{ + WASMRttType *rtt_type; + + bh_assert(defined_type_idx < rtt_type_count); + + os_mutex_lock(rtt_type_lock); + + if (rtt_types[defined_type_idx]) { + os_mutex_unlock(rtt_type_lock); + return rtt_types[defined_type_idx]; + } + + if ((rtt_type = wasm_runtime_malloc(sizeof(WASMRttType)))) { + rtt_type->type_flag = defined_type->type_flag; + rtt_type->inherit_depth = defined_type->inherit_depth; + rtt_type->defined_type = defined_type; + rtt_type->root_type = defined_type->root_type; + + rtt_types[defined_type_idx] = rtt_type; + } + + os_mutex_unlock(rtt_type_lock); + return rtt_type; +} + +static void * +gc_obj_malloc(void *heap_handle, uint64 size) +{ + void *mem; + + if (size >= UINT32_MAX + || !(mem = mem_allocator_malloc_with_gc(heap_handle, (uint32)size))) { + LOG_WARNING("warning: failed to allocate memory for gc object"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +static void * +get_gc_heap_handle(WASMExecEnv *exec_env) +{ + void *gc_heap_handle = NULL; + WASMModuleInstanceCommon *module_inst = exec_env->module_inst; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + gc_heap_handle = + ((WASMModuleInstance *)module_inst)->e->common.gc_heap_handle; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + gc_heap_handle = + ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->common.gc_heap_handle; +#endif + + bh_assert(gc_heap_handle); + return gc_heap_handle; +} + +WASMStructObjectRef +wasm_struct_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type) +{ + WASMStructObjectRef struct_obj; + WASMStructType *struct_type; + + bh_assert(rtt_type->type_flag == WASM_TYPE_STRUCT); + + struct_type = (WASMStructType *)rtt_type->defined_type; + if (!(struct_obj = gc_obj_malloc(heap_handle, struct_type->total_size))) { + return NULL; + } + + struct_obj->header = (WASMObjectHeader)rtt_type; + + return struct_obj; +} + +WASMStructObjectRef +wasm_struct_obj_new(WASMExecEnv *exec_env, WASMRttTypeRef rtt_type) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + return wasm_struct_obj_new_internal(heap_handle, rtt_type); +} + +void +wasm_struct_obj_set_field(WASMStructObjectRef struct_obj, uint32 field_idx, + const WASMValue *value) +{ + WASMRttTypeRef rtt_type = + (WASMRttTypeRef)wasm_object_header((WASMObjectRef)struct_obj); + WASMStructType *struct_type = (WASMStructType *)rtt_type->defined_type; + WASMStructFieldType *field; + uint8 field_size, *field_data; + + bh_assert(field_idx < struct_type->field_count); + + field = struct_type->fields + field_idx; + field_data = (uint8 *)struct_obj + field->field_offset; + field_size = field->field_size; + + if (field_size == 4) { + *(int32 *)field_data = value->i32; + } + else if (field_size == 8) { +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_X86_32) + *(int64 *)field_data = value->i64; +#else + PUT_I64_TO_ADDR((uint32 *)field_data, value->i64); +#endif + } + else if (field_size == 1) { + *(int8 *)field_data = (int8)value->i32; + } + else if (field_size == 2) { + *(int16 *)field_data = (int16)value->i32; + } + else { + bh_assert(0); + } +} + +void +wasm_struct_obj_get_field(const WASMStructObjectRef struct_obj, + uint32 field_idx, bool sign_extend, WASMValue *value) +{ + WASMRttTypeRef rtt_type = + (WASMRttTypeRef)wasm_object_header((WASMObjectRef)struct_obj); + WASMStructType *struct_type = (WASMStructType *)rtt_type->defined_type; + WASMStructFieldType *field; + uint8 *field_data, field_size; + + bh_assert(field_idx < struct_type->field_count); + + field = struct_type->fields + field_idx; + field_data = (uint8 *)struct_obj + field->field_offset; + field_size = field->field_size; + + if (field_size == 4) { + value->i32 = *(int32 *)field_data; + } + else if (field_size == 8) { +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_X86_32) + value->i64 = *(int64 *)field_data; +#else + value->i64 = GET_I64_FROM_ADDR((uint32 *)field_data); +#endif + } + else if (field_size == 1) { + if (sign_extend) + value->i32 = (int32)(*(int8 *)field_data); + else + value->u32 = (uint32)(*(uint8 *)field_data); + } + else if (field_size == 2) { + if (sign_extend) + value->i32 = (int32)(*(int16 *)field_data); + else + value->u32 = (uint32)(*(uint16 *)field_data); + } + else { + bh_assert(0); + } +} + +uint32 +wasm_struct_obj_get_field_count(const WASMStructObjectRef struct_obj) +{ + WASMRttTypeRef rtt_type = + (WASMRttTypeRef)wasm_object_header((WASMObjectRef)struct_obj); + WASMStructType *struct_type = (WASMStructType *)rtt_type->defined_type; + + return struct_type->field_count; +} + +WASMArrayObjectRef +wasm_array_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type, + uint32 length, WASMValue *init_value) +{ + WASMArrayObjectRef array_obj; + WASMArrayType *array_type; + uint64 total_size; + uint32 elem_size, elem_size_log, i; + + bh_assert(rtt_type->type_flag == WASM_TYPE_ARRAY); + + if (length >= (1 << 29)) + return NULL; + + array_type = (WASMArrayType *)rtt_type->defined_type; + if (array_type->elem_type == PACKED_TYPE_I8) { + elem_size = 1; + elem_size_log = 0; + } + else if (array_type->elem_type == PACKED_TYPE_I16) { + elem_size = 2; + elem_size_log = 1; + } + else { + elem_size = wasm_value_type_size(array_type->elem_type); + elem_size_log = (elem_size == 4) ? 2 : 3; + } + + total_size = + offsetof(WASMArrayObject, elem_data) + (uint64)elem_size * length; + if (!(array_obj = gc_obj_malloc(heap_handle, total_size))) { + return NULL; + } + + array_obj->header = (WASMObjectHeader)rtt_type; + array_obj->length = (length << 2) | elem_size_log; + + if (init_value != NULL) { + for (i = 0; i < length; i++) { + if (wasm_is_type_reftype(array_type->elem_type)) { + uint32 *elem_addr = + (uint32 *)array_obj->elem_data + REF_CELL_NUM * i; + PUT_REF_TO_ADDR(elem_addr, init_value->gc_obj); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32) { + ((int32 *)array_obj->elem_data)[i] = init_value->i32; + } + else if (array_type->elem_type == PACKED_TYPE_I8) { + ((int8 *)array_obj->elem_data)[i] = (int8)init_value->i32; + } + else if (array_type->elem_type == PACKED_TYPE_I16) { + ((int16 *)array_obj->elem_data)[i] = (int16)init_value->i32; + } + else { + uint32 *elem_addr = (uint32 *)array_obj->elem_data + 2 * i; + PUT_I64_TO_ADDR(elem_addr, init_value->i64); + } + } + } + + return array_obj; +} + +WASMArrayObjectRef +wasm_array_obj_new(WASMExecEnv *exec_env, WASMRttTypeRef rtt_type, + uint32 length, WASMValue *init_value) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + return wasm_array_obj_new_internal(heap_handle, rtt_type, length, + init_value); +} + +void +wasm_array_obj_set_elem(WASMArrayObjectRef array_obj, uint32 elem_idx, + const WASMValue *value) +{ + uint8 *elem_data = wasm_array_obj_elem_addr(array_obj, elem_idx); + uint32 elem_size = 1 << wasm_array_obj_elem_size_log(array_obj); + + switch (elem_size) { + case 1: + *(int8 *)elem_data = (int8)value->i32; + break; + case 2: + *(int16 *)elem_data = (int16)value->i32; + break; + case 4: + *(int32 *)elem_data = value->i32; + break; + case 8: + PUT_I64_TO_ADDR((uint32 *)elem_data, value->i64); + break; + } +} + +void +wasm_array_obj_get_elem(const WASMArrayObjectRef array_obj, uint32 elem_idx, + bool sign_extend, WASMValue *value) +{ + uint8 *elem_data = wasm_array_obj_elem_addr(array_obj, elem_idx); + uint32 elem_size = 1 << wasm_array_obj_elem_size_log(array_obj); + + switch (elem_size) { + case 1: + value->i32 = sign_extend ? (int32)(*(int8 *)elem_data) + : (int32)(uint32)(*(uint8 *)elem_data); + break; + case 2: + value->i32 = sign_extend ? (int32)(*(int16 *)elem_data) + : (int32)(uint32)(*(uint16 *)elem_data); + break; + case 4: + value->i32 = *(int32 *)elem_data; + break; + case 8: + value->i64 = GET_I64_FROM_ADDR((uint32 *)elem_data); + break; + } +} + +void +wasm_array_obj_fill(const WASMArrayObjectRef array_obj, uint32 elem_idx, + uint32 len, WASMValue *value) +{ + uint32 i; + uint8 *elem_data = wasm_array_obj_elem_addr(array_obj, elem_idx); + uint32 elem_size = 1 << wasm_array_obj_elem_size_log(array_obj); + + if (elem_size == 1) { + memset(elem_data, (int8)value->i32, len); + return; + } + + for (i = 0; i < len; i++) { + switch (elem_size) { + case 2: + *(int16 *)elem_data = (int16)value->i32; + break; + case 4: + *(int32 *)elem_data = value->i32; + break; + case 8: + PUT_I64_TO_ADDR((uint32 *)elem_data, value->i64); + break; + } + elem_data += elem_size; + } +} + +void +wasm_array_obj_copy(WASMArrayObjectRef dst_obj, uint32 dst_idx, + WASMArrayObjectRef src_obj, uint32 src_idx, uint32 len) +{ + uint8 *dst_data = wasm_array_obj_elem_addr(dst_obj, dst_idx); + uint8 *src_data = wasm_array_obj_elem_addr(src_obj, src_idx); + uint32 elem_size = 1 << wasm_array_obj_elem_size_log(dst_obj); + + bh_memmove_s(dst_data, elem_size * len, src_data, elem_size * len); +} + +uint32 +wasm_array_obj_length(const WASMArrayObjectRef array_obj) +{ + return array_obj->length >> WASM_ARRAY_LENGTH_SHIFT; +} + +void * +wasm_array_obj_first_elem_addr(const WASMArrayObjectRef array_obj) +{ + return array_obj->elem_data; +} + +void * +wasm_array_obj_elem_addr(const WASMArrayObjectRef array_obj, uint32 elem_idx) +{ + return array_obj->elem_data + + (elem_idx << wasm_array_obj_elem_size_log(array_obj)); +} + +WASMFuncObjectRef +wasm_func_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type, + uint32 func_idx_bound) +{ + WASMFuncObjectRef func_obj; + uint64 total_size; + + bh_assert(rtt_type->type_flag == WASM_TYPE_FUNC); + + total_size = sizeof(WASMFuncObject); + if (!(func_obj = gc_obj_malloc(heap_handle, total_size))) { + return NULL; + } + + func_obj->header = (WASMObjectHeader)rtt_type; + func_obj->func_idx_bound = func_idx_bound; + + return func_obj; +} + +WASMFuncObjectRef +wasm_func_obj_new(WASMExecEnv *exec_env, WASMRttTypeRef rtt_type, + uint32 func_idx_bound) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + return wasm_func_obj_new_internal(heap_handle, rtt_type, func_idx_bound); +} + +uint32 +wasm_func_obj_get_func_idx_bound(const WASMFuncObjectRef func_obj) +{ + return func_obj->func_idx_bound; +} + +WASMFuncType * +wasm_func_obj_get_func_type(const WASMFuncObjectRef func_obj) +{ + WASMRttTypeRef rtt_type = + (WASMRttTypeRef)wasm_object_header((WASMObjectRef)func_obj); + bh_assert(rtt_type->type_flag == WASM_TYPE_FUNC); + return (WASMFuncType *)rtt_type->defined_type; +} + +WASMExternrefObjectRef +wasm_externref_obj_new(WASMExecEnv *exec_env, const void *host_obj) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + WASMAnyrefObjectRef anyref_obj; + WASMExternrefObjectRef externref_obj; + WASMLocalObjectRef local_ref; + + if (!(anyref_obj = gc_obj_malloc(heap_handle, sizeof(WASMAnyrefObject)))) { + return NULL; + } + + anyref_obj->header = WASM_OBJ_ANYREF_OBJ_FLAG; + anyref_obj->host_obj = host_obj; + + /* Lock anyref_obj in case it is reclaimed when allocating memory below */ + wasm_runtime_push_local_obj_ref(exec_env, &local_ref); + local_ref.val = (WASMObjectRef)anyref_obj; + + if (!(externref_obj = + gc_obj_malloc(heap_handle, sizeof(WASMExternrefObject)))) { + wasm_runtime_pop_local_obj_ref(exec_env); + return NULL; + } + + externref_obj->header = WASM_OBJ_EXTERNREF_OBJ_FLAG; + externref_obj->internal_obj = (WASMObjectRef)anyref_obj; + + wasm_runtime_pop_local_obj_ref(exec_env); + return externref_obj; +} + +WASMAnyrefObjectRef +wasm_anyref_obj_new(WASMExecEnv *exec_env, const void *host_obj) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + WASMAnyrefObjectRef anyref_obj; + + if (!(anyref_obj = gc_obj_malloc(heap_handle, sizeof(WASMAnyrefObject)))) { + return NULL; + } + + anyref_obj->header = WASM_OBJ_ANYREF_OBJ_FLAG; + anyref_obj->host_obj = host_obj; + + return anyref_obj; +} + +WASMObjectRef +wasm_externref_obj_to_internal_obj(WASMExternrefObjectRef externref_obj) +{ + return externref_obj->internal_obj; +} + +WASMExternrefObjectRef +wasm_internal_obj_to_externref_obj(WASMExecEnv *exec_env, + WASMObjectRef internal_obj) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + WASMExternrefObjectRef externref_obj; + + if (!(externref_obj = + gc_obj_malloc(heap_handle, sizeof(WASMExternrefObject)))) { + return NULL; + } + + externref_obj->header = WASM_OBJ_EXTERNREF_OBJ_FLAG; + externref_obj->internal_obj = internal_obj; + + return externref_obj; +} + +const void * +wasm_anyref_obj_get_value(WASMAnyrefObjectRef anyref_obj) +{ + return anyref_obj->host_obj; +} + +const void * +wasm_externref_obj_get_value(const WASMExternrefObjectRef externref_obj) +{ + if (wasm_obj_is_anyref_obj(externref_obj->internal_obj)) + return ((WASMAnyrefObjectRef)externref_obj->internal_obj)->host_obj; + else + return externref_obj->internal_obj; +} + +WASMI31ObjectRef +wasm_i31_obj_new(uint32 i31_value) +{ + return (WASMI31ObjectRef)((i31_value << 1) | 1); +} + +uint32 +wasm_i31_obj_get_value(WASMI31ObjectRef i31_obj, bool sign_extend) +{ + uint32 i31_value = (uint32)(((uintptr_t)i31_obj) >> 1); + if (sign_extend && (i31_value & 0x40000000)) /* bit 30 is 1 */ + /* set bit 31 to 1 */ + i31_value |= 0x80000000; + return i31_value; +} + +bool +wasm_obj_is_i31_obj(WASMObjectRef obj) +{ + bh_assert(obj); + return (((uintptr_t)obj) & 1) ? true : false; +} + +bool +wasm_obj_is_externref_obj(WASMObjectRef obj) +{ + bh_assert(obj); + return (!wasm_obj_is_i31_obj(obj) + && (obj->header & WASM_OBJ_EXTERNREF_OBJ_FLAG)) + ? true + : false; +} + +bool +wasm_obj_is_anyref_obj(WASMObjectRef obj) +{ + bh_assert(obj); + return (!wasm_obj_is_i31_obj(obj) + && (obj->header & WASM_OBJ_ANYREF_OBJ_FLAG)) + ? true + : false; +} + +bool +wasm_obj_is_i31_externref_or_anyref_obj(WASMObjectRef obj) +{ + bh_assert(obj); + return (wasm_obj_is_i31_obj(obj) + || (obj->header + & (WASM_OBJ_EXTERNREF_OBJ_FLAG | WASM_OBJ_ANYREF_OBJ_FLAG))) + ? true + : false; +} + +bool +wasm_obj_is_struct_obj(WASMObjectRef obj) +{ + WASMRttTypeRef rtt_type; + + bh_assert(obj); + + if (wasm_obj_is_i31_externref_or_anyref_obj(obj)) + return false; + + rtt_type = (WASMRttTypeRef)wasm_object_header(obj); + return rtt_type->type_flag == WASM_TYPE_STRUCT ? true : false; +} + +bool +wasm_obj_is_array_obj(WASMObjectRef obj) +{ + WASMRttTypeRef rtt_type; + + bh_assert(obj); + + if (wasm_obj_is_i31_externref_or_anyref_obj(obj)) + return false; + + rtt_type = (WASMRttTypeRef)wasm_object_header(obj); + return rtt_type->type_flag == WASM_TYPE_ARRAY ? true : false; +} + +bool +wasm_obj_is_func_obj(WASMObjectRef obj) +{ + WASMRttTypeRef rtt_type; + + bh_assert(obj); + + if (wasm_obj_is_i31_externref_or_anyref_obj(obj)) + return false; + + rtt_type = (WASMRttTypeRef)wasm_object_header(obj); + return rtt_type->type_flag == WASM_TYPE_FUNC ? true : false; +} + +bool +wasm_obj_is_internal_obj(WASMObjectRef obj) +{ + WASMRttTypeRef rtt_type; + + bh_assert(obj); + + if (wasm_obj_is_i31_obj(obj)) + return true; + else if (obj->header & WASM_OBJ_ANYREF_OBJ_FLAG) + return true; + else if (obj->header & WASM_OBJ_EXTERNREF_OBJ_FLAG) + return false; + else { + rtt_type = (WASMRttTypeRef)wasm_object_header(obj); + return (rtt_type->type_flag == WASM_TYPE_STRUCT + || rtt_type->type_flag == WASM_TYPE_ARRAY) + ? true + : false; + } +} + +bool +wasm_obj_is_eq_obj(WASMObjectRef obj) +{ + WASMRttTypeRef rtt_type; + + bh_assert(obj); + + if (wasm_obj_is_i31_obj(obj)) + return true; + else if ((obj->header & WASM_OBJ_ANYREF_OBJ_FLAG) + || (obj->header & WASM_OBJ_EXTERNREF_OBJ_FLAG)) + return false; + else { + rtt_type = (WASMRttTypeRef)wasm_object_header(obj); + return (rtt_type->type_flag == WASM_TYPE_STRUCT + || rtt_type->type_flag == WASM_TYPE_ARRAY) + ? true + : false; + } +} + +bool +wasm_obj_is_instance_of(WASMObjectRef obj, uint32 type_idx, WASMType **types, + uint32 type_count) +{ + WASMRttTypeRef rtt_type_sub; + WASMType *type_sub, *type_parent; + uint32 distance, i; + + bh_assert(obj); + bh_assert(type_idx < type_count); + + if (wasm_obj_is_i31_externref_or_anyref_obj(obj)) + return false; + + rtt_type_sub = (WASMRttTypeRef)wasm_object_header(obj); + type_parent = types[type_idx]; + + if (!(rtt_type_sub->root_type == type_parent->root_type + && rtt_type_sub->inherit_depth >= type_parent->inherit_depth)) + return false; + + type_sub = rtt_type_sub->defined_type; + distance = type_sub->inherit_depth - type_parent->inherit_depth; + + for (i = 0; i < distance; i++) { + type_sub = type_sub->parent_type; + } + + return (type_sub == type_parent) ? true : false; +} + +bool +wasm_obj_is_type_of(WASMObjectRef obj, int32 heap_type) +{ + bh_assert(obj); + + switch (heap_type) { + case HEAP_TYPE_FUNC: + return wasm_obj_is_func_obj(obj); + case HEAP_TYPE_EXTERN: + return wasm_obj_is_externref_obj(obj); + case HEAP_TYPE_ANY: + return wasm_obj_is_internal_obj(obj); + case HEAP_TYPE_EQ: + return wasm_obj_is_eq_obj(obj); + case HEAP_TYPE_I31: + return wasm_obj_is_i31_obj(obj); + case HEAP_TYPE_STRUCT: + return wasm_obj_is_struct_obj(obj); + case HEAP_TYPE_ARRAY: + return wasm_obj_is_array_obj(obj); +#if WASM_ENABLE_STRINGREF != 0 + case HEAP_TYPE_STRINGREF: + return wasm_obj_is_stringref_obj(obj); + case HEAP_TYPE_STRINGVIEWWTF8: + return wasm_obj_is_stringview_wtf8_obj(obj); + case HEAP_TYPE_STRINGVIEWWTF16: + return wasm_obj_is_stringview_wtf16_obj(obj); +#endif + case HEAP_TYPE_NONE: + case HEAP_TYPE_NOFUNC: + case HEAP_TYPE_NOEXTERN: + return false; + default: + bh_assert(0); + break; + } + return false; +} + +bool +wasm_obj_equal(WASMObjectRef obj1, WASMObjectRef obj2) +{ + /* TODO: do we need to compare the internal details of the objects */ + return obj1 == obj2 ? true : false; +} + +bool +wasm_object_get_ref_list(WASMObjectRef obj, bool *p_is_compact_mode, + uint32 *p_ref_num, uint16 **p_ref_list, + uint32 *p_ref_start_offset) +{ + WASMRttTypeRef rtt_type; + + bh_assert(wasm_obj_is_created_from_heap(obj)); + + rtt_type = (WASMRttTypeRef)wasm_object_header(obj); + + if (obj->header & WASM_OBJ_EXTERNREF_OBJ_FLAG) { + /* externref object */ + static uint16 externref_obj_ref_list[] = { (uint16)offsetof( + WASMExternrefObject, internal_obj) }; + *p_is_compact_mode = false; + *p_ref_num = 1; + *p_ref_list = externref_obj_ref_list; + return true; + } + else if (obj->header & WASM_OBJ_ANYREF_OBJ_FLAG) { + /* anyref object */ + *p_is_compact_mode = false; + *p_ref_num = 0; + *p_ref_list = NULL; + return true; + } +#if WASM_ENABLE_STRINGREF != 0 + else if (rtt_type->type_flag == WASM_TYPE_STRINGREF + || rtt_type->type_flag == WASM_TYPE_STRINGVIEWWTF8 + || rtt_type->type_flag == WASM_TYPE_STRINGVIEWWTF16 + || rtt_type->type_flag == WASM_TYPE_STRINGVIEWITER) { + /* stringref/stringview_wtf8/stringview_wtf16/stringview_iter object */ + *p_is_compact_mode = false; + *p_ref_num = 0; + *p_ref_list = NULL; + return true; + } +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + else if (rtt_type->defined_type->type_flag == WASM_TYPE_FUNC) { + /* function object */ + *p_is_compact_mode = false; + *p_ref_num = 0; + *p_ref_list = NULL; + return true; + } + else if (rtt_type->defined_type->type_flag == WASM_TYPE_STRUCT) { + /* struct object */ + WASMStructType *type = (WASMStructType *)rtt_type->defined_type; + *p_is_compact_mode = false; + *p_ref_num = *type->reference_table; + *p_ref_list = type->reference_table + 1; + return true; + } + else if (rtt_type->defined_type->type_flag == WASM_TYPE_ARRAY) { + /* array object */ + WASMArrayType *type = (WASMArrayType *)rtt_type->defined_type; + if (wasm_is_type_reftype(type->elem_type)) { + *p_is_compact_mode = true; + *p_ref_num = wasm_array_obj_length((WASMArrayObjectRef)obj); + *p_ref_start_offset = (uint16)offsetof(WASMArrayObject, elem_data); + } + else { + *p_is_compact_mode = false; + *p_ref_num = 0; + *p_ref_list = NULL; + } + + return true; + } + else { + bh_assert(0); + return false; + } +} + +bool +wasm_obj_set_gc_finalizer(wasm_exec_env_t exec_env, const wasm_obj_t obj, + wasm_obj_finalizer_t cb, void *data) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + return mem_allocator_set_gc_finalizer(heap_handle, obj, (gc_finalizer_t)cb, + data); +} + +void +wasm_obj_unset_gc_finalizer(wasm_exec_env_t exec_env, void *obj) +{ + void *heap_handle = get_gc_heap_handle(exec_env); + mem_allocator_unset_gc_finalizer(heap_handle, obj); +} + +#if WASM_ENABLE_STRINGREF != 0 +WASMRttTypeRef +wasm_stringref_rtt_type_new(uint16 type_flag, WASMRttType **rtt_types, + korp_mutex *rtt_type_lock) +{ + WASMRttType *rtt_type; + uint32 index; + + bh_assert(type_flag >= WASM_TYPE_STRINGREF + && type_flag <= WASM_TYPE_STRINGVIEWITER); + + index = type_flag - WASM_TYPE_STRINGREF; + + os_mutex_lock(rtt_type_lock); + + if (rtt_types[index]) { + os_mutex_unlock(rtt_type_lock); + return rtt_types[index]; + } + + if ((rtt_type = wasm_runtime_malloc(sizeof(WASMRttType)))) { + memset(rtt_type, 0, sizeof(WASMRttType)); + rtt_type->type_flag = type_flag; + + rtt_types[index] = rtt_type; + } + + os_mutex_unlock(rtt_type_lock); + return rtt_type; +} + +static void +wasm_stringref_obj_finalizer(WASMStringrefObjectRef stringref_obj, void *data) +{ + wasm_string_destroy( + (WASMString)wasm_stringref_obj_get_value(stringref_obj)); +} + +static void +wasm_stringview_wtf8_obj_finalizer(WASMStringviewWTF8ObjectRef stringref_obj, + void *data) +{ + wasm_string_destroy( + (WASMString)wasm_stringview_wtf8_obj_get_value(stringref_obj)); +} + +static void +wasm_stringview_wtf16_obj_finalizer(WASMStringviewWTF16ObjectRef stringref_obj, + void *data) +{ + wasm_string_destroy( + (WASMString)wasm_stringview_wtf16_obj_get_value(stringref_obj)); +} + +static void +wasm_stringview_iter_obj_finalizer(WASMStringviewIterObjectRef stringref_obj, + void *data) +{ + wasm_string_destroy( + (WASMString)wasm_stringview_iter_obj_get_value(stringref_obj)); +} + +static WASMObjectRef +stringref_obj_new(WASMExecEnv *exec_env, uint32 type, const void *str_obj, + int32 pos) +{ + WASMObjectRef stringref_obj = NULL; + void *heap_handle = get_gc_heap_handle(exec_env); + WASMModuleInstanceCommon *module_inst = + wasm_runtime_get_module_inst(exec_env); + WASMRttTypeRef rtt_type = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModule *module = ((WASMModuleInstance *)module_inst)->module; + rtt_type = wasm_stringref_rtt_type_new(type, module->stringref_rtts, + &module->rtt_type_lock); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)module_inst)->module; + rtt_type = wasm_stringref_rtt_type_new(type, module->stringref_rtts, + &module->rtt_type_lock); + } +#endif + + if (!rtt_type) { + return NULL; + } + + if (type == WASM_TYPE_STRINGREF) { + if (!(stringref_obj = + gc_obj_malloc(heap_handle, sizeof(WASMStringrefObject)))) { + return NULL; + } + ((WASMStringrefObjectRef)stringref_obj)->header = + (WASMObjectHeader)rtt_type; + ((WASMStringrefObjectRef)stringref_obj)->str_obj = str_obj; + wasm_obj_set_gc_finalizer( + exec_env, (wasm_obj_t)stringref_obj, + (wasm_obj_finalizer_t)wasm_stringref_obj_finalizer, NULL); + } + else if (type == WASM_TYPE_STRINGVIEWWTF8) { + if (!(stringref_obj = gc_obj_malloc( + heap_handle, sizeof(WASMStringviewWTF8Object)))) { + return NULL; + } + ((WASMStringviewWTF8ObjectRef)stringref_obj)->header = + (WASMObjectHeader)rtt_type; + ((WASMStringviewWTF8ObjectRef)stringref_obj)->str_obj = str_obj; + wasm_obj_set_gc_finalizer( + exec_env, (wasm_obj_t)stringref_obj, + (wasm_obj_finalizer_t)wasm_stringview_wtf8_obj_finalizer, NULL); + } + else if (type == WASM_TYPE_STRINGVIEWWTF16) { + if (!(stringref_obj = gc_obj_malloc( + heap_handle, sizeof(WASMStringviewWTF16Object)))) { + return NULL; + } + ((WASMStringviewWTF16ObjectRef)stringref_obj)->header = + (WASMObjectHeader)rtt_type; + ((WASMStringviewWTF16ObjectRef)stringref_obj)->str_obj = str_obj; + wasm_obj_set_gc_finalizer( + exec_env, (wasm_obj_t)stringref_obj, + (wasm_obj_finalizer_t)wasm_stringview_wtf16_obj_finalizer, NULL); + } + else if (type == WASM_TYPE_STRINGVIEWITER) { + if (!(stringref_obj = gc_obj_malloc( + heap_handle, sizeof(WASMStringviewIterObject)))) { + return NULL; + } + ((WASMStringviewIterObjectRef)stringref_obj)->header = + (WASMObjectHeader)rtt_type; + ((WASMStringviewIterObjectRef)stringref_obj)->str_obj = str_obj; + ((WASMStringviewIterObjectRef)stringref_obj)->pos = pos; + wasm_obj_set_gc_finalizer( + exec_env, (wasm_obj_t)stringref_obj, + (wasm_obj_finalizer_t)wasm_stringview_iter_obj_finalizer, NULL); + } + + return stringref_obj; +} + +WASMStringrefObjectRef +wasm_stringref_obj_new(WASMExecEnv *exec_env, const void *str_obj) +{ + WASMStringrefObjectRef stringref_obj; + + stringref_obj = (WASMStringrefObjectRef)stringref_obj_new( + exec_env, WASM_TYPE_STRINGREF, str_obj, 0); + + return stringref_obj; +} + +WASMStringviewWTF8ObjectRef +wasm_stringview_wtf8_obj_new(WASMExecEnv *exec_env, const void *str_obj) +{ + WASMStringviewWTF8ObjectRef stringview_wtf8_obj; + + stringview_wtf8_obj = (WASMStringviewWTF8ObjectRef)stringref_obj_new( + exec_env, WASM_TYPE_STRINGVIEWWTF8, str_obj, 0); + + return stringview_wtf8_obj; +} + +WASMStringviewWTF16ObjectRef +wasm_stringview_wtf16_obj_new(WASMExecEnv *exec_env, const void *str_obj) +{ + WASMStringviewWTF16ObjectRef stringview_wtf16_obj; + + stringview_wtf16_obj = (WASMStringviewWTF16ObjectRef)stringref_obj_new( + exec_env, WASM_TYPE_STRINGVIEWWTF16, str_obj, 0); + + return stringview_wtf16_obj; +} + +WASMStringviewIterObjectRef +wasm_stringview_iter_obj_new(WASMExecEnv *exec_env, const void *str_obj, + int32 pos) +{ + WASMStringviewIterObjectRef stringview_iter_obj; + + stringview_iter_obj = (WASMStringviewIterObjectRef)stringref_obj_new( + exec_env, WASM_TYPE_STRINGVIEWITER, str_obj, pos); + + return stringview_iter_obj; +} + +const void * +wasm_stringref_obj_get_value(WASMStringrefObjectRef stringref_obj) +{ + return stringref_obj->str_obj; +} + +const void * +wasm_stringview_wtf8_obj_get_value( + WASMStringviewWTF8ObjectRef stringview_wtf8_obj) +{ + return stringview_wtf8_obj->str_obj; +} + +const void * +wasm_stringview_wtf16_obj_get_value( + WASMStringviewWTF16ObjectRef stringview_wtf16_obj) +{ + return stringview_wtf16_obj->str_obj; +} + +const void * +wasm_stringview_iter_obj_get_value( + WASMStringviewIterObjectRef stringview_iter_obj) +{ + return stringview_iter_obj->str_obj; +} + +int32 +wasm_stringview_iter_obj_get_pos( + WASMStringviewIterObjectRef stringview_iter_obj) +{ + return stringview_iter_obj->pos; +} + +void +wasm_stringview_iter_obj_update_pos( + WASMStringviewIterObjectRef stringview_iter_obj, int32 pos) +{ + stringview_iter_obj->pos = pos; +} + +#define WASM_OBJ_IS_STRINGREF_IMPL(flag) \ + WASMRttTypeRef rtt_type; \ + \ + bh_assert(obj); \ + \ + if (wasm_obj_is_i31_externref_or_anyref_obj(obj)) \ + return false; \ + \ + rtt_type = (WASMRttTypeRef)wasm_object_header(obj); \ + return rtt_type->type_flag == flag ? true : false + +bool +wasm_obj_is_stringref_obj(WASMObjectRef obj) +{ + WASM_OBJ_IS_STRINGREF_IMPL(WASM_TYPE_STRINGREF); +} + +bool +wasm_obj_is_stringview_wtf8_obj(WASMObjectRef obj) +{ + WASM_OBJ_IS_STRINGREF_IMPL(WASM_TYPE_STRINGVIEWWTF8); +} + +bool +wasm_obj_is_stringview_wtf16_obj(WASMObjectRef obj) +{ + WASM_OBJ_IS_STRINGREF_IMPL(WASM_TYPE_STRINGVIEWWTF16); +} +#undef WASM_OBJ_IS_STRINGREF_IMPL + +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ diff --git a/wasm-micro-runtime/core/iwasm/common/gc/gc_object.h b/wasm-micro-runtime/core/iwasm/common/gc/gc_object.h new file mode 100644 index 0000000..2d13368 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/gc_object.h @@ -0,0 +1,388 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _GC_OBJECT_H_ +#define _GC_OBJECT_H_ + +#include "gc_type.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Object header of a WASM object, as the adddress of allocated memory + * must be 8-byte aligned, the lowest 3 bits are zero, we use them to + * mark the object: + * bits[0] is 1: the object is an externref object + * bits[1] is 1: the object is an anyref object + * bits[2] is 1: the object has extra information + * if both bits[0] and bits[1] are 0, then this object header must + * be a pointer of a WASMRttType, denotes that the object is a + * struct object, or an array object, or a function object + */ +typedef uintptr_t WASMObjectHeader; + +#define WASM_OBJ_HEADER_MASK (~((uintptr_t)7)) + +#define WASM_OBJ_EXTERNREF_OBJ_FLAG (((uintptr_t)1) << 0) + +#define WASM_OBJ_ANYREF_OBJ_FLAG (((uintptr_t)1) << 1) + +#define WASM_OBJ_EXTRA_INFO_FLAG (((uintptr_t)1) << 2) + +/* Representation of WASM objects */ +typedef struct WASMObject { + WASMObjectHeader header; +} WASMObject, *WASMObjectRef; + +/* Representation of WASM rtt types */ +typedef struct WASMRttType { + /* type_flag must be WASM_TYPE_FUNC/STRUCT/ARRAY to + denote an object of func, struct or array */ + uint32 type_flag; + uint32 inherit_depth; + WASMType *defined_type; + WASMType *root_type; +} WASMRttType, *WASMRttTypeRef; + +/* Representation of WASM externref objects */ +typedef struct WASMExternrefObject { + /* bits[0] must be 1, denotes an externref object */ + WASMObjectHeader header; + /* an object of WASMAnyrefObjectRef which encapsulates the external + object passed from host, or other internal objects passed to + opcode extern.externalize */ + WASMObjectRef internal_obj; +} WASMExternrefObject, *WASMExternrefObjectRef; + +/* Representation of WASM anyref objects which encapsulate the + external object passed from host */ +typedef struct WASMAnyrefObject { + /* bits[1] must be 1, denotes an anyref object */ + WASMObjectHeader header; + const void *host_obj; +} WASMAnyrefObject, *WASMAnyrefObjectRef; + +/** + * Representation of WASM i31 objects, the lowest bit is 1: + * for a pointer of WASMObjectRef, if the lowest bit is 1, then it is an + * i31 object and bits[1..31] stores the actual i31 value, otherwise + * it is a normal object of rtt/externref/struct/array/func */ +typedef uintptr_t WASMI31ObjectRef; + +/* Representation of WASM struct objects */ +typedef struct WASMStructObject { + /* Must be pointer of WASMRttObject of struct type */ + WASMObjectHeader header; + uint8 field_data[1]; +} WASMStructObject, *WASMStructObjectRef; + +/* Representation of WASM array objects */ +typedef struct WASMArrayObject { + /* Must be pointer of WASMRttObject of array type */ + WASMObjectHeader header; + /* ( << 2) | , + * elem_count = lenght >> 2 + * elem_size = 2 ^ (length & 0x3) + */ + uint32 length; + uint8 elem_data[1]; +} WASMArrayObject, *WASMArrayObjectRef; + +#define WASM_ARRAY_LENGTH_SHIFT 2 +#define WASM_ARRAY_ELEM_SIZE_MASK 3 + +/* Representation of WASM function objects */ +typedef struct WASMFuncObject { + /* must be pointer of WASMRttObject of func type */ + WASMObjectHeader header; + uint32 func_idx_bound; +} WASMFuncObject, *WASMFuncObjectRef; + +/* Representation of WASM stringref objects */ +typedef struct WASMStringrefObject { + WASMObjectHeader header; + const void *str_obj; +} WASMStringrefObject, *WASMStringrefObjectRef; + +typedef struct WASMStringviewWTF8Object { + WASMObjectHeader header; + const void *str_obj; +} WASMStringviewWTF8Object, *WASMStringviewWTF8ObjectRef; + +typedef struct WASMStringviewWTF16Object { + WASMObjectHeader header; + const void *str_obj; +} WASMStringviewWTF16Object, *WASMStringviewWTF16ObjectRef; + +typedef struct WASMStringviewIterObject { + WASMObjectHeader header; + const void *str_obj; + int32 pos; +} WASMStringviewIterObject, *WASMStringviewIterObjectRef; + +struct WASMExecEnv; + +inline static WASMObjectHeader +wasm_object_header(const WASMObjectRef obj) +{ + return (obj->header & WASM_OBJ_HEADER_MASK); +} + +WASMRttTypeRef +wasm_rtt_type_new(WASMType *defined_type, uint32 defined_type_idx, + WASMRttType **rtt_types, uint32 rtt_type_count, + korp_mutex *rtt_type_lock); + +inline static WASMType * +wasm_rtt_type_get_defined_type(const WASMRttTypeRef rtt_type) +{ + return rtt_type->defined_type; +} + +WASMStructObjectRef +wasm_struct_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type); + +WASMStructObjectRef +wasm_struct_obj_new(struct WASMExecEnv *exec_env, WASMRttTypeRef rtt_type); + +void +wasm_struct_obj_set_field(WASMStructObjectRef struct_obj, uint32 field_idx, + const WASMValue *value); + +void +wasm_struct_obj_get_field(const WASMStructObjectRef struct_obj, + uint32 field_idx, bool sign_extend, WASMValue *value); + +/** + * Return the field count of the WASM struct object. + * + * @param struct_obj the WASM struct object + * + * @return the field count of the WASM struct object + */ +uint32 +wasm_struct_obj_get_field_count(const WASMStructObjectRef struct_obj); + +WASMArrayObjectRef +wasm_array_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type, + uint32 length, WASMValue *init_value); + +WASMArrayObjectRef +wasm_array_obj_new(struct WASMExecEnv *exec_env, WASMRttTypeRef rtt_type, + uint32 length, WASMValue *init_value); + +void +wasm_array_obj_set_elem(WASMArrayObjectRef array_obj, uint32 elem_idx, + const WASMValue *value); + +void +wasm_array_obj_get_elem(const WASMArrayObjectRef array_obj, uint32 elem_idx, + bool sign_extend, WASMValue *value); + +void +wasm_array_obj_fill(const WASMArrayObjectRef array_obj, uint32 elem_idx, + uint32 len, WASMValue *value); + +void +wasm_array_obj_copy(WASMArrayObjectRef dst_obj, uint32 dst_idx, + WASMArrayObjectRef src_obj, uint32 src_idx, uint32 len); + +/** + * Return the logarithm of the size of array element. + * + * @param array the WASM array object + * + * @return log(size of the array element) + */ +inline static uint32 +wasm_array_obj_elem_size_log(const WASMArrayObjectRef array_obj) +{ + return (array_obj->length & WASM_ARRAY_ELEM_SIZE_MASK); +} + +/** + * Return the length of the array. + * + * @param array_obj the WASM array object + * + * @return the length of the array + */ +uint32 +wasm_array_obj_length(const WASMArrayObjectRef array_obj); + +/** + * Return the address of the first element of an array object. + * + * @param array_obj the WASM array object + * + * @return the address of the first element of the array object + */ +void * +wasm_array_obj_first_elem_addr(const WASMArrayObjectRef array_obj); + +/** + * Return the address of the i-th element of an array object. + * + * @param array_obj the WASM array object + * @param index the index of the element + * + * @return the address of the i-th element of the array object + */ +void * +wasm_array_obj_elem_addr(const WASMArrayObjectRef array_obj, uint32 elem_idx); + +WASMFuncObjectRef +wasm_func_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type, + uint32 func_idx_bound); + +WASMFuncObjectRef +wasm_func_obj_new(struct WASMExecEnv *exec_env, WASMRttTypeRef rtt_type, + uint32 func_idx_bound); + +uint32 +wasm_func_obj_get_func_idx_bound(const WASMFuncObjectRef func_obj); + +WASMFuncType * +wasm_func_obj_get_func_type(const WASMFuncObjectRef func_obj); + +WASMExternrefObjectRef +wasm_externref_obj_new(struct WASMExecEnv *exec_env, const void *host_obj); + +WASMAnyrefObjectRef +wasm_anyref_obj_new(struct WASMExecEnv *exec_env, const void *host_obj); + +/* Implementation of opcode extern.internalize */ +WASMObjectRef +wasm_externref_obj_to_internal_obj(const WASMExternrefObjectRef externref_obj); + +/* Implementation of opcode extern.externalize */ +WASMExternrefObjectRef +wasm_internal_obj_to_externref_obj(struct WASMExecEnv *exec_env, + const WASMObjectRef internal_obj); + +const void * +wasm_anyref_obj_get_value(const WASMAnyrefObjectRef anyref_obj); + +const void * +wasm_externref_obj_get_value(const WASMExternrefObjectRef externref_obj); + +WASMI31ObjectRef +wasm_i31_obj_new(uint32 i31_value); + +uint32 +wasm_i31_obj_get_value(WASMI31ObjectRef i31_obj, bool sign_extend); + +bool +wasm_obj_is_i31_obj(WASMObjectRef obj); + +bool +wasm_obj_is_externref_obj(WASMObjectRef obj); + +bool +wasm_obj_is_anyref_obj(WASMObjectRef obj); + +bool +wasm_obj_is_i31_externref_or_anyref_obj(WASMObjectRef obj); + +bool +wasm_obj_is_struct_obj(WASMObjectRef obj); + +bool +wasm_obj_is_array_obj(WASMObjectRef obj); + +bool +wasm_obj_is_func_obj(WASMObjectRef obj); + +bool +wasm_obj_is_internal_obj(WASMObjectRef obj); + +bool +wasm_obj_is_eq_obj(WASMObjectRef obj); + +inline static bool +wasm_obj_is_null_obj(WASMObjectRef obj) +{ + return obj == NULL_REF ? true : false; +} + +inline static bool +wasm_obj_is_created_from_heap(WASMObjectRef obj) +{ + if (obj == NULL || (((uintptr_t)obj) & 1)) + /* null object or i31 object */ + return false; + return true; +} + +bool +wasm_obj_is_instance_of(WASMObjectRef obj, uint32 type_idx, WASMType **types, + uint32 type_count); + +bool +wasm_obj_is_type_of(WASMObjectRef obj, int32 heap_type); + +bool +wasm_obj_equal(WASMObjectRef obj1, WASMObjectRef obj2); + +bool +wasm_object_get_ref_list(WASMObjectRef obj, bool *p_is_compact_mode, + uint32 *p_ref_num, uint16 **p_ref_list, + uint32 *p_ref_start_offset); + +#if WASM_ENABLE_STRINGREF != 0 +WASMStringrefObjectRef +wasm_stringref_obj_new(struct WASMExecEnv *exec_env, const void *str_obj); + +WASMStringviewWTF8ObjectRef +wasm_stringview_wtf8_obj_new(struct WASMExecEnv *exec_env, const void *str_obj); + +WASMStringviewWTF16ObjectRef +wasm_stringview_wtf16_obj_new(struct WASMExecEnv *exec_env, + const void *str_obj); + +WASMStringviewIterObjectRef +wasm_stringview_iter_obj_new(struct WASMExecEnv *exec_env, const void *str_obj, + int32 pos); + +const void * +wasm_stringref_obj_get_value(WASMStringrefObjectRef stringref_obj); + +const void * +wasm_stringview_wtf8_obj_get_value( + WASMStringviewWTF8ObjectRef stringview_wtf8_obj); + +const void * +wasm_stringview_wtf16_obj_get_value( + WASMStringviewWTF16ObjectRef stringview_wtf16_obj); + +const void * +wasm_stringview_iter_obj_get_value( + WASMStringviewIterObjectRef stringview_iter_obj); + +int32 +wasm_stringview_iter_obj_get_pos( + WASMStringviewIterObjectRef stringview_iter_obj); + +void +wasm_stringview_iter_obj_update_pos( + WASMStringviewIterObjectRef stringview_iter_obj, int32 pos); + +bool +wasm_obj_is_stringref_obj(WASMObjectRef obj); + +bool +wasm_obj_is_stringview_wtf8_obj(WASMObjectRef obj); + +bool +wasm_obj_is_stringview_wtf16_obj(WASMObjectRef obj); +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _GC_OBJECT_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/common/gc/gc_type.c b/wasm-micro-runtime/core/iwasm/common/gc/gc_type.c new file mode 100644 index 0000000..5ade1cb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/gc_type.c @@ -0,0 +1,1359 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "gc_type.h" + +void +wasm_dump_value_type(uint8 type, const WASMRefType *ref_type) +{ + switch (type) { + case VALUE_TYPE_I32: + os_printf("i32"); + break; + case VALUE_TYPE_I64: + os_printf("i64"); + break; + case VALUE_TYPE_F32: + os_printf("f32"); + break; + case VALUE_TYPE_F64: + os_printf("f64"); + break; + case VALUE_TYPE_V128: + os_printf("v128"); + break; + case PACKED_TYPE_I8: + os_printf("i8"); + break; + case PACKED_TYPE_I16: + os_printf("i16"); + break; + case REF_TYPE_FUNCREF: + os_printf("funcref"); + break; + case REF_TYPE_EXTERNREF: + os_printf("externref"); + break; + case REF_TYPE_ANYREF: + os_printf("anyref"); + break; + case REF_TYPE_EQREF: + os_printf("eqref"); + break; + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + { + os_printf("(ref "); + if (ref_type->ref_ht_common.nullable) + os_printf("null "); + if (wasm_is_refheaptype_common(&ref_type->ref_ht_common)) { + switch (ref_type->ref_ht_common.heap_type) { + case HEAP_TYPE_FUNC: + os_printf("func"); + break; + case HEAP_TYPE_EXTERN: + os_printf("extern"); + break; + case HEAP_TYPE_ANY: + os_printf("any"); + break; + case HEAP_TYPE_EQ: + os_printf("eq"); + break; + case HEAP_TYPE_I31: + os_printf("i31"); + break; + case HEAP_TYPE_STRUCT: + os_printf("struct"); + break; + case HEAP_TYPE_ARRAY: + os_printf("array"); + break; + case HEAP_TYPE_NONE: + os_printf("none"); + break; + case HEAP_TYPE_NOFUNC: + os_printf("nofunc"); + break; + case HEAP_TYPE_NOEXTERN: + os_printf("noextern"); + break; + default: + bh_assert(0); + break; + } + } + else if (wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common)) { + os_printf("%" PRId32, ref_type->ref_ht_typeidx.type_idx); + } + else { + bh_assert(0); + } + os_printf(")"); + break; + } + case REF_TYPE_I31REF: + os_printf("i31ref"); + break; + case REF_TYPE_STRUCTREF: + os_printf("structref"); + break; + case REF_TYPE_ARRAYREF: + os_printf("arrayref"); + break; + case REF_TYPE_NULLREF: + os_printf("nullref"); + break; + case REF_TYPE_NULLFUNCREF: + os_printf("nullfuncref"); + break; + case REF_TYPE_NULLEXTERNREF: + os_printf("nullexternref"); + break; + default: + bh_assert(0); + } +} + +void +wasm_dump_func_type(const WASMFuncType *type) +{ + uint32 i, j = 0; + const WASMRefType *ref_type = NULL; + + if (type->base_type.parent_type_idx != (uint32)-1) { + if (!type->base_type.is_sub_final) + os_printf("sub "); + else + os_printf("sub final "); + os_printf("%" PRIu32 " ", type->base_type.parent_type_idx); + } + + os_printf("func ["); + + for (i = 0; i < type->param_count; i++) { + if (wasm_is_type_multi_byte_type(type->types[i])) { + bh_assert(j < type->ref_type_map_count); + bh_assert(i == type->ref_type_maps[j].index); + ref_type = type->ref_type_maps[j++].ref_type; + } + else + ref_type = NULL; + wasm_dump_value_type(type->types[i], ref_type); + if (i < (uint32)type->param_count - 1) + os_printf(" "); + } + + os_printf("] -> ["); + + for (; i < (uint32)(type->param_count + type->result_count); i++) { + if (wasm_is_type_multi_byte_type(type->types[i])) { + bh_assert(j < type->ref_type_map_count); + bh_assert(i == type->ref_type_maps[j].index); + ref_type = type->ref_type_maps[j++].ref_type; + } + else + ref_type = NULL; + wasm_dump_value_type(type->types[i], ref_type); + if (i < (uint32)type->param_count + type->result_count - 1) + os_printf(" "); + } + + os_printf("]\n"); +} + +void +wasm_dump_struct_type(const WASMStructType *type) +{ + uint32 i, j = 0; + const WASMRefType *ref_type = NULL; + + if (type->base_type.parent_type_idx != (uint32)-1) { + if (!type->base_type.is_sub_final) + os_printf("sub "); + else + os_printf("sub final "); + os_printf("%" PRIu32 " ", type->base_type.parent_type_idx); + } + + os_printf("struct"); + + for (i = 0; i < type->field_count; i++) { + os_printf(" (field "); + if (type->fields[i].field_flags & 1) + os_printf("(mut "); + if (wasm_is_type_multi_byte_type(type->fields[i].field_type)) { + bh_assert(j < type->ref_type_map_count); + bh_assert(i == type->ref_type_maps[j].index); + ref_type = type->ref_type_maps[j++].ref_type; + } + else + ref_type = NULL; + wasm_dump_value_type(type->fields[i].field_type, ref_type); + if (type->fields[i].field_flags & 1) + os_printf(")"); + os_printf(")"); + } + + os_printf("\n"); +} + +void +wasm_dump_array_type(const WASMArrayType *type) +{ + if (type->base_type.parent_type_idx != (uint32)-1) { + if (!type->base_type.is_sub_final) + os_printf("sub "); + else + os_printf("sub final "); + os_printf("%" PRIu32 " ", type->base_type.parent_type_idx); + } + + os_printf("array "); + + if (type->elem_flags & 1) + os_printf("(mut "); + wasm_dump_value_type(type->elem_type, type->elem_ref_type); + if (type->elem_flags & 1) + os_printf(")"); + os_printf("\n"); +} + +bool +wasm_value_types_is_subtype_of(const uint8 *types1, + const WASMRefTypeMap *ref_type_maps1, + const uint8 *types2, + const WASMRefTypeMap *ref_type_maps2, + uint32 value_type_count, + const WASMTypePtr *types, uint32 type_count) +{ + uint32 i; + WASMRefType *ref_type1, *ref_type2; + + for (i = 0; i < value_type_count; i++) { + ref_type1 = ref_type2 = NULL; + if (wasm_is_type_multi_byte_type(types1[i])) { + ref_type1 = ref_type_maps1->ref_type; + ref_type_maps1++; + } + if (wasm_is_type_multi_byte_type(types2[i])) { + ref_type2 = ref_type_maps2->ref_type; + ref_type_maps2++; + } + if (!wasm_reftype_is_subtype_of(types1[i], ref_type1, types2[i], + ref_type2, types, type_count)) { + return false; + } + } + return true; +} + +static bool +rec_ref_type_equal(const WASMRefType *ref_type1, const WASMRefType *ref_type2, + uint32 rec_begin_type_idx1, uint32 rec_begin_type_idx2, + uint32 rec_count, const WASMTypePtr *types, + uint32 type_count) +{ + uint32 type_idx1, type_idx2; + + if (!wasm_is_refheaptype_typeidx(&ref_type1->ref_ht_common) + || !wasm_is_refheaptype_typeidx(&ref_type2->ref_ht_common)) + return ref_type1->ref_ht_common.heap_type + == ref_type2->ref_ht_common.heap_type + ? true + : false; + + /* Now both ref types are type of (ref type_idx) */ + type_idx1 = ref_type1->ref_ht_typeidx.type_idx; + type_idx2 = ref_type2->ref_ht_typeidx.type_idx; + + if (type_idx1 >= rec_begin_type_idx1 + && type_idx1 < rec_begin_type_idx1 + rec_count) { + /* The converted iso-recursive types should be the same */ + bool ret = (type_idx2 >= rec_begin_type_idx2 + && type_idx2 < rec_begin_type_idx2 + rec_count + && type_idx1 - rec_begin_type_idx1 + == type_idx2 - rec_begin_type_idx2) + ? true + : false; + return ret; + } + else if (type_idx2 >= rec_begin_type_idx2 + && type_idx2 < rec_begin_type_idx2 + rec_count) { + /* The converted iso-recursive types should be the same */ + bool ret = (type_idx1 >= rec_begin_type_idx1 + && type_idx1 < rec_begin_type_idx1 + rec_count + && type_idx1 - rec_begin_type_idx1 + == type_idx2 - rec_begin_type_idx2) + ? true + : false; + return ret; + } + + return types[type_idx1] == types[type_idx2] ? true : false; +} + +bool +wasm_func_type_equal(const WASMFuncType *type1, const WASMFuncType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + uint32 i, j = 0; + + if (type1 == type2) + return true; + + if (type1->param_count != type2->param_count + || type1->result_count != type2->result_count + || type1->ref_type_map_count != type2->ref_type_map_count) + return false; + + for (i = 0; i < (uint32)(type1->param_count + type1->result_count); i++) { + if (type1->types[i] != type2->types[i]) + return false; + + if (wasm_is_type_multi_byte_type(type1->types[i])) { + const WASMRefType *ref_type1, *ref_type2; + + bh_assert(j < type1->ref_type_map_count); + bh_assert(i == type1->ref_type_maps[j].index + && i == type2->ref_type_maps[j].index); + + ref_type1 = type1->ref_type_maps[j].ref_type; + ref_type2 = type2->ref_type_maps[j].ref_type; + + if (!rec_ref_type_equal( + ref_type1, ref_type2, type1->base_type.rec_begin_type_idx, + type2->base_type.rec_begin_type_idx, + type1->base_type.rec_count, types, type_count)) + return false; + + j++; + } + } + + return true; +} + +bool +wasm_struct_type_equal(const WASMStructType *type1, const WASMStructType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + uint32 i, j = 0; + + if (type1 == type2) + return true; + + if (type1->field_count != type2->field_count + || type1->ref_type_map_count != type2->ref_type_map_count) + return false; + + for (i = 0; i < type1->field_count; i++) { + if (type1->fields[i].field_type != type2->fields[i].field_type + || type1->fields[i].field_flags != type2->fields[i].field_flags) + return false; + + if (wasm_is_type_multi_byte_type(type1->fields[i].field_type)) { + const WASMRefType *ref_type1, *ref_type2; + + bh_assert(j < type1->ref_type_map_count); + bh_assert(i == type1->ref_type_maps[j].index + && i == type2->ref_type_maps[j].index); + + ref_type1 = type1->ref_type_maps[j].ref_type; + ref_type2 = type2->ref_type_maps[j].ref_type; + + if (!rec_ref_type_equal( + ref_type1, ref_type2, type1->base_type.rec_begin_type_idx, + type2->base_type.rec_begin_type_idx, + type1->base_type.rec_count, types, type_count)) + return false; + + j++; + } + } + + return true; +} + +bool +wasm_array_type_equal(const WASMArrayType *type1, const WASMArrayType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + if (type1 == type2) + return true; + + if (type1->elem_flags != type2->elem_flags) + return false; + + if (type1->elem_type != type2->elem_type) + return false; + + if (!wasm_is_type_multi_byte_type(type1->elem_type)) + return true; + + return rec_ref_type_equal(type1->elem_ref_type, type2->elem_ref_type, + type1->base_type.rec_begin_type_idx, + type2->base_type.rec_begin_type_idx, + type1->base_type.rec_count, types, type_count); +} + +bool +wasm_type_equal(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + uint32 rec_begin_type_idx1 = type1->rec_begin_type_idx; + uint32 rec_begin_type_idx2 = type2->rec_begin_type_idx; + uint32 parent_type_idx1, parent_type_idx2, rec_count; + + if (type1 == type2) + return true; + + if (!(type1->type_flag == type2->type_flag + && type1->is_sub_final == type2->is_sub_final + && type1->rec_count == type2->rec_count + && type1->rec_idx == type2->rec_idx)) + return false; + + rec_count = type1->rec_count; + + parent_type_idx1 = type1->parent_type_idx; + parent_type_idx2 = type2->parent_type_idx; + + if (parent_type_idx1 >= rec_begin_type_idx1 + && parent_type_idx1 < rec_begin_type_idx1 + rec_count) { + /* The converted iso-recursive types should be the same */ + if (!(parent_type_idx2 >= rec_begin_type_idx2 + && parent_type_idx2 < rec_begin_type_idx2 + rec_count + && parent_type_idx1 - rec_begin_type_idx1 + == parent_type_idx2 - rec_begin_type_idx2)) { + return false; + } + } + else if (parent_type_idx2 >= rec_begin_type_idx2 + && parent_type_idx2 < rec_begin_type_idx2 + rec_count) { + /* The converted iso-recursive types should be the same */ + if (!(parent_type_idx1 >= rec_begin_type_idx1 + && parent_type_idx1 < rec_begin_type_idx1 + rec_count + && parent_type_idx1 - rec_begin_type_idx1 + == parent_type_idx2 - rec_begin_type_idx2)) { + return false; + } + } + else if (type1->parent_type != type2->parent_type) { + /* The parent types should be same since they have been + normalized and equivalence types with different type + indexes are referring to a same WASMType */ + return false; + } + + if (wasm_type_is_func_type(type1)) + return wasm_func_type_equal((WASMFuncType *)type1, + (WASMFuncType *)type2, types, type_count); + else if (wasm_type_is_struct_type(type1)) + return wasm_struct_type_equal((WASMStructType *)type1, + (WASMStructType *)type2, types, + type_count); + else if (wasm_type_is_array_type(type1)) + return wasm_array_type_equal((WASMArrayType *)type1, + (WASMArrayType *)type2, types, type_count); + + bh_assert(0); + return false; +} + +bool +wasm_func_type_is_subtype_of(const WASMFuncType *type1, + const WASMFuncType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + const WASMRefType *ref_type1 = NULL, *ref_type2 = NULL; + uint32 i, j1 = 0, j2 = 0; + + if (type1 == type2) + return true; + + if (type1->param_count != type2->param_count + || type1->result_count != type2->result_count) + return false; + + for (i = 0; i < type1->param_count; i++) { + if (wasm_is_type_multi_byte_type(type1->types[i])) { + bh_assert(j1 < type1->ref_type_map_count); + ref_type1 = type1->ref_type_maps[j1++].ref_type; + } + if (wasm_is_type_multi_byte_type(type2->types[i])) { + bh_assert(j2 < type2->ref_type_map_count); + ref_type2 = type2->ref_type_maps[j2++].ref_type; + } + if (!wasm_reftype_is_subtype_of(type2->types[i], ref_type2, + type1->types[i], ref_type1, types, + type_count)) { + return false; + } + } + + for (; i < (uint32)(type1->param_count + type1->result_count); i++) { + if (wasm_is_type_multi_byte_type(type1->types[i])) { + bh_assert(j1 < type1->ref_type_map_count); + ref_type1 = type1->ref_type_maps[j1++].ref_type; + } + if (wasm_is_type_multi_byte_type(type2->types[i])) { + bh_assert(j2 < type2->ref_type_map_count); + ref_type2 = type2->ref_type_maps[j2++].ref_type; + } + if (!wasm_reftype_is_subtype_of(type1->types[i], ref_type1, + type2->types[i], ref_type2, types, + type_count)) { + return false; + } + } + + return true; +} + +bool +wasm_func_type_result_is_subtype_of(const WASMFuncType *type1, + const WASMFuncType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + const WASMRefType *ref_type1 = NULL, *ref_type2 = NULL; + const WASMRefTypeMap *ref_type_map1, *ref_type_map2; + uint32 i; + + if (type1 == type2) + return true; + + if (type1->result_count != type2->result_count) + return false; + + ref_type_map1 = type1->result_ref_type_maps; + ref_type_map2 = type2->result_ref_type_maps; + + for (i = 0; i < type1->result_count; i++) { + ref_type1 = ref_type2 = NULL; + if (wasm_is_type_multi_byte_type( + type1->types[type1->param_count + i])) { + bh_assert(ref_type_map1 + && ref_type_map1->index == type1->param_count + i); + ref_type1 = ref_type_map1->ref_type; + ref_type_map1++; + } + if (wasm_is_type_multi_byte_type( + type2->types[type2->param_count + i])) { + bh_assert(ref_type_map2 + && ref_type_map2->index == type1->param_count + i); + ref_type2 = ref_type_map2->ref_type; + ref_type_map2++; + } + if (!wasm_reftype_is_subtype_of(type1->types[type1->param_count + i], + ref_type1, + type2->types[type2->param_count + i], + ref_type2, types, type_count)) { + return false; + } + } + return true; +} + +bool +wasm_struct_type_is_subtype_of(const WASMStructType *type1, + const WASMStructType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + const WASMRefType *ref_type1 = NULL, *ref_type2 = NULL; + uint32 i, j1 = 0, j2 = 0; + + /** + * A structure type is a supertype of another structure type if + * its field list is a prefix of the other (width subtyping). + * A structure type also is a supertype of another structure type + * if they have the same fields and for each field type: + * The field is mutable in both types and the storage types + * are the same. + * The field is immutable in both types and their storage types + * are in (covariant) subtype relation (depth subtyping). + */ + + if (type1 == type2) + return true; + + if (type1->field_count > type2->field_count) { + /* Check whether type1's field list is a prefix of type2 */ + for (i = 0; i < type2->field_count; i++) { + if (type1->fields[i].field_flags != type2->fields[i].field_flags) + return false; + if (wasm_is_type_multi_byte_type(type1->fields[i].field_type)) { + bh_assert(j1 < type1->ref_type_map_count); + ref_type1 = type1->ref_type_maps[j1++].ref_type; + } + if (wasm_is_type_multi_byte_type(type2->fields[i].field_type)) { + bh_assert(j2 < type2->ref_type_map_count); + ref_type2 = type2->ref_type_maps[j2++].ref_type; + } + if (!wasm_reftype_is_subtype_of(type1->fields[i].field_type, + ref_type1, + type2->fields[i].field_type, + ref_type2, types, type_count)) { + return false; + } + } + return true; + } + else if (type1->field_count == type2->field_count) { + /* Check each field's flag and type */ + for (i = 0; i < type1->field_count; i++) { + if (type1->fields[i].field_flags != type2->fields[i].field_flags) + return false; + + if (type1->fields[i].field_flags & 1) { + /* The field is mutable in both types: the storage types + must be the same */ + if (type1->fields[i].field_type != type2->fields[i].field_type) + return false; + if (wasm_is_type_multi_byte_type(type1->fields[i].field_type)) { + bh_assert(j1 < type1->ref_type_map_count); + bh_assert(j2 < type2->ref_type_map_count); + ref_type1 = type1->ref_type_maps[j1++].ref_type; + ref_type2 = type2->ref_type_maps[j2++].ref_type; + if (!wasm_reftype_equal(ref_type1->ref_type, ref_type1, + ref_type2->ref_type, ref_type2, + types, type_count)) + return false; + } + } + else { + /* The field is immutable in both types: their storage types + must be in (covariant) subtype relation (depth subtyping) */ + if (wasm_is_type_multi_byte_type(type1->fields[i].field_type)) { + bh_assert(j1 < type1->ref_type_map_count); + ref_type1 = type1->ref_type_maps[j1++].ref_type; + } + if (wasm_is_type_multi_byte_type(type2->fields[i].field_type)) { + bh_assert(j2 < type2->ref_type_map_count); + ref_type2 = type2->ref_type_maps[j2++].ref_type; + } + if (!wasm_reftype_is_subtype_of(type1->fields[i].field_type, + ref_type1, + type2->fields[i].field_type, + ref_type2, types, type_count)) + return false; + } + } + return true; + } + + return false; +} + +bool +wasm_array_type_is_subtype_of(const WASMArrayType *type1, + const WASMArrayType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + /** + * An array type is a supertype of another array type if: + * Both element types are mutable and the storage types are the same. + * Both element types are immutable and their storage types are in + * (covariant) subtype relation (depth subtyping). + */ + + if (type1->elem_flags != type2->elem_flags) + return false; + + if (type1->elem_flags & 1) { + /* The elem is mutable in both types: the storage types + must be the same */ + return wasm_reftype_equal(type1->elem_type, type1->elem_ref_type, + type2->elem_type, type2->elem_ref_type, types, + type_count); + } + else { + /* The elem is immutable in both types: their storage types + must be in (covariant) subtype relation (depth subtyping) */ + return wasm_reftype_is_subtype_of( + type1->elem_type, type1->elem_ref_type, type2->elem_type, + type2->elem_ref_type, types, type_count); + } + return false; +} + +bool +wasm_type_is_subtype_of(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + if (type1 == type2) + return true; + + if (type1->type_flag != type2->type_flag) + return false; + + if (wasm_type_is_func_type(type1)) + return wasm_func_type_is_subtype_of( + (WASMFuncType *)type1, (WASMFuncType *)type2, types, type_count); + else if (wasm_type_is_struct_type(type1)) + return wasm_struct_type_is_subtype_of((WASMStructType *)type1, + (WASMStructType *)type2, types, + type_count); + else if (wasm_type_is_array_type(type1)) + return wasm_array_type_is_subtype_of( + (WASMArrayType *)type1, (WASMArrayType *)type2, types, type_count); + + bh_assert(0); + return false; +} + +uint32 +wasm_reftype_size(uint8 type) +{ + if (type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32) + return 4; + else if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64) + return 8; + else if ((type >= (uint8)REF_TYPE_ARRAYREF + && type <= (uint8)REF_TYPE_NULLFUNCREF) + || (type >= (uint8)REF_TYPE_HT_NULLABLE + && type <= (uint8)REF_TYPE_HT_NON_NULLABLE) +#if WASM_ENABLE_STRINGREF != 0 + || (type >= (uint8)REF_TYPE_STRINGVIEWWTF8 + && type <= (uint8)REF_TYPE_STRINGREF) + || (type >= (uint8)REF_TYPE_STRINGVIEWITER + && type <= (uint8)REF_TYPE_STRINGVIEWWTF16) +#endif + ) + return sizeof(uintptr_t); + else if (type == PACKED_TYPE_I8) + return 1; + else if (type == PACKED_TYPE_I16) + return 2; + else if (type == VALUE_TYPE_V128) + return 16; + else { + bh_assert(0); + return 0; + } + + return 0; +} + +uint32 +wasm_reftype_struct_size(const WASMRefType *ref_type) +{ + bh_assert(wasm_is_reftype_htref_nullable(ref_type->ref_type) + || wasm_is_reftype_htref_non_nullable(ref_type->ref_type)); + bh_assert(wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common) + || wasm_is_refheaptype_common(&ref_type->ref_ht_common)); + + return (uint32)sizeof(RefHeapType_Common); +} + +bool +wasm_refheaptype_equal(const RefHeapType_Common *ref_heap_type1, + const RefHeapType_Common *ref_heap_type2, + const WASMTypePtr *types, uint32 type_count) +{ + if (ref_heap_type1 == ref_heap_type2) + return true; + + if (ref_heap_type1->ref_type != ref_heap_type2->ref_type) + return false; + + if (ref_heap_type1->heap_type != ref_heap_type2->heap_type) { + if (wasm_is_refheaptype_typeidx(ref_heap_type1) + && wasm_is_refheaptype_typeidx(ref_heap_type2)) { + if (ref_heap_type1->heap_type == ref_heap_type2->heap_type) + return true; + else + /* the type_count may be 0 when called from reftype_equal */ + return ((uint32)ref_heap_type1->heap_type < type_count + && (uint32)ref_heap_type2->heap_type < type_count + && types[ref_heap_type1->heap_type] + == types[ref_heap_type2->heap_type]) + ? true + : false; + } + return false; + } + + /* No need to check extra info for common types and (type i) + as their heap_types are the same */ + return true; +} + +bool +wasm_reftype_equal(uint8 type1, const WASMRefType *reftype1, uint8 type2, + const WASMRefType *reftype2, const WASMTypePtr *types, + uint32 type_count) +{ + /* For (ref null func/extern/any/eq/i31/struct/array/none/nofunc/noextern), + they are same as funcref/externref/anyref/eqref/i31ref/structref/arayref/ + nullref/nullfuncref/nullexternref, and have been converted into to the + related one-byte type when loading, so here we don't consider the + situations again: + one is (ref null func/extern/any/eq/i31/struct/array/..), + the other is + funcref/externref/anyref/eqref/i31ref/structref/arrayref/.. */ + if (type1 != type2) + return false; + + if (!wasm_is_type_multi_byte_type(type1)) + /* one byte type */ + return true; + + bh_assert(type1 == (uint8)REF_TYPE_HT_NULLABLE + || type1 == (uint8)REF_TYPE_HT_NON_NULLABLE); + + /* (ref null ht) or (ref ht) */ + return wasm_refheaptype_equal((RefHeapType_Common *)reftype1, + (RefHeapType_Common *)reftype2, types, + type_count); +} + +inline static bool +wasm_is_reftype_supers_of_eq(uint8 type) +{ + return (type == REF_TYPE_EQREF || type == REF_TYPE_ANYREF) ? true : false; +} + +inline static bool +wasm_is_reftype_supers_of_i31(uint8 type) +{ + return (type == REF_TYPE_I31REF || wasm_is_reftype_supers_of_eq(type)) + ? true + : false; +} + +inline static bool +wasm_is_reftype_supers_of_struct(uint8 type) +{ + return (type == REF_TYPE_STRUCTREF || wasm_is_reftype_supers_of_eq(type)) + ? true + : false; +} + +inline static bool +wasm_is_reftype_supers_of_array(uint8 type) +{ + return (type == REF_TYPE_ARRAYREF || wasm_is_reftype_supers_of_eq(type)) + ? true + : false; +} + +inline static bool +wasm_is_reftype_supers_of_func(uint8 type) +{ + return (type == REF_TYPE_FUNCREF) ? true : false; +} + +inline static bool +wasm_is_reftype_supers_of_extern(uint8 type) +{ + return (type == REF_TYPE_EXTERNREF) ? true : false; +} + +#if WASM_ENABLE_STRINGREF != 0 +inline static bool +wasm_is_reftype_supers_of_string(uint8 type) +{ + return (type == REF_TYPE_STRINGREF || type == REF_TYPE_ANYREF) ? true + : false; +} +#endif + +inline static bool +wasm_is_reftype_supers_of_none(uint8 type, const WASMRefType *ref_type, + const WASMTypePtr *types, uint32 type_count) +{ + if (type == REF_TYPE_NULLREF || type == REF_TYPE_I31REF + || type == REF_TYPE_STRUCTREF || type == REF_TYPE_ARRAYREF + || wasm_is_reftype_supers_of_eq(type) +#if WASM_ENABLE_STRINGREF != 0 + || type == REF_TYPE_STRINGREF +#endif + ) + return true; + + if (type == REF_TYPE_HT_NULLABLE && ref_type != NULL + && wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common) + && (types[ref_type->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_STRUCT + || types[ref_type->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_ARRAY)) + return true; + + return false; +} + +inline static bool +wasm_is_reftype_supers_of_nofunc(uint8 type, const WASMRefType *ref_type, + const WASMTypePtr *types, uint32 type_count) +{ + if (type == REF_TYPE_NULLFUNCREF || type == REF_TYPE_FUNCREF) + return true; + + if (type == REF_TYPE_HT_NULLABLE && ref_type != NULL + && wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common) + && (types[ref_type->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_FUNC)) + return true; + + return false; +} + +inline static bool +wasm_is_reftype_supers_of_noextern(uint8 type) +{ + return (type == REF_TYPE_NULLEXTERNREF || type == REF_TYPE_EXTERNREF) + ? true + : false; +} + +/* Whether type1 is one of super types of type2 */ +static bool +wasm_type_is_supers_of(const WASMType *type1, const WASMType *type2) +{ + uint32 i, inherit_depth_diff; + + if (type1 == type2) + return true; + + if (!(type1->root_type == type2->root_type + && type1->inherit_depth < type2->inherit_depth)) + return false; + + inherit_depth_diff = type2->inherit_depth - type1->inherit_depth; + for (i = 0; i < inherit_depth_diff; i++) { + type2 = type2->parent_type; + if (type2 == type1) + return true; + } + + return false; +} + +bool +wasm_func_type_is_super_of(const WASMFuncType *type1, const WASMFuncType *type2) +{ + return wasm_type_is_supers_of((const WASMType *)type1, + (const WASMType *)type2); +} + +bool +wasm_reftype_is_subtype_of(uint8 type1, const WASMRefType *ref_type1, + uint8 type2, const WASMRefType *ref_type2, + const WASMTypePtr *types, uint32 type_count) +{ + if (type1 >= PACKED_TYPE_I16 && type1 <= VALUE_TYPE_I32) { + /* Primitive types (I32/I64/F32/F64/V128/I8/I16) are not + subtypes of each other */ + return type1 == type2 ? true : false; + } + + /** + * Check subtype relationship of two ref types, the ref type hierarchy can + * be described as: + * + * anyref -> eqref + * |-> i31ref + * |-> structref -> (ref null $t) -> (ref $t), $t is struct + * |-> arrayref -> (ref null $t) -> (ref $t), $t is array + * + * funcref -> (ref null $t) -> (ref $t), $t is func + * externref + */ + + if (type1 == REF_TYPE_ANYREF) { + /* any <: any */ + return type2 == REF_TYPE_ANYREF ? true : false; + } + else if (type1 == REF_TYPE_FUNCREF) { + /* func <: func */ + return type2 == REF_TYPE_FUNCREF ? true : false; + } + else if (type1 == REF_TYPE_EXTERNREF) { + /* extern <: extern */ + return type2 == REF_TYPE_EXTERNREF ? true : false; + } + else if (type1 == REF_TYPE_EQREF) { + /* eq <: [eq, any] */ + return wasm_is_reftype_supers_of_eq(type2); + } + else if (type1 == REF_TYPE_I31REF) { + /* i31 <: [i31, eq, any] */ + return wasm_is_reftype_supers_of_i31(type2); + } + else if (type1 == REF_TYPE_STRUCTREF) { + /* struct <: [struct, eq, any] */ + return wasm_is_reftype_supers_of_struct(type2); + } + else if (type1 == REF_TYPE_ARRAYREF) { + /* array <: [array, eq, any] */ + return wasm_is_reftype_supers_of_array(type2); + } + else if (type1 == REF_TYPE_NULLREF) { + return wasm_is_reftype_supers_of_none(type2, ref_type2, types, + type_count); + } + else if (type1 == REF_TYPE_NULLFUNCREF) { + return wasm_is_reftype_supers_of_nofunc(type2, ref_type2, types, + type_count); + } + else if (type1 == REF_TYPE_NULLEXTERNREF) { + return wasm_is_reftype_supers_of_noextern(type2); + } +#if WASM_ENABLE_STRINGREF != 0 + else if (type1 == REF_TYPE_STRINGREF) { + return wasm_is_reftype_supers_of_string(type2); + } + else if (type1 == REF_TYPE_STRINGVIEWWTF8) { + return type2 == REF_TYPE_STRINGVIEWWTF8 ? true : false; + } + else if (type1 == REF_TYPE_STRINGVIEWWTF16) { + return type2 == REF_TYPE_STRINGVIEWWTF16 ? true : false; + } + else if (type1 == REF_TYPE_STRINGVIEWITER) { + return type2 == REF_TYPE_STRINGVIEWITER ? true : false; + } +#endif + else if (type1 == REF_TYPE_HT_NULLABLE) { + if (wasm_is_refheaptype_typeidx(&ref_type1->ref_ht_common)) { + bh_assert((uint32)ref_type1->ref_ht_typeidx.type_idx < type_count); + /* reftype1 is (ref null $t) */ + if (type2 == REF_TYPE_HT_NULLABLE && ref_type2 != NULL + && wasm_is_refheaptype_typeidx(&ref_type2->ref_ht_common)) { + bh_assert((uint32)ref_type2->ref_ht_typeidx.type_idx + < type_count); + return wasm_type_is_supers_of( + types[ref_type2->ref_ht_typeidx.type_idx], + types[ref_type1->ref_ht_typeidx.type_idx]); + } + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_STRUCT) + return wasm_is_reftype_supers_of_struct(type2); + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_ARRAY) + return wasm_is_reftype_supers_of_array(type2); + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_FUNC) + return wasm_is_reftype_supers_of_func(type2); +#if WASM_ENABLE_STRINGREF != 0 + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_STRINGREF) + return wasm_is_reftype_supers_of_string(type2); + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_STRINGVIEWWTF8) { + return type2 == REF_TYPE_STRINGVIEWWTF8 ? true : false; + } + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_STRINGVIEWWTF16) { + return type2 == REF_TYPE_STRINGVIEWWTF16 ? true : false; + } + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_STRINGVIEWITER) { + return type2 == REF_TYPE_STRINGVIEWITER ? true : false; + } +#endif + else + return false; + } + else { + /* (ref null func/extern/any/eq/i31/struct/array/..) have been + converted into + funcref/externref/anyref/eqref/i31ref/structref/arrayref/.. + when loading */ + bh_assert(0); + } + } + else if (type1 == REF_TYPE_HT_NON_NULLABLE) { + bh_assert(ref_type1); + if (wasm_is_refheaptype_typeidx(&ref_type1->ref_ht_common)) { + bh_assert((uint32)ref_type1->ref_ht_typeidx.type_idx < type_count); + /* reftype1 is (ref $t) */ + if ((type2 == REF_TYPE_HT_NULLABLE + || type2 == REF_TYPE_HT_NON_NULLABLE) + && ref_type2 != NULL + && wasm_is_refheaptype_typeidx(&ref_type2->ref_ht_common)) { + bh_assert((uint32)ref_type2->ref_ht_typeidx.type_idx + < type_count); + return wasm_type_is_supers_of( + types[ref_type2->ref_ht_typeidx.type_idx], + types[ref_type1->ref_ht_typeidx.type_idx]); + } + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_STRUCT) { + /* the super type is (ref null struct) or (ref struct) */ + if (type2 == REF_TYPE_HT_NULLABLE + || type2 == REF_TYPE_HT_NON_NULLABLE) { + bh_assert(ref_type2); + uint8 ref_type = + (uint8)(ref_type2->ref_ht_common.heap_type + + REF_TYPE_FUNCREF - HEAP_TYPE_FUNC); + return wasm_is_reftype_supers_of_struct(ref_type); + } + else + /* the super type is structref or anyref */ + return wasm_is_reftype_supers_of_struct(type2); + } + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_ARRAY) { + /* the super type is (ref null array) or (ref array) */ + if (type2 == REF_TYPE_HT_NULLABLE + || type2 == REF_TYPE_HT_NON_NULLABLE) { + bh_assert(ref_type2); + uint8 ref_type = + (uint8)(ref_type2->ref_ht_common.heap_type + + REF_TYPE_FUNCREF - HEAP_TYPE_FUNC); + return wasm_is_reftype_supers_of_array(ref_type); + } + else + /* the super type is arrayref, eqref or anyref */ + return wasm_is_reftype_supers_of_array(type2); + } + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == WASM_TYPE_FUNC) { + /* the super type is (ref null func) or (ref func) */ + if (type2 == REF_TYPE_HT_NULLABLE + || type2 == REF_TYPE_HT_NON_NULLABLE) { + bh_assert(ref_type2); + uint8 ref_type = + (uint8)(ref_type2->ref_ht_common.heap_type + + REF_TYPE_FUNCREF - HEAP_TYPE_FUNC); + return wasm_is_reftype_supers_of_func(ref_type); + } + else + /* the super type is funcref */ + return wasm_is_reftype_supers_of_func(type2); + } + else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag + == REF_TYPE_I31REF) { + /* the super type is (ref null i31) or (ref i31) */ + if (type2 == REF_TYPE_HT_NULLABLE + || type2 == REF_TYPE_HT_NON_NULLABLE) { + bh_assert(ref_type2); + uint8 ref_type = + (uint8)(ref_type2->ref_ht_common.heap_type + + REF_TYPE_FUNCREF - HEAP_TYPE_FUNC); + return wasm_is_reftype_supers_of_i31(ref_type); + } + else + /* the super type is i31ref, eqref or anyref */ + return wasm_is_reftype_supers_of_i31(type2); + } + else { + return false; + } + } + else if (wasm_is_refheaptype_common(&ref_type1->ref_ht_common)) { + /* reftype1 is (ref func/extern/any/eq/i31/struct/array/..) */ + if (wasm_reftype_equal(type1, ref_type1, type2, ref_type2, types, + type_count)) + return true; + else { + int32 heap_type = ref_type1->ref_ht_common.heap_type; + if (heap_type == HEAP_TYPE_ANY) { + /* (ref any) <: anyref */ + return type2 == REF_TYPE_ANYREF ? true : false; + } + else if (heap_type == HEAP_TYPE_EXTERN) { + /* (ref extern) <: externref */ + return type2 == REF_TYPE_EXTERNREF ? true : false; + } + else if (heap_type == HEAP_TYPE_EQ) { + /* (ref eq) <: [eqref, anyref] */ + return wasm_is_reftype_supers_of_eq(type2); + } + else if (heap_type == HEAP_TYPE_I31) { + /* (ref i31) <: [i31ref, eqref, anyref] */ + return wasm_is_reftype_supers_of_i31(type2); + } + else if (heap_type == HEAP_TYPE_STRUCT) { + /* (ref struct) <: [structref, eqref, anyref] */ + return wasm_is_reftype_supers_of_struct(type2); + } + else if (heap_type == HEAP_TYPE_ARRAY) { + /* (ref array) <: [arrayref, eqref, anyref] */ + return wasm_is_reftype_supers_of_array(type2); + } + else if (heap_type == HEAP_TYPE_FUNC) { + /* (ref func) <: [funcref] */ + return wasm_is_reftype_supers_of_func(type2); + } +#if WASM_ENABLE_STRINGREF != 0 + else if (heap_type == HEAP_TYPE_STRINGREF) { + return wasm_is_reftype_supers_of_string(type2); + } + else if (heap_type == HEAP_TYPE_STRINGVIEWWTF8) { + return type2 == REF_TYPE_STRINGVIEWWTF8 ? true : false; + } + else if (heap_type == HEAP_TYPE_STRINGVIEWWTF16) { + return type2 == REF_TYPE_STRINGVIEWWTF16 ? true : false; + } + else if (heap_type == HEAP_TYPE_STRINGVIEWITER) { + return type2 == REF_TYPE_STRINGVIEWITER ? true : false; + } +#endif + else if (heap_type == HEAP_TYPE_NONE) { + /* (ref none) */ + /* TODO */ + bh_assert(0); + } + else if (heap_type == HEAP_TYPE_NOEXTERN) { + /* (ref noextern) */ + /* TODO */ + bh_assert(0); + } + else if (heap_type == HEAP_TYPE_NOFUNC) { + /* (ref nofunc) */ + /* TODO */ + bh_assert(0); + } + else { + bh_assert(0); + } + } + } + else { + /* unknown type detected */ + LOG_ERROR("unknown sub type 0x%02x", type1); + bh_assert(0); + } + } + else { + bh_assert(0); + } + + return false; +} + +static uint32 +reftype_hash(const void *key) +{ + WASMRefType *reftype = (WASMRefType *)key; + + switch (reftype->ref_type) { + case (uint8)REF_TYPE_HT_NULLABLE: + case (uint8)REF_TYPE_HT_NON_NULLABLE: + { + RefHeapType_Common *ref_heap_type = (RefHeapType_Common *)reftype; + + if (wasm_is_refheaptype_common(ref_heap_type) + /* type indexes of defined type are same */ + || wasm_is_refheaptype_typeidx(ref_heap_type)) { + return (uint32)reftype->ref_type + ^ (uint32)ref_heap_type->heap_type; + } + + break; + } + + default: + break; + } + + bh_assert(0); + return 0; +} + +static bool +reftype_equal(void *type1, void *type2) +{ + WASMRefType *reftype1 = (WASMRefType *)type1; + WASMRefType *reftype2 = (WASMRefType *)type2; + + return wasm_reftype_equal(reftype1->ref_type, reftype1, reftype2->ref_type, + reftype2, NULL, 0); +} + +WASMRefType * +wasm_reftype_dup(const WASMRefType *ref_type) +{ + if (wasm_is_reftype_htref_nullable(ref_type->ref_type) + || wasm_is_reftype_htref_non_nullable(ref_type->ref_type)) { + if (wasm_is_refheaptype_common(&ref_type->ref_ht_common) + || wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common)) { + RefHeapType_Common *ht_common; + if (!(ht_common = wasm_runtime_malloc(sizeof(RefHeapType_Common)))) + return NULL; + + ht_common->ref_type = ref_type->ref_ht_common.ref_type; + ht_common->nullable = ref_type->ref_ht_common.nullable; + ht_common->heap_type = ref_type->ref_ht_common.heap_type; + return (WASMRefType *)ht_common; + } + } + + bh_assert(0); + return NULL; +} + +void +wasm_set_refheaptype_typeidx(RefHeapType_TypeIdx *ref_ht_typeidx, bool nullable, + int32 type_idx) +{ + ref_ht_typeidx->ref_type = + nullable ? REF_TYPE_HT_NULLABLE : REF_TYPE_HT_NON_NULLABLE; + ref_ht_typeidx->nullable = nullable; + ref_ht_typeidx->type_idx = type_idx; +} + +void +wasm_set_refheaptype_common(RefHeapType_Common *ref_ht_common, bool nullable, + int32 heap_type) +{ + ref_ht_common->ref_type = + nullable ? REF_TYPE_HT_NULLABLE : REF_TYPE_HT_NON_NULLABLE; + ref_ht_common->nullable = nullable; + ref_ht_common->heap_type = heap_type; +} + +WASMRefType * +wasm_reftype_map_find(WASMRefTypeMap *ref_type_maps, uint32 ref_type_map_count, + uint32 index_to_find) +{ + int low = 0, mid; + int high = (int32)ref_type_map_count - 1; + uint32 index; + + while (low <= high) { + mid = (low + high) / 2; + index = ref_type_maps[mid].index; + if (index_to_find == index) { + return ref_type_maps[mid].ref_type; + } + else if (index_to_find < index) + high = mid - 1; + else + low = mid + 1; + } + + return NULL; +} + +HashMap * +wasm_reftype_set_create(uint32 size) +{ + HashMap *ref_type_set = bh_hash_map_create( + size, false, reftype_hash, reftype_equal, NULL, wasm_runtime_free); + + return ref_type_set; +} + +WASMRefType * +wasm_reftype_set_insert(HashMap *ref_type_set, const WASMRefType *ref_type) +{ + WASMRefType *ref_type_ret; + + if ((ref_type_ret = bh_hash_map_find(ref_type_set, (void *)ref_type))) + return ref_type_ret; + + if (!(ref_type_ret = wasm_reftype_dup(ref_type))) + return NULL; + + if (!bh_hash_map_insert(ref_type_set, ref_type_ret, ref_type_ret)) { + wasm_runtime_free(ref_type_ret); + return NULL; + } + + return ref_type_ret; +} diff --git a/wasm-micro-runtime/core/iwasm/common/gc/gc_type.h b/wasm-micro-runtime/core/iwasm/common/gc/gc_type.h new file mode 100644 index 0000000..919c8e5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/gc_type.h @@ -0,0 +1,384 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _GC_TYPE_H_ +#define _GC_TYPE_H_ + +#include "../interpreter/wasm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void +wasm_dump_value_type(uint8 type, const WASMRefType *ref_type); + +void +wasm_dump_func_type(const WASMFuncType *type); + +void +wasm_dump_struct_type(const WASMStructType *type); + +void +wasm_dump_array_type(const WASMArrayType *type); + +/* Whether a group of value types is subtype of + another group of value types */ +bool +wasm_value_types_is_subtype_of(const uint8 *types1, + const WASMRefTypeMap *ref_type_maps1, + const uint8 *types2, + const WASMRefTypeMap *ref_type_maps2, + uint32 value_type_count, + const WASMTypePtr *types, uint32 type_count); + +/* Operations of function type */ + +/* Whether two function types are equal */ +bool +wasm_func_type_equal(const WASMFuncType *type1, const WASMFuncType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Whether func type1 is subtype of func type2 */ +bool +wasm_func_type_is_subtype_of(const WASMFuncType *type1, + const WASMFuncType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Whether func type1 is one of super types of func type2, + used for the func type check in call_indirect/call_ref opcodes */ +bool +wasm_func_type_is_super_of(const WASMFuncType *type1, + const WASMFuncType *type2); + +/* Whether func type1's result types are subtype of + func type2's result types */ +bool +wasm_func_type_result_is_subtype_of(const WASMFuncType *type, + const WASMFuncType *type2, + const WASMTypePtr *types, + uint32 type_count); + +/* Operations of struct type */ + +/* Whether two struct types are equal */ +bool +wasm_struct_type_equal(const WASMStructType *type1, const WASMStructType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Whether struct type1 is subtype of struct type2 */ +bool +wasm_struct_type_is_subtype_of(const WASMStructType *type1, + const WASMStructType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Operations of array type */ + +/* Whether two array types are equal */ +bool +wasm_array_type_equal(const WASMArrayType *type1, const WASMArrayType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Whether array type1 is subtype of array type2 */ +bool +wasm_array_type_is_subtype_of(const WASMArrayType *type1, + const WASMArrayType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Operations of wasm type */ + +/* Whether a wasm type is a function type */ +inline static bool +wasm_type_is_func_type(const WASMType *type) +{ + return type->type_flag == WASM_TYPE_FUNC ? true : false; +} + +/* Whether a wasm type is a struct type */ +inline static bool +wasm_type_is_struct_type(const WASMType *type) +{ + return type->type_flag == WASM_TYPE_STRUCT ? true : false; +} + +/* Whether a wasm type is an array type */ +inline static bool +wasm_type_is_array_type(const WASMType *type) +{ + return type->type_flag == WASM_TYPE_ARRAY ? true : false; +} + +/* Whether two wasm types are equal */ +bool +wasm_type_equal(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Whether wasm type1 is subtype of wasm type2 */ +bool +wasm_type_is_subtype_of(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count); + +/* Operations of reference type */ + +/* Whether a value type is a reference type */ +inline static bool +wasm_is_type_reftype(uint8 type) +{ + return ((type >= (uint8)REF_TYPE_ARRAYREF + && type <= (uint8)REF_TYPE_NULLFUNCREF) + || (type >= (uint8)REF_TYPE_HT_NULLABLE + && type <= (uint8)REF_TYPE_HT_NON_NULLABLE) +#if WASM_ENABLE_STRINGREF != 0 + || (type >= (uint8)REF_TYPE_STRINGVIEWWTF8 + && type <= (uint8)REF_TYPE_STRINGREF) + || (type >= (uint8)REF_TYPE_STRINGVIEWITER + && type <= (uint8)REF_TYPE_STRINGVIEWWTF16) +#endif + ) + ? true + : false; +} + +/* Whether a negative value is a valid heap type */ +inline static bool +wasm_is_valid_heap_type(int32 heap_type) +{ + return ((heap_type <= HEAP_TYPE_NOFUNC && heap_type >= HEAP_TYPE_ARRAY) +#if WASM_ENABLE_STRINGREF != 0 + || heap_type == HEAP_TYPE_STRINGREF + || heap_type == HEAP_TYPE_STRINGVIEWWTF8 + || heap_type == HEAP_TYPE_STRINGVIEWWTF16 + || heap_type == HEAP_TYPE_STRINGVIEWITER +#endif + ) + ? true + : false; +} + +/* Whether a value type is multi-byte type, or, requires ref type map + to retrieve extra info */ +inline static bool +wasm_is_type_multi_byte_type(uint8 type) +{ + return (type == (uint8)REF_TYPE_HT_NULLABLE + || type == (uint8)REF_TYPE_HT_NON_NULLABLE) + ? true + : false; +} + +/* Whether a reference type is a funcref type */ +inline static bool +wasm_is_reftype_funcref(uint8 type) +{ + return type == (uint8)REF_TYPE_FUNCREF ? true : false; +} + +/* Whether a reference type is an externref type */ +inline static bool +wasm_is_reftype_externref(uint8 type) +{ + return type == (uint8)REF_TYPE_EXTERNREF ? true : false; +} + +/* Whether a reference type is an anyref type */ +inline static bool +wasm_is_reftype_anyref(uint8 type) +{ + return type == (uint8)REF_TYPE_ANYREF ? true : false; +} + +/* Whether a reference type is an eqref type */ +inline static bool +wasm_is_reftype_eqref(uint8 type) +{ + return type == (uint8)REF_TYPE_EQREF ? true : false; +} + +/* Whether a reference type is a (ref null ht) type */ +inline static bool +wasm_is_reftype_htref_nullable(uint8 type) +{ + return type == (uint8)REF_TYPE_HT_NULLABLE ? true : false; +} + +/* Whether a reference type is a (ref ht) type */ +inline static bool +wasm_is_reftype_htref_non_nullable(uint8 type) +{ + return type == (uint8)REF_TYPE_HT_NON_NULLABLE ? true : false; +} + +/* Whether a reference type is an i31ref type */ +inline static bool +wasm_is_reftype_i31ref(uint8 type) +{ + return type == (uint8)REF_TYPE_I31REF ? true : false; +} + +/* Whether a reference type is a structref type */ +inline static bool +wasm_is_reftype_structref(uint8 type) +{ + return type == (uint8)REF_TYPE_STRUCTREF ? true : false; +} + +/* Whether a reference type is an arrayref type */ +inline static bool +wasm_is_reftype_arrayref(uint8 type) +{ + return type == (uint8)REF_TYPE_ARRAYREF ? true : false; +} + +/* Whether a reference type is a nullref type */ +inline static bool +wasm_is_reftype_nullref(uint8 type) +{ + return type == (uint8)REF_TYPE_NULLREF ? true : false; +} + +/* Whether a reference type is a nullfuncref type */ +inline static bool +wasm_is_reftype_nullfuncref(uint8 type) +{ + return type == (uint8)REF_TYPE_NULLFUNCREF ? true : false; +} + +/* Whether a reference type is a nullexternref type */ +inline static bool +wasm_is_reftype_nullexternref(uint8 type) +{ + return type == (uint8)REF_TYPE_NULLEXTERNREF ? true : false; +} + +/* Return the size of a reference type */ +uint32 +wasm_reftype_size(uint8 type); + +/* Return the actual WASMRefType struct size required of a reference type */ +uint32 +wasm_reftype_struct_size(const WASMRefType *ref_type); + +/* Operations of ref heap type */ + +/* Whether a ref heap type is (type i), i : typeidx, >= 0 */ +inline static bool +wasm_is_refheaptype_typeidx(const RefHeapType_Common *ref_heap_type) +{ + return ref_heap_type->heap_type >= 0 ? true : false; +} + +/* Whether a ref heap type is a common type: func/any/eq/i31/data, + not (type i) or (rtt n i) or (rtt i) */ +inline static bool +wasm_is_refheaptype_common(const RefHeapType_Common *ref_heap_type) +{ + return ((ref_heap_type->heap_type >= (int32)HEAP_TYPE_ARRAY + && ref_heap_type->heap_type <= (int32)HEAP_TYPE_NONE) +#if WASM_ENABLE_STRINGREF != 0 + || (ref_heap_type->heap_type >= (int32)HEAP_TYPE_STRINGVIEWITER + && ref_heap_type->heap_type <= (int32)HEAP_TYPE_I31) +#endif + ) + ? true + : false; +} + +/* Whether a ref heap type is a func type */ +inline static bool +wasm_is_refheaptype_func(const RefHeapType_Common *ref_heap_type) +{ + return ref_heap_type->heap_type == (int32)HEAP_TYPE_FUNC ? true : false; +} + +/* Whether a ref heap type is an any type */ +inline static bool +wasm_is_refheaptype_any(const RefHeapType_Common *ref_heap_type) +{ + return ref_heap_type->heap_type == (int32)HEAP_TYPE_ANY ? true : false; +} + +/* Whether a ref heap type is an eq type */ +inline static bool +wasm_is_refheaptype_eq(const RefHeapType_Common *ref_heap_type) +{ + return ref_heap_type->heap_type == (int32)HEAP_TYPE_EQ ? true : false; +} + +/* Whether a ref heap type is an i31 type */ +inline static bool +wasm_is_refheaptype_i31(const RefHeapType_Common *ref_heap_type) +{ + return ref_heap_type->heap_type == (int32)HEAP_TYPE_I31 ? true : false; +} + +/* Whether a ref heap type is an array type */ +inline static bool +wasm_is_refheaptype_array(const RefHeapType_Common *ref_heap_type) +{ + return ref_heap_type->heap_type == (int32)HEAP_TYPE_ARRAY ? true : false; +} + +#if WASM_ENABLE_STRINGREF != 0 +inline static bool +wasm_is_refheaptype_stringrefs(const RefHeapType_Common *ref_heap_type) +{ + return ref_heap_type->heap_type <= (int32)HEAP_TYPE_STRINGREF + && ref_heap_type->heap_type >= HEAP_TYPE_STRINGVIEWITER + ? true + : false; +} +#endif + +/* Whether two ref heap types are equal */ +bool +wasm_refheaptype_equal(const RefHeapType_Common *ref_heap_type1, + const RefHeapType_Common *ref_heap_type2, + const WASMTypePtr *types, uint32 type_count); + +/* Whether two ref types are equal */ +bool +wasm_reftype_equal(uint8 type1, const WASMRefType *reftype1, uint8 type2, + const WASMRefType *reftype2, const WASMTypePtr *types, + uint32 type_count); + +/* Whether ref type1 is subtype of ref type2 */ +bool +wasm_reftype_is_subtype_of(uint8 type1, const WASMRefType *reftype1, + uint8 type2, const WASMRefType *reftype2, + const WASMTypePtr *types, uint32 type_count); + +/* Returns a new reference type which is a duplication of ref_type, + the caller should use wasm_runtime_free() to free the new ref type */ +WASMRefType * +wasm_reftype_dup(const WASMRefType *ref_type); + +/* Set fields of RefHeapType_TypeIdx */ +void +wasm_set_refheaptype_typeidx(RefHeapType_TypeIdx *ref_ht_typeidx, bool nullable, + int32 type_idx); + +/* Set fields of RefHeapType_Common */ +void +wasm_set_refheaptype_common(RefHeapType_Common *ref_ht_common, bool nullable, + int32 heap_type); + +/* Find the related reftype in reftype map array with index */ +WASMRefType * +wasm_reftype_map_find(WASMRefTypeMap *ref_type_maps, uint32 ref_type_map_count, + uint32 index_to_find); + +/* Create a new hash set of reference type */ +HashMap * +wasm_reftype_set_create(uint32 size); + +/* Insert a reference type into the hash set */ +WASMRefType * +wasm_reftype_set_insert(HashMap *ref_type_set, const WASMRefType *ref_type); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _GC_TYPE_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/common/gc/iwasm_gc.cmake b/wasm-micro-runtime/core/iwasm/common/gc/iwasm_gc.cmake new file mode 100644 index 0000000..5e243c3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/iwasm_gc.cmake @@ -0,0 +1,36 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (IWASM_GC_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_GC=1) + +if (WAMR_TEST_GC EQUAL 1) + add_definitions (-DGC_MANUALLY=1 -DGC_IN_EVERY_ALLOCATION=1) +endif () + +include_directories (${IWASM_GC_DIR}) + +file (GLOB source_all ${IWASM_GC_DIR}/*.c) + +set (IWASM_GC_SOURCE ${source_all}) + +if (WAMR_BUILD_STRINGREF EQUAL 1) + set (IWASM_STRINGREF_DIR ${CMAKE_CURRENT_LIST_DIR}/stringref) + + add_definitions (-DWASM_ENABLE_STRINGREF=1) + + include_directories (${IWASM_STRINGREF_DIR}) + + if (NOT DEFINED WAMR_STRINGREF_IMPL_SOURCE) + message(FATAL_ERROR "stringref feature enabled, but WAMR_STRINGREF_IMPL_SOURCE not set" ) + else () + if (${WAMR_STRINGREF_IMPL_SOURCE} STREQUAL "STUB") + set (IWASM_STRINGREF_SOURCE ${IWASM_STRINGREF_DIR}/stringref_stub.c) + else() + set (IWASM_STRINGREF_SOURCE ${WAMR_STRINGREF_IMPL_SOURCE}) + endif() + endif () + + set (IWASM_GC_SOURCE ${IWASM_GC_SOURCE} ${IWASM_STRINGREF_SOURCE}) +endif () diff --git a/wasm-micro-runtime/core/iwasm/common/gc/stringref/string_object.h b/wasm-micro-runtime/core/iwasm/common/gc/stringref/string_object.h new file mode 100644 index 0000000..88135a6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/stringref/string_object.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _STRING_OBJECT_H_ +#define _STRING_OBJECT_H_ + +#include "wasm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum EncodingFlag { + UTF8, + WTF8, + WTF16, + LOSSY_UTF8, +} EncodingFlag; + +typedef enum StringViewType { + STRING_VIEW_WTF8, + STRING_VIEW_WTF16, + STRING_VIEW_ITER, +} StringViewType; + +typedef enum ErrorCode { + Insufficient_Space = -3, + Encode_Fail = -2, + Isolated_Surrogate = -1, +} ErrorCode; + +/******************* gc finalizer *****************/ +void +wasm_string_destroy(WASMString str_obj); + +/******************* opcode functions *****************/ + +/* string.const */ +WASMString +wasm_string_new_const(const char *content, uint32 length); + +/* string.new_xx8/new_wtf16 */ +/* string.new_xx8_array */ +/* string.new_wtf16_array */ +WASMString +wasm_string_new_with_encoding(void *addr, uint32 count, EncodingFlag flag); + +/* string.measure */ +int32 +wasm_string_measure(WASMString str_obj, EncodingFlag flag); + +/* stringview_wtf16.length */ +int32 +wasm_string_wtf16_get_length(WASMString str_obj); + +/* string.encode_xx8 */ +/* string.encode_wtf16 */ +/* stringview_wtf8.encode_xx */ +/* stringview_wtf16.encode */ +/* string.encode_xx8_array */ +/* string.encode_wtf16_array */ +int32 +wasm_string_encode(WASMString str_obj, uint32 pos, uint32 count, void *addr, + uint32 *next_pos, EncodingFlag flag); + +/* string.concat */ +WASMString +wasm_string_concat(WASMString str_obj1, WASMString str_obj2); + +/* string.eq */ +int32 +wasm_string_eq(WASMString str_obj1, WASMString str_obj2); + +/* string.is_usv_sequence */ +int32 +wasm_string_is_usv_sequence(WASMString str_obj); + +/* string.as_wtf8 */ +/* string.as_wtf16 */ +/* string.as_iter */ +WASMString +wasm_string_create_view(WASMString str_obj, StringViewType type); + +/* stringview_wtf8.advance */ +/* stringview_iter.advance */ +int32 +wasm_string_advance(WASMString str_obj, uint32 pos, uint32 count, + uint32 *target_pos); + +/* stringview_wtf8.slice */ +/* stringview_wtf16.slice */ +/* stringview_iter.slice */ +WASMString +wasm_string_slice(WASMString str_obj, uint32 start, uint32 end, + StringViewType type); + +/* stringview_wtf16.get_codeunit */ +int16 +wasm_string_get_wtf16_codeunit(WASMString str_obj, int32 pos); + +/* stringview_iter.next */ +uint32 +wasm_string_next_codepoint(WASMString str_obj, uint32 pos); + +/* stringview_iter.rewind */ +uint32 +wasm_string_rewind(WASMString str_obj, uint32 pos, uint32 count, + uint32 *target_pos); + +/******************* application functions *****************/ + +void +wasm_string_dump(WASMString str_obj); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _STRING_OBJECT_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/common/gc/stringref/stringref_stub.c b/wasm-micro-runtime/core/iwasm/common/gc/stringref/stringref_stub.c new file mode 100644 index 0000000..8a6d624 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/gc/stringref/stringref_stub.c @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* This file is the stub for stringref implementation, only used for wamrc + * compiler. The runtime embedder SHOULD NOT use this file */ + +#include "string_object.h" + +/******************* gc finalizer *****************/ +void +wasm_string_destroy(WASMString str_obj) +{} + +/******************* opcode functions *****************/ + +/* string.const */ +WASMString +wasm_string_new_const(const char *str, uint32 length) +{ + return NULL; +} + +/* string.new_xx8 */ +/* string.new_wtf16 */ +/* string.new_xx8_array */ +/* string.new_wtf16_array */ +WASMString +wasm_string_new_with_encoding(void *addr, uint32 count, EncodingFlag flag) +{ + return NULL; +} + +/* string.measure */ +/* stringview_wtf16.length */ +int32 +wasm_string_measure(WASMString str_obj, EncodingFlag flag) +{ + return 0; +} + +/* stringview_wtf16.length */ +int32 +wasm_string_wtf16_get_length(WASMString str_obj) +{ + return 0; +} + +/* string.encode_xx8 */ +/* string.encode_wtf16 */ +/* stringview_wtf8.encode_xx */ +/* stringview_wtf16.encode */ +/* string.encode_xx8_array */ +/* string.encode_wtf16_array */ +int32 +wasm_string_encode(WASMString str_obj, uint32 pos, uint32 count, void *addr, + uint32 *next_pos, EncodingFlag flag) +{ + return 0; +} + +/* string.concat */ +WASMString +wasm_string_concat(WASMString str_obj1, WASMString str_obj2) +{ + return NULL; +} + +/* string.eq */ +int32 +wasm_string_eq(WASMString str_obj1, WASMString str_obj2) +{ + return 0; +} + +/* string.is_usv_sequence */ +int32 +wasm_string_is_usv_sequence(WASMString str_obj) +{ + return 0; +} + +/* string.as_wtf8 */ +/* string.as_wtf16 */ +/* string.as_iter */ +WASMString +wasm_string_create_view(WASMString str_obj, StringViewType type) +{ + return NULL; +} + +/* stringview_wtf8.advance */ +/* stringview_iter.advance */ +int32 +wasm_string_advance(WASMString str_obj, uint32 pos, uint32 count, + uint32 *consumed) +{ + return 0; +} + +/* stringview_wtf8.slice */ +/* stringview_wtf16.slice */ +/* stringview_iter.slice */ +WASMString +wasm_string_slice(WASMString str_obj, uint32 start, uint32 end, + StringViewType type) +{ + return NULL; +} + +/* stringview_wtf16.get_codeunit */ +int16 +wasm_string_get_wtf16_codeunit(WASMString str_obj, int32 pos) +{ + return 0; +} + +/* stringview_iter.next */ +uint32 +wasm_string_next_codepoint(WASMString str_obj, uint32 pos) +{ + return 0; +} + +/* stringview_iter.rewind */ +uint32 +wasm_string_rewind(WASMString str_obj, uint32 pos, uint32 count, + uint32 *consumed) +{ + return 0; +} + +void +wasm_string_dump(WASMString str_obj) +{} diff --git a/wasm-micro-runtime/core/iwasm/common/iwasm_common.cmake b/wasm-micro-runtime/core/iwasm/common/iwasm_common.cmake new file mode 100644 index 0000000..15895b8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/iwasm_common.cmake @@ -0,0 +1,99 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (IWASM_COMMON_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include_directories (${IWASM_COMMON_DIR}) + +add_definitions(-DBH_MALLOC=wasm_runtime_malloc) +add_definitions(-DBH_FREE=wasm_runtime_free) + +file (GLOB c_source_all ${IWASM_COMMON_DIR}/*.c) + +if (WAMR_DISABLE_APP_ENTRY EQUAL 1) + list(REMOVE_ITEM c_source_all "${IWASM_COMMON_DIR}/wasm_application.c") +endif () + +if (CMAKE_OSX_ARCHITECTURES) + string(TOLOWER "${CMAKE_OSX_ARCHITECTURES}" OSX_ARCHS) + + list(FIND OSX_ARCHS arm64 OSX_AARCH64) + list(FIND OSX_ARCHS x86_64 OSX_X86_64) + + if (NOT "${OSX_AARCH64}" STREQUAL "-1" AND NOT "${OSX_X86_64}" STREQUAL "-1") + set(OSX_UNIVERSAL_BUILD 1) + endif() +endif() + +if (WAMR_BUILD_INVOKE_NATIVE_GENERAL EQUAL 1) + # Use invokeNative C version instead of asm code version + # if WAMR_BUILD_INVOKE_NATIVE_GENERAL is explicitly set. + # Note: + # the maximum number of native arguments is limited to 20, + # and there are possible issues when passing arguments to + # native function for some cpus, e.g. int64 and double arguments + # in arm and mips need to be 8-bytes aligned, and some arguments + # of x86_64 are passed by registers but not stack + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_general.c) +elseif (OSX_UNIVERSAL_BUILD EQUAL 1) + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_osx_universal.s) +elseif (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT WAMR_BUILD_SIMD EQUAL 1) + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + if (NOT MINGW) + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_em64.asm) + else () + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_mingw_x64.s) + endif () + else () + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_em64.s) + endif () + else () + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + if (NOT MINGW) + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_em64_simd.asm) + else () + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_mingw_x64_simd.s) + endif () + else() + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_em64_simd.s) + endif() + endif () +elseif (WAMR_BUILD_TARGET STREQUAL "X86_32") + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_ia32.asm) + else () + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_ia32.s) + endif () +elseif (WAMR_BUILD_TARGET MATCHES "ARM.*") + if (WAMR_BUILD_TARGET MATCHES "ARM.*_VFP") + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_arm_vfp.s) + else () + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_arm.s) + endif () +elseif (WAMR_BUILD_TARGET MATCHES "THUMB.*") + if (WAMR_BUILD_TARGET MATCHES "THUMB.*_VFP") + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_thumb_vfp.s) + else () + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_thumb.s) + endif () +elseif (WAMR_BUILD_TARGET MATCHES "AARCH64.*") + if (NOT WAMR_BUILD_SIMD EQUAL 1) + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_aarch64.s) + else() + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_aarch64_simd.s) + endif() +elseif (WAMR_BUILD_TARGET STREQUAL "MIPS") + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_mips.s) +elseif (WAMR_BUILD_TARGET STREQUAL "XTENSA") + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_xtensa.s) +elseif (WAMR_BUILD_TARGET MATCHES "RISCV*") + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_riscv.S) +elseif (WAMR_BUILD_TARGET STREQUAL "ARC") + set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_arc.s) +else () + message (FATAL_ERROR "Build target isn't set") +endif () + +set (IWASM_COMMON_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_application.c b/wasm-micro-runtime/core/iwasm/common/wasm_application.c new file mode 100644 index 0000000..f19cb00 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_application.c @@ -0,0 +1,910 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif +#if WASM_ENABLE_GC != 0 +#include "gc/gc_object.h" +#if WASM_ENABLE_STRINGREF != 0 +#include "string_object.h" +#endif +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +#include "../../shared/mem-alloc/mem_alloc.h" +#endif +#endif + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +static void * +runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst, + char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + if (module_inst != NULL) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + } + else if (error_buf != NULL) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + } + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) /* NOLINT */ + +/** + * Implementation of wasm_application_execute_main() + */ +static bool +check_main_func_type(const WASMFuncType *type, bool is_memory64) +{ + if (!(type->param_count == 0 || type->param_count == 2) + || type->result_count > 1) { + LOG_ERROR( + "WASM execute application failed: invalid main function type.\n"); + return false; + } + + if (type->param_count == 2 + && !(type->types[0] == VALUE_TYPE_I32 + && type->types[1] + == (is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32))) { + LOG_ERROR( + "WASM execute application failed: invalid main function type.\n"); + return false; + } + + if (type->result_count + && type->types[type->param_count] != VALUE_TYPE_I32) { + LOG_ERROR( + "WASM execute application failed: invalid main function type.\n"); + return false; + } + + return true; +} + +static bool +execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[]) +{ + WASMFunctionInstanceCommon *func; + WASMFuncType *func_type = NULL; + WASMExecEnv *exec_env = NULL; + uint32 argc1 = 0, argv1[3] = { 0 }; + uint32 total_argv_size = 0; + uint64 total_size; + uint64 argv_buf_offset = 0; + int32 i; + char *argv_buf, *p, *p_end; + uint32 *argv_offsets, module_type; + bool ret, is_import_func = true, is_memory64 = false; +#if WASM_ENABLE_MEMORY64 != 0 + WASMModuleInstance *wasm_module_inst = (WASMModuleInstance *)module_inst; + is_memory64 = wasm_module_inst->memories[0]->is_memory64; +#endif + + exec_env = wasm_runtime_get_exec_env_singleton(module_inst); + if (!exec_env) { + wasm_runtime_set_exception(module_inst, + "create singleton exec_env failed"); + return false; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + /* In wasi mode, we should call the function named "_start" + which initializes the wasi envrionment and then calls + the actual main function. Directly calling main function + may cause exception thrown. */ + if ((func = wasm_runtime_lookup_wasi_start_function(module_inst))) { + const char *wasi_proc_exit_exception = "wasi proc exit"; + + ret = wasm_runtime_call_wasm(exec_env, func, 0, NULL); +#if WASM_ENABLE_THREAD_MGR != 0 + if (ret) { + /* On a successful return from the `_start` function, + we terminate other threads by mimicing wasi:proc_exit(0). + + Note: + - A return from the `main` function is an equivalent of + exit(). (C standard) + - When exit code is 0, wasi-libc's `_start` function just + returns w/o calling `proc_exit`. + - A process termination should terminate threads in + the process. */ + + wasm_runtime_set_exception(module_inst, wasi_proc_exit_exception); + /* exit_code is zero-initialized */ + ret = false; + } +#endif + /* report wasm proc exit as a success */ + WASMModuleInstance *inst = (WASMModuleInstance *)module_inst; + if (!ret && strstr(inst->cur_exception, wasi_proc_exit_exception)) { + inst->cur_exception[0] = 0; + ret = true; + } + return ret; + } +#endif /* end of WASM_ENABLE_LIBC_WASI */ + + if (!(func = wasm_runtime_lookup_function(module_inst, "main")) + && !(func = + wasm_runtime_lookup_function(module_inst, "__main_argc_argv")) + && !(func = wasm_runtime_lookup_function(module_inst, "_main"))) { +#if WASM_ENABLE_LIBC_WASI != 0 + wasm_runtime_set_exception( + module_inst, "lookup the entry point symbol (like _start, main, " + "_main, __main_argc_argv) failed"); +#else + wasm_runtime_set_exception(module_inst, + "lookup the entry point symbol (like main, " + "_main, __main_argc_argv) failed"); +#endif + return false; + } + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + is_import_func = ((WASMFunctionInstance *)func)->is_import_func; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + is_import_func = ((AOTFunctionInstance *)func)->is_import_func; + } +#endif + + if (is_import_func) { + wasm_runtime_set_exception(module_inst, "lookup main function failed"); + return false; + } + + module_type = module_inst->module_type; + func_type = wasm_runtime_get_function_type(func, module_type); + + if (!func_type) { + LOG_ERROR("invalid module instance type"); + return false; + } + + if (!check_main_func_type(func_type, is_memory64)) { + wasm_runtime_set_exception(module_inst, + "invalid function type of main function"); + return false; + } + + if (func_type->param_count) { + for (i = 0; i < argc; i++) + total_argv_size += (uint32)(strlen(argv[i]) + 1); + total_argv_size = align_uint(total_argv_size, 4); + + total_size = (uint64)total_argv_size + sizeof(int32) * (uint64)argc; + + if (total_size >= UINT32_MAX + || !(argv_buf_offset = wasm_runtime_module_malloc( + module_inst, total_size, (void **)&argv_buf))) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + return false; + } + + p = argv_buf; + argv_offsets = (uint32 *)(p + total_argv_size); + p_end = p + total_size; + + for (i = 0; i < argc; i++) { + bh_memcpy_s(p, (uint32)(p_end - p), argv[i], + (uint32)(strlen(argv[i]) + 1)); + argv_offsets[i] = (uint32)argv_buf_offset + (uint32)(p - argv_buf); + p += strlen(argv[i]) + 1; + } + + argv1[0] = (uint32)argc; +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + argc1 = 3; + uint64 app_addr = + wasm_runtime_addr_native_to_app(module_inst, argv_offsets); + PUT_I64_TO_ADDR(&argv[1], app_addr); + } + else +#endif + { + argc1 = 2; + argv1[1] = (uint32)wasm_runtime_addr_native_to_app(module_inst, + argv_offsets); + } + } + + ret = wasm_runtime_call_wasm(exec_env, func, argc1, argv1); + if (ret && func_type->result_count > 0 && argc > 0 && argv) + /* copy the return value */ + *(int *)argv = (int)argv1[0]; + + if (argv_buf_offset) + wasm_runtime_module_free(module_inst, argv_buf_offset); + + return ret; +} + +bool +wasm_application_execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, + char *argv[]) +{ + bool ret; +#if (WASM_ENABLE_MEMORY_PROFILING != 0) + WASMExecEnv *exec_env; +#endif + + ret = execute_main(module_inst, argc, argv); + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + exec_env = wasm_runtime_get_exec_env_singleton(module_inst); + if (exec_env) { + wasm_runtime_dump_mem_consumption(exec_env); + } +#endif + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 + void *handle = wasm_runtime_get_gc_heap_handle(module_inst); + mem_allocator_dump_perf_profiling(handle); +#endif + +#if WASM_ENABLE_PERF_PROFILING != 0 + wasm_runtime_dump_perf_profiling(module_inst); +#endif + + if (ret) + ret = wasm_runtime_get_exception(module_inst) == NULL; + + return ret; +} + +/** + * Implementation of wasm_application_execute_func() + */ + +union ieee754_float { + float f; + + /* This is the IEEE 754 single-precision format. */ + union { + struct { + unsigned int negative : 1; + unsigned int exponent : 8; + unsigned int mantissa : 23; + } ieee_big_endian; + struct { + unsigned int mantissa : 23; + unsigned int exponent : 8; + unsigned int negative : 1; + } ieee_little_endian; + } ieee; +}; + +union ieee754_double { + double d; + + /* This is the IEEE 754 double-precision format. */ + union { + struct { + unsigned int negative : 1; + unsigned int exponent : 11; + /* Together these comprise the mantissa. */ + unsigned int mantissa0 : 20; + unsigned int mantissa1 : 32; + } ieee_big_endian; + + struct { + /* Together these comprise the mantissa. */ + unsigned int mantissa1 : 32; + unsigned int mantissa0 : 20; + unsigned int exponent : 11; + unsigned int negative : 1; + } ieee_little_endian; + } ieee; +}; + +static bool +execute_func(WASMModuleInstanceCommon *module_inst, const char *name, + int32 argc, char *argv[]) +{ + WASMFunctionInstanceCommon *target_func; + WASMFuncType *type = NULL; + WASMExecEnv *exec_env = NULL; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *ref_type_map; + WASMLocalObjectRef *local_ref; + uint32 num_local_ref_pushed = 0; +#endif + uint32 argc1, *argv1 = NULL, cell_num = 0, j, k = 0; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + uint32 param_size_in_double_world = 0, result_size_in_double_world = 0; +#endif + int32 i, p, module_type; + uint64 total_size; + char buf[128]; + + bh_assert(argc >= 0); + LOG_DEBUG("call a function \"%s\" with %d arguments", name, argc); + + if (!(target_func = wasm_runtime_lookup_function(module_inst, name))) { + snprintf(buf, sizeof(buf), "lookup function %s failed", name); + wasm_runtime_set_exception(module_inst, buf); + goto fail; + } + + module_type = module_inst->module_type; + type = wasm_runtime_get_function_type(target_func, module_type); + + if (!type) { + LOG_ERROR("invalid module instance type"); + return false; + } + + if (type->param_count != (uint32)argc) { + wasm_runtime_set_exception(module_inst, "invalid input argument count"); + goto fail; + } + + exec_env = wasm_runtime_get_exec_env_singleton(module_inst); + if (!exec_env) { + wasm_runtime_set_exception(module_inst, + "create singleton exec_env failed"); + goto fail; + } + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + for (i = 0; i < type->param_count; i++) { + param_size_in_double_world += + wasm_value_type_cell_num_outside(type->types[i]); + } + for (i = 0; i < type->result_count; i++) { + result_size_in_double_world += wasm_value_type_cell_num_outside( + type->types[type->param_count + i]); + } + argc1 = param_size_in_double_world; + cell_num = (param_size_in_double_world >= result_size_in_double_world) + ? param_size_in_double_world + : result_size_in_double_world; +#else + argc1 = type->param_cell_num; + cell_num = (argc1 > type->ret_cell_num) ? argc1 : type->ret_cell_num; +#endif + + total_size = sizeof(uint32) * (uint64)(cell_num > 2 ? cell_num : 2); + if ((!(argv1 = runtime_malloc((uint32)total_size, module_inst, NULL, 0)))) { + goto fail; + } + +#if WASM_ENABLE_GC != 0 + ref_type_map = type->ref_type_maps; +#endif + /* Parse arguments */ + for (i = 0, p = 0; i < argc; i++) { + char *endptr = NULL; + bh_assert(argv[i] != NULL); + if (argv[i][0] == '\0') { + snprintf(buf, sizeof(buf), "invalid input argument %" PRId32, i); + wasm_runtime_set_exception(module_inst, buf); + goto fail; + } + switch (type->types[i]) { + case VALUE_TYPE_I32: + argv1[p++] = (uint32)strtoul(argv[i], &endptr, 0); + break; + case VALUE_TYPE_I64: + { + union { + uint64 val; + uint32 parts[2]; + } u; + u.val = strtoull(argv[i], &endptr, 0); + argv1[p++] = u.parts[0]; + argv1[p++] = u.parts[1]; + break; + } + case VALUE_TYPE_F32: + { + float32 f32 = strtof(argv[i], &endptr); + if (isnan(f32)) { +#ifdef _MSC_VER + /* + * Spec tests require the binary representation of NaN to be + * 0x7fc00000 for float and 0x7ff8000000000000 for float; + * however, in MSVC compiler, strtof doesn't return this + * exact value, causing some of the spec test failures. We + * use the value returned by nan/nanf as it is the one + * expected by spec tests. + * + */ + f32 = nanf(""); +#endif + if (argv[i][0] == '-') { + union ieee754_float u; + u.f = f32; + if (is_little_endian()) + u.ieee.ieee_little_endian.negative = 1; + else + u.ieee.ieee_big_endian.negative = 1; + bh_memcpy_s(&f32, sizeof(float), &u.f, sizeof(float)); + } + if (endptr[0] == ':') { + uint32 sig; + union ieee754_float u; + sig = (uint32)strtoul(endptr + 1, &endptr, 0); + u.f = f32; + if (is_little_endian()) + u.ieee.ieee_little_endian.mantissa = sig; + else + u.ieee.ieee_big_endian.mantissa = sig; + bh_memcpy_s(&f32, sizeof(float), &u.f, sizeof(float)); + } + } + bh_memcpy_s(&argv1[p], (uint32)total_size - p, &f32, + (uint32)sizeof(float)); + p++; + break; + } + case VALUE_TYPE_F64: + { + union { + float64 val; + uint32 parts[2]; + } u; + u.val = strtod(argv[i], &endptr); + if (isnan(u.val)) { +#ifdef _MSC_VER + u.val = nan(""); +#endif + if (argv[i][0] == '-') { + union ieee754_double ud; + ud.d = u.val; + if (is_little_endian()) + ud.ieee.ieee_little_endian.negative = 1; + else + ud.ieee.ieee_big_endian.negative = 1; + bh_memcpy_s(&u.val, sizeof(double), &ud.d, + sizeof(double)); + } + if (endptr[0] == ':') { + uint64 sig; + union ieee754_double ud; + sig = strtoull(endptr + 1, &endptr, 0); + ud.d = u.val; + if (is_little_endian()) { + ud.ieee.ieee_little_endian.mantissa0 = sig >> 32; + ud.ieee.ieee_little_endian.mantissa1 = (uint32)sig; + } + else { + ud.ieee.ieee_big_endian.mantissa0 = sig >> 32; + ud.ieee.ieee_big_endian.mantissa1 = (uint32)sig; + } + bh_memcpy_s(&u.val, sizeof(double), &ud.d, + sizeof(double)); + } + } + argv1[p++] = u.parts[0]; + argv1[p++] = u.parts[1]; + break; + } +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + { + /* it likes 0x123\0x234 or 123\234 */ + /* retrive first i64 */ + *(uint64 *)(argv1 + p) = strtoull(argv[i], &endptr, 0); + /* skip \ */ + endptr++; + /* retrive second i64 */ + *(uint64 *)(argv1 + p + 2) = strtoull(endptr, &endptr, 0); + p += 4; + break; + } +#endif /* WASM_ENABLE_SIMD != 0 */ +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#if UINTPTR_MAX == UINT32_MAX + case VALUE_TYPE_EXTERNREF: +#endif + { + if (strncasecmp(argv[i], "null", 4) == 0) { + argv1[p++] = (uint32)-1; + } + else { + argv1[p++] = (uint32)strtoul(argv[i], &endptr, 0); + } + break; + } +#if UINTPTR_MAX == UINT64_MAX + case VALUE_TYPE_EXTERNREF: + { + union { + uintptr_t val; + uint32 parts[2]; + } u; + if (strncasecmp(argv[i], "null", 4) == 0) { + u.val = (uintptr_t)-1LL; + } + else { + u.val = strtoull(argv[i], &endptr, 0); + } + argv1[p++] = u.parts[0]; + argv1[p++] = u.parts[1]; + break; + } +#endif +#endif /* WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */ + default: + { +#if WASM_ENABLE_GC != 0 + bool is_extern_ref = false; + bool is_anyref = false; + + if (wasm_is_type_reftype(type->types[i])) { + if (strncasecmp(argv[i], "null", 4) == 0) { + PUT_REF_TO_ADDR(argv1 + p, NULL_REF); + p += REF_CELL_NUM; + break; + } + else if (type->types[i] == VALUE_TYPE_EXTERNREF) { + is_extern_ref = true; + } + else if (type->types[i] == VALUE_TYPE_ANYREF) { + is_anyref = true; + } + + if (wasm_is_type_multi_byte_type(type->types[i])) { + WASMRefType *ref_type = ref_type_map->ref_type; + if (wasm_is_refheaptype_common( + &ref_type->ref_ht_common)) { + int32 heap_type = ref_type->ref_ht_common.heap_type; + if (heap_type == HEAP_TYPE_EXTERN) { + is_extern_ref = true; + } + else if (heap_type == HEAP_TYPE_ANY) { + is_anyref = true; + } + } + + ref_type_map++; + } + + if (is_extern_ref) { + WASMExternrefObjectRef gc_obj; + void *extern_obj = + (void *)(uintptr_t)strtoull(argv[i], &endptr, 0); + gc_obj = wasm_externref_obj_new(exec_env, extern_obj); + if (!gc_obj) { + wasm_runtime_set_exception( + module_inst, "create extern object failed"); + goto fail; + } + if (!(local_ref = + runtime_malloc(sizeof(WASMLocalObjectRef), + module_inst, NULL, 0))) { + goto fail; + } + wasm_runtime_push_local_obj_ref(exec_env, local_ref); + local_ref->val = (WASMObjectRef)gc_obj; + num_local_ref_pushed++; + PUT_REF_TO_ADDR(argv1 + p, gc_obj); + p += REF_CELL_NUM; + } + else if (is_anyref) { + /* If a parameter type is (ref null? any) and its value + * is not null, then we treat the value as host ptr */ + WASMAnyrefObjectRef gc_obj; + void *host_obj = + (void *)(uintptr_t)strtoull(argv[i], &endptr, 0); + gc_obj = wasm_anyref_obj_new(exec_env, host_obj); + if (!gc_obj) { + wasm_runtime_set_exception( + module_inst, "create anyref object failed"); + goto fail; + } + if (!(local_ref = + runtime_malloc(sizeof(WASMLocalObjectRef), + module_inst, NULL, 0))) { + goto fail; + } + wasm_runtime_push_local_obj_ref(exec_env, local_ref); + local_ref->val = (WASMObjectRef)gc_obj; + num_local_ref_pushed++; + PUT_REF_TO_ADDR(argv1 + p, gc_obj); + p += REF_CELL_NUM; + } + + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + bh_assert(0); + break; + } + } + if (endptr && *endptr != '\0' && *endptr != '_') { + snprintf(buf, sizeof(buf), "invalid input argument %" PRId32 ": %s", + i, argv[i]); + wasm_runtime_set_exception(module_inst, buf); + goto fail; + } + } + + wasm_runtime_set_exception(module_inst, NULL); +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + bh_assert(p == (int32)argc1); +#endif + + if (!wasm_runtime_call_wasm(exec_env, target_func, argc1, argv1)) { + goto fail; + } + +#if WASM_ENABLE_GC != 0 + ref_type_map = type->result_ref_type_maps; +#endif + /* print return value */ + for (j = 0; j < type->result_count; j++) { + switch (type->types[type->param_count + j]) { + case VALUE_TYPE_I32: + { + os_printf("0x%" PRIx32 ":i32", argv1[k]); + k++; + break; + } + case VALUE_TYPE_I64: + { + union { + uint64 val; + uint32 parts[2]; + } u; + u.parts[0] = argv1[k]; + u.parts[1] = argv1[k + 1]; + k += 2; + os_printf("0x%" PRIx64 ":i64", u.val); + break; + } + case VALUE_TYPE_F32: + { + os_printf("%.7g:f32", *(float32 *)(argv1 + k)); + k++; + break; + } + case VALUE_TYPE_F64: + { + union { + float64 val; + uint32 parts[2]; + } u; + u.parts[0] = argv1[k]; + u.parts[1] = argv1[k + 1]; + k += 2; + os_printf("%.7g:f64", u.val); + break; + } +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + { + if (argv1[k] != NULL_REF) + os_printf("%" PRIu32 ":ref.func", argv1[k]); + else + os_printf("func:ref.null"); + k++; + break; + } + case VALUE_TYPE_EXTERNREF: + { +#if UINTPTR_MAX == UINT32_MAX + if (argv1[k] != 0 && argv1[k] != (uint32)-1) + os_printf("0x%" PRIxPTR ":ref.extern", (uintptr_t)argv1[k]); + else + os_printf("extern:ref.null"); + k++; +#else + union { + uintptr_t val; + uint32 parts[2]; + } u; + u.parts[0] = argv1[k]; + u.parts[1] = argv1[k + 1]; + k += 2; + if (u.val && u.val != (uintptr_t)-1LL) + os_printf("0x%" PRIxPTR ":ref.extern", u.val); + else + os_printf("extern:ref.null"); +#endif + break; + } +#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */ +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + { + uint64 *v = (uint64 *)(argv1 + k); + os_printf("<0x%016" PRIx64 " 0x%016" PRIx64 ">:v128", *v, + *(v + 1)); + k += 4; + break; + } +#endif /* WASM_ENABLE_SIMD != 0 */ + default: + { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(type->types[type->param_count + j])) { + void *gc_obj = GET_REF_FROM_ADDR(argv1 + k); + k += REF_CELL_NUM; + if (!gc_obj) { + uint8 type1 = type->types[type->param_count + j]; + WASMRefType *ref_type1 = NULL; + WASMType **types = NULL; + uint32 type_count = 0; + + if (wasm_is_type_multi_byte_type( + type->types[type->param_count + j])) + ref_type1 = ref_type_map->ref_type; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModule *module = + ((WASMModuleInstance *)module_inst)->module; + types = module->types; + type_count = module->type_count; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)module_inst) + ->module; + types = module->types; + type_count = module->type_count; + } +#endif + bh_assert(type); + if (wasm_reftype_is_subtype_of(type1, ref_type1, + REF_TYPE_ANYREF, NULL, + types, type_count)) + os_printf("any:"); + else if (wasm_reftype_is_subtype_of( + type1, ref_type1, REF_TYPE_FUNCREF, NULL, + types, type_count)) + os_printf("func:"); + if (wasm_reftype_is_subtype_of(type1, ref_type1, + REF_TYPE_EXTERNREF, NULL, + types, type_count)) + os_printf("extern:"); + os_printf("ref.null"); + } + else if (wasm_obj_is_func_obj(gc_obj)) + os_printf("ref.func"); +#if WASM_ENABLE_STRINGREF != 0 + else if (wasm_obj_is_stringref_obj(gc_obj) + || wasm_obj_is_stringview_wtf8_obj(gc_obj)) { + wasm_string_dump( + (WASMString)wasm_stringref_obj_get_value(gc_obj)); + } + else if (wasm_obj_is_stringview_wtf16_obj(gc_obj)) { + wasm_string_dump( + (WASMString)wasm_stringview_wtf16_obj_get_value( + gc_obj)); + } +#endif + else if (wasm_obj_is_externref_obj(gc_obj)) { +#if WASM_ENABLE_SPEC_TEST != 0 + WASMObjectRef obj = wasm_externref_obj_to_internal_obj( + (WASMExternrefObjectRef)gc_obj); + if (wasm_obj_is_anyref_obj(obj)) + os_printf("0x%" PRIxPTR ":ref.extern", + (uintptr_t)wasm_anyref_obj_get_value( + (WASMAnyrefObjectRef)obj)); + else +#endif + os_printf("ref.extern"); + } + else if (wasm_obj_is_i31_obj(gc_obj)) + os_printf("ref.i31"); + else if (wasm_obj_is_array_obj(gc_obj)) + os_printf("ref.array"); + else if (wasm_obj_is_struct_obj(gc_obj)) + os_printf("ref.struct"); + else if (wasm_obj_is_eq_obj(gc_obj)) + os_printf("ref.eq"); + else if (wasm_obj_is_anyref_obj(gc_obj)) + os_printf("0x%" PRIxPTR ":ref.host", + (uintptr_t)wasm_anyref_obj_get_value( + (WASMAnyrefObjectRef)gc_obj)); + else if (wasm_obj_is_internal_obj(gc_obj)) + os_printf("ref.any"); + + if (wasm_is_type_multi_byte_type( + type->types[type->param_count + j])) + ref_type_map++; + + break; + } +#endif /* endof WASM_ENABLE_GC != 0 */ + bh_assert(0); + break; + } + } + if (j < (uint32)(type->result_count - 1)) + os_printf(","); + } + os_printf("\n"); + +#if WASM_ENABLE_GC != 0 + for (j = 0; j < num_local_ref_pushed; j++) { + local_ref = wasm_runtime_pop_local_obj_ref(exec_env); + wasm_runtime_free(local_ref); + } +#endif + + wasm_runtime_free(argv1); + return true; + +fail: + if (argv1) + wasm_runtime_free(argv1); + +#if WASM_ENABLE_GC != 0 + for (j = 0; j < num_local_ref_pushed; j++) { + local_ref = wasm_runtime_pop_local_obj_ref(exec_env); + wasm_runtime_free(local_ref); + } +#endif + + bh_assert(wasm_runtime_get_exception(module_inst)); + return false; +} + +bool +wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, + const char *name, int32 argc, char *argv[]) +{ + bool ret; +#if WASM_ENABLE_MEMORY_PROFILING != 0 + WASMExecEnv *exec_env; +#endif + + ret = execute_func(module_inst, name, argc, argv); + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + exec_env = wasm_runtime_get_exec_env_singleton(module_inst); + if (exec_env) { + wasm_runtime_dump_mem_consumption(exec_env); + } +#endif + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 + void *handle = wasm_runtime_get_gc_heap_handle(module_inst); + mem_allocator_dump_perf_profiling(handle); +#endif + +#if WASM_ENABLE_PERF_PROFILING != 0 + wasm_runtime_dump_perf_profiling(module_inst); +#endif + + return (ret && !wasm_runtime_get_exception(module_inst)) ? true : false; +} diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_blocking_op.c b/wasm-micro-runtime/core/iwasm/common/wasm_blocking_op.c new file mode 100644 index 0000000..25777c8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_blocking_op.c @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_runtime_common.h" + +#include "bh_platform.h" +#include "bh_common.h" +#include "bh_assert.h" + +#if WASM_ENABLE_THREAD_MGR != 0 && defined(OS_ENABLE_WAKEUP_BLOCKING_OP) + +#define LOCK(env) WASM_SUSPEND_FLAGS_LOCK((env)->wait_lock) +#define UNLOCK(env) WASM_SUSPEND_FLAGS_UNLOCK((env)->wait_lock) + +#define ISSET(env, bit) \ + ((WASM_SUSPEND_FLAGS_GET((env)->suspend_flags) & WASM_SUSPEND_FLAG_##bit) \ + != 0) +#define SET(env, bit) \ + WASM_SUSPEND_FLAGS_FETCH_OR((env)->suspend_flags, WASM_SUSPEND_FLAG_##bit) +#define CLR(env, bit) \ + WASM_SUSPEND_FLAGS_FETCH_AND((env)->suspend_flags, ~WASM_SUSPEND_FLAG_##bit) + +bool +wasm_runtime_begin_blocking_op(wasm_exec_env_t env) +{ + LOCK(env); + bh_assert(!ISSET(env, BLOCKING)); + SET(env, BLOCKING); + if (ISSET(env, TERMINATE)) { + CLR(env, BLOCKING); + UNLOCK(env); + return false; + } + UNLOCK(env); + os_begin_blocking_op(); + return true; +} + +void +wasm_runtime_end_blocking_op(wasm_exec_env_t env) +{ + int saved_errno = errno; + LOCK(env); + bh_assert(ISSET(env, BLOCKING)); + CLR(env, BLOCKING); + UNLOCK(env); + os_end_blocking_op(); + errno = saved_errno; +} + +void +wasm_runtime_interrupt_blocking_op(wasm_exec_env_t env) +{ + /* + * ISSET(BLOCKING) here means that the target thread + * is in somewhere between wasm_begin_blocking_op and + * wasm_end_blocking_op. + * keep waking it up until it reaches wasm_end_blocking_op, + * which clears the BLOCKING bit. + * + * this dumb loop is necessary because posix doesn't provide + * a way to unmask signal and block atomically. + */ + + LOCK(env); + SET(env, TERMINATE); + while (ISSET(env, BLOCKING)) { + UNLOCK(env); + os_wakeup_blocking_op(env->handle); + + /* relax a bit */ + os_usleep(50 * 1000); + LOCK(env); + } + UNLOCK(env); +} + +#else /* WASM_ENABLE_THREAD_MGR && OS_ENABLE_WAKEUP_BLOCKING_OP */ + +bool +wasm_runtime_begin_blocking_op(wasm_exec_env_t env) +{ + return true; +} + +void +wasm_runtime_end_blocking_op(wasm_exec_env_t env) +{} + +#endif /* WASM_ENABLE_THREAD_MGR && OS_ENABLE_WAKEUP_BLOCKING_OP */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_c_api.c b/wasm-micro-runtime/core/iwasm/common/wasm_c_api.c new file mode 100644 index 0000000..456ce50 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_c_api.c @@ -0,0 +1,5345 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_log.h" +#include "wasm_c_api_internal.h" + +#include "bh_assert.h" +#include "wasm_export.h" +#include "wasm_memory.h" +#if WASM_ENABLE_INTERP != 0 +#include "wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "aot_runtime.h" +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 +#include "aot.h" +#include "aot_llvm.h" +#endif /*WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0*/ +#endif /*WASM_ENABLE_AOT != 0*/ + +#if WASM_ENABLE_WASM_CACHE != 0 +#include +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "thread_manager.h" +#endif + +/* + * Thread Model: + * - Only one wasm_engine_t in one process + * - One wasm_store_t is only accessed by one thread. wasm_store_t can't be + * shared in threads + * - wasm_module_t can be shared in threads + * - wasm_instance_t can not be shared in threads + */ + +#define ASSERT_NOT_IMPLEMENTED() bh_assert(!"not implemented") +#define UNREACHABLE() bh_assert(!"unreachable") + +typedef struct wasm_module_ex_t { + struct WASMModuleCommon *module_comm_rt; + wasm_byte_vec_t *binary; + korp_mutex lock; + uint32 ref_count; +#if WASM_ENABLE_WASM_CACHE != 0 + char hash[SHA256_DIGEST_LENGTH]; +#endif +} wasm_module_ex_t; + +#ifndef os_thread_local_attribute +typedef struct thread_local_stores { + korp_tid tid; + unsigned stores_num; +} thread_local_stores; +#endif + +static void +wasm_module_delete_internal(wasm_module_t *); + +static void +wasm_instance_delete_internal(wasm_instance_t *); + +/* temporarily put stubs here */ +static wasm_store_t * +wasm_store_copy(const wasm_store_t *src) +{ + (void)src; + LOG_WARNING("in the stub of %s", __FUNCTION__); + return NULL; +} + +wasm_module_t * +wasm_module_copy(const wasm_module_t *src) +{ + (void)src; + LOG_WARNING("in the stub of %s", __FUNCTION__); + return NULL; +} + +wasm_instance_t * +wasm_instance_copy(const wasm_instance_t *src) +{ + (void)src; + LOG_WARNING("in the stub of %s", __FUNCTION__); + return NULL; +} + +/* ---------------------------------------------------------------------- */ +static inline void * +malloc_internal(uint64 size) +{ + void *mem = NULL; + + if (size < UINT32_MAX && (mem = wasm_runtime_malloc((uint32)size))) { + memset(mem, 0, size); + } + + return mem; +} + +/* clang-format off */ +#define RETURN_OBJ(obj, obj_del_func) \ + return obj; \ +failed: \ + obj_del_func(obj); \ + return NULL; + +#define RETURN_VOID(obj, obj_del_func) \ + return; \ +failed: \ + obj_del_func(obj); \ + return; +/* clang-format on */ + +/* Vectors */ +#define INIT_VEC(vector_p, init_func, ...) \ + do { \ + if (!(vector_p = malloc_internal(sizeof(*(vector_p))))) { \ + goto failed; \ + } \ + \ + init_func(vector_p, ##__VA_ARGS__); \ + if (vector_p->size && !vector_p->data) { \ + LOG_DEBUG("%s failed", #init_func); \ + goto failed; \ + } \ + } while (false) + +#define DEINIT_VEC(vector_p, deinit_func) \ + if ((vector_p)) { \ + deinit_func(vector_p); \ + wasm_runtime_free(vector_p); \ + vector_p = NULL; \ + } + +#define WASM_DEFINE_VEC(name) \ + void wasm_##name##_vec_new_empty(own wasm_##name##_vec_t *out) \ + { \ + wasm_##name##_vec_new_uninitialized(out, 0); \ + } \ + void wasm_##name##_vec_new_uninitialized(own wasm_##name##_vec_t *out, \ + size_t size) \ + { \ + wasm_##name##_vec_new(out, size, NULL); \ + } + +/* vectors with no ownership management of elements */ +#define WASM_DEFINE_VEC_PLAIN(name) \ + WASM_DEFINE_VEC(name) \ + void wasm_##name##_vec_new(own wasm_##name##_vec_t *out, size_t size, \ + own wasm_##name##_t const data[]) \ + { \ + if (!out) { \ + return; \ + } \ + \ + memset(out, 0, sizeof(wasm_##name##_vec_t)); \ + \ + if (!size) { \ + return; \ + } \ + \ + if (!bh_vector_init((Vector *)out, size, sizeof(wasm_##name##_t), \ + true)) { \ + LOG_DEBUG("bh_vector_init failed"); \ + goto failed; \ + } \ + \ + if (data) { \ + uint32 size_in_bytes = 0; \ + size_in_bytes = (uint32)(size * sizeof(wasm_##name##_t)); \ + bh_memcpy_s(out->data, size_in_bytes, data, size_in_bytes); \ + out->num_elems = size; \ + } \ + \ + RETURN_VOID(out, wasm_##name##_vec_delete) \ + } \ + void wasm_##name##_vec_copy(wasm_##name##_vec_t *out, \ + const wasm_##name##_vec_t *src) \ + { \ + if (!src) { \ + return; \ + } \ + wasm_##name##_vec_new(out, src->size, src->data); \ + } \ + void wasm_##name##_vec_delete(wasm_##name##_vec_t *v) \ + { \ + if (v) { \ + bh_vector_destroy((Vector *)v); \ + } \ + } + +/* vectors that own their elements */ +#define WASM_DEFINE_VEC_OWN(name, elem_destroy_func) \ + WASM_DEFINE_VEC(name) \ + void wasm_##name##_vec_new(own wasm_##name##_vec_t *out, size_t size, \ + own wasm_##name##_t *const data[]) \ + { \ + if (!out) { \ + return; \ + } \ + \ + memset(out, 0, sizeof(wasm_##name##_vec_t)); \ + \ + if (!size) { \ + return; \ + } \ + \ + if (!bh_vector_init((Vector *)out, size, sizeof(wasm_##name##_t *), \ + true)) { \ + LOG_DEBUG("bh_vector_init failed"); \ + goto failed; \ + } \ + \ + if (data) { \ + uint32 size_in_bytes = 0; \ + size_in_bytes = (uint32)(size * sizeof(wasm_##name##_t *)); \ + bh_memcpy_s(out->data, size_in_bytes, data, size_in_bytes); \ + out->num_elems = size; \ + } \ + \ + RETURN_VOID(out, wasm_##name##_vec_delete) \ + } \ + void wasm_##name##_vec_copy(own wasm_##name##_vec_t *out, \ + const wasm_##name##_vec_t *src) \ + { \ + size_t i = 0; \ + \ + if (!out) { \ + return; \ + } \ + memset(out, 0, sizeof(Vector)); \ + \ + if (!src || !src->size) { \ + return; \ + } \ + \ + if (!bh_vector_init((Vector *)out, src->size, \ + sizeof(wasm_##name##_t *), true)) { \ + LOG_DEBUG("bh_vector_init failed"); \ + goto failed; \ + } \ + \ + for (i = 0; i != src->num_elems; ++i) { \ + if (!(out->data[i] = wasm_##name##_copy(src->data[i]))) { \ + LOG_DEBUG("wasm_%s_copy failed", #name); \ + goto failed; \ + } \ + } \ + out->num_elems = src->num_elems; \ + \ + RETURN_VOID(out, wasm_##name##_vec_delete) \ + } \ + void wasm_##name##_vec_delete(wasm_##name##_vec_t *v) \ + { \ + size_t i = 0; \ + if (!v) { \ + return; \ + } \ + for (i = 0; i != v->num_elems && v->data; ++i) { \ + elem_destroy_func(*(v->data + i)); \ + } \ + bh_vector_destroy((Vector *)v); \ + } + +WASM_DEFINE_VEC_PLAIN(byte) +WASM_DEFINE_VEC_PLAIN(val) + +WASM_DEFINE_VEC_OWN(exporttype, wasm_exporttype_delete) +WASM_DEFINE_VEC_OWN(extern, wasm_extern_delete) +WASM_DEFINE_VEC_OWN(frame, wasm_frame_delete) +WASM_DEFINE_VEC_OWN(functype, wasm_functype_delete) +WASM_DEFINE_VEC_OWN(importtype, wasm_importtype_delete) +WASM_DEFINE_VEC_OWN(instance, wasm_instance_delete_internal) +WASM_DEFINE_VEC_OWN(module, wasm_module_delete_internal) +WASM_DEFINE_VEC_OWN(store, wasm_store_delete) +WASM_DEFINE_VEC_OWN(valtype, wasm_valtype_delete) + +#ifndef NDEBUG +#if WASM_ENABLE_MEMORY_PROFILING != 0 +#define WASM_C_DUMP_PROC_MEM() LOG_PROC_MEM() +#else +#define WASM_C_DUMP_PROC_MEM() (void)0 +#endif +#else +#define WASM_C_DUMP_PROC_MEM() (void)0 +#endif + +/* Runtime Environment */ +own wasm_config_t * +wasm_config_new(void) +{ + /* since wasm_runtime_malloc is not ready */ + wasm_config_t *config = os_malloc(sizeof(wasm_config_t)); + if (!config) + return NULL; + + memset(config, 0, sizeof(wasm_config_t)); + config->mem_alloc_type = Alloc_With_System_Allocator; + + return config; +} + +void +wasm_config_delete(own wasm_config_t *config) +{ + if (config) + os_free(config); +} + +wasm_config_t * +wasm_config_set_mem_alloc_opt(wasm_config_t *config, + mem_alloc_type_t mem_alloc_type, + MemAllocOption *mem_alloc_option) +{ + if (!config) + return NULL; + + config->mem_alloc_type = mem_alloc_type; + if (mem_alloc_option) + memcpy(&config->mem_alloc_option, mem_alloc_option, + sizeof(MemAllocOption)); + return config; +} + +wasm_config_t * +wasm_config_set_linux_perf_opt(wasm_config_t *config, bool enable) +{ + if (!config) + return NULL; + + config->enable_linux_perf = enable; + return config; +} + +wasm_config_t * +wasm_config_set_segue_flags(wasm_config_t *config, uint32 segue_flags) +{ + if (!config) + return NULL; + + config->segue_flags = segue_flags; + return config; +} + +static void +wasm_engine_delete_internal(wasm_engine_t *engine) +{ + if (engine) { + /* clean all created wasm_module_t and their locks */ + unsigned i; + + for (i = 0; i < engine->modules.num_elems; i++) { + wasm_module_ex_t *module; + if (bh_vector_get(&engine->modules, i, &module)) { + os_mutex_destroy(&module->lock); + wasm_runtime_free(module); + } + } + + bh_vector_destroy(&engine->modules); + +#ifndef os_thread_local_attribute + bh_vector_destroy(&engine->stores_by_tid); +#endif + + wasm_runtime_free(engine); + } + + wasm_runtime_destroy(); +} + +static wasm_engine_t * +wasm_engine_new_internal(wasm_config_t *config) +{ + wasm_engine_t *engine = NULL; + /* init runtime */ + RuntimeInitArgs init_args = { 0 }; +#if WASM_ENABLE_JIT != 0 + LLVMJITOptions *jit_options = wasm_runtime_get_llvm_jit_options(); +#endif + +#ifndef NDEBUG + bh_log_set_verbose_level(BH_LOG_LEVEL_VERBOSE); +#else + bh_log_set_verbose_level(BH_LOG_LEVEL_WARNING); +#endif + + WASM_C_DUMP_PROC_MEM(); + + /* wasm_config_t->MemAllocOption -> RuntimeInitArgs->MemAllocOption */ + init_args.mem_alloc_type = config->mem_alloc_type; + memcpy(&init_args.mem_alloc_option, &config->mem_alloc_option, + sizeof(MemAllocOption)); + init_args.enable_linux_perf = config->enable_linux_perf; + init_args.segue_flags = config->segue_flags; + +#if WASM_ENABLE_JIT != 0 + jit_options->quick_invoke_c_api_import = true; +#endif + + if (!wasm_runtime_full_init(&init_args)) { + LOG_DEBUG("wasm_runtime_full_init failed"); + goto failed; + } + + /* create wasm_engine_t */ + if (!(engine = malloc_internal(sizeof(wasm_engine_t)))) { + goto failed; + } + + if (!bh_vector_init(&engine->modules, DEFAULT_VECTOR_INIT_SIZE, + sizeof(wasm_module_ex_t *), true)) + goto failed; + +#ifndef os_thread_local_attribute + if (!bh_vector_init(&engine->stores_by_tid, DEFAULT_VECTOR_INIT_SIZE, + sizeof(thread_local_stores), true)) + goto failed; +#endif + + engine->ref_count = 1; + + WASM_C_DUMP_PROC_MEM(); + + RETURN_OBJ(engine, wasm_engine_delete_internal) +} + +/* global engine instance */ +static wasm_engine_t *singleton_engine; +#ifdef os_thread_local_attribute +/* categorize wasm_store_t as threads*/ +static os_thread_local_attribute unsigned thread_local_stores_num = 0; +#endif +#if defined(OS_THREAD_MUTEX_INITIALIZER) +/** + * lock for the singleton_engine + * Note: if the platform has mutex initializer, we use a global lock to + * lock the operations of the singleton_engine, otherwise when there are + * operations happening simultaneously in multiple threads, developer + * must create the lock by himself, and use it to lock the operations + */ +static korp_mutex engine_lock = OS_THREAD_MUTEX_INITIALIZER; +#endif + +own wasm_engine_t * +wasm_engine_new() +{ + wasm_config_t config = { 0 }; + wasm_config_set_mem_alloc_opt(&config, Alloc_With_System_Allocator, NULL); + wasm_engine_t *engine = wasm_engine_new_with_config(&config); + return engine; +} + +own wasm_engine_t * +wasm_engine_new_with_config(wasm_config_t *config) +{ +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (!singleton_engine) + singleton_engine = wasm_engine_new_internal(config); + else + singleton_engine->ref_count++; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + + return singleton_engine; +} + +own wasm_engine_t * +wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts) +{ + wasm_config_t config = { 0 }; + config.mem_alloc_type = type; + memcpy(&config.mem_alloc_option, opts, sizeof(MemAllocOption)); + return wasm_engine_new_with_config(&config); +} + +void +wasm_engine_delete(wasm_engine_t *engine) +{ + if (!engine) + return; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (!singleton_engine) { +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + return; + } + + bh_assert(engine == singleton_engine); + bh_assert(singleton_engine->ref_count > 0); + + singleton_engine->ref_count--; + if (singleton_engine->ref_count == 0) { + wasm_engine_delete_internal(engine); + singleton_engine = NULL; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif +} + +#ifndef os_thread_local_attribute +static bool +search_thread_local_store_num(Vector *stores_by_tid, korp_tid tid, + thread_local_stores *out_ts, unsigned *out_i) +{ + unsigned i; + + for (i = 0; i < stores_by_tid->num_elems; i++) { + bool ret = bh_vector_get(stores_by_tid, i, out_ts); + bh_assert(ret); + (void)ret; + + if (out_ts->tid == tid) { + *out_i = i; + return true; + } + } + + return false; +} +#endif + +static unsigned +retrive_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) +{ +#ifndef os_thread_local_attribute + unsigned i = 0; + thread_local_stores ts = { 0 }; + unsigned ret = 0; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (search_thread_local_store_num(stores_by_tid, tid, &ts, &i)) + ret = ts.stores_num; + else + ret = 0; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + + return ret; +#else + (void)stores_by_tid; + (void)tid; + + return thread_local_stores_num; +#endif +} + +static bool +increase_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) +{ +#ifndef os_thread_local_attribute + unsigned i = 0; + thread_local_stores ts = { 0 }; + bool ret = false; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (search_thread_local_store_num(stores_by_tid, tid, &ts, &i)) { + /* just in case if integer overflow */ + if (ts.stores_num + 1 < ts.stores_num) { + ret = false; + } + else { + ts.stores_num++; + ret = bh_vector_set(stores_by_tid, i, &ts); + bh_assert(ret); + } + } + else { + ts.tid = tid; + ts.stores_num = 1; + ret = bh_vector_append(stores_by_tid, &ts); + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + return ret; +#else + (void)stores_by_tid; + (void)tid; + + /* just in case if integer overflow */ + if (thread_local_stores_num + 1 < thread_local_stores_num) + return false; + + thread_local_stores_num++; + return true; +#endif +} + +static bool +decrease_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) +{ +#ifndef os_thread_local_attribute + unsigned i = 0; + thread_local_stores ts = { 0 }; + bool ret = false; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + ret = search_thread_local_store_num(stores_by_tid, tid, &ts, &i); + bh_assert(ret); + + /* just in case if integer overflow */ + if (ts.stores_num - 1 > ts.stores_num) { + ret = false; + } + else { + ts.stores_num--; + ret = bh_vector_set(stores_by_tid, i, &ts); + bh_assert(ret); + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + + return ret; +#else + (void)stores_by_tid; + (void)tid; + + /* just in case if integer overflow */ + if (thread_local_stores_num - 1 > thread_local_stores_num) + return false; + + thread_local_stores_num--; + return true; +#endif +} + +wasm_store_t * +wasm_store_new(wasm_engine_t *engine) +{ + wasm_store_t *store = NULL; + + WASM_C_DUMP_PROC_MEM(); + + if (!engine || singleton_engine != engine) + return NULL; + + if (!retrive_thread_local_store_num(&engine->stores_by_tid, + os_self_thread())) { + if (!wasm_runtime_init_thread_env()) { + LOG_ERROR("init thread environment failed"); + return NULL; + } + + if (!increase_thread_local_store_num(&engine->stores_by_tid, + os_self_thread())) { + wasm_runtime_destroy_thread_env(); + return NULL; + } + + if (!(store = malloc_internal(sizeof(wasm_store_t)))) { + decrease_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread()); + wasm_runtime_destroy_thread_env(); + return NULL; + } + } + else { + if (!increase_thread_local_store_num(&engine->stores_by_tid, + os_self_thread())) + return NULL; + + if (!(store = malloc_internal(sizeof(wasm_store_t)))) { + decrease_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread()); + return NULL; + } + } + + /* new a vector, and new its data */ + INIT_VEC(store->modules, wasm_module_vec_new_uninitialized, + DEFAULT_VECTOR_INIT_LENGTH); + INIT_VEC(store->instances, wasm_instance_vec_new_uninitialized, + DEFAULT_VECTOR_INIT_LENGTH); + + if (!(store->foreigns = malloc_internal(sizeof(Vector))) + || !(bh_vector_init(store->foreigns, 24, sizeof(wasm_foreign_t *), + true))) { + goto failed; + } + + WASM_C_DUMP_PROC_MEM(); + + return store; +failed: + wasm_store_delete(store); + return NULL; +} + +void +wasm_store_delete(wasm_store_t *store) +{ + if (!store) { + return; + } + + DEINIT_VEC(store->instances, wasm_instance_vec_delete); + DEINIT_VEC(store->modules, wasm_module_vec_delete); + if (store->foreigns) { + bh_vector_destroy(store->foreigns); + wasm_runtime_free(store->foreigns); + } + + wasm_runtime_free(store); + + if (decrease_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread())) { + if (!retrive_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread())) { + wasm_runtime_destroy_thread_env(); + } + } +} + +/* Type Representations */ +static inline wasm_valkind_t +val_type_rt_2_valkind(uint8 val_type_rt) +{ + switch (val_type_rt) { +#define WAMR_VAL_TYPE_2_WASM_VAL_KIND(name) \ + case VALUE_TYPE_##name: \ + return WASM_##name; + + WAMR_VAL_TYPE_2_WASM_VAL_KIND(I32) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(I64) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(F32) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(F64) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(FUNCREF) +#undef WAMR_VAL_TYPE_2_WASM_VAL_KIND + + default: + return WASM_ANYREF; + } +} + +static wasm_valtype_t * +wasm_valtype_new_internal(uint8 val_type_rt) +{ + return wasm_valtype_new(val_type_rt_2_valkind(val_type_rt)); +} + +wasm_valtype_t * +wasm_valtype_new(wasm_valkind_t kind) +{ + wasm_valtype_t *val_type; + + if (kind > WASM_F64 && WASM_FUNCREF != kind +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + && WASM_ANYREF != kind +#endif + ) { + return NULL; + } + + if (!(val_type = malloc_internal(sizeof(wasm_valtype_t)))) { + return NULL; + } + + val_type->kind = kind; + + return val_type; +} + +void +wasm_valtype_delete(wasm_valtype_t *val_type) +{ + if (val_type) { + wasm_runtime_free(val_type); + } +} + +wasm_valtype_t * +wasm_valtype_copy(const wasm_valtype_t *src) +{ + return src ? wasm_valtype_new(src->kind) : NULL; +} + +wasm_valkind_t +wasm_valtype_kind(const wasm_valtype_t *val_type) +{ + return val_type ? val_type->kind : WASM_ANYREF; +} + +static wasm_functype_t * +wasm_functype_new_internal(WASMFuncType *type_rt) +{ + wasm_functype_t *type = NULL; + wasm_valtype_t *param_type = NULL, *result_type = NULL; + uint32 i = 0; + + if (!type_rt) { + return NULL; + } + + if (!(type = malloc_internal(sizeof(wasm_functype_t)))) { + return NULL; + } + + type->extern_kind = WASM_EXTERN_FUNC; + + /* WASMFuncType->types[0 : type_rt->param_count) -> type->params */ + INIT_VEC(type->params, wasm_valtype_vec_new_uninitialized, + type_rt->param_count); + for (i = 0; i < type_rt->param_count; ++i) { + if (!(param_type = wasm_valtype_new_internal(*(type_rt->types + i)))) { + goto failed; + } + + if (!bh_vector_append((Vector *)type->params, ¶m_type)) { + LOG_DEBUG("bh_vector_append failed"); + goto failed; + } + } + + /* WASMFuncType->types[type_rt->param_count : type_rt->result_count) -> + * type->results */ + INIT_VEC(type->results, wasm_valtype_vec_new_uninitialized, + type_rt->result_count); + for (i = 0; i < type_rt->result_count; ++i) { + if (!(result_type = wasm_valtype_new_internal( + *(type_rt->types + type_rt->param_count + i)))) { + goto failed; + } + + if (!bh_vector_append((Vector *)type->results, &result_type)) { + LOG_DEBUG("bh_vector_append failed"); + goto failed; + } + } + + return type; + +failed: + wasm_valtype_delete(param_type); + wasm_valtype_delete(result_type); + wasm_functype_delete(type); + return NULL; +} + +wasm_functype_t * +wasm_functype_new(own wasm_valtype_vec_t *params, + own wasm_valtype_vec_t *results) +{ + wasm_functype_t *type = NULL; + + if (!(type = malloc_internal(sizeof(wasm_functype_t)))) { + goto failed; + } + + type->extern_kind = WASM_EXTERN_FUNC; + + /* take ownership */ + if (!(type->params = malloc_internal(sizeof(wasm_valtype_vec_t)))) { + goto failed; + } + if (params) { + bh_memcpy_s(type->params, sizeof(wasm_valtype_vec_t), params, + sizeof(wasm_valtype_vec_t)); + } + + if (!(type->results = malloc_internal(sizeof(wasm_valtype_vec_t)))) { + goto failed; + } + if (results) { + bh_memcpy_s(type->results, sizeof(wasm_valtype_vec_t), results, + sizeof(wasm_valtype_vec_t)); + } + + return type; + +failed: + wasm_functype_delete(type); + return NULL; +} + +wasm_functype_t * +wasm_functype_copy(const wasm_functype_t *src) +{ + wasm_functype_t *functype; + wasm_valtype_vec_t params = { 0 }, results = { 0 }; + + if (!src) { + return NULL; + } + + wasm_valtype_vec_copy(¶ms, src->params); + if (src->params->size && !params.data) { + goto failed; + } + + wasm_valtype_vec_copy(&results, src->results); + if (src->results->size && !results.data) { + goto failed; + } + + if (!(functype = wasm_functype_new(¶ms, &results))) { + goto failed; + } + + return functype; + +failed: + wasm_valtype_vec_delete(¶ms); + wasm_valtype_vec_delete(&results); + return NULL; +} + +void +wasm_functype_delete(wasm_functype_t *func_type) +{ + if (!func_type) { + return; + } + + DEINIT_VEC(func_type->params, wasm_valtype_vec_delete); + DEINIT_VEC(func_type->results, wasm_valtype_vec_delete); + + wasm_runtime_free(func_type); +} + +const wasm_valtype_vec_t * +wasm_functype_params(const wasm_functype_t *func_type) +{ + if (!func_type) { + return NULL; + } + + return func_type->params; +} + +const wasm_valtype_vec_t * +wasm_functype_results(const wasm_functype_t *func_type) +{ + if (!func_type) { + return NULL; + } + + return func_type->results; +} + +static bool +cmp_val_kind_with_val_type(wasm_valkind_t v_k, uint8 v_t) +{ + return (v_k == WASM_I32 && v_t == VALUE_TYPE_I32) + || (v_k == WASM_I64 && v_t == VALUE_TYPE_I64) + || (v_k == WASM_F32 && v_t == VALUE_TYPE_F32) + || (v_k == WASM_F64 && v_t == VALUE_TYPE_F64) + || (v_k == WASM_ANYREF && v_t == VALUE_TYPE_EXTERNREF) + || (v_k == WASM_FUNCREF && v_t == VALUE_TYPE_FUNCREF); +} + +/* + *to compare a function type of wasm-c-api with a function type of wasm_runtime + */ +static bool +wasm_functype_same_internal(const wasm_functype_t *type, + const WASMFuncType *type_intl) +{ + uint32 i = 0; + + if (!type || !type_intl || type->params->num_elems != type_intl->param_count + || type->results->num_elems != type_intl->result_count) + return false; + + for (i = 0; i < type->params->num_elems; i++) { + wasm_valtype_t *v_t = type->params->data[i]; + if (!cmp_val_kind_with_val_type(wasm_valtype_kind(v_t), + type_intl->types[i])) + return false; + } + + for (i = 0; i < type->results->num_elems; i++) { + wasm_valtype_t *v_t = type->results->data[i]; + if (!cmp_val_kind_with_val_type( + wasm_valtype_kind(v_t), + type_intl->types[i + type->params->num_elems])) + return false; + } + + return true; +} + +wasm_globaltype_t * +wasm_globaltype_new(own wasm_valtype_t *val_type, wasm_mutability_t mut) +{ + wasm_globaltype_t *global_type = NULL; + + if (!val_type) { + return NULL; + } + + if (!(global_type = malloc_internal(sizeof(wasm_globaltype_t)))) { + return NULL; + } + + global_type->extern_kind = WASM_EXTERN_GLOBAL; + global_type->val_type = val_type; + global_type->mutability = mut; + + return global_type; +} + +wasm_globaltype_t * +wasm_globaltype_new_internal(uint8 val_type_rt, bool is_mutable) +{ + wasm_globaltype_t *globaltype; + wasm_valtype_t *val_type; + + if (!(val_type = wasm_valtype_new(val_type_rt_2_valkind(val_type_rt)))) { + return NULL; + } + + if (!(globaltype = wasm_globaltype_new( + val_type, is_mutable ? WASM_VAR : WASM_CONST))) { + wasm_valtype_delete(val_type); + } + + return globaltype; +} + +void +wasm_globaltype_delete(wasm_globaltype_t *global_type) +{ + if (!global_type) { + return; + } + + if (global_type->val_type) { + wasm_valtype_delete(global_type->val_type); + global_type->val_type = NULL; + } + + wasm_runtime_free(global_type); +} + +wasm_globaltype_t * +wasm_globaltype_copy(const wasm_globaltype_t *src) +{ + wasm_globaltype_t *global_type; + wasm_valtype_t *val_type; + + if (!src) { + return NULL; + } + + if (!(val_type = wasm_valtype_copy(src->val_type))) { + return NULL; + } + + if (!(global_type = wasm_globaltype_new(val_type, src->mutability))) { + wasm_valtype_delete(val_type); + } + + return global_type; +} + +const wasm_valtype_t * +wasm_globaltype_content(const wasm_globaltype_t *global_type) +{ + if (!global_type) { + return NULL; + } + + return global_type->val_type; +} + +wasm_mutability_t +wasm_globaltype_mutability(const wasm_globaltype_t *global_type) +{ + if (!global_type) { + return false; + } + + return global_type->mutability; +} + +static wasm_tabletype_t * +wasm_tabletype_new_internal(uint8 val_type_rt, uint32 init_size, + uint32 max_size) +{ + wasm_tabletype_t *table_type; + wasm_limits_t limits = { init_size, max_size }; + wasm_valtype_t *val_type; + + if (!(val_type = wasm_valtype_new_internal(val_type_rt))) { + return NULL; + } + + if (!(table_type = wasm_tabletype_new(val_type, &limits))) { + wasm_valtype_delete(val_type); + } + + return table_type; +} + +wasm_tabletype_t * +wasm_tabletype_new(own wasm_valtype_t *val_type, const wasm_limits_t *limits) +{ + wasm_tabletype_t *table_type = NULL; + + if (!val_type || !limits) { + return NULL; + } + + if (wasm_valtype_kind(val_type) != WASM_FUNCREF +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + && wasm_valtype_kind(val_type) != WASM_ANYREF +#endif + ) { + return NULL; + } + + if (!(table_type = malloc_internal(sizeof(wasm_tabletype_t)))) { + return NULL; + } + + table_type->extern_kind = WASM_EXTERN_TABLE; + table_type->val_type = val_type; + table_type->limits.min = limits->min; + table_type->limits.max = limits->max; + + return table_type; +} + +wasm_tabletype_t * +wasm_tabletype_copy(const wasm_tabletype_t *src) +{ + wasm_tabletype_t *table_type; + wasm_valtype_t *val_type; + + if (!src) { + return NULL; + } + + if (!(val_type = wasm_valtype_copy(src->val_type))) { + return NULL; + } + + if (!(table_type = wasm_tabletype_new(val_type, &src->limits))) { + wasm_valtype_delete(val_type); + } + + return table_type; +} + +void +wasm_tabletype_delete(wasm_tabletype_t *table_type) +{ + if (!table_type) { + return; + } + + if (table_type->val_type) { + wasm_valtype_delete(table_type->val_type); + table_type->val_type = NULL; + } + + wasm_runtime_free(table_type); +} + +const wasm_valtype_t * +wasm_tabletype_element(const wasm_tabletype_t *table_type) +{ + if (!table_type) { + return NULL; + } + + return table_type->val_type; +} + +const wasm_limits_t * +wasm_tabletype_limits(const wasm_tabletype_t *table_type) +{ + if (!table_type) { + return NULL; + } + + return &(table_type->limits); +} + +static wasm_memorytype_t * +wasm_memorytype_new_internal(uint32 min_pages, uint32 max_pages) +{ + wasm_limits_t limits = { min_pages, max_pages }; + return wasm_memorytype_new(&limits); +} + +wasm_memorytype_t * +wasm_memorytype_new(const wasm_limits_t *limits) +{ + wasm_memorytype_t *memory_type = NULL; + + if (!limits) { + return NULL; + } + + if (!(memory_type = malloc_internal(sizeof(wasm_memorytype_t)))) { + return NULL; + } + + memory_type->extern_kind = WASM_EXTERN_MEMORY; + memory_type->limits.min = limits->min; + memory_type->limits.max = limits->max; + + return memory_type; +} + +wasm_memorytype_t * +wasm_memorytype_copy(const wasm_memorytype_t *src) +{ + if (!src) { + return NULL; + } + + return wasm_memorytype_new(&src->limits); +} + +void +wasm_memorytype_delete(wasm_memorytype_t *memory_type) +{ + if (memory_type) { + wasm_runtime_free(memory_type); + } +} + +const wasm_limits_t * +wasm_memorytype_limits(const wasm_memorytype_t *memory_type) +{ + if (!memory_type) { + return NULL; + } + + return &(memory_type->limits); +} + +wasm_externkind_t +wasm_externtype_kind(const wasm_externtype_t *extern_type) +{ + if (!extern_type) { + return WASM_EXTERN_FUNC; + } + + return extern_type->extern_kind; +} + +#define BASIC_FOUR_TYPE_LIST(V) \ + V(functype) \ + V(globaltype) \ + V(memorytype) \ + V(tabletype) + +#define WASM_EXTERNTYPE_AS_OTHERTYPE(name) \ + wasm_##name##_t *wasm_externtype_as_##name(wasm_externtype_t *extern_type) \ + { \ + return (wasm_##name##_t *)extern_type; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_EXTERNTYPE_AS_OTHERTYPE) +#undef WASM_EXTERNTYPE_AS_OTHERTYPE + +#define WASM_OTHERTYPE_AS_EXTERNTYPE(name) \ + wasm_externtype_t *wasm_##name##_as_externtype(wasm_##name##_t *other) \ + { \ + return (wasm_externtype_t *)other; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_OTHERTYPE_AS_EXTERNTYPE) +#undef WASM_OTHERTYPE_AS_EXTERNTYPE + +#define WASM_EXTERNTYPE_AS_OTHERTYPE_CONST(name) \ + const wasm_##name##_t *wasm_externtype_as_##name##_const( \ + const wasm_externtype_t *extern_type) \ + { \ + return (const wasm_##name##_t *)extern_type; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_EXTERNTYPE_AS_OTHERTYPE_CONST) +#undef WASM_EXTERNTYPE_AS_OTHERTYPE_CONST + +#define WASM_OTHERTYPE_AS_EXTERNTYPE_CONST(name) \ + const wasm_externtype_t *wasm_##name##_as_externtype_const( \ + const wasm_##name##_t *other) \ + { \ + return (const wasm_externtype_t *)other; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_OTHERTYPE_AS_EXTERNTYPE_CONST) +#undef WASM_OTHERTYPE_AS_EXTERNTYPE_CONST + +wasm_externtype_t * +wasm_externtype_copy(const wasm_externtype_t *src) +{ + wasm_externtype_t *extern_type = NULL; + + if (!src) { + return NULL; + } + + switch (src->extern_kind) { +#define COPY_EXTERNTYPE(NAME, name) \ + case WASM_EXTERN_##NAME: \ + { \ + extern_type = wasm_##name##_as_externtype( \ + wasm_##name##_copy(wasm_externtype_as_##name##_const(src))); \ + break; \ + } + COPY_EXTERNTYPE(FUNC, functype) + COPY_EXTERNTYPE(GLOBAL, globaltype) + COPY_EXTERNTYPE(MEMORY, memorytype) + COPY_EXTERNTYPE(TABLE, tabletype) +#undef COPY_EXTERNTYPE + default: + LOG_WARNING("%s meets unsupported kind %u", __FUNCTION__, + src->extern_kind); + break; + } + return extern_type; +} + +void +wasm_externtype_delete(wasm_externtype_t *extern_type) +{ + if (!extern_type) { + return; + } + + switch (wasm_externtype_kind(extern_type)) { + case WASM_EXTERN_FUNC: + wasm_functype_delete(wasm_externtype_as_functype(extern_type)); + break; + case WASM_EXTERN_GLOBAL: + wasm_globaltype_delete(wasm_externtype_as_globaltype(extern_type)); + break; + case WASM_EXTERN_MEMORY: + wasm_memorytype_delete(wasm_externtype_as_memorytype(extern_type)); + break; + case WASM_EXTERN_TABLE: + wasm_tabletype_delete(wasm_externtype_as_tabletype(extern_type)); + break; + default: + LOG_WARNING("%s meets unsupported type %u", __FUNCTION__, + wasm_externtype_kind(extern_type)); + break; + } +} + +own wasm_importtype_t * +wasm_importtype_new(own wasm_byte_vec_t *module_name, + own wasm_byte_vec_t *field_name, + own wasm_externtype_t *extern_type) +{ + wasm_importtype_t *import_type = NULL; + + if (!module_name || !field_name || !extern_type) { + return NULL; + } + + if (!(import_type = malloc_internal(sizeof(wasm_importtype_t)))) { + return NULL; + } + + /* take ownership */ + if (!(import_type->module_name = + malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + bh_memcpy_s(import_type->module_name, sizeof(wasm_byte_vec_t), module_name, + sizeof(wasm_byte_vec_t)); + + if (!(import_type->name = malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + bh_memcpy_s(import_type->name, sizeof(wasm_byte_vec_t), field_name, + sizeof(wasm_byte_vec_t)); + + import_type->extern_type = extern_type; + + return import_type; +failed: + wasm_importtype_delete(import_type); + return NULL; +} + +void +wasm_importtype_delete(own wasm_importtype_t *import_type) +{ + if (!import_type) { + return; + } + + DEINIT_VEC(import_type->module_name, wasm_byte_vec_delete); + DEINIT_VEC(import_type->name, wasm_byte_vec_delete); + wasm_externtype_delete(import_type->extern_type); + import_type->extern_type = NULL; + wasm_runtime_free(import_type); +} + +own wasm_importtype_t * +wasm_importtype_copy(const wasm_importtype_t *src) +{ + wasm_byte_vec_t module_name = { 0 }, name = { 0 }; + wasm_externtype_t *extern_type = NULL; + wasm_importtype_t *import_type = NULL; + + if (!src) { + return NULL; + } + + wasm_byte_vec_copy(&module_name, src->module_name); + if (src->module_name->size && !module_name.data) { + goto failed; + } + + wasm_byte_vec_copy(&name, src->name); + if (src->name->size && !name.data) { + goto failed; + } + + if (!(extern_type = wasm_externtype_copy(src->extern_type))) { + goto failed; + } + + if (!(import_type = + wasm_importtype_new(&module_name, &name, extern_type))) { + goto failed; + } + + return import_type; + +failed: + wasm_byte_vec_delete(&module_name); + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); + wasm_importtype_delete(import_type); + return NULL; +} + +const wasm_byte_vec_t * +wasm_importtype_module(const wasm_importtype_t *import_type) +{ + if (!import_type) { + return NULL; + } + + return import_type->module_name; +} + +const wasm_byte_vec_t * +wasm_importtype_name(const wasm_importtype_t *import_type) +{ + if (!import_type) { + return NULL; + } + + return import_type->name; +} + +const wasm_externtype_t * +wasm_importtype_type(const wasm_importtype_t *import_type) +{ + if (!import_type) { + return NULL; + } + + return import_type->extern_type; +} + +bool +wasm_importtype_is_linked(const wasm_importtype_t *import_type) +{ + if (!import_type) + return false; + + const wasm_name_t *module_name = wasm_importtype_module(import_type); + const wasm_name_t *field_name = wasm_importtype_name(import_type); + + switch (wasm_externtype_kind(wasm_importtype_type(import_type))) { + case WASM_EXTERN_FUNC: + return wasm_runtime_is_import_func_linked(module_name->data, + field_name->data); + case WASM_EXTERN_GLOBAL: + return wasm_runtime_is_import_global_linked(module_name->data, + field_name->data); + case WASM_EXTERN_MEMORY: + case WASM_EXTERN_TABLE: + default: + break; + } + return false; +} + +own wasm_exporttype_t * +wasm_exporttype_new(own wasm_byte_vec_t *name, + own wasm_externtype_t *extern_type) +{ + wasm_exporttype_t *export_type = NULL; + + if (!name || !extern_type) { + return NULL; + } + + if (!(export_type = malloc_internal(sizeof(wasm_exporttype_t)))) { + return NULL; + } + + if (!(export_type->name = malloc_internal(sizeof(wasm_byte_vec_t)))) { + wasm_exporttype_delete(export_type); + return NULL; + } + bh_memcpy_s(export_type->name, sizeof(wasm_byte_vec_t), name, + sizeof(wasm_byte_vec_t)); + + export_type->extern_type = extern_type; + + return export_type; +} + +wasm_exporttype_t * +wasm_exporttype_copy(const wasm_exporttype_t *src) +{ + wasm_exporttype_t *export_type; + wasm_byte_vec_t name = { 0 }; + wasm_externtype_t *extern_type = NULL; + + if (!src) { + return NULL; + } + + wasm_byte_vec_copy(&name, src->name); + if (src->name->size && !name.data) { + goto failed; + } + + if (!(extern_type = wasm_externtype_copy(src->extern_type))) { + goto failed; + } + + if (!(export_type = wasm_exporttype_new(&name, extern_type))) { + goto failed; + } + + return export_type; +failed: + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); + return NULL; +} + +void +wasm_exporttype_delete(wasm_exporttype_t *export_type) +{ + if (!export_type) { + return; + } + + DEINIT_VEC(export_type->name, wasm_byte_vec_delete); + + wasm_externtype_delete(export_type->extern_type); + + wasm_runtime_free(export_type); +} + +const wasm_byte_vec_t * +wasm_exporttype_name(const wasm_exporttype_t *export_type) +{ + if (!export_type) { + return NULL; + } + return export_type->name; +} + +const wasm_externtype_t * +wasm_exporttype_type(const wasm_exporttype_t *export_type) +{ + if (!export_type) { + return NULL; + } + return export_type->extern_type; +} + +/* Runtime Objects */ +void +wasm_val_delete(wasm_val_t *v) +{ + if (v) + wasm_runtime_free(v); +} + +void +wasm_val_copy(wasm_val_t *out, const wasm_val_t *src) +{ + if (!out || !src) { + return; + } + + bh_memcpy_s(out, sizeof(wasm_val_t), src, sizeof(wasm_val_t)); +} + +bool +rt_val_to_wasm_val(const uint8 *data, uint8 val_type_rt, wasm_val_t *out) +{ + bool ret = true; + switch (val_type_rt) { + case VALUE_TYPE_I32: + out->kind = WASM_I32; + out->of.i32 = *((int32 *)data); + break; + case VALUE_TYPE_F32: + out->kind = WASM_F32; + out->of.f32 = *((float32 *)data); + break; + case VALUE_TYPE_I64: + out->kind = WASM_I64; + out->of.i64 = *((int64 *)data); + break; + case VALUE_TYPE_F64: + out->kind = WASM_F64; + out->of.f64 = *((float64 *)data); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + out->kind = WASM_ANYREF; + if (NULL_REF == *(uint32 *)data) { + out->of.ref = NULL; + } + else { + ret = wasm_externref_ref2obj(*(uint32 *)data, + (void **)&out->of.ref); + } + break; +#endif + default: + LOG_WARNING("unexpected value type %d", val_type_rt); + ret = false; + } + return ret; +} + +bool +wasm_val_to_rt_val(WASMModuleInstanceCommon *inst_comm_rt, uint8 val_type_rt, + const wasm_val_t *v, uint8 *data) +{ + bool ret = true; + switch (val_type_rt) { + case VALUE_TYPE_I32: + bh_assert(WASM_I32 == v->kind); + *((int32 *)data) = v->of.i32; + break; + case VALUE_TYPE_F32: + bh_assert(WASM_F32 == v->kind); + *((float32 *)data) = v->of.f32; + break; + case VALUE_TYPE_I64: + bh_assert(WASM_I64 == v->kind); + *((int64 *)data) = v->of.i64; + break; + case VALUE_TYPE_F64: + bh_assert(WASM_F64 == v->kind); + *((float64 *)data) = v->of.f64; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + bh_assert(WASM_ANYREF == v->kind); + ret = + wasm_externref_obj2ref(inst_comm_rt, v->of.ref, (uint32 *)data); + break; +#endif + default: + LOG_WARNING("unexpected value type %d", val_type_rt); + ret = false; + break; + } + + (void)inst_comm_rt; + return ret; +} + +wasm_ref_t * +wasm_ref_new_internal(wasm_store_t *store, enum wasm_reference_kind kind, + uint32 ref_idx_rt, WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_ref_t *ref; + + if (!store) { + return NULL; + } + + if (!(ref = malloc_internal(sizeof(wasm_ref_t)))) { + return NULL; + } + + ref->store = store; + ref->kind = kind; + ref->ref_idx_rt = ref_idx_rt; + ref->inst_comm_rt = inst_comm_rt; + + /* workaround */ + if (WASM_REF_foreign == kind) { + wasm_foreign_t *foreign; + + if (!(bh_vector_get(ref->store->foreigns, ref->ref_idx_rt, &foreign)) + || !foreign) { + wasm_runtime_free(ref); + return NULL; + } + + foreign->ref_cnt++; + } + /* others doesn't include ref counters */ + + return ref; +} + +own wasm_ref_t * +wasm_ref_copy(const wasm_ref_t *src) +{ + if (!src) + return NULL; + + /* host_info are different in wasm_ref_t(s) */ + return wasm_ref_new_internal(src->store, src->kind, src->ref_idx_rt, + src->inst_comm_rt); +} + +#define DELETE_HOST_INFO(obj) \ + if (obj->host_info.info) { \ + if (obj->host_info.finalizer) { \ + obj->host_info.finalizer(obj->host_info.info); \ + } \ + } + +void +wasm_ref_delete(own wasm_ref_t *ref) +{ + if (!ref || !ref->store) + return; + + DELETE_HOST_INFO(ref); + + if (WASM_REF_foreign == ref->kind) { + wasm_foreign_t *foreign = NULL; + + if (bh_vector_get(ref->store->foreigns, ref->ref_idx_rt, &foreign) + && foreign) { + wasm_foreign_delete(foreign); + } + } + + wasm_runtime_free(ref); +} + +#define WASM_DEFINE_REF_BASE(name) \ + bool wasm_##name##_same(const wasm_##name##_t *o1, \ + const wasm_##name##_t *o2) \ + { \ + return (!o1 && !o2) ? true \ + : (!o1 || !o2) ? false \ + : (o1->kind != o2->kind) \ + ? false \ + : o1->name##_idx_rt == o2->name##_idx_rt; \ + } \ + \ + void *wasm_##name##_get_host_info(const wasm_##name##_t *obj) \ + { \ + return obj ? obj->host_info.info : NULL; \ + } \ + \ + void wasm_##name##_set_host_info(wasm_##name##_t *obj, void *host_info) \ + { \ + if (obj) { \ + obj->host_info.info = host_info; \ + obj->host_info.finalizer = NULL; \ + } \ + } \ + \ + void wasm_##name##_set_host_info_with_finalizer( \ + wasm_##name##_t *obj, void *host_info, void (*finalizer)(void *)) \ + { \ + if (obj) { \ + obj->host_info.info = host_info; \ + obj->host_info.finalizer = finalizer; \ + } \ + } + +#define WASM_DEFINE_REF(name) \ + WASM_DEFINE_REF_BASE(name) \ + \ + wasm_ref_t *wasm_##name##_as_ref(wasm_##name##_t *name) \ + { \ + if (!name) { \ + return NULL; \ + } \ + \ + return wasm_ref_new_internal(name->store, WASM_REF_##name, \ + name->name##_idx_rt, name->inst_comm_rt); \ + } \ + \ + const wasm_ref_t *wasm_##name##_as_ref_const(const wasm_##name##_t *name) \ + { \ + if (!name) { \ + return NULL; \ + } \ + \ + return wasm_ref_new_internal(name->store, WASM_REF_##name, \ + name->name##_idx_rt, name->inst_comm_rt); \ + } \ + \ + wasm_##name##_t *wasm_ref_as_##name(wasm_ref_t *ref) \ + { \ + if (!ref || WASM_REF_##name != ref->kind) { \ + return NULL; \ + } \ + \ + return wasm_##name##_new_internal(ref->store, ref->ref_idx_rt, \ + ref->inst_comm_rt); \ + } \ + \ + const wasm_##name##_t *wasm_ref_as_##name##_const(const wasm_ref_t *ref) \ + { \ + if (!ref || WASM_REF_##name != ref->kind) { \ + return NULL; \ + } \ + \ + return wasm_##name##_new_internal(ref->store, ref->ref_idx_rt, \ + ref->inst_comm_rt); \ + } + +WASM_DEFINE_REF_BASE(ref) +WASM_DEFINE_REF(foreign) +WASM_DEFINE_REF(func) +WASM_DEFINE_REF(global) +WASM_DEFINE_REF(memory) +WASM_DEFINE_REF(table) + +static wasm_frame_t * +wasm_frame_new(wasm_instance_t *instance, size_t module_offset, + uint32 func_index, size_t func_offset) +{ + wasm_frame_t *frame; + + if (!(frame = malloc_internal(sizeof(wasm_frame_t)))) { + return NULL; + } + + frame->instance = instance; + frame->module_offset = (uint32)module_offset; + frame->func_index = func_index; + frame->func_offset = (uint32)func_offset; + return frame; +} + +own wasm_frame_t * +wasm_frame_copy(const wasm_frame_t *src) +{ + if (!src) { + return NULL; + } + + return wasm_frame_new(src->instance, src->module_offset, src->func_index, + src->func_offset); +} + +void +wasm_frame_delete(own wasm_frame_t *frame) +{ + if (frame) { + wasm_runtime_free(frame); + } +} + +struct wasm_instance_t * +wasm_frame_instance(const wasm_frame_t *frame) +{ + return frame ? frame->instance : NULL; +} + +size_t +wasm_frame_module_offset(const wasm_frame_t *frame) +{ + return frame ? frame->module_offset : 0; +} + +uint32_t +wasm_frame_func_index(const wasm_frame_t *frame) +{ + return frame ? frame->func_index : 0; +} + +size_t +wasm_frame_func_offset(const wasm_frame_t *frame) +{ + return frame ? frame->func_offset : 0; +} + +void +wasm_frame_vec_clone_internal(Vector *src, Vector *out) +{ + if (src->num_elems == 0) { + bh_vector_destroy(out); + return; + } + + if (!bh_vector_destroy(out) + || !bh_vector_init(out, src->num_elems, sizeof(WASMCApiFrame), false)) { + return; + } + + bh_memcpy_s(out->data, (uint32)(src->num_elems * sizeof(WASMCApiFrame)), + src->data, (uint32)(src->num_elems * sizeof(WASMCApiFrame))); + out->num_elems = src->num_elems; +} + +static wasm_trap_t * +wasm_trap_new_internal(wasm_store_t *store, + WASMModuleInstanceCommon *inst_comm_rt, + const char *error_info, Vector *cluster_frames) +{ + wasm_trap_t *trap; +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + wasm_instance_vec_t *instances; + wasm_instance_t *frame_instance = NULL; + uint32 i; +#endif + + if (!singleton_engine) + return NULL; + + if (!(trap = malloc_internal(sizeof(wasm_trap_t)))) { + return NULL; + } + + /* fill in message */ + if (error_info && strlen(error_info) > 0) { + if (!(trap->message = malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + + wasm_name_new_from_string_nt(trap->message, error_info); + if (!trap->message->data) { + goto failed; + } + } + + /* fill in frames */ +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + trap->frames = cluster_frames + ? cluster_frames + : ((WASMModuleInstance *)inst_comm_rt)->frames; + + if (trap->frames) { + /* fill in instances */ + instances = store->instances; + bh_assert(instances != NULL); + + for (i = 0; i < instances->num_elems; i++) { + if (instances->data[i]->inst_comm_rt == inst_comm_rt) { + frame_instance = instances->data[i]; + break; + } + } + + for (i = 0; i < trap->frames->num_elems; i++) { + (((wasm_frame_t *)trap->frames->data) + i)->instance = + frame_instance; + } + } +#else + (void)store; + (void)inst_comm_rt; +#endif /* WASM_ENABLE_DUMP_CALL_STACK != 0 */ + + return trap; +failed: + wasm_trap_delete(trap); + return NULL; +} + +wasm_trap_t * +wasm_trap_new(wasm_store_t *store, const wasm_message_t *message) +{ + wasm_trap_t *trap; + + if (!store) { + return NULL; + } + + if (!(trap = malloc_internal(sizeof(wasm_trap_t)))) { + return NULL; + } + + if (message) { + INIT_VEC(trap->message, wasm_byte_vec_new, message->size, + message->data); + } + + return trap; +failed: + wasm_trap_delete(trap); + return NULL; +} + +void +wasm_trap_delete(wasm_trap_t *trap) +{ + if (!trap) { + return; + } + + DEINIT_VEC(trap->message, wasm_byte_vec_delete); + /* reuse frames of WASMModuleInstance, do not free it here */ + + wasm_runtime_free(trap); +} + +void +wasm_trap_message(const wasm_trap_t *trap, own wasm_message_t *out) +{ + if (!trap || !out) { + return; + } + + wasm_byte_vec_copy(out, trap->message); +} + +own wasm_frame_t * +wasm_trap_origin(const wasm_trap_t *trap) +{ + wasm_frame_t *latest_frame; + + if (!trap || !trap->frames || !trap->frames->num_elems) { + return NULL; + } + + /* first frame is the latest frame */ + latest_frame = (wasm_frame_t *)trap->frames->data; + return wasm_frame_copy(latest_frame); +} + +void +wasm_trap_trace(const wasm_trap_t *trap, own wasm_frame_vec_t *out) +{ + uint32 i; + + if (!trap || !out) { + return; + } + + if (!trap->frames || !trap->frames->num_elems) { + wasm_frame_vec_new_empty(out); + return; + } + + wasm_frame_vec_new_uninitialized(out, trap->frames->num_elems); + if (out->size == 0 || !out->data) { + return; + } + + for (i = 0; i < trap->frames->num_elems; i++) { + wasm_frame_t *frame = ((wasm_frame_t *)trap->frames->data) + i; + if (!(out->data[i] = + wasm_frame_new(frame->instance, frame->module_offset, + frame->func_index, frame->func_offset))) { + goto failed; + } + out->num_elems++; + } + + return; +failed: + for (i = 0; i < out->num_elems; i++) { + if (out->data[i]) { + wasm_runtime_free(out->data[i]); + } + } + + wasm_runtime_free(out->data); +} + +wasm_foreign_t * +wasm_foreign_new_internal(wasm_store_t *store, uint32 foreign_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_foreign_t *foreign = NULL; + + if (!store || !store->foreigns) + return NULL; + + if (!(bh_vector_get(store->foreigns, foreign_idx_rt, &foreign)) + || !foreign) { + return NULL; + } + + foreign->ref_cnt++; + (void)inst_comm_rt; + return foreign; +} + +own wasm_foreign_t * +wasm_foreign_new(wasm_store_t *store) +{ + wasm_foreign_t *foreign; + + if (!store) + return NULL; + + if (!(foreign = malloc_internal(sizeof(wasm_foreign_t)))) + return NULL; + + foreign->store = store; + foreign->kind = WASM_REF_foreign; + foreign->foreign_idx_rt = (uint32)bh_vector_size(store->foreigns); + if (!(bh_vector_append(store->foreigns, &foreign))) { + wasm_runtime_free(foreign); + return NULL; + } + + return foreign; +} + +void +wasm_foreign_delete(wasm_foreign_t *foreign) +{ + if (!foreign) + return; + + if (foreign->ref_cnt < 1) { + return; + } + + foreign->ref_cnt--; + if (!foreign->ref_cnt) { + wasm_runtime_free(foreign); + } +} + +static inline wasm_module_t * +module_ext_to_module(wasm_module_ex_t *module_ex) +{ + return (wasm_module_t *)module_ex; +} + +static inline wasm_module_ex_t * +module_to_module_ext(wasm_module_t *module) +{ + return (wasm_module_ex_t *)module; +} + +#if WASM_ENABLE_INTERP != 0 +#define MODULE_INTERP(module_comm) ((WASMModule *)(*module_comm)) +#endif + +#if WASM_ENABLE_AOT != 0 +#define MODULE_AOT(module_comm) ((AOTModule *)(*module_comm)) +#endif + +#if WASM_ENABLE_WASM_CACHE != 0 +static wasm_module_ex_t * +check_loaded_module(Vector *modules, char *binary_hash) +{ + unsigned i; + wasm_module_ex_t *module = NULL; + + for (i = 0; i < modules->num_elems; i++) { + bh_vector_get(modules, i, &module); + if (!module) { + LOG_ERROR("Unexpected failure at %d\n", __LINE__); + return NULL; + } + + if (!module->ref_count) + /* deleted */ + continue; + + if (memcmp(module->hash, binary_hash, SHA256_DIGEST_LENGTH) == 0) + return module; + } + return NULL; +} + +static wasm_module_ex_t * +try_reuse_loaded_module(wasm_store_t *store, char *binary_hash) +{ + wasm_module_ex_t *cached = NULL; + wasm_module_ex_t *ret = NULL; + + cached = check_loaded_module(&singleton_engine->modules, binary_hash); + if (!cached) + goto quit; + + os_mutex_lock(&cached->lock); + if (!cached->ref_count) + goto unlock; + + if (!bh_vector_append((Vector *)store->modules, &cached)) + goto unlock; + + cached->ref_count += 1; + ret = cached; + +unlock: + os_mutex_unlock(&cached->lock); +quit: + return ret; +} +#endif /* WASM_ENABLE_WASM_CACHE != 0 */ + +wasm_module_t * +wasm_module_new_ex(wasm_store_t *store, const wasm_byte_vec_t *binary, + const LoadArgs *args) +{ + char error_buf[128] = { 0 }; + wasm_module_ex_t *module_ex = NULL; +#if WASM_ENABLE_WASM_CACHE != 0 + char binary_hash[SHA256_DIGEST_LENGTH] = { 0 }; +#endif + + bh_assert(singleton_engine); + + if (!store || !binary || binary->size == 0 || binary->size > UINT32_MAX) + goto quit; + + /* whether the combination of compilation flags are compatable with the + * package type */ + { + PackageType pkg_type; + pkg_type = + get_package_type((uint8 *)binary->data, (uint32)binary->size); + bool result = false; +#if WASM_ENABLE_INTERP != 0 + result = (pkg_type == Wasm_Module_Bytecode); +#endif + +#if WASM_ENABLE_AOT != 0 + result = result || (pkg_type == Wasm_Module_AoT); +#endif + if (!result) { + LOG_VERBOSE("current building isn't compatiable with the module," + "may need recompile"); + goto quit; + } + } + +#if WASM_ENABLE_WASM_CACHE != 0 + /* if cached */ + SHA256((void *)binary->data, binary->num_elems, (uint8_t *)binary_hash); + module_ex = try_reuse_loaded_module(store, binary_hash); + if (module_ex) + return module_ext_to_module(module_ex); +#endif + + WASM_C_DUMP_PROC_MEM(); + + module_ex = malloc_internal(sizeof(wasm_module_ex_t)); + if (!module_ex) + goto quit; + + module_ex->binary = malloc_internal(sizeof(wasm_byte_vec_t)); + if (!module_ex->binary) + goto free_module; + + wasm_byte_vec_copy(module_ex->binary, binary); + if (!module_ex->binary->data) + goto free_binary; + + module_ex->module_comm_rt = wasm_runtime_load_ex( + (uint8 *)module_ex->binary->data, (uint32)module_ex->binary->size, args, + error_buf, (uint32)sizeof(error_buf)); + if (!(module_ex->module_comm_rt)) { + LOG_ERROR("%s", error_buf); + goto free_vec; + } + + /* append it to a watching list in store */ + if (!bh_vector_append((Vector *)store->modules, &module_ex)) + goto unload; + + if (os_mutex_init(&module_ex->lock) != BHT_OK) + goto remove_last; + + if (!bh_vector_append(&singleton_engine->modules, &module_ex)) + goto destroy_lock; + +#if WASM_ENABLE_WASM_CACHE != 0 + bh_memcpy_s(module_ex->hash, sizeof(module_ex->hash), binary_hash, + sizeof(binary_hash)); +#endif + + module_ex->ref_count = 1; + + WASM_C_DUMP_PROC_MEM(); + + return module_ext_to_module(module_ex); + +destroy_lock: + os_mutex_destroy(&module_ex->lock); +remove_last: + bh_vector_remove((Vector *)store->modules, + (uint32)(store->modules->num_elems - 1), NULL); +unload: + wasm_runtime_unload(module_ex->module_comm_rt); +free_vec: + wasm_byte_vec_delete(module_ex->binary); +free_binary: + wasm_runtime_free(module_ex->binary); +free_module: + wasm_runtime_free(module_ex); +quit: + LOG_ERROR("%s failed", __FUNCTION__); + return NULL; +} + +wasm_module_t * +wasm_module_new(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + LoadArgs args = { 0 }; + args.name = ""; + return wasm_module_new_ex(store, binary, &args); +} + +bool +wasm_module_validate(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + wasm_byte_vec_t local_binary = { 0 }; + struct WASMModuleCommon *module_rt; + char error_buf[128] = { 0 }; + bool ret; + + bh_assert(singleton_engine); + + if (!store || !binary || binary->size > UINT32_MAX) { + LOG_ERROR("%s failed", __FUNCTION__); + return false; + } + + /* make a copy of binary */ + wasm_byte_vec_copy(&local_binary, binary); + + if (binary->size && !local_binary.data) + return false; + + module_rt = wasm_runtime_load((uint8 *)local_binary.data, + (uint32)local_binary.size, error_buf, 128); + wasm_byte_vec_delete(&local_binary); + if (module_rt) { + wasm_runtime_unload(module_rt); + ret = true; + } + else { + ret = false; + LOG_VERBOSE("%s", error_buf); + } + + return ret; +} + +static void +wasm_module_delete_internal(wasm_module_t *module) +{ + wasm_module_ex_t *module_ex; + + if (!module) { + return; + } + + module_ex = module_to_module_ext(module); + + os_mutex_lock(&module_ex->lock); + + /* N -> N-1 -> 0 -> UINT32_MAX */ + module_ex->ref_count--; + if (module_ex->ref_count > 0) { + os_mutex_unlock(&module_ex->lock); + return; + } + + DEINIT_VEC(module_ex->binary, wasm_byte_vec_delete); + + if (module_ex->module_comm_rt) { + wasm_runtime_unload(module_ex->module_comm_rt); + module_ex->module_comm_rt = NULL; + } + +#if WASM_ENABLE_WASM_CACHE != 0 + memset(module_ex->hash, 0, sizeof(module_ex->hash)); +#endif + + os_mutex_unlock(&module_ex->lock); +} + +void +wasm_module_delete(wasm_module_t *module) +{ + /* the module will be released when releasing the store */ + (void)module; +} + +void +wasm_module_imports(const wasm_module_t *module, own wasm_importtype_vec_t *out) +{ + uint32 i, import_func_count = 0, import_memory_count = 0, + import_global_count = 0, import_table_count = 0, import_count = 0; + wasm_byte_vec_t module_name = { 0 }, name = { 0 }; + wasm_externtype_t *extern_type = NULL; + wasm_importtype_t *import_type = NULL; + + if (!module || !out) { + return; + } + + if (((const wasm_module_ex_t *)(module))->ref_count == 0) + return; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + import_func_count = MODULE_INTERP(module)->import_function_count; + import_global_count = MODULE_INTERP(module)->import_global_count; + import_memory_count = MODULE_INTERP(module)->import_memory_count; + import_table_count = MODULE_INTERP(module)->import_table_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + import_func_count = MODULE_AOT(module)->import_func_count; + import_global_count = MODULE_AOT(module)->import_global_count; + import_memory_count = MODULE_AOT(module)->import_memory_count; + import_table_count = MODULE_AOT(module)->import_table_count; + } +#endif + + import_count = import_func_count + import_global_count + import_table_count + + import_memory_count; + + wasm_importtype_vec_new_uninitialized(out, import_count); + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!out->data) { + return; + } + + for (i = 0; i != import_count; ++i) { + char *module_name_rt = NULL, *field_name_rt = NULL; + + memset(&module_name, 0, sizeof(wasm_val_vec_t)); + memset(&name, 0, sizeof(wasm_val_vec_t)); + extern_type = NULL; + import_type = NULL; + + if (i < import_func_count) { + wasm_functype_t *type = NULL; + WASMFuncType *type_rt = NULL; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = + MODULE_INTERP(module)->import_functions + i; + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + type_rt = import->u.function.func_type; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportFunc *import = MODULE_AOT(module)->import_funcs + i; + module_name_rt = import->module_name; + field_name_rt = import->func_name; + type_rt = import->func_type; + } +#endif + + if (!module_name_rt || !field_name_rt || !type_rt) { + continue; + } + + if (!(type = wasm_functype_new_internal(type_rt))) { + goto failed; + } + + extern_type = wasm_functype_as_externtype(type); + } + else if (i < import_func_count + import_global_count) { + wasm_globaltype_t *type = NULL; + uint8 val_type_rt = 0; + bool mutability_rt = 0; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = MODULE_INTERP(module)->import_globals + + (i - import_func_count); + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + val_type_rt = import->u.global.type; + mutability_rt = import->u.global.is_mutable; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportGlobal *import = MODULE_AOT(module)->import_globals + + (i - import_func_count); + module_name_rt = import->module_name; + field_name_rt = import->global_name; + val_type_rt = import->type; + mutability_rt = import->is_mutable; + } +#endif + + if (!module_name_rt || !field_name_rt) { + continue; + } + + if (!(type = wasm_globaltype_new_internal(val_type_rt, + mutability_rt))) { + goto failed; + } + + extern_type = wasm_globaltype_as_externtype(type); + } + else if (i < import_func_count + import_global_count + + import_memory_count) { + wasm_memorytype_t *type = NULL; + uint32 min_page = 0, max_page = 0; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = + MODULE_INTERP(module)->import_memories + + (i - import_func_count - import_global_count); + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + min_page = import->u.memory.init_page_count; + max_page = import->u.memory.max_page_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportMemory *import = + MODULE_AOT(module)->import_memories + + (i - import_func_count - import_global_count); + module_name_rt = import->module_name; + field_name_rt = import->memory_name; + min_page = import->mem_init_page_count; + max_page = import->mem_max_page_count; + } +#endif + + if (!module_name_rt || !field_name_rt) { + continue; + } + + if (!(type = wasm_memorytype_new_internal(min_page, max_page))) { + goto failed; + } + + extern_type = wasm_memorytype_as_externtype(type); + } + else { + wasm_tabletype_t *type = NULL; + uint8 elem_type_rt = 0; + uint32 min_size = 0, max_size = 0; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = + MODULE_INTERP(module)->import_tables + + (i - import_func_count - import_global_count + - import_memory_count); + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + elem_type_rt = import->u.table.elem_type; + min_size = import->u.table.init_size; + max_size = import->u.table.max_size; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportTable *import = + MODULE_AOT(module)->import_tables + + (i - import_func_count - import_global_count + - import_memory_count); + module_name_rt = import->module_name; + field_name_rt = import->table_name; + elem_type_rt = import->elem_type; + min_size = import->table_init_size; + max_size = import->table_max_size; + } +#endif + + if (!module_name_rt || !field_name_rt) { + continue; + } + + if (!(type = wasm_tabletype_new_internal(elem_type_rt, min_size, + max_size))) { + goto failed; + } + + extern_type = wasm_tabletype_as_externtype(type); + } + + bh_assert(extern_type); + + wasm_name_new_from_string_nt(&module_name, module_name_rt); + if (strlen(module_name_rt) && !module_name.data) { + goto failed; + } + + wasm_name_new_from_string_nt(&name, field_name_rt); + if (strlen(field_name_rt) && !name.data) { + goto failed; + } + + if (!(import_type = + wasm_importtype_new(&module_name, &name, extern_type))) { + goto failed; + } + + if (!bh_vector_append((Vector *)out, &import_type)) { + goto failed_importtype_new; + } + + continue; + + failed: + wasm_byte_vec_delete(&module_name); + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); + failed_importtype_new: + wasm_importtype_delete(import_type); + } +} + +void +wasm_module_exports(const wasm_module_t *module, wasm_exporttype_vec_t *out) +{ + uint32 i, export_count = 0; + wasm_byte_vec_t name = { 0 }; + wasm_externtype_t *extern_type = NULL; + wasm_exporttype_t *export_type = NULL; + + if (!module || !out) { + return; + } + + if (((const wasm_module_ex_t *)(module))->ref_count == 0) + return; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + export_count = MODULE_INTERP(module)->export_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + export_count = MODULE_AOT(module)->export_count; + } +#endif + + wasm_exporttype_vec_new_uninitialized(out, export_count); + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!out->data) { + return; + } + + for (i = 0; i != export_count; i++) { + WASMExport *export = NULL; +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + export = MODULE_INTERP(module)->exports + i; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + export = MODULE_AOT(module)->exports + i; + } +#endif + + if (!export) { + continue; + } + + /* byte* -> wasm_byte_vec_t */ + wasm_name_new_from_string_nt(&name, export->name); + if (strlen(export->name) && !name.data) { + goto failed; + } + + /* WASMExport -> (WASMFuncType, (uint8, bool)) -> (wasm_functype_t, + * wasm_globaltype_t) -> wasm_externtype_t*/ + switch (export->kind) { + case EXPORT_KIND_FUNC: + { + wasm_functype_t *type = NULL; + WASMFuncType *type_rt; + + if (!wasm_runtime_get_export_func_type(*module, export, + &type_rt)) { + goto failed; + } + + if (!(type = wasm_functype_new_internal(type_rt))) { + goto failed; + } + + extern_type = wasm_functype_as_externtype(type); + break; + } + case EXPORT_KIND_GLOBAL: + { + wasm_globaltype_t *type = NULL; + uint8 val_type_rt = 0; + bool mutability_rt = 0; + + if (!wasm_runtime_get_export_global_type( + *module, export, &val_type_rt, &mutability_rt)) { + goto failed; + } + + if (!(type = wasm_globaltype_new_internal(val_type_rt, + mutability_rt))) { + goto failed; + } + + extern_type = wasm_globaltype_as_externtype(type); + break; + } + case EXPORT_KIND_MEMORY: + { + wasm_memorytype_t *type = NULL; + uint32 min_page = 0, max_page = 0; + + if (!wasm_runtime_get_export_memory_type( + *module, export, &min_page, &max_page)) { + goto failed; + } + + if (!(type = + wasm_memorytype_new_internal(min_page, max_page))) { + goto failed; + } + + extern_type = wasm_memorytype_as_externtype(type); + break; + } + case EXPORT_KIND_TABLE: + { + wasm_tabletype_t *type = NULL; + uint8 elem_type_rt = 0; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type_rt; +#endif + uint32 min_size = 0, max_size = 0; + + if (!wasm_runtime_get_export_table_type(*module, export, + &elem_type_rt, +#if WASM_ENABLE_GC != 0 + &elem_ref_type_rt, +#endif + &min_size, &max_size)) { + goto failed; + } +#if WASM_ENABLE_GC != 0 + (void)elem_ref_type_rt; /* TODO */ +#endif + + if (!(type = wasm_tabletype_new_internal(elem_type_rt, min_size, + max_size))) { + goto failed; + } + + extern_type = wasm_tabletype_as_externtype(type); + break; + } + default: + { + LOG_WARNING("%s meets unsupported type %u", __FUNCTION__, + export->kind); + break; + } + } + + if (!(export_type = wasm_exporttype_new(&name, extern_type))) { + goto failed; + } + + if (!(bh_vector_append((Vector *)out, &export_type))) { + goto failed_exporttype_new; + } + } + + return; + +failed: + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); +failed_exporttype_new: + wasm_exporttype_delete(export_type); + wasm_exporttype_vec_delete(out); +} + +#if WASM_ENABLE_JIT == 0 || WASM_ENABLE_LAZY_JIT != 0 +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + (void)module; + (void)out; + LOG_ERROR("only supported serialization in JIT with eager compilation"); +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *module, const wasm_byte_vec_t *binary) +{ + (void)module; + (void)binary; + LOG_ERROR("only supported deserialization in JIT with eager compilation"); + return NULL; +} +#else + +extern uint8 * +aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data, + uint32 *p_aot_file_size); +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + wasm_module_ex_t *module_ex; + AOTCompContext *comp_ctx; + AOTCompData *comp_data; + uint8 *aot_file_buf = NULL; + uint32 aot_file_size = 0; + + if (!module || !out) + return; + + if (((const wasm_module_ex_t *)(module))->ref_count == 0) + return; + + module_ex = module_to_module_ext(module); + comp_ctx = ((WASMModule *)(module_ex->module_comm_rt))->comp_ctx; + comp_data = ((WASMModule *)(module_ex->module_comm_rt))->comp_data; + bh_assert(comp_ctx != NULL && comp_data != NULL); + + aot_file_buf = aot_emit_aot_file_buf(comp_ctx, comp_data, &aot_file_size); + if (!aot_file_buf) + return; + + wasm_byte_vec_new(out, aot_file_size, (wasm_byte_t *)aot_file_buf); + wasm_runtime_free(aot_file_buf); + return; +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + return wasm_module_new(store, binary); +} +#endif + +wasm_module_t * +wasm_module_obtain(wasm_store_t *store, wasm_shared_module_t *shared_module) +{ + wasm_module_ex_t *module_ex = NULL; + + if (!store || !shared_module) + return NULL; + + module_ex = (wasm_module_ex_t *)shared_module; + + os_mutex_lock(&module_ex->lock); + + /* deleting the module... */ + if (module_ex->ref_count == 0) { + LOG_WARNING("wasm_module_obtain re-enter a module under deleting."); + os_mutex_unlock(&module_ex->lock); + return NULL; + } + + /* add it to a watching list in store */ + if (!bh_vector_append((Vector *)store->modules, &module_ex)) { + os_mutex_unlock(&module_ex->lock); + return NULL; + } + + module_ex->ref_count++; + os_mutex_unlock(&module_ex->lock); + + return (wasm_module_t *)shared_module; +} + +wasm_shared_module_t * +wasm_module_share(wasm_module_t *module) +{ + wasm_module_ex_t *module_ex = NULL; + + if (!module) + return NULL; + + module_ex = (wasm_module_ex_t *)module; + + os_mutex_lock(&module_ex->lock); + + /* deleting the module... */ + if (module_ex->ref_count == 0) { + LOG_WARNING("wasm_module_share re-enter a module under deleting."); + os_mutex_unlock(&module_ex->lock); + return NULL; + } + + module_ex->ref_count++; + + os_mutex_unlock(&module_ex->lock); + + return (wasm_shared_module_t *)module; +} + +void +wasm_shared_module_delete(own wasm_shared_module_t *shared_module) +{ + wasm_module_delete_internal((wasm_module_t *)shared_module); +} + +bool +wasm_module_set_name(wasm_module_t *module, const char *name) +{ + char error_buf[256] = { 0 }; + wasm_module_ex_t *module_ex = NULL; + + if (!module) + return false; + + module_ex = module_to_module_ext(module); + bool ret = wasm_runtime_set_module_name(module_ex->module_comm_rt, name, + error_buf, sizeof(error_buf) - 1); + if (!ret) + LOG_WARNING("set module name failed: %s", error_buf); + return ret; +} + +const char * +wasm_module_get_name(wasm_module_t *module) +{ + wasm_module_ex_t *module_ex = NULL; + if (!module) + return ""; + + module_ex = module_to_module_ext(module); + return wasm_runtime_get_module_name(module_ex->module_comm_rt); +} + +static wasm_func_t * +wasm_func_new_basic(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_t func_callback) +{ + wasm_func_t *func = NULL; + + if (!type) { + goto failed; + } + + if (!(func = malloc_internal(sizeof(wasm_func_t)))) { + goto failed; + } + + func->store = store; + func->kind = WASM_EXTERN_FUNC; + func->func_idx_rt = (uint16)-1; + func->with_env = false; + func->u.cb = func_callback; + + if (!(func->type = wasm_functype_copy(type))) { + goto failed; + } + /* func type's param_count and result_count were checked in + loader and are no larger than UINT16_MAX */ + func->param_count = (uint16)func->type->params->num_elems; + func->result_count = (uint16)func->type->results->num_elems; + + RETURN_OBJ(func, wasm_func_delete) +} + +static wasm_func_t * +wasm_func_new_with_env_basic(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_with_env_t callback, void *env, + void (*finalizer)(void *)) +{ + wasm_func_t *func = NULL; + + if (!type) { + goto failed; + } + + if (!(func = malloc_internal(sizeof(wasm_func_t)))) { + goto failed; + } + + func->store = store; + func->kind = WASM_EXTERN_FUNC; + func->func_idx_rt = (uint16)-1; + func->with_env = true; + func->u.cb_env.cb = callback; + func->u.cb_env.env = env; + func->u.cb_env.finalizer = finalizer; + + if (!(func->type = wasm_functype_copy(type))) { + goto failed; + } + /* func type's param_count and result_count were checked in + loader and are no larger than UINT16_MAX */ + func->param_count = (uint16)func->type->params->num_elems; + func->result_count = (uint16)func->type->results->num_elems; + + RETURN_OBJ(func, wasm_func_delete) +} + +wasm_func_t * +wasm_func_new(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_t callback) +{ + bh_assert(singleton_engine); + if (!callback) { + return NULL; + } + return wasm_func_new_basic(store, type, callback); +} + +wasm_func_t * +wasm_func_new_with_env(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_with_env_t callback, void *env, + void (*finalizer)(void *)) +{ + bh_assert(singleton_engine); + if (!callback) { + return NULL; + } + return wasm_func_new_with_env_basic(store, type, callback, env, finalizer); +} + +wasm_func_t * +wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_func_t *func = NULL; + WASMFuncType *type_rt = NULL; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + func = malloc_internal(sizeof(wasm_func_t)); + if (!func) { + goto failed; + } + + func->kind = WASM_EXTERN_FUNC; + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + bh_assert(func_idx_rt + < ((WASMModuleInstance *)inst_comm_rt)->e->function_count); + WASMFunctionInstance *func_interp = + ((WASMModuleInstance *)inst_comm_rt)->e->functions + func_idx_rt; + type_rt = func_interp->is_import_func + ? func_interp->u.func_import->func_type + : func_interp->u.func->func_type; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + /* use same index to trace the function type in AOTFuncType **func_types + */ + AOTModule *module_aot = + (AOTModule *)((AOTModuleInstance *)inst_comm_rt)->module; + if (func_idx_rt < module_aot->import_func_count) { + type_rt = (module_aot->import_funcs + func_idx_rt)->func_type; + } + else { + type_rt = + (AOTFuncType *)module_aot + ->types[module_aot->func_type_indexes + [func_idx_rt - module_aot->import_func_count]]; + } + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!type_rt) { + goto failed; + } + + func->type = wasm_functype_new_internal(type_rt); + if (!func->type) { + goto failed; + } + /* func type's param_count and result_count were checked in + loader and are no larger than UINT16_MAX */ + func->param_count = (uint16)func->type->params->num_elems; + func->result_count = (uint16)func->type->results->num_elems; + + /* will add name information when processing "exports" */ + func->store = store; + func->module_name = NULL; + func->name = NULL; + func->func_idx_rt = func_idx_rt; + func->inst_comm_rt = inst_comm_rt; + return func; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_func_delete(func); + return NULL; +} + +static wasm_func_t * +wasm_func_new_empty(wasm_store_t *store) +{ + wasm_func_t *func = NULL; + + if (!(func = malloc_internal(sizeof(wasm_func_t)))) + goto failed; + + func->store = store; + func->kind = WASM_EXTERN_FUNC; + + RETURN_OBJ(func, wasm_func_delete) +} + +void +wasm_func_delete(wasm_func_t *func) +{ + if (!func) { + return; + } + + if (func->type) { + wasm_functype_delete(func->type); + func->type = NULL; + } + + if (func->with_env) { + if (func->u.cb_env.finalizer) { + func->u.cb_env.finalizer(func->u.cb_env.env); + func->u.cb_env.finalizer = NULL; + func->u.cb_env.env = NULL; + } + } + + DELETE_HOST_INFO(func) + + wasm_runtime_free(func); +} + +own wasm_func_t * +wasm_func_copy(const wasm_func_t *func) +{ + wasm_func_t *cloned = NULL; + + if (!func) { + return NULL; + } + + if (!(cloned = func->with_env ? wasm_func_new_with_env_basic( + func->store, func->type, func->u.cb_env.cb, + func->u.cb_env.env, func->u.cb_env.finalizer) + : wasm_func_new_basic(func->store, func->type, + func->u.cb))) { + goto failed; + } + + cloned->func_idx_rt = func->func_idx_rt; + cloned->inst_comm_rt = func->inst_comm_rt; + + RETURN_OBJ(cloned, wasm_func_delete) +} + +own wasm_functype_t * +wasm_func_type(const wasm_func_t *func) +{ + if (!func) { + return NULL; + } + return wasm_functype_copy(func->type); +} + +static bool +params_to_argv(const wasm_val_vec_t *params, + const wasm_valtype_vec_t *param_defs, uint32 *argv, + uint32 *ptr_argc) +{ + uint32 *argv_org = argv; + const wasm_val_t *param, *param_end; + + bh_assert(params && params->num_elems && params->size && params->data); + + param = params->data; + param_end = param + param_defs->num_elems; + + for (; param < param_end; param++) { + switch (param->kind) { + case WASM_I32: + case WASM_F32: + *(int32 *)argv = param->of.i32; + argv += 1; + break; + case WASM_I64: + case WASM_F64: + *(int64 *)argv = param->of.i64; + argv += 2; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case WASM_ANYREF: + *(uintptr_t *)argv = (uintptr_t)param->of.ref; + argv += sizeof(uintptr_t) / sizeof(uint32); + break; +#endif + default: + LOG_WARNING("unexpected parameter val type %d", param->kind); + return false; + } + } + + *ptr_argc = (uint32)(argv - argv_org); + return true; +} + +static bool +argv_to_results(const uint32 *argv, const wasm_valtype_vec_t *result_defs, + wasm_val_vec_t *results) +{ + wasm_valtype_t **result_def, **result_def_end; + wasm_val_t *result; + + bh_assert(results && results->size && results->data); + + result_def = result_defs->data; + result_def_end = result_def + result_defs->num_elems; + result = results->data; + + for (; result_def < result_def_end; result_def++, result++) { + result->kind = result_def[0]->kind; + switch (result->kind) { + case WASM_I32: + case WASM_F32: + result->of.i32 = *(int32 *)argv; + argv += 1; + break; + case WASM_I64: + case WASM_F64: + result->of.i64 = *(int64 *)argv; + argv += 2; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case WASM_ANYREF: + result->of.ref = (struct wasm_ref_t *)(*(uintptr_t *)argv); + argv += sizeof(uintptr_t) / sizeof(uint32); + break; +#endif + default: + LOG_WARNING("%s meets unsupported type: %d", __FUNCTION__, + result->kind); + return false; + } + } + + return true; +} + +wasm_trap_t * +wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, + wasm_val_vec_t *results) +{ + /* parameters count as if all are uint32 */ + /* a int64 or float64 parameter means 2 */ + uint32 argc = 0; + /* a parameter list and a return value list */ + uint32 argv_buf[32] = { 0 }, *argv = argv_buf; + WASMFunctionInstanceCommon *func_comm_rt = NULL; + WASMExecEnv *exec_env = NULL; + size_t param_count, result_count, alloc_count; + Vector *cluster_frames = NULL; + + bh_assert(func && func->type); + + if (!func->inst_comm_rt) { + wasm_name_t message = { 0 }; + wasm_trap_t *trap; + + wasm_name_new_from_string_nt(&message, + "failed to call unlinked function"); + trap = wasm_trap_new(func->store, &message); + wasm_byte_vec_delete(&message); + + return trap; + } + + if (func->inst_comm_rt->module_type == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + func_comm_rt = ((WASMModuleInstance *)func->inst_comm_rt)->e->functions + + func->func_idx_rt; +#endif + } + else if (func->inst_comm_rt->module_type == Wasm_Module_AoT) { +#if WASM_ENABLE_AOT != 0 + if (!(func_comm_rt = func->func_comm_rt)) { + AOTModuleInstance *inst_aot = + (AOTModuleInstance *)func->inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + uint32 export_i = 0, export_func_j = 0; + + for (; export_i < module_aot->export_count; ++export_i) { + AOTExport *export = module_aot->exports + export_i; + if (export->kind == EXPORT_KIND_FUNC) { + if (export->index == func->func_idx_rt) { + func_comm_rt = + (AOTFunctionInstance *)inst_aot->export_functions + + export_func_j; + ((wasm_func_t *)func)->func_comm_rt = func_comm_rt; + break; + } + export_func_j++; + } + } + } +#endif + } + + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!func_comm_rt) { + goto failed; + } + + param_count = wasm_func_param_arity(func); + result_count = wasm_func_result_arity(func); + + alloc_count = (param_count > result_count) ? param_count : result_count; + if (alloc_count > (size_t)sizeof(argv_buf) / sizeof(uint64)) { + if (!(argv = malloc_internal(sizeof(uint64) * alloc_count))) { + goto failed; + } + } + + /* copy parametes */ + if (param_count + && !params_to_argv(params, wasm_functype_params(func->type), argv, + &argc)) { + goto failed; + } + +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = wasm_runtime_get_exec_env_tls(); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) { + exec_env = wasm_clusters_search_exec_env(func->inst_comm_rt); + } +#endif + if (!exec_env) { + exec_env = wasm_runtime_get_exec_env_singleton(func->inst_comm_rt); + } + if (!exec_env) { + goto failed; + } + + wasm_runtime_set_exception(func->inst_comm_rt, NULL); + if (!wasm_runtime_call_wasm(exec_env, func_comm_rt, argc, argv)) { + if (wasm_runtime_get_exception(func->inst_comm_rt)) { + LOG_DEBUG("%s", wasm_runtime_get_exception(func->inst_comm_rt)); + goto failed; + } + } + + /* copy results */ + if (result_count) { + if (!argv_to_results(argv, wasm_functype_results(func->type), + results)) { + goto failed; + } + results->num_elems = result_count; + results->size = result_count; + } + + if (argv != argv_buf) + wasm_runtime_free(argv); + return NULL; + +failed: + if (argv != argv_buf) + wasm_runtime_free(argv); + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 && WASM_ENABLE_THREAD_MGR != 0 + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + cluster_frames = &cluster->exception_frames; + wasm_cluster_traverse_lock(exec_env); +#endif + + wasm_trap_t *trap = wasm_trap_new_internal( + func->store, func->inst_comm_rt, + wasm_runtime_get_exception(func->inst_comm_rt), cluster_frames); + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 && WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_unlock(exec_env); +#endif + return trap; +} + +size_t +wasm_func_param_arity(const wasm_func_t *func) +{ + return func->param_count; +} + +size_t +wasm_func_result_arity(const wasm_func_t *func) +{ + return func->result_count; +} + +wasm_global_t * +wasm_global_new(wasm_store_t *store, const wasm_globaltype_t *global_type, + const wasm_val_t *init) +{ + wasm_global_t *global = NULL; + + bh_assert(singleton_engine); + + if (!global_type || !init) { + goto failed; + } + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) { + goto failed; + } + + global->store = store; + global->kind = WASM_EXTERN_GLOBAL; + global->type = wasm_globaltype_copy(global_type); + if (!global->type) { + goto failed; + } + + global->init = malloc_internal(sizeof(wasm_val_t)); + if (!global->init) { + goto failed; + } + + wasm_val_copy(global->init, init); + /* TODO: how to check if above is failed */ + + return global; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +static wasm_global_t * +wasm_global_new_empty(wasm_store_t *store) +{ + wasm_global_t *global = NULL; + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) + goto failed; + + global->store = store; + global->kind = WASM_EXTERN_GLOBAL; + + return global; +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +/* almost same with wasm_global_new */ +wasm_global_t * +wasm_global_copy(const wasm_global_t *src) +{ + wasm_global_t *global = NULL; + + if (!src) { + return NULL; + } + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) { + goto failed; + } + + global->kind = WASM_EXTERN_GLOBAL; + global->type = wasm_globaltype_copy(src->type); + if (!global->type) { + goto failed; + } + + global->init = malloc_internal(sizeof(wasm_val_t)); + if (!global->init) { + goto failed; + } + + wasm_val_copy(global->init, src->init); + + global->global_idx_rt = src->global_idx_rt; + global->inst_comm_rt = src->inst_comm_rt; + + return global; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +void +wasm_global_delete(wasm_global_t *global) +{ + if (!global) { + return; + } + + if (global->init) { + wasm_val_delete(global->init); + global->init = NULL; + } + + if (global->type) { + wasm_globaltype_delete(global->type); + global->type = NULL; + } + + DELETE_HOST_INFO(global) + + wasm_runtime_free(global); +} + +#if WASM_ENABLE_INTERP != 0 +static bool +interp_global_set(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, + const wasm_val_t *v) +{ + const WASMGlobalInstance *global_interp = + inst_interp->e->globals + global_idx_rt; + uint8 val_type_rt = global_interp->type; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint8 *data = global_interp->import_global_inst + ? global_interp->import_module_inst->global_data + + global_interp->import_global_inst->data_offset + : inst_interp->global_data + global_interp->data_offset; +#else + uint8 *data = inst_interp->global_data + global_interp->data_offset; +#endif + + return wasm_val_to_rt_val((WASMModuleInstanceCommon *)inst_interp, + val_type_rt, v, data); +} + +static bool +interp_global_get(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, + wasm_val_t *out) +{ + WASMGlobalInstance *global_interp = inst_interp->e->globals + global_idx_rt; + uint8 val_type_rt = global_interp->type; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint8 *data = global_interp->import_global_inst + ? global_interp->import_module_inst->global_data + + global_interp->import_global_inst->data_offset + : inst_interp->global_data + global_interp->data_offset; +#else + uint8 *data = inst_interp->global_data + global_interp->data_offset; +#endif + + return rt_val_to_wasm_val(data, val_type_rt, out); +} +#endif + +#if WASM_ENABLE_AOT != 0 +static bool +aot_global_set(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, + const wasm_val_t *v) +{ + AOTModule *module_aot = (AOTModule *)inst_aot->module; + uint8 val_type_rt = 0; + uint32 data_offset = 0; + void *data = NULL; + + if (global_idx_rt < module_aot->import_global_count) { + data_offset = module_aot->import_globals[global_idx_rt].data_offset; + val_type_rt = module_aot->import_globals[global_idx_rt].type; + } + else { + data_offset = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .data_offset; + val_type_rt = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .type; + } + + data = (void *)(inst_aot->global_data + data_offset); + return wasm_val_to_rt_val((WASMModuleInstanceCommon *)inst_aot, val_type_rt, + v, data); +} + +static bool +aot_global_get(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, + wasm_val_t *out) +{ + AOTModule *module_aot = (AOTModule *)inst_aot->module; + uint8 val_type_rt = 0; + uint32 data_offset = 0; + uint8 *data = NULL; + + if (global_idx_rt < module_aot->import_global_count) { + data_offset = module_aot->import_globals[global_idx_rt].data_offset; + val_type_rt = module_aot->import_globals[global_idx_rt].type; + } + else { + data_offset = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .data_offset; + val_type_rt = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .type; + } + + data = inst_aot->global_data + data_offset; + return rt_val_to_wasm_val(data, val_type_rt, out); +} +#endif + +void +wasm_global_set(wasm_global_t *global, const wasm_val_t *v) +{ + if (!global || !v || !global->inst_comm_rt) { + return; + } + +#if WASM_ENABLE_INTERP != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + (void)interp_global_set((WASMModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, v); + return; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_AoT) { + (void)aot_global_set((AOTModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, v); + return; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + UNREACHABLE(); +} + +void +wasm_global_get(const wasm_global_t *global, wasm_val_t *out) +{ + if (!global || !out) { + return; + } + + if (!global->inst_comm_rt) { + return; + } + + memset(out, 0, sizeof(wasm_val_t)); + +#if WASM_ENABLE_INTERP != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + (void)interp_global_get((WASMModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, out); + return; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_AoT) { + (void)aot_global_get((AOTModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, out); + return; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + UNREACHABLE(); +} + +wasm_global_t * +wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_global_t *global = NULL; + uint8 val_type_rt = 0; + bool is_mutable = 0; + bool init = false; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) { + goto failed; + } + + global->store = store; + global->kind = WASM_EXTERN_GLOBAL; + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMGlobalInstance *global_interp = + ((WASMModuleInstance *)inst_comm_rt)->e->globals + global_idx_rt; + val_type_rt = global_interp->type; + is_mutable = global_interp->is_mutable; + init = true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + + init = true; + + if (global_idx_rt < module_aot->import_global_count) { + AOTImportGlobal *global_import_aot = + module_aot->import_globals + global_idx_rt; + val_type_rt = global_import_aot->type; + is_mutable = global_import_aot->is_mutable; + } + else { + AOTGlobal *global_aot = + module_aot->globals + + (global_idx_rt - module_aot->import_global_count); + val_type_rt = global_aot->type; + is_mutable = global_aot->is_mutable; + } + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!init) { + goto failed; + } + + global->type = wasm_globaltype_new_internal(val_type_rt, is_mutable); + if (!global->type) { + goto failed; + } + + global->init = malloc_internal(sizeof(wasm_val_t)); + if (!global->init) { + goto failed; + } + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + interp_global_get((WASMModuleInstance *)inst_comm_rt, global_idx_rt, + global->init); + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + aot_global_get((AOTModuleInstance *)inst_comm_rt, global_idx_rt, + global->init); + } +#endif + + global->inst_comm_rt = inst_comm_rt; + global->global_idx_rt = global_idx_rt; + + return global; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +wasm_globaltype_t * +wasm_global_type(const wasm_global_t *global) +{ + if (!global) { + return NULL; + } + return wasm_globaltype_copy(global->type); +} + +static wasm_table_t * +wasm_table_new_basic(wasm_store_t *store, const wasm_tabletype_t *type) +{ + wasm_table_t *table = NULL; + + if (!(table = malloc_internal(sizeof(wasm_table_t)))) { + goto failed; + } + + table->store = store; + table->kind = WASM_EXTERN_TABLE; + + if (!(table->type = wasm_tabletype_copy(type))) { + goto failed; + } + + RETURN_OBJ(table, wasm_table_delete); +} + +wasm_table_t * +wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_table_t *table = NULL; + uint8 val_type_rt = 0; +#if WASM_ENABLE_GC != 0 + WASMRefType *val_ref_type_rt; +#endif + uint32 init_size = 0, max_size = 0; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + if (!(table = malloc_internal(sizeof(wasm_table_t)))) { + goto failed; + } + + table->store = store; + table->kind = WASM_EXTERN_TABLE; + + if (!wasm_runtime_get_table_inst_elem_type(inst_comm_rt, table_idx_rt, + &val_type_rt, +#if WASM_ENABLE_GC != 0 + &val_ref_type_rt, +#endif + &init_size, &max_size)) { + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + goto failed; + } +#if WASM_ENABLE_GC != 0 + (void)val_ref_type_rt; /* TODO */ +#endif + + if (!(table->type = + wasm_tabletype_new_internal(val_type_rt, init_size, max_size))) { + goto failed; + } + + table->inst_comm_rt = inst_comm_rt; + table->table_idx_rt = table_idx_rt; + + RETURN_OBJ(table, wasm_table_delete); +} + +/* will not actually apply this new table into the runtime */ +wasm_table_t * +wasm_table_new(wasm_store_t *store, const wasm_tabletype_t *table_type, + wasm_ref_t *init) +{ + wasm_table_t *table; + (void)init; + + bh_assert(singleton_engine); + + if ((table = wasm_table_new_basic(store, table_type))) { + table->store = store; + } + + return table; +} + +wasm_table_t * +wasm_table_copy(const wasm_table_t *src) +{ + wasm_table_t *table; + + if (!(table = wasm_table_new_basic(src->store, src->type))) { + return NULL; + } + + table->table_idx_rt = src->table_idx_rt; + table->inst_comm_rt = src->inst_comm_rt; + return table; +} + +void +wasm_table_delete(wasm_table_t *table) +{ + if (!table) { + return; + } + + if (table->type) { + wasm_tabletype_delete(table->type); + table->type = NULL; + } + + DELETE_HOST_INFO(table) + + wasm_runtime_free(table); +} + +wasm_tabletype_t * +wasm_table_type(const wasm_table_t *table) +{ + if (!table) { + return NULL; + } + return wasm_tabletype_copy(table->type); +} + +#if WASM_ENABLE_GC == 0 +own wasm_ref_t * +wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) +{ + uint32 ref_idx = NULL_REF; + + if (!table || !table->inst_comm_rt) { + return NULL; + } + +#if WASM_ENABLE_INTERP != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMTableInstance *table_interp = + ((WASMModuleInstance *)table->inst_comm_rt) + ->tables[table->table_idx_rt]; + if (index >= table_interp->cur_size) { + return NULL; + } + ref_idx = (uint32)table_interp->elems[index]; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; + if (index >= table_aot->cur_size) { + return NULL; + } + ref_idx = (uint32)table_aot->elems[index]; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (ref_idx == NULL_REF) { + return NULL; + } + +#if WASM_ENABLE_REF_TYPES != 0 + if (table->type->val_type->kind == WASM_ANYREF) { + void *externref_obj; + if (!wasm_externref_ref2obj(ref_idx, &externref_obj)) { + return NULL; + } + + return externref_obj; + } + else +#endif + { + return wasm_ref_new_internal(table->store, WASM_REF_func, ref_idx, + table->inst_comm_rt); + } +} + +bool +wasm_table_set(wasm_table_t *table, wasm_table_size_t index, + own wasm_ref_t *ref) +{ + uint32 *p_ref_idx = NULL; + uint32 function_count = 0; + + if (!table || !table->inst_comm_rt) { + return false; + } + + if (ref +#if WASM_ENABLE_REF_TYPES != 0 + && !(WASM_REF_foreign == ref->kind + && WASM_ANYREF == table->type->val_type->kind) +#endif + && !(WASM_REF_func == ref->kind + && WASM_FUNCREF == table->type->val_type->kind)) { + return false; + } + +#if WASM_ENABLE_INTERP != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMTableInstance *table_interp = + ((WASMModuleInstance *)table->inst_comm_rt) + ->tables[table->table_idx_rt]; + + if (index >= table_interp->cur_size) { + return false; + } + + p_ref_idx = (uint32 *)(table_interp->elems + index); + function_count = + ((WASMModuleInstance *)table->inst_comm_rt)->e->function_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; + + if (index >= table_aot->cur_size) { + return false; + } + + p_ref_idx = (uint32 *)(table_aot->elems + index); + function_count = module_aot->func_count; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!p_ref_idx) { + return false; + } + +#if WASM_ENABLE_REF_TYPES != 0 + if (table->type->val_type->kind == WASM_ANYREF) { + return wasm_externref_obj2ref(table->inst_comm_rt, ref, p_ref_idx); + } + else +#endif + { + if (ref) { + if (NULL_REF != ref->ref_idx_rt) { + if (ref->ref_idx_rt >= function_count) { + return false; + } + } + *p_ref_idx = ref->ref_idx_rt; + wasm_ref_delete(ref); + } + else { + *p_ref_idx = NULL_REF; + } + } + + return true; +} +#else /* else of WASM_ENABLE_GC == 0 */ +own wasm_ref_t * +wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) +{ + /* TODO */ + return NULL; +} + +bool +wasm_table_set(wasm_table_t *table, wasm_table_size_t index, + own wasm_ref_t *ref) +{ + /* TODO */ + return false; +} +#endif /* end of WASM_ENABLE_GC == 0 */ + +wasm_table_size_t +wasm_table_size(const wasm_table_t *table) +{ + if (!table || !table->inst_comm_rt) { + return 0; + } + +#if WASM_ENABLE_INTERP != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMTableInstance *table_interp = + ((WASMModuleInstance *)table->inst_comm_rt) + ->tables[table->table_idx_rt]; + return table_interp->cur_size; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + + if (table->table_idx_rt < module_aot->import_table_count) { + AOTImportTable *table_aot = + module_aot->import_tables + table->table_idx_rt; + return table_aot->table_init_size; + } + else { + AOTTable *table_aot = + module_aot->tables + + (table->table_idx_rt - module_aot->import_table_count); + return table_aot->table_init_size; + } + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return 0; +} + +bool +wasm_table_grow(wasm_table_t *table, wasm_table_size_t delta, + own wasm_ref_t *init) +{ + (void)table; + (void)delta; + (void)init; + LOG_WARNING("Calling wasm_table_grow() by host is not supported." + "Only allow growing a table via the opcode table.grow"); + return false; +} + +static wasm_memory_t * +wasm_memory_new_basic(wasm_store_t *store, const wasm_memorytype_t *type) +{ + wasm_memory_t *memory = NULL; + + if (!type) { + goto failed; + } + + if (!(memory = malloc_internal(sizeof(wasm_memory_t)))) { + goto failed; + } + + memory->store = store; + memory->kind = WASM_EXTERN_MEMORY; + memory->type = wasm_memorytype_copy(type); + + RETURN_OBJ(memory, wasm_memory_delete) +} + +wasm_memory_t * +wasm_memory_new(wasm_store_t *store, const wasm_memorytype_t *type) +{ + bh_assert(singleton_engine); + return wasm_memory_new_basic(store, type); +} + +wasm_memory_t * +wasm_memory_copy(const wasm_memory_t *src) +{ + wasm_memory_t *dst = NULL; + + if (!src) { + return NULL; + } + + if (!(dst = wasm_memory_new_basic(src->store, src->type))) { + goto failed; + } + + dst->memory_idx_rt = src->memory_idx_rt; + dst->inst_comm_rt = src->inst_comm_rt; + + RETURN_OBJ(dst, wasm_memory_delete) +} + +wasm_memory_t * +wasm_memory_new_internal(wasm_store_t *store, uint16 memory_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_memory_t *memory = NULL; + uint32 min_pages = 0, max_pages = 0; + bool init_flag = false; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + if (!(memory = malloc_internal(sizeof(wasm_memory_t)))) { + goto failed; + } + + memory->store = store; + memory->kind = WASM_EXTERN_MEMORY; + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMMemoryInstance *memory_interp = + ((WASMModuleInstance *)inst_comm_rt)->memories[memory_idx_rt]; + min_pages = memory_interp->cur_page_count; + max_pages = memory_interp->max_page_count; + init_flag = true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + + if (memory_idx_rt < module_aot->import_memory_count) { + min_pages = module_aot->import_memories->mem_init_page_count; + max_pages = module_aot->import_memories->mem_max_page_count; + } + else { + min_pages = module_aot->memories->mem_init_page_count; + max_pages = module_aot->memories->mem_max_page_count; + } + init_flag = true; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!init_flag) { + goto failed; + } + + if (!(memory->type = wasm_memorytype_new_internal(min_pages, max_pages))) { + goto failed; + } + + memory->inst_comm_rt = inst_comm_rt; + memory->memory_idx_rt = memory_idx_rt; + + RETURN_OBJ(memory, wasm_memory_delete); +} + +void +wasm_memory_delete(wasm_memory_t *memory) +{ + if (!memory) { + return; + } + + if (memory->type) { + wasm_memorytype_delete(memory->type); + memory->type = NULL; + } + + DELETE_HOST_INFO(memory) + + wasm_runtime_free(memory); +} + +wasm_memorytype_t * +wasm_memory_type(const wasm_memory_t *memory) +{ + if (!memory) { + return NULL; + } + + return wasm_memorytype_copy(memory->type); +} + +byte_t * +wasm_memory_data(wasm_memory_t *memory) +{ + WASMModuleInstanceCommon *module_inst_comm; + + if (!memory || !memory->inst_comm_rt) { + return NULL; + } + + module_inst_comm = memory->inst_comm_rt; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst = + module_inst->memories[memory->memory_idx_rt]; + return (byte_t *)memory_inst->memory_data; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + AOTMemoryInstance *memory_inst = + ((AOTMemoryInstance **) + module_inst->memories)[memory->memory_idx_rt]; + return (byte_t *)memory_inst->memory_data; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return NULL; +} + +size_t +wasm_memory_data_size(const wasm_memory_t *memory) +{ + WASMModuleInstanceCommon *module_inst_comm; + + if (!memory || !memory->inst_comm_rt) { + return 0; + } + + module_inst_comm = memory->inst_comm_rt; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst = + module_inst->memories[memory->memory_idx_rt]; + return (size_t)memory_inst->cur_page_count + * memory_inst->num_bytes_per_page; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + AOTMemoryInstance *memory_inst = + ((AOTMemoryInstance **) + module_inst->memories)[memory->memory_idx_rt]; + return (size_t)memory_inst->cur_page_count + * memory_inst->num_bytes_per_page; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return 0; +} + +wasm_memory_pages_t +wasm_memory_size(const wasm_memory_t *memory) +{ + WASMModuleInstanceCommon *module_inst_comm; + + if (!memory || !memory->inst_comm_rt) { + return 0; + } + + module_inst_comm = memory->inst_comm_rt; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst = + module_inst->memories[memory->memory_idx_rt]; + return memory_inst->cur_page_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + AOTMemoryInstance *memory_inst = + ((AOTMemoryInstance **) + module_inst->memories)[memory->memory_idx_rt]; + return memory_inst->cur_page_count; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return 0; +} + +bool +wasm_memory_grow(wasm_memory_t *memory, wasm_memory_pages_t delta) +{ + (void)memory; + (void)delta; + LOG_WARNING("Calling wasm_memory_grow() by host is not supported." + "Only allow growing a memory via the opcode memory.grow"); + return false; +} + +#if WASM_ENABLE_INTERP != 0 +static bool +interp_link_func(const wasm_instance_t *inst, const WASMModule *module_interp, + uint16 func_idx_rt, wasm_func_t *import) +{ + WASMImport *imported_func_interp = NULL; + + bh_assert(inst && module_interp && import); + bh_assert(func_idx_rt < module_interp->import_function_count); + bh_assert(WASM_EXTERN_FUNC == import->kind); + + imported_func_interp = module_interp->import_functions + func_idx_rt; + bh_assert(imported_func_interp); + bh_assert(imported_func_interp->kind == IMPORT_KIND_FUNC); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + /* type comparison */ + if (!wasm_functype_same_internal( + import->type, imported_func_interp->u.function.func_type)) + return false; + + imported_func_interp->u.function.call_conv_wasm_c_api = true; + /* only set func_ptr_linked to avoid unlink warning during instantiation, + func_ptr_linked, with_env and env will be stored in module instance's + c_api_func_imports later and used when calling import function */ + if (import->with_env) + imported_func_interp->u.function.func_ptr_linked = import->u.cb_env.cb; + else + imported_func_interp->u.function.func_ptr_linked = import->u.cb; + bh_assert(imported_func_interp->u.function.func_ptr_linked); + + import->func_idx_rt = func_idx_rt; + + (void)inst; + return true; +} + +static bool +interp_link_global(const WASMModule *module_interp, uint16 global_idx_rt, + wasm_global_t *import) +{ + WASMImport *imported_global_interp = NULL; + + bh_assert(module_interp && import); + bh_assert(global_idx_rt < module_interp->import_global_count); + bh_assert(WASM_EXTERN_GLOBAL == import->kind); + + imported_global_interp = module_interp->import_globals + global_idx_rt; + bh_assert(imported_global_interp); + bh_assert(imported_global_interp->kind == IMPORT_KIND_GLOBAL); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + /* type comparison */ + if (!cmp_val_kind_with_val_type(wasm_valtype_kind(import->type->val_type), + imported_global_interp->u.global.type)) + return false; + + /* set init value */ + bh_assert(import->init); + switch (wasm_valtype_kind(import->type->val_type)) { + case WASM_I32: + imported_global_interp->u.global.global_data_linked.i32 = + import->init->of.i32; + break; + case WASM_I64: + imported_global_interp->u.global.global_data_linked.i64 = + import->init->of.i64; + break; + case WASM_F32: + imported_global_interp->u.global.global_data_linked.f32 = + import->init->of.f32; + break; + case WASM_F64: + imported_global_interp->u.global.global_data_linked.f64 = + import->init->of.f64; + break; + default: + return false; + } + + import->global_idx_rt = global_idx_rt; + imported_global_interp->u.global.is_linked = true; + return true; +} + +static bool +interp_process_export(wasm_store_t *store, + const WASMModuleInstance *inst_interp, + wasm_extern_vec_t *externals) +{ + WASMExport *exports = NULL; + WASMExport *export = NULL; + wasm_extern_t *external = NULL; + uint32 export_cnt = 0; + uint32 i = 0; + + bh_assert(store && inst_interp && inst_interp->module && externals); + + exports = inst_interp->module->exports; + export_cnt = inst_interp->module->export_count; + + for (i = 0; i < export_cnt; ++i) { + export = exports + i; + + switch (export->kind) { + case EXPORT_KIND_FUNC: + { + wasm_func_t *func; + if (!(func = wasm_func_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_func_as_extern(func); + break; + } + case EXPORT_KIND_GLOBAL: + { + wasm_global_t *global; + if (!(global = wasm_global_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_global_as_extern(global); + break; + } + case EXPORT_KIND_TABLE: + { + wasm_table_t *table; + if (!(table = wasm_table_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_table_as_extern(table); + break; + } + case EXPORT_KIND_MEMORY: + { + wasm_memory_t *memory; + if (!(memory = wasm_memory_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_memory_as_extern(memory); + break; + } + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + export->kind); + goto failed; + } + + if (!bh_vector_append((Vector *)externals, &external)) { + goto failed; + } + } + + return true; + +failed: + wasm_extern_delete(external); + return false; +} +#endif /* WASM_ENABLE_INTERP */ + +#if WASM_ENABLE_AOT != 0 +static bool +aot_link_func(const wasm_instance_t *inst, const AOTModule *module_aot, + uint32 import_func_idx_rt, wasm_func_t *import) +{ + AOTImportFunc *import_aot_func = NULL; + + bh_assert(inst && module_aot && import); + + import_aot_func = module_aot->import_funcs + import_func_idx_rt; + bh_assert(import_aot_func); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + /* type comparison */ + if (!wasm_functype_same_internal(import->type, import_aot_func->func_type)) + return false; + + import_aot_func->call_conv_wasm_c_api = true; + /* only set func_ptr_linked to avoid unlink warning during instantiation, + func_ptr_linked, with_env and env will be stored in module instance's + c_api_func_imports later and used when calling import function */ + if (import->with_env) + import_aot_func->func_ptr_linked = import->u.cb_env.cb; + else + import_aot_func->func_ptr_linked = import->u.cb; + bh_assert(import_aot_func->func_ptr_linked); + + import->func_idx_rt = import_func_idx_rt; + + return true; +} + +static bool +aot_link_global(const AOTModule *module_aot, uint16 global_idx_rt, + wasm_global_t *import) +{ + AOTImportGlobal *import_aot_global = NULL; + const wasm_valtype_t *val_type = NULL; + + bh_assert(module_aot && import); + + import_aot_global = module_aot->import_globals + global_idx_rt; + bh_assert(import_aot_global); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + val_type = wasm_globaltype_content(import->type); + bh_assert(val_type); + + if (!cmp_val_kind_with_val_type(wasm_valtype_kind(val_type), + import_aot_global->type)) + return false; + + bh_assert(import->init); + switch (wasm_valtype_kind(val_type)) { + case WASM_I32: + import_aot_global->global_data_linked.i32 = import->init->of.i32; + break; + case WASM_I64: + import_aot_global->global_data_linked.i64 = import->init->of.i64; + break; + case WASM_F32: + import_aot_global->global_data_linked.f32 = import->init->of.f32; + break; + case WASM_F64: + import_aot_global->global_data_linked.f64 = import->init->of.f64; + break; + default: + goto failed; + } + + import->global_idx_rt = global_idx_rt; + import_aot_global->is_linked = true; + return true; +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + return false; +} + +static bool +aot_process_export(wasm_store_t *store, const AOTModuleInstance *inst_aot, + wasm_extern_vec_t *externals) +{ + uint32 i; + wasm_extern_t *external = NULL; + AOTModule *module_aot = NULL; + + bh_assert(store && inst_aot && externals); + + module_aot = (AOTModule *)inst_aot->module; + bh_assert(module_aot); + + for (i = 0; i < module_aot->export_count; ++i) { + AOTExport *export = module_aot->exports + i; + + switch (export->kind) { + case EXPORT_KIND_FUNC: + { + wasm_func_t *func = NULL; + if (!(func = wasm_func_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_func_as_extern(func); + break; + } + case EXPORT_KIND_GLOBAL: + { + wasm_global_t *global = NULL; + if (!(global = wasm_global_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_global_as_extern(global); + break; + } + case EXPORT_KIND_TABLE: + { + wasm_table_t *table; + if (!(table = wasm_table_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_table_as_extern(table); + break; + } + case EXPORT_KIND_MEMORY: + { + wasm_memory_t *memory; + if (!(memory = wasm_memory_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_memory_as_extern(memory); + break; + } + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + export->kind); + goto failed; + } + + if (!(external->name = malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + + wasm_name_new_from_string_nt(external->name, export->name); + if (strlen(export->name) && !external->name->data) { + goto failed; + } + + if (!bh_vector_append((Vector *)externals, &external)) { + goto failed; + } + } + + return true; + +failed: + wasm_extern_delete(external); + return false; +} +#endif /* WASM_ENABLE_AOT */ + +static bool +do_link(const wasm_instance_t *inst, const wasm_module_t *module, + const wasm_extern_vec_t *imports) +{ + uint32 i, import_func_i, import_global_i; + + bh_assert(inst && module); + + /* we have run a module_type check before. */ + + for (i = 0, import_func_i = 0, import_global_i = 0; i < imports->num_elems; + i++) { + wasm_extern_t *import = imports->data[i]; + + if (!import) { + LOG_ERROR("imports[%d] is NULL and it is fatal\n", i); + goto failed; + } + + switch (wasm_extern_kind(import)) { + case WASM_EXTERN_FUNC: + { + bool ret = false; +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + ret = interp_link_func(inst, MODULE_INTERP(module), + import_func_i, + wasm_extern_as_func(import)); + } +#endif +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + ret = aot_link_func(inst, MODULE_AOT(module), import_func_i, + wasm_extern_as_func(import)); + } +#endif + if (!ret) { + LOG_WARNING("link function #%d failed", import_func_i); + goto failed; + } + + import_func_i++; + break; + } + case WASM_EXTERN_GLOBAL: + { + bool ret = false; +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + ret = interp_link_global(MODULE_INTERP(module), + import_global_i, + wasm_extern_as_global(import)); + } +#endif +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + ret = aot_link_global(MODULE_AOT(module), import_global_i, + wasm_extern_as_global(import)); + } +#endif + if (!ret) { + LOG_WARNING("link global #%d failed", import_global_i); + goto failed; + } + + import_global_i++; + break; + } + case WASM_EXTERN_MEMORY: + case WASM_EXTERN_TABLE: + { + LOG_WARNING("doesn't support import memories and tables for " + "now, ignore them"); + break; + } + default: + { + UNREACHABLE(); + break; + } + } + } + + return true; +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + return false; +} + +wasm_instance_t * +wasm_instance_new(wasm_store_t *store, const wasm_module_t *module, + const wasm_extern_vec_t *imports, own wasm_trap_t **trap) +{ + return wasm_instance_new_with_args(store, module, imports, trap, + KILOBYTE(32), KILOBYTE(32)); +} + +wasm_instance_t * +wasm_instance_new_with_args(wasm_store_t *store, const wasm_module_t *module, + const wasm_extern_vec_t *imports, + own wasm_trap_t **trap, const uint32 stack_size, + const uint32 heap_size) +{ + InstantiationArgs inst_args = { 0 }; + inst_args.default_stack_size = stack_size; + inst_args.host_managed_heap_size = heap_size; + return wasm_instance_new_with_args_ex(store, module, imports, trap, + &inst_args); +} + +wasm_instance_t * +wasm_instance_new_with_args_ex(wasm_store_t *store, const wasm_module_t *module, + const wasm_extern_vec_t *imports, + own wasm_trap_t **trap, + const InstantiationArgs *inst_args) +{ + char sub_error_buf[128] = { 0 }; + char error_buf[256] = { 0 }; + wasm_instance_t *instance = NULL; + CApiFuncImport *func_import = NULL, **p_func_imports = NULL; + uint32 i = 0, import_func_count = 0; + uint64 total_size; + bool build_exported = false; + + bh_assert(singleton_engine); + + if (!module) + return NULL; + + /* + * will do the check at the end of wasm_runtime_instantiate + */ + + WASM_C_DUMP_PROC_MEM(); + + instance = malloc_internal(sizeof(wasm_instance_t)); + if (!instance) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to malloc instance"); + goto failed; + } + + /* executes the instantiate-time linking if provided */ + if (imports) { + if (!do_link(instance, module, imports)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to validate imports"); + goto failed; + } + } + /* + * will do the linking result check at the end of wasm_runtime_instantiate + */ + + instance->inst_comm_rt = wasm_runtime_instantiate_ex( + *module, inst_args, sub_error_buf, sizeof(sub_error_buf)); + if (!instance->inst_comm_rt) { + goto failed; + } + + if (!wasm_runtime_create_exec_env_singleton(instance->inst_comm_rt)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to create exec env singleton"); + goto failed; + } + + /* create the c-api func import list */ +#if WASM_ENABLE_INTERP != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *wasm_module_inst = + (WASMModuleInstance *)instance->inst_comm_rt; + p_func_imports = &(wasm_module_inst->c_api_func_imports); + import_func_count = MODULE_INTERP(module)->import_function_count; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_module_inst = + (AOTModuleInstance *)instance->inst_comm_rt; + p_func_imports = &(aot_module_inst->c_api_func_imports); + import_func_count = MODULE_AOT(module)->import_func_count; + } +#endif + bh_assert(p_func_imports); + + total_size = (uint64)sizeof(CApiFuncImport) * import_func_count; + if (total_size > 0 + && !(*p_func_imports = func_import = malloc_internal(total_size))) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to create wasm-c-api func imports"); + goto failed; + } + + /* fill in module_inst->c_api_func_imports */ + for (i = 0; imports && i < imports->num_elems; i++) { + wasm_func_t *func_host = NULL; + wasm_extern_t *in = imports->data[i]; + bh_assert(in); + + if (wasm_extern_kind(in) != WASM_EXTERN_FUNC) + continue; + + func_host = wasm_extern_as_func(in); + /* it is a placeholder and let's skip it*/ + if (!func_host->type) { + func_import++; + continue; + } + + func_import->with_env_arg = func_host->with_env; + if (func_host->with_env) { + func_import->func_ptr_linked = func_host->u.cb_env.cb; + func_import->env_arg = func_host->u.cb_env.env; + } + else { + func_import->func_ptr_linked = func_host->u.cb; + func_import->env_arg = NULL; + } + bh_assert(func_import->func_ptr_linked); + + func_import++; + } + + /* fill with inst */ + for (i = 0; imports && imports->data && i < imports->num_elems; ++i) { + wasm_extern_t *import = imports->data[i]; + bh_assert(import); + + switch (import->kind) { + case WASM_EXTERN_FUNC: + wasm_extern_as_func(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + case WASM_EXTERN_GLOBAL: + wasm_extern_as_global(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + case WASM_EXTERN_MEMORY: + wasm_extern_as_memory(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + case WASM_EXTERN_TABLE: + wasm_extern_as_table(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + default: + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Unknown import kind"); + goto failed; + } + } + + /* build the exports list */ +#if WASM_ENABLE_INTERP != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + uint32 export_cnt = ((WASMModuleInstance *)instance->inst_comm_rt) + ->module->export_count; + + INIT_VEC(instance->exports, wasm_extern_vec_new_uninitialized, + export_cnt); + + if (!interp_process_export(store, + (WASMModuleInstance *)instance->inst_comm_rt, + instance->exports)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Interpreter failed to process exports"); + goto failed; + } + + build_exported = true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_AoT) { + uint32 export_cnt = + ((AOTModuleInstance *)instance->inst_comm_rt)->export_func_count + + ((AOTModuleInstance *)instance->inst_comm_rt)->export_global_count + + ((AOTModuleInstance *)instance->inst_comm_rt)->export_table_count + + ((AOTModuleInstance *)instance->inst_comm_rt) + ->export_memory_count; + + INIT_VEC(instance->exports, wasm_extern_vec_new_uninitialized, + export_cnt); + + if (!aot_process_export(store, + (AOTModuleInstance *)instance->inst_comm_rt, + instance->exports)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "AOT failed to process exports"); + goto failed; + } + + build_exported = true; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!build_exported) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Incorrect filetype and compilation flags"); + goto failed; + } + + /* add it to a watching list in store */ + if (!bh_vector_append((Vector *)store->instances, &instance)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to add to store instances"); + goto failed; + } + + WASM_C_DUMP_PROC_MEM(); + + return instance; + +failed: + snprintf(error_buf, sizeof(error_buf), "%s failed: %s", __FUNCTION__, + sub_error_buf); + if (trap != NULL) { + wasm_message_t message = { 0 }; + wasm_name_new_from_string_nt(&message, error_buf); + *trap = wasm_trap_new(store, &message); + wasm_byte_vec_delete(&message); + } + LOG_DEBUG("%s", error_buf); + wasm_instance_delete_internal(instance); + return NULL; +} + +static void +wasm_instance_delete_internal(wasm_instance_t *instance) +{ + if (!instance) { + return; + } + + DEINIT_VEC(instance->exports, wasm_extern_vec_delete); + + if (instance->inst_comm_rt) { + wasm_runtime_deinstantiate(instance->inst_comm_rt); + instance->inst_comm_rt = NULL; + } + wasm_runtime_free(instance); +} + +void +wasm_instance_delete(wasm_instance_t *inst) +{ + DELETE_HOST_INFO(inst) + /* will release instance when releasing the store */ +} + +void +wasm_instance_exports(const wasm_instance_t *instance, + own wasm_extern_vec_t *out) +{ + if (!instance || !out) { + return; + } + wasm_extern_vec_copy(out, instance->exports); +} + +wasm_extern_t * +wasm_extern_copy(const wasm_extern_t *src) +{ + wasm_extern_t *dst = NULL; + + if (!src) { + return NULL; + } + + switch (wasm_extern_kind(src)) { + case WASM_EXTERN_FUNC: + dst = wasm_func_as_extern( + wasm_func_copy(wasm_extern_as_func_const(src))); + break; + case WASM_EXTERN_GLOBAL: + dst = wasm_global_as_extern( + wasm_global_copy(wasm_extern_as_global_const(src))); + break; + case WASM_EXTERN_MEMORY: + dst = wasm_memory_as_extern( + wasm_memory_copy(wasm_extern_as_memory_const(src))); + break; + case WASM_EXTERN_TABLE: + dst = wasm_table_as_extern( + wasm_table_copy(wasm_extern_as_table_const(src))); + break; + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + src->kind); + break; + } + + if (!dst) { + goto failed; + } + + return dst; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_extern_delete(dst); + return NULL; +} + +void +wasm_extern_delete(wasm_extern_t *external) +{ + if (!external) { + return; + } + + if (external->name) { + wasm_byte_vec_delete(external->name); + wasm_runtime_free(external->name); + external->name = NULL; + } + + switch (wasm_extern_kind(external)) { + case WASM_EXTERN_FUNC: + wasm_func_delete(wasm_extern_as_func(external)); + break; + case WASM_EXTERN_GLOBAL: + wasm_global_delete(wasm_extern_as_global(external)); + break; + case WASM_EXTERN_MEMORY: + wasm_memory_delete(wasm_extern_as_memory(external)); + break; + case WASM_EXTERN_TABLE: + wasm_table_delete(wasm_extern_as_table(external)); + break; + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + external->kind); + break; + } +} + +wasm_externkind_t +wasm_extern_kind(const wasm_extern_t *external) +{ + if (!external) { + return WASM_ANYREF; + } + + return external->kind; +} + +own wasm_externtype_t * +wasm_extern_type(const wasm_extern_t *external) +{ + if (!external) { + return NULL; + } + + switch (wasm_extern_kind(external)) { + case WASM_EXTERN_FUNC: + return wasm_functype_as_externtype( + wasm_func_type(wasm_extern_as_func_const(external))); + case WASM_EXTERN_GLOBAL: + return wasm_globaltype_as_externtype( + wasm_global_type(wasm_extern_as_global_const(external))); + case WASM_EXTERN_MEMORY: + return wasm_memorytype_as_externtype( + wasm_memory_type(wasm_extern_as_memory_const(external))); + case WASM_EXTERN_TABLE: + return wasm_tabletype_as_externtype( + wasm_table_type(wasm_extern_as_table_const(external))); + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + external->kind); + break; + } + return NULL; +} + +#define BASIC_FOUR_LIST(V) \ + V(func) \ + V(global) \ + V(memory) \ + V(table) + +#define WASM_EXTERN_AS_OTHER(name) \ + wasm_##name##_t *wasm_extern_as_##name(wasm_extern_t *external) \ + { \ + return (wasm_##name##_t *)external; \ + } + +BASIC_FOUR_LIST(WASM_EXTERN_AS_OTHER) +#undef WASM_EXTERN_AS_OTHER + +#define WASM_OTHER_AS_EXTERN(name) \ + wasm_extern_t *wasm_##name##_as_extern(wasm_##name##_t *other) \ + { \ + return (wasm_extern_t *)other; \ + } + +BASIC_FOUR_LIST(WASM_OTHER_AS_EXTERN) +#undef WASM_OTHER_AS_EXTERN + +#define WASM_EXTERN_AS_OTHER_CONST(name) \ + const wasm_##name##_t *wasm_extern_as_##name##_const( \ + const wasm_extern_t *external) \ + { \ + return (const wasm_##name##_t *)external; \ + } + +BASIC_FOUR_LIST(WASM_EXTERN_AS_OTHER_CONST) +#undef WASM_EXTERN_AS_OTHER_CONST + +#define WASM_OTHER_AS_EXTERN_CONST(name) \ + const wasm_extern_t *wasm_##name##_as_extern_const( \ + const wasm_##name##_t *other) \ + { \ + return (const wasm_extern_t *)other; \ + } + +BASIC_FOUR_LIST(WASM_OTHER_AS_EXTERN_CONST) +#undef WASM_OTHER_AS_EXTERN_CONST + +wasm_extern_t * +wasm_extern_new_empty(wasm_store_t *store, wasm_externkind_t extern_kind) +{ + if (extern_kind == WASM_EXTERN_FUNC) + return wasm_func_as_extern(wasm_func_new_empty(store)); + + if (extern_kind == WASM_EXTERN_GLOBAL) + return wasm_global_as_extern(wasm_global_new_empty(store)); + + LOG_ERROR("Don't support linking table and memory for now"); + return NULL; +} diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_c_api_internal.h b/wasm-micro-runtime/core/iwasm/common/wasm_c_api_internal.h new file mode 100644 index 0000000..3967bb2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_c_api_internal.h @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_C_API_INTERNAL_H +#define _WASM_C_API_INTERNAL_H + +#include "../include/wasm_c_api.h" +#include "wasm_runtime_common.h" + +#ifndef own +#define own +#endif + +/* Vectors */ +/* we will malloc resource for the vector's data field */ +/* we will release resource of data */ +/* caller needs to take care resource for the vector itself */ +#define DEFAULT_VECTOR_INIT_LENGTH (64) + +WASM_DECLARE_VEC(instance, *) +WASM_DECLARE_VEC(module, *) +WASM_DECLARE_VEC(store, *) + +/* Runtime Environment */ +struct wasm_engine_t { + uint32 ref_count; + /* list of wasm_module_ex_t */ + Vector modules; + /* list of stores which are classified according to tids */ + Vector stores_by_tid; +}; + +struct wasm_store_t { + /* maybe should remove the list */ + wasm_module_vec_t *modules; + wasm_instance_vec_t *instances; + Vector *foreigns; +}; + +/* Type Representations */ +struct wasm_valtype_t { + wasm_valkind_t kind; +}; + +struct wasm_functype_t { + uint32 extern_kind; + /* gona to new and delete own */ + wasm_valtype_vec_t *params; + wasm_valtype_vec_t *results; +}; + +struct wasm_globaltype_t { + uint32 extern_kind; + /* gona to new and delete own */ + wasm_valtype_t *val_type; + wasm_mutability_t mutability; +}; + +struct wasm_tabletype_t { + uint32 extern_kind; + wasm_valtype_t *val_type; + wasm_limits_t limits; +}; + +struct wasm_memorytype_t { + uint32 extern_kind; + wasm_limits_t limits; +}; + +struct wasm_externtype_t { + uint32 extern_kind; + /* reservered space */ + uint8 data[1]; +}; + +struct wasm_importtype_t { + wasm_name_t *module_name; + wasm_name_t *name; + wasm_externtype_t *extern_type; +}; + +struct wasm_exporttype_t { + wasm_name_t *name; + wasm_externtype_t *extern_type; +}; + +/* Runtime Objects */ +enum wasm_reference_kind { + WASM_REF_foreign, + WASM_REF_func, + WASM_REF_global, + WASM_REF_memory, + WASM_REF_table, +}; + +struct wasm_host_info { + void *info; + void (*finalizer)(void *); +}; + +struct wasm_ref_t { + wasm_store_t *store; + enum wasm_reference_kind kind; + struct wasm_host_info host_info; + uint32 ref_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_trap_t { + wasm_byte_vec_t *message; + Vector *frames; +}; + +struct wasm_foreign_t { + wasm_store_t *store; + enum wasm_reference_kind kind; + struct wasm_host_info host_info; + int32 ref_cnt; + uint32 foreign_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_func_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_functype_t *type; + uint16 param_count; + uint16 result_count; + + bool with_env; + union { + wasm_func_callback_t cb; + struct callback_ext { + void *env; + wasm_func_callback_with_env_t cb; + void (*finalizer)(void *); + } cb_env; + } u; + /* + * an index in both functions runtime instance lists + * of interpreter mode and aot mode + */ + uint16 func_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; + WASMFunctionInstanceCommon *func_comm_rt; +}; + +struct wasm_global_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_globaltype_t *type; + wasm_val_t *init; + /* + * an index in both global runtime instance lists + * of interpreter mode and aot mode + */ + uint16 global_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_memory_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_memorytype_t *type; + /* + * an index in both memory runtime instance lists + * of interpreter mode and aot mode + */ + uint16 memory_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_table_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_tabletype_t *type; + /* + * an index in both table runtime instance lists + * of interpreter mode and aot mode + */ + uint16 table_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_extern_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + wasm_externkind_t kind; + /* reservered space */ + uint8 data[1]; +}; + +struct wasm_instance_t { + wasm_store_t *store; + wasm_extern_vec_t *exports; + struct wasm_host_info host_info; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +wasm_ref_t * +wasm_ref_new_internal(wasm_store_t *store, enum wasm_reference_kind kind, + uint32 obj_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_foreign_t * +wasm_foreign_new_internal(wasm_store_t *store, uint32 foreign_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_func_t * +wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_global_t * +wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_memory_t * +wasm_memory_new_internal(wasm_store_t *store, uint16 memory_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_table_t * +wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +void +wasm_frame_vec_clone_internal(Vector *src, Vector *out); +#endif /* _WASM_C_API_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_exec_env.c b/wasm-micro-runtime/core/iwasm/common/wasm_exec_env.c new file mode 100644 index 0000000..373ac46 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_exec_env.c @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_exec_env.h" +#include "wasm_runtime_common.h" +#if WASM_ENABLE_GC != 0 +#include "mem_alloc.h" +#endif +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif + +#if WASM_ENABLE_AOT != 0 +#include "aot_runtime.h" +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "../libraries/debug-engine/debug_engine.h" +#endif +#endif + +WASMExecEnv * +wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size) +{ + uint64 total_size = + offsetof(WASMExecEnv, wasm_stack_u.bottom) + (uint64)stack_size; + WASMExecEnv *exec_env; + + if (total_size >= UINT32_MAX + || !(exec_env = wasm_runtime_malloc((uint32)total_size))) + return NULL; + + memset(exec_env, 0, (uint32)total_size); + +#if WASM_ENABLE_AOT != 0 + if (!(exec_env->argv_buf = wasm_runtime_malloc(sizeof(uint32) * 64))) { + goto fail1; + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + if (os_mutex_init(&exec_env->wait_lock) != 0) + goto fail2; + + if (os_cond_init(&exec_env->wait_cond) != 0) + goto fail3; + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!(exec_env->current_status = wasm_cluster_create_exenv_status())) + goto fail4; +#endif +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!(exec_env->exce_check_guard_page = + os_mmap(NULL, os_getpagesize(), MMAP_PROT_NONE, MMAP_MAP_NONE, + os_get_invalid_handle()))) + goto fail5; +#endif + + exec_env->module_inst = module_inst; + exec_env->wasm_stack_size = stack_size; + exec_env->wasm_stack.bottom = exec_env->wasm_stack_u.bottom; + exec_env->wasm_stack.top_boundary = + exec_env->wasm_stack.bottom + stack_size; + exec_env->wasm_stack.top = exec_env->wasm_stack.bottom; + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *i = (AOTModuleInstance *)module_inst; + AOTModule *m = (AOTModule *)i->module; + exec_env->native_symbol = m->native_symbol_list; + } +#endif + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_exec_env_mem_consumption(exec_env); +#endif + + return exec_env; + +#ifdef OS_ENABLE_HW_BOUND_CHECK +fail5: +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_destroy_exenv_status(exec_env->current_status); +#endif +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#if WASM_ENABLE_DEBUG_INTERP != 0 +fail4: + os_cond_destroy(&exec_env->wait_cond); +#endif +fail3: + os_mutex_destroy(&exec_env->wait_lock); +fail2: +#endif +#if WASM_ENABLE_AOT != 0 + wasm_runtime_free(exec_env->argv_buf); +fail1: +#endif + wasm_runtime_free(exec_env); + return NULL; +} + +void +wasm_exec_env_destroy_internal(WASMExecEnv *exec_env) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_munmap(exec_env->exce_check_guard_page, os_getpagesize()); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + os_mutex_destroy(&exec_env->wait_lock); + os_cond_destroy(&exec_env->wait_cond); +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_destroy_exenv_status(exec_env->current_status); +#endif +#endif +#if WASM_ENABLE_AOT != 0 + wasm_runtime_free(exec_env->argv_buf); +#endif + wasm_runtime_free(exec_env); +} + +WASMExecEnv * +wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + WASMCluster *cluster; +#endif + WASMExecEnv *exec_env = + wasm_exec_env_create_internal(module_inst, stack_size); +#if WASM_ENABLE_GC != 0 + void *gc_heap_handle = NULL; +#endif + + if (!exec_env) + return NULL; + +#if WASM_ENABLE_INTERP != 0 + /* Set the aux_stack_boundary and aux_stack_bottom */ + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModule *module = ((WASMModuleInstance *)module_inst)->module; + exec_env->aux_stack_bottom = (uintptr_t)module->aux_stack_bottom; + exec_env->aux_stack_boundary = + (uintptr_t)module->aux_stack_bottom - module->aux_stack_size; +#if WASM_ENABLE_GC != 0 + gc_heap_handle = + ((WASMModuleInstance *)module_inst)->e->common.gc_heap_pool; +#endif + } +#endif +#if WASM_ENABLE_AOT != 0 + /* Set the aux_stack_boundary and aux_stack_bottom */ + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)module_inst)->module; + exec_env->aux_stack_bottom = (uintptr_t)module->aux_stack_bottom; + exec_env->aux_stack_boundary = + (uintptr_t)module->aux_stack_bottom - module->aux_stack_size; +#if WASM_ENABLE_GC != 0 + gc_heap_handle = + ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->common.gc_heap_handle; +#endif + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Create a new cluster for this exec_env */ + if (!(cluster = wasm_cluster_create(exec_env))) { + wasm_exec_env_destroy_internal(exec_env); + return NULL; + } +#if WASM_ENABLE_GC != 0 + mem_allocator_enable_gc_reclaim(gc_heap_handle, cluster); +#endif +#else +#if WASM_ENABLE_GC != 0 + mem_allocator_enable_gc_reclaim(gc_heap_handle, exec_env); +#endif +#endif /* end of WASM_ENABLE_THREAD_MGR */ + + return exec_env; +} + +void +wasm_exec_env_destroy(WASMExecEnv *exec_env) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + /* Wait for all sub-threads */ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + if (cluster) { + wasm_cluster_wait_for_all_except_self(cluster, exec_env); +#if WASM_ENABLE_DEBUG_INTERP != 0 + /* Must fire exit event after other threads exits, otherwise + the stopped thread will be overrided by other threads */ + wasm_cluster_thread_exited(exec_env); +#endif + /* We have waited for other threads, this is the only alive thread, so + * we don't acquire cluster->lock because the cluster will be destroyed + * inside this function */ + wasm_cluster_del_exec_env(cluster, exec_env); + } +#endif /* end of WASM_ENABLE_THREAD_MGR */ + + wasm_exec_env_destroy_internal(exec_env); +} + +WASMModuleInstanceCommon * +wasm_exec_env_get_module_inst(WASMExecEnv *exec_env) +{ + return exec_env->module_inst; +} + +void +wasm_exec_env_set_module_inst(WASMExecEnv *exec_env, + WASMModuleInstanceCommon *const module_inst) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_lock(exec_env); +#endif + exec_env->module_inst = module_inst; +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_unlock(exec_env); +#endif +} + +void +wasm_exec_env_restore_module_inst( + WASMExecEnv *exec_env, WASMModuleInstanceCommon *const module_inst_common) +{ + WASMModuleInstanceCommon *old_module_inst_common = exec_env->module_inst; + WASMModuleInstance *old_module_inst = + (WASMModuleInstance *)old_module_inst_common; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_common; + char cur_exception[EXCEPTION_BUF_LEN]; + +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_lock(exec_env); +#endif + exec_env->module_inst = module_inst_common; + /* + * propagate an exception if any. + */ + exception_lock(old_module_inst); + if (old_module_inst->cur_exception[0] != '\0') { + bh_memcpy_s(cur_exception, sizeof(cur_exception), + old_module_inst->cur_exception, + sizeof(old_module_inst->cur_exception)); + } + else { + cur_exception[0] = '\0'; + } + exception_unlock(old_module_inst); +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_unlock(exec_env); +#endif + if (cur_exception[0] != '\0') { + exception_lock(module_inst); + bh_memcpy_s(module_inst->cur_exception, + sizeof(module_inst->cur_exception), cur_exception, + sizeof(cur_exception)); + exception_unlock(module_inst); + } +} + +void +wasm_exec_env_set_thread_info(WASMExecEnv *exec_env) +{ + uint8 *stack_boundary = os_thread_get_stack_boundary(); + +#if WASM_ENABLE_THREAD_MGR != 0 + os_mutex_lock(&exec_env->wait_lock); +#endif + exec_env->handle = os_self_thread(); + exec_env->native_stack_boundary = + stack_boundary ? stack_boundary + WASM_STACK_GUARD_SIZE : NULL; + exec_env->native_stack_top_min = (void *)UINTPTR_MAX; +#if WASM_ENABLE_THREAD_MGR != 0 + os_mutex_unlock(&exec_env->wait_lock); +#endif +} + +#if WASM_ENABLE_THREAD_MGR != 0 +void * +wasm_exec_env_get_thread_arg(WASMExecEnv *exec_env) +{ + return exec_env->thread_arg; +} + +void +wasm_exec_env_set_thread_arg(WASMExecEnv *exec_env, void *thread_arg) +{ + exec_env->thread_arg = thread_arg; +} +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +void +wasm_exec_env_push_jmpbuf(WASMExecEnv *exec_env, WASMJmpBuf *jmpbuf) +{ + jmpbuf->prev = exec_env->jmpbuf_stack_top; + exec_env->jmpbuf_stack_top = jmpbuf; +} + +WASMJmpBuf * +wasm_exec_env_pop_jmpbuf(WASMExecEnv *exec_env) +{ + WASMJmpBuf *stack_top = exec_env->jmpbuf_stack_top; + + if (stack_top) { + exec_env->jmpbuf_stack_top = stack_top->prev; + return stack_top; + } + + return NULL; +} +#endif diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_exec_env.h b/wasm-micro-runtime/core/iwasm/common/wasm_exec_env.h new file mode 100644 index 0000000..53d2487 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_exec_env.h @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_EXEC_ENV_H +#define _WASM_EXEC_ENV_H + +#include "bh_assert.h" +#include "wasm_suspend_flags.h" +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +struct WASMModuleInstanceCommon; +struct WASMInterpFrame; + +#if WASM_ENABLE_THREAD_MGR != 0 +typedef struct WASMCluster WASMCluster; +#if WASM_ENABLE_DEBUG_INTERP != 0 +typedef struct WASMCurrentEnvStatus WASMCurrentEnvStatus; +#endif +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +typedef struct WASMJmpBuf { + struct WASMJmpBuf *prev; + korp_jmpbuf jmpbuf; +} WASMJmpBuf; +#endif + +/* Execution environment */ +typedef struct WASMExecEnv { + /* Next thread's exec env of a WASM module instance. */ + struct WASMExecEnv *next; + + /* Current interpreter/AOT frame of current thread */ + struct WASMInterpFrame *cur_frame; + + /* Note: field module_inst, argv_buf, native_stack_boundary, + suspend_flags, aux_stack_boundary, aux_stack_bottom, and + native_symbol are used by AOTed code, don't change the + places of them */ + + /* The WASM module instance of current thread */ + struct WASMModuleInstanceCommon *module_inst; + +#if WASM_ENABLE_AOT != 0 + uint32 *argv_buf; +#endif + + /* The boundary of native stack. When runtime detects that native + frame may overrun this boundary, it throws stack overflow + exception. */ + uint8 *native_stack_boundary; + + /* Used to terminate or suspend current thread */ + WASMSuspendFlags suspend_flags; + + /* Auxiliary stack boundary */ + uintptr_t aux_stack_boundary; + + /* Auxiliary stack bottom */ + uintptr_t aux_stack_bottom; + +#if WASM_ENABLE_AOT != 0 + /* Native symbol list, reserved */ + void **native_symbol; +#endif + + /* + * The lowest stack pointer value observed. + * Assumption: native stack grows to the lower address. + */ + uint8 *native_stack_top_min; + + struct { + /* The top boundary of the stack. */ + uint8 *top_boundary; + /* The top to of the wasm stack which is free. */ + uint8 *top; + /* The bottom of the wasm stack. */ + uint8 *bottom; + } wasm_stack; + +#if WASM_ENABLE_FAST_JIT != 0 + /** + * Cache for + * - jit native operations in 32-bit target which hasn't 64-bit + * int/float registers, mainly for the operations of double and int64, + * such as F64TOI64, F32TOI64, I64 MUL/REM, and so on. + * - SSE instructions. + **/ + uint64 jit_cache[2]; +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + /* thread return value */ + void *thread_ret_value; + + /* Must be provided by thread library */ + void *(*thread_start_routine)(void *); + void *thread_arg; + + /* pointer to the cluster */ + WASMCluster *cluster; + + /* used to support debugger */ + korp_mutex wait_lock; + korp_cond wait_cond; + /* the count of threads which are joining current thread */ + uint32 wait_count; + + /* whether current thread is detached */ + bool thread_is_detached; + + /* whether the aux stack is allocated */ + bool is_aux_stack_allocated; +#endif + +#if WASM_ENABLE_GC != 0 + /* Current local object reference variable */ + struct WASMLocalObjectRef *cur_local_object_ref; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + WASMCurrentEnvStatus *current_status; +#endif + + /* attachment for native function */ + void *attachment; + + void *user_data; + + /* The native thread handle of current thread */ + korp_tid handle; + +#if WASM_ENABLE_INTERP != 0 && WASM_ENABLE_FAST_INTERP == 0 + BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE]; +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMJmpBuf *jmpbuf_stack_top; + /* One guard page for the exception check */ + uint8 *exce_check_guard_page; +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + uint32 max_wasm_stack_used; +#endif + + /* The WASM stack size */ + uint32 wasm_stack_size; + + /* The WASM stack of current thread */ + union { + uint64 __make_it_8_byte_aligned_; + /* The WASM stack. */ + uint8 bottom[1]; + } wasm_stack_u; +} WASMExecEnv; + +#if WASM_ENABLE_MEMORY_PROFILING != 0 +#define RECORD_STACK_USAGE(e, p) \ + do { \ + if ((e)->native_stack_top_min > (p)) { \ + (e)->native_stack_top_min = (p); \ + } \ + } while (0) +#else +#define RECORD_STACK_USAGE(e, p) (void)0 +#endif + +WASMExecEnv * +wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size); + +void +wasm_exec_env_destroy_internal(WASMExecEnv *exec_env); + +WASMExecEnv * +wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size); + +void +wasm_exec_env_destroy(WASMExecEnv *exec_env); + +static inline bool +wasm_exec_env_is_aux_stack_managed_by_runtime(WASMExecEnv *exec_env) +{ + return exec_env->aux_stack_boundary != 0 || exec_env->aux_stack_bottom != 0; +} + +/** + * Allocate a WASM frame from the WASM stack. + * + * @param exec_env the current execution environment + * @param size size of the WASM frame, it must be a multiple of 4 + * + * @return the WASM frame if there is enough space in the stack area + * with a protection area, NULL otherwise + */ +static inline void * +wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size) +{ + uint8 *addr = exec_env->wasm_stack.top; + + bh_assert(!(size & 3)); + + /* For classic interpreter, the outs area doesn't contain the const cells, + its size cannot be larger than the frame size, so here checking stack + overflow with multiplying by 2 is enough. For fast interpreter, since + the outs area contains const cells, its size may be larger than current + frame size, we should check again before putting the function arguments + into the outs area. */ + if (size * 2 + > (uint32)(uintptr_t)(exec_env->wasm_stack.top_boundary - addr)) { + /* WASM stack overflow. */ + return NULL; + } + + exec_env->wasm_stack.top += size; + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + { + uint32 wasm_stack_used = + exec_env->wasm_stack.top - exec_env->wasm_stack.bottom; + if (wasm_stack_used > exec_env->max_wasm_stack_used) + exec_env->max_wasm_stack_used = wasm_stack_used; + } +#endif + return addr; +} + +static inline void +wasm_exec_env_free_wasm_frame(WASMExecEnv *exec_env, void *prev_top) +{ + bh_assert((uint8 *)prev_top >= exec_env->wasm_stack.bottom); + exec_env->wasm_stack.top = (uint8 *)prev_top; +} + +/** + * Get the current WASM stack top pointer. + * + * @param exec_env the current execution environment + * + * @return the current WASM stack top pointer + */ +static inline void * +wasm_exec_env_wasm_stack_top(WASMExecEnv *exec_env) +{ + return exec_env->wasm_stack.top; +} + +/** + * Set the current frame pointer. + * + * @param exec_env the current execution environment + * @param frame the WASM frame to be set for the current exec env + */ +static inline void +wasm_exec_env_set_cur_frame(WASMExecEnv *exec_env, + struct WASMInterpFrame *frame) +{ + exec_env->cur_frame = frame; +} + +/** + * Get the current frame pointer. + * + * @param exec_env the current execution environment + * + * @return the current frame pointer + */ +static inline struct WASMInterpFrame * +wasm_exec_env_get_cur_frame(WASMExecEnv *exec_env) +{ + return exec_env->cur_frame; +} + +struct WASMModuleInstanceCommon * +wasm_exec_env_get_module_inst(WASMExecEnv *exec_env); + +void +wasm_exec_env_set_module_inst( + WASMExecEnv *exec_env, struct WASMModuleInstanceCommon *const module_inst); + +void +wasm_exec_env_restore_module_inst( + WASMExecEnv *exec_env, struct WASMModuleInstanceCommon *const module_inst); + +void +wasm_exec_env_set_thread_info(WASMExecEnv *exec_env); + +#if WASM_ENABLE_THREAD_MGR != 0 +void * +wasm_exec_env_get_thread_arg(WASMExecEnv *exec_env); + +void +wasm_exec_env_set_thread_arg(WASMExecEnv *exec_env, void *thread_arg); +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +void +wasm_exec_env_push_jmpbuf(WASMExecEnv *exec_env, WASMJmpBuf *jmpbuf); + +WASMJmpBuf * +wasm_exec_env_pop_jmpbuf(WASMExecEnv *exec_env); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_EXEC_ENV_H */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_memory.c b/wasm-micro-runtime/core/iwasm/common/wasm_memory.c new file mode 100644 index 0000000..c49c182 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_memory.c @@ -0,0 +1,1037 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_runtime_common.h" +#include "../interpreter/wasm_runtime.h" +#include "../aot/aot_runtime.h" +#include "mem_alloc.h" +#include "wasm_memory.h" + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif + +typedef enum Memory_Mode { + MEMORY_MODE_UNKNOWN = 0, + MEMORY_MODE_POOL, + MEMORY_MODE_ALLOCATOR, + MEMORY_MODE_SYSTEM_ALLOCATOR +} Memory_Mode; + +static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN; + +static mem_allocator_t pool_allocator = NULL; + +static enlarge_memory_error_callback_t enlarge_memory_error_cb; +static void *enlarge_memory_error_user_data; + +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 +static void *allocator_user_data = NULL; +#endif + +static void *(*malloc_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + unsigned int size) = NULL; + +static void *(*realloc_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, bool full_size_mmaped, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr, unsigned int size) = NULL; + +static void (*free_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr) = NULL; + +static unsigned int global_pool_size; + +static uint64 +align_as_and_cast(uint64 size, uint64 alignment) +{ + uint64 aligned_size = (size + alignment - 1) & ~(alignment - 1); + + return aligned_size; +} + +static bool +wasm_memory_init_with_pool(void *mem, unsigned int bytes) +{ + mem_allocator_t allocator = mem_allocator_create(mem, bytes); + + if (allocator) { + memory_mode = MEMORY_MODE_POOL; + pool_allocator = allocator; + global_pool_size = bytes; + return true; + } + LOG_ERROR("Init memory with pool (%p, %u) failed.\n", mem, bytes); + return false; +} + +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 +static bool +wasm_memory_init_with_allocator(void *_user_data, void *_malloc_func, + void *_realloc_func, void *_free_func) +{ + if (_malloc_func && _free_func && _malloc_func != _free_func) { + memory_mode = MEMORY_MODE_ALLOCATOR; + allocator_user_data = _user_data; + malloc_func = _malloc_func; + realloc_func = _realloc_func; + free_func = _free_func; + return true; + } + LOG_ERROR("Init memory with allocator (%p, %p, %p, %p) failed.\n", + _user_data, _malloc_func, _realloc_func, _free_func); + return false; +} +#else +static bool +wasm_memory_init_with_allocator(void *malloc_func_ptr, void *realloc_func_ptr, + void *free_func_ptr) +{ + if (malloc_func_ptr && free_func_ptr && malloc_func_ptr != free_func_ptr) { + memory_mode = MEMORY_MODE_ALLOCATOR; + malloc_func = malloc_func_ptr; + realloc_func = realloc_func_ptr; + free_func = free_func_ptr; + return true; + } + LOG_ERROR("Init memory with allocator (%p, %p, %p) failed.\n", + malloc_func_ptr, realloc_func_ptr, free_func_ptr); + return false; +} +#endif + +static inline bool +is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + if (!module_inst) { + return true; + } + + return wasm_runtime_is_bounds_checks_enabled(module_inst); +#else + return true; +#endif +} + +bool +wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, + const MemAllocOption *alloc_option) +{ + if (mem_alloc_type == Alloc_With_Pool) { + return wasm_memory_init_with_pool(alloc_option->pool.heap_buf, + alloc_option->pool.heap_size); + } + else if (mem_alloc_type == Alloc_With_Allocator) { + return wasm_memory_init_with_allocator( +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + alloc_option->allocator.user_data, +#endif + alloc_option->allocator.malloc_func, + alloc_option->allocator.realloc_func, + alloc_option->allocator.free_func); + } + else if (mem_alloc_type == Alloc_With_System_Allocator) { + memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; + return true; + } + else { + return false; + } +} + +void +wasm_runtime_memory_destroy() +{ + if (memory_mode == MEMORY_MODE_POOL) { +#if BH_ENABLE_GC_VERIFY == 0 + (void)mem_allocator_destroy(pool_allocator); +#else + int ret = mem_allocator_destroy(pool_allocator); + if (ret != 0) { + /* Memory leak detected */ + exit(-1); + } +#endif + } + memory_mode = MEMORY_MODE_UNKNOWN; +} + +unsigned +wasm_runtime_memory_pool_size() +{ + if (memory_mode == MEMORY_MODE_POOL) + return global_pool_size; + else + return UINT32_MAX; +} + +static inline void * +wasm_runtime_malloc_internal(unsigned int size) +{ + if (memory_mode == MEMORY_MODE_UNKNOWN) { + LOG_WARNING( + "wasm_runtime_malloc failed: memory hasn't been initialize.\n"); + return NULL; + } + else if (memory_mode == MEMORY_MODE_POOL) { + return mem_allocator_malloc(pool_allocator, size); + } + else if (memory_mode == MEMORY_MODE_ALLOCATOR) { + return malloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + size); + } + else { + return os_malloc(size); + } +} + +static inline void * +wasm_runtime_realloc_internal(void *ptr, unsigned int size) +{ + if (memory_mode == MEMORY_MODE_UNKNOWN) { + LOG_WARNING( + "wasm_runtime_realloc failed: memory hasn't been initialize.\n"); + return NULL; + } + else if (memory_mode == MEMORY_MODE_POOL) { + return mem_allocator_realloc(pool_allocator, ptr, size); + } + else if (memory_mode == MEMORY_MODE_ALLOCATOR) { + if (realloc_func) + return realloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, false, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + ptr, size); + else + return NULL; + } + else { + return os_realloc(ptr, size); + } +} + +static inline void +wasm_runtime_free_internal(void *ptr) +{ + if (!ptr) { + LOG_WARNING("warning: wasm_runtime_free with NULL pointer\n"); +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif + return; + } + + if (memory_mode == MEMORY_MODE_UNKNOWN) { + LOG_WARNING("warning: wasm_runtime_free failed: " + "memory hasn't been initialize.\n"); + } + else if (memory_mode == MEMORY_MODE_POOL) { + mem_allocator_free(pool_allocator, ptr); + } + else if (memory_mode == MEMORY_MODE_ALLOCATOR) { + free_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + ptr); + } + else { + os_free(ptr); + } +} + +void * +wasm_runtime_malloc(unsigned int size) +{ + if (size == 0) { + LOG_WARNING("warning: wasm_runtime_malloc with size zero\n"); + /* At lease alloc 1 byte to avoid malloc failed */ + size = 1; +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif + } + + return wasm_runtime_malloc_internal(size); +} + +void * +wasm_runtime_realloc(void *ptr, unsigned int size) +{ + return wasm_runtime_realloc_internal(ptr, size); +} + +void +wasm_runtime_free(void *ptr) +{ + wasm_runtime_free_internal(ptr); +} + +bool +wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info) +{ + if (memory_mode == MEMORY_MODE_POOL) { + return mem_allocator_get_alloc_info(pool_allocator, mem_alloc_info); + } + return false; +} + +bool +wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_offset, uint64 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!is_bounds_checks_enabled(module_inst_comm)) { + return true; + } + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif + /* boundary overflow check */ + if (size > max_linear_memory_size + || app_offset > max_linear_memory_size - size) { + goto fail; + } + + SHARED_MEMORY_LOCK(memory_inst); + + if (app_offset + size <= memory_inst->memory_data_size) { + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_str_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; + char *str, *str_end; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!is_bounds_checks_enabled(module_inst_comm)) { + return true; + } + + if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL, + &app_end_offset)) + goto fail; + +#if WASM_ENABLE_MEMORY64 != 0 + if (module_inst->memories[0]->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif + /* boundary overflow check, max start offset can only be size - 1, while end + * offset can be size */ + if (app_str_offset >= max_linear_memory_size + || app_end_offset > max_linear_memory_size) + goto fail; + + str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); + str_end = str + (app_end_offset - app_str_offset); + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + + return true; +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr, uint64 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!is_bounds_checks_enabled(module_inst_comm)) { + return true; + } + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif + /* boundary overflow check */ + if (size > max_linear_memory_size || (uintptr_t)addr > UINTPTR_MAX - size) { + goto fail; + } + + SHARED_MEMORY_LOCK(memory_inst); + + if (memory_inst->memory_data <= addr + && addr + size <= memory_inst->memory_data_end) { + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +void * +wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr; + bool bounds_checks; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + bounds_checks = is_bounds_checks_enabled(module_inst_comm); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return NULL; + } + + SHARED_MEMORY_LOCK(memory_inst); + + addr = memory_inst->memory_data + (uintptr_t)app_offset; + + if (bounds_checks) { + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + SHARED_MEMORY_UNLOCK(memory_inst); + return addr; + } + SHARED_MEMORY_UNLOCK(memory_inst); + return NULL; + } + + /* If bounds checks is disabled, return the address directly */ + SHARED_MEMORY_UNLOCK(memory_inst); + return addr; +} + +uint64 +wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + bool bounds_checks; + uint64 ret; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + bounds_checks = is_bounds_checks_enabled(module_inst_comm); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return 0; + } + + SHARED_MEMORY_LOCK(memory_inst); + + if (bounds_checks) { + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + ret = (uint64)(addr - memory_inst->memory_data); + SHARED_MEMORY_UNLOCK(memory_inst); + return ret; + } + } + /* If bounds checks is disabled, return the offset directly */ + else if (addr != NULL) { + ret = (uint64)(addr - memory_inst->memory_data); + SHARED_MEMORY_UNLOCK(memory_inst); + return ret; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + return 0; +} + +bool +wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_offset, uint64 *p_app_start_offset, + uint64 *p_app_end_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint64 memory_data_size; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + SHARED_MEMORY_LOCK(memory_inst); + + memory_data_size = memory_inst->memory_data_size; + + if (app_offset < memory_data_size) { + if (p_app_start_offset) + *p_app_start_offset = 0; + if (p_app_end_offset) + *p_app_end_offset = memory_data_size; + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + return false; +} + +bool +wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint8 *native_ptr, + uint8 **p_native_start_addr, + uint8 **p_native_end_addr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + SHARED_MEMORY_LOCK(memory_inst); + + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + if (p_native_start_addr) + *p_native_start_addr = memory_inst->memory_data; + if (p_native_end_addr) + *p_native_end_addr = memory_inst->memory_data_end; + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + return false; +} + +bool +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr) +{ + WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); + uint8 *native_addr; + bool bounds_checks; + + bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); + + if (!memory_inst) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + + bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst); + + if (!bounds_checks) { + if (app_buf_addr == 0) { + native_addr = NULL; + } + goto success; + } + + /* No need to check the app_offset and buf_size if memory access + boundary check with hardware trap is enabled */ +#ifndef OS_ENABLE_HW_BOUND_CHECK + SHARED_MEMORY_LOCK(memory_inst); + + if (app_buf_addr >= memory_inst->memory_data_size) { + goto fail; + } + + if (!is_str) { + if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { + goto fail; + } + } + else { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)memory_inst->memory_data_end; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + } + + SHARED_MEMORY_UNLOCK(memory_inst); +#endif + +success: + *p_native_addr = (void *)native_addr; + return true; + +#ifndef OS_ENABLE_HW_BOUND_CHECK +fail: + SHARED_MEMORY_UNLOCK(memory_inst); + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +#endif +} + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst) +{ + if (module_inst->memories) + return module_inst->memories[0]; + else + return NULL; +} + +void +wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory, + uint64 memory_data_size) +{ +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 || WASM_ENABLE_AOT != 0 +#if UINTPTR_MAX == UINT64_MAX + memory->mem_bound_check_1byte.u64 = memory_data_size - 1; + memory->mem_bound_check_2bytes.u64 = memory_data_size - 2; + memory->mem_bound_check_4bytes.u64 = memory_data_size - 4; + memory->mem_bound_check_8bytes.u64 = memory_data_size - 8; + memory->mem_bound_check_16bytes.u64 = memory_data_size - 16; +#else + memory->mem_bound_check_1byte.u32[0] = (uint32)memory_data_size - 1; + memory->mem_bound_check_2bytes.u32[0] = (uint32)memory_data_size - 2; + memory->mem_bound_check_4bytes.u32[0] = (uint32)memory_data_size - 4; + memory->mem_bound_check_8bytes.u32[0] = (uint32)memory_data_size - 8; + memory->mem_bound_check_16bytes.u32[0] = (uint32)memory_data_size - 16; +#endif +#endif +} + +static void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size) +{ +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(mapped_mem, commit_size); +#else + (void)commit_size; +#endif + os_munmap(mapped_mem, map_size); +} + +static void * +wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size, + uint64 commit_size) +{ + void *new_mem; + + bh_assert(new_size > 0); + bh_assert(new_size > old_size); + + if (mapped_mem) { + new_mem = os_mremap(mapped_mem, old_size, new_size); + } + else { + new_mem = os_mmap(NULL, new_size, MMAP_PROT_NONE, MMAP_MAP_NONE, + os_get_invalid_handle()); + } + if (!new_mem) { + return NULL; + } + +#ifdef BH_PLATFORM_WINDOWS + if (commit_size > 0 + && !os_mem_commit(new_mem, commit_size, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + os_munmap(new_mem, new_size); + return NULL; + } +#endif + + if (os_mprotect(new_mem, commit_size, MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { + wasm_munmap_linear_memory(new_mem, new_size, new_size); + return NULL; + } + + return new_mem; +} + +static void * +wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size) +{ + return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size); +} + +bool +wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); + uint8 *memory_data_old, *memory_data_new, *heap_data_old; + uint32 num_bytes_per_page, heap_size; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_old = 0, total_size_new; + bool ret = true, full_size_mmaped; + enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR; + + if (!memory) { + ret = false; + goto return_func; + } + +#ifdef OS_ENABLE_HW_BOUND_CHECK + full_size_mmaped = true; +#elif WASM_ENABLE_SHARED_MEMORY != 0 + full_size_mmaped = shared_memory_is_shared(memory); +#else + full_size_mmaped = false; +#endif + + memory_data_old = memory->memory_data; + total_size_old = memory->memory_data_size; + + heap_data_old = memory->heap_data; + heap_size = (uint32)(memory->heap_data_end - memory->heap_data); + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; + + if (inc_page_count <= 0) + /* No need to enlarge memory */ + return true; + + if (total_page_count < cur_page_count) { /* integer overflow */ + ret = false; + goto return_func; + } + + if (total_page_count > max_page_count) { + failure_reason = MAX_SIZE_REACHED; + ret = false; + goto return_func; + } + + bh_assert(total_size_new + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); + +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + if (!(memory_data_new = + realloc_func(Alloc_For_LinearMemory, full_size_mmaped, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + NULL, +#endif + memory_data_old, total_size_new))) { + ret = false; + goto return_func; + } + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + ret = false; + } + } + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data = memory_data_new; +#else + if (full_size_mmaped) { +#ifdef BH_PLATFORM_WINDOWS + if (!os_mem_commit(memory->memory_data_end, + (mem_offset_t)(total_size_new - total_size_old), + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + ret = false; + goto return_func; + } +#endif + + if (os_mprotect(memory->memory_data_end, + (mem_offset_t)(total_size_new - total_size_old), + MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(memory->memory_data_end, + (mem_offset_t)(total_size_new - total_size_old)); +#endif + ret = false; + goto return_func; + } + } + else { + if (heap_size > 0) { + if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + ret = false; + goto return_func; + } + } + + if (!(memory_data_new = + wasm_mremap_linear_memory(memory_data_old, total_size_old, + total_size_new, total_size_new))) { + ret = false; + goto return_func; + } + + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + /* Don't return here as memory->memory_data is obsolete and + must be updated to be correctly used later. */ + ret = false; + } + } + + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data = memory_data_new; +#if defined(os_writegsbase) + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_data_new); +#endif + } +#endif /* end of WASM_MEM_ALLOC_WITH_USAGE */ + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + SET_LINEAR_MEMORY_SIZE(memory, total_size_new); + memory->memory_data_end = memory->memory_data + total_size_new; + + wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new); + +return_func: + if (!ret && enlarge_memory_error_cb) { + WASMExecEnv *exec_env = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + exec_env = ((WASMModuleInstance *)module)->cur_exec_env; +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + exec_env = ((AOTModuleInstance *)module)->cur_exec_env; +#endif + + enlarge_memory_error_cb(inc_page_count, total_size_old, 0, + failure_reason, + (WASMModuleInstanceCommon *)module, exec_env, + enlarge_memory_error_user_data); + } + + return ret; +} + +void +wasm_runtime_set_enlarge_mem_error_callback( + const enlarge_memory_error_callback_t callback, void *user_data) +{ + enlarge_memory_error_cb = callback; + enlarge_memory_error_user_data = user_data; +} + +bool +wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) +{ + bool ret = false; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (module->memory_count > 0) + shared_memory_lock(module->memories[0]); +#endif + ret = wasm_enlarge_memory_internal(module, inc_page_count); +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (module->memory_count > 0) + shared_memory_unlock(module->memories[0]); +#endif + + return ret; +} + +void +wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst) +{ + uint64 map_size; + + bh_assert(memory_inst); + bh_assert(memory_inst->memory_data); + +#ifndef OS_ENABLE_HW_BOUND_CHECK +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (shared_memory_is_shared(memory_inst)) { + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->max_page_count; + } + else +#endif + { + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->cur_page_count; + } +#else + map_size = 8 * (uint64)BH_GB; +#endif + +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + (void)map_size; + free_func(Alloc_For_LinearMemory, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + NULL, +#endif + memory_inst->memory_data); +#else + wasm_munmap_linear_memory(memory_inst->memory_data, + memory_inst->memory_data_size, map_size); +#endif + + memory_inst->memory_data = NULL; +} + +int +wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, + bool is_memory64, uint64 num_bytes_per_page, + uint64 init_page_count, uint64 max_page_count, + uint64 *memory_data_size) +{ + uint64 map_size, page_size; + + bh_assert(data); + bh_assert(memory_data_size); + +#ifndef OS_ENABLE_HW_BOUND_CHECK +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + /* Allocate maximum memory size when memory is shared */ + map_size = max_page_count * num_bytes_per_page; + } + else +#endif + { + map_size = init_page_count * num_bytes_per_page; + } +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + + page_size = os_getpagesize(); + *memory_data_size = init_page_count * num_bytes_per_page; + +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + bh_assert(*memory_data_size <= MAX_LINEAR_MEM64_MEMORY_SIZE); + } + else +#endif + { + bh_assert(*memory_data_size <= MAX_LINEAR_MEMORY_SIZE); + } + *memory_data_size = align_as_and_cast(*memory_data_size, page_size); + + if (map_size > 0) { +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + (void)wasm_mmap_linear_memory; + if (!(*data = malloc_func(Alloc_For_LinearMemory, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + NULL, +#endif + *memory_data_size))) { + return BHT_ERROR; + } +#else + if (!(*data = wasm_mmap_linear_memory(map_size, *memory_data_size))) { + return BHT_ERROR; + } +#endif + } + + return BHT_OK; +} diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_memory.h b/wasm-micro-runtime/core/iwasm/common/wasm_memory.h new file mode 100644 index 0000000..a5dfefa --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_memory.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_MEMORY_H +#define _WASM_MEMORY_H + +#include "bh_common.h" +#include "../include/wasm_export.h" +#include "../interpreter/wasm_runtime.h" +#include "../common/wasm_shared_memory.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 && BH_ATOMIC_64_IS_ATOMIC != 0 +#define GET_LINEAR_MEMORY_SIZE(memory) \ + BH_ATOMIC_64_LOAD(memory->memory_data_size) +#define SET_LINEAR_MEMORY_SIZE(memory, size) \ + BH_ATOMIC_64_STORE(memory->memory_data_size, size) +#elif WASM_ENABLE_SHARED_MEMORY != 0 +static inline uint64 +GET_LINEAR_MEMORY_SIZE(const WASMMemoryInstance *memory) +{ + SHARED_MEMORY_LOCK(memory); + uint64 memory_data_size = BH_ATOMIC_64_LOAD(memory->memory_data_size); + SHARED_MEMORY_UNLOCK(memory); + return memory_data_size; +} +static inline void +SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) +{ + SHARED_MEMORY_LOCK(memory); + BH_ATOMIC_64_STORE(memory->memory_data_size, size); + SHARED_MEMORY_UNLOCK(memory); +} +#else +#define GET_LINEAR_MEMORY_SIZE(memory) memory->memory_data_size +#define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size +#endif + +bool +wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, + const MemAllocOption *alloc_option); + +void +wasm_runtime_memory_destroy(); + +unsigned +wasm_runtime_memory_pool_size(); + +void +wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory, + uint64 memory_data_size); + +void +wasm_runtime_set_enlarge_mem_error_callback( + const enlarge_memory_error_callback_t callback, void *user_data); + +void +wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst); + +int +wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, + bool is_memory64, uint64 num_bytes_per_page, + uint64 init_page_count, uint64 max_page_count, + uint64 *memory_data_size); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_MEMORY_H */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_native.c b/wasm-micro-runtime/core/iwasm/common/wasm_native.c new file mode 100644 index 0000000..394dfd2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_native.c @@ -0,0 +1,1507 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_native.h" +#include "wasm_runtime_common.h" +#include "bh_log.h" +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif + +static NativeSymbolsList g_native_symbols_list = NULL; + +#if WASM_ENABLE_LIBC_WASI != 0 +static void *g_wasi_context_key; +#endif /* WASM_ENABLE_LIBC_WASI */ + +uint32 +get_libc_builtin_export_apis(NativeSymbol **p_libc_builtin_apis); + +#if WASM_ENABLE_SPEC_TEST != 0 +uint32 +get_spectest_export_apis(NativeSymbol **p_libc_builtin_apis); +#endif + +uint32 +get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis); + +uint32_t +get_wasi_nn_export_apis(NativeSymbol **p_libc_wasi_apis); + +uint32 +get_base_lib_export_apis(NativeSymbol **p_base_lib_apis); + +uint32 +get_ext_lib_export_apis(NativeSymbol **p_ext_lib_apis); + +#if WASM_ENABLE_LIB_PTHREAD != 0 +bool +lib_pthread_init(); + +void +lib_pthread_destroy(); + +uint32 +get_lib_pthread_export_apis(NativeSymbol **p_lib_pthread_apis); +#endif + +#if WASM_ENABLE_LIB_WASI_THREADS != 0 +bool +lib_wasi_threads_init(void); + +void +lib_wasi_threads_destroy(void); + +uint32 +get_lib_wasi_threads_export_apis(NativeSymbol **p_lib_wasi_threads_apis); +#endif + +uint32 +get_libc_emcc_export_apis(NativeSymbol **p_libc_emcc_apis); + +uint32 +get_lib_rats_export_apis(NativeSymbol **p_lib_rats_apis); + +static bool +compare_type_with_signautre(uint8 type, const char signature) +{ + const char num_sig_map[] = { 'F', 'f', 'I', 'i' }; + + if (VALUE_TYPE_F64 <= type && type <= VALUE_TYPE_I32 + && signature == num_sig_map[type - VALUE_TYPE_F64]) { + return true; + } + +#if WASM_ENABLE_REF_TYPES != 0 + if ('r' == signature +#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_STRINGREF != 0 + && (type >= REF_TYPE_STRINGVIEWITER && type <= REF_TYPE_NULLFUNCREF) +#else + && (type >= REF_TYPE_HT_NULLABLE && type <= REF_TYPE_NULLFUNCREF) +#endif +#else + && type == VALUE_TYPE_EXTERNREF +#endif + ) + return true; +#endif + + /* TODO: a v128 parameter */ + return false; +} + +static bool +check_symbol_signature(const WASMFuncType *type, const char *signature) +{ + const char *p = signature, *p_end; + char sig; + uint32 i = 0; + + if (!p || strlen(p) < 2) + return false; + + p_end = p + strlen(signature); + + if (*p++ != '(') + return false; + + if ((uint32)(p_end - p) < (uint32)(type->param_count + 1)) + /* signatures of parameters, and ')' */ + return false; + + for (i = 0; i < type->param_count; i++) { + sig = *p++; + + /* a f64/f32/i64/i32/externref parameter */ + if (compare_type_with_signautre(type->types[i], sig)) + continue; + + /* a pointer/string paramter */ + if (type->types[i] != VALUE_TYPE_I32) + /* pointer and string must be i32 type */ + return false; + + if (sig == '*') { + /* it is a pointer */ + if (i + 1 < type->param_count + && type->types[i + 1] == VALUE_TYPE_I32 && *p == '~') { + /* pointer length followed */ + i++; + p++; + } + } + else if (sig == '$') { + /* it is a string */ + } + else { + /* invalid signature */ + return false; + } + } + + if (*p++ != ')') + return false; + + if (type->result_count) { + if (p >= p_end) + return false; + + /* result types includes: f64,f32,i64,i32,externref */ + if (!compare_type_with_signautre(type->types[i], *p)) + return false; + + p++; + } + + if (*p != '\0') + return false; + + return true; +} + +static int +native_symbol_cmp(const void *native_symbol1, const void *native_symbol2) +{ + return strcmp(((const NativeSymbol *)native_symbol1)->symbol, + ((const NativeSymbol *)native_symbol2)->symbol); +} + +static void * +lookup_symbol(NativeSymbol *native_symbols, uint32 n_native_symbols, + const char *symbol, const char **p_signature, void **p_attachment) +{ + NativeSymbol *native_symbol, key = { 0 }; + + key.symbol = symbol; + + if ((native_symbol = bsearch(&key, native_symbols, n_native_symbols, + sizeof(NativeSymbol), native_symbol_cmp))) { + *p_signature = native_symbol->signature; + *p_attachment = native_symbol->attachment; + return native_symbol->func_ptr; + } + + return NULL; +} + +/** + * allow func_type and all outputs, like p_signature, p_attachment and + * p_call_conv_raw to be NULL + */ +void * +wasm_native_resolve_symbol(const char *module_name, const char *field_name, + const WASMFuncType *func_type, + const char **p_signature, void **p_attachment, + bool *p_call_conv_raw) +{ + NativeSymbolsNode *node, *node_next; + const char *signature = NULL; + void *func_ptr = NULL, *attachment = NULL; + + node = g_native_symbols_list; + while (node) { + node_next = node->next; + if (!strcmp(node->module_name, module_name)) { + if ((func_ptr = + lookup_symbol(node->native_symbols, node->n_native_symbols, + field_name, &signature, &attachment)) + || (field_name[0] == '_' + && (func_ptr = lookup_symbol( + node->native_symbols, node->n_native_symbols, + field_name + 1, &signature, &attachment)))) + break; + } + node = node_next; + } + + if (!p_signature || !p_attachment || !p_call_conv_raw) + return func_ptr; + + if (func_ptr) { + if (signature && signature[0] != '\0') { + /* signature is not empty, check its format */ + if (!func_type || !check_symbol_signature(func_type, signature)) { +#if WASM_ENABLE_WAMR_COMPILER == 0 + /* Output warning except running aot compiler */ + LOG_WARNING("failed to check signature '%s' and resolve " + "pointer params for import function (%s %s)\n", + signature, module_name, field_name); +#endif + return NULL; + } + else + /* Save signature for runtime to do pointer check and + address conversion */ + *p_signature = signature; + } + else + /* signature is empty */ + *p_signature = NULL; + + *p_attachment = attachment; + *p_call_conv_raw = node->call_conv_raw; + } + + return func_ptr; +} + +static bool +register_natives(const char *module_name, NativeSymbol *native_symbols, + uint32 n_native_symbols, bool call_conv_raw) +{ + NativeSymbolsNode *node; + + if (!(node = wasm_runtime_malloc(sizeof(NativeSymbolsNode)))) + return false; +#if WASM_ENABLE_MEMORY_TRACING != 0 + os_printf("Register native, size: %u\n", sizeof(NativeSymbolsNode)); +#endif + + node->module_name = module_name; + node->native_symbols = native_symbols; + node->n_native_symbols = n_native_symbols; + node->call_conv_raw = call_conv_raw; + + /* Add to list head */ + node->next = g_native_symbols_list; + g_native_symbols_list = node; + + qsort(native_symbols, n_native_symbols, sizeof(NativeSymbol), + native_symbol_cmp); + + return true; +} + +bool +wasm_native_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols) +{ + return register_natives(module_name, native_symbols, n_native_symbols, + false); +} + +bool +wasm_native_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols) +{ + return register_natives(module_name, native_symbols, n_native_symbols, + true); +} + +bool +wasm_native_unregister_natives(const char *module_name, + NativeSymbol *native_symbols) +{ + NativeSymbolsNode **prevp; + NativeSymbolsNode *node; + + prevp = &g_native_symbols_list; + while ((node = *prevp) != NULL) { + if (node->native_symbols == native_symbols + && !strcmp(node->module_name, module_name)) { + *prevp = node->next; + wasm_runtime_free(node); + return true; + } + prevp = &node->next; + } + return false; +} + +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 +static uint32 +context_key_to_idx(void *key) +{ + bh_assert(key != NULL); + uint32 idx = (uint32)(uintptr_t)key; + bh_assert(idx > 0); + bh_assert(idx <= WASM_MAX_INSTANCE_CONTEXTS); + return idx - 1; +} + +static void * +context_idx_to_key(uint32 idx) +{ + bh_assert(idx < WASM_MAX_INSTANCE_CONTEXTS); + return (void *)(uintptr_t)(idx + 1); +} + +typedef void (*dtor_t)(WASMModuleInstanceCommon *, void *); +static dtor_t g_context_dtors[WASM_MAX_INSTANCE_CONTEXTS]; + +static void +dtor_noop(WASMModuleInstanceCommon *inst, void *ctx) +{} + +void * +wasm_native_create_context_key(void (*dtor)(WASMModuleInstanceCommon *inst, + void *ctx)) +{ + uint32 i; + for (i = 0; i < WASM_MAX_INSTANCE_CONTEXTS; i++) { + if (g_context_dtors[i] == NULL) { + if (dtor == NULL) { + dtor = dtor_noop; + } + g_context_dtors[i] = dtor; + return context_idx_to_key(i); + } + } + LOG_ERROR("failed to allocate instance context key"); + return NULL; +} + +void +wasm_native_destroy_context_key(void *key) +{ + uint32 idx = context_key_to_idx(key); + bh_assert(g_context_dtors[idx] != NULL); + g_context_dtors[idx] = NULL; +} + +static WASMModuleInstanceExtraCommon * +wasm_module_inst_extra_common(WASMModuleInstanceCommon *inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (inst->module_type == Wasm_Module_Bytecode) { + return &((WASMModuleInstance *)inst)->e->common; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (inst->module_type == Wasm_Module_AoT) { + return &((AOTModuleInstanceExtra *)((AOTModuleInstance *)inst)->e) + ->common; + } +#endif + bh_assert(false); + return NULL; +} + +void +wasm_native_set_context(WASMModuleInstanceCommon *inst, void *key, void *ctx) +{ + uint32 idx = context_key_to_idx(key); + WASMModuleInstanceExtraCommon *common = wasm_module_inst_extra_common(inst); + common->contexts[idx] = ctx; +} + +void +wasm_native_set_context_spread(WASMModuleInstanceCommon *inst, void *key, + void *ctx) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_set_context(inst, key, ctx); +#else + wasm_native_set_context(inst, key, ctx); +#endif +} + +void * +wasm_native_get_context(WASMModuleInstanceCommon *inst, void *key) +{ + uint32 idx = context_key_to_idx(key); + WASMModuleInstanceExtraCommon *common = wasm_module_inst_extra_common(inst); + return common->contexts[idx]; +} + +void +wasm_native_call_context_dtors(WASMModuleInstanceCommon *inst) +{ + WASMModuleInstanceExtraCommon *common = wasm_module_inst_extra_common(inst); + uint32 i; + for (i = 0; i < WASM_MAX_INSTANCE_CONTEXTS; i++) { + dtor_t dtor = g_context_dtors[i]; + if (dtor != NULL) { + dtor(inst, common->contexts[i]); + } + } +} + +void +wasm_native_inherit_contexts(WASMModuleInstanceCommon *child, + WASMModuleInstanceCommon *parent) +{ + WASMModuleInstanceExtraCommon *parent_common = + wasm_module_inst_extra_common(parent); + WASMModuleInstanceExtraCommon *child_common = + wasm_module_inst_extra_common(child); + bh_memcpy_s(child_common->contexts, + sizeof(*child_common->contexts) * WASM_MAX_INSTANCE_CONTEXTS, + parent_common->contexts, + sizeof(*parent_common->contexts) * WASM_MAX_INSTANCE_CONTEXTS); +} +#endif /* WASM_ENABLE_MODULE_INST_CONTEXT != 0 */ + +#if WASM_ENABLE_LIBC_WASI != 0 +WASIContext * +wasm_runtime_get_wasi_ctx(WASMModuleInstanceCommon *module_inst_comm) +{ + return wasm_native_get_context(module_inst_comm, g_wasi_context_key); +} + +void +wasm_runtime_set_wasi_ctx(WASMModuleInstanceCommon *module_inst_comm, + WASIContext *wasi_ctx) +{ + wasm_native_set_context(module_inst_comm, g_wasi_context_key, wasi_ctx); +} + +static void +wasi_context_dtor(WASMModuleInstanceCommon *inst, void *ctx) +{ + if (ctx == NULL) { + return; + } + wasm_runtime_destroy_wasi(inst); +} +#endif /* end of WASM_ENABLE_LIBC_WASI */ + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +static bool +quick_aot_entry_init(); +#endif + +bool +wasm_native_init() +{ +#if WASM_ENABLE_SPEC_TEST != 0 || WASM_ENABLE_LIBC_BUILTIN != 0 \ + || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0 \ + || WASM_ENABLE_LIB_RATS != 0 || WASM_ENABLE_WASI_NN != 0 \ + || WASM_ENABLE_APP_FRAMEWORK != 0 || WASM_ENABLE_LIBC_WASI != 0 \ + || WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + NativeSymbol *native_symbols; + uint32 n_native_symbols; +#endif + +#if WASM_ENABLE_LIBC_BUILTIN != 0 + n_native_symbols = get_libc_builtin_export_apis(&native_symbols); + if (!wasm_native_register_natives("env", native_symbols, n_native_symbols)) + goto fail; +#endif /* WASM_ENABLE_LIBC_BUILTIN */ + +#if WASM_ENABLE_SPEC_TEST + n_native_symbols = get_spectest_export_apis(&native_symbols); + if (!wasm_native_register_natives("spectest", native_symbols, + n_native_symbols)) + goto fail; +#endif /* WASM_ENABLE_SPEC_TEST */ + +#if WASM_ENABLE_LIBC_WASI != 0 + g_wasi_context_key = wasm_native_create_context_key(wasi_context_dtor); + if (g_wasi_context_key == NULL) { + goto fail; + } + n_native_symbols = get_libc_wasi_export_apis(&native_symbols); + if (!wasm_native_register_natives("wasi_unstable", native_symbols, + n_native_symbols)) + goto fail; + if (!wasm_native_register_natives("wasi_snapshot_preview1", native_symbols, + n_native_symbols)) + goto fail; +#endif + +#if WASM_ENABLE_BASE_LIB != 0 + n_native_symbols = get_base_lib_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif + +#if WASM_ENABLE_APP_FRAMEWORK != 0 + n_native_symbols = get_ext_lib_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif + +#if WASM_ENABLE_LIB_PTHREAD != 0 + if (!lib_pthread_init()) + goto fail; + + n_native_symbols = get_lib_pthread_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif + +#if WASM_ENABLE_LIB_WASI_THREADS != 0 + if (!lib_wasi_threads_init()) + goto fail; + + n_native_symbols = get_lib_wasi_threads_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("wasi", native_symbols, + n_native_symbols)) + goto fail; +#endif + +#if WASM_ENABLE_LIBC_EMCC != 0 + n_native_symbols = get_libc_emcc_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif /* WASM_ENABLE_LIBC_EMCC */ + +#if WASM_ENABLE_LIB_RATS != 0 + n_native_symbols = get_lib_rats_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif /* WASM_ENABLE_LIB_RATS */ + +#if WASM_ENABLE_WASI_NN != 0 + n_native_symbols = get_wasi_nn_export_apis(&native_symbols); +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +#define wasi_nn_module_name "wasi_ephemeral_nn" +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +#define wasi_nn_module_name "wasi_nn" +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + if (!wasm_native_register_natives(wasi_nn_module_name, native_symbols, + n_native_symbols)) + goto fail; +#endif + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + if (!quick_aot_entry_init()) { +#if WASM_ENABLE_SPEC_TEST != 0 || WASM_ENABLE_LIBC_BUILTIN != 0 \ + || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0 \ + || WASM_ENABLE_LIB_RATS != 0 || WASM_ENABLE_WASI_NN != 0 \ + || WASM_ENABLE_APP_FRAMEWORK != 0 || WASM_ENABLE_LIBC_WASI != 0 \ + || WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + goto fail; +#else + return false; +#endif + } +#endif + + return true; +#if WASM_ENABLE_SPEC_TEST != 0 || WASM_ENABLE_LIBC_BUILTIN != 0 \ + || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0 \ + || WASM_ENABLE_LIB_RATS != 0 || WASM_ENABLE_WASI_NN != 0 \ + || WASM_ENABLE_APP_FRAMEWORK != 0 || WASM_ENABLE_LIBC_WASI != 0 \ + || WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 +fail: + wasm_native_destroy(); + return false; +#endif +} + +void +wasm_native_destroy() +{ + NativeSymbolsNode *node, *node_next; + +#if WASM_ENABLE_LIBC_WASI != 0 + if (g_wasi_context_key != NULL) { + wasm_native_destroy_context_key(g_wasi_context_key); + g_wasi_context_key = NULL; + } +#endif +#if WASM_ENABLE_LIB_PTHREAD != 0 + lib_pthread_destroy(); +#endif + +#if WASM_ENABLE_LIB_WASI_THREADS != 0 + lib_wasi_threads_destroy(); +#endif + + node = g_native_symbols_list; + while (node) { + node_next = node->next; + wasm_runtime_free(node); + node = node_next; + } + + g_native_symbols_list = NULL; +} + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +static void +invoke_no_args_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *) = func_ptr; + native_code(exec_env); +} +static void +invoke_no_args_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *) = func_ptr; + argv_ret[0] = native_code(exec_env); +} +static void +invoke_no_args_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *) = func_ptr; + int64 ret = native_code(exec_env); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_i_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32) = func_ptr; + native_code(exec_env, argv[0]); +} +static void +invoke_i_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0]); +} +static void +invoke_i_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32) = func_ptr; + int64 ret = native_code(exec_env, argv[0]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_I_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv)); +} +static void +invoke_I_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv)); +} +static void +invoke_I_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_ii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32) = func_ptr; + native_code(exec_env, argv[0], argv[1]); +} +static void +invoke_ii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1]); +} +static void +invoke_ii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32) = func_ptr; + int64 ret = native_code(exec_env, argv[0], argv[1]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iI_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int64) = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1)); +} +static void +invoke_iI_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int64) = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1)); +} +static void +invoke_iI_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int64) = func_ptr; + int64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Ii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int32) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2]); +} +static void +invoke_Ii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int32) = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2]); +} +static void +invoke_Ii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int32) = func_ptr; + int64 ret = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_II_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2)); +} +static void +invoke_II_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2)); +} +static void +invoke_II_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int64) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32, int32) = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2]); +} +static void +invoke_iii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2]); +} +static void +invoke_iii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32, int32) = func_ptr; + int64 ret = native_code(exec_env, argv[0], argv[1], argv[2]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiI_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32, int64) = func_ptr; + native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2)); +} +static void +invoke_iiI_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2)); +} +static void +invoke_iiI_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32, int64) = func_ptr; + int64 ret = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIi_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int64, int32) = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + argv[3]); +} +static void +invoke_iIi_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int64, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], + GET_I64_FROM_ADDR((uint32 *)argv + 1), argv[3]); +} +static void +invoke_iIi_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int64, int32) = func_ptr; + int64 ret = native_code(exec_env, argv[0], + GET_I64_FROM_ADDR((uint32 *)argv + 1), argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iII_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int64, int64) = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3)); +} +static void +invoke_iII_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int64, int64) = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3)); +} +static void +invoke_iII_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int64, int64) = func_ptr; + int64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Iii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int32, int32) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], argv[3]); +} +static void +invoke_Iii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int32, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], argv[3]); +} +static void +invoke_Iii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int32, int32) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiI_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int32, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3)); +} +static void +invoke_IiI_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int32, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], GET_I64_FROM_ADDR((uint32 *)argv + 3)); +} +static void +invoke_IiI_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int32, int64) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], GET_I64_FROM_ADDR((uint32 *)argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIi_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int64, int32) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4]); +} +static void +invoke_IIi_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int64, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4]); +} +static void +invoke_IIi_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int64, int32) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_III_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int64, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} +static void +invoke_III_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int64, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} +static void +invoke_III_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int64, int64) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32, int32, int32) = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); +} +static void +invoke_iiii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32, int32, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); +} +static void +invoke_iiii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32, int32, int32) = func_ptr; + int64 ret = native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiiI_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32, int32, int64) = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3)); +} +static void +invoke_iiiI_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32, int32, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3)); +} +static void +invoke_iiiI_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32, int32, int64) = func_ptr; + int64 ret = native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiIi_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32, int64, int32) = func_ptr; + native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4]); +} +static void +invoke_iiIi_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32, int64, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4]); +} +static void +invoke_iiIi_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32, int64, int32) = func_ptr; + int64 ret = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiII_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32, int64, int64) = func_ptr; + native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} +static void +invoke_iiII_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32, int64, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} +static void +invoke_iiII_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32, int64, int64) = func_ptr; + int64 ret = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int64, int32, int32) = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + argv[3], argv[4]); +} +static void +invoke_iIii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int64, int32, int32) = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + argv[3], argv[4]); +} +static void +invoke_iIii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int64, int32, int32) = func_ptr; + int64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIiI_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int64, int32, int64) = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + argv[3], GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} +static void +invoke_iIiI_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int64, int32, int64) = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + argv[3], GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} +static void +invoke_iIiI_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int64, int32, int64) = func_ptr; + int64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + argv[3], GET_I64_FROM_ADDR((uint32 *)argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIIi_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int64, int64, int32) = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3), argv[5]); +} +static void +invoke_iIIi_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int64, int64, int32) = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3), argv[5]); +} +static void +invoke_iIIi_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int64, int64, int32) = func_ptr; + int64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3), argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIII_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int64, int64, int64) = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3), + GET_I64_FROM_ADDR((uint32 *)argv + 5)); +} +static void +invoke_iIII_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int64, int64, int64) = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3), + GET_I64_FROM_ADDR((uint32 *)argv + 5)); +} +static void +invoke_iIII_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int64, int64, int64) = func_ptr; + int64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR((uint32 *)argv + 1), + GET_I64_FROM_ADDR((uint32 *)argv + 3), + GET_I64_FROM_ADDR((uint32 *)argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Iiii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int32, int32, int32) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], argv[3], + argv[4]); +} +static void +invoke_Iiii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int32, int32, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], argv[3], argv[4]); +} +static void +invoke_Iiii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int32, int32, int32) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiiI_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int32, int32, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], argv[3], + GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} + +static void +invoke_IiiI_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int32, int32, int64) = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], + argv[3], GET_I64_FROM_ADDR((uint32 *)argv + 4)); +} + +static void +invoke_IiiI_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int32, int32, int64) = func_ptr; + int64 ret = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], + argv[3], GET_I64_FROM_ADDR((uint32 *)argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiIi_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int32, int64, int32) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3), argv[5]); +} +static void +invoke_IiIi_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int32, int64, int32) = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3), argv[5]); +} +static void +invoke_IiIi_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int32, int64, int32) = func_ptr; + int64 ret = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3), argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiII_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int32, int64, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), argv[2], + GET_I64_FROM_ADDR((uint32 *)argv + 3), + GET_I64_FROM_ADDR((uint32 *)argv + 5)); +} +static void +invoke_IiII_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int32, int64, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], GET_I64_FROM_ADDR((uint32 *)argv + 3), + GET_I64_FROM_ADDR((uint32 *)argv + 5)); +} +static void +invoke_IiII_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int32, int64, int64) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + argv[2], GET_I64_FROM_ADDR((uint32 *)argv + 3), + GET_I64_FROM_ADDR((uint32 *)argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int64, int32, int32) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4], argv[5]); +} +static void +invoke_IIii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int64, int32, int32) = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4], argv[5]); +} +static void +invoke_IIii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int64, int32, int32) = func_ptr; + int64 ret = + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4], argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIiI_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int64, int32, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4], + GET_I64_FROM_ADDR((uint32 *)argv + 5)); +} +static void +invoke_IIiI_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int64, int32, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4], + GET_I64_FROM_ADDR((uint32 *)argv + 5)); +} +static void +invoke_IIiI_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int64, int32, int64) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), argv[4], + GET_I64_FROM_ADDR((uint32 *)argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIIi_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int64, int64, int32) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4), argv[6]); +} +static void +invoke_IIIi_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int64, int64, int32) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4), argv[6]); +} +static void +invoke_IIIi_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int64, int64, int32) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4), argv[6]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIII_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int64, int64, int64, int64) = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4), + GET_I64_FROM_ADDR((uint32 *)argv + 6)); +} +static void +invoke_IIII_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int64, int64, int64, int64) = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4), + GET_I64_FROM_ADDR((uint32 *)argv + 6)); +} +static void +invoke_IIII_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int64, int64, int64, int64) = func_ptr; + int64 ret = native_code(exec_env, GET_I64_FROM_ADDR((uint32 *)argv), + GET_I64_FROM_ADDR((uint32 *)argv + 2), + GET_I64_FROM_ADDR((uint32 *)argv + 4), + GET_I64_FROM_ADDR((uint32 *)argv + 6)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiiii_v(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + void (*native_code)(WASMExecEnv *, int32, int32, int32, int32, int32) = + func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); +} +static void +invoke_iiiii_i(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int32 (*native_code)(WASMExecEnv *, int32, int32, int32, int32, int32) = + func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); +} +static void +invoke_iiiii_I(void *func_ptr, void *exec_env, int32 *argv, int32 *argv_ret) +{ + int64 (*native_code)(WASMExecEnv *, int32, int32, int32, int32, int32) = + func_ptr; + int64 ret = + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +typedef struct QuickAOTEntry { + const char *signature; + void *func_ptr; +} QuickAOTEntry; + +/* clang-format off */ +static QuickAOTEntry quick_aot_entries[] = { + { "()v", invoke_no_args_v }, + { "()i", invoke_no_args_i }, + { "()I", invoke_no_args_I }, + + { "(i)v", invoke_i_v }, { "(i)i", invoke_i_i }, { "(i)I", invoke_i_I }, + { "(I)v", invoke_I_v }, { "(I)i", invoke_I_i }, { "(I)I", invoke_I_I }, + + { "(ii)v", invoke_ii_v }, { "(ii)i", invoke_ii_i }, { "(ii)I", invoke_ii_I }, + { "(iI)v", invoke_iI_v }, { "(iI)i", invoke_iI_i }, { "(iI)I", invoke_iI_I }, + { "(Ii)v", invoke_Ii_v }, { "(Ii)i", invoke_Ii_i }, { "(Ii)I", invoke_Ii_I }, + { "(II)v", invoke_II_v }, { "(II)i", invoke_II_i }, { "(II)I", invoke_II_I }, + + { "(iii)v", invoke_iii_v }, { "(iii)i", invoke_iii_i }, { "(iii)I", invoke_iii_I }, + { "(iiI)v", invoke_iiI_v }, { "(iiI)i", invoke_iiI_i }, { "(iiI)I", invoke_iiI_I }, + { "(iIi)v", invoke_iIi_v }, { "(iIi)i", invoke_iIi_i }, { "(iIi)I", invoke_iIi_I }, + { "(iII)v", invoke_iII_v }, { "(iII)i", invoke_iII_i }, { "(iII)I", invoke_iII_I }, + { "(Iii)v", invoke_Iii_v }, { "(Iii)i", invoke_Iii_i }, { "(Iii)I", invoke_Iii_I }, + { "(IiI)v", invoke_IiI_v }, { "(IiI)i", invoke_IiI_i }, { "(IiI)I", invoke_IiI_I }, + { "(IIi)v", invoke_IIi_v }, { "(IIi)i", invoke_IIi_i }, { "(IIi)I", invoke_IIi_I }, + { "(III)v", invoke_III_v }, { "(III)i", invoke_III_i }, { "(III)I", invoke_III_I }, + + { "(iiii)v", invoke_iiii_v }, { "(iiii)i", invoke_iiii_i }, { "(iiii)I", invoke_iiii_I }, + { "(iiiI)v", invoke_iiiI_v }, { "(iiiI)i", invoke_iiiI_i }, { "(iiiI)I", invoke_iiiI_I }, + { "(iiIi)v", invoke_iiIi_v }, { "(iiIi)i", invoke_iiIi_i }, { "(iiIi)I", invoke_iiIi_I }, + { "(iiII)v", invoke_iiII_v }, { "(iiII)i", invoke_iiII_i }, { "(iiII)I", invoke_iiII_I }, + { "(iIii)v", invoke_iIii_v }, { "(iIii)i", invoke_iIii_i }, { "(iIii)I", invoke_iIii_I }, + { "(iIiI)v", invoke_iIiI_v }, { "(iIiI)i", invoke_iIiI_i }, { "(iIiI)I", invoke_iIiI_I }, + { "(iIIi)v", invoke_iIIi_v }, { "(iIIi)i", invoke_iIIi_i }, { "(iIIi)I", invoke_iIIi_I }, + { "(iIII)v", invoke_iIII_v }, { "(iIII)i", invoke_iIII_i }, { "(iIII)I", invoke_iIII_I }, + { "(Iiii)v", invoke_Iiii_v }, { "(Iiii)i", invoke_Iiii_i }, { "(Iiii)I", invoke_Iiii_I }, + { "(IiiI)v", invoke_IiiI_v }, { "(IiiI)i", invoke_IiiI_i }, { "(IiiI)I", invoke_IiiI_I }, + { "(IiIi)v", invoke_IiIi_v }, { "(IiIi)i", invoke_IiIi_i }, { "(IiIi)I", invoke_IiIi_I }, + { "(IiII)v", invoke_IiII_v }, { "(IiII)i", invoke_IiII_i }, { "(IiII)I", invoke_IiII_I }, + { "(IIii)v", invoke_IIii_v }, { "(IIii)i", invoke_IIii_i }, { "(IIii)I", invoke_IIii_I }, + { "(IIiI)v", invoke_IIiI_v }, { "(IIiI)i", invoke_IIiI_i }, { "(IIiI)I", invoke_IIiI_I }, + { "(IIIi)v", invoke_IIIi_v }, { "(IIIi)i", invoke_IIIi_i }, { "(IIIi)I", invoke_IIIi_I }, + { "(IIII)v", invoke_IIII_v }, { "(IIII)i", invoke_IIII_i }, { "(IIII)I", invoke_IIII_I }, + + { "(iiiii)v", invoke_iiiii_v }, { "(iiiii)i", invoke_iiiii_i }, { "(iiiii)I", invoke_iiiii_I }, +}; +/* clang-format on */ + +static int +quick_aot_entry_cmp(const void *quick_aot_entry1, const void *quick_aot_entry2) +{ + return strcmp(((const QuickAOTEntry *)quick_aot_entry1)->signature, + ((const QuickAOTEntry *)quick_aot_entry2)->signature); +} + +static bool +quick_aot_entry_init() +{ + qsort(quick_aot_entries, sizeof(quick_aot_entries) / sizeof(QuickAOTEntry), + sizeof(QuickAOTEntry), quick_aot_entry_cmp); + + return true; +} + +void * +wasm_native_lookup_quick_aot_entry(const WASMFuncType *func_type) +{ + char signature[16] = { 0 }; + uint32 param_count = func_type->param_count; + uint32 result_count = func_type->result_count, i, j = 0; + const uint8 *types = func_type->types; + QuickAOTEntry *quick_aot_entry, key = { 0 }; + + if (param_count > 5 || result_count > 1) + return NULL; + + signature[j++] = '('; + + for (i = 0; i < param_count; i++) { + if (types[i] == VALUE_TYPE_I32) + signature[j++] = 'i'; + else if (types[i] == VALUE_TYPE_I64) + signature[j++] = 'I'; + else + return NULL; + } + + signature[j++] = ')'; + + if (result_count == 0) { + signature[j++] = 'v'; + } + else { + if (types[i] == VALUE_TYPE_I32) + signature[j++] = 'i'; + else if (types[i] == VALUE_TYPE_I64) + signature[j++] = 'I'; + else + return NULL; + } + + key.signature = signature; + if ((quick_aot_entry = + bsearch(&key, quick_aot_entries, + sizeof(quick_aot_entries) / sizeof(QuickAOTEntry), + sizeof(QuickAOTEntry), quick_aot_entry_cmp))) { + return quick_aot_entry->func_ptr; + } + + return NULL; +} +#endif /* end of WASM_ENABLE_QUICK_AOT_ENTRY != 0 */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_native.h b/wasm-micro-runtime/core/iwasm/common/wasm_native.h new file mode 100644 index 0000000..5cb78bf --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_native.h @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_NATIVE_H +#define _WASM_NATIVE_H + +#include "bh_common.h" +#include "../include/wasm_export.h" +#include "../interpreter/wasm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct NativeSymbolsNode { + struct NativeSymbolsNode *next; + const char *module_name; + NativeSymbol *native_symbols; + uint32 n_native_symbols; + bool call_conv_raw; +} NativeSymbolsNode, *NativeSymbolsList; + +/** + * Lookup global variable of a given import global + * from libc builtin globals + * + * @param module_name the module name of the import global + * @param global_name the global name of the import global + * @param global return the global data + * + * @param true if success, false otherwise + */ +bool +wasm_native_lookup_libc_builtin_global(const char *module_name, + const char *global_name, + WASMGlobalImport *global); + +/** + * Resolve native symbol in all libraries, including libc-builtin, libc-wasi, + * base lib and extension lib, and user registered natives + * function, which can be auto checked by vm before calling native function + * + * @param module_name the module name of the import function + * @param func_name the function name of the import function + * @param func_type the function prototype of the import function + * @param p_signature output the signature if resolve success + * + * @return the native function pointer if success, NULL otherwise + */ +void * +wasm_native_resolve_symbol(const char *module_name, const char *field_name, + const WASMFuncType *func_type, + const char **p_signature, void **p_attachment, + bool *p_call_conv_raw); + +bool +wasm_native_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +bool +wasm_native_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +bool +wasm_native_unregister_natives(const char *module_name, + NativeSymbol *native_symbols); + +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 +struct WASMModuleInstanceCommon; + +void * +wasm_native_create_context_key( + void (*dtor)(struct WASMModuleInstanceCommon *inst, void *ctx)); + +void +wasm_native_destroy_context_key(void *key); + +void +wasm_native_set_context(struct WASMModuleInstanceCommon *inst, void *key, + void *ctx); +void +wasm_native_set_context_spread(struct WASMModuleInstanceCommon *inst, void *key, + void *ctx); +void * +wasm_native_get_context(struct WASMModuleInstanceCommon *inst, void *key); + +void +wasm_native_call_context_dtors(struct WASMModuleInstanceCommon *inst); + +void +wasm_native_inherit_contexts(struct WASMModuleInstanceCommon *child, + struct WASMModuleInstanceCommon *parent); +#else /* WASM_ENABLE_MODULE_INST_CONTEXT */ +#define wasm_native_call_context_dtors(inst) (void)(inst) +#define wasm_native_inherit_contexts(child, parent) (void)(parent) +#endif /* WASM_ENABLE_MODULE_INST_CONTEXT */ + +bool +wasm_native_init(); + +void +wasm_native_destroy(); + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +void * +wasm_native_lookup_quick_aot_entry(const WASMFuncType *func_type); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_NATIVE_H */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_runtime_common.c b/wasm-micro-runtime/core/iwasm/common/wasm_runtime_common.c new file mode 100644 index 0000000..ad40d30 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_runtime_common.c @@ -0,0 +1,7017 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" +#include "bh_common.h" +#include "bh_assert.h" +#include "bh_log.h" +#include "wasm_native.h" +#include "wasm_runtime_common.h" +#include "wasm_memory.h" +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#if WASM_ENABLE_DEBUG_AOT != 0 +#include "../aot/debug/jit_debug.h" +#endif +#endif +#if WASM_ENABLE_GC != 0 +#include "gc/gc_object.h" +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "../libraries/debug-engine/debug_engine.h" +#endif +#endif +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "wasm_shared_memory.h" +#endif +#if WASM_ENABLE_FAST_JIT != 0 +#include "../fast-jit/jit_compiler.h" +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 +#include "../compilation/aot_llvm.h" +#endif +#include "../common/wasm_c_api_internal.h" +#include "../../version.h" + +/** + * For runtime build, BH_MALLOC/BH_FREE should be defined as + * wasm_runtime_malloc/wasm_runtime_free. + */ +#define CHECK(a) CHECK1(a) +#define CHECK1(a) SHOULD_BE_##a + +#define SHOULD_BE_wasm_runtime_malloc 1 +#if !CHECK(BH_MALLOC) +#error unexpected BH_MALLOC +#endif +#undef SHOULD_BE_wasm_runtime_malloc + +#define SHOULD_BE_wasm_runtime_free 1 +#if !CHECK(BH_FREE) +#error unexpected BH_FREE +#endif +#undef SHOULD_BE_wasm_runtime_free + +#undef CHECK +#undef CHECK1 + +#if WASM_ENABLE_MULTI_MODULE != 0 +/** + * A safety insurance to prevent + * circular dependencies which leads stack overflow + * try to break early + */ +typedef struct LoadingModule { + bh_list_link l; + /* point to a string pool */ + const char *module_name; +} LoadingModule; + +static bh_list loading_module_list_head; +static bh_list *const loading_module_list = &loading_module_list_head; +static korp_mutex loading_module_list_lock; + +/** + * A list to store all exported functions/globals/memories/tables + * of every fully loaded module + */ +static bh_list registered_module_list_head; +static bh_list *const registered_module_list = ®istered_module_list_head; +static korp_mutex registered_module_list_lock; +static void +wasm_runtime_destroy_registered_module_list(); +#endif /* WASM_ENABLE_MULTI_MODULE */ + +#define E_TYPE_XIP 4 + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 +/* Initialize externref hashmap */ +static bool +wasm_externref_map_init(); + +/* Destroy externref hashmap */ +static void +wasm_externref_map_destroy(); +#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */ + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +static void * +runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst, + char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + if (module_inst != NULL) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + } + else if (error_buf != NULL) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + } + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +/* TODO: Let loader_malloc be a general API both for AOT and WASM. */ + +#define loader_malloc(size, error_buf, error_buf_size) \ + runtime_malloc(size, NULL, error_buf, error_buf_size) + +static void +set_error_buf_v(const WASMModuleCommon *module, char *error_buf, + uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + if (module->module_type == Wasm_Module_AoT) { + snprintf(error_buf, error_buf_size, "AOT module load failed: %s", + buf); + } + else if (module->module_type == Wasm_Module_Bytecode) { + snprintf(error_buf, error_buf_size, "WASM module load failed: %s", + buf); + } + } +} +#endif + +#if WASM_ENABLE_FAST_JIT != 0 +static JitCompOptions jit_options = { 0 }; +#endif + +#if WASM_ENABLE_JIT != 0 +/* opt_level: 3, size_level: 3, segue-flags: 0, + quick_invoke_c_api_import: false */ +static LLVMJITOptions llvm_jit_options = { 3, 3, 0, false }; +#endif + +#if WASM_ENABLE_GC != 0 +static uint32 gc_heap_size_default = GC_HEAP_SIZE_DEFAULT; +#endif + +static RunningMode runtime_running_mode = Mode_Default; + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* The exec_env of thread local storage, set before calling function + and used in signal handler, as we cannot get it from the argument + of signal handler */ +static os_thread_local_attribute WASMExecEnv *exec_env_tls = NULL; + +#ifndef BH_PLATFORM_WINDOWS +static void +runtime_signal_handler(void *sig_addr) +{ + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint32 page_size = os_getpagesize(); +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + uint8 *stack_min_addr; + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; +#endif + + /* Check whether current thread is running wasm function */ + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + /* Get mapped mem info of current instance */ + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + /* Get the default memory instance */ + memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) { + mapped_mem_start_addr = memory_inst->memory_data; + mapped_mem_end_addr = memory_inst->memory_data + 8 * (uint64)BH_GB; + } + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + /* Get stack info of current thread */ + stack_min_addr = os_thread_get_stack_boundary(); +#endif + + if (memory_inst + && (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr)) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions */ + wasm_set_exception(module_inst, "out of bounds memory access"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + else if (stack_min_addr - page_size <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < stack_min_addr + page_size * guard_page_count) { + /* The address which causes segmentation fault is inside + native thread's guard page */ + wasm_set_exception(module_inst, "native stack overflow"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } +#endif + else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < exec_env_tls->exce_check_guard_page + page_size) { + bh_assert(wasm_copy_exception(module_inst, NULL)); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } + } +} +#else /* else of BH_PLATFORM_WINDOWS */ + +#if WASM_ENABLE_AOT != 0 +#include + +static uint32 +decode_insn(uint8 *insn) +{ + uint8 *data = (uint8 *)insn; + uint32 length = 32; /* reserve enough size */ + + /* Initialize decoder context */ + ZydisDecoder decoder; + ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, + ZYDIS_STACK_WIDTH_64); + + /* Initialize formatter */ + ZydisFormatter formatter; + ZydisFormatterInit(&formatter, ZYDIS_FORMATTER_STYLE_INTEL); + + /* Loop over the instructions in our buffer */ + ZyanU64 runtime_address = (ZyanU64)(uintptr_t)data; + ZyanUSize offset = 0; + ZydisDecodedInstruction instruction; + ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT_VISIBLE]; + char buffer[256]; + + if (ZYAN_SUCCESS(ZydisDecoderDecodeFull( + &decoder, data + offset, length - offset, &instruction, operands, + ZYDIS_MAX_OPERAND_COUNT_VISIBLE, + ZYDIS_DFLAG_VISIBLE_OPERANDS_ONLY))) { + + /* Format & print the binary instruction structure to + human readable format */ + ZydisFormatterFormatInstruction(&formatter, &instruction, operands, + instruction.operand_count_visible, + buffer, sizeof(buffer), + runtime_address); + +#if 0 + /* Print current instruction */ + os_printf("%012" PRIX64 " ", runtime_address); + puts(buffer); +#endif + + return instruction.length; + } + + /* Decode failed */ + return 0; +} +#endif /* end of WASM_ENABLE_AOT != 0 */ + +static LONG +next_action(WASMModuleInstance *module_inst, EXCEPTION_POINTERS *exce_info) +{ +#if WASM_ENABLE_AOT != 0 + uint32 insn_size; +#endif + + if (module_inst->module_type == Wasm_Module_Bytecode + && module_inst->e->running_mode == Mode_Interp) { + /* Continue to search next exception handler for + interpreter mode as it can be caught by + `__try { .. } __except { .. }` sentences in + wasm_runtime.c */ + return EXCEPTION_CONTINUE_SEARCH; + } + +#if WASM_ENABLE_AOT != 0 + /* Skip current instruction and continue to run for AOT/JIT mode. + TODO: implement unwind support for AOT/JIT code in Windows platform */ + insn_size = decode_insn((uint8 *)exce_info->ContextRecord->Rip); + if (insn_size > 0) { + exce_info->ContextRecord->Rip += insn_size; + return EXCEPTION_CONTINUE_EXECUTION; + } +#endif + + /* return different value from EXCEPTION_CONTINUE_SEARCH (= 0) + and EXCEPTION_CONTINUE_EXECUTION (= -1) */ + return -2; +} + +static LONG +runtime_exception_handler(EXCEPTION_POINTERS *exce_info) +{ + PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; + uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint32 page_size = os_getpagesize(); + LONG ret; + + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + /* Get the default memory instance */ + memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) { + mapped_mem_start_addr = memory_inst->memory_data; + mapped_mem_end_addr = + memory_inst->memory_data + 8 * (uint64)BH_GB; + } + + if (memory_inst && mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions. + Set exception and let the wasm func continue to run, when + the wasm func returns, the caller will check whether the + exception is thrown and return to runtime. */ + wasm_set_exception(module_inst, "out of bounds memory access"); + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; + } + else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < exec_env_tls->exce_check_guard_page + page_size) { + bh_assert(wasm_copy_exception(module_inst, NULL)); + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; + } + } +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { + /* Set stack overflow exception and let the wasm func continue + to run, when the wasm func returns, the caller will check + whether the exception is thrown and return to runtime, and + the damaged stack will be recovered by _resetstkoflw(). */ + wasm_set_exception(module_inst, "native stack overflow"); + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; + } +#endif + } + + LOG_ERROR("Unhandled exception thrown: exception code: 0x%lx, " + "exception address: %p, exception information: %p\n", + ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, + sig_addr); + return EXCEPTION_CONTINUE_SEARCH; +} +#endif /* end of BH_PLATFORM_WINDOWS */ + +static bool +runtime_signal_init() +{ +#ifndef BH_PLATFORM_WINDOWS + return os_thread_signal_init(runtime_signal_handler) == 0 ? true : false; +#else + if (os_thread_signal_init() != 0) + return false; + + if (!AddVectoredExceptionHandler(1, runtime_exception_handler)) { + os_thread_signal_destroy(); + return false; + } +#endif + return true; +} + +static void +runtime_signal_destroy() +{ +#ifdef BH_PLATFORM_WINDOWS + RemoveVectoredExceptionHandler(runtime_exception_handler); +#endif + os_thread_signal_destroy(); +} + +void +wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env) +{ + exec_env_tls = exec_env; +} + +WASMExecEnv * +wasm_runtime_get_exec_env_tls() +{ + return exec_env_tls; +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + +static bool +wasm_runtime_env_init() +{ + if (bh_platform_init() != 0) + return false; + + if (wasm_native_init() == false) { + goto fail1; + } + +#if WASM_ENABLE_MULTI_MODULE + if (BHT_OK != os_mutex_init(®istered_module_list_lock)) { + goto fail2; + } + + if (BHT_OK != os_mutex_init(&loading_module_list_lock)) { + goto fail3; + } +#endif + +#if WASM_ENABLE_SHARED_MEMORY + if (!wasm_shared_memory_init()) { + goto fail4; + } +#endif + +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) + if (!thread_manager_init()) { + goto fail5; + } +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!runtime_signal_init()) { + goto fail6; + } +#endif + +#if WASM_ENABLE_AOT != 0 +#if WASM_ENABLE_DEBUG_AOT != 0 + if (!jit_debug_engine_init()) { + goto fail7; + } +#endif +#endif + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + if (!wasm_externref_map_init()) { + goto fail8; + } +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + if (!jit_compiler_init(&jit_options)) { + goto fail9; + } +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + if (!aot_compiler_init()) { + goto fail10; + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 && defined(OS_ENABLE_WAKEUP_BLOCKING_OP) + if (os_blocking_op_init() != BHT_OK) { + goto fail11; + } + os_end_blocking_op(); +#endif + + return true; + +#if WASM_ENABLE_THREAD_MGR != 0 && defined(OS_ENABLE_WAKEUP_BLOCKING_OP) +fail11: +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + aot_compiler_destroy(); +#endif +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 +fail10: +#if WASM_ENABLE_FAST_JIT != 0 + jit_compiler_destroy(); +#endif +#endif +#if WASM_ENABLE_FAST_JIT != 0 +fail9: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + wasm_externref_map_destroy(); +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 +fail8: +#endif +#if WASM_ENABLE_AOT != 0 +#if WASM_ENABLE_DEBUG_AOT != 0 + jit_debug_engine_destroy(); +fail7: +#endif +#endif +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); +fail6: +#endif +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) + thread_manager_destroy(); +fail5: +#endif +#if WASM_ENABLE_SHARED_MEMORY + wasm_shared_memory_destroy(); +fail4: +#endif +#if WASM_ENABLE_MULTI_MODULE + os_mutex_destroy(&loading_module_list_lock); +fail3: + os_mutex_destroy(®istered_module_list_lock); +fail2: +#endif + wasm_native_destroy(); +fail1: + bh_platform_destroy(); + + return false; +} + +static bool +wasm_runtime_exec_env_check(WASMExecEnv *exec_env) +{ + return exec_env && exec_env->module_inst && exec_env->wasm_stack_size > 0 + && exec_env->wasm_stack.top_boundary + == exec_env->wasm_stack.bottom + exec_env->wasm_stack_size + && exec_env->wasm_stack.top <= exec_env->wasm_stack.top_boundary; +} + +#if defined(OS_THREAD_MUTEX_INITIALIZER) +/** + * lock for wasm_runtime_init/wasm_runtime_full_init and runtime_ref_count + * Note: if the platform has mutex initializer, we use a global lock to + * lock the operations of runtime init/full_init, otherwise when there are + * operations happening simultaneously in multiple threads, developer + * must create the lock by himself, and use it to lock the operations + */ +static korp_mutex runtime_lock = OS_THREAD_MUTEX_INITIALIZER; +#endif +static int32 runtime_ref_count = 0; + +static bool +wasm_runtime_init_internal() +{ + if (!wasm_runtime_memory_init(Alloc_With_System_Allocator, NULL)) + return false; + + if (!wasm_runtime_env_init()) { + wasm_runtime_memory_destroy(); + return false; + } + + return true; +} + +bool +wasm_runtime_init() +{ + bool ret = true; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count >= 0); + if (runtime_ref_count == 0) { + ret = wasm_runtime_init_internal(); + } + if (ret) { + runtime_ref_count++; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif + + return ret; +} + +static void +wasm_runtime_destroy_internal() +{ +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + wasm_externref_map_destroy(); +#endif + +#if WASM_ENABLE_AOT != 0 +#if WASM_ENABLE_DEBUG_AOT != 0 + jit_debug_engine_destroy(); +#endif +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); +#endif + + /* runtime env destroy */ +#if WASM_ENABLE_MULTI_MODULE + wasm_runtime_destroy_loading_module_list(); + os_mutex_destroy(&loading_module_list_lock); + + wasm_runtime_destroy_registered_module_list(); + os_mutex_destroy(®istered_module_list_lock); +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + /* Destroy LLVM-JIT compiler after destroying the modules + * loaded by multi-module feature, since these modules may + * create backend threads to compile the wasm functions, + * which may access the LLVM resources. We wait until they + * finish the compilation to avoid accessing the destroyed + * resources in the compilation threads. + */ + aot_compiler_destroy(); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + /* Destroy Fast-JIT compiler after destroying the modules + * loaded by multi-module feature, since the Fast JIT's + * code cache allocator may be used by these modules. + */ + jit_compiler_destroy(); +#endif + +#if WASM_ENABLE_SHARED_MEMORY + wasm_shared_memory_destroy(); +#endif + +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_debug_engine_destroy(); +#endif + thread_manager_destroy(); +#endif + + wasm_native_destroy(); + bh_platform_destroy(); + + wasm_runtime_memory_destroy(); +} + +void +wasm_runtime_destroy() +{ +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count > 0); + runtime_ref_count--; + if (runtime_ref_count == 0) { + wasm_runtime_destroy_internal(); + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif +} + +RunningMode +wasm_runtime_get_default_running_mode(void) +{ + return runtime_running_mode; +} + +#if WASM_ENABLE_JIT != 0 +LLVMJITOptions * +wasm_runtime_get_llvm_jit_options(void) +{ + return &llvm_jit_options; +} +#endif + +#if WASM_ENABLE_GC != 0 +uint32 +wasm_runtime_get_gc_heap_size_default(void) +{ + return gc_heap_size_default; +} +#endif + +static bool +wasm_runtime_full_init_internal(RuntimeInitArgs *init_args) +{ + if (!wasm_runtime_memory_init(init_args->mem_alloc_type, + &init_args->mem_alloc_option)) + return false; + + if (!wasm_runtime_set_default_running_mode(init_args->running_mode)) { + wasm_runtime_memory_destroy(); + return false; + } + +#if WASM_ENABLE_FAST_JIT != 0 + jit_options.code_cache_size = init_args->fast_jit_code_cache_size; +#endif + +#if WASM_ENABLE_GC != 0 + gc_heap_size_default = init_args->gc_heap_size; +#endif + +#if WASM_ENABLE_JIT != 0 + llvm_jit_options.size_level = init_args->llvm_jit_size_level; + llvm_jit_options.opt_level = init_args->llvm_jit_opt_level; + llvm_jit_options.segue_flags = init_args->segue_flags; +#endif + +#if WASM_ENABLE_LINUX_PERF != 0 + wasm_runtime_set_linux_perf(init_args->enable_linux_perf); +#else + if (init_args->enable_linux_perf) + LOG_WARNING("warning: to enable linux perf support, please recompile " + "with -DWAMR_BUILD_LINUX_PERF=1"); +#endif + + if (!wasm_runtime_env_init()) { + wasm_runtime_memory_destroy(); + return false; + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (strlen(init_args->ip_addr)) + if (!wasm_debug_engine_init(init_args->ip_addr, + init_args->instance_port)) { + wasm_runtime_destroy(); + return false; + } +#endif + + if (init_args->n_native_symbols > 0 + && !wasm_runtime_register_natives(init_args->native_module_name, + init_args->native_symbols, + init_args->n_native_symbols)) { + wasm_runtime_destroy(); + return false; + } + +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_set_max_thread_num(init_args->max_thread_num); +#endif + + return true; +} + +bool +wasm_runtime_full_init(RuntimeInitArgs *init_args) +{ + bool ret = true; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count >= 0); + if (runtime_ref_count == 0) { + ret = wasm_runtime_full_init_internal(init_args); + } + if (ret) { + runtime_ref_count++; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif + + return ret; +} + +void +wasm_runtime_set_log_level(log_level_t level) +{ + bh_log_set_verbose_level(level); +} + +bool +wasm_runtime_is_running_mode_supported(RunningMode running_mode) +{ + if (running_mode == Mode_Default) { + return true; + } + else if (running_mode == Mode_Interp) { +#if WASM_ENABLE_INTERP != 0 + return true; +#endif + } + else if (running_mode == Mode_Fast_JIT) { +#if WASM_ENABLE_FAST_JIT != 0 + return true; +#endif + } + else if (running_mode == Mode_LLVM_JIT) { +#if WASM_ENABLE_JIT != 0 + return true; +#endif + } + else if (running_mode == Mode_Multi_Tier_JIT) { +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + return true; +#endif + } + + return false; +} + +bool +wasm_runtime_set_default_running_mode(RunningMode running_mode) +{ + if (wasm_runtime_is_running_mode_supported(running_mode)) { + runtime_running_mode = running_mode; + return true; + } + return false; +} + +PackageType +get_package_type(const uint8 *buf, uint32 size) +{ +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + uint32 buf32 = *(uint32 *)buf; + buf = (const uint8 *)&buf32; +#endif + if (buf && size >= 4) { + if (buf[0] == '\0' && buf[1] == 'a' && buf[2] == 's' && buf[3] == 'm') + return Wasm_Module_Bytecode; + if (buf[0] == '\0' && buf[1] == 'a' && buf[2] == 'o' && buf[3] == 't') + return Wasm_Module_AoT; + } + return Package_Type_Unknown; +} + +#if WASM_ENABLE_AOT != 0 +static uint8 * +align_ptr(const uint8 *p, uint32 b) +{ + uintptr_t v = (uintptr_t)p; + uintptr_t m = b - 1; + return (uint8 *)((v + m) & ~m); +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if ((uintptr_t)buf + length < (uintptr_t)buf \ + || (uintptr_t)buf + length > (uintptr_t)buf_end) \ + return false; \ + } while (0) + +/* NOLINTNEXTLINE */ +#define read_uint16(p, p_end, res) \ + do { \ + p = (uint8 *)align_ptr(p, sizeof(uint16)); \ + CHECK_BUF(p, p_end, sizeof(uint16)); \ + res = *(uint16 *)p; \ + p += sizeof(uint16); \ + } while (0) + +/* NOLINTNEXTLINE */ +#define read_uint32(p, p_end, res) \ + do { \ + p = (uint8 *)align_ptr(p, sizeof(uint32)); \ + CHECK_BUF(p, p_end, sizeof(uint32)); \ + res = *(uint32 *)p; \ + p += sizeof(uint32); \ + } while (0) + +bool +wasm_runtime_is_xip_file(const uint8 *buf, uint32 size) +{ + const uint8 *p = buf, *p_end = buf + size; + uint32 section_type, section_size; + uint16 e_type; + + if (get_package_type(buf, size) != Wasm_Module_AoT) + return false; + + CHECK_BUF(p, p_end, 8); + p += 8; + while (p < p_end) { + read_uint32(p, p_end, section_type); + read_uint32(p, p_end, section_size); + CHECK_BUF(p, p_end, section_size); + + if (section_type == AOT_SECTION_TYPE_TARGET_INFO) { + p += 4; + read_uint16(p, p_end, e_type); + return (e_type == E_TYPE_XIP) ? true : false; + } + else if (section_type >= AOT_SECTION_TYPE_SIGANATURE) { + return false; + } + p += section_size; + } + + return false; +} +#endif /* end of WASM_ENABLE_AOT */ + +#if (WASM_ENABLE_THREAD_MGR != 0) && (WASM_ENABLE_DEBUG_INTERP != 0) +uint32 +wasm_runtime_start_debug_instance_with_port(WASMExecEnv *exec_env, int32_t port) +{ + WASMModuleInstanceCommon *module_inst = + wasm_runtime_get_module_inst(exec_env); + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(module_inst); + bh_assert(cluster); + + if (module_inst->module_type != Wasm_Module_Bytecode) { + LOG_WARNING("Attempt to create a debug instance for an AOT module"); + return 0; + } + + if (cluster->debug_inst) { + LOG_WARNING("Cluster already bind to a debug instance"); + return cluster->debug_inst->control_thread->port; + } + + if (wasm_debug_instance_create(cluster, port)) { + return cluster->debug_inst->control_thread->port; + } + + return 0; +} + +uint32 +wasm_runtime_start_debug_instance(WASMExecEnv *exec_env) +{ + return wasm_runtime_start_debug_instance_with_port(exec_env, -1); +} +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +static module_reader reader; +static module_destroyer destroyer; +void +wasm_runtime_set_module_reader(const module_reader reader_cb, + const module_destroyer destroyer_cb) +{ + reader = reader_cb; + destroyer = destroyer_cb; +} + +module_reader +wasm_runtime_get_module_reader() +{ + return reader; +} + +module_destroyer +wasm_runtime_get_module_destroyer() +{ + return destroyer; +} + +static WASMRegisteredModule * +wasm_runtime_find_module_registered_by_reference(WASMModuleCommon *module) +{ + WASMRegisteredModule *reg_module = NULL; + + os_mutex_lock(®istered_module_list_lock); + reg_module = bh_list_first_elem(registered_module_list); + while (reg_module && module != reg_module->module) { + reg_module = bh_list_elem_next(reg_module); + } + os_mutex_unlock(®istered_module_list_lock); + + return reg_module; +} + +bool +wasm_runtime_register_module_internal(const char *module_name, + WASMModuleCommon *module, + uint8 *orig_file_buf, + uint32 orig_file_buf_size, + char *error_buf, uint32 error_buf_size) +{ + WASMRegisteredModule *node = NULL; + + node = wasm_runtime_find_module_registered_by_reference(module); + if (node) { /* module has been registered */ + if (node->module_name) { /* module has name */ + if (!module_name || strcmp(node->module_name, module_name)) { + /* module has different name */ + LOG_DEBUG("module(%p) has been registered with name %s", module, + node->module_name); + set_error_buf(error_buf, error_buf_size, + "Register module failed: " + "failed to rename the module"); + return false; + } + else { + /* module has the same name */ + LOG_DEBUG( + "module(%p) has been registered with the same name %s", + module, node->module_name); + return true; + } + } + else { + /* module has empyt name, reset it */ + node->module_name = module_name; + return true; + } + } + + /* module hasn't been registered */ + node = runtime_malloc(sizeof(WASMRegisteredModule), NULL, NULL, 0); + if (!node) { + LOG_DEBUG("malloc WASMRegisteredModule failed. SZ=%zu", + sizeof(WASMRegisteredModule)); + return false; + } + + /* share the string and the module */ + node->module_name = module_name; + node->module = module; + node->orig_file_buf = orig_file_buf; + node->orig_file_buf_size = orig_file_buf_size; + + os_mutex_lock(®istered_module_list_lock); + bh_list_status ret = bh_list_insert(registered_module_list, node); + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + os_mutex_unlock(®istered_module_list_lock); + return true; +} + +bool +wasm_runtime_register_module(const char *module_name, WASMModuleCommon *module, + char *error_buf, uint32 error_buf_size) +{ + if (!error_buf || !error_buf_size) { + LOG_ERROR("error buffer is required"); + return false; + } + + if (!module_name || !module) { + LOG_DEBUG("module_name and module are required"); + set_error_buf(error_buf, error_buf_size, + "Register module failed: " + "module_name and module are required"); + return false; + } + + if (wasm_runtime_is_built_in_module(module_name)) { + LOG_DEBUG("%s is a built-in module name", module_name); + set_error_buf(error_buf, error_buf_size, + "Register module failed: " + "can not register as a built-in module"); + return false; + } + + return wasm_runtime_register_module_internal(module_name, module, NULL, 0, + error_buf, error_buf_size); +} + +void +wasm_runtime_unregister_module(const WASMModuleCommon *module) +{ + WASMRegisteredModule *registered_module = NULL; + + os_mutex_lock(®istered_module_list_lock); + registered_module = bh_list_first_elem(registered_module_list); + while (registered_module && module != registered_module->module) { + registered_module = bh_list_elem_next(registered_module); + } + + /* it does not matter if it is not exist. after all, it is gone */ + if (registered_module) { + bh_list_remove(registered_module_list, registered_module); + wasm_runtime_free(registered_module); + } + os_mutex_unlock(®istered_module_list_lock); +} + +WASMModuleCommon * +wasm_runtime_find_module_registered(const char *module_name) +{ + WASMRegisteredModule *module = NULL, *module_next; + + os_mutex_lock(®istered_module_list_lock); + module = bh_list_first_elem(registered_module_list); + while (module) { + module_next = bh_list_elem_next(module); + if (module->module_name && !strcmp(module_name, module->module_name)) { + break; + } + module = module_next; + } + os_mutex_unlock(®istered_module_list_lock); + + return module ? module->module : NULL; +} + +/* + * simply destroy all + */ +static void +wasm_runtime_destroy_registered_module_list() +{ + WASMRegisteredModule *reg_module = NULL; + + os_mutex_lock(®istered_module_list_lock); + reg_module = bh_list_first_elem(registered_module_list); + while (reg_module) { + WASMRegisteredModule *next_reg_module = bh_list_elem_next(reg_module); + + bh_list_remove(registered_module_list, reg_module); + + /* now, it is time to release every module in the runtime */ + if (reg_module->module->module_type == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + wasm_unload((WASMModule *)reg_module->module); +#endif + } + else { +#if WASM_ENABLE_AOT != 0 + aot_unload((AOTModule *)reg_module->module); +#endif + } + + /* destroy the file buffer */ + if (destroyer && reg_module->orig_file_buf) { + destroyer(reg_module->orig_file_buf, + reg_module->orig_file_buf_size); + reg_module->orig_file_buf = NULL; + reg_module->orig_file_buf_size = 0; + } + + wasm_runtime_free(reg_module); + reg_module = next_reg_module; + } + os_mutex_unlock(®istered_module_list_lock); +} + +bool +wasm_runtime_add_loading_module(const char *module_name, char *error_buf, + uint32 error_buf_size) +{ + LOG_DEBUG("add %s into a loading list", module_name); + LoadingModule *loadingModule = + runtime_malloc(sizeof(LoadingModule), NULL, error_buf, error_buf_size); + + if (!loadingModule) { + return false; + } + + /* share the incoming string */ + loadingModule->module_name = module_name; + + os_mutex_lock(&loading_module_list_lock); + bh_list_status ret = bh_list_insert(loading_module_list, loadingModule); + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + os_mutex_unlock(&loading_module_list_lock); + return true; +} + +void +wasm_runtime_delete_loading_module(const char *module_name) +{ + LOG_DEBUG("delete %s from a loading list", module_name); + + LoadingModule *module = NULL; + + os_mutex_lock(&loading_module_list_lock); + module = bh_list_first_elem(loading_module_list); + while (module && strcmp(module->module_name, module_name)) { + module = bh_list_elem_next(module); + } + + /* it does not matter if it is not exist. after all, it is gone */ + if (module) { + bh_list_remove(loading_module_list, module); + wasm_runtime_free(module); + } + os_mutex_unlock(&loading_module_list_lock); +} + +bool +wasm_runtime_is_loading_module(const char *module_name) +{ + LOG_DEBUG("find %s in a loading list", module_name); + + LoadingModule *module = NULL; + + os_mutex_lock(&loading_module_list_lock); + module = bh_list_first_elem(loading_module_list); + while (module && strcmp(module_name, module->module_name)) { + module = bh_list_elem_next(module); + } + os_mutex_unlock(&loading_module_list_lock); + + return module != NULL; +} + +void +wasm_runtime_destroy_loading_module_list() +{ + LoadingModule *module = NULL; + + os_mutex_lock(&loading_module_list_lock); + module = bh_list_first_elem(loading_module_list); + while (module) { + LoadingModule *next_module = bh_list_elem_next(module); + + bh_list_remove(loading_module_list, module); + /* + * will not free the module_name since it is + * shared one of the const string pool + */ + wasm_runtime_free(module); + + module = next_module; + } + + os_mutex_unlock(&loading_module_list_lock); +} +#endif /* WASM_ENABLE_MULTI_MODULE */ + +bool +wasm_runtime_is_built_in_module(const char *module_name) +{ + return (!strcmp("env", module_name) || !strcmp("wasi_unstable", module_name) + || !strcmp("wasi_snapshot_preview1", module_name) +#if WASM_ENABLE_SPEC_TEST != 0 + || !strcmp("spectest", module_name) +#endif + || !strcmp("", module_name)); +} + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_exec_env_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, + uint32 size) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_set_aux_stack(exec_env, start_offset, size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_set_aux_stack(exec_env, start_offset, size); + } +#endif + return false; +} + +bool +wasm_exec_env_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, + uint32 *size) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_get_aux_stack(exec_env, start_offset, size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_get_aux_stack(exec_env, start_offset, size); + } +#endif + return false; +} + +void +wasm_runtime_set_max_thread_num(uint32 num) +{ + wasm_cluster_set_max_thread_num(num); +} +#endif /* end of WASM_ENABLE_THREAD_MGR */ + +static WASMModuleCommon * +register_module_with_null_name(WASMModuleCommon *module_common, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module_common) { + if (!wasm_runtime_register_module_internal(NULL, module_common, NULL, 0, + error_buf, error_buf_size)) { + wasm_runtime_unload(module_common); + return NULL; + } + return module_common; + } + else + return NULL; +#else + return module_common; +#endif +} + +WASMModuleCommon * +wasm_runtime_load_ex(uint8 *buf, uint32 size, const LoadArgs *args, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_common = NULL; + + if (!args) { + return NULL; + } + + if (get_package_type(buf, size) == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + module_common = + (WASMModuleCommon *)wasm_load(buf, size, +#if WASM_ENABLE_MULTI_MODULE != 0 + true, +#endif + args, error_buf, error_buf_size); +#endif + } + else if (get_package_type(buf, size) == Wasm_Module_AoT) { +#if WASM_ENABLE_AOT != 0 + module_common = (WASMModuleCommon *)aot_load_from_aot_file( + buf, size, args, error_buf, error_buf_size); +#endif + } + else { + if (size < 4) + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: unexpected end"); + else + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: magic header not detected"); + return NULL; + } + if (!module_common) { + LOG_DEBUG("WASM module load failed"); + return NULL; + } + + /*TODO: use file name as name and register with name? */ + return register_module_with_null_name(module_common, error_buf, + error_buf_size); +} + +WASMModuleCommon * +wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, + uint32 error_buf_size) +{ + LoadArgs args = { 0 }; + args.name = ""; + return wasm_runtime_load_ex(buf, size, &args, error_buf, error_buf_size); +} + +WASMModuleCommon * +wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_common; + + if (!is_aot) { +#if WASM_ENABLE_INTERP != 0 + module_common = (WASMModuleCommon *)wasm_load_from_sections( + section_list, error_buf, error_buf_size); + if (!module_common) { + LOG_DEBUG("WASM module load failed from sections"); + return NULL; + } + return register_module_with_null_name(module_common, error_buf, + error_buf_size); +#endif + } + else { +#if WASM_ENABLE_AOT != 0 + module_common = (WASMModuleCommon *)aot_load_from_sections( + section_list, error_buf, error_buf_size); + if (!module_common) { + LOG_DEBUG("WASM module load failed from sections"); + return NULL; + } + return register_module_with_null_name(module_common, error_buf, + error_buf_size); +#endif + } + +#if WASM_ENABLE_INTERP == 0 || WASM_ENABLE_AOT == 0 + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: invalid section list type"); + return NULL; +#endif +} + +void +wasm_runtime_unload(WASMModuleCommon *module) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + /** + * since we will unload and free all module when runtime_destroy() + * we don't want users to unwillingly disrupt it + */ + return; +#endif + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + wasm_unload((WASMModule *)module); + return; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + aot_unload((AOTModule *)module); + return; + } +#endif +} + +uint32 +wasm_runtime_get_max_mem(uint32 max_memory_pages, uint32 module_init_page_count, + uint32 module_max_page_count) +{ + if (max_memory_pages == 0) { + /* Max memory not overwritten by runtime, use value from wasm module */ + return module_max_page_count; + } + + if (max_memory_pages < module_init_page_count) { + LOG_WARNING("Cannot override max memory with value lower than module " + "initial memory"); + return module_init_page_count; + } + + if (max_memory_pages > module_max_page_count) { + LOG_WARNING("Cannot override max memory with value greater than module " + "max memory"); + return module_max_page_count; + } + + return max_memory_pages; +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate_internal(WASMModuleCommon *module, + WASMModuleInstanceCommon *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, + char *error_buf, uint32 error_buf_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + return (WASMModuleInstanceCommon *)wasm_instantiate( + (WASMModule *)module, (WASMModuleInstance *)parent, exec_env_main, + stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + return (WASMModuleInstanceCommon *)aot_instantiate( + (AOTModule *)module, (AOTModuleInstance *)parent, exec_env_main, + stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); +#endif + set_error_buf(error_buf, error_buf_size, + "Instantiate module failed, invalid module type"); + return NULL; +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate(WASMModuleCommon *module, uint32 stack_size, + uint32 heap_size, char *error_buf, + uint32 error_buf_size) +{ + return wasm_runtime_instantiate_internal(module, NULL, NULL, stack_size, + heap_size, 0, error_buf, + error_buf_size); +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex(WASMModuleCommon *module, + const InstantiationArgs *args, char *error_buf, + uint32 error_buf_size) +{ + return wasm_runtime_instantiate_internal( + module, NULL, NULL, args->default_stack_size, + args->host_managed_heap_size, args->max_memory_pages, error_buf, + error_buf_size); +} + +void +wasm_runtime_deinstantiate_internal(WASMModuleInstanceCommon *module_inst, + bool is_sub_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_deinstantiate((WASMModuleInstance *)module_inst, is_sub_inst); + return; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_deinstantiate((AOTModuleInstance *)module_inst, is_sub_inst); + return; + } +#endif +} + +bool +wasm_runtime_set_running_mode(wasm_module_inst_t module_inst, + RunningMode running_mode) +{ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return true; +#endif + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst_interp = + (WASMModuleInstance *)module_inst; + + return wasm_set_running_mode(module_inst_interp, running_mode); + } +#endif + + return false; +} + +RunningMode +wasm_runtime_get_running_mode(wasm_module_inst_t module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst_interp = + (WASMModuleInstance *)module_inst; + return module_inst_interp->e->running_mode; + } +#endif + + return Mode_Default; +} + +void +wasm_runtime_deinstantiate(WASMModuleInstanceCommon *module_inst) +{ + wasm_runtime_deinstantiate_internal(module_inst, false); +} + +WASMModuleCommon * +wasm_runtime_get_module(WASMModuleInstanceCommon *module_inst) +{ + return (WASMModuleCommon *)((WASMModuleInstance *)module_inst)->module; +} + +WASMExecEnv * +wasm_runtime_create_exec_env(WASMModuleInstanceCommon *module_inst, + uint32 stack_size) +{ + return wasm_exec_env_create(module_inst, stack_size); +} + +void +wasm_runtime_destroy_exec_env(WASMExecEnv *exec_env) +{ + wasm_exec_env_destroy(exec_env); +} + +bool +wasm_runtime_init_thread_env(void) +{ +#ifdef BH_PLATFORM_WINDOWS + if (os_thread_env_init() != 0) + return false; +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!runtime_signal_init()) { +#ifdef BH_PLATFORM_WINDOWS + os_thread_env_destroy(); +#endif + return false; + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 && defined(OS_ENABLE_WAKEUP_BLOCKING_OP) + os_end_blocking_op(); +#endif + + return true; +} + +void +wasm_runtime_destroy_thread_env(void) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); +#endif + +#ifdef BH_PLATFORM_WINDOWS + os_thread_env_destroy(); +#endif +} + +bool +wasm_runtime_thread_env_inited(void) +{ +#ifdef BH_PLATFORM_WINDOWS + if (!os_thread_env_inited()) + return false; +#endif + +#if WASM_ENABLE_AOT != 0 +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!os_thread_signal_inited()) + return false; +#endif +#endif + return true; +} + +#if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0) +void +wasm_runtime_dump_module_mem_consumption(const WASMModuleCommon *module) +{ + WASMModuleMemConsumption mem_conspn = { 0 }; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + wasm_get_module_mem_consumption((WASMModule *)module, &mem_conspn); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + aot_get_module_mem_consumption((AOTModule *)module, &mem_conspn); + } +#endif + + os_printf("WASM module memory consumption, total size: %u\n", + mem_conspn.total_size); + os_printf(" module struct size: %u\n", mem_conspn.module_struct_size); + os_printf(" types size: %u\n", mem_conspn.types_size); + os_printf(" imports size: %u\n", mem_conspn.imports_size); + os_printf(" funcs size: %u\n", mem_conspn.functions_size); + os_printf(" tables size: %u\n", mem_conspn.tables_size); + os_printf(" memories size: %u\n", mem_conspn.memories_size); + os_printf(" globals size: %u\n", mem_conspn.globals_size); + os_printf(" exports size: %u\n", mem_conspn.exports_size); + os_printf(" table segs size: %u\n", mem_conspn.table_segs_size); + os_printf(" data segs size: %u\n", mem_conspn.data_segs_size); + os_printf(" const strings size: %u\n", mem_conspn.const_strs_size); +#if WASM_ENABLE_AOT != 0 + os_printf(" aot code size: %u\n", mem_conspn.aot_code_size); +#endif +} + +void +wasm_runtime_dump_module_inst_mem_consumption( + const WASMModuleInstanceCommon *module_inst) +{ + WASMModuleInstMemConsumption mem_conspn = { 0 }; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_get_module_inst_mem_consumption((WASMModuleInstance *)module_inst, + &mem_conspn); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_get_module_inst_mem_consumption((AOTModuleInstance *)module_inst, + &mem_conspn); + } +#endif + + os_printf("WASM module inst memory consumption, total size: %lu\n", + mem_conspn.total_size); + os_printf(" module inst struct size: %u\n", + mem_conspn.module_inst_struct_size); + os_printf(" memories size: %lu\n", mem_conspn.memories_size); + os_printf(" app heap size: %u\n", mem_conspn.app_heap_size); + os_printf(" tables size: %u\n", mem_conspn.tables_size); + os_printf(" functions size: %u\n", mem_conspn.functions_size); + os_printf(" globals size: %u\n", mem_conspn.globals_size); + os_printf(" exports size: %u\n", mem_conspn.exports_size); +} + +void +wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env) +{ + uint32 total_size = + offsetof(WASMExecEnv, wasm_stack_u.bottom) + exec_env->wasm_stack_size; + + os_printf("Exec env memory consumption, total size: %u\n", total_size); + os_printf(" exec env struct size: %u\n", + offsetof(WASMExecEnv, wasm_stack_u.bottom)); +#if WASM_ENABLE_INTERP != 0 && WASM_ENABLE_FAST_INTERP == 0 + os_printf(" block addr cache size: %u\n", + sizeof(exec_env->block_addr_cache)); +#endif + os_printf(" stack size: %u\n", exec_env->wasm_stack_size); +} + +uint32 +gc_get_heap_highmark_size(void *heap); + +void +wasm_runtime_dump_mem_consumption(WASMExecEnv *exec_env) +{ + WASMModuleInstMemConsumption module_inst_mem_consps; + WASMModuleMemConsumption module_mem_consps; + WASMModuleInstanceCommon *module_inst_common; + WASMModuleCommon *module_common = NULL; + void *heap_handle = NULL; + uint32 app_heap_peak_size = 0; + uint32 max_aux_stack_used = -1; + uint64 total_size = 0; + + module_inst_common = exec_env->module_inst; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_common->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *wasm_module_inst = + (WASMModuleInstance *)module_inst_common; + WASMModule *wasm_module = wasm_module_inst->module; + module_common = (WASMModuleCommon *)wasm_module; + if (wasm_module_inst->memories) { + heap_handle = wasm_module_inst->memories[0]->heap_handle; + } + wasm_get_module_inst_mem_consumption(wasm_module_inst, + &module_inst_mem_consps); + wasm_get_module_mem_consumption(wasm_module, &module_mem_consps); + if (wasm_module_inst->module->aux_stack_top_global_index != (uint32)-1) + max_aux_stack_used = wasm_module_inst->e->max_aux_stack_used; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_common->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_module_inst = + (AOTModuleInstance *)module_inst_common; + AOTModule *aot_module = (AOTModule *)aot_module_inst->module; + module_common = (WASMModuleCommon *)aot_module; + if (aot_module_inst->memories) { + AOTMemoryInstance **memories = aot_module_inst->memories; + heap_handle = memories[0]->heap_handle; + } + aot_get_module_inst_mem_consumption(aot_module_inst, + &module_inst_mem_consps); + aot_get_module_mem_consumption(aot_module, &module_mem_consps); + } +#endif + + bh_assert(module_common != NULL); + + if (heap_handle) { + app_heap_peak_size = gc_get_heap_highmark_size(heap_handle); + } + + total_size = offsetof(WASMExecEnv, wasm_stack_u.bottom) + + exec_env->wasm_stack_size + module_mem_consps.total_size + + module_inst_mem_consps.total_size; + + os_printf("\nMemory consumption summary (bytes):\n"); + wasm_runtime_dump_module_mem_consumption(module_common); + wasm_runtime_dump_module_inst_mem_consumption(module_inst_common); + wasm_runtime_dump_exec_env_mem_consumption(exec_env); + os_printf("\nTotal memory consumption of module, module inst and " + "exec env: %" PRIu64 "\n", + total_size); + os_printf("Total interpreter stack used: %u\n", + exec_env->max_wasm_stack_used); + + if (max_aux_stack_used != (uint32)-1) + os_printf("Total auxiliary stack used: %u\n", max_aux_stack_used); + else + os_printf("Total aux stack used: no enough info to profile\n"); + + /* + * Report the native stack usage estimation. + * + * Unlike the aux stack above, we report the amount unused + * because we don't know the stack "bottom". + * + * Note that this is just about what the runtime itself observed. + * It doesn't cover host func implementations, signal handlers, etc. + */ + if (exec_env->native_stack_top_min != (void *)UINTPTR_MAX) + os_printf("Native stack left: %zd\n", + exec_env->native_stack_top_min + - exec_env->native_stack_boundary); + else + os_printf("Native stack left: no enough info to profile\n"); + + os_printf("Total app heap used: %u\n", app_heap_peak_size); +} +#endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \ + || (WASM_ENABLE_MEMORY_TRACING != 0) */ + +#if WASM_ENABLE_PERF_PROFILING != 0 +void +wasm_runtime_dump_perf_profiling(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_dump_perf_profiling((WASMModuleInstance *)module_inst); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_dump_perf_profiling((AOTModuleInstance *)module_inst); + } +#endif +} + +double +wasm_runtime_sum_wasm_exec_time(WASMModuleInstanceCommon *inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (inst->module_type == Wasm_Module_Bytecode) + return wasm_summarize_wasm_execute_time((WASMModuleInstance *)inst); +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst->module_type == Wasm_Module_AoT) + return aot_summarize_wasm_execute_time((AOTModuleInstance *)inst); +#endif + + return 0.0; +} + +double +wasm_runtime_get_wasm_func_exec_time(WASMModuleInstanceCommon *inst, + const char *func_name) +{ +#if WASM_ENABLE_INTERP != 0 + if (inst->module_type == Wasm_Module_Bytecode) + return wasm_get_wasm_func_exec_time((WASMModuleInstance *)inst, + func_name); +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst->module_type == Wasm_Module_AoT) + return aot_get_wasm_func_exec_time((AOTModuleInstance *)inst, + func_name); +#endif + + return 0.0; +} +#endif /* WASM_ENABLE_PERF_PROFILING != 0 */ + +WASMModuleInstanceCommon * +wasm_runtime_get_module_inst(WASMExecEnv *exec_env) +{ + return wasm_exec_env_get_module_inst(exec_env); +} + +void +wasm_runtime_set_module_inst(WASMExecEnv *exec_env, + WASMModuleInstanceCommon *const module_inst) +{ + wasm_exec_env_set_module_inst(exec_env, module_inst); +} + +void * +wasm_runtime_get_function_attachment(WASMExecEnv *exec_env) +{ + return exec_env->attachment; +} + +void +wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data) +{ + exec_env->user_data = user_data; +} + +void * +wasm_runtime_get_user_data(WASMExecEnv *exec_env) +{ + return exec_env->user_data; +} + +#ifdef OS_ENABLE_HW_BOUND_CHECK +void +wasm_runtime_access_exce_check_guard_page() +{ + if (exec_env_tls && exec_env_tls->handle == os_self_thread()) { + uint32 page_size = os_getpagesize(); + memset(exec_env_tls->exce_check_guard_page, 0, page_size); + } +} +#endif + +WASMFuncType * +wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function, + uint32 module_type) +{ + WASMFuncType *type = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module_type == Wasm_Module_Bytecode) { + WASMFunctionInstance *wasm_func = (WASMFunctionInstance *)function; + type = wasm_func->is_import_func ? wasm_func->u.func_import->func_type + : wasm_func->u.func->func_type; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_type == Wasm_Module_AoT) { + AOTFunctionInstance *aot_func = (AOTFunctionInstance *)function; + type = aot_func->is_import_func ? aot_func->u.func_import->func_type + : aot_func->u.func.func_type; + } +#endif + + return type; +} + +WASMFunctionInstanceCommon * +wasm_runtime_lookup_function(WASMModuleInstanceCommon *const module_inst, + const char *name) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return (WASMFunctionInstanceCommon *)wasm_lookup_function( + (const WASMModuleInstance *)module_inst, name); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return (WASMFunctionInstanceCommon *)aot_lookup_function( + (const AOTModuleInstance *)module_inst, name); +#endif + return NULL; +} + +uint32 +wasm_func_get_param_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + bh_assert(type); + + return type->param_count; +} + +uint32 +wasm_func_get_result_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + bh_assert(type); + + return type->result_count; +} + +static uint8 +val_type_to_val_kind(uint8 value_type) +{ + switch (value_type) { + case VALUE_TYPE_I32: + return WASM_I32; + case VALUE_TYPE_I64: + return WASM_I64; + case VALUE_TYPE_F32: + return WASM_F32; + case VALUE_TYPE_F64: + return WASM_F64; + case VALUE_TYPE_FUNCREF: + return WASM_FUNCREF; + case VALUE_TYPE_EXTERNREF: + return WASM_ANYREF; + default: + bh_assert(0); + return 0; + } +} + +void +wasm_func_get_param_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *param_types) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + uint32 i; + + bh_assert(type); + + for (i = 0; i < type->param_count; i++) { + param_types[i] = val_type_to_val_kind(type->types[i]); + } +} + +void +wasm_func_get_result_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *result_types) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + uint32 i; + + bh_assert(type); + + for (i = 0; i < type->result_count; i++) { + result_types[i] = + val_type_to_val_kind(type->types[type->param_count + i]); + } +} + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 +/* (uintptr_t)externref -> (uint32)index */ +/* argv -> *ret_argv */ +static bool +wasm_runtime_prepare_call_function(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 *argv, uint32 argc, uint32 **ret_argv, + uint32 *ret_argc_param, + uint32 *ret_argc_result) +{ + uint32 *new_argv = NULL, argv_i = 0, new_argv_i = 0, param_i = 0, + result_i = 0; + bool need_param_transform = false, need_result_transform = false; + uint64 size = 0; + WASMFuncType *func_type = wasm_runtime_get_function_type( + function, exec_env->module_inst->module_type); + + bh_assert(func_type); + + *ret_argc_param = func_type->param_cell_num; + *ret_argc_result = func_type->ret_cell_num; + for (param_i = 0; param_i < func_type->param_count; param_i++) { + if (VALUE_TYPE_EXTERNREF == func_type->types[param_i]) { + need_param_transform = true; + } + } + + for (result_i = 0; result_i < func_type->result_count; result_i++) { + if (VALUE_TYPE_EXTERNREF + == func_type->types[func_type->param_count + result_i]) { + need_result_transform = true; + } + } + + if (!need_param_transform && !need_result_transform) { + *ret_argv = argv; + return true; + } + + if (func_type->param_cell_num >= func_type->ret_cell_num) { + size = sizeof(uint32) * func_type->param_cell_num; + } + else { + size = sizeof(uint32) * func_type->ret_cell_num; + } + + if (!(new_argv = runtime_malloc(size, exec_env->module_inst, NULL, 0))) { + return false; + } + + if (!need_param_transform) { + bh_memcpy_s(new_argv, (uint32)size, argv, (uint32)size); + } + else { + for (param_i = 0; param_i < func_type->param_count && argv_i < argc + && new_argv_i < func_type->param_cell_num; + param_i++) { + uint8 param_type = func_type->types[param_i]; + if (VALUE_TYPE_EXTERNREF == param_type) { + void *externref_obj; + uint32 externref_index; + +#if UINTPTR_MAX == UINT32_MAX + externref_obj = (void *)argv[argv_i]; +#else + union { + uintptr_t val; + uint32 parts[2]; + } u; + + u.parts[0] = argv[argv_i]; + u.parts[1] = argv[argv_i + 1]; + externref_obj = (void *)u.val; +#endif + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_index)) { + wasm_runtime_free(new_argv); + return false; + } + + new_argv[new_argv_i] = externref_index; + argv_i += sizeof(uintptr_t) / sizeof(uint32); + new_argv_i++; + } + else { + uint16 param_cell_num = wasm_value_type_cell_num(param_type); + uint32 param_size = sizeof(uint32) * param_cell_num; + bh_memcpy_s(new_argv + new_argv_i, param_size, argv + argv_i, + param_size); + argv_i += param_cell_num; + new_argv_i += param_cell_num; + } + } + } + + *ret_argv = new_argv; + return true; +} + +/* (uintptr_t)externref <- (uint32)index */ +/* argv <- new_argv */ +static bool +wasm_runtime_finalize_call_function(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 *argv, uint32 argc, uint32 *ret_argv) +{ + uint32 argv_i = 0, result_i = 0, ret_argv_i = 0; + WASMFuncType *func_type; + + bh_assert((argv && ret_argv) || (argc == 0)); + + if (argv == ret_argv) { + /* no need to transfrom externref results */ + return true; + } + + func_type = wasm_runtime_get_function_type( + function, exec_env->module_inst->module_type); + bh_assert(func_type); + + for (result_i = 0; result_i < func_type->result_count && argv_i < argc; + result_i++) { + uint8 result_type = func_type->types[func_type->param_count + result_i]; + if (result_type == VALUE_TYPE_EXTERNREF) { + void *externref_obj; +#if UINTPTR_MAX != UINT32_MAX + union { + uintptr_t val; + uint32 parts[2]; + } u; +#endif + + if (!wasm_externref_ref2obj(argv[argv_i], &externref_obj)) { + wasm_runtime_free(argv); + return false; + } + +#if UINTPTR_MAX == UINT32_MAX + ret_argv[ret_argv_i] = (uintptr_t)externref_obj; +#else + u.val = (uintptr_t)externref_obj; + ret_argv[ret_argv_i] = u.parts[0]; + ret_argv[ret_argv_i + 1] = u.parts[1]; +#endif + argv_i += 1; + ret_argv_i += sizeof(uintptr_t) / sizeof(uint32); + } + else { + uint16 result_cell_num = wasm_value_type_cell_num(result_type); + uint32 result_size = sizeof(uint32) * result_cell_num; + bh_memcpy_s(ret_argv + ret_argv_i, result_size, argv + argv_i, + result_size); + argv_i += result_cell_num; + ret_argv_i += result_cell_num; + } + } + + wasm_runtime_free(argv); + return true; +} +#endif + +bool +wasm_runtime_call_wasm(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, uint32 argc, + uint32 argv[]) +{ + bool ret = false; + uint32 *new_argv = NULL, param_argc; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + uint32 result_argc = 0; +#endif + + if (!wasm_runtime_exec_env_check(exec_env)) { + LOG_ERROR("Invalid exec env stack info."); + return false; + } + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + if (!wasm_runtime_prepare_call_function(exec_env, function, argv, argc, + &new_argv, ¶m_argc, + &result_argc)) { + wasm_runtime_set_exception(exec_env->module_inst, + "the arguments conversion is failed"); + return false; + } +#else + new_argv = argv; + param_argc = argc; +#endif + +#if WASM_ENABLE_INTERP != 0 + if (exec_env->module_inst->module_type == Wasm_Module_Bytecode) + ret = wasm_call_function(exec_env, (WASMFunctionInstance *)function, + param_argc, new_argv); +#endif +#if WASM_ENABLE_AOT != 0 + if (exec_env->module_inst->module_type == Wasm_Module_AoT) + ret = aot_call_function(exec_env, (AOTFunctionInstance *)function, + param_argc, new_argv); +#endif + if (!ret) { + if (new_argv != argv) { + wasm_runtime_free(new_argv); + } + return false; + } + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + if (!wasm_runtime_finalize_call_function(exec_env, function, new_argv, + result_argc, argv)) { + wasm_runtime_set_exception(exec_env->module_inst, + "the result conversion is failed"); + return false; + } +#endif + + return ret; +} + +static void +parse_args_to_uint32_array(WASMFuncType *type, wasm_val_t *args, + uint32 *out_argv) +{ + uint32 i, p; + + for (i = 0, p = 0; i < type->param_count; i++) { + switch (args[i].kind) { + case WASM_I32: + out_argv[p++] = args[i].of.i32; + break; + case WASM_I64: + { + union { + uint64 val; + uint32 parts[2]; + } u; + u.val = args[i].of.i64; + out_argv[p++] = u.parts[0]; + out_argv[p++] = u.parts[1]; + break; + } + case WASM_F32: + { + union { + float32 val; + uint32 part; + } u; + u.val = args[i].of.f32; + out_argv[p++] = u.part; + break; + } + case WASM_F64: + { + union { + float64 val; + uint32 parts[2]; + } u; + u.val = args[i].of.f64; + out_argv[p++] = u.parts[0]; + out_argv[p++] = u.parts[1]; + break; + } +#if WASM_ENABLE_REF_TYPES != 0 +#if WASM_ENABLE_GC == 0 + case WASM_FUNCREF: + { + out_argv[p++] = args[i].of.i32; + break; + } +#else + case WASM_FUNCREF: +#endif + case WASM_ANYREF: + { +#if UINTPTR_MAX == UINT32_MAX + out_argv[p++] = args[i].of.foreign; +#else + union { + uintptr_t val; + uint32 parts[2]; + } u; + + u.val = (uintptr_t)args[i].of.foreign; + out_argv[p++] = u.parts[0]; + out_argv[p++] = u.parts[1]; +#endif + break; + } +#endif + default: + bh_assert(0); + break; + } + } +} + +static void +parse_uint32_array_to_results(WASMFuncType *type, uint32 *argv, + wasm_val_t *out_results) +{ + uint32 i, p; + + for (i = 0, p = 0; i < type->result_count; i++) { + switch (type->types[type->param_count + i]) { + case VALUE_TYPE_I32: + out_results[i].kind = WASM_I32; + out_results[i].of.i32 = (int32)argv[p++]; + break; + case VALUE_TYPE_I64: + { + union { + uint64 val; + uint32 parts[2]; + } u; + u.parts[0] = argv[p++]; + u.parts[1] = argv[p++]; + out_results[i].kind = WASM_I64; + out_results[i].of.i64 = u.val; + break; + } + case VALUE_TYPE_F32: + { + union { + float32 val; + uint32 part; + } u; + u.part = argv[p++]; + out_results[i].kind = WASM_F32; + out_results[i].of.f32 = u.val; + break; + } + case VALUE_TYPE_F64: + { + union { + float64 val; + uint32 parts[2]; + } u; + u.parts[0] = argv[p++]; + u.parts[1] = argv[p++]; + out_results[i].kind = WASM_F64; + out_results[i].of.f64 = u.val; + break; + } +#if WASM_ENABLE_REF_TYPES != 0 +#if WASM_ENABLE_GC == 0 + case VALUE_TYPE_FUNCREF: + { + out_results[i].kind = WASM_I32; + out_results[i].of.i32 = (int32)argv[p++]; + break; + } + case VALUE_TYPE_EXTERNREF: +#else + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#endif /* end of WASM_ENABLE_GC == 0 */ + { +#if UINTPTR_MAX == UINT32_MAX + out_results[i].kind = WASM_ANYREF; + out_results[i].of.foreign = (uintptr_t)argv[p++]; +#else + union { + uintptr_t val; + uint32 parts[2]; + } u; + u.parts[0] = argv[p++]; + u.parts[1] = argv[p++]; + out_results[i].kind = WASM_ANYREF; + out_results[i].of.foreign = u.val; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 */ + default: + bh_assert(0); + break; + } + } +} + +bool +wasm_runtime_call_wasm_a(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t results[], + uint32 num_args, wasm_val_t args[]) +{ + uint32 argc, argv_buf[16] = { 0 }, *argv = argv_buf, cell_num, module_type; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + uint32 i, param_size_in_double_world = 0, result_size_in_double_world = 0; +#endif + uint64 total_size; + WASMFuncType *type; + bool ret = false; + + module_type = exec_env->module_inst->module_type; + type = wasm_runtime_get_function_type(function, module_type); + + if (!type) { + LOG_ERROR("Function type get failed, WAMR Interpreter and AOT must be " + "enabled at least one."); + goto fail1; + } + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + for (i = 0; i < type->param_count; i++) { + param_size_in_double_world += + wasm_value_type_cell_num_outside(type->types[i]); + } + for (i = 0; i < type->result_count; i++) { + result_size_in_double_world += wasm_value_type_cell_num_outside( + type->types[type->param_count + i]); + } + argc = param_size_in_double_world; + cell_num = (argc >= result_size_in_double_world) + ? argc + : result_size_in_double_world; +#else + argc = type->param_cell_num; + cell_num = (argc > type->ret_cell_num) ? argc : type->ret_cell_num; +#endif + + if (num_results != type->result_count) { + LOG_ERROR( + "The result value number does not match the function declaration."); + goto fail1; + } + + if (num_args != type->param_count) { + LOG_ERROR("The argument value number does not match the function " + "declaration."); + goto fail1; + } + + total_size = sizeof(uint32) * (uint64)(cell_num > 2 ? cell_num : 2); + if (total_size > sizeof(argv_buf)) { + if (!(argv = + runtime_malloc(total_size, exec_env->module_inst, NULL, 0))) { + goto fail1; + } + } + + parse_args_to_uint32_array(type, args, argv); + if (!(ret = wasm_runtime_call_wasm(exec_env, function, argc, argv))) + goto fail2; + + parse_uint32_array_to_results(type, argv, results); + +fail2: + if (argv != argv_buf) + wasm_runtime_free(argv); +fail1: + return ret; +} + +bool +wasm_runtime_call_wasm_v(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t results[], + uint32 num_args, ...) +{ + wasm_val_t args_buf[8] = { 0 }, *args = args_buf; + WASMFuncType *type = NULL; + bool ret = false; + uint64 total_size; + uint32 i = 0, module_type; + va_list vargs; + + module_type = exec_env->module_inst->module_type; + type = wasm_runtime_get_function_type(function, module_type); + + if (!type) { + LOG_ERROR("Function type get failed, WAMR Interpreter and AOT " + "must be enabled at least one."); + goto fail1; + } + + if (num_args != type->param_count) { + LOG_ERROR("The argument value number does not match the " + "function declaration."); + goto fail1; + } + + total_size = sizeof(wasm_val_t) * (uint64)num_args; + if (total_size > sizeof(args_buf)) { + if (!(args = + runtime_malloc(total_size, exec_env->module_inst, NULL, 0))) { + goto fail1; + } + } + + va_start(vargs, num_args); + for (i = 0; i < num_args; i++) { + switch (type->types[i]) { + case VALUE_TYPE_I32: + args[i].kind = WASM_I32; + args[i].of.i32 = va_arg(vargs, uint32); + break; + case VALUE_TYPE_I64: + args[i].kind = WASM_I64; + args[i].of.i64 = va_arg(vargs, uint64); + break; + case VALUE_TYPE_F32: + args[i].kind = WASM_F32; + args[i].of.f32 = (float32)va_arg(vargs, float64); + break; + case VALUE_TYPE_F64: + args[i].kind = WASM_F64; + args[i].of.f64 = va_arg(vargs, float64); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + { + args[i].kind = WASM_FUNCREF; + args[i].of.i32 = va_arg(vargs, uint32); + break; + } + case VALUE_TYPE_EXTERNREF: + { + args[i].kind = WASM_ANYREF; + args[i].of.foreign = va_arg(vargs, uintptr_t); + break; + } +#endif + default: + bh_assert(0); + break; + } + } + va_end(vargs); + + ret = wasm_runtime_call_wasm_a(exec_env, function, num_results, results, + num_args, args); + if (args != args_buf) + wasm_runtime_free(args); + +fail1: + return ret; +} + +bool +wasm_runtime_create_exec_env_singleton( + WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMExecEnv *exec_env = NULL; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (module_inst->exec_env_singleton) { + return true; + } + + exec_env = wasm_exec_env_create(module_inst_comm, + module_inst->default_wasm_stack_size); + if (exec_env) + module_inst->exec_env_singleton = exec_env; + + return exec_env ? true : false; +} + +WASMExecEnv * +wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!module_inst->exec_env_singleton) { + wasm_runtime_create_exec_env_singleton(module_inst_comm); + } + return module_inst->exec_env_singleton; +} + +static void +wasm_set_exception_local(WASMModuleInstance *module_inst, const char *exception) +{ + exception_lock(module_inst); + if (exception) { + snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), + "Exception: %s", exception); + } + else { + module_inst->cur_exception[0] = '\0'; + } + exception_unlock(module_inst); +} + +void +wasm_set_exception(WASMModuleInstance *module_inst, const char *exception) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + WASMExecEnv *exec_env = + wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst); + if (exec_env) { + wasm_cluster_set_exception(exec_env, exception); + } + else { + wasm_set_exception_local(module_inst, exception); + } +#else + wasm_set_exception_local(module_inst, exception); +#endif +} + +/* clang-format off */ +static const char *exception_msgs[] = { + "unreachable", /* EXCE_UNREACHABLE */ + "allocate memory failed", /* EXCE_OUT_OF_MEMORY */ + "out of bounds memory access", /* EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS */ + "integer overflow", /* EXCE_INTEGER_OVERFLOW */ + "integer divide by zero", /* EXCE_INTEGER_DIVIDE_BY_ZERO */ + "invalid conversion to integer", /* EXCE_INVALID_CONVERSION_TO_INTEGER */ + "indirect call type mismatch", /* EXCE_INVALID_FUNCTION_TYPE_INDEX */ + "invalid function index", /* EXCE_INVALID_FUNCTION_INDEX */ + "undefined element", /* EXCE_UNDEFINED_ELEMENT */ + "uninitialized element", /* EXCE_UNINITIALIZED_ELEMENT */ + "failed to call unlinked import function", /* EXCE_CALL_UNLINKED_IMPORT_FUNC */ + "native stack overflow", /* EXCE_NATIVE_STACK_OVERFLOW */ + "unaligned atomic", /* EXCE_UNALIGNED_ATOMIC */ + "wasm auxiliary stack overflow", /* EXCE_AUX_STACK_OVERFLOW */ + "wasm auxiliary stack underflow", /* EXCE_AUX_STACK_UNDERFLOW */ + "out of bounds table access", /* EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */ + "wasm operand stack overflow", /* EXCE_OPERAND_STACK_OVERFLOW */ + "failed to compile fast jit function", /* EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC */ + /* GC related exceptions */ + "null function object", /* EXCE_NULL_FUNC_OBJ */ + "null structure object", /* EXCE_NULL_STRUCT_OBJ */ + "null array reference", /* EXCE_NULL_ARRAY_OBJ */ + "null i31 reference", /* EXCE_NULL_I31_OBJ */ + "null reference", /* EXCE_NULL_REFERENCE */ + "create rtt type failed", /* EXCE_FAILED_TO_CREATE_RTT_TYPE */ + "create struct object failed", /* EXCE_FAILED_TO_CREATE_STRUCT_OBJ */ + "create array object failed", /* EXCE_FAILED_TO_CREATE_ARRAY_OBJ */ + "create externref object failed", /* EXCE_FAILED_TO_CREATE_EXTERNREF_OBJ */ + "cast failure", /* EXCE_CAST_FAILURE */ + "out of bounds array access", /* EXCE_ARRAY_IDX_OOB */ + /* stringref related exceptions */ + "create string object failed", /* EXCE_FAILED_TO_CREATE_STRING */ + "create stringref failed", /* EXCE_FAILED_TO_CREATE_STRINGREF */ + "create stringview failed", /* EXCE_FAILED_TO_CREATE_STRINGVIEW */ + "encode failed", /* EXCE_FAILED_TO_ENCODE_STRING */ + "", /* EXCE_ALREADY_THROWN */ +}; +/* clang-format on */ + +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + if (id < EXCE_NUM) + wasm_set_exception(module_inst, exception_msgs[id]); + else + wasm_set_exception(module_inst, "unknown exception"); +} + +const char * +wasm_get_exception(WASMModuleInstance *module_inst) +{ + if (module_inst->cur_exception[0] == '\0') + return NULL; + else + return module_inst->cur_exception; +} + +bool +wasm_copy_exception(WASMModuleInstance *module_inst, char *exception_buf) +{ + bool has_exception = false; + + exception_lock(module_inst); + if (module_inst->cur_exception[0] != '\0') { + /* NULL is passed if the caller is not interested in getting the + * exception content, but only in knowing if an exception has been + * raised + */ + if (exception_buf != NULL) + bh_memcpy_s(exception_buf, sizeof(module_inst->cur_exception), + module_inst->cur_exception, + sizeof(module_inst->cur_exception)); + has_exception = true; + } + exception_unlock(module_inst); + + return has_exception; +} + +void +wasm_runtime_set_exception(WASMModuleInstanceCommon *module_inst_comm, + const char *exception) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_set_exception(module_inst, exception); +} + +const char * +wasm_runtime_get_exception(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return wasm_get_exception(module_inst); +} + +bool +wasm_runtime_copy_exception(WASMModuleInstanceCommon *module_inst_comm, + char *exception_buf) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return wasm_copy_exception(module_inst, exception_buf); +} + +void +wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst_comm) +{ + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_runtime_set_exception(module_inst_comm, NULL); +} + +void +wasm_runtime_terminate(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_set_exception(module_inst, "terminated by user"); +} + +void +wasm_runtime_set_custom_data_internal( + WASMModuleInstanceCommon *module_inst_comm, void *custom_data) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + module_inst->custom_data = custom_data; +} + +void +wasm_runtime_set_custom_data(WASMModuleInstanceCommon *module_inst, + void *custom_data) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_spread_custom_data(module_inst, custom_data); +#else + wasm_runtime_set_custom_data_internal(module_inst, custom_data); +#endif +} + +void * +wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return module_inst->custom_data; +} + +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 +void +wasm_runtime_set_bounds_checks(WASMModuleInstanceCommon *module_inst, + bool enable) +{ + /* Alwary disable bounds checks if hw bounds checks enabled */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + enable = false; +#endif +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e) + ->common.disable_bounds_checks = enable ? false : true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->common.disable_bounds_checks = enable ? false : true; + } +#endif +} + +bool +wasm_runtime_is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) +{ + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return !((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst) + ->e) + ->common.disable_bounds_checks; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return !((AOTModuleInstanceExtra *)((WASMModuleInstance *)module_inst) + ->e) + ->common.disable_bounds_checks; + } +#endif + + return true; +} +#endif + +uint64 +wasm_runtime_module_malloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_malloc_internal((WASMModuleInstance *)module_inst, + exec_env, size, p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_malloc_internal((AOTModuleInstance *)module_inst, + exec_env, size, p_native_addr); +#endif + return 0; +} + +uint64 +wasm_runtime_module_realloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr, + uint64 size, void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_realloc_internal((WASMModuleInstance *)module_inst, + exec_env, ptr, size, p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_realloc_internal((AOTModuleInstance *)module_inst, + exec_env, ptr, size, p_native_addr); +#endif + return 0; +} + +void +wasm_runtime_module_free_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_module_free_internal((WASMModuleInstance *)module_inst, exec_env, + ptr); + return; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_module_free_internal((AOTModuleInstance *)module_inst, exec_env, + ptr); + return; + } +#endif +} + +uint64 +wasm_runtime_module_malloc(WASMModuleInstanceCommon *module_inst, uint64 size, + void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_malloc((WASMModuleInstance *)module_inst, size, + p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_malloc((AOTModuleInstance *)module_inst, size, + p_native_addr); +#endif + return 0; +} + +uint64 +wasm_runtime_module_realloc(WASMModuleInstanceCommon *module_inst, uint64 ptr, + uint64 size, void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_realloc((WASMModuleInstance *)module_inst, ptr, size, + p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_realloc((AOTModuleInstance *)module_inst, ptr, size, + p_native_addr); +#endif + return 0; +} + +void +wasm_runtime_module_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_module_free((WASMModuleInstance *)module_inst, ptr); + return; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_module_free((AOTModuleInstance *)module_inst, ptr); + return; + } +#endif +} + +uint64 +wasm_runtime_module_dup_data(WASMModuleInstanceCommon *module_inst, + const char *src, uint64 size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_module_dup_data((WASMModuleInstance *)module_inst, src, + size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_module_dup_data((AOTModuleInstance *)module_inst, src, size); + } +#endif + return 0; +} + +#if WASM_ENABLE_LIBC_WASI != 0 + +static WASIArguments * +get_wasi_args_from_module(wasm_module_t module) +{ + WASIArguments *wasi_args = NULL; + +#if WASM_ENABLE_INTERP != 0 || WASM_ENABLE_JIT != 0 + if (module->module_type == Wasm_Module_Bytecode) + wasi_args = &((WASMModule *)module)->wasi_args; +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + wasi_args = &((AOTModule *)module)->wasi_args; +#endif + + return wasi_args; +} + +void +wasm_runtime_set_wasi_args_ex(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc, + int64 stdinfd, int64 stdoutfd, int64 stderrfd) +{ + WASIArguments *wasi_args = get_wasi_args_from_module(module); + + bh_assert(wasi_args); + + wasi_args->dir_list = dir_list; + wasi_args->dir_count = dir_count; + wasi_args->map_dir_list = map_dir_list; + wasi_args->map_dir_count = map_dir_count; + wasi_args->env = env_list; + wasi_args->env_count = env_count; + wasi_args->argv = argv; + wasi_args->argc = (uint32)argc; + wasi_args->stdio[0] = (os_raw_file_handle)stdinfd; + wasi_args->stdio[1] = (os_raw_file_handle)stdoutfd; + wasi_args->stdio[2] = (os_raw_file_handle)stderrfd; + +#if WASM_ENABLE_MULTI_MODULE != 0 +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + wasm_propagate_wasi_args((WASMModule *)module); + } +#endif +#endif +} + +void +wasm_runtime_set_wasi_args(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc) +{ + wasm_runtime_set_wasi_args_ex(module, dir_list, dir_count, map_dir_list, + map_dir_count, env_list, env_count, argv, + argc, -1, -1, -1); +} + +void +wasm_runtime_set_wasi_addr_pool(wasm_module_t module, const char *addr_pool[], + uint32 addr_pool_size) +{ + WASIArguments *wasi_args = get_wasi_args_from_module(module); + + if (wasi_args) { + wasi_args->addr_pool = addr_pool; + wasi_args->addr_count = addr_pool_size; + } +} + +void +wasm_runtime_set_wasi_ns_lookup_pool(wasm_module_t module, + const char *ns_lookup_pool[], + uint32 ns_lookup_pool_size) +{ + WASIArguments *wasi_args = get_wasi_args_from_module(module); + + if (wasi_args) { + wasi_args->ns_lookup_pool = ns_lookup_pool; + wasi_args->ns_lookup_count = ns_lookup_pool_size; + } +} + +#if WASM_ENABLE_UVWASI == 0 +static bool +copy_string_array(const char *array[], uint32 array_size, char **buf_ptr, + char ***list_ptr, uint64 *out_buf_size) +{ + uint64 buf_size = 0, total_size; + uint32 buf_offset = 0, i; + char *buf = NULL, **list = NULL; + + for (i = 0; i < array_size; i++) + buf_size += strlen(array[i]) + 1; + + /* We add +1 to generate null-terminated array of strings */ + total_size = sizeof(char *) * ((uint64)array_size + 1); + if (total_size >= UINT32_MAX + /* total_size must be larger than 0, don' check it again */ + || !(list = wasm_runtime_malloc((uint32)total_size)) + || buf_size >= UINT32_MAX + || (buf_size > 0 && !(buf = wasm_runtime_malloc((uint32)buf_size)))) { + + if (buf) + wasm_runtime_free(buf); + if (list) + wasm_runtime_free(list); + return false; + } + + for (i = 0; i < array_size; i++) { + list[i] = buf + buf_offset; + bh_strcpy_s(buf + buf_offset, (uint32)buf_size - buf_offset, array[i]); + buf_offset += (uint32)(strlen(array[i]) + 1); + } + list[array_size] = NULL; + + *list_ptr = list; + *buf_ptr = buf; + if (out_buf_size) + *out_buf_size = buf_size; + + return true; +} + +bool +wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, + const char *dir_list[], uint32 dir_count, + const char *map_dir_list[], uint32 map_dir_count, + const char *env[], uint32 env_count, + const char *addr_pool[], uint32 addr_pool_size, + const char *ns_lookup_pool[], uint32 ns_lookup_pool_size, + char *argv[], uint32 argc, os_raw_file_handle stdinfd, + os_raw_file_handle stdoutfd, os_raw_file_handle stderrfd, + char *error_buf, uint32 error_buf_size) +{ + WASIContext *wasi_ctx; + char *argv_buf = NULL; + char **argv_list = NULL; + char *env_buf = NULL; + char **env_list = NULL; + char *ns_lookup_buf = NULL; + char **ns_lookup_list = NULL; + uint64 argv_buf_size = 0, env_buf_size = 0; + struct fd_table *curfds = NULL; + struct fd_prestats *prestats = NULL; + struct argv_environ_values *argv_environ = NULL; + struct addr_pool *apool = NULL; + bool fd_table_inited = false, fd_prestats_inited = false; + bool argv_environ_inited = false; + bool addr_pool_inited = false; + __wasi_fd_t wasm_fd = 3; + os_file_handle file_handle; + char *path, resolved_path[PATH_MAX]; + uint32 i; + + if (!(wasi_ctx = runtime_malloc(sizeof(WASIContext), NULL, error_buf, + error_buf_size))) { + return false; + } + + wasm_runtime_set_wasi_ctx(module_inst, wasi_ctx); + + /* process argv[0], trip the path and suffix, only keep the program name + */ + if (!copy_string_array((const char **)argv, argc, &argv_buf, &argv_list, + &argv_buf_size)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + if (!copy_string_array(env, env_count, &env_buf, &env_list, + &env_buf_size)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + if (!(curfds = wasm_runtime_malloc(sizeof(struct fd_table))) + || !(prestats = wasm_runtime_malloc(sizeof(struct fd_prestats))) + || !(argv_environ = + wasm_runtime_malloc(sizeof(struct argv_environ_values))) + || !(apool = wasm_runtime_malloc(sizeof(struct addr_pool)))) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + if (!fd_table_init(curfds)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init fd table failed"); + goto fail; + } + fd_table_inited = true; + + if (!fd_prestats_init(prestats)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init fd prestats failed"); + goto fail; + } + fd_prestats_inited = true; + + if (!argv_environ_init(argv_environ, argv_buf, argv_buf_size, argv_list, + argc, env_buf, env_buf_size, env_list, env_count)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init argument environment failed"); + goto fail; + } + argv_environ_inited = true; + + if (!addr_pool_init(apool)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init the address pool failed"); + goto fail; + } + addr_pool_inited = true; + + os_file_handle stdin_file_handle = os_convert_stdin_handle(stdinfd); + os_file_handle stdout_file_handle = os_convert_stdout_handle(stdoutfd); + os_file_handle stderr_file_handle = os_convert_stderr_handle(stderrfd); + + if (!os_is_handle_valid(&stdin_file_handle) + || !os_is_handle_valid(&stdout_file_handle) + || !os_is_handle_valid(&stderr_file_handle)) + goto fail; + + /* Prepopulate curfds with stdin, stdout, and stderr file descriptors. */ + if (!fd_table_insert_existing(curfds, 0, stdin_file_handle, true) + || !fd_table_insert_existing(curfds, 1, stdout_file_handle, true) + || !fd_table_insert_existing(curfds, 2, stderr_file_handle, true)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: init fd table failed"); + goto fail; + } + + wasm_fd = 3; + for (i = 0; i < dir_count; i++, wasm_fd++) { + path = os_realpath(dir_list[i], resolved_path); + if (!path) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening directory %s: %d\n", + dir_list[i], errno); + goto fail; + } + + __wasi_errno_t error = os_open_preopendir(path, &file_handle); + + if (error != __WASI_ESUCCESS) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening directory %s: %d\n", + dir_list[i], error); + goto fail; + } + + if (!fd_table_insert_existing(curfds, wasm_fd, file_handle, false)) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error inserting preopen fd %u (directory %s) into fd " + "table", + (unsigned int)wasm_fd, dir_list[i]); + goto fail; + } + + if (!fd_prestats_insert(prestats, dir_list[i], wasm_fd)) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error inserting preopen fd %u (directory %s) into " + "prestats table", + (unsigned int)wasm_fd, dir_list[i]); + goto fail; + } + } + + for (i = 0; i < map_dir_count; i++, wasm_fd++) { + char mapping_copy_buf[256]; + char *mapping_copy = mapping_copy_buf; + char *map_mapped = NULL, *map_host = NULL; + const unsigned long max_len = strlen(map_dir_list[i]) * 2 + 3; + + /* Allocation limit for runtime environments with reduced stack size */ + if (max_len > 256) { + if (!(mapping_copy = wasm_runtime_malloc(max_len))) { + snprintf(error_buf, error_buf_size, + "error while allocating for directory mapping\n"); + goto fail; + } + } + + bh_memcpy_s(mapping_copy, max_len, map_dir_list[i], + (uint32)(strlen(map_dir_list[i]) + 1)); + map_mapped = strtok(mapping_copy, "::"); + map_host = strtok(NULL, "::"); + + if (!map_mapped || !map_host) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory: " + "invalid map\n"); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + path = os_realpath(map_host, resolved_path); + if (!path) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory %s: %d\n", + map_host, errno); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + __wasi_errno_t error = os_open_preopendir(path, &file_handle); + if (error != __WASI_ESUCCESS) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory %s: %d\n", + map_host, errno); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + if (!fd_table_insert_existing(curfds, wasm_fd, file_handle, false) + || !fd_prestats_insert(prestats, map_mapped, wasm_fd)) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory %s: " + "insertion failed\n", + dir_list[i]); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + } + + /* addr_pool(textual) -> apool */ + for (i = 0; i < addr_pool_size; i++) { + char *cp, *address, *mask; + bool ret = false; + + cp = bh_strdup(addr_pool[i]); + if (!cp) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: copy address failed"); + goto fail; + } + + address = strtok(cp, "/"); + mask = strtok(NULL, "/"); + + ret = addr_pool_insert(apool, address, (uint8)(mask ? atoi(mask) : 0)); + wasm_runtime_free(cp); + if (!ret) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: store address failed"); + goto fail; + } + } + + if (!copy_string_array(ns_lookup_pool, ns_lookup_pool_size, &ns_lookup_buf, + &ns_lookup_list, NULL)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + wasi_ctx->curfds = curfds; + wasi_ctx->prestats = prestats; + wasi_ctx->argv_environ = argv_environ; + wasi_ctx->addr_pool = apool; + wasi_ctx->argv_buf = argv_buf; + wasi_ctx->argv_list = argv_list; + wasi_ctx->env_buf = env_buf; + wasi_ctx->env_list = env_list; + wasi_ctx->ns_lookup_buf = ns_lookup_buf; + wasi_ctx->ns_lookup_list = ns_lookup_list; + + return true; + +fail: + if (argv_environ_inited) + argv_environ_destroy(argv_environ); + if (fd_prestats_inited) + fd_prestats_destroy(prestats); + if (fd_table_inited) + fd_table_destroy(curfds); + if (addr_pool_inited) + addr_pool_destroy(apool); + if (curfds) + wasm_runtime_free(curfds); + if (prestats) + wasm_runtime_free(prestats); + if (argv_environ) + wasm_runtime_free(argv_environ); + if (apool) + wasm_runtime_free(apool); + if (argv_buf) + wasm_runtime_free(argv_buf); + if (argv_list) + wasm_runtime_free(argv_list); + if (env_buf) + wasm_runtime_free(env_buf); + if (env_list) + wasm_runtime_free(env_list); + if (ns_lookup_buf) + wasm_runtime_free(ns_lookup_buf); + if (ns_lookup_list) + wasm_runtime_free(ns_lookup_list); + return false; +} +#else /* else of WASM_ENABLE_UVWASI == 0 */ +static void * +wasm_uvwasi_malloc(size_t size, void *mem_user_data) +{ + return runtime_malloc(size, NULL, NULL, 0); + (void)mem_user_data; +} + +static void +wasm_uvwasi_free(void *ptr, void *mem_user_data) +{ + if (ptr) + wasm_runtime_free(ptr); + (void)mem_user_data; +} + +static void * +wasm_uvwasi_calloc(size_t nmemb, size_t size, void *mem_user_data) +{ + uint64 total_size = (uint64)nmemb * size; + return runtime_malloc(total_size, NULL, NULL, 0); + (void)mem_user_data; +} + +static void * +wasm_uvwasi_realloc(void *ptr, size_t size, void *mem_user_data) +{ + if (size >= UINT32_MAX) { + return NULL; + } + return wasm_runtime_realloc(ptr, (uint32)size); +} + +/* clang-format off */ +static uvwasi_mem_t uvwasi_allocator = { + .mem_user_data = 0, + .malloc = wasm_uvwasi_malloc, + .free = wasm_uvwasi_free, + .calloc = wasm_uvwasi_calloc, + .realloc = wasm_uvwasi_realloc +}; +/* clang-format on */ + +bool +wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, + const char *dir_list[], uint32 dir_count, + const char *map_dir_list[], uint32 map_dir_count, + const char *env[], uint32 env_count, + const char *addr_pool[], uint32 addr_pool_size, + const char *ns_lookup_pool[], uint32 ns_lookup_pool_size, + char *argv[], uint32 argc, os_raw_file_handle stdinfd, + os_raw_file_handle stdoutfd, os_raw_file_handle stderrfd, + char *error_buf, uint32 error_buf_size) +{ + WASIContext *ctx; + uvwasi_t *uvwasi; + uvwasi_options_t init_options; + const char **envp = NULL; + uint64 total_size; + uint32 i; + bool ret = false; + + ctx = runtime_malloc(sizeof(*ctx), module_inst, error_buf, error_buf_size); + if (!ctx) + return false; + uvwasi = &ctx->uvwasi; + + /* Setup the initialization options */ + uvwasi_options_init(&init_options); + init_options.allocator = &uvwasi_allocator; + init_options.argc = argc; + init_options.argv = (const char **)argv; + init_options.in = (stdinfd != -1) ? (uvwasi_fd_t)stdinfd : init_options.in; + init_options.out = + (stdoutfd != -1) ? (uvwasi_fd_t)stdoutfd : init_options.out; + init_options.err = + (stderrfd != -1) ? (uvwasi_fd_t)stderrfd : init_options.err; + + if (dir_count > 0) { + init_options.preopenc = dir_count; + + total_size = sizeof(uvwasi_preopen_t) * (uint64)init_options.preopenc; + init_options.preopens = (uvwasi_preopen_t *)runtime_malloc( + total_size, module_inst, error_buf, error_buf_size); + if (init_options.preopens == NULL) + goto fail; + + for (i = 0; i < init_options.preopenc; i++) { + init_options.preopens[i].real_path = dir_list[i]; + init_options.preopens[i].mapped_path = + (i < map_dir_count) ? map_dir_list[i] : dir_list[i]; + } + } + + if (env_count > 0) { + total_size = sizeof(char *) * (uint64)(env_count + 1); + envp = + runtime_malloc(total_size, module_inst, error_buf, error_buf_size); + if (envp == NULL) + goto fail; + + for (i = 0; i < env_count; i++) { + envp[i] = env[i]; + } + envp[env_count] = NULL; + init_options.envp = envp; + } + + if (UVWASI_ESUCCESS != uvwasi_init(uvwasi, &init_options)) { + set_error_buf(error_buf, error_buf_size, "uvwasi init failed"); + goto fail; + } + + wasm_runtime_set_wasi_ctx(module_inst, ctx); + + ret = true; + +fail: + if (envp) + wasm_runtime_free((void *)envp); + + if (init_options.preopens) + wasm_runtime_free(init_options.preopens); + + if (!ret && uvwasi) + wasm_runtime_free(uvwasi); + + return ret; +} +#endif /* end of WASM_ENABLE_UVWASI */ + +bool +wasm_runtime_is_wasi_mode(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode + && ((WASMModuleInstance *)module_inst)->module->import_wasi_api) + return true; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT + && ((AOTModule *)((AOTModuleInstance *)module_inst)->module) + ->import_wasi_api) + return true; +#endif + return false; +} + +WASMFunctionInstanceCommon * +wasm_runtime_lookup_wasi_start_function(WASMModuleInstanceCommon *module_inst) +{ + uint32 i; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *wasm_inst = (WASMModuleInstance *)module_inst; + WASMFunctionInstance *func; + for (i = 0; i < wasm_inst->export_func_count; i++) { + if (!strcmp(wasm_inst->export_functions[i].name, "_start")) { + func = wasm_inst->export_functions[i].function; + if (func->u.func->func_type->param_count != 0 + || func->u.func->func_type->result_count != 0) { + LOG_ERROR("Lookup wasi _start function failed: " + "invalid function type.\n"); + return NULL; + } + return (WASMFunctionInstanceCommon *)func; + } + } + return NULL; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; + AOTFunctionInstance *export_funcs = + (AOTFunctionInstance *)aot_inst->export_functions; + for (i = 0; i < aot_inst->export_func_count; i++) { + if (!strcmp(export_funcs[i].func_name, "_start")) { + AOTFuncType *func_type = export_funcs[i].u.func.func_type; + if (func_type->param_count != 0 + || func_type->result_count != 0) { + LOG_ERROR("Lookup wasi _start function failed: " + "invalid function type.\n"); + return NULL; + } + return (WASMFunctionInstanceCommon *)&export_funcs[i]; + } + } + return NULL; + } +#endif /* end of WASM_ENABLE_AOT */ + + return NULL; +} + +#if WASM_ENABLE_UVWASI == 0 +void +wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst) +{ + WASIContext *wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst); + + if (wasi_ctx) { + if (wasi_ctx->argv_environ) { + argv_environ_destroy(wasi_ctx->argv_environ); + wasm_runtime_free(wasi_ctx->argv_environ); + } + if (wasi_ctx->curfds) { + fd_table_destroy(wasi_ctx->curfds); + wasm_runtime_free(wasi_ctx->curfds); + } + if (wasi_ctx->prestats) { + fd_prestats_destroy(wasi_ctx->prestats); + wasm_runtime_free(wasi_ctx->prestats); + } + if (wasi_ctx->addr_pool) { + addr_pool_destroy(wasi_ctx->addr_pool); + wasm_runtime_free(wasi_ctx->addr_pool); + } + if (wasi_ctx->argv_buf) + wasm_runtime_free(wasi_ctx->argv_buf); + if (wasi_ctx->argv_list) + wasm_runtime_free(wasi_ctx->argv_list); + if (wasi_ctx->env_buf) + wasm_runtime_free(wasi_ctx->env_buf); + if (wasi_ctx->env_list) + wasm_runtime_free(wasi_ctx->env_list); + if (wasi_ctx->ns_lookup_buf) + wasm_runtime_free(wasi_ctx->ns_lookup_buf); + if (wasi_ctx->ns_lookup_list) + wasm_runtime_free(wasi_ctx->ns_lookup_list); + + wasm_runtime_free(wasi_ctx); + } +} +#else +void +wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst) +{ + WASIContext *wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst); + + if (wasi_ctx) { + uvwasi_destroy(&wasi_ctx->uvwasi); + wasm_runtime_free(wasi_ctx); + } +} +#endif + +uint32_t +wasm_runtime_get_wasi_exit_code(WASMModuleInstanceCommon *module_inst) +{ + WASIContext *wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst); +#if WASM_ENABLE_THREAD_MGR != 0 + WASMCluster *cluster; + WASMExecEnv *exec_env; + + exec_env = wasm_runtime_get_exec_env_singleton(module_inst); + if (exec_env && (cluster = wasm_exec_env_get_cluster(exec_env))) { + /** + * The main thread may exit earlier than other threads, and + * the exit_code of wasi_ctx may be changed by other thread + * when it runs into wasi_proc_exit, here we wait until all + * other threads exit to avoid getting invalid exit_code. + */ + wasm_cluster_wait_for_all_except_self(cluster, exec_env); + } +#endif + return wasi_ctx->exit_code; +} +#endif /* end of WASM_ENABLE_LIBC_WASI */ + +WASMModuleCommon * +wasm_exec_env_get_module(WASMExecEnv *exec_env) +{ + WASMModuleInstanceCommon *module_inst_comm = + wasm_runtime_get_module_inst(exec_env); + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return (WASMModuleCommon *)module_inst->module; +} + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +const uint8 * +wasm_runtime_get_custom_section(WASMModuleCommon *const module_comm, + const char *name, uint32 *len) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) + return wasm_loader_get_custom_section((WASMModule *)module_comm, name, + len); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) + return aot_get_custom_section((AOTModule *)module_comm, name, len); +#endif + return NULL; +} +#endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 */ + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) /* NOLINT */ + +int32 +wasm_runtime_get_import_count(WASMModuleCommon *const module) +{ + if (!module) { + bh_assert(0); + return -1; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + return (int32)(aot_module->import_func_count + + aot_module->import_global_count + + aot_module->import_table_count + + aot_module->import_memory_count); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + return (int32)wasm_module->import_count; + } +#endif + + return -1; +} + +void +wasm_runtime_get_import_type(WASMModuleCommon *const module, int32 import_index, + wasm_import_type *import_type) +{ + if (!import_type) { + bh_assert(0); + return; + } + + memset(import_type, 0, sizeof(wasm_import_type)); + + if (!module) { + bh_assert(0); + return; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + + uint32 func_index = (uint32)import_index; + if (func_index < aot_module->import_func_count) { + const AOTImportFunc *aot_import_func = + &aot_module->import_funcs[func_index]; + import_type->module_name = aot_import_func->module_name; + import_type->name = aot_import_func->func_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_FUNC; + import_type->linked = + aot_import_func->func_ptr_linked ? true : false; + return; + } + + uint32 global_index = func_index - aot_module->import_func_count; + if (global_index < aot_module->import_global_count) { + const AOTImportGlobal *aot_import_global = + &aot_module->import_globals[global_index]; + import_type->module_name = aot_import_global->module_name; + import_type->name = aot_import_global->global_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_GLOBAL; + import_type->linked = aot_import_global->is_linked; + return; + } + + uint32 table_index = global_index - aot_module->import_global_count; + if (table_index < aot_module->import_table_count) { + const AOTImportTable *aot_import_table = + &aot_module->import_tables[table_index]; + import_type->module_name = aot_import_table->module_name; + import_type->name = aot_import_table->table_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_TABLE; + import_type->linked = false; + return; + } + + uint32 memory_index = table_index - aot_module->import_table_count; + if (memory_index < aot_module->import_memory_count) { + const AOTImportMemory *aot_import_memory = + &aot_module->import_memories[memory_index]; + import_type->module_name = aot_import_memory->module_name; + import_type->name = aot_import_memory->memory_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_MEMORY; + import_type->linked = false; + return; + } + + bh_assert(0); + return; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + + if ((uint32)import_index >= wasm_module->import_count) { + bh_assert(0); + return; + } + + const WASMImport *wasm_import = &wasm_module->imports[import_index]; + + import_type->module_name = wasm_import->u.names.module_name; + import_type->name = wasm_import->u.names.field_name; + import_type->kind = wasm_import->kind; + switch (import_type->kind) { + case WASM_IMPORT_EXPORT_KIND_FUNC: + import_type->linked = wasm_import->u.function.func_ptr_linked; + break; + case WASM_IMPORT_EXPORT_KIND_GLOBAL: + import_type->linked = wasm_import->u.global.is_linked; + break; + case WASM_IMPORT_EXPORT_KIND_TABLE: + /* not supported */ + import_type->linked = false; + break; + case WASM_IMPORT_EXPORT_KIND_MEMORY: + /* not supported */ + import_type->linked = false; + break; + default: + bh_assert(0); + break; + } + + return; + } +#endif +} + +int32 +wasm_runtime_get_export_count(WASMModuleCommon *const module) +{ + if (!module) { + bh_assert(0); + return -1; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + return (int32)aot_module->export_count; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + return (int32)wasm_module->export_count; + } +#endif + + return -1; +} + +void +wasm_runtime_get_export_type(WASMModuleCommon *const module, int32 export_index, + wasm_export_type *export_type) +{ + if (!export_type) { + bh_assert(0); + return; + } + + memset(export_type, 0, sizeof(wasm_export_type)); + + if (!module) { + bh_assert(0); + return; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + + if ((uint32)export_index >= aot_module->export_count) { + bh_assert(0); + return; + } + + const AOTExport *aot_export = &aot_module->exports[export_index]; + export_type->name = aot_export->name; + export_type->kind = aot_export->kind; + return; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + + if ((uint32)export_index >= wasm_module->export_count) { + bh_assert(0); + return; + } + + const WASMExport *wasm_export = &wasm_module->exports[export_index]; + export_type->name = wasm_export->name; + export_type->kind = wasm_export->kind; + return; + } +#endif +} + +bool +wasm_runtime_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols) +{ + return wasm_native_register_natives(module_name, native_symbols, + n_native_symbols); +} + +bool +wasm_runtime_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols) +{ + return wasm_native_register_natives_raw(module_name, native_symbols, + n_native_symbols); +} + +bool +wasm_runtime_unregister_natives(const char *module_name, + NativeSymbol *native_symbols) +{ + return wasm_native_unregister_natives(module_name, native_symbols); +} + +bool +wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, + const char *signature, void *attachment, + uint32 *argv, uint32 argc, uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); + typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *); + NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr; + uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size, arg_i64; + uint32 *argv_src = argv, i, argc1, ptr_len; + uint32 arg_i32; + bool ret = false; + + argc1 = func_type->param_count; + if (argc1 > sizeof(argv_buf) / sizeof(uint64)) { + size = sizeof(uint64) * (uint64)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + + argv_dst = argv1; + + /* Traverse secondly to fill in each argument */ + for (i = 0; i < func_type->param_count; i++, argv_dst++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + *(uint32 *)argv_dst = arg_i32 = *argv_src++; + /* TODO: memory64 if future there is a way for supporting + * wasm64 and wasm32 in libc at the same time, remove the + * macro control */ +#if WASM_ENABLE_MEMORY64 == 0 + if (signature) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + *(uintptr_t *)argv_dst = + (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + *(uintptr_t *)argv_dst = + (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } +#endif + break; + } + case VALUE_TYPE_I64: +#if WASM_ENABLE_MEMORY64 != 0 + { + PUT_I64_TO_ADDR((uint32 *)argv_dst, + GET_I64_FROM_ADDR(argv_src)); + argv_src += 2; + arg_i64 = *argv_dst; + if (signature) { + /* TODO: memory64 pointer with length need a new symbol + * to represent type i64, with '~' still represent i32 + * length */ + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr(module, arg_i64, + (uint64)ptr_len)) + goto fail; + + *argv_dst = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr(module, + arg_i64)) + goto fail; + + *argv_dst = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + } + break; + } +#endif + case VALUE_TYPE_F64: + bh_memcpy_s(argv_dst, sizeof(uint64), argv_src, + sizeof(uint32) * 2); + argv_src += 2; + break; + case VALUE_TYPE_F32: + *(float32 *)argv_dst = *(float32 *)argv_src++; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv_src++; + + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + bh_memcpy_s(argv_dst, sizeof(uintptr_t), argv_src, + sizeof(uintptr_t)); + break; + } +#endif +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + { + bh_memcpy_s(argv_dst, sizeof(uintptr_t), argv_src, + sizeof(uintptr_t)); + argv_src += sizeof(uintptr_t) / sizeof(uint32); + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + exec_env->attachment = attachment; + invoke_native_raw(exec_env, argv1); + exec_env->attachment = NULL; + + if (func_type->result_count > 0) { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = *(uint32 *)argv1; + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = *(float32 *)argv1; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + bh_memcpy_s(argv_ret, sizeof(uint32) * 2, argv1, + sizeof(uint64)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx; + uint64 externref_obj; + + bh_memcpy_s(&externref_obj, sizeof(uint64), argv1, + sizeof(uint64)); + + if (!wasm_externref_obj2ref(exec_env->module_inst, + (void *)(uintptr_t)externref_obj, + &externref_idx)) + goto fail; + argv_ret[0] = externref_idx; + break; + } +#endif +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + { + bh_memcpy_s(argv_ret, sizeof(uintptr_t), argv1, + sizeof(uintptr_t)); + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + ret = !wasm_runtime_copy_exception(module, NULL); + +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); +#if WASM_ENABLE_MEMORY64 == 0 + (void)arg_i64; +#endif + return ret; +} + +/** + * Implementation of wasm_runtime_invoke_native() + */ + +/* The invoke native implementation on ARM platform with VFP co-processor */ +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) \ + || defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) +typedef void (*GenericFunctionPointer)(); +void +invokeNative(GenericFunctionPointer f, uint32 *args, uint32 n_stacks); + +typedef float64 (*Float64FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef float32 (*Float32FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef int64 (*Int64FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef int32 (*Int32FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef void (*VoidFuncPtr)(GenericFunctionPointer, uint32 *, uint32); + +static volatile Float64FuncPtr invokeNative_Float64 = + (Float64FuncPtr)(uintptr_t)invokeNative; +static volatile Float32FuncPtr invokeNative_Float32 = + (Float32FuncPtr)(uintptr_t)invokeNative; +static volatile Int64FuncPtr invokeNative_Int64 = + (Int64FuncPtr)(uintptr_t)invokeNative; +static volatile Int32FuncPtr invokeNative_Int32 = + (Int32FuncPtr)(uintptr_t)invokeNative; +static volatile VoidFuncPtr invokeNative_Void = + (VoidFuncPtr)(uintptr_t)invokeNative; + +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) +#define MAX_REG_INTS 4 +#define MAX_REG_FLOATS 16 +#else +#define MAX_REG_INTS 8 +#define MAX_REG_FLOATS 8 +#endif + +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); + /* argv buf layout: int args(fix cnt) + float args(fix cnt) + stack args + */ + uint32 argv_buf[32], *argv1 = argv_buf, *ints, *stacks, size; + uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0; + uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret = false; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + bool is_aot_func = (NULL == signature); +#endif +#if !defined(BUILD_TARGET_RISCV32_ILP32) && !defined(BUILD_TARGET_ARC) + uint32 *fps; + int n_fps = 0; +#else +#define fps ints +#define n_fps n_ints +#endif + + n_ints++; /* exec env */ + + /* Traverse firstly to calculate stack args count */ + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + if (n_ints < MAX_REG_INTS) + n_ints++; + else + n_stacks++; + break; + case VALUE_TYPE_I64: + if (n_ints < MAX_REG_INTS - 1) { +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_ints & 1) + n_ints++; +#endif + n_ints += 2; + } +#if defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) || defined(BUILD_TARGET_ARC) + /* part in register, part in stack */ + else if (n_ints == MAX_REG_INTS - 1) { + n_ints++; + n_stacks++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + n_stacks += 2; + } + break; +#if !defined(BUILD_TARGET_RISCV32_ILP32D) + case VALUE_TYPE_F32: + if (n_fps < MAX_REG_FLOATS) + n_fps++; + else + n_stacks++; + break; + case VALUE_TYPE_F64: + if (n_fps < MAX_REG_FLOATS - 1) { +#if !defined(BUILD_TARGET_RISCV32_ILP32) && !defined(BUILD_TARGET_ARC) + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_fps & 1) + n_fps++; +#endif + n_fps += 2; + } +#if defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) + else if (n_fps == MAX_REG_FLOATS - 1) { + n_fps++; + n_stacks++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + n_stacks += 2; + } + break; +#else /* BUILD_TARGET_RISCV32_ILP32D */ + case VALUE_TYPE_F32: + case VALUE_TYPE_F64: + if (n_fps < MAX_REG_FLOATS) { + n_fps++; + } + else if (func_type->types[i] == VALUE_TYPE_F32 + && n_ints < MAX_REG_INTS) { + /* use int reg firstly if available */ + n_ints++; + } + else if (func_type->types[i] == VALUE_TYPE_F64 + && n_ints < MAX_REG_INTS - 1) { + /* use int regs firstly if available */ + if (n_ints & 1) + n_ints++; + ints += 2; + } + else { + /* 64-bit data in stack must be 8 bytes aligned in riscv32 + */ + if (n_stacks & 1) + n_stacks++; + n_stacks += 2; + } + break; +#endif /* BUILD_TARGET_RISCV32_ILP32D */ + default: + bh_assert(0); + break; + } + } + + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + n_ints++; + else + n_stacks++; + } + +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + argc1 = MAX_REG_INTS + MAX_REG_FLOATS + n_stacks; +#elif defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) + argc1 = MAX_REG_INTS + n_stacks; +#else /* for BUILD_TARGET_RISCV32_ILP32D */ + argc1 = MAX_REG_INTS + MAX_REG_FLOATS * 2 + n_stacks; +#endif + + if (argc1 > sizeof(argv_buf) / sizeof(uint32)) { + size = sizeof(uint32) * (uint32)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + + ints = argv1; +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + fps = ints + MAX_REG_INTS; + stacks = fps + MAX_REG_FLOATS; +#elif defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) + stacks = ints + MAX_REG_INTS; +#else /* for BUILD_TARGET_RISCV32_ILP32D */ + fps = ints + MAX_REG_INTS; + stacks = fps + MAX_REG_FLOATS * 2; +#endif + + n_ints = 0; + n_fps = 0; + n_stacks = 0; + ints[n_ints++] = (uint32)(uintptr_t)exec_env; + + /* Traverse secondly to fill in each argument */ + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + arg_i32 = *argv_src++; + + if (signature) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } + + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = arg_i32; + else + stacks[n_stacks++] = arg_i32; + break; + } + case VALUE_TYPE_I64: + { + if (n_ints < MAX_REG_INTS - 1) { +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_ints & 1) + n_ints++; +#endif + ints[n_ints++] = *argv_src++; + ints[n_ints++] = *argv_src++; + } +#if defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) || defined(BUILD_TARGET_ARC) + else if (n_ints == MAX_REG_INTS - 1) { + ints[n_ints++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + stacks[n_stacks++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } + break; + } +#if !defined(BUILD_TARGET_RISCV32_ILP32D) + case VALUE_TYPE_F32: + { + if (n_fps < MAX_REG_FLOATS) + *(float32 *)&fps[n_fps++] = *(float32 *)argv_src++; + else + *(float32 *)&stacks[n_stacks++] = *(float32 *)argv_src++; + break; + } + case VALUE_TYPE_F64: + { + if (n_fps < MAX_REG_FLOATS - 1) { +#if !defined(BUILD_TARGET_RISCV32_ILP32) && !defined(BUILD_TARGET_ARC) + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_fps & 1) + n_fps++; +#endif + fps[n_fps++] = *argv_src++; + fps[n_fps++] = *argv_src++; + } +#if defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) + else if (n_fps == MAX_REG_FLOATS - 1) { + fps[n_fps++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + stacks[n_stacks++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } + break; + } +#else /* BUILD_TARGET_RISCV32_ILP32D */ + case VALUE_TYPE_F32: + case VALUE_TYPE_F64: + { + if (n_fps < MAX_REG_FLOATS) { + if (func_type->types[i] == VALUE_TYPE_F32) { + *(float32 *)&fps[n_fps * 2] = *(float32 *)argv_src++; + /* NaN boxing, the upper bits of a valid NaN-boxed + value must be all 1s. */ + fps[n_fps * 2 + 1] = 0xFFFFFFFF; + } + else { + *(float64 *)&fps[n_fps * 2] = *(float64 *)argv_src; + argv_src += 2; + } + n_fps++; + } + else if (func_type->types[i] == VALUE_TYPE_F32 + && n_ints < MAX_REG_INTS) { + /* use int reg firstly if available */ + *(float32 *)&ints[n_ints++] = *(float32 *)argv_src++; + } + else if (func_type->types[i] == VALUE_TYPE_F64 + && n_ints < MAX_REG_INTS - 1) { + /* use int regs firstly if available */ + if (n_ints & 1) + n_ints++; + *(float64 *)&ints[n_ints] = *(float64 *)argv_src; + n_ints += 2; + argv_src += 2; + } + else { + /* 64-bit data in stack must be 8 bytes aligned in riscv32 + */ + if (n_stacks & 1) + n_stacks++; + if (func_type->types[i] == VALUE_TYPE_F32) { + *(float32 *)&stacks[n_stacks++] = + *(float32 *)argv_src++; + } + else { + *(float64 *)&stacks[n_stacks] = *(float64 *)argv_src; + argv_src += 2; + n_stacks += 2; + } + } + break; + } +#endif /* BUILD_TARGET_RISCV32_ILP32D */ +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv_src++; + + if (is_aot_func) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = externref_idx; + else + stacks[n_stacks++] = externref_idx; + } + else { + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = (uintptr_t)externref_obj; + else + stacks[n_stacks++] = (uintptr_t)externref_obj; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + /* Save extra result values' address to argv1 */ + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint32 *)argv_src++; + else + stacks[n_stacks++] = *(uint32 *)argv_src++; + } + + exec_env->attachment = attachment; + if (func_type->result_count == 0) { + invokeNative_Void(func_ptr, argv1, n_stacks); + } + else { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = + (uint32)invokeNative_Int32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_I64: + PUT_I64_TO_ADDR(argv_ret, + invokeNative_Int64(func_ptr, argv1, n_stacks)); + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = + invokeNative_Float32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_F64: + PUT_F64_TO_ADDR( + argv_ret, invokeNative_Float64(func_ptr, argv1, n_stacks)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + if (is_aot_func) { + uint32 externref_idx = + (uint32)invokeNative_Int32(func_ptr, argv1, argc1); + argv_ret[0] = externref_idx; + } + else { + uint32 externref_idx; + void *externref_obj; + + externref_obj = (void *)(uintptr_t)invokeNative_Int32( + func_ptr, argv1, argc1); + + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_idx)) + goto fail; + + argv_ret[0] = externref_idx; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + exec_env->attachment = NULL; + + ret = !wasm_runtime_copy_exception(module, NULL); + +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); + return ret; +} +#endif /* end of defined(BUILD_TARGET_ARM_VFP) \ + || defined(BUILD_TARGET_THUMB_VFP) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) \ + || defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_ARC) */ + +#if defined(BUILD_TARGET_X86_32) || defined(BUILD_TARGET_ARM) \ + || defined(BUILD_TARGET_THUMB) || defined(BUILD_TARGET_MIPS) \ + || defined(BUILD_TARGET_XTENSA) +typedef void (*GenericFunctionPointer)(); +void +invokeNative(GenericFunctionPointer f, uint32 *args, uint32 sz); + +typedef float64 (*Float64FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef float32 (*Float32FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef int64 (*Int64FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef int32 (*Int32FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef void (*VoidFuncPtr)(GenericFunctionPointer f, uint32 *, uint32); + +static volatile Int64FuncPtr invokeNative_Int64 = + (Int64FuncPtr)(uintptr_t)invokeNative; +static volatile Int32FuncPtr invokeNative_Int32 = + (Int32FuncPtr)(uintptr_t)invokeNative; +static volatile Float64FuncPtr invokeNative_Float64 = + (Float64FuncPtr)(uintptr_t)invokeNative; +static volatile Float32FuncPtr invokeNative_Float32 = + (Float32FuncPtr)(uintptr_t)invokeNative; +static volatile VoidFuncPtr invokeNative_Void = + (VoidFuncPtr)(uintptr_t)invokeNative; + +static inline void +word_copy(uint32 *dest, uint32 *src, unsigned num) +{ + for (; num > 0; num--) + *dest++ = *src++; +} + +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); + uint32 argv_buf[32], *argv1 = argv_buf, argc1, i, j = 0; + uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + uint64 size; + bool ret = false; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + bool is_aot_func = (NULL == signature); +#endif + +#if defined(BUILD_TARGET_X86_32) + argc1 = argc + ext_ret_count + 2; +#else + /* arm/thumb/mips/xtensa, 64-bit data must be 8 bytes aligned, + so we need to allocate more memory. */ + argc1 = func_type->param_count * 2 + ext_ret_count + 2; +#endif + + if (argc1 > sizeof(argv_buf) / sizeof(uint32)) { + size = sizeof(uint32) * (uint64)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + + for (i = 0; i < sizeof(WASMExecEnv *) / sizeof(uint32); i++) + argv1[j++] = ((uint32 *)&exec_env)[i]; + + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + arg_i32 = *argv++; + + if (signature) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } + + argv1[j++] = arg_i32; + break; + } + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: +#if !defined(BUILD_TARGET_X86_32) + /* 64-bit data must be 8 bytes aligned in arm, thumb, mips + and xtensa */ + if (j & 1) + j++; +#endif + argv1[j++] = *argv++; + argv1[j++] = *argv++; + break; + case VALUE_TYPE_F32: + argv1[j++] = *argv++; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv++; + if (is_aot_func) { + argv1[j++] = externref_idx; + } + else { + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + argv1[j++] = (uintptr_t)externref_obj; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + /* Save extra result values' address to argv1 */ + word_copy(argv1 + j, argv, ext_ret_count); + + argc1 = j + ext_ret_count; + exec_env->attachment = attachment; + if (func_type->result_count == 0) { + invokeNative_Void(func_ptr, argv1, argc1); + } + else { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = + (uint32)invokeNative_Int32(func_ptr, argv1, argc1); + break; + case VALUE_TYPE_I64: + PUT_I64_TO_ADDR(argv_ret, + invokeNative_Int64(func_ptr, argv1, argc1)); + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = + invokeNative_Float32(func_ptr, argv1, argc1); + break; + case VALUE_TYPE_F64: + PUT_F64_TO_ADDR(argv_ret, + invokeNative_Float64(func_ptr, argv1, argc1)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + if (is_aot_func) { + uint32 externref_idx = + (uint32)invokeNative_Int32(func_ptr, argv1, argc1); + argv_ret[0] = externref_idx; + } + else { + void *externref_obj = (void *)(uintptr_t)invokeNative_Int32( + func_ptr, argv1, argc1); + uint32 externref_idx; + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_idx)) + goto fail; + argv_ret[0] = externref_idx; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + exec_env->attachment = NULL; + + ret = !wasm_runtime_copy_exception(module, NULL); + +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); + return ret; +} + +#endif /* end of defined(BUILD_TARGET_X86_32) \ + || defined(BUILD_TARGET_ARM) \ + || defined(BUILD_TARGET_THUMB) \ + || defined(BUILD_TARGET_MIPS) \ + || defined(BUILD_TARGET_XTENSA) */ + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + +#if WASM_ENABLE_SIMD != 0 +#ifdef v128 +#undef v128 +#endif + +#if defined(_WIN32) || defined(_WIN32_) +typedef union __declspec(intrin_type) __declspec(align(8)) v128 { + __int8 m128i_i8[16]; + __int16 m128i_i16[8]; + __int32 m128i_i32[4]; + __int64 m128i_i64[2]; + unsigned __int8 m128i_u8[16]; + unsigned __int16 m128i_u16[8]; + unsigned __int32 m128i_u32[4]; + unsigned __int64 m128i_u64[2]; +} v128; +#elif defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) +typedef long long v128 + __attribute__((__vector_size__(16), __may_alias__, __aligned__(1))); +#elif defined(BUILD_TARGET_AARCH64) +#include +typedef uint32x4_t __m128i; +#define v128 __m128i +#endif + +#endif /* end of WASM_ENABLE_SIMD != 0 */ + +typedef void (*GenericFunctionPointer)(); +void +invokeNative(GenericFunctionPointer f, uint64 *args, uint64 n_stacks); + +typedef float64 (*Float64FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef float32 (*Float32FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef int64 (*Int64FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef int32 (*Int32FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef void (*VoidFuncPtr)(GenericFunctionPointer, uint64 *, uint64); + +/* NOLINTBEGIN */ +static volatile Float64FuncPtr invokeNative_Float64 = + (Float64FuncPtr)(uintptr_t)invokeNative; +static volatile Float32FuncPtr invokeNative_Float32 = + (Float32FuncPtr)(uintptr_t)invokeNative; +static volatile Int64FuncPtr invokeNative_Int64 = + (Int64FuncPtr)(uintptr_t)invokeNative; +static volatile Int32FuncPtr invokeNative_Int32 = + (Int32FuncPtr)(uintptr_t)invokeNative; +static volatile VoidFuncPtr invokeNative_Void = + (VoidFuncPtr)(uintptr_t)invokeNative; + +#if WASM_ENABLE_SIMD != 0 +typedef v128 (*V128FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +static V128FuncPtr invokeNative_V128 = (V128FuncPtr)(uintptr_t)invokeNative; +#endif +/* NOLINTEND */ + +#if defined(_WIN32) || defined(_WIN32_) +#define MAX_REG_FLOATS 4 +#define MAX_REG_INTS 4 +#else /* else of defined(_WIN32) || defined(_WIN32_) */ +#define MAX_REG_FLOATS 8 +#if defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) +#define MAX_REG_INTS 8 +#else +#define MAX_REG_INTS 6 +#endif /* end of defined(BUILD_TARGET_AARCH64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) */ +#endif /* end of defined(_WIN32) || defined(_WIN32_) */ + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); + uint64 argv_buf[32] = { 0 }, *argv1 = argv_buf, *ints, *stacks, size, + arg_i64; + uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0; + uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret = false; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + bool is_aot_func = (NULL == signature); +#endif +#ifndef BUILD_TARGET_RISCV64_LP64 +#if WASM_ENABLE_SIMD == 0 + uint64 *fps; +#else + v128 *fps; +#endif +#else /* else of BUILD_TARGET_RISCV64_LP64 */ +#define fps ints +#endif /* end of BUILD_TARGET_RISCV64_LP64 */ + +#if defined(_WIN32) || defined(_WIN32_) || defined(BUILD_TARGET_RISCV64_LP64) + /* important difference in calling conventions */ +#define n_fps n_ints +#else + int n_fps = 0; +#endif + +#if WASM_ENABLE_SIMD == 0 + argc1 = 1 + MAX_REG_FLOATS + (uint32)func_type->param_count + ext_ret_count; +#else + argc1 = 1 + MAX_REG_FLOATS * 2 + (uint32)func_type->param_count * 2 + + ext_ret_count; +#endif + if (argc1 > sizeof(argv_buf) / sizeof(uint64)) { + size = sizeof(uint64) * (uint64)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + +#ifndef BUILD_TARGET_RISCV64_LP64 +#if WASM_ENABLE_SIMD == 0 + fps = argv1; + ints = fps + MAX_REG_FLOATS; +#else + fps = (v128 *)argv1; + ints = (uint64 *)(fps + MAX_REG_FLOATS); +#endif +#else /* else of BUILD_TARGET_RISCV64_LP64 */ + ints = argv1; +#endif /* end of BUILD_TARGET_RISCV64_LP64 */ + stacks = ints + MAX_REG_INTS; + + ints[n_ints++] = (uint64)(uintptr_t)exec_env; + + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + arg_i32 = *argv_src++; + arg_i64 = arg_i32; + /* TODO: memory64 if future there is a way for supporting + * wasm64 and wasm32 in libc at the same time, remove the + * macro control */ +#if WASM_ENABLE_MEMORY64 == 0 + if (signature) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + arg_i64 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + arg_i64 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } +#endif + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = arg_i64; + else + stacks[n_stacks++] = arg_i64; + break; + } + case VALUE_TYPE_I64: +#if WASM_ENABLE_MEMORY64 != 0 + { + arg_i64 = GET_I64_FROM_ADDR(argv_src); + argv_src += 2; + if (signature) { + /* TODO: memory64 pointer with length need a new symbol + * to represent type i64, with '~' still represent i32 + * length */ + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr(module, arg_i64, + (uint64)ptr_len)) + goto fail; + + arg_i64 = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr(module, + arg_i64)) + goto fail; + + arg_i64 = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + } + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = arg_i64; + else + stacks[n_stacks++] = arg_i64; + break; + } +#endif +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint64 *)argv_src; + else + stacks[n_stacks++] = *(uint64 *)argv_src; + argv_src += 2; + break; + case VALUE_TYPE_F32: + if (n_fps < MAX_REG_FLOATS) { + *(float32 *)&fps[n_fps++] = *(float32 *)argv_src++; + } + else { + *(float32 *)&stacks[n_stacks++] = *(float32 *)argv_src++; + } + break; + case VALUE_TYPE_F64: + if (n_fps < MAX_REG_FLOATS) { + *(float64 *)&fps[n_fps++] = *(float64 *)argv_src; + } + else { + *(float64 *)&stacks[n_stacks++] = *(float64 *)argv_src; + } + argv_src += 2; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv_src++; + if (is_aot_func) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = externref_idx; + else + stacks[n_stacks++] = externref_idx; + } + else { + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = (uintptr_t)externref_obj; + else + stacks[n_stacks++] = (uintptr_t)externref_obj; + } + break; + } +#endif +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + if (n_fps < MAX_REG_FLOATS) { + *(v128 *)&fps[n_fps++] = *(v128 *)argv_src; + } + else { + *(v128 *)&stacks[n_stacks++] = *(v128 *)argv_src; + n_stacks++; + } + argv_src += 4; + break; +#endif + default: + bh_assert(0); + break; + } + } + + /* Save extra result values' address to argv1 */ + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint64 *)argv_src; + else + stacks[n_stacks++] = *(uint64 *)argv_src; + argv_src += 2; + } + + exec_env->attachment = attachment; + if (result_count == 0) { + invokeNative_Void(func_ptr, argv1, n_stacks); + } + else { + /* Invoke the native function and get the first result value */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = + (uint32)invokeNative_Int32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_I64: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif + PUT_I64_TO_ADDR(argv_ret, + invokeNative_Int64(func_ptr, argv1, n_stacks)); + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = + invokeNative_Float32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_F64: + PUT_F64_TO_ADDR( + argv_ret, invokeNative_Float64(func_ptr, argv1, n_stacks)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + if (is_aot_func) { + argv_ret[0] = invokeNative_Int32(func_ptr, argv1, n_stacks); + } + else { + uint32 externref_idx; + void *externref_obj = (void *)(uintptr_t)invokeNative_Int64( + func_ptr, argv1, n_stacks); + + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_idx)) + goto fail; + + argv_ret[0] = externref_idx; + } + break; + } +#endif +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + *(v128 *)argv_ret = + invokeNative_V128(func_ptr, argv1, n_stacks); + break; +#endif + default: + bh_assert(0); + break; + } + } + exec_env->attachment = NULL; + + ret = !wasm_runtime_copy_exception(module, NULL); +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); + + return ret; +} + +#endif /* end of defined(BUILD_TARGET_X86_64) \ + || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) */ + +bool +wasm_runtime_call_indirect(WASMExecEnv *exec_env, uint32 element_index, + uint32 argc, uint32 argv[]) +{ + bool ret = false; + + if (!wasm_runtime_exec_env_check(exec_env)) { + LOG_ERROR("Invalid exec env stack info."); + return false; + } + + /* this function is called from native code, so exec_env->handle and + exec_env->native_stack_boundary must have been set, we don't set + it again */ + +#if WASM_ENABLE_INTERP != 0 + if (exec_env->module_inst->module_type == Wasm_Module_Bytecode) + ret = wasm_call_indirect(exec_env, 0, element_index, argc, argv); +#endif +#if WASM_ENABLE_AOT != 0 + if (exec_env->module_inst->module_type == Wasm_Module_AoT) + ret = aot_call_indirect(exec_env, 0, element_index, argc, argv); +#endif + + return ret; +} + +static void +exchange_uint32(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 3); + *(p_data + 3) = value; + + value = *(p_data + 1); + *(p_data + 1) = *(p_data + 2); + *(p_data + 2) = value; +} + +static void +exchange_uint64(uint8 *p_data) +{ + uint32 value; + + value = *(uint32 *)p_data; + *(uint32 *)p_data = *(uint32 *)(p_data + 4); + *(uint32 *)(p_data + 4) = value; + exchange_uint32(p_data); + exchange_uint32(p_data + 4); +} + +void +wasm_runtime_read_v128(const uint8 *bytes, uint64 *ret1, uint64 *ret2) +{ + uint64 u1, u2; + + bh_memcpy_s(&u1, 8, bytes, 8); + bh_memcpy_s(&u2, 8, bytes + 8, 8); + + if (!is_little_endian()) { + exchange_uint64((uint8 *)&u1); + exchange_uint64((uint8 *)&u2); + *ret1 = u2; + *ret2 = u1; + } + else { + *ret1 = u1; + *ret2 = u2; + } +} + +#if WASM_ENABLE_THREAD_MGR != 0 +typedef struct WASMThreadArg { + WASMExecEnv *new_exec_env; + wasm_thread_callback_t callback; + void *arg; +} WASMThreadArg; + +WASMExecEnv * +wasm_runtime_spawn_exec_env(WASMExecEnv *exec_env) +{ + return wasm_cluster_spawn_exec_env(exec_env); +} + +void +wasm_runtime_destroy_spawned_exec_env(WASMExecEnv *exec_env) +{ + wasm_cluster_destroy_spawned_exec_env(exec_env); +} + +static void * +wasm_runtime_thread_routine(void *arg) +{ + WASMThreadArg *thread_arg = (WASMThreadArg *)arg; + void *ret; + + bh_assert(thread_arg->new_exec_env); + ret = thread_arg->callback(thread_arg->new_exec_env, thread_arg->arg); + + wasm_runtime_destroy_spawned_exec_env(thread_arg->new_exec_env); + wasm_runtime_free(thread_arg); + + os_thread_exit(ret); + return ret; +} + +int32 +wasm_runtime_spawn_thread(WASMExecEnv *exec_env, wasm_thread_t *tid, + wasm_thread_callback_t callback, void *arg) +{ + WASMExecEnv *new_exec_env = wasm_runtime_spawn_exec_env(exec_env); + WASMThreadArg *thread_arg; + int32 ret; + + if (!new_exec_env) + return -1; + + if (!(thread_arg = wasm_runtime_malloc(sizeof(WASMThreadArg)))) { + wasm_runtime_destroy_spawned_exec_env(new_exec_env); + return -1; + } + + thread_arg->new_exec_env = new_exec_env; + thread_arg->callback = callback; + thread_arg->arg = arg; + + ret = os_thread_create((korp_tid *)tid, wasm_runtime_thread_routine, + thread_arg, APP_THREAD_STACK_SIZE_DEFAULT); + + if (ret != 0) { + wasm_runtime_destroy_spawned_exec_env(new_exec_env); + wasm_runtime_free(thread_arg); + } + + return ret; +} + +int32 +wasm_runtime_join_thread(wasm_thread_t tid, void **retval) +{ + return os_thread_join((korp_tid)tid, retval); +} + +#endif /* end of WASM_ENABLE_THREAD_MGR */ + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + +static korp_mutex externref_lock; +static uint32 externref_global_id = 1; +static HashMap *externref_map; + +typedef struct ExternRefMapNode { + /* The extern object from runtime embedder */ + void *extern_obj; + /* The module instance it belongs to */ + WASMModuleInstanceCommon *module_inst; + /* Whether it is retained */ + bool retained; + /* Whether it is marked by runtime */ + bool marked; + /* cleanup function called when the externref is freed */ + void (*cleanup)(void *); +} ExternRefMapNode; + +static uint32 +wasm_externref_hash(const void *key) +{ + uint32 externref_idx = (uint32)(uintptr_t)key; + return externref_idx; +} + +static bool +wasm_externref_equal(void *key1, void *key2) +{ + uint32 externref_idx1 = (uint32)(uintptr_t)key1; + uint32 externref_idx2 = (uint32)(uintptr_t)key2; + return externref_idx1 == externref_idx2 ? true : false; +} + +static bool +wasm_externref_map_init() +{ + if (os_mutex_init(&externref_lock) != 0) + return false; + + if (!(externref_map = bh_hash_map_create(32, false, wasm_externref_hash, + wasm_externref_equal, NULL, + wasm_runtime_free))) { + os_mutex_destroy(&externref_lock); + return false; + } + + externref_global_id = 1; + return true; +} + +static void +wasm_externref_map_destroy() +{ + bh_hash_map_destroy(externref_map); + os_mutex_destroy(&externref_lock); +} + +typedef struct LookupExtObj_UserData { + ExternRefMapNode node; + bool found; + uint32 externref_idx; +} LookupExtObj_UserData; + +static void +lookup_extobj_callback(void *key, void *value, void *user_data) +{ + uint32 externref_idx = (uint32)(uintptr_t)key; + ExternRefMapNode *node = (ExternRefMapNode *)value; + LookupExtObj_UserData *user_data_lookup = + (LookupExtObj_UserData *)user_data; + + if (node->extern_obj == user_data_lookup->node.extern_obj + && node->module_inst == user_data_lookup->node.module_inst) { + user_data_lookup->found = true; + user_data_lookup->externref_idx = externref_idx; + } +} + +static void +delete_externref(void *key, ExternRefMapNode *node) +{ + bh_hash_map_remove(externref_map, key, NULL, NULL); + if (node->cleanup) { + (*node->cleanup)(node->extern_obj); + } + wasm_runtime_free(node); +} + +static void +delete_extobj_callback(void *key, void *value, void *user_data) +{ + ExternRefMapNode *node = (ExternRefMapNode *)value; + LookupExtObj_UserData *lookup_user_data = + (LookupExtObj_UserData *)user_data; + + if (node->extern_obj == lookup_user_data->node.extern_obj + && node->module_inst == lookup_user_data->node.module_inst) { + lookup_user_data->found = true; + delete_externref(key, node); + } +} + +bool +wasm_externref_objdel(WASMModuleInstanceCommon *module_inst, void *extern_obj) +{ + LookupExtObj_UserData lookup_user_data = { 0 }; + bool ok = false; + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, delete_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + ok = true; + } + os_mutex_unlock(&externref_lock); + + return ok; +} + +bool +wasm_externref_set_cleanup(WASMModuleInstanceCommon *module_inst, + void *extern_obj, void (*extern_obj_cleanup)(void *)) +{ + + LookupExtObj_UserData lookup_user_data = { 0 }; + bool ok = false; + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, lookup_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + void *key = (void *)(uintptr_t)lookup_user_data.externref_idx; + ExternRefMapNode *node = bh_hash_map_find(externref_map, key); + bh_assert(node); + node->cleanup = extern_obj_cleanup; + ok = true; + } + os_mutex_unlock(&externref_lock); + + return ok; +} + +bool +wasm_externref_obj2ref(WASMModuleInstanceCommon *module_inst, void *extern_obj, + uint32 *p_externref_idx) +{ + LookupExtObj_UserData lookup_user_data = { 0 }; + ExternRefMapNode *node; + uint32 externref_idx; + + /* + * to catch a parameter from `wasm_application_execute_func`, + * which represents a string 'null' + */ +#if UINTPTR_MAX == UINT32_MAX + if ((uint32)-1 == (uintptr_t)extern_obj) { +#else + if ((uint64)-1LL == (uintptr_t)extern_obj) { +#endif + *p_externref_idx = NULL_REF; + return true; + } + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, lookup_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + *p_externref_idx = lookup_user_data.externref_idx; + os_mutex_unlock(&externref_lock); + return true; + } + + /* Not found in hashmap */ + if (externref_global_id == NULL_REF || externref_global_id == 0) { + goto fail1; + } + + if (!(node = wasm_runtime_malloc(sizeof(ExternRefMapNode)))) { + goto fail1; + } + + memset(node, 0, sizeof(ExternRefMapNode)); + node->extern_obj = extern_obj; + node->module_inst = module_inst; + node->cleanup = NULL; + + externref_idx = externref_global_id; + + if (!bh_hash_map_insert(externref_map, (void *)(uintptr_t)externref_idx, + (void *)node)) { + goto fail2; + } + + externref_global_id++; + *p_externref_idx = externref_idx; + os_mutex_unlock(&externref_lock); + return true; +fail2: + wasm_runtime_free(node); +fail1: + os_mutex_unlock(&externref_lock); + return false; +} + +bool +wasm_externref_ref2obj(uint32 externref_idx, void **p_extern_obj) +{ + ExternRefMapNode *node; + + /* catch a `ref.null` vairable */ + if (externref_idx == NULL_REF) { + *p_extern_obj = NULL; + return true; + } + + os_mutex_lock(&externref_lock); + node = bh_hash_map_find(externref_map, (void *)(uintptr_t)externref_idx); + os_mutex_unlock(&externref_lock); + + if (!node) + return false; + + *p_extern_obj = node->extern_obj; + return true; +} + +static void +reclaim_extobj_callback(void *key, void *value, void *user_data) +{ + ExternRefMapNode *node = (ExternRefMapNode *)value; + WASMModuleInstanceCommon *module_inst = + (WASMModuleInstanceCommon *)user_data; + + if (node->module_inst == module_inst) { + if (!node->marked && !node->retained) { + delete_externref(key, node); + } + else { + node->marked = false; + } + } +} + +static void +mark_externref(uint32 externref_idx) +{ + ExternRefMapNode *node; + + if (externref_idx != NULL_REF) { + node = + bh_hash_map_find(externref_map, (void *)(uintptr_t)externref_idx); + if (node) { + node->marked = true; + } + } +} + +#if WASM_ENABLE_INTERP != 0 +static void +interp_mark_all_externrefs(WASMModuleInstance *module_inst) +{ + uint32 i, j, externref_idx; + table_elem_type_t *table_data; + uint8 *global_data = module_inst->global_data; + WASMGlobalInstance *global; + WASMTableInstance *table; + + global = module_inst->e->globals; + for (i = 0; i < module_inst->e->global_count; i++, global++) { + if (global->type == VALUE_TYPE_EXTERNREF) { + externref_idx = *(uint32 *)(global_data + global->data_offset); + mark_externref(externref_idx); + } + } + + for (i = 0; i < module_inst->table_count; i++) { + uint8 elem_type = 0; + uint32 init_size, max_size; + + table = wasm_get_table_inst(module_inst, i); + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, i, &elem_type, &init_size, + &max_size); + + if (elem_type == VALUE_TYPE_EXTERNREF) { + table_data = table->elems; + for (j = 0; j < table->cur_size; j++) { + externref_idx = table_data[j]; + mark_externref(externref_idx); + } + } + (void)init_size; + (void)max_size; + } +} +#endif + +#if WASM_ENABLE_AOT != 0 +static void +aot_mark_all_externrefs(AOTModuleInstance *module_inst) +{ + uint32 i = 0, j = 0; + const AOTModule *module = (AOTModule *)module_inst->module; + const AOTTable *table = module->tables; + const AOTGlobal *global = module->globals; + const AOTTableInstance *table_inst; + + for (i = 0; i < module->global_count; i++, global++) { + if (global->type == VALUE_TYPE_EXTERNREF) { + mark_externref( + *(uint32 *)(module_inst->global_data + global->data_offset)); + } + } + + for (i = 0; i < module->table_count; i++) { + table_inst = module_inst->tables[i]; + if ((table + i)->elem_type == VALUE_TYPE_EXTERNREF) { + while (j < table_inst->cur_size) { + mark_externref(table_inst->elems[j++]); + } + } + } +} +#endif + +void +wasm_externref_reclaim(WASMModuleInstanceCommon *module_inst) +{ + os_mutex_lock(&externref_lock); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + interp_mark_all_externrefs((WASMModuleInstance *)module_inst); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + aot_mark_all_externrefs((AOTModuleInstance *)module_inst); +#endif + + bh_hash_map_traverse(externref_map, reclaim_extobj_callback, + (void *)module_inst); + os_mutex_unlock(&externref_lock); +} + +static void +cleanup_extobj_callback(void *key, void *value, void *user_data) +{ + ExternRefMapNode *node = (ExternRefMapNode *)value; + WASMModuleInstanceCommon *module_inst = + (WASMModuleInstanceCommon *)user_data; + + if (node->module_inst == module_inst) { + delete_externref(key, node); + } +} + +void +wasm_externref_cleanup(WASMModuleInstanceCommon *module_inst) +{ + os_mutex_lock(&externref_lock); + bh_hash_map_traverse(externref_map, cleanup_extobj_callback, + (void *)module_inst); + os_mutex_unlock(&externref_lock); +} + +bool +wasm_externref_retain(uint32 externref_idx) +{ + ExternRefMapNode *node; + + os_mutex_lock(&externref_lock); + + if (externref_idx != NULL_REF) { + node = + bh_hash_map_find(externref_map, (void *)(uintptr_t)externref_idx); + if (node) { + node->retained = true; + os_mutex_unlock(&externref_lock); + return true; + } + } + + os_mutex_unlock(&externref_lock); + return false; +} +#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +uint32 +wasm_runtime_dump_line_buf_impl(const char *line_buf, bool dump_or_print, + char **buf, uint32 *len) +{ + if (dump_or_print) { + return (uint32)os_printf("%s", line_buf); + } + else if (*buf) { + uint32 dump_len; + + dump_len = snprintf(*buf, *len, "%s", line_buf); + if (dump_len >= *len) { + dump_len = *len; + } + + *len = *len - dump_len; + *buf = *buf + dump_len; + return dump_len; + } + else { + return (uint32)strlen(line_buf); + } +} + +void +wasm_runtime_dump_call_stack(WASMExecEnv *exec_env) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_interp_dump_call_stack(exec_env, true, NULL, 0); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_dump_call_stack(exec_env, true, NULL, 0); + } +#endif +} + +uint32 +wasm_runtime_get_call_stack_buf_size(wasm_exec_env_t exec_env) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_interp_dump_call_stack(exec_env, false, NULL, 0); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_dump_call_stack(exec_env, false, NULL, 0); + } +#endif + + return 0; +} + +uint32 +wasm_runtime_dump_call_stack_to_buf(wasm_exec_env_t exec_env, char *buf, + uint32 len) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_interp_dump_call_stack(exec_env, false, buf, len); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_dump_call_stack(exec_env, false, buf, len); + } +#endif + + return 0; +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ + +#if WASM_ENABLE_STATIC_PGO != 0 +uint32 +wasm_runtime_get_pgo_prof_data_size(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; + return aot_get_pgo_prof_data_size(aot_inst); + } +#endif + return 0; +} + +uint32 +wasm_runtime_dump_pgo_prof_data_to_buf(WASMModuleInstanceCommon *module_inst, + char *buf, uint32 len) +{ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; + return aot_dump_pgo_prof_data_to_buf(aot_inst, buf, len); + } +#endif + return 0; +} +#endif /* end of WASM_ENABLE_STATIC_PGO != 0 */ + +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (table_idx < module->import_table_count) { + WASMTableImport *import_table = + &((module->import_tables + table_idx)->u.table); + *out_elem_type = import_table->elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = import_table->elem_ref_type; +#endif + *out_min_size = import_table->init_size; + *out_max_size = import_table->max_size; + } + else { + WASMTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = table->elem_ref_type; +#endif + *out_min_size = table->init_size; + *out_max_size = table->max_size; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (table_idx < module->import_table_count) { + AOTImportTable *import_table = module->import_tables + table_idx; + *out_elem_type = import_table->elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = NULL; /* TODO */ +#endif + *out_min_size = import_table->table_init_size; + *out_max_size = import_table->table_max_size; + } + else { + AOTTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = NULL; /* TODO */ +#endif + *out_min_size = table->table_init_size; + *out_max_size = table->table_max_size; + } + return true; + } +#endif + + return false; +} + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + return wasm_runtime_get_table_elem_type( + (WASMModuleCommon *)module_inst->module, table_idx, out_elem_type, +#if WASM_ENABLE_GC != 0 + out_ref_type, +#endif + out_min_size, out_max_size); +} + +bool +wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, + const WASMExport *export, WASMFuncType **out) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (export->index < module->import_function_count) { + *out = module->import_functions[export->index].u.function.func_type; + } + else { + *out = + module->functions[export->index - module->import_function_count] + ->func_type; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (export->index < module->import_func_count) { + *out = (WASMFuncType *) + module->types[module->import_funcs[export->index] + .func_type_index]; + } + else { + *out = (WASMFuncType *)module + ->types[module->func_type_indexes + [export->index - module->import_func_count]]; + } + return true; + } +#endif + return false; +} + +bool +wasm_runtime_get_export_global_type(const WASMModuleCommon *module_comm, + const WASMExport *export, + uint8 *out_val_type, bool *out_mutability) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (export->index < module->import_global_count) { + WASMGlobalImport *import_global = + &((module->import_globals + export->index)->u.global); + *out_val_type = import_global->type; + *out_mutability = import_global->is_mutable; + } + else { + WASMGlobal *global = + module->globals + (export->index - module->import_global_count); + *out_val_type = global->type; + *out_mutability = global->is_mutable; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (export->index < module->import_global_count) { + AOTImportGlobal *import_global = + module->import_globals + export->index; + *out_val_type = import_global->type; + *out_mutability = import_global->is_mutable; + } + else { + AOTGlobal *global = + module->globals + (export->index - module->import_global_count); + *out_val_type = global->type; + *out_mutability = global->is_mutable; + } + return true; + } +#endif + return false; +} + +bool +wasm_runtime_get_export_memory_type(const WASMModuleCommon *module_comm, + const WASMExport *export, + uint32 *out_min_page, uint32 *out_max_page) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (export->index < module->import_memory_count) { + WASMMemoryImport *import_memory = + &((module->import_memories + export->index)->u.memory); + *out_min_page = import_memory->init_page_count; + *out_max_page = import_memory->max_page_count; + } + else { + WASMMemory *memory = + module->memories + + (export->index - module->import_memory_count); + *out_min_page = memory->init_page_count; + *out_max_page = memory->max_page_count; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (export->index < module->import_memory_count) { + AOTImportMemory *import_memory = + module->import_memories + export->index; + *out_min_page = import_memory->mem_init_page_count; + *out_max_page = import_memory->mem_max_page_count; + } + else { + AOTMemory *memory = module->memories + + (export->index - module->import_memory_count); + *out_min_page = memory->mem_init_page_count; + *out_max_page = memory->mem_max_page_count; + } + return true; + } +#endif + return false; +} + +bool +wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm, + const WASMExport *export, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size) +{ + return wasm_runtime_get_table_elem_type(module_comm, export->index, + out_elem_type, +#if WASM_ENABLE_GC != 0 + out_ref_type, +#endif + out_min_size, out_max_size); +} + +static inline bool +argv_to_params(wasm_val_t *out_params, const uint32 *argv, + WASMFuncType *func_type) +{ + wasm_val_t *param = out_params; + uint32 i = 0, *u32; + + for (i = 0; i < func_type->param_count; i++, param++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: + param->kind = WASM_I32; + param->of.i32 = *argv++; + break; + case VALUE_TYPE_I64: + param->kind = WASM_I64; + u32 = (uint32 *)¶m->of.i64; + u32[0] = *argv++; + u32[1] = *argv++; + break; + case VALUE_TYPE_F32: + param->kind = WASM_F32; + param->of.f32 = *(float32 *)argv++; + break; + case VALUE_TYPE_F64: + param->kind = WASM_F64; + u32 = (uint32 *)¶m->of.i64; + u32[0] = *argv++; + u32[1] = *argv++; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + param->kind = WASM_ANYREF; + + if (!wasm_externref_ref2obj(*argv, + (void **)¶m->of.foreign)) { + return false; + } + + argv++; + break; +#endif + default: + return false; + } + } + + return true; +} + +static inline bool +results_to_argv(WASMModuleInstanceCommon *module_inst, uint32 *out_argv, + const wasm_val_t *results, WASMFuncType *func_type) +{ + const wasm_val_t *result = results; + uint32 *argv = out_argv, *u32, i; + uint8 *result_types = func_type->types + func_type->param_count; + + for (i = 0; i < func_type->result_count; i++, result++) { + switch (result_types[i]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + *(int32 *)argv++ = result->of.i32; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + u32 = (uint32 *)&result->of.i64; + *argv++ = u32[0]; + *argv++ = u32[1]; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + if (!wasm_externref_obj2ref(module_inst, + (void *)result->of.foreign, + (uint32 *)argv)) { + return false; + } + argv++; + break; +#endif + default: + return false; + } + } + + return true; +} + +bool +wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, + void *func_ptr, WASMFuncType *func_type, + uint32 argc, uint32 *argv, bool with_env, + void *wasm_c_api_env) +{ + wasm_val_t params_buf[16] = { 0 }, results_buf[4] = { 0 }; + wasm_val_t *params = params_buf, *results = results_buf; + wasm_trap_t *trap = NULL; + bool ret = false; + wasm_val_vec_t params_vec = { 0 }, results_vec = { 0 }; + + if (func_type->param_count > 16) { + if (!(params = + runtime_malloc(sizeof(wasm_val_t) * func_type->param_count, + module_inst, NULL, 0))) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + + if (!argv_to_params(params, argv, func_type)) { + wasm_runtime_set_exception(module_inst, "unsupported param type"); + goto fail; + } + + if (func_type->result_count > 4) { + if (!(results = + runtime_malloc(sizeof(wasm_val_t) * func_type->result_count, + module_inst, NULL, 0))) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + goto fail; + } + } + + params_vec.data = params; + params_vec.num_elems = func_type->param_count; + params_vec.size = func_type->param_count; + + results_vec.data = results; + results_vec.num_elems = 0; + results_vec.size = func_type->result_count; + + if (!with_env) { + wasm_func_callback_t callback = (wasm_func_callback_t)func_ptr; + trap = callback(¶ms_vec, &results_vec); + } + else { + wasm_func_callback_with_env_t callback = + (wasm_func_callback_with_env_t)func_ptr; + trap = callback(wasm_c_api_env, ¶ms_vec, &results_vec); + } + + if (trap) { + if (trap->message->data) { + /* since trap->message->data does not end with '\0' */ + char trap_message[108] = { 0 }; + uint32 max_size_to_copy = (uint32)sizeof(trap_message) - 1; + uint32 size_to_copy = (trap->message->size < max_size_to_copy) + ? (uint32)trap->message->size + : max_size_to_copy; + bh_memcpy_s(trap_message, (uint32)sizeof(trap_message), + trap->message->data, size_to_copy); + wasm_runtime_set_exception(module_inst, trap_message); + } + else { + wasm_runtime_set_exception( + module_inst, "native function throw unknown exception"); + } + wasm_trap_delete(trap); + goto fail; + } + + if (!results_to_argv(module_inst, argv, results, func_type)) { + wasm_runtime_set_exception(module_inst, "unsupported result type"); + goto fail; + } + ret = true; + +fail: + if (params != params_buf) + wasm_runtime_free(params); + if (results != results_buf) + wasm_runtime_free(results); + return ret; +} + +bool +wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *inst_comm, + CApiFuncImport *c_api_import, + wasm_val_t *params, uint32 param_count, + wasm_val_t *results, uint32 result_count) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)inst_comm; + void *func_ptr = c_api_import->func_ptr_linked; + bool with_env_arg = c_api_import->with_env_arg, ret = true; + wasm_val_vec_t params_vec = { 0 }, results_vec = { 0 }; + wasm_trap_t *trap = NULL; + + params_vec.data = params; + params_vec.num_elems = param_count; + params_vec.size = param_count; + + results_vec.data = results; + results_vec.num_elems = 0; + results_vec.size = result_count; + + if (!func_ptr) { + wasm_set_exception_with_id(module_inst, EXCE_CALL_UNLINKED_IMPORT_FUNC); + ret = false; + goto fail; + } + + if (!with_env_arg) { + wasm_func_callback_t callback = (wasm_func_callback_t)func_ptr; + trap = callback(¶ms_vec, &results_vec); + } + else { + void *wasm_c_api_env = c_api_import->env_arg; + wasm_func_callback_with_env_t callback = + (wasm_func_callback_with_env_t)func_ptr; + trap = callback(wasm_c_api_env, ¶ms_vec, &results_vec); + } + + if (trap) { + if (trap->message->data) { + /* since trap->message->data does not end with '\0' */ + char trap_message[108] = { 0 }; + uint32 max_size_to_copy = (uint32)sizeof(trap_message) - 1; + uint32 size_to_copy = (trap->message->size < max_size_to_copy) + ? (uint32)trap->message->size + : max_size_to_copy; + bh_memcpy_s(trap_message, (uint32)sizeof(trap_message), + trap->message->data, size_to_copy); + wasm_set_exception(module_inst, trap_message); + } + else { + wasm_set_exception(module_inst, + "native function throw unknown exception"); + } + wasm_trap_delete(trap); + ret = false; + } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + +void +wasm_runtime_show_app_heap_corrupted_prompt() +{ + LOG_ERROR("Error: app heap is corrupted, if the wasm file " + "is compiled by wasi-sdk-12.0 or higher version, " + "please add -Wl,--export=malloc -Wl,--export=free " + "to export malloc and free functions. If it is " + "compiled by asc, please add --exportRuntime to " + "export the runtime helpers."); +} + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +void +wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list) +{ + WASMCustomSection *section = section_list, *next; + while (section) { + next = section->next; + wasm_runtime_free(section); + section = next; + } +} +#endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION */ + +void +wasm_runtime_get_version(uint32_t *major, uint32_t *minor, uint32_t *patch) +{ + *major = WAMR_VERSION_MAJOR; + *minor = WAMR_VERSION_MINOR; + *patch = WAMR_VERSION_PATCH; +} + +bool +wasm_runtime_is_import_func_linked(const char *module_name, + const char *func_name) +{ + return wasm_native_resolve_symbol(module_name, func_name, NULL, NULL, NULL, + NULL); +} + +bool +wasm_runtime_is_import_global_linked(const char *module_name, + const char *global_name) +{ +#if WASM_ENABLE_LIBC_BUILTIN != 0 + WASMGlobalImport global = { 0 }; + return wasm_native_lookup_libc_builtin_global(module_name, global_name, + &global); +#else + return false; +#endif +} + +#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0 +WASMExport * +loader_find_export(const WASMModuleCommon *module, const char *module_name, + const char *field_name, uint8 export_kind, char *error_buf, + uint32 error_buf_size) +{ + WASMExport *exports = NULL, *result = NULL, *export; + uint32 export_count = 0, i; +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + exports = (WASMExport *)aot_module->exports; + export_count = aot_module->export_count; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + exports = wasm_module->exports; + export_count = wasm_module->export_count; + } +#endif + for (i = 0, export = exports; i < export_count; ++i, ++export) { + if (export->kind == export_kind && !strcmp(field_name, export->name)) { + result = export; + goto exit; + } + } + if (i == export_count) { + LOG_DEBUG("can not find an export %d named %s in the module %s", + export_kind, field_name, module_name); + set_error_buf(error_buf, error_buf_size, + "unknown import or incompatible import type"); + } +exit: + return result; +} +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASMModuleCommon * +wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name) +{ + WASMRegisteredModule *node = NULL; +#if WASM_ENABLE_AOT != 0 + if (parent_module->module_type == Wasm_Module_AoT) { + node = bh_list_first_elem( + ((AOTModule *)parent_module)->import_module_list); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (parent_module->module_type == Wasm_Module_Bytecode) { + node = bh_list_first_elem( + ((WASMModule *)parent_module)->import_module_list); + } +#endif + while (node && strcmp(sub_module_name, node->module_name)) { + node = bh_list_elem_next(node); + } + return node ? node->module : NULL; +} + +bool +wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, + WASMModuleCommon *sub_module) +{ + /* register sub_module into its parent sub module list */ + WASMRegisteredModule *node = NULL; + bh_list_status ret = BH_LIST_ERROR; + + if (wasm_runtime_search_sub_module(parent_module, sub_module_name)) { + LOG_DEBUG("%s has been registered in its parent", sub_module_name); + return true; + } + + node = loader_malloc(sizeof(WASMRegisteredModule), NULL, 0); + if (!node) { + return false; + } + + node->module_name = sub_module_name; + node->module = sub_module; +#if WASM_ENABLE_AOT != 0 + if (parent_module->module_type == Wasm_Module_AoT) { + ret = bh_list_insert(((AOTModule *)parent_module)->import_module_list, + node); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (parent_module->module_type == Wasm_Module_Bytecode) { + ret = bh_list_insert(((WASMModule *)parent_module)->import_module_list, + node); + } +#endif + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + return true; +} + +WASMModuleCommon * +wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *sub_module = NULL; + bool ret = false; + uint8 *buffer = NULL; + uint32 buffer_size = 0; + LoadArgs args = { 0 }; + + /* check the registered module list of the parent */ + sub_module = wasm_runtime_search_sub_module(parent_module, sub_module_name); + if (sub_module) { + LOG_DEBUG("%s has been loaded before", sub_module_name); + return sub_module; + } + + /* check the global registered module list */ + sub_module = wasm_runtime_find_module_registered(sub_module_name); + if (sub_module) { + LOG_DEBUG("%s has been loaded", sub_module_name); + goto wasm_runtime_register_sub_module; + } + LOG_VERBOSE("loading %s", sub_module_name); + if (!reader) { + set_error_buf_v(parent_module, error_buf, error_buf_size, + "no sub module reader to load %s", sub_module_name); + return NULL; + } + /* start to maintain a loading module list */ + ret = wasm_runtime_is_loading_module(sub_module_name); + if (ret) { + set_error_buf_v(parent_module, error_buf, error_buf_size, + "found circular dependency on %s", sub_module_name); + return NULL; + } + ret = wasm_runtime_add_loading_module(sub_module_name, error_buf, + error_buf_size); + if (!ret) { + LOG_DEBUG("can not add %s into loading module list\n", sub_module_name); + return NULL; + } + + ret = reader(parent_module->module_type, sub_module_name, &buffer, + &buffer_size); + if (!ret) { + LOG_DEBUG("read the file of %s failed", sub_module_name); + set_error_buf_v(parent_module, error_buf, error_buf_size, + "unknown import %s", sub_module_name); + goto delete_loading_module; + } + if (get_package_type(buffer, buffer_size) != parent_module->module_type) { + LOG_DEBUG("moudle %s type error", sub_module_name); + goto destroy_file_buffer; + } + + args.name = (char *)sub_module_name; + if (get_package_type(buffer, buffer_size) == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + sub_module = (WASMModuleCommon *)wasm_load( + buffer, buffer_size, false, &args, error_buf, error_buf_size); +#endif + } + else if (get_package_type(buffer, buffer_size) == Wasm_Module_AoT) { +#if WASM_ENABLE_AOT != 0 + sub_module = (WASMModuleCommon *)aot_load_from_aot_file( + buffer, buffer_size, &args, error_buf, error_buf_size); +#endif + } + if (!sub_module) { + LOG_DEBUG("error: can not load the sub_module %s", sub_module_name); + /* others will be destroyed in runtime_destroy() */ + goto destroy_file_buffer; + } + wasm_runtime_delete_loading_module(sub_module_name); + /* register on a global list */ + ret = wasm_runtime_register_module_internal( + sub_module_name, (WASMModuleCommon *)sub_module, buffer, buffer_size, + error_buf, error_buf_size); + if (!ret) { + LOG_DEBUG("error: can not register module %s globally\n", + sub_module_name); + /* others will be unloaded in runtime_destroy() */ + goto unload_module; + } + + /* register into its parent list */ +wasm_runtime_register_sub_module: + ret = wasm_runtime_register_sub_module(parent_module, sub_module_name, + sub_module); + if (!ret) { + set_error_buf_v(parent_module, error_buf, error_buf_size, + "failed to register sub module %s", sub_module_name); + /* since it is in the global module list, no need to + * unload the module. the runtime_destroy() will do it + */ + return NULL; + } + + return sub_module; + +unload_module: + wasm_runtime_unload(sub_module); + +destroy_file_buffer: + if (destroyer) { + destroyer(buffer, buffer_size); + } + else { + LOG_WARNING("need to release the reading buffer of %s manually", + sub_module_name); + } + +delete_loading_module: + wasm_runtime_delete_loading_module(sub_module_name); + return NULL; +} + +bool +wasm_runtime_sub_module_instantiate(WASMModuleCommon *module, + WASMModuleInstanceCommon *module_inst, + uint32 stack_size, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + bh_list *sub_module_inst_list = NULL; + WASMRegisteredModule *sub_module_list_node = NULL; + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + sub_module_inst_list = + ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + sub_module_list_node = + bh_list_first_elem(((AOTModule *)module)->import_module_list); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + sub_module_inst_list = + ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + sub_module_list_node = + bh_list_first_elem(((WASMModule *)module)->import_module_list); + } +#endif + while (sub_module_list_node) { + WASMSubModInstNode *sub_module_inst_list_node = NULL; + WASMModuleCommon *sub_module = sub_module_list_node->module; + WASMModuleInstanceCommon *sub_module_inst = NULL; + sub_module_inst = wasm_runtime_instantiate_internal( + sub_module, NULL, NULL, stack_size, heap_size, max_memory_pages, + error_buf, error_buf_size); + if (!sub_module_inst) { + LOG_DEBUG("instantiate %s failed", + sub_module_list_node->module_name); + return false; + } + sub_module_inst_list_node = loader_malloc(sizeof(WASMSubModInstNode), + error_buf, error_buf_size); + if (!sub_module_inst_list_node) { + LOG_DEBUG("Malloc WASMSubModInstNode failed, SZ: %zu", + sizeof(WASMSubModInstNode)); + if (sub_module_inst) + wasm_runtime_deinstantiate_internal(sub_module_inst, false); + return false; + } + sub_module_inst_list_node->module_inst = + (WASMModuleInstance *)sub_module_inst; + sub_module_inst_list_node->module_name = + sub_module_list_node->module_name; + bh_list_status ret = + bh_list_insert(sub_module_inst_list, sub_module_inst_list_node); + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + sub_module_list_node = bh_list_elem_next(sub_module_list_node); + } + + return true; +} + +void +wasm_runtime_sub_module_deinstantiate(WASMModuleInstanceCommon *module_inst) +{ + bh_list *list = NULL; +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + list = ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + list = + ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + } +#endif + + WASMSubModInstNode *node = bh_list_first_elem(list); + while (node) { + WASMSubModInstNode *next_node = bh_list_elem_next(node); + bh_list_remove(list, node); + wasm_runtime_deinstantiate_internal( + (WASMModuleInstanceCommon *)node->module_inst, false); + wasm_runtime_free(node); + node = next_node; + } +} +#endif /* end of WASM_ENABLE_MULTI_MODULE */ +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 +void * +wasm_runtime_create_context_key(void (*dtor)(WASMModuleInstanceCommon *inst, + void *ctx)) +{ + return wasm_native_create_context_key(dtor); +} + +void +wasm_runtime_destroy_context_key(void *key) +{ + wasm_native_destroy_context_key(key); +} + +void +wasm_runtime_set_context(WASMModuleInstanceCommon *inst, void *key, void *ctx) +{ + wasm_native_set_context(inst, key, ctx); +} + +void +wasm_runtime_set_context_spread(WASMModuleInstanceCommon *inst, void *key, + void *ctx) +{ + wasm_native_set_context_spread(inst, key, ctx); +} + +void * +wasm_runtime_get_context(WASMModuleInstanceCommon *inst, void *key) +{ + return wasm_native_get_context(inst, key); +} +#endif /* WASM_ENABLE_MODULE_INST_CONTEXT != 0 */ + +#if WASM_ENABLE_LINUX_PERF != 0 +static bool enable_linux_perf = false; + +bool +wasm_runtime_get_linux_perf(void) +{ + return enable_linux_perf; +} + +void +wasm_runtime_set_linux_perf(bool flag) +{ + enable_linux_perf = flag; +} +#endif + +bool +wasm_runtime_set_module_name(wasm_module_t module, const char *name, + char *error_buf, uint32_t error_buf_size) +{ + if (!module) + return false; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + return wasm_set_module_name((WASMModule *)module, name, error_buf, + error_buf_size); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + return aot_set_module_name((AOTModule *)module, name, error_buf, + error_buf_size); +#endif + + return false; +} + +const char * +wasm_runtime_get_module_name(wasm_module_t module) +{ + if (!module) + return ""; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + return wasm_get_module_name((WASMModule *)module); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + return aot_get_module_name((AOTModule *)module); +#endif + + return ""; +} diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_runtime_common.h b/wasm-micro-runtime/core/iwasm/common/wasm_runtime_common.h new file mode 100644 index 0000000..62c3547 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_runtime_common.h @@ -0,0 +1,1204 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_COMMON_H +#define _WASM_COMMON_H + +#include "bh_platform.h" +#include "bh_common.h" +#include "wasm_exec_env.h" +#include "wasm_native.h" +#include "../include/wasm_export.h" +#include "../interpreter/wasm.h" +#if WASM_ENABLE_GC != 0 +#include "gc/gc_object.h" +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 +#if WASM_ENABLE_UVWASI == 0 +#include "posix.h" +#else +#include "uvwasi.h" +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Internal use for setting default running mode */ +#define Mode_Default 0 + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + +#define PUT_I64_TO_ADDR(addr, value) \ + do { \ + *(int64 *)(addr) = (int64)(value); \ + } while (0) +#define PUT_F64_TO_ADDR(addr, value) \ + do { \ + *(float64 *)(addr) = (float64)(value); \ + } while (0) +#define PUT_REF_TO_ADDR(addr, value) \ + do { \ + *(void **)(addr) = (void *)(value); \ + } while (0) + +#define GET_I64_FROM_ADDR(addr) (*(int64 *)(addr)) +#define GET_F64_FROM_ADDR(addr) (*(float64 *)(addr)) +#define GET_REF_FROM_ADDR(addr) (*(void **)(addr)) + +/* For STORE opcodes */ +#define STORE_I64 PUT_I64_TO_ADDR +static inline void +STORE_U32(void *addr, uint32_t value) +{ + *(uint32_t *)(addr) = (uint32_t)(value); +} +static inline void +STORE_U16(void *addr, uint16_t value) +{ + *(uint16_t *)(addr) = (uint16_t)(value); +} +static inline void +STORE_U8(void *addr, uint8_t value) +{ + *(uint8 *)addr = value; +} + +/* For LOAD opcodes */ +#define LOAD_I64(addr) (*(int64 *)(addr)) +#define LOAD_F64(addr) (*(float64 *)(addr)) +#define LOAD_I32(addr) (*(int32 *)(addr)) +#define LOAD_U32(addr) (*(uint32 *)(addr)) +#define LOAD_I16(addr) (*(int16 *)(addr)) +#define LOAD_U16(addr) (*(uint16 *)(addr)) + +#define STORE_PTR(addr, ptr) \ + do { \ + *(void **)addr = (void *)ptr; \ + } while (0) + +#else /* WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 */ + +#define PUT_I64_TO_ADDR(addr, value) \ + do { \ + uint32 *addr_u32 = (uint32 *)(addr); \ + union { \ + int64 val; \ + uint32 parts[2]; \ + } u; \ + u.val = (int64)(value); \ + addr_u32[0] = u.parts[0]; \ + addr_u32[1] = u.parts[1]; \ + } while (0) +#define PUT_F64_TO_ADDR(addr, value) \ + do { \ + uint32 *addr_u32 = (uint32 *)(addr); \ + union { \ + float64 val; \ + uint32 parts[2]; \ + } u; \ + u.val = (value); \ + addr_u32[0] = u.parts[0]; \ + addr_u32[1] = u.parts[1]; \ + } while (0) +#if UINTPTR_MAX == UINT64_MAX +#define PUT_REF_TO_ADDR(addr, value) \ + do { \ + uint32 *addr_u32 = (uint32 *)(addr); \ + union { \ + void *val; \ + uint32 parts[2]; \ + } u; \ + u.val = (void *)(value); \ + addr_u32[0] = u.parts[0]; \ + addr_u32[1] = u.parts[1]; \ + } while (0) +#else +#define PUT_REF_TO_ADDR(addr, value) \ + do { \ + *(void **)(addr) = (void *)(value); \ + } while (0) +#endif + +static inline int64 +GET_I64_FROM_ADDR(uint32 *addr) +{ + union { + int64 val; + uint32 parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} + +static inline float64 +GET_F64_FROM_ADDR(uint32 *addr) +{ + union { + float64 val; + uint32 parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} + +#if UINTPTR_MAX == UINT64_MAX +static inline void * +GET_REF_FROM_ADDR(uint32 *addr) +{ + union { + void *val; + uint32 parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} +#else +#define GET_REF_FROM_ADDR(addr) (*(void **)(addr)) +#endif + +/* For STORE opcodes */ +#define STORE_I64(addr, value) \ + do { \ + uintptr_t addr_ = (uintptr_t)(addr); \ + union { \ + int64 val; \ + uint32 u32[2]; \ + uint16 u16[4]; \ + uint8 u8[8]; \ + } u; \ + if ((addr_ & (uintptr_t)7) == 0) \ + *(int64 *)(addr) = (int64)(value); \ + else { \ + u.val = (int64)(value); \ + if ((addr_ & (uintptr_t)3) == 0) { \ + ((uint32 *)(addr))[0] = u.u32[0]; \ + ((uint32 *)(addr))[1] = u.u32[1]; \ + } \ + else if ((addr_ & (uintptr_t)1) == 0) { \ + ((uint16 *)(addr))[0] = u.u16[0]; \ + ((uint16 *)(addr))[1] = u.u16[1]; \ + ((uint16 *)(addr))[2] = u.u16[2]; \ + ((uint16 *)(addr))[3] = u.u16[3]; \ + } \ + else { \ + int32 t; \ + for (t = 0; t < 8; t++) \ + ((uint8 *)(addr))[t] = u.u8[t]; \ + } \ + } \ + } while (0) + +static inline void +STORE_U32(void *addr, uint32_t value) +{ + uintptr_t addr_ = (uintptr_t)(addr); + union { + uint32_t val; + uint16_t u16[2]; + uint8_t u8[4]; + } u; + if ((addr_ & (uintptr_t)3) == 0) + *(uint32_t *)(addr) = (uint32_t)(value); + else { + u.val = (uint32_t)(value); + if ((addr_ & (uintptr_t)1) == 0) { + ((uint16_t *)(addr))[0] = u.u16[0]; + ((uint16_t *)(addr))[1] = u.u16[1]; + } + else { + ((uint8_t *)(addr))[0] = u.u8[0]; + ((uint8_t *)(addr))[1] = u.u8[1]; + ((uint8_t *)(addr))[2] = u.u8[2]; + ((uint8_t *)(addr))[3] = u.u8[3]; + } + } +} + +static inline void +STORE_U8(void *addr, uint8_t value) +{ + *(uint8 *)addr = value; +} + +static inline void +STORE_U16(void *addr, uint16_t value) +{ + union { + uint16_t val; + uint8_t u8[2]; + } u; + u.val = (uint16_t)(value); + ((uint8_t *)(addr))[0] = u.u8[0]; + ((uint8_t *)(addr))[1] = u.u8[1]; +} +/* For LOAD opcodes */ +static inline int64 +LOAD_I64(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + int64 val; + uint32 u32[2]; + uint16 u16[4]; + uint8 u8[8]; + } u; + if ((addr1 & (uintptr_t)7) == 0) + return *(int64 *)addr; + + if ((addr1 & (uintptr_t)3) == 0) { + u.u32[0] = ((uint32 *)addr)[0]; + u.u32[1] = ((uint32 *)addr)[1]; + } + else if ((addr1 & (uintptr_t)1) == 0) { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + u.u16[2] = ((uint16 *)addr)[2]; + u.u16[3] = ((uint16 *)addr)[3]; + } + else { + int32 t; + for (t = 0; t < 8; t++) + u.u8[t] = ((uint8 *)addr)[t]; + } + return u.val; +} + +static inline float64 +LOAD_F64(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + float64 val; + uint32 u32[2]; + uint16 u16[4]; + uint8 u8[8]; + } u; + if ((addr1 & (uintptr_t)7) == 0) + return *(float64 *)addr; + + if ((addr1 & (uintptr_t)3) == 0) { + u.u32[0] = ((uint32 *)addr)[0]; + u.u32[1] = ((uint32 *)addr)[1]; + } + else if ((addr1 & (uintptr_t)1) == 0) { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + u.u16[2] = ((uint16 *)addr)[2]; + u.u16[3] = ((uint16 *)addr)[3]; + } + else { + int32 t; + for (t = 0; t < 8; t++) + u.u8[t] = ((uint8 *)addr)[t]; + } + return u.val; +} + +static inline int32 +LOAD_I32(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + int32 val; + uint16 u16[2]; + uint8 u8[4]; + } u; + if ((addr1 & (uintptr_t)3) == 0) + return *(int32 *)addr; + + if ((addr1 & (uintptr_t)1) == 0) { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + } + else { + u.u8[0] = ((uint8 *)addr)[0]; + u.u8[1] = ((uint8 *)addr)[1]; + u.u8[2] = ((uint8 *)addr)[2]; + u.u8[3] = ((uint8 *)addr)[3]; + } + return u.val; +} + +static inline int16 +LOAD_I16(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + int16 val; + uint8 u8[2]; + } u; + if ((addr1 & (uintptr_t)1)) { + u.u8[0] = ((uint8 *)addr)[0]; + u.u8[1] = ((uint8 *)addr)[1]; + return u.val; + } + return *(int16 *)addr; +} + +#define LOAD_U32(addr) ((uint32)LOAD_I32(addr)) +#define LOAD_U16(addr) ((uint16)LOAD_I16(addr)) + +#if UINTPTR_MAX == UINT32_MAX +#define STORE_PTR(addr, ptr) STORE_U32(addr, (uintptr_t)ptr) +#elif UINTPTR_MAX == UINT64_MAX +#define STORE_PTR(addr, ptr) STORE_I64(addr, (uintptr_t)ptr) +#endif + +#endif /* WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define SHARED_MEMORY_LOCK(memory) shared_memory_lock(memory) +#define SHARED_MEMORY_UNLOCK(memory) shared_memory_unlock(memory) +#else +#define SHARED_MEMORY_LOCK(memory) (void)0 +#define SHARED_MEMORY_UNLOCK(memory) (void)0 +#endif + +typedef struct WASMModuleCommon { + /* Module type, for module loaded from WASM bytecode binary, + this field is Wasm_Module_Bytecode, and this structure should + be treated as WASMModule structure; + for module loaded from AOT binary, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModule structure. */ + uint32 module_type; + + /* The following uint8[1] member is a dummy just to indicate + some module_type dependent members follow. + Typically, it should be accessed by casting to the corresponding + actual module_type dependent structure, not via this member. */ + uint8 module_data[1]; +} WASMModuleCommon; + +typedef struct WASMModuleInstanceCommon { + /* Module instance type, for module instance loaded from WASM + bytecode binary, this field is Wasm_Module_Bytecode, and this + structure should be treated as WASMModuleInstance structure; + for module instance loaded from AOT binary, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModuleInstance structure. */ + uint32 module_type; + + /* The following uint8[1] member is a dummy just to indicate + some module_type dependent members follow. + Typically, it should be accessed by casting to the corresponding + actual module_type dependent structure, not via this member. */ + uint8 module_inst_data[1]; +} WASMModuleInstanceCommon; + +typedef struct WASMModuleMemConsumption { + uint32 total_size; + uint32 module_struct_size; + uint32 types_size; + uint32 imports_size; + uint32 functions_size; + uint32 tables_size; + uint32 memories_size; + uint32 globals_size; + uint32 exports_size; + uint32 table_segs_size; + uint32 data_segs_size; + uint32 const_strs_size; +#if WASM_ENABLE_AOT != 0 + uint32 aot_code_size; +#endif +} WASMModuleMemConsumption; + +typedef struct WASMModuleInstMemConsumption { + uint64 total_size; + uint32 module_inst_struct_size; + uint32 app_heap_size; + uint64 memories_size; + uint32 tables_size; + uint32 globals_size; + uint32 functions_size; + uint32 exports_size; +} WASMModuleInstMemConsumption; + +#if WASM_ENABLE_LIBC_WASI != 0 +#if WASM_ENABLE_UVWASI == 0 +typedef struct WASIContext { + struct fd_table *curfds; + struct fd_prestats *prestats; + struct argv_environ_values *argv_environ; + struct addr_pool *addr_pool; + char *ns_lookup_buf; + char **ns_lookup_list; + char *argv_buf; + char **argv_list; + char *env_buf; + char **env_list; + uint32_t exit_code; +} WASIContext; +#else +typedef struct WASIContext { + uvwasi_t uvwasi; + uint32_t exit_code; +} WASIContext; +#endif +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +typedef struct WASMRegisteredModule { + bh_list_link l; + /* point to a string pool */ + const char *module_name; + WASMModuleCommon *module; + /* to store the original module file buffer address */ + uint8 *orig_file_buf; + uint32 orig_file_buf_size; +} WASMRegisteredModule; +#endif + +typedef package_type_t PackageType; +typedef wasm_section_t WASMSection, AOTSection; + +typedef struct wasm_frame_t { + /* wasm_instance_t */ + void *instance; + uint32 module_offset; + uint32 func_index; + uint32 func_offset; + const char *func_name_wp; + + uint32 *sp; + uint8 *frame_ref; + uint32 *lp; +} WASMCApiFrame; + +#if WASM_ENABLE_JIT != 0 +typedef struct LLVMJITOptions { + uint32 opt_level; + uint32 size_level; + uint32 segue_flags; + bool quick_invoke_c_api_import; +} LLVMJITOptions; +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* Signal info passing to interp/aot signal handler */ +typedef struct WASMSignalInfo { + WASMExecEnv *exec_env_tls; +#ifndef BH_PLATFORM_WINDOWS + void *sig_addr; +#else + EXCEPTION_POINTERS *exce_info; +#endif +} WASMSignalInfo; + +/* Set exec_env of thread local storage */ +void +wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env); + +/* Get exec_env of thread local storage */ +WASMExecEnv * +wasm_runtime_get_exec_env_tls(void); +#endif + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_init(void); + +/* Internal API */ +RunningMode +wasm_runtime_get_default_running_mode(void); + +#if WASM_ENABLE_JIT != 0 +/* Internal API */ +LLVMJITOptions * +wasm_runtime_get_llvm_jit_options(void); +#endif + +#if WASM_ENABLE_GC != 0 +/* Internal API */ +uint32 +wasm_runtime_get_gc_heap_size_default(void); +#endif + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_full_init(RuntimeInitArgs *init_args); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_running_mode_supported(RunningMode running_mode); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_default_running_mode(RunningMode running_mode); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy(void); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN PackageType +get_package_type(const uint8 *buf, uint32 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_xip_file(const uint8 *buf, uint32 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleCommon * +wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, + uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleCommon * +wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot, + char *error_buf, uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_unload(WASMModuleCommon *module); + +/* Internal API */ +uint32 +wasm_runtime_get_max_mem(uint32 max_memory_pages, uint32 module_init_page_count, + uint32 module_max_page_count); + +/* Internal API */ +WASMModuleInstanceCommon * +wasm_runtime_instantiate_internal(WASMModuleCommon *module, + WASMModuleInstanceCommon *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, + char *error_buf, uint32 error_buf_size); + +/* Internal API */ +void +wasm_runtime_deinstantiate_internal(WASMModuleInstanceCommon *module_inst, + bool is_sub_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_instantiate(WASMModuleCommon *module, uint32 default_stack_size, + uint32 host_managed_heap_size, char *error_buf, + uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex(WASMModuleCommon *module, + const InstantiationArgs *args, char *error_buf, + uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_running_mode(wasm_module_inst_t module_inst, + RunningMode running_mode); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN RunningMode +wasm_runtime_get_running_mode(wasm_module_inst_t module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_deinstantiate(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleCommon * +wasm_runtime_get_module(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMFunctionInstanceCommon * +wasm_runtime_lookup_function(WASMModuleInstanceCommon *const module_inst, + const char *name); + +/* Internal API */ +WASMFuncType * +wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function, + uint32 module_type); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_func_get_param_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_func_get_result_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_param_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *param_types); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_result_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *result_types); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMExecEnv * +wasm_runtime_create_exec_env(WASMModuleInstanceCommon *module_inst, + uint32 stack_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_exec_env(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_get_module_inst(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_inst(WASMExecEnv *exec_env, + WASMModuleInstanceCommon *const module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_function_attachment(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_user_data(WASMExecEnv *exec_env); + +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_bounds_checks(WASMModuleInstanceCommon *module_inst, + bool enable); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst); +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* Access exception check guard page to trigger the signal handler */ +void +wasm_runtime_access_exce_check_guard_page(); +#endif + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, uint32 argc, + uint32 argv[]); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_a(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t *results, + uint32 num_args, wasm_val_t *args); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_v(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t *results, + uint32 num_args, ...); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_indirect(WASMExecEnv *exec_env, uint32 element_index, + uint32 argc, uint32 argv[]); + +#if WASM_ENABLE_DEBUG_INTERP != 0 +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_runtime_start_debug_instance_with_port(WASMExecEnv *exec_env, + int32_t port); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_runtime_start_debug_instance(WASMExecEnv *exec_env); +#endif + +bool +wasm_runtime_create_exec_env_singleton(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMExecEnv * +wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, + char *argv[]); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, + const char *name, int32 argc, char *argv[]); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_exception(WASMModuleInstanceCommon *module, + const char *exception); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN const char * +wasm_runtime_get_exception(WASMModuleInstanceCommon *module); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_terminate(WASMModuleInstanceCommon *module); + +/* Internal API */ +void +wasm_runtime_set_custom_data_internal(WASMModuleInstanceCommon *module_inst, + void *custom_data); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_custom_data(WASMModuleInstanceCommon *module_inst, + void *custom_data); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst); + +/* Internal API */ +uint64 +wasm_runtime_module_malloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr); + +/* Internal API */ +uint64 +wasm_runtime_module_realloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr, + uint64 size, void **p_native_addr); + +/* Internal API */ +void +wasm_runtime_module_free_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_module_malloc(WASMModuleInstanceCommon *module_inst, uint64 size, + void **p_native_addr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_module_free(WASMModuleInstanceCommon *module_inst, uint64 ptr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_module_dup_data(WASMModuleInstanceCommon *module_inst, + const char *src, uint64 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst, + uint64 app_offset, uint64 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst, + uint64 app_str_offset); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst, + void *native_ptr, uint64 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst, + uint64 app_offset); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst, + void *native_ptr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst, + uint64 app_offset, uint64 *p_app_start_offset, + uint64 *p_app_end_offset); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst, + uint8 *native_ptr, + uint8 **p_native_start_addr, + uint8 **p_native_end_addr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN const uint8 * +wasm_runtime_get_custom_section(WASMModuleCommon *const module_comm, + const char *name, uint32 *len); + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_reader(const module_reader reader, + const module_destroyer destroyer); + +module_reader +wasm_runtime_get_module_reader(); + +module_destroyer +wasm_runtime_get_module_destroyer(); + +bool +wasm_runtime_register_module_internal(const char *module_name, + WASMModuleCommon *module, + uint8 *orig_file_buf, + uint32 orig_file_buf_size, + char *error_buf, uint32 error_buf_size); + +void +wasm_runtime_unregister_module(const WASMModuleCommon *module); + +WASMModuleCommon * +wasm_runtime_find_module_registered(const char *module_name); + +bool +wasm_runtime_add_loading_module(const char *module_name, char *error_buf, + uint32 error_buf_size); + +void +wasm_runtime_delete_loading_module(const char *module_name); + +bool +wasm_runtime_is_loading_module(const char *module_name); + +void +wasm_runtime_destroy_loading_module_list(); + +WASMModuleCommon * +wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name); + +bool +wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, + WASMModuleCommon *sub_module); + +WASMModuleCommon * +wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, char *error_buf, + uint32 error_buf_size); + +bool +wasm_runtime_sub_module_instantiate(WASMModuleCommon *module, + WASMModuleInstanceCommon *module_inst, + uint32 stack_size, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size); +void +wasm_runtime_sub_module_deinstantiate(WASMModuleInstanceCommon *module_inst); +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0 +WASMExport * +loader_find_export(const WASMModuleCommon *module, const char *module_name, + const char *field_name, uint8 export_kind, char *error_buf, + uint32 error_buf_size); +#endif /* WASM_ENALBE_MULTI_MODULE */ + +bool +wasm_runtime_is_built_in_module(const char *module_name); + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_exec_env_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, + uint32 *size); + +bool +wasm_exec_env_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, + uint32 size); +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args_ex(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc, + int64 stdinfd, int64 stdoutfd, int64 stderrfd); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_wasi_mode(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMFunctionInstanceCommon * +wasm_runtime_lookup_wasi_start_function(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_wasi_exit_code(WASMModuleInstanceCommon *module_inst); + +bool +wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, + const char *dir_list[], uint32 dir_count, + const char *map_dir_list[], uint32 map_dir_count, + const char *env[], uint32 env_count, + const char *addr_pool[], uint32 addr_pool_size, + const char *ns_lookup_pool[], uint32 ns_lookup_pool_size, + char *argv[], uint32 argc, os_raw_file_handle stdinfd, + os_raw_file_handle stdoutfd, os_raw_file_handle stderrfd, + char *error_buf, uint32 error_buf_size); + +void +wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst); + +void +wasm_runtime_set_wasi_ctx(WASMModuleInstanceCommon *module_inst, + WASIContext *wasi_ctx); + +WASIContext * +wasm_runtime_get_wasi_ctx(WASMModuleInstanceCommon *module_inst); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_addr_pool(wasm_module_t module, const char *addr_pool[], + uint32 addr_pool_size); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_ns_lookup_pool(wasm_module_t module, + const char *ns_lookup_pool[], + uint32 ns_lookup_pool_size); +#endif /* end of WASM_ENABLE_LIBC_WASI */ + +#if WASM_ENABLE_GC != 0 +void +wasm_runtime_set_gc_heap_handle(WASMModuleInstanceCommon *module_inst, + void *gc_heap_handle); + +void * +wasm_runtime_get_gc_heap_handle(WASMModuleInstanceCommon *module_inst); +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_obj2ref(WASMModuleInstanceCommon *module_inst, void *extern_obj, + uint32 *p_externref_idx); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_ref2obj(uint32 externref_idx, void **p_extern_obj); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_retain(uint32 externref_idx); + +/** + * Reclaim the externref objects/indexes which are not used by + * module instance + */ +void +wasm_externref_reclaim(WASMModuleInstanceCommon *module_inst); + +/** + * Cleanup the externref objects/indexes of the module instance + */ +void +wasm_externref_cleanup(WASMModuleInstanceCommon *module_inst); +#endif /* end of WASM_ENABLE_REF_TYPES */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +/** + * @brief Internal implementation for dumping or printing callstack line + * + * @note if dump_or_print is true, then print to stdout directly; + * if dump_or_print is false, but *buf is NULL, then return the length of the + * line; + * if dump_or_print is false, and *buf is not NULL, then dump content to + * the memory pointed by *buf, and adjust *buf and *len according to actual + * bytes dumped, and return the actual dumped length + * + * @param line_buf current line to dump or print + * @param dump_or_print whether to print to stdout or dump to buf + * @param buf [INOUT] pointer to the buffer + * @param len [INOUT] pointer to remaining length + * @return bytes printed to stdout or dumped to buf + */ +uint32 +wasm_runtime_dump_line_buf_impl(const char *line_buf, bool dump_or_print, + char **buf, uint32 *len); +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 */ + +/* Get module of the current exec_env */ +WASMModuleCommon * +wasm_exec_env_get_module(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_unregister_natives(const char *module_name, + NativeSymbol *native_symbols); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_create_context_key(void (*dtor)(WASMModuleInstanceCommon *inst, + void *ctx)); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_context_key(void *key); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context(WASMModuleInstanceCommon *inst, void *key, void *ctx); +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context_spread(WASMModuleInstanceCommon *inst, void *key, + void *ctx); +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_context(WASMModuleInstanceCommon *inst, void *key); + +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *ret); + +bool +wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, + const char *signature, void *attachment, + uint32 *argv, uint32 argc, uint32 *ret); + +void +wasm_runtime_read_v128(const uint8 *bytes, uint64 *ret1, uint64 *ret2); + +void +wasm_runtime_dump_module_mem_consumption(const WASMModuleCommon *module); + +void +wasm_runtime_dump_module_inst_mem_consumption( + const WASMModuleInstanceCommon *module_inst); + +void +wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env); + +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + WASMFuncType **out); + +bool +wasm_runtime_get_export_global_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + uint8 *out_val_type, bool *out_mutability); + +bool +wasm_runtime_get_export_memory_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + uint32 *out_min_page, uint32 *out_max_page); + +bool +wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, + void *func_ptr, WASMFuncType *func_type, + uint32 argc, uint32 *argv, bool with_env, + void *wasm_c_api_env); + +struct CApiFuncImport; +/* A quick version of wasm_runtime_invoke_c_api_native to directly invoke + wasm-c-api import function from jitted code to improve performance */ +bool +wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, + struct CApiFuncImport *c_api_import, + wasm_val_t *params, uint32 param_count, + wasm_val_t *results, + uint32 result_count); + +void +wasm_runtime_show_app_heap_corrupted_prompt(); + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +void +wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list); +#endif + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_func_linked(const char *module_name, + const char *func_name); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_global_linked(const char *module_name, + const char *global_name); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_begin_blocking_op(WASMExecEnv *exec_env); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_end_blocking_op(WASMExecEnv *exec_env); + +void +wasm_runtime_interrupt_blocking_op(WASMExecEnv *exec_env); + +#if WASM_ENABLE_LINUX_PERF != 0 +bool +wasm_runtime_get_linux_perf(void); + +void +wasm_runtime_set_linux_perf(bool flag); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_COMMON_H */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_shared_memory.c b/wasm-micro-runtime/core/iwasm/common/wasm_shared_memory.c new file mode 100644 index 0000000..3856b7d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_shared_memory.c @@ -0,0 +1,436 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_log.h" +#include "wasm_shared_memory.h" +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif + +/* + * Note: this lock can be per memory. + * + * For now, just use a global because: + * - it's a bit cumbersome to extend WASMMemoryInstance w/o breaking + * the AOT ABI. + * - If you care performance, it's better to make the interpreters + * use atomic ops. + */ +korp_mutex g_shared_memory_lock; + +/* clang-format off */ +enum { + S_WAITING, + S_NOTIFIED +}; +/* clang-format on */ + +typedef struct AtomicWaitInfo { + bh_list wait_list_head; + bh_list *wait_list; + /* WARNING: insert to the list allowed only in acquire_wait_info + otherwise there will be data race as described in PR #2016 */ +} AtomicWaitInfo; + +typedef struct AtomicWaitNode { + bh_list_link l; + uint8 status; + korp_cond wait_cond; +} AtomicWaitNode; + +/* Atomic wait map */ +static HashMap *wait_map; + +static uint32 +wait_address_hash(const void *address); + +static bool +wait_address_equal(void *h1, void *h2); + +static void +destroy_wait_info(void *wait_info); + +bool +wasm_shared_memory_init() +{ + if (os_mutex_init(&g_shared_memory_lock) != 0) + return false; + /* wait map not exists, create new map */ + if (!(wait_map = bh_hash_map_create(32, true, (HashFunc)wait_address_hash, + (KeyEqualFunc)wait_address_equal, NULL, + destroy_wait_info))) { + os_mutex_destroy(&g_shared_memory_lock); + return false; + } + return true; +} + +void +wasm_shared_memory_destroy() +{ + bh_hash_map_destroy(wait_map); + os_mutex_destroy(&g_shared_memory_lock); +} + +uint16 +shared_memory_inc_reference(WASMMemoryInstance *memory) +{ + bh_assert(shared_memory_is_shared(memory)); + uint16 old; +#if BH_ATOMIC_16_IS_ATOMIC == 0 + os_mutex_lock(&g_shared_memory_lock); +#endif + old = BH_ATOMIC_16_FETCH_ADD(memory->ref_count, 1); +#if BH_ATOMIC_16_IS_ATOMIC == 0 + os_mutex_unlock(&g_shared_memory_lock); +#endif + bh_assert(old >= 1); + bh_assert(old < UINT16_MAX); + return old + 1; +} + +uint16 +shared_memory_dec_reference(WASMMemoryInstance *memory) +{ + bh_assert(shared_memory_is_shared(memory)); + uint16 old; +#if BH_ATOMIC_16_IS_ATOMIC == 0 + os_mutex_lock(&g_shared_memory_lock); +#endif + old = BH_ATOMIC_16_FETCH_SUB(memory->ref_count, 1); +#if BH_ATOMIC_16_IS_ATOMIC == 0 + os_mutex_unlock(&g_shared_memory_lock); +#endif + bh_assert(old > 0); + return old - 1; +} + +static korp_mutex * +shared_memory_get_lock_pointer(WASMMemoryInstance *memory) +{ + bh_assert(memory != NULL); + return &g_shared_memory_lock; +} + +/* Atomics wait && notify APIs */ +static uint32 +wait_address_hash(const void *address) +{ + return (uint32)(uintptr_t)address; +} + +static bool +wait_address_equal(void *h1, void *h2) +{ + return h1 == h2 ? true : false; +} + +static bool +is_wait_node_exists(bh_list *wait_list, AtomicWaitNode *node) +{ + AtomicWaitNode *curr; + curr = bh_list_first_elem(wait_list); + + while (curr) { + if (curr == node) { + return true; + } + curr = bh_list_elem_next(curr); + } + + return false; +} + +static uint32 +notify_wait_list(bh_list *wait_list, uint32 count) +{ + AtomicWaitNode *node, *next; + uint32 i, notify_count = count; + + if (count > wait_list->len) + notify_count = wait_list->len; + + node = bh_list_first_elem(wait_list); + if (!node) + return 0; + + for (i = 0; i < notify_count; i++) { + bh_assert(node); + next = bh_list_elem_next(node); + + node->status = S_NOTIFIED; + /* wakeup */ + os_cond_signal(&node->wait_cond); + + node = next; + } + + return notify_count; +} + +static AtomicWaitInfo * +acquire_wait_info(void *address, AtomicWaitNode *wait_node) +{ + AtomicWaitInfo *wait_info = NULL; + bh_list_status ret; + + bh_assert(address != NULL); + + wait_info = (AtomicWaitInfo *)bh_hash_map_find(wait_map, address); + + if (!wait_node) { + return wait_info; + } + + /* No wait info on this address, create new info */ + if (!wait_info) { + if (!(wait_info = (AtomicWaitInfo *)wasm_runtime_malloc( + sizeof(AtomicWaitInfo)))) { + return NULL; + } + memset(wait_info, 0, sizeof(AtomicWaitInfo)); + + /* init wait list */ + wait_info->wait_list = &wait_info->wait_list_head; + ret = bh_list_init(wait_info->wait_list); + bh_assert(ret == BH_LIST_SUCCESS); + (void)ret; + + if (!bh_hash_map_insert(wait_map, address, (void *)wait_info)) { + wasm_runtime_free(wait_info); + return NULL; + } + } + + ret = bh_list_insert(wait_info->wait_list, wait_node); + bh_assert(ret == BH_LIST_SUCCESS); + (void)ret; + + return wait_info; +} + +static void +destroy_wait_info(void *wait_info) +{ + AtomicWaitNode *node, *next; + + if (wait_info) { + + node = bh_list_first_elem(((AtomicWaitInfo *)wait_info)->wait_list); + + while (node) { + next = bh_list_elem_next(node); + os_cond_destroy(&node->wait_cond); + wasm_runtime_free(node); + node = next; + } + + wasm_runtime_free(wait_info); + } +} + +static void +map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info, + void *address) +{ + if (wait_info->wait_list->len > 0) { + return; + } + + bh_hash_map_remove(wait_hash_map, address, NULL, NULL); + destroy_wait_info(wait_info); +} + +uint32 +wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, + uint64 expect, int64 timeout, bool wait64) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module; + AtomicWaitInfo *wait_info; + AtomicWaitNode *wait_node; + korp_mutex *lock; +#if WASM_ENABLE_THREAD_MGR != 0 + WASMExecEnv *exec_env; +#endif + uint64 timeout_left, timeout_wait, timeout_1sec; + bool check_ret, is_timeout, no_wait; + + bh_assert(module->module_type == Wasm_Module_Bytecode + || module->module_type == Wasm_Module_AoT); + + if (wasm_copy_exception(module_inst, NULL)) { + return -1; + } + + /* Currently we have only one memory instance */ + if (!shared_memory_is_shared(module_inst->memories[0])) { + wasm_runtime_set_exception(module, "expected shared memory"); + return -1; + } + + shared_memory_lock(module_inst->memories[0]); + if ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + (wait64 ? 8 : 4) + > module_inst->memories[0]->memory_data_end) { + shared_memory_unlock(module_inst->memories[0]); + wasm_runtime_set_exception(module, "out of bounds memory access"); + return -1; + } + shared_memory_unlock(module_inst->memories[0]); + +#if WASM_ENABLE_THREAD_MGR != 0 + exec_env = + wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst); + bh_assert(exec_env); +#endif + + lock = shared_memory_get_lock_pointer(module_inst->memories[0]); + + /* Lock the shared_mem_lock for the whole atomic wait process, + and use it to os_cond_reltimedwait */ + os_mutex_lock(lock); + + no_wait = (!wait64 && *(uint32 *)address != (uint32)expect) + || (wait64 && *(uint64 *)address != expect); + + if (no_wait) { + os_mutex_unlock(lock); + return 1; + } + + if (!(wait_node = wasm_runtime_malloc(sizeof(AtomicWaitNode)))) { + os_mutex_unlock(lock); + wasm_runtime_set_exception(module, "failed to create wait node"); + return -1; + } + memset(wait_node, 0, sizeof(AtomicWaitNode)); + + if (0 != os_cond_init(&wait_node->wait_cond)) { + os_mutex_unlock(lock); + wasm_runtime_free(wait_node); + wasm_runtime_set_exception(module, "failed to init wait cond"); + return -1; + } + + wait_node->status = S_WAITING; + + /* Acquire the wait info, create new one if not exists */ + wait_info = acquire_wait_info(address, wait_node); + + if (!wait_info) { + os_mutex_unlock(lock); + os_cond_destroy(&wait_node->wait_cond); + wasm_runtime_free(wait_node); + wasm_runtime_set_exception(module, "failed to acquire wait_info"); + return -1; + } + + /* unit of timeout is nsec, convert it to usec */ + timeout_left = (uint64)timeout / 1000; + timeout_1sec = (uint64)1e6; + + while (1) { + if (timeout < 0) { + /* wait forever until it is notified or terminatied + here we keep waiting and checking every second */ + os_cond_reltimedwait(&wait_node->wait_cond, lock, + (uint64)timeout_1sec); + if (wait_node->status == S_NOTIFIED /* notified by atomic.notify */ +#if WASM_ENABLE_THREAD_MGR != 0 + /* terminated by other thread */ + || wasm_cluster_is_thread_terminated(exec_env) +#endif + ) { + break; + } + } + else { + timeout_wait = + timeout_left < timeout_1sec ? timeout_left : timeout_1sec; + os_cond_reltimedwait(&wait_node->wait_cond, lock, timeout_wait); + if (wait_node->status == S_NOTIFIED /* notified by atomic.notify */ + || timeout_left <= timeout_wait /* time out */ +#if WASM_ENABLE_THREAD_MGR != 0 + /* terminated by other thread */ + || wasm_cluster_is_thread_terminated(exec_env) +#endif + ) { + break; + } + timeout_left -= timeout_wait; + } + } + + is_timeout = wait_node->status == S_WAITING ? true : false; + + check_ret = is_wait_node_exists(wait_info->wait_list, wait_node); + bh_assert(check_ret); + (void)check_ret; + + /* Remove wait node from wait list */ + bh_list_remove(wait_info->wait_list, wait_node); + os_cond_destroy(&wait_node->wait_cond); + wasm_runtime_free(wait_node); + + /* Release wait info if no wait nodes are attached */ + map_try_release_wait_info(wait_map, wait_info, address); + + os_mutex_unlock(lock); + + return is_timeout ? 2 : 0; +} + +uint32 +wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, + uint32 count) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module; + uint32 notify_result; + AtomicWaitInfo *wait_info; + korp_mutex *lock; + bool out_of_bounds; + + bh_assert(module->module_type == Wasm_Module_Bytecode + || module->module_type == Wasm_Module_AoT); + + shared_memory_lock(module_inst->memories[0]); + out_of_bounds = + ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end); + shared_memory_unlock(module_inst->memories[0]); + + if (out_of_bounds) { + wasm_runtime_set_exception(module, "out of bounds memory access"); + return -1; + } + + /* Currently we have only one memory instance */ + if (!shared_memory_is_shared(module_inst->memories[0])) { + /* Always return 0 for ushared linear memory since there is + no way to create a waiter on it */ + return 0; + } + + lock = shared_memory_get_lock_pointer(module_inst->memories[0]); + + /* Lock the shared_mem_lock for the whole atomic notify process, + and use it to os_cond_signal */ + os_mutex_lock(lock); + + wait_info = acquire_wait_info(address, NULL); + + /* Nobody wait on this address */ + if (!wait_info) { + os_mutex_unlock(lock); + return 0; + } + + /* Notify each wait node in the wait list */ + notify_result = notify_wait_list(wait_info->wait_list, count); + + os_mutex_unlock(lock); + + return notify_result; +} diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_shared_memory.h b/wasm-micro-runtime/core/iwasm/common/wasm_shared_memory.h new file mode 100644 index 0000000..8bbc4a8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_shared_memory.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_SHARED_MEMORY_H +#define _WASM_SHARED_MEMORY_H + +#include "bh_common.h" +#include "../interpreter/wasm_runtime.h" +#include "wasm_runtime_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern korp_mutex g_shared_memory_lock; + +bool +wasm_shared_memory_init(); + +void +wasm_shared_memory_destroy(); + +uint16 +shared_memory_inc_reference(WASMMemoryInstance *memory); + +uint16 +shared_memory_dec_reference(WASMMemoryInstance *memory); + +#define shared_memory_is_shared(memory) memory->is_shared_memory + +#define shared_memory_lock(memory) \ + do { \ + /* \ + * Note: exception logic is currently abusing this lock. \ + * cf. \ + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/2407 \ + */ \ + bh_assert(memory != NULL); \ + if (memory->is_shared_memory) \ + os_mutex_lock(&g_shared_memory_lock); \ + } while (0) + +#define shared_memory_unlock(memory) \ + do { \ + if (memory->is_shared_memory) \ + os_mutex_unlock(&g_shared_memory_lock); \ + } while (0) + +uint32 +wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, + uint64 expect, int64 timeout, bool wait64); + +uint32 +wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, + uint32 count); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_SHARED_MEMORY_H */ diff --git a/wasm-micro-runtime/core/iwasm/common/wasm_suspend_flags.h b/wasm-micro-runtime/core/iwasm/common/wasm_suspend_flags.h new file mode 100644 index 0000000..92661b7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/common/wasm_suspend_flags.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_SUSPEND_FLAGS_H +#define _WASM_SUSPEND_FLAGS_H + +#include "bh_atomic.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Need to terminate */ +#define WASM_SUSPEND_FLAG_TERMINATE 0x1 +/* Need to suspend */ +#define WASM_SUSPEND_FLAG_SUSPEND 0x2 +/* Need to go into breakpoint */ +#define WASM_SUSPEND_FLAG_BREAKPOINT 0x4 +/* Return from pthread_exit */ +#define WASM_SUSPEND_FLAG_EXIT 0x8 +/* The thread might be blocking */ +#define WASM_SUSPEND_FLAG_BLOCKING 0x10 + +typedef union WASMSuspendFlags { + bh_atomic_32_t flags; + uintptr_t __padding__; +} WASMSuspendFlags; + +#define WASM_SUSPEND_FLAGS_IS_ATOMIC BH_ATOMIC_32_IS_ATOMIC +#define WASM_SUSPEND_FLAGS_GET(s_flags) BH_ATOMIC_32_LOAD(s_flags.flags) +#define WASM_SUSPEND_FLAGS_FETCH_OR(s_flags, val) \ + BH_ATOMIC_32_FETCH_OR(s_flags.flags, val) +#define WASM_SUSPEND_FLAGS_FETCH_AND(s_flags, val) \ + BH_ATOMIC_32_FETCH_AND(s_flags.flags, val) + +#define WASM_SUSPEND_FLAG_INHERIT_MASK (~WASM_SUSPEND_FLAG_BLOCKING) + +#if WASM_SUSPEND_FLAGS_IS_ATOMIC != 0 +#define WASM_SUSPEND_FLAGS_LOCK(lock) (void)0 +#define WASM_SUSPEND_FLAGS_UNLOCK(lock) (void)0 +#else /* else of WASM_SUSPEND_FLAGS_IS_ATOMIC */ +#define WASM_SUSPEND_FLAGS_LOCK(lock) os_mutex_lock(&lock) +#define WASM_SUSPEND_FLAGS_UNLOCK(lock) os_mutex_unlock(&lock); +#endif /* WASM_SUSPEND_FLAGS_IS_ATOMIC */ + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_SUSPEND_FLAGS_H */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot.c b/wasm-micro-runtime/core/iwasm/compilation/aot.c new file mode 100644 index 0000000..f4532a5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot.c @@ -0,0 +1,774 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot.h" + +static char aot_error[128]; + +char * +aot_get_last_error() +{ + return aot_error[0] == '\0' ? "" : aot_error; +} + +void +aot_set_last_error_v(const char *format, ...) +{ + va_list args; + va_start(args, format); + vsnprintf(aot_error, sizeof(aot_error), format, args); + va_end(args); +} + +void +aot_set_last_error(const char *error) +{ + if (error) + snprintf(aot_error, sizeof(aot_error), "Error: %s", error); + else + aot_error[0] = '\0'; +} + +static void +aot_destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count) +{ + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) + wasm_runtime_free(data_list[i]); + wasm_runtime_free(data_list); +} + +static AOTMemInitData ** +aot_create_mem_init_data_list(const WASMModule *module) +{ + AOTMemInitData **data_list; + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTMemInitData *) * (uint64)module->data_seg_count; + if (size >= UINT32_MAX + || !(data_list = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(data_list, 0, size); + + /* Create each memory data segment */ + for (i = 0; i < module->data_seg_count; i++) { + size = offsetof(AOTMemInitData, bytes) + + (uint64)module->data_segments[i]->data_length; + if (size >= UINT32_MAX + || !(data_list[i] = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + data_list[i]->is_passive = module->data_segments[i]->is_passive; + data_list[i]->memory_index = module->data_segments[i]->memory_index; +#endif + data_list[i]->offset = module->data_segments[i]->base_offset; + data_list[i]->byte_count = module->data_segments[i]->data_length; + memcpy(data_list[i]->bytes, module->data_segments[i]->data, + module->data_segments[i]->data_length); + } + + return data_list; + +fail: + aot_destroy_mem_init_data_list(data_list, module->data_seg_count); + return NULL; +} + +static void +aot_destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count) +{ + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) + wasm_runtime_free(data_list[i]); + wasm_runtime_free(data_list); +} + +static AOTTableInitData ** +aot_create_table_init_data_list(const WASMModule *module) +{ + AOTTableInitData **data_list; + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTTableInitData *) * (uint64)module->table_seg_count; + if (size >= UINT32_MAX + || !(data_list = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(data_list, 0, size); + + /* Create each table data segment */ + for (i = 0; i < module->table_seg_count; i++) { + size = offsetof(AOTTableInitData, init_values) + + sizeof(InitializerExpression) + * (uint64)module->table_segments[i].value_count; + if (size >= UINT32_MAX + || !(data_list[i] = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + + data_list[i]->offset = module->table_segments[i].base_offset; + data_list[i]->value_count = module->table_segments[i].value_count; + data_list[i]->mode = module->table_segments[i].mode; + data_list[i]->elem_type = module->table_segments[i].elem_type; + /* runtime control it */ + data_list[i]->table_index = module->table_segments[i].table_index; + bh_memcpy_s(&data_list[i]->offset, sizeof(AOTInitExpr), + &module->table_segments[i].base_offset, + sizeof(AOTInitExpr)); + data_list[i]->value_count = module->table_segments[i].value_count; +#if WASM_ENABLE_GC != 0 + data_list[i]->elem_ref_type = module->table_segments[i].elem_ref_type; +#endif + bh_memcpy_s(data_list[i]->init_values, + sizeof(InitializerExpression) + * module->table_segments[i].value_count, + module->table_segments[i].init_values, + sizeof(InitializerExpression) + * module->table_segments[i].value_count); + } + + return data_list; + +fail: + aot_destroy_table_init_data_list(data_list, module->table_seg_count); + return NULL; +} + +static void +get_value_type_size(uint8 value_type, bool gc_enabled, uint32 *p_size_64bit, + uint32 *p_size_32bit) +{ + uint32 size_64bit = 0, size_32bit = 0; + + if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_F32) + size_64bit = size_32bit = sizeof(int32); + else if (value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F64) + size_64bit = size_32bit = sizeof(int64); + else if (value_type == VALUE_TYPE_V128) + size_64bit = size_32bit = sizeof(int64) * 2; + else if (!gc_enabled + && (value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF)) + size_64bit = size_32bit = sizeof(int32); + else if (gc_enabled + && ((value_type >= (uint8)REF_TYPE_ARRAYREF + && value_type <= (uint8)REF_TYPE_NULLFUNCREF) + || (value_type >= (uint8)REF_TYPE_HT_NULLABLE + && value_type <= (uint8)REF_TYPE_HT_NON_NULLABLE) +#if WASM_ENABLE_STRINGREF != 0 + || (value_type >= (uint8)REF_TYPE_STRINGVIEWWTF8 + && value_type <= (uint8)REF_TYPE_STRINGREF) + || (value_type >= (uint8)REF_TYPE_STRINGVIEWITER + && value_type <= (uint8)REF_TYPE_STRINGVIEWWTF16) +#endif + )) { + size_64bit = sizeof(uint64); + size_32bit = sizeof(uint32); + } + else if (gc_enabled && value_type == PACKED_TYPE_I8) { + size_64bit = size_32bit = sizeof(int8); + } + else if (gc_enabled && value_type == PACKED_TYPE_I16) { + size_64bit = size_32bit = sizeof(int16); + } + else { + bh_assert(0); + } + + *p_size_64bit = size_64bit; + *p_size_32bit = size_32bit; +} + +static AOTImportGlobal * +aot_create_import_globals(const WASMModule *module, bool gc_enabled, + uint32 *p_import_global_data_size_64bit, + uint32 *p_import_global_data_size_32bit) +{ + AOTImportGlobal *import_globals; + uint64 size; + uint32 i, data_offset_64bit = 0, data_offset_32bit = 0; + uint32 value_size_64bit, value_size_32bit; + + /* Allocate memory */ + size = sizeof(AOTImportGlobal) * (uint64)module->import_global_count; + if (size >= UINT32_MAX + || !(import_globals = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(import_globals, 0, (uint32)size); + + /* Create each import global */ + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *import_global = &module->import_globals[i].u.global; + import_globals[i].module_name = import_global->module_name; + import_globals[i].global_name = import_global->field_name; + import_globals[i].type = import_global->type; + import_globals[i].is_mutable = import_global->is_mutable; + import_globals[i].global_data_linked = + import_global->global_data_linked; + + import_globals[i].data_offset_64bit = data_offset_64bit; + import_globals[i].data_offset_32bit = data_offset_32bit; + + get_value_type_size(import_global->type, gc_enabled, &value_size_64bit, + &value_size_32bit); + + import_globals[i].size_64bit = value_size_64bit; + import_globals[i].size_32bit = value_size_32bit; + data_offset_64bit += value_size_64bit; + data_offset_32bit += value_size_32bit; + } + + *p_import_global_data_size_64bit = data_offset_64bit; + *p_import_global_data_size_32bit = data_offset_32bit; + return import_globals; +} + +static AOTGlobal * +aot_create_globals(const WASMModule *module, bool gc_enabled, + uint32 global_data_start_offset_64bit, + uint32 global_data_start_offset_32bit, + uint32 *p_global_data_size_64bit, + uint32 *p_global_data_size_32bit) +{ + AOTGlobal *globals; + uint64 size; + uint32 i; + uint32 data_offset_64bit = global_data_start_offset_64bit; + uint32 data_offset_32bit = global_data_start_offset_32bit; + uint32 value_size_64bit, value_size_32bit; + + /* Allocate memory */ + size = sizeof(AOTGlobal) * (uint64)module->global_count; + if (size >= UINT32_MAX || !(globals = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(globals, 0, (uint32)size); + + /* Create each global */ + for (i = 0; i < module->global_count; i++) { + WASMGlobal *global = &module->globals[i]; + globals[i].type = global->type; + globals[i].is_mutable = global->is_mutable; + memcpy(&globals[i].init_expr, &global->init_expr, + sizeof(global->init_expr)); + + globals[i].data_offset_64bit = data_offset_64bit; + globals[i].data_offset_32bit = data_offset_32bit; + + get_value_type_size(global->type, gc_enabled, &value_size_64bit, + &value_size_32bit); + + globals[i].size_64bit = value_size_64bit; + globals[i].size_32bit = value_size_32bit; + data_offset_64bit += value_size_64bit; + data_offset_32bit += value_size_32bit; + } + + *p_global_data_size_64bit = + data_offset_64bit - global_data_start_offset_64bit; + *p_global_data_size_32bit = + data_offset_32bit - global_data_start_offset_32bit; + return globals; +} + +static AOTImportFunc * +aot_create_import_funcs(const WASMModule *module) +{ + AOTImportFunc *import_funcs; + uint64 size; + uint32 i, j; + + /* Allocate memory */ + size = sizeof(AOTImportFunc) * (uint64)module->import_function_count; + if (size >= UINT32_MAX + || !(import_funcs = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + /* Create each import function */ + for (i = 0; i < module->import_function_count; i++) { + WASMFunctionImport *import_func = + &module->import_functions[i].u.function; + import_funcs[i].module_name = import_func->module_name; + import_funcs[i].func_name = import_func->field_name; + import_funcs[i].func_ptr_linked = import_func->func_ptr_linked; + import_funcs[i].func_type = import_func->func_type; + import_funcs[i].signature = import_func->signature; + import_funcs[i].attachment = import_func->attachment; + import_funcs[i].call_conv_raw = import_func->call_conv_raw; + import_funcs[i].call_conv_wasm_c_api = false; + /* Resolve function type index */ + for (j = 0; j < module->type_count; j++) + if (import_func->func_type == (WASMFuncType *)module->types[j]) { + import_funcs[i].func_type_index = j; + break; + } + } + + return import_funcs; +} + +static void +aot_destroy_funcs(AOTFunc **funcs, uint32 count) +{ + uint32 i; + + for (i = 0; i < count; i++) + if (funcs[i]) { + if (funcs[i]->local_offsets) + wasm_runtime_free(funcs[i]->local_offsets); + wasm_runtime_free(funcs[i]); + } + wasm_runtime_free(funcs); +} + +static AOTFunc ** +aot_create_funcs(const WASMModule *module, uint32 pointer_size) +{ + AOTFunc **funcs; + uint64 size; + uint32 i, j; + + /* Allocate memory */ + size = sizeof(AOTFunc *) * (uint64)module->function_count; + if (size >= UINT32_MAX || !(funcs = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(funcs, 0, size); + + /* Create each function */ + for (i = 0; i < module->function_count; i++) { + WASMFunction *func = module->functions[i]; + AOTFuncType *func_type = NULL; + AOTFunc *aot_func = NULL; + uint64 total_size; + uint32 offset = 0; + + size = sizeof(AOTFunc); + if (!(aot_func = funcs[i] = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + memset(aot_func, 0, sizeof(AOTFunc)); + + func_type = aot_func->func_type = func->func_type; + + /* Resolve function type index */ + for (j = 0; j < module->type_count; j++) { + if (func_type == (WASMFuncType *)module->types[j]) { + aot_func->func_type_index = j; + break; + } + } + + total_size = sizeof(uint16) + * ((uint64)func_type->param_count + func->local_count); + if ((total_size > 0) + && (total_size >= UINT32_MAX + || !(aot_func->local_offsets = + wasm_runtime_malloc((uint32)total_size)))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + + /* Resolve local variable info and code info */ + aot_func->local_count = func->local_count; + aot_func->local_types_wp = func->local_types; + aot_func->code = func->code; + aot_func->code_size = func->code_size; + + /* Resolve local offsets */ + for (j = 0; j < func_type->param_count; j++) { + aot_func->local_offsets[j] = (uint16)offset; + offset += wasm_value_type_cell_num_internal(func_type->types[j], + pointer_size); + } + aot_func->param_cell_num = offset; + + for (j = 0; j < func->local_count; j++) { + aot_func->local_offsets[func_type->param_count + j] = + (uint16)offset; + offset += wasm_value_type_cell_num_internal(func->local_types[j], + pointer_size); + } + aot_func->local_cell_num = offset - aot_func->param_cell_num; + + aot_func->max_stack_cell_num = func->max_stack_cell_num; + /* We use max_stack_cell_num calculated from wasm_loader, which is based + * on wamrc's target type. + * - If the wamrc is compiled as 64bit, then the number is enough for + * both 32bit and 64bit runtime target + * - If the wamrc is compiled as 32bit, then we multiply this number by + * two to avoid overflow on 64bit runtime target */ + if (sizeof(uintptr_t) == 4) { + aot_func->max_stack_cell_num *= 2; + } + } + + return funcs; + +fail: + aot_destroy_funcs(funcs, module->function_count); + return NULL; +} + +#if WASM_ENABLE_GC != 0 +static void +calculate_struct_field_sizes_offsets(AOTCompData *comp_data, bool is_target_x86, + bool gc_enabled) +{ + uint32 i; + + for (i = 0; i < comp_data->type_count; i++) { + if (comp_data->types[i]->type_flag == WASM_TYPE_STRUCT) { + WASMStructType *struct_type = (WASMStructType *)comp_data->types[i]; + WASMStructFieldType *fields = struct_type->fields; + uint32 field_offset_64bit, field_offset_32bit; + uint32 field_size_64bit, field_size_32bit, j; + + /* offsetof(WASMStructObject, field_data) in 64-bit */ + field_offset_64bit = sizeof(uint64); + + /* offsetof(WASMStructObject, field_data) in 32-bit */ + field_offset_32bit = sizeof(uint32); + + for (j = 0; j < struct_type->field_count; j++) { + get_value_type_size(fields[j].field_type, gc_enabled, + &field_size_64bit, &field_size_32bit); + + fields[j].field_size_64bit = field_size_64bit; + fields[j].field_size_32bit = field_size_32bit; + + if (!is_target_x86) { + if (field_size_64bit == 2) + field_offset_64bit = align_uint(field_offset_64bit, 2); + else if (field_size_64bit >= 4) + field_offset_64bit = align_uint(field_offset_64bit, 4); + + if (field_size_32bit == 2) + field_offset_32bit = align_uint(field_offset_32bit, 2); + else if (field_size_32bit >= 4) + field_offset_32bit = align_uint(field_offset_32bit, 4); + } + + fields[j].field_offset_64bit = field_offset_64bit; + fields[j].field_offset_32bit = field_offset_32bit; + + field_offset_64bit += field_size_64bit; + field_offset_32bit += field_size_32bit; + } + } + } +} +#endif + +AOTCompData * +aot_create_comp_data(WASMModule *module, const char *target_arch, + bool gc_enabled) +{ + AOTCompData *comp_data; + uint32 import_global_data_size_64bit = 0, global_data_size_64bit = 0, i, j; + uint32 import_global_data_size_32bit = 0, global_data_size_32bit = 0; + uint64 size; + bool is_64bit_target = false; +#if WASM_ENABLE_GC != 0 + bool is_target_x86 = false; +#endif + +#if WASM_ENABLE_GC != 0 + if (!target_arch) { +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_X86_32) + is_target_x86 = true; +#endif + } + else { + if (!strncmp(target_arch, "x86_64", 6) + || !strncmp(target_arch, "i386", 4)) + is_target_x86 = true; + } +#endif + + if (!target_arch) { +#if UINTPTR_MAX == UINT64_MAX + is_64bit_target = true; +#endif + } + else { + /* All 64bit targets contains "64" string in their target name */ + if (strstr(target_arch, "64") != NULL) { + is_64bit_target = true; + } + } + + /* Allocate memory */ + if (!(comp_data = wasm_runtime_malloc(sizeof(AOTCompData)))) { + aot_set_last_error("create compile data failed.\n"); + return NULL; + } + + memset(comp_data, 0, sizeof(AOTCompData)); + + comp_data->memory_count = + module->import_memory_count + module->memory_count; + + /* TODO: create import memories */ + + /* Allocate memory for memory array, reserve one AOTMemory space at least */ + if (!comp_data->memory_count) + comp_data->memory_count = 1; + + size = (uint64)comp_data->memory_count * sizeof(AOTMemory); + if (size >= UINT32_MAX + || !(comp_data->memories = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("create memories array failed.\n"); + goto fail; + } + memset(comp_data->memories, 0, size); + + if (!(module->import_memory_count + module->memory_count)) { + comp_data->memories[0].num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + } + + /* Set memory page count */ + for (i = 0; i < module->import_memory_count + module->memory_count; i++) { + if (i < module->import_memory_count) { + comp_data->memories[i].memory_flags = + module->import_memories[i].u.memory.flags; + comp_data->memories[i].num_bytes_per_page = + module->import_memories[i].u.memory.num_bytes_per_page; + comp_data->memories[i].mem_init_page_count = + module->import_memories[i].u.memory.init_page_count; + comp_data->memories[i].mem_max_page_count = + module->import_memories[i].u.memory.max_page_count; + comp_data->memories[i].num_bytes_per_page = + module->import_memories[i].u.memory.num_bytes_per_page; + } + else { + j = i - module->import_memory_count; + comp_data->memories[i].memory_flags = module->memories[j].flags; + comp_data->memories[i].num_bytes_per_page = + module->memories[j].num_bytes_per_page; + comp_data->memories[i].mem_init_page_count = + module->memories[j].init_page_count; + comp_data->memories[i].mem_max_page_count = + module->memories[j].max_page_count; + comp_data->memories[i].num_bytes_per_page = + module->memories[j].num_bytes_per_page; + } + } + + /* Create memory data segments */ + comp_data->mem_init_data_count = module->data_seg_count; + if (comp_data->mem_init_data_count > 0 + && !(comp_data->mem_init_data_list = + aot_create_mem_init_data_list(module))) + goto fail; + + /* Create tables */ + comp_data->table_count = module->import_table_count + module->table_count; + + if (comp_data->table_count > 0) { + size = sizeof(AOTTable) * (uint64)comp_data->table_count; + if (size >= UINT32_MAX + || !(comp_data->tables = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("create memories array failed.\n"); + goto fail; + } + memset(comp_data->tables, 0, size); + for (i = 0; i < comp_data->table_count; i++) { + if (i < module->import_table_count) { + comp_data->tables[i].elem_type = + module->import_tables[i].u.table.elem_type; + comp_data->tables[i].table_flags = + module->import_tables[i].u.table.flags; + comp_data->tables[i].table_init_size = + module->import_tables[i].u.table.init_size; + comp_data->tables[i].table_max_size = + module->import_tables[i].u.table.max_size; +#if WASM_ENABLE_GC != 0 + comp_data->tables[i].elem_ref_type = + module->import_tables[i].u.table.elem_ref_type; +#endif + comp_data->tables[i].possible_grow = + module->import_tables[i].u.table.possible_grow; + } + else { + j = i - module->import_table_count; + comp_data->tables[i].elem_type = module->tables[j].elem_type; + comp_data->tables[i].table_flags = module->tables[j].flags; + comp_data->tables[i].table_init_size = + module->tables[j].init_size; + comp_data->tables[i].table_max_size = + module->tables[j].max_size; + comp_data->tables[i].possible_grow = + module->tables[j].possible_grow; +#if WASM_ENABLE_GC != 0 + comp_data->tables[j].elem_ref_type = + module->tables[j].elem_ref_type; + /* Note: if the init_expr contains extra data for struct/array + * initialization information (init_expr.u.data), the pointer is + * copied. + * The pointers should still belong to wasm module, so DO NOT + * free the pointers copied to comp_data */ + comp_data->tables[j].init_expr = module->tables[j].init_expr; +#endif + } + } + } + + /* Create table data segments */ + comp_data->table_init_data_count = module->table_seg_count; + if (comp_data->table_init_data_count > 0 + && !(comp_data->table_init_data_list = + aot_create_table_init_data_list(module))) + goto fail; + + /* Create import globals */ + comp_data->import_global_count = module->import_global_count; + if (comp_data->import_global_count > 0 + && !(comp_data->import_globals = aot_create_import_globals( + module, gc_enabled, &import_global_data_size_64bit, + &import_global_data_size_32bit))) + goto fail; + + /* Create globals */ + comp_data->global_count = module->global_count; + if (comp_data->global_count + && !(comp_data->globals = aot_create_globals( + module, gc_enabled, import_global_data_size_64bit, + import_global_data_size_32bit, &global_data_size_64bit, + &global_data_size_32bit))) + goto fail; + + comp_data->global_data_size_64bit = + import_global_data_size_64bit + global_data_size_64bit; + comp_data->global_data_size_32bit = + import_global_data_size_32bit + global_data_size_32bit; + + /* Create types, they are checked by wasm loader */ + comp_data->type_count = module->type_count; + comp_data->types = module->types; +#if WASM_ENABLE_GC != 0 + /* Calculate the field sizes and field offsets for 64-bit and 32-bit + targets since they may vary in 32-bit target and 64-bit target */ + calculate_struct_field_sizes_offsets(comp_data, is_target_x86, gc_enabled); +#endif + + /* Create import functions */ + comp_data->import_func_count = module->import_function_count; + if (comp_data->import_func_count + && !(comp_data->import_funcs = aot_create_import_funcs(module))) + goto fail; + + /* Create functions */ + comp_data->func_count = module->function_count; + if (comp_data->func_count + && !(comp_data->funcs = + aot_create_funcs(module, is_64bit_target ? 8 : 4))) + goto fail; + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + /* Create custom name section */ + comp_data->name_section_buf = module->name_section_buf; + comp_data->name_section_buf_end = module->name_section_buf_end; +#endif + + /* Create aux data/heap/stack information */ + comp_data->aux_data_end_global_index = module->aux_data_end_global_index; + comp_data->aux_data_end = module->aux_data_end; + comp_data->aux_heap_base_global_index = module->aux_heap_base_global_index; + comp_data->aux_heap_base = module->aux_heap_base; + comp_data->aux_stack_top_global_index = module->aux_stack_top_global_index; + comp_data->aux_stack_bottom = module->aux_stack_bottom; + comp_data->aux_stack_size = module->aux_stack_size; + + comp_data->start_func_index = module->start_function; + comp_data->malloc_func_index = module->malloc_function; + comp_data->free_func_index = module->free_function; + comp_data->retain_func_index = module->retain_function; + +#if WASM_ENABLE_STRINGREF != 0 + comp_data->string_literal_count = module->string_literal_count; + comp_data->string_literal_ptrs_wp = module->string_literal_ptrs; + comp_data->string_literal_lengths_wp = module->string_literal_lengths; +#endif + + comp_data->wasm_module = module; + + return comp_data; + +fail: + + aot_destroy_comp_data(comp_data); + return NULL; +} + +void +aot_destroy_comp_data(AOTCompData *comp_data) +{ + if (!comp_data) + return; + + if (comp_data->import_memories) + wasm_runtime_free(comp_data->import_memories); + + if (comp_data->memories) + wasm_runtime_free(comp_data->memories); + + if (comp_data->mem_init_data_list) + aot_destroy_mem_init_data_list(comp_data->mem_init_data_list, + comp_data->mem_init_data_count); + + if (comp_data->import_tables) + wasm_runtime_free(comp_data->import_tables); + + if (comp_data->tables) + wasm_runtime_free(comp_data->tables); + + if (comp_data->table_init_data_list) + aot_destroy_table_init_data_list(comp_data->table_init_data_list, + comp_data->table_init_data_count); + + if (comp_data->import_globals) + wasm_runtime_free(comp_data->import_globals); + + if (comp_data->globals) + wasm_runtime_free(comp_data->globals); + + if (comp_data->import_funcs) + wasm_runtime_free(comp_data->import_funcs); + + if (comp_data->funcs) + aot_destroy_funcs(comp_data->funcs, comp_data->func_count); + + if (comp_data->aot_name_section_buf) + wasm_runtime_free(comp_data->aot_name_section_buf); + + wasm_runtime_free(comp_data); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot.h b/wasm-micro-runtime/core/iwasm/compilation/aot.h new file mode 100644 index 0000000..c0b68e0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot.h @@ -0,0 +1,401 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_H_ +#define _AOT_H_ + +#include "bh_platform.h" +#include "bh_assert.h" +#include "../common/wasm_runtime_common.h" +#include "../interpreter/wasm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef AOT_FUNC_PREFIX +#define AOT_FUNC_PREFIX "aot_func#" +#endif + +#ifndef AOT_FUNC_INTERNAL_PREFIX +#define AOT_FUNC_INTERNAL_PREFIX "aot_func_internal#" +#endif + +#ifndef AOT_STACK_SIZES_NAME +#define AOT_STACK_SIZES_NAME "aot_stack_sizes" +#endif +extern const char *aot_stack_sizes_name; + +#ifndef AOT_STACK_SIZES_ALIAS_NAME +#define AOT_STACK_SIZES_ALIAS_NAME "aot_stack_sizes_alias" +#endif +extern const char *aot_stack_sizes_alias_name; + +#ifndef AOT_STACK_SIZES_SECTION_NAME +#define AOT_STACK_SIZES_SECTION_NAME ".aot_stack_sizes" +#endif +extern const char *aot_stack_sizes_section_name; + +typedef InitializerExpression AOTInitExpr; +typedef WASMType AOTType; +typedef WASMFuncType AOTFuncType; +#if WASM_ENABLE_GC != 0 +typedef WASMStructType AOTStructType; +typedef WASMArrayType AOTArrayType; +#endif +typedef WASMExport AOTExport; + +#if WASM_ENABLE_DEBUG_AOT != 0 +typedef void *dwarf_extractor_handle_t; +#endif + +typedef enum AOTIntCond { + INT_EQZ = 0, + INT_EQ, + INT_NE, + INT_LT_S, + INT_LT_U, + INT_GT_S, + INT_GT_U, + INT_LE_S, + INT_LE_U, + INT_GE_S, + INT_GE_U +} AOTIntCond; + +typedef enum AOTFloatCond { + FLOAT_EQ = 0, + FLOAT_NE, + FLOAT_LT, + FLOAT_GT, + FLOAT_LE, + FLOAT_GE, + FLOAT_UNO +} AOTFloatCond; + +/** + * Import memory + */ +typedef struct AOTImportMemory { + char *module_name; + char *memory_name; + uint32 memory_flags; + uint32 num_bytes_per_page; + uint32 mem_init_page_count; + uint32 mem_max_page_count; +} AOTImportMemory; + +/** + * Memory information + */ +typedef struct AOTMemory { + /* memory info */ + uint32 memory_flags; + uint32 num_bytes_per_page; + uint32 mem_init_page_count; + uint32 mem_max_page_count; +} AOTMemory; + +/** + * A segment of memory init data + */ +typedef struct AOTMemInitData { +#if WASM_ENABLE_BULK_MEMORY != 0 + /* Passive flag */ + bool is_passive; + /* memory index */ + uint32 memory_index; +#endif + /* Start address of init data */ + AOTInitExpr offset; + /* Byte count */ + uint32 byte_count; + /* Byte array */ + uint8 bytes[1]; +} AOTMemInitData; + +/** + * Import table + */ +typedef struct AOTImportTable { + char *module_name; + char *table_name; + uint8 elem_type; + uint8 table_flags; + bool possible_grow; + uint32 table_init_size; + uint32 table_max_size; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif +} AOTImportTable; + +/** + * Table + */ +typedef struct AOTTable { + uint8 elem_type; + uint8 table_flags; + bool possible_grow; + uint32 table_init_size; + uint32 table_max_size; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; + /* init expr for the whole table */ + InitializerExpression init_expr; +#endif +} AOTTable; + +/** + * A segment of table init data + */ +typedef struct AOTTableInitData { + /* 0 to 7 */ + uint32 mode; + /* funcref or externref, elemkind will be considered as funcref */ + uint32 elem_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif + /* optional, only for active */ + uint32 table_index; + /* Start address of init data */ + AOTInitExpr offset; + /* Function index count */ + uint32 value_count; + /* Function index array */ + InitializerExpression init_values[1]; +} AOTTableInitData; + +/** + * Import global variable + */ +typedef struct AOTImportGlobal { + char *module_name; + char *global_name; + /* VALUE_TYPE_I32/I64/F32/F64 */ + uint8 type; + bool is_mutable; + uint32 size; + /* The data offset of current global in global data */ + uint32 data_offset; +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + /* + * The data size and data offset of a wasm global may vary + * in 32-bit target and 64-bit target, e.g., the size of a + * GC obj is 4 bytes in the former and 8 bytes in the + * latter, the AOT compiler needs to use the correct data + * offset according to the target info. + */ + uint32 size_64bit; + uint32 size_32bit; + uint32 data_offset_64bit; + uint32 data_offset_32bit; +#endif + /* global data after linked */ + WASMValue global_data_linked; + bool is_linked; +} AOTImportGlobal; + +/** + * Global variable + */ +typedef struct AOTGlobal { + /* VALUE_TYPE_I32/I64/F32/F64 */ + uint8 type; + bool is_mutable; + uint32 size; + /* The data offset of current global in global data */ + uint32 data_offset; +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + /* See comments in AOTImportGlobal */ + uint32 size_64bit; + uint32 size_32bit; + uint32 data_offset_64bit; + uint32 data_offset_32bit; +#endif + AOTInitExpr init_expr; +} AOTGlobal; + +/** + * Import function + */ +typedef struct AOTImportFunc { + char *module_name; + char *func_name; + AOTFuncType *func_type; + uint32 func_type_index; + /* function pointer after linked */ + void *func_ptr_linked; + /* signature from registered native symbols */ + const char *signature; + /* attachment */ + void *attachment; + bool call_conv_raw; + bool call_conv_wasm_c_api; + bool wasm_c_api_with_env; +} AOTImportFunc; + +/** + * Function + */ +typedef struct AOTFunc { + AOTFuncType *func_type; + uint32 func_type_index; + uint32 local_count; + uint8 *local_types_wp; + uint16 param_cell_num; + uint16 local_cell_num; + uint32 max_stack_cell_num; + uint32 code_size; + uint8 *code; + /* offset of each local, including function parameters + and local variables */ + uint16 *local_offsets; +} AOTFunc; + +typedef struct AOTCompData { + /* Import memories */ + uint32 import_memory_count; + AOTImportMemory *import_memories; + + /* Memories */ + uint32 memory_count; + AOTMemory *memories; + + /* Memory init data info */ + uint32 mem_init_data_count; + AOTMemInitData **mem_init_data_list; + + /* Import tables */ + uint32 import_table_count; + AOTImportTable *import_tables; + + /* Tables */ + uint32 table_count; + AOTTable *tables; + + /* Table init data info */ + uint32 table_init_data_count; + AOTTableInitData **table_init_data_list; + + /* Import globals */ + uint32 import_global_count; + AOTImportGlobal *import_globals; + + /* Globals */ + uint32 global_count; + AOTGlobal *globals; + + /* Function types */ + uint32 type_count; + AOTType **types; + + /* Import functions */ + uint32 import_func_count; + AOTImportFunc *import_funcs; + + /* Functions */ + uint32 func_count; + AOTFunc **funcs; + + /* Custom name sections */ + const uint8 *name_section_buf; + const uint8 *name_section_buf_end; + uint8 *aot_name_section_buf; + uint32 aot_name_section_size; + + uint32 global_data_size_64bit; + uint32 global_data_size_32bit; + + uint32 start_func_index; + uint32 malloc_func_index; + uint32 free_func_index; + uint32 retain_func_index; + + uint32 aux_data_end_global_index; + uint64 aux_data_end; + uint32 aux_heap_base_global_index; + uint64 aux_heap_base; + uint32 aux_stack_top_global_index; + uint64 aux_stack_bottom; + uint32 aux_stack_size; + +#if WASM_ENABLE_STRINGREF != 0 + uint32 string_literal_count; + uint32 *string_literal_lengths_wp; + const uint8 **string_literal_ptrs_wp; +#endif + + WASMModule *wasm_module; +#if WASM_ENABLE_DEBUG_AOT != 0 + dwarf_extractor_handle_t extractor; +#endif +} AOTCompData; + +typedef struct AOTNativeSymbol { + bh_list_link link; + char symbol[32]; + int32 index; +} AOTNativeSymbol; + +AOTCompData * +aot_create_comp_data(WASMModule *module, const char *target_arch, + bool gc_enabled); + +void +aot_destroy_comp_data(AOTCompData *comp_data); + +char * +aot_get_last_error(); + +void +aot_set_last_error(const char *error); + +void +aot_set_last_error_v(const char *format, ...); + +#if BH_DEBUG != 0 +#define HANDLE_FAILURE(callee) \ + do { \ + aot_set_last_error_v("call %s failed in %s:%d", (callee), \ + __FUNCTION__, __LINE__); \ + } while (0) +#else +#define HANDLE_FAILURE(callee) \ + do { \ + aot_set_last_error_v("call %s failed", (callee)); \ + } while (0) +#endif + +static inline uint32 +aot_get_imp_tbl_data_slots(const AOTImportTable *tbl, bool is_jit_mode) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + if (is_jit_mode) + return tbl->table_max_size; +#else + (void)is_jit_mode; +#endif + return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size; +} + +static inline uint32 +aot_get_tbl_data_slots(const AOTTable *tbl, bool is_jit_mode) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + if (is_jit_mode) + return tbl->table_max_size; +#else + (void)is_jit_mode; +#endif + return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size; +} + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_compiler.c b/wasm-micro-runtime/core/iwasm/compilation/aot_compiler.c new file mode 100644 index 0000000..5c25774 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_compiler.c @@ -0,0 +1,4266 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_compiler.h" +#include "aot_emit_compare.h" +#include "aot_emit_conversion.h" +#include "aot_emit_memory.h" +#include "aot_emit_variable.h" +#include "aot_emit_const.h" +#include "aot_emit_exception.h" +#include "aot_emit_numberic.h" +#include "aot_emit_control.h" +#include "aot_emit_function.h" +#include "aot_emit_parametric.h" +#include "aot_emit_table.h" +#include "aot_emit_gc.h" +#include "simd/simd_access_lanes.h" +#include "simd/simd_bitmask_extracts.h" +#include "simd/simd_bit_shifts.h" +#include "simd/simd_bitwise_ops.h" +#include "simd/simd_bool_reductions.h" +#include "simd/simd_comparisons.h" +#include "simd/simd_conversions.h" +#include "simd/simd_construct_values.h" +#include "simd/simd_conversions.h" +#include "simd/simd_floating_point.h" +#include "simd/simd_int_arith.h" +#include "simd/simd_load_store.h" +#include "simd/simd_sat_int_arith.h" +#include "../aot/aot_runtime.h" +#include "../interpreter/wasm_opcode.h" +#include + +#if WASM_ENABLE_DEBUG_AOT != 0 +#include "debug/dwarf_extractor.h" +#endif + +#if WASM_ENABLE_STRINGREF != 0 +#include "string_object.h" +#include "aot_emit_stringref.h" +#endif + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if (buf + length > buf_end) { \ + aot_set_last_error("read leb failed: unexpected end."); \ + return false; \ + } \ + } while (0) + +static bool +read_leb(const uint8 *buf, const uint8 *buf_end, uint32 *p_offset, + uint32 maxbits, bool sign, uint64 *p_result) +{ + uint64 result = 0; + uint32 shift = 0; + uint32 bcnt = 0; + uint64 byte; + + while (true) { + CHECK_BUF(buf, buf_end, 1); + byte = buf[*p_offset]; + *p_offset += 1; + result |= ((byte & 0x7f) << shift); + shift += 7; + if ((byte & 0x80) == 0) { + break; + } + bcnt += 1; + } + if (bcnt > (maxbits + 6) / 7) { + aot_set_last_error("read leb failed: " + "integer representation too long"); + return false; + } + if (sign && (shift < maxbits) && (byte & 0x40)) { + /* Sign extend */ + result |= (~((uint64)0)) << shift; + } + *p_result = result; + return true; +} + +/* NOLINTNEXTLINE */ +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(p, p_end, &off, 32, false, &res64)) \ + return false; \ + p += off; \ + res = (uint32)res64; \ + } while (0) + +/* NOLINTNEXTLINE */ +#define read_leb_int32(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(p, p_end, &off, 32, true, &res64)) \ + return false; \ + p += off; \ + res = (int32)res64; \ + } while (0) + +/* NOLINTNEXTLINE */ +#define read_leb_int64(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(p, p_end, &off, 64, true, &res64)) \ + return false; \ + p += off; \ + res = (int64)res64; \ + } while (0) + +/** + * Since wamrc uses a full feature Wasm loader, + * add a post-validator here to run checks according + * to options, like enable_tail_call, enable_ref_types, + * and so on. + */ +static bool +aot_validate_wasm(AOTCompContext *comp_ctx) +{ + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + /* Doesn't support multiple tables unless enabling reference type */ + if (comp_ctx->comp_data->import_table_count + + comp_ctx->comp_data->table_count + > 1) { + aot_set_last_error("multiple tables"); + return false; + } + } + + return true; +} + +#define COMPILE_ATOMIC_RMW(OP, NAME) \ + case WASM_OP_ATOMIC_RMW_I32_##NAME: \ + bytes = 4; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME: \ + bytes = 8; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I32_##NAME##8_U: \ + bytes = 1; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I32_##NAME##16_U: \ + bytes = 2; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##8_U: \ + bytes = 1; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##16_U: \ + bytes = 2; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##32_U: \ + bytes = 4; \ + op_type = VALUE_TYPE_I64; \ + OP_ATOMIC_##OP : bin_op = LLVMAtomicRMWBinOp##OP; \ + goto build_atomic_rmw; + +uint32 +offset_of_local_in_outs_area(AOTCompContext *comp_ctx, unsigned n) +{ + AOTCompFrame *frame = comp_ctx->aot_frame; + return frame->cur_frame_size + offset_of_local(comp_ctx, n); +} + +static bool +store_value(AOTCompContext *comp_ctx, LLVMValueRef value, uint8 value_type, + LLVMValueRef cur_frame, uint32 offset) +{ + LLVMValueRef value_offset, value_addr, value_ptr = NULL, res; + LLVMTypeRef value_ptr_type = NULL; + + if (!(value_offset = I32_CONST(offset))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "value_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + switch (value_type) { + case VALUE_TYPE_I32: + value_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + value_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F32: + value_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_F64: + value_ptr_type = F64_PTR_TYPE; + break; + case VALUE_TYPE_V128: + value_ptr_type = V128_PTR_TYPE; + break; +#if WASM_ENABLE_GC != 0 + case VALUE_TYPE_GC_REF: + value_ptr_type = GC_REF_PTR_TYPE; + break; +#endif + default: + bh_assert(0); + break; + } + + if (!(value_ptr = LLVMBuildBitCast(comp_ctx->builder, value_addr, + value_ptr_type, "value_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(res = LLVMBuildStore(comp_ctx->builder, value, value_ptr))) { + aot_set_last_error("llvm build store failed"); + return false; + } + + LLVMSetAlignment(res, 4); + + return true; +} + +bool +aot_frame_store_value(AOTCompContext *comp_ctx, LLVMValueRef value, + uint8 value_type, LLVMValueRef cur_frame, uint32 offset) +{ + return store_value(comp_ctx, value, value_type, cur_frame, offset); +} + +static bool +store_ref(AOTCompContext *comp_ctx, uint32 ref, LLVMValueRef cur_frame, + uint32 offset, uint32 nbytes) +{ + LLVMValueRef value_ref = NULL, value_offset, value_addr, res; + + if (!(value_offset = I32_CONST(offset))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "value_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + switch (nbytes) { + case 1: + if (!(value_ref = I8_CONST((uint8)ref))) { + aot_set_last_error("llvm build const failed"); + } + break; + case 2: + ref = (ref << 8) | ref; + + if (!(value_ref = LLVMConstInt(INT16_TYPE, (uint16)ref, false))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildBitCast(comp_ctx->builder, value_addr, + INT16_PTR_TYPE, "value_addr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + break; + case 4: + ref = (ref << 24) | (ref << 16) | (ref << 8) | ref; + + if (!(value_ref = I32_CONST(ref))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildBitCast(comp_ctx->builder, value_addr, + INT32_PTR_TYPE, "value_addr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + break; + default: + bh_assert(0); + break; + } + + if (!(res = LLVMBuildStore(comp_ctx->builder, value_ref, value_addr))) { + aot_set_last_error("llvm build store failed"); + return false; + } + LLVMSetAlignment(res, 1); + + return true; +} + +bool +aot_gen_commit_values(AOTCompFrame *frame) +{ + AOTCompContext *comp_ctx = frame->comp_ctx; + AOTFuncContext *func_ctx = frame->func_ctx; + AOTValueSlot *p, *end; + LLVMValueRef value; + uint32 n; + + /* First, commit reference flags + * For LLVM JIT, iterate all local and stack ref flags + * For AOT, ignore local(params + locals) ref flags */ + for (p = comp_ctx->is_jit_mode ? frame->lp + : frame->lp + frame->max_local_cell_num; + p < frame->sp; p++) { + if (!p->dirty) + continue; + + n = (uint32)(p - frame->lp); + + /* Commit reference flag */ + if (comp_ctx->enable_gc) { + switch (p->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + case VALUE_TYPE_I1: + if (p->ref != p->committed_ref - 1) { + if (!store_ref(comp_ctx, p->ref, func_ctx->cur_frame, + offset_of_ref(comp_ctx, n), 1)) + return false; + p->committed_ref = p->ref + 1; + } + break; + + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + bh_assert(p->ref == (p + 1)->ref); + if (p->ref != p->committed_ref - 1 + || p->ref != (p + 1)->committed_ref - 1) { + if (!store_ref(comp_ctx, p->ref, func_ctx->cur_frame, + offset_of_ref(comp_ctx, n), 2)) + return false; + p->committed_ref = (p + 1)->committed_ref = p->ref + 1; + } + p++; + break; + + case VALUE_TYPE_V128: + bh_assert(p->ref == (p + 1)->ref && p->ref == (p + 2)->ref + && p->ref == (p + 3)->ref); + if (p->ref != p->committed_ref - 1 + || p->ref != (p + 1)->committed_ref - 1 + || p->ref != (p + 2)->committed_ref - 1 + || p->ref != (p + 3)->committed_ref - 1) { + if (!store_ref(comp_ctx, p->ref, func_ctx->cur_frame, + offset_of_ref(comp_ctx, n), 4)) + return false; + p->committed_ref = (p + 1)->committed_ref = + (p + 2)->committed_ref = (p + 3)->committed_ref = + p->ref + 1; + } + p += 3; + break; + + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + case VALUE_TYPE_GC_REF: + if (comp_ctx->pointer_size == sizeof(uint64)) { + bh_assert(p->ref == (p + 1)->ref); + if (p->ref != p->committed_ref - 1 + || p->ref != (p + 1)->committed_ref - 1) { + if (!store_ref(comp_ctx, p->ref, + func_ctx->cur_frame, + offset_of_ref(comp_ctx, n), 2)) + return false; + p->committed_ref = (p + 1)->committed_ref = + p->ref + 1; + } + p++; + } + else { + if (p->ref != p->committed_ref - 1) { + if (!store_ref(comp_ctx, p->ref, + func_ctx->cur_frame, + offset_of_ref(comp_ctx, n), 1)) + return false; + p->committed_ref = p->ref + 1; + } + } + break; + + default: + bh_assert(0); + break; + } + } + } + + /* Second, commit all values */ + for (p = frame->lp; p < frame->sp; p++) { + if (!p->dirty) + continue; + + p->dirty = 0; + n = (uint32)(p - frame->lp); + + /* Commit values */ + switch (p->type) { + case VALUE_TYPE_I32: + if (!store_value(comp_ctx, p->value, VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_I64: + (++p)->dirty = 0; + if (!store_value(comp_ctx, p->value, VALUE_TYPE_I64, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_F32: + if (!store_value(comp_ctx, p->value, VALUE_TYPE_F32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_F64: + (++p)->dirty = 0; + if (!store_value(comp_ctx, p->value, VALUE_TYPE_F64, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_V128: + (++p)->dirty = 0; + (++p)->dirty = 0; + (++p)->dirty = 0; + if (!store_value(comp_ctx, p->value, VALUE_TYPE_V128, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_I1: + if (!(value = LLVMBuildZExt(comp_ctx->builder, p->value, + I32_TYPE, "i32_val"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (!store_value(comp_ctx, value, VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + if (comp_ctx->enable_ref_types) { + if (!store_value(comp_ctx, p->value, VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + } +#if WASM_ENABLE_GC != 0 + else if (comp_ctx->enable_gc) { + if (comp_ctx->pointer_size == sizeof(uint64)) + (++p)->dirty = 0; + if (!store_value(comp_ctx, p->value, VALUE_TYPE_GC_REF, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + } +#endif + else { + bh_assert(0); + } + break; +#if WASM_ENABLE_GC != 0 + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + /* case REF_TYPE_FUNCREF: */ + /* case REF_TYPE_EXTERNREF: */ + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case VALUE_TYPE_GC_REF: + if (comp_ctx->pointer_size == sizeof(uint64)) + (++p)->dirty = 0; + if (!store_value(comp_ctx, p->value, VALUE_TYPE_GC_REF, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; +#endif + default: + bh_assert(0); + break; + } + } + + if (comp_ctx->enable_gc) { + end = frame->lp + frame->max_local_cell_num + frame->max_stack_cell_num; + + /* Clear reference flags for unused stack slots. */ + for (p = frame->sp; p < end; p++) { + bh_assert(!p->ref); + n = (uint32)(p - frame->lp); + + /* Commit reference flag. */ + if (p->ref != p->committed_ref - 1) { + if (!store_ref(comp_ctx, p->ref, func_ctx->cur_frame, + offset_of_ref(comp_ctx, n), 1)) + return false; + p->committed_ref = 1 + p->ref; + } + } + } + + return true; +} + +bool +aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) +{ + AOTCompContext *comp_ctx = frame->comp_ctx; + AOTFuncContext *func_ctx = frame->func_ctx; + LLVMValueRef cur_frame = func_ctx->cur_frame; + LLVMValueRef value_offset, value_addr, value_ptr, value; + LLVMTypeRef int8_ptr_ptr_type; + uint32 offset_ip, offset_sp, n; + bool is_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; + const AOTValueSlot *sp = frame->sp; + const uint8 *ip = frame->frame_ip; + + if (!comp_ctx->is_jit_mode) { + offset_ip = frame->comp_ctx->pointer_size * 4; + offset_sp = frame->comp_ctx->pointer_size * 5; + } + else { + offset_ip = offsetof(WASMInterpFrame, ip); + offset_sp = offsetof(WASMInterpFrame, sp); + } + + if (commit_ip) { + if (!(value_offset = I32_CONST(offset_ip))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "ip_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(value_ptr = LLVMBuildBitCast( + comp_ctx->builder, value_addr, + is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!comp_ctx->is_jit_mode) { + WASMModule *module = comp_ctx->comp_data->wasm_module; + if (is_64bit) + value = I64_CONST((uint64)(uintptr_t)(ip - module->load_addr)); + else + value = I32_CONST((uint32)(uintptr_t)(ip - module->load_addr)); + } + else { + if (is_64bit) + value = I64_CONST((uint64)(uintptr_t)ip); + else + value = I32_CONST((uint32)(uintptr_t)ip); + } + + if (!value) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!LLVMBuildStore(comp_ctx->builder, value, value_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + + if (commit_sp) { + n = (uint32)(sp - frame->lp); + value = I32_CONST(offset_of_local(comp_ctx, n)); + if (!value) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + cur_frame, &value, 1, "sp"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(value_offset = I32_CONST(offset_sp))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "sp_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(int8_ptr_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0))) { + aot_set_last_error("llvm build pointer type failed"); + return false; + } + + if (!(value_ptr = LLVMBuildBitCast(comp_ctx->builder, value_addr, + int8_ptr_ptr_type, "sp_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!LLVMBuildStore(comp_ctx->builder, value, value_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + + return true; +} + +static uint32 +get_cur_frame_size(const AOTCompContext *comp_ctx, uint32 max_local_cell_num, + uint32 max_stack_cell_num) +{ + uint32 all_cell_num = max_local_cell_num + max_stack_cell_num; + uint32 frame_size; + + if (!comp_ctx->is_jit_mode) { + /* Refer to aot_alloc_frame */ + if (!comp_ctx->enable_gc) + frame_size = comp_ctx->pointer_size + * (offsetof(AOTFrame, lp) / sizeof(uintptr_t)) + + all_cell_num * 4; + else + frame_size = comp_ctx->pointer_size + * (offsetof(AOTFrame, lp) / sizeof(uintptr_t)) + + align_uint(all_cell_num * 5, 4); + } + else { + /* Refer to wasm_interp_interp_frame_size */ + if (!comp_ctx->enable_gc) + frame_size = offsetof(WASMInterpFrame, lp) + all_cell_num * 4; + else + frame_size = + offsetof(WASMInterpFrame, lp) + align_uint(all_cell_num * 5, 4); + } + + return frame_size; +} + +static bool +init_comp_frame(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 func_idx) +{ + AOTCompFrame *aot_frame; + AOTFunc *aot_func = func_ctx->aot_func; + AOTFuncType *func_type = aot_func->func_type; + AOTBlock *block = func_ctx->block_stack.block_list_end; + LLVMValueRef local_value; + uint32 max_local_cell_num = + aot_func->param_cell_num + aot_func->local_cell_num; + uint32 max_stack_cell_num = aot_func->max_stack_cell_num; + uint32 all_cell_num = max_local_cell_num + max_stack_cell_num; + uint32 i, n; + uint64 total_size; + uint8 local_type; + + /* Free aot_frame if it was allocated previously for + compiling other functions */ + if (comp_ctx->aot_frame) { + wasm_runtime_free(comp_ctx->aot_frame); + comp_ctx->aot_frame = NULL; + } + + /* Allocate extra 2 cells since some operations may push more + operands than the number calculated in wasm loader, such as + PUSH_F64(F64_CONST(1.0)) in aot_compile_op_f64_promote_f32 */ + all_cell_num += 2; + total_size = offsetof(AOTCompFrame, lp) + + (uint64)sizeof(AOTValueSlot) * all_cell_num; + + if (total_size > UINT32_MAX + || !(comp_ctx->aot_frame = aot_frame = + wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + return false; + } + memset(aot_frame, 0, (uint32)total_size); + + aot_frame->comp_ctx = comp_ctx; + aot_frame->func_ctx = func_ctx; + + aot_frame->max_local_cell_num = max_local_cell_num; + aot_frame->max_stack_cell_num = max_stack_cell_num; + aot_frame->cur_frame_size = + get_cur_frame_size(comp_ctx, max_local_cell_num, max_stack_cell_num); + + aot_frame->sp = aot_frame->lp + max_local_cell_num; + + /* Init the frame_sp_begin and frame_sp_max_reached + of the function block */ + block->frame_sp_begin = block->frame_sp_max_reached = aot_frame->sp; + + n = 0; + + /* Set all params dirty since they were set to llvm value but + haven't been committed to the AOT/JIT stack frame */ + for (i = 0; i < func_type->param_count; i++) { + local_type = func_type->types[i]; + local_value = LLVMGetParam(func_ctx->func, i + 1); + + switch (local_type) { + case VALUE_TYPE_I32: + set_local_i32(comp_ctx->aot_frame, n, local_value); + n++; + break; + case VALUE_TYPE_I64: + set_local_i64(comp_ctx->aot_frame, n, local_value); + n += 2; + break; + case VALUE_TYPE_F32: + set_local_f32(comp_ctx->aot_frame, n, local_value); + n++; + break; + case VALUE_TYPE_F64: + set_local_f64(comp_ctx->aot_frame, n, local_value); + n += 2; + break; + case VALUE_TYPE_V128: + set_local_v128(comp_ctx->aot_frame, n, local_value); + n += 4; + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + { + if (comp_ctx->enable_ref_types) { + set_local_ref(comp_ctx->aot_frame, n, local_value, + local_type); + n++; + } +#if WASM_ENABLE_GC != 0 + else if (comp_ctx->enable_gc) { + set_local_gc_ref(comp_ctx->aot_frame, n, local_value, + VALUE_TYPE_GC_REF); + n += comp_ctx->pointer_size / sizeof(uint32); + } +#endif + else { + bh_assert(0); + } + break; + } +#if WASM_ENABLE_GC != 0 + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + /* case REF_TYPE_FUNCREF: */ + /* case REF_TYPE_EXTERNREF: */ + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + bh_assert(comp_ctx->enable_gc); + set_local_gc_ref(comp_ctx->aot_frame, n, local_value, + VALUE_TYPE_GC_REF); + n += comp_ctx->pointer_size / sizeof(uint32); + break; +#endif + default: + bh_assert(0); + break; + } + } + + /* TODO: re-calculate param_cell_num according to the build target + after creating comp_ctx */ + /* bh_assert(n == aot_func->param_cell_num); */ + + /* Set all locals dirty since they were set to llvm value but + haven't been committed to the AOT/JIT stack frame */ + for (i = 0; i < aot_func->local_count; i++) { + local_type = aot_func->local_types_wp[i]; + + switch (local_type) { + case VALUE_TYPE_I32: + set_local_i32(comp_ctx->aot_frame, n, I32_ZERO); + n++; + break; + case VALUE_TYPE_I64: + set_local_i64(comp_ctx->aot_frame, n, I64_ZERO); + n += 2; + break; + case VALUE_TYPE_F32: + set_local_f32(comp_ctx->aot_frame, n, F32_ZERO); + n++; + break; + case VALUE_TYPE_F64: + set_local_f64(comp_ctx->aot_frame, n, F64_ZERO); + n += 2; + break; + case VALUE_TYPE_V128: + set_local_v128(comp_ctx->aot_frame, n, V128_f64x2_ZERO); + n += 4; + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + { + if (comp_ctx->enable_ref_types) { + set_local_ref(comp_ctx->aot_frame, n, I32_ZERO, local_type); + n++; + } +#if WASM_ENABLE_GC != 0 + else if (comp_ctx->enable_gc) { + set_local_gc_ref(comp_ctx->aot_frame, n, GC_REF_NULL, + VALUE_TYPE_GC_REF); + n += comp_ctx->pointer_size / sizeof(uint32); + } +#endif + else { + bh_assert(0); + } + break; + } +#if WASM_ENABLE_GC != 0 + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + /* case REF_TYPE_FUNCREF: */ + /* case REF_TYPE_EXTERNREF: */ + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + bh_assert(comp_ctx->enable_gc); + set_local_gc_ref(comp_ctx->aot_frame, n, GC_REF_NULL, + VALUE_TYPE_GC_REF); + n += comp_ctx->pointer_size / sizeof(uint32); + break; +#endif + default: + bh_assert(0); + break; + } + } + + /* TODO: re-calculate local_cell_num according to the build target + after creating comp_ctx */ + /* bh_assert(n == aot_func->param_cell_num + aot_func->local_cell_num); */ + + /* No need to initialize aot_frame all cells' committed_ref flags + and all stack cells' ref flags since they have been initialized + as 0 (uncommitted and not-reference) by the memset above */ + + return true; +} + +static bool +aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) +{ + AOTFuncContext *func_ctx = comp_ctx->func_ctxes[func_index]; + uint8 *frame_ip = func_ctx->aot_func->code, opcode, *p_f32, *p_f64; + uint8 *frame_ip_end = frame_ip + func_ctx->aot_func->code_size; + uint8 *param_types = NULL; + uint8 *result_types = NULL; + uint8 value_type; + uint16 param_count; + uint16 result_count; + uint32 br_depth, *br_depths, br_count; + uint32 func_idx, type_idx, mem_idx, local_idx, global_idx, i; + uint32 bytes = 4, align, offset; + uint32 type_index; + bool sign = true; + int32 i32_const; + int64 i64_const; + float32 f32_const; + float64 f64_const; + AOTFuncType *func_type = NULL; +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMMetadataRef location; +#endif + + if (comp_ctx->enable_aux_stack_frame) { + if (!init_comp_frame(comp_ctx, func_ctx, func_index)) { + return false; + } + } + + /* Start to translate the opcodes */ + LLVMPositionBuilderAtEnd( + comp_ctx->builder, + func_ctx->block_stack.block_list_head->llvm_entry_block); + while (frame_ip < frame_ip_end) { + opcode = *frame_ip++; + + if (comp_ctx->aot_frame) { + comp_ctx->aot_frame->frame_ip = frame_ip - 1; + } + +#if WASM_ENABLE_DEBUG_AOT != 0 + location = dwarf_gen_location( + comp_ctx, func_ctx, + (frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); + if (location != NULL) { + LLVMSetCurrentDebugLocation2(comp_ctx->builder, location); + } +#endif + + switch (opcode) { + case WASM_OP_UNREACHABLE: + if (!aot_compile_op_unreachable(comp_ctx, func_ctx, &frame_ip)) + return false; + break; + + case WASM_OP_NOP: + break; + + case WASM_OP_BLOCK: + case WASM_OP_LOOP: + case WASM_OP_IF: + { + value_type = *frame_ip++; + if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_I64 + || value_type == VALUE_TYPE_F32 + || value_type == VALUE_TYPE_F64 + || value_type == VALUE_TYPE_V128 + || value_type == VALUE_TYPE_VOID + || (comp_ctx->enable_ref_types + && (value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF)) + || (comp_ctx->enable_gc /* single byte type */ + && aot_is_type_gc_reftype(value_type))) { + param_count = 0; + param_types = NULL; + if (value_type == VALUE_TYPE_VOID) { + result_count = 0; + result_types = NULL; + } + else { + if (comp_ctx->enable_gc + && aot_is_type_gc_reftype(value_type)) + value_type = VALUE_TYPE_GC_REF; + result_count = 1; + result_types = &value_type; + } + } + else { + frame_ip--; + read_leb_uint32(frame_ip, frame_ip_end, type_index); + func_type = + (AOTFuncType *)comp_ctx->comp_data->types[type_index]; + param_count = func_type->param_count; + param_types = func_type->types; + result_count = func_type->result_count; + result_types = func_type->types + param_count; + } + if (!aot_compile_op_block( + comp_ctx, func_ctx, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - WASM_OP_BLOCK), + param_count, param_types, result_count, result_types)) + return false; + break; + } + + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + { + read_leb_uint32(frame_ip, frame_ip_end, type_index); + func_type = + (AOTFuncType *)comp_ctx->comp_data->types[type_index]; + param_count = func_type->param_count; + param_types = func_type->types; + result_count = func_type->result_count; + result_types = func_type->types + param_count; + if (!aot_compile_op_block( + comp_ctx, func_ctx, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - EXT_OP_BLOCK), + param_count, param_types, result_count, result_types)) + return false; + break; + } + + case WASM_OP_ELSE: + if (!aot_compile_op_else(comp_ctx, func_ctx, &frame_ip)) + return false; + break; + + case WASM_OP_END: + if (!aot_compile_op_end(comp_ctx, func_ctx, &frame_ip)) + return false; + break; + + case WASM_OP_BR: + { + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!aot_compile_op_br(comp_ctx, func_ctx, br_depth, &frame_ip)) + return false; + break; + } + + case WASM_OP_BR_IF: + { + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!aot_compile_op_br_if(comp_ctx, func_ctx, br_depth, + &frame_ip)) + return false; + break; + } + + case WASM_OP_BR_TABLE: + { + read_leb_uint32(frame_ip, frame_ip_end, br_count); + if (!(br_depths = wasm_runtime_malloc((uint32)sizeof(uint32) + * (br_count + 1)))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + for (i = 0; i <= br_count; i++) + read_leb_uint32(frame_ip, frame_ip_end, br_depths[i]); +#else + for (i = 0; i <= br_count; i++) + br_depths[i] = *frame_ip++; +#endif + + if (!aot_compile_op_br_table(comp_ctx, func_ctx, br_depths, + br_count, &frame_ip)) { + wasm_runtime_free(br_depths); + return false; + } + + wasm_runtime_free(br_depths); + break; + } + +#if WASM_ENABLE_FAST_INTERP == 0 + case EXT_OP_BR_TABLE_CACHE: + { + BrTableCache *node = bh_list_first_elem( + comp_ctx->comp_data->wasm_module->br_table_cache_list); + BrTableCache *node_next; + const uint8 *frame_ip_org = frame_ip - 1; + + read_leb_uint32(frame_ip, frame_ip_end, br_count); + + while (node) { + node_next = bh_list_elem_next(node); + if (node->br_table_op_addr == frame_ip_org) { + br_depths = node->br_depths; + if (!aot_compile_op_br_table(comp_ctx, func_ctx, + br_depths, br_count, + &frame_ip)) { + return false; + } + break; + } + node = node_next; + } + bh_assert(node); + + break; + } +#endif + + case WASM_OP_RETURN: + if (!aot_compile_op_return(comp_ctx, func_ctx, &frame_ip)) + return false; + break; + + case WASM_OP_CALL: + { + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + if (!aot_compile_op_call(comp_ctx, func_ctx, func_idx, false)) + return false; + break; + } + + case WASM_OP_CALL_INDIRECT: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + + if (comp_ctx->enable_gc || comp_ctx->enable_ref_types) { + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + } + else { + frame_ip++; + tbl_idx = 0; + } + + if (!aot_compile_op_call_indirect(comp_ctx, func_ctx, type_idx, + tbl_idx)) + return false; + break; + } + +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: + { + if (!comp_ctx->enable_tail_call) { + aot_set_last_error("unsupported opcode"); + return false; + } + + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + if (!aot_compile_op_call(comp_ctx, func_ctx, func_idx, true)) + return false; + if (!aot_compile_op_return(comp_ctx, func_ctx, &frame_ip)) + return false; + break; + } + + case WASM_OP_RETURN_CALL_INDIRECT: + { + uint32 tbl_idx; + + if (!comp_ctx->enable_tail_call) { + aot_set_last_error("unsupported opcode"); + return false; + } + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + if (comp_ctx->enable_gc || comp_ctx->enable_ref_types) { + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + } + else { + frame_ip++; + tbl_idx = 0; + } + + if (!aot_compile_op_call_indirect(comp_ctx, func_ctx, type_idx, + tbl_idx)) + return false; + if (!aot_compile_op_return(comp_ctx, func_ctx, &frame_ip)) + return false; + break; + } +#endif /* end of WASM_ENABLE_TAIL_CALL */ + + case WASM_OP_DROP: + if (!aot_compile_op_drop(comp_ctx, func_ctx, true)) + return false; + break; + + case WASM_OP_DROP_64: + if (!aot_compile_op_drop(comp_ctx, func_ctx, false)) + return false; + break; + + case WASM_OP_SELECT: + if (!aot_compile_op_select(comp_ctx, func_ctx, true)) + return false; + break; + + case WASM_OP_SELECT_64: + if (!aot_compile_op_select(comp_ctx, func_ctx, false)) + return false; + break; + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_SELECT_T: + { + uint32 vec_len; + + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + goto unsupport_gc_and_ref_types; + } + + read_leb_uint32(frame_ip, frame_ip_end, vec_len); + bh_assert(vec_len == 1); + (void)vec_len; + + type_idx = *frame_ip++; + if (!aot_compile_op_select( + comp_ctx, func_ctx, + (type_idx != VALUE_TYPE_I64) + && (type_idx != VALUE_TYPE_F64) +#if WASM_ENABLE_GC != 0 + && !(comp_ctx->enable_gc + && comp_ctx->pointer_size == sizeof(uint64) + && wasm_is_type_reftype(type_idx)) +#endif + )) + return false; + + break; + } + case WASM_OP_TABLE_GET: + { + uint32 tbl_idx; + + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + goto unsupport_gc_and_ref_types; + } + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!aot_compile_op_table_get(comp_ctx, func_ctx, tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_SET: + { + uint32 tbl_idx; + + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + goto unsupport_gc_and_ref_types; + } + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!aot_compile_op_table_set(comp_ctx, func_ctx, tbl_idx)) + return false; + break; + } + case WASM_OP_REF_NULL: + { + uint32 type; + + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + goto unsupport_gc_and_ref_types; + } + + read_leb_uint32(frame_ip, frame_ip_end, type); + + if (!aot_compile_op_ref_null(comp_ctx, func_ctx)) + return false; + + (void)type; + break; + } + case WASM_OP_REF_IS_NULL: + { + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + goto unsupport_gc_and_ref_types; + } + + if (!aot_compile_op_ref_is_null(comp_ctx, func_ctx)) + return false; + break; + } + case WASM_OP_REF_FUNC: + { + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + goto unsupport_gc_and_ref_types; + } + + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + if (!aot_compile_op_ref_func(comp_ctx, func_ctx, func_idx)) + return false; + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + case WASM_OP_CALL_REF: + { + if (!comp_ctx->enable_gc) { + goto unsupport_gc; + } + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + if (!aot_compile_op_call_ref(comp_ctx, func_ctx, type_idx, + false)) + return false; + break; + } + + case WASM_OP_RETURN_CALL_REF: + { + if (!comp_ctx->enable_gc) { + goto unsupport_gc; + } + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + if (!aot_compile_op_call_ref(comp_ctx, func_ctx, type_idx, + true)) + return false; + if (!aot_compile_op_return(comp_ctx, func_ctx, &frame_ip)) + return false; + break; + } + + case WASM_OP_REF_EQ: + if (!comp_ctx->enable_gc) { + goto unsupport_gc; + } + + if (!aot_compile_op_ref_eq(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_REF_AS_NON_NULL: + if (!comp_ctx->enable_gc) { + goto unsupport_gc; + } + + if (!aot_compile_op_ref_as_non_null(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_BR_ON_NULL: + { + if (!comp_ctx->enable_gc) { + goto unsupport_gc; + } + + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!aot_compile_op_br_on_null(comp_ctx, func_ctx, br_depth, + &frame_ip)) + return false; + break; + } + + case WASM_OP_BR_ON_NON_NULL: + { + if (!comp_ctx->enable_gc) { + goto unsupport_gc; + } + + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!aot_compile_op_br_on_non_null(comp_ctx, func_ctx, br_depth, + &frame_ip)) + return false; + break; + } + + case WASM_OP_GC_PREFIX: + { + uint32 opcode1, field_idx, data_seg_idx, array_len; + + if (!comp_ctx->enable_gc) { + goto unsupport_gc; + } + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + if (!aot_compile_op_struct_new( + comp_ctx, func_ctx, type_index, + opcode == WASM_OP_STRUCT_NEW_DEFAULT)) + return false; + break; + + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, field_idx); + if (!aot_compile_op_struct_get( + comp_ctx, func_ctx, type_index, field_idx, + opcode == WASM_OP_STRUCT_GET_S)) + return false; + break; + + case WASM_OP_STRUCT_SET: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, field_idx); + if (!aot_compile_op_struct_set(comp_ctx, func_ctx, + type_index, field_idx)) + return false; + break; + + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + if (opcode == WASM_OP_ARRAY_NEW_FIXED) + read_leb_uint32(frame_ip, frame_ip_end, array_len); + else + array_len = 0; + if (!aot_compile_op_array_new( + comp_ctx, func_ctx, type_index, + opcode == WASM_OP_ARRAY_NEW_DEFAULT, + opcode == WASM_OP_ARRAY_NEW_FIXED, array_len)) + return false; + break; + + case WASM_OP_ARRAY_NEW_DATA: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, data_seg_idx); + if (!aot_compile_op_array_new_data( + comp_ctx, func_ctx, type_index, data_seg_idx)) + return false; + break; + + case WASM_OP_ARRAY_NEW_ELEM: + /* TODO */ + aot_set_last_error("unsupported opcode"); + return false; + + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + if (!aot_compile_op_array_get( + comp_ctx, func_ctx, type_index, + opcode == WASM_OP_ARRAY_GET_S)) + return false; + break; + + case WASM_OP_ARRAY_SET: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + if (!aot_compile_op_array_set(comp_ctx, func_ctx, + type_index)) + return false; + break; + + case WASM_OP_ARRAY_FILL: + read_leb_uint32(frame_ip, frame_ip_end, type_index); + if (!aot_compile_op_array_fill(comp_ctx, func_ctx, + type_index)) + return false; + break; + + case WASM_OP_ARRAY_COPY: + { + uint32 src_type_index; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, src_type_index); + if (!aot_compile_op_array_copy( + comp_ctx, func_ctx, type_index, src_type_index)) + return false; + break; + } + + case WASM_OP_ARRAY_LEN: + if (!aot_compile_op_array_len(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_REF_I31: + if (!aot_compile_op_i31_new(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + if (!aot_compile_op_i31_get( + comp_ctx, func_ctx, + opcode == WASM_OP_I31_GET_S ? true : false)) + return false; + break; + + case WASM_OP_REF_TEST: + case WASM_OP_REF_TEST_NULLABLE: + { + int32 heap_type; + + read_leb_int32(frame_ip, frame_ip_end, heap_type); + if (!aot_compile_op_ref_test( + comp_ctx, func_ctx, heap_type, + opcode == WASM_OP_REF_TEST_NULLABLE ? true + : false)) + return false; + break; + } + + case WASM_OP_REF_CAST: + case WASM_OP_REF_CAST_NULLABLE: + { + int32 heap_type; + + read_leb_int32(frame_ip, frame_ip_end, heap_type); + if (!aot_compile_op_ref_cast( + comp_ctx, func_ctx, heap_type, + opcode == WASM_OP_REF_CAST_NULLABLE ? true + : false)) + return false; + break; + } + + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + { + uint8 castflags; + int32 heap_type, dst_heap_type; + + CHECK_BUF(frame_ip, frame_ip_end, 1); + castflags = *frame_ip++; + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + read_leb_int32(frame_ip, frame_ip_end, heap_type); + read_leb_int32(frame_ip, frame_ip_end, dst_heap_type); + + /* + * castflags should be 0~3: + * 0: (non-null, non-null) + * 1: (null, non-null) + * 2: (non-null, null) + * 3: (null, null) + * The nullability of source type has been checked in + * wasm loader, here we just need the dst nullability + */ + if (!aot_compile_op_br_on_cast( + comp_ctx, func_ctx, dst_heap_type, + castflags & 0x02, + opcode == WASM_OP_BR_ON_CAST_FAIL, br_depth, + &frame_ip)) + return false; + + (void)heap_type; + break; + } + + case WASM_OP_ANY_CONVERT_EXTERN: + if (!aot_compile_op_extern_internalize(comp_ctx, + func_ctx)) + return false; + break; + + case WASM_OP_EXTERN_CONVERT_ANY: + if (!aot_compile_op_extern_externalize(comp_ctx, + func_ctx)) + return false; + break; + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + { + EncodingFlag flag = WTF8; + + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + bh_assert(mem_idx == 0); + + if (opcode == WASM_OP_STRING_NEW_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_NEW_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8) { + flag = WTF8; + } + + if (!aot_compile_op_string_new(comp_ctx, func_ctx, + flag)) + return false; + break; + } + case WASM_OP_STRING_CONST: + { + uint32 contents; + read_leb_uint32(frame_ip, frame_ip_end, contents); + + if (!aot_compile_op_string_const(comp_ctx, func_ctx, + contents)) + return false; + break; + } + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + { + EncodingFlag flag = WTF8; + + if (opcode == WASM_OP_STRING_MEASURE_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_MEASURE_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_MEASURE_WTF8) { + flag = LOSSY_UTF8; + } + + if (!aot_compile_op_string_measure(comp_ctx, func_ctx, + flag)) + return false; + break; + } + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + { + EncodingFlag flag = WTF8; + + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + bh_assert(mem_idx == 0); + + if (opcode == WASM_OP_STRING_ENCODE_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_ENCODE_WTF8) { + flag = WTF8; + } + + if (!aot_compile_op_string_encode(comp_ctx, func_ctx, + mem_idx, flag)) + return false; + break; + } + case WASM_OP_STRING_CONCAT: + if (!aot_compile_op_string_concat(comp_ctx, func_ctx)) + return false; + break; + case WASM_OP_STRING_EQ: + if (!aot_compile_op_string_eq(comp_ctx, func_ctx)) + return false; + break; + case WASM_OP_STRING_IS_USV_SEQUENCE: + if (!aot_compile_op_string_is_usv_sequence(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRING_AS_WTF8: + if (!aot_compile_op_string_as_wtf8(comp_ctx, func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + if (!aot_compile_op_stringview_wtf8_advance(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + { + EncodingFlag flag = WTF8; + + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + bh_assert(mem_idx == 0); + + if (opcode == WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8) { + flag = WTF8; + } + + if (!aot_compile_op_stringview_wtf8_encode( + comp_ctx, func_ctx, mem_idx, flag)) + return false; + break; + } + case WASM_OP_STRINGVIEW_WTF8_SLICE: + if (!aot_compile_op_stringview_wtf8_slice(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRING_AS_WTF16: + if (!aot_compile_op_string_as_wtf16(comp_ctx, func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + if (!aot_compile_op_stringview_wtf16_length(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + if (!aot_compile_op_stringview_wtf16_get_codeunit( + comp_ctx, func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + { + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + bh_assert(mem_idx == 0); + + if (!aot_compile_op_stringview_wtf16_encode( + comp_ctx, func_ctx, mem_idx)) + return false; + break; + } + case WASM_OP_STRINGVIEW_WTF16_SLICE: + if (!aot_compile_op_stringview_wtf16_slice(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRING_AS_ITER: + if (!aot_compile_op_string_as_iter(comp_ctx, func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_ITER_NEXT: + if (!aot_compile_op_stringview_iter_next(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + if (!aot_compile_op_stringview_iter_advance(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_ITER_REWIND: + if (!aot_compile_op_stringview_iter_rewind(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRINGVIEW_ITER_SLICE: + if (!aot_compile_op_stringview_iter_slice(comp_ctx, + func_ctx)) + return false; + break; + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + { + EncodingFlag flag = WTF8; + + if (opcode == WASM_OP_STRING_NEW_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_NEW_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8) { + flag = WTF8; + } + + if (!aot_compile_op_string_new_array(comp_ctx, func_ctx, + flag)) + return false; + + break; + } + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + { + EncodingFlag flag = WTF8; + + if (opcode == WASM_OP_STRING_ENCODE_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_ENCODE_WTF8) { + flag = WTF8; + } + + if (!aot_compile_op_string_encode_array(comp_ctx, + func_ctx, flag)) + return false; + break; + } +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + + default: + aot_set_last_error("unsupported opcode"); + return false; + } + break; + } + +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_GET_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!aot_compile_op_get_local(comp_ctx, func_ctx, local_idx)) + return false; + break; + + case WASM_OP_SET_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!aot_compile_op_set_local(comp_ctx, func_ctx, local_idx)) + return false; + break; + + case WASM_OP_TEE_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!aot_compile_op_tee_local(comp_ctx, func_ctx, local_idx)) + return false; + break; + + case WASM_OP_GET_GLOBAL: + case WASM_OP_GET_GLOBAL_64: + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + if (!aot_compile_op_get_global(comp_ctx, func_ctx, global_idx)) + return false; + break; + + case WASM_OP_SET_GLOBAL: + case WASM_OP_SET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_AUX_STACK: + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + if (!aot_compile_op_set_global( + comp_ctx, func_ctx, global_idx, + opcode == WASM_OP_SET_GLOBAL_AUX_STACK ? true : false)) + return false; + break; + + case WASM_OP_I32_LOAD: + bytes = 4; + sign = true; + goto op_i32_load; + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + bytes = 1; + sign = (opcode == WASM_OP_I32_LOAD8_S) ? true : false; + goto op_i32_load; + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + bytes = 2; + sign = (opcode == WASM_OP_I32_LOAD16_S) ? true : false; + op_i32_load: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_i32_load(comp_ctx, func_ctx, align, offset, + bytes, sign, false)) + return false; + break; + + case WASM_OP_I64_LOAD: + bytes = 8; + sign = true; + goto op_i64_load; + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + bytes = 1; + sign = (opcode == WASM_OP_I64_LOAD8_S) ? true : false; + goto op_i64_load; + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + bytes = 2; + sign = (opcode == WASM_OP_I64_LOAD16_S) ? true : false; + goto op_i64_load; + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + bytes = 4; + sign = (opcode == WASM_OP_I64_LOAD32_S) ? true : false; + op_i64_load: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_i64_load(comp_ctx, func_ctx, align, offset, + bytes, sign, false)) + return false; + break; + + case WASM_OP_F32_LOAD: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_f32_load(comp_ctx, func_ctx, align, offset)) + return false; + break; + + case WASM_OP_F64_LOAD: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_f64_load(comp_ctx, func_ctx, align, offset)) + return false; + break; + + case WASM_OP_I32_STORE: + bytes = 4; + goto op_i32_store; + case WASM_OP_I32_STORE8: + bytes = 1; + goto op_i32_store; + case WASM_OP_I32_STORE16: + bytes = 2; + op_i32_store: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_i32_store(comp_ctx, func_ctx, align, offset, + bytes, false)) + return false; + break; + + case WASM_OP_I64_STORE: + bytes = 8; + goto op_i64_store; + case WASM_OP_I64_STORE8: + bytes = 1; + goto op_i64_store; + case WASM_OP_I64_STORE16: + bytes = 2; + goto op_i64_store; + case WASM_OP_I64_STORE32: + bytes = 4; + op_i64_store: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_i64_store(comp_ctx, func_ctx, align, offset, + bytes, false)) + return false; + break; + + case WASM_OP_F32_STORE: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_f32_store(comp_ctx, func_ctx, align, + offset)) + return false; + break; + + case WASM_OP_F64_STORE: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_op_f64_store(comp_ctx, func_ctx, align, + offset)) + return false; + break; + + case WASM_OP_MEMORY_SIZE: + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!aot_compile_op_memory_size(comp_ctx, func_ctx)) + return false; + (void)mem_idx; + break; + + case WASM_OP_MEMORY_GROW: + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!aot_compile_op_memory_grow(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_CONST: + read_leb_int32(frame_ip, frame_ip_end, i32_const); + if (!aot_compile_op_i32_const(comp_ctx, func_ctx, i32_const)) + return false; + break; + + case WASM_OP_I64_CONST: + read_leb_int64(frame_ip, frame_ip_end, i64_const); + if (!aot_compile_op_i64_const(comp_ctx, func_ctx, i64_const)) + return false; + break; + + case WASM_OP_F32_CONST: + p_f32 = (uint8 *)&f32_const; + for (i = 0; i < sizeof(float32); i++) + *p_f32++ = *frame_ip++; + if (!aot_compile_op_f32_const(comp_ctx, func_ctx, f32_const)) + return false; + break; + + case WASM_OP_F64_CONST: + p_f64 = (uint8 *)&f64_const; + for (i = 0; i < sizeof(float64); i++) + *p_f64++ = *frame_ip++; + if (!aot_compile_op_f64_const(comp_ctx, func_ctx, f64_const)) + return false; + break; + + case WASM_OP_I32_EQZ: + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + if (!aot_compile_op_i32_compare( + comp_ctx, func_ctx, INT_EQZ + opcode - WASM_OP_I32_EQZ)) + return false; + break; + + case WASM_OP_I64_EQZ: + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + if (!aot_compile_op_i64_compare( + comp_ctx, func_ctx, INT_EQZ + opcode - WASM_OP_I64_EQZ)) + return false; + break; + + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + if (!aot_compile_op_f32_compare( + comp_ctx, func_ctx, FLOAT_EQ + opcode - WASM_OP_F32_EQ)) + return false; + break; + + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + if (!aot_compile_op_f64_compare( + comp_ctx, func_ctx, FLOAT_EQ + opcode - WASM_OP_F64_EQ)) + return false; + break; + + case WASM_OP_I32_CLZ: + if (!aot_compile_op_i32_clz(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_CTZ: + if (!aot_compile_op_i32_ctz(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_POPCNT: + if (!aot_compile_op_i32_popcnt(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + if (!aot_compile_op_i32_arithmetic( + comp_ctx, func_ctx, INT_ADD + opcode - WASM_OP_I32_ADD, + &frame_ip)) + return false; + break; + + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + if (!aot_compile_op_i32_bitwise( + comp_ctx, func_ctx, INT_SHL + opcode - WASM_OP_I32_AND)) + return false; + break; + + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + if (!aot_compile_op_i32_shift( + comp_ctx, func_ctx, INT_SHL + opcode - WASM_OP_I32_SHL)) + return false; + break; + + case WASM_OP_I64_CLZ: + if (!aot_compile_op_i64_clz(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I64_CTZ: + if (!aot_compile_op_i64_ctz(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I64_POPCNT: + if (!aot_compile_op_i64_popcnt(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + if (!aot_compile_op_i64_arithmetic( + comp_ctx, func_ctx, INT_ADD + opcode - WASM_OP_I64_ADD, + &frame_ip)) + return false; + break; + + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + if (!aot_compile_op_i64_bitwise( + comp_ctx, func_ctx, INT_SHL + opcode - WASM_OP_I64_AND)) + return false; + break; + + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + if (!aot_compile_op_i64_shift( + comp_ctx, func_ctx, INT_SHL + opcode - WASM_OP_I64_SHL)) + return false; + break; + + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + if (!aot_compile_op_f32_math(comp_ctx, func_ctx, + FLOAT_ABS + opcode + - WASM_OP_F32_ABS)) + return false; + break; + + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + if (!aot_compile_op_f32_arithmetic(comp_ctx, func_ctx, + FLOAT_ADD + opcode + - WASM_OP_F32_ADD)) + return false; + break; + + case WASM_OP_F32_COPYSIGN: + if (!aot_compile_op_f32_copysign(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + if (!aot_compile_op_f64_math(comp_ctx, func_ctx, + FLOAT_ABS + opcode + - WASM_OP_F64_ABS)) + return false; + break; + + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + if (!aot_compile_op_f64_arithmetic(comp_ctx, func_ctx, + FLOAT_ADD + opcode + - WASM_OP_F64_ADD)) + return false; + break; + + case WASM_OP_F64_COPYSIGN: + if (!aot_compile_op_f64_copysign(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_WRAP_I64: + if (!aot_compile_op_i32_wrap_i64(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + sign = (opcode == WASM_OP_I32_TRUNC_S_F32) ? true : false; + if (!aot_compile_op_i32_trunc_f32(comp_ctx, func_ctx, sign, + false)) + return false; + break; + + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + sign = (opcode == WASM_OP_I32_TRUNC_S_F64) ? true : false; + if (!aot_compile_op_i32_trunc_f64(comp_ctx, func_ctx, sign, + false)) + return false; + break; + + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + sign = (opcode == WASM_OP_I64_EXTEND_S_I32) ? true : false; + if (!aot_compile_op_i64_extend_i32(comp_ctx, func_ctx, sign)) + return false; + break; + + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + sign = (opcode == WASM_OP_I64_TRUNC_S_F32) ? true : false; + if (!aot_compile_op_i64_trunc_f32(comp_ctx, func_ctx, sign, + false)) + return false; + break; + + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + sign = (opcode == WASM_OP_I64_TRUNC_S_F64) ? true : false; + if (!aot_compile_op_i64_trunc_f64(comp_ctx, func_ctx, sign, + false)) + return false; + break; + + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + sign = (opcode == WASM_OP_F32_CONVERT_S_I32) ? true : false; + if (!aot_compile_op_f32_convert_i32(comp_ctx, func_ctx, sign)) + return false; + break; + + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + sign = (opcode == WASM_OP_F32_CONVERT_S_I64) ? true : false; + if (!aot_compile_op_f32_convert_i64(comp_ctx, func_ctx, sign)) + return false; + break; + + case WASM_OP_F32_DEMOTE_F64: + if (!aot_compile_op_f32_demote_f64(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + sign = (opcode == WASM_OP_F64_CONVERT_S_I32) ? true : false; + if (!aot_compile_op_f64_convert_i32(comp_ctx, func_ctx, sign)) + return false; + break; + + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + sign = (opcode == WASM_OP_F64_CONVERT_S_I64) ? true : false; + if (!aot_compile_op_f64_convert_i64(comp_ctx, func_ctx, sign)) + return false; + break; + + case WASM_OP_F64_PROMOTE_F32: + if (!aot_compile_op_f64_promote_f32(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_REINTERPRET_F32: + if (!aot_compile_op_i32_reinterpret_f32(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I64_REINTERPRET_F64: + if (!aot_compile_op_i64_reinterpret_f64(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_F32_REINTERPRET_I32: + if (!aot_compile_op_f32_reinterpret_i32(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_F64_REINTERPRET_I64: + if (!aot_compile_op_f64_reinterpret_i64(comp_ctx, func_ctx)) + return false; + break; + + case WASM_OP_I32_EXTEND8_S: + if (!aot_compile_op_i32_extend_i32(comp_ctx, func_ctx, 8)) + return false; + break; + + case WASM_OP_I32_EXTEND16_S: + if (!aot_compile_op_i32_extend_i32(comp_ctx, func_ctx, 16)) + return false; + break; + + case WASM_OP_I64_EXTEND8_S: + if (!aot_compile_op_i64_extend_i64(comp_ctx, func_ctx, 8)) + return false; + break; + + case WASM_OP_I64_EXTEND16_S: + if (!aot_compile_op_i64_extend_i64(comp_ctx, func_ctx, 16)) + return false; + break; + + case WASM_OP_I64_EXTEND32_S: + if (!aot_compile_op_i64_extend_i64(comp_ctx, func_ctx, 32)) + return false; + break; + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (WASM_OP_MEMORY_INIT <= opcode + && opcode <= WASM_OP_MEMORY_FILL + && !comp_ctx->enable_bulk_memory) { + goto unsupport_bulk_memory; + } +#endif + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + if (WASM_OP_TABLE_INIT <= opcode && opcode <= WASM_OP_TABLE_FILL + && (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc)) { + goto unsupport_ref_types; + } +#endif + + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + sign = (opcode == WASM_OP_I32_TRUNC_SAT_S_F32) ? true + : false; + if (!aot_compile_op_i32_trunc_f32(comp_ctx, func_ctx, + sign, true)) + return false; + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + sign = (opcode == WASM_OP_I32_TRUNC_SAT_S_F64) ? true + : false; + if (!aot_compile_op_i32_trunc_f64(comp_ctx, func_ctx, + sign, true)) + return false; + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + sign = (opcode == WASM_OP_I64_TRUNC_SAT_S_F32) ? true + : false; + if (!aot_compile_op_i64_trunc_f32(comp_ctx, func_ctx, + sign, true)) + return false; + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + sign = (opcode == WASM_OP_I64_TRUNC_SAT_S_F64) ? true + : false; + if (!aot_compile_op_i64_trunc_f64(comp_ctx, func_ctx, + sign, true)) + return false; + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + uint32 seg_index; + read_leb_uint32(frame_ip, frame_ip_end, seg_index); + frame_ip++; + if (!aot_compile_op_memory_init(comp_ctx, func_ctx, + seg_index)) + return false; + break; + } + case WASM_OP_DATA_DROP: + { + uint32 seg_index; + read_leb_uint32(frame_ip, frame_ip_end, seg_index); + if (!aot_compile_op_data_drop(comp_ctx, func_ctx, + seg_index)) + return false; + break; + } + case WASM_OP_MEMORY_COPY: + { + frame_ip += 2; + if (!aot_compile_op_memory_copy(comp_ctx, func_ctx)) + return false; + break; + } + case WASM_OP_MEMORY_FILL: + { + frame_ip++; + if (!aot_compile_op_memory_fill(comp_ctx, func_ctx)) + return false; + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_TABLE_INIT: + { + uint32 tbl_idx, tbl_seg_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_seg_idx); + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!aot_compile_op_table_init(comp_ctx, func_ctx, + tbl_idx, tbl_seg_idx)) + return false; + break; + } + case WASM_OP_ELEM_DROP: + { + uint32 tbl_seg_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_seg_idx); + if (!aot_compile_op_elem_drop(comp_ctx, func_ctx, + tbl_seg_idx)) + return false; + break; + } + case WASM_OP_TABLE_COPY: + { + uint32 src_tbl_idx, dst_tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, dst_tbl_idx); + read_leb_uint32(frame_ip, frame_ip_end, src_tbl_idx); + if (!aot_compile_op_table_copy( + comp_ctx, func_ctx, src_tbl_idx, dst_tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_GROW: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!aot_compile_op_table_grow(comp_ctx, func_ctx, + tbl_idx)) + return false; + break; + } + + case WASM_OP_TABLE_SIZE: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!aot_compile_op_table_size(comp_ctx, func_ctx, + tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_FILL: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!aot_compile_op_table_fill(comp_ctx, func_ctx, + tbl_idx)) + return false; + break; + } +#endif /* WASM_ENABLE_REF_TYPES || WASM_ENABLE_GC */ + default: + aot_set_last_error("unsupported opcode"); + return false; + } + break; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + uint8 bin_op, op_type; + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + if (opcode != WASM_OP_ATOMIC_FENCE) { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + } + switch (opcode) { + case WASM_OP_ATOMIC_WAIT32: + if (!aot_compile_op_atomic_wait(comp_ctx, func_ctx, + VALUE_TYPE_I32, align, + offset, 4)) + return false; + break; + case WASM_OP_ATOMIC_WAIT64: + if (!aot_compile_op_atomic_wait(comp_ctx, func_ctx, + VALUE_TYPE_I64, align, + offset, 8)) + return false; + break; + case WASM_OP_ATOMIC_NOTIFY: + if (!aot_compiler_op_atomic_notify( + comp_ctx, func_ctx, align, offset, bytes)) + return false; + break; + case WASM_OP_ATOMIC_FENCE: + /* Skip memory index */ + frame_ip++; + if (!aot_compiler_op_atomic_fence(comp_ctx, func_ctx)) + return false; + break; + case WASM_OP_ATOMIC_I32_LOAD: + bytes = 4; + goto op_atomic_i32_load; + case WASM_OP_ATOMIC_I32_LOAD8_U: + bytes = 1; + goto op_atomic_i32_load; + case WASM_OP_ATOMIC_I32_LOAD16_U: + bytes = 2; + op_atomic_i32_load: + if (!aot_compile_op_i32_load(comp_ctx, func_ctx, align, + offset, bytes, sign, true)) + return false; + break; + + case WASM_OP_ATOMIC_I64_LOAD: + bytes = 8; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD8_U: + bytes = 1; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD16_U: + bytes = 2; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD32_U: + bytes = 4; + op_atomic_i64_load: + if (!aot_compile_op_i64_load(comp_ctx, func_ctx, align, + offset, bytes, sign, true)) + return false; + break; + + case WASM_OP_ATOMIC_I32_STORE: + bytes = 4; + goto op_atomic_i32_store; + case WASM_OP_ATOMIC_I32_STORE8: + bytes = 1; + goto op_atomic_i32_store; + case WASM_OP_ATOMIC_I32_STORE16: + bytes = 2; + op_atomic_i32_store: + if (!aot_compile_op_i32_store(comp_ctx, func_ctx, align, + offset, bytes, true)) + return false; + break; + + case WASM_OP_ATOMIC_I64_STORE: + bytes = 8; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE8: + bytes = 1; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE16: + bytes = 2; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE32: + bytes = 4; + op_atomic_i64_store: + if (!aot_compile_op_i64_store(comp_ctx, func_ctx, align, + offset, bytes, true)) + return false; + break; + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + bytes = 4; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + bytes = 8; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + bytes = 1; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + bytes = 2; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + bytes = 1; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + bytes = 2; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + bytes = 4; + op_type = VALUE_TYPE_I64; + op_atomic_cmpxchg: + if (!aot_compile_op_atomic_cmpxchg(comp_ctx, func_ctx, + op_type, align, + offset, bytes)) + return false; + break; + + COMPILE_ATOMIC_RMW(Add, ADD); + COMPILE_ATOMIC_RMW(Sub, SUB); + COMPILE_ATOMIC_RMW(And, AND); + COMPILE_ATOMIC_RMW(Or, OR); + COMPILE_ATOMIC_RMW(Xor, XOR); + COMPILE_ATOMIC_RMW(Xchg, XCHG); + + build_atomic_rmw: + if (!aot_compile_op_atomic_rmw(comp_ctx, func_ctx, + bin_op, op_type, align, + offset, bytes)) + return false; + break; + + default: + aot_set_last_error("unsupported opcode"); + return false; + } + break; + } +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + +#if WASM_ENABLE_SIMD != 0 + case WASM_OP_SIMD_PREFIX: + { + uint32 opcode1; + + if (!comp_ctx->enable_simd) { + goto unsupport_simd; + } + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + /* follow the order of enum WASMSimdEXTOpcode in + wasm_opcode.h */ + switch (opcode) { + /* Memory instruction */ + case SIMD_v128_load: + { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_simd_v128_load(comp_ctx, func_ctx, + align, offset)) + return false; + break; + } + + case SIMD_v128_load8x8_s: + case SIMD_v128_load8x8_u: + case SIMD_v128_load16x4_s: + case SIMD_v128_load16x4_u: + case SIMD_v128_load32x2_s: + case SIMD_v128_load32x2_u: + { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_simd_load_extend( + comp_ctx, func_ctx, opcode, align, offset)) + return false; + break; + } + + case SIMD_v128_load8_splat: + case SIMD_v128_load16_splat: + case SIMD_v128_load32_splat: + case SIMD_v128_load64_splat: + { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_simd_load_splat(comp_ctx, func_ctx, + opcode, align, offset)) + return false; + break; + } + + case SIMD_v128_store: + { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_simd_v128_store(comp_ctx, func_ctx, + align, offset)) + return false; + break; + } + + /* Basic operation */ + case SIMD_v128_const: + { + if (!aot_compile_simd_v128_const(comp_ctx, func_ctx, + frame_ip)) + return false; + frame_ip += 16; + break; + } + + case SIMD_v8x16_shuffle: + { + if (!aot_compile_simd_shuffle(comp_ctx, func_ctx, + frame_ip)) + return false; + frame_ip += 16; + break; + } + + case SIMD_v8x16_swizzle: + { + if (!aot_compile_simd_swizzle(comp_ctx, func_ctx)) + return false; + break; + } + + /* Splat operation */ + case SIMD_i8x16_splat: + case SIMD_i16x8_splat: + case SIMD_i32x4_splat: + case SIMD_i64x2_splat: + case SIMD_f32x4_splat: + case SIMD_f64x2_splat: + { + if (!aot_compile_simd_splat(comp_ctx, func_ctx, opcode)) + return false; + break; + } + + /* Lane operation */ + case SIMD_i8x16_extract_lane_s: + case SIMD_i8x16_extract_lane_u: + { + if (!aot_compile_simd_extract_i8x16( + comp_ctx, func_ctx, *frame_ip++, + SIMD_i8x16_extract_lane_s == opcode)) + return false; + break; + } + + case SIMD_i8x16_replace_lane: + { + if (!aot_compile_simd_replace_i8x16(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_i16x8_extract_lane_s: + case SIMD_i16x8_extract_lane_u: + { + if (!aot_compile_simd_extract_i16x8( + comp_ctx, func_ctx, *frame_ip++, + SIMD_i16x8_extract_lane_s == opcode)) + return false; + break; + } + + case SIMD_i16x8_replace_lane: + { + if (!aot_compile_simd_replace_i16x8(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_i32x4_extract_lane: + { + if (!aot_compile_simd_extract_i32x4(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_i32x4_replace_lane: + { + if (!aot_compile_simd_replace_i32x4(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_i64x2_extract_lane: + { + if (!aot_compile_simd_extract_i64x2(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_i64x2_replace_lane: + { + if (!aot_compile_simd_replace_i64x2(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_f32x4_extract_lane: + { + if (!aot_compile_simd_extract_f32x4(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_f32x4_replace_lane: + { + if (!aot_compile_simd_replace_f32x4(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_f64x2_extract_lane: + { + if (!aot_compile_simd_extract_f64x2(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + case SIMD_f64x2_replace_lane: + { + if (!aot_compile_simd_replace_f64x2(comp_ctx, func_ctx, + *frame_ip++)) + return false; + break; + } + + /* i8x16 Cmp */ + case SIMD_i8x16_eq: + case SIMD_i8x16_ne: + case SIMD_i8x16_lt_s: + case SIMD_i8x16_lt_u: + case SIMD_i8x16_gt_s: + case SIMD_i8x16_gt_u: + case SIMD_i8x16_le_s: + case SIMD_i8x16_le_u: + case SIMD_i8x16_ge_s: + case SIMD_i8x16_ge_u: + { + if (!aot_compile_simd_i8x16_compare( + comp_ctx, func_ctx, + INT_EQ + opcode - SIMD_i8x16_eq)) + return false; + break; + } + + /* i16x8 Cmp */ + case SIMD_i16x8_eq: + case SIMD_i16x8_ne: + case SIMD_i16x8_lt_s: + case SIMD_i16x8_lt_u: + case SIMD_i16x8_gt_s: + case SIMD_i16x8_gt_u: + case SIMD_i16x8_le_s: + case SIMD_i16x8_le_u: + case SIMD_i16x8_ge_s: + case SIMD_i16x8_ge_u: + { + if (!aot_compile_simd_i16x8_compare( + comp_ctx, func_ctx, + INT_EQ + opcode - SIMD_i16x8_eq)) + return false; + break; + } + + /* i32x4 Cmp */ + case SIMD_i32x4_eq: + case SIMD_i32x4_ne: + case SIMD_i32x4_lt_s: + case SIMD_i32x4_lt_u: + case SIMD_i32x4_gt_s: + case SIMD_i32x4_gt_u: + case SIMD_i32x4_le_s: + case SIMD_i32x4_le_u: + case SIMD_i32x4_ge_s: + case SIMD_i32x4_ge_u: + { + if (!aot_compile_simd_i32x4_compare( + comp_ctx, func_ctx, + INT_EQ + opcode - SIMD_i32x4_eq)) + return false; + break; + } + + /* f32x4 Cmp */ + case SIMD_f32x4_eq: + case SIMD_f32x4_ne: + case SIMD_f32x4_lt: + case SIMD_f32x4_gt: + case SIMD_f32x4_le: + case SIMD_f32x4_ge: + { + if (!aot_compile_simd_f32x4_compare( + comp_ctx, func_ctx, + FLOAT_EQ + opcode - SIMD_f32x4_eq)) + return false; + break; + } + + /* f64x2 Cmp */ + case SIMD_f64x2_eq: + case SIMD_f64x2_ne: + case SIMD_f64x2_lt: + case SIMD_f64x2_gt: + case SIMD_f64x2_le: + case SIMD_f64x2_ge: + { + if (!aot_compile_simd_f64x2_compare( + comp_ctx, func_ctx, + FLOAT_EQ + opcode - SIMD_f64x2_eq)) + return false; + break; + } + + /* v128 Op */ + case SIMD_v128_not: + case SIMD_v128_and: + case SIMD_v128_andnot: + case SIMD_v128_or: + case SIMD_v128_xor: + case SIMD_v128_bitselect: + { + if (!aot_compile_simd_v128_bitwise(comp_ctx, func_ctx, + V128_NOT + opcode + - SIMD_v128_not)) + return false; + break; + } + + case SIMD_v128_any_true: + { + if (!aot_compile_simd_v128_any_true(comp_ctx, func_ctx)) + return false; + break; + } + + /* Load Lane Op */ + case SIMD_v128_load8_lane: + case SIMD_v128_load16_lane: + case SIMD_v128_load32_lane: + case SIMD_v128_load64_lane: + { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_simd_load_lane(comp_ctx, func_ctx, + opcode, align, offset, + *frame_ip++)) + return false; + break; + } + + case SIMD_v128_store8_lane: + case SIMD_v128_store16_lane: + case SIMD_v128_store32_lane: + case SIMD_v128_store64_lane: + { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_simd_store_lane(comp_ctx, func_ctx, + opcode, align, offset, + *frame_ip++)) + return false; + break; + } + + case SIMD_v128_load32_zero: + case SIMD_v128_load64_zero: + { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!aot_compile_simd_load_zero(comp_ctx, func_ctx, + opcode, align, offset)) + return false; + break; + } + + /* Float conversion */ + case SIMD_f32x4_demote_f64x2_zero: + { + if (!aot_compile_simd_f64x2_demote(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f64x2_promote_low_f32x4_zero: + { + if (!aot_compile_simd_f32x4_promote(comp_ctx, func_ctx)) + return false; + break; + } + + /* i8x16 Op */ + case SIMD_i8x16_abs: + { + if (!aot_compile_simd_i8x16_abs(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i8x16_neg: + { + if (!aot_compile_simd_i8x16_neg(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i8x16_popcnt: + { + if (!aot_compile_simd_i8x16_popcnt(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i8x16_all_true: + { + if (!aot_compile_simd_i8x16_all_true(comp_ctx, + func_ctx)) + return false; + break; + } + + case SIMD_i8x16_bitmask: + { + if (!aot_compile_simd_i8x16_bitmask(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i8x16_narrow_i16x8_s: + case SIMD_i8x16_narrow_i16x8_u: + { + if (!aot_compile_simd_i8x16_narrow_i16x8( + comp_ctx, func_ctx, + (opcode == SIMD_i8x16_narrow_i16x8_s))) + return false; + break; + } + + case SIMD_f32x4_ceil: + { + if (!aot_compile_simd_f32x4_ceil(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f32x4_floor: + { + if (!aot_compile_simd_f32x4_floor(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f32x4_trunc: + { + if (!aot_compile_simd_f32x4_trunc(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f32x4_nearest: + { + if (!aot_compile_simd_f32x4_nearest(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i8x16_shl: + case SIMD_i8x16_shr_s: + case SIMD_i8x16_shr_u: + { + if (!aot_compile_simd_i8x16_shift(comp_ctx, func_ctx, + INT_SHL + opcode + - SIMD_i8x16_shl)) + return false; + break; + } + + case SIMD_i8x16_add: + { + if (!aot_compile_simd_i8x16_arith(comp_ctx, func_ctx, + V128_ADD)) + return false; + break; + } + + case SIMD_i8x16_add_sat_s: + case SIMD_i8x16_add_sat_u: + { + if (!aot_compile_simd_i8x16_saturate( + comp_ctx, func_ctx, V128_ADD, + opcode == SIMD_i8x16_add_sat_s)) + return false; + break; + } + + case SIMD_i8x16_sub: + { + if (!aot_compile_simd_i8x16_arith(comp_ctx, func_ctx, + V128_SUB)) + return false; + break; + } + + case SIMD_i8x16_sub_sat_s: + case SIMD_i8x16_sub_sat_u: + { + if (!aot_compile_simd_i8x16_saturate( + comp_ctx, func_ctx, V128_SUB, + opcode == SIMD_i8x16_sub_sat_s)) + return false; + break; + } + + case SIMD_f64x2_ceil: + { + if (!aot_compile_simd_f64x2_ceil(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f64x2_floor: + { + if (!aot_compile_simd_f64x2_floor(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i8x16_min_s: + case SIMD_i8x16_min_u: + { + if (!aot_compile_simd_i8x16_cmp( + comp_ctx, func_ctx, V128_MIN, + opcode == SIMD_i8x16_min_s)) + return false; + break; + } + + case SIMD_i8x16_max_s: + case SIMD_i8x16_max_u: + { + if (!aot_compile_simd_i8x16_cmp( + comp_ctx, func_ctx, V128_MAX, + opcode == SIMD_i8x16_max_s)) + return false; + break; + } + + case SIMD_f64x2_trunc: + { + if (!aot_compile_simd_f64x2_trunc(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i8x16_avgr_u: + { + if (!aot_compile_simd_i8x16_avgr_u(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i16x8_extadd_pairwise_i8x16_s: + case SIMD_i16x8_extadd_pairwise_i8x16_u: + { + if (!aot_compile_simd_i16x8_extadd_pairwise_i8x16( + comp_ctx, func_ctx, + SIMD_i16x8_extadd_pairwise_i8x16_s == opcode)) + return false; + break; + } + + case SIMD_i32x4_extadd_pairwise_i16x8_s: + case SIMD_i32x4_extadd_pairwise_i16x8_u: + { + if (!aot_compile_simd_i32x4_extadd_pairwise_i16x8( + comp_ctx, func_ctx, + SIMD_i32x4_extadd_pairwise_i16x8_s == opcode)) + return false; + break; + } + + /* i16x8 Op */ + case SIMD_i16x8_abs: + { + if (!aot_compile_simd_i16x8_abs(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i16x8_neg: + { + if (!aot_compile_simd_i16x8_neg(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i16x8_q15mulr_sat_s: + { + if (!aot_compile_simd_i16x8_q15mulr_sat(comp_ctx, + func_ctx)) + return false; + break; + } + + case SIMD_i16x8_all_true: + { + if (!aot_compile_simd_i16x8_all_true(comp_ctx, + func_ctx)) + return false; + break; + } + + case SIMD_i16x8_bitmask: + { + if (!aot_compile_simd_i16x8_bitmask(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i16x8_narrow_i32x4_s: + case SIMD_i16x8_narrow_i32x4_u: + { + if (!aot_compile_simd_i16x8_narrow_i32x4( + comp_ctx, func_ctx, + SIMD_i16x8_narrow_i32x4_s == opcode)) + return false; + break; + } + + case SIMD_i16x8_extend_low_i8x16_s: + case SIMD_i16x8_extend_high_i8x16_s: + { + if (!aot_compile_simd_i16x8_extend_i8x16( + comp_ctx, func_ctx, + SIMD_i16x8_extend_low_i8x16_s == opcode, true)) + return false; + break; + } + + case SIMD_i16x8_extend_low_i8x16_u: + case SIMD_i16x8_extend_high_i8x16_u: + { + if (!aot_compile_simd_i16x8_extend_i8x16( + comp_ctx, func_ctx, + SIMD_i16x8_extend_low_i8x16_u == opcode, false)) + return false; + break; + } + + case SIMD_i16x8_shl: + case SIMD_i16x8_shr_s: + case SIMD_i16x8_shr_u: + { + if (!aot_compile_simd_i16x8_shift(comp_ctx, func_ctx, + INT_SHL + opcode + - SIMD_i16x8_shl)) + return false; + break; + } + + case SIMD_i16x8_add: + { + if (!aot_compile_simd_i16x8_arith(comp_ctx, func_ctx, + V128_ADD)) + return false; + break; + } + + case SIMD_i16x8_add_sat_s: + case SIMD_i16x8_add_sat_u: + { + if (!aot_compile_simd_i16x8_saturate( + comp_ctx, func_ctx, V128_ADD, + opcode == SIMD_i16x8_add_sat_s ? true : false)) + return false; + break; + } + + case SIMD_i16x8_sub: + { + if (!aot_compile_simd_i16x8_arith(comp_ctx, func_ctx, + V128_SUB)) + return false; + break; + } + + case SIMD_i16x8_sub_sat_s: + case SIMD_i16x8_sub_sat_u: + { + if (!aot_compile_simd_i16x8_saturate( + comp_ctx, func_ctx, V128_SUB, + opcode == SIMD_i16x8_sub_sat_s ? true : false)) + return false; + break; + } + + case SIMD_f64x2_nearest: + { + if (!aot_compile_simd_f64x2_nearest(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i16x8_mul: + { + if (!aot_compile_simd_i16x8_arith(comp_ctx, func_ctx, + V128_MUL)) + return false; + break; + } + + case SIMD_i16x8_min_s: + case SIMD_i16x8_min_u: + { + if (!aot_compile_simd_i16x8_cmp( + comp_ctx, func_ctx, V128_MIN, + opcode == SIMD_i16x8_min_s)) + return false; + break; + } + + case SIMD_i16x8_max_s: + case SIMD_i16x8_max_u: + { + if (!aot_compile_simd_i16x8_cmp( + comp_ctx, func_ctx, V128_MAX, + opcode == SIMD_i16x8_max_s)) + return false; + break; + } + + case SIMD_i16x8_avgr_u: + { + if (!aot_compile_simd_i16x8_avgr_u(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i16x8_extmul_low_i8x16_s: + case SIMD_i16x8_extmul_high_i8x16_s: + { + if (!(aot_compile_simd_i16x8_extmul_i8x16( + comp_ctx, func_ctx, + SIMD_i16x8_extmul_low_i8x16_s == opcode, true))) + return false; + break; + } + + case SIMD_i16x8_extmul_low_i8x16_u: + case SIMD_i16x8_extmul_high_i8x16_u: + { + if (!(aot_compile_simd_i16x8_extmul_i8x16( + comp_ctx, func_ctx, + SIMD_i16x8_extmul_low_i8x16_u == opcode, + false))) + return false; + break; + } + + /* i32x4 Op */ + case SIMD_i32x4_abs: + { + if (!aot_compile_simd_i32x4_abs(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i32x4_neg: + { + if (!aot_compile_simd_i32x4_neg(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i32x4_all_true: + { + if (!aot_compile_simd_i32x4_all_true(comp_ctx, + func_ctx)) + return false; + break; + } + + case SIMD_i32x4_bitmask: + { + if (!aot_compile_simd_i32x4_bitmask(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i32x4_extend_low_i16x8_s: + case SIMD_i32x4_extend_high_i16x8_s: + { + if (!aot_compile_simd_i32x4_extend_i16x8( + comp_ctx, func_ctx, + SIMD_i32x4_extend_low_i16x8_s == opcode, true)) + return false; + break; + } + + case SIMD_i32x4_extend_low_i16x8_u: + case SIMD_i32x4_extend_high_i16x8_u: + { + if (!aot_compile_simd_i32x4_extend_i16x8( + comp_ctx, func_ctx, + SIMD_i32x4_extend_low_i16x8_u == opcode, false)) + return false; + break; + } + + case SIMD_i32x4_shl: + case SIMD_i32x4_shr_s: + case SIMD_i32x4_shr_u: + { + if (!aot_compile_simd_i32x4_shift(comp_ctx, func_ctx, + INT_SHL + opcode + - SIMD_i32x4_shl)) + return false; + break; + } + + case SIMD_i32x4_add: + { + if (!aot_compile_simd_i32x4_arith(comp_ctx, func_ctx, + V128_ADD)) + return false; + break; + } + + case SIMD_i32x4_sub: + { + if (!aot_compile_simd_i32x4_arith(comp_ctx, func_ctx, + V128_SUB)) + return false; + break; + } + + case SIMD_i32x4_mul: + { + if (!aot_compile_simd_i32x4_arith(comp_ctx, func_ctx, + V128_MUL)) + return false; + break; + } + + case SIMD_i32x4_min_s: + case SIMD_i32x4_min_u: + { + if (!aot_compile_simd_i32x4_cmp( + comp_ctx, func_ctx, V128_MIN, + SIMD_i32x4_min_s == opcode)) + return false; + break; + } + + case SIMD_i32x4_max_s: + case SIMD_i32x4_max_u: + { + if (!aot_compile_simd_i32x4_cmp( + comp_ctx, func_ctx, V128_MAX, + SIMD_i32x4_max_s == opcode)) + return false; + break; + } + + case SIMD_i32x4_dot_i16x8_s: + { + if (!aot_compile_simd_i32x4_dot_i16x8(comp_ctx, + func_ctx)) + return false; + break; + } + + case SIMD_i32x4_extmul_low_i16x8_s: + case SIMD_i32x4_extmul_high_i16x8_s: + { + if (!aot_compile_simd_i32x4_extmul_i16x8( + comp_ctx, func_ctx, + SIMD_i32x4_extmul_low_i16x8_s == opcode, true)) + return false; + break; + } + + case SIMD_i32x4_extmul_low_i16x8_u: + case SIMD_i32x4_extmul_high_i16x8_u: + { + if (!aot_compile_simd_i32x4_extmul_i16x8( + comp_ctx, func_ctx, + SIMD_i32x4_extmul_low_i16x8_u == opcode, false)) + return false; + break; + } + + /* i64x2 Op */ + case SIMD_i64x2_abs: + { + if (!aot_compile_simd_i64x2_abs(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i64x2_neg: + { + if (!aot_compile_simd_i64x2_neg(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i64x2_all_true: + { + if (!aot_compile_simd_i64x2_all_true(comp_ctx, + func_ctx)) + return false; + break; + } + + case SIMD_i64x2_bitmask: + { + if (!aot_compile_simd_i64x2_bitmask(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_i64x2_extend_low_i32x4_s: + case SIMD_i64x2_extend_high_i32x4_s: + { + if (!aot_compile_simd_i64x2_extend_i32x4( + comp_ctx, func_ctx, + SIMD_i64x2_extend_low_i32x4_s == opcode, true)) + return false; + break; + } + + case SIMD_i64x2_extend_low_i32x4_u: + case SIMD_i64x2_extend_high_i32x4_u: + { + if (!aot_compile_simd_i64x2_extend_i32x4( + comp_ctx, func_ctx, + SIMD_i64x2_extend_low_i32x4_u == opcode, false)) + return false; + break; + } + + case SIMD_i64x2_shl: + case SIMD_i64x2_shr_s: + case SIMD_i64x2_shr_u: + { + if (!aot_compile_simd_i64x2_shift(comp_ctx, func_ctx, + INT_SHL + opcode + - SIMD_i64x2_shl)) + return false; + break; + } + + case SIMD_i64x2_add: + { + if (!aot_compile_simd_i64x2_arith(comp_ctx, func_ctx, + V128_ADD)) + return false; + break; + } + + case SIMD_i64x2_sub: + { + if (!aot_compile_simd_i64x2_arith(comp_ctx, func_ctx, + V128_SUB)) + return false; + break; + } + + case SIMD_i64x2_mul: + { + if (!aot_compile_simd_i64x2_arith(comp_ctx, func_ctx, + V128_MUL)) + return false; + break; + } + + case SIMD_i64x2_eq: + case SIMD_i64x2_ne: + case SIMD_i64x2_lt_s: + case SIMD_i64x2_gt_s: + case SIMD_i64x2_le_s: + case SIMD_i64x2_ge_s: + { + IntCond icond[] = { INT_EQ, INT_NE, INT_LT_S, + INT_GT_S, INT_LE_S, INT_GE_S }; + if (!aot_compile_simd_i64x2_compare( + comp_ctx, func_ctx, + icond[opcode - SIMD_i64x2_eq])) + return false; + break; + } + + case SIMD_i64x2_extmul_low_i32x4_s: + case SIMD_i64x2_extmul_high_i32x4_s: + { + if (!aot_compile_simd_i64x2_extmul_i32x4( + comp_ctx, func_ctx, + SIMD_i64x2_extmul_low_i32x4_s == opcode, true)) + return false; + break; + } + + case SIMD_i64x2_extmul_low_i32x4_u: + case SIMD_i64x2_extmul_high_i32x4_u: + { + if (!aot_compile_simd_i64x2_extmul_i32x4( + comp_ctx, func_ctx, + SIMD_i64x2_extmul_low_i32x4_u == opcode, false)) + return false; + break; + } + + /* f32x4 Op */ + case SIMD_f32x4_abs: + { + if (!aot_compile_simd_f32x4_abs(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f32x4_neg: + { + if (!aot_compile_simd_f32x4_neg(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f32x4_sqrt: + { + if (!aot_compile_simd_f32x4_sqrt(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f32x4_add: + case SIMD_f32x4_sub: + case SIMD_f32x4_mul: + case SIMD_f32x4_div: + { + if (!aot_compile_simd_f32x4_arith(comp_ctx, func_ctx, + FLOAT_ADD + opcode + - SIMD_f32x4_add)) + return false; + break; + } + + case SIMD_f32x4_min: + case SIMD_f32x4_max: + { + if (!aot_compile_simd_f32x4_min_max( + comp_ctx, func_ctx, SIMD_f32x4_min == opcode)) + return false; + break; + } + + case SIMD_f32x4_pmin: + case SIMD_f32x4_pmax: + { + if (!aot_compile_simd_f32x4_pmin_pmax( + comp_ctx, func_ctx, SIMD_f32x4_pmin == opcode)) + return false; + break; + } + + /* f64x2 Op */ + + case SIMD_f64x2_abs: + { + if (!aot_compile_simd_f64x2_abs(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f64x2_neg: + { + if (!aot_compile_simd_f64x2_neg(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f64x2_sqrt: + { + if (!aot_compile_simd_f64x2_sqrt(comp_ctx, func_ctx)) + return false; + break; + } + + case SIMD_f64x2_add: + case SIMD_f64x2_sub: + case SIMD_f64x2_mul: + case SIMD_f64x2_div: + { + if (!aot_compile_simd_f64x2_arith(comp_ctx, func_ctx, + FLOAT_ADD + opcode + - SIMD_f64x2_add)) + return false; + break; + } + + case SIMD_f64x2_min: + case SIMD_f64x2_max: + { + if (!aot_compile_simd_f64x2_min_max( + comp_ctx, func_ctx, SIMD_f64x2_min == opcode)) + return false; + break; + } + + case SIMD_f64x2_pmin: + case SIMD_f64x2_pmax: + { + if (!aot_compile_simd_f64x2_pmin_pmax( + comp_ctx, func_ctx, SIMD_f64x2_pmin == opcode)) + return false; + break; + } + + /* Conversion Op */ + case SIMD_i32x4_trunc_sat_f32x4_s: + case SIMD_i32x4_trunc_sat_f32x4_u: + { + if (!aot_compile_simd_i32x4_trunc_sat_f32x4( + comp_ctx, func_ctx, + SIMD_i32x4_trunc_sat_f32x4_s == opcode)) + return false; + break; + } + + case SIMD_f32x4_convert_i32x4_s: + case SIMD_f32x4_convert_i32x4_u: + { + if (!aot_compile_simd_f32x4_convert_i32x4( + comp_ctx, func_ctx, + SIMD_f32x4_convert_i32x4_s == opcode)) + return false; + break; + } + + case SIMD_i32x4_trunc_sat_f64x2_s_zero: + case SIMD_i32x4_trunc_sat_f64x2_u_zero: + { + if (!aot_compile_simd_i32x4_trunc_sat_f64x2( + comp_ctx, func_ctx, + SIMD_i32x4_trunc_sat_f64x2_s_zero == opcode)) + return false; + break; + } + + case SIMD_f64x2_convert_low_i32x4_s: + case SIMD_f64x2_convert_low_i32x4_u: + { + if (!aot_compile_simd_f64x2_convert_i32x4( + comp_ctx, func_ctx, + SIMD_f64x2_convert_low_i32x4_s == opcode)) + return false; + break; + } + + default: + aot_set_last_error("unsupported SIMD opcode"); + return false; + } + break; + } +#endif /* end of WASM_ENABLE_SIMD */ + + default: + aot_set_last_error("unsupported opcode"); + return false; + } + } + + /* Move func_return block to the bottom */ + if (func_ctx->func_return_block) { + LLVMBasicBlockRef last_block = LLVMGetLastBasicBlock(func_ctx->func); + if (last_block != func_ctx->func_return_block) + LLVMMoveBasicBlockAfter(func_ctx->func_return_block, last_block); + } + + /* Move got_exception block to the bottom */ + if (func_ctx->got_exception_block) { + LLVMBasicBlockRef last_block = LLVMGetLastBasicBlock(func_ctx->func); + if (last_block != func_ctx->got_exception_block) + LLVMMoveBasicBlockAfter(func_ctx->got_exception_block, last_block); + } + return true; + +#if WASM_ENABLE_SIMD != 0 +unsupport_simd: + aot_set_last_error("SIMD instruction was found, " + "try removing --disable-simd option"); + return false; +#endif + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +unsupport_ref_types: + aot_set_last_error("reference type instruction was found, " + "try removing --disable-ref-types option " + "or adding --enable-gc option"); + return false; +#endif + +#if WASM_ENABLE_GC != 0 +unsupport_gc: + aot_set_last_error("GC instruction was found, " + "try adding --enable-gc option"); + return false; +#endif + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +unsupport_gc_and_ref_types: + aot_set_last_error( + "reference type or gc instruction was found, try removing " + "--disable-ref-types option or adding --enable-gc option"); + return false; +#endif + +#if WASM_ENABLE_BULK_MEMORY != 0 +unsupport_bulk_memory: + aot_set_last_error("bulk memory instruction was found, " + "try removing --disable-bulk-memory option"); + return false; +#endif + +fail: + return false; +} + +static bool +verify_module(AOTCompContext *comp_ctx) +{ + char *msg = NULL; + bool ret; + + ret = LLVMVerifyModule(comp_ctx->module, LLVMPrintMessageAction, &msg); + if (!ret && msg) { + if (msg[0] != '\0') { + aot_set_last_error(msg); + LLVMDisposeMessage(msg); + return false; + } + LLVMDisposeMessage(msg); + } + + return true; +} + +bool +aot_compile_wasm(AOTCompContext *comp_ctx) +{ + uint32 i; + + if (!aot_validate_wasm(comp_ctx)) { + return false; + } + + bh_print_time("Begin to compile WASM bytecode to LLVM IR"); + for (i = 0; i < comp_ctx->func_ctx_count; i++) { + if (!aot_compile_func(comp_ctx, i)) { + return false; + } + } + +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMDIBuilderFinalize(comp_ctx->debug_builder); +#endif + + /* Disable LLVM module verification for jit mode to speedup + the compilation process */ + if (!comp_ctx->is_jit_mode) { + bh_print_time("Begin to verify LLVM module"); + if (!verify_module(comp_ctx)) { + return false; + } + } + + /* Run IR optimization before feeding in ORCJIT and AOT codegen */ + if (comp_ctx->optimize) { + /* Run passes for AOT/JIT mode. + TODO: Apply these passes in the do_ir_transform callback of + TransformLayer when compiling each jit function, so as to + speedup the launch process. Now there are two issues in the + JIT: one is memory leak in do_ir_transform, the other is + possible core dump. */ + bh_print_time("Begin to run llvm optimization passes"); + aot_apply_llvm_new_pass_manager(comp_ctx, comp_ctx->module); + bh_print_time("Finish llvm optimization passes"); + } + +#ifdef DUMP_MODULE + LLVMDumpModule(comp_ctx->module); + os_printf("\n"); +#endif + + if (comp_ctx->is_jit_mode) { + LLVMErrorRef err; + LLVMOrcJITDylibRef orc_main_dylib; + LLVMOrcThreadSafeModuleRef orc_thread_safe_module; + + orc_main_dylib = LLVMOrcLLLazyJITGetMainJITDylib(comp_ctx->orc_jit); + if (!orc_main_dylib) { + aot_set_last_error( + "failed to get orc orc_jit main dynmaic library"); + return false; + } + + orc_thread_safe_module = LLVMOrcCreateNewThreadSafeModule( + comp_ctx->module, comp_ctx->orc_thread_safe_context); + if (!orc_thread_safe_module) { + aot_set_last_error("failed to create thread safe module"); + return false; + } + + if ((err = LLVMOrcLLLazyJITAddLLVMIRModule( + comp_ctx->orc_jit, orc_main_dylib, orc_thread_safe_module))) { + /* If adding the ThreadSafeModule fails then we need to clean it up + by ourselves, otherwise the orc orc_jit will manage the memory. + */ + LLVMOrcDisposeThreadSafeModule(orc_thread_safe_module); + aot_handle_llvm_errmsg("failed to addIRModule", err); + return false; + } + + if (comp_ctx->stack_sizes != NULL) { + LLVMOrcJITTargetAddress addr; + if ((err = LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &addr, + aot_stack_sizes_alias_name))) { + aot_handle_llvm_errmsg("failed to look up stack_sizes", err); + return false; + } + comp_ctx->jit_stack_sizes = (uint32 *)addr; + } + } + + return true; +} + +#if !(defined(_WIN32) || defined(_WIN32_)) +char * +aot_generate_tempfile_name(const char *prefix, const char *extension, + char *buffer, uint32 len) +{ + int fd, name_len; + + name_len = snprintf(buffer, len, "%s-XXXXXX", prefix); + + if ((fd = mkstemp(buffer)) <= 0) { + aot_set_last_error("make temp file failed."); + return NULL; + } + + /* close and remove temp file */ + close(fd); + unlink(buffer); + + /* Check if buffer length is enough */ + /* name_len + '.' + extension + '\0' */ + if (name_len + 1 + strlen(extension) + 1 > len) { + aot_set_last_error("temp file name too long."); + return NULL; + } + + snprintf(buffer + name_len, len - name_len, ".%s", extension); + return buffer; +} +#else + +errno_t +_mktemp_s(char *nameTemplate, size_t sizeInChars); + +char * +aot_generate_tempfile_name(const char *prefix, const char *extension, + char *buffer, uint32 len) +{ + int name_len; + + name_len = snprintf(buffer, len, "%s-XXXXXX", prefix); + + if (_mktemp_s(buffer, name_len + 1) != 0) { + return NULL; + } + + /* Check if buffer length is enough */ + /* name_len + '.' + extension + '\0' */ + if (name_len + 1 + strlen(extension) + 1 > len) { + aot_set_last_error("temp file name too long."); + return NULL; + } + + snprintf(buffer + name_len, len - name_len, ".%s", extension); + return buffer; +} +#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */ + +bool +aot_emit_llvm_file(AOTCompContext *comp_ctx, const char *file_name) +{ + char *err = NULL; + + bh_print_time("Begin to emit LLVM IR file"); + + if (LLVMPrintModuleToFile(comp_ctx->module, file_name, &err) != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("emit llvm ir to file failed."); + return false; + } + + return true; +} + +static bool +aot_move_file(const char *dest, const char *src) +{ + FILE *dfp = fopen(dest, "w"); + FILE *sfp = fopen(src, "r"); + size_t rsz; + char buf[128]; + bool success = false; + + if (dfp == NULL || sfp == NULL) { + LOG_DEBUG("open error %s %s", dest, src); + goto fail; + } + do { + rsz = fread(buf, 1, sizeof(buf), sfp); + if (rsz > 0) { + size_t wsz = fwrite(buf, 1, rsz, dfp); + if (wsz < rsz) { + LOG_DEBUG("write error"); + goto fail; + } + } + if (rsz < sizeof(buf)) { + if (ferror(sfp)) { + LOG_DEBUG("read error"); + goto fail; + } + } + } while (rsz > 0); + success = true; +fail: + if (dfp != NULL) { + if (fclose(dfp)) { + LOG_DEBUG("close error"); + success = false; + } + if (!success) { + (void)unlink(dest); + } + } + if (sfp != NULL) { + (void)fclose(sfp); + } + if (success) { + (void)unlink(src); + } + return success; +} + +bool +aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name) +{ + char *err = NULL; + LLVMCodeGenFileType file_type = LLVMObjectFile; + LLVMTargetRef target = LLVMGetTargetMachineTarget(comp_ctx->target_machine); + + bh_print_time("Begin to emit object file"); + +#if !(defined(_WIN32) || defined(_WIN32_)) + if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) { + char cmd[1024]; + int ret; + + if (comp_ctx->external_llc_compiler) { + const char *stack_usage_flag = ""; + char bc_file_name[64]; + char su_file_name[65]; /* See the comment below */ + + if (comp_ctx->stack_usage_file != NULL) { + /* + * Note: we know the caller uses 64 byte buffer for + * file_name. It will get 1 byte longer because we + * replace ".o" with ".su". + */ + size_t len = strlen(file_name); + bh_assert(len + 1 <= sizeof(su_file_name)); + bh_assert(len > 3); + bh_assert(file_name[len - 2] == '.'); + bh_assert(file_name[len - 1] == 'o'); + snprintf(su_file_name, sizeof(su_file_name), "%.*s.su", + (int)(len - 2), file_name); + stack_usage_flag = " -fstack-usage"; + } + + if (!aot_generate_tempfile_name("wamrc-bc", "bc", bc_file_name, + sizeof(bc_file_name))) { + return false; + } + + if (LLVMWriteBitcodeToFile(comp_ctx->module, bc_file_name) != 0) { + aot_set_last_error("emit llvm bitcode file failed."); + return false; + } + + snprintf(cmd, sizeof(cmd), "%s%s %s -o %s %s", + comp_ctx->external_llc_compiler, stack_usage_flag, + comp_ctx->llc_compiler_flags ? comp_ctx->llc_compiler_flags + : "-O3 -c", + file_name, bc_file_name); + LOG_VERBOSE("invoking external LLC compiler:\n\t%s", cmd); + + ret = system(cmd); + /* remove temp bitcode file */ + unlink(bc_file_name); + + if (ret != 0) { + aot_set_last_error("failed to compile LLVM bitcode to obj file " + "with external LLC compiler."); + return false; + } + if (comp_ctx->stack_usage_file != NULL) { + /* + * move the temporary .su file to the specified location. + * + * Note: the former is automatimally inferred from the output + * filename (file_name here) by clang. + * + * Note: the latter might be user-specified. + * (wamrc --stack-usage=) + */ + if (!aot_move_file(comp_ctx->stack_usage_file, su_file_name)) { + aot_set_last_error("failed to move su file."); + (void)unlink(su_file_name); + return false; + } + } + } + else if (comp_ctx->external_asm_compiler) { + char asm_file_name[64]; + + if (!aot_generate_tempfile_name("wamrc-asm", "s", asm_file_name, + sizeof(asm_file_name))) { + return false; + } + + if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine, + comp_ctx->module, asm_file_name, + LLVMAssemblyFile, &err) + != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("emit elf to assembly file failed."); + return false; + } + + snprintf(cmd, sizeof(cmd), "%s %s -o %s %s", + comp_ctx->external_asm_compiler, + comp_ctx->asm_compiler_flags ? comp_ctx->asm_compiler_flags + : "-O3 -c", + file_name, asm_file_name); + LOG_VERBOSE("invoking external ASM compiler:\n\t%s", cmd); + + ret = system(cmd); + /* remove temp assembly file */ + unlink(asm_file_name); + + if (ret != 0) { + aot_set_last_error("failed to compile Assembly file to obj " + "file with external ASM compiler."); + return false; + } + } + + return true; + } +#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */ + + if (!strncmp(LLVMGetTargetName(target), "arc", 3)) + /* Emit to assmelby file instead for arc target + as it cannot emit to object file */ + file_type = LLVMAssemblyFile; + + if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine, comp_ctx->module, + file_name, file_type, &err) + != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("emit elf to object file failed."); + return false; + } + + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_compiler.h b/wasm-micro-runtime/core/iwasm/compilation/aot_compiler.h new file mode 100644 index 0000000..5038a41 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_compiler.h @@ -0,0 +1,789 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_COMPILER_H_ +#define _AOT_COMPILER_H_ + +#include "aot.h" +#include "aot_llvm.h" +#include "../interpreter/wasm_interp.h" +#include "../aot/aot_runtime.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef AOTIntCond IntCond; +typedef AOTFloatCond FloatCond; + +typedef enum IntArithmetic { + INT_ADD = 0, + INT_SUB, + INT_MUL, + INT_DIV_S, + INT_DIV_U, + INT_REM_S, + INT_REM_U +} IntArithmetic; + +typedef enum V128Arithmetic { + V128_ADD = 0, + V128_SUB, + V128_MUL, + V128_DIV, + V128_NEG, + V128_MIN, + V128_MAX, +} V128Arithmetic; + +typedef enum IntBitwise { + INT_AND = 0, + INT_OR, + INT_XOR, +} IntBitwise; + +typedef enum V128Bitwise { + V128_NOT, + V128_AND, + V128_ANDNOT, + V128_OR, + V128_XOR, + V128_BITSELECT, +} V128Bitwise; + +typedef enum IntShift { + INT_SHL = 0, + INT_SHR_S, + INT_SHR_U, + INT_ROTL, + INT_ROTR +} IntShift; + +typedef enum FloatMath { + FLOAT_ABS = 0, + FLOAT_NEG, + FLOAT_CEIL, + FLOAT_FLOOR, + FLOAT_TRUNC, + FLOAT_NEAREST, + FLOAT_SQRT +} FloatMath; + +typedef enum FloatArithmetic { + FLOAT_ADD = 0, + FLOAT_SUB, + FLOAT_MUL, + FLOAT_DIV, + FLOAT_MIN, + FLOAT_MAX, +} FloatArithmetic; + +/** + * Check whether a value type is a GC reference type, + * don't use wasm_is_type_reftype since it requires + * GC feature and may result in compilation error when + * GC feature isn't compiled + */ +static inline bool +aot_is_type_gc_reftype(uint8 type) +{ + return ((type >= (uint8)REF_TYPE_ARRAYREF + && type <= (uint8)REF_TYPE_NULLFUNCREF) + || (type >= (uint8)REF_TYPE_HT_NULLABLE + && type <= (uint8)REF_TYPE_HT_NON_NULLABLE) +#if WASM_ENABLE_STRINGREF != 0 + || (type >= (uint8)REF_TYPE_STRINGVIEWWTF8 + && type <= (uint8)REF_TYPE_STRINGREF) + || (type >= (uint8)REF_TYPE_STRINGVIEWITER + && type <= (uint8)REF_TYPE_STRINGVIEWWTF16) +#endif + ) + ? true + : false; +} + +static inline bool +check_type_compatible(const AOTCompContext *comp_ctx, uint8 src_type, + uint8 dst_type) +{ + if (src_type == dst_type) { + return true; + } + + /* ext i1 to i32 */ + if (src_type == VALUE_TYPE_I1 && dst_type == VALUE_TYPE_I32) { + return true; + } + + /* i32 <==> func.ref, i32 <==> extern.ref */ + if (src_type == VALUE_TYPE_I32 + && (comp_ctx->enable_ref_types + && (dst_type == VALUE_TYPE_EXTERNREF + || dst_type == VALUE_TYPE_FUNCREF))) { + return true; + } + + if (dst_type == VALUE_TYPE_I32 + && (comp_ctx->enable_ref_types + && (src_type == VALUE_TYPE_FUNCREF + || src_type == VALUE_TYPE_EXTERNREF))) { + return true; + } + + return false; +} + +/** + * Operations for AOTCompFrame + */ + +/** + * Get the offset from frame pointer to the n-th local variable slot. + * + * @param n the index to the local variable array + * + * @return the offset from frame pointer to the local variable slot + */ +static inline uint32 +offset_of_local(AOTCompContext *comp_ctx, unsigned n) +{ + if (!comp_ctx->is_jit_mode) + /* In AOTFrame, there are 7 pointers before field lp */ + return comp_ctx->pointer_size + * (offsetof(AOTFrame, lp) / sizeof(uintptr_t)) + + sizeof(uint32) * n; + else + return offsetof(WASMInterpFrame, lp) + sizeof(uint32) * n; +} + +uint32 +offset_of_local_in_outs_area(AOTCompContext *comp_ctx, unsigned n); + +/** + * Get the offset from frame pointer to the n-th local variable's + * reference flag slot. + * + * @param n the index to the local variable array + * + * @return the offset from frame pointer to the local variable slot + */ +static inline unsigned +offset_of_ref(AOTCompContext *comp_ctx, unsigned n) +{ + AOTCompFrame *frame = comp_ctx->aot_frame; + uint32 all_cell_num = frame->max_local_cell_num + frame->max_stack_cell_num; + return offset_of_local(comp_ctx, all_cell_num) + n; +} + +/** + * Generate instructions to commit computation result to the frame. + * The general principle is to only commit values that will be used + * through the frame. + * + * @param frame the frame information + */ +bool +aot_gen_commit_values(AOTCompFrame *frame); + +/** + * Generate instructions to commit SP and IP pointers to the frame. + * + * @param frame the frame information + */ +bool +aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip); + +bool +aot_frame_store_value(AOTCompContext *comp_ctx, LLVMValueRef value, + uint8 value_type, LLVMValueRef cur_frame, uint32 offset); + +static inline void +push_32bit(AOTCompFrame *frame, AOTValue *aot_value) +{ + frame->sp->value = aot_value->value; + frame->sp->type = aot_value->type; + frame->sp->dirty = 1; + frame->sp++; +} + +static inline void +push_64bit(AOTCompFrame *frame, AOTValue *aot_value) +{ + push_32bit(frame, aot_value); + push_32bit(frame, aot_value); +} + +static inline void +push_i32(AOTCompFrame *frame, AOTValue *aot_value) +{ + bh_assert(aot_value->type == VALUE_TYPE_I32 + || aot_value->type == VALUE_TYPE_I1); + push_32bit(frame, aot_value); +} + +static inline void +push_i64(AOTCompFrame *frame, AOTValue *aot_value) +{ + bh_assert(aot_value->type == VALUE_TYPE_I64); + push_64bit(frame, aot_value); +} + +static inline void +push_f32(AOTCompFrame *frame, AOTValue *aot_value) +{ + bh_assert(aot_value->type == VALUE_TYPE_F32); + push_32bit(frame, aot_value); +} + +static inline void +push_f64(AOTCompFrame *frame, AOTValue *aot_value) +{ + bh_assert(aot_value->type == VALUE_TYPE_F64); + push_64bit(frame, aot_value); +} + +static inline void +push_v128(AOTCompFrame *frame, AOTValue *aot_value) +{ + bh_assert(aot_value->type == VALUE_TYPE_V128); + push_64bit(frame, aot_value); + push_64bit(frame, aot_value); +} + +static inline void +push_ref(AOTCompFrame *frame, AOTValue *aot_value) +{ + bh_assert(frame->comp_ctx->enable_ref_types); + push_32bit(frame, aot_value); +} + +#if WASM_ENABLE_GC != 0 +static inline void +push_gc_ref(AOTCompFrame *frame, AOTValue *aot_value) +{ + bh_assert(frame->comp_ctx->enable_gc); + bh_assert(aot_value->type == VALUE_TYPE_GC_REF); + if (frame->comp_ctx->pointer_size == sizeof(uint64)) { + push_64bit(frame, aot_value); + (frame->sp - 1)->ref = (frame->sp - 2)->ref = 1; + } + else { + push_32bit(frame, aot_value); + (frame->sp - 1)->ref = 1; + } +} +#endif + +/* Clear value slots except ref and committed_ref */ +static inline void +clear_frame_value_slots(AOTValueSlot *slots, uint32 n) +{ + uint32 i; + for (i = 0; i < n; i++) { + slots[i].value = 0; + slots[i].type = 0; + slots[i].dirty = 0; + } +} + +static inline void +pop_i32(AOTCompFrame *frame) +{ + bh_assert(frame->sp - frame->lp >= 1); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_I32 + || (frame->sp - 1)->type == VALUE_TYPE_I1); + frame->sp--; + clear_frame_value_slots(frame->sp, 1); +} + +static inline void +pop_i64(AOTCompFrame *frame) +{ + bh_assert(frame->sp - frame->lp >= 2); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_I64 + && (frame->sp - 2)->type == VALUE_TYPE_I64); + frame->sp -= 2; + clear_frame_value_slots(frame->sp, 2); +} + +static inline void +pop_f32(AOTCompFrame *frame) +{ + bh_assert(frame->sp - frame->lp >= 1); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_F32); + frame->sp--; + clear_frame_value_slots(frame->sp, 1); +} + +static inline void +pop_f64(AOTCompFrame *frame) +{ + bh_assert(frame->sp - frame->lp >= 2); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_F64 + && (frame->sp - 2)->type == VALUE_TYPE_F64); + frame->sp -= 2; + clear_frame_value_slots(frame->sp, 2); +} + +static inline void +pop_v128(AOTCompFrame *frame) +{ + bh_assert(frame->sp - frame->lp >= 4); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_V128 + && (frame->sp - 2)->type == VALUE_TYPE_V128 + && (frame->sp - 3)->type == VALUE_TYPE_V128 + && (frame->sp - 4)->type == VALUE_TYPE_V128); + frame->sp -= 4; + clear_frame_value_slots(frame->sp, 4); +} + +static inline void +pop_ref(AOTCompFrame *frame) +{ + bh_assert(frame->sp - frame->lp >= 1); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_FUNCREF + || (frame->sp - 1)->type == VALUE_TYPE_EXTERNREF); + frame->sp -= 1; + clear_frame_value_slots(frame->sp, 1); +} + +#if WASM_ENABLE_GC != 0 +static inline void +pop_gc_ref(AOTCompFrame *frame) +{ + bh_assert(frame->sp - frame->lp >= 1); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_GC_REF); + frame->sp -= 1; + clear_frame_value_slots(frame->sp, 1); + frame->sp->ref = 0; + if (frame->comp_ctx->pointer_size == sizeof(uint64)) { + bh_assert(frame->sp - frame->lp >= 1); + bh_assert((frame->sp - 1)->type == VALUE_TYPE_GC_REF); + frame->sp -= 1; + clear_frame_value_slots(frame->sp, 1); + frame->sp->ref = 0; + } +} +#endif + +static inline void +set_local_i32(AOTCompFrame *frame, int n, LLVMValueRef value) +{ + frame->lp[n].value = value; + frame->lp[n].type = VALUE_TYPE_I32; + frame->lp[n].dirty = 1; +} + +static inline void +set_local_i64(AOTCompFrame *frame, int n, LLVMValueRef value) +{ + frame->lp[n].value = value; + frame->lp[n].type = VALUE_TYPE_I64; + frame->lp[n].dirty = 1; + frame->lp[n + 1].value = value; + frame->lp[n + 1].type = VALUE_TYPE_I64; + frame->lp[n + 1].dirty = 1; +} + +static inline void +set_local_f32(AOTCompFrame *frame, int n, LLVMValueRef value) +{ + frame->lp[n].value = value; + frame->lp[n].type = VALUE_TYPE_F32; + frame->lp[n].dirty = 1; +} + +static inline void +set_local_f64(AOTCompFrame *frame, int n, LLVMValueRef value) +{ + frame->lp[n].value = value; + frame->lp[n].type = VALUE_TYPE_F64; + frame->lp[n].dirty = 1; + frame->lp[n + 1].value = value; + frame->lp[n + 1].type = VALUE_TYPE_F64; + frame->lp[n + 1].dirty = 1; +} + +static inline void +set_local_v128(AOTCompFrame *frame, int n, LLVMValueRef value) +{ + uint32 i; + for (i = 0; i < 4; i++) { + frame->lp[n + i].value = value; + frame->lp[n + i].type = VALUE_TYPE_V128; + frame->lp[n + i].dirty = 1; + } +} + +static inline void +set_local_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type) +{ + bh_assert(frame->comp_ctx->enable_ref_types); + frame->lp[n].value = value; + frame->lp[n].type = ref_type; + frame->lp[n].dirty = 1; +} + +#if WASM_ENABLE_GC != 0 +static inline void +set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type) +{ + bh_assert(frame->comp_ctx->enable_gc); + bh_assert(ref_type == VALUE_TYPE_GC_REF); + frame->lp[n].value = value; + frame->lp[n].type = ref_type; + frame->lp[n].dirty = 1; + frame->lp[n].ref = 1; + if (frame->comp_ctx->pointer_size == sizeof(uint64)) { + frame->lp[n + 1].value = value; + frame->lp[n + 1].type = ref_type; + frame->lp[n + 1].dirty = 1; + frame->lp[n + 1].ref = 1; + } +} +#endif + +#define CHECK_STACK() \ + do { \ + if (!func_ctx->block_stack.block_list_end) { \ + aot_set_last_error("WASM block stack underflow."); \ + goto fail; \ + } \ + if (!func_ctx->block_stack.block_list_end->value_stack \ + .value_list_end) { \ + aot_set_last_error("WASM data stack underflow."); \ + goto fail; \ + } \ + } while (0) + +#if WASM_ENABLE_GC != 0 + +#define GET_GC_REF_FROM_STACK(llvm_value) \ + do { \ + AOTValue *aot_value; \ + CHECK_STACK(); \ + aot_value = \ + func_ctx->block_stack.block_list_end->value_stack.value_list_end; \ + if (aot_value->type != VALUE_TYPE_GC_REF) { \ + aot_set_last_error("WASM stack data type is not reference"); \ + goto fail; \ + } \ + llvm_value = aot_value->value; \ + } while (0) + +#endif + +#define POP(llvm_value, value_type) \ + do { \ + AOTValue *aot_value; \ + uint8 val_type_to_pop = value_type; \ + CHECK_STACK(); \ + aot_value = aot_value_stack_pop( \ + comp_ctx, &func_ctx->block_stack.block_list_end->value_stack); \ + if (comp_ctx->enable_gc && aot_is_type_gc_reftype(value_type)) \ + val_type_to_pop = VALUE_TYPE_GC_REF; \ + if (!check_type_compatible(comp_ctx, aot_value->type, \ + val_type_to_pop)) { \ + aot_set_last_error("invalid WASM stack data type."); \ + wasm_runtime_free(aot_value); \ + goto fail; \ + } \ + if (aot_value->type == val_type_to_pop) \ + llvm_value = aot_value->value; \ + else { \ + if (aot_value->type == VALUE_TYPE_I1) { \ + if (!(llvm_value = \ + LLVMBuildZExt(comp_ctx->builder, aot_value->value, \ + I32_TYPE, "i1toi32"))) { \ + aot_set_last_error("invalid WASM stack data type."); \ + wasm_runtime_free(aot_value); \ + goto fail; \ + } \ + } \ + else { \ + bh_assert( \ + aot_value->type == VALUE_TYPE_I32 \ + || (comp_ctx->enable_ref_types \ + && (aot_value->type == VALUE_TYPE_FUNCREF \ + || aot_value->type == VALUE_TYPE_EXTERNREF))); \ + bh_assert( \ + val_type_to_pop == VALUE_TYPE_I32 \ + || (comp_ctx->enable_ref_types \ + && (val_type_to_pop == VALUE_TYPE_FUNCREF \ + || val_type_to_pop == VALUE_TYPE_EXTERNREF))); \ + llvm_value = aot_value->value; \ + } \ + } \ + wasm_runtime_free(aot_value); \ + } while (0) + +#define POP_I32(v) POP(v, VALUE_TYPE_I32) +#define POP_I64(v) POP(v, VALUE_TYPE_I64) +#define POP_F32(v) POP(v, VALUE_TYPE_F32) +#define POP_F64(v) POP(v, VALUE_TYPE_F64) +#define POP_V128(v) POP(v, VALUE_TYPE_V128) +#define POP_FUNCREF(v) POP(v, VALUE_TYPE_FUNCREF) +#define POP_EXTERNREF(v) POP(v, VALUE_TYPE_EXTERNREF) +#define POP_GC_REF(v) POP(v, VALUE_TYPE_GC_REF) + +#define POP_COND(llvm_value) \ + do { \ + AOTValue *aot_value; \ + CHECK_STACK(); \ + aot_value = aot_value_stack_pop( \ + comp_ctx, &func_ctx->block_stack.block_list_end->value_stack); \ + if (aot_value->type != VALUE_TYPE_I1 \ + && aot_value->type != VALUE_TYPE_I32) { \ + aot_set_last_error("invalid WASM stack data type."); \ + wasm_runtime_free(aot_value); \ + goto fail; \ + } \ + if (aot_value->type == VALUE_TYPE_I1) \ + llvm_value = aot_value->value; \ + else { \ + if (!(llvm_value = \ + LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, \ + aot_value->value, I32_ZERO, "i1_cond"))) { \ + aot_set_last_error("llvm build trunc failed."); \ + wasm_runtime_free(aot_value); \ + goto fail; \ + } \ + } \ + wasm_runtime_free(aot_value); \ + } while (0) + +#define PUSH(llvm_value, value_type) \ + do { \ + AOTValue *aot_value; \ + if (!func_ctx->block_stack.block_list_end) { \ + aot_set_last_error("WASM block stack underflow."); \ + goto fail; \ + } \ + aot_value = wasm_runtime_malloc(sizeof(AOTValue)); \ + if (!aot_value) { \ + aot_set_last_error("allocate memory failed."); \ + goto fail; \ + } \ + memset(aot_value, 0, sizeof(AOTValue)); \ + if (comp_ctx->enable_gc && aot_is_type_gc_reftype(value_type)) \ + aot_value->type = VALUE_TYPE_GC_REF; \ + else if (comp_ctx->enable_ref_types \ + && (value_type == VALUE_TYPE_FUNCREF \ + || value_type == VALUE_TYPE_EXTERNREF)) \ + aot_value->type = VALUE_TYPE_I32; \ + else \ + aot_value->type = value_type; \ + aot_value->value = llvm_value; \ + aot_value_stack_push( \ + comp_ctx, &func_ctx->block_stack.block_list_end->value_stack, \ + aot_value); \ + } while (0) + +#define PUSH_I32(v) PUSH(v, VALUE_TYPE_I32) +#define PUSH_I64(v) PUSH(v, VALUE_TYPE_I64) +#define PUSH_F32(v) PUSH(v, VALUE_TYPE_F32) +#define PUSH_F64(v) PUSH(v, VALUE_TYPE_F64) +#define PUSH_V128(v) PUSH(v, VALUE_TYPE_V128) +#define PUSH_COND(v) PUSH(v, VALUE_TYPE_I1) +#define PUSH_FUNCREF(v) PUSH(v, VALUE_TYPE_FUNCREF) +#define PUSH_EXTERNREF(v) PUSH(v, VALUE_TYPE_EXTERNREF) +#define PUSH_GC_REF(v) PUSH(v, VALUE_TYPE_GC_REF) + +#define TO_LLVM_TYPE(wasm_type) \ + wasm_type_to_llvm_type(comp_ctx, &comp_ctx->basic_types, wasm_type) + +#define I32_TYPE comp_ctx->basic_types.int32_type +#define I64_TYPE comp_ctx->basic_types.int64_type +#define F32_TYPE comp_ctx->basic_types.float32_type +#define F64_TYPE comp_ctx->basic_types.float64_type +#define VOID_TYPE comp_ctx->basic_types.void_type +#define INT1_TYPE comp_ctx->basic_types.int1_type +#define INT8_TYPE comp_ctx->basic_types.int8_type +#define INT16_TYPE comp_ctx->basic_types.int16_type +#define INTPTR_T_TYPE comp_ctx->basic_types.intptr_t_type +#define MD_TYPE comp_ctx->basic_types.meta_data_type +#define INT8_PTR_TYPE comp_ctx->basic_types.int8_ptr_type +#define INT16_PTR_TYPE comp_ctx->basic_types.int16_ptr_type +#define INT32_PTR_TYPE comp_ctx->basic_types.int32_ptr_type +#define INT64_PTR_TYPE comp_ctx->basic_types.int64_ptr_type +#define INTPTR_T_PTR_TYPE comp_ctx->basic_types.intptr_t_ptr_type +#define F32_PTR_TYPE comp_ctx->basic_types.float32_ptr_type +#define F64_PTR_TYPE comp_ctx->basic_types.float64_ptr_type +#define FUNC_REF_TYPE comp_ctx->basic_types.funcref_type +#define EXTERN_REF_TYPE comp_ctx->basic_types.externref_type +#define GC_REF_TYPE comp_ctx->basic_types.gc_ref_type +#define GC_REF_PTR_TYPE comp_ctx->basic_types.gc_ref_ptr_type + +#define INT8_PTR_TYPE_GS comp_ctx->basic_types.int8_ptr_type_gs +#define INT16_PTR_TYPE_GS comp_ctx->basic_types.int16_ptr_type_gs +#define INT32_PTR_TYPE_GS comp_ctx->basic_types.int32_ptr_type_gs +#define INT64_PTR_TYPE_GS comp_ctx->basic_types.int64_ptr_type_gs +#define F32_PTR_TYPE_GS comp_ctx->basic_types.float32_ptr_type_gs +#define F64_PTR_TYPE_GS comp_ctx->basic_types.float64_ptr_type_gs + +#define I32_CONST(v) LLVMConstInt(I32_TYPE, v, true) +#define I64_CONST(v) LLVMConstInt(I64_TYPE, v, true) +#define F32_CONST(v) LLVMConstReal(F32_TYPE, v) +#define F64_CONST(v) LLVMConstReal(F64_TYPE, v) +#define I8_CONST(v) LLVMConstInt(INT8_TYPE, v, true) + +#define LLVM_CONST(name) (comp_ctx->llvm_consts.name) +#define I1_ZERO LLVM_CONST(i1_zero) +#define I1_ONE LLVM_CONST(i1_one) +#define I8_ZERO LLVM_CONST(i8_zero) +#define I8_ONE LLVM_CONST(i8_one) +#define I32_ZERO LLVM_CONST(i32_zero) +#define I64_ZERO LLVM_CONST(i64_zero) +#define F32_ZERO LLVM_CONST(f32_zero) +#define F64_ZERO LLVM_CONST(f64_zero) +#define I32_ONE LLVM_CONST(i32_one) +#define I32_TWO LLVM_CONST(i32_two) +#define I32_THREE LLVM_CONST(i32_three) +#define I32_FOUR LLVM_CONST(i32_four) +#define I32_FIVE LLVM_CONST(i32_five) +#define I32_SIX LLVM_CONST(i32_six) +#define I32_SEVEN LLVM_CONST(i32_seven) +#define I32_EIGHT LLVM_CONST(i32_eight) +#define I32_NINE LLVM_CONST(i32_nine) +#define I32_TEN LLVM_CONST(i32_ten) +#define I32_ELEVEN LLVM_CONST(i32_eleven) +#define I32_TWELVE LLVM_CONST(i32_twelve) +#define I32_NEG_ONE LLVM_CONST(i32_neg_one) +#define I64_NEG_ONE LLVM_CONST(i64_neg_one) +#define I32_MIN LLVM_CONST(i32_min) +#define I64_MIN LLVM_CONST(i64_min) +#define I32_31 LLVM_CONST(i32_31) +#define I32_32 LLVM_CONST(i32_32) +#define I64_63 LLVM_CONST(i64_63) +#define I64_64 LLVM_CONST(i64_64) +#define REF_NULL I32_NEG_ONE +#define GC_REF_NULL LLVM_CONST(gc_ref_null) +#define I8_PTR_NULL LLVM_CONST(i8_ptr_null) + +#define V128_TYPE comp_ctx->basic_types.v128_type +#define V128_PTR_TYPE comp_ctx->basic_types.v128_ptr_type +#define V128_PTR_TYPE_GS comp_ctx->basic_types.v128_ptr_type_gs +#define V128_i8x16_TYPE comp_ctx->basic_types.i8x16_vec_type +#define V128_i16x8_TYPE comp_ctx->basic_types.i16x8_vec_type +#define V128_i32x4_TYPE comp_ctx->basic_types.i32x4_vec_type +#define V128_i64x2_TYPE comp_ctx->basic_types.i64x2_vec_type +#define V128_f32x4_TYPE comp_ctx->basic_types.f32x4_vec_type +#define V128_f64x2_TYPE comp_ctx->basic_types.f64x2_vec_type + +#define V128_i8x16_ZERO LLVM_CONST(i8x16_vec_zero) +#define V128_i16x8_ZERO LLVM_CONST(i16x8_vec_zero) +#define V128_i32x4_ZERO LLVM_CONST(i32x4_vec_zero) +#define V128_i64x2_ZERO LLVM_CONST(i64x2_vec_zero) +#define V128_f32x4_ZERO LLVM_CONST(f32x4_vec_zero) +#define V128_f64x2_ZERO LLVM_CONST(f64x2_vec_zero) + +#define TO_V128_i8x16(v) \ + LLVMBuildBitCast(comp_ctx->builder, v, V128_i8x16_TYPE, "i8x16_val") +#define TO_V128_i16x8(v) \ + LLVMBuildBitCast(comp_ctx->builder, v, V128_i16x8_TYPE, "i16x8_val") +#define TO_V128_i32x4(v) \ + LLVMBuildBitCast(comp_ctx->builder, v, V128_i32x4_TYPE, "i32x4_val") +#define TO_V128_i64x2(v) \ + LLVMBuildBitCast(comp_ctx->builder, v, V128_i64x2_TYPE, "i64x2_val") +#define TO_V128_f32x4(v) \ + LLVMBuildBitCast(comp_ctx->builder, v, V128_f32x4_TYPE, "f32x4_val") +#define TO_V128_f64x2(v) \ + LLVMBuildBitCast(comp_ctx->builder, v, V128_f64x2_TYPE, "f64x2_val") + +#define CHECK_LLVM_CONST(v) \ + do { \ + if (!v) { \ + aot_set_last_error("create llvm const failed."); \ + goto fail; \ + } \ + } while (0) + +#define GET_AOT_FUNCTION(name, argc) \ + do { \ + if (!(func_type = \ + LLVMFunctionType(ret_type, param_types, argc, false))) { \ + aot_set_last_error("llvm add function type failed."); \ + goto fail; \ + } \ + if (comp_ctx->is_jit_mode) { \ + /* JIT mode, call the function directly */ \ + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { \ + aot_set_last_error("llvm add pointer type failed."); \ + goto fail; \ + } \ + if (!(value = I64_CONST((uint64)(uintptr_t)name)) \ + || !(func = LLVMConstIntToPtr(value, func_ptr_type))) { \ + aot_set_last_error("create LLVM value failed."); \ + goto fail; \ + } \ + } \ + else if (comp_ctx->is_indirect_mode) { \ + int32 func_index; \ + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { \ + aot_set_last_error("create LLVM function type failed."); \ + goto fail; \ + } \ + \ + func_index = aot_get_native_symbol_index(comp_ctx, #name); \ + if (func_index < 0) { \ + goto fail; \ + } \ + if (!(func = aot_get_func_from_table( \ + comp_ctx, func_ctx->native_symbol, func_ptr_type, \ + func_index))) { \ + goto fail; \ + } \ + } \ + else { \ + char *func_name = #name; \ + /* AOT mode, delcare the function */ \ + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) \ + && !(func = LLVMAddFunction(func_ctx->module, func_name, \ + func_type))) { \ + aot_set_last_error("llvm add function failed."); \ + goto fail; \ + } \ + } \ + } while (0) + +/* if val is a constant integer and its value is not undef or poison */ +static inline bool +LLVMIsEfficientConstInt(LLVMValueRef val) +{ + return LLVMIsConstant(val) + && LLVMGetValueKind(val) == LLVMConstantIntValueKind + && !LLVMIsUndef(val) +#if LLVM_VERSION_NUMBER >= 12 + && !LLVMIsPoison(addr) +#endif + ; +} + +bool +aot_compile_wasm(AOTCompContext *comp_ctx); + +bool +aot_emit_llvm_file(AOTCompContext *comp_ctx, const char *file_name); + +bool +aot_emit_aot_file(AOTCompContext *comp_ctx, AOTCompData *comp_data, + const char *file_name); + +uint8 * +aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data, + uint32 *p_aot_file_size); + +bool +aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name); + +char * +aot_generate_tempfile_name(const char *prefix, const char *extension, + char *buffer, uint32 len); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_COMPILER_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_aot_file.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_aot_file.c new file mode 100644 index 0000000..4261719 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_aot_file.c @@ -0,0 +1,4579 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_compiler.h" +#include "../aot/aot_runtime.h" + +#define PUT_U64_TO_ADDR(addr, value) \ + do { \ + union { \ + uint64 val; \ + uint32 parts[2]; \ + } u; \ + u.val = (value); \ + ((uint32 *)(addr))[0] = u.parts[0]; \ + ((uint32 *)(addr))[1] = u.parts[1]; \ + } while (0) + +#define CHECK_SIZE(size) \ + do { \ + if (size == (uint32)-1) { \ + aot_set_last_error("get symbol size failed."); \ + return (uint32)-1; \ + } \ + } while (0) + +/* Internal function in object file */ +typedef struct AOTObjectFunc { + char *func_name; + /* text offset of aot_func#n */ + uint64 text_offset; + /* text offset of aot_func_internal#n */ + uint64 text_offset_of_aot_func_internal; +} AOTObjectFunc; + +/* Symbol table list node */ +typedef struct AOTSymbolNode { + struct AOTSymbolNode *next; + uint32 str_len; + char *symbol; +} AOTSymbolNode; + +typedef struct AOTSymbolList { + AOTSymbolNode *head; + AOTSymbolNode *end; + uint32 len; +} AOTSymbolList; + +/* AOT object data */ +typedef struct AOTObjectData { + AOTCompContext *comp_ctx; + + LLVMMemoryBufferRef mem_buf; + LLVMBinaryRef binary; + + AOTTargetInfo target_info; + + void *text; + uint32 text_size; + + void *text_unlikely; + uint32 text_unlikely_size; + + void *text_hot; + uint32 text_hot_size; + + /* literal data and size */ + void *literal; + uint32 literal_size; + + AOTObjectDataSection *data_sections; + uint32 data_sections_count; + + AOTObjectFunc *funcs; + uint32 func_count; + + AOTSymbolList symbol_list; + AOTRelocationGroup *relocation_groups; + uint32 relocation_group_count; + + const char *stack_sizes_section_name; + uint32 stack_sizes_offset; + uint32 *stack_sizes; +} AOTObjectData; + +#if 0 +static void dump_buf(uint8 *buf, uint32 size, char *title) +{ + int i; + printf("------ %s -------", title); + for (i = 0; i < size; i++) { + if ((i % 16) == 0) + printf("\n"); + printf("%02x ", (unsigned char)buf[i]); + } + printf("\n\n"); +} +#endif + +static bool +is_32bit_binary(const AOTObjectData *obj_data) +{ + /* bit 1: 0 is 32-bit, 1 is 64-bit */ + return obj_data->target_info.bin_type & 2 ? false : true; +} + +static bool +is_little_endian_binary(const AOTObjectData *obj_data) +{ + /* bit 0: 0 is little-endian, 1 is big-endian */ + return obj_data->target_info.bin_type & 1 ? false : true; +} + +static bool +need_call_wrapped_indirect(const AOTObjectData *obj_data) +{ + const bool need_precheck = obj_data->comp_ctx->enable_stack_bound_check + || obj_data->comp_ctx->enable_stack_estimation; + + return obj_data->comp_ctx->is_indirect_mode && need_precheck + && !strncmp(obj_data->comp_ctx->target_arch, "xtensa", 6); +} + +static bool +str_starts_with(const char *str, const char *prefix) +{ + size_t len_pre = strlen(prefix), len_str = strlen(str); + return (len_str >= len_pre) && !memcmp(str, prefix, len_pre); +} + +static uint32 +get_file_header_size() +{ + /* magic number (4 bytes) + version (4 bytes) */ + return sizeof(uint32) + sizeof(uint32); +} + +static uint32 +get_string_size(AOTCompContext *comp_ctx, const char *s) +{ + /* string size (2 bytes) + string content + '\0' */ + return (uint32)sizeof(uint16) + (uint32)strlen(s) + 1; +} + +static uint32 +get_target_info_section_size() +{ + return sizeof(AOTTargetInfo); +} + +static uint32 +get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data, + InitializerExpression *expr); + +static uint32 +get_mem_init_data_size(AOTCompContext *comp_ctx, AOTMemInitData *mem_init_data) +{ + /* init expr type (4 bytes) + * + init expr value (4 bytes, valid value can only be i32/get_global) + * + byte count (4 bytes) + bytes */ + uint32 total_size = + (uint32)(get_init_expr_size(comp_ctx, comp_ctx->comp_data, + &mem_init_data->offset) + + sizeof(uint32) + mem_init_data->byte_count); + + /* bulk_memory enabled: + is_passive (4 bytes) + memory_index (4 bytes) + bulk memory disabled: + placeholder (4 bytes) + placeholder (4 bytes) + */ + total_size += (sizeof(uint32) + sizeof(uint32)); + + return total_size; +} + +static uint32 +get_mem_init_data_list_size(AOTCompContext *comp_ctx, + AOTMemInitData **mem_init_data_list, + uint32 mem_init_data_count) +{ + AOTMemInitData **mem_init_data = mem_init_data_list; + uint32 size = 0, i; + + for (i = 0; i < mem_init_data_count; i++, mem_init_data++) { + size = align_uint(size, 4); + size += get_mem_init_data_size(comp_ctx, *mem_init_data); + } + return size; +} + +static uint32 +get_import_memory_size(AOTCompData *comp_data) +{ + /* currently we only emit import_memory_count = 0 */ + return sizeof(uint32); +} + +static uint32 +get_memory_size(AOTCompData *comp_data) +{ + /* memory_count + count * (memory_flags + num_bytes_per_page + + init_page_count + max_page_count) */ + return (uint32)(sizeof(uint32) + + comp_data->memory_count * sizeof(uint32) * 4); +} + +static uint32 +get_mem_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ + /* import_memory_size + memory_size + + init_data_count + init_data_list */ + return get_import_memory_size(comp_data) + get_memory_size(comp_data) + + (uint32)sizeof(uint32) + + get_mem_init_data_list_size(comp_ctx, + comp_data->mem_init_data_list, + comp_data->mem_init_data_count); +} + +static uint32 +get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data, + InitializerExpression *expr) +{ + /* init_expr_type */ + uint32 size = sizeof(uint32); +#if WASM_ENABLE_GC != 0 + WASMModule *module = comp_data->wasm_module; +#endif + + /* + init value size */ + switch (expr->init_expr_type) { + case INIT_EXPR_NONE: + /* no init value, used in table initializer */ + break; + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_F32_CONST: + case INIT_EXPR_TYPE_GET_GLOBAL: + size += sizeof(uint32); + break; + case INIT_EXPR_TYPE_I64_CONST: + case INIT_EXPR_TYPE_F64_CONST: + size += sizeof(uint64); + break; + case INIT_EXPR_TYPE_V128_CONST: + size += sizeof(uint64) * 2; + break; + case INIT_EXPR_TYPE_FUNCREF_CONST: + case INIT_EXPR_TYPE_REFNULL_CONST: + /* ref_index */ + size += sizeof(uint32); + break; +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_I31_NEW: + /* i32 */ + size += sizeof(uint32); + break; + case INIT_EXPR_TYPE_STRUCT_NEW: + { + uint32 i; + WASMStructNewInitValues *struct_new_init_values = + (WASMStructNewInitValues *)expr->u.data; + + /* type_index + field_count + fields */ + size += sizeof(uint32) + sizeof(uint32); + + bh_assert(struct_new_init_values->type_idx < module->type_count); + + for (i = 0; i < struct_new_init_values->count; i++) { + WASMStructType *struct_type = + (WASMStructType *) + module->types[struct_new_init_values->type_idx]; + uint32 field_size; + + bh_assert(struct_type); + bh_assert(struct_type->field_count + == struct_new_init_values->count); + + field_size = wasm_value_type_size_internal( + struct_type->fields[i].field_type, comp_ctx->pointer_size); + if (field_size < sizeof(uint32)) + field_size = sizeof(uint32); + size += field_size; + } + break; + } + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + /* type_index */ + size += sizeof(uint32); + break; + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + /* array_elem_type + type_index + len */ + size += sizeof(uint32) * 3; + break; + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + WASMArrayNewInitValues *array_new_init_values = + (WASMArrayNewInitValues *)expr->u.data; + WASMArrayType *array_type = NULL; + uint32 value_count; + + array_type = + (WASMArrayType *)module->types[array_new_init_values->type_idx]; + + bh_assert(array_type); + bh_assert(array_new_init_values->type_idx < module->type_count); + + value_count = + (expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) + ? array_new_init_values->length + : 1; + + /* array_elem_type + type_index + len + elems */ + size += sizeof(uint32) * 3 + + wasm_value_type_size_internal(array_type->elem_type, + comp_ctx->pointer_size) + * value_count; + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + bh_assert(0); + } + + return size; +} + +static uint32 +get_table_init_data_size(AOTCompContext *comp_ctx, + AOTTableInitData *table_init_data) +{ + uint32 size, i; + + /* + * mode (4 bytes), elem_type (4 bytes) + * + * table_index(4 bytes) + init expr type (4 bytes) + init expr value (8 + * bytes) + */ + size = (uint32)(sizeof(uint32) * 2 + sizeof(uint32) + sizeof(uint32) + + sizeof(uint64)) + /* Size of WasmRefType - inner padding (ref type + nullable + + heap_type) */ + + 8; + + /* + value count/func index count (4 bytes) + init_values */ + size += sizeof(uint32); + for (i = 0; i < table_init_data->value_count; i++) { + size += get_init_expr_size(comp_ctx, comp_ctx->comp_data, + &table_init_data->init_values[i]); + } + + return size; +} + +static uint32 +get_table_init_data_list_size(AOTCompContext *comp_ctx, + AOTTableInitData **table_init_data_list, + uint32 table_init_data_count) +{ + /* + * ------------------------------ + * | table_init_data_count + * ------------------------------ + * | | U32 mode + * | AOTTableInitData[N] | U32 elem_type + * | | U32 table_index + * | | U32 offset.init_expr_type + * | | U64 offset.u.i64 + * | | U32 func_index_count / elem_count + * | | UINTPTR [func_index_count] / [elem_count] + * ------------------------------ + */ + AOTTableInitData **table_init_data = table_init_data_list; + uint32 size = 0, i; + + /* table_init_data_count(4 bytes) */ + size = (uint32)sizeof(uint32); + + for (i = 0; i < table_init_data_count; i++, table_init_data++) { + size = align_uint(size, 4); + size += get_table_init_data_size(comp_ctx, *table_init_data); + } + return size; +} + +static uint32 +get_import_table_size(const AOTCompContext *comp_ctx, + const AOTCompData *comp_data) +{ + /* + * ------------------------------ + * | import_table_count + * ------------------------------ + * | | U8 elem_type + * | | U8 table_flags + * | | U8 possible_grow + * | AOTImportTable[N] | U8 elem_ref_type.nullable (for GC only) + * | | U32 table_init_size + * | | U32 table_max_size + * | | U32 elem_ref_type.heap_type (for GC only) + * ------------------------------ + */ + uint32 size = 0, i; + + size = (uint32)sizeof(uint32); + for (i = 0; i < comp_data->import_table_count; i++) { + size += sizeof(uint32) * 3; +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc && comp_data->import_tables[i].elem_ref_type) + size += sizeof(uint32); +#endif + } + return size; +} + +static uint32 +get_table_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data) +{ + /* + * ------------------------------ + * | table_count + * ------------------------------ + * | | U8 elem_type + * | | U8 table_flags + * | | U8 possible_grow + * | AOTTable[N] | U8 elem_ref_type.nullable (for GC only) + * | | U32 table_init_size + * | | U32 table_max_size + * | | U32 elem_ref_type.heap_type (for GC only) + * | | N init_expr (for GC only) + * ------------------------------ + */ + uint32 size = 0, i; + + size = (uint32)sizeof(uint32); + for (i = 0; i < comp_data->table_count; i++) { + size += sizeof(uint32) * 3; +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + if (comp_data->tables[i].elem_ref_type) { + size += sizeof(uint32); + } + size += get_init_expr_size(comp_ctx, comp_data, + &comp_data->tables[i].init_expr); + } +#endif + } + return size; +} + +static uint32 +get_table_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ + /* + * ------------------------------ + * | import_table_count + * ------------------------------ + * | + * | AOTImportTable[import_table_count] + * | + * ------------------------------ + * | table_count + * ------------------------------ + * | + * | AOTTable[table_count] + * | + * ------------------------------ + * | table_init_data_count + * ------------------------------ + * | + * | AOTTableInitData*[table_init_data_count] + * | + * ------------------------------ + */ + return get_import_table_size(comp_ctx, comp_data) + + get_table_size(comp_ctx, comp_data) + + get_table_init_data_list_size(comp_ctx, + comp_data->table_init_data_list, + comp_data->table_init_data_count); +} + +static uint32 +get_func_type_size(AOTCompContext *comp_ctx, AOTFuncType *func_type) +{ +#if WASM_ENABLE_GC != 0 + /* type flag + equivalence type flag + is_sub_final + parent_type_idx + + rec_count + rec_idx + param count + result count + + ref_type_map_count + types + context of ref_type_map */ + if (comp_ctx->enable_gc) { + uint32 size = 0; + + /* type flag */ + size += sizeof(func_type->base_type.type_flag); + /* equivalence type flag + is_sub_final */ + size += sizeof(uint16); + /* parent_type_idx */ + size += sizeof(func_type->base_type.parent_type_idx); + /* rec_count */ + size += sizeof(func_type->base_type.rec_count); + /* rec_idx */ + size += sizeof(func_type->base_type.rec_idx); + /* param count */ + size += sizeof(func_type->param_count); + /* result count */ + size += sizeof(func_type->result_count); + /* ref_type_map_count */ + size += sizeof(func_type->ref_type_map_count); + /* param and result types */ + size += func_type->param_count + func_type->result_count; + /* align size */ + size = align_uint(size, 4); + /* ref_type_map */ + size += func_type->ref_type_map_count * 8; + + return size; + } + else +#endif + { + /* type flag + param count + result count + types */ + return (uint32)sizeof(uint16) * 3 + func_type->param_count + + func_type->result_count; + } +} + +#if WASM_ENABLE_GC != 0 +static uint32 +get_struct_type_size(AOTCompContext *comp_ctx, AOTStructType *struct_type) +{ + uint32 size = 0; + /* type flag + equivalence type flag + is_sub_final + parent_type_idx + + rec_count + rec_idx + field count + fields */ + + /* type flag */ + size += sizeof(struct_type->base_type.type_flag); + /* equivalence type flag + is_sub_final */ + size += sizeof(uint16); + /* parent_type_idx */ + size += sizeof(struct_type->base_type.parent_type_idx); + /* rec_count */ + size += sizeof(struct_type->base_type.rec_count); + /* rec_idx */ + size += sizeof(struct_type->base_type.rec_idx); + /* field count */ + size += sizeof(struct_type->field_count); + /* field types */ + size += struct_type->field_count * 2; + /* ref_type_map_count */ + size += sizeof(struct_type->ref_type_map_count); + size = align_uint(size, 4); + /* ref_type_map */ + size += struct_type->ref_type_map_count * 8; + return size; +} + +static uint32 +get_array_type_size(AOTCompContext *comp_ctx, AOTArrayType *array_type) +{ + uint32 size = 0; + /* type flag + equivalence type flag + is_sub_final + parent_type_idx + + rec_count + rec_idx + elem_flags + elem_type + elem_ref_type */ + + /* type flag */ + size += sizeof(array_type->base_type.type_flag); + /* equivalence type flag + is_sub_final */ + size += sizeof(uint16); + /* parent_type_idx (u32) */ + size += sizeof(array_type->base_type.parent_type_idx); + /* rec_count */ + size += sizeof(array_type->base_type.rec_count); + /* rec_idx */ + size += sizeof(array_type->base_type.rec_idx); + /* elem_flags (u16) */ + size += sizeof(array_type->elem_flags); + /* elem_type (u8) */ + size += sizeof(array_type->elem_type); + /* elem_ref_type */ + if (array_type->elem_ref_type) { + /* nullable (u8) */ + size += sizeof(uint8); + /* heap type (u32) */ + size += sizeof(uint32); + } + + return size; +} +#endif + +static uint32 +get_type_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ + /* Initial size with size of type count */ + uint32 size = 4; + uint32 i; + +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + for (i = 0; i < comp_data->type_count; i++) { + uint32 j; + + size = align_uint(size, 4); + + /* Emit simple info if there is an equivalence type */ + for (j = 0; j < i; j++) { + if (comp_data->types[j] == comp_data->types[i]) { + /* type_flag (2 bytes) + equivalence type flag (1 byte) + + padding (1 byte) + equivalence type index */ + size += 8; + break; + } + } + if (j < i) + continue; + + if (comp_data->types[i]->type_flag == WASM_TYPE_FUNC) + size += get_func_type_size(comp_ctx, + (AOTFuncType *)comp_data->types[i]); + else if (comp_data->types[i]->type_flag == WASM_TYPE_STRUCT) + size += get_struct_type_size( + comp_ctx, (AOTStructType *)comp_data->types[i]); + else if (comp_data->types[i]->type_flag == WASM_TYPE_ARRAY) + size += get_array_type_size( + comp_ctx, (AOTArrayType *)comp_data->types[i]); + else + bh_assert(0); + } + } + else +#endif + { + for (i = 0; i < comp_data->type_count; i++) { + size = align_uint(size, 4); + size += get_func_type_size(comp_ctx, + (AOTFuncType *)comp_data->types[i]); + } + } + + return size; +} + +static uint32 +get_import_global_size(AOTCompContext *comp_ctx, AOTImportGlobal *import_global) +{ + /* type (1 byte) + is_mutable (1 byte) + module_name + global_name */ + uint32 size = (uint32)sizeof(uint8) * 2 + + get_string_size(comp_ctx, import_global->module_name); + size = align_uint(size, 2); + size += get_string_size(comp_ctx, import_global->global_name); + return size; +} + +static uint32 +get_import_globals_size(AOTCompContext *comp_ctx, + AOTImportGlobal *import_globals, + uint32 import_global_count) +{ + AOTImportGlobal *import_global = import_globals; + uint32 size = 0, i; + + for (i = 0; i < import_global_count; i++, import_global++) { + size = align_uint(size, 2); + size += get_import_global_size(comp_ctx, import_global); + } + return size; +} + +static uint32 +get_import_global_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ + /* import global count + import globals */ + return (uint32)sizeof(uint32) + + get_import_globals_size(comp_ctx, comp_data->import_globals, + comp_data->import_global_count); +} + +static uint32 +get_global_size(AOTCompContext *comp_ctx, AOTGlobal *global) +{ + /* type (1 byte) + is_mutable (1 byte) + padding (2 bytes) + + init expr value (include init expr type) */ + return sizeof(uint8) * 2 + sizeof(uint8) * 2 + + get_init_expr_size(comp_ctx, comp_ctx->comp_data, + &global->init_expr); +} + +static uint32 +get_globals_size(AOTCompContext *comp_ctx, AOTGlobal *globals, + uint32 global_count) +{ + AOTGlobal *global = globals; + uint32 size = 0, i; + + for (i = 0; i < global_count; i++, global++) { + size = align_uint(size, 4); + size += get_global_size(comp_ctx, global); + } + return size; +} + +static uint32 +get_global_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ + /* global count + globals */ + return (uint32)sizeof(uint32) + + get_globals_size(comp_ctx, comp_data->globals, + comp_data->global_count); +} + +static uint32 +get_import_func_size(AOTCompContext *comp_ctx, AOTImportFunc *import_func) +{ + /* type index (2 bytes) + module_name + func_name */ + uint32 size = (uint32)sizeof(uint16) + + get_string_size(comp_ctx, import_func->module_name); + size = align_uint(size, 2); + size += get_string_size(comp_ctx, import_func->func_name); + return size; +} + +static uint32 +get_import_funcs_size(AOTCompContext *comp_ctx, AOTImportFunc *import_funcs, + uint32 import_func_count) +{ + AOTImportFunc *import_func = import_funcs; + uint32 size = 0, i; + + for (i = 0; i < import_func_count; i++, import_func++) { + size = align_uint(size, 2); + size += get_import_func_size(comp_ctx, import_func); + } + return size; +} + +static uint32 +get_import_func_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ + /* import func count + import funcs */ + return (uint32)sizeof(uint32) + + get_import_funcs_size(comp_ctx, comp_data->import_funcs, + comp_data->import_func_count); +} + +static uint32 +get_object_data_sections_size(AOTCompContext *comp_ctx, + AOTObjectDataSection *data_sections, + uint32 data_sections_count) +{ + AOTObjectDataSection *data_section = data_sections; + uint32 size = 0, i; + + for (i = 0; i < data_sections_count; i++, data_section++) { + /* name + size + data */ + size = align_uint(size, 2); + size += get_string_size(comp_ctx, data_section->name); + size = align_uint(size, 4); + size += (uint32)sizeof(uint32); + size += data_section->size; + } + return size; +} + +static uint32 +get_object_data_section_info_size(AOTCompContext *comp_ctx, + AOTObjectData *obj_data) +{ + /* data sections count + data sections */ + return (uint32)sizeof(uint32) + + get_object_data_sections_size(comp_ctx, obj_data->data_sections, + obj_data->data_sections_count); +} + +static uint32 +get_init_data_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 size = 0; + + size += get_mem_info_size(comp_ctx, comp_data); + + size = align_uint(size, 4); + size += get_table_info_size(comp_ctx, comp_data); + + size = align_uint(size, 4); + size += get_type_info_size(comp_ctx, comp_data); + + size = align_uint(size, 4); + size += get_import_global_info_size(comp_ctx, comp_data); + + size = align_uint(size, 4); + size += get_global_info_size(comp_ctx, comp_data); + + size = align_uint(size, 4); + size += get_import_func_info_size(comp_ctx, comp_data); + + /* func count + start func index */ + size = align_uint(size, 4); + size += (uint32)sizeof(uint32) * 2; + + /* aux data/heap/stack data */ + size += sizeof(uint32) * 10; + + size += get_object_data_section_info_size(comp_ctx, obj_data); + return size; +} + +static uint32 +get_text_section_size(AOTObjectData *obj_data) +{ + return sizeof(uint32) + align_uint(obj_data->literal_size, 4) + + align_uint(obj_data->text_size, 4) + + align_uint(obj_data->text_unlikely_size, 4) + + align_uint(obj_data->text_hot_size, 4); +} + +static uint32 +get_func_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 size = 0; + + /* text offsets */ + if (is_32bit_binary(obj_data)) + size = (uint32)sizeof(uint32) * comp_data->func_count; + else + size = (uint32)sizeof(uint64) * comp_data->func_count; + + /* function type indexes */ + size += (uint32)sizeof(uint32) * comp_data->func_count; + + /* aot_func#xxx + aot_func_internal#xxx in XIP mode for xtensa */ + if (need_call_wrapped_indirect(obj_data)) + size *= 2; + + /* max_local_cell_nums */ + size += (uint32)sizeof(uint32) * comp_data->func_count; + + /* max_stack_cell_nums */ + size += (uint32)sizeof(uint32) * comp_data->func_count; + +#if WASM_ENABLE_GC != 0 + /* func_local_ref_flags */ + if (comp_ctx->enable_gc) { + AOTFuncType *func_type; + uint32 i, j, local_ref_flags_cell_num; + + for (i = 0; i < comp_data->import_func_count; i++) { + func_type = comp_data->import_funcs[i].func_type; + /* recalculate cell_num based on target pointer size */ + local_ref_flags_cell_num = 0; + for (j = 0; j < func_type->param_count; j++) { + local_ref_flags_cell_num += wasm_value_type_cell_num_internal( + func_type->types[j], comp_ctx->pointer_size); + } + local_ref_flags_cell_num = + local_ref_flags_cell_num > 2 ? local_ref_flags_cell_num : 2; + + size = align_uint(size, 4); + size += (uint32)sizeof(uint32); + size += (uint32)sizeof(uint8) * local_ref_flags_cell_num; + } + + for (i = 0; i < comp_data->func_count; i++) { + func_type = comp_data->funcs[i]->func_type; + local_ref_flags_cell_num = comp_data->funcs[i]->param_cell_num + + comp_data->funcs[i]->local_cell_num; + + size = align_uint(size, 4); + size += (uint32)sizeof(uint32); + size += (uint32)sizeof(uint8) * local_ref_flags_cell_num; + } + } +#endif + + return size; +} + +static uint32 +get_export_size(AOTCompContext *comp_ctx, AOTExport *export) +{ + /* export index + export kind + 1 byte padding + export name */ + return (uint32)sizeof(uint32) + sizeof(uint8) + 1 + + get_string_size(comp_ctx, export->name); +} + +static uint32 +get_exports_size(AOTCompContext *comp_ctx, AOTExport *exports, + uint32 export_count) +{ + AOTExport *export = exports; + uint32 size = 0, i; + + for (i = 0; i < export_count; i++, export ++) { + size = align_uint(size, 4); + size += get_export_size(comp_ctx, export); + } + return size; +} + +static uint32 +get_export_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ + /* export count + exports */ + return (uint32)sizeof(uint32) + + get_exports_size(comp_ctx, comp_data->wasm_module->exports, + comp_data->wasm_module->export_count); +} + +static uint32 +get_relocation_size(AOTRelocation *relocation, bool is_32bin) +{ + /* offset + addend + relocation type + symbol name */ + uint32 size = 0; + if (is_32bin) + size = sizeof(uint32) * 2; /* offset and addend */ + else + size = sizeof(uint64) * 2; /* offset and addend */ + size += (uint32)sizeof(uint32); /* relocation type */ + size += (uint32)sizeof(uint32); /* symbol name index */ + return size; +} + +static uint32 +get_relocations_size(AOTObjectData *obj_data, + AOTRelocationGroup *relocation_group, + AOTRelocation *relocations, uint32 relocation_count, + bool is_32bin) +{ + AOTRelocation *relocation = relocations; + uint32 size = 0, i; + + for (i = 0; i < relocation_count; i++, relocation++) { + /* ignore the relocations to aot_func_internal#n in text section + for windows platform since they will be applied in + aot_emit_text_section */ + if ((!strcmp(relocation_group->section_name, ".text") + || !strcmp(relocation_group->section_name, ".ltext")) + && !strncmp(relocation->symbol_name, AOT_FUNC_INTERNAL_PREFIX, + strlen(AOT_FUNC_INTERNAL_PREFIX)) + && ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6) + /* Windows AOT_COFF64_BIN_TYPE */ + && obj_data->target_info.bin_type == 6 + /* IMAGE_REL_AMD64_REL32 in windows x86_64 */ + && relocation->relocation_type == 4) + || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4) + /* Windows AOT_COFF32_BIN_TYPE */ + && obj_data->target_info.bin_type == 4 + /* IMAGE_REL_I386_REL32 in windows x86_32 */ + && relocation->relocation_type == 20))) { + continue; + } + size = align_uint(size, 4); + size += get_relocation_size(relocation, is_32bin); + } + return size; +} + +static uint32 +get_relocation_group_size(AOTObjectData *obj_data, + AOTRelocationGroup *relocation_group, bool is_32bin) +{ + uint32 size = 0; + /* section name index + relocation count + relocations */ + size += (uint32)sizeof(uint32); + size += (uint32)sizeof(uint32); + size += get_relocations_size(obj_data, relocation_group, + relocation_group->relocations, + relocation_group->relocation_count, is_32bin); + return size; +} + +static uint32 +get_relocation_groups_size(AOTObjectData *obj_data, + AOTRelocationGroup *relocation_groups, + uint32 relocation_group_count, bool is_32bin) +{ + AOTRelocationGroup *relocation_group = relocation_groups; + uint32 size = 0, i; + + for (i = 0; i < relocation_group_count; i++, relocation_group++) { + size = align_uint(size, 4); + size += get_relocation_group_size(obj_data, relocation_group, is_32bin); + } + return size; +} + +/* return the index (in order of insertion) of the symbol, + create if not exits, -1 if failed */ +static uint32 +get_relocation_symbol_index(const char *symbol_name, bool *is_new, + AOTSymbolList *symbol_list) +{ + AOTSymbolNode *sym; + uint32 index = 0; + + sym = symbol_list->head; + while (sym) { + if (!strcmp(sym->symbol, symbol_name)) { + if (is_new) + *is_new = false; + return index; + } + + sym = sym->next; + index++; + } + + /* Not found in symbol_list, add it */ + sym = wasm_runtime_malloc(sizeof(AOTSymbolNode)); + if (!sym) { + return (uint32)-1; + } + + memset(sym, 0, sizeof(AOTSymbolNode)); + sym->symbol = (char *)symbol_name; + sym->str_len = (uint32)strlen(symbol_name); + + if (!symbol_list->head) { + symbol_list->head = symbol_list->end = sym; + } + else { + symbol_list->end->next = sym; + symbol_list->end = sym; + } + symbol_list->len++; + + if (is_new) + *is_new = true; + return index; +} + +static uint32 +get_relocation_symbol_size(AOTCompContext *comp_ctx, AOTRelocation *relocation, + AOTSymbolList *symbol_list) +{ + uint32 size = 0, index = 0; + bool is_new = false; + + index = get_relocation_symbol_index(relocation->symbol_name, &is_new, + symbol_list); + CHECK_SIZE(index); + + if (is_new) { + size += get_string_size(comp_ctx, relocation->symbol_name); + size = align_uint(size, 2); + } + + relocation->symbol_index = index; + return size; +} + +static uint32 +get_relocations_symbol_size(AOTCompContext *comp_ctx, + AOTRelocation *relocations, uint32 relocation_count, + AOTSymbolList *symbol_list) +{ + AOTRelocation *relocation = relocations; + uint32 size = 0, curr_size, i; + + for (i = 0; i < relocation_count; i++, relocation++) { + curr_size = + get_relocation_symbol_size(comp_ctx, relocation, symbol_list); + CHECK_SIZE(curr_size); + + size += curr_size; + } + return size; +} + +static uint32 +get_relocation_group_symbol_size(AOTCompContext *comp_ctx, + AOTRelocationGroup *relocation_group, + AOTSymbolList *symbol_list) +{ + uint32 size = 0, index = 0, curr_size; + bool is_new = false; + + index = get_relocation_symbol_index(relocation_group->section_name, &is_new, + symbol_list); + CHECK_SIZE(index); + + if (is_new) { + size += get_string_size(comp_ctx, relocation_group->section_name); + size = align_uint(size, 2); + } + + relocation_group->name_index = index; + + curr_size = get_relocations_symbol_size( + comp_ctx, relocation_group->relocations, + relocation_group->relocation_count, symbol_list); + CHECK_SIZE(curr_size); + size += curr_size; + + return size; +} + +static uint32 +get_relocation_groups_symbol_size(AOTCompContext *comp_ctx, + AOTRelocationGroup *relocation_groups, + uint32 relocation_group_count, + AOTSymbolList *symbol_list) +{ + AOTRelocationGroup *relocation_group = relocation_groups; + uint32 size = 0, curr_size, i; + + for (i = 0; i < relocation_group_count; i++, relocation_group++) { + curr_size = get_relocation_group_symbol_size(comp_ctx, relocation_group, + symbol_list); + CHECK_SIZE(curr_size); + size += curr_size; + } + return size; +} + +static uint32 +get_symbol_size_from_symbol_list(AOTCompContext *comp_ctx, + AOTSymbolList *symbol_list) +{ + AOTSymbolNode *sym; + uint32 size = 0; + + sym = symbol_list->head; + while (sym) { + /* (uint16)str_len + str */ + size += get_string_size(comp_ctx, sym->symbol); + size = align_uint(size, 2); + sym = sym->next; + } + + return size; +} + +static uint32 +get_relocation_section_symbol_size(AOTCompContext *comp_ctx, + AOTObjectData *obj_data) +{ + AOTRelocationGroup *relocation_groups = obj_data->relocation_groups; + uint32 relocation_group_count = obj_data->relocation_group_count; + uint32 string_count = 0, symbol_table_size = 0; + + /* section size will be calculated twice, + get symbol size from symbol list directly in the second calculation */ + if (obj_data->symbol_list.len > 0) { + symbol_table_size = + get_symbol_size_from_symbol_list(comp_ctx, &obj_data->symbol_list); + } + else { + symbol_table_size = get_relocation_groups_symbol_size( + comp_ctx, relocation_groups, relocation_group_count, + &obj_data->symbol_list); + } + CHECK_SIZE(symbol_table_size); + string_count = obj_data->symbol_list.len; + + /* string_count + string_offsets + total_string_len + + [str (string_len + str)] */ + return (uint32)(sizeof(uint32) + sizeof(uint32) * string_count + + sizeof(uint32) + symbol_table_size); +} + +static uint32 +get_relocation_section_size(AOTCompContext *comp_ctx, AOTObjectData *obj_data) +{ + AOTRelocationGroup *relocation_groups = obj_data->relocation_groups; + uint32 relocation_group_count = obj_data->relocation_group_count; + uint32 symbol_table_size = 0; + + symbol_table_size = get_relocation_section_symbol_size(comp_ctx, obj_data); + CHECK_SIZE(symbol_table_size); + symbol_table_size = align_uint(symbol_table_size, 4); + + /* relocation group count + symbol_table + relocation groups */ + return (uint32)sizeof(uint32) + symbol_table_size + + get_relocation_groups_size(obj_data, relocation_groups, + relocation_group_count, + is_32bit_binary(obj_data)); +} + +static uint32 +get_native_symbol_list_size(AOTCompContext *comp_ctx) +{ + uint32 len = 0; + AOTNativeSymbol *sym = NULL; + + sym = bh_list_first_elem(&comp_ctx->native_symbols); + + while (sym) { + len = align_uint(len, 2); + len += get_string_size(comp_ctx, sym->symbol); + sym = bh_list_elem_next(sym); + } + + return len; +} + +#if WASM_ENABLE_STRINGREF != 0 +static uint32 +get_string_literal_section_size(AOTCompContext *comp_ctx, + AOTCompData *comp_data); +#endif + +static uint32 +get_custom_sections_size(AOTCompContext *comp_ctx, AOTCompData *comp_data); + +static uint32 +get_aot_file_size(AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 size = 0; + uint32 size_custom_section = 0; +#if WASM_ENABLE_STRINGREF != 0 + uint32 size_string_literal_section = 0; +#endif + + /* aot file header */ + size += get_file_header_size(); + + /* target info section */ + size = align_uint(size, 4); + /* section id + section size */ + size += (uint32)sizeof(uint32) * 2; + size += get_target_info_section_size(); + + /* init data section */ + size = align_uint(size, 4); + /* section id + section size */ + size += (uint32)sizeof(uint32) * 2; + size += get_init_data_section_size(comp_ctx, comp_data, obj_data); + + /* text section */ + size = align_uint(size, 4); + /* section id + section size */ + size += (uint32)sizeof(uint32) * 2; + size += get_text_section_size(obj_data); + + /* function section */ + size = align_uint(size, 4); + /* section id + section size */ + size += (uint32)sizeof(uint32) * 2; + size += get_func_section_size(comp_ctx, comp_data, obj_data); + + /* export section */ + size = align_uint(size, 4); + /* section id + section size */ + size += (uint32)sizeof(uint32) * 2; + size += get_export_section_size(comp_ctx, comp_data); + + /* relocation section */ + size = align_uint(size, 4); + /* section id + section size */ + size += (uint32)sizeof(uint32) * 2; + size += get_relocation_section_size(comp_ctx, obj_data); + + if (get_native_symbol_list_size(comp_ctx) > 0) { + /* emit only when there are native symbols */ + size = align_uint(size, 4); + /* section id + section size + sub section id + symbol count */ + size += (uint32)sizeof(uint32) * 4; + size += get_native_symbol_list_size(comp_ctx); + } + + size_custom_section = get_custom_sections_size(comp_ctx, comp_data); + if (size_custom_section > 0) { + size = align_uint(size, 4); + size += size_custom_section; + } + +#if WASM_ENABLE_STRINGREF != 0 + /* string literal section */ + size_string_literal_section = + get_string_literal_section_size(comp_ctx, comp_data); + if (size_string_literal_section > 0) { + size = align_uint(size, 4); + /* section id + section size + sub section id */ + size += (uint32)sizeof(uint32) * 3; + size += size_string_literal_section; + } +#endif + + return size; +} + +#define exchange_uint8(p_data) (void)0 + +static void +exchange_uint16(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 1); + *(p_data + 1) = value; +} + +static void +exchange_uint32(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 3); + *(p_data + 3) = value; + + value = *(p_data + 1); + *(p_data + 1) = *(p_data + 2); + *(p_data + 2) = value; +} + +static void +exchange_uint64(uint8 *p_data) +{ + uint32 value; + + value = *(uint32 *)p_data; + *(uint32 *)p_data = *(uint32 *)(p_data + 4); + *(uint32 *)(p_data + 4) = value; + exchange_uint32(p_data); + exchange_uint32(p_data + 4); +} + +static void +exchange_uint128(uint8 *p_data) +{ + /* swap high 64bit and low 64bit */ + uint64 value = *(uint64 *)p_data; + *(uint64 *)p_data = *(uint64 *)(p_data + 8); + *(uint64 *)(p_data + 8) = value; + /* exchange high 64bit */ + exchange_uint64(p_data); + /* exchange low 64bit */ + exchange_uint64(p_data + 8); +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +#define CHECK_BUF(length) \ + do { \ + if (buf + offset + length > buf_end) { \ + aot_set_last_error("buf overflow"); \ + return false; \ + } \ + } while (0) + +#define EMIT_U8(v) \ + do { \ + CHECK_BUF(1); \ + *(uint8 *)(buf + offset) = (uint8)v; \ + offset++; \ + } while (0) + +#define EMIT_U16(v) \ + do { \ + uint16 t = (uint16)v; \ + CHECK_BUF(2); \ + if (!is_little_endian()) \ + exchange_uint16((uint8 *)&t); \ + *(uint16 *)(buf + offset) = t; \ + offset += (uint32)sizeof(uint16); \ + } while (0) + +#define EMIT_U32(v) \ + do { \ + uint32 t = (uint32)v; \ + CHECK_BUF(4); \ + if (!is_little_endian()) \ + exchange_uint32((uint8 *)&t); \ + *(uint32 *)(buf + offset) = t; \ + offset += (uint32)sizeof(uint32); \ + } while (0) + +#define EMIT_U64(v) \ + do { \ + uint64 t = (uint64)v; \ + CHECK_BUF(8); \ + if (!is_little_endian()) \ + exchange_uint64((uint8 *)&t); \ + PUT_U64_TO_ADDR(buf + offset, t); \ + offset += (uint32)sizeof(uint64); \ + } while (0) + +#define EMIT_V128(v) \ + do { \ + uint64 *t = (uint64 *)v.i64x2; \ + CHECK_BUF(16); \ + if (!is_little_endian()) \ + exchange_uint128((uint8 *)t); \ + PUT_U64_TO_ADDR(buf + offset, t[0]); \ + offset += (uint32)sizeof(uint64); \ + PUT_U64_TO_ADDR(buf + offset, t[1]); \ + offset += (uint32)sizeof(uint64); \ + } while (0) + +#define EMIT_BUF(v, len) \ + do { \ + CHECK_BUF(len); \ + memcpy(buf + offset, v, len); \ + offset += len; \ + } while (0) + +/* Emit string with '\0' + */ +#define EMIT_STR(s) \ + do { \ + uint32 str_len = (uint32)strlen(s) + 1; \ + if (str_len > INT16_MAX) { \ + aot_set_last_error("emit string failed: " \ + "string too long"); \ + return false; \ + } \ + EMIT_U16(str_len); \ + EMIT_BUF(s, str_len); \ + } while (0) + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +static bool +read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, + uint64 *p_result) +{ + const uint8 *buf = *p_buf; + uint64 result = 0; + uint32 shift = 0; + uint32 offset = 0, bcnt = 0; + uint64 byte; + + while (true) { + /* uN or SN must not exceed ceil(N/7) bytes */ + if (bcnt + 1 > (maxbits + 6) / 7) { + aot_set_last_error("integer representation too long"); + return false; + } + + if (buf + offset + 1 > buf_end) { + aot_set_last_error("unexpected end of section or function"); + return false; + } + byte = buf[offset]; + offset += 1; + result |= ((byte & 0x7f) << shift); + shift += 7; + bcnt += 1; + if ((byte & 0x80) == 0) { + break; + } + } + + if (!sign && maxbits == 32 && shift >= maxbits) { + /* The top bits set represent values > 32 bits */ + if (((uint8)byte) & 0xf0) + goto fail_integer_too_large; + } + else if (sign && maxbits == 32) { + if (shift < maxbits) { + /* Sign extend, second highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x8; + int top_bits = ((uint8)byte) & 0xf0; + if ((sign_bit_set && top_bits != 0x70) + || (!sign_bit_set && top_bits != 0)) + goto fail_integer_too_large; + } + } + else if (sign && maxbits == 64) { + if (shift < maxbits) { + /* Sign extend, second highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x1; + int top_bits = ((uint8)byte) & 0xfe; + + if ((sign_bit_set && top_bits != 0x7e) + || (!sign_bit_set && top_bits != 0)) + goto fail_integer_too_large; + } + } + + *p_buf += offset; + *p_result = result; + return true; + +fail_integer_too_large: + aot_set_last_error("integer too large"); + return false; +} + +/* NOLINTNEXTLINE */ +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, 32, false, &res64)) \ + goto fail; \ + res = (uint32)res64; \ + } while (0) + +/* + * - transfer .name section in .wasm (comp_data->name_section_buf) to + * aot buf (comp_data->aot_name_section_buf) + * - leb128 to u32 + * - add `\0` at the end of every name, and adjust length(+1) + */ +static uint32 +get_name_section_size(AOTCompData *comp_data) +{ + /* original name section content in .wasm */ + const uint8 *p = comp_data->name_section_buf, + *p_end = comp_data->name_section_buf_end; + uint8 *buf, *buf_end; + uint32 name_type, subsection_size; + uint32 previous_name_type = 0; + uint32 num_func_name; + uint32 func_index; + uint32 previous_func_index = ~0U; + uint32 func_name_len; + uint32 name_index; + int i = 0; + uint32 name_len; + uint32 offset = 0; + uint32 max_aot_buf_size = 0; + + if (p >= p_end) { + aot_set_last_error("unexpected end"); + return 0; + } + + max_aot_buf_size = 4 * (uint32)(p_end - p); + if (!(buf = comp_data->aot_name_section_buf = + wasm_runtime_malloc(max_aot_buf_size))) { + aot_set_last_error("allocate memory for custom name section failed."); + return 0; + } + memset(buf, 0, (uint32)max_aot_buf_size); + buf_end = buf + max_aot_buf_size; + + /* the size of "name". it should be 4 */ + read_leb_uint32(p, p_end, name_len); + offset = align_uint(offset, 4); + EMIT_U32(name_len); + + if (name_len != 4 || p + name_len > p_end) { + aot_set_last_error("unexpected end"); + return 0; + } + + /* "name" */ + if (memcmp(p, "name", 4) != 0) { + aot_set_last_error("invalid custom name section"); + return 0; + } + EMIT_BUF(p, name_len); + p += name_len; + + while (p < p_end) { + read_leb_uint32(p, p_end, name_type); + if (i != 0) { + if (name_type == previous_name_type) { + aot_set_last_error("duplicate sub-section"); + return 0; + } + if (name_type < previous_name_type) { + aot_set_last_error("out-of-order sub-section"); + return 0; + } + } + previous_name_type = name_type; + read_leb_uint32(p, p_end, subsection_size); + switch (name_type) { + case SUB_SECTION_TYPE_FUNC: + if (subsection_size) { + offset = align_uint(offset, 4); + EMIT_U32(name_type); + EMIT_U32(subsection_size); + + read_leb_uint32(p, p_end, num_func_name); + EMIT_U32(num_func_name); + + for (name_index = 0; name_index < num_func_name; + name_index++) { + read_leb_uint32(p, p_end, func_index); + offset = align_uint(offset, 4); + EMIT_U32(func_index); + if (func_index == previous_func_index) { + aot_set_last_error("duplicate function name"); + return 0; + } + if (func_index < previous_func_index + && previous_func_index != ~0U) { + aot_set_last_error("out-of-order function index "); + return 0; + } + previous_func_index = func_index; + read_leb_uint32(p, p_end, func_name_len); + offset = align_uint(offset, 2); + + /* emit a string ends with `\0` */ + if (func_name_len + 1 > UINT16_MAX) { + aot_set_last_error( + "emit string failed: string too long"); + goto fail; + } + /* extra 1 byte for \0 */ + EMIT_U16(func_name_len + 1); + EMIT_BUF(p, func_name_len); + p += func_name_len; + EMIT_U8(0); + } + } + break; + case SUB_SECTION_TYPE_MODULE: /* TODO: Parse for module subsection + */ + case SUB_SECTION_TYPE_LOCAL: /* TODO: Parse for local subsection */ + default: + p = p + subsection_size; + break; + } + i++; + } + + return offset; +fail: + return 0; +} +#endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 */ + +#if WASM_ENABLE_STRINGREF != 0 +static uint32 +get_string_literal_section_size(AOTCompContext *comp_ctx, + AOTCompData *comp_data) +{ + uint32 i; + uint32 size = 0; + uint32 string_count = comp_data->string_literal_count; + + if (string_count == 0) { + return 0; + } + + /* reserved slot + string count + string_lengths */ + size += sizeof(uint32) * (2 + string_count); + + for (i = 0; i < string_count; i++) { + size += comp_data->string_literal_lengths_wp[i]; + } + + return size; +} +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + +static uint32 +get_custom_sections_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) +{ +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + uint32 size = 0, i; + + for (i = 0; i < comp_ctx->custom_sections_count; i++) { + const char *section_name = comp_ctx->custom_sections_wp[i]; + const uint8 *content = NULL; + uint32 length = 0; + + if (strcmp(section_name, "name") == 0) { + /* custom name section */ + comp_data->aot_name_section_size = get_name_section_size(comp_data); + if (comp_data->aot_name_section_size == 0) { + LOG_WARNING("Can't find custom section [name], ignore it"); + continue; + } + + size = align_uint(size, 4); + /* section id + section size + sub section id */ + size += (uint32)sizeof(uint32) * 3; + size += comp_data->aot_name_section_size; + continue; + } + + content = wasm_loader_get_custom_section(comp_data->wasm_module, + section_name, &length); + if (!content) { + LOG_WARNING("Can't find custom section [%s], ignore it", + section_name); + continue; + } + + size = align_uint(size, 4); + /* section id + section size + sub section id */ + size += (uint32)sizeof(uint32) * 3; + /* section name and len */ + size += get_string_size(comp_ctx, section_name); + /* section content */ + size += length; + } + + return size; +#else + return 0; +#endif +} + +static bool +aot_emit_file_header(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompData *comp_data, AOTObjectData *obj_data) +{ + uint32 offset = *p_offset; + uint32 aot_curr_version = AOT_CURRENT_VERSION; + + EMIT_U8('\0'); + EMIT_U8('a'); + EMIT_U8('o'); + EMIT_U8('t'); + + EMIT_U32(aot_curr_version); + + *p_offset = offset; + return true; +} + +static bool +aot_emit_target_info_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompData *comp_data, AOTObjectData *obj_data) +{ + uint32 offset = *p_offset; + uint32 section_size = get_target_info_section_size(); + AOTTargetInfo *target_info = &obj_data->target_info; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_TARGET_INFO); + EMIT_U32(section_size); + + EMIT_U16(target_info->bin_type); + EMIT_U16(target_info->abi_type); + EMIT_U16(target_info->e_type); + EMIT_U16(target_info->e_machine); + EMIT_U32(target_info->e_version); + EMIT_U32(target_info->e_flags); + EMIT_U64(target_info->feature_flags); + EMIT_U64(target_info->reserved); + EMIT_BUF(target_info->arch, sizeof(target_info->arch)); + + if (offset - *p_offset != section_size + sizeof(uint32) * 2) { + aot_set_last_error("emit target info failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, InitializerExpression *expr); + +static bool +aot_emit_mem_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + AOTMemInitData **init_datas = comp_data->mem_init_data_list; + + *p_offset = offset = align_uint(offset, 4); + + /* Emit import memory count, only emit 0 currently. + TODO: emit the actual import memory count and + the full import memory info. */ + EMIT_U32(0); + + /* Emit memory count */ + EMIT_U32(comp_data->memory_count); + /* Emit memory items */ + for (i = 0; i < comp_data->memory_count; i++) { + EMIT_U32(comp_data->memories[i].memory_flags); + EMIT_U32(comp_data->memories[i].num_bytes_per_page); + EMIT_U32(comp_data->memories[i].mem_init_page_count); + EMIT_U32(comp_data->memories[i].mem_max_page_count); + } + + /* Emit mem init data count */ + EMIT_U32(comp_data->mem_init_data_count); + /* Emit mem init data items */ + for (i = 0; i < comp_data->mem_init_data_count; i++) { + offset = align_uint(offset, 4); +#if WASM_ENABLE_BULK_MEMORY != 0 + if (comp_ctx->enable_bulk_memory) { + EMIT_U32(init_datas[i]->is_passive); + EMIT_U32(init_datas[i]->memory_index); + } + else +#endif + { + /* emit two placeholder to keep the same size */ + EMIT_U32(0); + EMIT_U32(0); + } + if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx, + &init_datas[i]->offset)) + return false; + EMIT_U32(init_datas[i]->byte_count); + EMIT_BUF(init_datas[i]->bytes, init_datas[i]->byte_count); + } + + if (offset - *p_offset != get_mem_info_size(comp_ctx, comp_data)) { + aot_set_last_error("emit memory info failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, InitializerExpression *expr) +{ + uint32 offset = *p_offset; +#if WASM_ENABLE_GC != 0 + WASMModule *module = comp_ctx->comp_data->wasm_module; +#endif + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(expr->init_expr_type); + switch (expr->init_expr_type) { + case INIT_EXPR_NONE: + break; + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_F32_CONST: + EMIT_U32(expr->u.i32); + break; + case INIT_EXPR_TYPE_I64_CONST: + case INIT_EXPR_TYPE_F64_CONST: + EMIT_U64(expr->u.i64); + break; + case INIT_EXPR_TYPE_V128_CONST: + EMIT_V128(expr->u.v128); + break; + case INIT_EXPR_TYPE_GET_GLOBAL: + EMIT_U32(expr->u.global_index); + break; + case INIT_EXPR_TYPE_FUNCREF_CONST: + case INIT_EXPR_TYPE_REFNULL_CONST: + EMIT_U32(expr->u.ref_index); + break; +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_I31_NEW: + EMIT_U32(expr->u.i32); + break; + case INIT_EXPR_TYPE_STRUCT_NEW: + { + uint32 i; + WASMStructNewInitValues *init_values = + (WASMStructNewInitValues *)expr->u.data; + WASMStructType *struct_type = NULL; + + EMIT_U32(init_values->type_idx); + EMIT_U32(init_values->count); + + bh_assert(init_values->type_idx < module->type_count); + + struct_type = + (WASMStructType *)module->types[init_values->type_idx]; + + bh_assert(struct_type); + bh_assert(struct_type->field_count == init_values->count); + + for (i = 0; i < init_values->count; i++) { + uint32 field_size = wasm_value_type_size_internal( + struct_type->fields[i].field_type, comp_ctx->pointer_size); + if (field_size <= sizeof(uint32)) + EMIT_U32(init_values->fields[i].u32); + else if (field_size == sizeof(uint64)) + EMIT_U64(init_values->fields[i].u64); + else if (field_size == sizeof(uint64) * 2) + EMIT_V128(init_values->fields[i].v128); + else { + bh_assert(0); + } + } + + break; + } + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + EMIT_U32(expr->u.type_index); + break; + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + { + WASMArrayType *array_type = NULL; + + bh_assert(expr->u.array_new_default.type_index + < module->type_count); + array_type = + (WASMArrayType *) + module->types[expr->u.array_new_default.type_index]; + + EMIT_U32(array_type->elem_type); + EMIT_U32(expr->u.array_new_default.type_index); + EMIT_U32(expr->u.array_new_default.length); + break; + } + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + uint32 value_count, i, field_size; + WASMArrayNewInitValues *init_values = + (WASMArrayNewInitValues *)expr->u.data; + WASMArrayType *array_type = NULL; + + bh_assert(init_values->type_idx < module->type_count); + array_type = (WASMArrayType *)module->types[init_values->type_idx]; + + EMIT_U32(array_type->elem_type); + EMIT_U32(init_values->type_idx); + EMIT_U32(init_values->length); + + value_count = + (expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) + ? init_values->length + : 1; + + field_size = wasm_value_type_size_internal(array_type->elem_type, + comp_ctx->pointer_size); + + for (i = 0; i < value_count; i++) { + if (field_size <= sizeof(uint32)) + EMIT_U32(init_values->elem_data[i].u32); + else if (field_size == sizeof(uint64)) + EMIT_U64(init_values->elem_data[i].u64); + else if (field_size == sizeof(uint64) * 2) + EMIT_V128(init_values->elem_data[i].v128); + else { + bh_assert(0); + } + } + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + aot_set_last_error("invalid init expr type."); + return false; + } + + *p_offset = offset; + return true; +} + +static bool +aot_emit_table_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i, j; + AOTTableInitData **init_datas = comp_data->table_init_data_list; + + *p_offset = offset = align_uint(offset, 4); + + /* Emit import table count */ + EMIT_U32(comp_data->import_table_count); + /* Emit table items */ + for (i = 0; i < comp_data->import_table_count; i++) { + /* TODO: + * EMIT_STR(comp_data->import_tables[i].module_name ); + * EMIT_STR(comp_data->import_tables[i].table_name); + */ + EMIT_U8(comp_data->import_tables[i].elem_type); + EMIT_U8(comp_data->import_tables[i].table_flags); + EMIT_U8(comp_data->import_tables[i].possible_grow); +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc && comp_data->import_tables[i].elem_ref_type) { + EMIT_U8(comp_data->import_tables[i] + .elem_ref_type->ref_ht_common.nullable); + } + else +#endif + { + /* emit one placeholder to keep the same size */ + EMIT_U8(0); + } + EMIT_U32(comp_data->import_tables[i].table_init_size); + EMIT_U32(comp_data->import_tables[i].table_max_size); +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc && comp_data->import_tables[i].elem_ref_type) { + bh_assert(wasm_is_type_multi_byte_type( + comp_data->import_tables[i].elem_type)); + EMIT_U32(comp_data->import_tables[i] + .elem_ref_type->ref_ht_common.heap_type); + } +#endif + } + + /* Emit table count */ + EMIT_U32(comp_data->table_count); + /* Emit table items */ + for (i = 0; i < comp_data->table_count; i++) { + EMIT_U8(comp_data->tables[i].elem_type); + EMIT_U8(comp_data->tables[i].table_flags); + EMIT_U8(comp_data->tables[i].possible_grow); +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc && comp_data->tables[i].elem_ref_type) { + EMIT_U8(comp_data->tables[i].elem_ref_type->ref_ht_common.nullable); + } + else +#endif + { + /* emit one placeholder to keep the same size */ + EMIT_U8(0); + } + EMIT_U32(comp_data->tables[i].table_init_size); + EMIT_U32(comp_data->tables[i].table_max_size); +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + if (comp_data->tables[i].elem_ref_type) { + bh_assert(wasm_is_type_multi_byte_type( + comp_data->tables[i].elem_type)); + EMIT_U32(comp_data->tables[i] + .elem_ref_type->ref_ht_common.heap_type); + } + if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx, + &comp_data->tables[i].init_expr)) { + return false; + } + } +#endif + } + + /* Emit table init data count */ + EMIT_U32(comp_data->table_init_data_count); + /* Emit table init data items */ + for (i = 0; i < comp_data->table_init_data_count; i++) { + offset = align_uint(offset, 4); + EMIT_U32(init_datas[i]->mode); + EMIT_U32(init_datas[i]->elem_type); + EMIT_U32(init_datas[i]->table_index); + EMIT_U32(init_datas[i]->offset.init_expr_type); + EMIT_U64(init_datas[i]->offset.u.i64); +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc && init_datas[i]->elem_ref_type) { + EMIT_U16(init_datas[i]->elem_ref_type->ref_ht_common.ref_type); + EMIT_U16(init_datas[i]->elem_ref_type->ref_ht_common.nullable); + EMIT_U32(init_datas[i]->elem_ref_type->ref_ht_common.heap_type); + } + else +#endif + { + EMIT_U16(init_datas[i]->elem_type); + EMIT_U16(0); + EMIT_U32(0); + } + EMIT_U32(init_datas[i]->value_count); + for (j = 0; j < init_datas[i]->value_count; j++) { + if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx, + &init_datas[i]->init_values[j])) + return false; + } + } + + if (offset - *p_offset != get_table_info_size(comp_ctx, comp_data)) { + aot_set_last_error("emit table info failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +#if WASM_ENABLE_GC != 0 +static bool +aot_emit_reftype_map(uint8 *buf, uint8 *buf_end, uint32 *p_offset, uint32 count, + WASMRefTypeMap *refmap) +{ + uint32 offset = *p_offset, i; + + for (i = 0; i < count; i++) { + EMIT_U16(refmap->index); + WASMRefType *ref_type = refmap->ref_type; + + /* Note: WASMRefType is a union type */ + EMIT_U8(ref_type->ref_ht_common.ref_type); + EMIT_U8(ref_type->ref_ht_common.nullable); + EMIT_U32(ref_type->ref_ht_common.heap_type); + + refmap++; + } + + *p_offset = offset; + return true; +} +#endif + +static bool +aot_emit_type_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(comp_data->type_count); + +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + AOTType **types = comp_data->types; + int32 idx; + uint32 j; + + for (i = 0; i < comp_data->type_count; i++) { + offset = align_uint(offset, 4); + + /* Emit simple info if there is an equivalence type */ + for (j = 0; j < i; j++) { + if (types[j] == types[i]) { + EMIT_U16(types[i]->type_flag); + /* equivalence type flag is true */ + EMIT_U8(1); + EMIT_U8(0); + /* equivalence type index */ + EMIT_U32(j); + break; + } + } + if (j < i) + continue; + + EMIT_U16(types[i]->type_flag); + /* equivalence type flag is false */ + EMIT_U8(0); + EMIT_U8(types[i]->is_sub_final); + EMIT_U32(types[i]->parent_type_idx); + + EMIT_U16(types[i]->rec_count); + EMIT_U16(types[i]->rec_idx); + + /* Emit WASM_TYPE_FUNC */ + if (types[i]->type_flag == WASM_TYPE_FUNC) { + AOTFuncType *func_type = (AOTFuncType *)types[i]; + EMIT_U16(func_type->param_count); + EMIT_U16(func_type->result_count); + EMIT_U16(func_type->ref_type_map_count); + EMIT_BUF(func_type->types, + func_type->param_count + func_type->result_count); + + offset = align_uint(offset, 4); + + aot_emit_reftype_map(buf, buf_end, &offset, + func_type->ref_type_map_count, + func_type->ref_type_maps); + } + /* Emit WASM_TYPE_STRUCT */ + else if (types[i]->type_flag == WASM_TYPE_STRUCT) { + AOTStructType *struct_type = (AOTStructType *)types[i]; + EMIT_U16(struct_type->field_count); + EMIT_U16(struct_type->ref_type_map_count); + + for (idx = 0; idx < struct_type->field_count; idx++) { + EMIT_U8(struct_type->fields[idx].field_flags); + EMIT_U8(struct_type->fields[idx].field_type); + } + + offset = align_uint(offset, 4); + + aot_emit_reftype_map(buf, buf_end, &offset, + struct_type->ref_type_map_count, + struct_type->ref_type_maps); + } + /* Emit WASM_TYPE_ARRAY */ + else if (types[i]->type_flag == WASM_TYPE_ARRAY) { + AOTArrayType *array_type = (AOTArrayType *)types[i]; + EMIT_U16(array_type->elem_flags); + EMIT_U8(array_type->elem_type); + if (array_type->elem_ref_type) { + bh_assert( + wasm_is_type_multi_byte_type(array_type->elem_type)); + EMIT_U8(array_type->elem_ref_type->ref_ht_common.nullable); + EMIT_U32( + array_type->elem_ref_type->ref_ht_common.heap_type); + } + } + else { + aot_set_last_error("invalid type flag."); + return false; + } + } + + if (offset - *p_offset != get_type_info_size(comp_ctx, comp_data)) { + aot_set_last_error("emit function type info failed."); + return false; + } + + *p_offset = offset; + } + else +#endif + { + AOTFuncType **func_types = (AOTFuncType **)comp_data->types; + + for (i = 0; i < comp_data->type_count; i++) { + offset = align_uint(offset, 4); + /* If GC is disabled, only emit function type info */ + EMIT_U16(WASM_TYPE_FUNC); + /* Omit to emit dummy padding for is_sub_final, + * parent_type_index, rec_count, rec_idx, 10 bytes in total */ + EMIT_U16(func_types[i]->param_count); + EMIT_U16(func_types[i]->result_count); + /* Omit to emit dummy padding for ref_type_map_count, 2 bytes in + * total */ + EMIT_BUF(func_types[i]->types, + func_types[i]->param_count + func_types[i]->result_count); + } + + if (offset - *p_offset != get_type_info_size(comp_ctx, comp_data)) { + aot_set_last_error("emit function type info failed."); + return false; + } + + *p_offset = offset; + } + + return true; +} + +static bool +aot_emit_import_global_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + AOTImportGlobal *import_global = comp_data->import_globals; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(comp_data->import_global_count); + + for (i = 0; i < comp_data->import_global_count; i++, import_global++) { + offset = align_uint(offset, 2); + EMIT_U8(import_global->type); + EMIT_U8(import_global->is_mutable); + EMIT_STR(import_global->module_name); + offset = align_uint(offset, 2); + EMIT_STR(import_global->global_name); + } + + if (offset - *p_offset + != get_import_global_info_size(comp_ctx, comp_data)) { + aot_set_last_error("emit import global info failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_global_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + AOTGlobal *global = comp_data->globals; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(comp_data->global_count); + + for (i = 0; i < comp_data->global_count; i++, global++) { + offset = align_uint(offset, 4); + EMIT_U8(global->type); + EMIT_U8(global->is_mutable); + + offset = align_uint(offset, 4); + if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx, + &global->init_expr)) + return false; + } + + if (offset - *p_offset != get_global_info_size(comp_ctx, comp_data)) { + aot_set_last_error("emit global info failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_import_func_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + AOTImportFunc *import_func = comp_data->import_funcs; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(comp_data->import_func_count); + + for (i = 0; i < comp_data->import_func_count; i++, import_func++) { + offset = align_uint(offset, 2); + EMIT_U16(import_func->func_type_index); + EMIT_STR(import_func->module_name); + offset = align_uint(offset, 2); + EMIT_STR(import_func->func_name); + } + + if (offset - *p_offset != get_import_func_info_size(comp_ctx, comp_data)) { + aot_set_last_error("emit import function info failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_object_data_section_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + AOTObjectDataSection *data_section = obj_data->data_sections; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(obj_data->data_sections_count); + + for (i = 0; i < obj_data->data_sections_count; i++, data_section++) { + offset = align_uint(offset, 2); + EMIT_STR(data_section->name); + offset = align_uint(offset, 4); + EMIT_U32(data_section->size); + if (obj_data->stack_sizes_section_name != NULL + && !strcmp(obj_data->stack_sizes_section_name, + data_section->name)) { + uint32 ss_offset = obj_data->stack_sizes_offset; + uint32 ss_size = + obj_data->func_count * sizeof(*obj_data->stack_sizes); + LOG_VERBOSE("Replacing stack_sizes in %s section, offset %" PRIu32 + ", size %" PRIu32, + obj_data->stack_sizes_section_name, ss_offset, ss_size); + bh_assert(ss_offset + ss_size <= data_section->size); + /* 0 .. ss_offset */ + if (ss_offset > 0) { + EMIT_BUF(data_section->data, ss_offset); + } + /* ss_offset .. ss_offset+ss_size */ + EMIT_BUF(obj_data->stack_sizes, ss_size); + /* ss_offset+ss_size .. data_section->size */ + if (data_section->size > ss_offset + ss_size) { + EMIT_BUF(data_section->data + ss_offset + ss_size, + data_section->size - (ss_offset + ss_size)); + } + } + else { + EMIT_BUF(data_section->data, data_section->size); + } + } + + if (offset - *p_offset + != get_object_data_section_info_size(comp_ctx, obj_data)) { + aot_set_last_error("emit object data section info failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_init_data_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 section_size = + get_init_data_section_size(comp_ctx, comp_data, obj_data); + uint32 offset = *p_offset; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_INIT_DATA); + EMIT_U32(section_size); + + if (!aot_emit_mem_info(buf, buf_end, &offset, comp_ctx, comp_data, obj_data) + || !aot_emit_table_info(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data) + || !aot_emit_type_info(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data) + || !aot_emit_import_global_info(buf, buf_end, &offset, comp_ctx, + comp_data, obj_data) + || !aot_emit_global_info(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data) + || !aot_emit_import_func_info(buf, buf_end, &offset, comp_ctx, + comp_data, obj_data)) + return false; + + offset = align_uint(offset, 4); + EMIT_U32(comp_data->func_count); + EMIT_U32(comp_data->start_func_index); + + EMIT_U32(comp_data->aux_data_end_global_index); + EMIT_U64(comp_data->aux_data_end); + EMIT_U32(comp_data->aux_heap_base_global_index); + EMIT_U64(comp_data->aux_heap_base); + EMIT_U32(comp_data->aux_stack_top_global_index); + EMIT_U64(comp_data->aux_stack_bottom); + EMIT_U32(comp_data->aux_stack_size); + + if (!aot_emit_object_data_section_info(buf, buf_end, &offset, comp_ctx, + obj_data)) + return false; + + if (offset - *p_offset != section_size + sizeof(uint32) * 2) { + aot_set_last_error("emit init data section failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_text_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompData *comp_data, AOTObjectData *obj_data) +{ + uint32 section_size = get_text_section_size(obj_data); + uint32 offset = *p_offset; + uint8 placeholder = 0; + AOTRelocationGroup *relocation_group; + AOTRelocation *relocation; + uint32 i, j, relocation_count; + uint8 *text; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_TEXT); + EMIT_U32(section_size); + EMIT_U32(obj_data->literal_size); + + if (obj_data->literal_size > 0) { + EMIT_BUF(obj_data->literal, obj_data->literal_size); + while (offset & 3) + EMIT_BUF(&placeholder, 1); + } + + text = buf + offset; + + if (obj_data->text_size > 0) { + EMIT_BUF(obj_data->text, obj_data->text_size); + while (offset & 3) + EMIT_BUF(&placeholder, 1); + } + if (obj_data->text_unlikely_size > 0) { + EMIT_BUF(obj_data->text_unlikely, obj_data->text_unlikely_size); + while (offset & 3) + EMIT_BUF(&placeholder, 1); + } + if (obj_data->text_hot_size > 0) { + EMIT_BUF(obj_data->text_hot, obj_data->text_hot_size); + while (offset & 3) + EMIT_BUF(&placeholder, 1); + } + + if (offset - *p_offset != section_size + sizeof(uint32) * 2) { + aot_set_last_error("emit text section failed."); + return false; + } + + /* apply relocations to aot_func_internal#n in text section for + windows platform */ + if ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6) + /* Windows AOT_COFF64_BIN_TYPE */ + && obj_data->target_info.bin_type == 6) + || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4) + /* Windows AOT_COFF32_BIN_TYPE */ + && obj_data->target_info.bin_type == 4)) { + relocation_group = obj_data->relocation_groups; + for (i = 0; i < obj_data->relocation_group_count; + i++, relocation_group++) { + /* relocation in text section */ + if ((!strcmp(relocation_group->section_name, ".text") + || !strcmp(relocation_group->section_name, ".ltext"))) { + relocation = relocation_group->relocations; + relocation_count = relocation_group->relocation_count; + for (j = 0; j < relocation_count; j++) { + /* relocation to aot_func_internal#n */ + if (str_starts_with(relocation->symbol_name, + AOT_FUNC_INTERNAL_PREFIX) + && ((obj_data->target_info.bin_type + == 6 /* AOT_COFF64_BIN_TYPE */ + && relocation->relocation_type + == 4 /* IMAGE_REL_AMD64_REL32 */) + || (obj_data->target_info.bin_type + == 4 /* AOT_COFF32_BIN_TYPE */ + && relocation->relocation_type + == 20 /* IMAGE_REL_I386_REL32 */))) { + uint32 func_idx = + atoi(relocation->symbol_name + + strlen(AOT_FUNC_INTERNAL_PREFIX)); + uint64 text_offset, reloc_offset, reloc_addend; + + bh_assert(func_idx < obj_data->func_count); + + text_offset = obj_data->funcs[func_idx] + .text_offset_of_aot_func_internal; + reloc_offset = relocation->relocation_offset; + reloc_addend = relocation->relocation_addend; + /* S + A - P */ + *(uint32 *)(text + reloc_offset) = + (uint32)(text_offset + reloc_addend - reloc_offset + - 4); + + /* remove current relocation as it has been applied */ + if (j < relocation_count - 1) { + uint32 move_size = + (uint32)(sizeof(AOTRelocation) + * (relocation_count - 1 - j)); + bh_memmove_s(relocation, move_size, relocation + 1, + move_size); + } + relocation_group->relocation_count--; + } + else { + relocation++; + } + } + } + } + } + + *p_offset = offset; + + return true; +} + +#if WASM_ENABLE_GC != 0 +static bool +aot_emit_ref_flag(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + uint8 pointer_size, int8 type) +{ + uint32 j, offset = *p_offset; + uint16 value_type_cell_num; + + if (wasm_is_type_reftype(type) && !wasm_is_reftype_i31ref(type)) { + EMIT_U8(1); + if (pointer_size == sizeof(uint64)) + EMIT_U8(1); + } + else { + value_type_cell_num = wasm_value_type_cell_num(type); + for (j = 0; j < value_type_cell_num; j++) + EMIT_U8(0); + } + + *p_offset = offset; + return true; +} +#endif + +static bool +aot_emit_func_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 section_size = get_func_section_size(comp_ctx, comp_data, obj_data); + uint32 i, offset = *p_offset; + AOTObjectFunc *func = obj_data->funcs; + AOTFunc **funcs = comp_data->funcs; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_FUNCTION); + EMIT_U32(section_size); + + for (i = 0; i < obj_data->func_count; i++, func++) { + if (is_32bit_binary(obj_data)) + EMIT_U32(func->text_offset); + else + EMIT_U64(func->text_offset); + } + + if (need_call_wrapped_indirect(obj_data)) { + /* + * Explicitly emit aot_func_internal#xxx for Xtensa XIP, therefore, + * for aot_func#xxx, func_indexes ranged from 0 ~ func_count, + * for aot_func_internal#xxxx, from func_count + 1 ~ 2 * func_count. + */ + for (i = 0, func = obj_data->funcs; i < obj_data->func_count; + i++, func++) { + if (is_32bit_binary(obj_data)) + EMIT_U32(func->text_offset_of_aot_func_internal); + else + EMIT_U64(func->text_offset_of_aot_func_internal); + } + } + + for (i = 0; i < comp_data->func_count; i++) + EMIT_U32(funcs[i]->func_type_index); + + if (need_call_wrapped_indirect(obj_data)) { + /* func_type_index for aot_func_internal#xxxx */ + for (i = 0; i < comp_data->func_count; i++) + EMIT_U32(funcs[i]->func_type_index); + } + + for (i = 0; i < comp_data->func_count; i++) { + uint32 max_local_cell_num = + funcs[i]->param_cell_num + funcs[i]->local_cell_num; + EMIT_U32(max_local_cell_num); + } + + for (i = 0; i < comp_data->func_count; i++) + EMIT_U32(funcs[i]->max_stack_cell_num); + +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + /* emit func_local_ref_flag arrays for both import and AOTed funcs */ + AOTFuncType *func_type; + uint32 j, local_ref_flags_cell_num, paddings; + + for (i = 0; i < comp_data->import_func_count; i++) { + func_type = comp_data->import_funcs[i].func_type; + /* recalculate cell_num based on target pointer size */ + local_ref_flags_cell_num = 0; + for (j = 0; j < func_type->param_count; j++) { + local_ref_flags_cell_num += wasm_value_type_cell_num_internal( + func_type->types[j], comp_ctx->pointer_size); + } + paddings = + local_ref_flags_cell_num < 2 ? 2 - local_ref_flags_cell_num : 0; + local_ref_flags_cell_num = + local_ref_flags_cell_num > 2 ? local_ref_flags_cell_num : 2; + + offset = align_uint(offset, 4); + EMIT_U32(local_ref_flags_cell_num); + for (j = 0; j < func_type->param_count; j++) { + if (!aot_emit_ref_flag(buf, buf_end, &offset, + comp_ctx->pointer_size, + func_type->types[j])) + return false; + } + for (j = 0; j < paddings; j++) + EMIT_U8(0); + } + + for (i = 0; i < comp_data->func_count; i++) { + func_type = funcs[i]->func_type; + local_ref_flags_cell_num = + funcs[i]->param_cell_num + funcs[i]->local_cell_num; + + offset = align_uint(offset, 4); + EMIT_U32(local_ref_flags_cell_num); + /* emit local_ref_flag for param variables */ + for (j = 0; j < func_type->param_count; j++) { + if (!aot_emit_ref_flag(buf, buf_end, &offset, + comp_ctx->pointer_size, + func_type->types[j])) + return false; + } + /* emit local_ref_flag for local variables */ + for (j = 0; j < funcs[i]->local_count; j++) { + if (!aot_emit_ref_flag(buf, buf_end, &offset, + comp_ctx->pointer_size, + funcs[i]->local_types_wp[j])) + return false; + } + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + if (offset - *p_offset != section_size + sizeof(uint32) * 2) { + aot_set_last_error("emit function section failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_export_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 section_size = get_export_section_size(comp_ctx, comp_data); + AOTExport *export = comp_data->wasm_module->exports; + uint32 export_count = comp_data->wasm_module->export_count; + uint32 i, offset = *p_offset; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_EXPORT); + EMIT_U32(section_size); + EMIT_U32(export_count); + + for (i = 0; i < export_count; i++, export ++) { + offset = align_uint(offset, 4); + EMIT_U32(export->index); + EMIT_U8(export->kind); + EMIT_U8(0); + EMIT_STR(export->name); + } + + if (offset - *p_offset != section_size + sizeof(uint32) * 2) { + aot_set_last_error("emit export section failed."); + return false; + } + + *p_offset = offset; + + return true; +} + +static bool +aot_emit_relocation_symbol_table(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, + AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 symbol_offset = 0, total_string_len = 0; + uint32 offset = *p_offset; + AOTSymbolNode *sym; + + EMIT_U32(obj_data->symbol_list.len); + + /* emit symbol offsets */ + sym = (AOTSymbolNode *)(obj_data->symbol_list.head); + while (sym) { + EMIT_U32(symbol_offset); + /* string_len + str[0 .. string_len - 1] */ + symbol_offset += get_string_size(comp_ctx, sym->symbol); + symbol_offset = align_uint(symbol_offset, 2); + sym = sym->next; + } + + /* emit total string len */ + total_string_len = symbol_offset; + EMIT_U32(total_string_len); + + /* emit symbols */ + sym = (AOTSymbolNode *)(obj_data->symbol_list.head); + while (sym) { + EMIT_STR(sym->symbol); + offset = align_uint(offset, 2); + sym = sym->next; + } + + *p_offset = offset; + return true; +} + +static bool +aot_emit_relocation_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 section_size = get_relocation_section_size(comp_ctx, obj_data); + uint32 i, offset = *p_offset; + AOTRelocationGroup *relocation_group = obj_data->relocation_groups; + + if (section_size == (uint32)-1) + return false; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_RELOCATION); + EMIT_U32(section_size); + + aot_emit_relocation_symbol_table(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data); + + offset = align_uint(offset, 4); + EMIT_U32(obj_data->relocation_group_count); + + /* emit each relocation group */ + for (i = 0; i < obj_data->relocation_group_count; i++, relocation_group++) { + AOTRelocation *relocation = relocation_group->relocations; + uint32 j; + + offset = align_uint(offset, 4); + EMIT_U32(relocation_group->name_index); + offset = align_uint(offset, 4); + EMIT_U32(relocation_group->relocation_count); + + /* emit each relocation */ + for (j = 0; j < relocation_group->relocation_count; j++, relocation++) { + offset = align_uint(offset, 4); + if (is_32bit_binary(obj_data)) { + EMIT_U32(relocation->relocation_offset); + EMIT_U32(relocation->relocation_addend); + } + else { + EMIT_U64(relocation->relocation_offset); + EMIT_U64(relocation->relocation_addend); + } + EMIT_U32(relocation->relocation_type); + EMIT_U32(relocation->symbol_index); + } + } + + if (offset - *p_offset != section_size + sizeof(uint32) * 2) { + aot_set_last_error("emit relocation section failed."); + return false; + } + + *p_offset = offset; + return true; +} + +static bool +aot_emit_native_symbol(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx) +{ + uint32 offset = *p_offset; + AOTNativeSymbol *sym = NULL; + + if (bh_list_length(&comp_ctx->native_symbols) == 0) + /* emit only when there are native symbols */ + return true; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_CUSTOM); + /* sub section id + symbol count + symbol list */ + EMIT_U32(sizeof(uint32) * 2 + get_native_symbol_list_size(comp_ctx)); + EMIT_U32(AOT_CUSTOM_SECTION_NATIVE_SYMBOL); + EMIT_U32(bh_list_length(&comp_ctx->native_symbols)); + + sym = bh_list_first_elem(&comp_ctx->native_symbols); + + while (sym) { + offset = align_uint(offset, 2); + EMIT_STR(sym->symbol); + sym = bh_list_elem_next(sym); + } + + *p_offset = offset; + + return true; +} + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +static bool +aot_emit_name_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompData *comp_data, AOTCompContext *comp_ctx) +{ + uint32 offset = *p_offset; + + if (comp_data->aot_name_section_size == 0) + return true; + + offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_CUSTOM); + /* sub section id + name section size */ + EMIT_U32(sizeof(uint32) * 1 + comp_data->aot_name_section_size); + EMIT_U32(AOT_CUSTOM_SECTION_NAME); + bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf), + comp_data->aot_name_section_buf, + (uint32)comp_data->aot_name_section_size); + offset += comp_data->aot_name_section_size; + + *p_offset = offset; + + LOG_DEBUG("emit name section"); + return true; +} +#endif + +#if WASM_ENABLE_STRINGREF != 0 +static bool +aot_emit_string_literal_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompData *comp_data, + AOTCompContext *comp_ctx) +{ + uint32 string_count = comp_data->string_literal_count; + + if (string_count > 0) { + uint32 offset = *p_offset; + uint32 i; + + *p_offset = offset = align_uint(offset, 4); + + EMIT_U32(AOT_SECTION_TYPE_CUSTOM); + /* sub section id + string literal section size */ + EMIT_U32(sizeof(uint32) * 1 + + get_string_literal_section_size(comp_ctx, comp_data)); + EMIT_U32(AOT_CUSTOM_SECTION_STRING_LITERAL); + + /* reserved */ + EMIT_U32(0); + + /* string literal count */ + EMIT_U32(string_count); + + for (i = 0; i < string_count; i++) { + EMIT_U32(comp_data->string_literal_lengths_wp[i]); + } + + for (i = 0; i < string_count; i++) { + uint32 string_length = comp_data->string_literal_lengths_wp[i]; + bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf), + comp_data->string_literal_ptrs_wp[i], string_length); + offset += string_length; + } + + *p_offset = offset; + } + + return true; +} +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + +static bool +aot_emit_custom_sections(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompData *comp_data, AOTCompContext *comp_ctx) +{ +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + uint32 offset = *p_offset, i; + + for (i = 0; i < comp_ctx->custom_sections_count; i++) { + const char *section_name = comp_ctx->custom_sections_wp[i]; + const uint8 *content = NULL; + uint32 length = 0; + + if (strcmp(section_name, "name") == 0) { + *p_offset = offset; + if (!aot_emit_name_section(buf, buf_end, p_offset, comp_data, + comp_ctx)) + return false; + + offset = *p_offset; + continue; + } + + content = wasm_loader_get_custom_section(comp_data->wasm_module, + section_name, &length); + if (!content) { + /* Warning has been reported during calculating size */ + continue; + } + + offset = align_uint(offset, 4); + EMIT_U32(AOT_SECTION_TYPE_CUSTOM); + /* sub section id + content */ + EMIT_U32(sizeof(uint32) * 1 + get_string_size(comp_ctx, section_name) + + length); + EMIT_U32(AOT_CUSTOM_SECTION_RAW); + EMIT_STR(section_name); + bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf), content, + length); + offset += length; + } + + *p_offset = offset; +#endif + + return true; +} + +typedef uint32 U32; +typedef int32 I32; +typedef uint16 U16; +typedef uint8 U8; + +struct coff_hdr { + U16 u16Machine; + U16 u16NumSections; + U32 u32DateTimeStamp; + U32 u32SymTblPtr; + U32 u32NumSymbols; + U16 u16PeHdrSize; + U16 u16Characs; +}; + +#define E_TYPE_REL 1 +#define E_TYPE_XIP 4 + +#define IMAGE_FILE_MACHINE_AMD64 0x8664 +#define IMAGE_FILE_MACHINE_I386 0x014c +#define IMAGE_FILE_MACHINE_IA64 0x0200 + +#define AOT_COFF32_BIN_TYPE 4 /* 32-bit little endian */ +#define AOT_COFF64_BIN_TYPE 6 /* 64-bit little endian */ + +#define EI_NIDENT 16 + +typedef uint32 elf32_word; +typedef int32 elf32_sword; +typedef uint16 elf32_half; +typedef uint32 elf32_off; +typedef uint32 elf32_addr; + +struct elf32_ehdr { + unsigned char e_ident[EI_NIDENT]; /* ident bytes */ + elf32_half e_type; /* file type */ + elf32_half e_machine; /* target machine */ + elf32_word e_version; /* file version */ + elf32_addr e_entry; /* start address */ + elf32_off e_phoff; /* phdr file offset */ + elf32_off e_shoff; /* shdr file offset */ + elf32_word e_flags; /* file flags */ + elf32_half e_ehsize; /* sizeof ehdr */ + elf32_half e_phentsize; /* sizeof phdr */ + elf32_half e_phnum; /* number phdrs */ + elf32_half e_shentsize; /* sizeof shdr */ + elf32_half e_shnum; /* number shdrs */ + elf32_half e_shstrndx; /* shdr string index */ +}; + +struct elf32_rel { + elf32_addr r_offset; + elf32_word r_info; +} elf32_rel; + +struct elf32_rela { + elf32_addr r_offset; + elf32_word r_info; + elf32_sword r_addend; +} elf32_rela; + +typedef uint32 elf64_word; +typedef int32 elf64_sword; +typedef uint64 elf64_xword; +typedef int64 elf64_sxword; +typedef uint16 elf64_half; +typedef uint64 elf64_off; +typedef uint64 elf64_addr; + +struct elf64_ehdr { + unsigned char e_ident[EI_NIDENT]; /* ident bytes */ + elf64_half e_type; /* file type */ + elf64_half e_machine; /* target machine */ + elf64_word e_version; /* file version */ + elf64_addr e_entry; /* start address */ + elf64_off e_phoff; /* phdr file offset */ + elf64_off e_shoff; /* shdr file offset */ + elf64_word e_flags; /* file flags */ + elf64_half e_ehsize; /* sizeof ehdr */ + elf64_half e_phentsize; /* sizeof phdr */ + elf64_half e_phnum; /* number phdrs */ + elf64_half e_shentsize; /* sizeof shdr */ + elf64_half e_shnum; /* number shdrs */ + elf64_half e_shstrndx; /* shdr string index */ +}; + +typedef struct elf64_rel { + elf64_addr r_offset; + elf64_xword r_info; +} elf64_rel; + +typedef struct elf64_rela { + elf64_addr r_offset; + elf64_xword r_info; + elf64_sxword r_addend; +} elf64_rela; + +#define SET_TARGET_INFO(f, v, type, little) \ + do { \ + type tmp = elf_header->v; \ + if ((little && !is_little_endian()) \ + || (!little && is_little_endian())) \ + exchange_##type((uint8 *)&tmp); \ + obj_data->target_info.f = tmp; \ + } while (0) + +static bool +aot_resolve_target_info(AOTCompContext *comp_ctx, AOTObjectData *obj_data) +{ + LLVMBinaryType bin_type = LLVMBinaryGetType(obj_data->binary); + const uint8 *elf_buf = (uint8 *)LLVMGetBufferStart(obj_data->mem_buf); + uint32 elf_size = (uint32)LLVMGetBufferSize(obj_data->mem_buf); + + if (bin_type != LLVMBinaryTypeCOFF && bin_type != LLVMBinaryTypeELF32L + && bin_type != LLVMBinaryTypeELF32B && bin_type != LLVMBinaryTypeELF64L + && bin_type != LLVMBinaryTypeELF64B + && bin_type != LLVMBinaryTypeMachO32L + && bin_type != LLVMBinaryTypeMachO32B + && bin_type != LLVMBinaryTypeMachO64L + && bin_type != LLVMBinaryTypeMachO64B) { + aot_set_last_error("invaid llvm binary bin_type."); + return false; + } + + obj_data->target_info.bin_type = bin_type - LLVMBinaryTypeELF32L; + + if (bin_type == LLVMBinaryTypeCOFF) { + struct coff_hdr *coff_header; + + if (!elf_buf || elf_size < sizeof(struct coff_hdr)) { + aot_set_last_error("invalid coff_hdr buffer."); + return false; + } + coff_header = (struct coff_hdr *)elf_buf; + + /* Emit eXecute In Place file type while in indirect mode */ + if (comp_ctx->is_indirect_mode) + obj_data->target_info.e_type = E_TYPE_XIP; + else + obj_data->target_info.e_type = E_TYPE_REL; + + obj_data->target_info.e_machine = coff_header->u16Machine; + obj_data->target_info.e_version = 1; + obj_data->target_info.e_flags = 0; + + if (coff_header->u16Machine == IMAGE_FILE_MACHINE_AMD64 + || coff_header->u16Machine == IMAGE_FILE_MACHINE_IA64) + obj_data->target_info.bin_type = AOT_COFF64_BIN_TYPE; + else if (coff_header->u16Machine == IMAGE_FILE_MACHINE_I386) + obj_data->target_info.bin_type = AOT_COFF32_BIN_TYPE; + } + else if (bin_type == LLVMBinaryTypeELF32L + || bin_type == LLVMBinaryTypeELF32B) { + struct elf32_ehdr *elf_header; + bool is_little_bin = bin_type == LLVMBinaryTypeELF32L; + + if (!elf_buf || elf_size < sizeof(struct elf32_ehdr)) { + aot_set_last_error("invalid elf32 buffer."); + return false; + } + + elf_header = (struct elf32_ehdr *)elf_buf; + + /* Emit eXecute In Place file type while in indirect mode */ + if (comp_ctx->is_indirect_mode) + elf_header->e_type = E_TYPE_XIP; + + SET_TARGET_INFO(e_type, e_type, uint16, is_little_bin); + SET_TARGET_INFO(e_machine, e_machine, uint16, is_little_bin); + SET_TARGET_INFO(e_version, e_version, uint32, is_little_bin); + SET_TARGET_INFO(e_flags, e_flags, uint32, is_little_bin); + } + else if (bin_type == LLVMBinaryTypeELF64L + || bin_type == LLVMBinaryTypeELF64B) { + struct elf64_ehdr *elf_header; + bool is_little_bin = bin_type == LLVMBinaryTypeELF64L; + + if (!elf_buf || elf_size < sizeof(struct elf64_ehdr)) { + aot_set_last_error("invalid elf64 buffer."); + return false; + } + + elf_header = (struct elf64_ehdr *)elf_buf; + + /* Emit eXecute In Place file type while in indirect mode */ + if (comp_ctx->is_indirect_mode) + elf_header->e_type = E_TYPE_XIP; + + SET_TARGET_INFO(e_type, e_type, uint16, is_little_bin); + SET_TARGET_INFO(e_machine, e_machine, uint16, is_little_bin); + SET_TARGET_INFO(e_version, e_version, uint32, is_little_bin); + SET_TARGET_INFO(e_flags, e_flags, uint32, is_little_bin); + } + else if (bin_type == LLVMBinaryTypeMachO32L + || bin_type == LLVMBinaryTypeMachO32B) { + /* TODO: parse file type of Mach-O 32 */ + aot_set_last_error("invaid llvm binary bin_type."); + return false; + } + else if (bin_type == LLVMBinaryTypeMachO64L + || bin_type == LLVMBinaryTypeMachO64B) { + /* TODO: parse file type of Mach-O 64 */ + aot_set_last_error("invaid llvm binary bin_type."); + return false; + } + + bh_assert(sizeof(obj_data->target_info.arch) + == sizeof(comp_ctx->target_arch)); + bh_memcpy_s(obj_data->target_info.arch, sizeof(obj_data->target_info.arch), + comp_ctx->target_arch, sizeof(comp_ctx->target_arch)); + + return true; +} + +static bool +aot_resolve_text(AOTObjectData *obj_data) +{ +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMBinaryType bin_type = LLVMBinaryGetType(obj_data->binary); + if (bin_type == LLVMBinaryTypeELF32L || bin_type == LLVMBinaryTypeELF64L) { + obj_data->text = (char *)LLVMGetBufferStart(obj_data->mem_buf); + obj_data->text_size = (uint32)LLVMGetBufferSize(obj_data->mem_buf); + } + else +#endif + { + LLVMSectionIteratorRef sec_itr; + char *name; + + if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + return false; + } + while ( + !LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) { + if ((name = (char *)LLVMGetSectionName(sec_itr))) { + if (!strcmp(name, ".text") || !strcmp(name, ".ltext")) { + obj_data->text = (char *)LLVMGetSectionContents(sec_itr); + obj_data->text_size = (uint32)LLVMGetSectionSize(sec_itr); + } + else if (!strcmp(name, ".text.unlikely.") + || !strcmp(name, ".ltext.unlikely.")) { + obj_data->text_unlikely = + (char *)LLVMGetSectionContents(sec_itr); + obj_data->text_unlikely_size = + (uint32)LLVMGetSectionSize(sec_itr); + } + else if (!strcmp(name, ".text.hot.") + || !strcmp(name, ".ltext.hot.")) { + obj_data->text_hot = + (char *)LLVMGetSectionContents(sec_itr); + obj_data->text_hot_size = + (uint32)LLVMGetSectionSize(sec_itr); + } + } + LLVMMoveToNextSection(sec_itr); + } + LLVMDisposeSectionIterator(sec_itr); + } + + return true; +} + +static bool +aot_resolve_literal(AOTObjectData *obj_data) +{ + LLVMSectionIteratorRef sec_itr; + char *name; + + if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + return false; + } + while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) { + if ((name = (char *)LLVMGetSectionName(sec_itr)) + && !strcmp(name, ".literal")) { + obj_data->literal = (char *)LLVMGetSectionContents(sec_itr); + obj_data->literal_size = (uint32)LLVMGetSectionSize(sec_itr); + break; + } + LLVMMoveToNextSection(sec_itr); + } + LLVMDisposeSectionIterator(sec_itr); + + return true; +} + +static bool +get_relocations_count(LLVMSectionIteratorRef sec_itr, uint32 *p_count); + +static bool +is_data_section(AOTObjectData *obj_data, LLVMSectionIteratorRef sec_itr, + char *section_name) +{ + uint32 relocation_count = 0; + + return (!strcmp(section_name, ".data") || !strcmp(section_name, ".sdata") + || !strcmp(section_name, ".rodata") + /* ".rodata.cst4/8/16/.." */ + || !strncmp(section_name, ".rodata.cst", strlen(".rodata.cst")) + /* ".rodata.strn.m" */ + || !strncmp(section_name, ".rodata.str", strlen(".rodata.str")) + || (!strcmp(section_name, ".rdata") + && get_relocations_count(sec_itr, &relocation_count) + && relocation_count > 0) + || !strcmp(section_name, aot_stack_sizes_section_name) + || (obj_data->comp_ctx->enable_llvm_pgo + && (!strncmp(section_name, "__llvm_prf_cnts", 15) + || !strncmp(section_name, "__llvm_prf_data", 15) + || !strncmp(section_name, "__llvm_prf_names", 16)))); +} + +static bool +get_object_data_sections_count(AOTObjectData *obj_data, uint32 *p_count) +{ + LLVMSectionIteratorRef sec_itr; + char *name; + uint32 count = 0; + + if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + return false; + } + while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) { + if ((name = (char *)LLVMGetSectionName(sec_itr)) + && (is_data_section(obj_data, sec_itr, name))) { + count++; + } + LLVMMoveToNextSection(sec_itr); + } + LLVMDisposeSectionIterator(sec_itr); + + *p_count = count; + return true; +} + +static bool +aot_resolve_object_data_sections(AOTObjectData *obj_data) +{ + LLVMSectionIteratorRef sec_itr; + char *name; + AOTObjectDataSection *data_section; + uint32 sections_count; + uint32 size; + + if (!get_object_data_sections_count(obj_data, §ions_count)) { + return false; + } + + if (sections_count > 0) { + uint32 llvm_prf_cnts_idx = 0, llvm_prf_data_idx = 0; + char buf[32]; + + size = (uint32)sizeof(AOTObjectDataSection) * sections_count; + if (!(data_section = obj_data->data_sections = + wasm_runtime_malloc(size))) { + aot_set_last_error("allocate memory for data sections failed."); + return false; + } + memset(obj_data->data_sections, 0, size); + obj_data->data_sections_count = sections_count; + + if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + return false; + } + while ( + !LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) { + if ((name = (char *)LLVMGetSectionName(sec_itr)) + && (is_data_section(obj_data, sec_itr, name))) { + data_section->name = name; + if (obj_data->comp_ctx->enable_llvm_pgo + && !strcmp(name, "__llvm_prf_cnts")) { + snprintf(buf, sizeof(buf), "%s%u", name, + llvm_prf_cnts_idx++); + size = (uint32)(strlen(buf) + 1); + if (!(data_section->name = wasm_runtime_malloc(size))) { + aot_set_last_error( + "allocate memory for data section name failed."); + return false; + } + bh_memcpy_s(data_section->name, size, buf, size); + data_section->is_name_allocated = true; + } + else if (obj_data->comp_ctx->enable_llvm_pgo + && !strcmp(name, "__llvm_prf_data")) { + snprintf(buf, sizeof(buf), "%s%u", name, + llvm_prf_data_idx++); + size = (uint32)(strlen(buf) + 1); + if (!(data_section->name = wasm_runtime_malloc(size))) { + aot_set_last_error( + "allocate memory for data section name failed."); + return false; + } + bh_memcpy_s(data_section->name, size, buf, size); + data_section->is_name_allocated = true; + } + + if (obj_data->comp_ctx->enable_llvm_pgo + && !strcmp(name, "__llvm_prf_names")) { + data_section->data = (uint8 *)aot_compress_aot_func_names( + obj_data->comp_ctx, &data_section->size); + data_section->is_data_allocated = true; + } + else { + data_section->data = + (uint8 *)LLVMGetSectionContents(sec_itr); + data_section->size = (uint32)LLVMGetSectionSize(sec_itr); + } + data_section++; + } + LLVMMoveToNextSection(sec_itr); + } + LLVMDisposeSectionIterator(sec_itr); + } + + return true; +} + +static bool +read_stack_usage_file(const AOTCompContext *comp_ctx, const char *filename, + uint32 *sizes, uint32 count) +{ + FILE *fp = NULL; + if (filename == NULL) { + aot_set_last_error("no stack usage file is specified."); + return false; + } + fp = fopen(filename, "r"); + if (fp == NULL) { + LOG_ERROR("failed to open stack usage file: %s", filename); + goto fail; + } + /* + * the file consists of lines like: + * + * WASM Module:aot_func#9 72 static + */ + const char *aot_func_prefix = AOT_FUNC_PREFIX; + const char *aot_func_internal_prefix = AOT_FUNC_INTERNAL_PREFIX; + uint32 precheck_found = 0; + uint32 precheck_stack_size_max = 0; + uint32 precheck_stack_size_min = UINT32_MAX; + uint32 found = 0; + while (true) { + const char *prefix; + char line[100]; + char *cp = fgets(line, sizeof(line), fp); + char *fn; + char *colon; + uintmax_t func_idx; + uintmax_t sz; + int ret; + + if (cp == NULL) { + break; + } + /* + * Note: strrchr (not strchr) because a module name can contain + * colons. + */ + colon = strrchr(cp, ':'); + if (colon == NULL) { + goto fail; + } + fn = strstr(colon, aot_func_prefix); + if (fn != NULL) { + prefix = aot_func_prefix; + } + else { + fn = strstr(colon, aot_func_internal_prefix); + if (fn == NULL) { + LOG_ERROR("failed to parse stack usage line: %s", cp); + goto fail; + } + prefix = aot_func_internal_prefix; + } + ret = sscanf(fn + strlen(prefix), "%ju %ju static", &func_idx, &sz); + if (ret != 2) { + goto fail; + } + if (sz > UINT32_MAX) { + goto fail; + } + if (func_idx > UINT32_MAX) { + goto fail; + } + if (func_idx >= count) { + goto fail; + } + if (prefix == aot_func_prefix) { + if (sz < precheck_stack_size_min) { + precheck_stack_size_min = (uint32)sz; + } + if (sz > precheck_stack_size_max) { + precheck_stack_size_max = (uint32)sz; + } + precheck_found++; + continue; + } + sizes[func_idx] = (uint32)sz; + found++; + } + fclose(fp); + if (precheck_found != count) { + LOG_ERROR("%" PRIu32 " precheck entries found while %" PRIu32 + " entries are expected", + precheck_found, count); + return false; + } + if (found != count) { + /* + * LLVM seems to eliminate calls to an empty function + * (and eliminate the function) even if it's marked noinline. + */ + LOG_VERBOSE("%" PRIu32 " entries found while %" PRIu32 + " entries are expected. Maybe LLVM optimization eliminated " + "some functions.", + found, count); + } + if (precheck_stack_size_min != precheck_stack_size_max) { + /* + * Note: this is too strict. + * + * actually, the stack consumption of the precheck functions + * can depend on the type of them. + * that is, depending on various factors including + * calling conventions and compilers, a function with many + * parameters can consume more stack, even if it merely does + * a tail-call to another function. + */ + bool musttail = aot_target_precheck_can_use_musttail(comp_ctx); + if (musttail) { + LOG_WARNING( + "precheck functions use variable amount of stack. (%" PRIu32 + " - %" PRIu32 ")", + precheck_stack_size_min, precheck_stack_size_max); + } + else { + LOG_VERBOSE("precheck functions use %" PRIu32 " - %" PRIu32 + " bytes of stack.", + precheck_stack_size_min, precheck_stack_size_max); + } + } + else { + LOG_VERBOSE("precheck functions use %" PRIu32 " bytes of stack.", + precheck_stack_size_max); + } + if (precheck_stack_size_max >= 1024) { + LOG_WARNING("precheck functions themselves consume relatively large " + "amount of stack (%" PRIu32 + "). Please ensure the runtime has large enough " + "WASM_STACK_GUARD_SIZE.", + precheck_stack_size_max); + } + return true; +fail: + if (fp != NULL) + fclose(fp); + aot_set_last_error("failed to read stack usage file."); + return false; +} + +static bool +aot_resolve_stack_sizes(AOTCompContext *comp_ctx, AOTObjectData *obj_data) +{ + LLVMSectionIteratorRef sec_itr = NULL; + LLVMSymbolIteratorRef sym_itr; + const char *name; + + if (!(sym_itr = LLVMObjectFileCopySymbolIterator(obj_data->binary))) { + aot_set_last_error("llvm get symbol iterator failed."); + return false; + } + + while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) { + if ((name = LLVMGetSymbolName(sym_itr)) + && (!strcmp(name, aot_stack_sizes_alias_name) + /* symbol of COFF32 starts with "_" */ + || (obj_data->target_info.bin_type == AOT_COFF32_BIN_TYPE + && !strncmp(name, "_", 1) + && !strcmp(name + 1, aot_stack_sizes_alias_name)))) { +#if 0 /* cf. https://github.com/llvm/llvm-project/issues/67765 */ + uint64 sz = LLVMGetSymbolSize(sym_itr); + if (sz != sizeof(uint32) * obj_data->func_count + /* sz of COFF64/COFF32 is 0, ignore the check */ + && obj_data->target_info.bin_type != AOT_COFF64_BIN_TYPE + && obj_data->target_info.bin_type != AOT_COFF32_BIN_TYPE) { + aot_set_last_error("stack_sizes had unexpected size."); + goto fail; + } +#endif + uint64 addr = LLVMGetSymbolAddress(sym_itr); + if (!(sec_itr = + LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + goto fail; + } + LLVMMoveToContainingSection(sec_itr, sym_itr); + const char *sec_name = LLVMGetSectionName(sec_itr); + LOG_VERBOSE("stack_sizes found in section %s offset %" PRIu64 ".", + sec_name, addr); + if (strcmp(sec_name, aot_stack_sizes_section_name) || addr != 0) { + aot_set_last_error( + "stack_sizes found at an unexpected location."); + goto fail; + } + /* + * Note: We can't always modify stack_sizes in-place. + * E.g. When WAMRC_LLC_COMPILER is used, LLVM sometimes uses + * read-only mmap of the temporary file to back + * LLVMGetSectionContents. + */ + const uint32 *ro_stack_sizes = + (const uint32 *)(LLVMGetSectionContents(sec_itr) + addr); + uint32 i; + for (i = 0; i < obj_data->func_count; i++) { + /* Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes */ + if (ro_stack_sizes[i] != (uint32)-1) { + aot_set_last_error("unexpected data in stack_sizes."); + goto fail; + } + } + /* + * Record section/offset and construct a copy of stack_sizes. + * aot_emit_object_data_section_info will emit this copy. + */ + obj_data->stack_sizes_section_name = sec_name; + obj_data->stack_sizes_offset = (uint32)addr; + obj_data->stack_sizes = wasm_runtime_malloc( + obj_data->func_count * sizeof(*obj_data->stack_sizes)); + if (obj_data->stack_sizes == NULL) { + aot_set_last_error("failed to allocate memory."); + goto fail; + } + uint32 *stack_sizes = obj_data->stack_sizes; + for (i = 0; i < obj_data->func_count; i++) { + stack_sizes[i] = (uint32)-1; + } + if (!read_stack_usage_file(comp_ctx, comp_ctx->stack_usage_file, + stack_sizes, obj_data->func_count)) { + goto fail; + } + for (i = 0; i < obj_data->func_count; i++) { + const AOTFuncContext *func_ctx = comp_ctx->func_ctxes[i]; + bool musttail = aot_target_precheck_can_use_musttail(comp_ctx); + unsigned int stack_consumption_to_call_wrapped_func = + musttail ? 0 + : aot_estimate_stack_usage_for_function_call( + comp_ctx, func_ctx->aot_func->func_type); + + /* + * LLVM seems to eliminate calls to an empty function + * (and eliminate the function) even if it's marked noinline. + * + * Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes + */ + if (stack_sizes[i] == (uint32)-1) { + if (func_ctx->stack_consumption_for_func_call != 0) { + /* + * This happens if a function calling another + * function has been optimized out. + * + * for example, + * + * (func $func + * (local i32) + * local.get 0 + * if + * call $another + * end + * ) + */ + LOG_VERBOSE("AOT func#%" PRIu32 + " had call(s) but eliminated?", + i); + } + else { + LOG_VERBOSE("AOT func#%" PRIu32 " eliminated?", i); + } + stack_sizes[i] = 0; + } + else { + LOG_VERBOSE("AOT func#%" PRIu32 " stack_size %u + %" PRIu32 + " + %u", + i, stack_consumption_to_call_wrapped_func, + stack_sizes[i], + func_ctx->stack_consumption_for_func_call); + if (UINT32_MAX - stack_sizes[i] + < func_ctx->stack_consumption_for_func_call) { + aot_set_last_error("stack size overflow."); + goto fail; + } + stack_sizes[i] += func_ctx->stack_consumption_for_func_call; + if (UINT32_MAX - stack_sizes[i] + < stack_consumption_to_call_wrapped_func) { + aot_set_last_error("stack size overflow."); + goto fail; + } + stack_sizes[i] += stack_consumption_to_call_wrapped_func; + } + } + LLVMDisposeSectionIterator(sec_itr); + LLVMDisposeSymbolIterator(sym_itr); + return true; + } + LLVMMoveToNextSymbol(sym_itr); + } + aot_set_last_error("stack_sizes not found."); +fail: + if (sec_itr) + LLVMDisposeSectionIterator(sec_itr); + LLVMDisposeSymbolIterator(sym_itr); + return false; +} + +static bool +aot_resolve_functions(AOTCompContext *comp_ctx, AOTObjectData *obj_data) +{ + AOTObjectFunc *func; + LLVMSymbolIteratorRef sym_itr; + char *name, *prefix = AOT_FUNC_PREFIX; + uint32 func_index, total_size; + + /* allocate memory for aot function */ + obj_data->func_count = comp_ctx->comp_data->func_count; + if (obj_data->func_count) { + if ((comp_ctx->enable_stack_bound_check + || comp_ctx->enable_stack_estimation) + && !aot_resolve_stack_sizes(comp_ctx, obj_data)) + return false; + total_size = (uint32)sizeof(AOTObjectFunc) * obj_data->func_count; + if (!(obj_data->funcs = wasm_runtime_malloc(total_size))) { + aot_set_last_error("allocate memory for functions failed."); + return false; + } + memset(obj_data->funcs, 0, total_size); + } + + if (!(sym_itr = LLVMObjectFileCopySymbolIterator(obj_data->binary))) { + aot_set_last_error("llvm get symbol iterator failed."); + return false; + } + + while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) { + if ((name = (char *)LLVMGetSymbolName(sym_itr)) + && str_starts_with(name, prefix)) { + /* symbol aot_func#n */ + func_index = (uint32)atoi(name + strlen(prefix)); + if (func_index < obj_data->func_count) { + LLVMSectionIteratorRef contain_section; + char *contain_section_name; + + func = obj_data->funcs + func_index; + func->func_name = name; + + if (!(contain_section = LLVMObjectFileCopySectionIterator( + obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + LLVMDisposeSymbolIterator(sym_itr); + return false; + } + LLVMMoveToContainingSection(contain_section, sym_itr); + contain_section_name = + (char *)LLVMGetSectionName(contain_section); + LLVMDisposeSectionIterator(contain_section); + + if (!strcmp(contain_section_name, ".text.unlikely.") + || !strcmp(contain_section_name, ".ltext.unlikely.")) { + func->text_offset = align_uint(obj_data->text_size, 4) + + LLVMGetSymbolAddress(sym_itr); + } + else if (!strcmp(contain_section_name, ".text.hot.") + || !strcmp(contain_section_name, ".ltext.hot.")) { + func->text_offset = + align_uint(obj_data->text_size, 4) + + align_uint(obj_data->text_unlikely_size, 4) + + LLVMGetSymbolAddress(sym_itr); + } + else { + func->text_offset = LLVMGetSymbolAddress(sym_itr); + } + } + } + else if ((name = (char *)LLVMGetSymbolName(sym_itr)) + && str_starts_with(name, AOT_FUNC_INTERNAL_PREFIX)) { + /* symbol aot_func_internal#n */ + func_index = (uint32)atoi(name + strlen(AOT_FUNC_INTERNAL_PREFIX)); + if (func_index < obj_data->func_count) { + LLVMSectionIteratorRef contain_section; + char *contain_section_name; + + func = obj_data->funcs + func_index; + + if (!(contain_section = LLVMObjectFileCopySectionIterator( + obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + LLVMDisposeSymbolIterator(sym_itr); + return false; + } + LLVMMoveToContainingSection(contain_section, sym_itr); + contain_section_name = + (char *)LLVMGetSectionName(contain_section); + LLVMDisposeSectionIterator(contain_section); + + if (!strcmp(contain_section_name, ".text.unlikely.") + || !strcmp(contain_section_name, ".ltext.unlikely.")) { + func->text_offset_of_aot_func_internal = + align_uint(obj_data->text_size, 4) + + LLVMGetSymbolAddress(sym_itr); + } + else if (!strcmp(contain_section_name, ".text.hot.") + || !strcmp(contain_section_name, ".ltext.hot.")) { + func->text_offset_of_aot_func_internal = + align_uint(obj_data->text_size, 4) + + align_uint(obj_data->text_unlikely_size, 4) + + LLVMGetSymbolAddress(sym_itr); + } + else { + func->text_offset_of_aot_func_internal = + LLVMGetSymbolAddress(sym_itr); + } + } + } + LLVMMoveToNextSymbol(sym_itr); + } + LLVMDisposeSymbolIterator(sym_itr); + + return true; +} + +static bool +get_relocations_count(LLVMSectionIteratorRef sec_itr, uint32 *p_count) +{ + uint32 relocation_count = 0; + LLVMRelocationIteratorRef rel_itr; + + if (!(rel_itr = LLVMGetRelocations(sec_itr))) { + aot_set_last_error("llvm get relocations failed."); + LLVMDisposeSectionIterator(sec_itr); + return false; + } + + while (!LLVMIsRelocationIteratorAtEnd(sec_itr, rel_itr)) { + relocation_count++; + LLVMMoveToNextRelocation(rel_itr); + } + LLVMDisposeRelocationIterator(rel_itr); + + *p_count = relocation_count; + return true; +} + +static bool +aot_resolve_object_relocation_group(AOTObjectData *obj_data, + AOTRelocationGroup *group, + LLVMSectionIteratorRef rel_sec) +{ + LLVMRelocationIteratorRef rel_itr; + AOTRelocation *relocation = group->relocations; + uint32 size; + bool is_binary_32bit = is_32bit_binary(obj_data); + bool is_binary_little_endian = is_little_endian_binary(obj_data); + bool has_addend = str_starts_with(group->section_name, ".rela"); + uint8 *rela_content = NULL; + + /* calculate relocations count and allocate memory */ + if (!get_relocations_count(rel_sec, &group->relocation_count)) + return false; + if (group->relocation_count == 0) { + aot_set_last_error("invalid relocations count"); + return false; + } + size = (uint32)sizeof(AOTRelocation) * group->relocation_count; + if (!(relocation = group->relocations = wasm_runtime_malloc(size))) { + aot_set_last_error("allocate memory for relocations failed."); + return false; + } + memset(group->relocations, 0, size); + + if (has_addend) { + uint64 rela_content_size; + /* LLVM doesn't provide C API to get relocation addend. So we have to + * parse it manually. */ + rela_content = (uint8 *)LLVMGetSectionContents(rel_sec); + rela_content_size = LLVMGetSectionSize(rel_sec); + if (is_binary_32bit) + size = (uint32)sizeof(struct elf32_rela) * group->relocation_count; + else + size = (uint32)sizeof(struct elf64_rela) * group->relocation_count; + if (rela_content_size != (uint64)size) { + aot_set_last_error("invalid relocation section content."); + return false; + } + } + + /* pares each relocation */ + if (!(rel_itr = LLVMGetRelocations(rel_sec))) { + aot_set_last_error("llvm get relocations failed."); + return false; + } + while (!LLVMIsRelocationIteratorAtEnd(rel_sec, rel_itr)) { + uint64 offset = LLVMGetRelocationOffset(rel_itr); + uint64 type = LLVMGetRelocationType(rel_itr); + LLVMSymbolIteratorRef rel_sym = LLVMGetRelocationSymbol(rel_itr); + + if (!rel_sym) { + aot_set_last_error("llvm get relocation symbol failed."); + goto fail; + } + + /* parse relocation addend from relocation content */ + if (has_addend) { + if (is_binary_32bit) { + int32 addend = + (int32)(((struct elf32_rela *)rela_content)->r_addend); + if (is_binary_little_endian != is_little_endian()) + exchange_uint32((uint8 *)&addend); + relocation->relocation_addend = (int64)addend; + rela_content += sizeof(struct elf32_rela); + } + else { + int64 addend = + (int64)(((struct elf64_rela *)rela_content)->r_addend); + if (is_binary_little_endian != is_little_endian()) + exchange_uint64((uint8 *)&addend); + relocation->relocation_addend = addend; + rela_content += sizeof(struct elf64_rela); + } + } + + /* set relocation fields */ + relocation->relocation_type = (uint32)type; + relocation->symbol_name = (char *)LLVMGetSymbolName(rel_sym); + relocation->relocation_offset = offset; + if (!strcmp(group->section_name, ".rela.text.unlikely.") + || !strcmp(group->section_name, ".rel.text.unlikely.")) { + relocation->relocation_offset += align_uint(obj_data->text_size, 4); + } + else if (!strcmp(group->section_name, ".rela.text.hot.") + || !strcmp(group->section_name, ".rel.text.hot.")) { + relocation->relocation_offset += + align_uint(obj_data->text_size, 4) + + align_uint(obj_data->text_unlikely_size, 4); + } + if (!strcmp(relocation->symbol_name, ".text.unlikely.")) { + relocation->symbol_name = ".text"; + relocation->relocation_addend += align_uint(obj_data->text_size, 4); + } + if (!strcmp(relocation->symbol_name, ".text.hot.")) { + relocation->symbol_name = ".text"; + relocation->relocation_addend += + align_uint(obj_data->text_size, 4) + + align_uint(obj_data->text_unlikely_size, 4); + } + + /* + * Note: aot_stack_sizes_section_name section only contains + * stack_sizes table. + */ + if (!strcmp(relocation->symbol_name, aot_stack_sizes_name) + /* in windows 32, the symbol name may start with '_' */ + || (strlen(relocation->symbol_name) > 0 + && relocation->symbol_name[0] == '_' + && !strcmp(relocation->symbol_name + 1, + aot_stack_sizes_name))) { + /* discard const */ + relocation->symbol_name = (char *)aot_stack_sizes_section_name; + } + + if (obj_data->comp_ctx->enable_llvm_pgo + && (!strcmp(relocation->symbol_name, "__llvm_prf_cnts") + || !strcmp(relocation->symbol_name, "__llvm_prf_data"))) { + LLVMSectionIteratorRef sec_itr; + char buf[32], *section_name; + uint32 prof_section_idx = 0; + + if (!(sec_itr = + LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + LLVMDisposeSymbolIterator(rel_sym); + goto fail; + } + while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, + sec_itr)) { + section_name = (char *)LLVMGetSectionName(sec_itr); + if (section_name + && !strcmp(section_name, relocation->symbol_name)) { + if (LLVMGetSectionContainsSymbol(sec_itr, rel_sym)) + break; + prof_section_idx++; + } + LLVMMoveToNextSection(sec_itr); + } + LLVMDisposeSectionIterator(sec_itr); + + if (!strcmp(group->section_name, ".rela.text") + || !strcmp(group->section_name, ".rel.text")) { + snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name, + prof_section_idx); + size = (uint32)(strlen(buf) + 1); + if (!(relocation->symbol_name = wasm_runtime_malloc(size))) { + aot_set_last_error( + "allocate memory for relocation symbol name failed."); + LLVMDisposeSymbolIterator(rel_sym); + goto fail; + } + bh_memcpy_s(relocation->symbol_name, size, buf, size); + relocation->is_symbol_name_allocated = true; + } + else if (!strncmp(group->section_name, ".rela__llvm_prf_data", 20) + || !strncmp(group->section_name, ".rel__llvm_prf_data", + 19)) { + snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name, + prof_section_idx); + size = (uint32)(strlen(buf) + 1); + if (!(relocation->symbol_name = wasm_runtime_malloc(size))) { + aot_set_last_error( + "allocate memory for relocation symbol name failed."); + LLVMDisposeSymbolIterator(rel_sym); + goto fail; + } + bh_memcpy_s(relocation->symbol_name, size, buf, size); + relocation->is_symbol_name_allocated = true; + } + } + + /* for ".LCPIxxx", ".LJTIxxx", ".LBBxxx" and switch lookup table + * relocation, transform the symbol name to real section name and set + * addend to the offset of the symbol in the real section */ + if (relocation->symbol_name + && (str_starts_with(relocation->symbol_name, ".LCPI") + || str_starts_with(relocation->symbol_name, ".LJTI") + || str_starts_with(relocation->symbol_name, ".LBB") + || str_starts_with(relocation->symbol_name, + ".Lswitch.table."))) { + /* change relocation->relocation_addend and + relocation->symbol_name */ + LLVMSectionIteratorRef contain_section; + if (!(contain_section = + LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + goto fail; + } + LLVMMoveToContainingSection(contain_section, rel_sym); + if (LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, + contain_section)) { + LLVMDisposeSectionIterator(contain_section); + aot_set_last_error("llvm get containing section failed."); + goto fail; + } + relocation->relocation_addend += LLVMGetSymbolAddress(rel_sym); + relocation->symbol_name = + (char *)LLVMGetSectionName(contain_section); + LLVMDisposeSectionIterator(contain_section); + } + + LLVMDisposeSymbolIterator(rel_sym); + LLVMMoveToNextRelocation(rel_itr); + relocation++; + } + LLVMDisposeRelocationIterator(rel_itr); + return true; + +fail: + LLVMDisposeRelocationIterator(rel_itr); + return false; +} + +static bool +is_relocation_section_name(AOTObjectData *obj_data, char *section_name) +{ + return (!strcmp(section_name, ".rela.text") + || !strcmp(section_name, ".rel.text") + || !strcmp(section_name, ".rela.text.unlikely.") + || !strcmp(section_name, ".rel.text.unlikely.") + || !strcmp(section_name, ".rela.text.hot.") + || !strcmp(section_name, ".rel.text.hot.") + || !strcmp(section_name, ".rela.ltext") + || !strcmp(section_name, ".rel.ltext") + || !strcmp(section_name, ".rela.ltext.unlikely.") + || !strcmp(section_name, ".rel.ltext.unlikely.") + || !strcmp(section_name, ".rela.ltext.hot.") + || !strcmp(section_name, ".rel.ltext.hot.") + || !strcmp(section_name, ".rela.literal") + || !strcmp(section_name, ".rela.data") + || !strcmp(section_name, ".rel.data") + || !strcmp(section_name, ".rela.sdata") + || !strcmp(section_name, ".rel.sdata") + || !strcmp(section_name, ".rela.rodata") + || !strcmp(section_name, ".rel.rodata") + || (obj_data->comp_ctx->enable_llvm_pgo + && (!strcmp(section_name, ".rela__llvm_prf_data") + || !strcmp(section_name, ".rel__llvm_prf_data"))) + /* ".rela.rodata.cst4/8/16/.." */ + || !strncmp(section_name, ".rela.rodata.cst", + strlen(".rela.rodata.cst")) + /* ".rel.rodata.cst4/8/16/.." */ + || !strncmp(section_name, ".rel.rodata.cst", + strlen(".rel.rodata.cst"))); +} + +static bool +is_relocation_section(AOTObjectData *obj_data, LLVMSectionIteratorRef sec_itr) +{ + uint32 count = 0; + char *name = (char *)LLVMGetSectionName(sec_itr); + if (name) { + if (is_relocation_section_name(obj_data, name)) + return true; + else if ((!strcmp(name, ".text") || !strcmp(name, ".text.unlikely.") + || !strcmp(name, ".text.hot.") || !strcmp(name, ".rdata")) + && get_relocations_count(sec_itr, &count) && count > 0) + return true; + } + return false; +} + +static bool +is_readonly_section(const char *name) +{ + return !strcmp(name, ".rel.text") || !strcmp(name, ".rela.text") + || !strcmp(name, ".rel.ltext") || !strcmp(name, ".rela.ltext") + || !strcmp(name, ".rela.literal") || !strcmp(name, ".text") + || !strcmp(name, ".ltext"); +} + +static bool +get_relocation_groups_count(AOTObjectData *obj_data, uint32 *p_count) +{ + uint32 count = 0; + LLVMSectionIteratorRef sec_itr; + + if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + return false; + } + while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) { + if (is_relocation_section(obj_data, sec_itr)) { + count++; + } + LLVMMoveToNextSection(sec_itr); + } + LLVMDisposeSectionIterator(sec_itr); + + *p_count = count; + return true; +} + +static bool +aot_resolve_object_relocation_groups(AOTObjectData *obj_data) +{ + LLVMSectionIteratorRef sec_itr; + AOTRelocationGroup *relocation_group; + uint32 group_count, llvm_prf_data_idx = 0; + char *name; + uint32 size; + + /* calculate relocation groups count and allocate memory */ + if (!get_relocation_groups_count(obj_data, &group_count)) + return false; + + if (0 == (obj_data->relocation_group_count = group_count)) + return true; + + size = (uint32)sizeof(AOTRelocationGroup) * group_count; + if (!(relocation_group = obj_data->relocation_groups = + wasm_runtime_malloc(size))) { + aot_set_last_error("allocate memory for relocation groups failed."); + return false; + } + + memset(obj_data->relocation_groups, 0, size); + + /* resolve each relocation group */ + if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + return false; + } + while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) { + if (is_relocation_section(obj_data, sec_itr)) { + name = (char *)LLVMGetSectionName(sec_itr); + relocation_group->section_name = name; + + if (obj_data->comp_ctx->enable_llvm_pgo + && (!strcmp(name, ".rela__llvm_prf_data") + || !strcmp(name, ".rel__llvm_prf_data"))) { + char buf[32]; + snprintf(buf, sizeof(buf), "%s%u", name, llvm_prf_data_idx); + size = (uint32)(strlen(buf) + 1); + if (!(relocation_group->section_name = + wasm_runtime_malloc(size))) { + aot_set_last_error( + "allocate memory for section name failed."); + LLVMDisposeSectionIterator(sec_itr); + return false; + } + bh_memcpy_s(relocation_group->section_name, size, buf, size); + relocation_group->is_section_name_allocated = true; + } + + if (!aot_resolve_object_relocation_group(obj_data, relocation_group, + sec_itr)) { + LLVMDisposeSectionIterator(sec_itr); + return false; + } + + if (obj_data->comp_ctx->enable_llvm_pgo + && (!strcmp(name, ".rela__llvm_prf_data") + || !strcmp(name, ".rel__llvm_prf_data"))) { + llvm_prf_data_idx++; + } + + if (!strcmp(relocation_group->section_name, ".rela.text.unlikely.") + || !strcmp(relocation_group->section_name, ".rela.text.hot.")) { + relocation_group->section_name = ".rela.text"; + } + else if (!strcmp(relocation_group->section_name, + ".rela.ltext.unlikely.") + || !strcmp(relocation_group->section_name, + ".rela.ltext.hot.")) { + relocation_group->section_name = ".rela.ltext"; + } + else if (!strcmp(relocation_group->section_name, + ".rel.text.unlikely.") + || !strcmp(relocation_group->section_name, + ".rel.text.hot.")) { + relocation_group->section_name = ".rel.text"; + } + else if (!strcmp(relocation_group->section_name, + ".rel.ltext.unlikely.") + || !strcmp(relocation_group->section_name, + ".rel.ltext.hot.")) { + relocation_group->section_name = ".rel.ltext"; + } + + /* + * Relocations in read-only sections are problematic, + * especially for XIP on platforms which don't have + * copy-on-write mappings. + */ + if (obj_data->comp_ctx->is_indirect_mode + && is_readonly_section(relocation_group->section_name)) { + LOG_WARNING("%" PRIu32 + " text relocations in %s section for indirect mode", + relocation_group->relocation_count, + relocation_group->section_name); + } + + relocation_group++; + } + LLVMMoveToNextSection(sec_itr); + } + LLVMDisposeSectionIterator(sec_itr); + + return true; +} + +static void +destroy_relocation_groups(AOTRelocationGroup *relocation_groups, + uint32 relocation_group_count) +{ + uint32 i, j; + AOTRelocationGroup *relocation_group = relocation_groups; + + for (i = 0; i < relocation_group_count; i++, relocation_group++) { + if (relocation_group->relocations) { + for (j = 0; j < relocation_group->relocation_count; j++) { + if (relocation_group->relocations[j].is_symbol_name_allocated) + wasm_runtime_free( + relocation_group->relocations[j].symbol_name); + } + wasm_runtime_free(relocation_group->relocations); + } + if (relocation_group->is_section_name_allocated) + wasm_runtime_free(relocation_group->section_name); + } + wasm_runtime_free(relocation_groups); +} + +static void +destroy_relocation_symbol_list(AOTSymbolList *symbol_list) +{ + AOTSymbolNode *elem; + + elem = symbol_list->head; + while (elem) { + AOTSymbolNode *next = elem->next; + wasm_runtime_free(elem); + elem = next; + } +} + +static void +aot_obj_data_destroy(AOTObjectData *obj_data) +{ + if (obj_data->binary) + LLVMDisposeBinary(obj_data->binary); + if (obj_data->mem_buf) + LLVMDisposeMemoryBuffer(obj_data->mem_buf); + if (obj_data->funcs) + wasm_runtime_free(obj_data->funcs); + if (obj_data->data_sections) { + uint32 i; + for (i = 0; i < obj_data->data_sections_count; i++) { + if (obj_data->data_sections[i].name + && obj_data->data_sections[i].is_name_allocated) { + wasm_runtime_free(obj_data->data_sections[i].name); + } + if (obj_data->data_sections[i].data + && obj_data->data_sections[i].is_data_allocated) { + wasm_runtime_free(obj_data->data_sections[i].data); + } + } + wasm_runtime_free(obj_data->data_sections); + } + if (obj_data->relocation_groups) + destroy_relocation_groups(obj_data->relocation_groups, + obj_data->relocation_group_count); + if (obj_data->symbol_list.len) + destroy_relocation_symbol_list(&obj_data->symbol_list); + if (obj_data->stack_sizes) + wasm_runtime_free(obj_data->stack_sizes); + wasm_runtime_free(obj_data); +} + +static AOTObjectData * +aot_obj_data_create(AOTCompContext *comp_ctx) +{ + char *err = NULL; + AOTObjectData *obj_data; + LLVMTargetRef target = LLVMGetTargetMachineTarget(comp_ctx->target_machine); + + bh_print_time("Begin to emit object file to buffer"); + + if (!(obj_data = wasm_runtime_malloc(sizeof(AOTObjectData)))) { + aot_set_last_error("allocate memory failed."); + return false; + } + memset(obj_data, 0, sizeof(AOTObjectData)); + obj_data->comp_ctx = comp_ctx; + + bh_print_time("Begin to emit object file"); + if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) { +#if defined(_WIN32) || defined(_WIN32_) + aot_set_last_error("external toolchain not supported on Windows"); + goto fail; +#else + /* Generate a temp file name */ + int ret; + char obj_file_name[64]; + + if (!aot_generate_tempfile_name("wamrc-obj", "o", obj_file_name, + sizeof(obj_file_name))) { + goto fail; + } + + if (!aot_emit_object_file(comp_ctx, obj_file_name)) { + goto fail; + } + + /* create memory buffer from object file */ + ret = LLVMCreateMemoryBufferWithContentsOfFile( + obj_file_name, &obj_data->mem_buf, &err); + /* remove temp object file */ + unlink(obj_file_name); + + if (ret != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("create mem buffer with file failed."); + goto fail; + } +#endif /* end of defined(_WIN32) || defined(_WIN32_) */ + } + else if (!strncmp(LLVMGetTargetName(target), "arc", 3)) { +#if defined(_WIN32) || defined(_WIN32_) + aot_set_last_error("emit object file on Windows is unsupported."); + goto fail; +#else + /* Emit to assembly file instead for arc target + as it cannot emit to object file */ + char file_name[] = "wasm-XXXXXX", buf[128]; + int fd, ret; + + if ((fd = mkstemp(file_name)) <= 0) { + aot_set_last_error("make temp file failed."); + goto fail; + } + + /* close and remove temp file */ + close(fd); + unlink(file_name); + + snprintf(buf, sizeof(buf), "%s%s", file_name, ".s"); + if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine, + comp_ctx->module, buf, LLVMAssemblyFile, + &err) + != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("emit elf to object file failed."); + goto fail; + } + + /* call arc gcc to compile assembly file to object file */ + /* TODO: get arc gcc from environment variable firstly + and check whether the toolchain exists actually */ + snprintf(buf, sizeof(buf), "%s%s%s%s%s%s", + "/opt/zephyr-sdk/arc-zephyr-elf/bin/arc-zephyr-elf-gcc ", + "-mcpu=arcem -o ", file_name, ".o -c ", file_name, ".s"); + /* TODO: use try..catch to handle possible exceptions */ + ret = system(buf); + /* remove temp assembly file */ + snprintf(buf, sizeof(buf), "%s%s", file_name, ".s"); + unlink(buf); + + if (ret != 0) { + aot_set_last_error("failed to compile asm file to obj file " + "with arc gcc toolchain."); + goto fail; + } + + /* create memory buffer from object file */ + snprintf(buf, sizeof(buf), "%s%s", file_name, ".o"); + ret = LLVMCreateMemoryBufferWithContentsOfFile(buf, &obj_data->mem_buf, + &err); + /* remove temp object file */ + snprintf(buf, sizeof(buf), "%s%s", file_name, ".o"); + unlink(buf); + + if (ret != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("create mem buffer with file failed."); + goto fail; + } +#endif /* end of defined(_WIN32) || defined(_WIN32_) */ + } + else { + if (LLVMTargetMachineEmitToMemoryBuffer( + comp_ctx->target_machine, comp_ctx->module, LLVMObjectFile, + &err, &obj_data->mem_buf) + != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("llvm emit to memory buffer failed."); + goto fail; + } + } + + if (!(obj_data->binary = LLVMCreateBinary(obj_data->mem_buf, NULL, &err))) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("llvm create binary failed."); + goto fail; + } + + /* Create wasm feature flags form compile options */ + obj_data->target_info.feature_flags = 0; + if (comp_ctx->enable_simd) { + obj_data->target_info.feature_flags |= WASM_FEATURE_SIMD_128BIT; + } + if (comp_ctx->enable_bulk_memory) { + obj_data->target_info.feature_flags |= WASM_FEATURE_BULK_MEMORY; + } + if (comp_ctx->enable_thread_mgr) { + obj_data->target_info.feature_flags |= WASM_FEATURE_MULTI_THREAD; + } + if (comp_ctx->enable_ref_types) { + obj_data->target_info.feature_flags |= WASM_FEATURE_REF_TYPES; + } + if (comp_ctx->enable_gc) { + obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION; + } + + bh_print_time("Begin to resolve object file info"); + + /* resolve target info/text/relocations/functions */ + if (!aot_resolve_target_info(comp_ctx, obj_data) + || !aot_resolve_text(obj_data) || !aot_resolve_literal(obj_data) + || !aot_resolve_object_data_sections(obj_data) + || !aot_resolve_functions(comp_ctx, obj_data) + || !aot_resolve_object_relocation_groups(obj_data)) + goto fail; + + return obj_data; + +fail: + aot_obj_data_destroy(obj_data); + return NULL; +} + +uint8 * +aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data, + uint32 *p_aot_file_size) +{ + AOTObjectData *obj_data = aot_obj_data_create(comp_ctx); + uint8 *aot_file_buf, *buf, *buf_end; + uint32 aot_file_size, offset = 0; + + if (!obj_data) + return NULL; + + aot_file_size = get_aot_file_size(comp_ctx, comp_data, obj_data); + if (aot_file_size == 0) { + aot_set_last_error("get aot file size failed"); + goto fail1; + } + + if (!(buf = aot_file_buf = wasm_runtime_malloc(aot_file_size))) { + aot_set_last_error("allocate memory failed."); + goto fail1; + } + + memset(aot_file_buf, 0, aot_file_size); + buf_end = buf + aot_file_size; + + if (!aot_emit_file_header(buf, buf_end, &offset, comp_data, obj_data) + || !aot_emit_target_info_section(buf, buf_end, &offset, comp_data, + obj_data) + || !aot_emit_init_data_section(buf, buf_end, &offset, comp_ctx, + comp_data, obj_data) + || !aot_emit_text_section(buf, buf_end, &offset, comp_data, obj_data) + || !aot_emit_func_section(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data) + || !aot_emit_export_section(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data) + || !aot_emit_relocation_section(buf, buf_end, &offset, comp_ctx, + comp_data, obj_data) + || !aot_emit_native_symbol(buf, buf_end, &offset, comp_ctx) + || !aot_emit_custom_sections(buf, buf_end, &offset, comp_data, comp_ctx) +#if WASM_ENABLE_STRINGREF != 0 + || !aot_emit_string_literal_section(buf, buf_end, &offset, comp_data, + comp_ctx) +#endif + ) + goto fail2; + +#if 0 + dump_buf(buf, offset, "sections"); +#endif + + if (offset != aot_file_size) { + aot_set_last_error("emit aot file failed."); + goto fail2; + } + + *p_aot_file_size = aot_file_size; + + aot_obj_data_destroy(obj_data); + return aot_file_buf; + +fail2: + wasm_runtime_free(aot_file_buf); + +fail1: + aot_obj_data_destroy(obj_data); + return NULL; +} + +bool +aot_emit_aot_file(AOTCompContext *comp_ctx, AOTCompData *comp_data, + const char *file_name) +{ + uint8 *aot_file_buf; + uint32 aot_file_size; + bool ret = false; + FILE *file; + + bh_print_time("Begin to emit AOT file"); + + if (!(aot_file_buf = + aot_emit_aot_file_buf(comp_ctx, comp_data, &aot_file_size))) { + return false; + } + + /* write buffer to file */ + if (!(file = fopen(file_name, "wb"))) { + aot_set_last_error("open or create aot file failed."); + goto fail1; + } + if (!fwrite(aot_file_buf, aot_file_size, 1, file)) { + aot_set_last_error("write to aot file failed."); + goto fail2; + } + + ret = true; + +fail2: + fclose(file); + +fail1: + wasm_runtime_free(aot_file_buf); + + return ret; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_compare.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_compare.c new file mode 100644 index 0000000..c57bdee --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_compare.c @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_compare.h" +#include "../aot/aot_intrinsic.h" + +static bool +int_cond_to_llvm_op(IntCond cond, LLVMIntPredicate *op) +{ + if (cond < INT_EQZ || cond > INT_GE_U) + return false; + + switch (cond) { + case INT_EQZ: + case INT_EQ: + *op = LLVMIntEQ; + break; + case INT_NE: + *op = LLVMIntNE; + break; + case INT_LT_S: + *op = LLVMIntSLT; + break; + case INT_LT_U: + *op = LLVMIntULT; + break; + case INT_GT_S: + *op = LLVMIntSGT; + break; + case INT_GT_U: + *op = LLVMIntUGT; + break; + case INT_LE_S: + *op = LLVMIntSLE; + break; + case INT_LE_U: + *op = LLVMIntULE; + break; + case INT_GE_S: + *op = LLVMIntSGE; + break; + case INT_GE_U: + *op = LLVMIntUGE; + break; + default: + return false; + } + + return true; +} + +static bool +float_cond_to_llvm_op(FloatCond cond, LLVMRealPredicate *op) +{ + if (cond < FLOAT_EQ || cond > FLOAT_GE) + return false; + + switch (cond) { + case FLOAT_EQ: + *op = LLVMRealOEQ; + break; + case FLOAT_NE: + *op = LLVMRealUNE; + break; + case FLOAT_LT: + *op = LLVMRealOLT; + break; + case FLOAT_GT: + *op = LLVMRealOGT; + break; + case FLOAT_LE: + *op = LLVMRealOLE; + break; + case FLOAT_GE: + *op = LLVMRealOGE; + break; + default: + return false; + } + + return true; +} + +bool +aot_compile_op_i32_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntCond cond) +{ + LLVMIntPredicate op; + LLVMValueRef lhs, rhs, res; + + if (!int_cond_to_llvm_op(cond, &op)) { + aot_set_last_error("invalid WASM condition opcode"); + return false; + } + + if (cond == INT_EQZ) + rhs = I32_ZERO; + else + POP_I32(rhs); + + POP_I32(lhs); + + if (!(res = LLVMBuildICmp(comp_ctx->builder, op, lhs, rhs, "i32_cmp"))) { + aot_set_last_error("llvm build compare failed."); + return false; + } + + PUSH_COND(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_i64_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntCond cond) +{ + LLVMIntPredicate op; + LLVMValueRef lhs, rhs, res; + + if (!int_cond_to_llvm_op(cond, &op)) { + aot_set_last_error("invalid WASM condition opcode"); + return false; + } + + if (cond == INT_EQZ) + rhs = I64_CONST(0); + else + POP_I64(rhs); + + POP_I64(lhs); + + if (!(res = LLVMBuildICmp(comp_ctx->builder, op, lhs, rhs, "i64_cmp"))) { + aot_set_last_error("llvm build compare failed."); + return false; + } + + PUSH_COND(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_f32_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatCond cond) +{ + LLVMRealPredicate op; + LLVMValueRef lhs, rhs, res; + + if (!float_cond_to_llvm_op(cond, &op)) { + aot_set_last_error("invalid WASM condition opcode"); + return false; + } + + POP_F32(rhs); + POP_F32(lhs); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, "f32_cmp")) { + LLVMTypeRef param_types[3]; + LLVMValueRef opcond = LLVMConstInt(I32_TYPE, cond, true); + param_types[0] = I32_TYPE; + param_types[1] = F32_TYPE; + param_types[2] = F32_TYPE; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, "f32_cmp", I32_TYPE, + param_types, 3, opcond, lhs, rhs); + if (!res) { + goto fail; + } + res = LLVMBuildIntCast(comp_ctx->builder, res, INT1_TYPE, "bit_cast"); + } + else { + res = LLVMBuildFCmp(comp_ctx->builder, op, lhs, rhs, "f32_cmp"); + } + + if (!res) { + aot_set_last_error("llvm build compare failed."); + return false; + } + + PUSH_COND(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatCond cond) +{ + LLVMRealPredicate op; + LLVMValueRef lhs, rhs, res; + + if (!float_cond_to_llvm_op(cond, &op)) { + aot_set_last_error("invalid WASM condition opcode"); + return false; + } + + POP_F64(rhs); + POP_F64(lhs); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, "f64_cmp")) { + LLVMTypeRef param_types[3]; + LLVMValueRef opcond = LLVMConstInt(I32_TYPE, cond, true); + param_types[0] = I32_TYPE; + param_types[1] = F64_TYPE; + param_types[2] = F64_TYPE; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, "f64_cmp", I32_TYPE, + param_types, 3, opcond, lhs, rhs); + if (!res) { + goto fail; + } + res = LLVMBuildIntCast(comp_ctx->builder, res, INT1_TYPE, "bit_cast"); + } + else { + res = LLVMBuildFCmp(comp_ctx->builder, op, lhs, rhs, "f64_cmp"); + } + + if (!res) { + aot_set_last_error("llvm build compare failed."); + return false; + } + + PUSH_COND(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_ref_eq(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef gc_obj1 = NULL, gc_obj2 = NULL, res; + + POP_GC_REF(gc_obj1); + POP_GC_REF(gc_obj2); + + /* LLVM pointer values pointers are compared using LLVMBuildICmp */ + res = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, gc_obj1, gc_obj2, + "cmp_gc_obj_eq"); + + if (!res) { + aot_set_last_error("llvm build compare failed."); + return false; + } + + PUSH_COND(res); + + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_compare.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_compare.h new file mode 100644 index 0000000..f0bfa8a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_compare.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_COMPARE_H_ +#define _AOT_EMIT_COMPARE_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_i32_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntCond cond); + +bool +aot_compile_op_i64_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntCond cond); + +bool +aot_compile_op_f32_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatCond cond); + +bool +aot_compile_op_f64_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatCond cond); + +#if WASM_ENABLE_GC != 0 + +bool +aot_compile_op_ref_eq(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_COMPARE_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_const.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_const.c new file mode 100644 index 0000000..4b38aa9 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_const.c @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_const.h" +#include "../aot/aot_intrinsic.h" + +bool +aot_compile_op_i32_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 i32_const) +{ + LLVMValueRef value; + + if (comp_ctx->is_indirect_mode + && aot_intrinsic_check_capability(comp_ctx, "i32.const")) { + WASMValue wasm_value; + wasm_value.i32 = i32_const; + value = aot_load_const_from_table(comp_ctx, func_ctx->native_symbol, + &wasm_value, VALUE_TYPE_I32); + if (!value) { + return false; + } + } + else { + value = I32_CONST((uint32)i32_const); + CHECK_LLVM_CONST(value); + } + + PUSH_I32(value); + return true; +fail: + return false; +} + +bool +aot_compile_op_i64_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int64 i64_const) +{ + LLVMValueRef value; + + if (comp_ctx->is_indirect_mode + && aot_intrinsic_check_capability(comp_ctx, "i64.const")) { + WASMValue wasm_value; + wasm_value.i64 = i64_const; + value = aot_load_const_from_table(comp_ctx, func_ctx->native_symbol, + &wasm_value, VALUE_TYPE_I64); + if (!value) { + return false; + } + } + else { + value = I64_CONST((uint64)i64_const); + CHECK_LLVM_CONST(value); + } + + PUSH_I64(value); + return true; +fail: + return false; +} + +bool +aot_compile_op_f32_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + float32 f32_const) +{ + LLVMValueRef alloca, value; + + if (!isnan(f32_const)) { + if (comp_ctx->is_indirect_mode + && aot_intrinsic_check_capability(comp_ctx, "f32.const")) { + WASMValue wasm_value; + memcpy(&wasm_value.f32, &f32_const, sizeof(float32)); + value = aot_load_const_from_table(comp_ctx, func_ctx->native_symbol, + &wasm_value, VALUE_TYPE_F32); + if (!value) { + return false; + } + PUSH_F32(value); + } + else { + value = F32_CONST(f32_const); + CHECK_LLVM_CONST(value); + PUSH_F32(value); + } + } + else { + int32 i32_const; + memcpy(&i32_const, &f32_const, sizeof(int32)); + if (!(alloca = + LLVMBuildAlloca(comp_ctx->builder, I32_TYPE, "i32_ptr"))) { + aot_set_last_error("llvm build alloca failed."); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, I32_CONST((uint32)i32_const), + alloca)) { + aot_set_last_error("llvm build store failed."); + return false; + } + if (!(alloca = LLVMBuildBitCast(comp_ctx->builder, alloca, F32_PTR_TYPE, + "f32_ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + return false; + } + if (!(value = + LLVMBuildLoad2(comp_ctx->builder, F32_TYPE, alloca, ""))) { + aot_set_last_error("llvm build load failed."); + return false; + } + PUSH_F32(value); + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + float64 f64_const) +{ + LLVMValueRef alloca, value; + + if (!isnan(f64_const)) { + if (comp_ctx->is_indirect_mode + && aot_intrinsic_check_capability(comp_ctx, "f64.const")) { + WASMValue wasm_value; + memcpy(&wasm_value.f64, &f64_const, sizeof(float64)); + value = aot_load_const_from_table(comp_ctx, func_ctx->native_symbol, + &wasm_value, VALUE_TYPE_F64); + if (!value) { + return false; + } + PUSH_F64(value); + } + else { + value = F64_CONST(f64_const); + CHECK_LLVM_CONST(value); + PUSH_F64(value); + } + } + else { + int64 i64_const; + memcpy(&i64_const, &f64_const, sizeof(int64)); + if (!(alloca = + LLVMBuildAlloca(comp_ctx->builder, I64_TYPE, "i64_ptr"))) { + aot_set_last_error("llvm build alloca failed."); + return false; + } + value = I64_CONST((uint64)i64_const); + CHECK_LLVM_CONST(value); + if (!LLVMBuildStore(comp_ctx->builder, value, alloca)) { + aot_set_last_error("llvm build store failed."); + return false; + } + if (!(alloca = LLVMBuildBitCast(comp_ctx->builder, alloca, F64_PTR_TYPE, + "f64_ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + return false; + } + if (!(value = + LLVMBuildLoad2(comp_ctx->builder, F64_TYPE, alloca, ""))) { + aot_set_last_error("llvm build load failed."); + return false; + } + PUSH_F64(value); + } + + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_const.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_const.h new file mode 100644 index 0000000..0b56cb1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_const.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_CONST_H_ +#define _AOT_EMIT_CONST_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_i32_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 i32_const); + +bool +aot_compile_op_i64_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int64 i64_const); + +bool +aot_compile_op_f32_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + float32 f32_const); + +bool +aot_compile_op_f64_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + float64 f64_const); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_CONST_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_control.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_control.c new file mode 100644 index 0000000..24511ff --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_control.c @@ -0,0 +1,1701 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_control.h" +#include "aot_compiler.h" +#include "aot_emit_exception.h" +#if WASM_ENABLE_GC != 0 +#include "aot_emit_gc.h" +#endif +#include "../aot/aot_runtime.h" +#include "../interpreter/wasm_loader.h" + +#if WASM_ENABLE_DEBUG_AOT != 0 +#include "debug/dwarf_extractor.h" +#endif + +static char *block_name_prefix[] = { "block", "loop", "if" }; +static char *block_name_suffix[] = { "begin", "else", "end" }; + +/* clang-format off */ +enum { + LABEL_BEGIN = 0, + LABEL_ELSE, + LABEL_END +}; +/* clang-format on */ + +static void +format_block_name(char *name, uint32 name_size, uint32 block_index, + uint32 label_type, uint32 label_id) +{ + if (label_type != LABEL_TYPE_FUNCTION) + snprintf(name, name_size, "%s%d%s%s", block_name_prefix[label_type], + block_index, "_", block_name_suffix[label_id]); + else + snprintf(name, name_size, "%s", "func_end"); +} + +#define CREATE_BLOCK(new_llvm_block, name) \ + do { \ + if (!(new_llvm_block = LLVMAppendBasicBlockInContext( \ + comp_ctx->context, func_ctx->func, name))) { \ + aot_set_last_error("add LLVM basic block failed."); \ + goto fail; \ + } \ + } while (0) + +#define CURR_BLOCK() LLVMGetInsertBlock(comp_ctx->builder) + +#define MOVE_BLOCK_AFTER(llvm_block, llvm_block_after) \ + LLVMMoveBasicBlockAfter(llvm_block, llvm_block_after) + +#define MOVE_BLOCK_AFTER_CURR(llvm_block) \ + LLVMMoveBasicBlockAfter(llvm_block, CURR_BLOCK()) + +#define MOVE_BLOCK_BEFORE(llvm_block, llvm_block_before) \ + LLVMMoveBasicBlockBefore(llvm_block, llvm_block_before) + +#define BUILD_BR(llvm_block) \ + do { \ + if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \ + aot_set_last_error("llvm build br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \ + block_else)) { \ + aot_set_last_error("llvm build cond br failed."); \ + goto fail; \ + } \ + } while (0) + +#define SET_BUILDER_POS(llvm_block) \ + LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block) + +#define CREATE_RESULT_VALUE_PHIS(block) \ + do { \ + if (block->result_count && !block->result_phis) { \ + uint32 _i; \ + uint64 _size; \ + LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \ + /* Allocate memory */ \ + _size = sizeof(LLVMValueRef) * (uint64)block->result_count; \ + if (_size >= UINT32_MAX \ + || !(block->result_phis = \ + wasm_runtime_malloc((uint32)_size))) { \ + aot_set_last_error("allocate memory failed."); \ + goto fail; \ + } \ + SET_BUILDER_POS(block->llvm_end_block); \ + for (_i = 0; _i < block->result_count; _i++) { \ + if (!(block->result_phis[_i] = LLVMBuildPhi( \ + comp_ctx->builder, \ + TO_LLVM_TYPE(block->result_types[_i]), "phi"))) { \ + aot_set_last_error("llvm build phi failed."); \ + goto fail; \ + } \ + } \ + SET_BUILDER_POS(_block_curr); \ + } \ + } while (0) + +#define ADD_TO_RESULT_PHIS(block, value, idx) \ + do { \ + LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \ + LLVMTypeRef phi_ty = LLVMTypeOf(block->result_phis[idx]); \ + LLVMTypeRef value_ty = LLVMTypeOf(value); \ + bh_assert(LLVMGetTypeKind(phi_ty) == LLVMGetTypeKind(value_ty)); \ + bh_assert(LLVMGetTypeContext(phi_ty) == LLVMGetTypeContext(value_ty)); \ + LLVMAddIncoming(block->result_phis[idx], &value, &_block_curr, 1); \ + (void)phi_ty; \ + (void)value_ty; \ + } while (0) + +#define BUILD_ICMP(op, left, right, res, name) \ + do { \ + if (!(res = \ + LLVMBuildICmp(comp_ctx->builder, op, left, right, name))) { \ + aot_set_last_error("llvm build icmp failed."); \ + goto fail; \ + } \ + } while (0) + +#define ADD_TO_PARAM_PHIS(block, value, idx) \ + do { \ + LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \ + LLVMAddIncoming(block->param_phis[idx], &value, &_block_curr, 1); \ + } while (0) + +static LLVMBasicBlockRef +find_next_llvm_end_block(AOTBlock *block) +{ + block = block->prev; + while (block && !block->llvm_end_block) + block = block->prev; + return block ? block->llvm_end_block : NULL; +} + +static AOTBlock * +get_target_block(AOTFuncContext *func_ctx, uint32 br_depth) +{ + uint32 i = br_depth; + AOTBlock *block = func_ctx->block_stack.block_list_end; + + while (i-- > 0 && block) { + block = block->prev; + } + + if (!block) { + aot_set_last_error("WASM block stack underflow."); + return NULL; + } + return block; +} + +static void +clear_frame_locals(AOTCompFrame *aot_frame) +{ + uint32 i; + + for (i = 0; i < aot_frame->max_local_cell_num; i++) { + aot_frame->lp[i].dirty = 0; + aot_frame->lp[i].value = NULL; + if (aot_frame->comp_ctx->enable_gc) + /* Mark the ref flag as committed */ + aot_frame->lp[i].committed_ref = aot_frame->lp[i].ref + 1; + } +} + +static void +restore_frame_sp_for_op_else(AOTBlock *block, AOTCompFrame *aot_frame) +{ + uint32 all_cell_num = + aot_frame->max_local_cell_num + aot_frame->max_stack_cell_num; + AOTValueSlot *p_end = aot_frame->lp + all_cell_num, *p; + + /* Reset all the value slots from current frame sp for the else + branch since they be the same as starting to translate the + if branch */ + for (p = block->frame_sp_begin; p < p_end; p++) { + p->dirty = 0; + p->value = NULL; + p->type = 0; + if (aot_frame->comp_ctx->enable_gc) { + p->ref = 0; + p->committed_ref = 1; + } + } + + bh_assert(aot_frame->sp >= block->frame_sp_begin); + aot_frame->sp = block->frame_sp_begin; +} + +static void +restore_frame_sp_for_op_end(AOTBlock *block, AOTCompFrame *aot_frame) +{ + uint32 all_cell_num = + aot_frame->max_local_cell_num + aot_frame->max_stack_cell_num; + AOTValueSlot *p_end = aot_frame->lp + all_cell_num, *p; + + bh_assert(block->frame_sp_max_reached >= block->frame_sp_begin); + + /* Reset all the value slots from current frame sp to be same as + starting to translate this block, except for the frame ref + flags: set the flags to uncommitted before the max frame sp + ever reached, set the flags to committed non-ref after that */ + for (p = block->frame_sp_begin; p < p_end; p++) { + p->dirty = 0; + p->value = NULL; + p->type = 0; + if (aot_frame->comp_ctx->enable_gc) { + p->ref = 0; + if (p < block->frame_sp_max_reached) + p->committed_ref = 0; + else + p->committed_ref = 1; + } + } + + bh_assert(aot_frame->sp >= block->frame_sp_begin); + aot_frame->sp = block->frame_sp_begin; +} + +static bool +handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block = func_ctx->block_stack.block_list_end; + AOTBlock *block_prev; + AOTCompFrame *aot_frame = comp_ctx->aot_frame; + uint8 *frame_ip = NULL; + uint32 i; + AOTFuncType *func_type; + LLVMValueRef ret; +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMMetadataRef return_location; +#endif + + aot_checked_addr_list_destroy(func_ctx); + bh_assert(block); + +#if WASM_ENABLE_DEBUG_AOT != 0 + return_location = dwarf_gen_location( + comp_ctx, func_ctx, + (*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); +#endif + + if (aot_frame) { + /* Clear frame local variables since they have been committed */ + clear_frame_locals(aot_frame); + } + + if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block + && *p_frame_ip <= block->wasm_code_else) { + /* Clear value stack and start to translate else branch */ + aot_value_stack_destroy(comp_ctx, &block->value_stack); + + if (aot_frame) { + /* Restore the frame sp */ + restore_frame_sp_for_op_else(block, aot_frame); + } + + /* Recover parameters of else branch */ + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = block->wasm_code_else + 1; + return true; + } + + while (block && !block->is_reachable) { + block_prev = block->prev; + block = aot_block_stack_pop(&func_ctx->block_stack); + + if (block->label_type == LABEL_TYPE_IF) { + if (block->llvm_else_block && !block->skip_wasm_code_else + && *p_frame_ip <= block->wasm_code_else) { + /* Clear value stack and start to translate else branch */ + aot_value_stack_destroy(comp_ctx, &block->value_stack); + + if (aot_frame) { + /* Restore the frame sp */ + restore_frame_sp_for_op_else(block, aot_frame); + } + + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = block->wasm_code_else + 1; + /* Push back the block */ + aot_block_stack_push(&func_ctx->block_stack, block); + /* Recover parameters of else branch */ + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); + return true; + } + else if (block->llvm_end_block) { + /* Remove unreachable basic block */ + LLVMDeleteBasicBlock(block->llvm_end_block); + block->llvm_end_block = NULL; + } + } + + frame_ip = block->wasm_code_end; + aot_block_destroy(comp_ctx, block); + block = block_prev; + } + + if (!block) { + *p_frame_ip = frame_ip + 1; + return true; + } + + if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block + && !block->skip_wasm_code_else + && *p_frame_ip <= block->wasm_code_else) { + /* Clear value stack and start to translate else branch */ + aot_value_stack_destroy(comp_ctx, &block->value_stack); + + if (aot_frame) { + /* Restore the frame sp */ + restore_frame_sp_for_op_else(block, aot_frame); + } + + /* Recover parameters of else branch */ + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = block->wasm_code_else + 1; + return true; + } + + *p_frame_ip = block->wasm_code_end + 1; + SET_BUILDER_POS(block->llvm_end_block); + + /* Pop block, push its return value, and destroy the block */ + block = aot_block_stack_pop(&func_ctx->block_stack); + + if (aot_frame) { + /* Restore the frame sp */ + restore_frame_sp_for_op_end(block, aot_frame); + } + + func_type = func_ctx->aot_func->func_type; + for (i = 0; i < block->result_count; i++) { + bh_assert(block->result_phis[i]); + if (block->label_type != LABEL_TYPE_FUNCTION) { + PUSH(block->result_phis[i], block->result_types[i]); + } + else { + /* Store extra return values to function parameters */ + if (i != 0) { + LLVMValueRef res; + uint32 param_index = func_type->param_count + i; + if (!(res = LLVMBuildStore( + comp_ctx->builder, block->result_phis[i], + LLVMGetParam(func_ctx->func, param_index)))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + LLVMSetAlignment(res, 1); + } + } + } + if (block->label_type == LABEL_TYPE_FUNCTION) { + if (block->result_count) { + /* Return the first return value */ + if (!(ret = + LLVMBuildRet(comp_ctx->builder, block->result_phis[0]))) { + aot_set_last_error("llvm build return failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + if (return_location != NULL) { + LLVMInstructionSetDebugLoc(ret, return_location); + } +#endif + } + else { + if (!(ret = LLVMBuildRetVoid(comp_ctx->builder))) { + aot_set_last_error("llvm build return void failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + if (return_location != NULL) { + LLVMInstructionSetDebugLoc(ret, return_location); + } +#endif + } + } + aot_block_destroy(comp_ctx, block); + return true; +fail: + return false; +} + +static bool +push_aot_block_to_stack_and_pass_params(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + AOTBlock *block) +{ + uint32 i, param_index; + LLVMValueRef value, br_inst; + uint64 size; + char name[32]; + LLVMBasicBlockRef block_curr = CURR_BLOCK(); + + if (block->param_count) { + size = sizeof(LLVMValueRef) * (uint64)block->param_count; + if (size >= UINT32_MAX + || !(block->param_phis = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return false; + } + + if (block->label_type == LABEL_TYPE_IF && !block->skip_wasm_code_else + && !(block->else_param_phis = wasm_runtime_malloc((uint32)size))) { + wasm_runtime_free(block->param_phis); + block->param_phis = NULL; + aot_set_last_error("allocate memory failed."); + return false; + } + + /* Create param phis */ + for (i = 0; i < block->param_count; i++) { + SET_BUILDER_POS(block->llvm_entry_block); + snprintf(name, sizeof(name), "%s%d_phi%d", + block_name_prefix[block->label_type], block->block_index, + i); + if (!(block->param_phis[i] = LLVMBuildPhi( + comp_ctx->builder, TO_LLVM_TYPE(block->param_types[i]), + name))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + + if (block->label_type == LABEL_TYPE_IF + && !block->skip_wasm_code_else && block->llvm_else_block) { + /* Build else param phis */ + SET_BUILDER_POS(block->llvm_else_block); + snprintf(name, sizeof(name), "else%d_phi%d", block->block_index, + i); + if (!(block->else_param_phis[i] = LLVMBuildPhi( + comp_ctx->builder, + TO_LLVM_TYPE(block->param_types[i]), name))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + } + } + + /* At this point, the branch instruction was already built to jump to + * the new BB, to avoid generating zext instruction from the popped + * operand that would come after branch instruction, we should position + * the builder before the last branch instruction */ + br_inst = LLVMGetLastInstruction(block_curr); + bh_assert(LLVMGetInstructionOpcode(br_inst) == LLVMBr); + LLVMPositionBuilderBefore(comp_ctx->builder, br_inst); + + /* Pop param values from current block's + * value stack and add to param phis. + */ + for (i = 0; i < block->param_count; i++) { + param_index = block->param_count - 1 - i; + POP(value, block->param_types[param_index]); + if (block->llvm_entry_block) + /* Only add incoming phis if the entry block was created */ + ADD_TO_PARAM_PHIS(block, value, param_index); + if (block->label_type == LABEL_TYPE_IF + && !block->skip_wasm_code_else) { + if (block->llvm_else_block) { + /* has else branch, add to else param phis */ + LLVMAddIncoming(block->else_param_phis[param_index], &value, + &block_curr, 1); + } + else { + /* no else branch, add to result phis */ + CREATE_RESULT_VALUE_PHIS(block); + ADD_TO_RESULT_PHIS(block, value, param_index); + } + } + } + } + + /* Push the new block to block stack */ + aot_block_stack_push(&func_ctx->block_stack, block); + if (comp_ctx->aot_frame) { + block->frame_sp_begin = block->frame_sp_max_reached = + comp_ctx->aot_frame->sp; + } + + /* Push param phis to the new block */ + for (i = 0; i < block->param_count; i++) { + if (block->llvm_entry_block) + /* Push param phis if the entry basic block was created */ + PUSH(block->param_phis[i], block->param_types[i]); + else { + bh_assert(block->label_type == LABEL_TYPE_IF + && block->llvm_else_block && block->else_param_phis + && !block->skip_wasm_code_else); + /* Push else param phis if we start to translate the + else branch */ + PUSH(block->else_param_phis[i], block->param_types[i]); + } + } + + return true; + +fail: + if (block->param_phis) { + wasm_runtime_free(block->param_phis); + block->param_phis = NULL; + } + if (block->else_param_phis) { + wasm_runtime_free(block->else_param_phis); + block->else_param_phis = NULL; + } + return false; +} + +bool +aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip, uint8 *frame_ip_end, uint32 label_type, + uint32 param_count, uint8 *param_types, + uint32 result_count, uint8 *result_types) +{ + BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE]; + AOTBlock *block; + uint8 *else_addr, *end_addr; + LLVMValueRef value; + char name[32]; + + /* Check block stack */ + if (!func_ctx->block_stack.block_list_end) { + aot_set_last_error("WASM block stack underflow."); + return false; + } + + memset(block_addr_cache, 0, sizeof(block_addr_cache)); + + /* Get block info */ + if (!(wasm_loader_find_block_addr( + NULL, (BlockAddr *)block_addr_cache, *p_frame_ip, frame_ip_end, + (uint8)label_type, &else_addr, &end_addr))) { + aot_set_last_error("find block end addr failed."); + return false; + } + + /* Allocate memory */ + if (!(block = wasm_runtime_malloc(sizeof(AOTBlock)))) { + aot_set_last_error("allocate memory failed."); + return false; + } + memset(block, 0, sizeof(AOTBlock)); + if (param_count + && !(block->param_types = wasm_runtime_malloc(param_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + if (result_count) { + if (!(block->result_types = wasm_runtime_malloc(result_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + } + + /* Init aot block data */ + block->label_type = label_type; + block->param_count = param_count; + if (param_count) { + bh_memcpy_s(block->param_types, param_count, param_types, param_count); + } + block->result_count = result_count; + if (result_count) { + bh_memcpy_s(block->result_types, result_count, result_types, + result_count); + } + block->wasm_code_else = else_addr; + block->wasm_code_end = end_addr; + block->block_index = func_ctx->block_stack.block_index[label_type]; + func_ctx->block_stack.block_index[label_type]++; + + if (comp_ctx->aot_frame) { + if (label_type != LABEL_TYPE_BLOCK && comp_ctx->enable_gc + && !aot_gen_commit_values(comp_ctx->aot_frame)) { + goto fail; + } + } + + if (label_type == LABEL_TYPE_BLOCK || label_type == LABEL_TYPE_LOOP) { + /* Create block */ + format_block_name(name, sizeof(name), block->block_index, label_type, + LABEL_BEGIN); + CREATE_BLOCK(block->llvm_entry_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); + /* Jump to the entry block */ + BUILD_BR(block->llvm_entry_block); + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block)) + goto fail; + /* Start to translate the block */ + SET_BUILDER_POS(block->llvm_entry_block); + if (label_type == LABEL_TYPE_LOOP) + aot_checked_addr_list_destroy(func_ctx); + } + else if (label_type == LABEL_TYPE_IF) { + POP_COND(value); + + if (LLVMIsUndef(value) +#if LLVM_VERSION_NUMBER >= 12 + || LLVMIsPoison(value) +#endif + ) { + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, + false, NULL, NULL))) { + goto fail; + } + aot_block_destroy(comp_ctx, block); + return aot_handle_next_reachable_block(comp_ctx, func_ctx, + p_frame_ip); + } + + if (!LLVMIsEfficientConstInt(value)) { + /* Compare value is not constant, create condition br IR */ + /* Create entry block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_BEGIN); + CREATE_BLOCK(block->llvm_entry_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); + + /* Create end block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_END); + CREATE_BLOCK(block->llvm_end_block, name); + MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_entry_block); + + if (else_addr) { + /* Create else block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_ELSE); + CREATE_BLOCK(block->llvm_else_block, name); + MOVE_BLOCK_AFTER(block->llvm_else_block, + block->llvm_entry_block); + /* Create condition br IR */ + BUILD_COND_BR(value, block->llvm_entry_block, + block->llvm_else_block); + } + else { + /* Create condition br IR */ + BUILD_COND_BR(value, block->llvm_entry_block, + block->llvm_end_block); + block->is_reachable = true; + } + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, + block)) + goto fail; + /* Start to translate if branch of BLOCK if */ + SET_BUILDER_POS(block->llvm_entry_block); + } + else { + if ((int32)LLVMConstIntGetZExtValue(value) != 0) { + /* Compare value is not 0, condition is true, else branch of + BLOCK if cannot be reached */ + block->skip_wasm_code_else = true; + /* Create entry block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_BEGIN); + CREATE_BLOCK(block->llvm_entry_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); + /* Jump to the entry block */ + BUILD_BR(block->llvm_entry_block); + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, + block)) + goto fail; + /* Start to translate the if branch */ + SET_BUILDER_POS(block->llvm_entry_block); + } + else { + /* Compare value is not 0, condition is false, if branch of + BLOCK if cannot be reached */ + if (else_addr) { + /* Create else block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_ELSE); + CREATE_BLOCK(block->llvm_else_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_else_block); + /* Jump to the else block */ + BUILD_BR(block->llvm_else_block); + if (!push_aot_block_to_stack_and_pass_params( + comp_ctx, func_ctx, block)) + goto fail; + /* Start to translate the else branch */ + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = else_addr + 1; + } + else { + /* skip the block */ + aot_block_destroy(comp_ctx, block); + *p_frame_ip = end_addr + 1; + } + } + } + } + else { + aot_set_last_error("Invalid block type."); + goto fail; + } + + return true; +fail: + aot_block_destroy(comp_ctx, block); + return false; +} + +bool +aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block = func_ctx->block_stack.block_list_end; + LLVMValueRef value; + AOTCompFrame *aot_frame = comp_ctx->aot_frame; + char name[32]; + uint32 i, result_index; + + /* Check block */ + if (!block) { + aot_set_last_error("WASM block stack underflow."); + return false; + } + if (block->label_type != LABEL_TYPE_IF + || (!block->skip_wasm_code_else && !block->llvm_else_block)) { + aot_set_last_error("Invalid WASM block type."); + return false; + } + + /* Create end block if needed */ + if (!block->llvm_end_block) { + format_block_name(name, sizeof(name), block->block_index, + block->label_type, LABEL_END); + CREATE_BLOCK(block->llvm_end_block, name); + if (block->llvm_else_block) + MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_else_block); + else + MOVE_BLOCK_AFTER_CURR(block->llvm_end_block); + } + + block->is_reachable = true; + + /* Comes from the if branch of BLOCK if */ + CREATE_RESULT_VALUE_PHIS(block); + for (i = 0; i < block->result_count; i++) { + result_index = block->result_count - 1 - i; + POP(value, block->result_types[result_index]); + ADD_TO_RESULT_PHIS(block, value, result_index); + } + + if (aot_frame) { + bh_assert(block->frame_sp_begin == aot_frame->sp); + if (comp_ctx->enable_gc && !aot_gen_commit_values(aot_frame)) { + goto fail; + } + } + + /* Jump to end block */ + BUILD_BR(block->llvm_end_block); + + if (!block->skip_wasm_code_else && block->llvm_else_block) { + /* Clear value stack, recover param values + and start to translate else branch. */ + aot_value_stack_destroy(comp_ctx, &block->value_stack); + + if (comp_ctx->aot_frame) { + clear_frame_locals(aot_frame); + restore_frame_sp_for_op_else(block, aot_frame); + } + + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); + SET_BUILDER_POS(block->llvm_else_block); + aot_checked_addr_list_destroy(func_ctx); + return true; + } + + /* No else branch or no need to translate else branch */ + block->is_reachable = true; + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +bool +aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block; + LLVMValueRef value; + LLVMBasicBlockRef next_llvm_end_block; + char name[32]; + uint32 i, result_index; + + /* Check block stack */ + if (!(block = func_ctx->block_stack.block_list_end)) { + aot_set_last_error("WASM block stack underflow."); + return false; + } + + /* Create the end block */ + if (!block->llvm_end_block) { + format_block_name(name, sizeof(name), block->block_index, + block->label_type, LABEL_END); + CREATE_BLOCK(block->llvm_end_block, name); + if ((next_llvm_end_block = find_next_llvm_end_block(block))) + MOVE_BLOCK_BEFORE(block->llvm_end_block, next_llvm_end_block); + } + + if (comp_ctx->aot_frame) { + if (block->label_type != LABEL_TYPE_FUNCTION && comp_ctx->enable_gc + && !aot_gen_commit_values(comp_ctx->aot_frame)) { + return false; + } + } + + /* Handle block result values */ + CREATE_RESULT_VALUE_PHIS(block); + for (i = 0; i < block->result_count; i++) { + value = NULL; + result_index = block->result_count - 1 - i; + POP(value, block->result_types[result_index]); + bh_assert(value); + ADD_TO_RESULT_PHIS(block, value, result_index); + } + + if (comp_ctx->aot_frame) { + bh_assert(comp_ctx->aot_frame->sp == block->frame_sp_begin); + } + + /* Jump to the end block */ + BUILD_BR(block->llvm_end_block); + + block->is_reachable = true; + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +bool +check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool check_terminate_and_suspend) +{ + LLVMValueRef terminate_addr, terminate_flags, flag, offset, res; + LLVMBasicBlockRef terminate_block, non_terminate_block; + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + bool is_shared_memory = + comp_ctx->comp_data->memories[0].memory_flags & 0x02 ? true : false; + + /* Only need to check the suspend flags when memory is shared since + shared memory must be enabled for multi-threading */ + if (!is_shared_memory) { + return true; + } + + /* Offset of suspend_flags */ + offset = I32_FIVE; + + if (!(terminate_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1, + "terminate_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(terminate_addr = + LLVMBuildBitCast(comp_ctx->builder, terminate_addr, + INT32_PTR_TYPE, "terminate_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(terminate_flags = + LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, terminate_addr, + "terminate_flags"))) { + aot_set_last_error("llvm build LOAD failed"); + return false; + } + /* Set terminate_flags memory accecc to volatile, so that the value + will always be loaded from memory rather than register */ + LLVMSetVolatile(terminate_flags, true); + + if (!(flag = LLVMBuildAnd(comp_ctx->builder, terminate_flags, I32_ONE, + "termination_flag"))) { + aot_set_last_error("llvm build AND failed"); + return false; + } + + CREATE_BLOCK(non_terminate_block, "non_terminate"); + MOVE_BLOCK_AFTER_CURR(non_terminate_block); + + CREATE_BLOCK(terminate_block, "terminate"); + MOVE_BLOCK_AFTER_CURR(terminate_block); + + BUILD_ICMP(LLVMIntEQ, flag, I32_ZERO, res, "flag_terminate"); + BUILD_COND_BR(res, non_terminate_block, terminate_block); + + /* Move builder to terminate block */ + SET_BUILDER_POS(terminate_block); + if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + goto fail; + } + + /* Move builder to non terminate block */ + SET_BUILDER_POS(non_terminate_block); + return true; + +fail: + return false; +} + +bool +aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip) +{ + AOTBlock *block_dst; + LLVMValueRef value_ret, value_param; + LLVMBasicBlockRef next_llvm_end_block; + char name[32]; + uint32 i, param_index, result_index; + + if (!(block_dst = get_target_block(func_ctx, br_depth))) { + return false; + } + + if (comp_ctx->aot_frame) { + if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + if (comp_ctx->enable_thread_mgr) { + /* Commit sp when GC is enabled, don't commit ip */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, + comp_ctx->enable_gc, false)) + return false; + } + } + else { + if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached) + block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp; + } + } + + /* Terminate or suspend current thread only when this is a backward jump */ + if (comp_ctx->enable_thread_mgr + && block_dst->label_type == LABEL_TYPE_LOOP) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + } + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + /* Dest block is Loop block */ + /* Handle Loop parameters */ + for (i = 0; i < block_dst->param_count; i++) { + param_index = block_dst->param_count - 1 - i; + POP(value_param, block_dst->param_types[param_index]); + ADD_TO_PARAM_PHIS(block_dst, value_param, param_index); + } + BUILD_BR(block_dst->llvm_entry_block); + } + else { + /* Dest block is Block/If/Function block */ + /* Create the end block */ + if (!block_dst->llvm_end_block) { + format_block_name(name, sizeof(name), block_dst->block_index, + block_dst->label_type, LABEL_END); + CREATE_BLOCK(block_dst->llvm_end_block, name); + if ((next_llvm_end_block = find_next_llvm_end_block(block_dst))) + MOVE_BLOCK_BEFORE(block_dst->llvm_end_block, + next_llvm_end_block); + } + + block_dst->is_reachable = true; + + /* Handle result values */ + CREATE_RESULT_VALUE_PHIS(block_dst); + for (i = 0; i < block_dst->result_count; i++) { + result_index = block_dst->result_count - 1 - i; + POP(value_ret, block_dst->result_types[result_index]); + ADD_TO_RESULT_PHIS(block_dst, value_ret, result_index); + } + /* Jump to the end block */ + BUILD_BR(block_dst->llvm_end_block); + } + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +static bool +aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, LLVMValueRef value_cmp, + uint8 **p_frame_ip) +{ + AOTBlock *block_dst; + LLVMValueRef value, *values = NULL; + LLVMBasicBlockRef llvm_else_block, next_llvm_end_block; + char name[32]; + uint32 i, param_index, result_index; + uint64 size; + + if (!(block_dst = get_target_block(func_ctx, br_depth))) { + return false; + } + + if (comp_ctx->aot_frame) { + if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + if (comp_ctx->enable_thread_mgr) { + /* Commit sp when GC is enabled, don't commit ip */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, + comp_ctx->enable_gc, false)) + return false; + } + } + else { + if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached) + block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp; + } + } + + /* Terminate or suspend current thread only when this is + a backward jump */ + if (comp_ctx->enable_thread_mgr + && block_dst->label_type == LABEL_TYPE_LOOP) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + } + + if (LLVMIsUndef(value_cmp) +#if LLVM_VERSION_NUMBER >= 12 + || LLVMIsPoison(value_cmp) +#endif + ) { + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, + false, NULL, NULL))) { + goto fail; + } + return aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); + } + + if (!LLVMIsEfficientConstInt(value_cmp)) { + /* Compare value is not constant, create condition br IR */ + + /* Create llvm else block */ + CREATE_BLOCK(llvm_else_block, "br_if_else"); + MOVE_BLOCK_AFTER_CURR(llvm_else_block); + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + /* Dest block is Loop block */ + /* Handle Loop parameters */ + if (block_dst->param_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + for (i = 0; i < block_dst->param_count; i++) { + param_index = block_dst->param_count - 1 - i; + POP(value, block_dst->param_types[param_index]); + ADD_TO_PARAM_PHIS(block_dst, value, param_index); + values[param_index] = value; + } + for (i = 0; i < block_dst->param_count; i++) { + PUSH(values[i], block_dst->param_types[i]); + } + wasm_runtime_free(values); + values = NULL; + } + + BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block, + llvm_else_block); + + /* Move builder to else block */ + SET_BUILDER_POS(llvm_else_block); + } + else { + /* Dest block is Block/If/Function block */ + /* Create the end block */ + if (!block_dst->llvm_end_block) { + format_block_name(name, sizeof(name), block_dst->block_index, + block_dst->label_type, LABEL_END); + CREATE_BLOCK(block_dst->llvm_end_block, name); + if ((next_llvm_end_block = find_next_llvm_end_block(block_dst))) + MOVE_BLOCK_BEFORE(block_dst->llvm_end_block, + next_llvm_end_block); + } + + /* Set reachable flag and create condition br IR */ + block_dst->is_reachable = true; + + /* Handle result values */ + if (block_dst->result_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + CREATE_RESULT_VALUE_PHIS(block_dst); + for (i = 0; i < block_dst->result_count; i++) { + result_index = block_dst->result_count - 1 - i; + POP(value, block_dst->result_types[result_index]); + values[result_index] = value; + ADD_TO_RESULT_PHIS(block_dst, value, result_index); + } + for (i = 0; i < block_dst->result_count; i++) { + PUSH(values[i], block_dst->result_types[i]); + } + wasm_runtime_free(values); + values = NULL; + } + + /* Condition jump to end block */ + BUILD_COND_BR(value_cmp, block_dst->llvm_end_block, + llvm_else_block); + + /* Move builder to else block */ + SET_BUILDER_POS(llvm_else_block); + } + } + else { + if ((int32)LLVMConstIntGetZExtValue(value_cmp) != 0) { + /* Compare value is not 0, condition is true, same as op_br */ + return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip); + } + else { + /* Compare value is not 0, condition is false, skip br_if */ + return true; + } + } + return true; +fail: + if (values) + wasm_runtime_free(values); + return false; +} + +bool +aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip) +{ + LLVMValueRef value_cmp; + + POP_COND(value_cmp); + + return aot_compile_conditional_br(comp_ctx, func_ctx, br_depth, value_cmp, + p_frame_ip); +fail: + return false; +} + +bool +aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 *br_depths, uint32 br_count, uint8 **p_frame_ip) +{ + uint32 i, j; + LLVMValueRef value_switch, value_cmp, value_case, value, *values = NULL; + LLVMBasicBlockRef default_llvm_block = NULL, target_llvm_block; + LLVMBasicBlockRef next_llvm_end_block; + AOTBlock *target_block; + uint32 br_depth, depth_idx; + uint32 param_index, result_index; + uint64 size; + char name[32]; + + POP_I32(value_cmp); + + if (LLVMIsUndef(value_cmp) +#if LLVM_VERSION_NUMBER >= 12 + || LLVMIsPoison(value_cmp) +#endif + ) { + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, + false, NULL, NULL))) { + goto fail; + } + return aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); + } + + if (!LLVMIsEfficientConstInt(value_cmp)) { + if (comp_ctx->aot_frame) { + if (comp_ctx->enable_gc + && !aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (comp_ctx->enable_thread_mgr) { + /* Commit sp when GC is enabled, don't commit ip */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, + comp_ctx->enable_gc, false)) + return false; + } + + for (i = 0; i <= br_count; i++) { + target_block = get_target_block(func_ctx, br_depths[i]); + if (!target_block) + return false; + if (target_block->label_type != LABEL_TYPE_LOOP) { + if (comp_ctx->aot_frame->sp + > target_block->frame_sp_max_reached) + target_block->frame_sp_max_reached = + comp_ctx->aot_frame->sp; + } + } + } + + if (comp_ctx->enable_thread_mgr) { + for (i = 0; i <= br_count; i++) { + target_block = get_target_block(func_ctx, br_depths[i]); + if (!target_block) + return false; + /* Terminate or suspend current thread only when this is a + backward jump */ + if (target_block->label_type == LABEL_TYPE_LOOP) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + break; + } + } + } + + /* Compare value is not constant, create switch IR */ + for (i = 0; i <= br_count; i++) { + target_block = get_target_block(func_ctx, br_depths[i]); + if (!target_block) + return false; + + if (target_block->label_type != LABEL_TYPE_LOOP) { + /* Dest block is Block/If/Function block */ + /* Create the end block */ + if (!target_block->llvm_end_block) { + format_block_name(name, sizeof(name), + target_block->block_index, + target_block->label_type, LABEL_END); + CREATE_BLOCK(target_block->llvm_end_block, name); + if ((next_llvm_end_block = + find_next_llvm_end_block(target_block))) + MOVE_BLOCK_BEFORE(target_block->llvm_end_block, + next_llvm_end_block); + } + /* Handle result values */ + if (target_block->result_count) { + size = sizeof(LLVMValueRef) + * (uint64)target_block->result_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + CREATE_RESULT_VALUE_PHIS(target_block); + for (j = 0; j < target_block->result_count; j++) { + result_index = target_block->result_count - 1 - j; + POP(value, target_block->result_types[result_index]); + values[result_index] = value; + ADD_TO_RESULT_PHIS(target_block, value, result_index); + } + for (j = 0; j < target_block->result_count; j++) { + PUSH(values[j], target_block->result_types[j]); + } + wasm_runtime_free(values); + values = NULL; + } + target_block->is_reachable = true; + if (i == br_count) + default_llvm_block = target_block->llvm_end_block; + } + else { + /* Handle Loop parameters */ + if (target_block->param_count) { + size = sizeof(LLVMValueRef) + * (uint64)target_block->param_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + for (j = 0; j < target_block->param_count; j++) { + param_index = target_block->param_count - 1 - j; + POP(value, target_block->param_types[param_index]); + values[param_index] = value; + ADD_TO_PARAM_PHIS(target_block, value, param_index); + } + for (j = 0; j < target_block->param_count; j++) { + PUSH(values[j], target_block->param_types[j]); + } + wasm_runtime_free(values); + values = NULL; + } + if (i == br_count) + default_llvm_block = target_block->llvm_entry_block; + } + } + + /* Create switch IR */ + if (!(value_switch = LLVMBuildSwitch(comp_ctx->builder, value_cmp, + default_llvm_block, br_count))) { + aot_set_last_error("llvm build switch failed."); + return false; + } + + /* Add each case for switch IR */ + for (i = 0; i < br_count; i++) { + value_case = I32_CONST(i); + CHECK_LLVM_CONST(value_case); + target_block = get_target_block(func_ctx, br_depths[i]); + if (!target_block) + return false; + target_llvm_block = target_block->label_type != LABEL_TYPE_LOOP + ? target_block->llvm_end_block + : target_block->llvm_entry_block; + LLVMAddCase(value_switch, value_case, target_llvm_block); + } + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); + } + else { + /* Compare value is constant, create br IR */ + depth_idx = (uint32)LLVMConstIntGetZExtValue(value_cmp); + br_depth = br_depths[br_count]; + if (depth_idx < br_count) { + br_depth = br_depths[depth_idx]; + } + return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip); + } +fail: + if (values) + wasm_runtime_free(values); + return false; +} + +bool +aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block_func = func_ctx->block_stack.block_list_head; + LLVMValueRef value; + LLVMValueRef ret; + AOTFuncType *func_type; + uint32 i, param_index, result_index; +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMMetadataRef return_location; +#endif + + bh_assert(block_func); + func_type = func_ctx->aot_func->func_type; + +#if WASM_ENABLE_DEBUG_AOT != 0 + return_location = dwarf_gen_location( + comp_ctx, func_ctx, + (*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); +#endif + + if (block_func->result_count) { + /* Store extra result values to function parameters */ + for (i = 0; i < block_func->result_count - 1; i++) { + LLVMValueRef res; + result_index = block_func->result_count - 1 - i; + POP(value, block_func->result_types[result_index]); + param_index = func_type->param_count + result_index; + if (!(res = LLVMBuildStore( + comp_ctx->builder, value, + LLVMGetParam(func_ctx->func, param_index)))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + LLVMSetAlignment(res, 1); + } + /* Return the first result value */ + POP(value, block_func->result_types[0]); + if (!(ret = LLVMBuildRet(comp_ctx->builder, value))) { + aot_set_last_error("llvm build return failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMInstructionSetDebugLoc(ret, return_location); +#endif + } + else { + if (!(ret = LLVMBuildRetVoid(comp_ctx->builder))) { + aot_set_last_error("llvm build return void failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMInstructionSetDebugLoc(ret, return_location); +#endif + } + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +bool +aot_compile_op_unreachable(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_UNREACHABLE, false, NULL, + NULL)) + return false; + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +} + +bool +aot_handle_next_reachable_block(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 **p_frame_ip) +{ + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +} + +#if WASM_ENABLE_GC != 0 +static bool +commit_gc_and_check_suspend_flags(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 br_depth) +{ + AOTBlock *block_dst; + + if (!(block_dst = get_target_block(func_ctx, br_depth))) { + return false; + } + + if (comp_ctx->aot_frame) { + /* Note that GC is enabled, no need to check it again */ + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + if (comp_ctx->enable_thread_mgr) { + /* Note that GC is enabled, no need to check it again */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, false)) + return false; + } + } + else { + if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached) + block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp; + } + } + + /* Terminate or suspend current thread only when this is + a backward jump */ + if (comp_ctx->enable_thread_mgr + && block_dst->label_type == LABEL_TYPE_LOOP) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + } + + return true; +} + +static bool +compile_gc_cond_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, LLVMValueRef value_cmp) +{ + AOTBlock *block_dst; + LLVMValueRef value, *values = NULL; + LLVMBasicBlockRef llvm_else_block, next_llvm_end_block; + char name[32]; + uint32 i, param_index, result_index; + uint64 size; + + if (!(block_dst = get_target_block(func_ctx, br_depth))) { + return false; + } + + /* Create llvm else block */ + CREATE_BLOCK(llvm_else_block, "br_if_else"); + MOVE_BLOCK_AFTER_CURR(llvm_else_block); + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + /* Dest block is Loop block */ + /* Handle Loop parameters */ + if (block_dst->param_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + for (i = 0; i < block_dst->param_count; i++) { + param_index = block_dst->param_count - 1 - i; + POP(value, block_dst->param_types[param_index]); + ADD_TO_PARAM_PHIS(block_dst, value, param_index); + values[param_index] = value; + } + for (i = 0; i < block_dst->param_count; i++) { + PUSH(values[i], block_dst->param_types[i]); + } + wasm_runtime_free(values); + values = NULL; + } + + BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block, llvm_else_block); + + /* Move builder to else block */ + SET_BUILDER_POS(llvm_else_block); + } + else { + /* Dest block is Block/If/Function block */ + /* Create the end block */ + if (!block_dst->llvm_end_block) { + format_block_name(name, sizeof(name), block_dst->block_index, + block_dst->label_type, LABEL_END); + CREATE_BLOCK(block_dst->llvm_end_block, name); + if ((next_llvm_end_block = find_next_llvm_end_block(block_dst))) + MOVE_BLOCK_BEFORE(block_dst->llvm_end_block, + next_llvm_end_block); + } + + /* Set reachable flag and create condition br IR */ + block_dst->is_reachable = true; + + /* Handle result values */ + if (block_dst->result_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + CREATE_RESULT_VALUE_PHIS(block_dst); + for (i = 0; i < block_dst->result_count; i++) { + result_index = block_dst->result_count - 1 - i; + POP(value, block_dst->result_types[result_index]); + values[result_index] = value; + ADD_TO_RESULT_PHIS(block_dst, value, result_index); + } + for (i = 0; i < block_dst->result_count; i++) { + PUSH(values[i], block_dst->result_types[i]); + } + wasm_runtime_free(values); + values = NULL; + } + + /* Condition jump to end block */ + BUILD_COND_BR(value_cmp, block_dst->llvm_end_block, llvm_else_block); + + /* Move builder to else block */ + SET_BUILDER_POS(llvm_else_block); + } + + return true; +fail: + if (values) + wasm_runtime_free(values); + return false; +} + +bool +aot_compile_op_br_on_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip) +{ + LLVMValueRef gc_obj, value_cmp; + + if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) { + return false; + } + + POP_GC_REF(gc_obj); + + if (!(value_cmp = + LLVMBuildIsNull(comp_ctx->builder, gc_obj, "cmp_gc_obj"))) { + aot_set_last_error("llvm build isnull failed."); + goto fail; + } + + if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, value_cmp)) { + goto fail; + } + + PUSH_GC_REF(gc_obj); + return true; +fail: + return false; +} + +bool +aot_compile_op_br_on_non_null(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 br_depth, + uint8 **p_frame_ip) +{ + LLVMValueRef gc_obj, value_cmp; + + if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) { + return false; + } + + GET_GC_REF_FROM_STACK(gc_obj); + + if (!(value_cmp = + LLVMBuildIsNotNull(comp_ctx->builder, gc_obj, "cmp_gc_obj"))) { + aot_set_last_error("llvm build isnotnull failed."); + goto fail; + } + + if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, value_cmp)) { + goto fail; + } + + POP_GC_REF(gc_obj); + return true; +fail: + return false; +} + +bool +aot_compile_op_br_on_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 heap_type, bool nullable, bool br_on_fail, + uint32 br_depth, uint8 **p_frame_ip) +{ + LLVMValueRef gc_obj, is_null, castable, not_castable, br_if_phi; + LLVMBasicBlockRef block_curr, block_non_null, block_br_if; + + if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) { + return false; + } + + GET_GC_REF_FROM_STACK(gc_obj); + + block_curr = CURR_BLOCK(); + + CREATE_BLOCK(block_non_null, "obj_non_null"); + MOVE_BLOCK_AFTER_CURR(block_non_null); + CREATE_BLOCK(block_br_if, "br_if"); + MOVE_BLOCK_AFTER(block_br_if, block_non_null); + + SET_BUILDER_POS(block_br_if); + if (!(br_if_phi = + LLVMBuildPhi(comp_ctx->builder, INT1_TYPE, "br_if_phi"))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + + SET_BUILDER_POS(block_curr); + + if (!(is_null = LLVMBuildIsNull(comp_ctx->builder, gc_obj, "is_null"))) { + aot_set_last_error("llvm build isnull failed."); + goto fail; + } + + BUILD_COND_BR(is_null, block_br_if, block_non_null); + + if ((!br_on_fail && nullable) || (br_on_fail && !nullable)) { + LLVMAddIncoming(br_if_phi, &I1_ONE, &block_curr, 1); + } + else { /* (!br_on_fail && !nullable) || (br_on_fail && nullable)) */ + LLVMAddIncoming(br_if_phi, &I1_ZERO, &block_curr, 1); + } + + SET_BUILDER_POS(block_non_null); + if (heap_type >= 0) { + if (!aot_call_aot_obj_is_instance_of(comp_ctx, func_ctx, gc_obj, + I32_CONST(heap_type), &castable)) + goto fail; + } + else { + if (!aot_call_wasm_obj_is_type_of(comp_ctx, func_ctx, gc_obj, + I32_CONST(heap_type), &castable)) + goto fail; + } + + if (!br_on_fail) { + if (!(castable = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, castable, + I8_ZERO, "castable"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + LLVMAddIncoming(br_if_phi, &castable, &block_non_null, 1); + } + else { + if (!(not_castable = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, + castable, I8_ZERO, "castable"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + LLVMAddIncoming(br_if_phi, ¬_castable, &block_non_null, 1); + } + BUILD_BR(block_br_if); + + SET_BUILDER_POS(block_br_if); + if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, br_if_phi)) { + goto fail; + } + + return true; +fail: + return false; +} + +#endif /* End of WASM_ENABLE_GC != 0 */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_control.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_control.h new file mode 100644 index 0000000..fd53849 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_control.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_CONTROL_H_ +#define _AOT_EMIT_CONTROL_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip, uint8 *frame_ip_end, uint32 label_type, + uint32 param_count, uint8 *param_types, + uint32 result_count, uint8 *result_types); + +bool +aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip); + +bool +aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip); + +bool +aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip); + +bool +aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip); + +bool +aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 *br_depths, uint32 br_count, uint8 **p_frame_ip); + +bool +aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip); + +bool +aot_compile_op_unreachable(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip); + +bool +aot_handle_next_reachable_block(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 **p_frame_ip); + +bool +check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool check_terminate_and_suspend); + +#if WASM_ENABLE_GC != 0 +bool +aot_compile_op_br_on_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip); + +bool +aot_compile_op_br_on_non_null(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 br_depth, + + uint8 **p_frame_ip); + +bool +aot_compile_op_br_on_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 heap_type, bool nullable, bool br_on_fail, + uint32 br_depth, uint8 **p_frame_ip); +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_CONTROL_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_conversion.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_conversion.c new file mode 100644 index 0000000..c3dfa6b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_conversion.c @@ -0,0 +1,939 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_conversion.h" +#include "aot_emit_exception.h" +#include "aot_emit_numberic.h" +#include "../aot/aot_intrinsic.h" +#include "../aot/aot_runtime.h" + +static LLVMValueRef +call_fcmp_intrinsic(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + enum AOTFloatCond cond, LLVMRealPredicate op, + LLVMValueRef lhs, LLVMValueRef rhs, LLVMTypeRef src_type, + const char *name) +{ + LLVMValueRef res = NULL; + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability( + comp_ctx, src_type == F32_TYPE ? "f32_cmp" : "f64_cmp")) { + LLVMTypeRef param_types[3]; + LLVMValueRef opcond = LLVMConstInt(I32_TYPE, cond, true); + param_types[0] = I32_TYPE; + param_types[1] = src_type; + param_types[2] = src_type; + res = aot_call_llvm_intrinsic( + comp_ctx, func_ctx, src_type == F32_TYPE ? "f32_cmp" : "f64_cmp", + I32_TYPE, param_types, 3, opcond, lhs, rhs); + if (!res) { + goto fail; + } + res = LLVMBuildIntCast(comp_ctx->builder, res, INT1_TYPE, "bit_cast"); + } + else { + res = LLVMBuildFCmp(comp_ctx->builder, op, lhs, rhs, name); + } +fail: + return res; +} + +static bool +trunc_float_to_int(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef operand, LLVMTypeRef src_type, + LLVMTypeRef dest_type, LLVMValueRef min_value, + LLVMValueRef max_value, char *name, bool sign) +{ + LLVMBasicBlockRef check_nan_succ, check_overflow_succ; + LLVMValueRef is_less, is_greater, res; + + res = call_fcmp_intrinsic(comp_ctx, func_ctx, FLOAT_UNO, LLVMRealUNO, + operand, operand, src_type, "fcmp_is_nan"); + + if (!res) { + aot_set_last_error("llvm build fcmp failed."); + goto fail; + } + + if (!(check_nan_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_nan_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_nan_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_INVALID_CONVERSION_TO_INTEGER, true, res, + check_nan_succ))) + goto fail; + + is_less = + call_fcmp_intrinsic(comp_ctx, func_ctx, FLOAT_LE, LLVMRealOLE, operand, + min_value, src_type, "fcmp_min_value"); + + if (!is_less) { + aot_set_last_error("llvm build fcmp failed."); + goto fail; + } + + is_greater = + call_fcmp_intrinsic(comp_ctx, func_ctx, FLOAT_GE, LLVMRealOGE, operand, + max_value, src_type, "fcmp_min_value"); + + if (!is_greater) { + aot_set_last_error("llvm build fcmp failed."); + goto fail; + } + + if (!(res = LLVMBuildOr(comp_ctx->builder, is_less, is_greater, + "is_overflow"))) { + aot_set_last_error("llvm build logic and failed."); + goto fail; + } + + /* Check if float value out of range */ + if (!(check_overflow_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_overflow_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_overflow_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, true, + res, check_overflow_succ))) + goto fail; + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, name)) { + LLVMTypeRef param_types[1]; + param_types[0] = src_type; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, name, dest_type, + param_types, 1, operand); + } + else { + if (sign) + res = LLVMBuildFPToSI(comp_ctx->builder, operand, dest_type, name); + else + res = LLVMBuildFPToUI(comp_ctx->builder, operand, dest_type, name); + } + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + if (dest_type == I32_TYPE) + PUSH_I32(res); + else if (dest_type == I64_TYPE) + PUSH_I64(res); + return true; +fail: + return false; +} + +#define ADD_BASIC_BLOCK(block, name) \ + do { \ + if (!(block = LLVMAppendBasicBlockInContext(comp_ctx->context, \ + func_ctx->func, name))) { \ + aot_set_last_error("llvm add basic block failed."); \ + goto fail; \ + } \ + \ + LLVMMoveBasicBlockAfter(block, LLVMGetInsertBlock(comp_ctx->builder)); \ + } while (0) + +static bool +trunc_sat_float_to_int(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef operand, LLVMTypeRef src_type, + LLVMTypeRef dest_type, LLVMValueRef min_value, + LLVMValueRef max_value, char *name, bool sign) +{ + LLVMBasicBlockRef check_nan_succ, check_less_succ, check_greater_succ; + LLVMBasicBlockRef is_nan_block, is_less_block, is_greater_block, res_block; + LLVMValueRef is_less, is_greater, res, phi; + LLVMValueRef zero = (dest_type == I32_TYPE) ? I32_ZERO : I64_ZERO; + LLVMValueRef vmin, vmax; + + if (!(res = + call_fcmp_intrinsic(comp_ctx, func_ctx, FLOAT_UNO, LLVMRealUNO, + operand, operand, src_type, "fcmp_is_nan"))) { + aot_set_last_error("llvm build fcmp failed."); + goto fail; + } + + ADD_BASIC_BLOCK(check_nan_succ, "check_nan_succ"); + ADD_BASIC_BLOCK(is_nan_block, "is_nan_block"); + ADD_BASIC_BLOCK(check_less_succ, "check_less_succ"); + ADD_BASIC_BLOCK(is_less_block, "is_less_block"); + ADD_BASIC_BLOCK(check_greater_succ, "check_greater_succ"); + ADD_BASIC_BLOCK(is_greater_block, "is_greater_block"); + ADD_BASIC_BLOCK(res_block, "res_block"); + + if (!LLVMBuildCondBr(comp_ctx->builder, res, is_nan_block, + check_nan_succ)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* Start to translate is_nan block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, is_nan_block); + if (!LLVMBuildBr(comp_ctx->builder, res_block)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Start to translate check_nan_succ block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, check_nan_succ); + if (!(is_less = call_fcmp_intrinsic(comp_ctx, func_ctx, FLOAT_LE, + LLVMRealOLE, operand, min_value, + src_type, "fcmp_min_value"))) { + aot_set_last_error("llvm build fcmp failed."); + goto fail; + } + if (!LLVMBuildCondBr(comp_ctx->builder, is_less, is_less_block, + check_less_succ)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* Start to translate is_less block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, is_less_block); + if (!LLVMBuildBr(comp_ctx->builder, res_block)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Start to translate check_less_succ block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, check_less_succ); + if (!(is_greater = call_fcmp_intrinsic(comp_ctx, func_ctx, FLOAT_GE, + LLVMRealOGE, operand, max_value, + src_type, "fcmp_max_value"))) { + aot_set_last_error("llvm build fcmp failed."); + goto fail; + } + if (!LLVMBuildCondBr(comp_ctx->builder, is_greater, is_greater_block, + check_greater_succ)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* Start to translate is_greater block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, is_greater_block); + if (!LLVMBuildBr(comp_ctx->builder, res_block)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Start to translate check_greater_succ block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, check_greater_succ); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, name)) { + LLVMTypeRef param_types[1]; + param_types[0] = src_type; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, name, dest_type, + param_types, 1, operand); + } + else { + char intrinsic[128]; + + /* Integer width is always 32 or 64 here. */ + + snprintf(intrinsic, sizeof(intrinsic), "i%d_trunc_f%d_%c", + LLVMGetIntTypeWidth(dest_type), + LLVMGetTypeKind(src_type) == LLVMFloatTypeKind ? 32 : 64, + sign ? 's' : 'u'); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, intrinsic)) { + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, + dest_type, &src_type, 1, operand); + } + else { + if (sign) { + res = LLVMBuildFPToSI(comp_ctx->builder, operand, dest_type, + name); + } + else { + res = LLVMBuildFPToUI(comp_ctx->builder, operand, dest_type, + name); + } + } + } + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + if (!LLVMBuildBr(comp_ctx->builder, res_block)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Start to translate res_block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, res_block); + /* Create result phi */ + if (!(phi = LLVMBuildPhi(comp_ctx->builder, dest_type, + "trunc_sat_result_phi"))) { + aot_set_last_error("llvm build phi failed."); + return false; + } + + /* Add phi incoming values */ + if (dest_type == I32_TYPE) { + if (sign) { + vmin = I32_CONST(INT32_MIN); + vmax = I32_CONST(INT32_MAX); + } + else { + vmin = I32_CONST(0); + vmax = I32_CONST(UINT32_MAX); + } + } + else if (dest_type == I64_TYPE) { + if (sign) { + vmin = I64_CONST(INT64_MIN); + vmax = I64_CONST(INT64_MAX); + } + else { + vmin = I64_CONST(0); + vmax = I64_CONST(UINT64_MAX); + } + } + LLVMAddIncoming(phi, &zero, &is_nan_block, 1); + LLVMAddIncoming(phi, &vmin, &is_less_block, 1); + LLVMAddIncoming(phi, &vmax, &is_greater_block, 1); + LLVMAddIncoming(phi, &res, &check_greater_succ, 1); + + if (dest_type == I32_TYPE) + PUSH_I32(phi); + else if (dest_type == I64_TYPE) + PUSH_I64(phi); + return true; +fail: + return false; +} + +bool +aot_compile_op_i32_wrap_i64(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef value, res; + + POP_I64(value); + + if (!(res = LLVMBuildTrunc(comp_ctx->builder, value, I32_TYPE, + "i32_wrap_i64"))) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_I32(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_i32_trunc_f32(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating) +{ + LLVMValueRef value; + LLVMValueRef min_value, max_value; + + POP_F32(value); + + if (!comp_ctx->is_indirect_mode) { + if (sign) { + min_value = F32_CONST(-2147483904.0f); + max_value = F32_CONST(2147483648.0f); + } + else { + min_value = F32_CONST(-1.0f); + max_value = F32_CONST(4294967296.0f); + } + } + else { + WASMValue wasm_value; + if (sign) { + wasm_value.f32 = -2147483904.0f; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + wasm_value.f32 = 2147483648.0f; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + } + else { + wasm_value.f32 = -1.0f; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + wasm_value.f32 = 4294967296.0f; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + } + } + CHECK_LLVM_CONST(min_value); + CHECK_LLVM_CONST(max_value); + + if (!saturating) + return trunc_float_to_int( + comp_ctx, func_ctx, value, F32_TYPE, I32_TYPE, min_value, max_value, + sign ? "i32_trunc_f32_s" : "i32_trunc_f32_u", sign); + else + return trunc_sat_float_to_int( + comp_ctx, func_ctx, value, F32_TYPE, I32_TYPE, min_value, max_value, + sign ? "i32_trunc_sat_f32_s" : "i32_trunc_sat_f32_u", sign); +fail: + return false; +} + +bool +aot_compile_op_i32_trunc_f64(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating) +{ + LLVMValueRef value; + LLVMValueRef min_value, max_value; + + POP_F64(value); + + if (!comp_ctx->is_indirect_mode) { + if (sign) { + min_value = F64_CONST(-2147483649.0); + max_value = F64_CONST(2147483648.0); + } + else { + min_value = F64_CONST(-1.0); + max_value = F64_CONST(4294967296.0); + } + } + else { + WASMValue wasm_value; + if (sign) { + wasm_value.f64 = -2147483649.0; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + wasm_value.f64 = 2147483648.0; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + } + else { + wasm_value.f64 = -1.0; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + wasm_value.f64 = 4294967296.0; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + } + } + CHECK_LLVM_CONST(min_value); + CHECK_LLVM_CONST(max_value); + + if (!saturating) + return trunc_float_to_int( + comp_ctx, func_ctx, value, F64_TYPE, I32_TYPE, min_value, max_value, + sign ? "i32_trunc_f64_s" : "i32_trunc_f64_u", sign); + else + return trunc_sat_float_to_int( + comp_ctx, func_ctx, value, F64_TYPE, I32_TYPE, min_value, max_value, + sign ? "i32_trunc_sat_f64_s" : "i32_trunc_sat_f64_u", sign); +fail: + return false; +} + +bool +aot_compile_op_i64_extend_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign) +{ + LLVMValueRef value, res; + + POP_I32(value); + + if (sign) + res = LLVMBuildSExt(comp_ctx->builder, value, I64_TYPE, + "i64_extend_i32_s"); + else + res = LLVMBuildZExt(comp_ctx->builder, value, I64_TYPE, + "i64_extend_i32_u"); + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_I64(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_i64_extend_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, int8 bitwidth) +{ + LLVMValueRef value, res, cast_value = NULL; + + POP_I64(value); + + if (bitwidth == 8) { + cast_value = LLVMBuildIntCast2(comp_ctx->builder, value, INT8_TYPE, + true, "i8_intcast_i64"); + } + else if (bitwidth == 16) { + cast_value = LLVMBuildIntCast2(comp_ctx->builder, value, INT16_TYPE, + true, "i16_intcast_i64"); + } + else if (bitwidth == 32) { + cast_value = LLVMBuildIntCast2(comp_ctx->builder, value, I32_TYPE, true, + "i32_intcast_i64"); + } + + if (!cast_value) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + res = LLVMBuildSExt(comp_ctx->builder, cast_value, I64_TYPE, + "i64_extend_i64_s"); + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_I64(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_i32_extend_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, int8 bitwidth) +{ + LLVMValueRef value, res, cast_value = NULL; + + POP_I32(value); + + if (bitwidth == 8) { + cast_value = LLVMBuildIntCast2(comp_ctx->builder, value, INT8_TYPE, + true, "i8_intcast_i32"); + } + else if (bitwidth == 16) { + cast_value = LLVMBuildIntCast2(comp_ctx->builder, value, INT16_TYPE, + true, "i16_intcast_i32"); + } + + if (!cast_value) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + res = LLVMBuildSExt(comp_ctx->builder, cast_value, I32_TYPE, + "i32_extend_i32_s"); + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_I32(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_i64_trunc_f32(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating) +{ + LLVMValueRef value; + LLVMValueRef min_value, max_value; + + POP_F32(value); + + if (!comp_ctx->is_indirect_mode) { + if (sign) { + min_value = F32_CONST(-9223373136366403584.0f); + max_value = F32_CONST(9223372036854775808.0f); + } + else { + min_value = F32_CONST(-1.0f); + max_value = F32_CONST(18446744073709551616.0f); + } + } + else { + WASMValue wasm_value; + if (sign) { + wasm_value.f32 = -9223373136366403584.0f; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + wasm_value.f32 = 9223372036854775808.0f; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + } + else { + wasm_value.f32 = -1.0f; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + wasm_value.f32 = 18446744073709551616.0f; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F32); + } + } + CHECK_LLVM_CONST(min_value); + CHECK_LLVM_CONST(max_value); + + if (!saturating) + return trunc_float_to_int( + comp_ctx, func_ctx, value, F32_TYPE, I64_TYPE, min_value, max_value, + sign ? "i64_trunc_f32_s" : "i64_trunc_f32_u", sign); + else + return trunc_sat_float_to_int( + comp_ctx, func_ctx, value, F32_TYPE, I64_TYPE, min_value, max_value, + sign ? "i64_trunc_sat_f32_s" : "i64_trunc_sat_f32_u", sign); +fail: + return false; +} + +bool +aot_compile_op_i64_trunc_f64(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating) +{ + LLVMValueRef value; + LLVMValueRef min_value, max_value; + + POP_F64(value); + + if (!comp_ctx->is_indirect_mode) { + if (sign) { + min_value = F64_CONST(-9223372036854777856.0); + max_value = F64_CONST(9223372036854775808.0); + } + else { + min_value = F64_CONST(-1.0); + max_value = F64_CONST(18446744073709551616.0); + } + } + else { + WASMValue wasm_value; + if (sign) { + wasm_value.f64 = -9223372036854777856.0; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + wasm_value.f64 = 9223372036854775808.0; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + } + else { + wasm_value.f64 = -1.0; + min_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + wasm_value.f64 = 18446744073709551616.0; + max_value = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_F64); + } + } + CHECK_LLVM_CONST(min_value); + CHECK_LLVM_CONST(max_value); + + if (!saturating) + return trunc_float_to_int( + comp_ctx, func_ctx, value, F64_TYPE, I64_TYPE, min_value, max_value, + sign ? "i64_trunc_f64_s" : "i64_trunc_f64_u", sign); + else + return trunc_sat_float_to_int( + comp_ctx, func_ctx, value, F64_TYPE, I64_TYPE, min_value, max_value, + sign ? "i64_trunc_sat_f64_s" : "i64_trunc_sat_f64_u", sign); + +fail: + return false; +} + +bool +aot_compile_op_f32_convert_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign) +{ + LLVMValueRef value, res; + + POP_I32(value); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability( + comp_ctx, sign ? "f32_convert_i32_s" : "f32_convert_i32_u")) { + LLVMTypeRef param_types[1]; + param_types[0] = I32_TYPE; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, + sign ? "f32_convert_i32_s" + : "f32_convert_i32_u", + F32_TYPE, param_types, 1, value); + } + else { + if (sign) + res = LLVMBuildSIToFP(comp_ctx->builder, value, F32_TYPE, + "f32_convert_i32_s"); + else + res = LLVMBuildUIToFP(comp_ctx->builder, value, F32_TYPE, + "f32_convert_i32_u"); + } + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_F32(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_f32_convert_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign) +{ + LLVMValueRef value, res; + + POP_I64(value); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability( + comp_ctx, sign ? "f32_convert_i64_s" : "f32_convert_i64_u")) { + LLVMTypeRef param_types[1]; + param_types[0] = I64_TYPE; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, + sign ? "f32_convert_i64_s" + : "f32_convert_i64_u", + F32_TYPE, param_types, 1, value); + } + else { + if (sign) + res = LLVMBuildSIToFP(comp_ctx->builder, value, F32_TYPE, + "f32_convert_i64_s"); + else + res = LLVMBuildUIToFP(comp_ctx->builder, value, F32_TYPE, + "f32_convert_i64_u"); + } + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_F32(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_f32_demote_f64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef value, res; + + POP_F64(value); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, "f32_demote_f64")) { + LLVMTypeRef param_types[1]; + param_types[0] = F64_TYPE; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, "f32_demote_f64", + F32_TYPE, param_types, 1, value); + } + else { + res = LLVMBuildFPTrunc(comp_ctx->builder, value, F32_TYPE, + "f32_demote_f64"); + } + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_F32(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_convert_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign) +{ + LLVMValueRef value, res; + + POP_I32(value); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability( + comp_ctx, sign ? "f64_convert_i32_s" : "f64_convert_i32_u")) { + LLVMTypeRef param_types[1]; + param_types[0] = I32_TYPE; + + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, + sign ? "f64_convert_i32_s" + : "f64_convert_i32_u", + F64_TYPE, param_types, 1, value); + } + else { + if (sign) + res = LLVMBuildSIToFP(comp_ctx->builder, value, F64_TYPE, + "f64_convert_i32_s"); + else + res = LLVMBuildUIToFP(comp_ctx->builder, value, F64_TYPE, + "f64_convert_i32_u"); + } + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_F64(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_convert_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign) +{ + LLVMValueRef value, res; + + POP_I64(value); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability( + comp_ctx, sign ? "f64_convert_i64_s" : "f64_convert_i64_u")) { + LLVMTypeRef param_types[1]; + param_types[0] = I64_TYPE; + + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, + sign ? "f64_convert_i64_s" + : "f64_convert_i64_u", + F64_TYPE, param_types, 1, value); + } + else { + if (sign) + res = LLVMBuildSIToFP(comp_ctx->builder, value, F64_TYPE, + "f64_convert_i64_s"); + else + res = LLVMBuildUIToFP(comp_ctx->builder, value, F64_TYPE, + "f64_convert_i64_u"); + } + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_F64(res); + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_promote_f32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef value, res; + + POP_F32(value); + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, "f64_promote_f32")) { + LLVMTypeRef param_types[1]; + param_types[0] = F32_TYPE; + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, "f64_promote_f32", + F64_TYPE, param_types, 1, value); + } + else { + res = LLVMBuildFPExt(comp_ctx->builder, value, F64_TYPE, + "f64_promote_f32"); + } + + if (!res) { + aot_set_last_error("llvm build conversion failed."); + return false; + } + + PUSH_F64(res); + + /* Avoid the promote being optimized away */ + PUSH_F64(F64_CONST(1.0)); + return aot_compile_op_f64_arithmetic(comp_ctx, func_ctx, FLOAT_MUL); +fail: + return false; +} + +bool +aot_compile_op_i64_reinterpret_f64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef value; + POP_F64(value); + if (!(value = + LLVMBuildBitCast(comp_ctx->builder, value, I64_TYPE, "i64"))) { + aot_set_last_error("llvm build fp to si failed."); + return false; + } + PUSH_I64(value); + return true; +fail: + return false; +} + +bool +aot_compile_op_i32_reinterpret_f32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef value; + POP_F32(value); + if (!(value = + LLVMBuildBitCast(comp_ctx->builder, value, I32_TYPE, "i32"))) { + aot_set_last_error("llvm build fp to si failed."); + return false; + } + PUSH_I32(value); + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_reinterpret_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef value; + POP_I64(value); + if (!(value = + LLVMBuildBitCast(comp_ctx->builder, value, F64_TYPE, "f64"))) { + aot_set_last_error("llvm build si to fp failed."); + return false; + } + PUSH_F64(value); + return true; +fail: + return false; +} + +bool +aot_compile_op_f32_reinterpret_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef value; + POP_I32(value); + if (!(value = + LLVMBuildBitCast(comp_ctx->builder, value, F32_TYPE, "f32"))) { + aot_set_last_error("llvm build si to fp failed."); + return false; + } + PUSH_F32(value); + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_conversion.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_conversion.h new file mode 100644 index 0000000..a0e2fcb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_conversion.h @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_CONVERSION_H_ +#define _AOT_EMIT_CONVERSION_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_i32_wrap_i64(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i32_trunc_f32(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating); + +bool +aot_compile_op_i32_trunc_f64(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating); + +bool +aot_compile_op_i64_extend_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign); + +bool +aot_compile_op_i64_extend_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, int8 bitwidth); + +bool +aot_compile_op_i32_extend_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, int8 bitwidth); + +bool +aot_compile_op_i64_trunc_f32(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating); + +bool +aot_compile_op_i64_trunc_f64(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign, bool saturating); + +bool +aot_compile_op_f32_convert_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign); + +bool +aot_compile_op_f32_convert_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign); + +bool +aot_compile_op_f32_demote_f64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_f64_convert_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign); + +bool +aot_compile_op_f64_convert_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool sign); + +bool +aot_compile_op_f64_promote_f32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_i64_reinterpret_f64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_i32_reinterpret_f32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_f64_reinterpret_i64(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_f32_reinterpret_i32(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_CONVERSION_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_exception.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_exception.c new file mode 100644 index 0000000..d3dcf71 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_exception.c @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_exception.h" +#include "../interpreter/wasm_runtime.h" +#include "../aot/aot_runtime.h" + +static bool +commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef exce_ip, bool is_64bit) +{ + LLVMValueRef cur_frame = func_ctx->cur_frame; + LLVMValueRef value_offset, value_addr, value_ptr; + uint32 offset_ip; + + if (!comp_ctx->is_jit_mode) + offset_ip = comp_ctx->pointer_size * 4; + else + offset_ip = offsetof(WASMInterpFrame, ip); + + if (!(value_offset = I32_CONST(offset_ip))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "ip_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(value_ptr = LLVMBuildBitCast( + comp_ctx->builder, value_addr, + is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!LLVMBuildStore(comp_ctx->builder, exce_ip, value_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + return true; +} + +bool +aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 exception_id, bool is_cond_br, LLVMValueRef cond_br_if, + LLVMBasicBlockRef cond_br_else_block) +{ + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMValueRef exce_id = I32_CONST((uint32)exception_id), func_const, func; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + LLVMValueRef param_values[2]; + bool is_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; + + bh_assert(exception_id >= 0 && exception_id < EXCE_NUM); + + CHECK_LLVM_CONST(exce_id); + + /* Create got_exception block if needed */ + if (!func_ctx->got_exception_block) { + if (!(func_ctx->got_exception_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "got_exception"))) { + aot_set_last_error("add LLVM basic block failed."); + return false; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, + func_ctx->got_exception_block); + + /* Create exception id phi */ + if (!(func_ctx->exception_id_phi = LLVMBuildPhi( + comp_ctx->builder, I32_TYPE, "exception_id_phi"))) { + aot_set_last_error("llvm build phi failed."); + return false; + } + + if (comp_ctx->aot_frame) { + /* Create exception ip phi */ + if (!(func_ctx->exception_ip_phi = LLVMBuildPhi( + comp_ctx->builder, is_64bit ? I64_TYPE : I32_TYPE, + "exception_ip_phi"))) { + aot_set_last_error("llvm build phi failed."); + return false; + } + + /* Commit ip to current frame */ + if (!commit_ip(comp_ctx, func_ctx, func_ctx->exception_ip_phi, + is_64bit)) { + return false; + } + } + + /* Call aot_set_exception_with_id() to throw exception */ + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = VOID_TYPE; + + /* Create function type */ + if (!(func_type = LLVMFunctionType(ret_type, param_types, 2, false))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + if (comp_ctx->is_jit_mode) { + /* Create function type */ + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + /* Create LLVM function with const function pointer */ + if (!(func_const = + I64_CONST((uint64)(uintptr_t)jit_set_exception_with_id)) + || !(func = LLVMConstIntToPtr(func_const, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + func_index = aot_get_native_symbol_index( + comp_ctx, "aot_set_exception_with_id"); + if (func_index < 0) { + return false; + } + if (!(func = + aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + /* Create LLVM function with external function pointer */ + if (!(func = LLVMGetNamedFunction(func_ctx->module, + "aot_set_exception_with_id")) + && !(func = LLVMAddFunction(func_ctx->module, + "aot_set_exception_with_id", + func_type))) { + aot_set_last_error("add LLVM function failed."); + return false; + } + } + + /* Call the aot_set_exception_with_id() function */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = func_ctx->exception_id_phi; + if (!LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, 2, + "")) { + aot_set_last_error("llvm build call failed."); + return false; + } + + /* Create return IR */ + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + return false; + } + + /* Resume the builder position */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + } + + /* Add phi incoming value to got_exception block */ + LLVMAddIncoming(func_ctx->exception_id_phi, &exce_id, &block_curr, 1); + + if (comp_ctx->aot_frame) { + const uint8 *ip = comp_ctx->aot_frame->frame_ip; + LLVMValueRef exce_ip = NULL; + + if (!comp_ctx->is_jit_mode) { + WASMModule *module = comp_ctx->comp_data->wasm_module; + if (is_64bit) + exce_ip = + I64_CONST((uint64)(uintptr_t)(ip - module->load_addr)); + else + exce_ip = + I32_CONST((uint32)(uintptr_t)(ip - module->load_addr)); + } + else { + if (is_64bit) + exce_ip = I64_CONST((uint64)(uintptr_t)ip); + else + exce_ip = I32_CONST((uint32)(uintptr_t)ip); + } + + if (!exce_ip) { + aot_set_last_error("llvm build const failed"); + return false; + } + + /* Add phi incoming value to got_exception block */ + LLVMAddIncoming(func_ctx->exception_ip_phi, &exce_ip, &block_curr, 1); + } + + if (!is_cond_br) { + /* not condition br, create br IR */ + if (!LLVMBuildBr(comp_ctx->builder, func_ctx->got_exception_block)) { + aot_set_last_error("llvm build br failed."); + return false; + } + } + else { + /* Create condition br */ + if (!LLVMBuildCondBr(comp_ctx->builder, cond_br_if, + func_ctx->got_exception_block, + cond_br_else_block)) { + aot_set_last_error("llvm build cond br failed."); + return false; + } + /* Start to translate the else block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, cond_br_else_block); + } + + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_exception.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_exception.h new file mode 100644 index 0000000..91c8bd3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_exception.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_EXCEPTION_H_ +#define _AOT_EMIT_EXCEPTION_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 exception_id, bool is_cond_br, LLVMValueRef cond_br_if, + LLVMBasicBlockRef cond_br_else_block); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_EXCEPTION_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_function.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_function.c new file mode 100644 index 0000000..cf3824e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_function.c @@ -0,0 +1,3152 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_function.h" +#include "aot_emit_exception.h" +#include "aot_emit_control.h" +#include "aot_emit_table.h" +#include "../aot/aot_runtime.h" +#if WASM_ENABLE_GC != 0 +#include "aot_emit_gc.h" +#endif + +#define ADD_BASIC_BLOCK(block, name) \ + do { \ + if (!(block = LLVMAppendBasicBlockInContext(comp_ctx->context, \ + func_ctx->func, name))) { \ + aot_set_last_error("llvm add basic block failed."); \ + goto fail; \ + } \ + } while (0) + +static bool +is_win_platform(AOTCompContext *comp_ctx) +{ + char *triple = LLVMGetTargetMachineTriple(comp_ctx->target_machine); + bool ret; + + bh_assert(triple); + ret = (strstr(triple, "win32") || strstr(triple, "win")) ? true : false; + + LLVMDisposeMessage(triple); + + return ret; +} + +static bool +create_func_return_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + + /* Create function return block if it isn't created */ + if (!func_ctx->func_return_block) { + if (!(func_ctx->func_return_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "func_ret"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + + /* Create return IR */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, + func_ctx->func_return_block); + if (!comp_ctx->enable_bound_check) { + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ALREADY_THROWN, + false, NULL, NULL)) { + return false; + } + } + else if (!aot_build_zero_function_ret(comp_ctx, func_ctx, + aot_func_type)) { + return false; + } + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + return true; +} + +/* Check whether there was exception thrown, if yes, return directly */ +static bool +check_exception_thrown(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMBasicBlockRef block_curr, check_exce_succ; + LLVMValueRef value, cmp; + + /* Create function return block if it isn't created */ + if (!create_func_return_block(comp_ctx, func_ctx)) + return false; + + /* Load the first byte of aot_module_inst->cur_exception, and check + whether it is '\0'. If yes, no exception was thrown. */ + if (!(value = LLVMBuildLoad2(comp_ctx->builder, INT8_TYPE, + func_ctx->cur_exception, "exce_value")) + || !(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, value, I8_ZERO, + "cmp"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + + /* Add check exception success block */ + if (!(check_exce_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_exce_succ"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMMoveBasicBlockAfter(check_exce_succ, block_curr); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + /* Create condition br */ + if (!LLVMBuildCondBr(comp_ctx->builder, cmp, check_exce_succ, + func_ctx->func_return_block)) { + aot_set_last_error("llvm build cond br failed."); + return false; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, check_exce_succ); + return true; +} + +/* Check whether there was exception thrown, if yes, return directly */ +static bool +check_call_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef res) +{ + LLVMBasicBlockRef block_curr, check_call_succ; + LLVMValueRef cmp; + + /* Create function return block if it isn't created */ + if (!create_func_return_block(comp_ctx, func_ctx)) + return false; + + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, res, I8_ZERO, + "cmp"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + + /* Add check exception success block */ + if (!(check_call_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_call_succ"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMMoveBasicBlockAfter(check_call_succ, block_curr); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + /* Create condition br */ + if (!LLVMBuildCondBr(comp_ctx->builder, cmp, check_call_succ, + func_ctx->func_return_block)) { + aot_set_last_error("llvm build cond br failed."); + return false; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, check_call_succ); + return true; +} + +static bool +call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef func_idx, AOTFuncType *aot_func_type, + LLVMTypeRef *param_types, + LLVMValueRef *param_values, uint32 param_count, + uint32 param_cell_num, LLVMTypeRef ret_type, + uint8 wasm_ret_type, LLVMValueRef *p_value_ret, + LLVMValueRef *p_res) +{ + LLVMTypeRef func_type, func_ptr_type, func_param_types[4]; + LLVMTypeRef ret_ptr_type, elem_ptr_type; + LLVMValueRef func, elem_idx, elem_ptr; + LLVMValueRef func_param_values[4], value_ret = NULL, res; + char buf[32], *func_name = "aot_invoke_native"; + uint32 i, cell_num = 0; + + /* prepare function type of aot_invoke_native */ + func_param_types[0] = comp_ctx->exec_env_type; /* exec_env */ + func_param_types[1] = I32_TYPE; /* func_idx */ + func_param_types[2] = I32_TYPE; /* argc */ + func_param_types[3] = INT32_PTR_TYPE; /* argv */ + if (!(func_type = + LLVMFunctionType(INT8_TYPE, func_param_types, 4, false))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + /* prepare function pointer */ + if (comp_ctx->is_jit_mode) { + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + /* JIT mode, call the function directly */ + if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_invoke_native)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + func_index = aot_get_native_symbol_index(comp_ctx, func_name); + if (func_index < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) + && !(func = + LLVMAddFunction(func_ctx->module, func_name, func_type))) { + aot_set_last_error("add LLVM function failed."); + return false; + } + } + + if (param_cell_num > 64) { + aot_set_last_error("prepare native arguments failed: " + "maximum 64 parameter cell number supported."); + return false; + } + + /* prepare frame_lp */ + for (i = 0; i < param_count; i++) { + if (!(elem_idx = I32_CONST(cell_num)) + || !(elem_ptr_type = LLVMPointerType(param_types[i], 0))) { + aot_set_last_error("llvm add const or pointer type failed."); + return false; + } + + snprintf(buf, sizeof(buf), "%s%d", "elem", i); + if (!(elem_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, I32_TYPE, + func_ctx->argv_buf, &elem_idx, 1, buf)) + || !(elem_ptr = LLVMBuildBitCast(comp_ctx->builder, elem_ptr, + elem_ptr_type, buf))) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + if (!(res = LLVMBuildStore(comp_ctx->builder, param_values[i], + elem_ptr))) { + aot_set_last_error("llvm build store failed."); + return false; + } + LLVMSetAlignment(res, 1); + + cell_num += wasm_value_type_cell_num_internal(aot_func_type->types[i], + comp_ctx->pointer_size); + } + + func_param_values[0] = func_ctx->exec_env; + func_param_values[1] = func_idx; + func_param_values[2] = I32_CONST(param_cell_num); + func_param_values[3] = func_ctx->argv_buf; + + if (!func_param_values[2]) { + aot_set_last_error("llvm create const failed."); + return false; + } + + /* call aot_invoke_native() function */ + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, + func_param_values, 4, "res"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + /* get function return value */ + if (wasm_ret_type != VALUE_TYPE_VOID) { + if (!(ret_ptr_type = LLVMPointerType(ret_type, 0))) { + aot_set_last_error("llvm add pointer type failed."); + return false; + } + + if (!(value_ret = + LLVMBuildBitCast(comp_ctx->builder, func_ctx->argv_buf, + ret_ptr_type, "argv_ret"))) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + if (!(*p_value_ret = LLVMBuildLoad2(comp_ctx->builder, ret_type, + value_ret, "value_ret"))) { + aot_set_last_error("llvm build load failed."); + return false; + } + } + + *p_res = res; + return true; +} + +static bool +call_aot_invoke_c_api_native(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 import_func_idx, AOTFuncType *aot_func_type, + LLVMValueRef *params) +{ + LLVMTypeRef int8_ptr_type, param_types[6], ret_type; + LLVMTypeRef value_ptr_type = NULL, value_type = NULL; + LLVMTypeRef func_type, func_ptr_type; + LLVMValueRef param_values[6], res, func, value = NULL, offset; + LLVMValueRef c_api_func_imports, c_api_func_import; + LLVMValueRef c_api_params, c_api_results, value_ret; + LLVMValueRef c_api_param_kind, c_api_param_value; + LLVMValueRef c_api_result_value; + uint32 offset_c_api_func_imports, i; + uint32 offset_param_kind, offset_param_value; + char buf[16]; + + /* `int8 **` type */ + int8_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0); + if (!int8_ptr_type) { + aot_set_last_error("create llvm pointer type failed"); + return false; + } + + param_types[0] = INT8_PTR_TYPE; /* module_inst */ + param_types[1] = INT8_PTR_TYPE; /* CApiFuncImport *c_api_import */ + param_types[2] = INT8_PTR_TYPE; /* wasm_val_t *params */ + param_types[3] = I32_TYPE; /* uint32 param_count */ + param_types[4] = INT8_PTR_TYPE; /* wasm_val_t *results */ + param_types[5] = I32_TYPE; /* uint32 result_count */ + + ret_type = INT8_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_quick_invoke_c_api_native, 6); + + param_values[0] = func_ctx->aot_inst; + + /* Get module_inst->c_api_func_imports, jit mode WASMModuleInstance is the + * same layout with AOTModuleInstance */ + offset_c_api_func_imports = offsetof(AOTModuleInstance, c_api_func_imports); + offset = I32_CONST(offset_c_api_func_imports); + CHECK_LLVM_CONST(offset); + c_api_func_imports = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, + &offset, 1, "c_api_func_imports_addr"); + c_api_func_imports = + LLVMBuildBitCast(comp_ctx->builder, c_api_func_imports, int8_ptr_type, + "c_api_func_imports_ptr"); + c_api_func_imports = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, c_api_func_imports, + "c_api_func_imports"); + + /* Get &c_api_func_imports[func_idx], note size of CApiFuncImport + is pointer_size * 3 */ + offset = I32_CONST((comp_ctx->pointer_size * 3) * import_func_idx); + CHECK_LLVM_CONST(offset); + c_api_func_import = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_func_imports, + &offset, 1, "c_api_func_import"); + + param_values[1] = c_api_func_import; + param_values[2] = c_api_params = func_ctx->argv_buf; + param_values[3] = I32_CONST(aot_func_type->param_count); + CHECK_LLVM_CONST(param_values[3]); + + /* Ensure sizeof(wasm_val_t) is 16 bytes */ + offset = I32_CONST(sizeof(wasm_val_t) * aot_func_type->param_count); + c_api_results = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->argv_buf, + &offset, 1, "results"); + param_values[4] = c_api_results; + + param_values[5] = I32_CONST(aot_func_type->result_count); + CHECK_LLVM_CONST(param_values[5]); + + /* Set each c api param */ + for (i = 0; i < aot_func_type->param_count; i++) { + /* Ensure sizeof(wasm_val_t) is 16 bytes */ + offset_param_kind = sizeof(wasm_val_t) * i; + offset = I32_CONST(offset_param_kind); + CHECK_LLVM_CONST(offset); + c_api_param_kind = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_params, + &offset, 1, "c_api_param_kind_addr"); + c_api_param_kind = + LLVMBuildBitCast(comp_ctx->builder, c_api_param_kind, INT8_PTR_TYPE, + "c_api_param_kind_ptr"); + + switch (aot_func_type->types[i]) { + case VALUE_TYPE_I32: + value = I8_CONST(WASM_I32); + break; + case VALUE_TYPE_F32: + value = I8_CONST(WASM_F32); + break; + case VALUE_TYPE_I64: + value = I8_CONST(WASM_I64); + break; + case VALUE_TYPE_F64: + value = I8_CONST(WASM_F64); + break; + default: + bh_assert(0); + break; + } + CHECK_LLVM_CONST(value); + + LLVMBuildStore(comp_ctx->builder, value, c_api_param_kind); + + /* Ensure offsetof(wasm_val_t, of) is 8 bytes */ + offset_param_value = offset_param_kind + offsetof(wasm_val_t, of); + offset = I32_CONST(offset_param_value); + CHECK_LLVM_CONST(offset); + c_api_param_value = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_params, + &offset, 1, "c_api_param_value_addr"); + + switch (aot_func_type->types[i]) { + case VALUE_TYPE_I32: + value_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_F32: + value_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + value_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F64: + value_ptr_type = F64_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + c_api_param_value = + LLVMBuildBitCast(comp_ctx->builder, c_api_param_value, + value_ptr_type, "c_api_param_value_ptr"); + LLVMBuildStore(comp_ctx->builder, params[i], c_api_param_value); + } + + /* Call the function */ + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 6, "call"))) { + aot_set_last_error("LLVM build call failed."); + goto fail; + } + + /* Check whether exception was thrown when executing the function */ + if (comp_ctx->enable_bound_check + && !check_call_return(comp_ctx, func_ctx, res)) { + goto fail; + } + + for (i = 0; i < aot_func_type->result_count; i++) { + /* Ensure sizeof(wasm_val_t) is 16 bytes and + offsetof(wasm_val_t, of) is 8 bytes */ + uint32 offset_result_value = + sizeof(wasm_val_t) * i + offsetof(wasm_val_t, of); + + offset = I32_CONST(offset_result_value); + CHECK_LLVM_CONST(offset); + c_api_result_value = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_results, + &offset, 1, "c_api_result_value_addr"); + + switch (aot_func_type->types[aot_func_type->param_count + i]) { + case VALUE_TYPE_I32: + value_type = I32_TYPE; + value_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_F32: + value_type = F32_TYPE; + value_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + value_type = I64_TYPE; + value_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F64: + value_type = F64_TYPE; + value_ptr_type = F64_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + c_api_result_value = + LLVMBuildBitCast(comp_ctx->builder, c_api_result_value, + value_ptr_type, "c_api_result_value_ptr"); + snprintf(buf, sizeof(buf), "%s%u", "ret", i); + value_ret = LLVMBuildLoad2(comp_ctx->builder, value_type, + c_api_result_value, buf); + + PUSH(value_ret, aot_func_type->types[aot_func_type->param_count + i]); + } + + return true; +fail: + return false; +} + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 +static bool +call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef func_idx) +{ + LLVMValueRef param_values[2], ret_value, value, func; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMBasicBlockRef frame_alloc_fail, frame_alloc_success; + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + + param_types[0] = comp_ctx->exec_env_type; + param_types[1] = I32_TYPE; + ret_type = INT8_TYPE; + +#if WASM_ENABLE_JIT != 0 + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_alloc_frame, 2); + else +#endif + GET_AOT_FUNCTION(aot_alloc_frame, 2); + + param_values[0] = func_ctx->exec_env; + param_values[1] = func_idx; + + if (!(ret_value = + LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 2, "call_aot_alloc_frame"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + if (!(ret_value = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGT, ret_value, + I8_ZERO, "frame_alloc_ret"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + + ADD_BASIC_BLOCK(frame_alloc_fail, "frame_alloc_fail"); + ADD_BASIC_BLOCK(frame_alloc_success, "frame_alloc_success"); + + LLVMMoveBasicBlockAfter(frame_alloc_fail, block_curr); + LLVMMoveBasicBlockAfter(frame_alloc_success, block_curr); + + if (!LLVMBuildCondBr(comp_ctx->builder, ret_value, frame_alloc_success, + frame_alloc_fail)) { + aot_set_last_error("llvm build cond br failed."); + return false; + } + + /* If frame alloc failed, return this function + so the runtime can catch the exception */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, frame_alloc_fail); + if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + return false; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, frame_alloc_success); + + return true; + +fail: + return false; +} + +static bool +alloc_frame_for_aot_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 func_idx) +{ + LLVMValueRef wasm_stack_top_bound = func_ctx->wasm_stack_top_bound; + LLVMValueRef wasm_stack_top_ptr = func_ctx->wasm_stack_top_ptr, + wasm_stack_top; + LLVMValueRef wasm_stack_top_max, offset, cmp; + LLVMValueRef cur_frame, new_frame, prev_frame_ptr; + LLVMValueRef cur_frame_ptr = func_ctx->cur_frame_ptr; + LLVMValueRef func_idx_ptr, func_idx_val, func_inst_ptr, func_inst; + LLVMTypeRef int8_ptr_type; + LLVMBasicBlockRef check_wasm_stack_succ; + uint32 import_func_count = comp_ctx->comp_data->import_func_count; + uint32 param_cell_num = 0, local_cell_num = 0, i; + uint32 max_local_cell_num, max_stack_cell_num; + uint32 all_cell_num, frame_size, frame_size_with_outs_area; + uint32 aot_frame_ptr_num = offsetof(AOTFrame, lp) / sizeof(uintptr_t); + AOTImportFunc *import_funcs = comp_ctx->comp_data->import_funcs; + AOTFuncType *aot_func_type; + AOTFunc *aot_func = NULL; + + /* `int8 **` type */ + int8_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0); + if (!int8_ptr_type) { + aot_set_last_error("create llvm pointer type failed"); + return false; + } + + /* Get param_cell_num, local_cell_num and max_stack_cell_num */ + if (func_idx < import_func_count) { + aot_func_type = import_funcs[func_idx].func_type; + for (i = 0; i < aot_func_type->param_count; i++) + param_cell_num += wasm_value_type_cell_num_internal( + aot_func_type->types[i], comp_ctx->pointer_size); + max_local_cell_num = param_cell_num > 2 ? param_cell_num : 2; + max_stack_cell_num = 0; + } + else { + aot_func = comp_ctx->comp_data->funcs[func_idx - import_func_count]; + param_cell_num = aot_func->param_cell_num; + local_cell_num = aot_func->local_cell_num; + max_local_cell_num = param_cell_num + local_cell_num; + max_stack_cell_num = aot_func->max_stack_cell_num; + } + + all_cell_num = max_local_cell_num + max_stack_cell_num; + + /* Get size of the frame to allocate and get size with outs_area to + check whether wasm operand stack is overflow */ + if (!comp_ctx->is_jit_mode) { + /* Refer to aot_alloc_frame */ + if (!comp_ctx->enable_gc) { + frame_size = frame_size_with_outs_area = + comp_ctx->pointer_size * aot_frame_ptr_num; + } + else { + frame_size = comp_ctx->pointer_size * aot_frame_ptr_num + + align_uint(all_cell_num * 5, 4); + frame_size_with_outs_area = + frame_size + comp_ctx->pointer_size * aot_frame_ptr_num + + max_stack_cell_num * 4; + } + } + else { + /* Refer to wasm_interp_interp_frame_size */ + if (!comp_ctx->enable_gc) { + frame_size = frame_size_with_outs_area = + offsetof(WASMInterpFrame, lp); + } + else { + frame_size = + offsetof(WASMInterpFrame, lp) + align_uint(all_cell_num * 5, 4); + frame_size_with_outs_area = frame_size + + offsetof(WASMInterpFrame, lp) + + max_stack_cell_num * 4; + } + } + + cur_frame = func_ctx->cur_frame; + + if (!comp_ctx->enable_gc) { + offset = I32_CONST(frame_size); + CHECK_LLVM_CONST(offset); + if (!(wasm_stack_top = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &offset, 1, "wasm_stack_top"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + offset = I32_CONST(frame_size * 2); + CHECK_LLVM_CONST(offset); + if (!(wasm_stack_top_max = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &offset, 1, "wasm_stack_top_max"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + } + else { + /* Get exec_env->wasm_stack.top */ + if (!(wasm_stack_top = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, + wasm_stack_top_ptr, "wasm_stack_top"))) { + aot_set_last_error("load wasm_stack.top failed"); + return false; + } + + /* Check whether wasm operand stack is overflow */ + offset = I32_CONST(frame_size_with_outs_area); + CHECK_LLVM_CONST(offset); + if (!(wasm_stack_top_max = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, wasm_stack_top, &offset, 1, + "wasm_stack_top_max"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + } + + new_frame = wasm_stack_top; + + if (!(check_wasm_stack_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_wasm_stack_succ"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + + LLVMMoveBasicBlockAfter(check_wasm_stack_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGT, wasm_stack_top_max, + wasm_stack_top_bound, "cmp"))) { + aot_set_last_error("llvm build icmp failed"); + return false; + } + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_OPERAND_STACK_OVERFLOW, + true, cmp, check_wasm_stack_succ))) { + return false; + } + +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + LLVMValueRef wasm_stack_top_new, frame_ref, frame_ref_ptr; + uint32 j, k; + + /* exec_env->wasm_stack.top += frame_size */ + offset = I32_CONST(frame_size); + CHECK_LLVM_CONST(offset); + if (!(wasm_stack_top_new = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, wasm_stack_top, &offset, 1, + "wasm_stack_top_new"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, wasm_stack_top_new, + wasm_stack_top_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + if (func_idx < import_func_count) { + LLVMValueRef frame_sp, frame_sp_ptr; + + /* Only need to initialize new_frame->sp when it's import function + otherwise they will be committed in AOT code if needed */ + + /* new_frame->sp = new_frame->lp + max_local_cell_num */ + if (!comp_ctx->is_jit_mode) + offset = I32_CONST(comp_ctx->pointer_size * 5); + else + offset = I32_CONST(offsetof(WASMInterpFrame, sp)); + CHECK_LLVM_CONST(offset); + if (!(frame_sp_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, new_frame, &offset, 1, + "frame_sp_addr")) + || !(frame_sp_ptr = + LLVMBuildBitCast(comp_ctx->builder, frame_sp_ptr, + int8_ptr_type, "frame_sp_ptr"))) { + aot_set_last_error("llvm get frame_sp_ptr failed"); + return false; + } + + if (!comp_ctx->is_jit_mode) + offset = I32_CONST(comp_ctx->pointer_size * aot_frame_ptr_num + + max_local_cell_num * sizeof(uint32)); + else + offset = I32_CONST(offsetof(WASMInterpFrame, lp) + + max_local_cell_num * sizeof(uint32)); + CHECK_LLVM_CONST(offset); + if (!(frame_sp = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + new_frame, &offset, 1, + "frame_sp"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, frame_sp, frame_sp_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + + if (!comp_ctx->is_jit_mode) { + /* new_frame->frame_ref = new_frame->lp + max_local_cell_num + + max_stack_cell_num */ + offset = I32_CONST(comp_ctx->pointer_size * 6); + CHECK_LLVM_CONST(offset); + if (!(frame_ref_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, new_frame, &offset, 1, + "frame_ref_addr")) + || !(frame_ref_ptr = + LLVMBuildBitCast(comp_ctx->builder, frame_ref_ptr, + int8_ptr_type, "frame_ref_ptr"))) { + aot_set_last_error("llvm get frame_ref_ptr failed"); + return false; + } + + offset = I32_CONST(comp_ctx->pointer_size * aot_frame_ptr_num + + (max_local_cell_num + max_stack_cell_num) + * sizeof(uint32)); + CHECK_LLVM_CONST(offset); + if (!(frame_ref = LLVMBuildInBoundsGEP2(comp_ctx->builder, + INT8_TYPE, new_frame, + &offset, 1, "frame_ref"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, frame_ref, frame_ref_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + else { + /* Get frame_ref in WASMInterpFrame */ + offset = I32_CONST(offsetof(WASMInterpFrame, lp) + + (max_local_cell_num + max_stack_cell_num) + * sizeof(uint32)); + CHECK_LLVM_CONST(offset); + if (!(frame_ref = LLVMBuildInBoundsGEP2(comp_ctx->builder, + INT8_TYPE, new_frame, + &offset, 1, "frame_ref"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + } + + /* Initialize frame ref flags for import function only in JIT mode */ + if (func_idx < import_func_count && comp_ctx->is_jit_mode) { + aot_func_type = import_funcs[func_idx].func_type; + for (i = 0, j = 0; i < aot_func_type->param_count; i++) { + if (aot_is_type_gc_reftype(aot_func_type->types[i]) + && !wasm_is_reftype_i31ref(aot_func_type->types[i])) { + for (k = 0; k < comp_ctx->pointer_size / sizeof(uint32); + k++) { + /* frame_ref[j++] = 1 */ + offset = I32_CONST(j); + CHECK_LLVM_CONST(offset); + frame_ref_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, frame_ref, &offset, 1, + "frame_ref_ptr"); + if (!LLVMBuildStore(comp_ctx->builder, I8_ONE, + frame_ref_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + j++; + } + } + else { + uint32 value_type_cell_num = + wasm_value_type_cell_num_internal( + aot_func_type->types[i], comp_ctx->pointer_size); + for (k = 0; k < value_type_cell_num; k++) { + /* frame_ref[j++] = 0 */ + offset = I32_CONST(j); + CHECK_LLVM_CONST(offset); + frame_ref_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, frame_ref, &offset, 1, + "frame_ref_ptr"); + if (!LLVMBuildStore(comp_ctx->builder, I8_ZERO, + frame_ref_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + j++; + } + } + } + + for (; j < 2; j++) { + /* frame_ref[j++] = 0 */ + offset = I32_CONST(j); + CHECK_LLVM_CONST(offset); + frame_ref_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, frame_ref, &offset, 1, + "frame_ref_ptr"); + if (!LLVMBuildStore(comp_ctx->builder, I8_ZERO, + frame_ref_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + /* new_frame->prev_frame = cur_frame */ + if (!(prev_frame_ptr = LLVMBuildBitCast(comp_ctx->builder, new_frame, + int8_ptr_type, "prev_frame_ptr"))) { + aot_set_last_error("llvm build bitcast failed"); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, cur_frame, prev_frame_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + if (!comp_ctx->is_jit_mode) { + /* aot mode: new_frame->func_idx = func_idx */ + func_idx_val = comp_ctx->pointer_size == sizeof(uint64) + ? I64_CONST(func_idx) + : I32_CONST(func_idx); + offset = I32_CONST(comp_ctx->pointer_size); + CHECK_LLVM_CONST(func_idx_val); + CHECK_LLVM_CONST(offset); + if (!(func_idx_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, new_frame, + &offset, 1, "func_idx_addr")) + || !(func_idx_ptr = + LLVMBuildBitCast(comp_ctx->builder, func_idx_ptr, + INTPTR_T_PTR_TYPE, "func_idx_ptr"))) { + aot_set_last_error("llvm get func_idx_ptr failed"); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, func_idx_val, func_idx_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + else { + /* jit mode: frame->function = module_inst->e->functions + func_index */ + LLVMValueRef functions; + uint32 offset_functions = + get_module_inst_extra_offset(comp_ctx) + + offsetof(WASMModuleInstanceExtra, functions); + + offset = I32_CONST(offset_functions); + CHECK_LLVM_CONST(offset); + if (!(functions = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "functions_addr"))) { + aot_set_last_error("llvm build inbounds gep failed"); + return false; + } + + if (!(functions = LLVMBuildBitCast(comp_ctx->builder, functions, + int8_ptr_type, "functions_ptr"))) { + aot_set_last_error("llvm build bitcast failed"); + return false; + } + + if (!(functions = LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, + functions, "functions"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + offset = I32_CONST(sizeof(WASMFunctionInstance) * func_idx); + CHECK_LLVM_CONST(offset); + if (!(func_inst = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, functions, + &offset, 1, "func_inst"))) { + aot_set_last_error("llvm build inbounds gep failed"); + return false; + } + + offset = I32_CONST(offsetof(WASMInterpFrame, function)); + CHECK_LLVM_CONST(offset); + if (!(func_inst_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, new_frame, + &offset, 1, "func_inst_addr")) + || !(func_inst_ptr = + LLVMBuildBitCast(comp_ctx->builder, func_inst_ptr, + int8_ptr_type, "func_inst_ptr"))) { + aot_set_last_error("llvm get func_inst_ptr failed"); + return false; + } + + if (!LLVMBuildStore(comp_ctx->builder, func_inst, func_inst_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + + /* No need to initialize new_frame->sp and new_frame->ip_offset + since they will be committed in AOT/JIT code if needed */ + + /* exec_env->cur_frame = new_frame */ + if (!LLVMBuildStore(comp_ctx->builder, new_frame, cur_frame_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + if (comp_ctx->enable_perf_profiling || comp_ctx->enable_memory_profiling) { + LLVMTypeRef param_types[2], func_type, func_ptr_type; + LLVMValueRef param_values[2], func = NULL, res; + char *func_name = "aot_frame_update_profile_info"; + + /* Call aot_frame_update_profile_info for AOT or + llvm_jit_frame_update_profile_info for JIT */ + + param_types[0] = comp_ctx->exec_env_type; + param_types[1] = INT8_TYPE; + + if (!(func_type = LLVMFunctionType(VOID_TYPE, param_types, 2, false))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + if (comp_ctx->is_jit_mode) { + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + +#if WASM_ENABLE_JIT != 0 \ + && (WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_MEMORY_PROFILING != 0) + /* JIT mode, call the function directly */ + if (!(func = I64_CONST( + (uint64)(uintptr_t)llvm_jit_frame_update_profile_info)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } +#endif + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + func_index = aot_get_native_symbol_index(comp_ctx, func_name); + if (func_index < 0) { + return false; + } + if (!(func = + aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) + && !(func = LLVMAddFunction(func_ctx->module, func_name, + func_type))) { + aot_set_last_error("add LLVM function failed."); + return false; + } + } + + param_values[0] = func_ctx->exec_env; + param_values[1] = I8_ONE; + + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, ""))) { + aot_set_last_error("llvm build call failed."); + return false; + } + } + + return true; +fail: + + return false; +} + +static bool +free_frame_for_aot_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef cur_frame_ptr = func_ctx->cur_frame_ptr, cur_frame; + LLVMValueRef wasm_stack_top_ptr = func_ctx->wasm_stack_top_ptr; + LLVMTypeRef int8_ptr_type; + + /* `int8 **` type */ + int8_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0); + if (!int8_ptr_type) { + aot_set_last_error("create llvm pointer type failed"); + return false; + } + + if (comp_ctx->enable_perf_profiling) { + LLVMTypeRef param_types[2], func_type, func_ptr_type; + LLVMValueRef param_values[2], func = NULL, res; + char *func_name = "aot_frame_update_profile_info"; + + /* call aot_frame_update_profile_info for AOT or + llvm_jit_frame_update_profile_info for JIT */ + + param_types[0] = comp_ctx->exec_env_type; + param_types[1] = INT8_TYPE; + + if (!(func_type = LLVMFunctionType(VOID_TYPE, param_types, 2, false))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + if (comp_ctx->is_jit_mode) { + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + +#if WASM_ENABLE_JIT != 0 \ + && (WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_MEMORY_PROFILING != 0) + /* JIT mode, call the function directly */ + if (!(func = I64_CONST( + (uint64)(uintptr_t)llvm_jit_frame_update_profile_info)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } +#endif + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + func_index = aot_get_native_symbol_index(comp_ctx, func_name); + if (func_index < 0) { + return false; + } + if (!(func = + aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) + && !(func = LLVMAddFunction(func_ctx->module, func_name, + func_type))) { + aot_set_last_error("add LLVM function failed."); + return false; + } + } + + param_values[0] = func_ctx->exec_env; + param_values[1] = I8_ZERO; + + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, ""))) { + aot_set_last_error("llvm build call failed."); + return false; + } + } + + if (comp_ctx->enable_gc) { + /* cur_frame = exec_env->cur_frame */ + if (!(cur_frame = LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, + cur_frame_ptr, "cur_frame"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + /* exec_env->wasm_stack.top = cur_frame */ + if (!LLVMBuildStore(comp_ctx->builder, cur_frame, wasm_stack_top_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + + /* exec_env->cur_frame = prev_frame */ + if (!LLVMBuildStore(comp_ctx->builder, func_ctx->cur_frame, + cur_frame_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + return true; +} +#endif /* end of WASM_ENABLE_AOT_STACK_FRAME != 0 */ + +/** + * Check whether the app address and its buffer are inside the linear memory, + * if no, throw exception + */ +static bool +check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_str_arg, LLVMValueRef app_addr, + LLVMValueRef buf_size, + LLVMValueRef *p_native_addr_converted) +{ + LLVMTypeRef func_type, func_ptr_type, func_param_types[5]; + LLVMValueRef func, func_param_values[5], res, native_addr_ptr; + char *func_name = "aot_check_app_addr_and_convert"; + + /* prepare function type of aot_check_app_addr_and_convert */ + func_param_types[0] = comp_ctx->aot_inst_type; /* module_inst */ + func_param_types[1] = INT8_TYPE; /* is_str_arg */ + func_param_types[2] = I64_TYPE; /* app_offset */ + func_param_types[3] = I64_TYPE; /* buf_size */ + func_param_types[4] = + comp_ctx->basic_types.int8_pptr_type; /* p_native_addr */ + if (!(func_type = + LLVMFunctionType(INT8_TYPE, func_param_types, 5, false))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + /* prepare function pointer */ + if (comp_ctx->is_jit_mode) { + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + /* JIT mode, call the function directly */ + if (!(func = + I64_CONST((uint64)(uintptr_t)jit_check_app_addr_and_convert)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + func_index = aot_get_native_symbol_index(comp_ctx, func_name); + if (func_index < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) + && !(func = + LLVMAddFunction(func_ctx->module, func_name, func_type))) { + aot_set_last_error("add LLVM function failed."); + return false; + } + } + + if (!(native_addr_ptr = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->argv_buf, + comp_ctx->basic_types.int8_pptr_type, "p_native_addr"))) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + func_param_values[0] = func_ctx->aot_inst; + func_param_values[1] = I8_CONST(is_str_arg); + func_param_values[2] = app_addr; + func_param_values[3] = buf_size; + func_param_values[4] = native_addr_ptr; + + if (!func_param_values[1]) { + aot_set_last_error("llvm create const failed."); + return false; + } + + /* call aot_check_app_addr_and_convert() function */ + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, + func_param_values, 5, "res"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + /* Check whether exception was thrown when executing the function */ + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) + && !check_call_return(comp_ctx, func_ctx, res)) { + return false; + } + + if (!(*p_native_addr_converted = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, native_addr_ptr, + "native_addr"))) { + aot_set_last_error("llvm build load failed."); + return false; + } + + return true; +} + +static void +aot_estimate_and_record_stack_usage_for_function_call( + const AOTCompContext *comp_ctx, AOTFuncContext *caller_func_ctx, + const AOTFuncType *callee_func_type) +{ + unsigned int size; + + if (!(comp_ctx->enable_stack_bound_check + || comp_ctx->enable_stack_estimation)) { + return; + } + + size = + aot_estimate_stack_usage_for_function_call(comp_ctx, callee_func_type); + /* + * only record the max value, assuming that LLVM emits machine code + * which rewinds the stack before making the next call in the + * function. + */ + if (caller_func_ctx->stack_consumption_for_func_call < size) { + caller_func_ctx->stack_consumption_for_func_call = size; + } +} + +static bool +commit_params_to_frame_of_import_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + AOTFuncType *func_type, + const LLVMValueRef *param_values) +{ + uint32 i, n; + + for (i = 0, n = 0; i < func_type->param_count; i++, n++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: + if (!aot_frame_store_value( + comp_ctx, param_values[i], VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local_in_outs_area(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_I64: + if (!aot_frame_store_value( + comp_ctx, param_values[i], VALUE_TYPE_I64, + func_ctx->cur_frame, + offset_of_local_in_outs_area(comp_ctx, n))) + return false; + n++; + break; + case VALUE_TYPE_F32: + if (!aot_frame_store_value( + comp_ctx, param_values[i], VALUE_TYPE_F32, + func_ctx->cur_frame, + offset_of_local_in_outs_area(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_F64: + if (!aot_frame_store_value( + comp_ctx, param_values[i], VALUE_TYPE_F64, + func_ctx->cur_frame, + offset_of_local_in_outs_area(comp_ctx, n))) + return false; + n++; + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + if (comp_ctx->enable_ref_types) { + if (!aot_frame_store_value( + comp_ctx, param_values[i], VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local_in_outs_area(comp_ctx, n))) + return false; + } +#if WASM_ENABLE_GC != 0 + else if (comp_ctx->enable_gc) { + if (!aot_frame_store_value( + comp_ctx, param_values[i], VALUE_TYPE_GC_REF, + func_ctx->cur_frame, + offset_of_local_in_outs_area(comp_ctx, n))) + return false; + if (comp_ctx->pointer_size == sizeof(uint64)) + n++; + } +#endif + else { + bh_assert(0); + } + break; +#if WASM_ENABLE_GC != 0 + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + /* case REF_TYPE_FUNCREF: */ + /* case REF_TYPE_EXTERNREF: */ + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case VALUE_TYPE_GC_REF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + if (!aot_frame_store_value( + comp_ctx, param_values[i], VALUE_TYPE_GC_REF, + func_ctx->cur_frame, + offset_of_local_in_outs_area(comp_ctx, n))) + return false; + if (comp_ctx->pointer_size == sizeof(uint64)) + n++; + break; +#endif + default: + bh_assert(0); + break; + } + } + + return true; +} + +bool +aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 func_idx, bool tail_call) +{ + uint32 import_func_count = comp_ctx->comp_data->import_func_count; + AOTImportFunc *import_funcs = comp_ctx->comp_data->import_funcs; + uint32 func_count = comp_ctx->func_ctx_count, param_cell_num = 0; + uint32 ext_ret_cell_num = 0, cell_num = 0; + AOTFuncContext **func_ctxes = comp_ctx->func_ctxes; + AOTFuncType *func_type; + LLVMTypeRef *param_types = NULL, ret_type; + LLVMTypeRef ext_ret_ptr_type; + LLVMValueRef *param_values = NULL, value_ret = NULL, func; + LLVMValueRef import_func_idx, res; + LLVMValueRef ext_ret, ext_ret_ptr, ext_ret_idx; + int32 i, j = 0, param_count, result_count, ext_ret_count; + uint64 total_size; + uint8 wasm_ret_type; + uint8 *ext_ret_types = NULL; + const char *signature = NULL; + bool ret = false; + char buf[32]; + bool quick_invoke_c_api_import = false; + + /* Check function index */ + if (func_idx >= import_func_count + func_count) { + aot_set_last_error("Function index out of range."); + return false; + } + + /* Get function type */ + if (func_idx < import_func_count) { + func_type = import_funcs[func_idx].func_type; + signature = import_funcs[func_idx].signature; + } + else { + func_type = + func_ctxes[func_idx - import_func_count]->aot_func->func_type; + } + aot_estimate_and_record_stack_usage_for_function_call(comp_ctx, func_ctx, + func_type); + + /* Commit stack operands, sp and ip */ + if (comp_ctx->aot_frame) { + if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + /* Commit sp if gc is enabled and commit ip for func call */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, comp_ctx->enable_gc, + true)) + return false; + } + + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + } + + if (comp_ctx->enable_aux_stack_frame) { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + if (!alloc_frame_for_aot_func(comp_ctx, func_ctx, func_idx)) + return false; +#endif + } + + /* Get param cell number */ + param_cell_num = func_type->param_cell_num; + + /* Allocate memory for parameters. + * Parameters layout: + * - exec env + * - wasm function's parameters + * - extra results'(except the first one) addresses + */ + param_count = (int32)func_type->param_count; + result_count = (int32)func_type->result_count; + ext_ret_count = result_count > 1 ? result_count - 1 : 0; + total_size = + sizeof(LLVMValueRef) * (uint64)(param_count + 1 + ext_ret_count); + if (total_size >= UINT32_MAX + || !(param_values = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + return false; + } + + /* First parameter is exec env */ + param_values[j++] = func_ctx->exec_env; + + /* Pop parameters from stack */ + for (i = param_count - 1; i >= 0; i--) + POP(param_values[i + j], func_type->types[i]); + + /* Set parameters for multiple return values, the first return value + is returned by function return value, and the other return values + are returned by function parameters with pointer types */ + if (ext_ret_count > 0) { + ext_ret_types = func_type->types + param_count + 1; + ext_ret_cell_num = wasm_get_cell_num(ext_ret_types, ext_ret_count); + if (ext_ret_cell_num > 64) { + aot_set_last_error("prepare extra results's return " + "address arguments failed: " + "maximum 64 parameter cell number supported."); + goto fail; + } + + for (i = 0; i < ext_ret_count; i++) { + if (!(ext_ret_idx = I32_CONST(cell_num)) + || !(ext_ret_ptr_type = + LLVMPointerType(TO_LLVM_TYPE(ext_ret_types[i]), 0))) { + aot_set_last_error("llvm add const or pointer type failed."); + goto fail; + } + + snprintf(buf, sizeof(buf), "ext_ret%d_ptr", i); + if (!(ext_ret_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, I32_TYPE, func_ctx->argv_buf, + &ext_ret_idx, 1, buf))) { + aot_set_last_error("llvm build GEP failed."); + goto fail; + } + snprintf(buf, sizeof(buf), "ext_ret%d_ptr_cast", i); + if (!(ext_ret_ptr = LLVMBuildBitCast(comp_ctx->builder, ext_ret_ptr, + ext_ret_ptr_type, buf))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + param_values[param_count + 1 + i] = ext_ret_ptr; + cell_num += wasm_value_type_cell_num_internal( + ext_ret_types[i], comp_ctx->pointer_size); + } + } + + if (func_idx < import_func_count) { + if (comp_ctx->enable_aux_stack_frame + && !commit_params_to_frame_of_import_func( + comp_ctx, func_ctx, func_type, param_values + 1)) { + goto fail; + } + + if (!(import_func_idx = I32_CONST(func_idx))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + /* Initialize parameter types of the LLVM function */ + total_size = sizeof(LLVMTypeRef) * (uint64)(param_count + 1); + if (total_size >= UINT32_MAX + || !(param_types = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + + j = 0; + param_types[j++] = comp_ctx->exec_env_type; + + for (i = 0; i < param_count; i++, j++) { + param_types[j] = TO_LLVM_TYPE(func_type->types[i]); + + /* If the signature can be gotten, e.g. the signature of the builtin + native libraries, just check the app offset and buf size, and + then convert app offset to native addr and call the native func + directly, no need to call aot_invoke_native to call it */ + if (signature) { + LLVMValueRef native_addr, native_addr_size; + if (signature[i + 1] == '*' || signature[i + 1] == '$') { + param_types[j] = INT8_PTR_TYPE; + } + if (signature[i + 1] == '*') { + if (signature[i + 2] == '~') + native_addr_size = param_values[i + 2]; + else + native_addr_size = I64_CONST(1); + if (!(native_addr_size = LLVMBuildZExtOrBitCast( + comp_ctx->builder, native_addr_size, I64_TYPE, + "native_addr_size_i64"))) { + aot_set_last_error("llvm build zextOrBitCast failed."); + goto fail; + } + if (!(param_values[j] = LLVMBuildZExtOrBitCast( + comp_ctx->builder, param_values[j], I64_TYPE, + "native_addr_i64"))) { + aot_set_last_error("llvm build zextOrBitCast failed."); + goto fail; + } + if (!check_app_addr_and_convert( + comp_ctx, func_ctx, false, param_values[j], + native_addr_size, &native_addr)) { + goto fail; + } + param_values[j] = native_addr; + } + else if (signature[i + 1] == '$') { + native_addr_size = I64_ZERO; + if (!(param_values[j] = LLVMBuildZExtOrBitCast( + comp_ctx->builder, param_values[j], I64_TYPE, + "native_addr_i64"))) { + aot_set_last_error("llvm build zextOrBitCast failed."); + goto fail; + } + if (!check_app_addr_and_convert( + comp_ctx, func_ctx, true, param_values[j], + native_addr_size, &native_addr)) { + goto fail; + } + param_values[j] = native_addr; + } + } + } + + if (func_type->result_count) { + wasm_ret_type = func_type->types[func_type->param_count]; + ret_type = TO_LLVM_TYPE(wasm_ret_type); + } + else { + wasm_ret_type = VALUE_TYPE_VOID; + ret_type = VOID_TYPE; + } + + if (!signature) { + if (comp_ctx->quick_invoke_c_api_import) { + uint32 buf_size_needed = + sizeof(wasm_val_t) * (param_count + result_count); + + /* length of exec_env->argv_buf is 64 */ + if (buf_size_needed < sizeof(uint32) * 64) { + for (i = 0; i < param_count + result_count; i++) { + /* Only support i32/i64/f32/f64 now */ + if (!(func_type->types[i] == VALUE_TYPE_I32 + || func_type->types[i] == VALUE_TYPE_I64 + || func_type->types[i] == VALUE_TYPE_F32 + || func_type->types[i] == VALUE_TYPE_F64)) + break; + } + if (i == param_count + result_count) + quick_invoke_c_api_import = true; + } + } + if (quick_invoke_c_api_import) { + if (!call_aot_invoke_c_api_native(comp_ctx, func_ctx, func_idx, + func_type, param_values + 1)) + goto fail; + } + else { + /* call aot_invoke_native() */ + if (!call_aot_invoke_native_func( + comp_ctx, func_ctx, import_func_idx, func_type, + param_types + 1, param_values + 1, param_count, + param_cell_num, ret_type, wasm_ret_type, &value_ret, + &res)) + goto fail; + /* Check whether there was exception thrown when executing + the function */ + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) + && !check_call_return(comp_ctx, func_ctx, res)) + goto fail; + } + } + else { /* call native func directly */ + LLVMTypeRef native_func_type, func_ptr_type; + LLVMValueRef func_ptr; + + if (!(native_func_type = LLVMFunctionType( + ret_type, param_types, param_count + 1, false))) { + aot_set_last_error("llvm add function type failed."); + goto fail; + } + + if (!(func_ptr_type = LLVMPointerType(native_func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + goto fail; + } + + /* Load function pointer */ + if (!(func_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->func_ptrs, + &import_func_idx, 1, "native_func_ptr_tmp"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(func_ptr = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + func_ptr, "native_func_ptr"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(func = LLVMBuildBitCast(comp_ctx->builder, func_ptr, + func_ptr_type, "native_func"))) { + aot_set_last_error("llvm bit cast failed."); + goto fail; + } + + /* Call the function */ + if (!(value_ret = LLVMBuildCall2( + comp_ctx->builder, native_func_type, func, param_values, + (uint32)param_count + 1 + ext_ret_count, + (func_type->result_count > 0 ? "call" : "")))) { + aot_set_last_error("LLVM build call failed."); + goto fail; + } + + /* Check whether there was exception thrown when executing + the function */ + if (!check_exception_thrown(comp_ctx, func_ctx)) { + goto fail; + } + } + } + else { +#if LLVM_VERSION_MAJOR >= 14 + LLVMTypeRef llvm_func_type; +#endif + if (comp_ctx->is_indirect_mode) { + LLVMTypeRef func_ptr_type; + + if (!(func_ptr_type = LLVMPointerType( + func_ctxes[func_idx - import_func_count]->func_type, + 0))) { + aot_set_last_error("construct func ptr type failed."); + goto fail; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->func_ptrs, + func_ptr_type, func_idx))) { + goto fail; + } + } + else { + if (func_ctxes[func_idx - import_func_count] == func_ctx) { + /* recursive call */ + func = func_ctx->precheck_func; + } + else { + if (!comp_ctx->is_jit_mode) { + func = + func_ctxes[func_idx - import_func_count]->precheck_func; + } + else { +#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0) + func = + func_ctxes[func_idx - import_func_count]->precheck_func; +#else + /* JIT tier-up, load func ptr from func_ptrs[func_idx] */ + LLVMValueRef func_ptr, func_idx_const; + LLVMTypeRef func_ptr_type; + + if (!(func_idx_const = I32_CONST(func_idx))) { + aot_set_last_error("llvm build const failed."); + goto fail; + } + + if (!(func_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->func_ptrs, &func_idx_const, 1, + "func_ptr_tmp"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(func_ptr = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + func_ptr, "func_ptr"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(func_ptr_type = LLVMPointerType( + func_ctxes[func_idx - import_func_count] + ->func_type, + 0))) { + aot_set_last_error("construct func ptr type failed."); + goto fail; + } + + if (!(func = LLVMBuildBitCast(comp_ctx->builder, func_ptr, + func_ptr_type, + "indirect_func"))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } +#endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0) */ + } + } + } + +#if LLVM_VERSION_MAJOR >= 14 + llvm_func_type = func_ctxes[func_idx - import_func_count]->func_type; +#endif + + /* Call the function */ + if (!(value_ret = LLVMBuildCall2( + comp_ctx->builder, llvm_func_type, func, param_values, + (uint32)param_count + 1 + ext_ret_count, + (func_type->result_count > 0 ? "call" : "")))) { + aot_set_last_error("LLVM build call failed."); + goto fail; + } + + if (tail_call) + LLVMSetTailCall(value_ret, true); + + /* Check whether there was exception thrown when executing + the function */ + if (!tail_call + && (comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) + && !check_exception_thrown(comp_ctx, func_ctx)) + goto fail; + } + + if (func_type->result_count > 0 && !quick_invoke_c_api_import) { + /* Push the first result to stack */ + PUSH(value_ret, func_type->types[func_type->param_count]); + /* Load extra result from its address and push to stack */ + for (i = 0; i < ext_ret_count; i++) { + snprintf(buf, sizeof(buf), "func%d_ext_ret%d", func_idx, i); + if (!(ext_ret = LLVMBuildLoad2( + comp_ctx->builder, TO_LLVM_TYPE(ext_ret_types[i]), + param_values[1 + param_count + i], buf))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + PUSH(ext_ret, ext_ret_types[i]); + } + } + + if (comp_ctx->enable_aux_stack_frame) { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + if (!free_frame_for_aot_func(comp_ctx, func_ctx)) + goto fail; +#endif + } + + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx, false)) + goto fail; + } + + ret = true; +fail: + if (param_types) + wasm_runtime_free(param_types); + if (param_values) + wasm_runtime_free(param_values); + return ret; +} + +#if WASM_ENABLE_GC != 0 +static LLVMValueRef +call_aot_func_type_is_super_of_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef type_idx1, + LLVMValueRef type_idx2) +{ + LLVMValueRef param_values[3], ret_value, value, func; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + + param_types[0] = comp_ctx->aot_inst_type; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_TYPE; + +#if WASM_ENABLE_JIT != 0 + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_func_type_is_super_of, 3); + else +#endif + GET_AOT_FUNCTION(aot_func_type_is_super_of, 3); + + param_values[0] = func_ctx->aot_inst; + param_values[1] = type_idx1; + param_values[2] = type_idx2; + + if (!(ret_value = + LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 3, "call_aot_func_type_is_super_of"))) { + aot_set_last_error("llvm build call failed."); + return NULL; + } + + if (!(ret_value = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, ret_value, + I8_ZERO, "check_fail"))) { + aot_set_last_error("llvm build icmp failed."); + return NULL; + } + + return ret_value; + +fail: + return NULL; +} +#endif + +static bool +call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + AOTFuncType *aot_func_type, + LLVMValueRef func_type_idx, LLVMValueRef table_idx, + LLVMValueRef table_elem_idx, + LLVMTypeRef *param_types, + LLVMValueRef *param_values, uint32 param_count, + uint32 param_cell_num, uint32 result_count, + uint8 *wasm_ret_types, LLVMValueRef *value_rets, + LLVMValueRef *p_res) +{ + LLVMTypeRef func_type, func_ptr_type, func_param_types[6]; + LLVMTypeRef ret_type, ret_ptr_type, elem_ptr_type; + LLVMValueRef func, ret_idx, ret_ptr, elem_idx, elem_ptr; + LLVMValueRef func_param_values[6], res = NULL; + char buf[32], *func_name = "aot_call_indirect"; + uint32 i, cell_num = 0, ret_cell_num, argv_cell_num; + + /* prepare function type of aot_call_indirect */ + func_param_types[0] = comp_ctx->exec_env_type; /* exec_env */ + func_param_types[1] = I32_TYPE; /* table_idx */ + func_param_types[2] = I32_TYPE; /* table_elem_idx */ + func_param_types[3] = I32_TYPE; /* argc */ + func_param_types[4] = INT32_PTR_TYPE; /* argv */ + if (!(func_type = + LLVMFunctionType(INT8_TYPE, func_param_types, 5, false))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + /* prepare function pointer */ + if (comp_ctx->is_jit_mode) { + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + /* JIT mode, call the function directly */ + if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_call_indirect)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + func_index = aot_get_native_symbol_index(comp_ctx, func_name); + if (func_index < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) + && !(func = + LLVMAddFunction(func_ctx->module, func_name, func_type))) { + aot_set_last_error("add LLVM function failed."); + return false; + } + } + + ret_cell_num = wasm_get_cell_num(wasm_ret_types, result_count); + argv_cell_num = + param_cell_num > ret_cell_num ? param_cell_num : ret_cell_num; + if (argv_cell_num > 64) { + aot_set_last_error("prepare native arguments failed: " + "maximum 64 parameter cell number supported."); + return false; + } + + /* prepare frame_lp */ + for (i = 0; i < param_count; i++) { + if (!(elem_idx = I32_CONST(cell_num)) + || !(elem_ptr_type = LLVMPointerType(param_types[i], 0))) { + aot_set_last_error("llvm add const or pointer type failed."); + return false; + } + + snprintf(buf, sizeof(buf), "%s%d", "elem", i); + if (!(elem_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, I32_TYPE, + func_ctx->argv_buf, &elem_idx, 1, buf)) + || !(elem_ptr = LLVMBuildBitCast(comp_ctx->builder, elem_ptr, + elem_ptr_type, buf))) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + if (!(res = LLVMBuildStore(comp_ctx->builder, param_values[i], + elem_ptr))) { + aot_set_last_error("llvm build store failed."); + return false; + } + LLVMSetAlignment(res, 1); + + cell_num += wasm_value_type_cell_num_internal(aot_func_type->types[i], + comp_ctx->pointer_size); + } + + func_param_values[0] = func_ctx->exec_env; + func_param_values[1] = table_idx; + func_param_values[2] = table_elem_idx; + func_param_values[3] = I32_CONST(param_cell_num); + func_param_values[4] = func_ctx->argv_buf; + + if (!func_param_values[3]) { + aot_set_last_error("llvm create const failed."); + return false; + } + + /* call aot_call_indirect() function */ + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, + func_param_values, 5, "res"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + /* get function result values */ + cell_num = 0; + for (i = 0; i < result_count; i++) { + ret_type = TO_LLVM_TYPE(wasm_ret_types[i]); + if (!(ret_idx = I32_CONST(cell_num)) + || !(ret_ptr_type = LLVMPointerType(ret_type, 0))) { + aot_set_last_error("llvm add const or pointer type failed."); + return false; + } + + snprintf(buf, sizeof(buf), "argv_ret%d", i); + if (!(ret_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, I32_TYPE, + func_ctx->argv_buf, &ret_idx, 1, buf)) + || !(ret_ptr = LLVMBuildBitCast(comp_ctx->builder, ret_ptr, + ret_ptr_type, buf))) { + aot_set_last_error("llvm build GEP or bit cast failed."); + return false; + } + + snprintf(buf, sizeof(buf), "ret%d", i); + if (!(value_rets[i] = + LLVMBuildLoad2(comp_ctx->builder, ret_type, ret_ptr, buf))) { + aot_set_last_error("llvm build load failed."); + return false; + } + cell_num += wasm_value_type_cell_num_internal(wasm_ret_types[i], + comp_ctx->pointer_size); + } + + *p_res = res; + return true; +} + +bool +aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_idx, uint32 tbl_idx) +{ + AOTFuncType *func_type; + LLVMValueRef tbl_idx_value, elem_idx, func_idx; + LLVMValueRef table_elem_base, table_elem_addr, table_elem; + LLVMValueRef ftype_idx_ptr, ftype_idx, ftype_idx_const; + LLVMValueRef cmp_func_obj, cmp_elem_idx, cmp_func_idx, cmp_ftype_idx; + LLVMValueRef func, func_ptr, table_size_const; + LLVMValueRef ext_ret_offset, ext_ret_ptr, ext_ret, res; + LLVMValueRef *param_values = NULL, *value_rets = NULL; + LLVMValueRef *result_phis = NULL, value_ret, import_func_count; + LLVMTypeRef *param_types = NULL, ret_type; + LLVMTypeRef llvm_func_type, llvm_func_ptr_type; + LLVMTypeRef ext_ret_ptr_type; + LLVMBasicBlockRef check_func_obj_succ, check_elem_idx_succ, + check_ftype_idx_succ; + LLVMBasicBlockRef check_func_idx_succ, block_return, block_curr; + LLVMBasicBlockRef block_call_import, block_call_non_import; + LLVMValueRef offset; + uint32 total_param_count, func_param_count, func_result_count; + uint32 ext_cell_num, param_cell_num, i, j; + uint8 wasm_ret_type, *wasm_ret_types; + uint64 total_size; + char buf[32]; + bool ret = false; + + /* Check function type index */ + if (type_idx >= comp_ctx->comp_data->type_count) { + aot_set_last_error("function type index out of range"); + return false; + } + + if (!comp_ctx->enable_gc) { + /* Find the equivalent function type whose type index is the smallest: + the callee function's type index is also converted to the smallest + one in wasm loader, so we can just check whether the two type indexes + are equal (the type index of call_indirect opcode and callee func), + we don't need to check whether the whole function types are equal, + including param types and result types. */ + type_idx = wasm_get_smallest_type_idx( + (WASMTypePtr *)comp_ctx->comp_data->types, + comp_ctx->comp_data->type_count, type_idx); + } + else { + /* Call aot_func_type_is_super_of to check whether the func type + provided in the bytecode is a super type of the func type of + the function to call */ + } + + ftype_idx_const = I32_CONST(type_idx); + CHECK_LLVM_CONST(ftype_idx_const); + + func_type = (AOTFuncType *)comp_ctx->comp_data->types[type_idx]; + aot_estimate_and_record_stack_usage_for_function_call(comp_ctx, func_ctx, + func_type); + /* Commit stack operands, sp and ip */ + if (comp_ctx->aot_frame) { + if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + /* Commit sp if gc is enabled and always commit ip for call_indirect */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, comp_ctx->enable_gc, + true)) + return false; + } + + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + } + + func_param_count = func_type->param_count; + func_result_count = func_type->result_count; + + POP_I32(elem_idx); + + /* get the cur size of the table instance */ + if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) + + offsetof(AOTTableInstance, cur_size)))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(table_size_const = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, + 1, "cur_size_i8p"))) { + HANDLE_FAILURE("LLVMBuildGEP"); + goto fail; + } + + if (!(table_size_const = + LLVMBuildBitCast(comp_ctx->builder, table_size_const, + INT32_PTR_TYPE, "cur_siuze_i32p"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(table_size_const = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, + table_size_const, "cur_size"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + goto fail; + } + + /* Check if (uint32)elem index >= table size */ + if (!(cmp_elem_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, elem_idx, + table_size_const, "cmp_elem_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* Throw exception if elem index >= table size */ + if (!(check_elem_idx_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_elem_idx_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_elem_idx_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNDEFINED_ELEMENT, true, + cmp_elem_idx, check_elem_idx_succ))) + goto fail; + + /* load data as i32* */ + if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) + + offsetof(AOTTableInstance, elems)))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(table_elem_base = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, + 1, "table_elem_base_i8p"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + + /* Load function index */ + if (comp_ctx->enable_gc) { + /* table elem is func_obj when gc is enabled */ + if (!(table_elem_base = + LLVMBuildBitCast(comp_ctx->builder, table_elem_base, + GC_REF_PTR_TYPE, "table_elem_base"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(table_elem_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, GC_REF_TYPE, table_elem_base, &elem_idx, 1, + "table_elem_addr"))) { + HANDLE_FAILURE("LLVMBuildNUWAdd"); + goto fail; + } + + if (!(table_elem = LLVMBuildLoad2(comp_ctx->builder, GC_REF_TYPE, + table_elem_addr, "table_elem"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + /* Check if func object is NULL */ + if (!(cmp_func_obj = LLVMBuildIsNull(comp_ctx->builder, table_elem, + "cmp_func_obj"))) { + aot_set_last_error("llvm build isnull failed."); + goto fail; + } + + /* Throw exception if func object is NULL */ + if (!(check_func_obj_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_func_obj_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_func_obj_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNINITIALIZED_ELEMENT, + true, cmp_func_obj, check_func_obj_succ))) + goto fail; + + /* Get the func idx bound of the WASMFuncObject, the offset may be + * different in 32-bit runtime and 64-bit runtime since WASMObjectHeader + * is uintptr_t. Use comp_ctx->pointer_size as the + * offsetof(WASMFuncObject, func_idx_bound) + */ + if (!(offset = I32_CONST(comp_ctx->pointer_size))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(func_idx = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + table_elem, &offset, 1, + "func_idx_bound_i8p"))) { + HANDLE_FAILURE("LLVMBuildGEP"); + goto fail; + } + + if (!(func_idx = + LLVMBuildBitCast(comp_ctx->builder, func_idx, INT32_PTR_TYPE, + "func_idx_bound_i32p"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(func_idx = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, func_idx, + "func_idx_bound"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + goto fail; + } + } + else { + if (!(table_elem_base = + LLVMBuildBitCast(comp_ctx->builder, table_elem_base, + INTPTR_T_PTR_TYPE, "table_elem_base"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(table_elem_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INTPTR_T_TYPE, table_elem_base, &elem_idx, + 1, "table_elem_addr"))) { + HANDLE_FAILURE("LLVMBuildNUWAdd"); + goto fail; + } + + if (!(func_idx = LLVMBuildLoad2(comp_ctx->builder, INTPTR_T_TYPE, + table_elem_addr, "func_idx"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(func_idx = LLVMBuildIntCast2(comp_ctx->builder, func_idx, + I32_TYPE, true, "func_idx_i32"))) { + aot_set_last_error("llvm build int cast failed."); + goto fail; + } + + /* Check if func_idx == -1 */ + if (!(cmp_func_idx = + LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, func_idx, + I32_NEG_ONE, "cmp_func_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* Throw exception if func_idx == -1 */ + if (!(check_func_idx_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_func_idx_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_func_idx_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNINITIALIZED_ELEMENT, + true, cmp_func_idx, check_func_idx_succ))) + goto fail; + } + /* Load function type index */ + if (!(ftype_idx_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, I32_TYPE, func_ctx->func_type_indexes, + &func_idx, 1, "ftype_idx_ptr"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(ftype_idx = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, ftype_idx_ptr, + "ftype_idx"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + if (!(cmp_ftype_idx = call_aot_func_type_is_super_of_func( + comp_ctx, func_ctx, ftype_idx_const, ftype_idx))) { + goto fail; + } + } + else +#endif + { + /* Check if function type index not equal */ + if (!(cmp_ftype_idx = + LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, ftype_idx, + ftype_idx_const, "cmp_ftype_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + } + + /* Throw exception if ftype_idx != ftype_idx_const */ + if (!(check_ftype_idx_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_ftype_idx_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_ftype_idx_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_INVALID_FUNCTION_TYPE_INDEX, true, + cmp_ftype_idx, check_ftype_idx_succ))) + goto fail; + + /* Initialize parameter types of the LLVM function */ + total_param_count = 1 + func_param_count; + + /* Extra function results' addresses (except the first one) are + appended to aot function parameters. */ + if (func_result_count > 1) + total_param_count += func_result_count - 1; + + total_size = sizeof(LLVMTypeRef) * (uint64)total_param_count; + if (total_size >= UINT32_MAX + || !(param_types = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + + /* Prepare param types */ + j = 0; + param_types[j++] = comp_ctx->exec_env_type; + for (i = 0; i < func_param_count; i++) + param_types[j++] = TO_LLVM_TYPE(func_type->types[i]); + + for (i = 1; i < func_result_count; i++, j++) { + param_types[j] = TO_LLVM_TYPE(func_type->types[func_param_count + i]); + if (!(param_types[j] = LLVMPointerType(param_types[j], 0))) { + aot_set_last_error("llvm get pointer type failed."); + goto fail; + } + } + + /* Resolve return type of the LLVM function */ + if (func_result_count) { + wasm_ret_type = func_type->types[func_param_count]; + ret_type = TO_LLVM_TYPE(wasm_ret_type); + } + else { + wasm_ret_type = VALUE_TYPE_VOID; + ret_type = VOID_TYPE; + } + + /* Allocate memory for parameters */ + total_size = sizeof(LLVMValueRef) * (uint64)total_param_count; + if (total_size >= UINT32_MAX + || !(param_values = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + + /* First parameter is exec env */ + j = 0; + param_values[j++] = func_ctx->exec_env; + + /* Pop parameters from stack */ + for (i = func_param_count - 1; (int32)i >= 0; i--) + POP(param_values[i + j], func_type->types[i]); + + /* Prepare extra parameters */ + ext_cell_num = 0; + for (i = 1; i < func_result_count; i++) { + ext_ret_offset = I32_CONST(ext_cell_num); + CHECK_LLVM_CONST(ext_ret_offset); + + snprintf(buf, sizeof(buf), "ext_ret%d_ptr", i - 1); + if (!(ext_ret_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, I32_TYPE, + func_ctx->argv_buf, + &ext_ret_offset, 1, buf))) { + aot_set_last_error("llvm build GEP failed."); + goto fail; + } + + ext_ret_ptr_type = param_types[func_param_count + i]; + snprintf(buf, sizeof(buf), "ext_ret%d_ptr_cast", i - 1); + if (!(ext_ret_ptr = LLVMBuildBitCast(comp_ctx->builder, ext_ret_ptr, + ext_ret_ptr_type, buf))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + + param_values[func_param_count + i] = ext_ret_ptr; + ext_cell_num += wasm_value_type_cell_num_internal( + func_type->types[func_param_count + i], comp_ctx->pointer_size); + } + + if (ext_cell_num > 64) { + aot_set_last_error("prepare call-indirect arguments failed: " + "maximum 64 extra cell number supported."); + goto fail; + } + + if (comp_ctx->enable_aux_stack_frame) { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + /* TODO: use current frame instead of allocating new frame + for WASM_OP_RETURN_CALL_INDIRECT */ + if (!call_aot_alloc_frame_func(comp_ctx, func_ctx, func_idx)) + goto fail; +#endif + } + + /* Add basic blocks */ + block_call_import = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "call_import"); + block_call_non_import = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "call_non_import"); + block_return = LLVMAppendBasicBlockInContext(comp_ctx->context, + func_ctx->func, "func_return"); + if (!block_call_import || !block_call_non_import || !block_return) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(block_call_import, + LLVMGetInsertBlock(comp_ctx->builder)); + LLVMMoveBasicBlockAfter(block_call_non_import, block_call_import); + LLVMMoveBasicBlockAfter(block_return, block_call_non_import); + + import_func_count = I32_CONST(comp_ctx->comp_data->import_func_count); + CHECK_LLVM_CONST(import_func_count); + + /* Check if func_idx < import_func_count */ + if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntULT, func_idx, + import_func_count, "cmp_func_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* If func_idx < import_func_count, jump to call import block, + else jump to call non-import block */ + if (!LLVMBuildCondBr(comp_ctx->builder, cmp_func_idx, block_call_import, + block_call_non_import)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* Add result phis for return block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return); + + if (func_result_count > 0) { + total_size = sizeof(LLVMValueRef) * (uint64)func_result_count; + if (total_size >= UINT32_MAX + || !(result_phis = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + memset(result_phis, 0, (uint32)total_size); + for (i = 0; i < func_result_count; i++) { + LLVMTypeRef tmp_type = + TO_LLVM_TYPE(func_type->types[func_param_count + i]); + if (!(result_phis[i] = + LLVMBuildPhi(comp_ctx->builder, tmp_type, "phi"))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + } + } + + /* Translate call import block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_import); + + if (comp_ctx->enable_aux_stack_frame + && !commit_params_to_frame_of_import_func(comp_ctx, func_ctx, func_type, + param_values + 1)) { + goto fail; + } + + /* Allocate memory for result values */ + if (func_result_count > 0) { + total_size = sizeof(LLVMValueRef) * (uint64)func_result_count; + if (total_size >= UINT32_MAX + || !(value_rets = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + memset(value_rets, 0, (uint32)total_size); + } + + param_cell_num = func_type->param_cell_num; + wasm_ret_types = func_type->types + func_type->param_count; + + tbl_idx_value = I32_CONST(tbl_idx); + if (!tbl_idx_value) { + aot_set_last_error("llvm create const failed."); + goto fail; + } + + if (!call_aot_call_indirect_func( + comp_ctx, func_ctx, func_type, ftype_idx, tbl_idx_value, elem_idx, + param_types + 1, param_values + 1, func_param_count, param_cell_num, + func_result_count, wasm_ret_types, value_rets, &res)) + goto fail; + + /* Check whether exception was thrown when executing the function */ + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) + && !check_call_return(comp_ctx, func_ctx, res)) + goto fail; + + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + for (i = 0; i < func_result_count; i++) { + LLVMAddIncoming(result_phis[i], &value_rets[i], &block_curr, 1); + } + + if (!LLVMBuildBr(comp_ctx->builder, block_return)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Translate call non-import block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_non_import); + + /* Load function pointer */ + if (!(func_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->func_ptrs, &func_idx, 1, + "func_ptr_tmp"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(func_ptr = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, func_ptr, + "func_ptr"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(llvm_func_type = + LLVMFunctionType(ret_type, param_types, total_param_count, false)) + || !(llvm_func_ptr_type = LLVMPointerType(llvm_func_type, 0))) { + aot_set_last_error("llvm add function type failed."); + goto fail; + } + + if (!(func = LLVMBuildBitCast(comp_ctx->builder, func_ptr, + llvm_func_ptr_type, "indirect_func"))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + + if (!(value_ret = LLVMBuildCall2(comp_ctx->builder, llvm_func_type, func, + param_values, total_param_count, + func_result_count > 0 ? "ret" : ""))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + /* Check whether exception was thrown when executing the function */ + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) + && !check_exception_thrown(comp_ctx, func_ctx)) + goto fail; + + if (func_result_count > 0) { + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + + /* Push the first result to stack */ + LLVMAddIncoming(result_phis[0], &value_ret, &block_curr, 1); + + /* Load extra result from its address and push to stack */ + for (i = 1; i < func_result_count; i++) { + ret_type = TO_LLVM_TYPE(func_type->types[func_param_count + i]); + snprintf(buf, sizeof(buf), "ext_ret%d", i - 1); + if (!(ext_ret = LLVMBuildLoad2(comp_ctx->builder, ret_type, + param_values[func_param_count + i], + buf))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + LLVMAddIncoming(result_phis[i], &ext_ret, &block_curr, 1); + } + } + + if (!LLVMBuildBr(comp_ctx->builder, block_return)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Translate function return block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return); + + for (i = 0; i < func_result_count; i++) { + PUSH(result_phis[i], func_type->types[func_param_count + i]); + } + + if (comp_ctx->enable_aux_stack_frame) { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + if (!free_frame_for_aot_func(comp_ctx, func_ctx)) + goto fail; +#endif + } + + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx, false)) + goto fail; + } + + ret = true; + +fail: + if (param_values) + wasm_runtime_free(param_values); + if (param_types) + wasm_runtime_free(param_types); + if (value_rets) + wasm_runtime_free(value_rets); + if (result_phis) + wasm_runtime_free(result_phis); + return ret; +} + +bool +aot_compile_op_ref_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + if (comp_ctx->enable_gc) + PUSH_GC_REF(GC_REF_NULL); + else + PUSH_I32(REF_NULL); + + return true; +fail: + return false; +} + +bool +aot_compile_op_ref_is_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef lhs = NULL, res; + + if (comp_ctx->enable_gc) { + POP_GC_REF(lhs); + + if (!(res = LLVMBuildIsNull(comp_ctx->builder, lhs, "lhs is null"))) { + HANDLE_FAILURE("LLVMBuildIsNull"); + goto fail; + } + } + else { + POP_I32(lhs); + + if (!(res = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, lhs, REF_NULL, + "cmp_w_null"))) { + HANDLE_FAILURE("LLVMBuildICmp"); + goto fail; + } + } + + if (!(res = LLVMBuildZExt(comp_ctx->builder, res, I32_TYPE, "r_i"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + goto fail; + } + + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +aot_compile_op_ref_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 func_idx) +{ + LLVMValueRef ref_idx; +#if WASM_ENABLE_GC != 0 + LLVMValueRef gc_obj; +#endif + + if (!(ref_idx = I32_CONST(func_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + /* Commit sp and ip if gc is enabled */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + if (!aot_call_aot_create_func_obj(comp_ctx, func_ctx, ref_idx, + &gc_obj)) { + goto fail; + } + + PUSH_GC_REF(gc_obj); + } + else +#endif + { + PUSH_I32(ref_idx); + } + + return true; +fail: + return false; +} + +#if WASM_ENABLE_GC != 0 +bool +aot_compile_op_call_ref(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_idx, bool tail_call) +{ + AOTFuncType *func_type; + LLVMValueRef func_obj, func_idx; + LLVMValueRef cmp_func_obj, cmp_func_idx; + LLVMValueRef func, func_ptr; + LLVMValueRef ext_ret_offset, ext_ret_ptr, ext_ret, res; + LLVMValueRef *param_values = NULL; + LLVMValueRef *result_phis = NULL, value_ret, import_func_count; + LLVMTypeRef *param_types = NULL, ret_type; + LLVMTypeRef llvm_func_type, llvm_func_ptr_type; + LLVMTypeRef ext_ret_ptr_type; + LLVMBasicBlockRef check_func_obj_succ, block_return, block_curr; + LLVMBasicBlockRef block_call_import, block_call_non_import; + LLVMValueRef offset; + uint32 total_param_count, func_param_count, func_result_count; + uint32 ext_cell_num, param_cell_num, i, j; + uint8 wasm_ret_type; + uint64 total_size; + char buf[32]; + bool ret = false; + + /* Check function type index */ + bh_assert(type_idx < comp_ctx->comp_data->type_count); + + func_type = (AOTFuncType *)comp_ctx->comp_data->types[type_idx]; + aot_estimate_and_record_stack_usage_for_function_call(comp_ctx, func_ctx, + func_type); + func_param_count = func_type->param_count; + func_result_count = func_type->result_count; + param_cell_num = func_type->param_cell_num; + + /* Commit stack operands, sp and ip to aot frame */ + if (comp_ctx->aot_frame) { + /* Note that GC is enabled, no need to check it again */ + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + /* Commit sp if gc is enabled and always commit ip for call_ref */ + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + } + + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + } + + POP_GC_REF(func_obj); + + /* Check if func object is NULL */ + if (!(cmp_func_obj = + LLVMBuildIsNull(comp_ctx->builder, func_obj, "cmp_func_obj"))) { + aot_set_last_error("llvm build isnull failed."); + goto fail; + } + + /* Throw exception if func object is NULL */ + if (!(check_func_obj_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_func_obj_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_func_obj_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_FUNC_OBJ, true, + cmp_func_obj, check_func_obj_succ))) + goto fail; + + /* Get the func idx bound of the WASMFuncObject, the offset may be + * different in 32-bit runtime and 64-bit runtime since WASMObjectHeader + * is uintptr_t. Use comp_ctx->pointer_size as the + * offsetof(WASMFuncObject, func_idx_bound) */ + if (!(offset = I32_CONST(comp_ctx->pointer_size))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(func_idx = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_obj, + &offset, 1, "func_idx_bound_i8p"))) { + HANDLE_FAILURE("LLVMBuildGEP"); + goto fail; + } + + if (!(func_idx = LLVMBuildBitCast(comp_ctx->builder, func_idx, + INT32_PTR_TYPE, "func_idx_bound_i32p"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(func_idx = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, func_idx, + "func_idx_bound"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + goto fail; + } + + /* Initialize parameter types of the LLVM function */ + total_param_count = 1 + func_param_count; + + /* Extra function results' addresses (except the first one) are + appended to aot function parameters. */ + if (func_result_count > 1) + total_param_count += func_result_count - 1; + + total_size = sizeof(LLVMTypeRef) * (uint64)total_param_count; + if (total_size >= UINT32_MAX + || !(param_types = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + + /* Prepare param types */ + j = 0; + param_types[j++] = comp_ctx->exec_env_type; + for (i = 0; i < func_param_count; i++) + param_types[j++] = TO_LLVM_TYPE(func_type->types[i]); + + for (i = 1; i < func_result_count; i++, j++) { + param_types[j] = TO_LLVM_TYPE(func_type->types[func_param_count + i]); + if (!(param_types[j] = LLVMPointerType(param_types[j], 0))) { + aot_set_last_error("llvm get pointer type failed."); + goto fail; + } + } + + /* Resolve return type of the LLVM function */ + if (func_result_count) { + wasm_ret_type = func_type->types[func_param_count]; + ret_type = TO_LLVM_TYPE(wasm_ret_type); + } + else { + wasm_ret_type = VALUE_TYPE_VOID; + ret_type = VOID_TYPE; + } + + /* Allocate memory for parameters */ + total_size = sizeof(LLVMValueRef) * (uint64)total_param_count; + if (total_size >= UINT32_MAX + || !(param_values = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + + /* First parameter is exec env */ + j = 0; + param_values[j++] = func_ctx->exec_env; + + /* Pop parameters from stack */ + for (i = func_param_count - 1; (int32)i >= 0; i--) + POP(param_values[i + j], func_type->types[i]); + + /* Prepare extra parameters */ + ext_cell_num = 0; + for (i = 1; i < func_result_count; i++) { + ext_ret_offset = I32_CONST(ext_cell_num); + CHECK_LLVM_CONST(ext_ret_offset); + + snprintf(buf, sizeof(buf), "ext_ret%d_ptr", i - 1); + if (!(ext_ret_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, I32_TYPE, + func_ctx->argv_buf, + &ext_ret_offset, 1, buf))) { + aot_set_last_error("llvm build GEP failed."); + goto fail; + } + + ext_ret_ptr_type = param_types[func_param_count + i]; + snprintf(buf, sizeof(buf), "ext_ret%d_ptr_cast", i - 1); + if (!(ext_ret_ptr = LLVMBuildBitCast(comp_ctx->builder, ext_ret_ptr, + ext_ret_ptr_type, buf))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + + param_values[func_param_count + i] = ext_ret_ptr; + ext_cell_num += wasm_value_type_cell_num_internal( + func_type->types[func_param_count + i], comp_ctx->pointer_size); + } + + if (ext_cell_num > 64) { + aot_set_last_error("prepare call-indirect arguments failed: " + "maximum 64 extra cell number supported."); + goto fail; + } + + if (comp_ctx->enable_aux_stack_frame) { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + /* TODO: use current frame instead of allocating new frame + for WASM_OP_RETURN_CALL_REF */ + if (!call_aot_alloc_frame_func(comp_ctx, func_ctx, func_idx)) + goto fail; +#endif + } + + /* Add basic blocks */ + block_call_import = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "call_import"); + block_call_non_import = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "call_non_import"); + block_return = LLVMAppendBasicBlockInContext(comp_ctx->context, + func_ctx->func, "func_return"); + if (!block_call_import || !block_call_non_import || !block_return) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(block_call_import, + LLVMGetInsertBlock(comp_ctx->builder)); + LLVMMoveBasicBlockAfter(block_call_non_import, block_call_import); + LLVMMoveBasicBlockAfter(block_return, block_call_non_import); + + import_func_count = I32_CONST(comp_ctx->comp_data->import_func_count); + CHECK_LLVM_CONST(import_func_count); + + /* Check if func_idx < import_func_count */ + if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntULT, func_idx, + import_func_count, "cmp_func_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* If func_idx < import_func_count, jump to call import block, + else jump to call non-import block */ + if (!LLVMBuildCondBr(comp_ctx->builder, cmp_func_idx, block_call_import, + block_call_non_import)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* Add result phis for return block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return); + + if (func_result_count > 0) { + total_size = sizeof(LLVMValueRef) * (uint64)func_result_count; + if (total_size >= UINT32_MAX + || !(result_phis = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + memset(result_phis, 0, (uint32)total_size); + for (i = 0; i < func_result_count; i++) { + LLVMTypeRef tmp_type = + TO_LLVM_TYPE(func_type->types[func_param_count + i]); + if (!(result_phis[i] = + LLVMBuildPhi(comp_ctx->builder, tmp_type, "phi"))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + } + } + + /* Translate call import block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_import); + + if (comp_ctx->enable_aux_stack_frame + && !commit_params_to_frame_of_import_func(comp_ctx, func_ctx, func_type, + param_values + 1)) { + goto fail; + } + + /* Similar to opcode call_indirect, but for opcode ref.func needs to call + * aot_invoke_native_func instead */ + if (!call_aot_invoke_native_func(comp_ctx, func_ctx, func_idx, func_type, + param_types + 1, param_values + 1, + func_param_count, param_cell_num, ret_type, + wasm_ret_type, &value_ret, &res)) + goto fail; + + /* Check whether exception was thrown when executing the function */ + if (comp_ctx->enable_bound_check + && !check_call_return(comp_ctx, func_ctx, res)) + goto fail; + + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + + /* Get function return values, for aot_invoke_native_func, the extra ret + * values are put into param's array */ + if (func_result_count > 0) { + /* Push the first result to stack */ + LLVMAddIncoming(result_phis[0], &value_ret, &block_curr, 1); + + /* Load extra result from its address and push to stack */ + for (i = 1; i < func_result_count; i++) { + ret_type = TO_LLVM_TYPE(func_type->types[func_param_count + i]); + snprintf(buf, sizeof(buf), "ext_ret%d", i - 1); + if (!(ext_ret = LLVMBuildLoad2(comp_ctx->builder, ret_type, + param_values[func_param_count + i], + buf))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + LLVMAddIncoming(result_phis[i], &ext_ret, &block_curr, 1); + } + } + + if (!LLVMBuildBr(comp_ctx->builder, block_return)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Translate call non-import block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_non_import); + + /* Load function pointer */ + if (!(func_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->func_ptrs, &func_idx, 1, + "func_ptr_tmp"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(func_ptr = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, func_ptr, + "func_ptr"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(llvm_func_type = + LLVMFunctionType(ret_type, param_types, total_param_count, false)) + || !(llvm_func_ptr_type = LLVMPointerType(llvm_func_type, 0))) { + aot_set_last_error("llvm add function type failed."); + goto fail; + } + + if (!(func = LLVMBuildBitCast(comp_ctx->builder, func_ptr, + llvm_func_ptr_type, "indirect_func"))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + + if (!(value_ret = LLVMBuildCall2(comp_ctx->builder, llvm_func_type, func, + param_values, total_param_count, + func_result_count > 0 ? "ret" : ""))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + /* Set calling convention for the call with the func's calling + convention */ + LLVMSetInstructionCallConv(value_ret, LLVMGetFunctionCallConv(func)); + + if (tail_call) + LLVMSetTailCall(value_ret, true); + + /* Check whether exception was thrown when executing the function */ + if (!tail_call + && (comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) + && !check_exception_thrown(comp_ctx, func_ctx)) + goto fail; + + if (func_result_count > 0) { + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + + /* Push the first result to stack */ + LLVMAddIncoming(result_phis[0], &value_ret, &block_curr, 1); + + /* Load extra result from its address and push to stack */ + for (i = 1; i < func_result_count; i++) { + ret_type = TO_LLVM_TYPE(func_type->types[func_param_count + i]); + snprintf(buf, sizeof(buf), "ext_ret%d", i - 1); + if (!(ext_ret = LLVMBuildLoad2(comp_ctx->builder, ret_type, + param_values[func_param_count + i], + buf))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + LLVMAddIncoming(result_phis[i], &ext_ret, &block_curr, 1); + } + } + + if (!LLVMBuildBr(comp_ctx->builder, block_return)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Translate function return block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return); + + for (i = 0; i < func_result_count; i++) { + PUSH(result_phis[i], func_type->types[func_param_count + i]); + } + + if (comp_ctx->enable_aux_stack_frame) { +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + if (!free_frame_for_aot_func(comp_ctx, func_ctx)) + goto fail; +#endif + } + + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx, false)) + goto fail; + } + + ret = true; + +fail: + if (param_values) + wasm_runtime_free(param_values); + if (param_types) + wasm_runtime_free(param_types); + if (result_phis) + wasm_runtime_free(result_phis); + return ret; +} + +#endif /* end of WASM_ENABLE_GC != 0 */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_function.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_function.h new file mode 100644 index 0000000..45f7bbe --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_function.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_FUNCTION_H_ +#define _AOT_EMIT_FUNCTION_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 func_idx, bool tail_call); + +bool +aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_idx, uint32 tbl_idx); + +bool +aot_compile_op_ref_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_ref_is_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_ref_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 func_idx); + +#if WASM_ENABLE_GC != 0 +bool +aot_compile_op_call_ref(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_idx, bool tail_call); +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_FUNCTION_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_gc.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_gc.c new file mode 100644 index 0000000..55f1471 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_gc.c @@ -0,0 +1,2144 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_gc.h" +#include "aot_compiler.h" +#include "aot_emit_exception.h" + +#if WASM_ENABLE_GC != 0 + +#define BUILD_ISNULL(ptr, res, name) \ + do { \ + if (!(res = LLVMBuildIsNull(comp_ctx->builder, ptr, name))) { \ + aot_set_last_error("llvm build isnull failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_ISNOTNULL(ptr, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, ptr, name))) { \ + aot_set_last_error("llvm build isnotnull failed."); \ + goto fail; \ + } \ + } while (0) + +#define ADD_BASIC_BLOCK(block, name) \ + do { \ + if (!(block = LLVMAppendBasicBlockInContext(comp_ctx->context, \ + func_ctx->func, name))) { \ + aot_set_last_error("llvm add basic block failed."); \ + goto fail; \ + } \ + } while (0) + +#define CURR_BLOCK() LLVMGetInsertBlock(comp_ctx->builder) + +#define MOVE_BLOCK_AFTER(llvm_block, llvm_block_after) \ + LLVMMoveBasicBlockAfter(llvm_block, llvm_block_after) + +#define MOVE_BLOCK_AFTER_CURR(llvm_block) \ + LLVMMoveBasicBlockAfter(llvm_block, CURR_BLOCK()) + +#define MOVE_BLOCK_BEFORE(llvm_block, llvm_block_before) \ + LLVMMoveBasicBlockBefore(llvm_block, llvm_block_before) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \ + block_else)) { \ + aot_set_last_error("llvm build cond br failed."); \ + goto fail; \ + } \ + } while (0) + +#define SET_BUILDER_POS(llvm_block) \ + LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block) + +#define BUILD_BR(llvm_block) \ + do { \ + if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \ + aot_set_last_error("llvm build br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_ICMP(op, left, right, res, name) \ + do { \ + if (!(res = \ + LLVMBuildICmp(comp_ctx->builder, op, left, right, name))) { \ + aot_set_last_error("llvm build icmp failed."); \ + goto fail; \ + } \ + } while (0) + +static bool +is_target_x86(AOTCompContext *comp_ctx) +{ + return !strncmp(comp_ctx->target_arch, "x86_64", 6) + || !strncmp(comp_ctx->target_arch, "i386", 4); +} + +bool +aot_call_aot_create_func_obj(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef func_idx, LLVMValueRef *p_gc_obj) +{ + LLVMValueRef gc_obj, cmp_gc_obj, param_values[5], func, value; + LLVMTypeRef param_types[5], ret_type, func_type, func_ptr_type; + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMBasicBlockRef init_gc_obj_fail, init_gc_obj_succ; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = INT8_TYPE; + param_types[3] = INT8_PTR_TYPE; + param_types[4] = I32_TYPE; + ret_type = GC_REF_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_create_func_obj, 5); + else + GET_AOT_FUNCTION(aot_create_func_obj, 5); + + /* Call function llvm_jit/aot_create_func_obj() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = func_idx; + param_values[2] = I8_CONST(1); + param_values[3] = I8_PTR_NULL; + param_values[4] = I32_ZERO; + if (!(gc_obj = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 5, "call"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + BUILD_ISNOTNULL(gc_obj, cmp_gc_obj, "gc_obj_not_null"); + + ADD_BASIC_BLOCK(init_gc_obj_fail, "init_gc_obj_fail"); + ADD_BASIC_BLOCK(init_gc_obj_succ, "init_gc_obj_success"); + + LLVMMoveBasicBlockAfter(init_gc_obj_fail, block_curr); + LLVMMoveBasicBlockAfter(init_gc_obj_succ, block_curr); + + if (!LLVMBuildCondBr(comp_ctx->builder, cmp_gc_obj, init_gc_obj_succ, + init_gc_obj_fail)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* If init gc_obj failed, return this function + so the runtime can catch the exception */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, init_gc_obj_fail); + if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, init_gc_obj_succ); + *p_gc_obj = gc_obj; + + return true; +fail: + return false; +} + +bool +aot_call_aot_obj_is_instance_of(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, LLVMValueRef gc_obj, + LLVMValueRef heap_type, LLVMValueRef *castable) +{ + LLVMValueRef param_values[3], func, value, res; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = GC_REF_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_obj_is_instance_of, 3); + else + GET_AOT_FUNCTION(aot_obj_is_instance_of, 3); + + /* Call function aot_obj_is_instance_of() or llvm_jit_obj_is_instance_of() + */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = gc_obj; + param_values[2] = heap_type; + + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 3, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *castable = res; + + return true; +fail: + return false; +} + +bool +aot_call_wasm_obj_is_type_of(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef gc_obj, LLVMValueRef heap_type, + LLVMValueRef *castable) +{ + LLVMValueRef param_values[2], func, value, res; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + param_types[0] = GC_REF_TYPE; + param_types[1] = I32_TYPE; + ret_type = INT8_TYPE; + + GET_AOT_FUNCTION(wasm_obj_is_type_of, 2); + + /* Call function wasm_obj_is_type_of() */ + param_values[0] = gc_obj; + param_values[1] = heap_type; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 2, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *castable = res; + + return true; +fail: + return false; +} + +bool +aot_call_aot_rtt_type_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef type_index, LLVMValueRef *rtt_type) +{ + LLVMValueRef param_values[2], func, value, res; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = GC_REF_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_rtt_type_new, 2); + else + GET_AOT_FUNCTION(aot_rtt_type_new, 2); + + /* Call function llvm_jit/aot_rtt_type_new() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = type_index; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 2, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *rtt_type = res; + return true; +fail: + return false; +} + +bool +aot_compile_op_ref_as_non_null(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef gc_obj, cmp_gc_obj; + LLVMBasicBlockRef check_gc_obj_succ; + + GET_GC_REF_FROM_STACK(gc_obj); + + /* Check if gc object is NULL */ + BUILD_ISNULL(gc_obj, cmp_gc_obj, "cmp_gc_obj"); + + ADD_BASIC_BLOCK(check_gc_obj_succ, "check_gc_obj_succ"); + MOVE_BLOCK_AFTER_CURR(check_gc_obj_succ); + + /* Throw exception if it is NULL */ + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_REFERENCE, true, + cmp_gc_obj, check_gc_obj_succ)) + goto fail; + + return true; +fail: + return false; +} + +static bool +aot_call_wasm_struct_obj_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef rtt_type, LLVMValueRef *struct_obj) +{ + LLVMValueRef param_values[2], func, value, res; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + ret_type = GC_REF_TYPE; + + GET_AOT_FUNCTION(wasm_struct_obj_new, 2); + + /* Call function wasm_struct_obj_new() */ + param_values[0] = func_ctx->exec_env; + param_values[1] = rtt_type; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 2, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *struct_obj = res; + return true; +fail: + return false; +} + +static void +get_struct_field_data_types(const AOTCompContext *comp_ctx, uint8 field_type, + LLVMTypeRef *p_field_data_type, + LLVMTypeRef *p_field_data_ptr_type, + bool *p_trunc_or_extend) +{ + LLVMTypeRef field_data_type = NULL, field_data_ptr_type = NULL; + bool trunc_or_extend = false; + + if (wasm_is_type_reftype(field_type)) { + field_data_type = GC_REF_TYPE; + field_data_ptr_type = GC_REF_PTR_TYPE; + } + else { + switch (field_type) { + case VALUE_TYPE_I32: + field_data_type = I32_TYPE; + field_data_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + field_data_type = I64_TYPE; + field_data_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F32: + field_data_type = F32_TYPE; + field_data_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_F64: + field_data_type = F64_TYPE; + field_data_ptr_type = F64_PTR_TYPE; + break; + case PACKED_TYPE_I8: + field_data_type = INT8_TYPE; + field_data_ptr_type = INT8_PTR_TYPE; + trunc_or_extend = true; + break; + case PACKED_TYPE_I16: + field_data_type = INT16_TYPE; + field_data_ptr_type = INT16_PTR_TYPE; + trunc_or_extend = true; + break; + default: + bh_assert(0); + break; + } + } + + *p_field_data_type = field_data_type; + *p_field_data_ptr_type = field_data_ptr_type; + *p_trunc_or_extend = trunc_or_extend; +} + +static bool +aot_struct_obj_set_field(AOTCompContext *comp_ctx, LLVMValueRef struct_obj, + LLVMValueRef field_offset, LLVMValueRef field_value, + uint8 field_type) +{ + bool trunc = false; + LLVMValueRef field_data_ptr, res; + LLVMTypeRef field_data_type = NULL, field_data_ptr_type = NULL; + + get_struct_field_data_types(comp_ctx, field_type, &field_data_type, + &field_data_ptr_type, &trunc); + + /* Truncate field_value if necessary */ + if (trunc) { + if (!(field_value = + LLVMBuildTrunc(comp_ctx->builder, field_value, + field_data_type, "field_value_trunc"))) { + aot_set_last_error("llvm build trunc failed."); + goto fail; + } + } + + if (!(struct_obj = LLVMBuildBitCast(comp_ctx->builder, struct_obj, + INT8_PTR_TYPE, "struct_obj_i8p"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + /* Build field data ptr and store the value */ + if (!(field_data_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, struct_obj, + &field_offset, 1, "field_data_i8p"))) { + aot_set_last_error("llvm build gep failed."); + goto fail; + } + + /* Cast to the field data type ptr */ + if (!(field_data_ptr = + LLVMBuildBitCast(comp_ctx->builder, field_data_ptr, + field_data_ptr_type, "field_value_ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(res = + LLVMBuildStore(comp_ctx->builder, field_value, field_data_ptr))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + + if (!is_target_x86(comp_ctx) + && (field_data_type == I64_TYPE || field_data_type == F64_TYPE + || field_data_type == GC_REF_TYPE)) { + LLVMSetAlignment(res, 4); + } + + return true; +fail: + return false; +} + +static bool +aot_struct_obj_get_field(AOTCompContext *comp_ctx, LLVMValueRef struct_obj, + LLVMValueRef field_offset, LLVMValueRef *p_field_value, + uint8 field_type, bool sign_extend) +{ + bool extend = false; + LLVMValueRef field_value, field_data_ptr; + LLVMTypeRef field_data_type = NULL, field_data_ptr_type = NULL; + + get_struct_field_data_types(comp_ctx, field_type, &field_data_type, + &field_data_ptr_type, &extend); + + if (!(struct_obj = LLVMBuildBitCast(comp_ctx->builder, struct_obj, + INT8_PTR_TYPE, "struct_obj_i8p"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(field_data_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, struct_obj, + &field_offset, 1, "field_data_i8p"))) { + aot_set_last_error("llvm build gep failed."); + goto fail; + } + + if (!(field_data_ptr = + LLVMBuildBitCast(comp_ctx->builder, field_data_ptr, + field_data_ptr_type, "field_value_ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(field_value = LLVMBuildLoad2(comp_ctx->builder, field_data_type, + field_data_ptr, "field_value"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!is_target_x86(comp_ctx) + && (field_data_type == I64_TYPE || field_data_type == F64_TYPE + || field_data_type == GC_REF_TYPE)) { + LLVMSetAlignment(field_value, 4); + } + + if (extend) { + if (sign_extend) { + if (!(field_value = LLVMBuildSExt(comp_ctx->builder, field_value, + I32_TYPE, "field_value_sext"))) { + aot_set_last_error("llvm build signed ext failed."); + goto fail; + } + } + else { + if (!(field_value = LLVMBuildZExt(comp_ctx->builder, field_value, + I32_TYPE, "field_value_zext"))) { + aot_set_last_error("llvm build unsigned ext failed."); + goto fail; + } + } + } + + *p_field_value = field_value; + return true; +fail: + return false; +} + +static bool +struct_new_canon_init_fields(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, LLVMValueRef struct_obj) +{ + LLVMValueRef field_value = NULL; + /* Used in compile time, to distinguish what type of AOTValue POP, + * field_data offset, size */ + WASMStructType *compile_time_struct_type = + (WASMStructType *)comp_ctx->comp_data->types[type_index]; + WASMStructFieldType *fields = compile_time_struct_type->fields; + int32 field_count = (int32)compile_time_struct_type->field_count; + int32 field_idx; + uint32 field_offset; + uint8 field_type; + + for (field_idx = field_count - 1; field_idx >= 0; field_idx--) { + field_type = fields[field_idx].field_type; + field_offset = comp_ctx->pointer_size == sizeof(uint64) + ? fields[field_idx].field_offset_64bit + : fields[field_idx].field_offset_32bit; + + if (wasm_is_type_reftype(field_type)) { + POP_GC_REF(field_value); + } + else if (field_type == VALUE_TYPE_I32 || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + POP_I32(field_value); + } + else if (field_type == VALUE_TYPE_I64) { + POP_I64(field_value); + } + else if (field_type == VALUE_TYPE_F32) { + POP_F32(field_value); + } + else if (field_type == VALUE_TYPE_F64) { + POP_F64(field_value); + } + else { + bh_assert(0); + } + + if (!aot_struct_obj_set_field(comp_ctx, struct_obj, + I32_CONST(field_offset), field_value, + field_type)) + goto fail; + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_struct_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, bool init_with_default) +{ + LLVMValueRef rtt_type, struct_obj, cmp; + LLVMBasicBlockRef check_rtt_type_succ, check_struct_obj_succ; + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + /* Generate call wasm_rtt_type_new and check for exception */ + if (!aot_call_aot_rtt_type_new(comp_ctx, func_ctx, I32_CONST(type_index), + &rtt_type)) + goto fail; + + ADD_BASIC_BLOCK(check_rtt_type_succ, "check rtt type succ"); + MOVE_BLOCK_AFTER_CURR(check_rtt_type_succ); + + BUILD_ISNULL(rtt_type, cmp, "cmp_rtt_type"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_FAILED_TO_CREATE_RTT_TYPE, + true, cmp, check_rtt_type_succ)) + goto fail; + + /* Generate call wasm_struct_obj_new and check for exception */ + if (!aot_call_wasm_struct_obj_new(comp_ctx, func_ctx, rtt_type, + &struct_obj)) + goto fail; + + ADD_BASIC_BLOCK(check_struct_obj_succ, "check struct obj succ"); + MOVE_BLOCK_AFTER(check_struct_obj_succ, check_rtt_type_succ); + + BUILD_ISNULL(struct_obj, cmp, "cmp_struct_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_FAILED_TO_CREATE_STRUCT_OBJ, true, cmp, + check_struct_obj_succ)) + goto fail; + + SET_BUILDER_POS(check_struct_obj_succ); + + /* For WASM_OP_STRUCT_NEW, init filed with poped value */ + if (!init_with_default + && !struct_new_canon_init_fields(comp_ctx, func_ctx, type_index, + struct_obj)) { + goto fail; + } + + PUSH_GC_REF(struct_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_struct_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, uint32 field_idx, bool sign) +{ + LLVMValueRef struct_obj, cmp, field_value; + LLVMBasicBlockRef check_struct_obj_succ; + + /* Used in compile time, to distinguish what type of AOTValue PUSH, + * field_data offset, size */ + WASMStructType *compile_time_struct_type = + (WASMStructType *)comp_ctx->comp_data->types[type_index]; + WASMStructFieldType *field; + uint32 field_offset; + uint8 field_type; + + field = compile_time_struct_type->fields + field_idx; + field_type = field->field_type; + field_offset = comp_ctx->pointer_size == sizeof(uint64) + ? field->field_offset_64bit + : field->field_offset_32bit; + + if (field_idx >= compile_time_struct_type->field_count) { + aot_set_last_error("struct field index out of bounds"); + goto fail; + } + + POP_GC_REF(struct_obj); + + ADD_BASIC_BLOCK(check_struct_obj_succ, "check struct obj succ"); + MOVE_BLOCK_AFTER_CURR(check_struct_obj_succ); + + BUILD_ISNULL(struct_obj, cmp, "cmp_struct_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_STRUCT_OBJ, true, cmp, + check_struct_obj_succ)) + goto fail; + + if (!aot_struct_obj_get_field(comp_ctx, struct_obj, I32_CONST(field_offset), + &field_value, field_type, sign)) + goto fail; + + if (wasm_is_type_reftype(field_type)) { + PUSH_GC_REF(field_value); + } + else if (field_type == VALUE_TYPE_I32 || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + PUSH_I32(field_value); + } + else if (field_type == VALUE_TYPE_I64) { + PUSH_I64(field_value); + } + else if (field_type == VALUE_TYPE_F32) { + PUSH_F32(field_value); + } + else if (field_type == VALUE_TYPE_F64) { + PUSH_F64(field_value); + } + else { + bh_assert(0); + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_struct_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, uint32 field_idx) +{ + LLVMValueRef struct_obj, cmp, field_value = NULL; + LLVMBasicBlockRef check_struct_obj_succ; + /* Used in compile time, to distinguish what type of AOTValue POP, + * field_data offset, size */ + WASMStructType *compile_time_struct_type = + (WASMStructType *)comp_ctx->comp_data->types[type_index]; + WASMStructFieldType *field; + uint32 field_offset; + uint8 field_type; + + field = compile_time_struct_type->fields + field_idx; + field_type = field->field_type; + field_offset = comp_ctx->pointer_size == sizeof(uint64) + ? field->field_offset_64bit + : field->field_offset_32bit; + + if (field_idx >= compile_time_struct_type->field_count) { + aot_set_last_error("struct field index out of bounds"); + goto fail; + } + + if (wasm_is_type_reftype(field_type)) { + POP_GC_REF(field_value); + } + else if (field_type == VALUE_TYPE_I32 || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + POP_I32(field_value); + } + else if (field_type == VALUE_TYPE_I64) { + POP_I64(field_value); + } + else if (field_type == VALUE_TYPE_F32) { + POP_F32(field_value); + } + else if (field_type == VALUE_TYPE_F64) { + POP_F64(field_value); + } + else { + bh_assert(0); + } + + POP_GC_REF(struct_obj); + + ADD_BASIC_BLOCK(check_struct_obj_succ, "check struct obj succ"); + MOVE_BLOCK_AFTER_CURR(check_struct_obj_succ); + + BUILD_ISNULL(struct_obj, cmp, "cmp_struct_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_STRUCT_OBJ, true, cmp, + check_struct_obj_succ)) + goto fail; + + if (!aot_struct_obj_set_field(comp_ctx, struct_obj, I32_CONST(field_offset), + field_value, field_type)) + goto fail; + + return true; +fail: + return false; +} + +static bool +aot_call_wasm_array_obj_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef rtt_type, LLVMValueRef array_len, + LLVMValueRef array_elem, LLVMValueRef *array_obj) +{ + LLVMValueRef param_values[4], func, value, res, array_elem_ptr; + LLVMTypeRef param_types[4], ret_type, func_type, func_ptr_type; + + if (!(array_elem_ptr = LLVMBuildAlloca( + comp_ctx->builder, LLVMTypeOf(array_elem), "array_elem_ptr"))) { + aot_set_last_error("llvm build alloca failed."); + goto fail; + } + if (!LLVMBuildStore(comp_ctx->builder, array_elem, array_elem_ptr)) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + if (!(array_elem_ptr = LLVMBuildBitCast(comp_ctx->builder, array_elem_ptr, + INT8_PTR_TYPE, "array_elem_ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + ret_type = GC_REF_TYPE; + + GET_AOT_FUNCTION(wasm_array_obj_new, 4); + + /* Call function wasm_array_obj_new() */ + param_values[0] = func_ctx->exec_env; + param_values[1] = rtt_type; + param_values[2] = array_len; + param_values[3] = array_elem_ptr; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 4, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *array_obj = res; + return true; +fail: + return false; +} + +static uint32 +aot_array_obj_elem_size_log(AOTCompContext *comp_ctx, uint8 array_elem_type) +{ + uint32 elem_size_log = 0; + + if (wasm_is_type_reftype(array_elem_type)) { + elem_size_log = comp_ctx->pointer_size == sizeof(uint32) ? 2 : 3; + } + else if (array_elem_type == PACKED_TYPE_I8) { + elem_size_log = 0; + } + else if (array_elem_type == PACKED_TYPE_I16) { + elem_size_log = 1; + } + else if (array_elem_type == VALUE_TYPE_I32 + || array_elem_type == VALUE_TYPE_F32) { + elem_size_log = 2; + } + else if (array_elem_type == VALUE_TYPE_I64 + || array_elem_type == VALUE_TYPE_F64) { + elem_size_log = 3; + } + else { + bh_assert(0); + } + + return elem_size_log; +} + +/* array_obj->elem_data + (elem_idx << elem_size_log) */ +bool +aot_array_obj_elem_addr(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef array_obj, LLVMValueRef elem_idx, + LLVMValueRef *p_elem_data, uint8 array_elem_type) +{ + uint32 elem_size_log = 0; + LLVMValueRef start_offset, elem_offset, elem_data; + + elem_size_log = aot_array_obj_elem_size_log(comp_ctx, array_elem_type); + + /* Get the elem data start offset of the WASMArrayObject, the offset may be + * different in 32-bit runtime and 64-bit runtime since WASMObjectHeader + * is uintptr_t. Use comp_ctx->pointer_size + 4(uint32 for length) as the + * offsetof(WASMArrayObject, length)*/ + if (!(start_offset = I32_CONST(comp_ctx->pointer_size + sizeof(uint32)))) { + aot_set_last_error("llvm build const failed."); + goto fail; + } + + if (!(elem_offset = + LLVMBuildShl(comp_ctx->builder, elem_idx, + I32_CONST(elem_size_log), "elem_offset"))) { + aot_set_last_error("llvm build shl failed."); + goto fail; + } + + if (!(elem_offset = LLVMBuildAdd(comp_ctx->builder, start_offset, + elem_offset, "total_offset"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + + if (!(elem_data = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + array_obj, &elem_offset, 1, + "array_obj_elem_data_i8p"))) { + aot_set_last_error("llvm build gep failed."); + goto fail; + } + + *p_elem_data = elem_data; + return true; +fail: + return false; +} + +static bool +aot_array_obj_set_elem(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef array_obj, LLVMValueRef elem_idx, + LLVMValueRef array_elem, uint8 array_elem_type) +{ + bool trunc = false; + LLVMValueRef elem_data_ptr, res; + LLVMTypeRef elem_data_type = NULL, elem_data_ptr_type = NULL; + + if (!aot_array_obj_elem_addr(comp_ctx, func_ctx, array_obj, elem_idx, + &elem_data_ptr, array_elem_type)) + goto fail; + + if (wasm_is_type_reftype(array_elem_type)) { + elem_data_type = GC_REF_TYPE; + elem_data_ptr_type = GC_REF_PTR_TYPE; + } + else + switch (array_elem_type) { + case PACKED_TYPE_I8: + elem_data_type = INT8_TYPE; + elem_data_ptr_type = INT8_PTR_TYPE; + trunc = true; + break; + case PACKED_TYPE_I16: + elem_data_type = INT16_TYPE; + elem_data_ptr_type = INT16_PTR_TYPE; + trunc = true; + break; + case VALUE_TYPE_I32: + elem_data_type = I32_TYPE; + elem_data_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + elem_data_type = I64_TYPE; + elem_data_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F32: + elem_data_type = F32_TYPE; + elem_data_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_F64: + elem_data_type = F64_TYPE; + elem_data_ptr_type = F64_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + /* Based on elem_size, trunc array_elem if necessary */ + if (trunc) { + if (!(array_elem = + LLVMBuildTrunc(comp_ctx->builder, array_elem, elem_data_type, + "array_elem_trunc"))) { + aot_set_last_error("llvm build trunc failed."); + goto fail; + } + } + + /* Cast to the field data type ptr */ + if (!(elem_data_ptr = + LLVMBuildBitCast(comp_ctx->builder, elem_data_ptr, + elem_data_ptr_type, "elem_data_ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(res = LLVMBuildStore(comp_ctx->builder, array_elem, elem_data_ptr))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + + if (!is_target_x86(comp_ctx) + && (elem_data_type == I64_TYPE || elem_data_type == F64_TYPE + || elem_data_type == GC_REF_TYPE)) { + LLVMSetAlignment(res, 4); + } + + return true; +fail: + return false; +} + +static bool +aot_call_aot_array_init_with_data( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef seg_index, + LLVMValueRef data_seg_offset, LLVMValueRef array_obj, + LLVMValueRef elem_size, LLVMValueRef array_len) +{ + LLVMValueRef param_values[6], func, value, res, cmp; + LLVMTypeRef param_types[6], ret_type, func_type, func_ptr_type; + LLVMBasicBlockRef init_success; + + ADD_BASIC_BLOCK(init_success, "init success"); + MOVE_BLOCK_AFTER_CURR(init_success); + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + param_types[4] = I32_TYPE; + param_types[5] = I32_TYPE; + ret_type = INT8_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_array_init_with_data, 6); + else + GET_AOT_FUNCTION(aot_array_init_with_data, 6); + + /* Call function aot_array_init_with_data() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = seg_index; + param_values[2] = data_seg_offset; + param_values[3] = array_obj; + param_values[4] = elem_size; + param_values[5] = array_len; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 6, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + BUILD_ICMP(LLVMIntEQ, res, I8_ZERO, cmp, "array_init_ret"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ARRAY_IDX_OOB, true, cmp, + init_success)) + goto fail; + + return true; +fail: + return false; +} + +static bool +aot_call_wasm_array_get_elem(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef array_obj, LLVMValueRef elem_idx, + LLVMValueRef *p_array_elem, uint8 array_elem_type, + bool sign) +{ + bool extend = false; + LLVMValueRef elem_data_ptr, array_elem; + LLVMTypeRef elem_data_type = NULL, elem_data_ptr_type = NULL; + + if (!aot_array_obj_elem_addr(comp_ctx, func_ctx, array_obj, elem_idx, + &elem_data_ptr, array_elem_type)) + goto fail; + + if (wasm_is_type_reftype(array_elem_type)) { + elem_data_type = GC_REF_TYPE; + elem_data_ptr_type = GC_REF_PTR_TYPE; + } + else + switch (array_elem_type) { + case PACKED_TYPE_I8: + elem_data_type = INT8_TYPE; + elem_data_ptr_type = INT8_PTR_TYPE; + extend = true; + break; + case PACKED_TYPE_I16: + elem_data_type = INT16_TYPE; + elem_data_ptr_type = INT16_PTR_TYPE; + extend = true; + break; + case VALUE_TYPE_I32: + elem_data_type = I32_TYPE; + elem_data_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + elem_data_type = I64_TYPE; + elem_data_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F32: + elem_data_type = F32_TYPE; + elem_data_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_F64: + elem_data_type = F64_TYPE; + elem_data_ptr_type = F64_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + /* Based on elem_size, trunc array_elem if necessary */ + if (!(elem_data_ptr = + LLVMBuildBitCast(comp_ctx->builder, elem_data_ptr, + elem_data_ptr_type, "elem_data_ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(array_elem = LLVMBuildLoad2(comp_ctx->builder, elem_data_type, + elem_data_ptr, "array_elem"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!is_target_x86(comp_ctx) + && (elem_data_type == I64_TYPE || elem_data_type == F64_TYPE + || elem_data_type == GC_REF_TYPE)) { + LLVMSetAlignment(array_elem, 4); + } + + if (extend) { + if (sign) { + if (!(array_elem = LLVMBuildSExt(comp_ctx->builder, array_elem, + I32_TYPE, "array_elem_sext"))) { + aot_set_last_error("llvm build signed ext failed."); + goto fail; + } + } + else { + if (!(array_elem = LLVMBuildZExt(comp_ctx->builder, array_elem, + I32_TYPE, "array_elem_zext"))) { + aot_set_last_error("llvm build unsigned ext failed."); + goto fail; + } + } + } + + *p_array_elem = array_elem; + return true; +fail: + return false; +} + +/* array_obj->length >> WASM_ARRAY_LENGTH_SHIFT */ +bool +aot_array_obj_length(AOTCompContext *comp_ctx, LLVMValueRef array_obj, + LLVMValueRef *p_array_len) +{ + LLVMValueRef offset, array_len; + + /* Get the length of the WASMArrayObject, the offset may be + * different in 32-bit runtime and 64-bit runtime since WASMObjectHeader + * is uintptr_t. Use comp_ctx->pointer_size as the + * offsetof(WASMArrayObject, length)*/ + if (!(offset = I32_CONST(comp_ctx->pointer_size))) { + aot_set_last_error("llvm build const failed."); + goto fail; + } + + if (!(array_len = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, array_obj, + &offset, 1, "array_obj_length_i8p"))) { + aot_set_last_error("llvm build gep failed."); + goto fail; + } + + if (!(array_len = + LLVMBuildBitCast(comp_ctx->builder, array_len, INT32_PTR_TYPE, + "array_obj_length_i32ptr"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(array_len = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, array_len, + "array_obj_length"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(array_len = LLVMBuildLShr(comp_ctx->builder, array_len, + I32_CONST(WASM_ARRAY_LENGTH_SHIFT), + "array_obj_length_shr"))) { + aot_set_last_error("llvm build lshr failed."); + goto fail; + } + + *p_array_len = array_len; + return true; +fail: + return false; +} + +bool +aot_compile_op_array_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, bool init_with_default, + bool fixed_size, uint32 array_len) +{ + LLVMValueRef array_length, array_elem = NULL, array_obj; + LLVMValueRef rtt_type, cmp, elem_idx; + LLVMBasicBlockRef check_rtt_type_succ, check_array_obj_succ; + /* Use for distinguish what type of AOTValue POP */ + WASMArrayType *compile_time_array_type = + (WASMArrayType *)comp_ctx->comp_data->types[type_index]; + uint8 array_elem_type = compile_time_array_type->elem_type; + uint32 i; + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + /* Generate call aot_rtt_type_new and check for exception */ + if (!aot_call_aot_rtt_type_new(comp_ctx, func_ctx, I32_CONST(type_index), + &rtt_type)) + goto fail; + + ADD_BASIC_BLOCK(check_rtt_type_succ, "check rtt type succ"); + MOVE_BLOCK_AFTER_CURR(check_rtt_type_succ); + + BUILD_ISNULL(rtt_type, cmp, "cmp_rtt_type"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_FAILED_TO_CREATE_RTT_TYPE, + true, cmp, check_rtt_type_succ)) + goto fail; + + if (!fixed_size) + POP_I32(array_length); + else + array_length = I32_CONST(array_len); + + /* For WASM_OP_ARRAY_NEW */ + if (!fixed_size && !init_with_default) { + if (wasm_is_type_reftype(array_elem_type)) { + POP_GC_REF(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I32 + || array_elem_type == PACKED_TYPE_I8 + || array_elem_type == PACKED_TYPE_I16) { + POP_I32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I64) { + POP_I64(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F32) { + POP_F32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F64) { + POP_F64(array_elem); + } + else { + bh_assert(0); + } + } + else { + /* I64 will alloca large enough space for all union access includes + * array_elem.gc_ob, i32, i64 to be interpreted as 0*/ + array_elem = I64_ZERO; + } + + /* Generate call wasm_array_obj_new and check for exception */ + if (!aot_call_wasm_array_obj_new(comp_ctx, func_ctx, rtt_type, array_length, + array_elem, &array_obj)) + goto fail; + + ADD_BASIC_BLOCK(check_array_obj_succ, "check array obj succ"); + MOVE_BLOCK_AFTER(check_array_obj_succ, check_rtt_type_succ); + + BUILD_ISNULL(array_obj, cmp, "cmp_array_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_FAILED_TO_CREATE_ARRAY_OBJ, + true, cmp, check_array_obj_succ)) + goto fail; + + if (fixed_size) { + for (i = 0; i < array_len; i++) { + if (wasm_is_type_reftype(array_elem_type)) { + POP_GC_REF(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I32 + || array_elem_type == PACKED_TYPE_I8 + || array_elem_type == PACKED_TYPE_I16) { + POP_I32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I64) { + POP_I64(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F32) { + POP_F32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F64) { + POP_F64(array_elem); + } + else { + bh_assert(0); + } + + /* array_len - 1 - i */ + if (!(elem_idx = LLVMBuildSub(comp_ctx->builder, array_length, + I32_CONST(i + 1), "elem_idx"))) { + aot_set_last_error("llvm build sub failed."); + goto fail; + } + + if (!aot_array_obj_set_elem(comp_ctx, func_ctx, array_obj, elem_idx, + array_elem, array_elem_type)) + goto fail; + } + } + + PUSH_GC_REF(array_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_array_new_data(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 type_index, + uint32 data_seg_index) +{ + LLVMValueRef array_length, data_seg_offset, rtt_type, + elem_size = NULL, array_elem, array_obj, cmp; + LLVMBasicBlockRef check_rtt_type_succ, check_array_obj_succ; + /* Use for distinguish what type of element in array */ + WASMArrayType *compile_time_array_type = + (WASMArrayType *)comp_ctx->comp_data->types[type_index]; + uint8 array_elem_type = compile_time_array_type->elem_type; + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + /* Generate call aot_rtt_type_new and check for exception */ + if (!aot_call_aot_rtt_type_new(comp_ctx, func_ctx, I32_CONST(type_index), + &rtt_type)) + goto fail; + + ADD_BASIC_BLOCK(check_rtt_type_succ, "check rtt type succ"); + MOVE_BLOCK_AFTER_CURR(check_rtt_type_succ); + + BUILD_ISNULL(rtt_type, cmp, "cmp_rtt_type"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_FAILED_TO_CREATE_RTT_TYPE, + true, cmp, check_rtt_type_succ)) + goto fail; + + POP_I32(array_length); + POP_I32(data_seg_offset); + + switch (array_elem_type) { + case PACKED_TYPE_I8: + elem_size = I32_ONE; + break; + case PACKED_TYPE_I16: + elem_size = I32_TWO; + break; + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + elem_size = I32_FOUR; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + elem_size = I32_EIGHT; + break; + default: + bh_assert(0); + } + + if (elem_size == I32_EIGHT) + array_elem = I64_ZERO; + else + array_elem = I32_ZERO; + + /* Generate call wasm_array_obj_new and check for exception */ + if (!aot_call_wasm_array_obj_new(comp_ctx, func_ctx, rtt_type, array_length, + array_elem, &array_obj)) + goto fail; + + ADD_BASIC_BLOCK(check_array_obj_succ, "check array obj succ"); + MOVE_BLOCK_AFTER(check_array_obj_succ, check_rtt_type_succ); + + BUILD_ISNULL(array_obj, cmp, "cmp_array_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_FAILED_TO_CREATE_ARRAY_OBJ, + true, cmp, check_array_obj_succ)) + goto fail; + + if (!aot_call_aot_array_init_with_data( + comp_ctx, func_ctx, I32_CONST(data_seg_index), data_seg_offset, + array_obj, elem_size, array_length)) + goto fail; + + PUSH_GC_REF(array_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_array_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, bool sign) +{ + LLVMValueRef elem_idx, array_obj, cmp, array_len, array_elem; + LLVMBasicBlockRef check_array_obj_succ, check_boundary_succ; + /* Use for distinguish what type of AOTValue PUSH */ + WASMArrayType *compile_time_array_type = + (WASMArrayType *)comp_ctx->comp_data->types[type_index]; + uint8 array_elem_type = compile_time_array_type->elem_type; + + POP_I32(elem_idx); + POP_GC_REF(array_obj); + + ADD_BASIC_BLOCK(check_array_obj_succ, "check array obj succ"); + MOVE_BLOCK_AFTER_CURR(check_array_obj_succ); + + BUILD_ISNULL(array_obj, cmp, "cmp_array_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_ARRAY_OBJ, true, cmp, + check_array_obj_succ)) + goto fail; + + SET_BUILDER_POS(check_array_obj_succ); + if (!aot_array_obj_length(comp_ctx, array_obj, &array_len)) + goto fail; + + ADD_BASIC_BLOCK(check_boundary_succ, "check boundary succ"); + MOVE_BLOCK_AFTER(check_boundary_succ, check_array_obj_succ); + + BUILD_ICMP(LLVMIntUGE, elem_idx, array_len, cmp, "cmp_array_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ARRAY_IDX_OOB, true, cmp, + check_boundary_succ)) + goto fail; + + SET_BUILDER_POS(check_boundary_succ); + if (!aot_call_wasm_array_get_elem(comp_ctx, func_ctx, array_obj, elem_idx, + &array_elem, array_elem_type, sign)) + goto fail; + + if (wasm_is_type_reftype(array_elem_type)) { + PUSH_GC_REF(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I32 + || array_elem_type == PACKED_TYPE_I8 + || array_elem_type == PACKED_TYPE_I16) { + PUSH_I32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I64) { + PUSH_I64(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F32) { + PUSH_F32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F64) { + PUSH_F64(array_elem); + } + else { + bh_assert(0); + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_array_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index) +{ + LLVMValueRef elem_idx, array_obj, cmp, array_len, array_elem = NULL; + LLVMBasicBlockRef check_array_obj_succ, check_boundary_succ; + /* Use for distinguish what type of AOTValue POP */ + WASMArrayType *compile_time_array_type = + (WASMArrayType *)comp_ctx->comp_data->types[type_index]; + uint8 array_elem_type = compile_time_array_type->elem_type; + + /* Get LLVM type based on array_elem_type */ + if (wasm_is_type_reftype(array_elem_type)) { + POP_GC_REF(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I32 + || array_elem_type == PACKED_TYPE_I8 + || array_elem_type == PACKED_TYPE_I16) { + POP_I32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_I64) { + POP_I64(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F32) { + POP_F32(array_elem); + } + else if (array_elem_type == VALUE_TYPE_F64) { + POP_F64(array_elem); + } + else { + bh_assert(0); + } + + POP_I32(elem_idx); + POP_GC_REF(array_obj); + + ADD_BASIC_BLOCK(check_array_obj_succ, "check array obj succ"); + MOVE_BLOCK_AFTER_CURR(check_array_obj_succ); + + BUILD_ISNULL(array_obj, cmp, "cmp_array_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_ARRAY_OBJ, true, cmp, + check_array_obj_succ)) + goto fail; + + SET_BUILDER_POS(check_array_obj_succ); + if (!aot_array_obj_length(comp_ctx, array_obj, &array_len)) + goto fail; + + ADD_BASIC_BLOCK(check_boundary_succ, "check boundary succ"); + MOVE_BLOCK_AFTER(check_boundary_succ, check_array_obj_succ); + + BUILD_ICMP(LLVMIntUGE, elem_idx, array_len, cmp, "cmp_array_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ARRAY_IDX_OOB, true, cmp, + check_boundary_succ)) + goto fail; + + SET_BUILDER_POS(check_boundary_succ); + if (!aot_array_obj_set_elem(comp_ctx, func_ctx, array_obj, elem_idx, + array_elem, array_elem_type)) { + aot_set_last_error("llvm build alloca failed."); + goto fail; + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_array_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index) +{ + LLVMValueRef len, array_obj, fill_value = NULL, offset, array_len, cmp[2], + boundary, loop_counter_addr, loop_counter_val; + LLVMBasicBlockRef check_obj_succ, len_gt_zero, len_le_zero, inner_else; + LLVMBasicBlockRef fill_loop_header, fill_loop_body; + WASMArrayType *compile_time_array_type = + (WASMArrayType *)comp_ctx->comp_data->types[type_index]; + uint8 array_elem_type = compile_time_array_type->elem_type; + + POP_I32(len); + /* Get LLVM type based on array_elem_type */ + if (wasm_is_type_reftype(array_elem_type)) { + POP_GC_REF(fill_value); + } + else if (array_elem_type == VALUE_TYPE_I32 + || array_elem_type == PACKED_TYPE_I8 + || array_elem_type == PACKED_TYPE_I16) { + POP_I32(fill_value); + } + else if (array_elem_type == VALUE_TYPE_I64) { + POP_I64(fill_value); + } + else if (array_elem_type == VALUE_TYPE_F32) { + POP_F32(fill_value); + } + else if (array_elem_type == VALUE_TYPE_F64) { + POP_F64(fill_value); + } + else { + bh_assert(0); + } + + POP_I32(offset); + POP_GC_REF(array_obj); + + ADD_BASIC_BLOCK(check_obj_succ, "check array objs succ"); + MOVE_BLOCK_AFTER_CURR(check_obj_succ); + + BUILD_ISNULL(array_obj, cmp[0], "cmp_obj"); + + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_ARRAY_OBJ, true, + cmp[0], check_obj_succ)) + goto fail; + + /* Create if block */ + ADD_BASIC_BLOCK(len_gt_zero, "len_gt_zero"); + MOVE_BLOCK_AFTER_CURR(len_gt_zero); + + /* Create inner else block */ + ADD_BASIC_BLOCK(inner_else, "inner_else"); + MOVE_BLOCK_AFTER(inner_else, len_gt_zero); + + /* Create fill_loop_header block */ + ADD_BASIC_BLOCK(fill_loop_header, "fill_loop_header"); + MOVE_BLOCK_AFTER(fill_loop_header, len_gt_zero); + + /* Create fill_loop_body block */ + ADD_BASIC_BLOCK(fill_loop_body, "fill_loop_body"); + MOVE_BLOCK_AFTER(fill_loop_body, len_gt_zero); + + /* Create else(end) block */ + ADD_BASIC_BLOCK(len_le_zero, "len_le_zero"); + MOVE_BLOCK_AFTER(len_le_zero, len_gt_zero); + + BUILD_ICMP(LLVMIntSGT, len, I32_ZERO, cmp[0], "cmp_len"); + BUILD_COND_BR(cmp[0], len_gt_zero, len_le_zero); + + /* Move builder to len > 0 block */ + SET_BUILDER_POS(len_gt_zero); + /* dst_offset > UINT32_MAX - len */ + if (!(boundary = LLVMBuildAdd(comp_ctx->builder, offset, len, ""))) { + aot_set_last_error("llvm build failed."); + goto fail; + } + BUILD_ICMP(LLVMIntUGT, boundary, I32_CONST(UINT32_MAX), cmp[0], + "boundary_check1"); + /* dst_offset + len > wasm_array_obj_length(dst_obj) */ + if (!aot_array_obj_length(comp_ctx, array_obj, &array_len)) + goto fail; + BUILD_ICMP(LLVMIntUGT, boundary, array_len, cmp[1], "boundary_check2"); + + if (!(cmp[0] = LLVMBuildOr(comp_ctx->builder, cmp[0], cmp[1], ""))) { + aot_set_last_error("llvm build failed."); + goto fail; + } + + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ARRAY_IDX_OOB, true, + cmp[0], inner_else)) + goto fail; + + if (!(loop_counter_addr = LLVMBuildAlloca(comp_ctx->builder, I32_TYPE, + "fill_loop_counter"))) { + aot_set_last_error("llvm build alloc failed."); + goto fail; + } + + if (!is_target_x86(comp_ctx)) { + LLVMSetAlignment(loop_counter_addr, 4); + } + + if (!LLVMBuildStore(comp_ctx->builder, offset, loop_counter_addr)) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + + BUILD_BR(fill_loop_header); + SET_BUILDER_POS(fill_loop_header); + + if (!(loop_counter_val = + LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, loop_counter_addr, + "fill_loop_counter"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + BUILD_ICMP(LLVMIntULT, loop_counter_val, boundary, cmp[0], + "cmp_loop_counter"); + BUILD_COND_BR(cmp[0], fill_loop_body, len_le_zero); + + SET_BUILDER_POS(fill_loop_body); + + if (!aot_array_obj_set_elem(comp_ctx, func_ctx, array_obj, loop_counter_val, + fill_value, array_elem_type)) + goto fail; + + if (!(loop_counter_val = LLVMBuildAdd(comp_ctx->builder, loop_counter_val, + I32_ONE, "fill_loop_counter"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + + if (!LLVMBuildStore(comp_ctx->builder, loop_counter_val, + loop_counter_addr)) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + + BUILD_BR(fill_loop_header); + + SET_BUILDER_POS(len_le_zero); + + return true; +fail: + return false; +} + +static bool +aot_call_wasm_array_obj_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef dst_obj, LLVMValueRef dst_offset, + LLVMValueRef src_obj, LLVMValueRef src_offset, + LLVMValueRef len) +{ + LLVMValueRef param_values[5], func, value; + LLVMTypeRef param_types[5], ret_type, func_type, func_ptr_type; + + param_types[0] = GC_REF_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = GC_REF_TYPE; + param_types[3] = I32_TYPE; + param_types[4] = I32_TYPE; + ret_type = VOID_TYPE; + + GET_AOT_FUNCTION(wasm_array_obj_copy, 5); + + /* Call function wasm_array_obj_copy() */ + param_values[0] = dst_obj; + param_values[1] = dst_offset; + param_values[2] = src_obj; + param_values[3] = src_offset; + param_values[4] = len; + if (!LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, 5, + "")) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_array_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, uint32 src_type_index) +{ + LLVMValueRef len, src_offset, src_obj, dst_offset, dst_obj, array_len, + cmp[4], boundary; + LLVMBasicBlockRef check_objs_succ, len_gt_zero, len_le_zero, inner_else; + int i; + + POP_I32(len); + POP_I32(src_offset); + POP_GC_REF(src_obj); + POP_I32(dst_offset); + POP_GC_REF(dst_obj); + + ADD_BASIC_BLOCK(check_objs_succ, "check array objs succ"); + MOVE_BLOCK_AFTER_CURR(check_objs_succ); + + BUILD_ISNULL(src_obj, cmp[0], "cmp_src_obj"); + BUILD_ISNULL(dst_obj, cmp[1], "cmp_dst_obj"); + + /* src_obj is null or dst_obj is null, throw exception */ + if (!(cmp[0] = LLVMBuildOr(comp_ctx->builder, cmp[0], cmp[1], ""))) { + aot_set_last_error("llvm build or failed."); + goto fail; + } + + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_ARRAY_OBJ, true, + cmp[0], check_objs_succ)) + goto fail; + + /* Create if block */ + ADD_BASIC_BLOCK(len_gt_zero, "len_gt_zero"); + MOVE_BLOCK_AFTER_CURR(len_gt_zero); + + /* Create else(end) block */ + ADD_BASIC_BLOCK(len_le_zero, "len_le_zero"); + MOVE_BLOCK_AFTER(len_le_zero, len_gt_zero); + + /* Create inner else block */ + ADD_BASIC_BLOCK(inner_else, "inner_else"); + MOVE_BLOCK_AFTER(inner_else, len_gt_zero); + + BUILD_ICMP(LLVMIntSGT, len, I32_ZERO, cmp[0], "cmp_len"); + BUILD_COND_BR(cmp[0], len_gt_zero, len_le_zero); + + /* Move builder to len > 0 block */ + SET_BUILDER_POS(len_gt_zero); + /* dst_offset > UINT32_MAX - len */ + if (!(boundary = LLVMBuildAdd(comp_ctx->builder, dst_offset, len, ""))) { + aot_set_last_error("llvm build failed."); + goto fail; + } + BUILD_ICMP(LLVMIntUGT, boundary, I32_CONST(UINT32_MAX), cmp[0], + "boundary_check1"); + /* dst_offset + len > wasm_array_obj_length(dst_obj) */ + if (!aot_array_obj_length(comp_ctx, dst_obj, &array_len)) + goto fail; + BUILD_ICMP(LLVMIntUGT, boundary, array_len, cmp[1], "boundary_check2"); + /* src_offset > UINT32_MAX - len */ + if (!(boundary = LLVMBuildAdd(comp_ctx->builder, src_offset, len, ""))) { + aot_set_last_error("llvm build failed."); + goto fail; + } + BUILD_ICMP(LLVMIntUGT, boundary, I32_CONST(UINT32_MAX), cmp[2], + "boundary_check3"); + /* src_offset + len > wasm_array_obj_length(src_obj) */ + if (!aot_array_obj_length(comp_ctx, src_obj, &array_len)) + goto fail; + BUILD_ICMP(LLVMIntUGT, boundary, array_len, cmp[3], "boundary_check4"); + + /* logical or above 4 boundary checks */ + for (i = 1; i < 4; ++i) { + if (!(cmp[0] = LLVMBuildOr(comp_ctx->builder, cmp[0], cmp[i], ""))) { + aot_set_last_error("llvm build failed."); + goto fail; + } + } + + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ARRAY_IDX_OOB, true, + cmp[0], inner_else)) + goto fail; + + if (!aot_call_wasm_array_obj_copy(comp_ctx, func_ctx, dst_obj, dst_offset, + src_obj, src_offset, len)) + goto fail; + + BUILD_BR(len_le_zero); + SET_BUILDER_POS(len_le_zero); + + return true; +fail: + return false; +} + +bool +aot_compile_op_array_len(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef array_obj, cmp, array_len; + LLVMBasicBlockRef check_array_obj_succ; + + POP_GC_REF(array_obj); + + ADD_BASIC_BLOCK(check_array_obj_succ, "check array obj succ"); + MOVE_BLOCK_AFTER_CURR(check_array_obj_succ); + + BUILD_ISNULL(array_obj, cmp, "cmp_array_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_ARRAY_OBJ, true, cmp, + check_array_obj_succ)) + goto fail; + + if (!aot_array_obj_length(comp_ctx, array_obj, &array_len)) + goto fail; + + PUSH_I32(array_len); + + return true; +fail: + return false; +} + +bool +aot_compile_op_i31_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef i31_val, i31_obj; + + POP_I32(i31_val); + + /* i31_val <<= 1 */ + if (!(i31_val = LLVMBuildShl(comp_ctx->builder, i31_val, I32_ONE, + "i31_val_shl"))) { + aot_set_last_error("llvm build shl failed."); + goto fail; + } + + /* i31_val |= 1 */ + if (!(i31_val = + LLVMBuildOr(comp_ctx->builder, i31_val, I32_ONE, "i31_val_or"))) { + aot_set_last_error("llvm build or failed."); + goto fail; + } + + if (!(i31_obj = LLVMBuildIntToPtr(comp_ctx->builder, i31_val, GC_REF_TYPE, + "i31_obj"))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + + PUSH_GC_REF(i31_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_i31_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign) +{ + LLVMValueRef i31_obj, i31_val, cmp_i31_obj; + LLVMBasicBlockRef check_i31_obj_succ; + + POP_GC_REF(i31_obj); + + ADD_BASIC_BLOCK(check_i31_obj_succ, "check_i31_obj_succ"); + MOVE_BLOCK_AFTER_CURR(check_i31_obj_succ); + + /* Check if i31 object is NULL, throw exception if it is */ + BUILD_ISNULL(i31_obj, cmp_i31_obj, "cmp_i31_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NULL_I31_OBJ, true, + cmp_i31_obj, check_i31_obj_succ)) { + goto fail; + } + + if (!(i31_val = LLVMBuildPtrToInt(comp_ctx->builder, i31_obj, I32_TYPE, + "i31_val"))) { + aot_set_last_error("llvm build ptr to init failed."); + goto fail; + } + + if (!sign) { + if (!(i31_val = LLVMBuildLShr(comp_ctx->builder, i31_val, I32_ONE, + "i31_value"))) { + aot_set_last_error("llvm build lshr failed."); + goto fail; + } + } + else { + if (!(i31_val = LLVMBuildAShr(comp_ctx->builder, i31_val, I32_ONE, + "i31_value"))) { + aot_set_last_error("llvm build ashr failed."); + goto fail; + } + } + + PUSH_I32(i31_val); + + return true; +fail: + return false; +} + +bool +aot_compile_op_ref_test(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 heap_type, bool nullable) +{ + LLVMValueRef gc_obj, ref_test_phi, cmp, castable; + LLVMBasicBlockRef block_curr, block_obj_non_null, block_end; + + POP_GC_REF(gc_obj); + + block_curr = CURR_BLOCK(); + + /* Create non-null object block */ + ADD_BASIC_BLOCK(block_obj_non_null, "non_null_obj"); + MOVE_BLOCK_AFTER_CURR(block_obj_non_null); + + /* Create end block */ + ADD_BASIC_BLOCK(block_end, "ref_test_end"); + MOVE_BLOCK_AFTER(block_end, block_obj_non_null); + + /* Create ref test result phi */ + SET_BUILDER_POS(block_end); + if (!(ref_test_phi = + LLVMBuildPhi(comp_ctx->builder, INT1_TYPE, "ref_test_res"))) { + aot_set_last_error("llvm build phi failed"); + return false; + } + + /* Check if gc object is NULL */ + SET_BUILDER_POS(block_curr); + BUILD_ISNULL(gc_obj, cmp, "cmp_gc_obj"); + BUILD_COND_BR(cmp, block_end, block_obj_non_null); + + if (nullable) + LLVMAddIncoming(ref_test_phi, &I1_ONE, &block_curr, 1); + else + LLVMAddIncoming(ref_test_phi, &I1_ZERO, &block_curr, 1); + + /* Move builder to non-null object block */ + SET_BUILDER_POS(block_obj_non_null); + + if (heap_type >= 0) { + if (!aot_call_aot_obj_is_instance_of(comp_ctx, func_ctx, gc_obj, + I32_CONST(heap_type), &castable)) + return false; + } + else { + if (!aot_call_wasm_obj_is_type_of(comp_ctx, func_ctx, gc_obj, + I32_CONST(heap_type), &castable)) + return false; + } + + if (!(castable = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, castable, + I8_ZERO, "castable"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + + BUILD_BR(block_end); + LLVMAddIncoming(ref_test_phi, &castable, &block_obj_non_null, 1); + + SET_BUILDER_POS(block_end); + PUSH_COND(ref_test_phi); + + return true; +fail: + return false; +} + +bool +aot_compile_op_ref_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 heap_type, bool nullable) +{ + LLVMValueRef gc_obj, cmp, castable; + LLVMBasicBlockRef block_obj_non_null, block_end; + + GET_GC_REF_FROM_STACK(gc_obj); + + /* Create non null block */ + ADD_BASIC_BLOCK(block_obj_non_null, "non_null_obj"); + MOVE_BLOCK_AFTER_CURR(block_obj_non_null); + + /* Create end block */ + ADD_BASIC_BLOCK(block_end, "ref_cast_end"); + MOVE_BLOCK_AFTER(block_end, block_obj_non_null); + + BUILD_ISNULL(gc_obj, cmp, "obj_is_null"); + if (nullable) { + BUILD_COND_BR(cmp, block_end, block_obj_non_null); + } + else { + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_CAST_FAILURE, true, + cmp, block_obj_non_null)) { + return false; + } + } + + SET_BUILDER_POS(block_obj_non_null); + + if (heap_type >= 0) { + if (!aot_call_aot_obj_is_instance_of(comp_ctx, func_ctx, gc_obj, + I32_CONST(heap_type), &castable)) + return false; + } + else { + if (!aot_call_wasm_obj_is_type_of(comp_ctx, func_ctx, gc_obj, + I32_CONST(heap_type), &castable)) + return false; + } + + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, castable, I8_ZERO, + "is_uncastable"))) { + aot_set_last_error("llvm build not failed"); + return false; + } + + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_CAST_FAILURE, true, cmp, + block_end)) { + return false; + } + + SET_BUILDER_POS(block_end); + + return true; +fail: + return false; +} + +static bool +aot_call_wasm_externref_obj_to_internal_obj(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef externref_obj, + LLVMValueRef *gc_obj) +{ + LLVMValueRef param_values[1], func, value, res; + LLVMTypeRef param_types[1], ret_type, func_type, func_ptr_type; + + param_types[0] = GC_REF_TYPE; + ret_type = GC_REF_TYPE; + + GET_AOT_FUNCTION(wasm_externref_obj_to_internal_obj, 1); + + /* Call function wasm_externref_obj_to_internal_obj */ + param_values[0] = externref_obj; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 1, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *gc_obj = res; + + return true; +fail: + return false; +} + +bool +aot_compile_op_extern_internalize(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef externref_obj, gc_obj, cmp, internal_obj_phi; + LLVMBasicBlockRef block_curr, block_obj_non_null, block_end; + + POP_GC_REF(externref_obj); + + block_curr = CURR_BLOCK(); + + /* Create non-null object block */ + ADD_BASIC_BLOCK(block_obj_non_null, "non_null_obj"); + MOVE_BLOCK_AFTER_CURR(block_obj_non_null); + + /* Create end block */ + ADD_BASIC_BLOCK(block_end, "internalize_end"); + MOVE_BLOCK_AFTER(block_end, block_obj_non_null); + + /* Create internalized object phi */ + SET_BUILDER_POS(block_end); + if (!(internal_obj_phi = + LLVMBuildPhi(comp_ctx->builder, GC_REF_TYPE, "internal_obj"))) { + aot_set_last_error("llvm build phi failed"); + return false; + } + + /* Check if externref object is NULL */ + SET_BUILDER_POS(block_curr); + BUILD_ISNULL(externref_obj, cmp, "cmp_externref_obj"); + BUILD_COND_BR(cmp, block_end, block_obj_non_null); + LLVMAddIncoming(internal_obj_phi, &GC_REF_NULL, &block_curr, 1); + + /* Move builder to non-null object block */ + SET_BUILDER_POS(block_obj_non_null); + if (!aot_call_wasm_externref_obj_to_internal_obj(comp_ctx, func_ctx, + externref_obj, &gc_obj)) { + return false; + } + BUILD_BR(block_end); + LLVMAddIncoming(internal_obj_phi, &gc_obj, &block_obj_non_null, 1); + + /* Move builder to end block */ + SET_BUILDER_POS(block_end); + PUSH_GC_REF(internal_obj_phi); + + return true; +fail: + return false; +} + +static bool +aot_call_wasm_internal_obj_to_external_obj(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef gc_obj, + LLVMValueRef *externref_obj) +{ + LLVMValueRef param_values[2], func, value, res; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = GC_REF_TYPE; + ret_type = GC_REF_TYPE; + + GET_AOT_FUNCTION(wasm_internal_obj_to_externref_obj, 2); + + /* Call function wasm_internal_obj_to_externref_obj() */ + param_values[0] = func_ctx->exec_env; + param_values[1] = gc_obj; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 2, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *externref_obj = res; + + return true; +fail: + return false; +} + +bool +aot_compile_op_extern_externalize(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef gc_obj, cmp, external_obj_phi, externref_obj; + LLVMBasicBlockRef block_curr, block_obj_non_null, block_end; + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_GC_REF(gc_obj); + + block_curr = CURR_BLOCK(); + + /* Create non-null object block */ + ADD_BASIC_BLOCK(block_obj_non_null, "non_null_obj"); + MOVE_BLOCK_AFTER_CURR(block_obj_non_null); + + /* Create end block */ + ADD_BASIC_BLOCK(block_end, "externalize_end"); + MOVE_BLOCK_AFTER(block_end, block_obj_non_null); + + /* Create externalized object phi */ + SET_BUILDER_POS(block_end); + if (!(external_obj_phi = + LLVMBuildPhi(comp_ctx->builder, GC_REF_TYPE, "external_obj"))) { + aot_set_last_error("llvm build phi failed"); + return false; + } + + /* Check if gc object is NULL */ + SET_BUILDER_POS(block_curr); + BUILD_ISNULL(gc_obj, cmp, "cmp_gc_obj"); + BUILD_COND_BR(cmp, block_end, block_obj_non_null); + LLVMAddIncoming(external_obj_phi, &GC_REF_NULL, &block_curr, 1); + + /* Move builder to non-null object block */ + SET_BUILDER_POS(block_obj_non_null); + + if (!aot_call_wasm_internal_obj_to_external_obj(comp_ctx, func_ctx, gc_obj, + &externref_obj)) { + return false; + } + + /* Check whether failed to externalize */ + BUILD_ISNULL(externref_obj, cmp, "cmp_externref_obj"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_FAILED_TO_CREATE_EXTERNREF_OBJ, true, cmp, + block_end)) { + return false; + } + + LLVMAddIncoming(external_obj_phi, &externref_obj, &block_obj_non_null, 1); + + /* Move builder to end block */ + SET_BUILDER_POS(block_end); + PUSH_GC_REF(external_obj_phi); + + return true; +fail: + return false; +} + +#endif /* end of WASM_ENABLE_GC != 0 */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_gc.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_gc.h new file mode 100644 index 0000000..40a3c23 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_gc.h @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_GC_H_ +#define _AOT_EMIT_GC_H_ + +#include "aot_compiler.h" +#include "aot_runtime.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if WASM_ENABLE_GC != 0 + +bool +aot_call_aot_create_func_obj(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef func_idx, LLVMValueRef *p_gc_obj); + +bool +aot_call_aot_obj_is_instance_of(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, LLVMValueRef gc_obj, + LLVMValueRef heap_type, LLVMValueRef *castable); + +bool +aot_call_wasm_obj_is_type_of(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef gc_obj, LLVMValueRef heap_type, + LLVMValueRef *castable); + +bool +aot_call_aot_rtt_type_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef type_index, LLVMValueRef *rtt_type); + +bool +aot_compile_op_ref_as_non_null(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_struct_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, bool init_with_default); + +bool +aot_compile_op_struct_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, uint32 field_idx, bool sign); + +bool +aot_compile_op_struct_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, uint32 field_idx); + +bool +aot_compile_op_array_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, bool init_with_default, + bool fixed_size, uint32 array_len); + +bool +aot_compile_op_array_new_data(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 type_index, + uint32 data_seg_index); + +bool +aot_array_obj_length(AOTCompContext *comp_ctx, LLVMValueRef array_obj, + LLVMValueRef *p_array_len); + +bool +aot_array_obj_elem_addr(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef array_obj, LLVMValueRef elem_idx, + LLVMValueRef *p_elem_data, uint8 array_elem_type); + +bool +aot_compile_op_array_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, bool sign); + +bool +aot_compile_op_array_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index); + +bool +aot_compile_op_array_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index); + +bool +aot_compile_op_array_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 type_index, uint32 src_type_index); + +bool +aot_compile_op_array_len(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i31_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i31_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool sign); + +bool +aot_compile_op_ref_test(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 heap_type, bool nullable); + +bool +aot_compile_op_ref_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + int32 heap_type, bool nullable); + +bool +aot_compile_op_extern_internalize(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_extern_externalize(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_GC_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_memory.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_memory.c new file mode 100644 index 0000000..eedc542 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_memory.c @@ -0,0 +1,1567 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_memory.h" +#include "aot_compiler.h" +#include "aot_emit_exception.h" +#include "../aot/aot_runtime.h" +#include "aot_intrinsic.h" +#include "aot_emit_control.h" + +#define BUILD_ICMP(op, left, right, res, name) \ + do { \ + if (!(res = \ + LLVMBuildICmp(comp_ctx->builder, op, left, right, name))) { \ + aot_set_last_error("llvm build icmp failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_OP(Op, left, right, res, name) \ + do { \ + if (!(res = LLVMBuild##Op(comp_ctx->builder, left, right, name))) { \ + aot_set_last_error("llvm build " #Op " fail."); \ + goto fail; \ + } \ + } while (0) + +#define ADD_BASIC_BLOCK(block, name) \ + do { \ + if (!(block = LLVMAppendBasicBlockInContext(comp_ctx->context, \ + func_ctx->func, name))) { \ + aot_set_last_error("llvm add basic block failed."); \ + goto fail; \ + } \ + } while (0) + +#define SET_BUILD_POS(block) LLVMPositionBuilderAtEnd(comp_ctx->builder, block) + +static LLVMValueRef +get_memory_check_bound(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 bytes) +{ + LLVMValueRef mem_check_bound = NULL; + switch (bytes) { + case 1: + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_1byte; + break; + case 2: + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_2bytes; + break; + case 4: + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_4bytes; + break; + case 8: + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_8bytes; + break; + case 16: + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_16bytes; + break; + default: + bh_assert(0); + return NULL; + } + + if (func_ctx->mem_space_unchanged) + return mem_check_bound; + + if (!(mem_check_bound = LLVMBuildLoad2( + comp_ctx->builder, + (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE : I32_TYPE, + mem_check_bound, "mem_check_bound"))) { + aot_set_last_error("llvm build load failed."); + return NULL; + } + return mem_check_bound; +} + +static LLVMValueRef +get_memory_curr_page_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +LLVMValueRef +aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 offset, uint32 bytes, bool enable_segue) +{ + LLVMValueRef offset_const = I32_CONST(offset); + LLVMValueRef addr, maddr, offset1, cmp1, cmp2, cmp; + LLVMValueRef mem_base_addr, mem_check_bound; + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMBasicBlockRef check_succ; + AOTValue *aot_value_top; + uint32 local_idx_of_aot_value = 0; + bool is_target_64bit, is_local_of_aot_value = false; +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory = + comp_ctx->comp_data->memories[0].memory_flags & 0x02; +#endif + + is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; + + if (comp_ctx->is_indirect_mode + && aot_intrinsic_check_capability(comp_ctx, "i32.const")) { + WASMValue wasm_value; + wasm_value.i32 = offset; + offset_const = aot_load_const_from_table( + comp_ctx, func_ctx->native_symbol, &wasm_value, VALUE_TYPE_I32); + if (!offset_const) { + return NULL; + } + } + else { + CHECK_LLVM_CONST(offset_const); + } + + /* Get memory base address and memory data size */ + if (func_ctx->mem_space_unchanged +#if WASM_ENABLE_SHARED_MEMORY != 0 + || is_shared_memory +#endif + ) { + mem_base_addr = func_ctx->mem_info[0].mem_base_addr; + } + else { + if (!(mem_base_addr = LLVMBuildLoad2( + comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->mem_info[0].mem_base_addr, "mem_base"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + } + + aot_value_top = + func_ctx->block_stack.block_list_end->value_stack.value_list_end; + if (aot_value_top) { + /* aot_value_top is freed in the following POP_I32(addr), + so save its fields here for further use */ + is_local_of_aot_value = aot_value_top->is_local; + local_idx_of_aot_value = aot_value_top->local_idx; + } + + POP_I32(addr); + + /* + * Note: not throw the integer-overflow-exception here since it must + * have been thrown when converting float to integer before + */ + /* return addres directly if constant offset and inside memory space */ + if (LLVMIsEfficientConstInt(addr)) { + uint64 mem_offset = + (uint64)LLVMConstIntGetZExtValue(addr) + (uint64)offset; + uint32 num_bytes_per_page = + comp_ctx->comp_data->memories[0].num_bytes_per_page; + uint32 init_page_count = + comp_ctx->comp_data->memories[0].mem_init_page_count; + uint64 mem_data_size = (uint64)num_bytes_per_page * init_page_count; + + if (mem_offset + bytes <= mem_data_size) { + /* inside memory space */ + if (comp_ctx->pointer_size == sizeof(uint64)) + offset1 = I64_CONST((uint32)mem_offset); + else + offset1 = I32_CONST((uint32)mem_offset); + CHECK_LLVM_CONST(offset1); + if (!enable_segue) { + if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, + INT8_TYPE, mem_base_addr, + &offset1, 1, "maddr"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + } + else { + if (!(maddr = LLVMBuildIntToPtr(comp_ctx->builder, offset1, + INT8_PTR_TYPE_GS, "maddr"))) { + aot_set_last_error("llvm build IntToPtr failed."); + goto fail; + } + } + return maddr; + } + } + + if (is_target_64bit) { + if (!(offset_const = LLVMBuildZExt(comp_ctx->builder, offset_const, + I64_TYPE, "offset_i64")) + || !(addr = LLVMBuildZExt(comp_ctx->builder, addr, I64_TYPE, + "addr_i64"))) { + aot_set_last_error("llvm build zero extend failed."); + goto fail; + } + } + + /* offset1 = offset + addr; */ + BUILD_OP(Add, offset_const, addr, offset1, "offset1"); + + if (comp_ctx->enable_bound_check + && !(is_local_of_aot_value + && aot_checked_addr_list_find(func_ctx, local_idx_of_aot_value, + offset, bytes))) { + uint32 init_page_count = + comp_ctx->comp_data->memories[0].mem_init_page_count; + if (init_page_count == 0) { + LLVMValueRef mem_size; + + if (!(mem_size = get_memory_curr_page_count(comp_ctx, func_ctx))) { + goto fail; + } + BUILD_ICMP(LLVMIntEQ, mem_size, I32_ZERO, cmp, "is_zero"); + ADD_BASIC_BLOCK(check_succ, "check_mem_size_succ"); + LLVMMoveBasicBlockAfter(check_succ, block_curr); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; + } + + SET_BUILD_POS(check_succ); + block_curr = check_succ; + } + + if (!(mem_check_bound = + get_memory_check_bound(comp_ctx, func_ctx, bytes))) { + goto fail; + } + + if (is_target_64bit) { + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); + } + else { + /* Check integer overflow */ + BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); + BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); + } + + /* Add basic blocks */ + ADD_BASIC_BLOCK(check_succ, "check_succ"); + LLVMMoveBasicBlockAfter(check_succ, block_curr); + + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; + } + + SET_BUILD_POS(check_succ); + + if (is_local_of_aot_value) { + if (!aot_checked_addr_list_add(func_ctx, local_idx_of_aot_value, + offset, bytes)) + goto fail; + } + } + + if (!enable_segue) { + /* maddr = mem_base_addr + offset1 */ + if (!(maddr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + mem_base_addr, &offset1, 1, "maddr"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + } + else { + LLVMValueRef maddr_base; + + if (!(maddr_base = LLVMBuildIntToPtr(comp_ctx->builder, addr, + INT8_PTR_TYPE_GS, "maddr_base"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + maddr_base, &offset_const, 1, + "maddr"))) { + aot_set_last_error("llvm build inboundgep failed."); + goto fail; + } + } + return maddr; +fail: + return NULL; +} + +#define BUILD_PTR_CAST(ptr_type) \ + do { \ + if (!(maddr = LLVMBuildBitCast(comp_ctx->builder, maddr, ptr_type, \ + "data_ptr"))) { \ + aot_set_last_error("llvm build bit cast failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_LOAD(data_type) \ + do { \ + if (!(value = LLVMBuildLoad2(comp_ctx->builder, data_type, maddr, \ + "data"))) { \ + aot_set_last_error("llvm build load failed."); \ + goto fail; \ + } \ + LLVMSetAlignment(value, 1); \ + } while (0) + +#define BUILD_TRUNC(value, data_type) \ + do { \ + if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ + "val_trunc"))) { \ + aot_set_last_error("llvm build trunc failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_STORE() \ + do { \ + LLVMValueRef res; \ + if (!(res = LLVMBuildStore(comp_ctx->builder, value, maddr))) { \ + aot_set_last_error("llvm build store failed."); \ + goto fail; \ + } \ + LLVMSetAlignment(res, 1); \ + } while (0) + +#define BUILD_SIGN_EXT(dst_type) \ + do { \ + if (!(value = LLVMBuildSExt(comp_ctx->builder, value, dst_type, \ + "data_s_ext"))) { \ + aot_set_last_error("llvm build sign ext failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_ZERO_EXT(dst_type) \ + do { \ + if (!(value = LLVMBuildZExt(comp_ctx->builder, value, dst_type, \ + "data_z_ext"))) { \ + aot_set_last_error("llvm build zero ext failed."); \ + goto fail; \ + } \ + } while (0) + +#if WASM_ENABLE_SHARED_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 +bool +check_memory_alignment(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef addr, uint32 align) +{ + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMBasicBlockRef check_align_succ; + LLVMValueRef align_mask = I32_CONST(((uint32)1 << align) - 1); + LLVMValueRef res; + + CHECK_LLVM_CONST(align_mask); + + /* Convert pointer to int */ + if (!(addr = LLVMBuildPtrToInt(comp_ctx->builder, addr, I32_TYPE, + "address"))) { + aot_set_last_error("llvm build ptr to int failed."); + goto fail; + } + + /* The memory address should be aligned */ + BUILD_OP(And, addr, align_mask, res, "and"); + BUILD_ICMP(LLVMIntNE, res, I32_ZERO, res, "cmp"); + + /* Add basic blocks */ + ADD_BASIC_BLOCK(check_align_succ, "check_align_succ"); + LLVMMoveBasicBlockAfter(check_align_succ, block_curr); + + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_UNALIGNED_ATOMIC, true, + res, check_align_succ)) { + goto fail; + } + + SET_BUILD_POS(check_align_succ); + + return true; +fail: + return false; +} +#endif /* WASM_ENABLE_SHARED_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define BUILD_ATOMIC_LOAD(align, data_type) \ + do { \ + if (!(check_memory_alignment(comp_ctx, func_ctx, maddr, align))) { \ + goto fail; \ + } \ + if (!(value = LLVMBuildLoad2(comp_ctx->builder, data_type, maddr, \ + "data"))) { \ + aot_set_last_error("llvm build load failed."); \ + goto fail; \ + } \ + LLVMSetAlignment(value, 1 << align); \ + LLVMSetVolatile(value, true); \ + LLVMSetOrdering(value, LLVMAtomicOrderingSequentiallyConsistent); \ + } while (0) + +#define BUILD_ATOMIC_STORE(align) \ + do { \ + LLVMValueRef res; \ + if (!(check_memory_alignment(comp_ctx, func_ctx, maddr, align))) { \ + goto fail; \ + } \ + if (!(res = LLVMBuildStore(comp_ctx->builder, value, maddr))) { \ + aot_set_last_error("llvm build store failed."); \ + goto fail; \ + } \ + LLVMSetAlignment(res, 1 << align); \ + LLVMSetVolatile(res, true); \ + LLVMSetOrdering(res, LLVMAtomicOrderingSequentiallyConsistent); \ + } while (0) +#endif + +bool +aot_compile_op_i32_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, bool sign, + bool atomic) +{ + LLVMValueRef maddr, value = NULL; + LLVMTypeRef data_type; + bool enable_segue = comp_ctx->enable_segue_i32_load; + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + enable_segue))) + return false; + + switch (bytes) { + case 4: + if (!enable_segue) + BUILD_PTR_CAST(INT32_PTR_TYPE); + else + BUILD_PTR_CAST(INT32_PTR_TYPE_GS); +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + BUILD_ATOMIC_LOAD(align, I32_TYPE); + else +#endif + BUILD_LOAD(I32_TYPE); + break; + case 2: + case 1: + if (bytes == 2) { + if (!enable_segue) + BUILD_PTR_CAST(INT16_PTR_TYPE); + else + BUILD_PTR_CAST(INT16_PTR_TYPE_GS); + data_type = INT16_TYPE; + } + else { + if (!enable_segue) + BUILD_PTR_CAST(INT8_PTR_TYPE); + else + BUILD_PTR_CAST(INT8_PTR_TYPE_GS); + data_type = INT8_TYPE; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) { + BUILD_ATOMIC_LOAD(align, data_type); + BUILD_ZERO_EXT(I32_TYPE); + } + else +#endif + { + BUILD_LOAD(data_type); + if (sign) + BUILD_SIGN_EXT(I32_TYPE); + else + BUILD_ZERO_EXT(I32_TYPE); + } + break; + default: + bh_assert(0); + break; + } + + PUSH_I32(value); + (void)data_type; + return true; +fail: + return false; +} + +bool +aot_compile_op_i64_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, bool sign, + bool atomic) +{ + LLVMValueRef maddr, value = NULL; + LLVMTypeRef data_type; + bool enable_segue = comp_ctx->enable_segue_i64_load; + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + enable_segue))) + return false; + + switch (bytes) { + case 8: + if (!enable_segue) + BUILD_PTR_CAST(INT64_PTR_TYPE); + else + BUILD_PTR_CAST(INT64_PTR_TYPE_GS); +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + BUILD_ATOMIC_LOAD(align, I64_TYPE); + else +#endif + BUILD_LOAD(I64_TYPE); + break; + case 4: + case 2: + case 1: + if (bytes == 4) { + if (!enable_segue) + BUILD_PTR_CAST(INT32_PTR_TYPE); + else + BUILD_PTR_CAST(INT32_PTR_TYPE_GS); + data_type = I32_TYPE; + } + else if (bytes == 2) { + if (!enable_segue) + BUILD_PTR_CAST(INT16_PTR_TYPE); + else + BUILD_PTR_CAST(INT16_PTR_TYPE_GS); + data_type = INT16_TYPE; + } + else { + if (!enable_segue) + BUILD_PTR_CAST(INT8_PTR_TYPE); + else + BUILD_PTR_CAST(INT8_PTR_TYPE_GS); + data_type = INT8_TYPE; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) { + BUILD_ATOMIC_LOAD(align, data_type); + BUILD_ZERO_EXT(I64_TYPE); + } + else +#endif + { + BUILD_LOAD(data_type); + if (sign) + BUILD_SIGN_EXT(I64_TYPE); + else + BUILD_ZERO_EXT(I64_TYPE); + } + break; + default: + bh_assert(0); + break; + } + + PUSH_I64(value); + (void)data_type; + return true; +fail: + return false; +} + +bool +aot_compile_op_f32_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset) +{ + LLVMValueRef maddr, value; + bool enable_segue = comp_ctx->enable_segue_f32_load; + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, 4, + enable_segue))) + return false; + + if (!enable_segue) + BUILD_PTR_CAST(F32_PTR_TYPE); + else + BUILD_PTR_CAST(F32_PTR_TYPE_GS); + BUILD_LOAD(F32_TYPE); + + PUSH_F32(value); + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset) +{ + LLVMValueRef maddr, value; + bool enable_segue = comp_ctx->enable_segue_f64_load; + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, 8, + enable_segue))) + return false; + + if (!enable_segue) + BUILD_PTR_CAST(F64_PTR_TYPE); + else + BUILD_PTR_CAST(F64_PTR_TYPE_GS); + BUILD_LOAD(F64_TYPE); + + PUSH_F64(value); + return true; +fail: + return false; +} + +bool +aot_compile_op_i32_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, bool atomic) +{ + LLVMValueRef maddr, value; + bool enable_segue = comp_ctx->enable_segue_i32_store; + + POP_I32(value); + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + enable_segue))) + return false; + + switch (bytes) { + case 4: + if (!enable_segue) + BUILD_PTR_CAST(INT32_PTR_TYPE); + else + BUILD_PTR_CAST(INT32_PTR_TYPE_GS); + break; + case 2: + if (!enable_segue) + BUILD_PTR_CAST(INT16_PTR_TYPE); + else + BUILD_PTR_CAST(INT16_PTR_TYPE_GS); + BUILD_TRUNC(value, INT16_TYPE); + break; + case 1: + if (!enable_segue) + BUILD_PTR_CAST(INT8_PTR_TYPE); + else + BUILD_PTR_CAST(INT8_PTR_TYPE_GS); + BUILD_TRUNC(value, INT8_TYPE); + break; + default: + bh_assert(0); + break; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + BUILD_ATOMIC_STORE(align); + else +#endif + BUILD_STORE(); + return true; +fail: + return false; +} + +bool +aot_compile_op_i64_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, bool atomic) +{ + LLVMValueRef maddr, value; + bool enable_segue = comp_ctx->enable_segue_i64_store; + + POP_I64(value); + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + enable_segue))) + return false; + + switch (bytes) { + case 8: + if (!enable_segue) + BUILD_PTR_CAST(INT64_PTR_TYPE); + else + BUILD_PTR_CAST(INT64_PTR_TYPE_GS); + break; + case 4: + if (!enable_segue) + BUILD_PTR_CAST(INT32_PTR_TYPE); + else + BUILD_PTR_CAST(INT32_PTR_TYPE_GS); + BUILD_TRUNC(value, I32_TYPE); + break; + case 2: + if (!enable_segue) + BUILD_PTR_CAST(INT16_PTR_TYPE); + else + BUILD_PTR_CAST(INT16_PTR_TYPE_GS); + BUILD_TRUNC(value, INT16_TYPE); + break; + case 1: + if (!enable_segue) + BUILD_PTR_CAST(INT8_PTR_TYPE); + else + BUILD_PTR_CAST(INT8_PTR_TYPE_GS); + BUILD_TRUNC(value, INT8_TYPE); + break; + default: + bh_assert(0); + break; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + BUILD_ATOMIC_STORE(align); + else +#endif + BUILD_STORE(); + return true; +fail: + return false; +} + +bool +aot_compile_op_f32_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset) +{ + LLVMValueRef maddr, value; + bool enable_segue = comp_ctx->enable_segue_f32_store; + + POP_F32(value); + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, 4, + enable_segue))) + return false; + + if (!enable_segue) + BUILD_PTR_CAST(F32_PTR_TYPE); + else + BUILD_PTR_CAST(F32_PTR_TYPE_GS); + BUILD_STORE(); + return true; +fail: + return false; +} + +bool +aot_compile_op_f64_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset) +{ + LLVMValueRef maddr, value; + bool enable_segue = comp_ctx->enable_segue_f64_store; + + POP_F64(value); + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, 8, + enable_segue))) + return false; + + if (!enable_segue) + BUILD_PTR_CAST(F64_PTR_TYPE); + else + BUILD_PTR_CAST(F64_PTR_TYPE_GS); + BUILD_STORE(); + return true; +fail: + return false; +} + +static LLVMValueRef +get_memory_curr_page_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef mem_size; + + if (func_ctx->mem_space_unchanged) { + mem_size = func_ctx->mem_info[0].mem_cur_page_count_addr; + } + else { + if (!(mem_size = LLVMBuildLoad2( + comp_ctx->builder, I32_TYPE, + func_ctx->mem_info[0].mem_cur_page_count_addr, "mem_size"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + } + + return mem_size; +fail: + return NULL; +} + +bool +aot_compile_op_memory_size(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef mem_size = get_memory_curr_page_count(comp_ctx, func_ctx); + + if (mem_size) + PUSH_I32(mem_size); + return mem_size ? true : false; +fail: + return false; +} + +bool +aot_compile_op_memory_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef mem_size = get_memory_curr_page_count(comp_ctx, func_ctx); + LLVMValueRef delta, param_values[2], ret_value, func, value; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + int32 func_index; + + if (!mem_size) + return false; + + POP_I32(delta); + + /* Function type of aot_enlarge_memory() */ + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = INT8_TYPE; + + if (!(func_type = LLVMFunctionType(ret_type, param_types, 2, false))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + if (comp_ctx->is_jit_mode) { + /* JIT mode, call the function directly */ + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("llvm add pointer type failed."); + return false; + } + if (!(value = I64_CONST((uint64)(uintptr_t)wasm_enlarge_memory)) + || !(func = LLVMConstIntToPtr(value, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else if (comp_ctx->is_indirect_mode) { + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + func_index = + aot_get_native_symbol_index(comp_ctx, "aot_enlarge_memory"); + if (func_index < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + char *func_name = "aot_enlarge_memory"; + /* AOT mode, delcare the function */ + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) + && !(func = + LLVMAddFunction(func_ctx->module, func_name, func_type))) { + aot_set_last_error("llvm add function failed."); + return false; + } + } + + /* Call function aot_enlarge_memory() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = delta; + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "call"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + BUILD_ICMP(LLVMIntUGT, ret_value, I8_ZERO, ret_value, "mem_grow_ret"); + + /* ret_value = ret_value == true ? delta : pre_page_count */ + if (!(ret_value = LLVMBuildSelect(comp_ctx->builder, ret_value, mem_size, + I32_NEG_ONE, "mem_grow_ret"))) { + aot_set_last_error("llvm build select failed."); + return false; + } + + PUSH_I32(ret_value); + return true; +fail: + return false; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 +LLVMValueRef +check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef offset, LLVMValueRef bytes) +{ + LLVMValueRef maddr, max_addr, cmp; + LLVMValueRef mem_base_addr; + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMBasicBlockRef check_succ; + LLVMValueRef mem_size; + + /* Get memory base address and memory data size */ +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory = + comp_ctx->comp_data->memories[0].memory_flags & 0x02; + + if (func_ctx->mem_space_unchanged || is_shared_memory) { +#else + if (func_ctx->mem_space_unchanged) { +#endif + mem_base_addr = func_ctx->mem_info[0].mem_base_addr; + } + else { + if (!(mem_base_addr = LLVMBuildLoad2( + comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->mem_info[0].mem_base_addr, "mem_base"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + } + + /* + * Note: not throw the integer-overflow-exception here since it must + * have been thrown when converting float to integer before + */ + /* return addres directly if constant offset and inside memory space */ + if (LLVMIsEfficientConstInt(offset) && LLVMIsEfficientConstInt(bytes)) { + uint64 mem_offset = (uint64)LLVMConstIntGetZExtValue(offset); + uint64 mem_len = (uint64)LLVMConstIntGetZExtValue(bytes); + uint32 num_bytes_per_page = + comp_ctx->comp_data->memories[0].num_bytes_per_page; + uint32 init_page_count = + comp_ctx->comp_data->memories[0].mem_init_page_count; + uint64 mem_data_size = (uint64)num_bytes_per_page * init_page_count; + if (mem_data_size > 0 && mem_offset + mem_len <= mem_data_size) { + /* inside memory space */ + /* maddr = mem_base_addr + moffset */ + if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + mem_base_addr, &offset, 1, + "maddr"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + return maddr; + } + } + + if (func_ctx->mem_space_unchanged) { + mem_size = func_ctx->mem_info[0].mem_data_size_addr; + } + else { + if (!(mem_size = LLVMBuildLoad2( + comp_ctx->builder, I64_TYPE, + func_ctx->mem_info[0].mem_data_size_addr, "mem_size"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + } + + ADD_BASIC_BLOCK(check_succ, "check_succ"); + LLVMMoveBasicBlockAfter(check_succ, block_curr); + + offset = + LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); + bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); + + BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); + BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; + } + + /* maddr = mem_base_addr + offset */ + if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + mem_base_addr, &offset, 1, "maddr"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + return maddr; +fail: + return NULL; +} +#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 or WASM_ENABLE_STRINGREF != 0 */ + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +aot_compile_op_memory_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 seg_index) +{ + LLVMValueRef seg, offset, dst, len, param_values[5], ret_value, func, value; + LLVMTypeRef param_types[5], ret_type, func_type, func_ptr_type; + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMBasicBlockRef mem_init_fail, init_success; + + seg = I32_CONST(seg_index); + + POP_I32(len); + POP_I32(offset); + POP_I32(dst); + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = I32_TYPE; + param_types[4] = I32_TYPE; + ret_type = INT8_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_memory_init, 5); + else + GET_AOT_FUNCTION(aot_memory_init, 5); + + /* Call function aot_memory_init() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = seg; + param_values[2] = offset; + param_values[3] = len; + param_values[4] = dst; + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 5, "call"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + BUILD_ICMP(LLVMIntUGT, ret_value, I8_ZERO, ret_value, "mem_init_ret"); + + ADD_BASIC_BLOCK(mem_init_fail, "mem_init_fail"); + ADD_BASIC_BLOCK(init_success, "init_success"); + + LLVMMoveBasicBlockAfter(mem_init_fail, block_curr); + LLVMMoveBasicBlockAfter(init_success, block_curr); + + if (!LLVMBuildCondBr(comp_ctx->builder, ret_value, init_success, + mem_init_fail)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* If memory.init failed, return this function + so the runtime can catch the exception */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, mem_init_fail); + if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, init_success); + + return true; +fail: + return false; +} + +bool +aot_compile_op_data_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 seg_index) +{ + LLVMValueRef seg, param_values[2], ret_value, func, value; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + seg = I32_CONST(seg_index); + CHECK_LLVM_CONST(seg); + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = INT8_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_data_drop, 2); + else + GET_AOT_FUNCTION(aot_data_drop, 2); + + /* Call function aot_data_drop() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = seg; + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "call"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef src, dst, src_addr, dst_addr, len, res; + bool call_aot_memmove = false; + + POP_I32(len); + POP_I32(src); + POP_I32(dst); + + if (!(src_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, src, len))) + return false; + + if (!(dst_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, dst, len))) + return false; + + call_aot_memmove = comp_ctx->is_indirect_mode || comp_ctx->is_jit_mode; + if (call_aot_memmove) { + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + LLVMValueRef func, params[3]; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function pointer type failed."); + return false; + } + + if (comp_ctx->is_jit_mode) { + if (!(func = I64_CONST((uint64)(uintptr_t)aot_memmove)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else { + int32 func_index; + func_index = aot_get_native_symbol_index(comp_ctx, "memmove"); + if (func_index < 0) { + return false; + } + if (!(func = + aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + + params[0] = dst_addr; + params[1] = src_addr; + params[2] = len; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, params, + 3, "call_memmove"))) { + aot_set_last_error("llvm build memmove failed."); + return false; + } + } + else { + if (!(res = LLVMBuildMemMove(comp_ctx->builder, dst_addr, 1, src_addr, + 1, len))) { + aot_set_last_error("llvm build memmove failed."); + return false; + } + } + + return true; +fail: + return false; +} + +static void * +jit_memset(void *s, int c, size_t n) +{ + return memset(s, c, n); +} + +bool +aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef val, dst, dst_addr, len, res; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + LLVMValueRef func, params[3]; + + POP_I32(len); + POP_I32(val); + POP_I32(dst); + + if (!(dst_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, dst, len))) + return false; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function pointer type failed."); + return false; + } + + if (comp_ctx->is_jit_mode) { + if (!(func = I64_CONST((uint64)(uintptr_t)jit_memset)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + func_index = aot_get_native_symbol_index(comp_ctx, "memset"); + if (func_index < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + if (!(func = LLVMGetNamedFunction(func_ctx->module, "memset")) + && !(func = + LLVMAddFunction(func_ctx->module, "memset", func_type))) { + aot_set_last_error("llvm add function failed."); + return false; + } + } + + params[0] = dst_addr; + params[1] = val; + params[2] = len; + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, params, 3, + "call_memset"))) { + aot_set_last_error("llvm build memset failed."); + return false; + } + + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_BULK_MEMORY */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 +bool +aot_compile_op_atomic_rmw(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 atomic_op, uint8 op_type, uint32 align, + uint32 offset, uint32 bytes) +{ + LLVMValueRef maddr, value, result; + bool enable_segue = (op_type == VALUE_TYPE_I32) + ? comp_ctx->enable_segue_i32_load + && comp_ctx->enable_segue_i32_store + : comp_ctx->enable_segue_i64_load + && comp_ctx->enable_segue_i64_store; + + if (op_type == VALUE_TYPE_I32) + POP_I32(value); + else + POP_I64(value); + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + enable_segue))) + return false; + + if (!check_memory_alignment(comp_ctx, func_ctx, maddr, align)) + return false; + + switch (bytes) { + case 8: + if (!enable_segue) + BUILD_PTR_CAST(INT64_PTR_TYPE); + else + BUILD_PTR_CAST(INT64_PTR_TYPE_GS); + break; + case 4: + if (!enable_segue) + BUILD_PTR_CAST(INT32_PTR_TYPE); + else + BUILD_PTR_CAST(INT32_PTR_TYPE_GS); + if (op_type == VALUE_TYPE_I64) + BUILD_TRUNC(value, I32_TYPE); + break; + case 2: + if (!enable_segue) + BUILD_PTR_CAST(INT16_PTR_TYPE); + else + BUILD_PTR_CAST(INT16_PTR_TYPE_GS); + BUILD_TRUNC(value, INT16_TYPE); + break; + case 1: + if (!enable_segue) + BUILD_PTR_CAST(INT8_PTR_TYPE); + else + BUILD_PTR_CAST(INT8_PTR_TYPE_GS); + BUILD_TRUNC(value, INT8_TYPE); + break; + default: + bh_assert(0); + break; + } + + if (!(result = LLVMBuildAtomicRMW( + comp_ctx->builder, atomic_op, maddr, value, + LLVMAtomicOrderingSequentiallyConsistent, false))) { + goto fail; + } + + LLVMSetVolatile(result, true); + + if (op_type == VALUE_TYPE_I32) { + if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, + "result_i32"))) { + goto fail; + } + PUSH_I32(result); + } + else { + if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I64_TYPE, + "result_i64"))) { + goto fail; + } + PUSH_I64(result); + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_atomic_cmpxchg(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 op_type, + uint32 align, uint32 offset, uint32 bytes) +{ + LLVMValueRef maddr, value, expect, result; + bool enable_segue = (op_type == VALUE_TYPE_I32) + ? comp_ctx->enable_segue_i32_load + && comp_ctx->enable_segue_i32_store + : comp_ctx->enable_segue_i64_load + && comp_ctx->enable_segue_i64_store; + + if (op_type == VALUE_TYPE_I32) { + POP_I32(value); + POP_I32(expect); + } + else { + POP_I64(value); + POP_I64(expect); + } + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + enable_segue))) + return false; + + if (!check_memory_alignment(comp_ctx, func_ctx, maddr, align)) + return false; + + switch (bytes) { + case 8: + if (!enable_segue) + BUILD_PTR_CAST(INT64_PTR_TYPE); + else + BUILD_PTR_CAST(INT64_PTR_TYPE_GS); + break; + case 4: + if (!enable_segue) + BUILD_PTR_CAST(INT32_PTR_TYPE); + else + BUILD_PTR_CAST(INT32_PTR_TYPE_GS); + if (op_type == VALUE_TYPE_I64) { + BUILD_TRUNC(value, I32_TYPE); + BUILD_TRUNC(expect, I32_TYPE); + } + break; + case 2: + if (!enable_segue) + BUILD_PTR_CAST(INT16_PTR_TYPE); + else + BUILD_PTR_CAST(INT16_PTR_TYPE_GS); + BUILD_TRUNC(value, INT16_TYPE); + BUILD_TRUNC(expect, INT16_TYPE); + break; + case 1: + if (!enable_segue) + BUILD_PTR_CAST(INT8_PTR_TYPE); + else + BUILD_PTR_CAST(INT8_PTR_TYPE_GS); + BUILD_TRUNC(value, INT8_TYPE); + BUILD_TRUNC(expect, INT8_TYPE); + break; + default: + bh_assert(0); + break; + } + + if (!(result = LLVMBuildAtomicCmpXchg( + comp_ctx->builder, maddr, expect, value, + LLVMAtomicOrderingSequentiallyConsistent, + LLVMAtomicOrderingSequentiallyConsistent, false))) { + goto fail; + } + + LLVMSetVolatile(result, true); + + /* CmpXchg return {i32, i1} structure, + we need to extrack the previous_value from the structure */ + if (!(result = LLVMBuildExtractValue(comp_ctx->builder, result, 0, + "previous_value"))) { + goto fail; + } + + if (op_type == VALUE_TYPE_I32) { + if (LLVMTypeOf(result) != I32_TYPE) { + if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, + "result_i32"))) { + goto fail; + } + } + PUSH_I32(result); + } + else { + if (LLVMTypeOf(result) != I64_TYPE) { + if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I64_TYPE, + "result_i64"))) { + goto fail; + } + } + PUSH_I64(result); + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_atomic_wait(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 op_type, uint32 align, uint32 offset, + uint32 bytes) +{ + LLVMValueRef maddr, value, timeout, expect, cmp; + LLVMValueRef param_values[5], ret_value, func, is_wait64; + LLVMTypeRef param_types[5], ret_type, func_type, func_ptr_type; + LLVMBasicBlockRef wait_fail, wait_success; + LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + + POP_I64(timeout); + if (op_type == VALUE_TYPE_I32) { + POP_I32(expect); + is_wait64 = I8_CONST(false); + if (!(expect = LLVMBuildZExt(comp_ctx->builder, expect, I64_TYPE, + "expect_i64"))) { + goto fail; + } + } + else { + POP_I64(expect); + is_wait64 = I8_CONST(true); + } + + CHECK_LLVM_CONST(is_wait64); + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + false))) + return false; + + if (!check_memory_alignment(comp_ctx, func_ctx, maddr, align)) + return false; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + param_types[2] = I64_TYPE; + param_types[3] = I64_TYPE; + param_types[4] = INT8_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_atomic_wait, 5); + + /* Call function wasm_runtime_atomic_wait() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = maddr; + param_values[2] = expect; + param_values[3] = timeout; + param_values[4] = is_wait64; + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 5, "call"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + BUILD_ICMP(LLVMIntNE, ret_value, I32_NEG_ONE, cmp, "atomic_wait_ret"); + + ADD_BASIC_BLOCK(wait_fail, "atomic_wait_fail"); + ADD_BASIC_BLOCK(wait_success, "wait_success"); + + LLVMMoveBasicBlockAfter(wait_fail, block_curr); + LLVMMoveBasicBlockAfter(wait_success, block_curr); + + if (!LLVMBuildCondBr(comp_ctx->builder, cmp, wait_success, wait_fail)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* If atomic wait failed, return this function + so the runtime can catch the exception */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, wait_fail); + if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, wait_success); + + PUSH_I32(ret_value); + + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx, false)) + return false; + } + + return true; +fail: + return false; +} + +bool +aot_compiler_op_atomic_notify(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 align, + uint32 offset, uint32 bytes) +{ + LLVMValueRef maddr, value, count; + LLVMValueRef param_values[3], ret_value, func; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + + POP_I32(count); + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, bytes, + false))) + return false; + + if (!check_memory_alignment(comp_ctx, func_ctx, maddr, align)) + return false; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + param_types[2] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_atomic_notify, 3); + + /* Call function wasm_runtime_atomic_notify() */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = maddr; + param_values[2] = count; + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 3, "call"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + PUSH_I32(ret_value); + + return true; +fail: + return false; +} + +bool +aot_compiler_op_atomic_fence(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return LLVMBuildFence(comp_ctx->builder, + LLVMAtomicOrderingSequentiallyConsistent, false, "") + ? true + : false; +} + +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_memory.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_memory.h new file mode 100644 index 0000000..e174aa3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_memory.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_MEMORY_H_ +#define _AOT_EMIT_MEMORY_H_ + +#include "aot_compiler.h" +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "wasm_shared_memory.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_i32_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, bool sign, + bool atomic); + +bool +aot_compile_op_i64_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, bool sign, + bool atomic); + +bool +aot_compile_op_f32_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset); + +bool +aot_compile_op_f64_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset); + +bool +aot_compile_op_i32_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, + bool atomic); + +bool +aot_compile_op_i64_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset, uint32 bytes, + bool atomic); + +bool +aot_compile_op_f32_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset); + +bool +aot_compile_op_f64_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset); + +LLVMValueRef +aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 offset, uint32 bytes, bool enable_segue); + +bool +aot_compile_op_memory_size(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_memory_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +check_memory_alignment(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef addr, uint32 align); + +LLVMValueRef +check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef offset, LLVMValueRef bytes); + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +aot_compile_op_memory_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 seg_index); + +bool +aot_compile_op_data_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 seg_index); + +bool +aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +#endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 +bool +aot_compile_op_atomic_rmw(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 atomic_op, uint8 op_type, uint32 align, + uint32 offset, uint32 bytes); + +bool +aot_compile_op_atomic_cmpxchg(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 op_type, + uint32 align, uint32 offset, uint32 bytes); + +bool +aot_compile_op_atomic_wait(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 op_type, uint32 align, uint32 offset, + uint32 bytes); + +bool +aot_compiler_op_atomic_notify(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 align, + uint32 offset, uint32 bytes); + +bool +aot_compiler_op_atomic_fence(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_MEMORY_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_numberic.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_numberic.c new file mode 100644 index 0000000..8b6ec02 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_numberic.c @@ -0,0 +1,1250 @@ +/* + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_numberic.h" +#include "aot_emit_exception.h" +#include "aot_emit_control.h" +#include "../aot/aot_runtime.h" +#include "../aot/aot_intrinsic.h" + +#include + +#define LLVM_BUILD_ICMP(op, left, right, res, name) \ + do { \ + if (!(res = \ + LLVMBuildICmp(comp_ctx->builder, op, left, right, name))) { \ + aot_set_last_error("llvm build " name " fail."); \ + return false; \ + } \ + } while (0) + +#define LLVM_BUILD_OP(Op, left, right, res, name, err_ret) \ + do { \ + if (!(res = LLVMBuild##Op(comp_ctx->builder, left, right, name))) { \ + aot_set_last_error("llvm build " #name " fail."); \ + return err_ret; \ + } \ + } while (0) + +#define LLVM_BUILD_OP_OR_INTRINSIC(Op, left, right, res, intrinsic, name, \ + err_ret) \ + do { \ + if (comp_ctx->disable_llvm_intrinsics \ + && aot_intrinsic_check_capability(comp_ctx, intrinsic)) { \ + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, \ + param_types[0], param_types, 2, \ + left, right); \ + } \ + else { \ + LLVM_BUILD_OP(Op, left, right, res, name, false); \ + } \ + } while (0) + +#define ADD_BASIC_BLOCK(block, name) \ + do { \ + if (!(block = LLVMAppendBasicBlockInContext(comp_ctx->context, \ + func_ctx->func, name))) { \ + aot_set_last_error("llvm add basic block failed."); \ + goto fail; \ + } \ + \ + LLVMMoveBasicBlockAfter(block, LLVMGetInsertBlock(comp_ctx->builder)); \ + } while (0) + +#if LLVM_VERSION_NUMBER >= 12 +#define IS_CONST_ZERO(val) \ + (LLVMIsEfficientConstInt(val) \ + && ((is_i32 && (int32)LLVMConstIntGetZExtValue(val) == 0) \ + || (!is_i32 && (int64)LLVMConstIntGetSExtValue(val) == 0))) +#else +#define IS_CONST_ZERO(val) \ + (LLVMIsEfficientConstInt(val) \ + && ((is_i32 && (int32)LLVMConstIntGetZExtValue(val) == 0) \ + || (!is_i32 && (int64)LLVMConstIntGetSExtValue(val) == 0))) +#endif + +#define CHECK_INT_OVERFLOW(type) \ + do { \ + LLVMValueRef cmp_min_int, cmp_neg_one; \ + LLVM_BUILD_ICMP(LLVMIntEQ, left, type##_MIN, cmp_min_int, \ + "cmp_min_int"); \ + LLVM_BUILD_ICMP(LLVMIntEQ, right, type##_NEG_ONE, cmp_neg_one, \ + "cmp_neg_one"); \ + LLVM_BUILD_OP(And, cmp_min_int, cmp_neg_one, overflow, "overflow", \ + false); \ + } while (0) + +#define PUSH_INT(v) \ + do { \ + if (is_i32) \ + PUSH_I32(v); \ + else \ + PUSH_I64(v); \ + } while (0) + +#define POP_INT(v) \ + do { \ + if (is_i32) \ + POP_I32(v); \ + else \ + POP_I64(v); \ + } while (0) + +#define PUSH_FLOAT(v) \ + do { \ + if (is_f32) \ + PUSH_F32(v); \ + else \ + PUSH_F64(v); \ + } while (0) + +#define POP_FLOAT(v) \ + do { \ + if (is_f32) \ + POP_F32(v); \ + else \ + POP_F64(v); \ + } while (0) + +#define DEF_INT_UNARY_OP(op, err) \ + do { \ + LLVMValueRef res, operand; \ + POP_INT(operand); \ + if (!(res = op)) { \ + if (err) \ + aot_set_last_error(err); \ + return false; \ + } \ + PUSH_INT(res); \ + } while (0) + +#define DEF_INT_BINARY_OP(op, err) \ + do { \ + LLVMValueRef res, left, right; \ + POP_INT(right); \ + POP_INT(left); \ + if (!(res = op)) { \ + if (err) \ + aot_set_last_error(err); \ + return false; \ + } \ + PUSH_INT(res); \ + } while (0) + +#define DEF_FP_UNARY_OP(op, err) \ + do { \ + LLVMValueRef res, operand; \ + POP_FLOAT(operand); \ + if (!(res = op)) { \ + if (err) \ + aot_set_last_error(err); \ + return false; \ + } \ + PUSH_FLOAT(res); \ + } while (0) + +#define DEF_FP_BINARY_OP(op, err) \ + do { \ + LLVMValueRef res, left, right; \ + POP_FLOAT(right); \ + POP_FLOAT(left); \ + if (!(res = op)) { \ + if (err) \ + aot_set_last_error(err); \ + return false; \ + } \ + PUSH_FLOAT(res); \ + } while (0) + +#define SHIFT_COUNT_MASK \ + do { \ + /* LLVM has undefined behavior if shift count is greater than \ + * bits count while Webassembly spec requires the shift count \ + * be wrapped. \ + */ \ + LLVMValueRef shift_count_mask, bits_minus_one; \ + bits_minus_one = is_i32 ? I32_31 : I64_63; \ + LLVM_BUILD_OP(And, right, bits_minus_one, shift_count_mask, \ + "shift_count_mask", NULL); \ + right = shift_count_mask; \ + } while (0) + +/* Call llvm constrained floating-point intrinsic */ +static LLVMValueRef +call_llvm_float_experimental_constrained_intrinsic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_f32, + const char *intrinsic, ...) +{ + va_list param_value_list; + LLVMValueRef ret; + LLVMTypeRef param_types[4], ret_type = is_f32 ? F32_TYPE : F64_TYPE; + int param_count = (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, intrinsic)) + ? 2 + : 4; + + param_types[0] = param_types[1] = ret_type; + param_types[2] = param_types[3] = MD_TYPE; + + va_start(param_value_list, intrinsic); + + ret = aot_call_llvm_intrinsic_v(comp_ctx, func_ctx, intrinsic, ret_type, + param_types, param_count, param_value_list); + + va_end(param_value_list); + + return ret; +} + +/* Call llvm constrained libm-equivalent intrinsic */ +static LLVMValueRef +call_llvm_libm_experimental_constrained_intrinsic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_f32, + const char *intrinsic, ...) +{ + va_list param_value_list; + LLVMValueRef ret; + LLVMTypeRef param_types[3], ret_type = is_f32 ? F32_TYPE : F64_TYPE; + + param_types[0] = ret_type; + param_types[1] = param_types[2] = MD_TYPE; + + va_start(param_value_list, intrinsic); + + ret = aot_call_llvm_intrinsic_v(comp_ctx, func_ctx, intrinsic, ret_type, + param_types, 3, param_value_list); + + va_end(param_value_list); + + return ret; +} + +static LLVMValueRef +compile_op_float_min_max(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_f32, LLVMValueRef left, LLVMValueRef right, + bool is_min) +{ + LLVMTypeRef param_types[2], ret_type = is_f32 ? F32_TYPE : F64_TYPE, + int_type = is_f32 ? I32_TYPE : I64_TYPE; + LLVMValueRef cmp, is_eq, is_nan, ret, left_int, right_int, tmp, + nan = LLVMConstRealOfString(ret_type, "NaN"); + char *intrinsic = is_min ? (is_f32 ? "llvm.minnum.f32" : "llvm.minnum.f64") + : (is_f32 ? "llvm.maxnum.f32" : "llvm.maxnum.f64"); + CHECK_LLVM_CONST(nan); + + param_types[0] = param_types[1] = ret_type; + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, + is_f32 ? "f32_cmp" : "f64_cmp")) { + LLVMTypeRef param_types_intrinsic[3]; + LLVMValueRef opcond = LLVMConstInt(I32_TYPE, FLOAT_UNO, true); + param_types_intrinsic[0] = I32_TYPE; + param_types_intrinsic[1] = is_f32 ? F32_TYPE : F64_TYPE; + param_types_intrinsic[2] = param_types_intrinsic[1]; + is_nan = aot_call_llvm_intrinsic( + comp_ctx, func_ctx, is_f32 ? "f32_cmp" : "f64_cmp", I32_TYPE, + param_types_intrinsic, 3, opcond, left, right); + + opcond = LLVMConstInt(I32_TYPE, FLOAT_EQ, true); + is_eq = aot_call_llvm_intrinsic( + comp_ctx, func_ctx, is_f32 ? "f32_cmp" : "f64_cmp", I32_TYPE, + param_types_intrinsic, 3, opcond, left, right); + + if (!is_nan || !is_eq) { + return NULL; + } + + if (!(is_nan = LLVMBuildIntCast(comp_ctx->builder, is_nan, INT1_TYPE, + "bit_cast_is_nan"))) { + aot_set_last_error("llvm build is_nan bit cast fail."); + return NULL; + } + + if (!(is_eq = LLVMBuildIntCast(comp_ctx->builder, is_eq, INT1_TYPE, + "bit_cast_is_eq"))) { + aot_set_last_error("llvm build is_eq bit cast fail."); + return NULL; + } + } + else if (!(is_nan = LLVMBuildFCmp(comp_ctx->builder, LLVMRealUNO, left, + right, "is_nan")) + || !(is_eq = LLVMBuildFCmp(comp_ctx->builder, LLVMRealOEQ, left, + right, "is_eq"))) { + aot_set_last_error("llvm build fcmp fail."); + return NULL; + } + + /* If left and right are equal, they may be zero with different sign. + Webassembly spec assert -0 < +0. So do a bitwise here. */ + if (!(left_int = + LLVMBuildBitCast(comp_ctx->builder, left, int_type, "left_int")) + || !(right_int = LLVMBuildBitCast(comp_ctx->builder, right, int_type, + "right_int"))) { + aot_set_last_error("llvm build bitcast fail."); + return NULL; + } + + if (is_min) + LLVM_BUILD_OP_OR_INTRINSIC(Or, left_int, right_int, tmp, + is_f32 ? "i32.or" : "i64.or", "tmp_int", + false); + else + LLVM_BUILD_OP_OR_INTRINSIC(And, left_int, right_int, tmp, + is_f32 ? "i32.and" : "i64.and", "tmp_int", + false); + + if (!(tmp = LLVMBuildBitCast(comp_ctx->builder, tmp, ret_type, "tmp"))) { + aot_set_last_error("llvm build bitcast fail."); + return NULL; + } + + if (!(cmp = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, ret_type, + param_types, 2, left, right))) + return NULL; + + /* The result of XIP intrinsic is 0 or 1, should return it directly */ + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, + is_f32 ? "f32_cmp" : "f64_cmp")) { + return cmp; + } + + if (!(cmp = LLVMBuildSelect(comp_ctx->builder, is_eq, tmp, cmp, "cmp"))) { + aot_set_last_error("llvm build select fail."); + return NULL; + } + + if (!(ret = LLVMBuildSelect(comp_ctx->builder, is_nan, nan, cmp, + is_min ? "min" : "max"))) { + aot_set_last_error("llvm build select fail."); + return NULL; + } + + return ret; +fail: + return NULL; +} + +typedef enum BitCountType { + CLZ32 = 0, + CLZ64, + CTZ32, + CTZ64, + POP_CNT32, + POP_CNT64 +} BitCountType; + +/* clang-format off */ +static char *bit_cnt_llvm_intrinsic[] = { + "llvm.ctlz.i32", + "llvm.ctlz.i64", + "llvm.cttz.i32", + "llvm.cttz.i64", + "llvm.ctpop.i32", + "llvm.ctpop.i64", +}; +/* clang-format on */ + +static bool +aot_compile_int_bit_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + BitCountType type, bool is_i32) +{ + LLVMValueRef zero_undef; + LLVMTypeRef ret_type, param_types[2]; + + param_types[0] = ret_type = is_i32 ? I32_TYPE : I64_TYPE; + param_types[1] = LLVMInt1TypeInContext(comp_ctx->context); + + zero_undef = LLVMConstInt(param_types[1], false, true); + CHECK_LLVM_CONST(zero_undef); + + /* Call the LLVM intrinsic function */ + if (type < POP_CNT32) + DEF_INT_UNARY_OP(aot_call_llvm_intrinsic( + comp_ctx, func_ctx, bit_cnt_llvm_intrinsic[type], + ret_type, param_types, 2, operand, zero_undef), + NULL); + else + DEF_INT_UNARY_OP(aot_call_llvm_intrinsic( + comp_ctx, func_ctx, bit_cnt_llvm_intrinsic[type], + ret_type, param_types, 1, operand), + NULL); + + return true; + +fail: + return false; +} + +static bool +compile_rems(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef left, LLVMValueRef right, LLVMValueRef overflow_cond, + bool is_i32) +{ + LLVMValueRef phi, no_overflow_value, zero = is_i32 ? I32_ZERO : I64_ZERO; + LLVMBasicBlockRef block_curr, no_overflow_block, rems_end_block; + LLVMTypeRef param_types[2]; + + param_types[1] = param_types[0] = is_i32 ? I32_TYPE : I64_TYPE; + + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + + /* Add 2 blocks: no_overflow_block and rems_end block */ + ADD_BASIC_BLOCK(rems_end_block, "rems_end"); + ADD_BASIC_BLOCK(no_overflow_block, "rems_no_overflow"); + + /* Create condition br */ + if (!LLVMBuildCondBr(comp_ctx->builder, overflow_cond, rems_end_block, + no_overflow_block)) { + aot_set_last_error("llvm build cond br failed."); + return false; + } + + /* Translate no_overflow_block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, no_overflow_block); + + LLVM_BUILD_OP_OR_INTRINSIC(SRem, left, right, no_overflow_value, + is_i32 ? "i32.rem_s" : "i64.rem_s", "rem_s", + false); + + /* Jump to rems_end block */ + if (!LLVMBuildBr(comp_ctx->builder, rems_end_block)) { + aot_set_last_error("llvm build br failed."); + return false; + } + + /* Translate rems_end_block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, rems_end_block); + + /* Create result phi */ + if (!(phi = LLVMBuildPhi(comp_ctx->builder, is_i32 ? I32_TYPE : I64_TYPE, + "rems_result_phi"))) { + aot_set_last_error("llvm build phi failed."); + return false; + } + + /* Add phi incoming values */ + LLVMAddIncoming(phi, &no_overflow_value, &no_overflow_block, 1); + LLVMAddIncoming(phi, &zero, &block_curr, 1); + + if (is_i32) + PUSH_I32(phi); + else + PUSH_I64(phi); + + return true; + +fail: + return false; +} + +static bool +compile_int_div(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntArithmetic arith_op, bool is_i32, uint8 **p_frame_ip) +{ + LLVMValueRef left, right, cmp_div_zero, overflow, res; + LLVMBasicBlockRef check_div_zero_succ, check_overflow_succ; + LLVMTypeRef param_types[2]; + const char *intrinsic = NULL; + + param_types[1] = param_types[0] = is_i32 ? I32_TYPE : I64_TYPE; + + bh_assert(arith_op == INT_DIV_S || arith_op == INT_DIV_U + || arith_op == INT_REM_S || arith_op == INT_REM_U); + + POP_INT(right); + POP_INT(left); + + if (LLVMIsUndef(right) || LLVMIsUndef(left) +#if LLVM_VERSION_NUMBER >= 12 + || LLVMIsPoison(right) || LLVMIsPoison(left) +#endif + ) { + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, + false, NULL, NULL))) { + goto fail; + } + return aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); + } + + if (LLVMIsEfficientConstInt(right)) { + int64 right_val = (int64)LLVMConstIntGetSExtValue(right); + switch (right_val) { + case 0: + /* Directly throw exception if divided by zero */ + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_INTEGER_DIVIDE_BY_ZERO, false, + NULL, NULL))) + goto fail; + + return aot_handle_next_reachable_block(comp_ctx, func_ctx, + p_frame_ip); + case 1: + if (arith_op == INT_DIV_S || arith_op == INT_DIV_U) + PUSH_INT(left); + else + PUSH_INT(is_i32 ? I32_ZERO : I64_ZERO); + return true; + case -1: + if (arith_op == INT_DIV_S) { + LLVM_BUILD_ICMP(LLVMIntEQ, left, is_i32 ? I32_MIN : I64_MIN, + overflow, "overflow"); + ADD_BASIC_BLOCK(check_overflow_succ, + "check_overflow_success"); + + /* Throw conditional exception if overflow */ + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_INTEGER_OVERFLOW, true, + overflow, check_overflow_succ))) + goto fail; + + /* Push -(left) to stack */ + if (!(res = LLVMBuildNeg(comp_ctx->builder, left, "neg"))) { + aot_set_last_error("llvm build neg fail."); + return false; + } + PUSH_INT(res); + return true; + } + else if (arith_op == INT_REM_S) { + PUSH_INT(is_i32 ? I32_ZERO : I64_ZERO); + return true; + } + else { + /* fall to default */ + goto handle_default; + } + handle_default: + default: + /* Build div */ + switch (arith_op) { + case INT_DIV_S: + LLVM_BUILD_OP_OR_INTRINSIC( + SDiv, left, right, res, + is_i32 ? "i32.div_s" : "i64.div_s", "div_s", false); + break; + case INT_DIV_U: + LLVM_BUILD_OP_OR_INTRINSIC( + UDiv, left, right, res, + is_i32 ? "i32.div_u" : "i64.div_u", "div_u", false); + break; + case INT_REM_S: + LLVM_BUILD_OP_OR_INTRINSIC( + SRem, left, right, res, + is_i32 ? "i32.rem_s" : "i64.rem_s", "rem_s", false); + break; + case INT_REM_U: + LLVM_BUILD_OP_OR_INTRINSIC( + URem, left, right, res, + is_i32 ? "i32.rem_u" : "i64.rem_u", "rem_u", false); + break; + default: + bh_assert(0); + return false; + } + + PUSH_INT(res); + return true; + } + } + else { + /* Check divied by zero */ + LLVM_BUILD_ICMP(LLVMIntEQ, right, is_i32 ? I32_ZERO : I64_ZERO, + cmp_div_zero, "cmp_div_zero"); + ADD_BASIC_BLOCK(check_div_zero_succ, "check_div_zero_success"); + + /* Throw conditional exception if divided by zero */ + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_INTEGER_DIVIDE_BY_ZERO, true, + cmp_div_zero, check_div_zero_succ))) + goto fail; + + switch (arith_op) { + case INT_DIV_S: + /* Check integer overflow */ + if (is_i32) + CHECK_INT_OVERFLOW(I32); + else + CHECK_INT_OVERFLOW(I64); + + ADD_BASIC_BLOCK(check_overflow_succ, "check_overflow_success"); + + /* Throw conditional exception if integer overflow */ + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_INTEGER_OVERFLOW, true, overflow, + check_overflow_succ))) + goto fail; + + LLVM_BUILD_OP_OR_INTRINSIC(SDiv, left, right, res, + is_i32 ? "i32.div_s" : "i64.div_s", + "div_s", false); + PUSH_INT(res); + return true; + case INT_DIV_U: + intrinsic = is_i32 ? "i32.div_u" : "i64.div_u"; + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, intrinsic)) { + res = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, + param_types[0], param_types, + 2, left, right); + } + else { + LLVM_BUILD_OP(UDiv, left, right, res, "div_u", false); + } + PUSH_INT(res); + return true; + case INT_REM_S: + /* Webassembly spec requires it return 0 */ + if (is_i32) + CHECK_INT_OVERFLOW(I32); + else + CHECK_INT_OVERFLOW(I64); + return compile_rems(comp_ctx, func_ctx, left, right, overflow, + is_i32); + case INT_REM_U: + LLVM_BUILD_OP_OR_INTRINSIC(URem, left, right, res, + is_i32 ? "i32.rem_u" : "i64.rem_u", + "rem_u", false); + PUSH_INT(res); + return true; + default: + bh_assert(0); + return false; + } + } + +fail: + return false; +} + +static LLVMValueRef +compile_int_add(AOTCompContext *comp_ctx, LLVMValueRef left, LLVMValueRef right, + bool is_i32) +{ + /* If one of the operands is 0, just return the other */ + if (IS_CONST_ZERO(left)) + return right; + if (IS_CONST_ZERO(right)) + return left; + + /* Build add */ + return LLVMBuildAdd(comp_ctx->builder, left, right, "add"); +} + +static LLVMValueRef +compile_int_sub(AOTCompContext *comp_ctx, LLVMValueRef left, LLVMValueRef right, + bool is_i32) +{ + /* If the right operand is 0, just return the left */ + if (IS_CONST_ZERO(right)) + return left; + + /* Build sub */ + return LLVMBuildSub(comp_ctx->builder, left, right, "sub"); +} + +static LLVMValueRef +compile_int_mul(AOTCompContext *comp_ctx, LLVMValueRef left, LLVMValueRef right, + bool is_i32) +{ + /* If one of the operands is 0, just return constant 0 */ + if (IS_CONST_ZERO(left) || IS_CONST_ZERO(right)) + return is_i32 ? I32_ZERO : I64_ZERO; + + /* Build mul */ + return LLVMBuildMul(comp_ctx->builder, left, right, "mul"); +} + +static bool +compile_op_int_arithmetic(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntArithmetic arith_op, bool is_i32, + uint8 **p_frame_ip) +{ + switch (arith_op) { + case INT_ADD: + DEF_INT_BINARY_OP(compile_int_add(comp_ctx, left, right, is_i32), + "compile int add fail."); + return true; + case INT_SUB: + DEF_INT_BINARY_OP(compile_int_sub(comp_ctx, left, right, is_i32), + "compile int sub fail."); + return true; + case INT_MUL: + DEF_INT_BINARY_OP(compile_int_mul(comp_ctx, left, right, is_i32), + "compile int mul fail."); + return true; + case INT_DIV_S: + case INT_DIV_U: + case INT_REM_S: + case INT_REM_U: + return compile_int_div(comp_ctx, func_ctx, arith_op, is_i32, + p_frame_ip); + default: + bh_assert(0); + return false; + } + +fail: + return false; +} + +static bool +compile_op_int_bitwise(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntBitwise bitwise_op, bool is_i32) +{ + switch (bitwise_op) { + case INT_AND: + DEF_INT_BINARY_OP( + LLVMBuildAnd(comp_ctx->builder, left, right, "and"), + "llvm build and fail."); + return true; + case INT_OR: + DEF_INT_BINARY_OP(LLVMBuildOr(comp_ctx->builder, left, right, "or"), + "llvm build or fail."); + return true; + case INT_XOR: + DEF_INT_BINARY_OP( + LLVMBuildXor(comp_ctx->builder, left, right, "xor"), + "llvm build xor fail."); + return true; + default: + bh_assert(0); + return false; + } + +fail: + return false; +} + +static LLVMValueRef +compile_int_shl(AOTCompContext *comp_ctx, LLVMValueRef left, LLVMValueRef right, + bool is_i32) +{ + LLVMValueRef res; + + SHIFT_COUNT_MASK; + + /* Build shl */ + LLVM_BUILD_OP(Shl, left, right, res, "shl", NULL); + + return res; +} + +static LLVMValueRef +compile_int_shr_s(AOTCompContext *comp_ctx, LLVMValueRef left, + LLVMValueRef right, bool is_i32) +{ + LLVMValueRef res; + + SHIFT_COUNT_MASK; + + /* Build shl */ + LLVM_BUILD_OP(AShr, left, right, res, "shr_s", NULL); + + return res; +} + +static LLVMValueRef +compile_int_shr_u(AOTCompContext *comp_ctx, LLVMValueRef left, + LLVMValueRef right, bool is_i32) +{ + LLVMValueRef res; + + SHIFT_COUNT_MASK; + + /* Build shl */ + LLVM_BUILD_OP(LShr, left, right, res, "shr_u", NULL); + + return res; +} + +static LLVMValueRef +compile_int_rot(AOTCompContext *comp_ctx, LLVMValueRef left, LLVMValueRef right, + bool is_rotl, bool is_i32) +{ + LLVMValueRef bits_minus_shift_count, res, tmp_l, tmp_r; + char *name = is_rotl ? "rotl" : "rotr"; + + SHIFT_COUNT_MASK; + + /* rotl/rotr with 0 */ + if (IS_CONST_ZERO(right)) + return left; + + /* Calculate (bits - shift_count) */ + LLVM_BUILD_OP(Sub, is_i32 ? I32_32 : I64_64, right, bits_minus_shift_count, + "bits_minus_shift_count", NULL); + /* Calculate (bits - shift_count) & mask */ + bits_minus_shift_count = + LLVMBuildAnd(comp_ctx->builder, bits_minus_shift_count, + is_i32 ? I32_31 : I64_63, "bits_minus_shift_count_and"); + if (!bits_minus_shift_count) { + aot_set_last_error("llvm build and failed."); + return NULL; + } + + if (is_rotl) { + /* (left << count) | (left >> ((BITS - count) & mask)) */ + LLVM_BUILD_OP(Shl, left, right, tmp_l, "tmp_l", NULL); + LLVM_BUILD_OP(LShr, left, bits_minus_shift_count, tmp_r, "tmp_r", NULL); + } + else { + /* (left >> count) | (left << ((BITS - count) & mask)) */ + LLVM_BUILD_OP(LShr, left, right, tmp_l, "tmp_l", NULL); + LLVM_BUILD_OP(Shl, left, bits_minus_shift_count, tmp_r, "tmp_r", NULL); + } + + LLVM_BUILD_OP(Or, tmp_l, tmp_r, res, name, NULL); + + return res; +} + +static bool +compile_op_int_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op, bool is_i32) +{ + switch (shift_op) { + case INT_SHL: + DEF_INT_BINARY_OP(compile_int_shl(comp_ctx, left, right, is_i32), + NULL); + return true; + case INT_SHR_S: + DEF_INT_BINARY_OP(compile_int_shr_s(comp_ctx, left, right, is_i32), + NULL); + return true; + case INT_SHR_U: + DEF_INT_BINARY_OP(compile_int_shr_u(comp_ctx, left, right, is_i32), + NULL); + return true; + case INT_ROTL: + DEF_INT_BINARY_OP( + compile_int_rot(comp_ctx, left, right, true, is_i32), NULL); + return true; + case INT_ROTR: + DEF_INT_BINARY_OP( + compile_int_rot(comp_ctx, left, right, false, is_i32), NULL); + return true; + default: + bh_assert(0); + return false; + } + +fail: + return false; +} + +static bool +is_target_arm(AOTCompContext *comp_ctx) +{ + return !strncmp(comp_ctx->target_arch, "arm", 3) + || !strncmp(comp_ctx->target_arch, "aarch64", 7) + || !strncmp(comp_ctx->target_arch, "thumb", 5); +} + +static bool +is_target_x86(AOTCompContext *comp_ctx) +{ + return !strncmp(comp_ctx->target_arch, "x86_64", 6) + || !strncmp(comp_ctx->target_arch, "i386", 4); +} + +static bool +is_target_xtensa(AOTCompContext *comp_ctx) +{ + return !strncmp(comp_ctx->target_arch, "xtensa", 6); +} + +static bool +is_target_mips(AOTCompContext *comp_ctx) +{ + return !strncmp(comp_ctx->target_arch, "mips", 4); +} + +static bool +is_target_riscv(AOTCompContext *comp_ctx) +{ + return !strncmp(comp_ctx->target_arch, "riscv", 5); +} + +static bool +is_targeting_soft_float(AOTCompContext *comp_ctx, bool is_f32) +{ + bool ret = false; + char *feature_string; + + if (!(feature_string = + LLVMGetTargetMachineFeatureString(comp_ctx->target_machine))) { + aot_set_last_error("llvm get target machine feature string fail."); + return false; + } + + /* Note: + * LLVM CodeGen uses FPU Coprocessor registers by default, + * so user must specify '--cpu-features=+soft-float' to wamrc if the target + * doesn't have or enable FPU on arm, x86 or mips. */ + if (is_target_arm(comp_ctx) || is_target_x86(comp_ctx) + || is_target_mips(comp_ctx)) { + ret = strstr(feature_string, "+soft-float") ? true : false; + } + else if (is_target_xtensa(comp_ctx)) { + /* Note: + * 1. The Floating-Point Coprocessor Option of xtensa only support + * single-precision floating-point operations, so must use soft-float + * for f64(i.e. double). + * 2. LLVM CodeGen uses Floating-Point Coprocessor registers by default, + * so user must specify '--cpu-features=-fp' to wamrc if the target + * doesn't have or enable Floating-Point Coprocessor Option on xtensa. + */ + if (comp_ctx->disable_llvm_intrinsics) + ret = false; + else + ret = (!is_f32 || strstr(feature_string, "-fp")) ? true : false; + } + else if (is_target_riscv(comp_ctx)) { + /* + * Note: Use builtin intrinsics since hardware float operation + * will cause rodata relocation, this will try to use hardware + * float unit (by return false) but handled by software finally + */ + if (comp_ctx->disable_llvm_intrinsics) + ret = false; + else + ret = !strstr(feature_string, "+d") ? true : false; + } + else { + ret = true; + } + + LLVMDisposeMessage(feature_string); + return ret; +} + +static bool +compile_op_float_arithmetic(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatArithmetic arith_op, bool is_f32) +{ + switch (arith_op) { + case FLOAT_ADD: + if (is_targeting_soft_float(comp_ctx, is_f32)) + DEF_FP_BINARY_OP( + LLVMBuildFAdd(comp_ctx->builder, left, right, "fadd"), + "llvm build fadd fail."); + else + DEF_FP_BINARY_OP( + call_llvm_float_experimental_constrained_intrinsic( + comp_ctx, func_ctx, is_f32, + (is_f32 ? "llvm.experimental.constrained.fadd.f32" + : "llvm.experimental.constrained.fadd.f64"), + left, right, comp_ctx->fp_rounding_mode, + comp_ctx->fp_exception_behavior), + NULL); + return true; + case FLOAT_SUB: + if (is_targeting_soft_float(comp_ctx, is_f32)) + DEF_FP_BINARY_OP( + LLVMBuildFSub(comp_ctx->builder, left, right, "fsub"), + "llvm build fsub fail."); + else + DEF_FP_BINARY_OP( + call_llvm_float_experimental_constrained_intrinsic( + comp_ctx, func_ctx, is_f32, + (is_f32 ? "llvm.experimental.constrained.fsub.f32" + : "llvm.experimental.constrained.fsub.f64"), + left, right, comp_ctx->fp_rounding_mode, + comp_ctx->fp_exception_behavior), + NULL); + return true; + case FLOAT_MUL: + if (is_targeting_soft_float(comp_ctx, is_f32)) + DEF_FP_BINARY_OP( + LLVMBuildFMul(comp_ctx->builder, left, right, "fmul"), + "llvm build fmul fail."); + else + DEF_FP_BINARY_OP( + call_llvm_float_experimental_constrained_intrinsic( + comp_ctx, func_ctx, is_f32, + (is_f32 ? "llvm.experimental.constrained.fmul.f32" + : "llvm.experimental.constrained.fmul.f64"), + left, right, comp_ctx->fp_rounding_mode, + comp_ctx->fp_exception_behavior), + NULL); + return true; + case FLOAT_DIV: + if (is_targeting_soft_float(comp_ctx, is_f32)) + DEF_FP_BINARY_OP( + LLVMBuildFDiv(comp_ctx->builder, left, right, "fdiv"), + "llvm build fdiv fail."); + else + DEF_FP_BINARY_OP( + call_llvm_float_experimental_constrained_intrinsic( + comp_ctx, func_ctx, is_f32, + (is_f32 ? "llvm.experimental.constrained.fdiv.f32" + : "llvm.experimental.constrained.fdiv.f64"), + left, right, comp_ctx->fp_rounding_mode, + comp_ctx->fp_exception_behavior), + NULL); + return true; + case FLOAT_MIN: + DEF_FP_BINARY_OP(compile_op_float_min_max( + comp_ctx, func_ctx, is_f32, left, right, true), + NULL); + return true; + case FLOAT_MAX: + DEF_FP_BINARY_OP(compile_op_float_min_max(comp_ctx, func_ctx, + is_f32, left, right, + false), + NULL); + + return true; + default: + bh_assert(0); + return false; + } + +fail: + return false; +} + +static LLVMValueRef +call_llvm_float_math_intrinsic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_f32, + const char *intrinsic, ...) +{ + va_list param_value_list; + LLVMValueRef ret; + LLVMTypeRef param_type, ret_type = is_f32 ? F32_TYPE : F64_TYPE; + + param_type = ret_type; + + va_start(param_value_list, intrinsic); + + ret = aot_call_llvm_intrinsic_v(comp_ctx, func_ctx, intrinsic, ret_type, + ¶m_type, 1, param_value_list); + + va_end(param_value_list); + + return ret; +} + +static bool +compile_op_float_math(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatMath math_op, bool is_f32) +{ + switch (math_op) { + case FLOAT_ABS: + DEF_FP_UNARY_OP(call_llvm_float_math_intrinsic( + comp_ctx, func_ctx, is_f32, + is_f32 ? "llvm.fabs.f32" : "llvm.fabs.f64", + operand), + NULL); + return true; + case FLOAT_NEG: + DEF_FP_UNARY_OP(LLVMBuildFNeg(comp_ctx->builder, operand, "fneg"), + "llvm build fneg fail."); + return true; + + case FLOAT_CEIL: + DEF_FP_UNARY_OP(call_llvm_float_math_intrinsic( + comp_ctx, func_ctx, is_f32, + is_f32 ? "llvm.ceil.f32" : "llvm.ceil.f64", + operand), + NULL); + return true; + case FLOAT_FLOOR: + DEF_FP_UNARY_OP(call_llvm_float_math_intrinsic( + comp_ctx, func_ctx, is_f32, + is_f32 ? "llvm.floor.f32" : "llvm.floor.f64", + operand), + NULL); + return true; + case FLOAT_TRUNC: + DEF_FP_UNARY_OP(call_llvm_float_math_intrinsic( + comp_ctx, func_ctx, is_f32, + is_f32 ? "llvm.trunc.f32" : "llvm.trunc.f64", + operand), + NULL); + return true; + case FLOAT_NEAREST: + DEF_FP_UNARY_OP(call_llvm_float_math_intrinsic( + comp_ctx, func_ctx, is_f32, + is_f32 ? "llvm.rint.f32" : "llvm.rint.f64", + operand), + NULL); + return true; + case FLOAT_SQRT: + if (is_targeting_soft_float(comp_ctx, is_f32) + || comp_ctx->disable_llvm_intrinsics) + DEF_FP_UNARY_OP(call_llvm_float_math_intrinsic( + comp_ctx, func_ctx, is_f32, + is_f32 ? "llvm.sqrt.f32" : "llvm.sqrt.f64", + operand), + NULL); + else + DEF_FP_UNARY_OP( + call_llvm_libm_experimental_constrained_intrinsic( + comp_ctx, func_ctx, is_f32, + (is_f32 ? "llvm.experimental.constrained.sqrt.f32" + : "llvm.experimental.constrained.sqrt.f64"), + operand, comp_ctx->fp_rounding_mode, + comp_ctx->fp_exception_behavior), + NULL); + return true; + default: + bh_assert(0); + return false; + } + + return true; + +fail: + return false; +} + +static bool +compile_float_copysign(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_f32) +{ + LLVMTypeRef ret_type, param_types[2]; + + param_types[0] = param_types[1] = ret_type = is_f32 ? F32_TYPE : F64_TYPE; + + DEF_FP_BINARY_OP(aot_call_llvm_intrinsic( + comp_ctx, func_ctx, + is_f32 ? "llvm.copysign.f32" : "llvm.copysign.f64", + ret_type, param_types, 2, left, right), + NULL); + return true; + +fail: + return false; +} + +bool +aot_compile_op_i32_clz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return aot_compile_int_bit_count(comp_ctx, func_ctx, CLZ32, true); +} + +bool +aot_compile_op_i32_ctz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return aot_compile_int_bit_count(comp_ctx, func_ctx, CTZ32, true); +} + +bool +aot_compile_op_i32_popcnt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return aot_compile_int_bit_count(comp_ctx, func_ctx, POP_CNT32, true); +} + +bool +aot_compile_op_i64_clz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return aot_compile_int_bit_count(comp_ctx, func_ctx, CLZ64, false); +} + +bool +aot_compile_op_i64_ctz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return aot_compile_int_bit_count(comp_ctx, func_ctx, CTZ64, false); +} + +bool +aot_compile_op_i64_popcnt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return aot_compile_int_bit_count(comp_ctx, func_ctx, POP_CNT64, false); +} + +bool +aot_compile_op_i32_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntArithmetic arith_op, + uint8 **p_frame_ip) +{ + return compile_op_int_arithmetic(comp_ctx, func_ctx, arith_op, true, + p_frame_ip); +} + +bool +aot_compile_op_i64_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntArithmetic arith_op, + uint8 **p_frame_ip) +{ + return compile_op_int_arithmetic(comp_ctx, func_ctx, arith_op, false, + p_frame_ip); +} + +bool +aot_compile_op_i32_bitwise(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntBitwise bitwise_op) +{ + return compile_op_int_bitwise(comp_ctx, func_ctx, bitwise_op, true); +} + +bool +aot_compile_op_i64_bitwise(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntBitwise bitwise_op) +{ + return compile_op_int_bitwise(comp_ctx, func_ctx, bitwise_op, false); +} + +bool +aot_compile_op_i32_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op) +{ + return compile_op_int_shift(comp_ctx, func_ctx, shift_op, true); +} + +bool +aot_compile_op_i64_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op) +{ + return compile_op_int_shift(comp_ctx, func_ctx, shift_op, false); +} + +bool +aot_compile_op_f32_math(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatMath math_op) +{ + return compile_op_float_math(comp_ctx, func_ctx, math_op, true); +} + +bool +aot_compile_op_f64_math(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatMath math_op) +{ + return compile_op_float_math(comp_ctx, func_ctx, math_op, false); +} + +bool +aot_compile_op_f32_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + FloatArithmetic arith_op) +{ + return compile_op_float_arithmetic(comp_ctx, func_ctx, arith_op, true); +} + +bool +aot_compile_op_f64_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + FloatArithmetic arith_op) +{ + return compile_op_float_arithmetic(comp_ctx, func_ctx, arith_op, false); +} + +bool +aot_compile_op_f32_copysign(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return compile_float_copysign(comp_ctx, func_ctx, true); +} + +bool +aot_compile_op_f64_copysign(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return compile_float_copysign(comp_ctx, func_ctx, false); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_numberic.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_numberic.h new file mode 100644 index 0000000..7206315 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_numberic.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_NUMBERIC_H_ +#define _AOT_EMIT_NUMBERIC_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_i32_clz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i32_ctz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i32_popcnt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i64_clz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i64_ctz(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i64_popcnt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_i32_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntArithmetic arith_op, + uint8 **p_frame_ip); + +bool +aot_compile_op_i64_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntArithmetic arith_op, + uint8 **p_frame_ip); + +bool +aot_compile_op_i32_bitwise(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntBitwise bitwise_op); + +bool +aot_compile_op_i64_bitwise(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntBitwise bitwise_op); + +bool +aot_compile_op_i32_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op); + +bool +aot_compile_op_i64_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op); + +bool +aot_compile_op_f32_math(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatMath math_op); + +bool +aot_compile_op_f64_math(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatMath math_op); + +bool +aot_compile_op_f32_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + FloatArithmetic arith_op); + +bool +aot_compile_op_f64_arithmetic(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + FloatArithmetic arith_op); + +bool +aot_compile_op_f32_copysign(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_f64_copysign(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_NUMBERIC_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_parametric.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_parametric.c new file mode 100644 index 0000000..198e045 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_parametric.c @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_parametric.h" + +static bool +pop_value_from_wasm_stack(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef *p_value, bool is_32, uint8 *p_type) +{ + AOTValue *aot_value; + uint8 type; + + if (!func_ctx->block_stack.block_list_end) { + aot_set_last_error("WASM block stack underflow."); + return false; + } + if (!func_ctx->block_stack.block_list_end->value_stack.value_list_end) { + aot_set_last_error("WASM data stack underflow."); + return false; + } + + aot_value = aot_value_stack_pop( + comp_ctx, &func_ctx->block_stack.block_list_end->value_stack); + type = aot_value->type; + + if (aot_value->type == VALUE_TYPE_I1) { + if (!(aot_value->value = + LLVMBuildZExt(comp_ctx->builder, aot_value->value, I32_TYPE, + "val_s_ext"))) { + aot_set_last_error("llvm build sign ext failed."); + return false; + } + type = aot_value->type = VALUE_TYPE_I32; + } + + if (p_type != NULL) { + *p_type = aot_value->type; + } + if (p_value != NULL) { + *p_value = aot_value->value; + } + + wasm_runtime_free(aot_value); + + if (is_32) { + /* is_32: i32, f32, ref.func, ref.extern, v128, + or GC ref types */ + if (!(type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32 + || type == VALUE_TYPE_V128 + || (comp_ctx->enable_ref_types + && (type == VALUE_TYPE_FUNCREF + || type == VALUE_TYPE_EXTERNREF)) +#if WASM_ENABLE_GC != 0 + || (comp_ctx->enable_gc && type == VALUE_TYPE_GC_REF) +#endif + )) { + aot_set_last_error("invalid WASM stack data type."); + return false; + } + } + else { + /* !is_32: i64, f64, or GC ref types */ + if (!(type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64 +#if WASM_ENABLE_GC != 0 + || (comp_ctx->enable_gc && type == VALUE_TYPE_GC_REF) + /* may be i32 which denotes funcref/externref */ + || (!comp_ctx->enable_gc && type == VALUE_TYPE_I32) +#endif + )) { + aot_set_last_error("invalid WASM stack data type."); + return false; + } + } + + return true; +} + +bool +aot_compile_op_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_drop_32) +{ + if (!pop_value_from_wasm_stack(comp_ctx, func_ctx, NULL, is_drop_32, NULL)) + return false; + + return true; +} + +bool +aot_compile_op_select(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_select_32) +{ + LLVMValueRef val1, val2, cond, selected; + uint8 val1_type, val2_type; + + POP_COND(cond); + + if (!pop_value_from_wasm_stack(comp_ctx, func_ctx, &val2, is_select_32, + &val2_type) + || !pop_value_from_wasm_stack(comp_ctx, func_ctx, &val1, is_select_32, + &val1_type)) + return false; + + if (val1_type != val2_type) { + aot_set_last_error("invalid stack values with different type"); + return false; + } + + if (!(selected = + LLVMBuildSelect(comp_ctx->builder, cond, val1, val2, "select"))) { + aot_set_last_error("llvm build select failed."); + return false; + } + + PUSH(selected, val1_type); + + return true; + +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_parametric.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_parametric.h new file mode 100644 index 0000000..68fe8f1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_parametric.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_PARAMETRIC_H_ +#define _AOT_EMIT_PARAMETRIC_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_drop_32); + +bool +aot_compile_op_select(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_select_32); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_PARAMETRIC_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_stringref.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_stringref.c new file mode 100644 index 0000000..1642cee --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_stringref.c @@ -0,0 +1,1443 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#if WASM_ENABLE_STRINGREF != 0 + +#include "aot_emit_stringref.h" +#include "aot_emit_exception.h" +#include "aot_emit_memory.h" +#include "aot_emit_gc.h" +#include "aot.h" +#include "aot_compiler.h" +#include "aot_emit_memory.h" +#include "gc_object.h" +#include "string_object.h" + +#define BUILD_ISNULL(ptr, res, name) \ + do { \ + if (!(res = LLVMBuildIsNull(comp_ctx->builder, ptr, name))) { \ + aot_set_last_error("llvm build isnull failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_ISNOTNULL(ptr, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, ptr, name))) { \ + aot_set_last_error("llvm build isnotnull failed."); \ + goto fail; \ + } \ + } while (0) + +#define ADD_BASIC_BLOCK(block, name) \ + do { \ + if (!(block = LLVMAppendBasicBlockInContext(comp_ctx->context, \ + func_ctx->func, name))) { \ + aot_set_last_error("llvm add basic block failed."); \ + goto fail; \ + } \ + } while (0) + +#define CURR_BLOCK() LLVMGetInsertBlock(comp_ctx->builder) + +#define MOVE_BLOCK_AFTER(llvm_block, llvm_block_after) \ + LLVMMoveBasicBlockAfter(llvm_block, llvm_block_after) + +#define MOVE_BLOCK_AFTER_CURR(llvm_block) \ + LLVMMoveBasicBlockAfter(llvm_block, CURR_BLOCK()) + +#define DEFINE_STRINGREF_CHECK_VAR() \ + LLVMBasicBlockRef check_string_obj_succ, check_stringref_obj_succ; \ + LLVMValueRef cmp + +#define CHECK_STRING_OBJ(str_obj) \ + do { \ + ADD_BASIC_BLOCK(check_string_obj_succ, "check string obj succ"); \ + MOVE_BLOCK_AFTER_CURR(check_string_obj_succ); \ + \ + BUILD_ISNULL(str_obj, cmp, "cmp_string_obj"); \ + if (!aot_emit_exception(comp_ctx, func_ctx, \ + EXCE_FAILED_TO_CREATE_STRING, true, cmp, \ + check_string_obj_succ)) \ + goto fail; \ + } while (0) + +#define CHECK_STRINGREF_INTERNAL(stringref_obj, exce_id, name) \ + do { \ + ADD_BASIC_BLOCK(check_stringref_obj_succ, "check " name " obj succ"); \ + MOVE_BLOCK_AFTER(check_stringref_obj_succ, check_string_obj_succ); \ + \ + BUILD_ISNULL(stringref_obj, cmp, "cmp_" name "_obj"); \ + if (!aot_emit_exception(comp_ctx, func_ctx, exce_id, true, cmp, \ + check_stringref_obj_succ)) \ + goto fail; \ + } while (0) + +#define CHECK_STRINGREF_OBJ(stringref_obj) \ + CHECK_STRINGREF_INTERNAL(stringref_obj, EXCE_FAILED_TO_CREATE_STRINGREF, \ + "stringref") + +#define CHECK_STRINGVIEW_OBJ(stringview_obj) \ + CHECK_STRINGREF_INTERNAL(stringview_obj, EXCE_FAILED_TO_CREATE_STRINGVIEW, \ + "stringview") + +#define CHECK_STRING_ENCODE(value) \ + do { \ + ADD_BASIC_BLOCK(check_string_encode_succ, "check string encode succ"); \ + MOVE_BLOCK_AFTER_CURR(check_string_encode_succ); \ + \ + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntSLT, value, \ + I32_ZERO, "cmp_string_encode"))) { \ + aot_set_last_error("llvm build icmp failed."); \ + goto fail; \ + } \ + \ + if (!aot_emit_exception(comp_ctx, func_ctx, \ + EXCE_FAILED_TO_ENCODE_STRING, true, cmp, \ + check_string_encode_succ)) \ + goto fail; \ + } while (0) + +static bool +aot_call_wasm_stringref_obj_new(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, LLVMValueRef str_obj, + uint32 stringref_type, uint32 pos, + LLVMValueRef *stringref_obj) +{ + LLVMValueRef param_values[3], func, value, res; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + uint32 argc = 2; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + if (stringref_type == WASM_TYPE_STRINGREF) { + GET_AOT_FUNCTION(wasm_stringref_obj_new, argc); + } + else if (stringref_type == WASM_TYPE_STRINGVIEWWTF8) { + GET_AOT_FUNCTION(wasm_stringview_wtf8_obj_new, argc); + } + else if (stringref_type == WASM_TYPE_STRINGVIEWWTF16) { + GET_AOT_FUNCTION(wasm_stringview_wtf16_obj_new, argc); + } + else { + argc = 3; + GET_AOT_FUNCTION(wasm_stringview_iter_obj_new, argc); + } + + param_values[0] = func_ctx->exec_env; + param_values[1] = str_obj; + if (stringref_type == WASM_TYPE_STRINGVIEWITER) { + param_values[2] = I32_CONST(pos); + } + + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + argc, "create_stringref"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + *stringref_obj = res; + + return true; +fail: + return false; +} + +static LLVMValueRef +aot_stringref_obj_get_value(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef stringref_obj) +{ + LLVMValueRef str_obj_ptr, str_obj, host_ptr_offset; + + /* header */ + host_ptr_offset = I32_CONST(comp_ctx->pointer_size); + + if (!(stringref_obj = + LLVMBuildBitCast(comp_ctx->builder, stringref_obj, INT8_PTR_TYPE, + "stringref_obj_i8p"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(str_obj_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, stringref_obj, + &host_ptr_offset, 1, "str_obj_i8p"))) { + aot_set_last_error("llvm build gep failed."); + goto fail; + } + + if (!(str_obj_ptr = LLVMBuildBitCast(comp_ctx->builder, str_obj_ptr, + GC_REF_PTR_TYPE, "str_obj_gcref_p"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(str_obj = LLVMBuildLoad2(comp_ctx->builder, GC_REF_TYPE, str_obj_ptr, + "str_obj"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + LLVMSetAlignment(str_obj, 4); + + return str_obj; + +fail: + return NULL; +} + +static LLVMValueRef +get_stringview_iter_pos_addr(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef stringview_iter_obj) +{ + LLVMValueRef iter_pos_ptr, host_ptr_offset; + + /* header + str_obj */ + host_ptr_offset = I32_CONST(comp_ctx->pointer_size * 2); + + if (!(stringview_iter_obj = + LLVMBuildBitCast(comp_ctx->builder, stringview_iter_obj, + INT8_PTR_TYPE, "stringview_iter_obj_i8p"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + if (!(iter_pos_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, stringview_iter_obj, + &host_ptr_offset, 1, "iter_pos_i8p"))) { + aot_set_last_error("llvm build gep failed."); + goto fail; + } + + if (!(iter_pos_ptr = LLVMBuildBitCast(comp_ctx->builder, iter_pos_ptr, + INT32_PTR_TYPE, "iter_pos_i32p"))) { + aot_set_last_error("llvm build bitcast failed."); + goto fail; + } + + return iter_pos_ptr; + +fail: + return NULL; +} + +static LLVMValueRef +aot_call_wasm_string_measure(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef stringref_obj, uint32 encoding) +{ + LLVMValueRef param_values[3], func, value, str_obj; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_measure, 2); + + /* Call function wasm_string_measure() */ + param_values[0] = str_obj; + param_values[1] = I32_CONST(encoding); + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "string_measure"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + return value; +fail: + return NULL; +} + +static LLVMValueRef +aot_call_wasm_string_create_view(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef stringref_obj, uint32 encoding) +{ + LLVMValueRef param_values[3], func, value, str_obj; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + GET_AOT_FUNCTION(wasm_string_create_view, 2); + + /* Call function wasm_string_create_view() */ + param_values[0] = str_obj; + param_values[1] = I32_CONST(encoding); + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "string_create_view"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + return value; +fail: + return NULL; +} + +static LLVMValueRef +aot_call_wasm_string_advance(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef stringref_obj, LLVMValueRef bytes, + LLVMValueRef pos) +{ + LLVMValueRef param_values[4], func, value, str_obj; + LLVMTypeRef param_types[4], ret_type, func_type, func_ptr_type; + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT32_PTR_TYPE; + ret_type = INT8_PTR_TYPE; + + GET_AOT_FUNCTION(wasm_string_advance, 4); + + /* Call function wasm_string_advance() */ + param_values[0] = str_obj; + param_values[1] = pos; + param_values[2] = bytes; + param_values[3] = I8_PTR_NULL; + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 4, "string_advance"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + return value; +fail: + return NULL; +} + +static LLVMValueRef +aot_call_wasm_string_slice(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef stringref_obj, LLVMValueRef start, + LLVMValueRef end, StringViewType stringview_type) +{ + LLVMValueRef param_values[4], func, value, str_obj; + LLVMTypeRef param_types[4], ret_type, func_type, func_ptr_type; + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + GET_AOT_FUNCTION(wasm_string_slice, 4); + + /* Call function wasm_string_slice() */ + param_values[0] = str_obj; + param_values[1] = start; + param_values[2] = end; + param_values[3] = I32_CONST(stringview_type); + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 4, "string_slice"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + return value; +fail: + return NULL; +} + +bool +aot_compile_op_string_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 encoding) +{ + LLVMValueRef maddr, byte_length, offset, str_obj, stringref_obj; + LLVMValueRef param_values[5], func, value; + LLVMTypeRef param_types[5], ret_type, func_type, func_ptr_type; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_I32(byte_length); + POP_I32(offset); + + if (!(maddr = check_bulk_memory_overflow(comp_ctx, func_ctx, offset, + byte_length))) + goto fail; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + GET_AOT_FUNCTION(wasm_string_new_with_encoding, 3); + + /* Call function wasm_struct_obj_new() */ + param_values[0] = maddr; + param_values[1] = byte_length; + param_values[2] = I32_CONST(encoding); + + if (!(str_obj = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 3, "wasm_string_new"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + CHECK_STRING_OBJ(str_obj); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, str_obj, + WASM_TYPE_STRINGREF, 0, + &stringref_obj)) { + goto fail; + } + CHECK_STRINGREF_OBJ(stringref_obj); + + PUSH_GC_REF(stringref_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 contents) +{ + LLVMValueRef param_values[2], func, value, str_obj, stringref_obj; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + GET_AOT_FUNCTION(wasm_string_new_const, 2); + + bh_assert(contents < comp_ctx->comp_data->string_literal_count); + param_values[0] = LLVMConstIntToPtr( + I64_CONST((unsigned long long)(uintptr_t) + comp_ctx->comp_data->string_literal_ptrs_wp[contents]), + INT8_PTR_TYPE); + param_values[1] = + I32_CONST(comp_ctx->comp_data->string_literal_lengths_wp[contents]); + + if (!(str_obj = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "create_stringref"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + CHECK_STRING_OBJ(str_obj); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, str_obj, + WASM_TYPE_STRINGREF, 0, + &stringref_obj)) { + goto fail; + } + CHECK_STRINGREF_OBJ(stringref_obj); + + PUSH_GC_REF(stringref_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_measure(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 encoding) +{ + LLVMValueRef stringref_obj, value; + + POP_GC_REF(stringref_obj); + + if (!(value = aot_call_wasm_string_measure(comp_ctx, func_ctx, + stringref_obj, encoding))) { + goto fail; + } + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_encode(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 mem_idx, uint32 encoding) +{ + LLVMValueRef param_values[6], func, value, offset, length, maddr, str_obj, + stringref_obj; + LLVMTypeRef param_types[6], ret_type, func_type, func_ptr_type; + LLVMBasicBlockRef check_string_encode_succ; + LLVMValueRef cmp; + + POP_I32(offset); + POP_GC_REF(stringref_obj); + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + if (!(length = aot_call_wasm_string_measure(comp_ctx, func_ctx, + stringref_obj, encoding))) { + goto fail; + } + + if (!(maddr = + check_bulk_memory_overflow(comp_ctx, func_ctx, offset, length))) + goto fail; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + param_types[4] = INT8_PTR_TYPE; + param_types[5] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_encode, 6); + + /* Call function wasm_string_measure() */ + param_values[0] = str_obj; + param_values[1] = I32_ZERO; + param_values[2] = length; + param_values[3] = maddr; + param_values[4] = I8_PTR_NULL; + param_values[5] = I32_CONST(encoding); + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 6, "string_encode"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + CHECK_STRING_ENCODE(value); + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_concat(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef param_values[2], func, value, str_obj_lhs, str_obj_rhs, + stringref_obj_lhs, stringref_obj_rhs, stringref_obj_new; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_GC_REF(stringref_obj_rhs); + POP_GC_REF(stringref_obj_lhs); + + if (!(str_obj_lhs = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringref_obj_lhs))) { + goto fail; + } + + if (!(str_obj_rhs = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringref_obj_rhs))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + ret_type = INT8_PTR_TYPE; + + GET_AOT_FUNCTION(wasm_string_concat, 2); + + /* Call function wasm_string_concat() */ + param_values[0] = str_obj_lhs; + param_values[1] = str_obj_rhs; + + if (!(str_obj_lhs = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "string_concat"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + CHECK_STRING_OBJ(str_obj_lhs); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, str_obj_lhs, + WASM_TYPE_STRINGREF, 0, + &stringref_obj_new)) { + goto fail; + } + CHECK_STRINGREF_OBJ(stringref_obj_new); + + PUSH_GC_REF(stringref_obj_new); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_eq(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef param_values[2], func, value, str_obj_lhs, str_obj_rhs, + stringref_obj_lhs, stringref_obj_rhs; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + POP_GC_REF(stringref_obj_lhs); + POP_GC_REF(stringref_obj_rhs); + + if (!(str_obj_lhs = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringref_obj_lhs))) { + goto fail; + } + + if (!(str_obj_rhs = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringref_obj_rhs))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_eq, 2); + + /* Call function wasm_string_eq() */ + param_values[0] = str_obj_lhs; + param_values[1] = str_obj_rhs; + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "string_eq"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_is_usv_sequence(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef param_values[1], func, value, str_obj, stringref_obj; + LLVMTypeRef param_types[1], ret_type, func_type, func_ptr_type; + + POP_GC_REF(stringref_obj); + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_is_usv_sequence, 1); + + /* Call function wasm_string_is_usv_sequence() */ + param_values[0] = str_obj; + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 1, "string_is_usv_sequence"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_as_wtf8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef str_obj, stringref_obj, stringview_wtf8_obj; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_GC_REF(stringref_obj); + + if (!(str_obj = aot_call_wasm_string_create_view( + comp_ctx, func_ctx, stringref_obj, STRING_VIEW_WTF8))) { + goto fail; + } + CHECK_STRING_OBJ(str_obj); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, str_obj, + WASM_TYPE_STRINGVIEWWTF8, 0, + &stringview_wtf8_obj)) { + goto fail; + } + CHECK_STRINGVIEW_OBJ(stringref_obj); + + PUSH_GC_REF(stringview_wtf8_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_wtf8_advance(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef stringref_obj, bytes, pos, value; + + POP_I32(bytes); + POP_I32(pos); + POP_GC_REF(stringref_obj); + + if (!(value = aot_call_wasm_string_advance(comp_ctx, func_ctx, + stringref_obj, bytes, pos))) { + goto fail; + } + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_wtf8_encode(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 mem_idx, + uint32 encoding) +{ + LLVMValueRef param_values[6], func, value, offset, maddr, str_obj, + stringref_obj; + LLVMValueRef bytes, pos, next_pos; + LLVMTypeRef param_types[6], ret_type, func_type, func_ptr_type; + LLVMBasicBlockRef check_string_encode_succ; + LLVMValueRef cmp; + + POP_I32(bytes); + POP_I32(pos); + POP_I32(offset); + + next_pos = LLVMBuildAlloca(comp_ctx->builder, I32_TYPE, "next_pos"); + if (!next_pos) { + aot_set_last_error("failed to build alloca"); + goto fail; + } + + if (!(maddr = + check_bulk_memory_overflow(comp_ctx, func_ctx, offset, bytes))) + goto fail; + + POP_GC_REF(stringref_obj); + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + param_types[4] = INT8_PTR_TYPE; + param_types[5] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_encode, 6); + + /* Call function wasm_string_measure() */ + param_values[0] = str_obj; + param_values[1] = pos; + param_values[2] = bytes; + param_values[3] = maddr; + param_values[4] = next_pos; + param_values[5] = I32_CONST(encoding); + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 6, "string_encode"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + CHECK_STRING_ENCODE(value); + + next_pos = + LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, next_pos, "next_pos"); + if (!next_pos) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + LLVMSetAlignment(next_pos, 4); + + PUSH_I32(next_pos); + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_wtf8_slice(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef stringref_obj, start, end, stringref_obj_new, value; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_I32(start); + POP_I32(end); + POP_GC_REF(stringref_obj); + + if (!(value = aot_call_wasm_string_slice(comp_ctx, func_ctx, stringref_obj, + start, end, STRING_VIEW_WTF8))) { + goto fail; + } + CHECK_STRING_OBJ(value); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, value, + WASM_TYPE_STRINGREF, 0, + &stringref_obj_new)) { + goto fail; + } + CHECK_STRINGREF_OBJ(stringref_obj_new); + + PUSH_GC_REF(stringref_obj_new); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_as_wtf16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef str_obj, stringref_obj, stringview_wtf16_obj; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_GC_REF(stringref_obj); + + if (!(str_obj = aot_call_wasm_string_create_view( + comp_ctx, func_ctx, stringref_obj, STRING_VIEW_WTF16))) { + goto fail; + } + CHECK_STRING_OBJ(str_obj); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, str_obj, + WASM_TYPE_STRINGVIEWWTF16, 0, + &stringview_wtf16_obj)) { + goto fail; + } + CHECK_STRINGVIEW_OBJ(stringview_wtf16_obj); + + PUSH_GC_REF(stringview_wtf16_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_wtf16_length(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef param_values[2], func, value, str_obj, stringview_wtf16_obj; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + POP_GC_REF(stringview_wtf16_obj); + + if (!(str_obj = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringview_wtf16_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_wtf16_get_length, 6); + + /* Call function wasm_string_wtf16_get_length() */ + param_values[0] = str_obj; + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 1, "stringview_wtf16_length"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_wtf16_get_codeunit(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef param_values[2], func, value, str_obj, stringview_wtf16_obj, + pos; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + POP_I32(pos); + POP_GC_REF(stringview_wtf16_obj); + + if (!(str_obj = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringview_wtf16_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_get_wtf16_codeunit, 2); + + /* Call function wasm_string_get_wtf16_codeunit() */ + param_values[0] = str_obj; + param_values[1] = pos; + + if (!(value = + LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 2, "stringview_wtf16_get_codeunit"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_wtf16_encode(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 mem_idx) +{ + LLVMValueRef param_values[6], func, value, offset, maddr, str_obj, + stringref_obj; + LLVMValueRef len, pos; + LLVMTypeRef param_types[6], ret_type, func_type, func_ptr_type; + LLVMBasicBlockRef check_string_encode_succ; + LLVMValueRef cmp; + + POP_I32(len); + POP_I32(pos); + POP_I32(offset); + + if (!(maddr = check_bulk_memory_overflow( + comp_ctx, func_ctx, offset, + LLVMBuildMul(comp_ctx->builder, len, I32_CONST(2), "wtf16_len")))) + goto fail; + + POP_GC_REF(stringref_obj); + + if (!check_memory_alignment(comp_ctx, func_ctx, maddr, 2)) { + goto fail; + } + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + param_types[4] = INT8_PTR_TYPE; + param_types[5] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_encode, 6); + + /* Call function wasm_string_measure() */ + param_values[0] = str_obj; + param_values[1] = pos; + param_values[2] = len; + param_values[3] = maddr; + param_values[4] = I8_PTR_NULL; + param_values[5] = I32_CONST(WTF16); + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 6, "string_encode"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + CHECK_STRING_ENCODE(value); + + PUSH_I32(value); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_wtf16_slice(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef stringref_obj, start, end, stringref_obj_new, value; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_I32(end); + POP_I32(start); + POP_GC_REF(stringref_obj); + + if (!(value = aot_call_wasm_string_slice(comp_ctx, func_ctx, stringref_obj, + start, end, STRING_VIEW_WTF16))) { + goto fail; + } + CHECK_STRING_OBJ(value); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, value, + WASM_TYPE_STRINGREF, 0, + &stringref_obj_new)) { + goto fail; + } + CHECK_STRINGREF_OBJ(stringref_obj_new); + + PUSH_GC_REF(stringref_obj_new); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_as_iter(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef stringref_obj, stringview_iter_obj, str_obj; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_GC_REF(stringref_obj); + + if (!(str_obj = aot_call_wasm_string_create_view( + comp_ctx, func_ctx, stringref_obj, STRING_VIEW_WTF8))) { + goto fail; + } + CHECK_STRING_OBJ(str_obj); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, stringref_obj, + WASM_TYPE_STRINGVIEWITER, 0, + &stringview_iter_obj)) { + goto fail; + } + CHECK_STRINGVIEW_OBJ(stringview_iter_obj); + + PUSH_GC_REF(stringview_iter_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_stringview_iter_next(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef param_values[2], func, value, stringview_iter_obj, str_obj, + iter_pos_addr, pos; + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + + POP_GC_REF(stringview_iter_obj); + + if (!(str_obj = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringview_iter_obj))) { + goto fail; + } + + if (!(iter_pos_addr = get_stringview_iter_pos_addr(comp_ctx, func_ctx, + stringview_iter_obj))) { + goto fail; + } + + pos = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, iter_pos_addr, + "get_iter_pos"); + LLVMSetAlignment(pos, 4); + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_next_codepoint, 2); + + /* Call function wasm_string_measure() */ + param_values[0] = str_obj; + param_values[1] = pos; + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, "stringview_iter_next"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + PUSH_I32(value); + + return true; +fail: + return false; +} + +static bool +stringview_iter_advance_or_rewind(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_rewind) +{ + LLVMValueRef param_values[4], func, value, stringview_iter_obj, str_obj, + code_points_consumed, iter_pos_addr, pos, code_points_count, res; + LLVMTypeRef param_types[4], ret_type, func_type, func_ptr_type; + + POP_I32(code_points_count); + POP_GC_REF(stringview_iter_obj); + + if (!(str_obj = aot_stringref_obj_get_value(comp_ctx, func_ctx, + stringview_iter_obj))) { + goto fail; + } + + if (!(iter_pos_addr = get_stringview_iter_pos_addr(comp_ctx, func_ctx, + stringview_iter_obj))) { + goto fail; + } + + if (!(pos = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, iter_pos_addr, + "get_iter_pos"))) { + goto fail; + } + LLVMSetAlignment(pos, 4); + + if (!(code_points_consumed = LLVMBuildAlloca(comp_ctx->builder, I32_TYPE, + "code_points_consumed"))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT32_PTR_TYPE; + ret_type = I32_TYPE; + + if (is_rewind) { + GET_AOT_FUNCTION(wasm_string_rewind, 4); + } + else { + GET_AOT_FUNCTION(wasm_string_advance, 4); + } + + /* Call function wasm_string_advance() */ + param_values[0] = str_obj; + param_values[1] = pos; + param_values[2] = code_points_count; + param_values[3] = code_points_consumed; + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 4, "string_advance"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + if (!(code_points_consumed = + LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, code_points_consumed, + "get_code_points_consumed"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + LLVMSetAlignment(code_points_consumed, 4); + + if (!(res = LLVMBuildStore(comp_ctx->builder, code_points_consumed, + iter_pos_addr))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + LLVMSetAlignment(res, 4); + + PUSH_I32(code_points_consumed); +fail: + return false; +} + +bool +aot_compile_op_stringview_iter_advance(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return stringview_iter_advance_or_rewind(comp_ctx, func_ctx, false); +} + +bool +aot_compile_op_stringview_iter_rewind(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return stringview_iter_advance_or_rewind(comp_ctx, func_ctx, true); +} + +bool +aot_compile_op_stringview_iter_slice(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef stringview_iter_obj, start, end, stringref_obj_new, value, + iter_pos_addr, code_points_count; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_I32(code_points_count); + POP_GC_REF(stringview_iter_obj); + + if (!(iter_pos_addr = get_stringview_iter_pos_addr(comp_ctx, func_ctx, + stringview_iter_obj))) { + goto fail; + } + + if (!(start = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, iter_pos_addr, + "get_iter_pos"))) { + goto fail; + } + LLVMSetAlignment(start, 4); + + if (!(end = LLVMBuildAdd(comp_ctx->builder, start, code_points_count, + "calc_slice_end"))) { + goto fail; + } + + if (!(value = aot_call_wasm_string_slice(comp_ctx, func_ctx, + stringview_iter_obj, start, end, + STRING_VIEW_ITER))) { + goto fail; + } + CHECK_STRING_OBJ(value); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, value, + WASM_TYPE_STRINGREF, 0, + &stringref_obj_new)) { + goto fail; + } + CHECK_STRINGREF_OBJ(stringref_obj_new); + + PUSH_GC_REF(stringref_obj_new); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_new_array(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 encoding) +{ + LLVMValueRef start, end, count, str_obj, stringref_obj, array_obj, + elem_data_ptr; + LLVMValueRef param_values[5], func, value; + LLVMTypeRef param_types[5], ret_type, func_type, func_ptr_type; + DEFINE_STRINGREF_CHECK_VAR(); + + if (!aot_gen_commit_values(comp_ctx->aot_frame)) + return false; + + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, true)) + return false; + + POP_I32(end); + POP_I32(start); + POP_GC_REF(array_obj); + + if (!aot_array_obj_elem_addr( + comp_ctx, func_ctx, array_obj, start, &elem_data_ptr, + encoding == WTF16 ? PACKED_TYPE_I16 : PACKED_TYPE_I8)) { + goto fail; + } + + if (!(count = LLVMBuildSub(comp_ctx->builder, end, start, "calc_count"))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + GET_AOT_FUNCTION(wasm_string_new_with_encoding, 3); + + /* Call function wasm_struct_obj_new() */ + param_values[0] = elem_data_ptr; + param_values[1] = count; + param_values[2] = I32_CONST(encoding); + + if (!(str_obj = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 3, "wasm_string_new"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + CHECK_STRING_OBJ(str_obj); + + if (!aot_call_wasm_stringref_obj_new(comp_ctx, func_ctx, str_obj, + WASM_TYPE_STRINGREF, 0, + &stringref_obj)) { + goto fail; + } + CHECK_STRINGREF_OBJ(stringref_obj); + + PUSH_GC_REF(stringref_obj); + + return true; +fail: + return false; +} + +bool +aot_compile_op_string_encode_array(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 encoding) +{ + LLVMValueRef param_values[6], func, value, count, start, str_obj, + stringref_obj, array_obj, elem_data_ptr, array_len; + LLVMTypeRef param_types[6], ret_type, func_type, func_ptr_type; + LLVMBasicBlockRef check_string_encode_succ, check_array_index_succ; + LLVMValueRef cmp; + + POP_I32(start); + POP_GC_REF(array_obj); + POP_GC_REF(stringref_obj); + + if (!(str_obj = + aot_stringref_obj_get_value(comp_ctx, func_ctx, stringref_obj))) { + goto fail; + } + + if (!aot_array_obj_length(comp_ctx, array_obj, &array_len)) + goto fail; + + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, start, array_len, + "check_array_index"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + ADD_BASIC_BLOCK(check_array_index_succ, "check array index succ"); + MOVE_BLOCK_AFTER_CURR(check_array_index_succ); + + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ARRAY_IDX_OOB, true, cmp, + check_array_index_succ)) { + goto fail; + } + + if (!aot_array_obj_elem_addr( + comp_ctx, func_ctx, stringref_obj, start, &elem_data_ptr, + encoding == WTF16 ? PACKED_TYPE_I16 : PACKED_TYPE_I8)) { + goto fail; + } + + if (!(count = aot_call_wasm_string_measure(comp_ctx, func_ctx, + stringref_obj, encoding))) { + goto fail; + } + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + param_types[4] = INT8_PTR_TYPE; + param_types[5] = I32_TYPE; + ret_type = I32_TYPE; + + GET_AOT_FUNCTION(wasm_string_encode, 6); + + /* Call function wasm_string_measure() */ + param_values[0] = str_obj; + param_values[1] = start; + param_values[2] = count; + param_values[3] = elem_data_ptr; + param_values[4] = I8_PTR_NULL; + param_values[5] = I32_CONST(encoding); + + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 6, "string_encode"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + CHECK_STRING_ENCODE(value); + + PUSH_I32(value); + + return true; +fail: + return false; +} + +#endif /* WASM_ENABLE_STRINGREF != 0 */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_stringref.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_stringref.h new file mode 100644 index 0000000..a1b68a4 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_stringref.h @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_STRINGREF_H_ +#define _AOT_EMIT_STRINGREF_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_string_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 encoding); + +bool +aot_compile_op_string_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 contents); + +bool +aot_compile_op_string_measure(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 encoding); + +bool +aot_compile_op_string_encode(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 mem_idx, uint32 encoding); + +bool +aot_compile_op_string_concat(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_string_eq(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_op_string_is_usv_sequence(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_string_as_wtf8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_wtf8_advance(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_wtf8_encode(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 mem_idx, + uint32 encoding); + +bool +aot_compile_op_stringview_wtf8_slice(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_string_as_wtf16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_wtf16_length(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_wtf16_get_codeunit(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_wtf16_encode(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + uint32 mem_idx); + +bool +aot_compile_op_stringview_wtf16_slice(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_string_as_iter(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_iter_next(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_iter_advance(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_iter_rewind(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_stringview_iter_slice(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_op_string_new_array(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 encoding); + +bool +aot_compile_op_string_encode_array(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 encoding); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_STRINGREF_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_table.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_table.c new file mode 100644 index 0000000..6ce3295 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_table.c @@ -0,0 +1,618 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_table.h" +#include "aot_emit_exception.h" +#include "../aot/aot_runtime.h" +#if WASM_ENABLE_GC != 0 +#include "aot_emit_gc.h" +#endif + +uint64 +get_tbl_inst_offset(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, uint32 tbl_idx) +{ + uint64 offset = 0, i = 0; + AOTImportTable *imp_tbls = comp_ctx->comp_data->import_tables; + AOTTable *tbls = comp_ctx->comp_data->tables; + + offset = + offsetof(AOTModuleInstance, global_table_data.bytes) + + (uint64)comp_ctx->comp_data->memory_count * sizeof(AOTMemoryInstance) + /* Get global data size according to target info */ + + (comp_ctx->pointer_size == sizeof(uint64) + ? comp_ctx->comp_data->global_data_size_64bit + : comp_ctx->comp_data->global_data_size_32bit); + + while (i < tbl_idx && i < comp_ctx->comp_data->import_table_count) { + offset += offsetof(AOTTableInstance, elems); + /* avoid loading from current AOTTableInstance */ + offset += + (uint64)comp_ctx->pointer_size + * aot_get_imp_tbl_data_slots(imp_tbls + i, comp_ctx->is_jit_mode); + ++i; + } + + if (i == tbl_idx) { + return offset; + } + + tbl_idx -= comp_ctx->comp_data->import_table_count; + i -= comp_ctx->comp_data->import_table_count; + while (i < tbl_idx && i < comp_ctx->comp_data->table_count) { + offset += offsetof(AOTTableInstance, elems); + /* avoid loading from current AOTTableInstance */ + offset += (uint64)comp_ctx->pointer_size + * aot_get_tbl_data_slots(tbls + i, comp_ctx->is_jit_mode); + ++i; + } + + return offset; +} + +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx) +{ + const AOTCompData *comp_data = comp_ctx->comp_data; + uint32 table_count = comp_data->import_table_count + comp_data->table_count; + uint64 offset = get_tbl_inst_offset(comp_ctx, NULL, table_count); + uint32 offset_32 = (uint32)offset; + bh_assert(offset <= UINT32_MAX); + offset_32 = align_uint(offset_32, 8); + return offset_32; +} + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + +LLVMValueRef +aot_compile_get_tbl_inst(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx) +{ + LLVMValueRef offset, tbl_inst; + + if (!(offset = + I64_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx)))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(tbl_inst = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "tbl_inst"))) { + HANDLE_FAILURE("LLVMBuildInBoundsGEP"); + goto fail; + } + + return tbl_inst; +fail: + return NULL; +} + +bool +aot_compile_op_elem_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_seg_idx) +{ + LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; + LLVMValueRef param_values[2], ret_value, func, value; + + /* void aot_drop_table_seg(AOTModuleInstance *, uint32 ) */ + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + ret_type = VOID_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_drop_table_seg, 2); + else + GET_AOT_FUNCTION(aot_drop_table_seg, 2); + + param_values[0] = func_ctx->aot_inst; + if (!(param_values[1] = I32_CONST(tbl_seg_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + /* "" means return void */ + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 2, ""))) { + HANDLE_FAILURE("LLVMBuildCall"); + goto fail; + } + + return true; +fail: + return false; +} + +static bool +aot_check_table_access(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx, LLVMValueRef elem_idx) +{ + LLVMValueRef offset, tbl_sz, cmp_elem_idx; + LLVMBasicBlockRef check_elem_idx_succ; + + /* get the cur size of the table instance */ + if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) + + offsetof(AOTTableInstance, cur_size)))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(tbl_sz = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "cur_size_i8p"))) { + HANDLE_FAILURE("LLVMBuildInBoundsGEP"); + goto fail; + } + + if (!(tbl_sz = LLVMBuildBitCast(comp_ctx->builder, tbl_sz, INT32_PTR_TYPE, + "cur_siuze_i32p"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(tbl_sz = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, tbl_sz, + "cur_size"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + goto fail; + } + + /* Check if (uint32)elem index >= table size */ + if (!(cmp_elem_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, elem_idx, + tbl_sz, "cmp_elem_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* Throw exception if elem index >= table size */ + if (!(check_elem_idx_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "check_elem_idx_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_elem_idx_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, true, + cmp_elem_idx, check_elem_idx_succ))) + goto fail; + + return true; +fail: + return false; +} + +bool +aot_compile_op_table_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx) +{ + LLVMValueRef elem_idx, offset, func_idx; + LLVMValueRef table_elem_base, table_elem_addr, table_elem; + + POP_I32(elem_idx); + + if (!aot_check_table_access(comp_ctx, func_ctx, tbl_idx, elem_idx)) { + goto fail; + } + + /* load data as i32* */ + if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) + + offsetof(AOTTableInstance, elems)))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(table_elem_base = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, + 1, "table_elem_base_i8p"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + + /* Load function object reference or function index */ + if (comp_ctx->enable_gc) { + if (!(table_elem_base = + LLVMBuildBitCast(comp_ctx->builder, table_elem_base, + GC_REF_PTR_TYPE, "table_elem_base"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(table_elem_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, GC_REF_TYPE, table_elem_base, &elem_idx, 1, + "table_elem_addr"))) { + HANDLE_FAILURE("LLVMBuildNUWAdd"); + goto fail; + } + + if (!(table_elem = LLVMBuildLoad2(comp_ctx->builder, GC_REF_TYPE, + table_elem_addr, "table_elem"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + goto fail; + } + + PUSH_GC_REF(table_elem); + } + else { + if (!(table_elem_base = + LLVMBuildBitCast(comp_ctx->builder, table_elem_base, + INTPTR_T_PTR_TYPE, "table_elem_base"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(table_elem_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INTPTR_T_TYPE, table_elem_base, &elem_idx, + 1, "table_elem_addr"))) { + HANDLE_FAILURE("LLVMBuildNUWAdd"); + goto fail; + } + + if (!(table_elem = LLVMBuildLoad2(comp_ctx->builder, INTPTR_T_TYPE, + table_elem_addr, "table_elem"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + goto fail; + } + + if (!(func_idx = LLVMBuildIntCast2(comp_ctx->builder, table_elem, + I32_TYPE, true, "func_idx"))) { + HANDLE_FAILURE("LLVMBuildIntCast"); + goto fail; + } + + PUSH_I32(func_idx); + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_table_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx) +{ + LLVMValueRef val = NULL, elem_idx, offset, table_elem_base, table_elem_addr; + + if (comp_ctx->enable_gc) + POP_GC_REF(val); + else { + POP_I32(val); + + if (!(val = LLVMBuildIntCast2(comp_ctx->builder, val, INTPTR_T_TYPE, + true, "val_intptr_t"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + } + + POP_I32(elem_idx); + + if (!aot_check_table_access(comp_ctx, func_ctx, tbl_idx, elem_idx)) { + goto fail; + } + + /* load data as gc_obj_ref* or i32* */ + if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) + + offsetof(AOTTableInstance, elems)))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(table_elem_base = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, + 1, "table_elem_base_i8p"))) { + HANDLE_FAILURE("LLVMBuildInBoundsGEP"); + goto fail; + } + + if (comp_ctx->enable_gc) { + if (!(table_elem_base = + LLVMBuildBitCast(comp_ctx->builder, table_elem_base, + GC_REF_PTR_TYPE, "table_elem_base"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(table_elem_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, GC_REF_TYPE, table_elem_base, &elem_idx, 1, + "table_elem_addr"))) { + HANDLE_FAILURE("LLVMBuildInBoundsGEP"); + goto fail; + } + } + else { + if (!(table_elem_base = + LLVMBuildBitCast(comp_ctx->builder, table_elem_base, + INTPTR_T_PTR_TYPE, "table_elem_base"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(table_elem_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INTPTR_T_TYPE, table_elem_base, &elem_idx, + 1, "table_elem_addr"))) { + HANDLE_FAILURE("LLVMBuildInBoundsGEP"); + goto fail; + } + } + + if (!(LLVMBuildStore(comp_ctx->builder, val, table_elem_addr))) { + HANDLE_FAILURE("LLVMBuildStore"); + goto fail; + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_table_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx, uint32 tbl_seg_idx) + +{ + LLVMValueRef func, param_values[6], value; + LLVMTypeRef param_types[6], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = I32_TYPE; + param_types[4] = I32_TYPE; + param_types[5] = I32_TYPE; + ret_type = VOID_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_init, 6); + else + GET_AOT_FUNCTION(aot_table_init, 6); + + param_values[0] = func_ctx->aot_inst; + + if (!(param_values[1] = I32_CONST(tbl_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(param_values[2] = I32_CONST(tbl_seg_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + /* n */ + POP_I32(param_values[3]); + /* s */ + POP_I32(param_values[4]); + /* d */ + POP_I32(param_values[5]); + + /* "" means return void */ + if (!(LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, 6, + ""))) { + HANDLE_FAILURE("LLVMBuildCall"); + goto fail; + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_table_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 src_tbl_idx, uint32 dst_tbl_idx) +{ + LLVMTypeRef param_types[6], ret_type, func_type, func_ptr_type; + LLVMValueRef func, param_values[6], value; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = I32_TYPE; + param_types[4] = I32_TYPE; + param_types[5] = I32_TYPE; + ret_type = VOID_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_copy, 6); + else + GET_AOT_FUNCTION(aot_table_copy, 6); + + param_values[0] = func_ctx->aot_inst; + + if (!(param_values[1] = I32_CONST(src_tbl_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(param_values[2] = I32_CONST(dst_tbl_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + /* n */ + POP_I32(param_values[3]); + /* s */ + POP_I32(param_values[4]); + /* d */ + POP_I32(param_values[5]); + + /* "" means return void */ + if (!(LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, 6, + ""))) { + HANDLE_FAILURE("LLVMBuildCall"); + goto fail; + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_table_size(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx) +{ + LLVMValueRef offset, tbl_sz; + + if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) + + offsetof(AOTTableInstance, cur_size)))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(tbl_sz = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "tbl_sz_ptr_i8"))) { + HANDLE_FAILURE("LLVMBuildInBoundsGEP"); + goto fail; + } + + if (!(tbl_sz = LLVMBuildBitCast(comp_ctx->builder, tbl_sz, INT32_PTR_TYPE, + "tbl_sz_ptr"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + if (!(tbl_sz = + LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, tbl_sz, "tbl_sz"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + goto fail; + } + + PUSH_I32(tbl_sz); + + return true; +fail: + return false; +} + +bool +aot_compile_op_table_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx) +{ + LLVMTypeRef param_types[4], ret_type, func_type, func_ptr_type; + LLVMValueRef func, param_values[4], ret, value; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + ret_type = I32_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_grow, 4); + else + GET_AOT_FUNCTION(aot_table_grow, 4); + + param_values[0] = func_ctx->aot_inst; + + if (!(param_values[1] = I32_CONST(tbl_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + /* n */ + POP_I32(param_values[2]); + /* v */ + + if (comp_ctx->enable_gc) { + POP_GC_REF(param_values[3]); + if (!(param_values[3] = + LLVMBuildBitCast(comp_ctx->builder, param_values[3], + INT8_PTR_TYPE, "table_elem_i8p"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + } + else { + POP_I32(param_values[3]); + if (!(param_values[3] = + LLVMBuildIntToPtr(comp_ctx->builder, param_values[3], + INT8_PTR_TYPE, "table_elem_i8p"))) { + HANDLE_FAILURE("LLVMBuildIntToPtr"); + goto fail; + } + } + + if (!(ret = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 4, "table_grow"))) { + HANDLE_FAILURE("LLVMBuildCall"); + goto fail; + } + + PUSH_I32(ret); + + return true; +fail: + return false; +} + +bool +aot_compile_op_table_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx) +{ + LLVMTypeRef param_types[5], ret_type, func_type, func_ptr_type; + LLVMValueRef func, param_values[5], value; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + param_types[3] = INT8_PTR_TYPE; + param_types[4] = I32_TYPE; + ret_type = VOID_TYPE; + + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_fill, 5); + else + GET_AOT_FUNCTION(aot_table_fill, 5); + + param_values[0] = func_ctx->aot_inst; + + if (!(param_values[1] = I32_CONST(tbl_idx))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + /* n */ + POP_I32(param_values[2]); + /* v */ + + if (comp_ctx->enable_gc) { + POP_GC_REF(param_values[3]); + if (!(param_values[3] = + LLVMBuildBitCast(comp_ctx->builder, param_values[3], + INT8_PTR_TYPE, "table_elem_i8p"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + } + else { + POP_I32(param_values[3]); + if (!(param_values[3] = + LLVMBuildIntToPtr(comp_ctx->builder, param_values[3], + INT8_PTR_TYPE, "table_elem_i8p"))) { + HANDLE_FAILURE("LLVMBuildIntToPtr"); + goto fail; + } + } + /* i */ + POP_I32(param_values[4]); + + /* "" means return void */ + if (!(LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, 5, + ""))) { + HANDLE_FAILURE("LLVMBuildCall"); + goto fail; + } + + return true; +fail: + return false; +} + +#endif /* WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC !=0 */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_table.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_table.h new file mode 100644 index 0000000..f294cca --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_table.h @@ -0,0 +1,62 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_TABLE_H_ +#define _AOT_EMIT_TABLE_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_elem_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_seg_idx); + +bool +aot_compile_op_table_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx); + +bool +aot_compile_op_table_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx); + +bool +aot_compile_op_table_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx, uint32 tbl_seg_idx); + +bool +aot_compile_op_table_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 src_tbl_idx, uint32 dst_tbl_idx); + +bool +aot_compile_op_table_size(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx); + +bool +aot_compile_op_table_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx); + +bool +aot_compile_op_table_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx); + +uint64 +get_tbl_inst_offset(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, uint32 tbl_idx); + +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx); + +LLVMValueRef +aot_compile_get_tbl_inst(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 tbl_idx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif +#endif diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_variable.c b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_variable.c new file mode 100644 index 0000000..6cd3221 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_variable.c @@ -0,0 +1,331 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_variable.h" +#include "aot_emit_exception.h" +#include "../aot/aot_runtime.h" + +#define CHECK_LOCAL(idx) \ + do { \ + if (idx >= func_ctx->aot_func->func_type->param_count \ + + func_ctx->aot_func->local_count) { \ + aot_set_last_error("local index out of range"); \ + return false; \ + } \ + } while (0) + +static uint8 +get_local_type(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 local_idx) +{ + AOTFunc *aot_func = func_ctx->aot_func; + uint32 param_count = aot_func->func_type->param_count; + uint8 local_type; + + local_type = local_idx < param_count + ? aot_func->func_type->types[local_idx] + : aot_func->local_types_wp[local_idx - param_count]; + + if (comp_ctx->enable_gc && aot_is_type_gc_reftype(local_type)) + local_type = VALUE_TYPE_GC_REF; + + return local_type; +} + +bool +aot_compile_op_get_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 local_idx) +{ + char name[32]; + LLVMValueRef value; + AOTValue *aot_value_top; + uint8 local_type; + + CHECK_LOCAL(local_idx); + + local_type = get_local_type(comp_ctx, func_ctx, local_idx); + + snprintf(name, sizeof(name), "%s%d%s", "local", local_idx, "#"); + if (!(value = LLVMBuildLoad2(comp_ctx->builder, TO_LLVM_TYPE(local_type), + func_ctx->locals[local_idx], name))) { + aot_set_last_error("llvm build load fail"); + return false; + } + + PUSH(value, local_type); + + aot_value_top = + func_ctx->block_stack.block_list_end->value_stack.value_list_end; + aot_value_top->is_local = true; + aot_value_top->local_idx = local_idx; + return true; + +fail: + return false; +} + +static bool +aot_compile_op_set_or_tee_local(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint32 local_idx, + bool is_tee_local) +{ + LLVMValueRef value; + uint8 local_type; + uint32 n; + + CHECK_LOCAL(local_idx); + + local_type = get_local_type(comp_ctx, func_ctx, local_idx); + + POP(value, local_type); + + if (comp_ctx->aot_frame) { + /* Get the slot index */ + n = func_ctx->aot_func->local_offsets[local_idx]; + bh_assert(comp_ctx->aot_frame->lp[n].type == local_type); + + switch (local_type) { + case VALUE_TYPE_I32: + set_local_i32(comp_ctx->aot_frame, n, value); + break; + case VALUE_TYPE_I64: + set_local_i64(comp_ctx->aot_frame, n, value); + break; + case VALUE_TYPE_F32: + set_local_f32(comp_ctx->aot_frame, n, value); + break; + case VALUE_TYPE_F64: + set_local_f64(comp_ctx->aot_frame, n, value); + break; + case VALUE_TYPE_V128: + set_local_v128(comp_ctx->aot_frame, n, value); + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + set_local_ref(comp_ctx->aot_frame, n, value, local_type); + break; +#if WASM_ENABLE_GC != 0 + case VALUE_TYPE_GC_REF: + set_local_gc_ref(comp_ctx->aot_frame, n, value, local_type); + break; +#endif + default: + bh_assert(0); + break; + } + } + + if (!LLVMBuildStore(comp_ctx->builder, value, + func_ctx->locals[local_idx])) { + aot_set_last_error("llvm build store fail"); + return false; + } + + if (is_tee_local) { + PUSH(value, local_type); + } + + aot_checked_addr_list_del(func_ctx, local_idx); + return true; + +fail: + return false; +} + +bool +aot_compile_op_set_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 local_idx) +{ + return aot_compile_op_set_or_tee_local(comp_ctx, func_ctx, local_idx, + false); +} + +bool +aot_compile_op_tee_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 local_idx) +{ + return aot_compile_op_set_or_tee_local(comp_ctx, func_ctx, local_idx, true); +} + +static bool +compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 global_idx, bool is_set, bool is_aux_stack) +{ + const AOTCompData *comp_data = comp_ctx->comp_data; + uint32 import_global_count = comp_data->import_global_count; + uint32 global_base_offset; + uint32 global_offset; + uint8 global_type; + LLVMValueRef offset, global_ptr, global, res; + LLVMTypeRef ptr_type = NULL; + + global_base_offset = + offsetof(AOTModuleInstance, global_table_data.bytes) + + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count; + + bh_assert(global_idx < import_global_count + comp_data->global_count); + + if (global_idx < import_global_count) { + global_offset = + global_base_offset + /* Get global data offset according to target info */ + + (comp_ctx->pointer_size == sizeof(uint64) + ? comp_data->import_globals[global_idx].data_offset_64bit + : comp_data->import_globals[global_idx].data_offset_32bit); + global_type = comp_data->import_globals[global_idx].type; + } + else { + global_offset = + global_base_offset + /* Get global data offset according to target info */ + + (comp_ctx->pointer_size == sizeof(uint64) + ? comp_data->globals[global_idx - import_global_count] + .data_offset_64bit + : comp_data->globals[global_idx - import_global_count] + .data_offset_32bit); + global_type = comp_data->globals[global_idx - import_global_count].type; + } + + if (comp_ctx->enable_gc && aot_is_type_gc_reftype(global_type)) + global_type = VALUE_TYPE_GC_REF; + + offset = I32_CONST(global_offset); + if (!(global_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "global_ptr_tmp"))) { + aot_set_last_error("llvm build in bounds gep failed."); + return false; + } + + switch (global_type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: + ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F32: + ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_F64: + ptr_type = F64_PTR_TYPE; + break; + case VALUE_TYPE_V128: + ptr_type = V128_PTR_TYPE; + break; +#if WASM_ENABLE_GC != 0 + case VALUE_TYPE_GC_REF: + ptr_type = GC_REF_PTR_TYPE; + break; +#endif + default: + bh_assert("unknown type"); + break; + } + + if (!(global_ptr = LLVMBuildBitCast(comp_ctx->builder, global_ptr, ptr_type, + "global_ptr"))) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + if (!is_set) { + if (!(global = + LLVMBuildLoad2(comp_ctx->builder, TO_LLVM_TYPE(global_type), + global_ptr, "global"))) { + aot_set_last_error("llvm build load failed."); + return false; + } + /* All globals' data is 4-byte aligned */ + LLVMSetAlignment(global, 4); + PUSH(global, global_type); + } + else { + POP(global, global_type); + + if (is_aux_stack && comp_ctx->enable_aux_stack_check) { + LLVMBasicBlockRef block_curr = + LLVMGetInsertBlock(comp_ctx->builder); + LLVMBasicBlockRef check_overflow_succ, check_underflow_succ; + LLVMValueRef cmp, global_i64; + + /* Add basic blocks */ + if (!(check_overflow_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, + "check_overflow_succ"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + LLVMMoveBasicBlockAfter(check_overflow_succ, block_curr); + + if (!(check_underflow_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, + "check_underflow_succ"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + LLVMMoveBasicBlockAfter(check_underflow_succ, check_overflow_succ); + + if (!(global_i64 = LLVMBuildZExt(comp_ctx->builder, global, + I64_TYPE, "global_i64"))) { + aot_set_last_error("llvm build zext failed."); + return false; + } + + /* Check aux stack overflow */ + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntULE, global_i64, + func_ctx->aux_stack_bound, "cmp"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_AUX_STACK_OVERFLOW, + true, cmp, check_overflow_succ)) { + return false; + } + + /* Check aux stack underflow */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, check_overflow_succ); + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGT, global_i64, + func_ctx->aux_stack_bottom, "cmp"))) { + aot_set_last_error("llvm build icmp failed."); + return false; + } + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_AUX_STACK_UNDERFLOW, true, cmp, + check_underflow_succ)) { + return false; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, check_underflow_succ); + } + + if (!(res = LLVMBuildStore(comp_ctx->builder, global, global_ptr))) { + aot_set_last_error("llvm build store failed."); + return false; + } + /* All globals' data is 4-byte aligned */ + LLVMSetAlignment(res, 4); + } + + return true; +fail: + return false; +} + +bool +aot_compile_op_get_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 global_idx) +{ + return compile_global(comp_ctx, func_ctx, global_idx, false, false); +} + +bool +aot_compile_op_set_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 global_idx, bool is_aux_stack) +{ + return compile_global(comp_ctx, func_ctx, global_idx, true, is_aux_stack); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_emit_variable.h b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_variable.h new file mode 100644 index 0000000..28c0bd0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_emit_variable.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_EMIT_VARIABLE_H_ +#define _AOT_EMIT_VARIABLE_H_ + +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_op_get_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 local_idx); + +bool +aot_compile_op_set_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 local_idx); + +bool +aot_compile_op_tee_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 local_idx); + +bool +aot_compile_op_get_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 global_idx); + +bool +aot_compile_op_set_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 global_idx, bool is_aux_stack); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_EMIT_VARIABLE_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_llvm.c b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm.c new file mode 100644 index 0000000..3af56e8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm.c @@ -0,0 +1,3996 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_llvm.h" +#include "aot_llvm_extra2.h" +#include "aot_compiler.h" +#include "aot_emit_exception.h" +#include "aot_emit_table.h" +#include "../aot/aot_runtime.h" +#include "../aot/aot_intrinsic.h" +#include "../interpreter/wasm_runtime.h" + +#if WASM_ENABLE_DEBUG_AOT != 0 +#include "debug/dwarf_extractor.h" +#endif + +static bool +create_native_symbol(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +static bool +create_native_stack_bound(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); +static bool +create_native_stack_top_min(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); +static bool +create_func_ptrs(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +LLVMTypeRef +wasm_type_to_llvm_type(const AOTCompContext *comp_ctx, + const AOTLLVMTypes *llvm_types, uint8 wasm_type) +{ + switch (wasm_type) { + case VALUE_TYPE_I32: + return llvm_types->int32_type; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + if (comp_ctx->enable_ref_types) + return llvm_types->int32_type; + else { + bh_assert(comp_ctx->enable_gc); + return llvm_types->gc_ref_type; + } + case VALUE_TYPE_I64: + return llvm_types->int64_type; + case VALUE_TYPE_F32: + return llvm_types->float32_type; + case VALUE_TYPE_F64: + return llvm_types->float64_type; + case VALUE_TYPE_V128: + return llvm_types->i64x2_vec_type; + case VALUE_TYPE_VOID: + return llvm_types->void_type; + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + /* case REF_TYPE_FUNCREF: */ + /* case REF_TYPE_EXTERNREF: */ + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + case VALUE_TYPE_GC_REF: + bh_assert(comp_ctx->enable_gc); + return llvm_types->gc_ref_type; + default: + break; + } + bh_assert(0); + return NULL; +} + +static LLVMValueRef +aot_add_llvm_func1(const AOTCompContext *comp_ctx, LLVMModuleRef module, + uint32 func_index, uint32 param_count, LLVMTypeRef func_type, + const char *prefix) +{ + char func_name[48] = { 0 }; + LLVMValueRef func; + LLVMValueRef local_value; + uint32 i, j; + + /* Add LLVM function */ + snprintf(func_name, sizeof(func_name), "%s%d", prefix, func_index); + if (!(func = LLVMAddFunction(module, func_name, func_type))) { + aot_set_last_error("add LLVM function failed."); + return NULL; + } + + j = 0; + local_value = LLVMGetParam(func, j++); + LLVMSetValueName(local_value, "exec_env"); + + /* Set parameter names */ + for (i = 0; i < param_count; i++) { + local_value = LLVMGetParam(func, j++); + LLVMSetValueName(local_value, ""); + } + + return func; +} + +/* + * create a basic func_ctx enough to call aot_emit_exception. + * + * that is: + * - exec_env + * - aot_inst + * - native_symbol (if is_indirect_mode) + */ +static bool +create_basic_func_context(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef aot_inst_offset = I32_TWO, aot_inst_addr; + + /* Save the parameters for fast access */ + func_ctx->exec_env = LLVMGetParam(func_ctx->func, 0); + + /* Get aot inst address, the layout of exec_env is: + exec_env->next, exec_env->prev, exec_env->module_inst, and argv_buf */ + if (!(aot_inst_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, + &aot_inst_offset, 1, "aot_inst_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + goto fail; + } + + /* Load aot inst */ + if (!(func_ctx->aot_inst = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + aot_inst_addr, "aot_inst"))) { + aot_set_last_error("llvm build load failed"); + goto fail; + } + + if (comp_ctx->is_indirect_mode + && !create_native_symbol(comp_ctx, func_ctx)) { + goto fail; + } + + return true; +fail: + return false; +} + +/* + * return if the "precheck" wrapper function can use tail call optimization + */ +bool +aot_target_precheck_can_use_musttail(const AOTCompContext *comp_ctx) +{ + if (!strcmp(comp_ctx->target_arch, "xtensa")) { + /* + * xtensa windowed ABI doesn't have tail call optimization. + * + * Note: as of writing this, the xtensa version of LLVM + * simply ignores the musttail attribute. + * https://github.com/espressif/llvm-project/pull/73 + */ + return false; + } + if (!strcmp(comp_ctx->target_arch, "riscv32") + || !strcmp(comp_ctx->target_arch, "riscv64")) { + /* + * REVISIT: actually, riscv can use tail call optimization + * in some cases. I (yamamoto) don't know the exact conditions + * though. + */ + return false; + } + if (!strcmp(comp_ctx->target_arch, "mips")) { + /* + * cf. + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/2412 + */ + return false; + } + if (strstr(comp_ctx->target_arch, "thumb")) { + /* + * cf. + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/2412 + */ + return false; + } + /* + * x86-64/i386: true + * + * others: assume true for now + */ + return true; +} + +unsigned int +aot_estimate_stack_usage_for_function_call(const AOTCompContext *comp_ctx, + const AOTFuncType *callee_func_type) +{ + /* + * Estimate how much stack is necessary to make a function call. + * This does not include the stack consumption of the callee function. + * + * For precise estimation, ideally this function needs to be + * target-specific. + * However, this implementation aims to be target-independent, + * allowing a small overstimation, which is probably ok for our purpose. + * (overflow detection and memory profiling) + * On the other hand, an underestimation should be avoided as it + * can cause more serious problems like silent data corruptions. + * + * Assumptions: + * + * - the first result is returned via a register. + * + * - all parameters, including exec_env and pointers to non-first + * results, are passed via stack. + * (this is a bit pessimistic than many of real calling conventions, + * where some of parameters are passed via register.) + * + * - N-byte value needs N-byte alignment on stack. + * + * - a value smaller than a pointer is extended. + * (eg. 4 byte values are extended to 8 byte on x86-64.) + */ + + const unsigned int param_count = callee_func_type->param_count; + const unsigned int result_count = callee_func_type->result_count; + unsigned int size = 0; + unsigned int i; + unsigned int nb; + + if (!strcmp(comp_ctx->target_arch, "xtensa")) { + /* + * In the xtensa windowed ABI, outgoing arguments are already + * included in the callee's stack frame size, which equals to + * the operand of the ENTRY instruction and what LLVM + * MFI->getStackSize returns. + */ + return 0; + } + + /* exec_env */ + size = comp_ctx->pointer_size; + + /* parameters */ + for (i = 0; i < param_count; i++) { + nb = wasm_value_type_cell_num(callee_func_type->types[i]) * 4; + if (nb < comp_ctx->pointer_size) { + nb = comp_ctx->pointer_size; + } + size = align_uint(size, nb) + nb; + } + + /* pointers to results */ + nb = comp_ctx->pointer_size; + for (i = 1; i < result_count; i++) { + size = align_uint(size, nb) + nb; + } + + /* return address */ + nb = comp_ctx->pointer_size; + size = align_uint(size, nb) + nb; + + /* + * some extra for possible arch-dependent things like + * 16-byte alignment for x86_64. + */ + size += 16; + return size; +} + +/* + * a "precheck" function performs a few things before calling wrapped_func. + * + * - update native_stack_top_min if necessary + * - stack overflow check (if it does, trap) + */ +static bool +aot_build_precheck_function(AOTCompContext *comp_ctx, LLVMModuleRef module, + LLVMValueRef precheck_func, uint32 func_index, + LLVMTypeRef func_type, LLVMValueRef wrapped_func) +{ + LLVMBasicBlockRef begin = NULL; + LLVMBasicBlockRef check_top_block = NULL; + LLVMBasicBlockRef update_top_block = NULL; + LLVMBasicBlockRef stack_bound_check_block = NULL; + LLVMBasicBlockRef call_wrapped_func_block = NULL; + LLVMValueRef *params = NULL; + + begin = LLVMAppendBasicBlockInContext(comp_ctx->context, precheck_func, + "begin"); + check_top_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, precheck_func, "check_top_block"); + if (comp_ctx->enable_stack_estimation) { + update_top_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, precheck_func, "update_top_block"); + if (!update_top_block) { + goto fail; + } + } + stack_bound_check_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, precheck_func, "stack_bound_check_block"); + call_wrapped_func_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, precheck_func, "call_wrapped_func"); + if (!begin || !check_top_block || !stack_bound_check_block + || !call_wrapped_func_block) { + goto fail; + } + LLVMBuilderRef b = comp_ctx->builder; + LLVMPositionBuilderAtEnd(b, begin); + + /* create a temporary minimum func_ctx */ + AOTFuncContext tmp; + AOTFuncContext *func_ctx = &tmp; + memset(func_ctx, 0, sizeof(*func_ctx)); + func_ctx->func = precheck_func; + func_ctx->module = module; + func_ctx->aot_func = comp_ctx->comp_data->funcs[func_index]; +#if WASM_ENABLE_DEBUG_AOT != 0 + func_ctx->debug_func = NULL; +#endif + if (!create_basic_func_context(comp_ctx, func_ctx)) + goto fail; + if (comp_ctx->enable_stack_bound_check + && !create_native_stack_bound(comp_ctx, func_ctx)) + goto fail; + if (comp_ctx->enable_stack_estimation + && !create_native_stack_top_min(comp_ctx, func_ctx)) { + goto fail; + } + + uint32 param_count = LLVMCountParams(precheck_func); + uint32 sz = param_count * (uint32)sizeof(LLVMValueRef); + params = wasm_runtime_malloc(sz); + if (params == NULL) { + goto fail; + } + LLVMGetParams(precheck_func, params); + + const bool is_64bit = comp_ctx->pointer_size == sizeof(uint64); + LLVMTypeRef uintptr_type; + if (is_64bit) + uintptr_type = I64_TYPE; + else + uintptr_type = I32_TYPE; + + /* + * load the stack pointer + */ + LLVMValueRef sp_ptr = LLVMBuildAlloca(b, I32_TYPE, "sp_ptr"); + if (!sp_ptr) { + goto fail; + } + LLVMValueRef sp = LLVMBuildPtrToInt(b, sp_ptr, uintptr_type, "sp"); + if (!sp) { + goto fail; + } + + /* + * load the value for this wrapped function from the stack_sizes array + */ + LLVMValueRef stack_sizes; + if (comp_ctx->is_indirect_mode) { + uint32 offset_u32; + LLVMValueRef offset; + LLVMValueRef stack_sizes_p; + + offset_u32 = get_module_inst_extra_offset(comp_ctx); + offset_u32 += offsetof(AOTModuleInstanceExtra, stack_sizes); + offset = I32_CONST(offset_u32); + if (!offset) { + goto fail; + } + stack_sizes_p = + LLVMBuildInBoundsGEP2(b, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "aot_inst_stack_sizes_p"); + if (!stack_sizes_p) { + goto fail; + } + stack_sizes = + LLVMBuildLoad2(b, INT32_PTR_TYPE, stack_sizes_p, "stack_sizes"); + if (!stack_sizes) { + goto fail; + } + } + else { + stack_sizes = comp_ctx->stack_sizes; + } + LLVMValueRef func_index_const = I32_CONST(func_index); + LLVMValueRef sizes = + LLVMBuildBitCast(b, stack_sizes, INT32_PTR_TYPE, "sizes"); + if (!sizes) { + goto fail; + } + LLVMValueRef sizep = LLVMBuildInBoundsGEP2(b, I32_TYPE, sizes, + &func_index_const, 1, "sizep"); + if (!sizep) { + goto fail; + } + LLVMValueRef size32 = LLVMBuildLoad2(b, I32_TYPE, sizep, "size32"); + if (!size32) { + goto fail; + } + LLVMValueRef size; + if (is_64bit) { + size = LLVMBuildZExt(b, size32, uintptr_type, "size"); + if (!size) { + goto fail; + } + } + else { + size = size32; + } + /* + * calculate new sp + */ + LLVMValueRef underflow = + LLVMBuildICmp(b, LLVMIntULT, sp, size, "underflow"); + if (!underflow) { + goto fail; + } + LLVMValueRef new_sp = LLVMBuildSub(b, sp, size, "new_sp"); + if (!new_sp) { + goto fail; + } + if (!LLVMBuildBr(b, check_top_block)) { + goto fail; + } + + LLVMPositionBuilderAtEnd(b, check_top_block); + if (comp_ctx->enable_stack_estimation) { + /* + * load native_stack_top_min from the exec_env + */ + LLVMValueRef top_min = + LLVMBuildLoad2(b, OPQ_PTR_TYPE, func_ctx->native_stack_top_min_addr, + "native_stack_top_min"); + if (!top_min) { + goto fail; + } + LLVMValueRef top_min_int = LLVMBuildPtrToInt( + b, top_min, uintptr_type, "native_stack_top_min_int"); + if (!top_min_int) { + goto fail; + } + + bh_assert(update_top_block); + + /* + * update native_stack_top_min if + * new_sp = sp - size < native_stack_top_min + * + * Note: unless the stack has already overflown in this exec_env, + * native_stack_bound <= native_stack_top_min + */ + LLVMValueRef cmp_top = + LLVMBuildICmp(b, LLVMIntULT, new_sp, top_min_int, "cmp_top"); + if (!cmp_top) { + goto fail; + } + cmp_top = LLVMBuildOr(b, underflow, cmp_top, "cmp_top2"); + if (!cmp_top) { + goto fail; + } + if (!LLVMBuildCondBr(b, cmp_top, update_top_block, + call_wrapped_func_block)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* + * update native_stack_top_min + */ + LLVMPositionBuilderAtEnd(b, update_top_block); + LLVMValueRef new_sp_ptr = + LLVMBuildIntToPtr(b, new_sp, INT8_PTR_TYPE, "new_sp_ptr"); + if (!new_sp_ptr) { + goto fail; + } + if (!LLVMBuildStore(b, new_sp_ptr, + func_ctx->native_stack_top_min_addr)) { + goto fail; + } + if (!LLVMBuildBr(b, stack_bound_check_block)) { + goto fail; + } + } + else { + if (!LLVMBuildBr(b, stack_bound_check_block)) { + goto fail; + } + } + + LLVMPositionBuilderAtEnd(b, stack_bound_check_block); + if (comp_ctx->enable_stack_bound_check) { + /* + * trap if new_sp < native_stack_bound + */ + LLVMValueRef bound_int = LLVMBuildPtrToInt( + b, func_ctx->native_stack_bound, uintptr_type, "bound_base_int"); + if (!bound_int) { + goto fail; + } + LLVMValueRef cmp = + LLVMBuildICmp(b, LLVMIntULT, new_sp, bound_int, "cmp"); + if (!cmp) { + goto fail; + } + cmp = LLVMBuildOr(b, underflow, cmp, "cmp2"); + if (!cmp) { + goto fail; + } + /* todo: @llvm.expect.i1(i1 %cmp, i1 0) */ + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_NATIVE_STACK_OVERFLOW, + true, cmp, call_wrapped_func_block)) + goto fail; + } + else { + if (!LLVMBuildBr(b, call_wrapped_func_block)) { + goto fail; + } + } + + /* + * call the wrapped function + * use a tail-call if possible + */ + LLVMPositionBuilderAtEnd(b, call_wrapped_func_block); + const char *name = "tail_call"; + LLVMTypeRef ret_type = LLVMGetReturnType(func_type); + if (ret_type == VOID_TYPE) { + name = ""; + } + + LLVMValueRef retval; + if (comp_ctx->is_indirect_mode + && !strncmp(comp_ctx->target_arch, "xtensa", 6)) { + /* call wrapped_func indirectly */ + if (!create_func_ptrs(comp_ctx, func_ctx)) { + goto fail; + } + + LLVMTypeRef func_ptr_type; + LLVMValueRef wrapped_func_indirect; + uint32 import_func_count = comp_ctx->comp_data->import_func_count; + uint32 func_count = comp_ctx->func_ctx_count; + + /* Check function index */ + if (func_index >= import_func_count + func_count) { + aot_set_last_error("Function index out of range."); + goto fail; + } + + /* Get function type */ + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + goto fail; + } + + /* + * func_index layout : + * aot_func#xxx, range from 0 ~ func_conut - 1; + * aot_func#internal#xxx, range from func_conut ~ 2 * func_conut - 1; + */ + if (!(wrapped_func_indirect = aot_get_func_from_table( + comp_ctx, func_ctx->func_ptrs, func_ptr_type, + func_index + func_count + import_func_count))) { + goto fail; + } + + /* Call the function indirectly */ + retval = LLVMBuildCall2(b, func_type, wrapped_func_indirect, params, + param_count, name); + } + else + retval = LLVMBuildCall2(b, func_type, wrapped_func, params, param_count, + name); + + if (!retval) { + goto fail; + } + wasm_runtime_free(params); + params = NULL; + if (aot_target_precheck_can_use_musttail(comp_ctx)) { + LLVMSetTailCallKind(retval, LLVMTailCallKindMustTail); + } + else { + LLVMSetTailCallKind(retval, LLVMTailCallKindTail); + } + if (ret_type == VOID_TYPE) { + if (!LLVMBuildRetVoid(b)) { + goto fail; + } + } + else { + if (!LLVMBuildRet(b, retval)) { + goto fail; + } + } + + return true; +fail: + if (params != NULL) { + wasm_runtime_free(params); + } + aot_set_last_error("failed to build precheck wrapper function."); + return false; +} + +static bool +check_wasm_type(AOTCompContext *comp_ctx, uint8 type) +{ + if (type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF) { + if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) { + aot_set_last_error("funcref or externref type was found, " + "try removing --disable-ref-types option " + "or adding --enable-gc option."); + return false; + } + else + return true; + } + else if (aot_is_type_gc_reftype(type)) { + if (!comp_ctx->enable_gc) { + aot_set_last_error("GC reference type was found, " + "try adding --enable-gc option."); + return false; + } + else + return true; + } + else if (type == VALUE_TYPE_V128) { + if (!comp_ctx->enable_simd) { + aot_set_last_error("SIMD type was found, try removing " + " --disable-simd option."); + return false; + } + return true; + } + else if (type != VALUE_TYPE_I32 && type != VALUE_TYPE_I64 + && type != VALUE_TYPE_F32 && type != VALUE_TYPE_F64) { + bh_assert(0); + } + + return true; +} + +/** + * Add LLVM function + */ +static LLVMValueRef +aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module, + const AOTFuncType *aot_func_type, uint32 func_index, + LLVMTypeRef *p_func_type, LLVMValueRef *p_precheck_func) +{ + WASMFunction *aot_func = + comp_ctx->comp_data->wasm_module->functions[func_index]; + LLVMValueRef func = NULL; + LLVMTypeRef *param_types, ret_type, func_type; + LLVMTypeRef func_type_wrapper; + LLVMValueRef func_wrapper; + LLVMBasicBlockRef func_begin; + char func_name[48]; + uint64 size; + uint32 i, j = 0, param_count = (uint64)aot_func_type->param_count; + uint32 backend_thread_num, compile_thread_num; + + /* Check function parameter types and result types */ + for (i = 0; + i < (uint32)(aot_func_type->param_count + aot_func_type->result_count); + i++) { + if (!check_wasm_type(comp_ctx, aot_func_type->types[i])) + return NULL; + } + /* Check function local types */ + for (i = 0; i < aot_func->local_count; i++) { + if (!check_wasm_type(comp_ctx, aot_func->local_types[i])) + return NULL; + } + + /* exec env as first parameter */ + param_count++; + + /* Extra wasm function results(except the first one)'s address are + * appended to aot function parameters. */ + if (aot_func_type->result_count > 1) + param_count += aot_func_type->result_count - 1; + + /* Initialize parameter types of the LLVM function */ + size = sizeof(LLVMTypeRef) * ((uint64)param_count); + if (size >= UINT32_MAX + || !(param_types = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + /* exec env as first parameter */ + param_types[j++] = comp_ctx->exec_env_type; + for (i = 0; i < aot_func_type->param_count; i++) + param_types[j++] = TO_LLVM_TYPE(aot_func_type->types[i]); + /* Extra results' address */ + for (i = 1; i < aot_func_type->result_count; i++, j++) { + param_types[j] = + TO_LLVM_TYPE(aot_func_type->types[aot_func_type->param_count + i]); + if (!(param_types[j] = LLVMPointerType(param_types[j], 0))) { + aot_set_last_error("llvm get pointer type failed."); + goto fail; + } + } + + /* Resolve return type of the LLVM function */ + if (aot_func_type->result_count) + ret_type = + TO_LLVM_TYPE(aot_func_type->types[aot_func_type->param_count]); + else + ret_type = VOID_TYPE; + + /* Resolve function prototype */ + if (!(func_type = + LLVMFunctionType(ret_type, param_types, param_count, false))) { + aot_set_last_error("create LLVM function type failed."); + goto fail; + } + + bh_assert(func_index < comp_ctx->func_ctx_count); + bh_assert(LLVMGetReturnType(func_type) == ret_type); + + const char *prefix = AOT_FUNC_PREFIX; + const bool need_precheck = + comp_ctx->enable_stack_bound_check || comp_ctx->enable_stack_estimation; + LLVMValueRef precheck_func = NULL; + + if (need_precheck) { + precheck_func = aot_add_llvm_func1(comp_ctx, module, func_index, + aot_func_type->param_count, + func_type, AOT_FUNC_PREFIX); + if (!precheck_func) { + goto fail; + } + /* + * REVISIT: probably this breaks windows hw bound check + * (the RtlAddFunctionTable stuff) + */ + prefix = AOT_FUNC_INTERNAL_PREFIX; + } + if (!(func = aot_add_llvm_func1(comp_ctx, module, func_index, + aot_func_type->param_count, func_type, + prefix))) + goto fail; + + if (comp_ctx->is_indirect_mode) { + /* avoid LUT relocations ("switch-table") */ + LLVMAttributeRef attr_no_jump_tables = LLVMCreateStringAttribute( + comp_ctx->context, "no-jump-tables", + (uint32)strlen("no-jump-tables"), "true", (uint32)strlen("true")); + LLVMAddAttributeAtIndex(func, LLVMAttributeFunctionIndex, + attr_no_jump_tables); + } + + /* spread fp.all to every function */ + if (comp_ctx->emit_frame_pointer) { + const char *key = "frame-pointer"; + const char *val = "all"; + LLVMAttributeRef no_omit_fp = LLVMCreateStringAttribute( + comp_ctx->context, key, (unsigned)strlen(key), val, + (unsigned)strlen(val)); + if (!no_omit_fp) { + aot_set_last_error("create LLVM attribute (frame-pointer) failed."); + goto fail; + } + LLVMAddAttributeAtIndex(func, LLVMAttributeFunctionIndex, no_omit_fp); + } + + if (need_precheck) { + if (!comp_ctx->is_jit_mode + && !(comp_ctx->is_indirect_mode + && !strncmp(comp_ctx->target_arch, "xtensa", 6))) + LLVMSetLinkage(func, LLVMInternalLinkage); + unsigned int kind = + LLVMGetEnumAttributeKindForName("noinline", strlen("noinline")); + LLVMAttributeRef attr_noinline = + LLVMCreateEnumAttribute(comp_ctx->context, kind, 0); + LLVMAddAttributeAtIndex(func, LLVMAttributeFunctionIndex, + attr_noinline); + + if (!aot_build_precheck_function(comp_ctx, module, precheck_func, + func_index, func_type, func)) + goto fail; + LLVMAddAttributeAtIndex(precheck_func, LLVMAttributeFunctionIndex, + attr_noinline); + *p_precheck_func = precheck_func; + } + else { + *p_precheck_func = func; + } + + if (p_func_type) + *p_func_type = func_type; + + backend_thread_num = WASM_ORC_JIT_BACKEND_THREAD_NUM; + compile_thread_num = WASM_ORC_JIT_COMPILE_THREAD_NUM; + + /* Add the jit wrapper function with simple prototype, so that we + can easily call it to trigger its compilation and let LLVM JIT + compile the actual jit functions by adding them into the function + list in the PartitionFunction callback */ + if (comp_ctx->is_jit_mode + && (func_index % (backend_thread_num * compile_thread_num) + < backend_thread_num)) { + func_type_wrapper = LLVMFunctionType(VOID_TYPE, NULL, 0, false); + if (!func_type_wrapper) { + aot_set_last_error("create LLVM function type failed."); + goto fail; + } + + snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, + func_index, "_wrapper"); + if (!(func_wrapper = + LLVMAddFunction(module, func_name, func_type_wrapper))) { + aot_set_last_error("add LLVM function failed."); + goto fail; + } + + if (!(func_begin = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_wrapper, "func_begin"))) { + aot_set_last_error("add LLVM basic block failed."); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, func_begin); + if (!LLVMBuildRetVoid(comp_ctx->builder)) { + aot_set_last_error("llvm build ret failed."); + goto fail; + } + } + +fail: + wasm_runtime_free(param_types); + return func; +} + +static void +free_block_memory(AOTBlock *block) +{ + if (block->param_types) + wasm_runtime_free(block->param_types); + if (block->result_types) + wasm_runtime_free(block->result_types); + wasm_runtime_free(block); +} + +/** + * Create first AOTBlock, or function block for the function + */ +static AOTBlock * +aot_create_func_block(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, const AOTFunc *func, + const AOTFuncType *aot_func_type) +{ + AOTBlock *aot_block; + uint32 param_count = aot_func_type->param_count, + result_count = aot_func_type->result_count; + + /* Allocate memory */ + if (!(aot_block = wasm_runtime_malloc(sizeof(AOTBlock)))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + memset(aot_block, 0, sizeof(AOTBlock)); + if (param_count + && !(aot_block->param_types = wasm_runtime_malloc(param_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + if (result_count) { + if (!(aot_block->result_types = wasm_runtime_malloc(result_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + } + + /* Set block data */ + aot_block->label_type = LABEL_TYPE_FUNCTION; + aot_block->param_count = param_count; + if (param_count) { + bh_memcpy_s(aot_block->param_types, param_count, aot_func_type->types, + param_count); + } + aot_block->result_count = result_count; + if (result_count) { + bh_memcpy_s(aot_block->result_types, result_count, + aot_func_type->types + param_count, result_count); + } + aot_block->wasm_code_end = func->code + func->code_size; + + /* Add function entry block */ + if (!(aot_block->llvm_entry_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "func_begin"))) { + aot_set_last_error("add LLVM basic block failed."); + goto fail; + } + + return aot_block; + +fail: + free_block_memory(aot_block); + return NULL; +} + +static bool +create_argv_buf(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef argv_buf_offset = I32_THREE, argv_buf_addr; + LLVMTypeRef int32_ptr_type; + + /* Get argv buffer address */ + if (!(argv_buf_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, + &argv_buf_offset, 1, "argv_buf_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(int32_ptr_type = LLVMPointerType(INT32_PTR_TYPE, 0))) { + aot_set_last_error("llvm add pointer type failed"); + return false; + } + + /* Convert to int32 pointer type */ + if (!(argv_buf_addr = LLVMBuildBitCast(comp_ctx->builder, argv_buf_addr, + int32_ptr_type, "argv_buf_ptr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + if (!(func_ctx->argv_buf = LLVMBuildLoad2(comp_ctx->builder, INT32_PTR_TYPE, + argv_buf_addr, "argv_buf"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + return true; +} + +static bool +create_native_stack_bound(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef stack_bound_offset = I32_FOUR, stack_bound_addr; + + if (!(stack_bound_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, + &stack_bound_offset, 1, "stack_bound_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(func_ctx->native_stack_bound = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, stack_bound_addr, + "native_stack_bound"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + return true; +} + +static bool +create_native_stack_top_min(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef offset = I32_NINE; + + if (!(func_ctx->native_stack_top_min_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1, + "native_stack_top_min_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + return true; +} + +static bool +create_aux_stack_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef aux_stack_bound_offset = I32_SIX, aux_stack_bound_addr; + LLVMValueRef aux_stack_bottom_offset = I32_SEVEN, aux_stack_bottom_addr; + + /* Get aux stack boundary address */ + if (!(aux_stack_bound_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, + &aux_stack_bound_offset, 1, "aux_stack_bound_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(aux_stack_bound_addr = + LLVMBuildBitCast(comp_ctx->builder, aux_stack_bound_addr, + INTPTR_T_PTR_TYPE, "aux_stack_bound_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(func_ctx->aux_stack_bound = + LLVMBuildLoad2(comp_ctx->builder, INTPTR_T_TYPE, + aux_stack_bound_addr, "aux_stack_bound_intptr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + if (!(func_ctx->aux_stack_bound = + LLVMBuildZExt(comp_ctx->builder, func_ctx->aux_stack_bound, + I64_TYPE, "aux_stack_bound_i64"))) { + aot_set_last_error("llvm build truncOrBitCast failed."); + return false; + } + + /* Get aux stack bottom address */ + if (!(aux_stack_bottom_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, + &aux_stack_bottom_offset, 1, "aux_stack_bottom_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(aux_stack_bottom_addr = + LLVMBuildBitCast(comp_ctx->builder, aux_stack_bottom_addr, + INTPTR_T_PTR_TYPE, "aux_stack_bottom_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(func_ctx->aux_stack_bottom = + LLVMBuildLoad2(comp_ctx->builder, INTPTR_T_TYPE, + aux_stack_bottom_addr, "aux_stack_bottom"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + if (!(func_ctx->aux_stack_bottom = + LLVMBuildZExt(comp_ctx->builder, func_ctx->aux_stack_bottom, + I64_TYPE, "aux_stack_bottom_i64"))) { + aot_set_last_error("llvm build truncOrBitCast failed."); + return false; + } + + return true; +} + +static bool +create_aux_stack_frame(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef wasm_stack_top_bound_ptr, offset; + + offset = I32_ONE; + if (!(func_ctx->cur_frame_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1, + "cur_frame_ptr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(func_ctx->cur_frame = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->cur_frame_ptr, "cur_frame"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + /* Get exec_env->wasm_stack.top_boundary and its address */ + offset = I32_TEN; + if (!(wasm_stack_top_bound_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1, + "wasm_stack_top_bound_ptr")) + || !(func_ctx->wasm_stack_top_bound = LLVMBuildLoad2( + comp_ctx->builder, INT8_PTR_TYPE, wasm_stack_top_bound_ptr, + "wasm_stack_top_bound"))) { + aot_set_last_error("load wasm_stack.top_boundary failed"); + return false; + } + + offset = I32_ELEVEN; + if (!(func_ctx->wasm_stack_top_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1, + "wasm_stack_top_ptr"))) { + aot_set_last_error("llvm build inbounds gep failed"); + return false; + } + + return true; +} + +static bool +create_native_symbol(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef native_symbol_offset = I32_EIGHT, native_symbol_addr; + + if (!(native_symbol_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, + &native_symbol_offset, 1, "native_symbol_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(func_ctx->native_symbol = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + native_symbol_addr, "native_symbol_tmp"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(func_ctx->native_symbol = + LLVMBuildBitCast(comp_ctx->builder, func_ctx->native_symbol, + comp_ctx->exec_env_type, "native_symbol"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + return true; +} + +static bool +create_local_variables(const AOTCompData *comp_data, + const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + const AOTFunc *func) +{ + AOTFuncType *aot_func_type = + (AOTFuncType *)comp_data->types[func->func_type_index]; + char local_name[32]; + uint32 i, j = 1; + + for (i = 0; i < aot_func_type->param_count; i++, j++) { + snprintf(local_name, sizeof(local_name), "l%d", i); + func_ctx->locals[i] = + LLVMBuildAlloca(comp_ctx->builder, + TO_LLVM_TYPE(aot_func_type->types[i]), local_name); + if (!func_ctx->locals[i]) { + aot_set_last_error("llvm build alloca failed."); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, LLVMGetParam(func_ctx->func, j), + func_ctx->locals[i])) { + aot_set_last_error("llvm build store failed."); + return false; + } + } + + for (i = 0; i < func->local_count; i++) { + LLVMTypeRef local_type; + LLVMValueRef local_value = NULL; + snprintf(local_name, sizeof(local_name), "l%d", + aot_func_type->param_count + i); + local_type = TO_LLVM_TYPE(func->local_types_wp[i]); + func_ctx->locals[aot_func_type->param_count + i] = + LLVMBuildAlloca(comp_ctx->builder, local_type, local_name); + if (!func_ctx->locals[aot_func_type->param_count + i]) { + aot_set_last_error("llvm build alloca failed."); + return false; + } + switch (func->local_types_wp[i]) { + case VALUE_TYPE_I32: + local_value = I32_ZERO; + break; + case VALUE_TYPE_I64: + local_value = I64_ZERO; + break; + case VALUE_TYPE_F32: + local_value = F32_ZERO; + break; + case VALUE_TYPE_F64: + local_value = F64_ZERO; + break; + case VALUE_TYPE_V128: + local_value = V128_i64x2_ZERO; + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + if (!comp_ctx->enable_gc) + local_value = REF_NULL; + else + local_value = GC_REF_NULL; + break; +#if WASM_ENABLE_GC != 0 + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + /* case REF_TYPE_FUNCREF: */ + /* case REF_TYPE_EXTERNREF: */ + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + local_value = GC_REF_NULL; + break; +#endif + default: + bh_assert(0); + break; + } + if (!LLVMBuildStore(comp_ctx->builder, local_value, + func_ctx->locals[aot_func_type->param_count + i])) { + aot_set_last_error("llvm build store failed."); + return false; + } + } + + return true; +} + +static bool +create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef int8_ptr_type, uint32 func_index) +{ + LLVMValueRef offset, mem_info_base; + uint32 memory_count; + WASMModule *module = comp_ctx->comp_data->wasm_module; + WASMFunction *func = module->functions[func_index]; + LLVMTypeRef bound_check_type; + bool mem_space_unchanged = + (!func->has_op_memory_grow && !func->has_op_func_call) + || (!module->possible_memory_grow); +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory; +#endif + + func_ctx->mem_space_unchanged = mem_space_unchanged; + + memory_count = module->memory_count + module->import_memory_count; + /* If the module dosen't have memory, reserve + one mem_info space with empty content */ + if (memory_count == 0) + memory_count = 1; + + if (!(func_ctx->mem_info = + wasm_runtime_malloc(sizeof(AOTMemInfo) * memory_count))) { + return false; + } + memset(func_ctx->mem_info, 0, sizeof(AOTMemInfo)); + + /* Currently we only create memory info for memory 0 */ + /* Load memory base address */ +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared_memory = + comp_ctx->comp_data->memories[0].memory_flags & 0x02 ? true : false; + if (is_shared_memory) { + LLVMValueRef shared_mem_addr; + offset = I32_CONST(offsetof(AOTModuleInstance, memories)); + if (!offset) { + aot_set_last_error("create llvm const failed."); + return false; + } + + /* aot_inst->memories */ + if (!(shared_mem_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "shared_mem_addr_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(shared_mem_addr = + LLVMBuildBitCast(comp_ctx->builder, shared_mem_addr, + int8_ptr_type, "shared_mem_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + /* aot_inst->memories[0] */ + if (!(shared_mem_addr = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + shared_mem_addr, "shared_mem_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + if (!(shared_mem_addr = + LLVMBuildBitCast(comp_ctx->builder, shared_mem_addr, + int8_ptr_type, "shared_mem_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (!(shared_mem_addr = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + shared_mem_addr, "shared_mem_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + /* memories[0]->memory_data */ + offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data)); + if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, shared_mem_addr, &offset, 1, + "mem_base_addr_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + /* memories[0]->cur_page_count */ + offset = I32_CONST(offsetof(AOTMemoryInstance, cur_page_count)); + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + shared_mem_addr, &offset, 1, + "mem_cur_page_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + /* memories[0]->memory_data_size */ + offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data_size)); + if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, shared_mem_addr, &offset, 1, + "mem_data_size_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + } + else +#endif + { + uint32 offset_of_global_table_data; + + if (comp_ctx->is_jit_mode) + offset_of_global_table_data = + offsetof(WASMModuleInstance, global_table_data); + else + offset_of_global_table_data = + offsetof(AOTModuleInstance, global_table_data); + + offset = I32_CONST(offset_of_global_table_data + + offsetof(AOTMemoryInstance, memory_data)); + if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "mem_base_addr_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + offset = I32_CONST(offset_of_global_table_data + + offsetof(AOTMemoryInstance, cur_page_count)); + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "mem_cur_page_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + offset = I32_CONST(offset_of_global_table_data + + offsetof(AOTMemoryInstance, memory_data_size)); + if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "mem_data_size_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + } + /* Store mem info base address before cast */ + mem_info_base = func_ctx->mem_info[0].mem_base_addr; + + if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_base_addr, + int8_ptr_type, "mem_base_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_cur_page_count_addr, + INT32_PTR_TYPE, "mem_cur_page_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_data_size_addr, + INT64_PTR_TYPE, "mem_data_size_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (mem_space_unchanged) { + if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildLoad2( + comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->mem_info[0].mem_base_addr, "mem_base_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = + LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, + func_ctx->mem_info[0].mem_cur_page_count_addr, + "mem_cur_page_count"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildLoad2( + comp_ctx->builder, I64_TYPE, + func_ctx->mem_info[0].mem_data_size_addr, "mem_data_size"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + else if (is_shared_memory) { + /* The base address for shared memory will never changed, + we can load the value here */ + if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildLoad2( + comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->mem_info[0].mem_base_addr, "mem_base_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } +#endif + + bound_check_type = (comp_ctx->pointer_size == sizeof(uint64)) + ? INT64_PTR_TYPE + : INT32_PTR_TYPE; + + /* Load memory bound check constants */ + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_1byte) + - offsetof(AOTMemoryInstance, memory_data)); + if (!(func_ctx->mem_info[0].mem_bound_check_1byte = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, + &offset, 1, "bound_check_1byte_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_bound_check_1byte = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_1byte, + bound_check_type, "bound_check_1byte_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (mem_space_unchanged) { + if (!(func_ctx->mem_info[0].mem_bound_check_1byte = LLVMBuildLoad2( + comp_ctx->builder, + (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE + : I32_TYPE, + func_ctx->mem_info[0].mem_bound_check_1byte, + "bound_check_1byte"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } + + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_2bytes) + - offsetof(AOTMemoryInstance, memory_data)); + if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, + &offset, 1, "bound_check_2bytes_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_2bytes, + bound_check_type, "bound_check_2bytes_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (mem_space_unchanged) { + if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = LLVMBuildLoad2( + comp_ctx->builder, + (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE + : I32_TYPE, + func_ctx->mem_info[0].mem_bound_check_2bytes, + "bound_check_2bytes"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } + + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_4bytes) + - offsetof(AOTMemoryInstance, memory_data)); + if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, + &offset, 1, "bound_check_4bytes_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_4bytes, + bound_check_type, "bound_check_4bytes_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (mem_space_unchanged) { + if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = LLVMBuildLoad2( + comp_ctx->builder, + (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE + : I32_TYPE, + func_ctx->mem_info[0].mem_bound_check_4bytes, + "bound_check_4bytes"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } + + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_8bytes) + - offsetof(AOTMemoryInstance, memory_data)); + if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, + &offset, 1, "bound_check_8bytes_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_8bytes, + bound_check_type, "bound_check_8bytes_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (mem_space_unchanged) { + if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = LLVMBuildLoad2( + comp_ctx->builder, + (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE + : I32_TYPE, + func_ctx->mem_info[0].mem_bound_check_8bytes, + "bound_check_8bytes"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } + + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_16bytes) + - offsetof(AOTMemoryInstance, memory_data)); + if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, + "bound_check_16bytes_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_16bytes, + bound_check_type, "bound_check_16bytes_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (mem_space_unchanged) { + if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildLoad2( + comp_ctx->builder, + (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE + : I32_TYPE, + func_ctx->mem_info[0].mem_bound_check_16bytes, + "bound_check_16bytes"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } + + return true; +} + +static bool +create_cur_exception(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef offset; + + offset = I32_CONST(offsetof(AOTModuleInstance, cur_exception)); + func_ctx->cur_exception = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, + &offset, 1, "cur_exception"); + if (!func_ctx->cur_exception) { + aot_set_last_error("llvm build in bounds gep failed."); + return false; + } + return true; +} + +static bool +create_func_type_indexes(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef offset, func_type_indexes_ptr; + LLVMTypeRef int32_ptr_type; + + offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes)); + func_type_indexes_ptr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, + &offset, 1, "func_type_indexes_ptr"); + if (!func_type_indexes_ptr) { + aot_set_last_error("llvm build add failed."); + return false; + } + + if (!(int32_ptr_type = LLVMPointerType(INT32_PTR_TYPE, 0))) { + aot_set_last_error("llvm get pointer type failed."); + return false; + } + + func_ctx->func_type_indexes = + LLVMBuildBitCast(comp_ctx->builder, func_type_indexes_ptr, + int32_ptr_type, "func_type_indexes_tmp"); + if (!func_ctx->func_type_indexes) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + func_ctx->func_type_indexes = + LLVMBuildLoad2(comp_ctx->builder, INT32_PTR_TYPE, + func_ctx->func_type_indexes, "func_type_indexes"); + if (!func_ctx->func_type_indexes) { + aot_set_last_error("llvm build load failed."); + return false; + } + return true; +} + +static bool +create_func_ptrs(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef offset; + + offset = I32_CONST(offsetof(AOTModuleInstance, func_ptrs)); + func_ctx->func_ptrs = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, + &offset, 1, "func_ptrs_offset"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build in bounds gep failed."); + return false; + } + func_ctx->func_ptrs = + LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs, + comp_ctx->exec_env_type, "func_ptrs_tmp"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + func_ctx->func_ptrs = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, + func_ctx->func_ptrs, "func_ptrs_ptr"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build load failed."); + return false; + } + + func_ctx->func_ptrs = + LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs, + comp_ctx->exec_env_type, "func_ptrs"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + return true; +} + +const char *aot_stack_sizes_name = AOT_STACK_SIZES_NAME; +const char *aot_stack_sizes_alias_name = AOT_STACK_SIZES_ALIAS_NAME; +const char *aot_stack_sizes_section_name = AOT_STACK_SIZES_SECTION_NAME; + +static bool +aot_create_stack_sizes(const AOTCompData *comp_data, AOTCompContext *comp_ctx) +{ + LLVMValueRef stack_sizes, *values, array, alias; + LLVMTypeRef stack_sizes_type; +#if LLVM_VERSION_MAJOR <= 13 + LLVMTypeRef alias_type; +#endif + uint64 size; + uint32 i; + + stack_sizes_type = LLVMArrayType(I32_TYPE, comp_data->func_count); + if (!stack_sizes_type) { + aot_set_last_error("failed to create stack_sizes type."); + return false; + } + + stack_sizes = + LLVMAddGlobal(comp_ctx->module, stack_sizes_type, aot_stack_sizes_name); + if (!stack_sizes) { + aot_set_last_error("failed to create stack_sizes global."); + return false; + } + + size = sizeof(LLVMValueRef) * comp_data->func_count; + if (size >= UINT32_MAX || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return false; + } + + for (i = 0; i < comp_data->func_count; i++) { + /* + * This value is a placeholder, which will be replaced + * after the corresponding functions are compiled. + * + * Don't use zeros becasue LLVM can optimize them to + * zeroinitializer. + */ + values[i] = I32_NEG_ONE; + } + + array = LLVMConstArray(I32_TYPE, values, comp_data->func_count); + wasm_runtime_free(values); + if (!array) { + aot_set_last_error("failed to create stack_sizes initializer."); + return false; + } + LLVMSetInitializer(stack_sizes, array); + + /* + * create an alias so that aot_resolve_stack_sizes can find it. + */ +#if LLVM_VERSION_MAJOR > 13 + alias = LLVMAddAlias2(comp_ctx->module, stack_sizes_type, 0, stack_sizes, + aot_stack_sizes_alias_name); +#else + alias_type = LLVMPointerType(stack_sizes_type, 0); + if (!alias_type) { + aot_set_last_error("failed to create alias type."); + return false; + } + alias = LLVMAddAlias(comp_ctx->module, alias_type, stack_sizes, + aot_stack_sizes_alias_name); +#endif + if (!alias) { + aot_set_last_error("failed to create stack_sizes alias."); + return false; + } + + /* + * make the original symbol internal. we mainly use this version to + * avoid creating extra relocations in the precheck functions. + */ + LLVMSetLinkage(stack_sizes, LLVMInternalLinkage); + LLVMSetSection(stack_sizes, aot_stack_sizes_section_name); + comp_ctx->stack_sizes_type = stack_sizes_type; + comp_ctx->stack_sizes = stack_sizes; + return true; +} + +/** + * Create function compiler context + */ +static AOTFuncContext * +aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, + AOTFunc *func, uint32 func_index) +{ + AOTFuncContext *func_ctx; + AOTFuncType *aot_func_type = + (AOTFuncType *)comp_data->types[func->func_type_index]; + WASMModule *module = comp_ctx->comp_data->wasm_module; + WASMFunction *wasm_func = module->functions[func_index]; + AOTBlock *aot_block; + LLVMTypeRef int8_ptr_type; + uint64 size; + + /* Allocate memory for the function context */ + size = offsetof(AOTFuncContext, locals) + + sizeof(LLVMValueRef) + * ((uint64)aot_func_type->param_count + func->local_count); + if (size >= UINT32_MAX || !(func_ctx = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(func_ctx, 0, (uint32)size); + func_ctx->aot_func = func; + + func_ctx->module = comp_ctx->module; + + /* Add LLVM function */ + if (!(func_ctx->func = aot_add_llvm_func( + comp_ctx, func_ctx->module, aot_func_type, func_index, + &func_ctx->func_type, &func_ctx->precheck_func))) { + goto fail; + } + + /* Create function's first AOTBlock */ + if (!(aot_block = + aot_create_func_block(comp_ctx, func_ctx, func, aot_func_type))) { + goto fail; + } + +#if WASM_ENABLE_DEBUG_AOT != 0 + func_ctx->debug_func = dwarf_gen_func_info(comp_ctx, func_ctx); +#endif + + aot_block_stack_push(&func_ctx->block_stack, aot_block); + + /* Add local variables */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, aot_block->llvm_entry_block); + + if (!create_basic_func_context(comp_ctx, func_ctx)) { + goto fail; + } + + /* Get argv buffer address */ + if (wasm_func->has_op_func_call && !create_argv_buf(comp_ctx, func_ctx)) { + goto fail; + } + + /* Get auxiliary stack info */ + if (wasm_func->has_op_set_global_aux_stack + && !create_aux_stack_info(comp_ctx, func_ctx)) { + goto fail; + } + + if (comp_ctx->enable_aux_stack_frame + && !create_aux_stack_frame(comp_ctx, func_ctx)) { + goto fail; + } + + /* Create local variables */ + if (!create_local_variables(comp_data, comp_ctx, func_ctx, func)) { + goto fail; + } + + if (!(int8_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0))) { + aot_set_last_error("llvm add pointer type failed."); + goto fail; + } + + /* Create base addr, end addr, data size of mem, heap */ + if (wasm_func->has_memory_operations + && !create_memory_info(comp_ctx, func_ctx, int8_ptr_type, func_index)) { + goto fail; + } + + /* Load current exception */ + if (!create_cur_exception(comp_ctx, func_ctx)) { + goto fail; + } + + /* Load function type indexes */ + if (wasm_func->has_op_call_indirect + && !create_func_type_indexes(comp_ctx, func_ctx)) { + goto fail; + } + + /* Load function pointers */ + if (!create_func_ptrs(comp_ctx, func_ctx)) { + goto fail; + } + + return func_ctx; + +fail: + if (func_ctx->mem_info) + wasm_runtime_free(func_ctx->mem_info); + aot_block_stack_destroy(comp_ctx, &func_ctx->block_stack); + wasm_runtime_free(func_ctx); + return NULL; +} + +static void +aot_destroy_func_contexts(AOTCompContext *comp_ctx, AOTFuncContext **func_ctxes, + uint32 count) +{ + uint32 i; + + for (i = 0; i < count; i++) + if (func_ctxes[i]) { + if (func_ctxes[i]->mem_info) + wasm_runtime_free(func_ctxes[i]->mem_info); + aot_block_stack_destroy(comp_ctx, &func_ctxes[i]->block_stack); + aot_checked_addr_list_destroy(func_ctxes[i]); + wasm_runtime_free(func_ctxes[i]); + } + wasm_runtime_free(func_ctxes); +} + +/** + * Create function compiler contexts + */ +static AOTFuncContext ** +aot_create_func_contexts(const AOTCompData *comp_data, AOTCompContext *comp_ctx) +{ + AOTFuncContext **func_ctxes; + uint64 size; + uint32 i; + + if ((comp_ctx->enable_stack_bound_check + || comp_ctx->enable_stack_estimation) + && !aot_create_stack_sizes(comp_data, comp_ctx)) + return NULL; + + /* Allocate memory */ + size = sizeof(AOTFuncContext *) * (uint64)comp_data->func_count; + if (size >= UINT32_MAX + || !(func_ctxes = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(func_ctxes, 0, size); + + /* Create each function context */ + for (i = 0; i < comp_data->func_count; i++) { + AOTFunc *func = comp_data->funcs[i]; + if (!(func_ctxes[i] = + aot_create_func_context(comp_data, comp_ctx, func, i))) { + aot_destroy_func_contexts(comp_ctx, func_ctxes, + comp_data->func_count); + return NULL; + } + } + + return func_ctxes; +} + +static bool +aot_set_llvm_basic_types(AOTLLVMTypes *basic_types, LLVMContextRef context, + int pointer_size) +{ + basic_types->int1_type = LLVMInt1TypeInContext(context); + basic_types->int8_type = LLVMInt8TypeInContext(context); + basic_types->int16_type = LLVMInt16TypeInContext(context); + basic_types->int32_type = LLVMInt32TypeInContext(context); + basic_types->int64_type = LLVMInt64TypeInContext(context); + basic_types->float32_type = LLVMFloatTypeInContext(context); + basic_types->float64_type = LLVMDoubleTypeInContext(context); + basic_types->void_type = LLVMVoidTypeInContext(context); + + basic_types->meta_data_type = LLVMMetadataTypeInContext(context); + + basic_types->int8_ptr_type = LLVMPointerType(basic_types->int8_type, 0); + + if (basic_types->int8_ptr_type) { + basic_types->int8_pptr_type = + LLVMPointerType(basic_types->int8_ptr_type, 0); + } + + basic_types->int16_ptr_type = LLVMPointerType(basic_types->int16_type, 0); + basic_types->int32_ptr_type = LLVMPointerType(basic_types->int32_type, 0); + basic_types->int64_ptr_type = LLVMPointerType(basic_types->int64_type, 0); + basic_types->float32_ptr_type = + LLVMPointerType(basic_types->float32_type, 0); + basic_types->float64_ptr_type = + LLVMPointerType(basic_types->float64_type, 0); + + basic_types->i8x16_vec_type = LLVMVectorType(basic_types->int8_type, 16); + basic_types->i16x8_vec_type = LLVMVectorType(basic_types->int16_type, 8); + basic_types->i32x4_vec_type = LLVMVectorType(basic_types->int32_type, 4); + basic_types->i64x2_vec_type = LLVMVectorType(basic_types->int64_type, 2); + basic_types->f32x4_vec_type = LLVMVectorType(basic_types->float32_type, 4); + basic_types->f64x2_vec_type = LLVMVectorType(basic_types->float64_type, 2); + + basic_types->v128_type = basic_types->i64x2_vec_type; + basic_types->v128_ptr_type = LLVMPointerType(basic_types->v128_type, 0); + + basic_types->int8_ptr_type_gs = + LLVMPointerType(basic_types->int8_type, 256); + basic_types->int16_ptr_type_gs = + LLVMPointerType(basic_types->int16_type, 256); + basic_types->int32_ptr_type_gs = + LLVMPointerType(basic_types->int32_type, 256); + basic_types->int64_ptr_type_gs = + LLVMPointerType(basic_types->int64_type, 256); + basic_types->float32_ptr_type_gs = + LLVMPointerType(basic_types->float32_type, 256); + basic_types->float64_ptr_type_gs = + LLVMPointerType(basic_types->float64_type, 256); + basic_types->v128_ptr_type_gs = + LLVMPointerType(basic_types->v128_type, 256); + if (!basic_types->int8_ptr_type_gs || !basic_types->int16_ptr_type_gs + || !basic_types->int32_ptr_type_gs || !basic_types->int64_ptr_type_gs + || !basic_types->float32_ptr_type_gs + || !basic_types->float64_ptr_type_gs + || !basic_types->v128_ptr_type_gs) { + return false; + } + + basic_types->i1x2_vec_type = LLVMVectorType(basic_types->int1_type, 2); + + basic_types->funcref_type = LLVMInt32TypeInContext(context); + basic_types->externref_type = LLVMInt32TypeInContext(context); + + if (pointer_size == 4) { + basic_types->intptr_t_type = basic_types->int32_type; + basic_types->intptr_t_ptr_type = basic_types->int32_ptr_type; + } + else { + basic_types->intptr_t_type = basic_types->int64_type; + basic_types->intptr_t_ptr_type = basic_types->int64_ptr_type; + } + + basic_types->gc_ref_type = basic_types->int8_ptr_type; + basic_types->gc_ref_ptr_type = basic_types->int8_pptr_type; + + return (basic_types->int8_ptr_type && basic_types->int8_pptr_type + && basic_types->int16_ptr_type && basic_types->int32_ptr_type + && basic_types->int64_ptr_type && basic_types->intptr_t_type + && basic_types->intptr_t_ptr_type && basic_types->float32_ptr_type + && basic_types->float64_ptr_type && basic_types->i8x16_vec_type + && basic_types->i16x8_vec_type && basic_types->i32x4_vec_type + && basic_types->i64x2_vec_type && basic_types->f32x4_vec_type + && basic_types->f64x2_vec_type && basic_types->i1x2_vec_type + && basic_types->meta_data_type && basic_types->funcref_type + && basic_types->externref_type && basic_types->gc_ref_type + && basic_types->gc_ref_ptr_type) + ? true + : false; +} + +static bool +aot_create_llvm_consts(AOTLLVMConsts *consts, AOTCompContext *comp_ctx) +{ +#define CREATE_I1_CONST(name, value) \ + if (!(consts->i1_##name = \ + LLVMConstInt(comp_ctx->basic_types.int1_type, value, true))) \ + return false; + + CREATE_I1_CONST(zero, 0) + CREATE_I1_CONST(one, 1) +#undef CREATE_I1_CONST + + if (!(consts->i8_zero = I8_CONST(0))) + return false; + + if (!(consts->i8_one = I8_CONST(1))) + return false; + + if (!(consts->f32_zero = F32_CONST(0))) + return false; + + if (!(consts->f64_zero = F64_CONST(0))) + return false; + +#define CREATE_I32_CONST(name, value) \ + if (!(consts->i32_##name = LLVMConstInt(I32_TYPE, value, true))) \ + return false; + + CREATE_I32_CONST(min, (uint32)INT32_MIN) + CREATE_I32_CONST(neg_one, (uint32)-1) + CREATE_I32_CONST(zero, 0) + CREATE_I32_CONST(one, 1) + CREATE_I32_CONST(two, 2) + CREATE_I32_CONST(three, 3) + CREATE_I32_CONST(four, 4) + CREATE_I32_CONST(five, 5) + CREATE_I32_CONST(six, 6) + CREATE_I32_CONST(seven, 7) + CREATE_I32_CONST(eight, 8) + CREATE_I32_CONST(nine, 9) + CREATE_I32_CONST(ten, 10) + CREATE_I32_CONST(eleven, 11) + CREATE_I32_CONST(twelve, 12) + CREATE_I32_CONST(thirteen, 13) + CREATE_I32_CONST(fourteen, 14) + CREATE_I32_CONST(fifteen, 15) + CREATE_I32_CONST(31, 31) + CREATE_I32_CONST(32, 32) +#undef CREATE_I32_CONST + +#define CREATE_I64_CONST(name, value) \ + if (!(consts->i64_##name = LLVMConstInt(I64_TYPE, value, true))) \ + return false; + + CREATE_I64_CONST(min, (uint64)INT64_MIN) + CREATE_I64_CONST(neg_one, (uint64)-1) + CREATE_I64_CONST(zero, 0) + CREATE_I64_CONST(63, 63) + CREATE_I64_CONST(64, 64) +#undef CREATE_I64_CONST + +#define CREATE_V128_CONST(name, type) \ + if (!(consts->name##_vec_zero = LLVMConstNull(type))) \ + return false; \ + if (!(consts->name##_undef = LLVMGetUndef(type))) \ + return false; + + CREATE_V128_CONST(i8x16, V128_i8x16_TYPE) + CREATE_V128_CONST(i16x8, V128_i16x8_TYPE) + CREATE_V128_CONST(i32x4, V128_i32x4_TYPE) + CREATE_V128_CONST(i64x2, V128_i64x2_TYPE) + CREATE_V128_CONST(f32x4, V128_f32x4_TYPE) + CREATE_V128_CONST(f64x2, V128_f64x2_TYPE) +#undef CREATE_V128_CONST + +#define CREATE_VEC_ZERO_MASK(slot) \ + { \ + LLVMTypeRef type = LLVMVectorType(I32_TYPE, slot); \ + if (!type || !(consts->i32x##slot##_zero = LLVMConstNull(type))) \ + return false; \ + } + + CREATE_VEC_ZERO_MASK(16) + CREATE_VEC_ZERO_MASK(8) + CREATE_VEC_ZERO_MASK(4) + CREATE_VEC_ZERO_MASK(2) +#undef CREATE_VEC_ZERO_MASK + + if (!(consts->gc_ref_null = + LLVMConstNull(comp_ctx->basic_types.gc_ref_type))) + return false; + if (!(consts->i8_ptr_null = + LLVMConstNull(comp_ctx->basic_types.int8_ptr_type))) + return false; + + return true; +} + +typedef struct ArchItem { + char *arch; + bool support_eb; +} ArchItem; + +/* clang-format off */ +static ArchItem valid_archs[] = { + { "x86_64", false }, + { "i386", false }, + { "xtensa", false }, + { "mips", true }, + { "mipsel", false }, + { "aarch64v8", false }, + { "aarch64v8.1", false }, + { "aarch64v8.2", false }, + { "aarch64v8.3", false }, + { "aarch64v8.4", false }, + { "aarch64v8.5", false }, + { "aarch64_bev8", false }, /* big endian */ + { "aarch64_bev8.1", false }, + { "aarch64_bev8.2", false }, + { "aarch64_bev8.3", false }, + { "aarch64_bev8.4", false }, + { "aarch64_bev8.5", false }, + { "armv4", true }, + { "armv4t", true }, + { "armv5t", true }, + { "armv5te", true }, + { "armv5tej", true }, + { "armv6", true }, + { "armv6kz", true }, + { "armv6t2", true }, + { "armv6k", true }, + { "armv7", true }, + { "armv6m", true }, + { "armv6sm", true }, + { "armv7em", true }, + { "armv8a", true }, + { "armv8r", true }, + { "armv8m.base", true }, + { "armv8m.main", true }, + { "armv8.1m.main", true }, + { "thumbv4", true }, + { "thumbv4t", true }, + { "thumbv5t", true }, + { "thumbv5te", true }, + { "thumbv5tej", true }, + { "thumbv6", true }, + { "thumbv6kz", true }, + { "thumbv6t2", true }, + { "thumbv6k", true }, + { "thumbv7", true }, + { "thumbv6m", true }, + { "thumbv6sm", true }, + { "thumbv7em", true }, + { "thumbv8a", true }, + { "thumbv8r", true }, + { "thumbv8m.base", true }, + { "thumbv8m.main", true }, + { "thumbv8.1m.main", true }, + { "riscv32", true }, + { "riscv64", true }, + { "arc", true } +}; + +static const char *valid_abis[] = { + "gnu", + "eabi", + "eabihf", + "gnueabihf", + "msvc", + "ilp32", + "ilp32f", + "ilp32d", + "lp64", + "lp64f", + "lp64d" +}; +/* clang-format on */ + +static void +print_supported_targets() +{ + uint32 i; + const char *target_name; + + os_printf("Supported targets:\n"); + /* over the list of all available targets */ + for (LLVMTargetRef target = LLVMGetFirstTarget(); target != NULL; + target = LLVMGetNextTarget(target)) { + target_name = LLVMGetTargetName(target); + /* Skip mipsel, aarch64_be since prefix mips, aarch64 will cover them */ + if (strcmp(target_name, "mipsel") == 0) + continue; + else if (strcmp(target_name, "aarch64_be") == 0) + continue; + + if (strcmp(target_name, "x86-64") == 0) + os_printf(" x86_64\n"); + else if (strcmp(target_name, "x86") == 0) + os_printf(" i386\n"); + else { + for (i = 0; i < sizeof(valid_archs) / sizeof(ArchItem); i++) { + /* If target_name is prefix for valid_archs[i].arch */ + if ((strncmp(target_name, valid_archs[i].arch, + strlen(target_name)) + == 0)) + os_printf(" %s\n", valid_archs[i].arch); + } + } + } +} + +static void +print_supported_abis() +{ + uint32 i; + os_printf("Supported ABI: "); + for (i = 0; i < sizeof(valid_abis) / sizeof(const char *); i++) + os_printf("%s ", valid_abis[i]); + os_printf("\n"); +} + +static bool +check_target_arch(const char *target_arch) +{ + uint32 i; + char *arch; + bool support_eb; + + for (i = 0; i < sizeof(valid_archs) / sizeof(ArchItem); i++) { + arch = valid_archs[i].arch; + support_eb = valid_archs[i].support_eb; + + if (!strncmp(target_arch, arch, strlen(arch)) + && ((support_eb + && (!strcmp(target_arch + strlen(arch), "eb") + || !strcmp(target_arch + strlen(arch), ""))) + || (!support_eb && !strcmp(target_arch + strlen(arch), "")))) { + return true; + } + } + return false; +} + +static bool +check_target_abi(const char *target_abi) +{ + uint32 i; + for (i = 0; i < sizeof(valid_abis) / sizeof(char *); i++) { + if (!strcmp(target_abi, valid_abis[i])) + return true; + } + return false; +} + +static void +get_target_arch_from_triple(const char *triple, char *arch_buf, uint32 buf_size) +{ + uint32 i = 0; + while (*triple != '-' && *triple != '\0' && i < buf_size - 1) + arch_buf[i++] = *triple++; + /* Make sure buffer is long enough */ + bh_assert(*triple == '-' || *triple == '\0'); +} + +static bool +is_baremetal_target(const char *target, const char *cpu, const char *abi) +{ + /* TODO: support more baremetal targets */ + if (target) { + /* If target is thumbxxx, then it is baremetal target */ + if (!strncmp(target, "thumb", strlen("thumb"))) + return true; + } + return false; +} + +void +aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err) +{ + char *err_msg = LLVMGetErrorMessage(err); + aot_set_last_error_v("%s: %s", string, err_msg); + LLVMDisposeErrorMessage(err_msg); +} + +static bool +create_target_machine_detect_host(AOTCompContext *comp_ctx) +{ + char *triple = NULL; + LLVMTargetRef target = NULL; + char *err_msg = NULL; + char *cpu = NULL; + char *features = NULL; + LLVMTargetMachineRef target_machine = NULL; + bool ret = false; + + triple = LLVMGetDefaultTargetTriple(); + if (triple == NULL) { + aot_set_last_error("failed to get default target triple."); + goto fail; + } + + if (LLVMGetTargetFromTriple(triple, &target, &err_msg) != 0) { + aot_set_last_error_v("failed to get llvm target from triple %s.", + err_msg); + LLVMDisposeMessage(err_msg); + goto fail; + } + + if (!LLVMTargetHasJIT(target)) { + aot_set_last_error("unspported JIT on this platform."); + goto fail; + } + + cpu = LLVMGetHostCPUName(); + if (cpu == NULL) { + aot_set_last_error("failed to get host cpu information."); + goto fail; + } + + features = LLVMGetHostCPUFeatures(); + if (features == NULL) { + aot_set_last_error("failed to get host cpu features."); + goto fail; + } + + LOG_VERBOSE("LLVM ORCJIT detected CPU \"%s\", with features \"%s\"\n", cpu, + features); + + /* create TargetMachine */ + target_machine = LLVMCreateTargetMachine( + target, triple, cpu, features, LLVMCodeGenLevelDefault, + LLVMRelocDefault, LLVMCodeModelJITDefault); + if (!target_machine) { + aot_set_last_error("failed to create target machine."); + goto fail; + } + comp_ctx->target_machine = target_machine; + + /* Save target arch */ + get_target_arch_from_triple(triple, comp_ctx->target_arch, + sizeof(comp_ctx->target_arch)); + ret = true; + +fail: + if (triple) + LLVMDisposeMessage(triple); + if (features) + LLVMDisposeMessage(features); + if (cpu) + LLVMDisposeMessage(cpu); + + return ret; +} + +static void +jit_stack_size_callback(void *user_data, const char *name, size_t namelen, + size_t stack_size) +{ + AOTCompContext *comp_ctx = user_data; + /* + * Note: the longest name we care is + * something like "aot_func_internal#4294967295". + */ + char buf[64]; + uint32 func_idx; + const AOTFuncContext *func_ctx; + bool musttail; + unsigned int stack_consumption_to_call_wrapped_func; + unsigned int call_size; + int ret; + + bh_assert(comp_ctx != NULL); + bh_assert(comp_ctx->jit_stack_sizes != NULL); + + if (namelen >= sizeof(buf)) { + LOG_DEBUG("too long name: %.*s", (int)namelen, name); + return; + } + /* ensure NUL termination */ + bh_memcpy_s(buf, (uint32)sizeof(buf), name, (uint32)namelen); + buf[namelen] = 0; + + ret = sscanf(buf, AOT_FUNC_INTERNAL_PREFIX "%" SCNu32, &func_idx); + if (ret != 1) { + return; + } + + bh_assert(func_idx < comp_ctx->func_ctx_count); + func_ctx = comp_ctx->func_ctxes[func_idx]; + call_size = func_ctx->stack_consumption_for_func_call; + musttail = aot_target_precheck_can_use_musttail(comp_ctx); + stack_consumption_to_call_wrapped_func = + musttail ? 0 + : aot_estimate_stack_usage_for_function_call( + comp_ctx, func_ctx->aot_func->func_type); + LOG_VERBOSE("func %.*s stack %u + %zu + %u", (int)namelen, name, + stack_consumption_to_call_wrapped_func, stack_size, call_size); + + /* Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes */ + bh_assert(comp_ctx->jit_stack_sizes[func_idx] == (uint32)-1); + comp_ctx->jit_stack_sizes[func_idx] = (uint32)stack_size + call_size; +} + +static bool +orc_jit_create(AOTCompContext *comp_ctx) +{ + LLVMErrorRef err; + LLVMOrcLLLazyJITRef orc_jit = NULL; + LLVMOrcLLLazyJITBuilderRef builder = NULL; + LLVMOrcJITTargetMachineBuilderRef jtmb = NULL; + bool ret = false; + + builder = LLVMOrcCreateLLLazyJITBuilder(); + if (builder == NULL) { + aot_set_last_error("failed to create jit builder."); + goto fail; + } + + if (comp_ctx->enable_stack_bound_check || comp_ctx->enable_stack_estimation) + LLVMOrcLLJITBuilderSetCompileFuncitonCreatorWithStackSizesCallback( + builder, jit_stack_size_callback, comp_ctx); + + err = LLVMOrcJITTargetMachineBuilderDetectHost(&jtmb); + if (err != LLVMErrorSuccess) { + aot_handle_llvm_errmsg( + "quited to create LLVMOrcJITTargetMachineBuilderRef", err); + goto fail; + } + + LLVMOrcLLLazyJITBuilderSetNumCompileThreads( + builder, WASM_ORC_JIT_COMPILE_THREAD_NUM); + + /* Ownership transfer: + LLVMOrcJITTargetMachineBuilderRef -> LLVMOrcLLJITBuilderRef */ + LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder(builder, jtmb); + err = LLVMOrcCreateLLLazyJIT(&orc_jit, builder); + if (err != LLVMErrorSuccess) { + aot_handle_llvm_errmsg("quited to create llvm lazy orcjit instance", + err); + goto fail; + } + /* Ownership transfer: LLVMOrcLLJITBuilderRef -> LLVMOrcLLJITRef */ + builder = NULL; + +#if WASM_ENABLE_LINUX_PERF != 0 + if (wasm_runtime_get_linux_perf()) { + LOG_DEBUG("Enable linux perf support in JIT"); + LLVMOrcObjectLayerRef obj_linking_layer = + (LLVMOrcObjectLayerRef)LLVMOrcLLLazyJITGetObjLinkingLayer(orc_jit); + LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener( + obj_linking_layer, LLVMCreatePerfJITEventListener()); + } +#endif + + /* Ownership transfer: local -> AOTCompContext */ + comp_ctx->orc_jit = orc_jit; + orc_jit = NULL; + ret = true; + +fail: + if (builder) + LLVMOrcDisposeLLLazyJITBuilder(builder); + + if (orc_jit) + LLVMOrcDisposeLLLazyJIT(orc_jit); + return ret; +} + +bool +aot_compiler_init(void) +{ + /* Initialize LLVM environment */ +#if LLVM_VERSION_MAJOR < 17 + LLVMInitializeCore(LLVMGetGlobalPassRegistry()); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* Init environment of all targets for AOT compiler */ + LLVMInitializeAllTargetInfos(); + LLVMInitializeAllTargets(); + LLVMInitializeAllTargetMCs(); + LLVMInitializeAllAsmPrinters(); +#else + /* Init environment of native for JIT compiler */ + LLVMInitializeNativeTarget(); + LLVMInitializeNativeTarget(); + LLVMInitializeNativeAsmPrinter(); +#endif + + return true; +} + +void +aot_compiler_destroy(void) +{ + LLVMShutdown(); +} + +AOTCompContext * +aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) +{ + AOTCompContext *comp_ctx, *ret = NULL; + LLVMTargetRef target; + char *triple = NULL, *triple_norm, *arch, *abi; + char *cpu = NULL, *features, buf[128]; + char *triple_norm_new = NULL, *cpu_new = NULL; + char *err = NULL, *fp_round = "round.tonearest", + *fp_exce = "fpexcept.strict"; + char triple_buf[128] = { 0 }, features_buf[128] = { 0 }; + uint32 opt_level, size_level, i; + LLVMCodeModel code_model; + LLVMTargetDataRef target_data_ref; + + /* Allocate memory */ + if (!(comp_ctx = wasm_runtime_malloc(sizeof(AOTCompContext)))) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(comp_ctx, 0, sizeof(AOTCompContext)); + comp_ctx->comp_data = comp_data; + + /* Create LLVM context, module and builder */ + comp_ctx->orc_thread_safe_context = LLVMOrcCreateNewThreadSafeContext(); + if (!comp_ctx->orc_thread_safe_context) { + aot_set_last_error("create LLVM ThreadSafeContext failed."); + goto fail; + } + + /* Get a reference to the underlying LLVMContext, note: + different from non LAZY JIT mode, no need to dispose this context, + if will be disposed when the thread safe context is disposed */ + if (!(comp_ctx->context = LLVMOrcThreadSafeContextGetContext( + comp_ctx->orc_thread_safe_context))) { + aot_set_last_error("get context from LLVM ThreadSafeContext failed."); + goto fail; + } + + if (!(comp_ctx->builder = LLVMCreateBuilderInContext(comp_ctx->context))) { + aot_set_last_error("create LLVM builder failed."); + goto fail; + } + + /* Create LLVM module for each jit function, note: + different from non ORC JIT mode, no need to dispose it, + it will be disposed when the thread safe context is disposed */ + if (!(comp_ctx->module = LLVMModuleCreateWithNameInContext( + "WASM Module", comp_ctx->context))) { + aot_set_last_error("create LLVM module failed."); + goto fail; + } +#if LLVM_VERSION_MAJOR >= 19 + LLVMSetIsNewDbgInfoFormat(comp_ctx->module, true); +#endif + +#if WASM_ENABLE_LINUX_PERF != 0 + if (wasm_runtime_get_linux_perf()) { + /* FramePointerKind.All */ + LLVMMetadataRef val = + LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 2, false)); + const char *key = "frame-pointer"; + LLVMAddModuleFlag(comp_ctx->module, LLVMModuleFlagBehaviorWarning, key, + strlen(key), val); + + comp_ctx->emit_frame_pointer = true; + } +#endif + + if (BH_LIST_ERROR == bh_list_init(&comp_ctx->native_symbols)) { + goto fail; + } + +#if WASM_ENABLE_DEBUG_AOT != 0 + if (!(comp_ctx->debug_builder = LLVMCreateDIBuilder(comp_ctx->module))) { + aot_set_last_error("create LLVM Debug Infor builder failed."); + goto fail; + } + + LLVMAddModuleFlag( + comp_ctx->module, LLVMModuleFlagBehaviorWarning, "Debug Info Version", + strlen("Debug Info Version"), + LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 3, false))); + + comp_ctx->debug_file = dwarf_gen_file_info(comp_ctx); + if (!comp_ctx->debug_file) { + aot_set_last_error("dwarf generate file info failed"); + goto fail; + } + comp_ctx->debug_comp_unit = dwarf_gen_comp_unit_info(comp_ctx); + if (!comp_ctx->debug_comp_unit) { + aot_set_last_error("dwarf generate compile unit info failed"); + goto fail; + } +#endif + + if (option->enable_bulk_memory) + comp_ctx->enable_bulk_memory = true; + + if (option->enable_thread_mgr) + comp_ctx->enable_thread_mgr = true; + + if (option->enable_tail_call) + comp_ctx->enable_tail_call = true; + + if (option->enable_ref_types) + comp_ctx->enable_ref_types = true; + + if (option->enable_aux_stack_frame) + comp_ctx->enable_aux_stack_frame = true; + + if (option->enable_perf_profiling) + comp_ctx->enable_perf_profiling = true; + + if (option->enable_memory_profiling) + comp_ctx->enable_memory_profiling = true; + + if (option->enable_aux_stack_check) + comp_ctx->enable_aux_stack_check = true; + + if (option->is_indirect_mode) + comp_ctx->is_indirect_mode = true; + + if (option->disable_llvm_intrinsics) + comp_ctx->disable_llvm_intrinsics = true; + + if (option->disable_llvm_lto) + comp_ctx->disable_llvm_lto = true; + + if (option->enable_llvm_pgo) + comp_ctx->enable_llvm_pgo = true; + + if (option->use_prof_file) + comp_ctx->use_prof_file = option->use_prof_file; + + if (option->enable_stack_estimation) + comp_ctx->enable_stack_estimation = true; + + if (option->quick_invoke_c_api_import) + comp_ctx->quick_invoke_c_api_import = true; + + if (option->llvm_passes) + comp_ctx->llvm_passes = option->llvm_passes; + + if (option->builtin_intrinsics) + comp_ctx->builtin_intrinsics = option->builtin_intrinsics; + + if (option->enable_gc) + comp_ctx->enable_gc = true; + + comp_ctx->opt_level = option->opt_level; + comp_ctx->size_level = option->size_level; + + comp_ctx->custom_sections_wp = option->custom_sections; + comp_ctx->custom_sections_count = option->custom_sections_count; + + if (option->is_jit_mode) { + comp_ctx->is_jit_mode = true; + +#ifndef OS_ENABLE_HW_BOUND_CHECK + comp_ctx->enable_bound_check = true; + /* Always enable stack boundary check if `bounds-checks` + is enabled */ + comp_ctx->enable_stack_bound_check = true; +#else + comp_ctx->enable_bound_check = false; + /* When `bounds-checks` is disabled, we set stack boundary + check status according to the compilation option */ +#if WASM_DISABLE_STACK_HW_BOUND_CHECK != 0 + /* Native stack overflow check with hardware trap is disabled, + we need to enable the check by LLVM JITed/AOTed code */ + comp_ctx->enable_stack_bound_check = true; +#else + /* Native stack overflow check with hardware trap is enabled, + no need to enable the check by LLVM JITed/AOTed code */ + comp_ctx->enable_stack_bound_check = false; +#endif +#endif + + /* Create TargetMachine */ + if (!create_target_machine_detect_host(comp_ctx)) + goto fail; + + /* Create LLJIT Instance */ + if (!orc_jit_create(comp_ctx)) + goto fail; + } + else { + /* Create LLVM target machine */ + arch = option->target_arch; + abi = option->target_abi; + cpu = option->target_cpu; + features = option->cpu_features; + opt_level = option->opt_level; + size_level = option->size_level; + + /* verify external llc compiler */ + comp_ctx->external_llc_compiler = getenv("WAMRC_LLC_COMPILER"); + if (comp_ctx->external_llc_compiler) { +#if defined(_WIN32) || defined(_WIN32_) + comp_ctx->external_llc_compiler = NULL; + LOG_WARNING("External LLC compiler not supported on Windows."); +#else + if (access(comp_ctx->external_llc_compiler, X_OK) != 0) { + LOG_WARNING("WAMRC_LLC_COMPILER [%s] not found, fallback to " + "default pipeline", + comp_ctx->external_llc_compiler); + comp_ctx->external_llc_compiler = NULL; + } + else { + comp_ctx->llc_compiler_flags = getenv("WAMRC_LLC_FLAGS"); + LOG_VERBOSE("Using external LLC compiler [%s]", + comp_ctx->external_llc_compiler); + } +#endif + } + + /* verify external asm compiler */ + if (!comp_ctx->external_llc_compiler) { + comp_ctx->external_asm_compiler = getenv("WAMRC_ASM_COMPILER"); + if (comp_ctx->external_asm_compiler) { +#if defined(_WIN32) || defined(_WIN32_) + comp_ctx->external_asm_compiler = NULL; + LOG_WARNING("External ASM compiler not supported on Windows."); +#else + if (access(comp_ctx->external_asm_compiler, X_OK) != 0) { + LOG_WARNING( + "WAMRC_ASM_COMPILER [%s] not found, fallback to " + "default pipeline", + comp_ctx->external_asm_compiler); + comp_ctx->external_asm_compiler = NULL; + } + else { + comp_ctx->asm_compiler_flags = getenv("WAMRC_ASM_FLAGS"); + LOG_VERBOSE("Using external ASM compiler [%s]", + comp_ctx->external_asm_compiler); + } +#endif + } + } + + if (arch) { + /* Add default sub-arch if not specified */ + if (!strcmp(arch, "arm")) + arch = "armv4"; + else if (!strcmp(arch, "armeb")) + arch = "armv4eb"; + else if (!strcmp(arch, "thumb")) + arch = "thumbv4t"; + else if (!strcmp(arch, "thumbeb")) + arch = "thumbv4teb"; + else if (!strcmp(arch, "aarch64")) + arch = "aarch64v8"; + else if (!strcmp(arch, "aarch64_be")) + arch = "aarch64_bev8"; + } + + /* Check target arch */ + if (arch && !check_target_arch(arch)) { + if (!strcmp(arch, "help")) + print_supported_targets(); + else + aot_set_last_error( + "Invalid target. " + "Use --target=help to list all supported targets"); + goto fail; + } + + /* Check target ABI */ + if (abi && !check_target_abi(abi)) { + if (!strcmp(abi, "help")) + print_supported_abis(); + else + aot_set_last_error( + "Invalid target ABI. " + "Use --target-abi=help to list all supported ABI"); + goto fail; + } + + /* Set default abi for riscv target */ + if (arch && !strncmp(arch, "riscv", 5) && !abi) { + if (!strcmp(arch, "riscv64")) + abi = "lp64d"; + else + abi = "ilp32d"; + } + +#if defined(__APPLE__) || defined(__MACH__) + if (!abi) { + /* On MacOS platform, set abi to "gnu" to avoid generating + object file of Mach-O binary format which is unsupported */ + abi = "gnu"; + if (!arch && !cpu && !features) { + /* Get CPU name of the host machine to avoid checking + SIMD capability failed */ + if (!(cpu = cpu_new = LLVMGetHostCPUName())) { + aot_set_last_error("llvm get host cpu name failed."); + goto fail; + } + } + } +#endif + + if (abi) { + /* Construct target triple: --- */ + const char *vendor_sys; + char *arch1 = arch, default_arch[32] = { 0 }; + + if (!arch1) { + char *default_triple = LLVMGetDefaultTargetTriple(); + + if (!default_triple) { + aot_set_last_error( + "llvm get default target triple failed."); + goto fail; + } + + vendor_sys = strstr(default_triple, "-"); + bh_assert(vendor_sys); + bh_memcpy_s(default_arch, sizeof(default_arch), default_triple, + (uint32)(vendor_sys - default_triple)); + arch1 = default_arch; + + LLVMDisposeMessage(default_triple); + } + + /** + * Set - according to abi to generate the object file + * with the correct file format which might be different from the + * default object file format of the host, e.g., generating AOT file + * for Windows/MacOS under Linux host, or generating AOT file for + * Linux/MacOS under Windows host. + */ + + if (!strcmp(abi, "msvc")) { + if (!strcmp(arch1, "i386")) + vendor_sys = "-pc-win32-"; + else + vendor_sys = "-pc-windows-"; + } + else { + if (is_baremetal_target(arch, cpu, abi)) + vendor_sys = "-unknown-none-"; + else + vendor_sys = "-pc-linux-"; + } + + bh_assert(strlen(arch1) + strlen(vendor_sys) + strlen(abi) + < sizeof(triple_buf)); + bh_memcpy_s(triple_buf, (uint32)sizeof(triple_buf), arch1, + (uint32)strlen(arch1)); + bh_memcpy_s(triple_buf + strlen(arch1), + (uint32)(sizeof(triple_buf) - strlen(arch1)), + vendor_sys, (uint32)strlen(vendor_sys)); + bh_memcpy_s(triple_buf + strlen(arch1) + strlen(vendor_sys), + (uint32)(sizeof(triple_buf) - strlen(arch1) + - strlen(vendor_sys)), + abi, (uint32)strlen(abi)); + triple = triple_buf; + } + else if (arch) { + /* Construct target triple: --- */ + const char *vendor_sys; + char *default_triple = LLVMGetDefaultTargetTriple(); + + if (!default_triple) { + aot_set_last_error("llvm get default target triple failed."); + goto fail; + } + + if (strstr(default_triple, "windows")) { + vendor_sys = "-pc-windows-"; + if (!abi) + abi = "msvc"; + } + else if (strstr(default_triple, "win32")) { + vendor_sys = "-pc-win32-"; + if (!abi) + abi = "msvc"; + } + else if (is_baremetal_target(arch, cpu, abi)) { + vendor_sys = "-unknown-none-"; + if (!abi) + abi = "gnu"; + } + else { + vendor_sys = "-pc-linux-"; + if (!abi) + abi = "gnu"; + } + + LLVMDisposeMessage(default_triple); + + bh_assert(strlen(arch) + strlen(vendor_sys) + strlen(abi) + < sizeof(triple_buf)); + bh_memcpy_s(triple_buf, (uint32)sizeof(triple_buf), arch, + (uint32)strlen(arch)); + bh_memcpy_s(triple_buf + strlen(arch), + (uint32)(sizeof(triple_buf) - strlen(arch)), vendor_sys, + (uint32)strlen(vendor_sys)); + bh_memcpy_s(triple_buf + strlen(arch) + strlen(vendor_sys), + (uint32)(sizeof(triple_buf) - strlen(arch) + - strlen(vendor_sys)), + abi, (uint32)strlen(abi)); + triple = triple_buf; + } + + if (!cpu && features) { + aot_set_last_error("cpu isn't specified for cpu features."); + goto fail; + } + + if (!triple && !cpu) { + /* Get a triple for the host machine */ + if (!(triple_norm = triple_norm_new = + LLVMGetDefaultTargetTriple())) { + aot_set_last_error("llvm get default target triple failed."); + goto fail; + } + /* Get CPU name of the host machine */ + if (!(cpu = cpu_new = LLVMGetHostCPUName())) { + aot_set_last_error("llvm get host cpu name failed."); + goto fail; + } + } + else if (triple) { + /* Normalize a target triple */ + if (!(triple_norm = triple_norm_new = + LLVMNormalizeTargetTriple(triple))) { + snprintf(buf, sizeof(buf), + "llvm normlalize target triple (%s) failed.", triple); + aot_set_last_error(buf); + goto fail; + } + if (!cpu) + cpu = ""; + } + else { + /* triple is NULL, cpu isn't NULL */ + snprintf(buf, sizeof(buf), "target isn't specified for cpu %s.", + cpu); + aot_set_last_error(buf); + goto fail; + } + + /* Add module flag and cpu feature for riscv target */ + if (arch && !strncmp(arch, "riscv", 5)) { + LLVMMetadataRef meta_target_abi; + + if (!(meta_target_abi = LLVMMDStringInContext2(comp_ctx->context, + abi, strlen(abi)))) { + aot_set_last_error("create metadata string failed."); + goto fail; + } + LLVMAddModuleFlag(comp_ctx->module, LLVMModuleFlagBehaviorError, + "target-abi", strlen("target-abi"), + meta_target_abi); + + if (!strcmp(abi, "lp64d") || !strcmp(abi, "ilp32d")) { + if (features && !strstr(features, "+d")) { + snprintf(features_buf, sizeof(features_buf), "%s%s", + features, ",+d"); + features = features_buf; + } + else if (!features) { + features = "+d"; + } + } + } + + if (!features) + features = ""; + + /* Get target with triple, note that LLVMGetTargetFromTriple() + return 0 when success, but not true. */ + if (LLVMGetTargetFromTriple(triple_norm, &target, &err) != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + snprintf(buf, sizeof(buf), + "llvm get target from triple (%s) failed", triple_norm); + aot_set_last_error(buf); + goto fail; + } + + /* Save target arch */ + get_target_arch_from_triple(triple_norm, comp_ctx->target_arch, + sizeof(comp_ctx->target_arch)); + + if (option->bounds_checks == 1 || option->bounds_checks == 0) { + /* Set by user */ + comp_ctx->enable_bound_check = + (option->bounds_checks == 1) ? true : false; + } + else { + /* Unset by user, use default value */ + if (strstr(comp_ctx->target_arch, "64") + && !option->is_sgx_platform) { + comp_ctx->enable_bound_check = false; + } + else { + comp_ctx->enable_bound_check = true; + } + } + + if (comp_ctx->enable_bound_check) { + /* Always enable stack boundary check if `bounds-checks` + is enabled */ + comp_ctx->enable_stack_bound_check = true; + } + else { + /* When `bounds-checks` is disabled, we set stack boundary + check status according to the input option */ + comp_ctx->enable_stack_bound_check = + (option->stack_bounds_checks == 1) ? true : false; + } + + if ((comp_ctx->enable_stack_bound_check + || comp_ctx->enable_stack_estimation) + && option->stack_usage_file == NULL) { + if (!aot_generate_tempfile_name( + "wamrc-su", "su", comp_ctx->stack_usage_temp_file, + sizeof(comp_ctx->stack_usage_temp_file))) + goto fail; + comp_ctx->stack_usage_file = comp_ctx->stack_usage_temp_file; + } + else { + comp_ctx->stack_usage_file = option->stack_usage_file; + } + + os_printf("Create AoT compiler with:\n"); + os_printf(" target: %s\n", comp_ctx->target_arch); + os_printf(" target cpu: %s\n", cpu); + os_printf(" target triple: %s\n", triple_norm); + os_printf(" cpu features: %s\n", features); + os_printf(" opt level: %d\n", opt_level); + os_printf(" size level: %d\n", size_level); + switch (option->output_format) { + case AOT_LLVMIR_UNOPT_FILE: + os_printf(" output format: unoptimized LLVM IR\n"); + break; + case AOT_LLVMIR_OPT_FILE: + os_printf(" output format: optimized LLVM IR\n"); + break; + case AOT_FORMAT_FILE: + os_printf(" output format: AoT file\n"); + break; + case AOT_OBJECT_FILE: + os_printf(" output format: native object file\n"); + break; + } + + LLVMSetTarget(comp_ctx->module, triple_norm); + + if (!LLVMTargetHasTargetMachine(target)) { + snprintf(buf, sizeof(buf), + "no target machine for this target (%s).", triple_norm); + aot_set_last_error(buf); + goto fail; + } + + /* Report error if target isn't arc and hasn't asm backend. + For arc target, as it cannot emit to memory buffer of elf file + currently, we let it emit to assembly file instead, and then call + arc-gcc to compile + asm file to elf file, and read elf file to memory buffer. */ + if (strncmp(comp_ctx->target_arch, "arc", 3) + && !LLVMTargetHasAsmBackend(target)) { + snprintf(buf, sizeof(buf), "no asm backend for this target (%s).", + LLVMGetTargetName(target)); + aot_set_last_error(buf); + goto fail; + } + + /* Set code model */ + if (size_level == 0) + code_model = LLVMCodeModelLarge; + else if (size_level == 1) + code_model = LLVMCodeModelMedium; + else if (size_level == 2) + code_model = LLVMCodeModelKernel; + else + code_model = LLVMCodeModelSmall; + + /* Create the target machine */ + if (!(comp_ctx->target_machine = LLVMCreateTargetMachineWithOpts( + target, triple_norm, cpu, features, opt_level, + LLVMRelocStatic, code_model, false, + comp_ctx->stack_usage_file))) { + aot_set_last_error("create LLVM target machine failed."); + goto fail; + } + + /* If only to create target machine for querying information, early stop + */ + if ((arch && !strcmp(arch, "help")) || (abi && !strcmp(abi, "help")) + || (cpu && !strcmp(cpu, "help")) + || (features && !strcmp(features, "+help"))) { + LOG_DEBUG( + "create LLVM target machine only for printing help info."); + goto fail; + } + } + + triple = LLVMGetTargetMachineTriple(comp_ctx->target_machine); + if (!triple) { + aot_set_last_error("get target machine triple failed."); + goto fail; + } + if (strstr(triple, "linux") && !strcmp(comp_ctx->target_arch, "x86_64")) { + if (option->segue_flags) { + if (option->segue_flags & (1 << 0)) + comp_ctx->enable_segue_i32_load = true; + if (option->segue_flags & (1 << 1)) + comp_ctx->enable_segue_i64_load = true; + if (option->segue_flags & (1 << 2)) + comp_ctx->enable_segue_f32_load = true; + if (option->segue_flags & (1 << 3)) + comp_ctx->enable_segue_f64_load = true; + if (option->segue_flags & (1 << 4)) + comp_ctx->enable_segue_v128_load = true; + if (option->segue_flags & (1 << 8)) + comp_ctx->enable_segue_i32_store = true; + if (option->segue_flags & (1 << 9)) + comp_ctx->enable_segue_i64_store = true; + if (option->segue_flags & (1 << 10)) + comp_ctx->enable_segue_f32_store = true; + if (option->segue_flags & (1 << 11)) + comp_ctx->enable_segue_f64_store = true; + if (option->segue_flags & (1 << 12)) + comp_ctx->enable_segue_v128_store = true; + } + } + LLVMDisposeMessage(triple); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + WASMModule *wasm_module = (WASMModule *)comp_data->wasm_module; + + /* Return error if SIMD is disabled by command line but SIMD instructions + * are used */ + if (!option->enable_simd && wasm_module->is_simd_used) { + aot_set_last_error("SIMD is disabled by --disable-simd but SIMD " + "instructions are used in this module"); + goto fail; + } + + /* Disable features when they are not actually used */ + if (!wasm_module->is_simd_used) { + option->enable_simd = comp_ctx->enable_simd = false; + } + if (!wasm_module->is_ref_types_used) { + option->enable_ref_types = comp_ctx->enable_ref_types = false; + } + if (!wasm_module->is_bulk_memory_used) { + option->enable_bulk_memory = comp_ctx->enable_bulk_memory = false; + } +#endif + + if (option->enable_simd && strcmp(comp_ctx->target_arch, "x86_64") != 0 + && strncmp(comp_ctx->target_arch, "aarch64", 7) != 0) { + /* Disable simd if it isn't supported by target arch */ + option->enable_simd = false; + } + + if (option->enable_simd) { + char *tmp; + bool check_simd_ret; + + comp_ctx->enable_simd = true; + + if (!(tmp = LLVMGetTargetMachineCPU(comp_ctx->target_machine))) { + aot_set_last_error("get CPU from Target Machine fail"); + goto fail; + } + + check_simd_ret = + aot_check_simd_compatibility(comp_ctx->target_arch, tmp); + LLVMDisposeMessage(tmp); + if (!check_simd_ret) { + aot_set_last_error("SIMD compatibility check failed, " + "try adding --cpu= to specify a cpu " + "or adding --disable-simd to disable SIMD"); + goto fail; + } + } + + if (!(target_data_ref = + LLVMCreateTargetDataLayout(comp_ctx->target_machine))) { + aot_set_last_error("create LLVM target data layout failed."); + goto fail; + } + LLVMSetModuleDataLayout(comp_ctx->module, target_data_ref); + comp_ctx->pointer_size = LLVMPointerSize(target_data_ref); + LLVMDisposeTargetData(target_data_ref); + + comp_ctx->optimize = true; + if (option->output_format == AOT_LLVMIR_UNOPT_FILE) + comp_ctx->optimize = false; + + /* Create metadata for llvm float experimental constrained intrinsics */ + if (!(comp_ctx->fp_rounding_mode = LLVMMDStringInContext( + comp_ctx->context, fp_round, (uint32)strlen(fp_round))) + || !(comp_ctx->fp_exception_behavior = LLVMMDStringInContext( + comp_ctx->context, fp_exce, (uint32)strlen(fp_exce)))) { + aot_set_last_error("create float llvm metadata failed."); + goto fail; + } + + if (!aot_set_llvm_basic_types(&comp_ctx->basic_types, comp_ctx->context, + comp_ctx->pointer_size)) { + aot_set_last_error("create LLVM basic types failed."); + goto fail; + } + + if (!aot_create_llvm_consts(&comp_ctx->llvm_consts, comp_ctx)) { + aot_set_last_error("create LLVM const values failed."); + goto fail; + } + + /* set exec_env data type to int8** */ + comp_ctx->exec_env_type = comp_ctx->basic_types.int8_pptr_type; + + /* set aot_inst data type to int8* */ + comp_ctx->aot_inst_type = INT8_PTR_TYPE; + + /* Create function context for each function */ + comp_ctx->func_ctx_count = comp_data->func_count; + if (comp_data->func_count > 0 + && !(comp_ctx->func_ctxes = + aot_create_func_contexts(comp_data, comp_ctx))) + goto fail; + + if (cpu) { + uint32 len = (uint32)strlen(cpu) + 1; + if (!(comp_ctx->target_cpu = wasm_runtime_malloc(len))) { + aot_set_last_error("allocate memory failed"); + goto fail; + } + bh_memcpy_s(comp_ctx->target_cpu, len, cpu, len); + } + + if (comp_ctx->disable_llvm_intrinsics) + aot_intrinsic_fill_capability_flags(comp_ctx); + + ret = comp_ctx; + +fail: + if (triple_norm_new) + LLVMDisposeMessage(triple_norm_new); + + if (cpu_new) + LLVMDisposeMessage(cpu_new); + + if (!ret) + aot_destroy_comp_context(comp_ctx); + + (void)i; + return ret; +} + +void +aot_destroy_comp_context(AOTCompContext *comp_ctx) +{ + if (!comp_ctx) + return; + + if (comp_ctx->stack_usage_file == comp_ctx->stack_usage_temp_file) { + (void)unlink(comp_ctx->stack_usage_temp_file); + } + + if (comp_ctx->target_machine) + LLVMDisposeTargetMachine(comp_ctx->target_machine); + + if (comp_ctx->builder) + LLVMDisposeBuilder(comp_ctx->builder); + + if (comp_ctx->orc_thread_safe_context) + LLVMOrcDisposeThreadSafeContext(comp_ctx->orc_thread_safe_context); + + /* Note: don't dispose comp_ctx->context and comp_ctx->module as + they are disposed when disposing the thread safe context */ + + /* Has to be the last one */ + if (comp_ctx->orc_jit) + LLVMOrcDisposeLLLazyJIT(comp_ctx->orc_jit); + + if (comp_ctx->func_ctxes) + aot_destroy_func_contexts(comp_ctx, comp_ctx->func_ctxes, + comp_ctx->func_ctx_count); + + if (bh_list_length(&comp_ctx->native_symbols) > 0) { + AOTNativeSymbol *sym = bh_list_first_elem(&comp_ctx->native_symbols); + while (sym) { + AOTNativeSymbol *t = bh_list_elem_next(sym); + bh_list_remove(&comp_ctx->native_symbols, sym); + wasm_runtime_free(sym); + sym = t; + } + } + + if (comp_ctx->target_cpu) { + wasm_runtime_free(comp_ctx->target_cpu); + } + + if (comp_ctx->aot_frame) { + wasm_runtime_free(comp_ctx->aot_frame); + } + + wasm_runtime_free(comp_ctx); +} + +static bool +insert_native_symbol(AOTCompContext *comp_ctx, const char *symbol, int32 idx) +{ + AOTNativeSymbol *sym = wasm_runtime_malloc(sizeof(AOTNativeSymbol)); + + if (!sym) { + aot_set_last_error("alloc native symbol failed."); + return false; + } + + memset(sym, 0, sizeof(AOTNativeSymbol)); + bh_assert(strlen(symbol) <= sizeof(sym->symbol)); + snprintf(sym->symbol, sizeof(sym->symbol), "%s", symbol); + sym->index = idx; + + if (BH_LIST_ERROR == bh_list_insert(&comp_ctx->native_symbols, sym)) { + wasm_runtime_free(sym); + aot_set_last_error("insert native symbol to list failed."); + return false; + } + + return true; +} + +int32 +aot_get_native_symbol_index(AOTCompContext *comp_ctx, const char *symbol) +{ + int32 idx = -1; + AOTNativeSymbol *sym = NULL; + + sym = bh_list_first_elem(&comp_ctx->native_symbols); + + /* Lookup an existing symobl record */ + + while (sym) { + if (strcmp(sym->symbol, symbol) == 0) { + idx = sym->index; + break; + } + sym = bh_list_elem_next(sym); + } + + /* Given symbol is not exist in list, then we alloc a new index for it */ + + if (idx < 0) { + if (comp_ctx->pointer_size == sizeof(uint32) + && (!strncmp(symbol, "f64#", 4) || !strncmp(symbol, "i64#", 4))) { + idx = bh_list_length(&comp_ctx->native_symbols); + /* Add 4 bytes padding on 32-bit target to make sure that + the f64 const is stored on 8-byte aligned address */ + if (idx & 1) { + if (!insert_native_symbol(comp_ctx, "__ignore", idx)) { + return -1; + } + } + } + + idx = bh_list_length(&comp_ctx->native_symbols); + if (!insert_native_symbol(comp_ctx, symbol, idx)) { + return -1; + } + + if (comp_ctx->pointer_size == sizeof(uint32) + && (!strncmp(symbol, "f64#", 4) || !strncmp(symbol, "i64#", 4))) { + /* f64 const occupies 2 pointer slots on 32-bit target */ + if (!insert_native_symbol(comp_ctx, "__ignore", idx + 1)) { + return -1; + } + } + } + + return idx; +} + +void +aot_value_stack_push(const AOTCompContext *comp_ctx, AOTValueStack *stack, + AOTValue *value) +{ + if (!stack->value_list_head) + stack->value_list_head = stack->value_list_end = value; + else { + stack->value_list_end->next = value; + value->prev = stack->value_list_end; + stack->value_list_end = value; + } + + if (comp_ctx->aot_frame) { + switch (value->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_I1: + push_i32(comp_ctx->aot_frame, value); + break; + case VALUE_TYPE_I64: + push_i64(comp_ctx->aot_frame, value); + break; + case VALUE_TYPE_F32: + push_f32(comp_ctx->aot_frame, value); + break; + case VALUE_TYPE_F64: + push_f64(comp_ctx->aot_frame, value); + break; + case VALUE_TYPE_V128: + push_v128(comp_ctx->aot_frame, value); + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + push_ref(comp_ctx->aot_frame, value); + break; +#if WASM_ENABLE_GC != 0 + case VALUE_TYPE_GC_REF: + bh_assert(comp_ctx->enable_gc); + push_gc_ref(comp_ctx->aot_frame, value); + break; +#endif + default: + bh_assert(0); + break; + } + } +} + +AOTValue * +aot_value_stack_pop(const AOTCompContext *comp_ctx, AOTValueStack *stack) +{ + AOTValue *value = stack->value_list_end; + + bh_assert(stack->value_list_end); + + if (stack->value_list_head == stack->value_list_end) + stack->value_list_head = stack->value_list_end = NULL; + else { + stack->value_list_end = stack->value_list_end->prev; + stack->value_list_end->next = NULL; + value->prev = NULL; + } + + if (comp_ctx->aot_frame) { + bh_assert(value); + bh_assert(value->value == (comp_ctx->aot_frame->sp - 1)->value); + bh_assert(value->type == (comp_ctx->aot_frame->sp - 1)->type); + + switch (value->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_I1: + pop_i32(comp_ctx->aot_frame); + break; + case VALUE_TYPE_I64: + pop_i64(comp_ctx->aot_frame); + break; + case VALUE_TYPE_F32: + pop_f32(comp_ctx->aot_frame); + break; + case VALUE_TYPE_F64: + pop_f64(comp_ctx->aot_frame); + break; + case VALUE_TYPE_V128: + pop_v128(comp_ctx->aot_frame); + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + pop_ref(comp_ctx->aot_frame); + break; +#if WASM_ENABLE_GC != 0 + case VALUE_TYPE_GC_REF: + bh_assert(comp_ctx->enable_gc); + pop_gc_ref(comp_ctx->aot_frame); + break; +#endif + default: + bh_assert(0); + break; + } + } + + return value; +} + +void +aot_value_stack_destroy(AOTCompContext *comp_ctx, AOTValueStack *stack) +{ + AOTValue *value = stack->value_list_head, *p; + + while (value) { + p = value->next; + wasm_runtime_free(value); + value = p; + } + + stack->value_list_head = NULL; + stack->value_list_end = NULL; +} + +void +aot_block_stack_push(AOTBlockStack *stack, AOTBlock *block) +{ + if (!stack->block_list_head) + stack->block_list_head = stack->block_list_end = block; + else { + stack->block_list_end->next = block; + block->prev = stack->block_list_end; + stack->block_list_end = block; + } +} + +AOTBlock * +aot_block_stack_pop(AOTBlockStack *stack) +{ + AOTBlock *block = stack->block_list_end; + + bh_assert(stack->block_list_end); + + if (stack->block_list_head == stack->block_list_end) + stack->block_list_head = stack->block_list_end = NULL; + else { + stack->block_list_end = stack->block_list_end->prev; + stack->block_list_end->next = NULL; + block->prev = NULL; + } + + return block; +} + +void +aot_block_stack_destroy(AOTCompContext *comp_ctx, AOTBlockStack *stack) +{ + AOTBlock *block = stack->block_list_head, *p; + + while (block) { + p = block->next; + aot_value_stack_destroy(comp_ctx, &block->value_stack); + aot_block_destroy(comp_ctx, block); + block = p; + } + + stack->block_list_head = NULL; + stack->block_list_end = NULL; +} + +void +aot_block_destroy(AOTCompContext *comp_ctx, AOTBlock *block) +{ + aot_value_stack_destroy(comp_ctx, &block->value_stack); + if (block->param_types) + wasm_runtime_free(block->param_types); + if (block->param_phis) + wasm_runtime_free(block->param_phis); + if (block->else_param_phis) + wasm_runtime_free(block->else_param_phis); + if (block->result_types) + wasm_runtime_free(block->result_types); + if (block->result_phis) + wasm_runtime_free(block->result_phis); + wasm_runtime_free(block); +} + +bool +aot_checked_addr_list_add(AOTFuncContext *func_ctx, uint32 local_idx, + uint32 offset, uint32 bytes) +{ + AOTCheckedAddr *node = func_ctx->checked_addr_list; + + if (!(node = wasm_runtime_malloc(sizeof(AOTCheckedAddr)))) { + aot_set_last_error("allocate memory failed."); + return false; + } + + node->local_idx = local_idx; + node->offset = offset; + node->bytes = bytes; + + node->next = func_ctx->checked_addr_list; + func_ctx->checked_addr_list = node; + return true; +} + +void +aot_checked_addr_list_del(AOTFuncContext *func_ctx, uint32 local_idx) +{ + AOTCheckedAddr *node = func_ctx->checked_addr_list; + AOTCheckedAddr *node_prev = NULL, *node_next; + + while (node) { + node_next = node->next; + + if (node->local_idx == local_idx) { + if (!node_prev) + func_ctx->checked_addr_list = node_next; + else + node_prev->next = node_next; + wasm_runtime_free(node); + } + else { + node_prev = node; + } + + node = node_next; + } +} + +bool +aot_checked_addr_list_find(AOTFuncContext *func_ctx, uint32 local_idx, + uint32 offset, uint32 bytes) +{ + AOTCheckedAddr *node = func_ctx->checked_addr_list; + + while (node) { + if (node->local_idx == local_idx && node->offset == offset + && node->bytes >= bytes) { + return true; + } + node = node->next; + } + + return false; +} + +void +aot_checked_addr_list_destroy(AOTFuncContext *func_ctx) +{ + AOTCheckedAddr *node = func_ctx->checked_addr_list, *node_next; + + while (node) { + node_next = node->next; + wasm_runtime_free(node); + node = node_next; + } + + func_ctx->checked_addr_list = NULL; +} + +bool +aot_build_zero_function_ret(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, AOTFuncType *func_type) +{ + LLVMValueRef ret = NULL; + + if (func_type->result_count) { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + ret = LLVMBuildRet(comp_ctx->builder, I32_ZERO); + break; + case VALUE_TYPE_I64: + ret = LLVMBuildRet(comp_ctx->builder, I64_ZERO); + break; + case VALUE_TYPE_F32: + ret = LLVMBuildRet(comp_ctx->builder, F32_ZERO); + break; + case VALUE_TYPE_F64: + ret = LLVMBuildRet(comp_ctx->builder, F64_ZERO); + break; + case VALUE_TYPE_V128: + ret = + LLVMBuildRet(comp_ctx->builder, LLVM_CONST(i64x2_vec_zero)); + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + if (comp_ctx->enable_ref_types) + ret = LLVMBuildRet(comp_ctx->builder, REF_NULL); +#if WASM_ENABLE_GC != 0 + else if (comp_ctx->enable_gc) + ret = LLVMBuildRet(comp_ctx->builder, GC_REF_NULL); +#endif + else + bh_assert(0); + break; +#if WASM_ENABLE_GC != 0 + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_NULLREF: + /* case REF_TYPE_FUNCREF: */ + /* case REF_TYPE_EXTERNREF: */ + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + bh_assert(comp_ctx->enable_gc); + ret = LLVMBuildRet(comp_ctx->builder, GC_REF_NULL); + break; +#endif + default: + bh_assert(0); + } + } + else { + ret = LLVMBuildRetVoid(comp_ctx->builder); + } + + if (!ret) { + aot_set_last_error("llvm build ret failed."); + return false; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + /* debug_func is NULL for precheck function */ + if (func_ctx->debug_func != NULL) { + LLVMMetadataRef return_location = + dwarf_gen_func_ret_location(comp_ctx, func_ctx); + LLVMInstructionSetDebugLoc(ret, return_location); + } +#endif + return true; +} + +static LLVMValueRef +__call_llvm_intrinsic(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, const char *name, + LLVMTypeRef ret_type, LLVMTypeRef *param_types, + int param_count, LLVMValueRef *param_values) +{ + LLVMValueRef func, ret; + LLVMTypeRef func_type; + const char *symname; + int32 func_idx; + + if (comp_ctx->disable_llvm_intrinsics + && aot_intrinsic_check_capability(comp_ctx, name)) { + if (func_ctx == NULL) { + aot_set_last_error_v("invalid func_ctx for intrinsic: %s", name); + return NULL; + } + + if (!(func_type = LLVMFunctionType(ret_type, param_types, + (uint32)param_count, false))) { + aot_set_last_error("create LLVM intrinsic function type failed."); + return NULL; + } + if (!(func_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error( + "create LLVM intrinsic function pointer type failed."); + return NULL; + } + + if (!(symname = aot_intrinsic_get_symbol(name))) { + aot_set_last_error_v("runtime intrinsic not implemented: %s\n", + name); + return NULL; + } + + func_idx = + aot_get_native_symbol_index((AOTCompContext *)comp_ctx, symname); + if (func_idx < 0) { + aot_set_last_error_v("get runtime intrinsc index failed: %s\n", + name); + return NULL; + } + + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_type, func_idx))) { + aot_set_last_error_v("get runtime intrinsc failed: %s\n", name); + return NULL; + } + } + else { + /* Declare llvm intrinsic function if necessary */ + if (!(func = LLVMGetNamedFunction(func_ctx->module, name))) { + if (!(func_type = LLVMFunctionType(ret_type, param_types, + (uint32)param_count, false))) { + aot_set_last_error( + "create LLVM intrinsic function type failed."); + return NULL; + } + + if (!(func = LLVMAddFunction(func_ctx->module, name, func_type))) { + aot_set_last_error("add LLVM intrinsic function failed."); + return NULL; + } + } + } + +#if LLVM_VERSION_MAJOR >= 14 + func_type = + LLVMFunctionType(ret_type, param_types, (uint32)param_count, false); +#endif + + /* Call the LLVM intrinsic function */ + if (!(ret = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + (uint32)param_count, "call"))) { + aot_set_last_error("llvm build intrinsic call failed."); + return NULL; + } + + return ret; +} + +LLVMValueRef +aot_call_llvm_intrinsic(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, const char *intrinsic, + LLVMTypeRef ret_type, LLVMTypeRef *param_types, + int param_count, ...) +{ + LLVMValueRef *param_values, ret; + va_list argptr; + uint64 total_size; + int i = 0; + + /* Create param values */ + total_size = sizeof(LLVMValueRef) * (uint64)param_count; + if (total_size >= UINT32_MAX + || !(param_values = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory for param values failed."); + return false; + } + + /* Load each param value */ + va_start(argptr, param_count); + while (i < param_count) + param_values[i++] = va_arg(argptr, LLVMValueRef); + va_end(argptr); + + ret = __call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, ret_type, + param_types, param_count, param_values); + + wasm_runtime_free(param_values); + + return ret; +} + +LLVMValueRef +aot_call_llvm_intrinsic_v(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, const char *intrinsic, + LLVMTypeRef ret_type, LLVMTypeRef *param_types, + int param_count, va_list param_value_list) +{ + LLVMValueRef *param_values, ret; + uint64 total_size; + int i = 0; + + /* Create param values */ + total_size = sizeof(LLVMValueRef) * (uint64)param_count; + if (total_size >= UINT32_MAX + || !(param_values = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory for param values failed."); + return false; + } + + /* Load each param value */ + while (i < param_count) + param_values[i++] = va_arg(param_value_list, LLVMValueRef); + + ret = __call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, ret_type, + param_types, param_count, param_values); + + wasm_runtime_free(param_values); + + return ret; +} + +LLVMValueRef +aot_get_func_from_table(const AOTCompContext *comp_ctx, LLVMValueRef base, + LLVMTypeRef func_type, int32 index) +{ + LLVMValueRef func; + LLVMValueRef func_addr; + + if (!(func_addr = I32_CONST(index))) { + aot_set_last_error("construct function index failed."); + goto fail; + } + + if (!(func_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE, base, + &func_addr, 1, "func_addr"))) { + aot_set_last_error("get function addr by index failed."); + goto fail; + } + + func = + LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, func_addr, "func_tmp"); + + if (func == NULL) { + aot_set_last_error("get function pointer failed."); + goto fail; + } + + if (!(func = + LLVMBuildBitCast(comp_ctx->builder, func, func_type, "func"))) { + aot_set_last_error("cast function fialed."); + goto fail; + } + + return func; +fail: + return NULL; +} + +LLVMValueRef +aot_load_const_from_table(AOTCompContext *comp_ctx, LLVMValueRef base, + const WASMValue *value, uint8 value_type) +{ + LLVMValueRef const_index, const_addr, const_value; + LLVMTypeRef const_ptr_type, const_type; + char buf[128] = { 0 }; + int32 index; + + switch (value_type) { + case VALUE_TYPE_I32: + /* Store the raw int bits of i32 const as a hex string */ + snprintf(buf, sizeof(buf), "i32#%08" PRIX32, value->i32); + const_ptr_type = INT32_PTR_TYPE; + const_type = I32_TYPE; + break; + case VALUE_TYPE_I64: + /* Store the raw int bits of i64 const as a hex string */ + snprintf(buf, sizeof(buf), "i64#%016" PRIX64, value->i64); + const_ptr_type = INT64_PTR_TYPE; + const_type = I64_TYPE; + break; + case VALUE_TYPE_F32: + /* Store the raw int bits of f32 const as a hex string */ + snprintf(buf, sizeof(buf), "f32#%08" PRIX32, value->i32); + const_ptr_type = F32_PTR_TYPE; + const_type = F32_TYPE; + break; + case VALUE_TYPE_F64: + /* Store the raw int bits of f64 const as a hex string */ + snprintf(buf, sizeof(buf), "f64#%016" PRIX64, value->i64); + const_ptr_type = F64_PTR_TYPE; + const_type = F64_TYPE; + break; + default: + bh_assert(0); + return NULL; + } + + /* Load f32/f64 const from exec_env->native_symbol[index] */ + + index = aot_get_native_symbol_index(comp_ctx, buf); + if (index < 0) { + return NULL; + } + + if (!(const_index = I32_CONST(index))) { + aot_set_last_error("construct const index failed."); + return NULL; + } + + if (!(const_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE, base, + &const_index, 1, "const_addr_tmp"))) { + aot_set_last_error("get const addr by index failed."); + return NULL; + } + + if (!(const_addr = LLVMBuildBitCast(comp_ctx->builder, const_addr, + const_ptr_type, "const_addr"))) { + aot_set_last_error("cast const fialed."); + return NULL; + } + + if (!(const_value = LLVMBuildLoad2(comp_ctx->builder, const_type, + const_addr, "const_value"))) { + aot_set_last_error("load const failed."); + return NULL; + } + + (void)const_type; + return const_value; +} + +bool +aot_set_cond_br_weights(AOTCompContext *comp_ctx, LLVMValueRef cond_br, + int32 weights_true, int32 weights_false) +{ + LLVMMetadataRef md_nodes[3], meta_data; + LLVMValueRef meta_data_as_value; + + md_nodes[0] = LLVMMDStringInContext2(comp_ctx->context, "branch_weights", + strlen("branch_weights")); + md_nodes[1] = LLVMValueAsMetadata(I32_CONST(weights_true)); + md_nodes[2] = LLVMValueAsMetadata(I32_CONST(weights_false)); + + meta_data = LLVMMDNodeInContext2(comp_ctx->context, md_nodes, 3); + meta_data_as_value = LLVMMetadataAsValue(comp_ctx->context, meta_data); + + LLVMSetMetadata(cond_br, 2, meta_data_as_value); + + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_llvm.h b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm.h new file mode 100644 index 0000000..a47c054 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm.h @@ -0,0 +1,643 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_LLVM_H_ +#define _AOT_LLVM_H_ + +#include "aot.h" +#include "llvm/Config/llvm-config.h" +#include "llvm-c/Types.h" +#include "llvm-c/Target.h" +#include "llvm-c/Core.h" +#include "llvm-c/Object.h" +#include "llvm-c/OrcEE.h" +#include "llvm-c/ExecutionEngine.h" +#include "llvm-c/Analysis.h" +#include "llvm-c/BitWriter.h" +#if LLVM_VERSION_MAJOR < 17 +#include "llvm-c/Transforms/Utils.h" +#include "llvm-c/Transforms/Scalar.h" +#include "llvm-c/Transforms/Vectorize.h" +#include "llvm-c/Transforms/PassManagerBuilder.h" +#include "llvm-c/Initialization.h" +#endif + +#include "llvm-c/Orc.h" +#include "llvm-c/Error.h" +#include "llvm-c/Support.h" + +#include "llvm-c/TargetMachine.h" +#include "llvm-c/LLJIT.h" +#if WASM_ENABLE_DEBUG_AOT != 0 +#include "llvm-c/DebugInfo.h" +#endif + +#include "aot_orc_extra.h" +#include "aot_comp_option.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if LLVM_VERSION_MAJOR < 14 +#define LLVMBuildLoad2(builder, type, value, name) \ + LLVMBuildLoad(builder, value, name) + +#define LLVMBuildCall2(builder, type, func, args, num_args, name) \ + LLVMBuildCall(builder, func, args, num_args, name) + +#define LLVMBuildInBoundsGEP2(builder, type, ptr, indices, num_indices, name) \ + LLVMBuildInBoundsGEP(builder, ptr, indices, num_indices, name) +#else +/* Opaque pointer type */ +#define OPQ_PTR_TYPE INT8_PTR_TYPE +#endif + +#ifndef NDEBUG +#undef DEBUG_PASS +#undef DUMP_MODULE +// #define DEBUG_PASS +// #define DUMP_MODULE +#else +#undef DEBUG_PASS +#undef DUMP_MODULE +#endif + +struct AOTValueSlot; + +/** + * Value in the WASM operation stack, each stack element + * is an LLVM value + */ +typedef struct AOTValue { + struct AOTValue *next; + struct AOTValue *prev; + LLVMValueRef value; + /* VALUE_TYPE_I32/I64/F32/F64/VOID */ + uint8 type; + bool is_local; + uint32 local_idx; +} AOTValue; + +/** + * Value stack, represents stack elements in a WASM block + */ +typedef struct AOTValueStack { + AOTValue *value_list_head; + AOTValue *value_list_end; +} AOTValueStack; + +/* Record information of a value slot of local variable or stack + during translation */ +typedef struct AOTValueSlot { + /* The LLVM value of this slot */ + LLVMValueRef value; + + /* The value type of this slot */ + uint8 type; + + /* The dirty bit of the value slot. It's set if the value in + register is newer than the value in memory. */ + uint32 dirty : 1; + + /* Whether the new value in register is a reference, which is valid + only when the dirty bit is set. */ + uint32 ref : 1; + + /* Committed reference flag: + 0: uncommitted, 1: not-reference, 2: reference */ + uint32 committed_ref : 2; +} AOTValueSlot; + +/* Frame information for translation */ +typedef struct AOTCompFrame { + /* The current compilation context */ + struct AOTCompContext *comp_ctx; + /* The current function context */ + struct AOTFuncContext *func_ctx; + /* The current instruction pointer which is being compiled */ + const uint8 *frame_ip; + + /* Max local slot number */ + uint32 max_local_cell_num; + + /* Max operand stack slot number */ + uint32 max_stack_cell_num; + + /* Size of current AOTFrame/WASMInterpFrame */ + uint32 cur_frame_size; + + /* Stack top pointer */ + AOTValueSlot *sp; + + /* Local variables + stack operands */ + AOTValueSlot lp[1]; +} AOTCompFrame; + +typedef struct AOTBlock { + struct AOTBlock *next; + struct AOTBlock *prev; + + /* Block index */ + uint32 block_index; + /* LABEL_TYPE_BLOCK/LOOP/IF/FUNCTION */ + uint32 label_type; + /* Whether it is reachable */ + bool is_reachable; + /* Whether skip translation of wasm else branch */ + bool skip_wasm_code_else; + + /* code of else opcode of this block, if it is a IF block */ + uint8 *wasm_code_else; + /* code end of this block */ + uint8 *wasm_code_end; + + /* LLVM label points to code begin */ + LLVMBasicBlockRef llvm_entry_block; + /* LLVM label points to code else */ + LLVMBasicBlockRef llvm_else_block; + /* LLVM label points to code end */ + LLVMBasicBlockRef llvm_end_block; + + /* WASM operation stack */ + AOTValueStack value_stack; + + /* Param count/types/PHIs of this block */ + uint32 param_count; + uint8 *param_types; + LLVMValueRef *param_phis; + LLVMValueRef *else_param_phis; + + /* Result count/types/PHIs of this block */ + uint32 result_count; + uint8 *result_types; + LLVMValueRef *result_phis; + + /* The begin frame stack pointer of this block */ + AOTValueSlot *frame_sp_begin; + /* The max frame stack pointer that br/br_if/br_table/br_on_xxx + opcodes ever reached when they jumped to the end this block */ + AOTValueSlot *frame_sp_max_reached; +} AOTBlock; + +/** + * Block stack, represents WASM block stack elements + */ +typedef struct AOTBlockStack { + AOTBlock *block_list_head; + AOTBlock *block_list_end; + /* Current block index of each block type */ + uint32 block_index[3]; +} AOTBlockStack; + +typedef struct AOTCheckedAddr { + struct AOTCheckedAddr *next; + uint32 local_idx; + uint32 offset; + uint32 bytes; +} AOTCheckedAddr, *AOTCheckedAddrList; + +typedef struct AOTMemInfo { + LLVMValueRef mem_base_addr; + LLVMValueRef mem_data_size_addr; + LLVMValueRef mem_cur_page_count_addr; + LLVMValueRef mem_bound_check_1byte; + LLVMValueRef mem_bound_check_2bytes; + LLVMValueRef mem_bound_check_4bytes; + LLVMValueRef mem_bound_check_8bytes; + LLVMValueRef mem_bound_check_16bytes; +} AOTMemInfo; + +typedef struct AOTFuncContext { + AOTFunc *aot_func; + LLVMValueRef func; + LLVMValueRef precheck_func; + LLVMTypeRef func_type; + LLVMModuleRef module; + AOTBlockStack block_stack; + + LLVMValueRef exec_env; + LLVMValueRef aot_inst; + LLVMValueRef argv_buf; + LLVMValueRef native_stack_bound; + LLVMValueRef native_stack_top_min_addr; + LLVMValueRef aux_stack_bound; + LLVMValueRef aux_stack_bottom; + LLVMValueRef native_symbol; + LLVMValueRef func_ptrs; + + AOTMemInfo *mem_info; + + LLVMValueRef cur_exception; + + LLVMValueRef cur_frame; + LLVMValueRef cur_frame_ptr; + LLVMValueRef wasm_stack_top_bound; + LLVMValueRef wasm_stack_top_ptr; + + bool mem_space_unchanged; + AOTCheckedAddrList checked_addr_list; + + LLVMBasicBlockRef got_exception_block; + LLVMBasicBlockRef func_return_block; + LLVMValueRef exception_id_phi; + /* current ip when exception is thrown */ + LLVMValueRef exception_ip_phi; + LLVMValueRef func_type_indexes; +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMMetadataRef debug_func; +#endif + + unsigned int stack_consumption_for_func_call; + + LLVMValueRef locals[1]; +} AOTFuncContext; + +typedef struct AOTLLVMTypes { + LLVMTypeRef int1_type; + LLVMTypeRef int8_type; + LLVMTypeRef int16_type; + LLVMTypeRef int32_type; + LLVMTypeRef int64_type; + LLVMTypeRef intptr_t_type; + LLVMTypeRef float32_type; + LLVMTypeRef float64_type; + LLVMTypeRef void_type; + + LLVMTypeRef int8_ptr_type; + LLVMTypeRef int8_pptr_type; + LLVMTypeRef int16_ptr_type; + LLVMTypeRef int32_ptr_type; + LLVMTypeRef int64_ptr_type; + LLVMTypeRef intptr_t_ptr_type; + LLVMTypeRef float32_ptr_type; + LLVMTypeRef float64_ptr_type; + + LLVMTypeRef v128_type; + LLVMTypeRef v128_ptr_type; + LLVMTypeRef i8x16_vec_type; + LLVMTypeRef i16x8_vec_type; + LLVMTypeRef i32x4_vec_type; + LLVMTypeRef i64x2_vec_type; + LLVMTypeRef f32x4_vec_type; + LLVMTypeRef f64x2_vec_type; + + LLVMTypeRef int8_ptr_type_gs; + LLVMTypeRef int16_ptr_type_gs; + LLVMTypeRef int32_ptr_type_gs; + LLVMTypeRef int64_ptr_type_gs; + LLVMTypeRef float32_ptr_type_gs; + LLVMTypeRef float64_ptr_type_gs; + LLVMTypeRef v128_ptr_type_gs; + + LLVMTypeRef i1x2_vec_type; + + LLVMTypeRef meta_data_type; + + LLVMTypeRef funcref_type; + LLVMTypeRef externref_type; + LLVMTypeRef gc_ref_type; + LLVMTypeRef gc_ref_ptr_type; +} AOTLLVMTypes; + +typedef struct AOTLLVMConsts { + LLVMValueRef i1_zero; + LLVMValueRef i1_one; + LLVMValueRef i8_zero; + LLVMValueRef i8_one; + LLVMValueRef i32_zero; + LLVMValueRef i64_zero; + LLVMValueRef f32_zero; + LLVMValueRef f64_zero; + LLVMValueRef i32_one; + LLVMValueRef i32_two; + LLVMValueRef i32_three; + LLVMValueRef i32_four; + LLVMValueRef i32_five; + LLVMValueRef i32_six; + LLVMValueRef i32_seven; + LLVMValueRef i32_eight; + LLVMValueRef i32_nine; + LLVMValueRef i32_ten; + LLVMValueRef i32_eleven; + LLVMValueRef i32_twelve; + LLVMValueRef i32_thirteen; + LLVMValueRef i32_fourteen; + LLVMValueRef i32_fifteen; + LLVMValueRef i32_neg_one; + LLVMValueRef i64_neg_one; + LLVMValueRef i32_min; + LLVMValueRef i64_min; + LLVMValueRef i32_31; + LLVMValueRef i32_32; + LLVMValueRef i64_63; + LLVMValueRef i64_64; + LLVMValueRef i8x16_vec_zero; + LLVMValueRef i16x8_vec_zero; + LLVMValueRef i32x4_vec_zero; + LLVMValueRef i64x2_vec_zero; + LLVMValueRef f32x4_vec_zero; + LLVMValueRef f64x2_vec_zero; + LLVMValueRef i8x16_undef; + LLVMValueRef i16x8_undef; + LLVMValueRef i32x4_undef; + LLVMValueRef i64x2_undef; + LLVMValueRef f32x4_undef; + LLVMValueRef f64x2_undef; + LLVMValueRef i32x16_zero; + LLVMValueRef i32x8_zero; + LLVMValueRef i32x4_zero; + LLVMValueRef i32x2_zero; + LLVMValueRef gc_ref_null; + LLVMValueRef i8_ptr_null; +} AOTLLVMConsts; + +/** + * Compiler context + */ +typedef struct AOTCompContext { + const AOTCompData *comp_data; + + /* LLVM variables required to emit LLVM IR */ + LLVMContextRef context; + LLVMBuilderRef builder; +#if WASM_ENABLE_DEBUG_AOT + LLVMDIBuilderRef debug_builder; + LLVMMetadataRef debug_file; + LLVMMetadataRef debug_comp_unit; +#endif + LLVMTargetMachineRef target_machine; + char *target_cpu; + char target_arch[16]; + unsigned pointer_size; + + /* Hardware intrinsic compability flags */ + uint64 flags[8]; + + /* required by JIT */ + LLVMOrcLLLazyJITRef orc_jit; + LLVMOrcThreadSafeContextRef orc_thread_safe_context; + + LLVMModuleRef module; + + bool is_jit_mode; + + /* AOT indirect mode flag & symbol list */ + bool is_indirect_mode; + bh_list native_symbols; + + /* Bulk memory feature */ + bool enable_bulk_memory; + + /* Boundary Check */ + bool enable_bound_check; + + /* Native stack boundary Check */ + bool enable_stack_bound_check; + + /* Native stack usage estimation */ + bool enable_stack_estimation; + + /* 128-bit SIMD */ + bool enable_simd; + + /* Auxiliary stack overflow/underflow check */ + bool enable_aux_stack_check; + + /* Generate auxiliary stack frame */ + bool enable_aux_stack_frame; + + /* Function performance profiling */ + bool enable_perf_profiling; + + /* Memory usage profiling */ + bool enable_memory_profiling; + + /* Thread Manager */ + bool enable_thread_mgr; + + /* Tail Call */ + bool enable_tail_call; + + /* Reference Types */ + bool enable_ref_types; + + /* Disable LLVM built-in intrinsics */ + bool disable_llvm_intrinsics; + + /* Disable LLVM link time optimization */ + bool disable_llvm_lto; + + /* Enable LLVM PGO (Profile-Guided Optimization) */ + bool enable_llvm_pgo; + + /* Treat unknown import function as wasm-c-api import function + and allow to directly invoke it from AOT/JIT code */ + bool quick_invoke_c_api_import; + + /* Use profile file collected by LLVM PGO */ + char *use_prof_file; + + /* Enable to use segument register as the base addr + of linear memory for load/store operations */ + bool enable_segue_i32_load; + bool enable_segue_i64_load; + bool enable_segue_f32_load; + bool enable_segue_f64_load; + bool enable_segue_v128_load; + bool enable_segue_i32_store; + bool enable_segue_i64_store; + bool enable_segue_f32_store; + bool enable_segue_f64_store; + bool enable_segue_v128_store; + + /* Whether optimize the JITed code */ + bool optimize; + + bool emit_frame_pointer; + + /* Enable GC */ + bool enable_gc; + + uint32 opt_level; + uint32 size_level; + + /* LLVM floating-point rounding mode metadata */ + LLVMValueRef fp_rounding_mode; + + /* LLVM floating-point exception behavior metadata */ + LLVMValueRef fp_exception_behavior; + + /* a global array to store stack sizes */ + LLVMTypeRef stack_sizes_type; + LLVMValueRef stack_sizes; + uint32 *jit_stack_sizes; /* for JIT */ + + /* LLVM data types */ + AOTLLVMTypes basic_types; + LLVMTypeRef exec_env_type; + LLVMTypeRef aot_inst_type; + + /* LLVM const values */ + AOTLLVMConsts llvm_consts; + + /* Function contexts */ + AOTFuncContext **func_ctxes; + uint32 func_ctx_count; + char **custom_sections_wp; + uint32 custom_sections_count; + + /* 3rd-party toolchains */ + /* External llc compiler, if specified, wamrc will emit the llvm-ir file and + * invoke the llc compiler to generate object file. + * This can be used when we want to benefit from the optimization of other + * LLVM based toolchains */ + const char *external_llc_compiler; + const char *llc_compiler_flags; + /* External asm compiler, if specified, wamrc will emit the text-based + * assembly file (.s) and invoke the llc compiler to generate object file. + * This will be useful when the upstream LLVM doesn't support to emit object + * file for some architecture (such as arc) */ + const char *external_asm_compiler; + const char *asm_compiler_flags; + + const char *stack_usage_file; + char stack_usage_temp_file[64]; + const char *llvm_passes; + const char *builtin_intrinsics; + + /* Current frame information for translation */ + AOTCompFrame *aot_frame; +} AOTCompContext; + +enum { + AOT_FORMAT_FILE, + AOT_OBJECT_FILE, + AOT_LLVMIR_UNOPT_FILE, + AOT_LLVMIR_OPT_FILE, +}; + +bool +aot_compiler_init(void); + +void +aot_compiler_destroy(void); + +AOTCompContext * +aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option); + +void +aot_destroy_comp_context(AOTCompContext *comp_ctx); + +int32 +aot_get_native_symbol_index(AOTCompContext *comp_ctx, const char *symbol); + +bool +aot_compile_wasm(AOTCompContext *comp_ctx); + +uint8 * +aot_emit_elf_file(AOTCompContext *comp_ctx, uint32 *p_elf_file_size); + +void +aot_destroy_elf_file(uint8 *elf_file); + +void +aot_value_stack_push(const AOTCompContext *comp_ctx, AOTValueStack *stack, + AOTValue *value); + +AOTValue * +aot_value_stack_pop(const AOTCompContext *comp_ctx, AOTValueStack *stack); + +void +aot_value_stack_destroy(AOTCompContext *comp_ctx, AOTValueStack *stack); + +void +aot_block_stack_push(AOTBlockStack *stack, AOTBlock *block); + +AOTBlock * +aot_block_stack_pop(AOTBlockStack *stack); + +void +aot_block_stack_destroy(AOTCompContext *comp_ctx, AOTBlockStack *stack); + +void +aot_block_destroy(AOTCompContext *comp_ctx, AOTBlock *block); + +LLVMTypeRef +wasm_type_to_llvm_type(const AOTCompContext *comp_ctx, + const AOTLLVMTypes *llvm_types, uint8 wasm_type); + +bool +aot_checked_addr_list_add(AOTFuncContext *func_ctx, uint32 local_idx, + uint32 offset, uint32 bytes); + +void +aot_checked_addr_list_del(AOTFuncContext *func_ctx, uint32 local_idx); + +bool +aot_checked_addr_list_find(AOTFuncContext *func_ctx, uint32 local_idx, + uint32 offset, uint32 bytes); + +void +aot_checked_addr_list_destroy(AOTFuncContext *func_ctx); + +bool +aot_build_zero_function_ret(const AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, AOTFuncType *func_type); + +LLVMValueRef +aot_call_llvm_intrinsic(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, const char *intrinsic, + LLVMTypeRef ret_type, LLVMTypeRef *param_types, + int param_count, ...); + +LLVMValueRef +aot_call_llvm_intrinsic_v(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, const char *intrinsic, + LLVMTypeRef ret_type, LLVMTypeRef *param_types, + int param_count, va_list param_value_list); + +LLVMValueRef +aot_get_func_from_table(const AOTCompContext *comp_ctx, LLVMValueRef base, + LLVMTypeRef func_type, int32 index); + +LLVMValueRef +aot_load_const_from_table(AOTCompContext *comp_ctx, LLVMValueRef base, + const WASMValue *value, uint8 value_type); + +bool +aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str); + +void +aot_add_expand_memory_op_pass(LLVMPassManagerRef pass); + +void +aot_add_simple_loop_unswitch_pass(LLVMPassManagerRef pass); + +void +aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module); + +void +aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err); + +char * +aot_compress_aot_func_names(AOTCompContext *comp_ctx, uint32 *p_size); + +bool +aot_set_cond_br_weights(AOTCompContext *comp_ctx, LLVMValueRef cond_br, + int32 weights_true, int32 weights_false); + +bool +aot_target_precheck_can_use_musttail(const AOTCompContext *comp_ctx); + +unsigned int +aot_estimate_stack_usage_for_function_call(const AOTCompContext *comp_ctx, + const AOTFuncType *callee_func_type); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _AOT_LLVM_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra.cpp b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra.cpp new file mode 100644 index 0000000..ed9447a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra.cpp @@ -0,0 +1,430 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#if LLVM_VERSION_MAJOR < 17 +#include +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LLVM_VERSION_MAJOR < 17 +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LLVM_VERSION_MAJOR >= 17 +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LLVM_VERSION_MAJOR >= 12 +#include +#endif +#include + +#include +#include "../aot/aot_runtime.h" +#include "aot_llvm.h" + +using namespace llvm; +using namespace llvm::orc; + +#if LLVM_VERSION_MAJOR >= 17 +namespace llvm { +template +using Optional = std::optional; +} +#endif + +LLVM_C_EXTERN_C_BEGIN + +bool +aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str); + +void +aot_add_expand_memory_op_pass(LLVMPassManagerRef pass); + +void +aot_add_simple_loop_unswitch_pass(LLVMPassManagerRef pass); + +void +aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module); + +LLVM_C_EXTERN_C_END + +ExitOnError ExitOnErr; + +class ExpandMemoryOpPass : public PassInfoMixin +{ + public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +PreservedAnalyses +ExpandMemoryOpPass::run(Function &F, FunctionAnalysisManager &AM) +{ + SmallVector MemCalls; + + /* Iterate over all instructions in the function, looking for memcpy, + * memmove, and memset. When we find one, expand it into a loop. */ + + for (auto &BB : F) { + for (auto &Inst : BB) { + if (auto *Memcpy = dyn_cast_or_null(&Inst)) { + MemCalls.push_back(Memcpy); + } + else if (auto *Memmove = dyn_cast_or_null(&Inst)) { + MemCalls.push_back(Memmove); + } + else if (auto *Memset = dyn_cast_or_null(&Inst)) { + MemCalls.push_back(Memset); + } + } + } + + for (MemIntrinsic *MemCall : MemCalls) { + if (MemCpyInst *Memcpy = dyn_cast(MemCall)) { + Function *ParentFunc = Memcpy->getParent()->getParent(); + const TargetTransformInfo &TTI = + AM.getResult(*ParentFunc); + expandMemCpyAsLoop(Memcpy, TTI); + Memcpy->eraseFromParent(); + } + else if (MemMoveInst *Memmove = dyn_cast(MemCall)) { +#if LLVM_VERSION_MAJOR >= 17 + Function *ParentFunc = Memmove->getParent()->getParent(); + const TargetTransformInfo &TTI = + AM.getResult(*ParentFunc); + expandMemMoveAsLoop(Memmove, TTI); +#else + expandMemMoveAsLoop(Memmove); +#endif + Memmove->eraseFromParent(); + } + else if (MemSetInst *Memset = dyn_cast(MemCall)) { + expandMemSetAsLoop(Memset); + Memset->eraseFromParent(); + } + } + + PreservedAnalyses PA; + PA.preserveSet(); + + return PA; +} + +bool +aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str) +{ +#if WASM_ENABLE_SIMD != 0 + if (!arch_c_str || !cpu_c_str) { + return false; + } + + llvm::SmallVector targetAttributes; + llvm::Triple targetTriple(arch_c_str, "", ""); + auto targetMachine = + std::unique_ptr(llvm::EngineBuilder().selectTarget( + targetTriple, "", std::string(cpu_c_str), targetAttributes)); + if (!targetMachine) { + return false; + } + + const llvm::Triple::ArchType targetArch = + targetMachine->getTargetTriple().getArch(); + const llvm::MCSubtargetInfo *subTargetInfo = + targetMachine->getMCSubtargetInfo(); + if (subTargetInfo == nullptr) { + return false; + } + + if (targetArch == llvm::Triple::x86_64) { + return subTargetInfo->checkFeatures("+sse4.1"); + } + else if (targetArch == llvm::Triple::aarch64) { + return subTargetInfo->checkFeatures("+neon"); + } + else { + return false; + } +#else + (void)arch_c_str; + (void)cpu_c_str; + return true; +#endif /* WASM_ENABLE_SIMD */ +} + +void +aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) +{ + TargetMachine *TM = + reinterpret_cast(comp_ctx->target_machine); + PipelineTuningOptions PTO; + PTO.LoopVectorization = true; + PTO.SLPVectorization = true; + PTO.LoopUnrolling = true; + +#if LLVM_VERSION_MAJOR >= 16 + Optional PGO = std::nullopt; +#else + Optional PGO = llvm::None; +#endif + + if (comp_ctx->enable_llvm_pgo) { + /* Disable static counter allocation for value profiler, + it will be allocated by runtime */ + const char *argv[] = { "", "-vp-static-alloc=false" }; + cl::ParseCommandLineOptions(2, argv); +#if LLVM_VERSION_MAJOR < 17 + PGO = PGOOptions("", "", "", PGOOptions::IRInstr); +#else + auto FS = vfs::getRealFileSystem(); + PGO = PGOOptions("", "", "", "", FS, PGOOptions::IRInstr); +#endif + } + else if (comp_ctx->use_prof_file) { +#if LLVM_VERSION_MAJOR < 17 + PGO = PGOOptions(comp_ctx->use_prof_file, "", "", PGOOptions::IRUse); +#else + auto FS = vfs::getRealFileSystem(); + PGO = PGOOptions(comp_ctx->use_prof_file, "", "", "", FS, + PGOOptions::IRUse); +#endif + } + +#ifdef DEBUG_PASS + PassInstrumentationCallbacks PIC; + PassBuilder PB(TM, PTO, PGO, &PIC); +#else +#if LLVM_VERSION_MAJOR == 12 + PassBuilder PB(false, TM, PTO, PGO); +#else + PassBuilder PB(TM, PTO, PGO); +#endif +#endif + + /* Register all the basic analyses with the managers */ + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + /* Register the target library analysis directly and give it a + customized preset TLI */ + std::unique_ptr TLII( + new TargetLibraryInfoImpl(Triple(TM->getTargetTriple()))); + FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); + + /* Register the AA manager first so that our version is the one used */ + AAManager AA = PB.buildDefaultAAPipeline(); + FAM.registerPass([&] { return std::move(AA); }); + +#ifdef DEBUG_PASS + StandardInstrumentations SI(true, false); + SI.registerCallbacks(PIC, &FAM); +#endif + + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + +#if LLVM_VERSION_MAJOR <= 13 + PassBuilder::OptimizationLevel OL; + + switch (comp_ctx->opt_level) { + case 0: + OL = PassBuilder::OptimizationLevel::O0; + break; + case 1: + OL = PassBuilder::OptimizationLevel::O1; + break; + case 2: + OL = PassBuilder::OptimizationLevel::O2; + break; + case 3: + default: + OL = PassBuilder::OptimizationLevel::O3; + break; + } +#else + OptimizationLevel OL; + + switch (comp_ctx->opt_level) { + case 0: + OL = OptimizationLevel::O0; + break; + case 1: + OL = OptimizationLevel::O1; + break; + case 2: + OL = OptimizationLevel::O2; + break; + case 3: + default: + OL = OptimizationLevel::O3; + break; + } +#endif /* end of LLVM_VERSION_MAJOR */ + + bool disable_llvm_lto = comp_ctx->disable_llvm_lto; +#if WASM_ENABLE_SPEC_TEST != 0 + disable_llvm_lto = true; +#endif + + Module *M = reinterpret_cast(module); + if (disable_llvm_lto) { + for (Function &F : *M) { + F.addFnAttr("disable-tail-calls", "true"); + } + } + + ModulePassManager MPM; + + if (comp_ctx->is_jit_mode) { + const char *Passes = + "loop-vectorize,slp-vectorizer," + "load-store-vectorizer,vector-combine," + "mem2reg,instcombine,simplifycfg,jump-threading,indvars"; + ExitOnErr(PB.parsePassPipeline(MPM, Passes)); + } + else { + FunctionPassManager FPM; + + /* Apply Vectorize related passes for AOT mode */ + FPM.addPass(LoopVectorizePass()); + FPM.addPass(SLPVectorizerPass()); + FPM.addPass(LoadStoreVectorizerPass()); + FPM.addPass(VectorCombinePass()); + + if (comp_ctx->enable_llvm_pgo || comp_ctx->use_prof_file) { + /* LICM pass: loop invariant code motion, attempting to remove + as much code from the body of a loop as possible. Experiments + show it is good to enable it when pgo is enabled. */ +#if LLVM_VERSION_MAJOR >= 15 + LICMOptions licm_opt; + FPM.addPass( + createFunctionToLoopPassAdaptor(LICMPass(licm_opt), true)); +#else + FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), true)); +#endif + } + + /* + FPM.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass())); + FPM.addPass(createFunctionToLoopPassAdaptor(SimpleLoopUnswitchPass())); + */ + + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + + if (comp_ctx->llvm_passes) { + ExitOnErr(PB.parsePassPipeline(MPM, comp_ctx->llvm_passes)); + } + + if ( +#if LLVM_VERSION_MAJOR <= 13 + PassBuilder::OptimizationLevel::O0 == OL +#else + OptimizationLevel::O0 == OL +#endif + ) { + MPM.addPass(PB.buildO0DefaultPipeline(OL)); + } + else { + if (!disable_llvm_lto) { + /* Apply LTO for AOT mode */ + if (comp_ctx->comp_data->func_count >= 10 + || comp_ctx->enable_llvm_pgo || comp_ctx->use_prof_file) + /* Add the pre-link optimizations if the func count + is large enough or PGO is enabled */ + MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(OL)); + else + MPM.addPass(PB.buildLTODefaultPipeline(OL, NULL)); + } + else { + MPM.addPass(PB.buildPerModuleDefaultPipeline(OL)); + } + } + + /* Run specific passes for AOT indirect mode in last since general + optimization may create some intrinsic function calls like + llvm.memset, so let's remove these function calls here. */ + if (comp_ctx->is_indirect_mode) { + FunctionPassManager FPM1; + FPM1.addPass(ExpandMemoryOpPass()); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM1))); + } + } + + MPM.run(*M, MAM); +} + +char * +aot_compress_aot_func_names(AOTCompContext *comp_ctx, uint32 *p_size) +{ + std::vector NameStrs; + std::string Result; + char buf[32], *compressed_str; + uint32 compressed_str_len, i; + + for (i = 0; i < comp_ctx->func_ctx_count; i++) { + snprintf(buf, sizeof(buf), "%s%d", AOT_FUNC_PREFIX, i); + std::string str(buf); + NameStrs.push_back(str); + } + +#if LLVM_VERSION_MAJOR < 18 +#define collectGlobalObjectNameStrings collectPGOFuncNameStrings +#endif + if (collectGlobalObjectNameStrings(NameStrs, true, Result)) { + aot_set_last_error("collect pgo func name strings failed"); + return NULL; + } + + compressed_str_len = Result.size(); + if (!(compressed_str = (char *)wasm_runtime_malloc(compressed_str_len))) { + aot_set_last_error("allocate memory failed"); + return NULL; + } + + bh_memcpy_s(compressed_str, compressed_str_len, Result.c_str(), + compressed_str_len); + *p_size = compressed_str_len; + return compressed_str; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra2.cpp b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra2.cpp new file mode 100644 index 0000000..ccbccd1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra2.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (c)2023 YAMAMOTO Takashi. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#if LLVM_VERSION_MAJOR < 17 +#include +#include +#endif +#include +#if LLVM_VERSION_MAJOR >= 14 +#include +#else +#include +#endif +#include + +#include "bh_assert.h" + +#include "aot_llvm_extra2.h" + +#if LLVM_VERSION_MAJOR >= 17 +namespace llvm { +template +using Optional = std::optional; +} +#endif + +static llvm::Optional +convert(LLVMRelocMode reloc_mode) +{ + switch (reloc_mode) { + case LLVMRelocDefault: +#if LLVM_VERSION_MAJOR >= 16 + return std::nullopt; +#else + return llvm::None; +#endif + case LLVMRelocStatic: + return llvm::Reloc::Static; + case LLVMRelocPIC: + return llvm::Reloc::PIC_; + case LLVMRelocDynamicNoPic: + return llvm::Reloc::DynamicNoPIC; + case LLVMRelocROPI: + return llvm::Reloc::ROPI; + case LLVMRelocRWPI: + return llvm::Reloc::RWPI; + case LLVMRelocROPI_RWPI: + return llvm::Reloc::ROPI_RWPI; + } + bh_assert(0); +#if LLVM_VERSION_MAJOR >= 16 + return std::nullopt; +#else + return llvm::None; +#endif +} + +#if LLVM_VERSION_MAJOR < 18 +static llvm::CodeGenOpt::Level +convert(LLVMCodeGenOptLevel opt_level) +{ + switch (opt_level) { + case LLVMCodeGenLevelNone: + return llvm::CodeGenOpt::None; + case LLVMCodeGenLevelLess: + return llvm::CodeGenOpt::Less; + case LLVMCodeGenLevelDefault: + return llvm::CodeGenOpt::Default; + case LLVMCodeGenLevelAggressive: + return llvm::CodeGenOpt::Aggressive; + } + bh_assert(0); + return llvm::CodeGenOpt::None; +} +#else +static llvm::CodeGenOptLevel +convert(LLVMCodeGenOptLevel opt_level) +{ + switch (opt_level) { + case LLVMCodeGenLevelNone: + return llvm::CodeGenOptLevel::None; + case LLVMCodeGenLevelLess: + return llvm::CodeGenOptLevel::Less; + case LLVMCodeGenLevelDefault: + return llvm::CodeGenOptLevel::Default; + case LLVMCodeGenLevelAggressive: + return llvm::CodeGenOptLevel::Aggressive; + } + bh_assert(0); + return llvm::CodeGenOptLevel::None; +} +#endif + +static llvm::Optional +convert(LLVMCodeModel code_model, bool *jit) +{ + *jit = false; + switch (code_model) { + case LLVMCodeModelDefault: +#if LLVM_VERSION_MAJOR >= 16 + return std::nullopt; +#else + return llvm::None; +#endif + case LLVMCodeModelJITDefault: + *jit = true; +#if LLVM_VERSION_MAJOR >= 16 + return std::nullopt; +#else + return llvm::None; +#endif + case LLVMCodeModelTiny: + return llvm::CodeModel::Tiny; + case LLVMCodeModelSmall: + return llvm::CodeModel::Small; + case LLVMCodeModelKernel: + return llvm::CodeModel::Kernel; + case LLVMCodeModelMedium: + return llvm::CodeModel::Medium; + case LLVMCodeModelLarge: + return llvm::CodeModel::Large; + } + bh_assert(0); +#if LLVM_VERSION_MAJOR >= 16 + return std::nullopt; +#else + return llvm::None; +#endif +} + +LLVMTargetMachineRef +LLVMCreateTargetMachineWithOpts(LLVMTargetRef ctarget, const char *triple, + const char *cpu, const char *features, + LLVMCodeGenOptLevel opt_level, + LLVMRelocMode reloc_mode, + LLVMCodeModel code_model, + bool EmitStackSizeSection, + const char *StackUsageOutput) +{ + llvm::TargetOptions opts; + + // -fstack-size-section equiv + // emit it to ".stack_sizes" section in case of ELF + // you can read it with "llvm-readobj --stack-sizes" + opts.EmitStackSizeSection = EmitStackSizeSection; + + // -fstack-usage equiv + if (StackUsageOutput != NULL) { + opts.StackUsageOutput = StackUsageOutput; + } + + auto target = reinterpret_cast(ctarget); + auto rm = convert(reloc_mode); + auto ol = convert(opt_level); + bool jit; + auto cm = convert(code_model, &jit); + auto targetmachine = target->createTargetMachine(triple, cpu, features, + opts, rm, cm, ol, jit); + return reinterpret_cast(targetmachine); +} + +/* https://reviews.llvm.org/D153107 */ +#if LLVM_VERSION_MAJOR < 18 +using namespace llvm; + +LLVMTailCallKind +LLVMGetTailCallKind(LLVMValueRef Call) +{ + return (LLVMTailCallKind)unwrap(Call)->getTailCallKind(); +} + +void +LLVMSetTailCallKind(LLVMValueRef Call, LLVMTailCallKind kind) +{ + unwrap(Call)->setTailCallKind((CallInst::TailCallKind)kind); +} +#endif diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra2.h b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra2.h new file mode 100644 index 0000000..be89faa --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_llvm_extra2.h @@ -0,0 +1,34 @@ +/* + * Copyright (c)2023 YAMAMOTO Takashi. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +LLVM_C_EXTERN_C_BEGIN +LLVMTargetMachineRef +LLVMCreateTargetMachineWithOpts(LLVMTargetRef ctarget, const char *triple, + const char *cpu, const char *features, + LLVMCodeGenOptLevel opt_level, + LLVMRelocMode reloc_mode, + LLVMCodeModel code_model, + bool EmitStackSizeSection, + const char *StackUsageOutput); + +/* https://reviews.llvm.org/D153107 */ +#if LLVM_VERSION_MAJOR < 18 +typedef enum { + LLVMTailCallKindNone = 0, + LLVMTailCallKindTail = 1, + LLVMTailCallKindMustTail = 2, + LLVMTailCallKindNoTail = 3, +} LLVMTailCallKind; + +LLVMTailCallKind +LLVMGetTailCallKind(LLVMValueRef CallInst); +void +LLVMSetTailCallKind(LLVMValueRef CallInst, LLVMTailCallKind kind); +#endif + +LLVM_C_EXTERN_C_END diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra.cpp b/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra.cpp new file mode 100644 index 0000000..90dafe0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra.cpp @@ -0,0 +1,360 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "llvm-c/LLJIT.h" +#include "llvm-c/Orc.h" +#include "llvm-c/OrcEE.h" +#include "llvm-c/TargetMachine.h" + +#if LLVM_VERSION_MAJOR < 17 +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#endif +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/Support/CBindingWrapping.h" + +#include "aot_orc_extra.h" +#include "aot.h" + +#if LLVM_VERSION_MAJOR >= 17 +namespace llvm { +template +using Optional = std::optional; +} +#endif + +using namespace llvm; +using namespace llvm::orc; +using GlobalValueSet = std::set; + +namespace llvm { +namespace orc { + +class InProgressLookupState; + +class OrcV2CAPIHelper +{ + public: +#if LLVM_VERSION_MAJOR < 18 + using PoolEntry = SymbolStringPtr::PoolEntry; + using PoolEntryPtr = SymbolStringPtr::PoolEntryPtr; + + // Move from SymbolStringPtr to PoolEntryPtr (no change in ref count). + static PoolEntryPtr moveFromSymbolStringPtr(SymbolStringPtr S) + { + PoolEntryPtr Result = nullptr; + std::swap(Result, S.S); + return Result; + } + + // Move from a PoolEntryPtr to a SymbolStringPtr (no change in ref count). + static SymbolStringPtr moveToSymbolStringPtr(PoolEntryPtr P) + { + SymbolStringPtr S; + S.S = P; + return S; + } + + // Copy a pool entry to a SymbolStringPtr (increments ref count). + static SymbolStringPtr copyToSymbolStringPtr(PoolEntryPtr P) + { + return SymbolStringPtr(P); + } + + static PoolEntryPtr getRawPoolEntryPtr(const SymbolStringPtr &S) + { + return S.S; + } + + static void retainPoolEntry(PoolEntryPtr P) + { + SymbolStringPtr S(P); + S.S = nullptr; + } + + static void releasePoolEntry(PoolEntryPtr P) + { + SymbolStringPtr S; + S.S = P; + } + +#endif + static InProgressLookupState *extractLookupState(LookupState &LS) + { + return LS.IPLS.release(); + } + + static void resetLookupState(LookupState &LS, InProgressLookupState *IPLS) + { + return LS.reset(IPLS); + } +}; + +} // namespace orc +} // namespace llvm + +// ORC.h +#if LLVM_VERSION_MAJOR >= 18 +inline LLVMOrcSymbolStringPoolEntryRef +wrap(SymbolStringPoolEntryUnsafe E) +{ + return reinterpret_cast(E.rawPtr()); +} + +inline SymbolStringPoolEntryUnsafe +unwrap(LLVMOrcSymbolStringPoolEntryRef E) +{ + return reinterpret_cast(E); +} +#endif + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionSession, LLVMOrcExecutionSessionRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRTransformLayer, LLVMOrcIRTransformLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib, LLVMOrcJITDylibRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITTargetMachineBuilder, + LLVMOrcJITTargetMachineBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectTransformLayer, + LLVMOrcObjectTransformLayerRef) +#if LLVM_VERSION_MAJOR < 18 +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, + LLVMOrcSymbolStringPoolEntryRef) +#endif +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectLayer, LLVMOrcObjectLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SymbolStringPool, LLVMOrcSymbolStringPoolRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeModule, LLVMOrcThreadSafeModuleRef) + +// LLJIT.h +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLLazyJITBuilder, LLVMOrcLLLazyJITBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLLazyJIT, LLVMOrcLLLazyJITRef) + +void +LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef Builder, + unsigned NumCompileThreads) +{ + unwrap(Builder)->setNumCompileThreads(NumCompileThreads); +} + +LLVMOrcLLLazyJITBuilderRef +LLVMOrcCreateLLLazyJITBuilder(void) +{ + return wrap(new LLLazyJITBuilder()); +} + +void +LLVMOrcDisposeLLLazyJITBuilder(LLVMOrcLLLazyJITBuilderRef Builder) +{ + delete unwrap(Builder); +} + +void +LLVMOrcLLLazyJITBuilderSetNumCompileThreads(LLVMOrcLLLazyJITBuilderRef Builder, + unsigned NumCompileThreads) +{ + unwrap(Builder)->setNumCompileThreads(NumCompileThreads); +} + +void +LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder( + LLVMOrcLLLazyJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMP) +{ + unwrap(Builder)->setJITTargetMachineBuilder(*unwrap(JTMP)); + /* Destroy the JTMP, similar to + LLVMOrcLLJITBuilderSetJITTargetMachineBuilder */ + LLVMOrcDisposeJITTargetMachineBuilder(JTMP); +} + +static Optional +PartitionFunction(GlobalValueSet Requested) +{ + std::vector GVsToAdd; + + for (auto *GV : Requested) { + if (isa(GV) && GV->hasName()) { + auto &F = cast(*GV); /* get LLVM function */ + const Module *M = F.getParent(); /* get LLVM module */ + auto GVName = GV->getName(); /* get the function name */ + const char *gvname = GVName.begin(); /* C function name */ + const char *wrapper; + uint32 prefix_len = strlen(AOT_FUNC_PREFIX); + + LOG_DEBUG("requested func %s", gvname); + /* Convert "aot_func#n_wrapper" to "aot_func#n" */ + if (strstr(gvname, AOT_FUNC_PREFIX)) { + char buf[16] = { 0 }; + char func_name[64]; + int group_stride, i, j; + int num; + + /* + * if the jit wrapper (which has "_wrapper" suffix in + * the name) is requested, compile others in the group too. + * otherwise, only compile the requested one. + * (and possibly the correspondig wrapped function, + * which has AOT_FUNC_INTERNAL_PREFIX.) + */ + wrapper = strstr(gvname + prefix_len, "_wrapper"); + if (wrapper != NULL) { + num = WASM_ORC_JIT_COMPILE_THREAD_NUM; + } + else { + num = 1; + wrapper = strchr(gvname + prefix_len, 0); + } + bh_assert(wrapper - (gvname + prefix_len) > 0); + /* Get AOT function index */ + bh_memcpy_s(buf, (uint32)sizeof(buf), gvname + prefix_len, + (uint32)(wrapper - (gvname + prefix_len))); + i = atoi(buf); + + group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM; + + /* Compile some functions each time */ + for (j = 0; j < num; j++) { + Function *F1; + snprintf(func_name, sizeof(func_name), "%s%d", + AOT_FUNC_PREFIX, i + j * group_stride); + F1 = M->getFunction(func_name); + if (F1) { + LOG_DEBUG("compile func %s", func_name); + GVsToAdd.push_back(cast(F1)); + } + snprintf(func_name, sizeof(func_name), "%s%d", + AOT_FUNC_INTERNAL_PREFIX, i + j * group_stride); + F1 = M->getFunction(func_name); + if (F1) { + LOG_DEBUG("compile func %s", func_name); + GVsToAdd.push_back(cast(F1)); + } + } + } + } + } + + for (auto *GV : GVsToAdd) { + Requested.insert(GV); + } + + return Requested; +} + +LLVMErrorRef +LLVMOrcCreateLLLazyJIT(LLVMOrcLLLazyJITRef *Result, + LLVMOrcLLLazyJITBuilderRef Builder) +{ + assert(Result && "Result can not be null"); + + if (!Builder) + Builder = LLVMOrcCreateLLLazyJITBuilder(); + + auto J = unwrap(Builder)->create(); + LLVMOrcDisposeLLLazyJITBuilder(Builder); + + if (!J) { + Result = nullptr; + return 0; + } + + LLLazyJIT *lazy_jit = J->release(); + lazy_jit->setPartitionFunction(PartitionFunction); + + *Result = wrap(lazy_jit); + return LLVMErrorSuccess; +} + +LLVMErrorRef +LLVMOrcDisposeLLLazyJIT(LLVMOrcLLLazyJITRef J) +{ + delete unwrap(J); + return LLVMErrorSuccess; +} + +LLVMErrorRef +LLVMOrcLLLazyJITAddLLVMIRModule(LLVMOrcLLLazyJITRef J, LLVMOrcJITDylibRef JD, + LLVMOrcThreadSafeModuleRef TSM) +{ + std::unique_ptr TmpTSM(unwrap(TSM)); + return wrap(unwrap(J)->addLazyIRModule(*unwrap(JD), std::move(*TmpTSM))); +} + +LLVMErrorRef +LLVMOrcLLLazyJITLookup(LLVMOrcLLLazyJITRef J, LLVMOrcExecutorAddress *Result, + const char *Name) +{ + assert(Result && "Result can not be null"); + + auto Sym = unwrap(J)->lookup(Name); + if (!Sym) { + *Result = 0; + return wrap(Sym.takeError()); + } + +#if LLVM_VERSION_MAJOR < 15 + *Result = Sym->getAddress(); +#else + *Result = Sym->getValue(); +#endif + return LLVMErrorSuccess; +} + +LLVMOrcSymbolStringPoolEntryRef +LLVMOrcLLLazyJITMangleAndIntern(LLVMOrcLLLazyJITRef J, + const char *UnmangledName) +{ +#if LLVM_VERSION_MAJOR < 18 + return wrap(OrcV2CAPIHelper::moveFromSymbolStringPtr( + unwrap(J)->mangleAndIntern(UnmangledName))); +#else + return wrap(SymbolStringPoolEntryUnsafe::take( + unwrap(J)->mangleAndIntern(UnmangledName))); +#endif +} + +LLVMOrcJITDylibRef +LLVMOrcLLLazyJITGetMainJITDylib(LLVMOrcLLLazyJITRef J) +{ + return wrap(&unwrap(J)->getMainJITDylib()); +} + +const char * +LLVMOrcLLLazyJITGetTripleString(LLVMOrcLLLazyJITRef J) +{ + return unwrap(J)->getTargetTriple().str().c_str(); +} + +LLVMOrcExecutionSessionRef +LLVMOrcLLLazyJITGetExecutionSession(LLVMOrcLLLazyJITRef J) +{ + return wrap(&unwrap(J)->getExecutionSession()); +} + +LLVMOrcIRTransformLayerRef +LLVMOrcLLLazyJITGetIRTransformLayer(LLVMOrcLLLazyJITRef J) +{ + return wrap(&unwrap(J)->getIRTransformLayer()); +} + +LLVMOrcObjectTransformLayerRef +LLVMOrcLLLazyJITGetObjTransformLayer(LLVMOrcLLLazyJITRef J) +{ + return wrap(&unwrap(J)->getObjTransformLayer()); +} + +LLVMOrcObjectLayerRef +LLVMOrcLLLazyJITGetObjLinkingLayer(LLVMOrcLLLazyJITRef J) +{ + return wrap(&unwrap(J)->getObjLinkingLayer()); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra.h b/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra.h new file mode 100644 index 0000000..32ece4d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_ORC_LAZINESS_H_ +#define _AOT_ORC_LAZINESS_H_ + +#include "llvm-c/Error.h" +#include "llvm-c/ExternC.h" +#include "llvm-c/LLJIT.h" +#include "llvm-c/Orc.h" +#include "llvm-c/Types.h" + +LLVM_C_EXTERN_C_BEGIN + +typedef struct LLVMOrcOpaqueLLLazyJITBuilder *LLVMOrcLLLazyJITBuilderRef; +typedef struct LLVMOrcOpaqueLLLazyJIT *LLVMOrcLLLazyJITRef; + +// Extra bindings for LLJIT +void +LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef Builder, + unsigned NumCompileThreads); + +// Extra bindings for LLLazyJIT +LLVMOrcLLLazyJITBuilderRef +LLVMOrcCreateLLLazyJITBuilder(void); + +void +LLVMOrcDisposeLLLazyJITBuilder(LLVMOrcLLLazyJITBuilderRef Builder); + +void +LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder( + LLVMOrcLLLazyJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMP); + +void +LLVMOrcLLLazyJITBuilderSetNumCompileThreads(LLVMOrcLLLazyJITBuilderRef Builder, + unsigned NumCompileThreads); + +LLVMErrorRef +LLVMOrcCreateLLLazyJIT(LLVMOrcLLLazyJITRef *Result, + LLVMOrcLLLazyJITBuilderRef Builder); + +LLVMErrorRef +LLVMOrcDisposeLLLazyJIT(LLVMOrcLLLazyJITRef J); + +LLVMErrorRef +LLVMOrcLLLazyJITAddLLVMIRModule(LLVMOrcLLLazyJITRef J, LLVMOrcJITDylibRef JD, + LLVMOrcThreadSafeModuleRef TSM); + +LLVMErrorRef +LLVMOrcLLLazyJITLookup(LLVMOrcLLLazyJITRef J, LLVMOrcExecutorAddress *Result, + const char *Name); + +LLVMOrcSymbolStringPoolEntryRef +LLVMOrcLLLazyJITMangleAndIntern(LLVMOrcLLLazyJITRef J, + const char *UnmangledName); + +LLVMOrcJITDylibRef +LLVMOrcLLLazyJITGetMainJITDylib(LLVMOrcLLLazyJITRef J); + +const char * +LLVMOrcLLLazyJITGetTripleString(LLVMOrcLLLazyJITRef J); + +LLVMOrcExecutionSessionRef +LLVMOrcLLLazyJITGetExecutionSession(LLVMOrcLLLazyJITRef J); + +LLVMOrcIRTransformLayerRef +LLVMOrcLLLazyJITGetIRTransformLayer(LLVMOrcLLLazyJITRef J); + +LLVMOrcObjectTransformLayerRef +LLVMOrcLLLazyJITGetObjTransformLayer(LLVMOrcLLLazyJITRef J); + +void +LLVMOrcLLJITBuilderSetCompileFuncitonCreatorWithStackSizesCallback( + LLVMOrcLLLazyJITBuilderRef Builder, + void (*cb)(void *, const char *, size_t, size_t), void *cb_data); + +LLVMOrcObjectLayerRef +LLVMOrcLLLazyJITGetObjLinkingLayer(LLVMOrcLLLazyJITRef J); + +LLVM_C_EXTERN_C_END +#endif diff --git a/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra2.cpp b/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra2.cpp new file mode 100644 index 0000000..515f720 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/aot_orc_extra2.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunctionPass.h" + +#include "aot_orc_extra.h" +#include "bh_log.h" + +typedef void (*cb_t)(void *, const char *, size_t, size_t); + +class MyCompiler : public llvm::orc::IRCompileLayer::IRCompiler +{ + public: + MyCompiler(llvm::orc::JITTargetMachineBuilder JTMB, cb_t cb, void *cb_data); + llvm::Expected operator()( + llvm::Module &M) override; + + private: + llvm::orc::JITTargetMachineBuilder JTMB; + + cb_t cb; + void *cb_data; +}; + +MyCompiler::MyCompiler(llvm::orc::JITTargetMachineBuilder JTMB, cb_t cb, + void *cb_data) + : IRCompiler(llvm::orc::irManglingOptionsFromTargetOptions(JTMB.getOptions())) + , JTMB(std::move(JTMB)) + , cb(cb) + , cb_data(cb_data) +{} + +class PrintStackSizes : public llvm::MachineFunctionPass +{ + public: + PrintStackSizes(cb_t cb, void *cb_data); + bool runOnMachineFunction(llvm::MachineFunction &MF) override; + static char ID; + + private: + cb_t cb; + void *cb_data; +}; + +PrintStackSizes::PrintStackSizes(cb_t cb, void *cb_data) + : MachineFunctionPass(ID) + , cb(cb) + , cb_data(cb_data) +{} + +char PrintStackSizes::ID = 0; + +bool +PrintStackSizes::runOnMachineFunction(llvm::MachineFunction &MF) +{ + auto name = MF.getName(); + auto MFI = &MF.getFrameInfo(); + size_t sz = MFI->getStackSize(); + cb(cb_data, name.data(), name.size(), sz); + return false; +} + +class MyPassManager : public llvm::legacy::PassManager +{ + public: + void add(llvm::Pass *P) override; +}; + +void +MyPassManager::add(llvm::Pass *P) +{ + // a hack to avoid having a copy of the whole addPassesToEmitMC. + // we want to add PrintStackSizes before FreeMachineFunctionPass. + if (P->getPassName() == "Free MachineFunction") { + return; + } + llvm::legacy::PassManager::add(P); +} + +// a modified copy from llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp +llvm::Expected +MyCompiler::operator()(llvm::Module &M) +{ + auto TM = cantFail(JTMB.createTargetMachine()); + llvm::SmallVector ObjBufferSV; + + { + llvm::raw_svector_ostream ObjStream(ObjBufferSV); + + MyPassManager PM; + llvm::MCContext *Ctx; + if (TM->addPassesToEmitMC(PM, Ctx, ObjStream)) + return llvm::make_error( + "Target does not support MC emission", + llvm::inconvertibleErrorCode()); + PM.add(new PrintStackSizes(cb, cb_data)); + dynamic_cast(&PM)->add( + llvm::createFreeMachineFunctionPass()); + PM.run(M); + } + +#if LLVM_VERSION_MAJOR > 13 + auto ObjBuffer = std::make_unique( + std::move(ObjBufferSV), + M.getModuleIdentifier() + "-jitted-objectbuffer", + /*RequiresNullTerminator=*/false); +#else + auto ObjBuffer = std::make_unique( + std::move(ObjBufferSV), + M.getModuleIdentifier() + "-jitted-objectbuffer"); +#endif + + return std::move(ObjBuffer); +} + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::LLLazyJITBuilder, + LLVMOrcLLLazyJITBuilderRef) + +void +LLVMOrcLLJITBuilderSetCompileFuncitonCreatorWithStackSizesCallback( + LLVMOrcLLLazyJITBuilderRef Builder, + void (*cb)(void *, const char *, size_t, size_t), void *cb_data) +{ + auto b = unwrap(Builder); + b->setCompileFunctionCreator( + [cb, cb_data](llvm::orc::JITTargetMachineBuilder JTMB) + -> llvm::Expected< + std::unique_ptr> { + return std::make_unique( + MyCompiler(std::move(JTMB), cb, cb_data)); + }); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/debug/dwarf_extractor.cpp b/wasm-micro-runtime/core/iwasm/compilation/debug/dwarf_extractor.cpp new file mode 100644 index 0000000..e2e515b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/debug/dwarf_extractor.cpp @@ -0,0 +1,565 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "lldb/API/SBBlock.h" +#include "lldb/API/SBCompileUnit.h" +#include "lldb/API/SBCommandReturnObject.h" +#include "lldb/API/SBCommandInterpreter.h" +#include "lldb/API/SBBreakpointLocation.h" +#include "lldb/API/SBDebugger.h" +#include "lldb/API//SBFunction.h" +#include "lldb/API//SBModule.h" +#include "lldb/API//SBProcess.h" +#include "lldb/API//SBStream.h" +#include "lldb/API//SBSymbol.h" +#include "lldb/API//SBTarget.h" +#include "lldb/API//SBThread.h" +#include "lldb/API/SBDeclaration.h" + +#include "dwarf_extractor.h" +#include "../aot_llvm.h" + +#include "bh_log.h" +#include "../../aot/aot_runtime.h" + +#include "llvm/BinaryFormat/Dwarf.h" + +using namespace lldb; + +typedef struct dwarf_extractor { + SBDebugger debugger; + SBTarget target; + SBModule module; + +} dwarf_extractor; + +#define TO_HANDLE(extractor) (dwarf_extractor_handle_t)(extractor) + +#define TO_EXTACTOR(handle) (dwarf_extractor *)(handle) + +static bool is_debugger_initialized; + +dwarf_extractor_handle_t +create_dwarf_extractor(AOTCompData *comp_data, char *file_name) +{ + char *arch = NULL; + char *platform = NULL; + dwarf_extractor *extractor = NULL; + + //__attribute__((constructor)) may be better? + if (!is_debugger_initialized) { + SBError error = SBDebugger::InitializeWithErrorHandling(); + if (error.Fail()) { + LOG_ERROR("Init Dwarf Debugger failed"); + return TO_HANDLE(NULL); + } + is_debugger_initialized = true; + } + + SBError error; + SBFileSpec exe_file_spec(file_name, true); + + if (!(extractor = new dwarf_extractor())) { + LOG_ERROR("Create Dwarf Extractor error: failed to allocate memory"); + goto fail3; + } + + extractor->debugger = SBDebugger::Create(); + if (!extractor->debugger.IsValid()) { + LOG_ERROR("Create Dwarf Debugger failed"); + goto fail2; + } + + extractor->target = extractor->debugger.CreateTarget( + file_name, arch, platform, false, error); + + if (!error.Success()) { + LOG_ERROR("Create Dwarf target failed:%s", error.GetCString()); + goto fail1; + } + + if (!extractor->target.IsValid()) { + LOG_ERROR("Create Dwarf target not valid"); + goto fail1; + } + + extractor->module = extractor->target.FindModule(exe_file_spec); + comp_data->extractor = TO_HANDLE(extractor); + + return TO_HANDLE(extractor); + +fail1: + SBDebugger::Destroy(extractor->debugger); + +fail2: + wasm_runtime_free(extractor); + +fail3: + return TO_HANDLE(NULL); +} + +void +destroy_dwarf_extractor(dwarf_extractor_handle_t handle) +{ + dwarf_extractor *extractor = TO_EXTACTOR(handle); + if (!extractor) + return; + extractor->debugger.DeleteTarget(extractor->target); + SBDebugger::Destroy(extractor->debugger); + delete extractor; + SBDebugger::Terminate(); + is_debugger_initialized = false; +} + +LLVMMetadataRef +dwarf_gen_file_info(const AOTCompContext *comp_ctx) +{ + dwarf_extractor *extractor; + int units_number; + LLVMMetadataRef file_info = NULL; + const char *file_name; + const char *dir_name; + + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) + return NULL; + + units_number = extractor->module.GetNumCompileUnits(); + + if (units_number > 0) { + SBCompileUnit compile_unit = extractor->module.GetCompileUnitAtIndex(0); + auto filespec = compile_unit.GetFileSpec(); + file_name = filespec.GetFilename(); + dir_name = filespec.GetDirectory(); + if (file_name || dir_name) { + file_info = LLVMDIBuilderCreateFile( + comp_ctx->debug_builder, file_name, + file_name ? strlen(file_name) : 0, dir_name, + dir_name ? strlen(dir_name) : 0); + } + } + return file_info; +} + +#if 0 +void +dwarf_gen_mock_vm_info(AOTCompContext *comp_ctx) +{ + LLVMMetadataRef file_info = NULL; + LLVMMetadataRef comp_unit = NULL; + file_info = LLVMDIBuilderCreateFile(comp_ctx->debug_builder, + "ant_runtime_mock.c", 18, ".", 1); + + comp_unit = LLVMDIBuilderCreateCompileUnit( + comp_ctx->debug_builder, LLVMDWARFSourceLanguageC, file_info, + "WAMR AoT compiler", 12, 0, NULL, 0, 1, NULL, 0, LLVMDWARFEmissionFull, 0, 0, + 0, "/", 1, "", 0); + + LLVMTypeRef ParamTys[] = { + LLVMVoidType(), + }; + + LLVMTypeRef FuncTy = LLVMFunctionType(LLVMVoidType(), ParamTys, 0, 0); + + LLVMValueRef Function = + LLVMAddFunction(comp_ctx->module, "ant_runtime_mock", FuncTy); + + LLVMMetadataRef ParamTypes[0]; + LLVMMetadataRef FunctionTy = LLVMDIBuilderCreateSubroutineType( + comp_ctx->debug_builder, file_info, ParamTypes, 0, LLVMDIFlagZero); + + /* 0x0015 is subroutine_type */ + LLVMMetadataRef ReplaceableFunctionMetadata = + LLVMDIBuilderCreateReplaceableCompositeType( + comp_ctx->debug_builder, 0x15, "ant_runtime_mock", 16, file_info, + file_info, 2, 0, 0, 0, LLVMDIFlagFwdDecl, "", 0); + + LLVMMetadataRef FunctionMetadata = LLVMDIBuilderCreateFunction( + comp_ctx->debug_builder, file_info, "ant_runtime_mock", 16, + "ant_runtime_mock", 16, file_info, 2, FunctionTy, true, true, 2, LLVMDIFlagZero, + false); + + LLVMMetadataReplaceAllUsesWith(ReplaceableFunctionMetadata, + FunctionMetadata); + + LLVMSetSubprogram(Function, FunctionMetadata); + + comp_ctx->vm_debug_comp_unit = comp_unit; + comp_ctx->vm_debug_file = file_info; + comp_ctx->vm_debug_func = FunctionMetadata; +} +#endif + +LLVMMetadataRef +dwarf_gen_comp_unit_info(const AOTCompContext *comp_ctx) +{ + dwarf_extractor *extractor; + int units_number; + LLVMMetadataRef comp_unit = NULL; + + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) + return NULL; + + units_number = extractor->module.GetNumCompileUnits(); + + if (units_number > 0) { + SBCompileUnit compile_unit = extractor->module.GetCompileUnitAtIndex(0); + auto lang_type = compile_unit.GetLanguage(); + + comp_unit = LLVMDIBuilderCreateCompileUnit( + comp_ctx->debug_builder, LLDB_TO_LLVM_LANG_TYPE(lang_type), + comp_ctx->debug_file, "WAMR AoT compiler", 12, 0, NULL, 0, 1, NULL, + 0, LLVMDWARFEmissionFull, 0, 0, 0, "/", 1, "", 0); + } + return comp_unit; +} + +static LLVMDWARFTypeEncoding +lldb_get_basic_type_encoding(BasicType basic_type) +{ + LLVMDWARFTypeEncoding encoding = 0; + switch (basic_type) { + case eBasicTypeUnsignedChar: + encoding = llvm::dwarf::DW_ATE_unsigned_char; + break; + case eBasicTypeSignedChar: + encoding = llvm::dwarf::DW_ATE_signed_char; + break; + case eBasicTypeUnsignedInt: + case eBasicTypeUnsignedLong: + case eBasicTypeUnsignedLongLong: + case eBasicTypeUnsignedWChar: + case eBasicTypeUnsignedInt128: + case eBasicTypeUnsignedShort: + encoding = llvm::dwarf::DW_ATE_unsigned; + break; + case eBasicTypeInt: + case eBasicTypeLong: + case eBasicTypeLongLong: + case eBasicTypeWChar: + case eBasicTypeInt128: + case eBasicTypeShort: + encoding = llvm::dwarf::DW_ATE_signed; + break; + case eBasicTypeBool: + encoding = llvm::dwarf::DW_ATE_boolean; + break; + case eBasicTypeHalf: + case eBasicTypeFloat: + case eBasicTypeDouble: + case eBasicTypeLongDouble: + encoding = llvm::dwarf::DW_ATE_float; + break; + default: + break; + } + return encoding; +} + +static LLVMMetadataRef +lldb_type_to_type_dbi(const AOTCompContext *comp_ctx, SBType &type) +{ + LLVMMetadataRef type_info = NULL; + BasicType basic_type = type.GetBasicType(); + uint64_t bit_size = type.GetByteSize() * 8; + LLVMDIBuilderRef DIB = comp_ctx->debug_builder; + LLVMDWARFTypeEncoding encoding; + + if (basic_type != eBasicTypeInvalid) { + encoding = lldb_get_basic_type_encoding(basic_type); + type_info = LLVMDIBuilderCreateBasicType( + DIB, type.GetName(), strlen(type.GetName()), bit_size, encoding, + LLVMDIFlagZero); + } + else if (type.IsPointerType()) { + SBType pointee_type = type.GetPointeeType(); + type_info = LLVMDIBuilderCreatePointerType( + DIB, lldb_type_to_type_dbi(comp_ctx, pointee_type), bit_size, 0, 0, + "", 0); + } + + return type_info; +} + +static LLVMMetadataRef +lldb_function_to_function_dbi(const AOTCompContext *comp_ctx, + SBSymbolContext &sc, + const AOTFuncContext *func_ctx) +{ + SBFunction function(sc.GetFunction()); + const char *function_name = function.GetName(); + const char *link_name = function.GetName(); + SBTypeList function_args = function.GetType().GetFunctionArgumentTypes(); + SBType return_type = function.GetType().GetFunctionReturnType(); + const size_t num_function_args = function_args.GetSize(); + dwarf_extractor *extractor; + + /* + * Process only known languages. + * We have a few assumptions which might not be true for non-C functions. + * + * At least it's known broken for C++ and Rust: + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/3187 + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/3163 + */ + LanguageType language_type = function.GetLanguage(); + switch (language_type) { + case eLanguageTypeC89: + case eLanguageTypeC: + case eLanguageTypeC99: + case eLanguageTypeC11: + case eLanguageTypeC17: + break; + default: + LOG_WARNING("func %s has unsuppoted language_type 0x%x", + function_name, (int)language_type); + return NULL; + } + + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) + return NULL; + + LLVMDIBuilderRef DIB = comp_ctx->debug_builder; + LLVMMetadataRef File = comp_ctx->debug_file; /* a fallback */ + + LLVMMetadataRef ParamTypes[num_function_args + 1]; + + ParamTypes[0] = lldb_type_to_type_dbi(comp_ctx, return_type); + + for (uint32_t function_arg_idx = 0; function_arg_idx < num_function_args; + ++function_arg_idx) { + SBType function_arg_type = + function_args.GetTypeAtIndex(function_arg_idx); + + if (function_arg_type.IsValid()) { + ParamTypes[function_arg_idx + 1] = + lldb_type_to_type_dbi(comp_ctx, function_arg_type); + if (ParamTypes[function_arg_idx + 1] == NULL) { + LOG_WARNING( + "func %s arg %" PRIu32 + " has a type not implemented by lldb_type_to_type_dbi", + function_name, function_arg_idx); + } + } + else { + LOG_WARNING("func %s arg %" PRIu32 ": GetTypeAtIndex failed", + function_name, function_arg_idx); + ParamTypes[function_arg_idx + 1] = NULL; + } + } + + auto compile_unit = sc.GetCompileUnit(); + auto file_spec = compile_unit.GetFileSpec(); + const char *file_name = file_spec.GetFilename(); + const char *dir_name = file_spec.GetDirectory(); + LLVMMetadataRef file_info = NULL; + if (file_name || dir_name) { + file_info = + LLVMDIBuilderCreateFile(comp_ctx->debug_builder, file_name, + file_name ? strlen(file_name) : 0, dir_name, + dir_name ? strlen(dir_name) : 0); + } + if (file_info) { + File = file_info; + } + + LLVMMetadataRef FunctionTy = LLVMDIBuilderCreateSubroutineType( + DIB, File, ParamTypes, num_function_args + 1, LLVMDIFlagZero); + + auto line_entry = sc.GetLineEntry(); + LLVMMetadataRef ReplaceableFunctionMetadata = + LLVMDIBuilderCreateReplaceableCompositeType( + DIB, 0x15, function_name, strlen(function_name), File, File, + line_entry.GetLine(), 0, 0, 0, LLVMDIFlagFwdDecl, "", 0); + + LLVMMetadataRef FunctionMetadata = LLVMDIBuilderCreateFunction( + DIB, File, function_name, strlen(function_name), link_name, + strlen(link_name), File, line_entry.GetLine(), FunctionTy, true, true, + line_entry.GetLine(), LLVMDIFlagZero, false); + + LLVMMetadataReplaceAllUsesWith(ReplaceableFunctionMetadata, + FunctionMetadata); + + LLVMSetSubprogram(func_ctx->func, FunctionMetadata); + + LLVMMetadataRef ParamExpression = + LLVMDIBuilderCreateExpression(DIB, NULL, 0); + auto variable_list = + function.GetBlock().GetVariables(extractor->target, true, false, false); + if (num_function_args != variable_list.GetSize()) { + LOG_ERROR( + "function args number dismatch!:value number=%d, function args=%d", + variable_list.GetSize(), num_function_args); + } + + LLVMMetadataRef ParamLocation = LLVMDIBuilderCreateDebugLocation( + comp_ctx->context, line_entry.GetLine(), 0, FunctionMetadata, NULL); + + // TODO:change to void * or WasmExenv * ？ + LLVMMetadataRef voidtype = + LLVMDIBuilderCreateBasicType(DIB, "void", 4, 0, 0, LLVMDIFlagZero); + LLVMMetadataRef voidpionter = + LLVMDIBuilderCreatePointerType(DIB, voidtype, 64, 0, 0, "void *", 6); + + LLVMMetadataRef ParamVar = LLVMDIBuilderCreateParameterVariable( + DIB, FunctionMetadata, "exenv", 5, 1, + File, // starts form 1, and 1 is exenv, + line_entry.GetLine(), voidpionter, true, LLVMDIFlagZero); + LLVMValueRef Param = LLVMGetParam(func_ctx->func, 0); + LLVMBasicBlockRef block_curr = LLVMGetEntryBasicBlock(func_ctx->func); + LLVMDIBuilderInsertDbgValueAtEnd(DIB, Param, ParamVar, ParamExpression, + ParamLocation, block_curr); + + for (uint32_t function_arg_idx = 0; + function_arg_idx < variable_list.GetSize(); ++function_arg_idx) { + SBValue variable(variable_list.GetValueAtIndex(function_arg_idx)); + if (variable.IsValid() && ParamTypes[function_arg_idx + 1] != NULL) { + SBDeclaration dec(variable.GetDeclaration()); + auto valtype = variable.GetType(); + LLVMMetadataRef ParamLocation = LLVMDIBuilderCreateDebugLocation( + comp_ctx->context, dec.GetLine(), dec.GetColumn(), + FunctionMetadata, NULL); + const char *varname = variable.GetName(); + LLVMMetadataRef ParamVar = LLVMDIBuilderCreateParameterVariable( + DIB, FunctionMetadata, varname, varname ? strlen(varname) : 0, + function_arg_idx + 1 + 1, + File, // starts form 1, and 1 is exenv, + dec.GetLine(), ParamTypes[function_arg_idx + 1], true, + LLVMDIFlagZero); + LLVMValueRef Param = + LLVMGetParam(func_ctx->func, function_arg_idx + 1); + LLVMDIBuilderInsertDbgValueAtEnd(DIB, Param, ParamVar, + ParamExpression, ParamLocation, + block_curr); + } + } + + return FunctionMetadata; +} + +LLVMMetadataRef +dwarf_gen_func_info(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx) +{ + LLVMMetadataRef func_info = NULL; + dwarf_extractor *extractor; + uint64_t vm_offset; + AOTFunc *func = func_ctx->aot_func; + + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) + return NULL; + + // A code address in DWARF for WebAssembly is the offset of an + // instruction relative within the Code section of the WebAssembly file. + // For this reason Section::GetFileAddress() must return zero for the + // Code section. (refert to ObjectFileWasm.cpp) + vm_offset = func->code - comp_ctx->comp_data->wasm_module->buf_code; + + auto sbaddr = extractor->target.ResolveFileAddress(vm_offset); + SBSymbolContext sc(sbaddr.GetSymbolContext(eSymbolContextFunction + | eSymbolContextLineEntry)); + if (sc.IsValid()) { + SBFunction function(sc.GetFunction()); + if (function.IsValid()) { + func_info = lldb_function_to_function_dbi(comp_ctx, sc, func_ctx); + } + } + return func_info; +} + +void +dwarf_get_func_name(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, char *name, int len) +{ + LLVMMetadataRef func_info = NULL; + dwarf_extractor *extractor; + uint64_t vm_offset; + AOTFunc *func = func_ctx->aot_func; + + name[0] = '\0'; + + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) + return; + + // A code address in DWARF for WebAssembly is the offset of an + // instruction relative within the Code section of the WebAssembly file. + // For this reason Section::GetFileAddress() must return zero for the + // Code section. (refert to ObjectFileWasm.cpp) + vm_offset = func->code - comp_ctx->comp_data->wasm_module->buf_code; + + auto sbaddr = extractor->target.ResolveFileAddress(vm_offset); + SBSymbolContext sc(sbaddr.GetSymbolContext(eSymbolContextFunction + | eSymbolContextLineEntry)); + if (sc.IsValid()) { + SBFunction function(sc.GetFunction()); + if (function.IsValid()) { + bh_strcpy_s(name, len, function.GetName()); + } + } +} + +LLVMMetadataRef +dwarf_gen_location(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, uint64_t vm_offset) +{ + LLVMMetadataRef location_info = NULL; + dwarf_extractor *extractor; + AOTFunc *func = func_ctx->aot_func; + + if (func_ctx->debug_func == NULL) + return NULL; + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) + return NULL; + + auto sbaddr = extractor->target.ResolveFileAddress(vm_offset); + SBSymbolContext sc(sbaddr.GetSymbolContext(eSymbolContextFunction + | eSymbolContextLineEntry)); + if (sc.IsValid()) { + // TODO:need to check if the vm_offset is belong to + SBFunction function(sc.GetFunction()); + if (function.IsValid()) { + uint64_t start = func_ctx->aot_func->code + - comp_ctx->comp_data->wasm_module->buf_code; + uint64_t end = func_ctx->aot_func->code + - comp_ctx->comp_data->wasm_module->buf_code + + func_ctx->aot_func->code_size; + if (function.GetStartAddress().GetOffset() <= start + && end <= function.GetEndAddress().GetOffset()) { + auto line_entry = sc.GetLineEntry(); + location_info = LLVMDIBuilderCreateDebugLocation( + comp_ctx->context, line_entry.GetLine(), + line_entry.GetColumn(), func_ctx->debug_func, NULL); + // LOG_VERBOSE("Gen the location l:%d, c:%d at %lx", + // line_entry.GetLine(), line_entry.GetColumn(), vm_offset); + } + else + LOG_WARNING("the offset and function is not matched"); + } + } + return location_info; +} + +LLVMMetadataRef +dwarf_gen_func_ret_location(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx) +{ + LLVMMetadataRef func_info = NULL; + dwarf_extractor *extractor; + uint64_t vm_offset; + AOTFunc *func = func_ctx->aot_func; + LLVMMetadataRef location_info = NULL; + + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) + return NULL; + + // A code address in DWARF for WebAssembly is the offset of an + // instruction relative within the Code section of the WebAssembly file. + // For this reason Section::GetFileAddress() must return zero for the + // Code section. (refert to ObjectFileWasm.cpp) + vm_offset = (func->code + func->code_size - 1) + - comp_ctx->comp_data->wasm_module->buf_code; + location_info = dwarf_gen_location(comp_ctx, func_ctx, vm_offset); + + return location_info; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/debug/dwarf_extractor.h b/wasm-micro-runtime/core/iwasm/compilation/debug/dwarf_extractor.h new file mode 100644 index 0000000..0bacb97 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/debug/dwarf_extractor.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _DWARF_EXTRACTOR_H_ +#define _DWARF_EXTRACTOR_H_ + +#include "llvm-c/DebugInfo.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned int LLDBLangType; +#define LLDB_TO_LLVM_LANG_TYPE(lldb_lang_type) \ + (LLVMDWARFSourceLanguage)(((lldb_lang_type) > 0 ? (lldb_lang_type)-1 : 1)) + +struct AOTCompData; +typedef struct AOTCompData *aot_comp_data_t; +typedef void *dwarf_extractor_handle_t; + +struct AOTCompContext; +typedef struct AOTCompContext AOTCompContext; + +struct AOTFuncContext; + +typedef struct AOTFuncContext AOTFuncContext; +dwarf_extractor_handle_t +create_dwarf_extractor(aot_comp_data_t comp_data, char *file_name); + +LLVMMetadataRef +dwarf_gen_file_info(const AOTCompContext *comp_ctx); + +LLVMMetadataRef +dwarf_gen_comp_unit_info(const AOTCompContext *comp_ctx); + +LLVMMetadataRef +dwarf_gen_func_info(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx); + +LLVMMetadataRef +dwarf_gen_location(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, uint64_t vm_offset); + +LLVMMetadataRef +dwarf_gen_func_ret_location(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx); + +void +dwarf_get_func_name(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, char *name, int len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/compilation/iwasm_compl.cmake b/wasm-micro-runtime/core/iwasm/compilation/iwasm_compl.cmake new file mode 100644 index 0000000..4ec4603 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/iwasm_compl.cmake @@ -0,0 +1,26 @@ +set (IWASM_COMPL_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include_directories(${IWASM_COMPL_DIR}) + +if (WAMR_BUILD_DEBUG_AOT EQUAL 1) + file (GLOB_RECURSE source_all + ${IWASM_COMPL_DIR}/*.c + ${IWASM_COMPL_DIR}/*.cpp) +else() + file (GLOB source_all + ${IWASM_COMPL_DIR}/simd/*.c + ${IWASM_COMPL_DIR}/simd/*.cpp + ${IWASM_COMPL_DIR}/*.c + ${IWASM_COMPL_DIR}/*.cpp) +endif() + +set (IWASM_COMPL_SOURCE ${source_all}) + +# Disalbe rtti to works with LLVM + +if (MSVC) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-") +else() + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") +endif() + diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_access_lanes.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_access_lanes.c new file mode 100644 index 0000000..4f43c35 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_access_lanes.c @@ -0,0 +1,418 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_access_lanes.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +bool +aot_compile_simd_shuffle(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + const uint8 *frame_ip) +{ + LLVMValueRef vec1, vec2, mask, result; + uint8 imm[16] = { 0 }; + int values[16]; + unsigned i; + + wasm_runtime_read_v128(frame_ip, (uint64 *)imm, (uint64 *)(imm + 8)); + for (i = 0; i < 16; i++) { + values[i] = imm[i]; + } + + if (!(vec2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE, + "vec2"))) { + goto fail; + } + + if (!(vec1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE, + "vec1"))) { + goto fail; + } + + /* build a vector <16 x i32> */ + if (!(mask = simd_build_const_integer_vector(comp_ctx, I32_TYPE, values, + 16))) { + goto fail; + } + + if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, vec1, vec2, mask, + "new_vector"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + goto fail; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); + +fail: + return false; +} + +/*TODO: llvm.experimental.vector.*/ +/* shufflevector is not an option, since it requires *mask as a const */ +bool +aot_compile_simd_swizzle_x86(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef vector, mask, max_lanes, condition, mask_lanes, result; + LLVMTypeRef param_types[2]; + + if (!(mask = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE, + "mask"))) { + goto fail; + } + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_i8x16_TYPE, "vec"))) { + goto fail; + } + + /* icmp uge <16 x i8> mask, <16, 16, 16, 16, ...> */ + if (!(max_lanes = simd_build_splat_const_integer_vector(comp_ctx, INT8_TYPE, + 16, 16))) { + goto fail; + } + + /* if the highest bit of every i8 of mask is 1, means doesn't pick up + from vector */ + /* select <16 x i1> %condition, <16 x i8> <0x80, 0x80, ...>, + <16 x i8> %mask */ + if (!(mask_lanes = simd_build_splat_const_integer_vector( + comp_ctx, INT8_TYPE, 0x80, 16))) { + goto fail; + } + + if (!(condition = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, mask, + max_lanes, "compare_with_16"))) { + HANDLE_FAILURE("LLVMBuldICmp"); + goto fail; + } + + if (!(mask = LLVMBuildSelect(comp_ctx->builder, condition, mask_lanes, mask, + "mask"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + goto fail; + } + + param_types[0] = V128_i8x16_TYPE; + param_types[1] = V128_i8x16_TYPE; + if (!(result = aot_call_llvm_intrinsic( + comp_ctx, func_ctx, "llvm.x86.ssse3.pshuf.b.128", V128_i8x16_TYPE, + param_types, 2, vector, mask))) { + HANDLE_FAILURE("LLVMBuildCall"); + goto fail; + } + + if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE, + "ret"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + PUSH_V128(result); + + return true; +fail: + return false; +} + +static bool +aot_compile_simd_swizzle_common(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef vector, mask, default_lane_value, condition, max_lane_id, + result, idx, id, replace_with_zero, elem, elem_or_zero, undef; + uint8 i; + + if (!(mask = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE, + "mask"))) { + goto fail; + } + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_i8x16_TYPE, "vec"))) { + goto fail; + } + + if (!(undef = LLVMGetUndef(V128_i8x16_TYPE))) { + HANDLE_FAILURE("LLVMGetUndef"); + goto fail; + } + + /* icmp uge <16 x i8> mask, <16, 16, 16, 16, ...> */ + if (!(max_lane_id = simd_build_splat_const_integer_vector( + comp_ctx, INT8_TYPE, 16, 16))) { + goto fail; + } + + if (!(condition = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, mask, + max_lane_id, "out_of_range"))) { + HANDLE_FAILURE("LLVMBuldICmp"); + goto fail; + } + + /* if the id is out of range (>=16), set the id as 0 */ + if (!(default_lane_value = simd_build_splat_const_integer_vector( + comp_ctx, INT8_TYPE, 0, 16))) { + goto fail; + } + + if (!(idx = LLVMBuildSelect(comp_ctx->builder, condition, + default_lane_value, mask, "mask"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + goto fail; + } + + for (i = 0; i < 16; i++) { + if (!(id = LLVMBuildExtractElement(comp_ctx->builder, idx, I8_CONST(i), + "id"))) { + HANDLE_FAILURE("LLVMBuildExtractElement"); + goto fail; + } + + if (!(replace_with_zero = + LLVMBuildExtractElement(comp_ctx->builder, condition, + I8_CONST(i), "replace_with_zero"))) { + HANDLE_FAILURE("LLVMBuildExtractElement"); + goto fail; + } + + if (!(elem = LLVMBuildExtractElement(comp_ctx->builder, vector, id, + "vector[mask[i]]"))) { + HANDLE_FAILURE("LLVMBuildExtractElement"); + goto fail; + } + + if (!(elem_or_zero = + LLVMBuildSelect(comp_ctx->builder, replace_with_zero, + I8_CONST(0), elem, "elem_or_zero"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + goto fail; + } + + if (!(undef = + LLVMBuildInsertElement(comp_ctx->builder, undef, elem_or_zero, + I8_CONST(i), "new_vector"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + goto fail; + } + } + + if (!(result = LLVMBuildBitCast(comp_ctx->builder, undef, V128_i64x2_TYPE, + "ret"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + PUSH_V128(result); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_swizzle(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + if (is_target_x86(comp_ctx)) { + return aot_compile_simd_swizzle_x86(comp_ctx, func_ctx); + } + else { + return aot_compile_simd_swizzle_common(comp_ctx, func_ctx); + } +} + +static bool +aot_compile_simd_extract(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 lane_id, bool need_extend, bool is_signed, + LLVMTypeRef vector_type, LLVMTypeRef result_type, + unsigned aot_value_type) +{ + LLVMValueRef vector, lane, result; + + if (!(lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + /* bitcast <2 x i64> %0 to */ + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vec"))) { + goto fail; + } + + /* extractelement %vector, i8 lane_id*/ + if (!(result = LLVMBuildExtractElement(comp_ctx->builder, vector, lane, + "element"))) { + HANDLE_FAILURE("LLVMBuildExtractElement"); + goto fail; + } + + if (need_extend) { + if (is_signed) { + /* sext %element to */ + if (!(result = LLVMBuildSExt(comp_ctx->builder, result, result_type, + "ret"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + goto fail; + } + } + else { + /* sext %element to */ + if (!(result = LLVMBuildZExt(comp_ctx->builder, result, result_type, + "ret"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + goto fail; + } + } + } + + PUSH(result, aot_value_type); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_extract_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id, + bool is_signed) +{ + return aot_compile_simd_extract(comp_ctx, func_ctx, lane_id, true, + is_signed, V128_i8x16_TYPE, I32_TYPE, + VALUE_TYPE_I32); +} + +bool +aot_compile_simd_extract_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id, + bool is_signed) +{ + return aot_compile_simd_extract(comp_ctx, func_ctx, lane_id, true, + is_signed, V128_i16x8_TYPE, I32_TYPE, + VALUE_TYPE_I32); +} + +bool +aot_compile_simd_extract_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_extract(comp_ctx, func_ctx, lane_id, false, false, + V128_i32x4_TYPE, I32_TYPE, VALUE_TYPE_I32); +} + +bool +aot_compile_simd_extract_i64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_extract(comp_ctx, func_ctx, lane_id, false, false, + V128_i64x2_TYPE, I64_TYPE, VALUE_TYPE_I64); +} + +bool +aot_compile_simd_extract_f32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_extract(comp_ctx, func_ctx, lane_id, false, false, + V128_f32x4_TYPE, F32_TYPE, VALUE_TYPE_F32); +} + +bool +aot_compile_simd_extract_f64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_extract(comp_ctx, func_ctx, lane_id, false, false, + V128_f64x2_TYPE, F64_TYPE, VALUE_TYPE_F64); +} + +static bool +aot_compile_simd_replace(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 lane_id, unsigned new_value_type, + LLVMTypeRef vector_type, bool need_reduce, + LLVMTypeRef element_type) +{ + LLVMValueRef vector, new_value, lane, result; + + POP(new_value, new_value_type); + + if (!(lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id))) { + goto fail; + } + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vec"))) { + goto fail; + } + + /* trunc to */ + if (need_reduce) { + if (!(new_value = LLVMBuildTrunc(comp_ctx->builder, new_value, + element_type, "element"))) { + HANDLE_FAILURE("LLVMBuildTrunc"); + goto fail; + } + } + + /* insertelement %vector, %element, + i32 lane */ + if (!(result = LLVMBuildInsertElement(comp_ctx->builder, vector, new_value, + lane, "new_vector"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + goto fail; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "reesult"); + +fail: + return false; +} + +bool +aot_compile_simd_replace_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_replace(comp_ctx, func_ctx, lane_id, VALUE_TYPE_I32, + V128_i8x16_TYPE, true, INT8_TYPE); +} + +bool +aot_compile_simd_replace_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_replace(comp_ctx, func_ctx, lane_id, VALUE_TYPE_I32, + V128_i16x8_TYPE, true, INT16_TYPE); +} + +bool +aot_compile_simd_replace_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_replace(comp_ctx, func_ctx, lane_id, VALUE_TYPE_I32, + V128_i32x4_TYPE, false, I32_TYPE); +} + +bool +aot_compile_simd_replace_i64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_replace(comp_ctx, func_ctx, lane_id, VALUE_TYPE_I64, + V128_i64x2_TYPE, false, I64_TYPE); +} + +bool +aot_compile_simd_replace_f32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_replace(comp_ctx, func_ctx, lane_id, VALUE_TYPE_F32, + V128_f32x4_TYPE, false, F32_TYPE); +} + +bool +aot_compile_simd_replace_f64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id) +{ + return aot_compile_simd_replace(comp_ctx, func_ctx, lane_id, VALUE_TYPE_F64, + V128_f64x2_TYPE, false, F64_TYPE); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_access_lanes.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_access_lanes.h new file mode 100644 index 0000000..75ca71c --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_access_lanes.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_ACCESS_LANES_H_ +#define _SIMD_ACCESS_LANES_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_shuffle(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + const uint8 *frame_ip); + +bool +aot_compile_simd_swizzle(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_extract_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id, + bool is_signed); + +bool +aot_compile_simd_extract_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id, + bool is_signed); + +bool +aot_compile_simd_extract_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_extract_i64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_extract_f32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_extract_f64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_replace_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_replace_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_replace_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_replace_i64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_replace_f32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_replace_f64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 lane_id); + +bool +aot_compile_simd_load8_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 lane_id); + +bool +aot_compile_simd_load16_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 lane_id); + +bool +aot_compile_simd_load32_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 lane_id); + +bool +aot_compile_simd_load64_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 lane_id); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_ACCESS_LANES_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bit_shifts.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bit_shifts.c new file mode 100644 index 0000000..1d645ed --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bit_shifts.c @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_bit_shifts.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +enum integer_shift { + e_shift_i8x16, + e_shift_i16x8, + e_shift_i32x4, + e_shift_i64x2, +}; + +static bool +simd_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op, enum integer_shift itype) +{ + LLVMValueRef vector, offset, result = NULL; + LLVMTypeRef vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE, + V128_i32x4_TYPE, V128_i64x2_TYPE }; + LLVMTypeRef element_type[] = { INT8_TYPE, INT16_TYPE, I32_TYPE, I64_TYPE }; + + LLVMValueRef undef[] = { LLVM_CONST(i8x16_undef), LLVM_CONST(i16x8_undef), + LLVM_CONST(i32x4_undef), LLVM_CONST(i64x2_undef) }; + LLVMValueRef mask[] = { LLVM_CONST(i8x16_vec_zero), + LLVM_CONST(i16x8_vec_zero), + LLVM_CONST(i32x4_vec_zero), + LLVM_CONST(i64x2_vec_zero) }; + LLVMValueRef lane_shift_masks[] = { + LLVMConstInt(I32_TYPE, 7, true), + LLVMConstInt(I32_TYPE, 15, true), + LLVMConstInt(I32_TYPE, 31, true), + LLVMConstInt(I32_TYPE, 63, true), + }; + + POP_I32(offset); + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + vector_type[itype], "vec"))) { + return false; + } + + /* offset = offset & shift_mask */ + if (!lane_shift_masks[itype] + || !(offset = LLVMBuildAnd(comp_ctx->builder, offset, + lane_shift_masks[itype], "offset_fix"))) { + HANDLE_FAILURE("LLVMBuildAnd"); + return false; + } + + /* change type */ + if (itype < e_shift_i32x4) { + offset = LLVMBuildTrunc(comp_ctx->builder, offset, element_type[itype], + "offset_trunc"); + } + else if (itype == e_shift_i64x2) { + offset = LLVMBuildZExt(comp_ctx->builder, offset, element_type[itype], + "offset_ext"); + } + + if (!offset) { + HANDLE_FAILURE("LLVMBuildZext/LLVMBuildTrunc"); + return false; + } + + /* splat to a vector */ + if (!(offset = + LLVMBuildInsertElement(comp_ctx->builder, undef[itype], offset, + I32_ZERO, "offset_vector_base"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + return false; + } + + if (!(offset = + LLVMBuildShuffleVector(comp_ctx->builder, offset, undef[itype], + mask[itype], "offset_vector"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + switch (shift_op) { + case INT_SHL: + { + result = LLVMBuildShl(comp_ctx->builder, vector, offset, "shl"); + break; + } + case INT_SHR_S: + { + result = LLVMBuildAShr(comp_ctx->builder, vector, offset, "ashr"); + break; + } + case INT_SHR_U: + { + result = LLVMBuildLShr(comp_ctx->builder, vector, offset, "lshr"); + break; + } + default: + { + break; + } + } + + if (!result) { + HANDLE_FAILURE("LLVMBuildShl/LLVMBuildLShr/LLVMBuildAShr"); + goto fail; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); + +fail: + return false; +} + +bool +aot_compile_simd_i8x16_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op) +{ + return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i8x16); +} + +bool +aot_compile_simd_i16x8_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op) +{ + return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i16x8); +} + +bool +aot_compile_simd_i32x4_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op) +{ + return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i32x4); +} + +bool +aot_compile_simd_i64x2_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op) +{ + return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i64x2); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bit_shifts.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bit_shifts.h new file mode 100644 index 0000000..06e86ca --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bit_shifts.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_BIT_SHIFTS_H_ +#define _SIMD_BIT_SHIFTS_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_i8x16_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op); + +bool +aot_compile_simd_i16x8_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op); + +bool +aot_compile_simd_i32x4_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op); + +bool +aot_compile_simd_i64x2_shift(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntShift shift_op); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_BIT_SHIFTS_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitmask_extracts.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitmask_extracts.c new file mode 100644 index 0000000..67d9654 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitmask_extracts.c @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_bitmask_extracts.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +enum integer_bitmask_type { + e_bitmask_i8x16, + e_bitmask_i16x8, + e_bitmask_i32x4, + e_bitmask_i64x2, +}; + +/* TODO: should use a much clever intrinsic */ +static bool +simd_build_bitmask(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, + enum integer_bitmask_type itype) +{ + LLVMValueRef vector, mask, result; + uint8 i; + LLVMTypeRef vector_ext_type; + + uint32 lanes[] = { 16, 8, 4, 2 }; + uint32 lane_bits[] = { 8, 16, 32, 64 }; + LLVMTypeRef element_type[] = { INT8_TYPE, INT16_TYPE, I32_TYPE, I64_TYPE }; + LLVMTypeRef vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE, + V128_i32x4_TYPE, V128_i64x2_TYPE }; + int32 mask_element[16] = { 0 }; + const char *intrinsic[] = { + "llvm.vector.reduce.or.v16i64", + "llvm.vector.reduce.or.v8i64", + "llvm.vector.reduce.or.v4i64", + "llvm.vector.reduce.or.v2i64", + }; + + LLVMValueRef ashr_distance; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + vector_type[itype], "vec"))) { + goto fail; + } + + /* fill every bit in a lange with its sign bit */ + if (!(ashr_distance = simd_build_splat_const_integer_vector( + comp_ctx, element_type[itype], lane_bits[itype] - 1, + lanes[itype]))) { + goto fail; + } + + if (!(vector = LLVMBuildAShr(comp_ctx->builder, vector, ashr_distance, + "vec_ashr"))) { + HANDLE_FAILURE("LLVMBuildAShr"); + goto fail; + } + + if (!(vector_ext_type = LLVMVectorType(I64_TYPE, lanes[itype]))) { + HANDLE_FAILURE("LLVMVectorType"); + goto fail; + } + + if (e_bitmask_i64x2 != itype) { + if (!(vector = LLVMBuildSExt(comp_ctx->builder, vector, vector_ext_type, + "zext_to_i64"))) { + goto fail; + } + } + + for (i = 0; i < 16; i++) { + mask_element[i] = 0x1 << i; + } + + if (!(mask = simd_build_const_integer_vector(comp_ctx, I64_TYPE, + mask_element, lanes[itype]))) { + goto fail; + } + + if (!(vector = + LLVMBuildAnd(comp_ctx->builder, vector, mask, "mask_bits"))) { + HANDLE_FAILURE("LLVMBuildAnd"); + goto fail; + } + + if (!(result = + aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic[itype], + I64_TYPE, &vector_ext_type, 1, vector))) { + goto fail; + } + + if (!(result = + LLVMBuildTrunc(comp_ctx->builder, result, I32_TYPE, "to_i32"))) { + HANDLE_FAILURE("LLVMBuildTrunc"); + goto fail; + } + + PUSH_I32(result); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_i8x16_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i8x16); +} + +bool +aot_compile_simd_i16x8_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i16x8); +} + +bool +aot_compile_simd_i32x4_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i32x4); +} + +bool +aot_compile_simd_i64x2_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i64x2); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitmask_extracts.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitmask_extracts.h new file mode 100644 index 0000000..aac4cc2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitmask_extracts.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_BITMASK_EXTRACTS_H_ +#define _SIMD_BITMASK_EXTRACTS_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_i8x16_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i16x8_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i32x4_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i64x2_bitmask(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_BITMASK_EXTRACTS_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitwise_ops.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitwise_ops.c new file mode 100644 index 0000000..66aef36 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitwise_ops.c @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_bitwise_ops.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +static bool +v128_bitwise_two_component(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Bitwise bitwise_op) +{ + LLVMValueRef vector1, vector2, result; + + POP_V128(vector2); + POP_V128(vector1); + + switch (bitwise_op) { + case V128_AND: + if (!(result = LLVMBuildAnd(comp_ctx->builder, vector1, vector2, + "and"))) { + HANDLE_FAILURE("LLVMBuildAnd"); + goto fail; + } + break; + case V128_OR: + if (!(result = + LLVMBuildOr(comp_ctx->builder, vector1, vector2, "or"))) { + HANDLE_FAILURE("LLVMBuildAnd"); + goto fail; + } + break; + case V128_XOR: + if (!(result = LLVMBuildXor(comp_ctx->builder, vector1, vector2, + "xor"))) { + HANDLE_FAILURE("LLVMBuildAnd"); + goto fail; + } + break; + case V128_ANDNOT: + { + /* v128.and(a, v128.not(b)) */ + if (!(vector2 = LLVMBuildNot(comp_ctx->builder, vector2, "not"))) { + HANDLE_FAILURE("LLVMBuildNot"); + goto fail; + } + + if (!(result = LLVMBuildAnd(comp_ctx->builder, vector1, vector2, + "and"))) { + HANDLE_FAILURE("LLVMBuildAnd"); + goto fail; + } + + break; + } + default: + bh_assert(0); + goto fail; + } + + PUSH_V128(result); + return true; +fail: + return false; +} + +static bool +v128_bitwise_not(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef vector, result; + + POP_V128(vector); + + if (!(result = LLVMBuildNot(comp_ctx->builder, vector, "not"))) { + HANDLE_FAILURE("LLVMBuildNot"); + goto fail; + } + + PUSH_V128(result); + return true; +fail: + return false; +} + +/* v128.or(v128.and(v1, c), v128.and(v2, v128.not(c))) */ +static bool +v128_bitwise_bitselect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef vector1, vector2, vector3, result; + + POP_V128(vector3); + POP_V128(vector2); + POP_V128(vector1); + + if (!(vector1 = + LLVMBuildAnd(comp_ctx->builder, vector1, vector3, "a_and_c"))) { + HANDLE_FAILURE("LLVMBuildAdd"); + goto fail; + } + + if (!(vector3 = LLVMBuildNot(comp_ctx->builder, vector3, "not_c"))) { + HANDLE_FAILURE("LLVMBuildNot"); + goto fail; + } + + if (!(vector2 = + LLVMBuildAnd(comp_ctx->builder, vector2, vector3, "b_and_c"))) { + HANDLE_FAILURE("LLVMBuildAdd"); + goto fail; + } + + if (!(result = + LLVMBuildOr(comp_ctx->builder, vector1, vector2, "a_or_b"))) { + HANDLE_FAILURE("LLVMBuildOr"); + goto fail; + } + + PUSH_V128(result); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_v128_bitwise(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, V128Bitwise bitwise_op) +{ + switch (bitwise_op) { + case V128_AND: + case V128_OR: + case V128_XOR: + case V128_ANDNOT: + return v128_bitwise_two_component(comp_ctx, func_ctx, bitwise_op); + case V128_NOT: + return v128_bitwise_not(comp_ctx, func_ctx); + case V128_BITSELECT: + return v128_bitwise_bitselect(comp_ctx, func_ctx); + default: + bh_assert(0); + return false; + } +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitwise_ops.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitwise_ops.h new file mode 100644 index 0000000..ddf81c0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bitwise_ops.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_BITWISE_OPS_H_ +#define _SIMD_BITWISE_OPS_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_v128_bitwise(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, V128Bitwise bitwise_op); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_BITWISE_OPS_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bool_reductions.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bool_reductions.c new file mode 100644 index 0000000..4607d68 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bool_reductions.c @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_bool_reductions.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +enum integer_all_true { + e_int_all_true_v16i8, + e_int_all_true_v8i16, + e_int_all_true_v4i32, + e_int_all_true_v2i64, +}; + +static bool +simd_all_true(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + enum integer_all_true itype) +{ + LLVMValueRef vector, result; + LLVMTypeRef vector_i1_type; + LLVMTypeRef vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE, + V128_i32x4_TYPE, V128_i64x2_TYPE }; + uint32 lanes[] = { 16, 8, 4, 2 }; + const char *intrinsic[] = { + "llvm.vector.reduce.and.v16i1", + "llvm.vector.reduce.and.v8i1", + "llvm.vector.reduce.and.v4i1", + "llvm.vector.reduce.and.v2i1", + }; + LLVMValueRef zero[] = { + LLVM_CONST(i8x16_vec_zero), + LLVM_CONST(i16x8_vec_zero), + LLVM_CONST(i32x4_vec_zero), + LLVM_CONST(i64x2_vec_zero), + }; + + if (!(vector_i1_type = LLVMVectorType(INT1_TYPE, lanes[itype]))) { + HANDLE_FAILURE("LLVMVectorType"); + goto fail; + } + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + vector_type[itype], "vector"))) { + goto fail; + } + + /* compare with zero */ + if (!(result = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, vector, + zero[itype], "ne_zero"))) { + HANDLE_FAILURE("LLVMBuildICmp"); + goto fail; + } + + /* check zero */ + if (!(result = + aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic[itype], + INT1_TYPE, &vector_i1_type, 1, result))) { + goto fail; + } + + if (!(result = + LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, "to_i32"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + goto fail; + } + + PUSH_I32(result); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_i8x16_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v16i8); +} + +bool +aot_compile_simd_i16x8_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v8i16); +} + +bool +aot_compile_simd_i32x4_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v4i32); +} + +bool +aot_compile_simd_i64x2_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v2i64); +} + +bool +aot_compile_simd_v128_any_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMTypeRef vector_type; + LLVMValueRef vector, result; + + if (!(vector_type = LLVMVectorType(INT1_TYPE, 128))) { + return false; + } + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vector"))) { + goto fail; + } + + if (!(result = aot_call_llvm_intrinsic( + comp_ctx, func_ctx, "llvm.vector.reduce.or.v128i1", INT1_TYPE, + &vector_type, 1, vector))) { + goto fail; + } + + if (!(result = + LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, "to_i32"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + goto fail; + } + + PUSH_I32(result); + + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bool_reductions.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bool_reductions.h new file mode 100644 index 0000000..649d5a5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_bool_reductions.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_BOOL_REDUCTIONS_H_ +#define _SIMD_BOOL_REDUCTIONS_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_i8x16_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i16x8_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i32x4_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i64x2_all_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_v128_any_true(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_BOOL_REDUCTIONS_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_common.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_common.c new file mode 100644 index 0000000..95bcdfd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_common.c @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_common.h" + +LLVMValueRef +simd_pop_v128_and_bitcast(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, LLVMTypeRef vec_type, + const char *name) +{ + LLVMValueRef number; + + POP_V128(number); + + if (!(number = + LLVMBuildBitCast(comp_ctx->builder, number, vec_type, name))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + return number; +fail: + return NULL; +} + +bool +simd_bitcast_and_push_v128(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, LLVMValueRef vector, + const char *name) +{ + if (!(vector = LLVMBuildBitCast(comp_ctx->builder, vector, V128_i64x2_TYPE, + name))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + /* push result into the stack */ + PUSH_V128(vector); + + return true; +fail: + return false; +} + +LLVMValueRef +simd_lane_id_to_llvm_value(AOTCompContext *comp_ctx, uint8 lane_id) +{ + LLVMValueRef lane_indexes[] = { + LLVM_CONST(i32_zero), LLVM_CONST(i32_one), + LLVM_CONST(i32_two), LLVM_CONST(i32_three), + LLVM_CONST(i32_four), LLVM_CONST(i32_five), + LLVM_CONST(i32_six), LLVM_CONST(i32_seven), + LLVM_CONST(i32_eight), LLVM_CONST(i32_nine), + LLVM_CONST(i32_ten), LLVM_CONST(i32_eleven), + LLVM_CONST(i32_twelve), LLVM_CONST(i32_thirteen), + LLVM_CONST(i32_fourteen), LLVM_CONST(i32_fifteen), + }; + + return lane_id < 16 ? lane_indexes[lane_id] : NULL; +} + +LLVMValueRef +simd_build_const_integer_vector(const AOTCompContext *comp_ctx, + const LLVMTypeRef element_type, + const int *element_value, uint32 length) +{ + LLVMValueRef vector = NULL; + LLVMValueRef *elements; + unsigned i; + + if (!(elements = wasm_runtime_malloc(sizeof(LLVMValueRef) * length))) { + return NULL; + } + + for (i = 0; i < length; i++) { + if (!(elements[i] = + LLVMConstInt(element_type, element_value[i], true))) { + HANDLE_FAILURE("LLVMConstInst"); + goto fail; + } + } + + if (!(vector = LLVMConstVector(elements, length))) { + HANDLE_FAILURE("LLVMConstVector"); + goto fail; + } + +fail: + wasm_runtime_free(elements); + return vector; +} + +LLVMValueRef +simd_build_splat_const_integer_vector(const AOTCompContext *comp_ctx, + const LLVMTypeRef element_type, + const int64 element_value, uint32 length) +{ + LLVMValueRef vector = NULL, element; + LLVMValueRef *elements; + unsigned i; + + if (!(elements = wasm_runtime_malloc(sizeof(LLVMValueRef) * length))) { + return NULL; + } + + if (!(element = LLVMConstInt(element_type, element_value, true))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + for (i = 0; i < length; i++) { + elements[i] = element; + } + + if (!(vector = LLVMConstVector(elements, length))) { + HANDLE_FAILURE("LLVMConstVector"); + goto fail; + } + +fail: + wasm_runtime_free(elements); + return vector; +} + +LLVMValueRef +simd_build_splat_const_float_vector(const AOTCompContext *comp_ctx, + const LLVMTypeRef element_type, + const float element_value, uint32 length) +{ + LLVMValueRef vector = NULL, element; + LLVMValueRef *elements; + unsigned i; + + if (!(elements = wasm_runtime_malloc(sizeof(LLVMValueRef) * length))) { + return NULL; + } + + if (!(element = LLVMConstReal(element_type, element_value))) { + HANDLE_FAILURE("LLVMConstReal"); + goto fail; + } + + for (i = 0; i < length; i++) { + elements[i] = element; + } + + if (!(vector = LLVMConstVector(elements, length))) { + HANDLE_FAILURE("LLVMConstVector"); + goto fail; + } + +fail: + wasm_runtime_free(elements); + return vector; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_common.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_common.h new file mode 100644 index 0000000..c7a08db --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_common.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_COMMON_H_ +#define _SIMD_COMMON_H_ + +#include "../aot_compiler.h" + +static inline bool +is_target_x86(AOTCompContext *comp_ctx) +{ + return !strncmp(comp_ctx->target_arch, "x86_64", 6) + || !strncmp(comp_ctx->target_arch, "i386", 4); +} + +LLVMValueRef +simd_pop_v128_and_bitcast(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, LLVMTypeRef vec_type, + const char *name); + +bool +simd_bitcast_and_push_v128(const AOTCompContext *comp_ctx, + const AOTFuncContext *func_ctx, LLVMValueRef vector, + const char *name); + +LLVMValueRef +simd_lane_id_to_llvm_value(AOTCompContext *comp_ctx, uint8 lane_id); + +LLVMValueRef +simd_build_const_integer_vector(const AOTCompContext *comp_ctx, + const LLVMTypeRef element_type, + const int *element_value, uint32 length); + +LLVMValueRef +simd_build_splat_const_integer_vector(const AOTCompContext *comp_ctx, + const LLVMTypeRef element_type, + const int64 element_value, uint32 length); + +LLVMValueRef +simd_build_splat_const_float_vector(const AOTCompContext *comp_ctx, + const LLVMTypeRef element_type, + const float element_value, uint32 length); +#endif /* _SIMD_COMMON_H_ */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_comparisons.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_comparisons.c new file mode 100644 index 0000000..8a87ab2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_comparisons.c @@ -0,0 +1,229 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_comparisons.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +static bool +float_cond_2_predicate(FloatCond cond, LLVMRealPredicate *out) +{ + switch (cond) { + case FLOAT_EQ: + *out = LLVMRealOEQ; + break; + case FLOAT_NE: + *out = LLVMRealUNE; + break; + case FLOAT_LT: + *out = LLVMRealOLT; + break; + case FLOAT_GT: + *out = LLVMRealOGT; + break; + case FLOAT_LE: + *out = LLVMRealOLE; + break; + case FLOAT_GE: + *out = LLVMRealOGE; + break; + default: + bh_assert(0); + goto fail; + } + + return true; +fail: + return false; +} + +static bool +int_cond_2_predicate(IntCond cond, LLVMIntPredicate *out) +{ + switch (cond) { + case INT_EQZ: + case INT_EQ: + *out = LLVMIntEQ; + break; + case INT_NE: + *out = LLVMIntNE; + break; + case INT_LT_S: + *out = LLVMIntSLT; + break; + case INT_LT_U: + *out = LLVMIntULT; + break; + case INT_GT_S: + *out = LLVMIntSGT; + break; + case INT_GT_U: + *out = LLVMIntUGT; + break; + case INT_LE_S: + *out = LLVMIntSLE; + break; + case INT_LE_U: + *out = LLVMIntULE; + break; + case INT_GE_S: + *out = LLVMIntSGE; + break; + case INT_GE_U: + *out = LLVMIntUGE; + break; + default: + bh_assert(0); + goto fail; + } + + return true; +fail: + return false; +} + +static bool +interger_vector_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + IntCond cond, LLVMTypeRef vector_type) +{ + LLVMValueRef vec1, vec2, result; + LLVMIntPredicate int_pred; + + if (!(vec2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vec2"))) { + goto fail; + } + + if (!(vec1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vec1"))) { + goto fail; + } + + if (!int_cond_2_predicate(cond, &int_pred)) { + HANDLE_FAILURE("int_cond_2_predicate"); + goto fail; + } + /* icmp %vec1, %vec2 */ + if (!(result = + LLVMBuildICmp(comp_ctx->builder, int_pred, vec1, vec2, "cmp"))) { + HANDLE_FAILURE("LLVMBuildICmp"); + goto fail; + } + + /* sext %result to */ + if (!(result = + LLVMBuildSExt(comp_ctx->builder, result, vector_type, "ext"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + goto fail; + } + + /* bitcast %result to <2 x i64> */ + if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE, + "result"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + PUSH_V128(result); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_i8x16_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond) +{ + return interger_vector_compare(comp_ctx, func_ctx, cond, V128_i8x16_TYPE); +} + +bool +aot_compile_simd_i16x8_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond) +{ + return interger_vector_compare(comp_ctx, func_ctx, cond, V128_i16x8_TYPE); +} + +bool +aot_compile_simd_i32x4_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond) +{ + return interger_vector_compare(comp_ctx, func_ctx, cond, V128_i32x4_TYPE); +} + +bool +aot_compile_simd_i64x2_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond) +{ + return interger_vector_compare(comp_ctx, func_ctx, cond, V128_i64x2_TYPE); +} + +static bool +float_vector_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatCond cond, LLVMTypeRef vector_type, + LLVMTypeRef result_type) +{ + LLVMValueRef vec1, vec2, result; + LLVMRealPredicate real_pred; + + if (!(vec2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vec2"))) { + goto fail; + } + + if (!(vec1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vec1"))) { + goto fail; + } + + if (!float_cond_2_predicate(cond, &real_pred)) { + HANDLE_FAILURE("float_cond_2_predicate"); + goto fail; + } + /* fcmp %vec1, %vec2 */ + if (!(result = + LLVMBuildFCmp(comp_ctx->builder, real_pred, vec1, vec2, "cmp"))) { + HANDLE_FAILURE("LLVMBuildFCmp"); + goto fail; + } + + /* sext %result to */ + if (!(result = + LLVMBuildSExt(comp_ctx->builder, result, result_type, "ext"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + goto fail; + } + + /* bitcast %result to <2 x i64> */ + if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE, + "result"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + goto fail; + } + + PUSH_V128(result); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_f32x4_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, FloatCond cond) +{ + return float_vector_compare(comp_ctx, func_ctx, cond, V128_f32x4_TYPE, + V128_i32x4_TYPE); +} + +bool +aot_compile_simd_f64x2_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, FloatCond cond) +{ + return float_vector_compare(comp_ctx, func_ctx, cond, V128_f64x2_TYPE, + V128_i64x2_TYPE); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_comparisons.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_comparisons.h new file mode 100644 index 0000000..322ebef --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_comparisons.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_COMPARISONS_H_ +#define _SIMD_COMPARISONS_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_i8x16_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond); + +bool +aot_compile_simd_i16x8_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond); + +bool +aot_compile_simd_i32x4_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond); + +bool +aot_compile_simd_i64x2_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, IntCond cond); + +bool +aot_compile_simd_f32x4_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, FloatCond cond); + +bool +aot_compile_simd_f64x2_compare(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, FloatCond cond); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_COMPARISONS_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_construct_values.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_construct_values.c new file mode 100644 index 0000000..ceb09e3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_construct_values.c @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_common.h" +#include "simd_construct_values.h" +#include "../aot_emit_exception.h" +#include "../interpreter/wasm_opcode.h" +#include "../../aot/aot_runtime.h" + +bool +aot_compile_simd_v128_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + const uint8 *imm_bytes) +{ + uint64 imm1, imm2; + LLVMValueRef first_long, agg1, second_long, agg2; + + wasm_runtime_read_v128(imm_bytes, &imm1, &imm2); + + /* %agg1 = insertelement <2 x i64> undef, i16 0, i64 ${*imm} */ + if (!(first_long = I64_CONST(imm1))) { + HANDLE_FAILURE("LLVMConstInt"); + goto fail; + } + + if (!(agg1 = + LLVMBuildInsertElement(comp_ctx->builder, LLVM_CONST(i64x2_undef), + first_long, I32_ZERO, "agg1"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + goto fail; + } + + /* %agg2 = insertelement <2 x i64> %agg1, i16 1, i64 ${*(imm + 1)} */ + if (!(second_long = I64_CONST(imm2))) { + HANDLE_FAILURE("LLVMGetUndef"); + goto fail; + } + + if (!(agg2 = LLVMBuildInsertElement(comp_ctx->builder, agg1, second_long, + I32_ONE, "agg2"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + goto fail; + } + + PUSH_V128(agg2); + return true; +fail: + return false; +} + +bool +aot_compile_simd_splat(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode) +{ + uint32 opcode_index = opcode - SIMD_i8x16_splat; + LLVMValueRef value = NULL, base, new_vector; + LLVMValueRef undefs[] = { + LLVM_CONST(i8x16_undef), LLVM_CONST(i16x8_undef), + LLVM_CONST(i32x4_undef), LLVM_CONST(i64x2_undef), + LLVM_CONST(f32x4_undef), LLVM_CONST(f64x2_undef), + }; + LLVMValueRef masks[] = { + LLVM_CONST(i32x16_zero), LLVM_CONST(i32x8_zero), LLVM_CONST(i32x4_zero), + LLVM_CONST(i32x2_zero), LLVM_CONST(i32x4_zero), LLVM_CONST(i32x2_zero), + }; + + switch (opcode) { + case SIMD_i8x16_splat: + { + LLVMValueRef input; + POP_I32(input); + /* trunc i32 %input to i8 */ + value = + LLVMBuildTrunc(comp_ctx->builder, input, INT8_TYPE, "trunc"); + break; + } + case SIMD_i16x8_splat: + { + LLVMValueRef input; + POP_I32(input); + /* trunc i32 %input to i16 */ + value = + LLVMBuildTrunc(comp_ctx->builder, input, INT16_TYPE, "trunc"); + break; + } + case SIMD_i32x4_splat: + { + POP_I32(value); + break; + } + case SIMD_i64x2_splat: + { + POP(value, VALUE_TYPE_I64); + break; + } + case SIMD_f32x4_splat: + { + POP(value, VALUE_TYPE_F32); + break; + } + case SIMD_f64x2_splat: + { + POP(value, VALUE_TYPE_F64); + break; + } + default: + { + break; + } + } + + if (!value) { + goto fail; + } + + /* insertelement undef, ty %value, i32 0 */ + if (!(base = LLVMBuildInsertElement(comp_ctx->builder, undefs[opcode_index], + value, I32_ZERO, "base"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + goto fail; + } + + /* shufflevector %base, undef, zeroinitializer */ + if (!(new_vector = LLVMBuildShuffleVector( + comp_ctx->builder, base, undefs[opcode_index], + masks[opcode_index], "new_vector"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + goto fail; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, new_vector, "result"); +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_construct_values.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_construct_values.h new file mode 100644 index 0000000..8cd50c8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_construct_values.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_CONSTRUCT_VALUES_H_ +#define _SIMD_CONSTRUCT_VALUES_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_v128_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + const uint8 *imm_bytes); + +bool +aot_compile_simd_splat(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 splat_opcode); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_CONSTRUCT_VALUES_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_conversions.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_conversions.c new file mode 100644 index 0000000..042e280 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_conversions.c @@ -0,0 +1,738 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_conversions.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../aot_emit_numberic.h" +#include "../../aot/aot_runtime.h" + +static bool +simd_integer_narrow_x86(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef in_vector_type, LLVMTypeRef out_vector_type, + const char *instrinsic) +{ + LLVMValueRef vector1, vector2, result; + LLVMTypeRef param_types[2] = { in_vector_type, in_vector_type }; + + if (!(vector2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + in_vector_type, "vec2")) + || !(vector1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + in_vector_type, "vec1"))) { + return false; + } + + if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, instrinsic, + out_vector_type, param_types, 2, + vector1, vector2))) { + HANDLE_FAILURE("LLVMBuildCall"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +enum integer_sat_type { + e_sat_i16x8 = 0, + e_sat_i32x4, + e_sat_i64x2, + e_sat_i32x8, +}; + +static LLVMValueRef +simd_saturate(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + enum integer_sat_type itype, LLVMValueRef vector, + LLVMValueRef min, LLVMValueRef max, bool is_signed) +{ + LLVMValueRef result; + LLVMTypeRef vector_type; + + LLVMTypeRef param_types[][2] = { + { V128_i16x8_TYPE, V128_i16x8_TYPE }, + { V128_i32x4_TYPE, V128_i32x4_TYPE }, + { V128_i64x2_TYPE, V128_i64x2_TYPE }, + { 0 }, + }; + + const char *smin_intrinsic[] = { + "llvm.smin.v8i16", + "llvm.smin.v4i32", + "llvm.smin.v2i64", + "llvm.smin.v8i32", + }; + + const char *umin_intrinsic[] = { + "llvm.umin.v8i16", + "llvm.umin.v4i32", + "llvm.umin.v2i64", + "llvm.umin.v8i32", + }; + + const char *smax_intrinsic[] = { + "llvm.smax.v8i16", + "llvm.smax.v4i32", + "llvm.smax.v2i64", + "llvm.smax.v8i32", + }; + + const char *umax_intrinsic[] = { + "llvm.umax.v8i16", + "llvm.umax.v4i32", + "llvm.umax.v2i64", + "llvm.umax.v8i32", + }; + + if (e_sat_i32x8 == itype) { + if (!(vector_type = LLVMVectorType(I32_TYPE, 8))) { + HANDLE_FAILURE("LLVMVectorType"); + return NULL; + } + + param_types[itype][0] = vector_type; + param_types[itype][1] = vector_type; + } + + if (!(result = aot_call_llvm_intrinsic( + comp_ctx, func_ctx, + is_signed ? smin_intrinsic[itype] : umin_intrinsic[itype], + param_types[itype][0], param_types[itype], 2, vector, max)) + || !(result = aot_call_llvm_intrinsic( + comp_ctx, func_ctx, + is_signed ? smax_intrinsic[itype] : umax_intrinsic[itype], + param_types[itype][0], param_types[itype], 2, result, min))) { + return NULL; + } + + return result; +} + +static bool +simd_integer_narrow_common(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + enum integer_sat_type itype, bool is_signed) +{ + LLVMValueRef vec1, vec2, min, max, mask, result; + LLVMTypeRef in_vector_type[] = { V128_i16x8_TYPE, V128_i32x4_TYPE, + V128_i64x2_TYPE }; + LLVMTypeRef min_max_type[] = { INT16_TYPE, I32_TYPE, I64_TYPE }; + LLVMTypeRef trunc_type[3] = { 0 }; + uint8 length[] = { 8, 4, 2 }; + + int64 smin[] = { 0xff80, 0xffFF8000, 0xffFFffFF80000000 }; + int64 umin[] = { 0x0, 0x0, 0x0 }; + int64 smax[] = { 0x007f, 0x00007fff, 0x000000007fFFffFF }; + int64 umax[] = { 0x00ff, 0x0000ffff, 0x00000000ffFFffFF }; + + LLVMValueRef mask_element[] = { + LLVM_CONST(i32_zero), LLVM_CONST(i32_one), + LLVM_CONST(i32_two), LLVM_CONST(i32_three), + LLVM_CONST(i32_four), LLVM_CONST(i32_five), + LLVM_CONST(i32_six), LLVM_CONST(i32_seven), + LLVM_CONST(i32_eight), LLVM_CONST(i32_nine), + LLVM_CONST(i32_ten), LLVM_CONST(i32_eleven), + LLVM_CONST(i32_twelve), LLVM_CONST(i32_thirteen), + LLVM_CONST(i32_fourteen), LLVM_CONST(i32_fifteen), + }; + + if (!(trunc_type[0] = LLVMVectorType(INT8_TYPE, 8)) + || !(trunc_type[1] = LLVMVectorType(INT16_TYPE, 4)) + || !(trunc_type[2] = LLVMVectorType(I32_TYPE, 2))) { + HANDLE_FAILURE("LLVMVectorType"); + return false; + } + + if (!(vec2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + in_vector_type[itype], "vec2")) + || !(vec1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + in_vector_type[itype], "vec1"))) { + return false; + } + + if (!(max = simd_build_splat_const_integer_vector( + comp_ctx, min_max_type[itype], + is_signed ? smax[itype] : umax[itype], length[itype])) + || !(min = simd_build_splat_const_integer_vector( + comp_ctx, min_max_type[itype], + is_signed ? smin[itype] : umin[itype], length[itype]))) { + return false; + } + + /* Refer to: + * https://github.com/WebAssembly/spec/blob/main/proposals/simd/SIMD.md#integer-to-integer-narrowing + * Regardless of the whether the operation is signed or unsigned, the input + * lanes are interpreted as signed integers. + */ + if (!(vec1 = simd_saturate(comp_ctx, func_ctx, e_sat_i16x8, vec1, min, max, + true)) + || !(vec2 = simd_saturate(comp_ctx, func_ctx, e_sat_i16x8, vec2, min, + max, true))) { + return false; + } + + /* trunc */ + if (!(vec1 = LLVMBuildTrunc(comp_ctx->builder, vec1, trunc_type[itype], + "vec1_trunc")) + || !(vec2 = LLVMBuildTrunc(comp_ctx->builder, vec2, trunc_type[itype], + "vec2_trunc"))) { + HANDLE_FAILURE("LLVMBuildTrunc"); + return false; + } + + /* combine */ + if (!(mask = LLVMConstVector(mask_element, (length[itype] << 1)))) { + HANDLE_FAILURE("LLVMConstInt"); + return false; + } + + if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, vec1, vec2, mask, + "vec_shuffle"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i8x16_narrow_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed) +{ + if (is_target_x86(comp_ctx)) { + return simd_integer_narrow_x86( + comp_ctx, func_ctx, V128_i16x8_TYPE, V128_i8x16_TYPE, + is_signed ? "llvm.x86.sse2.packsswb.128" + : "llvm.x86.sse2.packuswb.128"); + } + else { + return simd_integer_narrow_common(comp_ctx, func_ctx, e_sat_i16x8, + is_signed); + } +} + +bool +aot_compile_simd_i16x8_narrow_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed) +{ + if (is_target_x86(comp_ctx)) { + return simd_integer_narrow_x86(comp_ctx, func_ctx, V128_i32x4_TYPE, + V128_i16x8_TYPE, + is_signed ? "llvm.x86.sse2.packssdw.128" + : "llvm.x86.sse41.packusdw"); + } + else { + return simd_integer_narrow_common(comp_ctx, func_ctx, e_sat_i32x4, + is_signed); + } +} + +enum integer_extend_type { + e_ext_i8x16, + e_ext_i16x8, + e_ext_i32x4, +}; + +static LLVMValueRef +simd_integer_extension(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + enum integer_extend_type itype, LLVMValueRef vector, + bool lower_half, bool is_signed) +{ + LLVMValueRef mask, sub_vector, result; + LLVMValueRef bits[] = { + LLVM_CONST(i32_zero), LLVM_CONST(i32_one), + LLVM_CONST(i32_two), LLVM_CONST(i32_three), + LLVM_CONST(i32_four), LLVM_CONST(i32_five), + LLVM_CONST(i32_six), LLVM_CONST(i32_seven), + LLVM_CONST(i32_eight), LLVM_CONST(i32_nine), + LLVM_CONST(i32_ten), LLVM_CONST(i32_eleven), + LLVM_CONST(i32_twelve), LLVM_CONST(i32_thirteen), + LLVM_CONST(i32_fourteen), LLVM_CONST(i32_fifteen), + }; + LLVMTypeRef out_vector_type[] = { V128_i16x8_TYPE, V128_i32x4_TYPE, + V128_i64x2_TYPE }; + LLVMValueRef undef[] = { LLVM_CONST(i8x16_undef), LLVM_CONST(i16x8_undef), + LLVM_CONST(i32x4_undef) }; + uint32 sub_vector_length[] = { 8, 4, 2 }; + + if (!(mask = lower_half ? LLVMConstVector(bits, sub_vector_length[itype]) + : LLVMConstVector(bits + sub_vector_length[itype], + sub_vector_length[itype]))) { + HANDLE_FAILURE("LLVMConstVector"); + return false; + } + + /* retrive the low or high half */ + if (!(sub_vector = LLVMBuildShuffleVector(comp_ctx->builder, vector, + undef[itype], mask, "half"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + if (is_signed) { + if (!(result = LLVMBuildSExt(comp_ctx->builder, sub_vector, + out_vector_type[itype], "sext"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + return false; + } + } + else { + if (!(result = LLVMBuildZExt(comp_ctx->builder, sub_vector, + out_vector_type[itype], "zext"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + return false; + } + } + + return result; +} + +static bool +simd_integer_extension_wrapper(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + enum integer_extend_type itype, bool lower_half, + bool is_signed) +{ + LLVMValueRef vector, result; + + LLVMTypeRef in_vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE, + V128_i32x4_TYPE }; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + in_vector_type[itype], "vec"))) { + return false; + } + + if (!(result = simd_integer_extension(comp_ctx, func_ctx, itype, vector, + lower_half, is_signed))) { + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i16x8_extend_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed) +{ + return simd_integer_extension_wrapper(comp_ctx, func_ctx, e_ext_i8x16, + lower_half, is_signed); +} + +bool +aot_compile_simd_i32x4_extend_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed) +{ + return simd_integer_extension_wrapper(comp_ctx, func_ctx, e_ext_i16x8, + lower_half, is_signed); +} + +bool +aot_compile_simd_i64x2_extend_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed) +{ + return simd_integer_extension_wrapper(comp_ctx, func_ctx, e_ext_i32x4, + lower_half, is_signed); +} + +static LLVMValueRef +simd_trunc_sat(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + const char *intrinsics, LLVMTypeRef in_vector_type, + LLVMTypeRef out_vector_type) +{ + LLVMValueRef vector, result; + LLVMTypeRef param_types[] = { in_vector_type }; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, in_vector_type, + "vector"))) { + return false; + } + + if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsics, + out_vector_type, param_types, 1, + vector))) { + return false; + } + + return result; +} + +bool +aot_compile_simd_i32x4_trunc_sat_f32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed) +{ + LLVMValueRef result; + if (!(result = simd_trunc_sat(comp_ctx, func_ctx, + is_signed ? "llvm.fptosi.sat.v4i32.v4f32" + : "llvm.fptoui.sat.v4i32.v4f32", + V128_f32x4_TYPE, V128_i32x4_TYPE))) { + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i32x4_trunc_sat_f64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed) +{ + LLVMValueRef result, zero, mask; + LLVMTypeRef out_vector_type; + LLVMValueRef lanes[] = { + LLVM_CONST(i32_zero), + LLVM_CONST(i32_one), + LLVM_CONST(i32_two), + LLVM_CONST(i32_three), + }; + + if (!(out_vector_type = LLVMVectorType(I32_TYPE, 2))) { + HANDLE_FAILURE("LLVMVectorType"); + return false; + } + + if (!(result = simd_trunc_sat(comp_ctx, func_ctx, + is_signed ? "llvm.fptosi.sat.v2i32.v2f64" + : "llvm.fptoui.sat.v2i32.v2f64", + V128_f64x2_TYPE, out_vector_type))) { + return false; + } + + if (!(zero = LLVMConstNull(out_vector_type))) { + HANDLE_FAILURE("LLVMConstNull"); + return false; + } + + /* v2i32 -> v4i32 */ + if (!(mask = LLVMConstVector(lanes, 4))) { + HANDLE_FAILURE("LLVMConstVector"); + return false; + } + + if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, result, zero, mask, + "extend"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +static LLVMValueRef +simd_integer_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_signed, LLVMValueRef vector, + LLVMTypeRef out_vector_type) + +{ + LLVMValueRef result; + result = is_signed ? LLVMBuildSIToFP(comp_ctx->builder, vector, + out_vector_type, "converted") + : LLVMBuildUIToFP(comp_ctx->builder, vector, + out_vector_type, "converted"); + if (!result) { + HANDLE_FAILURE("LLVMBuildSIToFP/LLVMBuildUIToFP"); + } + + return result; +} + +bool +aot_compile_simd_f32x4_convert_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed) +{ + LLVMValueRef vector, result; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_i32x4_TYPE, "vec"))) { + return false; + } + + if (!(result = simd_integer_convert(comp_ctx, func_ctx, is_signed, vector, + V128_f32x4_TYPE))) { + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_f64x2_convert_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed) +{ + LLVMValueRef vector, mask, result; + LLVMValueRef lanes[] = { + LLVM_CONST(i32_zero), + LLVM_CONST(i32_one), + }; + LLVMTypeRef out_vector_type; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_i32x4_TYPE, "vec"))) { + return false; + } + + if (!(out_vector_type = LLVMVectorType(F64_TYPE, 4))) { + HANDLE_FAILURE("LLVMVectorType"); + return false; + } + + if (!(result = simd_integer_convert(comp_ctx, func_ctx, is_signed, vector, + out_vector_type))) { + return false; + } + + /* v4f64 -> v2f64 */ + if (!(mask = LLVMConstVector(lanes, 2))) { + HANDLE_FAILURE("LLVMConstVector"); + return false; + } + + if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, result, result, + mask, "trunc"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +static bool +simd_extadd_pairwise(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef in_vector_type, LLVMTypeRef out_vector_type, + bool is_signed) +{ + LLVMValueRef vector, even_mask, odd_mask, sub_vector_even, sub_vector_odd, + result; + + LLVMValueRef even_element[] = { + LLVM_CONST(i32_zero), LLVM_CONST(i32_two), LLVM_CONST(i32_four), + LLVM_CONST(i32_six), LLVM_CONST(i32_eight), LLVM_CONST(i32_ten), + LLVM_CONST(i32_twelve), LLVM_CONST(i32_fourteen), + }; + + LLVMValueRef odd_element[] = { + LLVM_CONST(i32_one), LLVM_CONST(i32_three), + LLVM_CONST(i32_five), LLVM_CONST(i32_seven), + LLVM_CONST(i32_nine), LLVM_CONST(i32_eleven), + LLVM_CONST(i32_thirteen), LLVM_CONST(i32_fifteen), + }; + + /* assumption about i16x8 from i8x16 and i32x4 from i16x8 */ + uint8 mask_length = V128_i16x8_TYPE == out_vector_type ? 8 : 4; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, in_vector_type, + "vector"))) { + return false; + } + + if (!(even_mask = LLVMConstVector(even_element, mask_length)) + || !(odd_mask = LLVMConstVector(odd_element, mask_length))) { + HANDLE_FAILURE("LLVMConstVector"); + return false; + } + + /* shuffle a <16xi8> vector to two <8xi8> vectors */ + if (!(sub_vector_even = LLVMBuildShuffleVector( + comp_ctx->builder, vector, vector, even_mask, "pick_even")) + || !(sub_vector_odd = LLVMBuildShuffleVector( + comp_ctx->builder, vector, vector, odd_mask, "pick_odd"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + /* sext/zext <8xi8> to <8xi16> */ + if (is_signed) { + if (!(sub_vector_even = + LLVMBuildSExt(comp_ctx->builder, sub_vector_even, + out_vector_type, "even_sext")) + || !(sub_vector_odd = + LLVMBuildSExt(comp_ctx->builder, sub_vector_odd, + out_vector_type, "odd_sext"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + return false; + } + } + else { + if (!(sub_vector_even = + LLVMBuildZExt(comp_ctx->builder, sub_vector_even, + out_vector_type, "even_zext")) + || !(sub_vector_odd = + LLVMBuildZExt(comp_ctx->builder, sub_vector_odd, + out_vector_type, "odd_zext"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + return false; + } + } + + if (!(result = LLVMBuildAdd(comp_ctx->builder, sub_vector_even, + sub_vector_odd, "sum"))) { + HANDLE_FAILURE("LLVMBuildAdd"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i16x8_extadd_pairwise_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_signed) +{ + return simd_extadd_pairwise(comp_ctx, func_ctx, V128_i8x16_TYPE, + V128_i16x8_TYPE, is_signed); +} + +bool +aot_compile_simd_i32x4_extadd_pairwise_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_signed) +{ + return simd_extadd_pairwise(comp_ctx, func_ctx, V128_i16x8_TYPE, + V128_i32x4_TYPE, is_signed); +} + +bool +aot_compile_simd_i16x8_q15mulr_sat(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef lhs, rhs, pad, offset, min, max, result; + LLVMTypeRef vector_ext_type; + + if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i16x8_TYPE, + "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_i16x8_TYPE, "lhs"))) { + return false; + } + + if (!(vector_ext_type = LLVMVectorType(I32_TYPE, 8))) { + HANDLE_FAILURE("LLVMVectorType"); + return false; + } + + if (!(lhs = LLVMBuildSExt(comp_ctx->builder, lhs, vector_ext_type, + "lhs_v8i32")) + || !(rhs = LLVMBuildSExt(comp_ctx->builder, rhs, vector_ext_type, + "rhs_v8i32"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + return false; + } + + /* 0x4000 and 15*/ + if (!(pad = simd_build_splat_const_integer_vector(comp_ctx, I32_TYPE, + 0x4000, 8)) + || !(offset = simd_build_splat_const_integer_vector(comp_ctx, I32_TYPE, + 15, 8))) { + return false; + } + + /* TODO: looking for x86 intrinsics about integer"fused multiply-and-add" */ + /* S.SignedSaturate((x * y + 0x4000) >> 15) */ + if (!(result = LLVMBuildMul(comp_ctx->builder, lhs, rhs, "mul"))) { + HANDLE_FAILURE("LLVMBuildMul"); + return false; + } + + if (!(result = LLVMBuildAdd(comp_ctx->builder, result, pad, "add"))) { + HANDLE_FAILURE("LLVMBuildAdd"); + return false; + } + + if (!(result = LLVMBuildAShr(comp_ctx->builder, result, offset, "ashr"))) { + HANDLE_FAILURE("LLVMBuildAShr"); + return false; + } + + if (!(min = simd_build_splat_const_integer_vector(comp_ctx, I32_TYPE, + 0xffff8000, 8)) + || !(max = simd_build_splat_const_integer_vector(comp_ctx, I32_TYPE, + 0x00007fff, 8))) { + return false; + } + + /* sat after trunc will let *sat* part be optimized */ + if (!(result = simd_saturate(comp_ctx, func_ctx, e_sat_i32x8, result, min, + max, true))) { + return false; + } + + if (!(result = LLVMBuildTrunc(comp_ctx->builder, result, V128_i16x8_TYPE, + "down_to_v8i16"))) { + HANDLE_FAILURE("LLVMBuidlTrunc"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +enum integer_extmul_type { + e_i16x8_extmul_i8x16, + e_i32x4_extmul_i16x8, + e_i64x2_extmul_i32x4, +}; + +static bool +simd_integer_extmul(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool lower_half, bool is_signed, + enum integer_extmul_type itype) +{ + LLVMValueRef vec1, vec2, result; + enum integer_extend_type ext_type[] = { + e_ext_i8x16, + e_ext_i16x8, + e_ext_i32x4, + }; + LLVMTypeRef in_vector_type[] = { + V128_i8x16_TYPE, + V128_i16x8_TYPE, + V128_i32x4_TYPE, + }; + + if (!(vec1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + in_vector_type[itype], "vec1")) + || !(vec2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + in_vector_type[itype], "vec2"))) { + return false; + } + + if (!(vec1 = simd_integer_extension(comp_ctx, func_ctx, ext_type[itype], + vec1, lower_half, is_signed)) + || !(vec2 = simd_integer_extension(comp_ctx, func_ctx, ext_type[itype], + vec2, lower_half, is_signed))) { + return false; + } + + if (!(result = LLVMBuildMul(comp_ctx->builder, vec1, vec2, "product"))) { + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i16x8_extmul_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed) +{ + return simd_integer_extmul(comp_ctx, func_ctx, lower_half, is_signed, + e_i16x8_extmul_i8x16); +} + +bool +aot_compile_simd_i32x4_extmul_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed) +{ + return simd_integer_extmul(comp_ctx, func_ctx, lower_half, is_signed, + e_i32x4_extmul_i16x8); +} + +bool +aot_compile_simd_i64x2_extmul_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed) +{ + return simd_integer_extmul(comp_ctx, func_ctx, lower_half, is_signed, + e_i64x2_extmul_i32x4); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_conversions.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_conversions.h new file mode 100644 index 0000000..e3a1a35 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_conversions.h @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_CONVERSIONS_H_ +#define _SIMD_CONVERSIONS_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_i8x16_narrow_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed); + +bool +aot_compile_simd_i16x8_narrow_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed); + +bool +aot_compile_simd_i16x8_extend_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_low, + bool is_signed); + +bool +aot_compile_simd_i32x4_extend_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_low, + bool is_signed); + +bool +aot_compile_simd_i64x2_extend_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed); + +bool +aot_compile_simd_i32x4_trunc_sat_f32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_signed); + +bool +aot_compile_simd_i32x4_trunc_sat_f64x2(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_signed); + +bool +aot_compile_simd_f32x4_convert_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed); + +bool +aot_compile_simd_f64x2_convert_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_signed); +bool +aot_compile_simd_i16x8_extadd_pairwise_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_signed); + +bool +aot_compile_simd_i32x4_extadd_pairwise_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + bool is_signed); +bool +aot_compile_simd_i16x8_q15mulr_sat(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i16x8_extmul_i8x16(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_low, + bool is_signed); + +bool +aot_compile_simd_i32x4_extmul_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool is_low, + bool is_signed); + +bool +aot_compile_simd_i64x2_extmul_i32x4(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool lower_half, + bool is_signed); +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_CONVERSIONS_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_floating_point.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_floating_point.c new file mode 100644 index 0000000..536ef5b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_floating_point.c @@ -0,0 +1,533 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_floating_point.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../aot_emit_numberic.h" +#include "../../aot/aot_runtime.h" + +static bool +simd_v128_float_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatArithmetic arith_op, LLVMTypeRef vector_type) +{ + LLVMValueRef lhs, rhs, result = NULL; + + if (!(rhs = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "lhs"))) { + return false; + } + + switch (arith_op) { + case FLOAT_ADD: + result = LLVMBuildFAdd(comp_ctx->builder, lhs, rhs, "sum"); + break; + case FLOAT_SUB: + result = LLVMBuildFSub(comp_ctx->builder, lhs, rhs, "difference"); + break; + case FLOAT_MUL: + result = LLVMBuildFMul(comp_ctx->builder, lhs, rhs, "product"); + break; + case FLOAT_DIV: + result = LLVMBuildFDiv(comp_ctx->builder, lhs, rhs, "quotient"); + break; + default: + return false; + } + + if (!result) { + HANDLE_FAILURE( + "LLVMBuildFAdd/LLVMBuildFSub/LLVMBuildFMul/LLVMBuildFDiv"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_f32x4_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatArithmetic arith_op) +{ + return simd_v128_float_arith(comp_ctx, func_ctx, arith_op, V128_f32x4_TYPE); +} + +bool +aot_compile_simd_f64x2_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatArithmetic arith_op) +{ + return simd_v128_float_arith(comp_ctx, func_ctx, arith_op, V128_f64x2_TYPE); +} + +static bool +simd_v128_float_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef vector_type) +{ + LLVMValueRef vector, result; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vector"))) { + return false; + } + + if (!(result = LLVMBuildFNeg(comp_ctx->builder, vector, "neg"))) { + HANDLE_FAILURE("LLVMBuildFNeg"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_f32x4_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_v128_float_neg(comp_ctx, func_ctx, V128_f32x4_TYPE); +} + +bool +aot_compile_simd_f64x2_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_v128_float_neg(comp_ctx, func_ctx, V128_f64x2_TYPE); +} + +static bool +simd_float_intrinsic(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef vector_type, const char *intrinsic) +{ + LLVMValueRef vector, result; + LLVMTypeRef param_types[1] = { vector_type }; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vector"))) { + return false; + } + + if (!(result = + aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, + vector_type, param_types, 1, vector))) { + HANDLE_FAILURE("LLVMBuildCall"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_f32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE, + "llvm.fabs.v4f32"); +} + +bool +aot_compile_simd_f64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE, + "llvm.fabs.v2f64"); +} + +bool +aot_compile_simd_f32x4_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE, + "llvm.sqrt.v4f32"); +} + +bool +aot_compile_simd_f64x2_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE, + "llvm.sqrt.v2f64"); +} + +bool +aot_compile_simd_f32x4_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE, + "llvm.ceil.v4f32"); +} + +bool +aot_compile_simd_f64x2_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE, + "llvm.ceil.v2f64"); +} + +bool +aot_compile_simd_f32x4_floor(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE, + "llvm.floor.v4f32"); +} + +bool +aot_compile_simd_f64x2_floor(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE, + "llvm.floor.v2f64"); +} + +bool +aot_compile_simd_f32x4_trunc(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE, + "llvm.trunc.v4f32"); +} + +bool +aot_compile_simd_f64x2_trunc(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE, + "llvm.trunc.v2f64"); +} + +bool +aot_compile_simd_f32x4_nearest(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE, + "llvm.rint.v4f32"); +} + +bool +aot_compile_simd_f64x2_nearest(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE, + "llvm.rint.v2f64"); +} + +static bool +simd_float_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatArithmetic op, LLVMTypeRef vector_type) +{ + LLVMValueRef lhs, rhs, cmp, selected; + + if (!(rhs = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "lhs"))) { + return false; + } + + if (!(cmp = LLVMBuildFCmp(comp_ctx->builder, + op == FLOAT_MIN ? LLVMRealOLT : LLVMRealOGT, rhs, + lhs, "cmp"))) { + HANDLE_FAILURE("LLVMBuildFCmp"); + return false; + } + + if (!(selected = + LLVMBuildSelect(comp_ctx->builder, cmp, rhs, lhs, "selected"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, selected, "result"); +} + +static bool +simd_float_min(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef vector_type) +{ + LLVMValueRef lhs, rhs, lhs_nan, rhs_nan, olt_ret, ogt_ret, or_ret, ret1, + ret2, ret3, ret4; + + if (!(rhs = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "lhs"))) { + return false; + } + + if (!(lhs_nan = LLVMBuildFCmp(comp_ctx->builder, LLVMRealUNO, lhs, lhs, + "lhs_nan"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealUNO"); + return false; + } + + if (!(rhs_nan = LLVMBuildFCmp(comp_ctx->builder, LLVMRealUNO, rhs, rhs, + "rhs_nan"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealUNO"); + return false; + } + + if (!(olt_ret = LLVMBuildFCmp(comp_ctx->builder, LLVMRealOLT, lhs, rhs, + "olt_ret"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealOLT"); + return false; + } + + if (!(ogt_ret = LLVMBuildFCmp(comp_ctx->builder, LLVMRealOGT, lhs, rhs, + "ogt_ret"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealOGT"); + return false; + } + + /* lhs or rhs */ + { + LLVMValueRef integer_l, integer_r, integer_or; + + if (!(integer_l = LLVMBuildBitCast(comp_ctx->builder, lhs, + V128_i64x2_TYPE, "lhs_to_int"))) { + HANDLE_FAILURE("LLVMBuildBitCas"); + return false; + } + + if (!(integer_r = LLVMBuildBitCast(comp_ctx->builder, rhs, + V128_i64x2_TYPE, "rhs_to_int"))) { + HANDLE_FAILURE("LLVMBuildBitCas"); + return false; + } + + if (!(integer_or = + LLVMBuildOr(comp_ctx->builder, integer_l, integer_r, "or"))) { + HANDLE_FAILURE("LLVMBuildOr"); + return false; + } + + if (!(or_ret = LLVMBuildBitCast(comp_ctx->builder, integer_or, + vector_type, "holder"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + return false; + } + } + + if (!(ret1 = LLVMBuildSelect(comp_ctx->builder, olt_ret, lhs, or_ret, + "sel_olt"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + if (!(ret2 = LLVMBuildSelect(comp_ctx->builder, ogt_ret, rhs, ret1, + "sel_ogt"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + if (!(ret3 = LLVMBuildSelect(comp_ctx->builder, lhs_nan, lhs, ret2, + "sel_lhs_nan"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + if (!(ret4 = LLVMBuildSelect(comp_ctx->builder, rhs_nan, rhs, ret3, + "sel_rhs_nan"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, ret4, "result"); +} + +static bool +simd_float_max(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef vector_type) +{ + LLVMValueRef lhs, rhs, lhs_nan, rhs_nan, olt_ret, ogt_ret, and_ret, ret1, + ret2, ret3, ret4; + + if (!(rhs = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "lhs"))) { + return false; + } + + if (!(lhs_nan = LLVMBuildFCmp(comp_ctx->builder, LLVMRealUNO, lhs, lhs, + "lhs_nan"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealUNO"); + return false; + } + + if (!(rhs_nan = LLVMBuildFCmp(comp_ctx->builder, LLVMRealUNO, rhs, rhs, + "rhs_nan"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealUNO"); + return false; + } + + if (!(olt_ret = LLVMBuildFCmp(comp_ctx->builder, LLVMRealOLT, lhs, rhs, + "olt_ret"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealOLT"); + return false; + } + + if (!(ogt_ret = LLVMBuildFCmp(comp_ctx->builder, LLVMRealOGT, lhs, rhs, + "ogt_ret"))) { + HANDLE_FAILURE("LLVMBuildFCmp + LLVMRealOGT"); + return false; + } + + /* lhs and rhs */ + { + LLVMValueRef integer_l, integer_r, integer_and; + + if (!(integer_l = LLVMBuildBitCast(comp_ctx->builder, lhs, + V128_i64x2_TYPE, "lhs_to_int"))) { + HANDLE_FAILURE("LLVMBuildBitCas"); + return false; + } + + if (!(integer_r = LLVMBuildBitCast(comp_ctx->builder, rhs, + V128_i64x2_TYPE, "rhs_to_int"))) { + HANDLE_FAILURE("LLVMBuildBitCas"); + return false; + } + + if (!(integer_and = LLVMBuildAnd(comp_ctx->builder, integer_l, + integer_r, "and"))) { + HANDLE_FAILURE("LLVMBuildOr"); + return false; + } + + if (!(and_ret = LLVMBuildBitCast(comp_ctx->builder, integer_and, + vector_type, "holder"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + return false; + } + } + + if (!(ret1 = LLVMBuildSelect(comp_ctx->builder, ogt_ret, lhs, and_ret, + "sel_ogt"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + if (!(ret2 = LLVMBuildSelect(comp_ctx->builder, olt_ret, rhs, ret1, + "sel_olt"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + if (!(ret3 = LLVMBuildSelect(comp_ctx->builder, lhs_nan, lhs, ret2, + "sel_lhs_nan"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + if (!(ret4 = LLVMBuildSelect(comp_ctx->builder, rhs_nan, rhs, ret3, + "sel_rhs_nan"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, ret4, "result"); +} + +bool +aot_compile_simd_f32x4_min_max(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min) +{ + return run_min ? simd_float_min(comp_ctx, func_ctx, V128_f32x4_TYPE) + : simd_float_max(comp_ctx, func_ctx, V128_f32x4_TYPE); +} + +bool +aot_compile_simd_f64x2_min_max(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min) +{ + return run_min ? simd_float_min(comp_ctx, func_ctx, V128_f64x2_TYPE) + : simd_float_max(comp_ctx, func_ctx, V128_f64x2_TYPE); +} + +bool +aot_compile_simd_f32x4_pmin_pmax(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min) +{ + return simd_float_cmp(comp_ctx, func_ctx, run_min ? FLOAT_MIN : FLOAT_MAX, + V128_f32x4_TYPE); +} + +bool +aot_compile_simd_f64x2_pmin_pmax(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min) +{ + return simd_float_cmp(comp_ctx, func_ctx, run_min ? FLOAT_MIN : FLOAT_MAX, + V128_f64x2_TYPE); +} + +bool +aot_compile_simd_f64x2_demote(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef vector, elem_0, elem_1, result; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_f64x2_TYPE, "vector"))) { + return false; + } + + if (!(elem_0 = LLVMBuildExtractElement(comp_ctx->builder, vector, + LLVM_CONST(i32_zero), "elem_0")) + || !(elem_1 = LLVMBuildExtractElement(comp_ctx->builder, vector, + LLVM_CONST(i32_one), "elem_1"))) { + HANDLE_FAILURE("LLVMBuildExtractElement"); + return false; + } + + /* fptrunc elem to */ + if (!(elem_0 = LLVMBuildFPTrunc(comp_ctx->builder, elem_0, F32_TYPE, + "elem_0_trunc")) + || !(elem_1 = LLVMBuildFPTrunc(comp_ctx->builder, elem_1, F32_TYPE, + "elem_1_trunc"))) { + HANDLE_FAILURE("LLVMBuildFPTrunc"); + return false; + } + + if (!(result = LLVMBuildInsertElement(comp_ctx->builder, + LLVM_CONST(f32x4_vec_zero), elem_0, + LLVM_CONST(i32_zero), "new_vector_0")) + || !(result = + LLVMBuildInsertElement(comp_ctx->builder, result, elem_1, + LLVM_CONST(i32_one), "new_vector_1"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_f32x4_promote(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef vector, elem_0, elem_1, result; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_f32x4_TYPE, "vector"))) { + return false; + } + + if (!(elem_0 = LLVMBuildExtractElement(comp_ctx->builder, vector, + LLVM_CONST(i32_zero), "elem_0")) + || !(elem_1 = LLVMBuildExtractElement(comp_ctx->builder, vector, + LLVM_CONST(i32_one), "elem_1"))) { + HANDLE_FAILURE("LLVMBuildExtractElement"); + return false; + } + + /* fpext elem to */ + if (!(elem_0 = + LLVMBuildFPExt(comp_ctx->builder, elem_0, F64_TYPE, "elem_0_ext")) + || !(elem_1 = LLVMBuildFPExt(comp_ctx->builder, elem_1, F64_TYPE, + "elem_1_ext"))) { + HANDLE_FAILURE("LLVMBuildFPExt"); + return false; + } + + if (!(result = LLVMBuildInsertElement(comp_ctx->builder, + LLVM_CONST(f64x2_vec_zero), elem_0, + LLVM_CONST(i32_zero), "new_vector_0")) + || !(result = + LLVMBuildInsertElement(comp_ctx->builder, result, elem_1, + LLVM_CONST(i32_one), "new_vector_1"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_floating_point.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_floating_point.h new file mode 100644 index 0000000..39e37c8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_floating_point.h @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_FLOATING_POINT_H_ +#define _SIMD_FLOATING_POINT_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_f32x4_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatArithmetic arith_op); + +bool +aot_compile_simd_f64x2_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + FloatArithmetic arith_op); + +bool +aot_compile_simd_f32x4_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f64x2_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f64x2_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f64x2_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_floor(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f64x2_floor(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_trunc(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f64x2_trunc(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_nearest(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f64x2_nearest(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_min_max(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min); + +bool +aot_compile_simd_f64x2_min_max(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min); + +bool +aot_compile_simd_f32x4_pmin_pmax(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min); + +bool +aot_compile_simd_f64x2_pmin_pmax(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, bool run_min); + +bool +aot_compile_simd_f64x2_demote(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_f32x4_promote(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_FLOATING_POINT_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_int_arith.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_int_arith.c new file mode 100644 index 0000000..6a1902d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_int_arith.c @@ -0,0 +1,397 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_int_arith.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +static bool +simd_integer_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op, LLVMTypeRef vector_type) +{ + LLVMValueRef lhs, rhs, result = NULL; + + if (!(rhs = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "lhs"))) { + return false; + } + + switch (arith_op) { + case V128_ADD: + result = LLVMBuildAdd(comp_ctx->builder, lhs, rhs, "sum"); + break; + case V128_SUB: + result = LLVMBuildSub(comp_ctx->builder, lhs, rhs, "difference"); + break; + case V128_MUL: + result = LLVMBuildMul(comp_ctx->builder, lhs, rhs, "product"); + break; + default: + HANDLE_FAILURE("Unsupport arith_op"); + break; + } + + if (!result) { + HANDLE_FAILURE("LLVMBuildAdd/LLVMBuildSub/LLVMBuildMul"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i8x16_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op) +{ + return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i8x16_TYPE); +} + +bool +aot_compile_simd_i16x8_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op) +{ + return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i16x8_TYPE); +} + +bool +aot_compile_simd_i32x4_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op) +{ + return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i32x4_TYPE); +} + +bool +aot_compile_simd_i64x2_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op) +{ + return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i64x2_TYPE); +} + +static bool +simd_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMTypeRef type) +{ + LLVMValueRef vector, result; + + if (!(vector = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, type, "vector"))) { + return false; + } + + if (!(result = LLVMBuildNeg(comp_ctx->builder, vector, "neg"))) { + HANDLE_FAILURE("LLVMBuildNeg"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i8x16_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_neg(comp_ctx, func_ctx, V128_i8x16_TYPE); +} + +bool +aot_compile_simd_i16x8_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_neg(comp_ctx, func_ctx, V128_i16x8_TYPE); +} + +bool +aot_compile_simd_i32x4_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_neg(comp_ctx, func_ctx, V128_i32x4_TYPE); +} + +bool +aot_compile_simd_i64x2_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_neg(comp_ctx, func_ctx, V128_i64x2_TYPE); +} + +bool +aot_compile_simd_i8x16_popcnt(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef vector, result; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_i8x16_TYPE, "vector"))) { + return false; + } + + if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, + "llvm.ctpop.v16i8", V128_i8x16_TYPE, + &V128_i8x16_TYPE, 1, vector))) { + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +static bool +simd_v128_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef vector_type, V128Arithmetic arith_op, bool is_signed) +{ + LLVMValueRef lhs, rhs, result; + LLVMIntPredicate op; + + if (!(rhs = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "lhs"))) { + return false; + } + + if (V128_MIN == arith_op) { + op = is_signed ? LLVMIntSLT : LLVMIntULT; + } + else { + op = is_signed ? LLVMIntSGT : LLVMIntUGT; + } + + if (!(result = LLVMBuildICmp(comp_ctx->builder, op, lhs, rhs, "cmp"))) { + HANDLE_FAILURE("LLVMBuildICmp"); + return false; + } + + if (!(result = + LLVMBuildSelect(comp_ctx->builder, result, lhs, rhs, "select"))) { + HANDLE_FAILURE("LLVMBuildSelect"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i8x16_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed) +{ + return simd_v128_cmp(comp_ctx, func_ctx, V128_i8x16_TYPE, arith_op, + is_signed); +} + +bool +aot_compile_simd_i16x8_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed) +{ + return simd_v128_cmp(comp_ctx, func_ctx, V128_i16x8_TYPE, arith_op, + is_signed); +} + +bool +aot_compile_simd_i32x4_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed) +{ + return simd_v128_cmp(comp_ctx, func_ctx, V128_i32x4_TYPE, arith_op, + is_signed); +} + +/* llvm.abs.* */ +static bool +simd_v128_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + char *intrinsic, LLVMTypeRef vector_type) +{ + LLVMValueRef vector, result; + LLVMTypeRef param_types[] = { vector_type, INT1_TYPE }; + + if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "vec"))) { + return false; + } + + if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, + vector_type, param_types, 2, vector, + /* is_int_min_poison */ + LLVM_CONST(i1_zero)))) { + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i8x16_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v16i8", V128_i8x16_TYPE); +} + +bool +aot_compile_simd_i16x8_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v8i16", V128_i16x8_TYPE); +} + +bool +aot_compile_simd_i32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v4i32", V128_i32x4_TYPE); +} + +bool +aot_compile_simd_i64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v2i64", V128_i64x2_TYPE); +} + +enum integer_avgr_u { + e_avgr_u_i8x16, + e_avgr_u_i16x8, +}; + +/* TODO: try int_x86_mmx_pavg_b and int_x86_mmx_pavg_w */ +/* (v1 + v2 + 1) / 2 */ +static bool +simd_v128_avg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + enum integer_avgr_u itype) +{ + LLVMValueRef lhs, rhs, ones, result; + LLVMTypeRef vector_ext_type; + LLVMTypeRef vector_type[] = { + V128_i8x16_TYPE, + V128_i16x8_TYPE, + }; + unsigned lanes[] = { 16, 8 }; + + if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + vector_type[itype], "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + vector_type[itype], "lhs"))) { + return false; + } + + if (!(vector_ext_type = LLVMVectorType(I64_TYPE, lanes[itype]))) { + HANDLE_FAILURE("LLVMVectorType"); + return false; + } + + if (!(lhs = LLVMBuildZExt(comp_ctx->builder, lhs, vector_ext_type, + "zext_to_i64")) + || !(rhs = LLVMBuildZExt(comp_ctx->builder, rhs, vector_ext_type, + "zext_to_i64"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + return false; + } + + /* by default, add will do signed/unsigned overflow */ + if (!(result = LLVMBuildAdd(comp_ctx->builder, lhs, rhs, "l_add_r"))) { + HANDLE_FAILURE("LLVMBuildAdd"); + return false; + } + + if (!(ones = simd_build_splat_const_integer_vector(comp_ctx, I64_TYPE, 1, + lanes[itype]))) { + return false; + } + + if (!(result = LLVMBuildAdd(comp_ctx->builder, result, ones, "plus_1"))) { + HANDLE_FAILURE("LLVMBuildAdd"); + return false; + } + + if (!(result = LLVMBuildLShr(comp_ctx->builder, result, ones, "avg"))) { + HANDLE_FAILURE("LLVMBuildLShr"); + return false; + } + + if (!(result = LLVMBuildTrunc(comp_ctx->builder, result, vector_type[itype], + "to_orig_type"))) { + HANDLE_FAILURE("LLVMBuildTrunc"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i8x16_avgr_u(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_v128_avg(comp_ctx, func_ctx, e_avgr_u_i8x16); +} + +bool +aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + return simd_v128_avg(comp_ctx, func_ctx, e_avgr_u_i16x8); +} + +bool +aot_compile_simd_i32x4_dot_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef vec1, vec2, even_mask, odd_mask, zero, result; + LLVMTypeRef vector_ext_type; + LLVMValueRef even_element[] = { + LLVM_CONST(i32_zero), + LLVM_CONST(i32_two), + LLVM_CONST(i32_four), + LLVM_CONST(i32_six), + }; + LLVMValueRef odd_element[] = { + LLVM_CONST(i32_one), + LLVM_CONST(i32_three), + LLVM_CONST(i32_five), + LLVM_CONST(i32_seven), + }; + + if (!(vec1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i16x8_TYPE, + "vec1")) + || !(vec2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, + V128_i16x8_TYPE, "vec2"))) { + return false; + } + + if (!(vector_ext_type = LLVMVectorType(I32_TYPE, 8))) { + HANDLE_FAILURE("LLVMVectorType"); + return false; + } + + /* sext to */ + if (!(vec1 = LLVMBuildSExt(comp_ctx->builder, vec1, vector_ext_type, + "vec1_v8i32")) + || !(vec2 = LLVMBuildSExt(comp_ctx->builder, vec2, vector_ext_type, + "vec2_v8i32"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + return false; + } + + if (!(result = LLVMBuildMul(comp_ctx->builder, vec1, vec2, "product"))) { + HANDLE_FAILURE("LLVMBuildMul"); + return false; + } + + /* pick elements with even indexes and odd indexes */ + if (!(even_mask = LLVMConstVector(even_element, 4)) + || !(odd_mask = LLVMConstVector(odd_element, 4))) { + HANDLE_FAILURE("LLVMConstVector"); + return false; + } + + if (!(zero = simd_build_splat_const_integer_vector(comp_ctx, I32_TYPE, 0, + 8))) { + return false; + } + + if (!(vec1 = LLVMBuildShuffleVector(comp_ctx->builder, result, zero, + even_mask, "even_result")) + || !(vec2 = LLVMBuildShuffleVector(comp_ctx->builder, result, zero, + odd_mask, "odd_result"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + if (!(result = LLVMBuildAdd(comp_ctx->builder, vec1, vec2, "new_vec"))) { + HANDLE_FAILURE("LLVMBuildAdd"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_int_arith.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_int_arith.h new file mode 100644 index 0000000..49827d5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_int_arith.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_INT_ARITH_H_ +#define _SIMD_INT_ARITH_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_i8x16_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic cond); + +bool +aot_compile_simd_i16x8_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic cond); + +bool +aot_compile_simd_i32x4_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic cond); + +bool +aot_compile_simd_i64x2_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic cond); + +bool +aot_compile_simd_i8x16_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i16x8_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i32x4_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i64x2_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i8x16_popcnt(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i8x16_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed); + +bool +aot_compile_simd_i16x8_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed); + +bool +aot_compile_simd_i32x4_cmp(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed); + +bool +aot_compile_simd_i8x16_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i16x8_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i8x16_avgr_u(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_compile_simd_i32x4_dot_i16x8(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_INT_ARITH_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_load_store.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_load_store.c new file mode 100644 index 0000000..0e86972 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_load_store.c @@ -0,0 +1,361 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_common.h" +#include "simd_load_store.h" +#include "../aot_emit_exception.h" +#include "../aot_emit_memory.h" +#include "../../aot/aot_runtime.h" +#include "../../interpreter/wasm_opcode.h" + +/* data_length in bytes */ +static LLVMValueRef +simd_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 align, + uint32 offset, uint32 data_length, LLVMTypeRef ptr_type, + LLVMTypeRef data_type, bool enable_segue) +{ + LLVMValueRef maddr, data; + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, + data_length, enable_segue))) { + HANDLE_FAILURE("aot_check_memory_overflow"); + return NULL; + } + + if (!(maddr = LLVMBuildBitCast(comp_ctx->builder, maddr, ptr_type, + "data_ptr"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + return NULL; + } + + if (!(data = LLVMBuildLoad2(comp_ctx->builder, data_type, maddr, "data"))) { + HANDLE_FAILURE("LLVMBuildLoad"); + return NULL; + } + + LLVMSetAlignment(data, 1); + + return data; +} + +bool +aot_compile_simd_v128_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset) +{ + bool enable_segue = comp_ctx->enable_segue_v128_load; + LLVMTypeRef v128_ptr_type = enable_segue ? V128_PTR_TYPE_GS : V128_PTR_TYPE; + LLVMValueRef result; + + if (!(result = simd_load(comp_ctx, func_ctx, align, offset, 16, + v128_ptr_type, V128_TYPE, enable_segue))) { + return false; + } + + PUSH_V128(result); + + return true; +fail: + return false; +} + +bool +aot_compile_simd_load_extend(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset) +{ + LLVMValueRef sub_vector, result; + uint32 opcode_index = opcode - SIMD_v128_load8x8_s; + bool signeds[] = { true, false, true, false, true, false }; + LLVMTypeRef vector_types[] = { + V128_i16x8_TYPE, V128_i16x8_TYPE, V128_i32x4_TYPE, + V128_i32x4_TYPE, V128_i64x2_TYPE, V128_i64x2_TYPE, + }; + LLVMTypeRef sub_vector_types[] = { + LLVMVectorType(INT8_TYPE, 8), LLVMVectorType(INT8_TYPE, 8), + LLVMVectorType(INT16_TYPE, 4), LLVMVectorType(INT16_TYPE, 4), + LLVMVectorType(I32_TYPE, 2), LLVMVectorType(I32_TYPE, 2), + }; + LLVMTypeRef sub_vector_type, sub_vector_ptr_type; + bool enable_segue = comp_ctx->enable_segue_v128_load; + + bh_assert(opcode_index < 6); + + sub_vector_type = sub_vector_types[opcode_index]; + + /* to vector ptr type */ + if (!sub_vector_type + || !(sub_vector_ptr_type = + LLVMPointerType(sub_vector_type, enable_segue ? 256 : 0))) { + HANDLE_FAILURE("LLVMPointerType"); + return false; + } + + if (!(sub_vector = + simd_load(comp_ctx, func_ctx, align, offset, 8, + sub_vector_ptr_type, sub_vector_type, enable_segue))) { + return false; + } + + if (signeds[opcode_index]) { + if (!(result = LLVMBuildSExt(comp_ctx->builder, sub_vector, + vector_types[opcode_index], "vector"))) { + HANDLE_FAILURE("LLVMBuildSExt"); + return false; + } + } + else { + if (!(result = LLVMBuildZExt(comp_ctx->builder, sub_vector, + vector_types[opcode_index], "vector"))) { + HANDLE_FAILURE("LLVMBuildZExt"); + return false; + } + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_load_splat(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset) +{ + uint32 opcode_index = opcode - SIMD_v128_load8_splat; + LLVMValueRef element, result; + LLVMTypeRef element_ptr_types[] = { INT8_PTR_TYPE, INT16_PTR_TYPE, + INT32_PTR_TYPE, INT64_PTR_TYPE }; + LLVMTypeRef element_ptr_types_gs[] = { INT8_PTR_TYPE_GS, INT16_PTR_TYPE_GS, + INT32_PTR_TYPE_GS, + INT64_PTR_TYPE_GS }; + LLVMTypeRef element_data_types[] = { INT8_TYPE, INT16_TYPE, I32_TYPE, + I64_TYPE }; + uint32 data_lengths[] = { 1, 2, 4, 8 }; + LLVMValueRef undefs[] = { + LLVM_CONST(i8x16_undef), + LLVM_CONST(i16x8_undef), + LLVM_CONST(i32x4_undef), + LLVM_CONST(i64x2_undef), + }; + LLVMValueRef masks[] = { + LLVM_CONST(i32x16_zero), + LLVM_CONST(i32x8_zero), + LLVM_CONST(i32x4_zero), + LLVM_CONST(i32x2_zero), + }; + bool enable_segue = comp_ctx->enable_segue_v128_load; + + bh_assert(opcode_index < 4); + + if (!(element = simd_load( + comp_ctx, func_ctx, align, offset, data_lengths[opcode_index], + comp_ctx->enable_segue_v128_load + ? element_ptr_types_gs[opcode_index] + : element_ptr_types[opcode_index], + element_data_types[opcode_index], enable_segue))) { + return false; + } + + if (!(result = + LLVMBuildInsertElement(comp_ctx->builder, undefs[opcode_index], + element, I32_ZERO, "base"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + return false; + } + + if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, result, + undefs[opcode_index], + masks[opcode_index], "vector"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_load_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset, + uint8 lane_id) +{ + LLVMValueRef element, vector; + uint32 opcode_index = opcode - SIMD_v128_load8_lane; + uint32 data_lengths[] = { 1, 2, 4, 8 }; + LLVMTypeRef element_ptr_types[] = { INT8_PTR_TYPE, INT16_PTR_TYPE, + INT32_PTR_TYPE, INT64_PTR_TYPE }; + LLVMTypeRef element_ptr_types_gs[] = { INT8_PTR_TYPE_GS, INT16_PTR_TYPE_GS, + INT32_PTR_TYPE_GS, + INT64_PTR_TYPE_GS }; + LLVMTypeRef element_data_types[] = { INT8_TYPE, INT16_TYPE, I32_TYPE, + I64_TYPE }; + LLVMTypeRef vector_types[] = { V128_i8x16_TYPE, V128_i16x8_TYPE, + V128_i32x4_TYPE, V128_i64x2_TYPE }; + LLVMValueRef lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id); + bool enable_segue = comp_ctx->enable_segue_v128_load; + + bh_assert(opcode_index < 4); + + if (!(vector = simd_pop_v128_and_bitcast( + comp_ctx, func_ctx, vector_types[opcode_index], "src"))) { + return false; + } + + if (!(element = simd_load( + comp_ctx, func_ctx, align, offset, data_lengths[opcode_index], + comp_ctx->enable_segue_v128_load + ? element_ptr_types_gs[opcode_index] + : element_ptr_types[opcode_index], + element_data_types[opcode_index], enable_segue))) { + return false; + } + + if (!(vector = LLVMBuildInsertElement(comp_ctx->builder, vector, element, + lane, "dst"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, vector, "result"); +} + +bool +aot_compile_simd_load_zero(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset) +{ + LLVMValueRef element, result, mask; + uint32 opcode_index = opcode - SIMD_v128_load32_zero; + uint32 data_lengths[] = { 4, 8 }; + LLVMTypeRef element_ptr_types[] = { INT32_PTR_TYPE, INT64_PTR_TYPE }; + LLVMTypeRef element_ptr_types_gs[] = { INT32_PTR_TYPE_GS, + INT64_PTR_TYPE_GS }; + LLVMTypeRef element_data_types[] = { I32_TYPE, I64_TYPE }; + LLVMValueRef zero[] = { + LLVM_CONST(i32x4_vec_zero), + LLVM_CONST(i64x2_vec_zero), + }; + LLVMValueRef undef[] = { + LLVM_CONST(i32x4_undef), + LLVM_CONST(i64x2_undef), + }; + uint32 mask_length[] = { 4, 2 }; + LLVMValueRef mask_element[][4] = { + { LLVM_CONST(i32_zero), LLVM_CONST(i32_four), LLVM_CONST(i32_five), + LLVM_CONST(i32_six) }, + { LLVM_CONST(i32_zero), LLVM_CONST(i32_two) }, + }; + bool enable_segue = comp_ctx->enable_segue_v128_load; + + bh_assert(opcode_index < 2); + + if (!(element = simd_load( + comp_ctx, func_ctx, align, offset, data_lengths[opcode_index], + comp_ctx->enable_segue_v128_load + ? element_ptr_types_gs[opcode_index] + : element_ptr_types[opcode_index], + element_data_types[opcode_index], enable_segue))) { + return false; + } + + if (!(result = + LLVMBuildInsertElement(comp_ctx->builder, undef[opcode_index], + element, I32_ZERO, "vector"))) { + HANDLE_FAILURE("LLVMBuildInsertElement"); + return false; + } + + /* fill in other lanes with zero */ + if (!(mask = LLVMConstVector(mask_element[opcode_index], + mask_length[opcode_index]))) { + HANDLE_FAILURE("LLConstVector"); + return false; + } + + if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, result, + zero[opcode_index], mask, + "fill_in_zero"))) { + HANDLE_FAILURE("LLVMBuildShuffleVector"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +/* data_length in bytes */ +static bool +simd_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 align, + uint32 offset, uint32 data_length, LLVMValueRef value, + LLVMTypeRef value_ptr_type, bool enable_segue) +{ + LLVMValueRef maddr, result; + + if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, + data_length, enable_segue))) + return false; + + if (!(maddr = LLVMBuildBitCast(comp_ctx->builder, maddr, value_ptr_type, + "data_ptr"))) { + HANDLE_FAILURE("LLVMBuildBitCast"); + return false; + } + + if (!(result = LLVMBuildStore(comp_ctx->builder, value, maddr))) { + HANDLE_FAILURE("LLVMBuildStore"); + return false; + } + + LLVMSetAlignment(result, 1); + + return true; +} + +bool +aot_compile_simd_v128_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset) +{ + bool enable_segue = comp_ctx->enable_segue_v128_store; + LLVMTypeRef v128_ptr_type = enable_segue ? V128_PTR_TYPE_GS : V128_PTR_TYPE; + LLVMValueRef value; + + POP_V128(value); + + return simd_store(comp_ctx, func_ctx, align, offset, 16, value, + v128_ptr_type, enable_segue); +fail: + return false; +} + +bool +aot_compile_simd_store_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset, + uint8 lane_id) +{ + LLVMValueRef element, vector; + uint32 data_lengths[] = { 1, 2, 4, 8 }; + LLVMTypeRef element_ptr_types[] = { INT8_PTR_TYPE, INT16_PTR_TYPE, + INT32_PTR_TYPE, INT64_PTR_TYPE }; + LLVMTypeRef element_ptr_types_gs[] = { INT8_PTR_TYPE_GS, INT16_PTR_TYPE_GS, + INT32_PTR_TYPE_GS, + INT64_PTR_TYPE_GS }; + uint32 opcode_index = opcode - SIMD_v128_store8_lane; + LLVMTypeRef vector_types[] = { V128_i8x16_TYPE, V128_i16x8_TYPE, + V128_i32x4_TYPE, V128_i64x2_TYPE }; + LLVMValueRef lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id); + bool enable_segue = comp_ctx->enable_segue_v128_store; + + bh_assert(opcode_index < 4); + + if (!(vector = simd_pop_v128_and_bitcast( + comp_ctx, func_ctx, vector_types[opcode_index], "src"))) { + return false; + } + + if (!(element = LLVMBuildExtractElement(comp_ctx->builder, vector, lane, + "element"))) { + HANDLE_FAILURE("LLVMBuildExtractElement"); + return false; + } + + return simd_store(comp_ctx, func_ctx, align, offset, + data_lengths[opcode_index], element, + enable_segue ? element_ptr_types_gs[opcode_index] + : element_ptr_types[opcode_index], + enable_segue); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_load_store.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_load_store.h new file mode 100644 index 0000000..fd118ec --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_load_store.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_LOAD_STORE_H_ +#define _SIMD_LOAD_STORE_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_v128_load(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset); + +bool +aot_compile_simd_load_extend(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset); + +bool +aot_compile_simd_load_splat(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset); + +bool +aot_compile_simd_load_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset, + uint8 lane_id); + +bool +aot_compile_simd_load_zero(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset); + +bool +aot_compile_simd_v128_store(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 align, uint32 offset); + +bool +aot_compile_simd_store_lane(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 opcode, uint32 align, uint32 offset, + uint8 lane_id); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_LOAD_STORE_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_sat_int_arith.c b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_sat_int_arith.c new file mode 100644 index 0000000..ea250b7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_sat_int_arith.c @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "simd_sat_int_arith.h" +#include "simd_common.h" +#include "../aot_emit_exception.h" +#include "../../aot/aot_runtime.h" + +static bool +simd_sat_int_arith(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMTypeRef vector_type, const char *intrinsics) +{ + LLVMValueRef lhs, rhs, result; + LLVMTypeRef param_types[2]; + + if (!(rhs = + simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs")) + || !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, + "lhs"))) { + return false; + } + + param_types[0] = vector_type; + param_types[1] = vector_type; + + if (!(result = + aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsics, + vector_type, param_types, 2, lhs, rhs))) { + HANDLE_FAILURE("LLVMBuildCall"); + return false; + } + + return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result"); +} + +bool +aot_compile_simd_i8x16_saturate(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed) +{ + char *intrinsics[][2] = { + { "llvm.sadd.sat.v16i8", "llvm.uadd.sat.v16i8" }, + { "llvm.ssub.sat.v16i8", "llvm.usub.sat.v16i8" }, + }; + + return simd_sat_int_arith(comp_ctx, func_ctx, V128_i8x16_TYPE, + is_signed ? intrinsics[arith_op][0] + : intrinsics[arith_op][1]); +} + +bool +aot_compile_simd_i16x8_saturate(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed) +{ + char *intrinsics[][2] = { + { "llvm.sadd.sat.v8i16", "llvm.uadd.sat.v8i16" }, + { "llvm.ssub.sat.v8i16", "llvm.usub.sat.v8i16" }, + }; + + return simd_sat_int_arith(comp_ctx, func_ctx, V128_i16x8_TYPE, + is_signed ? intrinsics[arith_op][0] + : intrinsics[arith_op][1]); +} diff --git a/wasm-micro-runtime/core/iwasm/compilation/simd/simd_sat_int_arith.h b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_sat_int_arith.h new file mode 100644 index 0000000..67c602f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/compilation/simd/simd_sat_int_arith.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SIMD_SAT_INT_ARITH_H_ +#define _SIMD_SAT_INT_ARITH_H_ + +#include "../aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_compile_simd_i8x16_saturate(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed); + +bool +aot_compile_simd_i16x8_saturate(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + V128Arithmetic arith_op, bool is_signed); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _SIMD_SAT_INT_ARITH_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/doc/classic_interpreter.MD b/wasm-micro-runtime/core/iwasm/doc/classic_interpreter.MD new file mode 100644 index 0000000..a607758 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/classic_interpreter.MD @@ -0,0 +1,5 @@ +# Classic interpreter + +## stack format + +![](./images/stack_format_ci.svg) \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/export_function.excalidraw b/wasm-micro-runtime/core/iwasm/doc/images/export_function.excalidraw new file mode 100644 index 0000000..b983af0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/export_function.excalidraw @@ -0,0 +1,5695 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "elements": [ + { + "type": "rectangle", + "version": 469, + "versionNonce": 587617691, + "isDeleted": false, + "id": "YQFdEhDm9LI_5UD2YjLU-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 238.99996948242188, + "y": 207.16673278808577, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 221.3333740234375, + "height": 94.629648844401, + "seed": 1964509979, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 318, + "versionNonce": 929539477, + "isDeleted": false, + "id": "pDkkkqvgrizIP6AYdFbkj", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 239.66665649414062, + "y": 161.83335876464827, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 96, + "height": 19, + "seed": 278267925, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMModule", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule" + }, + { + "type": "rectangle", + "version": 592, + "versionNonce": 276752955, + "isDeleted": false, + "id": "awCpIZpFN-gQRBgh6iG1X", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 228.33334350585938, + "y": 191.50007629394514, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 247.11109754774304, + "height": 265.6667175292969, + "seed": 1908273083, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 411, + "versionNonce": 1793600245, + "isDeleted": false, + "id": "TOmX9MwwNOgbfjNJ8V8j7", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 239.4444953070747, + "y": 233.9815283881292, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 211, + "height": 56, + "seed": 86392181, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "2NQmRBF_NE2Myp3-jqLfT", + "type": "arrow" + } + ], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": " WASMTable *tables;\n WASMMemory *memories;\n WASMGlobal *globals;", + "baseline": 52, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": " WASMTable *tables;\n WASMMemory *memories;\n WASMGlobal *globals;" + }, + { + "type": "text", + "version": 390, + "versionNonce": 1283272731, + "isDeleted": false, + "id": "FPUGB-iP7ep91gfoTuV2d", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 289.3334045410156, + "y": 374.1667327880857, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 165, + "height": 19, + "seed": 504570933, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "OdDm5a5O_NIoZbF3u0c0H", + "type": "arrow" + } + ], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMExport *exports;", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMExport *exports;" + }, + { + "type": "rectangle", + "version": 371, + "versionNonce": 692171541, + "isDeleted": false, + "id": "iP-OL8X-L4CE8z9CTp9qG", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 243.66677856445312, + "y": 359.5001068115232, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 216, + "height": 42.6666259765625, + "seed": 1531454875, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 336, + "versionNonce": 1261860027, + "isDeleted": false, + "id": "jkgMB1VHPK6x-xD6Wuey7", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 258.3335266113281, + "y": 370.16679382324196, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 15.33331298828125, + "height": 22.66668701171875, + "seed": 2028656021, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 286, + "versionNonce": 1775181787, + "isDeleted": false, + "id": "v4paccURicZAp-WeQNnlQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 585.7036675347218, + "y": 278.944342719184, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 156, + "height": 128.33334350585938, + "seed": 766182741, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 176, + "versionNonce": 907517781, + "isDeleted": false, + "id": "blxQLl_yf7DMD76qHC5rc", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 587.9629041883677, + "y": 256.3147176106771, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 97, + "height": 19, + "seed": 1409110677, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "OdDm5a5O_NIoZbF3u0c0H", + "type": "arrow" + } + ], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMExport", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMExport" + }, + { + "type": "text", + "version": 216, + "versionNonce": 271871099, + "isDeleted": false, + "id": "xQPYBI_XdfyZkh2-U_YQB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 617.3703240288627, + "y": 289.2776862250434, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 88, + "height": 19, + "seed": 1776880725, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "YidAloK-3ikBBzvuu-S22", + "type": "arrow" + } + ], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "char *name;", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "char *name;" + }, + { + "type": "text", + "version": 241, + "versionNonce": 217246901, + "isDeleted": false, + "id": "lB_sRHE0gMYdFdpT2Dvja", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 614.7036370171439, + "y": 321.6110602484809, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 75, + "height": 19, + "seed": 1312575355, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "uint8 kind;", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "uint8 kind;" + }, + { + "type": "text", + "version": 230, + "versionNonce": 562504987, + "isDeleted": false, + "id": "SQI7khDAbJL0pLh0deiej", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 610.0369805230033, + "y": 354.944342719184, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 93, + "height": 19, + "seed": 2114894261, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "C_HvFqwDiW4wGe01QNKFg", + "type": "arrow" + } + ], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "uint32 index;", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "uint32 index;" + }, + { + "type": "rectangle", + "version": 188, + "versionNonce": 2042465813, + "isDeleted": false, + "id": "sqBZGoR4vKErEsSE7jSJk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 601.0370110405812, + "y": 284.9443579779731, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 120.66668701171875, + "height": 29.666671752929688, + "seed": 1817827573, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "YidAloK-3ikBBzvuu-S22", + "type": "arrow" + } + ], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 255, + "versionNonce": 892979643, + "isDeleted": false, + "id": "xy3rvXCHxwYSjuQZAaa0Y", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 601.7036675347218, + "y": 320.6110602484809, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 118, + "height": 26.33331298828125, + "seed": 1394813013, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 190, + "versionNonce": 728564597, + "isDeleted": false, + "id": "noIblgsWe1zKM-bENF8Ev", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 599.0370110405812, + "y": 351.61102973090277, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 126, + "height": 29.666656494140625, + "seed": 410891355, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 174, + "versionNonce": 1447480533, + "isDeleted": false, + "id": "q8XZjMjkc5oaR7KNqOcGF", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 654.3703545464408, + "y": 385.7777167426215, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 20, + "height": 19, + "seed": 1931744507, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "[0]", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "rectangle", + "version": 177, + "versionNonce": 804837115, + "isDeleted": false, + "id": "cywWwhg521rh-6jpDBbTE", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 586.3703240288625, + "y": 407.94437323676215, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 158, + "height": 42, + "seed": 19921979, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "8dd-iKzlb9Z4V1eqnx9a-" + } + ], + "updated": 1679555887720, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 152, + "versionNonce": 165702197, + "isDeleted": false, + "id": "8dd-iKzlb9Z4V1eqnx9a-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 655.3703240288627, + "y": 419.44437323676215, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 20, + "height": 19, + "seed": 1593993467, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887720, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "[1]", + "baseline": 15, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "cywWwhg521rh-6jpDBbTE", + "originalText": "[1]" + }, + { + "type": "rectangle", + "version": 191, + "versionNonce": 1381646235, + "isDeleted": false, + "id": "SsP3zaE3LuSyRa2Ma4ffB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 587.3703240288625, + "y": 450.94437323676215, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 158, + "height": 42, + "seed": 1216947029, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "Ux1Agae75_DS0veGHXMuH" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 169, + "versionNonce": 1193237397, + "isDeleted": false, + "id": "Ux1Agae75_DS0veGHXMuH", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 654.3703240288627, + "y": 462.44437323676215, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 24, + "height": 19, + "seed": 982329467, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "[...]", + "baseline": 15, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "SsP3zaE3LuSyRa2Ma4ffB", + "originalText": "[...]" + }, + { + "type": "text", + "version": 248, + "versionNonce": 1555249749, + "isDeleted": false, + "id": "o7FaW1SjzOKpt86xZNbly", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -71.58126068115234, + "y": 146.11661834716801, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 171, + "height": 19, + "seed": 1768482683, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMModuleInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance" + }, + { + "type": "rectangle", + "version": 131, + "versionNonce": 1297489444, + "isDeleted": false, + "id": "MC9rQuxIk7iVRtsas7ICs", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -73.99992370605469, + "y": 180.9666107177734, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 210.66668701171875, + "height": 314.33334350585943, + "seed": 1805099445, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558426546, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 473, + "versionNonce": 1484728732, + "isDeleted": false, + "id": "6pGqkyRY8AHL4k5LTJ0Yl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -51.000274658203125, + "y": 425.4667785644531, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174.66668701171875, + "height": 31.666625976562507, + "seed": 957429083, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558303924, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 499, + "versionNonce": 1368837668, + "isDeleted": false, + "id": "cDohxEkx3HhhoX-zcCc9Q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -45.666961669921875, + "y": 433.1334350585937, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 63676885, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558303924, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 208, + "versionNonce": 1386190364, + "isDeleted": false, + "id": "kOP_SYX2hDGyTbKOLrSY7", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -25.66693115234375, + "y": 429.9667785644531, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 115, + "height": 20, + "seed": 697815547, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "xGVX08X1n0JU0OA60_t9A", + "type": "arrow" + } + ], + "updated": 1679558303924, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "export_tables", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "export_tables" + }, + { + "type": "rectangle", + "version": 534, + "versionNonce": 1960518965, + "isDeleted": false, + "id": "FnISIa4MPRb3okZ9nUHSQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -58.66651916503906, + "y": 359.8333740234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174.66668701171875, + "height": 31.666625976562507, + "seed": 1723426171, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 562, + "versionNonce": 901559451, + "isDeleted": false, + "id": "KPco9Nm_8Olmg0Uq3NVQ1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -53.33320617675781, + "y": 367.5000305175781, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 135660469, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 260, + "versionNonce": 420132501, + "isDeleted": false, + "id": "8KE83CX20gDJQwJueCKwN", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -39.33323669433594, + "y": 365.66668701171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 131, + "height": 20, + "seed": 788856347, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "v2uiV8UbBxa6_yEOLAehf", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "export_memories", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "export_memories" + }, + { + "type": "rectangle", + "version": 477, + "versionNonce": 1527867707, + "isDeleted": false, + "id": "_CbJcjswexYEsNuWct5mC", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -51.99998474121094, + "y": 189.83334350585938, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174.66668701171875, + "height": 31.666625976562507, + "seed": 1344002453, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 505, + "versionNonce": 671161333, + "isDeleted": false, + "id": "hGw4Pp4LnIpkfSfU4PeRb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -46.66667175292969, + "y": 197.5, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 861402683, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 213, + "versionNonce": 690391515, + "isDeleted": false, + "id": "67MrbK1oKDTBrmUzIQlr1", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -26.666641235351562, + "y": 194.33334350585938, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 118, + "height": 20, + "seed": 1190518517, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "export_globals", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "export_globals" + }, + { + "type": "rectangle", + "version": 400, + "versionNonce": 1237570901, + "isDeleted": false, + "id": "eMcaE0yc0BCbsQ4MrLuGP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -51.33323669433594, + "y": 271.8333740234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174.66668701171875, + "height": 31.666625976562507, + "seed": 1337283029, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 429, + "versionNonce": 84842107, + "isDeleted": false, + "id": "rW76UbdBzelN0CWuqQjTc", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -45.99992370605469, + "y": 279.5000305175781, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 778117627, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "4eMPasZehGc58H7vVusCf", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 136, + "versionNonce": 162030261, + "isDeleted": false, + "id": "NfPonNQyhbRCCPh50Ed7I", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -25.999893188476562, + "y": 276.3333740234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 136, + "height": 20, + "seed": 1026969397, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "4eMPasZehGc58H7vVusCf", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "export_functions", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "export_functions" + }, + { + "type": "text", + "version": 121, + "versionNonce": 227282715, + "isDeleted": false, + "id": "ILDnCNxtBv4qdrhMb3QoZ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -365.33335876464844, + "y": 193.1666259765625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 193, + "height": 19, + "seed": 282525979, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMExportFuncInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMExportFuncInstance" + }, + { + "type": "rectangle", + "version": 201, + "versionNonce": 1817461781, + "isDeleted": false, + "id": "2agHBzF-91Fb2ws0hYObd", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -354.33335876464844, + "y": 223.83334350585938, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 177.33331298828122, + "height": 121.00006103515625, + "seed": 1290367509, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "4eMPasZehGc58H7vVusCf", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 176, + "versionNonce": 1961256348, + "isDeleted": false, + "id": "aZL7IU5LEszDJmyyWcmtp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -340.6666717529297, + "y": 239.83328247070312, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 117, + "height": 31, + "seed": 1714300347, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "P3_9LebxO4r1Nud8xISGk" + }, + { + "id": "YidAloK-3ikBBzvuu-S22", + "type": "arrow" + } + ], + "updated": 1679558297714, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 194, + "versionNonce": 2136035876, + "isDeleted": false, + "id": "P3_9LebxO4r1Nud8xISGk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -334.6666717529297, + "y": 244.83328247070312, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 105, + "height": 21, + "seed": 2004641653, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297727, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "char* name ", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "aZL7IU5LEszDJmyyWcmtp", + "originalText": "char* name " + }, + { + "type": "diamond", + "version": 159, + "versionNonce": 1002521691, + "isDeleted": false, + "id": "qdga26-o2AGoXkwH2yP4N", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -323.00001525878906, + "y": 284.1666259765625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 14.666656494140625, + "height": 17.666656494140625, + "seed": 444053083, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 216, + "versionNonce": 1328022229, + "isDeleted": false, + "id": "2onXRIpW3Znosk6O46QtY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -338.33335876464844, + "y": 276.49993896484375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 128, + "height": 38, + "seed": 852078805, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "Zrwh0TsrNxBSNhpJZIqbu" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 183, + "versionNonce": 1455378972, + "isDeleted": false, + "id": "Zrwh0TsrNxBSNhpJZIqbu", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -305.83335876464844, + "y": 285.49993896484375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63, + "height": 21, + "seed": 2089169659, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297728, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "function", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "2onXRIpW3Znosk6O46QtY", + "originalText": "function" + }, + { + "type": "arrow", + "version": 505, + "versionNonce": 2099961909, + "isDeleted": false, + "id": "nG4vs8WttuFFZPXDKIdNr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -320.33335876464844, + "y": 293.49993896484375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 111.00001525878906, + "height": 184.2470495648475, + "seed": 476499509, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "7JcYtZ-KQ0WF1SDFDwvcV", + "gap": 1.2531640581994319, + "focus": -0.8059388021902064 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -111.00001525878906, + 33.166717529296875 + ], + [ + -35.26115412319115, + 184.2470495648475 + ] + ] + }, + { + "type": "text", + "version": 531, + "versionNonce": 363532693, + "isDeleted": false, + "id": "7MNv-pmgV4-8yJ5lIFY1U", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 496.5186225043402, + "y": 71.31480577256946, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 235, + "height": 149, + "seed": 2074251419, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "YidAloK-3ikBBzvuu-S22", + "type": "arrow" + }, + { + "id": "2NQmRBF_NE2Myp3-jqLfT", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "name: shown to external\nindex: refer to item idx for export \n in current kind\nkind: total 4 export kinds:\n- function\n- globals\n- memory\n- table", + "baseline": 145, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "name: shown to external\nindex: refer to item idx for export \n in current kind\nkind: total 4 export kinds:\n- function\n- globals\n- memory\n- table" + }, + { + "type": "rectangle", + "version": 81, + "versionNonce": 997735995, + "isDeleted": false, + "id": "orpW0tFerpELyreVDruzO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -353.3333740234375, + "y": 343.99998474121094, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 940489147, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "KlcGzdua5BpLa_-0RGeFO" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 53, + "versionNonce": 1965780388, + "isDeleted": false, + "id": "KlcGzdua5BpLa_-0RGeFO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -274.3333740234375, + "y": 350.1666564941406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 22, + "height": 21, + "seed": 1399452699, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297729, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[1]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "orpW0tFerpELyreVDruzO", + "originalText": "[1]" + }, + { + "type": "text", + "version": 71, + "versionNonce": 1609219803, + "isDeleted": false, + "id": "da0XYl9R6gkV-BQ4X1JdF", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -278.166748046875, + "y": 324.33338928222656, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 20, + "seed": 848835675, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[0]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "rectangle", + "version": 109, + "versionNonce": 528755579, + "isDeleted": false, + "id": "oVhH9XqC6rs2uaPMH4x2B", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -353.0001220703125, + "y": 377.50006103515625, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 1099143605, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "wu6BDVXXPWLo-Z-l7TYGW" + }, + { + "id": "Eda9W8eaZom6PATRAm5EX", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 83, + "versionNonce": 1472638620, + "isDeleted": false, + "id": "wu6BDVXXPWLo-Z-l7TYGW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -278.5001220703125, + "y": 383.66673278808594, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 31, + "height": 21, + "seed": 1029949467, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297730, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[...]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "oVhH9XqC6rs2uaPMH4x2B", + "originalText": "[...]" + }, + { + "type": "rectangle", + "version": 134, + "versionNonce": 1882593572, + "isDeleted": false, + "id": "t8xOhAVyJcx51xJsmWC6j", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -352.33331298828125, + "y": 408.5001220703125, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 31, + "seed": 883564757, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "F09zazRYIO_G3oM7DmBbr" + }, + { + "id": "Eda9W8eaZom6PATRAm5EX", + "type": "arrow" + } + ], + "updated": 1679558297730, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 111, + "versionNonce": 1756265244, + "isDeleted": false, + "id": "F09zazRYIO_G3oM7DmBbr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -274.83331298828125, + "y": 418.5001220703125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 25, + "height": 21, + "seed": 1529135867, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297738, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[n]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "t8xOhAVyJcx51xJsmWC6j", + "originalText": "[n]" + }, + { + "type": "line", + "version": 284, + "versionNonce": 350603451, + "isDeleted": false, + "id": "XphCtXmoQONIDWT15UPmf", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 169.00015258789057, + "y": 21.389623853895444, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 1.66656494140625, + "height": 838.3330154418943, + "seed": 395797243, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + 1.66656494140625, + 838.3330154418943 + ] + ] + }, + { + "type": "arrow", + "version": 156, + "versionNonce": 1808036981, + "isDeleted": false, + "id": "4eMPasZehGc58H7vVusCf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -40.66667175292969, + "y": 278.5517677558588, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 133.01236173861167, + "height": 54.587889349881976, + "seed": 526956341, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "startBinding": { + "elementId": "NfPonNQyhbRCCPh50Ed7I", + "focus": -0.6897037086038879, + "gap": 14.666778564453125 + }, + "endBinding": { + "elementId": "2agHBzF-91Fb2ws0hYObd", + "focus": -1.0127197558064842, + "gap": 3.3210122848258408 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -133.01236173861167, + -54.587889349881976 + ] + ] + }, + { + "type": "rectangle", + "version": 227, + "versionNonce": 2017655131, + "isDeleted": false, + "id": "7JcYtZ-KQ0WF1SDFDwvcV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -362, + "y": 478.3334655761719, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 223.3334655761719, + "height": 33.00015258789055, + "seed": 857261429, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "nG4vs8WttuFFZPXDKIdNr", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 351, + "versionNonce": 1552424405, + "isDeleted": false, + "id": "QLOWRmBSKHUu_NrX66JPt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -355.3333740234375, + "y": 486.00006103515625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 208, + "height": 20, + "seed": 513338651, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "nG4vs8WttuFFZPXDKIdNr", + "type": "arrow" + }, + { + "id": "Vhva2LNBhtrohj4Y3bGV9", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(refer to function diagam)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(refer to function diagam)" + }, + { + "type": "text", + "version": 94, + "versionNonce": 636556795, + "isDeleted": false, + "id": "R3bNKi3-D-UlbcoafRHR5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -40.6666259765625, + "y": 316.9998779296875, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 154, + "height": 20, + "seed": 1038062939, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "Eda9W8eaZom6PATRAm5EX", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "export_func_count", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "export_func_count" + }, + { + "type": "arrow", + "version": 162, + "versionNonce": 1037015861, + "isDeleted": false, + "id": "Eda9W8eaZom6PATRAm5EX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -51.354154838890395, + "y": 333.9999084472656, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 192.10502044691845, + "height": 89.91317165306509, + "seed": 518031893, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "startBinding": { + "elementId": "mZobbVjOOfzbAK_1alOcK", + "focus": 0.6974503894293331, + "gap": 1 + }, + "endBinding": { + "elementId": "oVhH9XqC6rs2uaPMH4x2B", + "focus": 0.8116332021918504, + "gap": 14.079675559315092 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -81.97918866696898, + 54.666717529296875 + ], + [ + -192.10502044691845, + 89.91317165306509 + ] + ] + }, + { + "type": "rectangle", + "version": 71, + "versionNonce": 2013293717, + "isDeleted": false, + "id": "mZobbVjOOfzbAK_1alOcK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -50.66668701171875, + "y": 311.333251953125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 172.666748046875, + "height": 31.66668701171875, + "seed": 1172380085, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "Eda9W8eaZom6PATRAm5EX", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 417, + "versionNonce": 1255150069, + "isDeleted": false, + "id": "YyGWHpfM0OE4uzGAFwZ12", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 484.1481289333766, + "y": 65.53704833984384, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 287.70376925998266, + "height": 162.29637824164502, + "seed": 274906523, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "YidAloK-3ikBBzvuu-S22", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 1048, + "versionNonce": 392819675, + "isDeleted": false, + "id": "YidAloK-3ikBBzvuu-S22", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 743.579402430669, + "y": 225.04108862166348, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 21.48420572561895, + "height": 53.29342358325337, + "seed": 1380550619, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "startBinding": { + "elementId": "7MNv-pmgV4-8yJ5lIFY1U", + "focus": -0.6782971803849929, + "gap": 12.953770184814061 + }, + "endBinding": { + "elementId": "sqBZGoR4vKErEsSE7jSJk", + "focus": 0.7448990456524555, + "gap": 10.515841746169144 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 10.124343093419043, + 26.347898601643806 + ], + [ + -11.359862632199906, + 53.29342358325337 + ] + ] + }, + { + "type": "text", + "version": 264, + "versionNonce": 1665403733, + "isDeleted": false, + "id": "sRzZXOvjblsPsAM89sUXa", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -705.3333892822266, + "y": 113.58320617675781, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 192, + "height": 19, + "seed": 700308213, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMExportGlobInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMExportGlobInstance" + }, + { + "type": "rectangle", + "version": 348, + "versionNonce": 1800716411, + "isDeleted": false, + "id": "xWPtNuyAqazMMUyouKnhf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -694.3333892822266, + "y": 144.2499237060547, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 177.33331298828122, + "height": 121.00006103515625, + "seed": 1037646555, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "RxhU0Qr-hmqzklRZdxsvn", + "type": "arrow" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 323, + "versionNonce": 467621028, + "isDeleted": false, + "id": "7uYVutqCDMjhS1LtgrSpl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -680.6667022705078, + "y": 160.24986267089844, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 117, + "height": 31, + "seed": 328250453, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "PlRFeDiD1tywOAk_rJgHg" + } + ], + "updated": 1679558297741, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 337, + "versionNonce": 334018460, + "isDeleted": false, + "id": "PlRFeDiD1tywOAk_rJgHg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -674.6667022705078, + "y": 165.24986267089844, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 105, + "height": 21, + "seed": 843261819, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297749, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "char* name ", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "7uYVutqCDMjhS1LtgrSpl", + "originalText": "char* name " + }, + { + "type": "diamond", + "version": 301, + "versionNonce": 2143222293, + "isDeleted": false, + "id": "01IysLeEkWSJyxI9NbVb8", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -663.0000457763672, + "y": 204.5832061767578, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 14.666656494140625, + "height": 17.666656494140625, + "seed": 1686200757, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 360, + "versionNonce": 458563003, + "isDeleted": false, + "id": "lWjh1xjt4eB1XKwJJx59D", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -678.3333892822266, + "y": 196.91651916503906, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 128, + "height": 38, + "seed": 671514651, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "VQ0ymOLfMpjFiupR01ZFb" + } + ], + "updated": 1679555887721, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 327, + "versionNonce": 97853476, + "isDeleted": false, + "id": "VQ0ymOLfMpjFiupR01ZFb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -637.3333892822266, + "y": 205.91651916503906, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 46, + "height": 21, + "seed": 565535509, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297750, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "global", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "lWjh1xjt4eB1XKwJJx59D", + "originalText": "global" + }, + { + "type": "arrow", + "version": 624, + "versionNonce": 2147462747, + "isDeleted": false, + "id": "E_RGDiJbGUf1aNH6jDnvz", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -660.3333892822266, + "y": 213.91651916503906, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 111.00001525878906, + "height": 176.67764729852638, + "seed": 152421563, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "KWKN5CpCB6zVtYZd6txj9", + "focus": -0.8240578974308156, + "gap": 7.457318223231596 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -111.00001525878906, + 33.166717529296875 + ], + [ + -17.790676987880033, + 176.67764729852638 + ] + ] + }, + { + "type": "rectangle", + "version": 225, + "versionNonce": 1372149301, + "isDeleted": false, + "id": "uLMWTXI90CGAIKI4zHJML", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -693.3334045410156, + "y": 264.41656494140625, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 1208836565, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "AGXBiW0dSLHn3nGEzduCY" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 196, + "versionNonce": 1168598044, + "isDeleted": false, + "id": "AGXBiW0dSLHn3nGEzduCY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -614.3334045410156, + "y": 270.58323669433594, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 22, + "height": 21, + "seed": 324145659, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297750, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[1]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "uLMWTXI90CGAIKI4zHJML", + "originalText": "[1]" + }, + { + "type": "text", + "version": 213, + "versionNonce": 2064304021, + "isDeleted": false, + "id": "QvT_bIR-th1HBXIOu8j7Y", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -618.1667785644531, + "y": 244.74996948242188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 20, + "seed": 269246261, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[0]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "rectangle", + "version": 253, + "versionNonce": 630646843, + "isDeleted": false, + "id": "zom5ZNn_gBWzVI1PMMYQo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -693.0001525878906, + "y": 297.91664123535156, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 215227035, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "U2e2lWVjWzkRt9x6FkfeD" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 226, + "versionNonce": 1443199908, + "isDeleted": false, + "id": "U2e2lWVjWzkRt9x6FkfeD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -618.5001525878906, + "y": 304.08331298828125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 31, + "height": 21, + "seed": 1845801109, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297751, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[...]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "zom5ZNn_gBWzVI1PMMYQo", + "originalText": "[...]" + }, + { + "type": "rectangle", + "version": 278, + "versionNonce": 333217948, + "isDeleted": false, + "id": "IbfRch-fWorg9yld9Wih5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -692.3333435058594, + "y": 328.9167022705078, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 31, + "seed": 1716260667, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "So5vnnPiQUj6m6ZH6HSMP" + } + ], + "updated": 1679558297752, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 254, + "versionNonce": 111044388, + "isDeleted": false, + "id": "So5vnnPiQUj6m6ZH6HSMP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -614.8333435058594, + "y": 338.9167022705078, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 25, + "height": 21, + "seed": 2141967861, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297758, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[n]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "IbfRch-fWorg9yld9Wih5", + "originalText": "[n]" + }, + { + "type": "rectangle", + "version": 257, + "versionNonce": 494949755, + "isDeleted": false, + "id": "KWKN5CpCB6zVtYZd6txj9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -670.666748046875, + "y": 392.7500457763672, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 196.00015258789062, + "height": 39.33346557617187, + "seed": 35418075, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "E_RGDiJbGUf1aNH6jDnvz", + "type": "arrow" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 415, + "versionNonce": 964039605, + "isDeleted": false, + "id": "uEz53YqVhDB8JRuE5hMcU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -659.3334655761719, + "y": 402.41664123535156, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 164, + "height": 20, + "seed": 268914517, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMGlobalInstance", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMGlobalInstance" + }, + { + "type": "text", + "version": 236, + "versionNonce": 2005417979, + "isDeleted": false, + "id": "FiGf5f4dzHNrO5L3NMiwT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -604.5001068115234, + "y": 504.4166564941406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 193, + "height": 19, + "seed": 1644818613, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "v2uiV8UbBxa6_yEOLAehf", + "type": "arrow" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMExportMemInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMExportMemInstance" + }, + { + "type": "rectangle", + "version": 316, + "versionNonce": 1410863413, + "isDeleted": false, + "id": "tgz_9er4fEh_TbaYDZF6u", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -593.5001068115234, + "y": 535.0833740234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 177.33331298828122, + "height": 121.00006103515625, + "seed": 273834267, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 295, + "versionNonce": 104467740, + "isDeleted": false, + "id": "fZ94UaMm9ypc37Hwvn9SX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -579.8334197998047, + "y": 551.0833129882812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 117, + "height": 31, + "seed": 492048917, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "QBs13zYnt2LZX4M9cdFuH" + } + ], + "updated": 1679558297759, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 308, + "versionNonce": 470672036, + "isDeleted": false, + "id": "QBs13zYnt2LZX4M9cdFuH", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -573.8334197998047, + "y": 556.0833129882812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 105, + "height": 21, + "seed": 524125627, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297764, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "char* name ", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "fZ94UaMm9ypc37Hwvn9SX", + "originalText": "char* name " + }, + { + "type": "diamond", + "version": 271, + "versionNonce": 424590651, + "isDeleted": false, + "id": "nqJwiZ18KG3vCMfANM8aQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -562.1667633056641, + "y": 595.4166564941406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 14.666656494140625, + "height": 17.666656494140625, + "seed": 563283829, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 332, + "versionNonce": 1978861557, + "isDeleted": false, + "id": "HEJvi8QjWRiMWcoZPLHIl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -577.5001068115234, + "y": 587.7499694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 128, + "height": 38, + "seed": 983242331, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "HU6nOicEloKkqYD55hBWR" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 304, + "versionNonce": 1580337564, + "isDeleted": false, + "id": "HU6nOicEloKkqYD55hBWR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -540.5001068115234, + "y": 596.7499694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 54, + "height": 21, + "seed": 554455253, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297765, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "memory", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "HEJvi8QjWRiMWcoZPLHIl", + "originalText": "memory" + }, + { + "type": "arrow", + "version": 546, + "versionNonce": 1519927637, + "isDeleted": false, + "id": "O2ixOO7WSexT4tbgcUDGA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -559.5001068115234, + "y": 604.7499694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 111.00001525878906, + "height": 176.67764729852638, + "seed": 20812539, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "Sn4zW4Qj5F8AgPuajBnRy", + "focus": -0.8240578974308156, + "gap": 7.457318223231596 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -111.00001525878906, + 33.166717529296875 + ], + [ + -17.790676987880033, + 176.67764729852638 + ] + ] + }, + { + "type": "text", + "version": 371, + "versionNonce": 1795615355, + "isDeleted": false, + "id": "XjbCYdp81z8HsAXal748C", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -794.1667785644531, + "y": 589.7500152587891, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 190, + "height": 20, + "seed": 1767206453, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(WASMMemoryInstance*)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(WASMMemoryInstance*)" + }, + { + "type": "rectangle", + "version": 197, + "versionNonce": 1561284277, + "isDeleted": false, + "id": "8-N4VCgO26s4M3_joLoAg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -592.5001220703125, + "y": 655.2500152587891, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 329743259, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "_zHc4H95qdMAwSfBAe4ji" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 167, + "versionNonce": 1560407588, + "isDeleted": false, + "id": "_zHc4H95qdMAwSfBAe4ji", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -513.5001220703125, + "y": 661.4166870117188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 22, + "height": 21, + "seed": 2035539861, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297766, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[1]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "8-N4VCgO26s4M3_joLoAg", + "originalText": "[1]" + }, + { + "type": "text", + "version": 183, + "versionNonce": 201175061, + "isDeleted": false, + "id": "8MK4DV6gTxwniJIipRt0r", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -517.33349609375, + "y": 635.5834197998047, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 20, + "seed": 1577799739, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[0]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "rectangle", + "version": 225, + "versionNonce": 313275, + "isDeleted": false, + "id": "tIAX2kAFoZpVqZ5qUAx0Z", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -592.1668701171875, + "y": 688.7500915527344, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 1463624949, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "2puZDaevwGxRcBgZ-ptc6" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 197, + "versionNonce": 1593210396, + "isDeleted": false, + "id": "2puZDaevwGxRcBgZ-ptc6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -517.6668701171875, + "y": 694.9167633056641, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 31, + "height": 21, + "seed": 575377627, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297767, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[...]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "tIAX2kAFoZpVqZ5qUAx0Z", + "originalText": "[...]" + }, + { + "type": "rectangle", + "version": 250, + "versionNonce": 174623140, + "isDeleted": false, + "id": "N0OFLsAea9yT6OuliaHBO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -591.5000610351562, + "y": 719.7501525878906, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 31, + "seed": 647413333, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "q7ppugt7g7qcpJ5yt5OdE" + } + ], + "updated": 1679558297767, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 225, + "versionNonce": 360272540, + "isDeleted": false, + "id": "q7ppugt7g7qcpJ5yt5OdE", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -514.0000610351562, + "y": 729.7501525878906, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 25, + "height": 21, + "seed": 1106951547, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297771, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[n]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "N0OFLsAea9yT6OuliaHBO", + "originalText": "[n]" + }, + { + "type": "rectangle", + "version": 228, + "versionNonce": 647779579, + "isDeleted": false, + "id": "Sn4zW4Qj5F8AgPuajBnRy", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -569.8334655761719, + "y": 783.58349609375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 196.00015258789062, + "height": 39.33346557617187, + "seed": 2117479349, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "O2ixOO7WSexT4tbgcUDGA", + "type": "arrow" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 386, + "versionNonce": 1606143029, + "isDeleted": false, + "id": "ADaMm1yoqbZuEu8DJV90L", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -558.5001831054688, + "y": 793.2500915527344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 170, + "height": 20, + "seed": 1213945371, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMMemoryInstance", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMMemoryInstance" + }, + { + "type": "arrow", + "version": 134, + "versionNonce": 1991153820, + "isDeleted": false, + "id": "RxhU0Qr-hmqzklRZdxsvn", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -42.66673278808594, + "y": 209.66668701171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 470.2476914752049, + "height": 74.33332824707031, + "seed": 1569902293, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679558440686, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "xWPtNuyAqazMMUyouKnhf", + "focus": -0.7987290948340221, + "gap": 4.085652030654501 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -274, + -74.33332824707031 + ], + [ + -470.2476914752049, + -62.82886586813058 + ] + ] + }, + { + "type": "arrow", + "version": 214, + "versionNonce": 1329495445, + "isDeleted": false, + "id": "v2uiV8UbBxa6_yEOLAehf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -54.615555578359704, + "y": 386.91916605443436, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 362.7179252566011, + "height": 185.08100179224533, + "seed": 1266609179, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "startBinding": { + "elementId": "8KE83CX20gDJQwJueCKwN", + "focus": 1.000586020261827, + "gap": 15.33355712890625 + }, + "endBinding": { + "elementId": "FiGf5f4dzHNrO5L3NMiwT", + "focus": 0.2798025099622999, + "gap": 11.916763305664062 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -132.71792525660123, + 185.08100179224533 + ], + [ + -362.7179252566011, + 148.41425374537033 + ] + ] + }, + { + "type": "text", + "version": 48, + "versionNonce": 304349941, + "isDeleted": false, + "id": "VoO6IQkaaomgHGzvmdk9L", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -107.00004577636719, + "y": 551.3334197998047, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 184, + "height": 19, + "seed": 2114679797, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMExportTabInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMExportTabInstance" + }, + { + "type": "rectangle", + "version": 353, + "versionNonce": 340366043, + "isDeleted": false, + "id": "cfV2B-j5UbmISEWVZNm_5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -108.33344268798828, + "y": 571.0000915527344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 177.33331298828122, + "height": 121.00006103515625, + "seed": 2095295067, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 335, + "versionNonce": 393196836, + "isDeleted": false, + "id": "DJ8H7VUIn868NKonqg5gA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -94.66675567626953, + "y": 587.0000305175781, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 117, + "height": 31, + "seed": 1950809301, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "kVEW0vY1bUiCqI5IpX5ML" + }, + { + "id": "xGVX08X1n0JU0OA60_t9A", + "type": "arrow" + } + ], + "updated": 1679558297772, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 346, + "versionNonce": 543989532, + "isDeleted": false, + "id": "kVEW0vY1bUiCqI5IpX5ML", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -88.66675567626953, + "y": 592.0000305175781, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 105, + "height": 21, + "seed": 357803771, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297776, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "char* name ", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "DJ8H7VUIn868NKonqg5gA", + "originalText": "char* name " + }, + { + "type": "diamond", + "version": 308, + "versionNonce": 930974133, + "isDeleted": false, + "id": "gUolYbJIMeIJ_CDZqZFxE", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -77.0000991821289, + "y": 631.3333740234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 14.666656494140625, + "height": 17.666656494140625, + "seed": 1482332725, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 371, + "versionNonce": 809555995, + "isDeleted": false, + "id": "KIe-ngH8dNcYj_TNLGGa1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -92.33344268798828, + "y": 623.6666870117188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 128, + "height": 38, + "seed": 1675300763, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "NtF7gKcUrgH2P2tIg7Y1I" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 343, + "versionNonce": 1636135076, + "isDeleted": false, + "id": "NtF7gKcUrgH2P2tIg7Y1I", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -49.83344268798828, + "y": 632.6666870117188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 43, + "height": 21, + "seed": 651398037, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297777, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "table", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "KIe-ngH8dNcYj_TNLGGa1", + "originalText": "table" + }, + { + "type": "rectangle", + "version": 236, + "versionNonce": 1212488891, + "isDeleted": false, + "id": "1vaZwH7ZrMW37Kus4v6s8", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -107.33345794677734, + "y": 691.1667327880859, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 179070011, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "EcHCtg-lwToZUzRwQtRqU" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 205, + "versionNonce": 2017614748, + "isDeleted": false, + "id": "EcHCtg-lwToZUzRwQtRqU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -28.333457946777344, + "y": 697.3334045410156, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 22, + "height": 21, + "seed": 266817781, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297777, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[1]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "1vaZwH7ZrMW37Kus4v6s8", + "originalText": "[1]" + }, + { + "type": "text", + "version": 220, + "versionNonce": 748146011, + "isDeleted": false, + "id": "xnv1ofRPmnyf7Y75joBRd", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -32.166831970214844, + "y": 671.5001373291016, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 20, + "seed": 1101669595, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[0]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "rectangle", + "version": 264, + "versionNonce": 1671072213, + "isDeleted": false, + "id": "WhKSAa-6xEaKbFn82X5ru", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -107.00020599365234, + "y": 724.6668090820312, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 32.333343505859375, + "seed": 555444821, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "-gxwjWzmqS5WDjuFaNKFD" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 235, + "versionNonce": 1750580260, + "isDeleted": false, + "id": "-gxwjWzmqS5WDjuFaNKFD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -32.500205993652344, + "y": 730.8334808349609, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 31, + "height": 21, + "seed": 543609211, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297778, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[...]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "WhKSAa-6xEaKbFn82X5ru", + "originalText": "[...]" + }, + { + "type": "rectangle", + "version": 289, + "versionNonce": 925665308, + "isDeleted": false, + "id": "10EyNsT35ULiP_xM9qv8Y", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -106.3333969116211, + "y": 755.6668701171875, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 180, + "height": 31, + "seed": 457529269, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "x2msRpv4tJnWtZ_hp50wB" + } + ], + "updated": 1679558297778, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 263, + "versionNonce": 97818532, + "isDeleted": false, + "id": "x2msRpv4tJnWtZ_hp50wB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -28.833396911621094, + "y": 765.6668701171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 25, + "height": 21, + "seed": 673507867, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297782, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[n]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "10EyNsT35ULiP_xM9qv8Y", + "originalText": "[n]" + }, + { + "type": "text", + "version": 76, + "versionNonce": 375904405, + "isDeleted": false, + "id": "0XlRzavkxOV0TYo3Cru6q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -307.66673278808594, + "y": 651.0002899169922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 171, + "height": 19, + "seed": 1584858171, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "GXMgwVG5yviwCuQz9-Jdx", + "type": "arrow" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "(WASMTableInstance *)", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(WASMTableInstance *)" + }, + { + "type": "arrow", + "version": 29, + "versionNonce": 1192191803, + "isDeleted": false, + "id": "GXMgwVG5yviwCuQz9-Jdx", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -67.33335876464844, + "y": 639.0001068115234, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 69.3333740234375, + "height": 18.333343505859375, + "seed": 1221178005, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "0XlRzavkxOV0TYo3Cru6q", + "focus": 0.6054948422392628, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -69.3333740234375, + 18.333343505859375 + ] + ] + }, + { + "type": "arrow", + "version": 104, + "versionNonce": 1499556260, + "isDeleted": false, + "id": "xGVX08X1n0JU0OA60_t9A", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -41.0000457763672, + "y": 446.7786348431563, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 81.66662597656249, + "height": 127.22147196836715, + "seed": 1672312955, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679558303924, + "link": null, + "locked": false, + "startBinding": { + "elementId": "kOP_SYX2hDGyTbKOLrSY7", + "focus": 1.0138390872785434, + "gap": 15.333114624023438 + }, + "endBinding": { + "elementId": "DJ8H7VUIn868NKonqg5gA", + "focus": -0.9738124716848731, + "gap": 15.333290100097656 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -81.66662597656249, + 95.22147196836715 + ], + [ + -68.99999999999999, + 127.22147196836715 + ] + ] + }, + { + "type": "text", + "version": 42, + "versionNonce": 548718555, + "isDeleted": false, + "id": "HULjVKYZds5RYxESw_kpb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -343.66673278808594, + "y": 454.3334197998047, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 173, + "height": 19, + "seed": 2051995003, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887722, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMFunctionInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunctionInstance" + }, + { + "type": "rectangle", + "version": 116, + "versionNonce": 130053973, + "isDeleted": false, + "id": "kgRAri5TBI6AVeJYwZ0aO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 291.8626718521118, + "y": 615.9665649414064, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 229.3333740234375, + "height": 71.33331298828125, + "seed": 1363010869, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "GHvOda4nqju-pWyZr-_zS" + } + ], + "updated": 1679555887722, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 70, + "versionNonce": 1629299868, + "isDeleted": false, + "id": "GHvOda4nqju-pWyZr-_zS", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 393.02935886383057, + "y": 642.0332214355467, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 27, + "height": 21, + "seed": 1441133723, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679558297783, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[..]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "kgRAri5TBI6AVeJYwZ0aO", + "originalText": "[..]" + }, + { + "type": "text", + "version": 339, + "versionNonce": 1673577653, + "isDeleted": false, + "id": "oxCwJMeoYhRvkKNPtKeha", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 292.13729763031006, + "y": 587.3669616699217, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 211, + "height": 20, + "seed": 1528133269, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "Vhva2LNBhtrohj4Y3bGV9", + "type": "arrow" + }, + { + "id": "tV8skfel8ww2IgWRNSntj", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMFunction (per module)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunction (per module)" + }, + { + "type": "arrow", + "version": 77, + "versionNonce": 1227017499, + "isDeleted": false, + "id": "Vhva2LNBhtrohj4Y3bGV9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -140.00001525878906, + "y": 499.6667022705078, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 435.3333740234375, + "height": 121, + "seed": 975356629, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": { + "elementId": "QLOWRmBSKHUu_NrX66JPt", + "focus": -0.5172589859849054, + "gap": 7.3333587646484375 + }, + "endBinding": { + "elementId": "oxCwJMeoYhRvkKNPtKeha", + "focus": -1.2215352226224219, + "gap": 13.29974060058612 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 230, + 35.33331298828125 + ], + [ + 435.3333740234375, + 121 + ] + ] + }, + { + "type": "rectangle", + "version": 220, + "versionNonce": 1520753525, + "isDeleted": false, + "id": "kby0LOGKuGuYeMzR0L7Pt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 602.3332366943359, + "y": 559.4166564941406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 47414645, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 241, + "versionNonce": 12821717, + "isDeleted": false, + "id": "J-wnUXeWDAfVNvwQHeW06", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 601.9998626708984, + "y": 592.5832824707031, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 785254101, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 226, + "versionNonce": 441810683, + "isDeleted": false, + "id": "qpSAJ6K0S6TvU6i2Uflep", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 625.9998626708984, + "y": 600.2499389648438, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1914322171, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "euj5CRuv6EoS2abVhp14P", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 229, + "versionNonce": 2130854453, + "isDeleted": false, + "id": "yCRds2HRNFQhi2l_NRv1E", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 601.6665496826172, + "y": 625.9166259765625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 24142901, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "tV8skfel8ww2IgWRNSntj", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 238, + "versionNonce": 451012507, + "isDeleted": false, + "id": "XQ_n6PzQRK1pjOIbTF09C", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 600.3331756591797, + "y": 653.5832824707031, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1465836955, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 219, + "versionNonce": 1782985621, + "isDeleted": false, + "id": "dQpthmaEJfLeEUR4UsOsg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 622.9998626708984, + "y": 661.9166259765625, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 2124420501, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 231, + "versionNonce": 1796750395, + "isDeleted": false, + "id": "lRGlqO5nnxQXGumLJpnO9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 624.3332366943359, + "y": 630.8333282470703, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1262844475, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 90, + "versionNonce": 516225269, + "isDeleted": false, + "id": "bCPa9rRVzubIpa31jwl73", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 574.0000152587891, + "y": 532.0000305175781, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 177.33334350585938, + "seed": 294820597, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "6EZFlJGbMoYN4RDYkSEYP", + "type": "arrow" + }, + { + "id": "euj5CRuv6EoS2abVhp14P", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 534, + "versionNonce": 1457192155, + "isDeleted": false, + "id": "C_HvFqwDiW4wGe01QNKFg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 715.7036827935111, + "y": 367.61115180121527, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00552481453781, + "height": 1.0071746818260863, + "seed": 1379035003, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": { + "elementId": "SQI7khDAbJL0pLh0deiej", + "focus": 0.39512687910745115, + "gap": 12.666702270507812 + }, + "endBinding": { + "elementId": "2jRIgxhs5VlnfeDeDJXss", + "focus": 0.24394927767367422, + "gap": 2.171731894901793 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 70.00552481453781, + -1.0071746818260863 + ] + ] + }, + { + "type": "arrow", + "version": 48, + "versionNonce": 163359099, + "isDeleted": false, + "id": "tV8skfel8ww2IgWRNSntj", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 622.6667327880859, + "y": 610.3333892822266, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 105.3333740234375, + "height": 3.666656494140625, + "seed": 1040768149, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": { + "elementId": "yCRds2HRNFQhi2l_NRv1E", + "focus": 1.968489954492859, + "gap": 15.583236694335938 + }, + "endBinding": { + "elementId": "oxCwJMeoYhRvkKNPtKeha", + "focus": 1.5212851912013483, + "gap": 14.196061134338379 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -105.3333740234375, + 3.666656494140625 + ] + ] + }, + { + "type": "rectangle", + "version": 138, + "versionNonce": 2094596021, + "isDeleted": false, + "id": "lbEH3IbUKE-BUvPaOCKOp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 705.7037133110892, + "y": 358.277838812934, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 13.33331298828125, + "height": 11.666671752929688, + "seed": 638517691, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 133, + "versionNonce": 419414555, + "isDeleted": false, + "id": "dTPwE9k9B6K3ksCQBb5OL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 535.0371958414713, + "y": 506.51877678765186, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 176, + "height": 20, + "seed": 121765525, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModule::functions", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule::functions" + }, + { + "type": "rectangle", + "version": 456, + "versionNonce": 1301726907, + "isDeleted": false, + "id": "2J2sR-bPrGrF9OxiUqIF0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 868.2591145833333, + "y": 158.1759851243761, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 1128772501, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "hFcIrFVFFLvSiAWOYE-xZ", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 284, + "versionNonce": 154931515, + "isDeleted": false, + "id": "ZZLxhvBfAsVbp2PY6VOfR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 851.2592061360676, + "y": 129.7593591478136, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 153.00003051757812, + "seed": 1437253909, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "hFcIrFVFFLvSiAWOYE-xZ", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 356, + "versionNonce": 1161098229, + "isDeleted": false, + "id": "FdRdG15cLuW_kygoS9kFB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 862.5927598741318, + "y": 332.1297662523058, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 159, + "height": 20, + "seed": 1076709051, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModule::globals", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule::globals" + }, + { + "type": "rectangle", + "version": 458, + "versionNonce": 196882907, + "isDeleted": false, + "id": "ev_sIxuEomRo18wQcVOAi", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 863.5926411946614, + "y": 195.926084306505, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 1597294293, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "hFcIrFVFFLvSiAWOYE-xZ", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 469, + "versionNonce": 1119661397, + "isDeleted": false, + "id": "2AbM31Y4eYzNJdn6ccO7X", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 870.2593892415364, + "y": 231.25939729478625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 1887403259, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "6EZFlJGbMoYN4RDYkSEYP", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "ellipse", + "version": 193, + "versionNonce": 499530421, + "isDeleted": false, + "id": "2jRIgxhs5VlnfeDeDJXss", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 787.7037285698783, + "y": 359.277838812934, + "strokeColor": "#000000", + "backgroundColor": "#7950f2", + "width": 16, + "height": 19.000091552734375, + "seed": 1552112411, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "id": "C_HvFqwDiW4wGe01QNKFg", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 445, + "versionNonce": 1861688341, + "isDeleted": false, + "id": "hFcIrFVFFLvSiAWOYE-xZ", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 797.8519422743055, + "y": 365.2778930664062, + "strokeColor": "#000000", + "backgroundColor": "#7950f2", + "width": 60.463513970850386, + "height": 155.0348750393585, + "seed": 1134283957, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "ev_sIxuEomRo18wQcVOAi", + "focus": 0.8021785743382612, + "gap": 5.277184949505454 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 26.18546210394959, + -121.94443088107656 + ], + [ + 60.463513970850386, + -155.0348750393585 + ] + ] + }, + { + "type": "rectangle", + "version": 485, + "versionNonce": 1957174203, + "isDeleted": false, + "id": "qGE55eCk3NIsUQpSP-g6Y", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 888.4441562228732, + "y": 390.84254328409827, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 983648475, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 316, + "versionNonce": 507826549, + "isDeleted": false, + "id": "c3AswZE8rY-ZsD8gGAzgg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 871.4442477756076, + "y": 362.42591730753577, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 153.00003051757812, + "seed": 832079445, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 343, + "versionNonce": 1916988507, + "isDeleted": false, + "id": "6YL2S8HdLuD8PaiWGA-vU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 829.5925801595051, + "y": 102.42615678575305, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 156, + "height": 20, + "seed": 920221051, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModule::tables", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule::tables" + }, + { + "type": "rectangle", + "version": 542, + "versionNonce": 2029312725, + "isDeleted": false, + "id": "50mTV_vbZA4UToji5TAhb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 882.0739339192708, + "y": 431.5186000400119, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 1438577589, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "OjoiFuv7ZOtNkq4SpgOL-", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 553, + "versionNonce": 1635428603, + "isDeleted": false, + "id": "Aa3QPT7Ps924J0klBEGMw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 888.7406819661458, + "y": 466.85191302829315, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 547556891, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 424, + "versionNonce": 1473368475, + "isDeleted": false, + "id": "WMOLdrP9c0TFV1JLcmfzK", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 233.66656494140625, + "y": 236.33336639404277, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 17.3333740234375, + "height": 13.000015258789062, + "seed": 1272865237, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 442, + "versionNonce": 606202261, + "isDeleted": false, + "id": "XVo2iBeciuaFiIrP3DUw5", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 235.66656494140625, + "y": 252.66669464111305, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 17.3333740234375, + "height": 13.000015258789062, + "seed": 823202299, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "6EZFlJGbMoYN4RDYkSEYP", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 427, + "versionNonce": 377153083, + "isDeleted": false, + "id": "InatpictpQQa0Op1qByFJ", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 235.66656494140625, + "y": 268.33336639404274, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 17.3333740234375, + "height": 13.000015258789062, + "seed": 682826293, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "6EZFlJGbMoYN4RDYkSEYP", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 609, + "versionNonce": 1207751547, + "isDeleted": false, + "id": "6EZFlJGbMoYN4RDYkSEYP", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 257.1755919962603, + "y": 432.4837343704476, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 350.4818452518821, + "height": 113.7944575139108, + "seed": 1046151995, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "bCPa9rRVzubIpa31jwl73", + "focus": 0.6467798007705062, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -33.87912050646, + 40.47938005012526 + ], + [ + 316.60272474542205, + 113.7944575139108 + ] + ] + }, + { + "type": "text", + "version": 380, + "versionNonce": 1636332981, + "isDeleted": false, + "id": "qUXi_zQbiA8lmV5c_r8ta", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 875.0742831759982, + "y": 534.6482721964519, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 172, + "height": 20, + "seed": 2104268027, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModule::memories", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule::memories" + }, + { + "type": "rectangle", + "version": 511, + "versionNonce": 833268763, + "isDeleted": false, + "id": "rfTE9QxqtEjzOzoCNZT01", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 900.9256795247394, + "y": 593.3610492282444, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 1292864565, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "4NjPz6Olqru3m83OxGXGX", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 342, + "versionNonce": 347731733, + "isDeleted": false, + "id": "RzznX4k1tfSt8AyuJuF2D", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 883.9257710774738, + "y": 564.9444232516819, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 153.00003051757812, + "seed": 1990606235, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "4NjPz6Olqru3m83OxGXGX", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 566, + "versionNonce": 626290875, + "isDeleted": false, + "id": "GEQFwoUvMYOSit62JGwLM", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 894.5554572211371, + "y": 634.037105984158, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 1545223573, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 578, + "versionNonce": 1559514229, + "isDeleted": false, + "id": "IvFYTMyYUBhJe6Ob5YAGg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 901.2222052680121, + "y": 669.3704189724392, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70.00006103515625, + "height": 25.66668701171874, + "seed": 937546299, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 43, + "versionNonce": 1419551067, + "isDeleted": false, + "id": "ZMwf8VnKqa7gp45PnuQWi", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 780.3335876464843, + "y": 45.92599826388881, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 286.66666666666686, + "height": 707.0370313856337, + "seed": 876902971, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "YidAloK-3ikBBzvuu-S22", + "type": "arrow" + } + ], + "updated": 1679555887723, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 138, + "versionNonce": 1241314773, + "isDeleted": false, + "id": "2NQmRBF_NE2Myp3-jqLfT", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 418.1839375016565, + "y": 222.9630296495223, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 351.7376685654755, + "height": 202.59258694118918, + "seed": 1011740277, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": { + "elementId": "TOmX9MwwNOgbfjNJ8V8j7", + "focus": 0.48971428325717553, + "gap": 11.018498738606851 + }, + "endBinding": { + "elementId": "7MNv-pmgV4-8yJ5lIFY1U", + "focus": 1.3268339026620566, + "gap": 13.16658528645857 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 82.8903795827618, + -202.59258694118918 + ], + [ + 351.7376685654755, + -175.99734962527606 + ] + ] + }, + { + "type": "arrow", + "version": 58, + "versionNonce": 416660987, + "isDeleted": false, + "id": "OjoiFuv7ZOtNkq4SpgOL-", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 796.6298048231338, + "y": 366.6667785644529, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 82.96305338541652, + "height": 65.92593722873261, + "seed": 275987509, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "50mTV_vbZA4UToji5TAhb", + "focus": -0.4434608130384178, + "gap": 2.481075710720461 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 82.96305338541652, + 65.92593722873261 + ] + ] + }, + { + "type": "arrow", + "version": 131, + "versionNonce": 1458139957, + "isDeleted": false, + "id": "4NjPz6Olqru3m83OxGXGX", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 796.6298726399739, + "y": 372.96306355794246, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.51854112413207, + "height": 227.77777777777777, + "seed": 1888354747, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "rfTE9QxqtEjzOzoCNZT01", + "focus": -0.6184957089704255, + "gap": 5.7772657606334406 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 46.66673448350696, + 191.48159450954864 + ], + [ + 98.51854112413207, + 227.77777777777777 + ] + ] + }, + { + "type": "arrow", + "version": 105, + "versionNonce": 2097301147, + "isDeleted": false, + "id": "euj5CRuv6EoS2abVhp14P", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 792.9261237250435, + "y": 375.55566745334175, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 140, + "height": 233.33346896701386, + "seed": 856817851, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887723, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "qpSAJ6K0S6TvU6i2Uflep", + "focus": 1.8857122566967692, + "gap": 14.01957438916057 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -30.37034776475707, + 147.77781168619794 + ], + [ + -140, + 233.33346896701386 + ] + ] + }, + { + "type": "text", + "version": 404, + "versionNonce": 2049173, + "isDeleted": false, + "id": "0836mka_gP8EntAw7Pvzh", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 285.4076097276477, + "y": 424.0742221408418, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 198, + "height": 19, + "seed": 1302236763, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887724, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMFunction **functions;", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunction **functions;" + }, + { + "type": "rectangle", + "version": 385, + "versionNonce": 131382075, + "isDeleted": false, + "id": "Fv3xwjCOvQ9-pJ5ui2sV2", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 239.7409837510852, + "y": 409.4075961642793, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 216, + "height": 42.6666259765625, + "seed": 1276926165, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887724, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 350, + "versionNonce": 424929781, + "isDeleted": false, + "id": "IibWA_jM89ndxmrceQLmE", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 254.4077317979602, + "y": 420.07428317599806, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 15.33331298828125, + "height": 22.66668701171875, + "seed": 1427245819, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887724, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 133, + "versionNonce": 1169136603, + "isDeleted": false, + "id": "OdDm5a5O_NIoZbF3u0c0H", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 442.5558098687066, + "y": 370.37052747938344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 138.34606629993374, + "height": 88.15863628949046, + "seed": 967352763, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679555887724, + "link": null, + "locked": false, + "startBinding": { + "elementId": "FPUGB-iP7ep91gfoTuV2d", + "focus": 0.6557614629211577, + "gap": 3.7962053087022696 + }, + "endBinding": { + "elementId": "blxQLl_yf7DMD76qHC5rc", + "focus": -0.24942508137542688, + "gap": 9.870619032117872 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 61.48149278428815, + -72.22222222222217 + ], + [ + 138.34606629993374, + -88.15863628949046 + ] + ] + }, + { + "type": "text", + "version": 35, + "versionNonce": 1670177621, + "isDeleted": false, + "id": "armvxVMuT0bX77T30wPV4", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -39.66648017035561, + "y": 230.37051052517361, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 165, + "height": 20, + "seed": 587722005, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887724, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "export_global_count", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "export_global_count" + }, + { + "type": "rectangle", + "version": 502, + "versionNonce": 312830075, + "isDeleted": false, + "id": "nUlAmf5jmsJoZ2lStTbBw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -52.18503146701369, + "y": 226.01867336697043, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174.66668701171875, + "height": 25.37032402886292, + "seed": 1476753525, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679555887724, + "link": null, + "locked": false + }, + { + "id": "Mznw_08SMS746JVePQe3h", + "type": "text", + "x": -55.747947692871094, + "y": 396.450114440918, + "width": 174, + "height": 20, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 1392475036, + "version": 62, + "versionNonce": 452856732, + "isDeleted": false, + "boundElements": null, + "updated": 1679558398355, + "link": null, + "locked": false, + "text": "export_memory_count", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 14, + "containerId": null, + "originalText": "export_memory_count" + }, + { + "id": "wyGG1OIhy3X499IvtJM3Z", + "type": "rectangle", + "x": -58.081260681152344, + "y": 394.9499618530274, + "width": 182.33331298828125, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 611257380, + "version": 32, + "versionNonce": 1973769116, + "isDeleted": false, + "boundElements": null, + "updated": 1679558406221, + "link": null, + "locked": false + }, + { + "id": "Gk4JYjHayAFkRDbQWId6b", + "type": "text", + "x": -42.081260681152344, + "y": 463.28327484130864, + "width": 162, + "height": 20, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 804003748, + "version": 47, + "versionNonce": 591391772, + "isDeleted": false, + "boundElements": null, + "updated": 1679558418472, + "link": null, + "locked": false, + "text": "export_table_count", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 14, + "containerId": null, + "originalText": "export_table_count" + }, + { + "id": "Xb-aggFXlG4Dh_LtxGJ7l", + "type": "rectangle", + "x": -49.414573669433594, + "y": 462.28330535888676, + "width": 174.33331298828125, + "height": 24.33343505859375, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 572093348, + "version": 30, + "versionNonce": 1840351132, + "isDeleted": false, + "boundElements": null, + "updated": 1679558423043, + "link": null, + "locked": false + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/stack_format_ci.excalidraw b/wasm-micro-runtime/core/iwasm/doc/images/stack_format_ci.excalidraw new file mode 100644 index 0000000..f2ade3b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/stack_format_ci.excalidraw @@ -0,0 +1,2643 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "elements": [ + { + "type": "rectangle", + "version": 155, + "versionNonce": 1248561528, + "isDeleted": false, + "id": "fxSjNN3geJtdm-2MR7INN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 583.9999694824219, + "y": 276.33331298828114, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 427.66665649414057, + "height": 468.6667785644533, + "seed": 1226571556, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "zFlCnXyCuuvj85rH8yMcR", + "type": "arrow" + }, + { + "id": "KiC12zdfqG7zXRbctXGmT", + "type": "arrow" + } + ], + "updated": 1679706874848, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 31, + "versionNonce": 726044059, + "isDeleted": false, + "id": "kKMhPpSUI7zZU0hhfGfSr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 652, + "y": 310.5, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 91, + "height": 20, + "seed": 1323166748, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "v8qOyuaPyJmHPHCk-V90A", + "type": "arrow" + } + ], + "updated": 1679639568750, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "prev_frame", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "prev_frame" + }, + { + "type": "text", + "version": 31, + "versionNonce": 213374372, + "isDeleted": false, + "id": "97vkuGwuf9dOgnbXL4nZW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 652, + "y": 345.5, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 84, + "height": 20, + "seed": 123433892, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679617642914, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func_inst ", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func_inst " + }, + { + "type": "text", + "version": 31, + "versionNonce": 34601372, + "isDeleted": false, + "id": "IaZULBLAtP-VbIpB210WT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 652, + "y": 380.5, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 13, + "height": 20, + "seed": 1157761180, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679617674900, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "ip", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "ip" + }, + { + "type": "text", + "version": 37, + "versionNonce": 556583925, + "isDeleted": false, + "id": "H6AElOIjCOfKVDEzsEb8_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 648.3333435058594, + "y": 415.16668701171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 160, + "height": 20, + "seed": 175464228, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "bzZuzwhAslOM0VOMpIFi4", + "type": "arrow" + } + ], + "updated": 1679638914768, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "opnd_stack_bottom", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "opnd_stack_bottom" + }, + { + "type": "text", + "version": 31, + "versionNonce": 2114117339, + "isDeleted": false, + "id": "InrDJwTwyP1E7lpeGu0Tb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 652, + "y": 450.5, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 275, + "height": 20, + "seed": 158242076, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "bzZuzwhAslOM0VOMpIFi4", + "type": "arrow" + } + ], + "updated": 1679638694120, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "opnd_sp (point to opnd stack top)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "opnd_sp (point to opnd stack top)" + }, + { + "type": "text", + "version": 31, + "versionNonce": 1005029461, + "isDeleted": false, + "id": "04QcPfm0Sc1zQpYDxmPVl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 652, + "y": 485.5, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 163, + "height": 20, + "seed": 2051398308, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679638792159, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "label_stack_bottom", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "label_stack_bottom" + }, + { + "type": "text", + "version": 31, + "versionNonce": 1565468827, + "isDeleted": false, + "id": "GD4urgWTuYKcn7FcwA5uj", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 652, + "y": 520.5, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 280, + "height": 20, + "seed": 429232540, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "AhNgHym2Ox-2Je47FlwkG", + "type": "arrow" + } + ], + "updated": 1679638982669, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "label_sp (point to label stack top)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "label_sp (point to label stack top)" + }, + { + "type": "text", + "version": 123, + "versionNonce": 458495496, + "isDeleted": false, + "id": "9TW8AADnHJOkmP9yPw86T", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 626, + "y": 586.1666564941406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 125, + "height": 20, + "seed": 561702436, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706667761, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func arguments:", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func arguments:" + }, + { + "type": "text", + "version": 373, + "versionNonce": 180653176, + "isDeleted": false, + "id": "ZDnffzv8Hf65ecpMemz82", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 700.6666870117188, + "y": 669.1667785644531, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 131, + "height": 20, + "seed": 1157792164, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543900, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "" + }, + { + "type": "rectangle", + "version": 108, + "versionNonce": 628186232, + "isDeleted": false, + "id": "10UtZNhGhQIza4PqPLKs5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 622.6665954589844, + "y": 299.9999694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 378.3334655761719, + "height": 33.66670227050781, + "seed": 1281579428, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706899765, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 108, + "versionNonce": 1807107317, + "isDeleted": false, + "id": "lxiJENi5HY8EPwIeH7LA0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 621.9999389648438, + "y": 343.3332977294922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 377.6668090820312, + "height": 28.33332824707031, + "seed": 523845788, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679639010923, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 104, + "versionNonce": 114288661, + "isDeleted": false, + "id": "iBGNXVvcsdhIuK4rSyr24", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 621.333251953125, + "y": 377.9999694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 380.3333129882813, + "height": 24.666656494140625, + "seed": 1117299228, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679639014926, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 141, + "versionNonce": 1756531061, + "isDeleted": false, + "id": "hwgrctAuxGaA-SGuwFgEd", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 623.6665649414062, + "y": 413.66664123535156, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 190.33331298828125, + "height": 25.333358764648438, + "seed": 1885325596, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "2yVfQUtaAnW5BuEtoXRcA", + "type": "arrow" + } + ], + "updated": 1679638734395, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 135, + "versionNonce": 138906299, + "isDeleted": false, + "id": "V6DAmN9L5JPz2B1MGfa9C", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 624.6665954589844, + "y": 445.9999694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 377.6666564941406, + "height": 28.00003051757812, + "seed": 1058988452, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "bzZuzwhAslOM0VOMpIFi4", + "type": "arrow" + } + ], + "updated": 1679638756269, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 171, + "versionNonce": 755362101, + "isDeleted": false, + "id": "0m59R0bqJv_x7GbZ6hfyg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 624.6665954589844, + "y": 485.33331298828125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 197.33340454101562, + "height": 23.33328247070314, + "seed": 1040920092, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "bzZuzwhAslOM0VOMpIFi4", + "type": "arrow" + } + ], + "updated": 1679638908118, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 191, + "versionNonce": 944635675, + "isDeleted": false, + "id": "Itqs7rL1-S3eIrmvfrr6i", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 625.333251953125, + "y": 516.3333129882812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 349.66674804687506, + "height": 34.66665649414062, + "seed": 498379804, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679639029205, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 169, + "versionNonce": 1525847816, + "isDeleted": false, + "id": "Yx4QpHhmHHvf267ot4QcW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 623.333251953125, + "y": 582.6666259765625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 352.3333740234375, + "height": 31.666748046875, + "seed": 612410396, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543900, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 195, + "versionNonce": 335028795, + "isDeleted": false, + "id": "sWiDv3IP9Y4zn-pQPKT59", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 585.6668090820312, + "y": 203.99998474121094, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 424.3334045410156, + "height": 70.66668701171874, + "seed": 813710748, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "v8qOyuaPyJmHPHCk-V90A", + "type": "arrow" + } + ], + "updated": 1679639568750, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 255, + "versionNonce": 1953250680, + "isDeleted": false, + "id": "gS_VbgMS5NUoHmfxH0eEO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 752.0001525878906, + "y": 624.3333129882812, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 30.6666259765625, + "height": 19.000091552734375, + "seed": 2044515484, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543900, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 171, + "versionNonce": 227339384, + "isDeleted": false, + "id": "m89OexiP1cw5cN304gmQ-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 771.3336334228516, + "y": 587.8333282470703, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 37.999969482421875, + "height": 19.666748046875, + "seed": 2074557348, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706664521, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 174, + "versionNonce": 1205391112, + "isDeleted": false, + "id": "aAWksorEQDseQSDntTUKe", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 824.6669464111328, + "y": 589.1665802001953, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 35.999969482421875, + "height": 15.666778564453123, + "seed": 1851711260, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706659367, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 213, + "versionNonce": 1798283016, + "isDeleted": false, + "id": "IH4D4tHy91CnkLlP3kKJT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 878.6669464111328, + "y": 588.8333892822266, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 35.999969482421875, + "height": 16.333435058593754, + "seed": 1835107492, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706677882, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 28, + "versionNonce": 1189712156, + "isDeleted": false, + "id": "OGXe0pARYj6FFaTw_LGLs", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 916.6667785644531, + "y": 348.3333435058594, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 13.33331298828125, + "height": 15.66668701171875, + "seed": 703955236, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "zFlCnXyCuuvj85rH8yMcR", + "type": "arrow" + } + ], + "updated": 1679617654572, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 90, + "versionNonce": 1752012152, + "isDeleted": false, + "id": "zFlCnXyCuuvj85rH8yMcR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 930.9664201728779, + "y": 355.9093759200581, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 97.70041942673151, + "height": 3.156306335738577, + "seed": 1652552228, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706533823, + "link": null, + "locked": false, + "startBinding": { + "elementId": "OGXe0pARYj6FFaTw_LGLs", + "gap": 1, + "focus": -0.05520408889747474 + }, + "endBinding": { + "elementId": "fxSjNN3geJtdm-2MR7INN", + "gap": 17.000213623046875, + "focus": 0.6588609578759657 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 97.70041942673151, + 3.156306335738577 + ] + ] + }, + { + "type": "rectangle", + "version": 39, + "versionNonce": 2081828900, + "isDeleted": false, + "id": "cN9omEMJtZq2pKK3nPqeK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1033.3334655761719, + "y": 339.0000305175781, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 218, + "height": 37, + "seed": 1716587932, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "w-Fx4nTROv9qvnPqLigxc" + } + ], + "updated": 1679617661526, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 4, + "versionNonce": 727129976, + "isDeleted": false, + "id": "w-Fx4nTROv9qvnPqLigxc", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1051.3334655761719, + "y": 347.5000305175781, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 182, + "height": 20, + "seed": 1245620124, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706461629, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(current func instance)", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "cN9omEMJtZq2pKK3nPqeK", + "originalText": "(current func instance)" + }, + { + "type": "diamond", + "version": 19, + "versionNonce": 1517445412, + "isDeleted": false, + "id": "YAWb1D32tq9r2NA9BjPC0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 917.3334655761719, + "y": 381.3333740234375, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 12.6666259765625, + "height": 17.666656494140625, + "seed": 831940124, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679617681098, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 103, + "versionNonce": 437077525, + "isDeleted": false, + "id": "EUw5VCdUqXWdPh2cmFgTu", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1048.0000915527344, + "y": 390.0000305175781, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 224, + "height": 52, + "seed": 122854564, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "sQUYAdB_aJemE8MbtfUXU" + }, + { + "id": "7cT6qctQMJuzVHVpt9gAe", + "type": "arrow" + } + ], + "updated": 1679643416466, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 89, + "versionNonce": 818328584, + "isDeleted": false, + "id": "sQUYAdB_aJemE8MbtfUXU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1066.0000915527344, + "y": 395.0000305175781, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 188, + "height": 40, + "seed": 1881253020, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706461630, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": " (next bytecode in code\nfor return)", + "baseline": 34, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "EUw5VCdUqXWdPh2cmFgTu", + "originalText": " (next bytecode in code\nfor return)" + }, + { + "type": "arrow", + "version": 86, + "versionNonce": 44769845, + "isDeleted": false, + "id": "7cT6qctQMJuzVHVpt9gAe", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 928.0000915527344, + "y": 389.66668701171875, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 118.6890752940651, + "height": 18.605575795885613, + "seed": 2073584156, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643418552, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "EUw5VCdUqXWdPh2cmFgTu", + "gap": 1.3109247059348494, + "focus": -0.23038166167021662 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 118.6890752940651, + 18.605575795885613 + ] + ] + }, + { + "type": "rectangle", + "version": 2, + "versionNonce": 504035291, + "isDeleted": false, + "id": "6fkbi-hv-WF274Ggup6O2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1200.0001525878906, + "y": 619.8332824707031, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.333251953125, + "height": 0.333343505859375, + "seed": 1881722357, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1679638527376, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 237, + "versionNonce": 103104376, + "isDeleted": false, + "id": "TBY_IOcQ1nL2SbnoH6sFO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 627.0001373291016, + "y": 660.5000610351562, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 357.33331298828125, + "height": 30.33352661132812, + "seed": 1619080725, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "bzZuzwhAslOM0VOMpIFi4", + "type": "arrow" + }, + { + "id": "2yVfQUtaAnW5BuEtoXRcA", + "type": "arrow" + } + ], + "updated": 1679706543900, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 304, + "versionNonce": 1231721480, + "isDeleted": false, + "id": "2yVfQUtaAnW5BuEtoXRcA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1001.4951265607561, + "y": 435.4648501924554, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 69.86595546390652, + "height": 230.92727975758817, + "seed": 554350997, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Q10gr2u0Ozc3e0iafNjrU", + "focus": -0.9749635321679403, + "gap": 1 + }, + "endBinding": { + "elementId": "TBY_IOcQ1nL2SbnoH6sFO", + "gap": 11.467449077109102, + "focus": 0.9687666106437038 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 64.17172829764229, + 132.70180630168522 + ], + [ + -5.694227166264227, + 230.92727975758817 + ] + ] + }, + { + "type": "arrow", + "version": 579, + "versionNonce": 1040896776, + "isDeleted": false, + "id": "bzZuzwhAslOM0VOMpIFi4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 637.088614263612, + "y": 428.66374830753944, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 87.42185095794798, + "height": 243.67105886528202, + "seed": 1778263445, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": { + "elementId": "H6AElOIjCOfKVDEzsEb8_", + "focus": 1.022917371543073, + "gap": 11.244729242247331 + }, + "endBinding": { + "elementId": "zmGNXD3FjbK3cLzJQcTS7", + "focus": -0.9191849396644245, + "gap": 1.3880431063754486 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -87.42185095794798, + 127.50287766902306 + ], + [ + -10.80983302916718, + 243.67105886528202 + ] + ] + }, + { + "type": "rectangle", + "version": 146, + "versionNonce": 1195282552, + "isDeleted": false, + "id": "zmGNXD3FjbK3cLzJQcTS7", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 627.6668243408203, + "y": 662.8332824707031, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "width": 207.333251953125, + "height": 25.333343505859375, + "seed": 2075064181, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "bzZuzwhAslOM0VOMpIFi4", + "type": "arrow" + }, + { + "id": "PtPViQVeIOLGj_fGdcXae", + "type": "arrow" + } + ], + "updated": 1679706543901, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 252, + "versionNonce": 1380659720, + "isDeleted": false, + "id": "fdKDasbQmnSQhVWnuc6o0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 892.2284992953181, + "y": 676.9139215503409, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "width": 46.77163803378346, + "height": 0.7472955737783877, + "seed": 2138378229, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "points": [ + [ + 0, + 0 + ], + [ + 46.77163803378346, + -0.7472955737783877 + ] + ] + }, + { + "type": "text", + "version": 63, + "versionNonce": 2032367925, + "isDeleted": false, + "id": "ZwrESUGUwekFgJ8cot-D1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 821.6667327880859, + "y": 414.6666259765625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174, + "height": 20, + "seed": 795089589, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "2yVfQUtaAnW5BuEtoXRcA", + "type": "arrow" + } + ], + "updated": 1679638757542, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "opnd_stack_boundary", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "opnd_stack_boundary" + }, + { + "type": "rectangle", + "version": 32, + "versionNonce": 132303029, + "isDeleted": false, + "id": "VOinNwnsGZVJt9RUbM5gI", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 821.3334197998047, + "y": 409.8332977294922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 184.3333740234375, + "height": 30.333328247070312, + "seed": 1069875285, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679638823023, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 255, + "versionNonce": 453362040, + "isDeleted": false, + "id": "PtPViQVeIOLGj_fGdcXae", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 979.6667327880859, + "y": 460.4999694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 140.89037965104183, + "height": 211.49970410546382, + "seed": 472939093, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "zmGNXD3FjbK3cLzJQcTS7", + "focus": 0.7843520090394333, + "gap": 6.109589831379935 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 2.33331298828125, + 145 + ], + [ + -138.55706666276058, + 211.49970410546382 + ] + ] + }, + { + "type": "diamond", + "version": 19, + "versionNonce": 987485307, + "isDeleted": false, + "id": "u7bglMSX07f5CN_SyFAmd", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 967.0000457763672, + "y": 452.1666259765625, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 15, + "height": 15, + "seed": 764957403, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679638777911, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 43, + "versionNonce": 115610933, + "isDeleted": false, + "id": "shh1k5OswcF57jhAKAI6G", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 830.0000457763672, + "y": 486.8332824707031, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 176, + "height": 20, + "seed": 1994374395, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679638809874, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "label_stack_boundary", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "label_stack_boundary" + }, + { + "type": "rectangle", + "version": 33, + "versionNonce": 1511042715, + "isDeleted": false, + "id": "H4lcjS3aIarViIGAHsLBU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 827.6667327880859, + "y": 480.83331298828125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 181, + "height": 29.666656494140625, + "seed": 1492829339, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679638816247, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 372, + "versionNonce": 1764776968, + "isDeleted": false, + "id": "B1eH_aNspf4OoiXnvWu1V", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 630.0000915527344, + "y": 705.1667175292969, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 356.66662597656256, + "height": 31.333404541015625, + "seed": 710054491, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "X76to9wjER3VzqXb7tCwb", + "type": "arrow" + }, + { + "id": "1Urs2i0MGQSQ9XbOVdbCH", + "type": "arrow" + }, + { + "id": "KiC12zdfqG7zXRbctXGmT", + "type": "arrow" + } + ], + "updated": 1679706706153, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 332, + "versionNonce": 1981566984, + "isDeleted": false, + "id": "Tu7mkiWTjNwFrOsUa35N1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 633.3333740234375, + "y": 704.8334045410156, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "width": 250.33328247070318, + "height": 26.66656494140625, + "seed": 1201312981, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "AhNgHym2Ox-2Je47FlwkG", + "type": "arrow" + } + ], + "updated": 1679706543901, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 250, + "versionNonce": 1696927496, + "isDeleted": false, + "id": "xQf_fS4j5XD1gPYVmnyQi", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 687, + "y": 709.8335876464844, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 108, + "height": 20, + "seed": 1671092892, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "" + }, + { + "type": "diamond", + "version": 19, + "versionNonce": 167262267, + "isDeleted": false, + "id": "T43dNIeAePPWy65vsVjRw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 629.0000457763672, + "y": 420.8332977294922, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 13.66668701171875, + "height": 9, + "seed": 1938681499, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679638923749, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 26, + "versionNonce": 1796989173, + "isDeleted": false, + "id": "Q10gr2u0Ozc3e0iafNjrU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 988.5000762939453, + "y": 431.9999542236328, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 13.66668701171875, + "height": 9, + "seed": 1339643477, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "2yVfQUtaAnW5BuEtoXRcA", + "type": "arrow" + } + ], + "updated": 1679639079224, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 160, + "versionNonce": 1562882312, + "isDeleted": false, + "id": "X76to9wjER3VzqXb7tCwb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 633.6667327880859, + "y": 498.1665954589844, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 95, + "height": 226.4773119076067, + "seed": 332801877, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "B1eH_aNspf4OoiXnvWu1V", + "gap": 3.7751266782821933, + "focus": -0.9692106196282059 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -95, + 119 + ], + [ + -7.441767913633839, + 226.4773119076067 + ] + ] + }, + { + "type": "arrow", + "version": 202, + "versionNonce": 791528312, + "isDeleted": false, + "id": "1Urs2i0MGQSQ9XbOVdbCH", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 993.6667327880859, + "y": 500.833251953125, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 55.58884147480535, + "height": 215.3311147859538, + "seed": 385950459, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": { + "elementId": "_1a_M2hRRjsccFJTYD-i0", + "focus": -0.27052637703325505, + "gap": 2.7105616084241397 + }, + "endBinding": { + "elementId": "B1eH_aNspf4OoiXnvWu1V", + "focus": 0.9207393546998478, + "gap": 3.0779218308585996 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 51.666748046875, + 153.66668701171875 + ], + [ + -3.922093427930349, + 215.3311147859538 + ] + ] + }, + { + "type": "arrow", + "version": 252, + "versionNonce": 1150719096, + "isDeleted": false, + "id": "AhNgHym2Ox-2Je47FlwkG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 941.0000457763672, + "y": 533.833251953125, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 73.73891815660659, + "height": 177.63880524698254, + "seed": 17032283, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": { + "elementId": "GD4urgWTuYKcn7FcwA5uj", + "focus": -1.0502037458670375, + "gap": 9.000045776367188 + }, + "endBinding": { + "elementId": "Tu7mkiWTjNwFrOsUa35N1", + "focus": 0.7914007600477917, + "gap": 2.5944100904637253 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 18.99993896484375, + 133.33340454101562 + ], + [ + -54.73897919176284, + 177.63880524698254 + ] + ] + }, + { + "type": "diamond", + "version": 19, + "versionNonce": 987485307, + "isDeleted": false, + "id": "C2JNNEQlhQ4cFy2h16E4l", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 630.5000457763672, + "y": 491.333251953125, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 15, + "height": 15, + "seed": 2047712475, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679638991963, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 19, + "versionNonce": 987485307, + "isDeleted": false, + "id": "xsWhegnNTDUvjVk1Hl20d", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 931.5000457763672, + "y": 523.333251953125, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 15, + "height": 15, + "seed": 751530581, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679638994246, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 20, + "versionNonce": 2123101557, + "isDeleted": false, + "id": "_1a_M2hRRjsccFJTYD-i0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 987.5000457763672, + "y": 503.333251953125, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 15, + "height": 15, + "seed": 1838893435, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "id": "1Urs2i0MGQSQ9XbOVdbCH", + "type": "arrow" + } + ], + "updated": 1679639046142, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 122, + "versionNonce": 918373752, + "isDeleted": false, + "id": "_FzrYgawOp32aihqUkx8N", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 937.3333587646484, + "y": 722.4999084472656, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 31, + "height": 0.333343505859375, + "seed": 125027067, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "points": [ + [ + 0, + 0 + ], + [ + 31, + 0.333343505859375 + ] + ] + }, + { + "type": "arrow", + "version": 58, + "versionNonce": 1791510299, + "isDeleted": false, + "id": "v8qOyuaPyJmHPHCk-V90A", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 640.33349609375, + "y": 318.1665954589844, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 112.66668701171875, + "height": 114.41643468378521, + "seed": 1521072891, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679639586455, + "link": null, + "locked": false, + "startBinding": { + "elementId": "kKMhPpSUI7zZU0hhfGfSr", + "focus": -0.8312315870337287, + "gap": 11.66650390625 + }, + "endBinding": { + "elementId": "sWiDv3IP9Y4zn-pQPKT59", + "focus": 1.0049483520293248, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -112.66668701171875, + -61.99995422363281 + ], + [ + -55.63497828298284, + -114.41643468378521 + ] + ] + }, + { + "type": "diamond", + "version": 23, + "versionNonce": 1760592469, + "isDeleted": false, + "id": "rizowRaNCY3sscSKdUoZ_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 637.6668090820312, + "y": 307.49993896484375, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 11.66668701171875, + "height": 17, + "seed": 599455541, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679639578366, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 241, + "versionNonce": 1517501429, + "isDeleted": false, + "id": "Mv8lernCpH2jNrVrNfqm5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 585.8334503173828, + "y": 130.833251953125, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 424.3334045410156, + "height": 70.66668701171874, + "seed": 1040624571, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "cl6Duxg7hC4_1chb5YNau", + "type": "arrow" + } + ], + "updated": 1679639602774, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 41, + "versionNonce": 926996117, + "isDeleted": false, + "id": "eQrRfUHvRypTcFfQ80nK5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 647.0000610351562, + "y": 219.33328247070312, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 91, + "height": 20, + "seed": 1884598901, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "cl6Duxg7hC4_1chb5YNau", + "type": "arrow" + } + ], + "updated": 1679639602774, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "prev_frame", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "prev_frame" + }, + { + "type": "diamond", + "version": 32, + "versionNonce": 784602165, + "isDeleted": false, + "id": "w_RQs6dyVbTAM3iaX46G2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 632.6668701171875, + "y": 216.33322143554688, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 11.66668701171875, + "height": 17, + "seed": 1918737243, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679639598842, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 135, + "versionNonce": 1953671957, + "isDeleted": false, + "id": "cl6Duxg7hC4_1chb5YNau", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 639.0001525878906, + "y": 224.16659545898438, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 80.33334350585938, + "height": 95, + "seed": 509490587, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679639608498, + "link": null, + "locked": false, + "startBinding": { + "elementId": "eQrRfUHvRypTcFfQ80nK5", + "focus": -0.7568501325114791, + "gap": 7.999908447265625 + }, + "endBinding": { + "elementId": "Mv8lernCpH2jNrVrNfqm5", + "focus": 1.0067042515978792, + "gap": 1.666656494140625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -80.33334350585938, + -53.66668701171875 + ], + [ + -53.66668701171875, + -95 + ] + ] + }, + { + "type": "text", + "version": 196, + "versionNonce": 1763609493, + "isDeleted": false, + "id": "7pum3cjupgQ0Fz7eY_3xD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1026.33349609375, + "y": 123.83326721191406, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 109, + "height": 20, + "seed": 1300737371, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679640276534, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "Stack bottom", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Stack bottom" + }, + { + "type": "text", + "version": 310, + "versionNonce": 1972102043, + "isDeleted": false, + "id": "Su_2hWwLvhW5W8cNMx6hT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1030.666748046875, + "y": 266.8333206176758, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "width": 110, + "height": 20, + "seed": 2131950293, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679640279741, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "current frame", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "current frame" + }, + { + "type": "rectangle", + "version": 181, + "versionNonce": 1099516792, + "isDeleted": false, + "id": "jCBotdnPaPF-NkuwgrLyd", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 582.3333129882812, + "y": 744.8332977294922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 429.666748046875, + "height": 48.333343505859396, + "seed": 1136750491, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706699607, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 167, + "versionNonce": 647398264, + "isDeleted": false, + "id": "BRMXHqjjvaWADtilYg0NV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 784.3333740234375, + "y": 775.8333892822266, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 57, + "height": 20, + "seed": 415943189, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "KiC12zdfqG7zXRbctXGmT", + "type": "arrow" + } + ], + "updated": 1679706708714, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "Unused", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Unused" + }, + { + "type": "text", + "version": 179, + "versionNonce": 1210985736, + "isDeleted": false, + "id": "6YoFb1tMeQkuEDMy5H4FV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1018.6668090820312, + "y": 778.8333892822266, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 122, + "height": 20, + "seed": 1257872437, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706711595, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "stack boundary", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "stack boundary" + }, + { + "type": "text", + "version": 174, + "versionNonce": 314437128, + "isDeleted": false, + "id": "_e0Yy4p4UD9MNJUym2U_N", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1025.6666259765625, + "y": 732.8335113525391, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 80, + "height": 20, + "seed": 521361589, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "stack top", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "stack top" + }, + { + "type": "rectangle", + "version": 126, + "versionNonce": 1927665272, + "isDeleted": false, + "id": "ATaMGJS9emfaD_vgxw4Wi", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 625.5619915078412, + "y": 621.4944998254441, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 358.66670227050787, + "height": 25.999969482421875, + "seed": 1247140509, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 268, + "versionNonce": 1190911752, + "isDeleted": false, + "id": "qlKC-UwDGQErtLa5oG6C3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 625.3333740234375, + "y": 554.1666870117188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 236, + "height": 20, + "seed": 1957232156, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706639231, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "Wasm LOCALs (local.set/get):", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Wasm LOCALs (local.set/get):" + }, + { + "type": "rectangle", + "version": 264, + "versionNonce": 500451192, + "isDeleted": false, + "id": "CnBSoD9DNI5Himd9gJXwV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 787.8953006814252, + "y": 625.661102913823, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 30.6666259765625, + "height": 19.000091552734375, + "seed": 1737116886, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 259, + "versionNonce": 235932680, + "isDeleted": false, + "id": "z4Ye9_zQDUab5ewn2mujX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 827.5619876931439, + "y": 623.9944159021043, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 30.6666259765625, + "height": 19.000091552734375, + "seed": 1207868746, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 292, + "versionNonce": 600610936, + "isDeleted": false, + "id": "V3_r6rtZsp-pFDhmttdsh", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 865.5620487283002, + "y": 625.3277594079636, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 30.6666259765625, + "height": 19.000091552734375, + "seed": 974809482, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 266, + "versionNonce": 1061243656, + "isDeleted": false, + "id": "Tov8P7W3W8Zmma9a3S8gP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 904.5620487283002, + "y": 626.661102913823, + "strokeColor": "#000000", + "backgroundColor": "#40c057", + "width": 30.6666259765625, + "height": 19.000091552734375, + "seed": 1423773654, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679706543901, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 148, + "versionNonce": 1167673464, + "isDeleted": false, + "id": "KiC12zdfqG7zXRbctXGmT", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 806.4613995527118, + "y": 748.8279119958543, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 2.161352857758061, + "height": 20.33328247070324, + "seed": 167487754, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679706706153, + "link": null, + "locked": false, + "startBinding": { + "elementId": "B1eH_aNspf4OoiXnvWu1V", + "focus": 0.026931962058821583, + "gap": 12.327789925541765 + }, + "endBinding": { + "elementId": "fxSjNN3geJtdm-2MR7INN", + "focus": -0.06989783256192499, + "gap": 24.16110291382313 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 2.161352857758061, + 20.33328247070324 + ] + ] + }, + { + "type": "text", + "version": 151, + "versionNonce": 1430888312, + "isDeleted": false, + "id": "wBnV18_fVimstqpfPY0KR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 631.4456683895414, + "y": 624.6666107177734, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 91, + "height": 20, + "seed": 1961347080, + "groupIds": [], + "roundness": null, + "boundElements": null, + "updated": 1679706671751, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func locals:", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func locals:" + }, + { + "id": "8Y0Qz7jDPtVE84-d_UHu2", + "type": "rectangle", + "x": 613.2790118954008, + "y": 575.9999542236328, + "width": 384.0000305175781, + "height": 76.66668701171875, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": { + "type": 3 + }, + "seed": 743618568, + "version": 80, + "versionNonce": 129563000, + "isDeleted": false, + "boundElements": null, + "updated": 1679706644458, + "link": null, + "locked": false + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/stack_format_ci.svg b/wasm-micro-runtime/core/iwasm/doc/images/stack_format_ci.svg new file mode 100644 index 0000000..f054c43 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/stack_format_ci.svg @@ -0,0 +1,16 @@ + + + + + + + prev_framefunc_inst ipopnd_stack_bottomopnd_sp (point to opnd stack top)label_stack_bottomlabel_sp (point to label stack top)func arguments:<operand stack>(current func instance) (next bytecode in codefor return)opnd_stack_boundarylabel_stack_boundary<lable stack>prev_frameStack bottomcurrent frameUnusedstack boundarystack topWasm LOCALs (local.set/get):func locals: \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/wasm_exports.svg b/wasm-micro-runtime/core/iwasm/doc/images/wasm_exports.svg new file mode 100644 index 0000000..573f11c --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/wasm_exports.svg @@ -0,0 +1,16 @@ + + + + + + + WASMModule WASMTable *tables; WASMMemory *memories; WASMGlobal *globals;WASMExport *exports;WASMExportchar *name;uint8 kind;uint32 index;[0][1][...]WASMModuleInstanceexport_tablesexport_memoriesexport_globalsexport_functionsWASMExportFuncInstancechar* name functionname: shown to externalindex: refer to item idx for export in current kindkind: total 4 export kinds:- function- globals- memory- table[1][0][...][n](refer to function diagam)export_func_countWASMExportGlobInstancechar* name global[1][0][...][n]WASMGlobalInstanceWASMExportMemInstancechar* name memory(WASMMemoryInstance*)[1][0][...][n]WASMMemoryInstanceWASMExportTabInstancechar* name table[1][0][...][n](WASMTableInstance *)WASMFunctionInstance[..]WASMFunction (per module)WASMModule::functionsWASMModule::globalsWASMModule::tablesWASMModule::memoriesWASMFunction **functions;export_global_countexport_memory_countexport_table_count \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/wasm_function.excalidraw b/wasm-micro-runtime/core/iwasm/doc/images/wasm_function.excalidraw new file mode 100644 index 0000000..8c59bdb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/wasm_function.excalidraw @@ -0,0 +1,7754 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "elements": [ + { + "type": "rectangle", + "version": 215, + "versionNonce": 1880133867, + "isDeleted": false, + "id": "4o9JHPgrKQSqK-ibc0qs_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2254.5001678466797, + "y": 1663.1666412353516, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 150.66668701171875, + "height": 43.000030517578125, + "seed": 4270136, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "PsNatrhOR918YjbmvyT9x", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 230, + "versionNonce": 1135445829, + "isDeleted": false, + "id": "94gfo2BctxYsRP6cuuIbI", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1297.1946105957031, + "y": 1755.6666768391929, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 214.33333333333348, + "height": 122.66668701171875, + "seed": 305330883, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "9vnyulmvSUCDWXvSKKyJ6", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 162, + "versionNonce": 1027947403, + "isDeleted": false, + "id": "JWlL3nHzTP4pxrEVYolFx", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2226.416498184204, + "y": 1001.3333435058594, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 278.6666259765625, + "height": 244.00003051757812, + "seed": 1502608995, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "70jp9eV1jV2_kUBbN055m", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 115, + "versionNonce": 291919525, + "isDeleted": false, + "id": "Cc8lshSSJ7lAY4RPjzS5A", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1289.1666717529297, + "y": 1285.8333587646484, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 178, + "height": 20, + "seed": 411108936, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMFunctionInstance", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunctionInstance" + }, + { + "type": "rectangle", + "version": 100, + "versionNonce": 1607176747, + "isDeleted": false, + "id": "onh-wm6wgKrkH94fakl6O", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1291.1666107177734, + "y": 1309.1666717529297, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 259.33331298828125, + "height": 210.3333435058593, + "seed": 2073695304, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "fltFoJAyfls8KPkBw6X_P", + "type": "arrow" + }, + { + "id": "MLVyGZQLa4jU554J6bsmJ", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 113, + "versionNonce": 34486789, + "isDeleted": false, + "id": "eAtZfC7P3ZafizTDITzz-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1363.1666717529297, + "y": 1335.500015258789, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 10, + "height": 12.333343505859375, + "seed": 136923720, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 139, + "versionNonce": 411964619, + "isDeleted": false, + "id": "GmBiArFp8A3Q9NaPWxp3Q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1395.8332977294922, + "y": 1332.8333587646484, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 94, + "height": 20, + "seed": 258177848, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "nUF7GyfmAGZN3iZvBfYtq", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func_import", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func_import" + }, + { + "type": "text", + "version": 239, + "versionNonce": 245543269, + "isDeleted": false, + "id": "YdCXv7DFgrOOU2wowasgm", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1748.499984741211, + "y": 1142.4999237060547, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 164, + "height": 20, + "seed": 1678600520, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMFunctionImport:", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunctionImport:" + }, + { + "type": "rectangle", + "version": 254, + "versionNonce": 656884587, + "isDeleted": false, + "id": "1Oi1iFkA8pBbaQpvduCq1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1738.499984741211, + "y": 1161.1666717529297, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 176, + "height": 200.33340454101554, + "seed": 1244990536, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "nUF7GyfmAGZN3iZvBfYtq", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 701, + "versionNonce": 879836357, + "isDeleted": false, + "id": "nUF7GyfmAGZN3iZvBfYtq", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1502.280048689805, + "y": 1332.5712493918486, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 234.84228393034437, + "height": 170.17912641351631, + "seed": 213606712, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "startBinding": { + "elementId": "GmBiArFp8A3Q9NaPWxp3Q", + "focus": 0.8789861743715494, + "gap": 14.558826765976846 + }, + "endBinding": { + "elementId": "w7-3leJjFtbtDhwDpkUjF", + "focus": 0.9853865103923569, + "gap": 6.774518257019281 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 157.46964422706515, + -143.2377075217314 + ], + [ + 234.84228393034437, + -170.17912641351631 + ] + ] + }, + { + "type": "text", + "version": 197, + "versionNonce": 1944165899, + "isDeleted": false, + "id": "bM6INpWVFBvURve3zdbdf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1396.1666717529297, + "y": 1355.5000457763672, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 34, + "height": 20, + "seed": 379599688, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "78xSb96N8EcRm2LhdCNjJ", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func" + }, + { + "type": "diamond", + "version": 140, + "versionNonce": 1796693029, + "isDeleted": false, + "id": "3cBYbIbQEyvFuXBMoyida", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1364.8332977294922, + "y": 1360.3333435058594, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 10, + "height": 12.333343505859375, + "seed": 1837623096, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 310, + "versionNonce": 2042078379, + "isDeleted": false, + "id": "9KTm6XzaqX3iZ90_AL3RN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1900.8332977294922, + "y": 1530.166732788086, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 211, + "height": 20, + "seed": 1970637640, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "Bnf72M4RGMZjNgBlzk90B", + "type": "arrow" + }, + { + "id": "78xSb96N8EcRm2LhdCNjJ", + "type": "arrow" + }, + { + "id": "S1cN82-5SoSu4e4SWfAUw", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMFunction (per module)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunction (per module)" + }, + { + "type": "rectangle", + "version": 261, + "versionNonce": 1518505861, + "isDeleted": false, + "id": "0kWlc6iPzGzhrfIVEPeOM", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1902.1667938232422, + "y": 1559.6667175292969, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 225.33337402343744, + "height": 200.9999389648438, + "seed": 2020723016, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "78xSb96N8EcRm2LhdCNjJ", + "type": "arrow" + }, + { + "id": "Bnf72M4RGMZjNgBlzk90B", + "type": "arrow" + }, + { + "id": "uC3ZSm-IltHDllxDGLJ9v", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 675, + "versionNonce": 1884542795, + "isDeleted": false, + "id": "zzj3BPEivUiaW1sqpfx7J", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2266.8334197998047, + "y": 1673.8333282470703, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 129, + "height": 20, + "seed": 26708536, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "78xSb96N8EcRm2LhdCNjJ", + "type": "arrow" + }, + { + "id": "PsNatrhOR918YjbmvyT9x", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "internal function", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "internal function" + }, + { + "type": "arrow", + "version": 666, + "versionNonce": 1357928165, + "isDeleted": false, + "id": "78xSb96N8EcRm2LhdCNjJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1434.5331571443298, + "y": 1377.528759465866, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 458.6267427864543, + "height": 186.82387510648414, + "seed": 535299656, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "startBinding": { + "elementId": "evHonaxVjRvjwFP08UX9x", + "focus": -0.827785929496437, + "gap": 16.471286310501227 + }, + "endBinding": { + "elementId": "9KTm6XzaqX3iZ90_AL3RN", + "focus": -1.317866862333605, + "gap": 14.985901784264229 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 184.63351460859985, + 58.63791228706373 + ], + [ + 458.6267427864543, + 186.82387510648414 + ] + ] + }, + { + "type": "rectangle", + "version": 145, + "versionNonce": 907064811, + "isDeleted": false, + "id": "WfHCEtd4eDaOU42FFn9wo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1341.1666717529297, + "y": 1321.500015258789, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 186.00006103515625, + "height": 65, + "seed": 663703880, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "78xSb96N8EcRm2LhdCNjJ", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 166, + "versionNonce": 2028876357, + "isDeleted": false, + "id": "QbAKq7kA_EZo7yxdpqj6F", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1742.499984741211, + "y": 1166.8332977294922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 158.66668701171872, + "height": 45.33337402343752, + "seed": 757681480, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 170, + "versionNonce": 2046345355, + "isDeleted": false, + "id": "w7-3leJjFtbtDhwDpkUjF", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1715.499984741211, + "y": 1169.1666412353516, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 187, + "height": 40, + "seed": 1177365064, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "nUF7GyfmAGZN3iZvBfYtq", + "type": "arrow" + }, + { + "id": "70jp9eV1jV2_kUBbN055m", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": " char *module_name;\n char *field_name;", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": " char *module_name;\n char *field_name;" + }, + { + "type": "rectangle", + "version": 190, + "versionNonce": 1331274149, + "isDeleted": false, + "id": "m6kQPfRjltByoJapP3m-h", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1744.499984741211, + "y": 1234.499984741211, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 155.33337402343747, + "height": 35.666656494140625, + "seed": 613948728, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 184, + "versionNonce": 633763627, + "isDeleted": false, + "id": "IXwlC5RgaF1iJPCtg6-Er", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1768.4999237060547, + "y": 1243.499984741211, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 125, + "height": 20, + "seed": 664043080, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "l4S1IXvHmVx_wl8DPQXk3", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func_ptr_linked", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func_ptr_linked" + }, + { + "type": "rectangle", + "version": 230, + "versionNonce": 1499473157, + "isDeleted": false, + "id": "o1vzUOCoilIzaDJdTpt35", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1735.8333587646484, + "y": 932.4999542236328, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 155.33331298828125, + "height": 90.33334350585938, + "seed": 1170302520, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "l4S1IXvHmVx_wl8DPQXk3", + "type": "arrow" + }, + { + "id": "j_Tg3JOansfDRNxNBUMfi", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 211, + "versionNonce": 132405707, + "isDeleted": false, + "id": "Kpjtivj-7LYLq1nuvC4KS", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1742.5000457763672, + "y": 940.8332977294922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 123, + "height": 20, + "seed": 1541878344, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "l4S1IXvHmVx_wl8DPQXk3", + "type": "arrow" + }, + { + "id": "j_Tg3JOansfDRNxNBUMfi", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "native function:", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "native function:" + }, + { + "type": "arrow", + "version": 755, + "versionNonce": 444546149, + "isDeleted": false, + "id": "l4S1IXvHmVx_wl8DPQXk3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1751.6334408235434, + "y": 1247.8332977294922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 121.88354763506686, + "height": 307.57912212433075, + "seed": 203809096, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "startBinding": { + "elementId": "OpV38MM8YOexWG_e-5_hX", + "focus": -0.4013706262952343, + "gap": 1.943772417380456 + }, + "endBinding": { + "elementId": "Kpjtivj-7LYLq1nuvC4KS", + "focus": 1.1040701980735919, + "gap": 6.6751580517609455 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -121.88354763506686, + -191.16677856445312 + ], + [ + -15.808553098937182, + -307.57912212433075 + ] + ] + }, + { + "type": "rectangle", + "version": 185, + "versionNonce": 1030547563, + "isDeleted": false, + "id": "5EDlNHzjVbZ3Rx8ny3SUe", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1742.6665802001953, + "y": 1279.1666412353516, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 165.16677856445304, + "height": 73.666748046875, + "seed": 202955336, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 241, + "versionNonce": 1250191301, + "isDeleted": false, + "id": "wc-s6ecXMbmh2ViGrkOa4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1757.8332977294922, + "y": 1289.5000457763672, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 152, + "height": 40, + "seed": 1536408648, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "S1cN82-5SoSu4e4SWfAUw", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "import_module;\nimport_func_linked;", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "import_module;\nimport_func_linked;" + }, + { + "type": "arrow", + "version": 788, + "versionNonce": 1546298123, + "isDeleted": false, + "id": "S1cN82-5SoSu4e4SWfAUw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1741.963485458681, + "y": 1322.4954523286892, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 248.14874440629774, + "height": 238.16542426722026, + "seed": 1657735240, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "startBinding": { + "elementId": "-vw33v9u2Ko2ayilI0CXG", + "focus": 2.0635763351494516, + "gap": 4.200185998866319 + }, + "endBinding": { + "elementId": "9KTm6XzaqX3iZ90_AL3RN", + "focus": -1.1514950968199016, + "gap": 11.294143807823616 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -88.79684422332912, + 88.33793695353734 + ], + [ + 159.35190018296862, + 238.16542426722026 + ] + ] + }, + { + "type": "text", + "version": 234, + "versionNonce": 1267541797, + "isDeleted": false, + "id": "jF8UBxFom2hco1NronB-_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1639.2142922537669, + "y": 1510.404782976423, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 137, + "height": 20, + "seed": 548488008, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(WASMFunction *)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(WASMFunction *)" + }, + { + "type": "diamond", + "version": 76, + "versionNonce": 1553467819, + "isDeleted": false, + "id": "jJOCFAslIe3JYgWsRnIKx", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1915.8334197998047, + "y": 1577.1667022705078, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 17.33331298828125, + "height": 15.666656494140625, + "seed": 1470386504, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "id": "PsNatrhOR918YjbmvyT9x", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 88, + "versionNonce": 1611070085, + "isDeleted": false, + "id": "V1RSJPoudlOhc-GaAxPSa", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1947.8334197998047, + "y": 1577.500015258789, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 37, + "height": 20, + "seed": 1209849672, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "PsNatrhOR918YjbmvyT9x", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "code", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "code" + }, + { + "type": "rectangle", + "version": 217, + "versionNonce": 1138728011, + "isDeleted": false, + "id": "tXHmR9TBkCnxMdo_pd0XG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2255.833480834961, + "y": 1603.1666412353516, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 149.3333740234375, + "height": 114.33340454101562, + "seed": 321892664, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 182, + "versionNonce": 548364773, + "isDeleted": false, + "id": "Zsg-OsrfZ2doJiTuOOpPu", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2277.166793823242, + "y": 1576.1666107177734, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70, + "height": 20, + "seed": 40194872, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "bytecode", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "bytecode" + }, + { + "type": "arrow", + "version": 545, + "versionNonce": 232512235, + "isDeleted": false, + "id": "PsNatrhOR918YjbmvyT9x", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1990.5000762939453, + "y": 1589.1666717529297, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 264.9616597351928, + "height": 70.73153780068333, + "seed": 1716226360, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "startBinding": { + "elementId": "V1RSJPoudlOhc-GaAxPSa", + "focus": -0.13746287298855567, + "gap": 7.9146728515625 + }, + "endBinding": { + "elementId": "4o9JHPgrKQSqK-ibc0qs_", + "focus": 0.060950944035830956, + "gap": 3.268431681738548 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 149.9165802001953, + 22.1666259765625 + ], + [ + 191.24952507019043, + 48.83355712890625 + ], + [ + 264.9616597351928, + 70.73153780068333 + ] + ] + }, + { + "type": "diamond", + "version": 79, + "versionNonce": 848551237, + "isDeleted": false, + "id": "sWaZg1Dcn3g0Qfdc7onW_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1916.5001068115234, + "y": 1629.8333892822266, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 18.6666259765625, + "height": 15, + "seed": 238700600, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 84, + "versionNonce": 1461142923, + "isDeleted": false, + "id": "U5NUD0rdNNDuY14J1ZRMu", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1954.8333587646484, + "y": 1628.8333892822266, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 110, + "height": 20, + "seed": 744475464, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "code_compiled", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "code_compiled" + }, + { + "type": "rectangle", + "version": 431, + "versionNonce": 1024584869, + "isDeleted": false, + "id": "gk39IBXF-F8MrwhzL35C6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2462.5001068115234, + "y": 1639.5001373291016, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 114.66674804687503, + "height": 119.33334350585936, + "seed": 201839672, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 444, + "versionNonce": 1187798059, + "isDeleted": false, + "id": "ZRckuyHrC8P5etlpd8MWT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2443.166793823242, + "y": 1616.833480834961, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 161, + "height": 20, + "seed": 2016979784, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "precompiled-bytecode", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "precompiled-bytecode" + }, + { + "type": "rectangle", + "version": 437, + "versionNonce": 534188037, + "isDeleted": false, + "id": "qGjmy62MZbL7zALsNU2dL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2463.166793823242, + "y": 1699.5001373291016, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 114.00006103515625, + "height": 43.000030517578125, + "seed": 1165004088, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "iqe6U9xOE6ECb18moJnw-", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 68, + "versionNonce": 954572491, + "isDeleted": false, + "id": "SiZHwNA9YKd93iwc74wxB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1942.4999542236328, + "y": 1667.833480834961, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 175, + "height": 20, + "seed": 1708294472, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "HCvGV6j45DG_BGeI3J7ut", + "type": "arrow" + } + ], + "updated": 1679533929331, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "fast_jit_jitted_code", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "fast_jit_jitted_code" + }, + { + "type": "diamond", + "version": 85, + "versionNonce": 1759561573, + "isDeleted": false, + "id": "ODI38iO5_5uu6RobQgkVa", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1912.499984741211, + "y": 1672.3333892822266, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 18.6666259765625, + "height": 15, + "seed": 297387080, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 93, + "versionNonce": 670928235, + "isDeleted": false, + "id": "gNxfqnpn5KEfZX8dL5GX3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1913.499984741211, + "y": 1717.666732788086, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 18.6666259765625, + "height": 15, + "seed": 1391035448, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 99, + "versionNonce": 175936197, + "isDeleted": false, + "id": "sYSDwbrpZl0692xpkcG4z", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1946.499984741211, + "y": 1713.166763305664, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 143, + "height": 20, + "seed": 1305637176, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "RckpORzYftIA2k4VSkTaW", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "llvm_jit_func_ptr", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "llvm_jit_func_ptr" + }, + { + "type": "text", + "version": 450, + "versionNonce": 1465821195, + "isDeleted": false, + "id": "1iUf8pP_YoM8qCPpSEahX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2383.0000762939453, + "y": 1788.8333129882812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 142, + "height": 20, + "seed": 68451128, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "fast jitted coode", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "fast jitted coode" + }, + { + "type": "rectangle", + "version": 501, + "versionNonce": 1363669541, + "isDeleted": false, + "id": "qVsRlSPxTl9a8XrnRMl2a", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2405.6665802001953, + "y": 1827.4999389648438, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 103.33337402343747, + "height": 29.2, + "seed": 1374549064, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "HCvGV6j45DG_BGeI3J7ut", + "type": "arrow" + }, + { + "id": "BF2h7Ub5gAf2yYxLfQSFh", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 447, + "versionNonce": 1023635115, + "isDeleted": false, + "id": "bu6GnR96DeCSFWu3DhMIJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2419.499954223633, + "y": 1890.1666870117188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 133, + "height": 20, + "seed": 713214264, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "RckpORzYftIA2k4VSkTaW", + "type": "arrow" + }, + { + "id": "kjpM2qWJqDrV-jr9XK8QR", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "llvm jitted coode", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "llvm jitted coode" + }, + { + "type": "rectangle", + "version": 395, + "versionNonce": 439379333, + "isDeleted": false, + "id": "Z9cJSNSjDvW2uPNthdOW9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2406.1666412353516, + "y": 1916.1666870117188, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 150.66668701171875, + "height": 43.000030517578125, + "seed": 1255376456, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "4E9MghnYo6hn8E82pmPMe", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 266, + "versionNonce": 731275595, + "isDeleted": false, + "id": "evHonaxVjRvjwFP08UX9x", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1335.1666259765625, + "y": 1394.0000457763672, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 187.33343505859372, + "height": 73.666748046875, + "seed": 873572680, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "78xSb96N8EcRm2LhdCNjJ", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 258, + "versionNonce": 1215524069, + "isDeleted": false, + "id": "4R5zAaFl-qG-PwNUPfyXq", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1360.4999389648438, + "y": 1404.6667022705078, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 157, + "height": 40, + "seed": 237924152, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "import_module_inst;\nimport_func_inst;", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "import_module_inst;\nimport_func_inst;" + }, + { + "type": "diamond", + "version": 132, + "versionNonce": 1014104043, + "isDeleted": false, + "id": "-cXxAxf0DBRaXH85Wn82G", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1341.4998168945312, + "y": 1408.6666564941406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 10, + "height": 12.333343505859375, + "seed": 1993562680, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "MLVyGZQLa4jU554J6bsmJ", + "type": "arrow" + }, + { + "id": "9vnyulmvSUCDWXvSKKyJ6", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 136, + "versionNonce": 2073987141, + "isDeleted": false, + "id": "V3cEII-BWPnk8MmO8v9Hw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1342.8331909179688, + "y": 1431.6666564941406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 10, + "height": 12.333343505859375, + "seed": 374375752, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "MLVyGZQLa4jU554J6bsmJ", + "type": "arrow" + }, + { + "id": "9vnyulmvSUCDWXvSKKyJ6", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 196, + "versionNonce": 2040580747, + "isDeleted": false, + "id": "jccHI4GP5zADwZpJ6_F0e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1020.4999542236328, + "y": 1294.5002899169922, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 132.66668701171866, + "height": 34.3333740234375, + "seed": 1209325128, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "eyNKBEqdZDGI0jikxT-7-" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 28, + "versionNonce": 1577034445, + "isDeleted": false, + "id": "eyNKBEqdZDGI0jikxT-7-", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1050.8332977294922, + "y": 1302.066976928711, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 72, + "height": 20, + "seed": 13177699, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247195, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "functions", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "jccHI4GP5zADwZpJ6_F0e", + "originalText": "functions" + }, + { + "type": "diamond", + "version": 157, + "versionNonce": 1205682475, + "isDeleted": false, + "id": "DpQKmgYqIbva-oudQDKVA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1029.8332977294922, + "y": 1302.8336334228516, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1787304760, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "0dTwoTnwCJUbyz3e0bm1O", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 460, + "versionNonce": 544406277, + "isDeleted": false, + "id": "fltFoJAyfls8KPkBw6X_P", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1152.0382824623548, + "y": 1307.393701716202, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 140.3213191272912, + "height": 0.8215748529805751, + "seed": 1890946120, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "onh-wm6wgKrkH94fakl6O", + "focus": 1.0244280280971265, + "gap": 2.5945448897082315 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 140.3213191272912, + -0.8215748529805751 + ] + ] + }, + { + "type": "rectangle", + "version": 245, + "versionNonce": 277996491, + "isDeleted": false, + "id": "BpsyACMObLF20Fkti2Uqq", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1020.8332061767578, + "y": 1339.000228881836, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 130.00000000000009, + "height": 31, + "seed": 1664170040, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "22kjCR2ZOQmZQXYgnarEF" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 50, + "versionNonce": 1029340291, + "isDeleted": false, + "id": "22kjCR2ZOQmZQXYgnarEF", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1058.3332061767578, + "y": 1344.900228881836, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 55, + "height": 20, + "seed": 1753405955, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247196, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "globals", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "BpsyACMObLF20Fkti2Uqq", + "originalText": "globals" + }, + { + "type": "diamond", + "version": 201, + "versionNonce": 735654507, + "isDeleted": false, + "id": "66O-4o40vS3pLIeBqwFBW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1030.1665802001953, + "y": 1344.000228881836, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 231180104, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 336, + "versionNonce": 2136587717, + "isDeleted": false, + "id": "l2Sz8ohFcHQT7gWys1M9D", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 751.1665802001953, + "y": 1311.6668853759766, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 158, + "height": 32, + "seed": 1571295288, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "o-cON41NQVHvHqkgeW_6m" + }, + { + "id": "0dTwoTnwCJUbyz3e0bm1O", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 115, + "versionNonce": 956023085, + "isDeleted": false, + "id": "o-cON41NQVHvHqkgeW_6m", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 824.6665802001953, + "y": 1317.1668853759766, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 11, + "height": 20, + "seed": 1939939267, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247196, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "e", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "l2Sz8ohFcHQT7gWys1M9D", + "originalText": "e" + }, + { + "type": "diamond", + "version": 340, + "versionNonce": 659012901, + "isDeleted": false, + "id": "_eiwTSAQSXB8P2k-PDhNa", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 769.8332061767578, + "y": 1323.6668548583984, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1019883336, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "0dTwoTnwCJUbyz3e0bm1O", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 238, + "versionNonce": 536466347, + "isDeleted": false, + "id": "OLHiYmF0KqdbZBgecyb7T", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1017.1665802001953, + "y": 1430.6669158935547, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 131.33337402343759, + "height": 32.33331298828125, + "seed": 1564507192, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 172, + "versionNonce": 1409208453, + "isDeleted": false, + "id": "qUAVJJGhHiEU00yvj5Xwy", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1041.1665802001953, + "y": 1438.3335723876953, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1916699464, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 170, + "versionNonce": 209200715, + "isDeleted": false, + "id": "ZzKApm4TYPqV3EGJbMuQ3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1001.6664886474609, + "y": 1253.9168853759766, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 214, + "height": 20, + "seed": 1061991496, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModuleInstanceExtra", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstanceExtra" + }, + { + "type": "rectangle", + "version": 195, + "versionNonce": 859600869, + "isDeleted": false, + "id": "SWgPVXvgjtN7AVlTfBUDR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1009.2498474121094, + "y": 1279.9169082641602, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 155, + "height": 200.41667938232422, + "seed": 1419979080, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "0dTwoTnwCJUbyz3e0bm1O", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 389, + "versionNonce": 114500843, + "isDeleted": false, + "id": "3rcvtpnHrIvCrE__yw5GU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1339.5831756591797, + "y": 1031.000186920166, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 174, + "height": 40, + "seed": 1610110792, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "AuwWYqGK5XChc2C2ZDCOd", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModuleInstance::\nimport_func_ptrs", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance::\nimport_func_ptrs" + }, + { + "type": "rectangle", + "version": 137, + "versionNonce": 1594766149, + "isDeleted": false, + "id": "IK-a-uPI373j3VM1YHdKy", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1390.1666870117188, + "y": 1107.625286102295, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1938455864, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "dA4sHNYw9NqC0yRSl1ByC", + "type": "arrow" + }, + { + "id": "AuwWYqGK5XChc2C2ZDCOd", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 123, + "versionNonce": 110448523, + "isDeleted": false, + "id": "xEmeSz_qg3vcIV0Kjo_4r", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1414.1666870117188, + "y": 1115.2919425964355, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1045500488, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "AuwWYqGK5XChc2C2ZDCOd", + "type": "arrow" + }, + { + "id": "j_Tg3JOansfDRNxNBUMfi", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 158, + "versionNonce": 1406239397, + "isDeleted": false, + "id": "u6aTea5vKrjtv4E0SAr_D", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1389.8333129882812, + "y": 1140.7919120788574, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1488328248, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "j_Tg3JOansfDRNxNBUMfi", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 144, + "versionNonce": 771302955, + "isDeleted": false, + "id": "6petv8iQ_6W4RCCyGVs86", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1413.8333129882812, + "y": 1148.458568572998, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1174899016, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "j_Tg3JOansfDRNxNBUMfi", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 146, + "versionNonce": 640935429, + "isDeleted": false, + "id": "HWwjwJX9MC7vOni7CdfwU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1389.5, + "y": 1174.1252555847168, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 654741304, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "j_Tg3JOansfDRNxNBUMfi", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 158, + "versionNonce": 1944660171, + "isDeleted": false, + "id": "3V6VqEBHUeHRXm4rtMN9P", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1388.1666259765625, + "y": 1201.7919120788574, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 652208184, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 138, + "versionNonce": 2033376613, + "isDeleted": false, + "id": "s3LweJMlqb3u8zFFOtRWC", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1410.8333129882812, + "y": 1210.1252555847168, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 630970184, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 748, + "versionNonce": 234870635, + "isDeleted": false, + "id": "j_Tg3JOansfDRNxNBUMfi", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1423.3266279235036, + "y": 1116.6584335466105, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 310.5923414580757, + "height": 177.73696684706545, + "seed": 561180216, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "startBinding": { + "elementId": "xEmeSz_qg3vcIV0Kjo_4r", + "focus": -0.6524247058306003, + "gap": 2.5695135809208747 + }, + "endBinding": { + "elementId": "Kpjtivj-7LYLq1nuvC4KS", + "focus": 1.1530360747961412, + "gap": 8.581076394787942 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 125.75645618294175, + -55.99185334641493 + ], + [ + 310.5923414580757, + -177.73696684706545 + ] + ] + }, + { + "type": "diamond", + "version": 115, + "versionNonce": 874244293, + "isDeleted": false, + "id": "SosCUEMFvN64e8eYhtj4S", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1748.0836486816406, + "y": 1289.8335762023926, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 8.333282470703125, + "height": 13.75, + "seed": 1668126536, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "S1cN82-5SoSu4e4SWfAUw", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 119, + "versionNonce": 445100555, + "isDeleted": false, + "id": "-vw33v9u2Ko2ayilI0CXG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1745.5836486816406, + "y": 1319.8335762023926, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 10.833282470703125, + "height": 9.583320617675781, + "seed": 2012578120, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "S1cN82-5SoSu4e4SWfAUw", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 648, + "versionNonce": 2022475813, + "isDeleted": false, + "id": "MLVyGZQLa4jU554J6bsmJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1347.8331909179688, + "y": 1415.2017225151058, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 179.41659545898438, + "height": 304.1667256414903, + "seed": 1591654456, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "startBinding": { + "elementId": "V3cEII-BWPnk8MmO8v9Hw", + "focus": 3.669987091693772, + "gap": 10.369642504898593 + }, + "endBinding": { + "elementId": "Rcref7JZ-AhlcXLLdwY5D", + "focus": -0.4568695235814372, + "gap": 6.321478788304148 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -179.41659545898438, + 169.4649187202458 + ], + [ + -89.48586426600582, + 304.1667256414903 + ] + ] + }, + { + "type": "text", + "version": 807, + "versionNonce": 175847595, + "isDeleted": false, + "id": "tUs9waDW6sL0AbyoZYJ5Q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 867.8295186360679, + "y": 823.9724260965983, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 672, + "height": 160, + "seed": 709576760, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "this is the one actually referred during executing opcode\n\nduring model load, if import can be solved through the native api registeration,\nthe pointer of native function will be filled.\n\nc-api could change the pointer later, then it will point to a different native function\n\nNULL: means multi-module import, go to \"import_func_inst\" field for target function", + "baseline": 154, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "this is the one actually referred during executing opcode\n\nduring model load, if import can be solved through the native api registeration,\nthe pointer of native function will be filled.\n\nc-api could change the pointer later, then it will point to a different native function\n\nNULL: means multi-module import, go to \"import_func_inst\" field for target function" + }, + { + "type": "rectangle", + "version": 410, + "versionNonce": 1483881349, + "isDeleted": false, + "id": "kARKLR5va5hDwe-2JCl7d", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 855.623291015625, + "y": 824.4128579639253, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 691.1904471261159, + "height": 170.05954742431626, + "seed": 1355113288, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "AuwWYqGK5XChc2C2ZDCOd", + "type": "arrow" + }, + { + "id": "j_Tg3JOansfDRNxNBUMfi", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 1427, + "versionNonce": 1267002187, + "isDeleted": false, + "id": "AuwWYqGK5XChc2C2ZDCOd", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1132.2583561737392, + "y": 997.6315427986297, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 188.93306844281233, + "height": 54.67370578283135, + "seed": 1524992312, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "startBinding": { + "elementId": "kARKLR5va5hDwe-2JCl7d", + "gap": 3.1591374103880994, + "focus": 0.5842950344963632 + }, + "endBinding": { + "elementId": "3rcvtpnHrIvCrE__yw5GU", + "gap": 18.39175104262814, + "focus": -0.7039875993615579 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 188.93306844281233, + 54.67370578283135 + ] + ] + }, + { + "type": "text", + "version": 363, + "versionNonce": 1275757285, + "isDeleted": false, + "id": "_pYmb7H1lxRLE4Qw_MajA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1034.3336715698242, + "y": 1680.4170036315918, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 176, + "height": 20, + "seed": 764509256, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModuleInstance*", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance*" + }, + { + "type": "text", + "version": 314, + "versionNonce": 838814187, + "isDeleted": false, + "id": "GBXnnFKM_76tjt1WAlYnV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1250.0279070536296, + "y": 1605.2504641215007, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 198, + "height": 20, + "seed": 1630119496, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "9vnyulmvSUCDWXvSKKyJ6", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(WASMFunctionInstance*)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(WASMFunctionInstance*)" + }, + { + "type": "arrow", + "version": 881, + "versionNonce": 1742921285, + "isDeleted": false, + "id": "9vnyulmvSUCDWXvSKKyJ6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1347.3662637658592, + "y": 1437.871212796838, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 109.83801974730454, + "height": 307.7636946939249, + "seed": 422577480, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "startBinding": { + "elementId": "-cXxAxf0DBRaXH85Wn82G", + "focus": -3.9115852992052806, + "gap": 11.29853538112821 + }, + "endBinding": { + "elementId": "94gfo2BctxYsRP6cuuIbI", + "gap": 10.033975741878294, + "focus": -0.6520703073991437 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -109.83801974730454, + 135.3790961936404 + ], + [ + -51.493209521376, + 307.7636946939249 + ] + ] + }, + { + "type": "diamond", + "version": 207, + "versionNonce": 2116029227, + "isDeleted": false, + "id": "kliwot061_yUhDMDbVbQe", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1354.3614679972331, + "y": 1779.417065938314, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 10, + "height": 12.333343505859375, + "seed": 82951992, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 233, + "versionNonce": 2082066693, + "isDeleted": false, + "id": "mvfNSPpLgMxaEaQuXYPrl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1387.0280939737956, + "y": 1776.7504094441733, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 94, + "height": 20, + "seed": 1266085960, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func_import", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func_import" + }, + { + "type": "text", + "version": 265, + "versionNonce": 1206611403, + "isDeleted": false, + "id": "Oe13Ts1dEazuz8ilwnZiL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1398.6947809855144, + "y": 1806.7504094441733, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 34, + "height": 20, + "seed": 2145720376, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func" + }, + { + "type": "diamond", + "version": 230, + "versionNonce": 287533157, + "isDeleted": false, + "id": "nyquujDKZGyYvDs-a_GQ-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1356.0280939737956, + "y": 1811.5837071736655, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 10, + "height": 12.333343505859375, + "seed": 784443208, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 252, + "versionNonce": 326204523, + "isDeleted": false, + "id": "cGQYK5rXelrtuGolytFol", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1332.3614679972331, + "y": 1765.417065938314, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 154.00006103515616, + "height": 79, + "seed": 1426411832, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "9vnyulmvSUCDWXvSKKyJ6", + "type": "arrow" + } + ], + "updated": 1679533929332, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 521, + "versionNonce": 868422597, + "isDeleted": false, + "id": "VRweEUgFB9qqzjdTwpHnK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1323.694574991862, + "y": 1739.194897969564, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 117, + "height": 20, + "seed": 155802936, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929332, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMFunction ", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunction " + }, + { + "type": "arrow", + "version": 621, + "versionNonce": 746370827, + "isDeleted": false, + "id": "CUEfVWpVIuIHc_5h3VskN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1459.8716446745273, + "y": 1829.1380187988277, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 394.2706533635544, + "height": 82.19527893066447, + "seed": 1854588984, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "epVvbDyPF40MaERFnDDJy", + "focus": 0.2506598246466399, + "gap": 9.849883651733307 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 153.21163779617586, + 82.19527893066447 + ], + [ + 394.2706533635544, + 27.990782200797412 + ] + ] + }, + { + "type": "text", + "version": 309, + "versionNonce": 1880511269, + "isDeleted": false, + "id": "OTP338ttAjF_rIJwNKA1S", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1298.416748046875, + "y": 1343.3333587646484, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 38, + "height": 20, + "seed": 1994699981, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "union", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "union" + }, + { + "type": "rectangle", + "version": 62, + "versionNonce": 25664939, + "isDeleted": false, + "id": "7kmKkWcfD2eeTgLmci_TA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1289.75, + "y": 1517.3333282470703, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 260.66668701171875, + "height": 61.333343505859375, + "seed": 1810842211, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "fr9-3bNKWz24759an1jPs" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 13, + "versionNonce": 914224163, + "isDeleted": false, + "id": "fr9-3bNKWz24759an1jPs", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1404.5833435058594, + "y": 1538.4, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 31, + "height": 20, + "seed": 1884299875, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247200, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[...]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "7kmKkWcfD2eeTgLmci_TA", + "originalText": "[...]" + }, + { + "type": "line", + "version": 176, + "versionNonce": 1373083019, + "isDeleted": false, + "id": "pYd8BNqe32DQ1BK15gl1X", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1607.7500610351565, + "y": 911.2331604003898, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 2.2737367544323206e-13, + "height": 1207.7780151367188, + "seed": 1889947117, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533932559, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + -2.2737367544323206e-13, + 1207.7780151367188 + ] + ] + }, + { + "type": "text", + "version": 58, + "versionNonce": 2111671781, + "isDeleted": false, + "id": "-EA8TtLR5unZFdjT-k9mq", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1404.4167785644531, + "y": 1492.6665802001953, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 20, + "seed": 1174452173, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[0]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "text", + "version": 225, + "versionNonce": 49704683, + "isDeleted": false, + "id": "al3s0Ce-XZ_FiU84jmv0f", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 729.2799504597981, + "y": 1209.9554656982423, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 171, + "height": 19, + "seed": 1330045933, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMModuleInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance" + }, + { + "type": "rectangle", + "version": 102, + "versionNonce": 555518277, + "isDeleted": false, + "id": "TxSoCCktw7hU7jU0cN2he", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 723.0834655761719, + "y": 1240.9998931884766, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 210.66668701171875, + "height": 305.33334350585943, + "seed": 598232163, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 113, + "versionNonce": 1941920139, + "isDeleted": false, + "id": "uyhu5MiXRFgQansSAKgbz", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1858.638671875, + "y": 1857.11110941569, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 229.3333740234375, + "height": 71.33331298828125, + "seed": 1064313101, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "4W7-B8UsG2Kp0-6eEN0-h" + }, + { + "id": "CUEfVWpVIuIHc_5h3VskN", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 67, + "versionNonce": 1873089421, + "isDeleted": false, + "id": "4W7-B8UsG2Kp0-6eEN0-h", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1959.8053588867188, + "y": 1883.1777659098307, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 27, + "height": 20, + "seed": 1206387267, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247203, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[..]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "uyhu5MiXRFgQansSAKgbz", + "originalText": "[..]" + }, + { + "type": "arrow", + "version": 335, + "versionNonce": 1028780075, + "isDeleted": false, + "id": "0dTwoTnwCJUbyz3e0bm1O", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 780.7038274166374, + "y": 1329.8562414550568, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 227.1615634556406, + "height": 49.895659740248675, + "seed": 1282951939, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "startBinding": { + "elementId": "_eiwTSAQSXB8P2k-PDhNa", + "focus": -0.16162303890895813, + "gap": 1.5411549516792498 + }, + "endBinding": { + "elementId": "SWgPVXvgjtN7AVlTfBUDR", + "focus": 1.0022214255263049, + "gap": 1.3844565398313762 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 227.1615634556406, + -49.895659740248675 + ] + ] + }, + { + "type": "rectangle", + "version": 380, + "versionNonce": 1831123973, + "isDeleted": false, + "id": "EjfvS52mTJV_ntE7FAmqi", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 749.41650390625, + "y": 1265.500015258789, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 163, + "height": 38.333343505859375, + "seed": 908455875, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "0dTwoTnwCJUbyz3e0bm1O", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 372, + "versionNonce": 395899749, + "isDeleted": false, + "id": "0n8DknkuWXlRynFJmYm-N", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 760.0831298828125, + "y": 1276.500015258789, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1140391779, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "dA4sHNYw9NqC0yRSl1ByC", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 131, + "versionNonce": 169804459, + "isDeleted": false, + "id": "xwVumJFL4-d-8BM7nSKzG", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1406.5276896158855, + "y": 1849.9999745686848, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 25, + "height": 20, + "seed": 813534477, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[n]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[n]" + }, + { + "type": "arrow", + "version": 164, + "versionNonce": 1833439621, + "isDeleted": false, + "id": "dA4sHNYw9NqC0yRSl1ByC", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 767.0830841064453, + "y": 1278.6665496826172, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 617.3333435058594, + "height": 168.00003051757812, + "seed": 1360269091, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "startBinding": { + "elementId": "0n8DknkuWXlRynFJmYm-N", + "focus": -0.6970200254594874, + "gap": 1 + }, + "endBinding": { + "elementId": "IK-a-uPI373j3VM1YHdKy", + "focus": 0.91934006004951, + "gap": 5.7502593994140625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 283.33331298828125, + -86.66671752929688 + ], + [ + 617.3333435058594, + -168.00003051757812 + ] + ] + }, + { + "type": "text", + "version": 370, + "versionNonce": 2015327563, + "isDeleted": false, + "id": "1uhr4l-w9t4eVo0gQ06SH", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1628.0831146240234, + "y": 1047.333236694336, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 60, + "height": 20, + "seed": 2127932547, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "70jp9eV1jV2_kUBbN055m", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(void *)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(void *)" + }, + { + "type": "diamond", + "version": 110, + "versionNonce": 867539173, + "isDeleted": false, + "id": "OpV38MM8YOexWG_e-5_hX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1751.5831604003906, + "y": 1244.7915496826172, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 8.333282470703125, + "height": 13.75, + "seed": 1876974509, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "l4S1IXvHmVx_wl8DPQXk3", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 201, + "versionNonce": 1052283883, + "isDeleted": false, + "id": "0VYDhNUTpNaSbemsP54Zy", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 982.6778793334961, + "y": 1174.3998931884767, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 68, + "height": 20, + "seed": 2080172771, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(void **)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(void **)" + }, + { + "type": "text", + "version": 52, + "versionNonce": 364590149, + "isDeleted": false, + "id": "mEKNbQ7XuwA2N2CRqO86h", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1395.7501068115234, + "y": 1178.6665954589844, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 43, + "height": 20, + "seed": 95270211, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "NULL", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "NULL" + }, + { + "type": "rectangle", + "version": 348, + "versionNonce": 306207333, + "isDeleted": false, + "id": "FKonkUbaqKFXKyt5VSiGE", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 743.0831654866537, + "y": 1476.5000508626301, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174.66668701171875, + "height": 31.666625976562507, + "seed": 710825357, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "0dTwoTnwCJUbyz3e0bm1O", + "type": "arrow" + } + ], + "updated": 1679533956228, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 376, + "versionNonce": 1951705707, + "isDeleted": false, + "id": "xNodTFHQFtS6jmRfbZDpo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 748.4164784749349, + "y": 1484.1667073567708, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1417026541, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533956228, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 82, + "versionNonce": 678544837, + "isDeleted": false, + "id": "8i8vmtgsTLeEQt_E_hLrk", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 768.4165089925131, + "y": 1481.0000508626301, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 149, + "height": 20, + "seed": 1944091203, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "yu3un9i0kAGrZ8bAnVMa3", + "type": "arrow" + } + ], + "updated": 1679533956228, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func_type_indexes", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func_type_indexes" + }, + { + "type": "rectangle", + "version": 210, + "versionNonce": 1441855435, + "isDeleted": false, + "id": "IwrJYgHbhcHGM-MPt77v3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 907.3989664713541, + "y": 1609.9723409016926, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 919532995, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 230, + "versionNonce": 2006154853, + "isDeleted": false, + "id": "XRMQmYFkm-FtbS8PBuF_X", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 907.0655924479166, + "y": 1643.1389668782551, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 361842019, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 218, + "versionNonce": 473286251, + "isDeleted": false, + "id": "nsMfEXFd6IeqB7fL4ngUa", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 906.7322794596354, + "y": 1676.4723103841145, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1923653891, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 227, + "versionNonce": 1069574597, + "isDeleted": false, + "id": "R5ioSAhKEdUlCurXbrHuN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 905.3989054361979, + "y": 1704.1389668782551, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1469286563, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 323, + "versionNonce": 617227531, + "isDeleted": false, + "id": "yu3un9i0kAGrZ8bAnVMa3", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 753.0831654866537, + "y": 1487.6672379829788, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 200.16657758128372, + "height": 153.3329247774377, + "seed": 1439562723, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533956229, + "link": null, + "locked": false, + "startBinding": { + "elementId": "8i8vmtgsTLeEQt_E_hLrk", + "focus": 1.0014597510870404, + "gap": 15.333343505859375 + }, + "endBinding": { + "elementId": "p5TPteQC3PraRMJtt4XsT", + "focus": 0.36261877029011863, + "gap": 3.9746450546211918 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -60.000050862630246, + 26.332884087333696 + ], + [ + -40.88889567057288, + 105.11070594605758 + ], + [ + 62.88881429036462, + 153.3329247774377 + ], + [ + 140.16652671865347, + 150.58490939994954 + ] + ] + }, + { + "type": "text", + "version": 99, + "versionNonce": 2113464613, + "isDeleted": false, + "id": "a8gv4R6T3PYjKJhay9vMv", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 802.4165191650391, + "y": 1604.7780354817708, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 70, + "height": 20, + "seed": 453646061, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint32 *", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "uint32 *" + }, + { + "type": "text", + "version": 97, + "versionNonce": 692152235, + "isDeleted": false, + "id": "eNJLe1mhF_AWRUyt9lO6k", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2227.083185195923, + "y": 955.9999694824219, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 96, + "height": 19, + "seed": 1711184323, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMModule", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule" + }, + { + "type": "rectangle", + "version": 247, + "versionNonce": 329940101, + "isDeleted": false, + "id": "25KRboddeZem953trnn-R", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2216.416498184204, + "y": 985.9999389648438, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 308, + "height": 568, + "seed": 847170787, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 135, + "versionNonce": 799881803, + "isDeleted": false, + "id": "NnvrV9DcfaaiOCGPUdP78", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2229.083246231079, + "y": 1003.3333129882812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 263, + "height": 224, + "seed": 2117157197, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": " WASMImport *import_tables;\n WASMImport *import_memories;\n WASMImport *import_globals;\n\n WASMType **types;\n WASMImport *imports;\n WASMTable *tables;\n WASMMemory *memories;\n WASMGlobal *globals;\n WASMExport *exports;\n WASMTableSeg *table_segments;\n WASMDataSeg **data_segments;", + "baseline": 220, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": " WASMImport *import_tables;\n WASMImport *import_memories;\n WASMImport *import_globals;\n\n WASMType **types;\n WASMImport *imports;\n WASMTable *tables;\n WASMMemory *memories;\n WASMGlobal *globals;\n WASMExport *exports;\n WASMTableSeg *table_segments;\n WASMDataSeg **data_segments;" + }, + { + "type": "rectangle", + "version": 123, + "versionNonce": 1486005221, + "isDeleted": false, + "id": "O5y5oOsgUB5ue4vWSQB0i", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2232.4165592193604, + "y": 1301.3332824707031, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 273.3333740234375, + "height": 44, + "seed": 1988414157, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 145, + "versionNonce": 709212395, + "isDeleted": false, + "id": "6xufIeHeKpvDChXCTeKSb", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2294.4165592193604, + "y": 1314.6665954589844, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 198, + "height": 19, + "seed": 2026661485, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMFunction **functions;", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunction **functions;" + }, + { + "type": "diamond", + "version": 108, + "versionNonce": 1354686277, + "isDeleted": false, + "id": "s4lQvHIN1eAAubp7DkNrk", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2245.083246231079, + "y": 1318, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 26.6666259765625, + "height": 19.333343505859375, + "seed": 1489582509, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "Bnf72M4RGMZjNgBlzk90B", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 354, + "versionNonce": 1184195467, + "isDeleted": false, + "id": "Bnf72M4RGMZjNgBlzk90B", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2250.416437149048, + "y": 1332.0000305175781, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 163.1120623945253, + "height": 15.460472501937602, + "seed": 1550987139, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "startBinding": { + "elementId": "s4lQvHIN1eAAubp7DkNrk", + "focus": -0.36983487209914556, + "gap": 1 + }, + "endBinding": { + "elementId": "swt6lb3ztkUJAvM41XHBK", + "focus": -0.7544161254824635, + "gap": 5.555002272222737 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -163.1120623945253, + 15.460472501937602 + ] + ] + }, + { + "type": "text", + "version": 138, + "versionNonce": 579645093, + "isDeleted": false, + "id": "TfktbM2ODt0uHXCbkJtbX", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2277.4166202545166, + "y": 1269.3333435058594, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 229, + "height": 19, + "seed": 2140950979, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMImport *import_functions;", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMImport *import_functions;" + }, + { + "type": "rectangle", + "version": 99, + "versionNonce": 483834411, + "isDeleted": false, + "id": "sIxEmRk_EPaTumCGaxM6t", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2231.749994277954, + "y": 1254.6667175292969, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 276, + "height": 42.6666259765625, + "seed": 2140310499, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 86, + "versionNonce": 1286181381, + "isDeleted": false, + "id": "_er3JaiUwljITdxfog0w_", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2246.416742324829, + "y": 1265.3334045410156, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 15.33331298828125, + "height": 22.66668701171875, + "seed": 568056877, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 453, + "versionNonce": 1509178571, + "isDeleted": false, + "id": "70jp9eV1jV2_kUBbN055m", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2253.0103282895525, + "y": 1275.9391811320388, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 323.49287390894006, + "height": 187.53914675447618, + "seed": 638625069, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "startBinding": { + "elementId": "JWlL3nHzTP4pxrEVYolFx", + "focus": -1.0762119763220488, + "gap": 30.60580710860131 + }, + "endBinding": { + "elementId": "1rZk-xFL82XSp5Mpcqy4f", + "focus": -1.2708836305762516, + "gap": 14.434116596798958 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -146.59352492956714, + -83.93927268477319 + ], + [ + -323.49287390894006, + -187.53914675447618 + ] + ] + }, + { + "type": "rectangle", + "version": 145, + "versionNonce": 471009637, + "isDeleted": false, + "id": "4BwxH9WndrjsXga95YTvm", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1705.7500858306885, + "y": 1086.6665954589844, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 222.6666259765625, + "height": 303.33331298828125, + "seed": 803769283, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "70jp9eV1jV2_kUBbN055m", + "type": "arrow" + }, + { + "id": "nUF7GyfmAGZN3iZvBfYtq", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 270, + "versionNonce": 696806251, + "isDeleted": false, + "id": "RR6mIlX4wkxcfcv6RQ8Wa", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1708.416711807251, + "y": 1062.6665954589844, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 97, + "height": 19, + "seed": 2042686211, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 2, + "text": "WASMImport", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMImport" + }, + { + "type": "rectangle", + "version": 191, + "versionNonce": 1645647045, + "isDeleted": false, + "id": "1rZk-xFL82XSp5Mpcqy4f", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1745.749963760376, + "y": 1097.3333129882812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 169.3333740234375, + "height": 31, + "seed": 176484109, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "qkiSeaixB8ivmpNxhvY6l" + }, + { + "id": "70jp9eV1jV2_kUBbN055m", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 162, + "versionNonce": 395409347, + "isDeleted": false, + "id": "qkiSeaixB8ivmpNxhvY6l", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1750.749963760376, + "y": 1102.3333129882812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 79, + "height": 20, + "seed": 2146735565, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247208, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint8 kind", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "1rZk-xFL82XSp5Mpcqy4f", + "originalText": "uint8 kind" + }, + { + "type": "rectangle", + "version": 175, + "versionNonce": 1783739429, + "isDeleted": false, + "id": "R_f4RWhd20C8JXmaJofMm", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1706.7500247955322, + "y": 1390.6665954589844, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 221, + "height": 31, + "seed": 691699363, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "jXOu2RGl7KTvcY3-hwIj2" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 56, + "versionNonce": 2117409261, + "isDeleted": false, + "id": "jXOu2RGl7KTvcY3-hwIj2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1806.2500247955322, + "y": 1396.0665954589845, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 22, + "height": 20, + "seed": 776900557, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247209, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[1]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "R_f4RWhd20C8JXmaJofMm", + "originalText": "[1]" + }, + { + "type": "text", + "version": 51, + "versionNonce": 816965509, + "isDeleted": false, + "id": "JvLqnDwgx42bo19rINn18", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1807.083490371704, + "y": 1371.9998474121094, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 20, + "seed": 2020174093, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[0]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "rectangle", + "version": 185, + "versionNonce": 1499856715, + "isDeleted": false, + "id": "Xj5n84LklLY7rIUbMpV30", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1708.2501163482666, + "y": 1422.3332061767578, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 221, + "height": 31, + "seed": 1499618403, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "LOWTAqc1KaVYNxnLkXHB4" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 61, + "versionNonce": 2125694819, + "isDeleted": false, + "id": "LOWTAqc1KaVYNxnLkXHB4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1803.2501163482666, + "y": 1427.733206176758, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 31, + "height": 20, + "seed": 701124429, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247210, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[...]", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Xj5n84LklLY7rIUbMpV30", + "originalText": "[...]" + }, + { + "type": "rectangle", + "version": 50, + "versionNonce": 178118123, + "isDeleted": false, + "id": "qd-T97QOO_xa4SGTgkrAj", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1363.083200454712, + "y": 1076.0001373291016, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 96, + "height": 170.66668701171875, + "seed": 215655011, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "AuwWYqGK5XChc2C2ZDCOd", + "type": "arrow" + } + ], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 169, + "versionNonce": 1562872389, + "isDeleted": false, + "id": "EAEAQzFaB6T9-rCB0X-61", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2303.749662399292, + "y": 1418.0001983642578, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 160, + "height": 20, + "seed": 1651418499, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "fast_jit_func_ptrs", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "fast_jit_func_ptrs" + }, + { + "type": "rectangle", + "version": 195, + "versionNonce": 312535179, + "isDeleted": false, + "id": "n3LUTZSRU6GJ4nfF98WsH", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2233.7496013641357, + "y": 1409.6669158935547, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 273.3333740234375, + "height": 44, + "seed": 1123016931, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 173, + "versionNonce": 312998309, + "isDeleted": false, + "id": "SOlRVdTv-nHYKomLm0-Lr", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2246.749662399292, + "y": 1421.000244140625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 26.6666259765625, + "height": 19.333343505859375, + "seed": 381478531, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929333, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 61, + "versionNonce": 1193910059, + "isDeleted": false, + "id": "RckpORzYftIA2k4VSkTaW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2093.082914352417, + "y": 1731.3334503173828, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 314.6666564941406, + "height": 191.3333740234375, + "seed": 1221862445, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": { + "elementId": "sYSDwbrpZl0692xpkcG4z", + "focus": -0.8132904069013918, + "gap": 6.05506706237793 + }, + "endBinding": { + "elementId": "bu6GnR96DeCSFWu3DhMIJ", + "focus": -1.5131314965155, + "gap": 13.30013732910163 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 150.66665649414062, + 128.66668701171875 + ], + [ + 314.6666564941406, + 191.3333740234375 + ] + ] + }, + { + "type": "arrow", + "version": 66, + "versionNonce": 1361280261, + "isDeleted": false, + "id": "HCvGV6j45DG_BGeI3J7ut", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2116.4162578582764, + "y": 1692.6667938232422, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 288.25032234191895, + "height": 141.39862277829707, + "seed": 298119629, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": { + "elementId": "SiZHwNA9YKd93iwc74wxB", + "focus": -0.7153529191190878, + "gap": 5.633312988281318 + }, + "endBinding": { + "elementId": "qVsRlSPxTl9a8XrnRMl2a", + "focus": -0.10195339456743455, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 125.33331298828125, + 108.66665649414062 + ], + [ + 288.25032234191895, + 141.39862277829707 + ] + ] + }, + { + "type": "arrow", + "version": 81, + "versionNonce": 1654817227, + "isDeleted": false, + "id": "iqe6U9xOE6ECb18moJnw-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2083.0828227996826, + "y": 1640.0001068115234, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 380, + "height": 114.66668701171875, + "seed": 1655019469, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "qGjmy62MZbL7zALsNU2dL", + "focus": 0.8815432234830397, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 202.66668701171875, + 114.66668701171875 + ], + [ + 380, + 64 + ] + ] + }, + { + "type": "rectangle", + "version": 196, + "versionNonce": 1880593509, + "isDeleted": false, + "id": "pWu_lwlXIT6mrZDyT6QkK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2690.0828075408936, + "y": 1522.750244140625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 162247299, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "NSz4yfxdToa5c5At8YYeR", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 181, + "versionNonce": 1847988331, + "isDeleted": false, + "id": "rBPIesLQ3_hwRCCEZDo_K", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2714.0828075408936, + "y": 1530.4169006347656, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 2113989421, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 216, + "versionNonce": 2068955077, + "isDeleted": false, + "id": "1nU3Tx2BdlitDqcIjhR6O", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2689.749433517456, + "y": 1555.9168701171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 255355427, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 202, + "versionNonce": 1803471627, + "isDeleted": false, + "id": "qPuFVkGJxscoxDIlxjxO7", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2713.749433517456, + "y": 1563.5835266113281, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1853160845, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 205, + "versionNonce": 686784293, + "isDeleted": false, + "id": "ijtdWkKxrTh4Pnlp14iE0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2689.416120529175, + "y": 1589.2502136230469, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 495791555, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "BF2h7Ub5gAf2yYxLfQSFh", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 215, + "versionNonce": 1059533227, + "isDeleted": false, + "id": "jFcNDR-eUAEW7pedWkAdo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2688.0827465057373, + "y": 1616.9168701171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 796378093, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 196, + "versionNonce": 257839749, + "isDeleted": false, + "id": "q4AkedYi9svz1WOMHGyiP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2710.749433517456, + "y": 1625.2502136230469, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 2009628003, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 195, + "versionNonce": 1514783819, + "isDeleted": false, + "id": "NSz4yfxdToa5c5At8YYeR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2491.7495250701904, + "y": 1437.3335571289062, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 199.43811857564833, + "height": 81.5190719332436, + "seed": 909964067, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "pWu_lwlXIT6mrZDyT6QkK", + "focus": 0.37183947794184286, + "gap": 3.8976150784751553 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 199.43811857564833, + 81.5190719332436 + ] + ] + }, + { + "type": "arrow", + "version": 222, + "versionNonce": 458996197, + "isDeleted": false, + "id": "BF2h7Ub5gAf2yYxLfQSFh", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2718.826031777619, + "y": 1626.7287320369785, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 207.0765067074285, + "height": 202.09118637438291, + "seed": 358411853, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": { + "elementId": "D0065Oo1uIjEqNG6iDJjn", + "focus": -2.935287320516118, + "gap": 14.961736017351072 + }, + "endBinding": { + "elementId": "qVsRlSPxTl9a8XrnRMl2a", + "focus": 0.3455990175110858, + "gap": 2.749570846557617 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -84.40981969570976, + 140.60479457434963 + ], + [ + -207.0765067074285, + 202.09118637438291 + ] + ] + }, + { + "type": "diamond", + "version": 207, + "versionNonce": 616628971, + "isDeleted": false, + "id": "D0065Oo1uIjEqNG6iDJjn", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2712.0828075408936, + "y": 1594.1669158935547, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1454945635, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "BF2h7Ub5gAf2yYxLfQSFh", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 61, + "versionNonce": 1173697861, + "isDeleted": false, + "id": "OI5mleCPF3WYtCzvosMk1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2661.7495861053467, + "y": 1495.3336181640625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 177.33334350585938, + "seed": 617283619, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 81, + "versionNonce": 655846795, + "isDeleted": false, + "id": "4Q_PdspTfAcKBYqfwEOrl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2534.4162731170654, + "y": 1478.0003051757812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 68, + "height": 20, + "seed": 468048995, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(void **)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(void **)" + }, + { + "type": "text", + "version": 215, + "versionNonce": 176702629, + "isDeleted": false, + "id": "zEBN6RuZylHRtnzhMAsfm", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2302.082899093628, + "y": 1367.6669006347656, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 80, + "height": 20, + "seed": 1726260589, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func_ptrs", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "func_ptrs" + }, + { + "type": "rectangle", + "version": 240, + "versionNonce": 1955038251, + "isDeleted": false, + "id": "Q1RbJ8FG5PuoKZ0jGW-AG", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2232.0828380584717, + "y": 1359.3336181640625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 273.3333740234375, + "height": 44, + "seed": 1664381923, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 218, + "versionNonce": 482502661, + "isDeleted": false, + "id": "lKOq2jf5D0WXwfNx9mwVb", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2245.082899093628, + "y": 1370.6669464111328, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 26.6666259765625, + "height": 19.333343505859375, + "seed": 1869049805, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 197, + "versionNonce": 1425786571, + "isDeleted": false, + "id": "5_VLE0dE94H4gwxL9iUpC", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2851.4160289764404, + "y": 1432.750228881836, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1330766563, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "8jHbZMq4zvKK9AxPUw9Qf", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 181, + "versionNonce": 1587617637, + "isDeleted": false, + "id": "k1p6Nabm5uz-_RQk8vKUy", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2875.4160289764404, + "y": 1440.4168853759766, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1901883597, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 216, + "versionNonce": 789493099, + "isDeleted": false, + "id": "BH3ZK-RnoIcnosroWZ9_J", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2851.082654953003, + "y": 1465.9168548583984, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 897158787, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 202, + "versionNonce": 376564421, + "isDeleted": false, + "id": "cAmZfC7YbkEqrmByhjjHI", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2875.082654953003, + "y": 1473.583511352539, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 596561709, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 205, + "versionNonce": 839569419, + "isDeleted": false, + "id": "yAeZ1rNDP97dzjt5edJvJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2850.7493419647217, + "y": 1499.2501983642578, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1022019107, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 215, + "versionNonce": 1691503141, + "isDeleted": false, + "id": "h_AbFbWb_N0KfOV68sJf6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2849.415967941284, + "y": 1526.9168548583984, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1889952141, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 196, + "versionNonce": 1072338603, + "isDeleted": false, + "id": "OKOuC260x6EDIVInqBKer", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2872.082654953003, + "y": 1535.2501983642578, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 479463875, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 208, + "versionNonce": 2058337669, + "isDeleted": false, + "id": "zQXps3l6_CULfrKz0sxFV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2873.4160289764404, + "y": 1504.1669006347656, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 769435629, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "4E9MghnYo6hn8E82pmPMe", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 61, + "versionNonce": 391678283, + "isDeleted": false, + "id": "4EdLS1fcNDOsz52Mf7ATN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2823.0828075408936, + "y": 1405.3336029052734, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 177.33334350585938, + "seed": 697555299, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 108, + "versionNonce": 256361701, + "isDeleted": false, + "id": "8jHbZMq4zvKK9AxPUw9Qf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2495.082899093628, + "y": 1376.0003051757812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 355.3333740234375, + "height": 55.999969482421875, + "seed": 424364771, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "5_VLE0dE94H4gwxL9iUpC", + "focus": 0.6186308498480104, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 355.3333740234375, + 55.999969482421875 + ] + ] + }, + { + "type": "text", + "version": 44, + "versionNonce": 1034889195, + "isDeleted": false, + "id": "PamsTN-BPmxTRCvWvz1kZ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2636.5496044158936, + "y": 1377.7335876464845, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 68, + "height": 20, + "seed": 508079075, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(void **)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(void **)" + }, + { + "type": "arrow", + "version": 101, + "versionNonce": 371712069, + "isDeleted": false, + "id": "4E9MghnYo6hn8E82pmPMe", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2877.7495250701904, + "y": 1546.6669921875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 317.33331298828125, + "height": 370, + "seed": 625293229, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": { + "elementId": "zQXps3l6_CULfrKz0sxFV", + "focus": -3.2298254921305025, + "gap": 13.691591814926234 + }, + "endBinding": { + "elementId": "Z9cJSNSjDvW2uPNthdOW9", + "focus": 0.3950527937625778, + "gap": 3.582883834838867 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -144, + 266 + ], + [ + -317.33331298828125, + 370 + ] + ] + }, + { + "type": "rectangle", + "version": 219, + "versionNonce": 687403659, + "isDeleted": false, + "id": "swt6lb3ztkUJAvM41XHBK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2033.7493724822998, + "y": 1348.4170532226562, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 177857187, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "Bnf72M4RGMZjNgBlzk90B", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 203, + "versionNonce": 892178341, + "isDeleted": false, + "id": "MlaoCVMw-5S2Yi8P1HjDR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2057.7493724823, + "y": 1356.0837097167969, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1852132621, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 238, + "versionNonce": 703621419, + "isDeleted": false, + "id": "TiGonLf-juzwJEoH9VSZD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2033.4159984588623, + "y": 1381.5836791992188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 254514755, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 224, + "versionNonce": 2055623429, + "isDeleted": false, + "id": "3YRJV9k9ywr0i4pEZkHkN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2057.4159984588623, + "y": 1389.2503356933594, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 2075272045, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 227, + "versionNonce": 369101771, + "isDeleted": false, + "id": "33dfOgt2KTLk3Q5NBImFr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2033.082685470581, + "y": 1414.9170227050781, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1787155939, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 237, + "versionNonce": 1387244133, + "isDeleted": false, + "id": "92FFROdmhjPmxEpXFFQ5M", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2031.7493114471436, + "y": 1442.5836791992188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1238707661, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 218, + "versionNonce": 826432107, + "isDeleted": false, + "id": "QULGDwKUKXLejiOFZCB88", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2054.4159984588623, + "y": 1450.9170227050781, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1270292867, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 230, + "versionNonce": 1047991749, + "isDeleted": false, + "id": "-dLa28cPs8d1YMU273D-q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2055.7493724823, + "y": 1419.833724975586, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1206381613, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "uC3ZSm-IltHDllxDGLJ9v", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 83, + "versionNonce": 1484903691, + "isDeleted": false, + "id": "diotVPud8Nk4qCzgu2hJi", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2005.416151046753, + "y": 1321.0004272460938, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 177.33334350585938, + "seed": 2080241955, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 89, + "versionNonce": 1224348965, + "isDeleted": false, + "id": "uC3ZSm-IltHDllxDGLJ9v", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2061.749616622925, + "y": 1459.3337860107422, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 103.33331298828125, + "height": 104.66668701171875, + "seed": 687885357, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": { + "elementId": "-dLa28cPs8d1YMU273D-q", + "focus": 3.424460294999428, + "gap": 11.855942213284369 + }, + "endBinding": { + "elementId": "0kWlc6iPzGzhrfIVEPeOM", + "focus": 0.1363752482649436, + "gap": 1.5828227996826172 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 103.33331298828125, + 64.66668701171875 + ], + [ + 67.3333740234375, + 104.66668701171875 + ] + ] + }, + { + "type": "text", + "version": 107, + "versionNonce": 1798896555, + "isDeleted": false, + "id": "Ba0bmo-eQQnhNNi6zcCzg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2259.0828075408936, + "y": 1502.6670684814453, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 177, + "height": 40, + "seed": 1174246765, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": " uint8 *load_addr;\n uint64 load_size;", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": " uint8 *load_addr;\n uint64 load_size;" + }, + { + "type": "rectangle", + "version": 43, + "versionNonce": 1034299525, + "isDeleted": false, + "id": "SOEOh6pxx-gC9L5EGGFZb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2247.082929611206, + "y": 1502.6670684814453, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 256, + "height": 38, + "seed": 1478394307, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 17, + "versionNonce": 1972522571, + "isDeleted": false, + "id": "KcLwCwuZF-_XQTvbSOePf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2267.7495555877686, + "y": 1506.6670684814453, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 17.3333740234375, + "height": 13.333343505859375, + "seed": 1278561005, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 58, + "versionNonce": 507039717, + "isDeleted": false, + "id": "f7JIg_N3m3yBHDkQvlpUn", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2275.7495555877686, + "y": 1510.6670684814453, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.33331298828125, + "height": 94.00003051757812, + "seed": 646153155, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -63.33331298828125, + 70 + ], + [ + -19.33331298828125, + 94.00003051757812 + ] + ] + }, + { + "type": "text", + "version": 331, + "versionNonce": 443043051, + "isDeleted": false, + "id": "epVvbDyPF40MaERFnDDJy", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1858.9132976531982, + "y": 1828.5115061442057, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 238, + "height": 20, + "seed": 2133258755, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "CUEfVWpVIuIHc_5h3VskN", + "type": "arrow" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMFunction (second module)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMFunction (second module)" + }, + { + "type": "text", + "version": 36, + "versionNonce": 1488948037, + "isDeleted": false, + "id": "iSNS4LqcpEqsBWT3JrhTG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1985.4163494110107, + "y": 1302.0003509521484, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 176, + "height": 20, + "seed": 895067437, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModule::functions", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule::functions" + }, + { + "type": "text", + "version": 58, + "versionNonce": 1027962763, + "isDeleted": false, + "id": "zzptLpjImiiG1vPCDIGPS", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2788.5657176971436, + "y": 1386.4003204345704, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 185, + "height": 20, + "seed": 510558371, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModule::func_ptrs", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule::func_ptrs" + }, + { + "type": "text", + "version": 84, + "versionNonce": 1033362085, + "isDeleted": false, + "id": "QPiZaCnPYGIz7ApGqrktI", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 2629.160482406616, + "y": 1451.7336334228517, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 160, + "height": 40, + "seed": 1301033645, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModule::\nfast_jit_func_ptrs", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModule::\nfast_jit_func_ptrs" + }, + { + "type": "rectangle", + "version": 444, + "versionNonce": 1818512939, + "isDeleted": false, + "id": "bGS26pMiud1SV_QZasA4k", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 753.6687545776367, + "y": 1354.399984741211, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 140.66668701171875, + "height": 32.33331298828126, + "seed": 1104978987, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "E9Lf0GGwiMXE7OEKmLBD0" + } + ], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 223, + "versionNonce": 498422861, + "isDeleted": false, + "id": "E9Lf0GGwiMXE7OEKmLBD0", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 784.0020980834961, + "y": 1360.0666412353517, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 80, + "height": 20, + "seed": 1389316101, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679537247214, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "func_ptrs", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "bGS26pMiud1SV_QZasA4k", + "originalText": "func_ptrs" + }, + { + "type": "diamond", + "version": 480, + "versionNonce": 179261643, + "isDeleted": false, + "id": "3V5jOnV9GBHrydrWDjdlN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 764.3353805541992, + "y": 1366.7332977294923, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 2058151627, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 64, + "versionNonce": 927151461, + "isDeleted": false, + "id": "Ej1XTEhuVY9wBpKERi-1L", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 772.668815612793, + "y": 1274.0665191650392, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 140, + "height": 20, + "seed": 1144833349, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "import_func_ptrs", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "import_func_ptrs" + }, + { + "type": "rectangle", + "version": 521, + "versionNonce": 50715531, + "isDeleted": false, + "id": "OFyAqRR6a69cDApnQUNYF", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 735.2243474324545, + "y": 1412.3998321533204, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 185.3334045410156, + "height": 34.99996948242188, + "seed": 1243189643, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533960843, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 511, + "versionNonce": 2105521829, + "isDeleted": false, + "id": "Nv_mYV9MitkgQQoVIZJ9H", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 737.8910039265951, + "y": 1424.3998321533204, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1000679467, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533960843, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 85, + "versionNonce": 1685598763, + "isDeleted": false, + "id": "sB-lDN4LgjZtFuvGEHopU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 753.8910039265951, + "y": 1421.2331756591798, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 160, + "height": 20, + "seed": 967892165, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "WmAyzG_eOk5gu9ag2e4NN", + "type": "arrow" + } + ], + "updated": 1679533960843, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "fast_jit_func_ptrs", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "fast_jit_func_ptrs" + }, + { + "type": "rectangle", + "version": 247, + "versionNonce": 1175609515, + "isDeleted": false, + "id": "bEyCLX9k_ShKiQzS-ZFKc", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 871.6685104370117, + "y": 1862.1498245239259, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1755598347, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929334, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 231, + "versionNonce": 591500165, + "isDeleted": false, + "id": "mz6MyUFqd-cTtlX0HmAqX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 895.6685104370117, + "y": 1869.8164810180665, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 488090661, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 266, + "versionNonce": 1509364555, + "isDeleted": false, + "id": "i6UWgN6SrSs9asopho9X_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 871.3351364135742, + "y": 1895.3164505004884, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 581194923, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 252, + "versionNonce": 858708709, + "isDeleted": false, + "id": "OxO-Lw3MtI3B3_yxI_9BQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 895.3351364135742, + "y": 1902.983106994629, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 133750661, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 255, + "versionNonce": 2146202091, + "isDeleted": false, + "id": "Av0IXtuoDwPzsT4d4ZD9r", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 871.001823425293, + "y": 1928.6497940063477, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 942311243, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 265, + "versionNonce": 414987845, + "isDeleted": false, + "id": "zsHPiP1O3kL8Jo_YTK1SB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 869.6684494018555, + "y": 1956.3164505004884, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 454862565, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 246, + "versionNonce": 150797451, + "isDeleted": false, + "id": "QkKoENHz9noDS00xsmwGp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 892.3351364135742, + "y": 1964.6497940063477, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 798057963, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 258, + "versionNonce": 1321674149, + "isDeleted": false, + "id": "8h3LnYKSejC6NlmCDjN_T", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 893.6685104370117, + "y": 1933.5664962768556, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1494254149, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 127, + "versionNonce": 1469009605, + "isDeleted": false, + "id": "8FJ9nE4COwUuKYB0Fjze2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 843.3352890014648, + "y": 1851.0666336059571, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 160.99990844726562, + "seed": 1608741003, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "WmAyzG_eOk5gu9ag2e4NN", + "type": "arrow" + }, + { + "id": "Xr0h90XMpFNQFRcVNp-Qb", + "type": "arrow" + } + ], + "updated": 1679534054366, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 103, + "versionNonce": 410855685, + "isDeleted": false, + "id": "xkXUNjzkhjNlrNaWU9GPX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 826.6688613891602, + "y": 1811.3999618530274, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174, + "height": 40, + "seed": 380760485, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModuleInstance::\nfast_jit_func_ptrs", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance::\nfast_jit_func_ptrs" + }, + { + "type": "arrow", + "version": 98, + "versionNonce": 1677433349, + "isDeleted": false, + "id": "WmAyzG_eOk5gu9ag2e4NN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 742.8910344441732, + "y": 1433.2344728128833, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 187.33331298828125, + "height": 431.9987943990309, + "seed": 471597803, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533960844, + "link": null, + "locked": false, + "startBinding": { + "elementId": "sB-lDN4LgjZtFuvGEHopU", + "focus": 1.017117824752581, + "gap": 10.999969482421875 + }, + "endBinding": { + "elementId": "8FJ9nE4COwUuKYB0Fjze2", + "focus": 0.09986159259017957, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -87.5555318196615, + 110.66535934043713 + ], + [ + -41.555531819661496, + 281.99876388145276 + ], + [ + 99.77778116861975, + 431.9987943990309 + ] + ] + }, + { + "type": "rectangle", + "version": 261, + "versionNonce": 227189867, + "isDeleted": false, + "id": "AXS2auzqFNZS35F2ixu8Z", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1039.001808166504, + "y": 1963.98309173584, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1019623493, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 245, + "versionNonce": 1630563269, + "isDeleted": false, + "id": "cjLfuQX8dFdgByjdkn0sY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1063.001808166504, + "y": 1971.6497482299806, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 67902091, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 280, + "versionNonce": 1496375051, + "isDeleted": false, + "id": "O8ko0NdUbnEnqQSXbDcTn", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1038.6684341430664, + "y": 1997.1497177124024, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 641787813, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 266, + "versionNonce": 907586341, + "isDeleted": false, + "id": "MK_yFDygTjEm7c4A1RI_A", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1062.6684341430664, + "y": 2004.816374206543, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 140415275, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 269, + "versionNonce": 1433847211, + "isDeleted": false, + "id": "IpQhaQLL8LX1JsLCZTqHR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1038.3351211547852, + "y": 2030.4830612182618, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 527346437, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 279, + "versionNonce": 1988230789, + "isDeleted": false, + "id": "G_B8CuUKKojKnmDg-blQS", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1037.0017471313477, + "y": 2058.1497177124024, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 48, + "height": 29, + "seed": 1416180683, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 261, + "versionNonce": 696703051, + "isDeleted": false, + "id": "bubXHX2zzv6QsRGmmBlvp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1059.6684341430664, + "y": 2066.483061218262, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 1758265957, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "kjpM2qWJqDrV-jr9XK8QR", + "type": "arrow" + } + ], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 272, + "versionNonce": 959047141, + "isDeleted": false, + "id": "p2Ps7ouZR4NFea5dxTpPY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1061.001808166504, + "y": 2035.3997634887696, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 10.66668701171875, + "height": 17.666656494140625, + "seed": 407274091, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 142, + "versionNonce": 161780453, + "isDeleted": false, + "id": "PZQU4Sc5stYlZLaGSiYZg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1010.668586730957, + "y": 1952.8999008178712, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98.66668701171875, + "height": 160.99990844726562, + "seed": 1462332869, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "eXEUtswyqJzZHgjz8P_J5", + "type": "arrow" + }, + { + "id": "BcRoQEkrTOTzUxX-Uw8S8", + "type": "arrow" + } + ], + "updated": 1679534050516, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 119, + "versionNonce": 1629348165, + "isDeleted": false, + "id": "RolknO7AhmdfdQTNiXJ8F", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 994.0021591186523, + "y": 1913.2332290649415, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174, + "height": 40, + "seed": 482409739, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "eXEUtswyqJzZHgjz8P_J5", + "type": "arrow" + } + ], + "updated": 1679533929335, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModuleInstance::\nfunc_ptrs", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance::\nfunc_ptrs" + }, + { + "type": "rectangle", + "version": 222, + "versionNonce": 824148363, + "isDeleted": false, + "id": "p5TPteQC3PraRMJtt4XsT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 897.2243372599283, + "y": 1585.9554707845052, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 74.666748046875, + "height": 160.99990844726562, + "seed": 727269189, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "yu3un9i0kAGrZ8bAnVMa3", + "type": "arrow" + } + ], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 444, + "versionNonce": 841297547, + "isDeleted": false, + "id": "eXEUtswyqJzZHgjz8P_J5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 771.3355026245117, + "y": 1374.8998474121095, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 384.88897705078125, + "height": 633.6667022705078, + "seed": 539008875, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533948630, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "PZQU4Sc5stYlZLaGSiYZg", + "focus": 0.2318272147781851, + "gap": 3.3330230712890625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -148.888916015625, + 184.1111602783203 + ], + [ + 15.555613199869754, + 612.4444732666016 + ], + [ + 236.00006103515625, + 633.6667022705078 + ] + ] + }, + { + "type": "arrow", + "version": 83, + "versionNonce": 745347115, + "isDeleted": false, + "id": "kjpM2qWJqDrV-jr9XK8QR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1063.3354822794595, + "y": 2044.0108703613284, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 1343.333333333333, + "height": 143.33333333333326, + "seed": 738327429, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false, + "startBinding": { + "elementId": "bubXHX2zzv6QsRGmmBlvp", + "focus": -3.5383188444895577, + "gap": 13.041657453315548 + }, + "endBinding": { + "elementId": "bu6GnR96DeCSFWu3DhMIJ", + "focus": -0.1414139865892621, + "gap": 14.399755859375318 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 790, + 23.888905843098883 + ], + [ + 1343.333333333333, + -119.44442749023438 + ] + ] + }, + { + "type": "text", + "version": 258, + "versionNonce": 1857033221, + "isDeleted": false, + "id": "VCjoZw1mwV4c94ES2JChm", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 708.2243372599285, + "y": 1804.0109212239581, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 68, + "height": 20, + "seed": 1329151115, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "(void **)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "(void **)" + }, + { + "type": "text", + "version": 317, + "versionNonce": 588176075, + "isDeleted": false, + "id": "8pDOO3W9GlOqpH8OfM3H_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1274.502098083496, + "y": 1674.5109720865876, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 246, + "height": 19, + "seed": 73971563, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679533929335, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMModuleInstance(second)", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance(second)" + }, + { + "type": "rectangle", + "version": 271, + "versionNonce": 2113489765, + "isDeleted": false, + "id": "Rcref7JZ-AhlcXLLdwY5D", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1264.668805440267, + "y": 1697.4554453531891, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 270.66663614908845, + "height": 194.22224934895837, + "seed": 635278027, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "MLVyGZQLa4jU554J6bsmJ", + "type": "arrow" + } + ], + "updated": 1679533929335, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 226, + "versionNonce": 1289066309, + "isDeleted": false, + "id": "CZNopRssr82fjSim4TRBD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 601.1133956909175, + "y": 2080.3445597330715, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 335, + "height": 20, + "seed": 556524395, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679534038146, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "function pointer arrays for faster access", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "function pointer arrays for faster access" + }, + { + "type": "rectangle", + "version": 45, + "versionNonce": 815465317, + "isDeleted": false, + "id": "J2L3EeElp1XhI1X3IbanK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 578.8910547892249, + "y": 2055.122269694009, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 364.4444274902344, + "height": 58.33338419596339, + "seed": 1181100939, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "id": "BcRoQEkrTOTzUxX-Uw8S8", + "type": "arrow" + }, + { + "id": "Xr0h90XMpFNQFRcVNp-Qb", + "type": "arrow" + } + ], + "updated": 1679534054366, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 33, + "versionNonce": 57793355, + "isDeleted": false, + "id": "BcRoQEkrTOTzUxX-Uw8S8", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 944.4466272989905, + "y": 2095.1223205566394, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.33333333333326, + "height": 19.444478352864735, + "seed": 132937093, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679534050516, + "link": null, + "locked": false, + "startBinding": { + "elementId": "J2L3EeElp1XhI1X3IbanK", + "focus": 0.788606211312463, + "gap": 1.11114501953125 + }, + "endBinding": { + "elementId": "PZQU4Sc5stYlZLaGSiYZg", + "focus": -0.2743956700481259, + "gap": 2.8886260986332672 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 63.33333333333326, + -19.444478352864735 + ] + ] + }, + { + "type": "arrow", + "version": 24, + "versionNonce": 2098532715, + "isDeleted": false, + "id": "Xr0h90XMpFNQFRcVNp-Qb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 786.6688156127926, + "y": 2054.011175537108, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 55.555572509765625, + "height": 45.00000000000023, + "seed": 1053702635, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679534054366, + "link": null, + "locked": false, + "startBinding": { + "elementId": "J2L3EeElp1XhI1X3IbanK", + "focus": -0.054183297415777855, + "gap": 1.1110941569011175 + }, + "endBinding": { + "elementId": "8FJ9nE4COwUuKYB0Fjze2", + "focus": -0.3037089268567984, + "gap": 1.110900878906591 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 55.555572509765625, + -45.00000000000023 + ] + ] + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/wasm_function.svg b/wasm-micro-runtime/core/iwasm/doc/images/wasm_function.svg new file mode 100644 index 0000000..86fdf78 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/wasm_function.svg @@ -0,0 +1,16 @@ + + + + + + + WASMFunctionInstancefunc_importWASMFunctionImport:funcWASMFunction (per module)internal function char *module_name; char *field_name;func_ptr_linkednative function:import_module;import_func_linked;(WASMFunction *)codebytecodecode_compiledprecompiled-bytecodefast_jit_jitted_codellvm_jit_func_ptrfast jitted coodellvm jitted coodeimport_module_inst;import_func_inst;functionsglobalseWASMModuleInstanceExtraWASMModuleInstance::import_func_ptrsthis is the one actually referred during executing opcodeduring model load, if import can be solved through the native api registeration,the pointer of native function will be filled.c-api could change the pointer later, then it will point to a different native functionNULL: means multi-module import, go to "import_func_inst" field for target functionWASMModuleInstance*(WASMFunctionInstance*)func_importfuncWASMFunction union[...][0]WASMModuleInstance[..][n](void *)(void **)NULLfunc_type_indexesuint32 *WASMModule WASMImport *import_tables; WASMImport *import_memories; WASMImport *import_globals; WASMType **types; WASMImport *imports; WASMTable *tables; WASMMemory *memories; WASMGlobal *globals; WASMExport *exports; WASMTableSeg *table_segments; WASMDataSeg **data_segments;WASMFunction **functions;WASMImport *import_functions;WASMImportuint8 kind[1][0][...]fast_jit_func_ptrs(void **)func_ptrs(void **) uint8 *load_addr; uint64 load_size;WASMFunction (second module)WASMModule::functionsWASMModule::func_ptrsWASMModule::fast_jit_func_ptrsfunc_ptrsimport_func_ptrsfast_jit_func_ptrsWASMModuleInstance::fast_jit_func_ptrsWASMModuleInstance::func_ptrs(void **)WASMModuleInstance(second)function pointer arrays for faster access \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/wasm_globals.excalidraw b/wasm-micro-runtime/core/iwasm/doc/images/wasm_globals.excalidraw new file mode 100644 index 0000000..9471535 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/wasm_globals.excalidraw @@ -0,0 +1,2313 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "elements": [ + { + "type": "rectangle", + "version": 381, + "versionNonce": 2068900405, + "isDeleted": false, + "id": "D5Ay5TxydaAe4f80l_79L", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 982.833251953125, + "y": 298.00006103515625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 99.666748046875, + "height": 211.00007629394523, + "seed": 1123866381, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "5S0qe2-BjQRPwuEEQ_UsU", + "type": "arrow" + } + ], + "updated": 1679660991439, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 317, + "versionNonce": 1880855285, + "isDeleted": false, + "id": "4JTzPDASWmnx1PVSabO3A", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 914.833251953125, + "y": 236.3333282470703, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 103, + "height": 20, + "seed": 422161475, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661135316, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "global_data:", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "global_data:" + }, + { + "type": "text", + "version": 183, + "versionNonce": 1437992789, + "isDeleted": false, + "id": "HQjUeFzLlihuH1Lf8XZQq", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 528.1666870117188, + "y": 324.00001525878906, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 274, + "height": 20, + "seed": 2018585571, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661163124, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModuleInstanceExtra::globals", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstanceExtra::globals" + }, + { + "type": "arrow", + "version": 453, + "versionNonce": 92905717, + "isDeleted": false, + "id": "I86Qg5wzdmE77J5SbA7HP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 759.0470523835444, + "y": 472.4753157819668, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 223.44867715016005, + "height": 152.25013181880155, + "seed": 661894701, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "Qm7oDN7yJzVFQrCa0EE1G" + } + ], + "updated": 1679660991440, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "Sd34CflTFN9Elv1dedCI1", + "focus": 0.9595350520837825, + "gap": 4.004209431139316 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 223.44867715016005, + -152.25013181880155 + ] + ] + }, + { + "type": "text", + "version": 4, + "versionNonce": 2089047221, + "isDeleted": false, + "id": "Qm7oDN7yJzVFQrCa0EE1G", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 844.2713909586244, + "y": 386.350249872566, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 53, + "height": 20, + "seed": 1720249052, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661201718, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "offset", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "I86Qg5wzdmE77J5SbA7HP", + "originalText": "offset" + }, + { + "type": "rectangle", + "version": 213, + "versionNonce": 1454183483, + "isDeleted": false, + "id": "Sd34CflTFN9Elv1dedCI1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 986.4999389648438, + "y": 315.6666564941406, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "width": 91.66668701171875, + "height": 27.999999999999996, + "seed": 357984397, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "I86Qg5wzdmE77J5SbA7HP", + "type": "arrow" + } + ], + "updated": 1679660991440, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 192, + "versionNonce": 706008283, + "isDeleted": false, + "id": "X5nXXDxRcZputNDZp2WfW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 984.4998779296875, + "y": 350.83333587646484, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "width": 94.3333740234375, + "height": 19.333358764648438, + "seed": 1611395779, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "PFkOKGbMOcdhcpcv4DutQ", + "type": "arrow" + } + ], + "updated": 1679660991442, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 369, + "versionNonce": 1307818581, + "isDeleted": false, + "id": "PFkOKGbMOcdhcpcv4DutQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 753.8716651262948, + "y": 690.3334503173828, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 220.3341412661283, + "height": 341.60985534904495, + "seed": 1747362403, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "MPb6tVMlSBtnXhUTpZvIC" + } + ], + "updated": 1679660991442, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "X5nXXDxRcZputNDZp2WfW", + "gap": 5.794568769140314, + "focus": 1.2182487723741706 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 220.3341412661283, + -341.60985534904495 + ] + ] + }, + { + "type": "text", + "version": 4, + "versionNonce": 1008745755, + "isDeleted": false, + "id": "MPb6tVMlSBtnXhUTpZvIC", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 837.538735759359, + "y": 509.52852264286037, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 53, + "height": 20, + "seed": 1880081892, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661201719, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "offset", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "PFkOKGbMOcdhcpcv4DutQ", + "originalText": "offset" + }, + { + "type": "text", + "version": 429, + "versionNonce": 147921333, + "isDeleted": false, + "id": "5GnY6Vq9qPDS0ntZW3UOE", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 917.166748046875, + "y": 259.3333282470703, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 197, + "height": 20, + "seed": 1179957165, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661138960, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "hold values of GLOBALs", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "hold values of GLOBALs" + }, + { + "type": "rectangle", + "version": 234, + "versionNonce": 339097851, + "isDeleted": false, + "id": "FcG2LCAdKxw_z2SzLhPPr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 548.3333129882812, + "y": 351.666748046875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 228.66668701171866, + "height": 299.66668701171875, + "seed": 407083364, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "_acaSZ2N5FSiuPGQjH8RA", + "type": "arrow" + } + ], + "updated": 1679660870990, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 182, + "versionNonce": 1735336804, + "isDeleted": false, + "id": "OdBdX2K4Kcn9Hq7vInwKA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 571, + "y": 387.3333740234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 190, + "height": 30, + "seed": 175211612, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "OZDTfM4SDGXEVbxgCAbsx" + } + ], + "updated": 1679643912221, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 135, + "versionNonce": 1695691996, + "isDeleted": false, + "id": "OZDTfM4SDGXEVbxgCAbsx", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 621.5, + "y": 392.3333740234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 89, + "height": 20, + "seed": 1994750564, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643912221, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint8 type;", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "OdBdX2K4Kcn9Hq7vInwKA", + "originalText": "uint8 type;" + }, + { + "type": "text", + "version": 174, + "versionNonce": 2063524661, + "isDeleted": false, + "id": "w5el7zqw5vcK9RGIcDlFZ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 580.9999389648438, + "y": 357.8333435058594, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 171, + "height": 19, + "seed": 1831729636, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "_acaSZ2N5FSiuPGQjH8RA", + "type": "arrow" + } + ], + "updated": 1679660882880, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMGlobalInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMGlobalInstance" + }, + { + "type": "rectangle", + "version": 147, + "versionNonce": 1194277212, + "isDeleted": false, + "id": "gd9UxDYh5XZYBG_P0f0c2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 568.3333129882812, + "y": 424.666748046875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 195, + "height": 30, + "seed": 1018791012, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "YeoEjoPWRnxxyB2DjNi3g" + } + ], + "updated": 1679643912221, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 104, + "versionNonce": 1904253540, + "isDeleted": false, + "id": "YeoEjoPWRnxxyB2DjNi3g", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 600.8333129882812, + "y": 429.666748046875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 130, + "height": 20, + "seed": 839439588, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643912221, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "bool is_mutable;", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "gd9UxDYh5XZYBG_P0f0c2", + "originalText": "bool is_mutable;" + }, + { + "type": "rectangle", + "version": 91, + "versionNonce": 1480043996, + "isDeleted": false, + "id": "MB23InQWKES3OnYie8bbP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 569.6666870117188, + "y": 460.666748046875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 195, + "height": 30, + "seed": 1468762332, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "--vP8lM62PEnMBhgFRqTM" + } + ], + "updated": 1679643912221, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 63, + "versionNonce": 717729252, + "isDeleted": false, + "id": "--vP8lM62PEnMBhgFRqTM", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 582.6666870117188, + "y": 465.666748046875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 169, + "height": 20, + "seed": 695551844, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643912221, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint32 data_offset;", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "MB23InQWKES3OnYie8bbP", + "originalText": "uint32 data_offset;" + }, + { + "type": "rectangle", + "version": 111, + "versionNonce": 155030108, + "isDeleted": false, + "id": "pUUma4amJviNKdfn5otpb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 577.6666870117188, + "y": 497.33338928222656, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 180.33331298828125, + "height": 30, + "seed": 1550527844, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "JybtW1PyDYQa00JaSYsuo" + } + ], + "updated": 1679643912221, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 63, + "versionNonce": 1625313636, + "isDeleted": false, + "id": "JybtW1PyDYQa00JaSYsuo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 618.8333435058594, + "y": 502.33338928222656, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 98, + "height": 20, + "seed": 1952693084, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643912221, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "initial_value", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "pUUma4amJviNKdfn5otpb", + "originalText": "initial_value" + }, + { + "type": "rectangle", + "version": 91, + "versionNonce": 1494410972, + "isDeleted": false, + "id": "CewuQtNj0ZC6Ogsq68ite", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 569.6666870117188, + "y": 546.6667022705078, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 191, + "height": 30, + "seed": 10317668, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "4xSFF2vFZIoA0G7GTeC8-" + } + ], + "updated": 1679643912221, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 63, + "versionNonce": 588809444, + "isDeleted": false, + "id": "4xSFF2vFZIoA0G7GTeC8-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 588.1666870117188, + "y": 551.6667022705078, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 154, + "height": 20, + "seed": 1109403492, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643912221, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "import_module_inst", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "CewuQtNj0ZC6Ogsq68ite", + "originalText": "import_module_inst" + }, + { + "type": "rectangle", + "version": 123, + "versionNonce": 474590044, + "isDeleted": false, + "id": "AuHfz77U4KxNTCmKvW6bJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 573, + "y": 589.3333892822266, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 184, + "height": 30, + "seed": 2058570588, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "h7MfnIO2f08rV_U7840Zp" + } + ], + "updated": 1679643912221, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 75, + "versionNonce": 1727940708, + "isDeleted": false, + "id": "h7MfnIO2f08rV_U7840Zp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 591, + "y": 594.3333892822266, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 148, + "height": 20, + "seed": 1917882340, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643912221, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "import_global_inst", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "AuHfz77U4KxNTCmKvW6bJ", + "originalText": "import_global_inst" + }, + { + "type": "text", + "version": 134, + "versionNonce": 1361481211, + "isDeleted": false, + "id": "Clf1NqZqRXJuRl-CQgx_u", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 232.6667175292969, + "y": 454.6668395996094, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 237, + "height": 20, + "seed": 1711793252, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "_acaSZ2N5FSiuPGQjH8RA", + "type": "arrow" + } + ], + "updated": 1679661000688, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMGlobalInstance *globals;", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMGlobalInstance *globals;" + }, + { + "type": "text", + "version": 30, + "versionNonce": 1504826724, + "isDeleted": false, + "id": "Mj7l2FOHQ7NNAkTiGq7T_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 641, + "y": 631.0001373291016, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 20, + "seed": 1401608924, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643959727, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[0]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[0]" + }, + { + "type": "rectangle", + "version": 52, + "versionNonce": 894397020, + "isDeleted": false, + "id": "CGwcN3BpsJaehLlHjEn6l", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 548.6666259765625, + "y": 653.0000457763672, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 227, + "height": 87, + "seed": 253331684, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644065258, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 107, + "versionNonce": 1424372828, + "isDeleted": false, + "id": "YjTAPKLSA0k-ml5j0Ho59", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 565.1666259765625, + "y": 671.6667327880859, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 195, + "height": 30, + "seed": 192441436, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "jmlAM3nHo1CJVxGKppt1O" + } + ], + "updated": 1679643968867, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 78, + "versionNonce": 2064306020, + "isDeleted": false, + "id": "jmlAM3nHo1CJVxGKppt1O", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 578.1666259765625, + "y": 676.6667327880859, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 169, + "height": 20, + "seed": 1473779556, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643968868, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint32 data_offset;", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "YjTAPKLSA0k-ml5j0Ho59", + "originalText": "uint32 data_offset;" + }, + { + "type": "text", + "version": 32, + "versionNonce": 1512962780, + "isDeleted": false, + "id": "QnLh-KZNv_MoD4Ak5RsLk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 636.1666259765625, + "y": 719.0000457763672, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 22, + "height": 20, + "seed": 927449564, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679643979051, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[1]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[1]" + }, + { + "type": "rectangle", + "version": 62, + "versionNonce": 626330844, + "isDeleted": false, + "id": "F26JZDrwDBDSk5o5kcV_E", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 547.3333129882812, + "y": 739.6667327880859, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 227.666748046875, + "height": 60.666656494140625, + "seed": 725179740, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644057644, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 71, + "versionNonce": 1658992100, + "isDeleted": false, + "id": "H6cL4L0PUBLB5An3KDf-i", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 636.6666259765625, + "y": 784.0003204345703, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 31, + "height": 20, + "seed": 83711068, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644496025, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "[...]", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "[...]" + }, + { + "type": "rectangle", + "version": 158, + "versionNonce": 208125412, + "isDeleted": false, + "id": "M6cl8S-GRJkkRSQrUxzJV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 566.833251953125, + "y": 747.6667327880859, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 195, + "height": 30, + "seed": 997959652, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "DrpXGgZIT2RAhz4L6EM2n" + } + ], + "updated": 1679644019993, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 128, + "versionNonce": 493257308, + "isDeleted": false, + "id": "DrpXGgZIT2RAhz4L6EM2n", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 579.833251953125, + "y": 752.6667327880859, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 169, + "height": 20, + "seed": 157169756, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644019993, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint32 data_offset;", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "M6cl8S-GRJkkRSQrUxzJV", + "originalText": "uint32 data_offset;" + }, + { + "type": "rectangle", + "version": 223, + "versionNonce": 680765365, + "isDeleted": false, + "id": "nPxM8HePICecTvRXbdEeU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 983.333251953125, + "y": 378.1666793823242, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "width": 97.66668701171875, + "height": 30.000015258789062, + "seed": 1516156124, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "Y_Dw9hCEvutA3MjjHfyJK", + "type": "arrow" + } + ], + "updated": 1679660991443, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 165, + "versionNonce": 430152219, + "isDeleted": false, + "id": "Y_Dw9hCEvutA3MjjHfyJK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 756.6666259765625, + "y": 766.3333892822266, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 217.14296385054604, + "height": 385.92617569738337, + "seed": 703298780, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "MraNxq38RWxCBOb3EhIue" + } + ], + "updated": 1679660991443, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "nPxM8HePICecTvRXbdEeU", + "gap": 9.523662126016394, + "focus": 1.1442737969660202 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 217.14296385054604, + -385.92617569738337 + ] + ] + }, + { + "type": "text", + "version": 4, + "versionNonce": 1122104853, + "isDeleted": false, + "id": "MraNxq38RWxCBOb3EhIue", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 838.7381079018355, + "y": 563.3703014335349, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 53, + "height": 20, + "seed": 1058889828, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661201730, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "offset", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Y_Dw9hCEvutA3MjjHfyJK", + "originalText": "offset" + }, + { + "type": "text", + "version": 239, + "versionNonce": 551950564, + "isDeleted": false, + "id": "O5mHltv55Fs4-vM6wNcf-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 226.99996948242188, + "y": 655.0000457763672, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 164, + "height": 20, + "seed": 352849508, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "1QAmzR8Zkk8scUA3tqI0k", + "type": "arrow" + } + ], + "updated": 1679644753215, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMGlobalInstance", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMGlobalInstance" + }, + { + "type": "rectangle", + "version": 104, + "versionNonce": 219644132, + "isDeleted": false, + "id": "CpOJESdMD1E9wBe2Gkkvf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 216.66665649414062, + "y": 681.6666412353516, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 180.33331298828125, + "height": 46.666778564453125, + "seed": 705788380, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644728661, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 153, + "versionNonce": 296383867, + "isDeleted": false, + "id": "1QAmzR8Zkk8scUA3tqI0k", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 583.9999694824219, + "y": 602.3333587646484, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "width": 179.2173434297432, + "height": 75.12469349055664, + "seed": 869061340, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "N9fS0DKeDh1d_Ueyopb9v" + } + ], + "updated": 1679661068533, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "O5mHltv55Fs4-vM6wNcf-", + "focus": 1.1855962422681312, + "gap": 14.0001220703125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -179.2173434297432, + 75.12469349055664 + ] + ] + }, + { + "id": "N9fS0DKeDh1d_Ueyopb9v", + "type": "text", + "x": 449.39129776755027, + "y": 629.8957055099268, + "width": 90, + "height": 20, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 339183189, + "version": 24, + "versionNonce": 270159573, + "isDeleted": false, + "boundElements": null, + "updated": 1679661092509, + "link": null, + "locked": false, + "text": "when import", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 14, + "containerId": "1QAmzR8Zkk8scUA3tqI0k", + "originalText": "when import" + }, + { + "type": "rectangle", + "version": 59, + "versionNonce": 96174172, + "isDeleted": false, + "id": "oDs6sXYd2cCCUTlDuFyfK", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 199.66659545898438, + "y": 599.9999847412109, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 236.33343505859375, + "height": 196.3333740234375, + "seed": 1531972708, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644717838, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 72, + "versionNonce": 1711293284, + "isDeleted": false, + "id": "Gyz12ttmraGX-CaCTWl3h", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 191.99990844726562, + "y": 576.3333892822266, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 239, + "height": 20, + "seed": 683171044, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "957MNrV_zCOsf-ssBBkla", + "type": "arrow" + } + ], + "updated": 1679644717838, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "WASMModuleInstance (second)", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance (second)" + }, + { + "type": "text", + "version": 185, + "versionNonce": 1644000405, + "isDeleted": false, + "id": "7TYuQ_yCiwRN4oz8zkdGb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 38.33337402343753, + "y": 248.66676330566406, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 171, + "height": 19, + "seed": 1694546652, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661000688, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMModuleInstance", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstance" + }, + { + "type": "rectangle", + "version": 151, + "versionNonce": 1152705621, + "isDeleted": false, + "id": "DN3tWnhyoeDPQR_-BYeYI", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 36.666656494140625, + "y": 269.00010681152344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 202.00003051757815, + "height": 124.66665649414062, + "seed": 693300324, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "-n6vwSfIOzGpAxjpRyuNZ", + "type": "arrow" + } + ], + "updated": 1679661004485, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 150, + "versionNonce": 458299605, + "isDeleted": false, + "id": "K0JMv-qZGU4i9Or0Xz4Gg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 246.33328247070315, + "y": 399.33343505859375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 218, + "height": 19, + "seed": 881476572, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661029361, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "WASMModuleInstanceExtra", + "baseline": 15, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "WASMModuleInstanceExtra" + }, + { + "type": "rectangle", + "version": 169, + "versionNonce": 769392085, + "isDeleted": false, + "id": "CERoUCjzlaXjrdr4PSxtB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 225.33328247070312, + "y": 428.00010681152344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 256.3333740234375, + "height": 91.33331298828128, + "seed": 1750838620, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "-n6vwSfIOzGpAxjpRyuNZ", + "type": "arrow" + } + ], + "updated": 1679661027326, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 88, + "versionNonce": 1947089019, + "isDeleted": false, + "id": "H3BkjBVeDYM2F429PxOI9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 233.0000915527344, + "y": 447.00010681152344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 243.00006103515625, + "height": 31, + "seed": 258806116, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "-n6vwSfIOzGpAxjpRyuNZ", + "type": "arrow" + } + ], + "updated": 1679661000688, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 257, + "versionNonce": 314119835, + "isDeleted": false, + "id": "_acaSZ2N5FSiuPGQjH8RA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 463.97668298272777, + "y": 453.66683959960943, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 80.59006409386518, + "height": 100.23285752369344, + "seed": 1866235612, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661000688, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Clf1NqZqRXJuRl-CQgx_u", + "focus": 0.8216444271084713, + "gap": 1 + }, + "endBinding": { + "elementId": "FcG2LCAdKxw_z2SzLhPPr", + "focus": 1.009989878742988, + "gap": 3.7665659116883603 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 80.59006409386518, + -100.23285752369344 + ] + ] + }, + { + "type": "rectangle", + "version": 63, + "versionNonce": 1164697781, + "isDeleted": false, + "id": "NIObCr2apYl6JLLzA37G2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 232.66665649414065, + "y": 486.66676330566406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 244.33331298828125, + "height": 30, + "seed": 1813909212, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "sYEF77zbpRw-pcdX1ass6" + } + ], + "updated": 1679661000688, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 24, + "versionNonce": 4682011, + "isDeleted": false, + "id": "sYEF77zbpRw-pcdX1ass6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 272.33331298828125, + "y": 491.66676330566406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 165, + "height": 20, + "seed": 382274908, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661000688, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint32 global_count;", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "NIObCr2apYl6JLLzA37G2", + "originalText": "uint32 global_count;" + }, + { + "type": "rectangle", + "version": 142, + "versionNonce": 519569941, + "isDeleted": false, + "id": "7jX0wgP7v4HHkuITKOf89", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 94.3333435058594, + "y": 344.33351135253906, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 115, + "height": 30.333358764648438, + "seed": 1226666212, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "sK7ecOoz_3QIduVF0nHd7" + } + ], + "updated": 1679661000688, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 116, + "versionNonce": 1846580667, + "isDeleted": false, + "id": "sK7ecOoz_3QIduVF0nHd7", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 146.3333435058594, + "y": 349.5001907348633, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 11, + "height": 20, + "seed": 1178170724, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661000688, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "e", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "7jX0wgP7v4HHkuITKOf89", + "originalText": "e" + }, + { + "type": "diamond", + "version": 135, + "versionNonce": 691599221, + "isDeleted": false, + "id": "yTfoj623sdm1eh4Eusmvt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 174.66665649414065, + "y": 352.0001983642578, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 13.66668701171875, + "height": 12.333328247070312, + "seed": 559414236, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661000688, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 324, + "versionNonce": 1197545819, + "isDeleted": false, + "id": "-n6vwSfIOzGpAxjpRyuNZ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 182.00003051757815, + "y": 356.66676330566406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 45.12566323463764, + "height": 69.66665649414062, + "seed": 1850539876, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661027326, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "CERoUCjzlaXjrdr4PSxtB", + "gap": 1, + "focus": -0.6072997775389923 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 45.12566323463764, + 69.66665649414062 + ] + ] + }, + { + "type": "arrow", + "version": 68, + "versionNonce": 732107996, + "isDeleted": false, + "id": "957MNrV_zCOsf-ssBBkla", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 578.0000305175781, + "y": 557.6666717529297, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 144.0692777412945, + "height": 39.30743410761045, + "seed": 1605267676, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644717838, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "Gyz12ttmraGX-CaCTWl3h", + "focus": 1.0338080600507247, + "gap": 3.00006103515625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -144.0692777412945, + 39.30743410761045 + ] + ] + }, + { + "type": "diamond", + "version": 20, + "versionNonce": 825681252, + "isDeleted": false, + "id": "pKw8fr7S6f80J93nrJtkQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 575.8333129882812, + "y": 553.5000076293945, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 13.66668701171875, + "height": 12.333328247070312, + "seed": 222063588, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644481683, + "link": null, + "locked": false + }, + { + "type": "diamond", + "version": 20, + "versionNonce": 825681252, + "isDeleted": false, + "id": "XKwLDKrjsXcSogvJHSY-e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 573.8333129882812, + "y": 597.5000076293945, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 13.66668701171875, + "height": 12.333328247070312, + "seed": 1140569180, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644483164, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 215, + "versionNonce": 510935253, + "isDeleted": false, + "id": "oPjebDyI5NP26BNK4PLtX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 50.33343505859378, + "y": 290.33338928222656, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 180, + "height": 31, + "seed": 727213156, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "ZK3cjtpUxVpruHBkbgevo" + } + ], + "updated": 1679661000688, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 175, + "versionNonce": 1653383931, + "isDeleted": false, + "id": "ZK3cjtpUxVpruHBkbgevo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 58.33343505859378, + "y": 295.83338928222656, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 164, + "height": 20, + "seed": 1130592356, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661000688, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "uint8 * global_data", + "baseline": 14, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "oPjebDyI5NP26BNK4PLtX", + "originalText": "uint8 * global_data" + }, + { + "type": "arrow", + "version": 159, + "versionNonce": 345149237, + "isDeleted": false, + "id": "5S0qe2-BjQRPwuEEQ_UsU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 228.00003051757812, + "y": 299.9999694824219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 754.7517447227082, + "height": 17.8331298828125, + "seed": 1256331620, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679661008047, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "D5Ay5TxydaAe4f80l_79L", + "focus": 0.9720307648275319, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 385.0834503173828, + -17.8331298828125 + ], + [ + 754.7517447227082, + -2.996583692902334 + ] + ] + }, + { + "type": "diamond", + "version": 89, + "versionNonce": 1413439029, + "isDeleted": false, + "id": "szV9RT3yAJ51dUnP5JYMS", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 220.1667175292969, + "y": 294.50000762939453, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 13.66668701171875, + "height": 12.333328247070312, + "seed": 1642184284, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679661000688, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 160, + "versionNonce": 715957468, + "isDeleted": false, + "id": "DuwilIDd-fhNUxybFRKva", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 255.16659545898438, + "y": 750.6665496826172, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 99, + "height": 20, + "seed": 121250780, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644739787, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "global_data", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "global_data" + }, + { + "type": "rectangle", + "version": 143, + "versionNonce": 2046173532, + "isDeleted": false, + "id": "6CMun9I8IOX876t85IJ2X", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 215.83328247070312, + "y": 738.6664733886719, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 180.33331298828125, + "height": 46.666778564453125, + "seed": 2113088100, + "groupIds": [], + "roundness": null, + "boundElements": [ + { + "id": "igqxFPh3AgDb1kYMsORSZ", + "type": "arrow" + } + ], + "updated": 1679644784578, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 34, + "versionNonce": 1198703204, + "isDeleted": false, + "id": "cbSo4pUG_J3a4Pnk3ODCA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 276.6665954589844, + "y": 693.3332366943359, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 55, + "height": 20, + "seed": 328242020, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1679644771096, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "globals", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "globals" + }, + { + "type": "arrow", + "version": 81, + "versionNonce": 311117156, + "isDeleted": false, + "id": "igqxFPh3AgDb1kYMsORSZ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 247.33328247070312, + "y": 706.3332366943359, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 71.66665649414062, + "height": 58.33343505859375, + "seed": 1345639908, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1679644787747, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "6CMun9I8IOX876t85IJ2X", + "focus": -0.8452179527426857, + "gap": 1.499908447265625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -71.66665649414062, + 15.66668701171875 + ], + [ + -32.999908447265625, + 58.33343505859375 + ] + ] + }, + { + "type": "diamond", + "version": 135, + "versionNonce": 691599221, + "isDeleted": false, + "id": "koTtnTl85TIGelUbfKlR9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 459.58338928222656, + "y": 446.0001754760742, + "strokeColor": "#000000", + "backgroundColor": "#be4bdb", + "width": 13.66668701171875, + "height": 12.333328247070312, + "seed": 626707637, + "groupIds": [], + "roundness": null, + "boundElements": null, + "updated": 1679661018823, + "link": null, + "locked": false + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/images/wasm_globals.svg b/wasm-micro-runtime/core/iwasm/doc/images/wasm_globals.svg new file mode 100644 index 0000000..fa34617 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/images/wasm_globals.svg @@ -0,0 +1,16 @@ + + + + + + + global_data:WASMModuleInstanceExtra::globalsoffsetoffsethold values of GLOBALsuint8 type;WASMGlobalInstancebool is_mutable;uint32 data_offset;initial_valueimport_module_instimport_global_instWASMGlobalInstance *globals;[0]uint32 data_offset;[1][...]uint32 data_offset;offsetWASMGlobalInstancewhen importWASMModuleInstance (second)WASMModuleInstanceWASMModuleInstanceExtrauint32 global_count;euint8 * global_dataglobal_dataglobals \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/wasm_exports.MD b/wasm-micro-runtime/core/iwasm/doc/wasm_exports.MD new file mode 100644 index 0000000..70708cd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/wasm_exports.MD @@ -0,0 +1,22 @@ +# Wasm exports +The internal data structure for Wasm exports: +![](./images/wasm_exports.svg) + +## Setup exports for Module +The array data structure pointed by `WASMModule::exports` is setup during loading a module. Basically the runtime will load the exports sections from the module file content, and construct an array of C struct `WASMExport`. + +A `WASMExport` item contains three elements that map the Wasm file exports section structure: +- name: the name shown to external +- kind: total 4 export types: function, globals, memory, table +- index: As all the 4 export types are organized in array, this refers to index in target export type + +## Function exports +### use function exports +function exports are often used in two situations: + 1. **call by host**: runtime API `wasm_runtime_lookup_function` will walk through the array of `WASMModuleInstance::export_functions` and compare the exported name with given target symbol name in the function parameter. If any array item matches the name, it then returns the value of field `function` which points to associated function instance (WASMFunctionInstance) + 2. **import by another module**: During linking multiple modules, the runtime saves the pointer of exported WASMFunctionInstance in the local WASMFunctionInstance of importing module. + +### setup for instance +The data structure pointed by `WASMModuleInstance::export_functions` is set up during instantiating module instance. + +The runtime will walk through the `WASMModule::exports` array and find all the item with kind equal to "function". Create a node of `WASMExportFuncInstance` for each matching, find the associated `WASMFunctionInstance` object and save its address in the field `function`. diff --git a/wasm-micro-runtime/core/iwasm/doc/wasm_function.MD b/wasm-micro-runtime/core/iwasm/doc/wasm_function.MD new file mode 100644 index 0000000..da4dac2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/wasm_function.MD @@ -0,0 +1,47 @@ +# Wasm Function + +## Internal data structure + +![](./images/wasm_function.svg) + +## Module level data (function) +**WASMModule**: Data structure created for loading a module. +- `WASMImport *import_functions`: initialized from the Wasm file function section +- `WASMImport *import_functions`: initialized from the Wasm file import section. The runtime will try to solve the imports from the native API registration, refer to [Export native API to WASM application](../../../doc/export_native_api.md). + +**WASMFunction**: represent a Wasm function located in Wasm file code section. Track the links to the compiled function body. +**WASMImport**: represent a imported Wasm function which can be a solved as a native function or another Wasm module exported function. + +## Instance level data (function) +**WASMModuleInstance**: Data structure created for instantiating a module +- `WASMModuleInstanceExtra::functions`: combined the imported and internal functions into single array of structure `WASMFunctionInstance` +- `WASMModuleInstance::import_func_ptrs`: pointer array for solved function imports. This array is referred during calling imported native function. Note it is initialzed with the module level solved imports, but may points to different native function later due to c-api calls. + +## Execution paths +**Interpreter**: +- Execute internal bytecode function: + ``` + WASMModuleInstance::e + -> WASMModuleInstanceExtra::functions[..] + -> WASMFunctionInstance::func + -> WASMFunction::code + ``` + +- Execute imported function from other module: + ``` + WASMModuleInstance::e + -> WASMModuleInstanceExtra::functions[..] + (WASMFunctionInstance flag indicates an import) + -> WASMFunctionInstance::import_func_inst + -> WASMModuleInstance(second)::func + -> WASMFunction (second module)::code + ``` + +- Execute imported native function: + ``` + WASMModuleInstance::e + -> WASMModuleInstanceExtra::functions[..] + (flag indicates imported native) + WASMModuleInstance::import_func_ptrs[..] + -> native function + ``` \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/doc/wasm_globals.MD b/wasm-micro-runtime/core/iwasm/doc/wasm_globals.MD new file mode 100644 index 0000000..e5019a9 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/doc/wasm_globals.MD @@ -0,0 +1,4 @@ +# Wasm globals + +![](./images/wasm_globals.svg) + diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/asmjit_sgx_patch.diff b/wasm-micro-runtime/core/iwasm/fast-jit/asmjit_sgx_patch.diff new file mode 100644 index 0000000..465b9de --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/asmjit_sgx_patch.diff @@ -0,0 +1,42 @@ +diff --git a/src/asmjit/core/cpuinfo.cpp b/src/asmjit/core/cpuinfo.cpp +index 7bf7407..ae2160b 100644 +--- a/src/asmjit/core/cpuinfo.cpp ++++ b/src/asmjit/core/cpuinfo.cpp +@@ -9,13 +9,13 @@ + + #if !defined(_WIN32) + #include +- #include ++ //#include + #include + #endif + + // Required by `getauxval()` on Linux. + #if defined(__linux__) +- #include ++ //#include + #endif + + //! Required to detect CPU and features on Apple platforms. +diff --git a/src/asmjit/core/globals.cpp b/src/asmjit/core/globals.cpp +index 2bbd0c0..e6b69e5 100644 +--- a/src/asmjit/core/globals.cpp ++++ b/src/asmjit/core/globals.cpp +@@ -105,6 +105,8 @@ ASMJIT_FAVOR_SIZE const char* DebugUtils::errorAsString(Error err) noexcept { + #endif + } + ++extern "C" int os_printf(const char *message, ...); ++ + // DebugUtils - Debug Output + // ========================= + +@@ -112,7 +114,7 @@ ASMJIT_FAVOR_SIZE void DebugUtils::debugOutput(const char* str) noexcept { + #if defined(_WIN32) + ::OutputDebugStringA(str); + #else +- ::fputs(str, stderr); ++ os_printf(str); + #endif + } + diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/cg/LICENSE_ASMJIT b/wasm-micro-runtime/core/iwasm/fast-jit/cg/LICENSE_ASMJIT new file mode 100644 index 0000000..020a569 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/cg/LICENSE_ASMJIT @@ -0,0 +1,17 @@ +Copyright (c) 2008-2020 The AsmJit Authors + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/cg/LICENSE_ZYDIS b/wasm-micro-runtime/core/iwasm/fast-jit/cg/LICENSE_ZYDIS new file mode 100644 index 0000000..11185a5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/cg/LICENSE_ZYDIS @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2014-2021 Florian Bernd +Copyright (c) 2014-2021 Joel Höner + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp b/wasm-micro-runtime/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp new file mode 100644 index 0000000..79c7250 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp @@ -0,0 +1,9513 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_codegen.h" +#include "jit_codecache.h" +#include "jit_compiler.h" +#include "jit_frontend.h" +#include "jit_dump.h" + +#include +#include +#if WASM_ENABLE_FAST_JIT_DUMP != 0 +#include +#endif + +#define CODEGEN_CHECK_ARGS 1 +#define CODEGEN_DUMP 0 + +using namespace asmjit; + +static char *code_block_switch_to_jitted_from_interp = NULL; +static char *code_block_return_to_interp_from_jitted = NULL; +#if WASM_ENABLE_LAZY_JIT != 0 +static char *code_block_compile_fast_jit_and_then_call = NULL; +#endif + +typedef enum { + REG_BPL_IDX = 0, + REG_AXL_IDX, + REG_BXL_IDX, + REG_CXL_IDX, + REG_DXL_IDX, + REG_DIL_IDX, + REG_SIL_IDX, + REG_I8_FREE_IDX = REG_SIL_IDX +} RegIndexI8; + +typedef enum { + REG_BP_IDX = 0, + REG_AX_IDX, + REG_BX_IDX, + REG_CX_IDX, + REG_DX_IDX, + REG_DI_IDX, + REG_SI_IDX, + REG_I16_FREE_IDX = REG_SI_IDX +} RegIndexI16; + +typedef enum { + REG_EBP_IDX = 0, + REG_EAX_IDX, + REG_EBX_IDX, + REG_ECX_IDX, + REG_EDX_IDX, + REG_EDI_IDX, + REG_ESI_IDX, + REG_I32_FREE_IDX = REG_ESI_IDX +} RegIndexI32; + +typedef enum { + REG_RBP_IDX = 0, + REG_RAX_IDX, + REG_RBX_IDX, + REG_RCX_IDX, + REG_RDX_IDX, + REG_RDI_IDX, + REG_RSI_IDX, + REG_RSP_IDX, + REG_R8_IDX, + REG_R9_IDX, + REG_R10_IDX, + REG_R11_IDX, + REG_R12_IDX, + REG_R13_IDX, + REG_R14_IDX, + REG_R15_IDX, + REG_I64_FREE_IDX = REG_RSI_IDX +} RegIndexI64; + +/* clang-format off */ +x86::Gp regs_i8[] = { + x86::bpl, x86::al, x86::bl, x86::cl, + x86::dl, x86::dil, x86::sil, x86::spl, + x86::r8b, x86::r9b, x86::r10b, x86::r11b, + x86::r12b, x86::r13b, x86::r14b, x86::r15b +}; + +x86::Gp regs_i16[] = { + x86::bp, x86::ax, x86::bx, x86::cx, + x86::dx, x86::di, x86::si, x86::sp, + x86::r8w, x86::r9w, x86::r10w, x86::r11w, + x86::r12w, x86::r13w, x86::r14w, x86::r15w +}; + +x86::Gp regs_i32[] = { + x86::ebp, x86::eax, x86::ebx, x86::ecx, + x86::edx, x86::edi, x86::esi, x86::esp, + x86::r8d, x86::r9d, x86::r10d, x86::r11d, + x86::r12d, x86::r13d, x86::r14d, x86::r15d +}; + +x86::Gp regs_i64[] = { + x86::rbp, x86::rax, x86::rbx, x86::rcx, + x86::rdx, x86::rdi, x86::rsi, x86::rsp, + x86::r8, x86::r9, x86::r10, x86::r11, + x86::r12, x86::r13, x86::r14, x86::r15, +}; + +#define REG_F32_FREE_IDX 15 +#define REG_F64_FREE_IDX 15 + +x86::Xmm regs_float[] = { + x86::xmm0, + x86::xmm1, + x86::xmm2, + x86::xmm3, + x86::xmm4, + x86::xmm5, + x86::xmm6, + x86::xmm7, + x86::xmm8, + x86::xmm9, + x86::xmm10, + x86::xmm11, + x86::xmm12, + x86::xmm13, + x86::xmm14, + x86::xmm15, +}; +/* clang-format on */ + +int +jit_codegen_interp_jitted_glue(void *exec_env, JitInterpSwitchInfo *info, + uint32 func_idx, void *target) +{ + typedef int32 (*F)(const void *exec_env, void *info, uint32 func_idx, + const void *target); + union { + F f; + void *v; + } u; + + u.v = code_block_switch_to_jitted_from_interp; + return u.f(exec_env, info, func_idx, target); +} + +#define PRINT_LINE() LOG_VERBOSE("\n", __LINE__) + +#if CODEGEN_DUMP != 0 +#define GOTO_FAIL \ + do { \ + PRINT_LINE(); \ + goto fail; \ + } while (0) +#else +#define GOTO_FAIL goto fail +#endif + +#if CODEGEN_CHECK_ARGS == 0 + +#define CHECK_EQKIND(reg0, reg1) (void)0 +#define CHECK_CONST(reg0) (void)0 +#define CHECK_NCONST(reg0) (void)0 +#define CHECK_KIND(reg0, type) (void)0 +#define CHECK_REG_NO(no, kind) (void)0 +#else + +/* Check if two register's kind is equal */ +#define CHECK_EQKIND(reg0, reg1) \ + do { \ + if (jit_reg_kind(reg0) != jit_reg_kind(reg1)) { \ + PRINT_LINE(); \ + LOG_VERBOSE("reg type not equal:\n"); \ + jit_dump_reg(cc, reg0); \ + jit_dump_reg(cc, reg1); \ + GOTO_FAIL; \ + } \ + } while (0) + +/* Check if a register is an const */ +#define CHECK_CONST(reg0) \ + do { \ + if (!jit_reg_is_const(reg0)) { \ + PRINT_LINE(); \ + LOG_VERBOSE("reg is not const:\n"); \ + jit_dump_reg(cc, reg0); \ + GOTO_FAIL; \ + } \ + } while (0) + +/* Check if a register is not an const */ +#define CHECK_NCONST(reg0) \ + do { \ + if (jit_reg_is_const(reg0)) { \ + PRINT_LINE(); \ + LOG_VERBOSE("reg is const:\n"); \ + jit_dump_reg(cc, reg0); \ + GOTO_FAIL; \ + } \ + } while (0) + +/* Check if a register is a special type */ +#define CHECK_KIND(reg0, type) \ + do { \ + if (jit_reg_kind(reg0) != type) { \ + PRINT_LINE(); \ + LOG_VERBOSE("invalid reg type %d, expected is: %d", \ + jit_reg_kind(reg0), type); \ + jit_dump_reg(cc, reg0); \ + GOTO_FAIL; \ + } \ + } while (0) + +#define CHECK_I32_REG_NO(no) \ + do { \ + if ((uint32)no >= sizeof(regs_i32) / sizeof(regs_i32[0])) \ + GOTO_FAIL; \ + } while (0) + +#define CHECK_I64_REG_NO(no) \ + do { \ + if ((uint32)no >= sizeof(regs_i64) / sizeof(regs_i64[0])) \ + GOTO_FAIL; \ + } while (0) + +#define CHECK_F32_REG_NO(no) \ + do { \ + if ((uint32)no >= sizeof(regs_float) / sizeof(regs_float[0])) \ + GOTO_FAIL; \ + } while (0) + +#define CHECK_F64_REG_NO(no) \ + do { \ + if ((uint32)no >= sizeof(regs_float) / sizeof(regs_float[0])) \ + GOTO_FAIL; \ + } while (0) + +/* Check if a register number is valid */ +#define CHECK_REG_NO(no, kind) \ + do { \ + if (kind == JIT_REG_KIND_I32 || kind == JIT_REG_KIND_I64) { \ + CHECK_I32_REG_NO(no); \ + CHECK_I64_REG_NO(no); \ + } \ + else if (kind == JIT_REG_KIND_F32 || kind == JIT_REG_KIND_F64) { \ + CHECK_F32_REG_NO(no); \ + CHECK_F64_REG_NO(no); \ + } \ + else \ + GOTO_FAIL; \ + } while (0) + +#endif /* end of CODEGEN_CHECK_ARGS == 0 */ + +/* Load one operand from insn and check none */ +#define LOAD_1ARG() r0 = *jit_insn_opnd(insn, 0) + +/* Load two operands from insn and check if r0 is non-const */ +#define LOAD_2ARGS() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + CHECK_NCONST(r0) + +/* Load three operands from insn and check if r0 is non-const */ +#define LOAD_3ARGS() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + r2 = *jit_insn_opnd(insn, 2); \ + CHECK_NCONST(r0) + +/* Load three operands from insn and check none */ +#define LOAD_3ARGS_NO_ASSIGN() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + r2 = *jit_insn_opnd(insn, 2); + +/* Load four operands from insn and check if r0 is non-const */ +#define LOAD_4ARGS() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + r2 = *jit_insn_opnd(insn, 2); \ + r3 = *jit_insn_opnd(insn, 3); \ + CHECK_NCONST(r0) + +/* Load five operands from insn and check if r0 is non-const */ +#define LOAD_4ARGS_NO_ASSIGN() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + r2 = *jit_insn_opnd(insn, 2); \ + r3 = *jit_insn_opnd(insn, 3); + +class JitErrorHandler : public ErrorHandler +{ + public: + Error err; + + JitErrorHandler() + : err(kErrorOk) + {} + + void handleError(Error e, const char *msg, BaseEmitter *base) override + { + (void)msg; + (void)base; + this->err = e; + } +}; + +/* Alu opcode */ +typedef enum { ADD, SUB, MUL, DIV_S, REM_S, DIV_U, REM_U, MIN, MAX } ALU_OP; +/* Bit opcode */ +typedef enum { OR, XOR, AND } BIT_OP; +/* Shift opcode */ +typedef enum { SHL, SHRS, SHRU, ROTL, ROTR } SHIFT_OP; +/* Bitcount opcode */ +typedef enum { CLZ, CTZ, POPCNT } BITCOUNT_OP; +/* Condition opcode */ +typedef enum { EQ, NE, GTS, GES, LTS, LES, GTU, GEU, LTU, LEU } COND_OP; + +typedef union _cast_float_to_integer { + float f; + uint32 i; +} cast_float_to_integer; + +typedef union _cast_double_to_integer { + double d; + uint64 i; +} cast_double_to_integer; + +static uint32 +local_log2(uint32 data) +{ + uint32 ret = 0; + while (data >>= 1) { + ret++; + } + return ret; +} + +static uint64 +local_log2l(uint64 data) +{ + uint64 ret = 0; + while (data >>= 1) { + ret++; + } + return ret; +} + +/* Jmp type */ +typedef enum JmpType { + JMP_DST_LABEL_REL, /* jmp to dst label with relative addr */ + JMP_DST_LABEL_ABS, /* jmp to dst label with absolute addr */ + JMP_END_OF_CALLBC, /* jmp to end of CALLBC */ + JMP_LOOKUPSWITCH_BASE, /* LookupSwitch table base addr */ +} JmpType; + +/** + * Jmp info, save the info on first encoding pass, + * and replace the offset with exact offset when the code cache + * has been allocated actually. + */ +typedef struct JmpInfo { + bh_list_link link; + JmpType type; + uint32 label_src; + uint32 offset; + union { + uint32 label_dst; + } dst_info; +} JmpInfo; + +static bool +label_is_neighboring(JitCompContext *cc, int32 label_prev, int32 label_succ) +{ + return (label_prev == 0 && label_succ == 2) + || (label_prev >= 2 && label_succ == label_prev + 1) + || (label_prev == (int32)jit_cc_label_num(cc) - 1 + && label_succ == 1); +} + +static bool +label_is_ahead(JitCompContext *cc, int32 label_dst, int32 label_src) +{ + return (label_dst == 0 && label_src != 0) + || (label_dst != 1 && label_src == 1) + || (2 <= label_dst && label_dst < label_src + && label_src <= (int32)jit_cc_label_num(cc) - 1); +} + +/** + * Encode jumping from one label to the other label + * + * @param a the assembler to emit the code + * @param jmp_info_list the jmp info list + * @param label_dst the index of dst label + * @param label_src the index of src label + * + * @return true if success, false if failed + */ +static bool +jmp_from_label_to_label(x86::Assembler &a, bh_list *jmp_info_list, + int32 label_dst, int32 label_src) +{ + Imm imm(INT32_MAX); + JmpInfo *node; + + node = (JmpInfo *)jit_calloc(sizeof(JmpInfo)); + if (!node) + return false; + + node->type = JMP_DST_LABEL_REL; + node->label_src = label_src; + node->dst_info.label_dst = label_dst; + node->offset = a.code()->sectionById(0)->buffer().size() + 2; + bh_list_insert(jmp_info_list, node); + + a.jmp(imm); + return true; +} + +/** + * Encode detecting compare result register according to condition code + * and then jumping to suitable label when the condtion is met + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param jmp_info_list the jmp info list + * @param label_src the index of src label + * @param op the opcode of condition operation + * @param r1 the label info when condition is met + * @param r2 the label info when condition is unmet, do nonthing if VOID + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +cmp_r_and_jmp_label(JitCompContext *cc, x86::Assembler &a, + bh_list *jmp_info_list, int32 label_src, COND_OP op, + JitReg r1, JitReg r2, bool is_last_insn) +{ + Imm imm(INT32_MAX); + JmpInfo *node; + + node = (JmpInfo *)jit_malloc(sizeof(JmpInfo)); + if (!node) + return false; + + node->type = JMP_DST_LABEL_REL; + node->label_src = label_src; + node->dst_info.label_dst = jit_reg_no(r1); + node->offset = a.code()->sectionById(0)->buffer().size() + 2; + bh_list_insert(jmp_info_list, node); + + bool fp_cmp = cc->last_cmp_on_fp; + + bh_assert(!fp_cmp || (fp_cmp && (op == GTS || op == GES))); + + switch (op) { + case EQ: + { + a.je(imm); + break; + } + case NE: + { + a.jne(imm); + break; + } + case GTS: + { + if (fp_cmp) + a.ja(imm); + else + a.jg(imm); + break; + } + case LES: + { + a.jng(imm); + break; + } + case GES: + { + if (fp_cmp) + a.jae(imm); + else + a.jnl(imm); + break; + } + case LTS: + { + a.jl(imm); + break; + } + case GTU: + { + a.ja(imm); + break; + } + case LEU: + { + a.jna(imm); + break; + } + case GEU: + { + a.jnb(imm); + break; + } + case LTU: + { + a.jb(imm); + break; + } + default: + { + bh_assert(0); + break; + } + } + + if (r2) { + int32 label_dst = jit_reg_no(r2); + if (!(is_last_insn && label_is_neighboring(cc, label_src, label_dst))) + if (!jmp_from_label_to_label(a, jmp_info_list, label_dst, + label_src)) + return false; + } + + return true; +} + +#if WASM_ENABLE_FAST_JIT_DUMP != 0 +static void +dump_native(char *data, uint32 length) +{ + /* Initialize decoder context */ + ZydisDecoder decoder; + ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, + ZYDIS_STACK_WIDTH_64); + + /* Initialize formatter */ + ZydisFormatter formatter; + ZydisFormatterInit(&formatter, ZYDIS_FORMATTER_STYLE_INTEL); + + /* Loop over the instructions in our buffer */ + ZyanU64 runtime_address = (ZyanU64)(uintptr_t)data; + ZyanUSize offset = 0; + ZydisDecodedInstruction instruction; + ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT_VISIBLE]; + + while (ZYAN_SUCCESS(ZydisDecoderDecodeFull( + &decoder, data + offset, length - offset, &instruction, operands, + ZYDIS_MAX_OPERAND_COUNT_VISIBLE, ZYDIS_DFLAG_VISIBLE_OPERANDS_ONLY))) { + /* Print current instruction pointer */ + os_printf("%012" PRIX64 " ", runtime_address); + + /* Format & print the binary instruction structure to + human readable format */ + char buffer[256]; + ZydisFormatterFormatInstruction(&formatter, &instruction, operands, + instruction.operand_count_visible, + buffer, sizeof(buffer), + runtime_address); + puts(buffer); + + offset += instruction.length; + runtime_address += instruction.length; + } +} +#endif + +/** + * Encode extending register of byte to register of dword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r8_to_r32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + if (is_signed) { + a.movsx(regs_i32[reg_no_dst], regs_i8[reg_no_src]); + } + else { + a.movzx(regs_i32[reg_no_dst], regs_i8[reg_no_src]); + } + return true; +} +/** + * Encode extending register of word to register of dword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r16_to_r32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + if (is_signed) { + a.movsx(regs_i32[reg_no_dst], regs_i16[reg_no_src]); + } + else { + a.movzx(regs_i32[reg_no_dst], regs_i16[reg_no_src]); + } + return true; +} + +/** + * Encode extending register of byte to register of qword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r8_to_r64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + if (is_signed) { + a.movsx(regs_i64[reg_no_dst], regs_i8[reg_no_src]); + } + else { + a.movzx(regs_i64[reg_no_dst], regs_i8[reg_no_src]); + } + return true; +} + +/** + * Encode extending register of word to register of qword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r16_to_r64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + if (is_signed) { + a.movsx(regs_i64[reg_no_dst], regs_i16[reg_no_src]); + } + else { + a.movzx(regs_i64[reg_no_dst], regs_i16[reg_no_src]); + } + return true; +} + +/** + * Encode extending register of dword to register of qword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r32_to_r64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + if (is_signed) { + a.movsxd(regs_i64[reg_no_dst], regs_i32[reg_no_src]); + } + else { + /* + * The upper 32-bit will be zero-extended, ref to Intel document, + * 3.4.1.1 General-Purpose Registers: 32-bit operands generate + * a 32-bit result, zero-extended to a 64-bit result in the + * destination general-purpose register + */ + a.mov(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + } + return true; +} + +static bool +mov_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src); + +static bool +mov_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src); + +static void +mov_r_to_r(x86::Assembler &a, uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src) +{ + if (kind_dst == JIT_REG_KIND_I32) + mov_r_to_r_i32(a, reg_no_dst, reg_no_src); + else if (kind_dst == JIT_REG_KIND_I64) + mov_r_to_r_i64(a, reg_no_dst, reg_no_src); + else if (kind_dst == JIT_REG_KIND_F32) { + /* TODO */ + bh_assert(0); + } + else if (kind_dst == JIT_REG_KIND_F64) { + /* TODO */ + bh_assert(0); + } + else { + bh_assert(0); + } +} + +/** + * Encode moving memory to a register + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * skipped by float and double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed whether the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param m_src the memory operand which contains the source data + * + * @return true if success, false otherwise + */ +static bool +mov_m_to_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, bool is_signed, + int32 reg_no_dst, x86::Mem &m_src) +{ + if (kind_dst == JIT_REG_KIND_I32) { + switch (bytes_dst) { + case 1: + case 2: + if (is_signed) + a.movsx(regs_i32[reg_no_dst], m_src); + else + a.movzx(regs_i32[reg_no_dst], m_src); + break; + case 4: + a.mov(regs_i32[reg_no_dst], m_src); + break; + default: + bh_assert(0); + return false; + } + } + else if (kind_dst == JIT_REG_KIND_I64) { + switch (bytes_dst) { + case 1: + case 2: + if (is_signed) + a.movsx(regs_i64[reg_no_dst], m_src); + else + a.movzx(regs_i64[reg_no_dst], m_src); + break; + case 4: + if (is_signed) + a.movsxd(regs_i64[reg_no_dst], m_src); + else + /* + * The upper 32-bit will be zero-extended, ref to Intel + * document, 3.4.1.1 General-Purpose Registers: 32-bit + * operands generate a 32-bit result, zero-extended to + * a 64-bit result in the destination general-purpose + * register + */ + a.mov(regs_i32[reg_no_dst], m_src); + break; + case 8: + a.mov(regs_i64[reg_no_dst], m_src); + break; + default: + bh_assert(0); + return false; + } + } + else if (kind_dst == JIT_REG_KIND_F32) { + a.movss(regs_float[reg_no_dst], m_src); + } + else if (kind_dst == JIT_REG_KIND_F64) { + a.movsd(regs_float[reg_no_dst], m_src); + } + return true; +} + +/** + * Encode moving register to memory + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * skipped by float and double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed whether the data is signed or unsigned + * @param m_dst the dest memory operand + * @param reg_no_src the index of dest register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_m(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + x86::Mem &m_dst, int32 reg_no_src) +{ + if (kind_dst == JIT_REG_KIND_I32) { + bh_assert(reg_no_src < 16); + switch (bytes_dst) { + case 1: + a.mov(m_dst, regs_i8[reg_no_src]); + break; + case 2: + a.mov(m_dst, regs_i16[reg_no_src]); + break; + case 4: + a.mov(m_dst, regs_i32[reg_no_src]); + break; + default: + bh_assert(0); + return false; + } + } + else if (kind_dst == JIT_REG_KIND_I64) { + bh_assert(reg_no_src < 16); + switch (bytes_dst) { + case 1: + a.mov(m_dst, regs_i8[reg_no_src]); + break; + case 2: + a.mov(m_dst, regs_i16[reg_no_src]); + break; + case 4: + a.mov(m_dst, regs_i32[reg_no_src]); + break; + case 8: + a.mov(m_dst, regs_i64[reg_no_src]); + break; + default: + bh_assert(0); + return false; + } + } + else if (kind_dst == JIT_REG_KIND_F32) { + a.movss(m_dst, regs_float[reg_no_src]); + } + else if (kind_dst == JIT_REG_KIND_F64) { + a.movsd(m_dst, regs_float[reg_no_src]); + } + return true; +} + +/** + * Encode moving immediate data to memory + * + * @param m dst memory + * @param imm src immediate data + * + * @return new stream + */ +static bool +mov_imm_to_m(x86::Assembler &a, x86::Mem &m_dst, Imm imm_src, uint32 bytes_dst) +{ + if (bytes_dst == 8) { + int64 value = imm_src.value(); + if (value >= INT32_MIN && value <= INT32_MAX) { + imm_src.setValue((int32)value); + a.mov(m_dst, imm_src); + } + else { + /* There is no instruction `MOV m64, imm64`, we use + two instructions to implement it */ + a.mov(regs_i64[REG_I64_FREE_IDX], imm_src); + a.mov(m_dst, regs_i64[REG_I64_FREE_IDX]); + } + } + else + a.mov(m_dst, imm_src); + return true; +} + +#if WASM_ENABLE_SHARED_MEMORY != 0 +/** + * Encode exchange register with memory + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * skipped by float and double + * @param kind_dst the kind of data to move, could only be I32 or I64 + * @param m_dst the dest memory operand + * @param reg_no_src the index of dest register + * + * @return true if success, false otherwise + */ +static bool +xchg_r_to_m(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + x86::Mem &m_dst, int32 reg_no_src) +{ + bh_assert((kind_dst == JIT_REG_KIND_I32 && bytes_dst <= 4) + || kind_dst == JIT_REG_KIND_I64); + bh_assert(reg_no_src < 16); + switch (bytes_dst) { + case 1: + a.xchg(m_dst, regs_i8[reg_no_src]); + break; + case 2: + a.xchg(m_dst, regs_i16[reg_no_src]); + break; + case 4: + a.xchg(m_dst, regs_i32[reg_no_src]); + break; + case 8: + a.xchg(m_dst, regs_i64[reg_no_src]); + break; + default: + bh_assert(0); + return false; + } + return true; +} +#endif +/** + * Encode loading register data from memory with imm base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param base the base address of the memory + * @param offset the offset address of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_imm_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, bool is_signed, int32 reg_no_dst, + int32 base, int32 offset) +{ + x86::Mem m((uintptr_t)(base + offset), bytes_dst); + return mov_m_to_r(a, bytes_dst, kind_dst, is_signed, reg_no_dst, m); +} + +/** + * Encode loading register data from memory with imm base and register offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param base the base address of the memory + * @param reg_no_offset the no of register which stores the offset of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_imm_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, bool is_signed, int32 reg_no_dst, + int32 base, int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_offset], base, bytes_dst); + return mov_m_to_r(a, bytes_dst, kind_dst, is_signed, reg_no_dst, m); +} + +/** + * Encode loading register data from memory with register base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param reg_no_base the no of register which stores the base of the memory + * @param offset the offset address of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, bool is_signed, int32 reg_no_dst, + int32 reg_no_base, int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return mov_m_to_r(a, bytes_dst, kind_dst, is_signed, reg_no_dst, m); +} + +/** + * Encode loading register data from memory with register base and register + * offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param reg_no_base the no of register which stores the base of the memory + * @param reg_no_offset the no of register which stores the offset of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + bool is_signed, int32 reg_no_dst, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return mov_m_to_r(a, bytes_dst, kind_dst, is_signed, reg_no_dst, m); +} + +/** + * Encode storing register data to memory with imm base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param base the base address of the dst memory + * @param offset the offset address of the dst memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_imm_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_src, int32 base, + int32 offset, bool atomic) +{ + x86::Mem m((uintptr_t)(base + offset), bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +#endif + return mov_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +} + +/** + * Encode storing register data to memory with imm base and register offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param base the base address of the dst memory + * @param reg_no_offset the no of register which stores the offset of the dst + * memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_imm_offset_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_src, int32 base, int32 reg_no_offset, + bool atomic) +{ + x86::Mem m(regs_i64[reg_no_offset], base, bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +#endif + return mov_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +} + +/** + * Encode storing register data to memory with register base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param reg_no_base the no of register which stores the base of the dst memory + * @param offset the offset address of the dst memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_src, int32 reg_no_base, int32 offset, + bool atomic) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +#endif + return mov_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +} + +/** + * Encode storing register data to memory with register base and register offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param reg_no_base the no of register which stores the base of the dst memory + * @param reg_no_offset the no of register which stores the offset of the dst + * memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_src, int32 reg_no_base, + int32 reg_no_offset, bool atomic) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +#endif + return mov_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src); +} + +static void +imm_set_value(Imm &imm, void *data, uint32 bytes) +{ + switch (bytes) { + case 1: + imm.setValue(*(uint8 *)data); + break; + case 2: + imm.setValue(*(uint16 *)data); + break; + case 4: + imm.setValue(*(uint32 *)data); + break; + case 8: + imm.setValue(*(uint64 *)data); + break; + default: + bh_assert(0); + } +} + +#if WASM_ENABLE_SHARED_MEMORY != 0 +static uint32 +mov_imm_to_free_reg(x86::Assembler &a, Imm &imm, uint32 bytes) +{ + uint32 reg_no; + + switch (bytes) { + case 1: + reg_no = REG_I8_FREE_IDX; + a.mov(regs_i8[reg_no], imm); + break; + case 2: + reg_no = REG_I16_FREE_IDX; + a.mov(regs_i16[reg_no], imm); + break; + case 4: + reg_no = REG_I32_FREE_IDX; + a.mov(regs_i32[reg_no], imm); + break; + case 8: + reg_no = REG_I64_FREE_IDX; + a.mov(regs_i64[reg_no], imm); + break; + default: + bh_assert(0); + } + + return reg_no; +} +#endif + +/** + * Encode storing int32 imm data to memory with imm base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param base the base address of dst memory + * @param offset the offset address of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_imm_offset_imm(x86::Assembler &a, uint32 bytes_dst, + void *data_src, int32 base, int32 offset, + bool atomic) +{ + x86::Mem m((uintptr_t)(base + offset), bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + if (atomic) { + return xchg_r_to_m(a, bytes_dst, JIT_REG_KIND_I64, m, reg_no_src); + } +#endif + return mov_imm_to_m(a, m, imm, bytes_dst); +} + +/** + * Encode storing int32 imm data to memory with imm base and reg offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param base the base address of dst memory + * @param reg_no_offset the no of register that stores the offset address + * of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_imm_offset_r(x86::Assembler &a, uint32 bytes_dst, void *data_src, + int32 base, int32 reg_no_offset, bool atomic) +{ + x86::Mem m(regs_i64[reg_no_offset], base, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + if (atomic) { + return xchg_r_to_m(a, bytes_dst, JIT_REG_KIND_I64, m, reg_no_src); + } +#endif + return mov_imm_to_m(a, m, imm, bytes_dst); +} + +/** + * Encode storing int32 imm data to memory with reg base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param reg_no_base the no of register that stores the base address + * of dst memory + * @param offset the offset address of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, void *data_src, + int32 reg_no_base, int32 offset, bool atomic) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + if (atomic) { + return xchg_r_to_m(a, bytes_dst, JIT_REG_KIND_I64, m, reg_no_src); + } +#endif + return mov_imm_to_m(a, m, imm, bytes_dst); +} + +/** + * Encode storing int32 imm data to memory with reg base and reg offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param reg_no_base the no of register that stores the base address + * of dst memory + * @param reg_no_offset the no of register that stores the offset address + * of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, void *data_src, + int32 reg_no_base, int32 reg_no_offset, bool atomic) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); +#if WASM_ENABLE_SHARED_MEMORY != 0 + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + if (atomic) { + return xchg_r_to_m(a, bytes_dst, JIT_REG_KIND_I64, m, reg_no_src); + } +#endif + return mov_imm_to_m(a, m, imm, bytes_dst); +} + +/** + * Encode moving immediate int32 data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_i32(x86::Assembler &a, int32 reg_no, int32 data) +{ + Imm imm(data); + a.mov(regs_i32[reg_no], imm); + return true; +} + +/** + * Encode moving int32 data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + if (reg_no_dst != reg_no_src) + a.mov(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + return true; +} + +/** + * Encode moving immediate int64 data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_i64(x86::Assembler &a, int32 reg_no, int64 data) +{ + Imm imm(data); + a.mov(regs_i64[reg_no], imm); + return true; +} + +/** + * Encode moving int64 data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + if (reg_no_dst != reg_no_src) + a.mov(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + return true; +} + +/** + * Encode moving immediate float data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_f32(x86::Assembler &a, int32 reg_no, float data) +{ + /* imm -> gp -> xmm */ + cast_float_to_integer v = { .f = data }; + Imm imm(v.i); + a.mov(regs_i32[REG_I32_FREE_IDX], imm); + a.movd(regs_float[reg_no], regs_i32[REG_I32_FREE_IDX]); + return true; +} + +/** + * Encode moving float data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + if (reg_no_dst != reg_no_src) { + a.movss(regs_float[reg_no_dst], regs_float[reg_no_src]); + } + return true; +} + +/** + * Encode moving immediate double data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_f64(x86::Assembler &a, int32 reg_no, double data) +{ + cast_double_to_integer v = { .d = data }; + Imm imm(v.i); + a.mov(regs_i64[REG_I32_FREE_IDX], imm); + /* REG_I32_FREE_IDX == REG_I64_FREE_IDX */ + a.movq(regs_float[reg_no], regs_i64[REG_I64_FREE_IDX]); + return true; +} + +/** + * Encode moving double data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + if (reg_no_dst != reg_no_src) { + a.movsd(regs_float[reg_no_dst], regs_float[reg_no_src]); + } + return true; +} + +/* Let compiler do the conversation job as much as possible */ + +/** + * Encoding convert int8 immediate data to int32 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int32 + * @param data the src int8 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i8_to_r_i32(x86::Assembler &a, int32 reg_no, int8 data) +{ + return mov_imm_to_r_i32(a, reg_no, (int32)data); +} + +/** + * encoding convert int8 register to int32 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i8_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return extend_r8_to_r32(a, reg_no_dst, reg_no_src, true); +} + +/** + * encoding convert int8 immediate data to int64 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int64 + * @param data the src int8 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i8_to_r_i64(x86::Assembler &a, int32 reg_no, int8 data) +{ + return mov_imm_to_r_i64(a, reg_no, (int64)data); +} + +/** + * encoding convert int8 register to int64 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i8_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return extend_r8_to_r64(a, reg_no_dst, reg_no_src, true); +} + +/** + * Encoding convert int16 immediate data to int32 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int32 + * @param data the src int16 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i16_to_r_i32(x86::Assembler &a, int32 reg_no, int16 data) +{ + return mov_imm_to_r_i32(a, reg_no, (int32)data); +} + +/** + * encoding convert int16 register to int32 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i16_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return extend_r16_to_r32(a, reg_no_dst, reg_no_src, true); +} + +/** + * encoding convert int16 immediate data to int64 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int64 + * @param data the src int16 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i16_to_r_i64(x86::Assembler &a, int32 reg_no, int16 data) +{ + return mov_imm_to_r_i64(a, reg_no, (int64)data); +} + +/** + * encoding convert int16 register to int64 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i16_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return extend_r16_to_r64(a, reg_no_dst, reg_no_src, true); +} + +/** + * Encoding convert int32 immediate data to int8 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int8 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_i8(x86::Assembler &a, int32 reg_no, int32 data) +{ + /* (int32)(int8)data will do sign-extension */ + /* (int32)(uint32)(int8)data is longer */ + return mov_imm_to_r_i32(a, reg_no, data & 0x000000FF); +} + +/** + * Encoding convert int32 immediate data to int8 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to int8 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_i8(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + mov_r_to_r_i32(a, reg_no_dst, reg_no_src); + a.and_(regs_i32[reg_no_dst], 0x000000FF); + return true; +} + +/** + * Encoding convert int32 immediate data to uint8 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to uint8 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_u8(x86::Assembler &a, int32 reg_no, int32 data) +{ + return mov_imm_to_r_i32(a, reg_no, (uint8)data); +} + +/** + * Encoding convert int32 immediate data to uint8 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to uint8 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_u8(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return convert_r_i32_to_r_i8(a, reg_no_dst, reg_no_src); +} + +/** + * Encoding convert int32 immediate data to int16 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int16 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_i16(x86::Assembler &a, int32 reg_no, int32 data) +{ + /* (int32)(int16)data will do sign-extension */ + /* (int32)(uint32)(int16)data is longer */ + return mov_imm_to_r_i32(a, reg_no, data & 0x0000FFFF); +} + +/** + * Encoding convert int32 immediate data to int16 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to int16 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_i16(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + mov_r_to_r_i32(a, reg_no_dst, reg_no_src); + a.and_(regs_i32[reg_no_dst], 0x0000FFFF); + return true; +} + +/** + * Encoding convert int32 immediate data to uint16 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to uint16 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_u16(x86::Assembler &a, int32 reg_no, int32 data) +{ + return mov_imm_to_r_i32(a, reg_no, (uint16)data); +} + +/** + * Encoding convert int32 immediate data to uint16 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to uint16 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_u16(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return convert_r_i32_to_r_i16(a, reg_no_dst, reg_no_src); +} + +/** + * Encoding convert int32 immediate data to int64 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to uint64 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_i64(x86::Assembler &a, int32 reg_no, int32 data) +{ + return mov_imm_to_r_i64(a, reg_no, (int64)data); +} + +/** + * Encoding convert int32 register data to int64 register with signed extension + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to uint64 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return extend_r32_to_r64(a, reg_no_dst, reg_no_src, true); +} + +/** + * Encode converting int32 register data to float register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst float register + * @param reg_no_src the no of src int32 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvtsi2ss(regs_float[reg_no_dst], regs_i32[reg_no_src]); + return true; +} + +/** + * Encode converting int32 immediate data to float register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_f32(x86::Assembler &a, int32 reg_no, int32 data) +{ + mov_imm_to_r_i32(a, REG_I32_FREE_IDX, data); + return convert_r_i32_to_r_f32(a, reg_no, REG_I32_FREE_IDX); +} + +/** + * Encode converting int32 register data to double register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src int32 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvtsi2sd(regs_float[reg_no_dst], regs_i32[reg_no_src]); + return true; +} + +/** + * Encode converting int32 immediate data to double register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src immediate int32 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_f64(x86::Assembler &a, int32 reg_no, int32 data) +{ + mov_imm_to_r_i32(a, REG_I32_FREE_IDX, data); + return convert_r_i32_to_r_f64(a, reg_no, REG_I32_FREE_IDX); +} + +/** + * Encode converting int64 immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate int64 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i64_to_r_i32(x86::Assembler &a, int32 reg_no, int64 data) +{ + return mov_imm_to_r_i64(a, reg_no, (int32)data); +} + +/** + * Encode converting int64 register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src int64 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i64_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + mov_r_to_r_i64(a, reg_no_dst, reg_no_src); + a.and_(regs_i64[reg_no_dst], 0x00000000FFFFFFFFLL); + return true; +} + +/** + * Encode converting int64 immediate data to int8 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate int64 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i64_to_r_i8(x86::Assembler &a, int32 reg_no, int64 data) +{ + return mov_imm_to_r_i64(a, reg_no, (int8)data); +} + +/** + * Encode converting int64 register data to int8 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int8 register + * @param reg_no_src the no of src int64 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i64_to_r_i8(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + mov_r_to_r_i64(a, reg_no_dst, reg_no_src); + a.and_(regs_i64[reg_no_dst], 0x00000000000000FFLL); + return true; +} + +/** + * Encode converting int64 immediate data to int16 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate int64 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i64_to_r_i16(x86::Assembler &a, int32 reg_no, int64 data) +{ + return mov_imm_to_r_i64(a, reg_no, (int16)data); +} + +/** + * Encode converting int64 register data to int16 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int16 register + * @param reg_no_src the no of src int64 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i64_to_r_i16(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + mov_r_to_r_i64(a, reg_no_dst, reg_no_src); + a.and_(regs_i64[reg_no_dst], 0x000000000000FFFFLL); + return true; +} + +/** + * Encode converting uint32 immediate data to int64 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int64 register + * @param data the src immediate uint32 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_u32_to_r_i64(x86::Assembler &a, int32 reg_no, uint32 data) +{ + return mov_imm_to_r_i64(a, reg_no, (int64)(uint64)data); +} + +/** + * Encode converting uint32 register data to int64 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst uint32 register + * @param reg_no_src the no of src int64 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_u32_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return extend_r32_to_r64(a, reg_no_dst, reg_no_src, false); +} + +/** + * Encode converting uint32 immediate data to float register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src immediate uint32 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_u32_to_r_f32(x86::Assembler &a, int32 reg_no, uint32 data) +{ + mov_imm_to_r_i64(a, REG_I64_FREE_IDX, (int64)(uint64)data); + a.cvtsi2ss(regs_float[reg_no], regs_i64[REG_I64_FREE_IDX]); + return true; +} + +/** + * Encode converting uint32 register data to float register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst uint32 register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +convert_r_u32_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + extend_r32_to_r64(a, REG_I64_FREE_IDX, reg_no_src, false); + a.cvtsi2ss(regs_float[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); + return true; +} + +/** + * Encode converting uint32 immediate data to double register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src immediate uint32 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_u32_to_r_f64(x86::Assembler &a, int32 reg_no, uint32 data) +{ + mov_imm_to_r_i64(a, REG_I64_FREE_IDX, (int64)(uint64)data); + a.cvtsi2sd(regs_float[reg_no], regs_i64[REG_I64_FREE_IDX]); + return true; +} + +/** + * Encode converting uint32 register data to double register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst uint32 register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +convert_r_u32_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + extend_r32_to_r64(a, REG_I64_FREE_IDX, reg_no_src, false); + a.cvtsi2sd(regs_float[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); + return true; +} + +/** + * Encode converting int64 register data to float register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst float register + * @param reg_no_src the no of src int64 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i64_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvtsi2ss(regs_float[reg_no_dst], regs_i64[reg_no_src]); + return true; +} + +/** + * Encode converting int64 immediate data to float register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src immediate int64 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i64_to_r_f32(x86::Assembler &a, int32 reg_no, int64 data) +{ + mov_imm_to_r_i64(a, REG_I64_FREE_IDX, data); + return convert_r_i64_to_r_f32(a, reg_no, REG_I64_FREE_IDX); +} + +/** + * Encode converting int64 register data to double register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src int64 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i64_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvtsi2sd(regs_float[reg_no_dst], regs_i64[reg_no_src]); + return true; +} + +/** + * Encode converting int64 immediate data to double register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src immediate int64 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i64_to_r_f64(x86::Assembler &a, int32 reg_no, int64 data) +{ + mov_imm_to_r_i64(a, REG_I64_FREE_IDX, data); + return convert_r_i64_to_r_f64(a, reg_no, REG_I64_FREE_IDX); +} + +/** + * Encode converting float immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate float data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f32_to_r_i32(x86::Assembler &a, int32 reg_no, float data) +{ + return mov_imm_to_r_i32(a, reg_no, (int32)data); +} + +/** + * Encode converting float register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f32_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvttss2si(regs_i32[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode converting float immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate float data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f32_to_r_u32(x86::Assembler &a, int32 reg_no, float data) +{ + return mov_imm_to_r_i32(a, reg_no, (uint32)data); +} + +/** + * Encode converting float register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f32_to_r_u32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvttss2si(regs_i64[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode converting float immediate data to int64 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int64 register + * @param data the src immediate float data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f32_to_r_i64(x86::Assembler &a, int32 reg_no, float data) +{ + return mov_imm_to_r_i64(a, reg_no, (int64)data); +} + +/** + * Encode converting float register data to int64 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int64 register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f32_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvttss2si(regs_i64[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode converting float immediate data to double register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src immediate float data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f32_to_r_f64(x86::Assembler &a, int32 reg_no, float data) +{ + return mov_imm_to_r_f64(a, reg_no, (double)data); +} + +/** + * Encode converting float register data to double register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f32_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvtss2sd(regs_float[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode converting double immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate double data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f64_to_r_i32(x86::Assembler &a, int32 reg_no, double data) +{ + return mov_imm_to_r_i32(a, reg_no, (int32)data); +} + +/** + * Encode converting double register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f64_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvttsd2si(regs_i32[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode converting double immediate data to int64 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int64 register + * @param data the src immediate double data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f64_to_r_i64(x86::Assembler &a, int32 reg_no, double data) +{ + return mov_imm_to_r_i64(a, reg_no, (int64)data); +} + +/** + * Encode converting double register data to int64 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int64 register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f64_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvttsd2si(regs_i64[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode converting double immediate data to float register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src immediate double data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f64_to_r_f32(x86::Assembler &a, int32 reg_no, double data) +{ + return mov_imm_to_r_f32(a, reg_no, (float)data); +} + +/** + * Encode converting double register data to float register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst float register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f64_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvtsd2ss(regs_float[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode converting double immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate double data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f64_to_r_u32(x86::Assembler &a, int32 reg_no, double data) +{ + return mov_imm_to_r_i32(a, reg_no, (uint32)data); +} + +/** + * Encode converting double register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f64_to_r_u32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.cvttsd2si(regs_i64[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode making negative from int32 immediate data to int32 register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_i32(x86::Assembler &a, int32 reg_no, int32 data) +{ + Imm imm(-data); + a.mov(regs_i32[reg_no], imm); + return true; +} + +/** + * Encode making negative from int32 register to int32 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + mov_r_to_r_i32(a, reg_no_dst, reg_no_src); + a.neg(regs_i32[reg_no_dst]); + return true; +} + +/** + * Encode making negative from int64 immediate data to int64 register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the src int64 immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_i64(x86::Assembler &a, int32 reg_no, int64 data) +{ + Imm imm(-data); + a.mov(regs_i64[reg_no], imm); + return true; +} + +/** + * Encode making negative from int64 register to int64 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + mov_r_to_r_i64(a, reg_no_dst, reg_no_src); + a.neg(regs_i64[reg_no_dst]); + return true; +} + +/** + * Encode making negative from float immediate data to float register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src float immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_f32(x86::Assembler &a, int32 reg_no, float data) +{ + bh_assert(0); + (void)a; + (void)reg_no; + (void)data; + return false; +} + +/** + * Encode making negative from float register to float register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + bh_assert(0); + (void)a; + (void)reg_no_dst; + (void)reg_no_src; + return false; +} + +/** + * Encode making negative from double immediate data to double register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src double immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_f64(x86::Assembler &a, int32 reg_no, double data) +{ + bh_assert(0); + (void)a; + (void)reg_no; + (void)data; + return false; +} + +/** + * Encode making negative from double register to double register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + bh_assert(0); + (void)a; + (void)reg_no_dst; + (void)reg_no_src; + return false; +} + +static COND_OP +not_cond(COND_OP op) +{ + COND_OP not_list[] = { NE, EQ, LES, LTS, GES, GTS, LEU, LTU, GEU, GTU }; + + bh_assert(op <= LEU); + return not_list[op]; +} + +/** + * Encode int32 alu operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_imm_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no_src, int32 data) +{ + Imm imm(data); + + switch (op) { + case ADD: + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); + if (data == 1) + a.inc(regs_i32[reg_no_dst]); + else if (data == -1) + a.dec(regs_i32[reg_no_dst]); + else if (data != 0) + a.add(regs_i32[reg_no_dst], imm); + break; + case SUB: + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); + if (data == -1) + a.inc(regs_i32[reg_no_dst]); + else if (data == 1) + a.dec(regs_i32[reg_no_dst]); + else if (data != 0) + a.sub(regs_i32[reg_no_dst], imm); + break; + case MUL: + if (data == 0) + a.xor_(regs_i32[reg_no_dst], regs_i32[reg_no_dst]); + else if (data == -1) { + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); + a.neg(regs_i32[reg_no_dst]); + } + else if (data == 1) { + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); + } + else if (data > 0 && (data & (data - 1)) == 0x0) { + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); + data = (int32)local_log2(data); + imm.setValue(data); + a.shl(regs_i32[reg_no_dst], imm); + } + else { + a.imul(regs_i32[reg_no_dst], regs_i32[reg_no_src], imm); + } + break; + case DIV_S: + case REM_S: + bh_assert(reg_no_src == REG_EAX_IDX); + if (op == DIV_S) { + bh_assert(reg_no_dst == REG_EAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_EDX_IDX); + } + a.mov(regs_i32[REG_I32_FREE_IDX], imm); + /* signed extend eax to edx:eax */ + a.cdq(); + a.idiv(regs_i32[REG_I32_FREE_IDX]); + break; + case DIV_U: + case REM_U: + bh_assert(reg_no_src == REG_EAX_IDX); + if (op == DIV_U) { + bh_assert(reg_no_dst == REG_EAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_EDX_IDX); + } + a.mov(regs_i32[REG_I32_FREE_IDX], imm); + /* unsigned extend eax to edx:eax */ + a.xor_(regs_i32[REG_EDX_IDX], regs_i32[REG_EDX_IDX]); + a.div(regs_i32[REG_I32_FREE_IDX]); + break; + default: + bh_assert(0); + break; + } + + return true; +} + +/** + * Encode int32 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + switch (op) { + case ADD: + if (reg_no_dst != reg_no2_src) { + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no1_src); + a.add(regs_i32[reg_no_dst], regs_i32[reg_no2_src]); + } + else + a.add(regs_i32[reg_no2_src], regs_i32[reg_no1_src]); + break; + case SUB: + if (reg_no_dst != reg_no2_src) { + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no1_src); + a.sub(regs_i32[reg_no_dst], regs_i32[reg_no2_src]); + } + else { + a.sub(regs_i32[reg_no2_src], regs_i32[reg_no1_src]); + a.neg(regs_i32[reg_no2_src]); + } + break; + case MUL: + if (reg_no_dst != reg_no2_src) { + mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no1_src); + a.imul(regs_i32[reg_no_dst], regs_i32[reg_no2_src]); + } + else + a.imul(regs_i32[reg_no2_src], regs_i32[reg_no1_src]); + break; + case DIV_S: + case REM_S: + bh_assert(reg_no1_src == REG_EAX_IDX); + if (op == DIV_S) { + bh_assert(reg_no_dst == REG_EAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_EDX_IDX); + if (reg_no2_src == REG_EDX_IDX) { + /* convert `REM_S edx, eax, edx` into + `mov esi, edx` and `REM_S edx eax, rsi` to + avoid overwritting edx when a.cdq() */ + a.mov(regs_i32[REG_I32_FREE_IDX], regs_i32[REG_EDX_IDX]); + reg_no2_src = REG_I32_FREE_IDX; + } + } + /* signed extend eax to edx:eax */ + a.cdq(); + a.idiv(regs_i32[reg_no2_src]); + break; + case DIV_U: + case REM_U: + bh_assert(reg_no1_src == REG_EAX_IDX); + if (op == DIV_U) { + bh_assert(reg_no_dst == REG_EAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_EDX_IDX); + if (reg_no2_src == REG_EDX_IDX) { + /* convert `REM_U edx, eax, edx` into + `mov esi, edx` and `REM_U edx eax, rsi` to + avoid overwritting edx when unsigned extend + eax to edx:eax */ + a.mov(regs_i32[REG_I32_FREE_IDX], regs_i32[REG_EDX_IDX]); + reg_no2_src = REG_I32_FREE_IDX; + } + } + /* unsigned extend eax to edx:eax */ + a.xor_(regs_i32[REG_EDX_IDX], regs_i32[REG_EDX_IDX]); + a.div(regs_i32[reg_no2_src]); + break; + default: + bh_assert(0); + return false; + } + + return true; +} + +/** + * Encode int32 alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 data1_src, int32 data2_src) +{ + Imm imm; + int32 data = 0; + + switch (op) { + case ADD: + data = data1_src + data2_src; + break; + case SUB: + data = data1_src - data2_src; + break; + case MUL: + data = data1_src * data2_src; + break; + case DIV_S: + data = data1_src / data2_src; + break; + case REM_S: + data = data1_src % data2_src; + break; + case DIV_U: + data = (uint32)data1_src / (uint32)data2_src; + break; + case REM_U: + data = (uint32)data1_src % (uint32)data2_src; + break; + default: + bh_assert(0); + return false; + } + + imm.setValue(data); + a.mov(regs_i32[reg_no_dst], imm); + return true; +} + +/** + * Encode int32 alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 data1_src, int32 reg_no2_src) +{ + if (op == ADD || op == MUL) + return alu_r_r_imm_i32(a, op, reg_no_dst, reg_no2_src, data1_src); + else if (op == SUB) { + if (!alu_r_r_imm_i32(a, op, reg_no_dst, reg_no2_src, data1_src)) + return false; + a.neg(regs_i32[reg_no_dst]); + return true; + } + else { + if (reg_no_dst != reg_no2_src) { + if (!mov_imm_to_r_i32(a, reg_no_dst, data1_src) + || !alu_r_r_r_i32(a, op, reg_no_dst, reg_no_dst, reg_no2_src)) + return false; + return true; + } + else { + if (!mov_imm_to_r_i32(a, REG_I32_FREE_IDX, data1_src) + || !alu_r_r_r_i32(a, op, reg_no_dst, REG_I32_FREE_IDX, + reg_no2_src)) + return false; + return true; + } + } + + return true; +} + +/** + * Encode int32 alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 data2_src) +{ + return alu_r_r_imm_i32(a, op, reg_no_dst, reg_no1_src, data2_src); +} + +/** + * Encode int32 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return alu_r_r_r_i32(a, op, reg_no_dst, reg_no1_src, reg_no2_src); +} + +/** + * Encode int64 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + switch (op) { + case ADD: + if (reg_no_dst != reg_no2_src) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no1_src); + a.add(regs_i64[reg_no_dst], regs_i64[reg_no2_src]); + } + else + a.add(regs_i64[reg_no2_src], regs_i64[reg_no1_src]); + break; + case SUB: + if (reg_no_dst != reg_no2_src) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no1_src); + a.sub(regs_i64[reg_no_dst], regs_i64[reg_no2_src]); + } + else { + a.sub(regs_i64[reg_no2_src], regs_i64[reg_no1_src]); + a.neg(regs_i64[reg_no2_src]); + } + break; + case MUL: + if (reg_no_dst != reg_no2_src) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no1_src); + a.imul(regs_i64[reg_no_dst], regs_i64[reg_no2_src]); + } + else + a.imul(regs_i64[reg_no2_src], regs_i64[reg_no1_src]); + break; + case DIV_S: + case REM_S: + bh_assert(reg_no1_src == REG_RAX_IDX); + if (op == DIV_S) { + bh_assert(reg_no_dst == REG_RAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_RDX_IDX); + } + /* signed extend rax to rdx:rax */ + a.cqo(); + a.idiv(regs_i64[reg_no2_src]); + break; + case DIV_U: + case REM_U: + bh_assert(reg_no1_src == REG_RAX_IDX); + if (op == DIV_U) { + bh_assert(reg_no_dst == REG_RAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_RDX_IDX); + } + /* unsigned extend rax to rdx:rax */ + a.xor_(regs_i64[REG_RDX_IDX], regs_i64[REG_RDX_IDX]); + a.div(regs_i64[reg_no2_src]); + break; + default: + bh_assert(0); + break; + } + + return true; +} + +/** + * Encode int64 alu operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_imm_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no_src, int64 data) +{ + Imm imm(data); + + switch (op) { + case ADD: + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + if (data == 1) + a.inc(regs_i64[reg_no_dst]); + else if (data == -1) + a.dec(regs_i64[reg_no_dst]); + else if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.add(regs_i64[reg_no_dst], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.add(regs_i64[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); + } + } + break; + case SUB: + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + if (data == -1) + a.inc(regs_i64[reg_no_dst]); + else if (data == 1) + a.dec(regs_i64[reg_no_dst]); + else if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.sub(regs_i64[reg_no_dst], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.sub(regs_i64[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); + } + } + break; + case MUL: + if (data == 0) + a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_dst]); + else if (data == -1) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + a.neg(regs_i64[reg_no_dst]); + } + else if (data == 1) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + } + else if (data > 0 && (data & (data - 1)) == 0x0) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + data = (int64)local_log2l(data); + imm.setValue(data); + a.shl(regs_i64[reg_no_dst], imm); + } + else if (INT32_MIN <= data && data <= INT32_MAX) { + a.imul(regs_i64[reg_no_dst], regs_i64[reg_no_src], imm); + } + else { + mov_imm_to_r_i64( + a, reg_no_dst == reg_no_src ? REG_I64_FREE_IDX : reg_no_dst, + data); + alu_r_r_r_i64(a, op, reg_no_dst, + reg_no_dst == reg_no_src ? REG_I64_FREE_IDX + : reg_no_dst, + reg_no_src); + } + break; + case DIV_S: + case REM_S: + bh_assert(reg_no_src == REG_RAX_IDX); + if (op == DIV_S) { + bh_assert(reg_no_dst == REG_RAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_RDX_IDX); + } + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + /* signed extend rax to rdx:rax */ + a.cqo(); + a.idiv(regs_i64[REG_I64_FREE_IDX]); + break; + case DIV_U: + case REM_U: + bh_assert(reg_no_src == REG_RAX_IDX); + if (op == DIV_U) { + bh_assert(reg_no_dst == REG_RAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_RDX_IDX); + } + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + /* unsigned extend rax to rdx:rax */ + a.xor_(regs_i64[REG_RDX_IDX], regs_i64[REG_RDX_IDX]); + a.div(regs_i64[REG_I64_FREE_IDX]); + break; + default: + bh_assert(0); + break; + } + + return true; +} + +/** + * Encode int64 alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int64 data1_src, int64 data2_src) +{ + Imm imm; + int64 data = 0; + + switch (op) { + case ADD: + data = data1_src + data2_src; + break; + case SUB: + data = data1_src - data2_src; + break; + case MUL: + data = data1_src * data2_src; + break; + case DIV_S: + data = data1_src / data2_src; + break; + case REM_S: + data = data1_src % data2_src; + break; + case DIV_U: + data = (uint64)data1_src / (uint64)data2_src; + break; + case REM_U: + data = (uint64)data1_src % (uint64)data2_src; + break; + default: + bh_assert(0); + break; + } + + imm.setValue(data); + a.mov(regs_i64[reg_no_dst], imm); + return true; +} + +/** + * Encode int64 alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int64 data1_src, int32 reg_no2_src) +{ + if (op == ADD || op == MUL) + return alu_r_r_imm_i64(a, op, reg_no_dst, reg_no2_src, data1_src); + else if (op == SUB) { + if (!alu_r_r_imm_i64(a, op, reg_no_dst, reg_no2_src, data1_src)) + return false; + a.neg(regs_i64[reg_no_dst]); + return true; + } + else { + if (reg_no_dst != reg_no2_src) { + if (!mov_imm_to_r_i64(a, reg_no_dst, data1_src) + || !alu_r_r_r_i64(a, op, reg_no_dst, reg_no_dst, reg_no2_src)) + return false; + return true; + } + else { + if (!mov_imm_to_r_i64(a, REG_I64_FREE_IDX, data1_src) + || !alu_r_r_r_i64(a, op, reg_no_dst, REG_I64_FREE_IDX, + reg_no2_src)) + return false; + return true; + } + } + + return true; +} + +/** + * Encode int64 alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int64 data2_src) +{ + return alu_r_r_imm_i64(a, op, reg_no_dst, reg_no1_src, data2_src); +} + +/** + * Encode int64 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return alu_r_r_r_i64(a, op, reg_no_dst, reg_no1_src, reg_no2_src); +} + +/** + * Encode float alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + float data1_src, float data2_src) +{ + Imm imm; + float data = 0; + + switch (op) { + case ADD: + { + data = data1_src + data2_src; + break; + } + case SUB: + { + data = data1_src - data2_src; + break; + } + case MUL: + { + data = data1_src * data2_src; + break; + } + case DIV_S: + { + data = data1_src / data2_src; + break; + } + case MAX: + { + data = fmaxf(data1_src, data2_src); + break; + } + case MIN: + { + data = fminf(data1_src, data2_src); + break; + } + default: + { + bh_assert(0); + return false; + } + } + + return mov_imm_to_r_f32(a, reg_no_dst, data); +} + +static bool +alu_r_m_float(x86::Assembler &a, ALU_OP op, int32 reg_no, x86::Mem &m, + bool is_f32) +{ + switch (op) { + case ADD: + { + if (is_f32) + a.addss(regs_float[reg_no], m); + else + a.addsd(regs_float[reg_no], m); + break; + } + case SUB: + { + if (is_f32) + a.subss(regs_float[reg_no], m); + else + a.subsd(regs_float[reg_no], m); + break; + } + case MUL: + { + if (is_f32) + a.mulss(regs_float[reg_no], m); + else + a.mulsd(regs_float[reg_no], m); + break; + } + case DIV_S: + { + if (is_f32) + a.divss(regs_float[reg_no], m); + else + a.divsd(regs_float[reg_no], m); + break; + } + case MAX: + { + if (is_f32) + a.maxss(regs_float[reg_no], m); + else + a.maxsd(regs_float[reg_no], m); + break; + } + case MIN: + { + if (is_f32) + a.minss(regs_float[reg_no], m); + else + a.minsd(regs_float[reg_no], m); + break; + } + default: + { + bh_assert(0); + return false; + } + } + return true; +} + +/** + * Encode float alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + float data1_src, int32 reg_no2_src) +{ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + /* xmm -> m128 */ + x86::Mem cache = x86::xmmword_ptr(regs_i64[hreg_info->exec_env_hreg_index], + offsetof(WASMExecEnv, jit_cache)); + a.movups(cache, regs_float[reg_no2_src]); + + /* imm -> gp -> xmm */ + mov_imm_to_r_f32(a, reg_no_dst, data1_src); + + return alu_r_m_float(a, op, reg_no_dst, cache, true); +} + +/** + * Encode float alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, float data2_src) +{ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + /* imm -> m32 */ + x86::Mem cache = x86::dword_ptr(regs_i64[hreg_info->exec_env_hreg_index], + offsetof(WASMExecEnv, jit_cache)); + cast_float_to_integer v = { .f = data2_src }; + Imm imm(v.i); + mov_imm_to_m(a, cache, imm, 4); + + mov_r_to_r_f32(a, reg_no_dst, reg_no1_src); + return alu_r_m_float(a, op, reg_no_dst, cache, true); +} + +/** + * Encode float alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + bool store_result = false; + + /** + * - op r0,r0,r1. do nothing since instructions always store results in + * the first register + * + * - op r1,r0,r1. use FREE_REG to cache and replace r0, and then store + * results in r1 + * + * - op r0,r1,r2. use r0 to cache and replace r1, and accept the result + * naturally + **/ + if (reg_no_dst == reg_no2_src) { + store_result = true; + reg_no_dst = REG_F32_FREE_IDX; + } + mov_r_to_r_f32(a, reg_no_dst, reg_no1_src); + + switch (op) { + case ADD: + { + a.addss(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case SUB: + { + a.subss(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case MUL: + { + a.mulss(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case DIV_S: + { + a.divss(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case MAX: + { + a.maxss(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case MIN: + { + a.minss(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + default: + { + bh_assert(0); + return false; + } + } + + if (store_result) + mov_r_to_r_f32(a, reg_no2_src, REG_F32_FREE_IDX); + + return true; +} + +/** + * Encode double alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + double data1_src, double data2_src) +{ + Imm imm; + double data = 0; + + switch (op) { + case ADD: + { + data = data1_src + data2_src; + break; + } + case SUB: + { + data = data1_src - data2_src; + break; + } + case MUL: + { + data = data1_src * data2_src; + break; + } + case DIV_S: + { + data = data1_src / data2_src; + break; + } + case MAX: + { + data = fmax(data1_src, data2_src); + break; + } + case MIN: + { + data = fmin(data1_src, data2_src); + break; + } + default: + { + bh_assert(0); + return false; + } + } + + return mov_imm_to_r_f64(a, reg_no_dst, data); +} + +/** + * Encode double alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + double data1_src, int32 reg_no2_src) +{ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + /* xmm -> m128 */ + x86::Mem cache = x86::qword_ptr(regs_i64[hreg_info->exec_env_hreg_index], + offsetof(WASMExecEnv, jit_cache)); + a.movupd(cache, regs_float[reg_no2_src]); + + /* imm -> gp -> xmm */ + mov_imm_to_r_f64(a, reg_no_dst, data1_src); + + return alu_r_m_float(a, op, reg_no_dst, cache, false); +} + +/** + * Encode double alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, double data2_src) +{ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + /* imm -> m64 */ + x86::Mem cache = x86::qword_ptr(regs_i64[hreg_info->exec_env_hreg_index], + offsetof(WASMExecEnv, jit_cache)); + cast_double_to_integer v = { .d = data2_src }; + Imm imm(v.i); + mov_imm_to_m(a, cache, imm, 8); + + mov_r_to_r_f64(a, reg_no_dst, reg_no1_src); + return alu_r_m_float(a, op, reg_no_dst, cache, false); +} + +/** + * Encode double alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + bool store_result = false; + + /** + * - op r0,r0,r1. do nothing since instructions always store results in + * the first register + * + * - op r1,r0,r1. use FREE_REG to cache and replace r0, and then store + * results in r1 + * + * - op r0,r1,r2. use r0 to cache and replace r1, and accept the result + * naturally + **/ + if (reg_no_dst == reg_no2_src) { + store_result = true; + reg_no_dst = REG_F64_FREE_IDX; + } + mov_r_to_r_f64(a, reg_no_dst, reg_no1_src); + + switch (op) { + case ADD: + { + a.addsd(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case SUB: + { + a.subsd(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case MUL: + { + a.mulsd(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case DIV_S: + { + a.divsd(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case MAX: + { + a.maxsd(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + case MIN: + { + a.minsd(regs_float[reg_no_dst], regs_float[reg_no2_src]); + break; + } + default: + { + bh_assert(0); + return false; + } + } + + if (store_result) + mov_r_to_r_f64(a, reg_no2_src, REG_F64_FREE_IDX); + + return true; +} + +/** + * Encode int32 bit operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_i32(x86::Assembler &a, BIT_OP op, int32 reg_no, int32 data) +{ + Imm imm(data); + + switch (op) { + case OR: + if (data != 0) + a.or_(regs_i32[reg_no], imm); + break; + case XOR: + if (data == -1) + a.not_(regs_i32[reg_no]); + else if (data != 0) + a.xor_(regs_i32[reg_no], imm); + break; + case AND: + if (data != -1) + a.and_(regs_i32[reg_no], imm); + break; + default: + bh_assert(0); + break; + } + return true; +} + +/** + * Encode int32 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no_src) +{ + switch (op) { + case OR: + a.or_(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + break; + case XOR: + a.xor_(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + break; + case AND: + a.and_(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + break; + default: + bh_assert(0); + break; + } + return true; +} + +/** + * Encode int32 bit operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_imm_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 data1_src, int32 data2_src) +{ + Imm imm; + + switch (op) { + case OR: + imm.setValue(data1_src | data2_src); + break; + case XOR: + imm.setValue(data1_src ^ data2_src); + break; + case AND: + imm.setValue(data1_src & data2_src); + break; + default: + bh_assert(0); + break; + } + + a.mov(regs_i32[reg_no_dst], imm); + return true; +} + +/** + * Encode int32 bit operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_r_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 data1_src, int32 reg_no2_src) +{ + if (op == AND && data1_src == 0) + a.xor_(regs_i32[reg_no_dst], regs_i32[reg_no_dst]); + else if (op == OR && data1_src == -1) { + Imm imm(-1); + a.mov(regs_i32[reg_no_dst], imm); + } + else { + mov_r_to_r_i32(a, reg_no_dst, reg_no2_src); + return bit_r_imm_i32(a, op, reg_no_dst, data1_src); + } + return true; +} + +/** + * Encode int32 bit operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 data2_src) +{ + return bit_imm_r_to_r_i32(a, op, reg_no_dst, data2_src, reg_no1_src); +} + +/** + * Encode int32 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + if (reg_no_dst != reg_no2_src) { + mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); + return bit_r_r_i32(a, op, reg_no_dst, reg_no2_src); + } + else + return bit_r_r_i32(a, op, reg_no_dst, reg_no1_src); + return false; +} + +/** + * Encode int64 bit operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_i64(x86::Assembler &a, BIT_OP op, int32 reg_no, int64 data) +{ + Imm imm(data); + + switch (op) { + case OR: + if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.or_(regs_i64[reg_no], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.or_(regs_i64[reg_no], regs_i64[REG_I64_FREE_IDX]); + } + } + break; + case XOR: + if (data == -1LL) + a.not_(regs_i64[reg_no]); + else if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.xor_(regs_i64[reg_no], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.xor_(regs_i64[reg_no], regs_i64[REG_I64_FREE_IDX]); + } + } + break; + case AND: + if (data != -1LL) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.and_(regs_i64[reg_no], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.and_(regs_i64[reg_no], regs_i64[REG_I64_FREE_IDX]); + } + } + break; + default: + bh_assert(0); + break; + } + return true; +} + +/** + * Encode int64 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no_src) +{ + switch (op) { + case OR: + a.or_(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + case XOR: + a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + case AND: + a.and_(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + default: + bh_assert(0); + break; + } + return true; +} + +/** + * Encode int64 bit operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 data1_src, int64 data2_src) +{ + Imm imm; + + switch (op) { + case OR: + imm.setValue(data1_src | data2_src); + break; + case XOR: + imm.setValue(data1_src ^ data2_src); + break; + case AND: + imm.setValue(data1_src & data2_src); + break; + default: + bh_assert(0); + break; + } + + a.mov(regs_i64[reg_no_dst], imm); + return true; +} + +/** + * Encode int64 bit operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int64 data1_src, int32 reg_no2_src) +{ + if (op == AND && data1_src == 0) + a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_dst]); + else if (op == OR && data1_src == -1LL) { + Imm imm(-1LL); + a.mov(regs_i64[reg_no_dst], imm); + } + else { + mov_r_to_r_i64(a, reg_no_dst, reg_no2_src); + return bit_r_imm_i64(a, op, reg_no_dst, data1_src); + } + return true; +} + +/** + * Encode int64 bit operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int64 data2_src) +{ + return bit_imm_r_to_r_i64(a, op, reg_no_dst, data2_src, reg_no1_src); +} + +/** + * Encode int64 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + if (reg_no_dst != reg_no2_src) { + mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); + return bit_r_r_i64(a, op, reg_no_dst, reg_no2_src); + } + else + return bit_r_r_i64(a, op, reg_no_dst, reg_no1_src); + return false; +} + +/** + * Encode int32 shift operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 data1_src, int32 data2_src) +{ + int32 data; + switch (op) { + case SHL: + { + data = data1_src << data2_src; + break; + } + case SHRS: + { + data = data1_src >> data2_src; + break; + } + case SHRU: + { + data = ((uint32)data1_src) >> data2_src; + break; + } + case ROTL: + { + data = (data1_src << data2_src) + | (((uint32)data1_src) >> (32 - data2_src)); + break; + } + case ROTR: + { + data = (((uint32)data1_src) >> data2_src) + | (data1_src << (32 - data2_src)); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + return mov_imm_to_r_i32(a, reg_no_dst, data); +fail: + return false; +} + +/** + * Encode int32 shift operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 data1_src, int32 reg_no2_src) +{ + /* Should have been optimized by previous lower */ + bh_assert(0); + (void)a; + (void)op; + (void)reg_no_dst; + (void)data1_src; + (void)reg_no2_src; + return false; +} + +/** + * Encode int32 shift operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_r_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 data2_src) +{ + /* SHL/SHA/SHR r/m32, imm8 */ + Imm imm((uint8)data2_src); + + mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); + switch (op) { + case SHL: + { + a.shl(regs_i32[reg_no_dst], imm); + break; + } + case SHRS: + { + a.sar(regs_i32[reg_no_dst], imm); + break; + } + case SHRU: + { + a.shr(regs_i32[reg_no_dst], imm); + break; + } + case ROTL: + { + a.rol(regs_i32[reg_no_dst], imm); + break; + } + case ROTR: + { + a.ror(regs_i32[reg_no_dst], imm); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + return true; +fail: + return false; +} + +/** + * Encode int32 shift operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of shift operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_r_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + /* should be CL */ + if (reg_no2_src != REG_ECX_IDX) + return false; + + mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); + + switch (op) { + case SHL: + { + a.shl(regs_i32[reg_no_dst], x86::cl); + break; + } + case SHRS: + { + a.sar(regs_i32[reg_no_dst], x86::cl); + break; + } + case SHRU: + { + a.shr(regs_i32[reg_no_dst], x86::cl); + break; + } + case ROTL: + { + a.rol(regs_i32[reg_no_dst], x86::cl); + break; + } + case ROTR: + { + a.ror(regs_i32[reg_no_dst], x86::cl); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + return true; +fail: + return false; +} + +/** + * Encode int64 shift operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int64 data1_src, int64 data2_src) +{ + int64 data; + + switch (op) { + case SHL: + { + data = data1_src << data2_src; + break; + } + case SHRS: + { + data = data1_src >> data2_src; + break; + } + case SHRU: + { + data = ((uint64)data1_src) >> data2_src; + break; + } + case ROTL: + { + data = (data1_src << data2_src) + | (((uint64)data1_src) >> (64LL - data2_src)); + break; + } + case ROTR: + { + data = (((uint64)data1_src) >> data2_src) + | (data1_src << (64LL - data2_src)); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + return mov_imm_to_r_i64(a, reg_no_dst, data); +fail: + return false; +} + +/** + * Encode int64 shift operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int64 data1_src, int32 reg_no2_src) +{ + /* Should have been optimized by previous lower */ + bh_assert(0); + (void)a; + (void)op; + (void)reg_no_dst; + (void)data1_src; + (void)reg_no2_src; + return false; +} + +/** + * Encode int64 shift operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_r_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int64 data2_src) +{ + /* SHL/SHA/SHR r/m64, imm8 */ + Imm imm((uint8)data2_src); + + mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); + switch (op) { + case SHL: + { + a.shl(regs_i64[reg_no_dst], imm); + break; + } + case SHRS: + { + a.sar(regs_i64[reg_no_dst], imm); + break; + } + case SHRU: + { + a.shr(regs_i64[reg_no_dst], imm); + break; + } + case ROTL: + { + a.rol(regs_i64[reg_no_dst], imm); + break; + } + case ROTR: + { + a.ror(regs_i64[reg_no_dst], imm); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + return true; +fail: + return false; +} + +/** + * Encode int64 shift operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of shift operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_r_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + /* should be CL */ + if (reg_no2_src != REG_ECX_IDX) + return false; + + mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); + + switch (op) { + case SHL: + { + a.shl(regs_i64[reg_no_dst], x86::cl); + break; + } + case SHRS: + { + a.sar(regs_i64[reg_no_dst], x86::cl); + break; + } + case SHRU: + { + a.shr(regs_i64[reg_no_dst], x86::cl); + break; + } + case ROTL: + { + a.rol(regs_i64[reg_no_dst], x86::cl); + break; + } + case ROTR: + { + a.ror(regs_i64[reg_no_dst], x86::cl); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + return true; +fail: + return false; +} + +/** + * Encode int32 cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 data1_src, + int32 data2_src) +{ + Imm imm(data1_src); + a.mov(regs_i32[REG_I32_FREE_IDX], imm); + imm.setValue(data2_src); + a.cmp(regs_i32[REG_I32_FREE_IDX], imm); + (void)reg_no_dst; + return true; +} + +/** + * Encode int32 cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 data1_src, + int32 reg_no2_src) +{ + Imm imm(data1_src); + a.mov(regs_i32[REG_I32_FREE_IDX], imm); + a.cmp(regs_i32[REG_I32_FREE_IDX], regs_i32[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode int32 cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 data2_src) +{ + Imm imm(data2_src); + a.cmp(regs_i32[reg_no1_src], imm); + (void)reg_no_dst; + return true; +} + +/** + * Encode int32 cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + a.cmp(regs_i32[reg_no1_src], regs_i32[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode int64 cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int64 data1_src, + int64 data2_src) +{ + /* imm -> m64 */ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + x86::Mem mem = x86::qword_ptr(regs_i64[hreg_info->exec_env_hreg_index], + offsetof(WASMExecEnv, jit_cache)); + Imm imm(data2_src); + mov_imm_to_m(a, mem, imm, 8); + + a.mov(regs_i64[REG_I64_FREE_IDX], data1_src); + a.cmp(regs_i64[REG_I64_FREE_IDX], mem); + (void)reg_no_dst; + return true; +} + +/** + * Encode int64 cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int64 data1_src, + int32 reg_no2_src) +{ + Imm imm(data1_src); + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.cmp(regs_i64[REG_I64_FREE_IDX], regs_i64[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode int64 cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int64 data2_src) +{ + Imm imm(data2_src); + + if (data2_src >= INT32_MIN && data2_src <= INT32_MAX) { + imm.setValue((int32)data2_src); + a.cmp(regs_i64[reg_no1_src], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.cmp(regs_i64[reg_no1_src], regs_i64[REG_I64_FREE_IDX]); + } + (void)reg_no_dst; + return true; +} + +/** + * Encode int64 cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + a.cmp(regs_i64[reg_no1_src], regs_i64[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode float cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + a.comiss(regs_float[reg_no1_src], regs_float[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode float cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_f32(x86::Assembler &a, int32 reg_no_dst, float data1_src, + float data2_src) +{ + /* should have been optimized in the frontend */ + bh_assert(0); + (void)a; + (void)reg_no_dst; + (void)data1_src; + (void)data2_src; + return false; +} + +/** + * Encode float cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, float data1_src, + int32 reg_no2_src) +{ + mov_imm_to_r_f32(a, REG_F32_FREE_IDX, data1_src); + a.comiss(regs_float[REG_F32_FREE_IDX], regs_float[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode float cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + float data2_src) +{ + mov_imm_to_r_f32(a, REG_F32_FREE_IDX, data2_src); + a.comiss(regs_float[reg_no1_src], regs_float[REG_F32_FREE_IDX]); + (void)reg_no_dst; + return true; +} + +/** + * Encode double cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + a.comisd(regs_float[reg_no1_src], regs_float[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode double cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_f64(x86::Assembler &a, int32 reg_no_dst, double data1_src, + double data2_src) +{ + /* should have been optimized in the frontend */ + bh_assert(0); + (void)a; + (void)reg_no_dst; + (void)data1_src; + (void)data2_src; + return false; +} + +/** + * Encode double cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, double data1_src, + int32 reg_no2_src) +{ + mov_imm_to_r_f64(a, REG_F64_FREE_IDX, data1_src); + a.comisd(regs_float[REG_F64_FREE_IDX], regs_float[reg_no2_src]); + (void)reg_no_dst; + return true; +} + +/** + * Encode double cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + double data2_src) +{ + mov_imm_to_r_f64(a, REG_F64_FREE_IDX, data2_src); + a.comisd(regs_float[reg_no1_src], regs_float[REG_F64_FREE_IDX]); + (void)reg_no_dst; + return true; +} + +/** + * Encode insn ld: LD_type r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param bytes_dst the byte number of dst data + * @param is_signed the data is signed or unsigned + */ +#define LD_R_R_R(kind, bytes_dst, is_signed) \ + do { \ + int32 reg_no_dst = 0, reg_no_base = 0, reg_no_offset = 0; \ + int32 base = 0, offset = 0; \ + bool _ret = false; \ + \ + if (jit_reg_is_const(r1)) { \ + CHECK_KIND(r1, JIT_REG_KIND_I32); \ + } \ + else { \ + CHECK_KIND(r1, JIT_REG_KIND_I64); \ + } \ + if (jit_reg_is_const(r2)) { \ + CHECK_KIND(r2, JIT_REG_KIND_I32); \ + } \ + else { \ + CHECK_KIND(r2, JIT_REG_KIND_I64); \ + } \ + \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + if (jit_reg_is_const(r1)) \ + base = jit_cc_get_const_I32(cc, r1); \ + else { \ + reg_no_base = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_base, jit_reg_kind(r1)); \ + } \ + if (jit_reg_is_const(r2)) \ + offset = jit_cc_get_const_I32(cc, r2); \ + else { \ + reg_no_offset = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_offset, jit_reg_kind(r2)); \ + } \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = ld_r_from_base_imm_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + base, offset); \ + else \ + _ret = ld_r_from_base_imm_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + base, reg_no_offset); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = ld_r_from_base_r_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + reg_no_base, offset); \ + else \ + _ret = ld_r_from_base_r_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + reg_no_base, reg_no_offset); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode insn sd: ST_type r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param bytes_dst the byte number of dst data + * @param atomic whether it's atomic store + */ +#define ST_R_R_R(kind, type, bytes_dst, atomic) \ + do { \ + type data_src = 0; \ + int32 reg_no_src = 0, reg_no_base = 0, reg_no_offset = 0; \ + int32 base = 0, offset = 0; \ + bool _ret = false; \ + \ + if (jit_reg_is_const(r1)) { \ + CHECK_KIND(r1, JIT_REG_KIND_I32); \ + } \ + else { \ + CHECK_KIND(r1, JIT_REG_KIND_I64); \ + } \ + if (jit_reg_is_const(r2)) { \ + CHECK_KIND(r2, JIT_REG_KIND_I32); \ + } \ + else { \ + CHECK_KIND(r2, JIT_REG_KIND_I64); \ + } \ + \ + if (jit_reg_is_const(r0)) \ + data_src = jit_cc_get_const_##kind(cc, r0); \ + else { \ + reg_no_src = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_src, jit_reg_kind(r0)); \ + } \ + if (jit_reg_is_const(r1)) \ + base = jit_cc_get_const_I32(cc, r1); \ + else { \ + reg_no_base = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_base, jit_reg_kind(r1)); \ + } \ + if (jit_reg_is_const(r2)) \ + offset = jit_cc_get_const_I32(cc, r2); \ + else { \ + reg_no_offset = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_offset, jit_reg_kind(r2)); \ + } \ + \ + if (jit_reg_is_const(r0)) { \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = st_imm_to_base_imm_offset_imm( \ + a, bytes_dst, &data_src, base, offset, atomic); \ + else \ + _ret = st_imm_to_base_imm_offset_r( \ + a, bytes_dst, &data_src, base, reg_no_offset, atomic); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = st_imm_to_base_r_offset_imm( \ + a, bytes_dst, &data_src, reg_no_base, offset, atomic); \ + else \ + _ret = st_imm_to_base_r_offset_r(a, bytes_dst, &data_src, \ + reg_no_base, reg_no_offset, \ + atomic); \ + } \ + else if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = st_r_to_base_imm_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_src, base, \ + offset, atomic); \ + else \ + _ret = st_r_to_base_imm_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_src, base, \ + reg_no_offset, atomic); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = st_r_to_base_r_offset_imm(a, bytes_dst, \ + JIT_REG_KIND_##kind, reg_no_src, \ + reg_no_base, offset, atomic); \ + else \ + _ret = st_r_to_base_r_offset_r(a, bytes_dst, JIT_REG_KIND_##kind, \ + reg_no_src, reg_no_base, \ + reg_no_offset, atomic); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode insn mov: MOV r0, r1 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + * @param bytes_dst the byte number of dst data + */ +#define MOV_R_R(kind, Type, type) \ + do { \ + bool _ret = false; \ + int32 reg_no_dst = 0, reg_no_src = 0; \ + CHECK_EQKIND(r0, r1); \ + \ + CHECK_NCONST(r0); \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + \ + if (jit_reg_is_const(r1)) { \ + Type data = jit_cc_get_const_##kind(cc, r1); \ + _ret = mov_imm_to_r_##type(a, reg_no_dst, data); \ + } \ + else { \ + reg_no_src = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src, jit_reg_kind(r1)); \ + _ret = mov_r_to_r_##type(a, reg_no_dst, reg_no_src); \ + } \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode mov insn, MOV r0, r1 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the src operand info + * + * @return true if success, false if failed + */ +static bool +lower_mov(JitCompContext *cc, x86::Assembler &a, JitReg r0, JitReg r1) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + MOV_R_R(I32, int32, i32); + break; + case JIT_REG_KIND_I64: + MOV_R_R(I64, int64, i64); + break; + case JIT_REG_KIND_F32: + MOV_R_R(F32, float32, f32); + break; + case JIT_REG_KIND_F64: + MOV_R_R(F64, float64, f64); + break; + default: + LOG_VERBOSE("Invalid reg type of mov: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn neg: NEG r0, r1 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + */ +#define NEG_R_R(kind, Type, type) \ + do { \ + bool _ret = false; \ + int32 reg_no_dst = 0, reg_no_src = 0; \ + CHECK_EQKIND(r0, r1); \ + \ + CHECK_NCONST(r0); \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + \ + if (jit_reg_is_const(r1)) { \ + Type data = jit_cc_get_const_##kind(cc, r1); \ + _ret = neg_imm_to_r_##type(a, reg_no_dst, data); \ + } \ + else { \ + reg_no_src = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src, jit_reg_kind(r1)); \ + _ret = neg_r_to_r_##type(a, reg_no_dst, reg_no_src); \ + } \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode neg insn, NEG r0, r1 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the src operand info + * + * @return true if success, false if failed + */ +static bool +lower_neg(JitCompContext *cc, x86::Assembler &a, JitReg r0, JitReg r1) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + NEG_R_R(I32, int32, i32); + break; + case JIT_REG_KIND_I64: + NEG_R_R(I64, int64, i64); + break; + case JIT_REG_KIND_F32: + NEG_R_R(F32, float32, f32); + break; + case JIT_REG_KIND_F64: + NEG_R_R(F64, float64, f64); + break; + default: + LOG_VERBOSE("Invalid reg type of neg: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn convert: I32TOI8 r0, r1, or I32TOI16, I32TOF32, F32TOF64, etc. + * @param kind0 the dst JIT_REG_KIND, such as I32, I64, F32 and F64 + * @param kind1 the src JIT_REG_KIND, such as I32, I64, F32 and F64 + * @param type0 the dst data type, such as i8, u8, i16, u16, i32, f32, i64, f32, + * f64 + * @param type1 the src data type, such as i8, u8, i16, u16, i32, f32, i64, f32, + * f64 + */ +#define CONVERT_R_R(kind0, kind1, type0, type1, Type1) \ + do { \ + bool _ret = false; \ + int32 reg_no_dst = 0, reg_no_src = 0; \ + CHECK_KIND(r0, JIT_REG_KIND_##kind0); \ + CHECK_KIND(r1, JIT_REG_KIND_##kind1); \ + \ + CHECK_NCONST(r0); \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + \ + if (jit_reg_is_const(r1)) { \ + Type1 data = jit_cc_get_const_##kind1(cc, r1); \ + _ret = convert_imm_##type1##_to_r_##type0(a, reg_no_dst, data); \ + } \ + else { \ + reg_no_src = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src, jit_reg_kind(r1)); \ + _ret = \ + convert_r_##type1##_to_r_##type0(a, reg_no_dst, reg_no_src); \ + } \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode insn alu: ADD/SUB/MUL/DIV/REM r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + * @param op the opcode of alu + */ +#define ALU_R_R_R(kind, Type, type, op) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst = 0, reg_no_src1 = 0, reg_no_src2 = 0; \ + bool _ret = false; \ + \ + CHECK_EQKIND(r0, r1); \ + CHECK_EQKIND(r0, r2); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + else { \ + reg_no_src1 = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src1, jit_reg_kind(r1)); \ + } \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + else { \ + reg_no_src2 = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_src2, jit_reg_kind(r2)); \ + } \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = \ + alu_imm_imm_to_r_##type(a, op, reg_no_dst, data1, data2); \ + else \ + _ret = alu_imm_r_to_r_##type(a, op, reg_no_dst, data1, \ + reg_no_src2); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = \ + alu_r_imm_to_r_##type(a, op, reg_no_dst, reg_no_src1, data2); \ + else \ + _ret = alu_r_r_to_r_##type(a, op, reg_no_dst, reg_no_src1, \ + reg_no_src2); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode alu insn, ADD/SUB/MUL/DIV/REM r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the opcode of alu operations + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_alu(JitCompContext *cc, x86::Assembler &a, ALU_OP op, JitReg r0, + JitReg r1, JitReg r2) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + ALU_R_R_R(I32, int32, i32, op); + break; + case JIT_REG_KIND_I64: + ALU_R_R_R(I64, int64, i64, op); + break; + case JIT_REG_KIND_F32: + ALU_R_R_R(F32, float32, f32, op); + break; + case JIT_REG_KIND_F64: + ALU_R_R_R(F64, float64, f64, op); + break; + default: + LOG_VERBOSE("Invalid reg type of alu: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn bit: AND/OR/XOR r0, r1, r2 + * @param kind the data kind, such as I32, I64 + * @param Type the data type, such as int32, int64 + * @param type the abbreviation of data type, such as i32, i64 + * @param op the opcode of bit operation + */ +#define BIT_R_R_R(kind, Type, type, op) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst = 0, reg_no_src1 = 0, reg_no_src2 = 0; \ + bool _ret = false; \ + \ + CHECK_EQKIND(r0, r1); \ + CHECK_EQKIND(r0, r2); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + else { \ + reg_no_src1 = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src1, jit_reg_kind(r1)); \ + } \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + else { \ + reg_no_src2 = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_src2, jit_reg_kind(r2)); \ + } \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = \ + bit_imm_imm_to_r_##type(a, op, reg_no_dst, data1, data2); \ + else \ + _ret = bit_imm_r_to_r_##type(a, op, reg_no_dst, data1, \ + reg_no_src2); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = \ + bit_r_imm_to_r_##type(a, op, reg_no_dst, reg_no_src1, data2); \ + else \ + _ret = bit_r_r_to_r_##type(a, op, reg_no_dst, reg_no_src1, \ + reg_no_src2); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode bit insn, AND/OR/XOR r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the opcode of bit operations + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_bit(JitCompContext *cc, x86::Assembler &a, BIT_OP op, JitReg r0, + JitReg r1, JitReg r2) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + BIT_R_R_R(I32, int32, i32, op); + break; + case JIT_REG_KIND_I64: + BIT_R_R_R(I64, int64, i64, op); + break; + default: + LOG_VERBOSE("Invalid reg type of bit: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn shift: SHL/SHRS/SHRU r0, r1, r2 + * @param kind the data kind, such as I32, I64 + * @param Type the data type, such as int32, int64 + * @param type the abbreviation of data type, such as i32, i64 + * @param op the opcode of shift operation + */ +#define SHIFT_R_R_R(kind, Type, type, op) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst = 0, reg_no_src1 = 0, reg_no_src2 = 0; \ + bool _ret = false; \ + \ + CHECK_EQKIND(r0, r1); \ + CHECK_KIND(r2, JIT_REG_KIND_##kind); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + else { \ + reg_no_src1 = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src1, jit_reg_kind(r1)); \ + } \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + else { \ + reg_no_src2 = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_src2, jit_reg_kind(r2)); \ + } \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = shift_imm_imm_to_r_##type(a, op, reg_no_dst, data1, \ + data2); \ + else \ + _ret = shift_imm_r_to_r_##type(a, op, reg_no_dst, data1, \ + reg_no_src2); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = shift_r_imm_to_r_##type(a, op, reg_no_dst, reg_no_src1, \ + data2); \ + else \ + _ret = shift_r_r_to_r_##type(a, op, reg_no_dst, reg_no_src1, \ + reg_no_src2); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode shift insn, SHL/SHRS/SHRU r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the opcode of shift operations + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_shift(JitCompContext *cc, x86::Assembler &a, SHIFT_OP op, JitReg r0, + JitReg r1, JitReg r2) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + SHIFT_R_R_R(I32, int32, i32, op); + break; + case JIT_REG_KIND_I64: + SHIFT_R_R_R(I64, int64, i64, op); + break; + default: + LOG_VERBOSE("Invalid reg type of shift: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode int32 bitcount operation of reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BITCOUNT operation + * @param reg_no_dst the no of register + * @param reg_no_src the reg no of first src register data + * + * @return true if success, false otherwise + */ +static bool +bitcount_r_to_r_i32(x86::Assembler &a, BITCOUNT_OP op, int32 reg_no_dst, + int32 reg_no_src) +{ + switch (op) { + case CLZ: + a.lzcnt(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + break; + case CTZ: + a.tzcnt(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + break; + case POPCNT: + a.popcnt(regs_i32[reg_no_dst], regs_i32[reg_no_src]); + break; + default: + bh_assert(0); + return false; + } + return true; +} + +/** + * Encode int64 bitcount operation of reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BITCOUNT operation + * @param reg_no_dst the no of register + * @param reg_no_src the reg no of first src register data + * + * @return true if success, false otherwise + */ +static bool +bitcount_r_to_r_i64(x86::Assembler &a, BITCOUNT_OP op, int32 reg_no_dst, + int32 reg_no_src) +{ + switch (op) { + case CLZ: + a.lzcnt(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + case CTZ: + a.tzcnt(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + case POPCNT: + a.popcnt(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + default: + bh_assert(0); + return false; + } + return true; +} + +/** + * Encode insn bitcount: CLZ/CTZ/POPCNT r0, r1 + * @param kind the data kind, such as I32, I64 + * @param Type the data type, such as int32, int64 + * @param type the abbreviation of data type, such as i32, i64 + * @param op the opcode of bit operation + */ +#define BITCOUNT_R_R(kind, Type, type, op) \ + do { \ + int32 reg_no_dst = 0, reg_no_src = 0; \ + \ + CHECK_EQKIND(r0, r1); \ + CHECK_NCONST(r0); \ + CHECK_NCONST(r1); \ + \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + reg_no_src = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src, jit_reg_kind(r1)); \ + if (!bitcount_r_to_r_##type(a, op, reg_no_dst, reg_no_src)) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode bitcount insn, CLZ/CTZ/POPCNT r0, r1 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the opcode of bitcount operations + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the src operand info + * + * @return true if success, false if failed + */ +static bool +lower_bitcount(JitCompContext *cc, x86::Assembler &a, BITCOUNT_OP op, JitReg r0, + JitReg r1) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + BITCOUNT_R_R(I32, int32, i32, op); + break; + case JIT_REG_KIND_I64: + BITCOUNT_R_R(I64, int64, i64, op); + break; + default: + LOG_VERBOSE("Invalid reg type of bit: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn cmp: CMP r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + */ +#define CMP_R_R_R(kind, Type, type) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst = 0, reg_no_src1 = 0, reg_no_src2 = 0; \ + bool _ret = false; \ + \ + CHECK_KIND(r0, JIT_REG_KIND_I32); \ + CHECK_KIND(r1, JIT_REG_KIND_##kind); \ + CHECK_EQKIND(r1, r2); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + else { \ + reg_no_src1 = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src1, jit_reg_kind(r1)); \ + } \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + else { \ + reg_no_src2 = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_src2, jit_reg_kind(r2)); \ + } \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = cmp_imm_imm_to_r_##type(a, reg_no_dst, data1, data2); \ + else \ + _ret = \ + cmp_imm_r_to_r_##type(a, reg_no_dst, data1, reg_no_src2); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = cmp_r_imm_to_r_##type(a, reg_no_dst, reg_no_src1, data2); \ + else \ + _ret = \ + cmp_r_r_to_r_##type(a, reg_no_dst, reg_no_src1, reg_no_src2); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode cmp insn, CMP r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 condition jit register + * @param r2 src jit register that contains the first src operand info + * @param r3 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_cmp(JitCompContext *cc, x86::Assembler &a, JitReg r0, JitReg r1, + JitReg r2) +{ + switch (jit_reg_kind(r1)) { + case JIT_REG_KIND_I32: + CMP_R_R_R(I32, int32, i32); + cc->last_cmp_on_fp = false; + break; + case JIT_REG_KIND_I64: + CMP_R_R_R(I64, int64, i64); + cc->last_cmp_on_fp = false; + break; + case JIT_REG_KIND_F32: + CMP_R_R_R(F32, float32, f32); + cc->last_cmp_on_fp = true; + break; + case JIT_REG_KIND_F64: + CMP_R_R_R(F64, float64, f64); + cc->last_cmp_on_fp = true; + break; + default: + cc->last_cmp_on_fp = false; + LOG_VERBOSE("Invalid reg type of cmp: %d\n", jit_reg_kind(r1)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode detecting the cmp flags in reg, and jmp to the relative address + * according to the condition opcode + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the condition opcode to jmp + * @param offset the relative offset to jmp when the contidtion meeted + * + * @return return the next address of native code after encoded + */ +static bool +cmp_r_and_jmp_relative(JitCompContext *cc, x86::Assembler &a, COND_OP op, + int32 offset) +{ + Imm target(INT32_MAX); + char *stream; + bool fp_cmp = cc->last_cmp_on_fp; + + bh_assert(!fp_cmp || (fp_cmp && (op == GTS || op == GES))); + + switch (op) { + case EQ: + { + a.je(target); + break; + } + case NE: + { + a.jne(target); + break; + } + case GTS: + { + if (fp_cmp) { + a.ja(target); + } + else { + a.jg(target); + } + break; + } + case LES: + { + a.jng(target); + break; + } + case GES: + { + if (fp_cmp) { + a.jae(target); + } + else { + a.jnl(target); + } + break; + } + case LTS: + { + a.jl(target); + break; + } + case GTU: + { + a.ja(target); + break; + } + case LEU: + { + a.jna(target); + break; + } + case GEU: + { + a.jae(target); + break; + } + case LTU: + { + a.jb(target); + break; + } + default: + { + bh_assert(0); + break; + } + } + + JitErrorHandler *err_handler = (JitErrorHandler *)a.code()->errorHandler(); + + if (!err_handler->err) { + /* The offset written by asmjit is always 0, we patch it again */ + stream = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size() - 6; + *(int32 *)(stream + 2) = offset; + } + return true; +} + +/** + * Encode select insn, SELECT r0, r1, r2, r3 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +/* TODO: optimize with setcc */ +static bool +lower_select(JitCompContext *cc, x86::Assembler &a, COND_OP op, JitReg r0, + JitReg r1, JitReg r2, JitReg r3) +{ + JitErrorHandler err_handler; + Environment env(Arch::kX64); + CodeHolder code1, code2; + char *stream_mov1, *stream_mov2; + uint32 size_mov1, size_mov2; + + code1.init(env); + code1.setErrorHandler(&err_handler); + x86::Assembler a1(&code1); + + code2.init(env); + code2.setErrorHandler(&err_handler); + x86::Assembler a2(&code2); + + CHECK_NCONST(r0); + CHECK_NCONST(r1); + CHECK_KIND(r1, JIT_REG_KIND_I32); + + if (r0 == r3 && r0 != r2 && !cc->last_cmp_on_fp) { + JitReg r_tmp; + + /* For i32/i64, exchange r2 and r3 to make r0 equal to r2, + so as to decrease possible execution instructions. + For f32/f64 comparison, should not change the order as + the result of comparison with NaN may be different. */ + r_tmp = r2; + r2 = r3; + r3 = r_tmp; + op = not_cond(op); + } + + if (!lower_mov(cc, a1, r0, r2)) + GOTO_FAIL; + + if (!lower_mov(cc, a2, r0, r3)) + GOTO_FAIL; + + stream_mov1 = (char *)a1.code()->sectionById(0)->buffer().data(); + size_mov1 = a1.code()->sectionById(0)->buffer().size(); + stream_mov2 = (char *)a2.code()->sectionById(0)->buffer().data(); + size_mov2 = a2.code()->sectionById(0)->buffer().size(); + + if (r0 != r2) { + a.embedDataArray(TypeId::kInt8, stream_mov1, size_mov1); + } + + if (r3 && r0 != r3) { + if (!cmp_r_and_jmp_relative(cc, a, op, (int32)size_mov2)) + return false; + a.embedDataArray(TypeId::kInt8, stream_mov2, size_mov2); + } + + return true; +fail: + return false; +} + +/* jmp to dst label */ +#define JMP_TO_LABEL(label_dst, label_src) \ + do { \ + if (label_is_ahead(cc, label_dst, label_src)) { \ + JitErrorHandler *err_handler = \ + (JitErrorHandler *)a.code()->errorHandler(); \ + int32 _offset; \ + char *stream; \ + Imm imm(INT32_MAX); \ + a.jmp(imm); \ + if (!err_handler->err) { \ + /* The offset written by asmjit is always 0, we patch it \ + again, 6 is the size of jmp instruciton */ \ + stream = (char *)a.code()->sectionById(0)->buffer().data() \ + + a.code()->sectionById(0)->buffer().size() - 6; \ + _offset = label_offsets[label_dst] \ + - a.code()->sectionById(0)->buffer().size(); \ + *(int32 *)(stream + 2) = _offset; \ + } \ + } \ + else { \ + if (!jmp_from_label_to_label(a, jmp_info_list, label_dst, \ + label_src)) \ + GOTO_FAIL; \ + } \ + } while (0) + +/** + * Encode branch insn, BEQ/BNE/../BLTU r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param jmp_info_list the jmp info list + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lower_branch(JitCompContext *cc, x86::Assembler &a, bh_list *jmp_info_list, + int32 label_src, COND_OP op, JitReg r0, JitReg r1, JitReg r2, + bool is_last_insn) +{ + int32 label_dst; + + CHECK_NCONST(r0); + CHECK_KIND(r0, JIT_REG_KIND_I32); + CHECK_KIND(r1, JIT_REG_KIND_L32); + + CHECK_REG_NO(jit_reg_no(r0), jit_reg_kind(r0)); + + label_dst = jit_reg_no(r1); + if (label_dst < (int32)jit_cc_label_num(cc) - 1 && is_last_insn + && label_is_neighboring(cc, label_src, label_dst) + && !cc->last_cmp_on_fp) { + JitReg r_tmp; + + r_tmp = r1; + r1 = r2; + r2 = r_tmp; + op = not_cond(op); + } + + if (!cmp_r_and_jmp_label(cc, a, jmp_info_list, label_src, op, r1, r2, + is_last_insn)) + GOTO_FAIL; + + return true; +fail: + return false; +} + +/** + * Encode lookupswitch with key of immediate data + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param jmp_info_list the jmp info list + * @param label_offsets the offsets of each label + * @param label_src the index of src label + * @param key the entry key + * @param opnd the lookup switch operand + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lookupswitch_imm(JitCompContext *cc, x86::Assembler &a, bh_list *jmp_info_list, + uint32 *label_offsets, int32 label_src, int32 key, + const JitOpndLookupSwitch *opnd, bool is_last_insn) +{ + uint32 i; + int32 label_dst; + + for (i = 0; i < opnd->match_pairs_num; i++) + if (key == opnd->match_pairs[i].value) { + label_dst = jit_reg_no(opnd->match_pairs[i].target); + if (!(is_last_insn + && label_is_neighboring(cc, label_src, label_dst))) { + JMP_TO_LABEL(label_dst, label_src); + } + return true; + } + + if (opnd->default_target) { + label_dst = jit_reg_no(opnd->default_target); + if (!(is_last_insn && label_is_neighboring(cc, label_src, label_dst))) { + JMP_TO_LABEL(label_dst, label_src); + } + } + + return true; +fail: + return false; +} + +/** + * Encode detecting lookupswitch entry register and jumping to matched label + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param jmp_info_list the jmp info list + * @param label_offsets the offsets of each label + * @param label_src the index of src label + * @param reg_no the no of entry register + * @param opnd the lookup switch operand + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lookupswitch_r(JitCompContext *cc, x86::Assembler &a, bh_list *jmp_info_list, + uint32 *label_offsets, int32 label_src, int32 reg_no, + const JitOpndLookupSwitch *opnd, bool is_last_insn) +{ + JmpInfo *node; + Imm imm; + x86::Mem m; + uint32 i; + int32 label_dst = 0; + char *stream; + + if (opnd->match_pairs_num < 10) { + /* For small count of branches, it is better to compare + the key with branch value and jump one by one */ + for (i = 0; i < opnd->match_pairs_num; i++) { + imm.setValue(opnd->match_pairs[i].value); + a.cmp(regs_i32[reg_no], imm); + + node = (JmpInfo *)jit_malloc(sizeof(JmpInfo)); + if (!node) + GOTO_FAIL; + + node->type = JMP_DST_LABEL_REL; + node->label_src = label_src; + node->dst_info.label_dst = jit_reg_no(opnd->match_pairs[i].target); + node->offset = a.code()->sectionById(0)->buffer().size() + 2; + bh_list_insert(jmp_info_list, node); + + imm.setValue(INT32_MAX); + a.je(imm); + } + + if (opnd->default_target) { + label_dst = jit_reg_no(opnd->default_target); + if (!(is_last_insn + && label_is_neighboring(cc, label_src, label_dst))) + JMP_TO_LABEL(label_dst, label_src); + } + } + else { + /* For bigger count of branches, use indirect jump */ + /* unsigned extend to rsi */ + a.mov(regs_i32[REG_I32_FREE_IDX], regs_i32[reg_no]); + imm.setValue(opnd->match_pairs_num); + a.cmp(regs_i64[REG_I64_FREE_IDX], imm); + + /* Jump to default label if rsi >= br_count */ + stream = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); + imm.setValue(INT32_MAX); + a.jb(imm); + *(uint32 *)(stream + 2) = 6; + + node = (JmpInfo *)jit_calloc(sizeof(JmpInfo)); + if (!node) + goto fail; + + node->type = JMP_DST_LABEL_REL; + node->label_src = label_src; + node->dst_info.label_dst = jit_reg_no(opnd->default_target); + node->offset = a.code()->sectionById(0)->buffer().size() + 2; + bh_list_insert(jmp_info_list, node); + + imm.setValue(INT32_MAX); + a.jmp(imm); + + node = (JmpInfo *)jit_malloc(sizeof(JmpInfo)); + if (!node) + GOTO_FAIL; + + node->type = JMP_LOOKUPSWITCH_BASE; + node->offset = a.code()->sectionById(0)->buffer().size() + 2; + bh_list_insert(jmp_info_list, node); + + /* LookupSwitch table base addr */ + imm.setValue(INT64_MAX); + a.mov(regs_i64[reg_no], imm); + + /* jmp *(base_addr + rsi * 8) */ + m = x86::ptr(regs_i64[reg_no], regs_i64[REG_I64_FREE_IDX], 3); + a.jmp(m); + + /* Store each dst label absolute address */ + for (i = 0; i < opnd->match_pairs_num; i++) { + node = (JmpInfo *)jit_malloc(sizeof(JmpInfo)); + if (!node) + GOTO_FAIL; + + node->type = JMP_DST_LABEL_ABS; + node->dst_info.label_dst = jit_reg_no(opnd->match_pairs[i].target); + node->offset = a.code()->sectionById(0)->buffer().size(); + bh_list_insert(jmp_info_list, node); + + a.embedUInt64(UINT64_MAX); + } + } + + return true; +fail: + return false; +} + +/** + * Encode lookupswitch insn, LOOKUPSWITCH opnd + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param jmp_info_list the jmp info list + * @param label_offsets the offsets of each label + * @param label_src the index of src label + * @param opnd the lookup switch operand + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lower_lookupswitch(JitCompContext *cc, x86::Assembler &a, + bh_list *jmp_info_list, uint32 *label_offsets, + int32 label_src, const JitOpndLookupSwitch *opnd, + bool is_last_insn) +{ + JitReg r0 = opnd->value; + int32 key, reg_no; + + CHECK_KIND(r0, JIT_REG_KIND_I32); + CHECK_KIND(opnd->default_target, JIT_REG_KIND_L32); + + if (jit_reg_is_const(r0)) { + key = jit_cc_get_const_I32(cc, r0); + if (!lookupswitch_imm(cc, a, jmp_info_list, label_offsets, label_src, + key, opnd, is_last_insn)) + GOTO_FAIL; + } + else { + reg_no = jit_reg_no(r0); + CHECK_I32_REG_NO(reg_no); + if (!lookupswitch_r(cc, a, jmp_info_list, label_offsets, label_src, + reg_no, opnd, is_last_insn)) + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode callnative insn, CALLNATIVE r0, r1, ... + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param insn current insn info + * + * @return true if success, false if failed + */ +static bool +lower_callnative(JitCompContext *cc, x86::Assembler &a, JitInsn *insn) +{ + void (*func_ptr)(void); + JitReg ret_reg, func_reg, arg_reg; + /* the index of callee saved registers in regs_i64 */ + uint8 regs_arg_idx[] = { REG_RDI_IDX, REG_RSI_IDX, REG_RDX_IDX, + REG_RCX_IDX, REG_R8_IDX, REG_R9_IDX }; + Imm imm; + uint32 i, opnd_num; + int32 integer_reg_index = 0, floatpoint_reg_index = 0; + + ret_reg = *(jit_insn_opndv(insn, 0)); + func_reg = *(jit_insn_opndv(insn, 1)); + CHECK_KIND(func_reg, JIT_REG_KIND_I64); + CHECK_CONST(func_reg); + + func_ptr = (void (*)(void))jit_cc_get_const_I64(cc, func_reg); + + opnd_num = jit_insn_opndv_num(insn); + for (i = 0; i < opnd_num - 2; i++) { + /*TODO: if arguments number is greater than 6 */ + bh_assert(integer_reg_index < 6); + bh_assert(floatpoint_reg_index < 6); + + arg_reg = *(jit_insn_opndv(insn, i + 2)); + switch (jit_reg_kind(arg_reg)) { + case JIT_REG_KIND_I32: + { + int32 reg_no = regs_arg_idx[integer_reg_index++]; + CHECK_I64_REG_NO(reg_no); + if (jit_reg_is_const(arg_reg)) { + mov_imm_to_r_i64(a, reg_no, + (int64)jit_cc_get_const_I32(cc, arg_reg)); + } + else { + int32 arg_reg_no = jit_reg_no(arg_reg); + CHECK_I32_REG_NO(arg_reg_no); + extend_r32_to_r64(a, reg_no, arg_reg_no, true); + } + break; + } + case JIT_REG_KIND_I64: + { + int32 reg_no = regs_arg_idx[integer_reg_index++]; + CHECK_I64_REG_NO(reg_no); + if (jit_reg_is_const(arg_reg)) { + mov_imm_to_r_i64(a, reg_no, + jit_cc_get_const_I64(cc, arg_reg)); + } + else { + int32 arg_reg_no = jit_reg_no(arg_reg); + CHECK_I64_REG_NO(arg_reg_no); + mov_r_to_r_i64(a, reg_no, arg_reg_no); + } + break; + } + case JIT_REG_KIND_F32: + { + CHECK_F32_REG_NO((int32)floatpoint_reg_index); + if (jit_reg_is_const(arg_reg)) { + mov_imm_to_r_f32(a, floatpoint_reg_index, + jit_cc_get_const_F32(cc, arg_reg)); + } + else { + int32 arg_reg_no = jit_reg_no(arg_reg); + CHECK_F32_REG_NO(arg_reg_no); + mov_r_to_r_f32(a, floatpoint_reg_index, arg_reg_no); + } + floatpoint_reg_index++; + break; + } + case JIT_REG_KIND_F64: + { + CHECK_F64_REG_NO((int32)floatpoint_reg_index); + if (jit_reg_is_const(arg_reg)) { + mov_imm_to_r_f64(a, floatpoint_reg_index, + jit_cc_get_const_F64(cc, arg_reg)); + } + else { + int32 arg_reg_no = jit_reg_no(arg_reg); + CHECK_F64_REG_NO(arg_reg_no); + mov_r_to_r_f64(a, floatpoint_reg_index, arg_reg_no); + } + floatpoint_reg_index++; + break; + } + default: + { + + bh_assert(0); + goto fail; + } + } + } + + imm.setValue((uint64)func_ptr); + a.mov(regs_i64[REG_RAX_IDX], imm); + a.call(regs_i64[REG_RAX_IDX]); + + if (ret_reg) { + uint32 ret_reg_no = jit_reg_no(ret_reg); + if (jit_reg_kind(ret_reg) == JIT_REG_KIND_I64) { + CHECK_I64_REG_NO(ret_reg_no); + /* mov res, rax */ + mov_r_to_r_i64(a, ret_reg_no, REG_RAX_IDX); + } + else if (jit_reg_kind(ret_reg) == JIT_REG_KIND_F64) { + CHECK_F64_REG_NO(ret_reg_no); + /* mov res, xmm0_f64 */ + mov_r_to_r_f64(a, ret_reg_no, 0); + } + else { + bh_assert((jit_reg_kind(ret_reg) == JIT_REG_KIND_I32 + && ret_reg_no == REG_EAX_IDX) + || (jit_reg_kind(ret_reg) == JIT_REG_KIND_F32 + && ret_reg_no == 0)); + } + } + + return true; +fail: + return false; +} + +/** + * Encode callbc insn, CALLBC r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param jmp_info_list the jmp info list + * @param label_src the index of src label + * @param insn current insn info + * + * @return true if success, false if failed + */ +static bool +lower_callbc(JitCompContext *cc, x86::Assembler &a, bh_list *jmp_info_list, + int32 label_src, JitInsn *insn) +{ + JmpInfo *node; + Imm imm; + JitReg edx_hreg = jit_reg_new(JIT_REG_KIND_I32, REG_EDX_IDX); + JitReg rdx_hreg = jit_reg_new(JIT_REG_KIND_I64, REG_RDX_IDX); + JitReg xmm0_f32_hreg = jit_reg_new(JIT_REG_KIND_F32, 0); + JitReg xmm0_f64_hreg = jit_reg_new(JIT_REG_KIND_F64, 0); + JitReg ret_reg = *(jit_insn_opnd(insn, 0)); + JitReg func_reg = *(jit_insn_opnd(insn, 2)); + JitReg func_idx = *(jit_insn_opnd(insn, 3)); + JitReg src_reg; + int32 func_reg_no; + + /* Load return_jitted_addr from stack */ + x86::Mem m(x86::rbp, cc->jitted_return_address_offset); + + CHECK_KIND(func_reg, JIT_REG_KIND_I64); + func_reg_no = jit_reg_no(func_reg); + CHECK_I64_REG_NO(func_reg_no); + + CHECK_KIND(func_idx, JIT_REG_KIND_I32); + if (jit_reg_is_const(func_idx)) { + imm.setValue(jit_cc_get_const_I32(cc, func_idx)); + a.mov(regs_i64[REG_RDX_IDX], imm); + } + else { + a.movzx(regs_i64[REG_RDX_IDX], regs_i32[jit_reg_no(func_idx)]); + } + + node = (JmpInfo *)jit_malloc(sizeof(JmpInfo)); + if (!node) + GOTO_FAIL; + + node->type = JMP_END_OF_CALLBC; + node->label_src = label_src; + node->offset = a.code()->sectionById(0)->buffer().size() + 2; + bh_list_insert(jmp_info_list, node); + + /* Set next jited addr to glue_ret_jited_addr, 0 will be replaced with + actual offset after actual code cache is allocated */ + imm.setValue(INT64_MAX); + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.mov(m, regs_i64[REG_I64_FREE_IDX]); + a.jmp(regs_i64[func_reg_no]); + + if (ret_reg) { + switch (jit_reg_kind(ret_reg)) { + case JIT_REG_KIND_I32: + src_reg = edx_hreg; + break; + case JIT_REG_KIND_I64: + src_reg = rdx_hreg; + break; + case JIT_REG_KIND_F32: + src_reg = xmm0_f32_hreg; + break; + case JIT_REG_KIND_F64: + src_reg = xmm0_f64_hreg; + break; + default: + bh_assert(0); + return false; + } + + if (!lower_mov(cc, a, ret_reg, src_reg)) + return false; + } + return true; +fail: + return false; +} + +static bool +lower_returnbc(JitCompContext *cc, x86::Assembler &a, JitInsn *insn) +{ + JitReg edx_hreg = jit_reg_new(JIT_REG_KIND_I32, REG_EDX_IDX); + JitReg rdx_hreg = jit_reg_new(JIT_REG_KIND_I64, REG_RDX_IDX); + JitReg xmm0_f32_hreg = jit_reg_new(JIT_REG_KIND_F32, 0); + JitReg xmm0_f64_hreg = jit_reg_new(JIT_REG_KIND_F64, 0); + JitReg act_reg = *(jit_insn_opnd(insn, 0)); + JitReg ret_reg = *(jit_insn_opnd(insn, 1)); + JitReg dst_reg; + int32 act; + + CHECK_CONST(act_reg); + CHECK_KIND(act_reg, JIT_REG_KIND_I32); + + act = jit_cc_get_const_I32(cc, act_reg); + + if (ret_reg) { + switch (jit_reg_kind(ret_reg)) { + case JIT_REG_KIND_I32: + dst_reg = edx_hreg; + break; + case JIT_REG_KIND_I64: + dst_reg = rdx_hreg; + break; + case JIT_REG_KIND_F32: + dst_reg = xmm0_f32_hreg; + break; + case JIT_REG_KIND_F64: + dst_reg = xmm0_f64_hreg; + break; + default: + bh_assert(0); + return false; + } + if (!lower_mov(cc, a, dst_reg, ret_reg)) + return false; + } + + { + /* eax = act */ + Imm imm(act); + a.mov(x86::eax, imm); + + x86::Mem m(x86::rbp, cc->jitted_return_address_offset); + a.jmp(m); + } + return true; +fail: + return false; +} + +static bool +lower_return(JitCompContext *cc, x86::Assembler &a, JitInsn *insn) +{ + JitReg act_reg = *(jit_insn_opnd(insn, 0)); + int32 act; + + CHECK_CONST(act_reg); + CHECK_KIND(act_reg, JIT_REG_KIND_I32); + + act = jit_cc_get_const_I32(cc, act_reg); + { + /* eax = act */ + Imm imm(act); + a.mov(x86::eax, imm); + + imm.setValue((uintptr_t)code_block_return_to_interp_from_jitted); + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.jmp(regs_i64[REG_I64_FREE_IDX]); + } + return true; +fail: + return false; +} + +/** + * Replace all the jmp address pre-saved when the code cache hasn't been + * allocated with actual address after code cache allocated + * + * @param cc compiler context containting the allocated code cacha info + * @param jmp_info_list the jmp info list + */ +static void +patch_jmp_info_list(JitCompContext *cc, bh_list *jmp_info_list) +{ + JmpInfo *jmp_info, *jmp_info_next; + JitReg reg_dst; + char *stream; + + jmp_info = (JmpInfo *)bh_list_first_elem(jmp_info_list); + + while (jmp_info) { + jmp_info_next = (JmpInfo *)bh_list_elem_next(jmp_info); + + stream = (char *)cc->jitted_addr_begin + jmp_info->offset; + + if (jmp_info->type == JMP_DST_LABEL_REL) { + /* Jmp with relative address */ + reg_dst = + jit_reg_new(JIT_REG_KIND_L32, jmp_info->dst_info.label_dst); + *(int32 *)stream = + (int32)((uintptr_t)*jit_annl_jitted_addr(cc, reg_dst) + - (uintptr_t)stream) + - 4; + } + else if (jmp_info->type == JMP_DST_LABEL_ABS) { + /* Jmp with absolute address */ + reg_dst = + jit_reg_new(JIT_REG_KIND_L32, jmp_info->dst_info.label_dst); + *(uintptr_t *)stream = + (uintptr_t)*jit_annl_jitted_addr(cc, reg_dst); + } + else if (jmp_info->type == JMP_END_OF_CALLBC) { + /* 7 is the size of mov and jmp instruction */ + *(uintptr_t *)stream = (uintptr_t)stream + sizeof(uintptr_t) + 7; + } + else if (jmp_info->type == JMP_LOOKUPSWITCH_BASE) { + /* 11 is the size of 8-byte addr and 3-byte jmp instruction */ + *(uintptr_t *)stream = (uintptr_t)stream + 11; + } + + jmp_info = jmp_info_next; + } +} + +/* Free the jmp info list */ +static void +free_jmp_info_list(bh_list *jmp_info_list) +{ + void *cur_node = bh_list_first_elem(jmp_info_list); + + while (cur_node) { + void *next_node = bh_list_elem_next(cur_node); + + bh_list_remove(jmp_info_list, cur_node); + jit_free(cur_node); + cur_node = next_node; + } +} + +/** + * Encode cast int32 immediate data to float register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src immediate data + * + * @return true if success, false otherwise + */ +static bool +cast_imm_i32_to_r_f32(x86::Assembler &a, int32 reg_no, int32 data) +{ + Imm imm(data); + a.mov(regs_i32[REG_I32_FREE_IDX], imm); + a.movd(regs_float[reg_no], regs_i32[REG_I32_FREE_IDX]); + return true; +} + +/** + * Encode cast int32 register data to float register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst float register + * @param reg_no_src the no of src int32 register + * + * @return true if success, false otherwise + */ +static bool +cast_r_i32_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.movd(regs_float[reg_no_dst], regs_i32[reg_no_src]); + return true; +} + +/** + * Encode cast int64 immediate data to double register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src immediate data + * + * @return true if success, false otherwise + */ +static bool +cast_imm_i64_to_r_f64(x86::Assembler &a, int32 reg_no, int64 data) +{ + Imm imm(data); + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.movq(regs_float[reg_no], regs_i64[REG_I64_FREE_IDX]); + return true; +} + +/** + * Encode cast int64 register data to double register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src int64 register + * + * @return true if success, false otherwise + */ +static bool +cast_r_i64_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.movq(regs_float[reg_no_dst], regs_i64[reg_no_src]); + return true; +} + +/** + * Encode cast float immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate data + * + * @return true if success, false otherwise + */ +static bool +cast_imm_f32_to_r_i32(x86::Assembler &a, int32 reg_no, float data) +{ + cast_float_to_integer v = { .f = data }; + return mov_imm_to_r_i32(a, reg_no, v.i); +} + +/** + * Encode cast float register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +cast_r_f32_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.movd(regs_i32[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode cast double immediate data to int64 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int64 register + * @param data the src immediate data + * + * @return true if success, false otherwise + */ +static bool +cast_imm_f64_to_r_i64(x86::Assembler &a, int32 reg_no, double data) +{ + cast_double_to_integer v = { .d = data }; + return mov_imm_to_r_i64(a, reg_no, v.i); +} + +/** + * Encode cast float register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +cast_r_f64_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + a.movq(regs_i64[reg_no_dst], regs_float[reg_no_src]); + return true; +} + +/** + * Encode insn cast: F32CASTI32, + * @param kind0 the dst JIT_REG_KIND, such as I32, I64, F32 and F64 + * @param kind1 the src JIT_REG_KIND, such as I32, I64, F32 and F64 + * @param type0 the dst data type, such as i8, u8, i16, u16, i32, f32, i64, f32, + * f64 + * @param type1 the src data type, such as i8, u8, i16, u16, i32, f32, i64, f32, + * f64 + */ +#define CAST_R_R(kind0, kind1, type0, type1, Type1) \ + do { \ + bool _ret = false; \ + int32 reg_no_dst = 0, reg_no_src = 0; \ + CHECK_KIND(r0, JIT_REG_KIND_##kind0); \ + CHECK_KIND(r1, JIT_REG_KIND_##kind1); \ + \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, JIT_REG_KIND_##kind0); \ + if (jit_reg_is_const(r1)) { \ + Type1 data = jit_cc_get_const_##kind1(cc, r1); \ + _ret = cast_imm_##type1##_to_r_##type0(a, reg_no_dst, data); \ + } \ + else { \ + reg_no_src = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src, JIT_REG_KIND_##kind1); \ + _ret = cast_r_##type1##_to_r_##type0(a, reg_no_dst, reg_no_src); \ + } \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +#if WASM_ENABLE_SHARED_MEMORY != 0 + +/** + * Encode extend certain bytes in the src register to a I32 or I64 kind value in + * dst register + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * @param kind_dst the kind of data to extend to, could be I32, I64 + * @param reg_no_src the index of register hold src value + * + * @return true if success, false otherwise + */ +static bool +extend_r_to_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_src, int32 reg_no_dst) +{ + if (kind_dst == JIT_REG_KIND_I32) { + bh_assert(reg_no_src < 16 && reg_no_dst < 16); + switch (bytes_dst) { + case 1: + extend_r8_to_r32(a, reg_no_dst, reg_no_src, false); + break; + case 2: + extend_r16_to_r32(a, reg_no_dst, reg_no_src, false); + break; + case 4: + mov_r_to_r_i32(a, reg_no_dst, reg_no_src); + break; + default: + bh_assert(0); + return false; + } + } + else if (kind_dst == JIT_REG_KIND_I64) { + bh_assert(reg_no_src < 16 && reg_no_dst < 16); + switch (bytes_dst) { + case 1: + extend_r8_to_r64(a, reg_no_dst, reg_no_src, false); + break; + case 2: + extend_r16_to_r64(a, reg_no_dst, reg_no_src, false); + break; + case 4: + extend_r32_to_r64(a, reg_no_dst, reg_no_src, false); + break; + case 8: + mov_r_to_r_i64(a, reg_no_dst, reg_no_src); + break; + default: + bh_assert(0); + return false; + } + } + else { + bh_assert(0); + } + return true; +} + +/** + * Encode atomic compare and exchange, when calling this function, + * value for comparison should be already moved in register + * al/ax/eax/rax + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * @param kind_dst the kind of data to move, could be I32, I64 + * @param m_dst the dest memory operand + * @param reg_no_xchg the index of register hold exchange value + * + * @return true if success, false otherwise + */ +static bool +at_cmpxchg(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_xchg, x86::Mem &m_dst) +{ + bh_assert((kind_dst == JIT_REG_KIND_I32 && bytes_dst <= 4) + || kind_dst == JIT_REG_KIND_I64); + bh_assert(reg_no_xchg < 16); + switch (bytes_dst) { + case 1: + a.lock().cmpxchg(m_dst, regs_i8[reg_no_xchg]); + break; + case 2: + a.lock().cmpxchg(m_dst, regs_i16[reg_no_xchg]); + break; + case 4: + a.lock().cmpxchg(m_dst, regs_i32[reg_no_xchg]); + break; + case 8: + a.lock().cmpxchg(m_dst, regs_i64[reg_no_xchg]); + break; + default: + bh_assert(0); + return false; + } + return true; +} + +/** + * Encode atomic compare and exchange: load value into a register from + * memory with reg base and reg offset, compare (expected) reg data with the + * loaded value, if equal, store the (replacement) reg data to the same + * memory, else, do nothing. Either way, returns the loaded value + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_xchg the no of register that stores the conditionally + * replacement value + * @param reg_no_base the no of register that stores the base address + * of src&dst memory + * @param reg_no_offset the no of register that stores the offset address + * of src&dst memory + * @return true if success, false otherwise + */ +static bool +at_cmpxchg_r_ra_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_xchg, + int32 reg_no_base, int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return at_cmpxchg(a, bytes_dst, kind_dst, reg_no_xchg, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, REG_RAX_IDX); +} + +/** + * Encode atomic compare and exchange: load value into a register from + * memory with reg base and imm offset, compare (expected) reg data with the + * loaded value, if equal, store the (replacement) reg data to the same + * memory, else, do nothing. Either way, returns the loaded value + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_xchg the no of register that stores the conditionally + * replacement value + * @param reg_no_base the no of register that stores the base address + * of src&dst memory + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_cmpxchg_r_ra_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_xchg, + int32 reg_no_base, int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return at_cmpxchg(a, bytes_dst, kind_dst, reg_no_xchg, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, REG_RAX_IDX); +} + +/** + * Encode atomic compare and exchange: load value into a register from + * memory with reg base and reg offset, compare (expected) reg data with the + * loaded value, if equal, store the (replacement) imm data to the same + * memory, else, do nothing. Either way, returns the loaded value + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_xchg the immediate data for exchange(conditionally replacment + * value) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory + * @param reg_no_offset the no of register that stores the offset address + * of src&dst memory + * @return true if success, false otherwise + */ +static bool +at_cmpxchg_imm_ra_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, void *data_xchg, + int32 reg_no_base, int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_xchg, bytes_dst); + uint32 reg_no_xchg = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_cmpxchg(a, bytes_dst, kind_dst, reg_no_xchg, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, REG_RAX_IDX); +} + +/** + * Encode atomic compare and exchange: load value into a register from + * memory with reg base and imm offset, compare (expected) reg data with the + * loaded value, if equal, store the (replacement) imm data to the same + * memory, else, do nothing. Either way, returns the loaded value + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_xchg the immediate data for exchange(conditionally replacment + * value) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_cmpxchg_imm_ra_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, void *data_xchg, + int32 reg_no_base, int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_xchg, bytes_dst); + uint32 reg_no_xchg = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_cmpxchg(a, bytes_dst, kind_dst, reg_no_xchg, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, REG_RAX_IDX); +} + +/** + * Encode insn cmpxchg: CMPXCHG_type r0, r1, r2, r3, r4 + * @param kind the data kind, can only be I32 or I64 + * @param bytes_dst the byte number of dst data + */ +#define CMPXCHG_R_R_R_R_R(kind, type, bytes_dst) \ + do { \ + type data_xchg = 0; \ + int32 reg_no_xchg = 0, reg_no_cmp = 0, reg_no_base = 0, \ + reg_no_offset = 0; \ + int32 offset = 0; \ + bool _ret = false; \ + if (jit_reg_is_const(r3)) { \ + CHECK_KIND(r3, JIT_REG_KIND_I32); \ + } \ + else { \ + CHECK_KIND(r3, JIT_REG_KIND_I64); \ + } \ + /* r1: expected value(it must in register a) \ + * r2: memory base addr can't be const */ \ + CHECK_NCONST(r1); \ + reg_no_cmp = jit_reg_no(r1); \ + bh_assert(reg_no_cmp == REG_EAX_IDX || reg_no_cmp == REG_RAX_IDX); \ + CHECK_REG_NO(reg_no_cmp, jit_reg_kind(r1)); \ + CHECK_NCONST(r2); \ + reg_no_base = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_base, jit_reg_kind(r2)); \ + /* r0: replacement value r3: offset can be const */ \ + if (jit_reg_is_const(r0)) \ + data_xchg = jit_cc_get_const_##kind(cc, r0); \ + else { \ + reg_no_xchg = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_xchg, jit_reg_kind(r0)); \ + } \ + if (jit_reg_is_const(r3)) \ + offset = jit_cc_get_const_I32(cc, r3); \ + else { \ + reg_no_offset = jit_reg_no(r3); \ + CHECK_REG_NO(reg_no_offset, jit_reg_kind(r3)); \ + } \ + \ + if (jit_reg_is_const(r0)) { \ + if (jit_reg_is_const(r3)) \ + _ret = at_cmpxchg_imm_ra_base_r_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, &data_xchg, \ + reg_no_base, offset); \ + else \ + _ret = at_cmpxchg_imm_ra_base_r_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, &data_xchg, \ + reg_no_base, reg_no_offset); \ + } \ + else { \ + if (jit_reg_is_const(r3)) \ + _ret = at_cmpxchg_r_ra_base_r_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_xchg, \ + reg_no_base, offset); \ + else \ + _ret = at_cmpxchg_r_ra_base_r_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_xchg, \ + reg_no_base, reg_no_offset); \ + } \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode negate a value in the register + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * @param kind_dst the kind of data to move, could be I32, I64 + * @param reg_no_src the index of register hold src value + * + * @return true if success, false otherwise + */ +static bool +neg_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, int32 reg_no_src) +{ + bh_assert((kind_dst == JIT_REG_KIND_I32 && bytes_dst <= 4) + || kind_dst == JIT_REG_KIND_I64); + bh_assert(reg_no_src < 16); + switch (bytes_dst) { + case 1: + a.neg(regs_i8[reg_no_src]); + break; + case 2: + a.neg(regs_i16[reg_no_src]); + break; + case 4: + a.neg(regs_i32[reg_no_src]); + break; + case 8: + a.neg(regs_i64[reg_no_src]); + break; + default: + bh_assert(0); + return false; + } + return true; +} + +/** + * Encode atomic exchange and add + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * @param kind_dst the kind of data to move, could be I32, I64 + * @param reg_no_src the index of register hold operand value of add operation + * @param m_dst the dest memory operand + * + * @return true if success, false otherwise + */ +static bool +at_xadd(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, int32 reg_no_src, + x86::Mem &m_dst) +{ + bh_assert((kind_dst == JIT_REG_KIND_I32 && bytes_dst <= 4) + || kind_dst == JIT_REG_KIND_I64); + bh_assert(reg_no_src < 16); + switch (bytes_dst) { + case 1: + a.lock().xadd(m_dst, regs_i8[reg_no_src]); + break; + case 2: + a.lock().xadd(m_dst, regs_i16[reg_no_src]); + break; + case 4: + a.lock().xadd(m_dst, regs_i32[reg_no_src]); + break; + case 8: + a.lock().xadd(m_dst, regs_i64[reg_no_src]); + break; + default: + bh_assert(0); + return false; + } + + return true; +} + +/** + * Encode atomic rmw add: load value into a register from memory + * with reg base and reg offset, add loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(first operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(second operand&store back) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_add_imm_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw add: load value into a register from memory + * with reg base and reg offset, add loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_add_imm_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw add: load value into a register from memory + * with reg base and imm offset, add loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_add_r_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw add: load value into a register from memory + * with reg base and reg offset, add loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_add_r_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw sub: load value into a register from memory + * with reg base and reg offset, sub loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(first operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(second operand&store back) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_sub_imm_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return neg_r(a, bytes_dst, kind_dst, reg_no_src) + && at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw sub: load value into a register from memory + * with reg base and reg offset, sub loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_sub_imm_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return neg_r(a, bytes_dst, kind_dst, reg_no_src) + && at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw sub: load value into a register from memory + * with reg base and imm offset, sub loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_sub_r_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return neg_r(a, bytes_dst, kind_dst, reg_no_src) + && at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw sub: load value into a register from memory + * with reg base and reg offset, sub loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_sub_r_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return neg_r(a, bytes_dst, kind_dst, reg_no_src) + && at_xadd(a, bytes_dst, kind_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw xchg: load value into a register from memory + * with reg base and reg offset, exchange loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(first operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(second operand&store back) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xchg_imm_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw xchg: load value into a register from memory + * with reg base and reg offset, exchange loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xchg_imm_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw xchg: load value into a register from memory + * with reg base and imm offset, exchange loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xchg_r_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode atomic rmw xchg: load value into a register from memory + * with reg base and reg offset, exchange loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xchg_r_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return xchg_r_to_m(a, bytes_dst, kind_dst, m, reg_no_src) + && extend_r_to_r(a, bytes_dst, kind_dst, reg_no_src, reg_no_dst); +} + +/** + * Encode insn rmw logical operation: generate a loop to make sure it's atomic + * @param bin_op the operation, can be and/or/xor + * @param kind the data kind, can only be I32 or I64 + * @param bytes_dst the byte number of dst data + */ +#define AT_RMW_LOGICAL_LOOP(bin_op, kind, bytes_dst) \ + do { \ + bh_assert((kind_dst == JIT_REG_KIND_I32 && bytes_dst <= 4) \ + || kind_dst == JIT_REG_KIND_I64); \ + bh_assert(reg_no_src < 16 && reg_no_dst < 16); \ + /* read original value in memory(operand 1) to rax(expected) */ \ + mov_m_to_r(a, bytes_dst, kind_dst, false, REG_RAX_IDX, m_dst); \ + Label loop = a.newLabel(); \ + /* check whether loop is valid, and bind the loop label \ + * to the current position in the code. */ \ + if (!loop.isValid() || a.bind(loop) != kErrorOk) \ + return false; \ + /* move operand 1 to temp reg rb */ \ + mov_r_to_r(a, kind_dst, REG_RBX_IDX, REG_RAX_IDX); \ + /* actual logical operation with operand 2, result save to rbx */ \ + switch (bytes_dst) { \ + case 1: \ + a.bin_op##_(regs_i8[REG_RBX_IDX], regs_i8[reg_no_src]); \ + break; \ + case 2: \ + a.bin_op##_(regs_i16[REG_RBX_IDX], regs_i16[reg_no_src]); \ + break; \ + case 4: \ + a.bin_op##_(regs_i32[REG_RBX_IDX], regs_i32[reg_no_src]); \ + break; \ + case 8: \ + a.bin_op##_(regs_i64[REG_RBX_IDX], regs_i64[reg_no_src]); \ + break; \ + default: \ + bh_assert(0); \ + return false; \ + } \ + /* cmp with read value in RAX, try to change with result value in RBX \ + * REG, if change successfully, mem data is changed and exit loop(ZF \ + * is set) if not, loop again(ZF is clear) and tries to do logical ops \ + * atomically */ \ + at_cmpxchg(a, bytes_dst, kind_dst, REG_RBX_IDX, m_dst); \ + a.jne(loop); \ + return true; \ + } while (0) + +/** + * Encode atomic logical binary operation: and + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * @param kind_dst the kind of data to move, could be I32, I64 + * @param reg_no_dst the index of dest register + * @param reg_no_src the index of register hold operand value of add operation + * @param m_dst the dest memory operand + * + * @return true if success, false otherwise + */ +static bool +at_and(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, x86::Mem &m_dst) +{ + AT_RMW_LOGICAL_LOOP(and, kind_dst, bytes_dst); +} + +/** + * Encode atomic logical binary operation: or + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * @param kind_dst the kind of data to move, could be I32, I64 + * @param reg_no_dst the index of dest register + * @param reg_no_src the index of register hold operand value of add operation + * @param m_dst the dest memory operand + * + * @return true if success, false otherwise + */ +static bool +at_or(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, x86::Mem &m_dst) +{ + AT_RMW_LOGICAL_LOOP(or, kind_dst, bytes_dst); +} +/** + * Encode atomic logical binary operation: xor + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), + * @param kind_dst the kind of data to move, could be I32, I64 + * @param reg_no_dst the index of dest register + * @param reg_no_src the index of register hold operand value of add operation + * @param m_dst the dest memory operand + * + * @return true if success, false otherwise + */ +static bool +at_xor(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, x86::Mem &m_dst) +{ + AT_RMW_LOGICAL_LOOP(xor, kind_dst, bytes_dst); +} + +/** + * Encode atomic rmw and: load value into a register from memory with reg base + * and reg offset, bitwise and loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(first operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(second operand&store back) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_and_imm_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_and(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw and: load value into a register from memory with reg base + * and reg offset, bitwise and loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_and_imm_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_and(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw and: load value into a register from memory with reg base + * and imm offset, bitwise and value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_and_r_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return at_and(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw and: load value into a register from memory with reg base + * and reg offset, bitwise and loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_and_r_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return at_and(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw or: load value into a register from memory with reg base + * and reg offset, bitwise or loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(first operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(second operand&store back) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_or_imm_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_or(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw or: load value into a register from memory with reg base + * and reg offset, bitwise or loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_or_imm_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, void *data_src, + int32 reg_no_base, int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_or(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw or: load value into a register from memory with reg base + * and imm offset, bitwise or loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_or_r_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return at_or(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw or: load value into a register from memory with reg base + * and reg offset, bitwise or loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_or_r_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, int32 reg_no_src, + int32 reg_no_base, int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return at_or(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw xor: load value into a register from memory with reg base + * and reg offset, bitwise xor loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(first operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(second operand&store back) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xor_imm_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_xor(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw xor: load value into a register from memory with reg base + * and reg offset, bitwise xor loaded value with imm data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param data_src the immediate data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xor_imm_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + void *data_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + Imm imm; + imm_set_value(imm, data_src, bytes_dst); + uint32 reg_no_src = mov_imm_to_free_reg(a, imm, bytes_dst); + return at_xor(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw xor: load value into a register from memory with reg base + * and imm offset, bitwise xor exchange loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back location) + * @param offset the offset address of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xor_r_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 offset) +{ + x86::Mem m(regs_i64[reg_no_base], offset, bytes_dst); + return at_xor(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode atomic rmw xor: load value into a register from memory with reg base + * and reg offset, bitwise xor loaded value with reg data, store back + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data to actual operated on(load, + * compare, replacement) could be 1(byte), 2(short), 4(int32), 8(int64) + * @param reg_no_dst the no of register that stores the returned value + * @param reg_no_src the no of register store the src data(second operand) + * @param reg_no_base the no of register that stores the base address + * of src&dst memory(first operand&store back) + * @param reg_no_offset the no of register that stores the offset of the memory + * @return true if success, false otherwise + */ +static bool +at_rmw_xor_r_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_dst, + int32 reg_no_src, int32 reg_no_base, + int32 reg_no_offset) +{ + x86::Mem m(regs_i64[reg_no_base], regs_i64[reg_no_offset], 0, 0, bytes_dst); + return at_xor(a, bytes_dst, kind_dst, reg_no_dst, reg_no_src, m) + && extend_r_to_r(a, bytes_dst, kind_dst, REG_RAX_IDX, reg_no_dst); +} + +/** + * Encode insn rmw RMW_type r0, r1, r2, r3 + * @param bin_op the operation, can be add/sub/xchg/and/or/xor + * @param kind the data kind, can only be I32 or I64 + * @param bytes_dst the byte number of dst data + */ +#define AT_RMW_R_R_R_R(bin_op, kind, type, bytes_dst) \ + do { \ + type data_src = 0; \ + int32 reg_no_dst = 0, reg_no_src = 0, reg_no_base = 0, \ + reg_no_offset = 0; \ + int32 offset = 0; \ + bool _ret = false; \ + if (jit_reg_is_const(r3)) { \ + CHECK_KIND(r3, JIT_REG_KIND_I32); \ + } \ + else { \ + CHECK_KIND(r3, JIT_REG_KIND_I64); \ + } \ + /* r0: read/return value r2: memory base addr can't be const */ \ + /* already check it's not const in LOAD_4ARGS() */ \ + reg_no_dst = jit_reg_no(r0); \ + CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ + /* mem_data base address has to be non-const */ \ + CHECK_NCONST(r2); \ + reg_no_base = jit_reg_no(r2); \ + CHECK_REG_NO(reg_no_base, jit_reg_kind(r2)); \ + /* r1: source operand value r3: offset can be const */ \ + if (jit_reg_is_const(r1)) \ + data_src = jit_cc_get_const_##kind(cc, r1); \ + else { \ + reg_no_src = jit_reg_no(r1); \ + CHECK_REG_NO(reg_no_src, jit_reg_kind(r1)); \ + } \ + if (jit_reg_is_const(r3)) \ + offset = jit_cc_get_const_I32(cc, r3); \ + else { \ + reg_no_offset = jit_reg_no(r3); \ + CHECK_REG_NO(reg_no_offset, jit_reg_kind(r3)); \ + } \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r3)) \ + _ret = at_rmw_##bin_op##_imm_base_r_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_dst, &data_src, \ + reg_no_base, offset); \ + else \ + _ret = at_rmw_##bin_op##_imm_base_r_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_dst, &data_src, \ + reg_no_base, reg_no_offset); \ + } \ + else { \ + if (jit_reg_is_const(r3)) \ + _ret = at_rmw_##bin_op##_r_base_r_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_dst, reg_no_src, \ + reg_no_base, offset); \ + else \ + _ret = at_rmw_##bin_op##_r_base_r_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_dst, reg_no_src, \ + reg_no_base, reg_no_offset); \ + } \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode insn mfence + **/ +static void +fence(x86::Assembler &a) +{ + a.mfence(); +} + +/** + * Encode insn fence + */ +#define FENCE() fence(a) + +#endif + +bool +jit_codegen_gen_native(JitCompContext *cc) +{ + bool atomic; + JitBasicBlock *block; + JitInsn *insn; + JitReg r0, r1, r2, r3, r4; + JmpInfo jmp_info_head; + bh_list *jmp_info_list = (bh_list *)&jmp_info_head; + uint32 label_index, label_num, i; + uint32 *label_offsets = NULL, code_size; +#if CODEGEN_DUMP != 0 + uint32 code_offset = 0; +#endif + bool return_value = false, is_last_insn; + void **jitted_addr; + char *code_buf, *stream; + + JitErrorHandler err_handler; + Environment env(Arch::kX64); + CodeHolder code; + code.init(env); + code.setErrorHandler(&err_handler); + x86::Assembler a(&code); + + if (BH_LIST_SUCCESS != bh_list_init(jmp_info_list)) { + jit_set_last_error(cc, "init jmp info list failed"); + return false; + } + + label_num = jit_cc_label_num(cc); + + if (!(label_offsets = + (uint32 *)jit_calloc(((uint32)sizeof(uint32)) * label_num))) { + jit_set_last_error(cc, "allocate memory failed"); + goto fail; + } + + for (i = 0; i < label_num; i++) { + if (i == 0) + label_index = 0; + else if (i == label_num - 1) + label_index = 1; + else + label_index = i + 1; + + label_offsets[label_index] = code.sectionById(0)->buffer().size(); + + block = *jit_annl_basic_block( + cc, jit_reg_new(JIT_REG_KIND_L32, label_index)); + +#if CODEGEN_DUMP != 0 + os_printf("\nL%d:\n\n", label_index); +#endif + + JIT_FOREACH_INSN(block, insn) + { + is_last_insn = (insn->next == block) ? true : false; + +#if CODEGEN_DUMP != 0 + os_printf("\n"); + jit_dump_insn(cc, insn); +#endif + switch (insn->opcode) { + case JIT_OP_MOV: + LOAD_2ARGS(); + if (!lower_mov(cc, a, r0, r1)) + GOTO_FAIL; + break; + + case JIT_OP_I8TOI32: + LOAD_2ARGS(); + CONVERT_R_R(I32, I32, i32, i8, int8); + break; + + case JIT_OP_I8TOI64: + LOAD_2ARGS(); + CONVERT_R_R(I64, I32, i64, i8, int8); + break; + + case JIT_OP_I16TOI32: + LOAD_2ARGS(); + CONVERT_R_R(I32, I32, i32, i16, int16); + break; + + case JIT_OP_I16TOI64: + LOAD_2ARGS(); + CONVERT_R_R(I64, I32, i64, i16, int16); + break; + + case JIT_OP_I32TOI8: + LOAD_2ARGS(); + CONVERT_R_R(I32, I32, i8, i32, int32); + break; + + case JIT_OP_I32TOU8: + LOAD_2ARGS(); + CONVERT_R_R(I32, I32, u8, i32, int32); + break; + + case JIT_OP_I32TOI16: + LOAD_2ARGS(); + CONVERT_R_R(I32, I32, i16, i32, int32); + break; + + case JIT_OP_I32TOU16: + LOAD_2ARGS(); + CONVERT_R_R(I32, I32, u16, i32, int32); + break; + + case JIT_OP_I32TOI64: + LOAD_2ARGS(); + CONVERT_R_R(I64, I32, i64, i32, int32); + break; + + case JIT_OP_U32TOI64: + LOAD_2ARGS(); + CONVERT_R_R(I64, I32, i64, u32, int32); + break; + + case JIT_OP_I32TOF32: + LOAD_2ARGS(); + CONVERT_R_R(F32, I32, f32, i32, int32); + break; + + case JIT_OP_U32TOF32: + LOAD_2ARGS(); + CONVERT_R_R(F32, I32, f32, u32, uint32); + break; + + case JIT_OP_I32TOF64: + LOAD_2ARGS(); + CONVERT_R_R(F64, I32, f64, i32, int32); + break; + + case JIT_OP_U32TOF64: + LOAD_2ARGS(); + CONVERT_R_R(F64, I32, f64, u32, uint32); + break; + + case JIT_OP_I64TOI8: + LOAD_2ARGS(); + CONVERT_R_R(I32, I64, i8, i64, int64); + break; + + case JIT_OP_I64TOI16: + LOAD_2ARGS(); + CONVERT_R_R(I32, I64, i16, i64, int64); + break; + + case JIT_OP_I64TOI32: + LOAD_2ARGS(); + CONVERT_R_R(I32, I64, i32, i64, int64); + break; + + case JIT_OP_I64TOF32: + LOAD_2ARGS(); + CONVERT_R_R(F32, I64, f32, i64, int64); + break; + + case JIT_OP_I64TOF64: + LOAD_2ARGS(); + CONVERT_R_R(F64, I64, f64, i64, int64); + break; + + case JIT_OP_F32TOI32: + LOAD_2ARGS(); + CONVERT_R_R(I32, F32, i32, f32, float32); + break; + + case JIT_OP_F32TOI64: + LOAD_2ARGS(); + CONVERT_R_R(I64, F32, i64, f32, float32); + break; + + case JIT_OP_F32TOF64: + LOAD_2ARGS(); + CONVERT_R_R(F64, F32, f64, f32, float32); + break; + + case JIT_OP_F32TOU32: + LOAD_2ARGS(); + CONVERT_R_R(I32, F32, u32, f32, float32); + break; + + case JIT_OP_F64TOI32: + LOAD_2ARGS(); + CONVERT_R_R(I32, F64, i32, f64, float64); + break; + + case JIT_OP_F64TOI64: + LOAD_2ARGS(); + CONVERT_R_R(I64, F64, i64, f64, float64); + break; + + case JIT_OP_F64TOF32: + LOAD_2ARGS(); + CONVERT_R_R(F32, F64, f32, f64, float64); + break; + + case JIT_OP_F64TOU32: + LOAD_2ARGS(); + CONVERT_R_R(I32, F64, u32, f64, float64); + break; + + case JIT_OP_NEG: + LOAD_2ARGS(); + if (!lower_neg(cc, a, r0, r1)) + GOTO_FAIL; + break; + + case JIT_OP_ADD: + case JIT_OP_SUB: + case JIT_OP_MUL: + case JIT_OP_DIV_S: + case JIT_OP_REM_S: + case JIT_OP_DIV_U: + case JIT_OP_REM_U: + LOAD_3ARGS(); + if (!lower_alu(cc, a, + (ALU_OP)(ADD + (insn->opcode - JIT_OP_ADD)), + r0, r1, r2)) + GOTO_FAIL; + break; + + case JIT_OP_SHL: + case JIT_OP_SHRS: + case JIT_OP_SHRU: + case JIT_OP_ROTL: + case JIT_OP_ROTR: + LOAD_3ARGS(); + if (!lower_shift( + cc, a, + (SHIFT_OP)(SHL + (insn->opcode - JIT_OP_SHL)), r0, + r1, r2)) + GOTO_FAIL; + break; + + case JIT_OP_OR: + case JIT_OP_XOR: + case JIT_OP_AND: + LOAD_3ARGS(); + if (!lower_bit(cc, a, + (BIT_OP)(OR + (insn->opcode - JIT_OP_OR)), + r0, r1, r2)) + GOTO_FAIL; + break; + + case JIT_OP_CLZ: + case JIT_OP_CTZ: + case JIT_OP_POPCNT: + LOAD_2ARGS(); + if (!lower_bitcount( + cc, a, + (BITCOUNT_OP)(CLZ + (insn->opcode - JIT_OP_CLZ)), + r0, r1)) + GOTO_FAIL; + break; + + case JIT_OP_CMP: + LOAD_3ARGS(); + if (!lower_cmp(cc, a, r0, r1, r2)) + GOTO_FAIL; + break; + + case JIT_OP_SELECTEQ: + case JIT_OP_SELECTNE: + case JIT_OP_SELECTGTS: + case JIT_OP_SELECTGES: + case JIT_OP_SELECTLTS: + case JIT_OP_SELECTLES: + case JIT_OP_SELECTGTU: + case JIT_OP_SELECTGEU: + case JIT_OP_SELECTLTU: + case JIT_OP_SELECTLEU: + LOAD_4ARGS(); + if (!lower_select( + cc, a, + (COND_OP)(EQ + (insn->opcode - JIT_OP_SELECTEQ)), + r0, r1, r2, r3)) + GOTO_FAIL; + break; + + case JIT_OP_LDEXECENV: + LOAD_1ARG(); + CHECK_KIND(r0, JIT_REG_KIND_I32); + /* TODO */ + break; + + case JIT_OP_LDJITINFO: + LOAD_1ARG(); + CHECK_KIND(r0, JIT_REG_KIND_I32); + /* TODO */ + break; + + case JIT_OP_LDI8: + LOAD_3ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + LD_R_R_R(I32, 1, true); + else + LD_R_R_R(I64, 1, true); + break; + + case JIT_OP_LDU8: + LOAD_3ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + LD_R_R_R(I32, 1, false); + else + LD_R_R_R(I64, 1, false); + break; + + case JIT_OP_LDI16: + LOAD_3ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + LD_R_R_R(I32, 2, true); + else + LD_R_R_R(I64, 2, true); + break; + + case JIT_OP_LDU16: + LOAD_3ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + LD_R_R_R(I32, 2, false); + else + LD_R_R_R(I64, 2, false); + break; + + case JIT_OP_LDI32: + LOAD_3ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + LD_R_R_R(I32, 4, true); + else + LD_R_R_R(I64, 4, true); + break; + + case JIT_OP_LDU32: + LOAD_3ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + LD_R_R_R(I32, 4, false); + else + LD_R_R_R(I64, 4, false); + break; + + case JIT_OP_LDI64: + case JIT_OP_LDU64: + case JIT_OP_LDPTR: + LOAD_3ARGS(); + LD_R_R_R(I64, 8, false); + break; + + case JIT_OP_LDF32: + LOAD_3ARGS(); + LD_R_R_R(F32, 4, false); + break; + + case JIT_OP_LDF64: + LOAD_3ARGS(); + LD_R_R_R(F64, 8, false); + break; + + case JIT_OP_STI8: + LOAD_3ARGS_NO_ASSIGN(); + atomic = insn->flags_u8 & 0x1; + ST_R_R_R(I32, int32, 1, atomic); + break; + + case JIT_OP_STI16: + LOAD_3ARGS_NO_ASSIGN(); + atomic = insn->flags_u8 & 0x1; + ST_R_R_R(I32, int32, 2, atomic); + break; + + case JIT_OP_STI32: + LOAD_3ARGS_NO_ASSIGN(); + atomic = insn->flags_u8 & 0x1; + ST_R_R_R(I32, int32, 4, atomic); + break; + + case JIT_OP_STI64: + LOAD_3ARGS_NO_ASSIGN(); + atomic = insn->flags_u8 & 0x1; + ST_R_R_R(I64, int64, 8, atomic); + break; + + case JIT_OP_STPTR: + LOAD_3ARGS_NO_ASSIGN(); + ST_R_R_R(I64, int64, 8, false); + break; + + case JIT_OP_STF32: + LOAD_3ARGS_NO_ASSIGN(); + ST_R_R_R(F32, float32, 4, false); + break; + + case JIT_OP_STF64: + LOAD_3ARGS_NO_ASSIGN(); + ST_R_R_R(F64, float64, 8, false); + break; + + case JIT_OP_JMP: + LOAD_1ARG(); + CHECK_KIND(r0, JIT_REG_KIND_L32); + if (!(is_last_insn + && label_is_neighboring(cc, label_index, + jit_reg_no(r0)))) + JMP_TO_LABEL(jit_reg_no(r0), label_index); + break; + + case JIT_OP_BEQ: + case JIT_OP_BNE: + case JIT_OP_BGTS: + case JIT_OP_BGES: + case JIT_OP_BLTS: + case JIT_OP_BLES: + case JIT_OP_BGTU: + case JIT_OP_BGEU: + case JIT_OP_BLTU: + case JIT_OP_BLEU: + LOAD_3ARGS(); + if (!lower_branch( + cc, a, jmp_info_list, label_index, + (COND_OP)(EQ + (insn->opcode - JIT_OP_BEQ)), r0, r1, + r2, is_last_insn)) + GOTO_FAIL; + break; + + case JIT_OP_LOOKUPSWITCH: + { + JitOpndLookupSwitch *opnd = jit_insn_opndls(insn); + if (!lower_lookupswitch(cc, a, jmp_info_list, label_offsets, + label_index, opnd, is_last_insn)) + GOTO_FAIL; + break; + } + + case JIT_OP_CALLNATIVE: + if (!lower_callnative(cc, a, insn)) + GOTO_FAIL; + break; + + case JIT_OP_CALLBC: + if (!lower_callbc(cc, a, jmp_info_list, label_index, insn)) + GOTO_FAIL; + break; + + case JIT_OP_RETURNBC: + if (!lower_returnbc(cc, a, insn)) + GOTO_FAIL; + break; + + case JIT_OP_RETURN: + if (!lower_return(cc, a, insn)) + GOTO_FAIL; + break; + + case JIT_OP_I32CASTF32: + LOAD_2ARGS(); + CAST_R_R(F32, I32, f32, i32, int32); + break; + + case JIT_OP_I64CASTF64: + LOAD_2ARGS(); + CAST_R_R(F64, I64, f64, i64, int64); + break; + + case JIT_OP_F32CASTI32: + LOAD_2ARGS(); + CAST_R_R(I32, F32, i32, f32, float); + break; + + case JIT_OP_F64CASTI64: + LOAD_2ARGS(); + CAST_R_R(I64, F64, i64, f64, double); + break; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case JIT_OP_AT_CMPXCHGU8: + LOAD_4ARGS_NO_ASSIGN(); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + CMPXCHG_R_R_R_R_R(I32, int32, 1); + else + CMPXCHG_R_R_R_R_R(I64, int64, 1); + break; + + case JIT_OP_AT_CMPXCHGU16: + LOAD_4ARGS_NO_ASSIGN(); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + CMPXCHG_R_R_R_R_R(I32, int32, 2); + else + CMPXCHG_R_R_R_R_R(I64, int64, 2); + break; + + case JIT_OP_AT_CMPXCHGI32: + LOAD_4ARGS_NO_ASSIGN(); + CMPXCHG_R_R_R_R_R(I32, int32, 4); + break; + + case JIT_OP_AT_CMPXCHGU32: + LOAD_4ARGS_NO_ASSIGN(); + CMPXCHG_R_R_R_R_R(I64, int32, 4); + break; + + case JIT_OP_AT_CMPXCHGI64: + LOAD_4ARGS_NO_ASSIGN(); + CMPXCHG_R_R_R_R_R(I64, int64, 8); + break; + + case JIT_OP_AT_ADDU8: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(add, I32, int32, 1); + else + AT_RMW_R_R_R_R(add, I64, int64, 1); + break; + + case JIT_OP_AT_ADDU16: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(add, I32, int32, 2); + else + AT_RMW_R_R_R_R(add, I64, int64, 2); + break; + + case JIT_OP_AT_ADDI32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(add, I32, int32, 4); + break; + + case JIT_OP_AT_ADDU32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(add, I64, int64, 4); + break; + + case JIT_OP_AT_ADDI64: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(add, I64, int64, 8); + break; + + case JIT_OP_AT_SUBU8: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(sub, I32, int32, 1); + else + AT_RMW_R_R_R_R(sub, I64, int64, 1); + break; + + case JIT_OP_AT_SUBU16: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(sub, I32, int32, 2); + else + AT_RMW_R_R_R_R(sub, I64, int64, 2); + break; + + case JIT_OP_AT_SUBI32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(sub, I32, int32, 4); + break; + + case JIT_OP_AT_SUBU32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(sub, I64, int64, 4); + break; + + case JIT_OP_AT_SUBI64: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(sub, I64, int64, 8); + break; + + case JIT_OP_AT_XCHGU8: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(xchg, I32, int32, 1); + else + AT_RMW_R_R_R_R(xchg, I64, int64, 1); + break; + + case JIT_OP_AT_XCHGU16: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(xchg, I32, int32, 2); + else + AT_RMW_R_R_R_R(xchg, I64, int64, 2); + break; + + case JIT_OP_AT_XCHGI32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(xchg, I32, int32, 4); + break; + + case JIT_OP_AT_XCHGU32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(xchg, I64, int64, 4); + break; + + case JIT_OP_AT_XCHGI64: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(xchg, I64, int64, 8); + break; + + case JIT_OP_AT_ANDU8: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(and, I32, int32, 1); + else + AT_RMW_R_R_R_R(and, I64, int64, 1); + break; + + case JIT_OP_AT_ANDU16: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(and, I32, int32, 2); + else + AT_RMW_R_R_R_R(and, I64, int64, 2); + break; + + case JIT_OP_AT_ANDI32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(and, I32, int32, 4); + break; + + case JIT_OP_AT_ANDU32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(and, I64, int64, 4); + break; + + case JIT_OP_AT_ANDI64: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(and, I64, int64, 8); + break; + + case JIT_OP_AT_ORU8: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(or, I32, int32, 1); + else + AT_RMW_R_R_R_R(or, I64, int64, 1); + break; + + case JIT_OP_AT_ORU16: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(or, I32, int32, 2); + else + AT_RMW_R_R_R_R(or, I64, int64, 2); + break; + + case JIT_OP_AT_ORI32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(or, I32, int32, 4); + break; + + case JIT_OP_AT_ORU32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(or, I64, int64, 4); + break; + + case JIT_OP_AT_ORI64: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(or, I64, int64, 8); + break; + + case JIT_OP_AT_XORU8: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(xor, I32, int32, 1); + else + AT_RMW_R_R_R_R(xor, I64, int64, 1); + break; + + case JIT_OP_AT_XORU16: + LOAD_4ARGS(); + bh_assert(jit_reg_kind(r0) == JIT_REG_KIND_I32 + || jit_reg_kind(r0) == JIT_REG_KIND_I64); + if (jit_reg_kind(r0) == JIT_REG_KIND_I32) + AT_RMW_R_R_R_R(xor, I32, int32, 2); + else + AT_RMW_R_R_R_R(xor, I64, int64, 2); + break; + + case JIT_OP_AT_XORI32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(xor, I32, int32, 4); + break; + + case JIT_OP_AT_XORU32: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(xor, I64, int64, 4); + break; + + case JIT_OP_AT_XORI64: + LOAD_4ARGS(); + AT_RMW_R_R_R_R(xor, I64, int64, 8); + break; + + case JIT_OP_FENCE: + FENCE(); + break; + +#endif + + default: + jit_set_last_error_v(cc, "unsupported JIT opcode 0x%2x", + insn->opcode); + GOTO_FAIL; + } + + if (err_handler.err) { + jit_set_last_error_v(cc, + "failed to generate native code for JIT " + "opcode 0x%02x, ErrorCode is %u", + insn->opcode, err_handler.err); + GOTO_FAIL; + } + +#if CODEGEN_DUMP != 0 + dump_native((char *)code.sectionById(0)->buffer().data() + + code_offset, + code.sectionById(0)->buffer().size() - code_offset); + code_offset = code.sectionById(0)->buffer().size(); +#endif + } + } + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + if (!(stream = (char *)jit_code_cache_alloc(code_size))) { + jit_set_last_error(cc, "allocate memory failed"); + goto fail; + } + + bh_memcpy_s(stream, code_size, code_buf, code_size); + cc->jitted_addr_begin = stream; + cc->jitted_addr_end = stream + code_size; + + for (i = 0; i < label_num; i++) { + if (i == 0) + label_index = 0; + else if (i == label_num - 1) + label_index = 1; + else + label_index = i + 1; + + jitted_addr = jit_annl_jitted_addr( + cc, jit_reg_new(JIT_REG_KIND_L32, label_index)); + *jitted_addr = stream + label_offsets[label_index]; + } + + patch_jmp_info_list(cc, jmp_info_list); + return_value = true; + +fail: + + jit_free(label_offsets); + free_jmp_info_list(jmp_info_list); + return return_value; +} + +#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_JIT != 0 + +#define MAX_REG_INTS 6 +#define MAX_REG_FLOATS 8 + +void * +jit_codegen_compile_call_to_llvm_jit(const WASMType *func_type) +{ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + x86::Gp reg_lp = x86::r10, reg_res = x86::r12; + x86::Gp reg_tmp_i64 = x86::r11, reg_tmp_i32 = x86::r11d; + /* the index of integer argument registers */ + uint8 reg_idx_of_int_args[] = { REG_RDI_IDX, REG_RSI_IDX, REG_RDX_IDX, + REG_RCX_IDX, REG_R8_IDX, REG_R9_IDX }; + uint32 n_ints = 0, n_fps = 0, n_stacks = 0, n_pushed; + uint32 int_reg_idx = 0, fp_reg_idx = 0, stack_arg_idx = 0; + uint32 off_to_lp = 0, off_to_res = 0, code_size, i; + uint32 param_count = func_type->param_count; + uint32 result_count = func_type->result_count; + uint32 ext_result_count; + char *code_buf, *stream; + Imm imm; + + JitErrorHandler err_handler; + Environment env(Arch::kX64); + CodeHolder code; + code.init(env); + code.setErrorHandler(&err_handler); + x86::Assembler a(&code); + + /* Load the llvm jit function pointer */ + { + /* r11 = exec_env->module_inst */ + x86::Mem m1(regs_i64[hreg_info->exec_env_hreg_index], + (uint32)offsetof(WASMExecEnv, module_inst)); + a.mov(reg_tmp_i64, m1); + /* r11 = module_inst->func_ptrs */ + x86::Mem m2(reg_tmp_i64, + (uint32)offsetof(WASMModuleInstance, func_ptrs)); + a.mov(reg_tmp_i64, m2); + /* rax = func_ptrs[func_idx] */ + x86::Mem m3(reg_tmp_i64, x86::rdx, 3, 0); + a.mov(x86::rax, m3); + } + + n_ints++; /* exec_env */ + + for (i = 0; i < param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_I64: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + if (n_ints < MAX_REG_INTS) + n_ints++; + else + n_stacks++; + break; + case VALUE_TYPE_F32: + case VALUE_TYPE_F64: + if (n_fps < MAX_REG_FLOATS) + n_fps++; + else + n_stacks++; + break; + } + } + + ext_result_count = result_count > 1 ? result_count - 1 : 0; + + if (ext_result_count > 0) { + if (n_ints + ext_result_count <= MAX_REG_INTS) { + /* extra result pointers can be stored into int registers */ + n_ints += ext_result_count; + } + else { + /* part or all extra result pointers must be stored into stack */ + n_stacks += n_ints + ext_result_count - MAX_REG_INTS; + n_ints = MAX_REG_INTS; + } + } + + n_pushed = n_stacks; + if (n_stacks & 1) { + /* Align stack on 16 bytes */ + n_pushed++; + } + if (n_pushed > 0) { + imm.setValue(n_pushed * 8); + a.sub(x86::rsp, imm); + } + + /* r10 = outs_area->lp */ + { + x86::Mem m(regs_i64[hreg_info->exec_env_hreg_index], + (uint32)offsetof(WASMExecEnv, wasm_stack.top)); + a.mov(reg_lp, m); + a.add(reg_lp, (uint32)offsetof(WASMInterpFrame, lp)); + } + + /* rdi = exec_env */ + a.mov(regs_i64[reg_idx_of_int_args[int_reg_idx++]], + regs_i64[hreg_info->exec_env_hreg_index]); + + for (i = 0; i < param_count; i++) { + x86::Mem m_src(reg_lp, off_to_lp); + + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + { + if (int_reg_idx < MAX_REG_INTS) { + a.mov(regs_i32[reg_idx_of_int_args[int_reg_idx]], m_src); + int_reg_idx++; + } + else { + a.mov(reg_tmp_i32, m_src); + x86::Mem m_dst(x86::rsp, stack_arg_idx * 8); + a.mov(m_dst, reg_tmp_i32); + stack_arg_idx++; + } + off_to_lp += 4; + break; + } + case VALUE_TYPE_I64: + { + if (int_reg_idx < MAX_REG_INTS) { + a.mov(regs_i64[reg_idx_of_int_args[int_reg_idx]], m_src); + int_reg_idx++; + } + else { + a.mov(reg_tmp_i64, m_src); + x86::Mem m_dst(x86::rsp, stack_arg_idx * 8); + a.mov(m_dst, reg_tmp_i64); + stack_arg_idx++; + } + off_to_lp += 8; + break; + } + case VALUE_TYPE_F32: + { + if (fp_reg_idx < MAX_REG_FLOATS) { + a.movss(regs_float[fp_reg_idx], m_src); + fp_reg_idx++; + } + else { + a.mov(reg_tmp_i32, m_src); + x86::Mem m_dst(x86::rsp, stack_arg_idx * 8); + a.mov(m_dst, reg_tmp_i32); + stack_arg_idx++; + } + off_to_lp += 4; + break; + } + case VALUE_TYPE_F64: + { + if (fp_reg_idx < MAX_REG_FLOATS) { + a.movsd(regs_float[fp_reg_idx], m_src); + fp_reg_idx++; + } + else { + a.mov(reg_tmp_i64, m_src); + x86::Mem m_dst(x86::rsp, stack_arg_idx * 8); + a.mov(m_dst, reg_tmp_i64); + stack_arg_idx++; + } + off_to_lp += 8; + break; + } + } + } + + if (result_count > 0) { + switch (func_type->types[param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + case VALUE_TYPE_F32: + off_to_res = 4; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + off_to_res = 8; + break; + } + + /* r12 = cur_frame->sp */ + x86::Mem m(x86::rbp, (uint32)offsetof(WASMInterpFrame, sp)); + a.mov(reg_res, m); + + for (i = 0; i < ext_result_count; i++) { + x86::Mem m(reg_res, off_to_res); + + if (int_reg_idx < MAX_REG_INTS) { + a.lea(regs_i64[reg_idx_of_int_args[int_reg_idx]], m); + int_reg_idx++; + } + else { + a.lea(reg_tmp_i64, m); + x86::Mem m_dst(x86::rsp, stack_arg_idx * 8); + a.mov(m_dst, reg_tmp_i64); + stack_arg_idx++; + } + + switch (func_type->types[param_count + 1 + i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + case VALUE_TYPE_F32: + off_to_res += 4; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + off_to_res += 8; + break; + } + } + } + + bh_assert(int_reg_idx == n_ints); + bh_assert(fp_reg_idx == n_fps); + bh_assert(stack_arg_idx == n_stacks); + + /* Call the llvm jit function */ + a.call(x86::rax); + + /* Check if there was exception thrown */ + { + /* r11 = exec_env->module_inst */ + x86::Mem m1(regs_i64[hreg_info->exec_env_hreg_index], + (uint32)offsetof(WASMExecEnv, module_inst)); + a.mov(reg_tmp_i64, m1); + /* module_inst->cur_exception */ + x86::Mem m2(reg_tmp_i64, + (uint32)offsetof(WASMModuleInstance, cur_exception)); + /* bl = module_inst->cur_exception[0] */ + a.mov(x86::bl, m2); + + /* cur_exception[0] == 0 ? */ + Imm imm((uint8)0); + a.cmp(x86::bl, imm); + /* If yes, jump to `Get function result and return` */ + imm.setValue(INT32_MAX); + a.je(imm); + + char *stream = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); + + /* If no, set eax to JIT_INTERP_ACTION_THROWN, and + jump to code_block_return_to_interp_from_jitted to + return to interpreter */ + imm.setValue(JIT_INTERP_ACTION_THROWN); + a.mov(x86::eax, imm); + imm.setValue(code_block_return_to_interp_from_jitted); + a.mov(x86::rsi, imm); + a.jmp(x86::rsi); + + char *stream_new = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); + + *(int32 *)(stream - 4) = (uint32)(stream_new - stream); + } + + /* Get function result and return */ + + if (result_count > 0 && func_type->types[param_count] != VALUE_TYPE_F32 + && func_type->types[param_count] != VALUE_TYPE_F64) { + a.mov(x86::rdx, x86::rax); + } + + if (off_to_res > 0) { + imm.setValue(off_to_res); + a.add(reg_res, imm); + /* cur_frame->sp = r12 */ + x86::Mem m(x86::rbp, (uint32)offsetof(WASMInterpFrame, sp)); + a.mov(m, reg_res); + } + + if (n_pushed > 0) { + imm.setValue(n_pushed * 8); + a.add(x86::rsp, imm); + } + + /* Return to the caller */ + { + /* eax = action = JIT_INTERP_ACTION_NORMAL */ + Imm imm(0); + a.mov(x86::eax, imm); + + uint32 jitted_return_addr_offset = + jit_frontend_get_jitted_return_addr_offset(); + x86::Mem m(x86::rbp, jitted_return_addr_offset); + a.jmp(m); + } + + if (err_handler.err) + return NULL; + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + stream = (char *)jit_code_cache_alloc(code_size); + if (!stream) + return NULL; + + bh_memcpy_s(stream, code_size, code_buf, code_size); + +#if 0 + dump_native(stream, code_size); +#endif + + return stream; +} + +static WASMInterpFrame * +fast_jit_alloc_frame(WASMExecEnv *exec_env, uint32 param_cell_num, + uint32 ret_cell_num) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + WASMInterpFrame *frame; + uint32 size_frame1 = wasm_interp_interp_frame_size(ret_cell_num); + uint32 size_frame2 = wasm_interp_interp_frame_size(param_cell_num); + + /** + * Check whether we can allocate two frames: the first is an implied + * frame to store the function results from jit function to call, + * the second is the frame for the jit function + */ + if ((uint8 *)exec_env->wasm_stack.top + size_frame1 + size_frame2 + > exec_env->wasm_stack.top_boundary) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + return NULL; + } + + /* Allocate the frame */ + frame = (WASMInterpFrame *)exec_env->wasm_stack.top; + exec_env->wasm_stack.top += size_frame1; + + frame->function = NULL; + frame->ip = NULL; + frame->sp = frame->lp; + frame->prev_frame = wasm_exec_env_get_cur_frame(exec_env); + frame->jitted_return_addr = + (uint8 *)code_block_return_to_interp_from_jitted; + + wasm_exec_env_set_cur_frame(exec_env, frame); + + return frame; +} + +void * +jit_codegen_compile_call_to_fast_jit(const WASMModule *module, uint32 func_idx) +{ + uint32 func_idx_non_import = func_idx - module->import_function_count; + WASMType *func_type = module->functions[func_idx_non_import]->func_type; + /* the index of integer argument registers */ + uint8 reg_idx_of_int_args[] = { REG_RDI_IDX, REG_RSI_IDX, REG_RDX_IDX, + REG_RCX_IDX, REG_R8_IDX, REG_R9_IDX }; + uint32 int_reg_idx, fp_reg_idx, stack_arg_idx; + uint32 switch_info_offset, exec_env_offset, stack_arg_offset; + uint32 int_reg_offset, frame_lp_offset; + uint32 switch_info_size, code_size, i; + uint32 param_count = func_type->param_count; + uint32 result_count = func_type->result_count; + uint32 ext_result_count = result_count > 1 ? result_count - 1 : 0; + uint32 param_cell_num = func_type->param_cell_num; + uint32 ret_cell_num = + func_type->ret_cell_num > 2 ? func_type->ret_cell_num : 2; + char *code_buf, *stream; + Imm imm; + + JitErrorHandler err_handler; + Environment env(Arch::kX64); + CodeHolder code; + code.init(env); + code.setErrorHandler(&err_handler); + x86::Assembler a(&code); + + /** + * Push JitInterpSwitchInfo and make stack 16-byte aligned: + * the size pushed must be odd multiples of 8, as the stack pointer + * %rsp must be aligned to a 16-byte boundary before making a call, + * and when a function (including this llvm jit function) gets + * control, the %rsp is not 16-byte aligned (call instruction will + * push the ret address to stack). + */ + switch_info_size = align_uint((uint32)sizeof(JitInterpSwitchInfo), 16) + 8; + imm.setValue((uint64)switch_info_size); + a.sub(x86::rsp, imm); + + /* Push all integer argument registers since we will use them as + temporarily registers to load/store data */ + for (i = 0; i < MAX_REG_INTS; i++) { + a.push(regs_i64[reg_idx_of_int_args[MAX_REG_INTS - 1 - i]]); + } + + /* We don't push float/double register since we don't use them here */ + + /** + * Layout of the stack now: + * stack arguments + * ret address of the caller + * switch info + * int registers: r9, r8, rcx, rdx, rsi + * exec_env: rdi + */ + + /* offset of the first stack argument to the stack pointer, + add 8 to skip the ret address of the caller */ + stack_arg_offset = switch_info_size + 8 * MAX_REG_INTS + 8; + /* offset of jit interp switch info to the stack pointer */ + switch_info_offset = 8 * MAX_REG_INTS; + /* offset of the first int register to the stack pointer */ + int_reg_offset = 8; + /* offset of exec_env to the stack pointer */ + exec_env_offset = 0; + + /* Call fast_jit_alloc_frame to allocate the stack frame to + receive the results of the fast jit function to call */ + + /* rdi = exec_env, has been already set as exec_env is + the first argument of LLVM JIT function */ + /* rsi = param_cell_num */ + imm.setValue(param_cell_num); + a.mov(x86::rsi, imm); + /* rdx = ret_cell_num */ + imm.setValue(ret_cell_num); + a.mov(x86::rdx, imm); + /* call fast_jit_alloc_frame */ + imm.setValue((uint64)(uintptr_t)fast_jit_alloc_frame); + a.mov(x86::rax, imm); + a.call(x86::rax); + + /* Check the return value, note now rax is the allocated frame */ + { + /* Did fast_jit_alloc_frame return NULL? */ + Imm imm((uint64)0); + a.cmp(x86::rax, imm); + /* If no, jump to `Copy arguments to frame lp area` */ + imm.setValue(INT32_MAX); + a.jne(imm); + + char *stream = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); + + /* If yes, set eax to 0, return to caller */ + + /* Pop all integer arument registers */ + for (i = 0; i < MAX_REG_INTS; i++) { + a.pop(regs_i64[reg_idx_of_int_args[i]]); + } + /* Pop jit interp switch info */ + imm.setValue((uint64)switch_info_size); + a.add(x86::rsp, imm); + + /* Return to the caller, don't use leave as we didn't + `push rbp` and `mov rbp, rsp` */ + a.ret(); + + /* Patch the offset of jne instruction */ + char *stream_new = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); + *(int32 *)(stream - 4) = (int32)(stream_new - stream); + } + + int_reg_idx = 1; /* skip exec_env */ + fp_reg_idx = 0; + stack_arg_idx = 0; + + /* Offset of the dest arguments to outs area */ + frame_lp_offset = wasm_interp_interp_frame_size(ret_cell_num) + + (uint32)offsetof(WASMInterpFrame, lp); + + /* Copy arguments to frame lp area */ + for (i = 0; i < func_type->param_count; i++) { + x86::Mem m_dst(x86::rax, frame_lp_offset); + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + if (int_reg_idx < MAX_REG_INTS) { + /* Copy i32 argument from int register */ + x86::Mem m_src(x86::rsp, int_reg_offset); + a.mov(x86::esi, m_src); + a.mov(m_dst, x86::esi); + int_reg_offset += 8; + int_reg_idx++; + } + else { + /* Copy i32 argument from stack */ + x86::Mem m_src(x86::rsp, stack_arg_offset); + a.mov(x86::esi, m_src); + a.mov(m_dst, x86::esi); + stack_arg_offset += 8; + stack_arg_idx++; + } + frame_lp_offset += 4; + break; + case VALUE_TYPE_I64: + if (int_reg_idx < MAX_REG_INTS) { + /* Copy i64 argument from int register */ + x86::Mem m_src(x86::rsp, int_reg_offset); + a.mov(x86::rsi, m_src); + a.mov(m_dst, x86::rsi); + int_reg_offset += 8; + int_reg_idx++; + } + else { + /* Copy i64 argument from stack */ + x86::Mem m_src(x86::rsp, stack_arg_offset); + a.mov(x86::rsi, m_src); + a.mov(m_dst, x86::rsi); + stack_arg_offset += 8; + stack_arg_idx++; + } + frame_lp_offset += 8; + break; + case VALUE_TYPE_F32: + if (fp_reg_idx < MAX_REG_FLOATS) { + /* Copy f32 argument from fp register */ + a.movss(m_dst, regs_float[fp_reg_idx++]); + } + else { + /* Copy f32 argument from stack */ + x86::Mem m_src(x86::rsp, stack_arg_offset); + a.mov(x86::esi, m_src); + a.mov(m_dst, x86::esi); + stack_arg_offset += 8; + stack_arg_idx++; + } + frame_lp_offset += 4; + break; + case VALUE_TYPE_F64: + if (fp_reg_idx < MAX_REG_FLOATS) { + /* Copy f64 argument from fp register */ + a.movsd(m_dst, regs_float[fp_reg_idx++]); + } + else { + /* Copy f64 argument from stack */ + x86::Mem m_src(x86::rsp, stack_arg_offset); + a.mov(x86::rsi, m_src); + a.mov(m_dst, x86::rsi); + stack_arg_offset += 8; + stack_arg_idx++; + } + frame_lp_offset += 8; + break; + default: + bh_assert(0); + } + } + + /* Call the fast jit function */ + { + /* info = rsp + switch_info_offset */ + a.lea(x86::rsi, x86::ptr(x86::rsp, switch_info_offset)); + /* info.frame = frame = rax, or return of fast_jit_alloc_frame */ + x86::Mem m1(x86::rsi, (uint32)offsetof(JitInterpSwitchInfo, frame)); + a.mov(m1, x86::rax); + + /* Call code_block_switch_to_jitted_from_interp + with argument (exec_env, info, func_idx, pc) */ + /* rdi = exec_env */ + a.mov(x86::rdi, x86::ptr(x86::rsp, exec_env_offset)); + /* rsi = info, has been set */ + /* rdx = func_idx */ + imm.setValue(func_idx); + a.mov(x86::rdx, imm); + /* module_inst = exec_env->module_inst */ + a.mov(x86::rcx, + x86::ptr(x86::rdi, (uint32)offsetof(WASMExecEnv, module_inst))); + /* fast_jit_func_ptrs = module_inst->fast_jit_func_ptrs */ + a.mov(x86::rcx, + x86::ptr(x86::rcx, (uint32)offsetof(WASMModuleInstance, + fast_jit_func_ptrs))); + imm.setValue(func_idx_non_import); + a.mov(x86::rax, imm); + x86::Mem m3(x86::rcx, x86::rax, 3, 0); + /* rcx = module_inst->fast_jit_func_ptrs[func_idx_non_import] */ + a.mov(x86::rcx, m3); + + imm.setValue( + (uint64)(uintptr_t)code_block_switch_to_jitted_from_interp); + a.mov(x86::rax, imm); + a.call(x86::rax); + } + + /* No need to check exception thrown here as it will be checked + in the caller */ + + /* Copy function results */ + if (result_count > 0) { + frame_lp_offset = offsetof(WASMInterpFrame, lp); + + switch (func_type->types[param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + a.mov(x86::eax, x86::edx); + frame_lp_offset += 4; + break; + case VALUE_TYPE_I64: + a.mov(x86::rax, x86::rdx); + frame_lp_offset += 8; + break; + case VALUE_TYPE_F32: + /* The first result has been put to xmm0 */ + frame_lp_offset += 4; + break; + case VALUE_TYPE_F64: + /* The first result has been put to xmm0 */ + frame_lp_offset += 8; + break; + default: + bh_assert(0); + } + + /* Copy extra results from exec_env->cur_frame */ + if (ext_result_count > 0) { + /* rdi = exec_env */ + a.mov(x86::rdi, x86::ptr(x86::rsp, exec_env_offset)); + /* rsi = exec_env->cur_frame */ + a.mov(x86::rsi, + x86::ptr(x86::rdi, (uint32)offsetof(WASMExecEnv, cur_frame))); + + for (i = 0; i < ext_result_count; i++) { + switch (func_type->types[param_count + 1 + i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + case VALUE_TYPE_F32: + { + /* Copy 32-bit result */ + a.mov(x86::ecx, x86::ptr(x86::rsi, frame_lp_offset)); + if (int_reg_idx < MAX_REG_INTS) { + x86::Mem m1(x86::rsp, + exec_env_offset + int_reg_idx * 8); + a.mov(x86::rdx, m1); + x86::Mem m2(x86::rdx, 0); + a.mov(m2, x86::ecx); + int_reg_idx++; + } + else { + x86::Mem m1(x86::rsp, stack_arg_offset); + a.mov(x86::rdx, m1); + x86::Mem m2(x86::rdx, 0); + a.mov(m2, x86::ecx); + stack_arg_offset += 8; + stack_arg_idx++; + } + frame_lp_offset += 4; + break; + } + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + { + /* Copy 64-bit result */ + a.mov(x86::rcx, x86::ptr(x86::rsi, frame_lp_offset)); + if (int_reg_idx < MAX_REG_INTS) { + x86::Mem m1(x86::rsp, + exec_env_offset + int_reg_idx * 8); + a.mov(x86::rdx, m1); + x86::Mem m2(x86::rdx, 0); + a.mov(m2, x86::rcx); + int_reg_idx++; + } + else { + x86::Mem m1(x86::rsp, stack_arg_offset); + a.mov(x86::rdx, m1); + x86::Mem m2(x86::rdx, 0); + a.mov(m2, x86::rcx); + stack_arg_offset += 8; + stack_arg_idx++; + } + frame_lp_offset += 8; + break; + } + default: + bh_assert(0); + } + } + } + } + + /* Free the frame allocated */ + + /* rdi = exec_env */ + a.mov(x86::rdi, x86::ptr(x86::rsp, exec_env_offset)); + /* rsi = exec_env->cur_frame */ + a.mov(x86::rsi, + x86::ptr(x86::rdi, (uint32)offsetof(WASMExecEnv, cur_frame))); + /* rdx = exec_env->cur_frame->prev_frame */ + a.mov(x86::rdx, + x86::ptr(x86::rsi, (uint32)offsetof(WASMInterpFrame, prev_frame))); + /* exec_env->wasm_stack.top = cur_frame */ + { + x86::Mem m(x86::rdi, offsetof(WASMExecEnv, wasm_stack.top)); + a.mov(m, x86::rsi); + } + /* exec_env->cur_frame = prev_frame */ + { + x86::Mem m(x86::rdi, offsetof(WASMExecEnv, cur_frame)); + a.mov(m, x86::rdx); + } + + /* Pop all integer arument registers */ + for (i = 0; i < MAX_REG_INTS; i++) { + a.pop(regs_i64[reg_idx_of_int_args[i]]); + } + /* Pop jit interp switch info */ + imm.setValue((uint64)switch_info_size); + a.add(x86::rsp, imm); + + /* Return to the caller, don't use leave as we didn't + `push rbp` and `mov rbp, rsp` */ + a.ret(); + + if (err_handler.err) { + return NULL; + } + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + stream = (char *)jit_code_cache_alloc(code_size); + if (!stream) + return NULL; + + bh_memcpy_s(stream, code_size, code_buf, code_size); + +#if 0 + printf("Code of call to fast jit of func %u:\n", func_idx); + dump_native(stream, code_size); + printf("\n"); +#endif + + return stream; +} + +#endif /* end of WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_JIT != 0 */ + +bool +jit_codegen_lower(JitCompContext *cc) +{ + (void)cc; + return true; +} + +void +jit_codegen_free_native(JitCompContext *cc) +{ + (void)cc; +} + +void +jit_codegen_dump_native(void *begin_addr, void *end_addr) +{ +#if WASM_ENABLE_FAST_JIT_DUMP != 0 + os_printf("\n"); + dump_native((char *)begin_addr, (char *)end_addr - (char *)begin_addr); + os_printf("\n"); +#else + (void)begin_addr; + (void)end_addr; +#endif +} + +bool +jit_codegen_init() +{ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + char *code_buf, *stream; + uint32 code_size; + + JitErrorHandler err_handler; + Environment env(Arch::kX64); + CodeHolder code; + code.init(env); + code.setErrorHandler(&err_handler); + x86::Assembler a(&code); + + /* Initialize code_block_switch_to_jitted_from_interp */ + + /* push callee-save registers */ + a.push(x86::rbp); + a.push(x86::rbx); + a.push(x86::r12); + a.push(x86::r13); + a.push(x86::r14); + a.push(x86::r15); + /* push info */ + a.push(x86::rsi); + + /* Note: the number of register pushed must be odd, as the stack pointer + %rsp must be aligned to a 16-byte boundary before making a call, so + when a function (including this function) gets control, %rsp is not + aligned. We push odd number registers here to make %rsp happy before + calling native functions. */ + + /* exec_env_reg = exec_env */ + a.mov(regs_i64[hreg_info->exec_env_hreg_index], x86::rdi); + /* fp_reg = info->frame */ + a.mov(x86::rbp, x86::ptr(x86::rsi, offsetof(JitInterpSwitchInfo, frame))); + /* rdx = func_idx, is already set in the func_idx argument of + jit_codegen_interp_jitted_glue */ + /* jmp target, rcx = pc */ + a.jmp(x86::rcx); + + if (err_handler.err) + return false; + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + stream = (char *)jit_code_cache_alloc(code_size); + if (!stream) + return false; + + bh_memcpy_s(stream, code_size, code_buf, code_size); + code_block_switch_to_jitted_from_interp = stream; + +#if 0 + dump_native(stream, code_size); +#endif + + /* Initialize code_block_return_to_interp_from_jitted */ + + a.setOffset(0); + + /* pop info */ + a.pop(x86::rsi); + /* info->frame = fp_reg */ + { + x86::Mem m(x86::rsi, offsetof(JitInterpSwitchInfo, frame)); + a.mov(m, x86::rbp); + } + /* info->out.ret.ival[0, 1] = rdx */ + { + x86::Mem m(x86::rsi, offsetof(JitInterpSwitchInfo, out.ret.ival)); + a.mov(m, x86::rdx); + } + /* info->out.ret.fval[0, 1] = xmm0 */ + { + x86::Mem m(x86::rsi, offsetof(JitInterpSwitchInfo, out.ret.fval)); + a.movsd(m, x86::xmm0); + } + + /* pop callee-save registers */ + a.pop(x86::r15); + a.pop(x86::r14); + a.pop(x86::r13); + a.pop(x86::r12); + a.pop(x86::rbx); + a.pop(x86::rbp); + a.ret(); + + if (err_handler.err) + goto fail1; + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + stream = (char *)jit_code_cache_alloc(code_size); + if (!stream) + goto fail1; + + bh_memcpy_s(stream, code_size, code_buf, code_size); + code_block_return_to_interp_from_jitted = + jit_globals->return_to_interp_from_jitted = stream; + +#if 0 + dump_native(stream, code_size); +#endif + +#if WASM_ENABLE_LAZY_JIT != 0 + /* Initialize code_block_compile_fast_jit_and_then_call */ + + a.setOffset(0); + + /* Use rbx, r12, r13 to save func_dix, module_inst and module, + as they are callee-save registers */ + + /* Backup func_idx: rbx = rdx = func_idx, note that rdx has + been prepared in the caller: + callbc or code_block_switch_to_jitted_from_interp */ + a.mov(x86::rbx, x86::rdx); + /* r12 = module_inst = exec_env->module_inst */ + { + x86::Mem m(regs_i64[hreg_info->exec_env_hreg_index], + (uint32)offsetof(WASMExecEnv, module_inst)); + a.mov(x86::r12, m); + } + /* rdi = r13 = module_inst->module */ + { + x86::Mem m(x86::r12, (uint32)offsetof(WASMModuleInstance, module)); + a.mov(x86::rdi, m); + a.mov(x86::r13, x86::rdi); + } + /* rsi = rdx = func_idx */ + a.mov(x86::rsi, x86::rdx); + /* Call jit_compiler_compile(module, func_idx) */ + { + Imm imm((uint64)(uintptr_t)jit_compiler_compile); + a.mov(x86::rax, imm); + a.call(x86::rax); + } + + /* Check if failed to compile the jit function */ + { + /* Did jit_compiler_compile return false? */ + Imm imm((uint8)0); + a.cmp(x86::al, imm); + /* If no, jump to `Load compiled func ptr and call it` */ + imm.setValue(INT32_MAX); + a.jne(imm); + + char *stream_old = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); + + /* If yes, call jit_set_exception_with_id to throw exception, + and then set eax to JIT_INTERP_ACTION_THROWN, and jump to + code_block_return_to_interp_from_jitted to return */ + + /* rdi = module_inst */ + a.mov(x86::rdi, x86::r12); + /* rsi = EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC */ + imm.setValue(EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC); + a.mov(x86::rsi, imm); + /* Call jit_set_exception_with_id */ + imm.setValue((uint64)(uintptr_t)jit_set_exception_with_id); + a.mov(x86::rax, imm); + a.call(x86::rax); + /* Return to the caller */ + imm.setValue(JIT_INTERP_ACTION_THROWN); + a.mov(x86::eax, imm); + imm.setValue(code_block_return_to_interp_from_jitted); + a.mov(x86::rsi, imm); + a.jmp(x86::rsi); + + /* Patch the offset of jne instruction */ + char *stream_new = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); + *(int32 *)(stream_old - 4) = (int32)(stream_new - stream_old); + } + + /* Load compiled func ptr and call it */ + { + /* rsi = module->import_function_count */ + x86::Mem m1(x86::r13, + (uint32)offsetof(WASMModule, import_function_count)); + a.movzx(x86::rsi, m1); + /* rbx = rbx - module->import_function_count */ + a.sub(x86::rbx, x86::rsi); + /* rax = module->fast_jit_func_ptrs */ + x86::Mem m2(x86::r13, (uint32)offsetof(WASMModule, fast_jit_func_ptrs)); + a.mov(x86::rax, m2); + /* rax = fast_jit_func_ptrs[rbx] */ + x86::Mem m3(x86::rax, x86::rbx, 3, 0); + a.mov(x86::rax, m3); + a.jmp(x86::rax); + } + + if (err_handler.err) + goto fail2; + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + stream = (char *)jit_code_cache_alloc(code_size); + if (!stream) + goto fail2; + + bh_memcpy_s(stream, code_size, code_buf, code_size); + code_block_compile_fast_jit_and_then_call = + jit_globals->compile_fast_jit_and_then_call = stream; + +#if 0 + dump_native(stream, code_size); +#endif +#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */ + + return true; + +#if WASM_ENABLE_LAZY_JIT != 0 +fail2: + jit_code_cache_free(code_block_return_to_interp_from_jitted); +#endif +fail1: + jit_code_cache_free(code_block_switch_to_jitted_from_interp); + return false; +} + +void +jit_codegen_destroy() +{ +#if WASM_ENABLE_LAZY_JIT != 0 + jit_code_cache_free(code_block_compile_fast_jit_and_then_call); +#endif + jit_code_cache_free(code_block_return_to_interp_from_jitted); + jit_code_cache_free(code_block_switch_to_jitted_from_interp); +} + +/* clang-format off */ +static const uint8 hreg_info_I32[3][7] = { + /* ebp, eax, ebx, ecx, edx, edi, esi */ + { 1, 0, 0, 0, 0, 0, 1 }, /* fixed, esi is freely used */ + { 0, 1, 0, 1, 1, 1, 0 }, /* caller_saved_native */ + { 0, 1, 1, 1, 1, 1, 0 } /* caller_saved_jitted */ +}; + +static const uint8 hreg_info_I64[3][16] = { + /* rbp, rax, rbx, rcx, rdx, rdi, rsi, rsp, + r8, r9, r10, r11, r12, r13, r14, r15 */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 1 }, /* fixed, rsi is freely used */ + { 0, 1, 0, 1, 1, 1, 0, 0, + 1, 1, 1, 1, 0, 0, 0, 0 }, /* caller_saved_native */ + { 0, 1, 1, 1, 1, 1, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 0 }, /* caller_saved_jitted */ +}; + +/* System V AMD64 ABI Calling Conversion. [XYZ]MM0-7 */ +static uint8 hreg_info_F32[3][16] = { + /* xmm0 ~ xmm15 */ + { 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1 }, + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0 }, /* caller_saved_native */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0 }, /* caller_saved_jitted */ +}; + +/* System V AMD64 ABI Calling Conversion. [XYZ]MM0-7 */ +static uint8 hreg_info_F64[3][16] = { + /* xmm0 ~ xmm15 */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 1 }, + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0 }, /* caller_saved_native */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0 }, /* caller_saved_jitted */ +}; + +static const JitHardRegInfo g_hreg_info = { + { + { 0, NULL, NULL, NULL }, /* VOID */ + + { sizeof(hreg_info_I32[0]), /* I32 */ + hreg_info_I32[0], + hreg_info_I32[1], + hreg_info_I32[2] }, + + { sizeof(hreg_info_I64[0]), /* I64 */ + hreg_info_I64[0], + hreg_info_I64[1], + hreg_info_I64[2] }, + + { sizeof(hreg_info_F32[0]), /* F32 */ + hreg_info_F32[0], + hreg_info_F32[1], + hreg_info_F32[2] }, + + { sizeof(hreg_info_F64[0]), /* F64 */ + hreg_info_F64[0], + hreg_info_F64[1], + hreg_info_F64[2] }, + + { 0, NULL, NULL, NULL }, /* V8 */ + { 0, NULL, NULL, NULL }, /* V16 */ + { 0, NULL, NULL, NULL } /* V32 */ + }, + /* frame pointer hreg index: rbp */ + 0, + /* exec_env hreg index: r15 */ + 15, + /* cmp hreg index: esi */ + 6 +}; +/* clang-format on */ + +const JitHardRegInfo * +jit_codegen_get_hreg_info() +{ + return &g_hreg_info; +} + +static const char *reg_names_i32[] = { + "ebp", "eax", "ebx", "ecx", "edx", "edi", "esi", "esp", +}; + +static const char *reg_names_i64[] = { + "rbp", "rax", "rbx", "rcx", "rdx", "rdi", "rsi", "rsp", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", +}; + +static const char *reg_names_f32[] = { "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15" }; + +static const char *reg_names_f64[] = { + "xmm0_f64", "xmm1_f64", "xmm2_f64", "xmm3_f64", "xmm4_f64", "xmm5_f64", + "xmm6_f64", "xmm7_f64", "xmm8_f64", "xmm9_f64", "xmm10_f64", "xmm11_f64", + "xmm12_f64", "xmm13_f64", "xmm14_f64", "xmm15_f64" +}; + +JitReg +jit_codegen_get_hreg_by_name(const char *name) +{ + size_t i; + + if (name[0] == 'e') { + for (i = 0; i < sizeof(reg_names_i32) / sizeof(char *); i++) + if (!strcmp(reg_names_i32[i], name)) + return jit_reg_new(JIT_REG_KIND_I32, i); + } + else if (name[0] == 'r') { + for (i = 0; i < sizeof(reg_names_i64) / sizeof(char *); i++) + if (!strcmp(reg_names_i64[i], name)) + return jit_reg_new(JIT_REG_KIND_I64, i); + } + else if (!strncmp(name, "xmm", 3)) { + if (!strstr(name, "_f64")) { + for (i = 0; i < sizeof(reg_names_f32) / sizeof(char *); i++) + if (!strcmp(reg_names_f32[i], name)) + return jit_reg_new(JIT_REG_KIND_F32, i); + } + else { + for (i = 0; i < sizeof(reg_names_f64) / sizeof(char *); i++) + if (!strcmp(reg_names_f64[i], name)) + return jit_reg_new(JIT_REG_KIND_F64, i); + } + } + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_compare.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_compare.c new file mode 100644 index 0000000..8fe48f1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_compare.c @@ -0,0 +1,345 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_compare.h" +#include "jit_emit_function.h" +#include "../jit_frontend.h" +#include "../jit_codegen.h" + +static bool +jit_compile_op_compare_integer(JitCompContext *cc, IntCond cond, bool is64Bit) +{ + JitReg lhs, rhs, res, const_zero, const_one; + + if (cond < INT_EQZ || cond > INT_GE_U) { + jit_set_last_error(cc, "unsupported comparation operation"); + goto fail; + } + + res = jit_cc_new_reg_I32(cc); + const_zero = NEW_CONST(I32, 0); + const_one = NEW_CONST(I32, 1); + + if (is64Bit) { + if (INT_EQZ == cond) { + rhs = NEW_CONST(I64, 0); + } + else { + POP_I64(rhs); + } + POP_I64(lhs); + } + else { + if (INT_EQZ == cond) { + rhs = NEW_CONST(I32, 0); + } + else { + POP_I32(rhs); + } + POP_I32(lhs); + } + + GEN_INSN(CMP, cc->cmp_reg, lhs, rhs); + switch (cond) { + case INT_EQ: + case INT_EQZ: + { + GEN_INSN(SELECTEQ, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_NE: + { + GEN_INSN(SELECTNE, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_LT_S: + { + GEN_INSN(SELECTLTS, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_LT_U: + { + GEN_INSN(SELECTLTU, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_GT_S: + { + GEN_INSN(SELECTGTS, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_GT_U: + { + GEN_INSN(SELECTGTU, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_LE_S: + { + GEN_INSN(SELECTLES, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_LE_U: + { + GEN_INSN(SELECTLEU, res, cc->cmp_reg, const_one, const_zero); + break; + } + case INT_GE_S: + { + GEN_INSN(SELECTGES, res, cc->cmp_reg, const_one, const_zero); + break; + } + default: /* INT_GE_U */ + { + GEN_INSN(SELECTGEU, res, cc->cmp_reg, const_one, const_zero); + break; + } + } + + PUSH_I32(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_compare(JitCompContext *cc, IntCond cond) +{ + return jit_compile_op_compare_integer(cc, cond, false); +} + +bool +jit_compile_op_i64_compare(JitCompContext *cc, IntCond cond) +{ + return jit_compile_op_compare_integer(cc, cond, true); +} + +static int32 +float_cmp_eq(float f1, float f2) +{ + if (isnan(f1) || isnan(f2)) + return 0; + + return f1 == f2; +} + +static int32 +float_cmp_ne(float f1, float f2) +{ + if (isnan(f1) || isnan(f2)) + return 1; + + return f1 != f2; +} + +static int32 +double_cmp_eq(double d1, double d2) +{ + if (isnan(d1) || isnan(d2)) + return 0; + + return d1 == d2; +} + +static int32 +double_cmp_ne(double d1, double d2) +{ + if (isnan(d1) || isnan(d2)) + return 1; + + return d1 != d2; +} + +static bool +jit_compile_op_compare_float_point(JitCompContext *cc, FloatCond cond, + JitReg lhs, JitReg rhs) +{ + JitReg res, args[2], const_zero, const_one; + JitRegKind kind; + void *func; + + if (cond == FLOAT_EQ || cond == FLOAT_NE) { + kind = jit_reg_kind(lhs); + if (cond == FLOAT_EQ) + func = (kind == JIT_REG_KIND_F32) ? (void *)float_cmp_eq + : (void *)double_cmp_eq; + else + func = (kind == JIT_REG_KIND_F32) ? (void *)float_cmp_ne + : (void *)double_cmp_ne; + + res = jit_cc_new_reg_I32(cc); + args[0] = lhs; + args[1] = rhs; + + if (!jit_emit_callnative(cc, func, res, args, 2)) { + goto fail; + } + } + else { + res = jit_cc_new_reg_I32(cc); + const_zero = NEW_CONST(I32, 0); + const_one = NEW_CONST(I32, 1); + switch (cond) { + case FLOAT_LT: + { + GEN_INSN(CMP, cc->cmp_reg, rhs, lhs); + GEN_INSN(SELECTGTS, res, cc->cmp_reg, const_one, const_zero); + break; + } + case FLOAT_GT: + { + GEN_INSN(CMP, cc->cmp_reg, lhs, rhs); + GEN_INSN(SELECTGTS, res, cc->cmp_reg, const_one, const_zero); + break; + } + case FLOAT_LE: + { + GEN_INSN(CMP, cc->cmp_reg, rhs, lhs); + GEN_INSN(SELECTGES, res, cc->cmp_reg, const_one, const_zero); + break; + } + case FLOAT_GE: + { + GEN_INSN(CMP, cc->cmp_reg, lhs, rhs); + GEN_INSN(SELECTGES, res, cc->cmp_reg, const_one, const_zero); + break; + } + default: + { + bh_assert(!"unknown FloatCond"); + goto fail; + } + } + } + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_compare(JitCompContext *cc, FloatCond cond) +{ + JitReg res, const_zero, const_one; + JitReg lhs, rhs; + + POP_F32(rhs); + POP_F32(lhs); + + if (jit_reg_is_const(lhs) && jit_reg_is_const(rhs)) { + float32 lvalue = jit_cc_get_const_F32(cc, lhs); + float32 rvalue = jit_cc_get_const_F32(cc, rhs); + + const_zero = NEW_CONST(I32, 0); + const_one = NEW_CONST(I32, 1); + + switch (cond) { + case FLOAT_EQ: + { + res = (lvalue == rvalue) ? const_one : const_zero; + break; + } + case FLOAT_NE: + { + res = (lvalue != rvalue) ? const_one : const_zero; + break; + } + case FLOAT_LT: + { + res = (lvalue < rvalue) ? const_one : const_zero; + break; + } + case FLOAT_GT: + { + res = (lvalue > rvalue) ? const_one : const_zero; + break; + } + case FLOAT_LE: + { + res = (lvalue <= rvalue) ? const_one : const_zero; + break; + } + case FLOAT_GE: + { + res = (lvalue >= rvalue) ? const_one : const_zero; + break; + } + default: + { + bh_assert(!"unknown FloatCond"); + goto fail; + } + } + + PUSH_I32(res); + return true; + } + + return jit_compile_op_compare_float_point(cc, cond, lhs, rhs); +fail: + return false; +} + +bool +jit_compile_op_f64_compare(JitCompContext *cc, FloatCond cond) +{ + JitReg res, const_zero, const_one; + JitReg lhs, rhs; + + POP_F64(rhs); + POP_F64(lhs); + + if (jit_reg_is_const(lhs) && jit_reg_is_const(rhs)) { + float64 lvalue = jit_cc_get_const_F64(cc, lhs); + float64 rvalue = jit_cc_get_const_F64(cc, rhs); + + const_zero = NEW_CONST(I32, 0); + const_one = NEW_CONST(I32, 1); + + switch (cond) { + case FLOAT_EQ: + { + res = (lvalue == rvalue) ? const_one : const_zero; + break; + } + case FLOAT_NE: + { + res = (lvalue != rvalue) ? const_one : const_zero; + break; + } + case FLOAT_LT: + { + res = (lvalue < rvalue) ? const_one : const_zero; + break; + } + case FLOAT_GT: + { + res = (lvalue > rvalue) ? const_one : const_zero; + break; + } + case FLOAT_LE: + { + res = (lvalue <= rvalue) ? const_one : const_zero; + break; + } + case FLOAT_GE: + { + res = (lvalue >= rvalue) ? const_one : const_zero; + break; + } + default: + { + bh_assert(!"unknown FloatCond"); + goto fail; + } + } + + PUSH_I32(res); + return true; + } + + return jit_compile_op_compare_float_point(cc, cond, lhs, rhs); +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_compare.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_compare.h new file mode 100644 index 0000000..db905b5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_compare.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_COMPARE_H_ +#define _JIT_EMIT_COMPARE_H_ + +#include "../jit_compiler.h" +#include "../jit_frontend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_i32_compare(JitCompContext *cc, IntCond cond); + +bool +jit_compile_op_i64_compare(JitCompContext *cc, IntCond cond); + +bool +jit_compile_op_f32_compare(JitCompContext *cc, FloatCond cond); + +bool +jit_compile_op_f64_compare(JitCompContext *cc, FloatCond cond); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_COMPARE_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_const.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_const.c new file mode 100644 index 0000000..1bbc83c --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_const.c @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_const.h" +#include "../jit_frontend.h" + +bool +jit_compile_op_i32_const(JitCompContext *cc, int32 i32_const) +{ + JitReg value = NEW_CONST(I32, i32_const); + PUSH_I32(value); + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_const(JitCompContext *cc, int64 i64_const) +{ + JitReg value = NEW_CONST(I64, i64_const); + PUSH_I64(value); + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_const(JitCompContext *cc, float32 f32_const) +{ + JitReg value = NEW_CONST(F32, f32_const); + PUSH_F32(value); + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_const(JitCompContext *cc, float64 f64_const) +{ + JitReg value = NEW_CONST(F64, f64_const); + PUSH_F64(value); + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_const.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_const.h new file mode 100644 index 0000000..b753141 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_const.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_CONST_H_ +#define _JIT_EMIT_CONST_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_i32_const(JitCompContext *cc, int32 i32_const); + +bool +jit_compile_op_i64_const(JitCompContext *cc, int64 i64_const); + +bool +jit_compile_op_f32_const(JitCompContext *cc, float32 f32_const); + +bool +jit_compile_op_f64_const(JitCompContext *cc, float64 f64_const); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_CONST_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_control.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_control.c new file mode 100644 index 0000000..9274e72 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_control.c @@ -0,0 +1,1318 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_control.h" +#include "jit_emit_exception.h" +#include "jit_emit_function.h" +#include "../jit_frontend.h" +#include "../interpreter/wasm_loader.h" + +#define CREATE_BASIC_BLOCK(new_basic_block) \ + do { \ + bh_assert(!new_basic_block); \ + if (!(new_basic_block = jit_cc_new_basic_block(cc, 0))) { \ + jit_set_last_error(cc, "create basic block failed"); \ + goto fail; \ + } \ + } while (0) + +#define CURR_BASIC_BLOCK() cc->cur_basic_block + +#define BUILD_BR(target_block) \ + do { \ + if (!GEN_INSN(JMP, jit_basic_block_label(target_block))) { \ + jit_set_last_error(cc, "generate jmp insn failed"); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!GEN_INSN(CMP, cc->cmp_reg, value_if, NEW_CONST(I32, 0)) \ + || !GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(block_then), \ + jit_basic_block_label(block_else))) { \ + jit_set_last_error(cc, "generate bne insn failed"); \ + goto fail; \ + } \ + } while (0) + +#define SET_BUILDER_POS(basic_block) \ + do { \ + cc->cur_basic_block = basic_block; \ + } while (0) + +#define SET_BB_BEGIN_BCIP(basic_block, bcip) \ + do { \ + *(jit_annl_begin_bcip(cc, jit_basic_block_label(basic_block))) = bcip; \ + } while (0) + +#define SET_BB_END_BCIP(basic_block, bcip) \ + do { \ + *(jit_annl_end_bcip(cc, jit_basic_block_label(basic_block))) = bcip; \ + } while (0) + +static JitBlock * +get_target_block(JitCompContext *cc, uint32 br_depth) +{ + uint32 i = br_depth; + JitBlock *block = jit_block_stack_top(&cc->block_stack); + + while (i-- > 0 && block) { + block = block->prev; + } + + if (!block) { + jit_set_last_error(cc, "WASM block stack underflow"); + return NULL; + } + return block; +} + +static bool +load_block_params(JitCompContext *cc, JitBlock *block) +{ + JitFrame *jit_frame = cc->jit_frame; + uint32 offset, i; + JitReg value = 0; + + /* Clear jit frame's locals and stacks */ + clear_values(jit_frame); + + /* Restore jit frame's sp to block's sp begin */ + jit_frame->sp = block->frame_sp_begin; + + /* Load params to new block */ + offset = (uint32)(jit_frame->sp - jit_frame->lp); + for (i = 0; i < block->param_count; i++) { + switch (block->param_types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + value = gen_load_i32(jit_frame, offset); + offset++; + break; + case VALUE_TYPE_I64: + value = gen_load_i64(jit_frame, offset); + offset += 2; + break; + case VALUE_TYPE_F32: + value = gen_load_f32(jit_frame, offset); + offset++; + break; + case VALUE_TYPE_F64: + value = gen_load_f64(jit_frame, offset); + offset += 2; + break; + default: + bh_assert(0); + break; + } + PUSH(value, block->param_types[i]); + } + + return true; +fail: + return false; +} + +static bool +load_block_results(JitCompContext *cc, JitBlock *block) +{ + JitFrame *jit_frame = cc->jit_frame; + uint32 offset, i; + JitReg value = 0; + + /* Restore jit frame's sp to block's sp begin */ + jit_frame->sp = block->frame_sp_begin; + + /* Load results to new block */ + offset = (uint32)(jit_frame->sp - jit_frame->lp); + for (i = 0; i < block->result_count; i++) { + switch (block->result_types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + value = gen_load_i32(jit_frame, offset); + offset++; + break; + case VALUE_TYPE_I64: + value = gen_load_i64(jit_frame, offset); + offset += 2; + break; + case VALUE_TYPE_F32: + value = gen_load_f32(jit_frame, offset); + offset++; + break; + case VALUE_TYPE_F64: + value = gen_load_f64(jit_frame, offset); + offset += 2; + break; + default: + bh_assert(0); + break; + } + PUSH(value, block->result_types[i]); + } + + return true; +fail: + return false; +} + +static bool +jit_reg_is_i32_const(JitCompContext *cc, JitReg reg, int32 val) +{ + return (jit_reg_kind(reg) == JIT_REG_KIND_I32 && jit_reg_is_const(reg) + && jit_cc_get_const_I32(cc, reg) == val) + ? true + : false; +} + +/** + * get the last two insns: + * CMP cmp_reg, r0, r1 + * SELECTcc r2, cmp_reg, 1, 0 + */ +static void +get_last_cmp_and_selectcc(JitCompContext *cc, JitReg cond, JitInsn **p_insn_cmp, + JitInsn **p_insn_select) +{ + JitInsn *insn = jit_basic_block_last_insn(cc->cur_basic_block); + + if (insn && insn->prev && insn->prev->opcode == JIT_OP_CMP + && insn->opcode >= JIT_OP_SELECTEQ && insn->opcode <= JIT_OP_SELECTLEU + && *jit_insn_opnd(insn, 0) == cond + && jit_reg_is_i32_const(cc, *jit_insn_opnd(insn, 2), 1) + && jit_reg_is_i32_const(cc, *jit_insn_opnd(insn, 3), 0)) { + *p_insn_cmp = insn->prev; + *p_insn_select = insn; + } +} + +static bool +push_jit_block_to_stack_and_pass_params(JitCompContext *cc, JitBlock *block, + JitBasicBlock *basic_block, JitReg cond, + bool merge_cmp_and_if) +{ + JitFrame *jit_frame = cc->jit_frame; + JitValue *value_list_head = NULL, *value_list_end = NULL, *jit_value; + JitInsn *insn; + JitReg value; + uint32 i, param_index, cell_num; + + if (cc->cur_basic_block == basic_block) { + /* Reuse the current basic block and no need to commit values, + we just move param values from current block's value stack to + the new block's value stack */ + for (i = 0; i < block->param_count; i++) { + jit_value = jit_value_stack_pop( + &jit_block_stack_top(&cc->block_stack)->value_stack); + if (!value_list_head) { + value_list_head = value_list_end = jit_value; + jit_value->prev = jit_value->next = NULL; + } + else { + jit_value->prev = NULL; + jit_value->next = value_list_head; + value_list_head->prev = jit_value; + value_list_head = jit_value; + } + } + block->value_stack.value_list_head = value_list_head; + block->value_stack.value_list_end = value_list_end; + + /* Save block's begin frame sp */ + cell_num = wasm_get_cell_num(block->param_types, block->param_count); + block->frame_sp_begin = jit_frame->sp - cell_num; + + /* Push the new block to block stack */ + jit_block_stack_push(&cc->block_stack, block); + + /* Continue to translate current block */ + } + else { + JitInsn *insn_select = NULL, *insn_cmp = NULL; + + if (merge_cmp_and_if) { + get_last_cmp_and_selectcc(cc, cond, &insn_cmp, &insn_select); + } + + /* Commit register values to locals and stacks */ + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + + /* Pop param values from current block's value stack */ + for (i = 0; i < block->param_count; i++) { + param_index = block->param_count - 1 - i; + POP(value, block->param_types[param_index]); + } + + /* Clear frame values */ + clear_values(jit_frame); + /* Save block's begin frame sp */ + block->frame_sp_begin = jit_frame->sp; + + /* Push the new block to block stack */ + jit_block_stack_push(&cc->block_stack, block); + + if (block->label_type == LABEL_TYPE_LOOP) { + BUILD_BR(basic_block); + } + else { + /* IF block with condition br insn */ + if (insn_select && insn_cmp) { + /* Change `CMP + SELECTcc` into `CMP + Bcc` */ + if (!(insn = GEN_INSN(BEQ, cc->cmp_reg, + jit_basic_block_label(basic_block), 0))) { + jit_set_last_error(cc, "generate cond br failed"); + goto fail; + } + insn->opcode = + JIT_OP_BEQ + (insn_select->opcode - JIT_OP_SELECTEQ); + jit_insn_unlink(insn_select); + jit_insn_delete(insn_select); + } + else { + if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)) + || !(insn = + GEN_INSN(BNE, cc->cmp_reg, + jit_basic_block_label(basic_block), 0))) { + jit_set_last_error(cc, "generate cond br failed"); + goto fail; + } + } + + /* Don't create else basic block or end basic block now, just + save its incoming BNE insn, and patch the insn's else label + when the basic block is lazily created */ + if (block->wasm_code_else) { + block->incoming_insn_for_else_bb = insn; + } + else { + if (!jit_block_add_incoming_insn(block, insn, 2)) { + jit_set_last_error(cc, "add incoming insn failed"); + goto fail; + } + } + } + + /* Start to translate the block */ + SET_BUILDER_POS(basic_block); + + /* Push the block parameters */ + if (!load_block_params(cc, block)) { + goto fail; + } + } + return true; +fail: + return false; +} + +static void +copy_block_arities(JitCompContext *cc, JitReg dst_frame_sp, uint8 *dst_types, + uint32 dst_type_count, JitReg *p_first_res_reg) +{ + JitFrame *jit_frame; + uint32 offset_src, offset_dst, i; + JitReg value; + + jit_frame = cc->jit_frame; + offset_src = (uint32)(jit_frame->sp - jit_frame->lp) + - wasm_get_cell_num(dst_types, dst_type_count); + offset_dst = 0; + + /* pop values from stack and store to dest frame */ + for (i = 0; i < dst_type_count; i++) { + switch (dst_types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + value = gen_load_i32(jit_frame, offset_src); + if (i == 0 && p_first_res_reg) + *p_first_res_reg = value; + else + GEN_INSN(STI32, value, dst_frame_sp, + NEW_CONST(I32, offset_dst * 4)); + offset_src++; + offset_dst++; + break; + case VALUE_TYPE_I64: + value = gen_load_i64(jit_frame, offset_src); + if (i == 0 && p_first_res_reg) + *p_first_res_reg = value; + else + GEN_INSN(STI64, value, dst_frame_sp, + NEW_CONST(I32, offset_dst * 4)); + offset_src += 2; + offset_dst += 2; + break; + case VALUE_TYPE_F32: + value = gen_load_f32(jit_frame, offset_src); + if (i == 0 && p_first_res_reg) + *p_first_res_reg = value; + else + GEN_INSN(STF32, value, dst_frame_sp, + NEW_CONST(I32, offset_dst * 4)); + offset_src++; + offset_dst++; + break; + case VALUE_TYPE_F64: + value = gen_load_f64(jit_frame, offset_src); + if (i == 0 && p_first_res_reg) + *p_first_res_reg = value; + else + GEN_INSN(STF64, value, dst_frame_sp, + NEW_CONST(I32, offset_dst * 4)); + offset_src += 2; + offset_dst += 2; + break; + default: + bh_assert(0); + break; + } + } +} + +static bool +handle_func_return(JitCompContext *cc, JitBlock *block) +{ + JitReg prev_frame, prev_frame_sp; + JitReg ret_reg = 0; +#if WASM_ENABLE_PERF_PROFILING != 0 + JitReg func_inst = jit_cc_new_reg_ptr(cc); + JitReg time_start = jit_cc_new_reg_I64(cc); + JitReg time_end = jit_cc_new_reg_I64(cc); + JitReg cur_exec_time = jit_cc_new_reg_I64(cc); + JitReg total_exec_time = jit_cc_new_reg_I64(cc); + JitReg total_exec_cnt = jit_cc_new_reg_I32(cc); +#endif + +#if WASM_ENABLE_PERF_PROFILING != 0 + /* time_end = os_time_thread_cputime_us() */ + if (!jit_emit_callnative(cc, os_time_thread_cputime_us, time_end, NULL, + 0)) { + return false; + } + /* time_start = cur_frame->time_started */ + GEN_INSN(LDI64, time_start, cc->fp_reg, + NEW_CONST(I32, offsetof(WASMInterpFrame, time_started))); + /* cur_exec_time = time_end - time_start */ + GEN_INSN(SUB, cur_exec_time, time_end, time_start); + /* func_inst = cur_frame->function */ + GEN_INSN(LDPTR, func_inst, cc->fp_reg, + NEW_CONST(I32, offsetof(WASMInterpFrame, function))); + /* total_exec_time = func_inst->total_exec_time */ + GEN_INSN(LDI64, total_exec_time, func_inst, + NEW_CONST(I32, offsetof(WASMFunctionInstance, total_exec_time))); + /* total_exec_time += cur_exec_time */ + GEN_INSN(ADD, total_exec_time, total_exec_time, cur_exec_time); + /* func_inst->total_exec_time = total_exec_time */ + GEN_INSN(STI64, total_exec_time, func_inst, + NEW_CONST(I32, offsetof(WASMFunctionInstance, total_exec_time))); + /* totoal_exec_cnt = func_inst->total_exec_cnt */ + GEN_INSN(LDI32, total_exec_cnt, func_inst, + NEW_CONST(I32, offsetof(WASMFunctionInstance, total_exec_cnt))); + /* total_exec_cnt++ */ + GEN_INSN(ADD, total_exec_cnt, total_exec_cnt, NEW_CONST(I32, 1)); + /* func_inst->total_exec_cnt = total_exec_cnt */ + GEN_INSN(STI32, total_exec_cnt, func_inst, + NEW_CONST(I32, offsetof(WASMFunctionInstance, total_exec_cnt))); +#endif + + prev_frame = jit_cc_new_reg_ptr(cc); + prev_frame_sp = jit_cc_new_reg_ptr(cc); + + /* prev_frame = cur_frame->prev_frame */ + GEN_INSN(LDPTR, prev_frame, cc->fp_reg, + NEW_CONST(I32, offsetof(WASMInterpFrame, prev_frame))); + GEN_INSN(LDPTR, prev_frame_sp, prev_frame, + NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); + + if (block->result_count) { + uint32 cell_num = + wasm_get_cell_num(block->result_types, block->result_count); + + copy_block_arities(cc, prev_frame_sp, block->result_types, + block->result_count, &ret_reg); + /* prev_frame->sp += cell_num */ + GEN_INSN(ADD, prev_frame_sp, prev_frame_sp, + NEW_CONST(PTR, cell_num * 4)); + GEN_INSN(STPTR, prev_frame_sp, prev_frame, + NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); + } + + /* Free stack space of the current frame: + exec_env->wasm_stack.top = cur_frame */ + GEN_INSN(STPTR, cc->fp_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top))); + /* Set the prev_frame as the current frame: + exec_env->cur_frame = prev_frame */ + GEN_INSN(STPTR, prev_frame, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, cur_frame))); + /* fp_reg = prev_frame */ + GEN_INSN(MOV, cc->fp_reg, prev_frame); + /* return 0 */ + GEN_INSN(RETURNBC, NEW_CONST(I32, JIT_INTERP_ACTION_NORMAL), ret_reg, 0); + + return true; +} + +/** + * is_block_polymorphic: whether current block's stack is in polymorphic state, + * if the opcode is one of unreachable/br/br_table/return, stack is marked + * to polymorphic state until the block's 'end' opcode is processed + */ +static bool +handle_op_end(JitCompContext *cc, uint8 **p_frame_ip, bool is_block_polymorphic) +{ + JitFrame *jit_frame = cc->jit_frame; + JitBlock *block, *block_prev; + JitIncomingInsn *incoming_insn; + JitInsn *insn; + + /* Check block stack */ + if (!(block = jit_block_stack_top(&cc->block_stack))) { + jit_set_last_error(cc, "WASM block stack underflow"); + return false; + } + + if (!block->incoming_insns_for_end_bb) { + /* No other basic blocks jumping to this end, no need to + create the end basic block, just continue to translate + the following opcodes */ + if (block->label_type == LABEL_TYPE_FUNCTION) { + if (!handle_func_return(cc, block)) { + return false; + } + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + clear_values(jit_frame); + } + else if (block->result_count > 0) { + JitValue *value_list_head = NULL, *value_list_end = NULL; + JitValue *jit_value; + uint32 i; + + /* No need to change cc->jit_frame, just move result values + from current block's value stack to previous block's + value stack */ + block_prev = block->prev; + + for (i = 0; i < block->result_count; i++) { + jit_value = jit_value_stack_pop(&block->value_stack); + bh_assert(jit_value); + if (!value_list_head) { + value_list_head = value_list_end = jit_value; + jit_value->prev = jit_value->next = NULL; + } + else { + jit_value->prev = NULL; + jit_value->next = value_list_head; + value_list_head->prev = jit_value; + value_list_head = jit_value; + } + } + + if (!block_prev->value_stack.value_list_head) { + block_prev->value_stack.value_list_head = value_list_head; + block_prev->value_stack.value_list_end = value_list_end; + } + else { + /* Link to the end of previous block's value stack */ + block_prev->value_stack.value_list_end->next = value_list_head; + value_list_head->prev = block_prev->value_stack.value_list_end; + block_prev->value_stack.value_list_end = value_list_end; + } + } + + /* Pop block and destroy the block */ + block = jit_block_stack_pop(&cc->block_stack); + jit_block_destroy(block); + return true; + } + else { + /* Commit register values to locals and stacks */ + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + /* Clear frame values */ + clear_values(jit_frame); + + /* Create the end basic block */ + CREATE_BASIC_BLOCK(block->basic_block_end); + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + SET_BB_BEGIN_BCIP(block->basic_block_end, *p_frame_ip); + /* No need to create 'JMP' insn if block is in stack polymorphic + state, as previous br/br_table opcode has created 'JMP' insn + to this end basic block */ + if (!is_block_polymorphic) { + /* Jump to the end basic block */ + BUILD_BR(block->basic_block_end); + } + + /* Patch the INSNs which jump to this basic block */ + incoming_insn = block->incoming_insns_for_end_bb; + while (incoming_insn) { + insn = incoming_insn->insn; + + bh_assert( + insn->opcode == JIT_OP_JMP + || (insn->opcode >= JIT_OP_BEQ && insn->opcode <= JIT_OP_BLEU) + || insn->opcode == JIT_OP_LOOKUPSWITCH); + + if (insn->opcode == JIT_OP_JMP + || (insn->opcode >= JIT_OP_BEQ + && insn->opcode <= JIT_OP_BLEU)) { + *(jit_insn_opnd(insn, incoming_insn->opnd_idx)) = + jit_basic_block_label(block->basic_block_end); + } + else { + /* Patch LOOKUPSWITCH INSN */ + JitOpndLookupSwitch *opnd = jit_insn_opndls(insn); + if (incoming_insn->opnd_idx < opnd->match_pairs_num) { + opnd->match_pairs[incoming_insn->opnd_idx].target = + jit_basic_block_label(block->basic_block_end); + } + else { + opnd->default_target = + jit_basic_block_label(block->basic_block_end); + } + } + + incoming_insn = incoming_insn->next; + } + + SET_BUILDER_POS(block->basic_block_end); + + /* Pop block and load block results */ + block = jit_block_stack_pop(&cc->block_stack); + + if (block->label_type == LABEL_TYPE_FUNCTION) { + if (!handle_func_return(cc, block)) { + jit_block_destroy(block); + goto fail; + } + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + clear_values(jit_frame); + } + else { + if (!load_block_results(cc, block)) { + jit_block_destroy(block); + goto fail; + } + } + + jit_block_destroy(block); + return true; + } + return true; +fail: + return false; +} + +/** + * is_block_polymorphic: whether current block's stack is in polymorphic state, + * if the opcode is one of unreachable/br/br_table/return, stack is marked + * to polymorphic state until the block's 'end' opcode is processed + */ +static bool +handle_op_else(JitCompContext *cc, uint8 **p_frame_ip, + bool is_block_polymorphic) +{ + JitBlock *block = jit_block_stack_top(&cc->block_stack); + JitFrame *jit_frame = cc->jit_frame; + JitInsn *insn; + + /* Check block */ + if (!block) { + jit_set_last_error(cc, "WASM block stack underflow"); + return false; + } + if (block->label_type != LABEL_TYPE_IF) { + jit_set_last_error(cc, "Invalid WASM block type"); + return false; + } + + if (!block->incoming_insn_for_else_bb) { + /* The if branch is handled like OP_BLOCK (cond is const and != 0), + just skip the else branch and handle OP_END */ + *p_frame_ip = block->wasm_code_end + 1; + return handle_op_end(cc, p_frame_ip, false); + } + else { + /* Has else branch and need to translate else branch */ + + /* Commit register values to locals and stacks */ + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + /* Clear frame values */ + clear_values(jit_frame); + + /* No need to create 'JMP' insn if block is in stack polymorphic + state, as previous br/br_table opcode has created 'JMP' insn + to this end basic block */ + if (!is_block_polymorphic) { + /* Jump to end basic block */ + if (!(insn = GEN_INSN(JMP, 0))) { + jit_set_last_error(cc, "generate jmp insn failed"); + return false; + } + if (!jit_block_add_incoming_insn(block, insn, 0)) { + jit_set_last_error(cc, "add incoming insn failed"); + return false; + } + } + + /* Clear value stack, restore param values and + start to translate the else branch. */ + jit_value_stack_destroy(&block->value_stack); + + /* create else basic block */ + CREATE_BASIC_BLOCK(block->basic_block_else); + SET_BB_END_BCIP(block->basic_block_entry, *p_frame_ip - 1); + SET_BB_BEGIN_BCIP(block->basic_block_else, *p_frame_ip); + + /* Patch the insn which conditionly jumps to the else basic block */ + insn = block->incoming_insn_for_else_bb; + *(jit_insn_opnd(insn, 2)) = + jit_basic_block_label(block->basic_block_else); + + SET_BUILDER_POS(block->basic_block_else); + + /* Reload block parameters */ + if (!load_block_params(cc, block)) { + return false; + } + + return true; + } + return true; +fail: + return false; +} + +static bool +handle_next_reachable_block(JitCompContext *cc, uint8 **p_frame_ip) +{ + JitBlock *block = jit_block_stack_top(&cc->block_stack); + + bh_assert(block); + + do { + if (block->label_type == LABEL_TYPE_IF + && block->incoming_insn_for_else_bb + && *p_frame_ip <= block->wasm_code_else) { + /* Else branch hasn't been translated, + start to translate the else branch */ + *p_frame_ip = block->wasm_code_else + 1; + /* Restore jit frame's sp to block's sp begin */ + cc->jit_frame->sp = block->frame_sp_begin; + return handle_op_else(cc, p_frame_ip, true); + } + else if (block->incoming_insns_for_end_bb) { + *p_frame_ip = block->wasm_code_end + 1; + /* Restore jit frame's sp to block's sp end */ + cc->jit_frame->sp = + block->frame_sp_begin + + wasm_get_cell_num(block->result_types, block->result_count); + return handle_op_end(cc, p_frame_ip, true); + } + else { + *p_frame_ip = block->wasm_code_end + 1; + jit_block_stack_pop(&cc->block_stack); + jit_block_destroy(block); + block = jit_block_stack_top(&cc->block_stack); + } + } while (block != NULL); + + return true; +} + +bool +jit_compile_op_block(JitCompContext *cc, uint8 **p_frame_ip, + uint8 *frame_ip_end, uint32 label_type, uint32 param_count, + uint8 *param_types, uint32 result_count, + uint8 *result_types, bool merge_cmp_and_if) +{ + BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE]; + JitBlock *block; + JitReg value; + uint8 *else_addr, *end_addr; + + /* Check block stack */ + if (!jit_block_stack_top(&cc->block_stack)) { + jit_set_last_error(cc, "WASM block stack underflow"); + return false; + } + + memset(block_addr_cache, 0, sizeof(block_addr_cache)); + + /* Get block info */ + if (!(wasm_loader_find_block_addr( + NULL, (BlockAddr *)block_addr_cache, *p_frame_ip, frame_ip_end, + (uint8)label_type, &else_addr, &end_addr))) { + jit_set_last_error(cc, "find block end addr failed"); + return false; + } + + /* Allocate memory */ + if (!(block = jit_calloc(sizeof(JitBlock)))) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + if (param_count && !(block->param_types = jit_calloc(param_count))) { + jit_set_last_error(cc, "allocate memory failed"); + goto fail; + } + if (result_count && !(block->result_types = jit_calloc(result_count))) { + jit_set_last_error(cc, "allocate memory failed"); + goto fail; + } + + /* Initialize block data */ + block->label_type = label_type; + block->param_count = param_count; + if (param_count) { + bh_memcpy_s(block->param_types, param_count, param_types, param_count); + } + block->result_count = result_count; + if (result_count) { + bh_memcpy_s(block->result_types, result_count, result_types, + result_count); + } + block->wasm_code_else = else_addr; + block->wasm_code_end = end_addr; + + if (label_type == LABEL_TYPE_BLOCK) { + /* Push the new jit block to block stack and continue to + translate current basic block */ + if (!push_jit_block_to_stack_and_pass_params( + cc, block, cc->cur_basic_block, 0, false)) + goto fail; + } + else if (label_type == LABEL_TYPE_LOOP) { + CREATE_BASIC_BLOCK(block->basic_block_entry); + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + SET_BB_BEGIN_BCIP(block->basic_block_entry, *p_frame_ip); + /* Push the new jit block to block stack and continue to + translate the new basic block */ + if (!push_jit_block_to_stack_and_pass_params( + cc, block, block->basic_block_entry, 0, false)) + goto fail; + } + else if (label_type == LABEL_TYPE_IF) { + POP_I32(value); + + if (!jit_reg_is_const(value)) { + /* Compare value is not constant, create condition br IR */ + + /* Create entry block */ + CREATE_BASIC_BLOCK(block->basic_block_entry); + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + SET_BB_BEGIN_BCIP(block->basic_block_entry, *p_frame_ip); + + if (!push_jit_block_to_stack_and_pass_params( + cc, block, block->basic_block_entry, value, + merge_cmp_and_if)) + goto fail; + } + else { + if (jit_cc_get_const_I32(cc, value) != 0) { + /* Compare value is not 0, condition is true, else branch of + BASIC_BLOCK if cannot be reached, we treat it same as + LABEL_TYPE_BLOCK and start to translate if branch */ + if (!push_jit_block_to_stack_and_pass_params( + cc, block, cc->cur_basic_block, 0, false)) + goto fail; + } + else { + if (else_addr) { + /* Compare value is not 0, condition is false, if branch of + BASIC_BLOCK if cannot be reached, we treat it same as + LABEL_TYPE_BLOCK and start to translate else branch */ + if (!push_jit_block_to_stack_and_pass_params( + cc, block, cc->cur_basic_block, 0, false)) + goto fail; + *p_frame_ip = else_addr + 1; + } + else { + /* The whole if block cannot be reached, skip it */ + jit_block_destroy(block); + *p_frame_ip = end_addr + 1; + } + } + } + } + else { + jit_set_last_error(cc, "Invalid block type"); + goto fail; + } + + return true; +fail: + /* Only destroy the block if it hasn't been pushed into + the block stack, or if will be destroyed again when + destroying the block stack */ + if (jit_block_stack_top(&cc->block_stack) != block) + jit_block_destroy(block); + return false; +} + +bool +jit_compile_op_else(JitCompContext *cc, uint8 **p_frame_ip) +{ + return handle_op_else(cc, p_frame_ip, false); +} + +bool +jit_compile_op_end(JitCompContext *cc, uint8 **p_frame_ip) +{ + return handle_op_end(cc, p_frame_ip, false); +} + +/* Check whether need to copy arities when jumping from current block + to the dest block */ +static bool +check_copy_arities(const JitBlock *block_dst, JitFrame *jit_frame) +{ + JitValueSlot *frame_sp_src = NULL; + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + frame_sp_src = + jit_frame->sp + - wasm_get_cell_num(block_dst->param_types, block_dst->param_count); + /* There are parameters to copy and the src/dst addr are different */ + return (block_dst->param_count > 0 + && block_dst->frame_sp_begin != frame_sp_src) + ? true + : false; + } + else { + frame_sp_src = jit_frame->sp + - wasm_get_cell_num(block_dst->result_types, + block_dst->result_count); + /* There are results to copy and the src/dst addr are different */ + return (block_dst->result_count > 0 + && block_dst->frame_sp_begin != frame_sp_src) + ? true + : false; + } +} + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +jit_check_suspend_flags(JitCompContext *cc) +{ + JitReg exec_env, suspend_flags, terminate_flag, offset; + JitBasicBlock *terminate_block, *cur_basic_block; + JitFrame *jit_frame = cc->jit_frame; + + cur_basic_block = cc->cur_basic_block; + terminate_block = jit_cc_new_basic_block(cc, 0); + if (!terminate_block) { + return false; + } + + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + exec_env = cc->exec_env_reg; + suspend_flags = jit_cc_new_reg_I32(cc); + terminate_flag = jit_cc_new_reg_I32(cc); + + offset = jit_cc_new_const_I32(cc, offsetof(WASMExecEnv, suspend_flags)); + GEN_INSN(LDI32, suspend_flags, exec_env, offset); + GEN_INSN(AND, terminate_flag, suspend_flags, NEW_CONST(I32, 1)); + + GEN_INSN(CMP, cc->cmp_reg, terminate_flag, NEW_CONST(I32, 0)); + GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(terminate_block), 0); + + cc->cur_basic_block = terminate_block; + GEN_INSN(RETURN, NEW_CONST(I32, 0)); + + cc->cur_basic_block = cur_basic_block; + + return true; +} + +#endif + +static bool +handle_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip) +{ + JitFrame *jit_frame; + JitBlock *block_dst, *block; + JitReg frame_sp_dst; + JitInsn *insn; + bool copy_arities; + uint32 offset; + + /* Check block stack */ + if (!(block = jit_block_stack_top(&cc->block_stack))) { + jit_set_last_error(cc, "WASM block stack underflow"); + return false; + } + + if (!(block_dst = get_target_block(cc, br_depth))) { + return false; + } + + jit_frame = cc->jit_frame; + + /* Only opy parameters or results when their count > 0 and + the src/dst addr are different */ + copy_arities = check_copy_arities(block_dst, jit_frame); + + if (copy_arities) { + frame_sp_dst = jit_cc_new_reg_ptr(cc); + offset = offsetof(WASMInterpFrame, lp) + + (block_dst->frame_sp_begin - jit_frame->lp) * 4; + GEN_INSN(ADD, frame_sp_dst, cc->fp_reg, NEW_CONST(PTR, offset)); + + /* No need to commit results as they will be copied to dest block */ + gen_commit_values(jit_frame, jit_frame->lp, block->frame_sp_begin); + } + else { + /* Commit all including results as they won't be copied */ + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + } + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + if (copy_arities) { + /* Dest block is Loop block, copy loop parameters */ + copy_block_arities(cc, frame_sp_dst, block_dst->param_types, + block_dst->param_count, NULL); + } + + clear_values(jit_frame); + + /* Jump to the begin basic block */ + BUILD_BR(block_dst->basic_block_entry); + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + } + else { + if (copy_arities) { + /* Dest block is Block/If/Function block, copy block results */ + copy_block_arities(cc, frame_sp_dst, block_dst->result_types, + block_dst->result_count, NULL); + } + + clear_values(jit_frame); + + /* Jump to the end basic block */ + if (!(insn = GEN_INSN(JMP, 0))) { + jit_set_last_error(cc, "generate jmp insn failed"); + goto fail; + } + if (!jit_block_add_incoming_insn(block_dst, insn, 0)) { + jit_set_last_error(cc, "add incoming insn failed"); + goto fail; + } + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + } + + return true; +fail: + return false; +} + +bool +jit_compile_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip) +{ + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + return false; +#endif + + return handle_op_br(cc, br_depth, p_frame_ip) + && handle_next_reachable_block(cc, p_frame_ip); +} + +static JitFrame * +jit_frame_clone(const JitFrame *jit_frame) +{ + JitFrame *jit_frame_cloned; + uint32 max_locals = jit_frame->max_locals; + uint32 max_stacks = jit_frame->max_stacks; + uint32 total_size; + + total_size = (uint32)(offsetof(JitFrame, lp) + + sizeof(*jit_frame->lp) * (max_locals + max_stacks)); + + jit_frame_cloned = jit_calloc(total_size); + if (jit_frame_cloned) { + bh_memcpy_s(jit_frame_cloned, total_size, jit_frame, total_size); + jit_frame_cloned->sp = + jit_frame_cloned->lp + (jit_frame->sp - jit_frame->lp); + } + + return jit_frame_cloned; +} + +static void +jit_frame_copy(JitFrame *jit_frame_dst, const JitFrame *jit_frame_src) +{ + uint32 max_locals = jit_frame_src->max_locals; + uint32 max_stacks = jit_frame_src->max_stacks; + uint32 total_size; + + total_size = + (uint32)(offsetof(JitFrame, lp) + + sizeof(*jit_frame_src->lp) * (max_locals + max_stacks)); + bh_memcpy_s(jit_frame_dst, total_size, jit_frame_src, total_size); + jit_frame_dst->sp = + jit_frame_dst->lp + (jit_frame_src->sp - jit_frame_src->lp); +} + +bool +jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, + bool merge_cmp_and_br_if, uint8 **p_frame_ip) +{ + JitFrame *jit_frame, *jit_frame_cloned; + JitBlock *block_dst; + JitReg cond; + JitBasicBlock *cur_basic_block, *if_basic_block = NULL; + JitInsn *insn, *insn_select = NULL, *insn_cmp = NULL; + bool copy_arities; + + if (!(block_dst = get_target_block(cc, br_depth))) { + return false; + } + + /* append IF to current basic block */ + POP_I32(cond); + + if (merge_cmp_and_br_if) { + get_last_cmp_and_selectcc(cc, cond, &insn_cmp, &insn_select); + } + + jit_frame = cc->jit_frame; + cur_basic_block = cc->cur_basic_block; + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + + if (!(insn_select && insn_cmp)) { + if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0))) { + jit_set_last_error(cc, "generate cmp insn failed"); + goto fail; + } + } + + /* Only copy parameters or results when their count > 0 and + the src/dst addr are different */ + copy_arities = check_copy_arities(block_dst, jit_frame); + + if (!copy_arities) { + if (block_dst->label_type == LABEL_TYPE_LOOP) { + if (!(insn = GEN_INSN( + BNE, cc->cmp_reg, + jit_basic_block_label(block_dst->basic_block_entry), + 0))) { + jit_set_last_error(cc, "generate bne insn failed"); + goto fail; + } + } + else { + if (!(insn = GEN_INSN(BNE, cc->cmp_reg, 0, 0))) { + jit_set_last_error(cc, "generate bne insn failed"); + goto fail; + } + if (!jit_block_add_incoming_insn(block_dst, insn, 1)) { + jit_set_last_error(cc, "add incoming insn failed"); + goto fail; + } + } + if (insn_select && insn_cmp) { + /* Change `CMP + SELECTcc` into `CMP + Bcc` */ + insn->opcode = JIT_OP_BEQ + (insn_select->opcode - JIT_OP_SELECTEQ); + jit_insn_unlink(insn_select); + jit_insn_delete(insn_select); + } + return true; + } + + CREATE_BASIC_BLOCK(if_basic_block); + if (!(insn = GEN_INSN(BNE, cc->cmp_reg, + jit_basic_block_label(if_basic_block), 0))) { + jit_set_last_error(cc, "generate bne insn failed"); + goto fail; + } + if (insn_select && insn_cmp) { + /* Change `CMP + SELECTcc` into `CMP + Bcc` */ + insn->opcode = JIT_OP_BEQ + (insn_select->opcode - JIT_OP_SELECTEQ); + jit_insn_unlink(insn_select); + jit_insn_delete(insn_select); + } + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + return false; +#endif + + SET_BUILDER_POS(if_basic_block); + SET_BB_BEGIN_BCIP(if_basic_block, *p_frame_ip - 1); + + /* Clone current jit frame to a new jit fame */ + if (!(jit_frame_cloned = jit_frame_clone(jit_frame))) { + jit_set_last_error(cc, "allocate memory failed"); + goto fail; + } + + /* Clear current jit frame so that the registers + in the new basic block will be loaded again */ + clear_values(jit_frame); + if (!handle_op_br(cc, br_depth, p_frame_ip)) { + jit_free(jit_frame_cloned); + goto fail; + } + + /* Restore the jit frame so that the registers can + be used again in current basic block */ + jit_frame_copy(jit_frame, jit_frame_cloned); + jit_free(jit_frame_cloned); + + /* Continue processing opcodes after BR_IF */ + SET_BUILDER_POS(cur_basic_block); + return true; +fail: + return false; +} + +bool +jit_compile_op_br_table(JitCompContext *cc, uint32 *br_depths, uint32 br_count, + uint8 **p_frame_ip) +{ + JitBasicBlock *cur_basic_block; + JitReg value; + JitInsn *insn; + uint32 i = 0; + JitOpndLookupSwitch *opnd = NULL; + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + return false; +#endif + + cur_basic_block = cc->cur_basic_block; + + POP_I32(value); + + /* append LOOKUPSWITCH to current basic block */ + gen_commit_values(cc->jit_frame, cc->jit_frame->lp, cc->jit_frame->sp); + /* Clear frame values */ + clear_values(cc->jit_frame); + SET_BB_END_BCIP(cur_basic_block, *p_frame_ip - 1); + + /* prepare basic blocks for br */ + insn = GEN_INSN(LOOKUPSWITCH, value, br_count); + if (NULL == insn) { + jit_set_last_error(cc, "generate insn LOOKUPSWITCH failed"); + goto fail; + } + + for (i = 0, opnd = jit_insn_opndls(insn); i < br_count + 1; i++) { + JitBasicBlock *basic_block = NULL; + JitBlock *block_dst; + bool copy_arities; + + if (!(block_dst = get_target_block(cc, br_depths[i]))) { + goto fail; + } + + /* Only opy parameters or results when their count > 0 and + the src/dst addr are different */ + copy_arities = check_copy_arities(block_dst, cc->jit_frame); + + if (!copy_arities) { + /* No need to create new basic block, direclty jump to + the existing basic block when no need to copy arities */ + if (i == br_count) { + if (block_dst->label_type == LABEL_TYPE_LOOP) { + opnd->default_target = + jit_basic_block_label(block_dst->basic_block_entry); + } + else { + bh_assert(!block_dst->basic_block_end); + if (!jit_block_add_incoming_insn(block_dst, insn, i)) { + jit_set_last_error(cc, "add incoming insn failed"); + goto fail; + } + } + } + else { + opnd->match_pairs[i].value = i; + if (block_dst->label_type == LABEL_TYPE_LOOP) { + opnd->match_pairs[i].target = + jit_basic_block_label(block_dst->basic_block_entry); + } + else { + bh_assert(!block_dst->basic_block_end); + if (!jit_block_add_incoming_insn(block_dst, insn, i)) { + jit_set_last_error(cc, "add incoming insn failed"); + goto fail; + } + } + } + continue; + } + + /* Create new basic block when need to copy arities */ + CREATE_BASIC_BLOCK(basic_block); + SET_BB_BEGIN_BCIP(basic_block, *p_frame_ip - 1); + + if (i == br_count) { + opnd->default_target = jit_basic_block_label(basic_block); + } + else { + opnd->match_pairs[i].value = i; + opnd->match_pairs[i].target = jit_basic_block_label(basic_block); + } + + SET_BUILDER_POS(basic_block); + + if (!handle_op_br(cc, br_depths[i], p_frame_ip)) + goto fail; + } + + /* Search next available block to handle */ + return handle_next_reachable_block(cc, p_frame_ip); +fail: + return false; +} + +bool +jit_compile_op_return(JitCompContext *cc, uint8 **p_frame_ip) +{ + JitBlock *block_func = cc->block_stack.block_list_head; + + bh_assert(block_func); + + if (!handle_func_return(cc, block_func)) { + return false; + } + SET_BB_END_BCIP(cc->cur_basic_block, *p_frame_ip - 1); + clear_values(cc->jit_frame); + + return handle_next_reachable_block(cc, p_frame_ip); +} + +bool +jit_compile_op_unreachable(JitCompContext *cc, uint8 **p_frame_ip) +{ + if (!jit_emit_exception(cc, EXCE_UNREACHABLE, JIT_OP_JMP, 0, NULL)) + return false; + + return handle_next_reachable_block(cc, p_frame_ip); +} + +bool +jit_handle_next_reachable_block(JitCompContext *cc, uint8 **p_frame_ip) +{ + return handle_next_reachable_block(cc, p_frame_ip); +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_control.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_control.h new file mode 100644 index 0000000..e1bc09a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_control.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_CONTROL_H_ +#define _JIT_EMIT_CONTROL_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_block(JitCompContext *cc, uint8 **p_frame_ip, + uint8 *frame_ip_end, uint32 label_type, uint32 param_count, + uint8 *param_types, uint32 result_count, + uint8 *result_types, bool merge_cmp_and_if); + +bool +jit_compile_op_else(JitCompContext *cc, uint8 **p_frame_ip); + +bool +jit_compile_op_end(JitCompContext *cc, uint8 **p_frame_ip); + +bool +jit_compile_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip); + +bool +jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, + bool merge_cmp_and_br_if, uint8 **p_frame_ip); + +bool +jit_compile_op_br_table(JitCompContext *cc, uint32 *br_depths, uint32 br_count, + uint8 **p_frame_ip); + +bool +jit_compile_op_return(JitCompContext *cc, uint8 **p_frame_ip); + +bool +jit_compile_op_unreachable(JitCompContext *cc, uint8 **p_frame_ip); + +bool +jit_handle_next_reachable_block(JitCompContext *cc, uint8 **p_frame_ip); + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +jit_check_suspend_flags(JitCompContext *cc); +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_CONTROL_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_conversion.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_conversion.c new file mode 100644 index 0000000..89d84f1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_conversion.c @@ -0,0 +1,709 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_conversion.h" +#include "jit_emit_exception.h" +#include "jit_emit_function.h" +#include "../jit_codegen.h" +#include "../jit_frontend.h" + +#define F32_I32_S_MIN (-2147483904.0f) +#define F32_I32_S_MAX (2147483648.0f) +#define F32_I32_U_MIN (-1.0f) +#define F32_I32_U_MAX (4294967296.0f) +#define F32_I64_S_MIN (-9223373136366403584.0f) +#define F32_I64_S_MAX (9223372036854775808.0f) +#define F32_I64_U_MIN (-1.0f) +#define F32_I64_U_MAX (18446744073709551616.0f) + +#define F64_I32_S_MIN (-2147483649.0) +#define F64_I32_S_MAX (2147483648.0) +#define F64_I32_U_MIN (-1.0) +#define F64_I32_U_MAX (4294967296.0) +#define F64_I64_S_MIN (-9223372036854777856.0) +#define F64_I64_S_MAX (9223372036854775808.0) +#define F64_I64_U_MIN (-1.0) +#define F64_I64_U_MAX (18446744073709551616.0) + +#define FP_TO_INT(f_ty, i_ty, f_nm, i_nm) \ + static i_ty i_nm##_trunc_##f_nm(f_ty fp) + +#define INT_TO_FP(i_ty, f_ty, i_nm, f_nm) \ + static f_ty f_nm##_convert_##i_nm(i_ty i) + +#define FP_TO_INT_SAT(f_ty, i_ty, f_nm, i_nm) \ + static i_ty i_nm##_trunc_##f_nm##_sat(f_ty fp) + +static int +local_isnan(double x) +{ + return isnan(x); +} + +static int +local_isnanf(float x) +{ + return isnan(x); +} + +#define RETURN_IF_NANF(fp) \ + if (local_isnanf(fp)) { \ + return 0; \ + } + +#define RETURN_IF_NAN(fp) \ + if (local_isnan(fp)) { \ + return 0; \ + } + +#define RETURN_IF_INF(fp, i_min, i_max) \ + if (isinf(fp)) { \ + return fp < 0 ? i_min : i_max; \ + } + +#define RETURN_IF_MIN(fp, f_min, i_min) \ + if (fp <= f_min) { \ + return i_min; \ + } + +#define RETURN_IF_MAX(fp, f_max, i_max) \ + if (fp >= f_max) { \ + return i_max; \ + } + +FP_TO_INT_SAT(float, int32, f32, i32) +{ + RETURN_IF_NANF(fp) + RETURN_IF_INF(fp, INT32_MIN, INT32_MAX) + RETURN_IF_MIN(fp, F32_I32_S_MIN, INT32_MIN) + RETURN_IF_MAX(fp, F32_I32_S_MAX, INT32_MAX) + return (int32)fp; +} + +FP_TO_INT_SAT(float, uint32, f32, u32) +{ + RETURN_IF_NANF(fp) + RETURN_IF_INF(fp, 0, UINT32_MAX) + RETURN_IF_MIN(fp, F32_I32_U_MIN, 0) + RETURN_IF_MAX(fp, F32_I32_U_MAX, UINT32_MAX) + return (uint32)fp; +} + +FP_TO_INT_SAT(double, int32, f64, i32) +{ + RETURN_IF_NAN(fp) + RETURN_IF_INF(fp, INT32_MIN, INT32_MAX) + RETURN_IF_MIN(fp, F64_I32_S_MIN, INT32_MIN) + RETURN_IF_MAX(fp, F64_I32_S_MAX, INT32_MAX) + return (int32)fp; +} + +FP_TO_INT_SAT(double, uint32, f64, u32) +{ + RETURN_IF_NAN(fp) + RETURN_IF_INF(fp, 0, UINT32_MAX) + RETURN_IF_MIN(fp, F64_I32_U_MIN, 0) + RETURN_IF_MAX(fp, F64_I32_U_MAX, UINT32_MAX) + return (uint32)fp; +} + +FP_TO_INT_SAT(float, int64, f32, i64) +{ + RETURN_IF_NANF(fp) + RETURN_IF_INF(fp, INT64_MIN, INT64_MAX) + RETURN_IF_MIN(fp, F32_I64_S_MIN, INT64_MIN) + RETURN_IF_MAX(fp, F32_I64_S_MAX, INT64_MAX) + return (int64)fp; +} + +FP_TO_INT(float, uint64, f32, u64) +{ + return (uint64)fp; +} + +FP_TO_INT_SAT(float, uint64, f32, u64) +{ + RETURN_IF_NANF(fp) + RETURN_IF_INF(fp, 0, UINT64_MAX) + RETURN_IF_MIN(fp, F32_I64_U_MIN, 0) + RETURN_IF_MAX(fp, F32_I64_U_MAX, UINT64_MAX) + return (uint64)fp; +} + +FP_TO_INT_SAT(double, int64, f64, i64) +{ + RETURN_IF_NANF(fp) + RETURN_IF_INF(fp, INT64_MIN, INT64_MAX) + RETURN_IF_MIN(fp, F64_I64_S_MIN, INT64_MIN) + RETURN_IF_MAX(fp, F64_I64_S_MAX, INT64_MAX) + return (int64)fp; +} + +FP_TO_INT(double, uint64, f64, u64) +{ + return (uint64)fp; +} + +FP_TO_INT_SAT(double, uint64, f64, u64) +{ + RETURN_IF_NANF(fp) + RETURN_IF_INF(fp, 0, UINT64_MAX) + RETURN_IF_MIN(fp, F64_I64_U_MIN, 0) + RETURN_IF_MAX(fp, F64_I64_U_MAX, UINT64_MAX) + return (uint64)fp; +} + +INT_TO_FP(uint64, float, u64, f32) +{ + return (float)i; +} + +INT_TO_FP(uint64, double, u64, f64) +{ + return (double)i; +} + +bool +jit_compile_op_i32_wrap_i64(JitCompContext *cc) +{ + JitReg num, res; + + POP_I64(num); + + res = jit_cc_new_reg_I32(cc); + GEN_INSN(I64TOI32, res, num); + + PUSH_I32(res); + + return true; +fail: + return false; +} + +static bool +jit_compile_check_value_range(JitCompContext *cc, JitReg value, JitReg min_fp, + JitReg max_fp) +{ + JitReg nan_ret = jit_cc_new_reg_I32(cc); + JitRegKind kind = jit_reg_kind(value); + bool emit_ret = false; + + bh_assert(JIT_REG_KIND_F32 == kind || JIT_REG_KIND_F64 == kind); + + if (JIT_REG_KIND_F32 == kind && jit_reg_is_const(value)) { + /* value is an f32 const */ + float value_f32_const = jit_cc_get_const_F32(cc, value); + float min_fp_f32_const = jit_cc_get_const_F32(cc, min_fp); + float max_fp_f32_const = jit_cc_get_const_F32(cc, max_fp); + + if (isnan(value_f32_const)) { + /* throw exception if value is nan */ + if (!jit_emit_exception(cc, EXCE_INVALID_CONVERSION_TO_INTEGER, + JIT_OP_JMP, 0, NULL)) + goto fail; + } + + if (value_f32_const <= min_fp_f32_const + || value_f32_const >= max_fp_f32_const) { + /* throw exception if value is out of range */ + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_JMP, 0, + NULL)) + goto fail; + } + + /* value is in range, do nothing */ + return true; + } + else if (JIT_REG_KIND_F64 == kind && jit_reg_is_const(value)) { + /* value is an f64 const */ + double value_f64_const = jit_cc_get_const_F64(cc, value); + double min_fp_f64_const = jit_cc_get_const_F64(cc, min_fp); + double max_fp_f64_const = jit_cc_get_const_F64(cc, max_fp); + + if (isnan(value_f64_const)) { + /* throw exception if value is nan */ + if (!jit_emit_exception(cc, EXCE_INVALID_CONVERSION_TO_INTEGER, + JIT_OP_JMP, 0, NULL)) + goto fail; + } + + if (value_f64_const <= min_fp_f64_const + || value_f64_const >= max_fp_f64_const) { + /* throw exception if value is out of range */ + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_JMP, 0, + NULL)) + goto fail; + } + + /* value is in range, do nothing */ + return true; + } + + /* If value is NaN, throw exception */ + if (JIT_REG_KIND_F32 == kind) + emit_ret = jit_emit_callnative(cc, local_isnanf, nan_ret, &value, 1); + else + emit_ret = jit_emit_callnative(cc, local_isnan, nan_ret, &value, 1); + if (!emit_ret) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, nan_ret, NEW_CONST(I32, 1)); + if (!jit_emit_exception(cc, EXCE_INVALID_CONVERSION_TO_INTEGER, JIT_OP_BEQ, + cc->cmp_reg, NULL)) + goto fail; + + /* If value is out of integer range, throw exception */ + GEN_INSN(CMP, cc->cmp_reg, min_fp, value); + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, cc->cmp_reg, + NULL)) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, value, max_fp); + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_trunc_f32(JitCompContext *cc, bool sign, bool sat) +{ + JitReg value, res; + + POP_F32(value); + + res = jit_cc_new_reg_I32(cc); + if (!sat) { + JitReg min_fp = NEW_CONST(F32, sign ? F32_I32_S_MIN : F32_I32_U_MIN); + JitReg max_fp = NEW_CONST(F32, sign ? F32_I32_S_MAX : F32_I32_U_MAX); + + if (!jit_compile_check_value_range(cc, value, min_fp, max_fp)) + goto fail; + + if (sign) + GEN_INSN(F32TOI32, res, value); + else + GEN_INSN(F32TOU32, res, value); + } + else { + if (!jit_emit_callnative(cc, + sign ? (void *)i32_trunc_f32_sat + : (void *)u32_trunc_f32_sat, + res, &value, 1)) + goto fail; + } + + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_trunc_f64(JitCompContext *cc, bool sign, bool sat) +{ + JitReg value, res; + + POP_F64(value); + + res = jit_cc_new_reg_I32(cc); + if (!sat) { + JitReg min_fp = NEW_CONST(F64, sign ? F64_I32_S_MIN : F64_I32_U_MIN); + JitReg max_fp = NEW_CONST(F64, sign ? F64_I32_S_MAX : F64_I32_U_MAX); + + if (!jit_compile_check_value_range(cc, value, min_fp, max_fp)) + goto fail; + + if (sign) + GEN_INSN(F64TOI32, res, value); + else + GEN_INSN(F64TOU32, res, value); + } + else { + if (!jit_emit_callnative(cc, + sign ? (void *)i32_trunc_f64_sat + : (void *)u32_trunc_f64_sat, + res, &value, 1)) + goto fail; + } + + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_extend_i32(JitCompContext *cc, bool sign) +{ + JitReg num, res; + + POP_I32(num); + + res = jit_cc_new_reg_I64(cc); + if (sign) + GEN_INSN(I32TOI64, res, num); + else + GEN_INSN(U32TOI64, res, num); + + PUSH_I64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_extend_i64(JitCompContext *cc, int8 bitwidth) +{ + JitReg value, tmp, res; + + POP_I64(value); + + tmp = jit_cc_new_reg_I32(cc); + res = jit_cc_new_reg_I64(cc); + + switch (bitwidth) { + case 8: + { + GEN_INSN(I64TOI8, tmp, value); + GEN_INSN(I8TOI64, res, tmp); + break; + } + case 16: + { + GEN_INSN(I64TOI16, tmp, value); + GEN_INSN(I16TOI64, res, tmp); + break; + } + case 32: + { + GEN_INSN(I64TOI32, tmp, value); + GEN_INSN(I32TOI64, res, tmp); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + PUSH_I64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_extend_i32(JitCompContext *cc, int8 bitwidth) +{ + JitReg value, tmp, res; + + POP_I32(value); + + tmp = jit_cc_new_reg_I32(cc); + res = jit_cc_new_reg_I32(cc); + + switch (bitwidth) { + case 8: + { + GEN_INSN(I32TOI8, tmp, value); + GEN_INSN(I8TOI32, res, tmp); + break; + } + case 16: + { + GEN_INSN(I32TOI16, tmp, value); + GEN_INSN(I16TOI32, res, tmp); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_trunc_f32(JitCompContext *cc, bool sign, bool sat) +{ + JitReg value, res; + + POP_F32(value); + + res = jit_cc_new_reg_I64(cc); + if (!sat) { + JitReg min_fp = NEW_CONST(F32, sign ? F32_I64_S_MIN : F32_I64_U_MIN); + JitReg max_fp = NEW_CONST(F32, sign ? F32_I64_S_MAX : F32_I64_U_MAX); + + if (!jit_compile_check_value_range(cc, value, min_fp, max_fp)) + goto fail; + + if (sign) { + GEN_INSN(F32TOI64, res, value); + } + else { + if (!jit_emit_callnative(cc, u64_trunc_f32, res, &value, 1)) + goto fail; + } + } + else { + if (!jit_emit_callnative(cc, + sign ? (void *)i64_trunc_f32_sat + : (void *)u64_trunc_f32_sat, + res, &value, 1)) + goto fail; + } + + PUSH_I64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_trunc_f64(JitCompContext *cc, bool sign, bool sat) +{ + JitReg value, res; + + POP_F64(value); + + res = jit_cc_new_reg_I64(cc); + if (!sat) { + JitReg min_fp = NEW_CONST(F64, sign ? F64_I64_S_MIN : F64_I64_U_MIN); + JitReg max_fp = NEW_CONST(F64, sign ? F64_I64_S_MAX : F64_I64_U_MAX); + + if (!jit_compile_check_value_range(cc, value, min_fp, max_fp)) + goto fail; + + if (sign) { + GEN_INSN(F64TOI64, res, value); + } + else { + if (!jit_emit_callnative(cc, u64_trunc_f64, res, &value, 1)) + goto fail; + } + } + else { + if (!jit_emit_callnative(cc, + sign ? (void *)i64_trunc_f64_sat + : (void *)u64_trunc_f64_sat, + res, &value, 1)) + goto fail; + } + + PUSH_I64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_convert_i32(JitCompContext *cc, bool sign) +{ + JitReg value, res; + + POP_I32(value); + + res = jit_cc_new_reg_F32(cc); + if (sign) { + GEN_INSN(I32TOF32, res, value); + } + else { + GEN_INSN(U32TOF32, res, value); + } + + PUSH_F32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_convert_i64(JitCompContext *cc, bool sign) +{ + JitReg value, res; + + POP_I64(value); + + res = jit_cc_new_reg_F32(cc); + if (sign) { + GEN_INSN(I64TOF32, res, value); + } + else { + if (!jit_emit_callnative(cc, f32_convert_u64, res, &value, 1)) { + goto fail; + } + } + + PUSH_F32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_demote_f64(JitCompContext *cc) +{ + JitReg value, res; + + POP_F64(value); + + res = jit_cc_new_reg_F32(cc); + GEN_INSN(F64TOF32, res, value); + + PUSH_F32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_convert_i32(JitCompContext *cc, bool sign) +{ + JitReg value, res; + + POP_I32(value); + + res = jit_cc_new_reg_F64(cc); + if (sign) + GEN_INSN(I32TOF64, res, value); + else + GEN_INSN(U32TOF64, res, value); + + PUSH_F64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_convert_i64(JitCompContext *cc, bool sign) +{ + JitReg value, res; + + POP_I64(value); + + res = jit_cc_new_reg_F64(cc); + if (sign) { + GEN_INSN(I64TOF64, res, value); + } + else { + if (!jit_emit_callnative(cc, f64_convert_u64, res, &value, 1)) { + goto fail; + } + } + + PUSH_F64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_promote_f32(JitCompContext *cc) +{ + JitReg value, res; + + POP_F32(value); + + res = jit_cc_new_reg_F64(cc); + GEN_INSN(F32TOF64, res, value); + + PUSH_F64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_reinterpret_f64(JitCompContext *cc) +{ + JitReg value, res; + + POP_F64(value); + + res = jit_cc_new_reg_I64(cc); + GEN_INSN(F64CASTI64, res, value); + + PUSH_I64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_reinterpret_f32(JitCompContext *cc) +{ + JitReg value, res; + + POP_F32(value); + + res = jit_cc_new_reg_I32(cc); + GEN_INSN(F32CASTI32, res, value); + + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_reinterpret_i64(JitCompContext *cc) +{ + JitReg value, res; + + POP_I64(value); + + res = jit_cc_new_reg_F64(cc); + GEN_INSN(I64CASTF64, res, value); + + PUSH_F64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_reinterpret_i32(JitCompContext *cc) +{ + JitReg value, res; + + POP_I32(value); + + res = jit_cc_new_reg_F32(cc); + GEN_INSN(I32CASTF32, res, value); + + PUSH_F32(res); + + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_conversion.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_conversion.h new file mode 100644 index 0000000..28952fc --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_conversion.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_CONVERSION_H_ +#define _JIT_EMIT_CONVERSION_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_i32_wrap_i64(JitCompContext *cc); + +bool +jit_compile_op_i32_trunc_f32(JitCompContext *cc, bool sign, bool sat); + +bool +jit_compile_op_i32_trunc_f64(JitCompContext *cc, bool sign, bool sat); + +bool +jit_compile_op_i64_extend_i32(JitCompContext *comp_ctx, bool sign); + +bool +jit_compile_op_i64_extend_i64(JitCompContext *comp_ctx, int8 bitwidth); + +bool +jit_compile_op_i32_extend_i32(JitCompContext *comp_ctx, int8 bitwidth); + +bool +jit_compile_op_i64_trunc_f32(JitCompContext *cc, bool sign, bool sat); + +bool +jit_compile_op_i64_trunc_f64(JitCompContext *cc, bool sign, bool sat); + +bool +jit_compile_op_f32_convert_i32(JitCompContext *comp_ctx, bool sign); + +bool +jit_compile_op_f32_convert_i64(JitCompContext *comp_ctx, bool sign); + +bool +jit_compile_op_f32_demote_f64(JitCompContext *comp_ctx); + +bool +jit_compile_op_f64_convert_i32(JitCompContext *comp_ctx, bool sign); + +bool +jit_compile_op_f64_convert_i64(JitCompContext *comp_ctx, bool sign); + +bool +jit_compile_op_f64_promote_f32(JitCompContext *comp_ctx); + +bool +jit_compile_op_i64_reinterpret_f64(JitCompContext *comp_ctx); + +bool +jit_compile_op_i32_reinterpret_f32(JitCompContext *comp_ctx); + +bool +jit_compile_op_f64_reinterpret_i64(JitCompContext *comp_ctx); + +bool +jit_compile_op_f32_reinterpret_i32(JitCompContext *comp_ctx); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_CONVERSION_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_exception.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_exception.c new file mode 100644 index 0000000..2addb5c --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_exception.c @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_exception.h" +#include "../jit_frontend.h" + +bool +jit_emit_exception(JitCompContext *cc, int32 exception_id, uint8 jit_opcode, + JitReg cond_br_if, JitBasicBlock *cond_br_else_block) +{ + JitInsn *insn = NULL; + JitIncomingInsn *incoming_insn; + JitReg else_label; + + bh_assert(exception_id < EXCE_NUM); + + if (jit_opcode >= JIT_OP_BEQ && jit_opcode <= JIT_OP_BLEU) { + bh_assert(cond_br_if == cc->cmp_reg); + else_label = + cond_br_else_block ? jit_basic_block_label(cond_br_else_block) : 0; + switch (jit_opcode) { + case JIT_OP_BEQ: + insn = GEN_INSN(BEQ, cond_br_if, 0, else_label); + break; + case JIT_OP_BNE: + insn = GEN_INSN(BNE, cond_br_if, 0, else_label); + break; + case JIT_OP_BGTS: + insn = GEN_INSN(BGTS, cond_br_if, 0, else_label); + break; + case JIT_OP_BGES: + insn = GEN_INSN(BGES, cond_br_if, 0, else_label); + break; + case JIT_OP_BLTS: + insn = GEN_INSN(BLTS, cond_br_if, 0, else_label); + break; + case JIT_OP_BLES: + insn = GEN_INSN(BLES, cond_br_if, 0, else_label); + break; + case JIT_OP_BGTU: + insn = GEN_INSN(BGTU, cond_br_if, 0, else_label); + break; + case JIT_OP_BGEU: + insn = GEN_INSN(BGEU, cond_br_if, 0, else_label); + break; + case JIT_OP_BLTU: + insn = GEN_INSN(BLTU, cond_br_if, 0, else_label); + break; + case JIT_OP_BLEU: + insn = GEN_INSN(BLEU, cond_br_if, 0, else_label); + break; + } + if (!insn) { + jit_set_last_error(cc, "generate cond br insn failed"); + return false; + } + } + else if (jit_opcode == JIT_OP_JMP) { + insn = GEN_INSN(JMP, 0); + if (!insn) { + jit_set_last_error(cc, "generate jmp insn failed"); + return false; + } + } + + incoming_insn = jit_calloc(sizeof(JitIncomingInsn)); + if (!incoming_insn) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + incoming_insn->insn = insn; + incoming_insn->next = cc->incoming_insns_for_exec_bbs[exception_id]; + cc->incoming_insns_for_exec_bbs[exception_id] = incoming_insn; + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_exception.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_exception.h new file mode 100644 index 0000000..7aa393b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_exception.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_EXCEPTION_H_ +#define _JIT_EMIT_EXCEPTION_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_emit_exception(JitCompContext *cc, int32 exception_id, uint8 jit_opcode, + JitReg cond_br_if, JitBasicBlock *cond_br_else_block); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_EXCEPTION_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_function.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_function.c new file mode 100644 index 0000000..1e41994 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_function.c @@ -0,0 +1,984 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_function.h" +#include "jit_emit_exception.h" +#include "jit_emit_control.h" +#include "../jit_frontend.h" +#include "../jit_codegen.h" +#include "../../interpreter/wasm_runtime.h" + +static bool +emit_callnative(JitCompContext *cc, JitReg native_func_reg, JitReg res, + JitReg *params, uint32 param_count); + +/* Prepare parameters for the function to call */ +static bool +pre_call(JitCompContext *cc, const WASMType *func_type) +{ + JitReg value; + uint32 i, outs_off; + /* Prepare parameters for the function to call */ + outs_off = + cc->total_frame_size + offsetof(WASMInterpFrame, lp) + + wasm_get_cell_num(func_type->types, func_type->param_count) * 4; + + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[func_type->param_count - 1 - i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + POP_I32(value); + outs_off -= 4; + GEN_INSN(STI32, value, cc->fp_reg, NEW_CONST(I32, outs_off)); + break; + case VALUE_TYPE_I64: + POP_I64(value); + outs_off -= 8; + GEN_INSN(STI64, value, cc->fp_reg, NEW_CONST(I32, outs_off)); + break; + case VALUE_TYPE_F32: + POP_F32(value); + outs_off -= 4; + GEN_INSN(STF32, value, cc->fp_reg, NEW_CONST(I32, outs_off)); + break; + case VALUE_TYPE_F64: + POP_F64(value); + outs_off -= 8; + GEN_INSN(STF64, value, cc->fp_reg, NEW_CONST(I32, outs_off)); + break; + default: + bh_assert(0); + goto fail; + } + } + + /* Commit sp as the callee may use it to store the results */ + gen_commit_sp_ip(cc->jit_frame); + + return true; +fail: + return false; +} + +/* Push results */ +static bool +post_return(JitCompContext *cc, const WASMType *func_type, JitReg first_res, + bool update_committed_sp) +{ + uint32 i, n; + JitReg value; + + n = cc->jit_frame->sp - cc->jit_frame->lp; + for (i = 0; i < func_type->result_count; i++) { + switch (func_type->types[func_type->param_count + i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + if (i == 0 && first_res) { + bh_assert(jit_reg_kind(first_res) == JIT_REG_KIND_I32); + value = first_res; + } + else { + value = jit_cc_new_reg_I32(cc); + GEN_INSN(LDI32, value, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + PUSH_I32(value); + n++; + break; + case VALUE_TYPE_I64: + if (i == 0 && first_res) { + bh_assert(jit_reg_kind(first_res) == JIT_REG_KIND_I64); + value = first_res; + } + else { + value = jit_cc_new_reg_I64(cc); + GEN_INSN(LDI64, value, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + PUSH_I64(value); + n += 2; + break; + case VALUE_TYPE_F32: + if (i == 0 && first_res) { + bh_assert(jit_reg_kind(first_res) == JIT_REG_KIND_F32); + value = first_res; + } + else { + value = jit_cc_new_reg_F32(cc); + GEN_INSN(LDF32, value, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + PUSH_F32(value); + n++; + break; + case VALUE_TYPE_F64: + if (i == 0 && first_res) { + bh_assert(jit_reg_kind(first_res) == JIT_REG_KIND_F64); + value = first_res; + } + else { + value = jit_cc_new_reg_F64(cc); + GEN_INSN(LDF64, value, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + PUSH_F64(value); + n += 2; + break; + default: + bh_assert(0); + goto fail; + } + } + + if (update_committed_sp) + /* Update the committed_sp as the callee has updated the frame sp */ + cc->jit_frame->committed_sp = cc->jit_frame->sp; + + return true; +fail: + return false; +} + +static bool +pre_load(JitCompContext *cc, JitReg *argvs, const WASMType *func_type) +{ + JitReg value; + uint32 i; + + /* Prepare parameters for the function to call */ + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[func_type->param_count - 1 - i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + POP_I32(value); + argvs[func_type->param_count - 1 - i] = value; + break; + case VALUE_TYPE_I64: + POP_I64(value); + argvs[func_type->param_count - 1 - i] = value; + break; + case VALUE_TYPE_F32: + POP_F32(value); + argvs[func_type->param_count - 1 - i] = value; + break; + case VALUE_TYPE_F64: + POP_F64(value); + argvs[func_type->param_count - 1 - i] = value; + break; + default: + bh_assert(0); + goto fail; + } + } + + gen_commit_sp_ip(cc->jit_frame); + + return true; +fail: + return false; +} + +static JitReg +create_first_res_reg(JitCompContext *cc, const WASMType *func_type) +{ + if (func_type->result_count) { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + return jit_cc_new_reg_I32(cc); + case VALUE_TYPE_I64: + return jit_cc_new_reg_I64(cc); + case VALUE_TYPE_F32: + return jit_cc_new_reg_F32(cc); + case VALUE_TYPE_F64: + return jit_cc_new_reg_F64(cc); + default: + bh_assert(0); + return 0; + } + } + return 0; +} + +bool +jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) +{ + WASMModule *wasm_module = cc->cur_wasm_module; + WASMFunctionImport *func_import; + WASMFunction *func; + WASMType *func_type; + JitFrame *jit_frame = cc->jit_frame; + JitReg fast_jit_func_ptrs, jitted_code = 0; + JitReg native_func, *argvs = NULL, *argvs1 = NULL, func_params[5]; + JitReg native_addr_ptr, module_inst_reg, ret, res; + uint32 jitted_func_idx, i; + uint64 total_size; + const char *signature = NULL; + /* Whether the argument is a pointer/str argument and + need to call jit_check_app_addr_and_convert */ + bool is_pointer_arg; + bool return_value = false; + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + goto fail; +#endif + + if (func_idx < wasm_module->import_function_count) { + /* The function to call is an import function */ + func_import = &wasm_module->import_functions[func_idx].u.function; + func_type = func_import->func_type; + + /* Call fast_jit_invoke_native in some cases */ + if (!func_import->func_ptr_linked /* import func hasn't been linked */ + || func_import->call_conv_wasm_c_api /* linked by wasm_c_api */ + || func_import->call_conv_raw /* registered as raw mode */ + || func_type->param_count >= 5 /* registered as normal mode, but + jit_emit_callnative only supports + maximum 6 registers now + (include exec_nev) */) { + JitReg arg_regs[3]; + + if (!pre_call(cc, func_type)) { + goto fail; + } + + /* Call fast_jit_invoke_native */ + ret = jit_cc_new_reg_I32(cc); + arg_regs[0] = cc->exec_env_reg; + arg_regs[1] = NEW_CONST(I32, func_idx); + arg_regs[2] = cc->fp_reg; + if (!jit_emit_callnative(cc, fast_jit_invoke_native, ret, arg_regs, + 3)) { + goto fail; + } + + /* Convert the return value from bool to uint32 */ + GEN_INSN(AND, ret, ret, NEW_CONST(I32, 0xFF)); + + /* Check whether there is exception thrown */ + GEN_INSN(CMP, cc->cmp_reg, ret, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, + cc->cmp_reg, NULL)) { + goto fail; + } + + if (!post_return(cc, func_type, 0, true)) { + goto fail; + } + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + goto fail; +#endif + + return true; + } + + /* Import function was registered as normal mode, and its argument count + is no more than 5, we directly call it */ + + signature = func_import->signature; + bh_assert(signature); + + /* Allocate memory for argvs*/ + total_size = sizeof(JitReg) * (uint64)(func_type->param_count); + if (total_size > 0) { + if (total_size >= UINT32_MAX + || !(argvs = jit_malloc((uint32)total_size))) { + goto fail; + } + } + + /* Pop function params from stack and store them into argvs */ + if (!pre_load(cc, argvs, func_type)) { + goto fail; + } + + ret = jit_cc_new_reg_I32(cc); + func_params[0] = module_inst_reg = get_module_inst_reg(jit_frame); + func_params[4] = native_addr_ptr = jit_cc_new_reg_ptr(cc); + GEN_INSN(ADD, native_addr_ptr, cc->exec_env_reg, + NEW_CONST(PTR, offsetof(WASMExecEnv, jit_cache))); + + /* Traverse each pointer/str argument, call + jit_check_app_addr_and_convert to check whether it is + in the range of linear memory and and convert it from + app offset into native address */ + for (i = 0; i < func_type->param_count; i++) { + + is_pointer_arg = false; + + if (signature[i + 1] == '*') { + /* param is a pointer */ + is_pointer_arg = true; + func_params[1] = NEW_CONST(I32, false); /* is_str = false */ + func_params[2] = argvs[i]; + if (signature[i + 2] == '~') { + /* TODO: Memory64 no need to convert if mem idx type i64 */ + func_params[3] = jit_cc_new_reg_I64(cc); + /* pointer with length followed */ + GEN_INSN(I32TOI64, func_params[3], argvs[i + 1]); + } + else { + /* pointer with length followed */ + func_params[3] = NEW_CONST(I64, 1); + } + } + else if (signature[i + 1] == '$') { + /* param is a string */ + is_pointer_arg = true; + func_params[1] = NEW_CONST(I32, true); /* is_str = true */ + func_params[2] = argvs[i]; + func_params[3] = NEW_CONST(I64, 1); + } + + if (is_pointer_arg) { + JitReg native_addr_64 = jit_cc_new_reg_I64(cc); + /* TODO: Memory64 no need to convert if mem idx type i64 */ + GEN_INSN(I32TOI64, native_addr_64, func_params[2]); + func_params[2] = native_addr_64; + + if (!jit_emit_callnative(cc, jit_check_app_addr_and_convert, + ret, func_params, 5)) { + goto fail; + } + + /* Convert the return value from bool to uint32 */ + GEN_INSN(AND, ret, ret, NEW_CONST(I32, 0xFF)); + /* Check whether there is exception thrown */ + GEN_INSN(CMP, cc->cmp_reg, ret, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, + cc->cmp_reg, NULL)) { + return false; + } + + /* Load native addr from pointer of native addr, + or exec_env->jit_cache */ + argvs[i] = jit_cc_new_reg_ptr(cc); + GEN_INSN(LDPTR, argvs[i], native_addr_ptr, NEW_CONST(I32, 0)); + } + } + + res = create_first_res_reg(cc, func_type); + + /* Prepare arguments of the native function */ + if (!(argvs1 = + jit_calloc(sizeof(JitReg) * (func_type->param_count + 1)))) { + goto fail; + } + argvs1[0] = cc->exec_env_reg; + for (i = 0; i < func_type->param_count; i++) { + argvs1[i + 1] = argvs[i]; + } + + /* Call the native function */ + native_func = NEW_CONST(PTR, (uintptr_t)func_import->func_ptr_linked); + if (!emit_callnative(cc, native_func, res, argvs1, + func_type->param_count + 1)) { + jit_free(argvs1); + goto fail; + } + jit_free(argvs1); + + /* Check whether there is exception thrown */ + GEN_INSN(LDI8, ret, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, cur_exception))); + GEN_INSN(CMP, cc->cmp_reg, ret, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BNE, + cc->cmp_reg, NULL)) { + goto fail; + } + + if (!post_return(cc, func_type, res, false)) { + goto fail; + } + } + else { + /* The function to call is a bytecode function */ + func = wasm_module + ->functions[func_idx - wasm_module->import_function_count]; + func_type = func->func_type; + + /* jitted_code = func_ptrs[func_idx - import_function_count] */ + fast_jit_func_ptrs = get_fast_jit_func_ptrs_reg(jit_frame); + jitted_code = jit_cc_new_reg_ptr(cc); + jitted_func_idx = func_idx - wasm_module->import_function_count; + GEN_INSN(LDPTR, jitted_code, fast_jit_func_ptrs, + NEW_CONST(I32, (uint32)sizeof(void *) * jitted_func_idx)); + + if (!pre_call(cc, func_type)) { + goto fail; + } + + res = create_first_res_reg(cc, func_type); + + GEN_INSN(CALLBC, res, 0, jitted_code, NEW_CONST(I32, func_idx)); + + if (!post_return(cc, func_type, res, true)) { + goto fail; + } + } + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + goto fail; +#endif + + /* Clear part of memory regs and table regs as their values + may be changed in the function call */ + if (cc->cur_wasm_module->possible_memory_grow) + clear_memory_regs(jit_frame); + clear_table_regs(jit_frame); + + /* Ignore tail call currently */ + (void)tail_call; + + return_value = true; + +fail: + if (argvs) + jit_free(argvs); + + return return_value; +} + +static JitReg +pack_argv(JitCompContext *cc) +{ + /* reuse the stack of the next frame */ + uint32 stack_base; + JitReg argv; + + stack_base = cc->total_frame_size + offsetof(WASMInterpFrame, lp); + argv = jit_cc_new_reg_ptr(cc); + GEN_INSN(ADD, argv, cc->fp_reg, NEW_CONST(PTR, stack_base)); + if (jit_get_last_error(cc)) { + return (JitReg)0; + } + return argv; +} + +bool +jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, + uint32 tbl_idx) +{ + WASMModule *wasm_module = cc->cur_wasm_module; + JitBasicBlock *block_import, *block_nonimport, *func_return; + JitReg elem_idx, native_ret, argv, arg_regs[6]; + JitFrame *jit_frame = cc->jit_frame; + JitReg tbl_size, offset, offset_i32; + JitReg func_import, func_idx, tbl_elems, func_count; + JitReg func_type_indexes, func_type_idx, fast_jit_func_ptrs; + JitReg offset1_i32, offset1, func_type_idx1, res; + JitReg import_func_ptrs, jitted_code_idx, jitted_code; + WASMType *func_type; + uint32 n; + + POP_I32(elem_idx); + + /* check elem_idx */ + tbl_size = get_table_cur_size_reg(jit_frame, tbl_idx); + + GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_size); + if (!jit_emit_exception(cc, EXCE_UNDEFINED_ELEMENT, JIT_OP_BGEU, + cc->cmp_reg, NULL)) + goto fail; + + /* check func_idx */ + if (UINTPTR_MAX == UINT64_MAX) { + offset_i32 = jit_cc_new_reg_I32(cc); + offset = jit_cc_new_reg_I64(cc); + /* Calculate offset by pointer size (elem_idx * + * sizeof(table_elem_type_t)) */ + GEN_INSN(SHL, offset_i32, elem_idx, NEW_CONST(I32, 3)); + GEN_INSN(I32TOI64, offset, offset_i32); + } + else { + offset = jit_cc_new_reg_I32(cc); + GEN_INSN(SHL, offset, elem_idx, NEW_CONST(I32, 2)); + } + func_idx = jit_cc_new_reg_I32(cc); + tbl_elems = get_table_elems_reg(jit_frame, tbl_idx); + GEN_INSN(LDI32, func_idx, tbl_elems, offset); + + GEN_INSN(CMP, cc->cmp_reg, func_idx, NEW_CONST(I32, -1)); + if (!jit_emit_exception(cc, EXCE_UNINITIALIZED_ELEMENT, JIT_OP_BEQ, + cc->cmp_reg, NULL)) + goto fail; + + func_count = NEW_CONST(I32, wasm_module->import_function_count + + wasm_module->function_count); + GEN_INSN(CMP, cc->cmp_reg, func_idx, func_count); + if (!jit_emit_exception(cc, EXCE_INVALID_FUNCTION_INDEX, JIT_OP_BGTU, + cc->cmp_reg, NULL)) + goto fail; + + /* check func_type */ + /* get func_type_idx from func_type_indexes */ + if (UINTPTR_MAX == UINT64_MAX) { + offset1_i32 = jit_cc_new_reg_I32(cc); + offset1 = jit_cc_new_reg_I64(cc); + GEN_INSN(SHL, offset1_i32, func_idx, NEW_CONST(I32, 2)); + GEN_INSN(I32TOI64, offset1, offset1_i32); + } + else { + offset1 = jit_cc_new_reg_I32(cc); + GEN_INSN(SHL, offset1, func_idx, NEW_CONST(I32, 2)); + } + + func_type_indexes = get_func_type_indexes_reg(jit_frame); + func_type_idx = jit_cc_new_reg_I32(cc); + GEN_INSN(LDI32, func_type_idx, func_type_indexes, offset1); + + type_idx = wasm_get_smallest_type_idx(wasm_module->types, + wasm_module->type_count, type_idx); + func_type_idx1 = NEW_CONST(I32, type_idx); + GEN_INSN(CMP, cc->cmp_reg, func_type_idx, func_type_idx1); + if (!jit_emit_exception(cc, EXCE_INVALID_FUNCTION_TYPE_INDEX, JIT_OP_BNE, + cc->cmp_reg, NULL)) + goto fail; + + /* pop function arguments and store it to out area of callee stack frame */ + func_type = wasm_module->types[type_idx]; + if (!pre_call(cc, func_type)) { + goto fail; + } + + /* store elem_idx and func_idx to exec_env->jit_cache */ + GEN_INSN(STI32, elem_idx, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache))); + GEN_INSN(STI32, func_idx, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache) + 4)); + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + goto fail; +#endif + + block_import = jit_cc_new_basic_block(cc, 0); + block_nonimport = jit_cc_new_basic_block(cc, 0); + func_return = jit_cc_new_basic_block(cc, 0); + if (!block_import || !block_nonimport || !func_return) { + goto fail; + } + + /* Commit register values to locals and stacks */ + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + /* Clear frame values */ + clear_values(jit_frame); + + /* jump to block_import or block_nonimport */ + GEN_INSN(CMP, cc->cmp_reg, func_idx, + NEW_CONST(I32, cc->cur_wasm_module->import_function_count)); + GEN_INSN(BLTU, cc->cmp_reg, jit_basic_block_label(block_import), + jit_basic_block_label(block_nonimport)); + + /* block_import */ + cc->cur_basic_block = block_import; + + elem_idx = jit_cc_new_reg_I32(cc); + GEN_INSN(LDI32, elem_idx, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache))); + GEN_INSN(LDI32, func_idx, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache) + 4)); + + argv = pack_argv(cc); + if (!argv) { + goto fail; + } + native_ret = jit_cc_new_reg_I32(cc); + arg_regs[0] = cc->exec_env_reg; + arg_regs[1] = NEW_CONST(I32, tbl_idx); + arg_regs[2] = elem_idx; + arg_regs[3] = NEW_CONST(I32, type_idx); + arg_regs[4] = NEW_CONST(I32, func_type->param_cell_num); + arg_regs[5] = argv; + + import_func_ptrs = get_import_func_ptrs_reg(jit_frame); + func_import = jit_cc_new_reg_ptr(cc); + if (UINTPTR_MAX == UINT64_MAX) { + JitReg func_import_offset = jit_cc_new_reg_I32(cc); + JitReg func_import_offset_i64 = jit_cc_new_reg_I64(cc); + GEN_INSN(SHL, func_import_offset, func_idx, NEW_CONST(I32, 3)); + GEN_INSN(I32TOI64, func_import_offset_i64, func_import_offset); + GEN_INSN(LDPTR, func_import, import_func_ptrs, func_import_offset_i64); + } + else { + JitReg func_import_offset = jit_cc_new_reg_I32(cc); + GEN_INSN(SHL, func_import_offset, func_idx, NEW_CONST(I32, 2)); + GEN_INSN(LDPTR, func_import, import_func_ptrs, func_import_offset); + } + if (!jit_emit_callnative(cc, fast_jit_call_indirect, native_ret, arg_regs, + 6)) { + goto fail; + } + + /* Convert bool to uint32 */ + GEN_INSN(AND, native_ret, native_ret, NEW_CONST(I32, 0xFF)); + + /* Check whether there is exception thrown */ + GEN_INSN(CMP, cc->cmp_reg, native_ret, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, cc->cmp_reg, + NULL)) { + return false; + } + + /* Store res into current frame, so that post_return in + block func_return can get the value */ + n = cc->jit_frame->sp - cc->jit_frame->lp; + if (func_type->result_count > 0) { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + res = jit_cc_new_reg_I32(cc); + GEN_INSN(LDI32, res, argv, NEW_CONST(I32, 0)); + GEN_INSN(STI32, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + case VALUE_TYPE_I64: + res = jit_cc_new_reg_I64(cc); + GEN_INSN(LDI64, res, argv, NEW_CONST(I32, 0)); + GEN_INSN(STI64, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + case VALUE_TYPE_F32: + res = jit_cc_new_reg_F32(cc); + GEN_INSN(LDF32, res, argv, NEW_CONST(I32, 0)); + GEN_INSN(STF32, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + case VALUE_TYPE_F64: + res = jit_cc_new_reg_F64(cc); + GEN_INSN(LDF64, res, argv, NEW_CONST(I32, 0)); + GEN_INSN(STF64, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + default: + bh_assert(0); + goto fail; + } + } + + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + clear_values(jit_frame); + GEN_INSN(JMP, jit_basic_block_label(func_return)); + + /* basic_block non_import */ + cc->cur_basic_block = block_nonimport; + + GEN_INSN(LDI32, func_idx, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache) + 4)); + + /* get jitted_code */ + fast_jit_func_ptrs = get_fast_jit_func_ptrs_reg(jit_frame); + jitted_code_idx = jit_cc_new_reg_I32(cc); + jitted_code = jit_cc_new_reg_ptr(cc); + GEN_INSN(SUB, jitted_code_idx, func_idx, + NEW_CONST(I32, cc->cur_wasm_module->import_function_count)); + if (UINTPTR_MAX == UINT64_MAX) { + JitReg jitted_code_offset = jit_cc_new_reg_I32(cc); + JitReg jitted_code_offset_64 = jit_cc_new_reg_I64(cc); + GEN_INSN(SHL, jitted_code_offset, jitted_code_idx, NEW_CONST(I32, 3)); + GEN_INSN(I32TOI64, jitted_code_offset_64, jitted_code_offset); + GEN_INSN(LDPTR, jitted_code, fast_jit_func_ptrs, jitted_code_offset_64); + } + else { + JitReg jitted_code_offset = jit_cc_new_reg_I32(cc); + GEN_INSN(SHL, jitted_code_offset, jitted_code_idx, NEW_CONST(I32, 2)); + GEN_INSN(LDPTR, jitted_code, fast_jit_func_ptrs, jitted_code_offset); + } + + res = 0; + if (func_type->result_count > 0) { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + res = jit_cc_new_reg_I32(cc); + break; + case VALUE_TYPE_I64: + res = jit_cc_new_reg_I64(cc); + break; + case VALUE_TYPE_F32: + res = jit_cc_new_reg_F32(cc); + break; + case VALUE_TYPE_F64: + res = jit_cc_new_reg_F64(cc); + break; + default: + bh_assert(0); + goto fail; + } + } + GEN_INSN(CALLBC, res, 0, jitted_code, func_idx); + /* Store res into current frame, so that post_return in + block func_return can get the value */ + n = cc->jit_frame->sp - cc->jit_frame->lp; + if (func_type->result_count > 0) { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + GEN_INSN(STI32, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + case VALUE_TYPE_I64: + GEN_INSN(STI64, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + case VALUE_TYPE_F32: + GEN_INSN(STF32, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + case VALUE_TYPE_F64: + GEN_INSN(STF64, res, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + default: + bh_assert(0); + goto fail; + } + } + /* commit and clear jit frame, then jump to block func_ret */ + gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp); + clear_values(jit_frame); + GEN_INSN(JMP, jit_basic_block_label(func_return)); + + /* translate block func_return */ + cc->cur_basic_block = func_return; + if (!post_return(cc, func_type, 0, true)) { + goto fail; + } + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + goto fail; +#endif + + /* Clear part of memory regs and table regs as their values + may be changed in the function call */ + if (cc->cur_wasm_module->possible_memory_grow) + clear_memory_regs(cc->jit_frame); + clear_table_regs(cc->jit_frame); + return true; +fail: + return false; +} + +#if WASM_ENABLE_REF_TYPES != 0 +bool +jit_compile_op_ref_null(JitCompContext *cc, uint32 ref_type) +{ + PUSH_I32(NEW_CONST(I32, NULL_REF)); + (void)ref_type; + return true; +fail: + return false; +} + +bool +jit_compile_op_ref_is_null(JitCompContext *cc) +{ + JitReg ref, res; + + POP_I32(ref); + + GEN_INSN(CMP, cc->cmp_reg, ref, NEW_CONST(I32, NULL_REF)); + res = jit_cc_new_reg_I32(cc); + GEN_INSN(SELECTEQ, res, cc->cmp_reg, NEW_CONST(I32, 1), NEW_CONST(I32, 0)); + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_ref_func(JitCompContext *cc, uint32 func_idx) +{ + PUSH_I32(NEW_CONST(I32, func_idx)); + return true; +fail: + return false; +} +#endif + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +static bool +emit_callnative(JitCompContext *cc, JitReg native_func_reg, JitReg res, + JitReg *params, uint32 param_count) +{ + JitInsn *insn; + char *i32_arg_names[] = { "edi", "esi", "edx", "ecx" }; + char *i64_arg_names[] = { "rdi", "rsi", "rdx", "rcx", "r8", "r9" }; + char *f32_arg_names[] = { "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" }; + char *f64_arg_names[] = { "xmm0_f64", "xmm1_f64", "xmm2_f64", + "xmm3_f64", "xmm4_f64", "xmm5_f64" }; + JitReg i32_arg_regs[4], i64_arg_regs[6]; + JitReg f32_arg_regs[6], f64_arg_regs[6], res_reg = 0; + JitReg eax_hreg = jit_codegen_get_hreg_by_name("eax"); + JitReg xmm0_hreg = jit_codegen_get_hreg_by_name("xmm0"); + uint32 i, i64_reg_idx, float_reg_idx, lock_i32_reg_num; + + bh_assert(param_count <= 6); + + for (i = 0; i < 4; i++) { + i32_arg_regs[i] = jit_codegen_get_hreg_by_name(i32_arg_names[i]); + } + + for (i = 0; i < 6; i++) { + i64_arg_regs[i] = jit_codegen_get_hreg_by_name(i64_arg_names[i]); + f32_arg_regs[i] = jit_codegen_get_hreg_by_name(f32_arg_names[i]); + f64_arg_regs[i] = jit_codegen_get_hreg_by_name(f64_arg_names[i]); + } + + lock_i32_reg_num = param_count < 4 ? param_count : 4; + + /* + * Lock i32 registers so that they won't be allocated for the operand + * of below I32TOI64 insn, which may have been overwritten in the + * previous MOV, for example, in the below insns: + * MOV I5, I15 + * I32TOI64 I6, i5 + * CALLNATIVE VOID, native_func, I5, I6 + * i5 is used in the second insn, but it has been overwritten in I5 + * by the first insn + */ + for (i = 0; i < lock_i32_reg_num; i++) { + GEN_INSN(MOV, i32_arg_regs[i], i32_arg_regs[i]); + } + + i64_reg_idx = float_reg_idx = 0; + for (i = 0; i < param_count; i++) { + switch (jit_reg_kind(params[i])) { + case JIT_REG_KIND_I32: + GEN_INSN(I32TOI64, i64_arg_regs[i64_reg_idx++], params[i]); + break; + case JIT_REG_KIND_I64: + GEN_INSN(MOV, i64_arg_regs[i64_reg_idx++], params[i]); + break; + case JIT_REG_KIND_F32: + GEN_INSN(MOV, f32_arg_regs[float_reg_idx++], params[i]); + break; + case JIT_REG_KIND_F64: + GEN_INSN(MOV, f64_arg_regs[float_reg_idx++], params[i]); + break; + default: + bh_assert(0); + return false; + } + } + + /* + * Announce the locked i32 registers are being used, and do necessary + * spill ASAP + */ + for (i = 0; i < lock_i32_reg_num; i++) { + GEN_INSN(MOV, i32_arg_regs[i], i32_arg_regs[i]); + } + + if (res) { + switch (jit_reg_kind(res)) { + case JIT_REG_KIND_I32: + res_reg = eax_hreg; + break; + case JIT_REG_KIND_I64: + res_reg = res; + break; + case JIT_REG_KIND_F32: + res_reg = xmm0_hreg; + break; + case JIT_REG_KIND_F64: + res_reg = res; + break; + default: + bh_assert(0); + return false; + } + } + + insn = GEN_INSN(CALLNATIVE, res_reg, native_func_reg, param_count); + if (!insn) { + return false; + } + + i64_reg_idx = float_reg_idx = 0; + for (i = 0; i < param_count; i++) { + switch (jit_reg_kind(params[i])) { + case JIT_REG_KIND_I32: + case JIT_REG_KIND_I64: + *(jit_insn_opndv(insn, i + 2)) = i64_arg_regs[i64_reg_idx++]; + break; + case JIT_REG_KIND_F32: + *(jit_insn_opndv(insn, i + 2)) = f32_arg_regs[float_reg_idx++]; + break; + case JIT_REG_KIND_F64: + *(jit_insn_opndv(insn, i + 2)) = f64_arg_regs[float_reg_idx++]; + break; + default: + bh_assert(0); + return false; + } + } + + if (res && res != res_reg) { + GEN_INSN(MOV, res, res_reg); + } + + return true; +} +#else +static bool +emit_callnative(JitCompContext *cc, JitRef native_func_reg, JitReg res, + JitReg *params, uint32 param_count) +{ + JitInsn *insn; + uint32 i; + + bh_assert(param_count <= 6); + + insn = GEN_INSN(CALLNATIVE, res, native_func_reg, param_count); + if (!insn) + return false; + + for (i = 0; i < param_count; i++) { + *(jit_insn_opndv(insn, i + 2)) = params[i]; + } + return true; +} +#endif + +bool +jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res, + JitReg *params, uint32 param_count) +{ + return emit_callnative(cc, NEW_CONST(PTR, (uintptr_t)native_func), res, + params, param_count); +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_function.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_function.h new file mode 100644 index 0000000..7405f77 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_function.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_FUNCTION_H_ +#define _JIT_EMIT_FUNCTION_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call); + +bool +jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, + uint32 tbl_idx); + +bool +jit_compile_op_ref_null(JitCompContext *cc, uint32 ref_type); + +bool +jit_compile_op_ref_is_null(JitCompContext *cc); + +bool +jit_compile_op_ref_func(JitCompContext *cc, uint32 func_idx); + +bool +jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res, + JitReg *params, uint32 param_count); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_FUNCTION_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_memory.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_memory.c new file mode 100644 index 0000000..ea245ba --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_memory.c @@ -0,0 +1,1192 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_memory.h" +#include "jit_emit_exception.h" +#include "jit_emit_function.h" +#include "../jit_frontend.h" +#include "../jit_codegen.h" +#include "../../interpreter/wasm_runtime.h" +#include "jit_emit_control.h" + +#ifndef OS_ENABLE_HW_BOUND_CHECK +static JitReg +get_memory_boundary(JitCompContext *cc, uint32 mem_idx, uint32 bytes) +{ + JitReg memory_boundary; + + switch (bytes) { + case 1: + { + memory_boundary = + get_mem_bound_check_1byte_reg(cc->jit_frame, mem_idx); + break; + } + case 2: + { + memory_boundary = + get_mem_bound_check_2bytes_reg(cc->jit_frame, mem_idx); + break; + } + case 4: + { + memory_boundary = + get_mem_bound_check_4bytes_reg(cc->jit_frame, mem_idx); + break; + } + case 8: + { + memory_boundary = + get_mem_bound_check_8bytes_reg(cc->jit_frame, mem_idx); + break; + } + case 16: + { + memory_boundary = + get_mem_bound_check_16bytes_reg(cc->jit_frame, mem_idx); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + return memory_boundary; +fail: + return 0; +} +#endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 +static void +set_load_or_store_atomic(JitInsn *load_or_store_inst) +{ + load_or_store_inst->flags_u8 |= 0x1; +} +#endif + +#if UINTPTR_MAX == UINT64_MAX +static JitReg +check_and_seek_on_64bit_platform(JitCompContext *cc, JitReg addr, JitReg offset, + JitReg memory_boundary) +{ + JitReg long_addr, offset1; + + /* long_addr = (int64_t)addr */ + long_addr = jit_cc_new_reg_I64(cc); + GEN_INSN(U32TOI64, long_addr, addr); + + /* offset1 = offset + long_addr */ + offset1 = jit_cc_new_reg_I64(cc); + GEN_INSN(ADD, offset1, offset, long_addr); + +#ifndef OS_ENABLE_HW_BOUND_CHECK + /* if (offset1 > memory_boundary) goto EXCEPTION */ + GEN_INSN(CMP, cc->cmp_reg, offset1, memory_boundary); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { + goto fail; + } +#endif + + return offset1; +#ifndef OS_ENABLE_HW_BOUND_CHECK +fail: + return 0; +#endif +} +#else +static JitReg +check_and_seek_on_32bit_platform(JitCompContext *cc, JitReg addr, JitReg offset, + JitReg memory_boundary) +{ + JitReg offset1; + + /* offset1 = offset + addr */ + offset1 = jit_cc_new_reg_I32(cc); + GEN_INSN(ADD, offset1, offset, addr); + + /* if (offset1 < addr) goto EXCEPTION */ + GEN_INSN(CMP, cc->cmp_reg, offset1, addr); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BLTU, + cc->cmp_reg, NULL)) { + goto fail; + } + +#ifndef OS_ENABLE_HW_BOUND_CHECK + /* if (offset1 > memory_boundary) goto EXCEPTION */ + GEN_INSN(CMP, cc->cmp_reg, offset1, memory_boundary); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { + goto fail; + } +#endif + + return offset1; +fail: + return 0; +} +#endif + +static JitReg +check_and_seek(JitCompContext *cc, JitReg addr, uint32 offset, uint32 bytes) +{ + JitReg memory_boundary = 0, offset1; +#ifndef OS_ENABLE_HW_BOUND_CHECK + JitReg cur_page_count; + /* the default memory */ + uint32 mem_idx = 0; +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + /* ---------- check ---------- */ + /* 1. shortcut if the memory size is 0 */ + if (cc->cur_wasm_module->memories != NULL + && 0 == cc->cur_wasm_module->memories[mem_idx].init_page_count) { + + cur_page_count = get_cur_page_count_reg(cc->jit_frame, mem_idx); + + /* if (cur_mem_page_count == 0) goto EXCEPTION */ + GEN_INSN(CMP, cc->cmp_reg, cur_page_count, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + JIT_OP_BEQ, cc->cmp_reg, NULL)) { + goto fail; + } + } + + /* 2. a complete boundary check */ + memory_boundary = get_memory_boundary(cc, mem_idx, bytes); + if (!memory_boundary) + goto fail; +#endif + +#if UINTPTR_MAX == UINT64_MAX + offset1 = check_and_seek_on_64bit_platform(cc, addr, NEW_CONST(I64, offset), + memory_boundary); + if (!offset1) + goto fail; +#else + offset1 = check_and_seek_on_32bit_platform(cc, addr, NEW_CONST(I32, offset), + memory_boundary); + if (!offset1) + goto fail; +#endif + + return offset1; +fail: + return 0; +} + +#if UINTPTR_MAX == UINT64_MAX +#define CHECK_ALIGNMENT(offset1) \ + do { \ + JitReg align_mask = NEW_CONST(I64, ((uint64)1 << align) - 1); \ + JitReg AND_res = jit_cc_new_reg_I64(cc); \ + GEN_INSN(AND, AND_res, offset1, align_mask); \ + GEN_INSN(CMP, cc->cmp_reg, AND_res, NEW_CONST(I64, 0)); \ + if (!jit_emit_exception(cc, EXCE_UNALIGNED_ATOMIC, JIT_OP_BNE, \ + cc->cmp_reg, NULL)) \ + goto fail; \ + } while (0) +#else +#define CHECK_ALIGNMENT(offset1) \ + do { \ + JitReg align_mask = NEW_CONST(I32, (1 << align) - 1); \ + JitReg AND_res = jit_cc_new_reg_I32(cc); \ + GEN_INSN(AND, AND_res, offset1, align_mask); \ + GEN_INSN(CMP, cc->cmp_reg, AND_res, NEW_CONST(I32, 0)); \ + if (!jit_emit_exception(cc, EXCE_UNALIGNED_ATOMIC, JIT_OP_BNE, \ + cc->cmp_reg, NULL)) \ + goto fail; \ + } while (0) +#endif + +bool +jit_compile_op_i32_load(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool sign, bool atomic) +{ + JitReg addr, offset1, value, memory_data; + JitInsn *load_insn = NULL; + + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) { + goto fail; + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) { + CHECK_ALIGNMENT(offset1); + } +#endif + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + value = jit_cc_new_reg_I32(cc); + switch (bytes) { + case 1: + { + if (sign) { + load_insn = GEN_INSN(LDI8, value, memory_data, offset1); + } + else { + load_insn = GEN_INSN(LDU8, value, memory_data, offset1); + } + break; + } + case 2: + { + if (sign) { + load_insn = GEN_INSN(LDI16, value, memory_data, offset1); + } + else { + load_insn = GEN_INSN(LDU16, value, memory_data, offset1); + } + break; + } + case 4: + { + if (sign) { + load_insn = GEN_INSN(LDI32, value, memory_data, offset1); + } + else { + load_insn = GEN_INSN(LDU32, value, memory_data, offset1); + } + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic && load_insn) + set_load_or_store_atomic(load_insn); +#else + (void)load_insn; +#endif + + PUSH_I32(value); + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_load(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool sign, bool atomic) +{ + JitReg addr, offset1, value, memory_data; + JitInsn *load_insn = NULL; + + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) { + goto fail; + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) { + CHECK_ALIGNMENT(offset1); + } +#endif + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + value = jit_cc_new_reg_I64(cc); + switch (bytes) { + case 1: + { + if (sign) { + load_insn = GEN_INSN(LDI8, value, memory_data, offset1); + } + else { + load_insn = GEN_INSN(LDU8, value, memory_data, offset1); + } + break; + } + case 2: + { + if (sign) { + load_insn = GEN_INSN(LDI16, value, memory_data, offset1); + } + else { + load_insn = GEN_INSN(LDU16, value, memory_data, offset1); + } + break; + } + case 4: + { + if (sign) { + load_insn = GEN_INSN(LDI32, value, memory_data, offset1); + } + else { + load_insn = GEN_INSN(LDU32, value, memory_data, offset1); + } + break; + } + case 8: + { + if (sign) { + load_insn = GEN_INSN(LDI64, value, memory_data, offset1); + } + else { + load_insn = GEN_INSN(LDU64, value, memory_data, offset1); + } + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic && load_insn) + set_load_or_store_atomic(load_insn); +#else + (void)load_insn; +#endif + + PUSH_I64(value); + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_load(JitCompContext *cc, uint32 align, uint32 offset) +{ + JitReg addr, offset1, value, memory_data; + + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, 4); + if (!offset1) { + goto fail; + } + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + value = jit_cc_new_reg_F32(cc); + GEN_INSN(LDF32, value, memory_data, offset1); + + PUSH_F32(value); + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_load(JitCompContext *cc, uint32 align, uint32 offset) +{ + JitReg addr, offset1, value, memory_data; + + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, 8); + if (!offset1) { + goto fail; + } + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + value = jit_cc_new_reg_F64(cc); + GEN_INSN(LDF64, value, memory_data, offset1); + + PUSH_F64(value); + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_store(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool atomic) +{ + JitReg value, addr, offset1, memory_data; + JitInsn *store_insn = NULL; + + POP_I32(value); + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) { + goto fail; + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) { + CHECK_ALIGNMENT(offset1); + } +#endif + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + switch (bytes) { + case 1: + { + store_insn = GEN_INSN(STI8, value, memory_data, offset1); + break; + } + case 2: + { + store_insn = GEN_INSN(STI16, value, memory_data, offset1); + break; + } + case 4: + { + store_insn = GEN_INSN(STI32, value, memory_data, offset1); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic && store_insn) + set_load_or_store_atomic(store_insn); +#else + (void)store_insn; +#endif + + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_store(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool atomic) +{ + JitReg value, addr, offset1, memory_data; + JitInsn *store_insn = NULL; + + POP_I64(value); + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) { + goto fail; + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic) { + CHECK_ALIGNMENT(offset1); + } +#endif + + if (jit_reg_is_const(value) && bytes < 8) { + value = NEW_CONST(I32, (int32)jit_cc_get_const_I64(cc, value)); + } + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + switch (bytes) { + case 1: + { + store_insn = GEN_INSN(STI8, value, memory_data, offset1); + break; + } + case 2: + { + store_insn = GEN_INSN(STI16, value, memory_data, offset1); + break; + } + case 4: + { + store_insn = GEN_INSN(STI32, value, memory_data, offset1); + break; + } + case 8: + { + store_insn = GEN_INSN(STI64, value, memory_data, offset1); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (atomic && store_insn) + set_load_or_store_atomic(store_insn); +#else + (void)store_insn; +#endif + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_store(JitCompContext *cc, uint32 align, uint32 offset) +{ + JitReg value, addr, offset1, memory_data; + + POP_F32(value); + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, 4); + if (!offset1) { + goto fail; + } + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + GEN_INSN(STF32, value, memory_data, offset1); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_store(JitCompContext *cc, uint32 align, uint32 offset) +{ + JitReg value, addr, offset1, memory_data; + + POP_F64(value); + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, 8); + if (!offset1) { + goto fail; + } + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + GEN_INSN(STF64, value, memory_data, offset1); + + return true; +fail: + return false; +} + +bool +jit_compile_op_memory_size(JitCompContext *cc, uint32 mem_idx) +{ + JitReg cur_page_count; + + cur_page_count = get_cur_page_count_reg(cc->jit_frame, mem_idx); + + PUSH_I32(cur_page_count); + + return true; +fail: + return false; +} + +bool +jit_compile_op_memory_grow(JitCompContext *cc, uint32 mem_idx) +{ + JitReg grow_res, res; + JitReg prev_page_count, inc_page_count, args[2]; + + /* Get current page count as prev_page_count */ + prev_page_count = get_cur_page_count_reg(cc->jit_frame, mem_idx); + + /* Call wasm_enlarge_memory */ + POP_I32(inc_page_count); + + grow_res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = inc_page_count; + + if (!jit_emit_callnative(cc, wasm_enlarge_memory, grow_res, args, 2)) { + goto fail; + } + /* Convert bool to uint32 */ + GEN_INSN(AND, grow_res, grow_res, NEW_CONST(I32, 0xFF)); + + /* return different values according to memory.grow result */ + res = jit_cc_new_reg_I32(cc); + GEN_INSN(CMP, cc->cmp_reg, grow_res, NEW_CONST(I32, 0)); + GEN_INSN(SELECTNE, res, cc->cmp_reg, prev_page_count, + NEW_CONST(I32, (int32)-1)); + PUSH_I32(res); + + /* Ensure a refresh in next get memory related registers */ + clear_memory_regs(cc->jit_frame); + + return true; +fail: + return false; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +static int +wasm_init_memory(WASMModuleInstance *inst, uint32 mem_idx, uint32 seg_idx, + uint32 len, uint32 mem_offset, uint32 data_offset) +{ + WASMMemoryInstance *mem_inst; + WASMDataSeg *data_segment; + uint64 mem_size; + uint8 *mem_addr, *data_addr; + uint32 seg_len; + + /* if d + n > the length of mem.data */ + mem_inst = inst->memories[mem_idx]; + mem_size = mem_inst->cur_page_count * (uint64)mem_inst->num_bytes_per_page; + if (mem_size < mem_offset || mem_size - mem_offset < len) + goto out_of_bounds; + + /* if s + n > the length of data.data */ + bh_assert(seg_idx < inst->module->data_seg_count); + if (bh_bitmap_get_bit(inst->e->common.data_dropped, seg_idx)) { + seg_len = 0; + data_addr = NULL; + } + else { + data_segment = inst->module->data_segments[seg_idx]; + seg_len = data_segment->data_length; + data_addr = data_segment->data + data_offset; + } + if (seg_len < data_offset || seg_len - data_offset < len) + goto out_of_bounds; + + mem_addr = mem_inst->memory_data + mem_offset; + bh_memcpy_s(mem_addr, (uint32)(mem_size - mem_offset), data_addr, len); + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds memory access"); + return -1; +} + +bool +jit_compile_op_memory_init(JitCompContext *cc, uint32 mem_idx, uint32 seg_idx) +{ + JitReg len, mem_offset, data_offset, res; + JitReg args[6] = { 0 }; + + POP_I32(len); + POP_I32(data_offset); + POP_I32(mem_offset); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, mem_idx); + args[2] = NEW_CONST(I32, seg_idx); + args[3] = len; + args[4] = mem_offset; + args[5] = data_offset; + + if (!jit_emit_callnative(cc, wasm_init_memory, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: + return false; +} + +static void +wasm_data_drop(WASMModuleInstance *inst, uint32 seg_idx) +{ + bh_bitmap_set_bit(inst->e->common.data_dropped, seg_idx); +} + +bool +jit_compile_op_data_drop(JitCompContext *cc, uint32 seg_idx) +{ + JitReg args[2] = { 0 }; + + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, seg_idx); + + return jit_emit_callnative(cc, wasm_data_drop, 0, args, + sizeof(args) / sizeof(args[0])); +} + +static int +wasm_copy_memory(WASMModuleInstance *inst, uint32 src_mem_idx, + uint32 dst_mem_idx, uint32 len, uint32 src_offset, + uint32 dst_offset) +{ + WASMMemoryInstance *src_mem, *dst_mem; + uint64 src_mem_size, dst_mem_size; + uint8 *src_addr, *dst_addr; + + src_mem = inst->memories[src_mem_idx]; + dst_mem = inst->memories[dst_mem_idx]; + src_mem_size = + src_mem->cur_page_count * (uint64)src_mem->num_bytes_per_page; + dst_mem_size = + dst_mem->cur_page_count * (uint64)dst_mem->num_bytes_per_page; + + /* if s + n > the length of mem.data */ + if (src_mem_size < src_offset || src_mem_size - src_offset < len) + goto out_of_bounds; + + /* if d + n > the length of mem.data */ + if (dst_mem_size < dst_offset || dst_mem_size - dst_offset < len) + goto out_of_bounds; + + src_addr = src_mem->memory_data + src_offset; + dst_addr = dst_mem->memory_data + dst_offset; + /* allowing the destination and source to overlap */ + bh_memmove_s(dst_addr, (uint32)(dst_mem_size - dst_offset), src_addr, len); + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds memory access"); + return -1; +} + +bool +jit_compile_op_memory_copy(JitCompContext *cc, uint32 src_mem_idx, + uint32 dst_mem_idx) +{ + JitReg len, src, dst, res; + JitReg args[6] = { 0 }; + + POP_I32(len); + POP_I32(src); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, src_mem_idx); + args[2] = NEW_CONST(I32, dst_mem_idx); + args[3] = len; + args[4] = src; + args[5] = dst; + + if (!jit_emit_callnative(cc, wasm_copy_memory, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: + return false; +} + +static int +wasm_fill_memory(WASMModuleInstance *inst, uint32 mem_idx, uint32 len, + uint32 val, uint32 dst) +{ + WASMMemoryInstance *mem_inst; + uint64 mem_size; + uint8 *dst_addr; + + mem_inst = inst->memories[mem_idx]; + mem_size = mem_inst->cur_page_count * (uint64)mem_inst->num_bytes_per_page; + + if (mem_size < dst || mem_size - dst < len) + goto out_of_bounds; + + dst_addr = mem_inst->memory_data + dst; + memset(dst_addr, val, len); + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds memory access"); + return -1; +} + +bool +jit_compile_op_memory_fill(JitCompContext *cc, uint32 mem_idx) +{ + JitReg res, len, val, dst; + JitReg args[5] = { 0 }; + + POP_I32(len); + POP_I32(val); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, mem_idx); + args[2] = len; + args[3] = val; + args[4] = dst; + + if (!jit_emit_callnative(cc, wasm_fill_memory, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: + return false; +} +#endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define GEN_AT_RMW_INSN(op, op_type, bytes, result, value, memory_data, \ + offset1) \ + do { \ + switch (bytes) { \ + case 1: \ + { \ + insn = GEN_INSN(AT_##op##U8, result, value, memory_data, \ + offset1); \ + break; \ + } \ + case 2: \ + { \ + insn = GEN_INSN(AT_##op##U16, result, value, memory_data, \ + offset1); \ + break; \ + } \ + case 4: \ + { \ + if (op_type == VALUE_TYPE_I32) \ + insn = GEN_INSN(AT_##op##I32, result, value, memory_data, \ + offset1); \ + else \ + insn = GEN_INSN(AT_##op##U32, result, value, memory_data, \ + offset1); \ + break; \ + } \ + case 8: \ + { \ + insn = GEN_INSN(AT_##op##I64, result, value, memory_data, \ + offset1); \ + break; \ + } \ + default: \ + { \ + bh_assert(0); \ + goto fail; \ + } \ + } \ + } while (0) + +bool +jit_compile_op_atomic_rmw(JitCompContext *cc, uint8 atomic_op, uint8 op_type, + uint32 align, uint32 offset, uint32 bytes) +{ + JitReg addr, offset1, memory_data, value, result, eax_hreg, rax_hreg, + ebx_hreg, rbx_hreg; + JitInsn *insn = NULL; + bool is_i32 = op_type == VALUE_TYPE_I32; + bool is_logical_op = atomic_op == AtomicRMWBinOpAnd + || atomic_op == AtomicRMWBinOpOr + || atomic_op == AtomicRMWBinOpXor; + + /* currently we only implement atomic rmw on x86-64 target */ +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + + /* For atomic logical binary ops, it implicitly uses rax in cmpxchg + * instruction and implicitly uses rbx for storing temp value in the + * generated loop */ + eax_hreg = jit_codegen_get_hreg_by_name("eax"); + rax_hreg = jit_codegen_get_hreg_by_name("rax"); + ebx_hreg = jit_codegen_get_hreg_by_name("ebx"); + rbx_hreg = jit_codegen_get_hreg_by_name("rbx"); + + bh_assert(op_type == VALUE_TYPE_I32 || op_type == VALUE_TYPE_I64); + if (op_type == VALUE_TYPE_I32) { + POP_I32(value); + } + else { + POP_I64(value); + } + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) { + goto fail; + } + CHECK_ALIGNMENT(offset1); + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + if (op_type == VALUE_TYPE_I32) + result = jit_cc_new_reg_I32(cc); + else + result = jit_cc_new_reg_I64(cc); + + switch (atomic_op) { + case AtomicRMWBinOpAdd: + { + GEN_AT_RMW_INSN(ADD, op_type, bytes, result, value, memory_data, + offset1); + break; + } + case AtomicRMWBinOpSub: + { + GEN_AT_RMW_INSN(SUB, op_type, bytes, result, value, memory_data, + offset1); + break; + } + case AtomicRMWBinOpAnd: + { + GEN_AT_RMW_INSN(AND, op_type, bytes, result, value, memory_data, + offset1); + break; + } + case AtomicRMWBinOpOr: + { + GEN_AT_RMW_INSN(OR, op_type, bytes, result, value, memory_data, + offset1); + break; + } + case AtomicRMWBinOpXor: + { + GEN_AT_RMW_INSN(XOR, op_type, bytes, result, value, memory_data, + offset1); + break; + } + case AtomicRMWBinOpXchg: + { + GEN_AT_RMW_INSN(XCHG, op_type, bytes, result, value, memory_data, + offset1); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + if (is_logical_op + && (!insn + || !jit_lock_reg_in_insn(cc, insn, is_i32 ? eax_hreg : rax_hreg) + || !jit_lock_reg_in_insn(cc, insn, is_i32 ? ebx_hreg : rbx_hreg))) { + jit_set_last_error( + cc, "generate atomic logical insn or lock ra&rb hreg failed"); + goto fail; + } + + if (op_type == VALUE_TYPE_I32) + PUSH_I32(result); + else + PUSH_I64(result); + + return true; +#endif /* defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) */ + +fail: + return false; +} + +bool +jit_compile_op_atomic_cmpxchg(JitCompContext *cc, uint8 op_type, uint32 align, + uint32 offset, uint32 bytes) +{ + JitReg addr, offset1, memory_data, value, expect, result; + bool is_i32 = op_type == VALUE_TYPE_I32; + /* currently we only implement atomic cmpxchg on x86-64 target */ +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + /* cmpxchg will use register al/ax/eax/rax to store parameter expected + * value, and the read result will also be stored to al/ax/eax/rax */ + JitReg eax_hreg = jit_codegen_get_hreg_by_name("eax"); + JitReg rax_hreg = jit_codegen_get_hreg_by_name("rax"); + JitInsn *insn = NULL; + + bh_assert(op_type == VALUE_TYPE_I32 || op_type == VALUE_TYPE_I64); + if (is_i32) { + POP_I32(value); + POP_I32(expect); + result = jit_cc_new_reg_I32(cc); + } + else { + POP_I64(value); + POP_I64(expect); + result = jit_cc_new_reg_I64(cc); + } + POP_I32(addr); + + offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) { + goto fail; + } + CHECK_ALIGNMENT(offset1); + + memory_data = get_memory_data_reg(cc->jit_frame, 0); + + GEN_INSN(MOV, is_i32 ? eax_hreg : rax_hreg, expect); + switch (bytes) { + case 1: + { + insn = GEN_INSN(AT_CMPXCHGU8, value, is_i32 ? eax_hreg : rax_hreg, + memory_data, offset1); + break; + } + case 2: + { + insn = GEN_INSN(AT_CMPXCHGU16, value, is_i32 ? eax_hreg : rax_hreg, + memory_data, offset1); + break; + } + case 4: + { + if (op_type == VALUE_TYPE_I32) + insn = + GEN_INSN(AT_CMPXCHGI32, value, is_i32 ? eax_hreg : rax_hreg, + memory_data, offset1); + else + insn = + GEN_INSN(AT_CMPXCHGU32, value, is_i32 ? eax_hreg : rax_hreg, + memory_data, offset1); + break; + } + case 8: + { + insn = GEN_INSN(AT_CMPXCHGI64, value, is_i32 ? eax_hreg : rax_hreg, + memory_data, offset1); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + if (!insn + || !jit_lock_reg_in_insn(cc, insn, is_i32 ? eax_hreg : rax_hreg)) { + jit_set_last_error(cc, "generate cmpxchg insn or lock ra hreg failed"); + goto fail; + } + + GEN_INSN(MOV, result, is_i32 ? eax_hreg : rax_hreg); + + if (is_i32) + PUSH_I32(result); + else + PUSH_I64(result); + + return true; +#endif /* defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) */ + +fail: + return false; +} + +bool +jit_compile_op_atomic_wait(JitCompContext *cc, uint8 op_type, uint32 align, + uint32 offset, uint32 bytes) +{ + bh_assert(op_type == VALUE_TYPE_I32 || op_type == VALUE_TYPE_I64); + + // Pop atomic.wait arguments + JitReg timeout, expect, expect_64, addr; + POP_I64(timeout); + if (op_type == VALUE_TYPE_I32) { + POP_I32(expect); + expect_64 = jit_cc_new_reg_I64(cc); + GEN_INSN(I32TOI64, expect_64, expect); + } + else { + POP_I64(expect_64); + } + POP_I32(addr); + + // Get referenced address and store it in `maddr` + JitReg memory_data = get_memory_data_reg(cc->jit_frame, 0); + JitReg offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) + goto fail; + CHECK_ALIGNMENT(offset1); + + JitReg maddr = jit_cc_new_reg_ptr(cc); + GEN_INSN(ADD, maddr, memory_data, offset1); + + // Prepare `wasm_runtime_atomic_wait` arguments + JitReg res = jit_cc_new_reg_I32(cc); + JitReg args[5] = { 0 }; + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = maddr; + args[2] = expect_64; + args[3] = timeout; + args[4] = NEW_CONST(I32, false); + + if (!jit_emit_callnative(cc, wasm_runtime_atomic_wait, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + // Handle return code + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, -1)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, cc->cmp_reg, + NULL)) + goto fail; + + PUSH_I32(res); + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (!jit_check_suspend_flags(cc)) + goto fail; +#endif + return true; +fail: + return false; +} + +bool +jit_compiler_op_atomic_notify(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes) +{ + // Pop atomic.notify arguments + JitReg notify_count, addr; + POP_I32(notify_count); + POP_I32(addr); + + // Get referenced address and store it in `maddr` + JitReg memory_data = get_memory_data_reg(cc->jit_frame, 0); + JitReg offset1 = check_and_seek(cc, addr, offset, bytes); + if (!offset1) + goto fail; + CHECK_ALIGNMENT(offset1); + + JitReg maddr = jit_cc_new_reg_ptr(cc); + GEN_INSN(ADD, maddr, memory_data, offset1); + + // Prepare `wasm_runtime_atomic_notify` arguments + JitReg res = jit_cc_new_reg_I32(cc); + JitReg args[3] = { 0 }; + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = maddr; + args[2] = notify_count; + + if (!jit_emit_callnative(cc, wasm_runtime_atomic_notify, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + // Handle return code + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + PUSH_I32(res); + return true; +fail: + return false; +} + +bool +jit_compiler_op_atomic_fence(JitCompContext *cc) +{ + GEN_INSN(FENCE); + return true; +} +#endif diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_memory.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_memory.h new file mode 100644 index 0000000..6565cdc --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_memory.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_MEMORY_H_ +#define _JIT_EMIT_MEMORY_H_ + +#include "../jit_compiler.h" +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../../common/wasm_shared_memory.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_i32_load(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool sign, bool atomic); + +bool +jit_compile_op_i64_load(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool sign, bool atomic); + +bool +jit_compile_op_f32_load(JitCompContext *cc, uint32 align, uint32 offset); + +bool +jit_compile_op_f64_load(JitCompContext *cc, uint32 align, uint32 offset); + +bool +jit_compile_op_i32_store(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool atomic); + +bool +jit_compile_op_i64_store(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes, bool atomic); + +bool +jit_compile_op_f32_store(JitCompContext *cc, uint32 align, uint32 offset); + +bool +jit_compile_op_f64_store(JitCompContext *cc, uint32 align, uint32 offset); + +bool +jit_compile_op_memory_size(JitCompContext *cc, uint32 mem_idx); + +bool +jit_compile_op_memory_grow(JitCompContext *cc, uint32 mem_idx); + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +jit_compile_op_memory_init(JitCompContext *cc, uint32 mem_idx, uint32 seg_idx); + +bool +jit_compile_op_data_drop(JitCompContext *cc, uint32 seg_idx); + +bool +jit_compile_op_memory_copy(JitCompContext *cc, uint32 src_mem_idx, + uint32 dst_mem_idx); + +bool +jit_compile_op_memory_fill(JitCompContext *cc, uint32 mem_idx); +#endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 +bool +jit_compile_op_atomic_rmw(JitCompContext *cc, uint8 atomic_op, uint8 op_type, + uint32 align, uint32 offset, uint32 bytes); + +bool +jit_compile_op_atomic_cmpxchg(JitCompContext *cc, uint8 op_type, uint32 align, + uint32 offset, uint32 bytes); + +bool +jit_compile_op_atomic_wait(JitCompContext *cc, uint8 op_type, uint32 align, + uint32 offset, uint32 bytes); + +bool +jit_compiler_op_atomic_notify(JitCompContext *cc, uint32 align, uint32 offset, + uint32 bytes); + +bool +jit_compiler_op_atomic_fence(JitCompContext *cc); +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_MEMORY_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_numberic.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_numberic.c new file mode 100644 index 0000000..00f608f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_numberic.c @@ -0,0 +1,1719 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_numberic.h" +#include "jit_emit_exception.h" +#include "jit_emit_control.h" +#include "jit_emit_function.h" +#include "../jit_frontend.h" +#include "../jit_codegen.h" + +#define PUSH_INT(v) \ + do { \ + if (is_i32) \ + PUSH_I32(v); \ + else \ + PUSH_I64(v); \ + } while (0) + +#define POP_INT(v) \ + do { \ + if (is_i32) \ + POP_I32(v); \ + else \ + POP_I64(v); \ + } while (0) + +#define PUSH_FLOAT(v) \ + do { \ + if (is_f32) \ + PUSH_F32(v); \ + else \ + PUSH_F64(v); \ + } while (0) + +#define POP_FLOAT(v) \ + do { \ + if (is_f32) \ + POP_F32(v); \ + else \ + POP_F64(v); \ + } while (0) + +#define DEF_INT_UNARY_OP(op, err) \ + do { \ + JitReg res, operand; \ + POP_INT(operand); \ + if (!(res = op)) { \ + if (err) \ + jit_set_last_error(cc, err); \ + goto fail; \ + } \ + PUSH_INT(res); \ + } while (0) + +#define DEF_INT_BINARY_OP(op, err) \ + do { \ + JitReg res, left, right; \ + POP_INT(right); \ + POP_INT(left); \ + if (!(res = op)) { \ + if (err) \ + jit_set_last_error(cc, err); \ + goto fail; \ + } \ + PUSH_INT(res); \ + } while (0) + +#define DEF_FP_UNARY_OP(op, err) \ + do { \ + JitReg res, operand; \ + POP_FLOAT(operand); \ + if (!(res = op)) { \ + if (err) \ + jit_set_last_error(cc, err); \ + goto fail; \ + } \ + PUSH_FLOAT(res); \ + } while (0) + +#define DEF_FP_BINARY_OP(op, err) \ + do { \ + JitReg res, left, right; \ + POP_FLOAT(right); \ + POP_FLOAT(left); \ + if (!(res = op)) { \ + if (err) \ + jit_set_last_error(cc, err); \ + goto fail; \ + } \ + PUSH_FLOAT(res); \ + } while (0) + +static uint32 +clz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 0x80000000)) { + num++; + type <<= 1; + } + return num; +} + +static uint64 +clz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 0x8000000000000000LL)) { + num++; + type <<= 1; + } + return num; +} + +static uint32 +ctz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static uint64 +ctz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static uint32 +popcnt32(uint32 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static uint64 +popcnt64(uint64 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +bool +jit_compile_op_i32_clz(JitCompContext *cc) +{ + JitReg value, res; + + POP_I32(value); + if (jit_reg_is_const(value)) { + uint32 i32 = jit_cc_get_const_I32(cc, value); + PUSH_I32(NEW_CONST(I32, clz32(i32))); + return true; + } + + res = jit_cc_new_reg_I32(cc); + GEN_INSN(CLZ, res, value); + PUSH_I32(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_ctz(JitCompContext *cc) +{ + JitReg value, res = jit_cc_new_reg_I32(cc); + + POP_I32(value); + if (jit_reg_is_const(value)) { + uint32 i32 = jit_cc_get_const_I32(cc, value); + PUSH_I32(NEW_CONST(I32, ctz32(i32))); + return true; + } + + res = jit_cc_new_reg_I32(cc); + GEN_INSN(CTZ, res, value); + PUSH_I32(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_popcnt(JitCompContext *cc) +{ + JitReg value, res; + + POP_I32(value); + if (jit_reg_is_const(value)) { + uint32 i32 = jit_cc_get_const_I32(cc, value); + PUSH_I32(NEW_CONST(I32, popcnt32(i32))); + return true; + } + + res = jit_cc_new_reg_I32(cc); + GEN_INSN(POPCNT, res, value); + PUSH_I32(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_clz(JitCompContext *cc) +{ + JitReg value, res; + + POP_I64(value); + if (jit_reg_is_const(value)) { + uint64 i64 = jit_cc_get_const_I64(cc, value); + PUSH_I64(NEW_CONST(I64, clz64(i64))); + return true; + } + + res = jit_cc_new_reg_I64(cc); + GEN_INSN(CLZ, res, value); + PUSH_I64(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_ctz(JitCompContext *cc) +{ + JitReg value, res; + + POP_I64(value); + if (jit_reg_is_const(value)) { + uint64 i64 = jit_cc_get_const_I64(cc, value); + PUSH_I64(NEW_CONST(I64, ctz64(i64))); + return true; + } + + res = jit_cc_new_reg_I64(cc); + GEN_INSN(CTZ, res, value); + PUSH_I64(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i64_popcnt(JitCompContext *cc) +{ + JitReg value, res; + + POP_I64(value); + if (jit_reg_is_const(value)) { + uint64 i64 = jit_cc_get_const_I64(cc, value); + PUSH_I64(NEW_CONST(I64, popcnt64(i64))); + return true; + } + + res = jit_cc_new_reg_I64(cc); + GEN_INSN(POPCNT, res, value); + PUSH_I64(res); + return true; +fail: + return false; +} + +#define IS_CONST_ALL_ONE(val, is_i32) \ + (jit_reg_is_const(val) \ + && ((is_i32 && jit_cc_get_const_I32(cc, val) == -1) \ + || (!is_i32 && jit_cc_get_const_I64(cc, val) == -1LL))) + +#define IS_CONST_ZERO(val) \ + (jit_reg_is_const(val) \ + && ((is_i32 && jit_cc_get_const_I32(cc, val) == 0) \ + || (!is_i32 && jit_cc_get_const_I64(cc, val) == 0))) + +/* macros for integer binary operations (ibinop) */ + +#if defined(__GNUC__) +#define NO_SANITIZER_INTEGER \ + __attribute__((no_sanitize("signed-integer-overflow"))) +#else +#define NO_SANITIZER_INTEGER +#endif + +#define __DEF_BI_INT_CONST_OPS(bits, opname, op) \ + NO_SANITIZER_INTEGER \ + static int##bits do_i##bits##_const_##opname(int##bits lhs, int##bits rhs) \ + { \ + return lhs op rhs; \ + } + +#define DEF_BI_INT_CONST_OPS(opname, op) \ + __DEF_BI_INT_CONST_OPS(32, opname, op) \ + __DEF_BI_INT_CONST_OPS(64, opname, op) + +#define DEF_UNI_INT_CONST_OPS(opname) \ + static JitReg compile_int_##opname##_consts( \ + JitCompContext *cc, JitReg left, JitReg right, bool is_i32) + +typedef JitReg (*uni_const_handler)(JitCompContext *, JitReg, JitReg, bool); +typedef int32 (*bin_i32_consts_handler)(int32, int32); +typedef int64 (*bin_i64_consts_handler)(int64, int64); + +/* ibinopt for integer binary operations */ +static JitReg +compile_op_ibinopt_const(JitCompContext *cc, JitReg left, JitReg right, + bool is_i32, uni_const_handler handle_one_const, + bin_i32_consts_handler handle_two_i32_const, + bin_i64_consts_handler handle_two_i64_const) +{ + JitReg res; + + if (jit_reg_is_const(left) && jit_reg_is_const(right)) { + if (is_i32) { + int32 left_val = jit_cc_get_const_I32(cc, left); + int32 right_val = jit_cc_get_const_I32(cc, right); + res = NEW_CONST(I32, handle_two_i32_const(left_val, right_val)); + } + else { + int64 left_val = jit_cc_get_const_I64(cc, left); + int64 right_val = jit_cc_get_const_I64(cc, right); + res = NEW_CONST(I64, handle_two_i64_const(left_val, right_val)); + } + goto shortcut; + } + + if (jit_reg_is_const(left) || jit_reg_is_const(right)) { + res = handle_one_const(cc, left, right, is_i32); + if (res) + goto shortcut; + } + + return 0; +shortcut: + return res; +} + +#define CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, opname) \ + compile_op_ibinopt_const(cc, left, right, is_i32, \ + compile_int_##opname##_consts, \ + do_i32_const_##opname, do_i64_const_##opname) + +DEF_UNI_INT_CONST_OPS(add) +{ + /* If one of the operands is 0, just return the other */ + if (IS_CONST_ZERO(left)) + return right; + if (IS_CONST_ZERO(right)) + return left; + + return 0; +} + +DEF_BI_INT_CONST_OPS(add, +) + +static JitReg +compile_int_add(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, add); + if (res) + goto shortcut; + + /* Build add */ + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(ADD, res, left, right); + +shortcut: + return res; +} + +DEF_UNI_INT_CONST_OPS(sub) +{ + /* If the right operand is 0, just return the left */ + if (IS_CONST_ZERO(right)) + return left; + + return 0; +} + +DEF_BI_INT_CONST_OPS(sub, -) + +static JitReg +compile_int_sub(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, sub); + if (res) + goto shortcut; + + /* Build sub */ + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(SUB, res, left, right); + +shortcut: + return res; +} + +DEF_UNI_INT_CONST_OPS(mul) +{ + /* If one of the operands is 0, just return constant 0 */ + if (IS_CONST_ZERO(left) || IS_CONST_ZERO(right)) + return is_i32 ? NEW_CONST(I32, 0) : NEW_CONST(I64, 0); + + return 0; +} + +static int32 +do_i32_const_mul(int32 lhs, int32 rhs) +{ + return (int32)((uint64)lhs * (uint64)rhs); +} + +static int64 +do_i64_const_mul(int64 lhs, int64 rhs) +{ + return (int64)((uint64)lhs * (uint64)rhs); +} + +static JitReg +compile_int_mul(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, mul); + if (res) + goto shortcut; + + /* Build mul */ + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(MUL, res, left, right); + +shortcut: + return res; +} + +static bool +compile_int_div_no_check(JitCompContext *cc, IntArithmetic arith_op, + bool is_i32, JitReg left, JitReg right, JitReg res) +{ +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg eax_hreg = jit_codegen_get_hreg_by_name("eax"); + JitReg edx_hreg = jit_codegen_get_hreg_by_name("edx"); + JitReg rax_hreg = jit_codegen_get_hreg_by_name("rax"); + JitReg rdx_hreg = jit_codegen_get_hreg_by_name("rdx"); +#endif + + if (jit_reg_is_const(right) && jit_reg_is_const(left)) { + if (INT_DIV_S == arith_op || INT_REM_S == arith_op) { + if (is_i32) { + int32 lhs = jit_cc_get_const_I32(cc, left); + int32 rhs = jit_cc_get_const_I32(cc, right); + if (INT_DIV_S == arith_op) { + res = NEW_CONST(I32, lhs / rhs); + } + else { + res = NEW_CONST(I32, lhs % rhs); + } + PUSH_I32(res); + return true; + } + else { + int64 lhs = jit_cc_get_const_I64(cc, left); + int64 rhs = jit_cc_get_const_I64(cc, right); + if (INT_DIV_S == arith_op) { + res = NEW_CONST(I64, lhs / rhs); + } + else { + res = NEW_CONST(I64, lhs % rhs); + } + PUSH_I64(res); + return true; + } + } + else { + if (is_i32) { + uint32 lhs = (uint32)jit_cc_get_const_I32(cc, left); + uint32 rhs = (uint32)jit_cc_get_const_I32(cc, right); + if (INT_DIV_U == arith_op) { + res = NEW_CONST(I32, lhs / rhs); + } + else { + res = NEW_CONST(I32, lhs % rhs); + } + PUSH_I32(res); + return true; + } + else { + uint64 lhs = (uint64)jit_cc_get_const_I64(cc, left); + uint64 rhs = (uint64)jit_cc_get_const_I64(cc, right); + if (INT_DIV_U == arith_op) { + res = NEW_CONST(I64, lhs / rhs); + } + else { + res = NEW_CONST(I64, lhs % rhs); + } + PUSH_I64(res); + return true; + } + } + } + + switch (arith_op) { +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + case INT_DIV_S: + case INT_DIV_U: + { + JitInsn *insn = NULL, *insn1 = NULL; + + if (is_i32) { + GEN_INSN(MOV, eax_hreg, left); + if (arith_op == INT_DIV_S) + insn = GEN_INSN(DIV_S, eax_hreg, eax_hreg, right); + else + insn = GEN_INSN(DIV_U, eax_hreg, eax_hreg, right); + } + else { + GEN_INSN(MOV, rax_hreg, left); + if (arith_op == INT_DIV_S) + insn = GEN_INSN(DIV_S, rax_hreg, rax_hreg, right); + else + insn = GEN_INSN(DIV_U, rax_hreg, rax_hreg, right); + } + + if (!insn) { + goto fail; + } + if (!jit_lock_reg_in_insn(cc, insn, eax_hreg) + || !jit_lock_reg_in_insn(cc, insn, edx_hreg)) { + goto fail; + } + + if (is_i32) { + res = jit_cc_new_reg_I32(cc); + insn1 = jit_insn_new_MOV(res, eax_hreg); + } + else { + res = jit_cc_new_reg_I64(cc); + insn1 = jit_insn_new_MOV(res, rax_hreg); + } + + if (!insn1) { + jit_set_last_error(cc, "generate insn failed"); + goto fail; + } + + jit_insn_insert_after(insn, insn1); + break; + } + case INT_REM_S: + case INT_REM_U: + { + JitInsn *insn = NULL, *insn1 = NULL; + + if (is_i32) { + GEN_INSN(MOV, eax_hreg, left); + if (arith_op == INT_REM_S) + insn = GEN_INSN(REM_S, edx_hreg, eax_hreg, right); + else + insn = GEN_INSN(REM_U, edx_hreg, eax_hreg, right); + } + else { + GEN_INSN(MOV, rax_hreg, left); + if (arith_op == INT_REM_S) + insn = GEN_INSN(REM_S, rdx_hreg, rax_hreg, right); + else + insn = GEN_INSN(REM_U, rdx_hreg, rax_hreg, right); + } + + if (!insn) { + goto fail; + } + if (!jit_lock_reg_in_insn(cc, insn, eax_hreg) + || !jit_lock_reg_in_insn(cc, insn, edx_hreg)) { + goto fail; + } + + if (is_i32) { + res = jit_cc_new_reg_I32(cc); + insn1 = jit_insn_new_MOV(res, edx_hreg); + } + else { + res = jit_cc_new_reg_I64(cc); + insn1 = jit_insn_new_MOV(res, rdx_hreg); + } + + if (!insn1) { + jit_set_last_error(cc, "generate insn failed"); + goto fail; + } + + jit_insn_insert_after(insn, insn1); + break; + } +#else + case INT_DIV_S: + GEN_INSN(DIV_S, res, left, right); + break; + case INT_DIV_U: + GEN_INSN(DIV_U, res, left, right); + break; + case INT_REM_S: + GEN_INSN(REM_S, res, left, right); + break; + case INT_REM_U: + GEN_INSN(REM_U, res, left, right); + break; +#endif /* defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) */ + default: + bh_assert(0); + return false; + } + + if (is_i32) + PUSH_I32(res); + else + PUSH_I64(res); + return true; +fail: + return false; +} + +static bool +compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, + uint8 **p_frame_ip) +{ + JitReg left, right, res; + + bh_assert(arith_op == INT_DIV_S || arith_op == INT_DIV_U + || arith_op == INT_REM_S || arith_op == INT_REM_U); + + if (is_i32) { + POP_I32(right); + POP_I32(left); + res = jit_cc_new_reg_I32(cc); + } + else { + POP_I64(right); + POP_I64(left); + res = jit_cc_new_reg_I64(cc); + } + + if (jit_reg_is_const(right)) { + int64 right_val = is_i32 ? (int64)jit_cc_get_const_I32(cc, right) + : jit_cc_get_const_I64(cc, right); + + switch (right_val) { + case 0: + { + /* Directly throw exception if divided by zero */ + if (!(jit_emit_exception(cc, EXCE_INTEGER_DIVIDE_BY_ZERO, + JIT_OP_JMP, 0, NULL))) + goto fail; + + return jit_handle_next_reachable_block(cc, p_frame_ip); + } + case 1: + { + if (arith_op == INT_DIV_S || arith_op == INT_DIV_U) { + if (is_i32) + PUSH_I32(left); + else + PUSH_I64(left); + } + else { + if (is_i32) + PUSH_I32(NEW_CONST(I32, 0)); + else + PUSH_I64(NEW_CONST(I64, 0)); + } + return true; + } + case -1: + { + if (arith_op == INT_DIV_S) { + if (is_i32) + GEN_INSN(CMP, cc->cmp_reg, left, + NEW_CONST(I32, INT32_MIN)); + else + GEN_INSN(CMP, cc->cmp_reg, left, + NEW_CONST(I64, INT64_MIN)); + + /* Throw integer overflow exception if left is + INT32_MIN or INT64_MIN */ + if (!(jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, + JIT_OP_BEQ, cc->cmp_reg, NULL))) + goto fail; + + /* Push -(left) to stack */ + GEN_INSN(NEG, res, left); + if (is_i32) + PUSH_I32(res); + else + PUSH_I64(res); + return true; + } + else if (arith_op == INT_REM_S) { + if (is_i32) + PUSH_I32(NEW_CONST(I32, 0)); + else + PUSH_I64(NEW_CONST(I64, 0)); + return true; + } + else { + /* Build default div and rem */ + return compile_int_div_no_check(cc, arith_op, is_i32, left, + right, res); + } + } + default: + { + /* Build default div and rem */ + return compile_int_div_no_check(cc, arith_op, is_i32, left, + right, res); + } + } + } + else { + JitReg cmp1 = jit_cc_new_reg_I32(cc); + JitReg cmp2 = jit_cc_new_reg_I32(cc); + + GEN_INSN(CMP, cc->cmp_reg, right, + is_i32 ? NEW_CONST(I32, 0) : NEW_CONST(I64, 0)); + /* Throw integer divided by zero exception if right is zero */ + if (!(jit_emit_exception(cc, EXCE_INTEGER_DIVIDE_BY_ZERO, JIT_OP_BEQ, + cc->cmp_reg, NULL))) + goto fail; + + switch (arith_op) { + case INT_DIV_S: + { + /* Check integer overflow */ + GEN_INSN(CMP, cc->cmp_reg, left, + is_i32 ? NEW_CONST(I32, INT32_MIN) + : NEW_CONST(I64, INT64_MIN)); + GEN_INSN(SELECTEQ, cmp1, cc->cmp_reg, NEW_CONST(I32, 1), + NEW_CONST(I32, 0)); + GEN_INSN(CMP, cc->cmp_reg, right, + is_i32 ? NEW_CONST(I32, -1) : NEW_CONST(I64, -1LL)); + GEN_INSN(SELECTEQ, cmp2, cc->cmp_reg, NEW_CONST(I32, 1), + NEW_CONST(I32, 0)); + GEN_INSN(AND, cmp1, cmp1, cmp2); + GEN_INSN(CMP, cc->cmp_reg, cmp1, NEW_CONST(I32, 1)); + /* Throw integer overflow exception if left is INT32_MIN or + INT64_MIN, and right is -1 */ + if (!(jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BEQ, + cc->cmp_reg, NULL))) + goto fail; + + /* Build default div and rem */ + return compile_int_div_no_check(cc, arith_op, is_i32, left, + right, res); + } + case INT_REM_S: + { + JitReg left1 = + is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + + GEN_INSN(CMP, cc->cmp_reg, right, + is_i32 ? NEW_CONST(I32, -1) : NEW_CONST(I64, -1LL)); + /* Don't generate `SELECTEQ left, cmp_reg, 0, left` since + left might be const, use left1 instead */ + if (is_i32) + GEN_INSN(SELECTEQ, left1, cc->cmp_reg, NEW_CONST(I32, 0), + left); + else + GEN_INSN(SELECTEQ, left1, cc->cmp_reg, NEW_CONST(I64, 0), + left); + /* Build default div and rem */ + return compile_int_div_no_check(cc, arith_op, is_i32, left1, + right, res); + } + default: + { + /* Build default div and rem */ + return compile_int_div_no_check(cc, arith_op, is_i32, left, + right, res); + } + } + } + +fail: + return false; +} + +static bool +compile_op_int_arithmetic(JitCompContext *cc, IntArithmetic arith_op, + bool is_i32, uint8 **p_frame_ip) +{ + switch (arith_op) { + case INT_ADD: + DEF_INT_BINARY_OP(compile_int_add(cc, left, right, is_i32), + "compile int add fail."); + return true; + case INT_SUB: + DEF_INT_BINARY_OP(compile_int_sub(cc, left, right, is_i32), + "compile int sub fail."); + return true; + case INT_MUL: + DEF_INT_BINARY_OP(compile_int_mul(cc, left, right, is_i32), + "compile int mul fail."); + return true; + case INT_DIV_S: + case INT_DIV_U: + case INT_REM_S: + case INT_REM_U: + return compile_int_div(cc, arith_op, is_i32, p_frame_ip); + default: + bh_assert(0); + return false; + } + +fail: + return false; +} + +bool +jit_compile_op_i32_arithmetic(JitCompContext *cc, IntArithmetic arith_op, + uint8 **p_frame_ip) +{ + return compile_op_int_arithmetic(cc, arith_op, true, p_frame_ip); +} + +bool +jit_compile_op_i64_arithmetic(JitCompContext *cc, IntArithmetic arith_op, + uint8 **p_frame_ip) +{ + return compile_op_int_arithmetic(cc, arith_op, false, p_frame_ip); +} + +DEF_UNI_INT_CONST_OPS(and) +{ + JitReg res; + if (IS_CONST_ZERO(left) || IS_CONST_ZERO(right)) { + res = is_i32 ? NEW_CONST(I32, 0) : NEW_CONST(I64, 0); + goto shortcut; + } + + if (IS_CONST_ALL_ONE(left, is_i32)) { + res = right; + goto shortcut; + } + + if (IS_CONST_ALL_ONE(right, is_i32)) { + res = left; + goto shortcut; + } + + return 0; +shortcut: + return res; +} + +DEF_BI_INT_CONST_OPS(and, &) + +static JitReg +compile_int_and(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; + + /* shortcuts */ + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, and); + if (res) + goto shortcut; + + /* do and */ + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(AND, res, left, right); + +shortcut: + return res; +} + +DEF_UNI_INT_CONST_OPS(or) +{ + JitReg res; + + if (IS_CONST_ZERO(left)) { + res = right; + goto shortcut; + } + + if (IS_CONST_ZERO(right)) { + res = left; + goto shortcut; + } + + if (IS_CONST_ALL_ONE(left, is_i32) || IS_CONST_ALL_ONE(right, is_i32)) { + res = is_i32 ? NEW_CONST(I32, -1) : NEW_CONST(I64, -1LL); + goto shortcut; + } + + return 0; +shortcut: + return res; +} + +DEF_BI_INT_CONST_OPS(or, |) + +static JitReg +compile_int_or(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; + + /* shortcuts */ + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, or); + if (res) + goto shortcut; + + /* do or */ + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(OR, res, left, right); + +shortcut: + return res; +} + +DEF_UNI_INT_CONST_OPS(xor) +{ + if (IS_CONST_ZERO(left)) + return right; + + if (IS_CONST_ZERO(right)) + return left; + + return 0; +} + +DEF_BI_INT_CONST_OPS(xor, ^) + +static JitReg +compile_int_xor(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; + + /* shortcuts */ + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, xor); + if (res) + goto shortcut; + + /* do xor */ + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(XOR, res, left, right); + +shortcut: + return res; +} + +static bool +compile_op_int_bitwise(JitCompContext *cc, IntBitwise arith_op, bool is_i32) +{ + JitReg left, right, res; + + POP_INT(right); + POP_INT(left); + + switch (arith_op) { + case INT_AND: + { + res = compile_int_and(cc, left, right, is_i32); + break; + } + case INT_OR: + { + res = compile_int_or(cc, left, right, is_i32); + break; + } + case INT_XOR: + { + res = compile_int_xor(cc, left, right, is_i32); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + PUSH_INT(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_bitwise(JitCompContext *cc, IntBitwise bitwise_op) +{ + return compile_op_int_bitwise(cc, bitwise_op, true); +} + +bool +jit_compile_op_i64_bitwise(JitCompContext *cc, IntBitwise bitwise_op) +{ + return compile_op_int_bitwise(cc, bitwise_op, false); +} + +DEF_UNI_INT_CONST_OPS(shl) +{ + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) { + return left; + } + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(SHL, res, left, right); + return res; + } + return 0; +} + +DEF_UNI_INT_CONST_OPS(shrs) +{ + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left) + || IS_CONST_ALL_ONE(left, is_i32)) { + return left; + } + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(SHRS, res, left, right); + return res; + } + return 0; +} + +DEF_UNI_INT_CONST_OPS(shru) +{ + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) { + return left; + } + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(SHRU, res, left, right); + return res; + } + return 0; +} + +static int32 +do_i32_const_shl(int32 lhs, int32 rhs) +{ + rhs &= 31; + return (int32)((uint32)lhs << (uint32)rhs); +} + +static int64 +do_i64_const_shl(int64 lhs, int64 rhs) +{ + rhs &= 63LL; + return (uint64)lhs << (uint64)rhs; +} + +DEF_BI_INT_CONST_OPS(shrs, >>) + +static int32 +do_i32_const_shru(int32 lhs, int32 rhs) +{ + rhs &= 31; + return (uint32)lhs >> rhs; +} + +static int64 +do_i64_const_shru(int64 lhs, int64 rhs) +{ + rhs &= 63LL; + return (uint64)lhs >> rhs; +} + +typedef enum { SHL, SHRS, SHRU, ROTL, ROTR } SHIFT_OP; + +static JitReg +compile_int_shift_modulo(JitCompContext *cc, JitReg rhs, bool is_i32, + SHIFT_OP op) +{ + JitReg res; + + if (jit_reg_is_const(rhs)) { + if (is_i32) { + int32 val = jit_cc_get_const_I32(cc, rhs); + val = val & 0x1f; + res = NEW_CONST(I32, val); + } + else { + int64 val = jit_cc_get_const_I64(cc, rhs); + val = val & 0x3f; + res = NEW_CONST(I64, val); + } + } + else { + if (op == ROTL || op == ROTR) { + /* No need to generate AND insn as the result + is same for rotate shift */ + res = rhs; + } + else if (is_i32) { + res = jit_cc_new_reg_I32(cc); + GEN_INSN(AND, res, rhs, NEW_CONST(I32, 0x1f)); + } + else { + res = jit_cc_new_reg_I64(cc); + GEN_INSN(AND, res, rhs, NEW_CONST(I64, 0x3f)); + } + } + + return res; +} + +static JitReg +mov_left_to_reg(JitCompContext *cc, bool is_i32, JitReg left) +{ + JitReg res = left; + /* left needs to be a variable */ + if (jit_reg_is_const(left)) { + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(MOV, res, left); + } + return res; +} + +static JitReg +compile_int_shl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx"); + JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx"); + JitInsn *insn = NULL; +#endif + + right = compile_int_shift_modulo(cc, right, is_i32, SHL); + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, shl); + if (res) + goto shortcut; + + left = mov_left_to_reg(cc, is_i32, left); + + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right); + insn = GEN_INSN(SHL, res, left, is_i32 ? ecx_hreg : rcx_hreg); + if (jit_get_last_error(cc) || !jit_lock_reg_in_insn(cc, insn, ecx_hreg)) { + goto fail; + } +#else + GEN_INSN(SHL, res, left, right); + if (jit_get_last_error(cc)) { + goto fail; + } +#endif + +shortcut: + return res; +fail: + return (JitReg)0; +} + +static JitReg +compile_int_shrs(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx"); + JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx"); + JitInsn *insn = NULL; +#endif + + right = compile_int_shift_modulo(cc, right, is_i32, SHRS); + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, shrs); + if (res) + goto shortcut; + + left = mov_left_to_reg(cc, is_i32, left); + + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right); + insn = GEN_INSN(SHRS, res, left, is_i32 ? ecx_hreg : rcx_hreg); + if (jit_get_last_error(cc) || !jit_lock_reg_in_insn(cc, insn, ecx_hreg)) { + goto fail; + } +#else + GEN_INSN(SHRS, res, left, right); + if (jit_get_last_error(cc)) { + goto fail; + } +#endif + +shortcut: + return res; +fail: + return (JitReg)0; +} + +static JitReg +compile_int_shru(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx"); + JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx"); + JitInsn *insn = NULL; +#endif + + right = compile_int_shift_modulo(cc, right, is_i32, SHRU); + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, shru); + if (res) + goto shortcut; + + left = mov_left_to_reg(cc, is_i32, left); + + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right); + insn = GEN_INSN(SHRU, res, left, is_i32 ? ecx_hreg : rcx_hreg); + if (jit_get_last_error(cc) || !jit_lock_reg_in_insn(cc, insn, ecx_hreg)) { + goto fail; + } +#else + GEN_INSN(SHRU, res, left, right); + if (jit_get_last_error(cc)) { + goto fail; + } +#endif + +shortcut: + return res; +fail: + return (JitReg)0; +} + +DEF_UNI_INT_CONST_OPS(rotl) +{ + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left) + || IS_CONST_ALL_ONE(left, is_i32)) + return left; + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(ROTL, res, left, right); + return res; + } + + return 0; +} + +static int32 +do_i32_const_rotl(int32 lhs, int32 rhs) +{ + uint32 n = (uint32)lhs; + uint32 d = (uint32)rhs; + return (n << d) | (n >> (32 - d)); +} + +static int64 +do_i64_const_rotl(int64 lhs, int64 rhs) +{ + uint64 n = (uint64)lhs; + uint64 d = (uint64)rhs; + return (n << d) | (n >> (64 - d)); +} + +static JitReg +compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx"); + JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx"); + JitInsn *insn = NULL; +#endif + + right = compile_int_shift_modulo(cc, right, is_i32, ROTL); + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, rotl); + if (res) + goto shortcut; + + left = mov_left_to_reg(cc, is_i32, left); + + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right); + insn = GEN_INSN(ROTL, res, left, is_i32 ? ecx_hreg : rcx_hreg); + if (jit_get_last_error(cc) || !jit_lock_reg_in_insn(cc, insn, ecx_hreg)) { + goto fail; + } +#else + GEN_INSN(ROTL, res, left, right); + if (jit_get_last_error(cc)) { + goto fail; + } +#endif + +shortcut: + return res; +fail: + return (JitReg)0; +} + +DEF_UNI_INT_CONST_OPS(rotr) +{ + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left) + || IS_CONST_ALL_ONE(left, is_i32)) + return left; + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(ROTR, res, left, right); + return res; + } + + return 0; +} + +static int32 +do_i32_const_rotr(int32 lhs, int32 rhs) +{ + uint32 n = (uint32)lhs; + uint32 d = (uint32)rhs; + return (n >> d) | (n << (32 - d)); +} + +static int64 +do_i64_const_rotr(int64 lhs, int64 rhs) +{ + uint64 n = (uint64)lhs; + uint64 d = (uint64)rhs; + return (n >> d) | (n << (64 - d)); +} + +static JitReg +compile_int_rotr(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) +{ + JitReg res; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx"); + JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx"); + JitInsn *insn = NULL; +#endif + + right = compile_int_shift_modulo(cc, right, is_i32, ROTR); + + res = CHECK_AND_PROCESS_INT_CONSTS(cc, left, right, is_i32, rotr); + if (res) + goto shortcut; + + left = mov_left_to_reg(cc, is_i32, left); + + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right); + insn = GEN_INSN(ROTR, res, left, is_i32 ? ecx_hreg : rcx_hreg); + if (jit_get_last_error(cc) || !jit_lock_reg_in_insn(cc, insn, ecx_hreg)) { + goto fail; + } +#else + GEN_INSN(ROTR, res, left, right); + if (jit_get_last_error(cc)) { + goto fail; + } +#endif + +shortcut: + return res; +fail: + return (JitReg)0; +} + +static bool +compile_op_int_shift(JitCompContext *cc, IntShift shift_op, bool is_i32) +{ + JitReg left, right, res; + + POP_INT(right); + POP_INT(left); + + switch (shift_op) { + case INT_SHL: + { + res = compile_int_shl(cc, left, right, is_i32); + break; + } + case INT_SHR_S: + { + res = compile_int_shrs(cc, left, right, is_i32); + break; + } + case INT_SHR_U: + { + res = compile_int_shru(cc, left, right, is_i32); + break; + } + case INT_ROTL: + { + res = compile_int_rotl(cc, left, right, is_i32); + break; + } + case INT_ROTR: + { + res = compile_int_rotr(cc, left, right, is_i32); + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + PUSH_INT(res); + return true; +fail: + return false; +} + +bool +jit_compile_op_i32_shift(JitCompContext *cc, IntShift shift_op) +{ + return compile_op_int_shift(cc, shift_op, true); +} + +bool +jit_compile_op_i64_shift(JitCompContext *cc, IntShift shift_op) +{ + return compile_op_int_shift(cc, shift_op, false); +} + +static float32 +negf(float32 f32) +{ + return -f32; +} + +static float64 +neg(float64 f64) +{ + return -f64; +} + +static bool +compile_op_float_math(JitCompContext *cc, FloatMath math_op, bool is_f32) +{ + JitReg value, res; + void *func = NULL; + + if (is_f32) + res = jit_cc_new_reg_F32(cc); + else + res = jit_cc_new_reg_F64(cc); + + if (is_f32) + POP_F32(value); + else + POP_F64(value); + + switch (math_op) { + case FLOAT_ABS: + /* TODO: andps 0x7fffffffffffffff */ + func = is_f32 ? (void *)fabsf : (void *)fabs; + break; + case FLOAT_NEG: + /* TODO: xorps 0x8000000000000000 */ + func = is_f32 ? (void *)negf : (void *)neg; + break; + case FLOAT_CEIL: + func = is_f32 ? (void *)ceilf : (void *)ceil; + break; + case FLOAT_FLOOR: + func = is_f32 ? (void *)floorf : (void *)floor; + break; + case FLOAT_TRUNC: + func = is_f32 ? (void *)truncf : (void *)trunc; + break; + case FLOAT_NEAREST: + func = is_f32 ? (void *)rintf : (void *)rint; + break; + case FLOAT_SQRT: + func = is_f32 ? (void *)sqrtf : (void *)sqrt; + break; + default: + bh_assert(0); + goto fail; + } + + if (!jit_emit_callnative(cc, func, res, &value, 1)) { + goto fail; + } + + if (is_f32) + PUSH_F32(res); + else + PUSH_F64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_math(JitCompContext *cc, FloatMath math_op) +{ + return compile_op_float_math(cc, math_op, true); +} + +bool +jit_compile_op_f64_math(JitCompContext *cc, FloatMath math_op) +{ + return compile_op_float_math(cc, math_op, false); +} + +static float32 +f32_min(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static float32 +f32_max(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static float64 +f64_min(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static float64 +f64_max(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static bool +compile_op_float_min_max(JitCompContext *cc, FloatArithmetic arith_op, + bool is_f32, JitReg lhs, JitReg rhs, JitReg *out) +{ + JitReg res, args[2]; + void *func; + + res = is_f32 ? jit_cc_new_reg_F32(cc) : jit_cc_new_reg_F64(cc); + if (arith_op == FLOAT_MIN) + func = is_f32 ? (void *)f32_min : (void *)f64_min; + else + func = is_f32 ? (void *)f32_max : (void *)f64_max; + + args[0] = lhs; + args[1] = rhs; + if (!jit_emit_callnative(cc, func, res, args, 2)) + return false; + + *out = res; + return true; +} + +static bool +compile_op_float_arithmetic(JitCompContext *cc, FloatArithmetic arith_op, + bool is_f32) +{ + JitReg lhs, rhs, res; + + if (is_f32) { + POP_F32(rhs); + POP_F32(lhs); + res = jit_cc_new_reg_F32(cc); + } + else { + POP_F64(rhs); + POP_F64(lhs); + res = jit_cc_new_reg_F64(cc); + } + + switch (arith_op) { + case FLOAT_ADD: + { + GEN_INSN(ADD, res, lhs, rhs); + break; + } + case FLOAT_SUB: + { + GEN_INSN(SUB, res, lhs, rhs); + break; + } + case FLOAT_MUL: + { + GEN_INSN(MUL, res, lhs, rhs); + break; + } + case FLOAT_DIV: + { + GEN_INSN(DIV_S, res, lhs, rhs); + break; + } + case FLOAT_MIN: + case FLOAT_MAX: + { + if (!compile_op_float_min_max(cc, arith_op, is_f32, lhs, rhs, &res)) + goto fail; + break; + } + default: + { + bh_assert(0); + goto fail; + } + } + + if (is_f32) + PUSH_F32(res); + else + PUSH_F64(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f32_arithmetic(JitCompContext *cc, FloatArithmetic arith_op) +{ + return compile_op_float_arithmetic(cc, arith_op, true); +} + +bool +jit_compile_op_f64_arithmetic(JitCompContext *cc, FloatArithmetic arith_op) +{ + return compile_op_float_arithmetic(cc, arith_op, false); +} + +bool +jit_compile_op_f32_copysign(JitCompContext *cc) +{ + JitReg res; + JitReg args[2] = { 0 }; + + POP_F32(args[1]); + POP_F32(args[0]); + + res = jit_cc_new_reg_F32(cc); + if (!jit_emit_callnative(cc, copysignf, res, args, 2)) + goto fail; + + PUSH_F32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_f64_copysign(JitCompContext *cc) +{ + JitReg res; + JitReg args[2] = { 0 }; + + POP_F64(args[1]); + POP_F64(args[0]); + + res = jit_cc_new_reg_F64(cc); + if (!jit_emit_callnative(cc, copysign, res, args, 2)) + goto fail; + + PUSH_F64(res); + + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_numberic.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_numberic.h new file mode 100644 index 0000000..e73c3eb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_numberic.h @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_NUMBERIC_H_ +#define _JIT_EMIT_NUMBERIC_H_ + +#include "../jit_compiler.h" +#include "../jit_frontend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_i32_clz(JitCompContext *cc); + +bool +jit_compile_op_i32_ctz(JitCompContext *cc); + +bool +jit_compile_op_i32_popcnt(JitCompContext *cc); + +bool +jit_compile_op_i64_clz(JitCompContext *cc); + +bool +jit_compile_op_i64_ctz(JitCompContext *cc); + +bool +jit_compile_op_i64_popcnt(JitCompContext *cc); + +bool +jit_compile_op_i32_arithmetic(JitCompContext *cc, IntArithmetic arith_op, + uint8 **p_frame_ip); + +bool +jit_compile_op_i64_arithmetic(JitCompContext *cc, IntArithmetic arith_op, + uint8 **p_frame_ip); + +bool +jit_compile_op_i32_bitwise(JitCompContext *cc, IntBitwise bitwise_op); + +bool +jit_compile_op_i64_bitwise(JitCompContext *cc, IntBitwise bitwise_op); + +bool +jit_compile_op_i32_shift(JitCompContext *cc, IntShift shift_op); + +bool +jit_compile_op_i64_shift(JitCompContext *cc, IntShift shift_op); + +bool +jit_compile_op_f32_math(JitCompContext *cc, FloatMath math_op); + +bool +jit_compile_op_f64_math(JitCompContext *cc, FloatMath math_op); + +bool +jit_compile_op_f32_arithmetic(JitCompContext *cc, FloatArithmetic arith_op); + +bool +jit_compile_op_f64_arithmetic(JitCompContext *cc, FloatArithmetic arith_op); + +bool +jit_compile_op_f32_copysign(JitCompContext *cc); + +bool +jit_compile_op_f64_copysign(JitCompContext *cc); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_NUMBERIC_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_parametric.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_parametric.c new file mode 100644 index 0000000..df0b23a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_parametric.c @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_parametric.h" +#include "../jit_frontend.h" + +static bool +pop_value_from_wasm_stack(JitCompContext *cc, bool is_32bit, JitReg *p_value, + uint8 *p_type) +{ + JitValue *jit_value; + JitReg value; + uint8 type; + + if (!jit_block_stack_top(&cc->block_stack)) { + jit_set_last_error(cc, "WASM block stack underflow."); + return false; + } + if (!jit_block_stack_top(&cc->block_stack)->value_stack.value_list_end) { + jit_set_last_error(cc, "WASM data stack underflow."); + return false; + } + + jit_value = jit_value_stack_pop( + &jit_block_stack_top(&cc->block_stack)->value_stack); + type = jit_value->type; + + if (p_type != NULL) { + *p_type = jit_value->type; + } + + wasm_runtime_free(jit_value); + + /* is_32: i32, f32, ref.func, ref.extern, v128 */ + if (is_32bit + && !(type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32 +#if WASM_ENABLE_REF_TYPES != 0 + || type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF +#endif + || type == VALUE_TYPE_V128)) { + jit_set_last_error(cc, "invalid WASM stack data type."); + return false; + } + /* !is_32: i64, f64 */ + if (!is_32bit && !(type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64)) { + jit_set_last_error(cc, "invalid WASM stack data type."); + return false; + } + + switch (type) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + value = pop_i32(cc->jit_frame); + break; + case VALUE_TYPE_I64: + value = pop_i64(cc->jit_frame); + break; + case VALUE_TYPE_F32: + value = pop_f32(cc->jit_frame); + break; + case VALUE_TYPE_F64: + value = pop_f64(cc->jit_frame); + break; + default: + bh_assert(0); + return false; + } + + if (p_value != NULL) { + *p_value = value; + } + return true; +} + +bool +jit_compile_op_drop(JitCompContext *cc, bool is_drop_32) +{ + if (!pop_value_from_wasm_stack(cc, is_drop_32, NULL, NULL)) + return false; + return true; +} + +bool +jit_compile_op_select(JitCompContext *cc, bool is_select_32) +{ + JitReg val1, val2, cond, selected; + uint8 val1_type, val2_type; + + POP_I32(cond); + + if (!pop_value_from_wasm_stack(cc, is_select_32, &val2, &val2_type) + || !pop_value_from_wasm_stack(cc, is_select_32, &val1, &val1_type)) { + return false; + } + + if (val1_type != val2_type) { + jit_set_last_error(cc, "invalid stack values with different type"); + return false; + } + + switch (val1_type) { + case VALUE_TYPE_I32: + selected = jit_cc_new_reg_I32(cc); + break; + case VALUE_TYPE_I64: + selected = jit_cc_new_reg_I64(cc); + break; + case VALUE_TYPE_F32: + selected = jit_cc_new_reg_F32(cc); + break; + case VALUE_TYPE_F64: + selected = jit_cc_new_reg_F64(cc); + break; + default: + bh_assert(0); + return false; + } + + GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)); + GEN_INSN(SELECTNE, selected, cc->cmp_reg, val1, val2); + PUSH(selected, val1_type); + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_parametric.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_parametric.h new file mode 100644 index 0000000..40025ed --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_parametric.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_PARAMETRIC_H_ +#define _JIT_EMIT_PARAMETRIC_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_drop(JitCompContext *cc, bool is_drop_32); + +bool +jit_compile_op_select(JitCompContext *cc, bool is_select_32); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_PARAMETRIC_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_table.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_table.c new file mode 100644 index 0000000..6da5b82 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_table.c @@ -0,0 +1,333 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_table.h" +#include "jit_emit_exception.h" +#include "jit_emit_function.h" +#include "../../interpreter/wasm_runtime.h" +#include "../jit_frontend.h" + +#if WASM_ENABLE_REF_TYPES != 0 +static void +wasm_elem_drop(WASMModuleInstance *inst, uint32 tbl_seg_idx) +{ + bh_bitmap_set_bit(inst->e->common.elem_dropped, tbl_seg_idx); +} + +bool +jit_compile_op_elem_drop(JitCompContext *cc, uint32 tbl_seg_idx) +{ + JitReg args[2] = { 0 }; + + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_seg_idx); + + return jit_emit_callnative(cc, wasm_elem_drop, 0, args, + sizeof(args) / sizeof(args[0])); +} + +bool +jit_compile_op_table_get(JitCompContext *cc, uint32 tbl_idx) +{ + JitReg elem_idx, tbl_sz, tbl_elems, elem_idx_long, offset, res; + + POP_I32(elem_idx); + + /* if (elem_idx >= tbl_sz) goto exception; */ + tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) + goto fail; + + elem_idx_long = jit_cc_new_reg_I64(cc); + GEN_INSN(I32TOI64, elem_idx_long, elem_idx); + + offset = jit_cc_new_reg_I64(cc); + GEN_INSN(MUL, offset, elem_idx_long, + NEW_CONST(I64, sizeof(table_elem_type_t))); + + res = jit_cc_new_reg_I32(cc); + tbl_elems = get_table_elems_reg(cc->jit_frame, tbl_idx); + GEN_INSN(LDI32, res, tbl_elems, offset); + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_table_set(JitCompContext *cc, uint32 tbl_idx) +{ + JitReg elem_idx, elem_val, tbl_sz, tbl_elems, elem_idx_long, offset; + + POP_I32(elem_val); + POP_I32(elem_idx); + + /* if (elem_idx >= tbl_sz) goto exception; */ + tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) + goto fail; + + elem_idx_long = jit_cc_new_reg_I64(cc); + GEN_INSN(I32TOI64, elem_idx_long, elem_idx); + + offset = jit_cc_new_reg_I64(cc); + GEN_INSN(MUL, offset, elem_idx_long, + NEW_CONST(I64, sizeof(table_elem_type_t))); + + tbl_elems = get_table_elems_reg(cc->jit_frame, tbl_idx); + GEN_INSN(STI32, elem_val, tbl_elems, offset); + + return true; +fail: + return false; +} + +static int +wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 seg_idx, + uint32 dst_offset, uint32 len, uint32 src_offset) +{ + WASMTableInstance *tbl; + WASMTableSeg *tbl_seg = inst->module->table_segments + seg_idx; + InitializerExpression *tbl_seg_init_values = NULL, *init_values; + uint32 tbl_sz, tbl_seg_len = 0, i; + table_elem_type_t *addr; + + if (!bh_bitmap_get_bit(inst->e->common.elem_dropped, seg_idx)) { + /* table segment isn't dropped */ + tbl_seg_init_values = tbl_seg->init_values; + tbl_seg_len = tbl_seg->value_count; + } + + if (offset_len_out_of_bounds(src_offset, len, tbl_seg_len)) + goto out_of_bounds; + + tbl = inst->tables[tbl_idx]; + tbl_sz = tbl->cur_size; + if (offset_len_out_of_bounds(dst_offset, len, tbl_sz)) + goto out_of_bounds; + + if (!len) + return 0; + + addr = + (table_elem_type_t *)((uint8 *)tbl + offsetof(WASMTableInstance, elems) + + dst_offset * sizeof(table_elem_type_t)); + init_values = tbl_seg_init_values + src_offset; + for (i = 0; i < len; i++) { + addr[i] = (table_elem_type_t)(uintptr_t)init_values[+i].u.ref_index; + } + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds table access"); + return -1; +} + +bool +jit_compile_op_table_init(JitCompContext *cc, uint32 tbl_idx, + uint32 tbl_seg_idx) +{ + JitReg len, src, dst, res; + JitReg args[6] = { 0 }; + + POP_I32(len); + POP_I32(src); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_idx); + args[2] = NEW_CONST(I32, tbl_seg_idx); + args[3] = dst; + args[4] = len; + args[5] = src; + + if (!jit_emit_callnative(cc, wasm_init_table, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: + return false; +} + +static int +wasm_copy_table(WASMModuleInstance *inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 dst_offset, uint32 len, + uint32 src_offset) +{ + WASMTableInstance *src_tbl, *dst_tbl; + uint32 src_tbl_sz, dst_tbl_sz; + + dst_tbl = inst->tables[dst_tbl_idx]; + dst_tbl_sz = dst_tbl->cur_size; + if (offset_len_out_of_bounds(dst_offset, len, dst_tbl_sz)) + goto out_of_bounds; + + src_tbl = inst->tables[src_tbl_idx]; + src_tbl_sz = src_tbl->cur_size; + if (offset_len_out_of_bounds(src_offset, len, src_tbl_sz)) + goto out_of_bounds; + + bh_memmove_s( + (uint8 *)dst_tbl + offsetof(WASMTableInstance, elems) + + dst_offset * sizeof(table_elem_type_t), + (uint32)((dst_tbl_sz - dst_offset) * sizeof(table_elem_type_t)), + (uint8 *)src_tbl + offsetof(WASMTableInstance, elems) + + src_offset * sizeof(table_elem_type_t), + (uint32)(len * sizeof(table_elem_type_t))); + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds table access"); + return -1; +} + +bool +jit_compile_op_table_copy(JitCompContext *cc, uint32 src_tbl_idx, + uint32 dst_tbl_idx) +{ + JitReg len, src, dst, res; + JitReg args[6] = { 0 }; + + POP_I32(len); + POP_I32(src); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, src_tbl_idx); + args[2] = NEW_CONST(I32, dst_tbl_idx); + args[3] = dst; + args[4] = len; + args[5] = src; + + if (!jit_emit_callnative(cc, wasm_copy_table, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: + return false; +} + +bool +jit_compile_op_table_size(JitCompContext *cc, uint32 tbl_idx) +{ + JitReg res; + + res = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + PUSH_I32(res); + + return true; +fail: + return false; +} + +bool +jit_compile_op_table_grow(JitCompContext *cc, uint32 tbl_idx) +{ + JitReg tbl_sz, n, val, enlarge_ret, res; + JitReg args[4] = { 0 }; + + POP_I32(n); + POP_I32(val); + + tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + + enlarge_ret = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_idx); + args[2] = n; + args[3] = val; + + if (!jit_emit_callnative(cc, wasm_enlarge_table, enlarge_ret, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + /* Convert bool to uint32 */ + GEN_INSN(AND, enlarge_ret, enlarge_ret, NEW_CONST(I32, 0xFF)); + + res = jit_cc_new_reg_I32(cc); + GEN_INSN(CMP, cc->cmp_reg, enlarge_ret, NEW_CONST(I32, 1)); + GEN_INSN(SELECTEQ, res, cc->cmp_reg, tbl_sz, NEW_CONST(I32, -1)); + PUSH_I32(res); + + /* Ensure a refresh in next get memory related registers */ + clear_table_regs(cc->jit_frame); + return true; +fail: + return false; +} + +static int +wasm_fill_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 dst_offset, + uintptr_t val, uint32 len) +{ + WASMTableInstance *tbl; + uint32 tbl_sz; + + tbl = inst->tables[tbl_idx]; + tbl_sz = tbl->cur_size; + + if (offset_len_out_of_bounds(dst_offset, len, tbl_sz)) + goto out_of_bounds; + + for (; len != 0; dst_offset++, len--) { + tbl->elems[dst_offset] = val; + } + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds table access"); + return -1; +} + +bool +jit_compile_op_table_fill(JitCompContext *cc, uint32 tbl_idx) +{ + JitReg len, val, dst, res; + JitReg args[5] = { 0 }; + + POP_I32(len); + POP_I32(val); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_idx); + args[2] = dst; + args[3] = val; + args[4] = len; + + if (!jit_emit_callnative(cc, wasm_fill_table, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: + return false; +} +#endif diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_table.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_table.h new file mode 100644 index 0000000..acfb655 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_table.h @@ -0,0 +1,47 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_TABLE_H_ +#define _JIT_EMIT_TABLE_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +bool +jit_compile_op_elem_drop(JitCompContext *cc, uint32 tbl_seg_idx); + +bool +jit_compile_op_table_get(JitCompContext *cc, uint32 tbl_idx); + +bool +jit_compile_op_table_set(JitCompContext *cc, uint32 tbl_idx); + +bool +jit_compile_op_table_init(JitCompContext *cc, uint32 tbl_idx, + uint32 tbl_seg_idx); + +bool +jit_compile_op_table_copy(JitCompContext *cc, uint32 src_tbl_idx, + uint32 dst_tbl_idx); + +bool +jit_compile_op_table_size(JitCompContext *cc, uint32 tbl_idx); + +bool +jit_compile_op_table_grow(JitCompContext *cc, uint32 tbl_idx); + +bool +jit_compile_op_table_fill(JitCompContext *cc, uint32 tbl_idx); +#endif + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif +#endif diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_variable.c b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_variable.c new file mode 100644 index 0000000..ffbf06a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_variable.c @@ -0,0 +1,312 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_emit_variable.h" +#include "jit_emit_exception.h" +#include "../jit_frontend.h" + +#define CHECK_LOCAL(idx) \ + do { \ + if (idx \ + >= wasm_func->func_type->param_count + wasm_func->local_count) { \ + jit_set_last_error(cc, "local index out of range"); \ + goto fail; \ + } \ + } while (0) + +static uint8 +get_local_type(const WASMFunction *wasm_func, uint32 local_idx) +{ + uint32 param_count = wasm_func->func_type->param_count; + return local_idx < param_count + ? wasm_func->func_type->types[local_idx] + : wasm_func->local_types[local_idx - param_count]; +} + +bool +jit_compile_op_get_local(JitCompContext *cc, uint32 local_idx) +{ + WASMFunction *wasm_func = cc->cur_wasm_func; + uint16 *local_offsets = wasm_func->local_offsets; + uint16 local_offset; + uint8 local_type; + JitReg value = 0; + + CHECK_LOCAL(local_idx); + + local_offset = local_offsets[local_idx]; + local_type = get_local_type(wasm_func, local_idx); + + switch (local_type) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + value = local_i32(cc->jit_frame, local_offset); + + break; + case VALUE_TYPE_I64: + value = local_i64(cc->jit_frame, local_offset); + break; + case VALUE_TYPE_F32: + value = local_f32(cc->jit_frame, local_offset); + break; + case VALUE_TYPE_F64: + value = local_f64(cc->jit_frame, local_offset); + break; + default: + bh_assert(0); + break; + } + + PUSH(value, local_type); + return true; +fail: + return false; +} + +bool +jit_compile_op_set_local(JitCompContext *cc, uint32 local_idx) +{ + WASMFunction *wasm_func = cc->cur_wasm_func; + uint16 *local_offsets = wasm_func->local_offsets; + uint16 local_offset; + uint8 local_type; + JitReg value; + + CHECK_LOCAL(local_idx); + + local_offset = local_offsets[local_idx]; + local_type = get_local_type(wasm_func, local_idx); + + switch (local_type) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + POP_I32(value); + set_local_i32(cc->jit_frame, local_offset, value); + break; + case VALUE_TYPE_I64: + POP_I64(value); + set_local_i64(cc->jit_frame, local_offset, value); + break; + case VALUE_TYPE_F32: + POP_F32(value); + set_local_f32(cc->jit_frame, local_offset, value); + break; + case VALUE_TYPE_F64: + POP_F64(value); + set_local_f64(cc->jit_frame, local_offset, value); + break; + default: + bh_assert(0); + break; + } + + return true; +fail: + return false; +} + +bool +jit_compile_op_tee_local(JitCompContext *cc, uint32 local_idx) +{ + WASMFunction *wasm_func = cc->cur_wasm_func; + uint16 *local_offsets = wasm_func->local_offsets; + uint16 local_offset; + uint8 local_type; + JitReg value = 0; + + CHECK_LOCAL(local_idx); + + local_offset = local_offsets[local_idx]; + local_type = get_local_type(wasm_func, local_idx); + + switch (local_type) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + POP_I32(value); + set_local_i32(cc->jit_frame, local_offset, value); + PUSH_I32(value); + break; + case VALUE_TYPE_I64: + POP_I64(value); + set_local_i64(cc->jit_frame, local_offset, value); + PUSH_I64(value); + break; + case VALUE_TYPE_F32: + POP_F32(value); + set_local_f32(cc->jit_frame, local_offset, value); + PUSH_F32(value); + break; + case VALUE_TYPE_F64: + POP_F64(value); + set_local_f64(cc->jit_frame, local_offset, value); + PUSH_F64(value); + break; + default: + bh_assert(0); + goto fail; + } + + return true; +fail: + return false; +} + +static uint8 +get_global_type(const WASMModule *module, uint32 global_idx) +{ + if (global_idx < module->import_global_count) { + const WASMGlobalImport *import_global = + &((module->import_globals + global_idx)->u.global); + return import_global->type; + } + else { + const WASMGlobal *global = + module->globals + (global_idx - module->import_global_count); + return global->type; + } +} + +bool +jit_compile_op_get_global(JitCompContext *cc, uint32 global_idx) +{ + uint32 data_offset; + uint8 global_type = 0; + JitReg value = 0; + + bh_assert(global_idx < cc->cur_wasm_module->import_global_count + + cc->cur_wasm_module->global_count); + + data_offset = + jit_frontend_get_global_data_offset(cc->cur_wasm_module, global_idx); + global_type = get_global_type(cc->cur_wasm_module, global_idx); + + switch (global_type) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + { + value = jit_cc_new_reg_I32(cc); + GEN_INSN(LDI32, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + case VALUE_TYPE_I64: + { + value = jit_cc_new_reg_I64(cc); + GEN_INSN(LDI64, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + case VALUE_TYPE_F32: + { + value = jit_cc_new_reg_F32(cc); + GEN_INSN(LDF32, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + case VALUE_TYPE_F64: + { + value = jit_cc_new_reg_F64(cc); + GEN_INSN(LDF64, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + default: + { + jit_set_last_error(cc, "unexpected global type"); + goto fail; + } + } + + PUSH(value, global_type); + + return true; +fail: + return false; +} + +bool +jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx, + bool is_aux_stack) +{ + uint32 data_offset; + uint8 global_type = 0; + JitReg value = 0; + + bh_assert(global_idx < cc->cur_wasm_module->import_global_count + + cc->cur_wasm_module->global_count); + + data_offset = + jit_frontend_get_global_data_offset(cc->cur_wasm_module, global_idx); + global_type = get_global_type(cc->cur_wasm_module, global_idx); + + switch (global_type) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: +#endif + { + POP_I32(value); + if (is_aux_stack) { + JitReg aux_stack_bound = get_aux_stack_bound_reg(cc->jit_frame); + JitReg aux_stack_bottom = + get_aux_stack_bottom_reg(cc->jit_frame); + GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bound); + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_OVERFLOW, + JIT_OP_BLEU, cc->cmp_reg, NULL))) + goto fail; + GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bottom); + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_UNDERFLOW, + JIT_OP_BGTU, cc->cmp_reg, NULL))) + goto fail; + } + GEN_INSN(STI32, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + case VALUE_TYPE_I64: + { + POP_I64(value); + GEN_INSN(STI64, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + case VALUE_TYPE_F32: + { + POP_F32(value); + GEN_INSN(STF32, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + case VALUE_TYPE_F64: + { + POP_F64(value); + GEN_INSN(STF64, value, get_module_inst_reg(cc->jit_frame), + NEW_CONST(I32, data_offset)); + break; + } + default: + { + jit_set_last_error(cc, "unexpected global type"); + goto fail; + } + } + + return true; +fail: + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_variable.h b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_variable.h new file mode 100644 index 0000000..80a1051 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/fe/jit_emit_variable.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_EMIT_VARIABLE_H_ +#define _JIT_EMIT_VARIABLE_H_ + +#include "../jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_compile_op_get_local(JitCompContext *cc, uint32 local_idx); + +bool +jit_compile_op_set_local(JitCompContext *cc, uint32 local_idx); + +bool +jit_compile_op_tee_local(JitCompContext *cc, uint32 local_idx); + +bool +jit_compile_op_get_global(JitCompContext *cc, uint32 global_idx); + +bool +jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx, + bool is_aux_stack); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _JIT_EMIT_VARIABLE_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/iwasm_fast_jit.cmake b/wasm-micro-runtime/core/iwasm/fast-jit/iwasm_fast_jit.cmake new file mode 100644 index 0000000..cd880a3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/iwasm_fast_jit.cmake @@ -0,0 +1,97 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (IWASM_FAST_JIT_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DWASM_ENABLE_FAST_JIT=1) +if (WAMR_BUILD_FAST_JIT_DUMP EQUAL 1) + add_definitions(-DWASM_ENABLE_FAST_JIT_DUMP=1) +endif () + +include_directories (${IWASM_FAST_JIT_DIR}) + +if (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + include(FetchContent) + if (NOT WAMR_BUILD_PLATFORM STREQUAL "linux-sgx") + FetchContent_Declare( + asmjit + GIT_REPOSITORY https://github.com/asmjit/asmjit.git + GIT_TAG c1019f1642a588107148f64ba54584b0ae3ec8d1 + ) + else () + FetchContent_Declare( + asmjit + GIT_REPOSITORY https://github.com/asmjit/asmjit.git + GIT_TAG c1019f1642a588107148f64ba54584b0ae3ec8d1 + PATCH_COMMAND git apply ${IWASM_FAST_JIT_DIR}/asmjit_sgx_patch.diff + ) + endif () + FetchContent_GetProperties(asmjit) + if (NOT asmjit_POPULATED) + message ("-- Fetching asmjit ..") + FetchContent_Populate(asmjit) + add_definitions(-DASMJIT_STATIC) + add_definitions(-DASMJIT_NO_DEPRECATED) + add_definitions(-DASMJIT_NO_BUILDER) + add_definitions(-DASMJIT_NO_COMPILER) + add_definitions(-DASMJIT_NO_JIT) + add_definitions(-DASMJIT_NO_LOGGING) + add_definitions(-DASMJIT_NO_TEXT) + add_definitions(-DASMJIT_NO_VALIDATION) + add_definitions(-DASMJIT_NO_INTROSPECTION) + add_definitions(-DASMJIT_NO_INTRINSICS) + add_definitions(-DASMJIT_NO_AARCH64) + add_definitions(-DASMJIT_NO_AARCH32) + include_directories("${asmjit_SOURCE_DIR}/src") + add_subdirectory(${asmjit_SOURCE_DIR} ${asmjit_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE cpp_source_asmjit + ${asmjit_SOURCE_DIR}/src/asmjit/core/*.cpp + ${asmjit_SOURCE_DIR}/src/asmjit/x86/*.cpp + ) + endif () + if (WAMR_BUILD_FAST_JIT_DUMP EQUAL 1) + FetchContent_Declare( + zycore + GIT_REPOSITORY https://github.com/zyantific/zycore-c.git + ) + FetchContent_GetProperties(zycore) + if (NOT zycore_POPULATED) + message ("-- Fetching zycore ..") + FetchContent_Populate(zycore) + option(ZYDIS_BUILD_TOOLS "" OFF) + option(ZYDIS_BUILD_EXAMPLES "" OFF) + include_directories("${zycore_SOURCE_DIR}/include") + include_directories("${zycore_BINARY_DIR}") + add_subdirectory(${zycore_SOURCE_DIR} ${zycore_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zycore ${zycore_SOURCE_DIR}/src/*.c) + endif () + FetchContent_Declare( + zydis + GIT_REPOSITORY https://github.com/zyantific/zydis.git + GIT_TAG e14a07895136182a5b53e181eec3b1c6e0b434de + ) + FetchContent_GetProperties(zydis) + if (NOT zydis_POPULATED) + message ("-- Fetching zydis ..") + FetchContent_Populate(zydis) + option(ZYDIS_BUILD_TOOLS "" OFF) + option(ZYDIS_BUILD_EXAMPLES "" OFF) + include_directories("${zydis_BINARY_DIR}") + include_directories("${zydis_SOURCE_DIR}/include") + include_directories("${zydis_SOURCE_DIR}/src") + add_subdirectory(${zydis_SOURCE_DIR} ${zydis_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zydis ${zydis_SOURCE_DIR}/src/*.c) + endif () + endif () +endif () + +file (GLOB c_source_jit ${IWASM_FAST_JIT_DIR}/*.c ${IWASM_FAST_JIT_DIR}/fe/*.c) + +if (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + file (GLOB_RECURSE cpp_source_jit_cg ${IWASM_FAST_JIT_DIR}/cg/x86-64/*.cpp) +else () + message (FATAL_ERROR "Fast JIT codegen for target ${WAMR_BUILD_TARGET} isn't implemented") +endif () + +set (IWASM_FAST_JIT_SOURCE ${c_source_jit} ${cpp_source_jit_cg} + ${cpp_source_asmjit} ${c_source_zycore} ${c_source_zydis}) diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_codecache.c b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codecache.c new file mode 100644 index 0000000..ef20747 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codecache.c @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_codecache.h" +#include "mem_alloc.h" +#include "jit_compiler.h" + +static void *code_cache_pool = NULL; +static uint32 code_cache_pool_size = 0; +static mem_allocator_t code_cache_pool_allocator = NULL; + +bool +jit_code_cache_init(uint32 code_cache_size) +{ + int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC; + int map_flags = MMAP_MAP_NONE; + + if (!(code_cache_pool = os_mmap(NULL, code_cache_size, map_prot, map_flags, + os_get_invalid_handle()))) { + return false; + } + + if (!(code_cache_pool_allocator = + mem_allocator_create(code_cache_pool, code_cache_size))) { + os_munmap(code_cache_pool, code_cache_size); + code_cache_pool = NULL; + return false; + } + + code_cache_pool_size = code_cache_size; + return true; +} + +void +jit_code_cache_destroy() +{ + mem_allocator_destroy(code_cache_pool_allocator); + os_munmap(code_cache_pool, code_cache_pool_size); +} + +void * +jit_code_cache_alloc(uint32 size) +{ + return mem_allocator_malloc(code_cache_pool_allocator, size); +} + +void +jit_code_cache_free(void *ptr) +{ + if (ptr) + mem_allocator_free(code_cache_pool_allocator, ptr); +} + +bool +jit_pass_register_jitted_code(JitCompContext *cc) +{ + WASMModuleInstance *instance; + WASMModule *module = cc->cur_wasm_module; + WASMFunction *func = cc->cur_wasm_func; + uint32 jit_func_idx = cc->cur_wasm_func_idx - module->import_function_count; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + os_mutex_lock(&module->instance_list_lock); +#endif + + module->fast_jit_func_ptrs[jit_func_idx] = func->fast_jit_jitted_code = + cc->jitted_addr_begin; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + instance = module->instance_list; + while (instance) { + if (instance->e->running_mode == Mode_Fast_JIT) + instance->fast_jit_func_ptrs[jit_func_idx] = cc->jitted_addr_begin; + instance = instance->e->next; + } + + os_mutex_unlock(&module->instance_list_lock); +#else + (void)instance; +#endif + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_codecache.h b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codecache.h new file mode 100644 index 0000000..953026a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codecache.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_CODE_CACHE_H_ +#define _JIT_CODE_CACHE_H_ + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +jit_code_cache_init(uint32 code_cache_size); + +void +jit_code_cache_destroy(); + +void * +jit_code_cache_alloc(uint32 size); + +void +jit_code_cache_free(void *ptr); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _JIT_CODE_CACHE_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_codegen.c b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codegen.c new file mode 100644 index 0000000..2bd60bb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codegen.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_compiler.h" +#include "jit_codegen.h" + +bool +jit_pass_lower_cg(JitCompContext *cc) +{ + return jit_codegen_lower(cc); +} + +bool +jit_pass_codegen(JitCompContext *cc) +{ + if (!jit_annl_enable_jitted_addr(cc)) + return false; + + return jit_codegen_gen_native(cc); +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_codegen.h b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codegen.h new file mode 100644 index 0000000..735cdda --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_codegen.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_CODEGEN_H_ +#define _JIT_CODEGEN_H_ + +#include "bh_platform.h" +#include "jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Initialize codegen module, such as instruction encoder. + * + * @return true if succeeded; false if failed. + */ +bool +jit_codegen_init(); + +/** + * Destroy codegen module, such as instruction encoder. + */ +void +jit_codegen_destroy(); + +/** + * Get hard register information of each kind. + * + * @return the JitHardRegInfo array of each kind + */ +const JitHardRegInfo * +jit_codegen_get_hreg_info(); + +/** + * Get hard register by name. + * + * @param name the name of the hard register + * + * @return the hard register of the name + */ +JitReg +jit_codegen_get_hreg_by_name(const char *name); + +/** + * Generate native code for the given compilation context + * + * @param cc the compilation context that is ready to do codegen + * + * @return true if succeeds, false otherwise + */ +bool +jit_codegen_gen_native(JitCompContext *cc); + +/** + * lower unsupported operations to supported ones for the target. + * + * @param cc the compilation context that is ready to do codegen + * + * @return true if succeeds, false otherwise + */ +bool +jit_codegen_lower(JitCompContext *cc); + +#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_JIT != 0 +void * +jit_codegen_compile_call_to_llvm_jit(const WASMType *func_type); + +void * +jit_codegen_compile_call_to_fast_jit(const WASMModule *module, uint32 func_idx); +#endif + +/** + * Dump native code in the given range to assembly. + * + * @param begin_addr begin address of the native code + * @param end_addr end address of the native code + */ +void +jit_codegen_dump_native(void *begin_addr, void *end_addr); + +int +jit_codegen_interp_jitted_glue(void *self, JitInterpSwitchInfo *info, + uint32 func_idx, void *pc); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _JIT_CODEGEN_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_compiler.c b/wasm-micro-runtime/core/iwasm/fast-jit/jit_compiler.c new file mode 100644 index 0000000..2c686dd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_compiler.c @@ -0,0 +1,291 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_compiler.h" +#include "jit_ir.h" +#include "jit_codegen.h" +#include "jit_codecache.h" +#include "../interpreter/wasm.h" + +typedef struct JitCompilerPass { + /* Name of the pass */ + const char *name; + /* The entry of the compiler pass */ + bool (*run)(JitCompContext *cc); +} JitCompilerPass; + +/* clang-format off */ +static JitCompilerPass compiler_passes[] = { + { NULL, NULL }, +#define REG_PASS(name) { #name, jit_pass_##name } + REG_PASS(dump), + REG_PASS(update_cfg), + REG_PASS(frontend), + REG_PASS(lower_cg), + REG_PASS(regalloc), + REG_PASS(codegen), + REG_PASS(register_jitted_code) +#undef REG_PASS +}; + +/* Number of compiler passes */ +#define COMPILER_PASS_NUM (sizeof(compiler_passes) / sizeof(compiler_passes[0])) + +#if WASM_ENABLE_FAST_JIT_DUMP == 0 +static const uint8 compiler_passes_without_dump[] = { + 3, 4, 5, 6, 7, 0 +}; +#else +static const uint8 compiler_passes_with_dump[] = { + 3, 2, 1, 4, 1, 5, 1, 6, 1, 7, 0 +}; +#endif + +/* The exported global data of JIT compiler */ +static JitGlobals jit_globals = { +#if WASM_ENABLE_FAST_JIT_DUMP == 0 + .passes = compiler_passes_without_dump, +#else + .passes = compiler_passes_with_dump, +#endif + .return_to_interp_from_jitted = NULL, +#if WASM_ENABLE_LAZY_JIT != 0 + .compile_fast_jit_and_then_call = NULL, +#endif +}; +/* clang-format on */ + +static bool +apply_compiler_passes(JitCompContext *cc) +{ + const uint8 *p = jit_globals.passes; + + for (; *p; p++) { + /* Set the pass NO */ + cc->cur_pass_no = p - jit_globals.passes; + bh_assert(*p < COMPILER_PASS_NUM); + + if (!compiler_passes[*p].run(cc) || jit_get_last_error(cc)) { + LOG_VERBOSE("JIT: compilation failed at pass[%td] = %s\n", + p - jit_globals.passes, compiler_passes[*p].name); + return false; + } + } + + return true; +} + +bool +jit_compiler_init(const JitCompOptions *options) +{ + uint32 code_cache_size = options->code_cache_size > 0 + ? options->code_cache_size + : FAST_JIT_DEFAULT_CODE_CACHE_SIZE; + + LOG_VERBOSE("JIT: compiler init with code cache size: %u\n", + code_cache_size); + + if (!jit_code_cache_init(code_cache_size)) + return false; + + if (!jit_codegen_init()) + goto fail1; + + return true; + +fail1: + jit_code_cache_destroy(); + return false; +} + +void +jit_compiler_destroy() +{ + jit_codegen_destroy(); + + jit_code_cache_destroy(); +} + +JitGlobals * +jit_compiler_get_jit_globals() +{ + return &jit_globals; +} + +const char * +jit_compiler_get_pass_name(unsigned i) +{ + return i < COMPILER_PASS_NUM ? compiler_passes[i].name : NULL; +} + +bool +jit_compiler_compile(WASMModule *module, uint32 func_idx) +{ + JitCompContext *cc = NULL; + char *last_error; + bool ret = false; + uint32 i = func_idx - module->import_function_count; + uint32 j = i % WASM_ORC_JIT_BACKEND_THREAD_NUM; + + /* Lock to avoid duplicated compilation by other threads */ + os_mutex_lock(&module->fast_jit_thread_locks[j]); + + if (jit_compiler_is_compiled(module, func_idx)) { + /* Function has been compiled */ + os_mutex_unlock(&module->fast_jit_thread_locks[j]); + return true; + } + + /* Initialize the compilation context */ + if (!(cc = jit_calloc(sizeof(*cc)))) { + goto fail; + } + + if (!jit_cc_init(cc, 64)) { + goto fail; + } + + cc->cur_wasm_module = module; + cc->cur_wasm_func = module->functions[i]; + cc->cur_wasm_func_idx = func_idx; + cc->mem_space_unchanged = (!cc->cur_wasm_func->has_op_memory_grow + && !cc->cur_wasm_func->has_op_func_call) + || (!module->possible_memory_grow); + + /* Apply compiler passes */ + if (!apply_compiler_passes(cc) || jit_get_last_error(cc)) { + last_error = jit_get_last_error(cc); + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + char *function_name = cc->cur_wasm_func->field_name; + LOG_ERROR("fast jit compilation failed: %s (function_name=%s)\n", + last_error ? last_error : "unknown error", function_name); +#else + LOG_ERROR("fast jit compilation failed: %s\n", + last_error ? last_error : "unknown error"); +#endif + + goto fail; + } + + ret = true; + +fail: + /* Destroy the compilation context */ + if (cc) + jit_cc_delete(cc); + + os_mutex_unlock(&module->fast_jit_thread_locks[j]); + + return ret; +} + +bool +jit_compiler_compile_all(WASMModule *module) +{ + uint32 i; + + for (i = 0; i < module->function_count; i++) { + if (!jit_compiler_compile(module, module->import_function_count + i)) { + return false; + } + } + + return true; +} + +bool +jit_compiler_is_compiled(const WASMModule *module, uint32 func_idx) +{ + uint32 i = func_idx - module->import_function_count; + + bh_assert(func_idx >= module->import_function_count + && func_idx + < module->import_function_count + module->function_count); + +#if WASM_ENABLE_LAZY_JIT == 0 + return module->fast_jit_func_ptrs[i] ? true : false; +#else + return module->fast_jit_func_ptrs[i] + != jit_globals.compile_fast_jit_and_then_call + ? true + : false; +#endif +} + +#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_JIT != 0 +bool +jit_compiler_set_call_to_llvm_jit(WASMModule *module, uint32 func_idx) +{ + uint32 i = func_idx - module->import_function_count; + uint32 j = i % WASM_ORC_JIT_BACKEND_THREAD_NUM; + WASMType *func_type = module->functions[i]->func_type; + uint32 k = + ((uint32)(uintptr_t)func_type >> 3) % WASM_ORC_JIT_BACKEND_THREAD_NUM; + void *func_ptr = NULL; + + /* Compile code block of call_to_llvm_jit_from_fast_jit of + this kind of function type if it hasn't been compiled */ + if (!(func_ptr = func_type->call_to_llvm_jit_from_fast_jit)) { + os_mutex_lock(&module->fast_jit_thread_locks[k]); + if (!(func_ptr = func_type->call_to_llvm_jit_from_fast_jit)) { + if (!(func_ptr = func_type->call_to_llvm_jit_from_fast_jit = + jit_codegen_compile_call_to_llvm_jit(func_type))) { + os_mutex_unlock(&module->fast_jit_thread_locks[k]); + return false; + } + } + os_mutex_unlock(&module->fast_jit_thread_locks[k]); + } + + /* Switch current fast jit func ptr to the code block */ + os_mutex_lock(&module->fast_jit_thread_locks[j]); + module->fast_jit_func_ptrs[i] = func_ptr; + os_mutex_unlock(&module->fast_jit_thread_locks[j]); + return true; +} + +bool +jit_compiler_set_call_to_fast_jit(WASMModule *module, uint32 func_idx) +{ + void *func_ptr = NULL; + + func_ptr = jit_codegen_compile_call_to_fast_jit(module, func_idx); + if (func_ptr) { + uint32 i = func_idx - module->import_function_count; + module->functions[i]->call_to_fast_jit_from_llvm_jit = func_ptr; + jit_compiler_set_llvm_jit_func_ptr(module, func_idx, func_ptr); + } + + return func_ptr ? true : false; +} + +void +jit_compiler_set_llvm_jit_func_ptr(WASMModule *module, uint32 func_idx, + void *func_ptr) +{ + WASMModuleInstance *instance; + uint32 i = func_idx - module->import_function_count; + + os_mutex_lock(&module->instance_list_lock); + + module->func_ptrs[i] = func_ptr; + + instance = module->instance_list; + while (instance) { + if (instance->e->running_mode == Mode_Multi_Tier_JIT) + instance->func_ptrs[func_idx] = func_ptr; + instance = instance->e->next; + } + os_mutex_unlock(&module->instance_list_lock); +} +#endif /* end of WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_JIT != 0 */ + +int +jit_interp_switch_to_jitted(void *exec_env, JitInterpSwitchInfo *info, + uint32 func_idx, void *pc) +{ + return jit_codegen_interp_jitted_glue(exec_env, info, func_idx, pc); +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_compiler.h b/wasm-micro-runtime/core/iwasm/fast-jit/jit_compiler.h new file mode 100644 index 0000000..9a49cff --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_compiler.h @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_COMPILER_H_ +#define _JIT_COMPILER_H_ + +#include "bh_platform.h" +#include "../interpreter/wasm_runtime.h" +#include "jit_ir.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct JitGlobals { + /* Compiler pass sequence, the last element must be 0 */ + const uint8 *passes; + char *return_to_interp_from_jitted; +#if WASM_ENABLE_LAZY_JIT != 0 + char *compile_fast_jit_and_then_call; +#endif +} JitGlobals; + +/** + * Actions the interpreter should do when jitted code returns to + * interpreter. + */ +typedef enum JitInterpAction { + JIT_INTERP_ACTION_NORMAL, /* normal execution */ + JIT_INTERP_ACTION_THROWN, /* exception was thrown */ + JIT_INTERP_ACTION_CALL /* call wasm function */ +} JitInterpAction; + +/** + * Information exchanged between jitted code and interpreter. + */ +typedef struct JitInterpSwitchInfo { + /* Points to the frame that is passed to jitted code and the frame + that is returned from jitted code */ + void *frame; + + /* Output values from jitted code of different actions */ + union { + /* IP and SP offsets for NORMAL */ + struct { + int32 ip; + int32 sp; + } normal; + + /* Function called from jitted code for CALL */ + struct { + void *function; + } call; + + /* Returned integer and/or floating point values for RETURN. This + is also used to pass return values from interpreter to jitted + code if the caller is in jitted code and the callee is in + interpreter. */ + struct { + uint32 ival[2]; + uint32 fval[2]; + uint32 last_return_type; + } ret; + } out; +} JitInterpSwitchInfo; + +/* Jit compiler options */ +typedef struct JitCompOptions { + uint32 code_cache_size; + uint32 opt_level; +} JitCompOptions; + +bool +jit_compiler_init(const JitCompOptions *option); + +void +jit_compiler_destroy(); + +JitGlobals * +jit_compiler_get_jit_globals(); + +const char * +jit_compiler_get_pass_name(unsigned i); + +bool +jit_compiler_compile(WASMModule *module, uint32 func_idx); + +bool +jit_compiler_compile_all(WASMModule *module); + +bool +jit_compiler_is_compiled(const WASMModule *module, uint32 func_idx); + +#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_JIT != 0 +bool +jit_compiler_set_call_to_llvm_jit(WASMModule *module, uint32 func_idx); + +bool +jit_compiler_set_call_to_fast_jit(WASMModule *module, uint32 func_idx); + +void +jit_compiler_set_llvm_jit_func_ptr(WASMModule *module, uint32 func_idx, + void *func_ptr); +#endif + +int +jit_interp_switch_to_jitted(void *self, JitInterpSwitchInfo *info, + uint32 func_idx, void *pc); + +/* + * Pass declarations: + */ + +/** + * Dump the compilation context. + */ +bool +jit_pass_dump(JitCompContext *cc); + +/** + * Update CFG (usually before dump for better readability). + */ +bool +jit_pass_update_cfg(JitCompContext *cc); + +/** + * Translate profiling result into MIR. + */ +bool +jit_pass_frontend(JitCompContext *cc); + +/** + * Lower unsupported operations into supported ones. + */ +bool +jit_pass_lower_cg(JitCompContext *cc); + +/** + * Register allocation. + */ +bool +jit_pass_regalloc(JitCompContext *cc); + +/** + * Native code generation. + */ +bool +jit_pass_codegen(JitCompContext *cc); + +/** + * Register the jitted code so that it can be executed. + */ +bool +jit_pass_register_jitted_code(JitCompContext *cc); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _JIT_COMPILER_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_dump.c b/wasm-micro-runtime/core/iwasm/fast-jit/jit_dump.c new file mode 100644 index 0000000..d61ed5d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_dump.c @@ -0,0 +1,336 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_dump.h" +#include "jit_compiler.h" +#include "jit_codegen.h" + +void +jit_dump_reg(JitCompContext *cc, JitReg reg) +{ + unsigned kind = jit_reg_kind(reg); + unsigned no = jit_reg_no(reg); + + switch (kind) { + case JIT_REG_KIND_VOID: + os_printf("VOID"); + break; + + case JIT_REG_KIND_I32: + if (jit_reg_is_const(reg)) { + unsigned rel = jit_cc_get_const_I32_rel(cc, reg); + + os_printf("0x%x", jit_cc_get_const_I32(cc, reg)); + + if (rel) + os_printf("(rel: 0x%x)", rel); + } + else + os_printf("i%d", no); + break; + + case JIT_REG_KIND_I64: + if (jit_reg_is_const(reg)) + os_printf("0x%llxL", jit_cc_get_const_I64(cc, reg)); + else + os_printf("I%d", no); + break; + + case JIT_REG_KIND_F32: + if (jit_reg_is_const(reg)) + os_printf("%f", jit_cc_get_const_F32(cc, reg)); + else + os_printf("f%d", no); + break; + + case JIT_REG_KIND_F64: + if (jit_reg_is_const(reg)) + os_printf("%fL", jit_cc_get_const_F64(cc, reg)); + else + os_printf("D%d", no); + break; + + case JIT_REG_KIND_L32: + os_printf("L%d", no); + break; + + default: + bh_assert(!"Unsupported register kind."); + } +} + +static void +jit_dump_insn_Reg(JitCompContext *cc, JitInsn *insn, unsigned opnd_num) +{ + unsigned i; + + for (i = 0; i < opnd_num; i++) { + os_printf(i == 0 ? " " : ", "); + jit_dump_reg(cc, *(jit_insn_opnd(insn, i))); + } + + os_printf("\n"); +} + +static void +jit_dump_insn_VReg(JitCompContext *cc, JitInsn *insn, unsigned opnd_num) +{ + unsigned i; + + opnd_num = jit_insn_opndv_num(insn); + + for (i = 0; i < opnd_num; i++) { + os_printf(i == 0 ? " " : ", "); + jit_dump_reg(cc, *(jit_insn_opndv(insn, i))); + } + + os_printf("\n"); +} + +static void +jit_dump_insn_LookupSwitch(JitCompContext *cc, JitInsn *insn, unsigned opnd_num) +{ + unsigned i; + JitOpndLookupSwitch *opnd = jit_insn_opndls(insn); + + os_printf(" "); + jit_dump_reg(cc, opnd->value); + os_printf("\n%16s: ", "default"); + jit_dump_reg(cc, opnd->default_target); + os_printf("\n"); + + for (i = 0; i < opnd->match_pairs_num; i++) { + os_printf("%18d: ", opnd->match_pairs[i].value); + jit_dump_reg(cc, opnd->match_pairs[i].target); + os_printf("\n"); + } +} + +void +jit_dump_insn(JitCompContext *cc, JitInsn *insn) +{ + switch (insn->opcode) { +#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) \ + case JIT_OP_##NAME: \ + if (insn->flags_u8 & 0x1) \ + os_printf(" ATOMIC %-8s", #NAME); \ + else \ + os_printf(" %-15s", #NAME); \ + jit_dump_insn_##OPND_KIND(cc, insn, OPND_NUM); \ + break; +#include "jit_ir.def" +#undef INSN + } +} + +void +jit_dump_basic_block(JitCompContext *cc, JitBasicBlock *block) +{ + unsigned i, label_index; + void *begin_addr, *end_addr; + JitBasicBlock *block_next; + JitInsn *insn; + JitRegVec preds = jit_basic_block_preds(block); + JitRegVec succs = jit_basic_block_succs(block); + JitReg label = jit_basic_block_label(block), label_next; + JitReg *reg; + + jit_dump_reg(cc, label); + os_printf(":\n ; PREDS("); + + JIT_REG_VEC_FOREACH(preds, i, reg) + { + if (i > 0) + os_printf(" "); + jit_dump_reg(cc, *reg); + } + + os_printf(")\n ;"); + + if (jit_annl_is_enabled_begin_bcip(cc)) + os_printf(" BEGIN_BCIP=0x%04tx", + *(jit_annl_begin_bcip(cc, label)) + - (uint8 *)cc->cur_wasm_module->load_addr); + + if (jit_annl_is_enabled_end_bcip(cc)) + os_printf(" END_BCIP=0x%04tx", + *(jit_annl_end_bcip(cc, label)) + - (uint8 *)cc->cur_wasm_module->load_addr); + os_printf("\n"); + + if (jit_annl_is_enabled_jitted_addr(cc)) { + begin_addr = *(jit_annl_jitted_addr(cc, label)); + + if (label == cc->entry_label) { + block_next = cc->_ann._label_basic_block[2]; + label_next = jit_basic_block_label(block_next); + end_addr = *(jit_annl_jitted_addr(cc, label_next)); + } + else if (label == cc->exit_label) { + end_addr = cc->jitted_addr_end; + } + else { + label_index = jit_reg_no(label); + if (label_index < jit_cc_label_num(cc) - 1) + block_next = cc->_ann._label_basic_block[label_index + 1]; + else + block_next = cc->_ann._label_basic_block[1]; + label_next = jit_basic_block_label(block_next); + end_addr = *(jit_annl_jitted_addr(cc, label_next)); + } + + jit_codegen_dump_native(begin_addr, end_addr); + } + else { + /* Dump IR. */ + JIT_FOREACH_INSN(block, insn) jit_dump_insn(cc, insn); + } + + os_printf(" ; SUCCS("); + + JIT_REG_VEC_FOREACH(succs, i, reg) + { + if (i > 0) + os_printf(" "); + jit_dump_reg(cc, *reg); + } + + os_printf(")\n\n"); +} + +static void +dump_func_name(JitCompContext *cc) +{ + const char *func_name = NULL; + WASMModule *module = cc->cur_wasm_module; + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + func_name = cc->cur_wasm_func->field_name; +#endif + + /* if custom name section is not generated, + search symbols from export table */ + if (!func_name) { + uint32 i; + for (i = 0; i < module->export_count; i++) { + if (module->exports[i].kind == EXPORT_KIND_FUNC + && module->exports[i].index == cc->cur_wasm_func_idx) { + func_name = module->exports[i].name; + break; + } + } + } + + /* function name not exported, print number instead */ + if (func_name == NULL) { + os_printf("$f%d", cc->cur_wasm_func_idx); + } + else { + os_printf("%s", func_name); + } +} + +static void +dump_cc_ir(JitCompContext *cc) +{ + unsigned i, end; + JitBasicBlock *block; + JitReg label; + const char *kind_names[] = { "VOID", "I32", "I64", "F32", + "F64", "V64", "V128", "V256" }; + + os_printf("; Function: "); + dump_func_name(cc); + os_printf("\n"); + + os_printf("; Constant table sizes:"); + + for (i = 0; i < JIT_REG_KIND_L32; i++) + os_printf(" %s=%d", kind_names[i], cc->_const_val._num[i]); + + os_printf("\n; Label number: %d", jit_cc_label_num(cc)); + os_printf("\n; Instruction number: %d", jit_cc_insn_num(cc)); + os_printf("\n; Register numbers:"); + + for (i = 0; i < JIT_REG_KIND_L32; i++) + os_printf(" %s=%d", kind_names[i], jit_cc_reg_num(cc, i)); + + os_printf("\n; Label annotations:"); +#define ANN_LABEL(TYPE, NAME) \ + if (jit_annl_is_enabled_##NAME(cc)) \ + os_printf(" %s", #NAME); +#include "jit_ir.def" +#undef ANN_LABEL + + os_printf("\n; Instruction annotations:"); +#define ANN_INSN(TYPE, NAME) \ + if (jit_anni_is_enabled_##NAME(cc)) \ + os_printf(" %s", #NAME); +#include "jit_ir.def" +#undef ANN_INSN + + os_printf("\n; Register annotations:"); +#define ANN_REG(TYPE, NAME) \ + if (jit_annr_is_enabled_##NAME(cc)) \ + os_printf(" %s", #NAME); +#include "jit_ir.def" +#undef ANN_REG + + os_printf("\n\n"); + + if (jit_annl_is_enabled_next_label(cc)) { + /* Blocks have been reordered, use that order to dump. */ + for (label = cc->entry_label; label; + label = *(jit_annl_next_label(cc, label))) + jit_dump_basic_block(cc, *(jit_annl_basic_block(cc, label))); + } + else { + /* Otherwise, use the default order. */ + jit_dump_basic_block(cc, jit_cc_entry_basic_block(cc)); + + JIT_FOREACH_BLOCK(cc, i, end, block) jit_dump_basic_block(cc, block); + + jit_dump_basic_block(cc, jit_cc_exit_basic_block(cc)); + } +} + +void +jit_dump_cc(JitCompContext *cc) +{ + if (jit_cc_label_num(cc) <= 2) + return; + + dump_cc_ir(cc); +} + +bool +jit_pass_dump(JitCompContext *cc) +{ + const JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + const uint8 *passes = jit_globals->passes; + uint8 pass_no = cc->cur_pass_no; + const char *pass_name = + pass_no > 0 ? jit_compiler_get_pass_name(passes[pass_no - 1]) : "NULL"; + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + if (!strcmp(pass_name, "lower_cg")) + /* Ignore lower codegen pass as it does nothing in x86-64 */ + return true; +#endif + + os_printf("JIT.COMPILER.DUMP: PASS_NO=%d PREV_PASS=%s\n\n", pass_no, + pass_name); + + jit_dump_cc(cc); + + os_printf("\n"); + return true; +} + +bool +jit_pass_update_cfg(JitCompContext *cc) +{ + return jit_cc_update_cfg(cc); +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_dump.h b/wasm-micro-runtime/core/iwasm/fast-jit/jit_dump.h new file mode 100644 index 0000000..8e572b8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_dump.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_DUMP_H_ +#define _JIT_DUMP_H_ + +#include "jit_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Dump a register. + * + * @param cc compilation context of the register + * @param reg register to be dumped + */ +void +jit_dump_reg(JitCompContext *cc, JitReg reg); + +/** + * Dump an instruction. + * + * @param cc compilation context of the instruction + * @param insn instruction to be dumped + */ +void +jit_dump_insn(JitCompContext *cc, JitInsn *insn); + +/** + * Dump a block. + * + * @param cc compilation context of the block + * @param block block to be dumped + */ +void +jit_dump_block(JitCompContext *cc, JitBlock *block); + +/** + * Dump a compilation context. + * + * @param cc compilation context to be dumped + */ +void +jit_dump_cc(JitCompContext *cc); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _JIT_DUMP_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_frontend.c b/wasm-micro-runtime/core/iwasm/fast-jit/jit_frontend.c new file mode 100644 index 0000000..b8d40f9 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_frontend.c @@ -0,0 +1,2600 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_compiler.h" +#include "jit_frontend.h" +#include "fe/jit_emit_compare.h" +#include "fe/jit_emit_const.h" +#include "fe/jit_emit_control.h" +#include "fe/jit_emit_conversion.h" +#include "fe/jit_emit_exception.h" +#include "fe/jit_emit_function.h" +#include "fe/jit_emit_memory.h" +#include "fe/jit_emit_numberic.h" +#include "fe/jit_emit_parametric.h" +#include "fe/jit_emit_table.h" +#include "fe/jit_emit_variable.h" +#include "../interpreter/wasm_interp.h" +#include "../interpreter/wasm_opcode.h" +#include "../interpreter/wasm_runtime.h" +#include "../common/wasm_exec_env.h" + +static uint32 +get_global_base_offset(const WASMModule *module) +{ + uint32 module_inst_struct_size = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes); + uint32 mem_inst_size = + (uint32)sizeof(WASMMemoryInstance) + * (module->import_memory_count + module->memory_count); + +#if WASM_ENABLE_JIT != 0 + /* If the module dosen't have memory, reserve one mem_info space + with empty content to align with llvm jit compiler */ + if (mem_inst_size == 0) + mem_inst_size = (uint32)sizeof(WASMMemoryInstance); +#endif + + /* Size of module inst and memory instances */ + return module_inst_struct_size + mem_inst_size; +} + +static uint32 +get_first_table_inst_offset(const WASMModule *module) +{ + return get_global_base_offset(module) + module->global_data_size; +} + +uint32 +jit_frontend_get_global_data_offset(const WASMModule *module, uint32 global_idx) +{ + uint32 global_base_offset = get_global_base_offset(module); + + if (global_idx < module->import_global_count) { + const WASMGlobalImport *import_global = + &((module->import_globals + global_idx)->u.global); + return global_base_offset + import_global->data_offset; + } + else { + const WASMGlobal *global = + module->globals + (global_idx - module->import_global_count); + return global_base_offset + global->data_offset; + } +} + +uint32 +jit_frontend_get_table_inst_offset(const WASMModule *module, uint32 tbl_idx) +{ + uint32 offset, i = 0; + + offset = get_first_table_inst_offset(module); + + while (i < tbl_idx && i < module->import_table_count) { + WASMTableImport *import_table = &module->import_tables[i].u.table; + + offset += (uint32)offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + offset += (uint32)sizeof(uint32) * import_table->max_size; +#else + offset += (uint32)sizeof(uint32) + * (import_table->possible_grow ? import_table->max_size + : import_table->init_size); +#endif + + i++; + } + + if (i == tbl_idx) { + return offset; + } + + tbl_idx -= module->import_table_count; + i -= module->import_table_count; + while (i < tbl_idx && i < module->table_count) { + WASMTable *table = module->tables + i; + + offset += (uint32)offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + offset += (uint32)sizeof(table_elem_type_t) * table->max_size; +#else + offset += (uint32)sizeof(table_elem_type_t) + * (table->possible_grow ? table->max_size : table->init_size); +#endif + + i++; + } + + return offset; +} + +uint32 +jit_frontend_get_module_inst_extra_offset(const WASMModule *module) +{ + uint32 offset = jit_frontend_get_table_inst_offset( + module, module->import_table_count + module->table_count); + + return align_uint(offset, 8); +} + +JitReg +get_module_inst_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + + if (!frame->module_inst_reg) { + frame->module_inst_reg = cc->module_inst_reg; + GEN_INSN(LDPTR, frame->module_inst_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); + } + return frame->module_inst_reg; +} + +JitReg +get_module_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->module_reg) { + frame->module_reg = cc->module_reg; + GEN_INSN(LDPTR, frame->module_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, module))); + } + return frame->module_reg; +} + +JitReg +get_import_func_ptrs_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->import_func_ptrs_reg) { + frame->import_func_ptrs_reg = cc->import_func_ptrs_reg; + GEN_INSN( + LDPTR, frame->import_func_ptrs_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, import_func_ptrs))); + } + return frame->import_func_ptrs_reg; +} + +JitReg +get_fast_jit_func_ptrs_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->fast_jit_func_ptrs_reg) { + frame->fast_jit_func_ptrs_reg = cc->fast_jit_func_ptrs_reg; + GEN_INSN( + LDPTR, frame->fast_jit_func_ptrs_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, fast_jit_func_ptrs))); + } + return frame->fast_jit_func_ptrs_reg; +} + +JitReg +get_func_type_indexes_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->func_type_indexes_reg) { + frame->func_type_indexes_reg = cc->func_type_indexes_reg; + GEN_INSN( + LDPTR, frame->func_type_indexes_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, func_type_indexes))); + } + return frame->func_type_indexes_reg; +} + +JitReg +get_aux_stack_bound_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg tmp = jit_cc_new_reg_I32(cc); + + if (!frame->aux_stack_bound_reg) { + frame->aux_stack_bound_reg = cc->aux_stack_bound_reg; + GEN_INSN(LDPTR, frame->aux_stack_bound_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_boundary))); + /* TODO: Memory64 whether to convert depends on memory idx type */ + GEN_INSN(I64TOI32, tmp, frame->aux_stack_bound_reg); + frame->aux_stack_bound_reg = tmp; + } + return frame->aux_stack_bound_reg; +} + +JitReg +get_aux_stack_bottom_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg tmp = jit_cc_new_reg_I32(cc); + + if (!frame->aux_stack_bottom_reg) { + frame->aux_stack_bottom_reg = cc->aux_stack_bottom_reg; + GEN_INSN(LDPTR, frame->aux_stack_bottom_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_bottom))); + /* TODO: Memory64 whether to convert depends on memory idx type */ + GEN_INSN(I64TOI32, tmp, frame->aux_stack_bottom_reg); + frame->aux_stack_bottom_reg = tmp; + } + return frame->aux_stack_bottom_reg; +} + +#if WASM_ENABLE_SHARED_MEMORY != 0 +static bool +is_shared_memory(WASMModule *module, uint32 mem_idx) +{ + WASMMemory *memory; + WASMMemoryImport *memory_import; + bool is_shared; + + if (mem_idx < module->import_memory_count) { + memory_import = &(module->import_memories[mem_idx].u.memory); + is_shared = memory_import->flags & 0x02 ? true : false; + } + else { + memory = &module->memories[mem_idx - module->import_memory_count]; + is_shared = memory->flags & 0x02 ? true : false; + } + return is_shared; +} +#endif + +JitReg +get_memory_inst_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 memory_inst_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memories_addr; + uint32 memories_offset; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].memory_inst) + return frame->memory_regs[mem_idx].memory_inst; + + frame->memory_regs[mem_idx].memory_inst = + cc->memory_regs[mem_idx].memory_inst; + + bh_assert(mem_idx == 0); +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memories_addr = jit_cc_new_reg_ptr(cc); + memories_offset = (uint32)offsetof(WASMModuleInstance, memories); + /* module_inst->memories */ + GEN_INSN(LDPTR, memories_addr, module_inst_reg, + NEW_CONST(I32, memories_offset)); + /* module_inst->memories[mem_idx], mem_idx can only be 0 now */ + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_inst, memories_addr, + NEW_CONST(I32, mem_idx)); + } + else +#endif + { + memory_inst_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes); + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_inst, + module_inst_reg, NEW_CONST(I32, memory_inst_offset)); + } + + return frame->memory_regs[mem_idx].memory_inst; +} + +JitReg +get_cur_page_count_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 cur_page_count_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].cur_page_count) + return frame->memory_regs[mem_idx].cur_page_count; + + frame->memory_regs[mem_idx].cur_page_count = + cc->memory_regs[mem_idx].cur_page_count; + + /* Get current page count */ + bh_assert(mem_idx == 0); +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + cur_page_count_offset = + (uint32)offsetof(WASMMemoryInstance, cur_page_count); + /* memories[mem_idx]->cur_page_count_offset */ + GEN_INSN(LDI32, frame->memory_regs[mem_idx].cur_page_count, + memory_inst_reg, NEW_CONST(I32, cur_page_count_offset)); + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + cur_page_count_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, cur_page_count); + GEN_INSN(LDI32, frame->memory_regs[mem_idx].cur_page_count, + module_inst_reg, NEW_CONST(I32, cur_page_count_offset)); + } + + return frame->memory_regs[mem_idx].cur_page_count; +} + +JitReg +get_memory_data_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 memory_data_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].memory_data) + return frame->memory_regs[mem_idx].memory_data; + + frame->memory_regs[mem_idx].memory_data = + cc->memory_regs[mem_idx].memory_data; + + bh_assert(mem_idx == 0); +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + memory_data_offset = (uint32)offsetof(WASMMemoryInstance, memory_data); + /* memories[mem_idx]->memory_data */ + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data, + memory_inst_reg, NEW_CONST(I32, memory_data_offset)); + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + memory_data_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, memory_data); + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data, + module_inst_reg, NEW_CONST(I32, memory_data_offset)); + } + return frame->memory_regs[mem_idx].memory_data; +} + +JitReg +get_memory_data_end_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 memory_data_end_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].memory_data_end) + return frame->memory_regs[mem_idx].memory_data_end; + + frame->memory_regs[mem_idx].memory_data_end = + cc->memory_regs[mem_idx].memory_data_end; + + bh_assert(mem_idx == 0); +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + memory_data_end_offset = + (uint32)offsetof(WASMMemoryInstance, memory_data_end); + /* memories[mem_idx]->memory_data_end */ + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data_end, + memory_inst_reg, NEW_CONST(I32, memory_data_end_offset)); + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + memory_data_end_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, memory_data_end); + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data_end, + module_inst_reg, NEW_CONST(I32, memory_data_end_offset)); + } + return frame->memory_regs[mem_idx].memory_data_end; +} + +JitReg +get_mem_bound_check_1byte_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 mem_bound_check_1byte_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_1byte) + return frame->memory_regs[mem_idx].mem_bound_check_1byte; + + frame->memory_regs[mem_idx].mem_bound_check_1byte = + cc->memory_regs[mem_idx].mem_bound_check_1byte; + + bh_assert(mem_idx == 0); + +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_1byte_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_1byte); + /* memories[mem_idx]->mem_bound_check_1byte */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_1byte, + memory_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_1byte, + memory_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_1byte_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_1byte); +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_1byte, + module_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_1byte, + module_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_1byte; +} + +JitReg +get_mem_bound_check_2bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 mem_bound_check_2bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_2bytes) + return frame->memory_regs[mem_idx].mem_bound_check_2bytes; + + frame->memory_regs[mem_idx].mem_bound_check_2bytes = + cc->memory_regs[mem_idx].mem_bound_check_2bytes; + + bh_assert(mem_idx == 0); + +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_2bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_2bytes); + /* memories[mem_idx]->mem_bound_check_2bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_2bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_2bytes); +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_2bytes; +} + +JitReg +get_mem_bound_check_4bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 mem_bound_check_4bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_4bytes) + return frame->memory_regs[mem_idx].mem_bound_check_4bytes; + + frame->memory_regs[mem_idx].mem_bound_check_4bytes = + cc->memory_regs[mem_idx].mem_bound_check_4bytes; + + bh_assert(mem_idx == 0); + +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_4bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_4bytes); + /* memories[mem_idx]->mem_bound_check_4bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_4bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_4bytes); +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_4bytes; +} + +JitReg +get_mem_bound_check_8bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 mem_bound_check_8bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_8bytes) + return frame->memory_regs[mem_idx].mem_bound_check_8bytes; + + frame->memory_regs[mem_idx].mem_bound_check_8bytes = + cc->memory_regs[mem_idx].mem_bound_check_8bytes; + + bh_assert(mem_idx == 0); + +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_8bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_8bytes); + /* memories[mem_idx]->mem_bound_check_8bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_8bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_8bytes); +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_8bytes; +} + +JitReg +get_mem_bound_check_16bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 mem_bound_check_16bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_16bytes) + return frame->memory_regs[mem_idx].mem_bound_check_16bytes; + + frame->memory_regs[mem_idx].mem_bound_check_16bytes = + cc->memory_regs[mem_idx].mem_bound_check_16bytes; + + bh_assert(mem_idx == 0); + +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_16bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_16bytes); + /* memories[mem_idx]->mem_bound_check_16bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_16bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_16bytes); +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_16bytes; +} + +JitReg +get_table_elems_reg(JitFrame *frame, uint32 tbl_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst = get_module_inst_reg(frame); + uint32 offset = + jit_frontend_get_table_inst_offset(cc->cur_wasm_module, tbl_idx) + + (uint32)offsetof(WASMTableInstance, elems); + + if (!frame->table_regs[tbl_idx].table_elems) { + frame->table_regs[tbl_idx].table_elems = + cc->table_regs[tbl_idx].table_elems; + GEN_INSN(ADD, frame->table_regs[tbl_idx].table_elems, module_inst, + NEW_CONST(PTR, offset)); + } + return frame->table_regs[tbl_idx].table_elems; +} + +JitReg +get_table_cur_size_reg(JitFrame *frame, uint32 tbl_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst = get_module_inst_reg(frame); + uint32 offset = + jit_frontend_get_table_inst_offset(cc->cur_wasm_module, tbl_idx) + + (uint32)offsetof(WASMTableInstance, cur_size); + + if (!frame->table_regs[tbl_idx].table_cur_size) { + frame->table_regs[tbl_idx].table_cur_size = + cc->table_regs[tbl_idx].table_cur_size; + GEN_INSN(LDI32, frame->table_regs[tbl_idx].table_cur_size, module_inst, + NEW_CONST(I32, offset)); + } + return frame->table_regs[tbl_idx].table_cur_size; +} + +void +clear_fixed_virtual_regs(JitFrame *frame) +{ + WASMModule *module = frame->cc->cur_wasm_module; + uint32 count, i; + + frame->module_inst_reg = 0; + frame->module_reg = 0; + frame->import_func_ptrs_reg = 0; + frame->fast_jit_func_ptrs_reg = 0; + frame->func_type_indexes_reg = 0; + frame->aux_stack_bound_reg = 0; + frame->aux_stack_bottom_reg = 0; + + count = module->import_memory_count + module->memory_count; + for (i = 0; i < count; i++) { + frame->memory_regs[i].memory_inst = 0; + frame->memory_regs[i].cur_page_count = 0; + frame->memory_regs[i].memory_data = 0; + frame->memory_regs[i].memory_data_end = 0; + frame->memory_regs[i].mem_bound_check_1byte = 0; + frame->memory_regs[i].mem_bound_check_2bytes = 0; + frame->memory_regs[i].mem_bound_check_4bytes = 0; + frame->memory_regs[i].mem_bound_check_8bytes = 0; + frame->memory_regs[i].mem_bound_check_16bytes = 0; + } + + count = module->import_table_count + module->table_count; + for (i = 0; i < count; i++) { + frame->table_regs[i].table_elems = 0; + frame->table_regs[i].table_cur_size = 0; + } +} + +void +clear_memory_regs(JitFrame *frame) +{ + WASMModule *module = frame->cc->cur_wasm_module; + uint32 count, i; + + count = module->import_memory_count + module->memory_count; + for (i = 0; i < count; i++) { + frame->memory_regs[i].cur_page_count = 0; + frame->memory_regs[i].memory_data = 0; + frame->memory_regs[i].memory_data_end = 0; + frame->memory_regs[i].mem_bound_check_1byte = 0; + frame->memory_regs[i].mem_bound_check_2bytes = 0; + frame->memory_regs[i].mem_bound_check_4bytes = 0; + frame->memory_regs[i].mem_bound_check_8bytes = 0; + frame->memory_regs[i].mem_bound_check_16bytes = 0; + } +} + +void +clear_table_regs(JitFrame *frame) +{ + WASMModule *module = frame->cc->cur_wasm_module; + uint32 count, i; + + count = module->import_table_count + module->table_count; + for (i = 0; i < count; i++) { + frame->table_regs[i].table_cur_size = 0; + } +} + +JitReg +gen_load_i32(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = jit_cc_new_reg_I32(cc); + GEN_INSN(LDI32, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +JitReg +gen_load_i64(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = frame->lp[n + 1].reg = jit_cc_new_reg_I64(cc); + GEN_INSN(LDI64, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +JitReg +gen_load_f32(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = jit_cc_new_reg_F32(cc); + GEN_INSN(LDF32, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +JitReg +gen_load_f64(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = frame->lp[n + 1].reg = jit_cc_new_reg_F64(cc); + GEN_INSN(LDF64, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +void +gen_commit_values(JitFrame *frame, JitValueSlot *begin, JitValueSlot *end) +{ + JitCompContext *cc = frame->cc; + JitValueSlot *p; + int n; + + for (p = begin; p < end; p++) { + if (!p->dirty) + continue; + + p->dirty = 0; + n = p - frame->lp; + + switch (jit_reg_kind(p->reg)) { + case JIT_REG_KIND_I32: + GEN_INSN(STI32, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + + case JIT_REG_KIND_I64: + GEN_INSN(STI64, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + (++p)->dirty = 0; + break; + + case JIT_REG_KIND_F32: + GEN_INSN(STF32, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + + case JIT_REG_KIND_F64: + GEN_INSN(STF64, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + (++p)->dirty = 0; + break; + } + } +} + +/** + * Generate instructions to commit SP and IP pointers to the frame. + * + * @param frame the frame information + */ +void +gen_commit_sp_ip(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg sp; + + if (frame->sp != frame->committed_sp) { + sp = jit_cc_new_reg_ptr(cc); + GEN_INSN(ADD, sp, cc->fp_reg, + NEW_CONST(PTR, offset_of_local(frame->sp - frame->lp))); + GEN_INSN(STPTR, sp, cc->fp_reg, + NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); + frame->committed_sp = frame->sp; + } + +#if 0 /* Disable committing ip currently */ + if (frame->ip != frame->committed_ip) { + GEN_INSN(STPTR, NEW_CONST(PTR, (uintptr_t)frame->ip), cc->fp_reg, + NEW_CONST(I32, offsetof(WASMInterpFrame, ip))); + frame->committed_ip = frame->ip; + } +#endif +} + +static bool +create_fixed_virtual_regs(JitCompContext *cc) +{ + WASMModule *module = cc->cur_wasm_module; + uint64 total_size; + uint32 i, count; + + cc->module_inst_reg = jit_cc_new_reg_ptr(cc); + cc->module_reg = jit_cc_new_reg_ptr(cc); + cc->import_func_ptrs_reg = jit_cc_new_reg_ptr(cc); + cc->fast_jit_func_ptrs_reg = jit_cc_new_reg_ptr(cc); + cc->func_type_indexes_reg = jit_cc_new_reg_ptr(cc); + cc->aux_stack_bound_reg = jit_cc_new_reg_ptr(cc); + cc->aux_stack_bottom_reg = jit_cc_new_reg_ptr(cc); + + count = module->import_memory_count + module->memory_count; + if (count > 0) { + total_size = (uint64)sizeof(JitMemRegs) * count; + if (total_size > UINT32_MAX + || !(cc->memory_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + for (i = 0; i < count; i++) { + cc->memory_regs[i].memory_inst = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].cur_page_count = jit_cc_new_reg_I32(cc); + cc->memory_regs[i].memory_data = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].memory_data_end = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_1byte = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_2bytes = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_4bytes = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_8bytes = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_16bytes = jit_cc_new_reg_ptr(cc); + } + } + + count = module->import_table_count + module->table_count; + if (count > 0) { + total_size = (uint64)sizeof(JitTableRegs) * count; + if (total_size > UINT32_MAX + || !(cc->table_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + for (i = 0; i < count; i++) { + cc->table_regs[i].table_elems = jit_cc_new_reg_ptr(cc); + cc->table_regs[i].table_cur_size = jit_cc_new_reg_I32(cc); + } + } + + return true; +} + +static bool +form_and_translate_func(JitCompContext *cc) +{ + JitBasicBlock *func_entry_basic_block; + JitReg func_entry_label; + JitInsn *insn; + JitIncomingInsn *incoming_insn, *incoming_insn_next; + uint32 i; + + if (!create_fixed_virtual_regs(cc)) + return false; + + if (!(func_entry_basic_block = jit_frontend_translate_func(cc))) + return false; + + jit_cc_reset_insn_hash(cc); + + /* The label of the func entry basic block. */ + func_entry_label = jit_basic_block_label(func_entry_basic_block); + + /* Create a JMP instruction jumping to the func entry. */ + if (!(insn = jit_cc_new_insn(cc, JMP, func_entry_label))) + return false; + + /* Insert the instruction into the cc entry block. */ + jit_basic_block_append_insn(jit_cc_entry_basic_block(cc), insn); + + /* Patch INSNs jumping to exception basic blocks. */ + for (i = 0; i < EXCE_NUM; i++) { + incoming_insn = cc->incoming_insns_for_exec_bbs[i]; + if (incoming_insn) { + if (!(cc->exce_basic_blocks[i] = jit_cc_new_basic_block(cc, 0))) { + jit_set_last_error(cc, "create basic block failed"); + return false; + } + while (incoming_insn) { + incoming_insn_next = incoming_insn->next; + insn = incoming_insn->insn; + if (insn->opcode == JIT_OP_JMP) { + *(jit_insn_opnd(insn, 0)) = + jit_basic_block_label(cc->exce_basic_blocks[i]); + } + else if (insn->opcode >= JIT_OP_BEQ + && insn->opcode <= JIT_OP_BLEU) { + *(jit_insn_opnd(insn, 1)) = + jit_basic_block_label(cc->exce_basic_blocks[i]); + } + incoming_insn = incoming_insn_next; + } + cc->cur_basic_block = cc->exce_basic_blocks[i]; + if (i != EXCE_ALREADY_THROWN) { + JitReg module_inst_reg = jit_cc_new_reg_ptr(cc); + GEN_INSN(LDPTR, module_inst_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); + insn = GEN_INSN( + CALLNATIVE, 0, + NEW_CONST(PTR, (uintptr_t)jit_set_exception_with_id), 2); + if (insn) { + *(jit_insn_opndv(insn, 2)) = module_inst_reg; + *(jit_insn_opndv(insn, 3)) = NEW_CONST(I32, i); + } + } + GEN_INSN(RETURN, NEW_CONST(I32, JIT_INTERP_ACTION_THROWN)); + + *(jit_annl_begin_bcip(cc, + jit_basic_block_label(cc->cur_basic_block))) = + *(jit_annl_end_bcip( + cc, jit_basic_block_label(cc->cur_basic_block))) = + cc->cur_wasm_module->load_addr; + } + } + + *(jit_annl_begin_bcip(cc, cc->entry_label)) = + *(jit_annl_end_bcip(cc, cc->entry_label)) = + *(jit_annl_begin_bcip(cc, cc->exit_label)) = + *(jit_annl_end_bcip(cc, cc->exit_label)) = + cc->cur_wasm_module->load_addr; + + if (jit_get_last_error(cc)) { + return false; + } + return true; +} + +bool +jit_pass_frontend(JitCompContext *cc) +{ + /* Enable necessary annotations required at the current stage. */ + if (!jit_annl_enable_begin_bcip(cc) || !jit_annl_enable_end_bcip(cc) + || !jit_annl_enable_end_sp(cc) || !jit_annr_enable_def_insn(cc) + || !jit_cc_enable_insn_hash(cc, 127)) + return false; + + if (!(form_and_translate_func(cc))) + return false; + + /* Release the annotations after local CSE and translation. */ + jit_cc_disable_insn_hash(cc); + jit_annl_disable_end_sp(cc); + + return true; +} + +static JitFrame * +init_func_translation(JitCompContext *cc) +{ + JitFrame *jit_frame; + JitReg top, top_boundary, new_top, frame_boundary, frame_sp; + WASMModule *cur_wasm_module = cc->cur_wasm_module; + WASMFunction *cur_wasm_func = cc->cur_wasm_func; + uint32 cur_wasm_func_idx = cc->cur_wasm_func_idx; + uint32 max_locals = + cur_wasm_func->param_cell_num + cur_wasm_func->local_cell_num; + uint32 max_stacks = cur_wasm_func->max_stack_cell_num; + uint64 total_cell_num = + (uint64)cur_wasm_func->param_cell_num + + (uint64)cur_wasm_func->local_cell_num + + (uint64)cur_wasm_func->max_stack_cell_num + + ((uint64)cur_wasm_func->max_block_num) * sizeof(WASMBranchBlock) / 4; + uint32 frame_size, outs_size, local_size, count; + uint32 i, local_off; + uint64 total_size; +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 + JitReg module_inst, func_inst; + uint32 func_insts_offset; +#if WASM_ENABLE_PERF_PROFILING != 0 + JitReg time_started; +#endif +#endif + + if ((uint64)max_locals + (uint64)max_stacks >= UINT32_MAX + || total_cell_num >= UINT32_MAX + || !(jit_frame = jit_calloc(offsetof(JitFrame, lp) + + sizeof(*jit_frame->lp) + * (max_locals + max_stacks)))) { + LOG_ERROR("allocate jit frame failed\n"); + return NULL; + } + + count = + cur_wasm_module->import_memory_count + cur_wasm_module->memory_count; + if (count > 0) { + total_size = (uint64)sizeof(JitMemRegs) * count; + if (total_size > UINT32_MAX + || !(jit_frame->memory_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + jit_free(jit_frame); + return NULL; + } + } + + count = cur_wasm_module->import_table_count + cur_wasm_module->table_count; + if (count > 0) { + total_size = (uint64)sizeof(JitTableRegs) * count; + if (total_size > UINT32_MAX + || !(jit_frame->table_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + if (jit_frame->memory_regs) + jit_free(jit_frame->memory_regs); + jit_free(jit_frame); + return NULL; + } + } + + jit_frame->cur_wasm_module = cur_wasm_module; + jit_frame->cur_wasm_func = cur_wasm_func; + jit_frame->cur_wasm_func_idx = cur_wasm_func_idx; + jit_frame->cc = cc; + jit_frame->max_locals = max_locals; + jit_frame->max_stacks = max_stacks; + jit_frame->sp = jit_frame->lp + max_locals; + jit_frame->ip = cur_wasm_func->code; + + cc->jit_frame = jit_frame; + cc->cur_basic_block = jit_cc_entry_basic_block(cc); + cc->spill_cache_offset = wasm_interp_interp_frame_size(total_cell_num); + /* Set spill cache size according to max local cell num, max stack cell + num and virtual fixed register num */ + cc->spill_cache_size = (max_locals + max_stacks) * 4 + sizeof(void *) * 16; + cc->total_frame_size = cc->spill_cache_offset + cc->spill_cache_size; + cc->jitted_return_address_offset = + offsetof(WASMInterpFrame, jitted_return_addr); + cc->cur_basic_block = jit_cc_entry_basic_block(cc); + + frame_size = outs_size = cc->total_frame_size; + local_size = + (cur_wasm_func->param_cell_num + cur_wasm_func->local_cell_num) * 4; + + top = jit_cc_new_reg_ptr(cc); + top_boundary = jit_cc_new_reg_ptr(cc); + new_top = jit_cc_new_reg_ptr(cc); + frame_boundary = jit_cc_new_reg_ptr(cc); + frame_sp = jit_cc_new_reg_ptr(cc); + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 + module_inst = jit_cc_new_reg_ptr(cc); + func_inst = jit_cc_new_reg_ptr(cc); +#if WASM_ENABLE_PERF_PROFILING != 0 + time_started = jit_cc_new_reg_I64(cc); + /* Call os_time_thread_cputime_us() to get time_started firstly + as there is stack frame switching below, calling native in them + may cause register spilling work inproperly */ + if (!jit_emit_callnative(cc, os_time_thread_cputime_us, time_started, NULL, + 0)) { + return NULL; + } +#endif +#endif + + /* top = exec_env->wasm_stack.top */ + GEN_INSN(LDPTR, top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top))); + /* top_boundary = exec_env->wasm_stack.top_boundary */ + GEN_INSN(LDPTR, top_boundary, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top_boundary))); + /* frame_boundary = top + frame_size + outs_size */ + GEN_INSN(ADD, frame_boundary, top, NEW_CONST(PTR, frame_size + outs_size)); + /* if frame_boundary > top_boundary, throw stack overflow exception */ + GEN_INSN(CMP, cc->cmp_reg, frame_boundary, top_boundary); + if (!jit_emit_exception(cc, EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { + return NULL; + } + + /* Add first and then sub to reduce one used register */ + /* new_top = frame_boundary - outs_size = top + frame_size */ + GEN_INSN(SUB, new_top, frame_boundary, NEW_CONST(PTR, outs_size)); + /* exec_env->wasm_stack.top = new_top */ + GEN_INSN(STPTR, new_top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top))); + /* frame_sp = frame->lp + local_size */ + GEN_INSN(ADD, frame_sp, top, + NEW_CONST(PTR, offsetof(WASMInterpFrame, lp) + local_size)); + /* frame->sp = frame_sp */ + GEN_INSN(STPTR, frame_sp, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); + /* frame->prev_frame = fp_reg */ + GEN_INSN(STPTR, cc->fp_reg, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, prev_frame))); +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 + /* module_inst = exec_env->module_inst */ + GEN_INSN(LDPTR, module_inst, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); + func_insts_offset = + jit_frontend_get_module_inst_extra_offset(cur_wasm_module) + + (uint32)offsetof(WASMModuleInstanceExtra, functions); + /* func_inst = module_inst->e->functions */ + GEN_INSN(LDPTR, func_inst, module_inst, NEW_CONST(I32, func_insts_offset)); + /* func_inst = func_inst + cur_wasm_func_idx */ + GEN_INSN(ADD, func_inst, func_inst, + NEW_CONST(PTR, (uint32)sizeof(WASMFunctionInstance) + * cur_wasm_func_idx)); + /* frame->function = func_inst */ + GEN_INSN(STPTR, func_inst, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, function))); +#if WASM_ENABLE_PERF_PROFILING != 0 + /* frame->time_started = time_started */ + GEN_INSN(STI64, time_started, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, time_started))); +#endif +#endif + /* exec_env->cur_frame = top */ + GEN_INSN(STPTR, top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, cur_frame))); + /* fp_reg = top */ + GEN_INSN(MOV, cc->fp_reg, top); + + /* Initialize local variables, set them to 0 */ + local_off = (uint32)offsetof(WASMInterpFrame, lp) + + cur_wasm_func->param_cell_num * 4; + for (i = 0; i < cur_wasm_func->local_cell_num / 2; i++, local_off += 8) { + GEN_INSN(STI64, NEW_CONST(I64, 0), cc->fp_reg, + NEW_CONST(I32, local_off)); + } + if (cur_wasm_func->local_cell_num & 1) { + GEN_INSN(STI32, NEW_CONST(I32, 0), cc->fp_reg, + NEW_CONST(I32, local_off)); + } + + return jit_frame; +} + +static void +free_block_memory(JitBlock *block) +{ + if (block->param_types) + jit_free(block->param_types); + if (block->result_types) + jit_free(block->result_types); + jit_free(block); +} + +static JitBasicBlock * +create_func_block(JitCompContext *cc) +{ + JitBlock *jit_block; + WASMFunction *cur_func = cc->cur_wasm_func; + WASMType *func_type = cur_func->func_type; + uint32 param_count = func_type->param_count; + uint32 result_count = func_type->result_count; + + if (!(jit_block = jit_calloc(sizeof(JitBlock)))) { + return NULL; + } + + if (param_count && !(jit_block->param_types = jit_calloc(param_count))) { + goto fail; + } + if (result_count && !(jit_block->result_types = jit_calloc(result_count))) { + goto fail; + } + + /* Set block data */ + jit_block->label_type = LABEL_TYPE_FUNCTION; + jit_block->param_count = param_count; + if (param_count) { + bh_memcpy_s(jit_block->param_types, param_count, func_type->types, + param_count); + } + jit_block->result_count = result_count; + if (result_count) { + bh_memcpy_s(jit_block->result_types, result_count, + func_type->types + param_count, result_count); + } + jit_block->wasm_code_end = cur_func->code + cur_func->code_size; + jit_block->frame_sp_begin = cc->jit_frame->sp; + + /* Add function entry block */ + if (!(jit_block->basic_block_entry = jit_cc_new_basic_block(cc, 0))) { + goto fail; + } + *(jit_annl_begin_bcip( + cc, jit_basic_block_label(jit_block->basic_block_entry))) = + cur_func->code; + jit_block_stack_push(&cc->block_stack, jit_block); + cc->cur_basic_block = jit_block->basic_block_entry; + + return jit_block->basic_block_entry; + +fail: + free_block_memory(jit_block); + return NULL; +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if (buf + length > buf_end) { \ + jit_set_last_error(cc, "read leb failed: unexpected end."); \ + return false; \ + } \ + } while (0) + +static bool +read_leb(JitCompContext *cc, const uint8 *buf, const uint8 *buf_end, + uint32 *p_offset, uint32 maxbits, bool sign, uint64 *p_result) +{ + uint64 result = 0; + uint32 shift = 0; + uint32 bcnt = 0; + uint64 byte; + + while (true) { + CHECK_BUF(buf, buf_end, 1); + byte = buf[*p_offset]; + *p_offset += 1; + result |= ((byte & 0x7f) << shift); + shift += 7; + if ((byte & 0x80) == 0) { + break; + } + bcnt += 1; + } + if (bcnt > (maxbits + 6) / 7) { + jit_set_last_error(cc, "read leb failed: " + "integer representation too long"); + return false; + } + if (sign && (shift < maxbits) && (byte & 0x40)) { + /* Sign extend */ + result |= (~((uint64)0)) << shift; + } + *p_result = result; + return true; +} + +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(cc, p, p_end, &off, 32, false, &res64)) \ + return false; \ + p += off; \ + res = (uint32)res64; \ + } while (0) + +#define read_leb_int32(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(cc, p, p_end, &off, 32, true, &res64)) \ + return false; \ + p += off; \ + res = (int32)res64; \ + } while (0) + +#define read_leb_int64(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(cc, p, p_end, &off, 64, true, &res64)) \ + return false; \ + p += off; \ + res = (int64)res64; \ + } while (0) + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define COMPILE_ATOMIC_RMW(OP, NAME) \ + case WASM_OP_ATOMIC_RMW_I32_##NAME: \ + bytes = 4; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME: \ + bytes = 8; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I32_##NAME##8_U: \ + bytes = 1; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I32_##NAME##16_U: \ + bytes = 2; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##8_U: \ + bytes = 1; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##16_U: \ + bytes = 2; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##32_U: \ + bytes = 4; \ + op_type = VALUE_TYPE_I64; \ + OP_ATOMIC_##OP : bin_op = AtomicRMWBinOp##OP; \ + goto build_atomic_rmw; +#endif + +static bool +jit_compile_func(JitCompContext *cc) +{ + WASMFunction *cur_func = cc->cur_wasm_func; + WASMType *func_type = NULL; + uint8 *frame_ip = cur_func->code, opcode, *p_f32, *p_f64; + uint8 *frame_ip_end = frame_ip + cur_func->code_size; + uint8 *param_types = NULL, *result_types = NULL, value_type; + uint16 param_count, result_count; + uint32 br_depth, *br_depths, br_count; + uint32 func_idx, type_idx, mem_idx, local_idx, global_idx, i; + uint32 bytes = 4, align, offset; + bool merge_cmp_and_if = false, merge_cmp_and_br_if = false; + bool sign = true; + int32 i32_const; + int64 i64_const; + float32 f32_const; + float64 f64_const; + + while (frame_ip < frame_ip_end) { + cc->jit_frame->ip = frame_ip; + opcode = *frame_ip++; + +#if 0 /* TODO */ +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (cc->enable_thread_mgr) { + if (!check_suspend_flags(cc, func_ctx)) + return false; + } +#endif +#endif + + switch (opcode) { + case WASM_OP_UNREACHABLE: + if (!jit_compile_op_unreachable(cc, &frame_ip)) + return false; + break; + + case WASM_OP_NOP: + break; + + case WASM_OP_BLOCK: + case WASM_OP_LOOP: + case WASM_OP_IF: + { + value_type = *frame_ip++; + if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_I64 + || value_type == VALUE_TYPE_F32 + || value_type == VALUE_TYPE_F64 + || value_type == VALUE_TYPE_V128 + || value_type == VALUE_TYPE_VOID + || value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF) { + param_count = 0; + param_types = NULL; + if (value_type == VALUE_TYPE_VOID) { + result_count = 0; + result_types = NULL; + } + else { + result_count = 1; + result_types = &value_type; + } + } + else { + jit_set_last_error(cc, "unsupported value type"); + return false; + } + if (!jit_compile_op_block( + cc, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - WASM_OP_BLOCK), + param_count, param_types, result_count, result_types, + merge_cmp_and_if)) + return false; + /* Clear flag */ + merge_cmp_and_if = false; + break; + } + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + { + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + func_type = cc->cur_wasm_module->types[type_idx]; + param_count = func_type->param_count; + param_types = func_type->types; + result_count = func_type->result_count; + result_types = func_type->types + param_count; + if (!jit_compile_op_block( + cc, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - EXT_OP_BLOCK), + param_count, param_types, result_count, result_types, + merge_cmp_and_if)) + return false; + /* Clear flag */ + merge_cmp_and_if = false; + break; + } + + case WASM_OP_ELSE: + if (!jit_compile_op_else(cc, &frame_ip)) + return false; + break; + + case WASM_OP_END: + if (!jit_compile_op_end(cc, &frame_ip)) + return false; + break; + + case WASM_OP_BR: + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!jit_compile_op_br(cc, br_depth, &frame_ip)) + return false; + break; + + case WASM_OP_BR_IF: + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!jit_compile_op_br_if(cc, br_depth, merge_cmp_and_br_if, + &frame_ip)) + return false; + /* Clear flag */ + merge_cmp_and_br_if = false; + break; + + case WASM_OP_BR_TABLE: + read_leb_uint32(frame_ip, frame_ip_end, br_count); + if (!(br_depths = jit_calloc((uint32)sizeof(uint32) + * (br_count + 1)))) { + jit_set_last_error(cc, "allocate memory failed."); + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + for (i = 0; i <= br_count; i++) + read_leb_uint32(frame_ip, frame_ip_end, br_depths[i]); +#else + for (i = 0; i <= br_count; i++) + br_depths[i] = *frame_ip++; +#endif + + if (!jit_compile_op_br_table(cc, br_depths, br_count, + &frame_ip)) { + jit_free(br_depths); + return false; + } + + jit_free(br_depths); + break; + +#if WASM_ENABLE_FAST_INTERP == 0 + case EXT_OP_BR_TABLE_CACHE: + { + BrTableCache *node = bh_list_first_elem( + cc->cur_wasm_module->br_table_cache_list); + BrTableCache *node_next; + uint8 *p_opcode = frame_ip - 1; + + read_leb_uint32(frame_ip, frame_ip_end, br_count); + + while (node) { + node_next = bh_list_elem_next(node); + if (node->br_table_op_addr == p_opcode) { + br_depths = node->br_depths; + if (!jit_compile_op_br_table(cc, br_depths, br_count, + &frame_ip)) { + return false; + } + break; + } + node = node_next; + } + bh_assert(node); + + break; + } +#endif + + case WASM_OP_RETURN: + if (!jit_compile_op_return(cc, &frame_ip)) + return false; + break; + + case WASM_OP_CALL: + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + if (!jit_compile_op_call(cc, func_idx, false)) + return false; + break; + + case WASM_OP_CALL_INDIRECT: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + +#if WASM_ENABLE_REF_TYPES != 0 + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); +#else + frame_ip++; + tbl_idx = 0; +#endif + + if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx)) + return false; + break; + } + +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + + if (!jit_compile_op_call(cc, func_idx, true)) + return false; + if (!jit_compile_op_return(cc, &frame_ip)) + return false; + break; + + case WASM_OP_RETURN_CALL_INDIRECT: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); +#if WASM_ENABLE_REF_TYPES != 0 + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); +#else + frame_ip++; + tbl_idx = 0; +#endif + + if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx)) + return false; + if (!jit_compile_op_return(cc, &frame_ip)) + return false; + break; + } +#endif /* end of WASM_ENABLE_TAIL_CALL */ + + case WASM_OP_DROP: + if (!jit_compile_op_drop(cc, true)) + return false; + break; + + case WASM_OP_DROP_64: + if (!jit_compile_op_drop(cc, false)) + return false; + break; + + case WASM_OP_SELECT: + if (!jit_compile_op_select(cc, true)) + return false; + break; + + case WASM_OP_SELECT_64: + if (!jit_compile_op_select(cc, false)) + return false; + break; + +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_SELECT_T: + { + uint32 vec_len; + + read_leb_uint32(frame_ip, frame_ip_end, vec_len); + bh_assert(vec_len == 1); + (void)vec_len; + + type_idx = *frame_ip++; + if (!jit_compile_op_select(cc, + (type_idx != VALUE_TYPE_I64) + && (type_idx != VALUE_TYPE_F64))) + return false; + break; + } + case WASM_OP_TABLE_GET: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_get(cc, tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_SET: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_set(cc, tbl_idx)) + return false; + break; + } + case WASM_OP_REF_NULL: + { + uint32 ref_type; + read_leb_uint32(frame_ip, frame_ip_end, ref_type); + if (!jit_compile_op_ref_null(cc, ref_type)) + return false; + break; + } + case WASM_OP_REF_IS_NULL: + { + if (!jit_compile_op_ref_is_null(cc)) + return false; + break; + } + case WASM_OP_REF_FUNC: + { + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + if (!jit_compile_op_ref_func(cc, func_idx)) + return false; + break; + } +#endif + + case WASM_OP_GET_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!jit_compile_op_get_local(cc, local_idx)) + return false; + break; + + case WASM_OP_SET_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!jit_compile_op_set_local(cc, local_idx)) + return false; + break; + + case WASM_OP_TEE_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!jit_compile_op_tee_local(cc, local_idx)) + return false; + break; + + case WASM_OP_GET_GLOBAL: + case WASM_OP_GET_GLOBAL_64: + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + if (!jit_compile_op_get_global(cc, global_idx)) + return false; + break; + + case WASM_OP_SET_GLOBAL: + case WASM_OP_SET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_AUX_STACK: + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + if (!jit_compile_op_set_global( + cc, global_idx, + opcode == WASM_OP_SET_GLOBAL_AUX_STACK ? true : false)) + return false; + break; + + case WASM_OP_I32_LOAD: + bytes = 4; + sign = true; + goto op_i32_load; + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + bytes = 1; + sign = (opcode == WASM_OP_I32_LOAD8_S) ? true : false; + goto op_i32_load; + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + bytes = 2; + sign = (opcode == WASM_OP_I32_LOAD16_S) ? true : false; + op_i32_load: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i32_load(cc, align, offset, bytes, sign, + false)) + return false; + break; + + case WASM_OP_I64_LOAD: + bytes = 8; + sign = true; + goto op_i64_load; + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + bytes = 1; + sign = (opcode == WASM_OP_I64_LOAD8_S) ? true : false; + goto op_i64_load; + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + bytes = 2; + sign = (opcode == WASM_OP_I64_LOAD16_S) ? true : false; + goto op_i64_load; + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + bytes = 4; + sign = (opcode == WASM_OP_I64_LOAD32_S) ? true : false; + op_i64_load: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i64_load(cc, align, offset, bytes, sign, + false)) + return false; + break; + + case WASM_OP_F32_LOAD: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f32_load(cc, align, offset)) + return false; + break; + + case WASM_OP_F64_LOAD: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f64_load(cc, align, offset)) + return false; + break; + + case WASM_OP_I32_STORE: + bytes = 4; + goto op_i32_store; + case WASM_OP_I32_STORE8: + bytes = 1; + goto op_i32_store; + case WASM_OP_I32_STORE16: + bytes = 2; + op_i32_store: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i32_store(cc, align, offset, bytes, false)) + return false; + break; + + case WASM_OP_I64_STORE: + bytes = 8; + goto op_i64_store; + case WASM_OP_I64_STORE8: + bytes = 1; + goto op_i64_store; + case WASM_OP_I64_STORE16: + bytes = 2; + goto op_i64_store; + case WASM_OP_I64_STORE32: + bytes = 4; + op_i64_store: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i64_store(cc, align, offset, bytes, false)) + return false; + break; + + case WASM_OP_F32_STORE: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f32_store(cc, align, offset)) + return false; + break; + + case WASM_OP_F64_STORE: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f64_store(cc, align, offset)) + return false; + break; + + case WASM_OP_MEMORY_SIZE: + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_size(cc, mem_idx)) + return false; + break; + + case WASM_OP_MEMORY_GROW: + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_grow(cc, mem_idx)) + return false; + break; + + case WASM_OP_I32_CONST: + read_leb_int32(frame_ip, frame_ip_end, i32_const); + if (!jit_compile_op_i32_const(cc, i32_const)) + return false; + break; + + case WASM_OP_I64_CONST: + read_leb_int64(frame_ip, frame_ip_end, i64_const); + if (!jit_compile_op_i64_const(cc, i64_const)) + return false; + break; + + case WASM_OP_F32_CONST: + p_f32 = (uint8 *)&f32_const; + for (i = 0; i < sizeof(float32); i++) + *p_f32++ = *frame_ip++; + if (!jit_compile_op_f32_const(cc, f32_const)) + return false; + break; + + case WASM_OP_F64_CONST: + p_f64 = (uint8 *)&f64_const; + for (i = 0; i < sizeof(float64); i++) + *p_f64++ = *frame_ip++; + if (!jit_compile_op_f64_const(cc, f64_const)) + return false; + break; + + case WASM_OP_I32_EQZ: + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + if (!jit_compile_op_i32_compare(cc, INT_EQZ + opcode + - WASM_OP_I32_EQZ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_I64_EQZ: + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + if (!jit_compile_op_i64_compare(cc, INT_EQZ + opcode + - WASM_OP_I64_EQZ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + if (!jit_compile_op_f32_compare(cc, FLOAT_EQ + opcode + - WASM_OP_F32_EQ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + if (!jit_compile_op_f64_compare(cc, FLOAT_EQ + opcode + - WASM_OP_F64_EQ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_I32_CLZ: + if (!jit_compile_op_i32_clz(cc)) + return false; + break; + + case WASM_OP_I32_CTZ: + if (!jit_compile_op_i32_ctz(cc)) + return false; + break; + + case WASM_OP_I32_POPCNT: + if (!jit_compile_op_i32_popcnt(cc)) + return false; + break; + + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + if (!jit_compile_op_i32_arithmetic( + cc, INT_ADD + opcode - WASM_OP_I32_ADD, &frame_ip)) + return false; + break; + + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + if (!jit_compile_op_i32_bitwise(cc, INT_SHL + opcode + - WASM_OP_I32_AND)) + return false; + break; + + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + if (!jit_compile_op_i32_shift(cc, INT_SHL + opcode + - WASM_OP_I32_SHL)) + return false; + break; + + case WASM_OP_I64_CLZ: + if (!jit_compile_op_i64_clz(cc)) + return false; + break; + + case WASM_OP_I64_CTZ: + if (!jit_compile_op_i64_ctz(cc)) + return false; + break; + + case WASM_OP_I64_POPCNT: + if (!jit_compile_op_i64_popcnt(cc)) + return false; + break; + + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + if (!jit_compile_op_i64_arithmetic( + cc, INT_ADD + opcode - WASM_OP_I64_ADD, &frame_ip)) + return false; + break; + + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + if (!jit_compile_op_i64_bitwise(cc, INT_SHL + opcode + - WASM_OP_I64_AND)) + return false; + break; + + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + if (!jit_compile_op_i64_shift(cc, INT_SHL + opcode + - WASM_OP_I64_SHL)) + return false; + break; + + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + if (!jit_compile_op_f32_math(cc, FLOAT_ABS + opcode + - WASM_OP_F32_ABS)) + return false; + break; + + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + if (!jit_compile_op_f32_arithmetic(cc, FLOAT_ADD + opcode + - WASM_OP_F32_ADD)) + return false; + break; + + case WASM_OP_F32_COPYSIGN: + if (!jit_compile_op_f32_copysign(cc)) + return false; + break; + + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + if (!jit_compile_op_f64_math(cc, FLOAT_ABS + opcode + - WASM_OP_F64_ABS)) + return false; + break; + + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + if (!jit_compile_op_f64_arithmetic(cc, FLOAT_ADD + opcode + - WASM_OP_F64_ADD)) + return false; + break; + + case WASM_OP_F64_COPYSIGN: + if (!jit_compile_op_f64_copysign(cc)) + return false; + break; + + case WASM_OP_I32_WRAP_I64: + if (!jit_compile_op_i32_wrap_i64(cc)) + return false; + break; + + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + sign = (opcode == WASM_OP_I32_TRUNC_S_F32) ? true : false; + if (!jit_compile_op_i32_trunc_f32(cc, sign, false)) + return false; + break; + + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + sign = (opcode == WASM_OP_I32_TRUNC_S_F64) ? true : false; + if (!jit_compile_op_i32_trunc_f64(cc, sign, false)) + return false; + break; + + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + sign = (opcode == WASM_OP_I64_EXTEND_S_I32) ? true : false; + if (!jit_compile_op_i64_extend_i32(cc, sign)) + return false; + break; + + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + sign = (opcode == WASM_OP_I64_TRUNC_S_F32) ? true : false; + if (!jit_compile_op_i64_trunc_f32(cc, sign, false)) + return false; + break; + + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + sign = (opcode == WASM_OP_I64_TRUNC_S_F64) ? true : false; + if (!jit_compile_op_i64_trunc_f64(cc, sign, false)) + return false; + break; + + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + sign = (opcode == WASM_OP_F32_CONVERT_S_I32) ? true : false; + if (!jit_compile_op_f32_convert_i32(cc, sign)) + return false; + break; + + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + sign = (opcode == WASM_OP_F32_CONVERT_S_I64) ? true : false; + if (!jit_compile_op_f32_convert_i64(cc, sign)) + return false; + break; + + case WASM_OP_F32_DEMOTE_F64: + if (!jit_compile_op_f32_demote_f64(cc)) + return false; + break; + + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + sign = (opcode == WASM_OP_F64_CONVERT_S_I32) ? true : false; + if (!jit_compile_op_f64_convert_i32(cc, sign)) + return false; + break; + + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + sign = (opcode == WASM_OP_F64_CONVERT_S_I64) ? true : false; + if (!jit_compile_op_f64_convert_i64(cc, sign)) + return false; + break; + + case WASM_OP_F64_PROMOTE_F32: + if (!jit_compile_op_f64_promote_f32(cc)) + return false; + break; + + case WASM_OP_I32_REINTERPRET_F32: + if (!jit_compile_op_i32_reinterpret_f32(cc)) + return false; + break; + + case WASM_OP_I64_REINTERPRET_F64: + if (!jit_compile_op_i64_reinterpret_f64(cc)) + return false; + break; + + case WASM_OP_F32_REINTERPRET_I32: + if (!jit_compile_op_f32_reinterpret_i32(cc)) + return false; + break; + + case WASM_OP_F64_REINTERPRET_I64: + if (!jit_compile_op_f64_reinterpret_i64(cc)) + return false; + break; + + case WASM_OP_I32_EXTEND8_S: + if (!jit_compile_op_i32_extend_i32(cc, 8)) + return false; + break; + + case WASM_OP_I32_EXTEND16_S: + if (!jit_compile_op_i32_extend_i32(cc, 16)) + return false; + break; + + case WASM_OP_I64_EXTEND8_S: + if (!jit_compile_op_i64_extend_i64(cc, 8)) + return false; + break; + + case WASM_OP_I64_EXTEND16_S: + if (!jit_compile_op_i64_extend_i64(cc, 16)) + return false; + break; + + case WASM_OP_I64_EXTEND32_S: + if (!jit_compile_op_i64_extend_i64(cc, 32)) + return false; + break; + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + sign = (opcode == WASM_OP_I32_TRUNC_SAT_S_F32) ? true + : false; + if (!jit_compile_op_i32_trunc_f32(cc, sign, true)) + return false; + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + sign = (opcode == WASM_OP_I32_TRUNC_SAT_S_F64) ? true + : false; + if (!jit_compile_op_i32_trunc_f64(cc, sign, true)) + return false; + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + sign = (opcode == WASM_OP_I64_TRUNC_SAT_S_F32) ? true + : false; + if (!jit_compile_op_i64_trunc_f32(cc, sign, true)) + return false; + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + sign = (opcode == WASM_OP_I64_TRUNC_SAT_S_F64) ? true + : false; + if (!jit_compile_op_i64_trunc_f64(cc, sign, true)) + return false; + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + uint32 seg_idx = 0; + read_leb_uint32(frame_ip, frame_ip_end, seg_idx); + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_init(cc, mem_idx, seg_idx)) + return false; + break; + } + case WASM_OP_DATA_DROP: + { + uint32 seg_idx; + read_leb_uint32(frame_ip, frame_ip_end, seg_idx); + if (!jit_compile_op_data_drop(cc, seg_idx)) + return false; + break; + } + case WASM_OP_MEMORY_COPY: + { + uint32 src_mem_idx, dst_mem_idx; + read_leb_uint32(frame_ip, frame_ip_end, src_mem_idx); + read_leb_uint32(frame_ip, frame_ip_end, dst_mem_idx); + if (!jit_compile_op_memory_copy(cc, src_mem_idx, + dst_mem_idx)) + return false; + break; + } + case WASM_OP_MEMORY_FILL: + { + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_fill(cc, mem_idx)) + return false; + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_TABLE_INIT: + { + uint32 tbl_idx, tbl_seg_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_seg_idx); + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_init(cc, tbl_idx, + tbl_seg_idx)) + return false; + break; + } + case WASM_OP_ELEM_DROP: + { + uint32 tbl_seg_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_seg_idx); + if (!jit_compile_op_elem_drop(cc, tbl_seg_idx)) + return false; + break; + } + case WASM_OP_TABLE_COPY: + { + uint32 src_tbl_idx, dst_tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, dst_tbl_idx); + read_leb_uint32(frame_ip, frame_ip_end, src_tbl_idx); + if (!jit_compile_op_table_copy(cc, src_tbl_idx, + dst_tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_GROW: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_grow(cc, tbl_idx)) + return false; + break; + } + + case WASM_OP_TABLE_SIZE: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_size(cc, tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_FILL: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_fill(cc, tbl_idx)) + return false; + break; + } +#endif /* WASM_ENABLE_REF_TYPES */ + default: + jit_set_last_error(cc, "unsupported opcode"); + return false; + } + break; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + uint8 bin_op, op_type; + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + if (opcode != WASM_OP_ATOMIC_FENCE) { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + } + switch (opcode) { + case WASM_OP_ATOMIC_WAIT32: + if (!jit_compile_op_atomic_wait(cc, VALUE_TYPE_I32, + align, offset, 4)) + return false; + break; + case WASM_OP_ATOMIC_WAIT64: + if (!jit_compile_op_atomic_wait(cc, VALUE_TYPE_I64, + align, offset, 8)) + return false; + break; + case WASM_OP_ATOMIC_NOTIFY: + if (!jit_compiler_op_atomic_notify(cc, align, offset, + bytes)) + return false; + break; + case WASM_OP_ATOMIC_FENCE: + /* Skip memory index */ + frame_ip++; + if (!jit_compiler_op_atomic_fence(cc)) + return false; + break; + case WASM_OP_ATOMIC_I32_LOAD: + bytes = 4; + goto op_atomic_i32_load; + case WASM_OP_ATOMIC_I32_LOAD8_U: + bytes = 1; + goto op_atomic_i32_load; + case WASM_OP_ATOMIC_I32_LOAD16_U: + bytes = 2; + op_atomic_i32_load: + if (!jit_compile_op_i32_load(cc, align, offset, bytes, + sign, true)) + return false; + break; + + case WASM_OP_ATOMIC_I64_LOAD: + bytes = 8; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD8_U: + bytes = 1; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD16_U: + bytes = 2; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD32_U: + bytes = 4; + op_atomic_i64_load: + if (!jit_compile_op_i64_load(cc, align, offset, bytes, + sign, true)) + return false; + break; + + case WASM_OP_ATOMIC_I32_STORE: + bytes = 4; + goto op_atomic_i32_store; + case WASM_OP_ATOMIC_I32_STORE8: + bytes = 1; + goto op_atomic_i32_store; + case WASM_OP_ATOMIC_I32_STORE16: + bytes = 2; + op_atomic_i32_store: + if (!jit_compile_op_i32_store(cc, align, offset, bytes, + true)) + return false; + break; + + case WASM_OP_ATOMIC_I64_STORE: + bytes = 8; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE8: + bytes = 1; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE16: + bytes = 2; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE32: + bytes = 4; + op_atomic_i64_store: + if (!jit_compile_op_i64_store(cc, align, offset, bytes, + true)) + return false; + break; + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + bytes = 4; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + bytes = 8; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + bytes = 1; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + bytes = 2; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + bytes = 1; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + bytes = 2; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + bytes = 4; + op_type = VALUE_TYPE_I64; + op_atomic_cmpxchg: + if (!jit_compile_op_atomic_cmpxchg(cc, op_type, align, + offset, bytes)) + return false; + break; + + COMPILE_ATOMIC_RMW(Add, ADD); + COMPILE_ATOMIC_RMW(Sub, SUB); + COMPILE_ATOMIC_RMW(And, AND); + COMPILE_ATOMIC_RMW(Or, OR); + COMPILE_ATOMIC_RMW(Xor, XOR); + COMPILE_ATOMIC_RMW(Xchg, XCHG); + + build_atomic_rmw: + if (!jit_compile_op_atomic_rmw(cc, bin_op, op_type, + align, offset, bytes)) + return false; + break; + + default: + jit_set_last_error(cc, "unsupported opcode"); + return false; + } + break; + } +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + + default: + jit_set_last_error(cc, "unsupported opcode"); + return false; + } + /* Error may occur when creating registers, basic blocks, insns, + consts and labels, in which the return value may be unchecked, + here we check again */ + if (jit_get_last_error(cc)) { + return false; + } + } + + (void)func_idx; + return true; +fail: + return false; +} + +JitBasicBlock * +jit_frontend_translate_func(JitCompContext *cc) +{ + JitFrame *jit_frame; + JitBasicBlock *basic_block_entry; + + if (!(jit_frame = init_func_translation(cc))) { + return NULL; + } + + if (!(basic_block_entry = create_func_block(cc))) { + return NULL; + } + + if (!jit_compile_func(cc)) { + return NULL; + } + + return basic_block_entry; +} + +uint32 +jit_frontend_get_jitted_return_addr_offset() +{ + return (uint32)offsetof(WASMInterpFrame, jitted_return_addr); +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_frontend.h b/wasm-micro-runtime/core/iwasm/fast-jit/jit_frontend.h new file mode 100644 index 0000000..9065d23 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_frontend.h @@ -0,0 +1,514 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_FRONTEND_H_ +#define _JIT_FRONTEND_H_ + +#include "jit_utils.h" +#include "jit_ir.h" +#include "../interpreter/wasm_interp.h" +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if WASM_ENABLE_AOT == 0 +typedef enum IntCond { + INT_EQZ = 0, + INT_EQ, + INT_NE, + INT_LT_S, + INT_LT_U, + INT_GT_S, + INT_GT_U, + INT_LE_S, + INT_LE_U, + INT_GE_S, + INT_GE_U +} IntCond; + +typedef enum FloatCond { + FLOAT_EQ = 0, + FLOAT_NE, + FLOAT_LT, + FLOAT_GT, + FLOAT_LE, + FLOAT_GE, + FLOAT_UNO +} FloatCond; +#else +#define IntCond AOTIntCond +#define FloatCond AOTFloatCond +#endif + +typedef enum IntArithmetic { + INT_ADD = 0, + INT_SUB, + INT_MUL, + INT_DIV_S, + INT_DIV_U, + INT_REM_S, + INT_REM_U +} IntArithmetic; + +typedef enum V128Arithmetic { + V128_ADD = 0, + V128_SUB, + V128_MUL, + V128_DIV, + V128_NEG, + V128_MIN, + V128_MAX, +} V128Arithmetic; + +typedef enum IntBitwise { + INT_AND = 0, + INT_OR, + INT_XOR, +} IntBitwise; + +typedef enum V128Bitwise { + V128_NOT, + V128_AND, + V128_ANDNOT, + V128_OR, + V128_XOR, + V128_BITSELECT, +} V128Bitwise; + +typedef enum IntShift { + INT_SHL = 0, + INT_SHR_S, + INT_SHR_U, + INT_ROTL, + INT_ROTR +} IntShift; + +typedef enum FloatMath { + FLOAT_ABS = 0, + FLOAT_NEG, + FLOAT_CEIL, + FLOAT_FLOOR, + FLOAT_TRUNC, + FLOAT_NEAREST, + FLOAT_SQRT +} FloatMath; + +typedef enum FloatArithmetic { + FLOAT_ADD = 0, + FLOAT_SUB, + FLOAT_MUL, + FLOAT_DIV, + FLOAT_MIN, + FLOAT_MAX, +} FloatArithmetic; + +#if WASM_ENABLE_SHARED_MEMORY != 0 +typedef enum AtomicRMWBinOp { + AtomicRMWBinOpAdd, + AtomicRMWBinOpSub, + AtomicRMWBinOpAnd, + AtomicRMWBinOpOr, + AtomicRMWBinOpXor, + AtomicRMWBinOpXchg +} AtomicRMWBinOp; +#endif + +/** + * Translate instructions in a function. The translated block must + * end with a branch instruction whose targets are offsets relating to + * the end bcip of the translated block, which are integral constants. + * If a target of a branch is really a constant value (which should be + * rare), put it into a register and then jump to the register instead + * of using the constant value directly in the target. In the + * translation process, don't create any new labels. The code bcip of + * the begin and end of the translated block is stored in the + * jit_annl_begin_bcip and jit_annl_end_bcip annotations of the label + * of the block, which must be the same as the bcips used in + * profiling. + * + * NOTE: the function must explicitly set SP to correct value when the + * entry's bcip is the function's entry address. + * + * @param cc containing compilation context of generated IR + * @param entry entry of the basic block to be translated. If its + * value is NULL, the function will clean up any pass local data that + * might be created previously. + * @param is_reached a bitmap recording which bytecode has been + * reached as a block entry + * + * @return IR block containing translated instructions if succeeds, + * NULL otherwise + */ +JitBasicBlock * +jit_frontend_translate_func(JitCompContext *cc); + +/** + * Lower the IR of the given compilation context. + * + * @param cc the compilation context + * + * @return true if succeeds, false otherwise + */ +bool +jit_frontend_lower(JitCompContext *cc); + +uint32 +jit_frontend_get_jitted_return_addr_offset(); + +uint32 +jit_frontend_get_global_data_offset(const WASMModule *module, + uint32 global_idx); + +uint32 +jit_frontend_get_table_inst_offset(const WASMModule *module, uint32 tbl_idx); + +uint32 +jit_frontend_get_module_inst_extra_offset(const WASMModule *module); + +JitReg +get_module_inst_reg(JitFrame *frame); + +JitReg +get_module_reg(JitFrame *frame); + +JitReg +get_import_func_ptrs_reg(JitFrame *frame); + +JitReg +get_fast_jit_func_ptrs_reg(JitFrame *frame); + +JitReg +get_func_type_indexes_reg(JitFrame *frame); + +JitReg +get_aux_stack_bound_reg(JitFrame *frame); + +JitReg +get_aux_stack_bottom_reg(JitFrame *frame); + +JitReg +get_memory_inst_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_cur_page_count_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_memory_data_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_memory_data_end_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_mem_bound_check_1byte_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_mem_bound_check_2bytes_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_mem_bound_check_4bytes_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_mem_bound_check_8bytes_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_mem_bound_check_16bytes_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_table_elems_reg(JitFrame *frame, uint32 table_idx); + +JitReg +get_table_cur_size_reg(JitFrame *frame, uint32 table_idx); + +void +clear_fixed_virtual_regs(JitFrame *frame); + +void +clear_memory_regs(JitFrame *frame); + +void +clear_table_regs(JitFrame *frame); + +/** + * Get the offset from frame pointer to the n-th local variable slot. + * + * @param n the index to the local variable array + * + * @return the offset from frame pointer to the local variable slot + */ +static inline unsigned +offset_of_local(unsigned n) +{ + return offsetof(WASMInterpFrame, lp) + n * 4; +} + +/** + * Generate instruction to load an integer from the frame. + * + * This and the below gen_load_X functions generate instructions to + * load values from the frame into registers if the values have not + * been loaded yet. + * + * @param frame the frame information + * @param n slot index to the local variable array + * + * @return register holding the loaded value + */ +JitReg +gen_load_i32(JitFrame *frame, unsigned n); + +/** + * Generate instruction to load a i64 integer from the frame. + * + * @param frame the frame information + * @param n slot index to the local variable array + * + * @return register holding the loaded value + */ +JitReg +gen_load_i64(JitFrame *frame, unsigned n); + +/** + * Generate instruction to load a floating point value from the frame. + * + * @param frame the frame information + * @param n slot index to the local variable array + * + * @return register holding the loaded value + */ +JitReg +gen_load_f32(JitFrame *frame, unsigned n); + +/** + * Generate instruction to load a double value from the frame. + * + * @param frame the frame information + * @param n slot index to the local variable array + * + * @return register holding the loaded value + */ +JitReg +gen_load_f64(JitFrame *frame, unsigned n); + +/** + * Generate instructions to commit computation result to the frame. + * The general principle is to only commit values that will be used + * through the frame. + * + * @param frame the frame information + * @param begin the begin value slot to commit + * @param end the end value slot to commit + */ +void +gen_commit_values(JitFrame *frame, JitValueSlot *begin, JitValueSlot *end); + +/** + * Generate instructions to commit SP and IP pointers to the frame. + * + * @param frame the frame information + */ +void +gen_commit_sp_ip(JitFrame *frame); + +/** + * Generate commit instructions for the block end. + * + * @param frame the frame information + */ +static inline void +gen_commit_for_branch(JitFrame *frame) +{ + gen_commit_values(frame, frame->lp, frame->sp); +} + +/** + * Generate commit instructions for exception checks. + * + * @param frame the frame information + */ +static inline void +gen_commit_for_exception(JitFrame *frame) +{ + gen_commit_values(frame, frame->lp, frame->lp + frame->max_locals); + gen_commit_sp_ip(frame); +} + +/** + * Generate commit instructions to commit all status. + * + * @param frame the frame information + */ +static inline void +gen_commit_for_all(JitFrame *frame) +{ + gen_commit_values(frame, frame->lp, frame->sp); + gen_commit_sp_ip(frame); +} + +static inline void +clear_values(JitFrame *frame) +{ + size_t total_size = + sizeof(JitValueSlot) * (frame->max_locals + frame->max_stacks); + memset(frame->lp, 0, total_size); + frame->committed_sp = NULL; + frame->committed_ip = NULL; + clear_fixed_virtual_regs(frame); +} + +static inline void +push_i32(JitFrame *frame, JitReg value) +{ + frame->sp->reg = value; + frame->sp->dirty = 1; + frame->sp++; +} + +static inline void +push_i64(JitFrame *frame, JitReg value) +{ + frame->sp->reg = value; + frame->sp->dirty = 1; + frame->sp++; + frame->sp->reg = value; + frame->sp->dirty = 1; + frame->sp++; +} + +static inline void +push_f32(JitFrame *frame, JitReg value) +{ + push_i32(frame, value); +} + +static inline void +push_f64(JitFrame *frame, JitReg value) +{ + push_i64(frame, value); +} + +static inline JitReg +pop_i32(JitFrame *frame) +{ + frame->sp--; + return gen_load_i32(frame, frame->sp - frame->lp); +} + +static inline JitReg +pop_i64(JitFrame *frame) +{ + frame->sp -= 2; + return gen_load_i64(frame, frame->sp - frame->lp); +} + +static inline JitReg +pop_f32(JitFrame *frame) +{ + frame->sp--; + return gen_load_f32(frame, frame->sp - frame->lp); +} + +static inline JitReg +pop_f64(JitFrame *frame) +{ + frame->sp -= 2; + return gen_load_f64(frame, frame->sp - frame->lp); +} + +static inline void +pop(JitFrame *frame, int n) +{ + frame->sp -= n; + memset(frame->sp, 0, n * sizeof(*frame->sp)); +} + +static inline JitReg +local_i32(JitFrame *frame, int n) +{ + return gen_load_i32(frame, n); +} + +static inline JitReg +local_i64(JitFrame *frame, int n) +{ + return gen_load_i64(frame, n); +} + +static inline JitReg +local_f32(JitFrame *frame, int n) +{ + return gen_load_f32(frame, n); +} + +static inline JitReg +local_f64(JitFrame *frame, int n) +{ + return gen_load_f64(frame, n); +} + +static void +set_local_i32(JitFrame *frame, int n, JitReg val) +{ + frame->lp[n].reg = val; + frame->lp[n].dirty = 1; +} + +static void +set_local_i64(JitFrame *frame, int n, JitReg val) +{ + frame->lp[n].reg = val; + frame->lp[n].dirty = 1; + frame->lp[n + 1].reg = val; + frame->lp[n + 1].dirty = 1; +} + +static inline void +set_local_f32(JitFrame *frame, int n, JitReg val) +{ + set_local_i32(frame, n, val); +} + +static inline void +set_local_f64(JitFrame *frame, int n, JitReg val) +{ + set_local_i64(frame, n, val); +} + +#define POP(jit_value, value_type) \ + do { \ + if (!jit_cc_pop_value(cc, value_type, &jit_value)) \ + goto fail; \ + } while (0) + +#define POP_I32(v) POP(v, VALUE_TYPE_I32) +#define POP_I64(v) POP(v, VALUE_TYPE_I64) +#define POP_F32(v) POP(v, VALUE_TYPE_F32) +#define POP_F64(v) POP(v, VALUE_TYPE_F64) +#define POP_FUNCREF(v) POP(v, VALUE_TYPE_FUNCREF) +#define POP_EXTERNREF(v) POP(v, VALUE_TYPE_EXTERNREF) + +#define PUSH(jit_value, value_type) \ + do { \ + if (!jit_value) \ + goto fail; \ + if (!jit_cc_push_value(cc, value_type, jit_value)) \ + goto fail; \ + } while (0) + +#define PUSH_I32(v) PUSH(v, VALUE_TYPE_I32) +#define PUSH_I64(v) PUSH(v, VALUE_TYPE_I64) +#define PUSH_F32(v) PUSH(v, VALUE_TYPE_F32) +#define PUSH_F64(v) PUSH(v, VALUE_TYPE_F64) +#define PUSH_FUNCREF(v) PUSH(v, VALUE_TYPE_FUNCREF) +#define PUSH_EXTERNREF(v) PUSH(v, VALUE_TYPE_EXTERNREF) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.c b/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.c new file mode 100644 index 0000000..68503e3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.c @@ -0,0 +1,1427 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_ir.h" +#include "jit_codegen.h" +#include "jit_frontend.h" + +/** + * Operand kinds of instructions. + */ +enum { + JIT_OPND_KIND_Reg, + JIT_OPND_KIND_VReg, + JIT_OPND_KIND_LookupSwitch, +}; + +/** + * Operand kind of each instruction. + */ +static const uint8 insn_opnd_kind[] = { +#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) JIT_OPND_KIND_##OPND_KIND, +#include "jit_ir.def" +#undef INSN +}; + +/** + * Operand number of each instruction. + */ +static const uint8 insn_opnd_num[] = { +#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) OPND_NUM, +#include "jit_ir.def" +#undef INSN +}; + +/** + * Operand number of each instruction. + */ +static const uint8 insn_opnd_first_use[] = { +#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) FIRST_USE, +#include "jit_ir.def" +#undef INSN +}; + +#define JIT_INSN_NEW_Reg(OPND_NUM) \ + jit_calloc(offsetof(JitInsn, _opnd) + sizeof(JitReg) * (OPND_NUM)) +#define JIT_INSN_NEW_VReg(OPND_NUM) \ + jit_calloc(offsetof(JitInsn, _opnd._opnd_VReg._reg) \ + + sizeof(JitReg) * (OPND_NUM)) + +JitInsn * +_jit_insn_new_Reg_0(JitOpcode opc) +{ + JitInsn *insn = JIT_INSN_NEW_Reg(0); + + if (insn) { + insn->opcode = opc; + } + + return insn; +} + +JitInsn * +_jit_insn_new_Reg_1(JitOpcode opc, JitReg r0) +{ + JitInsn *insn = JIT_INSN_NEW_Reg(1); + + if (insn) { + insn->opcode = opc; + *jit_insn_opnd(insn, 0) = r0; + } + + return insn; +} + +JitInsn * +_jit_insn_new_Reg_2(JitOpcode opc, JitReg r0, JitReg r1) +{ + JitInsn *insn = JIT_INSN_NEW_Reg(2); + + if (insn) { + insn->opcode = opc; + *jit_insn_opnd(insn, 0) = r0; + *jit_insn_opnd(insn, 1) = r1; + } + + return insn; +} + +JitInsn * +_jit_insn_new_Reg_3(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2) +{ + JitInsn *insn = JIT_INSN_NEW_Reg(3); + + if (insn) { + insn->opcode = opc; + *jit_insn_opnd(insn, 0) = r0; + *jit_insn_opnd(insn, 1) = r1; + *jit_insn_opnd(insn, 2) = r2; + } + + return insn; +} + +JitInsn * +_jit_insn_new_Reg_4(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2, JitReg r3) +{ + JitInsn *insn = JIT_INSN_NEW_Reg(4); + + if (insn) { + insn->opcode = opc; + *jit_insn_opnd(insn, 0) = r0; + *jit_insn_opnd(insn, 1) = r1; + *jit_insn_opnd(insn, 2) = r2; + *jit_insn_opnd(insn, 3) = r3; + } + + return insn; +} + +JitInsn * +_jit_insn_new_Reg_5(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2, JitReg r3, + JitReg r4) +{ + JitInsn *insn = JIT_INSN_NEW_Reg(5); + + if (insn) { + insn->opcode = opc; + *jit_insn_opnd(insn, 0) = r0; + *jit_insn_opnd(insn, 1) = r1; + *jit_insn_opnd(insn, 2) = r2; + *jit_insn_opnd(insn, 3) = r3; + *jit_insn_opnd(insn, 4) = r4; + } + + return insn; +} + +JitInsn * +_jit_insn_new_VReg_1(JitOpcode opc, JitReg r0, int n) +{ + JitInsn *insn = JIT_INSN_NEW_VReg(1 + n); + + if (insn) { + insn->opcode = opc; + insn->_opnd._opnd_VReg._reg_num = 1 + n; + *(jit_insn_opndv(insn, 0)) = r0; + } + + return insn; +} + +JitInsn * +_jit_insn_new_VReg_2(JitOpcode opc, JitReg r0, JitReg r1, int n) +{ + JitInsn *insn = JIT_INSN_NEW_VReg(2 + n); + + if (insn) { + insn->opcode = opc; + insn->_opnd._opnd_VReg._reg_num = 2 + n; + *(jit_insn_opndv(insn, 0)) = r0; + *(jit_insn_opndv(insn, 1)) = r1; + } + + return insn; +} + +JitInsn * +_jit_insn_new_LookupSwitch_1(JitOpcode opc, JitReg value, uint32 num) +{ + JitOpndLookupSwitch *opnd = NULL; + JitInsn *insn = + jit_calloc(offsetof(JitInsn, _opnd._opnd_LookupSwitch.match_pairs) + + sizeof(opnd->match_pairs[0]) * num); + + if (insn) { + insn->opcode = opc; + opnd = jit_insn_opndls(insn); + opnd->value = value; + opnd->match_pairs_num = num; + } + + return insn; +} + +#undef JIT_INSN_NEW_Reg +#undef JIT_INSN_NEW_VReg + +void +jit_insn_insert_before(JitInsn *insn1, JitInsn *insn2) +{ + bh_assert(insn1->prev); + insn1->prev->next = insn2; + insn2->prev = insn1->prev; + insn2->next = insn1; + insn1->prev = insn2; +} + +void +jit_insn_insert_after(JitInsn *insn1, JitInsn *insn2) +{ + bh_assert(insn1->next); + insn1->next->prev = insn2; + insn2->next = insn1->next; + insn2->prev = insn1; + insn1->next = insn2; +} + +void +jit_insn_unlink(JitInsn *insn) +{ + bh_assert(insn->prev); + insn->prev->next = insn->next; + bh_assert(insn->next); + insn->next->prev = insn->prev; + insn->prev = insn->next = NULL; +} + +unsigned +jit_insn_hash(JitInsn *insn) +{ + const uint8 opcode = insn->opcode; + unsigned hash = opcode, i; + + /* Currently, only instructions with Reg kind operand require + hashing. For others, simply use opcode as the hash value. */ + if (insn_opnd_kind[opcode] != JIT_OPND_KIND_Reg + || insn_opnd_num[opcode] < 1) + return hash; + + /* All the instructions with hashing support must be in the + assignment format, i.e. the first operand is the result (hence + being ignored) and all the others are operands. This is also + true for CHK instructions, whose first operand is the instruction + pointer. */ + for (i = 1; i < insn_opnd_num[opcode]; i++) + hash = ((hash << 5) - hash) + *(jit_insn_opnd(insn, i)); + + return hash; +} + +bool +jit_insn_equal(JitInsn *insn1, JitInsn *insn2) +{ + const uint8 opcode = insn1->opcode; + unsigned i; + + if (insn2->opcode != opcode) + return false; + + if (insn_opnd_kind[opcode] != JIT_OPND_KIND_Reg + || insn_opnd_num[opcode] < 1) + return false; + + for (i = 1; i < insn_opnd_num[opcode]; i++) + if (*(jit_insn_opnd(insn1, i)) != *(jit_insn_opnd(insn2, i))) + return false; + + return true; +} + +JitRegVec +jit_insn_opnd_regs(JitInsn *insn) +{ + JitRegVec vec = { 0 }; + JitOpndLookupSwitch *ls; + + vec._stride = 1; + + switch (insn_opnd_kind[insn->opcode]) { + case JIT_OPND_KIND_Reg: + vec.num = insn_opnd_num[insn->opcode]; + vec._base = jit_insn_opnd(insn, 0); + break; + + case JIT_OPND_KIND_VReg: + vec.num = jit_insn_opndv_num(insn); + vec._base = jit_insn_opndv(insn, 0); + break; + + case JIT_OPND_KIND_LookupSwitch: + ls = jit_insn_opndls(insn); + vec.num = ls->match_pairs_num + 2; + vec._base = &ls->value; + vec._stride = sizeof(ls->match_pairs[0]) / sizeof(*vec._base); + break; + } + + return vec; +} + +unsigned +jit_insn_opnd_first_use(JitInsn *insn) +{ + return insn_opnd_first_use[insn->opcode]; +} + +JitBasicBlock * +jit_basic_block_new(JitReg label, int n) +{ + JitBasicBlock *block = jit_insn_new_PHI(label, n); + if (!block) + return NULL; + + block->prev = block->next = block; + return block; +} + +void +jit_basic_block_delete(JitBasicBlock *block) +{ + JitInsn *insn, *next_insn, *end; + + if (!block) + return; + + insn = jit_basic_block_first_insn(block); + end = jit_basic_block_end_insn(block); + + for (; insn != end; insn = next_insn) { + next_insn = insn->next; + jit_insn_delete(insn); + } + + jit_insn_delete(block); +} + +JitRegVec +jit_basic_block_preds(JitBasicBlock *block) +{ + JitRegVec vec; + + vec.num = jit_insn_opndv_num(block) - 1; + vec._base = vec.num > 0 ? jit_insn_opndv(block, 1) : NULL; + vec._stride = 1; + + return vec; +} + +JitRegVec +jit_basic_block_succs(JitBasicBlock *block) +{ + JitInsn *last_insn = jit_basic_block_last_insn(block); + JitRegVec vec; + + vec.num = 0; + vec._base = NULL; + vec._stride = 1; + + switch (last_insn->opcode) { + case JIT_OP_JMP: + vec.num = 1; + vec._base = jit_insn_opnd(last_insn, 0); + break; + + case JIT_OP_BEQ: + case JIT_OP_BNE: + case JIT_OP_BGTS: + case JIT_OP_BGES: + case JIT_OP_BLTS: + case JIT_OP_BLES: + case JIT_OP_BGTU: + case JIT_OP_BGEU: + case JIT_OP_BLTU: + case JIT_OP_BLEU: + vec.num = 2; + vec._base = jit_insn_opnd(last_insn, 1); + break; + + case JIT_OP_LOOKUPSWITCH: + { + JitOpndLookupSwitch *opnd = jit_insn_opndls(last_insn); + vec.num = opnd->match_pairs_num + 1; + vec._base = &opnd->default_target; + vec._stride = sizeof(opnd->match_pairs[0]) / sizeof(*vec._base); + break; + } + + default: + vec._stride = 0; + } + + return vec; +} + +JitCompContext * +jit_cc_init(JitCompContext *cc, unsigned htab_size) +{ + JitBasicBlock *entry_block, *exit_block; + unsigned i, num; + + memset(cc, 0, sizeof(*cc)); + cc->_reference_count = 1; + jit_annl_enable_basic_block(cc); + + /* Create entry and exit blocks. They must be the first two + blocks respectively. */ + if (!(entry_block = jit_cc_new_basic_block(cc, 0))) + goto fail; + + if (!(exit_block = jit_cc_new_basic_block(cc, 0))) { + jit_basic_block_delete(entry_block); + goto fail; + } + + /* Record the entry and exit labels, whose indexes must be 0 and 1 + respectively. */ + cc->entry_label = jit_basic_block_label(entry_block); + cc->exit_label = jit_basic_block_label(exit_block); + bh_assert(jit_reg_no(cc->entry_label) == 0 + && jit_reg_no(cc->exit_label) == 1); + + if (!(cc->exce_basic_blocks = + jit_calloc(sizeof(JitBasicBlock *) * EXCE_NUM))) + goto fail; + + if (!(cc->incoming_insns_for_exec_bbs = + jit_calloc(sizeof(JitIncomingInsnList) * EXCE_NUM))) + goto fail; + + cc->hreg_info = jit_codegen_get_hreg_info(); + bh_assert(cc->hreg_info->info[JIT_REG_KIND_I32].num > 3); + + /* Initialize virtual registers for hard registers. */ + for (i = JIT_REG_KIND_VOID; i < JIT_REG_KIND_L32; i++) { + if ((num = cc->hreg_info->info[i].num)) { + /* Initialize the capacity to be large enough. */ + jit_cc_new_reg(cc, i); + bh_assert(cc->_ann._reg_capacity[i] > num); + cc->_ann._reg_num[i] = num; + } + } + + /* Create registers for frame pointer, exec_env and cmp. */ + cc->fp_reg = jit_reg_new(JIT_REG_KIND_PTR, cc->hreg_info->fp_hreg_index); + cc->exec_env_reg = + jit_reg_new(JIT_REG_KIND_PTR, cc->hreg_info->exec_env_hreg_index); + cc->cmp_reg = jit_reg_new(JIT_REG_KIND_I32, cc->hreg_info->cmp_hreg_index); + + cc->_const_val._hash_table_size = htab_size; + + if (!(cc->_const_val._hash_table = + jit_calloc(htab_size * sizeof(*cc->_const_val._hash_table)))) + goto fail; + + return cc; + +fail: + jit_cc_destroy(cc); + return NULL; +} + +void +jit_cc_destroy(JitCompContext *cc) +{ + unsigned i, end; + JitBasicBlock *block; + JitIncomingInsn *incoming_insn, *incoming_insn_next; + + jit_block_stack_destroy(&cc->block_stack); + + if (cc->jit_frame) { + if (cc->jit_frame->memory_regs) + jit_free(cc->jit_frame->memory_regs); + if (cc->jit_frame->table_regs) + jit_free(cc->jit_frame->table_regs); + jit_free(cc->jit_frame); + } + + if (cc->memory_regs) + jit_free(cc->memory_regs); + + if (cc->table_regs) + jit_free(cc->table_regs); + + jit_free(cc->_const_val._hash_table); + + /* Release the instruction hash table. */ + jit_cc_disable_insn_hash(cc); + + jit_free(cc->exce_basic_blocks); + + if (cc->incoming_insns_for_exec_bbs) { + for (i = 0; i < EXCE_NUM; i++) { + incoming_insn = cc->incoming_insns_for_exec_bbs[i]; + while (incoming_insn) { + incoming_insn_next = incoming_insn->next; + jit_free(incoming_insn); + incoming_insn = incoming_insn_next; + } + } + jit_free(cc->incoming_insns_for_exec_bbs); + } + + /* Release entry and exit blocks. */ + if (0 != cc->entry_label) + jit_basic_block_delete(jit_cc_entry_basic_block(cc)); + if (0 != cc->exit_label) + jit_basic_block_delete(jit_cc_exit_basic_block(cc)); + + /* clang-format off */ + /* Release blocks and instructions. */ + JIT_FOREACH_BLOCK(cc, i, end, block) + { + jit_basic_block_delete(block); + } + /* clang-format on */ + + /* Release constant values. */ + for (i = JIT_REG_KIND_VOID; i < JIT_REG_KIND_L32; i++) { + jit_free(cc->_const_val._value[i]); + jit_free(cc->_const_val._next[i]); + } + + /* Release storage of annotations. */ +#define ANN_LABEL(TYPE, NAME) jit_annl_disable_##NAME(cc); +#define ANN_INSN(TYPE, NAME) jit_anni_disable_##NAME(cc); +#define ANN_REG(TYPE, NAME) jit_annr_disable_##NAME(cc); +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG +} + +void +jit_cc_delete(JitCompContext *cc) +{ + if (cc && --cc->_reference_count == 0) { + jit_cc_destroy(cc); + jit_free(cc); + } +} + +/* + * Reallocate a memory block with the new_size. + * TODO: replace this with imported jit_realloc when it's available. + */ +static void * +_jit_realloc(void *ptr, unsigned new_size, unsigned old_size) +{ + void *new_ptr = jit_malloc(new_size); + + if (new_ptr) { + bh_assert(new_size > old_size); + + if (ptr) { + memcpy(new_ptr, ptr, old_size); + memset((uint8 *)new_ptr + old_size, 0, new_size - old_size); + jit_free(ptr); + } + else + memset(new_ptr, 0, new_size); + } + + return new_ptr; +} + +static unsigned +hash_of_const(unsigned kind, unsigned size, void *val) +{ + uint8 *p = (uint8 *)val, *end = p + size; + unsigned hash = kind; + + do + hash = ((hash << 5) - hash) + *p++; + while (p != end); + + return hash; +} + +static inline void * +address_of_const(JitCompContext *cc, JitReg reg, unsigned size) +{ + int kind = jit_reg_kind(reg); + unsigned no = jit_reg_no(reg); + unsigned idx = no & ~_JIT_REG_CONST_IDX_FLAG; + + bh_assert(kind < JIT_REG_KIND_L32); + bh_assert(jit_reg_is_const_idx(reg) && idx < cc->_const_val._num[kind]); + + return cc->_const_val._value[kind] + size * idx; +} + +static inline JitReg +next_of_const(JitCompContext *cc, JitReg reg) +{ + int kind = jit_reg_kind(reg); + unsigned no = jit_reg_no(reg); + unsigned idx = no & ~_JIT_REG_CONST_IDX_FLAG; + + bh_assert(kind < JIT_REG_KIND_L32); + bh_assert(jit_reg_is_const_idx(reg) && idx < cc->_const_val._num[kind]); + + return cc->_const_val._next[kind][idx]; +} + +/** + * Put a constant value into the compilation context. + * + * @param cc compilation context + * @param kind register kind + * @param size size of the value + * @param val pointer to value which must be aligned + * + * @return a constant register containing the value + */ +static JitReg +_jit_cc_new_const(JitCompContext *cc, int kind, unsigned size, void *val) +{ + unsigned num = cc->_const_val._num[kind], slot; + unsigned capacity = cc->_const_val._capacity[kind]; + uint8 *new_value; + JitReg r, *new_next; + + bh_assert(num <= capacity); + + /* Find the existing value first. */ + slot = hash_of_const(kind, size, val) % cc->_const_val._hash_table_size; + r = cc->_const_val._hash_table[slot]; + + for (; r; r = next_of_const(cc, r)) + if (jit_reg_kind(r) == kind + && !memcmp(val, address_of_const(cc, r, size), size)) + return r; + + if (num == capacity) { + /* Increase the space of value and next. */ + capacity = capacity > 0 ? (capacity + capacity / 2) : 16; + new_value = _jit_realloc(cc->_const_val._value[kind], size * capacity, + size * num); + new_next = + _jit_realloc(cc->_const_val._next[kind], + sizeof(*new_next) * capacity, sizeof(*new_next) * num); + + if (new_value && new_next) { + cc->_const_val._value[kind] = new_value; + cc->_const_val._next[kind] = new_next; + } + else { + jit_set_last_error(cc, "create const register failed"); + jit_free(new_value); + jit_free(new_next); + return 0; + } + + cc->_const_val._capacity[kind] = capacity; + } + + bh_assert(num + 1 < (uint32)_JIT_REG_CONST_IDX_FLAG); + r = jit_reg_new(kind, _JIT_REG_CONST_IDX_FLAG | num); + memcpy(cc->_const_val._value[kind] + size * num, val, size); + cc->_const_val._next[kind][num] = cc->_const_val._hash_table[slot]; + cc->_const_val._hash_table[slot] = r; + cc->_const_val._num[kind] = num + 1; + + return r; +} + +static inline int32 +get_const_val_in_reg(JitReg reg) +{ + int shift = 8 * sizeof(reg) - _JIT_REG_KIND_SHIFT + 1; + return ((int32)(reg << shift)) >> shift; +} + +#define _JIT_CC_NEW_CONST_HELPER(KIND, TYPE, val) \ + do { \ + JitReg reg = jit_reg_new( \ + JIT_REG_KIND_##KIND, \ + (_JIT_REG_CONST_VAL_FLAG | ((JitReg)val & ~_JIT_REG_KIND_MASK))); \ + \ + if ((TYPE)get_const_val_in_reg(reg) == val) \ + return reg; \ + return _jit_cc_new_const(cc, JIT_REG_KIND_##KIND, sizeof(val), &val); \ + } while (0) + +JitReg +jit_cc_new_const_I32_rel(JitCompContext *cc, int32 val, uint32 rel) +{ + uint64 val64 = (uint64)(uint32)val | ((uint64)rel << 32); + _JIT_CC_NEW_CONST_HELPER(I32, uint64, val64); +} + +JitReg +jit_cc_new_const_I64(JitCompContext *cc, int64 val) +{ + _JIT_CC_NEW_CONST_HELPER(I64, int64, val); +} + +JitReg +jit_cc_new_const_F32(JitCompContext *cc, float val) +{ + int32 float_neg_zero = 0x80000000; + + if (!memcmp(&val, &float_neg_zero, sizeof(float))) + /* Create const -0.0f */ + return _jit_cc_new_const(cc, JIT_REG_KIND_F32, sizeof(float), &val); + + _JIT_CC_NEW_CONST_HELPER(F32, float, val); +} + +JitReg +jit_cc_new_const_F64(JitCompContext *cc, double val) +{ + int64 double_neg_zero = 0x8000000000000000ll; + + if (!memcmp(&val, &double_neg_zero, sizeof(double))) + /* Create const -0.0d */ + return _jit_cc_new_const(cc, JIT_REG_KIND_F64, sizeof(double), &val); + + _JIT_CC_NEW_CONST_HELPER(F64, double, val); +} + +#undef _JIT_CC_NEW_CONST_HELPER + +#define _JIT_CC_GET_CONST_HELPER(KIND, TYPE) \ + do { \ + bh_assert(jit_reg_kind(reg) == JIT_REG_KIND_##KIND); \ + bh_assert(jit_reg_is_const(reg)); \ + \ + return (jit_reg_is_const_val(reg) \ + ? (TYPE)get_const_val_in_reg(reg) \ + : *(TYPE *)(address_of_const(cc, reg, sizeof(TYPE)))); \ + } while (0) + +static uint64 +jit_cc_get_const_I32_helper(JitCompContext *cc, JitReg reg) +{ + _JIT_CC_GET_CONST_HELPER(I32, uint64); +} + +uint32 +jit_cc_get_const_I32_rel(JitCompContext *cc, JitReg reg) +{ + return (uint32)(jit_cc_get_const_I32_helper(cc, reg) >> 32); +} + +int32 +jit_cc_get_const_I32(JitCompContext *cc, JitReg reg) +{ + return (int32)(jit_cc_get_const_I32_helper(cc, reg)); +} + +int64 +jit_cc_get_const_I64(JitCompContext *cc, JitReg reg) +{ + _JIT_CC_GET_CONST_HELPER(I64, int64); +} + +float +jit_cc_get_const_F32(JitCompContext *cc, JitReg reg) +{ + _JIT_CC_GET_CONST_HELPER(F32, float); +} + +double +jit_cc_get_const_F64(JitCompContext *cc, JitReg reg) +{ + _JIT_CC_GET_CONST_HELPER(F64, double); +} + +#undef _JIT_CC_GET_CONST_HELPER + +#define _JIT_REALLOC_ANN(TYPE, NAME, ANN, POSTFIX) \ + if (successful && cc->_ann._##ANN##_##NAME##_enabled) { \ + TYPE *ptr = _jit_realloc(cc->_ann._##ANN##_##NAME POSTFIX, \ + sizeof(TYPE) * capacity, sizeof(TYPE) * num); \ + if (ptr) \ + cc->_ann._##ANN##_##NAME POSTFIX = ptr; \ + else \ + successful = false; \ + } + +JitReg +jit_cc_new_label(JitCompContext *cc) +{ + unsigned num = cc->_ann._label_num; + unsigned capacity = cc->_ann._label_capacity; + bool successful = true; + + bh_assert(num <= capacity); + + if (num == capacity) { + capacity = capacity > 0 ? (capacity + capacity / 2) : 16; + +#define EMPTY_POSTFIX +#define ANN_LABEL(TYPE, NAME) _JIT_REALLOC_ANN(TYPE, NAME, label, EMPTY_POSTFIX) +#include "jit_ir.def" +#undef ANN_LABEL +#undef EMPTY_POSTFIX + + if (!successful) { + jit_set_last_error(cc, "create label register failed"); + return 0; + } + + cc->_ann._label_capacity = capacity; + } + + cc->_ann._label_num = num + 1; + + return jit_reg_new(JIT_REG_KIND_L32, num); +} + +JitBasicBlock * +jit_cc_new_basic_block(JitCompContext *cc, int n) +{ + JitReg label = jit_cc_new_label(cc); + JitBasicBlock *block = NULL; + + if (label && (block = jit_basic_block_new(label, n))) + /* Void 0 register indicates error in creation. */ + *(jit_annl_basic_block(cc, label)) = block; + else + jit_set_last_error(cc, "create basic block failed"); + + return block; +} + +JitBasicBlock * +jit_cc_resize_basic_block(JitCompContext *cc, JitBasicBlock *block, int n) +{ + JitReg label = jit_basic_block_label(block); + JitInsn *insn = jit_basic_block_first_insn(block); + JitBasicBlock *new_block = jit_basic_block_new(label, n); + + if (!new_block) { + jit_set_last_error(cc, "resize basic block failed"); + return NULL; + } + + jit_insn_unlink(block); + + if (insn != block) + jit_insn_insert_before(insn, new_block); + + bh_assert(*(jit_annl_basic_block(cc, label)) == block); + *(jit_annl_basic_block(cc, label)) = new_block; + jit_insn_delete(block); + + return new_block; +} + +bool +jit_cc_enable_insn_hash(JitCompContext *cc, unsigned n) +{ + if (jit_anni_is_enabled__hash_link(cc)) + return true; + + if (!jit_anni_enable__hash_link(cc)) + return false; + + /* The table must not exist. */ + bh_assert(!cc->_insn_hash_table._table); + + /* Integer overflow cannot happen because n << 4G (at most several + times of 64K in the most extreme case). */ + if (!(cc->_insn_hash_table._table = + jit_calloc(n * sizeof(*cc->_insn_hash_table._table)))) { + jit_anni_disable__hash_link(cc); + return false; + } + + cc->_insn_hash_table._size = n; + return true; +} + +void +jit_cc_disable_insn_hash(JitCompContext *cc) +{ + jit_anni_disable__hash_link(cc); + jit_free(cc->_insn_hash_table._table); + cc->_insn_hash_table._table = NULL; + cc->_insn_hash_table._size = 0; +} + +void +jit_cc_reset_insn_hash(JitCompContext *cc) +{ + if (jit_anni_is_enabled__hash_link(cc)) + memset(cc->_insn_hash_table._table, 0, + cc->_insn_hash_table._size + * sizeof(*cc->_insn_hash_table._table)); +} + +JitInsn * +jit_cc_set_insn_uid(JitCompContext *cc, JitInsn *insn) +{ + if (insn) { + unsigned num = cc->_ann._insn_num; + unsigned capacity = cc->_ann._insn_capacity; + bool successful = true; + + bh_assert(num <= capacity); + + if (num == capacity) { + capacity = capacity > 0 ? (capacity + capacity / 2) : 64; + +#define EMPTY_POSTFIX +#define ANN_INSN(TYPE, NAME) _JIT_REALLOC_ANN(TYPE, NAME, insn, EMPTY_POSTFIX) +#include "jit_ir.def" +#undef ANN_INSN +#undef EMPTY_POSTFIX + + if (!successful) { + jit_set_last_error(cc, "set insn uid failed"); + return NULL; + } + + cc->_ann._insn_capacity = capacity; + } + + cc->_ann._insn_num = num + 1; + insn->uid = num; + } + + return insn; +} + +JitInsn * +_jit_cc_set_insn_uid_for_new_insn(JitCompContext *cc, JitInsn *insn) +{ + if (jit_cc_set_insn_uid(cc, insn)) + return insn; + + jit_insn_delete(insn); + return NULL; +} + +JitReg +jit_cc_new_reg(JitCompContext *cc, unsigned kind) +{ + unsigned num = jit_cc_reg_num(cc, kind); + unsigned capacity = cc->_ann._reg_capacity[kind]; + bool successful = true; + + bh_assert(num <= capacity); + + if (num == capacity) { + capacity = (capacity == 0 + /* Initialize the capacity to be larger than hard + register number. */ + ? cc->hreg_info->info[kind].num + 16 + : capacity + capacity / 2); + +#define ANN_REG(TYPE, NAME) _JIT_REALLOC_ANN(TYPE, NAME, reg, [kind]) +#include "jit_ir.def" +#undef ANN_REG + + if (!successful) { + jit_set_last_error(cc, "create register failed"); + return 0; + } + + cc->_ann._reg_capacity[kind] = capacity; + } + + cc->_ann._reg_num[kind] = num + 1; + + return jit_reg_new(kind, num); +} + +#undef _JIT_REALLOC_ANN + +#define ANN_LABEL(TYPE, NAME) \ + bool jit_annl_enable_##NAME(JitCompContext *cc) \ + { \ + if (cc->_ann._label_##NAME##_enabled) \ + return true; \ + \ + if (cc->_ann._label_capacity > 0 \ + && !(cc->_ann._label_##NAME = \ + jit_calloc(cc->_ann._label_capacity * sizeof(TYPE)))) { \ + jit_set_last_error(cc, "annl enable " #NAME "failed"); \ + return false; \ + } \ + \ + cc->_ann._label_##NAME##_enabled = 1; \ + return true; \ + } +#define ANN_INSN(TYPE, NAME) \ + bool jit_anni_enable_##NAME(JitCompContext *cc) \ + { \ + if (cc->_ann._insn_##NAME##_enabled) \ + return true; \ + \ + if (cc->_ann._insn_capacity > 0 \ + && !(cc->_ann._insn_##NAME = \ + jit_calloc(cc->_ann._insn_capacity * sizeof(TYPE)))) { \ + jit_set_last_error(cc, "anni enable " #NAME "failed"); \ + return false; \ + } \ + \ + cc->_ann._insn_##NAME##_enabled = 1; \ + return true; \ + } +#define ANN_REG(TYPE, NAME) \ + bool jit_annr_enable_##NAME(JitCompContext *cc) \ + { \ + unsigned k; \ + \ + if (cc->_ann._reg_##NAME##_enabled) \ + return true; \ + \ + for (k = JIT_REG_KIND_VOID; k < JIT_REG_KIND_L32; k++) \ + if (cc->_ann._reg_capacity[k] > 0 \ + && !(cc->_ann._reg_##NAME[k] = jit_calloc( \ + cc->_ann._reg_capacity[k] * sizeof(TYPE)))) { \ + jit_set_last_error(cc, "annr enable " #NAME "failed"); \ + jit_annr_disable_##NAME(cc); \ + return false; \ + } \ + \ + cc->_ann._reg_##NAME##_enabled = 1; \ + return true; \ + } +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + +#define ANN_LABEL(TYPE, NAME) \ + void jit_annl_disable_##NAME(JitCompContext *cc) \ + { \ + jit_free(cc->_ann._label_##NAME); \ + cc->_ann._label_##NAME = NULL; \ + cc->_ann._label_##NAME##_enabled = 0; \ + } +#define ANN_INSN(TYPE, NAME) \ + void jit_anni_disable_##NAME(JitCompContext *cc) \ + { \ + jit_free(cc->_ann._insn_##NAME); \ + cc->_ann._insn_##NAME = NULL; \ + cc->_ann._insn_##NAME##_enabled = 0; \ + } +#define ANN_REG(TYPE, NAME) \ + void jit_annr_disable_##NAME(JitCompContext *cc) \ + { \ + unsigned k; \ + \ + for (k = JIT_REG_KIND_VOID; k < JIT_REG_KIND_L32; k++) { \ + jit_free(cc->_ann._reg_##NAME[k]); \ + cc->_ann._reg_##NAME[k] = NULL; \ + } \ + \ + cc->_ann._reg_##NAME##_enabled = 0; \ + } +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + +char * +jit_get_last_error(JitCompContext *cc) +{ + return cc->last_error[0] == '\0' ? NULL : cc->last_error; +} + +void +jit_set_last_error_v(JitCompContext *cc, const char *format, ...) +{ + va_list args; + va_start(args, format); + vsnprintf(cc->last_error, sizeof(cc->last_error), format, args); + va_end(args); +} + +void +jit_set_last_error(JitCompContext *cc, const char *error) +{ + if (error) + snprintf(cc->last_error, sizeof(cc->last_error), "Error: %s", error); + else + cc->last_error[0] = '\0'; +} + +bool +jit_cc_update_cfg(JitCompContext *cc) +{ + JitBasicBlock *block; + unsigned block_index, end, succ_index, idx; + JitReg *target; + bool retval = false; + + if (!jit_annl_enable_pred_num(cc)) + return false; + + /* Update pred_num of all blocks. */ + JIT_FOREACH_BLOCK_ENTRY_EXIT(cc, block_index, end, block) + { + JitRegVec succs = jit_basic_block_succs(block); + + JIT_REG_VEC_FOREACH(succs, succ_index, target) + if (jit_reg_is_kind(L32, *target)) + *(jit_annl_pred_num(cc, *target)) += 1; + } + + /* Resize predecessor vectors of body blocks. */ + JIT_FOREACH_BLOCK(cc, block_index, end, block) + { + if (!jit_cc_resize_basic_block( + cc, block, + *(jit_annl_pred_num(cc, jit_basic_block_label(block))))) + goto cleanup_and_return; + } + + /* Fill in predecessor vectors all blocks. */ + JIT_FOREACH_BLOCK_REVERSE_ENTRY_EXIT(cc, block_index, block) + { + JitRegVec succs = jit_basic_block_succs(block), preds; + + JIT_REG_VEC_FOREACH(succs, succ_index, target) + if (jit_reg_is_kind(L32, *target)) { + preds = jit_basic_block_preds(*(jit_annl_basic_block(cc, *target))); + bh_assert(*(jit_annl_pred_num(cc, *target)) > 0); + idx = *(jit_annl_pred_num(cc, *target)) - 1; + *(jit_annl_pred_num(cc, *target)) = idx; + *(jit_reg_vec_at(&preds, idx)) = jit_basic_block_label(block); + } + } + + retval = true; + +cleanup_and_return: + jit_annl_disable_pred_num(cc); + return retval; +} + +void +jit_value_stack_push(JitValueStack *stack, JitValue *value) +{ + if (!stack->value_list_head) + stack->value_list_head = stack->value_list_end = value; + else { + stack->value_list_end->next = value; + value->prev = stack->value_list_end; + stack->value_list_end = value; + } +} + +JitValue * +jit_value_stack_pop(JitValueStack *stack) +{ + JitValue *value = stack->value_list_end; + + bh_assert(stack->value_list_end); + + if (stack->value_list_head == stack->value_list_end) + stack->value_list_head = stack->value_list_end = NULL; + else { + stack->value_list_end = stack->value_list_end->prev; + stack->value_list_end->next = NULL; + value->prev = NULL; + } + + return value; +} + +void +jit_value_stack_destroy(JitValueStack *stack) +{ + JitValue *value = stack->value_list_head, *p; + + while (value) { + p = value->next; + jit_free(value); + value = p; + } + + stack->value_list_head = NULL; + stack->value_list_end = NULL; +} + +void +jit_block_stack_push(JitBlockStack *stack, JitBlock *block) +{ + if (!stack->block_list_head) + stack->block_list_head = stack->block_list_end = block; + else { + stack->block_list_end->next = block; + block->prev = stack->block_list_end; + stack->block_list_end = block; + } +} + +JitBlock * +jit_block_stack_top(JitBlockStack *stack) +{ + return stack->block_list_end; +} + +JitBlock * +jit_block_stack_pop(JitBlockStack *stack) +{ + JitBlock *block = stack->block_list_end; + + bh_assert(stack->block_list_end); + + if (stack->block_list_head == stack->block_list_end) + stack->block_list_head = stack->block_list_end = NULL; + else { + stack->block_list_end = stack->block_list_end->prev; + stack->block_list_end->next = NULL; + block->prev = NULL; + } + + return block; +} + +void +jit_block_stack_destroy(JitBlockStack *stack) +{ + JitBlock *block = stack->block_list_head, *p; + + while (block) { + p = block->next; + jit_value_stack_destroy(&block->value_stack); + jit_block_destroy(block); + block = p; + } + + stack->block_list_head = NULL; + stack->block_list_end = NULL; +} + +bool +jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn, uint32 opnd_idx) +{ + JitIncomingInsn *incoming_insn; + + if (!(incoming_insn = jit_calloc((uint32)sizeof(JitIncomingInsn)))) + return false; + + incoming_insn->insn = insn; + incoming_insn->opnd_idx = opnd_idx; + incoming_insn->next = block->incoming_insns_for_end_bb; + block->incoming_insns_for_end_bb = incoming_insn; + return true; +} + +void +jit_block_destroy(JitBlock *block) +{ + JitIncomingInsn *incoming_insn, *incoming_insn_next; + + jit_value_stack_destroy(&block->value_stack); + if (block->param_types) + jit_free(block->param_types); + if (block->result_types) + jit_free(block->result_types); + + incoming_insn = block->incoming_insns_for_end_bb; + while (incoming_insn) { + incoming_insn_next = incoming_insn->next; + jit_free(incoming_insn); + incoming_insn = incoming_insn_next; + } + + jit_free(block); +} + +static inline uint8 +to_stack_value_type(uint8 type) +{ +#if WASM_ENABLE_REF_TYPES != 0 + if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF) + return VALUE_TYPE_I32; +#endif + return type; +} + +bool +jit_cc_pop_value(JitCompContext *cc, uint8 type, JitReg *p_value) +{ + JitValue *jit_value = NULL; + JitReg value = 0; + + if (!jit_block_stack_top(&cc->block_stack)) { + jit_set_last_error(cc, "WASM block stack underflow"); + return false; + } + if (!jit_block_stack_top(&cc->block_stack)->value_stack.value_list_end) { + jit_set_last_error(cc, "WASM data stack underflow"); + return false; + } + + jit_value = jit_value_stack_pop( + &jit_block_stack_top(&cc->block_stack)->value_stack); + bh_assert(jit_value); + + if (jit_value->type != to_stack_value_type(type)) { + jit_set_last_error(cc, "invalid WASM stack data type"); + jit_free(jit_value); + return false; + } + + switch (jit_value->type) { + case VALUE_TYPE_I32: + value = pop_i32(cc->jit_frame); + break; + case VALUE_TYPE_I64: + value = pop_i64(cc->jit_frame); + break; + case VALUE_TYPE_F32: + value = pop_f32(cc->jit_frame); + break; + case VALUE_TYPE_F64: + value = pop_f64(cc->jit_frame); + break; + default: + bh_assert(0); + break; + } + + bh_assert(cc->jit_frame->sp == jit_value->value); + bh_assert(value == jit_value->value->reg); + *p_value = value; + jit_free(jit_value); + return true; +} + +bool +jit_cc_push_value(JitCompContext *cc, uint8 type, JitReg value) +{ + JitValue *jit_value; + + if (!jit_block_stack_top(&cc->block_stack)) { + jit_set_last_error(cc, "WASM block stack underflow"); + return false; + } + + if (!(jit_value = jit_calloc(sizeof(JitValue)))) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + bh_assert(value); + + jit_value->type = to_stack_value_type(type); + jit_value->value = cc->jit_frame->sp; + jit_value_stack_push(&jit_block_stack_top(&cc->block_stack)->value_stack, + jit_value); + + switch (jit_value->type) { + case VALUE_TYPE_I32: + push_i32(cc->jit_frame, value); + break; + case VALUE_TYPE_I64: + push_i64(cc->jit_frame, value); + break; + case VALUE_TYPE_F32: + push_f32(cc->jit_frame, value); + break; + case VALUE_TYPE_F64: + push_f64(cc->jit_frame, value); + break; + } + + return true; +} + +bool +_jit_insn_check_opnd_access_Reg(const JitInsn *insn, unsigned n) +{ + unsigned opcode = insn->opcode; + return (insn_opnd_kind[opcode] == JIT_OPND_KIND_Reg + && n < insn_opnd_num[opcode]); +} + +bool +_jit_insn_check_opnd_access_VReg(const JitInsn *insn, unsigned n) +{ + unsigned opcode = insn->opcode; + return (insn_opnd_kind[opcode] == JIT_OPND_KIND_VReg + && n < insn->_opnd._opnd_VReg._reg_num); +} + +bool +_jit_insn_check_opnd_access_LookupSwitch(const JitInsn *insn) +{ + unsigned opcode = insn->opcode; + return (insn_opnd_kind[opcode] == JIT_OPND_KIND_LookupSwitch); +} + +bool +jit_lock_reg_in_insn(JitCompContext *cc, JitInsn *the_insn, JitReg reg_to_lock) +{ + bool ret = false; + JitInsn *prevent_spill = NULL; + JitInsn *indicate_using = NULL; + + if (!the_insn) + goto just_return; + + if (jit_cc_is_hreg_fixed(cc, reg_to_lock)) { + ret = true; + goto just_return; + } + + /** + * give the virtual register of the locked hard register a minimum, non-zero + * distance, * so as to prevent it from being spilled out + */ + prevent_spill = jit_insn_new_MOV(reg_to_lock, reg_to_lock); + if (!prevent_spill) + goto just_return; + + jit_insn_insert_before(the_insn, prevent_spill); + + /** + * announce the locked hard register is being used, and do necessary spill + * ASAP + */ + indicate_using = jit_insn_new_MOV(reg_to_lock, reg_to_lock); + if (!indicate_using) + goto just_return; + + jit_insn_insert_after(the_insn, indicate_using); + + ret = true; + +just_return: + if (!ret) + jit_set_last_error(cc, "generate insn failed"); + return ret; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.def b/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.def new file mode 100644 index 0000000..046bea1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.def @@ -0,0 +1,346 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file jit-ir.def + * + * @brief Definition of JIT IR instructions and annotations. + */ + +/** + * @def INSN (NAME, OPND_KIND, OPND_NUM, FIRST_USE) + * + * Definition of IR instructions + * + * @param NAME name of the opcode + * @param OPND_KIND kind of the operand(s) + * @param OPND_NUM number of the operand(s) + * @param FIRST_USE index of the first use register + * + * @p OPND_KIND and @p OPND_NUM together determine the format of an + * instruction. There are four kinds of formats: + * + * 1) Reg: fixed-number register operands, @p OPND_NUM specifies the + * number of operands; + * + * 2) VReg: variable-number register operands, @p OPND_NUM specifies + * the number of fixed register operands; + * + * 3) TableSwitch: tableswitch instruction's format, @p OPND_NUM must + * be 1; + * + * 4) LookupSwitch: lookupswitch instruction's format, @p OPND_NUM + * must be 1. + * + * Instruction operands are all registers and they are organized in an + * order that all registers defined by the instruction, if any, appear + * before the registers used by the instruction. The @p FIRST_USE is + * the index of the first use register in the register vector sorted + * in this order. Use @c jit_insn_opnd_regs to get the register + * vector in this order and use @c jit_insn_opnd_first_use to get the + * index of the first use register. + * + * Every instruction with name @p NAME has the following definitions: + * + * @c JEFF_OP_NAME: the enum opcode of insn NAME + * @c jit_insn_new_NAME (...): creates a new instance of insn NAME + * + * An instruction is deleted by function: + * + * @c jit_insn_delete (@p insn) + * + * In the scope of this IR's terminology, operand and argument have + * different meanings. The operand is a general notation, which + * denotes every raw operand of an instruction, while the argument + * only denotes the variable part of operands of instructions of VReg + * kind. For example, a VReg instruction phi node "r0 = phi(r1, r2)" + * has three operands opnd[0]: r0, opnd[1]: r1 and opnd[2]: r2, but + * only two arguments arg[0]: r1 and arg[1]: r2. Operands or + * arguments of instructions with various formats can be access + * through the following APIs: + * + * @c jit_insn_opnd (@p insn, @p n): for Reg_N formats + * @c jit_insn_opndv (@p insn, @p n): for VReg_N formats + * @c jit_insn_opndv_num (@p insn): for VReg_N formats + * @c jit_insn_opndts (@p insn): for TableSwitch_1 format + * @c jit_insn_opndls (@p insn): for LookupSwitch_1 format + */ + +#ifndef INSN +#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) +#endif + +/* Move and conversion instructions that transfer values among + registers of the same kind (move) or different kinds (convert) */ +INSN(MOV, Reg, 2, 1) +INSN(PHI, VReg, 1, 1) + +/* conversion. will extend or truncate */ +INSN(I8TOI32, Reg, 2, 1) +INSN(I8TOI64, Reg, 2, 1) +INSN(I16TOI32, Reg, 2, 1) +INSN(I16TOI64, Reg, 2, 1) +INSN(I32TOI8, Reg, 2, 1) +INSN(I32TOU8, Reg, 2, 1) +INSN(I32TOI16, Reg, 2, 1) +INSN(I32TOU16, Reg, 2, 1) +INSN(I32TOI64, Reg, 2, 1) +INSN(I32TOF32, Reg, 2, 1) +INSN(I32TOF64, Reg, 2, 1) +INSN(U32TOI64, Reg, 2, 1) +INSN(U32TOF32, Reg, 2, 1) +INSN(U32TOF64, Reg, 2, 1) +INSN(I64TOI8, Reg, 2, 1) +INSN(I64TOI16, Reg, 2, 1) +INSN(I64TOI32, Reg, 2, 1) +INSN(I64TOF32, Reg, 2, 1) +INSN(I64TOF64, Reg, 2, 1) +INSN(F32TOI32, Reg, 2, 1) +INSN(F32TOI64, Reg, 2, 1) +INSN(F32TOF64, Reg, 2, 1) +INSN(F32TOU32, Reg, 2, 1) +INSN(F64TOI32, Reg, 2, 1) +INSN(F64TOI64, Reg, 2, 1) +INSN(F64TOF32, Reg, 2, 1) +INSN(F64TOU32, Reg, 2, 1) + +/** + * Re-interpret binary presentations: + * *(i32 *)&f32, *(i64 *)&f64, *(f32 *)&i32, *(f64 *)&i64 + */ +INSN(I32CASTF32, Reg, 2, 1) +INSN(I64CASTF64, Reg, 2, 1) +INSN(F32CASTI32, Reg, 2, 1) +INSN(F64CASTI64, Reg, 2, 1) + +/* Arithmetic and bitwise instructions: */ +INSN(NEG, Reg, 2, 1) +INSN(NOT, Reg, 2, 1) +INSN(ADD, Reg, 3, 1) +INSN(SUB, Reg, 3, 1) +INSN(MUL, Reg, 3, 1) +INSN(DIV_S, Reg, 3, 1) +INSN(REM_S, Reg, 3, 1) +INSN(DIV_U, Reg, 3, 1) +INSN(REM_U, Reg, 3, 1) +INSN(SHL, Reg, 3, 1) +INSN(SHRS, Reg, 3, 1) +INSN(SHRU, Reg, 3, 1) +INSN(ROTL, Reg, 3, 1) +INSN(ROTR, Reg, 3, 1) +INSN(OR, Reg, 3, 1) +INSN(XOR, Reg, 3, 1) +INSN(AND, Reg, 3, 1) +INSN(CMP, Reg, 3, 1) +INSN(MAX, Reg, 3, 1) +INSN(MIN, Reg, 3, 1) +INSN(CLZ, Reg, 2, 1) +INSN(CTZ, Reg, 2, 1) +INSN(POPCNT, Reg, 2, 1) + +/* Select instruction: */ +INSN(SELECTEQ, Reg, 4, 1) +INSN(SELECTNE, Reg, 4, 1) +INSN(SELECTGTS, Reg, 4, 1) +INSN(SELECTGES, Reg, 4, 1) +INSN(SELECTLTS, Reg, 4, 1) +INSN(SELECTLES, Reg, 4, 1) +INSN(SELECTGTU, Reg, 4, 1) +INSN(SELECTGEU, Reg, 4, 1) +INSN(SELECTLTU, Reg, 4, 1) +INSN(SELECTLEU, Reg, 4, 1) + +/* Memory access instructions: */ +INSN(LDEXECENV, Reg, 1, 1) +INSN(LDJITINFO, Reg, 1, 1) +INSN(LDI8, Reg, 3, 1) +INSN(LDU8, Reg, 3, 1) +INSN(LDI16, Reg, 3, 1) +INSN(LDU16, Reg, 3, 1) +INSN(LDI32, Reg, 3, 1) +INSN(LDU32, Reg, 3, 1) +INSN(LDI64, Reg, 3, 1) +INSN(LDU64, Reg, 3, 1) +INSN(LDF32, Reg, 3, 1) +INSN(LDF64, Reg, 3, 1) +INSN(LDPTR, Reg, 3, 1) +INSN(LDV64, Reg, 3, 1) +INSN(LDV128, Reg, 3, 1) +INSN(LDV256, Reg, 3, 1) +INSN(STI8, Reg, 3, 0) +INSN(STI16, Reg, 3, 0) +INSN(STI32, Reg, 3, 0) +INSN(STI64, Reg, 3, 0) +INSN(STF32, Reg, 3, 0) +INSN(STF64, Reg, 3, 0) +INSN(STPTR, Reg, 3, 0) +INSN(STV64, Reg, 3, 1) +INSN(STV128, Reg, 3, 1) +INSN(STV256, Reg, 3, 1) + +/* Control instructions */ +INSN(JMP, Reg, 1, 0) +INSN(BEQ, Reg, 3, 0) +INSN(BNE, Reg, 3, 0) +INSN(BGTS, Reg, 3, 0) +INSN(BGES, Reg, 3, 0) +INSN(BLTS, Reg, 3, 0) +INSN(BLES, Reg, 3, 0) +INSN(BGTU, Reg, 3, 0) +INSN(BGEU, Reg, 3, 0) +INSN(BLTU, Reg, 3, 0) +INSN(BLEU, Reg, 3, 0) +INSN(LOOKUPSWITCH, LookupSwitch, 1, 0) + +/* Call and return instructions */ +INSN(CALLNATIVE, VReg, 2, 1) +INSN(CALLBC, Reg, 4, 2) +INSN(RETURNBC, Reg, 3, 0) +INSN(RETURN, Reg, 1, 0) + +#if WASM_ENABLE_SHARED_MEMORY != 0 +/* Atomic Memory Accesses */ +/* op1(replacement val) op2(expected val) op3(mem data) op4(offset) + * and in x86, the result is stored in register al/ax/eax/rax */ +INSN(AT_CMPXCHGU8, Reg, 4, 0) +INSN(AT_CMPXCHGU16, Reg, 4, 0) +INSN(AT_CMPXCHGI32, Reg, 4, 0) +INSN(AT_CMPXCHGU32, Reg, 4, 0) +INSN(AT_CMPXCHGI64, Reg, 4, 0) +/* rmw operations: + * op1(read value) op2(operand value) op3(mem data) op4(offset) */ +INSN(AT_ADDU8, Reg, 4, 1) +INSN(AT_ADDU16, Reg, 4, 1) +INSN(AT_ADDI32, Reg, 4, 1) +INSN(AT_ADDU32, Reg, 4, 1) +INSN(AT_ADDI64, Reg, 4, 1) +INSN(AT_SUBU8, Reg, 4, 1) +INSN(AT_SUBU16, Reg, 4, 1) +INSN(AT_SUBI32, Reg, 4, 1) +INSN(AT_SUBU32, Reg, 4, 1) +INSN(AT_SUBI64, Reg, 4, 1) +INSN(AT_ANDU8, Reg, 4, 1) +INSN(AT_ANDU16, Reg, 4, 1) +INSN(AT_ANDI32, Reg, 4, 1) +INSN(AT_ANDU32, Reg, 4, 1) +INSN(AT_ANDI64, Reg, 4, 1) +INSN(AT_ORU8, Reg, 4, 1) +INSN(AT_ORU16, Reg, 4, 1) +INSN(AT_ORI32, Reg, 4, 1) +INSN(AT_ORU32, Reg, 4, 1) +INSN(AT_ORI64, Reg, 4, 1) +INSN(AT_XORU8, Reg, 4, 1) +INSN(AT_XORU16, Reg, 4, 1) +INSN(AT_XORI32, Reg, 4, 1) +INSN(AT_XORU32, Reg, 4, 1) +INSN(AT_XORI64, Reg, 4, 1) +INSN(AT_XCHGU8, Reg, 4, 1) +INSN(AT_XCHGU16, Reg, 4, 1) +INSN(AT_XCHGI32, Reg, 4, 1) +INSN(AT_XCHGU32, Reg, 4, 1) +INSN(AT_XCHGI64, Reg, 4, 1) +INSN(FENCE, Reg, 0, 0) +#endif + +#undef INSN + +/** + * @def ANN_LABEL (TYPE, NAME) + * + * Definition of label annotations. + * + * @param TYPE type of the annotation + * @param NAME name of the annotation + * + * Each defined annotation with name NAME has the following APIs: + * + * @c jit_annl_NAME (cc, label): accesses the annotation NAME of + * label @p label + * @c jit_annl_enable_NAME (cc): enables the annotation NAME + * @c jit_annl_disable_NAME (cc): disables the annotation NAME + * @c jit_annl_is_enabled_NAME (cc): check whether the annotation NAME + * is enabled + */ + +#ifndef ANN_LABEL +#define ANN_LABEL(TYPE, NAME) +#endif + +/* Basic Block of a label. */ +ANN_LABEL(JitBasicBlock *, basic_block) +/* Predecessor number of the block that is only used in + jit_cc_update_cfg for updating the CFG. */ +ANN_LABEL(uint16, pred_num) +/* Execution frequency of a block. We can split critical edges with + empty blocks so we don't need to store frequencies of edges. */ +ANN_LABEL(uint16, freq) +/* Begin bytecode instruction pointer of the block. */ +ANN_LABEL(uint8 *, begin_bcip) +/* End bytecode instruction pointer of the block. */ +ANN_LABEL(uint8 *, end_bcip) +/* Stack pointer offset at the end of the block. */ +ANN_LABEL(uint16, end_sp) +/* The label of the next physically adjacent block. */ +ANN_LABEL(JitReg, next_label) +/* Compiled code address of the block. */ +ANN_LABEL(void *, jitted_addr) + +#undef ANN_LABEL + +/** + * @def ANN_INSN (TYPE, NAME) + * + * Definition of instruction annotations. + * + * @param TYPE type of the annotation + * @param NAME name of the annotation + * + * Each defined annotation with name NAME has the following APIs: + * + * @c jit_anni_NAME (cc, insn): accesses the annotation NAME of + * instruction @p insn + * @c jit_anni_enable_NAME (cc): enables the annotation NAME + * @c jit_anni_disable_NAME (cc): disables the annotation NAME + * @c jit_anni_is_enabled_NAME (cc): check whether the annotation NAME + * is enabled + */ + +#ifndef ANN_INSN +#define ANN_INSN(TYPE, NAME) +#endif + +/* A private annotation for linking instructions with the same hash + value, which is only used by the compilation context's hash table + of instructions. */ +ANN_INSN(JitInsn *, _hash_link) + +#undef ANN_INSN + +/** + * @def ANN_REG (TYPE, NAME) + * + * Definition of register annotations. + * + * @param TYPE type of the annotation + * @param NAME name of the annotation + * + * Each defined annotation with name NAME has the following APIs: + * + * @c jit_annr_NAME (cc, reg): accesses the annotation NAME of + * register @p reg + * @c jit_annr_enable_NAME (cc): enables the annotation NAME + * @c jit_annr_disable_NAME (cc): disables the annotation NAME + * @c jit_annr_is_enabled_NAME (cc): check whether the annotation NAME + * is enabled + */ + +#ifndef ANN_REG +#define ANN_REG(TYPE, NAME) +#endif + +/* Defining instruction of registers satisfying SSA property. */ +ANN_REG(JitInsn *, def_insn) + +#undef ANN_REG diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.h b/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.h new file mode 100644 index 0000000..bef3fcc --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_ir.h @@ -0,0 +1,1882 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_IR_H_ +#define _JIT_IR_H_ + +#include "bh_platform.h" +#include "../interpreter/wasm.h" +#include "jit_utils.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Register (operand) representation of JIT IR. + * + * Encoding: [4-bit: kind, 28-bit register no.] + * + * Registers in JIT IR are classified into different kinds according + * to types of values they can hold. The classification is based on + * most processors' hardware register classifications, which include + * various sets of integer, floating point and vector registers with + * different sizes. These registers can be mapped onto corresponding + * kinds of hardware registers by register allocator. Instructions + * can only operate on allowed kinds of registers. For example, an + * integer instruction cannot operate on floating point or vector + * registers. Some encodings of these kinds of registers also + * represent immediate constant values and indexes to constant tables + * (see below). In that case, those registers are read-only. Writing + * to them is illegal. Reading from an immediate constant value + * register always returns the constant value encoded in the register + * no. Reading from a constant table index register always returns + * the constant value stored at the encoded index of the constant + * table of the register's kind. Immediate constant values and values + * indexed by constant table indexes can only be loaded into the + * corresponding kinds of registers if they must be loaded into + * registers. Besides these common kinds of registers, labels of + * basic blocks are also treated as registers of a special kind, which + * hold code addresses of basic block labels and are read-only. Each + * basic block is assigned one unique label register. With this + * unification, we can use the same set of load instructions to load + * values either from addresses stored in normal registers or from + * addresses of labels. Besides these register kinds, the void kind + * is a special kind of registers to denote some error occurs when a + * normal register is expected. Or it can be used as result operand + * of call and invoke instructions to denote no return values. The + * variable registers are classified into two sets: the hard registers + * whose register numbers are less than the hard register numbers of + * their kinds and the virtual registers whose register numbers are + * greater than or equal to the hard register numbers. Before + * register allocation is done, hard registers may appear in the IR + * due to special usages of passes frontend (e.g. fp_reg and exec_env_reg) + * or lower_cg. In the mean time (including during register + * allocation), those hard registers are treated same as virtual + * registers except that they may not be SSA and they can only be + * allocated to the hard registers of themselves. + * + * Classification of registers: + * + void register (kind == JIT_REG_KIND_VOID, no. must be 0) + * + label registers (kind == JIT_REG_KIND_L32) + * + value registers (kind == JIT_REG_KIND_I32/I64/F32/F64/V64/V128/V256) + * | + constants (_JIT_REG_CONST_VAL_FLAG | _JIT_REG_CONST_IDX_FLAG) + * | | + constant values (_JIT_REG_CONST_VAL_FLAG) + * | | + constant indexes (_JIT_REG_CONST_IDX_FLAG) + * | + variables (!(_JIT_REG_CONST_VAL_FLAG | _JIT_REG_CONST_IDX_FLAG)) + * | | + hard registers (no. < hard register number) + * | | + virtual registers (no. >= hard register number) + */ +typedef uint32 JitReg; + +/* + * Mask and shift bits of register kind. + */ +#define _JIT_REG_KIND_MASK 0xf0000000 +#define _JIT_REG_KIND_SHIFT 28 + +/* + * Mask of register no. which must be the least significant bits. + */ +#define _JIT_REG_NO_MASK (~_JIT_REG_KIND_MASK) + +/* + * Constant value flag (the most significant bit) of register + * no. field of integer, floating point and vector registers. If this + * flag is set in the register no., the rest bits of register + * no. represent a signed (27-bit) integer constant value of the + * corresponding type of the register and the register is read-only. + */ +#define _JIT_REG_CONST_VAL_FLAG ((_JIT_REG_NO_MASK >> 1) + 1) + +/* + * Constant index flag of non-constant-value (constant value flag is + * not set in register no. field) integer, floating point and vector + * regisers. If this flag is set, the rest bits of the register + * no. represent an index to the constant value table of the + * corresponding type of the register and the register is read-only. + */ +#define _JIT_REG_CONST_IDX_FLAG (_JIT_REG_CONST_VAL_FLAG >> 1) + +/** + * Register kinds. Don't change the order of the defined values. The + * L32 kind must be after all normal kinds (see _const_val and _reg_ann + * of JitCompContext). + */ +typedef enum JitRegKind { + JIT_REG_KIND_VOID = 0x00, /* void type */ + JIT_REG_KIND_I32 = 0x01, /* 32-bit signed or unsigned integer */ + JIT_REG_KIND_I64 = 0x02, /* 64-bit signed or unsigned integer */ + JIT_REG_KIND_F32 = 0x03, /* 32-bit floating point */ + JIT_REG_KIND_F64 = 0x04, /* 64-bit floating point */ + JIT_REG_KIND_V64 = 0x05, /* 64-bit vector */ + JIT_REG_KIND_V128 = 0x06, /* 128-bit vector */ + JIT_REG_KIND_V256 = 0x07, /* 256-bit vector */ + JIT_REG_KIND_L32 = 0x08, /* 32-bit label address */ + JIT_REG_KIND_NUM /* number of register kinds */ +} JitRegKind; + +#if UINTPTR_MAX == UINT64_MAX +#define JIT_REG_KIND_PTR JIT_REG_KIND_I64 +#else +#define JIT_REG_KIND_PTR JIT_REG_KIND_I32 +#endif + +/** + * Construct a new JIT IR register from the kind and no. + * + * @param reg_kind register kind + * @param reg_no register no. + * + * @return the new register with the given kind and no. + */ +static inline JitReg +jit_reg_new(unsigned reg_kind, unsigned reg_no) +{ + return (JitReg)((reg_kind << _JIT_REG_KIND_SHIFT) | reg_no); +} + +/** + * Get the register kind of the given register. + * + * @param r a JIT IR register + * + * @return the register kind of register r + */ +static inline int +jit_reg_kind(JitReg r) +{ + return (r & _JIT_REG_KIND_MASK) >> _JIT_REG_KIND_SHIFT; +} + +/** + * Get the register no. of the given JIT IR register. + * + * @param r a JIT IR register + * + * @return the register no. of register r + */ +static inline int +jit_reg_no(JitReg r) +{ + return r & _JIT_REG_NO_MASK; +} + +/** + * Check whether the given register is a normal value register. + * + * @param r a JIT IR register + * + * @return true iff the register is a normal value register + */ +static inline bool +jit_reg_is_value(JitReg r) +{ + unsigned kind = jit_reg_kind(r); + return kind > JIT_REG_KIND_VOID && kind < JIT_REG_KIND_L32; +} + +/** + * Check whether the given register is a constant value. + * + * @param r a JIT IR register + * + * @return true iff register r is a constant value + */ +static inline bool +jit_reg_is_const_val(JitReg r) +{ + return jit_reg_is_value(r) && (r & _JIT_REG_CONST_VAL_FLAG); +} + +/** + * Check whether the given register is a constant table index. + * + * @param r a JIT IR register + * + * @return true iff register r is a constant table index + */ +static inline bool +jit_reg_is_const_idx(JitReg r) +{ + return (jit_reg_is_value(r) && !jit_reg_is_const_val(r) + && (r & _JIT_REG_CONST_IDX_FLAG)); +} + +/** + * Check whether the given register is a constant. + * + * @param r a JIT IR register + * + * @return true iff register r is a constant + */ +static inline bool +jit_reg_is_const(JitReg r) +{ + return (jit_reg_is_value(r) + && (r & (_JIT_REG_CONST_VAL_FLAG | _JIT_REG_CONST_IDX_FLAG))); +} + +/** + * Check whether the given register is a normal variable register. + * + * @param r a JIT IR register + * + * @return true iff the register is a normal variable register + */ +static inline bool +jit_reg_is_variable(JitReg r) +{ + return (jit_reg_is_value(r) + && !(r & (_JIT_REG_CONST_VAL_FLAG | _JIT_REG_CONST_IDX_FLAG))); +} + +/** + * Test whether the register is the given kind. + * + * @param KIND register kind name + * @param R register + * + * @return true if the register is the given kind + */ +#define jit_reg_is_kind(KIND, R) (jit_reg_kind(R) == JIT_REG_KIND_##KIND) + +/** + * Construct a zero IR register with given the kind. + * + * @param kind the kind of the value + * + * @return a constant register of zero + */ +static inline JitReg +jit_reg_new_zero(unsigned kind) +{ + bh_assert(kind != JIT_REG_KIND_VOID && kind < JIT_REG_KIND_L32); + return jit_reg_new(kind, _JIT_REG_CONST_VAL_FLAG); +} + +/** + * Test whether the register is a zero constant value. + * + * @param reg an IR register + * + * @return true iff the register is a constant zero + */ +static inline JitReg +jit_reg_is_zero(JitReg reg) +{ + return (jit_reg_is_value(reg) + && jit_reg_no(reg) == _JIT_REG_CONST_VAL_FLAG); +} + +/** + * Operand of instructions with fixed-number register operand(s). + */ +typedef JitReg JitOpndReg; + +/** + * Operand of instructions with variable-number register operand(s). + */ +typedef struct JitOpndVReg { + uint32 _reg_num; + JitReg _reg[1]; +} JitOpndVReg; + +/** + * Operand of lookupswitch instruction. + */ +typedef struct JitOpndLookupSwitch { + /* NOTE: distance between JitReg operands must be the same (see + jit_insn_opnd_regs). */ + JitReg value; /* the value to be compared */ + uint32 match_pairs_num; /* match pairs number */ + /* NOTE: offset between adjacent targets must be sizeof + (match_pairs[0]) (see implementation of jit_basic_block_succs), + so the default_target field must be here. */ + JitReg default_target; /* default target BB */ + struct { + int32 value; /* match value of the match pair */ + JitReg target; /* target BB of the match pair */ + } match_pairs[1]; /* match pairs of the instruction */ +} JitOpndLookupSwitch; + +/** + * Instruction of JIT IR. + */ +typedef struct JitInsn { + /* Pointers to the previous and next instructions. */ + struct JitInsn *prev; + struct JitInsn *next; + + /* Opcode of the instruction. */ + uint16 opcode; + + /* Reserved field that may be used by optimizations locally. + * bit_0(Least Significant Bit) is atomic flag for load/store */ + uint8 flags_u8; + + /* The unique ID of the instruction. */ + uint16 uid; + + /* Operands for different kinds of instructions. */ + union { + /* For instructions with fixed-number register operand(s). */ + JitOpndReg _opnd_Reg[1]; + + /* For instructions with variable-number register operand(s). */ + JitOpndVReg _opnd_VReg; + + /* For lookupswitch instruction. */ + JitOpndLookupSwitch _opnd_LookupSwitch; + } _opnd; +} JitInsn; + +/** + * Opcodes of IR instructions. + */ +typedef enum JitOpcode { +#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) JIT_OP_##NAME, +#include "jit_ir.def" +#undef INSN + JIT_OP_OPCODE_NUMBER +} JitOpcode; + +/* + * Helper functions for creating new instructions. Don't call them + * directly. Use jit_insn_new_NAME, such as jit_insn_new_MOV instead. + */ + +JitInsn * +_jit_insn_new_Reg_0(JitOpcode opc); +JitInsn * +_jit_insn_new_Reg_1(JitOpcode opc, JitReg r0); +JitInsn * +_jit_insn_new_Reg_2(JitOpcode opc, JitReg r0, JitReg r1); +JitInsn * +_jit_insn_new_Reg_3(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2); +JitInsn * +_jit_insn_new_Reg_4(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2, JitReg r3); +JitInsn * +_jit_insn_new_Reg_5(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2, JitReg r3, + JitReg r4); +JitInsn * +_jit_insn_new_VReg_1(JitOpcode opc, JitReg r0, int n); +JitInsn * +_jit_insn_new_VReg_2(JitOpcode opc, JitReg r0, JitReg r1, int n); +JitInsn * +_jit_insn_new_LookupSwitch_1(JitOpcode opc, JitReg value, uint32 num); + +/* + * Instruction creation functions jit_insn_new_NAME, where NAME is the + * name of the instruction defined in jit_ir.def. + */ +#define ARG_DECL_Reg_0 +#define ARG_LIST_Reg_0 +#define ARG_DECL_Reg_1 JitReg r0 +#define ARG_LIST_Reg_1 , r0 +#define ARG_DECL_Reg_2 JitReg r0, JitReg r1 +#define ARG_LIST_Reg_2 , r0, r1 +#define ARG_DECL_Reg_3 JitReg r0, JitReg r1, JitReg r2 +#define ARG_LIST_Reg_3 , r0, r1, r2 +#define ARG_DECL_Reg_4 JitReg r0, JitReg r1, JitReg r2, JitReg r3 +#define ARG_LIST_Reg_4 , r0, r1, r2, r3 +#define ARG_DECL_Reg_5 JitReg r0, JitReg r1, JitReg r2, JitReg r3, JitReg r4 +#define ARG_LIST_Reg_5 , r0, r1, r2, r3, r4 +#define ARG_DECL_VReg_1 JitReg r0, int n +#define ARG_LIST_VReg_1 , r0, n +#define ARG_DECL_VReg_2 JitReg r0, JitReg r1, int n +#define ARG_LIST_VReg_2 , r0, r1, n +#define ARG_DECL_LookupSwitch_1 JitReg value, uint32 num +#define ARG_LIST_LookupSwitch_1 , value, num +#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) \ + static inline JitInsn *jit_insn_new_##NAME( \ + ARG_DECL_##OPND_KIND##_##OPND_NUM) \ + { \ + return _jit_insn_new_##OPND_KIND##_##OPND_NUM( \ + JIT_OP_##NAME ARG_LIST_##OPND_KIND##_##OPND_NUM); \ + } +#include "jit_ir.def" +#undef INSN +#undef ARG_DECL_Reg_0 +#undef ARG_LIST_Reg_0 +#undef ARG_DECL_Reg_1 +#undef ARG_LIST_Reg_1 +#undef ARG_DECL_Reg_2 +#undef ARG_LIST_Reg_2 +#undef ARG_DECL_Reg_3 +#undef ARG_LIST_Reg_3 +#undef ARG_DECL_Reg_4 +#undef ARG_LIST_Reg_4 +#undef ARG_DECL_Reg_5 +#undef ARG_LIST_Reg_5 +#undef ARG_DECL_VReg_1 +#undef ARG_LIST_VReg_1 +#undef ARG_DECL_VReg_2 +#undef ARG_LIST_VReg_2 +#undef ARG_DECL_LookupSwitch_1 +#undef ARG_LIST_LookupSwitch_1 + +/** + * Delete an instruction + * + * @param insn an instruction to be deleted + */ +static inline void +jit_insn_delete(JitInsn *insn) +{ + jit_free(insn); +} + +/* + * Runtime type check functions that check whether accessing the n-th + * operand is legal. They are only used for in self-verification + * mode. + * + * @param insn any JIT IR instruction + * @param n index of the operand to access + * + * @return true if the access is legal + */ +bool +_jit_insn_check_opnd_access_Reg(const JitInsn *insn, unsigned n); +bool +_jit_insn_check_opnd_access_VReg(const JitInsn *insn, unsigned n); +bool +_jit_insn_check_opnd_access_LookupSwitch(const JitInsn *insn); + +/** + * Get the pointer to the n-th register operand of the given + * instruction. The instruction format must be Reg. + * + * @param insn a Reg format instruction + * @param n index of the operand to get + * + * @return pointer to the n-th operand + */ +static inline JitReg * +jit_insn_opnd(JitInsn *insn, int n) +{ + bh_assert(_jit_insn_check_opnd_access_Reg(insn, n)); + return &insn->_opnd._opnd_Reg[n]; +} + +/** + * Get the pointer to the n-th register operand of the given + * instruction. The instruction format must be VReg. + * + * @param insn a VReg format instruction + * @param n index of the operand to get + * + * @return pointer to the n-th operand + */ +static inline JitReg * +jit_insn_opndv(JitInsn *insn, int n) +{ + bh_assert(_jit_insn_check_opnd_access_VReg(insn, n)); + return &insn->_opnd._opnd_VReg._reg[n]; +} + +/** + * Get the operand number of the given instruction. The instruction + * format must be VReg. + * + * @param insn a VReg format instruction + * + * @return operand number of the instruction + */ +static inline unsigned +jit_insn_opndv_num(const JitInsn *insn) +{ + bh_assert(_jit_insn_check_opnd_access_VReg(insn, 0)); + return insn->_opnd._opnd_VReg._reg_num; +} + +/** + * Get the pointer to the LookupSwitch operand of the given + * instruction. The instruction format must be LookupSwitch. + * + * @param insn a LookupSwitch format instruction + * + * @return pointer to the operand + */ +static inline JitOpndLookupSwitch * +jit_insn_opndls(JitInsn *insn) +{ + bh_assert(_jit_insn_check_opnd_access_LookupSwitch(insn)); + return &insn->_opnd._opnd_LookupSwitch; +} + +/** + * Insert instruction @p insn2 before instruction @p insn1. + * + * @param insn1 any instruction + * @param insn2 any instruction + */ +void +jit_insn_insert_before(JitInsn *insn1, JitInsn *insn2); + +/** + * Insert instruction @p insn2 after instruction @p insn1. + * + * @param insn1 any instruction + * @param insn2 any instruction + */ +void +jit_insn_insert_after(JitInsn *insn1, JitInsn *insn2); + +/** + * Unlink the instruction @p insn from the containing list. + * + * @param insn an instruction + */ +void +jit_insn_unlink(JitInsn *insn); + +/** + * Get the hash value of the comparable instruction (pure functions + * and exception check instructions). + * + * @param insn an instruction + * + * @return hash value of the instruction + */ +unsigned +jit_insn_hash(JitInsn *insn); + +/** + * Compare whether the two comparable instructions are the same. + * + * @param insn1 the first instruction + * @param insn2 the second instruction + * + * @return true if the two instructions are the same + */ +bool +jit_insn_equal(JitInsn *insn1, JitInsn *insn2); + +/** + * Register vector for accessing predecessors and successors of a + * basic block. + */ +typedef struct JitRegVec { + JitReg *_base; /* points to the first register */ + int32 _stride; /* stride to the next register */ + uint32 num; /* number of registers */ +} JitRegVec; + +/** + * Get the address of the i-th register in the register vector. + * + * @param vec a register vector + * @param i index to the register vector + * + * @return the address of the i-th register in the vector + */ +static inline JitReg * +jit_reg_vec_at(const JitRegVec *vec, unsigned i) +{ + bh_assert(i < vec->num); + return vec->_base + vec->_stride * i; +} + +/** + * Visit each element in a register vector. + * + * @param V (JitRegVec) the register vector + * @param I (unsigned) index variable in the vector + * @param R (JitReg *) resiger pointer variable + */ +#define JIT_REG_VEC_FOREACH(V, I, R) \ + for ((I) = 0, (R) = (V)._base; (I) < (V).num; (I)++, (R) += (V)._stride) + +/** + * Visit each register defined by an instruction. + * + * @param V (JitRegVec) register vector of the instruction + * @param I (unsigned) index variable in the vector + * @param R (JitReg *) resiger pointer variable + * @param F index of the first used register + */ +#define JIT_REG_VEC_FOREACH_DEF(V, I, R, F) \ + for ((I) = 0, (R) = (V)._base; (I) < (F); (I)++, (R) += (V)._stride) + +/** + * Visit each register used by an instruction. + * + * @param V (JitRegVec) register vector of the instruction + * @param I (unsigned) index variable in the vector + * @param R (JitReg *) resiger pointer variable + * @param F index of the first used register + */ +#define JIT_REG_VEC_FOREACH_USE(V, I, R, F) \ + for ((I) = (F), (R) = (V)._base + (F) * (V)._stride; (I) < (V).num; \ + (I)++, (R) += (V)._stride) + +/** + * Get a generic register vector that contains all register operands. + * The registers defined by the instruction, if any, appear before the + * registers used by the instruction. + * + * @param insn an instruction + * + * @return a register vector containing register operands + */ +JitRegVec +jit_insn_opnd_regs(JitInsn *insn); + +/** + * Get the index of the first use register in the register vector + * returned by jit_insn_opnd_regs. + * + * @param insn an instruction + * + * @return the index of the first use register in the register vector + */ +unsigned +jit_insn_opnd_first_use(JitInsn *insn); + +/** + * Basic Block of JIT IR. It is a basic block only if the IR is not in + * non-BB form. The block is represented by a special phi node, whose + * result and arguments are label registers. The result label is the + * containing block's label. The arguments are labels of predecessors + * of the block. Successor labels are stored in the last instruction, + * which must be a control flow instruction. Instructions of a block + * are linked in a circular linked list with the block phi node as the + * end of the list. The next and prev field of the block phi node + * point to the first and last instructions of the block. + */ +typedef JitInsn JitBasicBlock; + +/** + * Create a new basic block instance. + * + * @param label the label of the new basic block + * @param n number of predecessors + * + * @return the created new basic block instance + */ +JitBasicBlock * +jit_basic_block_new(JitReg label, int n); + +/** + * Delete a basic block instance and all instructions init. + * + * @param block the basic block to be deleted + */ +void +jit_basic_block_delete(JitBasicBlock *block); + +/** + * Get the label of the basic block. + * + * @param block a basic block instance + * + * @return the label of the basic block + */ +static inline JitReg +jit_basic_block_label(JitBasicBlock *block) +{ + return *(jit_insn_opndv(block, 0)); +} + +/** + * Get the first instruction of the basic block. + * + * @param block a basic block instance + * + * @return the first instruction of the basic block + */ +static inline JitInsn * +jit_basic_block_first_insn(JitBasicBlock *block) +{ + return block->next; +} + +/** + * Get the last instruction of the basic block. + * + * @param block a basic block instance + * + * @return the last instruction of the basic block + */ +static inline JitInsn * +jit_basic_block_last_insn(JitBasicBlock *block) +{ + return block->prev; +} + +/** + * Get the end of instruction list of the basic block (which is always + * the block itself). + * + * @param block a basic block instance + * + * @return the end of instruction list of the basic block + */ +static inline JitInsn * +jit_basic_block_end_insn(JitBasicBlock *block) +{ + return block; +} + +/** + * Visit each instruction in the block from the first to the last. In + * the code block, the instruction pointer @p I must be a valid + * pointer to an instruction in the block. That means if the + * instruction may be deleted, @p I must point to the previous or next + * valid instruction before the next iteration. + * + * @param B (JitBasicBlock *) the block + * @param I (JitInsn *) instruction visited + */ +#define JIT_FOREACH_INSN(B, I) \ + for (I = jit_basic_block_first_insn(B); I != jit_basic_block_end_insn(B); \ + I = I->next) + +/** + * Visit each instruction in the block from the last to the first. In + * the code block, the instruction pointer @p I must be a valid + * pointer to an instruction in the block. That means if the + * instruction may be deleted, @p I must point to the previous or next + * valid instruction before the next iteration. + * + * @param B (JitBasicBlock *) the block + * @param I (JitInsn *) instruction visited + */ +#define JIT_FOREACH_INSN_REVERSE(B, I) \ + for (I = jit_basic_block_last_insn(B); I != jit_basic_block_end_insn(B); \ + I = I->prev) + +/** + * Prepend an instruction in the front of the block. The position is + * just after the block phi node (the block instance itself). + * + * @param block a block + * @param insn an instruction to be prepended + */ +static inline void +jit_basic_block_prepend_insn(JitBasicBlock *block, JitInsn *insn) +{ + jit_insn_insert_after(block, insn); +} + +/** + * Append an instruction to the end of the basic block. + * + * @param block a basic block + * @param insn an instruction to be appended + */ +static inline void +jit_basic_block_append_insn(JitBasicBlock *block, JitInsn *insn) +{ + jit_insn_insert_before(block, insn); +} + +/** + * Get the register vector of predecessors of a basic block. + * + * @param block a JIT IR block + * + * @return register vector of the predecessors + */ +JitRegVec +jit_basic_block_preds(JitBasicBlock *block); + +/** + * Get the register vector of successors of a basic block. + * + * @param block a JIT IR basic block + * + * @return register vector of the successors + */ +JitRegVec +jit_basic_block_succs(JitBasicBlock *block); + +/** + * Hard register information of one kind. + */ +typedef struct JitHardRegInfo { + struct { + /* Hard register number of this kind. */ + uint32 num; + + /* Whether each register is fixed. */ + const uint8 *fixed; + + /* Whether each register is caller-saved in the native ABI. */ + const uint8 *caller_saved_native; + + /* Whether each register is caller-saved in the JITed ABI. */ + const uint8 *caller_saved_jitted; + } info[JIT_REG_KIND_L32]; + + /* The indexes of hard registers of frame pointer, exec_env and cmp. */ + uint32 fp_hreg_index; + uint32 exec_env_hreg_index; + uint32 cmp_hreg_index; +} JitHardRegInfo; + +struct JitBlock; +struct JitCompContext; +struct JitValueSlot; + +/** + * Value in the WASM operation stack, each stack element + * is a Jit register + */ +typedef struct JitValue { + struct JitValue *next; + struct JitValue *prev; + struct JitValueSlot *value; + /* VALUE_TYPE_I32/I64/F32/F64/VOID */ + uint8 type; +} JitValue; + +/** + * Value stack, represents stack elements in a WASM block + */ +typedef struct JitValueStack { + JitValue *value_list_head; + JitValue *value_list_end; +} JitValueStack; + +/* Record information of a value slot of local variable or stack + during translation. */ +typedef struct JitValueSlot { + /* The virtual register that holds the value of the slot if the + value of the slot is in register. */ + JitReg reg; + + /* The dirty bit of the value slot. It's set if the value in + register is newer than the value in memory. */ + uint32 dirty : 1; + + /* Whether the new value in register is a reference, which is valid + only when the dirty bit is set. */ + uint32 ref : 1; + + /* Committed reference flag. 0: unknown, 1: not-reference, 2: + reference. */ + uint32 committed_ref : 2; +} JitValueSlot; + +typedef struct JitMemRegs { + /* The following registers should be re-loaded after + memory.grow, callbc and callnative */ + JitReg memory_inst; + JitReg cur_page_count; + JitReg memory_data; + JitReg memory_data_end; + JitReg mem_bound_check_1byte; + JitReg mem_bound_check_2bytes; + JitReg mem_bound_check_4bytes; + JitReg mem_bound_check_8bytes; + JitReg mem_bound_check_16bytes; +} JitMemRegs; + +typedef struct JitTableRegs { + JitReg table_elems; + /* Should be re-loaded after table.grow, + callbc and callnative */ + JitReg table_cur_size; +} JitTableRegs; + +/* Frame information for translation */ +typedef struct JitFrame { + /* The current wasm module */ + WASMModule *cur_wasm_module; + /* The current wasm function */ + WASMFunction *cur_wasm_func; + /* The current wasm function index */ + uint32 cur_wasm_func_idx; + /* The current compilation context */ + struct JitCompContext *cc; + + /* Max local slot number. */ + uint32 max_locals; + + /* Max operand stack slot number. */ + uint32 max_stacks; + + /* Instruction pointer */ + uint8 *ip; + + /* Stack top pointer */ + JitValueSlot *sp; + + /* Committed instruction pointer */ + uint8 *committed_ip; + + /* Committed stack top pointer */ + JitValueSlot *committed_sp; + + /* WASM module instance */ + JitReg module_inst_reg; + /* WASM module */ + JitReg module_reg; + /* module_inst->import_func_ptrs */ + JitReg import_func_ptrs_reg; + /* module_inst->fast_jit_func_ptrs */ + JitReg fast_jit_func_ptrs_reg; + /* module_inst->func_type_indexes */ + JitReg func_type_indexes_reg; + /* Boundary of auxiliary stack */ + JitReg aux_stack_bound_reg; + /* Bottom of auxiliary stack */ + JitReg aux_stack_bottom_reg; + /* Data of memory instances */ + JitMemRegs *memory_regs; + /* Data of table instances */ + JitTableRegs *table_regs; + + /* Local variables */ + JitValueSlot lp[1]; +} JitFrame; + +typedef struct JitIncomingInsn { + struct JitIncomingInsn *next; + JitInsn *insn; + uint32 opnd_idx; +} JitIncomingInsn, *JitIncomingInsnList; + +typedef struct JitBlock { + struct JitBlock *next; + struct JitBlock *prev; + + /* The current Jit Block */ + struct JitCompContext *cc; + + /* LABEL_TYPE_BLOCK/LOOP/IF/FUNCTION */ + uint32 label_type; + + /* code of else opcode of this block, if it is a IF block */ + uint8 *wasm_code_else; + /* code of end opcode of this block */ + uint8 *wasm_code_end; + + /* JIT label points to code begin */ + JitBasicBlock *basic_block_entry; + /* JIT label points to code else */ + JitBasicBlock *basic_block_else; + /* JIT label points to code end */ + JitBasicBlock *basic_block_end; + + /* Incoming INSN for basic_block_else */ + JitInsn *incoming_insn_for_else_bb; + /* Incoming INSNs for basic_block_end */ + JitIncomingInsnList incoming_insns_for_end_bb; + + /* WASM operation stack */ + JitValueStack value_stack; + + /* Param count/types/PHIs of this block */ + uint32 param_count; + uint8 *param_types; + + /* Result count/types/PHIs of this block */ + uint32 result_count; + uint8 *result_types; + + /* The begin frame stack pointer of this block */ + JitValueSlot *frame_sp_begin; +} JitBlock; + +/** + * Block stack, represents WASM block stack elements + */ +typedef struct JitBlockStack { + JitBlock *block_list_head; + JitBlock *block_list_end; +} JitBlockStack; + +/** + * The JIT compilation context for one compilation process of a + * compilation unit. + */ +typedef struct JitCompContext { + /* Hard register information of each kind. */ + const JitHardRegInfo *hreg_info; + + /* No. of the pass to be applied. */ + uint8 cur_pass_no; + + /* The current wasm module */ + WASMModule *cur_wasm_module; + /* The current wasm function */ + WASMFunction *cur_wasm_func; + /* The current wasm function index */ + uint32 cur_wasm_func_idx; + /* The block stack */ + JitBlockStack block_stack; + + bool mem_space_unchanged; + + /* Entry and exit labels of the compilation unit, whose numbers must + be 0 and 1 respectively (see JIT_FOREACH_BLOCK). */ + JitReg entry_label; + JitReg exit_label; + JitBasicBlock **exce_basic_blocks; + JitIncomingInsnList *incoming_insns_for_exec_bbs; + + /* The current basic block to generate instructions */ + JitBasicBlock *cur_basic_block; + + /* Registers of frame pointer, exec_env and CMP result. */ + JitReg fp_reg; + JitReg exec_env_reg; + JitReg cmp_reg; + + /* WASM module instance */ + JitReg module_inst_reg; + /* WASM module */ + JitReg module_reg; + /* module_inst->import_func_ptrs */ + JitReg import_func_ptrs_reg; + /* module_inst->fast_jit_func_ptrs */ + JitReg fast_jit_func_ptrs_reg; + /* module_inst->func_type_indexes */ + JitReg func_type_indexes_reg; + /* Boundary of auxiliary stack */ + JitReg aux_stack_bound_reg; + /* Bottom of auxiliary stack */ + JitReg aux_stack_bottom_reg; + /* Data of memory instances */ + JitMemRegs *memory_regs; + /* Data of table instances */ + JitTableRegs *table_regs; + + /* Current frame information for translation */ + JitFrame *jit_frame; + + /* The total frame size of current function */ + uint32 total_frame_size; + + /* The spill cache offset to the interp frame */ + uint32 spill_cache_offset; + /* The spill cache size */ + uint32 spill_cache_size; + + /* The offset of jitted_return_address in the frame, which is set by + the pass frontend and used by the pass codegen. */ + uint32 jitted_return_address_offset; + + /* Begin and end addresses of the jitted code produced by the pass + codegen and consumed by the region registration after codegen and + the pass dump. */ + void *jitted_addr_begin; + void *jitted_addr_end; + + char last_error[128]; + + /* Below fields are all private. Don't access them directly. */ + + /* Reference count of the compilation context. */ + uint16 _reference_count; + + /* Constant values. */ + struct { + /* Number of constant values of each kind. */ + uint32 _num[JIT_REG_KIND_L32]; + + /* Capacity of register annotations of each kind. */ + uint32 _capacity[JIT_REG_KIND_L32]; + + /* Constant vallues of each kind. */ + uint8 *_value[JIT_REG_KIND_L32]; + + /* Next element on the list of values with the same hash code. */ + JitReg *_next[JIT_REG_KIND_L32]; + + /* Size of the hash table. */ + uint32 _hash_table_size; + + /* Map values to JIT register. */ + JitReg *_hash_table; + } _const_val; + + /* Annotations of labels, registers and instructions. */ + struct { + /* Number of all ever created labels. */ + uint32 _label_num; + + /* Capacity of label annotations. */ + uint32 _label_capacity; + + /* Number of all ever created instructions. */ + uint32 _insn_num; + + /* Capacity of instruction annotations. */ + uint32 _insn_capacity; + + /* Number of ever created registers of each kind. */ + uint32 _reg_num[JIT_REG_KIND_L32]; + + /* Capacity of register annotations of each kind. */ + uint32 _reg_capacity[JIT_REG_KIND_L32]; + + /* Storage of annotations. */ +#define ANN_LABEL(TYPE, NAME) TYPE *_label_##NAME; +#define ANN_INSN(TYPE, NAME) TYPE *_insn_##NAME; +#define ANN_REG(TYPE, NAME) TYPE *_reg_##NAME[JIT_REG_KIND_L32]; +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + + /* Flags of annotations. */ +#define ANN_LABEL(TYPE, NAME) uint32 _label_##NAME##_enabled : 1; +#define ANN_INSN(TYPE, NAME) uint32 _insn_##NAME##_enabled : 1; +#define ANN_REG(TYPE, NAME) uint32 _reg_##NAME##_enabled : 1; +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + } _ann; + + /* Instruction hash table. */ + struct { + /* Size of the hash table. */ + uint32 _size; + + /* The hash table. */ + JitInsn **_table; + } _insn_hash_table; + + /* indicate if the last comparision is about floating-point numbers or not + */ + bool last_cmp_on_fp; +} JitCompContext; + +/* + * Annotation accessing functions jit_annl_NAME, jit_anni_NAME and + * jit_annr_NAME. + */ +#define ANN_LABEL(TYPE, NAME) \ + static inline TYPE *jit_annl_##NAME(JitCompContext *cc, JitReg label) \ + { \ + unsigned idx = jit_reg_no(label); \ + bh_assert(jit_reg_kind(label) == JIT_REG_KIND_L32); \ + bh_assert(idx < cc->_ann._label_num); \ + bh_assert(cc->_ann._label_##NAME##_enabled); \ + return &cc->_ann._label_##NAME[idx]; \ + } +#define ANN_INSN(TYPE, NAME) \ + static inline TYPE *jit_anni_##NAME(JitCompContext *cc, JitInsn *insn) \ + { \ + unsigned uid = insn->uid; \ + bh_assert(uid < cc->_ann._insn_num); \ + bh_assert(cc->_ann._insn_##NAME##_enabled); \ + return &cc->_ann._insn_##NAME[uid]; \ + } +#define ANN_REG(TYPE, NAME) \ + static inline TYPE *jit_annr_##NAME(JitCompContext *cc, JitReg reg) \ + { \ + unsigned kind = jit_reg_kind(reg); \ + unsigned no = jit_reg_no(reg); \ + bh_assert(kind < JIT_REG_KIND_L32); \ + bh_assert(no < cc->_ann._reg_num[kind]); \ + bh_assert(cc->_ann._reg_##NAME##_enabled); \ + return &cc->_ann._reg_##NAME[kind][no]; \ + } +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + +/* + * Annotation enabling functions jit_annl_enable_NAME, + * jit_anni_enable_NAME and jit_annr_enable_NAME, which allocate + * sufficient memory for the annotations. + */ +#define ANN_LABEL(TYPE, NAME) bool jit_annl_enable_##NAME(JitCompContext *cc); +#define ANN_INSN(TYPE, NAME) bool jit_anni_enable_##NAME(JitCompContext *cc); +#define ANN_REG(TYPE, NAME) bool jit_annr_enable_##NAME(JitCompContext *cc); +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + +/* + * Annotation disabling functions jit_annl_disable_NAME, + * jit_anni_disable_NAME and jit_annr_disable_NAME, which release + * memory of the annotations. Before calling these functions, + * resources owned by the annotations must be explictely released. + */ +#define ANN_LABEL(TYPE, NAME) void jit_annl_disable_##NAME(JitCompContext *cc); +#define ANN_INSN(TYPE, NAME) void jit_anni_disable_##NAME(JitCompContext *cc); +#define ANN_REG(TYPE, NAME) void jit_annr_disable_##NAME(JitCompContext *cc); +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + +/* + * Functions jit_annl_is_enabled_NAME, jit_anni_is_enabled_NAME and + * jit_annr_is_enabled_NAME for checking whether an annotation is + * enabled. + */ +#define ANN_LABEL(TYPE, NAME) \ + static inline bool jit_annl_is_enabled_##NAME(JitCompContext *cc) \ + { \ + return !!cc->_ann._label_##NAME##_enabled; \ + } +#define ANN_INSN(TYPE, NAME) \ + static inline bool jit_anni_is_enabled_##NAME(JitCompContext *cc) \ + { \ + return !!cc->_ann._insn_##NAME##_enabled; \ + } +#define ANN_REG(TYPE, NAME) \ + static inline bool jit_annr_is_enabled_##NAME(JitCompContext *cc) \ + { \ + return !!cc->_ann._reg_##NAME##_enabled; \ + } +#include "jit_ir.def" +#undef ANN_LABEL +#undef ANN_INSN +#undef ANN_REG + +/** + * Initialize a compilation context. + * + * @param cc the compilation context + * @param htab_size the initial hash table size of constant pool + * + * @return cc if succeeds, NULL otherwise + */ +JitCompContext * +jit_cc_init(JitCompContext *cc, unsigned htab_size); + +/** + * Release all resources of a compilation context, which doesn't + * include the compilation context itself. + * + * @param cc the compilation context + */ +void +jit_cc_destroy(JitCompContext *cc); + +/** + * Increase the reference count of the compilation context. + * + * @param cc the compilation context + */ +static inline void +jit_cc_inc_ref(JitCompContext *cc) +{ + cc->_reference_count++; +} + +/** + * Decrease the reference_count and destroy and free the compilation + * context if the reference_count is decreased to zero. + * + * @param cc the compilation context + */ +void +jit_cc_delete(JitCompContext *cc); + +char * +jit_get_last_error(JitCompContext *cc); + +void +jit_set_last_error(JitCompContext *cc, const char *error); + +void +jit_set_last_error_v(JitCompContext *cc, const char *format, ...); + +/** + * Create a I32 constant value with relocatable into the compilation + * context. A constant value that has relocation info cannot be + * constant-folded as normal constants because its value depends on + * runtime context and may be different in different executions. + * + * @param cc compilation context + * @param val a I32 value + * @param rel relocation information + * + * @return a constant register containing the value + */ +JitReg +jit_cc_new_const_I32_rel(JitCompContext *cc, int32 val, uint32 rel); + +/** + * Create a I32 constant value without relocation info (0) into the + * compilation context. + * + * @param cc compilation context + * @param val a I32 value + * + * @return a constant register containing the value + */ +static inline JitReg +jit_cc_new_const_I32(JitCompContext *cc, int32 val) +{ + return jit_cc_new_const_I32_rel(cc, val, 0); +} + +/** + * Create a I64 constant value into the compilation context. + * + * @param cc compilation context + * @param val a I64 value + * + * @return a constant register containing the value + */ +JitReg +jit_cc_new_const_I64(JitCompContext *cc, int64 val); + +#if UINTPTR_MAX == UINT64_MAX +#define jit_cc_new_const_PTR jit_cc_new_const_I64 +#else +#define jit_cc_new_const_PTR jit_cc_new_const_I32 +#endif + +/** + * Create a F32 constant value into the compilation context. + * + * @param cc compilation context + * @param val a F32 value + * + * @return a constant register containing the value + */ +JitReg +jit_cc_new_const_F32(JitCompContext *cc, float val); + +/** + * Create a F64 constant value into the compilation context. + * + * @param cc compilation context + * @param val a F64 value + * + * @return a constant register containing the value + */ +JitReg +jit_cc_new_const_F64(JitCompContext *cc, double val); + +/** + * Get the relocation info of a I32 constant register. + * + * @param cc compilation context + * @param reg constant register + * + * @return the relocation info of the constant + */ +uint32 +jit_cc_get_const_I32_rel(JitCompContext *cc, JitReg reg); + +/** + * Get the constant value of a I32 constant register. + * + * @param cc compilation context + * @param reg constant register + * + * @return the constant value + */ +int32 +jit_cc_get_const_I32(JitCompContext *cc, JitReg reg); + +/** + * Get the constant value of a I64 constant register. + * + * @param cc compilation context + * @param reg constant register + * + * @return the constant value + */ +int64 +jit_cc_get_const_I64(JitCompContext *cc, JitReg reg); + +/** + * Get the constant value of a F32 constant register. + * + * @param cc compilation context + * @param reg constant register + * + * @return the constant value + */ +float +jit_cc_get_const_F32(JitCompContext *cc, JitReg reg); + +/** + * Get the constant value of a F64 constant register. + * + * @param cc compilation context + * @param reg constant register + * + * @return the constant value + */ +double +jit_cc_get_const_F64(JitCompContext *cc, JitReg reg); + +/** + * Get the number of total created labels. + * + * @param cc the compilation context + * + * @return the number of total created labels + */ +static inline unsigned +jit_cc_label_num(JitCompContext *cc) +{ + return cc->_ann._label_num; +} + +/** + * Get the number of total created instructions. + * + * @param cc the compilation context + * + * @return the number of total created instructions + */ +static inline unsigned +jit_cc_insn_num(JitCompContext *cc) +{ + return cc->_ann._insn_num; +} + +/** + * Get the number of total created registers. + * + * @param cc the compilation context + * @param kind the register kind + * + * @return the number of total created registers + */ +static inline unsigned +jit_cc_reg_num(JitCompContext *cc, unsigned kind) +{ + bh_assert(kind < JIT_REG_KIND_L32); + return cc->_ann._reg_num[kind]; +} + +/** + * Create a new label in the compilation context. + * + * @param cc the compilation context + * + * @return a new label in the compilation context + */ +JitReg +jit_cc_new_label(JitCompContext *cc); + +/** + * Create a new block with a new label in the compilation context. + * + * @param cc the compilation context + * @param n number of predecessors + * + * @return a new block with a new label in the compilation context + */ +JitBasicBlock * +jit_cc_new_basic_block(JitCompContext *cc, int n); + +/** + * Resize the predecessor number of a block. + * + * @param cc the containing compilation context + * @param block block to be resized + * @param n new number of predecessors + * + * @return the new block if succeeds, NULL otherwise + */ +JitBasicBlock * +jit_cc_resize_basic_block(JitCompContext *cc, JitBasicBlock *block, int n); + +/** + * Initialize the instruction hash table to the given size and enable + * the instruction's _hash_link annotation. + * + * @param cc the containing compilation context + * @param n size of the hash table + * + * @return true if succeeds, false otherwise + */ +bool +jit_cc_enable_insn_hash(JitCompContext *cc, unsigned n); + +/** + * Destroy the instruction hash table and disable the instruction's + * _hash_link annotation. + * + * @param cc the containing compilation context + */ +void +jit_cc_disable_insn_hash(JitCompContext *cc); + +/** + * Reset the hash table entries. + * + * @param cc the containing compilation context + */ +void +jit_cc_reset_insn_hash(JitCompContext *cc); + +/** + * Allocate a new instruction ID in the compilation context and set it + * to the given instruction. + * + * @param cc the compilation context + * @param insn IR instruction + * + * @return the insn with uid being set + */ +JitInsn * +jit_cc_set_insn_uid(JitCompContext *cc, JitInsn *insn); + +/* + * Similar to jit_cc_set_insn_uid except that if setting uid failed, + * delete the insn. Only used by jit_cc_new_insn + */ +JitInsn * +_jit_cc_set_insn_uid_for_new_insn(JitCompContext *cc, JitInsn *insn); + +/** + * Create a new instruction in the compilation context. + * + * @param cc the compilationo context + * @param NAME instruction name + * + * @return a new instruction in the compilation context + */ +#define jit_cc_new_insn(cc, NAME, ...) \ + _jit_cc_set_insn_uid_for_new_insn(cc, jit_insn_new_##NAME(__VA_ARGS__)) + +/* + * Helper function for jit_cc_new_insn_norm. + */ +JitInsn * +_jit_cc_new_insn_norm(JitCompContext *cc, JitReg *result, JitInsn *insn); + +/** + * Create a new instruction in the compilation context and normalize + * the instruction (constant folding and simplification etc.). If the + * instruction hashing is enabled (anni__hash_link is enabled), try to + * find the existing equivalent insruction first before adding a new + * one to the compilation contest. + * + * @param cc the compilationo context + * @param result returned result of the instruction. If the value is + * non-zero, it is the result of the constant-folding or an exsiting + * equivalent instruction, in which case no instruction is added into + * the compilation context. Otherwise, a new normalized instruction + * has been added into the compilation context. + * @param NAME instruction name + * + * @return a new or existing instruction in the compilation context + */ +#define jit_cc_new_insn_norm(cc, result, NAME, ...) \ + _jit_cc_new_insn_norm(cc, result, jit_insn_new_##NAME(__VA_ARGS__)) + +/** + * Helper function for GEN_INSN + * + * @param cc compilation context + * @param block the current block + * @param insn the new instruction + * + * @return the new instruction if inserted, NULL otherwise + */ +static inline JitInsn * +_gen_insn(JitCompContext *cc, JitInsn *insn) +{ + if (insn) + jit_basic_block_append_insn(cc->cur_basic_block, insn); + else + jit_set_last_error(cc, "generate insn failed"); + + return insn; +} + +/** + * Generate and append an instruction to the current block. + */ +#define GEN_INSN(...) _gen_insn(cc, jit_cc_new_insn(cc, __VA_ARGS__)) + +/** + * Create a constant register without relocation info. + * + * @param Type type of the register + * @param val the constant value + * + * @return the constant register if succeeds, 0 otherwise + */ +#define NEW_CONST(Type, val) jit_cc_new_const_##Type(cc, val) + +/** + * Create a new virtual register in the compilation context. + * + * @param cc the compilation context + * @param kind kind of the register + * + * @return a new label in the compilation context + */ +JitReg +jit_cc_new_reg(JitCompContext *cc, unsigned kind); + +/* + * Create virtual registers with specific types in the compilation + * context. They are more convenient than the above one. + */ + +static inline JitReg +jit_cc_new_reg_I32(JitCompContext *cc) +{ + return jit_cc_new_reg(cc, JIT_REG_KIND_I32); +} + +static inline JitReg +jit_cc_new_reg_I64(JitCompContext *cc) +{ + return jit_cc_new_reg(cc, JIT_REG_KIND_I64); +} + +#if UINTPTR_MAX == UINT64_MAX +#define jit_cc_new_reg_ptr jit_cc_new_reg_I64 +#else +#define jit_cc_new_reg_ptr jit_cc_new_reg_I32 +#endif + +static inline JitReg +jit_cc_new_reg_F32(JitCompContext *cc) +{ + return jit_cc_new_reg(cc, JIT_REG_KIND_F32); +} + +static inline JitReg +jit_cc_new_reg_F64(JitCompContext *cc) +{ + return jit_cc_new_reg(cc, JIT_REG_KIND_F64); +} + +static inline JitReg +jit_cc_new_reg_V64(JitCompContext *cc) +{ + return jit_cc_new_reg(cc, JIT_REG_KIND_V64); +} + +static inline JitReg +jit_cc_new_reg_V128(JitCompContext *cc) +{ + return jit_cc_new_reg(cc, JIT_REG_KIND_V128); +} + +static inline JitReg +jit_cc_new_reg_V256(JitCompContext *cc) +{ + return jit_cc_new_reg(cc, JIT_REG_KIND_V256); +} + +/** + * Get the hard register numbe of the given kind + * + * @param cc the compilation context + * @param kind the register kind + * + * @return number of hard registers of the given kind + */ +static inline unsigned +jit_cc_hreg_num(JitCompContext *cc, unsigned kind) +{ + bh_assert(kind < JIT_REG_KIND_L32); + return cc->hreg_info->info[kind].num; +} + +/** + * Check whether a given register is a hard register. + * + * @param cc the compilation context + * @param reg the register which must be a variable + * + * @return true if the register is a hard register + */ +static inline bool +jit_cc_is_hreg(JitCompContext *cc, JitReg reg) +{ + unsigned kind = jit_reg_kind(reg); + unsigned no = jit_reg_no(reg); + bh_assert(jit_reg_is_variable(reg)); + bh_assert(kind < JIT_REG_KIND_L32); + return no < cc->hreg_info->info[kind].num; +} + +/** + * Check whether the given hard register is fixed. + * + * @param cc the compilation context + * @param reg the hard register + * + * @return true if the hard register is fixed + */ +static inline bool +jit_cc_is_hreg_fixed(JitCompContext *cc, JitReg reg) +{ + unsigned kind = jit_reg_kind(reg); + unsigned no = jit_reg_no(reg); + bh_assert(jit_cc_is_hreg(cc, reg)); + bh_assert(kind < JIT_REG_KIND_L32); + return !!cc->hreg_info->info[kind].fixed[no]; +} + +/** + * Check whether the given hard register is caller-saved-native. + * + * @param cc the compilation context + * @param reg the hard register + * + * @return true if the hard register is caller-saved-native + */ +static inline bool +jit_cc_is_hreg_caller_saved_native(JitCompContext *cc, JitReg reg) +{ + unsigned kind = jit_reg_kind(reg); + unsigned no = jit_reg_no(reg); + bh_assert(jit_cc_is_hreg(cc, reg)); + bh_assert(kind < JIT_REG_KIND_L32); + return !!cc->hreg_info->info[kind].caller_saved_native[no]; +} + +/** + * Check whether the given hard register is caller-saved-jitted. + * + * @param cc the compilation context + * @param reg the hard register + * + * @return true if the hard register is caller-saved-jitted + */ +static inline bool +jit_cc_is_hreg_caller_saved_jitted(JitCompContext *cc, JitReg reg) +{ + unsigned kind = jit_reg_kind(reg); + unsigned no = jit_reg_no(reg); + bh_assert(jit_cc_is_hreg(cc, reg)); + bh_assert(kind < JIT_REG_KIND_L32); + return !!cc->hreg_info->info[kind].caller_saved_jitted[no]; +} + +/** + * Return the entry block of the compilation context. + * + * @param cc the compilation context + * + * @return the entry block of the compilation context + */ +static inline JitBasicBlock * +jit_cc_entry_basic_block(JitCompContext *cc) +{ + return *(jit_annl_basic_block(cc, cc->entry_label)); +} + +/** + * Return the exit block of the compilation context. + * + * @param cc the compilation context + * + * @return the exit block of the compilation context + */ +static inline JitBasicBlock * +jit_cc_exit_basic_block(JitCompContext *cc) +{ + return *(jit_annl_basic_block(cc, cc->exit_label)); +} + +void +jit_value_stack_push(JitValueStack *stack, JitValue *value); + +JitValue * +jit_value_stack_pop(JitValueStack *stack); + +void +jit_value_stack_destroy(JitValueStack *stack); + +JitBlock * +jit_block_stack_top(JitBlockStack *stack); + +void +jit_block_stack_push(JitBlockStack *stack, JitBlock *block); + +JitBlock * +jit_block_stack_pop(JitBlockStack *stack); + +void +jit_block_stack_destroy(JitBlockStack *stack); + +bool +jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn, uint32 opnd_idx); + +void +jit_block_destroy(JitBlock *block); + +bool +jit_cc_push_value(JitCompContext *cc, uint8 type, JitReg value); + +bool +jit_cc_pop_value(JitCompContext *cc, uint8 type, JitReg *p_value); + +bool +jit_lock_reg_in_insn(JitCompContext *cc, JitInsn *the_insn, JitReg reg_to_lock); + +/** + * Update the control flow graph after successors of blocks are + * changed so that the predecessor vector of each block represents the + * updated status. The predecessors may not be required by all + * passes, so we don't need to keep them always being updated. + * + * @param cc the compilation context + * + * @return true if succeeds, false otherwise + */ +bool +jit_cc_update_cfg(JitCompContext *cc); + +/** + * Visit each normal block (which is not entry nor exit block) in a + * compilation context. New blocks can be added in the loop body, but + * they won't be visited. Blocks can also be removed safely (by + * setting the label's block annotation to NULL) in the loop body. + * + * @param CC (JitCompContext *) the compilation context + * @param I (unsigned) index variable of the block (label no) + * @param E (unsigned) end index variable of block (last index + 1) + * @param B (JitBasicBlock *) block pointer variable + */ +#define JIT_FOREACH_BLOCK(CC, I, E, B) \ + for ((I) = 2, (E) = (CC)->_ann._label_num; (I) < (E); (I)++) \ + if (((B) = (CC)->_ann._label_basic_block[(I)])) + +/** + * The version that includes entry and exit block. + */ +#define JIT_FOREACH_BLOCK_ENTRY_EXIT(CC, I, E, B) \ + for ((I) = 0, (E) = (CC)->_ann._label_num; (I) < (E); (I)++) \ + if (((B) = (CC)->_ann._label_basic_block[(I)])) + +/** + * Visit each normal block (which is not entry nor exit block) in a + * compilation context in reverse order. New blocks can be added in + * the loop body, but they won't be visited. Blocks can also be + * removed safely (by setting the label's block annotation to NULL) in + * the loop body. + * + * @param CC (JitCompContext *) the compilation context + * @param I (unsigned) index of the block (label no) + * @param B (JitBasicBlock *) block pointer + */ +#define JIT_FOREACH_BLOCK_REVERSE(CC, I, B) \ + for ((I) = (CC)->_ann._label_num; (I) > 2; (I)--) \ + if (((B) = (CC)->_ann._label_basic_block[(I)-1])) + +/** + * The version that includes entry and exit block. + */ +#define JIT_FOREACH_BLOCK_REVERSE_ENTRY_EXIT(CC, I, B) \ + for ((I) = (CC)->_ann._label_num; (I) > 0; (I)--) \ + if (((B) = (CC)->_ann._label_basic_block[(I)-1])) + +#ifdef __cplusplus +} +#endif + +#endif /* end of _JIT_IR_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_regalloc.c b/wasm-micro-runtime/core/iwasm/fast-jit/jit_regalloc.c new file mode 100644 index 0000000..70ca228 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_regalloc.c @@ -0,0 +1,862 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_utils.h" +#include "jit_compiler.h" + +#if BH_DEBUG != 0 +#define VREG_DEF_SANITIZER +#endif + +/** + * A uint16 stack for storing distances of occurrences of virtual + * registers. + */ +typedef struct UintStack { + /* Capacity of the stack. */ + uint32 capacity; + + /* Top index of the stack. */ + uint32 top; + + /* Elements of the vector. */ + uint32 elem[1]; +} UintStack; + +static bool +uint_stack_push(UintStack **stack, unsigned val) +{ + unsigned capacity = *stack ? (*stack)->capacity : 0; + unsigned top = *stack ? (*stack)->top : 0; + + bh_assert(top <= capacity); + + if (top == capacity) { + const unsigned elem_size = sizeof((*stack)->elem[0]); + unsigned new_capacity = capacity ? capacity + capacity / 2 : 4; + UintStack *new_stack = + jit_malloc(offsetof(UintStack, elem) + elem_size * new_capacity); + + if (!new_stack) + return false; + + new_stack->capacity = new_capacity; + new_stack->top = top; + + if (*stack) + memcpy(new_stack->elem, (*stack)->elem, elem_size * top); + + jit_free(*stack); + *stack = new_stack; + } + + (*stack)->elem[(*stack)->top++] = val; + + return true; +} + +static int +uint_stack_top(UintStack *stack) +{ + return stack->elem[stack->top - 1]; +} + +static void +uint_stack_delete(UintStack **stack) +{ + jit_free(*stack); + *stack = NULL; +} + +static void +uint_stack_pop(UintStack **stack) +{ + bh_assert((*stack)->top > 0); + + /** + * TODO: the fact of empty distances stack means there is no instruction + * using current JitReg anymore. so shall we release the HardReg and clean + * VirtualReg information? + */ + if (--(*stack)->top == 0) + uint_stack_delete(stack); +} + +/** + * Information of a virtual register. + */ +typedef struct VirtualReg { + /* The hard register allocated to this virtual register. */ + JitReg hreg; + + /* The spill slot allocated to this virtual register. */ + JitReg slot; + + /* The hard register allocated to global virtual registers. It is 0 + for local registers, whose lifetime is within one basic block. */ + JitReg global_hreg; + + /* Distances from the beginning of basic block of all occurrences of the + virtual register in the basic block. */ + UintStack *distances; +} VirtualReg; + +/** + * Information of a hard register. + */ +typedef struct HardReg { + /* The virtual register this hard register is allocated to. */ + JitReg vreg; +} HardReg; + +/** + * Information of a spill slot. + */ +typedef struct SpillSlot { + /* The virtual register this spill slot is allocated to. */ + JitReg vreg; +} SpillSlot; + +typedef struct RegallocContext { + /* The compiler context. */ + JitCompContext *cc; + + /* Information of virtual registers. The register allocation must + not increase the virtual register number during the allocation + process. */ + VirtualReg *vregs[JIT_REG_KIND_L32]; + + /* Information of hard registers. */ + HardReg *hregs[JIT_REG_KIND_L32]; + + /* Number of elements in the spill_slots array. */ + uint32 spill_slot_num; + + /* Information of spill slots. */ + SpillSlot *spill_slots; + + /* The last define-released hard register. */ + JitReg last_def_released_hreg; +} RegallocContext; + +/** + * Get the VirtualReg structure of the given virtual register. + * + * @param rc the regalloc context + * @param vreg the virtual register + * + * @return the VirtualReg structure of the given virtual register + */ +static VirtualReg * +rc_get_vr(RegallocContext *rc, JitReg vreg) +{ + unsigned kind = jit_reg_kind(vreg); + unsigned no = jit_reg_no(vreg); + + bh_assert(jit_reg_is_variable(vreg)); + bh_assert(kind < JIT_REG_KIND_L32); + + return &rc->vregs[kind][no]; +} + +/** + * Get the HardReg structure of the given hard register. + * + * @param rc the regalloc context + * @param hreg the hard register + * + * @return the HardReg structure of the given hard register + */ +static HardReg * +rc_get_hr(RegallocContext *rc, JitReg hreg) +{ + unsigned kind = jit_reg_kind(hreg); + unsigned no = jit_reg_no(hreg); + + bh_assert(jit_reg_is_variable(hreg) && jit_cc_is_hreg(rc->cc, hreg)); + bh_assert(kind < JIT_REG_KIND_L32); + + return &rc->hregs[kind][no]; +} + +/** + * Get the SpillSlot structure of the given slot. + * + * @param rc the regalloc context + * @param slot the constant register representing the slot index + * + * @return the SpillSlot of the given slot + */ +static SpillSlot * +rc_get_spill_slot(RegallocContext *rc, JitReg slot) +{ + unsigned index = jit_cc_get_const_I32(rc->cc, slot); + + bh_assert(index < rc->spill_slot_num); + + return &rc->spill_slots[index]; +} + +/** + * Get the stride in the spill slots of the register. + * + * @param reg a virtual register + * + * @return stride in the spill slots + */ +static unsigned +get_reg_stride(JitReg reg) +{ + static const uint8 strides[] = { 0, 1, 2, 1, 2, 2, 4, 8, 0 }; + uint32 kind = jit_reg_kind(reg); + bh_assert(kind <= JIT_REG_KIND_L32); + return strides[kind]; +} + +/** + * Allocate a spill slot for the given virtual register. + * + * @param rc the regalloc context + * @param vreg the virtual register + * + * @return the spill slot encoded in a consant register + */ +static JitReg +rc_alloc_spill_slot(RegallocContext *rc, JitReg vreg) +{ + const unsigned stride = get_reg_stride(vreg); + unsigned mask, new_num, i, j; + SpillSlot *slots; + + bh_assert(stride > 0); + + for (i = 0; i < rc->spill_slot_num; i += stride) + for (j = i;; j++) { + if (j == i + stride) + /* Found a free slot for vreg. */ + goto found; + + if (rc->spill_slots[j].vreg) + break; + } + + /* No free slot, increase the slot number. */ + mask = stride - 1; + /* Align the slot index. */ + i = (rc->spill_slot_num + mask) & ~mask; + new_num = i == 0 ? 32 : i + i / 2; + + if (!(slots = jit_calloc(sizeof(*slots) * new_num))) + return 0; + + if (rc->spill_slots) + memcpy(slots, rc->spill_slots, sizeof(*slots) * rc->spill_slot_num); + + jit_free(rc->spill_slots); + rc->spill_slots = slots; + rc->spill_slot_num = new_num; + +found: + /* Now, i is the first slot for vreg. */ + if ((i + stride) * 4 > rc->cc->spill_cache_size) + /* No frame space for the spill area. */ + return 0; + + /* Allocate the slot(s) to vreg. */ + for (j = i; j < i + stride; j++) + rc->spill_slots[j].vreg = vreg; + + return jit_cc_new_const_I32(rc->cc, i); +} + +/** + * Free a spill slot. + * + * @param rc the regalloc context + * @param slot_reg the constant register representing the slot index + */ +static void +rc_free_spill_slot(RegallocContext *rc, JitReg slot_reg) +{ + if (slot_reg) { + SpillSlot *slot = rc_get_spill_slot(rc, slot_reg); + const JitReg vreg = slot->vreg; + const unsigned stride = get_reg_stride(vreg); + unsigned i; + + for (i = 0; i < stride; i++) + slot[i].vreg = 0; + } +} + +static void +rc_destroy(RegallocContext *rc) +{ + unsigned i, j; + + for (i = JIT_REG_KIND_VOID; i < JIT_REG_KIND_L32; i++) { + const unsigned vreg_num = jit_cc_reg_num(rc->cc, i); + + if (rc->vregs[i]) + for (j = 0; j < vreg_num; j++) + uint_stack_delete(&rc->vregs[i][j].distances); + + jit_free(rc->vregs[i]); + jit_free(rc->hregs[i]); + } + + jit_free(rc->spill_slots); +} + +static bool +rc_init(RegallocContext *rc, JitCompContext *cc) +{ + unsigned i, j; + + memset(rc, 0, sizeof(*rc)); + rc->cc = cc; + + for (i = JIT_REG_KIND_VOID; i < JIT_REG_KIND_L32; i++) { + const unsigned vreg_num = jit_cc_reg_num(cc, i); + const unsigned hreg_num = jit_cc_hreg_num(cc, i); + + if (vreg_num > 0 + && !(rc->vregs[i] = jit_calloc(sizeof(VirtualReg) * vreg_num))) + goto fail; + if (hreg_num > 0 + && !(rc->hregs[i] = jit_calloc(sizeof(HardReg) * hreg_num))) + goto fail; + + /* Hard registers can only be allocated to themselves. */ + for (j = 0; j < hreg_num; j++) + rc->vregs[i][j].global_hreg = jit_reg_new(i, j); + } + + return true; + +fail: + rc_destroy(rc); + + return false; +} + +/** + * Check whether the given register is an allocation candidate, which + * must be a variable register that is not fixed hard register. + * + * @param cc the compilation context + * @param reg the register + * + * @return true if the register is an allocation candidate + */ +static bool +is_alloc_candidate(JitCompContext *cc, JitReg reg) +{ + return (jit_reg_is_variable(reg) + && (!jit_cc_is_hreg(cc, reg) || !jit_cc_is_hreg_fixed(cc, reg))); +} + +#ifdef VREG_DEF_SANITIZER +static void +check_vreg_definition(RegallocContext *rc, JitInsn *insn) +{ + JitRegVec regvec = jit_insn_opnd_regs(insn); + JitReg *regp, reg_defined = 0; + unsigned i, first_use = jit_insn_opnd_first_use(insn); + + /* check if there is the definition of an vr before its references */ + JIT_REG_VEC_FOREACH(regvec, i, regp) + { + VirtualReg *vr = NULL; + + if (!is_alloc_candidate(rc->cc, *regp)) + continue; + + /* a strong assumption that there is only one defined reg */ + if (i < first_use) { + reg_defined = *regp; + continue; + } + + /** + * both definition and references are in one instruction, + * like MOV i3, i3 + */ + if (reg_defined == *regp) + continue; + + vr = rc_get_vr(rc, *regp); + bh_assert(vr->distances); + } +} +#endif + +/** + * Collect distances from the beginning of basic block of all occurrences of + * each virtual register. + * + * @param rc the regalloc context + * @param basic_block the basic block + * + * @return distance of the end instruction if succeeds, -1 otherwise + */ +static int +collect_distances(RegallocContext *rc, JitBasicBlock *basic_block) +{ + JitInsn *insn; + int distance = 1; + + JIT_FOREACH_INSN(basic_block, insn) + { +#if WASM_ENABLE_SHARED_MEMORY != 0 + /* fence insn doesn't have any operand, hence, no regs involved */ + if (insn->opcode == JIT_OP_FENCE) { + continue; + } +#endif + + JitRegVec regvec = jit_insn_opnd_regs(insn); + unsigned i; + JitReg *regp; + +#ifdef VREG_DEF_SANITIZER + check_vreg_definition(rc, insn); +#endif + + /* NOTE: the distance may be pushed more than once if the + virtual register occurs multiple times in the + instruction. */ + JIT_REG_VEC_FOREACH(regvec, i, regp) + if (is_alloc_candidate(rc->cc, *regp)) + if (!uint_stack_push(&(rc_get_vr(rc, *regp))->distances, distance)) + return -1; + + /* Integer overflow check, normally it won't happen, but + we had better add the check here */ + if (distance >= INT32_MAX) + return -1; + + distance++; + } + + return distance; +} + +static JitReg +offset_of_spill_slot(JitCompContext *cc, JitReg slot) +{ + return jit_cc_new_const_I32(cc, cc->spill_cache_offset + + jit_cc_get_const_I32(cc, slot) * 4); +} + +/** + * Reload the virtual register from memory. Reload instruction will + * be inserted after the given instruction. + * + * @param rc the regalloc context + * @param vreg the virtual register to be reloaded + * @param cur_insn the current instruction after which the reload + * insertion will be inserted + * + * @return the reload instruction if succeeds, NULL otherwise + */ +static JitInsn * +reload_vreg(RegallocContext *rc, JitReg vreg, JitInsn *cur_insn) +{ + VirtualReg *vr = rc_get_vr(rc, vreg); + HardReg *hr = rc_get_hr(rc, vr->hreg); + JitInsn *insn = NULL; + + if (vreg == rc->cc->exec_env_reg) + /* Reload exec_env_reg with LDEXECENV. */ + insn = jit_cc_new_insn(rc->cc, LDEXECENV, vr->hreg); + else + /* Allocate spill slot if not yet and reload from there. */ + { + JitReg fp_reg = rc->cc->fp_reg, offset; + + if (!vr->slot && !(vr->slot = rc_alloc_spill_slot(rc, vreg))) + /* Cannot allocte spill slot (due to OOM or frame size limit). */ + return NULL; + + offset = offset_of_spill_slot(rc->cc, vr->slot); + + switch (jit_reg_kind(vreg)) { + case JIT_REG_KIND_I32: + insn = jit_cc_new_insn(rc->cc, LDI32, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_I64: + insn = jit_cc_new_insn(rc->cc, LDI64, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_F32: + insn = jit_cc_new_insn(rc->cc, LDF32, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_F64: + insn = jit_cc_new_insn(rc->cc, LDF64, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_V64: + insn = jit_cc_new_insn(rc->cc, LDV64, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_V128: + insn = + jit_cc_new_insn(rc->cc, LDV128, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_V256: + insn = + jit_cc_new_insn(rc->cc, LDV256, vr->hreg, fp_reg, offset); + break; + default: + bh_assert(0); + } + } + + if (insn) + jit_insn_insert_after(cur_insn, insn); + + bh_assert(hr->vreg == vreg); + hr->vreg = vr->hreg = 0; + + return insn; +} + +/** + * Spill the virtual register (which cannot be exec_env_reg) to memory. + * Spill instruction will be inserted after the given instruction. + * + * @param rc the regalloc context + * @param vreg the virtual register to be reloaded + * @param cur_insn the current instruction after which the reload + * insertion will be inserted + * + * @return the spill instruction if succeeds, NULL otherwise + */ +static JitInsn * +spill_vreg(RegallocContext *rc, JitReg vreg, JitInsn *cur_insn) +{ + VirtualReg *vr = rc_get_vr(rc, vreg); + JitReg fp_reg = rc->cc->fp_reg, offset; + JitInsn *insn; + + /* There is no chance to spill exec_env_reg. */ + bh_assert(vreg != rc->cc->exec_env_reg); + bh_assert(vr->hreg && vr->slot); + offset = offset_of_spill_slot(rc->cc, vr->slot); + + switch (jit_reg_kind(vreg)) { + case JIT_REG_KIND_I32: + insn = jit_cc_new_insn(rc->cc, STI32, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_I64: + insn = jit_cc_new_insn(rc->cc, STI64, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_F32: + insn = jit_cc_new_insn(rc->cc, STF32, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_F64: + insn = jit_cc_new_insn(rc->cc, STF64, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_V64: + insn = jit_cc_new_insn(rc->cc, STV64, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_V128: + insn = jit_cc_new_insn(rc->cc, STV128, vr->hreg, fp_reg, offset); + break; + case JIT_REG_KIND_V256: + insn = jit_cc_new_insn(rc->cc, STV256, vr->hreg, fp_reg, offset); + break; + default: + bh_assert(0); + return NULL; + } + + if (insn) + jit_insn_insert_after(cur_insn, insn); + + return insn; +} + +/** + * Allocate a hard register for the virtual register. Necessary + * reloade instruction will be inserted after the given instruction. + * + * @param rc the regalloc context + * @param vreg the virtual register + * @param insn the instruction after which the reload insertion will + * be inserted + * @param distance the distance of the current instruction + * + * @return the hard register allocated if succeeds, 0 otherwise + */ +static JitReg +allocate_hreg(RegallocContext *rc, JitReg vreg, JitInsn *insn, int distance) +{ + const int kind = jit_reg_kind(vreg); + const HardReg *hregs; + unsigned hreg_num; + JitReg hreg, vreg_to_reload = 0; + int min_distance = distance, vr_distance; + VirtualReg *vr = rc_get_vr(rc, vreg); + unsigned i; + + bh_assert(kind < JIT_REG_KIND_L32); + hregs = rc->hregs[kind]; + hreg_num = jit_cc_hreg_num(rc->cc, kind); + + if (hreg_num == 0) + /* Unsupported hard register kind. */ + { + jit_set_last_error(rc->cc, "unsupported hard register kind"); + return 0; + } + + if (vr->global_hreg) + /* It has globally allocated register, we can only use it. */ + { + if ((vreg_to_reload = (rc_get_hr(rc, vr->global_hreg))->vreg)) + if (!reload_vreg(rc, vreg_to_reload, insn)) + return 0; + + return vr->global_hreg; + } + + /* Use the last define-released register if its kind is correct and + it's free so as to optimize for two-operand instructions. */ + if (jit_reg_kind(rc->last_def_released_hreg) == kind + && (rc_get_hr(rc, rc->last_def_released_hreg))->vreg == 0) + return rc->last_def_released_hreg; + + /* No hint given, just try to pick any free register. */ + for (i = 0; i < hreg_num; i++) { + hreg = jit_reg_new(kind, i); + + if (jit_cc_is_hreg_fixed(rc->cc, hreg)) + continue; + + if (hregs[i].vreg == 0) + /* Found a free one, return it. */ + return hreg; + } + + /* No free registers, need to spill and reload one. */ + for (i = 0; i < hreg_num; i++) { + if (jit_cc_is_hreg_fixed(rc->cc, jit_reg_new(kind, i))) + continue; + + vr = rc_get_vr(rc, hregs[i].vreg); + /* TODO: since the hregs[i] is in use, its distances should be valid */ + vr_distance = vr->distances ? uint_stack_top(vr->distances) : 0; + + if (vr_distance < min_distance) { + min_distance = vr_distance; + vreg_to_reload = hregs[i].vreg; + hreg = jit_reg_new(kind, i); + } + } + + bh_assert(min_distance < distance); + + if (!reload_vreg(rc, vreg_to_reload, insn)) + return 0; + + return hreg; +} + +/** + * Allocate a hard register for the virtual register if not allocated + * yet. Necessary spill and reloade instructions will be inserted + * before/after and after the given instruction. This operation will + * convert the virtual register's state from 1 or 3 to 2. + * + * @param rc the regalloc context + * @param vreg the virtual register + * @param insn the instruction after which the spill and reload + * insertions will be inserted + * @param distance the distance of the current instruction + * + * @return the hard register allocated to the virtual register if + * succeeds, 0 otherwise + */ +static JitReg +allocate_for_vreg(RegallocContext *rc, JitReg vreg, JitInsn *insn, int distance) +{ + VirtualReg *vr = rc_get_vr(rc, vreg); + + if (vr->hreg) + /* It has had a hard register, reuse it. */ + return vr->hreg; + + /* Not allocated yet. */ + if ((vr->hreg = allocate_hreg(rc, vreg, insn, distance))) + (rc_get_hr(rc, vr->hreg))->vreg = vreg; + + return vr->hreg; +} + +/** + * Clobber live registers. + * + * @param rc the regalloc context + * @param is_native whether it's native ABI or JITed ABI + * @param insn the instruction after which the reload insertion will + * be inserted + * + * @return true if succeeds, false otherwise + */ +static bool +clobber_live_regs(RegallocContext *rc, bool is_native, JitInsn *insn) +{ + unsigned i, j; + + for (i = JIT_REG_KIND_VOID; i < JIT_REG_KIND_L32; i++) { + const unsigned hreg_num = jit_cc_hreg_num(rc->cc, i); + + for (j = 0; j < hreg_num; j++) { + JitReg hreg = jit_reg_new(i, j); + bool caller_saved = + (is_native ? jit_cc_is_hreg_caller_saved_native(rc->cc, hreg) + : jit_cc_is_hreg_caller_saved_jitted(rc->cc, hreg)); + + if (caller_saved && rc->hregs[i][j].vreg) + if (!reload_vreg(rc, rc->hregs[i][j].vreg, insn)) + return false; + } + } + + return true; +} + +/** + * Do local register allocation for the given basic block + * + * @param rc the regalloc context + * @param basic_block the basic block + * @param distance the distance of the last instruction of the basic block + * + * @return true if succeeds, false otherwise + */ +static bool +allocate_for_basic_block(RegallocContext *rc, JitBasicBlock *basic_block, + int distance) +{ + JitInsn *insn; + + JIT_FOREACH_INSN_REVERSE(basic_block, insn) + { +#if WASM_ENABLE_SHARED_MEMORY != 0 + /* fence insn doesn't have any operand, hence, no regs involved */ + if (insn->opcode == JIT_OP_FENCE) { + continue; + } +#endif + + JitRegVec regvec = jit_insn_opnd_regs(insn); + unsigned first_use = jit_insn_opnd_first_use(insn); + unsigned i; + JitReg *regp; + + distance--; + + JIT_REG_VEC_FOREACH_DEF(regvec, i, regp, first_use) + if (is_alloc_candidate(rc->cc, *regp)) { + const JitReg vreg = *regp; + VirtualReg *vr = rc_get_vr(rc, vreg); + + if (!(*regp = allocate_for_vreg(rc, vreg, insn, distance))) + return false; + + /* Spill the register if required. */ + if (vr->slot && !spill_vreg(rc, vreg, insn)) + return false; + + bh_assert(uint_stack_top(vr->distances) == distance); + uint_stack_pop(&vr->distances); + /* Record the define-released hard register. */ + rc->last_def_released_hreg = vr->hreg; + /* Release the hreg and spill slot. */ + rc_free_spill_slot(rc, vr->slot); + (rc_get_hr(rc, vr->hreg))->vreg = 0; + vr->hreg = vr->slot = 0; + } + + if (insn->opcode == JIT_OP_CALLBC) { + if (!clobber_live_regs(rc, false, insn)) + return false; + + /* The exec_env_reg is implicitly used by the callee. */ + if (!allocate_for_vreg(rc, rc->cc->exec_env_reg, insn, distance)) + return false; + } + else if (insn->opcode == JIT_OP_CALLNATIVE) { + if (!clobber_live_regs(rc, true, insn)) + return false; + } + + JIT_REG_VEC_FOREACH_USE(regvec, i, regp, first_use) + if (is_alloc_candidate(rc->cc, *regp)) { + if (!allocate_for_vreg(rc, *regp, insn, distance)) + return false; + } + + JIT_REG_VEC_FOREACH_USE(regvec, i, regp, first_use) + if (is_alloc_candidate(rc->cc, *regp)) { + VirtualReg *vr = rc_get_vr(rc, *regp); + bh_assert(uint_stack_top(vr->distances) == distance); + uint_stack_pop(&vr->distances); + /* be sure that the hreg exists and hasn't been spilled out */ + bh_assert(vr->hreg != 0); + *regp = vr->hreg; + } + } + + return true; +} + +bool +jit_pass_regalloc(JitCompContext *cc) +{ + RegallocContext rc = { 0 }; + unsigned label_index, end_label_index; + JitBasicBlock *basic_block; + VirtualReg *self_vr; + bool retval = false; + + if (!rc_init(&rc, cc)) + return false; + + /* NOTE: don't allocate new virtual registers during allocation + because the rc->vregs array is fixed size. */ + + /* TODO: allocate hard registers for global virtual registers here. + Currently, exec_env_reg is the only global virtual register. */ + self_vr = rc_get_vr(&rc, cc->exec_env_reg); + + JIT_FOREACH_BLOCK_ENTRY_EXIT(cc, label_index, end_label_index, basic_block) + { + int distance; + + /* TODO: initialize hreg for live-out registers. */ + self_vr->hreg = self_vr->global_hreg; + (rc_get_hr(&rc, cc->exec_env_reg))->vreg = cc->exec_env_reg; + + /** + * TODO: the allocation of a basic block keeps using vregs[] + * and hregs[] from previous basic block + */ + if ((distance = collect_distances(&rc, basic_block)) < 0) + goto cleanup_and_return; + + if (!allocate_for_basic_block(&rc, basic_block, distance)) + goto cleanup_and_return; + + /* TODO: generate necessary spills for live-in registers. */ + } + + retval = true; + +cleanup_and_return: + rc_destroy(&rc); + + return retval; +} diff --git a/wasm-micro-runtime/core/iwasm/fast-jit/jit_utils.h b/wasm-micro-runtime/core/iwasm/fast-jit/jit_utils.h new file mode 100644 index 0000000..a533c70 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/fast-jit/jit_utils.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _JIT_UTILS_H_ +#define _JIT_UTILS_H_ + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void * +jit_malloc(unsigned int size) +{ + return wasm_runtime_malloc(size); +} + +static inline void * +jit_calloc(unsigned int size) +{ + void *ret = wasm_runtime_malloc(size); + if (ret) { + memset(ret, 0, size); + } + return ret; +} + +static inline void +jit_free(void *ptr) +{ + if (ptr) + wasm_runtime_free(ptr); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/include/aot_comp_option.h b/wasm-micro-runtime/core/iwasm/include/aot_comp_option.h new file mode 100644 index 0000000..617b68f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/include/aot_comp_option.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __AOT_COMP_OPTION_H__ +#define __AOT_COMP_OPTION_H__ + +typedef struct AOTCompOption { + bool is_jit_mode; + bool is_indirect_mode; + char *target_arch; + char *target_abi; + char *target_cpu; + char *cpu_features; + bool is_sgx_platform; + bool enable_bulk_memory; + bool enable_thread_mgr; + bool enable_tail_call; + bool enable_simd; + bool enable_ref_types; + bool enable_gc; + bool enable_aux_stack_check; + bool enable_aux_stack_frame; + bool enable_perf_profiling; + bool enable_memory_profiling; + bool disable_llvm_intrinsics; + bool disable_llvm_lto; + bool enable_llvm_pgo; + bool enable_stack_estimation; + bool quick_invoke_c_api_import; + char *use_prof_file; + uint32_t opt_level; + uint32_t size_level; + uint32_t output_format; + uint32_t bounds_checks; + uint32_t stack_bounds_checks; + uint32_t segue_flags; + char **custom_sections; + uint32_t custom_sections_count; + const char *stack_usage_file; + const char *llvm_passes; + const char *builtin_intrinsics; +} AOTCompOption, *aot_comp_option_t; + +#endif diff --git a/wasm-micro-runtime/core/iwasm/include/aot_export.h b/wasm-micro-runtime/core/iwasm/include/aot_export.h new file mode 100644 index 0000000..d06fef1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/include/aot_export.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file aot_export.h + * + * @brief This file defines the exported AOT compilation APIs + */ + +#ifndef _AOT_EXPORT_H +#define _AOT_EXPORT_H + +#include +#include + +#include "aot_comp_option.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct AOTCompData; +typedef struct AOTCompData *aot_comp_data_t; + +struct AOTCompContext; +typedef struct AOTCompContext *aot_comp_context_t; + +aot_comp_data_t +aot_create_comp_data(void *wasm_module, const char *target_arch, + bool gc_enabled); + +void +aot_destroy_comp_data(aot_comp_data_t comp_data); + +#if WASM_ENABLE_DEBUG_AOT != 0 +typedef void *dwarf_extractor_handle_t; +dwarf_extractor_handle_t +create_dwarf_extractor(aot_comp_data_t comp_data, char *file_name); +#endif + +enum { + AOT_FORMAT_FILE, + AOT_OBJECT_FILE, + AOT_LLVMIR_UNOPT_FILE, + AOT_LLVMIR_OPT_FILE, +}; + +bool +aot_compiler_init(void); + +void +aot_compiler_destroy(void); + +aot_comp_context_t +aot_create_comp_context(aot_comp_data_t comp_data, aot_comp_option_t option); + +void +aot_destroy_comp_context(aot_comp_context_t comp_ctx); + +bool +aot_compile_wasm(aot_comp_context_t comp_ctx); + +bool +aot_emit_llvm_file(aot_comp_context_t comp_ctx, const char *file_name); + +bool +aot_emit_object_file(aot_comp_context_t comp_ctx, const char *file_name); + +bool +aot_emit_aot_file(aot_comp_context_t comp_ctx, aot_comp_data_t comp_data, + const char *file_name); + +void +aot_destroy_aot_file(uint8_t *aot_file); + +char * +aot_get_last_error(); + +uint32_t +aot_get_plt_table_size(); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _AOT_EXPORT_H */ diff --git a/wasm-micro-runtime/core/iwasm/include/gc_export.h b/wasm-micro-runtime/core/iwasm/include/gc_export.h new file mode 100644 index 0000000..777551e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/include/gc_export.h @@ -0,0 +1,970 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file gc_export.h + * + * @brief This file defines the exported GC APIs + */ + +#ifndef _GC_EXPORT_H +#define _GC_EXPORT_H + +#include "wasm_export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint8_t wasm_value_type_t; + +typedef enum wasm_value_type_enum { + VALUE_TYPE_I32 = 0x7F, + VALUE_TYPE_I64 = 0x7E, + VALUE_TYPE_F32 = 0x7D, + VALUE_TYPE_F64 = 0x7C, + VALUE_TYPE_V128 = 0x7B, + /* GC Types */ + VALUE_TYPE_I8 = 0x78, + VALUE_TYPE_I16 = 0x77, + VALUE_TYPE_NULLFUNCREF = 0x73, + VALUE_TYPE_NULLEXTERNREF = 0x72, + VALUE_TYPE_NULLREF = 0x71, + VALUE_TYPE_FUNCREF = 0x70, + VALUE_TYPE_EXTERNREF = 0x6F, + VALUE_TYPE_ANYREF = 0x6E, + VALUE_TYPE_EQREF = 0x6D, + VALUE_TYPE_I31REF = 0x6C, + VALUE_TYPE_STRUCTREF = 0x6B, + VALUE_TYPE_ARRAYREF = 0x6A, + VALUE_TYPE_HT_NON_NULLABLE_REF = 0x64, + VALUE_TYPE_HT_NULLABLE_REF = 0x63, + /* Stringref Types */ + VALUE_TYPE_STRINGREF = 0X67, + VALUE_TYPE_STRINGVIEWWTF8 = 0x66, + VALUE_TYPE_STRINGVIEWWTF16 = 0x62, + VALUE_TYPE_STRINGVIEWITER = 0x61 +} wasm_value_type_enum; + +typedef int32_t wasm_heap_type_t; + +typedef enum wasm_heap_type_enum { + HEAP_TYPE_FUNC = -0x10, + HEAP_TYPE_EXTERN = -0x11, + HEAP_TYPE_ANY = -0x12, + HEAP_TYPE_EQ = -0x13, + HEAP_TYPE_I31 = -0x16, + HEAP_TYPE_NOFUNC = -0x17, + HEAP_TYPE_NOEXTERN = -0x18, + HEAP_TYPE_STRUCT = -0x19, + HEAP_TYPE_ARRAY = -0x1A, + HEAP_TYPE_NONE = -0x1B +} wasm_heap_type_enum; + +struct WASMObject; +typedef struct WASMObject *wasm_obj_t; + +#ifndef WASM_VALUE_DEFINED +#define WASM_VALUE_DEFINED +typedef union V128 { + int8_t i8x16[16]; + int16_t i16x8[8]; + int32_t i32x8[4]; + int64_t i64x2[2]; + float f32x4[4]; + double f64x2[2]; +} V128; + +typedef union WASMValue { + int32_t i32; + uint32_t u32; + uint32_t global_index; + uint32_t ref_index; + int64_t i64; + uint64_t u64; + float f32; + double f64; + V128 v128; + wasm_obj_t gc_obj; + uint32_t type_index; + struct { + uint32_t type_index; + uint32_t length; + } array_new_default; + /* pointer to a memory space holding more data, current usage: + * struct.new init value: WASMStructNewInitValues * + * array.new init value: WASMArrayNewInitValues * + */ + void *data; +} WASMValue; +#endif /* end of WASM_VALUE_DEFINED */ + +typedef union WASMValue wasm_value_t; + +/* Reference type, the layout is same as WasmRefType in wasm.h + * use wasm_ref_type_set_type_idx to initialize as concrete ref type + * use wasm_ref_type_set_heap_type to initialize as abstract ref type + */ +typedef struct wasm_ref_type_t { + wasm_value_type_t value_type; + bool nullable; + int32_t heap_type; +} wasm_ref_type_t; + +/** + * Local object reference that can be traced when GC occurs. All + * native functions that need to hold WASM objects which may not be + * referenced from other elements of GC root set may be hold with + * this type of variable so that they can be traced when GC occurs. + * Before using such a variable, it must be pushed onto the stack + * (implemented as a chain) of such variables, and before leaving the + * frame of the variables, they must be popped from the stack. + */ +typedef struct WASMLocalObjectRef { + /* Previous local object reference variable on the stack */ + struct WASMLocalObjectRef *prev; + /* The reference of WASM object hold by this variable */ + wasm_obj_t val; +} WASMLocalObjectRef, wasm_local_obj_ref_t; + +struct WASMType; +struct WASMFuncType; +struct WASMStructType; +struct WASMArrayType; + +typedef struct WASMType *wasm_defined_type_t; +typedef struct WASMFuncType *wasm_func_type_t; +typedef struct WASMStructType *wasm_struct_type_t; +typedef struct WASMArrayType *wasm_array_type_t; + +struct WASMExternrefObject; +struct WASMAnyrefObject; +struct WASMStructObject; +struct WASMArrayObject; +struct WASMFuncObject; + +typedef struct WASMExternrefObject *wasm_externref_obj_t; +typedef struct WASMAnyrefObject *wasm_anyref_obj_t; +typedef struct WASMStructObject *wasm_struct_obj_t; +typedef struct WASMArrayObject *wasm_array_obj_t; +typedef struct WASMFuncObject *wasm_func_obj_t; +typedef struct WASMStringrefObject *wasm_stringref_obj_t; +typedef uintptr_t wasm_i31_obj_t; + +typedef void (*wasm_obj_finalizer_t)(const wasm_obj_t obj, void *data); + +/* Defined type related operations */ + +/** + * Get number of defined types in the given wasm module + * + * @param module the wasm module + * + * @return defined type count + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_get_defined_type_count(const wasm_module_t module); + +/** + * Get defined type by type index + * + * @param module the wasm module + * @param index the type index + * + * @return defined type + */ +WASM_RUNTIME_API_EXTERN wasm_defined_type_t +wasm_get_defined_type(const wasm_module_t module, uint32_t index); + +/** + * Get defined type of the GC managed object, the object must be struct, + * array or func. + * + * @param obj the object + * + * @return defined type of the object. + */ +WASM_RUNTIME_API_EXTERN wasm_defined_type_t +wasm_obj_get_defined_type(const wasm_obj_t obj); + +/** + * Get defined type index of the GC managed object, the object must be struct, + * array or func. + * + * @param obj the object + * + * @return defined type index of the object. + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_obj_get_defined_type_idx(const wasm_module_t module, const wasm_obj_t obj); + +/** + * Check whether a defined type is a function type + * + * @param def_type the defined type to be checked + * + * @return true if the defined type is function type, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_defined_type_is_func_type(const wasm_defined_type_t def_type); + +/** + * Check whether a defined type is a struct type + * + * @param def_type the defined type to be checked + * + * @return true if the defined type is struct type, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_defined_type_is_struct_type(const wasm_defined_type_t def_type); + +/** + * Check whether a defined type is an array type + * + * @param def_type the defined type to be checked + * + * @return true if the defined type is array type, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_defined_type_is_array_type(const wasm_defined_type_t def_type); + +/** + * Get parameter count of a function type + * + * @param func_type the specified function type + * + * @return the param count of the specified function type + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_type_get_param_count(const wasm_func_type_t func_type); + +/** + * Get type of a specified parameter of a function type + * + * @param func_type the specified function type + * @param param_idx the specified param index + * + * @return the param type at the specified param index of the specified func + * type + */ +WASM_RUNTIME_API_EXTERN wasm_ref_type_t +wasm_func_type_get_param_type(const wasm_func_type_t func_type, + uint32_t param_idx); + +/** + * Get result count of a function type + * + * @param func_type the specified function type + * + * @return the result count of the specified function type + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_type_get_result_count(const wasm_func_type_t func_type); + +/** + * Get type of a specified result of a function type + * + * @param func_type the specified function type + * @param param_idx the specified result index + * + * @return the result type at the specified result index of the specified func + * type + */ +WASM_RUNTIME_API_EXTERN wasm_ref_type_t +wasm_func_type_get_result_type(const wasm_func_type_t func_type, + uint32_t result_idx); + +/** + * Get field count of a struct type + * + * @param struct_type the specified struct type + * + * @return the field count of the specified struct type + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_struct_type_get_field_count(const wasm_struct_type_t struct_type); + +/** + * Get type of a specified field of a struct type + * + * @param struct_type the specified struct type + * @param field_idx index of the specified field + * @param p_is_mutable if not NULL, output the mutability of the field + * + * @return the result type at the specified field index of the specified struct + */ +WASM_RUNTIME_API_EXTERN wasm_ref_type_t +wasm_struct_type_get_field_type(const wasm_struct_type_t struct_type, + uint32_t field_idx, bool *p_is_mutable); + +/** + * Get element type of an array type + * + * @param array_type the specified array type + * @param p_is_mutable if not NULL, output the mutability of the element type + * + * @return the ref type of array's elem type + */ +WASM_RUNTIME_API_EXTERN wasm_ref_type_t +wasm_array_type_get_elem_type(const wasm_array_type_t array_type, + bool *p_is_mutable); + +/** + * Check whether two defined types are equal + * + * @param def_type1 the specified defined type1 + * @param def_type2 the specified defined type2 + * @param module current wasm module + * + * @return true if the defined type1 is equal to the defined type2, + * false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_defined_type_equal(const wasm_defined_type_t def_type1, + const wasm_defined_type_t def_type2, + const wasm_module_t module); + +/** + * Check whether def_type1 is subtype of def_type2 + * + * @param def_type1 the specified defined type1 + * @param def_type2 the specified defined type2 + * @param module current wasm module + * + * @return true if the defined type1 is subtype of the defined type2, + * false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_defined_type_is_subtype_of(const wasm_defined_type_t def_type1, + const wasm_defined_type_t def_type2, + const wasm_module_t module); + +/* ref type related operations */ + +/** + * Set the ref_type to be (ref null? type_idx) + * + * @param ref_type the ref_type to be set + * @param nullable whether the ref_type is nullable + * @param type_idx the type index + */ +WASM_RUNTIME_API_EXTERN void +wasm_ref_type_set_type_idx(wasm_ref_type_t *ref_type, bool nullable, + int32_t type_idx); + +/** + * Set the ref_type to be (ref null? func/extern/any/eq/i31/struct/array/..) + * + * @param ref_type the ref_type to be set + * @param nullable whether the ref_type is nullable + * @param heap_type the heap type + */ +WASM_RUNTIME_API_EXTERN void +wasm_ref_type_set_heap_type(wasm_ref_type_t *ref_type, bool nullable, + int32_t heap_type); + +/** + * Check whether two ref types are equal + * + * @param ref_type1 the specified ref type1 + * @param ref_type2 the specified ref type2 + * @param module current wasm module + * + * @return true if the ref type1 is equal to the ref type2, + * false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_ref_type_equal(const wasm_ref_type_t *ref_type1, + const wasm_ref_type_t *ref_type2, + const wasm_module_t module); + +/** + * Check whether ref_type1 is subtype of ref_type2 + * + * @param ref_type1 the specified ref type1 + * @param ref_type2 the specified ref type2 + * @param module current wasm module + * + * @return true if the ref type1 is subtype of the ref type2, + * false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_ref_type_is_subtype_of(const wasm_ref_type_t *ref_type1, + const wasm_ref_type_t *ref_type2, + const wasm_module_t module); + +/* wasm object related operations */ + +/** + * Create a struct object with the index of defined type + * + * @param exec_env the execution environment + * @param type_idx index of the struct type + * + * @return wasm_struct_obj_t if create success, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_struct_obj_t +wasm_struct_obj_new_with_typeidx(wasm_exec_env_t exec_env, uint32_t type_idx); + +/** + * Create a struct object with the struct type + * + * @param exec_env the execution environment + * @param type defined struct type + * + * @return wasm_struct_obj_t if create success, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_struct_obj_t +wasm_struct_obj_new_with_type(wasm_exec_env_t exec_env, + const wasm_struct_type_t type); + +/** + * Set the field value of a struct object + * + * @param obj the struct object to set field + * @param field_idx the specified field index + * @param value wasm value to be set + */ +WASM_RUNTIME_API_EXTERN void +wasm_struct_obj_set_field(wasm_struct_obj_t obj, uint32_t field_idx, + const wasm_value_t *value); + +/** + * Get the field value of a struct object + * + * @param obj the struct object to get field + * @param field_idx the specified field index + * @param sign_extend whether to sign extend for i8 and i16 element types + * @param value output the wasm value + */ +WASM_RUNTIME_API_EXTERN void +wasm_struct_obj_get_field(const wasm_struct_obj_t obj, uint32_t field_idx, + bool sign_extend, wasm_value_t *value); + +/** + * Get the field count of the a struct object. + * + * @param obj the WASM struct object + * + * @return the field count of the a struct object + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_struct_obj_get_field_count(const wasm_struct_obj_t obj); + +/** + * Create an array object with the index of defined type, the obj's length is + * length, init value is init_value + * + * @param exec_env the execution environment + * @param type_idx the index of the specified type + * @param length the array's length + * @param init_value the array's init value + * + * @return the created array object + */ +WASM_RUNTIME_API_EXTERN wasm_array_obj_t +wasm_array_obj_new_with_typeidx(wasm_exec_env_t exec_env, uint32_t type_idx, + uint32_t length, wasm_value_t *init_value); + +/** + * Create an array object with the array type, the obj's length is length, init + * value is init_value + * + * @param exec_env the execution environment + * @param type the array's specified type + * @param length the array's length + * @param init_value the array's init value + * + * @return the created array object + */ +WASM_RUNTIME_API_EXTERN wasm_array_obj_t +wasm_array_obj_new_with_type(wasm_exec_env_t exec_env, + const wasm_array_type_t type, uint32_t length, + wasm_value_t *init_value); + +/** + * Set the specified element's value of an array object + * + * @param array_obj the array object to set element value + * @param elem_idx the specified element index + * @param value wasm value to be set + */ +WASM_RUNTIME_API_EXTERN void +wasm_array_obj_set_elem(wasm_array_obj_t array_obj, uint32_t elem_idx, + const wasm_value_t *value); + +/** + * Get the specified element's value of an array object + * + * @param array_obj the array object to get element value + * @param elem_idx the specified element index + * @param sign_extend whether to sign extend for i8 and i16 element types + * @param value output the wasm value + */ +WASM_RUNTIME_API_EXTERN void +wasm_array_obj_get_elem(const wasm_array_obj_t array_obj, uint32_t elem_idx, + bool sign_extend, wasm_value_t *value); + +/** + * Copy elements from one array to another + * + * @param dst_obj destination array object + * @param dst_idx target index in destination + * @param src_obj source array object + * @param src_idx start index in source + * @param len length of elements to copy + */ +WASM_RUNTIME_API_EXTERN void +wasm_array_obj_copy(wasm_array_obj_t dst_obj, uint32_t dst_idx, + const wasm_array_obj_t src_obj, uint32_t src_idx, + uint32_t len); + +/** + * Return the length of an array object + * + * @param array_obj the array object to get length + * + * @return length of the array object + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_array_obj_length(const wasm_array_obj_t array_obj); + +/** + * Get the address of the first element of an array object + * + * @param array_obj the array object to get element address + * + * @return address of the first element + */ +WASM_RUNTIME_API_EXTERN void * +wasm_array_obj_first_elem_addr(const wasm_array_obj_t array_obj); + +/** + * Get the address of the i-th element of an array object + * + * @param array_obj the array object to get element address + * @param elem_idx the specified element index + * + * @return address of the specified element + */ +WASM_RUNTIME_API_EXTERN void * +wasm_array_obj_elem_addr(const wasm_array_obj_t array_obj, uint32_t elem_idx); + +/** + * Create a function object with the index of defined type and the index of the + * function + * + * @param exec_env the execution environment + * @param type_idx the index of the specified type + * @param func_idx_bound the index of the function + * + * @return the created function object + */ +WASM_RUNTIME_API_EXTERN wasm_func_obj_t +wasm_func_obj_new_with_typeidx(wasm_exec_env_t exec_env, uint32_t type_idx, + uint32_t func_idx_bound); + +/** + * Create a function object with the function type and the index of the function + * + * @param exec_env the execution environment + * @param type the specified type + * @param func_idx_bound the index of the function + * + * @return the created function object + */ +WASM_RUNTIME_API_EXTERN wasm_func_obj_t +wasm_func_obj_new_with_type(wasm_exec_env_t exec_env, wasm_func_type_t type, + uint32_t func_idx_bound); + +/** + * Get the function index bound of a function object + * + * @param func_obj the function object + * + * @return the bound function index + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_obj_get_func_idx_bound(const wasm_func_obj_t func_obj); + +/** + * Get the function type of a function object + * + * @param func_obj the function object + * + * @return defined function type + */ +WASM_RUNTIME_API_EXTERN wasm_func_type_t +wasm_func_obj_get_func_type(const wasm_func_obj_t func_obj); + +/** + * Call the given WASM function object with arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param func_obj the function object to call + * @param argc total cell number that the function parameters occupy, + * a cell is a slot of the uint32 array argv[], e.g. i32/f32 argument + * occupies one cell, i64/f64 argument occupies two cells, note that + * it might be different from the parameter number of the function + * @param argv the arguments. If the function has return value, + * the first (or first two in case 64-bit return value) element of + * argv stores the return value of the called WASM function after this + * function returns. + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_func_ref(wasm_exec_env_t exec_env, + const wasm_func_obj_t func_obj, uint32_t argc, + uint32_t argv[]); + +/** + * Call the given WASM function object with provided results space + * and arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param func_obj the function object to call + * @param num_results the number of results + * @param results the pre-alloced pointer to get the results + * @param num_args the number of arguments + * @param args the arguments + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_func_ref_a(wasm_exec_env_t exec_env, + const wasm_func_obj_t func_obj, + uint32_t num_results, wasm_val_t results[], + uint32_t num_args, wasm_val_t *args); + +/** + * Call the given WASM function object with provided results space and + * variant arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param func_obj the function object to call + * @param num_results the number of results + * @param results the pre-alloced pointer to get the results + * @param num_args the number of arguments + * @param ... the variant arguments + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_func_ref_v(wasm_exec_env_t exec_env, + const wasm_func_obj_t func_obj, + uint32_t num_results, wasm_val_t results[], + uint32_t num_args, ...); + +/** + * Create an externref object with host object + * + * @param exec_env the execution environment + * @param host_obj host object pointer + * + * @return wasm_externref_obj_t if success, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_externref_obj_t +wasm_externref_obj_new(wasm_exec_env_t exec_env, const void *host_obj); + +/** + * Get the host value of an externref object + * + * @param externref_obj the externref object + * + * @return the stored host object pointer + */ +WASM_RUNTIME_API_EXTERN const void * +wasm_externref_obj_get_value(const wasm_externref_obj_t externref_obj); + +/** + * Create an anyref object with host object + * + * @param exec_env the execution environment + * @param host_obj host object pointer + * + * @return wasm_anyref_obj_t if success, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_anyref_obj_t +wasm_anyref_obj_new(wasm_exec_env_t exec_env, const void *host_obj); + +/** + * Get the host object value of an anyref object + * + * @param anyref_obj the anyref object + * + * @return the stored host object pointer + */ +WASM_RUNTIME_API_EXTERN const void * +wasm_anyref_obj_get_value(const wasm_anyref_obj_t anyref_obj); + +/** + * Get the internal object inside the externref object, same as + * the operation of opcode extern.internalize + * + * @param externref_obj the externref object + * + * @return internalized wasm_obj_t + */ +WASM_RUNTIME_API_EXTERN wasm_obj_t +wasm_externref_obj_to_internal_obj(const wasm_externref_obj_t externref_obj); + +/** + * Create an externref object from an internal object, same as + * the operation of opcode extern.externalize + * + * @param exec_env the execution environment + * @param internal_obj the internal object + * + * @return wasm_externref_obj_t if create success, NULL othersise + */ +WASM_RUNTIME_API_EXTERN wasm_externref_obj_t +wasm_internal_obj_to_externref_obj(wasm_exec_env_t exec_env, + const wasm_obj_t internal_obj); + +/** + * Create an i31 object + * + * @param i31_value the scalar value + * + * @return wasm_i31_obj_t + */ +WASM_RUNTIME_API_EXTERN wasm_i31_obj_t +wasm_i31_obj_new(uint32_t i31_value); + +/** + * Get value from an i31 object + * + * @param i31_obj the i31 object + * @param sign_extend whether to sign extend the value + * + * @return wasm_i31_obj_t + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_i31_obj_get_value(wasm_i31_obj_t i31_obj, bool sign_extend); + +/** + * Pin an object to make it traced during GC + * + * @param exec_env the execution environment + * @param obj the object to pin + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_pin_object(wasm_exec_env_t exec_env, wasm_obj_t obj); + +/** + * Unpin an object + * + * @param exec_env the execution environment + * @param obj the object to unpin + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_unpin_object(wasm_exec_env_t exec_env, wasm_obj_t obj); + +/** + * Check whether an object is a struct objectc + * + * @param obj the object to check + * + * @return true if the object is a struct, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_struct_obj(const wasm_obj_t obj); + +/** + * Check whether an object is an array object + * + * @param obj the object to check + * + * @return true if the object is a array, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_array_obj(const wasm_obj_t obj); + +/** + * Check whether an object is a function object + * + * @param obj the object to check + * + * @return true if the object is a function, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_func_obj(const wasm_obj_t obj); + +/** + * Check whether an object is an i31 object + * + * @param obj the object to check + * + * @return true if the object is an i32, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_i31_obj(const wasm_obj_t obj); + +/** + * Check whether an object is an externref object + * + * @param obj the object to check + * + * @return true if the object is an externref, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_externref_obj(const wasm_obj_t obj); + +/** + * Check whether an object is an anyref object + * + * @param obj the object to check + * + * @return true if the object is an anyref, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_anyref_obj(const wasm_obj_t obj); + +/** + * Check whether an object is a struct object, or, an i31/struct/array object + * + * @param obj the object to check + * + * @return true if the object is an internal object, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_internal_obj(const wasm_obj_t obj); + +/** + * Check whether an object is an eq object + * + * @param obj the object to check + * + * @return true if the object is an eq object, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_eq_obj(const wasm_obj_t obj); + +/** + * Check whether an object is an instance of a defined type + * + * @param obj the object to check + * @param defined_type the defined type + * @param module current wasm module + * + * @return true if the object is instance of the defined type, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_instance_of_defined_type(const wasm_obj_t obj, + const wasm_defined_type_t defined_type, + const wasm_module_t module); + +/** + * Check whether an object is an instance of a defined type with + * index type_idx + * + * @param obj the object to check + * @param type_idx the type index + * @param module current wasm module + * + * @return true if the object is instance of the defined type specified by + * type_idx, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_instance_of_type_idx(const wasm_obj_t obj, uint32_t type_idx, + const wasm_module_t module); + +/** + * Check whether an object is an instance of a ref type + * + * @param obj the object to check + * @param ref_type the ref type + * + * @return true if the object is instance of the ref type, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_obj_is_instance_of_ref_type(const wasm_obj_t obj, + const wasm_ref_type_t *ref_type); + +/** + * Push a local object ref into stack, note that we should set its value + * after pushing to retain it during GC, and should pop it from stack + * before returning from the current function + * + * @param exec_env the execution environment + * @param local_obj_ref the local object ref to push + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_push_local_obj_ref(wasm_exec_env_t exec_env, + wasm_local_obj_ref_t *local_obj_ref); + +/** + * Pop a local object ref from stack + * + * @param exec_env the execution environment + * + * @return the popped wasm_local_obj_ref_t + */ +WASM_RUNTIME_API_EXTERN wasm_local_obj_ref_t * +wasm_runtime_pop_local_obj_ref(wasm_exec_env_t exec_env); + +/** + * Pop n local object refs from stack + * + * @param exec_env the execution environment + * @param n number to pop + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_pop_local_obj_refs(wasm_exec_env_t exec_env, uint32_t n); + +/** + * Get current local object ref from stack + * + * @param exec_env the execution environment + * + * @return the wasm_local_obj_ref_t obj from the top of the stack, not change + * the state of the stack + */ +WASM_RUNTIME_API_EXTERN wasm_local_obj_ref_t * +wasm_runtime_get_cur_local_obj_ref(wasm_exec_env_t exec_env); + +/** + * Set finalizer to the given object, if another finalizer is set to the same + * object, the previous one will be cancelled + * + * @param exec_env the execution environment + * @param obj object to set finalizer + * @param cb finalizer function to be called before this object is freed + * @param data custom data to be passed to finalizer function + * + * @return true if success, false otherwise + */ +bool +wasm_obj_set_gc_finalizer(wasm_exec_env_t exec_env, const wasm_obj_t obj, + wasm_obj_finalizer_t cb, void *data); + +/** + * Unset finalizer to the given object + * + * @param exec_env the execution environment + * @param obj object to unset finalizer + */ +void +wasm_obj_unset_gc_finalizer(wasm_exec_env_t exec_env, void *obj); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _GC_EXPORT_H */ diff --git a/wasm-micro-runtime/core/iwasm/include/lib_export.h b/wasm-micro-runtime/core/iwasm/include/lib_export.h new file mode 100644 index 0000000..0ca668f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/include/lib_export.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file lib_export.h + * + */ + +#ifndef _LIB_EXPORT_H_ +#define _LIB_EXPORT_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct NativeSymbol { + const char *symbol; + void *func_ptr; + const char *signature; + /* attachment which can be retrieved in native API by + calling wasm_runtime_get_function_attachment(exec_env) */ + void *attachment; +} NativeSymbol; + +/* clang-format off */ +#define EXPORT_WASM_API(symbol) \ + { #symbol, (void *)symbol, NULL, NULL } +#define EXPORT_WASM_API2(symbol) \ + { #symbol, (void *)symbol##_wrapper, NULL, NULL } + +#define EXPORT_WASM_API_WITH_SIG(symbol, signature) \ + { #symbol, (void *)symbol, signature, NULL } +#define EXPORT_WASM_API_WITH_SIG2(symbol, signature) \ + { #symbol, (void *)symbol##_wrapper, signature, NULL } + +#define EXPORT_WASM_API_WITH_ATT(symbol, signature, attachment) \ + { #symbol, (void *)symbol, signature, attachment } +#define EXPORT_WASM_API_WITH_ATT2(symbol, signature, attachment) \ + { #symbol, (void *)symbol##_wrapper, signature, attachment } +/* clang-format on */ + +/** + * Get the exported APIs of base lib + * + * @param p_base_lib_apis return the exported API array of base lib + * + * @return the number of the exported API + */ +uint32_t +get_base_lib_export_apis(NativeSymbol **p_base_lib_apis); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _LIB_EXPORT_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/include/wasm_c_api.h b/wasm-micro-runtime/core/iwasm/include/wasm_c_api.h new file mode 100644 index 0000000..63d18f3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/include/wasm_c_api.h @@ -0,0 +1,889 @@ +// WebAssembly C API + +/** + * @file wasm_c_api.h + * + * @brief This file defines the WebAssembly C APIs + */ + +#ifndef _WASM_C_API_H_ +#define _WASM_C_API_H_ + +#include +#include +#include +#include +#include + +#ifndef WASM_API_EXTERN +#if defined(_MSC_BUILD) +#if defined(COMPILING_WASM_RUNTIME_API) +#define WASM_API_EXTERN __declspec(dllexport) +#else +#define WASM_API_EXTERN __declspec(dllimport) +#endif +#else +#define WASM_API_EXTERN +#endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define WASM_API_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define WASM_API_DEPRECATED __declspec(deprecated) +#else +#pragma message("WARNING: You need to implement DEPRECATED for this compiler") +#define WASM_API_DEPRECATED +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* clang-format off */ + +/////////////////////////////////////////////////////////////////////////////// +// Auxiliaries + +// Machine types +#if (__STDC_VERSION__) > 199901L +inline void assertions(void) { + static_assert(sizeof(float) == sizeof(uint32_t), "incompatible float type"); + static_assert(sizeof(double) == sizeof(uint64_t), "incompatible double type"); + static_assert(sizeof(intptr_t) == sizeof(uint32_t) || + sizeof(intptr_t) == sizeof(uint64_t), + "incompatible pointer type"); +} +#endif + +typedef char byte_t; +typedef float float32_t; +typedef double float64_t; + + +// Ownership + +#define own + +// The qualifier `own` is used to indicate ownership of data in this API. +// It is intended to be interpreted similar to a `const` qualifier: +// +// - `own wasm_xxx_t*` owns the pointed-to data +// - `own wasm_xxx_t` distributes to all fields of a struct or union `xxx` +// - `own wasm_xxx_vec_t` owns the vector as well as its elements(!) +// - an `own` function parameter passes ownership from caller to callee +// - an `own` function result passes ownership from callee to caller +// - an exception are `own` pointer parameters named `out`, which are copy-back +// output parameters passing back ownership from callee to caller +// +// Own data is created by `wasm_xxx_new` functions and some others. +// It must be released with the corresponding `wasm_xxx_delete` function. +// +// Deleting a reference does not necessarily delete the underlying object, +// it merely indicates that this owner no longer uses it. +// +// For vectors, `const wasm_xxx_vec_t` is used informally to indicate that +// neither the vector nor its elements should be modified. +// TODO: introduce proper `wasm_xxx_const_vec_t`? + + +#define WASM_DECLARE_OWN(name) \ + typedef struct wasm_##name##_t wasm_##name##_t; \ + \ + WASM_API_EXTERN void wasm_##name##_delete(own wasm_##name##_t*); + + +// Vectors +// size: capacity +// num_elems: current number of elements +// size_of_elem: size of one elemen +#define WASM_DECLARE_VEC(name, ptr_or_none) \ + typedef struct wasm_##name##_vec_t { \ + size_t size; \ + wasm_##name##_t ptr_or_none* data; \ + size_t num_elems; \ + size_t size_of_elem; \ + void *lock; \ + } wasm_##name##_vec_t; \ + \ + WASM_API_EXTERN void wasm_##name##_vec_new_empty(own wasm_##name##_vec_t* out); \ + WASM_API_EXTERN void wasm_##name##_vec_new_uninitialized( \ + own wasm_##name##_vec_t* out, size_t); \ + WASM_API_EXTERN void wasm_##name##_vec_new( \ + own wasm_##name##_vec_t* out, \ + size_t, own wasm_##name##_t ptr_or_none const[]); \ + WASM_API_EXTERN void wasm_##name##_vec_copy( \ + own wasm_##name##_vec_t* out, const wasm_##name##_vec_t*); \ + WASM_API_EXTERN void wasm_##name##_vec_delete(own wasm_##name##_vec_t*); + + +// Byte vectors + +typedef byte_t wasm_byte_t; +WASM_DECLARE_VEC(byte, ) + +typedef wasm_byte_vec_t wasm_name_t; + +#define wasm_name wasm_byte_vec +#define wasm_name_new wasm_byte_vec_new +#define wasm_name_new_empty wasm_byte_vec_new_empty +#define wasm_name_new_new_uninitialized wasm_byte_vec_new_uninitialized +#define wasm_name_copy wasm_byte_vec_copy +#define wasm_name_delete wasm_byte_vec_delete + +static inline void wasm_name_new_from_string( + own wasm_name_t* out, const char* s +) { + wasm_name_new(out, strlen(s), s); +} + +static inline void wasm_name_new_from_string_nt( + own wasm_name_t* out, const char* s +) { + wasm_name_new(out, strlen(s) + 1, s); +} + + +/////////////////////////////////////////////////////////////////////////////// +// Runtime Environment + +// Configuration + +WASM_DECLARE_OWN(config) + +#ifndef MEM_ALLOC_OPTION_DEFINED +#define MEM_ALLOC_OPTION_DEFINED +/* same definition from wasm_export.h */ +/* Memory allocator type */ +typedef enum { + /* pool mode, allocate memory from user defined heap buffer */ + Alloc_With_Pool = 0, + /* user allocator mode, allocate memory from user defined + malloc function */ + Alloc_With_Allocator, + /* system allocator mode, allocate memory from system allocator, + or, platform's os_malloc function */ + Alloc_With_System_Allocator, +} mem_alloc_type_t; + +/* Memory allocator option */ +typedef union MemAllocOption { + struct { + void *heap_buf; + uint32_t heap_size; + } pool; + struct { + void *malloc_func; + void *realloc_func; + void *free_func; + /* allocator user data, only used when + WASM_MEM_ALLOC_WITH_USER_DATA is defined */ + void *user_data; + } allocator; +} MemAllocOption; +#endif /* MEM_ALLOC_OPTION_DEFINED */ + +/* Runtime configration */ +struct wasm_config_t { + mem_alloc_type_t mem_alloc_type; + MemAllocOption mem_alloc_option; + uint32_t segue_flags; + bool enable_linux_perf; + /*TODO: wasi args*/ +}; + +#ifndef INSTANTIATION_ARGS_OPTION_DEFINED +#define INSTANTIATION_ARGS_OPTION_DEFINED +/* WASM module instantiation arguments */ +typedef struct InstantiationArgs { + uint32_t default_stack_size; + uint32_t host_managed_heap_size; + uint32_t max_memory_pages; +} InstantiationArgs; +#endif /* INSTANTIATION_ARGS_OPTION_DEFINED */ + +/* + * by default: + * - mem_alloc_type is Alloc_With_System_Allocator + * - mem_alloc_option is all 0 + * - enable_linux_perf is false + */ +WASM_API_EXTERN own wasm_config_t* wasm_config_new(void); + +// Embedders may provide custom functions for manipulating configs. +WASM_API_EXTERN own wasm_config_t* +wasm_config_set_mem_alloc_opt(wasm_config_t *, mem_alloc_type_t, MemAllocOption *); + +WASM_API_EXTERN own wasm_config_t* +wasm_config_set_linux_perf_opt(wasm_config_t *, bool); + +/** + * Enable using GS register as the base address of linear memory in linux x86_64, + * which may speedup the linear memory access for LLVM AOT/JIT: + * bit0 to bit4 denotes i32.load, i64.load, f32.load, f64.load, v128.load + * bit8 to bit12 denotes i32.store, i64.store, f32.store, f64.store, v128.store + * For example, 0x01 enables i32.load, 0x0100 enables i32.store. + * To enable all load/store operations, use 0x1F1F + */ +WASM_API_EXTERN wasm_config_t* +wasm_config_set_segue_flags(wasm_config_t *config, uint32_t segue_flags); + +// Engine + +WASM_DECLARE_OWN(engine) + +/** + * Create a new engine + * + * Note: for the engine new/delete operations, including this, + * wasm_engine_new_with_config, wasm_engine_new_with_args, and + * wasm_engine_delete, if the platform has mutex initializer, + * then they are thread-safe: we use a global lock to lock the + * operations of the engine. Otherwise they are not thread-safe: + * when there are engine new/delete operations happening + * simultaneously in multiple threads, developer must create + * the lock by himself, and add the lock when calling these + * functions. + */ +WASM_API_EXTERN own wasm_engine_t* wasm_engine_new(void); +WASM_API_EXTERN own wasm_engine_t* wasm_engine_new_with_config(wasm_config_t*); +WASM_API_DEPRECATED WASM_API_EXTERN own wasm_engine_t * +wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts); + +// Store + +WASM_DECLARE_OWN(store) + +WASM_API_EXTERN own wasm_store_t* wasm_store_new(wasm_engine_t*); + + +/////////////////////////////////////////////////////////////////////////////// +// Type Representations + +// Type attributes + +typedef uint8_t wasm_mutability_t; +enum wasm_mutability_enum { + WASM_CONST, + WASM_VAR, +}; + +typedef struct wasm_limits_t { + uint32_t min; + uint32_t max; +} wasm_limits_t; + +static const uint32_t wasm_limits_max_default = 0xffffffff; + + +// Generic + +#define WASM_DECLARE_TYPE(name) \ + WASM_DECLARE_OWN(name) \ + WASM_DECLARE_VEC(name, *) \ + \ + WASM_API_EXTERN own wasm_##name##_t* wasm_##name##_copy(const wasm_##name##_t*); + + +// Value Types + +WASM_DECLARE_TYPE(valtype) + +#ifndef WASM_VALKIND_T_DEFINED +#define WASM_VALKIND_T_DEFINED +typedef uint8_t wasm_valkind_t; +enum wasm_valkind_enum { + WASM_I32, + WASM_I64, + WASM_F32, + WASM_F64, + WASM_ANYREF = 128, + WASM_FUNCREF, +}; +#endif + +WASM_API_EXTERN own wasm_valtype_t* wasm_valtype_new(wasm_valkind_t); + +WASM_API_EXTERN wasm_valkind_t wasm_valtype_kind(const wasm_valtype_t*); + +static inline bool wasm_valkind_is_num(wasm_valkind_t k) { + return k < WASM_ANYREF; +} +static inline bool wasm_valkind_is_ref(wasm_valkind_t k) { + return k >= WASM_ANYREF; +} + +static inline bool wasm_valtype_is_num(const wasm_valtype_t* t) { + return wasm_valkind_is_num(wasm_valtype_kind(t)); +} +static inline bool wasm_valtype_is_ref(const wasm_valtype_t* t) { + return wasm_valkind_is_ref(wasm_valtype_kind(t)); +} + + +// Function Types + +WASM_DECLARE_TYPE(functype) + +WASM_API_EXTERN own wasm_functype_t* wasm_functype_new( + own wasm_valtype_vec_t* params, own wasm_valtype_vec_t* results); + +WASM_API_EXTERN const wasm_valtype_vec_t* wasm_functype_params(const wasm_functype_t*); +WASM_API_EXTERN const wasm_valtype_vec_t* wasm_functype_results(const wasm_functype_t*); + + +// Global Types + +WASM_DECLARE_TYPE(globaltype) + +WASM_API_EXTERN own wasm_globaltype_t* wasm_globaltype_new( + own wasm_valtype_t*, wasm_mutability_t); + +WASM_API_EXTERN const wasm_valtype_t* wasm_globaltype_content(const wasm_globaltype_t*); +WASM_API_EXTERN wasm_mutability_t wasm_globaltype_mutability(const wasm_globaltype_t*); + + +// Table Types + +WASM_DECLARE_TYPE(tabletype) + +WASM_API_EXTERN own wasm_tabletype_t* wasm_tabletype_new( + own wasm_valtype_t*, const wasm_limits_t*); + +WASM_API_EXTERN const wasm_valtype_t* wasm_tabletype_element(const wasm_tabletype_t*); +WASM_API_EXTERN const wasm_limits_t* wasm_tabletype_limits(const wasm_tabletype_t*); + + +// Memory Types + +WASM_DECLARE_TYPE(memorytype) + +WASM_API_EXTERN own wasm_memorytype_t* wasm_memorytype_new(const wasm_limits_t*); + +WASM_API_EXTERN const wasm_limits_t* wasm_memorytype_limits(const wasm_memorytype_t*); + + +// Extern Types + +WASM_DECLARE_TYPE(externtype) + +typedef uint8_t wasm_externkind_t; +enum wasm_externkind_enum { + WASM_EXTERN_FUNC, + WASM_EXTERN_GLOBAL, + WASM_EXTERN_TABLE, + WASM_EXTERN_MEMORY, +}; + +WASM_API_EXTERN wasm_externkind_t wasm_externtype_kind(const wasm_externtype_t*); + +WASM_API_EXTERN wasm_externtype_t* wasm_functype_as_externtype(wasm_functype_t*); +WASM_API_EXTERN wasm_externtype_t* wasm_globaltype_as_externtype(wasm_globaltype_t*); +WASM_API_EXTERN wasm_externtype_t* wasm_tabletype_as_externtype(wasm_tabletype_t*); +WASM_API_EXTERN wasm_externtype_t* wasm_memorytype_as_externtype(wasm_memorytype_t*); + +WASM_API_EXTERN wasm_functype_t* wasm_externtype_as_functype(wasm_externtype_t*); +WASM_API_EXTERN wasm_globaltype_t* wasm_externtype_as_globaltype(wasm_externtype_t*); +WASM_API_EXTERN wasm_tabletype_t* wasm_externtype_as_tabletype(wasm_externtype_t*); +WASM_API_EXTERN wasm_memorytype_t* wasm_externtype_as_memorytype(wasm_externtype_t*); + +WASM_API_EXTERN const wasm_externtype_t* wasm_functype_as_externtype_const(const wasm_functype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_globaltype_as_externtype_const(const wasm_globaltype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_tabletype_as_externtype_const(const wasm_tabletype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_memorytype_as_externtype_const(const wasm_memorytype_t*); + +WASM_API_EXTERN const wasm_functype_t* wasm_externtype_as_functype_const(const wasm_externtype_t*); +WASM_API_EXTERN const wasm_globaltype_t* wasm_externtype_as_globaltype_const(const wasm_externtype_t*); +WASM_API_EXTERN const wasm_tabletype_t* wasm_externtype_as_tabletype_const(const wasm_externtype_t*); +WASM_API_EXTERN const wasm_memorytype_t* wasm_externtype_as_memorytype_const(const wasm_externtype_t*); + + +// Import Types + +WASM_DECLARE_TYPE(importtype) + +WASM_API_EXTERN own wasm_importtype_t* wasm_importtype_new( + own wasm_name_t* module, own wasm_name_t* name, own wasm_externtype_t*); + +WASM_API_EXTERN const wasm_name_t* wasm_importtype_module(const wasm_importtype_t*); +WASM_API_EXTERN const wasm_name_t* wasm_importtype_name(const wasm_importtype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_importtype_type(const wasm_importtype_t*); +WASM_API_EXTERN bool wasm_importtype_is_linked(const wasm_importtype_t*); + + +// Export Types + +WASM_DECLARE_TYPE(exporttype) + +WASM_API_EXTERN own wasm_exporttype_t* wasm_exporttype_new( + own wasm_name_t*, own wasm_externtype_t*); + +WASM_API_EXTERN const wasm_name_t* wasm_exporttype_name(const wasm_exporttype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_exporttype_type(const wasm_exporttype_t*); + + +/////////////////////////////////////////////////////////////////////////////// +// Runtime Objects + +// Values + +#ifndef WASM_VAL_T_DEFINED +#define WASM_VAL_T_DEFINED +struct wasm_ref_t; + +typedef struct wasm_val_t { + wasm_valkind_t kind; + uint8_t __paddings[7]; + union { + int32_t i32; + int64_t i64; + float32_t f32; + float64_t f64; + struct wasm_ref_t* ref; + } of; +} wasm_val_t; +#endif + +WASM_API_EXTERN void wasm_val_delete(own wasm_val_t* v); +WASM_API_EXTERN void wasm_val_copy(own wasm_val_t* out, const wasm_val_t*); + +WASM_DECLARE_VEC(val, ) + + +// References + +#define WASM_DECLARE_REF_BASE(name) \ + WASM_DECLARE_OWN(name) \ + \ + WASM_API_EXTERN own wasm_##name##_t* wasm_##name##_copy(const wasm_##name##_t*); \ + WASM_API_EXTERN bool wasm_##name##_same(const wasm_##name##_t*, const wasm_##name##_t*); \ + \ + WASM_API_EXTERN void* wasm_##name##_get_host_info(const wasm_##name##_t*); \ + WASM_API_EXTERN void wasm_##name##_set_host_info(wasm_##name##_t*, void*); \ + WASM_API_EXTERN void wasm_##name##_set_host_info_with_finalizer( \ + wasm_##name##_t*, void*, void (*)(void*)); + +#define WASM_DECLARE_REF(name) \ + WASM_DECLARE_REF_BASE(name) \ + \ + WASM_API_EXTERN wasm_ref_t* wasm_##name##_as_ref(wasm_##name##_t*); \ + WASM_API_EXTERN wasm_##name##_t* wasm_ref_as_##name(wasm_ref_t*); \ + WASM_API_EXTERN const wasm_ref_t* wasm_##name##_as_ref_const(const wasm_##name##_t*); \ + WASM_API_EXTERN const wasm_##name##_t* wasm_ref_as_##name##_const(const wasm_ref_t*); + +#define WASM_DECLARE_SHARABLE_REF(name) \ + WASM_DECLARE_REF(name) \ + WASM_DECLARE_OWN(shared_##name) \ + \ + WASM_API_EXTERN own wasm_shared_##name##_t* wasm_##name##_share(const wasm_##name##_t*); \ + WASM_API_EXTERN own wasm_##name##_t* wasm_##name##_obtain(wasm_store_t*, const wasm_shared_##name##_t*); + + +WASM_DECLARE_REF_BASE(ref) + + +// Frames + +WASM_DECLARE_OWN(frame) +WASM_DECLARE_VEC(frame, *) +WASM_API_EXTERN own wasm_frame_t* wasm_frame_copy(const wasm_frame_t*); + +WASM_API_EXTERN struct wasm_instance_t* wasm_frame_instance(const wasm_frame_t*); +WASM_API_EXTERN uint32_t wasm_frame_func_index(const wasm_frame_t*); +WASM_API_EXTERN size_t wasm_frame_func_offset(const wasm_frame_t*); +WASM_API_EXTERN size_t wasm_frame_module_offset(const wasm_frame_t*); + + +// Traps + +typedef wasm_name_t wasm_message_t; // null terminated + +WASM_DECLARE_REF(trap) + +WASM_API_EXTERN own wasm_trap_t* wasm_trap_new(wasm_store_t* store, const wasm_message_t*); + +WASM_API_EXTERN void wasm_trap_message(const wasm_trap_t*, own wasm_message_t* out); +WASM_API_EXTERN own wasm_frame_t* wasm_trap_origin(const wasm_trap_t*); +WASM_API_EXTERN void wasm_trap_trace(const wasm_trap_t*, own wasm_frame_vec_t* out); + + +// Foreign Objects + +WASM_DECLARE_REF(foreign) + +WASM_API_EXTERN own wasm_foreign_t* wasm_foreign_new(wasm_store_t*); + + +// Modules +// WASM_DECLARE_SHARABLE_REF(module) + +#ifndef WASM_MODULE_T_DEFINED +#define WASM_MODULE_T_DEFINED +struct WASMModuleCommon; +typedef struct WASMModuleCommon *wasm_module_t; +#endif + +#ifndef LOAD_ARGS_OPTION_DEFINED +#define LOAD_ARGS_OPTION_DEFINED +typedef struct LoadArgs { + char *name; + /* TODO: more fields? */ +} LoadArgs; +#endif /* LOAD_ARGS_OPTION_DEFINED */ + +WASM_API_EXTERN own wasm_module_t* wasm_module_new( + wasm_store_t*, const wasm_byte_vec_t* binary); + +// please refer to wasm_runtime_load_ex(...) in core/iwasm/include/wasm_export.h +WASM_API_EXTERN own wasm_module_t* wasm_module_new_ex( + wasm_store_t*, const wasm_byte_vec_t* binary, const LoadArgs *args); + +WASM_API_EXTERN void wasm_module_delete(own wasm_module_t*); + +WASM_API_EXTERN bool wasm_module_validate(wasm_store_t*, const wasm_byte_vec_t* binary); + +WASM_API_EXTERN void wasm_module_imports(const wasm_module_t*, own wasm_importtype_vec_t* out); +WASM_API_EXTERN void wasm_module_exports(const wasm_module_t*, own wasm_exporttype_vec_t* out); + +WASM_API_EXTERN void wasm_module_serialize(wasm_module_t*, own wasm_byte_vec_t* out); +WASM_API_EXTERN own wasm_module_t* wasm_module_deserialize(wasm_store_t*, const wasm_byte_vec_t*); + +typedef wasm_module_t wasm_shared_module_t; +WASM_API_EXTERN own wasm_shared_module_t* wasm_module_share(wasm_module_t*); +WASM_API_EXTERN own wasm_module_t* wasm_module_obtain(wasm_store_t*, wasm_shared_module_t*); +WASM_API_EXTERN void wasm_shared_module_delete(own wasm_shared_module_t*); + +WASM_API_EXTERN bool wasm_module_set_name(wasm_module_t*, const char* name); +WASM_API_EXTERN const char *wasm_module_get_name(wasm_module_t*); + + +// Function Instances + +WASM_DECLARE_REF(func) + +typedef own wasm_trap_t* (*wasm_func_callback_t)( + const wasm_val_vec_t* args, own wasm_val_vec_t *results); +typedef own wasm_trap_t* (*wasm_func_callback_with_env_t)( + void* env, const wasm_val_vec_t *args, wasm_val_vec_t *results); + +WASM_API_EXTERN own wasm_func_t* wasm_func_new( + wasm_store_t*, const wasm_functype_t*, wasm_func_callback_t); +WASM_API_EXTERN own wasm_func_t* wasm_func_new_with_env( + wasm_store_t*, const wasm_functype_t* type, wasm_func_callback_with_env_t, + void* env, void (*finalizer)(void*)); + +WASM_API_EXTERN own wasm_functype_t* wasm_func_type(const wasm_func_t*); +WASM_API_EXTERN size_t wasm_func_param_arity(const wasm_func_t*); +WASM_API_EXTERN size_t wasm_func_result_arity(const wasm_func_t*); + +WASM_API_EXTERN own wasm_trap_t* wasm_func_call( + const wasm_func_t*, const wasm_val_vec_t* args, wasm_val_vec_t* results); + + +// Global Instances + +WASM_DECLARE_REF(global) + +WASM_API_EXTERN own wasm_global_t* wasm_global_new( + wasm_store_t*, const wasm_globaltype_t*, const wasm_val_t*); + +WASM_API_EXTERN own wasm_globaltype_t* wasm_global_type(const wasm_global_t*); + +WASM_API_EXTERN void wasm_global_get(const wasm_global_t*, own wasm_val_t* out); +WASM_API_EXTERN void wasm_global_set(wasm_global_t*, const wasm_val_t*); + + +// Table Instances + +WASM_DECLARE_REF(table) + +typedef uint32_t wasm_table_size_t; + +WASM_API_EXTERN own wasm_table_t* wasm_table_new( + wasm_store_t*, const wasm_tabletype_t*, wasm_ref_t* init); + +WASM_API_EXTERN own wasm_tabletype_t* wasm_table_type(const wasm_table_t*); + +WASM_API_EXTERN own wasm_ref_t* wasm_table_get(const wasm_table_t*, wasm_table_size_t index); +WASM_API_EXTERN bool wasm_table_set(wasm_table_t*, wasm_table_size_t index, wasm_ref_t*); + +WASM_API_EXTERN wasm_table_size_t wasm_table_size(const wasm_table_t*); +WASM_API_EXTERN bool wasm_table_grow(wasm_table_t*, wasm_table_size_t delta, wasm_ref_t* init); + + +// Memory Instances + +WASM_DECLARE_REF(memory) + +typedef uint32_t wasm_memory_pages_t; + +static const size_t MEMORY_PAGE_SIZE = 0x10000; + +WASM_API_EXTERN own wasm_memory_t* wasm_memory_new(wasm_store_t*, const wasm_memorytype_t*); + +WASM_API_EXTERN own wasm_memorytype_t* wasm_memory_type(const wasm_memory_t*); + +WASM_API_EXTERN byte_t* wasm_memory_data(wasm_memory_t*); +WASM_API_EXTERN size_t wasm_memory_data_size(const wasm_memory_t*); + +WASM_API_EXTERN wasm_memory_pages_t wasm_memory_size(const wasm_memory_t*); +WASM_API_EXTERN bool wasm_memory_grow(wasm_memory_t*, wasm_memory_pages_t delta); + + +// Externals + +WASM_DECLARE_REF(extern) +WASM_DECLARE_VEC(extern, *) + +WASM_API_EXTERN wasm_externkind_t wasm_extern_kind(const wasm_extern_t*); +WASM_API_EXTERN own wasm_externtype_t* wasm_extern_type(const wasm_extern_t*); + +WASM_API_EXTERN wasm_extern_t* wasm_func_as_extern(wasm_func_t*); +WASM_API_EXTERN wasm_extern_t* wasm_global_as_extern(wasm_global_t*); +WASM_API_EXTERN wasm_extern_t* wasm_table_as_extern(wasm_table_t*); +WASM_API_EXTERN wasm_extern_t* wasm_memory_as_extern(wasm_memory_t*); + +WASM_API_EXTERN wasm_func_t* wasm_extern_as_func(wasm_extern_t*); +WASM_API_EXTERN wasm_global_t* wasm_extern_as_global(wasm_extern_t*); +WASM_API_EXTERN wasm_table_t* wasm_extern_as_table(wasm_extern_t*); +WASM_API_EXTERN wasm_memory_t* wasm_extern_as_memory(wasm_extern_t*); + +WASM_API_EXTERN const wasm_extern_t* wasm_func_as_extern_const(const wasm_func_t*); +WASM_API_EXTERN const wasm_extern_t* wasm_global_as_extern_const(const wasm_global_t*); +WASM_API_EXTERN const wasm_extern_t* wasm_table_as_extern_const(const wasm_table_t*); +WASM_API_EXTERN const wasm_extern_t* wasm_memory_as_extern_const(const wasm_memory_t*); + +WASM_API_EXTERN const wasm_func_t* wasm_extern_as_func_const(const wasm_extern_t*); +WASM_API_EXTERN const wasm_global_t* wasm_extern_as_global_const(const wasm_extern_t*); +WASM_API_EXTERN const wasm_table_t* wasm_extern_as_table_const(const wasm_extern_t*); +WASM_API_EXTERN const wasm_memory_t* wasm_extern_as_memory_const(const wasm_extern_t*); + + +// Module Instances + +WASM_DECLARE_REF(instance) + +WASM_API_EXTERN own wasm_instance_t* wasm_instance_new( + wasm_store_t*, const wasm_module_t*, const wasm_extern_vec_t *imports, + own wasm_trap_t** trap +); + +// please refer to wasm_runtime_instantiate(...) in core/iwasm/include/wasm_export.h +WASM_API_EXTERN own wasm_instance_t* wasm_instance_new_with_args( + wasm_store_t*, const wasm_module_t*, const wasm_extern_vec_t *imports, + own wasm_trap_t** trap, const uint32_t stack_size, const uint32_t heap_size +); + +// please refer to wasm_runtime_instantiate_ex(...) in core/iwasm/include/wasm_export.h +WASM_API_EXTERN own wasm_instance_t* wasm_instance_new_with_args_ex( + wasm_store_t*, const wasm_module_t*, const wasm_extern_vec_t *imports, + own wasm_trap_t** trap, const InstantiationArgs *inst_args +); + +WASM_API_EXTERN void wasm_instance_exports(const wasm_instance_t*, own wasm_extern_vec_t* out); + + +/////////////////////////////////////////////////////////////////////////////// +// Convenience + +// Vectors + +#define WASM_EMPTY_VEC {0, NULL, 0, 0, NULL} +#define WASM_ARRAY_VEC(array) {sizeof(array)/sizeof(*(array)), array, sizeof(array)/sizeof(*(array)), sizeof(*(array)), NULL} + + +// Value Type construction short-hands + +static inline own wasm_valtype_t* wasm_valtype_new_i32(void) { + return wasm_valtype_new(WASM_I32); +} +static inline own wasm_valtype_t* wasm_valtype_new_i64(void) { + return wasm_valtype_new(WASM_I64); +} +static inline own wasm_valtype_t* wasm_valtype_new_f32(void) { + return wasm_valtype_new(WASM_F32); +} +static inline own wasm_valtype_t* wasm_valtype_new_f64(void) { + return wasm_valtype_new(WASM_F64); +} + +static inline own wasm_valtype_t* wasm_valtype_new_anyref(void) { + return wasm_valtype_new(WASM_ANYREF); +} +static inline own wasm_valtype_t* wasm_valtype_new_funcref(void) { + return wasm_valtype_new(WASM_FUNCREF); +} + + +// Function Types construction short-hands + +static inline own wasm_functype_t* wasm_functype_new_0_0(void) { + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new_empty(¶ms); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_1_0( + own wasm_valtype_t* p +) { + wasm_valtype_t* ps[1] = {p}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 1, ps); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_2_0( + own wasm_valtype_t* p1, own wasm_valtype_t* p2 +) { + wasm_valtype_t* ps[2] = {p1, p2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 2, ps); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_3_0( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* p3 +) { + wasm_valtype_t* ps[3] = {p1, p2, p3}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 3, ps); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_0_1( + own wasm_valtype_t* r +) { + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new_empty(¶ms); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_1_1( + own wasm_valtype_t* p, own wasm_valtype_t* r +) { + wasm_valtype_t* ps[1] = {p}; + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 1, ps); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_2_1( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* r +) { + wasm_valtype_t* ps[2] = {p1, p2}; + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 2, ps); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_3_1( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* p3, + own wasm_valtype_t* r +) { + wasm_valtype_t* ps[3] = {p1, p2, p3}; + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 3, ps); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_0_2( + own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new_empty(¶ms); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_1_2( + own wasm_valtype_t* p, own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* ps[1] = {p}; + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 1, ps); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_2_2( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, + own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* ps[2] = {p1, p2}; + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 2, ps); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_3_2( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* p3, + own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* ps[3] = {p1, p2, p3}; + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 3, ps); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + + +// Value construction short-hands + +static inline void wasm_val_init_ptr(own wasm_val_t* out, void* p) { +#if UINTPTR_MAX == UINT32_MAX + out->kind = WASM_I32; + out->of.i32 = (intptr_t)p; +#elif UINTPTR_MAX == UINT64_MAX + out->kind = WASM_I64; + out->of.i64 = (intptr_t)p; +#endif +} + +static inline void* wasm_val_ptr(const wasm_val_t* val) { +#if UINTPTR_MAX == UINT32_MAX + return (void*)(intptr_t)val->of.i32; +#elif UINTPTR_MAX == UINT64_MAX + return (void*)(intptr_t)val->of.i64; +#endif +} + +#define WASM_I32_VAL(i) {.kind = WASM_I32, .__paddings = {0}, .of = {.i32 = i}} +#define WASM_I64_VAL(i) {.kind = WASM_I64, .__paddings = {0}, .of = {.i64 = i}} +#define WASM_F32_VAL(z) {.kind = WASM_F32, .__paddings = {0}, .of = {.f32 = z}} +#define WASM_F64_VAL(z) {.kind = WASM_F64, .__paddings = {0}, .of = {.f64 = z}} +#define WASM_REF_VAL(r) {.kind = WASM_ANYREF, .__paddings = {0}, .of = {.ref = r}} +#define WASM_INIT_VAL {.kind = WASM_ANYREF, .__paddings = {0}, .of = {.ref = NULL}} + +#define KILOBYTE(n) ((n) * 1024) + +// Create placeholders filled in `wasm_externvec_t* imports` for `wasm_instance_new()` +WASM_API_EXTERN wasm_extern_t *wasm_extern_new_empty(wasm_store_t *, wasm_externkind_t); + +/////////////////////////////////////////////////////////////////////////////// + +#undef own + +/* clang-format on */ + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // #ifdef _WASM_C_API_H_ diff --git a/wasm-micro-runtime/core/iwasm/include/wasm_export.h b/wasm-micro-runtime/core/iwasm/include/wasm_export.h new file mode 100644 index 0000000..104a9e8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/include/wasm_export.h @@ -0,0 +1,1763 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file wasm_export.h + * + * @brief This file defines the exported common runtime APIs + */ + +#ifndef _WASM_EXPORT_H +#define _WASM_EXPORT_H + +#include +#include +#include "lib_export.h" + +#ifndef WASM_RUNTIME_API_EXTERN +#if defined(_MSC_BUILD) +#if defined(COMPILING_WASM_RUNTIME_API) +#define WASM_RUNTIME_API_EXTERN __declspec(dllexport) +#else +#define WASM_RUNTIME_API_EXTERN __declspec(dllimport) +#endif +#else +#define WASM_RUNTIME_API_EXTERN +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define get_module_inst(exec_env) wasm_runtime_get_module_inst(exec_env) + +#define validate_app_addr(offset, size) \ + wasm_runtime_validate_app_addr(module_inst, offset, size) + +#define validate_app_str_addr(offset) \ + wasm_runtime_validate_app_str_addr(module_inst, offset) + +#define addr_app_to_native(offset) \ + wasm_runtime_addr_app_to_native(module_inst, offset) + +#define addr_native_to_app(ptr) \ + wasm_runtime_addr_native_to_app(module_inst, ptr) + +#define module_malloc(size, p_native_addr) \ + wasm_runtime_module_malloc(module_inst, size, p_native_addr) + +#define module_free(offset) wasm_runtime_module_free(module_inst, offset) + +#define native_raw_return_type(type, args) type *raw_ret = (type *)(args) + +#define native_raw_get_arg(type, name, args) type name = *((type *)(args++)) + +#define native_raw_set_return(val) *raw_ret = (val) + +#ifndef WASM_MODULE_T_DEFINED +#define WASM_MODULE_T_DEFINED +/* Uninstantiated WASM module loaded from WASM binary file + or AoT binary file*/ +struct WASMModuleCommon; +typedef struct WASMModuleCommon *wasm_module_t; +#endif + +typedef enum { + WASM_IMPORT_EXPORT_KIND_FUNC, + WASM_IMPORT_EXPORT_KIND_TABLE, + WASM_IMPORT_EXPORT_KIND_MEMORY, + WASM_IMPORT_EXPORT_KIND_GLOBAL +} wasm_import_export_kind_t; + +typedef struct wasm_import_type { + const char *module_name; + const char *name; + wasm_import_export_kind_t kind; + bool linked; +} wasm_import_type; + +typedef struct wasm_export_type { + const char *name; + wasm_import_export_kind_t kind; +} wasm_export_type; + +/* Instantiated WASM module */ +struct WASMModuleInstanceCommon; +typedef struct WASMModuleInstanceCommon *wasm_module_inst_t; + +/* Function instance */ +typedef void WASMFunctionInstanceCommon; +typedef WASMFunctionInstanceCommon *wasm_function_inst_t; + +/* WASM section */ +typedef struct wasm_section_t { + struct wasm_section_t *next; + /* section type */ + int section_type; + /* section body, not include type and size */ + uint8_t *section_body; + /* section body size */ + uint32_t section_body_size; +} wasm_section_t, aot_section_t, *wasm_section_list_t, *aot_section_list_t; + +/* Execution environment, e.g. stack info */ +struct WASMExecEnv; +typedef struct WASMExecEnv *wasm_exec_env_t; + +/* Package Type */ +typedef enum { + Wasm_Module_Bytecode = 0, + Wasm_Module_AoT, + Package_Type_Unknown = 0xFFFF +} package_type_t; + +#ifndef MEM_ALLOC_OPTION_DEFINED +#define MEM_ALLOC_OPTION_DEFINED +/* Memory allocator type */ +typedef enum { + /* pool mode, allocate memory from user defined heap buffer */ + Alloc_With_Pool = 0, + /* user allocator mode, allocate memory from user defined + malloc function */ + Alloc_With_Allocator, + /* system allocator mode, allocate memory from system allocator, + or, platform's os_malloc function */ + Alloc_With_System_Allocator, +} mem_alloc_type_t; + +typedef enum { Alloc_For_Runtime, Alloc_For_LinearMemory } mem_alloc_usage_t; + +/* Memory allocator option */ +typedef union MemAllocOption { + struct { + void *heap_buf; + uint32_t heap_size; + } pool; + struct { + /* the function signature is varied when + WASM_MEM_ALLOC_WITH_USER_DATA and + WASM_MEM_ALLOC_WITH_USAGE are defined */ + void *malloc_func; + void *realloc_func; + void *free_func; + /* allocator user data, only used when + WASM_MEM_ALLOC_WITH_USER_DATA is defined */ + void *user_data; + } allocator; +} MemAllocOption; +#endif + +/* Memory pool info */ +typedef struct mem_alloc_info_t { + uint32_t total_size; + uint32_t total_free_size; + uint32_t highmark_size; +} mem_alloc_info_t; + +/* Running mode of runtime and module instance*/ +typedef enum RunningMode { + Mode_Interp = 1, + Mode_Fast_JIT, + Mode_LLVM_JIT, + Mode_Multi_Tier_JIT, +} RunningMode; + +/* WASM runtime initialize arguments */ +typedef struct RuntimeInitArgs { + mem_alloc_type_t mem_alloc_type; + MemAllocOption mem_alloc_option; + + const char *native_module_name; + NativeSymbol *native_symbols; + uint32_t n_native_symbols; + + /* maximum thread number, only used when + WASM_ENABLE_THREAD_MGR is defined */ + uint32_t max_thread_num; + + /* Debug settings, only used when + WASM_ENABLE_DEBUG_INTERP != 0 */ + char ip_addr[128]; + int unused; /* was platform_port */ + int instance_port; + + /* Fast JIT code cache size */ + uint32_t fast_jit_code_cache_size; + + /* Default GC heap size */ + uint32_t gc_heap_size; + + /* Default running mode of the runtime */ + RunningMode running_mode; + + /* LLVM JIT opt and size level */ + uint32_t llvm_jit_opt_level; + uint32_t llvm_jit_size_level; + /* Segue optimization flags for LLVM JIT */ + uint32_t segue_flags; + /** + * If enabled + * - llvm-jit will output a jitdump file for `perf inject` + * - aot will output a perf-${pid}.map for `perf record` + * - fast-jit. TBD + * - multi-tier-jit. TBD + * - interpreter. TBD + */ + bool enable_linux_perf; +} RuntimeInitArgs; + +#ifndef LOAD_ARGS_OPTION_DEFINED +#define LOAD_ARGS_OPTION_DEFINED +typedef struct LoadArgs { + char *name; + /* TODO: more fields? */ +} LoadArgs; +#endif /* LOAD_ARGS_OPTION_DEFINED */ + +#ifndef INSTANTIATION_ARGS_OPTION_DEFINED +#define INSTANTIATION_ARGS_OPTION_DEFINED +/* WASM module instantiation arguments */ +typedef struct InstantiationArgs { + uint32_t default_stack_size; + uint32_t host_managed_heap_size; + uint32_t max_memory_pages; +} InstantiationArgs; +#endif /* INSTANTIATION_ARGS_OPTION_DEFINED */ + +#ifndef WASM_VALKIND_T_DEFINED +#define WASM_VALKIND_T_DEFINED +typedef uint8_t wasm_valkind_t; +enum wasm_valkind_enum { + WASM_I32, + WASM_I64, + WASM_F32, + WASM_F64, + WASM_ANYREF = 128, + WASM_FUNCREF, +}; +#endif + +#ifndef WASM_VAL_T_DEFINED +#define WASM_VAL_T_DEFINED +struct wasm_ref_t; + +typedef struct wasm_val_t { + wasm_valkind_t kind; + uint8_t __paddings[7]; + union { + /* also represent a function index */ + int32_t i32; + int64_t i64; + float f32; + double f64; + /* represent a foreign object, aka externref in .wat */ + uintptr_t foreign; + struct wasm_ref_t *ref; + } of; +} wasm_val_t; +#endif + +typedef enum { + WASM_LOG_LEVEL_FATAL = 0, + WASM_LOG_LEVEL_ERROR = 1, + WASM_LOG_LEVEL_WARNING = 2, + WASM_LOG_LEVEL_DEBUG = 3, + WASM_LOG_LEVEL_VERBOSE = 4 +} log_level_t; + +/** + * Initialize the WASM runtime environment, and also initialize + * the memory allocator with system allocator, which calls os_malloc + * to allocate memory + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_init(void); + +/** + * Initialize the WASM runtime environment, WASM running mode, + * and also initialize the memory allocator and register native symbols, + * which are specified with init arguments + * + * @param init_args specifies the init arguments + * + * @return return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_full_init(RuntimeInitArgs *init_args); + +/** + * Set the log level. To be called after the runtime is initialized. + * + * @param level the log level to set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_log_level(log_level_t level); + +/** + * Query whether a certain running mode is supported for the runtime + * + * @param running_mode the running mode to query + * + * @return true if this running mode is supported, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_running_mode_supported(RunningMode running_mode); + +/** + * Set the default running mode for the runtime. It is inherited + * to set the running mode of a module instance when it is instantiated, + * and can be changed by calling wasm_runtime_set_running_mode + * + * @param running_mode the running mode to set + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_default_running_mode(RunningMode running_mode); + +/** + * Destroy the WASM runtime environment. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy(void); + +/** + * Allocate memory from runtime memory environment. + * + * @param size bytes need to allocate + * + * @return the pointer to memory allocated + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_malloc(unsigned int size); + +/** + * Reallocate memory from runtime memory environment + * + * @param ptr the original memory + * @param size bytes need to reallocate + * + * @return the pointer to memory reallocated + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_realloc(void *ptr, unsigned int size); + +/* + * Free memory to runtime memory environment. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_free(void *ptr); + +/* + * Get memory info, only pool mode is supported now. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info); + +/** + * Get the package type of a buffer. + * + * @param buf the package buffer + * @param size the package buffer size + * + * @return the package type, return Package_Type_Unknown if the type is unknown + */ +WASM_RUNTIME_API_EXTERN package_type_t +get_package_type(const uint8_t *buf, uint32_t size); + +/** + * Check whether a file is an AOT XIP (Execution In Place) file + * + * @param buf the package buffer + * @param size the package buffer size + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_xip_file(const uint8_t *buf, uint32_t size); + +/** + * Callback to load a module file into a buffer in multi-module feature + */ +typedef bool (*module_reader)(package_type_t module_type, + const char *module_name, uint8_t **p_buffer, + uint32_t *p_size); + +/** + * Callback to release the buffer loaded by module_reader callback + */ +typedef void (*module_destroyer)(uint8_t *buffer, uint32_t size); + +/** + * Setup callbacks for reading and releasing a buffer about a module file + * + * @param reader a callback to read a module file into a buffer + * @param destroyer a callback to release above buffer + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_reader(const module_reader reader, + const module_destroyer destroyer); +/** + * Give the "module" a name "module_name". + * Can not assign a new name to a module if it already has a name + * + * @param module_name indicate a name + * @param module the target module + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return true means success, false means failed + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_module(const char *module_name, wasm_module_t module, + char *error_buf, uint32_t error_buf_size); + +/** + * Check if there is already a loaded module named module_name in the + * runtime. Repeately loading a module with the same name is not allowed. + * + * @param module_name indicate a name + * + * @return return WASM module loaded, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_find_module_registered(const char *module_name); + +/** + * Load a WASM module from a specified byte buffer. The byte buffer can be + * WASM binary data when interpreter or JIT is enabled, or AOT binary data + * when AOT is enabled. If it is AOT binary data, it must be 4-byte aligned. + * + * Note: In case of AOT XIP modules, the runtime doesn't make modifications + * to the buffer. (Except the "Known issues" mentioned in doc/xip.md.) + * Otherwise, the runtime can make modifications to the buffer for its + * internal purposes. Thus, in general, it isn't safe to create multiple + * modules from a single buffer. + * + * @param buf the byte buffer which contains the WASM/AOT binary data, + * note that the byte buffer must be writable since runtime may + * change its content for footprint and performance purpose, and + * it must be referencable until wasm_runtime_unload is called + * @param size the size of the buffer + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return WASM module loaded, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_load(uint8_t *buf, uint32_t size, char *error_buf, + uint32_t error_buf_size); + +/** + * Load a WASM module with specified load argument. + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_load_ex(uint8_t *buf, uint32_t size, const LoadArgs *args, + char *error_buf, uint32_t error_buf_size); + +/** + * Load a WASM module from a specified WASM or AOT section list. + * + * @param section_list the section list which contains each section data + * @param is_aot whether the section list is AOT section list + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return WASM module loaded, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_load_from_sections(wasm_section_list_t section_list, bool is_aot, + char *error_buf, uint32_t error_buf_size); + +/** + * Unload a WASM module. + * + * @param module the module to be unloaded + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_unload(wasm_module_t module); + +/** + * Get the module hash of a WASM module, currently only available on + * linux-sgx platform when the remote attestation feature is enabled + * + * @param module the WASM module to retrieve + * + * @return the module hash of the WASM module + */ +char * +wasm_runtime_get_module_hash(wasm_module_t module); + +/** + * Set WASI parameters. + * + * While this API operates on a module, these parameters will be used + * only when the module is instantiated. That is, you can consider these + * as extra parameters for wasm_runtime_instantiate(). + * + * @param module The module to set WASI parameters. + * @param dir_list The list of directories to preopen. (real path) + * @param dir_count The number of elements in dir_list. + * @param map_dir_list The list of directories to preopen. (mapped path) + * Format for each map entry: :: + * @param map_dir_count The number of elements in map_dir_list. + * If map_dir_count is smaller than dir_count, + * mapped path is assumed to be same as the + * corresponding real path for the rest of entries. + * @param env The list of environment variables. + * @param env_count The number of elements in env. + * @param argv The list of command line arguments. + * @param argc The number of elements in argv. + * @param stdin_handle The raw host handle to back WASI STDIN_FILENO. + * If an invalid handle is specified (e.g. -1 on POSIX, + * INVALID_HANDLE_VALUE on Windows), the platform default + * for STDIN is used. + * @param stdoutfd The raw host handle to back WASI STDOUT_FILENO. + * If an invalid handle is specified (e.g. -1 on POSIX, + * INVALID_HANDLE_VALUE on Windows), the platform default + * for STDOUT is used. + * @param stderrfd The raw host handle to back WASI STDERR_FILENO. + * If an invalid handle is specified (e.g. -1 on POSIX, + * INVALID_HANDLE_VALUE on Windows), the platform default + * for STDERR is used. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args_ex(wasm_module_t module, const char *dir_list[], + uint32_t dir_count, const char *map_dir_list[], + uint32_t map_dir_count, const char *env[], + uint32_t env_count, char *argv[], int argc, + int64_t stdinfd, int64_t stdoutfd, + int64_t stderrfd); + +/** + * Set WASI parameters. + * + * Same as wasm_runtime_set_wasi_args_ex but with default stdio handles + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args(wasm_module_t module, const char *dir_list[], + uint32_t dir_count, const char *map_dir_list[], + uint32_t map_dir_count, const char *env[], + uint32_t env_count, char *argv[], int argc); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_addr_pool(wasm_module_t module, const char *addr_pool[], + uint32_t addr_pool_size); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_ns_lookup_pool(wasm_module_t module, + const char *ns_lookup_pool[], + uint32_t ns_lookup_pool_size); + +/** + * Instantiate a WASM module. + * + * @param module the WASM module to instantiate + * @param default_stack_size the default stack size of the module instance when + * the exec env's operation stack isn't created by user, e.g. API + * wasm_application_execute_main() and wasm_application_execute_func() + * create the operation stack internally with the stack size specified + * here. And API wasm_runtime_create_exec_env() creates the operation + * stack with stack size specified by its parameter, the stack size + * specified here is ignored. + * @param host_managed_heap_size the default heap size of the module instance, + * a heap will be created besides the app memory space. Both wasm app + * and native function can allocate memory from the heap. + * @param error_buf buffer to output the error info if failed + * @param error_buf_size the size of the error buffer + * + * @return return the instantiated WASM module instance, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_instantiate(const wasm_module_t module, + uint32_t default_stack_size, + uint32_t host_managed_heap_size, char *error_buf, + uint32_t error_buf_size); + +/** + * Instantiate a WASM module, with specified instantiation arguments + * + * Same as wasm_runtime_instantiate, but it also allows overwriting maximum + * memory + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_instantiate_ex(const wasm_module_t module, + const InstantiationArgs *args, char *error_buf, + uint32_t error_buf_size); + +/** + * Set the running mode of a WASM module instance, override the + * default running mode of the runtime. Note that it only makes sense when + * the input is a wasm bytecode file: for the AOT file, runtime always runs + * it with AOT engine, and this function always returns true. + * + * @param module_inst the WASM module instance to set running mode + * @param running_mode the running mode to set + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_running_mode(wasm_module_inst_t module_inst, + RunningMode running_mode); + +/** + * Get the running mode of a WASM module instance, if no running mode + * is explicitly set the default running mode of runtime will + * be used and returned. Note that it only makes sense when the input is a + * wasm bytecode file: for the AOT file, this function always returns 0. + * + * @param module_inst the WASM module instance to query for running mode + * + * @return the running mode this module instance currently use + */ +WASM_RUNTIME_API_EXTERN RunningMode +wasm_runtime_get_running_mode(wasm_module_inst_t module_inst); + +/** + * Deinstantiate a WASM module instance, destroy the resources. + * + * @param module_inst the WASM module instance to destroy + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_deinstantiate(wasm_module_inst_t module_inst); + +/** + * Get WASM module from WASM module instance + * + * @param module_inst the WASM module instance to retrieve + * + * @return the WASM module + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_get_module(wasm_module_inst_t module_inst); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_wasi_mode(wasm_module_inst_t module_inst); + +WASM_RUNTIME_API_EXTERN wasm_function_inst_t +wasm_runtime_lookup_wasi_start_function(wasm_module_inst_t module_inst); + +/** + * Get WASI exit code. + * + * After a WASI command completed its execution, an embedder can + * call this function to get its exit code. (that is, the value given + * to proc_exit.) + * + * @param module_inst the module instance + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_wasi_exit_code(wasm_module_inst_t module_inst); + +/** + * Lookup an exported function in the WASM module instance. + * + * @param module_inst the module instance + * @param name the name of the function + * + * @return the function instance found, NULL if not found + */ +WASM_RUNTIME_API_EXTERN wasm_function_inst_t +wasm_runtime_lookup_function(wasm_module_inst_t const module_inst, + const char *name); + +/** + * Get parameter count of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * + * @return the parameter count of the function instance + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_get_param_count(wasm_function_inst_t const func_inst, + wasm_module_inst_t const module_inst); + +/** + * Get result count of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * + * @return the result count of the function instance + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_get_result_count(wasm_function_inst_t const func_inst, + wasm_module_inst_t const module_inst); + +/** + * Get parameter types of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * @param param_types the parameter types returned + */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_param_types(wasm_function_inst_t const func_inst, + wasm_module_inst_t const module_inst, + wasm_valkind_t *param_types); + +/** + * Get result types of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * @param result_types the result types returned + */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_result_types(wasm_function_inst_t const func_inst, + wasm_module_inst_t const module_inst, + wasm_valkind_t *result_types); + +/** + * Create execution environment for a WASM module instance. + * + * @param module_inst the module instance + * @param stack_size the stack size to execute a WASM function + * + * @return the execution environment, NULL if failed, e.g. invalid + * stack size is passed + */ +WASM_RUNTIME_API_EXTERN wasm_exec_env_t +wasm_runtime_create_exec_env(wasm_module_inst_t module_inst, + uint32_t stack_size); + +/** + * Destroy the execution environment. + * + * @param exec_env the execution environment to destroy + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_exec_env(wasm_exec_env_t exec_env); + +/** + * Get the singleton execution environment for the instance. + * + * Note: The singleton execution environment is the execution + * environment used internally by the runtime for the API functions + * like wasm_application_execute_main, which don't take explicit + * execution environment. It's associated to the corresponding + * module instance and managed by the runtime. The API user should + * not destroy it with wasm_runtime_destroy_exec_env. + * + * @param module_inst the module instance + * + * @return exec_env the execution environment to destroy + */ +WASM_RUNTIME_API_EXTERN wasm_exec_env_t +wasm_runtime_get_exec_env_singleton(wasm_module_inst_t module_inst); + +/** + * Start debug instance based on given execution environment. + * Note: + * The debug instance will be destroyed during destroying the + * execution environment, developers don't need to destroy it + * manually. + * If the cluster of this execution environment has already + * been bound to a debug instance, this function will return true + * directly. + * If developer spawns some exec_env by wasm_runtime_spawn_exec_env, + * don't need to call this function for every spawned exec_env as + * they are sharing the same cluster with the main exec_env. + * + * @param exec_env the execution environment to start debug instance + * @param port the port for the debug server to listen on. + * 0 means automatic assignment. + * -1 means to use the global setting in RuntimeInitArgs. + * + * @return debug port if success, 0 otherwise. + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_start_debug_instance_with_port(wasm_exec_env_t exec_env, + int32_t port); + +/** + * Same as wasm_runtime_start_debug_instance_with_port(env, -1). + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_start_debug_instance(wasm_exec_env_t exec_env); + +/** + * Initialize the thread environment. + * Note: + * If developer creates a child thread by himself to call the + * the wasm function in that thread, he should call this API + * firstly before calling the wasm function and then call + * wasm_runtime_destroy_thread_env() after calling the wasm + * function. If the thread is created from the runtime API, + * it is unnecessary to call these two APIs. + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_init_thread_env(void); + +/** + * Destroy the thread environment + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_thread_env(void); + +/** + * Whether the thread environment is initialized + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_thread_env_inited(void); + +/** + * Get WASM module instance from execution environment + * + * @param exec_env the execution environment to retrieve + * + * @return the WASM module instance + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_get_module_inst(wasm_exec_env_t exec_env); + +/** + * Set WASM module instance of execution environment + * Caution: + * normally the module instance is bound with the execution + * environment one by one, if multiple module instances want + * to share to the same execution environment, developer should + * be responsible for the backup and restore of module instance + * + * @param exec_env the execution environment + * @param module_inst the WASM module instance to set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_inst(wasm_exec_env_t exec_env, + const wasm_module_inst_t module_inst); + +/** + * Call the given WASM function of a WASM module instance with + * arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param function the function to call + * @param argc total cell number that the function parameters occupy, + * a cell is a slot of the uint32 array argv[], e.g. i32/f32 argument + * occupies one cell, i64/f64 argument occupies two cells, note that + * it might be different from the parameter number of the function + * @param argv the arguments. If the function has return value, + * the first (or first two in case 64-bit return value) element of + * argv stores the return value of the called WASM function after this + * function returns. + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm(wasm_exec_env_t exec_env, wasm_function_inst_t function, + uint32_t argc, uint32_t argv[]); + +/** + * Call the given WASM function of a WASM module instance with + * provided results space and arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param function the function to call + * @param num_results the number of results + * @param results the pre-alloced pointer to get the results + * @param num_args the number of arguments + * @param args the arguments + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_a(wasm_exec_env_t exec_env, + wasm_function_inst_t function, uint32_t num_results, + wasm_val_t results[], uint32_t num_args, + wasm_val_t *args); + +/** + * Call the given WASM function of a WASM module instance with + * provided results space and variant arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param function the function to call + * @param num_results the number of results + * @param results the pre-alloced pointer to get the results + * @param num_args the number of arguments + * @param ... the variant arguments + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_v(wasm_exec_env_t exec_env, + wasm_function_inst_t function, uint32_t num_results, + wasm_val_t results[], uint32_t num_args, ...); + +/** + * Call a function reference of a given WASM runtime instance with + * arguments. + * + * Note: this can be used to call a function which is not exported + * by the module explicitly. You might consider it as an abstraction + * violation. + * + * @param exec_env the execution environment to call the function + * which must be created from wasm_create_exec_env() + * @param element_index the function reference index, usually + * prvovided by the caller of a registed native function + * @param argc the number of arguments + * @param argv the arguments. If the function method has return value, + * the first (or first two in case 64-bit return value) element of + * argv stores the return value of the called WASM function after this + * function returns. + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get exception info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_indirect(wasm_exec_env_t exec_env, uint32_t element_index, + uint32_t argc, uint32_t argv[]); + +/** + * Find the unique main function from a WASM module instance + * and execute that function. + * + * @param module_inst the WASM module instance + * @param argc the number of arguments + * @param argv the arguments array, if the main function has return value, + * *(int*)argv stores the return value of the called main function after + * this function returns. + * + * @return true if the main function is called, false otherwise and exception + * will be thrown, the caller can call wasm_runtime_get_exception to get + * the exception info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_main(wasm_module_inst_t module_inst, int32_t argc, + char *argv[]); + +/** + * Find the specified function in argv[0] from a WASM module instance + * and execute that function. + * + * @param module_inst the WASM module instance + * @param name the name of the function to execute. + * to indicate the module name via: $module_name$function_name + * or just a function name: function_name + * @param argc the number of arguments + * @param argv the arguments array + * + * @return true if the specified function is called, false otherwise and + * exception will be thrown, the caller can call wasm_runtime_get_exception + * to get the exception info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_func(wasm_module_inst_t module_inst, const char *name, + int32_t argc, char *argv[]); + +/** + * Get exception info of the WASM module instance. + * + * @param module_inst the WASM module instance + * + * @return the exception string + */ +WASM_RUNTIME_API_EXTERN const char * +wasm_runtime_get_exception(wasm_module_inst_t module_inst); + +/** + * Set exception info of the WASM module instance. + * + * @param module_inst the WASM module instance + * + * @param exception the exception string + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_exception(wasm_module_inst_t module_inst, + const char *exception); + +/** + * Clear exception info of the WASM module instance. + * + * @param module_inst the WASM module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_clear_exception(wasm_module_inst_t module_inst); + +/** + * Terminate the WASM module instance. + * + * This function causes the module instance fail as if it raised a trap. + * + * This is intended to be used in situations like: + * + * - A thread is executing the WASM module instance + * (eg. it's in the middle of `wasm_application_execute_main`) + * + * - Another thread has a copy of `wasm_module_inst_t` of + * the module instance and wants to terminate it asynchronously. + * + * @param module_inst the WASM module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_terminate(wasm_module_inst_t module_inst); + +/** + * Set custom data to WASM module instance. + * Note: + * If WAMR_BUILD_LIB_PTHREAD is enabled, this API + * will spread the custom data to all threads + * + * @param module_inst the WASM module instance + * @param custom_data the custom data to be set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_custom_data(wasm_module_inst_t module_inst, void *custom_data); + +/** + * Get the custom data within a WASM module instance. + * + * @param module_inst the WASM module instance + * + * @return the custom data (NULL if not set yet) + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_custom_data(wasm_module_inst_t module_inst); + +/** + * Set the memory bounds checks flag of a WASM module instance. + * + * @param module_inst the WASM module instance + * @param enable the flag to enable/disable the memory bounds checks + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_bounds_checks(wasm_module_inst_t module_inst, bool enable); + +/** + * Check if the memory bounds checks flag is enabled for a WASM module instance. + * + * @param module_inst the WASM module instance + * @return true if the memory bounds checks flag is enabled, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_bounds_checks_enabled(wasm_module_inst_t module_inst); + +/** + * Allocate memory from the heap of WASM module instance + * + * Note: wasm_runtime_module_malloc can call heap functions inside + * the module instance and thus cause a memory growth. + * This API needs to be used very carefully when you have a native + * pointers to the module instance memory obtained with + * wasm_runtime_addr_app_to_native or similar APIs. + * + * @param module_inst the WASM module instance which contains heap + * @param size the size bytes to allocate + * @param p_native_addr return native address of the allocated memory + * if it is not NULL, and return NULL if memory malloc failed + * + * @return the allocated memory address, which is a relative offset to the + * base address of the module instance's memory space. Note that + * it is not an absolute address. + * Return non-zero if success, zero if failed. + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_module_malloc(wasm_module_inst_t module_inst, uint64_t size, + void **p_native_addr); + +/** + * Free memory to the heap of WASM module instance + * + * @param module_inst the WASM module instance which contains heap + * @param ptr the pointer to free + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_module_free(wasm_module_inst_t module_inst, uint64_t ptr); + +/** + * Allocate memory from the heap of WASM module instance and initialize + * the memory with src + * + * @param module_inst the WASM module instance which contains heap + * @param src the source data to copy + * @param size the size of the source data + * + * @return the allocated memory address, which is a relative offset to the + * base address of the module instance's memory space. Note that + * it is not an absolute address. + * Return non-zero if success, zero if failed. + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_module_dup_data(wasm_module_inst_t module_inst, const char *src, + uint64_t size); + +/** + * Validate the app address, check whether it belongs to WASM module + * instance's address space, or in its heap space or memory space. + * + * @param module_inst the WASM module instance + * @param app_offset the app address to validate, which is a relative address + * @param size the size bytes of the app address + * + * @return true if success, false otherwise. If failed, an exception will + * be thrown. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_addr(wasm_module_inst_t module_inst, + uint64_t app_offset, uint64_t size); + +/** + * Similar to wasm_runtime_validate_app_addr(), except that the size parameter + * is not provided. This function validates the app string address, check + * whether it belongs to WASM module instance's address space, or in its heap + * space or memory space. Moreover, it checks whether it is the offset of a + * string that is end with '\0'. + * + * Note: The validation result, especially the NUL termination check, + * is not reliable for a module instance with multiple threads because + * other threads can modify the heap behind us. + * + * @param module_inst the WASM module instance + * @param app_str_offset the app address of the string to validate, which is a + * relative address + * + * @return true if success, false otherwise. If failed, an exception will + * be thrown. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_str_addr(wasm_module_inst_t module_inst, + uint64_t app_str_offset); + +/** + * Validate the native address, check whether it belongs to WASM module + * instance's address space, or in its heap space or memory space. + * + * @param module_inst the WASM module instance + * @param native_ptr the native address to validate, which is an absolute + * address + * @param size the size bytes of the app address + * + * @return true if success, false otherwise. If failed, an exception will + * be thrown. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_native_addr(wasm_module_inst_t module_inst, + void *native_ptr, uint64_t size); + +/** + * Convert app address(relative address) to native address(absolute address) + * + * Note that native addresses to module instance memory can be invalidated + * on a memory growth. (Except shared memory, whose native addresses are + * stable.) + * + * @param module_inst the WASM module instance + * @param app_offset the app adress + * + * @return the native address converted + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_addr_app_to_native(wasm_module_inst_t module_inst, + uint64_t app_offset); + +/** + * Convert native address(absolute address) to app address(relative address) + * + * @param module_inst the WASM module instance + * @param native_ptr the native address + * + * @return the app address converted + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_addr_native_to_app(wasm_module_inst_t module_inst, + void *native_ptr); + +/** + * Get the app address range (relative address) that a app address belongs to + * + * @param module_inst the WASM module instance + * @param app_offset the app address to retrieve + * @param p_app_start_offset buffer to output the app start offset if not NULL + * @param p_app_end_offset buffer to output the app end offset if not NULL + * + * @return true if success, false otherwise. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_app_addr_range(wasm_module_inst_t module_inst, + uint64_t app_offset, + uint64_t *p_app_start_offset, + uint64_t *p_app_end_offset); + +/** + * Get the native address range (absolute address) that a native address + * belongs to + * + * @param module_inst the WASM module instance + * @param native_ptr the native address to retrieve + * @param p_native_start_addr buffer to output the native start address + * if not NULL + * @param p_native_end_addr buffer to output the native end address + * if not NULL + * + * @return true if success, false otherwise. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_native_addr_range(wasm_module_inst_t module_inst, + uint8_t *native_ptr, + uint8_t **p_native_start_addr, + uint8_t **p_native_end_addr); + +/** + * Get the number of import items for a WASM module + * + * @param module the WASM module + * + * @return the number of imports (zero for none), or -1 for failure + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_get_import_count(const wasm_module_t module); + +/** + * Get information about a specific WASM module import + * + * @param module the WASM module + * @param import_index the desired import index + * @param import_type the location to store information about the import + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_get_import_type(const wasm_module_t module, int32_t import_index, + wasm_import_type *import_type); + +/** + * Get the number of export items for a WASM module + * + * @param module the WASM module + * + * @return the number of exports (zero for none), or -1 for failure + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_get_export_count(const wasm_module_t module); + +/** + * Get information about a specific WASM module export + * + * @param module the WASM module + * @param export_index the desired export index + * @param export_type the location to store information about the export + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_get_export_type(const wasm_module_t module, int32_t export_index, + wasm_export_type *export_type); + +/** + * Register native functions with same module name + * + * Note: The array `native_symbols` should not be read-only because the + * library can modify it in-place. + * + * Note: After successful call of this function, the array `native_symbols` + * is owned by the library. + * + * @param module_name the module name of the native functions + * @param native_symbols specifies an array of NativeSymbol structures which + * contain the names, function pointers and signatures + * Note: WASM runtime will not allocate memory to clone the data, so + * user must ensure the array can be used forever + * Meanings of letters in function signature: + * 'i': the parameter is i32 type + * 'I': the parameter is i64 type + * 'f': the parameter is f32 type + * 'F': the parameter is f64 type + * 'r': the parameter is externref type, it should be a uintptr_t + * in host + * '*': the parameter is a pointer (i32 in WASM), and runtime will + * auto check its boundary before calling the native function. + * If it is followed by '~', the checked length of the pointer + * is gotten from the following parameter, if not, the checked + * length of the pointer is 1. + * '~': the parameter is the pointer's length with i32 type, and must + * follow after '*' + * '$': the parameter is a string (i32 in WASM), and runtime will + * auto check its boundary before calling the native function + * @param n_native_symbols specifies the number of native symbols in the array + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32_t n_native_symbols); + +/** + * Register native functions with same module name, similar to + * wasm_runtime_register_natives, the difference is that runtime passes raw + * arguments to native API, which means that the native API should be defined as + * void foo(wasm_exec_env_t exec_env, uint64 *args); + * and native API should extract arguments one by one from args array with macro + * native_raw_get_arg + * and write the return value back to args[0] with macro + * native_raw_return_type and native_raw_set_return + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32_t n_native_symbols); + +/** + * Undo wasm_runtime_register_natives or wasm_runtime_register_natives_raw + * + * @param module_name Should be the same as the corresponding + * wasm_runtime_register_natives. + * (Same in term of strcmp.) + * + * @param native_symbols Should be the same as the corresponding + * wasm_runtime_register_natives. + * (Same in term of pointer comparison.) + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_unregister_natives(const char *module_name, + NativeSymbol *native_symbols); + +/** + * Get attachment of native function from execution environment + * + * @param exec_env the execution environment to retrieve + * + * @return the attachment of native function + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_function_attachment(wasm_exec_env_t exec_env); + +/** + * Set user data to execution environment. + * + * @param exec_env the execution environment + * @param user_data the user data to be set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_user_data(wasm_exec_env_t exec_env, void *user_data); + +/** + * Get the user data within execution environment. + * + * @param exec_env the execution environment + * + * @return the user data (NULL if not set yet) + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_user_data(wasm_exec_env_t exec_env); + +/** + * Dump runtime memory consumption, including: + * Exec env memory consumption + * WASM module memory consumption + * WASM module instance memory consumption + * stack and app heap used info + * + * @param exec_env the execution environment + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_dump_mem_consumption(wasm_exec_env_t exec_env); + +/** + * Dump runtime performance profiler data of each function + * + * @param module_inst the WASM module instance to profile + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_dump_perf_profiling(wasm_module_inst_t module_inst); + +/** + * Return total wasm functions' execution time in ms + * + * @param module_inst the WASM module instance to profile + */ +WASM_RUNTIME_API_EXTERN double +wasm_runtime_sum_wasm_exec_time(wasm_module_inst_t module_inst); + +/** + * Return execution time in ms of a given wasm funciton with + * func_name. If the function is not found, return 0. + * + * @param module_inst the WASM module instance to profile + * @param func_name could be an export name or a name in the + * name section + */ +WASM_RUNTIME_API_EXTERN double +wasm_runtime_get_wasm_func_exec_time(wasm_module_inst_t inst, + const char *func_name); + +/* wasm thread callback function type */ +typedef void *(*wasm_thread_callback_t)(wasm_exec_env_t, void *); +/* wasm thread type */ +typedef uintptr_t wasm_thread_t; + +/** + * Set the max thread num per cluster. + * + * @param num maximum thread num + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_max_thread_num(uint32_t num); + +/** + * Spawn a new exec_env, the spawned exec_env + * can be used in other threads + * + * @param num the original exec_env + * + * @return the spawned exec_env if success, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_exec_env_t +wasm_runtime_spawn_exec_env(wasm_exec_env_t exec_env); + +/** + * Destroy the spawned exec_env + * + * @param exec_env the spawned exec_env + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_spawned_exec_env(wasm_exec_env_t exec_env); + +/** + * Spawn a thread from the given exec_env + * + * @param exec_env the original exec_env + * @param tid thread id to be returned to the caller + * @param callback the callback function provided by the user + * @param arg the arguments passed to the callback + * + * @return 0 if success, -1 otherwise + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_spawn_thread(wasm_exec_env_t exec_env, wasm_thread_t *tid, + wasm_thread_callback_t callback, void *arg); + +/** + * Wait a spawned thread to terminate + * + * @param tid thread id + * @param retval if not NULL, output the return value of the thread + * + * @return 0 if success, error number otherwise + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_join_thread(wasm_thread_t tid, void **retval); + +/** + * Map external object to an internal externref index: if the index + * has been created, return it, otherwise create the index. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to be mapped + * @param p_externref_idx return externref index of the external object + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_obj2ref(wasm_module_inst_t module_inst, void *extern_obj, + uint32_t *p_externref_idx); + +/** + * Delete external object registered by `wasm_externref_obj2ref`. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to be deleted + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_objdel(wasm_module_inst_t module_inst, void *extern_obj); + +/** + * Set cleanup callback to release external object. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to which to set the + * `extern_obj_cleanup` cleanup callback. + * @param extern_obj_cleanup a callback to release `extern_obj` + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_set_cleanup(wasm_module_inst_t module_inst, void *extern_obj, + void (*extern_obj_cleanup)(void *)); + +/** + * Retrieve the external object from an internal externref index + * + * @param externref_idx the externref index to retrieve + * @param p_extern_obj return the mapped external object of + * the externref index + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_ref2obj(uint32_t externref_idx, void **p_extern_obj); + +/** + * Retain an extern object which is mapped to the internal externref + * so that the object won't be cleaned during extern object reclaim + * if it isn't used. + * + * @param externref_idx the externref index of an external object + * to retain + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_retain(uint32_t externref_idx); + +/** + * Dump the call stack to stdout + * + * @param exec_env the execution environment + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_dump_call_stack(wasm_exec_env_t exec_env); + +/** + * Get the size required to store the call stack contents, including + * the space for terminating null byte ('\0') + * + * @param exec_env the execution environment + * + * @return size required to store the contents, 0 means error + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_call_stack_buf_size(wasm_exec_env_t exec_env); + +/** + * Dump the call stack to buffer. + * + * @note this function is not thread-safe, please only use this API + * when the exec_env is not executing + * + * @param exec_env the execution environment + * @param buf buffer to store the dumped content + * @param len length of the buffer + * + * @return bytes dumped to the buffer, including the terminating null + * byte ('\0'), 0 means error and data in buf may be invalid + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_dump_call_stack_to_buf(wasm_exec_env_t exec_env, char *buf, + uint32_t len); + +/** + * Get the size required to store the LLVM PGO profile data + * + * @param module_inst the WASM module instance + * + * @return size required to store the contents, 0 means error + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_pgo_prof_data_size(wasm_module_inst_t module_inst); + +/** + * Dump the LLVM PGO profile data to buffer + * + * @param module_inst the WASM module instance + * @param buf buffer to store the dumped content + * @param len length of the buffer + * + * @return bytes dumped to the buffer, 0 means error and data in buf + * may be invalid + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_dump_pgo_prof_data_to_buf(wasm_module_inst_t module_inst, + char *buf, uint32_t len); + +/** + * Get a custom section by name + * + * @param module_comm the module to find + * @param name name of the custom section + * @param len return the length of the content if found + * + * @return Custom section content (not including the name length + * and name string) if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN const uint8_t * +wasm_runtime_get_custom_section(wasm_module_t const module_comm, + const char *name, uint32_t *len); + +/** + * Get WAMR semantic version + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_get_version(uint32_t *major, uint32_t *minor, uint32_t *patch); + +/** + * Check whether an import func `(import (func ...))` + * is linked or not with runtime registered natvie functions + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_func_linked(const char *module_name, + const char *func_name); + +/** + * Check whether an import global `(import + * (global ...))` is linked or not with runtime registered natvie globals + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_global_linked(const char *module_name, + const char *global_name); + +typedef enum { + INTERNAL_ERROR, + MAX_SIZE_REACHED, +} enlarge_memory_error_reason_t; + +typedef void (*enlarge_memory_error_callback_t)( + uint32_t inc_page_count, uint64_t current_memory_size, + uint32_t memory_index, enlarge_memory_error_reason_t failure_reason, + wasm_module_inst_t instance, wasm_exec_env_t exec_env, void *user_data); + +/** + * Setup callback invoked when memory.grow fails + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_enlarge_mem_error_callback( + const enlarge_memory_error_callback_t callback, void *user_data); + +/* + * module instance context APIs + * wasm_runtime_create_context_key + * wasm_runtime_destroy_context_key + * wasm_runtime_set_context + * wasm_runtime_set_context_spread + * wasm_runtime_get_context + * + * This set of APIs is intended to be used by an embedder which provides + * extra sets of native functions, which need per module instance state + * and are maintained outside of the WAMR tree. + * + * It's modelled after the pthread specific API. + * + * wasm_runtime_set_context_spread is similar to + * wasm_runtime_set_context, except that + * wasm_runtime_set_context_spread applies the change + * to all threads in the cluster. + * It's an undefined behavior if multiple threads in a cluster call + * wasm_runtime_set_context_spread on the same key + * simultaneously. It's a caller's resposibility to perform necessary + * serialization if necessary. For example: + * + * if (wasm_runtime_get_context(inst, key) == NULL) { + * newctx = alloc_and_init(...); + * lock(some_lock); + * if (wasm_runtime_get_context(inst, key) == NULL) { + * // this thread won the race + * wasm_runtime_set_context_spread(inst, key, newctx); + * newctx = NULL; + * } + * unlock(some_lock); + * if (newctx != NULL) { + * // this thread lost the race, free it + * cleanup_and_free(newctx); + * } + * } + * + * Note: dynamic key create/destroy while instances are live is not + * implemented as of writing this. + * it's caller's resposibility to ensure destorying all module instances + * before calling wasm_runtime_create_context_key or + * wasm_runtime_destroy_context_key. + * otherwise, it's an undefined behavior. + * + * Note about threads: + * - When spawning a thread, the contexts (the pointers given to + * wasm_runtime_set_context) are copied from the parent + * instance. + * - The destructor is called only on the main instance. + */ + +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_create_context_key(void (*dtor)(wasm_module_inst_t inst, + void *ctx)); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_context_key(void *key); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context(wasm_module_inst_t inst, void *key, void *ctx); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context_spread(wasm_module_inst_t inst, void *key, void *ctx); + +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_context(wasm_module_inst_t inst, void *key); + +/* + * wasm_runtime_begin_blocking_op/wasm_runtime_end_blocking_op + * + * These APIs are intended to be used by the implementations of + * host functions. It wraps an operation which possibly blocks for long + * to prepare for async termination. + * + * For simplicity, we recommend to wrap only the very minimum piece of + * the code with this. Ideally, just a single system call. + * + * eg. + * + * if (!wasm_runtime_begin_blocking_op(exec_env)) { + * return EINTR; + * } + * ret = possibly_blocking_op(); + * wasm_runtime_end_blocking_op(exec_env); + * return ret; + * + * If threading support (WASM_ENABLE_THREAD_MGR) is not enabled, + * these functions are no-op. + * + * If the underlying platform support (OS_ENABLE_WAKEUP_BLOCKING_OP) is + * not available, these functions are no-op. In that case, the runtime + * might not terminate a blocking thread in a timely manner. + * + * If the underlying platform support is available, it's used to wake up + * the thread for async termination. The expectation here is that a + * `os_wakeup_blocking_op` call makes the blocking operation + * (`possibly_blocking_op` in the above example) return in a timely manner. + * + * The actual wake up mechanism used by `os_wakeup_blocking_op` is + * platform-dependent. It might impose some platform-dependent restrictions + * on the implementation of the blocking opearation. + * + * For example, on POSIX-like platforms, a signal (by default SIGUSR1) is + * used. The signal delivery configurations (eg. signal handler, signal mask, + * etc) for the signal are set up by the runtime. You can change the signal + * to use for this purpose by calling os_set_signal_number_for_blocking_op + * before the runtime initialization. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_begin_blocking_op(wasm_exec_env_t exec_env); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_end_blocking_op(wasm_exec_env_t exec_env); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_module_name(wasm_module_t module, const char *name, + char *error_buf, uint32_t error_buf_size); + +/* return the most recently set module name or "" if never set before */ +WASM_RUNTIME_API_EXTERN const char * +wasm_runtime_get_module_name(wasm_module_t module); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_EXPORT_H */ diff --git a/wasm-micro-runtime/core/iwasm/interpreter/SConscript b/wasm-micro-runtime/core/iwasm/interpreter/SConscript new file mode 100644 index 0000000..7c0605e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/SConscript @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * + +cwd = GetCurrentDir() + +src = Split(''' +wasm_runtime.c +''') + +if GetDepend(['WAMR_BUILD_FAST_INTERP']): + src += ["wasm_interp_fast.c"] +else: + src += ["wasm_interp_classic.c"] + +if GetDepend(['WAMR_BUILD_MINI_LOADER']): + src += ["wasm_mini_loader.c"] +else: + src += ["wasm_loader.c"] + + +CPPPATH = [cwd] + +group = DefineGroup('iwasm_interpreter', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/interpreter/iwasm_interp.cmake b/wasm-micro-runtime/core/iwasm/interpreter/iwasm_interp.cmake new file mode 100644 index 0000000..e6e52e4 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/iwasm_interp.cmake @@ -0,0 +1,29 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (IWASM_INTERP_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_INTERP=1) + +include_directories(${IWASM_INTERP_DIR}) + +if (WAMR_BUILD_FAST_INTERP EQUAL 1) + set (INTERPRETER "wasm_interp_fast.c") +else () + set (INTERPRETER "wasm_interp_classic.c") +endif () + +if (WAMR_BUILD_MINI_LOADER EQUAL 1) + set (LOADER "wasm_mini_loader.c") +else () + set (LOADER "wasm_loader.c") +endif () + +file (GLOB_RECURSE source_all + ${IWASM_INTERP_DIR}/${LOADER} + ${IWASM_INTERP_DIR}/wasm_runtime.c + ${IWASM_INTERP_DIR}/${INTERPRETER} +) + +set (IWASM_INTERP_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm.h b/wasm-micro-runtime/core/iwasm/interpreter/wasm.h new file mode 100644 index 0000000..5c436cb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm.h @@ -0,0 +1,1425 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_H_ +#define _WASM_H_ + +#include "bh_platform.h" +#include "bh_hashmap.h" +#include "bh_assert.h" +#if WASM_ENABLE_GC != 0 +#include "gc_export.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Value Type */ +#define VALUE_TYPE_I32 0x7F +#define VALUE_TYPE_I64 0X7E +#define VALUE_TYPE_F32 0x7D +#define VALUE_TYPE_F64 0x7C +#define VALUE_TYPE_V128 0x7B +#define VALUE_TYPE_FUNCREF 0x70 +#define VALUE_TYPE_EXTERNREF 0x6F +#define VALUE_TYPE_VOID 0x40 + +/* Packed Types */ +#define PACKED_TYPE_I8 0x78 +#define PACKED_TYPE_I16 0x77 + +/* Reference Types */ +#define REF_TYPE_NULLFUNCREF 0x73 +#define REF_TYPE_NULLEXTERNREF 0x72 +#define REF_TYPE_NULLREF 0x71 +#define REF_TYPE_FUNCREF VALUE_TYPE_FUNCREF /* 0x70 */ +#define REF_TYPE_EXTERNREF VALUE_TYPE_EXTERNREF /* 0x6F */ +#define REF_TYPE_ANYREF 0x6E +#define REF_TYPE_EQREF 0x6D +#define REF_TYPE_I31REF 0x6C +#define REF_TYPE_STRUCTREF 0x6B +#define REF_TYPE_ARRAYREF 0x6A +#define REF_TYPE_HT_NON_NULLABLE 0x64 +#define REF_TYPE_HT_NULLABLE 0x63 +#define REF_TYPE_STRINGREF VALUE_TYPE_STRINGREF /* 0x67 */ +#define REF_TYPE_STRINGVIEWWTF8 VALUE_TYPE_STRINGVIEWWTF8 /* 0x66 */ +#define REF_TYPE_STRINGVIEWWTF16 VALUE_TYPE_STRINGVIEWWTF16 /* 0x62 */ +#define REF_TYPE_STRINGVIEWITER VALUE_TYPE_STRINGVIEWITER /* 0x61 */ + +/* Heap Types */ +#define HEAP_TYPE_NOFUNC (-0x0D) +#define HEAP_TYPE_NOEXTERN (-0x0E) +#define HEAP_TYPE_NONE (-0x0F) +#define HEAP_TYPE_FUNC (-0x10) +#define HEAP_TYPE_EXTERN (-0x11) +#define HEAP_TYPE_ANY (-0x12) +#define HEAP_TYPE_EQ (-0x13) +#define HEAP_TYPE_I31 (-0x14) +#define HEAP_TYPE_STRUCT (-0x15) +#define HEAP_TYPE_ARRAY (-0x16) +#define HEAP_TYPE_STRINGREF (-0x19) +#define HEAP_TYPE_STRINGVIEWWTF8 (-0x1A) +#define HEAP_TYPE_STRINGVIEWWTF16 (-0x1E) +#define HEAP_TYPE_STRINGVIEWITER (-0x1F) + +/* Defined Types */ +#define DEFINED_TYPE_FUNC 0x60 +#define DEFINED_TYPE_STRUCT 0x5F +#define DEFINED_TYPE_ARRAY 0x5E +#define DEFINED_TYPE_SUB 0x50 +#define DEFINED_TYPE_SUB_FINAL 0x4F +#define DEFINED_TYPE_REC 0x4E + +/* Used by AOT */ +#define VALUE_TYPE_I1 0x41 +/** + * Used by loader to represent any type of i32/i64/f32/f64/v128 + * and ref types, including funcref, externref, anyref, eqref, + * (ref null $ht), (ref $ht), i31ref, structref, arrayref, + * nullfuncref, nullexternref, nullref and stringref + */ +#define VALUE_TYPE_ANY 0x42 +/** + * Used by wamr compiler to represent object ref types, + * including func object ref, externref object ref, + * internal object ref, eq obect ref, i31 object ref, + * struct object ref, array obect ref + */ +#define VALUE_TYPE_GC_REF 0x43 + +#define MAX_PAGE_COUNT_FLAG 0x01 +#define SHARED_MEMORY_FLAG 0x02 +#define MEMORY64_FLAG 0x04 + +#define DEFAULT_NUM_BYTES_PER_PAGE 65536 +#define DEFAULT_MAX_PAGES 65536 +#define DEFAULT_MEM64_MAX_PAGES UINT32_MAX + +/* Max size of linear memory */ +#define MAX_LINEAR_MEMORY_SIZE (4 * (uint64)BH_GB) +/* Roughly 274 TB */ +#define MAX_LINEAR_MEM64_MEMORY_SIZE \ + (DEFAULT_MEM64_MAX_PAGES * (uint64)64 * (uint64)BH_KB) +/* Macro to check memory flag and return appropriate memory size */ +#define GET_MAX_LINEAR_MEMORY_SIZE(is_memory64) \ + (is_memory64 ? MAX_LINEAR_MEM64_MEMORY_SIZE : MAX_LINEAR_MEMORY_SIZE) + +#if WASM_ENABLE_GC == 0 +typedef uintptr_t table_elem_type_t; +#define NULL_REF (0xFFFFFFFF) +#else +typedef void *table_elem_type_t; +#define NULL_REF (NULL) +#define REF_CELL_NUM ((uint32)sizeof(uintptr_t) / sizeof(uint32)) +#endif + +#define INIT_EXPR_NONE 0x00 +#define INIT_EXPR_TYPE_I32_CONST 0x41 +#define INIT_EXPR_TYPE_I64_CONST 0x42 +#define INIT_EXPR_TYPE_F32_CONST 0x43 +#define INIT_EXPR_TYPE_F64_CONST 0x44 +#define INIT_EXPR_TYPE_V128_CONST 0xFD +#define INIT_EXPR_TYPE_GET_GLOBAL 0x23 +#define INIT_EXPR_TYPE_REFNULL_CONST 0xD0 +#define INIT_EXPR_TYPE_FUNCREF_CONST 0xD2 +#define INIT_EXPR_TYPE_STRUCT_NEW 0xD3 +#define INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT 0xD4 +#define INIT_EXPR_TYPE_ARRAY_NEW 0xD5 +#define INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT 0xD6 +#define INIT_EXPR_TYPE_ARRAY_NEW_FIXED 0xD7 +#define INIT_EXPR_TYPE_I31_NEW 0xD8 +#define INIT_EXPR_TYPE_ANY_CONVERT_EXTERN 0xD9 +#define INIT_EXPR_TYPE_EXTERN_CONVERT_ANY 0xDA + +#define WASM_MAGIC_NUMBER 0x6d736100 +#define WASM_CURRENT_VERSION 1 + +#define SECTION_TYPE_USER 0 +#define SECTION_TYPE_TYPE 1 +#define SECTION_TYPE_IMPORT 2 +#define SECTION_TYPE_FUNC 3 +#define SECTION_TYPE_TABLE 4 +#define SECTION_TYPE_MEMORY 5 +#define SECTION_TYPE_GLOBAL 6 +#define SECTION_TYPE_EXPORT 7 +#define SECTION_TYPE_START 8 +#define SECTION_TYPE_ELEM 9 +#define SECTION_TYPE_CODE 10 +#define SECTION_TYPE_DATA 11 +#if WASM_ENABLE_BULK_MEMORY != 0 +#define SECTION_TYPE_DATACOUNT 12 +#endif +#if WASM_ENABLE_TAGS != 0 +#define SECTION_TYPE_TAG 13 +#endif +#if WASM_ENABLE_STRINGREF != 0 +#define SECTION_TYPE_STRINGREF 14 +#endif + +#define SUB_SECTION_TYPE_MODULE 0 +#define SUB_SECTION_TYPE_FUNC 1 +#define SUB_SECTION_TYPE_LOCAL 2 + +#define IMPORT_KIND_FUNC 0 +#define IMPORT_KIND_TABLE 1 +#define IMPORT_KIND_MEMORY 2 +#define IMPORT_KIND_GLOBAL 3 +#if WASM_ENABLE_TAGS != 0 +#define IMPORT_KIND_TAG 4 +#endif + +#define EXPORT_KIND_FUNC 0 +#define EXPORT_KIND_TABLE 1 +#define EXPORT_KIND_MEMORY 2 +#define EXPORT_KIND_GLOBAL 3 +#if WASM_ENABLE_TAGS != 0 +#define EXPORT_KIND_TAG 4 +#endif + +#define LABEL_TYPE_BLOCK 0 +#define LABEL_TYPE_LOOP 1 +#define LABEL_TYPE_IF 2 +#define LABEL_TYPE_FUNCTION 3 +#if WASM_ENABLE_EXCE_HANDLING != 0 +#define LABEL_TYPE_TRY 4 +#define LABEL_TYPE_CATCH 5 +#define LABEL_TYPE_CATCH_ALL 6 +#endif + +#define WASM_TYPE_FUNC 0 +#define WASM_TYPE_STRUCT 1 +#define WASM_TYPE_ARRAY 2 + +#if WASM_ENABLE_STRINGREF != 0 +#define WASM_TYPE_STRINGREF 3 +#define WASM_TYPE_STRINGVIEWWTF8 4 +#define WASM_TYPE_STRINGVIEWWTF16 5 +#define WASM_TYPE_STRINGVIEWITER 6 +#endif + +/* In WasmGC, a table can start with [0x40 0x00] to indicate it has an + * initializer */ +#define TABLE_INIT_EXPR_FLAG 0x40 + +typedef struct WASMModule WASMModule; +typedef struct WASMFunction WASMFunction; +typedef struct WASMGlobal WASMGlobal; +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMTag WASMTag; +#endif + +#ifndef WASM_VALUE_DEFINED +#define WASM_VALUE_DEFINED +typedef union V128 { + int8 i8x16[16]; + int16 i16x8[8]; + int32 i32x8[4]; + int64 i64x2[2]; + float32 f32x4[4]; + float64 f64x2[2]; +} V128; + +typedef union WASMValue { + int32 i32; + uint32 u32; + uint32 global_index; + uint32 ref_index; + int64 i64; + uint64 u64; + float32 f32; + float64 f64; + V128 v128; +#if WASM_ENABLE_GC != 0 + wasm_obj_t gc_obj; + uint32 type_index; + struct { + uint32 type_index; + uint32 length; + } array_new_default; + /* pointer to a memory space holding more data, current usage: + * struct.new init value: WASMStructNewInitValues * + * array.new init value: WASMArrayNewInitValues * + */ + void *data; +#endif +} WASMValue; +#endif /* end of WASM_VALUE_DEFINED */ + +typedef struct WASMStructNewInitValues { + uint8 type_idx; + uint32 count; + WASMValue fields[1]; +} WASMStructNewInitValues; + +typedef struct WASMArrayNewInitValues { + uint8 type_idx; + uint32 length; + WASMValue elem_data[1]; +} WASMArrayNewInitValues; + +typedef struct InitializerExpression { + /* type of INIT_EXPR_TYPE_XXX, which is an instruction of + constant expression */ + uint8 init_expr_type; + WASMValue u; +} InitializerExpression; + +#if WASM_ENABLE_GC != 0 +/** + * Reference type of (ref null ht) or (ref ht), + * and heap type is defined type (type i), i >= 0 + */ +typedef struct RefHeapType_TypeIdx { + /* ref_type is REF_TYPE_HT_NULLABLE or + REF_TYPE_HT_NON_NULLABLE, (0x63 or 0x64) */ + uint8 ref_type; + /* true if ref_type is REF_TYPE_HT_NULLABLE */ + bool nullable; + /* heap type is defined type: type_index >= 0 */ + int32 type_idx; +} RefHeapType_TypeIdx; + +/** + * Reference type of (ref null ht) or (ref ht), + * and heap type is non-defined type + */ +typedef struct RefHeapType_Common { + /* ref_type is REF_TYPE_HT_NULLABLE or + REF_TYPE_HT_NON_NULLABLE (0x63 or 0x64) */ + uint8 ref_type; + /* true if ref_type is REF_TYPE_HT_NULLABLE */ + bool nullable; + /* Common heap type (not defined type): + -0x10 (func), -0x11 (extern), -0x12 (any), -0x13 (eq), + -0x16 (i31), -0x17 (nofunc), -0x18 (noextern), + -0x19 (struct), -0x20 (array), -0x21 (none) */ + int32 heap_type; +} RefHeapType_Common; + +/** + * Reference type + */ +typedef union WASMRefType { + uint8 ref_type; + RefHeapType_TypeIdx ref_ht_typeidx; + RefHeapType_Common ref_ht_common; +} WASMRefType; + +typedef struct WASMRefTypeMap { + /** + * The type index of a type array, which only stores + * the first byte of the type, e.g. WASMFuncType.types, + * WASMStructType.fields + */ + uint16 index; + /* The full type info if the type cannot be described + with one byte */ + WASMRefType *ref_type; +} WASMRefTypeMap; +#endif /* end of WASM_ENABLE_GC */ + +#if WASM_ENABLE_GC == 0 +typedef struct WASMFuncType WASMType; +typedef WASMType *WASMTypePtr; +#else +/** + * Common type, store the same fields of + * WASMFuncType, WASMStructType and WASMArrayType + */ +typedef struct WASMType { + /** + * type_flag must be WASM_TYPE_FUNC/STRUCT/ARRAY to + * denote that it is a WASMFuncType, WASMStructType or + * WASMArrayType + */ + uint16 type_flag; + + bool is_sub_final; + /* How many types are referring to this type */ + uint16 ref_count; + /* The inheritance depth */ + uint16 inherit_depth; + /* The root type */ + struct WASMType *root_type; + /* The parent type */ + struct WASMType *parent_type; + uint32 parent_type_idx; + + /* The number of internal types in the current rec group, and if + the type is not in a recursive group, rec_count is 1 since a + single type definition is reinterpreted as a short-hand for a + recursive group containing just one type */ + uint16 rec_count; + uint16 rec_idx; + /* The index of the begin type of this group */ + uint32 rec_begin_type_idx; +} WASMType, *WASMTypePtr; +#endif /* end of WASM_ENABLE_GC */ + +/* Function type */ +typedef struct WASMFuncType { +#if WASM_ENABLE_GC != 0 + WASMType base_type; +#endif + + uint16 param_count; + uint16 result_count; + uint16 param_cell_num; + uint16 ret_cell_num; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + /* Code block to call llvm jit functions of this + kind of function type from fast jit jitted code */ + void *call_to_llvm_jit_from_fast_jit; +#endif + +#if WASM_ENABLE_GC != 0 + uint16 ref_type_map_count; + WASMRefTypeMap *ref_type_maps; + WASMRefTypeMap *result_ref_type_maps; +#else + uint16 ref_count; +#endif + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick AOT/JIT entry of this func type */ + void *quick_aot_entry; +#endif + + /* types of params and results, only store the first byte + * of the type, if it cannot be described with one byte, + * then the full type info is stored in ref_type_maps */ + uint8 types[1]; +} WASMFuncType; + +#if WASM_ENABLE_GC != 0 +typedef struct WASMStructFieldType { + uint16 field_flags; + uint8 field_type; + uint8 field_size; + uint32 field_offset; +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + /* + * The field size and field offset of a wasm struct may vary + * in 32-bit target and 64-bit target, e.g., the size of a + * GC reference is 4 bytes in the former and 8 bytes in the + * latter, the AOT compiler needs to use the correct field + * offset according to the target info. + */ + uint8 field_size_64bit; + uint8 field_size_32bit; + uint32 field_offset_64bit; + uint32 field_offset_32bit; +#endif +} WASMStructFieldType; + +typedef struct WASMStructType { + WASMType base_type; + + /* total size of this struct object */ + uint32 total_size; + uint16 field_count; + + uint16 ref_type_map_count; + WASMRefTypeMap *ref_type_maps; + + /* Offsets of reference fields that need to be traced during GC. + The first element of the table is the number of such offsets. */ + uint16 *reference_table; + + /* Field info, note that fields[i]->field_type only stores + * the first byte of the field type, if it cannot be described + * with one byte, then the full field type info is stored in + * ref_type_maps */ + WASMStructFieldType fields[1]; +} WASMStructType; + +typedef struct WASMArrayType { + WASMType base_type; + + uint16 elem_flags; + uint8 elem_type; + /* The full elem type info if the elem type cannot be + described with one byte */ + WASMRefType *elem_ref_type; +} WASMArrayType; + +#if WASM_ENABLE_STRINGREF != 0 +/* stringref representation, we define it as a void * pointer here, the + * stringref implementation can use any structure */ +/* + WasmGC heap + +-----------------------+ + | | + | stringref | + | +----------+ | external string representation + | | host_ptr |--------o------+----->+------------+ + | +----------+ | | | | + | | | +------------+ + | stringview_wtf8/16 | | + | +----------+ | | + | | host_ptr |--------o------+ + | +----------+ | | + | | | + | stringview_iter | | + | +----------+ | | + | | host_ptr |--------o------+ + | +----------+ | + | | pos | | + | +----------+ | + | | + +-----------------------+ +*/ +typedef void *WASMString; + +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ +#endif /* end of WASM_ENABLE_GC != 0 */ + +typedef struct WASMTable { + uint8 elem_type; + /** + * 0: no max size and not shared + * 1: hax max size + * 2: shared + */ + uint8 flags; + bool possible_grow; + uint32 init_size; + /* specified if (flags & 1), else it is 0x10000 */ + uint32 max_size; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; + /* init expr for the whole table */ + InitializerExpression init_expr; +#endif +} WASMTable; + +#if WASM_ENABLE_MEMORY64 != 0 +typedef uint64 mem_offset_t; +#else +typedef uint32 mem_offset_t; +#endif + +typedef struct WASMMemory { + uint32 flags; + uint32 num_bytes_per_page; + uint32 init_page_count; + uint32 max_page_count; +} WASMMemory; + +typedef struct WASMTableImport { + char *module_name; + char *field_name; + /* 0: no max size, 1: has max size */ + uint8 elem_type; + uint8 flags; + bool possible_grow; + uint32 init_size; + /* specified if (flags & 1), else it is 0x10000 */ + uint32 max_size; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *import_module; + WASMTable *import_table_linked; +#endif +} WASMTableImport; + +typedef struct WASMMemoryImport { + char *module_name; + char *field_name; + uint32 flags; + uint32 num_bytes_per_page; + uint32 init_page_count; + uint32 max_page_count; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *import_module; + WASMMemory *import_memory_linked; +#endif +} WASMMemoryImport; + +typedef struct WASMFunctionImport { + char *module_name; + char *field_name; + /* function type */ + WASMFuncType *func_type; + /* native function pointer after linked */ + void *func_ptr_linked; + /* signature from registered native symbols */ + const char *signature; + /* attachment */ + void *attachment; +#if WASM_ENABLE_GC != 0 + /* the type index of this function's func_type */ + uint32 type_idx; +#endif + bool call_conv_raw; + bool call_conv_wasm_c_api; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *import_module; + WASMFunction *import_func_linked; +#endif +} WASMFunctionImport; + +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMTagImport { + char *module_name; + char *field_name; + uint8 attribute; /* the type of the tag (numerical) */ + uint32 type; /* the type of the catch function (numerical)*/ + WASMType *tag_type; + void *tag_ptr_linked; + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* imported tag pointer after linked */ + WASMModule *import_module; + WASMTag *import_tag_linked; + uint32 import_tag_index_linked; +#endif +} WASMTagImport; +#endif + +typedef struct WASMGlobalImport { + char *module_name; + char *field_name; + uint8 type; + bool is_mutable; + bool is_linked; + /* global data after linked */ + WASMValue global_data_linked; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + /* imported function pointer after linked */ + /* TODO: remove if not needed */ + WASMModule *import_module; + WASMGlobal *import_global_linked; +#endif +#if WASM_ENABLE_FAST_JIT != 0 + /* The data offset of current global in global data */ + uint32 data_offset; +#endif +} WASMGlobalImport; + +typedef struct WASMImport { + uint8 kind; + union { + WASMFunctionImport function; + WASMTableImport table; + WASMMemoryImport memory; +#if WASM_ENABLE_TAGS != 0 + WASMTagImport tag; +#endif + WASMGlobalImport global; + struct { + char *module_name; + char *field_name; + } names; + } u; +} WASMImport; + +struct WASMFunction { +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + char *field_name; +#endif + /* the type of function */ + WASMFuncType *func_type; + uint32 local_count; + uint8 *local_types; +#if WASM_ENABLE_GC != 0 + uint16 local_ref_type_map_count; + WASMRefTypeMap *local_ref_type_maps; +#endif + + /* cell num of parameters */ + uint16 param_cell_num; + /* cell num of return type */ + uint16 ret_cell_num; + /* cell num of local variables */ + uint16 local_cell_num; + /* offset of each local, including function parameters + and local variables */ + uint16 *local_offsets; + + uint32 max_stack_cell_num; + uint32 max_block_num; + uint32 code_size; + uint8 *code; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 code_compiled_size; + uint8 *code_compiled; + uint8 *consts; + uint32 const_cell_num; +#endif + +#if WASM_ENABLE_GC != 0 + /* the type index of this function's func_type */ + uint32 type_idx; +#endif + +#if WASM_ENABLE_EXCE_HANDLING != 0 + uint32 exception_handler_count; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + /* Whether function has opcode memory.grow */ + bool has_op_memory_grow; + /* Whether function has opcode call or call_indirect */ + bool has_op_func_call; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + /* Whether function has memory operation opcodes */ + bool has_memory_operations; + /* Whether function has opcode call_indirect */ + bool has_op_call_indirect; + /* Whether function has opcode set_global_aux_stack */ + bool has_op_set_global_aux_stack; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + /* The compiled fast jit jitted code block of this function */ + void *fast_jit_jitted_code; +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + /* The compiled llvm jit func ptr of this function */ + void *llvm_jit_func_ptr; + /* Code block to call fast jit jitted code of this function + from the llvm jit jitted code */ + void *call_to_fast_jit_from_llvm_jit; +#endif +#endif +}; + +#if WASM_ENABLE_TAGS != 0 +struct WASMTag { + uint8 attribute; /* the attribute property of the tag (expected to be 0) */ + uint32 type; /* the type of the tag (expected valid inden in type table) */ + WASMType *tag_type; +}; +#endif + +struct WASMGlobal { + uint8 type; + bool is_mutable; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + InitializerExpression init_expr; +#if WASM_ENABLE_FAST_JIT != 0 + /* The data offset of current global in global data */ + uint32 data_offset; +#endif +}; + +typedef struct WASMExport { + char *name; + uint8 kind; + uint32 index; +} WASMExport; + +typedef struct WASMTableSeg { + /* 0 to 7 */ + uint32 mode; + /* funcref or externref, elemkind will be considered as funcref */ + uint32 elem_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif + /* optional, only for active */ + uint32 table_index; + InitializerExpression base_offset; + uint32 value_count; + InitializerExpression *init_values; +} WASMTableSeg; + +typedef struct WASMDataSeg { + uint32 memory_index; + InitializerExpression base_offset; + uint32 data_length; +#if WASM_ENABLE_BULK_MEMORY != 0 + bool is_passive; +#endif + uint8 *data; +} WASMDataSeg; + +typedef struct BlockAddr { + const uint8 *start_addr; + uint8 *else_addr; + uint8 *end_addr; +} BlockAddr; + +#if WASM_ENABLE_LIBC_WASI != 0 +typedef struct WASIArguments { + const char **dir_list; + uint32 dir_count; + const char **map_dir_list; + uint32 map_dir_count; + const char **env; + uint32 env_count; + /* in CIDR noation */ + const char **addr_pool; + uint32 addr_count; + const char **ns_lookup_pool; + uint32 ns_lookup_count; + char **argv; + uint32 argc; + os_raw_file_handle stdio[3]; +} WASIArguments; +#endif + +typedef struct StringNode { + struct StringNode *next; + char *str; +} StringNode, *StringList; + +typedef struct BrTableCache { + struct BrTableCache *next; + /* Address of br_table opcode */ + uint8 *br_table_op_addr; + uint32 br_count; + uint32 br_depths[1]; +} BrTableCache; + +#if WASM_ENABLE_DEBUG_INTERP != 0 +typedef struct WASMFastOPCodeNode { + struct WASMFastOPCodeNode *next; + uint64 offset; + uint8 orig_op; +} WASMFastOPCodeNode; +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +typedef struct WASMCustomSection { + struct WASMCustomSection *next; + /* Start address of the section name */ + char *name_addr; + /* Length of the section name decoded from leb */ + uint32 name_len; + /* Start address of the content (name len and name skipped) */ + uint8 *content_addr; + uint32 content_len; +} WASMCustomSection; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 +struct AOTCompData; +struct AOTCompContext; + +/* Orc JIT thread arguments */ +typedef struct OrcJitThreadArg { +#if WASM_ENABLE_JIT != 0 + struct AOTCompContext *comp_ctx; +#endif + struct WASMModule *module; + uint32 group_idx; +} OrcJitThreadArg; +#endif + +struct WASMModuleInstance; + +struct WASMModule { + /* Module type, for module loaded from WASM bytecode binary, + this field is Wasm_Module_Bytecode; + for module loaded from AOT file, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModule structure. */ + uint32 module_type; + + uint32 type_count; + uint32 import_count; + uint32 function_count; + uint32 table_count; + uint32 memory_count; +#if WASM_ENABLE_TAGS != 0 + uint32 tag_count; +#endif + uint32 global_count; + uint32 export_count; + uint32 table_seg_count; + /* data seg count read from data segment section */ + uint32 data_seg_count; +#if WASM_ENABLE_BULK_MEMORY != 0 + /* data count read from datacount section */ + uint32 data_seg_count1; +#endif +#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_STRINGREF != 0 + uint32 string_literal_count; + uint32 *string_literal_lengths; + const uint8 **string_literal_ptrs; +#endif +#endif + + uint32 import_function_count; + uint32 import_table_count; + uint32 import_memory_count; +#if WASM_ENABLE_TAGS != 0 + uint32 import_tag_count; +#endif + uint32 import_global_count; + + WASMImport *import_functions; + WASMImport *import_tables; + WASMImport *import_memories; +#if WASM_ENABLE_TAGS != 0 + WASMImport *import_tags; +#endif + WASMImport *import_globals; + + WASMType **types; + WASMImport *imports; + WASMFunction **functions; + WASMTable *tables; + WASMMemory *memories; +#if WASM_ENABLE_TAGS != 0 + WASMTag **tags; +#endif + WASMGlobal *globals; + WASMExport *exports; + WASMTableSeg *table_segments; + WASMDataSeg **data_segments; + uint32 start_function; + + /* total global variable size */ + uint32 global_data_size; + + /* the index of auxiliary __data_end global, + -1 means unexported */ + uint32 aux_data_end_global_index; + /* auxiliary __data_end exported by wasm app */ + uint64 aux_data_end; + + /* the index of auxiliary __heap_base global, + -1 means unexported */ + uint32 aux_heap_base_global_index; + /* auxiliary __heap_base exported by wasm app */ + uint64 aux_heap_base; + + /* the index of auxiliary stack top global, + -1 means unexported */ + uint32 aux_stack_top_global_index; + /* auxiliary stack bottom resolved */ + uint64 aux_stack_bottom; + /* auxiliary stack size resolved */ + uint32 aux_stack_size; + + /* the index of malloc/free function, + -1 means unexported */ + uint32 malloc_function; + uint32 free_function; + + /* the index of __retain function, + -1 means unexported */ + uint32 retain_function; + + /* Whether there is possible memory grow, e.g. memory.grow opcode */ + bool possible_memory_grow; + + StringList const_str_list; +#if WASM_ENABLE_FAST_INTERP == 0 + bh_list br_table_cache_list_head; + bh_list *br_table_cache_list; +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 + WASIArguments wasi_args; + bool import_wasi_api; +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* TODO: add mutex for mutli-thread? */ + bh_list import_module_list_head; + bh_list *import_module_list; +#endif + +#if WASM_ENABLE_GC != 0 + /* Ref types hash set */ + HashMap *ref_type_set; + struct WASMRttType **rtt_types; + korp_mutex rtt_type_lock; +#if WASM_ENABLE_STRINGREF != 0 + /* special rtts for stringref types + - stringref + - stringview_wtf8 + - stringview_wtf16 + - stringview_iter + */ + struct WASMRttType *stringref_rtts[4]; +#endif +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0 + bh_list fast_opcode_list; + uint8 *buf_code; + uint64 buf_code_size; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_FAST_JIT != 0 \ + || WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_JIT != 0 + uint8 *load_addr; + uint64 load_size; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + /** + * List of instances referred to this module. When source debugging + * feature is enabled, the debugger may modify the code section of + * the module, so we need to report a warning if user create several + * instances based on the same module. + * + * Also add the instance to the list for Fast JIT to LLVM JIT + * tier-up, since we need to lazily update the LLVM func pointers + * in the instance. + */ + struct WASMModuleInstance *instance_list; + korp_mutex instance_list_lock; +#endif + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + const uint8 *name_section_buf; + const uint8 *name_section_buf_end; +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + WASMCustomSection *custom_section_list; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + /** + * func pointers of Fast JITed (un-imported) functions + * for non Multi-Tier JIT mode: + * (1) when lazy jit is disabled, each pointer is set to the compiled + * fast jit jitted code + * (2) when lazy jit is enabled, each pointer is firstly inited as + * jit_global->compile_fast_jit_and_then_call, and then set to the + * compiled fast jit jitted code when it is called (the stub will + * compile the jit function and then update itself) + * for Multi-Tier JIT mode: + * each pointer is firstly inited as compile_fast_jit_and_then_call, + * and then set to the compiled fast jit jitted code when it is called, + * and when the llvm jit func ptr of the same function is compiled, it + * will be set to call_to_llvm_jit_from_fast_jit of this function type + * (tier-up from fast-jit to llvm-jit) + */ + void **fast_jit_func_ptrs; + /* locks for Fast JIT lazy compilation */ + korp_mutex fast_jit_thread_locks[WASM_ORC_JIT_BACKEND_THREAD_NUM]; + bool fast_jit_thread_locks_inited[WASM_ORC_JIT_BACKEND_THREAD_NUM]; +#endif + +#if WASM_ENABLE_JIT != 0 + struct AOTCompData *comp_data; + struct AOTCompContext *comp_ctx; + /** + * func pointers of LLVM JITed (un-imported) functions + * for non Multi-Tier JIT mode: + * each pointer is set to the lookuped llvm jit func ptr, note that it + * is a stub and will trigger the actual compilation when it is called + * for Multi-Tier JIT mode: + * each pointer is inited as call_to_fast_jit code block, when the llvm + * jit func ptr is actually compiled, it is set to the compiled llvm jit + * func ptr + */ + void **func_ptrs; + /* whether the func pointers are compiled */ + bool *func_ptrs_compiled; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* backend compilation threads */ + korp_tid orcjit_threads[WASM_ORC_JIT_BACKEND_THREAD_NUM]; + /* backend thread arguments */ + OrcJitThreadArg orcjit_thread_args[WASM_ORC_JIT_BACKEND_THREAD_NUM]; + /* whether to stop the compilation of backend threads */ + bool orcjit_stop_compiling; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + /* wait lock/cond for the synchronization of + the llvm jit initialization */ + korp_mutex tierup_wait_lock; + korp_cond tierup_wait_cond; + bool tierup_wait_lock_inited; + korp_tid llvm_jit_init_thread; + /* whether the llvm jit is initialized */ + bool llvm_jit_inited; + /* Whether to enable llvm jit compilation: + it is set to true only when there is a module instance starts to + run with running mode Mode_LLVM_JIT or Mode_Multi_Tier_JIT, + since no need to enable llvm jit compilation for Mode_Interp and + Mode_Fast_JIT, so as to improve performance for them */ + bool enable_llvm_jit_compilation; + /* The count of groups which finish compiling the fast jit + functions in that group */ + uint32 fast_jit_ready_groups; +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + bool is_simd_used; + bool is_ref_types_used; + bool is_bulk_memory_used; +#endif + + /* user defined name */ + char *name; +}; + +typedef struct BlockType { + /* Block type may be expressed in one of two forms: + * either by the type of the single return value or + * by a type index of module. + */ + union { + struct { + uint8 type; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap ref_type_map; +#endif + } value_type; + WASMFuncType *type; + } u; + bool is_value_type; +} BlockType; + +typedef struct WASMBranchBlock { + uint8 *begin_addr; + uint8 *target_addr; + uint32 *frame_sp; + uint32 cell_num; +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* in exception handling, label_type needs to be stored to lookup exception + * handlers */ + uint8 label_type; +#endif +} WASMBranchBlock; + +/** + * Align an unsigned value on a alignment boundary. + * + * @param v the value to be aligned + * @param b the alignment boundary (2, 4, 8, ...) + * + * @return the aligned value + */ +inline static unsigned +align_uint(unsigned v, unsigned b) +{ + unsigned m = b - 1; + return (v + m) & ~m; +} + +/** + * Align an 64 bit unsigned value on a alignment boundary. + * + * @param v the value to be aligned + * @param b the alignment boundary (2, 4, 8, ...) + * + * @return the aligned value + */ +inline static uint64 +align_uint64(uint64 v, uint64 b) +{ + uint64 m = b - 1; + return (v + m) & ~m; +} + +/** + * Check whether a piece of data is out of range + * + * @param offset the offset that the data starts + * @param len the length of the data + * @param max_size the maximum size of the data range + * + * @return true if out of range, false otherwise + */ +inline static bool +offset_len_out_of_bounds(uint32 offset, uint32 len, uint32 max_size) +{ + if (offset + len < offset /* integer overflow */ + || offset + len > max_size) + return true; + return false; +} + +/** + * Return the hash value of c string. + */ +inline static uint32 +wasm_string_hash(const char *str) +{ + unsigned h = (unsigned)strlen(str); + const uint8 *p = (uint8 *)str; + const uint8 *end = p + h; + + while (p != end) + h = ((h << 5) - h) + *p++; + return h; +} + +/** + * Whether two c strings are equal. + */ +inline static bool +wasm_string_equal(const char *s1, const char *s2) +{ + return strcmp(s1, s2) == 0 ? true : false; +} + +/** + * Return the byte size of value type with specific pointer size. + * + * Note: Please use wasm_value_type_size for interpreter, only aot compiler + * can use this API directly to calculate type size for different target + */ +inline static uint32 +wasm_value_type_size_internal(uint8 value_type, uint8 pointer_size) +{ + if (value_type == VALUE_TYPE_VOID) + return 0; + else if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_F32 + || value_type == VALUE_TYPE_ANY) + return sizeof(int32); + else if (value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F64) + return sizeof(int64); +#if WASM_ENABLE_SIMD != 0 + else if (value_type == VALUE_TYPE_V128) + return sizeof(int64) * 2; +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + else if (value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF) + return sizeof(uint32); +#elif WASM_ENABLE_GC != 0 + else if ((value_type >= (uint8)REF_TYPE_ARRAYREF /* 0x6A */ + && value_type <= (uint8)REF_TYPE_NULLFUNCREF) /* 0x73 */ + || (value_type >= (uint8)REF_TYPE_HT_NULLABLE /* 0x63 */ + && value_type <= (uint8)REF_TYPE_HT_NON_NULLABLE) /* 0x64 */ +#if WASM_ENABLE_STRINGREF != 0 + || (value_type >= (uint8)REF_TYPE_STRINGVIEWWTF8 /* 0x66 */ + && value_type <= (uint8)REF_TYPE_STRINGREF) /* 0x67 */ + || (value_type >= (uint8)REF_TYPE_STRINGVIEWITER /* 0x61 */ + && value_type <= (uint8)REF_TYPE_STRINGVIEWWTF16) /* 0x62 */ +#endif + ) + return pointer_size; + else if (value_type == PACKED_TYPE_I8) + return sizeof(int8); + else if (value_type == PACKED_TYPE_I16) + return sizeof(int16); +#endif + else { + bh_assert(0); + } + return 0; +} + +/** + * Return the cell num of value type with specific pointer size. + * + * Note: Please use wasm_value_type_cell_num for interpreter, only aot compiler + * can use this API directly to calculate type cell num for different target + */ +inline static uint16 +wasm_value_type_cell_num_internal(uint8 value_type, uint8 pointer_size) +{ + return wasm_value_type_size_internal(value_type, pointer_size) / 4; +} + +/** + * Return the byte size of value type. + */ +inline static uint32 +wasm_value_type_size(uint8 value_type) +{ + return wasm_value_type_size_internal(value_type, sizeof(uintptr_t)); +} + +inline static uint16 +wasm_value_type_cell_num(uint8 value_type) +{ + return wasm_value_type_size(value_type) / 4; +} + +inline static uint32 +wasm_get_cell_num(const uint8 *types, uint32 type_count) +{ + uint32 cell_num = 0; + uint32 i; + for (i = 0; i < type_count; i++) + cell_num += wasm_value_type_cell_num(types[i]); + return cell_num; +} + +#if WASM_ENABLE_REF_TYPES != 0 +inline static uint16 +wasm_value_type_cell_num_outside(uint8 value_type) +{ + if (VALUE_TYPE_EXTERNREF == value_type) { + return sizeof(uintptr_t) / sizeof(uint32); + } + else { + return wasm_value_type_cell_num(value_type); + } +} +#endif + +#if WASM_ENABLE_GC == 0 +inline static bool +wasm_type_equal(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + const WASMFuncType *func_type1 = (const WASMFuncType *)type1; + const WASMFuncType *func_type2 = (const WASMFuncType *)type2; + + if (type1 == type2) { + return true; + } + + return (func_type1->param_count == func_type2->param_count + && func_type1->result_count == func_type2->result_count + && memcmp( + func_type1->types, func_type2->types, + (uint32)(func_type1->param_count + func_type1->result_count)) + == 0) + ? true + : false; + (void)types; + (void)type_count; +} +#else +/* implemented in gc_type.c */ +bool +wasm_type_equal(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count); +#endif + +inline static uint32 +wasm_get_smallest_type_idx(const WASMTypePtr *types, uint32 type_count, + uint32 cur_type_idx) +{ + uint32 i; + + for (i = 0; i < cur_type_idx; i++) { + if (wasm_type_equal(types[cur_type_idx], types[i], types, type_count)) + return i; + } + return cur_type_idx; +} + +#if WASM_ENABLE_GC == 0 +static inline uint32 +block_type_get_param_types(BlockType *block_type, uint8 **p_param_types) +#else +static inline uint32 +block_type_get_param_types(BlockType *block_type, uint8 **p_param_types, + WASMRefTypeMap **p_param_reftype_maps, + uint32 *p_param_reftype_map_count) +#endif +{ + uint32 param_count = 0; + if (!block_type->is_value_type) { + WASMFuncType *func_type = block_type->u.type; + *p_param_types = func_type->types; + param_count = func_type->param_count; +#if WASM_ENABLE_GC != 0 + *p_param_reftype_maps = func_type->ref_type_maps; + *p_param_reftype_map_count = (uint32)(func_type->result_ref_type_maps + - func_type->ref_type_maps); +#endif + } + else { + *p_param_types = NULL; + param_count = 0; +#if WASM_ENABLE_GC != 0 + *p_param_reftype_maps = NULL; + *p_param_reftype_map_count = 0; +#endif + } + + return param_count; +} + +#if WASM_ENABLE_GC == 0 +static inline uint32 +block_type_get_result_types(BlockType *block_type, uint8 **p_result_types) +#else +static inline uint32 +block_type_get_result_types(BlockType *block_type, uint8 **p_result_types, + WASMRefTypeMap **p_result_reftype_maps, + uint32 *p_result_reftype_map_count) +#endif +{ + uint32 result_count = 0; + uint8 *result_types = NULL; +#if WASM_ENABLE_GC != 0 + uint8 type; + uint32 result_reftype_map_count = 0; + WASMRefTypeMap *result_reftype_maps = NULL; +#endif + + if (block_type->is_value_type) { + if (block_type->u.value_type.type != VALUE_TYPE_VOID) { + result_types = &block_type->u.value_type.type; + result_count = 1; +#if WASM_ENABLE_GC != 0 + type = block_type->u.value_type.type; + if (type == (uint8)REF_TYPE_HT_NULLABLE + || type == (uint8)REF_TYPE_HT_NON_NULLABLE) { + result_reftype_maps = &block_type->u.value_type.ref_type_map; + result_reftype_map_count = 1; + } +#endif + } + } + else { + WASMFuncType *func_type = block_type->u.type; + result_types = func_type->types + func_type->param_count; + result_count = func_type->result_count; +#if WASM_ENABLE_GC != 0 + result_reftype_maps = func_type->result_ref_type_maps; + result_reftype_map_count = (uint32)(func_type->ref_type_map_count + - (func_type->result_ref_type_maps + - func_type->ref_type_maps)); +#endif + } + *p_result_types = result_types; +#if WASM_ENABLE_GC != 0 + *p_result_reftype_maps = result_reftype_maps; + *p_result_reftype_map_count = result_reftype_map_count; +#endif + return result_count; +} + +static inline uint32 +block_type_get_arity(const BlockType *block_type, uint8 label_type) +{ + if (label_type == LABEL_TYPE_LOOP) { + if (block_type->is_value_type) + return 0; + else + return block_type->u.type->param_count; + } + else { + if (block_type->is_value_type) { + return block_type->u.value_type.type != VALUE_TYPE_VOID ? 1 : 0; + } + else + return block_type->u.type->result_count; + } + return 0; +} + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _WASM_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp.h b/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp.h new file mode 100644 index 0000000..1416405 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp.h @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_INTERP_H +#define _WASM_INTERP_H + +#include "wasm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct WASMModuleInstance; +struct WASMFunctionInstance; +struct WASMExecEnv; + +typedef struct WASMInterpFrame { + /* The frame of the caller that are calling the current function. */ + struct WASMInterpFrame *prev_frame; + + /* The current WASM function. */ + struct WASMFunctionInstance *function; + + /* Instruction pointer of the bytecode array. */ + uint8 *ip; + +#if WASM_ENABLE_FAST_JIT != 0 + uint8 *jitted_return_addr; +#endif + +#if WASM_ENABLE_PERF_PROFILING != 0 + uint64 time_started; +#endif + +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* set to true if the callee returns an exception rather than + * result values on the stack + */ + bool exception_raised; + uint32 tag_index; +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Return offset of the first return value of current frame, + the callee will put return values here continuously */ + uint32 ret_offset; + uint32 *lp; +#if WASM_ENABLE_GC != 0 + uint8 *frame_ref; +#endif + uint32 operand[1]; +#else /* else of WASM_ENABLE_FAST_INTERP != 0 */ + /* Operand stack top pointer of the current frame. The bottom of + the stack is the next cell after the last local variable. */ + uint32 *sp_bottom; + uint32 *sp_boundary; + uint32 *sp; + + WASMBranchBlock *csp_bottom; + WASMBranchBlock *csp_boundary; + WASMBranchBlock *csp; + + /** + * Frame data, the layout is: + * lp: parameters and local variables + * sp_bottom to sp_boundary: wasm operand stack + * csp_bottom to csp_boundary: wasm label stack + * frame ref flags: only available for GC + * whether each cell in local and stack area is a GC obj + * jit spill cache: only available for fast jit + */ + uint32 lp[1]; +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ +} WASMInterpFrame; + +/** + * Calculate the size of interpreter area of frame of a function. + * + * @param all_cell_num number of all cells including local variables + * and the working stack slots + * + * @return the size of interpreter area of the frame + */ +static inline unsigned +wasm_interp_interp_frame_size(unsigned all_cell_num) +{ + unsigned frame_size; + +#if WASM_ENABLE_FAST_INTERP == 0 +#if WASM_ENABLE_GC == 0 + frame_size = (uint32)offsetof(WASMInterpFrame, lp) + all_cell_num * 4; +#else + frame_size = + (uint32)offsetof(WASMInterpFrame, lp) + align_uint(all_cell_num * 5, 4); +#endif +#else + frame_size = (uint32)offsetof(WASMInterpFrame, operand) + all_cell_num * 4; +#endif + return align_uint(frame_size, 4); +} + +void +wasm_interp_call_wasm(struct WASMModuleInstance *module_inst, + struct WASMExecEnv *exec_env, + struct WASMFunctionInstance *function, uint32 argc, + uint32 argv[]); + +#if WASM_ENABLE_GC != 0 +bool +wasm_interp_traverse_gc_rootset(struct WASMExecEnv *exec_env, void *heap); + +uint8 * +wasm_interp_get_frame_ref(WASMInterpFrame *frame); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_INTERP_H */ diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp_classic.c b/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp_classic.c new file mode 100644 index 0000000..026e9dd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp_classic.c @@ -0,0 +1,7260 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_interp.h" +#include "bh_log.h" +#include "wasm_runtime.h" +#include "wasm_opcode.h" +#include "wasm_loader.h" +#include "wasm_memory.h" +#include "../common/wasm_exec_env.h" +#if WASM_ENABLE_GC != 0 +#include "../common/gc/gc_object.h" +#include "mem_alloc.h" +#if WASM_ENABLE_STRINGREF != 0 +#include "string_object.h" +#endif +#endif +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#include "../libraries/debug-engine/debug_engine.h" +#endif +#if WASM_ENABLE_FAST_JIT != 0 +#include "../fast-jit/jit_compiler.h" +#endif + +typedef int32 CellType_I32; +typedef int64 CellType_I64; +typedef float32 CellType_F32; +typedef float64 CellType_F64; + +#define BR_TABLE_TMP_BUF_LEN 32 + +#if WASM_ENABLE_THREAD_MGR == 0 +#define get_linear_mem_size() linear_mem_size +#else +/** + * Load memory data size in each time boundary check in + * multi-threading mode since it may be changed by other + * threads in memory.grow + */ +#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) +#endif + +#if WASM_ENABLE_MEMORY64 == 0 + +#if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ + /* If offset1 is in valid range, maddr must also \ + be in valid range, no need to check it again. */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ + /* App heap space is not valid space for \ + bulk memory operation */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) +#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ + WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + maddr = memory->memory_data + offset1; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + maddr = memory->memory_data + (uint32)(start); \ + } while (0) +#endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ + WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ + +#else /* else of WASM_ENABLE_MEMORY64 == 0 */ + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint64)(start); \ + /* If memory64 is enabled, offset1 + bytes can overflow */ \ + if (disable_bounds_checks \ + || (offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + /* App heap space is not valid space for \ + bulk memory operation */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) + +#endif /* end of WASM_ENABLE_MEMORY64 == 0 */ + +#define CHECK_ATOMIC_MEMORY_ACCESS() \ + do { \ + if (((uintptr_t)maddr & (((uintptr_t)1 << align) - 1)) != 0) \ + goto unaligned_atomic; \ + } while (0) + +#if WASM_ENABLE_DEBUG_INTERP != 0 +#define TRIGGER_WATCHPOINT_SIGTRAP() \ + do { \ + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TRAP); \ + CHECK_SUSPEND_FLAGS(); \ + } while (0) + +#define CHECK_WATCHPOINT(list, current_addr) \ + do { \ + WASMDebugWatchPoint *watchpoint = bh_list_first_elem(list); \ + while (watchpoint) { \ + WASMDebugWatchPoint *next = bh_list_elem_next(watchpoint); \ + if (watchpoint->addr <= current_addr \ + && watchpoint->addr + watchpoint->length > current_addr) { \ + TRIGGER_WATCHPOINT_SIGTRAP(); \ + } \ + watchpoint = next; \ + } \ + } while (0) + +#define CHECK_READ_WATCHPOINT(addr, offset) \ + CHECK_WATCHPOINT(watch_point_list_read, WASM_ADDR_OFFSET(addr + offset)) +#define CHECK_WRITE_WATCHPOINT(addr, offset) \ + CHECK_WATCHPOINT(watch_point_list_write, WASM_ADDR_OFFSET(addr + offset)) +#else +#define CHECK_READ_WATCHPOINT(addr, offset) (void)0 +#define CHECK_WRITE_WATCHPOINT(addr, offset) (void)0 +#endif + +static inline uint32 +rotl32(uint32 n, uint32 c) +{ + const uint32 mask = (31); + c = c % 32; + c &= mask; + return (n << c) | (n >> ((0 - c) & mask)); +} + +static inline uint32 +rotr32(uint32 n, uint32 c) +{ + const uint32 mask = (31); + c = c % 32; + c &= mask; + return (n >> c) | (n << ((0 - c) & mask)); +} + +static inline uint64 +rotl64(uint64 n, uint64 c) +{ + const uint64 mask = (63); + c = c % 64; + c &= mask; + return (n << c) | (n >> ((0 - c) & mask)); +} + +static inline uint64 +rotr64(uint64 n, uint64 c) +{ + const uint64 mask = (63); + c = c % 64; + c &= mask; + return (n >> c) | (n << ((0 - c) & mask)); +} + +static inline float32 +f32_min(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static inline float32 +f32_max(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static inline float64 +f64_min(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static inline float64 +f64_max(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static inline uint32 +clz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 0x80000000)) { + num++; + type <<= 1; + } + return num; +} + +static inline uint32 +clz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 0x8000000000000000LL)) { + num++; + type <<= 1; + } + return num; +} + +static inline uint32 +ctz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static inline uint32 +ctz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static inline uint32 +popcount32(uint32 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static inline uint32 +popcount64(uint64 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static float +local_copysignf(float x, float y) +{ + union { + float f; + uint32 i; + } ux = { x }, uy = { y }; + ux.i &= 0x7fffffff; + ux.i |= uy.i & 0x80000000; + return ux.f; +} + +static double +local_copysign(double x, double y) +{ + union { + double f; + uint64 i; + } ux = { x }, uy = { y }; + ux.i &= UINT64_MAX / 2; + ux.i |= uy.i & 1ULL << 63; + return ux.f; +} + +static uint64 +read_leb(const uint8 *buf, uint32 *p_offset, uint32 maxbits, bool sign) +{ + uint64 result = 0, byte; + uint32 offset = *p_offset; + uint32 shift = 0; + + while (true) { + byte = buf[offset++]; + result |= ((byte & 0x7f) << shift); + shift += 7; + if ((byte & 0x80) == 0) { + break; + } + } + if (sign && (shift < maxbits) && (byte & 0x40)) { + /* Sign extend */ + result |= (~((uint64)0)) << shift; + } + *p_offset = offset; + return result; +} + +#if WASM_ENABLE_GC != 0 +static uint8 * +get_frame_ref(WASMInterpFrame *frame) +{ + WASMFunctionInstance *cur_func = frame->function; + uint32 all_cell_num; + + if (!cur_func) { + /* it's a glue frame created in wasm_interp_call_wasm, + no GC object will be traversed */ + return (uint8 *)frame->lp; + } + else if (!frame->ip) { + /* it's a native method frame created in + wasm_interp_call_func_native */ + all_cell_num = + cur_func->param_cell_num > 2 ? cur_func->param_cell_num : 2; + return (uint8 *)(frame->lp + all_cell_num); + } + else { +#if WASM_ENABLE_JIT == 0 + /* it's a wasm bytecode function frame */ + return (uint8 *)frame->csp_boundary; +#else + return (uint8 *)(frame->lp + cur_func->param_cell_num + + cur_func->local_cell_num + + cur_func->u.func->max_stack_cell_num); +#endif + } +} + +static void +init_frame_refs(uint8 *frame_ref, uint32 cell_num, WASMFunctionInstance *func) +{ + uint32 i, j; + + memset(frame_ref, 0, cell_num); + + for (i = 0, j = 0; i < func->param_count; i++) { + if (wasm_is_type_reftype(func->param_types[i]) + && !wasm_is_reftype_i31ref(func->param_types[i])) { + frame_ref[j++] = 1; +#if UINTPTR_MAX == UINT64_MAX + frame_ref[j++] = 1; +#endif + } + else { + j += wasm_value_type_cell_num(func->param_types[i]); + } + } + + for (i = 0; i < func->local_count; i++) { + if (wasm_is_type_reftype(func->local_types[i]) + && !wasm_is_reftype_i31ref(func->local_types[i])) { + frame_ref[j++] = 1; +#if UINTPTR_MAX == UINT64_MAX + frame_ref[j++] = 1; +#endif + } + else { + j += wasm_value_type_cell_num(func->local_types[i]); + } + } +} + +uint8 * +wasm_interp_get_frame_ref(WASMInterpFrame *frame) +{ + return get_frame_ref(frame); +} + +/* Return the corresponding ref slot of the given address of local + variable or stack pointer. */ + +#define COMPUTE_FRAME_REF(ref, lp, p) (ref + (unsigned)((uint32 *)p - lp)) + +#define FRAME_REF(p) COMPUTE_FRAME_REF(frame_ref, frame_lp, p) + +#define FRAME_REF_FOR(frame, p) \ + COMPUTE_FRAME_REF(get_frame_ref(frame), frame->lp, p) + +#define CLEAR_FRAME_REF(p, n) \ + do { \ + int32 ref_i, ref_n = (int32)(n); \ + uint8 *ref = FRAME_REF(p); \ + for (ref_i = 0; ref_i < ref_n; ref_i++) \ + ref[ref_i] = 0; \ + } while (0) +#else +#define CLEAR_FRAME_REF(p, n) (void)0 +#endif /* end of WASM_ENABLE_GC != 0 */ + +#define skip_leb(p) while (*p++ & 0x80) + +#define PUSH_I32(value) \ + do { \ + *(int32 *)frame_sp++ = (int32)(value); \ + } while (0) + +#define PUSH_F32(value) \ + do { \ + *(float32 *)frame_sp++ = (float32)(value); \ + } while (0) + +#define PUSH_I64(value) \ + do { \ + PUT_I64_TO_ADDR(frame_sp, value); \ + frame_sp += 2; \ + } while (0) + +#define PUSH_F64(value) \ + do { \ + PUT_F64_TO_ADDR(frame_sp, value); \ + frame_sp += 2; \ + } while (0) + +#if UINTPTR_MAX == UINT64_MAX +#define PUSH_REF(value) \ + do { \ + PUT_REF_TO_ADDR(frame_sp, value); \ + frame_ref_tmp = FRAME_REF(frame_sp); \ + *frame_ref_tmp = *(frame_ref_tmp + 1) = 1; \ + frame_sp += 2; \ + } while (0) +#define PUSH_I31REF(value) \ + do { \ + PUT_REF_TO_ADDR(frame_sp, value); \ + frame_sp += 2; \ + } while (0) +#else +#define PUSH_REF(value) \ + do { \ + PUT_REF_TO_ADDR(frame_sp, value); \ + frame_ref_tmp = FRAME_REF(frame_sp); \ + *frame_ref_tmp = 1; \ + frame_sp++; \ + } while (0) +#define PUSH_I31REF(value) \ + do { \ + PUT_REF_TO_ADDR(frame_sp, value); \ + frame_sp++; \ + } while (0) +#endif + +/* in exception handling, label_type needs to be stored to lookup exception + * handlers */ + +#if WASM_ENABLE_EXCE_HANDLING != 0 +#define SET_LABEL_TYPE(_label_type) frame_csp->label_type = _label_type +#else +#define SET_LABEL_TYPE(_label_type) (void)0 +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +#define PUSH_MEM_OFFSET(value) \ + do { \ + if (is_memory64) { \ + PUT_I64_TO_ADDR(frame_sp, value); \ + frame_sp += 2; \ + } \ + else { \ + *(int32 *)frame_sp++ = (int32)(value); \ + } \ + } while (0) +#else +#define PUSH_MEM_OFFSET(value) PUSH_I32(value) +#endif + +#define PUSH_PAGE_COUNT(value) PUSH_MEM_OFFSET(value) + +#define PUSH_CSP(_label_type, param_cell_num, cell_num, _target_addr) \ + do { \ + bh_assert(frame_csp < frame->csp_boundary); \ + SET_LABEL_TYPE(_label_type); \ + frame_csp->cell_num = cell_num; \ + frame_csp->begin_addr = frame_ip; \ + frame_csp->target_addr = _target_addr; \ + frame_csp->frame_sp = frame_sp - param_cell_num; \ + frame_csp++; \ + } while (0) + +#define POP_I32() (--frame_sp, *(int32 *)frame_sp) + +#define POP_F32() (--frame_sp, *(float32 *)frame_sp) + +#define POP_I64() (frame_sp -= 2, GET_I64_FROM_ADDR(frame_sp)) + +#define POP_F64() (frame_sp -= 2, GET_F64_FROM_ADDR(frame_sp)) + +#if UINTPTR_MAX == UINT64_MAX +#define POP_REF() \ + (frame_sp -= 2, frame_ref_tmp = FRAME_REF(frame_sp), \ + *frame_ref_tmp = *(frame_ref_tmp + 1) = 0, GET_REF_FROM_ADDR(frame_sp)) +#else +#define POP_REF() \ + (frame_sp--, frame_ref_tmp = FRAME_REF(frame_sp), *frame_ref_tmp = 0, \ + GET_REF_FROM_ADDR(frame_sp)) +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +#define POP_MEM_OFFSET() (is_memory64 ? POP_I64() : POP_I32()) +#else +#define POP_MEM_OFFSET() POP_I32() +#endif + +#define POP_PAGE_COUNT() POP_MEM_OFFSET() + +#define POP_CSP_CHECK_OVERFLOW(n) \ + do { \ + bh_assert(frame_csp - n >= frame->csp_bottom); \ + } while (0) + +#define POP_CSP() \ + do { \ + POP_CSP_CHECK_OVERFLOW(1); \ + --frame_csp; \ + } while (0) + +#define POP_CSP_N(n) \ + do { \ + uint32 *frame_sp_old = frame_sp; \ + uint32 cell_num_to_copy; \ + POP_CSP_CHECK_OVERFLOW(n + 1); \ + frame_csp -= n; \ + frame_ip = (frame_csp - 1)->target_addr; \ + /* copy arity values of block */ \ + frame_sp = (frame_csp - 1)->frame_sp; \ + cell_num_to_copy = (frame_csp - 1)->cell_num; \ + if (cell_num_to_copy > 0) { \ + word_copy(frame_sp, frame_sp_old - cell_num_to_copy, \ + cell_num_to_copy); \ + frame_ref_copy(FRAME_REF(frame_sp), \ + FRAME_REF(frame_sp_old - cell_num_to_copy), \ + cell_num_to_copy); \ + } \ + frame_sp += cell_num_to_copy; \ + CLEAR_FRAME_REF(frame_sp, frame_sp_old - frame_sp); \ + } while (0) + +/* Pop the given number of elements from the given frame's stack. */ +#define POP(N) \ + do { \ + int n = (N); \ + frame_sp -= n; \ + CLEAR_FRAME_REF(frame_sp, n); \ + } while (0) + +#if WASM_ENABLE_EXCE_HANDLING != 0 +/* unwind the CSP to a given label and optionally modify the labeltype */ +#define UNWIND_CSP(N, T) \ + do { \ + /* unwind to function frame */ \ + frame_csp -= N; \ + /* drop handlers and values pushd in try block */ \ + frame_sp = (frame_csp - 1)->frame_sp; \ + (frame_csp - 1)->label_type = T ? T : (frame_csp - 1)->label_type; \ + } while (0) +#endif + +#define SYNC_ALL_TO_FRAME() \ + do { \ + frame->sp = frame_sp; \ + frame->ip = frame_ip; \ + frame->csp = frame_csp; \ + } while (0) + +#define UPDATE_ALL_FROM_FRAME() \ + do { \ + frame_sp = frame->sp; \ + frame_ip = frame->ip; \ + frame_csp = frame->csp; \ + } while (0) + +#define read_leb_int64(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = (int64)_val; \ + if (_val & 0x40) \ + /* sign extend */ \ + res |= 0xFFFFFFFFFFFFFF80LL; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (int64)read_leb(p, &_off, 64, true); \ + p += _off; \ + } \ + } while (0) + +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = _val; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (uint32)read_leb(p, &_off, 32, false); \ + p += _off; \ + } \ + } while (0) + +#define read_leb_int32(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = (int32)_val; \ + if (_val & 0x40) \ + /* sign extend */ \ + res |= 0xFFFFFF80; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (int32)read_leb(p, &_off, 32, true); \ + p += _off; \ + } \ + } while (0) + +#if WASM_ENABLE_MEMORY64 != 0 +#define read_leb_mem_offset(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = (mem_offset_t)_val; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (mem_offset_t)read_leb(p, &_off, is_memory64 ? 64 : 32, \ + false); \ + p += _off; \ + } \ + } while (0) +#else +#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 +#define RECOVER_FRAME_IP_END() frame_ip_end = wasm_get_func_code_end(cur_func) +#else +#define RECOVER_FRAME_IP_END() (void)0 +#endif + +#if WASM_ENABLE_GC != 0 +#define RECOVER_FRAME_REF() frame_ref = (uint8 *)frame->csp_boundary +#else +#define RECOVER_FRAME_REF() (void)0 +#endif + +#define RECOVER_CONTEXT(new_frame) \ + do { \ + frame = (new_frame); \ + cur_func = frame->function; \ + prev_frame = frame->prev_frame; \ + frame_ip = frame->ip; \ + RECOVER_FRAME_IP_END(); \ + frame_lp = frame->lp; \ + frame_sp = frame->sp; \ + frame_csp = frame->csp; \ + RECOVER_FRAME_REF(); \ + } while (0) + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#define GET_OPCODE() opcode = *(frame_ip - 1); +#else +#define GET_OPCODE() (void)0 +#endif + +#define DEF_OP_I_CONST(ctype, src_op_type) \ + do { \ + ctype cval; \ + read_leb_##ctype(frame_ip, frame_ip_end, cval); \ + PUSH_##src_op_type(cval); \ + } while (0) + +#define DEF_OP_EQZ(src_op_type) \ + do { \ + int32 pop_val; \ + pop_val = POP_##src_op_type() == 0; \ + PUSH_I32(pop_val); \ + } while (0) + +#define DEF_OP_CMP(src_type, src_op_type, cond) \ + do { \ + uint32 res; \ + src_type val1, val2; \ + val2 = (src_type)POP_##src_op_type(); \ + val1 = (src_type)POP_##src_op_type(); \ + res = val1 cond val2; \ + PUSH_I32(res); \ + } while (0) + +#define DEF_OP_BIT_COUNT(src_type, src_op_type, operation) \ + do { \ + src_type val1, val2; \ + val1 = (src_type)POP_##src_op_type(); \ + val2 = (src_type)operation(val1); \ + PUSH_##src_op_type(val2); \ + } while (0) + +#define DEF_OP_NUMERIC(src_type1, src_type2, src_op_type, operation) \ + do { \ + frame_sp -= sizeof(src_type2) / sizeof(uint32); \ + *(src_type1 *)(frame_sp - sizeof(src_type1) / sizeof(uint32)) \ + operation## = *(src_type2 *)(frame_sp); \ + } while (0) + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define DEF_OP_NUMERIC_64 DEF_OP_NUMERIC +#else +#define DEF_OP_NUMERIC_64(src_type1, src_type2, src_op_type, operation) \ + do { \ + src_type1 val1; \ + src_type2 val2; \ + frame_sp -= 2; \ + val1 = (src_type1)GET_##src_op_type##_FROM_ADDR(frame_sp - 2); \ + val2 = (src_type2)GET_##src_op_type##_FROM_ADDR(frame_sp); \ + val1 operation## = val2; \ + PUT_##src_op_type##_TO_ADDR(frame_sp - 2, val1); \ + } while (0) +#endif + +#define DEF_OP_NUMERIC2(src_type1, src_type2, src_op_type, operation) \ + do { \ + frame_sp -= sizeof(src_type2) / sizeof(uint32); \ + *(src_type1 *)(frame_sp - sizeof(src_type1) / sizeof(uint32)) \ + operation## = (*(src_type2 *)(frame_sp) % 32); \ + } while (0) + +#define DEF_OP_NUMERIC2_64(src_type1, src_type2, src_op_type, operation) \ + do { \ + src_type1 val1; \ + src_type2 val2; \ + frame_sp -= 2; \ + val1 = (src_type1)GET_##src_op_type##_FROM_ADDR(frame_sp - 2); \ + val2 = (src_type2)GET_##src_op_type##_FROM_ADDR(frame_sp); \ + val1 operation## = (val2 % 64); \ + PUT_##src_op_type##_TO_ADDR(frame_sp - 2, val1); \ + } while (0) + +#define DEF_OP_MATH(src_type, src_op_type, method) \ + do { \ + src_type src_val; \ + src_val = POP_##src_op_type(); \ + PUSH_##src_op_type(method(src_val)); \ + } while (0) + +#define TRUNC_FUNCTION(func_name, src_type, dst_type, signed_type) \ + static dst_type func_name(src_type src_value, src_type src_min, \ + src_type src_max, dst_type dst_min, \ + dst_type dst_max, bool is_sign) \ + { \ + dst_type dst_value = 0; \ + if (!isnan(src_value)) { \ + if (src_value <= src_min) \ + dst_value = dst_min; \ + else if (src_value >= src_max) \ + dst_value = dst_max; \ + else { \ + if (is_sign) \ + dst_value = (dst_type)(signed_type)src_value; \ + else \ + dst_value = (dst_type)src_value; \ + } \ + } \ + return dst_value; \ + } + +TRUNC_FUNCTION(trunc_f32_to_i32, float32, uint32, int32) +TRUNC_FUNCTION(trunc_f32_to_i64, float32, uint64, int64) +TRUNC_FUNCTION(trunc_f64_to_i32, float64, uint32, int32) +TRUNC_FUNCTION(trunc_f64_to_i64, float64, uint64, int64) + +static bool +trunc_f32_to_int(WASMModuleInstance *module, uint32 *frame_sp, float32 src_min, + float32 src_max, bool saturating, bool is_i32, bool is_sign) +{ + float32 src_value = POP_F32(); + uint64 dst_value_i64; + uint32 dst_value_i32; + + if (!saturating) { + if (isnan(src_value)) { + wasm_set_exception(module, "invalid conversion to integer"); + return false; + } + else if (src_value <= src_min || src_value >= src_max) { + wasm_set_exception(module, "integer overflow"); + return false; + } + } + + if (is_i32) { + uint32 dst_min = is_sign ? INT32_MIN : 0; + uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX; + dst_value_i32 = trunc_f32_to_i32(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + PUSH_I32(dst_value_i32); + } + else { + uint64 dst_min = is_sign ? INT64_MIN : 0; + uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX; + dst_value_i64 = trunc_f32_to_i64(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + PUSH_I64(dst_value_i64); + } + return true; +} + +static bool +trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min, + float64 src_max, bool saturating, bool is_i32, bool is_sign) +{ + float64 src_value = POP_F64(); + uint64 dst_value_i64; + uint32 dst_value_i32; + + if (!saturating) { + if (isnan(src_value)) { + wasm_set_exception(module, "invalid conversion to integer"); + return false; + } + else if (src_value <= src_min || src_value >= src_max) { + wasm_set_exception(module, "integer overflow"); + return false; + } + } + + if (is_i32) { + uint32 dst_min = is_sign ? INT32_MIN : 0; + uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX; + dst_value_i32 = trunc_f64_to_i32(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + PUSH_I32(dst_value_i32); + } + else { + uint64 dst_min = is_sign ? INT64_MIN : 0; + uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX; + dst_value_i64 = trunc_f64_to_i64(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + PUSH_I64(dst_value_i64); + } + return true; +} + +#define DEF_OP_TRUNC_F32(min, max, is_i32, is_sign) \ + do { \ + if (!trunc_f32_to_int(module, frame_sp, min, max, false, is_i32, \ + is_sign)) \ + goto got_exception; \ + } while (0) + +#define DEF_OP_TRUNC_F64(min, max, is_i32, is_sign) \ + do { \ + if (!trunc_f64_to_int(module, frame_sp, min, max, false, is_i32, \ + is_sign)) \ + goto got_exception; \ + } while (0) + +#define DEF_OP_TRUNC_SAT_F32(min, max, is_i32, is_sign) \ + do { \ + (void)trunc_f32_to_int(module, frame_sp, min, max, true, is_i32, \ + is_sign); \ + } while (0) + +#define DEF_OP_TRUNC_SAT_F64(min, max, is_i32, is_sign) \ + do { \ + (void)trunc_f64_to_int(module, frame_sp, min, max, true, is_i32, \ + is_sign); \ + } while (0) + +#define DEF_OP_CONVERT(dst_type, dst_op_type, src_type, src_op_type) \ + do { \ + dst_type value = (dst_type)(src_type)POP_##src_op_type(); \ + PUSH_##dst_op_type(value); \ + } while (0) + +#define GET_LOCAL_INDEX_TYPE_AND_OFFSET() \ + do { \ + uint32 param_count = cur_func->param_count; \ + read_leb_uint32(frame_ip, frame_ip_end, local_idx); \ + bh_assert(local_idx < param_count + cur_func->local_count); \ + local_offset = cur_func->local_offsets[local_idx]; \ + if (local_idx < param_count) \ + local_type = cur_func->param_types[local_idx]; \ + else \ + local_type = cur_func->local_types[local_idx - param_count]; \ + } while (0) + +#define DEF_ATOMIC_RMW_OPCODE(OP_NAME, op) \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U: \ + { \ + uint32 readv, sval; \ + \ + sval = POP_I32(); \ + addr = POP_MEM_OFFSET(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \ + CHECK_MEMORY_OVERFLOW(1); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + shared_memory_lock(memory); \ + readv = (uint32)(*(uint8 *)maddr); \ + *(uint8 *)maddr = (uint8)(readv op sval); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ + CHECK_MEMORY_OVERFLOW(2); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + shared_memory_lock(memory); \ + readv = (uint32)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else { \ + CHECK_MEMORY_OVERFLOW(4); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + shared_memory_lock(memory); \ + readv = LOAD_I32(maddr); \ + STORE_U32(maddr, readv op sval); \ + shared_memory_unlock(memory); \ + } \ + PUSH_I32(readv); \ + break; \ + } \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U: \ + { \ + uint64 readv, sval; \ + \ + sval = (uint64)POP_I64(); \ + addr = POP_MEM_OFFSET(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \ + CHECK_MEMORY_OVERFLOW(1); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)(*(uint8 *)maddr); \ + *(uint8 *)maddr = (uint8)(readv op sval); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ + CHECK_MEMORY_OVERFLOW(2); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ + CHECK_MEMORY_OVERFLOW(4); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_U32(maddr); \ + STORE_U32(maddr, (uint32)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else { \ + uint64 op_result; \ + CHECK_MEMORY_OVERFLOW(8); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_I64(maddr); \ + op_result = readv op sval; \ + STORE_I64(maddr, op_result); \ + shared_memory_unlock(memory); \ + } \ + PUSH_I64(readv); \ + break; \ + } + +static inline int32 +sign_ext_8_32(int8 val) +{ + if (val & 0x80) + return (int32)val | (int32)0xffffff00; + return val; +} + +static inline int32 +sign_ext_16_32(int16 val) +{ + if (val & 0x8000) + return (int32)val | (int32)0xffff0000; + return val; +} + +static inline int64 +sign_ext_8_64(int8 val) +{ + if (val & 0x80) + return (int64)val | (int64)0xffffffffffffff00LL; + return val; +} + +static inline int64 +sign_ext_16_64(int16 val) +{ + if (val & 0x8000) + return (int64)val | (int64)0xffffffffffff0000LL; + return val; +} + +static inline int64 +sign_ext_32_64(int32 val) +{ + if (val & (int32)0x80000000) + return (int64)val | (int64)0xffffffff00000000LL; + return val; +} + +static inline void +word_copy(uint32 *dest, uint32 *src, unsigned num) +{ + bh_assert(dest != NULL); + bh_assert(src != NULL); + bh_assert(num > 0); + if (dest != src) { + /* No overlap buffer */ + bh_assert(!((src < dest) && (dest < src + num))); + for (; num > 0; num--) + *dest++ = *src++; + } +} + +#if WASM_ENABLE_GC != 0 +static inline void +frame_ref_copy(uint8 *frame_ref_dest, uint8 *frame_ref_src, unsigned num) +{ + if (frame_ref_dest != frame_ref_src) + for (; num > 0; num--) + *frame_ref_dest++ = *frame_ref_src++; +} +#else +#define frame_ref_copy(frame_ref_dst, frame_ref_src, num) (void)0 +#endif + +static inline WASMInterpFrame * +ALLOC_FRAME(WASMExecEnv *exec_env, uint32 size, WASMInterpFrame *prev_frame) +{ + WASMInterpFrame *frame = wasm_exec_env_alloc_wasm_frame(exec_env, size); + + if (frame) { + frame->prev_frame = prev_frame; +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = os_time_thread_cputime_us(); +#endif + } + else { + wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, + "wasm operand stack overflow"); + } + + return frame; +} + +static inline void +FREE_FRAME(WASMExecEnv *exec_env, WASMInterpFrame *frame) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + if (frame->function) { + WASMInterpFrame *prev_frame = frame->prev_frame; + uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started; + + frame->function->total_exec_time += time_elapsed; + frame->function->total_exec_cnt++; + + if (prev_frame && prev_frame->function) + prev_frame->function->children_exec_time += time_elapsed; + } +#endif + wasm_exec_env_free_wasm_frame(exec_env, frame); +} + +static void +wasm_interp_call_func_native(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMFunctionImport *func_import = cur_func->u.func_import; + CApiFuncImport *c_api_func_import = NULL; + unsigned local_cell_num = + cur_func->param_cell_num > 2 ? cur_func->param_cell_num : 2; + unsigned all_cell_num; + WASMInterpFrame *frame; + uint32 argv_ret[2], cur_func_index; + void *native_func_pointer = NULL; + char buf[128]; + bool ret; +#if WASM_ENABLE_GC != 0 + WASMFuncType *func_type; + uint8 *frame_ref; +#endif + + all_cell_num = local_cell_num; +#if WASM_ENABLE_GC != 0 + all_cell_num += (local_cell_num + 3) / 4; +#endif + + if (!(frame = + ALLOC_FRAME(exec_env, wasm_interp_interp_frame_size(all_cell_num), + prev_frame))) + return; + + frame->function = cur_func; + frame->ip = NULL; + frame->sp = frame->lp + local_cell_num; +#if WASM_ENABLE_GC != 0 + /* native function doesn't have operand stack and label stack */ + frame_ref = (uint8 *)frame->sp; + init_frame_refs(frame_ref, local_cell_num, cur_func); +#endif + + wasm_exec_env_set_cur_frame(exec_env, frame); + + cur_func_index = (uint32)(cur_func - module_inst->e->functions); + bh_assert(cur_func_index < module_inst->module->import_function_count); + if (!func_import->call_conv_wasm_c_api) { + native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; + } + else if (module_inst->c_api_func_imports) { + c_api_func_import = module_inst->c_api_func_imports + cur_func_index; + native_func_pointer = c_api_func_import->func_ptr_linked; + } + + if (!native_func_pointer) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + func_import->module_name, func_import->field_name); + wasm_set_exception(module_inst, buf); + return; + } + + if (func_import->call_conv_wasm_c_api) { + ret = wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, native_func_pointer, + func_import->func_type, cur_func->param_cell_num, frame->lp, + c_api_func_import->with_env_arg, c_api_func_import->env_arg); + if (ret) { + argv_ret[0] = frame->lp[0]; + argv_ret[1] = frame->lp[1]; + } + } + else if (!func_import->call_conv_raw) { + ret = wasm_runtime_invoke_native( + exec_env, native_func_pointer, func_import->func_type, + func_import->signature, func_import->attachment, frame->lp, + cur_func->param_cell_num, argv_ret); + } + else { + ret = wasm_runtime_invoke_native_raw( + exec_env, native_func_pointer, func_import->func_type, + func_import->signature, func_import->attachment, frame->lp, + cur_func->param_cell_num, argv_ret); + } + + if (!ret) + return; + +#if WASM_ENABLE_GC != 0 + func_type = cur_func->u.func_import->func_type; + if (func_type->result_count + && wasm_is_type_reftype(func_type->types[cur_func->param_count])) { + frame_ref = (uint8 *)prev_frame->csp_boundary + + (unsigned)(uintptr_t)(prev_frame->sp - prev_frame->lp); + if (!wasm_is_reftype_i31ref(func_type->types[cur_func->param_count])) { +#if UINTPTR_MAX == UINT64_MAX + *frame_ref = *(frame_ref + 1) = 1; +#else + *frame_ref = 1; +#endif + } + } +#endif + + if (cur_func->ret_cell_num == 1) { + prev_frame->sp[0] = argv_ret[0]; + prev_frame->sp++; + } + else if (cur_func->ret_cell_num == 2) { + prev_frame->sp[0] = argv_ret[0]; + prev_frame->sp[1] = argv_ret[1]; + prev_frame->sp += 2; + } + + FREE_FRAME(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); +} + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, + WASMInterpFrame *prev_frame) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + WASMFunctionInstance *cur_func = module_inst->e->functions + func_idx; + + wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame); + return wasm_copy_exception(module_inst, NULL) ? false : true; +} +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +static void +wasm_interp_call_func_bytecode(WASMModuleInstance *module, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame); + +static void +wasm_interp_call_func_import(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMModuleInstance *sub_module_inst = cur_func->import_module_inst; + WASMFunctionInstance *sub_func_inst = cur_func->import_func_inst; + WASMFunctionImport *func_import = cur_func->u.func_import; + uint8 *ip = prev_frame->ip; + char buf[128]; + WASMExecEnv *sub_module_exec_env = NULL; + uintptr_t aux_stack_origin_boundary = 0; + uintptr_t aux_stack_origin_bottom = 0; + + if (!sub_func_inst) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + func_import->module_name, func_import->field_name); + wasm_set_exception(module_inst, buf); + return; + } + + /* Switch exec_env but keep using the same one by replacing necessary + * variables */ + sub_module_exec_env = wasm_runtime_get_exec_env_singleton( + (WASMModuleInstanceCommon *)sub_module_inst); + if (!sub_module_exec_env) { + wasm_set_exception(module_inst, "create singleton exec_env failed"); + return; + } + + /* - module_inst */ + wasm_exec_env_set_module_inst(exec_env, + (WASMModuleInstanceCommon *)sub_module_inst); + /* - aux_stack_boundary */ + aux_stack_origin_boundary = exec_env->aux_stack_boundary; + exec_env->aux_stack_boundary = sub_module_exec_env->aux_stack_boundary; + /* - aux_stack_bottom */ + aux_stack_origin_bottom = exec_env->aux_stack_bottom; + exec_env->aux_stack_bottom = sub_module_exec_env->aux_stack_bottom; + + /* set ip NULL to make call_func_bytecode return after executing + this function */ + prev_frame->ip = NULL; + + /* call function of sub-module*/ + wasm_interp_call_func_bytecode(sub_module_inst, exec_env, sub_func_inst, + prev_frame); + + /* restore ip and other replaced */ + prev_frame->ip = ip; + exec_env->aux_stack_boundary = aux_stack_origin_boundary; + exec_env->aux_stack_bottom = aux_stack_origin_bottom; + wasm_exec_env_restore_module_inst(exec_env, + (WASMModuleInstanceCommon *)module_inst); +} +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +#if WASM_ENABLE_DEBUG_INTERP != 0 +#define CHECK_SUSPEND_FLAGS() \ + do { \ + os_mutex_lock(&exec_env->wait_lock); \ + if (IS_WAMR_TERM_SIG(exec_env->current_status->signal_flag)) { \ + os_mutex_unlock(&exec_env->wait_lock); \ + return; \ + } \ + if (IS_WAMR_STOP_SIG(exec_env->current_status->signal_flag)) { \ + SYNC_ALL_TO_FRAME(); \ + wasm_cluster_thread_waiting_run(exec_env); \ + } \ + os_mutex_unlock(&exec_env->wait_lock); \ + } while (0) +#else +#if WASM_SUSPEND_FLAGS_IS_ATOMIC != 0 +/* The lock is only needed when the suspend_flags is atomic; otherwise + the lock is already taken at the time when SUSPENSION_LOCK() is called. */ +#define SUSPENSION_LOCK() os_mutex_lock(&exec_env->wait_lock); +#define SUSPENSION_UNLOCK() os_mutex_unlock(&exec_env->wait_lock); +#else +#define SUSPENSION_LOCK() +#define SUSPENSION_UNLOCK() +#endif + +#define CHECK_SUSPEND_FLAGS() \ + do { \ + WASM_SUSPEND_FLAGS_LOCK(exec_env->wait_lock); \ + if (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) \ + & WASM_SUSPEND_FLAG_TERMINATE) { \ + /* terminate current thread */ \ + WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \ + return; \ + } \ + while (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) \ + & WASM_SUSPEND_FLAG_SUSPEND) { \ + /* suspend current thread */ \ + SUSPENSION_LOCK() \ + os_cond_wait(&exec_env->wait_cond, &exec_env->wait_lock); \ + SUSPENSION_UNLOCK() \ + } \ + WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \ + } while (0) +#endif /* WASM_ENABLE_DEBUG_INTERP */ +#endif /* WASM_ENABLE_THREAD_MGR */ + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 + +#define HANDLE_OP(opcode) HANDLE_##opcode: +#define FETCH_OPCODE_AND_DISPATCH() goto *handle_table[*frame_ip++] + +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0 +#define HANDLE_OP_END() \ + do { \ + /* Record the current frame_ip, so when exception occurs, \ + debugger can know the exact opcode who caused the exception */ \ + frame_ip_orig = frame_ip; \ + os_mutex_lock(&exec_env->wait_lock); \ + while (exec_env->current_status->signal_flag == WAMR_SIG_SINGSTEP \ + && exec_env->current_status->step_count++ == 1) { \ + exec_env->current_status->step_count = 0; \ + SYNC_ALL_TO_FRAME(); \ + wasm_cluster_thread_waiting_run(exec_env); \ + } \ + os_mutex_unlock(&exec_env->wait_lock); \ + goto *handle_table[*frame_ip++]; \ + } while (0) +#else +#define HANDLE_OP_END() FETCH_OPCODE_AND_DISPATCH() +#endif + +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#define HANDLE_OP(opcode) case opcode: +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0 +#define HANDLE_OP_END() \ + os_mutex_lock(&exec_env->wait_lock); \ + if (exec_env->current_status->signal_flag == WAMR_SIG_SINGSTEP \ + && exec_env->current_status->step_count++ == 2) { \ + exec_env->current_status->step_count = 0; \ + SYNC_ALL_TO_FRAME(); \ + wasm_cluster_thread_waiting_run(exec_env); \ + } \ + os_mutex_unlock(&exec_env->wait_lock); \ + continue +#else +#define HANDLE_OP_END() continue +#endif + +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + +static inline uint8 * +get_global_addr(uint8 *global_data, WASMGlobalInstance *global) +{ +#if WASM_ENABLE_MULTI_MODULE == 0 + return global_data + global->data_offset; +#else + return global->import_global_inst + ? global->import_module_inst->global_data + + global->import_global_inst->data_offset + : global_data + global->data_offset; +#endif +} + +static void +wasm_interp_call_func_bytecode(WASMModuleInstance *module, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + uint64 linear_mem_size = 0; + if (memory) +#if WASM_ENABLE_THREAD_MGR == 0 + linear_mem_size = memory->memory_data_size; +#else + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif +#endif + WASMFuncType **wasm_types = (WASMFuncType **)module->module->types; + WASMGlobalInstance *globals = module->e->globals, *global; + uint8 *global_data = module->global_data; + uint8 opcode_IMPDEP = WASM_OP_IMPDEP; + WASMInterpFrame *frame = NULL; + /* Points to this special opcode so as to jump to the + * call_method_from_entry. */ + register uint8 *frame_ip = &opcode_IMPDEP; /* cache of frame->ip */ + register uint32 *frame_lp = NULL; /* cache of frame->lp */ + register uint32 *frame_sp = NULL; /* cache of frame->sp */ +#if WASM_ENABLE_GC != 0 + register uint8 *frame_ref = NULL; /* cache of frame->ref */ + uint8 *frame_ref_tmp; +#endif + WASMBranchBlock *frame_csp = NULL; + BlockAddr *cache_items; + uint8 *frame_ip_end = frame_ip + 1; + uint8 opcode; + uint32 i, depth, cond, count, fidx, tidx, lidx, frame_size = 0; + uint32 all_cell_num = 0; + int32 val; + uint8 *else_addr, *end_addr, *maddr = NULL; + uint32 local_idx, local_offset, global_idx; + uint8 local_type, *global_addr; + uint32 cache_index, type_index, param_cell_num, cell_num; +#if WASM_ENABLE_EXCE_HANDLING != 0 + int32_t exception_tag_index; +#endif + uint8 value_type; +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + bool disable_bounds_checks = !wasm_runtime_is_bounds_checks_enabled( + (WASMModuleInstanceCommon *)module); +#else + bool disable_bounds_checks = false; +#endif +#endif +#if WASM_ENABLE_GC != 0 + WASMObjectRef gc_obj; + WASMStructObjectRef struct_obj; + WASMArrayObjectRef array_obj; + WASMFuncObjectRef func_obj; + WASMI31ObjectRef i31_obj; + WASMExternrefObjectRef externref_obj; +#if WASM_ENABLE_STRINGREF != 0 + WASMString str_obj = NULL; + WASMStringrefObjectRef stringref_obj; + WASMStringviewWTF8ObjectRef stringview_wtf8_obj; + WASMStringviewWTF16ObjectRef stringview_wtf16_obj; + WASMStringviewIterObjectRef stringview_iter_obj; +#endif +#endif +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + bool is_return_call = false; +#endif +#if WASM_ENABLE_MEMORY64 != 0 + /* TODO: multi-memories for now assuming the memory idx type is consistent + * across multi-memories */ + bool is_memory64 = false; + if (memory) + is_memory64 = memory->is_memory64; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + uint8 *frame_ip_orig = NULL; + WASMDebugInstance *debug_instance = wasm_exec_env_get_instance(exec_env); + bh_list *watch_point_list_read = + debug_instance ? &debug_instance->watch_point_list_read : NULL; + bh_list *watch_point_list_write = + debug_instance ? &debug_instance->watch_point_list_write : NULL; +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#define HANDLE_OPCODE(op) &&HANDLE_##op + DEFINE_GOTO_TABLE(const void *, handle_table); +#undef HANDLE_OPCODE +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + while (frame_ip < frame_ip_end) { + opcode = *frame_ip++; + switch (opcode) { +#else + FETCH_OPCODE_AND_DISPATCH(); +#endif + /* control instructions */ + HANDLE_OP(WASM_OP_UNREACHABLE) + { + wasm_set_exception(module, "unreachable"); + goto got_exception; + } + + HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } + +#if WASM_ENABLE_EXCE_HANDLING != 0 + HANDLE_OP(WASM_OP_RETHROW) + { + int32_t relative_depth; + read_leb_int32(frame_ip, frame_ip_end, relative_depth); + + /* No frame found with exception handler; validation should + * catch it */ + bh_assert(frame_csp >= frame->csp_bottom + relative_depth); + + /* go up the frame stack */ + WASMBranchBlock *tgtframe = (frame_csp - 1) - relative_depth; + + bh_assert(tgtframe->label_type == LABEL_TYPE_CATCH + || tgtframe->label_type == LABEL_TYPE_CATCH_ALL); + + /* tgtframe points to the frame containing a thrown + * exception */ + + uint32 *tgtframe_sp = tgtframe->frame_sp; + + /* frame sp of tgtframe points to catched exception */ + exception_tag_index = *((uint32 *)tgtframe_sp); + tgtframe_sp++; + + /* get tag type */ + uint8 tag_type_index = + module->module->tags[exception_tag_index]->type; + uint32 cell_num_to_copy = + wasm_types[tag_type_index]->param_cell_num; + + /* move exception parameters (if there are any) onto top + * of stack */ + if (cell_num_to_copy > 0) { + word_copy(frame_sp, tgtframe_sp - cell_num_to_copy, + cell_num_to_copy); + } + + frame_sp += cell_num_to_copy; + goto find_a_catch_handler; + } + + HANDLE_OP(WASM_OP_THROW) + { + read_leb_int32(frame_ip, frame_ip_end, exception_tag_index); + + /* landing pad for the rethrow ? */ + find_a_catch_handler: + { + WASMType *tag_type = NULL; + uint32 cell_num_to_copy = 0; + if (IS_INVALID_TAGINDEX(exception_tag_index)) { + /* + * invalid exception index, + * generated if a submodule throws an exception + * that has not been imported here + * + * This should result in a branch to the CATCH_ALL block, + * if there is one + */ + tag_type = NULL; + cell_num_to_copy = 0; + } + else { + if (module->e->tags[exception_tag_index].is_import_tag) { + tag_type = module->e->tags[exception_tag_index] + .u.tag_import->tag_type; + } + else { + tag_type = module->e->tags[exception_tag_index] + .u.tag->tag_type; + } + cell_num_to_copy = tag_type->param_cell_num; + } + + /* browse through frame stack */ + uint32 relative_depth = 0; + do { + POP_CSP_CHECK_OVERFLOW(relative_depth - 1); + WASMBranchBlock *tgtframe = frame_csp - relative_depth - 1; + + switch (tgtframe->label_type) { + case LABEL_TYPE_BLOCK: + case LABEL_TYPE_IF: + case LABEL_TYPE_LOOP: + case LABEL_TYPE_CATCH: + case LABEL_TYPE_CATCH_ALL: + /* + * skip that blocks in search + * BLOCK, IF and LOOP do not contain handlers and + * cannot catch exceptions. + * blocks marked as CATCH or + * CATCH_ALL did already caugth an exception and can + * only be a target for RETHROW, but cannot catch an + * exception again + */ + break; + case LABEL_TYPE_TRY: + { + uint32 handler_number = 0; + uint8 **handlers = (uint8 **)tgtframe->frame_sp; + uint8 *handler = NULL; + while ((handler = handlers[handler_number]) != 0) { + uint8 handler_opcode = *handler; + uint8 *target_addr = + handler + + 1; /* first instruction or leb-immediate + behind the handler opcode */ + switch (handler_opcode) { + case WASM_OP_CATCH: + { + int32 lookup_index = 0; + /* read the tag_index and advance + * target_addr to the first instruction + * in the block */ + read_leb_int32(target_addr, 0, + lookup_index); + + if (exception_tag_index + == lookup_index) { + /* set ip */ + frame_ip = target_addr; + /* save frame_sp (points to + * exception values) */ + uint32 *frame_sp_old = frame_sp; + + UNWIND_CSP(relative_depth, + LABEL_TYPE_CATCH); + + /* push exception_tag_index and + * exception values for rethrow */ + PUSH_I32(exception_tag_index); + if (cell_num_to_copy > 0) { + word_copy( + frame_sp, + frame_sp_old + - cell_num_to_copy, + cell_num_to_copy); + frame_sp += cell_num_to_copy; + /* push exception values for + * catch + */ + word_copy( + frame_sp, + frame_sp_old + - cell_num_to_copy, + cell_num_to_copy); + frame_sp += cell_num_to_copy; + } + + /* advance to handler */ + HANDLE_OP_END(); + } + break; + } + case WASM_OP_DELEGATE: + { + int32 lookup_depth = 0; + /* read the depth */ + read_leb_int32(target_addr, 0, + lookup_depth); + + /* save frame_sp (points to exception + * values) */ + uint32 *frame_sp_old = frame_sp; + + UNWIND_CSP(relative_depth, + LABEL_TYPE_CATCH); + + /* leave the block (the delegate is + * technically not inside the frame) */ + frame_csp--; + + /* unwind to delegated frame */ + frame_csp -= lookup_depth; + + /* push exception values for catch */ + if (cell_num_to_copy > 0) { + word_copy(frame_sp, + frame_sp_old + - cell_num_to_copy, + cell_num_to_copy); + frame_sp += cell_num_to_copy; + } + + /* tag_index is already stored in + * exception_tag_index */ + goto find_a_catch_handler; + } + case WASM_OP_CATCH_ALL: + { + /* no immediate */ + /* save frame_sp (points to exception + * values) */ + uint32 *frame_sp_old = frame_sp; + /* set ip */ + frame_ip = target_addr; + + UNWIND_CSP(relative_depth, + LABEL_TYPE_CATCH_ALL); + + /* push exception_tag_index and + * exception values for rethrow */ + PUSH_I32(exception_tag_index); + if (cell_num_to_copy > 0) { + word_copy(frame_sp, + frame_sp_old + - cell_num_to_copy, + cell_num_to_copy); + frame_sp += cell_num_to_copy; + } + /* catch_all has no exception values */ + + /* advance to handler */ + HANDLE_OP_END(); + } + default: + wasm_set_exception( + module, "WASM_OP_THROW found " + "unexpected handler type"); + goto got_exception; + } + handler_number++; + } + /* exception not catched in this frame */ + break; + } + case LABEL_TYPE_FUNCTION: + { + /* save frame_sp (points to exception values) */ + uint32 *frame_sp_old = frame_sp; + + UNWIND_CSP(relative_depth, LABEL_TYPE_FUNCTION); + /* push exception values for catch + * The values are copied to the CALLER FRAME + * (prev_frame->sp) same behvior ad WASM_OP_RETURN + */ + if (cell_num_to_copy > 0) { + word_copy(prev_frame->sp, + frame_sp_old - cell_num_to_copy, + cell_num_to_copy); + prev_frame->sp += cell_num_to_copy; + } + *((int32 *)(prev_frame->sp)) = exception_tag_index; + prev_frame->sp++; + + /* mark frame as raised exception */ + wasm_set_exception(module, + "uncaught wasm exception"); + + /* end of function, treat as WASM_OP_RETURN */ + goto return_func; + } + default: + wasm_set_exception( + module, + "unexpected or invalid label in THROW or " + "RETHROW when searching a catch handler"); + goto got_exception; + } + + relative_depth++; + + } while (1); + } + + /* something went wrong. normally, we should always find the + * func label. if not, stop the interpreter */ + wasm_set_exception( + module, "WASM_OP_THROW hit the bottom of the frame stack"); + goto got_exception; + } + + HANDLE_OP(EXT_OP_TRY) + { + /* read the blocktype */ + read_leb_uint32(frame_ip, frame_ip_end, type_index); + param_cell_num = wasm_types[type_index]->param_cell_num; + cell_num = wasm_types[type_index]->ret_cell_num; + goto handle_op_try; + } + + HANDLE_OP(WASM_OP_TRY) + { + value_type = *frame_ip++; + param_cell_num = 0; + cell_num = wasm_value_type_cell_num(value_type); + + handle_op_try: + + cache_index = ((uintptr_t)frame_ip) + & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + cache_items = exec_env->block_addr_cache[cache_index]; + if (cache_items[0].start_addr == frame_ip) { + cache_items[0].start_addr = 0; + } + if (cache_items[1].start_addr == frame_ip) { + cache_items[1].start_addr = 0; + } + + /* start at the first opcode following the try and its blocktype + */ + uint8 *lookup_cursor = frame_ip; + uint8 handler_opcode = WASM_OP_UNREACHABLE; + + /* target_addr filled in when END or DELEGATE is found */ + PUSH_CSP(LABEL_TYPE_TRY, param_cell_num, cell_num, 0); + + /* reset to begin of block */ + lookup_cursor = frame_ip; + do { + /* lookup the next CATCH, CATCH_ALL or END for this TRY */ + if (!wasm_loader_find_block_addr( + exec_env, (BlockAddr *)exec_env->block_addr_cache, + lookup_cursor, (uint8 *)-1, LABEL_TYPE_TRY, + &else_addr, &end_addr)) { + /* something went wrong */ + wasm_set_exception(module, "find block address failed"); + goto got_exception; + } + + /* place cursor for continuation past opcode */ + lookup_cursor = end_addr + 1; + + /* end_addr points to CATCH, CATCH_ALL, DELEGATE or END */ + handler_opcode = *end_addr; + switch (handler_opcode) { + case WASM_OP_CATCH: + skip_leb(lookup_cursor); /* skip tag_index */ + PUSH_I64(end_addr); + break; + case WASM_OP_CATCH_ALL: + PUSH_I64(end_addr); + break; + case WASM_OP_DELEGATE: + skip_leb(lookup_cursor); /* skip depth */ + PUSH_I64(end_addr); + /* patch target_addr */ + (frame_csp - 1)->target_addr = lookup_cursor; + break; + case WASM_OP_END: + PUSH_I64(0); + /* patch target_addr */ + (frame_csp - 1)->target_addr = end_addr; + break; + default: + /* something went wrong */ + wasm_set_exception(module, + "find block address returned an " + "unexpected opcode"); + goto got_exception; + } + /* ... search until the returned address is the END of the + * TRY block */ + } while (handler_opcode != WASM_OP_END + && handler_opcode != WASM_OP_DELEGATE); + /* handler setup on stack complete */ + + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_CATCH) + { + /* skip the tag_index */ + skip_leb(frame_ip); + /* leave the frame */ + POP_CSP_N(0); + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_CATCH_ALL) + { + /* leave the frame */ + POP_CSP_N(0); + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_DELEGATE) + { + /* skip the delegate depth */ + skip_leb(frame_ip); + /* leave the frame like WASM_OP_END */ + POP_CSP(); + HANDLE_OP_END(); + } +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + HANDLE_OP(EXT_OP_BLOCK) + { + read_leb_uint32(frame_ip, frame_ip_end, type_index); + param_cell_num = + ((WASMFuncType *)wasm_types[type_index])->param_cell_num; + cell_num = + ((WASMFuncType *)wasm_types[type_index])->ret_cell_num; + goto handle_op_block; + } + + HANDLE_OP(WASM_OP_BLOCK) + { + value_type = *frame_ip++; + param_cell_num = 0; + cell_num = wasm_value_type_cell_num(value_type); + handle_op_block: + cache_index = ((uintptr_t)frame_ip) + & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + cache_items = exec_env->block_addr_cache[cache_index]; + if (cache_items[0].start_addr == frame_ip) { + end_addr = cache_items[0].end_addr; + } + else if (cache_items[1].start_addr == frame_ip) { + end_addr = cache_items[1].end_addr; + } +#if WASM_ENABLE_DEBUG_INTERP != 0 + else if (!wasm_loader_find_block_addr( + exec_env, (BlockAddr *)exec_env->block_addr_cache, + frame_ip, (uint8 *)-1, LABEL_TYPE_BLOCK, + &else_addr, &end_addr)) { + wasm_set_exception(module, "find block address failed"); + goto got_exception; + } +#endif + else { + end_addr = NULL; + } + PUSH_CSP(LABEL_TYPE_BLOCK, param_cell_num, cell_num, end_addr); + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_LOOP) + { + read_leb_uint32(frame_ip, frame_ip_end, type_index); + param_cell_num = + ((WASMFuncType *)wasm_types[type_index])->param_cell_num; + cell_num = + ((WASMFuncType *)wasm_types[type_index])->param_cell_num; + goto handle_op_loop; + } + + HANDLE_OP(WASM_OP_LOOP) + { + value_type = *frame_ip++; + param_cell_num = 0; + cell_num = 0; + handle_op_loop: + PUSH_CSP(LABEL_TYPE_LOOP, param_cell_num, cell_num, frame_ip); + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_IF) + { + read_leb_uint32(frame_ip, frame_ip_end, type_index); + param_cell_num = + ((WASMFuncType *)wasm_types[type_index])->param_cell_num; + cell_num = + ((WASMFuncType *)wasm_types[type_index])->ret_cell_num; + goto handle_op_if; + } + + HANDLE_OP(WASM_OP_IF) + { + value_type = *frame_ip++; + param_cell_num = 0; + cell_num = wasm_value_type_cell_num(value_type); + handle_op_if: + cache_index = ((uintptr_t)frame_ip) + & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + cache_items = exec_env->block_addr_cache[cache_index]; + if (cache_items[0].start_addr == frame_ip) { + else_addr = cache_items[0].else_addr; + end_addr = cache_items[0].end_addr; + } + else if (cache_items[1].start_addr == frame_ip) { + else_addr = cache_items[1].else_addr; + end_addr = cache_items[1].end_addr; + } + else if (!wasm_loader_find_block_addr( + exec_env, (BlockAddr *)exec_env->block_addr_cache, + frame_ip, (uint8 *)-1, LABEL_TYPE_IF, &else_addr, + &end_addr)) { + wasm_set_exception(module, "find block address failed"); + goto got_exception; + } + + cond = (uint32)POP_I32(); + + if (cond) { /* if branch is met */ + PUSH_CSP(LABEL_TYPE_IF, param_cell_num, cell_num, end_addr); + } + else { /* if branch is not met */ + /* if there is no else branch, go to the end addr */ + if (else_addr == NULL) { + frame_ip = end_addr + 1; + } + /* if there is an else branch, go to the else addr */ + else { + PUSH_CSP(LABEL_TYPE_IF, param_cell_num, cell_num, + end_addr); + frame_ip = else_addr + 1; + } + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_ELSE) + { + /* comes from the if branch in WASM_OP_IF */ + frame_ip = (frame_csp - 1)->target_addr; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_END) + { + if (frame_csp > frame->csp_bottom + 1) { + POP_CSP(); + } + else { /* end of function, treat as WASM_OP_RETURN */ + frame_sp -= cur_func->ret_cell_num; + for (i = 0; i < cur_func->ret_cell_num; i++) { +#if WASM_ENABLE_GC != 0 + if (prev_frame->ip) { + /* prev frame is not a glue frame and has + the frame ref area */ + *FRAME_REF_FOR(prev_frame, prev_frame->sp) = + *FRAME_REF(frame_sp + i); + } +#endif + *prev_frame->sp++ = frame_sp[i]; + } + goto return_func; + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, depth); + label_pop_csp_n: + POP_CSP_N(depth); + if (!frame_ip) { /* must be label pushed by WASM_OP_BLOCK */ + if (!wasm_loader_find_block_addr( + exec_env, (BlockAddr *)exec_env->block_addr_cache, + (frame_csp - 1)->begin_addr, (uint8 *)-1, + LABEL_TYPE_BLOCK, &else_addr, &end_addr)) { + wasm_set_exception(module, "find block address failed"); + goto got_exception; + } + frame_ip = end_addr; + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_IF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, depth); + cond = (uint32)POP_I32(); + if (cond) + goto label_pop_csp_n; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_TABLE) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, count); + lidx = POP_I32(); + if (lidx > count) + lidx = count; + depth = frame_ip[lidx]; + goto label_pop_csp_n; + } + + HANDLE_OP(EXT_OP_BR_TABLE_CACHE) + { + BrTableCache *node_cache = + bh_list_first_elem(module->module->br_table_cache_list); + BrTableCache *node_next; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + lidx = POP_I32(); + + while (node_cache) { + node_next = bh_list_elem_next(node_cache); + if (node_cache->br_table_op_addr == frame_ip - 1) { + if (lidx > node_cache->br_count) + lidx = node_cache->br_count; + depth = node_cache->br_depths[lidx]; + goto label_pop_csp_n; + } + node_cache = node_next; + } + bh_assert(0); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_RETURN) + { + frame_sp -= cur_func->ret_cell_num; + for (i = 0; i < cur_func->ret_cell_num; i++) { +#if WASM_ENABLE_GC != 0 + if (prev_frame->ip) { + /* prev frame is not a glue frame and has + the frame ref area */ + *FRAME_REF_FOR(prev_frame, prev_frame->sp) = + *FRAME_REF(frame_sp + i); + } +#endif + *prev_frame->sp++ = frame_sp[i]; + } + goto return_func; + } + + HANDLE_OP(WASM_OP_CALL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, fidx); +#if WASM_ENABLE_MULTI_MODULE != 0 + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } +#endif + + cur_func = module->e->functions + fidx; + goto call_func_from_interp; + } + +#if WASM_ENABLE_TAIL_CALL != 0 + HANDLE_OP(WASM_OP_RETURN_CALL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, fidx); +#if WASM_ENABLE_MULTI_MODULE != 0 + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } +#endif + cur_func = module->e->functions + fidx; + + goto call_func_from_return_call; + } +#endif /* WASM_ENABLE_TAIL_CALL */ + + HANDLE_OP(WASM_OP_CALL_INDIRECT) +#if WASM_ENABLE_TAIL_CALL != 0 + HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT) +#endif + { + WASMFuncType *cur_type, *cur_func_type; + WASMTableInstance *tbl_inst; + uint32 tbl_idx; + +#if WASM_ENABLE_TAIL_CALL != 0 + opcode = *(frame_ip - 1); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + /** + * type check. compiler will make sure all like + * (call_indirect (type $x) (i32.const 1)) + * the function type has to be defined in the module also + * no matter it is used or not + */ + read_leb_uint32(frame_ip, frame_ip_end, tidx); + bh_assert(tidx < module->module->type_count); + cur_type = wasm_types[tidx]; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + val = POP_I32(); + if ((uint32)val >= tbl_inst->cur_size) { + wasm_set_exception(module, "undefined element"); + goto got_exception; + } + + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + fidx = tbl_inst->elems[val]; + if (fidx == (uint32)-1) { + wasm_set_exception(module, "uninitialized element"); + goto got_exception; + } +#else + func_obj = (WASMFuncObjectRef)tbl_inst->elems[val]; + if (!func_obj) { + wasm_set_exception(module, "uninitialized element"); + goto got_exception; + } + fidx = wasm_func_obj_get_func_idx_bound(func_obj); +#endif + /* clang-format on */ + + /* + * we might be using a table injected by host or + * another module. In that case, we don't validate + * the elem value while loading + */ + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } + + /* always call module own functions */ + cur_func = module->e->functions + fidx; + + if (cur_func->is_import_func) + cur_func_type = cur_func->u.func_import->func_type; + else + cur_func_type = cur_func->u.func->func_type; + + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + if (cur_type != cur_func_type) { + wasm_set_exception(module, "indirect call type mismatch"); + goto got_exception; + } +#else + if (!wasm_func_type_is_super_of(cur_type, cur_func_type)) { + wasm_set_exception(module, "indirect call type mismatch"); + goto got_exception; + } +#endif + /* clang-format on */ + +#if WASM_ENABLE_TAIL_CALL != 0 + if (opcode == WASM_OP_RETURN_CALL_INDIRECT) + goto call_func_from_return_call; +#endif + goto call_func_from_interp; + } + + /* parametric instructions */ + HANDLE_OP(WASM_OP_DROP) + { + frame_sp--; + +#if WASM_ENABLE_GC != 0 + frame_ref_tmp = FRAME_REF(frame_sp); + *frame_ref_tmp = 0; +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_DROP_64) + { + frame_sp -= 2; + +#if WASM_ENABLE_GC != 0 + frame_ref_tmp = FRAME_REF(frame_sp); + *frame_ref_tmp = 0; + *(frame_ref_tmp + 1) = 0; +#endif + + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SELECT) + { + cond = (uint32)POP_I32(); + frame_sp--; + if (!cond) + *(frame_sp - 1) = *frame_sp; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SELECT_64) + { + cond = (uint32)POP_I32(); + frame_sp -= 2; + if (!cond) { + *(frame_sp - 2) = *frame_sp; + *(frame_sp - 1) = *(frame_sp + 1); + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_SELECT_T) + { + uint32 vec_len; + uint8 type; + + read_leb_uint32(frame_ip, frame_ip_end, vec_len); + type = *frame_ip++; + + cond = (uint32)POP_I32(); + if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64 +#if WASM_ENABLE_GC != 0 && UINTPTR_MAX == UINT64_MAX + || wasm_is_type_reftype(type) +#endif + ) { + frame_sp -= 2; + if (!cond) { + *(frame_sp - 2) = *frame_sp; + *(frame_sp - 1) = *(frame_sp + 1); + } + } + else { + frame_sp--; + if (!cond) + *(frame_sp - 1) = *frame_sp; + } + +#if WASM_ENABLE_GC != 0 + frame_ref_tmp = FRAME_REF(frame_sp); + *frame_ref_tmp = 0; +#if UINTPTR_MAX == UINT64_MAX + *(frame_ref_tmp + 1) = 0; +#endif +#endif + (void)vec_len; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_TABLE_GET) + { + uint32 tbl_idx, elem_idx; + WASMTableInstance *tbl_inst; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + elem_idx = POP_I32(); + if (elem_idx >= tbl_inst->cur_size) { + wasm_set_exception(module, "out of bounds table access"); + goto got_exception; + } + +#if WASM_ENABLE_GC == 0 + PUSH_I32(tbl_inst->elems[elem_idx]); +#else + PUSH_REF(tbl_inst->elems[elem_idx]); +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_TABLE_SET) + { + WASMTableInstance *tbl_inst; + uint32 tbl_idx, elem_idx; + table_elem_type_t elem_val; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + +#if WASM_ENABLE_GC == 0 + elem_val = POP_I32(); +#else + elem_val = POP_REF(); +#endif + elem_idx = POP_I32(); + if (elem_idx >= tbl_inst->cur_size) { + wasm_set_exception(module, "out of bounds table access"); + goto got_exception; + } + + tbl_inst->elems[elem_idx] = elem_val; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_NULL) + { + uint32 ref_type; + read_leb_uint32(frame_ip, frame_ip_end, ref_type); +#if WASM_ENABLE_GC == 0 + PUSH_I32(NULL_REF); +#else + PUSH_REF(NULL_REF); +#endif + (void)ref_type; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_IS_NULL) + { +#if WASM_ENABLE_GC == 0 + uint32 ref_val; + ref_val = POP_I32(); +#else + void *ref_val; + ref_val = POP_REF(); +#endif + PUSH_I32(ref_val == NULL_REF ? 1 : 0); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_FUNC) + { + uint32 func_idx; + read_leb_uint32(frame_ip, frame_ip_end, func_idx); +#if WASM_ENABLE_GC == 0 + PUSH_I32(func_idx); +#else + SYNC_ALL_TO_FRAME(); + if (!(gc_obj = wasm_create_func_obj(module, func_idx, true, + NULL, 0))) { + goto got_exception; + } + PUSH_REF(gc_obj); +#endif + HANDLE_OP_END(); + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_CALL_REF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, type_index); + func_obj = POP_REF(); + if (!func_obj) { + wasm_set_exception(module, "null function object"); + goto got_exception; + } + + fidx = wasm_func_obj_get_func_idx_bound(func_obj); + cur_func = module->e->functions + fidx; + goto call_func_from_interp; + } + + HANDLE_OP(WASM_OP_RETURN_CALL_REF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, type_index); + func_obj = POP_REF(); + if (!func_obj) { + wasm_set_exception(module, "null function object"); + goto got_exception; + } + + fidx = wasm_func_obj_get_func_idx_bound(func_obj); + cur_func = module->e->functions + fidx; + goto call_func_from_return_call; + } + + HANDLE_OP(WASM_OP_REF_EQ) + { + WASMObjectRef gc_obj1, gc_obj2; + gc_obj2 = POP_REF(); + gc_obj1 = POP_REF(); + val = wasm_obj_equal(gc_obj1, gc_obj2); + PUSH_I32(val); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_AS_NON_NULL) + { + gc_obj = POP_REF(); + if (gc_obj == NULL_REF) { + wasm_set_exception(module, "null reference"); + goto got_exception; + } + PUSH_REF(gc_obj); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_ON_NULL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, depth); + gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM); + if (gc_obj == NULL_REF) { + frame_sp -= REF_CELL_NUM; + CLEAR_FRAME_REF(frame_sp, REF_CELL_NUM); + goto label_pop_csp_n; + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_ON_NON_NULL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + read_leb_uint32(frame_ip, frame_ip_end, depth); + gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM); + if (gc_obj != NULL_REF) { + goto label_pop_csp_n; + } + else { + frame_sp -= REF_CELL_NUM; + CLEAR_FRAME_REF(frame_sp, REF_CELL_NUM); + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GC_PREFIX) + { + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + { + WASMModule *wasm_module = module->module; + WASMStructType *struct_type; + WASMRttType *rtt_type; + WASMValue field_value = { 0 }; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + struct_type = + (WASMStructType *)module->module->types[type_index]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_index, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + struct_obj = wasm_struct_obj_new(exec_env, rtt_type); + if (!struct_obj) { + wasm_set_exception(module, + "create struct object failed"); + goto got_exception; + } + + if (opcode == WASM_OP_STRUCT_NEW) { + WASMStructFieldType *fields = struct_type->fields; + int32 field_count = (int32)struct_type->field_count; + int32 field_idx; + uint8 field_type; + + for (field_idx = field_count - 1; field_idx >= 0; + field_idx--) { + field_type = fields[field_idx].field_type; + if (wasm_is_type_reftype(field_type)) { + field_value.gc_obj = POP_REF(); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + field_value.i32 = POP_I32(); + } + else { + field_value.i64 = POP_I64(); + } + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + } + } + PUSH_REF(struct_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + { + WASMStructType *struct_type; + WASMValue field_value = { 0 }; + uint32 field_idx; + uint8 field_type; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, field_idx); + struct_type = + (WASMStructType *)module->module->types[type_index]; + + struct_obj = POP_REF(); + + if (!struct_obj) { + wasm_set_exception(module, "null structure object"); + goto got_exception; + } + + wasm_struct_obj_get_field( + struct_obj, field_idx, + opcode == WASM_OP_STRUCT_GET_S ? true : false, + &field_value); + + field_type = struct_type->fields[field_idx].field_type; + if (wasm_is_reftype_i31ref(field_type)) { + PUSH_I31REF(field_value.gc_obj); + } + else if (wasm_is_type_reftype(field_type)) { + PUSH_REF(field_value.gc_obj); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + PUSH_I32(field_value.i32); + } + else { + PUSH_I64(field_value.i64); + } + HANDLE_OP_END(); + } + case WASM_OP_STRUCT_SET: + { + WASMStructType *struct_type; + WASMValue field_value = { 0 }; + uint32 field_idx; + uint8 field_type; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, field_idx); + + struct_type = + (WASMStructType *)module->module->types[type_index]; + field_type = struct_type->fields[field_idx].field_type; + + if (wasm_is_type_reftype(field_type)) { + field_value.gc_obj = POP_REF(); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + field_value.i32 = POP_I32(); + } + else { + field_value.i64 = POP_I64(); + } + + struct_obj = POP_REF(); + if (!struct_obj) { + wasm_set_exception(module, "null structure object"); + goto got_exception; + } + + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + HANDLE_OP_END(); + } + + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + { + WASMModule *wasm_module = module->module; + WASMArrayType *array_type; + WASMRttType *rtt_type; + WASMValue array_elem = { 0 }; + uint32 array_len; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + array_type = + (WASMArrayType *)wasm_module->types[type_index]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_index, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + if (opcode != WASM_OP_ARRAY_NEW_FIXED) + array_len = POP_I32(); + else + read_leb_uint32(frame_ip, frame_ip_end, array_len); + + if (opcode == WASM_OP_ARRAY_NEW) { + if (wasm_is_type_reftype(array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type + == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + } + + SYNC_ALL_TO_FRAME(); + array_obj = wasm_array_obj_new(exec_env, rtt_type, + array_len, &array_elem); + if (!array_obj) { + wasm_set_exception(module, + "create array object failed"); + goto got_exception; + } + + if (opcode == WASM_OP_ARRAY_NEW_FIXED) { + for (i = 0; i < array_len; i++) { + if (wasm_is_type_reftype( + array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type + == VALUE_TYPE_F32 + || array_type->elem_type + == PACKED_TYPE_I8 + || array_type->elem_type + == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + wasm_array_obj_set_elem( + array_obj, array_len - 1 - i, &array_elem); + } + } + + PUSH_REF(array_obj); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW_DATA: + { + WASMModule *wasm_module = module->module; + WASMArrayType *array_type; + WASMRttType *rtt_type; + WASMValue array_elem = { 0 }; + WASMDataSeg *data_seg; + uint8 *array_elem_base; + uint32 array_len, data_seg_idx, data_seg_offset; + uint32 elem_size = 0; + uint64 total_size; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, data_seg_idx); + data_seg = wasm_module->data_segments[data_seg_idx]; + + array_type = + (WASMArrayType *)wasm_module->types[type_index]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_index, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + array_len = POP_I32(); + data_seg_offset = POP_I32(); + + switch (array_type->elem_type) { + case PACKED_TYPE_I8: + elem_size = 1; + break; + case PACKED_TYPE_I16: + elem_size = 2; + break; + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + elem_size = 4; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + elem_size = 8; + break; + default: + bh_assert(0); + } + + total_size = (uint64)elem_size * array_len; + if (data_seg_offset >= data_seg->data_length + || total_size + > data_seg->data_length - data_seg_offset) { + wasm_set_exception(module, + "data segment out of bounds"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + array_obj = wasm_array_obj_new(exec_env, rtt_type, + array_len, &array_elem); + if (!array_obj) { + wasm_set_exception(module, + "create array object failed"); + goto got_exception; + } + + array_elem_base = + (uint8 *)wasm_array_obj_first_elem_addr(array_obj); + bh_memcpy_s(array_elem_base, (uint32)total_size, + data_seg->data + data_seg_offset, + (uint32)total_size); + + PUSH_REF(array_obj); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW_ELEM: + { + /* TODO */ + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + { + WASMArrayType *array_type; + WASMValue array_elem = { 0 }; + uint32 elem_idx, elem_size_log; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + array_type = + (WASMArrayType *)module->module->types[type_index]; + + elem_idx = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + if (elem_idx >= wasm_array_obj_length(array_obj)) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_get_elem( + array_obj, elem_idx, + opcode == WASM_OP_ARRAY_GET_S ? true : false, + &array_elem); + elem_size_log = wasm_array_obj_elem_size_log(array_obj); + + if (wasm_is_reftype_i31ref(array_type->elem_type)) { + PUSH_I31REF(array_elem.gc_obj); + } + else if (wasm_is_type_reftype(array_type->elem_type)) { + PUSH_REF(array_elem.gc_obj); + } + else if (elem_size_log < 3) { + PUSH_I32(array_elem.i32); + } + else { + PUSH_I64(array_elem.i64); + } + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_SET: + { + WASMArrayType *array_type; + WASMValue array_elem = { 0 }; + uint32 elem_idx; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + array_type = + (WASMArrayType *)module->module->types[type_index]; + if (wasm_is_type_reftype(array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + + elem_idx = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + if (elem_idx >= wasm_array_obj_length(array_obj)) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_set_elem(array_obj, elem_idx, + &array_elem); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_LEN: + { + uint32 array_len; + + array_obj = POP_REF(); + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + array_len = wasm_array_obj_length(array_obj); + PUSH_I32(array_len); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_FILL: + { + WASMArrayType *array_type; + WASMValue fill_value = { 0 }; + uint32 start_offset, len; + read_leb_uint32(frame_ip, frame_ip_end, type_index); + + array_type = + (WASMArrayType *)module->module->types[type_index]; + + len = POP_I32(); + if (wasm_is_type_reftype(array_type->elem_type)) { + fill_value.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type == PACKED_TYPE_I16) { + fill_value.i32 = POP_I32(); + } + else { + fill_value.i64 = POP_I64(); + } + start_offset = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + + if (len > 0) { + if ((uint64)start_offset + len + >= wasm_array_obj_length(array_obj)) { + wasm_set_exception( + module, "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_fill(array_obj, start_offset, len, + &fill_value); + } + + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_COPY: + { + uint32 dst_offset, src_offset, len, src_type_index; + WASMArrayObjectRef src_obj, dst_obj; + + read_leb_uint32(frame_ip, frame_ip_end, type_index); + read_leb_uint32(frame_ip, frame_ip_end, src_type_index); + + len = POP_I32(); + src_offset = POP_I32(); + src_obj = POP_REF(); + dst_offset = POP_I32(); + dst_obj = POP_REF(); + + if (!src_obj || !dst_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + + if (len > 0) { + if ((dst_offset > UINT32_MAX - len) + || (dst_offset + len + > wasm_array_obj_length(dst_obj)) + || (src_offset > UINT32_MAX - len) + || (src_offset + len + > wasm_array_obj_length(src_obj))) { + wasm_set_exception( + module, "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_copy(dst_obj, dst_offset, src_obj, + src_offset, len); + } + + (void)src_type_index; + HANDLE_OP_END(); + } + + case WASM_OP_REF_I31: + { + uint32 i31_val; + + i31_val = POP_I32(); + i31_obj = wasm_i31_obj_new(i31_val); + PUSH_I31REF(i31_obj); + HANDLE_OP_END(); + } + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + { + uint32 i31_val; + + i31_obj = (WASMI31ObjectRef)POP_REF(); + if (!i31_obj) { + wasm_set_exception(module, "null i31 reference"); + goto got_exception; + } + i31_val = (uint32)(((uintptr_t)i31_obj) >> 1); + if (opcode == WASM_OP_I31_GET_S + && (i31_val & 0x40000000) /* bit 30 is 1 */) + /* set bit 31 to 1 */ + i31_val |= 0x80000000; + PUSH_I32(i31_val); + HANDLE_OP_END(); + } + + case WASM_OP_REF_TEST: + case WASM_OP_REF_CAST: + case WASM_OP_REF_TEST_NULLABLE: + case WASM_OP_REF_CAST_NULLABLE: + { + int32 heap_type; + + read_leb_int32(frame_ip, frame_ip_end, heap_type); + + gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM); + if (!gc_obj) { + if (opcode == WASM_OP_REF_TEST + || opcode == WASM_OP_REF_TEST_NULLABLE) { + (void)POP_REF(); + if (opcode == WASM_OP_REF_TEST) + PUSH_I32(0); + else + PUSH_I32(1); + } + else if (opcode == WASM_OP_REF_CAST) { + wasm_set_exception(module, "cast failure"); + goto got_exception; + } + else { + /* Do nothing for WASM_OP_REF_CAST_NULLABLE */ + } + } + else { + bool castable = false; + + if (heap_type >= 0) { + WASMModule *wasm_module = module->module; + castable = wasm_obj_is_instance_of( + gc_obj, (uint32)heap_type, + wasm_module->types, + wasm_module->type_count); + } + else { + castable = + wasm_obj_is_type_of(gc_obj, heap_type); + } + + if (opcode == WASM_OP_REF_TEST + || opcode == WASM_OP_REF_TEST_NULLABLE) { + (void)POP_REF(); + if (castable) + PUSH_I32(1); + else + PUSH_I32(0); + } + else if (!castable) { + wasm_set_exception(module, "cast failure"); + goto got_exception; + } + } + HANDLE_OP_END(); + } + + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + { + int32 heap_type, heap_type_dst; + uint8 castflags; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + castflags = *frame_ip++; + read_leb_uint32(frame_ip, frame_ip_end, depth); + read_leb_int32(frame_ip, frame_ip_end, heap_type); + read_leb_int32(frame_ip, frame_ip_end, heap_type_dst); + + gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM); + if (!gc_obj) { + /* + * castflags should be 0~3: + * 0: (non-null, non-null) + * 1: (null, non-null) + * 2: (non-null, null) + * 3: (null, null) + */ + if ( + /* op is BR_ON_CAST and dst reftype is nullable + */ + ((opcode1 == WASM_OP_BR_ON_CAST) + && ((castflags == 2) || (castflags == 3))) + /* op is BR_ON_CAST_FAIL and dst reftype is + non-nullable */ + || ((opcode1 == WASM_OP_BR_ON_CAST_FAIL) + && ((castflags == 0) || (castflags == 1)))) + goto label_pop_csp_n; + } + else { + bool castable = false; + + if (heap_type_dst >= 0) { + WASMModule *wasm_module = module->module; + castable = wasm_obj_is_instance_of( + gc_obj, (uint32)heap_type_dst, + wasm_module->types, + wasm_module->type_count); + } + else { + castable = + wasm_obj_is_type_of(gc_obj, heap_type_dst); + } + + if ((castable && (opcode == WASM_OP_BR_ON_CAST)) + || (!castable + && (opcode == WASM_OP_BR_ON_CAST_FAIL))) { + goto label_pop_csp_n; + } + } + + (void)heap_type; + HANDLE_OP_END(); + } + + case WASM_OP_ANY_CONVERT_EXTERN: + { + externref_obj = POP_REF(); + if (externref_obj == NULL_REF) + PUSH_REF(NULL_REF); + else { + gc_obj = wasm_externref_obj_to_internal_obj( + externref_obj); + PUSH_REF(gc_obj); + } + HANDLE_OP_END(); + } + case WASM_OP_EXTERN_CONVERT_ANY: + { + gc_obj = POP_REF(); + if (gc_obj == NULL_REF) + PUSH_REF(NULL_REF); + else { + if (!(externref_obj = + wasm_internal_obj_to_externref_obj( + exec_env, gc_obj))) { + wasm_set_exception( + module, "create externref object failed"); + goto got_exception; + } + PUSH_REF(externref_obj); + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + { + uint32 mem_idx, addr, bytes_length, offset = 0; + EncodingFlag flag = WTF8; + + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + bytes_length = POP_I32(); + addr = POP_I32(); + + CHECK_MEMORY_OVERFLOW(bytes_length); + + if (opcode == WASM_OP_STRING_NEW_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_NEW_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8) { + flag = WTF8; + } + + str_obj = wasm_string_new_with_encoding( + maddr, bytes_length, flag); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + + (void)mem_idx; + HANDLE_OP_END(); + } + case WASM_OP_STRING_CONST: + { + WASMModule *wasm_module = module->module; + uint32 contents; + + read_leb_uint32(frame_ip, frame_ip_end, contents); + + str_obj = wasm_string_new_const( + (const char *) + wasm_module->string_literal_ptrs[contents], + wasm_module->string_literal_lengths[contents]); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + { + int32 target_bytes_length; + EncodingFlag flag = WTF8; + + stringref_obj = POP_REF(); + + if (opcode == WASM_OP_STRING_MEASURE_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_MEASURE_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_MEASURE_WTF8) { + flag = LOSSY_UTF8; + } + target_bytes_length = wasm_string_measure( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + flag); + + PUSH_I32(target_bytes_length); + HANDLE_OP_END(); + } + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + { + uint32 mem_idx, addr; + int32 target_bytes_length; + WASMMemoryInstance *memory_inst; + EncodingFlag flag = WTF8; + + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + addr = POP_I32(); + stringref_obj = POP_REF(); + + str_obj = (WASMString)wasm_stringref_obj_get_value( + stringref_obj); + + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + + if (opcode == WASM_OP_STRING_ENCODE_WTF16) { + flag = WTF16; + count = wasm_string_measure(str_obj, flag); + target_bytes_length = wasm_string_encode( + str_obj, 0, count, maddr, NULL, flag); + } + else { + if (opcode == WASM_OP_STRING_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRING_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_ENCODE_WTF8) { + flag = WTF8; + } + count = wasm_string_measure(str_obj, flag); + target_bytes_length = wasm_string_encode( + str_obj, 0, count, maddr, NULL, flag); + + if (target_bytes_length == -1) { + wasm_set_exception( + module, "isolated surrogate is seen"); + goto got_exception; + } + } + if (target_bytes_length < 0) { + wasm_set_exception(module, + "stringref encode failed"); + goto got_exception; + } + + PUSH_I32(target_bytes_length); + HANDLE_OP_END(); + } + case WASM_OP_STRING_CONCAT: + { + WASMStringrefObjectRef stringref_obj1, stringref_obj2; + + stringref_obj2 = POP_REF(); + stringref_obj1 = POP_REF(); + + str_obj = wasm_string_concat( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj1), + (WASMString)wasm_stringref_obj_get_value( + stringref_obj2)); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_EQ: + { + WASMStringrefObjectRef stringref_obj1, stringref_obj2; + int32 is_eq; + + stringref_obj2 = POP_REF(); + stringref_obj1 = POP_REF(); + + is_eq = wasm_string_eq( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj1), + (WASMString)wasm_stringref_obj_get_value( + stringref_obj2)); + + PUSH_I32(is_eq); + HANDLE_OP_END(); + } + case WASM_OP_STRING_IS_USV_SEQUENCE: + { + int32 is_usv_sequence; + + stringref_obj = POP_REF(); + + is_usv_sequence = wasm_string_is_usv_sequence( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj)); + + PUSH_I32(is_usv_sequence); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_WTF8: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_WTF8); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_wtf8_obj = + wasm_stringview_wtf8_obj_new(exec_env, str_obj); + if (!stringview_wtf8_obj) { + wasm_set_exception(module, + "create stringview wtf8 failed"); + goto got_exception; + } + + PUSH_REF(stringview_wtf8_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + { + uint32 next_pos, bytes, pos; + + bytes = POP_I32(); + pos = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + next_pos = wasm_string_advance( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + pos, bytes, NULL); + + PUSH_I32(next_pos); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + { + uint32 mem_idx, addr, pos, bytes, next_pos; + int32 bytes_written; + WASMMemoryInstance *memory_inst; + EncodingFlag flag = WTF8; + + if (opcode == WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8) { + flag = WTF8; + } + + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + bytes = POP_I32(); + pos = POP_I32(); + addr = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + + bytes_written = wasm_string_encode( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + pos, bytes, maddr, &next_pos, flag); + + if (bytes_written < 0) { + if (bytes_written == Isolated_Surrogate) { + wasm_set_exception( + module, "isolated surrogate is seen"); + } + else { + wasm_set_exception(module, "encode failed"); + } + + goto got_exception; + } + + PUSH_I32(next_pos); + PUSH_I32(bytes_written); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_SLICE: + { + uint32 start, end; + + end = POP_I32(); + start = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + start, end, STRING_VIEW_WTF8); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_WTF16: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_WTF16); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_wtf16_obj = + wasm_stringview_wtf16_obj_new(exec_env, str_obj); + if (!stringview_wtf16_obj) { + wasm_set_exception( + module, "create stringview wtf16 failed"); + goto got_exception; + } + + PUSH_REF(stringview_wtf16_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + { + int32 code_units_length; + + stringview_wtf16_obj = POP_REF(); + + code_units_length = wasm_string_wtf16_get_length( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj)); + + PUSH_I32(code_units_length); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + { + int32 pos; + uint32 code_unit; + + pos = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + code_unit = (uint32)wasm_string_get_wtf16_codeunit( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + pos); + + PUSH_I32(code_unit); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + { + uint32 mem_idx, addr, pos, len, offset = 0; + int32 written_code_units = 0; + + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + len = POP_I32(); + pos = POP_I32(); + addr = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + CHECK_MEMORY_OVERFLOW(len * sizeof(uint16)); + + /* check 2-byte alignment */ + if (((uintptr_t)maddr & (((uintptr_t)1 << 2) - 1)) + != 0) { + wasm_set_exception(module, + "unaligned memory access"); + goto got_exception; + } + + written_code_units = wasm_string_encode( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + pos, len, maddr, NULL, WTF16); + if (written_code_units < 0) { + wasm_set_exception(module, "encode failed"); + goto got_exception; + } + + PUSH_I32(written_code_units); + (void)mem_idx; + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_SLICE: + { + uint32 start, end; + + end = POP_I32(); + start = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + start, end, STRING_VIEW_WTF16); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_ITER: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_ITER); + + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_iter_obj = + wasm_stringview_iter_obj_new(exec_env, str_obj, 0); + if (!stringview_iter_obj) { + wasm_set_exception(module, + "create stringview iter failed"); + goto got_exception; + } + + PUSH_REF(stringview_iter_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_NEXT: + { + uint32 code_point; + + stringview_iter_obj = POP_REF(); + + code_point = wasm_string_next_codepoint( + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj), + wasm_stringview_iter_obj_get_pos( + stringview_iter_obj)); + + PUSH_I32(code_point); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + case WASM_OP_STRINGVIEW_ITER_REWIND: + { + uint32 code_points_count, code_points_consumed = 0, + cur_pos, next_pos = 0; + + code_points_count = POP_I32(); + stringview_iter_obj = POP_REF(); + + str_obj = + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj); + cur_pos = wasm_stringview_iter_obj_get_pos( + stringview_iter_obj); + + if (opcode == WASM_OP_STRINGVIEW_ITER_ADVANCE) { + next_pos = wasm_string_advance( + str_obj, cur_pos, code_points_count, + &code_points_consumed); + } + else if (opcode == WASM_OP_STRINGVIEW_ITER_REWIND) { + next_pos = wasm_string_rewind( + str_obj, cur_pos, code_points_count, + &code_points_consumed); + } + + wasm_stringview_iter_obj_update_pos(stringview_iter_obj, + next_pos); + + PUSH_I32(code_points_consumed); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_SLICE: + { + uint32 code_points_count, cur_pos; + + code_points_count = POP_I32(); + stringview_iter_obj = POP_REF(); + + cur_pos = wasm_stringview_iter_obj_get_pos( + stringview_iter_obj); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj), + cur_pos, cur_pos + code_points_count, + STRING_VIEW_ITER); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + { + uint32 start, end, array_len; + EncodingFlag flag = WTF8; + WASMArrayType *array_type; + void *arr_start_addr; + + end = POP_I32(); + start = POP_I32(); + array_obj = POP_REF(); + + array_type = (WASMArrayType *)wasm_obj_get_defined_type( + (WASMObjectRef)array_obj); + arr_start_addr = + wasm_array_obj_elem_addr(array_obj, start); + array_len = wasm_array_obj_length(array_obj); + + if (start > end || end > array_len) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + if (opcode == WASM_OP_STRING_NEW_WTF16_ARRAY) { + if (array_type->elem_type != VALUE_TYPE_I16) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + flag = WTF16; + } + else { + if (array_type->elem_type != VALUE_TYPE_I8) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + if (opcode == WASM_OP_STRING_NEW_UTF8_ARRAY) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8_ARRAY) { + flag = WTF8; + } + else if (opcode + == WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY) { + flag = LOSSY_UTF8; + } + } + + str_obj = wasm_string_new_with_encoding( + arr_start_addr, (end - start), flag); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + { + uint32 start, array_len, count; + int32 bytes_written; + EncodingFlag flag = WTF8; + WASMArrayType *array_type; + void *arr_start_addr; + + start = POP_I32(); + array_obj = POP_REF(); + stringref_obj = POP_REF(); + + str_obj = (WASMString)wasm_stringref_obj_get_value( + stringref_obj); + + array_type = (WASMArrayType *)wasm_obj_get_defined_type( + (WASMObjectRef)array_obj); + arr_start_addr = + wasm_array_obj_elem_addr(array_obj, start); + array_len = wasm_array_obj_length(array_obj); + + if (start > array_len) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + if (opcode == WASM_OP_STRING_ENCODE_WTF16_ARRAY) { + if (array_type->elem_type != VALUE_TYPE_I16) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + flag = WTF16; + } + else { + if (array_type->elem_type != VALUE_TYPE_I8) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + if (opcode == WASM_OP_STRING_ENCODE_UTF8_ARRAY) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRING_ENCODE_WTF8_ARRAY) { + flag = WTF8; + } + else if ( + opcode + == WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY) { + flag = LOSSY_UTF8; + } + } + + count = wasm_string_measure(str_obj, flag); + + bytes_written = wasm_string_encode( + str_obj, 0, count, arr_start_addr, NULL, flag); + + if (bytes_written < 0) { + if (bytes_written == Isolated_Surrogate) { + wasm_set_exception( + module, "isolated surrogate is seen"); + } + else if (bytes_written == Insufficient_Space) { + wasm_set_exception( + module, "array space is insufficient"); + } + else { + wasm_set_exception(module, "encode failed"); + } + + goto got_exception; + } + + PUSH_I32(bytes_written); + HANDLE_OP_END(); + } +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + default: + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + /* variable instructions */ + HANDLE_OP(WASM_OP_GET_LOCAL) + { + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + + switch (local_type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + PUSH_I32(*(int32 *)(frame_lp + local_offset)); + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + PUSH_I64(GET_I64_FROM_ADDR(frame_lp + local_offset)); + break; + default: +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(local_type)) { + if (wasm_is_reftype_i31ref(local_type)) { + PUSH_I31REF( + GET_REF_FROM_ADDR(frame_lp + local_offset)); + } + else { + PUSH_REF( + GET_REF_FROM_ADDR(frame_lp + local_offset)); + } + } + else +#endif + { + wasm_set_exception(module, "invalid local type"); + goto got_exception; + } + } + + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_GET_LOCAL_FAST) + { + local_offset = *frame_ip++; + if (local_offset & 0x80) + PUSH_I64( + GET_I64_FROM_ADDR(frame_lp + (local_offset & 0x7F))); + else + PUSH_I32(*(int32 *)(frame_lp + local_offset)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_LOCAL) + { + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + + switch (local_type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + *(int32 *)(frame_lp + local_offset) = POP_I32(); + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset), + POP_I64()); + break; + default: +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(local_type)) { + PUT_REF_TO_ADDR(frame_lp + local_offset, POP_REF()); + } + else +#endif + { + wasm_set_exception(module, "invalid local type"); + goto got_exception; + } + } + + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_SET_LOCAL_FAST) + { + local_offset = *frame_ip++; + if (local_offset & 0x80) + PUT_I64_TO_ADDR( + (uint32 *)(frame_lp + (local_offset & 0x7F)), + POP_I64()); + else + *(int32 *)(frame_lp + local_offset) = POP_I32(); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_TEE_LOCAL) + { + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + + switch (local_type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + *(int32 *)(frame_lp + local_offset) = + *(int32 *)(frame_sp - 1); + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_I64_FROM_ADDR(frame_sp - 2)); + break; + default: +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(local_type)) { + PUT_REF_TO_ADDR( + frame_lp + local_offset, + GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM)); + } + else +#endif + { + wasm_set_exception(module, "invalid local type"); + goto got_exception; + } + } + + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_TEE_LOCAL_FAST) + { + local_offset = *frame_ip++; + if (local_offset & 0x80) + PUT_I64_TO_ADDR( + (uint32 *)(frame_lp + (local_offset & 0x7F)), + GET_I64_FROM_ADDR(frame_sp - 2)); + else + *(int32 *)(frame_lp + local_offset) = + *(int32 *)(frame_sp - 1); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GET_GLOBAL) + { + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + PUSH_I32(*(uint32 *)global_addr); +#else + if (!wasm_is_type_reftype(global->type)) { + PUSH_I32(*(uint32 *)global_addr); + } + else if (wasm_is_reftype_i31ref(global->type)) { + PUSH_I31REF(GET_REF_FROM_ADDR((uint32 *)global_addr)); + } + else { + PUSH_REF(GET_REF_FROM_ADDR((uint32 *)global_addr)); + } +#endif + /* clang-format on */ + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GET_GLOBAL_64) + { + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + PUSH_I64(GET_I64_FROM_ADDR((uint32 *)global_addr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL) + { + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + *(int32 *)global_addr = POP_I32(); +#else + if (!wasm_is_type_reftype(global->type)) + *(int32 *)global_addr = POP_I32(); + else + PUT_REF_TO_ADDR((uint32 *)global_addr, POP_REF()); +#endif + /* clang-format on */ + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL_AUX_STACK) + { + uint64 aux_stack_top; + + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + aux_stack_top = *(uint64 *)(frame_sp - 2); + } + else +#endif + { + aux_stack_top = (uint64)(*(uint32 *)(frame_sp - 1)); + } + if (aux_stack_top <= (uint64)exec_env->aux_stack_boundary) { + wasm_set_exception(module, "wasm auxiliary stack overflow"); + goto got_exception; + } + if (aux_stack_top > (uint64)exec_env->aux_stack_bottom) { + wasm_set_exception(module, + "wasm auxiliary stack underflow"); + goto got_exception; + } +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + *(uint64 *)global_addr = aux_stack_top; + frame_sp -= 2; + } + else +#endif + { + *(uint32 *)global_addr = (uint32)aux_stack_top; + frame_sp--; + } +#if WASM_ENABLE_MEMORY_PROFILING != 0 + if (module->module->aux_stack_top_global_index != (uint32)-1) { + uint32 aux_stack_used = + (uint32)(module->module->aux_stack_bottom + - *(uint32 *)global_addr); + if (aux_stack_used > module->e->max_aux_stack_used) + module->e->max_aux_stack_used = aux_stack_used; + } +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL_64) + { + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + PUT_I64_TO_ADDR((uint32 *)global_addr, POP_I64()); + HANDLE_OP_END(); + } + + /* memory load instructions */ + HANDLE_OP(WASM_OP_I32_LOAD) + HANDLE_OP(WASM_OP_F32_LOAD) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + PUSH_I32(LOAD_I32(maddr)); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD) + HANDLE_OP(WASM_OP_F64_LOAD) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(8); + PUSH_I64(LOAD_I64(maddr)); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD8_S) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUSH_I32(sign_ext_8_32(*(int8 *)maddr)); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD8_U) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUSH_I32((uint32)(*(uint8 *)maddr)); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD16_S) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUSH_I32(sign_ext_16_32(LOAD_I16(maddr))); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD16_U) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUSH_I32((uint32)(LOAD_U16(maddr))); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD8_S) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUSH_I64(sign_ext_8_64(*(int8 *)maddr)); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD8_U) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUSH_I64((uint64)(*(uint8 *)maddr)); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD16_S) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUSH_I64(sign_ext_16_64(LOAD_I16(maddr))); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD16_U) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUSH_I64((uint64)(LOAD_U16(maddr))); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD32_S) + { + uint32 flags; + mem_offset_t offset, addr; + + opcode = *(frame_ip - 1); + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + PUSH_I64(sign_ext_32_64(LOAD_I32(maddr))); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD32_U) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + PUSH_I64((uint64)(LOAD_U32(maddr))); + CHECK_READ_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + /* memory store instructions */ + HANDLE_OP(WASM_OP_I32_STORE) + HANDLE_OP(WASM_OP_F32_STORE) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + frame_sp--; + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + STORE_U32(maddr, frame_sp[2]); + } + else +#endif + { + STORE_U32(maddr, frame_sp[1]); + } + CHECK_WRITE_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE) + HANDLE_OP(WASM_OP_F64_STORE) + { + uint32 flags; + mem_offset_t offset, addr; + + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + frame_sp -= 2; + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(8); + +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + PUT_I64_TO_ADDR((mem_offset_t *)maddr, + GET_I64_FROM_ADDR(frame_sp + 2)); + } + else +#endif + { + PUT_I64_TO_ADDR((uint32 *)maddr, + GET_I64_FROM_ADDR(frame_sp + 1)); + } + CHECK_WRITE_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_STORE8) + HANDLE_OP(WASM_OP_I32_STORE16) + { + uint32 flags; + mem_offset_t offset, addr; + uint32 sval; + + opcode = *(frame_ip - 1); + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + sval = (uint32)POP_I32(); + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_I32_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + *(uint8 *)maddr = (uint8)sval; + } + else { + CHECK_MEMORY_OVERFLOW(2); + STORE_U16(maddr, (uint16)sval); + } + CHECK_WRITE_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE8) + HANDLE_OP(WASM_OP_I64_STORE16) + HANDLE_OP(WASM_OP_I64_STORE32) + { + uint32 flags; + mem_offset_t offset, addr; + uint64 sval; + + opcode = *(frame_ip - 1); + read_leb_uint32(frame_ip, frame_ip_end, flags); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + sval = (uint64)POP_I64(); + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_I64_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + *(uint8 *)maddr = (uint8)sval; + } + else if (opcode == WASM_OP_I64_STORE16) { + CHECK_MEMORY_OVERFLOW(2); + STORE_U16(maddr, (uint16)sval); + } + else { + CHECK_MEMORY_OVERFLOW(4); + STORE_U32(maddr, (uint32)sval); + } + CHECK_WRITE_WATCHPOINT(addr, offset); + (void)flags; + HANDLE_OP_END(); + } + + /* memory size and memory grow instructions */ + HANDLE_OP(WASM_OP_MEMORY_SIZE) + { + uint32 reserved; + read_leb_uint32(frame_ip, frame_ip_end, reserved); + PUSH_PAGE_COUNT(memory->cur_page_count); + (void)reserved; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_MEMORY_GROW) + { + uint32 reserved, delta, + prev_page_count = memory->cur_page_count; + + read_leb_uint32(frame_ip, frame_ip_end, reserved); + delta = (uint32)POP_PAGE_COUNT(); + + if (!wasm_enlarge_memory(module, delta)) { + /* failed to memory.grow, return -1 */ + PUSH_PAGE_COUNT(-1); + } + else { + /* success, return previous page count */ + PUSH_PAGE_COUNT(prev_page_count); + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif + } + + (void)reserved; + HANDLE_OP_END(); + } + + /* constant instructions */ + HANDLE_OP(WASM_OP_I32_CONST) + DEF_OP_I_CONST(int32, I32); + HANDLE_OP_END(); + + HANDLE_OP(WASM_OP_I64_CONST) + DEF_OP_I_CONST(int64, I64); + HANDLE_OP_END(); + + HANDLE_OP(WASM_OP_F32_CONST) + { + uint8 *p_float = (uint8 *)frame_sp++; + for (i = 0; i < sizeof(float32); i++) + *p_float++ = *frame_ip++; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONST) + { + uint8 *p_float = (uint8 *)frame_sp++; + frame_sp++; + for (i = 0; i < sizeof(float64); i++) + *p_float++ = *frame_ip++; + HANDLE_OP_END(); + } + + /* comparison instructions of i32 */ + HANDLE_OP(WASM_OP_I32_EQZ) + { + DEF_OP_EQZ(I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EQ) + { + DEF_OP_CMP(uint32, I32, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_NE) + { + DEF_OP_CMP(uint32, I32, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LT_S) + { + DEF_OP_CMP(int32, I32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LT_U) + { + DEF_OP_CMP(uint32, I32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GT_S) + { + DEF_OP_CMP(int32, I32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GT_U) + { + DEF_OP_CMP(uint32, I32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LE_S) + { + DEF_OP_CMP(int32, I32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LE_U) + { + DEF_OP_CMP(uint32, I32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GE_S) + { + DEF_OP_CMP(int32, I32, >=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GE_U) + { + DEF_OP_CMP(uint32, I32, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of i64 */ + HANDLE_OP(WASM_OP_I64_EQZ) + { + DEF_OP_EQZ(I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EQ) + { + DEF_OP_CMP(uint64, I64, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_NE) + { + DEF_OP_CMP(uint64, I64, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LT_S) + { + DEF_OP_CMP(int64, I64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LT_U) + { + DEF_OP_CMP(uint64, I64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GT_S) + { + DEF_OP_CMP(int64, I64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GT_U) + { + DEF_OP_CMP(uint64, I64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LE_S) + { + DEF_OP_CMP(int64, I64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LE_U) + { + DEF_OP_CMP(uint64, I64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GE_S) + { + DEF_OP_CMP(int64, I64, >=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GE_U) + { + DEF_OP_CMP(uint64, I64, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of f32 */ + HANDLE_OP(WASM_OP_F32_EQ) + { + DEF_OP_CMP(float32, F32, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NE) + { + DEF_OP_CMP(float32, F32, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_LT) + { + DEF_OP_CMP(float32, F32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_GT) + { + DEF_OP_CMP(float32, F32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_LE) + { + DEF_OP_CMP(float32, F32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_GE) + { + DEF_OP_CMP(float32, F32, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of f64 */ + HANDLE_OP(WASM_OP_F64_EQ) + { + DEF_OP_CMP(float64, F64, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NE) + { + DEF_OP_CMP(float64, F64, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_LT) + { + DEF_OP_CMP(float64, F64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_GT) + { + DEF_OP_CMP(float64, F64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_LE) + { + DEF_OP_CMP(float64, F64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_GE) + { + DEF_OP_CMP(float64, F64, >=); + HANDLE_OP_END(); + } + + /* numberic instructions of i32 */ + HANDLE_OP(WASM_OP_I32_CLZ) + { + DEF_OP_BIT_COUNT(uint32, I32, clz32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_CTZ) + { + DEF_OP_BIT_COUNT(uint32, I32, ctz32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_POPCNT) + { + DEF_OP_BIT_COUNT(uint32, I32, popcount32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ADD) + { + DEF_OP_NUMERIC(uint32, uint32, I32, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SUB) + { + DEF_OP_NUMERIC(uint32, uint32, I32, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_MUL) + { + DEF_OP_NUMERIC(uint32, uint32, I32, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_DIV_S) + { + int32 a, b; + + b = POP_I32(); + a = POP_I32(); + if (a == (int32)0x80000000 && b == -1) { + wasm_set_exception(module, "integer overflow"); + goto got_exception; + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I32(a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_DIV_U) + { + uint32 a, b; + + b = (uint32)POP_I32(); + a = (uint32)POP_I32(); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I32(a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_REM_S) + { + int32 a, b; + + b = POP_I32(); + a = POP_I32(); + if (a == (int32)0x80000000 && b == -1) { + PUSH_I32(0); + HANDLE_OP_END(); + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I32(a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_REM_U) + { + uint32 a, b; + + b = (uint32)POP_I32(); + a = (uint32)POP_I32(); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I32(a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_AND) + { + DEF_OP_NUMERIC(uint32, uint32, I32, &); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_OR) + { + DEF_OP_NUMERIC(uint32, uint32, I32, |); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_XOR) + { + DEF_OP_NUMERIC(uint32, uint32, I32, ^); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHL) + { + DEF_OP_NUMERIC2(uint32, uint32, I32, <<); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHR_S) + { + DEF_OP_NUMERIC2(int32, uint32, I32, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHR_U) + { + DEF_OP_NUMERIC2(uint32, uint32, I32, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ROTL) + { + uint32 a, b; + + b = (uint32)POP_I32(); + a = (uint32)POP_I32(); + PUSH_I32(rotl32(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ROTR) + { + uint32 a, b; + + b = (uint32)POP_I32(); + a = (uint32)POP_I32(); + PUSH_I32(rotr32(a, b)); + HANDLE_OP_END(); + } + + /* numberic instructions of i64 */ + HANDLE_OP(WASM_OP_I64_CLZ) + { + DEF_OP_BIT_COUNT(uint64, I64, clz64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_CTZ) + { + DEF_OP_BIT_COUNT(uint64, I64, ctz64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_POPCNT) + { + DEF_OP_BIT_COUNT(uint64, I64, popcount64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ADD) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SUB) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_MUL) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_DIV_S) + { + int64 a, b; + + b = POP_I64(); + a = POP_I64(); + if (a == (int64)0x8000000000000000LL && b == -1) { + wasm_set_exception(module, "integer overflow"); + goto got_exception; + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I64(a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_DIV_U) + { + uint64 a, b; + + b = (uint64)POP_I64(); + a = (uint64)POP_I64(); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I64(a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REM_S) + { + int64 a, b; + + b = POP_I64(); + a = POP_I64(); + if (a == (int64)0x8000000000000000LL && b == -1) { + PUSH_I64(0); + HANDLE_OP_END(); + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I64(a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REM_U) + { + uint64 a, b; + + b = (uint64)POP_I64(); + a = (uint64)POP_I64(); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUSH_I64(a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_AND) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, &); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_OR) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, |); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_XOR) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, ^); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHL) + { + DEF_OP_NUMERIC2_64(uint64, uint64, I64, <<); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHR_S) + { + DEF_OP_NUMERIC2_64(int64, uint64, I64, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHR_U) + { + DEF_OP_NUMERIC2_64(uint64, uint64, I64, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ROTL) + { + uint64 a, b; + + b = (uint64)POP_I64(); + a = (uint64)POP_I64(); + PUSH_I64(rotl64(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ROTR) + { + uint64 a, b; + + b = (uint64)POP_I64(); + a = (uint64)POP_I64(); + PUSH_I64(rotr64(a, b)); + HANDLE_OP_END(); + } + + /* numberic instructions of f32 */ + HANDLE_OP(WASM_OP_F32_ABS) + { + DEF_OP_MATH(float32, F32, fabsf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NEG) + { + uint32 u32 = frame_sp[-1]; + uint32 sign_bit = u32 & ((uint32)1 << 31); + if (sign_bit) + frame_sp[-1] = u32 & ~((uint32)1 << 31); + else + frame_sp[-1] = u32 | ((uint32)1 << 31); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CEIL) + { + DEF_OP_MATH(float32, F32, ceilf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_FLOOR) + { + DEF_OP_MATH(float32, F32, floorf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_TRUNC) + { + DEF_OP_MATH(float32, F32, truncf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NEAREST) + { + DEF_OP_MATH(float32, F32, rintf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_SQRT) + { + DEF_OP_MATH(float32, F32, sqrtf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_ADD) + { + DEF_OP_NUMERIC(float32, float32, F32, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_SUB) + { + DEF_OP_NUMERIC(float32, float32, F32, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MUL) + { + DEF_OP_NUMERIC(float32, float32, F32, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_DIV) + { + DEF_OP_NUMERIC(float32, float32, F32, /); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MIN) + { + float32 a, b; + + b = POP_F32(); + a = POP_F32(); + + PUSH_F32(f32_min(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MAX) + { + float32 a, b; + + b = POP_F32(); + a = POP_F32(); + + PUSH_F32(f32_max(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_COPYSIGN) + { + float32 a, b; + + b = POP_F32(); + a = POP_F32(); + PUSH_F32(local_copysignf(a, b)); + HANDLE_OP_END(); + } + + /* numberic instructions of f64 */ + HANDLE_OP(WASM_OP_F64_ABS) + { + DEF_OP_MATH(float64, F64, fabs); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NEG) + { + uint64 u64 = GET_I64_FROM_ADDR(frame_sp - 2); + uint64 sign_bit = u64 & (((uint64)1) << 63); + if (sign_bit) + PUT_I64_TO_ADDR(frame_sp - 2, (u64 & ~(((uint64)1) << 63))); + else + PUT_I64_TO_ADDR(frame_sp - 2, (u64 | (((uint64)1) << 63))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CEIL) + { + DEF_OP_MATH(float64, F64, ceil); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_FLOOR) + { + DEF_OP_MATH(float64, F64, floor); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_TRUNC) + { + DEF_OP_MATH(float64, F64, trunc); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NEAREST) + { + DEF_OP_MATH(float64, F64, rint); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_SQRT) + { + DEF_OP_MATH(float64, F64, sqrt); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_ADD) + { + DEF_OP_NUMERIC_64(float64, float64, F64, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_SUB) + { + DEF_OP_NUMERIC_64(float64, float64, F64, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MUL) + { + DEF_OP_NUMERIC_64(float64, float64, F64, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_DIV) + { + DEF_OP_NUMERIC_64(float64, float64, F64, /); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MIN) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + + PUSH_F64(f64_min(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MAX) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + + PUSH_F64(f64_max(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_COPYSIGN) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + PUSH_F64(local_copysign(a, b)); + HANDLE_OP_END(); + } + + /* conversions of i32 */ + HANDLE_OP(WASM_OP_I32_WRAP_I64) + { + int32 value = (int32)(POP_I64() & 0xFFFFFFFFLL); + PUSH_I32(value); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_S_F32) + { + /* We don't use INT32_MIN/INT32_MAX/UINT32_MIN/UINT32_MAX, + since float/double values of ieee754 cannot precisely + represent all int32/uint32/int64/uint64 values, e.g. + UINT32_MAX is 4294967295, but (float32)4294967295 is + 4294967296.0f, but not 4294967295.0f. */ + DEF_OP_TRUNC_F32(-2147483904.0f, 2147483648.0f, true, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_U_F32) + { + DEF_OP_TRUNC_F32(-1.0f, 4294967296.0f, true, false); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_S_F64) + { + DEF_OP_TRUNC_F64(-2147483649.0, 2147483648.0, true, true); + /* frame_sp can't be moved in trunc function, we need to + manually adjust it if src and dst op's cell num is + different */ + frame_sp--; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_U_F64) + { + DEF_OP_TRUNC_F64(-1.0, 4294967296.0, true, false); + frame_sp--; + HANDLE_OP_END(); + } + + /* conversions of i64 */ + HANDLE_OP(WASM_OP_I64_EXTEND_S_I32) + { + DEF_OP_CONVERT(int64, I64, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND_U_I32) + { + DEF_OP_CONVERT(int64, I64, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_S_F32) + { + DEF_OP_TRUNC_F32(-9223373136366403584.0f, + 9223372036854775808.0f, false, true); + frame_sp++; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_U_F32) + { + DEF_OP_TRUNC_F32(-1.0f, 18446744073709551616.0f, false, false); + frame_sp++; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_S_F64) + { + DEF_OP_TRUNC_F64(-9223372036854777856.0, 9223372036854775808.0, + false, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_U_F64) + { + DEF_OP_TRUNC_F64(-1.0, 18446744073709551616.0, false, false); + HANDLE_OP_END(); + } + + /* conversions of f32 */ + HANDLE_OP(WASM_OP_F32_CONVERT_S_I32) + { + DEF_OP_CONVERT(float32, F32, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_U_I32) + { + DEF_OP_CONVERT(float32, F32, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_S_I64) + { + DEF_OP_CONVERT(float32, F32, int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_U_I64) + { + DEF_OP_CONVERT(float32, F32, uint64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_DEMOTE_F64) + { + DEF_OP_CONVERT(float32, F32, float64, F64); + HANDLE_OP_END(); + } + + /* conversions of f64 */ + HANDLE_OP(WASM_OP_F64_CONVERT_S_I32) + { + DEF_OP_CONVERT(float64, F64, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_U_I32) + { + DEF_OP_CONVERT(float64, F64, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_S_I64) + { + DEF_OP_CONVERT(float64, F64, int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_U_I64) + { + DEF_OP_CONVERT(float64, F64, uint64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_PROMOTE_F32) + { + DEF_OP_CONVERT(float64, F64, float32, F32); + HANDLE_OP_END(); + } + + /* reinterpretations */ + HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) + HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) + HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) { HANDLE_OP_END(); } + + HANDLE_OP(WASM_OP_I32_EXTEND8_S) + { + DEF_OP_CONVERT(int32, I32, int8, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EXTEND16_S) + { + DEF_OP_CONVERT(int32, I32, int16, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND8_S) + { + DEF_OP_CONVERT(int64, I64, int8, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND16_S) + { + DEF_OP_CONVERT(int64, I64, int16, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND32_S) + { + DEF_OP_CONVERT(int64, I64, int32, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_MISC_PREFIX) + { + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + DEF_OP_TRUNC_SAT_F32(-2147483904.0f, 2147483648.0f, + true, true); + break; + case WASM_OP_I32_TRUNC_SAT_U_F32: + DEF_OP_TRUNC_SAT_F32(-1.0f, 4294967296.0f, true, false); + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + DEF_OP_TRUNC_SAT_F64(-2147483649.0, 2147483648.0, true, + true); + frame_sp--; + break; + case WASM_OP_I32_TRUNC_SAT_U_F64: + DEF_OP_TRUNC_SAT_F64(-1.0, 4294967296.0, true, false); + frame_sp--; + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + DEF_OP_TRUNC_SAT_F32(-9223373136366403584.0f, + 9223372036854775808.0f, false, + true); + frame_sp++; + break; + case WASM_OP_I64_TRUNC_SAT_U_F32: + DEF_OP_TRUNC_SAT_F32(-1.0f, 18446744073709551616.0f, + false, false); + frame_sp++; + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + DEF_OP_TRUNC_SAT_F64(-9223372036854777856.0, + 9223372036854775808.0, false, + true); + break; + case WASM_OP_I64_TRUNC_SAT_U_F64: + DEF_OP_TRUNC_SAT_F64(-1.0f, 18446744073709551616.0, + false, false); + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + uint32 segment; + mem_offset_t addr; + uint64 bytes, offset, seg_len; + uint8 *data; + + read_leb_uint32(frame_ip, frame_ip_end, segment); + /* skip memory index */ + frame_ip++; + + bytes = (uint64)(uint32)POP_I32(); + offset = (uint64)(uint32)POP_I32(); + addr = (mem_offset_t)POP_MEM_OFFSET(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); +#else + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; +#endif + + if (bh_bitmap_get_bit(module->e->common.data_dropped, + segment)) { + seg_len = 0; + data = NULL; + } + else { + seg_len = + (uint64)module->module->data_segments[segment] + ->data_length; + data = module->module->data_segments[segment]->data; + } + if (offset + bytes > seg_len) + goto out_of_bounds; + + bh_memcpy_s(maddr, (uint32)(linear_mem_size - addr), + data + offset, (uint32)bytes); + break; + } + case WASM_OP_DATA_DROP: + { + uint32 segment; + + read_leb_uint32(frame_ip, frame_ip_end, segment); + bh_bitmap_set_bit(module->e->common.data_dropped, + segment); + break; + } + case WASM_OP_MEMORY_COPY: + { + mem_offset_t dst, src, len; + uint8 *mdst, *msrc; + + frame_ip += 2; + len = POP_MEM_OFFSET(); + src = POP_MEM_OFFSET(); + dst = POP_MEM_OFFSET(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); + CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); +#else + if ((uint64)(uint32)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + (uint32)src; + + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; +#endif + + /* allowing the destination and source to overlap */ + bh_memmove_s(mdst, (uint32)(linear_mem_size - dst), + msrc, len); + break; + } + case WASM_OP_MEMORY_FILL: + { + mem_offset_t dst, len; + uint8 fill_val, *mdst; + frame_ip++; + + len = POP_MEM_OFFSET(); + fill_val = POP_I32(); + dst = POP_MEM_OFFSET(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); +#else + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; +#endif + + memset(mdst, fill_val, len); + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_TABLE_INIT: + { + uint32 tbl_idx, elem_idx; + uint32 n, s, d; + WASMTableInstance *tbl_inst; + table_elem_type_t *table_elems; + InitializerExpression *tbl_seg_init_values = NULL, + *init_values; + uint32 tbl_seg_len = 0; + + read_leb_uint32(frame_ip, frame_ip_end, elem_idx); + bh_assert(elem_idx < module->module->table_seg_count); + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + bh_assert(tbl_idx < module->module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + n = (uint32)POP_I32(); + s = (uint32)POP_I32(); + d = (uint32)POP_I32(); + + if (!bh_bitmap_get_bit(module->e->common.elem_dropped, + elem_idx)) { + /* table segment isn't dropped */ + tbl_seg_init_values = + module->module->table_segments[elem_idx] + .init_values; + tbl_seg_len = + module->module->table_segments[elem_idx] + .value_count; + } + + if (offset_len_out_of_bounds(s, n, tbl_seg_len) + || offset_len_out_of_bounds(d, n, + tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + if (!n) { + break; + } + + table_elems = tbl_inst->elems + d; + init_values = tbl_seg_init_values + s; +#if WASM_ENABLE_GC != 0 + SYNC_ALL_TO_FRAME(); +#endif + for (i = 0; i < n; i++) { + /* UINT32_MAX indicates that it is a null ref */ + bh_assert(init_values[i].init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST + || init_values[i].init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST); +#if WASM_ENABLE_GC == 0 + table_elems[i] = + (table_elem_type_t)init_values[i].u.ref_index; +#else + if (init_values[i].u.ref_index != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module, init_values[i].u.ref_index, + true, NULL, 0))) { + goto got_exception; + } + table_elems[i] = func_obj; + } + else { + table_elems[i] = NULL_REF; + } +#endif + } + break; + } + case WASM_OP_ELEM_DROP: + { + uint32 elem_idx; + read_leb_uint32(frame_ip, frame_ip_end, elem_idx); + bh_assert(elem_idx < module->module->table_seg_count); + + bh_bitmap_set_bit(module->e->common.elem_dropped, + elem_idx); + break; + } + case WASM_OP_TABLE_COPY: + { + uint32 src_tbl_idx, dst_tbl_idx; + uint32 n, s, d; + WASMTableInstance *src_tbl_inst, *dst_tbl_inst; + + read_leb_uint32(frame_ip, frame_ip_end, dst_tbl_idx); + bh_assert(dst_tbl_idx < module->table_count); + + dst_tbl_inst = wasm_get_table_inst(module, dst_tbl_idx); + + read_leb_uint32(frame_ip, frame_ip_end, src_tbl_idx); + bh_assert(src_tbl_idx < module->table_count); + + src_tbl_inst = wasm_get_table_inst(module, src_tbl_idx); + + n = (uint32)POP_I32(); + s = (uint32)POP_I32(); + d = (uint32)POP_I32(); + + if (offset_len_out_of_bounds(d, n, + dst_tbl_inst->cur_size) + || offset_len_out_of_bounds( + s, n, src_tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + /* if s >= d, copy from front to back */ + /* if s < d, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + + offsetof(WASMTableInstance, elems) + + d * sizeof(table_elem_type_t), + (uint32)((dst_tbl_inst->cur_size - d) + * sizeof(table_elem_type_t)), + (uint8 *)src_tbl_inst + + offsetof(WASMTableInstance, elems) + + s * sizeof(table_elem_type_t), + (uint32)(n * sizeof(table_elem_type_t))); + break; + } + case WASM_OP_TABLE_GROW: + { + WASMTableInstance *tbl_inst; + uint32 tbl_idx, n, orig_tbl_sz; + table_elem_type_t init_val; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + orig_tbl_sz = tbl_inst->cur_size; + + n = POP_I32(); +#if WASM_ENABLE_GC == 0 + init_val = POP_I32(); +#else + init_val = POP_REF(); +#endif + + if (!wasm_enlarge_table(module, tbl_idx, n, init_val)) { + PUSH_I32(-1); + } + else { + PUSH_I32(orig_tbl_sz); + } + break; + } + case WASM_OP_TABLE_SIZE: + { + uint32 tbl_idx; + WASMTableInstance *tbl_inst; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + PUSH_I32(tbl_inst->cur_size); + break; + } + case WASM_OP_TABLE_FILL: + { + uint32 tbl_idx, n; + WASMTableInstance *tbl_inst; + table_elem_type_t fill_val; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + n = POP_I32(); +#if WASM_ENABLE_GC == 0 + fill_val = POP_I32(); +#else + fill_val = POP_REF(); +#endif + i = POP_I32(); + + if (offset_len_out_of_bounds(i, n, + tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + for (; n != 0; i++, n--) { + tbl_inst->elems[i] = fill_val; + } + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + default: + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + HANDLE_OP(WASM_OP_ATOMIC_PREFIX) + { + mem_offset_t offset = 0, addr; + uint32 align = 0; + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + /* opcode1 was checked in loader and is no larger than + UINT8_MAX */ + opcode = (uint8)opcode1; + + if (opcode != WASM_OP_ATOMIC_FENCE) { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + } + + switch (opcode) { + case WASM_OP_ATOMIC_NOTIFY: + { + uint32 notify_count, ret; + + notify_count = POP_I32(); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + + ret = wasm_runtime_atomic_notify( + (WASMModuleInstanceCommon *)module, maddr, + notify_count); + if (ret == (uint32)-1) + goto got_exception; + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT32: + { + uint64 timeout; + uint32 expect, ret; + + timeout = POP_I64(); + expect = POP_I32(); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + + ret = wasm_runtime_atomic_wait( + (WASMModuleInstanceCommon *)module, maddr, + (uint64)expect, timeout, false); + if (ret == (uint32)-1) + goto got_exception; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT64: + { + uint64 timeout, expect; + uint32 ret; + + timeout = POP_I64(); + expect = POP_I64(); + addr = POP_MEM_OFFSET(); + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(); + + ret = wasm_runtime_atomic_wait( + (WASMModuleInstanceCommon *)module, maddr, expect, + timeout, true); + if (ret == (uint32)-1) + goto got_exception; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_FENCE: + { + /* Skip the memory index */ + frame_ip++; + os_atomic_thread_fence(os_memory_order_seq_cst); + break; + } + + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + { + uint32 readv; + + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + readv = (uint32)(*(uint8 *)maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + readv = (uint32)LOAD_U16(maddr); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + readv = LOAD_I32(maddr); + shared_memory_unlock(memory); + } + + PUSH_I32(readv); + break; + } + + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + { + uint64 readv; + + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + readv = (uint64)(*(uint8 *)maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + readv = (uint64)LOAD_U16(maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + readv = (uint64)LOAD_U32(maddr); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + readv = LOAD_I64(maddr); + shared_memory_unlock(memory); + } + + PUSH_I64(readv); + break; + } + + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + { + uint32 sval; + + sval = (uint32)POP_I32(); + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_ATOMIC_I32_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + *(uint8 *)maddr = (uint8)sval; + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + STORE_U16(maddr, (uint16)sval); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + STORE_U32(maddr, sval); + shared_memory_unlock(memory); + } + break; + } + + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + { + uint64 sval; + + sval = (uint64)POP_I64(); + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_ATOMIC_I64_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + *(uint8 *)maddr = (uint8)sval; + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + STORE_U16(maddr, (uint16)sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + STORE_U32(maddr, (uint32)sval); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(); + shared_memory_lock(memory); + STORE_I64(maddr, sval); + shared_memory_unlock(memory); + } + break; + } + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + { + uint32 readv, sval, expect; + + sval = POP_I32(); + expect = POP_I32(); + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(); + + expect = (uint8)expect; + shared_memory_lock(memory); + readv = (uint32)(*(uint8 *)maddr); + if (readv == expect) + *(uint8 *)maddr = (uint8)(sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(); + + expect = (uint16)expect; + shared_memory_lock(memory); + readv = (uint32)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + + shared_memory_lock(memory); + readv = LOAD_I32(maddr); + if (readv == expect) + STORE_U32(maddr, sval); + shared_memory_unlock(memory); + } + PUSH_I32(readv); + break; + } + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + { + uint64 readv, sval, expect; + + sval = (uint64)POP_I64(); + expect = (uint64)POP_I64(); + addr = POP_MEM_OFFSET(); + + if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(); + + expect = (uint8)expect; + shared_memory_lock(memory); + readv = (uint64)(*(uint8 *)maddr); + if (readv == expect) + *(uint8 *)maddr = (uint8)(sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(); + + expect = (uint16)expect; + shared_memory_lock(memory); + readv = (uint64)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(); + + expect = (uint32)expect; + shared_memory_lock(memory); + readv = (uint64)LOAD_U32(maddr); + if (readv == expect) + STORE_U32(maddr, (uint32)(sval)); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(); + + shared_memory_lock(memory); + readv = (uint64)LOAD_I64(maddr); + if (readv == expect) + STORE_I64(maddr, sval); + shared_memory_unlock(memory); + } + PUSH_I64(readv); + break; + } + + DEF_ATOMIC_RMW_OPCODE(ADD, +); + DEF_ATOMIC_RMW_OPCODE(SUB, -); + DEF_ATOMIC_RMW_OPCODE(AND, &); + DEF_ATOMIC_RMW_OPCODE(OR, |); + DEF_ATOMIC_RMW_OPCODE(XOR, ^); + /* xchg, ignore the read value, and store the given + value: readv * 0 + sval */ + DEF_ATOMIC_RMW_OPCODE(XCHG, *0 +); + } + + HANDLE_OP_END(); + } +#endif + + HANDLE_OP(WASM_OP_IMPDEP) + { + frame = prev_frame; + frame_ip = frame->ip; + frame_sp = frame->sp; + frame_csp = frame->csp; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif + goto call_func_from_entry; + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 + HANDLE_OP(DEBUG_OP_BREAK) + { + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TRAP); + WASM_SUSPEND_FLAGS_FETCH_OR(exec_env->suspend_flags, + WASM_SUSPEND_FLAG_SUSPEND); + frame_ip--; + SYNC_ALL_TO_FRAME(); + CHECK_SUSPEND_FLAGS(); + HANDLE_OP_END(); + } +#endif +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + default: + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 + HANDLE_OP(WASM_OP_UNUSED_0x0a) +#if WASM_ENABLE_TAIL_CALL == 0 + HANDLE_OP(WASM_OP_RETURN_CALL) + HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT) +#endif +#if WASM_ENABLE_SHARED_MEMORY == 0 + HANDLE_OP(WASM_OP_ATOMIC_PREFIX) +#endif +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + HANDLE_OP(WASM_OP_SELECT_T) + HANDLE_OP(WASM_OP_TABLE_GET) + HANDLE_OP(WASM_OP_TABLE_SET) + HANDLE_OP(WASM_OP_REF_NULL) + HANDLE_OP(WASM_OP_REF_IS_NULL) + HANDLE_OP(WASM_OP_REF_FUNC) +#endif +#if WASM_ENABLE_GC == 0 + HANDLE_OP(WASM_OP_CALL_REF) + HANDLE_OP(WASM_OP_RETURN_CALL_REF) + HANDLE_OP(WASM_OP_REF_EQ) + HANDLE_OP(WASM_OP_REF_AS_NON_NULL) + HANDLE_OP(WASM_OP_BR_ON_NULL) + HANDLE_OP(WASM_OP_BR_ON_NON_NULL) + HANDLE_OP(WASM_OP_GC_PREFIX) +#endif +#if WASM_ENABLE_EXCE_HANDLING == 0 + HANDLE_OP(WASM_OP_TRY) + HANDLE_OP(WASM_OP_CATCH) + HANDLE_OP(WASM_OP_THROW) + HANDLE_OP(WASM_OP_RETHROW) + HANDLE_OP(WASM_OP_DELEGATE) + HANDLE_OP(WASM_OP_CATCH_ALL) + HANDLE_OP(EXT_OP_TRY) +#endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SIMD != 0 + /* SIMD isn't supported by interpreter, but when JIT is + enabled, `iwasm --interp ` may be run to + trigger the SIMD opcode in interpreter */ + HANDLE_OP(WASM_OP_SIMD_PREFIX) +#endif + HANDLE_OP(WASM_OP_UNUSED_0x16) + HANDLE_OP(WASM_OP_UNUSED_0x17) + HANDLE_OP(WASM_OP_UNUSED_0x27) + /* Used by fast interpreter */ + HANDLE_OP(EXT_OP_SET_LOCAL_FAST_I64) + HANDLE_OP(EXT_OP_TEE_LOCAL_FAST_I64) + HANDLE_OP(EXT_OP_COPY_STACK_TOP) + HANDLE_OP(EXT_OP_COPY_STACK_TOP_I64) + HANDLE_OP(EXT_OP_COPY_STACK_VALUES) + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES != 0 */ + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + continue; +#else + FETCH_OPCODE_AND_DISPATCH(); +#endif + +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + call_func_from_return_call: + { + POP(cur_func->param_cell_num); + if (cur_func->param_cell_num > 0) { + word_copy(frame->lp, frame_sp, cur_func->param_cell_num); + } + FREE_FRAME(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); + is_return_call = true; + goto call_func_from_entry; + } +#endif + call_func_from_interp: + { + /* Only do the copy when it's called from interpreter. */ + WASMInterpFrame *outs_area = wasm_exec_env_wasm_stack_top(exec_env); + if (cur_func->param_cell_num > 0) { + POP(cur_func->param_cell_num); + word_copy(outs_area->lp, frame_sp, cur_func->param_cell_num); + } + SYNC_ALL_TO_FRAME(); + prev_frame = frame; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif + } + + call_func_from_entry: + { + if (cur_func->is_import_func) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (cur_func->import_func_inst) { + wasm_interp_call_func_import(module, exec_env, cur_func, + prev_frame); +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (is_return_call) { + /* the frame was freed before tail calling and + the prev_frame was set as exec_env's cur_frame, + so here we recover context from prev_frame */ + RECOVER_CONTEXT(prev_frame); + } + else +#endif + { + prev_frame = frame->prev_frame; + cur_func = frame->function; + UPDATE_ALL_FROM_FRAME(); + } + +#if WASM_ENABLE_EXCE_HANDLING != 0 + char uncaught_exception[128] = { 0 }; + bool has_exception = + wasm_copy_exception(module, uncaught_exception); + if (has_exception + && strstr(uncaught_exception, "uncaught wasm exception")) { + uint32 import_exception; + /* initialize imported exception index to be invalid */ + SET_INVALID_TAGINDEX(import_exception); + + /* pull external exception */ + uint32 ext_exception = POP_I32(); + + /* external function came back with an exception or trap */ + /* lookup exception in import tags */ + WASMTagInstance *tag = module->e->tags; + for (uint32 t = 0; t < module->module->import_tag_count; + tag++, t++) { + + /* compare the module and the external index with the + * imort tag data */ + if ((cur_func->u.func_import->import_module + == tag->u.tag_import->import_module) + && (ext_exception + == tag->u.tag_import + ->import_tag_index_linked)) { + /* set the import_exception to the import tag */ + import_exception = t; + break; + } + } + /* + * excange the thrown exception (index valid in submodule) + * with the imported exception index (valid in this module) + * if the module did not import the exception, + * that results in a "INVALID_TAGINDEX", that triggers + * an CATCH_ALL block, if there is one. + */ + PUSH_I32(import_exception); + } +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + } + else +#endif /* end of WASM_ENABLE_MULTI_MODULE != 0 */ + { + wasm_interp_call_func_native(module, exec_env, cur_func, + prev_frame); +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (is_return_call) { + /* the frame was freed before tail calling and + the prev_frame was set as exec_env's cur_frame, + so here we recover context from prev_frame */ + RECOVER_CONTEXT(prev_frame); + } + else +#endif + { + prev_frame = frame->prev_frame; + cur_func = frame->function; + UPDATE_ALL_FROM_FRAME(); + } + } + + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + if (memory) + linear_mem_size = get_linear_mem_size(); +#endif + if (wasm_copy_exception(module, NULL)) { +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* the caller raised an exception */ + char uncaught_exception[128] = { 0 }; + bool has_exception = + wasm_copy_exception(module, uncaught_exception); + + /* libc_builtin signaled a "exception thrown by stdc++" trap */ + if (has_exception + && strstr(uncaught_exception, + "exception thrown by stdc++")) { + wasm_set_exception(module, NULL); + + /* setup internal c++ rethrow */ + exception_tag_index = 0; + goto find_a_catch_handler; + } + + /* when throw hits the end of a function it signalles with a + * "uncaught wasm exception" trap */ + if (has_exception + && strstr(uncaught_exception, "uncaught wasm exception")) { + wasm_set_exception(module, NULL); + exception_tag_index = POP_I32(); + + /* rethrow the exception into that frame */ + goto find_a_catch_handler; + } +#endif /* WASM_ENABLE_EXCE_HANDLING != 0 */ + goto got_exception; + } + } + else { + WASMFunction *cur_wasm_func = cur_func->u.func; + WASMFuncType *func_type = cur_wasm_func->func_type; + uint32 max_stack_cell_num = cur_wasm_func->max_stack_cell_num; + uint32 cell_num_of_local_stack; + +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* account for exception handlers, bundle them here */ + uint32 eh_size = + cur_wasm_func->exception_handler_count * sizeof(uint8 *); + max_stack_cell_num += eh_size; +#endif + + cell_num_of_local_stack = cur_func->param_cell_num + + cur_func->local_cell_num + + max_stack_cell_num; + all_cell_num = cell_num_of_local_stack + + cur_wasm_func->max_block_num + * (uint32)sizeof(WASMBranchBlock) / 4; +#if WASM_ENABLE_GC != 0 + /* area of frame_ref */ + all_cell_num += (cell_num_of_local_stack + 3) / 4; +#endif + + /* param_cell_num, local_cell_num, max_stack_cell_num and + max_block_num are all no larger than UINT16_MAX (checked + in loader), all_cell_num must be smaller than 1MB */ + bh_assert(all_cell_num < 1 * BH_MB); + + frame_size = wasm_interp_interp_frame_size(all_cell_num); + if (!(frame = ALLOC_FRAME(exec_env, frame_size, prev_frame))) { + frame = prev_frame; + goto got_exception; + } + + /* Initialize the interpreter context. */ + frame->function = cur_func; + frame_ip = wasm_get_func_code(cur_func); + frame_ip_end = wasm_get_func_code_end(cur_func); + frame_lp = frame->lp; + + frame_sp = frame->sp_bottom = + frame_lp + cur_func->param_cell_num + cur_func->local_cell_num; + frame->sp_boundary = frame->sp_bottom + max_stack_cell_num; + + frame_csp = frame->csp_bottom = + (WASMBranchBlock *)frame->sp_boundary; + frame->csp_boundary = + frame->csp_bottom + cur_wasm_func->max_block_num; + +#if WASM_ENABLE_GC != 0 + /* frame->sp and frame->ip are used during GC root set enumeration, + * so we must initialized these fields here */ + frame->sp = frame_sp; + frame->ip = frame_ip; + frame_ref = (uint8 *)frame->csp_boundary; + init_frame_refs(frame_ref, (uint32)cell_num_of_local_stack, + cur_func); +#endif + + /* Initialize the local variables */ + memset(frame_lp + cur_func->param_cell_num, 0, + (uint32)(cur_func->local_cell_num * 4)); + + /* Push function block as first block */ + cell_num = func_type->ret_cell_num; + PUSH_CSP(LABEL_TYPE_FUNCTION, 0, cell_num, frame_ip_end - 1); + + wasm_exec_env_set_cur_frame(exec_env, frame); + } +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + HANDLE_OP_END(); + } + + return_func: + { + FREE_FRAME(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); + + if (!prev_frame->ip) { + /* Called from native. */ + return; + } + + RECOVER_CONTEXT(prev_frame); +#if WASM_ENABLE_EXCE_HANDLING != 0 + if (wasm_get_exception(module)) { + wasm_set_exception(module, NULL); + exception_tag_index = POP_I32(); + goto find_a_catch_handler; + } +#endif + HANDLE_OP_END(); + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + unaligned_atomic: + wasm_set_exception(module, "unaligned atomic"); + goto got_exception; +#endif + +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + out_of_bounds: + wasm_set_exception(module, "out of bounds memory access"); +#endif + + got_exception: +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (wasm_exec_env_get_instance(exec_env) != NULL) { + uint8 *frame_ip_temp = frame_ip; + frame_ip = frame_ip_orig; + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TRAP); + CHECK_SUSPEND_FLAGS(); + frame_ip = frame_ip_temp; + } +#endif + SYNC_ALL_TO_FRAME(); + return; + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + } +#else + FETCH_OPCODE_AND_DISPATCH(); +#endif +} + +#if WASM_ENABLE_GC != 0 +bool +wasm_interp_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMInterpFrame *frame; + WASMObjectRef gc_obj; + int i; + + frame = wasm_exec_env_get_cur_frame(exec_env); + for (; frame; frame = frame->prev_frame) { + uint8 *frame_ref = get_frame_ref(frame); + for (i = 0; i < frame->sp - frame->lp; i++) { + if (frame_ref[i]) { + gc_obj = GET_REF_FROM_ADDR(frame->lp + i); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) { + return false; + } + } +#if UINTPTR_MAX == UINT64_MAX + bh_assert(frame_ref[i + 1]); + i++; +#endif + } + } + } + return true; +} +#endif + +#if WASM_ENABLE_FAST_JIT != 0 +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +static void +fast_jit_call_func_bytecode(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *function, + WASMInterpFrame *frame) +{ + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + JitInterpSwitchInfo info; + WASMModule *module = module_inst->module; + WASMFuncType *func_type = function->u.func->func_type; + uint8 type = func_type->result_count + ? func_type->types[func_type->param_count] + : VALUE_TYPE_VOID; + uint32 func_idx = (uint32)(function - module_inst->e->functions); + uint32 func_idx_non_import = func_idx - module->import_function_count; + int32 action; + +#if WASM_ENABLE_REF_TYPES != 0 + if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF) + type = VALUE_TYPE_I32; +#endif + +#if WASM_ENABLE_LAZY_JIT != 0 + if (!jit_compiler_compile(module, func_idx)) { + wasm_set_exception(module_inst, "failed to compile fast jit function"); + return; + } +#endif + bh_assert(jit_compiler_is_compiled(module, func_idx)); + + /* Switch to jitted code to call the jit function */ + info.out.ret.last_return_type = type; + info.frame = frame; + frame->jitted_return_addr = + (uint8 *)jit_globals->return_to_interp_from_jitted; + action = jit_interp_switch_to_jitted( + exec_env, &info, func_idx, + module_inst->fast_jit_func_ptrs[func_idx_non_import]); + bh_assert(action == JIT_INTERP_ACTION_NORMAL + || (action == JIT_INTERP_ACTION_THROWN + && wasm_copy_exception( + (WASMModuleInstance *)exec_env->module_inst, NULL))); + + /* Get the return values form info.out.ret */ + if (func_type->result_count) { + switch (type) { + case VALUE_TYPE_I32: + *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0]; + break; + case VALUE_TYPE_I64: + *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0]; + *(frame->sp - function->ret_cell_num + 1) = + info.out.ret.ival[1]; + break; + case VALUE_TYPE_F32: + *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0]; + break; + case VALUE_TYPE_F64: + *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0]; + *(frame->sp - function->ret_cell_num + 1) = + info.out.ret.fval[1]; + break; + default: + bh_assert(0); + break; + } + } + (void)action; + (void)func_idx; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 */ + +#if WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 \ + || WASM_ENABLE_AOT_STACK_FRAME != 0 +#if WASM_ENABLE_GC == 0 +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + WASMInterpFrame *cur_frame, *frame; + uint32 size = (uint32)offsetof(WASMInterpFrame, lp); + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + cur_frame = exec_env->cur_frame; + if (!cur_frame) + frame = (WASMInterpFrame *)exec_env->wasm_stack.bottom; + else + frame = (WASMInterpFrame *)((uint8 *)cur_frame + size); + + if ((uint8 *)frame + size > exec_env->wasm_stack.top_boundary) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + return false; + } + + frame->function = module_inst->e->functions + func_index; + /* No need to initialize ip, it will be committed in jitted code + when needed */ + /* frame->ip = NULL; */ + frame->prev_frame = cur_frame; + +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = os_time_thread_cputime_us(); +#endif +#if WASM_ENABLE_MEMORY_PROFILING != 0 + { + uint32 wasm_stack_used = + (uint8 *)frame + size - exec_env->wasm_stack.bottom; + if (wasm_stack_used > exec_env->max_wasm_stack_used) + exec_env->max_wasm_stack_used = wasm_stack_used; + } +#endif + + exec_env->cur_frame = frame; + + return true; +} + +static inline void +llvm_jit_free_frame_internal(WASMExecEnv *exec_env) +{ + WASMInterpFrame *frame = exec_env->cur_frame; + WASMInterpFrame *prev_frame = frame->prev_frame; + + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); + +#if WASM_ENABLE_PERF_PROFILING != 0 + if (frame->function) { + uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started; + + frame->function->total_exec_time += time_elapsed; + frame->function->total_exec_cnt++; + + /* parent function */ + if (prev_frame) + prev_frame->function->children_exec_time += time_elapsed; + } +#endif + exec_env->cur_frame = prev_frame; +} + +void +llvm_jit_free_frame(WASMExecEnv *exec_env) +{ + llvm_jit_free_frame_internal(exec_env); +} + +#else /* else of WASM_ENABLE_GC == 0 */ + +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + WASMModuleInstance *module_inst; + WASMModule *module; + WASMInterpFrame *frame; + uint32 size, max_local_cell_num, max_stack_cell_num; + + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + module = module_inst->module; + + if (func_index >= func_index - module->import_function_count) { + WASMFunction *func = + module->functions[func_index - module->import_function_count]; + + max_local_cell_num = func->param_cell_num + func->local_cell_num; + max_stack_cell_num = func->max_stack_cell_num; + } + else { + WASMFunctionImport *func = + &((module->import_functions + func_index)->u.function); + + max_local_cell_num = func->func_type->param_cell_num > 2 + ? func->func_type->param_cell_num + : 2; + max_stack_cell_num = 0; + } + + size = + wasm_interp_interp_frame_size(max_local_cell_num + max_stack_cell_num); + + frame = wasm_exec_env_alloc_wasm_frame(exec_env, size); + if (!frame) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + return false; + } + + frame->function = module_inst->e->functions + func_index; +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = os_time_thread_cputime_us(); +#endif + frame->prev_frame = wasm_exec_env_get_cur_frame(exec_env); + + /* No need to initialize ip, it will be committed in jitted code + when needed */ + /* frame->ip = NULL; */ + +#if WASM_ENABLE_GC != 0 + frame->sp = frame->lp + max_local_cell_num; + + /* Initialize frame ref flags for import function */ + if (func_index < module->import_function_count) { + WASMFunctionImport *func = + &((module->import_functions + func_index)->u.function); + WASMFuncType *func_type = func->func_type; + /* native function doesn't have operand stack and label stack */ + uint8 *frame_ref = (uint8 *)frame->sp; + uint32 i, j, k, value_type_cell_num; + + for (i = 0, j = 0; i < func_type->param_count; i++) { + if (wasm_is_type_reftype(func_type->types[i]) + && !wasm_is_reftype_i31ref(func_type->types[i])) { + frame_ref[j++] = 1; +#if UINTPTR_MAX == UINT64_MAX + frame_ref[j++] = 1; +#endif + } + else { + value_type_cell_num = + wasm_value_type_cell_num(func_type->types[i]); + for (k = 0; k < value_type_cell_num; k++) + frame_ref[j++] = 0; + } + } + } +#endif + + wasm_exec_env_set_cur_frame(exec_env, frame); + + return true; +} + +static inline void +llvm_jit_free_frame_internal(WASMExecEnv *exec_env) +{ + WASMInterpFrame *frame; + WASMInterpFrame *prev_frame; + + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); + + frame = wasm_exec_env_get_cur_frame(exec_env); + prev_frame = frame->prev_frame; + +#if WASM_ENABLE_PERF_PROFILING != 0 + if (frame->function) { + uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started; + + frame->function->total_exec_time += time_elapsed; + frame->function->total_exec_cnt++; + + /* parent function */ + if (prev_frame) + prev_frame->function->children_exec_time += time_elapsed; + } +#endif + wasm_exec_env_free_wasm_frame(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); +} + +void +llvm_jit_free_frame(WASMExecEnv *exec_env) +{ + llvm_jit_free_frame_internal(exec_env); +} +#endif /* end of WASM_ENABLE_GC == 0 */ + +void +llvm_jit_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + WASMInterpFrame *cur_frame = exec_env->cur_frame; + + if (alloc_frame) { + cur_frame->time_started = os_time_thread_cputime_us(); + } + else { + if (cur_frame->function) { + WASMInterpFrame *prev_frame = cur_frame->prev_frame; + uint64 time_elapsed = + os_time_thread_cputime_us() - cur_frame->time_started; + + cur_frame->function->total_exec_time += time_elapsed; + cur_frame->function->total_exec_cnt++; + + /* parent function */ + if (prev_frame) + prev_frame->function->children_exec_time += time_elapsed; + } + } +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + if (alloc_frame) { +#if WASM_ENABLE_GC == 0 + uint32 wasm_stack_used = (uint8 *)exec_env->cur_frame + + (uint32)offsetof(WASMInterpFrame, lp) + - exec_env->wasm_stack.bottom; +#else + uint32 wasm_stack_used = + exec_env->wasm_stack.top - exec_env->wasm_stack.bottom; +#endif + if (wasm_stack_used > exec_env->max_wasm_stack_used) + exec_env->max_wasm_stack_used = wasm_stack_used; + } +#endif +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 \ + || WASM_ENABLE_PERF_PROFILING != 0 \ + || WASM_ENABLE_AOT_STACK_FRAME != 0 */ + +static bool +llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *function, uint32 argc, + uint32 argv[]) +{ + WASMFuncType *func_type = function->u.func->func_type; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + uint32 func_idx = (uint32)(function - module_inst->e->functions); + bool ret = false; + +#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) \ + || (WASM_ENABLE_AOT_STACK_FRAME != 0) + if (!llvm_jit_alloc_frame(exec_env, function - module_inst->e->functions)) { + /* wasm operand stack overflow has been thrown, + no need to throw again */ + return false; + } +#endif + + if (ext_ret_count > 0) { + uint32 cell_num = 0, i; + uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; + uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL; + uint32 *argv_ret = argv; + uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); + uint64 size; + + /* Allocate memory all arguments */ + size = + sizeof(uint32) * (uint64)argc /* original arguments */ + + sizeof(void *) + * (uint64)ext_ret_count /* extra result values' addr */ + + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */ + if (size > sizeof(argv1_buf)) { + if (size > UINT32_MAX + || !(argv1 = wasm_runtime_malloc((uint32)size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + ret = false; + goto fail; + } + } + + /* Copy original arguments */ + bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc); + + /* Get the extra result value's address */ + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + + /* Append each extra result value's address to original arguments */ + for (i = 0; i < ext_ret_count; i++) { + *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) = + (uintptr_t)(ext_rets + cell_num); + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + + ret = wasm_runtime_invoke_native( + exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, NULL, + argv1, argc, argv); + if (!ret) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + goto fail; + } + + /* Get extra result values */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + argv_ret++; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + argv_ret += 2; + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + argv_ret += 4; + break; +#endif + default: + bh_assert(0); + break; + } + + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets, + sizeof(uint32) * cell_num); + + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + ret = true; + } + else { +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick call if the quick jit entry is registered */ + if (func_type->quick_aot_entry) { + void (*invoke_native)(void *func_ptr, void *exec_env, uint32 *argv, + uint32 *argv_ret) = + func_type->quick_aot_entry; + invoke_native(module_inst->func_ptrs[func_idx], exec_env, argv, + argv); + ret = !wasm_copy_exception(module_inst, NULL); + } + else +#endif + { + ret = wasm_runtime_invoke_native( + exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, + NULL, argv, argc, argv); + + if (ret) + ret = !wasm_copy_exception(module_inst, NULL); + } + } + +fail: + +#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) \ + || (WASM_ENABLE_AOT_STACK_FRAME != 0) + llvm_jit_free_frame_internal(exec_env); +#endif + + return ret; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +void +wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, + WASMFunctionInstance *function, uint32 argc, + uint32 argv[]) +{ + WASMRuntimeFrame *frame = NULL, *prev_frame, *outs_area; + RunningMode running_mode = + wasm_runtime_get_running_mode((WASMModuleInstanceCommon *)module_inst); + /* Allocate sufficient cells for all kinds of return values. */ + bool alloc_frame = true; + + if (argc < function->param_cell_num) { + char buf[128]; + snprintf(buf, sizeof(buf), + "invalid argument count %" PRIu32 + ", must be no smaller than %u", + argc, function->param_cell_num); + wasm_set_exception(module_inst, buf); + return; + } + argc = function->param_cell_num; + + RECORD_STACK_USAGE(exec_env, (uint8 *)&prev_frame); +#if !(defined(OS_ENABLE_HW_BOUND_CHECK) \ + && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0) + if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) { + wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, + "native stack overflow"); + return; + } +#endif + + if (!function->is_import_func) { + /* No need to alloc frame when calling LLVM JIT function */ +#if WASM_ENABLE_JIT != 0 + if (running_mode == Mode_LLVM_JIT) { + alloc_frame = false; + } +#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 + else if (running_mode == Mode_Multi_Tier_JIT) { + /* Tier-up from Fast JIT to LLVM JIT, call llvm jit function + if it is compiled, else call fast jit function */ + uint32 func_idx = (uint32)(function - module_inst->e->functions); + if (module_inst->module->func_ptrs_compiled + [func_idx - module_inst->module->import_function_count]) { + alloc_frame = false; + } + } +#endif +#endif + } + + if (alloc_frame) { + unsigned all_cell_num = + function->ret_cell_num > 2 ? function->ret_cell_num : 2; + unsigned frame_size; + + prev_frame = wasm_exec_env_get_cur_frame(exec_env); + /* This frame won't be used by JITed code, so only allocate interp + frame here. */ + frame_size = wasm_interp_interp_frame_size(all_cell_num); + + if (!(frame = ALLOC_FRAME(exec_env, frame_size, prev_frame))) + return; + + outs_area = wasm_exec_env_wasm_stack_top(exec_env); + frame->function = NULL; + frame->ip = NULL; + /* There is no local variable. */ + frame->sp = frame->lp + 0; + + if ((uint8 *)(outs_area->lp + function->param_cell_num) + > exec_env->wasm_stack.top_boundary) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + return; + } + + if (argc > 0) + word_copy(outs_area->lp, argv, argc); + + wasm_exec_env_set_cur_frame(exec_env, frame); + } + +#if defined(os_writegsbase) + { + WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_inst->memory_data); + } +#endif + + if (function->is_import_func) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (function->import_module_inst) { + wasm_interp_call_func_import(module_inst, exec_env, function, + frame); + } + else +#endif + { + /* it is a native function */ + wasm_interp_call_func_native(module_inst, exec_env, function, + frame); + } + } + else { + if (running_mode == Mode_Interp) { + wasm_interp_call_func_bytecode(module_inst, exec_env, function, + frame); + } +#if WASM_ENABLE_FAST_JIT != 0 + else if (running_mode == Mode_Fast_JIT) { + fast_jit_call_func_bytecode(module_inst, exec_env, function, frame); + } +#endif +#if WASM_ENABLE_JIT != 0 + else if (running_mode == Mode_LLVM_JIT) { + llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc, + argv); + } +#endif +#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 \ + && WASM_ENABLE_JIT != 0 + else if (running_mode == Mode_Multi_Tier_JIT) { + /* Tier-up from Fast JIT to LLVM JIT, call llvm jit function + if it is compiled, else call fast jit function */ + uint32 func_idx = (uint32)(function - module_inst->e->functions); + if (module_inst->module->func_ptrs_compiled + [func_idx - module_inst->module->import_function_count]) { + llvm_jit_call_func_bytecode(module_inst, exec_env, function, + argc, argv); + } + else { + fast_jit_call_func_bytecode(module_inst, exec_env, function, + frame); + } + } +#endif + else { + /* There should always be a supported running mode selected */ + bh_assert(0); + } + + (void)wasm_interp_call_func_bytecode; +#if WASM_ENABLE_FAST_JIT != 0 + (void)fast_jit_call_func_bytecode; +#endif + } + + /* Output the return value to the caller */ + if (!wasm_copy_exception(module_inst, NULL)) { + if (alloc_frame) { + uint32 i; + for (i = 0; i < function->ret_cell_num; i++) { + argv[i] = *(frame->sp + i - function->ret_cell_num); + } + } + } + else { +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (wasm_interp_create_call_stack(exec_env)) { + wasm_interp_dump_call_stack(exec_env, true, NULL, 0); + } +#endif + } + + if (alloc_frame) { + wasm_exec_env_set_cur_frame(exec_env, prev_frame); + FREE_FRAME(exec_env, frame); + } +} diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp_fast.c b/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp_fast.c new file mode 100644 index 0000000..417e3b0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp_fast.c @@ -0,0 +1,6168 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_interp.h" +#include "bh_log.h" +#include "wasm_runtime.h" +#include "wasm_opcode.h" +#include "wasm_loader.h" +#include "wasm_memory.h" +#include "../common/wasm_exec_env.h" +#if WASM_ENABLE_GC != 0 +#include "../common/gc/gc_object.h" +#include "mem_alloc.h" +#if WASM_ENABLE_STRINGREF != 0 +#include "string_object.h" +#endif +#endif +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif + +typedef int32 CellType_I32; +typedef int64 CellType_I64; +typedef float32 CellType_F32; +typedef float64 CellType_F64; + +#if WASM_ENABLE_THREAD_MGR == 0 +#define get_linear_mem_size() linear_mem_size +#else +/** + * Load memory data size in each time boundary check in + * multi-threading mode since it may be changed by other + * threads in memory.grow + */ +#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) +#endif + +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ + /* If offset1 is in valid range, maddr must also \ + be in valid range, no need to check it again. */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ + /* App heap space is not valid space for \ + bulk memory operation */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) +#else +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + maddr = memory->memory_data + offset1; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + maddr = memory->memory_data + (uint32)(start); \ + } while (0) +#endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ + +#define CHECK_ATOMIC_MEMORY_ACCESS(align) \ + do { \ + if (((uintptr_t)maddr & (align - 1)) != 0) \ + goto unaligned_atomic; \ + } while (0) + +static inline uint32 +rotl32(uint32 n, uint32 c) +{ + const uint32 mask = (31); + c = c % 32; + c &= mask; + return (n << c) | (n >> ((0 - c) & mask)); +} + +static inline uint32 +rotr32(uint32 n, uint32 c) +{ + const uint32 mask = (31); + c = c % 32; + c &= mask; + return (n >> c) | (n << ((0 - c) & mask)); +} + +static inline uint64 +rotl64(uint64 n, uint64 c) +{ + const uint64 mask = (63); + c = c % 64; + c &= mask; + return (n << c) | (n >> ((0 - c) & mask)); +} + +static inline uint64 +rotr64(uint64 n, uint64 c) +{ + const uint64 mask = (63); + c = c % 64; + c &= mask; + return (n >> c) | (n << ((0 - c) & mask)); +} + +static inline float32 +f32_min(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static inline float32 +f32_max(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static inline float64 +f64_min(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static inline float64 +f64_max(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static inline uint32 +clz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 0x80000000)) { + num++; + type <<= 1; + } + return num; +} + +static inline uint32 +clz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 0x8000000000000000LL)) { + num++; + type <<= 1; + } + return num; +} + +static inline uint32 +ctz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static inline uint32 +ctz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static inline uint32 +popcount32(uint32 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static inline uint32 +popcount64(uint64 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static float +local_copysignf(float x, float y) +{ + union { + float f; + uint32 i; + } ux = { x }, uy = { y }; + ux.i &= 0x7fffffff; + ux.i |= uy.i & 0x80000000; + return ux.f; +} + +static double +local_copysign(double x, double y) +{ + union { + double f; + uint64 i; + } ux = { x }, uy = { y }; + ux.i &= UINT64_MAX / 2; + ux.i |= uy.i & 1ULL << 63; + return ux.f; +} + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define LOAD_U32_WITH_2U16S(addr) (*(uint32 *)(addr)) +#define LOAD_PTR(addr) (*(void **)(addr)) +#else /* else of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +static inline uint32 +LOAD_U32_WITH_2U16S(void *addr) +{ + union { + uint32 val; + uint16 u16[2]; + } u; + + bh_assert(((uintptr_t)addr & 1) == 0); + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + return u.val; +} +#if UINTPTR_MAX == UINT32_MAX +#define LOAD_PTR(addr) ((void *)LOAD_U32_WITH_2U16S(addr)) +#elif UINTPTR_MAX == UINT64_MAX +static inline void * +LOAD_PTR(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + void *val; + uint32 u32[2]; + uint16 u16[4]; + } u; + + bh_assert(((uintptr_t)addr & 1) == 0); + if ((addr1 & (uintptr_t)7) == 0) + return *(void **)addr; + + if ((addr1 & (uintptr_t)3) == 0) { + u.u32[0] = ((uint32 *)addr)[0]; + u.u32[1] = ((uint32 *)addr)[1]; + } + else { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + u.u16[2] = ((uint16 *)addr)[2]; + u.u16[3] = ((uint16 *)addr)[3]; + } + return u.val; +} +#endif /* end of UINTPTR_MAX */ +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ + +#if WASM_ENABLE_GC != 0 +static void +init_frame_refs(uint8 *frame_ref, uint32 cell_num, WASMFunctionInstance *func) +{ + uint32 i, j; + + memset(frame_ref, 0, cell_num); + + for (i = 0, j = 0; i < func->param_count; i++) { + if (wasm_is_type_reftype(func->param_types[i]) + && !wasm_is_reftype_i31ref(func->param_types[i])) { + frame_ref[j++] = 1; +#if UINTPTR_MAX == UINT64_MAX + frame_ref[j++] = 1; +#endif + } + else { + j += wasm_value_type_cell_num(func->param_types[i]); + } + } + + for (i = 0; i < func->local_count; i++) { + if (wasm_is_type_reftype(func->local_types[i]) + && !wasm_is_reftype_i31ref(func->local_types[i])) { + frame_ref[j++] = 1; +#if UINTPTR_MAX == UINT64_MAX + frame_ref[j++] = 1; +#endif + } + else { + j += wasm_value_type_cell_num(func->local_types[i]); + } + } +} + +uint8 * +wasm_interp_get_frame_ref(WASMInterpFrame *frame) +{ + return frame->frame_ref; +} + +/* Return the corresponding ref slot of the given slot of local + variable or stack pointer. */ + +#define COMPUTE_FRAME_REF(ref, off) (ref + (unsigned)(off)) + +#define FRAME_REF(off) COMPUTE_FRAME_REF(frame_ref, off) + +#if UINTPTR_MAX == UINT64_MAX +#define SET_FRAME_REF(off) *FRAME_REF(off) = *FRAME_REF(off + 1) = 1 +#define CLEAR_FRAME_REF(off) \ + (unsigned)off >= local_cell_num \ + ? (*FRAME_REF(off) = *FRAME_REF(off + 1) = 0) \ + : (void)0 +#else +#define SET_FRAME_REF(off) *FRAME_REF(off) = 1 +#define CLEAR_FRAME_REF(off) \ + (unsigned)off >= local_cell_num ? (*FRAME_REF(off) = 0) : (void)0 +#endif + +#define FRAME_REF_FOR(frame, p) \ + COMPUTE_FRAME_REF(frame->frame_ref, p - frame->lp) + +#define CLEAR_FRAME_REF_FOR(p, n) \ + do { \ + int32 ref_i, ref_n = (int32)(n); \ + uint8 *ref = FRAME_REF(p - frame_lp); \ + for (ref_i = 0; ref_i < ref_n; ref_i++) \ + ref[ref_i] = 0; \ + } while (0) +#endif /* end of WASM_ENABLE_GC != 0 */ + +#define read_uint32(p) \ + (p += sizeof(uint32), LOAD_U32_WITH_2U16S(p - sizeof(uint32))) + +#define GET_LOCAL_INDEX_TYPE_AND_OFFSET() \ + do { \ + uint32 param_count = cur_func->param_count; \ + local_idx = read_uint32(frame_ip); \ + bh_assert(local_idx < param_count + cur_func->local_count); \ + local_offset = cur_func->local_offsets[local_idx]; \ + if (local_idx < param_count) \ + local_type = cur_func->param_types[local_idx]; \ + else \ + local_type = cur_func->local_types[local_idx - param_count]; \ + } while (0) + +#define GET_OFFSET() (frame_ip += 2, *(int16 *)(frame_ip - 2)) + +#define SET_OPERAND_I32(off, value) \ + do { \ + *(uint32 *)(frame_lp + *(int16 *)(frame_ip + off)) = value; \ + } while (0) +#define SET_OPERAND_F32(off, value) \ + do { \ + *(float32 *)(frame_lp + *(int16 *)(frame_ip + off)) = value; \ + } while (0) +#define SET_OPERAND_I64(off, value) \ + do { \ + uint32 *addr_tmp = frame_lp + *(int16 *)(frame_ip + off); \ + PUT_I64_TO_ADDR(addr_tmp, value); \ + } while (0) +#define SET_OPERAND_F64(off, value) \ + do { \ + uint32 *addr_tmp = frame_lp + *(int16 *)(frame_ip + off); \ + PUT_F64_TO_ADDR(addr_tmp, value); \ + } while (0) +#define SET_OPERAND_REF(off, value) \ + do { \ + uint32 *addr_tmp; \ + opnd_off = *(int16 *)(frame_ip + off); \ + addr_tmp = frame_lp + opnd_off; \ + PUT_REF_TO_ADDR(addr_tmp, value); \ + SET_FRAME_REF(ond_off); \ + } while (0) + +#define SET_OPERAND(op_type, off, value) SET_OPERAND_##op_type(off, value) + +#define GET_OPERAND_I32(type, off) \ + *(type *)(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_F32(type, off) \ + *(type *)(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_I64(type, off) \ + (type) GET_I64_FROM_ADDR(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_F64(type, off) \ + (type) GET_F64_FROM_ADDR(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_REF(type, off) \ + (type) GET_REF_FROM_ADDR(frame_lp + *(int16 *)(frame_ip + off)) + +#define GET_OPERAND(type, op_type, off) GET_OPERAND_##op_type(type, off) + +#define PUSH_I32(value) \ + do { \ + *(int32 *)(frame_lp + GET_OFFSET()) = value; \ + } while (0) + +#define PUSH_F32(value) \ + do { \ + *(float32 *)(frame_lp + GET_OFFSET()) = value; \ + } while (0) + +#define PUSH_I64(value) \ + do { \ + uint32 *addr_tmp = frame_lp + GET_OFFSET(); \ + PUT_I64_TO_ADDR(addr_tmp, value); \ + } while (0) + +#define PUSH_F64(value) \ + do { \ + uint32 *addr_tmp = frame_lp + GET_OFFSET(); \ + PUT_F64_TO_ADDR(addr_tmp, value); \ + } while (0) + +#define PUSH_REF(value) \ + do { \ + uint32 *addr_tmp; \ + opnd_off = GET_OFFSET(); \ + addr_tmp = frame_lp + opnd_off; \ + PUT_REF_TO_ADDR(addr_tmp, value); \ + SET_FRAME_REF(opnd_off); \ + } while (0) + +#define PUSH_I31REF(value) \ + do { \ + uint32 *addr_tmp; \ + opnd_off = GET_OFFSET(); \ + addr_tmp = frame_lp + opnd_off; \ + PUT_REF_TO_ADDR(addr_tmp, value); \ + } while (0) + +#define POP_I32() (*(int32 *)(frame_lp + GET_OFFSET())) + +#define POP_F32() (*(float32 *)(frame_lp + GET_OFFSET())) + +#define POP_I64() (GET_I64_FROM_ADDR(frame_lp + GET_OFFSET())) + +#define POP_F64() (GET_F64_FROM_ADDR(frame_lp + GET_OFFSET())) + +#define POP_REF() \ + (opnd_off = GET_OFFSET(), CLEAR_FRAME_REF((unsigned)(opnd_off)), \ + GET_REF_FROM_ADDR(frame_lp + opnd_off)) + +#if WASM_ENABLE_GC != 0 +#define SYNC_FRAME_REF() frame->frame_ref = frame_ref +#define UPDATE_FRAME_REF() frame_ref = frame->frame_ref +#else +#define SYNC_FRAME_REF() (void)0 +#define UPDATE_FRAME_REF() (void)0 +#endif + +#define SYNC_ALL_TO_FRAME() \ + do { \ + frame->ip = frame_ip; \ + SYNC_FRAME_REF(); \ + } while (0) + +#define UPDATE_ALL_FROM_FRAME() \ + do { \ + frame_ip = frame->ip; \ + UPDATE_FRAME_REF(); \ + } while (0) + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#define UPDATE_FRAME_IP_END() (void)0 +#else +#define UPDATE_FRAME_IP_END() frame_ip_end = wasm_get_func_code_end(cur_func) +#endif + +#if WASM_ENABLE_GC != 0 +#define RECOVER_FRAME_REF() frame_ref = frame->frame_ref +#else +#define RECOVER_FRAME_REF() (void)0 +#endif + +#define RECOVER_CONTEXT(new_frame) \ + do { \ + frame = (new_frame); \ + cur_func = frame->function; \ + prev_frame = frame->prev_frame; \ + frame_ip = frame->ip; \ + UPDATE_FRAME_IP_END(); \ + frame_lp = frame->lp; \ + RECOVER_FRAME_REF(); \ + } while (0) + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define GET_OPCODE() opcode = *frame_ip++; +#else +#define GET_OPCODE() \ + opcode = *frame_ip; \ + frame_ip += 2; +#endif + +#define DEF_OP_EQZ(ctype, src_op_type) \ + do { \ + SET_OPERAND(I32, 2, (GET_OPERAND(ctype, src_op_type, 0) == 0)); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_CMP(src_type, src_op_type, cond) \ + do { \ + SET_OPERAND(I32, 4, \ + GET_OPERAND(src_type, src_op_type, 2) \ + cond GET_OPERAND(src_type, src_op_type, 0)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_OP_BIT_COUNT(src_type, src_op_type, operation) \ + do { \ + SET_OPERAND( \ + src_op_type, 2, \ + (src_type)operation(GET_OPERAND(src_type, src_op_type, 0))); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_NUMERIC(src_type1, src_type2, src_op_type, operation) \ + do { \ + SET_OPERAND(src_op_type, 4, \ + GET_OPERAND(src_type1, src_op_type, 2) \ + operation GET_OPERAND(src_type2, src_op_type, 0)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_OP_REINTERPRET(src_type, src_op_type) \ + do { \ + SET_OPERAND(src_op_type, 2, GET_OPERAND(src_type, src_op_type, 0)); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_NUMERIC_64 DEF_OP_NUMERIC + +#define DEF_OP_NUMERIC2(src_type1, src_type2, src_op_type, operation) \ + do { \ + SET_OPERAND(src_op_type, 4, \ + GET_OPERAND(src_type1, src_op_type, 2) operation( \ + GET_OPERAND(src_type2, src_op_type, 0) % 32)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_OP_NUMERIC2_64(src_type1, src_type2, src_op_type, operation) \ + do { \ + SET_OPERAND(src_op_type, 4, \ + GET_OPERAND(src_type1, src_op_type, 2) operation( \ + GET_OPERAND(src_type2, src_op_type, 0) % 64)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_ATOMIC_RMW_OPCODE(OP_NAME, op) \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U: \ + { \ + uint32 readv, sval; \ + \ + sval = POP_I32(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \ + CHECK_MEMORY_OVERFLOW(1); \ + CHECK_ATOMIC_MEMORY_ACCESS(1); \ + \ + shared_memory_lock(memory); \ + readv = (uint32)(*(uint8 *)maddr); \ + *(uint8 *)maddr = (uint8)(readv op sval); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ + CHECK_MEMORY_OVERFLOW(2); \ + CHECK_ATOMIC_MEMORY_ACCESS(2); \ + \ + shared_memory_lock(memory); \ + readv = (uint32)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else { \ + CHECK_MEMORY_OVERFLOW(4); \ + CHECK_ATOMIC_MEMORY_ACCESS(4); \ + \ + shared_memory_lock(memory); \ + readv = LOAD_I32(maddr); \ + STORE_U32(maddr, readv op sval); \ + shared_memory_unlock(memory); \ + } \ + PUSH_I32(readv); \ + break; \ + } \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U: \ + { \ + uint64 readv, sval; \ + \ + sval = (uint64)POP_I64(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \ + CHECK_MEMORY_OVERFLOW(1); \ + CHECK_ATOMIC_MEMORY_ACCESS(1); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)(*(uint8 *)maddr); \ + *(uint8 *)maddr = (uint8)(readv op sval); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ + CHECK_MEMORY_OVERFLOW(2); \ + CHECK_ATOMIC_MEMORY_ACCESS(2); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ + CHECK_MEMORY_OVERFLOW(4); \ + CHECK_ATOMIC_MEMORY_ACCESS(4); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_U32(maddr); \ + STORE_U32(maddr, (uint32)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else { \ + uint64 op_result; \ + CHECK_MEMORY_OVERFLOW(8); \ + CHECK_ATOMIC_MEMORY_ACCESS(8); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_I64(maddr); \ + op_result = readv op sval; \ + STORE_I64(maddr, op_result); \ + shared_memory_unlock(memory); \ + } \ + PUSH_I64(readv); \ + break; \ + } + +#define DEF_OP_MATH(src_type, src_op_type, method) \ + do { \ + SET_OPERAND(src_op_type, 2, \ + (src_type)method(GET_OPERAND(src_type, src_op_type, 0))); \ + frame_ip += 4; \ + } while (0) + +#define TRUNC_FUNCTION(func_name, src_type, dst_type, signed_type) \ + static dst_type func_name(src_type src_value, src_type src_min, \ + src_type src_max, dst_type dst_min, \ + dst_type dst_max, bool is_sign) \ + { \ + dst_type dst_value = 0; \ + if (!isnan(src_value)) { \ + if (src_value <= src_min) \ + dst_value = dst_min; \ + else if (src_value >= src_max) \ + dst_value = dst_max; \ + else { \ + if (is_sign) \ + dst_value = (dst_type)(signed_type)src_value; \ + else \ + dst_value = (dst_type)src_value; \ + } \ + } \ + return dst_value; \ + } + +TRUNC_FUNCTION(trunc_f32_to_i32, float32, uint32, int32) +TRUNC_FUNCTION(trunc_f32_to_i64, float32, uint64, int64) +TRUNC_FUNCTION(trunc_f64_to_i32, float64, uint32, int32) +TRUNC_FUNCTION(trunc_f64_to_i64, float64, uint64, int64) + +static bool +trunc_f32_to_int(WASMModuleInstance *module, uint8 *frame_ip, uint32 *frame_lp, + float32 src_min, float32 src_max, bool saturating, bool is_i32, + bool is_sign) +{ + float32 src_value = GET_OPERAND(float32, F32, 0); + uint64 dst_value_i64; + uint32 dst_value_i32; + + if (!saturating) { + if (isnan(src_value)) { + wasm_set_exception(module, "invalid conversion to integer"); + return false; + } + else if (src_value <= src_min || src_value >= src_max) { + wasm_set_exception(module, "integer overflow"); + return false; + } + } + + if (is_i32) { + uint32 dst_min = is_sign ? INT32_MIN : 0; + uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX; + dst_value_i32 = trunc_f32_to_i32(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I32, 2, dst_value_i32); + } + else { + uint64 dst_min = is_sign ? INT64_MIN : 0; + uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX; + dst_value_i64 = trunc_f32_to_i64(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I64, 2, dst_value_i64); + } + return true; +} + +static bool +trunc_f64_to_int(WASMModuleInstance *module, uint8 *frame_ip, uint32 *frame_lp, + float64 src_min, float64 src_max, bool saturating, bool is_i32, + bool is_sign) +{ + float64 src_value = GET_OPERAND(float64, F64, 0); + uint64 dst_value_i64; + uint32 dst_value_i32; + + if (!saturating) { + if (isnan(src_value)) { + wasm_set_exception(module, "invalid conversion to integer"); + return false; + } + else if (src_value <= src_min || src_value >= src_max) { + wasm_set_exception(module, "integer overflow"); + return false; + } + } + + if (is_i32) { + uint32 dst_min = is_sign ? INT32_MIN : 0; + uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX; + dst_value_i32 = trunc_f64_to_i32(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I32, 2, dst_value_i32); + } + else { + uint64 dst_min = is_sign ? INT64_MIN : 0; + uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX; + dst_value_i64 = trunc_f64_to_i64(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I64, 2, dst_value_i64); + } + return true; +} + +#define DEF_OP_TRUNC_F32(min, max, is_i32, is_sign) \ + do { \ + if (!trunc_f32_to_int(module, frame_ip, frame_lp, min, max, false, \ + is_i32, is_sign)) \ + goto got_exception; \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_TRUNC_F64(min, max, is_i32, is_sign) \ + do { \ + if (!trunc_f64_to_int(module, frame_ip, frame_lp, min, max, false, \ + is_i32, is_sign)) \ + goto got_exception; \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_TRUNC_SAT_F32(min, max, is_i32, is_sign) \ + do { \ + (void)trunc_f32_to_int(module, frame_ip, frame_lp, min, max, true, \ + is_i32, is_sign); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_TRUNC_SAT_F64(min, max, is_i32, is_sign) \ + do { \ + (void)trunc_f64_to_int(module, frame_ip, frame_lp, min, max, true, \ + is_i32, is_sign); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_CONVERT(dst_type, dst_op_type, src_type, src_op_type) \ + do { \ + dst_type value = (dst_type)(src_type)POP_##src_op_type(); \ + PUSH_##dst_op_type(value); \ + } while (0) + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define CELL_SIZE sizeof(uint8) +#else +#define CELL_SIZE (sizeof(uint8) * 2) +#endif + +static bool +copy_stack_values(WASMModuleInstance *module, uint32 *frame_lp, uint32 arity, +#if WASM_ENABLE_GC != 0 + uint8 *frame_ref, +#endif + uint32 total_cell_num, const uint8 *cells, + const int16 *src_offsets, const uint16 *dst_offsets) +{ + /* To avoid the overlap issue between src offsets and dst offset, + * we use 2 steps to do the copy. First step, copy the src values + * to a tmp buf. Second step, copy the values from tmp buf to dst. + */ + bool ret = false; + uint32 buf[16] = { 0 }, i; + uint32 *tmp_buf = buf; + uint8 cell; + int16 src, buf_index = 0; + uint16 dst; +#if WASM_ENABLE_GC != 0 + uint8 ref_buf[4]; + uint8 *tmp_ref_buf = ref_buf; +#endif + + /* Allocate memory if the buf is not large enough */ + if (total_cell_num > sizeof(buf) / sizeof(uint32)) { + uint64 total_size = sizeof(uint32) * (uint64)total_cell_num; + if (total_size >= UINT32_MAX + || !(tmp_buf = wasm_runtime_malloc((uint32)total_size))) { + wasm_set_exception(module, "allocate memory failed"); + goto fail; + } + } + +#if WASM_ENABLE_GC != 0 + if (total_cell_num > sizeof(ref_buf) / sizeof(uint8)) { + uint64 total_size = sizeof(uint8) * (uint64)total_cell_num; + if (total_size >= UINT32_MAX + || !(tmp_ref_buf = wasm_runtime_malloc((uint32)total_size))) { + wasm_set_exception(module, "allocate memory failed"); + goto fail; + } + } +#endif + + /* 1) Copy values from src to tmp buf */ + for (i = 0; i < arity; i++) { + cell = cells[i * CELL_SIZE]; + src = src_offsets[i]; + if (cell == 1) { + tmp_buf[buf_index] = frame_lp[src]; +#if WASM_ENABLE_GC != 0 + tmp_ref_buf[buf_index] = frame_ref[src]; + frame_ref[src] = 0; +#endif + } + else { + tmp_buf[buf_index] = frame_lp[src]; + tmp_buf[buf_index + 1] = frame_lp[src + 1]; +#if WASM_ENABLE_GC != 0 + tmp_ref_buf[buf_index] = frame_ref[src]; + tmp_ref_buf[buf_index + 1] = frame_ref[src + 1]; + frame_ref[src] = 0; + frame_ref[src + 1] = 0; +#endif + } + buf_index += cell; + } + + /* 2) Copy values from tmp buf to dest */ + buf_index = 0; + for (i = 0; i < arity; i++) { + cell = cells[i * CELL_SIZE]; + dst = dst_offsets[i]; + if (cell == 1) { + frame_lp[dst] = tmp_buf[buf_index]; +#if WASM_ENABLE_GC != 0 + frame_ref[dst] = tmp_ref_buf[buf_index]; +#endif + } + else { + frame_lp[dst] = tmp_buf[buf_index]; + frame_lp[dst + 1] = tmp_buf[buf_index + 1]; +#if WASM_ENABLE_GC != 0 + frame_ref[dst] = tmp_ref_buf[buf_index]; + frame_ref[dst + 1] = tmp_ref_buf[buf_index + 1]; +#endif + } + buf_index += cell; + } + + ret = true; + +fail: + if (tmp_buf != buf) { + wasm_runtime_free(tmp_buf); + } + +#if WASM_ENABLE_GC != 0 + if (tmp_ref_buf != ref_buf) { + wasm_runtime_free(tmp_ref_buf); + } +#endif + + return ret; +} + +#if WASM_ENABLE_GC != 0 +#define RECOVER_BR_INFO() \ + do { \ + uint32 arity; \ + /* read arity */ \ + arity = read_uint32(frame_ip); \ + if (arity) { \ + uint32 total_cell; \ + uint16 *dst_offsets = NULL; \ + uint8 *cells; \ + int16 *src_offsets = NULL; \ + /* read total cell num */ \ + total_cell = read_uint32(frame_ip); \ + /* cells */ \ + cells = (uint8 *)frame_ip; \ + frame_ip += arity * CELL_SIZE; \ + /* src offsets */ \ + src_offsets = (int16 *)frame_ip; \ + frame_ip += arity * sizeof(int16); \ + /* dst offsets */ \ + dst_offsets = (uint16 *)frame_ip; \ + frame_ip += arity * sizeof(uint16); \ + if (arity == 1) { \ + if (cells[0] == 1) { \ + frame_lp[dst_offsets[0]] = frame_lp[src_offsets[0]]; \ + /* Ignore constants because they are not reference */ \ + if (src_offsets[0] >= 0) { \ + CLEAR_FRAME_REF((unsigned)(src_offsets[0])); \ + SET_FRAME_REF(dst_offsets[0]); \ + } \ + } \ + else if (cells[0] == 2) { \ + frame_lp[dst_offsets[0]] = frame_lp[src_offsets[0]]; \ + frame_lp[dst_offsets[0] + 1] = \ + frame_lp[src_offsets[0] + 1]; \ + /* Ignore constants because they are not reference */ \ + if (src_offsets[0] >= 0) { \ + CLEAR_FRAME_REF((unsigned)src_offsets[0]); \ + CLEAR_FRAME_REF((unsigned)(src_offsets[0] + 1)); \ + SET_FRAME_REF((unsigned)dst_offsets[0]); \ + SET_FRAME_REF((unsigned)(dst_offsets[0] + 1)); \ + } \ + } \ + } \ + else { \ + if (!copy_stack_values(module, frame_lp, arity, frame_ref, \ + total_cell, cells, src_offsets, \ + dst_offsets)) \ + goto got_exception; \ + } \ + } \ + frame_ip = (uint8 *)LOAD_PTR(frame_ip); \ + } while (0) +#else +#define RECOVER_BR_INFO() \ + do { \ + uint32 arity; \ + /* read arity */ \ + arity = read_uint32(frame_ip); \ + if (arity) { \ + uint32 total_cell; \ + uint16 *dst_offsets = NULL; \ + uint8 *cells; \ + int16 *src_offsets = NULL; \ + /* read total cell num */ \ + total_cell = read_uint32(frame_ip); \ + /* cells */ \ + cells = (uint8 *)frame_ip; \ + frame_ip += arity * CELL_SIZE; \ + /* src offsets */ \ + src_offsets = (int16 *)frame_ip; \ + frame_ip += arity * sizeof(int16); \ + /* dst offsets */ \ + dst_offsets = (uint16 *)frame_ip; \ + frame_ip += arity * sizeof(uint16); \ + if (arity == 1) { \ + if (cells[0] == 1) \ + frame_lp[dst_offsets[0]] = frame_lp[src_offsets[0]]; \ + else if (cells[0] == 2) { \ + frame_lp[dst_offsets[0]] = frame_lp[src_offsets[0]]; \ + frame_lp[dst_offsets[0] + 1] = \ + frame_lp[src_offsets[0] + 1]; \ + } \ + } \ + else { \ + if (!copy_stack_values(module, frame_lp, arity, total_cell, \ + cells, src_offsets, dst_offsets)) \ + goto got_exception; \ + } \ + } \ + frame_ip = (uint8 *)LOAD_PTR(frame_ip); \ + } while (0) +#endif + +#define SKIP_BR_INFO() \ + do { \ + uint32 arity; \ + /* read and skip arity */ \ + arity = read_uint32(frame_ip); \ + if (arity) { \ + /* skip total cell num */ \ + frame_ip += sizeof(uint32); \ + /* skip cells, src offsets and dst offsets */ \ + frame_ip += (CELL_SIZE + sizeof(int16) + sizeof(uint16)) * arity; \ + } \ + /* skip target address */ \ + frame_ip += sizeof(uint8 *); \ + } while (0) + +static inline int32 +sign_ext_8_32(int8 val) +{ + if (val & 0x80) + return (int32)val | (int32)0xffffff00; + return val; +} + +static inline int32 +sign_ext_16_32(int16 val) +{ + if (val & 0x8000) + return (int32)val | (int32)0xffff0000; + return val; +} + +static inline int64 +sign_ext_8_64(int8 val) +{ + if (val & 0x80) + return (int64)val | (int64)0xffffffffffffff00LL; + return val; +} + +static inline int64 +sign_ext_16_64(int16 val) +{ + if (val & 0x8000) + return (int64)val | (int64)0xffffffffffff0000LL; + return val; +} + +static inline int64 +sign_ext_32_64(int32 val) +{ + if (val & (int32)0x80000000) + return (int64)val | (int64)0xffffffff00000000LL; + return val; +} + +static inline void +word_copy(uint32 *dest, uint32 *src, unsigned num) +{ + bh_assert(dest != NULL); + bh_assert(src != NULL); + bh_assert(num > 0); + if (dest != src) { + /* No overlap buffer */ + bh_assert(!((src < dest) && (dest < src + num))); + for (; num > 0; num--) + *dest++ = *src++; + } +} + +static inline WASMInterpFrame * +ALLOC_FRAME(WASMExecEnv *exec_env, uint32 size, WASMInterpFrame *prev_frame) +{ + WASMInterpFrame *frame = wasm_exec_env_alloc_wasm_frame(exec_env, size); + + if (frame) { + frame->prev_frame = prev_frame; +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = os_time_thread_cputime_us(); +#endif + } + else { + wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, + "wasm operand stack overflow"); + } + + return frame; +} + +static inline void +FREE_FRAME(WASMExecEnv *exec_env, WASMInterpFrame *frame) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + if (frame->function) { + WASMInterpFrame *prev_frame = frame->prev_frame; + uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started; + + frame->function->total_exec_time += time_elapsed; + frame->function->total_exec_cnt++; + + /* parent function */ + if (prev_frame && prev_frame->function) + prev_frame->function->children_exec_time += time_elapsed; + } +#endif + wasm_exec_env_free_wasm_frame(exec_env, frame); +} + +static void +wasm_interp_call_func_native(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMFunctionImport *func_import = cur_func->u.func_import; + CApiFuncImport *c_api_func_import = NULL; + unsigned local_cell_num = + cur_func->param_cell_num > 2 ? cur_func->param_cell_num : 2; + unsigned all_cell_num; + WASMInterpFrame *frame; + uint32 argv_ret[2], cur_func_index; + void *native_func_pointer = NULL; + bool ret; +#if WASM_ENABLE_GC != 0 + WASMFuncType *func_type; + uint8 *frame_ref; +#endif + + all_cell_num = local_cell_num; +#if WASM_ENABLE_GC != 0 + all_cell_num += (local_cell_num + 3) / 4; +#endif + + if (!(frame = + ALLOC_FRAME(exec_env, wasm_interp_interp_frame_size(all_cell_num), + prev_frame))) + return; + + frame->function = cur_func; + frame->ip = NULL; + frame->lp = frame->operand; +#if WASM_ENABLE_GC != 0 + frame->frame_ref = (uint8 *)(frame->lp + local_cell_num); + init_frame_refs(frame->frame_ref, local_cell_num, cur_func); +#endif + + wasm_exec_env_set_cur_frame(exec_env, frame); + + cur_func_index = (uint32)(cur_func - module_inst->e->functions); + bh_assert(cur_func_index < module_inst->module->import_function_count); + if (!func_import->call_conv_wasm_c_api) { + native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; + } + else if (module_inst->c_api_func_imports) { + c_api_func_import = module_inst->c_api_func_imports + cur_func_index; + native_func_pointer = c_api_func_import->func_ptr_linked; + } + + if (!native_func_pointer) { + char buf[128]; + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + func_import->module_name, func_import->field_name); + wasm_set_exception((WASMModuleInstance *)module_inst, buf); + return; + } + + if (func_import->call_conv_wasm_c_api) { + ret = wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, native_func_pointer, + func_import->func_type, cur_func->param_cell_num, frame->lp, + c_api_func_import->with_env_arg, c_api_func_import->env_arg); + if (ret) { + argv_ret[0] = frame->lp[0]; + argv_ret[1] = frame->lp[1]; + } + } + else if (!func_import->call_conv_raw) { + ret = wasm_runtime_invoke_native( + exec_env, native_func_pointer, func_import->func_type, + func_import->signature, func_import->attachment, frame->lp, + cur_func->param_cell_num, argv_ret); + } + else { + ret = wasm_runtime_invoke_native_raw( + exec_env, native_func_pointer, func_import->func_type, + func_import->signature, func_import->attachment, frame->lp, + cur_func->param_cell_num, argv_ret); + } + + if (!ret) + return; + +#if WASM_ENABLE_GC != 0 + func_type = cur_func->u.func_import->func_type; + if (func_type->result_count + && wasm_is_type_reftype(func_type->types[cur_func->param_count]) + && !wasm_is_reftype_i31ref(func_type->types[cur_func->param_count])) { + frame_ref = prev_frame->frame_ref + prev_frame->ret_offset; +#if UINTPTR_MAX == UINT64_MAX + *frame_ref = *(frame_ref + 1) = 1; +#else + *frame_ref = 1; +#endif + } +#endif + + if (cur_func->ret_cell_num == 1) { + prev_frame->lp[prev_frame->ret_offset] = argv_ret[0]; + } + else if (cur_func->ret_cell_num == 2) { + prev_frame->lp[prev_frame->ret_offset] = argv_ret[0]; + prev_frame->lp[prev_frame->ret_offset + 1] = argv_ret[1]; + } + + FREE_FRAME(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +static void +wasm_interp_call_func_bytecode(WASMModuleInstance *module, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame); + +static void +wasm_interp_call_func_import(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMModuleInstance *sub_module_inst = cur_func->import_module_inst; + WASMFunctionInstance *sub_func_inst = cur_func->import_func_inst; + WASMFunctionImport *func_import = cur_func->u.func_import; + uint8 *ip = prev_frame->ip; + char buf[128]; + WASMExecEnv *sub_module_exec_env = NULL; + uintptr_t aux_stack_origin_boundary = 0; + uintptr_t aux_stack_origin_bottom = 0; + + if (!sub_func_inst) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + func_import->module_name, func_import->field_name); + wasm_set_exception(module_inst, buf); + return; + } + + /* Switch exec_env but keep using the same one by replacing necessary + * variables */ + sub_module_exec_env = wasm_runtime_get_exec_env_singleton( + (WASMModuleInstanceCommon *)sub_module_inst); + if (!sub_module_exec_env) { + wasm_set_exception(module_inst, "create singleton exec_env failed"); + return; + } + + /* - module_inst */ + wasm_exec_env_set_module_inst(exec_env, + (WASMModuleInstanceCommon *)sub_module_inst); + /* - aux_stack_boundary */ + aux_stack_origin_boundary = exec_env->aux_stack_boundary; + exec_env->aux_stack_boundary = sub_module_exec_env->aux_stack_boundary; + /* - aux_stack_bottom */ + aux_stack_origin_bottom = exec_env->aux_stack_bottom; + exec_env->aux_stack_bottom = sub_module_exec_env->aux_stack_bottom; + + /* set ip NULL to make call_func_bytecode return after executing + this function */ + prev_frame->ip = NULL; + + /* call function of sub-module*/ + wasm_interp_call_func_bytecode(sub_module_inst, exec_env, sub_func_inst, + prev_frame); + + /* restore ip and other replaced */ + prev_frame->ip = ip; + exec_env->aux_stack_boundary = aux_stack_origin_boundary; + exec_env->aux_stack_bottom = aux_stack_origin_bottom; + wasm_exec_env_restore_module_inst(exec_env, + (WASMModuleInstanceCommon *)module_inst); +} +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +#define CHECK_SUSPEND_FLAGS() \ + do { \ + WASM_SUSPEND_FLAGS_LOCK(exec_env->wait_lock); \ + if (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) \ + & WASM_SUSPEND_FLAG_TERMINATE) { \ + /* terminate current thread */ \ + WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \ + return; \ + } \ + /* TODO: support suspend and breakpoint */ \ + WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \ + } while (0) +#endif + +#if WASM_ENABLE_OPCODE_COUNTER != 0 +typedef struct OpcodeInfo { + char *name; + uint64 count; +} OpcodeInfo; + +/* clang-format off */ +#define HANDLE_OPCODE(op) \ + { \ + #op, 0 \ + } +DEFINE_GOTO_TABLE(OpcodeInfo, opcode_table); +#undef HANDLE_OPCODE +/* clang-format on */ + +static void +wasm_interp_dump_op_count() +{ + uint32 i; + uint64 total_count = 0; + for (i = 0; i < WASM_OP_IMPDEP; i++) + total_count += opcode_table[i].count; + + os_printf("total opcode count: %ld\n", total_count); + for (i = 0; i < WASM_OP_IMPDEP; i++) + if (opcode_table[i].count > 0) + os_printf("\t\t%s count:\t\t%ld,\t\t%.2f%%\n", opcode_table[i].name, + opcode_table[i].count, + opcode_table[i].count * 100.0f / total_count); +} +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 + +/* #define HANDLE_OP(opcode) HANDLE_##opcode:printf(#opcode"\n"); */ +#if WASM_ENABLE_OPCODE_COUNTER != 0 +#define HANDLE_OP(opcode) HANDLE_##opcode : opcode_table[opcode].count++; +#else +#define HANDLE_OP(opcode) HANDLE_##opcode: +#endif +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define FETCH_OPCODE_AND_DISPATCH() \ + do { \ + const void *p_label_addr = *(void **)frame_ip; \ + frame_ip += sizeof(void *); \ + goto *p_label_addr; \ + } while (0) +#else +#if UINTPTR_MAX == UINT64_MAX +#define FETCH_OPCODE_AND_DISPATCH() \ + do { \ + const void *p_label_addr; \ + bh_assert(((uintptr_t)frame_ip & 1) == 0); \ + /* int32 relative offset was emitted in 64-bit target */ \ + p_label_addr = label_base + (int32)LOAD_U32_WITH_2U16S(frame_ip); \ + frame_ip += sizeof(int32); \ + goto *p_label_addr; \ + } while (0) +#else +#define FETCH_OPCODE_AND_DISPATCH() \ + do { \ + const void *p_label_addr; \ + bh_assert(((uintptr_t)frame_ip & 1) == 0); \ + /* uint32 label address was emitted in 32-bit target */ \ + p_label_addr = (void *)(uintptr_t)LOAD_U32_WITH_2U16S(frame_ip); \ + frame_ip += sizeof(int32); \ + goto *p_label_addr; \ + } while (0) +#endif +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#define HANDLE_OP_END() FETCH_OPCODE_AND_DISPATCH() + +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ + +#define HANDLE_OP(opcode) case opcode: +#define HANDLE_OP_END() continue + +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +static void **global_handle_table; +#endif + +static inline uint8 * +get_global_addr(uint8 *global_data, WASMGlobalInstance *global) +{ +#if WASM_ENABLE_MULTI_MODULE == 0 + return global_data + global->data_offset; +#else + return global->import_global_inst + ? global->import_module_inst->global_data + + global->import_global_inst->data_offset + : global_data + global->data_offset; +#endif +} + +static void +wasm_interp_call_func_bytecode(WASMModuleInstance *module, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + uint64 linear_mem_size = 0; + if (memory) +#if WASM_ENABLE_THREAD_MGR == 0 + linear_mem_size = memory->memory_data_size; +#else + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif +#endif + WASMGlobalInstance *globals = module->e ? module->e->globals : NULL; + WASMGlobalInstance *global; + uint8 *global_data = module->global_data; + uint8 opcode_IMPDEP = WASM_OP_IMPDEP; + WASMInterpFrame *frame = NULL; + /* Points to this special opcode so as to jump to the + * call_method_from_entry. */ + register uint8 *frame_ip = &opcode_IMPDEP; /* cache of frame->ip */ + register uint32 *frame_lp = NULL; /* cache of frame->lp */ +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 && UINTPTR_MAX == UINT64_MAX + /* cache of label base addr */ + register uint8 *label_base = &&HANDLE_WASM_OP_UNREACHABLE; +#endif +#endif +#if WASM_ENABLE_GC != 0 + register uint8 *frame_ref = NULL; /* cache of frame->ref */ + uint32 local_cell_num = 0; + int16 opnd_off; +#endif + uint8 *frame_ip_end = frame_ip + 1; + uint32 cond, count, fidx, tidx, frame_size = 0; + uint32 all_cell_num = 0; + int16 addr1, addr2, addr_ret = 0; + int32 didx, val; + uint8 *maddr = NULL; + uint32 local_idx, local_offset, global_idx; + uint8 opcode = 0, local_type, *global_addr; +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + bool disable_bounds_checks = !wasm_runtime_is_bounds_checks_enabled( + (WASMModuleInstanceCommon *)module); +#else + bool disable_bounds_checks = false; +#endif +#endif +#if WASM_ENABLE_GC != 0 + WASMObjectRef gc_obj; + WASMStructObjectRef struct_obj; + WASMArrayObjectRef array_obj; + WASMFuncObjectRef func_obj; + WASMI31ObjectRef i31_obj; + WASMExternrefObjectRef externref_obj; + uint32 type_idx; +#if WASM_ENABLE_STRINGREF != 0 + WASMString str_obj; + WASMStringrefObjectRef stringref_obj; + WASMStringviewWTF8ObjectRef stringview_wtf8_obj; + WASMStringviewWTF16ObjectRef stringview_wtf16_obj; + WASMStringviewIterObjectRef stringview_iter_obj; +#endif +#endif +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + bool is_return_call = false; +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#define HANDLE_OPCODE(op) &&HANDLE_##op + DEFINE_GOTO_TABLE(const void *, handle_table); +#undef HANDLE_OPCODE + if (exec_env == NULL) { + global_handle_table = (void **)handle_table; + return; + } +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + while (frame_ip < frame_ip_end) { + opcode = *frame_ip++; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + frame_ip++; +#endif + switch (opcode) { +#else + goto *handle_table[WASM_OP_IMPDEP]; +#endif + /* control instructions */ + HANDLE_OP(WASM_OP_UNREACHABLE) + { + wasm_set_exception(module, "unreachable"); + goto got_exception; + } + + HANDLE_OP(WASM_OP_IF) + { + cond = (uint32)POP_I32(); + + if (cond == 0) { + uint8 *else_addr = (uint8 *)LOAD_PTR(frame_ip); + if (else_addr == NULL) { + frame_ip = + (uint8 *)LOAD_PTR(frame_ip + sizeof(uint8 *)); + } + else { + frame_ip = else_addr; + } + } + else { + frame_ip += sizeof(uint8 *) * 2; + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_ELSE) + { + frame_ip = (uint8 *)LOAD_PTR(frame_ip); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + recover_br_info: + RECOVER_BR_INFO(); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_IF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + cond = frame_lp[GET_OFFSET()]; + + if (cond) + goto recover_br_info; + else + SKIP_BR_INFO(); + + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_TABLE) + { + uint32 arity, br_item_size; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + count = read_uint32(frame_ip); + didx = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + + if (!(didx >= 0 && (uint32)didx < count)) + didx = count; + + /* all br items must have the same arity and item size, + so we only calculate the first item size */ + arity = LOAD_U32_WITH_2U16S(frame_ip); + br_item_size = sizeof(uint32); /* arity */ + if (arity) { + /* total cell num */ + br_item_size += sizeof(uint32); + /* cells, src offsets and dst offsets */ + br_item_size += + (CELL_SIZE + sizeof(int16) + sizeof(uint16)) * arity; + } + /* target address */ + br_item_size += sizeof(uint8 *); + + frame_ip += br_item_size * didx; + goto recover_br_info; + } + + HANDLE_OP(WASM_OP_RETURN) + { + uint32 ret_idx; + WASMFuncType *func_type; + uint32 off, ret_offset; + uint8 *ret_types; + if (cur_func->is_import_func) + func_type = cur_func->u.func_import->func_type; + else + func_type = cur_func->u.func->func_type; + + /* types of each return value */ + ret_types = func_type->types + func_type->param_count; + ret_offset = prev_frame->ret_offset; + + for (ret_idx = 0, + off = sizeof(int16) * (func_type->result_count - 1); + ret_idx < func_type->result_count; + ret_idx++, off -= sizeof(int16)) { + if (ret_types[ret_idx] == VALUE_TYPE_I64 + || ret_types[ret_idx] == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR(prev_frame->lp + ret_offset, + GET_OPERAND(uint64, I64, off)); + ret_offset += 2; + } +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(ret_types[ret_idx])) { + PUT_REF_TO_ADDR(prev_frame->lp + ret_offset, + GET_OPERAND(void *, REF, off)); + if (!wasm_is_reftype_i31ref(ret_types[ret_idx])) { + *(prev_frame->frame_ref + ret_offset) = 1; +#if UINTPTR_MAX == UINT64_MAX + *(prev_frame->frame_ref + ret_offset + 1) = 1; +#endif + } + ret_offset += REF_CELL_NUM; + } +#endif + else { + prev_frame->lp[ret_offset] = + GET_OPERAND(uint32, I32, off); + ret_offset++; + } + } + goto return_func; + } + + HANDLE_OP(WASM_OP_CALL_INDIRECT) +#if WASM_ENABLE_TAIL_CALL != 0 + HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT) +#endif + { + WASMFuncType *cur_type, *cur_func_type; + WASMTableInstance *tbl_inst; + uint32 tbl_idx; + +#if WASM_ENABLE_TAIL_CALL != 0 + GET_OPCODE(); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + tidx = read_uint32(frame_ip); + cur_type = (WASMFuncType *)module->module->types[tidx]; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + val = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + + if ((uint32)val >= tbl_inst->cur_size) { + wasm_set_exception(module, "undefined element"); + goto got_exception; + } + + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + fidx = (uint32)tbl_inst->elems[val]; + if (fidx == (uint32)-1) { + wasm_set_exception(module, "uninitialized element"); + goto got_exception; + } +#else + func_obj = (WASMFuncObjectRef)tbl_inst->elems[val]; + if (!func_obj) { + wasm_set_exception(module, "uninitialized element"); + goto got_exception; + } + fidx = wasm_func_obj_get_func_idx_bound(func_obj); +#endif + /* clang-format on */ + + /* + * we might be using a table injected by host or + * another module. in that case, we don't validate + * the elem value while loading + */ + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } + + /* always call module own functions */ + cur_func = module->e->functions + fidx; + + if (cur_func->is_import_func) + cur_func_type = cur_func->u.func_import->func_type; + else + cur_func_type = cur_func->u.func->func_type; + + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + if (cur_type != cur_func_type) { + wasm_set_exception(module, "indirect call type mismatch"); + goto got_exception; + } +#else + if (!wasm_func_type_is_super_of(cur_type, cur_func_type)) { + wasm_set_exception(module, "indirect call type mismatch"); + goto got_exception; + } +#endif + /* clang-format on */ + +#if WASM_ENABLE_TAIL_CALL != 0 + if (opcode == WASM_OP_RETURN_CALL_INDIRECT) + goto call_func_from_return_call; +#endif + goto call_func_from_interp; + } + +#if WASM_ENABLE_EXCE_HANDLING != 0 + HANDLE_OP(WASM_OP_TRY) + HANDLE_OP(WASM_OP_CATCH) + HANDLE_OP(WASM_OP_THROW) + HANDLE_OP(WASM_OP_RETHROW) + HANDLE_OP(WASM_OP_DELEGATE) + HANDLE_OP(WASM_OP_CATCH_ALL) + HANDLE_OP(EXT_OP_TRY) + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif + + /* parametric instructions */ + HANDLE_OP(WASM_OP_SELECT) + { + cond = frame_lp[GET_OFFSET()]; + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + if (!cond) { + if (addr_ret != addr1) + frame_lp[addr_ret] = frame_lp[addr1]; + } + else { + if (addr_ret != addr2) + frame_lp[addr_ret] = frame_lp[addr2]; + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SELECT_64) + { + cond = frame_lp[GET_OFFSET()]; + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + if (!cond) { + if (addr_ret != addr1) + PUT_I64_TO_ADDR(frame_lp + addr_ret, + GET_I64_FROM_ADDR(frame_lp + addr1)); + } + else { + if (addr_ret != addr2) + PUT_I64_TO_ADDR(frame_lp + addr_ret, + GET_I64_FROM_ADDR(frame_lp + addr2)); + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_SELECT_T) + { + cond = frame_lp[GET_OFFSET()]; + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + if (!cond) { + if (addr_ret != addr1) + PUT_REF_TO_ADDR(frame_lp + addr_ret, + GET_REF_FROM_ADDR(frame_lp + addr1)); + } + else { + if (addr_ret != addr2) + PUT_REF_TO_ADDR(frame_lp + addr_ret, + GET_REF_FROM_ADDR(frame_lp + addr2)); + } + { + uint8 orig_ref = 0; + /* Ignore constants because they are not reference */ + if (addr1 >= 0) { + orig_ref = *FRAME_REF(addr1); + CLEAR_FRAME_REF(addr1); + } + if (addr2 >= 0) { + CLEAR_FRAME_REF(addr2); + } + if (orig_ref) { + SET_FRAME_REF(addr_ret); + } + } + + HANDLE_OP_END(); + } +#endif + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_TABLE_GET) + { + uint32 tbl_idx, elem_idx; + WASMTableInstance *tbl_inst; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + elem_idx = POP_I32(); + if (elem_idx >= tbl_inst->cur_size) { + wasm_set_exception(module, "out of bounds table access"); + goto got_exception; + } + +#if WASM_ENABLE_GC == 0 + PUSH_I32(tbl_inst->elems[elem_idx]); +#else + PUSH_REF(tbl_inst->elems[elem_idx]); +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_TABLE_SET) + { + uint32 tbl_idx, elem_idx; + WASMTableInstance *tbl_inst; + table_elem_type_t elem_val; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + +#if WASM_ENABLE_GC == 0 + elem_val = POP_I32(); +#else + elem_val = POP_REF(); +#endif + elem_idx = POP_I32(); + if (elem_idx >= tbl_inst->cur_size) { + wasm_set_exception(module, "out of bounds table access"); + goto got_exception; + } + + tbl_inst->elems[elem_idx] = elem_val; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_NULL) + { +#if WASM_ENABLE_GC == 0 + PUSH_I32(NULL_REF); +#else + PUSH_REF(NULL_REF); +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_IS_NULL) + { +#if WASM_ENABLE_GC == 0 + uint32 ref_val; + ref_val = POP_I32(); +#else + void *ref_val; + ref_val = POP_REF(); +#endif + PUSH_I32(ref_val == NULL_REF ? 1 : 0); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_FUNC) + { + uint32 func_idx = read_uint32(frame_ip); + +#if WASM_ENABLE_GC == 0 + PUSH_I32(func_idx); +#else + SYNC_ALL_TO_FRAME(); + if (!(gc_obj = wasm_create_func_obj(module, func_idx, true, + NULL, 0))) { + goto got_exception; + } + PUSH_REF(gc_obj); +#endif + HANDLE_OP_END(); + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_CALL_REF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + func_obj = POP_REF(); + if (!func_obj) { + wasm_set_exception(module, "null function object"); + goto got_exception; + } + + fidx = wasm_func_obj_get_func_idx_bound(func_obj); + cur_func = module->e->functions + fidx; + goto call_func_from_interp; + } + HANDLE_OP(WASM_OP_RETURN_CALL_REF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + func_obj = POP_REF(); + if (!func_obj) { + wasm_set_exception(module, "null function object"); + goto got_exception; + } + + fidx = wasm_func_obj_get_func_idx_bound(func_obj); + cur_func = module->e->functions + fidx; + goto call_func_from_return_call; + } + HANDLE_OP(WASM_OP_REF_AS_NON_NULL) + { + gc_obj = POP_REF(); + if (gc_obj == NULL_REF) { + wasm_set_exception(module, "null reference"); + goto got_exception; + } + PUSH_REF(gc_obj); + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_REF_EQ) + { + WASMObjectRef gc_obj1, gc_obj2; + gc_obj2 = POP_REF(); + gc_obj1 = POP_REF(); + val = wasm_obj_equal(gc_obj1, gc_obj2); + PUSH_I32(val); + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_BR_ON_NULL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + opnd_off = GET_OFFSET(); + gc_obj = GET_REF_FROM_ADDR(frame_lp + opnd_off); + if (gc_obj == NULL_REF) { + CLEAR_FRAME_REF(opnd_off); + goto recover_br_info; + } + else { + SKIP_BR_INFO(); + } + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_BR_ON_NON_NULL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + opnd_off = GET_OFFSET(); + gc_obj = GET_REF_FROM_ADDR(frame_lp + opnd_off); + if (gc_obj != NULL_REF) { + goto recover_br_info; + } + else { + CLEAR_FRAME_REF(opnd_off); + SKIP_BR_INFO(); + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GC_PREFIX) + { + GET_OPCODE(); + + switch (opcode) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + { + WASMModule *wasm_module = module->module; + WASMStructType *struct_type; + WASMRttType *rtt_type; + WASMValue field_value = { 0 }; + + type_idx = read_uint32(frame_ip); + struct_type = + (WASMStructType *)module->module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_idx, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + struct_obj = wasm_struct_obj_new(exec_env, rtt_type); + if (!struct_obj) { + wasm_set_exception(module, + "create struct object failed"); + goto got_exception; + } + + if (opcode == WASM_OP_STRUCT_NEW) { + WASMStructFieldType *fields = struct_type->fields; + int32 field_count = (int32)struct_type->field_count; + int32 field_idx; + uint8 field_type; + + for (field_idx = field_count - 1; field_idx >= 0; + field_idx--) { + field_type = fields[field_idx].field_type; + if (wasm_is_type_reftype(field_type)) { + field_value.gc_obj = POP_REF(); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + field_value.i32 = POP_I32(); + } + else { + field_value.i64 = POP_I64(); + } + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + } + } + PUSH_REF(struct_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + { + WASMStructType *struct_type; + WASMValue field_value = { 0 }; + uint32 field_idx; + uint8 field_type; + + type_idx = read_uint32(frame_ip); + field_idx = read_uint32(frame_ip); + + struct_type = + (WASMStructType *)module->module->types[type_idx]; + + struct_obj = POP_REF(); + + if (!struct_obj) { + wasm_set_exception(module, "null structure object"); + goto got_exception; + } + + wasm_struct_obj_get_field( + struct_obj, field_idx, + opcode == WASM_OP_STRUCT_GET_S ? true : false, + &field_value); + + field_type = struct_type->fields[field_idx].field_type; + if (wasm_is_reftype_i31ref(field_type)) { + PUSH_I31REF(field_value.gc_obj); + } + else if (wasm_is_type_reftype(field_type)) { + PUSH_REF(field_value.gc_obj); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + PUSH_I32(field_value.i32); + } + else { + PUSH_I64(field_value.i64); + } + HANDLE_OP_END(); + } + case WASM_OP_STRUCT_SET: + { + WASMStructType *struct_type; + WASMValue field_value = { 0 }; + uint32 field_idx; + uint8 field_type; + + type_idx = read_uint32(frame_ip); + field_idx = read_uint32(frame_ip); + + struct_type = + (WASMStructType *)module->module->types[type_idx]; + field_type = struct_type->fields[field_idx].field_type; + + if (wasm_is_type_reftype(field_type)) { + field_value.gc_obj = POP_REF(); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + field_value.i32 = POP_I32(); + } + else { + field_value.i64 = POP_I64(); + } + + struct_obj = POP_REF(); + if (!struct_obj) { + wasm_set_exception(module, "null structure object"); + goto got_exception; + } + + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + { + WASMModule *wasm_module = module->module; + WASMArrayType *array_type; + WASMRttType *rtt_type; + WASMValue array_elem = { 0 }; + uint32 array_len, i; + + type_idx = read_uint32(frame_ip); + array_type = + (WASMArrayType *)wasm_module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + if (opcode != WASM_OP_ARRAY_NEW_FIXED) + array_len = POP_I32(); + else + array_len = read_uint32(frame_ip); + + if (opcode == WASM_OP_ARRAY_NEW) { + if (wasm_is_type_reftype(array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type + == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + } + + SYNC_ALL_TO_FRAME(); + array_obj = wasm_array_obj_new(exec_env, rtt_type, + array_len, &array_elem); + if (!array_obj) { + wasm_set_exception(module, + "create array object failed"); + goto got_exception; + } + + if (opcode == WASM_OP_ARRAY_NEW_FIXED) { + for (i = 0; i < array_len; i++) { + if (wasm_is_type_reftype( + array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type + == VALUE_TYPE_F32 + || array_type->elem_type + == PACKED_TYPE_I8 + || array_type->elem_type + == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + wasm_array_obj_set_elem( + array_obj, array_len - 1 - i, &array_elem); + } + } + + PUSH_REF(array_obj); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW_DATA: + { + WASMModule *wasm_module = module->module; + WASMArrayType *array_type; + WASMRttType *rtt_type; + WASMValue array_elem = { 0 }; + WASMDataSeg *data_seg; + uint8 *array_elem_base; + uint32 array_len, data_seg_idx, data_seg_offset; + uint32 elem_size = 0; + uint64 total_size; + + type_idx = read_uint32(frame_ip); + data_seg_idx = read_uint32(frame_ip); + data_seg = wasm_module->data_segments[data_seg_idx]; + + array_type = + (WASMArrayType *)wasm_module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + array_len = POP_I32(); + data_seg_offset = POP_I32(); + + switch (array_type->elem_type) { + case PACKED_TYPE_I8: + elem_size = 1; + break; + case PACKED_TYPE_I16: + elem_size = 2; + break; + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + elem_size = 4; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + elem_size = 8; + break; + default: + bh_assert(0); + } + + total_size = (uint64)elem_size * array_len; + if (data_seg_offset >= data_seg->data_length + || total_size + > data_seg->data_length - data_seg_offset) { + wasm_set_exception(module, + "data segment out of bounds"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + array_obj = wasm_array_obj_new(exec_env, rtt_type, + array_len, &array_elem); + if (!array_obj) { + wasm_set_exception(module, + "create array object failed"); + goto got_exception; + } + + array_elem_base = + (uint8 *)wasm_array_obj_first_elem_addr(array_obj); + bh_memcpy_s(array_elem_base, (uint32)total_size, + data_seg->data + data_seg_offset, + (uint32)total_size); + + PUSH_REF(array_obj); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW_ELEM: + { + /* TODO */ + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + { + WASMArrayType *array_type; + WASMValue array_elem = { 0 }; + uint32 elem_idx, elem_size_log; + + type_idx = read_uint32(frame_ip); + array_type = + (WASMArrayType *)module->module->types[type_idx]; + + elem_idx = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + if (elem_idx >= wasm_array_obj_length(array_obj)) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_get_elem( + array_obj, elem_idx, + opcode == WASM_OP_ARRAY_GET_S ? true : false, + &array_elem); + elem_size_log = wasm_array_obj_elem_size_log(array_obj); + + if (wasm_is_reftype_i31ref(array_type->elem_type)) { + PUSH_I31REF(array_elem.gc_obj); + } + else if (wasm_is_type_reftype(array_type->elem_type)) { + PUSH_REF(array_elem.gc_obj); + } + else if (elem_size_log < 3) { + PUSH_I32(array_elem.i32); + } + else { + PUSH_I64(array_elem.i64); + } + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_SET: + { + WASMArrayType *array_type; + WASMValue array_elem = { 0 }; + uint32 elem_idx; + + type_idx = read_uint32(frame_ip); + array_type = + (WASMArrayType *)module->module->types[type_idx]; + if (wasm_is_type_reftype(array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + + elem_idx = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + if (elem_idx >= wasm_array_obj_length(array_obj)) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_set_elem(array_obj, elem_idx, + &array_elem); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_LEN: + { + uint32 array_len; + array_obj = POP_REF(); + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + array_len = wasm_array_obj_length(array_obj); + PUSH_I32(array_len); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_FILL: + { + WASMArrayType *array_type; + WASMValue fill_value = { 0 }; + uint32 start_offset, len; + + type_idx = read_uint32(frame_ip); + + array_type = + (WASMArrayType *)module->module->types[type_idx]; + + len = POP_I32(); + if (wasm_is_type_reftype(array_type->elem_type)) { + fill_value.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type == PACKED_TYPE_I16) { + fill_value.i32 = POP_I32(); + } + else { + fill_value.i64 = POP_I64(); + } + start_offset = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + + if (len > 0) { + if ((uint64)start_offset + len + >= wasm_array_obj_length(array_obj)) { + wasm_set_exception( + module, "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_fill(array_obj, start_offset, len, + &fill_value); + } + + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_COPY: + { + uint32 dst_offset, src_offset, len, src_type_index; + WASMArrayObjectRef src_obj, dst_obj; + + type_idx = read_uint32(frame_ip); + src_type_index = read_uint32(frame_ip); + + len = POP_I32(); + src_offset = POP_I32(); + src_obj = POP_REF(); + dst_offset = POP_I32(); + dst_obj = POP_REF(); + + if (!src_obj || !dst_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + + if (len > 0) { + if ((dst_offset > UINT32_MAX - len) + || (dst_offset + len + > wasm_array_obj_length(dst_obj)) + || (src_offset > UINT32_MAX - len) + || (src_offset + len + > wasm_array_obj_length(src_obj))) { + wasm_set_exception( + module, "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_copy(dst_obj, dst_offset, src_obj, + src_offset, len); + } + + (void)src_type_index; + HANDLE_OP_END(); + } + + case WASM_OP_REF_I31: + { + uint32 i31_val; + + i31_val = POP_I32(); + i31_obj = wasm_i31_obj_new(i31_val); + PUSH_I31REF(i31_obj); + HANDLE_OP_END(); + } + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + { + uint32 i31_val; + + i31_obj = (WASMI31ObjectRef)POP_REF(); + if (!i31_obj) { + wasm_set_exception(module, "null i31 reference"); + goto got_exception; + } + i31_val = (uint32)(((uintptr_t)i31_obj) >> 1); + if (opcode == WASM_OP_I31_GET_S + && (i31_val & 0x40000000) /* bit 30 is 1 */) + /* set bit 31 to 1 */ + i31_val |= 0x80000000; + PUSH_I32(i31_val); + HANDLE_OP_END(); + } + + case WASM_OP_REF_TEST: + case WASM_OP_REF_CAST: + case WASM_OP_REF_TEST_NULLABLE: + case WASM_OP_REF_CAST_NULLABLE: + { + int32 heap_type; + + heap_type = (int32)read_uint32(frame_ip); + + gc_obj = POP_REF(); + if (!gc_obj) { + if (opcode == WASM_OP_REF_TEST + || opcode == WASM_OP_REF_TEST_NULLABLE) { + if (opcode == WASM_OP_REF_TEST) + PUSH_I32(0); + else + PUSH_I32(1); + } + else if (opcode == WASM_OP_REF_CAST) { + wasm_set_exception(module, "cast failure"); + goto got_exception; + } + else { + PUSH_REF(gc_obj); + } + } + else { + bool castable = false; + + if (heap_type >= 0) { + WASMModule *wasm_module = module->module; + castable = wasm_obj_is_instance_of( + gc_obj, (uint32)heap_type, + wasm_module->types, + wasm_module->type_count); + } + else { + castable = + wasm_obj_is_type_of(gc_obj, heap_type); + } + + if (opcode == WASM_OP_REF_TEST + || opcode == WASM_OP_REF_TEST_NULLABLE) { + if (castable) + PUSH_I32(1); + else + PUSH_I32(0); + } + else if (!castable) { + wasm_set_exception(module, "cast failure"); + goto got_exception; + } + else { + PUSH_REF(gc_obj); + } + } + HANDLE_OP_END(); + } + + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + { + int32 heap_type, heap_type_dst; + uint8 castflags; + uint16 opnd_off_br; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + castflags = *frame_ip++; + heap_type = (int32)read_uint32(frame_ip); + heap_type_dst = (int32)read_uint32(frame_ip); + + opnd_off = GET_OFFSET(); + opnd_off_br = GET_OFFSET(); + gc_obj = GET_REF_FROM_ADDR(frame_lp + opnd_off); + PUT_REF_TO_ADDR(frame_lp + opnd_off_br, gc_obj); + + if (!gc_obj) { + /* + * castflags should be 0~3: + * 0: (non-null, non-null) + * 1: (null, non-null) + * 2: (non-null, null) + * 3: (null, null) + */ + if ( + /* op is BR_ON_CAST and dst reftype is nullable + */ + ((opcode == WASM_OP_BR_ON_CAST) + && ((castflags == 2) || (castflags == 3))) + /* op is BR_ON_CAST_FAIL and dst reftype is + non-nullable */ + || ((opcode == WASM_OP_BR_ON_CAST_FAIL) + && ((castflags == 0) + || (castflags == 1)))) { + CLEAR_FRAME_REF(opnd_off); + if (!wasm_is_reftype_i31ref(heap_type)) { + SET_FRAME_REF(opnd_off_br); + } + goto recover_br_info; + } + } + else { + bool castable = false; + + if (heap_type_dst >= 0) { + WASMModule *wasm_module = module->module; + castable = wasm_obj_is_instance_of( + gc_obj, (uint32)heap_type_dst, + wasm_module->types, + wasm_module->type_count); + } + else { + castable = + wasm_obj_is_type_of(gc_obj, heap_type_dst); + } + + if ((castable && (opcode == WASM_OP_BR_ON_CAST)) + || (!castable + && (opcode == WASM_OP_BR_ON_CAST_FAIL))) { + CLEAR_FRAME_REF(opnd_off); + if (!wasm_is_reftype_i31ref(heap_type)) { + SET_FRAME_REF(opnd_off_br); + } + goto recover_br_info; + } + } + SKIP_BR_INFO(); + + (void)heap_type_dst; + HANDLE_OP_END(); + } + + case WASM_OP_ANY_CONVERT_EXTERN: + { + externref_obj = POP_REF(); + if (externref_obj == NULL_REF) + PUSH_REF(NULL_REF); + else { + gc_obj = wasm_externref_obj_to_internal_obj( + externref_obj); + PUSH_REF(gc_obj); + } + HANDLE_OP_END(); + } + case WASM_OP_EXTERN_CONVERT_ANY: + { + gc_obj = POP_REF(); + if (gc_obj == NULL_REF) + PUSH_REF(NULL_REF); + else { + if (!(externref_obj = + wasm_internal_obj_to_externref_obj( + exec_env, gc_obj))) { + wasm_set_exception( + module, "create externref object failed"); + goto got_exception; + } + PUSH_REF(externref_obj); + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + { + uint32 mem_idx, addr, bytes_length, offset = 0; + EncodingFlag flag = WTF8; + + mem_idx = (uint32)read_uint32(frame_ip); + bytes_length = POP_I32(); + addr = POP_I32(); + + CHECK_MEMORY_OVERFLOW(bytes_length); + + if (opcode == WASM_OP_STRING_NEW_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_NEW_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8) { + flag = WTF8; + } + + str_obj = wasm_string_new_with_encoding( + maddr, bytes_length, flag); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + + (void)mem_idx; + HANDLE_OP_END(); + } + case WASM_OP_STRING_CONST: + { + WASMModule *wasm_module = module->module; + uint32 contents; + + contents = (uint32)read_uint32(frame_ip); + + str_obj = wasm_string_new_const( + (const char *) + wasm_module->string_literal_ptrs[contents], + wasm_module->string_literal_lengths[contents]); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!str_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + { + int32 target_bytes_length; + EncodingFlag flag = WTF8; + + stringref_obj = POP_REF(); + + if (opcode == WASM_OP_STRING_MEASURE_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_MEASURE_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_MEASURE_WTF8) { + flag = LOSSY_UTF8; + } + target_bytes_length = wasm_string_measure( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + flag); + + PUSH_I32(target_bytes_length); + HANDLE_OP_END(); + } + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + { + uint32 mem_idx, addr; + int32 target_bytes_length; + WASMMemoryInstance *memory_inst; + EncodingFlag flag = WTF8; + + mem_idx = (uint32)read_uint32(frame_ip); + addr = POP_I32(); + stringref_obj = POP_REF(); + + str_obj = (WASMString)wasm_stringref_obj_get_value( + stringref_obj); + + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + + if (opcode == WASM_OP_STRING_ENCODE_WTF16) { + flag = WTF16; + count = wasm_string_measure(str_obj, flag); + target_bytes_length = wasm_string_encode( + str_obj, 0, count, maddr, NULL, flag); + } + else { + if (opcode == WASM_OP_STRING_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRING_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_ENCODE_WTF8) { + flag = WTF8; + } + count = wasm_string_measure(str_obj, flag); + target_bytes_length = wasm_string_encode( + str_obj, 0, count, maddr, NULL, flag); + + if (target_bytes_length == -1) { + wasm_set_exception( + module, "isolated surrogate is seen"); + goto got_exception; + } + } + if (target_bytes_length < 0) { + wasm_set_exception(module, + "stringref encode failed"); + goto got_exception; + } + + PUSH_I32(target_bytes_length); + HANDLE_OP_END(); + } + case WASM_OP_STRING_CONCAT: + { + WASMStringrefObjectRef stringref_obj1, stringref_obj2; + + stringref_obj2 = POP_REF(); + stringref_obj1 = POP_REF(); + + str_obj = wasm_string_concat( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj1), + (WASMString)wasm_stringref_obj_get_value( + stringref_obj2)); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_EQ: + { + WASMStringrefObjectRef stringref_obj1, stringref_obj2; + int32 is_eq; + + stringref_obj2 = POP_REF(); + stringref_obj1 = POP_REF(); + + is_eq = wasm_string_eq( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj1), + (WASMString)wasm_stringref_obj_get_value( + stringref_obj2)); + + PUSH_I32(is_eq); + HANDLE_OP_END(); + } + case WASM_OP_STRING_IS_USV_SEQUENCE: + { + int32 is_usv_sequence; + + stringref_obj = POP_REF(); + + is_usv_sequence = wasm_string_is_usv_sequence( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj)); + + PUSH_I32(is_usv_sequence); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_WTF8: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_WTF8); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_wtf8_obj = + wasm_stringview_wtf8_obj_new(exec_env, str_obj); + if (!stringview_wtf8_obj) { + wasm_set_exception(module, + "create stringview wtf8 failed"); + goto got_exception; + } + + PUSH_REF(stringview_wtf8_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + { + uint32 next_pos, bytes, pos; + + bytes = POP_I32(); + pos = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + next_pos = wasm_string_advance( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + pos, bytes, NULL); + + PUSH_I32(next_pos); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + { + uint32 mem_idx, addr, pos, bytes, next_pos; + int32 bytes_written; + WASMMemoryInstance *memory_inst; + EncodingFlag flag = WTF8; + + if (opcode == WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8) { + flag = WTF8; + } + + mem_idx = (uint32)read_uint32(frame_ip); + bytes = POP_I32(); + pos = POP_I32(); + addr = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + + bytes_written = wasm_string_encode( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + pos, bytes, maddr, &next_pos, flag); + + if (bytes_written < 0) { + if (bytes_written == Isolated_Surrogate) { + wasm_set_exception( + module, "isolated surrogate is seen"); + } + else { + wasm_set_exception(module, "encode failed"); + } + + goto got_exception; + } + + PUSH_I32(next_pos); + PUSH_I32(bytes_written); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_SLICE: + { + uint32 start, end; + + end = POP_I32(); + start = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + start, end, STRING_VIEW_WTF8); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_WTF16: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_WTF16); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_wtf16_obj = + wasm_stringview_wtf16_obj_new(exec_env, str_obj); + if (!stringview_wtf16_obj) { + wasm_set_exception( + module, "create stringview wtf16 failed"); + goto got_exception; + } + + PUSH_REF(stringview_wtf16_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + { + int32 code_units_length; + + stringview_wtf16_obj = POP_REF(); + + code_units_length = wasm_string_wtf16_get_length( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj)); + + PUSH_I32(code_units_length); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + { + int32 pos; + uint32 code_unit; + + pos = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + code_unit = (uint32)wasm_string_get_wtf16_codeunit( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + pos); + + PUSH_I32(code_unit); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + { + uint32 mem_idx, addr, pos, len, offset = 0; + int32 written_code_units = 0; + + mem_idx = (uint32)read_uint32(frame_ip); + len = POP_I32(); + pos = POP_I32(); + addr = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + CHECK_MEMORY_OVERFLOW(len * sizeof(uint16)); + + /* check 2-byte alignment */ + if (((uintptr_t)maddr & (((uintptr_t)1 << 2) - 1)) + != 0) { + wasm_set_exception(module, + "unaligned memory access"); + goto got_exception; + } + + written_code_units = wasm_string_encode( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + pos, len, maddr, NULL, WTF16); + + PUSH_I32(written_code_units); + (void)mem_idx; + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_SLICE: + { + uint32 start, end; + + end = POP_I32(); + start = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + start, end, STRING_VIEW_WTF16); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_ITER: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_ITER); + + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_iter_obj = + wasm_stringview_iter_obj_new(exec_env, str_obj, 0); + if (!stringview_iter_obj) { + wasm_set_exception(module, + "create stringview iter failed"); + goto got_exception; + } + + PUSH_REF(stringview_iter_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_NEXT: + { + uint32 code_point; + + stringview_iter_obj = POP_REF(); + + code_point = wasm_string_next_codepoint( + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj), + wasm_stringview_iter_obj_get_pos( + stringview_iter_obj)); + + PUSH_I32(code_point); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + case WASM_OP_STRINGVIEW_ITER_REWIND: + { + uint32 code_points_count, code_points_consumed = 0, + cur_pos, next_pos = 0; + + code_points_count = POP_I32(); + stringview_iter_obj = POP_REF(); + + str_obj = + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj); + cur_pos = wasm_stringview_iter_obj_get_pos( + stringview_iter_obj); + + if (opcode == WASM_OP_STRINGVIEW_ITER_ADVANCE) { + next_pos = wasm_string_advance( + str_obj, cur_pos, code_points_count, + &code_points_consumed); + } + else if (opcode == WASM_OP_STRINGVIEW_ITER_REWIND) { + next_pos = wasm_string_rewind( + str_obj, cur_pos, code_points_count, + &code_points_consumed); + } + + wasm_stringview_iter_obj_update_pos(stringview_iter_obj, + next_pos); + + PUSH_I32(code_points_consumed); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_SLICE: + { + uint32 code_points_count, cur_pos; + + code_points_count = POP_I32(); + stringview_iter_obj = POP_REF(); + + cur_pos = wasm_stringview_iter_obj_get_pos( + stringview_iter_obj); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj), + cur_pos, cur_pos + code_points_count, + STRING_VIEW_ITER); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + { + uint32 start, end, array_len; + EncodingFlag flag = WTF8; + WASMArrayType *array_type; + void *arr_start_addr; + + end = POP_I32(); + start = POP_I32(); + array_obj = POP_REF(); + + array_type = (WASMArrayType *)wasm_obj_get_defined_type( + (WASMObjectRef)array_obj); + arr_start_addr = + wasm_array_obj_elem_addr(array_obj, start); + array_len = wasm_array_obj_length(array_obj); + + if (start > end || end > array_len) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + if (opcode == WASM_OP_STRING_NEW_WTF16_ARRAY) { + if (array_type->elem_type != VALUE_TYPE_I16) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + flag = WTF16; + } + else { + if (array_type->elem_type != VALUE_TYPE_I8) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + if (opcode == WASM_OP_STRING_NEW_UTF8_ARRAY) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8_ARRAY) { + flag = WTF8; + } + else if (opcode + == WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY) { + flag = LOSSY_UTF8; + } + } + + str_obj = wasm_string_new_with_encoding( + arr_start_addr, (end - start), flag); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + { + uint32 start, array_len, count; + int32 bytes_written; + EncodingFlag flag = WTF8; + WASMArrayType *array_type; + void *arr_start_addr; + + start = POP_I32(); + array_obj = POP_REF(); + stringref_obj = POP_REF(); + + str_obj = (WASMString)wasm_stringref_obj_get_value( + stringref_obj); + + array_type = (WASMArrayType *)wasm_obj_get_defined_type( + (WASMObjectRef)array_obj); + arr_start_addr = + wasm_array_obj_elem_addr(array_obj, start); + array_len = wasm_array_obj_length(array_obj); + + if (start > array_len) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + if (opcode == WASM_OP_STRING_ENCODE_WTF16_ARRAY) { + if (array_type->elem_type != VALUE_TYPE_I16) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + flag = WTF16; + } + else { + if (array_type->elem_type != VALUE_TYPE_I8) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + if (opcode == WASM_OP_STRING_ENCODE_UTF8_ARRAY) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRING_ENCODE_WTF8_ARRAY) { + flag = WTF8; + } + else if ( + opcode + == WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY) { + flag = LOSSY_UTF8; + } + } + + count = wasm_string_measure(str_obj, flag); + + bytes_written = wasm_string_encode( + str_obj, 0, count, arr_start_addr, NULL, flag); + + if (bytes_written < 0) { + if (bytes_written == Isolated_Surrogate) { + wasm_set_exception( + module, "isolated surrogate is seen"); + } + else if (bytes_written == Insufficient_Space) { + wasm_set_exception( + module, "array space is insufficient"); + } + else { + wasm_set_exception(module, "encode failed"); + } + + goto got_exception; + } + + PUSH_I32(bytes_written); + HANDLE_OP_END(); + } +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + + default: + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + /* variable instructions */ + HANDLE_OP(EXT_OP_SET_LOCAL_FAST) + HANDLE_OP(EXT_OP_TEE_LOCAL_FAST) + { + /* clang-format off */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + local_offset = *frame_ip++; +#else + local_offset = *frame_ip; + frame_ip += 2; +#endif + /* clang-format on */ + *(uint32 *)(frame_lp + local_offset) = + GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_SET_LOCAL_FAST_I64) + HANDLE_OP(EXT_OP_TEE_LOCAL_FAST_I64) + { + /* clang-format off */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + local_offset = *frame_ip++; +#else + local_offset = *frame_ip; + frame_ip += 2; +#endif + /* clang-format on */ + PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_OPERAND(uint64, I64, 0)); + frame_ip += 2; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GET_GLOBAL) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr_ret = GET_OFFSET(); + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + frame_lp[addr_ret] = *(uint32 *)global_addr; +#else + if (!wasm_is_type_reftype(global->type)) + frame_lp[addr_ret] = *(uint32 *)global_addr; + else { + PUT_REF_TO_ADDR(frame_lp + addr_ret, + GET_REF_FROM_ADDR((uint32 *)global_addr)); + if (!wasm_is_reftype_i31ref(global->type)) { + SET_FRAME_REF(addr_ret); + } + } +#endif + /* clang-format on */ + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GET_GLOBAL_64) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr_ret = GET_OFFSET(); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + GET_I64_FROM_ADDR((uint32 *)global_addr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr1 = GET_OFFSET(); + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + *(int32 *)global_addr = frame_lp[addr1]; +#else + if (!wasm_is_type_reftype(global->type)) + *(int32 *)global_addr = frame_lp[addr1]; + else { + PUT_REF_TO_ADDR((uint32 *)global_addr, + GET_REF_FROM_ADDR(frame_lp + addr1)); + CLEAR_FRAME_REF(addr1); + } +#endif + /* clang-format on */ + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL_AUX_STACK) + { + uint64 aux_stack_top; + + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + /* TODO: Memory64 the data type depends on mem idx type */ + aux_stack_top = (uint64)frame_lp[GET_OFFSET()]; + if (aux_stack_top <= (uint64)exec_env->aux_stack_boundary) { + wasm_set_exception(module, "wasm auxiliary stack overflow"); + goto got_exception; + } + if (aux_stack_top > (uint64)exec_env->aux_stack_bottom) { + wasm_set_exception(module, + "wasm auxiliary stack underflow"); + goto got_exception; + } + *(int32 *)global_addr = (uint32)aux_stack_top; +#if WASM_ENABLE_MEMORY_PROFILING != 0 + if (module->module->aux_stack_top_global_index != (uint32)-1) { + uint32 aux_stack_used = + (uint32)(module->module->aux_stack_bottom + - *(uint32 *)global_addr); + if (aux_stack_used > module->e->max_aux_stack_used) + module->e->max_aux_stack_used = aux_stack_used; + } +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL_64) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr1 = GET_OFFSET(); + PUT_I64_TO_ADDR((uint32 *)global_addr, + GET_I64_FROM_ADDR(frame_lp + addr1)); + HANDLE_OP_END(); + } + + /* memory load instructions */ + HANDLE_OP(WASM_OP_I32_LOAD) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + frame_lp[addr_ret] = LOAD_I32(maddr); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(8); + PUT_I64_TO_ADDR(frame_lp + addr_ret, LOAD_I64(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD8_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + frame_lp[addr_ret] = sign_ext_8_32(*(int8 *)maddr); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD8_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + frame_lp[addr_ret] = (uint32)(*(uint8 *)(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD16_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + frame_lp[addr_ret] = sign_ext_16_32(LOAD_I16(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD16_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + frame_lp[addr_ret] = (uint32)(LOAD_U16(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD8_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + sign_ext_8_64(*(int8 *)maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD8_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUT_I64_TO_ADDR(frame_lp + addr_ret, (uint64)(*(uint8 *)maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD16_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + sign_ext_16_64(LOAD_I16(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD16_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUT_I64_TO_ADDR(frame_lp + addr_ret, (uint64)(LOAD_U16(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD32_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + sign_ext_32_64(LOAD_I32(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD32_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + PUT_I64_TO_ADDR(frame_lp + addr_ret, (uint64)(LOAD_U32(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_STORE) + { + uint32 offset, addr; + uint32 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint32, I32, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(4); + STORE_U32(maddr, sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_STORE8) + { + uint32 offset, addr; + uint32 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint32, I32, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(1); + STORE_U8(maddr, (uint8_t)sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_STORE16) + { + uint32 offset, addr; + uint32 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint32, I32, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(2); + STORE_U16(maddr, (uint16)sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(8); + STORE_I64(maddr, sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE8) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(1); + *(uint8 *)maddr = (uint8)sval; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE16) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(2); + STORE_U16(maddr, (uint16)sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE32) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(4); + STORE_U32(maddr, (uint32)sval); + HANDLE_OP_END(); + } + + /* memory size and memory grow instructions */ + HANDLE_OP(WASM_OP_MEMORY_SIZE) + { + uint32 reserved; + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = memory->cur_page_count; + (void)reserved; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_MEMORY_GROW) + { + uint32 reserved, delta, + prev_page_count = memory->cur_page_count; + + addr1 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + delta = (uint32)frame_lp[addr1]; + + if (!wasm_enlarge_memory(module, delta)) { + /* failed to memory.grow, return -1 */ + frame_lp[addr_ret] = -1; + } + else { + /* success, return previous page count */ + frame_lp[addr_ret] = prev_page_count; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif + } + + (void)reserved; + HANDLE_OP_END(); + } + + /* constant instructions */ + HANDLE_OP(WASM_OP_F64_CONST) + HANDLE_OP(WASM_OP_I64_CONST) + { + uint8 *orig_ip = frame_ip; + + frame_ip += sizeof(uint64); + addr_ret = GET_OFFSET(); + + bh_memcpy_s(frame_lp + addr_ret, sizeof(uint64), orig_ip, + sizeof(uint64)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONST) + HANDLE_OP(WASM_OP_I32_CONST) + { + uint8 *orig_ip = frame_ip; + + frame_ip += sizeof(uint32); + addr_ret = GET_OFFSET(); + + bh_memcpy_s(frame_lp + addr_ret, sizeof(uint32), orig_ip, + sizeof(uint32)); + HANDLE_OP_END(); + } + + /* comparison instructions of i32 */ + HANDLE_OP(WASM_OP_I32_EQZ) + { + DEF_OP_EQZ(int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EQ) + { + DEF_OP_CMP(uint32, I32, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_NE) + { + DEF_OP_CMP(uint32, I32, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LT_S) + { + DEF_OP_CMP(int32, I32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LT_U) + { + DEF_OP_CMP(uint32, I32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GT_S) + { + DEF_OP_CMP(int32, I32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GT_U) + { + DEF_OP_CMP(uint32, I32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LE_S) + { + DEF_OP_CMP(int32, I32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LE_U) + { + DEF_OP_CMP(uint32, I32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GE_S) + { + DEF_OP_CMP(int32, I32, >=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GE_U) + { + DEF_OP_CMP(uint32, I32, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of i64 */ + HANDLE_OP(WASM_OP_I64_EQZ) + { + DEF_OP_EQZ(int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EQ) + { + DEF_OP_CMP(uint64, I64, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_NE) + { + DEF_OP_CMP(uint64, I64, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LT_S) + { + DEF_OP_CMP(int64, I64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LT_U) + { + DEF_OP_CMP(uint64, I64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GT_S) + { + DEF_OP_CMP(int64, I64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GT_U) + { + DEF_OP_CMP(uint64, I64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LE_S) + { + DEF_OP_CMP(int64, I64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LE_U) + { + DEF_OP_CMP(uint64, I64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GE_S) + { + DEF_OP_CMP(int64, I64, >=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GE_U) + { + DEF_OP_CMP(uint64, I64, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of f32 */ + HANDLE_OP(WASM_OP_F32_EQ) + { + DEF_OP_CMP(float32, F32, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NE) + { + DEF_OP_CMP(float32, F32, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_LT) + { + DEF_OP_CMP(float32, F32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_GT) + { + DEF_OP_CMP(float32, F32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_LE) + { + DEF_OP_CMP(float32, F32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_GE) + { + DEF_OP_CMP(float32, F32, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of f64 */ + HANDLE_OP(WASM_OP_F64_EQ) + { + DEF_OP_CMP(float64, F64, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NE) + { + DEF_OP_CMP(float64, F64, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_LT) + { + DEF_OP_CMP(float64, F64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_GT) + { + DEF_OP_CMP(float64, F64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_LE) + { + DEF_OP_CMP(float64, F64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_GE) + { + DEF_OP_CMP(float64, F64, >=); + HANDLE_OP_END(); + } + + /* numberic instructions of i32 */ + HANDLE_OP(WASM_OP_I32_CLZ) + { + DEF_OP_BIT_COUNT(uint32, I32, clz32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_CTZ) + { + DEF_OP_BIT_COUNT(uint32, I32, ctz32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_POPCNT) + { + DEF_OP_BIT_COUNT(uint32, I32, popcount32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ADD) + { + DEF_OP_NUMERIC(uint32, uint32, I32, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SUB) + { + DEF_OP_NUMERIC(uint32, uint32, I32, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_MUL) + { + DEF_OP_NUMERIC(uint32, uint32, I32, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_DIV_S) + { + int32 a, b; + + b = frame_lp[GET_OFFSET()]; + a = frame_lp[GET_OFFSET()]; + addr_ret = GET_OFFSET(); + if (a == (int32)0x80000000 && b == -1) { + wasm_set_exception(module, "integer overflow"); + goto got_exception; + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_DIV_U) + { + uint32 a, b; + + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + b = (uint32)frame_lp[addr1]; + a = (uint32)frame_lp[addr2]; + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_REM_S) + { + int32 a, b; + + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + b = frame_lp[addr1]; + a = frame_lp[addr2]; + if (a == (int32)0x80000000 && b == -1) { + frame_lp[addr_ret] = 0; + HANDLE_OP_END(); + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_REM_U) + { + uint32 a, b; + + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + b = (uint32)frame_lp[addr1]; + a = (uint32)frame_lp[addr2]; + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_AND) + { + DEF_OP_NUMERIC(uint32, uint32, I32, &); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_OR) + { + DEF_OP_NUMERIC(uint32, uint32, I32, |); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_XOR) + { + DEF_OP_NUMERIC(uint32, uint32, I32, ^); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHL) + { + DEF_OP_NUMERIC2(uint32, uint32, I32, <<); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHR_S) + { + DEF_OP_NUMERIC2(int32, uint32, I32, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHR_U) + { + DEF_OP_NUMERIC2(uint32, uint32, I32, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ROTL) + { + uint32 a, b; + + b = (uint32)frame_lp[GET_OFFSET()]; + a = (uint32)frame_lp[GET_OFFSET()]; + frame_lp[GET_OFFSET()] = rotl32(a, b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ROTR) + { + uint32 a, b; + + b = (uint32)frame_lp[GET_OFFSET()]; + a = (uint32)frame_lp[GET_OFFSET()]; + frame_lp[GET_OFFSET()] = rotr32(a, b); + HANDLE_OP_END(); + } + + /* numberic instructions of i64 */ + HANDLE_OP(WASM_OP_I64_CLZ) + { + DEF_OP_BIT_COUNT(uint64, I64, clz64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_CTZ) + { + DEF_OP_BIT_COUNT(uint64, I64, ctz64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_POPCNT) + { + DEF_OP_BIT_COUNT(uint64, I64, popcount64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ADD) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SUB) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_MUL) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_DIV_S) + { + int64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (a == (int64)0x8000000000000000LL && b == -1) { + wasm_set_exception(module, "integer overflow"); + goto got_exception; + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_DIV_U) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REM_S) + { + int64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (a == (int64)0x8000000000000000LL && b == -1) { + *(int64 *)(frame_lp + GET_OFFSET()) = 0; + HANDLE_OP_END(); + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REM_U) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_AND) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, &); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_OR) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, |); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_XOR) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, ^); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHL) + { + DEF_OP_NUMERIC2_64(uint64, uint64, I64, <<); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHR_S) + { + DEF_OP_NUMERIC2_64(int64, uint64, I64, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHR_U) + { + DEF_OP_NUMERIC2_64(uint64, uint64, I64, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ROTL) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), rotl64(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ROTR) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), rotr64(a, b)); + HANDLE_OP_END(); + } + + /* numberic instructions of f32 */ + HANDLE_OP(WASM_OP_F32_ABS) + { + DEF_OP_MATH(float32, F32, fabsf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NEG) + { + uint32 u32 = frame_lp[GET_OFFSET()]; + uint32 sign_bit = u32 & ((uint32)1 << 31); + addr_ret = GET_OFFSET(); + if (sign_bit) + frame_lp[addr_ret] = u32 & ~((uint32)1 << 31); + else + frame_lp[addr_ret] = u32 | ((uint32)1 << 31); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CEIL) + { + DEF_OP_MATH(float32, F32, ceilf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_FLOOR) + { + DEF_OP_MATH(float32, F32, floorf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_TRUNC) + { + DEF_OP_MATH(float32, F32, truncf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NEAREST) + { + DEF_OP_MATH(float32, F32, rintf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_SQRT) + { + DEF_OP_MATH(float32, F32, sqrtf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_ADD) + { + DEF_OP_NUMERIC(float32, float32, F32, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_SUB) + { + DEF_OP_NUMERIC(float32, float32, F32, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MUL) + { + DEF_OP_NUMERIC(float32, float32, F32, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_DIV) + { + DEF_OP_NUMERIC(float32, float32, F32, /); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MIN) + { + float32 a, b; + + b = *(float32 *)(frame_lp + GET_OFFSET()); + a = *(float32 *)(frame_lp + GET_OFFSET()); + + *(float32 *)(frame_lp + GET_OFFSET()) = f32_min(a, b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MAX) + { + float32 a, b; + + b = *(float32 *)(frame_lp + GET_OFFSET()); + a = *(float32 *)(frame_lp + GET_OFFSET()); + + *(float32 *)(frame_lp + GET_OFFSET()) = f32_max(a, b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_COPYSIGN) + { + float32 a, b; + + b = *(float32 *)(frame_lp + GET_OFFSET()); + a = *(float32 *)(frame_lp + GET_OFFSET()); + *(float32 *)(frame_lp + GET_OFFSET()) = local_copysignf(a, b); + HANDLE_OP_END(); + } + + /* numberic instructions of f64 */ + HANDLE_OP(WASM_OP_F64_ABS) + { + DEF_OP_MATH(float64, F64, fabs); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NEG) + { + uint64 u64 = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + uint64 sign_bit = u64 & (((uint64)1) << 63); + if (sign_bit) + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), + (u64 & ~(((uint64)1) << 63))); + else + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), + (u64 | (((uint64)1) << 63))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CEIL) + { + DEF_OP_MATH(float64, F64, ceil); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_FLOOR) + { + DEF_OP_MATH(float64, F64, floor); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_TRUNC) + { + DEF_OP_MATH(float64, F64, trunc); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NEAREST) + { + DEF_OP_MATH(float64, F64, rint); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_SQRT) + { + DEF_OP_MATH(float64, F64, sqrt); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_ADD) + { + DEF_OP_NUMERIC_64(float64, float64, F64, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_SUB) + { + DEF_OP_NUMERIC_64(float64, float64, F64, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MUL) + { + DEF_OP_NUMERIC_64(float64, float64, F64, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_DIV) + { + DEF_OP_NUMERIC_64(float64, float64, F64, /); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MIN) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + + PUSH_F64(f64_min(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MAX) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + + PUSH_F64(f64_max(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_COPYSIGN) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + PUSH_F64(local_copysign(a, b)); + HANDLE_OP_END(); + } + + /* conversions of i32 */ + HANDLE_OP(WASM_OP_I32_WRAP_I64) + { + int32 value = (int32)(POP_I64() & 0xFFFFFFFFLL); + PUSH_I32(value); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_S_F32) + { + /* We don't use INT32_MIN/INT32_MAX/UINT32_MIN/UINT32_MAX, + since float/double values of ieee754 cannot precisely + represent all int32/uint32/int64/uint64 values, e.g.: + UINT32_MAX is 4294967295, but (float32)4294967295 is + 4294967296.0f, but not 4294967295.0f. */ + DEF_OP_TRUNC_F32(-2147483904.0f, 2147483648.0f, true, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_U_F32) + { + DEF_OP_TRUNC_F32(-1.0f, 4294967296.0f, true, false); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_S_F64) + { + DEF_OP_TRUNC_F64(-2147483649.0, 2147483648.0, true, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_U_F64) + { + DEF_OP_TRUNC_F64(-1.0, 4294967296.0, true, false); + HANDLE_OP_END(); + } + + /* conversions of i64 */ + HANDLE_OP(WASM_OP_I64_EXTEND_S_I32) + { + DEF_OP_CONVERT(int64, I64, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND_U_I32) + { + DEF_OP_CONVERT(int64, I64, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_S_F32) + { + DEF_OP_TRUNC_F32(-9223373136366403584.0f, + 9223372036854775808.0f, false, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_U_F32) + { + DEF_OP_TRUNC_F32(-1.0f, 18446744073709551616.0f, false, false); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_S_F64) + { + DEF_OP_TRUNC_F64(-9223372036854777856.0, 9223372036854775808.0, + false, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_U_F64) + { + DEF_OP_TRUNC_F64(-1.0, 18446744073709551616.0, false, false); + HANDLE_OP_END(); + } + + /* conversions of f32 */ + HANDLE_OP(WASM_OP_F32_CONVERT_S_I32) + { + DEF_OP_CONVERT(float32, F32, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_U_I32) + { + DEF_OP_CONVERT(float32, F32, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_S_I64) + { + DEF_OP_CONVERT(float32, F32, int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_U_I64) + { + DEF_OP_CONVERT(float32, F32, uint64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_DEMOTE_F64) + { + DEF_OP_CONVERT(float32, F32, float64, F64); + HANDLE_OP_END(); + } + + /* conversions of f64 */ + HANDLE_OP(WASM_OP_F64_CONVERT_S_I32) + { + DEF_OP_CONVERT(float64, F64, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_U_I32) + { + DEF_OP_CONVERT(float64, F64, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_S_I64) + { + DEF_OP_CONVERT(float64, F64, int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_U_I64) + { + DEF_OP_CONVERT(float64, F64, uint64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_PROMOTE_F32) + { + DEF_OP_CONVERT(float64, F64, float32, F32); + HANDLE_OP_END(); + } + + /* reinterpretations */ + HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) + HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) + { + DEF_OP_REINTERPRET(uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) + { + DEF_OP_REINTERPRET(int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_COPY_STACK_TOP) + { + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + frame_lp[addr2] = frame_lp[addr1]; + +#if WASM_ENABLE_GC != 0 + /* Ignore constants because they are not reference */ + if (addr1 >= 0) { + if (*FRAME_REF(addr1)) { + CLEAR_FRAME_REF(addr1); + SET_FRAME_REF(addr2); + } + } +#endif + + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_COPY_STACK_TOP_I64) + { + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + + PUT_I64_TO_ADDR(frame_lp + addr2, + GET_I64_FROM_ADDR(frame_lp + addr1)); + +#if WASM_ENABLE_GC != 0 + /* Ignore constants because they are not reference */ + if (addr1 >= 0) { + if (*FRAME_REF(addr1)) { + CLEAR_FRAME_REF(addr1); + SET_FRAME_REF(addr2); + } + } +#endif + + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_COPY_STACK_VALUES) + { + uint32 values_count, total_cell; + uint8 *cells; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + + /* read values_count */ + values_count = read_uint32(frame_ip); + /* read total cell num */ + total_cell = read_uint32(frame_ip); + /* cells */ + cells = (uint8 *)frame_ip; + frame_ip += values_count * CELL_SIZE; + /* src offsets */ + src_offsets = (int16 *)frame_ip; + frame_ip += values_count * sizeof(int16); + /* dst offsets */ + dst_offsets = (uint16 *)frame_ip; + frame_ip += values_count * sizeof(uint16); + + if (!copy_stack_values(module, frame_lp, values_count, +#if WASM_ENABLE_GC != 0 + frame_ref, +#endif + total_cell, cells, src_offsets, + dst_offsets)) + goto got_exception; + + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_LOCAL) + { + opcode = WASM_OP_SET_LOCAL; + goto handle_op_set_tee_local; + } + HANDLE_OP(WASM_OP_TEE_LOCAL) + { + opcode = WASM_OP_TEE_LOCAL; + handle_op_set_tee_local: + + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + addr1 = GET_OFFSET(); + + if (local_type == VALUE_TYPE_I32 + || local_type == VALUE_TYPE_F32) { + *(int32 *)(frame_lp + local_offset) = frame_lp[addr1]; + } + else if (local_type == VALUE_TYPE_I64 + || local_type == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_I64_FROM_ADDR(frame_lp + addr1)); + } +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(local_type)) { + PUT_REF_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_REF_FROM_ADDR(frame_lp + addr1)); + if (opcode == WASM_OP_SET_LOCAL) { + CLEAR_FRAME_REF(addr1); + } + } +#endif + else { + wasm_set_exception(module, "invalid local type"); + goto got_exception; + } + + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EXTEND8_S) + { + DEF_OP_CONVERT(int32, I32, int8, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EXTEND16_S) + { + DEF_OP_CONVERT(int32, I32, int16, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND8_S) + { + DEF_OP_CONVERT(int64, I64, int8, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND16_S) + { + DEF_OP_CONVERT(int64, I64, int16, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND32_S) + { + DEF_OP_CONVERT(int64, I64, int32, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_MISC_PREFIX) + { + GET_OPCODE(); + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + DEF_OP_TRUNC_SAT_F32(-2147483904.0f, 2147483648.0f, + true, true); + break; + case WASM_OP_I32_TRUNC_SAT_U_F32: + DEF_OP_TRUNC_SAT_F32(-1.0f, 4294967296.0f, true, false); + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + DEF_OP_TRUNC_SAT_F64(-2147483649.0, 2147483648.0, true, + true); + break; + case WASM_OP_I32_TRUNC_SAT_U_F64: + DEF_OP_TRUNC_SAT_F64(-1.0, 4294967296.0, true, false); + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + DEF_OP_TRUNC_SAT_F32(-9223373136366403584.0f, + 9223372036854775808.0f, false, + true); + break; + case WASM_OP_I64_TRUNC_SAT_U_F32: + DEF_OP_TRUNC_SAT_F32(-1.0f, 18446744073709551616.0f, + false, false); + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + DEF_OP_TRUNC_SAT_F64(-9223372036854777856.0, + 9223372036854775808.0, false, + true); + break; + case WASM_OP_I64_TRUNC_SAT_U_F64: + DEF_OP_TRUNC_SAT_F64(-1.0, 18446744073709551616.0, + false, false); + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + uint32 addr, segment; + uint64 bytes, offset, seg_len; + uint8 *data; + + segment = read_uint32(frame_ip); + + bytes = (uint64)(uint32)POP_I32(); + offset = (uint64)(uint32)POP_I32(); + addr = POP_I32(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); +#else + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; +#endif + if (bh_bitmap_get_bit(module->e->common.data_dropped, + segment)) { + seg_len = 0; + data = NULL; + } + else { + seg_len = + (uint64)module->module->data_segments[segment] + ->data_length; + data = module->module->data_segments[segment]->data; + } + if (offset + bytes > seg_len) + goto out_of_bounds; + + bh_memcpy_s(maddr, (uint32)(linear_mem_size - addr), + data + offset, (uint32)bytes); + break; + } + case WASM_OP_DATA_DROP: + { + uint32 segment; + + segment = read_uint32(frame_ip); + bh_bitmap_set_bit(module->e->common.data_dropped, + segment); + break; + } + case WASM_OP_MEMORY_COPY: + { + uint32 dst, src, len; + uint8 *mdst, *msrc; + + len = POP_I32(); + src = POP_I32(); + dst = POP_I32(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); + CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); +#else + if ((uint64)(uint32)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + (uint32)src; + + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; +#endif + + /* allowing the destination and source to overlap */ + bh_memmove_s(mdst, (uint32)(linear_mem_size - dst), + msrc, len); + break; + } + case WASM_OP_MEMORY_FILL: + { + uint32 dst, len; + uint8 fill_val, *mdst; + + len = POP_I32(); + fill_val = POP_I32(); + dst = POP_I32(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); +#else + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; +#endif + + memset(mdst, fill_val, len); + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_TABLE_INIT: + { + uint32 tbl_idx, elem_idx; + uint32 n, s, d; + WASMTableInstance *tbl_inst; + table_elem_type_t *table_elems; + InitializerExpression *tbl_seg_init_values = NULL, + *init_values; + uint64 i; + uint32 tbl_seg_len = 0; + + elem_idx = read_uint32(frame_ip); + bh_assert(elem_idx < module->module->table_seg_count); + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + n = (uint32)POP_I32(); + s = (uint32)POP_I32(); + d = (uint32)POP_I32(); + + if (!bh_bitmap_get_bit(module->e->common.elem_dropped, + elem_idx)) { + /* table segment isn't dropped */ + tbl_seg_init_values = + module->module->table_segments[elem_idx] + .init_values; + tbl_seg_len = + module->module->table_segments[elem_idx] + .value_count; + } + + if (offset_len_out_of_bounds(s, n, tbl_seg_len) + || offset_len_out_of_bounds(d, n, + tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + if (!n) { + break; + } + + table_elems = tbl_inst->elems + d; + init_values = tbl_seg_init_values + s; +#if WASM_ENABLE_GC != 0 + SYNC_ALL_TO_FRAME(); +#endif + for (i = 0; i < n; i++) { + /* UINT32_MAX indicates that it is a null ref */ + bh_assert(init_values[i].init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST + || init_values[i].init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST); +#if WASM_ENABLE_GC == 0 + table_elems[i] = + (table_elem_type_t)init_values[i].u.ref_index; +#else + if (init_values[i].u.ref_index != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module, init_values[i].u.ref_index, + true, NULL, 0))) { + goto got_exception; + } + table_elems[i] = func_obj; + } + else { + table_elems[i] = NULL_REF; + } +#endif + } + + break; + } + case WASM_OP_ELEM_DROP: + { + uint32 elem_idx = read_uint32(frame_ip); + bh_assert(elem_idx < module->module->table_seg_count); + bh_bitmap_set_bit(module->e->common.elem_dropped, + elem_idx); + break; + } + case WASM_OP_TABLE_COPY: + { + uint32 src_tbl_idx, dst_tbl_idx; + uint32 n, s, d; + WASMTableInstance *src_tbl_inst, *dst_tbl_inst; + + dst_tbl_idx = read_uint32(frame_ip); + bh_assert(dst_tbl_idx < module->table_count); + + dst_tbl_inst = wasm_get_table_inst(module, dst_tbl_idx); + + src_tbl_idx = read_uint32(frame_ip); + bh_assert(src_tbl_idx < module->table_count); + + src_tbl_inst = wasm_get_table_inst(module, src_tbl_idx); + + n = (uint32)POP_I32(); + s = (uint32)POP_I32(); + d = (uint32)POP_I32(); + + if (offset_len_out_of_bounds(d, n, + dst_tbl_inst->cur_size) + || offset_len_out_of_bounds( + s, n, src_tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + /* if s >= d, copy from front to back */ + /* if s < d, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + + offsetof(WASMTableInstance, elems) + + d * sizeof(table_elem_type_t), + (uint32)((dst_tbl_inst->cur_size - d) + * sizeof(table_elem_type_t)), + (uint8 *)src_tbl_inst + + offsetof(WASMTableInstance, elems) + + s * sizeof(table_elem_type_t), + (uint32)(n * sizeof(table_elem_type_t))); + break; + } + case WASM_OP_TABLE_GROW: + { + uint32 tbl_idx, n, orig_tbl_sz; + WASMTableInstance *tbl_inst; + table_elem_type_t init_val; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + orig_tbl_sz = tbl_inst->cur_size; + + n = POP_I32(); +#if WASM_ENABLE_GC == 0 + init_val = POP_I32(); +#else + init_val = POP_REF(); +#endif + + if (!wasm_enlarge_table(module, tbl_idx, n, init_val)) { + PUSH_I32(-1); + } + else { + PUSH_I32(orig_tbl_sz); + } + + break; + } + case WASM_OP_TABLE_SIZE: + { + uint32 tbl_idx; + WASMTableInstance *tbl_inst; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + PUSH_I32(tbl_inst->cur_size); + break; + } + case WASM_OP_TABLE_FILL: + { + uint32 tbl_idx, n, i; + WASMTableInstance *tbl_inst; + table_elem_type_t fill_val; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + n = POP_I32(); +#if WASM_ENABLE_GC == 0 + fill_val = POP_I32(); +#else + fill_val = POP_REF(); +#endif + i = POP_I32(); + + if (offset_len_out_of_bounds(i, n, + tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + for (; n != 0; i++, n--) { + tbl_inst->elems[i] = fill_val; + } + + break; + } +#endif /* WASM_ENABLE_REF_TYPES */ + default: + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + HANDLE_OP(WASM_OP_ATOMIC_PREFIX) + { + uint32 offset = 0, addr; + + GET_OPCODE(); + + if (opcode != WASM_OP_ATOMIC_FENCE) { + offset = read_uint32(frame_ip); + } + + switch (opcode) { + case WASM_OP_ATOMIC_NOTIFY: + { + uint32 notify_count, ret; + + notify_count = POP_I32(); + addr = POP_I32(); + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + ret = wasm_runtime_atomic_notify( + (WASMModuleInstanceCommon *)module, maddr, + notify_count); + if (ret == (uint32)-1) + goto got_exception; + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT32: + { + uint64 timeout; + uint32 expect, ret; + + timeout = POP_I64(); + expect = POP_I32(); + addr = POP_I32(); + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + ret = wasm_runtime_atomic_wait( + (WASMModuleInstanceCommon *)module, maddr, + (uint64)expect, timeout, false); + if (ret == (uint32)-1) + goto got_exception; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT64: + { + uint64 timeout, expect; + uint32 ret; + + timeout = POP_I64(); + expect = POP_I64(); + addr = POP_I32(); + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + + ret = wasm_runtime_atomic_wait( + (WASMModuleInstanceCommon *)module, maddr, expect, + timeout, true); + if (ret == (uint32)-1) + goto got_exception; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_FENCE: + { + os_atomic_thread_fence(os_memory_order_seq_cst); + break; + } + + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + { + uint32 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + readv = (uint32)(*(uint8 *)maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + readv = (uint32)LOAD_U16(maddr); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + readv = LOAD_I32(maddr); + shared_memory_unlock(memory); + } + + PUSH_I32(readv); + break; + } + + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + { + uint64 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + readv = (uint64)(*(uint8 *)maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + readv = (uint64)LOAD_U16(maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + readv = (uint64)LOAD_U32(maddr); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + shared_memory_lock(memory); + readv = LOAD_I64(maddr); + shared_memory_unlock(memory); + } + + PUSH_I64(readv); + break; + } + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + { + uint32 sval; + + sval = (uint32)POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + *(uint8 *)maddr = (uint8)sval; + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + STORE_U16(maddr, (uint16)sval); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + STORE_U32(maddr, sval); + shared_memory_unlock(memory); + } + break; + } + + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + { + uint64 sval; + + sval = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + *(uint8 *)maddr = (uint8)sval; + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + STORE_U16(maddr, (uint16)sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + STORE_U32(maddr, (uint32)sval); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + shared_memory_lock(memory); + STORE_I64(maddr, sval); + shared_memory_unlock(memory); + } + break; + } + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + { + uint32 readv, sval, expect; + + sval = POP_I32(); + expect = POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + + expect = (uint8)expect; + shared_memory_lock(memory); + readv = (uint32)(*(uint8 *)maddr); + if (readv == expect) + *(uint8 *)maddr = (uint8)(sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + + expect = (uint16)expect; + shared_memory_lock(memory); + readv = (uint32)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + shared_memory_lock(memory); + readv = LOAD_I32(maddr); + if (readv == expect) + STORE_U32(maddr, sval); + shared_memory_unlock(memory); + } + PUSH_I32(readv); + break; + } + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + { + uint64 readv, sval, expect; + + sval = (uint64)POP_I64(); + expect = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + + expect = (uint8)expect; + shared_memory_lock(memory); + readv = (uint64)(*(uint8 *)maddr); + if (readv == expect) + *(uint8 *)maddr = (uint8)(sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + + expect = (uint16)expect; + shared_memory_lock(memory); + readv = (uint64)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + expect = (uint32)expect; + shared_memory_lock(memory); + readv = (uint64)LOAD_U32(maddr); + if (readv == expect) + STORE_U32(maddr, (uint32)(sval)); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + + shared_memory_lock(memory); + readv = (uint64)LOAD_I64(maddr); + if (readv == expect) + STORE_I64(maddr, sval); + shared_memory_unlock(memory); + } + PUSH_I64(readv); + break; + } + + DEF_ATOMIC_RMW_OPCODE(ADD, +); + DEF_ATOMIC_RMW_OPCODE(SUB, -); + DEF_ATOMIC_RMW_OPCODE(AND, &); + DEF_ATOMIC_RMW_OPCODE(OR, |); + DEF_ATOMIC_RMW_OPCODE(XOR, ^); + /* xchg, ignore the read value, and store the given + value: readv * 0 + sval */ + DEF_ATOMIC_RMW_OPCODE(XCHG, *0 +); + } + + HANDLE_OP_END(); + } +#endif + + HANDLE_OP(WASM_OP_IMPDEP) + { + frame = prev_frame; + frame_ip = frame->ip; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif + goto call_func_from_entry; + } + + HANDLE_OP(WASM_OP_CALL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + fidx = read_uint32(frame_ip); +#if WASM_ENABLE_MULTI_MODULE != 0 + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } +#endif + cur_func = module->e->functions + fidx; + goto call_func_from_interp; + } + +#if WASM_ENABLE_TAIL_CALL != 0 + HANDLE_OP(WASM_OP_RETURN_CALL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + fidx = read_uint32(frame_ip); +#if WASM_ENABLE_MULTI_MODULE != 0 + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } +#endif + cur_func = module->e->functions + fidx; + goto call_func_from_return_call; + } +#endif /* WASM_ENABLE_TAIL_CALL */ + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + default: + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 + HANDLE_OP(WASM_OP_UNUSED_0x0a) +#if WASM_ENABLE_TAIL_CALL == 0 + HANDLE_OP(WASM_OP_RETURN_CALL) + HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT) +#endif +#if WASM_ENABLE_SHARED_MEMORY == 0 + HANDLE_OP(WASM_OP_ATOMIC_PREFIX) +#endif +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + HANDLE_OP(WASM_OP_TABLE_GET) + HANDLE_OP(WASM_OP_TABLE_SET) + HANDLE_OP(WASM_OP_REF_NULL) + HANDLE_OP(WASM_OP_REF_IS_NULL) + HANDLE_OP(WASM_OP_REF_FUNC) +#endif +#if WASM_ENABLE_GC == 0 + /* SELECT_T is converted to SELECT or SELECT_64 */ + HANDLE_OP(WASM_OP_SELECT_T) + HANDLE_OP(WASM_OP_CALL_REF) + HANDLE_OP(WASM_OP_RETURN_CALL_REF) + HANDLE_OP(WASM_OP_REF_EQ) + HANDLE_OP(WASM_OP_REF_AS_NON_NULL) + HANDLE_OP(WASM_OP_BR_ON_NULL) + HANDLE_OP(WASM_OP_BR_ON_NON_NULL) + HANDLE_OP(WASM_OP_GC_PREFIX) +#endif +#if WASM_ENABLE_EXCE_HANDLING == 0 + /* if exception handling is disabled, these opcodes issue a trap */ + HANDLE_OP(WASM_OP_TRY) + HANDLE_OP(WASM_OP_CATCH) + HANDLE_OP(WASM_OP_THROW) + HANDLE_OP(WASM_OP_RETHROW) + HANDLE_OP(WASM_OP_DELEGATE) + HANDLE_OP(WASM_OP_CATCH_ALL) + HANDLE_OP(EXT_OP_TRY) +#endif + HANDLE_OP(WASM_OP_UNUSED_0x16) + HANDLE_OP(WASM_OP_UNUSED_0x17) + HANDLE_OP(WASM_OP_UNUSED_0x27) + /* optimized op code */ + HANDLE_OP(WASM_OP_F32_STORE) + HANDLE_OP(WASM_OP_F64_STORE) + HANDLE_OP(WASM_OP_F32_LOAD) + HANDLE_OP(WASM_OP_F64_LOAD) + HANDLE_OP(EXT_OP_GET_LOCAL_FAST) + HANDLE_OP(WASM_OP_GET_LOCAL) + HANDLE_OP(WASM_OP_DROP) + HANDLE_OP(WASM_OP_DROP_64) + HANDLE_OP(WASM_OP_BLOCK) + HANDLE_OP(WASM_OP_LOOP) + HANDLE_OP(WASM_OP_END) + HANDLE_OP(WASM_OP_NOP) + HANDLE_OP(EXT_OP_BLOCK) + HANDLE_OP(EXT_OP_LOOP) + HANDLE_OP(EXT_OP_IF) + HANDLE_OP(EXT_OP_BR_TABLE_CACHE) + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + continue; +#else + FETCH_OPCODE_AND_DISPATCH(); +#endif + +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + call_func_from_return_call: + { + uint32 *lp_base = NULL, *lp = NULL; + int i; + + if (cur_func->param_cell_num > 0 + && !(lp_base = lp = wasm_runtime_malloc(cur_func->param_cell_num + * sizeof(uint32)))) { + wasm_set_exception(module, "allocate memory failed"); + goto got_exception; + } + for (i = 0; i < cur_func->param_count; i++) { + if (cur_func->param_types[i] == VALUE_TYPE_I64 + || cur_func->param_types[i] == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR( + lp, GET_OPERAND(uint64, I64, + 2 * (cur_func->param_count - i - 1))); + lp += 2; + } + else { + *lp = GET_OPERAND(uint32, I32, + (2 * (cur_func->param_count - i - 1))); + lp++; + } + } + frame->lp = frame->operand + cur_func->const_cell_num; + if (lp - lp_base > 0) { + word_copy(frame->lp, lp_base, lp - lp_base); + } + if (lp_base) + wasm_runtime_free(lp_base); + FREE_FRAME(exec_env, frame); + frame_ip += cur_func->param_count * sizeof(int16); + wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame *)prev_frame); + is_return_call = true; + goto call_func_from_entry; + } +#endif /* WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 */ + + call_func_from_interp: + { + /* Only do the copy when it's called from interpreter. */ + WASMInterpFrame *outs_area = wasm_exec_env_wasm_stack_top(exec_env); + int i; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (cur_func->is_import_func) { + outs_area->lp = outs_area->operand + + (cur_func->import_func_inst + ? cur_func->import_func_inst->const_cell_num + : 0); + } + else +#endif + { + outs_area->lp = outs_area->operand + cur_func->const_cell_num; + } + + if ((uint8 *)(outs_area->lp + cur_func->param_cell_num) + > exec_env->wasm_stack.top_boundary) { + wasm_set_exception(module, "wasm operand stack overflow"); + goto got_exception; + } + + for (i = 0; i < cur_func->param_count; i++) { + if (cur_func->param_types[i] == VALUE_TYPE_I64 + || cur_func->param_types[i] == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR( + outs_area->lp, + GET_OPERAND(uint64, I64, + 2 * (cur_func->param_count - i - 1))); + outs_area->lp += 2; + } +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(cur_func->param_types[i])) { + PUT_REF_TO_ADDR( + outs_area->lp, + GET_OPERAND(void *, REF, + 2 * (cur_func->param_count - i - 1))); + CLEAR_FRAME_REF( + *(uint16 *)(frame_ip + + (2 * (cur_func->param_count - i - 1)))); + outs_area->lp += REF_CELL_NUM; + } +#endif + else { + *outs_area->lp = GET_OPERAND( + uint32, I32, (2 * (cur_func->param_count - i - 1))); + outs_area->lp++; + } + } + frame_ip += cur_func->param_count * sizeof(int16); + if (cur_func->ret_cell_num != 0) { + /* Get the first return value's offset. Since loader emit + * all return values' offset so we must skip remain return + * values' offsets. + */ + WASMFuncType *func_type; + if (cur_func->is_import_func) + func_type = cur_func->u.func_import->func_type; + else + func_type = cur_func->u.func->func_type; + frame->ret_offset = GET_OFFSET(); + frame_ip += 2 * (func_type->result_count - 1); + } + SYNC_ALL_TO_FRAME(); + prev_frame = frame; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif + } + + call_func_from_entry: + { + if (cur_func->is_import_func) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (cur_func->import_func_inst) { + wasm_interp_call_func_import(module, exec_env, cur_func, + prev_frame); + } + else +#endif + { + wasm_interp_call_func_native(module, exec_env, cur_func, + prev_frame); + } + +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (is_return_call) { + /* the frame was freed before tail calling and + the prev_frame was set as exec_env's cur_frame, + so here we recover context from prev_frame */ + RECOVER_CONTEXT(prev_frame); + } + else +#endif + { + prev_frame = frame->prev_frame; + cur_func = frame->function; + UPDATE_ALL_FROM_FRAME(); + } + + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + if (memory) + linear_mem_size = get_linear_mem_size(); +#endif + if (wasm_copy_exception(module, NULL)) + goto got_exception; + } + else { + WASMFunction *cur_wasm_func = cur_func->u.func; + uint32 cell_num_of_local_stack; + + cell_num_of_local_stack = cur_func->param_cell_num + + cur_func->local_cell_num + + cur_wasm_func->max_stack_cell_num; + all_cell_num = cur_func->const_cell_num + cell_num_of_local_stack; +#if WASM_ENABLE_GC != 0 + /* area of frame_ref */ + all_cell_num += (cell_num_of_local_stack + 3) / 4; + /* cells occupied by locals, POP_REF should not clear frame_ref for + * these cells */ + local_cell_num = + cur_func->param_cell_num + cur_func->local_cell_num; +#endif + /* param_cell_num, local_cell_num, const_cell_num and + max_stack_cell_num are all no larger than UINT16_MAX (checked + in loader), all_cell_num must be smaller than 1MB */ + bh_assert(all_cell_num < 1 * BH_MB); + + frame_size = wasm_interp_interp_frame_size(all_cell_num); + if (!(frame = ALLOC_FRAME(exec_env, frame_size, prev_frame))) { + frame = prev_frame; + goto got_exception; + } + + /* Initialize the interpreter context. */ + frame->function = cur_func; + frame_ip = wasm_get_func_code(cur_func); + frame_ip_end = wasm_get_func_code_end(cur_func); + + frame_lp = frame->lp = + frame->operand + cur_wasm_func->const_cell_num; + + /* Initialize the consts */ + if (cur_wasm_func->const_cell_num > 0) { + word_copy(frame->operand, (uint32 *)cur_wasm_func->consts, + cur_wasm_func->const_cell_num); + } + + /* Initialize the local variables */ + memset(frame_lp + cur_func->param_cell_num, 0, + (uint32)(cur_func->local_cell_num * 4)); + +#if WASM_ENABLE_GC != 0 + /* frame->ip is used during GC root set enumeration, so we must + * initialized this field here */ + frame->ip = frame_ip; + frame_ref = frame->frame_ref = + (uint8 *)(frame->lp + (uint32)cell_num_of_local_stack); + init_frame_refs(frame_ref, (uint32)cell_num_of_local_stack, + cur_func); +#endif + + wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame *)frame); + } +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + HANDLE_OP_END(); + } + + return_func: + { + FREE_FRAME(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame *)prev_frame); + + if (!prev_frame->ip) + /* Called from native. */ + return; + + RECOVER_CONTEXT(prev_frame); +#if WASM_ENABLE_GC != 0 + local_cell_num = cur_func->param_cell_num + cur_func->local_cell_num; +#endif + HANDLE_OP_END(); + } + + (void)frame_ip_end; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + unaligned_atomic: + wasm_set_exception(module, "unaligned atomic"); + goto got_exception; +#endif + +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + out_of_bounds: + wasm_set_exception(module, "out of bounds memory access"); +#endif + + got_exception: + SYNC_ALL_TO_FRAME(); + return; + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + } +#else + FETCH_OPCODE_AND_DISPATCH(); +#endif +} + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +void ** +wasm_interp_get_handle_table() +{ + WASMModuleInstance module; + memset(&module, 0, sizeof(WASMModuleInstance)); + wasm_interp_call_func_bytecode(&module, NULL, NULL, NULL); + return global_handle_table; +} +#endif + +#if WASM_ENABLE_GC != 0 +bool +wasm_interp_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMInterpFrame *frame; + WASMObjectRef gc_obj; + WASMFunctionInstance *cur_func; + uint8 *frame_ref; + uint32 local_cell_num, i; + + frame = wasm_exec_env_get_cur_frame(exec_env); + for (; frame; frame = frame->prev_frame) { + frame_ref = frame->frame_ref; + cur_func = frame->function; + + if (!cur_func) + continue; + + local_cell_num = cur_func->param_cell_num; + if (frame->ip) + local_cell_num += + cur_func->local_cell_num + cur_func->u.func->max_stack_cell_num; + + for (i = 0; i < local_cell_num; i++) { + if (frame_ref[i]) { + gc_obj = GET_REF_FROM_ADDR(frame->lp + i); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) { + return false; + } + } +#if UINTPTR_MAX == UINT64_MAX + bh_assert(frame_ref[i + 1]); + i++; +#endif + } + } + } + return true; +} +#endif + +void +wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, + WASMFunctionInstance *function, uint32 argc, + uint32 argv[]) +{ + WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env); + WASMInterpFrame *frame, *outs_area; + + /* Allocate sufficient cells for all kinds of return values. */ + unsigned all_cell_num = + function->ret_cell_num > 2 ? function->ret_cell_num : 2, + i; + /* This frame won't be used by JITed code, so only allocate interp + frame here. */ + unsigned frame_size; + +#if WASM_ENABLE_GC != 0 + all_cell_num += (all_cell_num + 3) / 4; +#endif + + frame_size = wasm_interp_interp_frame_size(all_cell_num); + + if (argc < function->param_cell_num) { + char buf[128]; + snprintf(buf, sizeof(buf), + "invalid argument count %" PRIu32 + ", must be no smaller than %" PRIu32, + argc, (uint32)function->param_cell_num); + wasm_set_exception(module_inst, buf); + return; + } + argc = function->param_cell_num; + + RECORD_STACK_USAGE(exec_env, (uint8 *)&prev_frame); +#if !(defined(OS_ENABLE_HW_BOUND_CHECK) \ + && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0) + if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) { + wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, + "native stack overflow"); + return; + } +#endif + + if (!(frame = + ALLOC_FRAME(exec_env, frame_size, (WASMInterpFrame *)prev_frame))) + return; + + outs_area = wasm_exec_env_wasm_stack_top(exec_env); + frame->function = NULL; + frame->ip = NULL; + /* There is no local variable. */ + frame->lp = frame->operand + 0; +#if WASM_ENABLE_GC != 0 + frame->frame_ref = + (uint8 *)(frame->lp + + (function->ret_cell_num > 2 ? function->ret_cell_num : 2)); +#endif + frame->ret_offset = 0; + + if ((uint8 *)(outs_area->operand + function->const_cell_num + argc) + > exec_env->wasm_stack.top_boundary) { + wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, + "wasm operand stack overflow"); + return; + } + + if (argc > 0) + word_copy(outs_area->operand + function->const_cell_num, argv, argc); + + wasm_exec_env_set_cur_frame(exec_env, frame); + +#if defined(os_writegsbase) + { + WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_inst->memory_data); + } +#endif + + if (function->is_import_func) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (function->import_module_inst) { + LOG_DEBUG("it is a function of a sub module"); + wasm_interp_call_func_import(module_inst, exec_env, function, + frame); + } + else +#endif + { + LOG_DEBUG("it is an native function"); + wasm_interp_call_func_native(module_inst, exec_env, function, + frame); + } + } + else { + wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame); + } + + /* Output the return value to the caller */ + if (!wasm_copy_exception(module_inst, NULL)) { + for (i = 0; i < function->ret_cell_num; i++) + argv[i] = *(frame->lp + i); + } + else { +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (wasm_interp_create_call_stack(exec_env)) { + wasm_interp_dump_call_stack(exec_env, true, NULL, 0); + } +#endif + } + + wasm_exec_env_set_cur_frame(exec_env, prev_frame); + FREE_FRAME(exec_env, frame); +#if WASM_ENABLE_OPCODE_COUNTER != 0 + wasm_interp_dump_op_count(); +#endif +} diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_loader.c b/wasm-micro-runtime/core/iwasm/interpreter/wasm_loader.c new file mode 100644 index 0000000..9aab9fe --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_loader.c @@ -0,0 +1,15643 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_loader.h" +#include "bh_common.h" +#include "bh_log.h" +#include "wasm.h" +#include "wasm_opcode.h" +#include "wasm_runtime.h" +#include "../common/wasm_native.h" +#include "../common/wasm_memory.h" +#if WASM_ENABLE_GC != 0 +#include "../common/gc/gc_type.h" +#include "../common/gc/gc_object.h" +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "../libraries/debug-engine/debug_engine.h" +#endif +#if WASM_ENABLE_FAST_JIT != 0 +#include "../fast-jit/jit_compiler.h" +#include "../fast-jit/jit_codecache.h" +#endif +#if WASM_ENABLE_JIT != 0 +#include "../compilation/aot_llvm.h" +#endif + +#ifndef TRACE_WASM_LOADER +#define TRACE_WASM_LOADER 0 +#endif + +/* Read a value of given type from the address pointed to by the given + pointer and increase the pointer to the position just after the + value being read. */ +#define TEMPLATE_READ_VALUE(Type, p) \ + (p += sizeof(Type), *(Type *)(p - sizeof(Type))) + +#if WASM_ENABLE_MEMORY64 != 0 +static bool +has_module_memory64(WASMModule *module) +{ + /* TODO: multi-memories for now assuming the memory idx type is consistent + * across multi-memories */ + if (module->import_memory_count > 0) + return !!(module->import_memories[0].u.memory.flags & MEMORY64_FLAG); + else if (module->memory_count > 0) + return !!(module->memories[0].flags & MEMORY64_FLAG); + + return false; +} +#endif + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, "WASM module load failed: %s", + string); + } +} + +#if WASM_ENABLE_MEMORY64 != 0 +static void +set_error_buf_mem_offset_out_of_range(char *error_buf, uint32 error_buf_size) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, "offset out of range"); + } +} +#endif + +static void +set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + snprintf(error_buf, error_buf_size, "WASM module load failed: %s", buf); + } +} + +static bool +check_buf(const uint8 *buf, const uint8 *buf_end, uint32 length, + char *error_buf, uint32 error_buf_size) +{ + if ((uintptr_t)buf + length < (uintptr_t)buf + || (uintptr_t)buf + length > (uintptr_t)buf_end) { + set_error_buf(error_buf, error_buf_size, + "unexpected end of section or function"); + return false; + } + return true; +} + +static bool +check_buf1(const uint8 *buf, const uint8 *buf_end, uint32 length, + char *error_buf, uint32 error_buf_size) +{ + if ((uintptr_t)buf + length < (uintptr_t)buf + || (uintptr_t)buf + length > (uintptr_t)buf_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + return true; +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if (!check_buf(buf, buf_end, length, error_buf, error_buf_size)) { \ + goto fail; \ + } \ + } while (0) + +#define CHECK_BUF1(buf, buf_end, length) \ + do { \ + if (!check_buf1(buf, buf_end, length, error_buf, error_buf_size)) { \ + goto fail; \ + } \ + } while (0) + +#define skip_leb(p) while (*p++ & 0x80) +#define skip_leb_int64(p, p_end) skip_leb(p) +#define skip_leb_uint32(p, p_end) skip_leb(p) +#define skip_leb_int32(p, p_end) skip_leb(p) +#define skip_leb_mem_offset(p, p_end) skip_leb(p) + +static bool +read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, + uint64 *p_result, char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + uint64 result = 0; + uint32 shift = 0; + uint32 offset = 0, bcnt = 0; + uint64 byte; + + while (true) { + /* uN or SN must not exceed ceil(N/7) bytes */ + if (bcnt + 1 > (maxbits + 6) / 7) { + set_error_buf(error_buf, error_buf_size, + "integer representation too long"); + return false; + } + + CHECK_BUF(buf, buf_end, offset + 1); + byte = buf[offset]; + offset += 1; + result |= ((byte & 0x7f) << shift); + shift += 7; + bcnt += 1; + if ((byte & 0x80) == 0) { + break; + } + } + + if (!sign && maxbits == 32 && shift >= maxbits) { + /* The top bits set represent values > 32 bits */ + if (((uint8)byte) & 0xf0) + goto fail_integer_too_large; + } + else if (sign && maxbits == 32) { + if (shift < maxbits) { + /* Sign extend, second-highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x8; + int top_bits = ((uint8)byte) & 0xf0; + if ((sign_bit_set && top_bits != 0x70) + || (!sign_bit_set && top_bits != 0)) + goto fail_integer_too_large; + } + } + else if (sign && maxbits == 64) { + if (shift < maxbits) { + /* Sign extend, second-highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x1; + int top_bits = ((uint8)byte) & 0xfe; + + if ((sign_bit_set && top_bits != 0x7e) + || (!sign_bit_set && top_bits != 0)) + goto fail_integer_too_large; + } + } + + *p_buf += offset; + *p_result = result; + return true; + +fail_integer_too_large: + set_error_buf(error_buf, error_buf_size, "integer too large"); +fail: + return false; +} + +#define read_uint8(p) TEMPLATE_READ_VALUE(uint8, p) +#define read_uint32(p) TEMPLATE_READ_VALUE(uint32, p) + +#define read_leb_int64(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, 64, true, &res64, error_buf, \ + error_buf_size)) \ + goto fail; \ + res = (int64)res64; \ + } while (0) + +#if WASM_ENABLE_MEMORY64 != 0 +#define read_leb_mem_offset(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, \ + &res64, error_buf, error_buf_size)) { \ + set_error_buf_mem_offset_out_of_range(error_buf, error_buf_size); \ + goto fail; \ + } \ + res = (mem_offset_t)res64; \ + } while (0) +#else +#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, 32, false, &res64, error_buf, \ + error_buf_size)) \ + goto fail; \ + res = (uint32)res64; \ + } while (0) + +#define read_leb_int32(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, 32, true, &res64, error_buf, \ + error_buf_size)) \ + goto fail; \ + res = (int32)res64; \ + } while (0) + +static char * +type2str(uint8 type) +{ + char *type_str[] = { "v128", "f64", "f32", "i64", "i32" }; +#if WASM_ENABLE_GC != 0 + char *type_str_ref[] = { "stringview_iter", + "stringview_wtf16", + "(ref null ht)", + "(ref ht)", + "", /* reserved */ + "stringview_wtf8", + "stringref", + "", /* reserved */ + "", /* reserved */ + "arrayref", + "structref", + "i32ref", + "eqref", + "anyref", + "externref", + "funcref", + "nullref", + "nullexternref", + "nullfuncref" }; +#endif + + if (type >= VALUE_TYPE_V128 && type <= VALUE_TYPE_I32) + return type_str[type - VALUE_TYPE_V128]; +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(type)) + return type_str_ref[type - REF_TYPE_STRINGVIEWITER]; +#endif + else if (type == VALUE_TYPE_FUNCREF) + return "funcref"; + else if (type == VALUE_TYPE_EXTERNREF) + return "externref"; + else + return "unknown type"; +} + +static bool +is_32bit_type(uint8 type) +{ + if (type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32 +#if WASM_ENABLE_GC != 0 + || (sizeof(uintptr_t) == 4 && wasm_is_type_reftype(type)) +#elif WASM_ENABLE_REF_TYPES != 0 + /* For reference types, we use uint32 index to represent + the funcref and externref */ + || type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF +#endif + ) + return true; + return false; +} + +static bool +is_64bit_type(uint8 type) +{ + if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64 +#if WASM_ENABLE_GC != 0 + || (sizeof(uintptr_t) == 8 && wasm_is_type_reftype(type)) +#endif + ) + return true; + return false; +} + +static bool +is_value_type(uint8 type) +{ + if (/* I32/I64/F32/F64, 0x7C to 0x7F */ + (type >= VALUE_TYPE_F64 && type <= VALUE_TYPE_I32) +#if WASM_ENABLE_GC != 0 + /* reference types, 0x65 to 0x70 */ + || wasm_is_type_reftype(type) +#elif WASM_ENABLE_REF_TYPES != 0 + || (type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF) +#endif +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + || type == VALUE_TYPE_V128 /* 0x7B */ +#endif +#endif + ) + return true; + return false; +} + +#if WASM_ENABLE_GC != 0 +static bool +is_packed_type(uint8 type) +{ + return (type == PACKED_TYPE_I8 || type == PACKED_TYPE_I16) ? true : false; +} +#endif + +static bool +is_byte_a_type(uint8 type) +{ + return (is_value_type(type) || (type == VALUE_TYPE_VOID)) ? true : false; +} + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) +static V128 +read_i8x16(uint8 *p_buf, char *error_buf, uint32 error_buf_size) +{ + V128 result; + uint8 i; + + for (i = 0; i != 16; ++i) { + result.i8x16[i] = read_uint8(p_buf); + } + + return result; +} +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +static void * +loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +static void * +memory_realloc(void *mem_old, uint32 size_old, uint32 size_new, char *error_buf, + uint32 error_buf_size) +{ + uint8 *mem_new; + bh_assert(size_new > size_old); + if ((mem_new = loader_malloc(size_new, error_buf, error_buf_size))) { + bh_memcpy_s(mem_new, size_new, mem_old, size_old); + memset(mem_new + size_old, 0, size_new - size_old); + wasm_runtime_free(mem_old); + } + return mem_new; +} + +#define MEM_REALLOC(mem, size_old, size_new) \ + do { \ + void *mem_new = memory_realloc(mem, size_old, size_new, error_buf, \ + error_buf_size); \ + if (!mem_new) \ + goto fail; \ + mem = mem_new; \ + } while (0) + +#if WASM_ENABLE_GC != 0 +static bool +check_type_index(const WASMModule *module, uint32 type_count, uint32 type_index, + char *error_buf, uint32 error_buf_size) +{ + if (type_index >= type_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown type %d", + type_index); + return false; + } + return true; +} + +static bool +check_array_type(const WASMModule *module, uint32 type_index, char *error_buf, + uint32 error_buf_size) +{ + if (!check_type_index(module, module->type_count, type_index, error_buf, + error_buf_size)) { + return false; + } + if (module->types[type_index]->type_flag != WASM_TYPE_ARRAY) { + set_error_buf(error_buf, error_buf_size, "unkown array type"); + return false; + } + + return true; +} +#endif + +static bool +check_function_index(const WASMModule *module, uint32 function_index, + char *error_buf, uint32 error_buf_size) +{ + if (function_index + >= module->import_function_count + module->function_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown function %u", + function_index); + return false; + } + return true; +} + +typedef struct InitValue { + uint8 type; + uint8 flag; +#if WASM_ENABLE_GC != 0 + uint8 gc_opcode; + WASMRefType ref_type; +#endif + WASMValue value; +} InitValue; + +typedef struct ConstExprContext { + uint32 sp; + uint32 size; + WASMModule *module; + InitValue *stack; + InitValue data[WASM_CONST_EXPR_STACK_SIZE]; +} ConstExprContext; + +static void +init_const_expr_stack(ConstExprContext *ctx, WASMModule *module) +{ + ctx->sp = 0; + ctx->module = module; + ctx->stack = ctx->data; + ctx->size = WASM_CONST_EXPR_STACK_SIZE; +} + +static bool +push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type, +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type, uint8 gc_opcode, +#endif + WASMValue *value, char *error_buf, uint32 error_buf_size) +{ + InitValue *cur_value; + + if (ctx->sp >= ctx->size) { + if (ctx->stack != ctx->data) { + MEM_REALLOC(ctx->stack, ctx->size * sizeof(InitValue), + (ctx->size + 4) * sizeof(InitValue)); + } + else { + if (!(ctx->stack = + loader_malloc((ctx->size + 4) * (uint64)sizeof(InitValue), + error_buf, error_buf_size))) { + goto fail; + } + } + ctx->size += 4; + } + + cur_value = &ctx->stack[ctx->sp++]; + cur_value->type = type; + cur_value->flag = flag; + cur_value->value = *value; + +#if WASM_ENABLE_GC != 0 + cur_value->gc_opcode = gc_opcode; + if (wasm_is_type_multi_byte_type(type)) { + bh_memcpy_s(&cur_value->ref_type, wasm_reftype_struct_size(ref_type), + ref_type, wasm_reftype_struct_size(ref_type)); + } +#endif + + return true; +fail: + return false; +} + +static bool +pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type, +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type, uint8 *p_gc_opcode, +#endif + WASMValue *p_value, char *error_buf, uint32 error_buf_size) +{ + InitValue *cur_value; + + if (ctx->sp == 0) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: const expr stack underflow"); + return false; + } + + cur_value = &ctx->stack[--ctx->sp]; + +#if WASM_ENABLE_GC == 0 + if (cur_value->type != type) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } +#else + if (!wasm_reftype_is_subtype_of(cur_value->type, &cur_value->ref_type, type, + ref_type, ctx->module->types, + ctx->module->type_count)) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + goto fail; + } + + if ((ctx->sp != 0) && (cur_value->flag == WASM_OP_GC_PREFIX) + && (cur_value->gc_opcode != WASM_OP_REF_I31)) { + /* To reduce complexity, we don't allow initialize struct fields/array + * element with references, so struct/array must be at the bottom of the + * init value stack */ + set_error_buf( + error_buf, error_buf_size, + "struct or array as field is not supported in constant expr"); + goto fail; + } +#endif + + if (p_flag) + *p_flag = cur_value->flag; + if (p_value) + *p_value = cur_value->value; +#if WASM_ENABLE_GC != 0 + if (p_gc_opcode) + *p_gc_opcode = cur_value->gc_opcode; +#endif + + return true; + +#if WASM_ENABLE_GC != 0 +fail: + if ((cur_value->flag == WASM_OP_GC_PREFIX) + && (cur_value->gc_opcode == WASM_OP_STRUCT_NEW + || cur_value->gc_opcode == WASM_OP_ARRAY_NEW + || cur_value->gc_opcode == WASM_OP_ARRAY_NEW_FIXED)) { + wasm_runtime_free(cur_value->value.data); + } + return false; +#endif +} + +static void +destroy_const_expr_stack(ConstExprContext *ctx) +{ +#if WASM_ENABLE_GC != 0 + uint32 i; + + for (i = 0; i < ctx->sp; i++) { + if ((ctx->stack[i].flag == WASM_OP_GC_PREFIX) + && (ctx->stack[i].gc_opcode == WASM_OP_STRUCT_NEW + || ctx->stack[i].gc_opcode == WASM_OP_ARRAY_NEW + || ctx->stack[i].gc_opcode == WASM_OP_ARRAY_NEW_FIXED)) { + wasm_runtime_free(ctx->stack[i].value.data); + } + } +#endif + + if (ctx->stack != ctx->data) { + wasm_runtime_free(ctx->stack); + } +} + +#if WASM_ENABLE_GC != 0 +static void +destroy_init_expr(InitializerExpression *expr) +{ + if (expr->init_expr_type == INIT_EXPR_TYPE_STRUCT_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + wasm_runtime_free(expr->u.data); + } +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static bool +load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, + InitializerExpression *init_expr, uint8 type, void *ref_type, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 flag, *p_float; + uint32 i; + ConstExprContext const_expr_ctx = { 0 }; + WASMValue cur_value; +#if WASM_ENABLE_GC != 0 + uint32 opcode1, type_idx; + uint8 opcode; + WASMRefType cur_ref_type = { 0 }; +#endif + + init_const_expr_stack(&const_expr_ctx, module); + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + while (flag != WASM_OP_END) { + switch (flag) { + /* i32.const */ + case INIT_EXPR_TYPE_I32_CONST: + read_leb_int32(p, p_end, cur_value.i32); + + if (!push_const_expr_stack( + &const_expr_ctx, flag, VALUE_TYPE_I32, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, error_buf, error_buf_size)) + goto fail; + break; + /* i64.const */ + case INIT_EXPR_TYPE_I64_CONST: + read_leb_int64(p, p_end, cur_value.i64); + + if (!push_const_expr_stack( + &const_expr_ctx, flag, VALUE_TYPE_I64, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, error_buf, error_buf_size)) + goto fail; + break; + /* f32.const */ + case INIT_EXPR_TYPE_F32_CONST: + CHECK_BUF(p, p_end, 4); + p_float = (uint8 *)&cur_value.f32; + for (i = 0; i < sizeof(float32); i++) + *p_float++ = *p++; + + if (!push_const_expr_stack( + &const_expr_ctx, flag, VALUE_TYPE_F32, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, error_buf, error_buf_size)) + goto fail; + break; + /* f64.const */ + case INIT_EXPR_TYPE_F64_CONST: + CHECK_BUF(p, p_end, 8); + p_float = (uint8 *)&cur_value.f64; + for (i = 0; i < sizeof(float64); i++) + *p_float++ = *p++; + + if (!push_const_expr_stack( + &const_expr_ctx, flag, VALUE_TYPE_F64, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, error_buf, error_buf_size)) + goto fail; + break; +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + /* v128.const */ + case INIT_EXPR_TYPE_V128_CONST: + { + uint64 high, low; + + CHECK_BUF(p, p_end, 1); + (void)read_uint8(p); + + CHECK_BUF(p, p_end, 16); + wasm_runtime_read_v128(p, &high, &low); + p += 16; + + cur_value.v128.i64x2[0] = high; + cur_value.v128.i64x2[1] = low; + + if (!push_const_expr_stack( + &const_expr_ctx, flag, VALUE_TYPE_V128, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, error_buf, error_buf_size)) + goto fail; +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* If any init_expr is v128.const, mark SIMD used */ + module->is_simd_used = true; +#endif + break; + } +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + /* ref.func */ + case INIT_EXPR_TYPE_FUNCREF_CONST: + { + uint32 func_idx; + read_leb_uint32(p, p_end, func_idx); + cur_value.ref_index = func_idx; + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_GC == 0 + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_FUNCREF, &cur_value, + error_buf, error_buf_size)) + goto fail; +#else + if (func_idx < module->import_function_count) { + type_idx = + module->import_functions[func_idx].u.function.type_idx; + } + else { + type_idx = module + ->functions[func_idx + - module->import_function_count] + ->type_idx; + } + wasm_set_refheaptype_typeidx(&cur_ref_type.ref_ht_typeidx, + false, type_idx); + if (!push_const_expr_stack(&const_expr_ctx, flag, + cur_ref_type.ref_type, &cur_ref_type, + 0, &cur_value, error_buf, + error_buf_size)) + goto fail; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + + /* ref.null */ + case INIT_EXPR_TYPE_REFNULL_CONST: + { + uint8 type1; + + CHECK_BUF(p, p_end, 1); + type1 = read_uint8(p); + +#if WASM_ENABLE_GC == 0 + cur_value.ref_index = NULL_REF; + if (!push_const_expr_stack(&const_expr_ctx, flag, type1, + &cur_value, error_buf, + error_buf_size)) + goto fail; +#else + cur_value.gc_obj = NULL_REF; + + if (!is_byte_a_type(type1)) { + p--; + read_leb_uint32(p, p_end, type_idx); + if (!check_type_index(module, module->type_count, type_idx, + error_buf, error_buf_size)) + goto fail; + + wasm_set_refheaptype_typeidx(&cur_ref_type.ref_ht_typeidx, + true, type_idx); + if (!push_const_expr_stack(&const_expr_ctx, flag, + cur_ref_type.ref_type, + &cur_ref_type, 0, &cur_value, + error_buf, error_buf_size)) + goto fail; + } + else { + if (!push_const_expr_stack(&const_expr_ctx, flag, type1, + NULL, 0, &cur_value, error_buf, + error_buf_size)) + goto fail; + } +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + + /* get_global */ + case INIT_EXPR_TYPE_GET_GLOBAL: + { + uint32 global_idx; + uint8 global_type; + + read_leb_uint32(p, p_end, cur_value.global_index); + global_idx = cur_value.global_index; + +#if WASM_ENABLE_GC == 0 + if (global_idx >= module->import_global_count) { + /** + * Currently, constant expressions occurring as initializers + * of globals are further constrained in that contained + * global.get instructions are + * only allowed to refer to imported globals. + */ + set_error_buf_v(error_buf, error_buf_size, + "unknown global %u", global_idx); + goto fail; + } + if (module->import_globals[global_idx].u.global.is_mutable) { + set_error_buf_v(error_buf, error_buf_size, + "constant expression required"); + goto fail; + } +#else + if (global_idx + >= module->import_global_count + module->global_count) { + set_error_buf_v(error_buf, error_buf_size, + "unknown global %u", global_idx); + goto fail; + } + if (global_idx < module->import_global_count + && module->import_globals[global_idx].u.global.is_mutable) { + set_error_buf_v(error_buf, error_buf_size, + "constant expression required"); + goto fail; + } +#endif + + if (global_idx < module->import_global_count) { + global_type = + module->import_globals[global_idx].u.global.type; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(global_type)) { + WASMRefType *global_ref_type = + module->import_globals[global_idx] + .u.global.ref_type; + bh_memcpy_s(&cur_ref_type, + wasm_reftype_struct_size(global_ref_type), + global_ref_type, + wasm_reftype_struct_size(global_ref_type)); + } +#endif + } + else { + global_type = + module + ->globals[global_idx - module->import_global_count] + .type; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(global_type)) { + WASMRefType *global_ref_type = + module + ->globals[global_idx + - module->import_global_count] + .ref_type; + bh_memcpy_s(&cur_ref_type, + wasm_reftype_struct_size(global_ref_type), + global_ref_type, + wasm_reftype_struct_size(global_ref_type)); + } +#endif + } + + if (!push_const_expr_stack(&const_expr_ctx, flag, global_type, +#if WASM_ENABLE_GC != 0 + &cur_ref_type, 0, +#endif + &cur_value, error_buf, + error_buf_size)) + goto fail; + + break; + } + +#if WASM_ENABLE_GC != 0 + /* struct.new and array.new */ + case WASM_OP_GC_PREFIX: + { + read_leb_uint32(p, p_end, opcode1); + + switch (opcode1) { + case WASM_OP_STRUCT_NEW: + { + WASMStructType *struct_type; + WASMStructNewInitValues *struct_init_values = NULL; + uint32 field_count; + read_leb_uint32(p, p_end, type_idx); + + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + struct_type = (WASMStructType *)module->types[type_idx]; + if (struct_type->base_type.type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unkown struct type"); + goto fail; + } + field_count = struct_type->field_count; + + if (!(struct_init_values = loader_malloc( + offsetof(WASMStructNewInitValues, fields) + + (uint64)field_count * sizeof(WASMValue), + error_buf, error_buf_size))) { + goto fail; + } + struct_init_values->count = field_count; + + for (i = field_count; i > 0; i--) { + WASMRefType *field_ref_type = NULL; + uint32 field_idx = i - 1; + uint8 field_type = + struct_type->fields[field_idx].field_type; + if (wasm_is_type_multi_byte_type(field_type)) { + field_ref_type = wasm_reftype_map_find( + struct_type->ref_type_maps, + struct_type->ref_type_map_count, field_idx); + } + + if (is_packed_type(field_type)) { + field_type = VALUE_TYPE_I32; + } + + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, field_type, + field_ref_type, NULL, + &struct_init_values->fields[field_idx], + error_buf, error_buf_size)) { + wasm_runtime_free(struct_init_values); + goto fail; + } + } + + cur_value.data = struct_init_values; + wasm_set_refheaptype_typeidx( + &cur_ref_type.ref_ht_typeidx, false, type_idx); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, + error_buf, error_buf_size)) { + wasm_runtime_free(struct_init_values); + goto fail; + } + break; + } + case WASM_OP_STRUCT_NEW_DEFAULT: + { + read_leb_uint32(p, p_end, cur_value.type_index); + type_idx = cur_value.type_index; + + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unkown struct type"); + goto fail; + } + + cur_value.type_index = type_idx; + wasm_set_refheaptype_typeidx( + &cur_ref_type.ref_ht_typeidx, false, type_idx); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, + error_buf, error_buf_size)) { + goto fail; + } + break; + } + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + { + WASMArrayNewInitValues *array_init_values = NULL; + WASMArrayType *array_type = NULL; + WASMRefType *elem_ref_type = NULL; + uint64 total_size; + uint8 elem_type; + + read_leb_uint32(p, p_end, cur_value.type_index); + type_idx = cur_value.type_index; + + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + array_type = (WASMArrayType *)module->types[type_idx]; + if (array_type->base_type.type_flag + != WASM_TYPE_ARRAY) { + set_error_buf(error_buf, error_buf_size, + "unkown array type"); + goto fail; + } + + if (opcode1 != WASM_OP_ARRAY_NEW_DEFAULT) { + elem_type = array_type->elem_type; + if (wasm_is_type_multi_byte_type(elem_type)) { + elem_ref_type = array_type->elem_ref_type; + } + + if (is_packed_type(elem_type)) { + elem_type = VALUE_TYPE_I32; + } + + if (opcode1 == WASM_OP_ARRAY_NEW) { + WASMValue len_val; + + if (!(array_init_values = loader_malloc( + sizeof(WASMArrayNewInitValues), + error_buf, error_buf_size))) { + goto fail; + } + array_init_values->type_idx = type_idx; + + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, VALUE_TYPE_I32, + NULL, NULL, &len_val, error_buf, + error_buf_size)) { + wasm_runtime_free(array_init_values); + goto fail; + } + array_init_values->length = len_val.i32; + + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, elem_type, + elem_ref_type, NULL, + &array_init_values->elem_data[0], + error_buf, error_buf_size)) { + wasm_runtime_free(array_init_values); + goto fail; + } + + cur_value.data = array_init_values; + } + else { + /* WASM_OP_ARRAY_NEW_FIXED */ + uint32 len; + read_leb_uint32(p, p_end, len); + + total_size = + (uint64)offsetof(WASMArrayNewInitValues, + elem_data) + + (uint64)sizeof(WASMValue) * len; + if (!(array_init_values = + loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + + array_init_values->type_idx = type_idx; + array_init_values->length = len; + + for (i = len; i > 0; i--) { + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, elem_type, + elem_ref_type, NULL, + &array_init_values + ->elem_data[i - 1], + error_buf, error_buf_size)) { + wasm_runtime_free(array_init_values); + goto fail; + } + } + + cur_value.data = array_init_values; + } + } + else { + /* WASM_OP_ARRAY_NEW_DEFAULT */ + WASMValue len_val; + uint32 len; + + /* POP(i32) */ + if (!pop_const_expr_stack(&const_expr_ctx, NULL, + VALUE_TYPE_I32, NULL, + NULL, &len_val, error_buf, + error_buf_size)) { + goto fail; + } + len = len_val.i32; + + cur_value.array_new_default.type_index = type_idx; + cur_value.array_new_default.length = len; + } + + wasm_set_refheaptype_typeidx( + &cur_ref_type.ref_ht_typeidx, false, type_idx); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, + error_buf, error_buf_size)) { + if (array_init_values) { + wasm_runtime_free(array_init_values); + } + goto fail; + } + break; + } + case WASM_OP_ANY_CONVERT_EXTERN: + { + set_error_buf(error_buf, error_buf_size, + "unsuppoted constant expression of " + "extern.internalize"); + goto fail; + } + case WASM_OP_EXTERN_CONVERT_ANY: + { + set_error_buf(error_buf, error_buf_size, + "unsuppoted constant expression of " + "extern.externalize"); + goto fail; + } + case WASM_OP_REF_I31: + { + /* POP(i32) */ + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, VALUE_TYPE_I32, NULL, + NULL, &cur_value, error_buf, error_buf_size)) { + goto fail; + } + + wasm_set_refheaptype_common(&cur_ref_type.ref_ht_common, + false, HEAP_TYPE_I31); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, + error_buf, error_buf_size)) { + goto fail; + } + break; + } + default: + set_error_buf( + error_buf, error_buf_size, + "type mismatch or constant expression required"); + goto fail; + } + + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + { + set_error_buf(error_buf, error_buf_size, + "illegal opcode " + "or constant expression required " + "or type mismatch"); + goto fail; + } + } + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + } + + /* There should be only one value left on the init value stack */ + if (!pop_const_expr_stack(&const_expr_ctx, &flag, type, +#if WASM_ENABLE_GC != 0 + ref_type, &opcode, +#endif + &cur_value, error_buf, error_buf_size)) { + goto fail; + } + + if (const_expr_ctx.sp != 0) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: illegal constant opcode sequence"); + goto fail; + } + + init_expr->init_expr_type = flag; + init_expr->u = cur_value; + +#if WASM_ENABLE_GC != 0 + if (init_expr->init_expr_type == WASM_OP_GC_PREFIX) { + switch (opcode) { + case WASM_OP_STRUCT_NEW: + init_expr->init_expr_type = INIT_EXPR_TYPE_STRUCT_NEW; + break; + case WASM_OP_STRUCT_NEW_DEFAULT: + init_expr->init_expr_type = INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT; + break; + case WASM_OP_ARRAY_NEW: + init_expr->init_expr_type = INIT_EXPR_TYPE_ARRAY_NEW; + break; + case WASM_OP_ARRAY_NEW_DEFAULT: + init_expr->init_expr_type = INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT; + break; + case WASM_OP_ARRAY_NEW_FIXED: + init_expr->init_expr_type = INIT_EXPR_TYPE_ARRAY_NEW_FIXED; + break; + case WASM_OP_REF_I31: + init_expr->init_expr_type = INIT_EXPR_TYPE_I31_NEW; + break; + default: + bh_assert(0); + break; + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + *p_buf = p; + destroy_const_expr_stack(&const_expr_ctx); + return true; + +fail: + destroy_const_expr_stack(&const_expr_ctx); + return false; +} + +static bool +check_mutability(uint8 mutable, char *error_buf, uint32 error_buf_size) +{ + if (mutable >= 2) { + set_error_buf(error_buf, error_buf_size, "invalid mutability"); + return false; + } + return true; +} + +#if WASM_ENABLE_GC != 0 +static void +destroy_func_type(WASMFuncType *type) +{ + /* Destroy the reference type hash set */ + if (type->ref_type_maps) + wasm_runtime_free(type->ref_type_maps); + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (type->call_to_llvm_jit_from_fast_jit) + jit_code_cache_free(type->call_to_llvm_jit_from_fast_jit); +#endif + /* Free the type */ + wasm_runtime_free(type); +} + +static void +destroy_struct_type(WASMStructType *type) +{ + if (type->ref_type_maps) + wasm_runtime_free(type->ref_type_maps); + + wasm_runtime_free(type); +} + +static void +destroy_array_type(WASMArrayType *type) +{ + wasm_runtime_free(type); +} + +static void +destroy_wasm_type(WASMType *type) +{ + if (type->ref_count > 1) { + /* The type is referenced by other types + of current wasm module */ + type->ref_count--; + return; + } + + if (type->type_flag == WASM_TYPE_FUNC) + destroy_func_type((WASMFuncType *)type); + else if (type->type_flag == WASM_TYPE_STRUCT) + destroy_struct_type((WASMStructType *)type); + else if (type->type_flag == WASM_TYPE_ARRAY) + destroy_array_type((WASMArrayType *)type); + else { + bh_assert(0); + } +} + +/* Resolve (ref null ht) or (ref ht) */ +static bool +resolve_reftype_htref(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, bool nullable, + WASMRefType *ref_type, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + + ref_type->ref_type = + nullable ? REF_TYPE_HT_NULLABLE : REF_TYPE_HT_NON_NULLABLE; + ref_type->ref_ht_common.nullable = nullable; + read_leb_int32(p, p_end, ref_type->ref_ht_common.heap_type); + + if (wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common)) { + /* heap type is (type i), i : typeidx, >= 0 */ + if (!check_type_index(module, type_count, + ref_type->ref_ht_typeidx.type_idx, error_buf, + error_buf_size)) { + return false; + } + } + else if (!wasm_is_refheaptype_common(&ref_type->ref_ht_common)) { + /* heap type is func, extern, any, eq, i31 or data */ + set_error_buf(error_buf, error_buf_size, "unknown heap type"); + return false; + } + + *p_buf = p; + return true; +fail: + return false; +} + +static bool +resolve_value_type(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, + bool *p_need_ref_type_map, WASMRefType *ref_type, + bool allow_packed_type, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 type; + + memset(ref_type, 0, sizeof(WASMRefType)); + + CHECK_BUF(p, p_end, 1); + type = read_uint8(p); + + if (wasm_is_reftype_htref_nullable(type)) { + /* (ref null ht) */ + if (!resolve_reftype_htref(&p, p_end, module, type_count, true, + ref_type, error_buf, error_buf_size)) + return false; + if (!wasm_is_refheaptype_common(&ref_type->ref_ht_common)) + *p_need_ref_type_map = true; + else { + /* For (ref null func/extern/any/eq/i31/data), they are same as + funcref/externref/anyref/eqref/i31ref/dataref, we convert the + multi-byte type to one-byte type to reduce the footprint and + the complexity of type equal/subtype checking */ + ref_type->ref_type = + (uint8)((int32)0x80 + ref_type->ref_ht_common.heap_type); + *p_need_ref_type_map = false; + } + } + else if (wasm_is_reftype_htref_non_nullable(type)) { + /* (ref ht) */ + if (!resolve_reftype_htref(&p, p_end, module, type_count, false, + ref_type, error_buf, error_buf_size)) + return false; + *p_need_ref_type_map = true; +#if WASM_ENABLE_STRINGREF != 0 + /* covert (ref string) to stringref */ + if (wasm_is_refheaptype_stringrefs(&ref_type->ref_ht_common)) { + ref_type->ref_type = + (uint8)((int32)0x80 + ref_type->ref_ht_common.heap_type); + *p_need_ref_type_map = false; + } +#endif + } + else { + /* type which can be represented by one byte */ + if (!is_value_type(type) + && !(allow_packed_type && is_packed_type(type))) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + ref_type->ref_type = type; + *p_need_ref_type_map = false; +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* If any value's type is v128, mark the module as SIMD used */ + if (type == VALUE_TYPE_V128) + module->is_simd_used = true; +#endif + } + + *p_buf = p; + return true; +fail: + return false; +} + +static WASMRefType * +reftype_set_insert(HashMap *ref_type_set, const WASMRefType *ref_type, + char *error_buf, uint32 error_buf_size) +{ + WASMRefType *ret = wasm_reftype_set_insert(ref_type_set, ref_type); + + if (!ret) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + } + return ret; +} + +static bool +resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, + uint32 type_count, uint32 type_idx, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; + uint32 param_count, result_count, i, j = 0; + uint32 param_cell_num, ret_cell_num; + uint32 ref_type_map_count = 0, result_ref_type_map_count = 0; + uint64 total_size; + bool need_ref_type_map; + WASMRefType ref_type; + WASMFuncType *type = NULL; + + /* Parse first time to resolve param count, result count and + ref type map count */ + read_leb_uint32(p, p_end, param_count); + p_org = p; + for (i = 0; i < param_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) + ref_type_map_count++; + } + + read_leb_uint32(p, p_end, result_count); + for (i = 0; i < result_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) { + ref_type_map_count++; + result_ref_type_map_count++; + } + } + + LOG_VERBOSE("type %u: func, param count: %d, result count: %d, " + "ref type map count: %d", + type_idx, param_count, result_count, ref_type_map_count); + + /* Parse second time to resolve param types, result types and + ref type map info */ + p = p_org; + + total_size = offsetof(WASMFuncType, types) + + sizeof(uint8) * (uint64)(param_count + result_count); + if (!(type = loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + if (ref_type_map_count > 0) { + total_size = sizeof(WASMRefTypeMap) * (uint64)ref_type_map_count; + if (!(type->ref_type_maps = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + } + + type->base_type.type_flag = WASM_TYPE_FUNC; + type->param_count = param_count; + type->result_count = result_count; + type->ref_type_map_count = ref_type_map_count; + if (ref_type_map_count > 0) { + type->result_ref_type_maps = type->ref_type_maps + ref_type_map_count + - result_ref_type_map_count; + } + + for (i = 0; i < param_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + goto fail; + } + type->types[i] = ref_type.ref_type; + if (need_ref_type_map) { + type->ref_type_maps[j].index = i; + if (!(type->ref_type_maps[j++].ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + } + + read_leb_uint32(p, p_end, result_count); + for (i = 0; i < result_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + goto fail; + } + type->types[param_count + i] = ref_type.ref_type; + if (need_ref_type_map) { + type->ref_type_maps[j].index = param_count + i; + if (!(type->ref_type_maps[j++].ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + } + + bh_assert(j == type->ref_type_map_count); +#if TRACE_WASM_LOADER != 0 + os_printf("type %d = ", type_idx); + wasm_dump_func_type(type); +#endif + + param_cell_num = wasm_get_cell_num(type->types, param_count); + ret_cell_num = wasm_get_cell_num(type->types + param_count, result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + goto fail; + } + type->param_cell_num = (uint16)param_cell_num; + type->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + for (i = 0; i < (uint32)(type->param_count + type->result_count); i++) { + if (type->types[i] == VALUE_TYPE_V128) + module->is_simd_used = true; + } +#endif + + *p_buf = p; + + module->types[type_idx] = (WASMType *)type; + return true; + +fail: + if (type) + destroy_func_type(type); + return false; +} + +static bool +resolve_struct_type(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, uint32 type_idx, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; + uint32 field_count, ref_type_map_count = 0, ref_field_count = 0; + uint32 i, j = 0, offset; + uint16 *reference_table; + uint64 total_size; + uint8 mutable; + bool need_ref_type_map; + WASMRefType ref_type; + WASMStructType *type = NULL; + + /* Parse first time to resolve field count and ref type map count */ + read_leb_uint32(p, p_end, field_count); + p_org = p; + for (i = 0; i < field_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, true, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) + ref_type_map_count++; + + if (wasm_is_type_reftype(ref_type.ref_type)) + ref_field_count++; + + CHECK_BUF(p, p_end, 1); + mutable = read_uint8(p); + if (!check_mutability(mutable, error_buf, error_buf_size)) { + return false; + } + } + + LOG_VERBOSE("type %u: struct, field count: %d, ref type map count: %d", + type_idx, field_count, ref_type_map_count); + + /* Parse second time to resolve field types and ref type map info */ + p = p_org; + + total_size = offsetof(WASMStructType, fields) + + sizeof(WASMStructFieldType) * (uint64)field_count + + sizeof(uint16) * (uint64)(ref_field_count + 1); + if (!(type = loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + if (ref_type_map_count > 0) { + total_size = sizeof(WASMRefTypeMap) * (uint64)ref_type_map_count; + if (!(type->ref_type_maps = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + } + + type->reference_table = reference_table = + (uint16 *)((uint8 *)type + offsetof(WASMStructType, fields) + + sizeof(WASMStructFieldType) * field_count); + *reference_table++ = ref_field_count; + + type->base_type.type_flag = WASM_TYPE_STRUCT; + type->field_count = field_count; + type->ref_type_map_count = ref_type_map_count; + + offset = (uint32)offsetof(WASMStructObject, field_data); + for (i = 0; i < field_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, true, error_buf, + error_buf_size)) { + goto fail; + } + type->fields[i].field_type = ref_type.ref_type; + if (need_ref_type_map) { + type->ref_type_maps[j].index = i; + if (!(type->ref_type_maps[j++].ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + + CHECK_BUF(p, p_end, 1); + type->fields[i].field_flags = read_uint8(p); + type->fields[i].field_size = + (uint8)wasm_reftype_size(ref_type.ref_type); +#if !(defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_X86_32)) + if (type->fields[i].field_size == 2) + offset = align_uint(offset, 2); + else if (type->fields[i].field_size >= 4) /* field size is 4 or 8 */ + offset = align_uint(offset, 4); +#endif + type->fields[i].field_offset = offset; + if (wasm_is_type_reftype(ref_type.ref_type)) + *reference_table++ = offset; + offset += type->fields[i].field_size; + + LOG_VERBOSE(" field: %d, flags: %d, type: %d", i, + type->fields[i].field_flags, type->fields[i].field_type); + } + type->total_size = offset; + + bh_assert(j == type->ref_type_map_count); +#if TRACE_WASM_LOADER != 0 + os_printf("type %d = ", type_idx); + wasm_dump_struct_type(type); +#endif + + *p_buf = p; + + module->types[type_idx] = (WASMType *)type; + return true; + +fail: + if (type) + destroy_struct_type(type); + return false; +} + +static bool +resolve_array_type(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, uint32 type_idx, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 mutable; + bool need_ref_type_map; + WASMRefType ref_type; + WASMArrayType *type = NULL; + + if (!resolve_value_type(&p, p_end, module, type_count, &need_ref_type_map, + &ref_type, true, error_buf, error_buf_size)) { + return false; + } + + CHECK_BUF(p, p_end, 1); + mutable = read_uint8(p); + if (!check_mutability(mutable, error_buf, error_buf_size)) { + return false; + } + + LOG_VERBOSE("type %u: array", type_idx); + + if (!(type = loader_malloc(sizeof(WASMArrayType), error_buf, + error_buf_size))) { + return false; + } + + type->base_type.type_flag = WASM_TYPE_ARRAY; + type->elem_flags = mutable; + type->elem_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(type->elem_ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, error_buf, + error_buf_size))) { + goto fail; + } + } + +#if TRACE_WASM_LOADER != 0 + os_printf("type %d = ", type_idx); + wasm_dump_array_type(type); +#endif + + *p_buf = p; + + module->types[type_idx] = (WASMType *)type; + return true; + +fail: + if (type) + destroy_array_type(type); + return false; +} + +static bool +init_ref_type(WASMModule *module, WASMRefType *ref_type, bool nullable, + int32 heap_type, char *error_buf, uint32 error_buf_size) +{ + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, heap_type, error_buf, + error_buf_size)) { + return false; + } + wasm_set_refheaptype_typeidx(&ref_type->ref_ht_typeidx, nullable, + heap_type); + } + else { + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + return false; + } + wasm_set_refheaptype_common(&ref_type->ref_ht_common, nullable, + heap_type); + if (nullable) { + /* For (ref null func/extern/any/eq/i31/data), + they are same as + funcref/externref/anyref/eqref/i31ref/dataref, + we convert the multi-byte type to one-byte + type to reduce the footprint and the + complexity of type equal/subtype checking */ + ref_type->ref_type = + (uint8)((int32)0x80 + ref_type->ref_ht_common.heap_type); + } + } + return true; +} + +static void +calculate_reftype_diff(WASMRefType *ref_type_diff, WASMRefType *ref_type1, + WASMRefType *ref_type2) +{ + /** + * The difference rt1 ∖ rt2 between two reference types is defined as + * follows: + * (ref null?1 ht1) ∖ (ref null ht2) = (ref ht1) (ref null?1 ht1) ∖ + * (ref ht2) = (ref null?1 ht1) + */ + if (wasm_is_type_multi_byte_type(ref_type1->ref_type)) { + bh_memcpy_s(ref_type_diff, wasm_reftype_struct_size(ref_type1), + ref_type1, wasm_reftype_struct_size(ref_type1)); + } + else { + ref_type_diff->ref_type = ref_type1->ref_type; + } + + if (ref_type2->ref_ht_common.nullable) { + if (wasm_is_type_reftype(ref_type_diff->ref_type) + && !(wasm_is_type_multi_byte_type(ref_type_diff->ref_type))) { + wasm_set_refheaptype_typeidx(&ref_type_diff->ref_ht_typeidx, false, + (int32)ref_type_diff->ref_type - 0x80); + } + else { + ref_type_diff->ref_ht_typeidx.nullable = false; + } + } +} +#else /* else of WASM_ENABLE_GC != 0 */ +static void +destroy_wasm_type(WASMType *type) +{ + if (type->ref_count > 1) { + /* The type is referenced by other types + of current wasm module */ + type->ref_count--; + return; + } + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (type->call_to_llvm_jit_from_fast_jit) + jit_code_cache_free(type->call_to_llvm_jit_from_fast_jit); +#endif + + wasm_runtime_free(type); +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static bool +load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 type_count, i; + uint64 total_size; + uint8 flag; +#if WASM_ENABLE_GC != 0 + uint32 processed_type_count = 0; +#endif + + read_leb_uint32(p, p_end, type_count); + + if (type_count) { + module->type_count = type_count; + total_size = sizeof(WASMType *) * (uint64)type_count; + if (!(module->types = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + +#if WASM_ENABLE_GC == 0 + for (i = 0; i < type_count; i++) { + WASMFuncType *type; + const uint8 *p_org; + uint32 param_count, result_count, j; + uint32 param_cell_num, ret_cell_num; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + if (flag != 0x60) { + set_error_buf(error_buf, error_buf_size, "invalid type flag"); + return false; + } + + read_leb_uint32(p, p_end, param_count); + + /* Resolve param count and result count firstly */ + p_org = p; + CHECK_BUF(p, p_end, param_count); + p += param_count; + read_leb_uint32(p, p_end, result_count); + CHECK_BUF(p, p_end, result_count); + p = p_org; + + if (param_count > UINT16_MAX || result_count > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + return false; + } + + total_size = offsetof(WASMFuncType, types) + + sizeof(uint8) * (uint64)(param_count + result_count); + if (!(type = module->types[i] = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Resolve param types and result types */ + type->ref_count = 1; + type->param_count = (uint16)param_count; + type->result_count = (uint16)result_count; + for (j = 0; j < param_count; j++) { + CHECK_BUF(p, p_end, 1); + type->types[j] = read_uint8(p); + } + read_leb_uint32(p, p_end, result_count); + for (j = 0; j < result_count; j++) { + CHECK_BUF(p, p_end, 1); + type->types[param_count + j] = read_uint8(p); + } + for (j = 0; j < param_count + result_count; j++) { + if (!is_value_type(type->types[j])) { + set_error_buf(error_buf, error_buf_size, + "unknown value type"); + return false; + } + } + + param_cell_num = wasm_get_cell_num(type->types, param_count); + ret_cell_num = + wasm_get_cell_num(type->types + param_count, result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + return false; + } + type->param_cell_num = (uint16)param_cell_num; + type->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + for (j = 0; j < type->param_count + type->result_count; j++) { + if (type->types[j] == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (type->types[j] == VALUE_TYPE_FUNCREF + || type->types[j] == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; + } +#endif + + /* If there is already a same type created, use it instead */ + for (j = 0; j < i; j++) { + if (wasm_type_equal(type, module->types[j], module->types, i)) { + if (module->types[j]->ref_count == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "wasm type's ref count too large"); + return false; + } + destroy_wasm_type(type); + module->types[i] = module->types[j]; + module->types[j]->ref_count++; + break; + } + } + } +#else /* else of WASM_ENABLE_GC == 0 */ + for (i = 0; i < type_count; i++) { + uint32 super_type_count = 0, parent_type_idx = (uint32)-1; + uint32 rec_count = 1, j; + bool is_sub_final = true; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + if (flag == DEFINED_TYPE_REC) { + read_leb_uint32(p, p_end, rec_count); + + if (rec_count > 1) { + uint64 new_total_size; + + /* integer overflow */ + if (rec_count - 1 > UINT32_MAX - module->type_count) { + set_error_buf(error_buf, error_buf_size, + "recursive type count too large"); + return false; + } + module->type_count += rec_count - 1; + new_total_size = + sizeof(WASMFuncType *) * (uint64)module->type_count; + if (new_total_size > UINT32_MAX) { + set_error_buf(error_buf, error_buf_size, + "allocate memory failed"); + return false; + } + MEM_REALLOC(module->types, (uint32)total_size, + (uint32)new_total_size); + total_size = new_total_size; + } + + LOG_VERBOSE("Processing rec group [%d-%d]", + processed_type_count, + processed_type_count + rec_count - 1); + } + else { + p--; + } + + for (j = 0; j < rec_count; j++) { + WASMType *cur_type = NULL; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + parent_type_idx = -1; + + if (flag == DEFINED_TYPE_SUB + || flag == DEFINED_TYPE_SUB_FINAL) { + read_leb_uint32(p, p_end, super_type_count); + if (super_type_count > 1) { + set_error_buf(error_buf, error_buf_size, + "super type count too large"); + return false; + } + + if (super_type_count > 0) { + read_leb_uint32(p, p_end, parent_type_idx); + if (parent_type_idx >= processed_type_count + j) { + set_error_buf_v(error_buf, error_buf_size, + "unknown type %d", parent_type_idx); + return false; + } + if (module->types[parent_type_idx]->is_sub_final) { + set_error_buf(error_buf, error_buf_size, + "sub type can not inherit from " + "a final super type"); + return false; + } + } + + if (flag == DEFINED_TYPE_SUB) + is_sub_final = false; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + } + + if (flag == DEFINED_TYPE_FUNC) { + if (!resolve_func_type(&p, buf_end, module, + processed_type_count + rec_count, + processed_type_count + j, error_buf, + error_buf_size)) { + return false; + } + } + else if (flag == DEFINED_TYPE_STRUCT) { + if (!resolve_struct_type(&p, buf_end, module, + processed_type_count + rec_count, + processed_type_count + j, + error_buf, error_buf_size)) { + return false; + } + } + else if (flag == DEFINED_TYPE_ARRAY) { + if (!resolve_array_type(&p, buf_end, module, + processed_type_count + rec_count, + processed_type_count + j, error_buf, + error_buf_size)) { + return false; + } + } + else { + set_error_buf(error_buf, error_buf_size, + "invalid type flag"); + return false; + } + + cur_type = module->types[processed_type_count + j]; + + cur_type->ref_count = 1; + cur_type->parent_type_idx = parent_type_idx; + cur_type->is_sub_final = is_sub_final; + + cur_type->rec_count = rec_count; + cur_type->rec_idx = j; + cur_type->rec_begin_type_idx = processed_type_count; + } + + /* resolve subtyping relationship in current rec group */ + for (j = 0; j < rec_count; j++) { + WASMType *cur_type = module->types[processed_type_count + j]; + + if (cur_type->parent_type_idx != (uint32)-1) { /* has parent */ + WASMType *parent_type = + module->types[cur_type->parent_type_idx]; + cur_type->parent_type = parent_type; + cur_type->root_type = parent_type->root_type; + if (parent_type->inherit_depth == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "parent type's inherit depth too large"); + return false; + } + cur_type->inherit_depth = parent_type->inherit_depth + 1; + } + else { + cur_type->parent_type = NULL; + cur_type->root_type = cur_type; + cur_type->inherit_depth = 0; + } + } + + for (j = 0; j < rec_count; j++) { + WASMType *cur_type = module->types[processed_type_count + j]; + + if (cur_type->parent_type_idx != (uint32)-1) { /* has parent */ + WASMType *parent_type = + module->types[cur_type->parent_type_idx]; + if (!wasm_type_is_subtype_of(cur_type, parent_type, + module->types, + module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "sub type does not match super type"); + return false; + } + } + } + + /* If there is already an equivalence type or a group of equivalence + recursive types created, use it or them instead */ + for (j = 0; j < processed_type_count;) { + WASMType *src_type = module->types[j]; + WASMType *cur_type = module->types[processed_type_count]; + uint32 k, src_rec_count; + + src_rec_count = src_type->rec_count; + if (src_rec_count != rec_count) { + /* no type equivalence */ + j += src_rec_count; + continue; + } + + for (k = 0; k < rec_count; k++) { + src_type = module->types[j + k]; + cur_type = module->types[processed_type_count + k]; + if (!wasm_type_equal(src_type, cur_type, module->types, + module->type_count)) { + break; + } + } + if (k < rec_count) { + /* no type equivalence */ + j += src_rec_count; + continue; + } + + /* type equivalence */ + for (k = 0; k < rec_count; k++) { + if (module->types[j + k]->ref_count == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "wasm type's ref count too large"); + return false; + } + destroy_wasm_type(module->types[processed_type_count + k]); + module->types[processed_type_count + k] = + module->types[j + k]; + module->types[j + k]->ref_count++; + } + break; + } + + if (rec_count > 1) { + LOG_VERBOSE("Finished processing rec group [%d-%d]", + processed_type_count, + processed_type_count + rec_count - 1); + } + + processed_type_count += rec_count; + } + + if (!(module->rtt_types = loader_malloc((uint64)sizeof(WASMRttType *) + * module->type_count, + error_buf, error_buf_size))) { + return false; + } +#endif /* end of WASM_ENABLE_GC == 0 */ + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load type section success.\n"); + return true; +fail: + return false; +} + +static void +adjust_table_max_size(uint32 init_size, uint32 max_size_flag, uint32 *max_size) +{ + uint32 default_max_size = init_size * 2 > WASM_TABLE_MAX_SIZE + ? init_size * 2 + : WASM_TABLE_MAX_SIZE; + + if (max_size_flag) { + /* module defines the table limitation */ + bh_assert(init_size <= *max_size); + + if (init_size < *max_size) { + *max_size = + *max_size < default_max_size ? *max_size : default_max_size; + } + } + else { + /* partial defined table limitation, gives a default value */ + *max_size = default_max_size; + } +} + +#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0 +/** + * Find export item of a module with export info: + * module name, field name and export kind + */ +static WASMExport * +wasm_loader_find_export(const WASMModule *module, const char *module_name, + const char *field_name, uint8 export_kind, + char *error_buf, uint32 error_buf_size) +{ + WASMExport *export = + loader_find_export((WASMModuleCommon *)module, module_name, field_name, + export_kind, error_buf, error_buf_size); + return export; +} +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +static WASMFunction * +wasm_loader_resolve_function(const char *module_name, const char *function_name, + const WASMFuncType *expected_function_type, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMFunction *function = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + WASMFuncType *target_function_type = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for function %s", module_name, + function_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, function_name, + EXPORT_KIND_FUNC, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve function type and function */ + if (export->index < module->import_function_count) { + target_function_type = + module->import_functions[export->index].u.function.func_type; + function = module->import_functions[export->index] + .u.function.import_func_linked; + } + else { + target_function_type = + module->functions[export->index - module->import_function_count] + ->func_type; + function = + module->functions[export->index - module->import_function_count]; + } + + /* check function type */ + if (!wasm_type_equal((WASMType *)expected_function_type, + (WASMType *)target_function_type, module->types, + module->type_count)) { + LOG_DEBUG("%s.%s failed the type check", module_name, function_name); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + + return function; +} + +static WASMTable * +wasm_loader_resolve_table(const char *module_name, const char *table_name, + uint32 init_size, uint32 max_size, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMTable *table = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for table", module_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, table_name, + EXPORT_KIND_TABLE, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve table and check the init/max size */ + if (export->index < module->import_table_count) { + table = + module->import_tables[export->index].u.table.import_table_linked; + } + else { + table = &(module->tables[export->index - module->import_table_count]); + } + if (table->init_size < init_size || table->max_size > max_size) { + LOG_DEBUG("%s,%s failed type check(%d-%d), expected(%d-%d)", + module_name, table_name, table->init_size, table->max_size, + init_size, max_size); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + + return table; +} + +static WASMMemory * +wasm_loader_resolve_memory(const char *module_name, const char *memory_name, + uint32 init_page_count, uint32 max_page_count, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMMemory *memory = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for memory", module_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, memory_name, + EXPORT_KIND_MEMORY, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve memory and check the init/max page count */ + if (export->index < module->import_memory_count) { + memory = module->import_memories[export->index] + .u.memory.import_memory_linked; + } + else { + memory = + &(module->memories[export->index - module->import_memory_count]); + } + if (memory->init_page_count < init_page_count + || memory->max_page_count > max_page_count) { + LOG_DEBUG("%s,%s failed type check(%d-%d), expected(%d-%d)", + module_name, memory_name, memory->init_page_count, + memory->max_page_count, init_page_count, max_page_count); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + return memory; +} + +static WASMGlobal * +wasm_loader_resolve_global(const char *module_name, const char *global_name, + uint8 type, bool is_mutable, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMGlobal *global = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for global", module_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, global_name, + EXPORT_KIND_GLOBAL, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve and check the global */ + if (export->index < module->import_global_count) { + global = + module->import_globals[export->index].u.global.import_global_linked; + } + else { + global = + &(module->globals[export->index - module->import_global_count]); + } + if (global->type != type || global->is_mutable != is_mutable) { + LOG_DEBUG("%s,%s failed type check(%d, %d), expected(%d, %d)", + module_name, global_name, global->type, global->is_mutable, + type, is_mutable); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + return global; +} + +#if WASM_ENABLE_TAGS != 0 +static WASMTag * +wasm_loader_resolve_tag(const char *module_name, const char *tag_name, + const WASMType *expected_tag_type, + uint32 *linked_tag_index, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMTag *tag = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for tag %s", module_name, + tag_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, tag_name, EXPORT_KIND_TAG, + error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve tag type and tag */ + if (export->index < module->import_tag_count) { + /* importing an imported tag from the submodule */ + tag = module->import_tags[export->index].u.tag.import_tag_linked; + } + else { + /* importing an section tag from the submodule */ + tag = module->tags[export->index - module->import_tag_count]; + } + + /* check function type */ + if (!wasm_type_equal(expected_tag_type, tag->tag_type)) { + LOG_DEBUG("%s.%s failed the type check", module_name, tag_name); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + + if (linked_tag_index != NULL) { + *linked_tag_index = export->index; + } + + return tag; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ +#endif /* end of WASM_ENABLE_MULTI_MODULE */ + +static bool +load_function_import(const uint8 **p_buf, const uint8 *buf_end, + const WASMModule *parent_module, + const char *sub_module_name, const char *function_name, + WASMFunctionImport *function, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 declare_type_index = 0; + WASMFuncType *declare_func_type = NULL; + WASMFunction *linked_func = NULL; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; +#endif + const char *linked_signature = NULL; + void *linked_attachment = NULL; + bool linked_call_conv_raw = false; + bool is_native_symbol = false; + + read_leb_uint32(p, p_end, declare_type_index); + *p_buf = p; + + if (declare_type_index >= parent_module->type_count) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + return false; + } + +#if WASM_ENABLE_GC != 0 + function->type_idx = declare_type_index; +#endif + +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + declare_type_index = wasm_get_smallest_type_idx( + parent_module->types, parent_module->type_count, declare_type_index); +#endif + + declare_func_type = + (WASMFuncType *)parent_module->types[declare_type_index]; + + /* lookup registered native symbols first */ + linked_func = wasm_native_resolve_symbol( + sub_module_name, function_name, declare_func_type, &linked_signature, + &linked_attachment, &linked_call_conv_raw); + if (linked_func) { + is_native_symbol = true; + } +#if WASM_ENABLE_MULTI_MODULE != 0 + else { + if (!wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (!sub_module) { + return false; + } + } + linked_func = wasm_loader_resolve_function( + sub_module_name, function_name, declare_func_type, error_buf, + error_buf_size); + } +#endif + + function->module_name = (char *)sub_module_name; + function->field_name = (char *)function_name; + function->func_type = declare_func_type; + /* func_ptr_linked is for native registered symbol */ + function->func_ptr_linked = is_native_symbol ? linked_func : NULL; + function->signature = linked_signature; + function->attachment = linked_attachment; + function->call_conv_raw = linked_call_conv_raw; +#if WASM_ENABLE_MULTI_MODULE != 0 + function->import_module = is_native_symbol ? NULL : sub_module; + function->import_func_linked = is_native_symbol ? NULL : linked_func; +#endif + return true; +fail: + return false; +} + +static bool +check_table_max_size(uint32 init_size, uint32 max_size, char *error_buf, + uint32 error_buf_size) +{ + if (max_size < init_size) { + set_error_buf(error_buf, error_buf_size, + "size minimum must not be greater than maximum"); + return false; + } + return true; +} + +static bool +load_table_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, const char *sub_module_name, + const char *table_name, WASMTableImport *table, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 declare_elem_type = 0, declare_max_size_flag = 0, + declare_init_size = 0, declare_max_size = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; + WASMTable *linked_table = NULL; +#endif +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; + bool need_ref_type_map; +#endif + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + /* 0x70 or 0x6F */ + declare_elem_type = read_uint8(p); + if (VALUE_TYPE_FUNCREF != declare_elem_type +#if WASM_ENABLE_REF_TYPES != 0 + && VALUE_TYPE_EXTERNREF != declare_elem_type +#endif + ) { + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return false; + } +#else /* else of WASM_ENABLE_GC == 0 */ + if (!resolve_value_type(&p, p_end, parent_module, parent_module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (wasm_is_reftype_htref_non_nullable(ref_type.ref_type)) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + declare_elem_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(table->elem_ref_type = + reftype_set_insert(parent_module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("import table type: "); + wasm_dump_value_type(declare_elem_type, table->elem_ref_type); + os_printf("\n"); +#endif +#endif /* end of WASM_ENABLE_GC == 0 */ + + read_leb_uint32(p, p_end, declare_max_size_flag); + if (declare_max_size_flag > 1) { + set_error_buf(error_buf, error_buf_size, "integer too large"); + return false; + } + + read_leb_uint32(p, p_end, declare_init_size); + + if (declare_max_size_flag) { + read_leb_uint32(p, p_end, declare_max_size); + if (!check_table_max_size(declare_init_size, declare_max_size, + error_buf, error_buf_size)) + return false; + } + + adjust_table_max_size(declare_init_size, declare_max_size_flag, + &declare_max_size); + + *p_buf = p; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (!sub_module) { + return false; + } + + linked_table = wasm_loader_resolve_table( + sub_module_name, table_name, declare_init_size, declare_max_size, + error_buf, error_buf_size); + if (!linked_table) { + return false; + } + + /* reset with linked table limit */ + declare_elem_type = linked_table->elem_type; + declare_init_size = linked_table->init_size; + declare_max_size = linked_table->max_size; + declare_max_size_flag = linked_table->flags; + table->import_table_linked = linked_table; + table->import_module = sub_module; + } +#endif /* WASM_ENABLE_MULTI_MODULE != 0 */ + + /* (table (export "table") 10 20 funcref) */ + /* we need this section working in wamrc */ + if (!strcmp("spectest", sub_module_name)) { + const uint32 spectest_table_init_size = 10; + const uint32 spectest_table_max_size = 20; + + if (strcmp("table", table_name)) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type or unknown import"); + return false; + } + + if (declare_init_size > spectest_table_init_size + || declare_max_size < spectest_table_max_size) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + + declare_init_size = spectest_table_init_size; + declare_max_size = spectest_table_max_size; + } + + /* now we believe all declaration are ok */ + table->elem_type = declare_elem_type; + table->init_size = declare_init_size; + table->flags = declare_max_size_flag; + table->max_size = declare_max_size; + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table->elem_type == VALUE_TYPE_EXTERNREF) + parent_module->is_ref_types_used = true; +#endif + (void)parent_module; + return true; +fail: + return false; +} + +static bool +check_memory_init_size(bool is_memory64, uint32 init_size, char *error_buf, + uint32 error_buf_size) +{ + uint32 default_max_size = + is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + + if (!is_memory64 && init_size > default_max_size) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return false; + } +#if WASM_ENABLE_MEMORY64 != 0 + else if (is_memory64 && init_size > default_max_size) { + set_error_buf( + error_buf, error_buf_size, + "memory size must be at most 4,294,967,295 pages (274 Terabyte)"); + return false; + } +#endif + return true; +} + +static bool +check_memory_max_size(bool is_memory64, uint32 init_size, uint32 max_size, + char *error_buf, uint32 error_buf_size) +{ + uint32 default_max_size = + is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + + if (max_size < init_size) { + set_error_buf(error_buf, error_buf_size, + "size minimum must not be greater than maximum"); + return false; + } + + if (!is_memory64 && max_size > default_max_size) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return false; + } +#if WASM_ENABLE_MEMORY64 != 0 + else if (is_memory64 && max_size > default_max_size) { + set_error_buf( + error_buf, error_buf_size, + "memory size must be at most 4,294,967,295 pages (274 Terabyte)"); + return false; + } +#endif + + return true; +} + +static bool +check_memory_flag(const uint8 mem_flag, char *error_buf, uint32 error_buf_size) +{ + /* Check whether certain features indicated by mem_flag are enabled in + * runtime */ + if (mem_flag > MAX_PAGE_COUNT_FLAG) { +#if WASM_ENABLE_SHARED_MEMORY == 0 + if (mem_flag & SHARED_MEMORY_FLAG) { + LOG_VERBOSE("shared memory flag was found, please enable shared " + "memory, lib-pthread or lib-wasi-threads"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } +#endif +#if WASM_ENABLE_MEMORY64 == 0 + if (mem_flag & MEMORY64_FLAG) { + LOG_VERBOSE("memory64 flag was found, please enable memory64"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } +#endif + } + + if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) { + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + else if ((mem_flag & SHARED_MEMORY_FLAG) + && !(mem_flag & MAX_PAGE_COUNT_FLAG)) { + set_error_buf(error_buf, error_buf_size, + "shared memory must have maximum"); + return false; + } + + return true; +} + +static bool +load_memory_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, const char *sub_module_name, + const char *memory_name, WASMMemoryImport *memory, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; +#if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); + uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT + / DEFAULT_NUM_BYTES_PER_PAGE; +#else + uint32 max_page_count; +#endif /* WASM_ENABLE_APP_FRAMEWORK */ + uint32 mem_flag = 0; + bool is_memory64 = false; + uint32 declare_init_page_count = 0; + uint32 declare_max_page_count = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; + WASMMemory *linked_memory = NULL; +#endif + + p_org = p; + read_leb_uint32(p, p_end, mem_flag); + is_memory64 = mem_flag & MEMORY64_FLAG; + if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + + if (!check_memory_flag(mem_flag, error_buf, error_buf_size)) { + return false; + } + + read_leb_uint32(p, p_end, declare_init_page_count); + if (!check_memory_init_size(is_memory64, declare_init_page_count, error_buf, + error_buf_size)) { + return false; + } + +#if WASM_ENABLE_APP_FRAMEWORK == 0 + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + if (mem_flag & MAX_PAGE_COUNT_FLAG) { + read_leb_uint32(p, p_end, declare_max_page_count); + if (!check_memory_max_size(is_memory64, declare_init_page_count, + declare_max_page_count, error_buf, + error_buf_size)) { + return false; + } + if (declare_max_page_count > max_page_count) { + declare_max_page_count = max_page_count; + } + } + else { + /* Limit the maximum memory size to max_page_count */ + declare_max_page_count = max_page_count; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (!sub_module) { +#if WASM_ENABLE_LIB_WASI_THREADS != 0 + /* Avoid memory import failure when wasi-threads is enabled + and the memory is shared */ + if (!(mem_flag & SHARED_MEMORY_FLAG)) + return false; +#else + return false; +#endif /* WASM_ENABLE_LIB_WASI_THREADS */ + } + else { + linked_memory = wasm_loader_resolve_memory( + sub_module_name, memory_name, declare_init_page_count, + declare_max_page_count, error_buf, error_buf_size); + if (!linked_memory) { + return false; + } + + /** + * reset with linked memory limit + */ + memory->import_module = sub_module; + memory->import_memory_linked = linked_memory; + declare_init_page_count = linked_memory->init_page_count; + declare_max_page_count = linked_memory->max_page_count; + } + } +#endif + + /* (memory (export "memory") 1 2) */ + if (!strcmp("spectest", sub_module_name)) { + uint32 spectest_memory_init_page = 1; + uint32 spectest_memory_max_page = 2; + + if (strcmp("memory", memory_name)) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type or unknown import"); + return false; + } + + if (declare_init_page_count > spectest_memory_init_page + || declare_max_page_count < spectest_memory_max_page) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + + declare_init_page_count = spectest_memory_init_page; + declare_max_page_count = spectest_memory_max_page; + } + + /* now we believe all declaration are ok */ + memory->flags = mem_flag; + memory->init_page_count = declare_init_page_count; + memory->max_page_count = declare_max_page_count; + memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + + *p_buf = p; + + (void)parent_module; + return true; +fail: + return false; +} + +#if WASM_ENABLE_TAGS != 0 +static bool +load_tag_import(const uint8 **p_buf, const uint8 *buf_end, + const WASMModule *parent_module, /* this module ! */ + const char *sub_module_name, const char *tag_name, + WASMTagImport *tag, /* structure to fill */ + char *error_buf, uint32 error_buf_size) +{ + /* attribute and type of the import statement */ + uint8 declare_tag_attribute; + uint32 declare_type_index; + const uint8 *p = *p_buf, *p_end = buf_end; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; +#endif + + /* get the one byte attribute */ + CHECK_BUF(p, p_end, 1); + declare_tag_attribute = read_uint8(p); + if (declare_tag_attribute != 0) { + set_error_buf(error_buf, error_buf_size, "unknown tag attribute"); + goto fail; + } + + /* get type */ + read_leb_uint32(p, p_end, declare_type_index); + /* compare against module->types */ + if (declare_type_index >= parent_module->type_count) { + set_error_buf(error_buf, error_buf_size, "unknown tag type"); + goto fail; + } + + WASMType *declare_tag_type = parent_module->types[declare_type_index]; + + /* check, that the type of the declared tag returns void */ + if (declare_tag_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "tag type signature does not return void"); + + goto fail; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (!sub_module) { + return false; + } + /* wasm_loader_resolve_tag checks, that the imported tag + * and the declared tag have the same type + */ + uint32 linked_tag_index = 0; + WASMTag *linked_tag = wasm_loader_resolve_tag( + sub_module_name, tag_name, declare_tag_type, + &linked_tag_index /* out */, error_buf, error_buf_size); + if (linked_tag) { + tag->import_module = sub_module; + tag->import_tag_linked = linked_tag; + tag->import_tag_index_linked = linked_tag_index; + } + } +#endif + /* store to module tag declarations */ + tag->attribute = declare_tag_attribute; + tag->type = declare_type_index; + + tag->module_name = (char *)sub_module_name; + tag->field_name = (char *)tag_name; + tag->tag_type = declare_tag_type; + + *p_buf = p; + (void)parent_module; + + LOG_VERBOSE("Load tag import success\n"); + + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +static bool +load_global_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, char *sub_module_name, + char *global_name, WASMGlobalImport *global, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 declare_type = 0; + uint8 declare_mutable = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; + WASMGlobal *linked_global = NULL; +#endif +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; + bool need_ref_type_map; +#endif + bool ret = false; + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 2); + declare_type = read_uint8(p); + declare_mutable = read_uint8(p); +#else + if (!resolve_value_type(&p, p_end, parent_module, parent_module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + declare_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(global->ref_type = + reftype_set_insert(parent_module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("import global type: "); + wasm_dump_value_type(declare_type, global->ref_type); + os_printf("\n"); +#endif + CHECK_BUF(p, p_end, 1); + declare_mutable = read_uint8(p); +#endif /* end of WASM_ENABLE_GC == 0 */ + + *p_buf = p; + + if (!check_mutability(declare_mutable, error_buf, error_buf_size)) { + return false; + } + +#if WASM_ENABLE_LIBC_BUILTIN != 0 + ret = wasm_native_lookup_libc_builtin_global(sub_module_name, global_name, + global); + if (ret) { + if (global->type != declare_type + || global->is_mutable != declare_mutable) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + global->is_linked = true; + } +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!global->is_linked + && !wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (!sub_module) { + return false; + } + + /* check sub modules */ + linked_global = wasm_loader_resolve_global( + sub_module_name, global_name, declare_type, declare_mutable, + error_buf, error_buf_size); + if (linked_global) { + global->import_module = sub_module; + global->import_global_linked = linked_global; + global->is_linked = true; + } + } +#endif + + global->module_name = sub_module_name; + global->field_name = global_name; + global->type = declare_type; + global->is_mutable = (declare_mutable == 1); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (global->type == VALUE_TYPE_V128) + parent_module->is_simd_used = true; + else if (global->type == VALUE_TYPE_EXTERNREF) + parent_module->is_ref_types_used = true; +#endif + (void)parent_module; + (void)ret; + return true; +fail: + return false; +} + +static bool +load_table(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, + WASMTable *table, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; + bool need_ref_type_map; +#endif + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + /* 0x70 or 0x6F */ + table->elem_type = read_uint8(p); + if (VALUE_TYPE_FUNCREF != table->elem_type +#if WASM_ENABLE_REF_TYPES != 0 + && VALUE_TYPE_EXTERNREF != table->elem_type +#endif + ) { + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return false; + } +#else /* else of WASM_ENABLE_GC == 0 */ + if (!resolve_value_type(&p, p_end, module, module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + table->elem_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(table->elem_ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, error_buf, + error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("table type: "); + wasm_dump_value_type(table->elem_type, table->elem_ref_type); + os_printf("\n"); +#endif +#endif /* end of WASM_ENABLE_GC == 0 */ + + p_org = p; + read_leb_uint32(p, p_end, table->flags); +#if WASM_ENABLE_SHARED_MEMORY == 0 + if (p - p_org > 1) { + set_error_buf(error_buf, error_buf_size, + "integer representation too long"); + return false; + } + if (table->flags > 1) { + set_error_buf(error_buf, error_buf_size, "integer too large"); + return false; + } +#else + if (p - p_org > 1) { + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + if (table->flags == 2) { + set_error_buf(error_buf, error_buf_size, "tables cannot be shared"); + return false; + } + if (table->flags > 1) { + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } +#endif + + read_leb_uint32(p, p_end, table->init_size); + + if (table->flags) { + read_leb_uint32(p, p_end, table->max_size); + if (!check_table_max_size(table->init_size, table->max_size, error_buf, + error_buf_size)) + return false; + } + + adjust_table_max_size(table->init_size, table->flags, &table->max_size); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table->elem_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + + *p_buf = p; + return true; +fail: + return false; +} + +static bool +load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; +#if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); + uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT + / DEFAULT_NUM_BYTES_PER_PAGE; +#else + uint32 max_page_count; +#endif + bool is_memory64 = false; + + p_org = p; + read_leb_uint32(p, p_end, memory->flags); + is_memory64 = memory->flags & MEMORY64_FLAG; + if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + + if (!check_memory_flag(memory->flags, error_buf, error_buf_size)) { + return false; + } + + read_leb_uint32(p, p_end, memory->init_page_count); + if (!check_memory_init_size(is_memory64, memory->init_page_count, error_buf, + error_buf_size)) + return false; + +#if WASM_ENABLE_APP_FRAMEWORK == 0 + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + if (memory->flags & 1) { + read_leb_uint32(p, p_end, memory->max_page_count); + if (!check_memory_max_size(is_memory64, memory->init_page_count, + memory->max_page_count, error_buf, + error_buf_size)) + return false; + if (memory->max_page_count > max_page_count) + memory->max_page_count = max_page_count; + } + else { + /* Limit the maximum memory size to max_page_count */ + memory->max_page_count = max_page_count; + } + + memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + + *p_buf = p; + return true; +fail: + return false; +} + +static bool +load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end, *p_old; + uint32 import_count, name_len, type_index, i, u32, flags; + uint64 total_size; + WASMImport *import; + WASMImport *import_functions = NULL, *import_tables = NULL; + WASMImport *import_memories = NULL, *import_globals = NULL; +#if WASM_ENABLE_TAGS != 0 + WASMImport *import_tags = NULL; +#endif + char *sub_module_name, *field_name; + uint8 u8, kind, global_type; + + read_leb_uint32(p, p_end, import_count); + + if (import_count) { + module->import_count = import_count; + total_size = sizeof(WASMImport) * (uint64)import_count; + if (!(module->imports = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + p_old = p; + + /* Scan firstly to get import count of each type */ + for (i = 0; i < import_count; i++) { + /* module name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + p += name_len; + + /* field name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + p += name_len; + + CHECK_BUF(p, p_end, 1); + /* 0x00/0x01/0x02/0x03/0x04 */ + kind = read_uint8(p); + + switch (kind) { + case IMPORT_KIND_FUNC: /* import function */ + read_leb_uint32(p, p_end, type_index); + module->import_function_count++; + break; + + case IMPORT_KIND_TABLE: /* import table */ + CHECK_BUF(p, p_end, 1); + /* 0x70 */ + u8 = read_uint8(p); + read_leb_uint32(p, p_end, flags); + read_leb_uint32(p, p_end, u32); + if (flags & 1) + read_leb_uint32(p, p_end, u32); + module->import_table_count++; + + if (module->import_table_count > 1) { +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + set_error_buf(error_buf, error_buf_size, + "multiple tables"); + return false; +#elif WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + } + break; + + case IMPORT_KIND_MEMORY: /* import memory */ + read_leb_uint32(p, p_end, flags); + read_leb_uint32(p, p_end, u32); + if (flags & 1) + read_leb_uint32(p, p_end, u32); + module->import_memory_count++; + if (module->import_memory_count > 1) { + set_error_buf(error_buf, error_buf_size, + "multiple memories"); + return false; + } + break; + +#if WASM_ENABLE_TAGS != 0 + case IMPORT_KIND_TAG: /* import tags */ + /* it only counts the number of tags to import */ + module->import_tag_count++; + CHECK_BUF(p, p_end, 1); + u8 = read_uint8(p); + read_leb_uint32(p, p_end, type_index); + break; +#endif + + case IMPORT_KIND_GLOBAL: /* import global */ +#if WASM_ENABLE_GC != 0 + /* valtype */ + CHECK_BUF(p, p_end, 1); + global_type = read_uint8(p); + if (wasm_is_type_multi_byte_type(global_type)) { + int32 heap_type; + read_leb_int32(p, p_end, heap_type); + (void)heap_type; + } + + /* mutability */ + CHECK_BUF(p, p_end, 1); + p += 1; +#else + CHECK_BUF(p, p_end, 2); + p += 2; +#endif + + (void)global_type; + module->import_global_count++; + break; + + default: + set_error_buf(error_buf, error_buf_size, + "invalid import kind"); + return false; + } + } + + if (module->import_function_count) + import_functions = module->import_functions = module->imports; + if (module->import_table_count) + import_tables = module->import_tables = + module->imports + module->import_function_count; + if (module->import_memory_count) + import_memories = module->import_memories = + module->imports + module->import_function_count + + module->import_table_count; + +#if WASM_ENABLE_TAGS != 0 + if (module->import_tag_count) + import_tags = module->import_tags = + module->imports + module->import_function_count + + module->import_table_count + module->import_memory_count; + if (module->import_global_count) + import_globals = module->import_globals = + module->imports + module->import_function_count + + module->import_table_count + module->import_memory_count + + module->import_tag_count; +#else + if (module->import_global_count) + import_globals = module->import_globals = + module->imports + module->import_function_count + + module->import_table_count + module->import_memory_count; +#endif + + p = p_old; + + /* Scan again to resolve the data */ + for (i = 0; i < import_count; i++) { + /* load module name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + if (!(sub_module_name = wasm_const_str_list_insert( + p, name_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + p += name_len; + + /* load field name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + if (!(field_name = wasm_const_str_list_insert( + p, name_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + p += name_len; + + CHECK_BUF(p, p_end, 1); + /* 0x00/0x01/0x02/0x03/0x4 */ + kind = read_uint8(p); + + switch (kind) { + case IMPORT_KIND_FUNC: /* import function */ + bh_assert(import_functions); + import = import_functions++; + if (!load_function_import( + &p, p_end, module, sub_module_name, field_name, + &import->u.function, error_buf, error_buf_size)) { + return false; + } + break; + + case IMPORT_KIND_TABLE: /* import table */ + bh_assert(import_tables); + import = import_tables++; + if (!load_table_import(&p, p_end, module, sub_module_name, + field_name, &import->u.table, + error_buf, error_buf_size)) { + LOG_DEBUG("can not import such a table (%s,%s)", + sub_module_name, field_name); + return false; + } + break; + + case IMPORT_KIND_MEMORY: /* import memory */ + bh_assert(import_memories); + import = import_memories++; + if (!load_memory_import(&p, p_end, module, sub_module_name, + field_name, &import->u.memory, + error_buf, error_buf_size)) { + return false; + } + break; + +#if WASM_ENABLE_TAGS != 0 + case IMPORT_KIND_TAG: + bh_assert(import_tags); + import = import_tags++; + if (!load_tag_import(&p, p_end, module, sub_module_name, + field_name, &import->u.tag, error_buf, + error_buf_size)) { + return false; + } + break; +#endif + + case IMPORT_KIND_GLOBAL: /* import global */ + bh_assert(import_globals); + import = import_globals++; + if (!load_global_import(&p, p_end, module, sub_module_name, + field_name, &import->u.global, + error_buf, error_buf_size)) { + return false; + } + break; + + default: + set_error_buf(error_buf, error_buf_size, + "invalid import kind"); + return false; + } + import->kind = kind; + import->u.names.module_name = sub_module_name; + import->u.names.field_name = field_name; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + import = module->import_functions; + for (i = 0; i < module->import_function_count; i++, import++) { + if (!strcmp(import->u.names.module_name, "wasi_unstable") + || !strcmp(import->u.names.module_name, + "wasi_snapshot_preview1")) { + module->import_wasi_api = true; + break; + } + } +#endif + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load import section success.\n"); + (void)u8; + (void)u32; + (void)type_index; + return true; +fail: + return false; +} + +static bool +init_function_local_offsets(WASMFunction *func, char *error_buf, + uint32 error_buf_size) +{ + WASMFuncType *param_type = func->func_type; + uint32 param_count = param_type->param_count; + uint8 *param_types = param_type->types; + uint32 local_count = func->local_count; + uint8 *local_types = func->local_types; + uint32 i, local_offset = 0; + uint64 total_size = sizeof(uint16) * ((uint64)param_count + local_count); + + /* + * Only allocate memory when total_size is not 0, + * or the return value of malloc(0) might be NULL on some platforms, + * which causes wasm loader return false. + */ + if (total_size > 0 + && !(func->local_offsets = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < param_count; i++) { + func->local_offsets[i] = (uint16)local_offset; + local_offset += wasm_value_type_cell_num(param_types[i]); + } + + for (i = 0; i < local_count; i++) { + func->local_offsets[param_count + i] = (uint16)local_offset; + local_offset += wasm_value_type_cell_num(local_types[i]); + } + + bh_assert(local_offset == func->param_cell_num + func->local_cell_num); + return true; +} + +static bool +load_function_section(const uint8 *buf, const uint8 *buf_end, + const uint8 *buf_code, const uint8 *buf_code_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + const uint8 *p_code = buf_code, *p_code_end, *p_code_save; + uint32 func_count; + uint64 total_size; + uint32 code_count = 0, code_size, type_index, i, j, k, local_type_index; + uint32 local_count, local_set_count, sub_local_count, local_cell_num; + uint8 type; + WASMFunction *func; +#if WASM_ENABLE_GC != 0 + bool need_ref_type_map; + WASMRefType ref_type; + uint32 ref_type_map_count = 0, t = 0, type_index_org; +#endif + + read_leb_uint32(p, p_end, func_count); + + if (buf_code) + read_leb_uint32(p_code, buf_code_end, code_count); + + if (func_count != code_count) { + set_error_buf(error_buf, error_buf_size, + "function and code section have inconsistent lengths or " + "unexpected end"); + return false; + } + + if (func_count) { + module->function_count = func_count; + total_size = sizeof(WASMFunction *) * (uint64)func_count; + if (!(module->functions = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < func_count; i++) { + /* Resolve function type */ + read_leb_uint32(p, p_end, type_index); + if (type_index >= module->type_count) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + return false; + } + +#if WASM_ENABLE_GC != 0 + type_index_org = type_index; +#endif + +#if (WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0) \ + && WASM_ENABLE_GC == 0 + type_index = wasm_get_smallest_type_idx( + module->types, module->type_count, type_index); +#endif + + read_leb_uint32(p_code, buf_code_end, code_size); + if (code_size == 0 || p_code + code_size > buf_code_end) { + set_error_buf(error_buf, error_buf_size, + "invalid function code size"); + return false; + } + + /* Resolve local set count */ + p_code_end = p_code + code_size; + local_count = 0; + read_leb_uint32(p_code, buf_code_end, local_set_count); + p_code_save = p_code; + +#if WASM_ENABLE_GC != 0 + ref_type_map_count = 0; +#endif + + /* Calculate total local count */ + for (j = 0; j < local_set_count; j++) { + read_leb_uint32(p_code, buf_code_end, sub_local_count); + if (sub_local_count > UINT32_MAX - local_count) { + set_error_buf(error_buf, error_buf_size, "too many locals"); + return false; + } +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p_code, buf_code_end, 1); + /* 0x7F/0x7E/0x7D/0x7C */ + type = read_uint8(p_code); + local_count += sub_local_count; +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* If any value's type is v128, mark the module as SIMD used */ + if (type == VALUE_TYPE_V128) + module->is_simd_used = true; +#endif +#else + if (!resolve_value_type(&p_code, buf_code_end, module, + module->type_count, &need_ref_type_map, + &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + local_count += sub_local_count; + if (need_ref_type_map) + ref_type_map_count += sub_local_count; +#endif + } + + /* Alloc memory, layout: function structure + local types */ + code_size = (uint32)(p_code_end - p_code); + + total_size = sizeof(WASMFunction) + (uint64)local_count; + if (!(func = module->functions[i] = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } +#if WASM_ENABLE_GC != 0 + if (ref_type_map_count > 0) { + total_size = + sizeof(WASMRefTypeMap) * (uint64)ref_type_map_count; + if (!(func->local_ref_type_maps = loader_malloc( + total_size, error_buf, error_buf_size))) { + return false; + } + func->local_ref_type_map_count = ref_type_map_count; + } +#endif + + /* Set function type, local count, code size and code body */ + func->func_type = (WASMFuncType *)module->types[type_index]; + func->local_count = local_count; + if (local_count > 0) + func->local_types = (uint8 *)func + sizeof(WASMFunction); + func->code_size = code_size; + /* + * we shall make a copy of code body [p_code, p_code + code_size] + * when we are worrying about inappropriate releasing behaviour. + * all code bodies are actually in a buffer which user allocates in + * his embedding environment and we don't have power on them. + * it will be like: + * code_body_cp = malloc(code_size); + * memcpy(code_body_cp, p_code, code_size); + * func->code = code_body_cp; + */ + func->code = (uint8 *)p_code; +#if WASM_ENABLE_GC != 0 + func->type_idx = type_index_org; +#endif + +#if WASM_ENABLE_GC != 0 + t = 0; +#endif + + /* Load each local type */ + p_code = p_code_save; + local_type_index = 0; + for (j = 0; j < local_set_count; j++) { + read_leb_uint32(p_code, buf_code_end, sub_local_count); + /* Note: sub_local_count is allowed to be 0 */ + if (local_type_index > UINT32_MAX - sub_local_count + || local_type_index + sub_local_count > local_count) { + set_error_buf(error_buf, error_buf_size, + "invalid local count"); + return false; + } +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p_code, buf_code_end, 1); + /* 0x7F/0x7E/0x7D/0x7C */ + type = read_uint8(p_code); + if (!is_value_type(type)) { + if (type == VALUE_TYPE_V128) + set_error_buf(error_buf, error_buf_size, + "v128 value type requires simd feature"); + else if (type == VALUE_TYPE_FUNCREF + || type == VALUE_TYPE_EXTERNREF) + set_error_buf(error_buf, error_buf_size, + "ref value type requires " + "reference types feature"); + else + set_error_buf_v(error_buf, error_buf_size, + "invalid local type 0x%02X", type); + return false; + } +#else + if (!resolve_value_type(&p_code, buf_code_end, module, + module->type_count, &need_ref_type_map, + &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) { + WASMRefType *ref_type_tmp; + if (!(ref_type_tmp = reftype_set_insert( + module->ref_type_set, &ref_type, error_buf, + error_buf_size))) { + return false; + } + for (k = 0; k < sub_local_count; k++) { + func->local_ref_type_maps[t + k].ref_type = + ref_type_tmp; + func->local_ref_type_maps[t + k].index = + local_type_index + k; + } + t += sub_local_count; + } + type = ref_type.ref_type; +#endif + for (k = 0; k < sub_local_count; k++) { + func->local_types[local_type_index++] = type; + } +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (type == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (type == VALUE_TYPE_FUNCREF + || type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + } + + bh_assert(local_type_index == func->local_count); +#if WASM_ENABLE_GC != 0 + bh_assert(t == func->local_ref_type_map_count); +#if TRACE_WASM_LOADER != 0 + os_printf("func %u, local types: [", i); + k = 0; + for (j = 0; j < func->local_count; j++) { + WASMRefType *ref_type_tmp = NULL; + if (wasm_is_type_multi_byte_type(func->local_types[j])) { + bh_assert(j == func->local_ref_type_maps[k].index); + ref_type_tmp = func->local_ref_type_maps[k++].ref_type; + } + wasm_dump_value_type(func->local_types[j], ref_type_tmp); + if (j < func->local_count - 1) + os_printf(" "); + } + os_printf("]\n"); +#endif +#endif + + func->param_cell_num = func->func_type->param_cell_num; + func->ret_cell_num = func->func_type->ret_cell_num; + local_cell_num = + wasm_get_cell_num(func->local_types, func->local_count); + + if (local_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "local count too large"); + return false; + } + + func->local_cell_num = (uint16)local_cell_num; + + if (!init_function_local_offsets(func, error_buf, error_buf_size)) + return false; + + p_code = p_code_end; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load function section success.\n"); + return true; +fail: + return false; +} + +static bool +load_table_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 table_count, i; + uint64 total_size; + WASMTable *table; + + read_leb_uint32(p, p_end, table_count); + if (module->import_table_count + table_count > 1) { +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + /* a total of one table is allowed */ + set_error_buf(error_buf, error_buf_size, "multiple tables"); + return false; +#elif WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + } + + if (table_count) { + module->table_count = table_count; + total_size = sizeof(WASMTable) * (uint64)table_count; + if (!(module->tables = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* load each table */ + table = module->tables; + for (i = 0; i < table_count; i++, table++) { +#if WASM_ENABLE_GC != 0 + uint8 flag; + bool has_init = false; + + CHECK_BUF(buf, buf_end, 1); + flag = read_uint8(p); + + if (flag == TABLE_INIT_EXPR_FLAG) { + CHECK_BUF(buf, buf_end, 1); + flag = read_uint8(p); + + if (flag != 0x00) { + set_error_buf(error_buf, error_buf_size, + "invalid leading bytes for table"); + return false; + } + has_init = true; + } + else { + p--; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + if (!load_table(&p, p_end, module, table, error_buf, + error_buf_size)) + return false; + +#if WASM_ENABLE_GC != 0 + if (has_init) { + if (!load_init_expr(module, &p, p_end, &table->init_expr, + table->elem_type, table->elem_ref_type, + error_buf, error_buf_size)) + return false; + if (table->init_expr.init_expr_type >= INIT_EXPR_TYPE_STRUCT_NEW + && table->init_expr.init_expr_type + <= INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + set_error_buf( + error_buf, error_buf_size, + "unsupported initializer expression for table"); + return false; + } + } + else { + if (wasm_is_reftype_htref_non_nullable(table->elem_type)) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: non-nullable table without init expr"); + return false; + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table->elem_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load table section success.\n"); + return true; +fail: + return false; +} + +static bool +load_memory_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 memory_count, i; + uint64 total_size; + WASMMemory *memory; + + read_leb_uint32(p, p_end, memory_count); + /* a total of one memory is allowed */ + if (module->import_memory_count + memory_count > 1) { + set_error_buf(error_buf, error_buf_size, "multiple memories"); + return false; + } + + if (memory_count) { + module->memory_count = memory_count; + total_size = sizeof(WASMMemory) * (uint64)memory_count; + if (!(module->memories = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* load each memory */ + memory = module->memories; + for (i = 0; i < memory_count; i++, memory++) + if (!load_memory(&p, p_end, memory, error_buf, error_buf_size)) + return false; + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load memory section success.\n"); + return true; +fail: + return false; +} + +static bool +load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 global_count, i; + uint64 total_size; + WASMGlobal *global; + uint8 mutable; +#if WASM_ENABLE_GC != 0 + bool need_ref_type_map; + WASMRefType ref_type; +#endif + + read_leb_uint32(p, p_end, global_count); + + module->global_count = 0; + if (global_count) { + total_size = sizeof(WASMGlobal) * (uint64)global_count; + if (!(module->globals = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + global = module->globals; + + for (i = 0; i < global_count; i++, global++) { +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 2); + global->type = read_uint8(p); + mutable = read_uint8(p); +#else + if (!resolve_value_type(&p, p_end, module, module->type_count, + &need_ref_type_map, &ref_type, false, + error_buf, error_buf_size)) { + return false; + } + global->type = ref_type.ref_type; + CHECK_BUF(p, p_end, 1); + mutable = read_uint8(p); +#endif /* end of WASM_ENABLE_GC */ + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (global->type == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (global->type == VALUE_TYPE_FUNCREF + || global->type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + + if (!check_mutability(mutable, error_buf, error_buf_size)) { + return false; + } + global->is_mutable = mutable ? true : false; + + /* initialize expression */ + if (!load_init_expr(module, &p, p_end, &(global->init_expr), + global->type, +#if WASM_ENABLE_GC == 0 + NULL, +#else + &ref_type, +#endif + error_buf, error_buf_size)) + return false; + +#if WASM_ENABLE_GC != 0 + if (global->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { + uint8 global_type; + WASMRefType *global_ref_type; + uint32 global_idx = global->init_expr.u.global_index; + + if (global->init_expr.u.global_index + >= module->import_global_count + i) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + return false; + } + + if (global_idx < module->import_global_count) { + global_type = + module->import_globals[global_idx].u.global.type; + global_ref_type = + module->import_globals[global_idx].u.global.ref_type; + } + else { + global_type = + module + ->globals[global_idx - module->import_global_count] + .type; + global_ref_type = + module + ->globals[global_idx - module->import_global_count] + .ref_type; + } + if (!wasm_reftype_is_subtype_of( + global_type, global_ref_type, global->type, + global->ref_type, module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + } + + if (need_ref_type_map) { + if (!(global->ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("global type: "); + wasm_dump_value_type(global->type, global->ref_type); + os_printf("\n"); +#endif +#endif + module->global_count++; + } + bh_assert(module->global_count == global_count); + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load global section success.\n"); + return true; +fail: + return false; +} + +static bool +load_export_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 export_count, i, j, index; + uint64 total_size; + uint32 str_len; + WASMExport *export; + const char *name; + + read_leb_uint32(p, p_end, export_count); + + if (export_count) { + module->export_count = export_count; + total_size = sizeof(WASMExport) * (uint64)export_count; + if (!(module->exports = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + export = module->exports; + for (i = 0; i < export_count; i++, export ++) { +#if WASM_ENABLE_THREAD_MGR == 0 + if (p == p_end) { + /* export section with inconsistent count: + n export declared, but less than n given */ + set_error_buf(error_buf, error_buf_size, + "length out of bounds"); + return false; + } +#endif + read_leb_uint32(p, p_end, str_len); + CHECK_BUF(p, p_end, str_len); + + for (j = 0; j < i; j++) { + name = module->exports[j].name; + if (strlen(name) == str_len && memcmp(name, p, str_len) == 0) { + set_error_buf(error_buf, error_buf_size, + "duplicate export name"); + return false; + } + } + + if (!(export->name = wasm_const_str_list_insert( + p, str_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + + p += str_len; + CHECK_BUF(p, p_end, 1); + export->kind = read_uint8(p); + read_leb_uint32(p, p_end, index); + export->index = index; + + switch (export->kind) { + /* function index */ + case EXPORT_KIND_FUNC: + if (index >= module->function_count + + module->import_function_count) { + set_error_buf(error_buf, error_buf_size, + "unknown function"); + return false; + } +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + /* TODO: check func type, if it has v128 param or result, + report error */ +#endif +#endif + break; + /* table index */ + case EXPORT_KIND_TABLE: + if (index + >= module->table_count + module->import_table_count) { + set_error_buf(error_buf, error_buf_size, + "unknown table"); + return false; + } + break; + /* memory index */ + case EXPORT_KIND_MEMORY: + if (index + >= module->memory_count + module->import_memory_count) { + set_error_buf(error_buf, error_buf_size, + "unknown memory"); + return false; + } + break; +#if WASM_ENABLE_TAGS != 0 + /* export tag */ + case EXPORT_KIND_TAG: + if (index >= module->tag_count + module->import_tag_count) { + set_error_buf(error_buf, error_buf_size, "unknown tag"); + return false; + } + break; +#endif + + /* global index */ + case EXPORT_KIND_GLOBAL: + if (index + >= module->global_count + module->import_global_count) { + set_error_buf(error_buf, error_buf_size, + "unknown global"); + return false; + } + break; + + default: + set_error_buf(error_buf, error_buf_size, + "invalid export kind"); + return false; + } + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load export section success.\n"); + return true; +fail: + return false; +} + +static bool +check_table_index(const WASMModule *module, uint32 table_index, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + if (table_index != 0) { + set_error_buf(error_buf, error_buf_size, "zero byte expected"); + return false; + } +#endif + + if (table_index >= module->import_table_count + module->table_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown table %d", + table_index); + return false; + } + return true; +} + +static bool +load_table_index(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, + uint32 *p_table_index, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 table_index; + + read_leb_uint32(p, p_end, table_index); + if (!check_table_index(module, table_index, error_buf, error_buf_size)) { + return false; + } + + *p_table_index = table_index; + *p_buf = p; + return true; +fail: + return false; +} + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +static bool +load_elem_type(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, + uint32 *p_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **p_elem_ref_type, +#endif + bool elemkind_zero, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 elem_type; +#if WASM_ENABLE_GC != 0 + WASMRefType elem_ref_type; + bool need_ref_type_map; +#endif + + CHECK_BUF(p, p_end, 1); + elem_type = read_uint8(p); + if (elemkind_zero) { + if (elem_type != 0) { + set_error_buf(error_buf, error_buf_size, + "invalid reference type or unknown type"); + return false; + } + else { + *p_elem_type = VALUE_TYPE_FUNCREF; + *p_buf = p; + return true; + } + } + +#if WASM_ENABLE_GC == 0 + if (elem_type != VALUE_TYPE_FUNCREF && elem_type != VALUE_TYPE_EXTERNREF) { + set_error_buf(error_buf, error_buf_size, + "invalid reference type or unknown type"); + return false; + } + *p_elem_type = elem_type; +#else + p--; + if (!resolve_value_type((const uint8 **)&p, p_end, module, + module->type_count, &need_ref_type_map, + &elem_ref_type, false, error_buf, error_buf_size)) { + return false; + } + if (!wasm_is_type_reftype(elem_ref_type.ref_type)) { + set_error_buf(error_buf, error_buf_size, + "invalid reference type or unknown type"); + return false; + } + *p_elem_type = elem_ref_type.ref_type; + if (need_ref_type_map) { + if (!(*p_elem_ref_type = + reftype_set_insert(module->ref_type_set, &elem_ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#endif + + *p_buf = p; + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +static bool +load_func_index_vec(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, WASMTableSeg *table_segment, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 function_count, function_index = 0, i; + uint64 total_size; + + read_leb_uint32(p, p_end, function_count); + table_segment->value_count = function_count; + total_size = sizeof(InitializerExpression) * (uint64)function_count; + if (total_size > 0 + && !(table_segment->init_values = + (InitializerExpression *)loader_malloc(total_size, error_buf, + error_buf_size))) { + return false; + } + + for (i = 0; i < function_count; i++) { + InitializerExpression *init_expr = &table_segment->init_values[i]; + + read_leb_uint32(p, p_end, function_index); + if (!check_function_index(module, function_index, error_buf, + error_buf_size)) { + return false; + } + + init_expr->init_expr_type = INIT_EXPR_TYPE_FUNCREF_CONST; + init_expr->u.ref_index = function_index; + } + + *p_buf = p; + return true; +fail: + return false; +} + +#if (WASM_ENABLE_GC != 0) || (WASM_ENABLE_REF_TYPES != 0) +static bool +load_init_expr_vec(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, WASMTableSeg *table_segment, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 ref_count, i; + uint64 total_size; + + read_leb_uint32(p, p_end, ref_count); + table_segment->value_count = ref_count; + total_size = sizeof(InitializerExpression) * (uint64)ref_count; + if (total_size > 0 + && !(table_segment->init_values = + (InitializerExpression *)loader_malloc(total_size, error_buf, + error_buf_size))) { + return false; + } + + for (i = 0; i < ref_count; i++) { + InitializerExpression *init_expr = &table_segment->init_values[i]; + + if (!load_init_expr(module, &p, p_end, init_expr, + table_segment->elem_type, +#if WASM_ENABLE_GC == 0 + NULL, +#else + table_segment->elem_ref_type, +#endif + error_buf, error_buf_size)) + return false; + + bh_assert((init_expr->init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) + || (init_expr->init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST) + || (init_expr->init_expr_type >= INIT_EXPR_TYPE_FUNCREF_CONST + && init_expr->init_expr_type + <= INIT_EXPR_TYPE_ARRAY_NEW_FIXED)); + } + + *p_buf = p; + return true; +fail: + return false; +} +#endif /* end of (WASM_ENABLE_GC != 0) || (WASM_ENABLE_REF_TYPES != 0) */ + +static bool +load_table_segment_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 table_segment_count, i; + uint64 total_size; + WASMTableSeg *table_segment; + + read_leb_uint32(p, p_end, table_segment_count); + + if (table_segment_count) { + module->table_seg_count = table_segment_count; + total_size = sizeof(WASMTableSeg) * (uint64)table_segment_count; + if (!(module->table_segments = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + table_segment = module->table_segments; + for (i = 0; i < table_segment_count; i++, table_segment++) { + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, + "invalid value type or " + "invalid elements segment kind"); + return false; + } + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + read_leb_uint32(p, p_end, table_segment->mode); + /* last three bits */ + table_segment->mode = table_segment->mode & 0x07; + switch (table_segment->mode) { + /* elemkind/elemtype + active */ + case 0: + case 4: + { +#if WASM_ENABLE_GC != 0 + if (table_segment->mode == 0) { + /* vec(funcidx), set elem type to (ref func) */ + WASMRefType elem_ref_type = { 0 }; + table_segment->elem_type = REF_TYPE_HT_NON_NULLABLE; + wasm_set_refheaptype_common( + &elem_ref_type.ref_ht_common, false, + HEAP_TYPE_FUNC); + if (!(table_segment->elem_ref_type = reftype_set_insert( + module->ref_type_set, &elem_ref_type, + error_buf, error_buf_size))) + return false; + } + else { + /* vec(expr), set elem type to funcref */ + table_segment->elem_type = VALUE_TYPE_FUNCREF; + } +#else + table_segment->elem_type = VALUE_TYPE_FUNCREF; +#endif + table_segment->table_index = 0; + + if (!check_table_index(module, table_segment->table_index, + error_buf, error_buf_size)) + return false; + if (!load_init_expr( + module, &p, p_end, &table_segment->base_offset, + VALUE_TYPE_I32, NULL, error_buf, error_buf_size)) + return false; + + if (table_segment->mode == 0) { + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + else { + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + + break; + } + /* elemkind + passive/declarative */ + case 1: + case 3: + if (!load_elem_type(module, &p, p_end, + &table_segment->elem_type, +#if WASM_ENABLE_GC != 0 + &table_segment->elem_ref_type, +#endif + true, error_buf, error_buf_size)) + return false; + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; + break; + /* elemkind/elemtype + table_idx + active */ + case 2: + case 6: + if (!load_table_index(&p, p_end, module, + &table_segment->table_index, + error_buf, error_buf_size)) + return false; + if (!load_init_expr( + module, &p, p_end, &table_segment->base_offset, + VALUE_TYPE_I32, NULL, error_buf, error_buf_size)) + return false; + if (!load_elem_type(module, &p, p_end, + &table_segment->elem_type, +#if WASM_ENABLE_GC != 0 + &table_segment->elem_ref_type, +#endif + table_segment->mode == 2 ? true : false, + error_buf, error_buf_size)) + return false; + + if (table_segment->mode == 2) { + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + else { + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + break; + case 5: + case 7: + if (!load_elem_type(module, &p, p_end, + &table_segment->elem_type, +#if WASM_ENABLE_GC != 0 + &table_segment->elem_ref_type, +#endif + false, error_buf, error_buf_size)) + return false; + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; + break; + default: + set_error_buf(error_buf, error_buf_size, + "unknown element segment kind"); + return false; + } +#else /* else of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + /* + * like: 00 41 05 0b 04 00 01 00 01 + * for: (elem 0 (offset (i32.const 5)) $f1 $f2 $f1 $f2) + */ + if (!load_table_index(&p, p_end, module, + &table_segment->table_index, error_buf, + error_buf_size)) + return false; + if (!load_init_expr(module, &p, p_end, &table_segment->base_offset, + VALUE_TYPE_I32, NULL, error_buf, + error_buf_size)) + return false; + if (!load_func_index_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table_segment->elem_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load table segment section success.\n"); + return true; +fail: + return false; +} + +static bool +load_data_segment_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 data_seg_count, i, mem_index, data_seg_len; + uint64 total_size; + WASMDataSeg *dataseg; + InitializerExpression init_expr; +#if WASM_ENABLE_BULK_MEMORY != 0 + bool is_passive = false; + uint32 mem_flag; +#endif + uint8 mem_offset_type = VALUE_TYPE_I32; + + read_leb_uint32(p, p_end, data_seg_count); + +#if WASM_ENABLE_BULK_MEMORY != 0 + if ((module->data_seg_count1 != 0) + && (data_seg_count != module->data_seg_count1)) { + set_error_buf(error_buf, error_buf_size, + "data count and data section have inconsistent lengths"); + return false; + } +#endif + + if (data_seg_count) { + module->data_seg_count = data_seg_count; + total_size = sizeof(WASMDataSeg *) * (uint64)data_seg_count; + if (!(module->data_segments = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < data_seg_count; i++) { + read_leb_uint32(p, p_end, mem_index); +#if WASM_ENABLE_BULK_MEMORY != 0 + is_passive = false; + mem_flag = mem_index & 0x03; + switch (mem_flag) { + case 0x01: + is_passive = true; +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + case 0x00: + /* no memory index, treat index as 0 */ + mem_index = 0; + goto check_mem_index; + case 0x02: + /* read following memory index */ + read_leb_uint32(p, p_end, mem_index); +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + check_mem_index: + if (mem_index + >= module->import_memory_count + module->memory_count) { + set_error_buf_v(error_buf, error_buf_size, + "unknown memory %d", mem_index); + return false; + } + break; + case 0x03: + default: + set_error_buf(error_buf, error_buf_size, "unknown memory"); + return false; + break; + } +#else + if (mem_index + >= module->import_memory_count + module->memory_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown memory %d", + mem_index); + return false; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif + { +#if WASM_ENABLE_MEMORY64 != 0 + /* This memory_flag is from memory instead of data segment */ + uint8 memory_flag; + if (module->import_memory_count > 0) { + memory_flag = + module->import_memories[mem_index].u.memory.flags; + } + else { + memory_flag = + module + ->memories[mem_index - module->import_memory_count] + .flags; + } + mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif + if (!load_init_expr(module, &p, p_end, &init_expr, + mem_offset_type, NULL, error_buf, + error_buf_size)) + return false; + + read_leb_uint32(p, p_end, data_seg_len); + + if (!(dataseg = module->data_segments[i] = loader_malloc( + sizeof(WASMDataSeg), error_buf, error_buf_size))) { + return false; + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + dataseg->is_passive = is_passive; + if (!is_passive) +#endif + { + bh_memcpy_s(&dataseg->base_offset, + sizeof(InitializerExpression), &init_expr, + sizeof(InitializerExpression)); + + dataseg->memory_index = mem_index; + } + + dataseg->data_length = data_seg_len; + CHECK_BUF(p, p_end, data_seg_len); + dataseg->data = (uint8 *)p; + p += data_seg_len; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load data segment section success.\n"); + return true; +fail: + return false; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +static bool +load_datacount_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 data_seg_count1 = 0; + + read_leb_uint32(p, p_end, data_seg_count1); + module->data_seg_count1 = data_seg_count1; + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + LOG_VERBOSE("Load datacount section success.\n"); + return true; +fail: + return false; +} +#endif + +#if WASM_ENABLE_TAGS != 0 +static bool +load_tag_section(const uint8 *buf, const uint8 *buf_end, const uint8 *buf_code, + const uint8 *buf_code_end, WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + (void)buf_code; + (void)buf_code_end; + + const uint8 *p = buf, *p_end = buf_end; + size_t total_size = 0; + /* number of tags defined in the section */ + uint32 section_tag_count = 0; + uint8 tag_attribute; + uint32 tag_type; + WASMTag *tag = NULL; + + /* get tag count */ + read_leb_uint32(p, p_end, section_tag_count); + module->tag_count = section_tag_count; + + if (section_tag_count) { + total_size = sizeof(WASMTag *) * module->tag_count; + if (!(module->tags = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + /* load each tag, imported tags precede the tags */ + uint32 tag_index; + for (tag_index = 0; tag_index < section_tag_count; tag_index++) { + + /* get the one byte attribute */ + CHECK_BUF(p, p_end, 1); + tag_attribute = read_uint8(p); + + /* get type */ + read_leb_uint32(p, p_end, tag_type); + /* compare against module->types */ + if (tag_type >= module->type_count) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + return false; + } + + /* get return type (must be 0) */ + /* check, that the type of the referred tag returns void */ + WASMType *func_type = (WASMType *)module->types[tag_type]; + if (func_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "non-empty tag result type"); + + goto fail; + } + + if (!(tag = module->tags[tag_index] = loader_malloc( + sizeof(WASMTag), error_buf, error_buf_size))) { + return false; + } + + /* store to module tag declarations */ + tag->attribute = tag_attribute; + tag->type = tag_type; + tag->tag_type = func_type; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load tag section success.\n"); + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +static bool +load_code_section(const uint8 *buf, const uint8 *buf_end, const uint8 *buf_func, + const uint8 *buf_func_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + const uint8 *p_func = buf_func; + uint32 func_count = 0, code_count; + + /* code has been loaded in function section, so pass it here, just check + * whether function and code section have inconsistent lengths */ + read_leb_uint32(p, p_end, code_count); + + if (buf_func) + read_leb_uint32(p_func, buf_func_end, func_count); + + if (func_count != code_count) { + set_error_buf(error_buf, error_buf_size, + "function and code section have inconsistent lengths"); + return false; + } + + LOG_VERBOSE("Load code segment section success.\n"); + (void)module; + return true; +fail: + return false; +} + +static bool +load_start_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + WASMFuncType *type; + uint32 start_function; + + read_leb_uint32(p, p_end, start_function); + + if (start_function + >= module->function_count + module->import_function_count) { + set_error_buf(error_buf, error_buf_size, "unknown function"); + return false; + } + + if (start_function < module->import_function_count) + type = module->import_functions[start_function].u.function.func_type; + else + type = module->functions[start_function - module->import_function_count] + ->func_type; + if (type->param_count != 0 || type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, "invalid start function"); + return false; + } + + module->start_function = start_function; + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load start section success.\n"); + return true; +fail: + return false; +} + +#if WASM_ENABLE_STRINGREF != 0 +static bool +load_stringref_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + int32 deferred_count, immediate_count, string_length, i; + uint64 total_size; + + read_leb_uint32(p, p_end, deferred_count); + read_leb_uint32(p, p_end, immediate_count); + + /* proposal set deferred_count for future extension */ + if (deferred_count != 0) { + goto fail; + } + + if (immediate_count > 0) { + total_size = sizeof(char *) * (uint64)immediate_count; + if (!(module->string_literal_ptrs = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + module->string_literal_count = immediate_count; + + total_size = sizeof(uint32) * (uint64)immediate_count; + if (!(module->string_literal_lengths = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + + for (i = 0; i < immediate_count; i++) { + read_leb_uint32(p, p_end, string_length); + + CHECK_BUF(p, p_end, string_length); + module->string_literal_ptrs[i] = p; + module->string_literal_lengths[i] = string_length; + p += string_length; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + goto fail; + } + + LOG_VERBOSE("Load stringref section success.\n"); + return true; + +fail: + return false; +} +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 +static bool +handle_name_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 name_type, subsection_size; + uint32 previous_name_type = 0; + uint32 num_func_name; + uint32 func_index; + uint32 previous_func_index = ~0U; + uint32 func_name_len; + uint32 name_index; + int i = 0; + + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + while (p < p_end) { + read_leb_uint32(p, p_end, name_type); + if (i != 0) { + if (name_type == previous_name_type) { + set_error_buf(error_buf, error_buf_size, + "duplicate sub-section"); + return false; + } + if (name_type < previous_name_type) { + set_error_buf(error_buf, error_buf_size, + "out-of-order sub-section"); + return false; + } + } + previous_name_type = name_type; + read_leb_uint32(p, p_end, subsection_size); + CHECK_BUF(p, p_end, subsection_size); + switch (name_type) { + case SUB_SECTION_TYPE_FUNC: + if (subsection_size) { + read_leb_uint32(p, p_end, num_func_name); + for (name_index = 0; name_index < num_func_name; + name_index++) { + read_leb_uint32(p, p_end, func_index); + if (func_index == previous_func_index) { + set_error_buf(error_buf, error_buf_size, + "duplicate function name"); + return false; + } + if (func_index < previous_func_index + && previous_func_index != ~0U) { + set_error_buf(error_buf, error_buf_size, + "out-of-order function index "); + return false; + } + previous_func_index = func_index; + read_leb_uint32(p, p_end, func_name_len); + CHECK_BUF(p, p_end, func_name_len); + /* Skip the import functions */ + if (func_index >= module->import_function_count) { + func_index -= module->import_function_count; + if (func_index >= module->function_count) { + set_error_buf(error_buf, error_buf_size, + "out-of-range function index"); + return false; + } + if (!(module->functions[func_index]->field_name = + wasm_const_str_list_insert( + p, func_name_len, module, +#if WASM_ENABLE_WAMR_COMPILER != 0 + false, +#else + is_load_from_file_buf, +#endif + error_buf, error_buf_size))) { + return false; + } + } + p += func_name_len; + } + } + break; + case SUB_SECTION_TYPE_MODULE: /* TODO: Parse for module subsection + */ + case SUB_SECTION_TYPE_LOCAL: /* TODO: Parse for local subsection */ + default: + p = p + subsection_size; + break; + } + i++; + } + + return true; +fail: + return false; +} +#endif + +static bool +load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + char section_name[32]; + uint32 name_len, buffer_len; + + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + read_leb_uint32(p, p_end, name_len); + + if (p + name_len > p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + if (!wasm_check_utf8_str(p, name_len)) { + set_error_buf(error_buf, error_buf_size, "invalid UTF-8 encoding"); + return false; + } + + buffer_len = sizeof(section_name); + memset(section_name, 0, buffer_len); + if (name_len < buffer_len) { + bh_memcpy_s(section_name, buffer_len, p, name_len); + } + else { + bh_memcpy_s(section_name, buffer_len, p, buffer_len - 4); + memset(section_name + buffer_len - 4, '.', 3); + } + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + if (name_len == 4 && memcmp(p, "name", 4) == 0) { + module->name_section_buf = buf; + module->name_section_buf_end = buf_end; + p += name_len; + handle_name_section(p, p_end, module, is_load_from_file_buf, error_buf, + error_buf_size); + LOG_VERBOSE("Load custom name section success."); + return true; + } +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + { + WASMCustomSection *section = + loader_malloc(sizeof(WASMCustomSection), error_buf, error_buf_size); + + if (!section) { + return false; + } + + section->name_addr = (char *)p; + section->name_len = name_len; + section->content_addr = (uint8 *)(p + name_len); + section->content_len = (uint32)(p_end - p - name_len); + + section->next = module->custom_section_list; + module->custom_section_list = section; + LOG_VERBOSE("Load custom section [%s] success.", section_name); + return true; + } +#endif + + LOG_VERBOSE("Ignore custom section [%s].", section_name); + + (void)is_load_from_file_buf; + (void)module; + return true; +fail: + return false; +} + +static void +calculate_global_data_offset(WASMModule *module) +{ + uint32 i, data_offset; + + data_offset = 0; + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *import_global = + &((module->import_globals + i)->u.global); +#if WASM_ENABLE_FAST_JIT != 0 + import_global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(import_global->type); + } + + for (i = 0; i < module->global_count; i++) { + WASMGlobal *global = module->globals + i; +#if WASM_ENABLE_FAST_JIT != 0 + global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(global->type); + } + + module->global_data_size = data_offset; +} + +#if WASM_ENABLE_FAST_JIT != 0 +static bool +init_fast_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_LAZY_JIT != 0 + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); +#endif + uint32 i; + + if (!module->function_count) + return true; + + if (!(module->fast_jit_func_ptrs = + loader_malloc(sizeof(void *) * module->function_count, error_buf, + error_buf_size))) { + return false; + } + +#if WASM_ENABLE_LAZY_JIT != 0 + for (i = 0; i < module->function_count; i++) { + module->fast_jit_func_ptrs[i] = + jit_globals->compile_fast_jit_and_then_call; + } +#endif + + for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) { + if (os_mutex_init(&module->fast_jit_thread_locks[i]) != 0) { + set_error_buf(error_buf, error_buf_size, + "init fast jit thread lock failed"); + return false; + } + module->fast_jit_thread_locks_inited[i] = true; + } + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 */ + +#if WASM_ENABLE_JIT != 0 +static bool +init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + LLVMJITOptions *llvm_jit_options = wasm_runtime_get_llvm_jit_options(); + AOTCompOption option = { 0 }; + char *aot_last_error; + uint64 size; +#if WASM_ENABLE_GC != 0 + bool gc_enabled = true; +#else + bool gc_enabled = false; +#endif + + if (module->function_count == 0) + return true; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (os_mutex_init(&module->tierup_wait_lock) != 0) { + set_error_buf(error_buf, error_buf_size, "init jit tierup lock failed"); + return false; + } + if (os_cond_init(&module->tierup_wait_cond) != 0) { + set_error_buf(error_buf, error_buf_size, "init jit tierup cond failed"); + os_mutex_destroy(&module->tierup_wait_lock); + return false; + } + module->tierup_wait_lock_inited = true; +#endif + + size = sizeof(void *) * (uint64)module->function_count + + sizeof(bool) * (uint64)module->function_count; + if (!(module->func_ptrs = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + module->func_ptrs_compiled = + (bool *)((uint8 *)module->func_ptrs + + sizeof(void *) * module->function_count); + + module->comp_data = aot_create_comp_data(module, NULL, gc_enabled); + if (!module->comp_data) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + option.is_jit_mode = true; + + option.opt_level = llvm_jit_options->opt_level; + option.size_level = llvm_jit_options->size_level; + option.segue_flags = llvm_jit_options->segue_flags; + option.quick_invoke_c_api_import = + llvm_jit_options->quick_invoke_c_api_import; + +#if WASM_ENABLE_BULK_MEMORY != 0 + option.enable_bulk_memory = true; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif +#if WASM_ENABLE_TAIL_CALL != 0 + option.enable_tail_call = true; +#endif +#if WASM_ENABLE_SIMD != 0 + option.enable_simd = true; +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + option.enable_ref_types = true; +#elif WASM_ENABLE_GC != 0 + option.enable_gc = true; +#endif + option.enable_aux_stack_check = true; +#if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ + || WASM_ENABLE_AOT_STACK_FRAME != 0 + option.enable_aux_stack_frame = true; +#endif +#if WASM_ENABLE_PERF_PROFILING != 0 + option.enable_perf_profiling = true; +#endif +#if WASM_ENABLE_MEMORY_PROFILING != 0 + option.enable_memory_profiling = true; + option.enable_stack_estimation = true; +#endif + + module->comp_ctx = aot_create_comp_context(module->comp_data, &option); + if (!module->comp_ctx) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + return true; +} + +static bool +init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + char *aot_last_error; + uint32 i; + + if (module->function_count == 0) + return true; + + if (!aot_compile_wasm(module->comp_ctx)) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (module->orcjit_stop_compiling) + return false; +#endif + + bh_print_time("Begin to lookup llvm jit functions"); + + for (i = 0; i < module->function_count; i++) { + LLVMOrcJITTargetAddress func_addr = 0; + LLVMErrorRef error; + char func_name[48]; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + error = LLVMOrcLLLazyJITLookup(module->comp_ctx->orc_jit, &func_addr, + func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + set_error_buf_v(error_buf, error_buf_size, + "failed to compile llvm jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + return false; + } + + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + module->functions[i]->llvm_jit_func_ptr = (void *)func_addr; + + if (module->orcjit_stop_compiling) + return false; +#endif + } + + bh_print_time("End lookup llvm jit functions"); + + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 +static void * +init_llvm_jit_functions_stage2_callback(void *arg) +{ + WASMModule *module = (WASMModule *)arg; + char error_buf[128]; + uint32 error_buf_size = (uint32)sizeof(error_buf); + + if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) { + module->orcjit_stop_compiling = true; + return NULL; + } + + os_mutex_lock(&module->tierup_wait_lock); + module->llvm_jit_inited = true; + os_cond_broadcast(&module->tierup_wait_cond); + os_mutex_unlock(&module->tierup_wait_lock); + + return NULL; +} +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 +/* The callback function to compile jit functions */ +static void * +orcjit_thread_callback(void *arg) +{ + OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg; +#if WASM_ENABLE_JIT != 0 + AOTCompContext *comp_ctx = thread_arg->comp_ctx; +#endif + WASMModule *module = thread_arg->module; + uint32 group_idx = thread_arg->group_idx; + uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM; + uint32 func_count = module->function_count; + uint32 i; + +#if WASM_ENABLE_FAST_JIT != 0 + /* Compile fast jit funcitons of this group */ + for (i = group_idx; i < func_count; i += group_stride) { + if (!jit_compiler_compile(module, i + module->import_function_count)) { + LOG_ERROR("failed to compile fast jit function %u\n", i); + break; + } + + if (module->orcjit_stop_compiling) { + return NULL; + } + } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + os_mutex_lock(&module->tierup_wait_lock); + module->fast_jit_ready_groups++; + os_mutex_unlock(&module->tierup_wait_lock); +#endif +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + /* For JIT tier-up, set each llvm jit func to call_to_fast_jit */ + for (i = group_idx; i < func_count; + i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) { + uint32 j; + + for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) { + if (i + j * group_stride < func_count) { + if (!jit_compiler_set_call_to_fast_jit( + module, + i + j * group_stride + module->import_function_count)) { + LOG_ERROR( + "failed to compile call_to_fast_jit for func %u\n", + i + j * group_stride + module->import_function_count); + module->orcjit_stop_compiling = true; + return NULL; + } + } + if (module->orcjit_stop_compiling) { + return NULL; + } + } + } + + /* Wait until init_llvm_jit_functions_stage2 finishes and all + fast jit functions are compiled */ + os_mutex_lock(&module->tierup_wait_lock); + while (!(module->llvm_jit_inited && module->enable_llvm_jit_compilation + && module->fast_jit_ready_groups >= group_stride)) { + os_cond_reltimedwait(&module->tierup_wait_cond, + &module->tierup_wait_lock, 10000); + if (module->orcjit_stop_compiling) { + /* init_llvm_jit_functions_stage2 failed */ + os_mutex_unlock(&module->tierup_wait_lock); + return NULL; + } + } + os_mutex_unlock(&module->tierup_wait_lock); +#endif + +#if WASM_ENABLE_JIT != 0 + /* Compile llvm jit functions of this group */ + for (i = group_idx; i < func_count; + i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) { + LLVMOrcJITTargetAddress func_addr = 0; + LLVMErrorRef error; + char func_name[48]; + typedef void (*F)(void); + union { + F f; + void *v; + } u; + uint32 j; + + snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i, + "_wrapper"); + LOG_DEBUG("compile llvm jit func %s", func_name); + error = + LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + LOG_ERROR("failed to compile llvm jit function %u: %s", i, err_msg); + LLVMDisposeErrorMessage(err_msg); + break; + } + + /* Call the jit wrapper function to trigger its compilation, so as + to compile the actual jit functions, since we add the latter to + function list in the PartitionFunction callback */ + u.v = (void *)func_addr; + u.f(); + + for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) { + if (i + j * group_stride < func_count) { + module->func_ptrs_compiled[i + j * group_stride] = true; +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, + i + j * group_stride); + error = LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, + func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + LOG_ERROR("failed to compile llvm jit function %u: %s", i, + err_msg); + LLVMDisposeErrorMessage(err_msg); + /* Ignore current llvm jit func, as its func ptr is + previous set to call_to_fast_jit, which also works */ + continue; + } + + jit_compiler_set_llvm_jit_func_ptr( + module, + i + j * group_stride + module->import_function_count, + (void *)func_addr); + + /* Try to switch to call this llvm jit funtion instead of + fast jit function from fast jit jitted code */ + jit_compiler_set_call_to_llvm_jit( + module, + i + j * group_stride + module->import_function_count); +#endif + } + } + + if (module->orcjit_stop_compiling) { + break; + } + } +#endif + + return NULL; +} + +static void +orcjit_stop_compile_threads(WASMModule *module) +{ + uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args) + / sizeof(OrcJitThreadArg)); + + module->orcjit_stop_compiling = true; + for (i = 0; i < thread_num; i++) { + if (module->orcjit_threads[i]) + os_thread_join(module->orcjit_threads[i], NULL); + } +} + +static bool +compile_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + uint32 thread_num = + (uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg)); + uint32 i, j; + + bh_print_time("Begin to compile jit functions"); + + /* Create threads to compile the jit functions */ + for (i = 0; i < thread_num && i < module->function_count; i++) { +#if WASM_ENABLE_JIT != 0 + module->orcjit_thread_args[i].comp_ctx = module->comp_ctx; +#endif + module->orcjit_thread_args[i].module = module; + module->orcjit_thread_args[i].group_idx = i; + + if (os_thread_create(&module->orcjit_threads[i], orcjit_thread_callback, + (void *)&module->orcjit_thread_args[i], + APP_THREAD_STACK_SIZE_DEFAULT) + != 0) { + set_error_buf(error_buf, error_buf_size, + "create orcjit compile thread failed"); + /* Terminate the threads created */ + module->orcjit_stop_compiling = true; + for (j = 0; j < i; j++) { + os_thread_join(module->orcjit_threads[j], NULL); + } + return false; + } + } + +#if WASM_ENABLE_LAZY_JIT == 0 + /* Wait until all jit functions are compiled for eager mode */ + for (i = 0; i < thread_num; i++) { + if (module->orcjit_threads[i]) + os_thread_join(module->orcjit_threads[i], NULL); + } + +#if WASM_ENABLE_FAST_JIT != 0 + /* Ensure all the fast-jit functions are compiled */ + for (i = 0; i < module->function_count; i++) { + if (!jit_compiler_is_compiled(module, + i + module->import_function_count)) { + set_error_buf(error_buf, error_buf_size, + "failed to compile fast jit function"); + return false; + } + } +#endif + +#if WASM_ENABLE_JIT != 0 + /* Ensure all the llvm-jit functions are compiled */ + for (i = 0; i < module->function_count; i++) { + if (!module->func_ptrs_compiled[i]) { + set_error_buf(error_buf, error_buf_size, + "failed to compile llvm jit function"); + return false; + } + } +#endif +#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */ + + bh_print_time("End compile jit functions"); + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */ + +static bool +wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, + uint32 cur_func_idx, char *error_buf, + uint32 error_buf_size); + +#if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 +void ** +wasm_interp_get_handle_table(); + +static void **handle_table; +#endif + +static bool +load_from_sections(WASMModule *module, WASMSection *sections, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + WASMExport *export; + WASMSection *section = sections; + const uint8 *buf, *buf_end, *buf_code = NULL, *buf_code_end = NULL, + *buf_func = NULL, *buf_func_end = NULL; + WASMGlobal *aux_data_end_global = NULL, *aux_heap_base_global = NULL; + WASMGlobal *aux_stack_top_global = NULL, *global; + uint64 aux_data_end = (uint64)-1LL, aux_heap_base = (uint64)-1LL, + aux_stack_top = (uint64)-1LL; + uint32 global_index, func_index, i; + uint32 aux_data_end_global_index = (uint32)-1; + uint32 aux_heap_base_global_index = (uint32)-1; + WASMFuncType *func_type; + uint8 malloc_free_io_type = VALUE_TYPE_I32; + + /* Find code and function sections if have */ + while (section) { + if (section->section_type == SECTION_TYPE_CODE) { + buf_code = section->section_body; + buf_code_end = buf_code + section->section_body_size; +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0 + module->buf_code = (uint8 *)buf_code; + module->buf_code_size = section->section_body_size; +#endif + } + else if (section->section_type == SECTION_TYPE_FUNC) { + buf_func = section->section_body; + buf_func_end = buf_func + section->section_body_size; + } + section = section->next; + } + + section = sections; + while (section) { + buf = section->section_body; + buf_end = buf + section->section_body_size; + switch (section->section_type) { + case SECTION_TYPE_USER: + /* unsupported user section, ignore it. */ + if (!load_user_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_TYPE: + if (!load_type_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_IMPORT: + if (!load_import_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_FUNC: + if (!load_function_section(buf, buf_end, buf_code, buf_code_end, + module, error_buf, error_buf_size)) + return false; + break; + case SECTION_TYPE_TABLE: + if (!load_table_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_MEMORY: + if (!load_memory_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; +#if WASM_ENABLE_TAGS != 0 + case SECTION_TYPE_TAG: + /* load tag declaration section */ + if (!load_tag_section(buf, buf_end, buf_code, buf_code_end, + module, error_buf, error_buf_size)) + return false; + break; +#endif + case SECTION_TYPE_GLOBAL: + if (!load_global_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_EXPORT: + if (!load_export_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_START: + if (!load_start_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_ELEM: + if (!load_table_segment_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_CODE: + if (!load_code_section(buf, buf_end, buf_func, buf_func_end, + module, error_buf, error_buf_size)) + return false; + break; + case SECTION_TYPE_DATA: + if (!load_data_segment_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case SECTION_TYPE_DATACOUNT: + if (!load_datacount_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; +#endif +#if WASM_ENABLE_STRINGREF != 0 + case SECTION_TYPE_STRINGREF: + if (!load_stringref_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; +#endif + default: + set_error_buf(error_buf, error_buf_size, "invalid section id"); + return false; + } + + section = section->next; + } + + module->aux_data_end_global_index = (uint32)-1; + module->aux_heap_base_global_index = (uint32)-1; + module->aux_stack_top_global_index = (uint32)-1; + + /* Resolve auxiliary data/stack/heap info and reset memory info */ + export = module->exports; + for (i = 0; i < module->export_count; i++, export ++) { + if (export->kind == EXPORT_KIND_GLOBAL) { + if (!strcmp(export->name, "__heap_base")) { + global_index = export->index - module->import_global_count; + global = module->globals + global_index; + if (global->type == VALUE_TYPE_I32 && !global->is_mutable + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST) { + aux_heap_base_global = global; + aux_heap_base = (uint64)(uint32)global->init_expr.u.i32; + aux_heap_base_global_index = export->index; + LOG_VERBOSE("Found aux __heap_base global, value: %" PRIu64, + aux_heap_base); + } + } + else if (!strcmp(export->name, "__data_end")) { + global_index = export->index - module->import_global_count; + global = module->globals + global_index; + if (global->type == VALUE_TYPE_I32 && !global->is_mutable + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST) { + aux_data_end_global = global; + aux_data_end = (uint64)(uint32)global->init_expr.u.i32; + aux_data_end_global_index = export->index; + LOG_VERBOSE("Found aux __data_end global, value: %" PRIu64, + aux_data_end); + + aux_data_end = align_uint64(aux_data_end, 16); + } + } + + /* For module compiled with -pthread option, the global is: + [0] stack_top <-- 0 + [1] tls_pointer + [2] tls_size + [3] data_end <-- 3 + [4] global_base + [5] heap_base <-- 5 + [6] dso_handle + + For module compiled without -pthread option: + [0] stack_top <-- 0 + [1] data_end <-- 1 + [2] global_base + [3] heap_base <-- 3 + [4] dso_handle + */ + if (aux_data_end_global && aux_heap_base_global + && aux_data_end <= aux_heap_base) { + module->aux_data_end_global_index = aux_data_end_global_index; + module->aux_data_end = aux_data_end; + module->aux_heap_base_global_index = aux_heap_base_global_index; + module->aux_heap_base = aux_heap_base; + + /* Resolve aux stack top global */ + for (global_index = 0; global_index < module->global_count; + global_index++) { + global = module->globals + global_index; + if (global->is_mutable /* heap_base and data_end is + not mutable */ + && global->type == VALUE_TYPE_I32 + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST + && (uint64)(uint32)global->init_expr.u.i32 + <= aux_heap_base) { + aux_stack_top_global = global; + aux_stack_top = (uint64)(uint32)global->init_expr.u.i32; + module->aux_stack_top_global_index = + module->import_global_count + global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = + aux_stack_top > aux_data_end + ? (uint32)(aux_stack_top - aux_data_end) + : (uint32)aux_stack_top; + LOG_VERBOSE( + "Found aux stack top global, value: %" PRIu64 ", " + "global index: %d, stack size: %d", + aux_stack_top, global_index, + module->aux_stack_size); + break; + } + } + if (!aux_stack_top_global) { + /* Auxiliary stack global isn't found, it must be unused + in the wasm app, as if it is used, the global must be + defined. Here we set it to __heap_base global and set + its size to 0. */ + aux_stack_top_global = aux_heap_base_global; + aux_stack_top = aux_heap_base; + module->aux_stack_top_global_index = + module->aux_heap_base_global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = 0; + } + break; + } + } + } + + module->malloc_function = (uint32)-1; + module->free_function = (uint32)-1; + module->retain_function = (uint32)-1; + + /* Resolve malloc/free function exported by wasm module */ +#if WASM_ENABLE_MEMORY64 != 0 + if (has_module_memory64(module)) + malloc_free_io_type = VALUE_TYPE_I64; +#endif + export = module->exports; + for (i = 0; i < module->export_count; i++, export ++) { + if (export->kind == EXPORT_KIND_FUNC) { + if (!strcmp(export->name, "malloc") + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == malloc_free_io_type) { + bh_assert(module->malloc_function == (uint32)-1); + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + export->name, export->index); + } + } + else if (!strcmp(export->name, "__new") + && export->index >= module->import_function_count) { + /* __new && __pin for AssemblyScript */ + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 2 && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == VALUE_TYPE_I32 + && func_type->types[2] == malloc_free_io_type) { + uint32 j; + WASMExport *export_tmp; + + bh_assert(module->malloc_function == (uint32)-1); + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + export->name, export->index); + + /* resolve retain function. + If not found, reset malloc function index */ + export_tmp = module->exports; + for (j = 0; j < module->export_count; j++, export_tmp++) { + if ((export_tmp->kind == EXPORT_KIND_FUNC) + && (!strcmp(export_tmp->name, "__retain") + || (!strcmp(export_tmp->name, "__pin"))) + && (export_tmp->index + >= module->import_function_count)) { + func_index = export_tmp->index + - module->import_function_count; + func_type = + module->functions[func_index]->func_type; + if (func_type->param_count == 1 + && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == malloc_free_io_type) { + bh_assert(module->retain_function + == (uint32)-1); + module->retain_function = export_tmp->index; + LOG_VERBOSE("Found retain function, name: %s, " + "index: %u", + export_tmp->name, + export_tmp->index); + break; + } + } + } + if (j == module->export_count) { + module->malloc_function = (uint32)-1; + LOG_VERBOSE("Can't find retain function," + "reset malloc function index to -1"); + } + } + } + else if (((!strcmp(export->name, "free")) + || (!strcmp(export->name, "__release")) + || (!strcmp(export->name, "__unpin"))) + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 && func_type->result_count == 0 + && func_type->types[0] == malloc_free_io_type) { + bh_assert(module->free_function == (uint32)-1); + module->free_function = export->index; + LOG_VERBOSE("Found free function, name: %s, index: %u", + export->name, export->index); + } + } + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 + handle_table = wasm_interp_get_handle_table(); +#endif + + for (i = 0; i < module->function_count; i++) { + WASMFunction *func = module->functions[i]; + if (!wasm_loader_prepare_bytecode(module, func, i, error_buf, + error_buf_size)) { + return false; + } + + if (i == module->function_count - 1 + && func->code + func->code_size != buf_code_end) { + set_error_buf(error_buf, error_buf_size, + "code section size mismatch"); + return false; + } + } + + if (!module->possible_memory_grow) { + WASMMemoryImport *memory_import; + WASMMemory *memory; + + if (aux_data_end_global && aux_heap_base_global + && aux_stack_top_global) { + uint64 init_memory_size; + uint64 shrunk_memory_size = align_uint64(aux_heap_base, 8); + + /* Only resize(shrunk) the memory size if num_bytes_per_page is in + * valid range of uint32 */ + if (shrunk_memory_size <= UINT32_MAX) { + if (module->import_memory_count) { + memory_import = &module->import_memories[0].u.memory; + init_memory_size = (uint64)memory_import->num_bytes_per_page + * memory_import->init_page_count; + if (shrunk_memory_size <= init_memory_size) { + /* Reset memory info to decrease memory usage */ + memory_import->num_bytes_per_page = + (uint32)shrunk_memory_size; + memory_import->init_page_count = 1; + LOG_VERBOSE("Shrink import memory size to %" PRIu64, + shrunk_memory_size); + } + } + + if (module->memory_count) { + memory = &module->memories[0]; + init_memory_size = (uint64)memory->num_bytes_per_page + * memory->init_page_count; + if (shrunk_memory_size <= init_memory_size) { + /* Reset memory info to decrease memory usage */ + memory->num_bytes_per_page = (uint32)shrunk_memory_size; + memory->init_page_count = 1; + LOG_VERBOSE("Shrink memory size to %" PRIu64, + shrunk_memory_size); + } + } + } + } + +#if WASM_ENABLE_MULTI_MODULE == 0 + if (module->import_memory_count) { + memory_import = &module->import_memories[0].u.memory; + /* Only resize the memory to one big page if num_bytes_per_page is + * in valid range of uint32 */ + if (memory_import->init_page_count < DEFAULT_MAX_PAGES) { + memory_import->num_bytes_per_page *= + memory_import->init_page_count; + + if (memory_import->init_page_count > 0) + memory_import->init_page_count = + memory_import->max_page_count = 1; + else + memory_import->init_page_count = + memory_import->max_page_count = 0; + } + } + if (module->memory_count) { + memory = &module->memories[0]; + /* Only resize(shrunk) the memory size if num_bytes_per_page is in + * valid range of uint32 */ + if (memory->init_page_count < DEFAULT_MAX_PAGES) { + memory->num_bytes_per_page *= memory->init_page_count; + if (memory->init_page_count > 0) + memory->init_page_count = memory->max_page_count = 1; + else + memory->init_page_count = memory->max_page_count = 0; + } + } +#endif + } + + calculate_global_data_offset(module); + +#if WASM_ENABLE_FAST_JIT != 0 + if (!init_fast_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif + +#if WASM_ENABLE_JIT != 0 + if (!init_llvm_jit_functions_stage1(module, error_buf, error_buf_size)) { + return false; + } +#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0) + if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) { + return false; + } +#else + /* Run aot_compile_wasm in a backend thread, so as not to block the main + thread fast jit execution, since applying llvm optimizations in + aot_compile_wasm may cost a lot of time. + Create thread with enough native stack to apply llvm optimizations */ + if (os_thread_create(&module->llvm_jit_init_thread, + init_llvm_jit_functions_stage2_callback, + (void *)module, APP_THREAD_STACK_SIZE_DEFAULT * 8) + != 0) { + set_error_buf(error_buf, error_buf_size, + "create orcjit compile thread failed"); + return false; + } +#endif +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* Create threads to compile the jit functions */ + if (!compile_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module); +#endif + return true; +} + +static WASMModule * +create_module(char *name, char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = + loader_malloc(sizeof(WASMModule), error_buf, error_buf_size); + bh_list_status ret; + + if (!module) { + return NULL; + } + + module->module_type = Wasm_Module_Bytecode; + + /* Set start_function to -1, means no start function */ + module->start_function = (uint32)-1; + + module->name = name; + +#if WASM_ENABLE_FAST_INTERP == 0 + module->br_table_cache_list = &module->br_table_cache_list_head; + ret = bh_list_init(module->br_table_cache_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + module->import_module_list = &module->import_module_list_head; + ret = bh_list_init(module->import_module_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + ret = bh_list_init(&module->fast_opcode_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + +#if WASM_ENABLE_GC != 0 + if (!(module->ref_type_set = + wasm_reftype_set_create(GC_REFTYPE_MAP_SIZE_DEFAULT))) { + set_error_buf(error_buf, error_buf_size, "create reftype map failed"); + goto fail1; + } + + if (os_mutex_init(&module->rtt_type_lock)) { + set_error_buf(error_buf, error_buf_size, "init rtt type lock failed"); + goto fail2; + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + if (os_mutex_init(&module->instance_list_lock) != 0) { + set_error_buf(error_buf, error_buf_size, + "init instance list lock failed"); + goto fail3; + } +#endif + + (void)ret; + return module; + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \ + && WASM_ENABLE_LAZY_JIT != 0) +fail3: +#endif +#if WASM_ENABLE_GC != 0 + os_mutex_destroy(&module->rtt_type_lock); +fail2: + bh_hash_map_destroy(module->ref_type_set); +fail1: +#endif + wasm_runtime_free(module); + return NULL; +} + +#if WASM_ENABLE_DEBUG_INTERP != 0 +static bool +record_fast_op(WASMModule *module, uint8 *pos, uint8 orig_op, char *error_buf, + uint32 error_buf_size) +{ + WASMFastOPCodeNode *fast_op = + loader_malloc(sizeof(WASMFastOPCodeNode), error_buf, error_buf_size); + if (fast_op) { + fast_op->offset = pos - module->load_addr; + fast_op->orig_op = orig_op; + bh_list_insert(&module->fast_opcode_list, fast_op); + } + return fast_op ? true : false; +} +#endif + +WASMModule * +wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size) +{ + WASMModule *module = create_module("", error_buf, error_buf_size); + if (!module) + return NULL; + + if (!load_from_sections(module, section_list, false, error_buf, + error_buf_size)) { + wasm_loader_unload(module); + return NULL; + } + + LOG_VERBOSE("Load module from sections success.\n"); + return module; +} + +static void +destroy_sections(WASMSection *section_list) +{ + WASMSection *section = section_list, *next; + while (section) { + next = section->next; + wasm_runtime_free(section); + section = next; + } +} + +/* clang-format off */ +static uint8 section_ids[] = { + SECTION_TYPE_USER, + SECTION_TYPE_TYPE, + SECTION_TYPE_IMPORT, + SECTION_TYPE_FUNC, + SECTION_TYPE_TABLE, + SECTION_TYPE_MEMORY, +#if WASM_ENABLE_TAGS != 0 + SECTION_TYPE_TAG, +#endif +#if WASM_ENABLE_STRINGREF != 0 + /* must immediately precede the global section, + or where the global section would be */ + SECTION_TYPE_STRINGREF, +#endif + SECTION_TYPE_GLOBAL, + SECTION_TYPE_EXPORT, + SECTION_TYPE_START, + SECTION_TYPE_ELEM, +#if WASM_ENABLE_BULK_MEMORY != 0 + SECTION_TYPE_DATACOUNT, +#endif + SECTION_TYPE_CODE, + SECTION_TYPE_DATA +}; +/* clang-format on */ + +static uint8 +get_section_index(uint8 section_type) +{ + uint8 max_id = sizeof(section_ids) / sizeof(uint8); + + for (uint8 i = 0; i < max_id; i++) { + if (section_type == section_ids[i]) + return i; + } + + return (uint8)-1; +} + +static bool +create_sections(const uint8 *buf, uint32 size, WASMSection **p_section_list, + char *error_buf, uint32 error_buf_size) +{ + WASMSection *section_list_end = NULL, *section; + const uint8 *p = buf, *p_end = buf + size /*, *section_body*/; + uint8 section_type, section_index, last_section_index = (uint8)-1; + uint32 section_size; + + bh_assert(!*p_section_list); + + p += 8; + while (p < p_end) { + CHECK_BUF(p, p_end, 1); + section_type = read_uint8(p); + section_index = get_section_index(section_type); + if (section_index != (uint8)-1) { + if (section_type != SECTION_TYPE_USER) { + /* Custom sections may be inserted at any place, + while other sections must occur at most once + and in prescribed order. */ + if (last_section_index != (uint8)-1 + && (section_index <= last_section_index)) { + set_error_buf(error_buf, error_buf_size, + "unexpected content after last section or " + "junk after last section"); + return false; + } + last_section_index = section_index; + } + read_leb_uint32(p, p_end, section_size); + CHECK_BUF1(p, p_end, section_size); + + if (!(section = loader_malloc(sizeof(WASMSection), error_buf, + error_buf_size))) { + return false; + } + + section->section_type = section_type; + section->section_body = (uint8 *)p; + section->section_body_size = section_size; + + if (!section_list_end) + *p_section_list = section_list_end = section; + else { + section_list_end->next = section; + section_list_end = section; + } + + p += section_size; + } + else { + set_error_buf(error_buf, error_buf_size, "invalid section id"); + return false; + } + } + + return true; +fail: + return false; +} + +static void +exchange32(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 3); + *(p_data + 3) = value; + + value = *(p_data + 1); + *(p_data + 1) = *(p_data + 2); + *(p_data + 2) = value; +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +static bool +load(const uint8 *buf, uint32 size, WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf_end = buf + size; + const uint8 *p = buf, *p_end = buf_end; + uint32 magic_number, version; + WASMSection *section_list = NULL; + + CHECK_BUF1(p, p_end, sizeof(uint32)); + magic_number = read_uint32(p); + if (!is_little_endian()) + exchange32((uint8 *)&magic_number); + + if (magic_number != WASM_MAGIC_NUMBER) { + set_error_buf(error_buf, error_buf_size, "magic header not detected"); + return false; + } + + CHECK_BUF1(p, p_end, sizeof(uint32)); + version = read_uint32(p); + if (!is_little_endian()) + exchange32((uint8 *)&version); + + if (version != WASM_CURRENT_VERSION) { + set_error_buf(error_buf, error_buf_size, "unknown binary version"); + return false; + } + + if (!create_sections(buf, size, §ion_list, error_buf, error_buf_size) + || !load_from_sections(module, section_list, true, error_buf, + error_buf_size)) { + destroy_sections(section_list); + return false; + } + + destroy_sections(section_list); + return true; +fail: + return false; +} + +#if WASM_ENABLE_LIBC_WASI != 0 +/** + * refer to + * https://github.com/WebAssembly/WASI/blob/main/design/application-abi.md + */ +static bool +check_wasi_abi_compatibility(const WASMModule *module, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + char *error_buf, uint32 error_buf_size) +{ + /** + * be careful with: + * wasi compatiable modules(command/reactor) which don't import any wasi + * APIs. Usually, a command has to import a "prox_exit" at least, but a + * reactor can depend on nothing. At the same time, each has its own entry + * point. + * + * observations: + * - clang always injects `_start` into a command + * - clang always injects `_initialize` into a reactor + * - `iwasm -f` allows to run a function in the reactor + * + * strong assumptions: + * - no one will define either `_start` or `_initialize` on purpose + * - `_start` should always be `void _start(void)` + * - `_initialize` should always be `void _initialize(void)` + * + */ + + /* clang-format off */ + /** + * + * | | import_wasi_api True | | import_wasi_api False | | + * | ----------- | -------------------- | ---------------- | --------------------- | ---------------- | + * | | \_initialize() Y | \_initialize() N | \_initialize() Y | \_initialize() N | + * | \_start() Y | N | COMMANDER | N | COMMANDER | + * | \_start() N | REACTOR | N | REACTOR | OTHERS | + */ + /* clang-format on */ + + WASMExport *initialize = NULL, *memory = NULL, *start = NULL; + uint32 import_function_count = module->import_function_count; + WASMFuncType *func_type; + + /* (func (export "_start") (...) */ + start = wasm_loader_find_export(module, "", "_start", EXPORT_KIND_FUNC, + error_buf, error_buf_size); + if (start) { + if (start->index < import_function_count) { + set_error_buf( + error_buf, error_buf_size, + "the builtin _start function can not be an import function"); + return false; + } + + func_type = + module->functions[start->index - import_function_count]->func_type; + if (func_type->param_count || func_type->result_count) { + set_error_buf(error_buf, error_buf_size, + "the signature of builtin _start function is wrong"); + return false; + } + } + else { + /* (func (export "_initialize") (...) */ + initialize = + wasm_loader_find_export(module, "", "_initialize", EXPORT_KIND_FUNC, + error_buf, error_buf_size); + + if (initialize) { + if (initialize->index < import_function_count) { + set_error_buf(error_buf, error_buf_size, + "the builtin _initialize function can not be an " + "import function"); + return false; + } + + func_type = + module->functions[initialize->index - import_function_count] + ->func_type; + if (func_type->param_count || func_type->result_count) { + set_error_buf( + error_buf, error_buf_size, + "the signature of builtin _initialize function is wrong"); + return false; + } + } + } + + /* filter out non-wasi compatiable modules */ + if (!module->import_wasi_api && !start && !initialize) { + return true; + } + + /* should have one at least */ + if (module->import_wasi_api && !start && !initialize) { + LOG_WARNING("warning: a module with WASI apis should be either " + "a command or a reactor"); + } + + /* + * there is at least one of `_start` and `_initialize` in below cases. + * according to the assumption, they should be all wasi compatiable + */ + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* filter out commands (with `_start`) cases */ + if (start && !main_module) { + set_error_buf( + error_buf, error_buf_size, + "a command (with _start function) can not be a sub-module"); + return false; + } +#endif + + /* + * it is ok a reactor acts as a main module, + * so skip the check about (with `_initialize`) + */ + + memory = wasm_loader_find_export(module, "", "memory", EXPORT_KIND_MEMORY, + error_buf, error_buf_size); + if (!memory +#if WASM_ENABLE_LIB_WASI_THREADS != 0 + /* + * with wasi-threads, it's still an open question if a memory + * should be exported. + * + * https://github.com/WebAssembly/wasi-threads/issues/22 + * https://github.com/WebAssembly/WASI/issues/502 + * + * Note: this code assumes the number of memories is at most 1. + */ + && module->import_memory_count == 0 +#endif + ) { + set_error_buf(error_buf, error_buf_size, + "a module with WASI apis must export memory by default"); + return false; + } + + return true; +} +#endif + +WASMModule * +wasm_loader_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *args, char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = create_module(args->name, error_buf, error_buf_size); + if (!module) { + return NULL; + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_FAST_JIT != 0 \ + || WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_JIT != 0 + module->load_addr = (uint8 *)buf; + module->load_size = size; +#endif + + if (!load(buf, size, module, error_buf, error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + /* Check the WASI application ABI */ + if (!check_wasi_abi_compatibility(module, +#if WASM_ENABLE_MULTI_MODULE != 0 + main_module, +#endif + error_buf, error_buf_size)) { + goto fail; + } +#endif + + LOG_VERBOSE("Load module success.\n"); + return module; + +fail: + wasm_loader_unload(module); + return NULL; +} + +void +wasm_loader_unload(WASMModule *module) +{ + uint32 i; + + if (!module) + return; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + module->orcjit_stop_compiling = true; + if (module->llvm_jit_init_thread) + os_thread_join(module->llvm_jit_init_thread, NULL); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* Stop Fast/LLVM JIT compilation firstly to avoid accessing + module internal data after they were freed */ + orcjit_stop_compile_threads(module); +#endif + +#if WASM_ENABLE_JIT != 0 + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + if (module->comp_ctx) + aot_destroy_comp_context(module->comp_ctx); + if (module->comp_data) + aot_destroy_comp_data(module->comp_data); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (module->tierup_wait_lock_inited) { + os_mutex_destroy(&module->tierup_wait_lock); + os_cond_destroy(&module->tierup_wait_cond); + } +#endif + + if (module->types) { + for (i = 0; i < module->type_count; i++) { + if (module->types[i]) + destroy_wasm_type(module->types[i]); + } + wasm_runtime_free(module->types); + } + + if (module->imports) + wasm_runtime_free(module->imports); + + if (module->functions) { + for (i = 0; i < module->function_count; i++) { + if (module->functions[i]) { + if (module->functions[i]->local_offsets) + wasm_runtime_free(module->functions[i]->local_offsets); +#if WASM_ENABLE_FAST_INTERP != 0 + if (module->functions[i]->code_compiled) + wasm_runtime_free(module->functions[i]->code_compiled); + if (module->functions[i]->consts) + wasm_runtime_free(module->functions[i]->consts); +#endif +#if WASM_ENABLE_FAST_JIT != 0 + if (module->functions[i]->fast_jit_jitted_code) { + jit_code_cache_free( + module->functions[i]->fast_jit_jitted_code); + } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (module->functions[i]->call_to_fast_jit_from_llvm_jit) { + jit_code_cache_free( + module->functions[i]->call_to_fast_jit_from_llvm_jit); + } +#endif +#endif +#if WASM_ENABLE_GC != 0 + if (module->functions[i]->local_ref_type_maps) { + wasm_runtime_free( + module->functions[i]->local_ref_type_maps); + } +#endif + wasm_runtime_free(module->functions[i]); + } + } + wasm_runtime_free(module->functions); + } + + if (module->tables) { +#if WASM_ENABLE_GC != 0 + for (i = 0; i < module->table_count; i++) { + destroy_init_expr(&module->tables[i].init_expr); + } +#endif + wasm_runtime_free(module->tables); + } + + if (module->memories) + wasm_runtime_free(module->memories); + + if (module->globals) { +#if WASM_ENABLE_GC != 0 + for (i = 0; i < module->global_count; i++) { + destroy_init_expr(&module->globals[i].init_expr); + } +#endif + wasm_runtime_free(module->globals); + } + +#if WASM_ENABLE_TAGS != 0 + if (module->tags) { + for (i = 0; i < module->tag_count; i++) { + if (module->tags[i]) + wasm_runtime_free(module->tags[i]); + } + wasm_runtime_free(module->tags); + } +#endif + + if (module->exports) + wasm_runtime_free(module->exports); + + if (module->table_segments) { + for (i = 0; i < module->table_seg_count; i++) { + if (module->table_segments[i].init_values) { +#if WASM_ENABLE_GC != 0 + uint32 j; + for (j = 0; j < module->table_segments[i].value_count; j++) { + destroy_init_expr( + &module->table_segments[i].init_values[j]); + } +#endif + wasm_runtime_free(module->table_segments[i].init_values); + } + } + wasm_runtime_free(module->table_segments); + } + + if (module->data_segments) { + for (i = 0; i < module->data_seg_count; i++) { + if (module->data_segments[i]) + wasm_runtime_free(module->data_segments[i]); + } + wasm_runtime_free(module->data_segments); + } + + if (module->const_str_list) { + StringNode *node = module->const_str_list, *node_next; + while (node) { + node_next = node->next; + wasm_runtime_free(node); + node = node_next; + } + } + +#if WASM_ENABLE_STRINGREF != 0 + if (module->string_literal_ptrs) { + wasm_runtime_free((void *)module->string_literal_ptrs); + } + if (module->string_literal_lengths) { + wasm_runtime_free(module->string_literal_lengths); + } +#endif + +#if WASM_ENABLE_FAST_INTERP == 0 + if (module->br_table_cache_list) { + BrTableCache *node = bh_list_first_elem(module->br_table_cache_list); + BrTableCache *node_next; + while (node) { + node_next = bh_list_elem_next(node); + wasm_runtime_free(node); + node = node_next; + } + } +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* just release the sub module list */ + if (module->import_module_list) { + WASMRegisteredModule *node = + bh_list_first_elem(module->import_module_list); + while (node) { + WASMRegisteredModule *next = bh_list_elem_next(node); + bh_list_remove(module->import_module_list, node); + /* + * unload(sub_module) will be trigged during runtime_destroy(). + * every module in the global module list will be unloaded one by + * one. so don't worry. + */ + wasm_runtime_free(node); + /* + * the module file reading buffer will be released + * in runtime_destroy() + */ + node = next; + } + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + WASMFastOPCodeNode *fast_opcode = + bh_list_first_elem(&module->fast_opcode_list); + while (fast_opcode) { + WASMFastOPCodeNode *next = bh_list_elem_next(fast_opcode); + wasm_runtime_free(fast_opcode); + fast_opcode = next; + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + os_mutex_destroy(&module->instance_list_lock); +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + wasm_runtime_destroy_custom_sections(module->custom_section_list); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + if (module->fast_jit_func_ptrs) { + wasm_runtime_free(module->fast_jit_func_ptrs); + } + + for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) { + if (module->fast_jit_thread_locks_inited[i]) { + os_mutex_destroy(&module->fast_jit_thread_locks[i]); + } + } +#endif + +#if WASM_ENABLE_GC != 0 + os_mutex_destroy(&module->rtt_type_lock); + bh_hash_map_destroy(module->ref_type_set); + if (module->rtt_types) { + for (i = 0; i < module->type_count; i++) { + if (module->rtt_types[i]) + wasm_runtime_free(module->rtt_types[i]); + } + wasm_runtime_free(module->rtt_types); + } +#if WASM_ENABLE_STRINGREF != 0 + for (i = 0; i < WASM_TYPE_STRINGVIEWITER - WASM_TYPE_STRINGREF + 1; i++) { + if (module->stringref_rtts[i]) + wasm_runtime_free(module->stringref_rtts[i]); + } +#endif +#endif + + wasm_runtime_free(module); +} + +bool +wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, + const uint8 *start_addr, const uint8 *code_end_addr, + uint8 label_type, uint8 **p_else_addr, + uint8 **p_end_addr) +{ + const uint8 *p = start_addr, *p_end = code_end_addr; + uint8 *else_addr = NULL; + char error_buf[128]; + uint32 block_nested_depth = 1, count, i, j, t; + uint32 error_buf_size = sizeof(error_buf); + uint8 opcode, u8; + BlockAddr block_stack[16] = { { 0 } }, *block; + + i = ((uintptr_t)start_addr) & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + block = block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; + + for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) { + if (block[j].start_addr == start_addr) { + /* Cache hit */ + *p_else_addr = block[j].else_addr; + *p_end_addr = block[j].end_addr; + return true; + } + } + + /* Cache unhit */ + block_stack[0].start_addr = start_addr; + + while (p < code_end_addr) { + opcode = *p++; +#if WASM_ENABLE_DEBUG_INTERP != 0 + op_break_retry: +#endif + switch (opcode) { + case WASM_OP_UNREACHABLE: + case WASM_OP_NOP: + break; + +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_TRY: + u8 = read_uint8(p); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + case EXT_OP_TRY: + skip_leb_uint32(p, p_end); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + case WASM_OP_CATCH: + if (block_nested_depth == 1) { + *p_end_addr = (uint8 *)(p - 1); + /* stop search and return the address of the catch block */ + return true; + } + break; + case WASM_OP_CATCH_ALL: + if (block_nested_depth == 1) { + *p_end_addr = (uint8 *)(p - 1); + /* stop search and return the address of the catch_all block + */ + return true; + } + break; + case WASM_OP_THROW: + /* skip tag_index */ + skip_leb(p); + break; + case WASM_OP_RETHROW: + /* skip depth */ + skip_leb(p); + break; + case WASM_OP_DELEGATE: + if (block_nested_depth == 1) { + *p_end_addr = (uint8 *)(p - 1); + return true; + } + else { + skip_leb(p); + /* the DELEGATE opcode ends the tryblock, */ + block_nested_depth--; + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth].end_addr = + (uint8 *)(p - 1); + } + break; +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + + case WASM_OP_BLOCK: + case WASM_OP_LOOP: + case WASM_OP_IF: + { + /* block result type: 0x40/0x7F/0x7E/0x7D/0x7C */ + u8 = read_uint8(p); + if (is_byte_a_type(u8)) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(u8)) { + /* the possible extra bytes of GC ref type have been + modified to OP_NOP, no need to resolve them again */ + } +#endif + } + else { + p--; + skip_leb_uint32(p, p_end); + } + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + } + + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + /* block type */ + skip_leb_uint32(p, p_end); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + + case WASM_OP_ELSE: + if (label_type == LABEL_TYPE_IF && block_nested_depth == 1) + else_addr = (uint8 *)(p - 1); + if (block_nested_depth - 1 + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth - 1].else_addr = + (uint8 *)(p - 1); + break; + + case WASM_OP_END: + if (block_nested_depth == 1) { + if (label_type == LABEL_TYPE_IF) + *p_else_addr = else_addr; + *p_end_addr = (uint8 *)(p - 1); + + block_stack[0].end_addr = (uint8 *)(p - 1); + for (t = 0; t < sizeof(block_stack) / sizeof(BlockAddr); + t++) { + start_addr = block_stack[t].start_addr; + if (start_addr) { + i = ((uintptr_t)start_addr) + & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + block = + block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; + for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) + if (!block[j].start_addr) + break; + + if (j == BLOCK_ADDR_CONFLICT_SIZE) { + memmove(block + 1, block, + (BLOCK_ADDR_CONFLICT_SIZE - 1) + * sizeof(BlockAddr)); + j = 0; + } + block[j].start_addr = block_stack[t].start_addr; + block[j].else_addr = block_stack[t].else_addr; + block[j].end_addr = block_stack[t].end_addr; + } + else + break; + } + return true; + } + else { + block_nested_depth--; + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth].end_addr = + (uint8 *)(p - 1); + } + break; + + case WASM_OP_BR: + case WASM_OP_BR_IF: + skip_leb_uint32(p, p_end); /* labelidx */ + break; + + case WASM_OP_BR_TABLE: + read_leb_uint32(p, p_end, count); /* lable num */ +#if WASM_ENABLE_FAST_INTERP != 0 + for (i = 0; i <= count; i++) /* lableidxs */ + skip_leb_uint32(p, p_end); +#else + p += count + 1; + while (*p == WASM_OP_NOP) + p++; +#endif + break; + +#if WASM_ENABLE_FAST_INTERP == 0 + case EXT_OP_BR_TABLE_CACHE: + read_leb_uint32(p, p_end, count); /* lable num */ + while (*p == WASM_OP_NOP) + p++; + break; +#endif + + case WASM_OP_RETURN: + break; + + case WASM_OP_CALL: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: +#endif + skip_leb_uint32(p, p_end); /* funcidx */ + break; + + case WASM_OP_CALL_INDIRECT: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL_INDIRECT: +#endif + skip_leb_uint32(p, p_end); /* typeidx */ +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + u8 = read_uint8(p); /* 0x00 */ +#else + skip_leb_uint32(p, p_end); /* talbeidx */ +#endif + break; + +#if WASM_ENABLE_GC != 0 + case WASM_OP_CALL_REF: + case WASM_OP_RETURN_CALL_REF: + skip_leb_uint32(p, p_end); /* typeidx */ + break; +#endif + + case WASM_OP_DROP: + case WASM_OP_SELECT: + case WASM_OP_DROP_64: + case WASM_OP_SELECT_64: + break; + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_SELECT_T: + { + skip_leb_uint32(p, p_end); /* vec length */ + u8 = read_uint8(p); /* typeidx */ + /* the possible extra bytes of GC ref type have been + modified to OP_NOP, no need to resolve them again */ + break; + } + + case WASM_OP_TABLE_GET: + case WASM_OP_TABLE_SET: + skip_leb_uint32(p, p_end); /* table index */ + break; + case WASM_OP_REF_NULL: + { + u8 = read_uint8(p); /* type */ + if (is_byte_a_type(u8)) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(u8)) { + /* the possible extra bytes of GC ref type have been + modified to OP_NOP, no need to resolve them again */ + } +#endif + } + else { + p--; + skip_leb_uint32(p, p_end); + } + break; + } + case WASM_OP_REF_IS_NULL: + break; + case WASM_OP_REF_FUNC: + skip_leb_uint32(p, p_end); /* func index */ + break; +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + case WASM_OP_REF_AS_NON_NULL: + case WASM_OP_REF_EQ: + break; + case WASM_OP_BR_ON_NULL: + case WASM_OP_BR_ON_NON_NULL: + skip_leb_uint32(p, p_end); /* label index */ + break; +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_GET_LOCAL: + case WASM_OP_SET_LOCAL: + case WASM_OP_TEE_LOCAL: + case WASM_OP_GET_GLOBAL: + case WASM_OP_SET_GLOBAL: + case WASM_OP_GET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_AUX_STACK: + skip_leb_uint32(p, p_end); /* local index */ + break; + + case EXT_OP_GET_LOCAL_FAST: + case EXT_OP_SET_LOCAL_FAST: + case EXT_OP_TEE_LOCAL_FAST: + CHECK_BUF(p, p_end, 1); + p++; + break; + + case WASM_OP_I32_LOAD: + case WASM_OP_I64_LOAD: + case WASM_OP_F32_LOAD: + case WASM_OP_F64_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + case WASM_OP_I32_STORE: + case WASM_OP_I64_STORE: + case WASM_OP_F32_STORE: + case WASM_OP_F64_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ + break; + + case WASM_OP_MEMORY_SIZE: + case WASM_OP_MEMORY_GROW: + skip_leb_uint32(p, p_end); /* 0x00 */ + break; + + case WASM_OP_I32_CONST: + skip_leb_int32(p, p_end); + break; + case WASM_OP_I64_CONST: + skip_leb_int64(p, p_end); + break; + case WASM_OP_F32_CONST: + p += sizeof(float32); + break; + case WASM_OP_F64_CONST: + p += sizeof(float64); + break; + + case WASM_OP_I32_EQZ: + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + case WASM_OP_I64_EQZ: + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + case WASM_OP_I32_CLZ: + case WASM_OP_I32_CTZ: + case WASM_OP_I32_POPCNT: + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + case WASM_OP_I64_CLZ: + case WASM_OP_I64_CTZ: + case WASM_OP_I64_POPCNT: + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + case WASM_OP_F32_COPYSIGN: + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + case WASM_OP_F64_COPYSIGN: + case WASM_OP_I32_WRAP_I64: + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + case WASM_OP_F32_DEMOTE_F64: + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + case WASM_OP_F64_PROMOTE_F32: + case WASM_OP_I32_REINTERPRET_F32: + case WASM_OP_I64_REINTERPRET_F64: + case WASM_OP_F32_REINTERPRET_I32: + case WASM_OP_F64_REINTERPRET_I64: + case WASM_OP_I32_EXTEND8_S: + case WASM_OP_I32_EXTEND16_S: + case WASM_OP_I64_EXTEND8_S: + case WASM_OP_I64_EXTEND16_S: + case WASM_OP_I64_EXTEND32_S: + break; + +#if WASM_ENABLE_GC != 0 + case WASM_OP_GC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + skip_leb_uint32(p, p_end); /* typeidx */ + break; + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + case WASM_OP_STRUCT_SET: + skip_leb_uint32(p, p_end); /* typeidx */ + skip_leb_uint32(p, p_end); /* fieldidx */ + break; + + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + case WASM_OP_ARRAY_SET: + case WASM_OP_ARRAY_FILL: + skip_leb_uint32(p, p_end); /* typeidx */ + break; + case WASM_OP_ARRAY_COPY: + skip_leb_uint32(p, p_end); /* typeidx1 */ + skip_leb_uint32(p, p_end); /* typeidx2 */ + break; + case WASM_OP_ARRAY_LEN: + break; + case WASM_OP_ARRAY_NEW_FIXED: + case WASM_OP_ARRAY_NEW_DATA: + case WASM_OP_ARRAY_NEW_ELEM: + skip_leb_uint32(p, p_end); /* typeidx */ + skip_leb_uint32(p, p_end); /* N/dataidx/elemidx */ + break; + + case WASM_OP_REF_I31: + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + break; + + case WASM_OP_REF_TEST: + case WASM_OP_REF_CAST: + case WASM_OP_REF_TEST_NULLABLE: + case WASM_OP_REF_CAST_NULLABLE: + skip_leb_int32(p, p_end); /* heaptype */ + break; + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + p += sizeof(uint8); /* castflag */ + skip_leb_uint32(p, p_end); /* labelidx */ + skip_leb_int32(p, p_end); /* heaptype */ + skip_leb_int32(p, p_end); /* heaptype2 */ + break; + + case WASM_OP_ANY_CONVERT_EXTERN: + case WASM_OP_EXTERN_CONVERT_ANY: + break; + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRING_CONST: + skip_leb_int32(p, p_end); /* contents */ + break; + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + break; + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRING_CONCAT: + case WASM_OP_STRING_EQ: + case WASM_OP_STRING_IS_USV_SEQUENCE: + case WASM_OP_STRING_AS_WTF8: + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + break; + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRINGVIEW_WTF8_SLICE: + case WASM_OP_STRING_AS_WTF16: + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + break; + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRINGVIEW_WTF16_SLICE: + case WASM_OP_STRING_AS_ITER: + case WASM_OP_STRINGVIEW_ITER_NEXT: + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + case WASM_OP_STRINGVIEW_ITER_REWIND: + case WASM_OP_STRINGVIEW_ITER_SLICE: + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + break; +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + default: + return false; + } + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + skip_leb_uint32(p, p_end); + /* skip memory idx */ + p++; + break; + case WASM_OP_DATA_DROP: + skip_leb_uint32(p, p_end); + break; + case WASM_OP_MEMORY_COPY: + /* skip two memory idx */ + p += 2; + break; + case WASM_OP_MEMORY_FILL: + /* skip memory idx */ + p++; + break; +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_TABLE_INIT: + case WASM_OP_TABLE_COPY: + /* tableidx */ + skip_leb_uint32(p, p_end); + /* elemidx */ + skip_leb_uint32(p, p_end); + break; + case WASM_OP_ELEM_DROP: + /* elemidx */ + skip_leb_uint32(p, p_end); + break; + case WASM_OP_TABLE_SIZE: + case WASM_OP_TABLE_GROW: + case WASM_OP_TABLE_FILL: + skip_leb_uint32(p, p_end); /* table idx */ + break; +#endif /* WASM_ENABLE_REF_TYPES */ + default: + return false; + } + break; + } + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + case WASM_OP_SIMD_PREFIX: + { + /* TODO: memory64 offset type changes */ + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + /* follow the order of enum WASMSimdEXTOpcode in wasm_opcode.h + */ + switch (opcode) { + case SIMD_v128_load: + case SIMD_v128_load8x8_s: + case SIMD_v128_load8x8_u: + case SIMD_v128_load16x4_s: + case SIMD_v128_load16x4_u: + case SIMD_v128_load32x2_s: + case SIMD_v128_load32x2_u: + case SIMD_v128_load8_splat: + case SIMD_v128_load16_splat: + case SIMD_v128_load32_splat: + case SIMD_v128_load64_splat: + case SIMD_v128_store: + /* memarg align */ + skip_leb_uint32(p, p_end); + /* memarg offset*/ + skip_leb_uint32(p, p_end); + break; + + case SIMD_v128_const: + case SIMD_v8x16_shuffle: + /* immByte[16] immLaneId[16] */ + CHECK_BUF1(p, p_end, 16); + p += 16; + break; + + case SIMD_i8x16_extract_lane_s: + case SIMD_i8x16_extract_lane_u: + case SIMD_i8x16_replace_lane: + case SIMD_i16x8_extract_lane_s: + case SIMD_i16x8_extract_lane_u: + case SIMD_i16x8_replace_lane: + case SIMD_i32x4_extract_lane: + case SIMD_i32x4_replace_lane: + case SIMD_i64x2_extract_lane: + case SIMD_i64x2_replace_lane: + case SIMD_f32x4_extract_lane: + case SIMD_f32x4_replace_lane: + case SIMD_f64x2_extract_lane: + case SIMD_f64x2_replace_lane: + /* ImmLaneId */ + CHECK_BUF(p, p_end, 1); + p++; + break; + + case SIMD_v128_load8_lane: + case SIMD_v128_load16_lane: + case SIMD_v128_load32_lane: + case SIMD_v128_load64_lane: + case SIMD_v128_store8_lane: + case SIMD_v128_store16_lane: + case SIMD_v128_store32_lane: + case SIMD_v128_store64_lane: + /* memarg align */ + skip_leb_uint32(p, p_end); + /* memarg offset*/ + skip_leb_uint32(p, p_end); + /* ImmLaneId */ + CHECK_BUF(p, p_end, 1); + p++; + break; + + case SIMD_v128_load32_zero: + case SIMD_v128_load64_zero: + /* memarg align */ + skip_leb_uint32(p, p_end); + /* memarg offset*/ + skip_leb_uint32(p, p_end); + break; + + default: + /* + * since latest SIMD specific used almost every value + * from 0x00 to 0xff, the default branch will present + * all opcodes without imm + * https://github.com/WebAssembly/simd/blob/main/proposals/simd/NewOpcodes.md + */ + break; + } + break; + } +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + /* TODO: memory64 offset type changes */ + uint32 opcode1; + + /* atomic_op (u32_leb) + memarg (2 u32_leb) */ + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + if (opcode != WASM_OP_ATOMIC_FENCE) { + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ + } + else { + /* atomic.fence doesn't have memarg */ + p++; + } + break; + } +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + case DEBUG_OP_BREAK: + { + WASMDebugInstance *debug_instance = + wasm_exec_env_get_instance(exec_env); + char orignal_opcode[1]; + uint64 size = 1; + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + uint64 offset = (p - 1) >= module_inst->module->load_addr + ? (p - 1) - module_inst->module->load_addr + : ~0; + if (debug_instance) { + if (wasm_debug_instance_get_obj_mem(debug_instance, offset, + orignal_opcode, &size) + && size == 1) { + LOG_VERBOSE("WASM loader find OP_BREAK , recover it " + "with %02x: ", + orignal_opcode[0]); + opcode = orignal_opcode[0]; + goto op_break_retry; + } + } + break; + } +#endif + + default: + return false; + } + } + + (void)u8; + (void)exec_env; + return false; +fail: + return false; +} + +#if WASM_ENABLE_FAST_INTERP != 0 + +#if WASM_DEBUG_PREPROCESSOR != 0 +#define LOG_OP(...) os_printf(__VA_ARGS__) +#else +#define LOG_OP(...) (void)0 +#endif + +#define PATCH_ELSE 0 +#define PATCH_END 1 +typedef struct BranchBlockPatch { + struct BranchBlockPatch *next; + uint8 patch_type; + uint8 *code_compiled; +} BranchBlockPatch; +#endif + +typedef struct BranchBlock { + uint8 label_type; + BlockType block_type; + uint8 *start_addr; + uint8 *else_addr; + uint8 *end_addr; + uint32 stack_cell_num; +#if WASM_ENABLE_GC != 0 + uint32 reftype_map_num; + /* Indicate which local is used inside current block, used to validate + * local.get with non-nullable ref types */ + uint8 *local_use_mask; + uint32 local_use_mask_size; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + uint16 dynamic_offset; + uint8 *code_compiled; + BranchBlockPatch *patch_list; + /* This is used to save params frame_offset of of if block */ + int16 *param_frame_offsets; + /* This is used to recover dynamic offset for else branch */ + uint16 start_dynamic_offset; +#endif + + /* Indicate the operand stack is in polymorphic state. + * If the opcode is one of unreachable/br/br_table/return, stack is marked + * to polymorphic state until the block's 'end' opcode is processed. + * If stack is in polymorphic state and stack is empty, instruction can + * pop any type of value directly without decreasing stack top pointer + * and stack cell num. */ + bool is_stack_polymorphic; +} BranchBlock; + +typedef struct WASMLoaderContext { + /* frame ref stack */ + uint8 *frame_ref; + uint8 *frame_ref_bottom; + uint8 *frame_ref_boundary; + uint32 frame_ref_size; + uint32 stack_cell_num; + uint32 max_stack_cell_num; + +#if WASM_ENABLE_GC != 0 + /* frame reftype map stack */ + WASMRefTypeMap *frame_reftype_map; + WASMRefTypeMap *frame_reftype_map_bottom; + WASMRefTypeMap *frame_reftype_map_boundary; + uint32 frame_reftype_map_size; + uint32 reftype_map_num; + uint32 max_reftype_map_num; + /* Current module */ + WASMModule *module; + /* Current module's ref_type_set */ + HashMap *ref_type_set; + /* Always point to local variable ref_type of + wasm_loader_prepare_bytecode */ + WASMRefType *ref_type_tmp; +#endif + + /* frame csp stack */ + BranchBlock *frame_csp; + BranchBlock *frame_csp_bottom; + BranchBlock *frame_csp_boundary; + uint32 frame_csp_size; + uint32 csp_num; + uint32 max_csp_num; + +#if WASM_ENABLE_FAST_INTERP != 0 + /* frame offset stack */ + int16 *frame_offset; + int16 *frame_offset_bottom; + int16 *frame_offset_boundary; + uint32 frame_offset_size; + int16 dynamic_offset; + int16 start_dynamic_offset; + int16 max_dynamic_offset; + + /* preserved local offset */ + int16 preserved_local_offset; + + /* const buffer */ + uint8 *const_buf; + uint16 num_const; + uint16 const_cell_num; + uint32 const_buf_size; + + /* processed code */ + uint8 *p_code_compiled; + uint8 *p_code_compiled_end; + uint32 code_compiled_size; + /* If the last opcode will be dropped, the peak memory usage will be larger + * than the final code_compiled_size, we record the peak size to ensure + * there will not be invalid memory access during second traverse */ + uint32 code_compiled_peak_size; +#endif +} WASMLoaderContext; + +typedef struct Const { + WASMValue value; + uint16 slot_index; + uint8 value_type; +} Const; + +#define CHECK_CSP_PUSH() \ + do { \ + if (ctx->frame_csp >= ctx->frame_csp_boundary) { \ + MEM_REALLOC( \ + ctx->frame_csp_bottom, ctx->frame_csp_size, \ + (uint32)(ctx->frame_csp_size + 8 * sizeof(BranchBlock))); \ + ctx->frame_csp_size += (uint32)(8 * sizeof(BranchBlock)); \ + ctx->frame_csp_boundary = \ + ctx->frame_csp_bottom \ + + ctx->frame_csp_size / sizeof(BranchBlock); \ + ctx->frame_csp = ctx->frame_csp_bottom + ctx->csp_num; \ + } \ + } while (0) + +#define CHECK_CSP_POP() \ + do { \ + if (ctx->csp_num < 1) { \ + set_error_buf(error_buf, error_buf_size, \ + "type mismatch: " \ + "expect data but block stack was empty"); \ + goto fail; \ + } \ + } while (0) + +#if WASM_ENABLE_FAST_INTERP != 0 +static bool +check_offset_push(WASMLoaderContext *ctx, char *error_buf, + uint32 error_buf_size) +{ + uint32 cell_num = (uint32)(ctx->frame_offset - ctx->frame_offset_bottom); + if (ctx->frame_offset >= ctx->frame_offset_boundary) { + MEM_REALLOC(ctx->frame_offset_bottom, ctx->frame_offset_size, + ctx->frame_offset_size + 16); + ctx->frame_offset_size += 16; + ctx->frame_offset_boundary = + ctx->frame_offset_bottom + ctx->frame_offset_size / sizeof(int16); + ctx->frame_offset = ctx->frame_offset_bottom + cell_num; + } + return true; +fail: + return false; +} + +static bool +check_offset_pop(WASMLoaderContext *ctx, uint32 cells) +{ + if (ctx->frame_offset - cells < ctx->frame_offset_bottom) + return false; + return true; +} + +static void +free_label_patch_list(BranchBlock *frame_csp) +{ + BranchBlockPatch *label_patch = frame_csp->patch_list; + BranchBlockPatch *next; + while (label_patch != NULL) { + next = label_patch->next; + wasm_runtime_free(label_patch); + label_patch = next; + } + frame_csp->patch_list = NULL; +} + +static void +free_all_label_patch_lists(BranchBlock *frame_csp, uint32 csp_num) +{ + BranchBlock *tmp_csp = frame_csp; + + for (uint32 i = 0; i < csp_num; i++) { + free_label_patch_list(tmp_csp); + tmp_csp++; + } +} + +#endif /* end of WASM_ENABLE_FAST_INTERP */ + +#if WASM_ENABLE_GC != 0 +static bool +wasm_loader_init_local_use_masks(WASMLoaderContext *ctx, uint32 local_count, + char *error_buf, uint32 error_buf_size) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + uint32 local_mask_size; + + if (local_count == 0) { + current_csp->local_use_mask_size = 0; + return true; + } + + /* if current_csp->local_use_mask is not NULL, then it is re-init masks for + * else branch, we don't need to allocate memory again */ + if (!current_csp->local_use_mask) { + local_mask_size = (local_count + 7) / sizeof(uint8); + if (!(current_csp->local_use_mask = + loader_malloc(local_mask_size, error_buf, error_buf_size))) { + return false; + } + current_csp->local_use_mask_size = local_mask_size; + } + else { + local_mask_size = current_csp->local_use_mask_size; + bh_assert(current_csp->label_type == LABEL_TYPE_IF); + } + + if (current_csp->label_type != LABEL_TYPE_FUNCTION) { + /* For non-function blocks, inherit the use status from parent block */ + BranchBlock *parent_csp = current_csp - 1; + + bh_assert(parent_csp >= ctx->frame_csp_bottom); + bh_assert(parent_csp->local_use_mask); + + bh_memcpy_s(current_csp->local_use_mask, local_mask_size, + parent_csp->local_use_mask, local_mask_size); + } + + return true; +} + +static void +wasm_loader_destroy_curr_local_use_masks(WASMLoaderContext *ctx) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + + bh_assert(current_csp->local_use_mask + || current_csp->local_use_mask_size == 0); + + if (current_csp->local_use_mask) { + wasm_runtime_free(current_csp->local_use_mask); + } + + current_csp->local_use_mask = NULL; + current_csp->local_use_mask_size = 0; +} + +static void +wasm_loader_clean_all_local_use_masks(WASMLoaderContext *ctx) +{ + BranchBlock *tmp_csp = ctx->frame_csp_bottom; + uint32 i; + + for (i = 0; i < ctx->csp_num; i++) { + if (tmp_csp->local_use_mask) { + wasm_runtime_free(tmp_csp->local_use_mask); + tmp_csp->local_use_mask = NULL; + tmp_csp->local_use_mask_size = 0; + } + tmp_csp++; + } +} + +static void +wasm_loader_mask_local(WASMLoaderContext *ctx, uint32 index) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + uint32 byte_offset = index / sizeof(uint8); + uint32 bit_offset = index % sizeof(uint8); + + bh_assert(byte_offset < current_csp->local_use_mask_size); + bh_assert(current_csp->local_use_mask); + + current_csp->local_use_mask[byte_offset] |= (1 << bit_offset); +} + +static bool +wasm_loader_get_local_status(WASMLoaderContext *ctx, uint32 index) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + uint32 byte_offset = index / sizeof(uint8); + uint32 bit_offset = index % sizeof(uint8); + + bh_assert(byte_offset < current_csp->local_use_mask_size); + bh_assert(current_csp->local_use_mask); + + return (current_csp->local_use_mask[byte_offset] & (1 << bit_offset)) + ? true + : false; +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static void +wasm_loader_ctx_destroy(WASMLoaderContext *ctx) +{ + if (ctx) { + if (ctx->frame_ref_bottom) + wasm_runtime_free(ctx->frame_ref_bottom); +#if WASM_ENABLE_GC != 0 + if (ctx->frame_reftype_map_bottom) + wasm_runtime_free(ctx->frame_reftype_map_bottom); +#endif + if (ctx->frame_csp_bottom) { +#if WASM_ENABLE_FAST_INTERP != 0 + free_all_label_patch_lists(ctx->frame_csp_bottom, ctx->csp_num); +#endif +#if WASM_ENABLE_GC != 0 + wasm_loader_clean_all_local_use_masks(ctx); +#endif + wasm_runtime_free(ctx->frame_csp_bottom); + } +#if WASM_ENABLE_FAST_INTERP != 0 + if (ctx->frame_offset_bottom) + wasm_runtime_free(ctx->frame_offset_bottom); + if (ctx->const_buf) + wasm_runtime_free(ctx->const_buf); +#endif + wasm_runtime_free(ctx); + } +} + +static WASMLoaderContext * +wasm_loader_ctx_init(WASMFunction *func, char *error_buf, uint32 error_buf_size) +{ + WASMLoaderContext *loader_ctx = + loader_malloc(sizeof(WASMLoaderContext), error_buf, error_buf_size); + if (!loader_ctx) + return NULL; + + loader_ctx->frame_ref_size = 32; + if (!(loader_ctx->frame_ref_bottom = loader_ctx->frame_ref = loader_malloc( + loader_ctx->frame_ref_size, error_buf, error_buf_size))) + goto fail; + loader_ctx->frame_ref_boundary = loader_ctx->frame_ref_bottom + 32; + +#if WASM_ENABLE_GC != 0 + loader_ctx->frame_reftype_map_size = sizeof(WASMRefTypeMap) * 16; + if (!(loader_ctx->frame_reftype_map_bottom = loader_ctx->frame_reftype_map = + loader_malloc(loader_ctx->frame_reftype_map_size, error_buf, + error_buf_size))) + goto fail; + loader_ctx->frame_reftype_map_boundary = + loader_ctx->frame_reftype_map_bottom + 16; +#endif + + loader_ctx->frame_csp_size = sizeof(BranchBlock) * 8; + if (!(loader_ctx->frame_csp_bottom = loader_ctx->frame_csp = loader_malloc( + loader_ctx->frame_csp_size, error_buf, error_buf_size))) + goto fail; + loader_ctx->frame_csp_boundary = loader_ctx->frame_csp_bottom + 8; + +#if WASM_ENABLE_EXCE_HANDLING != 0 + func->exception_handler_count = 0; +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + loader_ctx->frame_offset_size = sizeof(int16) * 32; + if (!(loader_ctx->frame_offset_bottom = loader_ctx->frame_offset = + loader_malloc(loader_ctx->frame_offset_size, error_buf, + error_buf_size))) + goto fail; + loader_ctx->frame_offset_boundary = loader_ctx->frame_offset_bottom + 32; + + loader_ctx->num_const = 0; + loader_ctx->const_buf_size = sizeof(Const) * 8; + if (!(loader_ctx->const_buf = loader_malloc(loader_ctx->const_buf_size, + error_buf, error_buf_size))) + goto fail; + + if (func->param_cell_num >= (int32)INT16_MAX - func->local_cell_num) { + set_error_buf(error_buf, error_buf_size, + "fast interpreter offset overflow"); + goto fail; + } + + loader_ctx->start_dynamic_offset = loader_ctx->dynamic_offset = + loader_ctx->max_dynamic_offset = + func->param_cell_num + func->local_cell_num; +#endif + return loader_ctx; + +fail: + wasm_loader_ctx_destroy(loader_ctx); + return NULL; +} + +static bool +check_stack_push(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + uint32 cell_num_needed = wasm_value_type_cell_num(type); + + if (ctx->frame_ref + cell_num_needed > ctx->frame_ref_boundary) { + /* Increase the frame ref stack */ + MEM_REALLOC(ctx->frame_ref_bottom, ctx->frame_ref_size, + ctx->frame_ref_size + 16); + ctx->frame_ref_size += 16; + ctx->frame_ref_boundary = ctx->frame_ref_bottom + ctx->frame_ref_size; + ctx->frame_ref = ctx->frame_ref_bottom + ctx->stack_cell_num; + } + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(type) + && ctx->frame_reftype_map >= ctx->frame_reftype_map_boundary) { + /* Increase the frame reftype map stack */ + bh_assert( + (uint32)((ctx->frame_reftype_map - ctx->frame_reftype_map_bottom) + * sizeof(WASMRefTypeMap)) + == ctx->frame_reftype_map_size); + MEM_REALLOC(ctx->frame_reftype_map_bottom, ctx->frame_reftype_map_size, + ctx->frame_reftype_map_size + + (uint32)sizeof(WASMRefTypeMap) * 8); + ctx->frame_reftype_map = + ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size / ((uint32)sizeof(WASMRefTypeMap)); + ctx->frame_reftype_map_size += (uint32)sizeof(WASMRefTypeMap) * 8; + ctx->frame_reftype_map_boundary = + ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size / ((uint32)sizeof(WASMRefTypeMap)); + } +#endif + return true; +fail: + return false; +} + +static bool +wasm_loader_push_frame_ref(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + uint32 type_cell_num = wasm_value_type_cell_num(type); + uint32 i; + + if (!check_stack_push(ctx, type, error_buf, error_buf_size)) + return false; + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(type)) { + WASMRefType *ref_type; + if (!(ref_type = + reftype_set_insert(ctx->ref_type_set, ctx->ref_type_tmp, + error_buf, error_buf_size))) { + return false; + } + + if (ctx->frame_reftype_map >= ctx->frame_reftype_map_boundary) { + /* Increase the frame reftype map stack */ + bh_assert((uint32)((ctx->frame_reftype_map + - ctx->frame_reftype_map_bottom) + * sizeof(WASMRefTypeMap)) + == ctx->frame_reftype_map_size); + MEM_REALLOC(ctx->frame_reftype_map_bottom, + ctx->frame_reftype_map_size, + ctx->frame_reftype_map_size + + (uint32)sizeof(WASMRefTypeMap) * 8); + ctx->frame_reftype_map = ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size + / ((uint32)sizeof(WASMRefTypeMap)); + ctx->frame_reftype_map_size += (uint32)sizeof(WASMRefTypeMap) * 8; + ctx->frame_reftype_map_boundary = + ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size + / ((uint32)sizeof(WASMRefTypeMap)); + } + + ctx->frame_reftype_map->index = ctx->stack_cell_num; + ctx->frame_reftype_map->ref_type = ref_type; + ctx->frame_reftype_map++; + ctx->reftype_map_num++; + if (ctx->reftype_map_num > ctx->max_reftype_map_num) + ctx->max_reftype_map_num = ctx->reftype_map_num; + } +#endif + + for (i = 0; i < type_cell_num; i++) + *ctx->frame_ref++ = type; + ctx->stack_cell_num += type_cell_num; + + if (ctx->stack_cell_num > ctx->max_stack_cell_num) { + ctx->max_stack_cell_num = ctx->stack_cell_num; + if (ctx->max_stack_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "operand stack depth limit exceeded"); + return false; + } + } + return true; +#if WASM_ENABLE_GC != 0 +fail: + return false; +#endif +} + +static bool +check_stack_top_values(WASMLoaderContext *ctx, uint8 *frame_ref, + int32 stack_cell_num, +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map, int32 reftype_map_num, +#endif + uint8 type, +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type, +#endif + char *error_buf, uint32 error_buf_size) +{ + int32 type_cell_num = (int32)wasm_value_type_cell_num(type), i; +#if WASM_ENABLE_GC != 0 + WASMRefType *frame_reftype = NULL; +#endif + + if (stack_cell_num < type_cell_num) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: expect data but stack was empty"); + return false; + } + +#if WASM_ENABLE_GC == 0 + for (i = 0; i < type_cell_num; i++) { + if (*(frame_ref - 1 - i) != type) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + return false; + } + } +#else + if (wasm_is_type_multi_byte_type(*(frame_ref - 1))) { + bh_assert(reftype_map_num > 0); + frame_reftype = (frame_reftype_map - 1)->ref_type; + } + if (!wasm_reftype_is_subtype_of(*(frame_ref - 1), frame_reftype, type, + ref_type, ctx->module->types, + ctx->module->type_count)) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + return false; + } + for (i = 0; i < type_cell_num - 1; i++) { + if (*(frame_ref - 2 - i) != *(frame_ref - 1)) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + return false; + } + } +#endif + + return true; +} + +static bool +check_stack_pop(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + int32 block_stack_cell_num = + (int32)(ctx->stack_cell_num - (ctx->frame_csp - 1)->stack_cell_num); +#if WASM_ENABLE_GC != 0 + int32 reftype_map_num = + (int32)(ctx->reftype_map_num - (ctx->frame_csp - 1)->reftype_map_num); +#endif + + if (block_stack_cell_num > 0) { + if (*(ctx->frame_ref - 1) == VALUE_TYPE_ANY) + /* the stack top is a value of any type, return success */ + return true; + } + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(type) && block_stack_cell_num > 0) { + uint8 stack_top_type = *(ctx->frame_ref - 1); + WASMRefType *stack_top_ref_type = NULL; + + if (wasm_is_type_multi_byte_type(stack_top_type)) { + bh_assert(reftype_map_num > 0); + stack_top_ref_type = (*(ctx->frame_reftype_map - 1)).ref_type; + } + + if (wasm_reftype_is_subtype_of(stack_top_type, stack_top_ref_type, type, + ctx->ref_type_tmp, ctx->module->types, + ctx->module->type_count)) { + if (wasm_is_type_multi_byte_type(stack_top_type)) { + uint32 ref_type_struct_size = + wasm_reftype_struct_size(stack_top_ref_type); + bh_memcpy_s(ctx->ref_type_tmp, (uint32)sizeof(WASMRefType), + stack_top_ref_type, ref_type_struct_size); + } + return true; + } + } +#endif + + if (!check_stack_top_values(ctx, ctx->frame_ref, block_stack_cell_num, +#if WASM_ENABLE_GC != 0 + ctx->frame_reftype_map, reftype_map_num, +#endif + type, +#if WASM_ENABLE_GC != 0 + ctx->ref_type_tmp, +#endif + error_buf, error_buf_size)) { + return false; + } + + return true; +} + +static bool +wasm_loader_pop_frame_ref(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + BranchBlock *cur_block = ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + uint32 cell_num_to_pop = wasm_value_type_cell_num(type); + + /* Directly return success if current block is in stack + polymorphic state while stack is empty. */ + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + return true; + + if (type == VALUE_TYPE_VOID) + return true; + + if (!check_stack_pop(ctx, type, error_buf, error_buf_size)) + return false; + + bh_assert(available_stack_cell > 0); + if (*(ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + type = VALUE_TYPE_ANY; + cell_num_to_pop = 1; + } + + ctx->frame_ref -= cell_num_to_pop; + ctx->stack_cell_num -= cell_num_to_pop; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(*ctx->frame_ref)) { + ctx->frame_reftype_map--; + ctx->reftype_map_num--; + } +#endif + + return true; +} + +#if WASM_ENABLE_GC != 0 +/* Get the stack top element of current block */ +static bool +wasm_loader_get_frame_ref_top(WASMLoaderContext *ctx, uint8 *p_type, + WASMRefType **p_ref_type, char *error_buf, + uint32 error_buf_size) +{ + BranchBlock *cur_block = ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + + if (available_stack_cell <= 0) { + /* Directly return success if current block is in stack + polymorphic state while stack is empty. */ + if (cur_block->is_stack_polymorphic) { + *p_type = VALUE_TYPE_ANY; + return true; + } + else { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: expect data but block stack was empty"); + return false; + } + } + + *p_type = *(ctx->frame_ref - 1); + if (wasm_is_type_multi_byte_type(*p_type)) { + int32 available_reftype_map = + (int32)(ctx->reftype_map_num + - (ctx->frame_csp - 1)->reftype_map_num); + bh_assert(available_reftype_map > 0); + (void)available_reftype_map; + *p_ref_type = (ctx->frame_reftype_map - 1)->ref_type; + } + + return true; +} + +#if WASM_ENABLE_FAST_INTERP != 0 +static bool +wasm_loader_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size); +#endif + +/* Check whether the stack top elem is a heap object, and if yes, + pop and return it */ +static bool +wasm_loader_pop_heap_obj(WASMLoaderContext *ctx, uint8 *p_type, + WASMRefType *ref_ht_ret, char *error_buf, + uint32 error_buf_size) +{ + uint8 type = 0; + WASMRefType *ref_type = NULL; + + /* Get stack top element */ + if (!wasm_loader_get_frame_ref_top(ctx, &type, &ref_type, error_buf, + error_buf_size)) { + return false; + } + + if (type != VALUE_TYPE_ANY /* block isn't in stack polymorphic state */ + /* stack top isn't a ref type */ + && !wasm_is_type_reftype(type)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: expect heap object but got others"); + return false; + } + + /* POP stack top */ + if (wasm_is_type_multi_byte_type(type)) { + bh_assert(ref_type); + bh_memcpy_s(ctx->ref_type_tmp, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!wasm_loader_pop_frame_ref_offset(ctx, type, error_buf, + error_buf_size)) { + return false; + } +#else + if (!wasm_loader_pop_frame_ref(ctx, type, error_buf, error_buf_size)) { + return false; + } +#endif + + if (p_type) + *p_type = type; + if (wasm_is_type_multi_byte_type(type) && ref_ht_ret) { + bh_memcpy_s(ref_ht_ret, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } + return true; +} + +/* Check whether the stack top elem is subtype of (ref null ht), + and if yes, pop it and return the converted (ref ht) */ +static bool +wasm_loader_pop_nullable_ht(WASMLoaderContext *ctx, uint8 *p_type, + WASMRefType *ref_ht_ret, char *error_buf, + uint32 error_buf_size) +{ + uint8 type = 0; + WASMRefType ref_type = { 0 }; + + if (!wasm_loader_pop_heap_obj(ctx, &type, &ref_type, error_buf, + error_buf_size)) { + return false; + } + + /* Convert to related (ref ht) and return */ + if (type >= REF_TYPE_ARRAYREF && type <= REF_TYPE_NULLFUNCREF) { + /* Return (ref array/struct/i31/eq/any/extern/func/none/noextern/nofunc) + */ + wasm_set_refheaptype_common(&ref_ht_ret->ref_ht_common, false, + HEAP_TYPE_ARRAY + + (type - REF_TYPE_ARRAYREF)); + type = ref_ht_ret->ref_type; + } + else if (wasm_is_reftype_htref_nullable(type) + || wasm_is_reftype_htref_non_nullable(type)) { + bh_memcpy_s(ref_ht_ret, (uint32)sizeof(WASMRefType), &ref_type, + wasm_reftype_struct_size(&ref_type)); + /* Convert to (ref ht) */ + ref_ht_ret->ref_ht_common.ref_type = REF_TYPE_HT_NON_NULLABLE; + ref_ht_ret->ref_ht_common.nullable = false; + type = ref_ht_ret->ref_type; + } + *p_type = type; + + return true; +} + +/* Check whether the stack top elem is (ref null $t) or (ref $t), + and if yes, pop it and return the type_idx */ +static bool +wasm_loader_pop_nullable_typeidx(WASMLoaderContext *ctx, uint8 *p_type, + uint32 *p_type_idx, char *error_buf, + uint32 error_buf_size) +{ + uint8 type = 0; + int32 type_idx = -1; + WASMRefType *ref_type = NULL; + + /* Get stack top element */ + if (!wasm_loader_get_frame_ref_top(ctx, &type, &ref_type, error_buf, + error_buf_size)) { + return false; + } + + if (type != VALUE_TYPE_ANY) { + /* stack top isn't (ref null $t) */ + if (!((wasm_is_reftype_htref_nullable(type) + || wasm_is_reftype_htref_non_nullable(type)) + && wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common))) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: expect (ref null $t) but got others"); + return false; + } + type_idx = ref_type->ref_ht_typeidx.type_idx; + + bh_memcpy_s(ctx->ref_type_tmp, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } + + /* POP stack top */ +#if WASM_ENABLE_FAST_INTERP != 0 + if (!wasm_loader_pop_frame_ref_offset(ctx, type, error_buf, + error_buf_size)) { + return false; + } +#else + if (!wasm_loader_pop_frame_ref(ctx, type, error_buf, error_buf_size)) { + return false; + } +#endif + + /* Convert to type_idx and return */ + *p_type = type; + if (type != VALUE_TYPE_ANY) + *p_type_idx = (uint32)type_idx; + return true; +} +#endif /* WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_FAST_INTERP == 0 +static bool +wasm_loader_push_pop_frame_ref(WASMLoaderContext *ctx, uint8 pop_cnt, + uint8 type_push, uint8 type_pop, char *error_buf, + uint32 error_buf_size) +{ + for (int i = 0; i < pop_cnt; i++) { + if (!wasm_loader_pop_frame_ref(ctx, type_pop, error_buf, + error_buf_size)) + return false; + } + if (!wasm_loader_push_frame_ref(ctx, type_push, error_buf, error_buf_size)) + return false; + return true; +} +#endif + +static bool +wasm_loader_push_frame_csp(WASMLoaderContext *ctx, uint8 label_type, + BlockType block_type, uint8 *start_addr, + char *error_buf, uint32 error_buf_size) +{ + CHECK_CSP_PUSH(); + memset(ctx->frame_csp, 0, sizeof(BranchBlock)); + ctx->frame_csp->label_type = label_type; + ctx->frame_csp->block_type = block_type; + ctx->frame_csp->start_addr = start_addr; + ctx->frame_csp->stack_cell_num = ctx->stack_cell_num; +#if WASM_ENABLE_GC != 0 + ctx->frame_csp->reftype_map_num = ctx->reftype_map_num; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + ctx->frame_csp->dynamic_offset = ctx->dynamic_offset; + ctx->frame_csp->patch_list = NULL; +#endif + ctx->frame_csp++; + ctx->csp_num++; + if (ctx->csp_num > ctx->max_csp_num) { + ctx->max_csp_num = ctx->csp_num; + if (ctx->max_csp_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "label stack depth limit exceeded"); + return false; + } + } + return true; +fail: + return false; +} + +static bool +wasm_loader_pop_frame_csp(WASMLoaderContext *ctx, char *error_buf, + uint32 error_buf_size) +{ + CHECK_CSP_POP(); +#if WASM_ENABLE_FAST_INTERP != 0 + if ((ctx->frame_csp - 1)->param_frame_offsets) + wasm_runtime_free((ctx->frame_csp - 1)->param_frame_offsets); +#endif + ctx->frame_csp--; + ctx->csp_num--; + + return true; +fail: + return false; +} + +#if WASM_ENABLE_FAST_INTERP != 0 + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define emit_label(opcode) \ + do { \ + wasm_loader_emit_ptr(loader_ctx, handle_table[opcode]); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(void *)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#else /* else of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#if UINTPTR_MAX == UINT64_MAX +#define emit_label(opcode) \ + do { \ + int32 offset = \ + (int32)((uint8 *)handle_table[opcode] - (uint8 *)handle_table[0]); \ + /* emit int32 relative offset in 64-bit target */ \ + wasm_loader_emit_uint32(loader_ctx, offset); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#else +#define emit_label(opcode) \ + do { \ + uint32 label_addr = (uint32)(uintptr_t)handle_table[opcode]; \ + /* emit uint32 label address in 32-bit target */ \ + wasm_loader_emit_uint32(loader_ctx, label_addr); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#endif +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(int32)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#define emit_label(opcode) \ + do { \ + wasm_loader_emit_uint8(loader_ctx, opcode); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint8)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + +#define emit_empty_label_addr_and_frame_ip(type) \ + do { \ + if (!add_label_patch_to_list(loader_ctx->frame_csp - 1, type, \ + loader_ctx->p_code_compiled, error_buf, \ + error_buf_size)) \ + goto fail; \ + /* label address, to be patched */ \ + wasm_loader_emit_ptr(loader_ctx, NULL); \ + } while (0) + +#define emit_br_info(frame_csp, is_br) \ + do { \ + if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, is_br, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define LAST_OP_OUTPUT_I32() \ + (last_op >= WASM_OP_I32_EQZ && last_op <= WASM_OP_I32_ROTR) \ + || (last_op == WASM_OP_I32_LOAD || last_op == WASM_OP_F32_LOAD) \ + || (last_op >= WASM_OP_I32_LOAD8_S && last_op <= WASM_OP_I32_LOAD16_U) \ + || (last_op >= WASM_OP_F32_ABS && last_op <= WASM_OP_F32_COPYSIGN) \ + || (last_op >= WASM_OP_I32_WRAP_I64 \ + && last_op <= WASM_OP_I32_TRUNC_U_F64) \ + || (last_op >= WASM_OP_F32_CONVERT_S_I32 \ + && last_op <= WASM_OP_F32_DEMOTE_F64) \ + || (last_op == WASM_OP_I32_REINTERPRET_F32) \ + || (last_op == WASM_OP_F32_REINTERPRET_I32) \ + || (last_op == EXT_OP_COPY_STACK_TOP) + +#define LAST_OP_OUTPUT_I64() \ + (last_op >= WASM_OP_I64_CLZ && last_op <= WASM_OP_I64_ROTR) \ + || (last_op >= WASM_OP_F64_ABS && last_op <= WASM_OP_F64_COPYSIGN) \ + || (last_op == WASM_OP_I64_LOAD || last_op == WASM_OP_F64_LOAD) \ + || (last_op >= WASM_OP_I64_LOAD8_S && last_op <= WASM_OP_I64_LOAD32_U) \ + || (last_op >= WASM_OP_I64_EXTEND_S_I32 \ + && last_op <= WASM_OP_I64_TRUNC_U_F64) \ + || (last_op >= WASM_OP_F64_CONVERT_S_I32 \ + && last_op <= WASM_OP_F64_PROMOTE_F32) \ + || (last_op == WASM_OP_I64_REINTERPRET_F64) \ + || (last_op == WASM_OP_F64_REINTERPRET_I64) \ + || (last_op == EXT_OP_COPY_STACK_TOP_I64) + +#define GET_CONST_OFFSET(type, val) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &val, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define GET_CONST_F32_OFFSET(type, fval) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &fval, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define GET_CONST_F64_OFFSET(type, fval) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &fval, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define emit_operand(ctx, offset) \ + do { \ + wasm_loader_emit_int16(ctx, offset); \ + LOG_OP("%d\t", offset); \ + } while (0) + +#define emit_byte(ctx, byte) \ + do { \ + wasm_loader_emit_uint8(ctx, byte); \ + LOG_OP("%d\t", byte); \ + } while (0) + +#define emit_uint32(ctx, value) \ + do { \ + wasm_loader_emit_uint32(ctx, value); \ + LOG_OP("%d\t", value); \ + } while (0) + +#define emit_uint64(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, false); \ + LOG_OP("%lld\t", value); \ + } while (0) + +#define emit_float32(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, true); \ + LOG_OP("%f\t", value); \ + } while (0) + +#define emit_float64(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, false); \ + LOG_OP("%f\t", value); \ + } while (0) + +static bool +wasm_loader_ctx_reinit(WASMLoaderContext *ctx) +{ + if (!(ctx->p_code_compiled = + loader_malloc(ctx->code_compiled_peak_size, NULL, 0))) + return false; + ctx->p_code_compiled_end = + ctx->p_code_compiled + ctx->code_compiled_peak_size; + + /* clean up frame ref */ + memset(ctx->frame_ref_bottom, 0, ctx->frame_ref_size); + ctx->frame_ref = ctx->frame_ref_bottom; + ctx->stack_cell_num = 0; + +#if WASM_ENABLE_GC != 0 + /* clean up reftype map */ + memset(ctx->frame_reftype_map_bottom, 0, ctx->frame_reftype_map_size); + ctx->frame_reftype_map = ctx->frame_reftype_map_bottom; + ctx->reftype_map_num = 0; +#endif + + /* clean up frame csp */ + memset(ctx->frame_csp_bottom, 0, ctx->frame_csp_size); + ctx->frame_csp = ctx->frame_csp_bottom; + ctx->csp_num = 0; + ctx->max_csp_num = 0; + + /* clean up frame offset */ + memset(ctx->frame_offset_bottom, 0, ctx->frame_offset_size); + ctx->frame_offset = ctx->frame_offset_bottom; + ctx->dynamic_offset = ctx->start_dynamic_offset; + + /* init preserved local offsets */ + ctx->preserved_local_offset = ctx->max_dynamic_offset; + + /* const buf is reserved */ + return true; +} + +static void +increase_compiled_code_space(WASMLoaderContext *ctx, int32 size) +{ + ctx->code_compiled_size += size; + if (ctx->code_compiled_size >= ctx->code_compiled_peak_size) { + ctx->code_compiled_peak_size = ctx->code_compiled_size; + } +} + +static void +wasm_loader_emit_const(WASMLoaderContext *ctx, void *value, bool is_32_bit) +{ + uint32 size = is_32_bit ? sizeof(uint32) : sizeof(uint64); + + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + bh_memcpy_s(ctx->p_code_compiled, + (uint32)(ctx->p_code_compiled_end - ctx->p_code_compiled), + value, size); + ctx->p_code_compiled += size; + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, size); + } +} + +static void +wasm_loader_emit_uint32(WASMLoaderContext *ctx, uint32 value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_U32(ctx->p_code_compiled, value); + ctx->p_code_compiled += sizeof(uint32); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(uint32)); + } +} + +static void +wasm_loader_emit_int16(WASMLoaderContext *ctx, int16 value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_U16(ctx->p_code_compiled, (uint16)value); + ctx->p_code_compiled += sizeof(int16); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(uint16)); + } +} + +static void +wasm_loader_emit_uint8(WASMLoaderContext *ctx, uint8 value) +{ + if (ctx->p_code_compiled) { + *(ctx->p_code_compiled) = value; + ctx->p_code_compiled += sizeof(uint8); +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + ctx->p_code_compiled++; + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + } + else { + increase_compiled_code_space(ctx, sizeof(uint8)); +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + increase_compiled_code_space(ctx, sizeof(uint8)); + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + } +} + +static void +wasm_loader_emit_ptr(WASMLoaderContext *ctx, void *value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_PTR(ctx->p_code_compiled, value); + ctx->p_code_compiled += sizeof(void *); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(void *)); + } +} + +static void +wasm_loader_emit_backspace(WASMLoaderContext *ctx, uint32 size) +{ + if (ctx->p_code_compiled) { + ctx->p_code_compiled -= size; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + if (size == sizeof(uint8)) { + ctx->p_code_compiled--; + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); + } +#endif + } + else { + ctx->code_compiled_size -= size; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + if (size == sizeof(uint8)) { + ctx->code_compiled_size--; + bh_assert((ctx->code_compiled_size & 1) == 0); + } +#endif + } +} + +static bool +preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode, + uint32 local_index, uint32 local_type, + bool *preserved, char *error_buf, + uint32 error_buf_size) +{ + uint32 i = 0; + int16 preserved_offset = (int16)local_index; + + *preserved = false; + while (i < loader_ctx->stack_cell_num) { + uint8 cur_type = loader_ctx->frame_ref_bottom[i]; + + /* move previous local into dynamic space before a set/tee_local opcode + */ + if (loader_ctx->frame_offset_bottom[i] == (int16)local_index) { + if (!(*preserved)) { + *preserved = true; + skip_label(); + preserved_offset = loader_ctx->preserved_local_offset; + if (loader_ctx->p_code_compiled) { + bh_assert(preserved_offset != (int16)local_index); + } + if (is_32bit_type(local_type)) { + /* Only increase preserve offset in the second traversal */ + if (loader_ctx->p_code_compiled) + loader_ctx->preserved_local_offset++; + emit_label(EXT_OP_COPY_STACK_TOP); + } + else { + if (loader_ctx->p_code_compiled) + loader_ctx->preserved_local_offset += 2; + emit_label(EXT_OP_COPY_STACK_TOP_I64); + } + emit_operand(loader_ctx, local_index); + emit_operand(loader_ctx, preserved_offset); + emit_label(opcode); + } + loader_ctx->frame_offset_bottom[i] = preserved_offset; + } + + if (is_32bit_type(cur_type)) + i++; + else + i += 2; + } + + (void)error_buf; + (void)error_buf_size; + return true; +} + +static bool +preserve_local_for_block(WASMLoaderContext *loader_ctx, uint8 opcode, + char *error_buf, uint32 error_buf_size) +{ + uint32 i = 0; + bool preserve_local; + + /* preserve locals before blocks to ensure that "tee/set_local" inside + blocks will not influence the value of these locals */ + while (i < loader_ctx->stack_cell_num) { + int16 cur_offset = loader_ctx->frame_offset_bottom[i]; + uint8 cur_type = loader_ctx->frame_ref_bottom[i]; + + if ((cur_offset < loader_ctx->start_dynamic_offset) + && (cur_offset >= 0)) { + if (!(preserve_referenced_local(loader_ctx, opcode, cur_offset, + cur_type, &preserve_local, + error_buf, error_buf_size))) + return false; + } + + if (is_32bit_type(cur_type)) { + i++; + } + else { + i += 2; + } + } + + return true; +} + +static bool +add_label_patch_to_list(BranchBlock *frame_csp, uint8 patch_type, + uint8 *p_code_compiled, char *error_buf, + uint32 error_buf_size) +{ + BranchBlockPatch *patch = + loader_malloc(sizeof(BranchBlockPatch), error_buf, error_buf_size); + if (!patch) { + return false; + } + patch->patch_type = patch_type; + patch->code_compiled = p_code_compiled; + if (!frame_csp->patch_list) { + frame_csp->patch_list = patch; + patch->next = NULL; + } + else { + patch->next = frame_csp->patch_list; + frame_csp->patch_list = patch; + } + return true; +} + +static void +apply_label_patch(WASMLoaderContext *ctx, uint8 depth, uint8 patch_type) +{ + BranchBlock *frame_csp = ctx->frame_csp - depth; + BranchBlockPatch *node = frame_csp->patch_list; + BranchBlockPatch *node_prev = NULL, *node_next; + + if (!ctx->p_code_compiled) + return; + + while (node) { + node_next = node->next; + if (node->patch_type == patch_type) { + STORE_PTR(node->code_compiled, ctx->p_code_compiled); + if (node_prev == NULL) { + frame_csp->patch_list = node_next; + } + else { + node_prev->next = node_next; + } + wasm_runtime_free(node); + } + else { + node_prev = node; + } + node = node_next; + } +} + +static bool +wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, + bool is_br, char *error_buf, uint32 error_buf_size) +{ + /* br info layout: + * a) arity of target block + * b) total cell num of arity values + * c) each arity value's cell num + * d) each arity value's src frame offset + * e) each arity values's dst dynamic offset + * f) branch target address + * + * Note: b-e are omitted when arity is 0 so that + * interpreter can recover the br info quickly. + */ + BlockType *block_type = &frame_csp->block_type; + uint8 *types = NULL, cell; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *reftype_maps; + uint32 reftype_map_count; +#endif + uint32 arity = 0; + int32 i; + int16 *frame_offset = ctx->frame_offset; + uint16 dynamic_offset; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ +#if WASM_ENABLE_GC == 0 + if (frame_csp->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(block_type, &types); + else + arity = block_type_get_result_types(block_type, &types); +#else + if (frame_csp->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(block_type, &types, &reftype_maps, + &reftype_map_count); + else + arity = block_type_get_result_types(block_type, &types, &reftype_maps, + &reftype_map_count); +#endif + + /* Part a */ + emit_uint32(ctx, arity); + + if (arity) { + /* Part b */ + emit_uint32(ctx, wasm_get_cell_num(types, arity)); + /* Part c */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + emit_byte(ctx, cell); + } + /* Part d */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + frame_offset -= cell; + emit_operand(ctx, *(int16 *)(frame_offset)); + } + /* Part e */ + dynamic_offset = + frame_csp->dynamic_offset + wasm_get_cell_num(types, arity); + if (is_br) + ctx->dynamic_offset = dynamic_offset; + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + dynamic_offset -= cell; + emit_operand(ctx, dynamic_offset); + } + } + + /* Part f */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) { + wasm_loader_emit_ptr(ctx, frame_csp->code_compiled); + } + else { + if (!add_label_patch_to_list(frame_csp, PATCH_END, ctx->p_code_compiled, + error_buf, error_buf_size)) + return false; + /* label address, to be patched */ + wasm_loader_emit_ptr(ctx, NULL); + } + + return true; +} + +static bool +wasm_loader_push_frame_offset(WASMLoaderContext *ctx, uint8 type, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + if (type == VALUE_TYPE_VOID) + return true; + + /* only check memory overflow in first traverse */ + if (ctx->p_code_compiled == NULL) { + if (!check_offset_push(ctx, error_buf, error_buf_size)) + return false; + } + + if (disable_emit) + *(ctx->frame_offset)++ = operand_offset; + else { + emit_operand(ctx, ctx->dynamic_offset); + *(ctx->frame_offset)++ = ctx->dynamic_offset; + ctx->dynamic_offset++; + if (ctx->dynamic_offset > ctx->max_dynamic_offset) { + ctx->max_dynamic_offset = ctx->dynamic_offset; + if (ctx->max_dynamic_offset >= INT16_MAX) { + goto fail; + } + } + } + + if (is_32bit_type(type)) + return true; + + if (ctx->p_code_compiled == NULL) { + if (!check_offset_push(ctx, error_buf, error_buf_size)) + return false; + } + + ctx->frame_offset++; + if (!disable_emit) { + ctx->dynamic_offset++; + if (ctx->dynamic_offset > ctx->max_dynamic_offset) { + ctx->max_dynamic_offset = ctx->dynamic_offset; + if (ctx->max_dynamic_offset >= INT16_MAX) { + goto fail; + } + } + } + return true; + +fail: + set_error_buf(error_buf, error_buf_size, + "fast interpreter offset overflow"); + return false; +} + +/* This function should be in front of wasm_loader_pop_frame_ref + as they both use ctx->stack_cell_num, and ctx->stack_cell_num + will be modified by wasm_loader_pop_frame_ref */ +static bool +wasm_loader_pop_frame_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size) +{ + /* if ctx->frame_csp equals ctx->frame_csp_bottom, + then current block is the function block */ + uint32 depth = ctx->frame_csp > ctx->frame_csp_bottom ? 1 : 0; + BranchBlock *cur_block = ctx->frame_csp - depth; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + + /* Directly return success if current block is in stack + * polymorphic state while stack is empty. */ + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + return true; + + if (type == VALUE_TYPE_VOID) + return true; + + if (is_32bit_type(type)) { + /* Check the offset stack bottom to ensure the frame offset + stack will not go underflow. But we don't thrown error + and return true here, because the error msg should be + given in wasm_loader_pop_frame_ref */ + if (!check_offset_pop(ctx, 1)) + return true; + + ctx->frame_offset -= 1; + if ((*(ctx->frame_offset) > ctx->start_dynamic_offset) + && (*(ctx->frame_offset) < ctx->max_dynamic_offset)) + ctx->dynamic_offset -= 1; + } + else { + if (!check_offset_pop(ctx, 2)) + return true; + + ctx->frame_offset -= 2; + if ((*(ctx->frame_offset) > ctx->start_dynamic_offset) + && (*(ctx->frame_offset) < ctx->max_dynamic_offset)) + ctx->dynamic_offset -= 2; + } + emit_operand(ctx, *(ctx->frame_offset)); + + (void)error_buf; + (void)error_buf_size; + return true; +} + +static bool +wasm_loader_push_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + if (!(wasm_loader_push_frame_offset(ctx, type, disable_emit, operand_offset, + error_buf, error_buf_size))) + return false; + if (!(wasm_loader_push_frame_ref(ctx, type, error_buf, error_buf_size))) + return false; + + return true; +} + +static bool +wasm_loader_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size) +{ + /* put wasm_loader_pop_frame_offset in front of wasm_loader_pop_frame_ref */ + if (!wasm_loader_pop_frame_offset(ctx, type, error_buf, error_buf_size)) + return false; + if (!wasm_loader_pop_frame_ref(ctx, type, error_buf, error_buf_size)) + return false; + + return true; +} + +static bool +wasm_loader_push_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 pop_cnt, + uint8 type_push, uint8 type_pop, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + uint8 i; + + for (i = 0; i < pop_cnt; i++) { + if (!wasm_loader_pop_frame_offset(ctx, type_pop, error_buf, + error_buf_size)) + return false; + + if (!wasm_loader_pop_frame_ref(ctx, type_pop, error_buf, + error_buf_size)) + return false; + } + + if (!wasm_loader_push_frame_offset(ctx, type_push, disable_emit, + operand_offset, error_buf, + error_buf_size)) + return false; + + if (!wasm_loader_push_frame_ref(ctx, type_push, error_buf, error_buf_size)) + return false; + + return true; +} + +static bool +wasm_loader_get_const_offset(WASMLoaderContext *ctx, uint8 type, void *value, + int16 *offset, char *error_buf, + uint32 error_buf_size) +{ + int8 bytes_to_increase; + int16 operand_offset = 0; + Const *c; + + /* Search existing constant */ + for (c = (Const *)ctx->const_buf; + (uint8 *)c < ctx->const_buf + ctx->num_const * sizeof(Const); c++) { + /* TODO: handle v128 type? */ + if ((type == c->value_type) + && ((type == VALUE_TYPE_I64 && *(int64 *)value == c->value.i64) + || (type == VALUE_TYPE_I32 && *(int32 *)value == c->value.i32) +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + || (type == VALUE_TYPE_FUNCREF + && *(int32 *)value == c->value.i32) + || (type == VALUE_TYPE_EXTERNREF + && *(int32 *)value == c->value.i32) +#endif + || (type == VALUE_TYPE_F64 + && (0 == memcmp(value, &(c->value.f64), sizeof(float64)))) + || (type == VALUE_TYPE_F32 + && (0 + == memcmp(value, &(c->value.f32), sizeof(float32)))))) { + operand_offset = c->slot_index; + break; + } + if (is_32bit_type(c->value_type)) + operand_offset += 1; + else + operand_offset += 2; + } + + if ((uint8 *)c == ctx->const_buf + ctx->num_const * sizeof(Const)) { + /* New constant, append to the const buffer */ + if ((type == VALUE_TYPE_F64) || (type == VALUE_TYPE_I64)) { + bytes_to_increase = 2; + } + else { + bytes_to_increase = 1; + } + + /* The max cell num of const buffer is 32768 since the valid index range + * is -32768 ~ -1. Return an invalid index 0 to indicate the buffer is + * full */ + if (ctx->const_cell_num > INT16_MAX - bytes_to_increase + 1) { + *offset = 0; + return true; + } + + if ((uint8 *)c == ctx->const_buf + ctx->const_buf_size) { + MEM_REALLOC(ctx->const_buf, ctx->const_buf_size, + ctx->const_buf_size + 4 * sizeof(Const)); + ctx->const_buf_size += 4 * sizeof(Const); + c = (Const *)(ctx->const_buf + ctx->num_const * sizeof(Const)); + } + c->value_type = type; + switch (type) { + case VALUE_TYPE_F64: + bh_memcpy_s(&(c->value.f64), sizeof(WASMValue), value, + sizeof(float64)); + ctx->const_cell_num += 2; + /* The const buf will be reversed, we use the second cell */ + /* of the i64/f64 const so the finnal offset is corrent */ + operand_offset++; + break; + case VALUE_TYPE_I64: + c->value.i64 = *(int64 *)value; + ctx->const_cell_num += 2; + operand_offset++; + break; + case VALUE_TYPE_F32: + bh_memcpy_s(&(c->value.f32), sizeof(WASMValue), value, + sizeof(float32)); + ctx->const_cell_num++; + break; + case VALUE_TYPE_I32: + c->value.i32 = *(int32 *)value; + ctx->const_cell_num++; + break; +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: + c->value.i32 = *(int32 *)value; + ctx->const_cell_num++; + break; +#endif + default: + break; + } + c->slot_index = operand_offset; + ctx->num_const++; + LOG_OP("#### new const [%d]: %ld\n", ctx->num_const, + (int64)c->value.i64); + } + /* use negetive index for const */ + operand_offset = -(operand_offset + 1); + *offset = operand_offset; + return true; +fail: + return false; +} + +/* + PUSH(POP)_XXX = push(pop) frame_ref + push(pop) frame_offset + -- Mostly used for the binary / compare operation + PUSH(POP)_OFFSET_TYPE only push(pop) the frame_offset stack + -- Mostly used in block / control instructions + + The POP will always emit the offset on the top of the frame_offset stack + PUSH can be used in two ways: + 1. directly PUSH: + PUSH_XXX(); + will allocate a dynamic space and emit + 2. silent PUSH: + operand_offset = xxx; disable_emit = true; + PUSH_XXX(); + only push the frame_offset stack, no emit +*/ + +#define TEMPLATE_PUSH(Type) \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, VALUE_TYPE_##Type, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define TEMPLATE_PUSH_REF(Type) \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, Type, disable_emit, \ + operand_offset, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP(Type) \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, VALUE_TYPE_##Type, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP_REF(Type) \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, Type, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_OFFSET_TYPE(type) \ + do { \ + if (!(wasm_loader_push_frame_offset(loader_ctx, type, disable_emit, \ + operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_OFFSET_TYPE(type) \ + do { \ + if (!(wasm_loader_pop_frame_offset(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref_offset( \ + loader_ctx, 1, type_push, type_pop, disable_emit, \ + operand_offset, error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +/* type of POPs should be the same */ +#define POP2_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref_offset( \ + loader_ctx, 2, type_push, type_pop, disable_emit, \ + operand_offset, error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#else /* WASM_ENABLE_FAST_INTERP */ + +#define TEMPLATE_PUSH(Type) \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, VALUE_TYPE_##Type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define TEMPLATE_PUSH_REF(Type) \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, Type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP(Type) \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_##Type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP_REF(Type) \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, Type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 1, type_push, \ + type_pop, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +/* type of POPs should be the same */ +#define POP2_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 2, type_push, \ + type_pop, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) +#endif /* WASM_ENABLE_FAST_INTERP */ + +#define PUSH_I32() TEMPLATE_PUSH(I32) +#define PUSH_F32() TEMPLATE_PUSH(F32) +#define PUSH_I64() TEMPLATE_PUSH(I64) +#define PUSH_F64() TEMPLATE_PUSH(F64) +#define PUSH_V128() TEMPLATE_PUSH(V128) +#define PUSH_FUNCREF() TEMPLATE_PUSH(FUNCREF) +#define PUSH_EXTERNREF() TEMPLATE_PUSH(EXTERNREF) +#define PUSH_REF(Type) TEMPLATE_PUSH_REF(Type) +#define POP_REF(Type) TEMPLATE_POP_REF(Type) +#define PUSH_MEM_OFFSET() TEMPLATE_PUSH_REF(mem_offset_type) +#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET() + +#define POP_I32() TEMPLATE_POP(I32) +#define POP_F32() TEMPLATE_POP(F32) +#define POP_I64() TEMPLATE_POP(I64) +#define POP_F64() TEMPLATE_POP(F64) +#define POP_V128() TEMPLATE_POP(V128) +#define POP_FUNCREF() TEMPLATE_POP(FUNCREF) +#define POP_EXTERNREF() TEMPLATE_POP(EXTERNREF) +#define POP_STRINGREF() TEMPLATE_POP(STRINGREF) +#define POP_MEM_OFFSET() TEMPLATE_POP_REF(mem_offset_type) + +#if WASM_ENABLE_FAST_INTERP != 0 + +static bool +reserve_block_ret(WASMLoaderContext *loader_ctx, uint8 opcode, + bool disable_emit, char *error_buf, uint32 error_buf_size) +{ + int16 operand_offset = 0; + BranchBlock *block = (opcode == WASM_OP_ELSE) ? loader_ctx->frame_csp - 1 + : loader_ctx->frame_csp; + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *reftype_maps = NULL; + uint32 reftype_map_count; +#endif + uint32 return_count = 0, value_count = 0, total_cel_num = 0; + int32 i = 0; + int16 dynamic_offset, dynamic_offset_org, *frame_offset = NULL, + *frame_offset_org = NULL; + +#if WASM_ENABLE_GC == 0 + return_count = block_type_get_result_types(block_type, &return_types); +#else + return_count = block_type_get_result_types( + block_type, &return_types, &reftype_maps, &reftype_map_count); +#endif + + /* If there is only one return value, use EXT_OP_COPY_STACK_TOP/_I64 instead + * of EXT_OP_COPY_STACK_VALUES for interpreter performance. */ + if (return_count == 1) { + uint8 cell = (uint8)wasm_value_type_cell_num(return_types[0]); + if (cell <= 2 /* V128 isn't supported whose cell num is 4 */ + && block->dynamic_offset != *(loader_ctx->frame_offset - cell)) { + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); + emit_label(cell == 1 ? EXT_OP_COPY_STACK_TOP + : EXT_OP_COPY_STACK_TOP_I64); + emit_operand(loader_ctx, *(loader_ctx->frame_offset - cell)); + emit_operand(loader_ctx, block->dynamic_offset); + + if (opcode == WASM_OP_ELSE) { + *(loader_ctx->frame_offset - cell) = block->dynamic_offset; + } + else { + loader_ctx->frame_offset -= cell; + loader_ctx->dynamic_offset = block->dynamic_offset; + PUSH_OFFSET_TYPE(return_types[0]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + } + return true; + } + + /* Copy stack top values to block's results which are in dynamic space. + * The instruction format: + * Part a: values count + * Part b: all values total cell num + * Part c: each value's cell_num, src offset and dst offset + * Part d: each value's src offset and dst offset + * Part e: each value's dst offset + */ + frame_offset = frame_offset_org = loader_ctx->frame_offset; + dynamic_offset = dynamic_offset_org = + block->dynamic_offset + wasm_get_cell_num(return_types, return_count); + + /* First traversal to get the count of values needed to be copied. */ + for (i = (int32)return_count - 1; i >= 0; i--) { + uint8 cells = (uint8)wasm_value_type_cell_num(return_types[i]); + + frame_offset -= cells; + dynamic_offset -= cells; + if (dynamic_offset != *frame_offset) { + value_count++; + total_cel_num += cells; + } + } + + if (value_count) { + uint32 j = 0; + uint8 *emit_data = NULL, *cells = NULL; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + uint64 size = + (uint64)value_count + * (sizeof(*cells) + sizeof(*src_offsets) + sizeof(*dst_offsets)); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + value_count); + dst_offsets = (uint16 *)(src_offsets + value_count); + + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, value_count); + /* Part b) */ + emit_uint32(loader_ctx, total_cel_num); + + /* Second traversal to get each value's cell num, src offset and dst + * offset. */ + frame_offset = frame_offset_org; + dynamic_offset = dynamic_offset_org; + for (i = (int32)return_count - 1, j = 0; i >= 0; i--) { + uint8 cell = (uint8)wasm_value_type_cell_num(return_types[i]); + frame_offset -= cell; + dynamic_offset -= cell; + if (dynamic_offset != *frame_offset) { + /* cell num */ + cells[j] = cell; + /* src offset */ + src_offsets[j] = *frame_offset; + /* dst offset */ + dst_offsets[j] = dynamic_offset; + j++; + } + if (opcode == WASM_OP_ELSE) { + *frame_offset = dynamic_offset; + } + else { + loader_ctx->frame_offset = frame_offset; + loader_ctx->dynamic_offset = dynamic_offset; + PUSH_OFFSET_TYPE(return_types[i]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + loader_ctx->frame_offset = frame_offset_org; + loader_ctx->dynamic_offset = dynamic_offset_org; + } + } + + bh_assert(j == value_count); + + /* Emit the cells, src_offsets and dst_offsets */ + for (j = 0; j < value_count; j++) + emit_byte(loader_ctx, cells[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, src_offsets[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, dst_offsets[j]); + + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + + wasm_runtime_free(emit_data); + } + + return true; + +fail: + return false; +} +#endif /* WASM_ENABLE_FAST_INTERP */ + +#define RESERVE_BLOCK_RET() \ + do { \ + if (!reserve_block_ret(loader_ctx, opcode, disable_emit, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_TYPE(type) \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_TYPE(type) \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#if WASM_ENABLE_GC == 0 +#define PUSH_CSP(label_type, block_type, _start_addr) \ + do { \ + if (!wasm_loader_push_frame_csp(loader_ctx, label_type, block_type, \ + _start_addr, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_CSP() \ + do { \ + if (!wasm_loader_pop_frame_csp(loader_ctx, error_buf, error_buf_size)) \ + goto fail; \ + } while (0) +#else +#define PUSH_CSP(label_type, block_type, _start_addr) \ + do { \ + if (!wasm_loader_push_frame_csp(loader_ctx, label_type, block_type, \ + _start_addr, error_buf, \ + error_buf_size)) \ + goto fail; \ + if (!wasm_loader_init_local_use_masks(loader_ctx, local_count, \ + error_buf, error_buf_size)) { \ + goto fail; \ + } \ + } while (0) + +#define POP_CSP() \ + do { \ + wasm_loader_destroy_curr_local_use_masks(loader_ctx); \ + if (!wasm_loader_pop_frame_csp(loader_ctx, error_buf, error_buf_size)) \ + goto fail; \ + } while (0) +#endif /* end of WASM_ENABLE_GC == 0 */ + +#if WASM_ENABLE_GC == 0 +#define GET_LOCAL_REFTYPE() (void)0 +#else +#define GET_LOCAL_REFTYPE() \ + do { \ + if (wasm_is_type_multi_byte_type(local_type)) { \ + WASMRefType *_ref_type; \ + if (local_idx < param_count) \ + _ref_type = wasm_reftype_map_find( \ + param_reftype_maps, param_reftype_map_count, local_idx); \ + else \ + _ref_type = wasm_reftype_map_find(local_reftype_maps, \ + local_reftype_map_count, \ + local_idx - param_count); \ + bh_assert(_ref_type); \ + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), _ref_type, \ + wasm_reftype_struct_size(_ref_type)); \ + } \ + } while (0) +#endif /* end of WASM_ENABLE_GC == 0 */ + +#define GET_LOCAL_INDEX_TYPE_AND_OFFSET() \ + do { \ + read_leb_uint32(p, p_end, local_idx); \ + if (local_idx >= param_count + local_count) { \ + set_error_buf(error_buf, error_buf_size, "unknown local"); \ + goto fail; \ + } \ + local_type = local_idx < param_count \ + ? param_types[local_idx] \ + : local_types[local_idx - param_count]; \ + local_offset = local_offsets[local_idx]; \ + GET_LOCAL_REFTYPE(); \ + } while (0) + +static bool +check_memory(WASMModule *module, char *error_buf, uint32 error_buf_size) +{ + if (module->memory_count == 0 && module->import_memory_count == 0) { + set_error_buf(error_buf, error_buf_size, "unknown memory"); + return false; + } + return true; +} + +#define CHECK_MEMORY() \ + do { \ + if (!check_memory(module, error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +static bool +check_memory_access_align(uint8 opcode, uint32 align, char *error_buf, + uint32 error_buf_size) +{ + uint8 mem_access_aligns[] = { + 2, 3, 2, 3, 0, 0, 1, 1, 0, 0, 1, 1, 2, 2, /* loads */ + 2, 3, 2, 3, 0, 1, 0, 1, 2 /* stores */ + }; + bh_assert(opcode >= WASM_OP_I32_LOAD && opcode <= WASM_OP_I64_STORE32); + if (align > mem_access_aligns[opcode - WASM_OP_I32_LOAD]) { + set_error_buf(error_buf, error_buf_size, + "alignment must not be larger than natural"); + return false; + } + return true; +} + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) +static bool +check_simd_memory_access_align(uint8 opcode, uint32 align, char *error_buf, + uint32 error_buf_size) +{ + uint8 mem_access_aligns[] = { + 4, /* load */ + 3, 3, 3, 3, 3, 3, /* load and extend */ + 0, 1, 2, 3, /* load and splat */ + 4, /* store */ + }; + + uint8 mem_access_aligns_load_lane[] = { + 0, 1, 2, 3, /* load lane */ + 0, 1, 2, 3, /* store lane */ + 2, 3 /* store zero */ + }; + + if (!((opcode <= SIMD_v128_store) + || (SIMD_v128_load8_lane <= opcode + && opcode <= SIMD_v128_load64_zero))) { + set_error_buf(error_buf, error_buf_size, + "the opcode doesn't include memarg"); + return false; + } + + if ((opcode <= SIMD_v128_store + && align > mem_access_aligns[opcode - SIMD_v128_load]) + || (SIMD_v128_load8_lane <= opcode && opcode <= SIMD_v128_load64_zero + && align > mem_access_aligns_load_lane[opcode + - SIMD_v128_load8_lane])) { + set_error_buf(error_buf, error_buf_size, + "alignment must not be larger than natural"); + return false; + } + + return true; +} + +static bool +check_simd_access_lane(uint8 opcode, uint8 lane, char *error_buf, + uint32 error_buf_size) +{ + switch (opcode) { + case SIMD_i8x16_extract_lane_s: + case SIMD_i8x16_extract_lane_u: + case SIMD_i8x16_replace_lane: + if (lane >= 16) { + goto fail; + } + break; + case SIMD_i16x8_extract_lane_s: + case SIMD_i16x8_extract_lane_u: + case SIMD_i16x8_replace_lane: + if (lane >= 8) { + goto fail; + } + break; + case SIMD_i32x4_extract_lane: + case SIMD_i32x4_replace_lane: + case SIMD_f32x4_extract_lane: + case SIMD_f32x4_replace_lane: + if (lane >= 4) { + goto fail; + } + break; + case SIMD_i64x2_extract_lane: + case SIMD_i64x2_replace_lane: + case SIMD_f64x2_extract_lane: + case SIMD_f64x2_replace_lane: + if (lane >= 2) { + goto fail; + } + break; + + case SIMD_v128_load8_lane: + case SIMD_v128_load16_lane: + case SIMD_v128_load32_lane: + case SIMD_v128_load64_lane: + case SIMD_v128_store8_lane: + case SIMD_v128_store16_lane: + case SIMD_v128_store32_lane: + case SIMD_v128_store64_lane: + case SIMD_v128_load32_zero: + case SIMD_v128_load64_zero: + { + uint8 max_lanes[] = { 16, 8, 4, 2, 16, 8, 4, 2, 4, 2 }; + if (lane >= max_lanes[opcode - SIMD_v128_load8_lane]) { + goto fail; + } + break; + } + default: + goto fail; + } + + return true; +fail: + set_error_buf(error_buf, error_buf_size, "invalid lane index"); + return false; +} + +static bool +check_simd_shuffle_mask(V128 mask, char *error_buf, uint32 error_buf_size) +{ + uint8 i; + for (i = 0; i != 16; ++i) { + if (mask.i8x16[i] < 0 || mask.i8x16[i] >= 32) { + set_error_buf(error_buf, error_buf_size, "invalid lane index"); + return false; + } + } + return true; +} +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 +static bool +check_memory_align_equal(uint8 opcode, uint32 align, char *error_buf, + uint32 error_buf_size) +{ + uint8 wait_notify_aligns[] = { 2, 2, 3 }; + uint8 mem_access_aligns[] = { + 2, 3, 0, 1, 0, 1, 2, + }; + uint8 expect; + + bh_assert((opcode <= WASM_OP_ATOMIC_WAIT64) + || (opcode >= WASM_OP_ATOMIC_I32_LOAD + && opcode <= WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U)); + if (opcode <= WASM_OP_ATOMIC_WAIT64) { + expect = wait_notify_aligns[opcode - WASM_OP_ATOMIC_NOTIFY]; + } + else { + /* 7 opcodes in every group */ + expect = mem_access_aligns[(opcode - WASM_OP_ATOMIC_I32_LOAD) % 7]; + } + if (align != expect) { + set_error_buf(error_buf, error_buf_size, + "alignment isn't equal to natural"); + return false; + } + return true; +} +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + +static bool +wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, + char *error_buf, uint32 error_buf_size) +{ + BranchBlock *target_block, *cur_block; + BlockType *target_block_type; + uint8 type, *types = NULL, *frame_ref; + uint32 arity = 0; + int32 i, available_stack_cell; + uint16 cell_num; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map; + WASMRefTypeMap *reftype_maps = NULL, *reftype_map = NULL; + WASMRefType *ref_type; + uint32 reftype_map_count = 0; + int32 available_reftype_map; + bool is_type_multi_byte; +#endif + + uint8 *frame_ref_old = loader_ctx->frame_ref; + uint8 *frame_ref_after_popped = NULL; + uint8 frame_ref_tmp[4] = { 0 }; + uint8 *frame_ref_buf = frame_ref_tmp; + uint32 stack_cell_num_old = loader_ctx->stack_cell_num; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map_old = loader_ctx->frame_reftype_map; + WASMRefTypeMap *frame_reftype_map_after_popped = NULL; + WASMRefTypeMap frame_reftype_map_tmp[4] = { 0 }; + WASMRefTypeMap *frame_reftype_map_buf = frame_reftype_map_tmp; + uint32 reftype_map_num_old = loader_ctx->reftype_map_num; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + int16 *frame_offset_old = loader_ctx->frame_offset; + int16 *frame_offset_after_popped = NULL; + int16 frame_offset_tmp[4] = { 0 }; + int16 *frame_offset_buf = frame_offset_tmp; + uint16 dynamic_offset_old = (loader_ctx->frame_csp - 1)->dynamic_offset; +#endif + bool ret = false; + + bh_assert(loader_ctx->csp_num > 0); + if (loader_ctx->csp_num - 1 < depth) { + set_error_buf(error_buf, error_buf_size, + "unknown label, " + "unexpected end of section or function"); + return false; + } + + cur_block = loader_ctx->frame_csp - 1; + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + frame_ref = loader_ctx->frame_ref; +#if WASM_ENABLE_GC != 0 + frame_reftype_map = loader_ctx->frame_reftype_map; +#endif + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ +#if WASM_ENABLE_GC == 0 + if (target_block->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(target_block_type, &types); + else + arity = block_type_get_result_types(target_block_type, &types); +#else + if (target_block->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(target_block_type, &types, + &reftype_maps, &reftype_map_count); + else + arity = block_type_get_result_types(target_block_type, &types, + &reftype_maps, &reftype_map_count); +#endif + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (cur_block->is_stack_polymorphic) { +#if WASM_ENABLE_GC != 0 + int32 j = reftype_map_count - 1; +#endif + for (i = (int32)arity - 1; i >= 0; i--) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(types[i])) { + bh_assert(reftype_maps[j].index == i); + bh_memcpy_s(loader_ctx->ref_type_tmp, sizeof(WASMRefType), + reftype_maps[j].ref_type, + wasm_reftype_struct_size(reftype_maps[j].ref_type)); + j--; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(types[i]); +#endif + POP_TYPE(types[i]); + } + + /* Backup stack data since it may be changed in the below + push operations, and the stack data may be used when + checking other target blocks of opcode br_table */ + if (opcode == WASM_OP_BR_TABLE) { + uint64 total_size; + + frame_ref_after_popped = loader_ctx->frame_ref; + total_size = (uint64)sizeof(uint8) + * (frame_ref_old - frame_ref_after_popped); + if (total_size > sizeof(frame_ref_tmp) + && !(frame_ref_buf = loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_ref_buf, (uint32)total_size, + frame_ref_after_popped, (uint32)total_size); + +#if WASM_ENABLE_GC != 0 + frame_reftype_map_after_popped = loader_ctx->frame_reftype_map; + total_size = + (uint64)sizeof(WASMRefTypeMap) + * (frame_reftype_map_old - frame_reftype_map_after_popped); + if (total_size > sizeof(frame_reftype_map_tmp) + && !(frame_reftype_map_buf = loader_malloc( + total_size, error_buf, error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_reftype_map_buf, (uint32)total_size, + frame_reftype_map_after_popped, (uint32)total_size); +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + frame_offset_after_popped = loader_ctx->frame_offset; + total_size = (uint64)sizeof(int16) + * (frame_offset_old - frame_offset_after_popped); + if (total_size > sizeof(frame_offset_tmp) + && !(frame_offset_buf = loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_offset_buf, (uint32)total_size, + frame_offset_after_popped, (uint32)total_size); +#endif + } + +#if WASM_ENABLE_GC != 0 + j = 0; +#endif + for (i = 0; i < (int32)arity; i++) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(types[i])) { + bh_assert(reftype_maps[j].index == i); + bh_memcpy_s(loader_ctx->ref_type_tmp, sizeof(WASMRefType), + reftype_maps[j].ref_type, + wasm_reftype_struct_size(reftype_maps[j].ref_type)); + j++; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(types[i]); +#endif + PUSH_TYPE(types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(target_block, opcode == WASM_OP_BR); +#endif + + /* Restore the stack data, note that frame_ref_bottom, + frame_reftype_map_bottom, frame_offset_bottom may be + re-allocated in the above push operations */ + if (opcode == WASM_OP_BR_TABLE) { + uint32 total_size; + + /* The stack operand num should not be smaller than before + after pop and push operations */ + bh_assert(loader_ctx->stack_cell_num >= stack_cell_num_old); + loader_ctx->stack_cell_num = stack_cell_num_old; + loader_ctx->frame_ref = + loader_ctx->frame_ref_bottom + stack_cell_num_old; + total_size = (uint32)(sizeof(uint8) + * (frame_ref_old - frame_ref_after_popped)); + bh_memcpy_s((uint8 *)loader_ctx->frame_ref - total_size, total_size, + frame_ref_buf, total_size); + +#if WASM_ENABLE_GC != 0 + /* The stack operand num should not be smaller than before + after pop and push operations */ + bh_assert(loader_ctx->reftype_map_num >= reftype_map_num_old); + loader_ctx->reftype_map_num = reftype_map_num_old; + loader_ctx->frame_reftype_map = + loader_ctx->frame_reftype_map_bottom + reftype_map_num_old; + total_size = (uint32)(sizeof(WASMRefTypeMap) + * (frame_reftype_map_old + - frame_reftype_map_after_popped)); + bh_memcpy_s((uint8 *)loader_ctx->frame_reftype_map - total_size, + total_size, frame_reftype_map_buf, total_size); +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + loader_ctx->frame_offset = + loader_ctx->frame_offset_bottom + stack_cell_num_old; + total_size = + (uint32)(sizeof(int16) + * (frame_offset_old - frame_offset_after_popped)); + bh_memcpy_s((uint8 *)loader_ctx->frame_offset - total_size, + total_size, frame_offset_buf, total_size); + (loader_ctx->frame_csp - 1)->dynamic_offset = dynamic_offset_old; +#endif + } + + ret = true; + goto cleanup_and_return; + } + + available_stack_cell = + (int32)(loader_ctx->stack_cell_num - cur_block->stack_cell_num); +#if WASM_ENABLE_GC != 0 + available_reftype_map = + (int32)(loader_ctx->reftype_map_num + - (loader_ctx->frame_csp - 1)->reftype_map_num); + reftype_map = reftype_maps ? reftype_maps + reftype_map_count - 1 : NULL; +#endif + + /* Check stack top values match target block type */ + for (i = (int32)arity - 1; i >= 0; i--) { + type = types[i]; +#if WASM_ENABLE_GC != 0 + ref_type = NULL; + is_type_multi_byte = wasm_is_type_multi_byte_type(type); + if (is_type_multi_byte) { + bh_assert(reftype_map); + ref_type = reftype_map->ref_type; + } +#endif + + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + break; + + if (!check_stack_top_values(loader_ctx, frame_ref, available_stack_cell, +#if WASM_ENABLE_GC != 0 + frame_reftype_map, available_reftype_map, +#endif + type, +#if WASM_ENABLE_GC != 0 + ref_type, +#endif + error_buf, error_buf_size)) { + goto fail; + } + cell_num = wasm_value_type_cell_num(types[i]); + frame_ref -= cell_num; + available_stack_cell -= cell_num; +#if WASM_ENABLE_GC != 0 + if (is_type_multi_byte) { + frame_reftype_map--; + available_reftype_map--; + reftype_map--; + } +#endif + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(target_block, opcode == WASM_OP_BR); +#endif + + ret = true; + +cleanup_and_return: +fail: + if (frame_ref_buf && frame_ref_buf != frame_ref_tmp) + wasm_runtime_free(frame_ref_buf); +#if WASM_ENABLE_GC != 0 + if (frame_reftype_map_buf && frame_reftype_map_buf != frame_reftype_map_tmp) + wasm_runtime_free(frame_reftype_map_buf); +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + if (frame_offset_buf && frame_offset_buf != frame_offset_tmp) + wasm_runtime_free(frame_offset_buf); +#endif + + return ret; +} + +static BranchBlock * +check_branch_block(WASMLoaderContext *loader_ctx, uint8 **p_buf, uint8 *buf_end, + uint8 opcode, char *error_buf, uint32 error_buf_size) +{ + uint8 *p = *p_buf, *p_end = buf_end; + BranchBlock *frame_csp_tmp; + uint32 depth; + + read_leb_uint32(p, p_end, depth); + if (!wasm_loader_check_br(loader_ctx, depth, opcode, error_buf, + error_buf_size)) { + goto fail; + } + + frame_csp_tmp = loader_ctx->frame_csp - depth - 1; + + *p_buf = p; + return frame_csp_tmp; +fail: + return NULL; +} + +#if WASM_ENABLE_EXCE_HANDLING != 0 +static BranchBlock * +check_branch_block_for_delegate(WASMLoaderContext *loader_ctx, uint8 **p_buf, + uint8 *buf_end, char *error_buf, + uint32 error_buf_size) +{ + uint8 *p = *p_buf, *p_end = buf_end; + BranchBlock *frame_csp_tmp; + uint32 depth; + + read_leb_uint32(p, p_end, depth); + /* + * Note: "delegate 0" means the surrounding block, not the + * try-delegate block itself. + * + * Note: the caller hasn't popped the try-delegate frame yet. + */ + bh_assert(loader_ctx->csp_num > 0); + if (loader_ctx->csp_num - 1 <= depth) { +#if WASM_ENABLE_SPEC_TEST == 0 + set_error_buf(error_buf, error_buf_size, "unknown delegate label"); +#else + set_error_buf(error_buf, error_buf_size, "unknown label"); +#endif + goto fail; + } + frame_csp_tmp = loader_ctx->frame_csp - depth - 2; +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(frame_csp_tmp, false); +#endif + + *p_buf = p; + return frame_csp_tmp; +fail: + return NULL; +} +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + +static bool +check_block_stack(WASMLoaderContext *loader_ctx, BranchBlock *block, + char *error_buf, uint32 error_buf_size) +{ + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0; + int32 available_stack_cell, return_cell_num, i; + uint8 *frame_ref = NULL; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map; + WASMRefTypeMap *return_reftype_maps = NULL, *return_reftype_map; + WASMRefType *ref_type; + uint32 param_count, return_reftype_map_count = 0; + int32 available_reftype_map = + (int32)(loader_ctx->reftype_map_num - block->reftype_map_num); +#endif + + available_stack_cell = + (int32)(loader_ctx->stack_cell_num - block->stack_cell_num); + +#if WASM_ENABLE_GC == 0 + return_count = block_type_get_result_types(block_type, &return_types); +#else + return_count = block_type_get_result_types(block_type, &return_types, + &return_reftype_maps, + &return_reftype_map_count); + param_count = + block_type->is_value_type ? 0 : block_type->u.type->param_count; + (void)param_count; +#endif + return_cell_num = + return_count > 0 ? wasm_get_cell_num(return_types, return_count) : 0; + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (block->is_stack_polymorphic) { +#if WASM_ENABLE_GC != 0 + int32 j = return_reftype_map_count - 1; +#endif + for (i = (int32)return_count - 1; i >= 0; i--) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(return_types[i])) { + bh_assert(return_reftype_maps[j].index == i + param_count); + bh_memcpy_s( + loader_ctx->ref_type_tmp, sizeof(WASMRefType), + return_reftype_maps[j].ref_type, + wasm_reftype_struct_size(return_reftype_maps[j].ref_type)); + j--; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(return_types[i]); +#endif + POP_TYPE(return_types[i]); + } + + /* Check stack is empty */ + if (loader_ctx->stack_cell_num != block->stack_cell_num) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: stack size does not match block type"); + goto fail; + } + +#if WASM_ENABLE_GC != 0 + j = 0; +#endif + for (i = 0; i < (int32)return_count; i++) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(return_types[i])) { + bh_assert(return_reftype_maps[j].index == i + param_count); + bh_memcpy_s( + loader_ctx->ref_type_tmp, sizeof(WASMRefType), + return_reftype_maps[j].ref_type, + wasm_reftype_struct_size(return_reftype_maps[j].ref_type)); + j++; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(return_types[i]); +#endif + PUSH_TYPE(return_types[i]); + } + return true; + } + + if (available_stack_cell != return_cell_num) { +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* testspec: this error message format is expected by try_catch.wast */ + snprintf( + error_buf, error_buf_size, "type mismatch: %s requires [%s]%s[%s]", + block->label_type == LABEL_TYPE_TRY + || (block->label_type == LABEL_TYPE_CATCH + && return_cell_num > 0) + ? "instruction" + : "block", + return_cell_num > 0 ? type2str(return_types[0]) : "", + " but stack has ", + available_stack_cell > 0 ? type2str(*(loader_ctx->frame_ref - 1)) + : ""); + goto fail; +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: stack size does not match block type"); + goto fail; +#endif + } + + /* Check stack values match return types */ + frame_ref = loader_ctx->frame_ref; +#if WASM_ENABLE_GC != 0 + frame_reftype_map = loader_ctx->frame_reftype_map; + return_reftype_map = + return_reftype_map_count + ? return_reftype_maps + return_reftype_map_count - 1 + : NULL; +#endif + for (i = (int32)return_count - 1; i >= 0; i--) { + uint8 type = return_types[i]; +#if WASM_ENABLE_GC != 0 + bool is_type_multi_byte = wasm_is_type_multi_byte_type(type); + ref_type = NULL; + if (is_type_multi_byte) { + bh_assert(return_reftype_map); + ref_type = return_reftype_map->ref_type; + } +#endif + if (!check_stack_top_values(loader_ctx, frame_ref, available_stack_cell, +#if WASM_ENABLE_GC != 0 + frame_reftype_map, available_reftype_map, +#endif + type, +#if WASM_ENABLE_GC != 0 + ref_type, +#endif + error_buf, error_buf_size)) + return false; + frame_ref -= wasm_value_type_cell_num(return_types[i]); + available_stack_cell -= wasm_value_type_cell_num(return_types[i]); +#if WASM_ENABLE_GC != 0 + if (is_type_multi_byte) { + frame_reftype_map--; + available_reftype_map--; + return_reftype_map--; + } +#endif + } + + return true; + +fail: + return false; +} + +#if WASM_ENABLE_FAST_INTERP != 0 +/* Copy parameters to dynamic space. + * 1) POP original parameter out; + * 2) Push and copy original values to dynamic space. + * The copy instruction format: + * Part a: param count + * Part b: all param total cell num + * Part c: each param's cell_num, src offset and dst offset + * Part d: each param's src offset + * Part e: each param's dst offset + */ +static bool +copy_params_to_dynamic_space(WASMLoaderContext *loader_ctx, bool is_if_block, + char *error_buf, uint32 error_buf_size) +{ + bool ret = false; + int16 *frame_offset = NULL; + uint8 *cells = NULL, cell; + int16 *src_offsets = NULL; + uint8 *emit_data = NULL; + uint32 i; + BranchBlock *block = loader_ctx->frame_csp - 1; + BlockType *block_type = &block->block_type; + WASMFuncType *wasm_type = block_type->u.type; + uint32 param_count = block_type->u.type->param_count; + int16 condition_offset = 0; + bool disable_emit = false; + int16 operand_offset = 0; + + uint64 size = (uint64)param_count * (sizeof(*cells) + sizeof(*src_offsets)); + bh_assert(size > 0); + + /* For if block, we also need copy the condition operand offset. */ + if (is_if_block) + size += sizeof(*cells) + sizeof(*src_offsets); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + param_count); + + if (is_if_block) + condition_offset = *loader_ctx->frame_offset; + + /* POP original parameter out */ + for (i = 0; i < param_count; i++) { + POP_OFFSET_TYPE(wasm_type->types[param_count - i - 1]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + frame_offset = loader_ctx->frame_offset; + + /* Get each param's cell num and src offset */ + for (i = 0; i < param_count; i++) { + cell = (uint8)wasm_value_type_cell_num(wasm_type->types[i]); + cells[i] = cell; + src_offsets[i] = *frame_offset; + frame_offset += cell; + } + /* emit copy instruction */ + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, is_if_block ? param_count + 1 : param_count); + /* Part b) */ + emit_uint32(loader_ctx, is_if_block ? wasm_type->param_cell_num + 1 + : wasm_type->param_cell_num); + /* Part c) */ + for (i = 0; i < param_count; i++) + emit_byte(loader_ctx, cells[i]); + if (is_if_block) + emit_byte(loader_ctx, 1); + + /* Part d) */ + for (i = 0; i < param_count; i++) + emit_operand(loader_ctx, src_offsets[i]); + if (is_if_block) + emit_operand(loader_ctx, condition_offset); + + /* Part e) */ + /* Push to dynamic space. The push will emit the dst offset. */ + for (i = 0; i < param_count; i++) + PUSH_OFFSET_TYPE(wasm_type->types[i]); + if (is_if_block) + PUSH_OFFSET_TYPE(VALUE_TYPE_I32); + + ret = true; + +fail: + /* Free the emit data */ + wasm_runtime_free(emit_data); + + return ret; +} +#endif + +#if WASM_ENABLE_GC == 0 +#define RESET_REFTYPE_MAP_STACK() (void)0 +#else +#define RESET_REFTYPE_MAP_STACK() \ + do { \ + loader_ctx->reftype_map_num = \ + (loader_ctx->frame_csp - 1)->reftype_map_num; \ + loader_ctx->frame_reftype_map = loader_ctx->frame_reftype_map_bottom \ + + loader_ctx->reftype_map_num; \ + } while (0) +#endif + +/* reset the stack to the state of before entering the last block */ +#if WASM_ENABLE_FAST_INTERP != 0 +#define RESET_STACK() \ + do { \ + loader_ctx->stack_cell_num = \ + (loader_ctx->frame_csp - 1)->stack_cell_num; \ + loader_ctx->frame_ref = \ + loader_ctx->frame_ref_bottom + loader_ctx->stack_cell_num; \ + loader_ctx->frame_offset = \ + loader_ctx->frame_offset_bottom + loader_ctx->stack_cell_num; \ + RESET_REFTYPE_MAP_STACK(); \ + } while (0) +#else +#define RESET_STACK() \ + do { \ + loader_ctx->stack_cell_num = \ + (loader_ctx->frame_csp - 1)->stack_cell_num; \ + loader_ctx->frame_ref = \ + loader_ctx->frame_ref_bottom + loader_ctx->stack_cell_num; \ + RESET_REFTYPE_MAP_STACK(); \ + } while (0) +#endif + +/* set current block's stack polymorphic state */ +#define SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(flag) \ + do { \ + BranchBlock *_cur_block = loader_ctx->frame_csp - 1; \ + _cur_block->is_stack_polymorphic = flag; \ + } while (0) + +#define BLOCK_HAS_PARAM(block_type) \ + (!block_type.is_value_type && block_type.u.type->param_count > 0) + +#define PRESERVE_LOCAL_FOR_BLOCK() \ + do { \ + if (!(preserve_local_for_block(loader_ctx, opcode, error_buf, \ + error_buf_size))) { \ + goto fail; \ + } \ + } while (0) + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +static bool +get_table_elem_type(const WASMModule *module, uint32 table_idx, + uint8 *p_elem_type, void **p_ref_type, char *error_buf, + uint32 error_buf_size) +{ + if (!check_table_index(module, table_idx, error_buf, error_buf_size)) { + return false; + } + + if (table_idx < module->import_table_count) { + if (p_elem_type) + *p_elem_type = module->import_tables[table_idx].u.table.elem_type; +#if WASM_ENABLE_GC != 0 + if (p_ref_type) + *((WASMRefType **)p_ref_type) = + module->import_tables[table_idx].u.table.elem_ref_type; +#endif + } + else { + if (p_elem_type) + *p_elem_type = + module->tables[module->import_table_count + table_idx] + .elem_type; +#if WASM_ENABLE_GC != 0 + if (p_ref_type) + *((WASMRefType **)p_ref_type) = + module->tables[module->import_table_count + table_idx] + .elem_ref_type; +#endif + } + return true; +} + +static bool +get_table_seg_elem_type(const WASMModule *module, uint32 table_seg_idx, + uint8 *p_elem_type, void **p_elem_ref_type, + char *error_buf, uint32 error_buf_size) +{ + if (table_seg_idx >= module->table_seg_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown elem segment %u", + table_seg_idx); + return false; + } + + if (p_elem_type) { + *p_elem_type = module->table_segments[table_seg_idx].elem_type; + } +#if WASM_ENABLE_GC != 0 + if (p_elem_ref_type) + *((WASMRefType **)p_elem_ref_type) = + module->table_segments[table_seg_idx].elem_ref_type; +#endif + return true; +} +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +const uint8 * +wasm_loader_get_custom_section(WASMModule *module, const char *name, + uint32 *len) +{ + WASMCustomSection *section = module->custom_section_list; + + while (section) { + if ((section->name_len == strlen(name)) + && (memcmp(section->name_addr, name, section->name_len) == 0)) { + if (len) { + *len = section->content_len; + } + return section->content_addr; + } + + section = section->next; + } + + return NULL; +} +#endif + +static bool +wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, + uint32 cur_func_idx, char *error_buf, + uint32 error_buf_size) +{ + uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org; + uint32 param_count, local_count, global_count; + uint8 *param_types, *local_types, local_type, global_type, mem_offset_type; + BlockType func_block_type; + uint16 *local_offsets, local_offset; + uint32 type_idx, func_idx, local_idx, global_idx, table_idx; + uint32 table_seg_idx, data_seg_idx, count, align, i; + mem_offset_t mem_offset; + int32 i32_const = 0; + int64 i64_const; + uint8 opcode; + bool return_value = false; + WASMLoaderContext *loader_ctx; + BranchBlock *frame_csp_tmp; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *param_reftype_maps, *local_reftype_maps; + uint32 param_reftype_map_count, local_reftype_map_count; + int32 heap_type; + WASMRefType wasm_ref_type = { 0 }; + bool need_ref_type_map; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *func_const_end, *func_const = NULL; + int16 operand_offset = 0; + uint8 last_op = 0; + bool disable_emit, preserve_local = false, if_condition_available = true; + float32 f32_const; + float64 f64_const; + + LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n", + func->param_cell_num, func->local_cell_num, func->ret_cell_num); +#endif +#if WASM_ENABLE_MEMORY64 != 0 + bool is_memory64 = has_module_memory64(module); + mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif + + global_count = module->import_global_count + module->global_count; + + param_count = func->func_type->param_count; + param_types = func->func_type->types; + + func_block_type.is_value_type = false; + func_block_type.u.type = func->func_type; + + local_count = func->local_count; + local_types = func->local_types; + local_offsets = func->local_offsets; + +#if WASM_ENABLE_GC != 0 + param_reftype_maps = func->func_type->ref_type_maps; + param_reftype_map_count = func->func_type->ref_type_map_count; + local_reftype_maps = func->local_ref_type_maps; + local_reftype_map_count = func->local_ref_type_map_count; +#endif + + if (!(loader_ctx = wasm_loader_ctx_init(func, error_buf, error_buf_size))) { + goto fail; + } +#if WASM_ENABLE_GC != 0 + loader_ctx->module = module; + loader_ctx->ref_type_set = module->ref_type_set; + loader_ctx->ref_type_tmp = &wasm_ref_type; +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* For the first traverse, the initial value of preserved_local_offset has + * not been determined, we use the INT16_MAX to represent that a slot has + * been copied to preserve space. For second traverse, this field will be + * set to the appropriate value in wasm_loader_ctx_reinit. + * This is for Issue #1230, + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/1230, the + * drop opcodes need to know which slots are preserved, so those slots will + * not be treated as dynamically allocated slots */ + loader_ctx->preserved_local_offset = INT16_MAX; + +re_scan: + if (loader_ctx->code_compiled_size > 0) { + if (!wasm_loader_ctx_reinit(loader_ctx)) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + goto fail; + } + p = func->code; + func->code_compiled = loader_ctx->p_code_compiled; + func->code_compiled_size = loader_ctx->code_compiled_size; + } +#endif + + PUSH_CSP(LABEL_TYPE_FUNCTION, func_block_type, p); + + while (p < p_end) { + opcode = *p++; +#if WASM_ENABLE_FAST_INTERP != 0 + p_org = p; + disable_emit = false; + emit_label(opcode); +#endif + switch (opcode) { + case WASM_OP_UNREACHABLE: + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + + case WASM_OP_NOP: +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); +#endif + break; + + case WASM_OP_IF: + { +#if WASM_ENABLE_FAST_INTERP != 0 + BranchBlock *parent_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - parent_block->stack_cell_num); + + if (available_stack_cell <= 0 + && parent_block->is_stack_polymorphic) + if_condition_available = false; + else + if_condition_available = true; + + PRESERVE_LOCAL_FOR_BLOCK(); +#endif +#if WASM_ENABLE_GC == 0 + POP_I32(); +#endif + goto handle_op_block_and_loop; + } + case WASM_OP_BLOCK: + case WASM_OP_LOOP: +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_TRY: + if (opcode == WASM_OP_TRY) { + /* + * keep track of exception handlers to account for + * memory allocation + */ + func->exception_handler_count++; + + /* + * try is a block + * do nothing special, but execution continues to + * to handle_op_block_and_loop, + * and that be pushes the csp + */ + } + +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + PRESERVE_LOCAL_FOR_BLOCK(); +#endif + handle_op_block_and_loop: + { + uint8 value_type; + BlockType block_type; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 available_params = 0; +#endif + + CHECK_BUF(p, p_end, 1); + value_type = read_uint8(p); + if (is_byte_a_type(value_type)) { + /* If the first byte is one of these special values: + * 0x40/0x7F/0x7E/0x7D/0x7C, take it as the type of + * the single return value. */ + block_type.is_value_type = true; + block_type.u.value_type.type = value_type; +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (value_type == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif +#if WASM_ENABLE_GC != 0 + if (value_type != VALUE_TYPE_VOID) { + p_org = p; + p--; + if (!resolve_value_type((const uint8 **)&p, p_end, + module, module->type_count, + &need_ref_type_map, + &wasm_ref_type, false, + error_buf, error_buf_size)) { + goto fail; + } + if (need_ref_type_map) { + block_type.u.value_type.ref_type_map.index = 0; + if (!(block_type.u.value_type.ref_type_map + .ref_type = reftype_set_insert( + module->ref_type_set, &wasm_ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + /* Set again as the type might be changed, e.g. + (ref null any) to anyref */ + block_type.u.value_type.type = wasm_ref_type.ref_type; +#if WASM_ENABLE_FAST_INTERP == 0 + while (p_org < p) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, + error_buf, error_buf_size)) { + goto fail; + } +#endif + /* Ignore extra bytes for interpreter */ + *p_org++ = WASM_OP_NOP; + } +#endif + } +#endif /* end of WASM_ENABLE_GC != 0 */ + } + else { + uint32 type_index; + /* Resolve the leb128 encoded type index as block type */ + p--; + p_org = p - 1; + read_leb_uint32(p, p_end, type_index); + if (type_index >= module->type_count) { + set_error_buf(error_buf, error_buf_size, + "unknown type"); + goto fail; + } + block_type.is_value_type = false; + block_type.u.type = + (WASMFuncType *)module->types[type_index]; +#if WASM_ENABLE_FAST_INTERP == 0 + /* If block use type index as block type, change the opcode + * to new extended opcode so that interpreter can resolve + * the block quickly. + */ +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = EXT_OP_BLOCK + (opcode - WASM_OP_BLOCK); +#endif + } + +#if WASM_ENABLE_GC != 0 + if (opcode == WASM_OP_IF) { + POP_I32(); + } +#endif + + /* Pop block parameters from stack */ + if (BLOCK_HAS_PARAM(block_type)) { + WASMFuncType *wasm_type = block_type.u.type; + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 cell_num; + available_params = block_type.u.type->param_count; +#endif + for (i = 0; i < block_type.u.type->param_count; i++) { + + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + if (available_stack_cell <= 0 + && cur_block->is_stack_polymorphic) { +#if WASM_ENABLE_FAST_INTERP != 0 + available_params = i; +#endif + break; + } + + POP_TYPE( + wasm_type->types[wasm_type->param_count - i - 1]); +#if WASM_ENABLE_FAST_INTERP != 0 + /* decrease the frame_offset pointer accordingly to keep + * consistent with frame_ref stack */ + cell_num = wasm_value_type_cell_num( + wasm_type->types[wasm_type->param_count - i - 1]); + loader_ctx->frame_offset -= cell_num; +#endif + } + } + PUSH_CSP(LABEL_TYPE_BLOCK + (opcode - WASM_OP_BLOCK), + block_type, p); + + /* Pass parameters to block */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 cell_num = wasm_value_type_cell_num( + block_type.u.type->types[i]); + if (i >= available_params) { + /* If there isn't enough data on stack, push a dummy + * offset to keep the stack consistent with + * frame_ref. + * Since the stack is already in polymorphic state, + * the opcode will not be executed, so the dummy + * offset won't cause any error */ + uint32 n; + + for (n = 0; n < cell_num; n++) { + if (loader_ctx->p_code_compiled == NULL) { + if (!check_offset_push(loader_ctx, + error_buf, + error_buf_size)) + goto fail; + } + *loader_ctx->frame_offset++ = 0; + } + } + else { + loader_ctx->frame_offset += cell_num; + } +#endif + PUSH_TYPE(block_type.u.type->types[i]); + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (opcode == WASM_OP_BLOCK || opcode == WASM_OP_LOOP) { + skip_label(); + + if (BLOCK_HAS_PARAM(block_type)) { + /* Make sure params are in dynamic space */ + if (!copy_params_to_dynamic_space( + loader_ctx, false, error_buf, error_buf_size)) + goto fail; + } + + if (opcode == WASM_OP_LOOP) { + (loader_ctx->frame_csp - 1)->code_compiled = + loader_ctx->p_code_compiled; + } + } +#if WASM_ENABLE_EXCE_HANDLING != 0 + else if (opcode == WASM_OP_TRY) { + skip_label(); + } +#endif + else if (opcode == WASM_OP_IF) { + BranchBlock *block = loader_ctx->frame_csp - 1; + /* If block has parameters, we should make sure they are in + * dynamic space. Otherwise, when else branch is missing, + * the later opcode may consume incorrect operand offset. + * Spec case: + * (func (export "params-id") (param i32) (result i32) + * (i32.const 1) + * (i32.const 2) + * (if (param i32 i32) (result i32 i32) (local.get 0) + * (then)) (i32.add) + * ) + * + * So we should emit a copy instruction before the if. + * + * And we also need to save the parameter offsets and + * recover them before entering else branch. + * + */ + if (BLOCK_HAS_PARAM(block_type)) { + uint64 size; + + /* In polymorphic state, there may be no if condition on + * the stack, so the offset may not emitted */ + if (if_condition_available) { + /* skip the if condition operand offset */ + wasm_loader_emit_backspace(loader_ctx, + sizeof(int16)); + } + /* skip the if label */ + skip_label(); + /* Emit a copy instruction */ + if (!copy_params_to_dynamic_space( + loader_ctx, true, error_buf, error_buf_size)) + goto fail; + + /* Emit the if instruction */ + emit_label(opcode); + /* Emit the new condition operand offset */ + POP_OFFSET_TYPE(VALUE_TYPE_I32); + + /* Save top param_count values of frame_offset stack, so + * that we can recover it before executing else branch + */ + size = sizeof(int16) + * (uint64)block_type.u.type->param_cell_num; + if (!(block->param_frame_offsets = loader_malloc( + size, error_buf, error_buf_size))) + goto fail; + bh_memcpy_s(block->param_frame_offsets, (uint32)size, + loader_ctx->frame_offset + - size / sizeof(int16), + (uint32)size); + } + + block->start_dynamic_offset = loader_ctx->dynamic_offset; + + emit_empty_label_addr_and_frame_ip(PATCH_ELSE); + emit_empty_label_addr_and_frame_ip(PATCH_END); + } +#endif + break; + } +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_THROW: + { + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + uint8 label_type = cur_block->label_type; + uint32 tag_index = 0; + read_leb_int32(p, p_end, tag_index); + + /* check validity of tag_index against module->tag_count */ + /* check tag index is within the tag index space */ + if (tag_index >= module->import_tag_count + module->tag_count) { + snprintf(error_buf, error_buf_size, "unknown tag %d", + tag_index); + goto fail; + } + + /* the tag_type is stored in either the WASMTag (section tags) + * or WASMTagImport (import tag) */ + WASMType *tag_type = NULL; + if (tag_index < module->import_tag_count) { + tag_type = module->import_tags[tag_index].u.tag.tag_type; + } + else { + tag_type = + module->tags[tag_index - module->import_tag_count] + ->tag_type; + } + + if (tag_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "tag type signature does not return void"); + goto fail; + } + + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + + /* Check stack values match return types by comparing tag param + * types with stack cells */ + uint8 *frame_ref = loader_ctx->frame_ref; + for (int tti = (int32)tag_type->param_count - 1; tti >= 0; + tti--) { + if (!check_stack_top_values( + loader_ctx, frame_ref, available_stack_cell, + tag_type->types[tti], error_buf, error_buf_size)) { + snprintf(error_buf, error_buf_size, + "type mismatch: instruction requires [%s] but " + "stack has [%s]", + tag_type->param_count > 0 + ? type2str(tag_type->types[tti]) + : "", + available_stack_cell > 0 + ? type2str(*(loader_ctx->frame_ref - 1)) + : ""); + goto fail; + } + frame_ref -= wasm_value_type_cell_num(tag_type->types[tti]); + available_stack_cell -= + wasm_value_type_cell_num(tag_type->types[tti]); + } + + /* throw is stack polymorphic */ + (void)label_type; + RESET_STACK(); + + break; + } + case WASM_OP_RETHROW: + { + /* must be done before checking branch block */ + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + /* check the target catching block: LABEL_TYPE_CATCH */ + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + if (frame_csp_tmp->label_type != LABEL_TYPE_CATCH + && frame_csp_tmp->label_type != LABEL_TYPE_CATCH_ALL) { + /* trap according to spectest (rethrow.wast) */ + set_error_buf(error_buf, error_buf_size, + "invalid rethrow label"); + goto fail; + } + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + uint8 label_type = cur_block->label_type; + (void)label_type; + /* rethrow is stack polymorphic */ + RESET_STACK(); + break; + } + case WASM_OP_DELEGATE: + { + /* check target block is valid */ + if (!(frame_csp_tmp = check_branch_block_for_delegate( + loader_ctx, &p, p_end, error_buf, error_buf_size))) + goto fail; + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + uint8 label_type = cur_block->label_type; + + (void)label_type; + /* DELEGATE ends the block */ + POP_CSP(); + break; + } + case WASM_OP_CATCH: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + uint8 label_type = cur_block->label_type; + uint32 tag_index = 0; + read_leb_int32(p, p_end, tag_index); + + /* check validity of tag_index against module->tag_count */ + /* check tag index is within the tag index space */ + if (tag_index >= module->import_tag_count + module->tag_count) { + LOG_VERBOSE("In %s, unknown tag at WASM_OP_CATCH\n", + __FUNCTION__); + set_error_buf(error_buf, error_buf_size, "unknown tag"); + goto fail; + } + + /* the tag_type is stored in either the WASMTag (section tags) + * or WASMTagImport (import tag) */ + WASMType *func_type = NULL; + if (tag_index < module->import_tag_count) { + func_type = module->import_tags[tag_index].u.tag.tag_type; + } + else { + func_type = + module->tags[tag_index - module->import_tag_count] + ->tag_type; + } + + if (func_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "tag type signature does not return void"); + goto fail; + } + + /* check validity of current label (expect LABEL_TYPE_TRY or + * LABEL_TYPE_CATCH) */ + if ((LABEL_TYPE_CATCH != label_type) + && (LABEL_TYPE_TRY != label_type)) { + set_error_buf(error_buf, error_buf_size, + "Unexpected block sequence encountered."); + goto fail; + } + + BlockType new_block_type; + new_block_type.is_value_type = false; + new_block_type.u.type = func_type; + + /* + * replace frame_csp by LABEL_TYPE_CATCH + */ + cur_block->label_type = LABEL_TYPE_CATCH; + + /* RESET_STACK removes the values pushed in TRY or pervious + * CATCH Blocks */ + RESET_STACK(); + + /* push types on the stack according to catched type */ + if (BLOCK_HAS_PARAM(new_block_type)) { + for (i = 0; i < new_block_type.u.type->param_count; i++) + PUSH_TYPE(new_block_type.u.type->types[i]); + } + break; + } + case WASM_OP_CATCH_ALL: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + /* expecting a TRY or CATCH, anything else will be considered an + * error */ + if ((LABEL_TYPE_CATCH != cur_block->label_type) + && (LABEL_TYPE_TRY != cur_block->label_type)) { + set_error_buf(error_buf, error_buf_size, + "Unexpected block sequence encountered."); + goto fail; + } + + /* no immediates */ + /* replace frame_csp by LABEL_TYPE_CATCH_ALL */ + cur_block->label_type = LABEL_TYPE_CATCH_ALL; + + /* RESET_STACK removes the values pushed in TRY or pervious + * CATCH Blocks */ + RESET_STACK(); + + /* catch_all has no tagtype and therefore no parameters */ + break; + } +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + case WASM_OP_ELSE: + { + BranchBlock *block = NULL; + BlockType block_type; + + if (loader_ctx->csp_num < 2 + /* the matched if isn't found */ + || (loader_ctx->frame_csp - 1)->label_type != LABEL_TYPE_IF + /* duplicated else is found */ + || (loader_ctx->frame_csp - 1)->else_addr) { + set_error_buf( + error_buf, error_buf_size, + "opcode else found without matched opcode if"); + goto fail; + } + block = loader_ctx->frame_csp - 1; + + /* check whether if branch's stack matches its result type */ + if (!check_block_stack(loader_ctx, block, error_buf, + error_buf_size)) + goto fail; + + block->else_addr = p - 1; + block_type = block->block_type; + +#if WASM_ENABLE_GC != 0 + if (!wasm_loader_init_local_use_masks( + loader_ctx, local_count, error_buf, error_buf_size)) { + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* if the result of if branch is in local or const area, add a + * copy op */ + RESERVE_BLOCK_RET(); + + emit_empty_label_addr_and_frame_ip(PATCH_END); + apply_label_patch(loader_ctx, 1, PATCH_ELSE); +#endif + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(false); + + /* Pass parameters to if-false branch */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) + PUSH_TYPE(block_type.u.type->types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Recover top param_count values of frame_offset stack */ + if (BLOCK_HAS_PARAM((block_type))) { + uint32 size; + size = sizeof(int16) * block_type.u.type->param_cell_num; + bh_memcpy_s(loader_ctx->frame_offset, size, + block->param_frame_offsets, size); + loader_ctx->frame_offset += (size / sizeof(int16)); + } + loader_ctx->dynamic_offset = block->start_dynamic_offset; +#endif + + break; + } + + case WASM_OP_END: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + /* check whether block stack matches its result type */ + if (!check_block_stack(loader_ctx, cur_block, error_buf, + error_buf_size)) + goto fail; + + /* if no else branch, and return types do not match param types, + report failure */ + if (cur_block->label_type == LABEL_TYPE_IF + && !cur_block->else_addr) { + uint32 block_param_count = 0, block_ret_count = 0; + uint8 *block_param_types = NULL, *block_ret_types = NULL; + BlockType *cur_block_type = &cur_block->block_type; +#if WASM_ENABLE_GC != 0 + uint32 block_param_reftype_map_count; + uint32 block_ret_reftype_map_count; + WASMRefTypeMap *block_param_reftype_maps; + WASMRefTypeMap *block_ret_reftype_maps; +#endif + + block_param_count = block_type_get_param_types( + cur_block_type, &block_param_types +#if WASM_ENABLE_GC != 0 + , + &block_param_reftype_maps, + &block_param_reftype_map_count +#endif + ); + block_ret_count = block_type_get_result_types( + cur_block_type, &block_ret_types +#if WASM_ENABLE_GC != 0 + , + &block_ret_reftype_maps, &block_ret_reftype_map_count +#endif + ); + + if (block_param_count != block_ret_count + || (block_param_count + && memcmp(block_param_types, block_ret_types, + block_param_count))) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: else branch missing"); + goto fail; + } +#if WASM_ENABLE_GC != 0 + if (block_param_reftype_map_count + != block_ret_reftype_map_count + || (block_param_reftype_map_count + && memcmp(block_param_reftype_maps, + block_ret_reftype_maps, + sizeof(WASMRefTypeMap) + * block_param_reftype_map_count))) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: else branch missing"); + goto fail; + } +#endif + } + + POP_CSP(); + +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + /* copy the result to the block return address */ + RESERVE_BLOCK_RET(); + + apply_label_patch(loader_ctx, 0, PATCH_END); + free_label_patch_list(loader_ctx->frame_csp); + if (loader_ctx->frame_csp->label_type == LABEL_TYPE_FUNCTION) { + int32 idx; + uint8 ret_type; + + emit_label(WASM_OP_RETURN); + for (idx = (int32)func->func_type->result_count - 1; + idx >= 0; idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); + POP_OFFSET_TYPE(ret_type); + } + } +#endif + if (loader_ctx->csp_num > 0) { + loader_ctx->frame_csp->end_addr = p - 1; + } + else { + /* end of function block, function will return */ + if (p < p_end) { + set_error_buf(error_buf, error_buf_size, + "section size mismatch"); + goto fail; + } + } + + break; + } + + case WASM_OP_BR: + { + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + } + + case WASM_OP_BR_IF: + { + POP_I32(); + + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + break; + } + + case WASM_OP_BR_TABLE: + { + uint32 depth = 0, default_arity, arity = 0; + BranchBlock *target_block; + BlockType *target_block_type; +#if WASM_ENABLE_FAST_INTERP == 0 + BrTableCache *br_table_cache = NULL; + uint8 *p_depth_begin, *p_depth, *p_opcode = p - 1; + uint32 j; +#endif + + read_leb_uint32(p, p_end, count); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, count); +#endif + POP_I32(); + + /* Get the default depth and check it */ + p_org = p; + for (i = 0; i <= count; i++) { + read_leb_uint32(p, p_end, depth); + } + if (loader_ctx->csp_num < depth + 1) { + set_error_buf(error_buf, error_buf_size, + "unknown label, " + "unexpected end of section or function"); + goto fail; + } + p = p_org; + + /* Get the default block's arity */ + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + default_arity = block_type_get_arity(target_block_type, + target_block->label_type); + +#if WASM_ENABLE_FAST_INTERP == 0 + p_depth_begin = p_depth = p; +#endif + for (i = 0; i <= count; i++) { + p_org = p; + read_leb_uint32(p, p_end, depth); + if (loader_ctx->csp_num < depth + 1) { + set_error_buf(error_buf, error_buf_size, + "unknown label, " + "unexpected end of section or function"); + goto fail; + } + p = p_org; + + /* Get the target block's arity and check it */ + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + arity = block_type_get_arity(target_block_type, + target_block->label_type); + if (arity != default_arity) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: br_table targets must " + "all use same result type"); + goto fail; + } + + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) { + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP == 0 + if (br_table_cache) { + br_table_cache->br_depths[i] = depth; + } + else { + if (depth > 255) { + /* The depth cannot be stored in one byte, + create br_table cache to store each depth */ +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_opcode, *p_opcode, + error_buf, error_buf_size)) { + goto fail; + } +#endif + if (!(br_table_cache = loader_malloc( + offsetof(BrTableCache, br_depths) + + sizeof(uint32) + * (uint64)(count + 1), + error_buf, error_buf_size))) { + goto fail; + } + *p_opcode = EXT_OP_BR_TABLE_CACHE; + br_table_cache->br_table_op_addr = p_opcode; + br_table_cache->br_count = count; + /* Copy previous depths which are one byte */ + for (j = 0; j < i; j++) { + br_table_cache->br_depths[j] = p_depth_begin[j]; + } + br_table_cache->br_depths[i] = depth; + bh_list_insert(module->br_table_cache_list, + br_table_cache); + } + else { + /* The depth can be stored in one byte, use the + byte of the leb to store it */ + *p_depth++ = (uint8)depth; + } + } +#endif + } + +#if WASM_ENABLE_FAST_INTERP == 0 + /* Set the tailing bytes to nop */ + if (br_table_cache) + p_depth = p_depth_begin; + while (p_depth < p) + *p_depth++ = WASM_OP_NOP; +#endif + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + } + + case WASM_OP_RETURN: + { + int32 idx; + uint8 ret_type; + for (idx = (int32)func->func_type->result_count - 1; idx >= 0; + idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); +#if WASM_ENABLE_FAST_INTERP != 0 + /* emit the offset after return opcode */ + POP_OFFSET_TYPE(ret_type); +#endif + POP_TYPE(ret_type); + } + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + break; + } + + case WASM_OP_CALL: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: +#endif +#if WASM_ENABLE_GC != 0 + case WASM_OP_CALL_REF: + case WASM_OP_RETURN_CALL_REF: +#endif + { + WASMFuncType *func_type; + uint8 type; + int32 idx; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; + uint32 type_idx1; + int32 j; +#endif + +#if WASM_ENABLE_GC != 0 + if (opcode == WASM_OP_CALL_REF + || opcode == WASM_OP_RETURN_CALL_REF) { + read_leb_uint32(p, p_end, type_idx1); + if (!check_type_index(module, module->type_count, type_idx1, + error_buf, error_buf_size)) { + goto fail; + } + if (module->types[type_idx1]->type_flag != WASM_TYPE_FUNC) { + set_error_buf(error_buf, error_buf_size, + "unkown function type"); + goto fail; + } + if (!wasm_loader_pop_nullable_typeidx(loader_ctx, &type, + &type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (type == VALUE_TYPE_ANY) { + type_idx = type_idx1; + } + if (!check_type_index(module, module->type_count, type_idx, + error_buf, error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag != WASM_TYPE_FUNC) { + set_error_buf(error_buf, error_buf_size, + "unkown function type"); + goto fail; + } + if (!wasm_func_type_is_super_of( + (WASMFuncType *)module->types[type_idx1], + (WASMFuncType *)module->types[type_idx])) { + set_error_buf(error_buf, error_buf_size, + "function type mismatch"); + goto fail; + } + func_type = (WASMFuncType *)module->types[type_idx]; + } + else +#endif + { + read_leb_uint32(p, p_end, func_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + /* we need to emit func_idx before arguments */ + emit_uint32(loader_ctx, func_idx); +#endif + + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + + if (func_idx < module->import_function_count) + func_type = module->import_functions[func_idx] + .u.function.func_type; + else + func_type = + module + ->functions[func_idx + - module->import_function_count] + ->func_type; + } + + if (func_type->param_count > 0) { +#if WASM_ENABLE_GC != 0 + j = (int32)(func_type->result_ref_type_maps + - func_type->ref_type_maps - 1); +#endif + for (idx = (int32)(func_type->param_count - 1); idx >= 0; + idx--) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type( + func_type->types[idx])) { + ref_type = func_type->ref_type_maps[j].ref_type; + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + j--; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(func_type->types[idx]); +#endif + POP_TYPE(func_type->types[idx]); + } + } + +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (opcode == WASM_OP_CALL || opcode == WASM_OP_CALL_REF) { +#endif +#if WASM_ENABLE_GC != 0 + j = (int32)(func_type->result_ref_type_maps + - func_type->ref_type_maps); +#endif + for (i = 0; i < func_type->result_count; i++) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type( + func_type->types[func_type->param_count + i])) { + ref_type = func_type->ref_type_maps[j].ref_type; + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + j++; + } +#endif + PUSH_TYPE(func_type->types[func_type->param_count + i]); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Here we emit each return value's dynamic_offset. But + * in fact these offsets are continuous, so interpreter + * only need to get the first return value's offset. + */ + PUSH_OFFSET_TYPE( + func_type->types[func_type->param_count + i]); +#endif + } +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + } + else { +#if WASM_ENABLE_GC == 0 + if (func_type->result_count + != func->func_type->result_count) { + set_error_buf_v(error_buf, error_buf_size, "%s%u%s", + "type mismatch: expect ", + func->func_type->result_count, + " return values but got other"); + goto fail; + } + for (i = 0; i < func_type->result_count; i++) { + type = func->func_type + ->types[func->func_type->param_count + i]; + if (func_type->types[func_type->param_count + i] + != type) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", + type2str(type), " but got other"); + goto fail; + } + } +#else + if (!wasm_func_type_result_is_subtype_of( + func_type, func->func_type, module->types, + module->type_count)) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: invalid func result types"); + goto fail; + } +#endif + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + } +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_func_call = true; +#endif + (void)type; + break; + } + + /* + * if disable reference type: call_indirect typeidx, 0x00 + * if enable reference type: call_indirect typeidx, tableidx + */ + case WASM_OP_CALL_INDIRECT: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL_INDIRECT: +#endif + { + int32 idx; + WASMFuncType *func_type; + + read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + read_leb_uint32(p, p_end, table_idx); +#else + CHECK_BUF(p, p_end, 1); + table_idx = read_uint8(p); +#endif + if (!check_table_index(module, table_idx, error_buf, + error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* we need to emit before arguments */ +#if WASM_ENABLE_TAIL_CALL != 0 + emit_byte(loader_ctx, opcode); +#endif + emit_uint32(loader_ctx, type_idx); + emit_uint32(loader_ctx, table_idx); +#endif + + /* skip elem idx */ + POP_I32(); + + if (type_idx >= module->type_count) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + goto fail; + } + + func_type = (WASMFuncType *)module->types[type_idx]; + + if (func_type->param_count > 0) { + for (idx = (int32)(func_type->param_count - 1); idx >= 0; + idx--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(func_type->types[idx]); +#endif + POP_TYPE(func_type->types[idx]); + } + } + +#if WASM_ENABLE_TAIL_CALL != 0 + if (opcode == WASM_OP_CALL_INDIRECT) { +#endif + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE( + func_type->types[func_type->param_count + i]); +#endif + } +#if WASM_ENABLE_TAIL_CALL != 0 + } + else { + uint8 type; + if (func_type->result_count + != func->func_type->result_count) { + set_error_buf_v(error_buf, error_buf_size, "%s%u%s", + "type mismatch: expect ", + func->func_type->result_count, + " return values but got other"); + goto fail; + } + for (i = 0; i < func_type->result_count; i++) { + type = func->func_type + ->types[func->func_type->param_count + i]; + if (func_type->types[func_type->param_count + i] + != type) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", + type2str(type), " but got other"); + goto fail; + } + } + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + } +#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_func_call = true; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_call_indirect = true; +#endif + break; + } + + case WASM_OP_DROP: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + + if (available_stack_cell <= 0 + && !cur_block->is_stack_polymorphic) { + set_error_buf(error_buf, error_buf_size, + "type mismatch, opcode drop was found " + "but stack was empty"); + goto fail; + } + + if (available_stack_cell > 0) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type( + *(loader_ctx->frame_ref - 1))) { + bh_assert((int32)(loader_ctx->reftype_map_num + - cur_block->reftype_map_num) + > 0); + loader_ctx->frame_reftype_map--; + loader_ctx->reftype_map_num--; + } +#endif + if (is_32bit_type(*(loader_ctx->frame_ref - 1)) + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + loader_ctx->frame_ref--; + loader_ctx->stack_cell_num--; +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + loader_ctx->frame_offset--; + if ((*(loader_ctx->frame_offset) + > loader_ctx->start_dynamic_offset) + && (*(loader_ctx->frame_offset) + < loader_ctx->max_dynamic_offset)) + loader_ctx->dynamic_offset--; +#endif + } + else if (is_64bit_type(*(loader_ctx->frame_ref - 1))) { + loader_ctx->frame_ref -= 2; + loader_ctx->stack_cell_num -= 2; +#if WASM_ENABLE_FAST_INTERP == 0 + *(p - 1) = WASM_OP_DROP_64; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + loader_ctx->frame_offset -= 2; + if ((*(loader_ctx->frame_offset) + > loader_ctx->start_dynamic_offset) + && (*(loader_ctx->frame_offset) + < loader_ctx->max_dynamic_offset)) + loader_ctx->dynamic_offset -= 2; +#endif + } +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + else if (*(loader_ctx->frame_ref - 1) == VALUE_TYPE_V128) { + loader_ctx->frame_ref -= 4; + loader_ctx->stack_cell_num -= 4; + } +#endif +#endif + else { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); +#endif + } + break; + } + + case WASM_OP_SELECT: + { + uint8 ref_type; + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell; +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif + + POP_I32(); + + available_stack_cell = (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + + if (available_stack_cell <= 0 + && !cur_block->is_stack_polymorphic) { + set_error_buf(error_buf, error_buf_size, + "type mismatch or invalid result arity, " + "opcode select was found " + "but stack was empty"); + goto fail; + } + + if (available_stack_cell > 0) { + switch (*(loader_ctx->frame_ref - 1)) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: +#if WASM_ENABLE_FAST_INTERP == 0 + *(p - 1) = WASM_OP_SELECT_64; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled) { + uint8 opcode_tmp = WASM_OP_SELECT_64; +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(void **)(p_code_compiled_tmp + - sizeof(void *)) = + handle_table[opcode_tmp]; +#else +#if UINTPTR_MAX == UINT64_MAX + /* emit int32 relative offset in 64-bit target + */ + int32 offset = + (int32)((uint8 *)handle_table[opcode_tmp] + - (uint8 *)handle_table[0]); + *(int32 *)(p_code_compiled_tmp + - sizeof(int32)) = offset; +#else + /* emit uint32 label address in 32-bit target */ + *(uint32 *)(p_code_compiled_tmp + - sizeof(uint32)) = + (uint32)(uintptr_t)handle_table[opcode_tmp]; +#endif +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(p_code_compiled_tmp - 1) = opcode_tmp; +#else + *(p_code_compiled_tmp - 2) = opcode_tmp; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + } +#endif /* end of WASM_ENABLE_FAST_INTERP */ + break; +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + case VALUE_TYPE_V128: + break; +#endif /* (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) */ +#endif /* WASM_ENABLE_SIMD != 0 */ + default: + { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + } + + ref_type = *(loader_ctx->frame_ref - 1); +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(ref_type); + POP_TYPE(ref_type); + POP_OFFSET_TYPE(ref_type); + POP_TYPE(ref_type); + PUSH_OFFSET_TYPE(ref_type); + PUSH_TYPE(ref_type); +#else + POP2_AND_PUSH(ref_type, ref_type); +#endif + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(VALUE_TYPE_ANY); +#endif + PUSH_TYPE(VALUE_TYPE_ANY); + } + break; + } + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_SELECT_T: + { + uint8 vec_len, type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type = NULL; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif + + read_leb_uint32(p, p_end, vec_len); + if (vec_len != 1) { + /* typed select must have exactly one result */ + set_error_buf(error_buf, error_buf_size, + "invalid result arity"); + goto fail; + } + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + type = read_uint8(p); + if (!is_value_type(type)) { + set_error_buf(error_buf, error_buf_size, + "unknown value type"); + goto fail; + } +#else + p_org = p + 1; + if (!resolve_value_type((const uint8 **)&p, p_end, module, + module->type_count, &need_ref_type_map, + &wasm_ref_type, false, error_buf, + error_buf_size)) { + goto fail; + } + type = wasm_ref_type.ref_type; + if (need_ref_type_map) { + if (!(ref_type = reftype_set_insert( + module->ref_type_set, &wasm_ref_type, error_buf, + error_buf_size))) { + goto fail; + } + } +#if WASM_ENABLE_FAST_INTERP == 0 + while (p_org < p) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + /* Ignore extra bytes for interpreter */ + *p_org++ = WASM_OP_NOP; + } +#endif +#endif /* end of WASM_ENABLE_GC == 0 */ + + POP_I32(); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled) { + uint8 opcode_tmp = WASM_OP_SELECT; + + if (type == VALUE_TYPE_V128) { +#if (WASM_ENABLE_SIMD == 0) \ + || ((WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0)) + set_error_buf(error_buf, error_buf_size, + "SIMD v128 type isn't supported"); + goto fail; +#endif + } + else { + if (type == VALUE_TYPE_F64 || type == VALUE_TYPE_I64) + opcode_tmp = WASM_OP_SELECT_64; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(type)) + opcode_tmp = WASM_OP_SELECT_T; +#endif +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(void **)(p_code_compiled_tmp - sizeof(void *)) = + handle_table[opcode_tmp]; +#else +#if UINTPTR_MAX == UINT64_MAX + /* emit int32 relative offset in 64-bit target */ + int32 offset = (int32)((uint8 *)handle_table[opcode_tmp] + - (uint8 *)handle_table[0]); + *(int32 *)(p_code_compiled_tmp - sizeof(int32)) = + offset; +#else + /* emit uint32 label address in 32-bit target */ + *(uint32 *)(p_code_compiled_tmp - sizeof(uint32)) = + (uint32)(uintptr_t)handle_table[opcode_tmp]; +#endif +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(p_code_compiled_tmp - 1) = opcode_tmp; +#else + *(p_code_compiled_tmp - 2) = opcode_tmp; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + } + } +#endif /* WASM_ENABLE_FAST_INTERP != 0 */ + + POP_REF(type); + +#if WASM_ENABLE_GC != 0 + if (need_ref_type_map) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + POP_REF(type); + +#if WASM_ENABLE_GC != 0 + if (need_ref_type_map) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + PUSH_REF(type); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + (void)vec_len; + break; + } + + /* table.get x. tables[x]. [i32] -> [t] */ + /* table.set x. tables[x]. [i32 t] -> [] */ + case WASM_OP_TABLE_GET: + case WASM_OP_TABLE_SET: + { + uint8 decl_ref_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + + read_leb_uint32(p, p_end, table_idx); + if (!get_table_elem_type(module, table_idx, &decl_ref_type, +#if WASM_ENABLE_GC != 0 + (void **)&ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(decl_ref_type)) { + bh_assert(ref_type); + bh_memcpy_s(&wasm_ref_type, (uint32)sizeof(WASMRefType), + ref_type, wasm_reftype_struct_size(ref_type)); + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + + if (opcode == WASM_OP_TABLE_GET) { + POP_I32(); +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(decl_ref_type); +#endif + PUSH_TYPE(decl_ref_type); + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(decl_ref_type); +#endif + POP_TYPE(decl_ref_type); + POP_I32(); + } + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_REF_NULL: + { + uint8 ref_type; + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + ref_type = read_uint8(p); + + if (ref_type != VALUE_TYPE_FUNCREF + && ref_type != VALUE_TYPE_EXTERNREF) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + goto fail; + } +#else + read_leb_int32(p, p_end, heap_type); + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, heap_type, + error_buf, error_buf_size)) { + goto fail; + } + wasm_set_refheaptype_typeidx(&wasm_ref_type.ref_ht_typeidx, + true, heap_type); + ref_type = wasm_ref_type.ref_type; + } + else { + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf(error_buf, error_buf_size, + "unknown type"); + goto fail; + } + ref_type = (uint8)((int32)0x80 + heap_type); + } +#endif /* end of WASM_ENABLE_GC == 0 */ + +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(ref_type); +#endif + PUSH_TYPE(ref_type); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_REF_IS_NULL: + { +#if WASM_ENABLE_GC == 0 +#if WASM_ENABLE_FAST_INTERP != 0 + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 block_stack_cell_num = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + if (block_stack_cell_num <= 0) { + if (!cur_block->is_stack_polymorphic) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: expect data but stack was empty"); + goto fail; + } + } + else { + if (*(loader_ctx->frame_ref - 1) == VALUE_TYPE_FUNCREF + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_EXTERNREF + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + if (!wasm_loader_pop_frame_ref_offset( + loader_ctx, *(loader_ctx->frame_ref - 1), + error_buf, error_buf_size)) { + goto fail; + } + } + else { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + } +#else + if (!wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_FUNCREF, + error_buf, error_buf_size) + && !wasm_loader_pop_frame_ref(loader_ctx, + VALUE_TYPE_EXTERNREF, + error_buf, error_buf_size)) { + goto fail; + } +#endif +#else /* else of WASM_ENABLE_GC == 0 */ + uint8 type; + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, &wasm_ref_type, + error_buf, error_buf_size)) { + goto fail; + } +#endif + PUSH_I32(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_REF_FUNC: + { + read_leb_uint32(p, p_end, func_idx); + + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + + /* Refer to a forward-declared function: + the function must be an import, exported, or present in + a table elem segment or global initializer to be used as + the operand to ref.func */ + if (func_idx >= module->import_function_count) { + WASMTableSeg *table_seg = module->table_segments; + bool func_declared = false; + uint32 j; + + for (i = 0; i < module->global_count; i++) { + if (module->globals[i].type == VALUE_TYPE_FUNCREF + && module->globals[i].init_expr.init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST + && module->globals[i].init_expr.u.u32 == func_idx) { + func_declared = true; + break; + } + } + + if (!func_declared) { + /* Check whether the function is declared in table segs, + note that it doesn't matter whether the table seg's + mode is passive, active or declarative. */ + for (i = 0; i < module->table_seg_count; + i++, table_seg++) { + if (table_seg->elem_type == VALUE_TYPE_FUNCREF +#if WASM_ENABLE_GC != 0 + /* elem type is (ref null? func) or + (ref null? $t) */ + || ((table_seg->elem_type + == REF_TYPE_HT_NON_NULLABLE + || table_seg->elem_type + == REF_TYPE_HT_NULLABLE) + && (table_seg->elem_ref_type->ref_ht_common + .heap_type + == HEAP_TYPE_FUNC + || table_seg->elem_ref_type + ->ref_ht_common.heap_type + > 0)) +#endif + ) { + for (j = 0; j < table_seg->value_count; j++) { + if (table_seg->init_values[j].u.ref_index + == func_idx) { + func_declared = true; + break; + } + } + } + } + } + + if (!func_declared) { + /* Check whether the function is exported */ + for (i = 0; i < module->export_count; i++) { + if (module->exports[i].kind == EXPORT_KIND_FUNC + && module->exports[i].index == func_idx) { + func_declared = true; + break; + } + } + } + + if (!func_declared) { + set_error_buf(error_buf, error_buf_size, + "undeclared function reference"); + goto fail; + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, func_idx); +#endif +#if WASM_ENABLE_GC == 0 + PUSH_FUNCREF(); +#else + if (func_idx < module->import_function_count) + type_idx = + module->import_functions[func_idx].u.function.type_idx; + else + type_idx = module + ->functions[func_idx + - module->import_function_count] + ->type_idx; + wasm_set_refheaptype_typeidx(&wasm_ref_type.ref_ht_typeidx, + false, type_idx); + PUSH_REF(wasm_ref_type.ref_type); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + case WASM_OP_REF_AS_NON_NULL: + case WASM_OP_BR_ON_NULL: + { + uint8 type; + WASMRefType ref_type; + + /* POP (ref null ht) and get the converted (ref ht) */ + if (!wasm_loader_pop_nullable_ht(loader_ctx, &type, &ref_type, + error_buf, error_buf_size)) { + goto fail; + } + + if (opcode == WASM_OP_BR_ON_NULL) { + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + disable_emit = true; +#endif + } + + /* PUSH the converted (ref ht) */ + if (type != VALUE_TYPE_ANY) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), &ref_type, + sizeof(WASMRefType)); + } + PUSH_REF(type); + break; + } + + case WASM_OP_BR_ON_NON_NULL: + { + uint8 type; + WASMRefType ref_type; + uint32 available_stack_cell = + loader_ctx->stack_cell_num + - (loader_ctx->frame_csp - 1)->stack_cell_num; + + /* POP (ref null ht) and get the converted (ref ht) */ + if (!wasm_loader_pop_nullable_ht(loader_ctx, &type, &ref_type, + error_buf, error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + disable_emit = true; +#endif + + /* Temporarily PUSH back (ref ht), check brach block and + then POP it */ + if (available_stack_cell + > 0) { /* stack isn't in polymorphic state */ + if (type != VALUE_TYPE_ANY) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + &ref_type, sizeof(WASMRefType)); + } + PUSH_REF(type); + } + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) { + goto fail; + } + if (available_stack_cell + > 0) { /* stack isn't in polymorphic state */ + POP_REF(type); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Erase the opnd offset emitted by POP_REF() */ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint16)); +#endif + } + break; + } + + case WASM_OP_REF_EQ: + POP_REF(REF_TYPE_EQREF); + POP_REF(REF_TYPE_EQREF); + PUSH_I32(); + break; +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_GET_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + PUSH_TYPE(local_type); + +#if WASM_ENABLE_GC != 0 + /* Cannot get a non-nullable and unset local */ + if (local_idx >= param_count + && wasm_is_reftype_htref_non_nullable(local_type) + && !wasm_loader_get_local_status(loader_ctx, + local_idx - param_count)) { + set_error_buf(error_buf, error_buf_size, + "uninitialized local"); + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Get Local is optimized out */ + skip_label(); + disable_emit = true; + operand_offset = local_offset; + PUSH_OFFSET_TYPE(local_type); +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0) + if (local_offset < 0x80 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + *p_org++ = EXT_OP_GET_LOCAL_FAST; + if (is_32bit_type(local_type)) { + *p_org++ = (uint8)local_offset; + } + else { + *p_org++ = (uint8)(local_offset | 0x80); + } + while (p_org < p) { + *p_org++ = WASM_OP_NOP; + } + } +#endif +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + break; + } + + case WASM_OP_SET_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!(preserve_referenced_local( + loader_ctx, opcode, local_offset, local_type, + &preserve_local, error_buf, error_buf_size))) + goto fail; + + if (local_offset < 256 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + skip_label(); + if ((!preserve_local) && (LAST_OP_OUTPUT_I32())) { + if (loader_ctx->p_code_compiled) + STORE_U16(loader_ctx->p_code_compiled - 2, + local_offset); + loader_ctx->frame_offset--; + loader_ctx->dynamic_offset--; + } + else if ((!preserve_local) && (LAST_OP_OUTPUT_I64())) { + if (loader_ctx->p_code_compiled) + STORE_U16(loader_ctx->p_code_compiled - 2, + local_offset); + loader_ctx->frame_offset -= 2; + loader_ctx->dynamic_offset -= 2; + } + else { + if (is_32bit_type(local_type)) { + emit_label(EXT_OP_SET_LOCAL_FAST); + emit_byte(loader_ctx, (uint8)local_offset); + } + else { + emit_label(EXT_OP_SET_LOCAL_FAST_I64); + emit_byte(loader_ctx, (uint8)local_offset); + } + POP_OFFSET_TYPE(local_type); + } + } + else { /* local index larger than 255, reserve leb */ + emit_uint32(loader_ctx, local_idx); + POP_OFFSET_TYPE(local_type); + } +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0) + if (local_offset < 0x80 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + *p_org++ = EXT_OP_SET_LOCAL_FAST; + if (is_32bit_type(local_type)) { + *p_org++ = (uint8)local_offset; + } + else { + *p_org++ = (uint8)(local_offset | 0x80); + } + while (p_org < p) { + *p_org++ = WASM_OP_NOP; + } + } +#endif +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + +#if WASM_ENABLE_GC != 0 + if (local_idx >= param_count) { + wasm_loader_mask_local(loader_ctx, local_idx - param_count); + } +#endif + + POP_TYPE(local_type); + break; + } + + case WASM_OP_TEE_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); +#if WASM_ENABLE_FAST_INTERP != 0 + /* If the stack is in polymorphic state, do fake pop and push on + offset stack to keep the depth of offset stack to be the + same with ref stack */ + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + if (cur_block->is_stack_polymorphic) { + POP_OFFSET_TYPE(local_type); + PUSH_OFFSET_TYPE(local_type); + } +#endif + POP_TYPE(local_type); + PUSH_TYPE(local_type); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!(preserve_referenced_local( + loader_ctx, opcode, local_offset, local_type, + &preserve_local, error_buf, error_buf_size))) + goto fail; + + if (local_offset < 256 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + skip_label(); + if (is_32bit_type(local_type)) { + emit_label(EXT_OP_TEE_LOCAL_FAST); + emit_byte(loader_ctx, (uint8)local_offset); + } + else { + emit_label(EXT_OP_TEE_LOCAL_FAST_I64); + emit_byte(loader_ctx, (uint8)local_offset); + } + } + else { /* local index larger than 255, reserve leb */ + emit_uint32(loader_ctx, local_idx); + } + emit_operand(loader_ctx, + *(loader_ctx->frame_offset + - wasm_value_type_cell_num(local_type))); +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0) + if (local_offset < 0x80 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + *p_org++ = EXT_OP_TEE_LOCAL_FAST; + if (is_32bit_type(local_type)) { + *p_org++ = (uint8)local_offset; + } + else { + *p_org++ = (uint8)(local_offset | 0x80); + } + while (p_org < p) { + *p_org++ = WASM_OP_NOP; + } + } +#endif +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + +#if WASM_ENABLE_GC != 0 + if (local_idx >= param_count) { + wasm_loader_mask_local(loader_ctx, local_idx - param_count); + } +#endif + break; + } + + case WASM_OP_GET_GLOBAL: + { +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + + p_org = p - 1; + read_leb_uint32(p, p_end, global_idx); + if (global_idx >= global_count) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + global_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.type + : module + ->globals[global_idx + - module->import_global_count] + .type; +#if WASM_ENABLE_GC != 0 + ref_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.ref_type + : module + ->globals[global_idx + - module->import_global_count] + .ref_type; + if (wasm_is_type_multi_byte_type(global_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + + PUSH_TYPE(global_type); + +#if WASM_ENABLE_FAST_INTERP == 0 + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = WASM_OP_GET_GLOBAL_64; + } +#else /* else of WASM_ENABLE_FAST_INTERP */ + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { + skip_label(); + emit_label(WASM_OP_GET_GLOBAL_64); + } + emit_uint32(loader_ctx, global_idx); + PUSH_OFFSET_TYPE(global_type); +#endif /* end of WASM_ENABLE_FAST_INTERP */ + break; + } + + case WASM_OP_SET_GLOBAL: + { + bool is_mutable = false; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + + p_org = p - 1; + read_leb_uint32(p, p_end, global_idx); + if (global_idx >= global_count) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + is_mutable = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.is_mutable + : module + ->globals[global_idx + - module->import_global_count] + .is_mutable; + if (!is_mutable) { +#if WASM_ENABLE_GC == 0 + set_error_buf(error_buf, error_buf_size, + "global is immutable"); +#else + set_error_buf(error_buf, error_buf_size, + "immutable global"); +#endif + goto fail; + } + + global_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.type + : module + ->globals[global_idx + - module->import_global_count] + .type; +#if WASM_ENABLE_GC != 0 + ref_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.ref_type + : module + ->globals[global_idx + - module->import_global_count] + .ref_type; + if (wasm_is_type_multi_byte_type(global_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + +#if WASM_ENABLE_FAST_INTERP == 0 + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = WASM_OP_SET_GLOBAL_64; + } + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = WASM_OP_SET_GLOBAL_AUX_STACK; +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_set_global_aux_stack = true; +#endif + } +#else /* else of WASM_ENABLE_FAST_INTERP */ + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { + skip_label(); + emit_label(WASM_OP_SET_GLOBAL_64); + } + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { + skip_label(); + emit_label(WASM_OP_SET_GLOBAL_AUX_STACK); + } + emit_uint32(loader_ctx, global_idx); + POP_OFFSET_TYPE(global_type); +#endif /* end of WASM_ENABLE_FAST_INTERP */ + + POP_TYPE(global_type); + + break; + } + + /* load */ + case WASM_OP_I32_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + case WASM_OP_I64_LOAD: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + case WASM_OP_F32_LOAD: + case WASM_OP_F64_LOAD: + /* store */ + case WASM_OP_I32_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + case WASM_OP_I64_STORE: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + case WASM_OP_F32_STORE: + case WASM_OP_F64_STORE: + { +#if WASM_ENABLE_FAST_INTERP != 0 + /* change F32/F64 into I32/I64 */ + if (opcode == WASM_OP_F32_LOAD) { + skip_label(); + emit_label(WASM_OP_I32_LOAD); + } + else if (opcode == WASM_OP_F64_LOAD) { + skip_label(); + emit_label(WASM_OP_I64_LOAD); + } + else if (opcode == WASM_OP_F32_STORE) { + skip_label(); + emit_label(WASM_OP_I32_STORE); + } + else if (opcode == WASM_OP_F64_STORE) { + skip_label(); + emit_label(WASM_OP_I64_STORE); + } +#endif + CHECK_MEMORY(); + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ + if (!check_memory_access_align(opcode, align, error_buf, + error_buf_size)) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + switch (opcode) { + /* load */ + case WASM_OP_I32_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); + break; + case WASM_OP_I64_LOAD: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); + break; + case WASM_OP_F32_LOAD: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32); + break; + case WASM_OP_F64_LOAD: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64); + break; + /* store */ + case WASM_OP_I32_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + POP_I32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_I64_STORE: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + POP_I64(); + POP_MEM_OFFSET(); + break; + case WASM_OP_F32_STORE: + POP_F32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_F64_STORE: + POP_F64(); + POP_MEM_OFFSET(); + break; + default: + break; + } + break; + } + + case WASM_OP_MEMORY_SIZE: + CHECK_MEMORY(); + /* reserved byte 0x00 */ + if (*p++ != 0x00) { + set_error_buf(error_buf, error_buf_size, + "zero byte expected"); + goto fail; + } + PUSH_PAGE_COUNT(); + + module->possible_memory_grow = true; +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + + case WASM_OP_MEMORY_GROW: + CHECK_MEMORY(); + /* reserved byte 0x00 */ + if (*p++ != 0x00) { + set_error_buf(error_buf, error_buf_size, + "zero byte expected"); + goto fail; + } + POP_AND_PUSH(mem_offset_type, mem_offset_type); + + module->possible_memory_grow = true; +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_memory_grow = true; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + + case WASM_OP_I32_CONST: + read_leb_int32(p, p_end, i32_const); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + GET_CONST_OFFSET(VALUE_TYPE_I32, i32_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_I32_CONST); + emit_uint32(loader_ctx, i32_const); + } +#else + (void)i32_const; +#endif + PUSH_I32(); + break; + + case WASM_OP_I64_CONST: + read_leb_int64(p, p_end, i64_const); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + GET_CONST_OFFSET(VALUE_TYPE_I64, i64_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_I64_CONST); + emit_uint64(loader_ctx, i64_const); + } +#endif + PUSH_I64(); + break; + + case WASM_OP_F32_CONST: + CHECK_BUF(p, p_end, sizeof(float32)); + p += sizeof(float32); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + bh_memcpy_s((uint8 *)&f32_const, sizeof(float32), p_org, + sizeof(float32)); + GET_CONST_F32_OFFSET(VALUE_TYPE_F32, f32_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_F32_CONST); + emit_float32(loader_ctx, f32_const); + } +#endif + PUSH_F32(); + break; + + case WASM_OP_F64_CONST: + CHECK_BUF(p, p_end, sizeof(float64)); + p += sizeof(float64); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + /* Some MCU may require 8-byte align */ + bh_memcpy_s((uint8 *)&f64_const, sizeof(float64), p_org, + sizeof(float64)); + GET_CONST_F64_OFFSET(VALUE_TYPE_F64, f64_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_F64_CONST); + emit_float64(loader_ctx, f64_const); + } +#endif + PUSH_F64(); + break; + + case WASM_OP_I32_EQZ: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EQZ: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + POP2_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + POP2_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + POP2_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_CLZ: + case WASM_OP_I32_CTZ: + case WASM_OP_I32_POPCNT: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_CLZ: + case WASM_OP_I64_CTZ: + case WASM_OP_I64_POPCNT: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + POP2_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + case WASM_OP_F32_COPYSIGN: + POP2_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + case WASM_OP_F64_COPYSIGN: + POP2_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_WRAP_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_DEMOTE_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_PROMOTE_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_REINTERPRET_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_REINTERPRET_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_REINTERPRET_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_REINTERPRET_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_EXTEND8_S: + case WASM_OP_I32_EXTEND16_S: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EXTEND8_S: + case WASM_OP_I64_EXTEND16_S: + case WASM_OP_I64_EXTEND32_S: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + +#if WASM_ENABLE_GC != 0 + case WASM_OP_GC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, ((uint8)opcode1)); +#endif + + switch (opcode1) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + { + read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unkown struct type"); + goto fail; + } + + if (opcode1 == WASM_OP_STRUCT_NEW) { + int32 j, k; + uint8 value_type; + uint32 ref_type_struct_size; + WASMStructType *struct_type = + (WASMStructType *)module->types[type_idx]; + + k = struct_type->ref_type_map_count - 1; + for (j = struct_type->field_count - 1; j >= 0; + j--) { + value_type = struct_type->fields[j].field_type; + if (wasm_is_type_reftype(value_type)) { + if (wasm_is_type_multi_byte_type( + value_type)) { + ref_type_struct_size = + wasm_reftype_struct_size( + struct_type->ref_type_maps[k] + .ref_type); + bh_memcpy_s( + &wasm_ref_type, + (uint32)sizeof(WASMRefType), + struct_type->ref_type_maps[k] + .ref_type, + ref_type_struct_size); + k--; + } + POP_REF(value_type); + } + else { + switch (value_type) { + case VALUE_TYPE_I32: + case PACKED_TYPE_I8: + case PACKED_TYPE_I16: + POP_I32(); + break; + case VALUE_TYPE_I64: + POP_I64(); + break; + case VALUE_TYPE_F32: + POP_F32(); + break; + case VALUE_TYPE_F64: + POP_F64(); + break; + default: + set_error_buf(error_buf, + error_buf_size, + "unknown type"); + goto fail; + } + } + } + } + + /* PUSH struct obj, (ref $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, false, type_idx); + PUSH_REF(wasm_ref_type.ref_type); + break; + } + + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + case WASM_OP_STRUCT_SET: + { + WASMStructType *struct_type; + WASMRefType *ref_type = NULL; + uint32 field_idx; + uint8 field_type; + + read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unknown struct type"); + goto fail; + } + struct_type = (WASMStructType *)module->types[type_idx]; + + read_leb_uint32(p, p_end, field_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, field_idx); +#endif + if (field_idx >= struct_type->field_count) { + set_error_buf(error_buf, error_buf_size, + "unknown struct field"); + goto fail; + } + + if (opcode1 == WASM_OP_STRUCT_SET + && !(struct_type->fields[field_idx].field_flags + & 1)) { + set_error_buf(error_buf, error_buf_size, + "field is immutable"); + goto fail; + } + + field_type = struct_type->fields[field_idx].field_type; + if (is_packed_type(field_type)) { + if (opcode1 == WASM_OP_STRUCT_GET) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + else { + field_type = VALUE_TYPE_I32; + } + } + if (wasm_is_type_multi_byte_type(field_type)) { + ref_type = wasm_reftype_map_find( + struct_type->ref_type_maps, + struct_type->ref_type_map_count, field_idx); + bh_assert(ref_type); + } + if (opcode1 == WASM_OP_STRUCT_SET) { + /* POP field */ + if (wasm_is_type_multi_byte_type(field_type)) { + bh_memcpy_s(&wasm_ref_type, + (uint32)sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + POP_REF(field_type); + /* POP struct obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + } + else { + /* POP struct obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + /* PUSH field */ + if (wasm_is_type_multi_byte_type(field_type)) { + bh_memcpy_s(&wasm_ref_type, + (uint32)sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + PUSH_REF(field_type); + } + break; + } + + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + case WASM_OP_ARRAY_NEW_DATA: + case WASM_OP_ARRAY_NEW_ELEM: + { + WASMArrayType *array_type; + uint8 elem_type; + uint32 u32 = 0; + + read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (opcode1 == WASM_OP_ARRAY_NEW_FIXED + || opcode1 == WASM_OP_ARRAY_NEW_DATA + || opcode1 == WASM_OP_ARRAY_NEW_ELEM) { + read_leb_uint32(p, p_end, u32); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, u32); +#endif + } + + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + if (opcode1 != WASM_OP_ARRAY_NEW_FIXED) { + /* length */ + POP_I32(); + } + + array_type = (WASMArrayType *)module->types[type_idx]; + elem_type = array_type->elem_type; + + if (opcode1 == WASM_OP_ARRAY_NEW + || opcode1 == WASM_OP_ARRAY_NEW_FIXED) { + if (wasm_is_type_multi_byte_type(elem_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + array_type->elem_ref_type, + wasm_reftype_struct_size( + array_type->elem_ref_type)); + } + if (is_packed_type(elem_type)) { + elem_type = VALUE_TYPE_I32; + } + + if (opcode1 == WASM_OP_ARRAY_NEW_FIXED) { + uint32 N = u32; + for (i = 0; i < N; i++) { + if (wasm_is_type_multi_byte_type( + elem_type)) { + bh_memcpy_s( + &wasm_ref_type, sizeof(WASMRefType), + array_type->elem_ref_type, + wasm_reftype_struct_size( + array_type->elem_ref_type)); + } + POP_REF(elem_type); + } + } + else + POP_REF(elem_type); + } + else if (opcode1 == WASM_OP_ARRAY_NEW_DATA) { + /* offset of data segment */ + POP_I32(); + + if (u32 >= module->data_seg_count) { + set_error_buf(error_buf, error_buf_size, + "unknown data segement"); + goto fail; + } + + if (wasm_is_type_reftype(elem_type)) { + set_error_buf(error_buf, error_buf_size, + "array elem type mismatch"); + goto fail; + } + } + else if (opcode1 == WASM_OP_ARRAY_NEW_ELEM) { + WASMTableSeg *table_seg = + module->table_segments + u32; + + /* offset of element segment */ + POP_I32(); + + if (u32 >= module->table_seg_count) { + set_error_buf(error_buf, error_buf_size, + "unknown element segement"); + goto fail; + } + if (!wasm_reftype_is_subtype_of( + table_seg->elem_type, + table_seg->elem_ref_type, elem_type, + array_type->elem_ref_type, module->types, + module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "array elem type mismatch"); + goto fail; + } + } + + /* PUSH array obj, (ref $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, false, type_idx); + PUSH_REF(wasm_ref_type.ref_type); + break; + } + + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + case WASM_OP_ARRAY_SET: + { + uint8 elem_type; + WASMArrayType *array_type; + WASMRefType *ref_type = NULL; + + read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + array_type = (WASMArrayType *)module->types[type_idx]; + + if (opcode1 == WASM_OP_ARRAY_SET + && !(array_type->elem_flags & 1)) { + set_error_buf(error_buf, error_buf_size, + "array is immutable"); + goto fail; + } + + elem_type = array_type->elem_type; + if (is_packed_type(elem_type)) { + if (opcode1 != WASM_OP_ARRAY_GET_S + && opcode1 != WASM_OP_ARRAY_GET_U + && opcode1 != WASM_OP_ARRAY_SET) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + else { + elem_type = VALUE_TYPE_I32; + } + } + ref_type = array_type->elem_ref_type; + + if (opcode1 == WASM_OP_ARRAY_SET) { + /* POP elem to set */ + if (wasm_is_type_multi_byte_type(elem_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + POP_REF(elem_type); + } + /* elem idx */ + POP_I32(); + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + if (opcode1 != WASM_OP_ARRAY_SET) { + /* PUSH elem */ + if (wasm_is_type_multi_byte_type(elem_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + PUSH_REF(elem_type); + } + break; + } + + case WASM_OP_ARRAY_LEN: + { + POP_REF(REF_TYPE_ARRAYREF); + /* length */ + PUSH_I32(); + break; + } + + case WASM_OP_ARRAY_FILL: + { + WASMArrayType *array_type; + uint8 elem_type; + /* typeidx */ + read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + array_type = (WASMArrayType *)module->types[type_idx]; + if (!(array_type->elem_flags & 1)) { + set_error_buf(error_buf, error_buf_size, + "array is immutable"); + goto fail; + } + + elem_type = array_type->elem_type; + if (is_packed_type(elem_type)) { + elem_type = VALUE_TYPE_I32; + } + + POP_I32(); /* length */ +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(elem_type); +#endif + POP_TYPE(elem_type); + POP_I32(); /* start */ + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + + break; + } + + case WASM_OP_ARRAY_COPY: + { + uint32 src_type_idx; + uint8 src_elem_type, dst_elem_type; + WASMRefType src_ref_type = { 0 }, + *src_elem_ref_type = NULL; + WASMRefType dst_ref_type = { 0 }, + *dst_elem_ref_type = NULL; + WASMArrayType *array_type; + + /* typeidx1 */ + read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + /* typeidx2 */ + read_leb_uint32(p, p_end, src_type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, src_type_idx); +#endif + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + if (!check_array_type(module, src_type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + POP_I32(); + POP_I32(); + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, src_type_idx); + POP_REF(wasm_ref_type.ref_type); + bh_memcpy_s(&src_ref_type, (uint32)sizeof(WASMRefType), + &wasm_ref_type, + wasm_reftype_struct_size(&wasm_ref_type)); + POP_I32(); + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + bh_memcpy_s(&dst_ref_type, (uint32)sizeof(WASMRefType), + &wasm_ref_type, + wasm_reftype_struct_size(&wasm_ref_type)); + + array_type = (WASMArrayType *)module->types[type_idx]; + if (!(array_type->elem_flags & 1)) { + set_error_buf(error_buf, error_buf_size, + "destination array is immutable"); + goto fail; + } + + dst_elem_type = array_type->elem_type; + if (wasm_is_type_multi_byte_type(dst_elem_type)) { + dst_elem_ref_type = array_type->elem_ref_type; + } + + array_type = + (WASMArrayType *)module->types[src_type_idx]; + src_elem_type = array_type->elem_type; + if (wasm_is_type_multi_byte_type(src_elem_type)) { + src_elem_ref_type = array_type->elem_ref_type; + } + + if (!wasm_reftype_is_subtype_of( + src_elem_type, src_elem_ref_type, dst_elem_type, + dst_elem_ref_type, module->types, + module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "array types do not match"); + goto fail; + } + + break; + } + + case WASM_OP_REF_I31: + { + POP_I32(); + wasm_set_refheaptype_common( + &wasm_ref_type.ref_ht_common, false, HEAP_TYPE_I31); + PUSH_REF(wasm_ref_type.ref_type); + break; + } + + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + { + POP_REF(REF_TYPE_I31REF); + PUSH_I32(); + break; + } + + case WASM_OP_REF_TEST: + case WASM_OP_REF_CAST: + case WASM_OP_REF_TEST_NULLABLE: + case WASM_OP_REF_CAST_NULLABLE: + { + uint8 type; + + read_leb_int32(p, p_end, heap_type); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)heap_type); +#endif + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, + heap_type, error_buf, + error_buf_size)) { + goto fail; + } + } + else { + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf(error_buf, error_buf_size, + "unknown type"); + goto fail; + } + } + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, + &wasm_ref_type, error_buf, + error_buf_size)) { + goto fail; + } + if (opcode1 == WASM_OP_REF_TEST + || opcode1 == WASM_OP_REF_TEST_NULLABLE) + PUSH_I32(); + else { + bool nullable = + (opcode1 == WASM_OP_REF_CAST_NULLABLE) ? true + : false; + if (heap_type >= 0 || !nullable) { + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, nullable, + heap_type); + PUSH_REF(wasm_ref_type.ref_type); + } + else { + PUSH_REF((uint8)((int32)0x80 + heap_type)); + } + } + break; + } + + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + { + WASMRefType ref_type_tmp = { 0 }, ref_type1 = { 0 }, + ref_type2 = { 0 }, ref_type_diff = { 0 }; + uint8 type_tmp, castflags; + uint32 depth; + int32 heap_type_dst; + bool src_nullable, dst_nullable; + + CHECK_BUF(p, p_end, 1); + castflags = read_uint8(p); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Emit heap_type firstly */ + emit_byte(loader_ctx, castflags); +#endif + + p_org = p; + read_leb_uint32(p, p_end, depth); + read_leb_int32(p, p_end, heap_type); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Emit heap_type firstly */ + emit_uint32(loader_ctx, (uint32)heap_type); +#endif + read_leb_int32(p, p_end, heap_type_dst); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Emit heap_type firstly */ + emit_uint32(loader_ctx, (uint32)heap_type_dst); +#endif + (void)depth; + + /* + * castflags should be 0~3: + * 0: (non-null, non-null) + * 1: (null, non-null) + * 2: (non-null, null) + * 3: (null, null) + */ + if (castflags > 3) { + set_error_buf(error_buf, error_buf_size, + "invalid castflags"); + break; + } + src_nullable = + (castflags == 1) || (castflags == 3) ? true : false; + dst_nullable = + (castflags == 2) || (castflags == 3) ? true : false; + + /* Pop and backup the stack top's ref type */ + if (!wasm_loader_pop_heap_obj(loader_ctx, &type_tmp, + &ref_type_tmp, error_buf, + error_buf_size)) { + goto fail; + } + + /* The reference type rt1 must be valid */ + if (!init_ref_type(module, &ref_type1, src_nullable, + heap_type, error_buf, + error_buf_size)) { + goto fail; + } + + /* The reference type rt2 must be valid. */ + if (!init_ref_type(module, &ref_type2, dst_nullable, + heap_type_dst, error_buf, + error_buf_size)) { + goto fail; + } + + calculate_reftype_diff(&ref_type_diff, &ref_type1, + &ref_type2); + + /* The reference type rt2 must match rt1. */ + if (!wasm_reftype_is_subtype_of( + ref_type2.ref_type, &ref_type2, + ref_type1.ref_type, &ref_type1, module->types, + module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + + p = p_org; + /* Push ref type casted for branch block check */ + if (opcode1 == WASM_OP_BR_ON_CAST) { + /* The reference type rt2 must match rt′. */ + type_tmp = ref_type2.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type2), + &ref_type2, + wasm_reftype_struct_size(&ref_type2)); + } + } + else { + /* The reference type rt′1 must match rt′. */ + type_tmp = ref_type_diff.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type_diff), + &ref_type_diff, + wasm_reftype_struct_size(&ref_type_diff)); + } + } + PUSH_REF(type_tmp); + if (!(frame_csp_tmp = check_branch_block( + loader_ctx, &p, p_end, opcode, error_buf, + error_buf_size))) { + goto fail; + } + /* Ignore heap_types */ + skip_leb_uint32(p, p_end); + skip_leb_uint32(p, p_end); + + /* Restore the original stack top's ref type */ + POP_REF(type_tmp); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Erase the opnd offset emitted by POP_REF() */ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint16)); +#endif + if (opcode1 == WASM_OP_BR_ON_CAST) { + type_tmp = ref_type_diff.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type_diff), + &ref_type_diff, + wasm_reftype_struct_size(&ref_type_diff)); + } + } + else { + type_tmp = ref_type2.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type2), + &ref_type2, + wasm_reftype_struct_size(&ref_type2)); + } + } + PUSH_REF(type_tmp); + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Erase the opnd offset emitted by PUSH_REF() */ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint16)); +#endif + break; + } + + case WASM_OP_ANY_CONVERT_EXTERN: + { + uint8 type; + + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, + &wasm_ref_type, error_buf, + error_buf_size)) { + goto fail; + } + if (!(type == REF_TYPE_EXTERNREF + || (type == REF_TYPE_HT_NON_NULLABLE + && wasm_ref_type.ref_ht_common.heap_type + == HEAP_TYPE_EXTERN) + || type == VALUE_TYPE_ANY)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + + if (type == REF_TYPE_EXTERNREF) + type = REF_TYPE_ANYREF; + else { + wasm_ref_type.ref_ht_common.heap_type = + HEAP_TYPE_ANY; + } + PUSH_REF(type); + break; + } + + case WASM_OP_EXTERN_CONVERT_ANY: + { + uint8 type; + + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, + &wasm_ref_type, error_buf, + error_buf_size)) { + goto fail; + } + if (type == REF_TYPE_EXTERNREF + || ((type == REF_TYPE_HT_NULLABLE + || type == REF_TYPE_HT_NON_NULLABLE) + && wasm_ref_type.ref_ht_common.heap_type + == HEAP_TYPE_EXTERN)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + + if (type != REF_TYPE_HT_NON_NULLABLE) { + /* push (ref null extern) */ + type = REF_TYPE_EXTERNREF; + } + else { + /* push (ref extern) */ + type = REF_TYPE_HT_NON_NULLABLE; + wasm_set_refheaptype_common( + &wasm_ref_type.ref_ht_common, false, + HEAP_TYPE_EXTERN); + } + PUSH_REF(type); + break; + } + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + { + uint32 memidx; + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_I32(); + PUSH_REF(REF_TYPE_STRINGREF); + (void)memidx; + break; + } + case WASM_OP_STRING_CONST: + { + uint32 contents; + + read_leb_uint32(p, p_end, contents); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)contents); +#endif + PUSH_REF(REF_TYPE_STRINGREF); + (void)contents; + break; + } + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + { + POP_STRINGREF(); + PUSH_I32(); + break; + } + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + { + uint32 memidx; + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_STRINGREF(); + PUSH_I32(); + (void)memidx; + break; + } + case WASM_OP_STRING_CONCAT: + { + POP_STRINGREF(); + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_EQ: + { + POP_STRINGREF(); + POP_STRINGREF(); + PUSH_I32(); + break; + } + case WASM_OP_STRING_IS_USV_SEQUENCE: + { + POP_STRINGREF(); + PUSH_I32(); + break; + } + case WASM_OP_STRING_AS_WTF8: + { + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGVIEWWTF8); + break; + } + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF8); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + { + uint32 memidx; + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF8); + PUSH_I32(); + PUSH_I32(); + (void)memidx; + break; + } + case WASM_OP_STRINGVIEW_WTF8_SLICE: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF8); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_AS_WTF16: + { + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGVIEWWTF16); + break; + } + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + { + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + { + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + { + uint32 memidx; + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_I32(); + (void)memidx; + break; + } + case WASM_OP_STRINGVIEW_WTF16_SLICE: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_AS_ITER: + { + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGVIEWITER); + break; + } + case WASM_OP_STRINGVIEW_ITER_NEXT: + { + POP_REF(REF_TYPE_STRINGVIEWITER); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + case WASM_OP_STRINGVIEW_ITER_REWIND: + { + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWITER); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_ITER_SLICE: + { + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWITER); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_ARRAYREF); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + { + POP_I32(); + POP_REF(REF_TYPE_ARRAYREF); + POP_STRINGREF(); + PUSH_I32(); + break; + } +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + default: + set_error_buf_v(error_buf, error_buf_size, + "%s %02x %02x", "unsupported opcode", + 0xfb, opcode1); + goto fail; + } + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, ((uint8)opcode1)); +#endif + switch (opcode1) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I64); + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + read_leb_uint32(p, p_end, data_seg_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, data_seg_idx); +#endif + if (module->import_memory_count == 0 + && module->memory_count == 0) + goto fail_unknown_memory; + + if (*p++ != 0x00) + goto fail_zero_byte_expected; + + if (data_seg_idx >= module->data_seg_count) { + set_error_buf_v(error_buf, error_buf_size, + "unknown data segment %d", + data_seg_idx); + goto fail; + } + + if (module->data_seg_count1 == 0) + goto fail_data_cnt_sec_require; + + POP_I32(); + POP_I32(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } + case WASM_OP_DATA_DROP: + { + read_leb_uint32(p, p_end, data_seg_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, data_seg_idx); +#endif + if (data_seg_idx >= module->data_seg_count) { + set_error_buf(error_buf, error_buf_size, + "unknown data segment"); + goto fail; + } + + if (module->data_seg_count1 == 0) + goto fail_data_cnt_sec_require; + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } + case WASM_OP_MEMORY_COPY: + { + CHECK_BUF(p, p_end, sizeof(int16)); + /* both src and dst memory index should be 0 */ + if (*(int16 *)p != 0x0000) + goto fail_zero_byte_expected; + p += 2; + + if (module->import_memory_count == 0 + && module->memory_count == 0) + goto fail_unknown_memory; + + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } + case WASM_OP_MEMORY_FILL: + { + if (*p++ != 0x00) { + goto fail_zero_byte_expected; + } + if (module->import_memory_count == 0 + && module->memory_count == 0) { + goto fail_unknown_memory; + } + POP_MEM_OFFSET(); + POP_I32(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } + fail_zero_byte_expected: + set_error_buf(error_buf, error_buf_size, + "zero byte expected"); + goto fail; + + fail_unknown_memory: + set_error_buf(error_buf, error_buf_size, + "unknown memory 0"); + goto fail; + fail_data_cnt_sec_require: + set_error_buf(error_buf, error_buf_size, + "data count section required"); + goto fail; +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_TABLE_INIT: + { + uint8 seg_type = 0, tbl_type = 0; +#if WASM_ENABLE_GC != 0 + WASMRefType *seg_ref_type = NULL, *tbl_ref_type = NULL; +#endif + + read_leb_uint32(p, p_end, table_seg_idx); + read_leb_uint32(p, p_end, table_idx); + + if (!get_table_elem_type(module, table_idx, &tbl_type, +#if WASM_ENABLE_GC != 0 + (void **)&tbl_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + + if (!get_table_seg_elem_type(module, table_seg_idx, + &seg_type, +#if WASM_ENABLE_GC != 0 + (void **)&seg_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_GC == 0 + if (seg_type != tbl_type) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#else + if (!wasm_reftype_is_subtype_of( + seg_type, seg_ref_type, tbl_type, tbl_ref_type, + module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_seg_idx); + emit_uint32(loader_ctx, table_idx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_ELEM_DROP: + { + read_leb_uint32(p, p_end, table_seg_idx); + if (!get_table_seg_elem_type(module, table_seg_idx, + NULL, NULL, error_buf, + error_buf_size)) + goto fail; +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_seg_idx); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_TABLE_COPY: + { + uint8 src_type, dst_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *src_ref_type = NULL, *dst_ref_type = NULL; +#endif + uint32 src_tbl_idx, dst_tbl_idx; + + read_leb_uint32(p, p_end, dst_tbl_idx); + if (!get_table_elem_type(module, dst_tbl_idx, &dst_type, +#if WASM_ENABLE_GC != 0 + (void **)&dst_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + + read_leb_uint32(p, p_end, src_tbl_idx); + if (!get_table_elem_type(module, src_tbl_idx, &src_type, +#if WASM_ENABLE_GC != 0 + (void **)&src_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_GC == 0 + if (src_type != dst_type) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#else + if (!wasm_reftype_is_subtype_of( + src_type, src_ref_type, dst_type, dst_ref_type, + module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, dst_tbl_idx); + emit_uint32(loader_ctx, src_tbl_idx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_TABLE_SIZE: + { + read_leb_uint32(p, p_end, table_idx); + /* TODO: shall we create a new function to check + table idx instead of using below function? */ + if (!get_table_elem_type(module, table_idx, NULL, NULL, + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + + PUSH_I32(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_TABLE_GROW: + case WASM_OP_TABLE_FILL: + { + uint8 decl_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type = NULL; +#endif + + read_leb_uint32(p, p_end, table_idx); + if (!get_table_elem_type(module, table_idx, &decl_type, +#if WASM_ENABLE_GC != 0 + (void **)&ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(decl_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + + if (opcode1 == WASM_OP_TABLE_GROW) { + if (table_idx < module->import_table_count) { + module->import_tables[table_idx] + .u.table.possible_grow = true; + } + else { + module + ->tables[table_idx + - module->import_table_count] + .possible_grow = true; + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + + POP_I32(); +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(decl_type); +#endif + POP_TYPE(decl_type); + if (opcode1 == WASM_OP_TABLE_GROW) + PUSH_I32(); + else + POP_I32(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + default: + set_error_buf_v(error_buf, error_buf_size, + "%s %02x %02x", "unsupported opcode", + 0xfc, opcode1); + goto fail; + } + break; + } + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + case WASM_OP_SIMD_PREFIX: + { + /* TODO: memory64 offset type changes */ + uint32 opcode1; + +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* Mark the SIMD instruction is used in this module */ + module->is_simd_used = true; +#endif + + read_leb_uint32(p, p_end, opcode1); + + /* follow the order of enum WASMSimdEXTOpcode in wasm_opcode.h + */ + switch (opcode1) { + /* memory instruction */ + case SIMD_v128_load: + case SIMD_v128_load8x8_s: + case SIMD_v128_load8x8_u: + case SIMD_v128_load16x4_s: + case SIMD_v128_load16x4_u: + case SIMD_v128_load32x2_s: + case SIMD_v128_load32x2_u: + case SIMD_v128_load8_splat: + case SIMD_v128_load16_splat: + case SIMD_v128_load32_splat: + case SIMD_v128_load64_splat: + { + CHECK_MEMORY(); + + read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + read_leb_uint32(p, p_end, mem_offset); /* offset */ + + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_V128); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + case SIMD_v128_store: + { + CHECK_MEMORY(); + + read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + read_leb_uint32(p, p_end, mem_offset); /* offset */ + + POP_V128(); + POP_I32(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + /* basic operation */ + case SIMD_v128_const: + { + CHECK_BUF1(p, p_end, 16); + p += 16; + PUSH_V128(); + break; + } + + case SIMD_v8x16_shuffle: + { + V128 mask; + + CHECK_BUF1(p, p_end, 16); + mask = read_i8x16(p, error_buf, error_buf_size); + p += 16; + if (!check_simd_shuffle_mask(mask, error_buf, + error_buf_size)) { + goto fail; + } + + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v8x16_swizzle: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* splat operation */ + case SIMD_i8x16_splat: + case SIMD_i16x8_splat: + case SIMD_i32x4_splat: + case SIMD_i64x2_splat: + case SIMD_f32x4_splat: + case SIMD_f64x2_splat: + { + uint8 pop_type[] = { VALUE_TYPE_I32, VALUE_TYPE_I32, + VALUE_TYPE_I32, VALUE_TYPE_I64, + VALUE_TYPE_F32, VALUE_TYPE_F64 }; + POP_AND_PUSH(pop_type[opcode1 - SIMD_i8x16_splat], + VALUE_TYPE_V128); + break; + } + + /* lane operation */ + case SIMD_i8x16_extract_lane_s: + case SIMD_i8x16_extract_lane_u: + case SIMD_i8x16_replace_lane: + case SIMD_i16x8_extract_lane_s: + case SIMD_i16x8_extract_lane_u: + case SIMD_i16x8_replace_lane: + case SIMD_i32x4_extract_lane: + case SIMD_i32x4_replace_lane: + case SIMD_i64x2_extract_lane: + case SIMD_i64x2_replace_lane: + case SIMD_f32x4_extract_lane: + case SIMD_f32x4_replace_lane: + case SIMD_f64x2_extract_lane: + case SIMD_f64x2_replace_lane: + { + uint8 lane; + /* clang-format off */ + uint8 replace[] = { + /*i8x16*/ 0x0, 0x0, VALUE_TYPE_I32, + /*i16x8*/ 0x0, 0x0, VALUE_TYPE_I32, + /*i32x4*/ 0x0, VALUE_TYPE_I32, + /*i64x2*/ 0x0, VALUE_TYPE_I64, + /*f32x4*/ 0x0, VALUE_TYPE_F32, + /*f64x2*/ 0x0, VALUE_TYPE_F64, + }; + uint8 push_type[] = { + /*i8x16*/ VALUE_TYPE_I32, VALUE_TYPE_I32, + VALUE_TYPE_V128, + /*i16x8*/ VALUE_TYPE_I32, VALUE_TYPE_I32, + VALUE_TYPE_V128, + /*i32x4*/ VALUE_TYPE_I32, VALUE_TYPE_V128, + /*i64x2*/ VALUE_TYPE_I64, VALUE_TYPE_V128, + /*f32x4*/ VALUE_TYPE_F32, VALUE_TYPE_V128, + /*f64x2*/ VALUE_TYPE_F64, VALUE_TYPE_V128, + }; + /* clang-format on */ + + CHECK_BUF(p, p_end, 1); + lane = read_uint8(p); + if (!check_simd_access_lane(opcode1, lane, error_buf, + error_buf_size)) { + goto fail; + } + + if (replace[opcode1 - SIMD_i8x16_extract_lane_s]) { + if (!(wasm_loader_pop_frame_ref( + loader_ctx, + replace[opcode1 + - SIMD_i8x16_extract_lane_s], + error_buf, error_buf_size))) + goto fail; + } + + POP_AND_PUSH( + VALUE_TYPE_V128, + push_type[opcode1 - SIMD_i8x16_extract_lane_s]); + break; + } + + /* i8x16 compare operation */ + case SIMD_i8x16_eq: + case SIMD_i8x16_ne: + case SIMD_i8x16_lt_s: + case SIMD_i8x16_lt_u: + case SIMD_i8x16_gt_s: + case SIMD_i8x16_gt_u: + case SIMD_i8x16_le_s: + case SIMD_i8x16_le_u: + case SIMD_i8x16_ge_s: + case SIMD_i8x16_ge_u: + /* i16x8 compare operation */ + case SIMD_i16x8_eq: + case SIMD_i16x8_ne: + case SIMD_i16x8_lt_s: + case SIMD_i16x8_lt_u: + case SIMD_i16x8_gt_s: + case SIMD_i16x8_gt_u: + case SIMD_i16x8_le_s: + case SIMD_i16x8_le_u: + case SIMD_i16x8_ge_s: + case SIMD_i16x8_ge_u: + /* i32x4 compare operation */ + case SIMD_i32x4_eq: + case SIMD_i32x4_ne: + case SIMD_i32x4_lt_s: + case SIMD_i32x4_lt_u: + case SIMD_i32x4_gt_s: + case SIMD_i32x4_gt_u: + case SIMD_i32x4_le_s: + case SIMD_i32x4_le_u: + case SIMD_i32x4_ge_s: + case SIMD_i32x4_ge_u: + /* f32x4 compare operation */ + case SIMD_f32x4_eq: + case SIMD_f32x4_ne: + case SIMD_f32x4_lt: + case SIMD_f32x4_gt: + case SIMD_f32x4_le: + case SIMD_f32x4_ge: + /* f64x2 compare operation */ + case SIMD_f64x2_eq: + case SIMD_f64x2_ne: + case SIMD_f64x2_lt: + case SIMD_f64x2_gt: + case SIMD_f64x2_le: + case SIMD_f64x2_ge: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* v128 operation */ + case SIMD_v128_not: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v128_and: + case SIMD_v128_andnot: + case SIMD_v128_or: + case SIMD_v128_xor: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v128_bitselect: + { + POP_V128(); + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v128_any_true: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + /* Load Lane Operation */ + case SIMD_v128_load8_lane: + case SIMD_v128_load16_lane: + case SIMD_v128_load32_lane: + case SIMD_v128_load64_lane: + case SIMD_v128_store8_lane: + case SIMD_v128_store16_lane: + case SIMD_v128_store32_lane: + case SIMD_v128_store64_lane: + { + uint8 lane; + + CHECK_MEMORY(); + + read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + read_leb_uint32(p, p_end, mem_offset); /* offset */ + + CHECK_BUF(p, p_end, 1); + lane = read_uint8(p); + if (!check_simd_access_lane(opcode1, lane, error_buf, + error_buf_size)) { + goto fail; + } + + POP_V128(); + POP_I32(); + if (opcode1 < SIMD_v128_store8_lane) { + PUSH_V128(); + } +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + case SIMD_v128_load32_zero: + case SIMD_v128_load64_zero: + { + CHECK_MEMORY(); + + read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + read_leb_uint32(p, p_end, mem_offset); /* offset */ + + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_V128); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + /* Float conversion */ + case SIMD_f32x4_demote_f64x2_zero: + case SIMD_f64x2_promote_low_f32x4_zero: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* i8x16 Operation */ + case SIMD_i8x16_abs: + case SIMD_i8x16_neg: + case SIMD_i8x16_popcnt: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_all_true: + case SIMD_i8x16_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i8x16_narrow_i16x8_s: + case SIMD_i8x16_narrow_i16x8_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f32x4_ceil: + case SIMD_f32x4_floor: + case SIMD_f32x4_trunc: + case SIMD_f32x4_nearest: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_shl: + case SIMD_i8x16_shr_s: + case SIMD_i8x16_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_add: + case SIMD_i8x16_add_sat_s: + case SIMD_i8x16_add_sat_u: + case SIMD_i8x16_sub: + case SIMD_i8x16_sub_sat_s: + case SIMD_i8x16_sub_sat_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_ceil: + case SIMD_f64x2_floor: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_min_s: + case SIMD_i8x16_min_u: + case SIMD_i8x16_max_s: + case SIMD_i8x16_max_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_trunc: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_avgr_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_extadd_pairwise_i8x16_s: + case SIMD_i16x8_extadd_pairwise_i8x16_u: + case SIMD_i32x4_extadd_pairwise_i16x8_s: + case SIMD_i32x4_extadd_pairwise_i16x8_u: + /* i16x8 operation */ + case SIMD_i16x8_abs: + case SIMD_i16x8_neg: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_q15mulr_sat_s: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_all_true: + case SIMD_i16x8_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i16x8_narrow_i32x4_s: + case SIMD_i16x8_narrow_i32x4_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_extend_low_i8x16_s: + case SIMD_i16x8_extend_high_i8x16_s: + case SIMD_i16x8_extend_low_i8x16_u: + case SIMD_i16x8_extend_high_i8x16_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_shl: + case SIMD_i16x8_shr_s: + case SIMD_i16x8_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_add: + case SIMD_i16x8_add_sat_s: + case SIMD_i16x8_add_sat_u: + case SIMD_i16x8_sub: + case SIMD_i16x8_sub_sat_s: + case SIMD_i16x8_sub_sat_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_nearest: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_mul: + case SIMD_i16x8_min_s: + case SIMD_i16x8_min_u: + case SIMD_i16x8_max_s: + case SIMD_i16x8_max_u: + case SIMD_i16x8_avgr_u: + case SIMD_i16x8_extmul_low_i8x16_s: + case SIMD_i16x8_extmul_high_i8x16_s: + case SIMD_i16x8_extmul_low_i8x16_u: + case SIMD_i16x8_extmul_high_i8x16_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* i32x4 operation */ + case SIMD_i32x4_abs: + case SIMD_i32x4_neg: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_all_true: + case SIMD_i32x4_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i32x4_extend_low_i16x8_s: + case SIMD_i32x4_extend_high_i16x8_s: + case SIMD_i32x4_extend_low_i16x8_u: + case SIMD_i32x4_extend_high_i16x8_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_shl: + case SIMD_i32x4_shr_s: + case SIMD_i32x4_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_add: + case SIMD_i32x4_sub: + case SIMD_i32x4_mul: + case SIMD_i32x4_min_s: + case SIMD_i32x4_min_u: + case SIMD_i32x4_max_s: + case SIMD_i32x4_max_u: + case SIMD_i32x4_dot_i16x8_s: + case SIMD_i32x4_extmul_low_i16x8_s: + case SIMD_i32x4_extmul_high_i16x8_s: + case SIMD_i32x4_extmul_low_i16x8_u: + case SIMD_i32x4_extmul_high_i16x8_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* i64x2 operation */ + case SIMD_i64x2_abs: + case SIMD_i64x2_neg: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i64x2_all_true: + case SIMD_i64x2_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i64x2_extend_low_i32x4_s: + case SIMD_i64x2_extend_high_i32x4_s: + case SIMD_i64x2_extend_low_i32x4_u: + case SIMD_i64x2_extend_high_i32x4_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i64x2_shl: + case SIMD_i64x2_shr_s: + case SIMD_i64x2_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i64x2_add: + case SIMD_i64x2_sub: + case SIMD_i64x2_mul: + case SIMD_i64x2_eq: + case SIMD_i64x2_ne: + case SIMD_i64x2_lt_s: + case SIMD_i64x2_gt_s: + case SIMD_i64x2_le_s: + case SIMD_i64x2_ge_s: + case SIMD_i64x2_extmul_low_i32x4_s: + case SIMD_i64x2_extmul_high_i32x4_s: + case SIMD_i64x2_extmul_low_i32x4_u: + case SIMD_i64x2_extmul_high_i32x4_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* f32x4 operation */ + case SIMD_f32x4_abs: + case SIMD_f32x4_neg: + case SIMD_f32x4_sqrt: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f32x4_add: + case SIMD_f32x4_sub: + case SIMD_f32x4_mul: + case SIMD_f32x4_div: + case SIMD_f32x4_min: + case SIMD_f32x4_max: + case SIMD_f32x4_pmin: + case SIMD_f32x4_pmax: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* f64x2 operation */ + case SIMD_f64x2_abs: + case SIMD_f64x2_neg: + case SIMD_f64x2_sqrt: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_add: + case SIMD_f64x2_sub: + case SIMD_f64x2_mul: + case SIMD_f64x2_div: + case SIMD_f64x2_min: + case SIMD_f64x2_max: + case SIMD_f64x2_pmin: + case SIMD_f64x2_pmax: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_trunc_sat_f32x4_s: + case SIMD_i32x4_trunc_sat_f32x4_u: + case SIMD_f32x4_convert_i32x4_s: + case SIMD_f32x4_convert_i32x4_u: + case SIMD_i32x4_trunc_sat_f64x2_s_zero: + case SIMD_i32x4_trunc_sat_f64x2_u_zero: + case SIMD_f64x2_convert_low_i32x4_s: + case SIMD_f64x2_convert_low_i32x4_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + default: + { + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, + "WASM module load failed: " + "invalid opcode 0xfd %02x.", + opcode1); + } + goto fail; + } + } + break; + } +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, opcode1); +#endif + if (opcode1 != WASM_OP_ATOMIC_FENCE) { + CHECK_MEMORY(); + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ + if (!check_memory_align_equal(opcode1, align, error_buf, + error_buf_size)) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif + } +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + switch (opcode1) { + case WASM_OP_ATOMIC_NOTIFY: + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_WAIT32: + POP_I64(); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_WAIT64: + POP_I64(); + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_FENCE: + /* reserved byte 0x00 */ + if (*p++ != 0x00) { + set_error_buf(error_buf, error_buf_size, + "zero byte expected"); + goto fail; + } + break; + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); + break; + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + POP_I32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); + break; + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + POP_I64(); + POP_MEM_OFFSET(); + break; + case WASM_OP_ATOMIC_RMW_I32_ADD: + case WASM_OP_ATOMIC_RMW_I32_ADD8_U: + case WASM_OP_ATOMIC_RMW_I32_ADD16_U: + case WASM_OP_ATOMIC_RMW_I32_SUB: + case WASM_OP_ATOMIC_RMW_I32_SUB8_U: + case WASM_OP_ATOMIC_RMW_I32_SUB16_U: + case WASM_OP_ATOMIC_RMW_I32_AND: + case WASM_OP_ATOMIC_RMW_I32_AND8_U: + case WASM_OP_ATOMIC_RMW_I32_AND16_U: + case WASM_OP_ATOMIC_RMW_I32_OR: + case WASM_OP_ATOMIC_RMW_I32_OR8_U: + case WASM_OP_ATOMIC_RMW_I32_OR16_U: + case WASM_OP_ATOMIC_RMW_I32_XOR: + case WASM_OP_ATOMIC_RMW_I32_XOR8_U: + case WASM_OP_ATOMIC_RMW_I32_XOR16_U: + case WASM_OP_ATOMIC_RMW_I32_XCHG: + case WASM_OP_ATOMIC_RMW_I32_XCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_XCHG16_U: + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_RMW_I64_ADD: + case WASM_OP_ATOMIC_RMW_I64_ADD8_U: + case WASM_OP_ATOMIC_RMW_I64_ADD16_U: + case WASM_OP_ATOMIC_RMW_I64_ADD32_U: + case WASM_OP_ATOMIC_RMW_I64_SUB: + case WASM_OP_ATOMIC_RMW_I64_SUB8_U: + case WASM_OP_ATOMIC_RMW_I64_SUB16_U: + case WASM_OP_ATOMIC_RMW_I64_SUB32_U: + case WASM_OP_ATOMIC_RMW_I64_AND: + case WASM_OP_ATOMIC_RMW_I64_AND8_U: + case WASM_OP_ATOMIC_RMW_I64_AND16_U: + case WASM_OP_ATOMIC_RMW_I64_AND32_U: + case WASM_OP_ATOMIC_RMW_I64_OR: + case WASM_OP_ATOMIC_RMW_I64_OR8_U: + case WASM_OP_ATOMIC_RMW_I64_OR16_U: + case WASM_OP_ATOMIC_RMW_I64_OR32_U: + case WASM_OP_ATOMIC_RMW_I64_XOR: + case WASM_OP_ATOMIC_RMW_I64_XOR8_U: + case WASM_OP_ATOMIC_RMW_I64_XOR16_U: + case WASM_OP_ATOMIC_RMW_I64_XOR32_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG: + case WASM_OP_ATOMIC_RMW_I64_XCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG32_U: + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I64(); + break; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + POP_I32(); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + POP_I64(); + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I64(); + break; + default: + set_error_buf_v(error_buf, error_buf_size, + "%s %02x %02x", "unsupported opcode", + 0xfe, opcode1); + goto fail; + } + break; + } +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + + default: + set_error_buf_v(error_buf, error_buf_size, "%s %02x", + "unsupported opcode", opcode); + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + last_op = opcode; +#endif + } + + if (loader_ctx->csp_num > 0) { + if (cur_func_idx < module->function_count - 1) + /* Function with missing end marker (between two functions) */ + set_error_buf(error_buf, error_buf_size, "END opcode expected"); + else + /* Function with missing end marker + (at EOF or end of code sections) */ + set_error_buf(error_buf, error_buf_size, + "unexpected end of section or function, " + "or section size mismatch"); + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled == NULL) + goto re_scan; + + func->const_cell_num = loader_ctx->const_cell_num; + if (func->const_cell_num > 0) { + int32 j; + + if (!(func->consts = func_const = loader_malloc( + func->const_cell_num * 4, error_buf, error_buf_size))) + goto fail; + + func_const_end = func->consts + func->const_cell_num * 4; + /* reverse the const buf */ + for (j = loader_ctx->num_const - 1; j >= 0; j--) { + Const *c = (Const *)(loader_ctx->const_buf + j * sizeof(Const)); + if (c->value_type == VALUE_TYPE_F64 + || c->value_type == VALUE_TYPE_I64) { + bh_memcpy_s(func_const, (uint32)(func_const_end - func_const), + &(c->value.f64), (uint32)sizeof(int64)); + func_const += sizeof(int64); + } + else { + bh_memcpy_s(func_const, (uint32)(func_const_end - func_const), + &(c->value.f32), (uint32)sizeof(int32)); + func_const += sizeof(int32); + } + } + } + + func->max_stack_cell_num = loader_ctx->preserved_local_offset + - loader_ctx->start_dynamic_offset + 1; +#else + func->max_stack_cell_num = loader_ctx->max_stack_cell_num; +#endif + func->max_block_num = loader_ctx->max_csp_num; + return_value = true; + +fail: + wasm_loader_ctx_destroy(loader_ctx); + + (void)table_idx; + (void)table_seg_idx; + (void)data_seg_idx; + (void)i64_const; + (void)local_offset; + (void)p_org; + (void)mem_offset; + (void)align; + return return_value; +} diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_loader.h b/wasm-micro-runtime/core/iwasm/interpreter/wasm_loader.h new file mode 100644 index 0000000..676770e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_loader.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#ifndef _WASM_LOADER_H +#define _WASM_LOADER_H + +#include "wasm.h" +#include "bh_hashmap.h" +#include "../common/wasm_runtime_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Load a WASM module from a specified byte buffer. + * + * @param buf the byte buffer which contains the WASM binary data + * @param size the size of the buffer + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return module loaded, NULL if failed + */ +WASMModule * +wasm_loader_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *args, char *error_buf, uint32 error_buf_size); + +/** + * Load a WASM module from a specified WASM section list. + * + * @param section_list the section list which contains each section data + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return WASM module loaded, NULL if failed + */ +WASMModule * +wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size); + +/** + * Unload a WASM module. + * + * @param module the module to be unloaded + */ +void +wasm_loader_unload(WASMModule *module); + +/** + * Find address of related else opcode and end opcode of opcode block/loop/if + * according to the start address of opcode. + * + * @param module the module to find + * @param start_addr the next address of opcode block/loop/if + * @param code_end_addr the end address of function code block + * @param block_type the type of block, 0/1/2 denotes block/loop/if + * @param p_else_addr returns the else addr if found + * @param p_end_addr returns the end addr if found + * @param error_buf returns the error log for this function + * @param error_buf_size returns the error log string length + * + * @return true if success, false otherwise + */ + +bool +wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, + const uint8 *start_addr, const uint8 *code_end_addr, + uint8 block_type, uint8 **p_else_addr, + uint8 **p_end_addr); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_LOADER_H */ diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_mini_loader.c b/wasm-micro-runtime/core/iwasm/interpreter/wasm_mini_loader.c new file mode 100644 index 0000000..d5282c6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_mini_loader.c @@ -0,0 +1,8119 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_loader.h" +#include "bh_common.h" +#include "bh_log.h" +#include "wasm.h" +#include "wasm_opcode.h" +#include "wasm_runtime.h" +#include "../common/wasm_native.h" +#include "../common/wasm_memory.h" +#if WASM_ENABLE_FAST_JIT != 0 +#include "../fast-jit/jit_compiler.h" +#include "../fast-jit/jit_codecache.h" +#endif +#if WASM_ENABLE_JIT != 0 +#include "../compilation/aot_llvm.h" +#endif + +/* Read a value of given type from the address pointed to by the given + pointer and increase the pointer to the position just after the + value being read. */ +#define TEMPLATE_READ_VALUE(Type, p) \ + (p += sizeof(Type), *(Type *)(p - sizeof(Type))) + +#if WASM_ENABLE_MEMORY64 != 0 +static bool +has_module_memory64(WASMModule *module) +{ + /* TODO: multi-memories for now assuming the memory idx type is consistent + * across multi-memories */ + if (module->import_memory_count > 0) + return !!(module->import_memories[0].u.memory.flags & MEMORY64_FLAG); + else if (module->memory_count > 0) + return !!(module->memories[0].flags & MEMORY64_FLAG); + + return false; +} +#endif + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "WASM module load failed: %s", + string); +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + bh_assert(buf + length >= buf && buf + length <= buf_end); \ + } while (0) + +#define CHECK_BUF1(buf, buf_end, length) \ + do { \ + bh_assert(buf + length >= buf && buf + length <= buf_end); \ + } while (0) + +#define skip_leb(p) while (*p++ & 0x80) +#define skip_leb_int64(p, p_end) skip_leb(p) +#define skip_leb_uint32(p, p_end) skip_leb(p) +#define skip_leb_int32(p, p_end) skip_leb(p) +#define skip_leb_mem_offset(p, p_end) skip_leb(p) + +static bool +is_32bit_type(uint8 type) +{ + if (type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32 +#if WASM_ENABLE_REF_TYPES != 0 + || type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF +#endif + ) + return true; + return false; +} + +static bool +is_64bit_type(uint8 type) +{ + if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64) + return true; + return false; +} + +static bool +is_value_type(uint8 type) +{ + if (type == VALUE_TYPE_I32 || type == VALUE_TYPE_I64 + || type == VALUE_TYPE_F32 || type == VALUE_TYPE_F64 +#if WASM_ENABLE_REF_TYPES != 0 + || type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF +#endif + ) + return true; + return false; +} + +static bool +is_byte_a_type(uint8 type) +{ + return is_value_type(type) || (type == VALUE_TYPE_VOID); +} + +static void +read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, + uint64 *p_result, char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + uint64 result = 0; + uint32 shift = 0; + uint32 offset = 0, bcnt = 0; + uint64 byte; + + while (true) { + bh_assert(bcnt + 1 <= (maxbits + 6) / 7); + CHECK_BUF(buf, buf_end, offset + 1); + byte = buf[offset]; + offset += 1; + result |= ((byte & 0x7f) << shift); + shift += 7; + bcnt += 1; + if ((byte & 0x80) == 0) { + break; + } + } + + if (!sign && maxbits == 32 && shift >= maxbits) { + /* The top bits set represent values > 32 bits */ + bh_assert(!(((uint8)byte) & 0xf0)); + } + else if (sign && maxbits == 32) { + if (shift < maxbits) { + /* Sign extend, second-highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x8; + int top_bits = ((uint8)byte) & 0xf0; + bh_assert(!((sign_bit_set && top_bits != 0x70) + || (!sign_bit_set && top_bits != 0))); + (void)top_bits; + (void)sign_bit_set; + } + } + else if (sign && maxbits == 64) { + if (shift < maxbits) { + /* Sign extend, second-highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x1; + int top_bits = ((uint8)byte) & 0xfe; + + bh_assert(!((sign_bit_set && top_bits != 0x7e) + || (!sign_bit_set && top_bits != 0))); + (void)top_bits; + (void)sign_bit_set; + } + } + + *p_buf += offset; + *p_result = result; +} + +#define read_uint8(p) TEMPLATE_READ_VALUE(uint8, p) +#define read_uint32(p) TEMPLATE_READ_VALUE(uint32, p) +#define read_bool(p) TEMPLATE_READ_VALUE(bool, p) + +#define read_leb_int64(p, p_end, res) \ + do { \ + uint64 res64; \ + read_leb((uint8 **)&p, p_end, 64, true, &res64, error_buf, \ + error_buf_size); \ + res = (int64)res64; \ + } while (0) + +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint64 res64; \ + read_leb((uint8 **)&p, p_end, 32, false, &res64, error_buf, \ + error_buf_size); \ + res = (uint32)res64; \ + } while (0) + +#define read_leb_int32(p, p_end, res) \ + do { \ + uint64 res64; \ + read_leb((uint8 **)&p, p_end, 32, true, &res64, error_buf, \ + error_buf_size); \ + res = (int32)res64; \ + } while (0) + +#if WASM_ENABLE_MEMORY64 != 0 +#define read_leb_mem_offset(p, p_end, res) \ + do { \ + uint64 res64; \ + read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, &res64, \ + error_buf, error_buf_size); \ + res = (mem_offset_t)res64; \ + } while (0) +#else +#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + +static void * +loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +static void * +memory_realloc(void *mem_old, uint32 size_old, uint32 size_new, char *error_buf, + uint32 error_buf_size) +{ + uint8 *mem_new; + bh_assert(size_new > size_old); + if ((mem_new = loader_malloc(size_new, error_buf, error_buf_size))) { + bh_memcpy_s(mem_new, size_new, mem_old, size_old); + memset(mem_new + size_old, 0, size_new - size_old); + wasm_runtime_free(mem_old); + } + return mem_new; +} + +#define MEM_REALLOC(mem, size_old, size_new) \ + do { \ + void *mem_new = memory_realloc(mem, size_old, size_new, error_buf, \ + error_buf_size); \ + if (!mem_new) \ + goto fail; \ + mem = mem_new; \ + } while (0) + +static void +destroy_wasm_type(WASMFuncType *type) +{ + if (type->ref_count > 1) { + /* The type is referenced by other types + of current wasm module */ + type->ref_count--; + return; + } + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (type->call_to_llvm_jit_from_fast_jit) + jit_code_cache_free(type->call_to_llvm_jit_from_fast_jit); +#endif + + wasm_runtime_free(type); +} + +static bool +check_function_index(const WASMModule *module, uint32 function_index, + char *error_buf, uint32 error_buf_size) +{ + return (function_index + < module->import_function_count + module->function_count); +} + +typedef struct InitValue { + uint8 type; + uint8 flag; + WASMValue value; +} InitValue; + +typedef struct ConstExprContext { + uint32 sp; + uint32 size; + WASMModule *module; + InitValue *stack; + InitValue data[WASM_CONST_EXPR_STACK_SIZE]; +} ConstExprContext; + +static void +init_const_expr_stack(ConstExprContext *ctx, WASMModule *module) +{ + ctx->sp = 0; + ctx->module = module; + ctx->stack = ctx->data; + ctx->size = WASM_CONST_EXPR_STACK_SIZE; +} + +static bool +push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type, + WASMValue *value, char *error_buf, uint32 error_buf_size) +{ + InitValue *cur_value; + + if (ctx->sp >= ctx->size) { + if (ctx->stack != ctx->data) { + MEM_REALLOC(ctx->stack, ctx->size * sizeof(InitValue), + (ctx->size + 4) * sizeof(InitValue)); + } + else { + if (!(ctx->stack = + loader_malloc((ctx->size + 4) * (uint64)sizeof(InitValue), + error_buf, error_buf_size))) { + return false; + } + } + ctx->size += 4; + } + + cur_value = &ctx->stack[ctx->sp++]; + cur_value->type = type; + cur_value->flag = flag; + cur_value->value = *value; + + return true; +fail: + return false; +} + +static bool +pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type, + WASMValue *p_value, char *error_buf, uint32 error_buf_size) +{ + InitValue *cur_value; + + if (ctx->sp == 0) { + return false; + } + + cur_value = &ctx->stack[--ctx->sp]; + + if (cur_value->type != type) { + return false; + } + + if (p_flag) + *p_flag = cur_value->flag; + if (p_value) + *p_value = cur_value->value; + + return true; +} + +static void +destroy_const_expr_stack(ConstExprContext *ctx) +{ + if (ctx->stack != ctx->data) { + wasm_runtime_free(ctx->stack); + } +} + +static bool +load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, + InitializerExpression *init_expr, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 flag, *p_float; + uint32 i; + ConstExprContext const_expr_ctx = { 0 }; + WASMValue cur_value = { 0 }; + + init_const_expr_stack(&const_expr_ctx, module); + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + while (flag != WASM_OP_END) { + switch (flag) { + /* i32.const */ + case INIT_EXPR_TYPE_I32_CONST: + read_leb_int32(p, p_end, cur_value.i32); + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_I32, &cur_value, + error_buf, error_buf_size)) { + bh_assert(0); + } + break; + /* i64.const */ + case INIT_EXPR_TYPE_I64_CONST: + read_leb_int64(p, p_end, cur_value.i64); + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_I64, &cur_value, + error_buf, error_buf_size)) { + bh_assert(0); + } + break; + /* f32.const */ + case INIT_EXPR_TYPE_F32_CONST: + CHECK_BUF(p, p_end, 4); + p_float = (uint8 *)&cur_value.f32; + for (i = 0; i < sizeof(float32); i++) + *p_float++ = *p++; + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_F32, &cur_value, + error_buf, error_buf_size)) { + bh_assert(0); + } + break; + /* f64.const */ + case INIT_EXPR_TYPE_F64_CONST: + CHECK_BUF(p, p_end, 8); + p_float = (uint8 *)&cur_value.f64; + for (i = 0; i < sizeof(float64); i++) + *p_float++ = *p++; + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_F64, &cur_value, + error_buf, error_buf_size)) { + bh_assert(0); + } + break; + +#if WASM_ENABLE_REF_TYPES != 0 + /* ref.func */ + case INIT_EXPR_TYPE_FUNCREF_CONST: + { + uint32 func_idx; + read_leb_uint32(p, p_end, func_idx); + cur_value.ref_index = func_idx; + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + bh_assert(0); + } + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_FUNCREF, &cur_value, + error_buf, error_buf_size)) { + bh_assert(0); + } + break; + } + + /* ref.null */ + case INIT_EXPR_TYPE_REFNULL_CONST: + { + uint8 type1; + + CHECK_BUF(p, p_end, 1); + type1 = read_uint8(p); + + cur_value.ref_index = UINT32_MAX; + if (!push_const_expr_stack(&const_expr_ctx, flag, type1, + &cur_value, error_buf, + error_buf_size)) { + bh_assert(0); + } + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 */ + + /* get_global */ + case INIT_EXPR_TYPE_GET_GLOBAL: + { + uint32 global_idx; + uint8 global_type; + + read_leb_uint32(p, p_end, cur_value.global_index); + global_idx = cur_value.global_index; + + bh_assert(global_idx < module->import_global_count); + bh_assert( + !module->import_globals[global_idx].u.global.is_mutable); + + if (global_idx < module->import_global_count) { + global_type = + module->import_globals[global_idx].u.global.type; + } + else { + global_type = + module + ->globals[global_idx - module->import_global_count] + .type; + } + + if (!push_const_expr_stack(&const_expr_ctx, flag, global_type, + &cur_value, error_buf, + error_buf_size)) + bh_assert(0); + + break; + } + default: + { + bh_assert(0); + } + } + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + } + + /* There should be only one value left on the init value stack */ + if (!pop_const_expr_stack(&const_expr_ctx, &flag, type, &cur_value, + error_buf, error_buf_size)) { + bh_assert(0); + } + + bh_assert(const_expr_ctx.sp == 0); + + init_expr->init_expr_type = flag; + init_expr->u = cur_value; + + *p_buf = p; + destroy_const_expr_stack(&const_expr_ctx); + return true; +} + +static bool +load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end, *p_org; + uint32 type_count, param_count, result_count, i, j; + uint32 param_cell_num, ret_cell_num; + uint64 total_size; + uint8 flag; + WASMFuncType *type; + + read_leb_uint32(p, p_end, type_count); + + if (type_count) { + module->type_count = type_count; + total_size = sizeof(WASMFuncType *) * (uint64)type_count; + if (!(module->types = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < type_count; i++) { + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + bh_assert(flag == 0x60); + + read_leb_uint32(p, p_end, param_count); + + /* Resolve param count and result count firstly */ + p_org = p; + CHECK_BUF(p, p_end, param_count); + p += param_count; + read_leb_uint32(p, p_end, result_count); + CHECK_BUF(p, p_end, result_count); + p = p_org; + + bh_assert(param_count <= UINT16_MAX && result_count <= UINT16_MAX); + + total_size = offsetof(WASMFuncType, types) + + sizeof(uint8) * (uint64)(param_count + result_count); + if (!(type = module->types[i] = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Resolve param types and result types */ + type->ref_count = 1; + type->param_count = (uint16)param_count; + type->result_count = (uint16)result_count; + for (j = 0; j < param_count; j++) { + CHECK_BUF(p, p_end, 1); + type->types[j] = read_uint8(p); + } + read_leb_uint32(p, p_end, result_count); + for (j = 0; j < result_count; j++) { + CHECK_BUF(p, p_end, 1); + type->types[param_count + j] = read_uint8(p); + } + for (j = 0; j < param_count + result_count; j++) { + bh_assert(is_value_type(type->types[j])); + } + + param_cell_num = wasm_get_cell_num(type->types, param_count); + ret_cell_num = + wasm_get_cell_num(type->types + param_count, result_count); + bh_assert(param_cell_num <= UINT16_MAX + && ret_cell_num <= UINT16_MAX); + type->param_cell_num = (uint16)param_cell_num; + type->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + + /* If there is already a same type created, use it instead */ + for (j = 0; j < i; ++j) { + if (wasm_type_equal(type, module->types[j], module->types, i)) { + bh_assert(module->types[j]->ref_count != UINT16_MAX); + destroy_wasm_type(type); + module->types[i] = module->types[j]; + module->types[j]->ref_count++; + break; + } + } + } + } + + bh_assert(p == p_end); + LOG_VERBOSE("Load type section success.\n"); + (void)flag; + return true; +} + +static void +adjust_table_max_size(uint32 init_size, uint32 max_size_flag, uint32 *max_size) +{ + uint32 default_max_size = init_size * 2 > WASM_TABLE_MAX_SIZE + ? init_size * 2 + : WASM_TABLE_MAX_SIZE; + + if (max_size_flag) { + /* module defines the table limitation */ + bh_assert(init_size <= *max_size); + + if (init_size < *max_size) { + *max_size = + *max_size < default_max_size ? *max_size : default_max_size; + } + } + else { + /* partial defined table limitation, gives a default value */ + *max_size = default_max_size; + } +} + +static bool +load_function_import(const uint8 **p_buf, const uint8 *buf_end, + const WASMModule *parent_module, + const char *sub_module_name, const char *function_name, + WASMFunctionImport *function, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 declare_type_index = 0; + WASMFuncType *declare_func_type = NULL; + WASMFunction *linked_func = NULL; + const char *linked_signature = NULL; + void *linked_attachment = NULL; + bool linked_call_conv_raw = false; + + read_leb_uint32(p, p_end, declare_type_index); + *p_buf = p; + + bh_assert(declare_type_index < parent_module->type_count); + + declare_func_type = parent_module->types[declare_type_index]; + + /* check built-in modules */ + linked_func = wasm_native_resolve_symbol( + sub_module_name, function_name, declare_func_type, &linked_signature, + &linked_attachment, &linked_call_conv_raw); + + function->module_name = (char *)sub_module_name; + function->field_name = (char *)function_name; + function->func_type = declare_func_type; + function->func_ptr_linked = linked_func; + function->signature = linked_signature; + function->attachment = linked_attachment; + function->call_conv_raw = linked_call_conv_raw; + return true; +} + +static bool +load_table_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, const char *sub_module_name, + const char *table_name, WASMTableImport *table, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 declare_elem_type = 0, declare_max_size_flag = 0, + declare_init_size = 0, declare_max_size = 0; + + CHECK_BUF(p, p_end, 1); + /* 0x70 or 0x6F */ + declare_elem_type = read_uint8(p); + bh_assert(VALUE_TYPE_FUNCREF == declare_elem_type +#if WASM_ENABLE_REF_TYPES != 0 + || VALUE_TYPE_EXTERNREF == declare_elem_type +#endif + ); + + read_leb_uint32(p, p_end, declare_max_size_flag); + read_leb_uint32(p, p_end, declare_init_size); + if (declare_max_size_flag & 1) { + read_leb_uint32(p, p_end, declare_max_size); + bh_assert(table->init_size <= table->max_size); + } + + adjust_table_max_size(declare_init_size, declare_max_size_flag, + &declare_max_size); + *p_buf = p; + + bh_assert( + !((declare_max_size_flag & 1) && declare_init_size > declare_max_size)); + + /* now we believe all declaration are ok */ + table->elem_type = declare_elem_type; + table->init_size = declare_init_size; + table->flags = declare_max_size_flag; + table->max_size = declare_max_size; + return true; +} + +static bool +check_memory_flag(const uint8 mem_flag) +{ + /* Check whether certain features indicated by mem_flag are enabled in + * runtime */ + if (mem_flag > MAX_PAGE_COUNT_FLAG) { +#if WASM_ENABLE_SHARED_MEMORY == 0 + if (mem_flag & SHARED_MEMORY_FLAG) { + LOG_VERBOSE("shared memory flag was found, please enable shared " + "memory, lib-pthread or lib-wasi-threads"); + return false; + } +#endif +#if WASM_ENABLE_MEMORY64 == 0 + if (mem_flag & MEMORY64_FLAG) { + LOG_VERBOSE("memory64 flag was found, please enable memory64"); + return false; + } +#endif + } + + if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) { + return false; + } + else if ((mem_flag & SHARED_MEMORY_FLAG) + && !(mem_flag & MAX_PAGE_COUNT_FLAG)) { + return false; + } + + return true; +} + +static bool +load_memory_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, const char *sub_module_name, + const char *memory_name, WASMMemoryImport *memory, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; +#if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); + uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT + / DEFAULT_NUM_BYTES_PER_PAGE; +#else + uint32 max_page_count; +#endif /* WASM_ENABLE_APP_FRAMEWORK */ + uint32 mem_flag = 0; + bool is_memory64 = false; + uint32 declare_init_page_count = 0; + uint32 declare_max_page_count = 0; + + read_leb_uint32(p, p_end, mem_flag); + bh_assert(check_memory_flag(mem_flag)); + +#if WASM_ENABLE_APP_FRAMEWORK == 0 + is_memory64 = mem_flag & MEMORY64_FLAG; + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + + read_leb_uint32(p, p_end, declare_init_page_count); + bh_assert(declare_init_page_count <= max_page_count); + + if (mem_flag & MAX_PAGE_COUNT_FLAG) { + read_leb_uint32(p, p_end, declare_max_page_count); + bh_assert(declare_init_page_count <= declare_max_page_count); + bh_assert(declare_max_page_count <= max_page_count); + if (declare_max_page_count > max_page_count) { + declare_max_page_count = max_page_count; + } + } + else { + /* Limit the maximum memory size to max_page_count */ + declare_max_page_count = max_page_count; + } + + /* now we believe all declaration are ok */ + memory->flags = mem_flag; + memory->init_page_count = declare_init_page_count; + memory->max_page_count = declare_max_page_count; + memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + + *p_buf = p; + (void)check_memory_flag; + return true; +} + +static bool +load_global_import(const uint8 **p_buf, const uint8 *buf_end, + const WASMModule *parent_module, char *sub_module_name, + char *global_name, WASMGlobalImport *global, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 declare_type = 0; + uint8 declare_mutable = 0; + bool is_mutable = false; + bool ret = false; + + CHECK_BUF(p, p_end, 2); + declare_type = read_uint8(p); + declare_mutable = read_uint8(p); + *p_buf = p; + + bh_assert(declare_mutable < 2); + + is_mutable = declare_mutable & 1 ? true : false; + +#if WASM_ENABLE_LIBC_BUILTIN != 0 + /* check built-in modules */ + ret = wasm_native_lookup_libc_builtin_global(sub_module_name, global_name, + global); + if (ret) { + bh_assert(global->type == declare_type + && global->is_mutable != declare_mutable); + } +#endif /* WASM_ENABLE_LIBC_BUILTIN */ + + global->is_linked = ret; + global->module_name = sub_module_name; + global->field_name = global_name; + global->type = declare_type; + global->is_mutable = is_mutable; + (void)p_end; + return true; +} + +static bool +load_table(const uint8 **p_buf, const uint8 *buf_end, WASMTable *table, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; + + CHECK_BUF(p, p_end, 1); + /* 0x70 or 0x6F */ + table->elem_type = read_uint8(p); + bh_assert((VALUE_TYPE_FUNCREF == table->elem_type) +#if WASM_ENABLE_REF_TYPES != 0 + || VALUE_TYPE_EXTERNREF == table->elem_type +#endif + ); + + p_org = p; + read_leb_uint32(p, p_end, table->flags); + bh_assert(p - p_org <= 1); + bh_assert(table->flags <= 1); + (void)p_org; + + read_leb_uint32(p, p_end, table->init_size); + if (table->flags == 1) { + read_leb_uint32(p, p_end, table->max_size); + bh_assert(table->init_size <= table->max_size); + } + + adjust_table_max_size(table->init_size, table->flags, &table->max_size); + + *p_buf = p; + return true; +} + +static bool +load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; +#if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); + uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT + / DEFAULT_NUM_BYTES_PER_PAGE; +#else + uint32 max_page_count; + bool is_memory64 = false; +#endif + + p_org = p; + read_leb_uint32(p, p_end, memory->flags); + bh_assert(p - p_org <= 1); + (void)p_org; + bh_assert(check_memory_flag(memory->flags)); + +#if WASM_ENABLE_APP_FRAMEWORK == 0 + is_memory64 = memory->flags & MEMORY64_FLAG; + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + + read_leb_uint32(p, p_end, memory->init_page_count); + bh_assert(memory->init_page_count <= max_page_count); + + if (memory->flags & 1) { + read_leb_uint32(p, p_end, memory->max_page_count); + bh_assert(memory->init_page_count <= memory->max_page_count); + bh_assert(memory->max_page_count <= max_page_count); + if (memory->max_page_count > max_page_count) + memory->max_page_count = max_page_count; + } + else { + /* Limit the maximum memory size to max_page_count */ + memory->max_page_count = max_page_count; + } + + memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + + *p_buf = p; + (void)check_memory_flag; + return true; +} + +static bool +load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end, *p_old; + uint32 import_count, name_len, type_index, i, u32, flags; + uint64 total_size; + WASMImport *import; + WASMImport *import_functions = NULL, *import_tables = NULL; + WASMImport *import_memories = NULL, *import_globals = NULL; + char *sub_module_name, *field_name; + uint8 u8, kind; + + read_leb_uint32(p, p_end, import_count); + + if (import_count) { + module->import_count = import_count; + total_size = sizeof(WASMImport) * (uint64)import_count; + if (!(module->imports = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + p_old = p; + + /* Scan firstly to get import count of each type */ + for (i = 0; i < import_count; i++) { + /* module name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + p += name_len; + + /* field name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + p += name_len; + + CHECK_BUF(p, p_end, 1); + /* 0x00/0x01/0x02/0x03 */ + kind = read_uint8(p); + + switch (kind) { + case IMPORT_KIND_FUNC: /* import function */ + read_leb_uint32(p, p_end, type_index); + module->import_function_count++; + break; + + case IMPORT_KIND_TABLE: /* import table */ + CHECK_BUF(p, p_end, 1); + /* 0x70 */ + u8 = read_uint8(p); + read_leb_uint32(p, p_end, flags); + read_leb_uint32(p, p_end, u32); + if (flags & 1) + read_leb_uint32(p, p_end, u32); + module->import_table_count++; +#if WASM_ENABLE_REF_TYPES == 0 + bh_assert(module->import_table_count <= 1); +#endif + break; + + case IMPORT_KIND_MEMORY: /* import memory */ + read_leb_uint32(p, p_end, flags); + read_leb_uint32(p, p_end, u32); + if (flags & 1) + read_leb_uint32(p, p_end, u32); + module->import_memory_count++; + bh_assert(module->import_memory_count <= 1); + break; + + case IMPORT_KIND_GLOBAL: /* import global */ + CHECK_BUF(p, p_end, 2); + p += 2; + module->import_global_count++; + break; + + default: + bh_assert(0); + break; + } + } + + if (module->import_function_count) + import_functions = module->import_functions = module->imports; + if (module->import_table_count) + import_tables = module->import_tables = + module->imports + module->import_function_count; + if (module->import_memory_count) + import_memories = module->import_memories = + module->imports + module->import_function_count + + module->import_table_count; + if (module->import_global_count) + import_globals = module->import_globals = + module->imports + module->import_function_count + + module->import_table_count + module->import_memory_count; + + p = p_old; + + /* Scan again to resolve the data */ + for (i = 0; i < import_count; i++) { + WASMModule *sub_module = NULL; + + /* load module name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + if (!(sub_module_name = wasm_const_str_list_insert( + p, name_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + p += name_len; + + /* load field name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + if (!(field_name = wasm_const_str_list_insert( + p, name_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + p += name_len; + + CHECK_BUF(p, p_end, 1); + /* 0x00/0x01/0x02/0x03 */ + kind = read_uint8(p); + + LOG_DEBUG("import #%d: (%s, %s), kind: %d", i, sub_module_name, + field_name, kind); + switch (kind) { + case IMPORT_KIND_FUNC: /* import function */ + bh_assert(import_functions); + import = import_functions++; + if (!load_function_import( + &p, p_end, module, sub_module_name, field_name, + &import->u.function, error_buf, error_buf_size)) { + return false; + } + break; + + case IMPORT_KIND_TABLE: /* import table */ + bh_assert(import_tables); + import = import_tables++; + if (!load_table_import(&p, p_end, module, sub_module_name, + field_name, &import->u.table, + error_buf, error_buf_size)) { + LOG_DEBUG("can not import such a table (%s,%s)", + sub_module_name, field_name); + return false; + } + break; + + case IMPORT_KIND_MEMORY: /* import memory */ + bh_assert(import_memories); + import = import_memories++; + if (!load_memory_import(&p, p_end, module, sub_module_name, + field_name, &import->u.memory, + error_buf, error_buf_size)) { + return false; + } + break; + + case IMPORT_KIND_GLOBAL: /* import global */ + bh_assert(import_globals); + import = import_globals++; + if (!load_global_import(&p, p_end, module, sub_module_name, + field_name, &import->u.global, + error_buf, error_buf_size)) { + return false; + } + break; + + default: + bh_assert(0); + import = NULL; + break; + } + import->kind = kind; + import->u.names.module_name = sub_module_name; + import->u.names.field_name = field_name; + (void)sub_module; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + import = module->import_functions; + for (i = 0; i < module->import_function_count; i++, import++) { + if (!strcmp(import->u.names.module_name, "wasi_unstable") + || !strcmp(import->u.names.module_name, + "wasi_snapshot_preview1")) { + module->import_wasi_api = true; + break; + } + } +#endif + } + + bh_assert(p == p_end); + + LOG_VERBOSE("Load import section success.\n"); + (void)u8; + (void)u32; + (void)type_index; + return true; +} + +static bool +init_function_local_offsets(WASMFunction *func, char *error_buf, + uint32 error_buf_size) +{ + WASMFuncType *param_type = func->func_type; + uint32 param_count = param_type->param_count; + uint8 *param_types = param_type->types; + uint32 local_count = func->local_count; + uint8 *local_types = func->local_types; + uint32 i, local_offset = 0; + uint64 total_size = sizeof(uint16) * ((uint64)param_count + local_count); + + if (total_size > 0 + && !(func->local_offsets = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < param_count; i++) { + func->local_offsets[i] = (uint16)local_offset; + local_offset += wasm_value_type_cell_num(param_types[i]); + } + + for (i = 0; i < local_count; i++) { + func->local_offsets[param_count + i] = (uint16)local_offset; + local_offset += wasm_value_type_cell_num(local_types[i]); + } + + bh_assert(local_offset == func->param_cell_num + func->local_cell_num); + return true; +} + +static bool +load_function_section(const uint8 *buf, const uint8 *buf_end, + const uint8 *buf_code, const uint8 *buf_code_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + const uint8 *p_code = buf_code, *p_code_end, *p_code_save; + uint32 func_count; + uint64 total_size; + uint32 code_count = 0, code_size, type_index, i, j, k, local_type_index; + uint32 local_count, local_set_count, sub_local_count, local_cell_num; + uint8 type; + WASMFunction *func; + + read_leb_uint32(p, p_end, func_count); + + if (buf_code) + read_leb_uint32(p_code, buf_code_end, code_count); + + bh_assert(func_count == code_count); + + if (func_count) { + module->function_count = func_count; + total_size = sizeof(WASMFunction *) * (uint64)func_count; + if (!(module->functions = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < func_count; i++) { + /* Resolve function type */ + read_leb_uint32(p, p_end, type_index); + bh_assert(type_index < module->type_count); + +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + type_index = wasm_get_smallest_type_idx( + module->types, module->type_count, type_index); +#endif + + read_leb_uint32(p_code, buf_code_end, code_size); + bh_assert(code_size > 0 && p_code + code_size <= buf_code_end); + + /* Resolve local set count */ + p_code_end = p_code + code_size; + local_count = 0; + read_leb_uint32(p_code, buf_code_end, local_set_count); + p_code_save = p_code; + + /* Calculate total local count */ + for (j = 0; j < local_set_count; j++) { + read_leb_uint32(p_code, buf_code_end, sub_local_count); + bh_assert(sub_local_count <= UINT32_MAX - local_count); + + CHECK_BUF(p_code, buf_code_end, 1); + /* 0x7F/0x7E/0x7D/0x7C */ + type = read_uint8(p_code); + local_count += sub_local_count; + } + + /* Alloc memory, layout: function structure + local types */ + code_size = (uint32)(p_code_end - p_code); + + total_size = sizeof(WASMFunction) + (uint64)local_count; + if (!(func = module->functions[i] = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Set function type, local count, code size and code body */ + func->func_type = module->types[type_index]; + func->local_count = local_count; + if (local_count > 0) + func->local_types = (uint8 *)func + sizeof(WASMFunction); + func->code_size = code_size; + /* + * we shall make a copy of code body [p_code, p_code + code_size] + * when we are worrying about inappropriate releasing behaviour. + * all code bodies are actually in a buffer which user allocates in + * his embedding environment and we don't have power on them. + * it will be like: + * code_body_cp = malloc(code_size); + * memcpy(code_body_cp, p_code, code_size); + * func->code = code_body_cp; + */ + func->code = (uint8 *)p_code; + + /* Load each local type */ + p_code = p_code_save; + local_type_index = 0; + for (j = 0; j < local_set_count; j++) { + read_leb_uint32(p_code, buf_code_end, sub_local_count); + /* Note: sub_local_count is allowed to be 0 */ + bh_assert(local_type_index <= UINT32_MAX - sub_local_count + && local_type_index + sub_local_count <= local_count); + + CHECK_BUF(p_code, buf_code_end, 1); + /* 0x7F/0x7E/0x7D/0x7C */ + type = read_uint8(p_code); + bh_assert(is_value_type(type)); + for (k = 0; k < sub_local_count; k++) { + func->local_types[local_type_index++] = type; + } + } + + func->param_cell_num = func->func_type->param_cell_num; + func->ret_cell_num = func->func_type->ret_cell_num; + local_cell_num = + wasm_get_cell_num(func->local_types, func->local_count); + bh_assert(local_cell_num <= UINT16_MAX); + + func->local_cell_num = (uint16)local_cell_num; + + if (!init_function_local_offsets(func, error_buf, error_buf_size)) + return false; + + p_code = p_code_end; + } + } + + bh_assert(p == p_end); + LOG_VERBOSE("Load function section success.\n"); + (void)code_count; + return true; +} + +static bool +load_table_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 table_count, i; + uint64 total_size; + WASMTable *table; + + read_leb_uint32(p, p_end, table_count); +#if WASM_ENABLE_REF_TYPES == 0 + bh_assert(module->import_table_count + table_count <= 1); +#endif + + if (table_count) { + module->table_count = table_count; + total_size = sizeof(WASMTable) * (uint64)table_count; + if (!(module->tables = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* load each table */ + table = module->tables; + for (i = 0; i < table_count; i++, table++) + if (!load_table(&p, p_end, table, error_buf, error_buf_size)) + return false; + } + + bh_assert(p == p_end); + LOG_VERBOSE("Load table section success.\n"); + return true; +} + +static bool +load_memory_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 memory_count, i; + uint64 total_size; + WASMMemory *memory; + + read_leb_uint32(p, p_end, memory_count); + bh_assert(module->import_memory_count + memory_count <= 1); + + if (memory_count) { + module->memory_count = memory_count; + total_size = sizeof(WASMMemory) * (uint64)memory_count; + if (!(module->memories = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* load each memory */ + memory = module->memories; + for (i = 0; i < memory_count; i++, memory++) + if (!load_memory(&p, p_end, memory, error_buf, error_buf_size)) + return false; + } + + bh_assert(p == p_end); + LOG_VERBOSE("Load memory section success.\n"); + return true; +} + +static bool +load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 global_count, i; + uint64 total_size; + WASMGlobal *global; + uint8 mutable; + + read_leb_uint32(p, p_end, global_count); + + module->global_count = 0; + if (global_count) { + total_size = sizeof(WASMGlobal) * (uint64)global_count; + if (!(module->globals = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + global = module->globals; + + for (i = 0; i < global_count; i++, global++) { + CHECK_BUF(p, p_end, 2); + global->type = read_uint8(p); + mutable = read_uint8(p); + bh_assert(mutable < 2); + global->is_mutable = mutable ? true : false; + + /* initialize expression */ + if (!load_init_expr(module, &p, p_end, &(global->init_expr), + global->type, error_buf, error_buf_size)) + return false; + + if (INIT_EXPR_TYPE_GET_GLOBAL == global->init_expr.init_expr_type) { + /** + * Currently, constant expressions occurring as initializers + * of globals are further constrained in that contained + * global.get instructions are + * only allowed to refer to imported globals. + */ + uint32 target_global_index = global->init_expr.u.global_index; + bh_assert(target_global_index < module->import_global_count); + (void)target_global_index; + } + else if (INIT_EXPR_TYPE_FUNCREF_CONST + == global->init_expr.init_expr_type) { + bh_assert(global->init_expr.u.ref_index + < module->import_function_count + + module->function_count); + } + + module->global_count++; + } + bh_assert(module->global_count == global_count); + } + + bh_assert(p == p_end); + LOG_VERBOSE("Load global section success.\n"); + return true; +} + +static bool +load_export_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 export_count, i, j, index; + uint64 total_size; + uint32 str_len; + WASMExport *export; + const char *name; + + read_leb_uint32(p, p_end, export_count); + + if (export_count) { + module->export_count = export_count; + total_size = sizeof(WASMExport) * (uint64)export_count; + if (!(module->exports = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + export = module->exports; + for (i = 0; i < export_count; i++, export ++) { + read_leb_uint32(p, p_end, str_len); + CHECK_BUF(p, p_end, str_len); + + for (j = 0; j < i; j++) { + name = module->exports[j].name; + bh_assert(!(strlen(name) == str_len + && memcmp(name, p, str_len) == 0)); + } + + if (!(export->name = wasm_const_str_list_insert( + p, str_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + + p += str_len; + CHECK_BUF(p, p_end, 1); + export->kind = read_uint8(p); + read_leb_uint32(p, p_end, index); + export->index = index; + + switch (export->kind) { + /* function index */ + case EXPORT_KIND_FUNC: + bh_assert(index < module->function_count + + module->import_function_count); + break; + /* table index */ + case EXPORT_KIND_TABLE: + bh_assert(index < module->table_count + + module->import_table_count); + break; + /* memory index */ + case EXPORT_KIND_MEMORY: + bh_assert(index < module->memory_count + + module->import_memory_count); + break; + /* global index */ + case EXPORT_KIND_GLOBAL: + bh_assert(index < module->global_count + + module->import_global_count); + break; + default: + bh_assert(0); + break; + } + } + } + + bh_assert(p == p_end); + LOG_VERBOSE("Load export section success.\n"); + (void)name; + return true; +} + +static bool +check_table_index(const WASMModule *module, uint32 table_index, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_REF_TYPES == 0 + if (table_index != 0) { + return false; + } +#endif + + if (table_index >= module->import_table_count + module->table_count) { + return false; + } + return true; +} + +static bool +load_table_index(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, + uint32 *p_table_index, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 table_index; + + read_leb_uint32(p, p_end, table_index); + if (!check_table_index(module, table_index, error_buf, error_buf_size)) { + return false; + } + + *p_table_index = table_index; + *p_buf = p; + return true; +} + +#if WASM_ENABLE_REF_TYPES != 0 +static bool +load_elem_type(const uint8 **p_buf, const uint8 *buf_end, uint32 *p_elem_type, + bool elemkind_zero, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 elem_type; + + CHECK_BUF(p, p_end, 1); + elem_type = read_uint8(p); + if ((elemkind_zero && elem_type != 0) + || (!elemkind_zero && elem_type != VALUE_TYPE_FUNCREF + && elem_type != VALUE_TYPE_EXTERNREF)) { + set_error_buf(error_buf, error_buf_size, "invalid reference type"); + return false; + } + + if (elemkind_zero) + *p_elem_type = VALUE_TYPE_FUNCREF; + else + *p_elem_type = elem_type; + *p_buf = p; + + (void)p_end; + return true; +} +#endif /* WASM_ENABLE_REF_TYPES != 0*/ + +static bool +load_func_index_vec(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, WASMTableSeg *table_segment, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 function_count, function_index = 0, i; + uint64 total_size; + + read_leb_uint32(p, p_end, function_count); + table_segment->value_count = function_count; + total_size = sizeof(InitializerExpression) * (uint64)function_count; + if (total_size > 0 + && !(table_segment->init_values = + (InitializerExpression *)loader_malloc(total_size, error_buf, + error_buf_size))) { + return false; + } + + for (i = 0; i < function_count; i++) { + InitializerExpression *init_expr = &table_segment->init_values[i]; + + read_leb_uint32(p, p_end, function_index); + if (!check_function_index(module, function_index, error_buf, + error_buf_size)) { + return false; + } + + init_expr->init_expr_type = INIT_EXPR_TYPE_FUNCREF_CONST; + init_expr->u.ref_index = function_index; + } + + *p_buf = p; + return true; +} + +#if WASM_ENABLE_REF_TYPES != 0 +static bool +load_init_expr_vec(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, WASMTableSeg *table_segment, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 ref_count, i; + uint64 total_size; + + read_leb_uint32(p, p_end, ref_count); + table_segment->value_count = ref_count; + total_size = sizeof(InitializerExpression) * (uint64)ref_count; + if (total_size > 0 + && !(table_segment->init_values = + (InitializerExpression *)loader_malloc(total_size, error_buf, + error_buf_size))) { + return false; + } + + for (i = 0; i < ref_count; i++) { + InitializerExpression *init_expr = &table_segment->init_values[i]; + + if (!load_init_expr(module, &p, p_end, init_expr, + table_segment->elem_type, error_buf, + error_buf_size)) + return false; + + bh_assert( + (init_expr->init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) + || (init_expr->init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST) + || (init_expr->init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST)); + } + + *p_buf = p; + return true; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 */ + +static bool +load_table_segment_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 table_segment_count, i, table_index, function_count; + uint64 total_size; + WASMTableSeg *table_segment; + + read_leb_uint32(p, p_end, table_segment_count); + + if (table_segment_count) { + module->table_seg_count = table_segment_count; + total_size = sizeof(WASMTableSeg) * (uint64)table_segment_count; + if (!(module->table_segments = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + table_segment = module->table_segments; + for (i = 0; i < table_segment_count; i++, table_segment++) { + bh_assert(p < p_end); + +#if WASM_ENABLE_REF_TYPES != 0 + read_leb_uint32(p, p_end, table_segment->mode); + /* last three bits */ + table_segment->mode = table_segment->mode & 0x07; + switch (table_segment->mode) { + /* elemkind/elemtype + active */ + case 0: + case 4: + table_segment->elem_type = VALUE_TYPE_FUNCREF; + table_segment->table_index = 0; + + if (!check_table_index(module, table_segment->table_index, + error_buf, error_buf_size)) + return false; + + if (!load_init_expr( + module, &p, p_end, &table_segment->base_offset, + VALUE_TYPE_I32, error_buf, error_buf_size)) + return false; + + if (table_segment->mode == 0) { + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + else { + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + break; + /* elemkind + passive/declarative */ + case 1: + case 3: + if (!load_elem_type(&p, p_end, &table_segment->elem_type, + true, error_buf, error_buf_size)) + return false; + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; + break; + /* elemkind/elemtype + table_idx + active */ + case 2: + case 6: + if (!load_table_index(&p, p_end, module, + &table_segment->table_index, + error_buf, error_buf_size)) + return false; + if (!load_init_expr( + module, &p, p_end, &table_segment->base_offset, + VALUE_TYPE_I32, error_buf, error_buf_size)) + return false; + if (!load_elem_type(&p, p_end, &table_segment->elem_type, + table_segment->mode == 2 ? true : false, + error_buf, error_buf_size)) + return false; + if (table_segment->mode == 2) { + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + else { + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + break; + case 5: + case 7: + if (!load_elem_type(&p, p_end, &table_segment->elem_type, + false, error_buf, error_buf_size)) + return false; + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; + break; + default: + return false; + } +#else + /* + * like: 00 41 05 0b 04 00 01 00 01 + * for: (elem 0 (offset (i32.const 5)) $f1 $f2 $f1 $f2) + */ + if (!load_table_index(&p, p_end, module, + &table_segment->table_index, error_buf, + error_buf_size)) + return false; + if (!load_init_expr(module, &p, p_end, &table_segment->base_offset, + VALUE_TYPE_I32, error_buf, error_buf_size)) + return false; + if (!load_func_index_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; +#endif /* WASM_ENABLE_REF_TYPES != 0 */ + } + } + + (void)table_index; + (void)function_count; + bh_assert(p == p_end); + LOG_VERBOSE("Load table segment section success.\n"); + return true; +} + +static bool +load_data_segment_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 data_seg_count, i, mem_index, data_seg_len; + uint64 total_size; + WASMDataSeg *dataseg; + InitializerExpression init_expr; +#if WASM_ENABLE_BULK_MEMORY != 0 + bool is_passive = false; + uint32 mem_flag; +#endif + uint8 mem_offset_type = VALUE_TYPE_I32; + + read_leb_uint32(p, p_end, data_seg_count); + +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_assert(module->data_seg_count1 == 0 + || data_seg_count == module->data_seg_count1); +#endif + + if (data_seg_count) { + module->data_seg_count = data_seg_count; + total_size = sizeof(WASMDataSeg *) * (uint64)data_seg_count; + if (!(module->data_segments = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < data_seg_count; i++) { + read_leb_uint32(p, p_end, mem_index); +#if WASM_ENABLE_BULK_MEMORY != 0 + is_passive = false; + mem_flag = mem_index & 0x03; + switch (mem_flag) { + case 0x01: + is_passive = true; + break; + case 0x00: + /* no memory index, treat index as 0 */ + mem_index = 0; + goto check_mem_index; + case 0x02: + /* read following memory index */ + read_leb_uint32(p, p_end, mem_index); + check_mem_index: + bh_assert(mem_index < module->import_memory_count + + module->memory_count); + break; + case 0x03: + default: + bh_assert(0); + break; + } +#else + bh_assert(mem_index + < module->import_memory_count + module->memory_count); +#endif /* WASM_ENABLE_BULK_MEMORY */ + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif /* WASM_ENABLE_BULK_MEMORY */ + { +#if WASM_ENABLE_MEMORY64 != 0 + /* This memory_flag is from memory instead of data segment */ + uint8 memory_flag; + if (module->import_memory_count > 0) { + memory_flag = + module->import_memories[mem_index].u.memory.flags; + } + else { + memory_flag = + module + ->memories[mem_index - module->import_memory_count] + .flags; + } + mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif /* WASM_ENABLE_MEMORY64 */ + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif + if (!load_init_expr(module, &p, p_end, &init_expr, + mem_offset_type, error_buf, error_buf_size)) + return false; + + read_leb_uint32(p, p_end, data_seg_len); + + if (!(dataseg = module->data_segments[i] = loader_malloc( + sizeof(WASMDataSeg), error_buf, error_buf_size))) { + return false; + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + dataseg->is_passive = is_passive; + if (!is_passive) +#endif + { + bh_memcpy_s(&dataseg->base_offset, + sizeof(InitializerExpression), &init_expr, + sizeof(InitializerExpression)); + + dataseg->memory_index = mem_index; + } + + dataseg->data_length = data_seg_len; + CHECK_BUF(p, p_end, data_seg_len); + dataseg->data = (uint8 *)p; + p += data_seg_len; + } + } + + bh_assert(p == p_end); + LOG_VERBOSE("Load data segment section success.\n"); + return true; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +static bool +load_datacount_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 data_seg_count1 = 0; + + read_leb_uint32(p, p_end, data_seg_count1); + module->data_seg_count1 = data_seg_count1; + + bh_assert(p == p_end); + LOG_VERBOSE("Load datacount section success.\n"); + return true; +} +#endif + +static bool +load_code_section(const uint8 *buf, const uint8 *buf_end, const uint8 *buf_func, + const uint8 *buf_func_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + const uint8 *p_func = buf_func; + uint32 func_count = 0, code_count; + + /* code has been loaded in function section, so pass it here, just check + * whether function and code section have inconsistent lengths */ + read_leb_uint32(p, p_end, code_count); + + if (buf_func) + read_leb_uint32(p_func, buf_func_end, func_count); + + bh_assert(func_count == code_count); + LOG_VERBOSE("Load code segment section success.\n"); + (void)code_count; + (void)func_count; + return true; +} + +static bool +load_start_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + WASMFuncType *type; + uint32 start_function; + + read_leb_uint32(p, p_end, start_function); + + bh_assert(start_function + < module->function_count + module->import_function_count); + + if (start_function < module->import_function_count) + type = module->import_functions[start_function].u.function.func_type; + else + type = module->functions[start_function - module->import_function_count] + ->func_type; + + bh_assert(type->param_count == 0 && type->result_count == 0); + + module->start_function = start_function; + + bh_assert(p == p_end); + LOG_VERBOSE("Load start section success.\n"); + (void)type; + return true; +} + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 +static bool +handle_name_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 name_type, subsection_size; + uint32 previous_name_type = 0; + uint32 num_func_name; + uint32 func_index; + uint32 previous_func_index = ~0U; + uint32 func_name_len; + uint32 name_index; + int i = 0; + + bh_assert(p < p_end); + + while (p < p_end) { + read_leb_uint32(p, p_end, name_type); + if (i != 0) { + bh_assert(name_type > previous_name_type); + } + previous_name_type = name_type; + read_leb_uint32(p, p_end, subsection_size); + CHECK_BUF(p, p_end, subsection_size); + switch (name_type) { + case SUB_SECTION_TYPE_FUNC: + if (subsection_size) { + read_leb_uint32(p, p_end, num_func_name); + for (name_index = 0; name_index < num_func_name; + name_index++) { + read_leb_uint32(p, p_end, func_index); + bh_assert(func_index > previous_func_index); + previous_func_index = func_index; + read_leb_uint32(p, p_end, func_name_len); + CHECK_BUF(p, p_end, func_name_len); + /* Skip the import functions */ + if (func_index >= module->import_function_count) { + func_index -= module->import_function_count; + bh_assert(func_index < module->function_count); + if (!(module->functions[func_index]->field_name = + wasm_const_str_list_insert( + p, func_name_len, module, + is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + } + p += func_name_len; + } + } + break; + case SUB_SECTION_TYPE_MODULE: /* TODO: Parse for module subsection + */ + case SUB_SECTION_TYPE_LOCAL: /* TODO: Parse for local subsection */ + default: + p = p + subsection_size; + break; + } + i++; + } + + (void)previous_name_type; + (void)previous_func_index; + return true; +} +#endif + +static bool +load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 name_len; + + bh_assert(p < p_end); + + read_leb_uint32(p, p_end, name_len); + + bh_assert(name_len > 0 && p + name_len <= p_end); + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + if (name_len == 4 && memcmp(p, "name", 4) == 0) { + p += name_len; + handle_name_section(p, p_end, module, is_load_from_file_buf, error_buf, + error_buf_size); + } +#endif + LOG_VERBOSE("Load custom section success.\n"); + (void)name_len; + return true; +} + +static void +calculate_global_data_offset(WASMModule *module) +{ + uint32 i, data_offset; + + data_offset = 0; + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *import_global = + &((module->import_globals + i)->u.global); +#if WASM_ENABLE_FAST_JIT != 0 + import_global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(import_global->type); + } + + for (i = 0; i < module->global_count; i++) { + WASMGlobal *global = module->globals + i; +#if WASM_ENABLE_FAST_JIT != 0 + global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(global->type); + } + + module->global_data_size = data_offset; +} + +#if WASM_ENABLE_FAST_JIT != 0 +static bool +init_fast_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_LAZY_JIT != 0 + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); +#endif + uint32 i; + + if (!module->function_count) + return true; + + if (!(module->fast_jit_func_ptrs = + loader_malloc(sizeof(void *) * module->function_count, error_buf, + error_buf_size))) { + return false; + } + +#if WASM_ENABLE_LAZY_JIT != 0 + for (i = 0; i < module->function_count; i++) { + module->fast_jit_func_ptrs[i] = + jit_globals->compile_fast_jit_and_then_call; + } +#endif + + for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) { + if (os_mutex_init(&module->fast_jit_thread_locks[i]) != 0) { + set_error_buf(error_buf, error_buf_size, + "init fast jit thread lock failed"); + return false; + } + module->fast_jit_thread_locks_inited[i] = true; + } + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 */ + +#if WASM_ENABLE_JIT != 0 +static bool +init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + LLVMJITOptions llvm_jit_options = wasm_runtime_get_llvm_jit_options(); + AOTCompOption option = { 0 }; + char *aot_last_error; + uint64 size; + bool gc_enabled = false; /* GC hasn't been enabled in mini loader */ + + if (module->function_count == 0) + return true; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (os_mutex_init(&module->tierup_wait_lock) != 0) { + set_error_buf(error_buf, error_buf_size, "init jit tierup lock failed"); + return false; + } + if (os_cond_init(&module->tierup_wait_cond) != 0) { + set_error_buf(error_buf, error_buf_size, "init jit tierup cond failed"); + os_mutex_destroy(&module->tierup_wait_lock); + return false; + } + module->tierup_wait_lock_inited = true; +#endif + + size = sizeof(void *) * (uint64)module->function_count + + sizeof(bool) * (uint64)module->function_count; + if (!(module->func_ptrs = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + module->func_ptrs_compiled = + (bool *)((uint8 *)module->func_ptrs + + sizeof(void *) * module->function_count); + + module->comp_data = aot_create_comp_data(module, NULL, gc_enabled); + if (!module->comp_data) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + option.is_jit_mode = true; + option.opt_level = llvm_jit_options->opt_level; + option.size_level = llvm_jit_options->size_level; + option.segue_flags = llvm_jit_options->segue_flags; + option.quick_invoke_c_api_import = + llvm_jit_options->quick_invoke_c_api_import; + +#if WASM_ENABLE_BULK_MEMORY != 0 + option.enable_bulk_memory = true; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif +#if WASM_ENABLE_TAIL_CALL != 0 + option.enable_tail_call = true; +#endif +#if WASM_ENABLE_SIMD != 0 + option.enable_simd = true; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + option.enable_ref_types = true; +#endif + option.enable_aux_stack_check = true; +#if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ + || WASM_ENABLE_AOT_STACK_FRAME != 0 + option.enable_aux_stack_frame = true; +#endif +#if WASM_ENABLE_PERF_PROFILING != 0 + option.enable_perf_profiling = true; +#endif +#if WASM_ENABLE_MEMORY_PROFILING != 0 + option.enable_memory_profiling = true; + option.enable_stack_estimation = true; +#endif + + module->comp_ctx = aot_create_comp_context(module->comp_data, &option); + if (!module->comp_ctx) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + return true; +} + +static bool +init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + char *aot_last_error; + uint32 i; + + if (module->function_count == 0) + return true; + + if (!aot_compile_wasm(module->comp_ctx)) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (module->orcjit_stop_compiling) + return false; +#endif + + bh_print_time("Begin to lookup llvm jit functions"); + + for (i = 0; i < module->function_count; i++) { + LLVMOrcJITTargetAddress func_addr = 0; + LLVMErrorRef error; + char func_name[48]; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + error = LLVMOrcLLLazyJITLookup(module->comp_ctx->orc_jit, &func_addr, + func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + char buf[96]; + snprintf(buf, sizeof(buf), + "failed to compile llvm jit function: %s", err_msg); + set_error_buf(error_buf, error_buf_size, buf); + LLVMDisposeErrorMessage(err_msg); + return false; + } + + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + module->functions[i]->llvm_jit_func_ptr = (void *)func_addr; + + if (module->orcjit_stop_compiling) + return false; +#endif + } + + bh_print_time("End lookup llvm jit functions"); + + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 +static void * +init_llvm_jit_functions_stage2_callback(void *arg) +{ + WASMModule *module = (WASMModule *)arg; + char error_buf[128]; + uint32 error_buf_size = (uint32)sizeof(error_buf); + + if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) { + module->orcjit_stop_compiling = true; + return NULL; + } + + os_mutex_lock(&module->tierup_wait_lock); + module->llvm_jit_inited = true; + os_cond_broadcast(&module->tierup_wait_cond); + os_mutex_unlock(&module->tierup_wait_lock); + + return NULL; +} +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 +/* The callback function to compile jit functions */ +static void * +orcjit_thread_callback(void *arg) +{ + OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg; +#if WASM_ENABLE_JIT != 0 + AOTCompContext *comp_ctx = thread_arg->comp_ctx; +#endif + WASMModule *module = thread_arg->module; + uint32 group_idx = thread_arg->group_idx; + uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM; + uint32 func_count = module->function_count; + uint32 i; + +#if WASM_ENABLE_FAST_JIT != 0 + /* Compile fast jit funcitons of this group */ + for (i = group_idx; i < func_count; i += group_stride) { + if (!jit_compiler_compile(module, i + module->import_function_count)) { + LOG_ERROR("failed to compile fast jit function %u\n", i); + break; + } + + if (module->orcjit_stop_compiling) { + return NULL; + } + } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + os_mutex_lock(&module->tierup_wait_lock); + module->fast_jit_ready_groups++; + os_mutex_unlock(&module->tierup_wait_lock); +#endif +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + /* For JIT tier-up, set each llvm jit func to call_to_fast_jit */ + for (i = group_idx; i < func_count; + i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) { + uint32 j; + + for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) { + if (i + j * group_stride < func_count) { + if (!jit_compiler_set_call_to_fast_jit( + module, + i + j * group_stride + module->import_function_count)) { + LOG_ERROR( + "failed to compile call_to_fast_jit for func %u\n", + i + j * group_stride + module->import_function_count); + module->orcjit_stop_compiling = true; + return NULL; + } + } + if (module->orcjit_stop_compiling) { + return NULL; + } + } + } + + /* Wait until init_llvm_jit_functions_stage2 finishes and all + fast jit functions are compiled */ + os_mutex_lock(&module->tierup_wait_lock); + while (!(module->llvm_jit_inited && module->enable_llvm_jit_compilation + && module->fast_jit_ready_groups >= group_stride)) { + os_cond_reltimedwait(&module->tierup_wait_cond, + &module->tierup_wait_lock, 10000); + if (module->orcjit_stop_compiling) { + /* init_llvm_jit_functions_stage2 failed */ + os_mutex_unlock(&module->tierup_wait_lock); + return NULL; + } + } + os_mutex_unlock(&module->tierup_wait_lock); +#endif + +#if WASM_ENABLE_JIT != 0 + /* Compile llvm jit functions of this group */ + for (i = group_idx; i < func_count; + i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) { + LLVMOrcJITTargetAddress func_addr = 0; + LLVMErrorRef error; + char func_name[48]; + typedef void (*F)(void); + union { + F f; + void *v; + } u; + uint32 j; + + snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i, + "_wrapper"); + LOG_DEBUG("compile llvm jit func %s", func_name); + error = + LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + LOG_ERROR("failed to compile llvm jit function %u: %s", i, err_msg); + LLVMDisposeErrorMessage(err_msg); + break; + } + + /* Call the jit wrapper function to trigger its compilation, so as + to compile the actual jit functions, since we add the latter to + function list in the PartitionFunction callback */ + u.v = (void *)func_addr; + u.f(); + + for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) { + if (i + j * group_stride < func_count) { + module->func_ptrs_compiled[i + j * group_stride] = true; +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, + i + j * group_stride); + error = LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, + func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + LOG_ERROR("failed to compile llvm jit function %u: %s", i, + err_msg); + LLVMDisposeErrorMessage(err_msg); + /* Ignore current llvm jit func, as its func ptr is + previous set to call_to_fast_jit, which also works */ + continue; + } + + jit_compiler_set_llvm_jit_func_ptr( + module, + i + j * group_stride + module->import_function_count, + (void *)func_addr); + + /* Try to switch to call this llvm jit funtion instead of + fast jit function from fast jit jitted code */ + jit_compiler_set_call_to_llvm_jit( + module, + i + j * group_stride + module->import_function_count); +#endif + } + } + + if (module->orcjit_stop_compiling) { + break; + } + } +#endif + + return NULL; +} + +static void +orcjit_stop_compile_threads(WASMModule *module) +{ + uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args) + / sizeof(OrcJitThreadArg)); + + module->orcjit_stop_compiling = true; + for (i = 0; i < thread_num; i++) { + if (module->orcjit_threads[i]) + os_thread_join(module->orcjit_threads[i], NULL); + } +} + +static bool +compile_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + uint32 thread_num = + (uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg)); + uint32 i, j; + + bh_print_time("Begin to compile jit functions"); + + /* Create threads to compile the jit functions */ + for (i = 0; i < thread_num && i < module->function_count; i++) { +#if WASM_ENABLE_JIT != 0 + module->orcjit_thread_args[i].comp_ctx = module->comp_ctx; +#endif + module->orcjit_thread_args[i].module = module; + module->orcjit_thread_args[i].group_idx = i; + + if (os_thread_create(&module->orcjit_threads[i], orcjit_thread_callback, + (void *)&module->orcjit_thread_args[i], + APP_THREAD_STACK_SIZE_DEFAULT) + != 0) { + set_error_buf(error_buf, error_buf_size, + "create orcjit compile thread failed"); + /* Terminate the threads created */ + module->orcjit_stop_compiling = true; + for (j = 0; j < i; j++) { + os_thread_join(module->orcjit_threads[j], NULL); + } + return false; + } + } + +#if WASM_ENABLE_LAZY_JIT == 0 + /* Wait until all jit functions are compiled for eager mode */ + for (i = 0; i < thread_num; i++) { + if (module->orcjit_threads[i]) + os_thread_join(module->orcjit_threads[i], NULL); + } + +#if WASM_ENABLE_FAST_JIT != 0 + /* Ensure all the fast-jit functions are compiled */ + for (i = 0; i < module->function_count; i++) { + if (!jit_compiler_is_compiled(module, + i + module->import_function_count)) { + set_error_buf(error_buf, error_buf_size, + "failed to compile fast jit function"); + return false; + } + } +#endif + +#if WASM_ENABLE_JIT != 0 + /* Ensure all the llvm-jit functions are compiled */ + for (i = 0; i < module->function_count; i++) { + if (!module->func_ptrs_compiled[i]) { + set_error_buf(error_buf, error_buf_size, + "failed to compile llvm jit function"); + return false; + } + } +#endif +#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */ + + bh_print_time("End compile jit functions"); + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_REF_TYPES != 0 +static bool +get_table_elem_type(const WASMModule *module, uint32 table_idx, + uint8 *p_elem_type, char *error_buf, uint32 error_buf_size) +{ + if (!check_table_index(module, table_idx, error_buf, error_buf_size)) { + return false; + } + + if (p_elem_type) { + if (table_idx < module->import_table_count) + *p_elem_type = module->import_tables[table_idx].u.table.elem_type; + else + *p_elem_type = + module->tables[module->import_table_count + table_idx] + .elem_type; + } + return true; +} + +static bool +get_table_seg_elem_type(const WASMModule *module, uint32 table_seg_idx, + uint8 *p_elem_type, char *error_buf, + uint32 error_buf_size) +{ + if (table_seg_idx >= module->table_seg_count) { + return false; + } + + if (p_elem_type) { + *p_elem_type = module->table_segments[table_seg_idx].elem_type; + } + return true; +} +#endif + +static bool +wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, + uint32 cur_func_idx, char *error_buf, + uint32 error_buf_size); + +#if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 +void ** +wasm_interp_get_handle_table(); + +static void **handle_table; +#endif + +static bool +load_from_sections(WASMModule *module, WASMSection *sections, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + WASMExport *export; + WASMSection *section = sections; + const uint8 *buf, *buf_end, *buf_code = NULL, *buf_code_end = NULL, + *buf_func = NULL, *buf_func_end = NULL; + WASMGlobal *aux_data_end_global = NULL, *aux_heap_base_global = NULL; + WASMGlobal *aux_stack_top_global = NULL, *global; + uint64 aux_data_end = (uint64)-1LL, aux_heap_base = (uint64)-1LL, + aux_stack_top = (uint64)-1LL; + uint32 global_index, func_index, i; + uint32 aux_data_end_global_index = (uint32)-1; + uint32 aux_heap_base_global_index = (uint32)-1; + WASMFuncType *func_type; + uint8 malloc_free_io_type = VALUE_TYPE_I32; + + /* Find code and function sections if have */ + while (section) { + if (section->section_type == SECTION_TYPE_CODE) { + buf_code = section->section_body; + buf_code_end = buf_code + section->section_body_size; + } + else if (section->section_type == SECTION_TYPE_FUNC) { + buf_func = section->section_body; + buf_func_end = buf_func + section->section_body_size; + } + section = section->next; + } + + section = sections; + while (section) { + buf = section->section_body; + buf_end = buf + section->section_body_size; + LOG_DEBUG("load section, type: %d", section->section_type); + switch (section->section_type) { + case SECTION_TYPE_USER: + /* unsupported user section, ignore it. */ + if (!load_user_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_TYPE: + if (!load_type_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_IMPORT: + if (!load_import_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_FUNC: + if (!load_function_section(buf, buf_end, buf_code, buf_code_end, + module, error_buf, error_buf_size)) + return false; + break; + case SECTION_TYPE_TABLE: + if (!load_table_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_MEMORY: + if (!load_memory_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_GLOBAL: + if (!load_global_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_EXPORT: + if (!load_export_section(buf, buf_end, module, + is_load_from_file_buf, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_START: + if (!load_start_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_ELEM: + if (!load_table_segment_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_CODE: + if (!load_code_section(buf, buf_end, buf_func, buf_func_end, + module, error_buf, error_buf_size)) + return false; + break; + case SECTION_TYPE_DATA: + if (!load_data_segment_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case SECTION_TYPE_DATACOUNT: + if (!load_datacount_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; +#endif + default: + set_error_buf(error_buf, error_buf_size, "invalid section id"); + return false; + } + + section = section->next; + } + + module->aux_data_end_global_index = (uint32)-1; + module->aux_heap_base_global_index = (uint32)-1; + module->aux_stack_top_global_index = (uint32)-1; + + /* Resolve auxiliary data/stack/heap info and reset memory info */ + export = module->exports; + for (i = 0; i < module->export_count; i++, export ++) { + if (export->kind == EXPORT_KIND_GLOBAL) { + if (!strcmp(export->name, "__heap_base")) { + global_index = export->index - module->import_global_count; + global = module->globals + global_index; + if (global->type == VALUE_TYPE_I32 && !global->is_mutable + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST) { + aux_heap_base_global = global; + aux_heap_base = (uint64)(uint32)global->init_expr.u.i32; + aux_heap_base_global_index = export->index; + LOG_VERBOSE("Found aux __heap_base global, value: %" PRIu64, + aux_heap_base); + } + } + else if (!strcmp(export->name, "__data_end")) { + global_index = export->index - module->import_global_count; + global = module->globals + global_index; + if (global->type == VALUE_TYPE_I32 && !global->is_mutable + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST) { + aux_data_end_global = global; + aux_data_end = (uint64)(uint32)global->init_expr.u.i32; + aux_data_end_global_index = export->index; + LOG_VERBOSE("Found aux __data_end global, value: %" PRIu64, + aux_data_end); + aux_data_end = align_uint64(aux_data_end, 16); + } + } + + /* For module compiled with -pthread option, the global is: + [0] stack_top <-- 0 + [1] tls_pointer + [2] tls_size + [3] data_end <-- 3 + [4] global_base + [5] heap_base <-- 5 + [6] dso_handle + + For module compiled without -pthread option: + [0] stack_top <-- 0 + [1] data_end <-- 1 + [2] global_base + [3] heap_base <-- 3 + [4] dso_handle + */ + if (aux_data_end_global && aux_heap_base_global + && aux_data_end <= aux_heap_base) { + module->aux_data_end_global_index = aux_data_end_global_index; + module->aux_data_end = aux_data_end; + module->aux_heap_base_global_index = aux_heap_base_global_index; + module->aux_heap_base = aux_heap_base; + + /* Resolve aux stack top global */ + for (global_index = 0; global_index < module->global_count; + global_index++) { + global = module->globals + global_index; + if (global->is_mutable /* heap_base and data_end is + not mutable */ + && global->type == VALUE_TYPE_I32 + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST + && (uint64)(uint32)global->init_expr.u.i32 + <= aux_heap_base) { + aux_stack_top_global = global; + aux_stack_top = (uint64)(uint32)global->init_expr.u.i32; + module->aux_stack_top_global_index = + module->import_global_count + global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = + aux_stack_top > aux_data_end + ? (uint32)(aux_stack_top - aux_data_end) + : (uint32)aux_stack_top; + LOG_VERBOSE( + "Found aux stack top global, value: %" PRIu64 ", " + "global index: %d, stack size: %d", + aux_stack_top, global_index, + module->aux_stack_size); + break; + } + } + if (!aux_stack_top_global) { + /* Auxiliary stack global isn't found, it must be unused + in the wasm app, as if it is used, the global must be + defined. Here we set it to __heap_base global and set + its size to 0. */ + aux_stack_top_global = aux_heap_base_global; + aux_stack_top = aux_heap_base; + module->aux_stack_top_global_index = + module->aux_heap_base_global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = 0; + } + break; + } + } + } + + module->malloc_function = (uint32)-1; + module->free_function = (uint32)-1; + module->retain_function = (uint32)-1; + + /* Resolve malloc/free function exported by wasm module */ +#if WASM_ENABLE_MEMORY64 != 0 + if (has_module_memory64(module)) + malloc_free_io_type = VALUE_TYPE_I64; +#endif + export = module->exports; + for (i = 0; i < module->export_count; i++, export ++) { + if (export->kind == EXPORT_KIND_FUNC) { + if (!strcmp(export->name, "malloc") + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == malloc_free_io_type) { + bh_assert(module->malloc_function == (uint32)-1); + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + export->name, export->index); + } + } + else if (!strcmp(export->name, "__new") + && export->index >= module->import_function_count) { + /* __new && __pin for AssemblyScript */ + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 2 && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == VALUE_TYPE_I32 + && func_type->types[2] == malloc_free_io_type) { + uint32 j; + WASMExport *export_tmp; + + bh_assert(module->malloc_function == (uint32)-1); + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + export->name, export->index); + + /* resolve retain function. + If not found, reset malloc function index */ + export_tmp = module->exports; + for (j = 0; j < module->export_count; j++, export_tmp++) { + if ((export_tmp->kind == EXPORT_KIND_FUNC) + && (!strcmp(export_tmp->name, "__retain") + || !strcmp(export_tmp->name, "__pin")) + && (export_tmp->index + >= module->import_function_count)) { + func_index = export_tmp->index + - module->import_function_count; + func_type = + module->functions[func_index]->func_type; + if (func_type->param_count == 1 + && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == malloc_free_io_type) { + bh_assert(module->retain_function + == (uint32)-1); + module->retain_function = export_tmp->index; + LOG_VERBOSE("Found retain function, name: %s, " + "index: %u", + export_tmp->name, + export_tmp->index); + break; + } + } + } + if (j == module->export_count) { + module->malloc_function = (uint32)-1; + LOG_VERBOSE("Can't find retain function," + "reset malloc function index to -1"); + } + } + } + else if (((!strcmp(export->name, "free")) + || (!strcmp(export->name, "__release")) + || (!strcmp(export->name, "__unpin"))) + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 && func_type->result_count == 0 + && func_type->types[0] == malloc_free_io_type) { + bh_assert(module->free_function == (uint32)-1); + module->free_function = export->index; + LOG_VERBOSE("Found free function, name: %s, index: %u", + export->name, export->index); + } + } + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 + handle_table = wasm_interp_get_handle_table(); +#endif + + for (i = 0; i < module->function_count; i++) { + WASMFunction *func = module->functions[i]; + if (!wasm_loader_prepare_bytecode(module, func, i, error_buf, + error_buf_size)) { + return false; + } + + if (i == module->function_count - 1) { + bh_assert(func->code + func->code_size == buf_code_end); + } + } + + if (!module->possible_memory_grow) { + WASMMemoryImport *memory_import; + WASMMemory *memory; + + if (aux_data_end_global && aux_heap_base_global + && aux_stack_top_global) { + uint64 init_memory_size; + uint64 shrunk_memory_size = align_uint64(aux_heap_base, 8); + + /* Only resize(shrunk) the memory size if num_bytes_per_page is in + * valid range of uint32 */ + if (shrunk_memory_size <= UINT32_MAX) { + if (module->import_memory_count) { + memory_import = &module->import_memories[0].u.memory; + init_memory_size = (uint64)memory_import->num_bytes_per_page + * memory_import->init_page_count; + if (shrunk_memory_size <= init_memory_size) { + /* Reset memory info to decrease memory usage */ + memory_import->num_bytes_per_page = shrunk_memory_size; + memory_import->init_page_count = 1; + LOG_VERBOSE("Shrink import memory size to %" PRIu64, + shrunk_memory_size); + } + } + + if (module->memory_count) { + memory = &module->memories[0]; + init_memory_size = (uint64)memory->num_bytes_per_page + * memory->init_page_count; + if (shrunk_memory_size <= init_memory_size) { + /* Reset memory info to decrease memory usage */ + memory->num_bytes_per_page = shrunk_memory_size; + memory->init_page_count = 1; + LOG_VERBOSE("Shrink memory size to %" PRIu64, + shrunk_memory_size); + } + } + } + } + + if (module->import_memory_count) { + memory_import = &module->import_memories[0].u.memory; + if (memory_import->init_page_count < DEFAULT_MAX_PAGES) { + memory_import->num_bytes_per_page *= + memory_import->init_page_count; + if (memory_import->init_page_count > 0) + memory_import->init_page_count = + memory_import->max_page_count = 1; + else + memory_import->init_page_count = + memory_import->max_page_count = 0; + } + } + + if (module->memory_count) { + memory = &module->memories[0]; + if (memory->init_page_count < DEFAULT_MAX_PAGES) { + memory->num_bytes_per_page *= memory->init_page_count; + if (memory->init_page_count > 0) + memory->init_page_count = memory->max_page_count = 1; + else + memory->init_page_count = memory->max_page_count = 0; + } + } + } + + calculate_global_data_offset(module); + +#if WASM_ENABLE_FAST_JIT != 0 + if (!init_fast_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif + +#if WASM_ENABLE_JIT != 0 + if (!init_llvm_jit_functions_stage1(module, error_buf, error_buf_size)) { + return false; + } +#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0) + if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) { + return false; + } +#else + /* Run aot_compile_wasm in a backend thread, so as not to block the main + thread fast jit execution, since applying llvm optimizations in + aot_compile_wasm may cost a lot of time. + Create thread with enough native stack to apply llvm optimizations */ + if (os_thread_create(&module->llvm_jit_init_thread, + init_llvm_jit_functions_stage2_callback, + (void *)module, APP_THREAD_STACK_SIZE_DEFAULT * 8) + != 0) { + set_error_buf(error_buf, error_buf_size, + "create orcjit compile thread failed"); + return false; + } +#endif +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* Create threads to compile the jit functions */ + if (!compile_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_module_mem_consumption(module); +#endif + return true; +} + +static WASMModule * +create_module(char *name, char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = + loader_malloc(sizeof(WASMModule), error_buf, error_buf_size); + bh_list_status ret; + + if (!module) { + return NULL; + } + + module->module_type = Wasm_Module_Bytecode; + + /* Set start_function to -1, means no start function */ + module->start_function = (uint32)-1; + + module->name = name; + +#if WASM_ENABLE_FAST_INTERP == 0 + module->br_table_cache_list = &module->br_table_cache_list_head; + ret = bh_list_init(module->br_table_cache_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (os_mutex_init(&module->instance_list_lock) != 0) { + set_error_buf(error_buf, error_buf_size, + "init instance list lock failed"); + wasm_runtime_free(module); + return NULL; + } +#endif + + (void)ret; + return module; +} + +WASMModule * +wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size) +{ + WASMModule *module = create_module("", error_buf, error_buf_size); + if (!module) + return NULL; + + if (!load_from_sections(module, section_list, false, error_buf, + error_buf_size)) { + wasm_loader_unload(module); + return NULL; + } + + LOG_VERBOSE("Load module from sections success.\n"); + return module; +} + +static void +destroy_sections(WASMSection *section_list) +{ + WASMSection *section = section_list, *next; + while (section) { + next = section->next; + wasm_runtime_free(section); + section = next; + } +} + +/* clang-format off */ +static uint8 section_ids[] = { + SECTION_TYPE_USER, + SECTION_TYPE_TYPE, + SECTION_TYPE_IMPORT, + SECTION_TYPE_FUNC, + SECTION_TYPE_TABLE, + SECTION_TYPE_MEMORY, + SECTION_TYPE_GLOBAL, + SECTION_TYPE_EXPORT, + SECTION_TYPE_START, + SECTION_TYPE_ELEM, +#if WASM_ENABLE_BULK_MEMORY != 0 + SECTION_TYPE_DATACOUNT, +#endif + SECTION_TYPE_CODE, + SECTION_TYPE_DATA +}; +/* clang-format on */ + +static uint8 +get_section_index(uint8 section_type) +{ + uint8 max_id = sizeof(section_ids) / sizeof(uint8); + + for (uint8 i = 0; i < max_id; i++) { + if (section_type == section_ids[i]) + return i; + } + + return (uint8)-1; +} + +static bool +create_sections(const uint8 *buf, uint32 size, WASMSection **p_section_list, + char *error_buf, uint32 error_buf_size) +{ + WASMSection *section_list_end = NULL, *section; + const uint8 *p = buf, *p_end = buf + size /*, *section_body*/; + uint8 section_type, section_index, last_section_index = (uint8)-1; + uint32 section_size; + + bh_assert(!*p_section_list); + + p += 8; + while (p < p_end) { + CHECK_BUF(p, p_end, 1); + section_type = read_uint8(p); + section_index = get_section_index(section_type); + if (section_index != (uint8)-1) { + if (section_type != SECTION_TYPE_USER) { + /* Custom sections may be inserted at any place, + while other sections must occur at most once + and in prescribed order. */ + bh_assert(last_section_index == (uint8)-1 + || last_section_index < section_index); + last_section_index = section_index; + } + read_leb_uint32(p, p_end, section_size); + CHECK_BUF1(p, p_end, section_size); + + if (!(section = loader_malloc(sizeof(WASMSection), error_buf, + error_buf_size))) { + return false; + } + + section->section_type = section_type; + section->section_body = (uint8 *)p; + section->section_body_size = section_size; + + if (!*p_section_list) + *p_section_list = section_list_end = section; + else { + section_list_end->next = section; + section_list_end = section; + } + + p += section_size; + } + else { + bh_assert(0); + } + } + + (void)last_section_index; + return true; +} + +static void +exchange32(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 3); + *(p_data + 3) = value; + + value = *(p_data + 1); + *(p_data + 1) = *(p_data + 2); + *(p_data + 2) = value; +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +static bool +load(const uint8 *buf, uint32 size, WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf_end = buf + size; + const uint8 *p = buf, *p_end = buf_end; + uint32 magic_number, version; + WASMSection *section_list = NULL; + + CHECK_BUF1(p, p_end, sizeof(uint32)); + magic_number = read_uint32(p); + if (!is_little_endian()) + exchange32((uint8 *)&magic_number); + + bh_assert(magic_number == WASM_MAGIC_NUMBER); + + CHECK_BUF1(p, p_end, sizeof(uint32)); + version = read_uint32(p); + if (!is_little_endian()) + exchange32((uint8 *)&version); + + if (version != WASM_CURRENT_VERSION) { + set_error_buf(error_buf, error_buf_size, "unknown binary version"); + return false; + } + + if (!create_sections(buf, size, §ion_list, error_buf, error_buf_size) + || !load_from_sections(module, section_list, true, error_buf, + error_buf_size)) { + destroy_sections(section_list); + return false; + } + + destroy_sections(section_list); + (void)p_end; + return true; +} + +WASMModule * +wasm_loader_load(uint8 *buf, uint32 size, const LoadArgs *args, char *error_buf, + uint32 error_buf_size) +{ + WASMModule *module = create_module(args->name, error_buf, error_buf_size); + if (!module) { + return NULL; + } + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ + || WASM_ENABLE_JIT != 0 + module->load_addr = (uint8 *)buf; + module->load_size = size; +#endif + + if (!load(buf, size, module, error_buf, error_buf_size)) { + goto fail; + } + + LOG_VERBOSE("Load module success.\n"); + return module; + +fail: + wasm_loader_unload(module); + return NULL; +} + +void +wasm_loader_unload(WASMModule *module) +{ + uint32 i; + + if (!module) + return; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + module->orcjit_stop_compiling = true; + if (module->llvm_jit_init_thread) + os_thread_join(module->llvm_jit_init_thread, NULL); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* Stop Fast/LLVM JIT compilation firstly to avoid accessing + module internal data after they were freed */ + orcjit_stop_compile_threads(module); +#endif + +#if WASM_ENABLE_JIT != 0 + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + if (module->comp_ctx) + aot_destroy_comp_context(module->comp_ctx); + if (module->comp_data) + aot_destroy_comp_data(module->comp_data); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (module->tierup_wait_lock_inited) { + os_mutex_destroy(&module->tierup_wait_lock); + os_cond_destroy(&module->tierup_wait_cond); + } +#endif + + if (module->types) { + for (i = 0; i < module->type_count; i++) { + if (module->types[i]) + destroy_wasm_type(module->types[i]); + } + wasm_runtime_free(module->types); + } + + if (module->imports) + wasm_runtime_free(module->imports); + + if (module->functions) { + for (i = 0; i < module->function_count; i++) { + if (module->functions[i]) { + if (module->functions[i]->local_offsets) + wasm_runtime_free(module->functions[i]->local_offsets); +#if WASM_ENABLE_FAST_INTERP != 0 + if (module->functions[i]->code_compiled) + wasm_runtime_free(module->functions[i]->code_compiled); + if (module->functions[i]->consts) + wasm_runtime_free(module->functions[i]->consts); +#endif +#if WASM_ENABLE_FAST_JIT != 0 + if (module->functions[i]->fast_jit_jitted_code) { + jit_code_cache_free( + module->functions[i]->fast_jit_jitted_code); + } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (module->functions[i]->call_to_fast_jit_from_llvm_jit) { + jit_code_cache_free( + module->functions[i]->call_to_fast_jit_from_llvm_jit); + } +#endif +#endif + wasm_runtime_free(module->functions[i]); + } + } + wasm_runtime_free(module->functions); + } + + if (module->tables) + wasm_runtime_free(module->tables); + + if (module->memories) + wasm_runtime_free(module->memories); + + if (module->globals) + wasm_runtime_free(module->globals); + + if (module->exports) + wasm_runtime_free(module->exports); + + if (module->table_segments) { + for (i = 0; i < module->table_seg_count; i++) { + if (module->table_segments[i].init_values) + wasm_runtime_free(module->table_segments[i].init_values); + } + wasm_runtime_free(module->table_segments); + } + + if (module->data_segments) { + for (i = 0; i < module->data_seg_count; i++) { + if (module->data_segments[i]) + wasm_runtime_free(module->data_segments[i]); + } + wasm_runtime_free(module->data_segments); + } + + if (module->const_str_list) { + StringNode *node = module->const_str_list, *node_next; + while (node) { + node_next = node->next; + wasm_runtime_free(node); + node = node_next; + } + } + +#if WASM_ENABLE_FAST_INTERP == 0 + if (module->br_table_cache_list) { + BrTableCache *node = bh_list_first_elem(module->br_table_cache_list); + BrTableCache *node_next; + while (node) { + node_next = bh_list_elem_next(node); + wasm_runtime_free(node); + node = node_next; + } + } +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + os_mutex_destroy(&module->instance_list_lock); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + if (module->fast_jit_func_ptrs) { + wasm_runtime_free(module->fast_jit_func_ptrs); + } + + for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) { + if (module->fast_jit_thread_locks_inited[i]) { + os_mutex_destroy(&module->fast_jit_thread_locks[i]); + } + } +#endif + + wasm_runtime_free(module); +} + +bool +wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, + const uint8 *start_addr, const uint8 *code_end_addr, + uint8 label_type, uint8 **p_else_addr, + uint8 **p_end_addr) +{ + const uint8 *p = start_addr, *p_end = code_end_addr; + uint8 *else_addr = NULL; + char error_buf[128]; + uint32 block_nested_depth = 1, count, i, j, t; + uint32 error_buf_size = sizeof(error_buf); + uint8 opcode, u8; + BlockAddr block_stack[16] = { 0 }, *block; + + i = ((uintptr_t)start_addr) & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + block = block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; + + for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) { + if (block[j].start_addr == start_addr) { + /* Cache hit */ + *p_else_addr = block[j].else_addr; + *p_end_addr = block[j].end_addr; + return true; + } + } + + /* Cache unhit */ + block_stack[0].start_addr = start_addr; + + while (p < code_end_addr) { + opcode = *p++; + + switch (opcode) { + case WASM_OP_UNREACHABLE: + case WASM_OP_NOP: + break; + + case WASM_OP_BLOCK: + case WASM_OP_LOOP: + case WASM_OP_IF: + /* block result type: 0x40/0x7F/0x7E/0x7D/0x7C */ + u8 = read_uint8(p); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + /* block type */ + skip_leb_uint32(p, p_end); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + + case WASM_OP_ELSE: + if (label_type == LABEL_TYPE_IF && block_nested_depth == 1) + else_addr = (uint8 *)(p - 1); + if (block_nested_depth - 1 + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth - 1].else_addr = + (uint8 *)(p - 1); + break; + + case WASM_OP_END: + if (block_nested_depth == 1) { + if (label_type == LABEL_TYPE_IF) + *p_else_addr = else_addr; + *p_end_addr = (uint8 *)(p - 1); + + block_stack[0].end_addr = (uint8 *)(p - 1); + for (t = 0; t < sizeof(block_stack) / sizeof(BlockAddr); + t++) { + start_addr = block_stack[t].start_addr; + if (start_addr) { + i = ((uintptr_t)start_addr) + & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + block = + block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; + for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) + if (!block[j].start_addr) + break; + + if (j == BLOCK_ADDR_CONFLICT_SIZE) { + memmove(block + 1, block, + (BLOCK_ADDR_CONFLICT_SIZE - 1) + * sizeof(BlockAddr)); + j = 0; + } + block[j].start_addr = block_stack[t].start_addr; + block[j].else_addr = block_stack[t].else_addr; + block[j].end_addr = block_stack[t].end_addr; + } + else + break; + } + return true; + } + else { + block_nested_depth--; + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth].end_addr = + (uint8 *)(p - 1); + } + break; + + case WASM_OP_BR: + case WASM_OP_BR_IF: + skip_leb_uint32(p, p_end); /* labelidx */ + break; + + case WASM_OP_BR_TABLE: + read_leb_uint32(p, p_end, count); /* lable num */ +#if WASM_ENABLE_FAST_INTERP != 0 + for (i = 0; i <= count; i++) /* lableidxs */ + skip_leb_uint32(p, p_end); +#else + p += count + 1; + while (*p == WASM_OP_NOP) + p++; +#endif + break; + +#if WASM_ENABLE_FAST_INTERP == 0 + case EXT_OP_BR_TABLE_CACHE: + read_leb_uint32(p, p_end, count); /* lable num */ + while (*p == WASM_OP_NOP) + p++; + break; +#endif + + case WASM_OP_RETURN: + break; + + case WASM_OP_CALL: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: +#endif + skip_leb_uint32(p, p_end); /* funcidx */ + break; + + case WASM_OP_CALL_INDIRECT: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL_INDIRECT: +#endif + skip_leb_uint32(p, p_end); /* typeidx */ + CHECK_BUF(p, p_end, 1); + u8 = read_uint8(p); /* 0x00 */ + break; + +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_TRY: + case WASM_OP_CATCH: + case WASM_OP_THROW: + case WASM_OP_RETHROW: + case WASM_OP_DELEGATE: + case WASM_OP_CATCH_ALL: + /* TODO */ + return false; +#endif + + case WASM_OP_DROP: + case WASM_OP_SELECT: + case WASM_OP_DROP_64: + case WASM_OP_SELECT_64: + break; +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_SELECT_T: + skip_leb_uint32(p, p_end); /* vec length */ + CHECK_BUF(p, p_end, 1); + u8 = read_uint8(p); /* typeidx */ + break; + case WASM_OP_TABLE_GET: + case WASM_OP_TABLE_SET: + skip_leb_uint32(p, p_end); /* table index */ + break; + case WASM_OP_REF_NULL: + CHECK_BUF(p, p_end, 1); + u8 = read_uint8(p); /* type */ + break; + case WASM_OP_REF_IS_NULL: + break; + case WASM_OP_REF_FUNC: + skip_leb_uint32(p, p_end); /* func index */ + break; +#endif /* WASM_ENABLE_REF_TYPES */ + case WASM_OP_GET_LOCAL: + case WASM_OP_SET_LOCAL: + case WASM_OP_TEE_LOCAL: + case WASM_OP_GET_GLOBAL: + case WASM_OP_SET_GLOBAL: + case WASM_OP_GET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_AUX_STACK: + skip_leb_uint32(p, p_end); /* localidx */ + break; + + case EXT_OP_GET_LOCAL_FAST: + case EXT_OP_SET_LOCAL_FAST: + case EXT_OP_TEE_LOCAL_FAST: + CHECK_BUF(p, p_end, 1); + p++; + break; + + case WASM_OP_I32_LOAD: + case WASM_OP_I64_LOAD: + case WASM_OP_F32_LOAD: + case WASM_OP_F64_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + case WASM_OP_I32_STORE: + case WASM_OP_I64_STORE: + case WASM_OP_F32_STORE: + case WASM_OP_F64_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ + break; + + case WASM_OP_MEMORY_SIZE: + case WASM_OP_MEMORY_GROW: + skip_leb_uint32(p, p_end); /* 0x00 */ + break; + + case WASM_OP_I32_CONST: + skip_leb_int32(p, p_end); + break; + case WASM_OP_I64_CONST: + skip_leb_int64(p, p_end); + break; + case WASM_OP_F32_CONST: + p += sizeof(float32); + break; + case WASM_OP_F64_CONST: + p += sizeof(float64); + break; + + case WASM_OP_I32_EQZ: + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + case WASM_OP_I64_EQZ: + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + case WASM_OP_I32_CLZ: + case WASM_OP_I32_CTZ: + case WASM_OP_I32_POPCNT: + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + case WASM_OP_I64_CLZ: + case WASM_OP_I64_CTZ: + case WASM_OP_I64_POPCNT: + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + case WASM_OP_F32_COPYSIGN: + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + case WASM_OP_F64_COPYSIGN: + case WASM_OP_I32_WRAP_I64: + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + case WASM_OP_F32_DEMOTE_F64: + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + case WASM_OP_F64_PROMOTE_F32: + case WASM_OP_I32_REINTERPRET_F32: + case WASM_OP_I64_REINTERPRET_F64: + case WASM_OP_F32_REINTERPRET_I32: + case WASM_OP_F64_REINTERPRET_I64: + case WASM_OP_I32_EXTEND8_S: + case WASM_OP_I32_EXTEND16_S: + case WASM_OP_I64_EXTEND8_S: + case WASM_OP_I64_EXTEND16_S: + case WASM_OP_I64_EXTEND32_S: + break; + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + skip_leb_uint32(p, p_end); + /* skip memory idx */ + p++; + break; + case WASM_OP_DATA_DROP: + skip_leb_uint32(p, p_end); + break; + case WASM_OP_MEMORY_COPY: + /* skip two memory idx */ + p += 2; + break; + case WASM_OP_MEMORY_FILL: + /* skip memory idx */ + p++; + break; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_TABLE_INIT: + case WASM_OP_TABLE_COPY: + /* tableidx */ + skip_leb_uint32(p, p_end); + /* elemidx */ + skip_leb_uint32(p, p_end); + break; + case WASM_OP_ELEM_DROP: + /* elemidx */ + skip_leb_uint32(p, p_end); + break; + case WASM_OP_TABLE_SIZE: + case WASM_OP_TABLE_GROW: + case WASM_OP_TABLE_FILL: + skip_leb_uint32(p, p_end); /* table idx */ + break; +#endif /* WASM_ENABLE_REF_TYPES */ + default: + bh_assert(0); + break; + } + break; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + /* TODO: memory64 offset type changes */ + uint32 opcode1; + + /* atomic_op (u32_leb) + memarg (2 u32_leb) */ + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + if (opcode != WASM_OP_ATOMIC_FENCE) { + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ + } + else { + /* atomic.fence doesn't have memarg */ + p++; + } + break; + } +#endif + + default: + bh_assert(0); + break; + } + } + + (void)u8; + return false; +} + +#define REF_I32 VALUE_TYPE_I32 +#define REF_F32 VALUE_TYPE_F32 +#define REF_I64_1 VALUE_TYPE_I64 +#define REF_I64_2 VALUE_TYPE_I64 +#define REF_F64_1 VALUE_TYPE_F64 +#define REF_F64_2 VALUE_TYPE_F64 +#define REF_ANY VALUE_TYPE_ANY + +#if WASM_ENABLE_FAST_INTERP != 0 + +#if WASM_DEBUG_PREPROCESSOR != 0 +#define LOG_OP(...) os_printf(__VA_ARGS__) +#else +#define LOG_OP(...) (void)0 +#endif + +#define PATCH_ELSE 0 +#define PATCH_END 1 +typedef struct BranchBlockPatch { + struct BranchBlockPatch *next; + uint8 patch_type; + uint8 *code_compiled; +} BranchBlockPatch; +#endif + +typedef struct BranchBlock { + uint8 label_type; + BlockType block_type; + uint8 *start_addr; + uint8 *else_addr; + uint8 *end_addr; + uint32 stack_cell_num; +#if WASM_ENABLE_FAST_INTERP != 0 + uint16 dynamic_offset; + uint8 *code_compiled; + BranchBlockPatch *patch_list; + /* This is used to save params frame_offset of of if block */ + int16 *param_frame_offsets; + /* This is used to store available param num for if/else branch, so the else + * opcode can know how many parameters should be copied to the stack */ + uint32 available_param_num; + /* This is used to recover dynamic offset for else branch */ + uint16 start_dynamic_offset; +#endif + + /* Indicate the operand stack is in polymorphic state. + * If the opcode is one of unreachable/br/br_table/return, stack is marked + * to polymorphic state until the block's 'end' opcode is processed. + * If stack is in polymorphic state and stack is empty, instruction can + * pop any type of value directly without decreasing stack top pointer + * and stack cell num. */ + bool is_stack_polymorphic; +} BranchBlock; + +typedef struct WASMLoaderContext { + /* frame ref stack */ + uint8 *frame_ref; + uint8 *frame_ref_bottom; + uint8 *frame_ref_boundary; + uint32 frame_ref_size; + uint32 stack_cell_num; + uint32 max_stack_cell_num; + + /* frame csp stack */ + BranchBlock *frame_csp; + BranchBlock *frame_csp_bottom; + BranchBlock *frame_csp_boundary; + uint32 frame_csp_size; + uint32 csp_num; + uint32 max_csp_num; + +#if WASM_ENABLE_FAST_INTERP != 0 + /* frame offset stack */ + int16 *frame_offset; + int16 *frame_offset_bottom; + int16 *frame_offset_boundary; + uint32 frame_offset_size; + int16 dynamic_offset; + int16 start_dynamic_offset; + int16 max_dynamic_offset; + + /* preserved local offset */ + int16 preserved_local_offset; + + /* const buffer */ + uint8 *const_buf; + uint16 num_const; + uint16 const_cell_num; + uint32 const_buf_size; + + /* processed code */ + uint8 *p_code_compiled; + uint8 *p_code_compiled_end; + uint32 code_compiled_size; + /* If the last opcode will be dropped, the peak memory usage will be larger + * than the final code_compiled_size, we record the peak size to ensure + * there will not be invalid memory access during second traverse */ + uint32 code_compiled_peak_size; +#endif +} WASMLoaderContext; + +typedef struct Const { + WASMValue value; + uint16 slot_index; + uint8 value_type; +} Const; + +#define CHECK_CSP_PUSH() \ + do { \ + if (ctx->frame_csp >= ctx->frame_csp_boundary) { \ + MEM_REALLOC( \ + ctx->frame_csp_bottom, ctx->frame_csp_size, \ + (uint32)(ctx->frame_csp_size + 8 * sizeof(BranchBlock))); \ + ctx->frame_csp_size += (uint32)(8 * sizeof(BranchBlock)); \ + ctx->frame_csp_boundary = \ + ctx->frame_csp_bottom \ + + ctx->frame_csp_size / sizeof(BranchBlock); \ + ctx->frame_csp = ctx->frame_csp_bottom + ctx->csp_num; \ + } \ + } while (0) + +#define CHECK_CSP_POP() \ + do { \ + bh_assert(ctx->csp_num >= 1); \ + } while (0) + +#if WASM_ENABLE_FAST_INTERP != 0 +static bool +check_offset_push(WASMLoaderContext *ctx, char *error_buf, + uint32 error_buf_size) +{ + uint32 cell_num = (uint32)(ctx->frame_offset - ctx->frame_offset_bottom); + if (ctx->frame_offset >= ctx->frame_offset_boundary) { + MEM_REALLOC(ctx->frame_offset_bottom, ctx->frame_offset_size, + ctx->frame_offset_size + 16); + ctx->frame_offset_size += 16; + ctx->frame_offset_boundary = + ctx->frame_offset_bottom + ctx->frame_offset_size / sizeof(int16); + ctx->frame_offset = ctx->frame_offset_bottom + cell_num; + } + return true; +fail: + return false; +} + +static bool +check_offset_pop(WASMLoaderContext *ctx, uint32 cells) +{ + if (ctx->frame_offset - cells < ctx->frame_offset_bottom) + return false; + return true; +} + +static void +free_label_patch_list(BranchBlock *frame_csp) +{ + BranchBlockPatch *label_patch = frame_csp->patch_list; + BranchBlockPatch *next; + while (label_patch != NULL) { + next = label_patch->next; + wasm_runtime_free(label_patch); + label_patch = next; + } + frame_csp->patch_list = NULL; +} + +static void +free_all_label_patch_lists(BranchBlock *frame_csp, uint32 csp_num) +{ + BranchBlock *tmp_csp = frame_csp; + + for (uint32 i = 0; i < csp_num; i++) { + free_label_patch_list(tmp_csp); + tmp_csp++; + } +} + +#endif + +static bool +check_stack_push(WASMLoaderContext *ctx, char *error_buf, uint32 error_buf_size) +{ + if (ctx->frame_ref >= ctx->frame_ref_boundary) { + MEM_REALLOC(ctx->frame_ref_bottom, ctx->frame_ref_size, + ctx->frame_ref_size + 16); + ctx->frame_ref_size += 16; + ctx->frame_ref_boundary = ctx->frame_ref_bottom + ctx->frame_ref_size; + ctx->frame_ref = ctx->frame_ref_bottom + ctx->stack_cell_num; + } + return true; +fail: + return false; +} + +static bool +check_stack_top_values(uint8 *frame_ref, int32 stack_cell_num, uint8 type, + char *error_buf, uint32 error_buf_size) +{ + bh_assert(!((is_32bit_type(type) && stack_cell_num < 1) + || (is_64bit_type(type) && stack_cell_num < 2))); + + bh_assert(!( + (type == VALUE_TYPE_I32 && *(frame_ref - 1) != REF_I32) + || (type == VALUE_TYPE_F32 && *(frame_ref - 1) != REF_F32) + || (type == VALUE_TYPE_I64 + && (*(frame_ref - 2) != REF_I64_1 || *(frame_ref - 1) != REF_I64_2)) + || (type == VALUE_TYPE_F64 + && (*(frame_ref - 2) != REF_F64_1 + || *(frame_ref - 1) != REF_F64_2)))); + return true; +} + +static bool +check_stack_pop(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + int32 block_stack_cell_num = + (int32)(ctx->stack_cell_num - (ctx->frame_csp - 1)->stack_cell_num); + + if (block_stack_cell_num > 0 && *(ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + /* the stack top is a value of any type, return success */ + return true; + } + + if (!check_stack_top_values(ctx->frame_ref, block_stack_cell_num, type, + error_buf, error_buf_size)) + return false; + + return true; +} + +static void +wasm_loader_ctx_destroy(WASMLoaderContext *ctx) +{ + if (ctx) { + if (ctx->frame_ref_bottom) + wasm_runtime_free(ctx->frame_ref_bottom); + if (ctx->frame_csp_bottom) { +#if WASM_ENABLE_FAST_INTERP != 0 + free_all_label_patch_lists(ctx->frame_csp_bottom, ctx->csp_num); +#endif + wasm_runtime_free(ctx->frame_csp_bottom); + } +#if WASM_ENABLE_FAST_INTERP != 0 + if (ctx->frame_offset_bottom) + wasm_runtime_free(ctx->frame_offset_bottom); + if (ctx->const_buf) + wasm_runtime_free(ctx->const_buf); +#endif + wasm_runtime_free(ctx); + } +} + +static WASMLoaderContext * +wasm_loader_ctx_init(WASMFunction *func, char *error_buf, uint32 error_buf_size) +{ + WASMLoaderContext *loader_ctx = + loader_malloc(sizeof(WASMLoaderContext), error_buf, error_buf_size); + if (!loader_ctx) + return NULL; + + loader_ctx->frame_ref_size = 32; + if (!(loader_ctx->frame_ref_bottom = loader_ctx->frame_ref = loader_malloc( + loader_ctx->frame_ref_size, error_buf, error_buf_size))) + goto fail; + loader_ctx->frame_ref_boundary = loader_ctx->frame_ref_bottom + 32; + + loader_ctx->frame_csp_size = sizeof(BranchBlock) * 8; + if (!(loader_ctx->frame_csp_bottom = loader_ctx->frame_csp = loader_malloc( + loader_ctx->frame_csp_size, error_buf, error_buf_size))) + goto fail; + loader_ctx->frame_csp_boundary = loader_ctx->frame_csp_bottom + 8; + +#if WASM_ENABLE_FAST_INTERP != 0 + loader_ctx->frame_offset_size = sizeof(int16) * 32; + if (!(loader_ctx->frame_offset_bottom = loader_ctx->frame_offset = + loader_malloc(loader_ctx->frame_offset_size, error_buf, + error_buf_size))) + goto fail; + loader_ctx->frame_offset_boundary = loader_ctx->frame_offset_bottom + 32; + + loader_ctx->num_const = 0; + loader_ctx->const_buf_size = sizeof(Const) * 8; + if (!(loader_ctx->const_buf = loader_malloc(loader_ctx->const_buf_size, + error_buf, error_buf_size))) + goto fail; + + if (func->param_cell_num >= (int32)INT16_MAX - func->local_cell_num) { + set_error_buf(error_buf, error_buf_size, + "fast interpreter offset overflow"); + goto fail; + } + + loader_ctx->start_dynamic_offset = loader_ctx->dynamic_offset = + loader_ctx->max_dynamic_offset = + func->param_cell_num + func->local_cell_num; +#endif + return loader_ctx; + +fail: + wasm_loader_ctx_destroy(loader_ctx); + return NULL; +} + +static bool +wasm_loader_push_frame_ref(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + if (type == VALUE_TYPE_VOID) + return true; + + if (!check_stack_push(ctx, error_buf, error_buf_size)) + return false; + + *ctx->frame_ref++ = type; + ctx->stack_cell_num++; + if (ctx->stack_cell_num > ctx->max_stack_cell_num) + ctx->max_stack_cell_num = ctx->stack_cell_num; + + if (is_32bit_type(type)) + return true; + + if (!check_stack_push(ctx, error_buf, error_buf_size)) + return false; + *ctx->frame_ref++ = type; + ctx->stack_cell_num++; + if (ctx->stack_cell_num > ctx->max_stack_cell_num) { + ctx->max_stack_cell_num = ctx->stack_cell_num; + bh_assert(ctx->max_stack_cell_num <= UINT16_MAX); + } + return true; +} + +static bool +wasm_loader_pop_frame_ref(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + BranchBlock *cur_block = ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + + /* Directly return success if current block is in stack + * polymorphic state while stack is empty. */ + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + return true; + + if (type == VALUE_TYPE_VOID) + return true; + + if (!check_stack_pop(ctx, type, error_buf, error_buf_size)) + return false; + + ctx->frame_ref--; + ctx->stack_cell_num--; + + if (is_32bit_type(type) || *ctx->frame_ref == VALUE_TYPE_ANY) + return true; + + ctx->frame_ref--; + ctx->stack_cell_num--; + return true; +} + +#if WASM_ENABLE_FAST_INTERP == 0 +static bool +wasm_loader_push_pop_frame_ref(WASMLoaderContext *ctx, uint8 pop_cnt, + uint8 type_push, uint8 type_pop, char *error_buf, + uint32 error_buf_size) +{ + for (int i = 0; i < pop_cnt; i++) { + if (!wasm_loader_pop_frame_ref(ctx, type_pop, error_buf, + error_buf_size)) + return false; + } + if (!wasm_loader_push_frame_ref(ctx, type_push, error_buf, error_buf_size)) + return false; + return true; +} +#endif + +static bool +wasm_loader_push_frame_csp(WASMLoaderContext *ctx, uint8 label_type, + BlockType block_type, uint8 *start_addr, + char *error_buf, uint32 error_buf_size) +{ + CHECK_CSP_PUSH(); + memset(ctx->frame_csp, 0, sizeof(BranchBlock)); + ctx->frame_csp->label_type = label_type; + ctx->frame_csp->block_type = block_type; + ctx->frame_csp->start_addr = start_addr; + ctx->frame_csp->stack_cell_num = ctx->stack_cell_num; +#if WASM_ENABLE_FAST_INTERP != 0 + ctx->frame_csp->dynamic_offset = ctx->dynamic_offset; + ctx->frame_csp->patch_list = NULL; +#endif + ctx->frame_csp++; + ctx->csp_num++; + if (ctx->csp_num > ctx->max_csp_num) { + ctx->max_csp_num = ctx->csp_num; + bh_assert(ctx->max_csp_num <= UINT16_MAX); + } + return true; +fail: + return false; +} + +static bool +wasm_loader_pop_frame_csp(WASMLoaderContext *ctx, char *error_buf, + uint32 error_buf_size) +{ + CHECK_CSP_POP(); +#if WASM_ENABLE_FAST_INTERP != 0 + if ((ctx->frame_csp - 1)->param_frame_offsets) + wasm_runtime_free((ctx->frame_csp - 1)->param_frame_offsets); +#endif + ctx->frame_csp--; + ctx->csp_num--; + return true; +} + +#if WASM_ENABLE_FAST_INTERP != 0 + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define emit_label(opcode) \ + do { \ + wasm_loader_emit_ptr(loader_ctx, handle_table[opcode]); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(void *)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#else /* else of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#if UINTPTR_MAX == UINT64_MAX +#define emit_label(opcode) \ + do { \ + int32 offset = \ + (int32)((uint8 *)handle_table[opcode] - (uint8 *)handle_table[0]); \ + /* emit int32 relative offset in 64-bit target */ \ + wasm_loader_emit_uint32(loader_ctx, offset); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#else +#define emit_label(opcode) \ + do { \ + uint32 label_addr = (uint32)(uintptr_t)handle_table[opcode]; \ + /* emit uint32 label address in 32-bit target */ \ + wasm_loader_emit_uint32(loader_ctx, label_addr); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#endif +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(int32)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#define emit_label(opcode) \ + do { \ + wasm_loader_emit_uint8(loader_ctx, opcode); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint8)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + +#define emit_empty_label_addr_and_frame_ip(type) \ + do { \ + if (!add_label_patch_to_list(loader_ctx->frame_csp - 1, type, \ + loader_ctx->p_code_compiled, error_buf, \ + error_buf_size)) \ + goto fail; \ + /* label address, to be patched */ \ + wasm_loader_emit_ptr(loader_ctx, NULL); \ + } while (0) + +#define emit_br_info(frame_csp, is_br) \ + do { \ + if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, is_br, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define LAST_OP_OUTPUT_I32() \ + (last_op >= WASM_OP_I32_EQZ && last_op <= WASM_OP_I32_ROTR) \ + || (last_op == WASM_OP_I32_LOAD || last_op == WASM_OP_F32_LOAD) \ + || (last_op >= WASM_OP_I32_LOAD8_S && last_op <= WASM_OP_I32_LOAD16_U) \ + || (last_op >= WASM_OP_F32_ABS && last_op <= WASM_OP_F32_COPYSIGN) \ + || (last_op >= WASM_OP_I32_WRAP_I64 \ + && last_op <= WASM_OP_I32_TRUNC_U_F64) \ + || (last_op >= WASM_OP_F32_CONVERT_S_I32 \ + && last_op <= WASM_OP_F32_DEMOTE_F64) \ + || (last_op == WASM_OP_I32_REINTERPRET_F32) \ + || (last_op == WASM_OP_F32_REINTERPRET_I32) \ + || (last_op == EXT_OP_COPY_STACK_TOP) + +#define LAST_OP_OUTPUT_I64() \ + (last_op >= WASM_OP_I64_CLZ && last_op <= WASM_OP_I64_ROTR) \ + || (last_op >= WASM_OP_F64_ABS && last_op <= WASM_OP_F64_COPYSIGN) \ + || (last_op == WASM_OP_I64_LOAD || last_op == WASM_OP_F64_LOAD) \ + || (last_op >= WASM_OP_I64_LOAD8_S && last_op <= WASM_OP_I64_LOAD32_U) \ + || (last_op >= WASM_OP_I64_EXTEND_S_I32 \ + && last_op <= WASM_OP_I64_TRUNC_U_F64) \ + || (last_op >= WASM_OP_F64_CONVERT_S_I32 \ + && last_op <= WASM_OP_F64_PROMOTE_F32) \ + || (last_op == WASM_OP_I64_REINTERPRET_F64) \ + || (last_op == WASM_OP_F64_REINTERPRET_I64) \ + || (last_op == EXT_OP_COPY_STACK_TOP_I64) + +#define GET_CONST_OFFSET(type, val) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &val, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define GET_CONST_F32_OFFSET(type, fval) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &fval, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define GET_CONST_F64_OFFSET(type, fval) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &fval, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define emit_operand(ctx, offset) \ + do { \ + wasm_loader_emit_int16(ctx, offset); \ + LOG_OP("%d\t", offset); \ + } while (0) + +#define emit_byte(ctx, byte) \ + do { \ + wasm_loader_emit_uint8(ctx, byte); \ + LOG_OP("%d\t", byte); \ + } while (0) + +#define emit_uint32(ctx, value) \ + do { \ + wasm_loader_emit_uint32(ctx, value); \ + LOG_OP("%d\t", value); \ + } while (0) + +#define emit_uint64(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, false); \ + LOG_OP("%lld\t", value); \ + } while (0) + +#define emit_float32(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, true); \ + LOG_OP("%f\t", value); \ + } while (0) + +#define emit_float64(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, false); \ + LOG_OP("%f\t", value); \ + } while (0) + +static bool +wasm_loader_ctx_reinit(WASMLoaderContext *ctx) +{ + if (!(ctx->p_code_compiled = + loader_malloc(ctx->code_compiled_peak_size, NULL, 0))) + return false; + ctx->p_code_compiled_end = + ctx->p_code_compiled + ctx->code_compiled_peak_size; + + /* clean up frame ref */ + memset(ctx->frame_ref_bottom, 0, ctx->frame_ref_size); + ctx->frame_ref = ctx->frame_ref_bottom; + ctx->stack_cell_num = 0; + + /* clean up frame csp */ + memset(ctx->frame_csp_bottom, 0, ctx->frame_csp_size); + ctx->frame_csp = ctx->frame_csp_bottom; + ctx->csp_num = 0; + ctx->max_csp_num = 0; + + /* clean up frame offset */ + memset(ctx->frame_offset_bottom, 0, ctx->frame_offset_size); + ctx->frame_offset = ctx->frame_offset_bottom; + ctx->dynamic_offset = ctx->start_dynamic_offset; + + /* init preserved local offsets */ + ctx->preserved_local_offset = ctx->max_dynamic_offset; + + /* const buf is reserved */ + return true; +} + +static void +increase_compiled_code_space(WASMLoaderContext *ctx, int32 size) +{ + ctx->code_compiled_size += size; + if (ctx->code_compiled_size >= ctx->code_compiled_peak_size) { + ctx->code_compiled_peak_size = ctx->code_compiled_size; + } +} + +static void +wasm_loader_emit_const(WASMLoaderContext *ctx, void *value, bool is_32_bit) +{ + uint32 size = is_32_bit ? sizeof(uint32) : sizeof(uint64); + + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + bh_memcpy_s(ctx->p_code_compiled, + (uint32)(ctx->p_code_compiled_end - ctx->p_code_compiled), + value, size); + ctx->p_code_compiled += size; + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, size); + } +} + +static void +wasm_loader_emit_uint32(WASMLoaderContext *ctx, uint32 value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_U32(ctx->p_code_compiled, value); + ctx->p_code_compiled += sizeof(uint32); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(uint32)); + } +} + +static void +wasm_loader_emit_int16(WASMLoaderContext *ctx, int16 value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_U16(ctx->p_code_compiled, (uint16)value); + ctx->p_code_compiled += sizeof(int16); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(uint16)); + } +} + +static void +wasm_loader_emit_uint8(WASMLoaderContext *ctx, uint8 value) +{ + if (ctx->p_code_compiled) { + *(ctx->p_code_compiled) = value; + ctx->p_code_compiled += sizeof(uint8); +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + ctx->p_code_compiled++; + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + } + else { + increase_compiled_code_space(ctx, sizeof(uint8)); +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + increase_compiled_code_space(ctx, sizeof(uint8)); + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + } +} + +static void +wasm_loader_emit_ptr(WASMLoaderContext *ctx, void *value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_PTR(ctx->p_code_compiled, value); + ctx->p_code_compiled += sizeof(void *); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(void *)); + } +} + +static void +wasm_loader_emit_backspace(WASMLoaderContext *ctx, uint32 size) +{ + if (ctx->p_code_compiled) { + ctx->p_code_compiled -= size; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + if (size == sizeof(uint8)) { + ctx->p_code_compiled--; + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); + } +#endif + } + else { + ctx->code_compiled_size -= size; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + if (size == sizeof(uint8)) { + ctx->code_compiled_size--; + bh_assert((ctx->code_compiled_size & 1) == 0); + } +#endif + } +} + +static bool +preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode, + uint32 local_index, uint32 local_type, + bool *preserved, char *error_buf, + uint32 error_buf_size) +{ + uint32 i = 0; + int16 preserved_offset = (int16)local_index; + + *preserved = false; + while (i < loader_ctx->stack_cell_num) { + uint8 cur_type = loader_ctx->frame_ref_bottom[i]; + + /* move previous local into dynamic space before a set/tee_local opcode + */ + if (loader_ctx->frame_offset_bottom[i] == (int16)local_index) { + if (!(*preserved)) { + *preserved = true; + skip_label(); + preserved_offset = loader_ctx->preserved_local_offset; + if (loader_ctx->p_code_compiled) { + bh_assert(preserved_offset != (int16)local_index); + } + if (is_32bit_type(local_type)) { + /* Only increase preserve offset in the second traversal */ + if (loader_ctx->p_code_compiled) + loader_ctx->preserved_local_offset++; + emit_label(EXT_OP_COPY_STACK_TOP); + } + else { + if (loader_ctx->p_code_compiled) + loader_ctx->preserved_local_offset += 2; + emit_label(EXT_OP_COPY_STACK_TOP_I64); + } + emit_operand(loader_ctx, local_index); + emit_operand(loader_ctx, preserved_offset); + emit_label(opcode); + } + loader_ctx->frame_offset_bottom[i] = preserved_offset; + } + + if (is_32bit_type(cur_type)) + i++; + else + i += 2; + } + + return true; +} + +static bool +preserve_local_for_block(WASMLoaderContext *loader_ctx, uint8 opcode, + char *error_buf, uint32 error_buf_size) +{ + uint32 i = 0; + bool preserve_local; + + /* preserve locals before blocks to ensure that "tee/set_local" inside + blocks will not influence the value of these locals */ + while (i < loader_ctx->stack_cell_num) { + int16 cur_offset = loader_ctx->frame_offset_bottom[i]; + uint8 cur_type = loader_ctx->frame_ref_bottom[i]; + + if ((cur_offset < loader_ctx->start_dynamic_offset) + && (cur_offset >= 0)) { + if (!(preserve_referenced_local(loader_ctx, opcode, cur_offset, + cur_type, &preserve_local, + error_buf, error_buf_size))) + return false; + } + + if (is_32bit_type(cur_type == VALUE_TYPE_I32)) { + i++; + } + else { + i += 2; + } + } + + return true; +} + +static bool +add_label_patch_to_list(BranchBlock *frame_csp, uint8 patch_type, + uint8 *p_code_compiled, char *error_buf, + uint32 error_buf_size) +{ + BranchBlockPatch *patch = + loader_malloc(sizeof(BranchBlockPatch), error_buf, error_buf_size); + if (!patch) { + return false; + } + patch->patch_type = patch_type; + patch->code_compiled = p_code_compiled; + if (!frame_csp->patch_list) { + frame_csp->patch_list = patch; + patch->next = NULL; + } + else { + patch->next = frame_csp->patch_list; + frame_csp->patch_list = patch; + } + return true; +} + +static void +apply_label_patch(WASMLoaderContext *ctx, uint8 depth, uint8 patch_type) +{ + BranchBlock *frame_csp = ctx->frame_csp - depth; + BranchBlockPatch *node = frame_csp->patch_list; + BranchBlockPatch *node_prev = NULL, *node_next; + + if (!ctx->p_code_compiled) + return; + + while (node) { + node_next = node->next; + if (node->patch_type == patch_type) { + STORE_PTR(node->code_compiled, ctx->p_code_compiled); + if (node_prev == NULL) { + frame_csp->patch_list = node_next; + } + else { + node_prev->next = node_next; + } + wasm_runtime_free(node); + } + else { + node_prev = node; + } + node = node_next; + } +} + +static bool +wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, + bool is_br, char *error_buf, uint32 error_buf_size) +{ + /* br info layout: + * a) arity of target block + * b) total cell num of arity values + * c) each arity value's cell num + * d) each arity value's src frame offset + * e) each arity values's dst dynamic offset + * f) branch target address + * + * Note: b-e are omitted when arity is 0 so that + * interpreter can recover the br info quickly. + */ + BlockType *block_type = &frame_csp->block_type; + uint8 *types = NULL, cell; + uint32 arity = 0; + int32 i; + int16 *frame_offset = ctx->frame_offset; + uint16 dynamic_offset; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(block_type, &types); + else + arity = block_type_get_result_types(block_type, &types); + + /* Part a */ + emit_uint32(ctx, arity); + + if (arity) { + /* Part b */ + emit_uint32(ctx, wasm_get_cell_num(types, arity)); + + /* Part c */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + emit_byte(ctx, cell); + } + /* Part d */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + frame_offset -= cell; + emit_operand(ctx, *(int16 *)(frame_offset)); + } + /* Part e */ + dynamic_offset = + frame_csp->dynamic_offset + wasm_get_cell_num(types, arity); + if (is_br) + ctx->dynamic_offset = dynamic_offset; + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + dynamic_offset -= cell; + emit_operand(ctx, dynamic_offset); + } + } + + /* Part f */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) { + wasm_loader_emit_ptr(ctx, frame_csp->code_compiled); + } + else { + if (!add_label_patch_to_list(frame_csp, PATCH_END, ctx->p_code_compiled, + error_buf, error_buf_size)) + return false; + /* label address, to be patched */ + wasm_loader_emit_ptr(ctx, NULL); + } + + return true; +} + +static bool +wasm_loader_push_frame_offset(WASMLoaderContext *ctx, uint8 type, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + if (type == VALUE_TYPE_VOID) + return true; + + /* only check memory overflow in first traverse */ + if (ctx->p_code_compiled == NULL) { + if (!check_offset_push(ctx, error_buf, error_buf_size)) + return false; + } + + if (disable_emit) + *(ctx->frame_offset)++ = operand_offset; + else { + emit_operand(ctx, ctx->dynamic_offset); + *(ctx->frame_offset)++ = ctx->dynamic_offset; + ctx->dynamic_offset++; + if (ctx->dynamic_offset > ctx->max_dynamic_offset) { + ctx->max_dynamic_offset = ctx->dynamic_offset; + bh_assert(ctx->max_dynamic_offset < INT16_MAX); + } + } + + if (is_32bit_type(type)) + return true; + + if (ctx->p_code_compiled == NULL) { + if (!check_offset_push(ctx, error_buf, error_buf_size)) + return false; + } + + ctx->frame_offset++; + if (!disable_emit) { + ctx->dynamic_offset++; + if (ctx->dynamic_offset > ctx->max_dynamic_offset) { + ctx->max_dynamic_offset = ctx->dynamic_offset; + bh_assert(ctx->max_dynamic_offset < INT16_MAX); + } + } + return true; +} + +/* This function should be in front of wasm_loader_pop_frame_ref + as they both use ctx->stack_cell_num, and ctx->stack_cell_num + will be modified by wasm_loader_pop_frame_ref */ +static bool +wasm_loader_pop_frame_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size) +{ + /* if ctx->frame_csp equals ctx->frame_csp_bottom, + then current block is the function block */ + uint32 depth = ctx->frame_csp > ctx->frame_csp_bottom ? 1 : 0; + BranchBlock *cur_block = ctx->frame_csp - depth; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + + /* Directly return success if current block is in stack + * polymorphic state while stack is empty. */ + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + return true; + + if (type == VALUE_TYPE_VOID) + return true; + + if (is_32bit_type(type)) { + /* Check the offset stack bottom to ensure the frame offset + stack will not go underflow. But we don't thrown error + and return true here, because the error msg should be + given in wasm_loader_pop_frame_ref */ + if (!check_offset_pop(ctx, 1)) + return true; + + ctx->frame_offset -= 1; + if ((*(ctx->frame_offset) > ctx->start_dynamic_offset) + && (*(ctx->frame_offset) < ctx->max_dynamic_offset)) + ctx->dynamic_offset -= 1; + } + else { + if (!check_offset_pop(ctx, 2)) + return true; + + ctx->frame_offset -= 2; + if ((*(ctx->frame_offset) > ctx->start_dynamic_offset) + && (*(ctx->frame_offset) < ctx->max_dynamic_offset)) + ctx->dynamic_offset -= 2; + } + emit_operand(ctx, *(ctx->frame_offset)); + return true; +} + +static bool +wasm_loader_push_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + if (!(wasm_loader_push_frame_offset(ctx, type, disable_emit, operand_offset, + error_buf, error_buf_size))) + return false; + if (!(wasm_loader_push_frame_ref(ctx, type, error_buf, error_buf_size))) + return false; + + return true; +} + +static bool +wasm_loader_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size) +{ + /* put wasm_loader_pop_frame_offset in front of wasm_loader_pop_frame_ref */ + if (!wasm_loader_pop_frame_offset(ctx, type, error_buf, error_buf_size)) + return false; + if (!wasm_loader_pop_frame_ref(ctx, type, error_buf, error_buf_size)) + return false; + + return true; +} + +static bool +wasm_loader_push_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 pop_cnt, + uint8 type_push, uint8 type_pop, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + uint8 i; + + for (i = 0; i < pop_cnt; i++) { + if (!wasm_loader_pop_frame_offset(ctx, type_pop, error_buf, + error_buf_size)) + return false; + + if (!wasm_loader_pop_frame_ref(ctx, type_pop, error_buf, + error_buf_size)) + return false; + } + + if (!wasm_loader_push_frame_offset(ctx, type_push, disable_emit, + operand_offset, error_buf, + error_buf_size)) + return false; + + if (!wasm_loader_push_frame_ref(ctx, type_push, error_buf, error_buf_size)) + return false; + + return true; +} + +static bool +wasm_loader_get_const_offset(WASMLoaderContext *ctx, uint8 type, void *value, + int16 *offset, char *error_buf, + uint32 error_buf_size) +{ + int8 bytes_to_increase; + int16 operand_offset = 0; + Const *c; + + /* Search existing constant */ + for (c = (Const *)ctx->const_buf; + (uint8 *)c < ctx->const_buf + ctx->num_const * sizeof(Const); c++) { + if ((type == c->value_type) + && ((type == VALUE_TYPE_I64 && *(int64 *)value == c->value.i64) + || (type == VALUE_TYPE_I32 && *(int32 *)value == c->value.i32) +#if WASM_ENABLE_REF_TYPES != 0 + || (type == VALUE_TYPE_FUNCREF + && *(int32 *)value == c->value.i32) + || (type == VALUE_TYPE_EXTERNREF + && *(int32 *)value == c->value.i32) +#endif + || (type == VALUE_TYPE_F64 + && (0 == memcmp(value, &(c->value.f64), sizeof(float64)))) + || (type == VALUE_TYPE_F32 + && (0 + == memcmp(value, &(c->value.f32), sizeof(float32)))))) { + operand_offset = c->slot_index; + break; + } + if (c->value_type == VALUE_TYPE_I64 || c->value_type == VALUE_TYPE_F64) + operand_offset += 2; + else + operand_offset += 1; + } + + if ((uint8 *)c == ctx->const_buf + ctx->num_const * sizeof(Const)) { + /* New constant, append to the const buffer */ + if ((type == VALUE_TYPE_F64) || (type == VALUE_TYPE_I64)) { + bytes_to_increase = 2; + } + else { + bytes_to_increase = 1; + } + + /* The max cell num of const buffer is 32768 since the valid index range + * is -32768 ~ -1. Return an invalid index 0 to indicate the buffer is + * full */ + if (ctx->const_cell_num > INT16_MAX - bytes_to_increase + 1) { + *offset = 0; + return true; + } + + if ((uint8 *)c == ctx->const_buf + ctx->const_buf_size) { + MEM_REALLOC(ctx->const_buf, ctx->const_buf_size, + ctx->const_buf_size + 4 * sizeof(Const)); + ctx->const_buf_size += 4 * sizeof(Const); + c = (Const *)(ctx->const_buf + ctx->num_const * sizeof(Const)); + } + c->value_type = type; + switch (type) { + case VALUE_TYPE_F64: + bh_memcpy_s(&(c->value.f64), sizeof(WASMValue), value, + sizeof(float64)); + ctx->const_cell_num += 2; + /* The const buf will be reversed, we use the second cell */ + /* of the i64/f64 const so the finnal offset is corrent */ + operand_offset++; + break; + case VALUE_TYPE_I64: + c->value.i64 = *(int64 *)value; + ctx->const_cell_num += 2; + operand_offset++; + break; + case VALUE_TYPE_F32: + bh_memcpy_s(&(c->value.f32), sizeof(WASMValue), value, + sizeof(float32)); + ctx->const_cell_num++; + break; + case VALUE_TYPE_I32: + c->value.i32 = *(int32 *)value; + ctx->const_cell_num++; + break; +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_FUNCREF: + c->value.i32 = *(int32 *)value; + ctx->const_cell_num++; + break; +#endif + default: + break; + } + c->slot_index = operand_offset; + ctx->num_const++; + LOG_OP("#### new const [%d]: %ld\n", ctx->num_const, + (int64)c->value.i64); + } + /* use negetive index for const */ + operand_offset = -(operand_offset + 1); + *offset = operand_offset; + return true; +fail: + return false; +} + +/* + PUSH(POP)_XXX = push(pop) frame_ref + push(pop) frame_offset + -- Mostly used for the binary / compare operation + PUSH(POP)_OFFSET_TYPE only push(pop) the frame_offset stack + -- Mostly used in block / control instructions + + The POP will always emit the offset on the top of the frame_offset stack + PUSH can be used in two ways: + 1. directly PUSH: + PUSH_XXX(); + will allocate a dynamic space and emit + 2. silent PUSH: + operand_offset = xxx; disable_emit = true; + PUSH_XXX(); + only push the frame_offset stack, no emit +*/ +#define PUSH_I32() \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, VALUE_TYPE_I32, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_F32() \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, VALUE_TYPE_F32, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_I64() \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, VALUE_TYPE_I64, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_F64() \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, VALUE_TYPE_F64, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_FUNCREF() \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, VALUE_TYPE_FUNCREF, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_I32() \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, VALUE_TYPE_I32, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_F32() \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, VALUE_TYPE_F32, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_I64() \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, VALUE_TYPE_I64, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_F64() \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, VALUE_TYPE_F64, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_OFFSET_TYPE(type) \ + do { \ + if (!(wasm_loader_push_frame_offset(loader_ctx, type, disable_emit, \ + operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_OFFSET_TYPE(type) \ + do { \ + if (!(wasm_loader_pop_frame_offset(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_MEM_OFFSET() \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, mem_offset_type, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) +#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET() + +#define POP_MEM_OFFSET() \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, mem_offset_type, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref_offset( \ + loader_ctx, 1, type_push, type_pop, disable_emit, \ + operand_offset, error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +/* type of POPs should be the same */ +#define POP2_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref_offset( \ + loader_ctx, 2, type_push, type_pop, disable_emit, \ + operand_offset, error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#else /* WASM_ENABLE_FAST_INTERP */ + +#define PUSH_I32() \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, VALUE_TYPE_I32, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_F32() \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, VALUE_TYPE_F32, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_I64() \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, VALUE_TYPE_I64, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_F64() \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, VALUE_TYPE_F64, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_FUNCREF() \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, VALUE_TYPE_FUNCREF, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_MEM_OFFSET() \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, mem_offset_type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET() + +#define POP_I32() \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_I32, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_F32() \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_F32, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_I64() \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_I64, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_F64() \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_F64, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_FUNCREF() \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_FUNCREF, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_MEM_OFFSET() \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, mem_offset_type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 1, type_push, \ + type_pop, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +/* type of POPs should be the same */ +#define POP2_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 2, type_push, \ + type_pop, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) +#endif /* WASM_ENABLE_FAST_INTERP */ + +#if WASM_ENABLE_FAST_INTERP != 0 + +static bool +reserve_block_ret(WASMLoaderContext *loader_ctx, uint8 opcode, + bool disable_emit, char *error_buf, uint32 error_buf_size) +{ + int16 operand_offset = 0; + BranchBlock *block = (opcode == WASM_OP_ELSE) ? loader_ctx->frame_csp - 1 + : loader_ctx->frame_csp; + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0, value_count = 0, total_cel_num = 0; + int32 i = 0; + int16 dynamic_offset, dynamic_offset_org, *frame_offset = NULL, + *frame_offset_org = NULL; + + return_count = block_type_get_result_types(block_type, &return_types); + + /* If there is only one return value, use EXT_OP_COPY_STACK_TOP/_I64 instead + * of EXT_OP_COPY_STACK_VALUES for interpreter performance. */ + if (return_count == 1) { + uint8 cell = (uint8)wasm_value_type_cell_num(return_types[0]); + if (cell <= 2 /* V128 isn't supported whose cell num is 4 */ + && block->dynamic_offset != *(loader_ctx->frame_offset - cell)) { + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); + emit_label(cell == 1 ? EXT_OP_COPY_STACK_TOP + : EXT_OP_COPY_STACK_TOP_I64); + emit_operand(loader_ctx, *(loader_ctx->frame_offset - cell)); + emit_operand(loader_ctx, block->dynamic_offset); + + if (opcode == WASM_OP_ELSE) { + *(loader_ctx->frame_offset - cell) = block->dynamic_offset; + } + else { + loader_ctx->frame_offset -= cell; + loader_ctx->dynamic_offset = block->dynamic_offset; + PUSH_OFFSET_TYPE(return_types[0]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + } + return true; + } + + /* Copy stack top values to block's results which are in dynamic space. + * The instruction format: + * Part a: values count + * Part b: all values total cell num + * Part c: each value's cell_num, src offset and dst offset + * Part d: each value's src offset and dst offset + * Part e: each value's dst offset + */ + frame_offset = frame_offset_org = loader_ctx->frame_offset; + dynamic_offset = dynamic_offset_org = + block->dynamic_offset + wasm_get_cell_num(return_types, return_count); + + /* First traversal to get the count of values needed to be copied. */ + for (i = (int32)return_count - 1; i >= 0; i--) { + uint8 cells = (uint8)wasm_value_type_cell_num(return_types[i]); + + frame_offset -= cells; + dynamic_offset -= cells; + if (dynamic_offset != *frame_offset) { + value_count++; + total_cel_num += cells; + } + } + + if (value_count) { + uint32 j = 0; + uint8 *emit_data = NULL, *cells = NULL; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + uint64 size = + (uint64)value_count + * (sizeof(*cells) + sizeof(*src_offsets) + sizeof(*dst_offsets)); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + value_count); + dst_offsets = (uint16 *)(src_offsets + value_count); + + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, value_count); + /* Part b) */ + emit_uint32(loader_ctx, total_cel_num); + + /* Second traversal to get each value's cell num, src offset and dst + * offset. */ + frame_offset = frame_offset_org; + dynamic_offset = dynamic_offset_org; + for (i = (int32)return_count - 1, j = 0; i >= 0; i--) { + uint8 cell = (uint8)wasm_value_type_cell_num(return_types[i]); + frame_offset -= cell; + dynamic_offset -= cell; + if (dynamic_offset != *frame_offset) { + /* cell num */ + cells[j] = cell; + /* src offset */ + src_offsets[j] = *frame_offset; + /* dst offset */ + dst_offsets[j] = dynamic_offset; + j++; + } + if (opcode == WASM_OP_ELSE) { + *frame_offset = dynamic_offset; + } + else { + loader_ctx->frame_offset = frame_offset; + loader_ctx->dynamic_offset = dynamic_offset; + PUSH_OFFSET_TYPE(return_types[i]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + loader_ctx->frame_offset = frame_offset_org; + loader_ctx->dynamic_offset = dynamic_offset_org; + } + } + + bh_assert(j == value_count); + + /* Emit the cells, src_offsets and dst_offsets */ + for (j = 0; j < value_count; j++) + emit_byte(loader_ctx, cells[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, src_offsets[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, dst_offsets[j]); + + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + + wasm_runtime_free(emit_data); + } + + return true; + +fail: + return false; +} + +#endif /* WASM_ENABLE_FAST_INTERP */ + +#define RESERVE_BLOCK_RET() \ + do { \ + if (!reserve_block_ret(loader_ctx, opcode, disable_emit, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_TYPE(type) \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_TYPE(type) \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_CSP(label_type, block_type, _start_addr) \ + do { \ + if (!wasm_loader_push_frame_csp(loader_ctx, label_type, block_type, \ + _start_addr, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_CSP() \ + do { \ + if (!wasm_loader_pop_frame_csp(loader_ctx, error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define GET_LOCAL_INDEX_TYPE_AND_OFFSET() \ + do { \ + read_leb_uint32(p, p_end, local_idx); \ + bh_assert(local_idx < param_count + local_count); \ + local_type = local_idx < param_count \ + ? param_types[local_idx] \ + : local_types[local_idx - param_count]; \ + local_offset = local_offsets[local_idx]; \ + } while (0) + +#define CHECK_MEMORY() \ + do { \ + bh_assert(module->import_memory_count + module->memory_count > 0); \ + } while (0) + +static bool +wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, + char *error_buf, uint32 error_buf_size) +{ + BranchBlock *target_block, *cur_block; + BlockType *target_block_type; + uint8 *types = NULL, *frame_ref; + uint32 arity = 0; + int32 i, available_stack_cell; + uint16 cell_num; + + uint8 *frame_ref_old = loader_ctx->frame_ref; + uint8 *frame_ref_after_popped = NULL; + uint8 frame_ref_tmp[4] = { 0 }; + uint8 *frame_ref_buf = frame_ref_tmp; + uint32 stack_cell_num_old = loader_ctx->stack_cell_num; +#if WASM_ENABLE_FAST_INTERP != 0 + int16 *frame_offset_old = loader_ctx->frame_offset; + int16 *frame_offset_after_popped = NULL; + int16 frame_offset_tmp[4] = { 0 }; + int16 *frame_offset_buf = frame_offset_tmp; + uint16 dynamic_offset_old = (loader_ctx->frame_csp - 1)->dynamic_offset; +#endif + bool ret = false; + + bh_assert(loader_ctx->csp_num > 0); + if (loader_ctx->csp_num - 1 < depth) { + set_error_buf(error_buf, error_buf_size, + "unknown label, " + "unexpected end of section or function"); + return false; + } + + cur_block = loader_ctx->frame_csp - 1; + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + frame_ref = loader_ctx->frame_ref; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ + if (target_block->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(target_block_type, &types); + else + arity = block_type_get_result_types(target_block_type, &types); + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (cur_block->is_stack_polymorphic) { + for (i = (int32)arity - 1; i >= 0; i--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(types[i]); +#endif + POP_TYPE(types[i]); + } + + /* Backup stack data since it may be changed in the below + push operations, and the stack data may be used when + checking other target blocks of opcode br_table */ + if (opcode == WASM_OP_BR_TABLE) { + uint64 total_size; + + frame_ref_after_popped = loader_ctx->frame_ref; + total_size = (uint64)sizeof(uint8) + * (frame_ref_old - frame_ref_after_popped); + if (total_size > sizeof(frame_ref_tmp) + && !(frame_ref_buf = loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_ref_buf, (uint32)total_size, + frame_ref_after_popped, (uint32)total_size); + +#if WASM_ENABLE_FAST_INTERP != 0 + frame_offset_after_popped = loader_ctx->frame_offset; + total_size = (uint64)sizeof(int16) + * (frame_offset_old - frame_offset_after_popped); + if (total_size > sizeof(frame_offset_tmp) + && !(frame_offset_buf = loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_offset_buf, (uint32)total_size, + frame_offset_after_popped, (uint32)total_size); +#endif + } + + for (i = 0; i < (int32)arity; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(types[i]); +#endif + PUSH_TYPE(types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(target_block, opcode == WASM_OP_BR); +#endif + + /* Restore the stack data, note that frame_ref_bottom, + frame_reftype_map_bottom, frame_offset_bottom may be + re-allocated in the above push operations */ + if (opcode == WASM_OP_BR_TABLE) { + uint32 total_size; + + /* The stack operand num should not be smaller than before + after pop and push operations */ + bh_assert(loader_ctx->stack_cell_num >= stack_cell_num_old); + loader_ctx->stack_cell_num = stack_cell_num_old; + loader_ctx->frame_ref = + loader_ctx->frame_ref_bottom + stack_cell_num_old; + total_size = (uint32)sizeof(uint8) + * (frame_ref_old - frame_ref_after_popped); + bh_memcpy_s((uint8 *)loader_ctx->frame_ref - total_size, total_size, + frame_ref_buf, total_size); + +#if WASM_ENABLE_FAST_INTERP != 0 + loader_ctx->frame_offset = + loader_ctx->frame_offset_bottom + stack_cell_num_old; + total_size = (uint32)sizeof(int16) + * (frame_offset_old - frame_offset_after_popped); + bh_memcpy_s((uint8 *)loader_ctx->frame_offset - total_size, + total_size, frame_offset_buf, total_size); + (loader_ctx->frame_csp - 1)->dynamic_offset = dynamic_offset_old; +#endif + } + + ret = true; + goto cleanup_and_return; + } + + available_stack_cell = + (int32)(loader_ctx->stack_cell_num - cur_block->stack_cell_num); + + /* Check stack top values match target block type */ + for (i = (int32)arity - 1; i >= 0; i--) { + if (!check_stack_top_values(frame_ref, available_stack_cell, types[i], + error_buf, error_buf_size)) { + goto fail; + } + cell_num = wasm_value_type_cell_num(types[i]); + frame_ref -= cell_num; + available_stack_cell -= cell_num; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(target_block, opcode == WASM_OP_BR); +#endif + + ret = true; + +cleanup_and_return: +fail: + if (frame_ref_buf && frame_ref_buf != frame_ref_tmp) + wasm_runtime_free(frame_ref_buf); +#if WASM_ENABLE_FAST_INTERP != 0 + if (frame_offset_buf && frame_offset_buf != frame_offset_tmp) + wasm_runtime_free(frame_offset_buf); +#endif + + return ret; +} + +static BranchBlock * +check_branch_block(WASMLoaderContext *loader_ctx, uint8 **p_buf, uint8 *buf_end, + uint8 opcode, char *error_buf, uint32 error_buf_size) +{ + uint8 *p = *p_buf, *p_end = buf_end; + BranchBlock *frame_csp_tmp; + uint32 depth; + + read_leb_uint32(p, p_end, depth); + if (!wasm_loader_check_br(loader_ctx, depth, opcode, error_buf, + error_buf_size)) { + goto fail; + } + + frame_csp_tmp = loader_ctx->frame_csp - depth - 1; + + *p_buf = p; + return frame_csp_tmp; +fail: + return NULL; +} + +static bool +check_block_stack(WASMLoaderContext *loader_ctx, BranchBlock *block, + char *error_buf, uint32 error_buf_size) +{ + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0; + int32 available_stack_cell, return_cell_num, i; + uint8 *frame_ref = NULL; + + available_stack_cell = + (int32)(loader_ctx->stack_cell_num - block->stack_cell_num); + + return_count = block_type_get_result_types(block_type, &return_types); + return_cell_num = + return_count > 0 ? wasm_get_cell_num(return_types, return_count) : 0; + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (block->is_stack_polymorphic) { + for (i = (int32)return_count - 1; i >= 0; i--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(return_types[i]); +#endif + POP_TYPE(return_types[i]); + } + + /* Check stack is empty */ + bh_assert(loader_ctx->stack_cell_num == block->stack_cell_num); + + for (i = 0; i < (int32)return_count; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(return_types[i]); +#endif + PUSH_TYPE(return_types[i]); + } + return true; + } + + /* Check stack cell num equals return cell num */ + bh_assert(available_stack_cell == return_cell_num); + + /* Check stack values match return types */ + frame_ref = loader_ctx->frame_ref; + for (i = (int32)return_count - 1; i >= 0; i--) { + if (!check_stack_top_values(frame_ref, available_stack_cell, + return_types[i], error_buf, error_buf_size)) + return false; + frame_ref -= wasm_value_type_cell_num(return_types[i]); + available_stack_cell -= wasm_value_type_cell_num(return_types[i]); + } + + (void)return_cell_num; + return true; + +fail: + return false; +} + +#if WASM_ENABLE_FAST_INTERP != 0 +/* Copy parameters to dynamic space. + * 1) POP original parameter out; + * 2) Push and copy original values to dynamic space. + * The copy instruction format: + * Part a: param count + * Part b: all param total cell num + * Part c: each param's cell_num, src offset and dst offset + * Part d: each param's src offset + * Part e: each param's dst offset + */ +static bool +copy_params_to_dynamic_space(WASMLoaderContext *loader_ctx, bool is_if_block, + char *error_buf, uint32 error_buf_size) +{ + bool ret = false; + int16 *frame_offset = NULL; + uint8 *cells = NULL, cell; + int16 *src_offsets = NULL; + uint8 *emit_data = NULL; + uint32 i; + BranchBlock *block = loader_ctx->frame_csp - 1; + BlockType *block_type = &block->block_type; + WASMFuncType *wasm_type = block_type->u.type; + uint32 param_count = block_type->u.type->param_count; + int16 condition_offset = 0; + bool disable_emit = false; + int16 operand_offset = 0; + + uint64 size = (uint64)param_count * (sizeof(*cells) + sizeof(*src_offsets)); + bh_assert(size > 0); + + /* For if block, we also need copy the condition operand offset. */ + if (is_if_block) + size += sizeof(*cells) + sizeof(*src_offsets); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + param_count); + + if (is_if_block) + condition_offset = *loader_ctx->frame_offset; + + /* POP original parameter out */ + for (i = 0; i < param_count; i++) { + POP_OFFSET_TYPE(wasm_type->types[param_count - i - 1]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + frame_offset = loader_ctx->frame_offset; + + /* Get each param's cell num and src offset */ + for (i = 0; i < param_count; i++) { + cell = (uint8)wasm_value_type_cell_num(wasm_type->types[i]); + cells[i] = cell; + src_offsets[i] = *frame_offset; + frame_offset += cell; + } + /* emit copy instruction */ + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, is_if_block ? param_count + 1 : param_count); + /* Part b) */ + emit_uint32(loader_ctx, is_if_block ? wasm_type->param_cell_num + 1 + : wasm_type->param_cell_num); + /* Part c) */ + for (i = 0; i < param_count; i++) + emit_byte(loader_ctx, cells[i]); + if (is_if_block) + emit_byte(loader_ctx, 1); + + /* Part d) */ + for (i = 0; i < param_count; i++) + emit_operand(loader_ctx, src_offsets[i]); + if (is_if_block) + emit_operand(loader_ctx, condition_offset); + + /* Part e) */ + /* Push to dynamic space. The push will emit the dst offset. */ + for (i = 0; i < param_count; i++) + PUSH_OFFSET_TYPE(wasm_type->types[i]); + if (is_if_block) + PUSH_OFFSET_TYPE(VALUE_TYPE_I32); + + ret = true; + +fail: + /* Free the emit data */ + wasm_runtime_free(emit_data); + + return ret; +} +#endif + +/* reset the stack to the state of before entering the last block */ +#if WASM_ENABLE_FAST_INTERP != 0 +#define RESET_STACK() \ + do { \ + loader_ctx->stack_cell_num = \ + (loader_ctx->frame_csp - 1)->stack_cell_num; \ + loader_ctx->frame_ref = \ + loader_ctx->frame_ref_bottom + loader_ctx->stack_cell_num; \ + loader_ctx->frame_offset = \ + loader_ctx->frame_offset_bottom + loader_ctx->stack_cell_num; \ + } while (0) +#else +#define RESET_STACK() \ + do { \ + loader_ctx->stack_cell_num = \ + (loader_ctx->frame_csp - 1)->stack_cell_num; \ + loader_ctx->frame_ref = \ + loader_ctx->frame_ref_bottom + loader_ctx->stack_cell_num; \ + } while (0) +#endif + +/* set current block's stack polymorphic state */ +#define SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(flag) \ + do { \ + BranchBlock *_cur_block = loader_ctx->frame_csp - 1; \ + _cur_block->is_stack_polymorphic = flag; \ + } while (0) + +#define BLOCK_HAS_PARAM(block_type) \ + (!block_type.is_value_type && block_type.u.type->param_count > 0) + +#define PRESERVE_LOCAL_FOR_BLOCK() \ + do { \ + if (!(preserve_local_for_block(loader_ctx, opcode, error_buf, \ + error_buf_size))) { \ + goto fail; \ + } \ + } while (0) + +static bool +wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, + uint32 cur_func_idx, char *error_buf, + uint32 error_buf_size) +{ + uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org; + uint32 param_count, local_count, global_count; + uint8 *param_types, *local_types, local_type, global_type, mem_offset_type; + BlockType func_block_type; + uint16 *local_offsets, local_offset; + uint32 count, local_idx, global_idx, u32, align, i; + mem_offset_t mem_offset; + int32 i32, i32_const = 0; + int64 i64_const; + uint8 opcode, u8; + bool return_value = false; + WASMLoaderContext *loader_ctx; + BranchBlock *frame_csp_tmp; +#if WASM_ENABLE_BULK_MEMORY != 0 + uint32 segment_index; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *func_const_end, *func_const = NULL; + int16 operand_offset = 0; + uint8 last_op = 0; + bool disable_emit, preserve_local = false, if_condition_available = true; + ; + float32 f32_const; + float64 f64_const; + + LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n", + func->param_cell_num, func->local_cell_num, func->ret_cell_num); +#endif +#if WASM_ENABLE_MEMORY64 != 0 + bool is_memory64 = has_module_memory64(module); + mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif + + global_count = module->import_global_count + module->global_count; + + param_count = func->func_type->param_count; + param_types = func->func_type->types; + + func_block_type.is_value_type = false; + func_block_type.u.type = func->func_type; + + local_count = func->local_count; + local_types = func->local_types; + local_offsets = func->local_offsets; + + if (!(loader_ctx = wasm_loader_ctx_init(func, error_buf, error_buf_size))) { + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* For the first traverse, the initial value of preserved_local_offset has + * not been determined, we use the INT16_MAX to represent that a slot has + * been copied to preserve space. For second traverse, this field will be + * set to the appropriate value in wasm_loader_ctx_reinit. + * This is for Issue #1230, + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/1230, the + * drop opcodes need to know which slots are preserved, so those slots will + * not be treated as dynamically allocated slots */ + loader_ctx->preserved_local_offset = INT16_MAX; + +re_scan: + if (loader_ctx->code_compiled_size > 0) { + if (!wasm_loader_ctx_reinit(loader_ctx)) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + goto fail; + } + p = func->code; + func->code_compiled = loader_ctx->p_code_compiled; + func->code_compiled_size = loader_ctx->code_compiled_size; + } +#endif + + PUSH_CSP(LABEL_TYPE_FUNCTION, func_block_type, p); + + while (p < p_end) { + opcode = *p++; +#if WASM_ENABLE_FAST_INTERP != 0 + p_org = p; + disable_emit = false; + emit_label(opcode); +#endif + + switch (opcode) { + case WASM_OP_UNREACHABLE: + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + + case WASM_OP_NOP: +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); +#endif + break; + + case WASM_OP_IF: + { +#if WASM_ENABLE_FAST_INTERP != 0 + BranchBlock *parent_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - parent_block->stack_cell_num); + + if (available_stack_cell <= 0 + && parent_block->is_stack_polymorphic) + if_condition_available = false; + else + if_condition_available = true; + PRESERVE_LOCAL_FOR_BLOCK(); +#endif + POP_I32(); + goto handle_op_block_and_loop; + } + case WASM_OP_BLOCK: + case WASM_OP_LOOP: +#if WASM_ENABLE_FAST_INTERP != 0 + PRESERVE_LOCAL_FOR_BLOCK(); +#endif + handle_op_block_and_loop: + { + uint8 value_type; + BlockType block_type; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 available_params = 0; +#endif + + p_org = p - 1; + value_type = read_uint8(p); + if (is_byte_a_type(value_type)) { + /* If the first byte is one of these special values: + * 0x40/0x7F/0x7E/0x7D/0x7C, take it as the type of + * the single return value. */ + block_type.is_value_type = true; + block_type.u.value_type.type = value_type; + } + else { + uint32 type_index; + /* Resolve the leb128 encoded type index as block type */ + p--; + read_leb_uint32(p, p_end, type_index); + bh_assert(type_index < module->type_count); + block_type.is_value_type = false; + block_type.u.type = module->types[type_index]; +#if WASM_ENABLE_FAST_INTERP == 0 + /* If block use type index as block type, change the opcode + * to new extended opcode so that interpreter can resolve + * the block quickly. + */ + *p_org = EXT_OP_BLOCK + (opcode - WASM_OP_BLOCK); +#endif + } + + /* Pop block parameters from stack */ + if (BLOCK_HAS_PARAM(block_type)) { + WASMFuncType *wasm_type = block_type.u.type; + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 cell_num; + available_params = block_type.u.type->param_count; +#endif + for (i = 0; i < block_type.u.type->param_count; i++) { + + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + if (available_stack_cell <= 0 + && cur_block->is_stack_polymorphic) { +#if WASM_ENABLE_FAST_INTERP != 0 + available_params = i; +#endif + break; + } + + POP_TYPE( + wasm_type->types[wasm_type->param_count - i - 1]); +#if WASM_ENABLE_FAST_INTERP != 0 + /* decrease the frame_offset pointer accordingly to keep + * consistent with frame_ref stack */ + cell_num = wasm_value_type_cell_num( + wasm_type->types[wasm_type->param_count - i - 1]); + loader_ctx->frame_offset -= cell_num; +#endif + } + } + + PUSH_CSP(LABEL_TYPE_BLOCK + (opcode - WASM_OP_BLOCK), + block_type, p); + + /* Pass parameters to block */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 cell_num = wasm_value_type_cell_num( + block_type.u.type->types[i]); + if (i >= available_params) { + /* If there isn't enough data on stack, push a dummy + * offset to keep the stack consistent with + * frame_ref. + * Since the stack is already in polymorphic state, + * the opcode will not be executed, so the dummy + * offset won't cause any error */ + uint32 n; + + for (n = 0; n < cell_num; n++) { + if (loader_ctx->p_code_compiled == NULL) { + if (!check_offset_push(loader_ctx, + error_buf, + error_buf_size)) + goto fail; + } + *loader_ctx->frame_offset++ = 0; + } + } + else { + loader_ctx->frame_offset += cell_num; + } +#endif + PUSH_TYPE(block_type.u.type->types[i]); + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (opcode == WASM_OP_BLOCK || opcode == WASM_OP_LOOP) { + skip_label(); + if (BLOCK_HAS_PARAM(block_type)) { + /* Make sure params are in dynamic space */ + if (!copy_params_to_dynamic_space( + loader_ctx, false, error_buf, error_buf_size)) + goto fail; + } + if (opcode == WASM_OP_LOOP) { + (loader_ctx->frame_csp - 1)->code_compiled = + loader_ctx->p_code_compiled; + } + } + else if (opcode == WASM_OP_IF) { + BranchBlock *block = loader_ctx->frame_csp - 1; + /* If block has parameters, we should make sure they are in + * dynamic space. Otherwise, when else branch is missing, + * the later opcode may consume incorrect operand offset. + * Spec case: + * (func (export "params-id") (param i32) (result i32) + * (i32.const 1) + * (i32.const 2) + * (if (param i32 i32) (result i32 i32) (local.get 0) + * (then)) (i32.add) + * ) + * + * So we should emit a copy instruction before the if. + * + * And we also need to save the parameter offsets and + * recover them before entering else branch. + * + */ + if (BLOCK_HAS_PARAM(block_type)) { + uint64 size; + + /* In polymorphic state, there may be no if condition on + * the stack, so the offset may not emitted */ + if (if_condition_available) { + /* skip the if condition operand offset */ + wasm_loader_emit_backspace(loader_ctx, + sizeof(int16)); + } + /* skip the if label */ + skip_label(); + /* Emit a copy instruction */ + if (!copy_params_to_dynamic_space( + loader_ctx, true, error_buf, error_buf_size)) + goto fail; + + /* Emit the if instruction */ + emit_label(opcode); + /* Emit the new condition operand offset */ + POP_OFFSET_TYPE(VALUE_TYPE_I32); + + /* Save top param_count values of frame_offset stack, so + * that we can recover it before executing else branch + */ + size = sizeof(int16) + * (uint64)block_type.u.type->param_cell_num; + if (!(block->param_frame_offsets = loader_malloc( + size, error_buf, error_buf_size))) + goto fail; + bh_memcpy_s(block->param_frame_offsets, (uint32)size, + loader_ctx->frame_offset + - size / sizeof(int16), + (uint32)size); + } + + block->start_dynamic_offset = loader_ctx->dynamic_offset; + + emit_empty_label_addr_and_frame_ip(PATCH_ELSE); + emit_empty_label_addr_and_frame_ip(PATCH_END); + } +#endif + break; + } + + case WASM_OP_ELSE: + { + BranchBlock *block = NULL; + BlockType block_type = (loader_ctx->frame_csp - 1)->block_type; + bh_assert(loader_ctx->csp_num >= 2 + /* the matched if is found */ + && (loader_ctx->frame_csp - 1)->label_type + == LABEL_TYPE_IF + /* duplicated else isn't found */ + && !(loader_ctx->frame_csp - 1)->else_addr); + block = loader_ctx->frame_csp - 1; + + /* check whether if branch's stack matches its result type */ + if (!check_block_stack(loader_ctx, block, error_buf, + error_buf_size)) + goto fail; + + block->else_addr = p - 1; + +#if WASM_ENABLE_FAST_INTERP != 0 + /* if the result of if branch is in local or const area, add a + * copy op */ + RESERVE_BLOCK_RET(); + + emit_empty_label_addr_and_frame_ip(PATCH_END); + apply_label_patch(loader_ctx, 1, PATCH_ELSE); +#endif + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(false); + + /* Pass parameters to if-false branch */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) + PUSH_TYPE(block_type.u.type->types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Recover top param_count values of frame_offset stack */ + if (BLOCK_HAS_PARAM((block_type))) { + uint32 size; + size = sizeof(int16) * block_type.u.type->param_cell_num; + bh_memcpy_s(loader_ctx->frame_offset, size, + block->param_frame_offsets, size); + loader_ctx->frame_offset += (size / sizeof(int16)); + } + loader_ctx->dynamic_offset = block->start_dynamic_offset; +#endif + + break; + } + + case WASM_OP_END: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + /* check whether block stack matches its result type */ + if (!check_block_stack(loader_ctx, cur_block, error_buf, + error_buf_size)) + goto fail; + + /* if no else branch, and return types do not match param types, + * fail */ + if (cur_block->label_type == LABEL_TYPE_IF + && !cur_block->else_addr) { + uint32 block_param_count = 0, block_ret_count = 0; + uint8 *block_param_types = NULL, *block_ret_types = NULL; + BlockType *cur_block_type = &cur_block->block_type; + + block_param_count = block_type_get_param_types( + cur_block_type, &block_param_types); + block_ret_count = block_type_get_result_types( + cur_block_type, &block_ret_types); + bh_assert(block_param_count == block_ret_count + && (!block_param_count + || !memcmp(block_param_types, block_ret_types, + block_param_count))); + (void)block_ret_types; + (void)block_ret_count; + (void)block_param_types; + (void)block_param_count; + } + + POP_CSP(); + +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + /* copy the result to the block return address */ + RESERVE_BLOCK_RET(); + + apply_label_patch(loader_ctx, 0, PATCH_END); + free_label_patch_list(loader_ctx->frame_csp); + if (loader_ctx->frame_csp->label_type == LABEL_TYPE_FUNCTION) { + int32 idx; + uint8 ret_type; + + emit_label(WASM_OP_RETURN); + for (idx = (int32)func->func_type->result_count - 1; + idx >= 0; idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); + POP_OFFSET_TYPE(ret_type); + } + } +#endif + if (loader_ctx->csp_num > 0) { + loader_ctx->frame_csp->end_addr = p - 1; + } + else { + /* end of function block, function will return */ + bh_assert(p == p_end); + } + + break; + } + + case WASM_OP_BR: + { + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + } + + case WASM_OP_BR_IF: + { + POP_I32(); + + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + break; + } + + case WASM_OP_BR_TABLE: + { + uint8 *ret_types = NULL; + uint32 ret_count = 0; +#if WASM_ENABLE_FAST_INTERP == 0 + uint8 *p_depth_begin, *p_depth; + uint32 depth, j; + BrTableCache *br_table_cache = NULL; + + p_org = p - 1; +#endif + + read_leb_uint32(p, p_end, count); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, count); +#endif + POP_I32(); + +#if WASM_ENABLE_FAST_INTERP == 0 + p_depth_begin = p_depth = p; +#endif + for (i = 0; i <= count; i++) { + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + +#if WASM_ENABLE_FAST_INTERP == 0 + depth = (uint32)(loader_ctx->frame_csp - 1 - frame_csp_tmp); + if (br_table_cache) { + br_table_cache->br_depths[i] = depth; + } + else { + if (depth > 255) { + /* The depth cannot be stored in one byte, + create br_table cache to store each depth */ + if (!(br_table_cache = loader_malloc( + offsetof(BrTableCache, br_depths) + + sizeof(uint32) + * (uint64)(count + 1), + error_buf, error_buf_size))) { + goto fail; + } + *p_org = EXT_OP_BR_TABLE_CACHE; + br_table_cache->br_table_op_addr = p_org; + br_table_cache->br_count = count; + /* Copy previous depths which are one byte */ + for (j = 0; j < i; j++) { + br_table_cache->br_depths[j] = p_depth_begin[j]; + } + br_table_cache->br_depths[i] = depth; + bh_list_insert(module->br_table_cache_list, + br_table_cache); + } + else { + /* The depth can be stored in one byte, use the + byte of the leb to store it */ + *p_depth++ = (uint8)depth; + } + } +#endif + } + +#if WASM_ENABLE_FAST_INTERP == 0 + /* Set the tailing bytes to nop */ + if (br_table_cache) + p_depth = p_depth_begin; + while (p_depth < p) + *p_depth++ = WASM_OP_NOP; +#endif + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + (void)ret_count; + (void)ret_types; + break; + } + + case WASM_OP_RETURN: + { + int32 idx; + uint8 ret_type; + for (idx = (int32)func->func_type->result_count - 1; idx >= 0; + idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); +#if WASM_ENABLE_FAST_INTERP != 0 + /* emit the offset after return opcode */ + POP_OFFSET_TYPE(ret_type); +#endif + POP_TYPE(ret_type); + } + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + break; + } + + case WASM_OP_CALL: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: +#endif + { + WASMFuncType *func_type; + uint32 func_idx; + int32 idx; + + read_leb_uint32(p, p_end, func_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + /* we need to emit func_idx before arguments */ + emit_uint32(loader_ctx, func_idx); +#endif + + bh_assert(func_idx < module->import_function_count + + module->function_count); + + if (func_idx < module->import_function_count) + func_type = + module->import_functions[func_idx].u.function.func_type; + else + func_type = module + ->functions[func_idx + - module->import_function_count] + ->func_type; + + if (func_type->param_count > 0) { + for (idx = (int32)(func_type->param_count - 1); idx >= 0; + idx--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(func_type->types[idx]); +#endif + POP_TYPE(func_type->types[idx]); + } + } + +#if WASM_ENABLE_TAIL_CALL != 0 + if (opcode == WASM_OP_CALL) { +#endif + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Here we emit each return value's dynamic_offset. But + * in fact these offsets are continuous, so interpreter + * only need to get the first return value's offset. + */ + PUSH_OFFSET_TYPE( + func_type->types[func_type->param_count + i]); +#endif + } +#if WASM_ENABLE_TAIL_CALL != 0 + } + else { + bh_assert(func_type->result_count + == func->func_type->result_count); + for (i = 0; i < func_type->result_count; i++) { + bh_assert( + func_type->types[func_type->param_count + i] + == func->func_type + ->types[func->func_type->param_count + i]); + } + } +#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_func_call = true; +#endif + break; + } + + case WASM_OP_CALL_INDIRECT: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL_INDIRECT: +#endif + { + int32 idx; + WASMFuncType *func_type; + uint32 type_idx, table_idx; + + bh_assert(module->import_table_count + module->table_count > 0); + + read_leb_uint32(p, p_end, type_idx); + +#if WASM_ENABLE_REF_TYPES != 0 + read_leb_uint32(p, p_end, table_idx); +#else + CHECK_BUF(p, p_end, 1); + table_idx = read_uint8(p); +#endif + if (!check_table_index(module, table_idx, error_buf, + error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* we need to emit before arguments */ + emit_uint32(loader_ctx, type_idx); + emit_uint32(loader_ctx, table_idx); +#endif + + /* skip elem idx */ + POP_I32(); + + bh_assert(type_idx < module->type_count); + + func_type = module->types[type_idx]; + + if (func_type->param_count > 0) { + for (idx = (int32)(func_type->param_count - 1); idx >= 0; + idx--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(func_type->types[idx]); +#endif + POP_TYPE(func_type->types[idx]); + } + } + +#if WASM_ENABLE_TAIL_CALL != 0 + if (opcode == WASM_OP_CALL) { +#endif + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE( + func_type->types[func_type->param_count + i]); +#endif + } +#if WASM_ENABLE_TAIL_CALL != 0 + } + else { + bh_assert(func_type->result_count + == func->func_type->result_count); + for (i = 0; i < func_type->result_count; i++) { + bh_assert( + func_type->types[func_type->param_count + i] + == func->func_type + ->types[func->func_type->param_count + i]); + } + } +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_func_call = true; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_call_indirect = true; +#endif + break; + } + +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_TRY: + case WASM_OP_CATCH: + case WASM_OP_THROW: + case WASM_OP_RETHROW: + case WASM_OP_DELEGATE: + case WASM_OP_CATCH_ALL: + /* TODO */ + set_error_buf(error_buf, error_buf_size, "unsupported opcode"); + goto fail; +#endif + + case WASM_OP_DROP: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + + bh_assert(!(available_stack_cell <= 0 + && !cur_block->is_stack_polymorphic)); + + if (available_stack_cell > 0) { + if (is_32bit_type(*(loader_ctx->frame_ref - 1)) + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + loader_ctx->frame_ref--; + loader_ctx->stack_cell_num--; +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + loader_ctx->frame_offset--; + if ((*(loader_ctx->frame_offset) + > loader_ctx->start_dynamic_offset) + && (*(loader_ctx->frame_offset) + < loader_ctx->max_dynamic_offset)) + loader_ctx->dynamic_offset--; +#endif + } + else if (is_64bit_type(*(loader_ctx->frame_ref - 1))) { + loader_ctx->frame_ref -= 2; + loader_ctx->stack_cell_num -= 2; +#if WASM_ENABLE_FAST_INTERP == 0 + *(p - 1) = WASM_OP_DROP_64; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + loader_ctx->frame_offset -= 2; + if ((*(loader_ctx->frame_offset) + > loader_ctx->start_dynamic_offset) + && (*(loader_ctx->frame_offset) + < loader_ctx->max_dynamic_offset)) + loader_ctx->dynamic_offset -= 2; +#endif + } + else { + bh_assert(0); + } + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); +#endif + } + break; + } + + case WASM_OP_SELECT: + { + uint8 ref_type; + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell; +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif + + POP_I32(); + + available_stack_cell = (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + + bh_assert(!(available_stack_cell <= 0 + && !cur_block->is_stack_polymorphic)); + + if (available_stack_cell > 0) { + switch (*(loader_ctx->frame_ref - 1)) { + case REF_I32: + case REF_F32: + break; + case REF_I64_2: + case REF_F64_2: +#if WASM_ENABLE_FAST_INTERP == 0 + *(p - 1) = WASM_OP_SELECT_64; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled) { + uint8 opcode_tmp = WASM_OP_SELECT_64; +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(void **)(p_code_compiled_tmp + - sizeof(void *)) = + handle_table[opcode_tmp]; +#else +#if UINTPTR_MAX == UINT64_MAX + /* emit int32 relative offset in 64-bit target + */ + int32 offset = + (int32)((uint8 *)handle_table[opcode_tmp] + - (uint8 *)handle_table[0]); + *(int32 *)(p_code_compiled_tmp + - sizeof(int32)) = offset; +#else + /* emit uint32 label address in 32-bit target */ + *(uint32 *)(p_code_compiled_tmp + - sizeof(uint32)) = + (uint32)(uintptr_t)handle_table[opcode_tmp]; +#endif +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(p_code_compiled_tmp - 1) = opcode_tmp; +#else + *(p_code_compiled_tmp - 2) = opcode_tmp; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + } +#endif + break; + } + + ref_type = *(loader_ctx->frame_ref - 1); +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(ref_type); +#endif + POP_TYPE(ref_type); +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(ref_type); +#endif + POP_TYPE(ref_type); +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(ref_type); +#endif + PUSH_TYPE(ref_type); + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(VALUE_TYPE_ANY); +#endif + PUSH_TYPE(VALUE_TYPE_ANY); + } + break; + } + +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_SELECT_T: + { + uint8 vec_len, ref_type; +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif + + read_leb_uint32(p, p_end, vec_len); + if (vec_len != 1) { + /* typed select must have exactly one result */ + set_error_buf(error_buf, error_buf_size, + "invalid result arity"); + goto fail; + } + + CHECK_BUF(p, p_end, 1); + ref_type = read_uint8(p); + if (!is_value_type(ref_type)) { + set_error_buf(error_buf, error_buf_size, + "unknown value type"); + goto fail; + } + + POP_I32(); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled) { + uint8 opcode_tmp = WASM_OP_SELECT; + + if (ref_type == VALUE_TYPE_F64 + || ref_type == VALUE_TYPE_I64) + opcode_tmp = WASM_OP_SELECT_64; + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(void **)(p_code_compiled_tmp - sizeof(void *)) = + handle_table[opcode_tmp]; +#else +#if UINTPTR_MAX == UINT64_MAX + /* emit int32 relative offset in 64-bit target */ + int32 offset = (int32)((uint8 *)handle_table[opcode_tmp] + - (uint8 *)handle_table[0]); + *(int32 *)(p_code_compiled_tmp - sizeof(int32)) = offset; +#else + /* emit uint32 label address in 32-bit target */ + *(uint32 *)(p_code_compiled_tmp - sizeof(uint32)) = + (uint32)(uintptr_t)handle_table[opcode_tmp]; +#endif +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(p_code_compiled_tmp - 1) = opcode_tmp; +#else + *(p_code_compiled_tmp - 2) = opcode_tmp; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + } +#endif /* WASM_ENABLE_FAST_INTERP != 0 */ + +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(ref_type); + POP_TYPE(ref_type); + POP_OFFSET_TYPE(ref_type); + POP_TYPE(ref_type); + PUSH_OFFSET_TYPE(ref_type); + PUSH_TYPE(ref_type); +#else + POP2_AND_PUSH(ref_type, ref_type); +#endif /* WASM_ENABLE_FAST_INTERP != 0 */ + + (void)vec_len; + break; + } + + /* table.get x. tables[x]. [i32] -> [t] */ + /* table.set x. tables[x]. [i32 t] -> [] */ + case WASM_OP_TABLE_GET: + case WASM_OP_TABLE_SET: + { + uint8 decl_ref_type; + uint32 table_idx; + + read_leb_uint32(p, p_end, table_idx); + if (!get_table_elem_type(module, table_idx, &decl_ref_type, + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + + if (opcode == WASM_OP_TABLE_GET) { + POP_I32(); +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(decl_ref_type); +#endif + PUSH_TYPE(decl_ref_type); + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(decl_ref_type); +#endif + POP_TYPE(decl_ref_type); + POP_I32(); + } + break; + } + case WASM_OP_REF_NULL: + { + uint8 ref_type; + + CHECK_BUF(p, p_end, 1); + ref_type = read_uint8(p); + if (ref_type != VALUE_TYPE_FUNCREF + && ref_type != VALUE_TYPE_EXTERNREF) { + set_error_buf(error_buf, error_buf_size, + "unknown value type"); + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(ref_type); +#endif + PUSH_TYPE(ref_type); + break; + } + case WASM_OP_REF_IS_NULL: + { +#if WASM_ENABLE_FAST_INTERP != 0 + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 block_stack_cell_num = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + if (block_stack_cell_num <= 0) { + if (!cur_block->is_stack_polymorphic) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: expect data but stack was empty"); + goto fail; + } + } + else { + if (*(loader_ctx->frame_ref - 1) == VALUE_TYPE_FUNCREF + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_EXTERNREF + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + if (!wasm_loader_pop_frame_ref_offset( + loader_ctx, *(loader_ctx->frame_ref - 1), + error_buf, error_buf_size)) { + goto fail; + } + } + else { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + } +#else + if (!wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_FUNCREF, + error_buf, error_buf_size) + && !wasm_loader_pop_frame_ref(loader_ctx, + VALUE_TYPE_EXTERNREF, + error_buf, error_buf_size)) { + goto fail; + } +#endif + PUSH_I32(); + break; + } + case WASM_OP_REF_FUNC: + { + uint32 func_idx = 0; + read_leb_uint32(p, p_end, func_idx); + + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + + /* Refer to a forward-declared function: + the function must be an import, exported, or present in + a table elem segment or global initializer to be used as + the operand to ref.func */ + if (func_idx >= module->import_function_count) { + WASMTableSeg *table_seg = module->table_segments; + bool func_declared = false; + uint32 j; + + for (i = 0; i < module->global_count; i++) { + if (module->globals[i].type == VALUE_TYPE_FUNCREF + && module->globals[i].init_expr.init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST + && module->globals[i].init_expr.u.u32 == func_idx) { + func_declared = true; + break; + } + } + + if (!func_declared) { + /* Check whether the function is declared in table segs, + note that it doesn't matter whether the table seg's + mode is passive, active or declarative. */ + for (i = 0; i < module->table_seg_count; + i++, table_seg++) { + if (table_seg->elem_type == VALUE_TYPE_FUNCREF) { + for (j = 0; j < table_seg->value_count; j++) { + if (table_seg->init_values[j].u.ref_index + == func_idx) { + func_declared = true; + break; + } + } + } + } + } + + if (!func_declared) { + /* Check whether the function is exported */ + for (i = 0; i < module->export_count; i++) { + if (module->exports[i].kind == EXPORT_KIND_FUNC + && module->exports[i].index == func_idx) { + func_declared = true; + break; + } + } + } + bh_assert(func_declared); + (void)func_declared; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, func_idx); +#endif + PUSH_FUNCREF(); + break; + } +#endif /* WASM_ENABLE_REF_TYPES */ + + case WASM_OP_GET_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + PUSH_TYPE(local_type); + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Get Local is optimized out */ + skip_label(); + disable_emit = true; + operand_offset = local_offset; + PUSH_OFFSET_TYPE(local_type); +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) + if (local_offset < 0x80) { + *p_org++ = EXT_OP_GET_LOCAL_FAST; + if (is_32bit_type(local_type)) + *p_org++ = (uint8)local_offset; + else + *p_org++ = (uint8)(local_offset | 0x80); + while (p_org < p) + *p_org++ = WASM_OP_NOP; + } +#endif +#endif + break; + } + + case WASM_OP_SET_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!(preserve_referenced_local( + loader_ctx, opcode, local_offset, local_type, + &preserve_local, error_buf, error_buf_size))) + goto fail; + + if (local_offset < 256) { + skip_label(); + if ((!preserve_local) && (LAST_OP_OUTPUT_I32())) { + if (loader_ctx->p_code_compiled) + STORE_U16(loader_ctx->p_code_compiled - 2, + local_offset); + loader_ctx->frame_offset--; + loader_ctx->dynamic_offset--; + } + else if ((!preserve_local) && (LAST_OP_OUTPUT_I64())) { + if (loader_ctx->p_code_compiled) + STORE_U16(loader_ctx->p_code_compiled - 2, + local_offset); + loader_ctx->frame_offset -= 2; + loader_ctx->dynamic_offset -= 2; + } + else { + if (is_32bit_type(local_type)) { + emit_label(EXT_OP_SET_LOCAL_FAST); + emit_byte(loader_ctx, (uint8)local_offset); + } + else { + emit_label(EXT_OP_SET_LOCAL_FAST_I64); + emit_byte(loader_ctx, (uint8)local_offset); + } + POP_OFFSET_TYPE(local_type); + } + } + else { /* local index larger than 255, reserve leb */ + emit_uint32(loader_ctx, local_idx); + POP_OFFSET_TYPE(local_type); + } +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) + if (local_offset < 0x80) { + *p_org++ = EXT_OP_SET_LOCAL_FAST; + if (is_32bit_type(local_type)) + *p_org++ = (uint8)local_offset; + else + *p_org++ = (uint8)(local_offset | 0x80); + while (p_org < p) + *p_org++ = WASM_OP_NOP; + } +#endif +#endif + POP_TYPE(local_type); + break; + } + + case WASM_OP_TEE_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); +#if WASM_ENABLE_FAST_INTERP != 0 + /* If the stack is in polymorphic state, do fake pop and push on + offset stack to keep the depth of offset stack to be the + same with ref stack */ + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + if (cur_block->is_stack_polymorphic) { + POP_OFFSET_TYPE(local_type); + PUSH_OFFSET_TYPE(local_type); + } +#endif + POP_TYPE(local_type); + PUSH_TYPE(local_type); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!(preserve_referenced_local( + loader_ctx, opcode, local_offset, local_type, + &preserve_local, error_buf, error_buf_size))) + goto fail; + + if (local_offset < 256) { + skip_label(); + if (is_32bit_type(local_type)) { + emit_label(EXT_OP_TEE_LOCAL_FAST); + emit_byte(loader_ctx, (uint8)local_offset); + } + else { + emit_label(EXT_OP_TEE_LOCAL_FAST_I64); + emit_byte(loader_ctx, (uint8)local_offset); + } + } + else { /* local index larger than 255, reserve leb */ + emit_uint32(loader_ctx, local_idx); + } + emit_operand(loader_ctx, + *(loader_ctx->frame_offset + - wasm_value_type_cell_num(local_type))); +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) + if (local_offset < 0x80) { + *p_org++ = EXT_OP_TEE_LOCAL_FAST; + if (is_32bit_type(local_type)) + *p_org++ = (uint8)local_offset; + else + *p_org++ = (uint8)(local_offset | 0x80); + while (p_org < p) + *p_org++ = WASM_OP_NOP; + } +#endif +#endif + break; + } + + case WASM_OP_GET_GLOBAL: + { + p_org = p - 1; + read_leb_uint32(p, p_end, global_idx); + bh_assert(global_idx < global_count); + + global_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.type + : module + ->globals[global_idx + - module->import_global_count] + .type; + + PUSH_TYPE(global_type); + +#if WASM_ENABLE_FAST_INTERP == 0 + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { + *p_org = WASM_OP_GET_GLOBAL_64; + } +#else /* else of WASM_ENABLE_FAST_INTERP */ + if (is_64bit_type(global_type)) { + skip_label(); + emit_label(WASM_OP_GET_GLOBAL_64); + } + emit_uint32(loader_ctx, global_idx); + PUSH_OFFSET_TYPE(global_type); +#endif /* end of WASM_ENABLE_FAST_INTERP */ + break; + } + + case WASM_OP_SET_GLOBAL: + { + bool is_mutable = false; + + p_org = p - 1; + read_leb_uint32(p, p_end, global_idx); + bh_assert(global_idx < global_count); + + is_mutable = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.is_mutable + : module + ->globals[global_idx + - module->import_global_count] + .is_mutable; + bh_assert(is_mutable); + + global_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.type + : module + ->globals[global_idx + - module->import_global_count] + .type; + +#if WASM_ENABLE_FAST_INTERP == 0 + if (is_64bit_type(global_type)) { + *p_org = WASM_OP_SET_GLOBAL_64; + } + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { + *p_org = WASM_OP_SET_GLOBAL_AUX_STACK; +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_set_global_aux_stack = true; +#endif + } +#else /* else of WASM_ENABLE_FAST_INTERP */ + if (is_64bit_type(global_type)) { + skip_label(); + emit_label(WASM_OP_SET_GLOBAL_64); + } + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { + skip_label(); + emit_label(WASM_OP_SET_GLOBAL_AUX_STACK); + } + emit_uint32(loader_ctx, global_idx); + POP_OFFSET_TYPE(global_type); +#endif /* end of WASM_ENABLE_FAST_INTERP */ + + POP_TYPE(global_type); + + (void)is_mutable; + break; + } + + /* load */ + case WASM_OP_I32_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + case WASM_OP_I64_LOAD: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + case WASM_OP_F32_LOAD: + case WASM_OP_F64_LOAD: + /* store */ + case WASM_OP_I32_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + case WASM_OP_I64_STORE: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + case WASM_OP_F32_STORE: + case WASM_OP_F64_STORE: + { +#if WASM_ENABLE_FAST_INTERP != 0 + /* change F32/F64 into I32/I64 */ + if (opcode == WASM_OP_F32_LOAD) { + skip_label(); + emit_label(WASM_OP_I32_LOAD); + } + else if (opcode == WASM_OP_F64_LOAD) { + skip_label(); + emit_label(WASM_OP_I64_LOAD); + } + else if (opcode == WASM_OP_F32_STORE) { + skip_label(); + emit_label(WASM_OP_I32_STORE); + } + else if (opcode == WASM_OP_F64_STORE) { + skip_label(); + emit_label(WASM_OP_I64_STORE); + } +#endif + CHECK_MEMORY(); + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + switch (opcode) { + /* load */ + case WASM_OP_I32_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); + break; + case WASM_OP_I64_LOAD: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); + break; + case WASM_OP_F32_LOAD: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32); + break; + case WASM_OP_F64_LOAD: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64); + break; + /* store */ + case WASM_OP_I32_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + POP_I32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_I64_STORE: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + POP_I64(); + POP_MEM_OFFSET(); + break; + case WASM_OP_F32_STORE: + POP_F32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_F64_STORE: + POP_F64(); + POP_MEM_OFFSET(); + break; + default: + break; + } + break; + } + + case WASM_OP_MEMORY_SIZE: + CHECK_MEMORY(); + /* reserved byte 0x00 */ + bh_assert(*p == 0x00); + p++; + PUSH_PAGE_COUNT(); + + module->possible_memory_grow = true; +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + + case WASM_OP_MEMORY_GROW: + CHECK_MEMORY(); + /* reserved byte 0x00 */ + bh_assert(*p == 0x00); + p++; + POP_AND_PUSH(mem_offset_type, mem_offset_type); + + module->possible_memory_grow = true; +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_memory_grow = true; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + + case WASM_OP_I32_CONST: + read_leb_int32(p, p_end, i32_const); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + GET_CONST_OFFSET(VALUE_TYPE_I32, i32_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_I32_CONST); + emit_uint32(loader_ctx, i32_const); + } +#else + (void)i32_const; +#endif + PUSH_I32(); + break; + + case WASM_OP_I64_CONST: + read_leb_int64(p, p_end, i64_const); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + GET_CONST_OFFSET(VALUE_TYPE_I64, i64_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_I64_CONST); + emit_uint64(loader_ctx, i64_const); + } +#endif + PUSH_I64(); + break; + + case WASM_OP_F32_CONST: + CHECK_BUF(p, p_end, sizeof(float32)); + p += sizeof(float32); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + bh_memcpy_s((uint8 *)&f32_const, sizeof(float32), p_org, + sizeof(float32)); + GET_CONST_F32_OFFSET(VALUE_TYPE_F32, f32_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_F32_CONST); + emit_float32(loader_ctx, f32_const); + } +#endif + PUSH_F32(); + break; + + case WASM_OP_F64_CONST: + CHECK_BUF(p, p_end, sizeof(float64)); + p += sizeof(float64); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + /* Some MCU may require 8-byte align */ + bh_memcpy_s((uint8 *)&f64_const, sizeof(float64), p_org, + sizeof(float64)); + GET_CONST_F64_OFFSET(VALUE_TYPE_F64, f64_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_F64_CONST); + emit_float64(loader_ctx, f64_const); + } +#endif + PUSH_F64(); + break; + + case WASM_OP_I32_EQZ: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EQZ: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + POP2_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + POP2_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + POP2_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_CLZ: + case WASM_OP_I32_CTZ: + case WASM_OP_I32_POPCNT: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_CLZ: + case WASM_OP_I64_CTZ: + case WASM_OP_I64_POPCNT: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + POP2_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + case WASM_OP_F32_COPYSIGN: + POP2_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + case WASM_OP_F64_COPYSIGN: + POP2_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_WRAP_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_DEMOTE_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_PROMOTE_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_REINTERPRET_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_REINTERPRET_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_REINTERPRET_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_REINTERPRET_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_EXTEND8_S: + case WASM_OP_I32_EXTEND16_S: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EXTEND8_S: + case WASM_OP_I64_EXTEND16_S: + case WASM_OP_I64_EXTEND32_S: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, ((uint8)opcode1)); +#endif + switch (opcode1) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I64); + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + read_leb_uint32(p, p_end, segment_index); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, segment_index); +#endif + bh_assert(module->import_memory_count + + module->memory_count + > 0); + + bh_assert(*p == 0x00); + p++; + + bh_assert(segment_index < module->data_seg_count); + bh_assert(module->data_seg_count1 > 0); + + POP_I32(); + POP_I32(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + case WASM_OP_DATA_DROP: + { + read_leb_uint32(p, p_end, segment_index); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, segment_index); +#endif + bh_assert(segment_index < module->data_seg_count); + bh_assert(module->data_seg_count1 > 0); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + case WASM_OP_MEMORY_COPY: + { + CHECK_BUF(p, p_end, sizeof(int16)); + /* both src and dst memory index should be 0 */ + bh_assert(*(int16 *)p == 0x0000); + p += 2; + + bh_assert(module->import_memory_count + + module->memory_count + > 0); + + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + case WASM_OP_MEMORY_FILL: + { + bh_assert(*p == 0); + p++; + + bh_assert(module->import_memory_count + + module->memory_count + > 0); + + POP_MEM_OFFSET(); + POP_I32(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_TABLE_INIT: + { + uint8 seg_ref_type, tbl_ref_type; + uint32 table_seg_idx, table_idx; + + read_leb_uint32(p, p_end, table_seg_idx); + read_leb_uint32(p, p_end, table_idx); + + if (!get_table_elem_type(module, table_idx, + &tbl_ref_type, error_buf, + error_buf_size)) + goto fail; + + if (!get_table_seg_elem_type(module, table_seg_idx, + &seg_ref_type, error_buf, + error_buf_size)) + goto fail; + + if (seg_ref_type != tbl_ref_type) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_seg_idx); + emit_uint32(loader_ctx, table_idx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + break; + } + case WASM_OP_ELEM_DROP: + { + uint32 table_seg_idx; + read_leb_uint32(p, p_end, table_seg_idx); + if (!get_table_seg_elem_type(module, table_seg_idx, + NULL, error_buf, + error_buf_size)) + goto fail; +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_seg_idx); +#endif + break; + } + case WASM_OP_TABLE_COPY: + { + uint8 src_ref_type, dst_ref_type; + uint32 src_tbl_idx, dst_tbl_idx; + + read_leb_uint32(p, p_end, src_tbl_idx); + if (!get_table_elem_type(module, src_tbl_idx, + &src_ref_type, error_buf, + error_buf_size)) + goto fail; + + read_leb_uint32(p, p_end, dst_tbl_idx); + if (!get_table_elem_type(module, dst_tbl_idx, + &dst_ref_type, error_buf, + error_buf_size)) + goto fail; + + if (src_ref_type != dst_ref_type) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, src_tbl_idx); + emit_uint32(loader_ctx, dst_tbl_idx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + break; + } + case WASM_OP_TABLE_SIZE: + { + uint32 table_idx; + + read_leb_uint32(p, p_end, table_idx); + /* TODO: shall we create a new function to check + table idx instead of using below function? */ + if (!get_table_elem_type(module, table_idx, NULL, + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + + PUSH_I32(); + break; + } + case WASM_OP_TABLE_GROW: + case WASM_OP_TABLE_FILL: + { + uint8 decl_ref_type; + uint32 table_idx; + + read_leb_uint32(p, p_end, table_idx); + if (!get_table_elem_type(module, table_idx, + &decl_ref_type, error_buf, + error_buf_size)) + goto fail; + + if (opcode1 == WASM_OP_TABLE_GROW) { + if (table_idx < module->import_table_count) { + module->import_tables[table_idx] + .u.table.possible_grow = true; + } + else { + module + ->tables[table_idx + - module->import_table_count] + .possible_grow = true; + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + + POP_I32(); +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(decl_ref_type); +#endif + POP_TYPE(decl_ref_type); + if (opcode1 == WASM_OP_TABLE_GROW) + PUSH_I32(); + else + POP_I32(); + break; + } +#endif /* WASM_ENABLE_REF_TYPES */ + default: + bh_assert(0); + break; + } + break; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, opcode1); +#endif + if (opcode1 != WASM_OP_ATOMIC_FENCE) { + CHECK_MEMORY(); + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif + } +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + switch (opcode1) { + case WASM_OP_ATOMIC_NOTIFY: + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_WAIT32: + POP_I64(); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_WAIT64: + POP_I64(); + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_FENCE: + /* reserved byte 0x00 */ + bh_assert(*p == 0x00); + p++; + break; + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); + break; + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + POP_I32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); + break; + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + POP_I64(); + POP_MEM_OFFSET(); + break; + case WASM_OP_ATOMIC_RMW_I32_ADD: + case WASM_OP_ATOMIC_RMW_I32_ADD8_U: + case WASM_OP_ATOMIC_RMW_I32_ADD16_U: + case WASM_OP_ATOMIC_RMW_I32_SUB: + case WASM_OP_ATOMIC_RMW_I32_SUB8_U: + case WASM_OP_ATOMIC_RMW_I32_SUB16_U: + case WASM_OP_ATOMIC_RMW_I32_AND: + case WASM_OP_ATOMIC_RMW_I32_AND8_U: + case WASM_OP_ATOMIC_RMW_I32_AND16_U: + case WASM_OP_ATOMIC_RMW_I32_OR: + case WASM_OP_ATOMIC_RMW_I32_OR8_U: + case WASM_OP_ATOMIC_RMW_I32_OR16_U: + case WASM_OP_ATOMIC_RMW_I32_XOR: + case WASM_OP_ATOMIC_RMW_I32_XOR8_U: + case WASM_OP_ATOMIC_RMW_I32_XOR16_U: + case WASM_OP_ATOMIC_RMW_I32_XCHG: + case WASM_OP_ATOMIC_RMW_I32_XCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_XCHG16_U: + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_RMW_I64_ADD: + case WASM_OP_ATOMIC_RMW_I64_ADD8_U: + case WASM_OP_ATOMIC_RMW_I64_ADD16_U: + case WASM_OP_ATOMIC_RMW_I64_ADD32_U: + case WASM_OP_ATOMIC_RMW_I64_SUB: + case WASM_OP_ATOMIC_RMW_I64_SUB8_U: + case WASM_OP_ATOMIC_RMW_I64_SUB16_U: + case WASM_OP_ATOMIC_RMW_I64_SUB32_U: + case WASM_OP_ATOMIC_RMW_I64_AND: + case WASM_OP_ATOMIC_RMW_I64_AND8_U: + case WASM_OP_ATOMIC_RMW_I64_AND16_U: + case WASM_OP_ATOMIC_RMW_I64_AND32_U: + case WASM_OP_ATOMIC_RMW_I64_OR: + case WASM_OP_ATOMIC_RMW_I64_OR8_U: + case WASM_OP_ATOMIC_RMW_I64_OR16_U: + case WASM_OP_ATOMIC_RMW_I64_OR32_U: + case WASM_OP_ATOMIC_RMW_I64_XOR: + case WASM_OP_ATOMIC_RMW_I64_XOR8_U: + case WASM_OP_ATOMIC_RMW_I64_XOR16_U: + case WASM_OP_ATOMIC_RMW_I64_XOR32_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG: + case WASM_OP_ATOMIC_RMW_I64_XCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG32_U: + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I64(); + break; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + POP_I32(); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + POP_I64(); + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I64(); + break; + default: + bh_assert(0); + break; + } + break; + } +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + + default: + bh_assert(0); + break; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + last_op = opcode; +#endif + } + + if (loader_ctx->csp_num > 0) { + set_error_buf(error_buf, error_buf_size, + "function body must end with END opcode"); + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled == NULL) + goto re_scan; + + func->const_cell_num = loader_ctx->const_cell_num; + if (func->const_cell_num > 0) { + int32 j; + + if (!(func->consts = func_const = loader_malloc( + func->const_cell_num * 4, error_buf, error_buf_size))) + goto fail; + + func_const_end = func->consts + func->const_cell_num * 4; + /* reverse the const buf */ + for (j = loader_ctx->num_const - 1; j >= 0; j--) { + Const *c = (Const *)(loader_ctx->const_buf + j * sizeof(Const)); + if (c->value_type == VALUE_TYPE_F64 + || c->value_type == VALUE_TYPE_I64) { + bh_memcpy_s(func_const, (uint32)(func_const_end - func_const), + &(c->value.f64), (uint32)sizeof(int64)); + func_const += sizeof(int64); + } + else { + bh_memcpy_s(func_const, (uint32)(func_const_end - func_const), + &(c->value.f32), (uint32)sizeof(int32)); + func_const += sizeof(int32); + } + } + } + + func->max_stack_cell_num = loader_ctx->preserved_local_offset + - loader_ctx->start_dynamic_offset + 1; +#else + func->max_stack_cell_num = loader_ctx->max_stack_cell_num; +#endif + func->max_block_num = loader_ctx->max_csp_num; + return_value = true; + +fail: + wasm_loader_ctx_destroy(loader_ctx); + + (void)u8; + (void)u32; + (void)i32; + (void)i64_const; + (void)global_count; + (void)local_count; + (void)local_offset; + (void)p_org; + (void)mem_offset; + (void)align; +#if WASM_ENABLE_BULK_MEMORY != 0 + (void)segment_index; +#endif + return return_value; +} diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_opcode.h b/wasm-micro-runtime/core/iwasm/interpreter/wasm_opcode.h new file mode 100644 index 0000000..db5e5e4 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_opcode.h @@ -0,0 +1,1030 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_OPCODE_H +#define _WASM_OPCODE_H + +#include "wasm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum WASMOpcode { + /* control instructions */ + WASM_OP_UNREACHABLE = 0x00, /* unreachable */ + WASM_OP_NOP = 0x01, /* nop */ + WASM_OP_BLOCK = 0x02, /* block */ + WASM_OP_LOOP = 0x03, /* loop */ + WASM_OP_IF = 0x04, /* if */ + WASM_OP_ELSE = 0x05, /* else */ + WASM_OP_TRY = 0x06, /* try */ + WASM_OP_CATCH = 0x07, /* catch* */ + WASM_OP_THROW = 0x08, /* throw of a try catch */ + WASM_OP_RETHROW = 0x09, /* rethrow of a try catch */ + WASM_OP_UNUSED_0x0a = 0x0a, + + WASM_OP_END = 0x0b, /* end */ + WASM_OP_BR = 0x0c, /* br */ + WASM_OP_BR_IF = 0x0d, /* br if */ + WASM_OP_BR_TABLE = 0x0e, /* br table */ + WASM_OP_RETURN = 0x0f, /* return */ + WASM_OP_CALL = 0x10, /* call */ + WASM_OP_CALL_INDIRECT = 0x11, /* call_indirect */ + WASM_OP_RETURN_CALL = 0x12, /* return_call */ + WASM_OP_RETURN_CALL_INDIRECT = 0x13, /* return_call_indirect */ + WASM_OP_CALL_REF = 0x14, /* call_ref */ + WASM_OP_RETURN_CALL_REF = 0x15, /* return_call_ref */ + + WASM_OP_UNUSED_0x16 = 0x16, + WASM_OP_UNUSED_0x17 = 0x17, + + WASM_OP_DELEGATE = 0x18, /* delegate block of the try catch*/ + WASM_OP_CATCH_ALL = 0x19, /* a catch_all handler in a try block */ + + /* parametric instructions */ + WASM_OP_DROP = 0x1a, /* drop */ + WASM_OP_SELECT = 0x1b, /* select */ + WASM_OP_SELECT_T = 0x1c, /* select t */ + + WASM_OP_GET_GLOBAL_64 = 0x1d, + WASM_OP_SET_GLOBAL_64 = 0x1e, + WASM_OP_SET_GLOBAL_AUX_STACK = 0x1f, + + /* variable instructions */ + WASM_OP_GET_LOCAL = 0x20, /* get_local */ + WASM_OP_SET_LOCAL = 0x21, /* set_local */ + WASM_OP_TEE_LOCAL = 0x22, /* tee_local */ + WASM_OP_GET_GLOBAL = 0x23, /* get_global */ + WASM_OP_SET_GLOBAL = 0x24, /* set_global */ + + WASM_OP_TABLE_GET = 0x25, /* table.get */ + WASM_OP_TABLE_SET = 0x26, /* table.set */ + WASM_OP_UNUSED_0x27 = 0x27, + + /* memory instructions */ + WASM_OP_I32_LOAD = 0x28, /* i32.load */ + WASM_OP_I64_LOAD = 0x29, /* i64.load */ + WASM_OP_F32_LOAD = 0x2a, /* f32.load */ + WASM_OP_F64_LOAD = 0x2b, /* f64.load */ + WASM_OP_I32_LOAD8_S = 0x2c, /* i32.load8_s */ + WASM_OP_I32_LOAD8_U = 0x2d, /* i32.load8_u */ + WASM_OP_I32_LOAD16_S = 0x2e, /* i32.load16_s */ + WASM_OP_I32_LOAD16_U = 0x2f, /* i32.load16_u */ + WASM_OP_I64_LOAD8_S = 0x30, /* i64.load8_s */ + WASM_OP_I64_LOAD8_U = 0x31, /* i64.load8_u */ + WASM_OP_I64_LOAD16_S = 0x32, /* i64.load16_s */ + WASM_OP_I64_LOAD16_U = 0x33, /* i64.load16_u */ + WASM_OP_I64_LOAD32_S = 0x34, /* i32.load32_s */ + WASM_OP_I64_LOAD32_U = 0x35, /* i32.load32_u */ + WASM_OP_I32_STORE = 0x36, /* i32.store */ + WASM_OP_I64_STORE = 0x37, /* i64.store */ + WASM_OP_F32_STORE = 0x38, /* f32.store */ + WASM_OP_F64_STORE = 0x39, /* f64.store */ + WASM_OP_I32_STORE8 = 0x3a, /* i32.store8 */ + WASM_OP_I32_STORE16 = 0x3b, /* i32.store16 */ + WASM_OP_I64_STORE8 = 0x3c, /* i64.store8 */ + WASM_OP_I64_STORE16 = 0x3d, /* i64.sotre16 */ + WASM_OP_I64_STORE32 = 0x3e, /* i64.store32 */ + WASM_OP_MEMORY_SIZE = 0x3f, /* memory.size */ + WASM_OP_MEMORY_GROW = 0x40, /* memory.grow */ + + /* constant instructions */ + WASM_OP_I32_CONST = 0x41, /* i32.const */ + WASM_OP_I64_CONST = 0x42, /* i64.const */ + WASM_OP_F32_CONST = 0x43, /* f32.const */ + WASM_OP_F64_CONST = 0x44, /* f64.const */ + + /* comparison instructions */ + WASM_OP_I32_EQZ = 0x45, /* i32.eqz */ + WASM_OP_I32_EQ = 0x46, /* i32.eq */ + WASM_OP_I32_NE = 0x47, /* i32.ne */ + WASM_OP_I32_LT_S = 0x48, /* i32.lt_s */ + WASM_OP_I32_LT_U = 0x49, /* i32.lt_u */ + WASM_OP_I32_GT_S = 0x4a, /* i32.gt_s */ + WASM_OP_I32_GT_U = 0x4b, /* i32.gt_u */ + WASM_OP_I32_LE_S = 0x4c, /* i32.le_s */ + WASM_OP_I32_LE_U = 0x4d, /* i32.le_u */ + WASM_OP_I32_GE_S = 0x4e, /* i32.ge_s */ + WASM_OP_I32_GE_U = 0x4f, /* i32.ge_u */ + + WASM_OP_I64_EQZ = 0x50, /* i64.eqz */ + WASM_OP_I64_EQ = 0x51, /* i64.eq */ + WASM_OP_I64_NE = 0x52, /* i64.ne */ + WASM_OP_I64_LT_S = 0x53, /* i64.lt_s */ + WASM_OP_I64_LT_U = 0x54, /* i64.lt_u */ + WASM_OP_I64_GT_S = 0x55, /* i64.gt_s */ + WASM_OP_I64_GT_U = 0x56, /* i64.gt_u */ + WASM_OP_I64_LE_S = 0x57, /* i64.le_s */ + WASM_OP_I64_LE_U = 0x58, /* i64.le_u */ + WASM_OP_I64_GE_S = 0x59, /* i64.ge_s */ + WASM_OP_I64_GE_U = 0x5a, /* i64.ge_u */ + + WASM_OP_F32_EQ = 0x5b, /* f32.eq */ + WASM_OP_F32_NE = 0x5c, /* f32.ne */ + WASM_OP_F32_LT = 0x5d, /* f32.lt */ + WASM_OP_F32_GT = 0x5e, /* f32.gt */ + WASM_OP_F32_LE = 0x5f, /* f32.le */ + WASM_OP_F32_GE = 0x60, /* f32.ge */ + + WASM_OP_F64_EQ = 0x61, /* f64.eq */ + WASM_OP_F64_NE = 0x62, /* f64.ne */ + WASM_OP_F64_LT = 0x63, /* f64.lt */ + WASM_OP_F64_GT = 0x64, /* f64.gt */ + WASM_OP_F64_LE = 0x65, /* f64.le */ + WASM_OP_F64_GE = 0x66, /* f64.ge */ + + /* numeric operators */ + WASM_OP_I32_CLZ = 0x67, /* i32.clz */ + WASM_OP_I32_CTZ = 0x68, /* i32.ctz */ + WASM_OP_I32_POPCNT = 0x69, /* i32.popcnt */ + WASM_OP_I32_ADD = 0x6a, /* i32.add */ + WASM_OP_I32_SUB = 0x6b, /* i32.sub */ + WASM_OP_I32_MUL = 0x6c, /* i32.mul */ + WASM_OP_I32_DIV_S = 0x6d, /* i32.div_s */ + WASM_OP_I32_DIV_U = 0x6e, /* i32.div_u */ + WASM_OP_I32_REM_S = 0x6f, /* i32.rem_s */ + WASM_OP_I32_REM_U = 0x70, /* i32.rem_u */ + WASM_OP_I32_AND = 0x71, /* i32.and */ + WASM_OP_I32_OR = 0x72, /* i32.or */ + WASM_OP_I32_XOR = 0x73, /* i32.xor */ + WASM_OP_I32_SHL = 0x74, /* i32.shl */ + WASM_OP_I32_SHR_S = 0x75, /* i32.shr_s */ + WASM_OP_I32_SHR_U = 0x76, /* i32.shr_u */ + WASM_OP_I32_ROTL = 0x77, /* i32.rotl */ + WASM_OP_I32_ROTR = 0x78, /* i32.rotr */ + + WASM_OP_I64_CLZ = 0x79, /* i64.clz */ + WASM_OP_I64_CTZ = 0x7a, /* i64.ctz */ + WASM_OP_I64_POPCNT = 0x7b, /* i64.popcnt */ + WASM_OP_I64_ADD = 0x7c, /* i64.add */ + WASM_OP_I64_SUB = 0x7d, /* i64.sub */ + WASM_OP_I64_MUL = 0x7e, /* i64.mul */ + WASM_OP_I64_DIV_S = 0x7f, /* i64.div_s */ + WASM_OP_I64_DIV_U = 0x80, /* i64.div_u */ + WASM_OP_I64_REM_S = 0x81, /* i64.rem_s */ + WASM_OP_I64_REM_U = 0x82, /* i64.rem_u */ + WASM_OP_I64_AND = 0x83, /* i64.and */ + WASM_OP_I64_OR = 0x84, /* i64.or */ + WASM_OP_I64_XOR = 0x85, /* i64.xor */ + WASM_OP_I64_SHL = 0x86, /* i64.shl */ + WASM_OP_I64_SHR_S = 0x87, /* i64.shr_s */ + WASM_OP_I64_SHR_U = 0x88, /* i64.shr_u */ + WASM_OP_I64_ROTL = 0x89, /* i64.rotl */ + WASM_OP_I64_ROTR = 0x8a, /* i64.rotr */ + + WASM_OP_F32_ABS = 0x8b, /* f32.abs */ + WASM_OP_F32_NEG = 0x8c, /* f32.neg */ + WASM_OP_F32_CEIL = 0x8d, /* f32.ceil */ + WASM_OP_F32_FLOOR = 0x8e, /* f32.floor */ + WASM_OP_F32_TRUNC = 0x8f, /* f32.trunc */ + WASM_OP_F32_NEAREST = 0x90, /* f32.nearest */ + WASM_OP_F32_SQRT = 0x91, /* f32.sqrt */ + WASM_OP_F32_ADD = 0x92, /* f32.add */ + WASM_OP_F32_SUB = 0x93, /* f32.sub */ + WASM_OP_F32_MUL = 0x94, /* f32.mul */ + WASM_OP_F32_DIV = 0x95, /* f32.div */ + WASM_OP_F32_MIN = 0x96, /* f32.min */ + WASM_OP_F32_MAX = 0x97, /* f32.max */ + WASM_OP_F32_COPYSIGN = 0x98, /* f32.copysign */ + + WASM_OP_F64_ABS = 0x99, /* f64.abs */ + WASM_OP_F64_NEG = 0x9a, /* f64.neg */ + WASM_OP_F64_CEIL = 0x9b, /* f64.ceil */ + WASM_OP_F64_FLOOR = 0x9c, /* f64.floor */ + WASM_OP_F64_TRUNC = 0x9d, /* f64.trunc */ + WASM_OP_F64_NEAREST = 0x9e, /* f64.nearest */ + WASM_OP_F64_SQRT = 0x9f, /* f64.sqrt */ + WASM_OP_F64_ADD = 0xa0, /* f64.add */ + WASM_OP_F64_SUB = 0xa1, /* f64.sub */ + WASM_OP_F64_MUL = 0xa2, /* f64.mul */ + WASM_OP_F64_DIV = 0xa3, /* f64.div */ + WASM_OP_F64_MIN = 0xa4, /* f64.min */ + WASM_OP_F64_MAX = 0xa5, /* f64.max */ + WASM_OP_F64_COPYSIGN = 0xa6, /* f64.copysign */ + + /* conversions */ + WASM_OP_I32_WRAP_I64 = 0xa7, /* i32.wrap/i64 */ + WASM_OP_I32_TRUNC_S_F32 = 0xa8, /* i32.trunc_s/f32 */ + WASM_OP_I32_TRUNC_U_F32 = 0xa9, /* i32.trunc_u/f32 */ + WASM_OP_I32_TRUNC_S_F64 = 0xaa, /* i32.trunc_s/f64 */ + WASM_OP_I32_TRUNC_U_F64 = 0xab, /* i32.trunc_u/f64 */ + + WASM_OP_I64_EXTEND_S_I32 = 0xac, /* i64.extend_s/i32 */ + WASM_OP_I64_EXTEND_U_I32 = 0xad, /* i64.extend_u/i32 */ + WASM_OP_I64_TRUNC_S_F32 = 0xae, /* i64.trunc_s/f32 */ + WASM_OP_I64_TRUNC_U_F32 = 0xaf, /* i64.trunc_u/f32 */ + WASM_OP_I64_TRUNC_S_F64 = 0xb0, /* i64.trunc_s/f64 */ + WASM_OP_I64_TRUNC_U_F64 = 0xb1, /* i64.trunc_u/f64 */ + + WASM_OP_F32_CONVERT_S_I32 = 0xb2, /* f32.convert_s/i32 */ + WASM_OP_F32_CONVERT_U_I32 = 0xb3, /* f32.convert_u/i32 */ + WASM_OP_F32_CONVERT_S_I64 = 0xb4, /* f32.convert_s/i64 */ + WASM_OP_F32_CONVERT_U_I64 = 0xb5, /* f32.convert_u/i64 */ + WASM_OP_F32_DEMOTE_F64 = 0xb6, /* f32.demote/f64 */ + + WASM_OP_F64_CONVERT_S_I32 = 0xb7, /* f64.convert_s/i32 */ + WASM_OP_F64_CONVERT_U_I32 = 0xb8, /* f64.convert_u/i32 */ + WASM_OP_F64_CONVERT_S_I64 = 0xb9, /* f64.convert_s/i64 */ + WASM_OP_F64_CONVERT_U_I64 = 0xba, /* f64.convert_u/i64 */ + WASM_OP_F64_PROMOTE_F32 = 0xbb, /* f64.promote/f32 */ + + /* reinterpretations */ + WASM_OP_I32_REINTERPRET_F32 = 0xbc, /* i32.reinterpret/f32 */ + WASM_OP_I64_REINTERPRET_F64 = 0xbd, /* i64.reinterpret/f64 */ + WASM_OP_F32_REINTERPRET_I32 = 0xbe, /* f32.reinterpret/i32 */ + WASM_OP_F64_REINTERPRET_I64 = 0xbf, /* f64.reinterpret/i64 */ + + WASM_OP_I32_EXTEND8_S = 0xc0, /* i32.extend8_s */ + WASM_OP_I32_EXTEND16_S = 0xc1, /* i32.extend16_s */ + WASM_OP_I64_EXTEND8_S = 0xc2, /* i64.extend8_s */ + WASM_OP_I64_EXTEND16_S = 0xc3, /* i64.extend16_s */ + WASM_OP_I64_EXTEND32_S = 0xc4, /* i64.extend32_s */ + + /* drop/select specified types*/ + WASM_OP_DROP_64 = 0xc5, + WASM_OP_SELECT_64 = 0xc6, + + /* extend op code */ + EXT_OP_GET_LOCAL_FAST = 0xc7, + EXT_OP_SET_LOCAL_FAST_I64 = 0xc8, + EXT_OP_SET_LOCAL_FAST = 0xc9, + EXT_OP_TEE_LOCAL_FAST = 0xca, + EXT_OP_TEE_LOCAL_FAST_I64 = 0xcb, + EXT_OP_COPY_STACK_TOP = 0xcc, + EXT_OP_COPY_STACK_TOP_I64 = 0xcd, + EXT_OP_COPY_STACK_VALUES = 0xce, + + WASM_OP_IMPDEP = 0xcf, + + WASM_OP_REF_NULL = 0xd0, /* ref.null */ + WASM_OP_REF_IS_NULL = 0xd1, /* ref.is_null */ + WASM_OP_REF_FUNC = 0xd2, /* ref.func */ + WASM_OP_REF_EQ = 0xd3, /* ref.eq */ + WASM_OP_REF_AS_NON_NULL = 0xd4, /* ref.as_non_null */ + WASM_OP_BR_ON_NULL = 0xd5, /* br_on_null */ + WASM_OP_BR_ON_NON_NULL = 0xd6, /* br_on_non_null */ + + EXT_OP_BLOCK = 0xd7, /* block with blocktype */ + EXT_OP_LOOP = 0xd8, /* loop with blocktype */ + EXT_OP_IF = 0xd9, /* if with blocktype */ + EXT_OP_BR_TABLE_CACHE = 0xda, /* br_table from cache */ + + EXT_OP_TRY = 0xdb, /* try block with blocktype */ + +#if WASM_ENABLE_DEBUG_INTERP != 0 + DEBUG_OP_BREAK = 0xdc, /* debug break point */ +#endif + + /* Post-MVP extend op prefix */ + WASM_OP_GC_PREFIX = 0xfb, + WASM_OP_MISC_PREFIX = 0xfc, + WASM_OP_SIMD_PREFIX = 0xfd, + WASM_OP_ATOMIC_PREFIX = 0xfe, +} WASMOpcode; + +typedef enum WASMGCEXTOpcode { + WASM_OP_STRUCT_NEW = 0x00, /* struct.new */ + WASM_OP_STRUCT_NEW_DEFAULT = 0x01, /* struct.new_default */ + WASM_OP_STRUCT_GET = 0x02, /* struct.get */ + WASM_OP_STRUCT_GET_S = 0x03, /* struct.get_s */ + WASM_OP_STRUCT_GET_U = 0x04, /* struct.get_u */ + WASM_OP_STRUCT_SET = 0x05, /* struct.set */ + + WASM_OP_ARRAY_NEW = 0x06, /* array.new */ + WASM_OP_ARRAY_NEW_DEFAULT = 0x07, /* array.new_default */ + WASM_OP_ARRAY_NEW_FIXED = 0x08, /* array.new_fixed */ + WASM_OP_ARRAY_NEW_DATA = 0x09, /* array.new_data */ + WASM_OP_ARRAY_NEW_ELEM = 0x0A, /* array.new_elem */ + WASM_OP_ARRAY_GET = 0x0B, /* array.get */ + WASM_OP_ARRAY_GET_S = 0x0C, /* array.get_s */ + WASM_OP_ARRAY_GET_U = 0x0D, /* array.get_u */ + WASM_OP_ARRAY_SET = 0x0E, /* array.set */ + WASM_OP_ARRAY_LEN = 0x0F, /* array.len */ + WASM_OP_ARRAY_FILL = 0x10, /* array.fill */ + WASM_OP_ARRAY_COPY = 0x11, /* array.copy */ + WASM_OP_ARRAY_INIT_DATA = 0x12, + /* array.init_data */ /* TODO */ + WASM_OP_ARRAY_INIT_ELEM = 0x13, + /* array.init_elem */ /* TODO */ + + WASM_OP_REF_TEST = 0x14, /* ref.test */ + WASM_OP_REF_TEST_NULLABLE = 0x15, /* ref.test_nullable */ + WASM_OP_REF_CAST = 0x16, /* ref.cast */ + WASM_OP_REF_CAST_NULLABLE = 0x17, /* ref.cast_nullable */ + + WASM_OP_BR_ON_CAST = 0x18, /* br_on_cast */ + WASM_OP_BR_ON_CAST_FAIL = 0x19, /* br_on_cast_fail */ + + WASM_OP_ANY_CONVERT_EXTERN = 0x1A, /* any.convert_extern */ + WASM_OP_EXTERN_CONVERT_ANY = 0x1B, /* extern.covert_any */ + + WASM_OP_REF_I31 = 0x1C, /* ref.i31 */ + WASM_OP_I31_GET_S = 0x1D, /* i31.get_s */ + WASM_OP_I31_GET_U = 0x1E, /* i31.get_u */ + + /* stringref related opcoded */ + WASM_OP_STRING_NEW_UTF8 = 0x80, /* string.new_utf8 */ + WASM_OP_STRING_NEW_WTF16 = 0x81, /* string.new_wtf16 */ + WASM_OP_STRING_CONST = 0x82, /* string.const */ + WASM_OP_STRING_MEASURE_UTF8 = 0x83, /* string.measure_utf8 */ + WASM_OP_STRING_MEASURE_WTF8 = 0x84, /* string.measure_wtf8 */ + WASM_OP_STRING_MEASURE_WTF16 = 0x85, /* string.measure_wtf16 */ + WASM_OP_STRING_ENCODE_UTF8 = 0x86, /* string.encode_utf8 */ + WASM_OP_STRING_ENCODE_WTF16 = 0x87, /* string.encode_wtf16 */ + WASM_OP_STRING_CONCAT = 0x88, /* string.concat */ + WASM_OP_STRING_EQ = 0x89, /* string.eq */ + WASM_OP_STRING_IS_USV_SEQUENCE = 0x8a, /* string.is_usv_sequence */ + WASM_OP_STRING_NEW_LOSSY_UTF8 = 0x8b, /* string.new_lossy_utf8 */ + WASM_OP_STRING_NEW_WTF8 = 0x8c, /* string.new_wtf8 */ + WASM_OP_STRING_ENCODE_LOSSY_UTF8 = 0x8d, /* string.encode_lossy_utf8 */ + WASM_OP_STRING_ENCODE_WTF8 = 0x8e, /* string.encode_wtf8 */ + + WASM_OP_STRING_AS_WTF8 = 0x90, /* string.as_wtf8 */ + WASM_OP_STRINGVIEW_WTF8_ADVANCE = 0x91, /* stringview_wtf8.advance */ + WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8 = + 0x92, /* stringview_wtf8.encode_utf8 */ + WASM_OP_STRINGVIEW_WTF8_SLICE = 0x93, /* stringview_wtf8.slice */ + WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8 = + 0x94, /* stringview_wtf8.encode_lossy_utf8 */ + WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8 = + 0x95, /* stringview_wtf8.encode_wtf8 */ + + WASM_OP_STRING_AS_WTF16 = 0x98, /* string.as_wtf16 */ + WASM_OP_STRINGVIEW_WTF16_LENGTH = 0x99, /* stringview_wtf16.length */ + WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT = + 0x9a, /* stringview_wtf16.get_codeunit */ + WASM_OP_STRINGVIEW_WTF16_ENCODE = 0x9b, /* stringview_wtf16.encode */ + WASM_OP_STRINGVIEW_WTF16_SLICE = 0x9c, /* stringview_wtf16.slice */ + + WASM_OP_STRING_AS_ITER = 0xa0, /* string.as_iter */ + WASM_OP_STRINGVIEW_ITER_NEXT = 0xa1, /* stringview_iter.next */ + WASM_OP_STRINGVIEW_ITER_ADVANCE = 0xa2, /* stringview_iter.advance */ + WASM_OP_STRINGVIEW_ITER_REWIND = 0xa3, /* stringview_iter.rewind */ + WASM_OP_STRINGVIEW_ITER_SLICE = 0xa4, /* stringview_iter.slice */ + + WASM_OP_STRING_NEW_UTF8_ARRAY = 0xb0, /* string.new_utf8_array */ + WASM_OP_STRING_NEW_WTF16_ARRAY = 0xb1, /* string.new_wtf16_array */ + WASM_OP_STRING_ENCODE_UTF8_ARRAY = 0xb2, /* string.encode_utf8_array */ + WASM_OP_STRING_ENCODE_WTF16_ARRAY = 0xb3, /* string.encode_wtf16_array */ + WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY = + 0xb4, /* string.new_lossy_utf8_array */ + WASM_OP_STRING_NEW_WTF8_ARRAY = 0xb5, /* string.new_wtf8_array */ + WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY = + 0xb6, /* string.encode_lossy_utf8_array */ + WASM_OP_STRING_ENCODE_WTF8_ARRAY = 0xb7, /* string.encode_wtf8_array */ +} WASMGCEXTOpcode; + +typedef enum WASMMiscEXTOpcode { + WASM_OP_I32_TRUNC_SAT_S_F32 = 0x00, + WASM_OP_I32_TRUNC_SAT_U_F32 = 0x01, + WASM_OP_I32_TRUNC_SAT_S_F64 = 0x02, + WASM_OP_I32_TRUNC_SAT_U_F64 = 0x03, + WASM_OP_I64_TRUNC_SAT_S_F32 = 0x04, + WASM_OP_I64_TRUNC_SAT_U_F32 = 0x05, + WASM_OP_I64_TRUNC_SAT_S_F64 = 0x06, + WASM_OP_I64_TRUNC_SAT_U_F64 = 0x07, + WASM_OP_MEMORY_INIT = 0x08, + WASM_OP_DATA_DROP = 0x09, + WASM_OP_MEMORY_COPY = 0x0a, + WASM_OP_MEMORY_FILL = 0x0b, + WASM_OP_TABLE_INIT = 0x0c, + WASM_OP_ELEM_DROP = 0x0d, + WASM_OP_TABLE_COPY = 0x0e, + WASM_OP_TABLE_GROW = 0x0f, + WASM_OP_TABLE_SIZE = 0x10, + WASM_OP_TABLE_FILL = 0x11, +} WASMMiscEXTOpcode; + +typedef enum WASMSimdEXTOpcode { + /* memory instruction */ + SIMD_v128_load = 0x00, + SIMD_v128_load8x8_s = 0x01, + SIMD_v128_load8x8_u = 0x02, + SIMD_v128_load16x4_s = 0x03, + SIMD_v128_load16x4_u = 0x04, + SIMD_v128_load32x2_s = 0x05, + SIMD_v128_load32x2_u = 0x06, + SIMD_v128_load8_splat = 0x07, + SIMD_v128_load16_splat = 0x08, + SIMD_v128_load32_splat = 0x09, + SIMD_v128_load64_splat = 0x0a, + SIMD_v128_store = 0x0b, + + /* basic operation */ + SIMD_v128_const = 0x0c, + SIMD_v8x16_shuffle = 0x0d, + SIMD_v8x16_swizzle = 0x0e, + + /* splat operation */ + SIMD_i8x16_splat = 0x0f, + SIMD_i16x8_splat = 0x10, + SIMD_i32x4_splat = 0x11, + SIMD_i64x2_splat = 0x12, + SIMD_f32x4_splat = 0x13, + SIMD_f64x2_splat = 0x14, + + /* lane operation */ + SIMD_i8x16_extract_lane_s = 0x15, + SIMD_i8x16_extract_lane_u = 0x16, + SIMD_i8x16_replace_lane = 0x17, + SIMD_i16x8_extract_lane_s = 0x18, + SIMD_i16x8_extract_lane_u = 0x19, + SIMD_i16x8_replace_lane = 0x1a, + SIMD_i32x4_extract_lane = 0x1b, + SIMD_i32x4_replace_lane = 0x1c, + SIMD_i64x2_extract_lane = 0x1d, + SIMD_i64x2_replace_lane = 0x1e, + SIMD_f32x4_extract_lane = 0x1f, + SIMD_f32x4_replace_lane = 0x20, + SIMD_f64x2_extract_lane = 0x21, + SIMD_f64x2_replace_lane = 0x22, + + /* i8x16 compare operation */ + SIMD_i8x16_eq = 0x23, + SIMD_i8x16_ne = 0x24, + SIMD_i8x16_lt_s = 0x25, + SIMD_i8x16_lt_u = 0x26, + SIMD_i8x16_gt_s = 0x27, + SIMD_i8x16_gt_u = 0x28, + SIMD_i8x16_le_s = 0x29, + SIMD_i8x16_le_u = 0x2a, + SIMD_i8x16_ge_s = 0x2b, + SIMD_i8x16_ge_u = 0x2c, + + /* i16x8 compare operation */ + SIMD_i16x8_eq = 0x2d, + SIMD_i16x8_ne = 0x2e, + SIMD_i16x8_lt_s = 0x2f, + SIMD_i16x8_lt_u = 0x30, + SIMD_i16x8_gt_s = 0x31, + SIMD_i16x8_gt_u = 0x32, + SIMD_i16x8_le_s = 0x33, + SIMD_i16x8_le_u = 0x34, + SIMD_i16x8_ge_s = 0x35, + SIMD_i16x8_ge_u = 0x36, + + /* i32x4 compare operation */ + SIMD_i32x4_eq = 0x37, + SIMD_i32x4_ne = 0x38, + SIMD_i32x4_lt_s = 0x39, + SIMD_i32x4_lt_u = 0x3a, + SIMD_i32x4_gt_s = 0x3b, + SIMD_i32x4_gt_u = 0x3c, + SIMD_i32x4_le_s = 0x3d, + SIMD_i32x4_le_u = 0x3e, + SIMD_i32x4_ge_s = 0x3f, + SIMD_i32x4_ge_u = 0x40, + + /* f32x4 compare operation */ + SIMD_f32x4_eq = 0x41, + SIMD_f32x4_ne = 0x42, + SIMD_f32x4_lt = 0x43, + SIMD_f32x4_gt = 0x44, + SIMD_f32x4_le = 0x45, + SIMD_f32x4_ge = 0x46, + + /* f64x2 compare operation */ + SIMD_f64x2_eq = 0x47, + SIMD_f64x2_ne = 0x48, + SIMD_f64x2_lt = 0x49, + SIMD_f64x2_gt = 0x4a, + SIMD_f64x2_le = 0x4b, + SIMD_f64x2_ge = 0x4c, + + /* v128 operation */ + SIMD_v128_not = 0x4d, + SIMD_v128_and = 0x4e, + SIMD_v128_andnot = 0x4f, + SIMD_v128_or = 0x50, + SIMD_v128_xor = 0x51, + SIMD_v128_bitselect = 0x52, + SIMD_v128_any_true = 0x53, + + /* Load Lane Operation */ + SIMD_v128_load8_lane = 0x54, + SIMD_v128_load16_lane = 0x55, + SIMD_v128_load32_lane = 0x56, + SIMD_v128_load64_lane = 0x57, + SIMD_v128_store8_lane = 0x58, + SIMD_v128_store16_lane = 0x59, + SIMD_v128_store32_lane = 0x5a, + SIMD_v128_store64_lane = 0x5b, + SIMD_v128_load32_zero = 0x5c, + SIMD_v128_load64_zero = 0x5d, + + /* Float conversion */ + SIMD_f32x4_demote_f64x2_zero = 0x5e, + SIMD_f64x2_promote_low_f32x4_zero = 0x5f, + + /* i8x16 Operation */ + SIMD_i8x16_abs = 0x60, + SIMD_i8x16_neg = 0x61, + SIMD_i8x16_popcnt = 0x62, + SIMD_i8x16_all_true = 0x63, + SIMD_i8x16_bitmask = 0x64, + SIMD_i8x16_narrow_i16x8_s = 0x65, + SIMD_i8x16_narrow_i16x8_u = 0x66, + SIMD_f32x4_ceil = 0x67, + SIMD_f32x4_floor = 0x68, + SIMD_f32x4_trunc = 0x69, + SIMD_f32x4_nearest = 0x6a, + SIMD_i8x16_shl = 0x6b, + SIMD_i8x16_shr_s = 0x6c, + SIMD_i8x16_shr_u = 0x6d, + SIMD_i8x16_add = 0x6e, + SIMD_i8x16_add_sat_s = 0x6f, + SIMD_i8x16_add_sat_u = 0x70, + SIMD_i8x16_sub = 0x71, + SIMD_i8x16_sub_sat_s = 0x72, + SIMD_i8x16_sub_sat_u = 0x73, + SIMD_f64x2_ceil = 0x74, + SIMD_f64x2_floor = 0x75, + SIMD_i8x16_min_s = 0x76, + SIMD_i8x16_min_u = 0x77, + SIMD_i8x16_max_s = 0x78, + SIMD_i8x16_max_u = 0x79, + SIMD_f64x2_trunc = 0x7a, + SIMD_i8x16_avgr_u = 0x7b, + SIMD_i16x8_extadd_pairwise_i8x16_s = 0x7c, + SIMD_i16x8_extadd_pairwise_i8x16_u = 0x7d, + SIMD_i32x4_extadd_pairwise_i16x8_s = 0x7e, + SIMD_i32x4_extadd_pairwise_i16x8_u = 0x7f, + + /* i16x8 operation */ + SIMD_i16x8_abs = 0x80, + SIMD_i16x8_neg = 0x81, + SIMD_i16x8_q15mulr_sat_s = 0x82, + SIMD_i16x8_all_true = 0x83, + SIMD_i16x8_bitmask = 0x84, + SIMD_i16x8_narrow_i32x4_s = 0x85, + SIMD_i16x8_narrow_i32x4_u = 0x86, + SIMD_i16x8_extend_low_i8x16_s = 0x87, + SIMD_i16x8_extend_high_i8x16_s = 0x88, + SIMD_i16x8_extend_low_i8x16_u = 0x89, + SIMD_i16x8_extend_high_i8x16_u = 0x8a, + SIMD_i16x8_shl = 0x8b, + SIMD_i16x8_shr_s = 0x8c, + SIMD_i16x8_shr_u = 0x8d, + SIMD_i16x8_add = 0x8e, + SIMD_i16x8_add_sat_s = 0x8f, + SIMD_i16x8_add_sat_u = 0x90, + SIMD_i16x8_sub = 0x91, + SIMD_i16x8_sub_sat_s = 0x92, + SIMD_i16x8_sub_sat_u = 0x93, + SIMD_f64x2_nearest = 0x94, + SIMD_i16x8_mul = 0x95, + SIMD_i16x8_min_s = 0x96, + SIMD_i16x8_min_u = 0x97, + SIMD_i16x8_max_s = 0x98, + SIMD_i16x8_max_u = 0x99, + /* placeholder = 0x9a */ + SIMD_i16x8_avgr_u = 0x9b, + SIMD_i16x8_extmul_low_i8x16_s = 0x9c, + SIMD_i16x8_extmul_high_i8x16_s = 0x9d, + SIMD_i16x8_extmul_low_i8x16_u = 0x9e, + SIMD_i16x8_extmul_high_i8x16_u = 0x9f, + + /* i32x4 operation */ + SIMD_i32x4_abs = 0xa0, + SIMD_i32x4_neg = 0xa1, + /* placeholder = 0xa2 */ + SIMD_i32x4_all_true = 0xa3, + SIMD_i32x4_bitmask = 0xa4, + /* placeholder = 0xa5 */ + /* placeholder = 0xa6 */ + SIMD_i32x4_extend_low_i16x8_s = 0xa7, + SIMD_i32x4_extend_high_i16x8_s = 0xa8, + SIMD_i32x4_extend_low_i16x8_u = 0xa9, + SIMD_i32x4_extend_high_i16x8_u = 0xaa, + SIMD_i32x4_shl = 0xab, + SIMD_i32x4_shr_s = 0xac, + SIMD_i32x4_shr_u = 0xad, + SIMD_i32x4_add = 0xae, + /* placeholder = 0xaf */ + /* placeholder = 0xb0 */ + SIMD_i32x4_sub = 0xb1, + /* placeholder = 0xb2 */ + /* placeholder = 0xb3 */ + /* placeholder = 0xb4 */ + SIMD_i32x4_mul = 0xb5, + SIMD_i32x4_min_s = 0xb6, + SIMD_i32x4_min_u = 0xb7, + SIMD_i32x4_max_s = 0xb8, + SIMD_i32x4_max_u = 0xb9, + SIMD_i32x4_dot_i16x8_s = 0xba, + /* placeholder = 0xbb */ + SIMD_i32x4_extmul_low_i16x8_s = 0xbc, + SIMD_i32x4_extmul_high_i16x8_s = 0xbd, + SIMD_i32x4_extmul_low_i16x8_u = 0xbe, + SIMD_i32x4_extmul_high_i16x8_u = 0xbf, + + /* i64x2 operation */ + SIMD_i64x2_abs = 0xc0, + SIMD_i64x2_neg = 0xc1, + /* placeholder = 0xc2 */ + SIMD_i64x2_all_true = 0xc3, + SIMD_i64x2_bitmask = 0xc4, + /* placeholder = 0xc5 */ + /* placeholder = 0xc6 */ + SIMD_i64x2_extend_low_i32x4_s = 0xc7, + SIMD_i64x2_extend_high_i32x4_s = 0xc8, + SIMD_i64x2_extend_low_i32x4_u = 0xc9, + SIMD_i64x2_extend_high_i32x4_u = 0xca, + SIMD_i64x2_shl = 0xcb, + SIMD_i64x2_shr_s = 0xcc, + SIMD_i64x2_shr_u = 0xcd, + SIMD_i64x2_add = 0xce, + /* placeholder = 0xcf */ + /* placeholder = 0xd0 */ + SIMD_i64x2_sub = 0xd1, + /* placeholder = 0xd2 */ + /* placeholder = 0xd3 */ + /* placeholder = 0xd4 */ + SIMD_i64x2_mul = 0xd5, + SIMD_i64x2_eq = 0xd6, + SIMD_i64x2_ne = 0xd7, + SIMD_i64x2_lt_s = 0xd8, + SIMD_i64x2_gt_s = 0xd9, + SIMD_i64x2_le_s = 0xda, + SIMD_i64x2_ge_s = 0xdb, + SIMD_i64x2_extmul_low_i32x4_s = 0xdc, + SIMD_i64x2_extmul_high_i32x4_s = 0xdd, + SIMD_i64x2_extmul_low_i32x4_u = 0xde, + SIMD_i64x2_extmul_high_i32x4_u = 0xdf, + + /* f32x4 operation */ + SIMD_f32x4_abs = 0xe0, + SIMD_f32x4_neg = 0xe1, + /* placeholder = 0xe2 */ + SIMD_f32x4_sqrt = 0xe3, + SIMD_f32x4_add = 0xe4, + SIMD_f32x4_sub = 0xe5, + SIMD_f32x4_mul = 0xe6, + SIMD_f32x4_div = 0xe7, + SIMD_f32x4_min = 0xe8, + SIMD_f32x4_max = 0xe9, + SIMD_f32x4_pmin = 0xea, + SIMD_f32x4_pmax = 0xeb, + + /* f64x2 operation */ + SIMD_f64x2_abs = 0xec, + SIMD_f64x2_neg = 0xed, + /* placeholder = 0xee */ + SIMD_f64x2_sqrt = 0xef, + SIMD_f64x2_add = 0xf0, + SIMD_f64x2_sub = 0xf1, + SIMD_f64x2_mul = 0xf2, + SIMD_f64x2_div = 0xf3, + SIMD_f64x2_min = 0xf4, + SIMD_f64x2_max = 0xf5, + SIMD_f64x2_pmin = 0xf6, + SIMD_f64x2_pmax = 0xf7, + + /* conversion operation */ + SIMD_i32x4_trunc_sat_f32x4_s = 0xf8, + SIMD_i32x4_trunc_sat_f32x4_u = 0xf9, + SIMD_f32x4_convert_i32x4_s = 0xfa, + SIMD_f32x4_convert_i32x4_u = 0xfb, + SIMD_i32x4_trunc_sat_f64x2_s_zero = 0xfc, + SIMD_i32x4_trunc_sat_f64x2_u_zero = 0xfd, + SIMD_f64x2_convert_low_i32x4_s = 0xfe, + SIMD_f64x2_convert_low_i32x4_u = 0xff, +} WASMSimdEXTOpcode; + +typedef enum WASMAtomicEXTOpcode { + /* atomic wait and notify */ + WASM_OP_ATOMIC_NOTIFY = 0x00, + WASM_OP_ATOMIC_WAIT32 = 0x01, + WASM_OP_ATOMIC_WAIT64 = 0x02, + WASM_OP_ATOMIC_FENCE = 0x03, + /* atomic load and store */ + WASM_OP_ATOMIC_I32_LOAD = 0x10, + WASM_OP_ATOMIC_I64_LOAD = 0x11, + WASM_OP_ATOMIC_I32_LOAD8_U = 0x12, + WASM_OP_ATOMIC_I32_LOAD16_U = 0x13, + WASM_OP_ATOMIC_I64_LOAD8_U = 0x14, + WASM_OP_ATOMIC_I64_LOAD16_U = 0x15, + WASM_OP_ATOMIC_I64_LOAD32_U = 0x16, + WASM_OP_ATOMIC_I32_STORE = 0x17, + WASM_OP_ATOMIC_I64_STORE = 0x18, + WASM_OP_ATOMIC_I32_STORE8 = 0x19, + WASM_OP_ATOMIC_I32_STORE16 = 0x1a, + WASM_OP_ATOMIC_I64_STORE8 = 0x1b, + WASM_OP_ATOMIC_I64_STORE16 = 0x1c, + WASM_OP_ATOMIC_I64_STORE32 = 0x1d, + /* atomic add */ + WASM_OP_ATOMIC_RMW_I32_ADD = 0x1e, + WASM_OP_ATOMIC_RMW_I64_ADD = 0x1f, + WASM_OP_ATOMIC_RMW_I32_ADD8_U = 0x20, + WASM_OP_ATOMIC_RMW_I32_ADD16_U = 0x21, + WASM_OP_ATOMIC_RMW_I64_ADD8_U = 0x22, + WASM_OP_ATOMIC_RMW_I64_ADD16_U = 0x23, + WASM_OP_ATOMIC_RMW_I64_ADD32_U = 0x24, + /* atomic sub */ + WASM_OP_ATOMIC_RMW_I32_SUB = 0x25, + WASM_OP_ATOMIC_RMW_I64_SUB = 0x26, + WASM_OP_ATOMIC_RMW_I32_SUB8_U = 0x27, + WASM_OP_ATOMIC_RMW_I32_SUB16_U = 0x28, + WASM_OP_ATOMIC_RMW_I64_SUB8_U = 0x29, + WASM_OP_ATOMIC_RMW_I64_SUB16_U = 0x2a, + WASM_OP_ATOMIC_RMW_I64_SUB32_U = 0x2b, + /* atomic and */ + WASM_OP_ATOMIC_RMW_I32_AND = 0x2c, + WASM_OP_ATOMIC_RMW_I64_AND = 0x2d, + WASM_OP_ATOMIC_RMW_I32_AND8_U = 0x2e, + WASM_OP_ATOMIC_RMW_I32_AND16_U = 0x2f, + WASM_OP_ATOMIC_RMW_I64_AND8_U = 0x30, + WASM_OP_ATOMIC_RMW_I64_AND16_U = 0x31, + WASM_OP_ATOMIC_RMW_I64_AND32_U = 0x32, + /* atomic or */ + WASM_OP_ATOMIC_RMW_I32_OR = 0x33, + WASM_OP_ATOMIC_RMW_I64_OR = 0x34, + WASM_OP_ATOMIC_RMW_I32_OR8_U = 0x35, + WASM_OP_ATOMIC_RMW_I32_OR16_U = 0x36, + WASM_OP_ATOMIC_RMW_I64_OR8_U = 0x37, + WASM_OP_ATOMIC_RMW_I64_OR16_U = 0x38, + WASM_OP_ATOMIC_RMW_I64_OR32_U = 0x39, + /* atomic xor */ + WASM_OP_ATOMIC_RMW_I32_XOR = 0x3a, + WASM_OP_ATOMIC_RMW_I64_XOR = 0x3b, + WASM_OP_ATOMIC_RMW_I32_XOR8_U = 0x3c, + WASM_OP_ATOMIC_RMW_I32_XOR16_U = 0x3d, + WASM_OP_ATOMIC_RMW_I64_XOR8_U = 0x3e, + WASM_OP_ATOMIC_RMW_I64_XOR16_U = 0x3f, + WASM_OP_ATOMIC_RMW_I64_XOR32_U = 0x40, + /* atomic xchg */ + WASM_OP_ATOMIC_RMW_I32_XCHG = 0x41, + WASM_OP_ATOMIC_RMW_I64_XCHG = 0x42, + WASM_OP_ATOMIC_RMW_I32_XCHG8_U = 0x43, + WASM_OP_ATOMIC_RMW_I32_XCHG16_U = 0x44, + WASM_OP_ATOMIC_RMW_I64_XCHG8_U = 0x45, + WASM_OP_ATOMIC_RMW_I64_XCHG16_U = 0x46, + WASM_OP_ATOMIC_RMW_I64_XCHG32_U = 0x47, + /* atomic cmpxchg */ + WASM_OP_ATOMIC_RMW_I32_CMPXCHG = 0x48, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG = 0x49, + WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U = 0x4a, + WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U = 0x4b, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U = 0x4c, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U = 0x4d, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U = 0x4e, +} WASMAtomicEXTOpcode; + +#if WASM_ENABLE_DEBUG_INTERP != 0 +#define DEF_DEBUG_BREAK_HANDLE() \ + [DEBUG_OP_BREAK] = HANDLE_OPCODE(DEBUG_OP_BREAK), /* 0xdb */ +#else +#define DEF_DEBUG_BREAK_HANDLE() +#endif + +#define SET_GOTO_TABLE_ELEM(opcode) [opcode] = HANDLE_OPCODE(opcode) + +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SIMD != 0 +#define SET_GOTO_TABLE_SIMD_PREFIX_ELEM() \ + SET_GOTO_TABLE_ELEM(WASM_OP_SIMD_PREFIX), +#else +#define SET_GOTO_TABLE_SIMD_PREFIX_ELEM() +#endif + +/* + * Macro used to generate computed goto tables for the C interpreter. + */ +#define WASM_INSTRUCTION_NUM 256 + +#define DEFINE_GOTO_TABLE(type, _name) \ + static type _name[WASM_INSTRUCTION_NUM] = { \ + HANDLE_OPCODE(WASM_OP_UNREACHABLE), /* 0x00 */ \ + HANDLE_OPCODE(WASM_OP_NOP), /* 0x01 */ \ + HANDLE_OPCODE(WASM_OP_BLOCK), /* 0x02 */ \ + HANDLE_OPCODE(WASM_OP_LOOP), /* 0x03 */ \ + HANDLE_OPCODE(WASM_OP_IF), /* 0x04 */ \ + HANDLE_OPCODE(WASM_OP_ELSE), /* 0x05 */ \ + HANDLE_OPCODE(WASM_OP_TRY), /* 0x06 */ \ + HANDLE_OPCODE(WASM_OP_CATCH), /* 0x07 */ \ + HANDLE_OPCODE(WASM_OP_THROW), /* 0x08 */ \ + HANDLE_OPCODE(WASM_OP_RETHROW), /* 0x09 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x0a), /* 0x0a */ \ + HANDLE_OPCODE(WASM_OP_END), /* 0x0b */ \ + HANDLE_OPCODE(WASM_OP_BR), /* 0x0c */ \ + HANDLE_OPCODE(WASM_OP_BR_IF), /* 0x0d */ \ + HANDLE_OPCODE(WASM_OP_BR_TABLE), /* 0x0e */ \ + HANDLE_OPCODE(WASM_OP_RETURN), /* 0x0f */ \ + HANDLE_OPCODE(WASM_OP_CALL), /* 0x10 */ \ + HANDLE_OPCODE(WASM_OP_CALL_INDIRECT), /* 0x11 */ \ + HANDLE_OPCODE(WASM_OP_RETURN_CALL), /* 0x12 */ \ + HANDLE_OPCODE(WASM_OP_RETURN_CALL_INDIRECT), /* 0x13 */ \ + HANDLE_OPCODE(WASM_OP_CALL_REF), /* 0x14 */ \ + HANDLE_OPCODE(WASM_OP_RETURN_CALL_REF), /* 0x15 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x16), /* 0x16 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x17), /* 0x17 */ \ + HANDLE_OPCODE(WASM_OP_DELEGATE), /* 0x18 */ \ + HANDLE_OPCODE(WASM_OP_CATCH_ALL), /* 0x19 */ \ + HANDLE_OPCODE(WASM_OP_DROP), /* 0x1a */ \ + HANDLE_OPCODE(WASM_OP_SELECT), /* 0x1b */ \ + HANDLE_OPCODE(WASM_OP_SELECT_T), /* 0x1c */ \ + HANDLE_OPCODE(WASM_OP_GET_GLOBAL_64), /* 0x1d */ \ + HANDLE_OPCODE(WASM_OP_SET_GLOBAL_64), /* 0x1e */ \ + HANDLE_OPCODE(WASM_OP_SET_GLOBAL_AUX_STACK), /* 0x1f */ \ + HANDLE_OPCODE(WASM_OP_GET_LOCAL), /* 0x20 */ \ + HANDLE_OPCODE(WASM_OP_SET_LOCAL), /* 0x21 */ \ + HANDLE_OPCODE(WASM_OP_TEE_LOCAL), /* 0x22 */ \ + HANDLE_OPCODE(WASM_OP_GET_GLOBAL), /* 0x23 */ \ + HANDLE_OPCODE(WASM_OP_SET_GLOBAL), /* 0x24 */ \ + HANDLE_OPCODE(WASM_OP_TABLE_GET), /* 0x25 */ \ + HANDLE_OPCODE(WASM_OP_TABLE_SET), /* 0x26 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x27), /* 0x27 */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD), /* 0x28 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD), /* 0x29 */ \ + HANDLE_OPCODE(WASM_OP_F32_LOAD), /* 0x2a */ \ + HANDLE_OPCODE(WASM_OP_F64_LOAD), /* 0x2b */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD8_S), /* 0x2c */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD8_U), /* 0x2d */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD16_S), /* 0x2e */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD16_U), /* 0x2f */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD8_S), /* 0x30 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD8_U), /* 0x31 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD16_S), /* 0x32 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD16_U), /* 0x33 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD32_S), /* 0x34 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD32_U), /* 0x35 */ \ + HANDLE_OPCODE(WASM_OP_I32_STORE), /* 0x36 */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE), /* 0x37 */ \ + HANDLE_OPCODE(WASM_OP_F32_STORE), /* 0x38 */ \ + HANDLE_OPCODE(WASM_OP_F64_STORE), /* 0x39 */ \ + HANDLE_OPCODE(WASM_OP_I32_STORE8), /* 0x3a */ \ + HANDLE_OPCODE(WASM_OP_I32_STORE16), /* 0x3b */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE8), /* 0x3c */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE16), /* 0x3d */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE32), /* 0x3e */ \ + HANDLE_OPCODE(WASM_OP_MEMORY_SIZE), /* 0x3f */ \ + HANDLE_OPCODE(WASM_OP_MEMORY_GROW), /* 0x40 */ \ + HANDLE_OPCODE(WASM_OP_I32_CONST), /* 0x41 */ \ + HANDLE_OPCODE(WASM_OP_I64_CONST), /* 0x42 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONST), /* 0x43 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONST), /* 0x44 */ \ + HANDLE_OPCODE(WASM_OP_I32_EQZ), /* 0x45 */ \ + HANDLE_OPCODE(WASM_OP_I32_EQ), /* 0x46 */ \ + HANDLE_OPCODE(WASM_OP_I32_NE), /* 0x47 */ \ + HANDLE_OPCODE(WASM_OP_I32_LT_S), /* 0x48 */ \ + HANDLE_OPCODE(WASM_OP_I32_LT_U), /* 0x49 */ \ + HANDLE_OPCODE(WASM_OP_I32_GT_S), /* 0x4a */ \ + HANDLE_OPCODE(WASM_OP_I32_GT_U), /* 0x4b */ \ + HANDLE_OPCODE(WASM_OP_I32_LE_S), /* 0x4c */ \ + HANDLE_OPCODE(WASM_OP_I32_LE_U), /* 0x4d */ \ + HANDLE_OPCODE(WASM_OP_I32_GE_S), /* 0x4e */ \ + HANDLE_OPCODE(WASM_OP_I32_GE_U), /* 0x4f */ \ + HANDLE_OPCODE(WASM_OP_I64_EQZ), /* 0x50 */ \ + HANDLE_OPCODE(WASM_OP_I64_EQ), /* 0x51 */ \ + HANDLE_OPCODE(WASM_OP_I64_NE), /* 0x52 */ \ + HANDLE_OPCODE(WASM_OP_I64_LT_S), /* 0x53 */ \ + HANDLE_OPCODE(WASM_OP_I64_LT_U), /* 0x54 */ \ + HANDLE_OPCODE(WASM_OP_I64_GT_S), /* 0x55 */ \ + HANDLE_OPCODE(WASM_OP_I64_GT_U), /* 0x56 */ \ + HANDLE_OPCODE(WASM_OP_I64_LE_S), /* 0x57 */ \ + HANDLE_OPCODE(WASM_OP_I64_LE_U), /* 0x58 */ \ + HANDLE_OPCODE(WASM_OP_I64_GE_S), /* 0x59 */ \ + HANDLE_OPCODE(WASM_OP_I64_GE_U), /* 0x5a */ \ + HANDLE_OPCODE(WASM_OP_F32_EQ), /* 0x5b */ \ + HANDLE_OPCODE(WASM_OP_F32_NE), /* 0x5c */ \ + HANDLE_OPCODE(WASM_OP_F32_LT), /* 0x5d */ \ + HANDLE_OPCODE(WASM_OP_F32_GT), /* 0x5e */ \ + HANDLE_OPCODE(WASM_OP_F32_LE), /* 0x5f */ \ + HANDLE_OPCODE(WASM_OP_F32_GE), /* 0x60 */ \ + HANDLE_OPCODE(WASM_OP_F64_EQ), /* 0x61 */ \ + HANDLE_OPCODE(WASM_OP_F64_NE), /* 0x62 */ \ + HANDLE_OPCODE(WASM_OP_F64_LT), /* 0x63 */ \ + HANDLE_OPCODE(WASM_OP_F64_GT), /* 0x64 */ \ + HANDLE_OPCODE(WASM_OP_F64_LE), /* 0x65 */ \ + HANDLE_OPCODE(WASM_OP_F64_GE), /* 0x66 */ \ + HANDLE_OPCODE(WASM_OP_I32_CLZ), /* 0x67 */ \ + HANDLE_OPCODE(WASM_OP_I32_CTZ), /* 0x68 */ \ + HANDLE_OPCODE(WASM_OP_I32_POPCNT), /* 0x69 */ \ + HANDLE_OPCODE(WASM_OP_I32_ADD), /* 0x6a */ \ + HANDLE_OPCODE(WASM_OP_I32_SUB), /* 0x6b */ \ + HANDLE_OPCODE(WASM_OP_I32_MUL), /* 0x6c */ \ + HANDLE_OPCODE(WASM_OP_I32_DIV_S), /* 0x6d */ \ + HANDLE_OPCODE(WASM_OP_I32_DIV_U), /* 0x6e */ \ + HANDLE_OPCODE(WASM_OP_I32_REM_S), /* 0x6f */ \ + HANDLE_OPCODE(WASM_OP_I32_REM_U), /* 0x70 */ \ + HANDLE_OPCODE(WASM_OP_I32_AND), /* 0x71 */ \ + HANDLE_OPCODE(WASM_OP_I32_OR), /* 0x72 */ \ + HANDLE_OPCODE(WASM_OP_I32_XOR), /* 0x73 */ \ + HANDLE_OPCODE(WASM_OP_I32_SHL), /* 0x74 */ \ + HANDLE_OPCODE(WASM_OP_I32_SHR_S), /* 0x75 */ \ + HANDLE_OPCODE(WASM_OP_I32_SHR_U), /* 0x76 */ \ + HANDLE_OPCODE(WASM_OP_I32_ROTL), /* 0x77 */ \ + HANDLE_OPCODE(WASM_OP_I32_ROTR), /* 0x78 */ \ + HANDLE_OPCODE(WASM_OP_I64_CLZ), /* 0x79 */ \ + HANDLE_OPCODE(WASM_OP_I64_CTZ), /* 0x7a */ \ + HANDLE_OPCODE(WASM_OP_I64_POPCNT), /* 0x7b */ \ + HANDLE_OPCODE(WASM_OP_I64_ADD), /* 0x7c */ \ + HANDLE_OPCODE(WASM_OP_I64_SUB), /* 0x7d */ \ + HANDLE_OPCODE(WASM_OP_I64_MUL), /* 0x7e */ \ + HANDLE_OPCODE(WASM_OP_I64_DIV_S), /* 0x7f */ \ + HANDLE_OPCODE(WASM_OP_I64_DIV_U), /* 0x80 */ \ + HANDLE_OPCODE(WASM_OP_I64_REM_S), /* 0x81 */ \ + HANDLE_OPCODE(WASM_OP_I64_REM_U), /* 0x82 */ \ + HANDLE_OPCODE(WASM_OP_I64_AND), /* 0x83 */ \ + HANDLE_OPCODE(WASM_OP_I64_OR), /* 0x84 */ \ + HANDLE_OPCODE(WASM_OP_I64_XOR), /* 0x85 */ \ + HANDLE_OPCODE(WASM_OP_I64_SHL), /* 0x86 */ \ + HANDLE_OPCODE(WASM_OP_I64_SHR_S), /* 0x87 */ \ + HANDLE_OPCODE(WASM_OP_I64_SHR_U), /* 0x88 */ \ + HANDLE_OPCODE(WASM_OP_I64_ROTL), /* 0x89 */ \ + HANDLE_OPCODE(WASM_OP_I64_ROTR), /* 0x8a */ \ + HANDLE_OPCODE(WASM_OP_F32_ABS), /* 0x8b */ \ + HANDLE_OPCODE(WASM_OP_F32_NEG), /* 0x8c */ \ + HANDLE_OPCODE(WASM_OP_F32_CEIL), /* 0x8d */ \ + HANDLE_OPCODE(WASM_OP_F32_FLOOR), /* 0x8e */ \ + HANDLE_OPCODE(WASM_OP_F32_TRUNC), /* 0x8f */ \ + HANDLE_OPCODE(WASM_OP_F32_NEAREST), /* 0x90 */ \ + HANDLE_OPCODE(WASM_OP_F32_SQRT), /* 0x91 */ \ + HANDLE_OPCODE(WASM_OP_F32_ADD), /* 0x92 */ \ + HANDLE_OPCODE(WASM_OP_F32_SUB), /* 0x93 */ \ + HANDLE_OPCODE(WASM_OP_F32_MUL), /* 0x94 */ \ + HANDLE_OPCODE(WASM_OP_F32_DIV), /* 0x95 */ \ + HANDLE_OPCODE(WASM_OP_F32_MIN), /* 0x96 */ \ + HANDLE_OPCODE(WASM_OP_F32_MAX), /* 0x97 */ \ + HANDLE_OPCODE(WASM_OP_F32_COPYSIGN), /* 0x98 */ \ + HANDLE_OPCODE(WASM_OP_F64_ABS), /* 0x99 */ \ + HANDLE_OPCODE(WASM_OP_F64_NEG), /* 0x9a */ \ + HANDLE_OPCODE(WASM_OP_F64_CEIL), /* 0x9b */ \ + HANDLE_OPCODE(WASM_OP_F64_FLOOR), /* 0x9c */ \ + HANDLE_OPCODE(WASM_OP_F64_TRUNC), /* 0x9d */ \ + HANDLE_OPCODE(WASM_OP_F64_NEAREST), /* 0x9e */ \ + HANDLE_OPCODE(WASM_OP_F64_SQRT), /* 0x9f */ \ + HANDLE_OPCODE(WASM_OP_F64_ADD), /* 0xa0 */ \ + HANDLE_OPCODE(WASM_OP_F64_SUB), /* 0xa1 */ \ + HANDLE_OPCODE(WASM_OP_F64_MUL), /* 0xa2 */ \ + HANDLE_OPCODE(WASM_OP_F64_DIV), /* 0xa3 */ \ + HANDLE_OPCODE(WASM_OP_F64_MIN), /* 0xa4 */ \ + HANDLE_OPCODE(WASM_OP_F64_MAX), /* 0xa5 */ \ + HANDLE_OPCODE(WASM_OP_F64_COPYSIGN), /* 0xa6 */ \ + HANDLE_OPCODE(WASM_OP_I32_WRAP_I64), /* 0xa7 */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_S_F32), /* 0xa8 */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_U_F32), /* 0xa9 */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_S_F64), /* 0xaa */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_U_F64), /* 0xab */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND_S_I32), /* 0xac */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND_U_I32), /* 0xad */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_S_F32), /* 0xae */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_U_F32), /* 0xaf */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_S_F64), /* 0xb0 */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_U_F64), /* 0xb1 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_S_I32), /* 0xb2 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_U_I32), /* 0xb3 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_S_I64), /* 0xb4 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_U_I64), /* 0xb5 */ \ + HANDLE_OPCODE(WASM_OP_F32_DEMOTE_F64), /* 0xb6 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_S_I32), /* 0xb7 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_U_I32), /* 0xb8 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_S_I64), /* 0xb9 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_U_I64), /* 0xba */ \ + HANDLE_OPCODE(WASM_OP_F64_PROMOTE_F32), /* 0xbb */ \ + HANDLE_OPCODE(WASM_OP_I32_REINTERPRET_F32), /* 0xbc */ \ + HANDLE_OPCODE(WASM_OP_I64_REINTERPRET_F64), /* 0xbd */ \ + HANDLE_OPCODE(WASM_OP_F32_REINTERPRET_I32), /* 0xbe */ \ + HANDLE_OPCODE(WASM_OP_F64_REINTERPRET_I64), /* 0xbf */ \ + HANDLE_OPCODE(WASM_OP_I32_EXTEND8_S), /* 0xc0 */ \ + HANDLE_OPCODE(WASM_OP_I32_EXTEND16_S), /* 0xc1 */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND8_S), /* 0xc2 */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND16_S), /* 0xc3 */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND32_S), /* 0xc4 */ \ + HANDLE_OPCODE(WASM_OP_DROP_64), /* 0xc5 */ \ + HANDLE_OPCODE(WASM_OP_SELECT_64), /* 0xc6 */ \ + HANDLE_OPCODE(EXT_OP_GET_LOCAL_FAST), /* 0xc7 */ \ + HANDLE_OPCODE(EXT_OP_SET_LOCAL_FAST_I64), /* 0xc8 */ \ + HANDLE_OPCODE(EXT_OP_SET_LOCAL_FAST), /* 0xc9 */ \ + HANDLE_OPCODE(EXT_OP_TEE_LOCAL_FAST), /* 0xca */ \ + HANDLE_OPCODE(EXT_OP_TEE_LOCAL_FAST_I64), /* 0xcb */ \ + HANDLE_OPCODE(EXT_OP_COPY_STACK_TOP), /* 0xcc */ \ + HANDLE_OPCODE(EXT_OP_COPY_STACK_TOP_I64), /* 0xcd */ \ + HANDLE_OPCODE(EXT_OP_COPY_STACK_VALUES), /* 0xce */ \ + HANDLE_OPCODE(WASM_OP_IMPDEP), /* 0xcf */ \ + HANDLE_OPCODE(WASM_OP_REF_NULL), /* 0xd0 */ \ + HANDLE_OPCODE(WASM_OP_REF_IS_NULL), /* 0xd1 */ \ + HANDLE_OPCODE(WASM_OP_REF_FUNC), /* 0xd2 */ \ + HANDLE_OPCODE(WASM_OP_REF_EQ), /* 0xd3 */ \ + HANDLE_OPCODE(WASM_OP_REF_AS_NON_NULL), /* 0xd4 */ \ + HANDLE_OPCODE(WASM_OP_BR_ON_NULL), /* 0xd5 */ \ + HANDLE_OPCODE(WASM_OP_BR_ON_NON_NULL), /* 0xd6 */ \ + HANDLE_OPCODE(EXT_OP_BLOCK), /* 0xd7 */ \ + HANDLE_OPCODE(EXT_OP_LOOP), /* 0xd8 */ \ + HANDLE_OPCODE(EXT_OP_IF), /* 0xd9 */ \ + HANDLE_OPCODE(EXT_OP_BR_TABLE_CACHE), /* 0xda */ \ + HANDLE_OPCODE(EXT_OP_TRY), /* 0xdb */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_GC_PREFIX), /* 0xfb */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_MISC_PREFIX), /* 0xfc */ \ + SET_GOTO_TABLE_SIMD_PREFIX_ELEM() /* 0xfd */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_ATOMIC_PREFIX), /* 0xfe */ \ + DEF_DEBUG_BREAK_HANDLE() \ + }; + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_OPCODE_H */ diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_runtime.c b/wasm-micro-runtime/core/iwasm/interpreter/wasm_runtime.c new file mode 100644 index 0000000..c08e09a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_runtime.c @@ -0,0 +1,4664 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_runtime.h" +#include "wasm.h" +#include "wasm_loader.h" +#include "wasm_interp.h" +#include "bh_common.h" +#include "bh_log.h" +#include "mem_alloc.h" +#include "../common/wasm_runtime_common.h" +#include "../common/wasm_memory.h" +#if WASM_ENABLE_GC != 0 +#include "../common/gc/gc_object.h" +#endif +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "../libraries/debug-engine/debug_engine.h" +#endif +#if WASM_ENABLE_FAST_JIT != 0 +#include "../fast-jit/jit_compiler.h" +#endif +#if WASM_ENABLE_JIT != 0 +#include "../aot/aot_runtime.h" +#endif + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, + "WASM module instantiate failed: %s", string); + } +} + +static void +set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + snprintf(error_buf, error_buf_size, + "WASM module instantiate failed: %s", buf); + } +} + +WASMModule * +wasm_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *name, char *error_buf, uint32 error_buf_size) +{ + return wasm_loader_load(buf, size, +#if WASM_ENABLE_MULTI_MODULE != 0 + main_module, +#endif + name, error_buf, error_buf_size); +} + +WASMModule * +wasm_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size) +{ + return wasm_loader_load_from_sections(section_list, error_buf, + error_buf_size); +} + +void +wasm_unload(WASMModule *module) +{ + wasm_loader_unload(module); +} + +static void * +runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +static WASMModuleInstance * +get_sub_module_inst(const WASMModuleInstance *parent_module_inst, + const WASMModule *sub_module) +{ + bh_list *sub_module_inst_list = parent_module_inst->e->sub_module_inst_list; + WASMSubModInstNode *node = bh_list_first_elem(sub_module_inst_list); + + while (node && sub_module != node->module_inst->module) { + node = bh_list_elem_next(node); + } + return node ? node->module_inst : NULL; +} +#endif + +/** + * Destroy memory instances. + */ +static void +memories_deinstantiate(WASMModuleInstance *module_inst, + WASMMemoryInstance **memories, uint32 count) +{ + uint32 i; + if (memories) { + for (i = 0; i < count; i++) { + if (memories[i]) { +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *module = module_inst->module; + if (i < module->import_memory_count + && module->import_memories[i].u.memory.import_module) { + continue; + } +#endif +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (shared_memory_is_shared(memories[i])) { + uint32 ref_count = shared_memory_dec_reference(memories[i]); + /* if the reference count is not zero, + don't free the memory */ + if (ref_count > 0) + continue; + } +#endif + if (memories[i]->heap_handle) { + mem_allocator_destroy(memories[i]->heap_handle); + wasm_runtime_free(memories[i]->heap_handle); + memories[i]->heap_handle = NULL; + } + if (memories[i]->memory_data) { + wasm_deallocate_linear_memory(memories[i]); + } + } + } + wasm_runtime_free(memories); + } + (void)module_inst; +} + +static WASMMemoryInstance * +memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, + WASMMemoryInstance *memory, uint32 memory_idx, + uint32 num_bytes_per_page, uint32 init_page_count, + uint32 max_page_count, uint32 heap_size, uint32 flags, + char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = module_inst->module; + uint32 inc_page_count, global_idx, default_max_page; + uint32 bytes_of_last_page, bytes_to_page_end; + uint64 aux_heap_base, + heap_offset = (uint64)num_bytes_per_page * init_page_count; + uint64 memory_data_size, max_memory_data_size; + uint8 *global_addr; + + bool is_shared_memory = false; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared_memory = flags & SHARED_MEMORY_FLAG ? true : false; + + /* shared memory */ + if (is_shared_memory && parent != NULL) { + bh_assert(parent->memory_count > memory_idx); + memory = parent->memories[memory_idx]; + shared_memory_inc_reference(memory); + return memory; + } +#else + (void)parent; + (void)memory_idx; + (void)flags; +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + +#if WASM_ENABLE_MEMORY64 != 0 + if (flags & MEMORY64_FLAG) { + memory->is_memory64 = 1; + } +#endif + default_max_page = + memory->is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + + if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1 + && module_inst->module->free_function != (uint32)-1) { + /* Disable app heap, use malloc/free function exported + by wasm app to allocate/free memory instead */ + heap_size = 0; + } + + /* If initial memory is the largest size allowed, disallowing insert host + * managed heap */ + if (heap_size > 0 + && heap_offset == GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + + if (init_page_count == max_page_count && init_page_count == 1) { + /* If only one page and at most one page, we just append + the app heap to the end of linear memory, enlarge the + num_bytes_per_page, and don't change the page count */ + heap_offset = num_bytes_per_page; + num_bytes_per_page += heap_size; + if (num_bytes_per_page < heap_size) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + } + else if (heap_size > 0) { + if (init_page_count == max_page_count && init_page_count == 0) { + /* If the memory data size is always 0, we resize it to + one page for app heap */ + num_bytes_per_page = heap_size; + heap_offset = 0; + inc_page_count = 1; + } + else if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base + < (uint64)num_bytes_per_page * init_page_count) { + /* Insert app heap before __heap_base */ + aux_heap_base = module->aux_heap_base; + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + inc_page_count = + (heap_size - bytes_to_page_end + num_bytes_per_page - 1) + / num_bytes_per_page; + heap_offset = aux_heap_base; + aux_heap_base += heap_size; + + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + if (bytes_to_page_end < 1 * BH_KB) { + aux_heap_base += 1 * BH_KB; + inc_page_count++; + } + + /* Adjust __heap_base global value */ + global_idx = module->aux_heap_base_global_index; + bh_assert(module_inst->e->globals + && global_idx < module_inst->e->global_count); + global_addr = module_inst->global_data + + module_inst->e->globals[global_idx].data_offset; +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + /* For memory64, the global value should be i64 */ + *(uint64 *)global_addr = aux_heap_base; + } + else +#endif + { + /* For memory32, the global value should be i32 */ + *(uint32 *)global_addr = (uint32)aux_heap_base; + } + LOG_VERBOSE("Reset __heap_base global to %" PRIu64, aux_heap_base); + } + else { + /* Insert app heap before new page */ + inc_page_count = + (heap_size + num_bytes_per_page - 1) / num_bytes_per_page; + heap_offset = (uint64)num_bytes_per_page * init_page_count; + heap_size = (uint64)num_bytes_per_page * inc_page_count; + if (heap_size > 0) + heap_size -= 1 * BH_KB; + } + init_page_count += inc_page_count; + max_page_count += inc_page_count; + if (init_page_count > default_max_page) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + + if (max_page_count > default_max_page) + max_page_count = default_max_page; + } + + LOG_VERBOSE("Memory instantiate:"); + LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", + num_bytes_per_page, init_page_count, max_page_count); + LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %u\n", heap_offset, + heap_size); + + max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; + bh_assert(max_memory_data_size + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); + (void)max_memory_data_size; + + bh_assert(memory != NULL); + + if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory, + memory->is_memory64, num_bytes_per_page, + init_page_count, max_page_count, + &memory_data_size) + != BHT_OK) { + set_error_buf(error_buf, error_buf_size, + "allocate linear memory failed"); + return NULL; + } + + memory->module_type = Wasm_Module_Bytecode; + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = init_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = memory_data_size; + + memory->heap_data = memory->memory_data + heap_offset; + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data_end = memory->memory_data + memory_data_size; + + /* Initialize heap */ + if (heap_size > 0) { + uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); + + if (!(memory->heap_handle = runtime_malloc( + (uint64)heap_struct_size, error_buf, error_buf_size))) { + goto fail1; + } + if (!mem_allocator_create_with_struct_and_pool( + memory->heap_handle, heap_struct_size, memory->heap_data, + heap_size)) { + set_error_buf(error_buf, error_buf_size, "init app heap failed"); + goto fail2; + } + } + + if (memory_data_size > 0) { + wasm_runtime_set_mem_bound_check_bytes(memory, memory_data_size); + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + memory->is_shared_memory = 1; + memory->ref_count = 1; + } +#endif + + LOG_VERBOSE("Memory instantiate success."); + return memory; + +fail2: + if (heap_size > 0) + wasm_runtime_free(memory->heap_handle); +fail1: + if (memory->memory_data) + wasm_deallocate_linear_memory(memory); + + return NULL; +} + +/** + * Instantiate memories in a module. + */ +static WASMMemoryInstance ** +memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + WASMModuleInstance *parent, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + WASMImport *import; + uint32 mem_index = 0, i, + memory_count = module->import_memory_count + module->memory_count; + uint64 total_size; + WASMMemoryInstance **memories, *memory; + + total_size = sizeof(WASMMemoryInstance *) * (uint64)memory_count; + + if (!(memories = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + memory = module_inst->global_table_data.memory_instances; + + /* instantiate memories from import section */ + import = module->import_memories; + for (i = 0; i < module->import_memory_count; i++, import++, memory++) { + uint32 num_bytes_per_page = import->u.memory.num_bytes_per_page; + uint32 init_page_count = import->u.memory.init_page_count; + uint32 max_page_count = wasm_runtime_get_max_mem( + max_memory_pages, import->u.memory.init_page_count, + import->u.memory.max_page_count); + uint32 flags = import->u.memory.flags; + uint32 actual_heap_size = heap_size; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->u.memory.import_module != NULL) { + WASMModuleInstance *module_inst_linked; + + if (!(module_inst_linked = get_sub_module_inst( + module_inst, import->u.memory.import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown memory"); + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + + if (!(memories[mem_index++] = wasm_lookup_memory( + module_inst_linked, import->u.memory.field_name))) { + set_error_buf(error_buf, error_buf_size, "unknown memory"); + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + } + else +#endif + { + if (!(memories[mem_index] = memory_instantiate( + module_inst, parent, memory, mem_index, + num_bytes_per_page, init_page_count, max_page_count, + actual_heap_size, flags, error_buf, error_buf_size))) { + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + mem_index++; + } + } + + /* instantiate memories from memory section */ + for (i = 0; i < module->memory_count; i++, memory++) { + uint32 max_page_count = wasm_runtime_get_max_mem( + max_memory_pages, module->memories[i].init_page_count, + module->memories[i].max_page_count); + if (!(memories[mem_index] = memory_instantiate( + module_inst, parent, memory, mem_index, + module->memories[i].num_bytes_per_page, + module->memories[i].init_page_count, max_page_count, + heap_size, module->memories[i].flags, error_buf, + error_buf_size))) { + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + mem_index++; + } + + bh_assert(mem_index == memory_count); + (void)module_inst; + return memories; +} + +/** + * Destroy table instances. + */ +static void +tables_deinstantiate(WASMModuleInstance *module_inst) +{ + if (module_inst->tables) { + wasm_runtime_free(module_inst->tables); + } +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module_inst->e->table_insts_linked) { + wasm_runtime_free(module_inst->e->table_insts_linked); + } +#endif +} + +/** + * Instantiate tables in a module. + */ +static WASMTableInstance ** +tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + WASMTableInstance *first_table, char *error_buf, + uint32 error_buf_size) +{ + WASMImport *import; + uint32 table_index = 0, i; + uint32 table_count = module->import_table_count + module->table_count; + uint64 total_size = (uint64)sizeof(WASMTableInstance *) * table_count; + WASMTableInstance **tables, *table = first_table; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint64 total_size_of_tables_linked = + (uint64)sizeof(WASMTableInstance *) * module->import_table_count; + WASMTableInstance **table_linked = NULL; +#endif + + if (!(tables = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module->import_table_count > 0 + && !(module_inst->e->table_insts_linked = table_linked = runtime_malloc( + total_size_of_tables_linked, error_buf, error_buf_size))) { + goto fail; + } +#endif + + /* instantiate tables from import section */ + import = module->import_tables; + for (i = 0; i < module->import_table_count; i++, import++) { + uint32 max_size_fixed = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMTableInstance *table_inst_linked = NULL; + WASMModuleInstance *module_inst_linked = NULL; + + if (import->u.table.import_module) { + if (!(module_inst_linked = get_sub_module_inst( + module_inst, import->u.table.import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown table"); + goto fail; + } + + if (!(table_inst_linked = wasm_lookup_table( + module_inst_linked, import->u.table.field_name))) { + set_error_buf(error_buf, error_buf_size, "unknown table"); + goto fail; + } + + total_size = offsetof(WASMTableInstance, elems); + } + else +#endif + { + /* in order to save memory, alloc resource as few as possible */ + max_size_fixed = import->u.table.possible_grow + ? import->u.table.max_size + : import->u.table.init_size; + + /* it is a built-in table, every module has its own */ + total_size = offsetof(WASMTableInstance, elems); + /* store function indexes for non-gc, object pointers for gc */ + total_size += (uint64)sizeof(table_elem_type_t) * max_size_fixed; + } + + tables[table_index++] = table; + +#if WASM_ENABLE_GC == 0 + /* Set all elements to -1 to mark them as uninitialized elements */ + memset(table, -1, (uint32)total_size); +#else + /* For GC, all elements have already been set to NULL_REF (0) as + uninitialized elements */ +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + *table_linked = table_inst_linked; + if (table_inst_linked != NULL) { +#if WASM_ENABLE_GC != 0 + table->elem_type = table_inst_linked->elem_type; + table->elem_ref_type = table_inst_linked->elem_ref_type; +#endif + table->cur_size = table_inst_linked->cur_size; + table->max_size = table_inst_linked->max_size; + } + else +#endif + { +#if WASM_ENABLE_GC != 0 + table->elem_type = import->u.table.elem_type; + table->elem_ref_type.elem_ref_type = import->u.table.elem_ref_type; +#endif + table->cur_size = import->u.table.init_size; + table->max_size = max_size_fixed; + } + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_linked++; +#endif + } + + /* instantiate tables from table section */ + for (i = 0; i < module->table_count; i++) { + uint32 max_size_fixed = 0; + + total_size = offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + /* in case, a module which imports this table will grow it */ + max_size_fixed = module->tables[i].max_size; +#else + max_size_fixed = module->tables[i].possible_grow + ? module->tables[i].max_size + : module->tables[i].init_size; +#endif +#if WASM_ENABLE_GC == 0 + /* Store function indexes */ + total_size += sizeof(uintptr_t) * (uint64)max_size_fixed; +#else + /* Store object pointers */ + total_size += sizeof(uintptr_t) * (uint64)max_size_fixed; +#endif + + tables[table_index++] = table; + +#if WASM_ENABLE_GC == 0 + /* Set all elements to -1 to mark them as uninitialized elements */ + memset(table, -1, (uint32)total_size); +#else + /* For GC, all elements have already been set to NULL_REF (0) as + uninitialized elements */ +#endif +#if WASM_ENABLE_GC != 0 + table->elem_type = module->tables[i].elem_type; + table->elem_ref_type.elem_ref_type = module->tables[i].elem_ref_type; +#endif + table->cur_size = module->tables[i].init_size; + table->max_size = max_size_fixed; + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); + } + + bh_assert(table_index == table_count); + (void)module_inst; + return tables; +#if WASM_ENABLE_MULTI_MODULE != 0 +fail: + wasm_runtime_free(tables); + return NULL; +#endif +} + +/** + * Destroy function instances. + */ +static void +functions_deinstantiate(WASMFunctionInstance *functions) +{ + if (functions) { + wasm_runtime_free(functions); + } +} + +/** + * Instantiate functions in a module. + */ +static WASMFunctionInstance * +functions_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + char *error_buf, uint32 error_buf_size) +{ + WASMImport *import; + uint32 i, + function_count = module->import_function_count + module->function_count; + uint64 total_size = sizeof(WASMFunctionInstance) * (uint64)function_count; + WASMFunctionInstance *functions, *function; + + if (!(functions = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + total_size = sizeof(void *) * (uint64)module->import_function_count; + if (total_size > 0 + && !(module_inst->import_func_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + wasm_runtime_free(functions); + return NULL; + } + + /* instantiate functions from import section */ + function = functions; + import = module->import_functions; + for (i = 0; i < module->import_function_count; i++, import++) { + function->is_import_func = true; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->u.function.import_module) { + function->import_module_inst = get_sub_module_inst( + module_inst, import->u.function.import_module); + + if (function->import_module_inst) { + function->import_func_inst = + wasm_lookup_function(function->import_module_inst, + import->u.function.field_name); + } + } +#endif /* WASM_ENABLE_MULTI_MODULE */ + function->u.func_import = &import->u.function; + function->param_cell_num = import->u.function.func_type->param_cell_num; + function->ret_cell_num = import->u.function.func_type->ret_cell_num; + function->param_count = + (uint16)function->u.func_import->func_type->param_count; + function->param_types = function->u.func_import->func_type->types; + function->local_cell_num = 0; + function->local_count = 0; + function->local_types = NULL; + + /* Copy the function pointer to current instance */ + module_inst->import_func_ptrs[i] = + function->u.func_import->func_ptr_linked; + + function++; + } + + /* instantiate functions from function section */ + for (i = 0; i < module->function_count; i++) { + function->is_import_func = false; + function->u.func = module->functions[i]; + + function->param_cell_num = function->u.func->param_cell_num; + function->ret_cell_num = function->u.func->ret_cell_num; + function->local_cell_num = function->u.func->local_cell_num; + + function->param_count = + (uint16)function->u.func->func_type->param_count; + function->local_count = (uint16)function->u.func->local_count; + function->param_types = function->u.func->func_type->types; + function->local_types = function->u.func->local_types; + + function->local_offsets = function->u.func->local_offsets; + +#if WASM_ENABLE_FAST_INTERP != 0 + function->const_cell_num = function->u.func->const_cell_num; +#endif + + function++; + } + bh_assert((uint32)(function - functions) == function_count); + +#if WASM_ENABLE_FAST_JIT != 0 + module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs; +#endif + + return functions; +} + +#if WASM_ENABLE_TAGS != 0 +/** + * Destroy tags instances. + */ +static void +tags_deinstantiate(WASMTagInstance *tags, void **import_tag_ptrs) +{ + if (tags) { + wasm_runtime_free(tags); + } + if (import_tag_ptrs) { + wasm_runtime_free(import_tag_ptrs); + } +} + +/** + * Instantiate tags in a module. + */ +static WASMTagInstance * +tags_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + char *error_buf, uint32 error_buf_size) +{ + WASMImport *import; + uint32 i, tag_count = module->import_tag_count + module->tag_count; + uint64 total_size = sizeof(WASMTagInstance) * (uint64)tag_count; + WASMTagInstance *tags, *tag; + + if (!(tags = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + total_size = sizeof(void *) * (uint64)module->import_tag_count; + if (total_size > 0 + && !(module_inst->e->import_tag_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + wasm_runtime_free(tags); + return NULL; + } + + /* instantiate tags from import section */ + tag = tags; + import = module->import_tags; + for (i = 0; i < module->import_tag_count; i++, import++) { + tag->is_import_tag = true; + tag->u.tag_import = &import->u.tag; + tag->type = import->u.tag.type; + tag->attribute = import->u.tag.attribute; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->u.tag.import_module) { + if (!(tag->import_module_inst = get_sub_module_inst( + module_inst, import->u.tag.import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown tag"); + goto fail; + } + + if (!(tag->import_tag_inst = + wasm_lookup_tag(tag->import_module_inst, + import->u.tag.field_name, NULL))) { + set_error_buf(error_buf, error_buf_size, "unknown tag"); + goto fail; + } + + /* Copy the imported tag to current instance */ + module_inst->e->import_tag_ptrs[i] = + tag->u.tag_import->import_tag_linked; + } +#endif + tag++; + } + + /* instantiate tags from tag section */ + for (i = 0; i < module->tag_count; i++) { + tag->is_import_tag = false; + tag->type = module->tags[i]->type; + tag->u.tag = module->tags[i]; + +#if WASM_ENABLE_FAST_INTERP != 0 + /* tag->const_cell_num = function->u.func->const_cell_num; */ +#endif + tag++; + } + bh_assert((uint32)(tag - tags) == tag_count); + + return tags; + +#if WASM_ENABLE_MULTI_MODULE != 0 +fail: + tags_deinstantiate(tags, module_inst->e->import_tag_ptrs); + /* clean up */ + module_inst->e->import_tag_ptrs = NULL; + return NULL; +#endif +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +/** + * Destroy global instances. + */ +static void +globals_deinstantiate(WASMGlobalInstance *globals) +{ + if (globals) + wasm_runtime_free(globals); +} + +static bool +check_global_init_expr(const WASMModule *module, uint32 global_index, + char *error_buf, uint32 error_buf_size) +{ + if (global_index >= module->import_global_count + module->global_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown global %d", + global_index); + return false; + } + +#if WASM_ENABLE_GC == 0 + /** + * Currently, constant expressions occurring as initializers of + * globals are further constrained in that contained global.get + * instructions are only allowed to refer to imported globals. + * + * And initializer expression cannot reference a mutable global. + */ + if (global_index >= module->import_global_count + || (module->import_globals + global_index)->u.global.is_mutable) { + set_error_buf(error_buf, error_buf_size, + "constant expression required"); + return false; + } +#endif + + return true; +} + +/** + * Instantiate globals in a module. + */ +static WASMGlobalInstance * +globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst, + char *error_buf, uint32 error_buf_size) +{ + WASMImport *import; + uint32 global_data_offset = 0; + uint32 i, global_count = module->import_global_count + module->global_count; + uint64 total_size = sizeof(WASMGlobalInstance) * (uint64)global_count; + WASMGlobalInstance *globals, *global; + + if (!(globals = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + /* instantiate globals from import section */ + global = globals; + import = module->import_globals; + for (i = 0; i < module->import_global_count; i++, import++) { + WASMGlobalImport *global_import = &import->u.global; + global->type = global_import->type; + global->is_mutable = global_import->is_mutable; +#if WASM_ENABLE_GC != 0 + global->ref_type = global_import->ref_type; +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + if (global_import->import_module) { + if (!(global->import_module_inst = get_sub_module_inst( + module_inst, global_import->import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + if (!(global->import_global_inst = wasm_lookup_global( + global->import_module_inst, global_import->field_name))) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + /* The linked global instance has been initialized, we + just need to copy the value. */ + bh_memcpy_s(&(global->initial_value), sizeof(WASMValue), + &(global_import->import_global_linked->init_expr), + sizeof(WASMValue)); + } + else +#endif + { + /* native globals share their initial_values in one module */ + bh_memcpy_s(&(global->initial_value), sizeof(WASMValue), + &(global_import->global_data_linked), + sizeof(WASMValue)); + } +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == global_import->data_offset); +#endif + global->data_offset = global_data_offset; + global_data_offset += wasm_value_type_size(global->type); + + global++; + } + + /* instantiate globals from global section */ + for (i = 0; i < module->global_count; i++) { + InitializerExpression *init_expr = &(module->globals[i].init_expr); + uint8 flag = init_expr->init_expr_type; + + global->type = module->globals[i].type; + global->is_mutable = module->globals[i].is_mutable; +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == module->globals[i].data_offset); +#endif + global->data_offset = global_data_offset; + global_data_offset += wasm_value_type_size(global->type); +#if WASM_ENABLE_GC != 0 + global->ref_type = module->globals[i].ref_type; +#endif + + switch (flag) { + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr(module, init_expr->u.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + bh_memcpy_s( + &(global->initial_value), sizeof(WASMValue), + &(globals[init_expr->u.global_index].initial_value), + sizeof(globals[init_expr->u.global_index].initial_value)); + break; + } +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_STRUCT_NEW: + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + { + WASMRttType *rtt_type; + WASMStructObjectRef struct_obj; + WASMStructType *struct_type; + WASMStructNewInitValues *init_values = NULL; + uint32 type_idx; + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + init_values = (WASMStructNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + } + else { + type_idx = init_expr->u.type_index; + } + + struct_type = (WASMStructType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + goto fail; + } + + if (!(struct_obj = wasm_struct_obj_new_internal( + module_inst->e->common.gc_heap_handle, rtt_type))) { + set_error_buf(error_buf, error_buf_size, + "create struct object failed"); + goto fail; + } + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + uint32 field_idx; + + bh_assert(init_values->count == struct_type->field_count); + + for (field_idx = 0; field_idx < init_values->count; + field_idx++) { + wasm_struct_obj_set_field( + struct_obj, field_idx, + &init_values->fields[field_idx]); + } + } + + global->initial_value.gc_obj = (void *)struct_obj; + break; + } + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + WASMRttType *rtt_type; + WASMArrayObjectRef array_obj; + WASMArrayType *array_type; + WASMArrayNewInitValues *init_values = NULL; + WASMValue *arr_init_val = NULL, empty_val = { 0 }; + uint32 type_idx, len; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { + type_idx = init_expr->u.array_new_default.type_index; + len = init_expr->u.array_new_default.length; + arr_init_val = &empty_val; + } + else { + init_values = (WASMArrayNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + len = init_values->length; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW) { + arr_init_val = init_values->elem_data; + } + } + + array_type = (WASMArrayType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, module->rtt_types, + module->type_count, &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + goto fail; + } + + if (!(array_obj = wasm_array_obj_new_internal( + module_inst->e->common.gc_heap_handle, rtt_type, len, + arr_init_val))) { + set_error_buf(error_buf, error_buf_size, + "create array object failed"); + goto fail; + } + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + uint32 elem_idx; + + bh_assert(init_values); + + for (elem_idx = 0; elem_idx < len; elem_idx++) { + wasm_array_obj_set_elem( + array_obj, elem_idx, + &init_values->elem_data[elem_idx]); + } + } + + global->initial_value.gc_obj = (void *)array_obj; + + break; + } + case INIT_EXPR_TYPE_I31_NEW: + { + global->initial_value.gc_obj = + (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32); + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + bh_memcpy_s(&(global->initial_value), sizeof(WASMValue), + &(init_expr->u), sizeof(init_expr->u)); + break; + } + + global++; + } + + bh_assert((uint32)(global - globals) == global_count); + bh_assert(global_data_offset == module->global_data_size); + (void)module_inst; + return globals; +fail: + wasm_runtime_free(globals); + return NULL; +} + +/** + * Return export function count in module export section. + */ +static uint32 +get_export_count(const WASMModule *module, uint8 kind) +{ + WASMExport *export = module->exports; + uint32 count = 0, i; + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == kind) + count++; + + return count; +} + +/** + * Destroy export function instances. + */ +static void +export_functions_deinstantiate(WASMExportFuncInstance *functions) +{ + if (functions) + wasm_runtime_free(functions); +} + +/** + * Instantiate export functions in a module. + */ +static WASMExportFuncInstance * +export_functions_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_func_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportFuncInstance *export_funcs, *export_func; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportFuncInstance) * (uint64)export_func_count; + + if (!(export_func = export_funcs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_FUNC) { + export_func->name = export->name; + export_func->function = &module_inst->e->functions[export->index]; + export_func++; + } + + bh_assert((uint32)(export_func - export_funcs) == export_func_count); + return export_funcs; +} + +#if WASM_ENABLE_TAGS != 0 +/** + * Destroy export function instances. + */ +static void +export_tags_deinstantiate(WASMExportTagInstance *tags) +{ + if (tags) + wasm_runtime_free(tags); +} + +/** + * Instantiate export functions in a module. + */ +static WASMExportTagInstance * +export_tags_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_tag_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportTagInstance *export_tags, *export_tag; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportTagInstance) * (uint64)export_tag_count; + + if (!(export_tag = export_tags = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_TAG) { + export_tag->name = export->name; + + bh_assert(module_inst->e->tags); + + export_tag->tag = &module_inst->e->tags[export->index]; + export_tag++; + } + + bh_assert((uint32)(export_tag - export_tags) == export_tag_count); + return export_tags; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +#if WASM_ENABLE_MULTI_MODULE != 0 +static void +export_globals_deinstantiate(WASMExportGlobInstance *globals) +{ + if (globals) + wasm_runtime_free(globals); +} + +static WASMExportGlobInstance * +export_globals_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_glob_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportGlobInstance *export_globals, *export_global; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportGlobInstance) * (uint64)export_glob_count; + + if (!(export_global = export_globals = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_GLOBAL) { + export_global->name = export->name; + export_global->global = &module_inst->e->globals[export->index]; + export_global++; + } + + bh_assert((uint32)(export_global - export_globals) == export_glob_count); + return export_globals; +} +#endif + +static WASMFunctionInstance * +lookup_post_instantiate_func(WASMModuleInstance *module_inst, + const char *func_name) +{ + WASMFunctionInstance *func; + WASMFuncType *func_type; + + if (!(func = wasm_lookup_function(module_inst, func_name))) + /* Not found */ + return NULL; + + func_type = func->u.func->func_type; + if (!(func_type->param_count == 0 && func_type->result_count == 0)) + /* Not a valid function type, ignore it */ + return NULL; + + return func; +} + +static bool +execute_post_instantiate_functions(WASMModuleInstance *module_inst, + bool is_sub_inst, WASMExecEnv *exec_env_main) +{ + WASMFunctionInstance *start_func = module_inst->e->start_function; + WASMFunctionInstance *initialize_func = NULL; + WASMFunctionInstance *post_inst_func = NULL; + WASMFunctionInstance *call_ctors_func = NULL; +#if WASM_ENABLE_LIBC_WASI != 0 + WASMModule *module = module_inst->module; +#endif + WASMModuleInstanceCommon *module_inst_main = NULL; +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env = NULL, *exec_env_created = NULL; + bool ret = false; + +#if WASM_ENABLE_LIBC_WASI != 0 + /* + * WASI reactor instances may assume that _initialize will be called by + * the environment at most once, and that none of their other exports + * are accessed before that call. + */ + if (!is_sub_inst && module->import_wasi_api) { + initialize_func = + lookup_post_instantiate_func(module_inst, "_initialize"); + } +#endif + + /* Execute possible "__post_instantiate" function if wasm app is + compiled by emsdk's early version */ + if (!is_sub_inst) { + post_inst_func = + lookup_post_instantiate_func(module_inst, "__post_instantiate"); + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + /* Only execute the memory init function for main instance since + the data segments will be dropped once initialized */ + if (!is_sub_inst +#if WASM_ENABLE_LIBC_WASI != 0 + && !module->import_wasi_api +#endif + ) { + call_ctors_func = + lookup_post_instantiate_func(module_inst, "__wasm_call_ctors"); + } +#endif + + if (!start_func && !initialize_func && !post_inst_func + && !call_ctors_func) { + /* No post instantiation functions to call */ + return true; + } + + if (is_sub_inst) { + bh_assert(exec_env_main); +#ifdef OS_ENABLE_HW_BOUND_CHECK + bh_assert(exec_env_tls == exec_env_main); + (void)exec_env_tls; +#endif + exec_env = exec_env_main; + + /* Temporarily replace parent exec_env's module inst to current + module inst to avoid checking failure when calling the + wasm functions, and ensure that the exec_env's module inst + is the correct one. */ + module_inst_main = exec_env_main->module_inst; + wasm_exec_env_set_module_inst(exec_env, + (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_main = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + /* Execute start function for both main insance and sub instance */ + if (start_func && !wasm_call_function(exec_env, start_func, 0, NULL)) { + goto fail; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + if (initialize_func + && !wasm_call_function(exec_env, initialize_func, 0, NULL)) { + goto fail; + } +#else + (void)initialize_func; +#endif + + if (post_inst_func + && !wasm_call_function(exec_env, post_inst_func, 0, NULL)) { + goto fail; + } + + if (call_ctors_func + && !wasm_call_function(exec_env, call_ctors_func, 0, NULL)) { + goto fail; + } + + ret = true; + +fail: + if (is_sub_inst) { + /* Restore the parent exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env_main, module_inst_main); + } + else { + if (module_inst_main) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_main); + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + } + + return ret; +} + +static bool +execute_malloc_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, + WASMFunctionInstance *malloc_func, + WASMFunctionInstance *retain_func, uint64 size, + uint64 *p_result) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env_created = NULL; + WASMModuleInstanceCommon *module_inst_old = NULL; + uint32 argv[3], argc; + bool ret; +#if WASM_ENABLE_MEMORY64 != 0 + bool is_memory64 = module_inst->memories[0]->is_memory64; + if (is_memory64) { + argc = 2; + PUT_I64_TO_ADDR(&argv[0], size); + } + else +#endif + { + argc = 1; + argv[0] = size; + } + + /* if __retain is exported, then this module is compiled by + assemblyscript, the memory should be managed by as's runtime, + in this case we need to call the retain function after malloc + the memory */ + if (retain_func) { + /* the malloc function from assemblyscript is: + function __new(size: usize, id: u32) + id = 0 means this is an ArrayBuffer object */ + argv[argc] = 0; + argc++; + } + + if (exec_env) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls) { + bh_assert(exec_env_tls == exec_env); + } +#endif + bh_assert(exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_old = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + ret = wasm_call_function(exec_env, malloc_func, argc, argv); + + if (retain_func && ret) + ret = wasm_call_function(exec_env, retain_func, 1, argv); + + if (module_inst_old) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_old); + + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + + if (ret) { +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) + *p_result = GET_I64_FROM_ADDR(&argv[0]); + else +#endif + { + *p_result = argv[0]; + } + } + return ret; +} + +static bool +execute_free_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, + WASMFunctionInstance *free_func, uint64 offset) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env_created = NULL; + WASMModuleInstanceCommon *module_inst_old = NULL; + uint32 argv[2], argc; + bool ret; + +#if WASM_ENABLE_MEMORY64 != 0 + if (module_inst->memories[0]->is_memory64) { + PUT_I64_TO_ADDR(&argv[0], offset); + argc = 2; + } + else +#endif + { + argv[0] = (uint32)offset; + argc = 1; + } + + if (exec_env) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls) { + bh_assert(exec_env_tls == exec_env); + } +#endif + bh_assert(exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_old = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + ret = wasm_call_function(exec_env, free_func, argc, argv); + + if (module_inst_old) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_old); + + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + + return ret; +} + +static bool +check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf, + uint32 error_buf_size) +{ + WASMModule *module = module_inst->module; + uint32 i; + + for (i = 0; i < module->import_function_count; i++) { + WASMFunctionImport *func = + &((module->import_functions + i)->u.function); + if (!func->func_ptr_linked +#if WASM_ENABLE_MULTI_MODULE != 0 + && !func->import_func_linked +#endif + ) { +#if WASM_ENABLE_WAMR_COMPILER == 0 + LOG_WARNING("warning: failed to link import function (%s, %s)", + func->module_name, func->field_name); + /* will throw exception only if calling */ +#else + /* do nothing to avoid confused message */ +#endif /* WASM_ENABLE_WAMR_COMPILER == 0 */ + } + } + + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *global = &((module->import_globals + i)->u.global); + if (!global->is_linked) { +#if WASM_ENABLE_SPEC_TEST != 0 + set_error_buf(error_buf, error_buf_size, + "unknown import or incompatible import type"); + return false; +#else +#if WASM_ENABLE_WAMR_COMPILER == 0 + set_error_buf_v(error_buf, error_buf_size, + "failed to link import global (%s, %s)", + global->module_name, global->field_name); + return false; +#else + /* do nothing to avoid confused message */ +#endif /* WASM_ENABLE_WAMR_COMPILER == 0 */ +#endif /* WASM_ENABLE_SPEC_TEST != 0 */ + } + } + + return true; +} + +#if WASM_ENABLE_JIT != 0 +static bool +init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + void **func_ptrs; + uint64 total_size = (uint64)sizeof(void *) * module_inst->e->function_count; + + /* Allocate memory */ + if (!(func_ptrs = module_inst->func_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Set import function pointers */ + for (i = 0; i < module->import_function_count; i++, func_ptrs++) { + WASMFunctionImport *import_func = + &module->import_functions[i].u.function; + /* TODO: handle multi module */ + *func_ptrs = import_func->func_ptr_linked; + } + + /* The defined function pointers will be set in + wasm_runtime_set_running_mode, no need to set them here */ + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 +static uint32 +get_smallest_type_idx(WASMModule *module, WASMFuncType *func_type) +{ + uint32 i; + + for (i = 0; i < module->type_count; i++) { + if (func_type == (WASMFuncType *)module->types[i]) + return i; + } + + bh_assert(0); + return -1; +} + +static bool +init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, + uint32 error_buf_size) +{ + uint32 i; + uint64 total_size = (uint64)sizeof(uint32) * module_inst->e->function_count; + + /* Allocate memory */ + if (!(module_inst->func_type_indexes = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module_inst->e->function_count; i++) { + WASMFunctionInstance *func_inst = module_inst->e->functions + i; + WASMFuncType *func_type = func_inst->is_import_func + ? func_inst->u.func_import->func_type + : func_inst->u.func->func_type; + module_inst->func_type_indexes[i] = + get_smallest_type_idx(module_inst->module, func_type); + } + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +wasm_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = module_inst->module; + WASMRttTypeRef rtt_type; + WASMFuncObjectRef func_obj; + WASMFunctionInstance *func_inst; + WASMFuncType *func_type; + uint32 type_idx; + + if (throw_exce) { + error_buf = module_inst->cur_exception; + error_buf_size = sizeof(module_inst->cur_exception); + } + + if (func_idx >= module_inst->e->function_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown function %d", + func_idx); + return NULL; + } + + func_inst = &module_inst->e->functions[func_idx]; + func_type = func_inst->is_import_func ? func_inst->u.func_import->func_type + : func_inst->u.func->func_type; + type_idx = func_inst->is_import_func ? func_inst->u.func_import->type_idx + : func_inst->u.func->type_idx; + + if (!(rtt_type = wasm_rtt_type_new((WASMType *)func_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, "create rtt object failed"); + return NULL; + } + + if (!(func_obj = wasm_func_obj_new_internal( + module_inst->e->common.gc_heap_handle, rtt_type, func_idx))) { + set_error_buf(error_buf, error_buf_size, "create func object failed"); + return NULL; + } + + return func_obj; +} + +static bool +wasm_global_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap) +{ + WASMGlobalInstance *global = module_inst->e->globals; + WASMGlobalInstance *global_end = global + module_inst->e->global_count; + uint8 *global_data = module_inst->global_data; + WASMObjectRef gc_obj; + + while (global < global_end) { + if (wasm_is_type_reftype(global->type)) { + gc_obj = GET_REF_FROM_ADDR( + (uint32 *)(global_data + global->data_offset)); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + global++; + } + return true; +} + +static bool +wasm_table_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap) +{ + WASMTableInstance **tables = module_inst->tables, *table; + uint32 table_count = module_inst->table_count, i, j; + WASMObjectRef gc_obj, *table_elems; + + for (i = 0; i < table_count; i++) { + table = tables[i]; + table_elems = (WASMObjectRef *)table->elems; + for (j = 0; j < table->cur_size; j++) { + gc_obj = table_elems[j]; + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + } + + return true; +} + +static bool +local_object_refs_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMLocalObjectRef *r; + WASMObjectRef gc_obj; + + for (r = exec_env->cur_local_object_ref; r; r = r->prev) { + gc_obj = r->val; + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + return true; +} + +bool +wasm_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + bool ret; + + ret = wasm_global_traverse_gc_rootset(module_inst, heap); + if (!ret) + return ret; + + ret = wasm_table_traverse_gc_rootset(module_inst, heap); + if (!ret) + return ret; + + ret = local_object_refs_traverse_gc_rootset(exec_env, heap); + if (!ret) + return ret; + + return wasm_interp_traverse_gc_rootset(exec_env, heap); +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static bool +set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode, + bool first_time_set) +{ + WASMModule *module = module_inst->module; + + if (running_mode == Mode_Default) { +#if WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT == 0 + running_mode = Mode_Interp; +#elif WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT == 0 + running_mode = Mode_Fast_JIT; +#elif WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT != 0 + running_mode = Mode_LLVM_JIT; +#else /* WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 */ +#if WASM_ENABLE_LAZY_JIT == 0 + running_mode = Mode_LLVM_JIT; +#else + running_mode = Mode_Multi_Tier_JIT; +#endif +#endif + } + + if (!wasm_runtime_is_running_mode_supported(running_mode)) + return false; + +#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */ + (void)first_time_set; + module_inst->e->running_mode = running_mode; + + if (running_mode == Mode_Interp) { + /* Do nothing for Mode_Interp */ + } + else if (running_mode == Mode_Fast_JIT) { + /* Do nothing for Mode_Fast_JIT since + module_inst->fast_jit_func_ptrs is same as + module->fast_jit_func_ptrs */ + } +#if WASM_ENABLE_JIT != 0 + else if (running_mode == Mode_LLVM_JIT) { + /* Set defined function pointers */ + bh_memcpy_s(module_inst->func_ptrs + module->import_function_count, + sizeof(void *) * module->function_count, module->func_ptrs, + sizeof(void *) * module->function_count); + } +#endif + else { + bh_assert(0); + } +#else /* Possible multi-tier JIT */ + os_mutex_lock(&module->instance_list_lock); + + module_inst->e->running_mode = running_mode; + + if (running_mode == Mode_Interp) { + /* Do nothing for Mode_Interp */ + } +#if WASM_ENABLE_FAST_JIT != 0 + else if (running_mode == Mode_Fast_JIT) { + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + uint32 i; + + /* Allocate memory for fast_jit_func_ptrs if needed */ + if (!module_inst->fast_jit_func_ptrs + || module_inst->fast_jit_func_ptrs == module->fast_jit_func_ptrs) { + uint64 total_size = (uint64)sizeof(void *) * module->function_count; + if (!(module_inst->fast_jit_func_ptrs = + runtime_malloc(total_size, NULL, 0))) { + os_mutex_unlock(&module->instance_list_lock); + return false; + } + } + + for (i = 0; i < module->function_count; i++) { + if (module->functions[i]->fast_jit_jitted_code) { + /* current fast jit function has been compiled */ + module_inst->fast_jit_func_ptrs[i] = + module->functions[i]->fast_jit_jitted_code; + } + else { + module_inst->fast_jit_func_ptrs[i] = + jit_globals->compile_fast_jit_and_then_call; + } + } + } +#endif +#if WASM_ENABLE_JIT != 0 + else if (running_mode == Mode_LLVM_JIT) { + void **llvm_jit_func_ptrs; + uint32 i; + + /* Notify backend threads to start llvm jit compilation */ + module->enable_llvm_jit_compilation = true; + + /* Wait until llvm jit finishes initialization */ + os_mutex_lock(&module->tierup_wait_lock); + while (!module->llvm_jit_inited) { + os_cond_reltimedwait(&module->tierup_wait_cond, + &module->tierup_wait_lock, 10000); + if (module->orcjit_stop_compiling) { + /* init_llvm_jit_functions_stage2 failed */ + os_mutex_unlock(&module->tierup_wait_lock); + os_mutex_unlock(&module->instance_list_lock); + return false; + } + } + os_mutex_unlock(&module->tierup_wait_lock); + + llvm_jit_func_ptrs = + module_inst->func_ptrs + module->import_function_count; + for (i = 0; i < module->function_count; i++) { + llvm_jit_func_ptrs[i] = module->functions[i]->llvm_jit_func_ptr; + } + } +#endif + else if (running_mode == Mode_Multi_Tier_JIT) { + /* Notify backend threads to start llvm jit compilation */ + module->enable_llvm_jit_compilation = true; + + /* Free fast_jit_func_ptrs if it is allocated before */ + if (module_inst->fast_jit_func_ptrs + && module_inst->fast_jit_func_ptrs != module->fast_jit_func_ptrs) { + wasm_runtime_free(module_inst->fast_jit_func_ptrs); + } + module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs; + + /* Copy all llvm jit func ptrs from the module */ + bh_memcpy_s(module_inst->func_ptrs + module->import_function_count, + sizeof(void *) * module->function_count, module->func_ptrs, + sizeof(void *) * module->function_count); + } + else { + bh_assert(0); + } + + /* Add module instance into module's instance list if not added */ + if (first_time_set) { + bool found = false; + WASMModuleInstance *node = module->instance_list; + + while (node) { + if (node == module_inst) { + found = true; + break; + } + node = node->e->next; + } + + if (!found) { + module_inst->e->next = module->instance_list; + module->instance_list = module_inst; + } + } + + os_mutex_unlock(&module->instance_list_lock); +#endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) */ + + (void)module; + return true; +} + +bool +wasm_set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode) +{ + return set_running_mode(module_inst, running_mode, false); +} + +/** + * Instantiate module + */ +WASMModuleInstance * +wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleInstance *module_inst; + WASMGlobalInstance *globals = NULL, *global; + WASMTableInstance *first_table; + uint32 global_count, i; + uint32 length, extra_info_offset; + mem_offset_t base_offset; + uint32 module_inst_struct_size = + offsetof(WASMModuleInstance, global_table_data.bytes); + uint64 module_inst_mem_inst_size; + uint64 total_size, table_size = 0; + uint8 *global_data, *global_data_end; +#if WASM_ENABLE_MULTI_MODULE != 0 + bool ret = false; +#endif + const bool is_sub_inst = parent != NULL; + + if (!module) + return NULL; + + /* Check the heap size */ + heap_size = align_uint(heap_size, 8); + if (heap_size > APP_HEAP_SIZE_MAX) + heap_size = APP_HEAP_SIZE_MAX; + + module_inst_mem_inst_size = + (uint64)sizeof(WASMMemoryInstance) + * (module->import_memory_count + module->memory_count); + +#if WASM_ENABLE_JIT != 0 + /* If the module dosen't have memory, reserve one mem_info space + with empty content to align with llvm jit compiler */ + if (module_inst_mem_inst_size == 0) + module_inst_mem_inst_size = (uint64)sizeof(WASMMemoryInstance); +#endif + + /* Size of module inst, memory instances and global data */ + total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size + + module->global_data_size; + + /* Calculate the size of table data */ + for (i = 0; i < module->import_table_count; i++) { + WASMTableImport *import_table = &module->import_tables[i].u.table; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += + (uint64)sizeof(table_elem_type_t) * import_table->max_size; +#else + table_size += (uint64)sizeof(table_elem_type_t) + * (import_table->possible_grow ? import_table->max_size + : import_table->init_size); +#endif + } + for (i = 0; i < module->table_count; i++) { + WASMTable *table = module->tables + i; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += (uint64)sizeof(table_elem_type_t) * table->max_size; +#else + table_size += + (uint64)sizeof(table_elem_type_t) + * (table->possible_grow ? table->max_size : table->init_size); +#endif + } + total_size += table_size; + + /* The offset of WASMModuleInstanceExtra, make it 8-byte aligned */ + total_size = (total_size + 7LL) & ~7LL; + extra_info_offset = (uint32)total_size; + total_size += sizeof(WASMModuleInstanceExtra); + + /* Allocate the memory for module instance with memory instances, + global data, table data appended at the end */ + if (!(module_inst = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + module_inst->module_type = Wasm_Module_Bytecode; + module_inst->module = module; + module_inst->e = + (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset); + +#if WASM_ENABLE_MULTI_MODULE != 0 + module_inst->e->sub_module_inst_list = + &module_inst->e->sub_module_inst_list_head; + ret = wasm_runtime_sub_module_instantiate( + (WASMModuleCommon *)module, (WASMModuleInstanceCommon *)module_inst, + stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); + if (!ret) { + LOG_DEBUG("build a sub module list failed"); + goto fail; + } +#endif + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (module->data_seg_count > 0) { + module_inst->e->common.data_dropped = + bh_bitmap_new(0, module->data_seg_count); + if (module_inst->e->common.data_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + for (i = 0; i < module->data_seg_count; i++) { + if (!module->data_segments[i]->is_passive) + bh_bitmap_set_bit(module_inst->e->common.data_dropped, i); + } + } +#endif +#if WASM_ENABLE_REF_TYPES != 0 + if (module->table_seg_count > 0) { + module_inst->e->common.elem_dropped = + bh_bitmap_new(0, module->table_seg_count); + if (module_inst->e->common.elem_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + for (i = 0; i < module->table_seg_count; i++) { + if (wasm_elem_is_active(module->table_segments[i].mode)) + bh_bitmap_set_bit(module_inst->e->common.elem_dropped, i); + } + } +#endif + +#if WASM_ENABLE_GC != 0 + if (!is_sub_inst) { + uint32 gc_heap_size = wasm_runtime_get_gc_heap_size_default(); + + if (gc_heap_size < GC_HEAP_SIZE_MIN) + gc_heap_size = GC_HEAP_SIZE_MIN; + if (gc_heap_size > GC_HEAP_SIZE_MAX) + gc_heap_size = GC_HEAP_SIZE_MAX; + + module_inst->e->common.gc_heap_pool = + runtime_malloc(gc_heap_size, error_buf, error_buf_size); + if (!module_inst->e->common.gc_heap_pool) + goto fail; + + module_inst->e->common.gc_heap_handle = mem_allocator_create( + module_inst->e->common.gc_heap_pool, gc_heap_size); + if (!module_inst->e->common.gc_heap_handle) + goto fail; + } +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (!(module_inst->frames = runtime_malloc((uint64)sizeof(Vector), + error_buf, error_buf_size))) { + goto fail; + } +#endif + + /* Instantiate global firstly to get the mutable data size */ + global_count = module->import_global_count + module->global_count; + if (global_count + && !(globals = globals_instantiate(module, module_inst, error_buf, + error_buf_size))) { + goto fail; + } + module_inst->e->global_count = global_count; + module_inst->e->globals = globals; + module_inst->global_data = (uint8 *)module_inst + module_inst_struct_size + + module_inst_mem_inst_size; + module_inst->global_data_size = module->global_data_size; + first_table = (WASMTableInstance *)(module_inst->global_data + + module->global_data_size); + + module_inst->memory_count = + module->import_memory_count + module->memory_count; + module_inst->table_count = module->import_table_count + module->table_count; + module_inst->e->function_count = + module->import_function_count + module->function_count; +#if WASM_ENABLE_TAGS != 0 + module_inst->e->tag_count = module->import_tag_count + module->tag_count; +#endif + + /* export */ + module_inst->export_func_count = get_export_count(module, EXPORT_KIND_FUNC); +#if WASM_ENABLE_MULTI_MODULE != 0 + module_inst->export_table_count = + get_export_count(module, EXPORT_KIND_TABLE); + module_inst->export_memory_count = + get_export_count(module, EXPORT_KIND_MEMORY); +#if WASM_ENABLE_TAGS != 0 + module_inst->e->export_tag_count = + get_export_count(module, EXPORT_KIND_TAG); +#endif + module_inst->export_global_count = + get_export_count(module, EXPORT_KIND_GLOBAL); +#endif + + /* Instantiate memories/tables/functions/tags */ + if ((module_inst->memory_count > 0 + && !(module_inst->memories = memories_instantiate( + module, module_inst, parent, heap_size, max_memory_pages, + error_buf, error_buf_size))) + || (module_inst->table_count > 0 + && !(module_inst->tables = + tables_instantiate(module, module_inst, first_table, + error_buf, error_buf_size))) + || (module_inst->e->function_count > 0 + && !(module_inst->e->functions = functions_instantiate( + module, module_inst, error_buf, error_buf_size))) + || (module_inst->export_func_count > 0 + && !(module_inst->export_functions = export_functions_instantiate( + module, module_inst, module_inst->export_func_count, + error_buf, error_buf_size))) +#if WASM_ENABLE_TAGS != 0 + || (module_inst->e->tag_count > 0 + && !(module_inst->e->tags = tags_instantiate( + module, module_inst, error_buf, error_buf_size))) + || (module_inst->e->export_tag_count > 0 + && !(module_inst->e->export_tags = export_tags_instantiate( + module, module_inst, module_inst->e->export_tag_count, + error_buf, error_buf_size))) +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + || (module_inst->export_global_count > 0 + && !(module_inst->export_globals = export_globals_instantiate( + module, module_inst, module_inst->export_global_count, + error_buf, error_buf_size))) +#endif +#if WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 + && !init_func_ptrs(module_inst, module, error_buf, error_buf_size)) +#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 + && !init_func_type_indexes(module_inst, error_buf, error_buf_size)) +#endif + ) { + goto fail; + } + if (global_count > 0) { + /* Initialize the global data */ + global_data = module_inst->global_data; + global_data_end = global_data + module->global_data_size; + global = globals; + for (i = 0; i < global_count; i++, global++) { + switch (global->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + *(int32 *)global_data = global->initial_value.i32; + global_data += sizeof(int32); + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + bh_memcpy_s(global_data, + (uint32)(global_data_end - global_data), + &global->initial_value.i64, sizeof(int64)); + global_data += sizeof(int64); + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + bh_memcpy_s(global_data, (uint32)sizeof(V128), + &global->initial_value.v128, sizeof(V128)); + global_data += sizeof(V128); + break; +#endif +#if WASM_ENABLE_GC != 0 + case VALUE_TYPE_EXTERNREF: + /* the initial value should be a null reference */ + bh_assert(global->initial_value.gc_obj == NULL_REF); + STORE_PTR((void **)global_data, NULL_REF); + global_data += sizeof(void *); + break; +#endif + default: + { +#if WASM_ENABLE_GC != 0 + InitializerExpression *global_init = NULL; + bh_assert(wasm_is_type_reftype(global->type)); + + if (i >= module->import_global_count) { + global_init = + &module->globals[i - module->import_global_count] + .init_expr; + } + + if (global->type == REF_TYPE_NULLFUNCREF + || global->type == REF_TYPE_NULLEXTERNREF + || global->type == REF_TYPE_NULLREF) { + STORE_PTR((void **)global_data, NULL_REF); + global_data += sizeof(void *); + break; + } + + /* We can't create funcref obj during global instantiation + * since the functions are not instantiated yet, so we need + * to defer the initialization here */ + if (global_init + && (global_init->init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST) + && wasm_reftype_is_subtype_of( + global->type, global->ref_type, REF_TYPE_FUNCREF, + NULL, module_inst->module->types, + module_inst->module->type_count)) { + WASMFuncObjectRef func_obj = NULL; + /* UINT32_MAX indicates that it is a null reference */ + if ((uint32)global->initial_value.i32 != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module_inst, global->initial_value.i32, + false, error_buf, error_buf_size))) + goto fail; + } + STORE_PTR((void **)global_data, func_obj); + global_data += sizeof(void *); + /* Also update the inital_value since other globals may + * refer to this */ + global->initial_value.gc_obj = (wasm_obj_t)func_obj; + break; + } + else { + STORE_PTR((void **)global_data, + global->initial_value.gc_obj); + global_data += sizeof(void *); + break; + } +#endif + bh_assert(0); + break; + } + } + } + bh_assert(global_data == global_data_end); + } + + if (!check_linked_symbol(module_inst, error_buf, error_buf_size)) { + goto fail; + } + + /* Initialize the memory data with data segment section */ + for (i = 0; i < module->data_seg_count; i++) { + WASMMemoryInstance *memory = NULL; + uint8 *memory_data = NULL; + uint64 memory_size = 0; + WASMDataSeg *data_seg = module->data_segments[i]; + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (data_seg->is_passive) + continue; +#endif + if (is_sub_inst) + /* Ignore setting memory init data if the memory has been + initialized */ + continue; + + /* has check it in loader */ + memory = module_inst->memories[data_seg->memory_index]; + bh_assert(memory); + + memory_data = memory->memory_data; + memory_size = + (uint64)memory->num_bytes_per_page * memory->cur_page_count; + bh_assert(memory_data || memory_size == 0); + + bh_assert( + data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL + || (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST + && !memory->is_memory64) + || (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I64_CONST + && memory->is_memory64)); + + if (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { + if (!check_global_init_expr(module, + data_seg->base_offset.u.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + if (!globals + || globals[data_seg->base_offset.u.global_index].type + != (memory->is_memory64 ? VALUE_TYPE_I64 + : VALUE_TYPE_I32)) { + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); + goto fail; + } + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + base_offset = + (uint64)globals[data_seg->base_offset.u.global_index] + .initial_value.i64; + } + else +#endif + { + base_offset = + (uint32)globals[data_seg->base_offset.u.global_index] + .initial_value.i32; + } + } + else { +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + base_offset = (uint64)data_seg->base_offset.u.i64; + } + else +#endif + { + base_offset = (uint32)data_seg->base_offset.u.i32; + } + } + + /* check offset */ + if (base_offset > memory_size) { +#if WASM_ENABLE_MEMORY64 != 0 + LOG_DEBUG("base_offset(%" PRIu64 ") > memory_size(%" PRIu64 ")", + base_offset, memory_size); +#else + LOG_DEBUG("base_offset(%u) > memory_size(%" PRIu64 ")", base_offset, + memory_size); +#endif +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds memory access"); +#else + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); +#endif + goto fail; + } + + /* check offset + length(could be zero) */ + length = data_seg->data_length; + if ((uint64)base_offset + length > memory_size) { +#if WASM_ENABLE_MEMORY64 != 0 + LOG_DEBUG("base_offset(%" PRIu64 + ") + length(%d) > memory_size(%" PRIu64 ")", + base_offset, length, memory_size); +#else + LOG_DEBUG("base_offset(%u) + length(%d) > memory_size(%" PRIu64 ")", + base_offset, length, memory_size); +#endif +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds memory access"); +#else + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); +#endif + goto fail; + } + + if (memory_data) { + bh_memcpy_s(memory_data + base_offset, + (uint32)memory_size - base_offset, data_seg->data, + length); + } + } + +#if WASM_ENABLE_GC != 0 + /* Initialize the table data with init expr */ + for (i = 0; i < module->table_count; i++) { + WASMTable *table = module->tables + i; + WASMTableInstance *table_inst = module_inst->tables[i]; + table_elem_type_t *table_data; + uint32 j; + + if (table->init_expr.init_expr_type == 0) { + /* No table initializer */ + continue; + } + + table_data = table_inst->elems; + + bh_assert( + table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL + || table->init_expr.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST + || table->init_expr.init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST); + + if (table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { + if (!check_global_init_expr(module, table->init_expr.u.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + table->init_expr.u.gc_obj = + globals[table->init_expr.u.global_index].initial_value.gc_obj; + } + else if (table->init_expr.init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST) { + uint32 func_idx = table->init_expr.u.ref_index; + if (func_idx != UINT32_MAX) { + if (!(table->init_expr.u.gc_obj = + wasm_create_func_obj(module_inst, func_idx, false, + error_buf, error_buf_size))) + goto fail; + } + else { + table->init_expr.u.gc_obj = NULL_REF; + } + } + else if (table->init_expr.init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST) { + table->init_expr.u.gc_obj = NULL_REF; + } + + LOG_DEBUG("Init table [%d] elements from [%d] to [%d] as: %p", i, 0, + table_inst->cur_size, (void *)table->init_expr.u.gc_obj); + for (j = 0; j < table_inst->cur_size; j++) { + *(table_data + j) = table->init_expr.u.gc_obj; + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + /* Initialize the table data with table segment section */ + for (i = 0; module_inst->table_count > 0 && i < module->table_seg_count; + i++) { + WASMTableSeg *table_seg = module->table_segments + i; + /* has check it in loader */ + WASMTableInstance *table = module_inst->tables[table_seg->table_index]; + table_elem_type_t *table_data; + uint32 j; +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + uint8 tbl_elem_type; + uint32 tbl_init_size, tbl_max_size; +#endif +#if WASM_ENABLE_GC != 0 + WASMRefType *tbl_elem_ref_type; +#endif + + bh_assert(table); + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, table_seg->table_index, + &tbl_elem_type, +#if WASM_ENABLE_GC != 0 + &tbl_elem_ref_type, +#endif + &tbl_init_size, &tbl_max_size); + +#if WASM_ENABLE_GC == 0 + if (tbl_elem_type != VALUE_TYPE_FUNCREF + && tbl_elem_type != VALUE_TYPE_EXTERNREF) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); + goto fail; + } +#elif WASM_ENABLE_GC != 0 + if (!wasm_elem_is_declarative(table_seg->mode) + && !wasm_reftype_is_subtype_of( + table_seg->elem_type, table_seg->elem_ref_type, + table->elem_type, table->elem_ref_type.elem_ref_type, + module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); + goto fail; + } +#endif + (void)tbl_init_size; + (void)tbl_max_size; +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + + table_data = table->elems; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (table_seg->table_index < module->import_table_count + && module_inst->e->table_insts_linked[table_seg->table_index]) { + table_data = + module_inst->e->table_insts_linked[table_seg->table_index] + ->elems; + } +#endif + bh_assert(table_data); + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + if (!wasm_elem_is_active(table_seg->mode)) + continue; +#endif + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + bh_assert(table_seg->base_offset.init_expr_type + == INIT_EXPR_TYPE_I32_CONST + || table_seg->base_offset.init_expr_type + == INIT_EXPR_TYPE_GET_GLOBAL + || table_seg->base_offset.init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST + || table_seg->base_offset.init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST); +#else + bh_assert(table_seg->base_offset.init_expr_type + == INIT_EXPR_TYPE_I32_CONST + || table_seg->base_offset.init_expr_type + == INIT_EXPR_TYPE_GET_GLOBAL); +#endif + + /* init vec(funcidx) or vec(expr) */ + if (table_seg->base_offset.init_expr_type + == INIT_EXPR_TYPE_GET_GLOBAL) { + if (!check_global_init_expr(module, + table_seg->base_offset.u.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + if (!globals + || globals[table_seg->base_offset.u.global_index].type + != VALUE_TYPE_I32) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); + goto fail; + } + + table_seg->base_offset.u.i32 = + globals[table_seg->base_offset.u.global_index] + .initial_value.i32; + } + + /* check offset since length might negative */ + if ((uint32)table_seg->base_offset.u.i32 > table->cur_size) { + LOG_DEBUG("base_offset(%d) > table->cur_size(%d)", + table_seg->base_offset.u.i32, table->cur_size); +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); +#endif + goto fail; + } + + /* check offset + length(could be zero) */ + length = table_seg->value_count; + if ((uint32)table_seg->base_offset.u.i32 + length > table->cur_size) { + LOG_DEBUG("base_offset(%d) + length(%d)> table->cur_size(%d)", + table_seg->base_offset.u.i32, length, table->cur_size); +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); +#endif + goto fail; + } + + for (j = 0; j < length; j++) { + InitializerExpression *init_expr = &table_seg->init_values[j]; + uint8 flag = init_expr->init_expr_type; + void *ref = NULL; + + /* const and get global init values should be resolved during + * loading */ + bh_assert((flag == INIT_EXPR_TYPE_GET_GLOBAL) + || (flag == INIT_EXPR_TYPE_REFNULL_CONST) + || ((flag >= INIT_EXPR_TYPE_FUNCREF_CONST) + && (flag <= INIT_EXPR_TYPE_EXTERN_CONVERT_ANY))); + + switch (flag) { + case INIT_EXPR_TYPE_REFNULL_CONST: + ref = NULL; + break; + case INIT_EXPR_TYPE_FUNCREF_CONST: + { +#if WASM_ENABLE_GC == 0 + ref = (void *)(uintptr_t)init_expr->u.ref_index; +#else + WASMFuncObjectRef func_obj; + uint32 func_idx = init_expr->u.ref_index; + /* UINT32_MAX indicates that it is a null reference */ + if (func_idx != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module_inst, func_idx, false, error_buf, + error_buf_size))) { + goto fail; + } + ref = func_obj; + } + else { + ref = NULL_REF; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + break; + } +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr(module, + init_expr->u.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + ref = + globals[init_expr->u.global_index].initial_value.gc_obj; + break; + } + case INIT_EXPR_TYPE_STRUCT_NEW: + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + { + WASMRttType *rtt_type; + WASMStructObjectRef struct_obj; + WASMStructType *struct_type; + WASMStructNewInitValues *init_values = NULL; + uint32 type_idx; + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + init_values = + (WASMStructNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + } + else { + type_idx = init_expr->u.type_index; + } + + struct_type = (WASMStructType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + goto fail; + } + + if (!(struct_obj = wasm_struct_obj_new_internal( + module_inst->e->common.gc_heap_handle, + rtt_type))) { + set_error_buf(error_buf, error_buf_size, + "create struct object failed"); + goto fail; + } + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + uint32 field_idx; + + bh_assert(init_values->count + == struct_type->field_count); + + for (field_idx = 0; field_idx < init_values->count; + field_idx++) { + wasm_struct_obj_set_field( + struct_obj, field_idx, + &init_values->fields[field_idx]); + } + } + + ref = struct_obj; + break; + } + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + WASMRttType *rtt_type; + WASMArrayObjectRef array_obj; + WASMArrayType *array_type; + WASMArrayNewInitValues *init_values = NULL; + WASMValue *arr_init_val = NULL, empty_val = { 0 }; + uint32 type_idx, len; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { + type_idx = init_expr->u.array_new_default.type_index; + len = init_expr->u.array_new_default.length; + arr_init_val = &empty_val; + } + else { + init_values = + (WASMArrayNewInitValues *)init_expr->u.data; + type_idx = init_values->type_idx; + len = init_values->length; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + arr_init_val = init_values->elem_data; + } + } + + array_type = (WASMArrayType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + goto fail; + } + + if (!(array_obj = wasm_array_obj_new_internal( + module_inst->e->common.gc_heap_handle, rtt_type, + len, arr_init_val))) { + set_error_buf(error_buf, error_buf_size, + "create array object failed"); + goto fail; + } + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + uint32 elem_idx; + + bh_assert(init_values); + + for (elem_idx = 0; elem_idx < len; elem_idx++) { + wasm_array_obj_set_elem( + array_obj, elem_idx, + &init_values->elem_data[elem_idx]); + } + } + + ref = array_obj; + + break; + } + case INIT_EXPR_TYPE_I31_NEW: + { + ref = (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32); + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + } + + *(table_data + table_seg->base_offset.u.i32 + j) = + (table_elem_type_t)ref; + } + } + + /* Initialize the thread related data */ + if (stack_size == 0) + stack_size = DEFAULT_WASM_STACK_SIZE; +#if WASM_ENABLE_SPEC_TEST != 0 +#if WASM_ENABLE_TAIL_CALL == 0 + if (stack_size < 128 * 1024) + stack_size = 128 * 1024; +#else + /* Some tail-call cases require large operand stack */ + if (stack_size < 10 * 1024 * 1024) + stack_size = 10 * 1024 * 1024; +#endif +#endif + module_inst->default_wasm_stack_size = stack_size; + + if (module->malloc_function != (uint32)-1) { + module_inst->e->malloc_function = + &module_inst->e->functions[module->malloc_function]; + } + + if (module->free_function != (uint32)-1) { + module_inst->e->free_function = + &module_inst->e->functions[module->free_function]; + } + + if (module->retain_function != (uint32)-1) { + module_inst->e->retain_function = + &module_inst->e->functions[module->retain_function]; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + /* The sub-instance will get the wasi_ctx from main-instance */ + if (!is_sub_inst) { + if (!wasm_runtime_init_wasi( + (WASMModuleInstanceCommon *)module_inst, + module->wasi_args.dir_list, module->wasi_args.dir_count, + module->wasi_args.map_dir_list, module->wasi_args.map_dir_count, + module->wasi_args.env, module->wasi_args.env_count, + module->wasi_args.addr_pool, module->wasi_args.addr_count, + module->wasi_args.ns_lookup_pool, + module->wasi_args.ns_lookup_count, module->wasi_args.argv, + module->wasi_args.argc, module->wasi_args.stdio[0], + module->wasi_args.stdio[1], module->wasi_args.stdio[2], + error_buf, error_buf_size)) { + goto fail; + } + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!is_sub_inst) { + /* Add module instance into module's instance list */ + os_mutex_lock(&module->instance_list_lock); + if (module->instance_list) { + LOG_WARNING( + "warning: multiple instances referencing to the same module " + "may cause unexpected behaviour during debugging"); + } + module_inst->e->next = module->instance_list; + module->instance_list = module_inst; + os_mutex_unlock(&module->instance_list_lock); + } +#endif + + /* Set running mode before executing wasm functions */ + if (!set_running_mode(module_inst, wasm_runtime_get_default_running_mode(), + true)) { + set_error_buf(error_buf, error_buf_size, + "set instance running mode failed"); + goto fail; + } + + if (module->start_function != (uint32)-1) { + /* TODO: fix start function can be import function issue */ + if (module->start_function >= module->import_function_count) + module_inst->e->start_function = + &module_inst->e->functions[module->start_function]; + } + + if (!execute_post_instantiate_functions(module_inst, is_sub_inst, + exec_env_main)) { + set_error_buf(error_buf, error_buf_size, module_inst->cur_exception); + goto fail; + } + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_module_inst_mem_consumption( + (WASMModuleInstanceCommon *)module_inst); +#endif + + (void)global_data_end; + return module_inst; + +fail: + wasm_deinstantiate(module_inst, false); + return NULL; +} + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +static void +destroy_c_api_frames(Vector *frames) +{ + WASMCApiFrame frame = { 0 }; + uint32 i, total_frames, ret; + + total_frames = (uint32)bh_vector_size(frames); + + for (i = 0; i < total_frames; i++) { + ret = bh_vector_get(frames, i, &frame); + bh_assert(ret); + + if (frame.lp) + wasm_runtime_free(frame.lp); + } + + ret = bh_vector_destroy(frames); + bh_assert(ret); + (void)ret; +} +#endif + +void +wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) +{ + if (!module_inst) + return; + + if (module_inst->exec_env_singleton) { + /* wasm_exec_env_destroy will call + wasm_cluster_wait_for_all_except_self to wait for other + threads, so as to destroy their exec_envs and module + instances first, and avoid accessing the shared resources + of current module instance after it is deinstantiated. */ + wasm_exec_env_destroy(module_inst->exec_env_singleton); + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + /* Remove instance from module's instance list before freeing + func_ptrs and fast_jit_func_ptrs of the instance, to avoid + accessing the freed memory in the jit backend compilation + threads */ + { + WASMModule *module = module_inst->module; + WASMModuleInstance *instance_prev = NULL, *instance; + os_mutex_lock(&module->instance_list_lock); + + instance = module->instance_list; + while (instance) { + if (instance == module_inst) { + if (!instance_prev) + module->instance_list = instance->e->next; + else + instance_prev->e->next = instance->e->next; + break; + } + instance_prev = instance; + instance = instance->e->next; + } + + os_mutex_unlock(&module->instance_list_lock); + } +#endif + +#if WASM_ENABLE_JIT != 0 + if (module_inst->func_ptrs) + wasm_runtime_free(module_inst->func_ptrs); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (module_inst->fast_jit_func_ptrs + && module_inst->fast_jit_func_ptrs + != module_inst->module->fast_jit_func_ptrs) + wasm_runtime_free(module_inst->fast_jit_func_ptrs); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + if (module_inst->func_type_indexes) + wasm_runtime_free(module_inst->func_type_indexes); +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + wasm_runtime_sub_module_deinstantiate( + (WASMModuleInstanceCommon *)module_inst); +#endif + + if (module_inst->memory_count > 0) + memories_deinstantiate(module_inst, module_inst->memories, + module_inst->memory_count); + + if (module_inst->import_func_ptrs) { + wasm_runtime_free(module_inst->import_func_ptrs); + } + + tables_deinstantiate(module_inst); + functions_deinstantiate(module_inst->e->functions); +#if WASM_ENABLE_TAGS != 0 + tags_deinstantiate(module_inst->e->tags, module_inst->e->import_tag_ptrs); +#endif + globals_deinstantiate(module_inst->e->globals); + export_functions_deinstantiate(module_inst->export_functions); +#if WASM_ENABLE_TAGS != 0 + export_tags_deinstantiate(module_inst->e->export_tags); +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + export_globals_deinstantiate(module_inst->export_globals); +#endif + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + wasm_externref_cleanup((WASMModuleInstanceCommon *)module_inst); +#endif + +#if WASM_ENABLE_GC != 0 + if (!is_sub_inst) { + if (module_inst->e->common.gc_heap_handle) + mem_allocator_destroy(module_inst->e->common.gc_heap_handle); + if (module_inst->e->common.gc_heap_pool) + wasm_runtime_free(module_inst->e->common.gc_heap_pool); + } +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (module_inst->frames) { + destroy_c_api_frames(module_inst->frames); + wasm_runtime_free(module_inst->frames); + module_inst->frames = NULL; + } +#endif + + if (module_inst->c_api_func_imports) + wasm_runtime_free(module_inst->c_api_func_imports); + + if (!is_sub_inst) { +#if WASM_ENABLE_WASI_NN != 0 + wasi_nn_destroy(module_inst); +#endif + wasm_native_call_context_dtors((WASMModuleInstanceCommon *)module_inst); + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap_delete(module_inst->e->common.data_dropped); +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap_delete(module_inst->e->common.elem_dropped); +#endif + + wasm_runtime_free(module_inst); +} + +WASMFunctionInstance * +wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name) +{ + uint32 i; + for (i = 0; i < module_inst->export_func_count; i++) + if (!strcmp(module_inst->export_functions[i].name, name)) + return module_inst->export_functions[i].function; + return NULL; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASMGlobalInstance * +wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name) +{ + uint32 i; + for (i = 0; i < module_inst->export_global_count; i++) + if (!strcmp(module_inst->export_globals[i].name, name)) + return module_inst->export_globals[i].global; + return NULL; +} + +WASMMemoryInstance * +wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name) +{ + /** + * using a strong assumption that one module instance only has + * one memory instance + */ + (void)module_inst->export_memories; + return module_inst->memories[0]; +} + +WASMTableInstance * +wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name) +{ + /** + * using a strong assumption that one module instance only has + * one table instance + */ + (void)module_inst->export_tables; + return module_inst->tables[0]; +} + +#if WASM_ENABLE_TAGS != 0 +WASMTagInstance * +wasm_lookup_tag(const WASMModuleInstance *module_inst, const char *name, + const char *signature) +{ + uint32 i; + for (i = 0; i < module_inst->e->export_tag_count; i++) + if (!strcmp(module_inst->e->export_tags[i].name, name)) + return module_inst->e->export_tags[i].tag; + (void)signature; + return NULL; +} +#endif + +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +static void +call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *function, unsigned argc, + uint32 argv[]) +{ + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); + WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env); + uint8 *prev_top = exec_env->wasm_stack.top; +#ifdef BH_PLATFORM_WINDOWS + int result; + bool has_exception; + char exception[EXCEPTION_BUF_LEN]; +#endif + bool ret = true; + + /* Check native stack overflow firstly to ensure we have enough + native stack to run the following codes before actually calling + the aot function in invokeNative function. */ + RECORD_STACK_USAGE(exec_env, (uint8 *)&exec_env_tls); + if ((uint8 *)&exec_env_tls < exec_env->native_stack_boundary + + page_size * (guard_page_count + 1)) { + wasm_set_exception(module_inst, "native stack overflow"); + return; + } + + if (!exec_env_tls) { + if (!os_thread_signal_inited()) { + wasm_set_exception(module_inst, "thread signal env not inited"); + return; + } + + /* Set thread handle and stack boundary if they haven't been set */ + wasm_exec_env_set_thread_info(exec_env); + + wasm_runtime_set_exec_env_tls(exec_env); + } + else { + if (exec_env_tls != exec_env) { + wasm_set_exception(module_inst, "invalid exec env"); + return; + } + } + + wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); + + if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { +#ifndef BH_PLATFORM_WINDOWS + wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); +#else + __try { + wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); + } __except (wasm_copy_exception(module_inst, NULL) + ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { + /* Exception was thrown in wasm_exception_handler */ + ret = false; + } + has_exception = wasm_copy_exception(module_inst, exception); + if (has_exception && strstr(exception, "native stack overflow")) { + /* After a stack overflow, the stack was left + in a damaged state, let the CRT repair it */ + result = _resetstkoflw(); + bh_assert(result != 0); + } +#endif + } + else { + /* Exception has been set in signal handler before calling longjmp */ + ret = false; + } + + /* Note: can't check wasm_get_exception(module_inst) here, there may be + * exception which is not caught by hardware (e.g. uninitialized elements), + * then the stack-frame is already freed inside wasm_interp_call_wasm */ + if (!ret) { +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (wasm_interp_create_call_stack(exec_env)) { + wasm_interp_dump_call_stack(exec_env, true, NULL, 0); + } +#endif + /* Restore operand frames */ + wasm_exec_env_set_cur_frame(exec_env, prev_frame); + exec_env->wasm_stack.top = prev_top; + } + + jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env); + bh_assert(&jmpbuf_node == jmpbuf_node_pop); + if (!exec_env->jmpbuf_stack_top) { + wasm_runtime_set_exec_env_tls(NULL); + } + if (!ret) { + os_sigreturn(); + os_signal_unmask(); + } + (void)jmpbuf_node_pop; +} +#define interp_call_wasm call_wasm_with_hw_bound_check +#else +#define interp_call_wasm wasm_interp_call_wasm +#endif + +bool +wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function, + unsigned argc, uint32 argv[]) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + +#ifndef OS_ENABLE_HW_BOUND_CHECK + /* Set thread handle and stack boundary */ + wasm_exec_env_set_thread_info(exec_env); +#else + /* Set thread info in call_wasm_with_hw_bound_check when + hw bound check is enabled */ +#endif + + /* Set exec env, so it can be later retrieved from instance */ + module_inst->cur_exec_env = exec_env; + + interp_call_wasm(module_inst, exec_env, function, argc, argv); + return !wasm_copy_exception(module_inst, NULL); +} + +#if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 +/* look for the function name */ +static char * +get_func_name_from_index(const WASMModuleInstance *inst, uint32 func_index) +{ + char *func_name = NULL; + WASMFunctionInstance *func_inst = inst->e->functions + func_index; + + if (func_inst->is_import_func) { + func_name = func_inst->u.func_import->field_name; + } + else { +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + func_name = func_inst->u.func->field_name; +#endif + /* if custom name section is not generated, + search symbols from export table */ + if (!func_name) { + unsigned j; + for (j = 0; j < inst->export_func_count; j++) { + WASMExportFuncInstance *export_func = + inst->export_functions + j; + if (export_func->function == func_inst) { + func_name = export_func->name; + break; + } + } + } + } + + return func_name; +} +#endif /*WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0*/ + +#if WASM_ENABLE_PERF_PROFILING != 0 +void +wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) +{ + WASMFunctionInstance *func_inst; + char *func_name; + uint32 i; + + os_printf("Performance profiler data:\n"); + for (i = 0; i < module_inst->e->function_count; i++) { + func_inst = module_inst->e->functions + i; + + if (func_inst->total_exec_cnt == 0) + continue; + + func_name = get_func_name_from_index(module_inst, i); + if (func_name) + os_printf( + " func %s, execution time: %.3f ms, execution count: %" PRIu32 + " times, children execution time: %.3f ms\n", + func_name, func_inst->total_exec_time / 1000.0f, + func_inst->total_exec_cnt, + func_inst->children_exec_time / 1000.0f); + else + os_printf(" func %" PRIu32 + ", execution time: %.3f ms, execution count: %" PRIu32 + " times, children execution time: %.3f ms\n", + i, func_inst->total_exec_time / 1000.0f, + func_inst->total_exec_cnt, + func_inst->children_exec_time / 1000.0f); + } +} + +double +wasm_summarize_wasm_execute_time(const WASMModuleInstance *inst) +{ + double ret = 0; + + unsigned i; + for (i = 0; i < inst->e->function_count; i++) { + WASMFunctionInstance *func = inst->e->functions + i; + ret += (func->total_exec_time - func->children_exec_time) / 1000.0f; + } + + return ret; +} + +double +wasm_get_wasm_func_exec_time(const WASMModuleInstance *inst, + const char *func_name) +{ + unsigned i; + for (i = 0; i < inst->e->function_count; i++) { + char *name_in_wasm = get_func_name_from_index(inst, i); + if (name_in_wasm && strcmp(name_in_wasm, func_name) == 0) { + WASMFunctionInstance *func = inst->e->functions + i; + return (func->total_exec_time - func->children_exec_time) / 1000.0f; + } + } + + return -1.0; +} +#endif /*WASM_ENABLE_PERF_PROFILING != 0*/ + +uint64 +wasm_module_malloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); + uint8 *addr = NULL; + uint64 offset = 0; + + /* TODO: Memory64 size check based on memory idx type */ + bh_assert(size <= UINT32_MAX); + + if (!memory) { + wasm_set_exception(module_inst, "uninitialized memory"); + return 0; + } + + if (memory->heap_handle) { + addr = mem_allocator_malloc(memory->heap_handle, (uint32)size); + } + else if (module_inst->e->malloc_function && module_inst->e->free_function) { + if (!execute_malloc_function( + module_inst, exec_env, module_inst->e->malloc_function, + module_inst->e->retain_function, size, &offset)) { + return 0; + } + /* If we use app's malloc function, + the default memory may be changed while memory growing */ + memory = wasm_get_default_memory(module_inst); + addr = offset ? memory->memory_data + offset : NULL; + } + + if (!addr) { + if (memory->heap_handle + && mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + wasm_set_exception(module_inst, "app heap corrupted"); + } + else { + LOG_WARNING("warning: allocate %" PRIu64 " bytes memory failed", + size); + } + return 0; + } + if (p_native_addr) + *p_native_addr = addr; + + return (uint64)(addr - memory->memory_data); +} + +uint64 +wasm_module_realloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr, uint64 size, + void **p_native_addr) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); + uint8 *addr = NULL; + + /* TODO: Memory64 ptr and size check based on memory idx type */ + bh_assert(ptr <= UINT32_MAX); + bh_assert(size <= UINT32_MAX); + + if (!memory) { + wasm_set_exception(module_inst, "uninitialized memory"); + return 0; + } + + if (memory->heap_handle) { + addr = mem_allocator_realloc( + memory->heap_handle, + (uint32)ptr ? memory->memory_data + (uint32)ptr : NULL, + (uint32)size); + } + + /* Only support realloc in WAMR's app heap */ + (void)exec_env; + + if (!addr) { + if (memory->heap_handle + && mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_set_exception(module_inst, "app heap corrupted"); + } + else { + wasm_set_exception(module_inst, "out of memory"); + } + return 0; + } + if (p_native_addr) + *p_native_addr = addr; + + return (uint64)(addr - memory->memory_data); +} + +void +wasm_module_free_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); + + /* TODO: Memory64 ptr and size check based on memory idx type */ + bh_assert(ptr <= UINT32_MAX); + + if (!memory) { + return; + } + + if (ptr) { + uint8 *addr = memory->memory_data + (uint32)ptr; + uint8 *memory_data_end; + + /* memory->memory_data_end may be changed in memory grow */ + SHARED_MEMORY_LOCK(memory); + memory_data_end = memory->memory_data_end; + SHARED_MEMORY_UNLOCK(memory); + + if (memory->heap_handle && memory->heap_data <= addr + && addr < memory->heap_data_end) { + mem_allocator_free(memory->heap_handle, addr); + } + else if (module_inst->e->malloc_function + && module_inst->e->free_function && memory->memory_data <= addr + && addr < memory_data_end) { + execute_free_function(module_inst, exec_env, + module_inst->e->free_function, ptr); + } + } +} + +uint64 +wasm_module_malloc(WASMModuleInstance *module_inst, uint64 size, + void **p_native_addr) +{ + return wasm_module_malloc_internal(module_inst, NULL, size, p_native_addr); +} + +uint64 +wasm_module_realloc(WASMModuleInstance *module_inst, uint64 ptr, uint64 size, + void **p_native_addr) +{ + return wasm_module_realloc_internal(module_inst, NULL, ptr, size, + p_native_addr); +} + +void +wasm_module_free(WASMModuleInstance *module_inst, uint64 ptr) +{ + wasm_module_free_internal(module_inst, NULL, ptr); +} + +uint64 +wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, + uint64 size) +{ + char *buffer; + uint64 buffer_offset; + + /* TODO: Memory64 size check based on memory idx type */ + bh_assert(size <= UINT32_MAX); + + buffer_offset = wasm_module_malloc(module_inst, size, (void **)&buffer); + + if (buffer_offset != 0) { + buffer = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, buffer_offset); + bh_memcpy_s(buffer, (uint32)size, src, (uint32)size); + } + return buffer_offset; +} + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +bool +wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, + uint32 inc_size, table_elem_type_t init_val) +{ + uint32 total_size, i; + table_elem_type_t *new_table_data_start; + WASMTableInstance *table_inst; + + if (!inc_size) { + return true; + } + + bh_assert(table_idx < module_inst->table_count); + table_inst = wasm_get_table_inst(module_inst, table_idx); + if (!table_inst) { + return false; + } + + if (inc_size > UINT32_MAX - table_inst->cur_size) { + return false; + } + + total_size = table_inst->cur_size + inc_size; + if (total_size > table_inst->max_size) { + return false; + } + + /* fill in */ + new_table_data_start = table_inst->elems + table_inst->cur_size; + for (i = 0; i < inc_size; ++i) { + new_table_data_start[i] = init_val; + } + + table_inst->cur_size = total_size; + return true; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +static bool +call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 tbl_elem_idx, + uint32 argc, uint32 argv[], bool check_type_idx, uint32 type_idx) +{ + WASMModuleInstance *module_inst = NULL; + WASMTableInstance *table_inst = NULL; + table_elem_type_t tbl_elem_val = NULL_REF; + uint32 func_idx = 0; + WASMFunctionInstance *func_inst = NULL; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + bh_assert(module_inst); + + table_inst = module_inst->tables[tbl_idx]; + if (!table_inst) { + wasm_set_exception(module_inst, "unknown table"); + goto got_exception; + } + + if (tbl_elem_idx >= table_inst->cur_size) { + wasm_set_exception(module_inst, "undefined element"); + goto got_exception; + } + + tbl_elem_val = ((table_elem_type_t *)table_inst->elems)[tbl_elem_idx]; + if (tbl_elem_val == NULL_REF) { + wasm_set_exception(module_inst, "uninitialized element"); + goto got_exception; + } + +#if WASM_ENABLE_GC == 0 + func_idx = (uint32)tbl_elem_val; +#else + func_idx = + wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val); +#endif + + /** + * we insist to call functions owned by the module itself + **/ + if (func_idx >= module_inst->e->function_count) { + wasm_set_exception(module_inst, "unknown function"); + goto got_exception; + } + + func_inst = module_inst->e->functions + func_idx; + + if (check_type_idx) { + WASMType *cur_type = module_inst->module->types[type_idx]; + WASMType *cur_func_type; + + if (func_inst->is_import_func) + cur_func_type = (WASMType *)func_inst->u.func_import->func_type; + else + cur_func_type = (WASMType *)func_inst->u.func->func_type; + + if (cur_type != cur_func_type) { + wasm_set_exception(module_inst, "indirect call type mismatch"); + goto got_exception; + } + } + + interp_call_wasm(module_inst, exec_env, func_inst, argc, argv); + + return !wasm_copy_exception(module_inst, NULL); + +got_exception: + return false; +} + +bool +wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 argv[]) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +} + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, uint32 size) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + uint32 stack_top_idx = module_inst->module->aux_stack_top_global_index; + +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0 + /* Check the aux stack space */ + uint64 data_end = module_inst->module->aux_data_end; + uint64 stack_bottom = module_inst->module->aux_stack_bottom; + bool is_stack_before_data = stack_bottom < data_end ? true : false; + if ((is_stack_before_data && (size > start_offset)) + || ((!is_stack_before_data) && (start_offset - data_end < size))) + return false; +#endif + + if (stack_top_idx != (uint32)-1) { + /* The aux stack top is a wasm global, + set the initial value for the global */ + uint8 *global_addr = + module_inst->global_data + + module_inst->e->globals[stack_top_idx].data_offset; + *(int32 *)global_addr = (uint32)start_offset; + /* The aux stack boundary is a constant value, + set the value to exec_env */ + exec_env->aux_stack_boundary = (uintptr_t)start_offset - size; + exec_env->aux_stack_bottom = (uintptr_t)start_offset; + return true; + } + + return false; +} + +bool +wasm_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, uint32 *size) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + + /* The aux stack information is resolved in loader + and store in module */ + uint64 stack_bottom = module_inst->module->aux_stack_bottom; + uint32 total_aux_stack_size = module_inst->module->aux_stack_size; + + if (stack_bottom != 0 && total_aux_stack_size != 0) { + if (start_offset) + *start_offset = stack_bottom; + if (size) + *size = total_aux_stack_size; + return true; + } + return false; +} +#endif + +#if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0) +void +wasm_get_module_mem_consumption(const WASMModule *module, + WASMModuleMemConsumption *mem_conspn) +{ + uint32 i, size; + + memset(mem_conspn, 0, sizeof(*mem_conspn)); + + mem_conspn->module_struct_size = sizeof(WASMModule); + + mem_conspn->types_size = sizeof(WASMFuncType *) * module->type_count; + for (i = 0; i < module->type_count; i++) { + WASMFuncType *type = module->types[i]; + size = offsetof(WASMFuncType, types) + + sizeof(uint8) * (type->param_count + type->result_count); + mem_conspn->types_size += size; + } + + mem_conspn->imports_size = sizeof(WASMImport) * module->import_count; + + mem_conspn->functions_size = + sizeof(WASMFunction *) * module->function_count; + for (i = 0; i < module->function_count; i++) { + WASMFunction *func = module->functions[i]; + WASMFuncType *type = func->func_type; + size = sizeof(WASMFunction) + func->local_count + + sizeof(uint16) * (type->param_count + func->local_count); +#if WASM_ENABLE_FAST_INTERP != 0 + size += + func->code_compiled_size + sizeof(uint32) * func->const_cell_num; +#endif + mem_conspn->functions_size += size; + } + + mem_conspn->tables_size = sizeof(WASMTable) * module->table_count; + mem_conspn->memories_size = sizeof(WASMMemory) * module->memory_count; + mem_conspn->globals_size = sizeof(WASMGlobal) * module->global_count; + mem_conspn->exports_size = sizeof(WASMExport) * module->export_count; + + mem_conspn->table_segs_size = + sizeof(WASMTableSeg) * module->table_seg_count; + for (i = 0; i < module->table_seg_count; i++) { + WASMTableSeg *table_seg = &module->table_segments[i]; + mem_conspn->tables_size += + sizeof(InitializerExpression *) * table_seg->value_count; + } + + mem_conspn->data_segs_size = sizeof(WASMDataSeg *) * module->data_seg_count; + for (i = 0; i < module->data_seg_count; i++) { + mem_conspn->data_segs_size += sizeof(WASMDataSeg); + } + + if (module->const_str_list) { + StringNode *node = module->const_str_list, *node_next; + while (node) { + node_next = node->next; + mem_conspn->const_strs_size += + sizeof(StringNode) + strlen(node->str) + 1; + node = node_next; + } + } + + mem_conspn->total_size += mem_conspn->module_struct_size; + mem_conspn->total_size += mem_conspn->types_size; + mem_conspn->total_size += mem_conspn->imports_size; + mem_conspn->total_size += mem_conspn->functions_size; + mem_conspn->total_size += mem_conspn->tables_size; + mem_conspn->total_size += mem_conspn->memories_size; + mem_conspn->total_size += mem_conspn->globals_size; + mem_conspn->total_size += mem_conspn->exports_size; + mem_conspn->total_size += mem_conspn->table_segs_size; + mem_conspn->total_size += mem_conspn->data_segs_size; + mem_conspn->total_size += mem_conspn->const_strs_size; +} + +void +wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst, + WASMModuleInstMemConsumption *mem_conspn) +{ + uint32 i; + uint64 size; + + memset(mem_conspn, 0, sizeof(*mem_conspn)); + + mem_conspn->module_inst_struct_size = (uint8 *)module_inst->e + - (uint8 *)module_inst + + sizeof(WASMModuleInstanceExtra); + + mem_conspn->memories_size = + sizeof(WASMMemoryInstance *) * module_inst->memory_count; + for (i = 0; i < module_inst->memory_count; i++) { + WASMMemoryInstance *memory = module_inst->memories[i]; + size = memory->num_bytes_per_page * memory->cur_page_count; + mem_conspn->memories_size += size; + mem_conspn->app_heap_size += memory->heap_data_end - memory->heap_data; + /* size of app heap structure */ + mem_conspn->memories_size += mem_allocator_get_heap_struct_size(); + /* Module instance structures have been appened into the end of + module instance */ + } + + mem_conspn->tables_size = + sizeof(WASMTableInstance *) * module_inst->table_count; + /* Table instance structures and table elements have been appened into + the end of module instance */ + + mem_conspn->functions_size = + sizeof(WASMFunctionInstance) * module_inst->e->function_count; + + mem_conspn->globals_size = + sizeof(WASMGlobalInstance) * module_inst->e->global_count; + /* Global data has been appened into the end of module instance */ + + mem_conspn->exports_size = + sizeof(WASMExportFuncInstance) * module_inst->export_func_count; + + mem_conspn->total_size += mem_conspn->module_inst_struct_size; + mem_conspn->total_size += mem_conspn->memories_size; + mem_conspn->total_size += mem_conspn->functions_size; + mem_conspn->total_size += mem_conspn->tables_size; + mem_conspn->total_size += mem_conspn->globals_size; + mem_conspn->total_size += mem_conspn->exports_size; +} +#endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \ + || (WASM_ENABLE_MEMORY_TRACING != 0) */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +bool +wasm_interp_create_call_stack(struct WASMExecEnv *exec_env) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env); + WASMModule *module = module_inst->module; + WASMInterpFrame *first_frame, + *cur_frame = wasm_exec_env_get_cur_frame(exec_env); + uint32 n = 0; + + /* count frames includes a function */ + first_frame = cur_frame; + while (cur_frame) { + if (cur_frame->function) { + n++; + } + cur_frame = cur_frame->prev_frame; + } + + /* release previous stack frames and create new ones */ + destroy_c_api_frames(module_inst->frames); + if (!bh_vector_init(module_inst->frames, n, sizeof(WASMCApiFrame), false)) { + return false; + } + + cur_frame = first_frame; + n = 0; + + while (cur_frame) { + WASMCApiFrame frame = { 0 }; + WASMFunctionInstance *func_inst = cur_frame->function; + const char *func_name = NULL; + const uint8 *func_code_base = NULL; + uint32 max_local_cell_num, max_stack_cell_num; + uint32 all_cell_num, lp_size; + + if (!func_inst) { + cur_frame = cur_frame->prev_frame; + continue; + } + + /* place holder, will overwrite it in wasm_c_api */ + frame.instance = module_inst; + frame.module_offset = 0; + frame.func_index = (uint32)(func_inst - module_inst->e->functions); + + func_code_base = wasm_get_func_code(func_inst); + if (!cur_frame->ip || !func_code_base) { + frame.func_offset = 0; + } + else { +#if WASM_ENABLE_FAST_INTERP == 0 + frame.func_offset = (uint32)(cur_frame->ip - module->load_addr); +#else + frame.func_offset = (uint32)(cur_frame->ip - func_code_base); +#endif + } + + func_name = get_func_name_from_index(module_inst, frame.func_index); + frame.func_name_wp = func_name; + + if (frame.func_index >= module->import_function_count) { + uint32 wasm_func_idx = + frame.func_index - module->import_function_count; + max_local_cell_num = + module->functions[wasm_func_idx]->param_cell_num + + module->functions[wasm_func_idx]->local_cell_num; + max_stack_cell_num = + module->functions[wasm_func_idx]->max_stack_cell_num; + all_cell_num = max_local_cell_num + max_stack_cell_num; +#if WASM_ENABLE_FAST_INTERP != 0 + all_cell_num += module->functions[wasm_func_idx]->const_cell_num; +#endif + } + else { + WASMFuncType *func_type = + module->import_functions[frame.func_index].u.function.func_type; + max_local_cell_num = + func_type->param_cell_num > 2 ? func_type->param_cell_num : 2; + max_stack_cell_num = 0; + all_cell_num = max_local_cell_num + max_stack_cell_num; + } + +#if WASM_ENABLE_GC == 0 + lp_size = all_cell_num * 4; +#else + lp_size = align_uint(all_cell_num * 5, 4); +#endif + if (lp_size > 0) { + if (!(frame.lp = wasm_runtime_malloc(lp_size))) { + destroy_c_api_frames(module_inst->frames); + return false; + } + bh_memcpy_s(frame.lp, lp_size, cur_frame->lp, lp_size); + +#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_FAST_INTERP == 0 + frame.sp = frame.lp + (cur_frame->sp - cur_frame->lp); +#else + /* for fast-interp, let frame sp point to the end of the frame */ + frame.sp = frame.lp + all_cell_num; +#endif + frame.frame_ref = (uint8 *)frame.lp + + (wasm_interp_get_frame_ref(cur_frame) + - (uint8 *)cur_frame->lp); +#endif + } + + if (!bh_vector_append(module_inst->frames, &frame)) { + if (frame.lp) + wasm_runtime_free(frame.lp); + destroy_c_api_frames(module_inst->frames); + return false; + } + + cur_frame = cur_frame->prev_frame; + n++; + } + + return true; +} + +#define PRINT_OR_DUMP() \ + do { \ + total_len += \ + wasm_runtime_dump_line_buf_impl(line_buf, print, &buf, &len); \ + if ((!print) && buf && (len == 0)) { \ + exception_unlock(module_inst); \ + return total_len; \ + } \ + } while (0) + +uint32 +wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf, + uint32 len) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env); + uint32 n = 0, total_len = 0, total_frames; + /* reserve 256 bytes for line buffer, any line longer than 256 bytes + * will be truncated */ + char line_buf[256]; + + if (!module_inst->frames) { + return 0; + } + + total_frames = (uint32)bh_vector_size(module_inst->frames); + if (total_frames == 0) { + return 0; + } + + exception_lock(module_inst); + snprintf(line_buf, sizeof(line_buf), "\n"); + PRINT_OR_DUMP(); + + while (n < total_frames) { + WASMCApiFrame frame = { 0 }; + uint32 line_length, i; + + if (!bh_vector_get(module_inst->frames, n, &frame)) { + exception_unlock(module_inst); + return 0; + } + +#if WASM_ENABLE_FAST_JIT != 0 + /* Fast JIT doesn't support committing ip (instruction pointer) yet */ + if (module_inst->e->running_mode == Mode_Fast_JIT + || module_inst->e->running_mode == Mode_Multi_Tier_JIT) { + /* function name not exported, print number instead */ + if (frame.func_name_wp == NULL) { + line_length = snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 " $f%" PRIu32 "\n", n, + frame.func_index); + } + else { + line_length = + snprintf(line_buf, sizeof(line_buf), "#%02" PRIu32 " %s\n", + n, frame.func_name_wp); + } + } + else +#endif + { + /* function name not exported, print number instead */ + if (frame.func_name_wp == NULL) { + line_length = + snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 ": 0x%04x - $f%" PRIu32 "\n", n, + frame.func_offset, frame.func_index); + } + else { + line_length = snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 ": 0x%04x - %s\n", n, + frame.func_offset, frame.func_name_wp); + } + } + + if (line_length >= sizeof(line_buf)) { + uint32 line_buffer_len = sizeof(line_buf); + /* If line too long, ensure the last character is '\n' */ + for (i = line_buffer_len - 5; i < line_buffer_len - 2; i++) { + line_buf[i] = '.'; + } + line_buf[line_buffer_len - 2] = '\n'; + } + + PRINT_OR_DUMP(); + + n++; + } + snprintf(line_buf, sizeof(line_buf), "\n"); + PRINT_OR_DUMP(); + exception_unlock(module_inst); + + return total_len + 1; +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + if (id != EXCE_ALREADY_THROWN) + wasm_set_exception_with_id(module_inst, id); +#ifdef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_access_exce_check_guard_page(); +#endif +} + +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr) +{ + bool ret = wasm_check_app_addr_and_convert( + module_inst, is_str, app_buf_addr, app_buf_size, p_native_addr); + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + + return ret; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true, + type_idx); +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 */ + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv) +{ + bool ret; + + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); + + ret = call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv) +{ + WASMModuleInstance *module_inst; + WASMModule *module; + uint32 *func_type_indexes; + uint32 func_type_idx; + WASMFuncType *func_type; + void *func_ptr; + WASMFunctionImport *import_func; + CApiFuncImport *c_api_func_import = NULL; + const char *signature; + void *attachment; + char buf[96]; + bool ret = false; + + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); + + module_inst = (WASMModuleInstance *)wasm_runtime_get_module_inst(exec_env); + module = module_inst->module; + func_type_indexes = module_inst->func_type_indexes; + func_type_idx = func_type_indexes[func_idx]; + func_type = (WASMFuncType *)module->types[func_type_idx]; + func_ptr = module_inst->func_ptrs[func_idx]; + + bh_assert(func_idx < module->import_function_count); + + import_func = &module->import_functions[func_idx].u.function; + if (import_func->call_conv_wasm_c_api) { + if (module_inst->c_api_func_imports) { + c_api_func_import = module_inst->c_api_func_imports + func_idx; + func_ptr = c_api_func_import->func_ptr_linked; + } + else { + c_api_func_import = NULL; + func_ptr = NULL; + } + } + + if (!func_ptr) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + import_func->module_name, import_func->field_name); + wasm_set_exception(module_inst, buf); + goto fail; + } + + attachment = import_func->attachment; + if (import_func->call_conv_wasm_c_api) { + ret = wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc, + argv, c_api_func_import->with_env_arg, c_api_func_import->env_arg); + } + else if (!import_func->call_conv_raw) { + signature = import_func->signature; + ret = + wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv); + } + else { + signature = import_func->signature; + ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); + } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, uint32 dst) +{ + WASMMemoryInstance *memory_inst; + WASMModule *module; + uint8 *data; + uint8 *maddr; + uint64 seg_len; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + memory_inst = wasm_get_default_memory(module_inst); + + if (bh_bitmap_get_bit(module_inst->e->common.data_dropped, seg_index)) { + seg_len = 0; + data = NULL; + } + else { + module = module_inst->module; + seg_len = module->data_segments[seg_index]->data_length; + data = module->data_segments[seg_index]->data; + } + + if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, + (uint64)dst, (uint64)len)) + return false; + + if ((uint64)offset + (uint64)len > seg_len) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + maddr = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, (uint64)dst); + + SHARED_MEMORY_LOCK(memory_inst); + bh_memcpy_s(maddr, (uint32)(memory_inst->memory_data_size - dst), + data + offset, len); + SHARED_MEMORY_UNLOCK(memory_inst); + return true; +} + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index) +{ + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + bh_bitmap_set_bit(module_inst->e->common.data_dropped, seg_index); + /* Currently we can't free the dropped data segment + as they are stored in wasm bytecode */ + return true; +} +#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 */ + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx) +{ + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + bh_bitmap_set_bit(module_inst->e->common.elem_dropped, tbl_seg_idx); +} + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *tbl_inst; + WASMTableSeg *tbl_seg; + table_elem_type_t *table_elems; + InitializerExpression *tbl_seg_init_values = NULL, *init_values; + uint32 i, tbl_seg_len = 0; +#if WASM_ENABLE_GC != 0 + void *func_obj; +#endif + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + tbl_seg = module_inst->module->table_segments + tbl_seg_idx; + + bh_assert(tbl_inst); + bh_assert(tbl_seg); + + if (!bh_bitmap_get_bit(module_inst->e->common.elem_dropped, tbl_seg_idx)) { + /* table segment isn't dropped */ + tbl_seg_init_values = tbl_seg->init_values; + tbl_seg_len = tbl_seg->value_count; + } + + if (offset_len_out_of_bounds(src_offset, length, tbl_seg_len) + || offset_len_out_of_bounds(dst_offset, length, tbl_inst->cur_size)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + if (!length) { + return; + } + + table_elems = tbl_inst->elems + dst_offset; + init_values = tbl_seg_init_values + src_offset; + + for (i = 0; i < length; i++) { +#if WASM_ENABLE_GC != 0 + /* UINT32_MAX indicates that it is a null ref */ + if (init_values[i].u.ref_index != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj(module_inst, + init_values[i].u.ref_index, + true, NULL, 0))) { + wasm_set_exception(module_inst, "null function object"); + return; + } + table_elems[i] = func_obj; + } + else { + table_elems[i] = NULL_REF; + } +#else + table_elems[i] = init_values[i].u.ref_index; +#endif + } +} + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *src_tbl_inst; + WASMTableInstance *dst_tbl_inst; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + src_tbl_inst = wasm_get_table_inst(module_inst, src_tbl_idx); + dst_tbl_inst = wasm_get_table_inst(module_inst, dst_tbl_idx); + bh_assert(src_tbl_inst); + bh_assert(dst_tbl_inst); + + if (offset_len_out_of_bounds(dst_offset, length, dst_tbl_inst->cur_size) + || offset_len_out_of_bounds(src_offset, length, + src_tbl_inst->cur_size)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + /* if src_offset >= dst_offset, copy from front to back */ + /* if src_offset < dst_offset, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(table_elem_type_t) * dst_offset, + (uint32)sizeof(table_elem_type_t) + * (dst_tbl_inst->cur_size - dst_offset), + (uint8 *)src_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(table_elem_type_t) * src_offset, + (uint32)sizeof(table_elem_type_t) * length); +} + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uintptr_t val, uint32 data_offset) +{ + WASMTableInstance *tbl_inst; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + bh_assert(tbl_inst); + + if (offset_len_out_of_bounds(data_offset, length, tbl_inst->cur_size)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + for (; length != 0; data_offset++, length--) { + tbl_inst->elems[data_offset] = (table_elem_type_t)val; + } +} + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_size, uintptr_t init_val) +{ + WASMTableInstance *tbl_inst; + uint32 i, orig_size, total_size; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + if (!tbl_inst) { + return (uint32)-1; + } + + orig_size = tbl_inst->cur_size; + + if (!inc_size) { + return orig_size; + } + + if (tbl_inst->cur_size > UINT32_MAX - inc_size) { /* integer overflow */ +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of integer overflow", + tbl_inst->cur_size, inc_size); +#endif + return (uint32)-1; + } + + total_size = tbl_inst->cur_size + inc_size; + if (total_size > tbl_inst->max_size) { +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of over max size", + tbl_inst->cur_size, inc_size); +#endif + return (uint32)-1; + } + + /* fill in */ + for (i = 0; i < inc_size; ++i) { + tbl_inst->elems[tbl_inst->cur_size + i] = (table_elem_type_t)init_val; + } + + tbl_inst->cur_size = total_size; + return orig_size; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +llvm_jit_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, + uint32 error_buf_size) +{ + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + return wasm_create_func_obj(module_inst, func_idx, throw_exce, error_buf, + error_buf_size); +} + +bool +llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst, + WASMObjectRef gc_obj, uint32 type_index) +{ + WASMModule *module = module_inst->module; + WASMType **types = module->types; + uint32 type_count = module->type_count; + + return wasm_obj_is_instance_of(gc_obj, type_index, types, type_count); +} + +bool +llvm_jit_func_type_is_super_of(WASMModuleInstance *module_inst, + uint32 type_idx1, uint32 type_idx2) +{ + WASMModule *module = module_inst->module; + WASMType **types = module->types; + + if (type_idx1 == type_idx2) + return true; + + bh_assert(types[type_idx1]->type_flag == WASM_TYPE_FUNC); + bh_assert(types[type_idx2]->type_flag == WASM_TYPE_FUNC); + return wasm_func_type_is_super_of((WASMFuncType *)types[type_idx1], + (WASMFuncType *)types[type_idx2]); +} + +WASMRttTypeRef +llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index) +{ + WASMModule *module = module_inst->module; + WASMType *defined_type = module->types[type_index]; + WASMRttType **rtt_types = module->rtt_types; + uint32 rtt_type_count = module->type_count; + korp_mutex *rtt_type_lock = &module->rtt_type_lock; + + return wasm_rtt_type_new(defined_type, type_index, rtt_types, + rtt_type_count, rtt_type_lock); +} + +bool +llvm_array_init_with_data(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 data_seg_offset, WASMArrayObjectRef array_obj, + uint32 elem_size, uint32 array_len) +{ + WASMModule *wasm_module = module_inst->module; + WASMDataSeg *data_seg; + uint8 *array_elem_base; + uint64 total_size; + + data_seg = wasm_module->data_segments[seg_index]; + total_size = (int64)elem_size * array_len; + + if (data_seg_offset >= data_seg->data_length + || total_size > data_seg->data_length - data_seg_offset) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + array_elem_base = (uint8 *)wasm_array_obj_first_elem_addr(array_obj); + bh_memcpy_s(array_elem_base, (uint32)total_size, + data_seg->data + data_seg_offset, (uint32)total_size); + + return true; +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_LIBC_WASI != 0 && WASM_ENABLE_MULTI_MODULE != 0 +void +wasm_propagate_wasi_args(WASMModule *module) +{ + if (!module->import_count) + return; + + bh_assert(&module->import_module_list_head); + + WASMRegisteredModule *node = + bh_list_first_elem(&module->import_module_list_head); + while (node) { + WASIArguments *wasi_args_impt_mod = + &((WASMModule *)(node->module))->wasi_args; + bh_assert(wasi_args_impt_mod); + + bh_memcpy_s(wasi_args_impt_mod, sizeof(WASIArguments), + &module->wasi_args, sizeof(WASIArguments)); + node = bh_list_elem_next(node); + } +} +#endif + +bool +wasm_check_utf8_str(const uint8 *str, uint32 len) +{ + /* The valid ranges are taken from page 125, below link + https://www.unicode.org/versions/Unicode9.0.0/ch03.pdf */ + const uint8 *p = str, *p_end = str + len; + uint8 chr; + + while (p < p_end) { + chr = *p; + + if (chr == 0) { + LOG_WARNING( + "LIMITATION: a string which contains '\\00' is unsupported"); + return false; + } + else if (chr < 0x80) { + p++; + } + else if (chr >= 0xC2 && chr <= 0xDF && p + 1 < p_end) { + if (p[1] < 0x80 || p[1] > 0xBF) { + return false; + } + p += 2; + } + else if (chr >= 0xE0 && chr <= 0xEF && p + 2 < p_end) { + if (chr == 0xE0) { + if (p[1] < 0xA0 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) { + return false; + } + } + else if (chr == 0xED) { + if (p[1] < 0x80 || p[1] > 0x9F || p[2] < 0x80 || p[2] > 0xBF) { + return false; + } + } + else { /* chr >= 0xE1 && chr <= 0xEF */ + if (p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) { + return false; + } + } + p += 3; + } + else if (chr >= 0xF0 && chr <= 0xF4 && p + 3 < p_end) { + if (chr == 0xF0) { + if (p[1] < 0x90 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF + || p[3] < 0x80 || p[3] > 0xBF) { + return false; + } + } + else if (chr <= 0xF3) { /* and also chr >= 0xF1 */ + if (p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF + || p[3] < 0x80 || p[3] > 0xBF) { + return false; + } + } + else { /* chr == 0xF4 */ + if (p[1] < 0x80 || p[1] > 0x8F || p[2] < 0x80 || p[2] > 0xBF + || p[3] < 0x80 || p[3] > 0xBF) { + return false; + } + } + p += 4; + } + else { + return false; + } + } + return (p == p_end); +} + +char * +wasm_const_str_list_insert(const uint8 *str, uint32 len, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + StringNode *node, *node_next; + + if (!wasm_check_utf8_str(str, len)) { + set_error_buf(error_buf, error_buf_size, "invalid UTF-8 encoding"); + return NULL; + } + + if (len == 0) { + return ""; + } + else if (is_load_from_file_buf) { + /* As the file buffer can be referred to after loading, we use + the previous byte of leb encoded size to adjust the string: + move string 1 byte backward and then append '\0' */ + char *c_str = (char *)str - 1; + bh_memmove_s(c_str, len + 1, c_str + 1, len); + c_str[len] = '\0'; + return c_str; + } + + /* Search const str list */ + node = module->const_str_list; + while (node) { + node_next = node->next; + if (strlen(node->str) == len && !memcmp(node->str, str, len)) + break; + node = node_next; + } + + if (node) { + return node->str; + } + + if (!(node = runtime_malloc(sizeof(StringNode) + len + 1, error_buf, + error_buf_size))) { + return NULL; + } + + node->str = ((char *)node) + sizeof(StringNode); + bh_memcpy_s(node->str, len + 1, str, len); + node->str[len] = '\0'; + + if (!module->const_str_list) { + /* set as head */ + module->const_str_list = node; + node->next = NULL; + } + else { + /* insert it */ + node->next = module->const_str_list; + module->const_str_list = node; + } + + return node->str; +} + +bool +wasm_set_module_name(WASMModule *module, const char *name, char *error_buf, + uint32_t error_buf_size) +{ + if (!name) + return false; + + module->name = + wasm_const_str_list_insert((const uint8 *)name, (uint32)strlen(name), + module, false, error_buf, error_buf_size); + return module->name != NULL; +} + +const char * +wasm_get_module_name(WASMModule *module) +{ + return module->name; +} diff --git a/wasm-micro-runtime/core/iwasm/interpreter/wasm_runtime.h b/wasm-micro-runtime/core/iwasm/interpreter/wasm_runtime.h new file mode 100644 index 0000000..4249eb5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/interpreter/wasm_runtime.h @@ -0,0 +1,863 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_RUNTIME_H +#define _WASM_RUNTIME_H + +#include "wasm.h" +#include "bh_atomic.h" +#include "bh_bitmap.h" +#include "bh_hashmap.h" +#include "../common/wasm_runtime_common.h" +#include "../common/wasm_exec_env.h" + +#if WASM_ENABLE_WASI_NN != 0 +#include "../libraries/wasi-nn/src/wasi_nn_private.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define EXCEPTION_BUF_LEN 128 + +typedef struct WASMModuleInstance WASMModuleInstance; +typedef struct WASMFunctionInstance WASMFunctionInstance; +typedef struct WASMMemoryInstance WASMMemoryInstance; +typedef struct WASMTableInstance WASMTableInstance; +typedef struct WASMGlobalInstance WASMGlobalInstance; +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMTagInstance WASMTagInstance; +#endif + +/** + * When LLVM JIT, WAMR compiler or AOT is enabled, we should ensure that + * some offsets of the same field in the interpreter module instance and + * aot module instance are the same, so that the LLVM JITed/AOTed code + * can smoothly access the interpreter module instance. + * Same for the memory instance and table instance. + * We use the macro DefPointer to define some related pointer fields. + */ +#if (WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 \ + || WASM_ENABLE_AOT != 0) \ + && UINTPTR_MAX == UINT32_MAX +/* Add u32 padding if LLVM JIT, WAMR compiler or AOT is enabled on + 32-bit platform */ +#define DefPointer(type, field) \ + type field; \ + uint32 field##_padding +#else +#define DefPointer(type, field) type field +#endif + +typedef enum WASMExceptionID { + EXCE_UNREACHABLE = 0, + EXCE_OUT_OF_MEMORY, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + EXCE_INTEGER_OVERFLOW, + EXCE_INTEGER_DIVIDE_BY_ZERO, + EXCE_INVALID_CONVERSION_TO_INTEGER, + EXCE_INVALID_FUNCTION_TYPE_INDEX, + EXCE_INVALID_FUNCTION_INDEX, + EXCE_UNDEFINED_ELEMENT, + EXCE_UNINITIALIZED_ELEMENT, + EXCE_CALL_UNLINKED_IMPORT_FUNC, + EXCE_NATIVE_STACK_OVERFLOW, + EXCE_UNALIGNED_ATOMIC, + EXCE_AUX_STACK_OVERFLOW, + EXCE_AUX_STACK_UNDERFLOW, + EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, + EXCE_OPERAND_STACK_OVERFLOW, + EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC, + /* GC related exceptions */ + EXCE_NULL_FUNC_OBJ, + EXCE_NULL_STRUCT_OBJ, + EXCE_NULL_ARRAY_OBJ, + EXCE_NULL_I31_OBJ, + EXCE_NULL_REFERENCE, + EXCE_FAILED_TO_CREATE_RTT_TYPE, + EXCE_FAILED_TO_CREATE_STRUCT_OBJ, + EXCE_FAILED_TO_CREATE_ARRAY_OBJ, + EXCE_FAILED_TO_CREATE_EXTERNREF_OBJ, + EXCE_CAST_FAILURE, + EXCE_ARRAY_IDX_OOB, + EXCE_FAILED_TO_CREATE_STRING, + EXCE_FAILED_TO_CREATE_STRINGREF, + EXCE_FAILED_TO_CREATE_STRINGVIEW, + EXCE_FAILED_TO_ENCODE_STRING, + EXCE_ALREADY_THROWN, + EXCE_NUM, +} WASMExceptionID; + +typedef union { + uint64 u64; + uint32 u32[2]; +} MemBound; + +struct WASMMemoryInstance { + /* Module type */ + uint32 module_type; + + /* Whether the memory is shared */ + uint8 is_shared_memory; + + /* Whether the memory has 64-bit memory addresses */ + uint8 is_memory64; + + /* Reference count of the memory instance: + 0: non-shared memory, > 0: shared memory */ + bh_atomic_16_t ref_count; + + /* Four-byte paddings to ensure the layout of WASMMemoryInstance is the same + * in both 64-bit and 32-bit */ + uint8 __paddings[4]; + + /* Number bytes per page */ + uint32 num_bytes_per_page; + /* Current page count */ + uint32 cur_page_count; + /* Maximum page count */ + uint32 max_page_count; + /* Memory data size */ + uint64 memory_data_size; + /** + * Memory data begin address, Note: + * the app-heap might be inserted in to the linear memory, + * when memory is re-allocated, the heap data and memory data + * must be copied to new memory also + */ + DefPointer(uint8 *, memory_data); + /* Memory data end address */ + DefPointer(uint8 *, memory_data_end); + + /* Heap data base address */ + DefPointer(uint8 *, heap_data); + /* Heap data end address */ + DefPointer(uint8 *, heap_data_end); + /* The heap created */ + DefPointer(void *, heap_handle); + /* TODO: use it to replace the g_shared_memory_lock */ + DefPointer(korp_mutex *, memory_lock); + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_AOT != 0 + MemBound mem_bound_check_1byte; + MemBound mem_bound_check_2bytes; + MemBound mem_bound_check_4bytes; + MemBound mem_bound_check_8bytes; + MemBound mem_bound_check_16bytes; +#endif +}; + +/* WASMTableInstance is used to represent table instance in + * runtime, to compute the table element address with index + * we need to know the element type and the element ref type. + * For pointer type, it's 32-bit or 64-bit, align up to 8 bytes + * to simplify the computation. + * And each struct member should be 4-byte or 8-byte aligned. + */ +struct WASMTableInstance { + /* The element type */ + uint8 elem_type; + uint8 __padding__[7]; + union { +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif + uint64 __padding__; + } elem_ref_type; + /* Current size */ + uint32 cur_size; + /* Maximum size */ + uint32 max_size; + /* Table elements */ + table_elem_type_t elems[1]; +}; + +struct WASMGlobalInstance { + /* value type, VALUE_TYPE_I32/I64/F32/F64 */ + uint8 type; + /* mutable or constant */ + bool is_mutable; + /* data offset to the address of initial_value, started from the end of + * WASMMemoryInstance(start of WASMGlobalInstance)*/ + uint32 data_offset; + /* initial value */ + WASMValue initial_value; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + /* just for import, keep the reference here */ + WASMModuleInstance *import_module_inst; + WASMGlobalInstance *import_global_inst; +#endif +}; + +struct WASMFunctionInstance { + /* whether it is import function or WASM function */ + bool is_import_func; + /* parameter count */ + uint16 param_count; + /* local variable count, 0 for import function */ + uint16 local_count; + /* cell num of parameters */ + uint16 param_cell_num; + /* cell num of return type */ + uint16 ret_cell_num; + /* cell num of local variables, 0 for import function */ + uint16 local_cell_num; +#if WASM_ENABLE_FAST_INTERP != 0 + /* cell num of consts */ + uint16 const_cell_num; +#endif + uint16 *local_offsets; + /* parameter types */ + uint8 *param_types; + /* local types, NULL for import function */ + uint8 *local_types; + union { + WASMFunctionImport *func_import; + WASMFunction *func; + } u; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModuleInstance *import_module_inst; + WASMFunctionInstance *import_func_inst; +#endif +#if WASM_ENABLE_PERF_PROFILING != 0 + /* total execution time */ + uint64 total_exec_time; + /* total execution count */ + uint32 total_exec_cnt; + /* children execution time */ + uint64 children_exec_time; +#endif +}; + +#if WASM_ENABLE_TAGS != 0 +struct WASMTagInstance { + bool is_import_tag; + /* tag attribute */ + uint8 attribute; + /* tag type index */ + uint32 type; + union { + WASMTagImport *tag_import; + WASMTag *tag; + } u; + +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModuleInstance *import_module_inst; + WASMTagInstance *import_tag_inst; +#endif +}; +#endif + +#if WASM_ENABLE_EXCE_HANDLING != 0 +#define INVALID_TAGINDEX ((uint32)0xFFFFFFFF) +#define SET_INVALID_TAGINDEX(tag) (tag = INVALID_TAGINDEX) +#define IS_INVALID_TAGINDEX(tag) ((tag & INVALID_TAGINDEX) == INVALID_TAGINDEX) +#endif +typedef struct WASMExportFuncInstance { + char *name; + WASMFunctionInstance *function; +} WASMExportFuncInstance; + +typedef struct WASMExportGlobInstance { + char *name; + WASMGlobalInstance *global; +} WASMExportGlobInstance; + +typedef struct WASMExportTabInstance { + char *name; + WASMTableInstance *table; +} WASMExportTabInstance; + +typedef struct WASMExportMemInstance { + char *name; + WASMMemoryInstance *memory; +} WASMExportMemInstance; + +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMExportTagInstance { + char *name; + WASMTagInstance *tag; +} WASMExportTagInstance; +#endif + +/* wasm-c-api import function info */ +typedef struct CApiFuncImport { + /* host func pointer after linked */ + void *func_ptr_linked; + /* whether the host func has env argument */ + bool with_env_arg; + /* the env argument of the host func */ + void *env_arg; +} CApiFuncImport; + +/* The common part of WASMModuleInstanceExtra and AOTModuleInstanceExtra */ +typedef struct WASMModuleInstanceExtraCommon { +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 + void *contexts[WASM_MAX_INSTANCE_CONTEXTS]; +#endif +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + /* Disable bounds checks or not */ + bool disable_bounds_checks; +#endif +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap *data_dropped; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap *elem_dropped; +#endif + +#if WASM_ENABLE_GC != 0 + /* The gc heap memory pool */ + uint8 *gc_heap_pool; + /* The gc heap created */ + void *gc_heap_handle; +#endif +} WASMModuleInstanceExtraCommon; + +/* Extra info of WASM module instance for interpreter/jit mode */ +typedef struct WASMModuleInstanceExtra { + WASMModuleInstanceExtraCommon common; + + WASMGlobalInstance *globals; + WASMFunctionInstance *functions; + + uint32 global_count; + uint32 function_count; + + WASMFunctionInstance *start_function; + WASMFunctionInstance *malloc_function; + WASMFunctionInstance *free_function; + WASMFunctionInstance *retain_function; + + RunningMode running_mode; + +#if WASM_ENABLE_MULTI_MODULE != 0 + bh_list sub_module_inst_list_head; + bh_list *sub_module_inst_list; + /* linked table instances of import table instances */ + WASMTableInstance **table_insts_linked; +#endif + +#if WASM_ENABLE_TAGS != 0 + uint32 tag_count; + uint32 export_tag_count; + WASMTagInstance *tags; + WASMExportTagInstance *export_tags; + void **import_tag_ptrs; +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + uint32 max_aux_stack_used; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + WASMModuleInstance *next; +#endif +} WASMModuleInstanceExtra; + +struct AOTFuncPerfProfInfo; + +struct WASMModuleInstance { + /* Module instance type, for module instance loaded from + WASM bytecode binary, this field is Wasm_Module_Bytecode; + for module instance loaded from AOT file, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModuleInstance structure. */ + uint32 module_type; + + uint32 memory_count; + DefPointer(WASMMemoryInstance **, memories); + + /* global and table info */ + uint32 global_data_size; + uint32 table_count; + DefPointer(uint8 *, global_data); + /* For AOTModuleInstance, it denotes `AOTTableInstance *` */ + DefPointer(WASMTableInstance **, tables); + + /* import func ptrs + llvm jit func ptrs */ + DefPointer(void **, func_ptrs); + + /* function type indexes */ + DefPointer(uint32 *, func_type_indexes); + + uint32 export_func_count; + uint32 export_global_count; + uint32 export_memory_count; + uint32 export_table_count; + /* For AOTModuleInstance, it denotes `AOTFunctionInstance *` */ + DefPointer(WASMExportFuncInstance *, export_functions); + DefPointer(WASMExportGlobInstance *, export_globals); + DefPointer(WASMExportMemInstance *, export_memories); + DefPointer(WASMExportTabInstance *, export_tables); + + /* The exception buffer of wasm interpreter for current thread. */ + char cur_exception[EXCEPTION_BUF_LEN]; + + /* The WASM module or AOT module, for AOTModuleInstance, + it denotes `AOTModule *` */ + DefPointer(WASMModule *, module); + + DefPointer(WASMExecEnv *, exec_env_singleton); + /* Array of function pointers to import functions, + not available in AOTModuleInstance */ + DefPointer(void **, import_func_ptrs); + /* Array of function pointers to fast jit functions, + not available in AOTModuleInstance: + Only when the multi-tier JIT macros are all enabled and the running + mode of current module instance is set to Mode_Fast_JIT, runtime + will allocate new memory for it, otherwise it always points to the + module->fast_jit_func_ptrs */ + DefPointer(void **, fast_jit_func_ptrs); + /* The custom data that can be set/get by wasm_{get|set}_custom_data */ + DefPointer(void *, custom_data); + /* Stack frames, used in call stack dump and perf profiling */ + DefPointer(Vector *, frames); + /* Function performance profiling info list, only available + in AOTModuleInstance */ + DefPointer(struct AOTFuncPerfProfInfo *, func_perf_profilings); + DefPointer(CApiFuncImport *, c_api_func_imports); + /* Pointer to the exec env currently used */ + DefPointer(WASMExecEnv *, cur_exec_env); + /* WASM/AOT module extra info, for AOTModuleInstance, + it denotes `AOTModuleInstanceExtra *` */ + DefPointer(WASMModuleInstanceExtra *, e); + + /* Default WASM operand stack size */ + uint32 default_wasm_stack_size; + uint32 reserved[7]; + + /* + * +------------------------------+ <-- memories + * | WASMMemoryInstance[mem_count], mem_count is always 1 for LLVM JIT/AOT + * +------------------------------+ <-- global_data + * | global data + * +------------------------------+ <-- tables + * | WASMTableInstance[table_count] + * +------------------------------+ <-- e + * | WASMModuleInstanceExtra + * +------------------------------+ + */ + union { + uint64 _make_it_8_byte_aligned_; + WASMMemoryInstance memory_instances[1]; + uint8 bytes[1]; + } global_table_data; +}; + +struct WASMInterpFrame; +typedef struct WASMInterpFrame WASMRuntimeFrame; + +#if WASM_ENABLE_MULTI_MODULE != 0 +typedef struct WASMSubModInstNode { + bh_list_link l; + /* point to a string pool */ + const char *module_name; + WASMModuleInstance *module_inst; +} WASMSubModInstNode; +#endif + +/** + * Return the code block of a function. + * + * @param func the WASM function instance + * + * @return the code block of the function + */ +static inline uint8 * +wasm_get_func_code(WASMFunctionInstance *func) +{ +#if WASM_ENABLE_FAST_INTERP == 0 + return func->is_import_func ? NULL : func->u.func->code; +#else + return func->is_import_func ? NULL : func->u.func->code_compiled; +#endif +} + +/** + * Return the code block end of a function. + * + * @param func the WASM function instance + * + * @return the code block end of the function + */ +static inline uint8 * +wasm_get_func_code_end(WASMFunctionInstance *func) +{ +#if WASM_ENABLE_FAST_INTERP == 0 + return func->is_import_func ? NULL + : func->u.func->code + func->u.func->code_size; +#else + return func->is_import_func + ? NULL + : func->u.func->code_compiled + func->u.func->code_compiled_size; +#endif +} + +WASMModule * +wasm_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *args, char *error_buf, uint32 error_buf_size); + +WASMModule * +wasm_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size); + +void +wasm_unload(WASMModule *module); + +WASMModuleInstance * +wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size); + +void +wasm_dump_perf_profiling(const WASMModuleInstance *module_inst); + +double +wasm_summarize_wasm_execute_time(const WASMModuleInstance *inst); + +double +wasm_get_wasm_func_exec_time(const WASMModuleInstance *inst, + const char *func_name); + +void +wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst); + +bool +wasm_set_running_mode(WASMModuleInstance *module_inst, + RunningMode running_mode); + +WASMFunctionInstance * +wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name); + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASMGlobalInstance * +wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name); + +WASMMemoryInstance * +wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name); + +WASMTableInstance * +wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name); + +#if WASM_ENABLE_TAGS != 0 +WASMTagInstance * +wasm_lookup_tag(const WASMModuleInstance *module_inst, const char *name, + const char *signature); +#endif + +#endif + +bool +wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function, + unsigned argc, uint32 argv[]); + +void +wasm_set_exception(WASMModuleInstance *module, const char *exception); + +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + +const char * +wasm_get_exception(WASMModuleInstance *module); + +/** + * @brief Copy exception in buffer passed as parameter. Thread-safe version of + * `wasm_get_exception()` + * @note Buffer size must be no smaller than EXCEPTION_BUF_LEN + * @return true if exception found + */ +bool +wasm_copy_exception(WASMModuleInstance *module_inst, char *exception_buf); + +uint64 +wasm_module_malloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr); + +uint64 +wasm_module_realloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr, uint64 size, + void **p_native_addr); + +void +wasm_module_free_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr); + +uint64 +wasm_module_malloc(WASMModuleInstance *module_inst, uint64 size, + void **p_native_addr); + +uint64 +wasm_module_realloc(WASMModuleInstance *module_inst, uint64 ptr, uint64 size, + void **p_native_addr); + +void +wasm_module_free(WASMModuleInstance *module_inst, uint64 ptr); + +uint64 +wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, + uint64 size); + +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr); + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst); + +bool +wasm_enlarge_memory(WASMModuleInstance *module_inst, uint32 inc_page_count); + +bool +wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 argv[]); + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, uint32 size); + +bool +wasm_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, uint32 *size); +#endif + +void +wasm_get_module_mem_consumption(const WASMModule *module, + WASMModuleMemConsumption *mem_conspn); + +void +wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module, + WASMModuleInstMemConsumption *mem_conspn); + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +static inline bool +wasm_elem_is_active(uint32 mode) +{ + return (mode & 0x1) == 0x0; +} + +static inline bool +wasm_elem_is_passive(uint32 mode) +{ + return (mode & 0x1) == 0x1; +} + +static inline bool +wasm_elem_is_declarative(uint32 mode) +{ + return (mode & 0x3) == 0x3; +} + +bool +wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, + uint32 inc_entries, table_elem_type_t init_val); +#endif /* WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +wasm_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, uint32 error_buf_size); + +bool +wasm_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap); + +#endif + +static inline WASMTableInstance * +wasm_get_table_inst(const WASMModuleInstance *module_inst, uint32 tbl_idx) +{ + /* careful, it might be a table in another module */ + WASMTableInstance *tbl_inst = module_inst->tables[tbl_idx]; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (tbl_idx < module_inst->module->import_table_count + && module_inst->e->table_insts_linked[tbl_idx]) { + tbl_inst = module_inst->e->table_insts_linked[tbl_idx]; + } +#endif + bh_assert(tbl_inst); + return tbl_inst; +} + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +bool +wasm_interp_create_call_stack(struct WASMExecEnv *exec_env); + +/** + * @brief Dump wasm call stack or get the size + * + * @param exec_env the execution environment + * @param print whether to print to stdout or not + * @param buf buffer to store the dumped content + * @param len length of the buffer + * + * @return when print is true, return the bytes printed out to stdout; when + * print is false and buf is NULL, return the size required to store the + * callstack content; when print is false and buf is not NULL, return the size + * dumped to the buffer, 0 means error and data in buf may be invalid + */ +uint32 +wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf, + uint32 len); +#endif + +const uint8 * +wasm_loader_get_custom_section(WASMModule *module, const char *name, + uint32 *len); + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr); +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv); + +bool +fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, + struct WASMInterpFrame *prev_frame); +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv); + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv); + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, uint32 dst); + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index); +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx); + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uintptr_t val, uint32 data_offset); + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_entries, uintptr_t init_val); +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 \ + || WASM_ENABLE_AOT_STACK_FRAME != 0 +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); + +void +llvm_jit_free_frame(WASMExecEnv *exec_env); + +void +llvm_jit_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame); +#endif + +#if WASM_ENABLE_GC != 0 +void * +llvm_jit_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, + uint32 error_buf_size); + +bool +llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst, + WASMObjectRef gc_obj, uint32 type_index); + +/* Whether func type1 is one of super types of func type2 */ +bool +llvm_jit_func_type_is_super_of(WASMModuleInstance *module_inst, + uint32 type_idx1, uint32 type_idx2); + +WASMRttTypeRef +llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index); + +bool +llvm_array_init_with_data(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 data_seg_offset, WASMArrayObjectRef array_obj, + uint32 elem_size, uint32 array_len); +#endif +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_LIBC_WASI != 0 && WASM_ENABLE_MULTI_MODULE != 0 +void +wasm_propagate_wasi_args(WASMModule *module); +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +void +exception_lock(WASMModuleInstance *module_inst); +void +exception_unlock(WASMModuleInstance *module_inst); +#else +#define exception_lock(module_inst) (void)(module_inst) +#define exception_unlock(module_inst) (void)(module_inst) +#endif + +bool +wasm_check_utf8_str(const uint8 *str, uint32 len); + +char * +wasm_const_str_list_insert(const uint8 *str, uint32 len, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size); + +bool +wasm_set_module_name(WASMModule *module, const char *name, char *error_buf, + uint32_t error_buf_size); + +const char * +wasm_get_module_name(WASMModule *module); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_RUNTIME_H */ diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.c b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.c new file mode 100644 index 0000000..0ffc78a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.c @@ -0,0 +1,1433 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "debug_engine.h" +#include "gdbserver.h" +#include "handler.h" +#include "bh_platform.h" +#include "wasm_interp.h" +#include "wasm_opcode.h" +#include "wasm_runtime.h" + +static const uint8 break_instr[] = { DEBUG_OP_BREAK }; + +typedef struct WASMDebugEngine { + struct WASMDebugEngine *next; + WASMDebugControlThread *control_thread; + char ip_addr[128]; + int32 process_base_port; + bh_list debug_instance_list; + korp_mutex instance_list_lock; +} WASMDebugEngine; + +void +on_thread_stop_event(WASMDebugInstance *debug_inst, WASMExecEnv *exec_env) +{ + os_mutex_lock(&debug_inst->wait_lock); + debug_inst->stopped_thread = exec_env; + + if (debug_inst->current_state == DBG_LAUNCHING) { + /* In launching phase, send a signal so that handle_threadstop_request + * can be woken up */ + os_cond_signal(&debug_inst->wait_cond); + } + os_mutex_unlock(&debug_inst->wait_lock); +} + +void +on_thread_exit_event(WASMDebugInstance *debug_inst, WASMExecEnv *exec_env) +{ + os_mutex_lock(&debug_inst->wait_lock); + + /* DBG_LAUNCHING: exit when debugger detached, + * DBG_ERROR: exit when debugger error */ + if (debug_inst->current_state != DBG_LAUNCHING + && debug_inst->current_state != DBG_ERROR) { + /* only when exit normally the debugger thread will participate in + * teardown phase */ + debug_inst->stopped_thread = exec_env; + } + + os_mutex_unlock(&debug_inst->wait_lock); +} + +static WASMDebugEngine *g_debug_engine; + +static uint32 current_instance_id = 1; + +static uint32 +allocate_instance_id() +{ + uint32 id; + + bh_assert(g_debug_engine); + + os_mutex_lock(&g_debug_engine->instance_list_lock); + id = current_instance_id++; + os_mutex_unlock(&g_debug_engine->instance_list_lock); + + return id; +} + +static bool +is_thread_running(WASMDebugControlThread *control_thread) +{ + return control_thread->status == RUNNING; +} + +static bool +is_thread_stopped(WASMDebugControlThread *control_thread) +{ + return control_thread->status == STOPPED; +} + +static bool +is_thread_detached(WASMDebugControlThread *control_thread) +{ + return control_thread->status == DETACHED; +} + +static void * +control_thread_routine(void *arg) +{ + WASMDebugInstance *debug_inst = (WASMDebugInstance *)arg; + WASMDebugControlThread *control_thread = NULL; + + control_thread = debug_inst->control_thread; + bh_assert(control_thread); + + os_mutex_lock(&debug_inst->wait_lock); + + control_thread->status = RUNNING; + + debug_inst->id = allocate_instance_id(); + + control_thread->debug_engine = g_debug_engine; + control_thread->debug_instance = debug_inst; + bh_strcpy_s(control_thread->ip_addr, sizeof(control_thread->ip_addr), + g_debug_engine->ip_addr); + if (control_thread->port == -1) { + control_thread->port = + (g_debug_engine->process_base_port == 0) + ? 0 + : g_debug_engine->process_base_port + debug_inst->id - 1; + } + + LOG_WARNING("control thread of debug object %p start\n", debug_inst); + + control_thread->server = + wasm_create_gdbserver(control_thread->ip_addr, &control_thread->port); + + if (!control_thread->server) { + LOG_ERROR("Failed to create debug server\n"); + control_thread->port = 0; + os_cond_signal(&debug_inst->wait_cond); + os_mutex_unlock(&debug_inst->wait_lock); + return NULL; + } + + control_thread->server->thread = control_thread; + + /* + * wasm gdbserver created, the execution thread + * doesn't need to wait for the debugger connection, + * so we wake up the execution thread before listen + */ + os_cond_signal(&debug_inst->wait_cond); + os_mutex_unlock(&debug_inst->wait_lock); + + if (!wasm_gdbserver_listen(control_thread->server)) { + LOG_ERROR("Failed while listening for debugger\n"); + goto fail; + } + + /* outer infinite loop: try to connect with the debugger */ + while (true) { + /* wait lldb client to connect */ + if (!wasm_gdbserver_accept(control_thread->server)) { + LOG_ERROR("Failed while accepting debugger connection\n"); + goto fail; + } + + control_thread->status = RUNNING; + /* when reattached, send signal */ + wasm_cluster_send_signal_all(debug_inst->cluster, WAMR_SIG_SINGSTEP); + + /* inner infinite loop: keep serving until detach */ + while (true) { + os_mutex_lock(&control_thread->wait_lock); + if (is_thread_running(control_thread)) { + /* send thread stop reply */ + if (debug_inst->stopped_thread + && debug_inst->current_state == APP_RUNNING) { + uint32 status; + korp_tid tid; + + status = (uint32)debug_inst->stopped_thread->current_status + ->signal_flag; + tid = debug_inst->stopped_thread->handle; + + if (debug_inst->stopped_thread->current_status + ->running_status + == STATUS_EXIT) { + /* If the thread exits, report "W00" if it's the last + * thread in the cluster, otherwise ignore this event */ + status = 0; + + /* By design, all the other threads should have been + * stopped at this moment, so it is safe to access the + * exec_env_list.len without lock */ + if (debug_inst->cluster->exec_env_list.len != 1) { + debug_inst->stopped_thread = NULL; + /* The exiting thread may wait for the signal */ + os_cond_signal(&debug_inst->wait_cond); + os_mutex_unlock(&control_thread->wait_lock); + continue; + } + } + + wasm_debug_instance_set_cur_thread( + debug_inst, debug_inst->stopped_thread->handle); + + send_thread_stop_status(control_thread->server, status, + tid); + + debug_inst->current_state = APP_STOPPED; + debug_inst->stopped_thread = NULL; + + if (status == 0) { + /* The exiting thread may wait for the signal */ + os_cond_signal(&debug_inst->wait_cond); + } + } + + /* Processing incoming requests */ + if (!wasm_gdbserver_handle_packet(control_thread->server)) { + control_thread->status = STOPPED; + LOG_ERROR("An error occurs when handling a packet\n"); + os_mutex_unlock(&control_thread->wait_lock); + goto fail; + } + } + else if (is_thread_detached(control_thread)) { + os_mutex_unlock(&control_thread->wait_lock); + break; + } + else if (is_thread_stopped(control_thread)) { + os_mutex_unlock(&control_thread->wait_lock); + return NULL; + } + os_mutex_unlock(&control_thread->wait_lock); + } + } +fail: + wasm_debug_instance_on_failure(debug_inst); + LOG_VERBOSE("control thread of debug object [%p] stopped with failure\n", + debug_inst); + return NULL; +} + +static WASMDebugControlThread * +wasm_debug_control_thread_create(WASMDebugInstance *debug_instance, int32 port) +{ + WASMDebugControlThread *control_thread; + + if (!(control_thread = + wasm_runtime_malloc(sizeof(WASMDebugControlThread)))) { + LOG_ERROR("WASM Debug Engine error: failed to allocate memory"); + return NULL; + } + memset(control_thread, 0, sizeof(WASMDebugControlThread)); + control_thread->port = port; + + if (os_mutex_init(&control_thread->wait_lock) != 0) + goto fail; + + debug_instance->control_thread = control_thread; + + os_mutex_lock(&debug_instance->wait_lock); + + if (0 + != os_thread_create(&control_thread->tid, control_thread_routine, + debug_instance, APP_THREAD_STACK_SIZE_DEFAULT)) { + os_mutex_unlock(&debug_instance->wait_lock); + goto fail1; + } + + /* wait until the debug control thread ready */ + os_cond_wait(&debug_instance->wait_cond, &debug_instance->wait_lock); + os_mutex_unlock(&debug_instance->wait_lock); + if (!control_thread->server) { + os_thread_join(control_thread->tid, NULL); + goto fail1; + } + + os_mutex_lock(&g_debug_engine->instance_list_lock); + /* create control thread success, append debug instance to debug engine */ + bh_list_insert(&g_debug_engine->debug_instance_list, debug_instance); + os_mutex_unlock(&g_debug_engine->instance_list_lock); + + /* If we set WAMR_SIG_STOP here, the VSCode debugger adaptor will raise an + * exception in the UI. We use WAMR_SIG_SINGSTEP to avoid this exception for + * better user experience */ + wasm_cluster_send_signal_all(debug_instance->cluster, WAMR_SIG_SINGSTEP); + + return control_thread; + +fail1: + os_mutex_destroy(&control_thread->wait_lock); +fail: + wasm_runtime_free(control_thread); + return NULL; +} + +static void +wasm_debug_control_thread_destroy(WASMDebugInstance *debug_instance) +{ + WASMDebugControlThread *control_thread = debug_instance->control_thread; + + LOG_VERBOSE("stopping control thread of debug object [%p]\n", + debug_instance); + control_thread->status = STOPPED; + os_mutex_lock(&control_thread->wait_lock); + wasm_close_gdbserver(control_thread->server); + os_mutex_unlock(&control_thread->wait_lock); + os_thread_join(control_thread->tid, NULL); + wasm_runtime_free(control_thread->server); + + os_mutex_destroy(&control_thread->wait_lock); + wasm_runtime_free(control_thread); +} + +static WASMDebugEngine * +wasm_debug_engine_create() +{ + WASMDebugEngine *engine; + + if (!(engine = wasm_runtime_malloc(sizeof(WASMDebugEngine)))) { + LOG_ERROR("WASM Debug Engine error: failed to allocate memory"); + return NULL; + } + memset(engine, 0, sizeof(WASMDebugEngine)); + + if (os_mutex_init(&engine->instance_list_lock) != 0) { + wasm_runtime_free(engine); + LOG_ERROR("WASM Debug Engine error: failed to init mutex"); + return NULL; + } + + /* reset current instance id */ + current_instance_id = 1; + + bh_list_init(&engine->debug_instance_list); + return engine; +} + +void +wasm_debug_engine_destroy() +{ + if (g_debug_engine) { + wasm_debug_handler_deinit(); + os_mutex_destroy(&g_debug_engine->instance_list_lock); + wasm_runtime_free(g_debug_engine); + g_debug_engine = NULL; + } +} + +bool +wasm_debug_engine_init(char *ip_addr, int32 process_port) +{ + if (wasm_debug_handler_init() != 0) { + return false; + } + + if (g_debug_engine == NULL) { + g_debug_engine = wasm_debug_engine_create(); + } + + if (g_debug_engine) { + g_debug_engine->process_base_port = + (process_port > 0) ? process_port : 0; + if (ip_addr) + snprintf(g_debug_engine->ip_addr, sizeof(g_debug_engine->ip_addr), + "%s", ip_addr); + else + snprintf(g_debug_engine->ip_addr, sizeof(g_debug_engine->ip_addr), + "%s", "127.0.0.1"); + } + else { + wasm_debug_handler_deinit(); + } + + return g_debug_engine != NULL ? true : false; +} + +/* A debug Instance is a debug "process" in gdb remote protocol + and bound to a runtime cluster */ +WASMDebugInstance * +wasm_debug_instance_create(WASMCluster *cluster, int32 port) +{ + WASMDebugInstance *instance; + WASMExecEnv *exec_env = NULL; + wasm_module_inst_t module_inst = NULL; + + if (!g_debug_engine) { + return NULL; + } + + if (!(instance = wasm_runtime_malloc(sizeof(WASMDebugInstance)))) { + LOG_ERROR("WASM Debug Engine error: failed to allocate memory"); + return NULL; + } + memset(instance, 0, sizeof(WASMDebugInstance)); + + if (os_mutex_init(&instance->wait_lock) != 0) { + goto fail1; + } + + if (os_cond_init(&instance->wait_cond) != 0) { + goto fail2; + } + + bh_list_init(&instance->break_point_list); + bh_list_init(&instance->watch_point_list_read); + bh_list_init(&instance->watch_point_list_write); + + instance->cluster = cluster; + exec_env = bh_list_first_elem(&cluster->exec_env_list); + bh_assert(exec_env); + + instance->current_tid = exec_env->handle; + + module_inst = wasm_runtime_get_module_inst(exec_env); + bh_assert(module_inst); + + /* Allocate linear memory for evaluating expressions during debugging. If + * the allocation failed, the debugger will not be able to evaluate + * expressions */ + instance->exec_mem_info.size = DEBUG_EXECUTION_MEMORY_SIZE; + instance->exec_mem_info.start_offset = wasm_runtime_module_malloc( + module_inst, (uint64)instance->exec_mem_info.size, NULL); + if (instance->exec_mem_info.start_offset == 0) { + LOG_WARNING( + "WASM Debug Engine warning: failed to allocate linear memory for " + "execution. \n" + "Will not be able to evaluate expressions during " + "debugging"); + } + instance->exec_mem_info.current_pos = instance->exec_mem_info.start_offset; + + if (!wasm_debug_control_thread_create(instance, port)) { + LOG_ERROR("WASM Debug Engine error: failed to create control thread"); + goto fail3; + } + + wasm_cluster_set_debug_inst(cluster, instance); + + return instance; + +fail3: + os_cond_destroy(&instance->wait_cond); +fail2: + os_mutex_destroy(&instance->wait_lock); +fail1: + wasm_runtime_free(instance); + + return NULL; +} + +static void +wasm_debug_instance_destroy_breakpoints(WASMDebugInstance *instance) +{ + WASMDebugBreakPoint *breakpoint, *next_bp; + + breakpoint = bh_list_first_elem(&instance->break_point_list); + while (breakpoint) { + next_bp = bh_list_elem_next(breakpoint); + + bh_list_remove(&instance->break_point_list, breakpoint); + wasm_runtime_free(breakpoint); + + breakpoint = next_bp; + } +} + +static void +wasm_debug_instance_destroy_watchpoints(WASMDebugInstance *instance, + bh_list *watchpoints) +{ + WASMDebugWatchPoint *watchpoint, *next; + + watchpoint = bh_list_first_elem(watchpoints); + while (watchpoint) { + next = bh_list_elem_next(watchpoint); + + bh_list_remove(watchpoints, watchpoint); + wasm_runtime_free(watchpoint); + + watchpoint = next; + } +} + +void +wasm_debug_instance_destroy(WASMCluster *cluster) +{ + WASMDebugInstance *instance = NULL; + + if (!g_debug_engine) { + return; + } + + instance = cluster->debug_inst; + if (instance) { + /* destroy control thread */ + wasm_debug_control_thread_destroy(instance); + + os_mutex_lock(&g_debug_engine->instance_list_lock); + bh_list_remove(&g_debug_engine->debug_instance_list, instance); + os_mutex_unlock(&g_debug_engine->instance_list_lock); + + /* destroy all breakpoints */ + wasm_debug_instance_destroy_breakpoints(instance); + wasm_debug_instance_destroy_watchpoints( + instance, &instance->watch_point_list_read); + wasm_debug_instance_destroy_watchpoints( + instance, &instance->watch_point_list_write); + + os_mutex_destroy(&instance->wait_lock); + os_cond_destroy(&instance->wait_cond); + + wasm_runtime_free(instance); + cluster->debug_inst = NULL; + } +} + +WASMExecEnv * +wasm_debug_instance_get_current_env(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env = NULL; + + if (instance) { + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + while (exec_env) { + if (exec_env->handle == instance->current_tid) + break; + exec_env = bh_list_elem_next(exec_env); + } + } + return exec_env; +} + +#if WASM_ENABLE_LIBC_WASI != 0 +bool +wasm_debug_instance_get_current_object_name(WASMDebugInstance *instance, + char name_buffer[], uint32 len) +{ + WASMExecEnv *exec_env; + WASIArguments *wasi_args; + WASMModuleInstance *module_inst; + + if (!instance) + return false; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + wasi_args = &module_inst->module->wasi_args; + if (wasi_args && wasi_args->argc > 0) { + char *argv_name = wasi_args->argv[0]; + uint32 name_len = (uint32)strlen(argv_name); + + printf("the module name is %s\n", argv_name); + if (len - 1 >= name_len) + bh_strcpy_s(name_buffer, len, argv_name); + else + bh_strcpy_s(name_buffer, len, argv_name + (name_len + 1 - len)); + return true; + } + return false; +} +#endif + +uint64 +wasm_debug_instance_get_pid(WASMDebugInstance *instance) +{ + if (instance != NULL) { + return (uint64)instance->id; + } + return (uint64)0; +} + +korp_tid +wasm_debug_instance_get_tid(WASMDebugInstance *instance) +{ + if (instance != NULL) { + return instance->current_tid; + } + return (korp_tid)(uintptr_t)0; +} + +uint32 +wasm_debug_instance_get_tids(WASMDebugInstance *instance, korp_tid tids[], + uint32 len) +{ + WASMExecEnv *exec_env; + uint32 i = 0, threads_num = 0; + + if (!instance) + return 0; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + while (exec_env && i < len) { + /* Some threads may not be ready */ + if (exec_env->handle != 0) { + tids[i++] = exec_env->handle; + threads_num++; + } + exec_env = bh_list_elem_next(exec_env); + } + LOG_VERBOSE("find %d tids\n", threads_num); + return threads_num; +} + +uint32 +wasm_debug_instance_get_thread_status(WASMDebugInstance *instance, korp_tid tid) +{ + WASMExecEnv *exec_env = NULL; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + while (exec_env) { + if (exec_env->handle == tid) { + return (uint32)exec_env->current_status->signal_flag; + } + exec_env = bh_list_elem_next(exec_env); + } + + return 0; +} + +void +wasm_debug_instance_set_cur_thread(WASMDebugInstance *instance, korp_tid tid) +{ + instance->current_tid = tid; +} + +uint64 +wasm_debug_instance_get_pc(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env; + + if (!instance) + return 0; + + exec_env = wasm_debug_instance_get_current_env(instance); + if ((exec_env != NULL) && (exec_env->cur_frame != NULL) + && (exec_env->cur_frame->ip != NULL)) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + return WASM_ADDR( + WasmObj, instance->id, + (exec_env->cur_frame->ip - module_inst->module->load_addr)); + } + return 0; +} + +uint64 +wasm_debug_instance_get_load_addr(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env; + + if (!instance) + return WASM_ADDR(WasmInvalid, 0, 0); + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (exec_env) { + return WASM_ADDR(WasmObj, instance->id, 0); + } + + return WASM_ADDR(WasmInvalid, 0, 0); +} + +WASMDebugMemoryInfo * +wasm_debug_instance_get_memregion(WASMDebugInstance *instance, uint64 addr) +{ + WASMDebugMemoryInfo *mem_info; + WASMExecEnv *exec_env; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory; + uint32 num_bytes_per_page; + uint32 linear_mem_size = 0; + + if (!instance) + return NULL; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return NULL; + + if (!(mem_info = wasm_runtime_malloc(sizeof(WASMDebugMemoryInfo)))) { + LOG_ERROR("WASM Debug Engine error: failed to allocate memory"); + return NULL; + } + memset(mem_info, 0, sizeof(WASMDebugMemoryInfo)); + mem_info->start = WASM_ADDR(WasmInvalid, 0, 0); + mem_info->size = 0; + mem_info->name[0] = '\0'; + mem_info->permisson[0] = '\0'; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + + switch (WASM_ADDR_TYPE(addr)) { + case WasmObj: + if (WASM_ADDR_OFFSET(addr) < module_inst->module->load_size) { + mem_info->start = WASM_ADDR(WasmObj, instance->id, 0); + mem_info->size = module_inst->module->load_size; + snprintf(mem_info->name, sizeof(mem_info->name), "%s", + "module"); + snprintf(mem_info->permisson, sizeof(mem_info->permisson), "%s", + "rx"); + } + break; + case WasmMemory: + { + memory = wasm_get_default_memory(module_inst); + + if (memory) { + num_bytes_per_page = memory->num_bytes_per_page; + linear_mem_size = num_bytes_per_page * memory->cur_page_count; + } + if (WASM_ADDR_OFFSET(addr) < linear_mem_size) { + mem_info->start = WASM_ADDR(WasmMemory, instance->id, 0); + mem_info->size = linear_mem_size; + snprintf(mem_info->name, sizeof(mem_info->name), "%s", + "memory"); + snprintf(mem_info->permisson, sizeof(mem_info->permisson), "%s", + "rw"); + } + break; + } + default: + mem_info->start = WASM_ADDR(WasmInvalid, 0, 0); + mem_info->size = 0; + } + return mem_info; +} + +void +wasm_debug_instance_destroy_memregion(WASMDebugInstance *instance, + WASMDebugMemoryInfo *mem_info) +{ + wasm_runtime_free(mem_info); +} + +bool +wasm_debug_instance_get_obj_mem(WASMDebugInstance *instance, uint64 offset, + char *buf, uint64 *size) +{ + WASMExecEnv *exec_env; + WASMModuleInstance *module_inst; + WASMDebugBreakPoint *breakpoint; + WASMFastOPCodeNode *fast_opcode; + + if (!instance) + return false; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + + if (offset + *size > module_inst->module->load_size) { + LOG_VERBOSE("wasm_debug_instance_get_data_mem size over flow!\n"); + *size = module_inst->module->load_size >= offset + ? module_inst->module->load_size - offset + : 0; + } + + bh_memcpy_s(buf, (uint32)*size, module_inst->module->load_addr + offset, + (uint32)*size); + + breakpoint = bh_list_first_elem(&instance->break_point_list); + while (breakpoint) { + if (offset <= breakpoint->addr && breakpoint->addr < offset + *size) { + bh_memcpy_s(buf + (breakpoint->addr - offset), sizeof(break_instr), + &breakpoint->orignal_data, sizeof(break_instr)); + } + breakpoint = bh_list_elem_next(breakpoint); + } + + fast_opcode = bh_list_first_elem(&module_inst->module->fast_opcode_list); + while (fast_opcode) { + if (offset <= fast_opcode->offset + && fast_opcode->offset < offset + *size) { + *(uint8 *)(buf + (fast_opcode->offset - offset)) = + fast_opcode->orig_op; + } + fast_opcode = bh_list_elem_next(fast_opcode); + } + + return true; +} + +bool +wasm_debug_instance_get_linear_mem(WASMDebugInstance *instance, uint64 offset, + char *buf, uint64 *size) +{ + WASMExecEnv *exec_env; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory; + uint32 num_bytes_per_page; + uint32 linear_mem_size; + + if (!instance) + return false; + + exec_env = wasm_debug_instance_get_current_env(instance); + if (!exec_env) + return false; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + memory = wasm_get_default_memory(module_inst); + if (memory) { + num_bytes_per_page = memory->num_bytes_per_page; + linear_mem_size = num_bytes_per_page * memory->cur_page_count; + if (offset + *size > linear_mem_size) { + LOG_VERBOSE("wasm_debug_instance_get_linear_mem size over flow!\n"); + *size = linear_mem_size >= offset ? linear_mem_size - offset : 0; + } + bh_memcpy_s(buf, (uint32)*size, memory->memory_data + offset, + (uint32)*size); + return true; + } + return false; +} + +bool +wasm_debug_instance_set_linear_mem(WASMDebugInstance *instance, uint64 offset, + char *buf, uint64 *size) +{ + WASMExecEnv *exec_env; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory; + uint32 num_bytes_per_page; + uint32 linear_mem_size; + + if (!instance) + return false; + + exec_env = wasm_debug_instance_get_current_env(instance); + if (!exec_env) + return false; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + memory = wasm_get_default_memory(module_inst); + if (memory) { + num_bytes_per_page = memory->num_bytes_per_page; + linear_mem_size = num_bytes_per_page * memory->cur_page_count; + if (offset + *size > linear_mem_size) { + LOG_VERBOSE("wasm_debug_instance_get_linear_mem size over flow!\n"); + *size = linear_mem_size >= offset ? linear_mem_size - offset : 0; + } + bh_memcpy_s(memory->memory_data + offset, (uint32)*size, buf, + (uint32)*size); + return true; + } + return false; +} + +bool +wasm_debug_instance_get_mem(WASMDebugInstance *instance, uint64 addr, char *buf, + uint64 *size) +{ + switch (WASM_ADDR_TYPE(addr)) { + case WasmMemory: + return wasm_debug_instance_get_linear_mem( + instance, WASM_ADDR_OFFSET(addr), buf, size); + break; + case WasmObj: + return wasm_debug_instance_get_obj_mem( + instance, WASM_ADDR_OFFSET(addr), buf, size); + break; + default: + return false; + } +} + +bool +wasm_debug_instance_set_mem(WASMDebugInstance *instance, uint64 addr, char *buf, + uint64 *size) +{ + switch (WASM_ADDR_TYPE(addr)) { + case WasmMemory: + return wasm_debug_instance_set_linear_mem( + instance, WASM_ADDR_OFFSET(addr), buf, size); + break; + case WasmObj: + default: + return false; + } +} + +WASMDebugInstance * +wasm_exec_env_get_instance(WASMExecEnv *exec_env) +{ + WASMDebugInstance *instance = NULL; + + if (!g_debug_engine) { + return NULL; + } + + os_mutex_lock(&g_debug_engine->instance_list_lock); + instance = bh_list_first_elem(&g_debug_engine->debug_instance_list); + while (instance) { + if (instance->cluster == exec_env->cluster) + break; + instance = bh_list_elem_next(instance); + } + + os_mutex_unlock(&g_debug_engine->instance_list_lock); + return instance; +} + +uint32 +wasm_debug_instance_get_call_stack_pcs(WASMDebugInstance *instance, + korp_tid tid, uint64 buf[], uint64 size) +{ + WASMExecEnv *exec_env; + struct WASMInterpFrame *frame; + uint32 i = 0; + + if (!instance) + return 0; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + while (exec_env) { + if (exec_env->handle == tid) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + frame = exec_env->cur_frame; + while (frame && i < size) { + if (frame->ip != NULL) { + buf[i++] = + WASM_ADDR(WasmObj, instance->id, + (frame->ip - module_inst->module->load_addr)); + } + frame = frame->prev_frame; + } + return i; + } + exec_env = bh_list_elem_next(exec_env); + } + return 0; +} + +bool +wasm_debug_instance_add_breakpoint(WASMDebugInstance *instance, uint64 addr, + uint64 length) +{ + WASMExecEnv *exec_env; + WASMModuleInstance *module_inst; + uint64 offset; + + if (!instance) + return false; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + if (WASM_ADDR_TYPE(addr) != WasmObj) + return false; + + offset = WASM_ADDR_OFFSET(addr); + + if (length >= sizeof(break_instr)) { + if (offset + sizeof(break_instr) <= module_inst->module->load_size) { + WASMDebugBreakPoint *breakpoint; + if (!(breakpoint = + wasm_runtime_malloc(sizeof(WASMDebugBreakPoint)))) { + LOG_ERROR("WASM Debug Engine error: failed to allocate memory"); + return false; + } + memset(breakpoint, 0, sizeof(WASMDebugBreakPoint)); + breakpoint->addr = offset; + /* TODO: how to if more than one breakpoints are set + at the same addr? */ + bh_memcpy_s(&breakpoint->orignal_data, (uint32)sizeof(break_instr), + module_inst->module->load_addr + offset, + (uint32)sizeof(break_instr)); + + bh_memcpy_s(module_inst->module->load_addr + offset, + (uint32)sizeof(break_instr), break_instr, + (uint32)sizeof(break_instr)); + + bh_list_insert(&instance->break_point_list, breakpoint); + return true; + } + } + return false; +} + +bool +wasm_debug_instance_remove_breakpoint(WASMDebugInstance *instance, uint64 addr, + uint64 length) +{ + WASMExecEnv *exec_env; + WASMModuleInstance *module_inst; + uint64 offset; + + if (!instance) + return false; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + + if (WASM_ADDR_TYPE(addr) != WasmObj) + return false; + offset = WASM_ADDR_OFFSET(addr); + + if (length >= sizeof(break_instr)) { + if (offset + sizeof(break_instr) <= module_inst->module->load_size) { + WASMDebugBreakPoint *breakpoint = + bh_list_first_elem(&instance->break_point_list); + while (breakpoint) { + WASMDebugBreakPoint *next_break = bh_list_elem_next(breakpoint); + if (breakpoint->addr == offset) { + /* TODO: how to if more than one breakpoints are set + at the same addr? */ + bh_memcpy_s(module_inst->module->load_addr + offset, + (uint32)sizeof(break_instr), + &breakpoint->orignal_data, + (uint32)sizeof(break_instr)); + bh_list_remove(&instance->break_point_list, breakpoint); + wasm_runtime_free(breakpoint); + } + breakpoint = next_break; + } + } + } + return true; +} + +static bool +add_watchpoint(bh_list *list, uint64 addr, uint64 length) +{ + WASMDebugWatchPoint *watchpoint; + if (!(watchpoint = wasm_runtime_malloc(sizeof(WASMDebugWatchPoint)))) { + LOG_ERROR("WASM Debug Engine error: failed to allocate memory for " + "watchpoint"); + return false; + } + memset(watchpoint, 0, sizeof(WASMDebugWatchPoint)); + watchpoint->addr = addr; + watchpoint->length = length; + bh_list_insert(list, watchpoint); + return true; +} + +static bool +remove_watchpoint(bh_list *list, uint64 addr, uint64 length) +{ + WASMDebugWatchPoint *watchpoint = bh_list_first_elem(list); + while (watchpoint) { + WASMDebugWatchPoint *next = bh_list_elem_next(watchpoint); + if (watchpoint->addr == addr && watchpoint->length == length) { + bh_list_remove(list, watchpoint); + wasm_runtime_free(watchpoint); + } + watchpoint = next; + } + return true; +} + +bool +wasm_debug_instance_watchpoint_write_add(WASMDebugInstance *instance, + uint64 addr, uint64 length) +{ + return add_watchpoint(&instance->watch_point_list_write, addr, length); +} + +bool +wasm_debug_instance_watchpoint_write_remove(WASMDebugInstance *instance, + uint64 addr, uint64 length) +{ + return remove_watchpoint(&instance->watch_point_list_write, addr, length); +} + +bool +wasm_debug_instance_watchpoint_read_add(WASMDebugInstance *instance, + uint64 addr, uint64 length) +{ + return add_watchpoint(&instance->watch_point_list_read, addr, length); +} + +bool +wasm_debug_instance_watchpoint_read_remove(WASMDebugInstance *instance, + uint64 addr, uint64 length) +{ + return remove_watchpoint(&instance->watch_point_list_read, addr, length); +} + +bool +wasm_debug_instance_on_failure(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env; + + if (!instance) + return false; + + os_mutex_lock(&instance->wait_lock); + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) { + os_mutex_unlock(&instance->wait_lock); + return false; + } + + if (instance->stopped_thread == NULL + && instance->current_state == DBG_LAUNCHING) { + /* if fail in start stage: may need wait for main thread to notify it */ + os_cond_wait(&instance->wait_cond, &instance->wait_lock); + } + instance->current_state = DBG_ERROR; + instance->stopped_thread = NULL; + + /* terminate the wasm execution thread */ + while (exec_env) { + /* Resume all threads so they can receive the TERM signal */ + os_mutex_lock(&exec_env->wait_lock); + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TERM); + exec_env->current_status->running_status = STATUS_RUNNING; + os_cond_signal(&exec_env->wait_cond); + os_mutex_unlock(&exec_env->wait_lock); + exec_env = bh_list_elem_next(exec_env); + } + os_mutex_unlock(&instance->wait_lock); + + return true; +} + +bool +wasm_debug_instance_continue(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env; + + if (!instance) + return false; + + if (instance->current_state == APP_RUNNING) { + LOG_VERBOSE("Already in running state, ignore continue request"); + return false; + } + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + while (exec_env) { + wasm_cluster_thread_continue(exec_env); + exec_env = bh_list_elem_next(exec_env); + } + + instance->current_state = APP_RUNNING; + + return true; +} + +bool +wasm_debug_instance_interrupt_all_threads(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env; + + if (!instance) + return false; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + while (exec_env) { + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TRAP); + exec_env = bh_list_elem_next(exec_env); + } + return true; +} + +bool +wasm_debug_instance_detach(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env; + + if (!instance) + return false; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + wasm_gdbserver_detach(instance->control_thread->server); + + while (exec_env) { + if (instance->current_state == APP_STOPPED) { + /* Resume all threads since remote debugger detached*/ + wasm_cluster_thread_continue(exec_env); + } + exec_env = bh_list_elem_next(exec_env); + } + + /* relaunch, accept new debug connection */ + instance->current_state = DBG_LAUNCHING; + instance->control_thread->status = DETACHED; + instance->stopped_thread = NULL; + + return true; +} + +bool +wasm_debug_instance_kill(WASMDebugInstance *instance) +{ + WASMExecEnv *exec_env; + + if (!instance) + return false; + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + while (exec_env) { + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TERM); + if (instance->current_state == APP_STOPPED) { + /* Resume all threads so they can receive the TERM signal */ + os_mutex_lock(&exec_env->wait_lock); + exec_env->current_status->running_status = STATUS_RUNNING; + os_cond_signal(&exec_env->wait_cond); + os_mutex_unlock(&exec_env->wait_lock); + } + exec_env = bh_list_elem_next(exec_env); + } + + instance->current_state = APP_RUNNING; + return true; +} + +bool +wasm_debug_instance_singlestep(WASMDebugInstance *instance, korp_tid tid) +{ + WASMExecEnv *exec_env; + + if (!instance) + return false; + + if (instance->current_state == APP_RUNNING) { + LOG_VERBOSE("Already in running state, ignore step request"); + return false; + } + + exec_env = bh_list_first_elem(&instance->cluster->exec_env_list); + if (!exec_env) + return false; + + while (exec_env) { + if (exec_env->handle == tid || tid == (korp_tid)(uintptr_t)~0LL) { + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_SINGSTEP); + wasm_cluster_thread_step(exec_env); + } + exec_env = bh_list_elem_next(exec_env); + } + + instance->current_state = APP_RUNNING; + + return true; +} + +bool +wasm_debug_instance_get_local(WASMDebugInstance *instance, int32 frame_index, + int32 local_index, char buf[], int32 *size) +{ + WASMExecEnv *exec_env; + struct WASMInterpFrame *frame; + WASMFunctionInstance *cur_func; + uint8 local_type = 0xFF; + uint32 local_offset; + int32 param_count; + int32 fi = 0; + + if (!instance) + return false; + + exec_env = wasm_debug_instance_get_current_env(instance); + if (!exec_env) + return false; + + frame = exec_env->cur_frame; + while (frame && fi++ != frame_index) { + frame = frame->prev_frame; + } + + if (!frame) + return false; + cur_func = frame->function; + if (!cur_func) + return false; + + param_count = cur_func->param_count; + + if (local_index >= param_count + cur_func->local_count) + return false; + + local_offset = cur_func->local_offsets[local_index]; + if (local_index < param_count) + local_type = cur_func->param_types[local_index]; + else if (local_index < cur_func->local_count + param_count) + local_type = cur_func->local_types[local_index - param_count]; + + switch (local_type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + *size = 4; + bh_memcpy_s(buf, 4, (char *)(frame->lp + local_offset), 4); + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + *size = 8; + bh_memcpy_s(buf, 8, (char *)(frame->lp + local_offset), 8); + break; + default: + *size = 0; + break; + } + return true; +} + +bool +wasm_debug_instance_get_global(WASMDebugInstance *instance, int32 frame_index, + int32 global_index, char buf[], int32 *size) +{ + WASMExecEnv *exec_env; + struct WASMInterpFrame *frame; + WASMModuleInstance *module_inst; + WASMGlobalInstance *globals, *global; + uint8 *global_addr; + uint8 global_type = 0xFF; + uint8 *global_data; + int32 fi = 0; + + if (!instance) + return false; + + exec_env = wasm_debug_instance_get_current_env(instance); + if (!exec_env) + return false; + + frame = exec_env->cur_frame; + while (frame && fi++ != frame_index) { + frame = frame->prev_frame; + } + + if (!frame) + return false; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + global_data = module_inst->global_data; + globals = module_inst->e->globals; + + if ((global_index < 0) + || ((uint32)global_index >= module_inst->e->global_count)) { + return false; + } + global = globals + global_index; + +#if WASM_ENABLE_MULTI_MODULE == 0 + global_addr = global_data + global->data_offset; +#else + global_addr = global->import_global_inst + ? global->import_module_inst->global_data + + global->import_global_inst->data_offset + : global_data + global->data_offset; +#endif + global_type = global->type; + + switch (global_type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + *size = 4; + bh_memcpy_s(buf, 4, (char *)(global_addr), 4); + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + *size = 8; + bh_memcpy_s(buf, 8, (char *)(global_addr), 8); + break; + default: + *size = 0; + break; + } + return true; +} + +uint64 +wasm_debug_instance_mmap(WASMDebugInstance *instance, uint32 size, + int32 map_prot) +{ + WASMExecEnv *exec_env; + uint32 offset = 0; + (void)map_prot; + + if (!instance) + return 0; + + exec_env = wasm_debug_instance_get_current_env(instance); + if (!exec_env) + return 0; + + if (instance->exec_mem_info.start_offset == 0) { + return 0; + } + + if (instance->exec_mem_info.current_pos + - instance->exec_mem_info.start_offset + size + <= (uint64)instance->exec_mem_info.size) { + offset = instance->exec_mem_info.current_pos; + instance->exec_mem_info.current_pos += size; + } + + if (offset == 0) { + LOG_WARNING("the memory may be not enough for debug, try use larger " + "--heap-size"); + return 0; + } + + return WASM_ADDR(WasmMemory, 0, offset); +} + +bool +wasm_debug_instance_ummap(WASMDebugInstance *instance, uint64 addr) +{ + WASMExecEnv *exec_env; + + if (!instance) + return false; + + exec_env = wasm_debug_instance_get_current_env(instance); + if (!exec_env) + return false; + + if (instance->exec_mem_info.start_offset == 0) { + return false; + } + + (void)addr; + + /* Currently we don't support to free the execution memory, simply return + * true here */ + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.cmake b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.cmake new file mode 100644 index 0000000..914ddd6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.cmake @@ -0,0 +1,12 @@ +# Copyright (C) 2021 Ant Group. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (DEBUG_ENGINE_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_DEBUG_INTERP=1) + +include_directories(${DEBUG_ENGINE_DIR}) + +file (GLOB source_all ${DEBUG_ENGINE_DIR}/*.c) + +set (DEBUG_ENGINE_SOURCE ${source_all}) diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.h b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.h new file mode 100644 index 0000000..6873821 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/debug_engine.h @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _DEBUG_ENGINE_H +#define _DEBUG_ENGINE_H + +#include "bh_list.h" +#include "gdbserver.h" +#include "thread_manager.h" + +typedef enum WASMDebugControlThreadStatus { + RUNNING, + DETACHED, + STOPPED, +} WASMDebugControlThreadStatus; + +struct WASMDebugEngine; +struct WASMDebugInstance; + +typedef struct WASMDebugControlThread { + WASMGDBServer *server; + korp_tid tid; + korp_mutex wait_lock; + char ip_addr[128]; + int port; + WASMDebugControlThreadStatus status; + struct WASMDebugEngine *debug_engine; + struct WASMDebugInstance *debug_instance; +} WASMDebugControlThread; + +typedef struct WASMDebugBreakPoint { + struct WASMDebugBreakPoint *next; + uint64 addr; + uint64 orignal_data; +} WASMDebugBreakPoint; + +typedef struct WASMDebugWatchPoint { + bh_list_link next; + uint64 addr; + uint64 length; +} WASMDebugWatchPoint; + +typedef enum debug_state_t { + /* Debugger state conversion sequence: + * DBG_LAUNCHING ---> APP_STOPPED <---> APP_RUNNING + */ + DBG_LAUNCHING, + APP_RUNNING, + APP_STOPPED, + DBG_ERROR +} debug_state_t; + +typedef struct WASMDebugExecutionMemory { + uint64 start_offset; + uint64 current_pos; + uint32 size; +} WASMDebugExecutionMemory; + +struct WASMDebugInstance { + struct WASMDebugInstance *next; + WASMDebugControlThread *control_thread; + bh_list break_point_list; + bh_list watch_point_list_read; + bh_list watch_point_list_write; + WASMCluster *cluster; + uint32 id; + korp_tid current_tid; + korp_mutex wait_lock; + korp_cond wait_cond; + /* Last stopped thread, it should be set to NULL when sending + * out the thread stop reply */ + WASMExecEnv *volatile stopped_thread; + /* Currently status of the debug instance, it will be set to + * RUNNING when receiving STEP/CONTINUE commands, and set to + * STOPPED when any thread stopped */ + volatile debug_state_t current_state; + /* Execution memory info. During debugging, the debug client may request to + * malloc a memory space to evaluate user expressions. We preserve a buffer + * during creating debug instance, and use a simple bump pointer allocator + * to serve lldb's memory request */ + WASMDebugExecutionMemory exec_mem_info; +}; + +typedef enum WASMDebugEventKind { + BREAK_POINT_ADD, + BREAK_POINT_REMOVE +} WASMDebugEventKind; + +typedef struct WASMDebugEvent { + WASMDebugEventKind kind; + unsigned char metadata[0]; +} WASMDebugEvent; + +typedef struct WASMDebugMemoryInfo { + uint64 start; + uint64 size; + char name[128]; + char permisson[4]; +} WASMDebugMemoryInfo; + +typedef enum WasmAddressType { + WasmMemory = 0x00, + WasmObj = 0x01, + WasmInvalid = 0x03 +} WasmAddressType; + +#define WASM_ADDR(type, id, offset) \ + (((uint64)type << 62) | ((uint64)0 << 32) | ((uint64)offset << 0)) + +#define WASM_ADDR_TYPE(addr) (((addr)&0xC000000000000000) >> 62) +#define WASM_ADDR_OFFSET(addr) (((addr)&0x00000000FFFFFFFF)) + +#define INVALIED_ADDR (0xFFFFFFFFFFFFFFFF) + +void +on_thread_stop_event(WASMDebugInstance *debug_inst, WASMExecEnv *exec_env); + +void +on_thread_exit_event(WASMDebugInstance *debug_inst, WASMExecEnv *exec_env); + +WASMDebugInstance * +wasm_debug_instance_create(WASMCluster *cluster, int32 port); + +void +wasm_debug_instance_destroy(WASMCluster *cluster); + +WASMDebugInstance * +wasm_exec_env_get_instance(WASMExecEnv *exec_env); + +bool +wasm_debug_engine_init(char *ip_addr, int32 process_port); + +void +wasm_debug_engine_destroy(); + +WASMExecEnv * +wasm_debug_instance_get_current_env(WASMDebugInstance *instance); + +uint64 +wasm_debug_instance_get_pid(WASMDebugInstance *instance); + +korp_tid +wasm_debug_instance_get_tid(WASMDebugInstance *instance); + +uint32 +wasm_debug_instance_get_tids(WASMDebugInstance *instance, korp_tid tids[], + uint32 len); + +void +wasm_debug_instance_set_cur_thread(WASMDebugInstance *instance, korp_tid tid); + +uint64 +wasm_debug_instance_get_pc(WASMDebugInstance *instance); + +uint64 +wasm_debug_instance_get_load_addr(WASMDebugInstance *instance); + +WASMDebugMemoryInfo * +wasm_debug_instance_get_memregion(WASMDebugInstance *instance, uint64 addr); + +void +wasm_debug_instance_destroy_memregion(WASMDebugInstance *instance, + WASMDebugMemoryInfo *mem_info); + +bool +wasm_debug_instance_get_obj_mem(WASMDebugInstance *instance, uint64 addr, + char *buf, uint64 *size); + +bool +wasm_debug_instance_get_linear_mem(WASMDebugInstance *instance, uint64 addr, + char *buf, uint64 *size); + +bool +wasm_debug_instance_get_mem(WASMDebugInstance *instance, uint64 addr, char *buf, + uint64 *size); + +bool +wasm_debug_instance_set_mem(WASMDebugInstance *instance, uint64 addr, char *buf, + uint64 *size); + +uint32 +wasm_debug_instance_get_call_stack_pcs(WASMDebugInstance *instance, + korp_tid tid, uint64 buf[], uint64 size); + +bool +wasm_debug_instance_add_breakpoint(WASMDebugInstance *instance, uint64 addr, + uint64 length); + +bool +wasm_debug_instance_remove_breakpoint(WASMDebugInstance *instance, uint64 addr, + uint64 length); + +bool +wasm_debug_instance_watchpoint_write_add(WASMDebugInstance *instance, + uint64 addr, uint64 length); + +bool +wasm_debug_instance_watchpoint_write_remove(WASMDebugInstance *instance, + uint64 addr, uint64 length); + +bool +wasm_debug_instance_watchpoint_read_add(WASMDebugInstance *instance, + uint64 addr, uint64 length); + +bool +wasm_debug_instance_watchpoint_read_remove(WASMDebugInstance *instance, + uint64 addr, uint64 length); + +bool +wasm_debug_instance_on_failure(WASMDebugInstance *instance); + +bool +wasm_debug_instance_interrupt_all_threads(WASMDebugInstance *instance); + +bool +wasm_debug_instance_continue(WASMDebugInstance *instance); + +bool +wasm_debug_instance_detach(WASMDebugInstance *instance); + +bool +wasm_debug_instance_kill(WASMDebugInstance *instance); + +uint32 +wasm_debug_instance_get_thread_status(WASMDebugInstance *instance, + korp_tid tid); + +bool +wasm_debug_instance_singlestep(WASMDebugInstance *instance, korp_tid tid); + +bool +wasm_debug_instance_get_local(WASMDebugInstance *instance, int32 frame_index, + int32 local_index, char buf[], int32 *size); + +bool +wasm_debug_instance_get_global(WASMDebugInstance *instance, int32 frame_index, + int32 global_index, char buf[], int32 *size); + +#if WASM_ENABLE_LIBC_WASI != 0 +bool +wasm_debug_instance_get_current_object_name(WASMDebugInstance *instance, + char name_buffer[], uint32 len); +#endif + +uint64 +wasm_debug_instance_mmap(WASMDebugInstance *instance, uint32 size, + int32 map_prot); + +bool +wasm_debug_instance_ummap(WASMDebugInstance *instance, uint64 addr); +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/gdbserver.c b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/gdbserver.c new file mode 100644 index 0000000..fbd876a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/gdbserver.c @@ -0,0 +1,330 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" +#include "gdbserver.h" +#include "handler.h" +#include "packets.h" +#include "utils.h" + +typedef void (*PacketHandler)(WASMGDBServer *server, char *payload); + +struct packet_handler_elem { + char request; + PacketHandler handler; +}; + +#define DEL_HANDLER(r, h) [r] = { .request = r, .handler = h } + +static const struct packet_handler_elem packet_handler_table[255] = { + DEL_HANDLER('Q', handle_general_set), + DEL_HANDLER('q', handle_general_query), + DEL_HANDLER('v', handle_v_packet), + DEL_HANDLER('?', handle_threadstop_request), + DEL_HANDLER('H', handle_set_current_thread), + DEL_HANDLER('p', handle_get_register), + DEL_HANDLER('j', handle_get_json_request), + DEL_HANDLER('m', handle_get_read_memory), + DEL_HANDLER('M', handle_get_write_memory), + DEL_HANDLER('x', handle_get_read_binary_memory), + DEL_HANDLER('Z', handle_add_break), + DEL_HANDLER('z', handle_remove_break), + DEL_HANDLER('c', handle_continue_request), + DEL_HANDLER('k', handle_kill_request), + DEL_HANDLER('_', handle____request), + DEL_HANDLER('D', handle_detach_request), +}; + +WASMGDBServer * +wasm_create_gdbserver(const char *host, int32 *port) +{ + bh_socket_t listen_fd = (bh_socket_t)-1; + WASMGDBServer *server; + + bh_assert(port); + + if (!(server = wasm_runtime_malloc(sizeof(WASMGDBServer)))) { + LOG_ERROR("wasm gdb server error: failed to allocate memory"); + return NULL; + } + + memset(server, 0, sizeof(WASMGDBServer)); + + if (!(server->receive_ctx = + wasm_runtime_malloc(sizeof(rsp_recv_context_t)))) { + LOG_ERROR("wasm gdb server error: failed to allocate memory"); + goto fail; + } + + memset(server->receive_ctx, 0, sizeof(rsp_recv_context_t)); + + if (0 != os_socket_create(&listen_fd, true, true)) { + LOG_ERROR("wasm gdb server error: create socket failed"); + goto fail; + } + + if (0 != os_socket_bind(listen_fd, host, port)) { + LOG_ERROR("wasm gdb server error: socket bind failed"); + goto fail; + } + + LOG_WARNING("Debug server listening on %s:%" PRIu32 "\n", host, *port); + server->listen_fd = listen_fd; + + return server; + +fail: + if (listen_fd >= 0) { + os_socket_shutdown(listen_fd); + os_socket_close(listen_fd); + } + if (server->receive_ctx) + wasm_runtime_free(server->receive_ctx); + if (server) + wasm_runtime_free(server); + return NULL; +} + +bool +wasm_gdbserver_listen(WASMGDBServer *server) +{ + int32 ret; + + ret = os_socket_listen(server->listen_fd, 1); + if (ret != 0) { + LOG_ERROR("wasm gdb server error: socket listen failed"); + goto fail; + } + + LOG_VERBOSE("listen for gdb client"); + return true; + +fail: + os_socket_shutdown(server->listen_fd); + os_socket_close(server->listen_fd); + return false; +} + +bool +wasm_gdbserver_accept(WASMGDBServer *server) +{ + + bh_socket_t sockt_fd = (bh_socket_t)-1; + + LOG_VERBOSE("waiting for gdb client to connect..."); + + os_socket_accept(server->listen_fd, &sockt_fd, NULL, NULL); + if (sockt_fd < 0) { + LOG_ERROR("wasm gdb server error: socket accept failed"); + goto fail; + } + + LOG_VERBOSE("accept gdb client"); + server->socket_fd = sockt_fd; + server->noack = false; + return true; + +fail: + os_socket_shutdown(server->listen_fd); + os_socket_close(server->listen_fd); + return false; +} + +void +wasm_gdbserver_detach(WASMGDBServer *server) +{ + if (server->socket_fd > 0) { + os_socket_shutdown(server->socket_fd); + os_socket_close(server->socket_fd); + } +} + +void +wasm_close_gdbserver(WASMGDBServer *server) +{ + if (server->receive_ctx) { + wasm_runtime_free(server->receive_ctx); + } + if (server->socket_fd > 0) { + os_socket_shutdown(server->socket_fd); + os_socket_close(server->socket_fd); + } + if (server->listen_fd > 0) { + os_socket_shutdown(server->listen_fd); + os_socket_close(server->listen_fd); + } +} + +static inline void +handle_packet(WASMGDBServer *server, char request, char *payload) +{ + if (packet_handler_table[(int)request].handler != NULL) + packet_handler_table[(int)request].handler(server, payload); +} + +static void +process_packet(WASMGDBServer *server) +{ + uint8 *inbuf = (uint8 *)server->receive_ctx->receive_buffer; + char request; + char *payload = NULL; + + request = inbuf[0]; + + if (request == '\0') { + LOG_VERBOSE("ignore empty request"); + return; + } + + payload = (char *)&inbuf[1]; + + LOG_VERBOSE("receive request:%c %s\n", request, payload); + handle_packet(server, request, payload); +} + +static inline void +push_byte(rsp_recv_context_t *ctx, unsigned char ch, bool checksum) +{ + if (ctx->receive_index >= sizeof(ctx->receive_buffer)) { + LOG_ERROR("RSP message buffer overflow"); + bh_assert(false); + return; + } + + ctx->receive_buffer[ctx->receive_index++] = ch; + + if (checksum) { + ctx->check_sum += ch; + } +} + +/** + * The packet layout is: + * 1. Normal packet: + * '$' + payload + '#' + checksum(2bytes) + * ^ + * packetend + * 2. Interrupt: + * 0x03 + */ + +/* return: + * 0: incomplete message received + * 1: complete message received + * 2: interrupt message received + */ +static int +on_rsp_byte_arrive(unsigned char ch, rsp_recv_context_t *ctx) +{ + if (ctx->phase == Phase_Idle) { + ctx->receive_index = 0; + ctx->check_sum = 0; + + if (ch == 0x03) { + LOG_VERBOSE("Receive interrupt package"); + return 2; + } + else if (ch == '$') { + ctx->phase = Phase_Payload; + } + + return 0; + } + else if (ctx->phase == Phase_Payload) { + if (ch == '#') { + ctx->phase = Phase_Checksum; + push_byte(ctx, ch, false); + } + else { + push_byte(ctx, ch, true); + } + + return 0; + } + else if (ctx->phase == Phase_Checksum) { + ctx->size_in_phase++; + push_byte(ctx, ch, false); + + if (ctx->size_in_phase == 2) { + ctx->size_in_phase = 0; + + bh_assert(ctx->receive_index >= 3); + + if ((hex(ctx->receive_buffer[ctx->receive_index - 2]) << 4 + | hex(ctx->receive_buffer[ctx->receive_index - 1])) + != ctx->check_sum) { + LOG_WARNING("RSP package checksum error, ignore it"); + ctx->phase = Phase_Idle; + return 0; + } + else { + /* Change # to \0 */ + ctx->receive_buffer[ctx->receive_index - 3] = '\0'; + ctx->phase = Phase_Idle; + return 1; + } + } + + return 0; + } + + /* Should never reach here */ + bh_assert(false); + return 0; +} + +bool +wasm_gdbserver_handle_packet(WASMGDBServer *server) +{ + int32 n; + char buf[1024]; + + if (os_socket_settimeout(server->socket_fd, 1000) != 0) { + LOG_ERROR("Set socket recv timeout failed"); + return false; + } + + n = os_socket_recv(server->socket_fd, buf, sizeof(buf)); + + if (n == 0) { + handle_detach_request(server, NULL); + LOG_VERBOSE("Debugger disconnected, waiting for debugger reconnection"); + return true; + } + else if (n < 0) { +#if defined(BH_PLATFORM_WINDOWS) + if (WSAGetLastError() == WSAETIMEDOUT) +#else + if (errno == EAGAIN || errno == EWOULDBLOCK) +#endif + { + /* No bytes arrived */ + return true; + } + else { + LOG_ERROR("Socket receive error"); + return false; + } + } + else { + int32 i, ret; + + for (i = 0; i < n; i++) { + ret = on_rsp_byte_arrive(buf[i], server->receive_ctx); + + if (ret == 1) { + if (!server->noack) + write_data_raw(server, (uint8 *)"+", 1); + + process_packet(server); + } + else if (ret == 2) { + handle_interrupt(server); + } + } + } + + return true; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/gdbserver.h b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/gdbserver.h new file mode 100644 index 0000000..9e279a2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/gdbserver.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _GDB_SERVER_H +#define _GDB_SERVER_H + +#include "bh_platform.h" + +#define PACKET_BUF_SIZE 0x8000 + +enum GDBStoppointType { + eStoppointInvalid = -1, + eBreakpointSoftware = 0, + eBreakpointHardware, + eWatchpointWrite, + eWatchpointRead, + eWatchpointReadWrite +}; + +typedef enum rsp_recv_phase_t { + Phase_Idle, + Phase_Payload, + Phase_Checksum +} rsp_recv_phase_t; + +/* Remote Serial Protocol Receive Context */ +typedef struct rsp_recv_context_t { + rsp_recv_phase_t phase; + uint16 receive_index; + uint16 size_in_phase; + uint8 check_sum; + /* RSP packet should not be too long */ + char receive_buffer[1024]; +} rsp_recv_context_t; + +typedef struct WasmDebugPacket { + unsigned char buf[PACKET_BUF_SIZE]; + uint32 size; +} WasmDebugPacket; + +struct WASMDebugControlThread; +typedef struct WASMGDBServer { + bh_socket_t listen_fd; + bh_socket_t socket_fd; + WasmDebugPacket pkt; + bool noack; + struct WASMDebugControlThread *thread; + rsp_recv_context_t *receive_ctx; +} WASMGDBServer; + +WASMGDBServer * +wasm_create_gdbserver(const char *host, int32 *port); + +bool +wasm_gdbserver_listen(WASMGDBServer *server); + +bool +wasm_gdbserver_accept(WASMGDBServer *server); + +void +wasm_gdbserver_detach(WASMGDBServer *server); + +void +wasm_close_gdbserver(WASMGDBServer *server); + +bool +wasm_gdbserver_handle_packet(WASMGDBServer *server); +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/handler.c b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/handler.c new file mode 100644 index 0000000..905ca2f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/handler.c @@ -0,0 +1,878 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" +#include "handler.h" +#include "debug_engine.h" +#include "packets.h" +#include "utils.h" +#include "wasm_runtime.h" + +/* + * Note: A moderate MAX_PACKET_SIZE is ok because + * LLDB queries our buffer size (via qSupported PacketSize) + * and limits packet sizes accordingly. + */ + +#if defined(DEBUG_MAX_PACKET_SIZE) +#define MAX_PACKET_SIZE DEBUG_MAX_PACKET_SIZE +#else +#define MAX_PACKET_SIZE (4096) +#endif + +/* + * Note: It's assumed that MAX_PACKET_SIZE is reasonably large. + * See GetWorkingDir, WasmCallStack, etc. + */ +#if MAX_PACKET_SIZE < PATH_MAX || MAX_PACKET_SIZE < (2048 + 1) +#error MAX_PACKET_SIZE is too small +#endif + +static char *tmpbuf; +static korp_mutex tmpbuf_lock; + +int +wasm_debug_handler_init() +{ + int ret; + tmpbuf = wasm_runtime_malloc(MAX_PACKET_SIZE); + if (tmpbuf == NULL) { + LOG_ERROR("debug-engine: Packet buffer allocation failure"); + return BHT_ERROR; + } + ret = os_mutex_init(&tmpbuf_lock); + if (ret != BHT_OK) { + wasm_runtime_free(tmpbuf); + tmpbuf = NULL; + } + return ret; +} + +void +wasm_debug_handler_deinit() +{ + wasm_runtime_free(tmpbuf); + tmpbuf = NULL; + os_mutex_destroy(&tmpbuf_lock); +} + +void +handle_interrupt(WASMGDBServer *server) +{ + wasm_debug_instance_interrupt_all_threads(server->thread->debug_instance); +} + +void +handle_general_set(WASMGDBServer *server, char *payload) +{ + const char *name; + char *args; + + args = strchr(payload, ':'); + if (args) + *args++ = '\0'; + + name = payload; + LOG_VERBOSE("%s:%s\n", __FUNCTION__, payload); + + if (!strcmp(name, "StartNoAckMode")) { + server->noack = true; + write_packet(server, "OK"); + } + if (!strcmp(name, "ThreadSuffixSupported")) { + write_packet(server, ""); + } + if (!strcmp(name, "ListThreadsInStopReply")) { + write_packet(server, ""); + } + if (!strcmp(name, "EnableErrorStrings")) { + write_packet(server, "OK"); + } +} + +static void +process_xfer(WASMGDBServer *server, const char *name, char *args) +{ + const char *mode = args; + + args = strchr(args, ':'); + if (args) + *args++ = '\0'; + + if (!strcmp(name, "libraries") && !strcmp(mode, "read")) { + // TODO: how to get current wasm file name? + uint64 addr = wasm_debug_instance_get_load_addr( + (WASMDebugInstance *)server->thread->debug_instance); + os_mutex_lock(&tmpbuf_lock); +#if WASM_ENABLE_LIBC_WASI != 0 + char objname[128]; + if (!wasm_debug_instance_get_current_object_name( + (WASMDebugInstance *)server->thread->debug_instance, objname, + 128)) { + objname[0] = 0; /* use an empty string */ + } + snprintf(tmpbuf, MAX_PACKET_SIZE, + "l
", + objname, addr); +#else + snprintf(tmpbuf, MAX_PACKET_SIZE, + "l
", + "nobody.wasm", addr); +#endif + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } +} + +void +process_wasm_local(WASMGDBServer *server, char *args) +{ + int32 frame_index; + int32 local_index; + char buf[16]; + int32 size = 16; + bool ret; + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "E01"); + if (sscanf(args, "%" PRId32 ";%" PRId32, &frame_index, &local_index) == 2) { + ret = wasm_debug_instance_get_local( + (WASMDebugInstance *)server->thread->debug_instance, frame_index, + local_index, buf, &size); + if (ret && size > 0) { + mem2hex(buf, tmpbuf, size); + } + } + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +void +process_wasm_global(WASMGDBServer *server, char *args) +{ + int32 frame_index; + int32 global_index; + char buf[16]; + int32 size = 16; + bool ret; + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "E01"); + if (sscanf(args, "%" PRId32 ";%" PRId32, &frame_index, &global_index) + == 2) { + ret = wasm_debug_instance_get_global( + (WASMDebugInstance *)server->thread->debug_instance, frame_index, + global_index, buf, &size); + if (ret && size > 0) { + mem2hex(buf, tmpbuf, size); + } + } + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +void +handle_general_query(WASMGDBServer *server, char *payload) +{ + const char *name; + char *args; + char triple[256]; + + args = strchr(payload, ':'); + if (args) + *args++ = '\0'; + name = payload; + LOG_VERBOSE("%s:%s\n", __FUNCTION__, payload); + + if (!strcmp(name, "C")) { + uint64 pid, tid; + pid = wasm_debug_instance_get_pid( + (WASMDebugInstance *)server->thread->debug_instance); + tid = (uint64)(uintptr_t)wasm_debug_instance_get_tid( + (WASMDebugInstance *)server->thread->debug_instance); + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "QCp%" PRIx64 ".%" PRIx64 "", pid, + tid); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } + if (!strcmp(name, "Supported")) { + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, + "qXfer:libraries:read+;PacketSize=%" PRIx32 ";", + MAX_PACKET_SIZE); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } + + if (!strcmp(name, "Xfer")) { + name = args; + + if (!args) { + LOG_ERROR("payload parse error during handle_general_query"); + return; + } + + args = strchr(args, ':'); + + if (args) { + *args++ = '\0'; + process_xfer(server, name, args); + } + } + + if (!strcmp(name, "HostInfo")) { + mem2hex("wasm32-wamr-wasi-wasm", triple, + strlen("wasm32-wamr-wasi-wasm")); + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, + "vendor:wamr;ostype:wasi;arch:wasm32;" + "triple:%s;endian:little;ptrsize:4;", + triple); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } + if (!strcmp(name, "ModuleInfo")) { + write_packet(server, ""); + } + if (!strcmp(name, "GetWorkingDir")) { + os_mutex_lock(&tmpbuf_lock); + if (getcwd(tmpbuf, PATH_MAX)) + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } + if (!strcmp(name, "QueryGDBServer")) { + write_packet(server, ""); + } + if (!strcmp(name, "VAttachOrWaitSupported")) { + write_packet(server, ""); + } + if (!strcmp(name, "ProcessInfo")) { + // Todo: process id parent-pid + uint64 pid; + pid = wasm_debug_instance_get_pid( + (WASMDebugInstance *)server->thread->debug_instance); + mem2hex("wasm32-wamr-wasi-wasm", triple, + strlen("wasm32-wamr-wasi-wasm")); + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, + "pid:%" PRIx64 ";parent-pid:%" PRIx64 + ";vendor:wamr;ostype:wasi;arch:wasm32;" + "triple:%s;endian:little;ptrsize:4;", + pid, pid, triple); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } + if (!strcmp(name, "RegisterInfo0")) { + os_mutex_lock(&tmpbuf_lock); + snprintf( + tmpbuf, MAX_PACKET_SIZE, + "name:pc;alt-name:pc;bitsize:64;offset:0;encoding:uint;format:hex;" + "set:General Purpose Registers;gcc:16;dwarf:16;generic:pc;"); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } + else if (!strncmp(name, "RegisterInfo", strlen("RegisterInfo"))) { + write_packet(server, "E45"); + } + if (!strcmp(name, "StructuredDataPlugins")) { + write_packet(server, ""); + } + + if (args && (!strcmp(name, "MemoryRegionInfo"))) { + uint64 addr = strtoll(args, NULL, 16); + WASMDebugMemoryInfo *mem_info = wasm_debug_instance_get_memregion( + (WASMDebugInstance *)server->thread->debug_instance, addr); + if (mem_info) { + char name_buf[256]; + mem2hex(mem_info->name, name_buf, strlen(mem_info->name)); + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, + "start:%" PRIx64 ";size:%" PRIx64 + ";permissions:%s;name:%s;", + (uint64)mem_info->start, mem_info->size, + mem_info->permisson, name_buf); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + + wasm_debug_instance_destroy_memregion( + (WASMDebugInstance *)server->thread->debug_instance, mem_info); + } + } + + if (!strcmp(name, "WasmData")) { + write_packet(server, ""); + } + + if (!strcmp(name, "WasmMem")) { + write_packet(server, ""); + } + + if (!strcmp(name, "Symbol")) { + write_packet(server, ""); + } + + if (args && (!strcmp(name, "WasmCallStack"))) { + uint64 tid = strtoll(args, NULL, 16); + uint64 buf[1024 / sizeof(uint64)]; + uint32 count = wasm_debug_instance_get_call_stack_pcs( + (WASMDebugInstance *)server->thread->debug_instance, + (korp_tid)(uintptr_t)tid, buf, 1024 / sizeof(uint64)); + + if (count > 0) { + os_mutex_lock(&tmpbuf_lock); + mem2hex((char *)buf, tmpbuf, count * sizeof(uint64)); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } + else + write_packet(server, ""); + } + + if (args && (!strcmp(name, "WasmLocal"))) { + process_wasm_local(server, args); + } + + if (args && (!strcmp(name, "WasmGlobal"))) { + process_wasm_global(server, args); + } + + if (!strcmp(name, "Offsets")) { + write_packet(server, ""); + } + + if (!strncmp(name, "ThreadStopInfo", strlen("ThreadStopInfo"))) { + int32 prefix_len = strlen("ThreadStopInfo"); + uint64 tid_number = strtoll(name + prefix_len, NULL, 16); + korp_tid tid = (korp_tid)(uintptr_t)tid_number; + uint32 status; + + status = wasm_debug_instance_get_thread_status( + server->thread->debug_instance, tid); + + send_thread_stop_status(server, status, tid); + } + + if (!strcmp(name, "WatchpointSupportInfo")) { + os_mutex_lock(&tmpbuf_lock); + // Any uint32 is OK for the watchpoint support + snprintf(tmpbuf, MAX_PACKET_SIZE, "num:32;"); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + } +} + +void +send_thread_stop_status(WASMGDBServer *server, uint32 status, korp_tid tid) +{ + int32 len = 0; + uint64 pc; + korp_tid tids[20]; + char pc_string[17]; + uint32 tids_count, i = 0; + uint32 gdb_status = status; + WASMExecEnv *exec_env; + const char *exception; + + if (status == 0) { + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "W%02x", status); + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); + return; + } + tids_count = wasm_debug_instance_get_tids( + (WASMDebugInstance *)server->thread->debug_instance, tids, 20); + pc = wasm_debug_instance_get_pc( + (WASMDebugInstance *)server->thread->debug_instance); + + if (status == WAMR_SIG_SINGSTEP) { + gdb_status = WAMR_SIG_TRAP; + } + + os_mutex_lock(&tmpbuf_lock); + // TODO: how name a wasm thread? + len += snprintf(tmpbuf, MAX_PACKET_SIZE, "T%02xthread:%" PRIx64 ";name:%s;", + gdb_status, (uint64)(uintptr_t)tid, "nobody"); + if (tids_count > 0) { + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, "threads:"); + while (i < tids_count) { + if (i == tids_count - 1) + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, + "%" PRIx64 ";", (uint64)(uintptr_t)tids[i]); + else + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, + "%" PRIx64 ",", (uint64)(uintptr_t)tids[i]); + i++; + } + } + mem2hex((void *)&pc, pc_string, 8); + pc_string[8 * 2] = '\0'; + + exec_env = wasm_debug_instance_get_current_env( + (WASMDebugInstance *)server->thread->debug_instance); + bh_assert(exec_env); + + exception = + wasm_runtime_get_exception(wasm_runtime_get_module_inst(exec_env)); + if (exception) { + /* When exception occurs, use reason:exception so the description can be + * correctly processed by LLDB */ + uint32 exception_len = strlen(exception); + len += + snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, + "thread-pcs:%" PRIx64 ";00:%s;reason:%s;description:", pc, + pc_string, "exception"); + /* The description should be encoded as HEX */ + for (i = 0; i < exception_len; i++) { + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, "%02x", + exception[i]); + } + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, ";"); + } + else { + if (status == WAMR_SIG_TRAP) { + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, + "thread-pcs:%" PRIx64 ";00:%s;reason:%s;", pc, + pc_string, "breakpoint"); + } + else if (status == WAMR_SIG_SINGSTEP) { + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, + "thread-pcs:%" PRIx64 ";00:%s;reason:%s;", pc, + pc_string, "trace"); + } + else { /* status > 0 (== 0 is checked at the function beginning) */ + len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, + "thread-pcs:%" PRIx64 ";00:%s;reason:%s;", pc, + pc_string, "signal"); + } + } + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +void +handle_v_packet(WASMGDBServer *server, char *payload) +{ + const char *name; + char *args; + + args = strchr(payload, ';'); + if (args) + *args++ = '\0'; + name = payload; + LOG_VERBOSE("%s:%s\n", __FUNCTION__, payload); + + if (!strcmp("Cont?", name)) + write_packet(server, "vCont;c;C;s;S;"); + + if (!strcmp("Cont", name)) { + if (args) { + if (args[0] == 's' || args[0] == 'c') { + char *numstring = strchr(args, ':'); + if (numstring) { + uint64 tid_number; + korp_tid tid; + + *numstring++ = '\0'; + tid_number = strtoll(numstring, NULL, 16); + tid = (korp_tid)(uintptr_t)tid_number; + wasm_debug_instance_set_cur_thread( + (WASMDebugInstance *)server->thread->debug_instance, + tid); + + if (args[0] == 's') { + wasm_debug_instance_singlestep( + (WASMDebugInstance *)server->thread->debug_instance, + tid); + } + else { + wasm_debug_instance_continue( + (WASMDebugInstance *) + server->thread->debug_instance); + } + } + } + } + } +} + +void +handle_threadstop_request(WASMGDBServer *server, char *payload) +{ + korp_tid tid; + uint32 status; + WASMDebugInstance *debug_inst = + (WASMDebugInstance *)server->thread->debug_instance; + bh_assert(debug_inst); + + /* According to + https://sourceware.org/gdb/onlinedocs/gdb/Packets.html#Packets, the "?" + package should be sent when connection is first established to query the + reason the target halted */ + bh_assert(debug_inst->current_state == DBG_LAUNCHING); + + /* Waiting for the stop event */ + os_mutex_lock(&debug_inst->wait_lock); + while (!debug_inst->stopped_thread) { + os_cond_wait(&debug_inst->wait_cond, &debug_inst->wait_lock); + } + os_mutex_unlock(&debug_inst->wait_lock); + + tid = debug_inst->stopped_thread->handle; + status = (uint32)debug_inst->stopped_thread->current_status->signal_flag; + + wasm_debug_instance_set_cur_thread(debug_inst, tid); + + send_thread_stop_status(server, status, tid); + + debug_inst->current_state = APP_STOPPED; + debug_inst->stopped_thread = NULL; +} + +void +handle_set_current_thread(WASMGDBServer *server, char *payload) +{ + LOG_VERBOSE("%s:%s\n", __FUNCTION__, payload); + if ('g' == *payload++) { + uint64 tid = strtoll(payload, NULL, 16); + if (tid > 0) + wasm_debug_instance_set_cur_thread( + (WASMDebugInstance *)server->thread->debug_instance, + (korp_tid)(uintptr_t)tid); + } + write_packet(server, "OK"); +} + +void +handle_get_register(WASMGDBServer *server, char *payload) +{ + uint64 regdata; + int32 i = strtol(payload, NULL, 16); + + if (i != 0) { + write_packet(server, "E01"); + return; + } + regdata = wasm_debug_instance_get_pc( + (WASMDebugInstance *)server->thread->debug_instance); + + os_mutex_lock(&tmpbuf_lock); + mem2hex((void *)®data, tmpbuf, 8); + tmpbuf[8 * 2] = '\0'; + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +void +handle_get_json_request(WASMGDBServer *server, char *payload) +{ + char *args; + + args = strchr(payload, ':'); + if (args) + *args++ = '\0'; + write_packet(server, ""); +} + +void +handle_get_read_binary_memory(WASMGDBServer *server, char *payload) +{ + write_packet(server, ""); +} + +void +handle_get_read_memory(WASMGDBServer *server, char *payload) +{ + uint64 maddr, mlen; + bool ret; + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "%s", ""); + if (sscanf(payload, "%" SCNx64 ",%" SCNx64, &maddr, &mlen) == 2) { + char *buff; + + if (mlen * 2 > MAX_PACKET_SIZE) { + LOG_ERROR("Buffer overflow!"); + mlen = MAX_PACKET_SIZE / 2; + } + + buff = wasm_runtime_malloc(mlen); + if (buff) { + ret = wasm_debug_instance_get_mem( + (WASMDebugInstance *)server->thread->debug_instance, maddr, + buff, &mlen); + if (ret) { + mem2hex(buff, tmpbuf, mlen); + } + wasm_runtime_free(buff); + } + } + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +void +handle_get_write_memory(WASMGDBServer *server, char *payload) +{ + size_t hex_len; + int32 offset, act_len; + uint64 maddr, mlen; + char *buff; + bool ret; + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "%s", ""); + if (sscanf(payload, "%" SCNx64 ",%" SCNx64 ":%n", &maddr, &mlen, &offset) + == 2) { + payload += offset; + hex_len = strlen(payload); + act_len = hex_len / 2 < mlen ? hex_len / 2 : mlen; + + buff = wasm_runtime_malloc(act_len); + if (buff) { + hex2mem(payload, buff, act_len); + ret = wasm_debug_instance_set_mem( + (WASMDebugInstance *)server->thread->debug_instance, maddr, + buff, &mlen); + if (ret) { + snprintf(tmpbuf, MAX_PACKET_SIZE, "%s", "OK"); + } + wasm_runtime_free(buff); + } + } + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +void +handle_breakpoint_software_add(WASMGDBServer *server, uint64 addr, + size_t length) +{ + bool ret = wasm_debug_instance_add_breakpoint( + (WASMDebugInstance *)server->thread->debug_instance, addr, length); + write_packet(server, ret ? "OK" : "EO1"); +} + +void +handle_breakpoint_software_remove(WASMGDBServer *server, uint64 addr, + size_t length) +{ + bool ret = wasm_debug_instance_remove_breakpoint( + (WASMDebugInstance *)server->thread->debug_instance, addr, length); + write_packet(server, ret ? "OK" : "EO1"); +} + +void +handle_watchpoint_write_add(WASMGDBServer *server, uint64 addr, size_t length) +{ + bool ret = wasm_debug_instance_watchpoint_write_add( + (WASMDebugInstance *)server->thread->debug_instance, addr, length); + write_packet(server, ret ? "OK" : "EO1"); +} + +void +handle_watchpoint_write_remove(WASMGDBServer *server, uint64 addr, + size_t length) +{ + bool ret = wasm_debug_instance_watchpoint_write_remove( + (WASMDebugInstance *)server->thread->debug_instance, addr, length); + write_packet(server, ret ? "OK" : "EO1"); +} + +void +handle_watchpoint_read_add(WASMGDBServer *server, uint64 addr, size_t length) +{ + bool ret = wasm_debug_instance_watchpoint_read_add( + (WASMDebugInstance *)server->thread->debug_instance, addr, length); + write_packet(server, ret ? "OK" : "EO1"); +} + +void +handle_watchpoint_read_remove(WASMGDBServer *server, uint64 addr, size_t length) +{ + bool ret = wasm_debug_instance_watchpoint_read_remove( + (WASMDebugInstance *)server->thread->debug_instance, addr, length); + write_packet(server, ret ? "OK" : "EO1"); +} + +void +handle_add_break(WASMGDBServer *server, char *payload) +{ + int arg_c; + size_t type, length; + uint64 addr; + + if ((arg_c = sscanf(payload, "%zx,%" SCNx64 ",%zx", &type, &addr, &length)) + != 3) { + LOG_ERROR("Unsupported number of add break arguments %d", arg_c); + write_packet(server, ""); + return; + } + + switch (type) { + case eBreakpointSoftware: + handle_breakpoint_software_add(server, addr, length); + break; + case eWatchpointWrite: + handle_watchpoint_write_add(server, addr, length); + break; + case eWatchpointRead: + handle_watchpoint_read_add(server, addr, length); + break; + case eWatchpointReadWrite: + handle_watchpoint_write_add(server, addr, length); + handle_watchpoint_read_add(server, addr, length); + break; + default: + LOG_ERROR("Unsupported breakpoint type %zu", type); + write_packet(server, ""); + break; + } +} + +void +handle_remove_break(WASMGDBServer *server, char *payload) +{ + int arg_c; + size_t type, length; + uint64 addr; + + if ((arg_c = sscanf(payload, "%zx,%" SCNx64 ",%zx", &type, &addr, &length)) + != 3) { + LOG_ERROR("Unsupported number of remove break arguments %d", arg_c); + write_packet(server, ""); + return; + } + + switch (type) { + case eBreakpointSoftware: + handle_breakpoint_software_remove(server, addr, length); + break; + case eWatchpointWrite: + handle_watchpoint_write_remove(server, addr, length); + break; + case eWatchpointRead: + handle_watchpoint_read_remove(server, addr, length); + break; + case eWatchpointReadWrite: + handle_watchpoint_write_remove(server, addr, length); + handle_watchpoint_read_remove(server, addr, length); + break; + default: + LOG_ERROR("Unsupported breakpoint type %zu", type); + write_packet(server, ""); + break; + } +} + +void +handle_continue_request(WASMGDBServer *server, char *payload) +{ + wasm_debug_instance_continue( + (WASMDebugInstance *)server->thread->debug_instance); +} + +void +handle_kill_request(WASMGDBServer *server, char *payload) +{ + wasm_debug_instance_kill( + (WASMDebugInstance *)server->thread->debug_instance); +} + +static void +handle_malloc(WASMGDBServer *server, char *payload) +{ + char *args; + uint64 addr, size; + int32 map_prot = MMAP_PROT_NONE; + + args = strstr(payload, ","); + if (args) { + *args++ = '\0'; + } + else { + LOG_ERROR("Payload parse error during handle malloc"); + return; + } + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "%s", "E03"); + + size = strtoll(payload, NULL, 16); + if (size > 0) { + while (*args) { + if (*args == 'r') { + map_prot |= MMAP_PROT_READ; + } + if (*args == 'w') { + map_prot |= MMAP_PROT_WRITE; + } + if (*args == 'x') { + map_prot |= MMAP_PROT_EXEC; + } + args++; + } + addr = wasm_debug_instance_mmap( + (WASMDebugInstance *)server->thread->debug_instance, size, + map_prot); + if (addr) { + snprintf(tmpbuf, MAX_PACKET_SIZE, "%" PRIx64, addr); + } + } + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +static void +handle_free(WASMGDBServer *server, char *payload) +{ + uint64 addr; + bool ret; + + os_mutex_lock(&tmpbuf_lock); + snprintf(tmpbuf, MAX_PACKET_SIZE, "%s", "E03"); + addr = strtoll(payload, NULL, 16); + + ret = wasm_debug_instance_ummap( + (WASMDebugInstance *)server->thread->debug_instance, addr); + if (ret) { + snprintf(tmpbuf, MAX_PACKET_SIZE, "%s", "OK"); + } + + write_packet(server, tmpbuf); + os_mutex_unlock(&tmpbuf_lock); +} + +void +handle____request(WASMGDBServer *server, char *payload) +{ + char *args; + + if (payload[0] == 'M') { + args = payload + 1; + handle_malloc(server, args); + } + if (payload[0] == 'm') { + args = payload + 1; + handle_free(server, args); + } +} + +void +handle_detach_request(WASMGDBServer *server, char *payload) +{ + if (payload != NULL) { + write_packet(server, "OK"); + } + wasm_debug_instance_detach( + (WASMDebugInstance *)server->thread->debug_instance); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/handler.h b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/handler.h new file mode 100644 index 0000000..af2566d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/handler.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef HANDLER_H +#define HANDLER_H + +#include "gdbserver.h" + +int +wasm_debug_handler_init(); + +void +wasm_debug_handler_deinit(); + +void +handle_interrupt(WASMGDBServer *server); + +void +handle_general_set(WASMGDBServer *server, char *payload); + +void +handle_general_query(WASMGDBServer *server, char *payload); + +void +handle_v_packet(WASMGDBServer *server, char *payload); + +void +handle_threadstop_request(WASMGDBServer *server, char *payload); + +void +handle_set_current_thread(WASMGDBServer *server, char *payload); + +void +handle_get_register(WASMGDBServer *server, char *payload); + +void +handle_get_json_request(WASMGDBServer *server, char *payload); + +void +handle_get_read_binary_memory(WASMGDBServer *server, char *payload); + +void +handle_get_read_memory(WASMGDBServer *server, char *payload); + +void +handle_get_write_memory(WASMGDBServer *server, char *payload); + +void +handle_add_break(WASMGDBServer *server, char *payload); + +void +handle_remove_break(WASMGDBServer *server, char *payload); + +void +handle_continue_request(WASMGDBServer *server, char *payload); + +void +handle_kill_request(WASMGDBServer *server, char *payload); + +void +handle____request(WASMGDBServer *server, char *payload); + +void +handle_detach_request(WASMGDBServer *server, char *payload); + +void +send_thread_stop_status(WASMGDBServer *server, uint32 status, korp_tid tid); +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/packets.c b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/packets.c new file mode 100644 index 0000000..1bdb3d2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/packets.c @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" +#include "packets.h" +#include "gdbserver.h" + +void +write_data_raw(WASMGDBServer *gdbserver, const uint8 *data, ssize_t len) +{ + ssize_t nwritten; + + nwritten = os_socket_send(gdbserver->socket_fd, data, len); + if (nwritten < 0) { + LOG_ERROR("Write error\n"); + exit(-2); + } +} + +void +write_hex(WASMGDBServer *gdbserver, unsigned long hex) +{ + char buf[32]; + size_t len; + + len = snprintf(buf, sizeof(buf) - 1, "%02lx", hex); + write_data_raw(gdbserver, (uint8 *)buf, len); +} + +void +write_packet_bytes(WASMGDBServer *gdbserver, const uint8 *data, + size_t num_bytes) +{ + uint8 checksum; + size_t i; + + write_data_raw(gdbserver, (uint8 *)"$", 1); + for (i = 0, checksum = 0; i < num_bytes; ++i) + checksum += data[i]; + write_data_raw(gdbserver, (uint8 *)data, num_bytes); + write_data_raw(gdbserver, (uint8 *)"#", 1); + write_hex(gdbserver, checksum); +} + +void +write_packet(WASMGDBServer *gdbserver, const char *data) +{ + LOG_VERBOSE("send replay:%s", data); + write_packet_bytes(gdbserver, (const uint8 *)data, strlen(data)); +} + +void +write_binary_packet(WASMGDBServer *gdbserver, const char *pfx, + const uint8 *data, ssize_t num_bytes) +{ + uint8 *buf; + ssize_t pfx_num_chars = strlen(pfx); + ssize_t buf_num_bytes = 0, total_size; + int32 i; + + total_size = 2 * num_bytes + pfx_num_chars; + buf = wasm_runtime_malloc(total_size); + if (!buf) { + LOG_ERROR("Failed to allocate memory for binary packet"); + return; + } + + memset(buf, 0, total_size); + memcpy(buf, pfx, pfx_num_chars); + buf_num_bytes += pfx_num_chars; + + for (i = 0; i < num_bytes; ++i) { + uint8 b = data[i]; + switch (b) { + case '#': + case '$': + case '}': + case '*': + buf[buf_num_bytes++] = '}'; + buf[buf_num_bytes++] = b ^ 0x20; + break; + default: + buf[buf_num_bytes++] = b; + break; + } + } + write_packet_bytes(gdbserver, buf, buf_num_bytes); + wasm_runtime_free(buf); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/packets.h b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/packets.h new file mode 100644 index 0000000..b358893 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/packets.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef PACKETS_H +#define PACKETS_H + +#include "gdbserver.h" + +void +write_data_raw(WASMGDBServer *gdbserver, const uint8 *data, ssize_t len); + +void +write_packet(WASMGDBServer *gdbserver, const char *data); + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/utils.c b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/utils.c new file mode 100644 index 0000000..4d9299c --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/utils.c @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "utils.h" + +static const char hexchars[] = "0123456789abcdef"; + +int32 +hex(char ch) +{ + if ((ch >= 'a') && (ch <= 'f')) + return (ch - 'a' + 10); + if ((ch >= '0') && (ch <= '9')) + return (ch - '0'); + if ((ch >= 'A') && (ch <= 'F')) + return (ch - 'A' + 10); + return (-1); +} + +char * +mem2hex(char *mem, char *buf, int32 count) +{ + uint8 ch; + + for (int i = 0; i < count; i++) { + ch = *(mem++); + *buf++ = hexchars[ch >> 4]; + *buf++ = hexchars[ch % 16]; + } + *buf = 0; + return (buf); +} + +char * +hex2mem(char *buf, char *mem, int32 count) +{ + uint8 ch; + + for (int i = 0; i < count; i++) { + ch = hex(*buf++) << 4; + ch = ch + hex(*buf++); + *(mem++) = ch; + } + return (mem); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/debug-engine/utils.h b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/utils.h new file mode 100644 index 0000000..1c75808 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/debug-engine/utils.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2021 Ant Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef UTILS_H +#define UTILS_H + +#include "bh_platform.h" + +int32 +hex(char ch); + +char * +mem2hex(char *mem, char *buf, int32 count); + +char * +hex2mem(char *buf, char *mem, int32 count); + +int32 +unescape(char *msg, int32 len); + +#endif /* UTILS_H */ diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/SConscript b/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/SConscript new file mode 100644 index 0000000..d03936c --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/SConscript @@ -0,0 +1,20 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * + +cwd = GetCurrentDir() + +src = Split(''' +libc_pthread_wrapper.c +''') + +CPPPATH = [cwd] + + +group = DefineGroup('iwasm_libc_pthread', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/lib_pthread.cmake b/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/lib_pthread.cmake new file mode 100644 index 0000000..134edf0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/lib_pthread.cmake @@ -0,0 +1,17 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIB_PTHREAD_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_LIB_PTHREAD=1) + +if (WAMR_BUILD_LIB_PTHREAD_SEMAPHORE EQUAL 1) +add_definitions (-DWASM_ENABLE_LIB_PTHREAD_SEMAPHORE=1) +endif() + +include_directories(${LIB_PTHREAD_DIR}) + +file (GLOB source_all ${LIB_PTHREAD_DIR}/*.c) + +set (LIB_PTHREAD_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c b/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c new file mode 100644 index 0000000..b3fa57d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c @@ -0,0 +1,1358 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "../interpreter/wasm.h" +#include "../common/wasm_runtime_common.h" +#include "thread_manager.h" + +#if WASM_ENABLE_INTERP != 0 +#include "wasm_runtime.h" +#endif + +#if WASM_ENABLE_AOT != 0 +#include "aot_runtime.h" +#endif + +#define WAMR_PTHREAD_KEYS_MAX 32 + +/* clang-format off */ +#define get_module(exec_env) \ + wasm_exec_env_get_module(exec_env) + +#define get_module_inst(exec_env) \ + wasm_runtime_get_module_inst(exec_env) + +#define get_thread_arg(exec_env) \ + wasm_exec_env_get_thread_arg(exec_env) + +#define get_wasi_ctx(module_inst) \ + wasm_runtime_get_wasi_ctx(module_inst) + +#define validate_app_addr(offset, size) \ + wasm_runtime_validate_app_addr(module_inst, offset, size) + +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define addr_app_to_native(offset) \ + wasm_runtime_addr_app_to_native(module_inst, offset) + +#define addr_native_to_app(ptr) \ + wasm_runtime_addr_native_to_app(module_inst, ptr) +/* clang-format on */ + +enum { + T_THREAD, + T_MUTEX, + T_COND, + T_SEM, +}; + +enum thread_status_t { + THREAD_INIT, + THREAD_RUNNING, + THREAD_CANCELLED, + THREAD_EXIT, +}; + +enum mutex_status_t { + MUTEX_CREATED, + MUTEX_DESTROYED, +}; + +enum cond_status_t { + COND_CREATED, + COND_DESTROYED, +}; + +enum sem_status_t { + SEM_CREATED, + SEM_CLOSED, + SEM_DESTROYED, +}; + +typedef struct ThreadKeyValueNode { + bh_list_link l; + wasm_exec_env_t exec_env; + int32 thread_key_values[WAMR_PTHREAD_KEYS_MAX]; +} ThreadKeyValueNode; + +typedef struct KeyData { + int32 destructor_func; + bool is_created; +} KeyData; + +typedef struct ClusterInfoNode { + bh_list_link l; + WASMCluster *cluster; + HashMap *thread_info_map; + /* Key data list */ + KeyData key_data_list[WAMR_PTHREAD_KEYS_MAX]; + korp_mutex key_data_list_lock; + /* Every node contains the key value list for a thread */ + bh_list thread_list_head; + bh_list *thread_list; +} ClusterInfoNode; + +typedef struct ThreadInfoNode { + wasm_exec_env_t parent_exec_env; + wasm_exec_env_t exec_env; + /* the id returned to app */ + uint32 handle; + /* type can be [THREAD | MUTEX | CONDITION] */ + uint32 type; + /* Thread status, this variable should be volatile + as its value may be changed in different threads */ + volatile uint32 status; + bool joinable; + union { + korp_tid thread; + korp_mutex *mutex; + korp_cond *cond; +#if WASM_ENABLE_LIB_PTHREAD_SEMAPHORE != 0 + korp_sem *sem; +#endif + /* A copy of the thread return value */ + void *ret; + } u; +} ThreadInfoNode; + +typedef struct { + ThreadInfoNode *info_node; + /* table elem index of the app's entry function */ + uint32 elem_index; + /* arg of the app's entry function */ + uint32 arg; + wasm_module_inst_t module_inst; +} ThreadRoutineArgs; + +typedef struct { + uint32 handle; + ThreadInfoNode *node; +} SemCallbackArgs; + +static bh_list cluster_info_list; +#if WASM_ENABLE_LIB_PTHREAD_SEMAPHORE != 0 +static HashMap *sem_info_map; +#endif +static korp_mutex thread_global_lock; +static uint32 handle_id = 1; + +static void +lib_pthread_destroy_callback(WASMCluster *cluster); + +static uint32 +thread_handle_hash(void *handle) +{ + return (uint32)(uintptr_t)handle; +} + +static bool +thread_handle_equal(void *h1, void *h2) +{ + return (uint32)(uintptr_t)h1 == (uint32)(uintptr_t)h2 ? true : false; +} + +static void +thread_info_destroy(void *node) +{ + ThreadInfoNode *info_node = (ThreadInfoNode *)node; + + os_mutex_lock(&thread_global_lock); + if (info_node->type == T_MUTEX) { + if (info_node->status != MUTEX_DESTROYED) + os_mutex_destroy(info_node->u.mutex); + wasm_runtime_free(info_node->u.mutex); + } + else if (info_node->type == T_COND) { + if (info_node->status != COND_DESTROYED) + os_cond_destroy(info_node->u.cond); + wasm_runtime_free(info_node->u.cond); + } +#if WASM_ENABLE_LIB_PTHREAD_SEMAPHORE != 0 + else if (info_node->type == T_SEM) { + if (info_node->status != SEM_DESTROYED) + os_sem_close(info_node->u.sem); + } +#endif + wasm_runtime_free(info_node); + os_mutex_unlock(&thread_global_lock); +} + +bool +lib_pthread_init() +{ + if (0 != os_mutex_init(&thread_global_lock)) + return false; + bh_list_init(&cluster_info_list); + if (!wasm_cluster_register_destroy_callback(lib_pthread_destroy_callback)) { + os_mutex_destroy(&thread_global_lock); + return false; + } +#if WASM_ENABLE_LIB_PTHREAD_SEMAPHORE != 0 + if (!(sem_info_map = bh_hash_map_create( + 32, true, (HashFunc)wasm_string_hash, + (KeyEqualFunc)wasm_string_equal, NULL, thread_info_destroy))) { + os_mutex_destroy(&thread_global_lock); + return false; + } +#endif + return true; +} + +void +lib_pthread_destroy() +{ +#if WASM_ENABLE_LIB_PTHREAD_SEMAPHORE != 0 + bh_hash_map_destroy(sem_info_map); +#endif + os_mutex_destroy(&thread_global_lock); +} + +static ClusterInfoNode * +get_cluster_info(WASMCluster *cluster) +{ + ClusterInfoNode *node; + + os_mutex_lock(&thread_global_lock); + node = bh_list_first_elem(&cluster_info_list); + + while (node) { + if (cluster == node->cluster) { + os_mutex_unlock(&thread_global_lock); + return node; + } + node = bh_list_elem_next(node); + } + os_mutex_unlock(&thread_global_lock); + + return NULL; +} + +static KeyData * +key_data_list_lookup(wasm_exec_env_t exec_env, int32 key) +{ + ClusterInfoNode *node; + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + + if ((node = get_cluster_info(cluster))) { + return (key >= 0 && key < WAMR_PTHREAD_KEYS_MAX + && node->key_data_list[key].is_created) + ? &(node->key_data_list[key]) + : NULL; + } + + return NULL; +} + +/** + * Lookup the thread key value node for a thread, create a new one if failed + * This design will reduce the memory usage. If the thread doesn't use the + * local storage, it will not occupy memory space. + */ +static int32 * +key_value_list_lookup_or_create(wasm_exec_env_t exec_env, ClusterInfoNode *info, + int32 key) +{ + KeyData *key_node; + ThreadKeyValueNode *data; + + /* Check if the key is valid */ + key_node = key_data_list_lookup(exec_env, key); + if (!key_node) { + return NULL; + } + + /* Find key values node */ + data = bh_list_first_elem(info->thread_list); + while (data) { + if (data->exec_env == exec_env) + return data->thread_key_values; + data = bh_list_elem_next(data); + } + + /* If not found, create a new node for this thread */ + if (!(data = wasm_runtime_malloc(sizeof(ThreadKeyValueNode)))) + return NULL; + memset(data, 0, sizeof(ThreadKeyValueNode)); + data->exec_env = exec_env; + + if (bh_list_insert(info->thread_list, data) != 0) { + wasm_runtime_free(data); + return NULL; + } + + return data->thread_key_values; +} + +static void +call_key_destructor(wasm_exec_env_t exec_env) +{ + int32 i; + uint32 destructor_index; + KeyData *key_node; + ThreadKeyValueNode *value_node; + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + ClusterInfoNode *info = get_cluster_info(cluster); + + if (!info) { + return; + } + + value_node = bh_list_first_elem(info->thread_list); + while (value_node) { + if (value_node->exec_env == exec_env) + break; + value_node = bh_list_elem_next(value_node); + } + + /* This thread hasn't created key value node */ + if (!value_node) + return; + + /* Destroy key values */ + for (i = 0; i < WAMR_PTHREAD_KEYS_MAX; i++) { + if (value_node->thread_key_values[i] != 0) { + int32 value = value_node->thread_key_values[i]; + os_mutex_lock(&info->key_data_list_lock); + + if ((key_node = key_data_list_lookup(exec_env, i))) + destructor_index = key_node->destructor_func; + else + destructor_index = 0; + os_mutex_unlock(&info->key_data_list_lock); + + /* reset key value */ + value_node->thread_key_values[i] = 0; + + /* Call the destructor func provided by app */ + if (destructor_index) { + uint32 argv[1]; + + argv[0] = value; + wasm_runtime_call_indirect(exec_env, destructor_index, 1, argv); + } + } + } + + bh_list_remove(info->thread_list, value_node); + wasm_runtime_free(value_node); +} + +static void +destroy_thread_key_value_list(bh_list *list) +{ + ThreadKeyValueNode *node, *next; + + /* There should be only one node for main thread */ + bh_assert(list->len <= 1); + + if (list->len) { + node = bh_list_first_elem(list); + while (node) { + next = bh_list_elem_next(node); + call_key_destructor(node->exec_env); + node = next; + } + } +} + +static ClusterInfoNode * +create_cluster_info(WASMCluster *cluster) +{ + ClusterInfoNode *node; + bh_list_status ret; + + if (!(node = wasm_runtime_malloc(sizeof(ClusterInfoNode)))) { + return NULL; + } + memset(node, 0, sizeof(ClusterInfoNode)); + + node->thread_list = &node->thread_list_head; + ret = bh_list_init(node->thread_list); + bh_assert(ret == BH_LIST_SUCCESS); + + if (os_mutex_init(&node->key_data_list_lock) != 0) { + wasm_runtime_free(node); + return NULL; + } + + node->cluster = cluster; + if (!(node->thread_info_map = bh_hash_map_create( + 32, true, (HashFunc)thread_handle_hash, + (KeyEqualFunc)thread_handle_equal, NULL, thread_info_destroy))) { + os_mutex_destroy(&node->key_data_list_lock); + wasm_runtime_free(node); + return NULL; + } + os_mutex_lock(&thread_global_lock); + ret = bh_list_insert(&cluster_info_list, node); + bh_assert(ret == BH_LIST_SUCCESS); + os_mutex_unlock(&thread_global_lock); + + (void)ret; + return node; +} + +static bool +destroy_cluster_info(WASMCluster *cluster) +{ + ClusterInfoNode *node = get_cluster_info(cluster); + if (node) { + bh_hash_map_destroy(node->thread_info_map); + destroy_thread_key_value_list(node->thread_list); + os_mutex_destroy(&node->key_data_list_lock); + + /* Remove from the cluster info list */ + os_mutex_lock(&thread_global_lock); + bh_list_remove(&cluster_info_list, node); + wasm_runtime_free(node); + os_mutex_unlock(&thread_global_lock); + return true; + } + return false; +} + +static void +lib_pthread_destroy_callback(WASMCluster *cluster) +{ + destroy_cluster_info(cluster); +} + +static void +delete_thread_info_node(ThreadInfoNode *thread_info) +{ + ClusterInfoNode *node; + bool ret; + WASMCluster *cluster = wasm_exec_env_get_cluster(thread_info->exec_env); + + if ((node = get_cluster_info(cluster))) { + ret = bh_hash_map_remove(node->thread_info_map, + (void *)(uintptr_t)thread_info->handle, NULL, + NULL); + (void)ret; + } + + thread_info_destroy(thread_info); +} + +static bool +append_thread_info_node(ThreadInfoNode *thread_info) +{ + ClusterInfoNode *node; + WASMCluster *cluster = wasm_exec_env_get_cluster(thread_info->exec_env); + + if (!(node = get_cluster_info(cluster))) { + if (!(node = create_cluster_info(cluster))) { + return false; + } + } + + if (!bh_hash_map_insert(node->thread_info_map, + (void *)(uintptr_t)thread_info->handle, + thread_info)) { + return false; + } + + return true; +} + +static ThreadInfoNode * +get_thread_info(wasm_exec_env_t exec_env, uint32 handle) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + ClusterInfoNode *info = get_cluster_info(cluster); + + if (!info || !handle) { + return NULL; + } + + return bh_hash_map_find(info->thread_info_map, (void *)(uintptr_t)handle); +} + +static uint32 +allocate_handle() +{ + uint32 id; + os_mutex_lock(&thread_global_lock); + id = handle_id++; + os_mutex_unlock(&thread_global_lock); + return id; +} + +static void * +pthread_start_routine(void *arg) +{ + wasm_exec_env_t exec_env = (wasm_exec_env_t)arg; + wasm_exec_env_t parent_exec_env; + ThreadRoutineArgs *routine_args = exec_env->thread_arg; + ThreadInfoNode *info_node = routine_args->info_node; + uint32 argv[1]; + + parent_exec_env = info_node->parent_exec_env; + os_mutex_lock(&parent_exec_env->wait_lock); + info_node->exec_env = exec_env; + info_node->u.thread = exec_env->handle; + if (!append_thread_info_node(info_node)) { + delete_thread_info_node(info_node); + os_cond_signal(&parent_exec_env->wait_cond); + os_mutex_unlock(&parent_exec_env->wait_lock); + return NULL; + } + + info_node->status = THREAD_RUNNING; + os_cond_signal(&parent_exec_env->wait_cond); + os_mutex_unlock(&parent_exec_env->wait_lock); + + wasm_exec_env_set_thread_info(exec_env); + argv[0] = routine_args->arg; + + if (!wasm_runtime_call_indirect(exec_env, routine_args->elem_index, 1, + argv)) { + /* Exception has already been spread during throwing */ + } + + /* destroy pthread key values */ + call_key_destructor(exec_env); + + wasm_runtime_free(routine_args); + + /* if the thread is joinable, store the result in its info node, + if the other threads join this thread after exited, then we + can return the stored result */ + if (!info_node->joinable) { + delete_thread_info_node(info_node); + } + else { + info_node->u.ret = (void *)(uintptr_t)argv[0]; +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) + & WASM_SUSPEND_FLAG_EXIT) + /* argv[0] isn't set after longjmp(1) to + invoke_native_with_hw_bound_check */ + info_node->u.ret = exec_env->thread_ret_value; +#endif + /* Update node status after ret value was set */ + info_node->status = THREAD_EXIT; + } + + return (void *)(uintptr_t)argv[0]; +} + +static int +pthread_create_wrapper(wasm_exec_env_t exec_env, + uint32 *thread, /* thread_handle */ + const void *attr, /* not supported */ + uint32 elem_index, /* entry function */ + uint32 arg) /* arguments buffer */ +{ + wasm_module_t module = get_module(exec_env); + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasm_module_inst_t new_module_inst = NULL; + ThreadInfoNode *info_node = NULL; + ThreadRoutineArgs *routine_args = NULL; + uint32 thread_handle; + uint32 stack_size = 8192; + uint32 aux_stack_size; + uint64 aux_stack_start = 0; + int32 ret = -1; + + bh_assert(module); + bh_assert(module_inst); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + stack_size = + ((WASMModuleInstance *)module_inst)->default_wasm_stack_size; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + stack_size = + ((AOTModuleInstance *)module_inst)->default_wasm_stack_size; + } +#endif + + if (!(new_module_inst = wasm_runtime_instantiate_internal( + module, module_inst, exec_env, stack_size, 0, 0, NULL, 0))) + return -1; + + /* Set custom_data to new module instance */ + wasm_runtime_set_custom_data_internal( + new_module_inst, wasm_runtime_get_custom_data(module_inst)); + + wasm_native_inherit_contexts(new_module_inst, module_inst); + + if (!(wasm_cluster_dup_c_api_imports(new_module_inst, module_inst))) + goto fail; + + if (!(info_node = wasm_runtime_malloc(sizeof(ThreadInfoNode)))) + goto fail; + + memset(info_node, 0, sizeof(ThreadInfoNode)); + thread_handle = allocate_handle(); + info_node->parent_exec_env = exec_env; + info_node->handle = thread_handle; + info_node->type = T_THREAD; + info_node->status = THREAD_INIT; + info_node->joinable = true; + + if (!(routine_args = wasm_runtime_malloc(sizeof(ThreadRoutineArgs)))) + goto fail; + + routine_args->arg = arg; + routine_args->elem_index = elem_index; + routine_args->info_node = info_node; + routine_args->module_inst = new_module_inst; + + /* Allocate aux stack previously since exec_env->wait_lock is acquired + below, and if the stack is allocated in wasm_cluster_create_thread, + runtime may call the exported malloc function to allocate the stack, + which acquires exec_env->wait again in wasm_exec_env_set_thread_info, + and recursive lock (or hang) occurs */ + if (!wasm_cluster_allocate_aux_stack(exec_env, &aux_stack_start, + &aux_stack_size)) { + LOG_ERROR("thread manager error: " + "failed to allocate aux stack space for new thread"); + goto fail; + } + + os_mutex_lock(&exec_env->wait_lock); + ret = wasm_cluster_create_thread( + exec_env, new_module_inst, true, aux_stack_start, aux_stack_size, + pthread_start_routine, (void *)routine_args); + if (ret != 0) { + os_mutex_unlock(&exec_env->wait_lock); + goto fail; + } + + /* Wait for the thread routine to assign the exec_env to + thread_info_node, otherwise the exec_env in the thread + info node may be NULL in the next pthread API call */ + os_cond_wait(&exec_env->wait_cond, &exec_env->wait_lock); + os_mutex_unlock(&exec_env->wait_lock); + + if (thread) + *thread = thread_handle; + + return 0; + +fail: + if (new_module_inst) + wasm_runtime_deinstantiate_internal(new_module_inst, true); + if (info_node) + wasm_runtime_free(info_node); + if (routine_args) + wasm_runtime_free(routine_args); + if (aux_stack_start) + wasm_cluster_free_aux_stack(exec_env, aux_stack_start); + return ret; +} + +static int32 +pthread_join_wrapper(wasm_exec_env_t exec_env, uint32 thread, + int32 retval_offset) /* void **retval */ +{ + uint32 *ret; + int32 join_ret; + void **retval; + ThreadInfoNode *node; + wasm_module_inst_t module_inst; + wasm_exec_env_t target_exec_env; + + module_inst = get_module_inst(exec_env); + + /* validate addr, we can use current thread's + module instance here as the memory is shared */ + if (!validate_app_addr((uint64)retval_offset, (uint64)sizeof(int32))) { + /* Join failed, but we don't want to terminate all threads, + do not spread exception here */ + wasm_runtime_set_exception(module_inst, NULL); + return -1; + } + + retval = (void **)addr_app_to_native((uint64)retval_offset); + + node = get_thread_info(exec_env, thread); + if (!node) { + /* The thread has exited and not joinable, return 0 to app */ + return 0; + } + + target_exec_env = node->exec_env; + bh_assert(target_exec_env); + + if (node->status != THREAD_EXIT) { + /* if the thread is still running, call the platforms join API */ + join_ret = wasm_cluster_join_thread(target_exec_env, (void **)&ret); + } + else { + /* if the thread has exited, return stored results */ + + /* this thread must be joinable, otherwise the + info_node should be destroyed once exit */ + bh_assert(node->joinable); + join_ret = 0; + ret = node->u.ret; + + /* The target thread changes the node's status before calling + wasm_cluster_exit_thread to exit, so here its resources may + haven't been destroyed yet, we wait enough time to ensure that + they are actually destroyed to avoid unexpected behavior. */ + os_mutex_lock(&exec_env->wait_lock); + os_cond_reltimedwait(&exec_env->wait_cond, &exec_env->wait_lock, 1000); + os_mutex_unlock(&exec_env->wait_lock); + } + + if (retval_offset != 0) + *(uint32 *)retval = (uint32)(uintptr_t)ret; + + return join_ret; +} + +static int32 +pthread_detach_wrapper(wasm_exec_env_t exec_env, uint32 thread) +{ + ThreadInfoNode *node; + wasm_exec_env_t target_exec_env; + + node = get_thread_info(exec_env, thread); + if (!node) + return 0; + + node->joinable = false; + + target_exec_env = node->exec_env; + bh_assert(target_exec_env != NULL); + + return wasm_cluster_detach_thread(target_exec_env); +} + +static int32 +pthread_cancel_wrapper(wasm_exec_env_t exec_env, uint32 thread) +{ + ThreadInfoNode *node; + wasm_exec_env_t target_exec_env; + + node = get_thread_info(exec_env, thread); + if (!node) + return 0; + + node->status = THREAD_CANCELLED; + node->joinable = false; + + target_exec_env = node->exec_env; + bh_assert(target_exec_env != NULL); + + return wasm_cluster_cancel_thread(target_exec_env); +} + +static int32 +pthread_self_wrapper(wasm_exec_env_t exec_env) +{ + ThreadRoutineArgs *args = get_thread_arg(exec_env); + /* If thread_arg is NULL, it's the exec_env of the main thread, + return id 0 to app */ + if (!args) + return 0; + + return args->info_node->handle; +} + +/* emcc use __pthread_self rather than pthread_self */ +static int32 +__pthread_self_wrapper(wasm_exec_env_t exec_env) +{ + return pthread_self_wrapper(exec_env); +} + +static void +pthread_exit_wrapper(wasm_exec_env_t exec_env, int32 retval_offset) +{ + ThreadRoutineArgs *args = get_thread_arg(exec_env); + /* Currently exit main thread is not allowed */ + if (!args) + return; + +#if defined(OS_ENABLE_HW_BOUND_CHECK) && !defined(BH_PLATFORM_WINDOWS) + /* If hardware bound check enabled, don't deinstantiate module inst + and thread info node here for AoT module, as they will be freed + in pthread_start_routine */ + if (exec_env->jmpbuf_stack_top) { + wasm_cluster_exit_thread(exec_env, (void *)(uintptr_t)retval_offset); + } +#endif + + /* destroy pthread key values */ + call_key_destructor(exec_env); + + if (!args->info_node->joinable) { + delete_thread_info_node(args->info_node); + } + else { + args->info_node->u.ret = (void *)(uintptr_t)retval_offset; + /* Update node status after ret value was set */ + args->info_node->status = THREAD_EXIT; + } + + wasm_runtime_free(args); + + /* Don't destroy exec_env->module_inst in this functuntion since + it will be destroyed in wasm_cluster_exit_thread */ + wasm_cluster_exit_thread(exec_env, (void *)(uintptr_t)retval_offset); +} + +static int32 +pthread_mutex_init_wrapper(wasm_exec_env_t exec_env, uint32 *mutex, void *attr) +{ + korp_mutex *pmutex; + ThreadInfoNode *info_node; + + if (!(pmutex = wasm_runtime_malloc(sizeof(korp_mutex)))) { + return -1; + } + + if (os_mutex_init(pmutex) != 0) { + goto fail1; + } + + if (!(info_node = wasm_runtime_malloc(sizeof(ThreadInfoNode)))) + goto fail2; + + memset(info_node, 0, sizeof(ThreadInfoNode)); + info_node->exec_env = exec_env; + info_node->handle = allocate_handle(); + info_node->type = T_MUTEX; + info_node->u.mutex = pmutex; + info_node->status = MUTEX_CREATED; + + if (!append_thread_info_node(info_node)) + goto fail3; + + /* Return the mutex handle to app */ + if (mutex) + *(uint32 *)mutex = info_node->handle; + + return 0; + +fail3: + delete_thread_info_node(info_node); +fail2: + os_mutex_destroy(pmutex); +fail1: + wasm_runtime_free(pmutex); + + return -1; +} + +static int32 +pthread_mutex_lock_wrapper(wasm_exec_env_t exec_env, uint32 *mutex) +{ + ThreadInfoNode *info_node = get_thread_info(exec_env, *mutex); + if (!info_node || info_node->type != T_MUTEX) + return -1; + + return os_mutex_lock(info_node->u.mutex); +} + +static int32 +pthread_mutex_unlock_wrapper(wasm_exec_env_t exec_env, uint32 *mutex) +{ + ThreadInfoNode *info_node = get_thread_info(exec_env, *mutex); + if (!info_node || info_node->type != T_MUTEX) + return -1; + + return os_mutex_unlock(info_node->u.mutex); +} + +static int32 +pthread_mutex_destroy_wrapper(wasm_exec_env_t exec_env, uint32 *mutex) +{ + int32 ret_val; + ThreadInfoNode *info_node = get_thread_info(exec_env, *mutex); + if (!info_node || info_node->type != T_MUTEX) + return -1; + + ret_val = os_mutex_destroy(info_node->u.mutex); + + info_node->status = MUTEX_DESTROYED; + delete_thread_info_node(info_node); + + return ret_val; +} + +static int32 +pthread_cond_init_wrapper(wasm_exec_env_t exec_env, uint32 *cond, void *attr) +{ + korp_cond *pcond; + ThreadInfoNode *info_node; + + if (!(pcond = wasm_runtime_malloc(sizeof(korp_cond)))) { + return -1; + } + + if (os_cond_init(pcond) != 0) { + goto fail1; + } + + if (!(info_node = wasm_runtime_malloc(sizeof(ThreadInfoNode)))) + goto fail2; + + memset(info_node, 0, sizeof(ThreadInfoNode)); + info_node->exec_env = exec_env; + info_node->handle = allocate_handle(); + info_node->type = T_COND; + info_node->u.cond = pcond; + info_node->status = COND_CREATED; + + if (!append_thread_info_node(info_node)) + goto fail3; + + /* Return the cond handle to app */ + if (cond) + *(uint32 *)cond = info_node->handle; + + return 0; + +fail3: + delete_thread_info_node(info_node); +fail2: + os_cond_destroy(pcond); +fail1: + wasm_runtime_free(pcond); + + return -1; +} + +static int32 +pthread_cond_wait_wrapper(wasm_exec_env_t exec_env, uint32 *cond, uint32 *mutex) +{ + ThreadInfoNode *cond_info_node, *mutex_info_node; + + cond_info_node = get_thread_info(exec_env, *cond); + if (!cond_info_node || cond_info_node->type != T_COND) + return -1; + + mutex_info_node = get_thread_info(exec_env, *mutex); + if (!mutex_info_node || mutex_info_node->type != T_MUTEX) + return -1; + + return os_cond_wait(cond_info_node->u.cond, mutex_info_node->u.mutex); +} + +/** + * Currently we don't support struct timespec in built-in libc, + * so the pthread_cond_timedwait use useconds instead + */ +static int32 +pthread_cond_timedwait_wrapper(wasm_exec_env_t exec_env, uint32 *cond, + uint32 *mutex, uint64 useconds) +{ + ThreadInfoNode *cond_info_node, *mutex_info_node; + + cond_info_node = get_thread_info(exec_env, *cond); + if (!cond_info_node || cond_info_node->type != T_COND) + return -1; + + mutex_info_node = get_thread_info(exec_env, *mutex); + if (!mutex_info_node || mutex_info_node->type != T_MUTEX) + return -1; + + return os_cond_reltimedwait(cond_info_node->u.cond, + mutex_info_node->u.mutex, useconds); +} + +static int32 +pthread_cond_signal_wrapper(wasm_exec_env_t exec_env, uint32 *cond) +{ + ThreadInfoNode *info_node = get_thread_info(exec_env, *cond); + if (!info_node || info_node->type != T_COND) + return -1; + + return os_cond_signal(info_node->u.cond); +} + +static int32 +pthread_cond_broadcast_wrapper(wasm_exec_env_t exec_env, uint32 *cond) +{ + ThreadInfoNode *info_node = get_thread_info(exec_env, *cond); + if (!info_node || info_node->type != T_COND) + return -1; + + return os_cond_broadcast(info_node->u.cond); +} + +static int32 +pthread_cond_destroy_wrapper(wasm_exec_env_t exec_env, uint32 *cond) +{ + int32 ret_val; + ThreadInfoNode *info_node = get_thread_info(exec_env, *cond); + if (!info_node || info_node->type != T_COND) + return -1; + + ret_val = os_cond_destroy(info_node->u.cond); + + info_node->status = COND_DESTROYED; + delete_thread_info_node(info_node); + + return ret_val; +} + +static int32 +pthread_key_create_wrapper(wasm_exec_env_t exec_env, int32 *key, + int32 destructor_elem_index) +{ + uint32 i; + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + ClusterInfoNode *info = get_cluster_info(cluster); + + if (!info) { + /* The user may call pthread_key_create in main thread, + in this case the cluster info hasn't been created */ + if (!(info = create_cluster_info(cluster))) { + return -1; + } + } + + os_mutex_lock(&info->key_data_list_lock); + for (i = 0; i < WAMR_PTHREAD_KEYS_MAX; i++) { + if (!info->key_data_list[i].is_created) { + break; + } + } + + if (i == WAMR_PTHREAD_KEYS_MAX) { + os_mutex_unlock(&info->key_data_list_lock); + return -1; + } + + info->key_data_list[i].destructor_func = destructor_elem_index; + info->key_data_list[i].is_created = true; + *key = i; + os_mutex_unlock(&info->key_data_list_lock); + + return 0; +} + +static int32 +pthread_setspecific_wrapper(wasm_exec_env_t exec_env, int32 key, + int32 value_offset) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + ClusterInfoNode *info = get_cluster_info(cluster); + int32 *key_values; + + if (!info) + return -1; + + os_mutex_lock(&info->key_data_list_lock); + + key_values = key_value_list_lookup_or_create(exec_env, info, key); + if (!key_values) { + os_mutex_unlock(&info->key_data_list_lock); + return -1; + } + + key_values[key] = value_offset; + os_mutex_unlock(&info->key_data_list_lock); + + return 0; +} + +static int32 +pthread_getspecific_wrapper(wasm_exec_env_t exec_env, int32 key) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + ClusterInfoNode *info = get_cluster_info(cluster); + int32 ret, *key_values; + + if (!info) + return 0; + + os_mutex_lock(&info->key_data_list_lock); + + key_values = key_value_list_lookup_or_create(exec_env, info, key); + if (!key_values) { + os_mutex_unlock(&info->key_data_list_lock); + return 0; + } + + ret = key_values[key]; + os_mutex_unlock(&info->key_data_list_lock); + + return ret; +} + +static int32 +pthread_key_delete_wrapper(wasm_exec_env_t exec_env, int32 key) +{ + KeyData *data; + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + ClusterInfoNode *info = get_cluster_info(cluster); + + if (!info) + return -1; + + os_mutex_lock(&info->key_data_list_lock); + data = key_data_list_lookup(exec_env, key); + if (!data) { + os_mutex_unlock(&info->key_data_list_lock); + return -1; + } + + memset(data, 0, sizeof(KeyData)); + os_mutex_unlock(&info->key_data_list_lock); + + return 0; +} + +/** + * Currently the memory allocator doesn't support alloc specific aligned + * space, we wrap posix_memalign to simply malloc memory + */ +static int32 +posix_memalign_wrapper(wasm_exec_env_t exec_env, void **memptr, int32 align, + int32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + void *p = NULL; + + /* TODO: for memory 64, module_malloc may return uint64 offset */ + *((uint32 *)memptr) = (uint32)module_malloc(size, (void **)&p); + if (!p) + return -1; + + return 0; +} + +#if WASM_ENABLE_LIB_PTHREAD_SEMAPHORE != 0 + +static int32 +sem_open_wrapper(wasm_exec_env_t exec_env, const char *name, int32 oflags, + int32 mode, int32 val) +{ + korp_sem *psem = NULL; + ThreadInfoNode *info_node = NULL; + + /** + * For RTOS, global semaphore map is safe for share the same semaphore + * between task/pthread. + * For Unix like system, it's dedicated for multiple processes. + */ + + if (!name) { /* avoid passing NULL to bh_hash_map_find and os_sem_open */ + return -1; + } + + if ((info_node = bh_hash_map_find(sem_info_map, (void *)name))) { + return info_node->handle; + } + + if (!(psem = os_sem_open(name, oflags, mode, val))) { + goto fail1; + } + + if (!(info_node = wasm_runtime_malloc(sizeof(ThreadInfoNode)))) + goto fail2; + + memset(info_node, 0, sizeof(ThreadInfoNode)); + info_node->exec_env = exec_env; + info_node->handle = allocate_handle(); + info_node->type = T_SEM; + info_node->u.sem = psem; + info_node->status = SEM_CREATED; + + if (!bh_hash_map_insert(sem_info_map, (void *)name, info_node)) + goto fail3; + + return info_node->handle; + +fail3: + wasm_runtime_free(info_node); +fail2: + os_sem_close(psem); +fail1: + return -1; +} + +void +sem_fetch_cb(void *key, void *value, void *user_data) +{ + (void)key; + SemCallbackArgs *args = user_data; + ThreadInfoNode *info_node = value; + if (args->handle == info_node->handle && info_node->status == SEM_CREATED) { + args->node = info_node; + } +} + +static int32 +sem_close_wrapper(wasm_exec_env_t exec_env, uint32 sem) +{ + (void)exec_env; + int ret = -1; + SemCallbackArgs args = { sem, NULL }; + + bh_hash_map_traverse(sem_info_map, sem_fetch_cb, &args); + + if (args.node) { + ret = os_sem_close(args.node->u.sem); + if (ret == 0) { + args.node->status = SEM_CLOSED; + } + } + + return ret; +} + +static int32 +sem_wait_wrapper(wasm_exec_env_t exec_env, uint32 sem) +{ + (void)exec_env; + SemCallbackArgs args = { sem, NULL }; + + bh_hash_map_traverse(sem_info_map, sem_fetch_cb, &args); + + if (args.node) { + return os_sem_wait(args.node->u.sem); + } + + return -1; +} + +static int32 +sem_trywait_wrapper(wasm_exec_env_t exec_env, uint32 sem) +{ + (void)exec_env; + SemCallbackArgs args = { sem, NULL }; + + bh_hash_map_traverse(sem_info_map, sem_fetch_cb, &args); + + if (args.node) { + return os_sem_trywait(args.node->u.sem); + } + + return -1; +} + +static int32 +sem_post_wrapper(wasm_exec_env_t exec_env, uint32 sem) +{ + (void)exec_env; + SemCallbackArgs args = { sem, NULL }; + + bh_hash_map_traverse(sem_info_map, sem_fetch_cb, &args); + + if (args.node) { + return os_sem_post(args.node->u.sem); + } + + return -1; +} + +static int32 +sem_getvalue_wrapper(wasm_exec_env_t exec_env, uint32 sem, int32 *sval) +{ + int32 ret = -1; + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + (void)exec_env; + SemCallbackArgs args = { sem, NULL }; + + if (validate_native_addr(sval, (uint64)sizeof(int32))) { + + bh_hash_map_traverse(sem_info_map, sem_fetch_cb, &args); + + if (args.node) { + ret = os_sem_getvalue(args.node->u.sem, sval); + } + } + return ret; +} + +static int32 +sem_unlink_wrapper(wasm_exec_env_t exec_env, const char *name) +{ + (void)exec_env; + int32 ret_val; + + ThreadInfoNode *info_node; + + if (!name) { /* avoid passing NULL to bh_hash_map_find */ + return -1; + } + + info_node = bh_hash_map_find(sem_info_map, (void *)name); + if (!info_node || info_node->type != T_SEM) + return -1; + + if (info_node->status != SEM_CLOSED) { + ret_val = os_sem_close(info_node->u.sem); + if (ret_val != 0) { + return ret_val; + } + } + + ret_val = os_sem_unlink(name); + + if (ret_val == 0) { + bh_hash_map_remove(sem_info_map, (void *)name, NULL, NULL); + info_node->status = SEM_DESTROYED; + thread_info_destroy(info_node); + } + return ret_val; +} + +#endif + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_lib_pthread[] = { + REG_NATIVE_FUNC(pthread_create, "(**ii)i"), + REG_NATIVE_FUNC(pthread_join, "(ii)i"), + REG_NATIVE_FUNC(pthread_detach, "(i)i"), + REG_NATIVE_FUNC(pthread_cancel, "(i)i"), + REG_NATIVE_FUNC(pthread_self, "()i"), + REG_NATIVE_FUNC(__pthread_self, "()i"), + REG_NATIVE_FUNC(pthread_exit, "(i)"), + REG_NATIVE_FUNC(pthread_mutex_init, "(**)i"), + REG_NATIVE_FUNC(pthread_mutex_lock, "(*)i"), + REG_NATIVE_FUNC(pthread_mutex_unlock, "(*)i"), + REG_NATIVE_FUNC(pthread_mutex_destroy, "(*)i"), + REG_NATIVE_FUNC(pthread_cond_init, "(**)i"), + REG_NATIVE_FUNC(pthread_cond_wait, "(**)i"), + REG_NATIVE_FUNC(pthread_cond_timedwait, "(**I)i"), + REG_NATIVE_FUNC(pthread_cond_signal, "(*)i"), + REG_NATIVE_FUNC(pthread_cond_broadcast, "(*)i"), + REG_NATIVE_FUNC(pthread_cond_destroy, "(*)i"), + REG_NATIVE_FUNC(pthread_key_create, "(*i)i"), + REG_NATIVE_FUNC(pthread_setspecific, "(ii)i"), + REG_NATIVE_FUNC(pthread_getspecific, "(i)i"), + REG_NATIVE_FUNC(pthread_key_delete, "(i)i"), + REG_NATIVE_FUNC(posix_memalign, "(*ii)i"), +#if WASM_ENABLE_LIB_PTHREAD_SEMAPHORE != 0 + REG_NATIVE_FUNC(sem_open, "($iii)i"), + REG_NATIVE_FUNC(sem_close, "(i)i"), + REG_NATIVE_FUNC(sem_wait, "(i)i"), + REG_NATIVE_FUNC(sem_trywait, "(i)i"), + REG_NATIVE_FUNC(sem_post, "(i)i"), + REG_NATIVE_FUNC(sem_getvalue, "(i*)i"), + REG_NATIVE_FUNC(sem_unlink, "($)i"), +#endif +}; + +uint32 +get_lib_pthread_export_apis(NativeSymbol **p_lib_pthread_apis) +{ + *p_lib_pthread_apis = native_symbols_lib_pthread; + return sizeof(native_symbols_lib_pthread) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats.cmake b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats.cmake new file mode 100644 index 0000000..e17a6b2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats.cmake @@ -0,0 +1,53 @@ +# Copyright (c) 2022 Intel Corporation +# Copyright (c) 2020-2021 Alibaba Cloud +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIB_RATS_DIR ${CMAKE_CURRENT_LIST_DIR}) + +if ("$ENV{SGX_SSL_DIR}" STREQUAL "") + set (SGX_SSL_DIR "/opt/intel/sgxssl") +else() + set (SGX_SSL_DIR $ENV{SGX_SSL_DIR}) +endif() + +if (NOT EXISTS ${SGX_SSL_DIR}) + message(FATAL_ERROR "Can not find SGX_SSL, please install it first") +endif() + +add_definitions (-DWASM_ENABLE_LIB_RATS=1) + +include_directories(${LIB_RATS_DIR} ${SGX_SSL_DIR}/include) + +include(FetchContent) + +set(RATS_BUILD_MODE "sgx" + CACHE INTERNAL "Select build mode for librats(host|occlum|sgx｜wasm)") +set(RATS_INSTALL_PATH "${CMAKE_BINARY_DIR}/librats" CACHE INTERNAL "") +set(BUILD_SAMPLES OFF CACHE BOOL "Disable de compilation of the librats samples" FORCE) + +FetchContent_Declare( + librats + GIT_REPOSITORY https://github.com/inclavare-containers/librats + GIT_TAG master +) +FetchContent_GetProperties(librats) +if (NOT librats_POPULATED) + message("-- Fetching librats ..") + FetchContent_Populate(librats) + include_directories("${librats_SOURCE_DIR}/include") + + # Prevent the propagation of the CMAKE_C_FLAGS of WAMR into librats + set(SAVED_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + set(CMAKE_C_FLAGS "") + + # Import the building scripts of librats + add_subdirectory(${librats_SOURCE_DIR} ${librats_BINARY_DIR} EXCLUDE_FROM_ALL) + + # Restore the CMAKE_C_FLAGS of WAMR + set(CMAKE_C_FLAGS ${SAVED_CMAKE_C_FLAGS}) + +endif() + +file (GLOB source_all ${LIB_RATS_DIR}/*.c) + +set (LIB_RATS_SOURCE ${source_all}) \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_common.h b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_common.h new file mode 100644 index 0000000..434c2ab --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_common.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 Intel Corporation + * Copyright (c) 2020-2021 Alibaba Cloud + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _RATS_WAMR_COMMON_H +#define _RATS_WAMR_COMMON_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Enclave Flags Bit Masks */ +/* If set, then the enclave is initialized */ +#define SGX_FLAGS_INITTED 0x001ULL +/* If set, then the enclave is debug */ +#define SGX_FLAGS_DEBUG 0x002ULL +/* If set, then the enclave is 64 bit */ +#define SGX_FLAGS_MODE64BIT 0x004ULL +/* If set, then the enclave has access to provision key */ +#define SGX_FLAGS_PROVISION_KEY 0x010ULL +/* If set, then the enclave has access to EINITTOKEN key */ +#define SGX_FLAGS_EINITTOKEN_KEY 0x020ULL +/* If set, then the enclave uses KSS */ +#define SGX_FLAGS_KSS 0x080ULL +/* If set, then the enclave enables AEX Notify */ +#define SGX_FLAGS_AEX_NOTIFY 0x400ULL + +#define SGX_QUOTE_MAX_SIZE 8192 +#define SGX_USER_DATA_SIZE 64 +#define SGX_MEASUREMENT_SIZE 32 + +/* clang-format off */ +typedef struct rats_sgx_evidence { + uint8_t quote[SGX_QUOTE_MAX_SIZE]; /* The quote of the Enclave */ + uint32_t quote_size; /* The size of the quote */ + uint8_t user_data[SGX_USER_DATA_SIZE]; /* The custom data in the quote */ + uint32_t product_id; /* Product ID of the Enclave */ + uint8_t mr_enclave[SGX_MEASUREMENT_SIZE]; /* The MRENCLAVE of the Enclave */ + uint32_t security_version; /* Security Version of the Enclave */ + uint8_t mr_signer[SGX_MEASUREMENT_SIZE]; /* The MRSIGNER of the Enclave */ + uint64_t att_flags; /* Flags of the Enclave in attributes */ + uint64_t att_xfrm; /* XSAVE Feature Request Mask */ +} rats_sgx_evidence_t; +/* clang-format on */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_wrapper.c b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_wrapper.c new file mode 100644 index 0000000..bdacc25 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_wrapper.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2022 Intel Corporation + * Copyright (c) 2020-2021 Alibaba Cloud + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +#include "sgx_quote_3.h" +#include "wasm_export.h" +#include "bh_common.h" +#include "lib_rats_common.h" + +static int +librats_collect_wrapper(wasm_exec_env_t exec_env, uint32_t *evidence_json, + const char *buffer, uint32_t buffer_size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasm_module_t module = wasm_runtime_get_module(module_inst); + char *wasm_module_hash = wasm_runtime_get_module_hash(module); + + char *json, *str_ret; + uint32_t str_ret_offset; + uint8_t final_hash[SHA256_DIGEST_LENGTH]; + + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, wasm_module_hash, SHA256_DIGEST_LENGTH); + if (buffer != NULL) + SHA256_Update(&sha256, buffer, buffer_size); + SHA256_Final(final_hash, &sha256); + + int ret_code = librats_collect_evidence_to_json(final_hash, &json); + if (ret_code != 0) { + return ret_code; + } + + uint32_t json_size = strlen(json) + 1; + str_ret_offset = module_malloc(json_size, (void **)&str_ret); + if (!str_ret_offset) { + free(json); + return (int)RATS_ATTESTER_ERR_NO_MEM; + } + bh_memcpy_s(str_ret, json_size, json, json_size); + *evidence_json = str_ret_offset; + free(json); + + return 0; +} + +static int +librats_verify_wrapper(wasm_exec_env_t exec_env, const char *evidence_json, + uint32_t evidence_size, const uint8_t *hash, + uint32_t hash_size) +{ + return librats_verify_evidence_from_json(evidence_json, hash); +} + +static int +librats_parse_evidence_wrapper(wasm_exec_env_t exec_env, + const char *evidence_json, uint32_t json_size, + rats_sgx_evidence_t *evidence, + uint32_t evidence_size) +{ + attestation_evidence_t att_ev; + + if (get_evidence_from_json(evidence_json, &att_ev) != 0) { + return -1; + } + + // Only supports parsing sgx evidence currently + if (strcmp(att_ev.type, "sgx_ecdsa") != 0) { + return -1; + } + + sgx_quote3_t *quote_ptr = (sgx_quote3_t *)att_ev.ecdsa.quote; + bh_memcpy_s(evidence->quote, att_ev.ecdsa.quote_len, att_ev.ecdsa.quote, + att_ev.ecdsa.quote_len); + evidence->quote_size = att_ev.ecdsa.quote_len; + bh_memcpy_s(evidence->user_data, SGX_REPORT_DATA_SIZE, + quote_ptr->report_body.report_data.d, SGX_REPORT_DATA_SIZE); + bh_memcpy_s(evidence->mr_enclave, sizeof(sgx_measurement_t), + quote_ptr->report_body.mr_enclave.m, sizeof(sgx_measurement_t)); + bh_memcpy_s(evidence->mr_signer, sizeof(sgx_measurement_t), + quote_ptr->report_body.mr_signer.m, sizeof(sgx_measurement_t)); + evidence->product_id = quote_ptr->report_body.isv_prod_id; + evidence->security_version = quote_ptr->report_body.isv_svn; + evidence->att_flags = quote_ptr->report_body.attributes.flags; + evidence->att_xfrm = quote_ptr->report_body.attributes.flags; + + return 0; +} + +static void +librats_dispose_evidence_json_wrapper(wasm_exec_env_t exec_env, + uint32_t evidence_json) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + module_free(evidence_json); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_lib_rats[] = { + REG_NATIVE_FUNC(librats_collect, "(**~)i"), + REG_NATIVE_FUNC(librats_verify, "(*~*~)i"), + REG_NATIVE_FUNC(librats_parse_evidence, "(*~*~)i"), + REG_NATIVE_FUNC(librats_dispose_evidence_json, "(i)") +}; + +uint32_t +get_lib_rats_export_apis(NativeSymbol **p_lib_rats_apis) +{ + *p_lib_rats_apis = native_symbols_lib_rats; + return sizeof(native_symbols_lib_rats) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_wrapper.h b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_wrapper.h new file mode 100644 index 0000000..9286451 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-rats/lib_rats_wrapper.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 Intel Corporation + * Copyright (c) 2020-2021 Alibaba Cloud + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _RATS_WAMR_API_H +#define _RATS_WAMR_API_H + +#include +#include + +#include "lib_rats_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int +librats_collect(char **evidence_json, const char *buffer, uint32_t buffer_size); + +int +librats_verify(const char *evidence_json, uint32_t evidence_size, + const uint8_t *hash, uint32_t hash_size); + +int +librats_parse_evidence(const char *evidence_json, uint32_t json_size, + rats_sgx_evidence_t *evidence, uint32_t evidence_size); + +#define librats_collect(evidence_json, buffer) \ + librats_collect(evidence_json, buffer, buffer ? strlen(buffer) + 1 : 0) + +#define librats_verify(evidence_json, hash) \ + librats_verify(evidence_json, \ + evidence_json ? strlen(evidence_json) + 1 : 0, hash, \ + hash ? strlen((const char *)hash) + 1 : 0) + +#define librats_parse_evidence(evidence_json, evidence) \ + librats_parse_evidence(evidence_json, \ + evidence_json ? strlen(evidence_json) + 1 : 0, \ + evidence, sizeof(rats_sgx_evidence_t)) + +void +librats_dispose_evidence_json(char *evidence_json); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h new file mode 100644 index 0000000..98429bb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h @@ -0,0 +1,1001 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASI_SOCKET_EXT_H_ +#define _WASI_SOCKET_EXT_H_ + +#include +#include +#include + +/*Be a part of */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + /* Used only for sock_addr_resolve hints */ + SOCKET_ANY = -1, + SOCKET_DGRAM = 0, + SOCKET_STREAM, +} __wasi_sock_type_t; + +typedef uint16_t __wasi_ip_port_t; + +typedef enum { IPv4 = 0, IPv6 } __wasi_addr_type_t; + +/* + n0.n1.n2.n3 + Example: + IP Address: 127.0.0.1 + Structure: {n0: 127, n1: 0, n2: 0, n3: 1} +*/ +typedef struct __wasi_addr_ip4_t { + uint8_t n0; + uint8_t n1; + uint8_t n2; + uint8_t n3; +} __wasi_addr_ip4_t; + +typedef struct __wasi_addr_ip4_port_t { + __wasi_addr_ip4_t addr; + __wasi_ip_port_t port; /* host byte order */ +} __wasi_addr_ip4_port_t; + +/* + n0:n1:n2:n3:h0:h1:h2:h3, each 16bit value uses host byte order + Example (little-endian system) + IP Address fe80::3ba2:893b:4be0:e3dd + Structure: { + n0: 0xfe80, n1:0x0, n2: 0x0, n3: 0x0, + h0: 0x3ba2, h1: 0x893b, h2: 0x4be0, h3: 0xe3dd + } +*/ +typedef struct __wasi_addr_ip6_t { + uint16_t n0; + uint16_t n1; + uint16_t n2; + uint16_t n3; + uint16_t h0; + uint16_t h1; + uint16_t h2; + uint16_t h3; +} __wasi_addr_ip6_t; + +typedef struct __wasi_addr_ip6_port_t { + __wasi_addr_ip6_t addr; + __wasi_ip_port_t port; /* host byte order */ +} __wasi_addr_ip6_port_t; + +typedef struct __wasi_addr_ip_t { + __wasi_addr_type_t kind; + union { + __wasi_addr_ip4_t ip4; + __wasi_addr_ip6_t ip6; + } addr; +} __wasi_addr_ip_t; + +typedef struct __wasi_addr_t { + __wasi_addr_type_t kind; + union { + __wasi_addr_ip4_port_t ip4; + __wasi_addr_ip6_port_t ip6; + } addr; +} __wasi_addr_t; + +typedef enum { INET4 = 0, INET6, INET_UNSPEC } __wasi_address_family_t; + +typedef struct __wasi_addr_info_t { + __wasi_addr_t addr; + __wasi_sock_type_t type; +} __wasi_addr_info_t; + +typedef struct __wasi_addr_info_hints_t { + __wasi_sock_type_t type; + __wasi_address_family_t family; + // this is to workaround lack of optional parameters + uint8_t hints_enabled; +} __wasi_addr_info_hints_t; + +#ifdef __wasi__ +/** + * Reimplement below POSIX APIs with __wasi_sock_XXX functions. + * + * Keep sync with + * + * + */ +#define SO_REUSEADDR 2 +#define SO_BROADCAST 6 +#define SO_SNDBUF 7 +#define SO_RCVBUF 8 +#define SO_KEEPALIVE 9 +#define SO_LINGER 13 +#define SO_REUSEPORT 15 +#define SO_RCVTIMEO 20 +#define SO_SNDTIMEO 21 + +#define TCP_NODELAY 1 +#define TCP_KEEPIDLE 4 +#define TCP_KEEPINTVL 5 +#define TCP_QUICKACK 12 +#define TCP_FASTOPEN_CONNECT 30 + +#define IP_TTL 2 +#define IP_MULTICAST_TTL 33 +#define IP_MULTICAST_LOOP 34 +#define IP_ADD_MEMBERSHIP 35 +#define IP_DROP_MEMBERSHIP 36 + +#define IPV6_MULTICAST_LOOP 19 +#define IPV6_JOIN_GROUP 20 +#define IPV6_LEAVE_GROUP 21 +#define IPV6_V6ONLY 26 + +struct addrinfo { + int ai_flags; /* Input flags. */ + int ai_family; /* Protocol family for socket. */ + int ai_socktype; /* Socket type. */ + int ai_protocol; /* Protocol for socket. */ + socklen_t ai_addrlen; /* Length of socket address. */ + struct sockaddr *ai_addr; /* Socket address for socket. */ + char *ai_canonname; /* Canonical name for service location. */ + struct addrinfo *ai_next; /* Pointer to next in list. */ +}; + +#ifndef __WASI_RIGHTS_SOCK_ACCEPT +int +accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen); +#endif + +int +bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen); + +int +connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen); + +int +listen(int sockfd, int backlog); + +ssize_t +recvmsg(int sockfd, struct msghdr *msg, int flags); + +ssize_t +sendmsg(int sockfd, const struct msghdr *msg, int flags); + +ssize_t +sendto(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen); + +ssize_t +recvfrom(int sockfd, void *buf, size_t len, int flags, + struct sockaddr *src_addr, socklen_t *addrlen); + +int +socket(int domain, int type, int protocol); + +int +getsockname(int sockfd, struct sockaddr *addr, socklen_t *addrlen); + +int +getpeername(int sockfd, struct sockaddr *addr, socklen_t *addrlen); + +int +getsockopt(int sockfd, int level, int optname, void *__restrict optval, + socklen_t *__restrict optlen); + +int +setsockopt(int sockfd, int level, int optname, const void *optval, + socklen_t optlen); + +int +getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, + struct addrinfo **res); + +void +freeaddrinfo(struct addrinfo *res); +#endif + +/** + * __wasi_sock_accept was introduced in wasi-sdk v15. To + * temporarily maintain backward compatibility with the old + * wasi-sdk, we explicitly add that implementation here so it works + * with older versions of the SDK. + */ +#ifndef __WASI_RIGHTS_SOCK_ACCEPT +/** + * Accept a connection on a socket + * Note: This is similar to `accept` + */ +int32_t +__imported_wasi_snapshot_preview1_sock_accept(int32_t arg0, int32_t arg1, + int32_t arg2) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_accept"))); + +static inline __wasi_errno_t +__wasi_sock_accept(__wasi_fd_t fd, __wasi_fdflags_t flags, __wasi_fd_t *fd_new) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_accept( + (int32_t)fd, (int32_t)flags, (int32_t)fd_new); +} +#endif + +/** + * Returns the local address to which the socket is bound. + * + * Note: This is similar to `getsockname` in POSIX + * + * When successful, the contents of the output buffer consist of an IP address, + * either IP4 or IP6. + */ +int32_t +__imported_wasi_snapshot_preview1_sock_addr_local(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_addr_local"))); + +static inline __wasi_errno_t +__wasi_sock_addr_local(__wasi_fd_t fd, __wasi_addr_t *addr) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_addr_local( + (int32_t)fd, (int32_t)addr); +} + +/** + * Returns the remote address to which the socket is connected to. + * + * Note: This is similar to `getpeername` in POSIX + * + * When successful, the contents of the output buffer consist of an IP address, + * either IP4 or IP6. + */ +int32_t +__imported_wasi_snapshot_preview1_sock_addr_remote(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_addr_remote"))); + +static inline __wasi_errno_t +__wasi_sock_addr_remote(__wasi_fd_t fd, __wasi_addr_t *addr) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_addr_remote( + (int32_t)fd, (int32_t)addr); +} + +/** + * Resolve a hostname and a service to one or more IP addresses. Service is + * optional and you can pass empty string in most cases, it is used as a hint + * for protocol. + * + * Note: This is similar to `getaddrinfo` in POSIX + * + * When successful, the contents of the output buffer consist of a sequence of + * IPv4 and/or IPv6 addresses. Each address entry consists of a wasi_addr_t + * object. + * + * This function fills the output buffer as much as possible, truncating the + * entries that didn't fit into the buffer. A number of available addresses + * will be returned through the last parameter. + */ +int32_t +__imported_wasi_snapshot_preview1_sock_addr_resolve(int32_t arg0, int32_t arg1, + int32_t arg2, int32_t arg3, + int32_t arg4, int32_t arg5) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_addr_resolve"))); + +static inline __wasi_errno_t +__wasi_sock_addr_resolve(const char *host, const char *service, + __wasi_addr_info_hints_t *hints, + __wasi_addr_info_t *addr_info, + __wasi_size_t addr_info_size, + __wasi_size_t *max_info_size) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_addr_resolve( + (int32_t)host, (int32_t)service, (int32_t)hints, (int32_t)addr_info, + (int32_t)addr_info_size, (int32_t)max_info_size); +} + +/** + * Bind a socket + * Note: This is similar to `bind` in POSIX using PF_INET + */ +int32_t +__imported_wasi_snapshot_preview1_sock_bind(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_bind"))); + +static inline __wasi_errno_t +__wasi_sock_bind(__wasi_fd_t fd, __wasi_addr_t *addr) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_bind( + (int32_t)fd, (int32_t)addr); +} + +/** + * Send data to a specific target + * Note: This is similar to `sendto` in POSIX + */ +int32_t +__imported_wasi_snapshot_preview1_sock_send_to(int32_t arg0, int32_t arg1, + int32_t arg2, int32_t arg3, + int32_t arg4, int32_t arg5) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_send_to"))); + +static inline __wasi_errno_t +__wasi_sock_send_to(__wasi_fd_t fd, const __wasi_ciovec_t *si_data, + uint32_t si_data_len, __wasi_siflags_t si_flags, + const __wasi_addr_t *dest_addr, uint32_t *so_data_len) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_send_to( + (int32_t)fd, (int32_t)si_data, (int32_t)si_data_len, (int32_t)si_flags, + (uint32_t)dest_addr, (uint32_t)so_data_len); +} + +/** + * Receives data from a socket + * Note: This is similar to `recvfrom` in POSIX + */ +int32_t +__imported_wasi_snapshot_preview1_sock_recv_from(int32_t arg0, int32_t arg1, + int32_t arg2, int32_t arg3, + int32_t arg4, int32_t arg5) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_recv_from"))); + +static inline __wasi_errno_t +__wasi_sock_recv_from(__wasi_fd_t fd, __wasi_ciovec_t *ri_data, + uint32_t ri_data_len, __wasi_riflags_t ri_flags, + __wasi_addr_t *src_addr, uint32_t *ro_data_len) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_recv_from( + (int32_t)fd, (int32_t)ri_data, (int32_t)ri_data_len, (int32_t)ri_flags, + (uint32_t)src_addr, (uint32_t)ro_data_len); +} + +/** + * Close a socket (this is an alias for `fd_close`) + * Note: This is similar to `close` in POSIX. + */ +int32_t +__imported_wasi_snapshot_preview1_sock_close(int32_t arg0) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_close"))); + +static inline __wasi_errno_t +__wasi_sock_close(__wasi_fd_t fd) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_close( + (int32_t)fd); +} + +/** + * Initiate a connection on a socket to the specified address + * Note: This is similar to `connect` in POSIX + */ + +int32_t +__imported_wasi_snapshot_preview1_sock_connect(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_connect"))); + +static inline __wasi_errno_t +__wasi_sock_connect(__wasi_fd_t fd, __wasi_addr_t *addr) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_connect( + (int32_t)fd, (int32_t)addr); +} +/** + * Retrieve the size of the receive buffer + * Note: This is similar to `getsockopt` in POSIX for SO_RCVBUF + */ + +int32_t +__imported_wasi_snapshot_preview1_sock_get_recv_buf_size(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_recv_buf_size"))); + +static inline __wasi_errno_t +__wasi_sock_get_recv_buf_size(__wasi_fd_t fd, __wasi_size_t *size) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_recv_buf_size((int32_t)fd, + (int32_t)size); +} +/** + * Retrieve status of address reuse on a socket + * Note: This is similar to `getsockopt` in POSIX for SO_REUSEADDR + */ +int32_t +__imported_wasi_snapshot_preview1_sock_get_reuse_addr(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_reuse_addr"))); + +static inline __wasi_errno_t +__wasi_sock_get_reuse_addr(__wasi_fd_t fd, bool *reuse) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_reuse_addr((int32_t)fd, + (int32_t)reuse); +} + +/** + * Retrieve status of port reuse on a socket + * Note: This is similar to `getsockopt` in POSIX for SO_REUSEPORT + */ +int32_t +__imported_wasi_snapshot_preview1_sock_get_reuse_port(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_reuse_port"))); + +static inline __wasi_errno_t +__wasi_sock_get_reuse_port(__wasi_fd_t fd, bool *reuse) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_reuse_port((int32_t)fd, + (int32_t)reuse); +} + +/** + * Retrieve the size of the send buffer + * Note: This is similar to `getsockopt` in POSIX for SO_SNDBUF + */ +int32_t +__imported_wasi_snapshot_preview1_sock_get_send_buf_size(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_send_buf_size"))); + +static inline __wasi_errno_t +__wasi_sock_get_send_buf_size(__wasi_fd_t fd, __wasi_size_t *size) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_send_buf_size((int32_t)fd, + (int32_t)size); +} + +/** + * Listen for connections on a socket + * Note: This is similar to `listen` + */ +int32_t +__imported_wasi_snapshot_preview1_sock_listen(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_listen"))); + +static inline __wasi_errno_t +__wasi_sock_listen(__wasi_fd_t fd, __wasi_size_t backlog) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_listen( + (int32_t)fd, (int32_t)backlog); +} + +/** + * Open a socket + + * The first argument to this function is a handle to an + * address pool. The address pool determines what actions can + * be performed and at which addresses they can be performed to. + + * The address pool cannot be re-assigned. You will need to close + * the socket and open a new one to use a different address pool. + + * Note: This is similar to `socket` in POSIX using PF_INET + */ + +int32_t +__imported_wasi_snapshot_preview1_sock_open(int32_t arg0, int32_t arg1, + int32_t arg2, int32_t arg3) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_open"))); + +static inline __wasi_errno_t +__wasi_sock_open(__wasi_fd_t fd, __wasi_address_family_t af, + __wasi_sock_type_t socktype, __wasi_fd_t *sockfd) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_open( + (int32_t)fd, (int32_t)af, (int32_t)socktype, (int32_t)sockfd); +} + +/** + * Set size of receive buffer + * Note: This is similar to `setsockopt` in POSIX for SO_RCVBUF + */ +int32_t +__imported_wasi_snapshot_preview1_sock_set_recv_buf_size(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_recv_buf_size"))); + +static inline __wasi_errno_t +__wasi_sock_set_recv_buf_size(__wasi_fd_t fd, __wasi_size_t size) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_recv_buf_size((int32_t)fd, + (int32_t)size); +} + +/** + * Enable/disable address reuse on a socket + * Note: This is similar to `setsockopt` in POSIX for SO_REUSEADDR + */ +int32_t +__imported_wasi_snapshot_preview1_sock_set_reuse_addr(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_reuse_addr"))); + +static inline __wasi_errno_t +__wasi_sock_set_reuse_addr(__wasi_fd_t fd, bool reuse) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_reuse_addr((int32_t)fd, + (int32_t)reuse); +} + +/** + * Enable port reuse on a socket + * Note: This is similar to `setsockopt` in POSIX for SO_REUSEPORT + */ +int32_t +__imported_wasi_snapshot_preview1_sock_set_reuse_port(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_reuse_port"))); + +static inline __wasi_errno_t +__wasi_sock_set_reuse_port(__wasi_fd_t fd, bool reuse) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_reuse_port((int32_t)fd, + (int32_t)reuse); +} + +/** + * Set size of send buffer + * Note: This is similar to `setsockopt` in POSIX for SO_SNDBUF + */ +int32_t +__imported_wasi_snapshot_preview1_sock_set_send_buf_size(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_send_buf_size"))); + +static inline __wasi_errno_t +__wasi_sock_set_send_buf_size(__wasi_fd_t fd, __wasi_size_t buf_len) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_send_buf_size( + (int32_t)fd, (int32_t)buf_len); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_recv_timeout(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_recv_timeout"))); + +static inline __wasi_errno_t +__wasi_sock_get_recv_timeout(__wasi_fd_t fd, uint64_t *timeout_us) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_recv_timeout( + (int32_t)fd, (int32_t)timeout_us); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_recv_timeout(int32_t arg0, + int64_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_recv_timeout"))); + +static inline __wasi_errno_t +__wasi_sock_set_recv_timeout(__wasi_fd_t fd, uint64_t timeout_us) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_recv_timeout( + (int32_t)fd, (int64_t)timeout_us); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_send_timeout(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_send_timeout"))); + +static inline __wasi_errno_t +__wasi_sock_get_send_timeout(__wasi_fd_t fd, uint64_t *timeout_us) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_send_timeout( + (int32_t)fd, (int32_t)timeout_us); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_send_timeout(int32_t arg0, + int64_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_send_timeout"))); + +static inline __wasi_errno_t +__wasi_sock_set_send_timeout(__wasi_fd_t fd, uint64_t timeout_us) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_send_timeout( + (int32_t)fd, (int64_t)timeout_us); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_keep_alive(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_keep_alive"))); + +static inline __wasi_errno_t +__wasi_sock_set_keep_alive(__wasi_fd_t fd, bool option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_keep_alive((int32_t)fd, + (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_keep_alive(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_keep_alive"))); + +static inline __wasi_errno_t +__wasi_sock_get_keep_alive(__wasi_fd_t fd, bool *option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_keep_alive((int32_t)fd, + (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_linger(int32_t arg0, int32_t arg1, + int32_t arg2) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_linger"))); + +static inline __wasi_errno_t +__wasi_sock_set_linger(__wasi_fd_t fd, bool is_enabled, int linger_s) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_set_linger( + (int32_t)fd, (int32_t)is_enabled, (int32_t)linger_s); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_linger(int32_t arg0, int32_t arg1, + int32_t arg2) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_linger"))); + +static inline __wasi_errno_t +__wasi_sock_get_linger(__wasi_fd_t fd, bool *is_enabled, int *linger_s) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_get_linger( + (int32_t)fd, (int32_t)is_enabled, (int32_t)linger_s); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_tcp_keep_idle(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_tcp_keep_idle"))); + +static inline __wasi_errno_t +__wasi_sock_set_tcp_keep_idle(__wasi_fd_t fd, uint32_t time_s) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_tcp_keep_idle( + (int32_t)fd, (int32_t)time_s); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_tcp_keep_idle(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_tcp_keep_idle"))); + +static inline __wasi_errno_t +__wasi_sock_get_tcp_keep_idle(__wasi_fd_t fd, uint32_t *time_s) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_tcp_keep_idle( + (int32_t)fd, (int32_t)time_s); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_tcp_keep_intvl(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_tcp_keep_intvl"))); + +static inline __wasi_errno_t +__wasi_sock_set_tcp_keep_intvl(__wasi_fd_t fd, uint32_t time_s) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_tcp_keep_intvl( + (int32_t)fd, (int32_t)time_s); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_tcp_keep_intvl(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_tcp_keep_intvl"))); + +static inline __wasi_errno_t +__wasi_sock_get_tcp_keep_intvl(__wasi_fd_t fd, uint32_t *time_s) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_tcp_keep_intvl( + (int32_t)fd, (int32_t)time_s); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_tcp_fastopen_connect(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_tcp_fastopen_connect"))); + +static inline __wasi_errno_t +__wasi_sock_set_tcp_fastopen_connect(__wasi_fd_t fd, bool option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_tcp_fastopen_connect( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_tcp_fastopen_connect(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_tcp_fastopen_connect"))); + +static inline __wasi_errno_t +__wasi_sock_get_tcp_fastopen_connect(__wasi_fd_t fd, bool *option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_tcp_fastopen_connect( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_ip_multicast_loop(int32_t arg0, + int32_t arg1, + int32_t arg2) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_ip_multicast_loop"))); + +static inline __wasi_errno_t +__wasi_sock_set_ip_multicast_loop(__wasi_fd_t fd, bool ipv6, bool option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_ip_multicast_loop( + (int32_t)fd, (int32_t)ipv6, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_ip_multicast_loop(int32_t arg0, + int32_t arg1, + int32_t arg2) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_ip_multicast_loop"))); + +static inline __wasi_errno_t +__wasi_sock_get_ip_multicast_loop(__wasi_fd_t fd, bool ipv6, bool *option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_ip_multicast_loop( + (int32_t)fd, (int32_t)ipv6, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_ip_multicast_ttl(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_ip_multicast_ttl"))); + +static inline __wasi_errno_t +__wasi_sock_set_ip_multicast_ttl(__wasi_fd_t fd, uint8_t option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_ip_multicast_ttl( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_ip_multicast_ttl(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_ip_multicast_ttl"))); + +static inline __wasi_errno_t +__wasi_sock_get_ip_multicast_ttl(__wasi_fd_t fd, uint8_t *option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_ip_multicast_ttl( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_ip_add_membership(int32_t arg0, + int32_t arg1, + int32_t arg2) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_ip_add_membership"))); + +static inline __wasi_errno_t +__wasi_sock_set_ip_add_membership(__wasi_fd_t fd, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_ip_add_membership( + (int32_t)fd, (int32_t)imr_multiaddr, (int32_t)imr_interface); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_ip_drop_membership(int32_t arg0, + int32_t arg1, + int32_t arg2) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_ip_drop_membership"))); + +static inline __wasi_errno_t +__wasi_sock_set_ip_drop_membership(__wasi_fd_t fd, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_ip_drop_membership( + (int32_t)fd, (int32_t)imr_multiaddr, (int32_t)imr_interface); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_broadcast(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_broadcast"))); + +static inline __wasi_errno_t +__wasi_sock_set_broadcast(__wasi_fd_t fd, bool option) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_set_broadcast( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_broadcast(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_broadcast"))); + +static inline __wasi_errno_t +__wasi_sock_get_broadcast(__wasi_fd_t fd, bool *option) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_get_broadcast( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_tcp_no_delay(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_tcp_no_delay"))); + +static inline __wasi_errno_t +__wasi_sock_set_tcp_no_delay(__wasi_fd_t fd, bool option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_tcp_no_delay( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_tcp_no_delay(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_tcp_no_delay"))); + +static inline __wasi_errno_t +__wasi_sock_get_tcp_no_delay(__wasi_fd_t fd, bool *option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_tcp_no_delay( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_tcp_quick_ack(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_tcp_quick_ack"))); + +static inline __wasi_errno_t +__wasi_sock_set_tcp_quick_ack(__wasi_fd_t fd, bool option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_set_tcp_quick_ack( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_tcp_quick_ack(int32_t arg0, + int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_tcp_quick_ack"))); + +static inline __wasi_errno_t +__wasi_sock_get_tcp_quick_ack(__wasi_fd_t fd, bool *option) +{ + return (__wasi_errno_t) + __imported_wasi_snapshot_preview1_sock_get_tcp_quick_ack( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_ip_ttl(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_ip_ttl"))); + +static inline __wasi_errno_t +__wasi_sock_set_ip_ttl(__wasi_fd_t fd, uint8_t option) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_set_ip_ttl( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_ip_ttl(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_ip_ttl"))); + +static inline __wasi_errno_t +__wasi_sock_get_ip_ttl(__wasi_fd_t fd, uint8_t *option) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_get_ip_ttl( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_set_ipv6_only(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_set_ipv6_only"))); + +static inline __wasi_errno_t +__wasi_sock_set_ipv6_only(__wasi_fd_t fd, bool option) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_set_ipv6_only( + (int32_t)fd, (int32_t)option); +} + +int32_t +__imported_wasi_snapshot_preview1_sock_get_ipv6_only(int32_t arg0, int32_t arg1) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("sock_get_ipv6_only"))); + +static inline __wasi_errno_t +__wasi_sock_get_ipv6_only(__wasi_fd_t fd, bool *option) +{ + return (__wasi_errno_t)__imported_wasi_snapshot_preview1_sock_get_ipv6_only( + (int32_t)fd, (int32_t)option); +} +/** + * TODO: modify recv() and send() + * since don't want to re-compile the wasi-libc, + * we tend to keep original implentations of recv() and send(). + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-socket/lib_socket_wasi.cmake b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/lib_socket_wasi.cmake new file mode 100644 index 0000000..8ddddff --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/lib_socket_wasi.cmake @@ -0,0 +1,9 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.8...3.16) + +project(socket_wasi_ext LANGUAGES C) + +add_library(${PROJECT_NAME} STATIC ${CMAKE_CURRENT_LIST_DIR}/src/wasi/wasi_socket_ext.c) +target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/inc/) diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c new file mode 100644 index 0000000..1172d0a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c @@ -0,0 +1,1014 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define HANDLE_ERROR(error) \ + if (error != __WASI_ERRNO_SUCCESS) { \ + errno = error; \ + return -1; \ + } + +static void +ipv4_addr_to_wasi_ip4_addr(uint32_t addr_num, __wasi_addr_ip4_t *out) +{ + addr_num = ntohl(addr_num); + out->n0 = (addr_num & 0xFF000000) >> 24; + out->n1 = (addr_num & 0x00FF0000) >> 16; + out->n2 = (addr_num & 0x0000FF00) >> 8; + out->n3 = (addr_num & 0x000000FF); +} + +/* addr_num and port are in network order */ +static void +ipv4_addr_to_wasi_addr(uint32_t addr_num, uint16_t port, __wasi_addr_t *out) +{ + out->kind = IPv4; + out->addr.ip4.port = ntohs(port); + ipv4_addr_to_wasi_ip4_addr(addr_num, &(out->addr.ip4.addr)); +} + +static void +ipv6_addr_to_wasi_ipv6_addr(uint16_t *addr, __wasi_addr_ip6_t *out) +{ + out->n0 = ntohs(addr[0]); + out->n1 = ntohs(addr[1]); + out->n2 = ntohs(addr[2]); + out->n3 = ntohs(addr[3]); + out->h0 = ntohs(addr[4]); + out->h1 = ntohs(addr[5]); + out->h2 = ntohs(addr[6]); + out->h3 = ntohs(addr[7]); +} + +static void +ipv6_addr_to_wasi_addr(uint16_t *addr, uint16_t port, __wasi_addr_t *out) +{ + out->kind = IPv6; + out->addr.ip6.port = ntohs(port); + ipv6_addr_to_wasi_ipv6_addr(addr, &(out->addr.ip6.addr)); +} + +static __wasi_errno_t +sockaddr_to_wasi_addr(const struct sockaddr *sock_addr, socklen_t addrlen, + __wasi_addr_t *wasi_addr) +{ + __wasi_errno_t ret = __WASI_ERRNO_SUCCESS; + if (AF_INET == sock_addr->sa_family) { + assert(sizeof(struct sockaddr_in) <= addrlen); + + ipv4_addr_to_wasi_addr( + ((struct sockaddr_in *)sock_addr)->sin_addr.s_addr, + ((struct sockaddr_in *)sock_addr)->sin_port, wasi_addr); + } + else if (AF_INET6 == sock_addr->sa_family) { + assert(sizeof(struct sockaddr_in6) <= addrlen); + ipv6_addr_to_wasi_addr( + (uint16_t *)((struct sockaddr_in6 *)sock_addr)->sin6_addr.s6_addr, + ((struct sockaddr_in6 *)sock_addr)->sin6_port, wasi_addr); + } + else { + ret = __WASI_ERRNO_AFNOSUPPORT; + } + + return ret; +} + +static __wasi_errno_t +wasi_addr_to_sockaddr(const __wasi_addr_t *wasi_addr, + struct sockaddr *sock_addr, socklen_t *addrlen) +{ + switch (wasi_addr->kind) { + case IPv4: + { + struct sockaddr_in sock_addr_in; + uint32_t s_addr; + + memset(&sock_addr_in, 0, sizeof(sock_addr_in)); + + s_addr = (wasi_addr->addr.ip4.addr.n0 << 24) + | (wasi_addr->addr.ip4.addr.n1 << 16) + | (wasi_addr->addr.ip4.addr.n2 << 8) + | wasi_addr->addr.ip4.addr.n3; + + sock_addr_in.sin_family = AF_INET; + sock_addr_in.sin_addr.s_addr = htonl(s_addr); + sock_addr_in.sin_port = htons(wasi_addr->addr.ip4.port); + memcpy(sock_addr, &sock_addr_in, sizeof(sock_addr_in)); + + *addrlen = sizeof(sock_addr_in); + break; + } + case IPv6: + { + struct sockaddr_in6 sock_addr_in6; + + memset(&sock_addr_in6, 0, sizeof(sock_addr_in6)); + + uint16_t *addr_buf = (uint16_t *)sock_addr_in6.sin6_addr.s6_addr; + + addr_buf[0] = htons(wasi_addr->addr.ip6.addr.n0); + addr_buf[1] = htons(wasi_addr->addr.ip6.addr.n1); + addr_buf[2] = htons(wasi_addr->addr.ip6.addr.n2); + addr_buf[3] = htons(wasi_addr->addr.ip6.addr.n3); + addr_buf[4] = htons(wasi_addr->addr.ip6.addr.h0); + addr_buf[5] = htons(wasi_addr->addr.ip6.addr.h1); + addr_buf[6] = htons(wasi_addr->addr.ip6.addr.h2); + addr_buf[7] = htons(wasi_addr->addr.ip6.addr.h3); + + sock_addr_in6.sin6_family = AF_INET6; + sock_addr_in6.sin6_port = htons(wasi_addr->addr.ip6.port); + memcpy(sock_addr, &sock_addr_in6, sizeof(sock_addr_in6)); + + *addrlen = sizeof(sock_addr_in6); + break; + } + default: + return __WASI_ERRNO_AFNOSUPPORT; + } + return __WASI_ERRNO_SUCCESS; +} + +int +accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) +{ + __wasi_addr_t wasi_addr; + __wasi_fd_t new_sockfd; + __wasi_errno_t error; + + memset(&wasi_addr, 0, sizeof(wasi_addr)); + + error = __wasi_sock_accept(sockfd, 0, &new_sockfd); + HANDLE_ERROR(error) + + if (getpeername(new_sockfd, addr, addrlen) == -1) { + return -1; + } + + return new_sockfd; +} + +int +bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen) +{ + __wasi_addr_t wasi_addr; + __wasi_errno_t error; + + memset(&wasi_addr, 0, sizeof(wasi_addr)); + + error = sockaddr_to_wasi_addr(addr, addrlen, &wasi_addr); + HANDLE_ERROR(error) + + error = __wasi_sock_bind(sockfd, &wasi_addr); + HANDLE_ERROR(error) + + return __WASI_ERRNO_SUCCESS; +} + +int +connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) +{ + __wasi_addr_t wasi_addr; + __wasi_errno_t error; + + memset(&wasi_addr, 0, sizeof(wasi_addr)); + + if (NULL == addr) { + HANDLE_ERROR(__WASI_ERRNO_INVAL) + } + + error = sockaddr_to_wasi_addr(addr, addrlen, &wasi_addr); + HANDLE_ERROR(error) + + error = __wasi_sock_connect(sockfd, &wasi_addr); + HANDLE_ERROR(error) + + return __WASI_ERRNO_SUCCESS; +} + +int +listen(int sockfd, int backlog) +{ + __wasi_errno_t error = __wasi_sock_listen(sockfd, backlog); + HANDLE_ERROR(error) + return __WASI_ERRNO_SUCCESS; +} + +ssize_t +recvmsg(int sockfd, struct msghdr *msg, int flags) +{ + // Prepare input parameters. + __wasi_iovec_t *ri_data = NULL; + size_t i = 0; + size_t ro_datalen = 0; + __wasi_roflags_t ro_flags = 0; + + if (NULL == msg) { + HANDLE_ERROR(__WASI_ERRNO_INVAL) + } + + // Validate flags. + if (flags != 0) { + HANDLE_ERROR(__WASI_ERRNO_NOPROTOOPT) + } + + // __wasi_ciovec_t -> struct iovec + if (!(ri_data = (__wasi_iovec_t *)malloc(sizeof(__wasi_iovec_t) + * msg->msg_iovlen))) { + HANDLE_ERROR(__WASI_ERRNO_NOMEM) + } + + for (i = 0; i < msg->msg_iovlen; i++) { + ri_data[i].buf = (uint8_t *)msg->msg_iov[i].iov_base; + ri_data[i].buf_len = msg->msg_iov[i].iov_len; + } + + // Perform system call. + __wasi_errno_t error = __wasi_sock_recv(sockfd, ri_data, msg->msg_iovlen, 0, + &ro_datalen, &ro_flags); + free(ri_data); + HANDLE_ERROR(error) + + return ro_datalen; +} + +ssize_t +sendmsg(int sockfd, const struct msghdr *msg, int flags) +{ + // Prepare input parameters. + __wasi_ciovec_t *si_data = NULL; + size_t so_datalen = 0; + size_t i = 0; + + if (NULL == msg) { + HANDLE_ERROR(__WASI_ERRNO_INVAL) + } + + // This implementation does not support any flags. + if (flags != 0) { + HANDLE_ERROR(__WASI_ERRNO_NOPROTOOPT) + } + + // struct iovec -> __wasi_ciovec_t + if (!(si_data = (__wasi_ciovec_t *)malloc(sizeof(__wasi_ciovec_t) + * msg->msg_iovlen))) { + HANDLE_ERROR(__WASI_ERRNO_NOMEM) + } + + for (i = 0; i < msg->msg_iovlen; i++) { + si_data[i].buf = (uint8_t *)msg->msg_iov[i].iov_base; + si_data[i].buf_len = msg->msg_iov[i].iov_len; + } + + // Perform system call. + __wasi_errno_t error = + __wasi_sock_send(sockfd, si_data, msg->msg_iovlen, 0, &so_datalen); + free(si_data); + HANDLE_ERROR(error) + + return so_datalen; +} + +ssize_t +sendto(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ + // Prepare input parameters. + __wasi_ciovec_t iov = { .buf = (uint8_t *)buf, .buf_len = len }; + uint32_t so_datalen = 0; + __wasi_addr_t wasi_addr; + __wasi_errno_t error; + size_t si_data_len = 1; + __wasi_siflags_t si_flags = 0; + + // This implementation does not support any flags. + if (flags != 0) { + HANDLE_ERROR(__WASI_ERRNO_NOPROTOOPT) + } + + error = sockaddr_to_wasi_addr(dest_addr, addrlen, &wasi_addr); + HANDLE_ERROR(error); + + // Perform system call. + error = __wasi_sock_send_to(sockfd, &iov, si_data_len, si_flags, &wasi_addr, + &so_datalen); + HANDLE_ERROR(error) + + return so_datalen; +} + +ssize_t +recvfrom(int sockfd, void *buf, size_t len, int flags, + struct sockaddr *src_addr, socklen_t *addrlen) +{ + // Prepare input parameters. + __wasi_ciovec_t iov = { .buf = (uint8_t *)buf, .buf_len = len }; + uint32_t so_datalen = 0; + __wasi_addr_t wasi_addr; + __wasi_errno_t error; + size_t si_data_len = 1; + __wasi_siflags_t si_flags = 0; + + // This implementation does not support any flags. + if (flags != 0) { + HANDLE_ERROR(__WASI_ERRNO_NOPROTOOPT) + } + + if (!src_addr) { + return recv(sockfd, buf, len, flags); + } + + // Perform system call. + error = __wasi_sock_recv_from(sockfd, &iov, si_data_len, si_flags, + &wasi_addr, &so_datalen); + HANDLE_ERROR(error); + + error = wasi_addr_to_sockaddr(&wasi_addr, src_addr, addrlen); + HANDLE_ERROR(error); + + return so_datalen; +} + +int +socket(int domain, int type, int protocol) +{ + // the stub of address pool fd + __wasi_fd_t poolfd = -1; + __wasi_fd_t sockfd; + __wasi_errno_t error; + __wasi_address_family_t af; + __wasi_sock_type_t socktype; + + if (AF_INET == domain) { + af = INET4; + } + else if (AF_INET6 == domain) { + af = INET6; + } + else { + return __WASI_ERRNO_NOPROTOOPT; + } + + if (SOCK_DGRAM == type) { + socktype = SOCKET_DGRAM; + } + else if (SOCK_STREAM == type) { + socktype = SOCKET_STREAM; + } + else { + return __WASI_ERRNO_NOPROTOOPT; + } + + error = __wasi_sock_open(poolfd, af, socktype, &sockfd); + HANDLE_ERROR(error) + + return sockfd; +} + +int +getsockname(int sockfd, struct sockaddr *addr, socklen_t *addrlen) +{ + __wasi_addr_t wasi_addr; + __wasi_errno_t error; + + memset(&wasi_addr, 0, sizeof(wasi_addr)); + + error = __wasi_sock_addr_local(sockfd, &wasi_addr); + HANDLE_ERROR(error) + + error = wasi_addr_to_sockaddr(&wasi_addr, addr, addrlen); + HANDLE_ERROR(error) + + return __WASI_ERRNO_SUCCESS; +} + +int +getpeername(int sockfd, struct sockaddr *addr, socklen_t *addrlen) +{ + __wasi_addr_t wasi_addr; + __wasi_errno_t error; + + memset(&wasi_addr, 0, sizeof(wasi_addr)); + + error = __wasi_sock_addr_remote(sockfd, &wasi_addr); + HANDLE_ERROR(error) + + error = wasi_addr_to_sockaddr(&wasi_addr, addr, addrlen); + HANDLE_ERROR(error) + + return __WASI_ERRNO_SUCCESS; +} + +struct aibuf { + struct addrinfo ai; + union sa { + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + } sa; +}; + +static __wasi_errno_t +addrinfo_hints_to_wasi_hints(const struct addrinfo *hints, + __wasi_addr_info_hints_t *wasi_hints) +{ + if (hints) { + wasi_hints->hints_enabled = 1; + + switch (hints->ai_family) { + case AF_INET: + wasi_hints->family = INET4; + break; + case AF_INET6: + wasi_hints->family = INET6; + break; + case AF_UNSPEC: + wasi_hints->family = INET_UNSPEC; + break; + default: + return __WASI_ERRNO_AFNOSUPPORT; + } + switch (hints->ai_socktype) { + case SOCK_STREAM: + wasi_hints->type = SOCKET_STREAM; + break; + case SOCK_DGRAM: + wasi_hints->type = SOCKET_DGRAM; + break; + case 0: + wasi_hints->type = SOCKET_ANY; + default: + return __WASI_ERRNO_NOTSUP; + } + + if (hints->ai_protocol != 0) { + return __WASI_ERRNO_NOTSUP; + } + + if (hints->ai_flags != 0) { + return __WASI_ERRNO_NOTSUP; + } + } + else { + wasi_hints->hints_enabled = 0; + } + + return __WASI_ERRNO_SUCCESS; +} + +static __wasi_errno_t +wasi_addr_info_to_addr_info(const __wasi_addr_info_t *addr_info, + struct addrinfo *ai) +{ + ai->ai_socktype = + addr_info->type == SOCKET_DGRAM ? SOCK_DGRAM : SOCK_STREAM; + ai->ai_protocol = 0; + ai->ai_canonname = NULL; + + if (addr_info->addr.kind == IPv4) { + ai->ai_family = AF_INET; + ai->ai_addrlen = sizeof(struct sockaddr_in); + } + else { + ai->ai_family = AF_INET6; + ai->ai_addrlen = sizeof(struct sockaddr_in6); + } + + return wasi_addr_to_sockaddr(&addr_info->addr, ai->ai_addr, + &ai->ai_addrlen); // TODO err handling +} + +int +getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, + struct addrinfo **res) +{ + __wasi_addr_info_hints_t wasi_hints; + __wasi_addr_info_t *addr_info = NULL; + __wasi_size_t addr_info_size, i; + __wasi_size_t max_info_size = 16; + __wasi_errno_t error; + struct aibuf *aibuf_res; + + error = addrinfo_hints_to_wasi_hints(hints, &wasi_hints); + HANDLE_ERROR(error) + + do { + if (addr_info) + free(addr_info); + + addr_info_size = max_info_size; + addr_info = (__wasi_addr_info_t *)malloc(addr_info_size + * sizeof(__wasi_addr_info_t)); + + if (!addr_info) { + HANDLE_ERROR(__WASI_ERRNO_NOMEM) + } + + error = __wasi_sock_addr_resolve(node, service == NULL ? "" : service, + &wasi_hints, addr_info, addr_info_size, + &max_info_size); + if (error != __WASI_ERRNO_SUCCESS) { + free(addr_info); + HANDLE_ERROR(error); + } + } while (max_info_size > addr_info_size); + + if (addr_info_size == 0) { + free(addr_info); + *res = NULL; + return __WASI_ERRNO_SUCCESS; + } + + aibuf_res = + (struct aibuf *)calloc(1, addr_info_size * sizeof(struct aibuf)); + if (!aibuf_res) { + free(addr_info); + HANDLE_ERROR(__WASI_ERRNO_NOMEM) + } + + *res = &aibuf_res[0].ai; + + if (addr_info_size) { + addr_info_size = max_info_size; + } + + for (i = 0; i < addr_info_size; i++) { + struct addrinfo *ai = &aibuf_res[i].ai; + ai->ai_addr = (struct sockaddr *)&aibuf_res[i].sa; + + error = wasi_addr_info_to_addr_info(&addr_info[i], ai); + if (error != __WASI_ERRNO_SUCCESS) { + free(addr_info); + free(aibuf_res); + HANDLE_ERROR(error) + } + ai->ai_next = i == addr_info_size - 1 ? NULL : &aibuf_res[i + 1].ai; + } + + free(addr_info); + + return __WASI_ERRNO_SUCCESS; +} + +void +freeaddrinfo(struct addrinfo *res) +{ + /* res is a pointer to a first field in the first element + * of aibuf array allocated in getaddrinfo, therefore this call + * frees the memory of the entire array. */ + free(res); +} + +static struct timeval +time_us_to_timeval(uint64_t time_us) +{ + struct timeval tv; + tv.tv_sec = time_us / 1000000UL; + tv.tv_usec = time_us % 1000000UL; + return tv; +} + +static uint64_t +timeval_to_time_us(struct timeval tv) +{ + return (tv.tv_sec * 1000000UL) + tv.tv_usec; +} + +static int +get_sol_socket_option(int sockfd, int optname, void *__restrict optval, + socklen_t *__restrict optlen) +{ + __wasi_errno_t error; + uint64_t timeout_us; + bool is_linger_enabled; + int linger_s; + + switch (optname) { + case SO_RCVTIMEO: + assert(*optlen == sizeof(struct timeval)); + error = __wasi_sock_get_recv_timeout(sockfd, &timeout_us); + HANDLE_ERROR(error); + *(struct timeval *)optval = time_us_to_timeval(timeout_us); + return error; + case SO_SNDTIMEO: + assert(*optlen == sizeof(struct timeval)); + error = __wasi_sock_get_send_timeout(sockfd, &timeout_us); + HANDLE_ERROR(error); + *(struct timeval *)optval = time_us_to_timeval(timeout_us); + return error; + case SO_SNDBUF: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_send_buf_size(sockfd, (size_t *)optval); + HANDLE_ERROR(error); + return error; + case SO_RCVBUF: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_recv_buf_size(sockfd, (size_t *)optval); + HANDLE_ERROR(error); + return error; + case SO_KEEPALIVE: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_keep_alive(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + case SO_REUSEADDR: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_reuse_addr(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + case SO_REUSEPORT: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_reuse_port(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + case SO_LINGER: + assert(*optlen == sizeof(struct linger)); + error = + __wasi_sock_get_linger(sockfd, &is_linger_enabled, &linger_s); + HANDLE_ERROR(error); + ((struct linger *)optval)->l_onoff = (int)is_linger_enabled; + ((struct linger *)optval)->l_linger = linger_s; + return error; + case SO_BROADCAST: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_broadcast(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +static int +get_ipproto_tcp_option(int sockfd, int optname, void *__restrict optval, + socklen_t *__restrict optlen) +{ + __wasi_errno_t error; + switch (optname) { + case TCP_KEEPIDLE: + assert(*optlen == sizeof(uint32_t)); + error = __wasi_sock_get_tcp_keep_idle(sockfd, (uint32_t *)optval); + HANDLE_ERROR(error); + return error; + case TCP_KEEPINTVL: + assert(*optlen == sizeof(uint32_t)); + error = __wasi_sock_get_tcp_keep_intvl(sockfd, (uint32_t *)optval); + HANDLE_ERROR(error); + return error; + case TCP_FASTOPEN_CONNECT: + assert(*optlen == sizeof(int)); + error = + __wasi_sock_get_tcp_fastopen_connect(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + case TCP_NODELAY: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_tcp_no_delay(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + case TCP_QUICKACK: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_tcp_quick_ack(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +static int +get_ipproto_ip_option(int sockfd, int optname, void *__restrict optval, + socklen_t *__restrict optlen) +{ + __wasi_errno_t error; + + switch (optname) { + case IP_MULTICAST_LOOP: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_ip_multicast_loop(sockfd, false, + (bool *)optval); + HANDLE_ERROR(error); + return error; + case IP_TTL: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_ip_ttl(sockfd, (uint8_t *)optval); + HANDLE_ERROR(error); + return error; + case IP_MULTICAST_TTL: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_ip_multicast_ttl(sockfd, (uint8_t *)optval); + HANDLE_ERROR(error); + return error; + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +static int +get_ipproto_ipv6_option(int sockfd, int optname, void *__restrict optval, + socklen_t *__restrict optlen) +{ + __wasi_errno_t error; + + switch (optname) { + case IPV6_V6ONLY: + assert(*optlen == sizeof(int)); + error = __wasi_sock_get_ipv6_only(sockfd, (bool *)optval); + HANDLE_ERROR(error); + return error; + case IPV6_MULTICAST_LOOP: + assert(*optlen == sizeof(int)); + error = + __wasi_sock_get_ip_multicast_loop(sockfd, true, (bool *)optval); + HANDLE_ERROR(error); + return error; + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +int +getsockopt(int sockfd, int level, int optname, void *__restrict optval, + socklen_t *__restrict optlen) +{ + __wasi_errno_t error; + + switch (level) { + case SOL_SOCKET: + return get_sol_socket_option(sockfd, optname, optval, optlen); + case IPPROTO_TCP: + return get_ipproto_tcp_option(sockfd, optname, optval, optlen); + case IPPROTO_IP: + return get_ipproto_ip_option(sockfd, optname, optval, optlen); + case IPPROTO_IPV6: + return get_ipproto_ipv6_option(sockfd, optname, optval, optlen); + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +static int +set_sol_socket_option(int sockfd, int optname, const void *optval, + socklen_t optlen) +{ + __wasi_errno_t error; + uint64_t timeout_us; + + switch (optname) { + case SO_RCVTIMEO: + { + assert(optlen == sizeof(struct timeval)); + timeout_us = timeval_to_time_us(*(struct timeval *)optval); + error = __wasi_sock_set_recv_timeout(sockfd, timeout_us); + HANDLE_ERROR(error); + return error; + } + case SO_SNDTIMEO: + { + assert(optlen == sizeof(struct timeval)); + timeout_us = timeval_to_time_us(*(struct timeval *)optval); + error = __wasi_sock_set_send_timeout(sockfd, timeout_us); + HANDLE_ERROR(error); + return error; + } + case SO_SNDBUF: + { + assert(optlen == sizeof(int)); + error = __wasi_sock_set_send_buf_size(sockfd, *(size_t *)optval); + HANDLE_ERROR(error); + return error; + } + case SO_RCVBUF: + { + assert(optlen == sizeof(int)); + error = __wasi_sock_set_recv_buf_size(sockfd, *(size_t *)optval); + HANDLE_ERROR(error); + return error; + } + case SO_KEEPALIVE: + { + assert(optlen == sizeof(int)); + error = __wasi_sock_set_keep_alive(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + } + case SO_REUSEADDR: + { + assert(optlen == sizeof(int)); + error = __wasi_sock_set_reuse_addr(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + } + case SO_REUSEPORT: + { + assert(optlen == sizeof(int)); + error = __wasi_sock_set_reuse_port(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + } + case SO_LINGER: + { + assert(optlen == sizeof(struct linger)); + struct linger *linger_opt = ((struct linger *)optval); + error = __wasi_sock_set_linger(sockfd, (bool)linger_opt->l_onoff, + linger_opt->l_linger); + HANDLE_ERROR(error); + return error; + } + case SO_BROADCAST: + { + assert(optlen == sizeof(int)); + error = __wasi_sock_set_broadcast(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + } + default: + { + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } + } +} + +static int +set_ipproto_tcp_option(int sockfd, int optname, const void *optval, + socklen_t optlen) +{ + __wasi_errno_t error; + + switch (optname) { + case TCP_NODELAY: + assert(optlen == sizeof(int)); + error = __wasi_sock_set_tcp_no_delay(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + case TCP_KEEPIDLE: + assert(optlen == sizeof(uint32_t)); + error = __wasi_sock_set_tcp_keep_idle(sockfd, *(uint32_t *)optval); + HANDLE_ERROR(error); + return error; + case TCP_KEEPINTVL: + assert(optlen == sizeof(uint32_t)); + error = __wasi_sock_set_tcp_keep_intvl(sockfd, *(uint32_t *)optval); + HANDLE_ERROR(error); + return error; + case TCP_FASTOPEN_CONNECT: + assert(optlen == sizeof(int)); + error = + __wasi_sock_set_tcp_fastopen_connect(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + case TCP_QUICKACK: + assert(optlen == sizeof(int)); + error = __wasi_sock_set_tcp_quick_ack(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +static int +set_ipproto_ip_option(int sockfd, int optname, const void *optval, + socklen_t optlen) +{ + __wasi_errno_t error; + __wasi_addr_ip_t imr_multiaddr; + struct ip_mreq *ip_mreq_opt; + + switch (optname) { + case IP_MULTICAST_LOOP: + assert(optlen == sizeof(int)); + error = __wasi_sock_set_ip_multicast_loop(sockfd, false, + *(bool *)optval); + HANDLE_ERROR(error); + return error; + case IP_ADD_MEMBERSHIP: + assert(optlen == sizeof(struct ip_mreq)); + ip_mreq_opt = (struct ip_mreq *)optval; + imr_multiaddr.kind = IPv4; + ipv4_addr_to_wasi_ip4_addr(ip_mreq_opt->imr_multiaddr.s_addr, + &imr_multiaddr.addr.ip4); + error = __wasi_sock_set_ip_add_membership( + sockfd, &imr_multiaddr, ip_mreq_opt->imr_interface.s_addr); + HANDLE_ERROR(error); + return error; + case IP_DROP_MEMBERSHIP: + assert(optlen == sizeof(struct ip_mreq)); + ip_mreq_opt = (struct ip_mreq *)optval; + imr_multiaddr.kind = IPv4; + ipv4_addr_to_wasi_ip4_addr(ip_mreq_opt->imr_multiaddr.s_addr, + &imr_multiaddr.addr.ip4); + error = __wasi_sock_set_ip_drop_membership( + sockfd, &imr_multiaddr, ip_mreq_opt->imr_interface.s_addr); + HANDLE_ERROR(error); + return error; + case IP_TTL: + assert(optlen == sizeof(int)); + error = __wasi_sock_set_ip_ttl(sockfd, *(uint8_t *)optval); + HANDLE_ERROR(error); + return error; + case IP_MULTICAST_TTL: + assert(optlen == sizeof(int)); + error = + __wasi_sock_set_ip_multicast_ttl(sockfd, *(uint8_t *)optval); + HANDLE_ERROR(error); + return error; + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +static int +set_ipproto_ipv6_option(int sockfd, int optname, const void *optval, + socklen_t optlen) +{ + __wasi_errno_t error; + struct ipv6_mreq *ipv6_mreq_opt; + __wasi_addr_ip_t imr_multiaddr; + + switch (optname) { + case IPV6_V6ONLY: + assert(optlen == sizeof(int)); + error = __wasi_sock_set_ipv6_only(sockfd, *(bool *)optval); + HANDLE_ERROR(error); + return error; + case IPV6_MULTICAST_LOOP: + assert(optlen == sizeof(int)); + error = __wasi_sock_set_ip_multicast_loop(sockfd, true, + *(bool *)optval); + HANDLE_ERROR(error); + return error; + case IPV6_JOIN_GROUP: + assert(optlen == sizeof(struct ipv6_mreq)); + ipv6_mreq_opt = (struct ipv6_mreq *)optval; + imr_multiaddr.kind = IPv6; + ipv6_addr_to_wasi_ipv6_addr( + (uint16_t *)ipv6_mreq_opt->ipv6mr_multiaddr.s6_addr, + &imr_multiaddr.addr.ip6); + error = __wasi_sock_set_ip_add_membership( + sockfd, &imr_multiaddr, ipv6_mreq_opt->ipv6mr_interface); + HANDLE_ERROR(error); + return error; + case IPV6_LEAVE_GROUP: + assert(optlen == sizeof(struct ipv6_mreq)); + ipv6_mreq_opt = (struct ipv6_mreq *)optval; + imr_multiaddr.kind = IPv6; + ipv6_addr_to_wasi_ipv6_addr( + (uint16_t *)ipv6_mreq_opt->ipv6mr_multiaddr.s6_addr, + &imr_multiaddr.addr.ip6); + error = __wasi_sock_set_ip_drop_membership( + sockfd, &imr_multiaddr, ipv6_mreq_opt->ipv6mr_interface); + HANDLE_ERROR(error); + return error; + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} + +int +setsockopt(int sockfd, int level, int optname, const void *optval, + socklen_t optlen) +{ + __wasi_errno_t error; + + switch (level) { + case SOL_SOCKET: + return set_sol_socket_option(sockfd, optname, optval, optlen); + case IPPROTO_TCP: + return set_ipproto_tcp_option(sockfd, optname, optval, optlen); + case IPPROTO_IP: + return set_ipproto_ip_option(sockfd, optname, optval, optlen); + case IPPROTO_IPV6: + return set_ipproto_ipv6_option(sockfd, optname, optval, optlen); + default: + error = __WASI_ERRNO_NOTSUP; + HANDLE_ERROR(error); + return error; + } +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/build.sh b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/build.sh new file mode 100755 index 0000000..24f5ee6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -eo pipefail +CC="${CC:=/opt/wasi-sdk/bin/clang}" +files=("tcp_udp.c" "nslookup.c") + +for file in "${files[@]}" +do + echo $file + $CC \ + --target=wasm32-wasi-threads \ + -I../inc \ + ../src/wasi/wasi_socket_ext.c -pthread -ftls-model=local-exec \ + -Wl,--allow-undefined \ + -Wl,--strip-all,--no-entry \ + -Wl,--export=__heap_base \ + -Wl,--export=__data_end \ + -Wl,--shared-memory,--max-memory=10485760 \ + -Wl,--export=malloc \ + -Wl,--export=free \ + -o "${file%.*}.wasm" "$file" +done \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/manifest.json b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/manifest.json new file mode 100644 index 0000000..b0afd1d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/manifest.json @@ -0,0 +1,3 @@ +{ + "name": "WAMR lib-socket tests" +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/nslookup.c b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/nslookup.c new file mode 100644 index 0000000..8e64d06 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/nslookup.c @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#ifdef __wasi__ +#include +#include +#include +#include +#else +#include +#endif + +void +test_nslookup(int af) +{ + struct addrinfo *res; + int count = 0; + struct addrinfo hints; + char *url = "google-public-dns-a.google.com"; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = af; + hints.ai_socktype = SOCK_STREAM; + int ret = getaddrinfo(url, 0, &hints, &res); + assert(ret == 0); + struct addrinfo *address = res; + while (address) { + assert(address->ai_family == af); + assert(address->ai_socktype == SOCK_STREAM); + count++; + address = address->ai_next; + } + + assert(count > 0); + freeaddrinfo(res); +} + +void * +test_nslookup_mt(void *params) +{ + int *af = (int *)params; + test_nslookup(*af); + return NULL; +} + +int +main() +{ + int afs[] = { AF_INET, AF_INET6 }; + + for (int i = 0; i < sizeof(afs) / sizeof(afs[0]); i++) { + pthread_t th; + + printf("Testing %d in main thread...\n", afs[i]); + test_nslookup(afs[i]); + printf("Testing %d in a new thread...\n", afs[i]); + pthread_create(&th, NULL, test_nslookup_mt, &afs[i]); + pthread_join(th, NULL); + } + + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/tcp_udp.c b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/tcp_udp.c new file mode 100644 index 0000000..0ed0312 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-socket/test/tcp_udp.c @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#include +#include +#endif +#include +#include +#include + +#define SERVER_MSG "Message from server." +#define PORT 8989 + +pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + +int server_init_complete = 0; + +typedef struct { + struct sockaddr_storage addr; + socklen_t addr_len; + int sock; + int protocol; +} socket_info_t; + +void +wait_for_server(int wait_time_seconds) +{ + int res = 0; + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += wait_time_seconds; + + pthread_mutex_lock(&mut); + while (server_init_complete == 0) { + res = pthread_cond_timedwait(&cond, &mut, &ts); + if (res == ETIMEDOUT) + break; + } + pthread_mutex_unlock(&mut); + + assert(res == 0); +} + +void +notify_server_started() +{ + pthread_mutex_lock(&mut); + server_init_complete = 1; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&mut); +} + +socket_info_t +init_socket_addr(int family, int protocol) +{ + socket_info_t info; + + info.sock = socket(family, protocol, 0); + assert(info.sock != -1); + info.protocol = protocol; + + memset(&info.addr, 0, sizeof(info.addr)); + + if (family == AF_INET) { + struct sockaddr_in *addr = (struct sockaddr_in *)&info.addr; + addr->sin_family = AF_INET; + addr->sin_port = htons(PORT); + addr->sin_addr.s_addr = htonl(INADDR_LOOPBACK); + info.addr_len = sizeof(struct sockaddr_in); + } + else if (family == AF_INET6) { + struct sockaddr_in6 *addr = (struct sockaddr_in6 *)&info.addr; + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(PORT); + addr->sin6_addr = in6addr_loopback; + info.addr_len = sizeof(struct sockaddr_in6); + } + + return info; +} + +void * +server(void *arg) +{ + char buffer[sizeof(SERVER_MSG) + 1] = { 0 }; + struct sockaddr_storage client_addr; + socket_info_t *info = (socket_info_t *)arg; + struct sockaddr *server_addr = (struct sockaddr *)&info->addr; + int server_sock = info->sock; + + int optval = 1; + assert(setsockopt(server_sock, SOL_SOCKET, SO_REUSEADDR, &optval, + sizeof(optval)) + == 0); + + assert(bind(server_sock, server_addr, info->addr_len) == 0); + + if (info->protocol == SOCK_STREAM) + listen(server_sock, 1); + notify_server_started(); + + socklen_t addr_size = info->addr_len; + if (info->protocol == SOCK_STREAM) { + int client_sock = + accept(server_sock, (struct sockaddr *)&client_addr, &addr_size); + assert(client_sock >= 0); + assert(recv(client_sock, buffer, sizeof(buffer), 0) > 0); + strcpy(buffer, SERVER_MSG); + assert(send(client_sock, buffer, sizeof(buffer), 0) > 0); + assert(recv(client_sock, buffer, sizeof(buffer), 0) > 0); + } + else { + assert(recvfrom(server_sock, buffer, sizeof(buffer), 0, + (struct sockaddr *)&client_addr, &addr_size) + > 0); + strcpy(buffer, SERVER_MSG); + assert(sendto(server_sock, buffer, strlen(buffer), 0, + (struct sockaddr *)&client_addr, addr_size) + > 0); + assert(recvfrom(server_sock, buffer, sizeof(buffer), 0, + (struct sockaddr *)&client_addr, &addr_size) + > 0); + } + assert(close(server_sock) == 0); + + return NULL; +} + +void * +client(void *arg) +{ + char buffer[sizeof(SERVER_MSG) + 1]; + socket_info_t *info = (socket_info_t *)arg; + int sock = info->sock; + struct sockaddr *addr = (struct sockaddr *)&info->addr; + + wait_for_server(1); + + if (info->protocol == SOCK_STREAM) { + assert(connect(sock, addr, info->addr_len) != -1); + } + + assert(sendto(sock, "open", strlen("open"), 0, addr, info->addr_len) > 0); + assert(recv(sock, buffer, sizeof(buffer), 0) > 0); + assert(strncmp(buffer, SERVER_MSG, strlen(SERVER_MSG)) == 0); + assert(sendto(sock, "close", sizeof("close"), 0, addr, info->addr_len) > 0); + assert(close(sock) == 0); + + return NULL; +} + +void +test_protocol(int family, int protocol) +{ + pthread_t server_thread, client_thread; + socket_info_t server_info = init_socket_addr(family, protocol); + socket_info_t client_info = init_socket_addr(family, protocol); + + printf("Testing address family: %d protocol: %d\n", family, protocol); + + server_init_complete = 0; + + assert(pthread_create(&server_thread, NULL, server, (void *)&server_info) + == 0); + assert(pthread_create(&client_thread, NULL, client, (void *)&client_info) + == 0); + assert(pthread_join(server_thread, NULL) == 0); + assert(pthread_join(client_thread, NULL) == 0); +} + +int +main(int argc, char **argv) +{ + /* test tcp with ipv4 and ipv6 */ + test_protocol(AF_INET, SOCK_STREAM); + test_protocol(AF_INET6, SOCK_STREAM); + + /* test udp with ipv4 and ipv6 */ + test_protocol(AF_INET, SOCK_DGRAM); + test_protocol(AF_INET6, SOCK_DGRAM); + + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads.cmake b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads.cmake new file mode 100644 index 0000000..54d2ba9 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads.cmake @@ -0,0 +1,12 @@ +# Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIB_WASI_THREADS_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_LIB_WASI_THREADS=1 -DWASM_ENABLE_HEAP_AUX_STACK_ALLOCATION=1) + +include_directories(${LIB_WASI_THREADS_DIR}) + +set (LIB_WASI_THREADS_SOURCE + ${LIB_WASI_THREADS_DIR}/lib_wasi_threads_wrapper.c + ${LIB_WASI_THREADS_DIR}/tid_allocator.c) \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c new file mode 100644 index 0000000..aeaafce --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_log.h" +#include "thread_manager.h" +#include "tid_allocator.h" + +#if WASM_ENABLE_INTERP != 0 +#include "wasm_runtime.h" +#endif + +#if WASM_ENABLE_AOT != 0 +#include "aot_runtime.h" +#endif + +static const char *THREAD_START_FUNCTION = "wasi_thread_start"; +static korp_mutex thread_id_lock; +static TidAllocator tid_allocator; + +typedef struct { + /* app's entry function */ + wasm_function_inst_t start_func; + /* arg of the app's entry function */ + uint32 arg; + /* thread id passed to the app */ + int32 thread_id; +} ThreadStartArg; + +static int32 +allocate_thread_id() +{ + os_mutex_lock(&thread_id_lock); + int32 id = tid_allocator_get_tid(&tid_allocator); + os_mutex_unlock(&thread_id_lock); + + return id; +} + +void +deallocate_thread_id(int32 thread_id) +{ + os_mutex_lock(&thread_id_lock); + tid_allocator_release_tid(&tid_allocator, thread_id); + os_mutex_unlock(&thread_id_lock); +} + +static void * +thread_start(void *arg) +{ + wasm_exec_env_t exec_env = (wasm_exec_env_t)arg; + ThreadStartArg *thread_arg = exec_env->thread_arg; + uint32 argv[2]; + + wasm_exec_env_set_thread_info(exec_env); + argv[0] = thread_arg->thread_id; + argv[1] = thread_arg->arg; + + if (!wasm_runtime_call_wasm(exec_env, thread_arg->start_func, 2, argv)) { + /* Exception has already been spread during throwing */ + } + + // Routine exit + deallocate_thread_id(thread_arg->thread_id); + wasm_runtime_free(thread_arg); + exec_env->thread_arg = NULL; + + return NULL; +} + +static int32 +thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg) +{ + wasm_module_t module = wasm_exec_env_get_module(exec_env); + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasm_module_inst_t new_module_inst = NULL; + ThreadStartArg *thread_start_arg = NULL; + wasm_function_inst_t start_func; + int32 thread_id; + uint32 stack_size = 8192; + int32 ret = -1; + + bh_assert(module); + bh_assert(module_inst); + + stack_size = ((WASMModuleInstance *)module_inst)->default_wasm_stack_size; + + if (!(new_module_inst = wasm_runtime_instantiate_internal( + module, module_inst, exec_env, stack_size, 0, 0, NULL, 0))) + return -1; + + wasm_runtime_set_custom_data_internal( + new_module_inst, wasm_runtime_get_custom_data(module_inst)); + + if (!(wasm_cluster_dup_c_api_imports(new_module_inst, module_inst))) + goto thread_preparation_fail; + + wasm_native_inherit_contexts(new_module_inst, module_inst); + + start_func = + wasm_runtime_lookup_function(new_module_inst, THREAD_START_FUNCTION); + if (!start_func) { + LOG_ERROR("Failed to find thread start function %s", + THREAD_START_FUNCTION); + goto thread_preparation_fail; + } + + if (!(thread_start_arg = wasm_runtime_malloc(sizeof(ThreadStartArg)))) { + LOG_ERROR("Runtime args allocation failed"); + goto thread_preparation_fail; + } + + thread_start_arg->thread_id = thread_id = allocate_thread_id(); + if (thread_id < 0) { + LOG_ERROR("Failed to get thread identifier"); + goto thread_preparation_fail; + } + thread_start_arg->arg = start_arg; + thread_start_arg->start_func = start_func; + + ret = wasm_cluster_create_thread(exec_env, new_module_inst, false, 0, 0, + thread_start, thread_start_arg); + if (ret != 0) { + LOG_ERROR("Failed to spawn a new thread"); + goto thread_spawn_fail; + } + + return thread_id; + +thread_spawn_fail: + deallocate_thread_id(thread_id); + +thread_preparation_fail: + if (new_module_inst) + wasm_runtime_deinstantiate_internal(new_module_inst, true); + if (thread_start_arg) + wasm_runtime_free(thread_start_arg); + + return -1; +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(name, func_name, signature) \ + { name, func_name##_wrapper, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_lib_wasi_threads[] = { REG_NATIVE_FUNC( + "thread-spawn", thread_spawn, "(i)i") }; + +uint32 +get_lib_wasi_threads_export_apis(NativeSymbol **p_lib_wasi_threads_apis) +{ + *p_lib_wasi_threads_apis = native_symbols_lib_wasi_threads; + return sizeof(native_symbols_lib_wasi_threads) / sizeof(NativeSymbol); +} + +bool +lib_wasi_threads_init(void) +{ + if (0 != os_mutex_init(&thread_id_lock)) + return false; + + if (!tid_allocator_init(&tid_allocator)) { + os_mutex_destroy(&thread_id_lock); + return false; + } + + return true; +} + +void +lib_wasi_threads_destroy(void) +{ + tid_allocator_deinit(&tid_allocator); + os_mutex_destroy(&thread_id_lock); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/build.sh b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/build.sh new file mode 100755 index 0000000..1ea95cb --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# +# Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +set -eo pipefail +CC=${CC:=/opt/wasi-sdk/bin/clang} +WAMR_DIR=../../../../.. + +show_usage() { + echo "Usage: $0 [--sysroot PATH_TO_SYSROOT]" + echo "--sysroot PATH_TO_SYSROOT specify to build with custom sysroot for wasi-libc" +} + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --sysroot) + sysroot_path="$2" + shift + shift + ;; + --help) + show_usage + exit + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +rm -rf *.wasm +rm -rf *.aot + +for test_c in *.c; do + test_wasm="$(basename $test_c .c).wasm" + + if [[ -n "$sysroot_path" ]]; then + if [ ! -d "$sysroot_path" ]; then + echo "Directory $sysroot_path doesn't exist. Aborting" + exit 1 + fi + sysroot_command="--sysroot $sysroot_path" + fi + + echo "Compiling $test_c to $test_wasm" + $CC \ + -target wasm32-wasi-threads \ + -O2 \ + -Wall \ + -pthread \ + -z stack-size=32768 \ + -Wl,--export=__heap_base \ + -Wl,--export=__data_end \ + -Wl,--shared-memory,--max-memory=1966080 \ + -Wl,--export=wasi_thread_start \ + -Wl,--export=malloc \ + -Wl,--export=free \ + -Wl,--export=test \ + $sysroot_command \ + $test_c -o $test_wasm +done diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/errorcheck_mutex_stress_test.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/errorcheck_mutex_stress_test.c new file mode 100644 index 0000000..946e8bd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/errorcheck_mutex_stress_test.c @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include "mutex_common.h" + +void +test() +{ + pthread_mutex_t mutex; + + // Set mutex type to errorcheck. This type provides some additional checks + // (for example returns EDEADLK instead of deadlocking in some cases) + pthread_mutexattr_t mutex_attr; + pthread_mutexattr_init(&mutex_attr); + pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_ERRORCHECK); + + pthread_mutex_init(&mutex, &mutex_attr); + pthread_mutexattr_destroy(&mutex_attr); + + run_common_tests(&mutex); + fprintf(stderr, "Errorcheck mutex test is completed\n"); + pthread_mutex_destroy(&mutex); +} + +int +main() +{ + test(); + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/manifest.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/manifest.json new file mode 100644 index 0000000..bb91ad0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/manifest.json @@ -0,0 +1,3 @@ +{ + "name": "lib-wasi-threads stress tests" +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/mutex_common.h b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/mutex_common.h new file mode 100644 index 0000000..d57ff7d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/mutex_common.h @@ -0,0 +1,229 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef MUTEX_COMMON_H +#define MUTEX_COMMON_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum Constants { + NUM_ITER = 250000, + NUM_THREADS = 12, + NUM_RETRY = 8, + RETRY_SLEEP_TIME_US = 1000, +}; + +// We're counting how many times each thread was called using this array +// Main thread is also counted here so we need to make arrays bigger +typedef struct { + int tids[NUM_THREADS + 1]; + int calls[NUM_THREADS + 1]; +} StatCollector; + +typedef struct { + pthread_mutex_t *mutex; + StatCollector stat; + int counter; + bool is_sleeping; +} MutexCounter; + +// This enum defines whether thread should sleep to increase contention +enum SleepState { + NON_SLEEP = 0, + SLEEP = 1, +}; + +void +mutex_counter_init(MutexCounter *mutex_counter, pthread_mutex_t *mutex, + enum SleepState is_sleeping) +{ + memset(mutex_counter, 0, sizeof(*mutex_counter)); + mutex_counter->mutex = mutex; + mutex_counter->is_sleeping = is_sleeping; +} + +// This function spawns the thread using exponential retries if it receives +// EAGAIN +static inline void +spawn_thread(pthread_t *tid, void *func, void *arg) +{ + int status_code = -1; + int timeout_us = RETRY_SLEEP_TIME_US; + for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) { + status_code = pthread_create(tid, NULL, (void *(*)(void *))func, arg); + assert(status_code == 0 || status_code == EAGAIN); + if (status_code == EAGAIN) { + usleep(timeout_us); + timeout_us *= 2; + } + } + + assert(status_code == 0 && "Thread creation should succeed"); +} + +// This function adds tid to our stat +static inline void +add_to_stat(StatCollector *stat, int tid) +{ + int tid_num = 0; + for (; tid_num < NUM_THREADS + 1 && stat->tids[tid_num] != 0; ++tid_num) { + if (stat->tids[tid_num] == tid) { + stat->calls[tid_num]++; + return; + } + } + + assert(tid_num < NUM_THREADS + 1); + stat->tids[tid_num] = tid; + stat->calls[tid_num] = 1; +} + +// This function prints number of calls by TID +static inline void +print_stat(StatCollector *stat) +{ + fprintf(stderr, "Thread calls count by TID\n"); + for (int i = 0; i < NUM_THREADS + 1; ++i) { + if (stat->tids[i] != 0) { + fprintf(stderr, "TID: %d; Calls: %d\n", stat->tids[i], + stat->calls[i]); + } + } +} + +// This function is run by the threads, it increases counter in a loop and then +// sleeps after unlocking the mutex to provide better contention +static inline void * +inc_shared_variable(void *arg) +{ + MutexCounter *mutex_counter = (MutexCounter *)(arg); + int sleep_us = 0; + while (!pthread_mutex_lock(mutex_counter->mutex) + && mutex_counter->counter < NUM_ITER) { + mutex_counter->counter++; + add_to_stat(&mutex_counter->stat, (int)(pthread_self())); + if (mutex_counter->is_sleeping) { + sleep_us = rand() % 1000; + } + + assert(pthread_mutex_unlock(mutex_counter->mutex) == 0 + && "Should be able to unlock a mutex"); + if (mutex_counter->is_sleeping) { + usleep(sleep_us); + } + } + + assert(mutex_counter->counter == NUM_ITER); + assert(pthread_mutex_unlock(mutex_counter->mutex) == 0 + && "Should be able to unlock the mutex after test execution"); + + return NULL; +} + +// Locking and unlocking a mutex in a single thread. +static inline void * +same_thread_lock_unlock_test(void *mutex) +{ + for (int i = 0; i < NUM_ITER; ++i) { + assert(pthread_mutex_lock(mutex) == 0 + && "Main thread should be able to lock a mutex"); + assert(pthread_mutex_unlock(mutex) == 0 + && "Main thread should be able to unlock a mutex"); + } + + return NULL; +} + +// This function spawns a thread that locks and unlocks a mutex `NUM_ITER` times +// in a row +static inline void +same_non_main_thread_lock_unlock_test(pthread_mutex_t *mutex) +{ + pthread_t tid = 0; + spawn_thread(&tid, same_thread_lock_unlock_test, mutex); + + assert(tid != 0 && "TID can't be 0 after successful thread creation"); + assert(pthread_join(tid, NULL) == 0 + && "Thread should be joined successfully"); +} + +// This function checks basic contention between main and non-main thread +// increasing the shared variable +static inline void +two_threads_inc_test(pthread_mutex_t *mutex) +{ + MutexCounter mutex_counter; + mutex_counter_init(&mutex_counter, mutex, false); + + pthread_t tid = 0; + spawn_thread(&tid, inc_shared_variable, &mutex_counter); + + assert(tid != 0 && "TID can't be 0 after successful thread creation"); + inc_shared_variable(&mutex_counter); + assert(pthread_join(tid, NULL) == 0 + && "Thread should be joined without errors"); + assert(mutex_counter.counter == NUM_ITER); +} + +// This function creates number of threads specified by NUM_THREADS and run +// concurrent increasing of shared variable +static inline void +max_threads_inc_test(pthread_mutex_t *mutex, int threads_num, + enum SleepState is_sleeping) +{ + MutexCounter mutex_counter; + mutex_counter_init(&mutex_counter, mutex, is_sleeping); + + pthread_t tids[threads_num]; + for (int i = 0; i < threads_num; ++i) { + spawn_thread(&tids[i], inc_shared_variable, &mutex_counter); + } + + inc_shared_variable(&mutex_counter); + + for (int i = 0; i < threads_num; ++i) { + assert(pthread_join(tids[i], NULL) == 0 + && "Thread should be joined without errors"); + } + + print_stat(&mutex_counter.stat); +} + +// This function just runs all the tests described above +static inline void +run_common_tests(pthread_mutex_t *mutex) +{ + srand(time(NULL)); + + fprintf(stderr, "Starting same_thread_lock_unlock_test test\n"); + same_thread_lock_unlock_test(mutex); + fprintf(stderr, "Finished same_thread_lock_unlock_test test\n"); + + fprintf(stderr, "Starting same_non_main_thread_lock_unlock_test test\n"); + same_non_main_thread_lock_unlock_test(mutex); + fprintf(stderr, "Finished same_non_main_thread_lock_unlock_test test\n"); + + fprintf(stderr, "Starting two_threads_inc_test test\n"); + two_threads_inc_test(mutex); + fprintf(stderr, "Finished two_threads_inc_test test\n"); + + fprintf(stderr, "Starting max_threads_inc_test_sleep test\n"); + max_threads_inc_test(mutex, NUM_THREADS, SLEEP); + fprintf(stderr, "Finished concurrent_inc sleep test\n"); + + fprintf(stderr, "Starting max_threads_inc_test_non_sleep test\n"); + max_threads_inc_test(mutex, NUM_THREADS, NON_SLEEP); + fprintf(stderr, "Finished max_threads_inc_test test\n"); +} + +#endif // MUTEX_COMMON_H diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/normal_mutex_stress_test.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/normal_mutex_stress_test.c new file mode 100644 index 0000000..c7ffae2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/normal_mutex_stress_test.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include "mutex_common.h" + +void +test() +{ + pthread_mutex_t mutex; + pthread_mutex_init(&mutex, NULL); + + run_common_tests(&mutex); + + fprintf(stderr, "Normal mutex test is completed\n"); + pthread_mutex_destroy(&mutex); +} + +int +main() +{ + test(); + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/recursive_mutex_stress_test.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/recursive_mutex_stress_test.c new file mode 100644 index 0000000..5874372 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/recursive_mutex_stress_test.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include "mutex_common.h" + +void +multiple_same_thread_lock(void *mutex) +{ + for (int i = 0; i < 100; ++i) { + assert(pthread_mutex_lock(mutex) == 0 + && "Recursive mutex should allow multiple locking"); + } + + for (int i = 0; i < 100; ++i) { + assert(pthread_mutex_unlock(mutex) == 0 + && "Recursive mutex should allow multiple unlocking"); + } +} + +void * +same_thread_multiple_rec_mutex_lock(void *mutex) +{ + for (int i = 0; i < NUM_ITER; ++i) { + multiple_same_thread_lock(mutex); + } + + return NULL; +} + +void +test() +{ + pthread_mutex_t mutex; + + // Set mutex type to recursive. This type allows multiple locking and + // unlocking within the same thread + pthread_mutexattr_t mutex_attr; + pthread_mutexattr_init(&mutex_attr); + pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); + + pthread_mutex_init(&mutex, &mutex_attr); + pthread_mutexattr_destroy(&mutex_attr); + + run_common_tests(&mutex); + + fprintf(stderr, "Starting same_thread_multiple_rec_mutex_lock test\n"); + same_thread_multiple_rec_mutex_lock(&mutex); + fprintf(stderr, "Finished same_thread_multiple_rec_mutex_lock test\n"); + + fprintf(stderr, "Starting same_thread_multiple_rec_mutex_lock test in " + "non-main thread\n"); + pthread_t tid; + spawn_thread(&tid, same_thread_multiple_rec_mutex_lock, &mutex); + assert(pthread_join(tid, NULL) == 0 + && "Non-main thread should be joined successfully"); + fprintf(stderr, "Finished same_thread_multiple_rec_mutex_lock test in " + "non-main thread\n"); + + fprintf(stderr, "Recursive mutex test is completed\n"); + pthread_mutex_destroy(&mutex); +} + +int +main() +{ + test(); + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/spawn_stress_test.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/spawn_stress_test.c new file mode 100644 index 0000000..8cb61a2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/spawn_stress_test.c @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +unsigned prime_numbers_count = 0; + +bool +is_prime(unsigned int num) +{ + for (unsigned int i = 2; i <= (unsigned int)(sqrt(num)); ++i) { + if (num % i == 0) { + return false; + } + } + + return true; +} + +void * +check_if_prime(void *value) +{ + unsigned int *num = (unsigned int *)(value); + usleep(10000); + if (is_prime(*num)) { + __atomic_fetch_add(&prime_numbers_count, 1, __ATOMIC_SEQ_CST); + } + return NULL; +} + +unsigned int +validate(int iter_num) +{ + unsigned int counter = 0; + for (unsigned int i = 2; i <= iter_num; ++i) { + counter += is_prime(i); + } + + return counter; +} + +void +spawn_thread(pthread_t *thread, int retry_time_us, int retry_num, + unsigned int *arg) +{ + int status_code = -1; + int timeout_us = retry_time_us; + for (int tries = 0; status_code != 0 && tries < retry_num; ++tries) { + status_code = pthread_create(thread, NULL, &check_if_prime, arg); + assert(status_code == 0 || status_code == EAGAIN); + if (status_code == EAGAIN) { + usleep(timeout_us); + timeout_us *= 2; + } + } + + assert(status_code == 0 && "Thread creation should succeed"); +} + +void +test(int iter_num, int retry_num, int max_threads_num, int retry_time_us) +{ + pthread_t threads[max_threads_num]; + unsigned int args[max_threads_num]; + double percentage = 0.1; + + for (unsigned int factorised_number = 2; factorised_number < iter_num; + ++factorised_number) { + if (factorised_number > iter_num * percentage) { + fprintf(stderr, "Stress test is %d%% finished\n", + (unsigned int)(percentage * 100)); + percentage += 0.1; + } + + unsigned int thread_num = factorised_number % max_threads_num; + if (threads[thread_num] != 0) { + assert(pthread_join(threads[thread_num], NULL) == 0); + } + + args[thread_num] = factorised_number; + + usleep(retry_time_us); + spawn_thread(&threads[thread_num], retry_time_us, retry_num, + &args[thread_num]); + assert(threads[thread_num] != 0); + } + + for (int i = 0; i < max_threads_num; ++i) { + assert(threads[i] == 0 || pthread_join(threads[i], NULL) == 0); + } + + // Check the test results + assert( + prime_numbers_count == validate(iter_num) + && "Answer mismatch between tested code and reference implementation"); + + fprintf(stderr, "Stress test finished successfully\n"); +} + +enum DEFAULT_PARAMETERS { + ITER_NUM = 20000, + RETRY_NUM = 8, + MAX_THREADS_NUM = 12, + RETRY_SLEEP_TIME_US = 2000, +}; + +int +main(int argc, char **argv) +{ + test(ITER_NUM, RETRY_NUM, MAX_THREADS_NUM, RETRY_SLEEP_TIME_US); + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/stress_test_threads_creation.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/stress_test_threads_creation.c new file mode 100644 index 0000000..c4c2d28 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/stress-test/stress_test_threads_creation.c @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +int threads_executed = 0; +unsigned int threads_creation_tried = 0; +unsigned int threads_in_use = 0; + +void * +thread_func(void *arg) +{ + (void)(arg); + __atomic_fetch_add(&threads_executed, 1, __ATOMIC_RELAXED); + __atomic_fetch_sub(&threads_in_use, 1, __ATOMIC_SEQ_CST); + return NULL; +} + +void +spawn_thread(pthread_t *thread, int retry_time, int iter_num) +{ + int status_code = -1; + int timeout_us = retry_time; + for (int tries = 0; status_code != 0 && tries < iter_num; ++tries) { + status_code = pthread_create(thread, NULL, &thread_func, NULL); + __atomic_fetch_add(&threads_creation_tried, 1, __ATOMIC_RELAXED); + + assert(status_code == 0 || status_code == EAGAIN); + if (status_code == EAGAIN) { + usleep(timeout_us); + timeout_us *= 2; + } + } + + assert(status_code == 0 && "Thread creation should succeed"); +} + +void +test(int iter_num, int max_threads_num, int retry_num, int retry_time_us) +{ + double percentage = 0.1; + int second_us = 1000 * 1000 * 1000; // 1 second in us + + for (int iter = 0; iter < iter_num; ++iter) { + if (iter > iter_num * percentage) { + fprintf(stderr, "Spawning stress test is %d%% finished\n", + (unsigned int)(percentage * 100)); + percentage += 0.1; + } + while (__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) + == max_threads_num) { + usleep(100); + } + + __atomic_fetch_add(&threads_in_use, 1, __ATOMIC_SEQ_CST); + pthread_t tmp; + spawn_thread(&tmp, retry_time_us, iter_num); + pthread_detach(tmp); + } + + while ((__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) != 0)) { + // Casting to int* to supress compiler warning + __builtin_wasm_memory_atomic_wait32((int *)(&threads_in_use), 0, + second_us); + } + + assert(__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) == 0); + + // Validation + assert(threads_creation_tried >= threads_executed + && "Test executed more threads than were created"); + assert((1. * threads_creation_tried) / threads_executed < 2.5 + && "Ensuring that we're retrying thread creation less than 2.5 " + "times on average "); + + fprintf(stderr, + "Spawning stress test finished successfully executed %d threads " + "with retry ratio %f\n", + threads_creation_tried, + (1. * threads_creation_tried) / threads_executed); +} + +enum DEFAULT_PARAMETERS { + ITER_NUM = 50000, + RETRY_NUM = 8, + MAX_NUM_THREADS = 12, + RETRY_SLEEP_TIME_US = 4000, +}; + +int +main(int argc, char **argv) +{ + test(ITER_NUM, MAX_NUM_THREADS, RETRY_NUM, RETRY_SLEEP_TIME_US); + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/build.sh b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/build.sh new file mode 100755 index 0000000..608dd22 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/build.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# +# Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +set -eo pipefail +CC=${CC:=/opt/wasi-sdk/bin/clang} +WAMR_DIR=../../../../.. + +show_usage() { + echo "Usage: $0 [--sysroot PATH_TO_SYSROOT]" + echo "--sysroot PATH_TO_SYSROOT specify to build with custom sysroot for wasi-libc" +} + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --sysroot) + sysroot_path="$2" + shift + shift + ;; + --help) + show_usage + exit + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Stress tests names +thread_start_file_exclusions=("linear_memory_size_update.wasm") + +rm -rf *.wasm +rm -rf *.aot + +for test_c in *.c; do + test_wasm="$(basename $test_c .c).wasm" + + if [[ " ${thread_start_file_exclusions[@]} " =~ " ${test_wasm} " ]] ; then + thread_start_file="" + else + thread_start_file=$WAMR_DIR/samples/wasi-threads/wasm-apps/wasi_thread_start.S + fi + + if [[ -n "$sysroot_path" ]]; then + if [ ! -d "$sysroot_path" ]; then + echo "Directory $sysroot_path doesn't exist. Aborting" + exit 1 + fi + sysroot_command="--sysroot $sysroot_path" + fi + + echo "Compiling $test_c to $test_wasm" + $CC \ + -target wasm32-wasi-threads \ + -O2 \ + -pthread -ftls-model=local-exec \ + -z stack-size=32768 \ + -Wl,--export=__heap_base \ + -Wl,--export=__data_end \ + -Wl,--shared-memory,--max-memory=1966080 \ + -Wl,--export=wasi_thread_start \ + -Wl,--export=malloc \ + -Wl,--export=free \ + -I $WAMR_DIR/samples/wasi-threads/wasm-apps \ + $sysroot_command \ + $thread_start_file \ + $test_c -o $test_wasm +done \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/common.h b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/common.h new file mode 100644 index 0000000..01ca932 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/common.h @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include + +#if USE_CUSTOM_SYNC_PRIMITIVES != 0 +#include "sync_primitives.h" +#else +#include +#endif + +#include "wasi_thread_start.h" + +typedef enum { + BLOCKING_TASK_BUSY_WAIT, + BLOCKING_TASK_ATOMIC_WAIT, + BLOCKING_TASK_POLL_ONEOFF +} blocking_task_type_t; + +/* Parameter to change test behavior */ +static bool termination_by_trap; +static bool termination_in_main_thread; +static blocking_task_type_t blocking_task_type; + +#define NUM_THREADS 3 +static pthread_barrier_t barrier; + +typedef struct { + start_args_t base; + bool throw_exception; +} shared_t; + +void +run_long_task() +{ + if (blocking_task_type == BLOCKING_TASK_BUSY_WAIT) { + for (;;) { + } + } + else if (blocking_task_type == BLOCKING_TASK_ATOMIC_WAIT) { + __builtin_wasm_memory_atomic_wait32(0, 0, -1); + } + else { + sleep(UINT_MAX); + } +} + +void +start_job() +{ + /* Wait for all threads (including the main thread) to be ready */ + pthread_barrier_wait(&barrier); + run_long_task(); /* Task to be interrupted */ + assert(false && "Thread termination test failed"); +} + +void +terminate_process() +{ + /* Wait for all threads (including the main thread) to be ready */ + pthread_barrier_wait(&barrier); + + if (termination_by_trap) + __builtin_trap(); + else + __wasi_proc_exit(33); +} + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + shared_t *data = (shared_t *)start_arg; + + if (data->throw_exception) { + terminate_process(); + } + else { + start_job(); + } +} + +void +test_termination(bool trap, bool main, blocking_task_type_t task_type) +{ + termination_by_trap = trap; + termination_in_main_thread = main; + blocking_task_type = task_type; + + int thread_id = -1, i; + shared_t data[NUM_THREADS] = { 0 }; + assert(pthread_barrier_init(&barrier, NULL, NUM_THREADS + 1) == 0 + && "Failed to init barrier"); + + for (i = 0; i < NUM_THREADS; i++) { + /* No graceful memory free to simplify the test */ + assert(start_args_init(&data[i].base) + && "Failed to allocate thread's stack"); + } + + /* Create a thread that forces termination through trap or `proc_exit` */ + data[0].throw_exception = !termination_in_main_thread; + thread_id = __wasi_thread_spawn(&data[0]); + assert(thread_id > 0 && "Failed to create thread"); + + /* Create two additional threads to test exception propagation */ + data[1].throw_exception = false; + thread_id = __wasi_thread_spawn(&data[1]); + assert(thread_id > 0 && "Failed to create thread"); + data[2].throw_exception = false; + thread_id = __wasi_thread_spawn(&data[2]); + assert(thread_id > 0 && "Failed to create thread"); + + if (termination_in_main_thread) { + terminate_process(); + } + else { + start_job(); + } +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/create_threads_until_limit.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/create_threads_until_limit.c new file mode 100644 index 0000000..94ffa0f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/create_threads_until_limit.c @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include +#include + +#include "wasi_thread_start.h" + +enum CONSTANTS { + MAX_NUM_THREADS = 4, /* Should be the same as "--max-threads" */ + NUM_RETRY = 5, + SECOND = 1000 * 1000 * 1000, /* 1 second */ + TIMEOUT = 10LL * SECOND +}; + +int g_count = 0; + +typedef struct { + start_args_t base; + int th_ready; + int th_continue; + int th_done; + bool no_ops; +} shared_t; + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + shared_t *data = (shared_t *)start_arg; + + if (data->no_ops) { + __builtin_wasm_memory_atomic_wait32(NULL, 0, 2 * SECOND); + return; + } + + __atomic_store_n(&data->th_ready, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data->th_ready, 1); + + if (__builtin_wasm_memory_atomic_wait32(&data->th_continue, 0, TIMEOUT) + == 2) { + assert(false && "Wait should not time out"); + } + + __atomic_fetch_add(&g_count, 1, __ATOMIC_SEQ_CST); + + __atomic_store_n(&data->th_done, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data->th_done, 1); +} + +int +main(int argc, char **argv) +{ + shared_t data[MAX_NUM_THREADS] = { 0 }; + int thread_ids[MAX_NUM_THREADS]; + + for (int i = 0; i < MAX_NUM_THREADS; i++) { + assert(start_args_init(&data[i].base)); + thread_ids[i] = __wasi_thread_spawn(&data[i]); + printf("Thread created with id=%d\n", thread_ids[i]); + ASSERT_VALID_TID(thread_ids[i]); + + for (int j = 0; j < i; j++) { + assert(thread_ids[i] != thread_ids[j] && "Duplicated TIDs"); + } + + if (__builtin_wasm_memory_atomic_wait32(&data[i].th_ready, 0, TIMEOUT) + == 2) { + assert(false && "Wait should not time out"); + } + } + + printf("Attempt to create thread when not possible\n"); + shared_t data_fail = { 0 }; + assert(start_args_init(&data_fail.base)); + int thread_id = __wasi_thread_spawn(&data_fail); + start_args_deinit(&data_fail.base); + assert(thread_id < 0 && "Thread creation should fail"); + + printf("Unlock created threads\n"); + for (int i = 0; i < MAX_NUM_THREADS; i++) { + __atomic_store_n(&data[i].th_continue, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data[i].th_continue, 1); + } + + printf("Wait for threads to finish\n"); + for (int i = 0; i < MAX_NUM_THREADS; i++) { + if (__builtin_wasm_memory_atomic_wait32(&data[i].th_done, 0, TIMEOUT) + == 2) { + assert(false && "Wait should not time out"); + } + + start_args_deinit(&data[i].base); + } + + printf("Value of count after update: %d\n", g_count); + assert(g_count == (MAX_NUM_THREADS) + && "Global count not updated correctly"); + + /* --------------------------------------------------- */ + + printf("Create new threads without waiting from them to finish\n"); + shared_t data_no_join[MAX_NUM_THREADS] = { 0 }; + for (int i = 0; i < MAX_NUM_THREADS; i++) { + /* No graceful memory free to simplify the test */ + assert(start_args_init(&data_no_join[i].base)); + data_no_join[i].no_ops = true; + + int thread_id = -1; + for (int j = 0; j < NUM_RETRY && thread_id < 0; j++) { + thread_id = __wasi_thread_spawn(&data_no_join[i]); + if (thread_id < 0) + __builtin_wasm_memory_atomic_wait32(NULL, 0, SECOND); + } + + printf("Thread created with id=%d\n", thread_id); + assert(thread_id > 0 && "Thread creation should succeed"); + } + + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/global_atomic.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/global_atomic.c new file mode 100644 index 0000000..7e1e8e0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/global_atomic.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include +#include + +#include "wasi_thread_start.h" + +enum CONSTANTS { + NUM_THREADS = 4, + NUM_ITER = 1000, + SECOND = 1000 * 1000 * 1000, /* 1 second */ + TIMEOUT = 10LL * SECOND +}; + +int g_count = 0; + +typedef struct { + start_args_t base; + int th_done; +} shared_t; + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + shared_t *data = (shared_t *)start_arg; + + for (int i = 0; i < NUM_ITER; i++) + __atomic_fetch_add(&g_count, 1, __ATOMIC_SEQ_CST); + + __atomic_store_n(&data->th_done, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data->th_done, 1); +} + +int +main(int argc, char **argv) +{ + shared_t data[NUM_THREADS] = { 0 }; + int thread_ids[NUM_THREADS]; + + for (int i = 0; i < NUM_THREADS; i++) { + assert(start_args_init(&data[i].base)); + thread_ids[i] = __wasi_thread_spawn(&data[i]); + ASSERT_VALID_TID(thread_ids[i]); + } + + printf("Wait for threads to finish\n"); + for (int i = 0; i < NUM_THREADS; i++) { + if (__builtin_wasm_memory_atomic_wait32(&data[i].th_done, 0, TIMEOUT) + == 2) { + assert(false && "Wait should not time out"); + } + + start_args_deinit(&data[i].base); + } + + printf("Value of count after update: %d\n", g_count); + assert(g_count == (NUM_THREADS * NUM_ITER) + && "Global count not updated correctly"); + + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/global_lock.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/global_lock.c new file mode 100644 index 0000000..fb33802 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/global_lock.c @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include +#include + +#if USE_CUSTOM_SYNC_PRIMITIVES != 0 +#include "sync_primitives.h" +#else +#include +#endif + +#include "wasi_thread_start.h" + +enum CONSTANTS { + NUM_THREADS = 4, + NUM_ITER = 200, + SECOND = 1000 * 1000 * 1000, /* 1 second */ + TIMEOUT = 10LL * SECOND +}; + +pthread_mutex_t mutex; +int g_count = 0; + +typedef struct { + start_args_t base; + int th_done; +} shared_t; + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + shared_t *data = (shared_t *)start_arg; + + for (int i = 0; i < NUM_ITER; i++) { + pthread_mutex_lock(&mutex); + g_count++; + pthread_mutex_unlock(&mutex); + } + + __atomic_store_n(&data->th_done, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data->th_done, 1); +} + +int +main(int argc, char **argv) +{ + shared_t data[NUM_THREADS] = { 0 }; + int thread_ids[NUM_THREADS]; + + assert(pthread_mutex_init(&mutex, NULL) == 0 && "Failed to init mutex"); + + for (int i = 0; i < NUM_THREADS; i++) { + assert(start_args_init(&data[i].base)); + thread_ids[i] = __wasi_thread_spawn(&data[i]); + ASSERT_VALID_TID(thread_ids[i]); + } + + printf("Wait for threads to finish\n"); + for (int i = 0; i < NUM_THREADS; i++) { + if (__builtin_wasm_memory_atomic_wait32(&data[i].th_done, 0, TIMEOUT) + == 2) { + assert(false && "Wait should not time out"); + } + + start_args_deinit(&data[i].base); + } + + printf("Value of count after update: %d\n", g_count); + assert(g_count == (NUM_THREADS * NUM_ITER) + && "Global count not updated correctly"); + + assert(pthread_mutex_destroy(&mutex) == 0 && "Failed to destroy mutex"); + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/linear_memory_size_update.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/linear_memory_size_update.c new file mode 100644 index 0000000..9dcb34a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/linear_memory_size_update.c @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include +#include + +typedef enum { + APP_STARTED, + THREAD_STARTED, + MEMORY_ALLOCATED, +} app_state_t; +typedef struct { + + pthread_cond_t cond; + pthread_mutex_t mutex; + app_state_t state; + char *data; +} context_t; + +void +context_init(context_t *ctx) +{ + pthread_cond_init(&ctx->cond, NULL); + pthread_mutex_init(&ctx->mutex, NULL); + ctx->state = APP_STARTED; + ctx->data = NULL; +} + +void +context_destroy(context_t *ctx) +{ + pthread_cond_destroy(&ctx->cond); + pthread_mutex_destroy(&ctx->mutex); + if (ctx->data) { + free(ctx->data); + } +} + +void +context_set_state(context_t *ctx, app_state_t state) +{ + pthread_mutex_lock(&ctx->mutex); + ctx->state = state; + pthread_mutex_unlock(&ctx->mutex); + pthread_cond_signal(&ctx->cond); +} + +void +context_wait_for_state(context_t *ctx, app_state_t state) +{ + pthread_mutex_lock(&ctx->mutex); + while (ctx->state != state) { + pthread_cond_wait(&ctx->cond, &ctx->mutex); + } + pthread_mutex_unlock(&ctx->mutex); +} + +void * +fnc(void *p) +{ + context_t *ctx = (context_t *)p; + context_set_state(ctx, THREAD_STARTED); + + context_wait_for_state(ctx, MEMORY_ALLOCATED); + + // trigger memory.copy + __builtin_memcpy(ctx->data + 512 * 1024, ctx->data + 1024, 1024); + + return NULL; +} + +int +main() +{ + context_t ctx; + context_init(&ctx); + + pthread_t th; + pthread_create(&th, NULL, fnc, &ctx); + + context_wait_for_state(&ctx, THREAD_STARTED); + + // trigger memory.grow + ctx.data = calloc(1024 * 1024, 1); + + context_set_state(&ctx, MEMORY_ALLOCATED); + + pthread_join(th, NULL); + + context_destroy(&ctx); + + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_busy.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_busy.c new file mode 100644 index 0000000..19d3ec2 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_busy.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(false, true, BLOCKING_TASK_BUSY_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_busy.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_busy.json new file mode 100644 index 0000000..5370f66 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_busy.json @@ -0,0 +1,3 @@ +{ + "exit_code": 33 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_sleep.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_sleep.c new file mode 100644 index 0000000..a667e91 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_sleep.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(false, true, BLOCKING_TASK_POLL_ONEOFF); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_sleep.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_sleep.json new file mode 100644 index 0000000..5370f66 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_sleep.json @@ -0,0 +1,3 @@ +{ + "exit_code": 33 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_wait.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_wait.c new file mode 100644 index 0000000..dc8615a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_wait.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(false, true, BLOCKING_TASK_ATOMIC_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_wait.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_wait.json new file mode 100644 index 0000000..5370f66 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_proc_exit_wait.json @@ -0,0 +1,3 @@ +{ + "exit_code": 33 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_busy.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_busy.c new file mode 100644 index 0000000..bb0ac8f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_busy.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(true, true, BLOCKING_TASK_BUSY_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_busy.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_busy.json new file mode 100644 index 0000000..07689a1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_busy.json @@ -0,0 +1,3 @@ +{ + "exit_code": 1 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_sleep.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_sleep.c new file mode 100644 index 0000000..a2c2488 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_sleep.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(true, true, BLOCKING_TASK_POLL_ONEOFF); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_sleep.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_sleep.json new file mode 100644 index 0000000..07689a1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_sleep.json @@ -0,0 +1,3 @@ +{ + "exit_code": 1 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_wait.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_wait.c new file mode 100644 index 0000000..0904f34 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_wait.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(true, true, BLOCKING_TASK_ATOMIC_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_wait.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_wait.json new file mode 100644 index 0000000..07689a1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/main_trap_wait.json @@ -0,0 +1,3 @@ +{ + "exit_code": 1 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/manifest.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/manifest.json new file mode 100644 index 0000000..cd2cc76 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/manifest.json @@ -0,0 +1,3 @@ +{ + "name": "lib-wasi-threads tests" +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_busy.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_busy.c new file mode 100644 index 0000000..71fdcb8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_busy.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(false, false, BLOCKING_TASK_BUSY_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_busy.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_busy.json new file mode 100644 index 0000000..5370f66 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_busy.json @@ -0,0 +1,3 @@ +{ + "exit_code": 33 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_sleep.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_sleep.c new file mode 100644 index 0000000..14352cf --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_sleep.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(false, false, BLOCKING_TASK_POLL_ONEOFF); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_sleep.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_sleep.json new file mode 100644 index 0000000..5370f66 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_sleep.json @@ -0,0 +1,3 @@ +{ + "exit_code": 33 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_wait.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_wait.c new file mode 100644 index 0000000..0963aa0 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_wait.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(false, false, BLOCKING_TASK_ATOMIC_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_wait.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_wait.json new file mode 100644 index 0000000..5370f66 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_proc_exit_wait.json @@ -0,0 +1,3 @@ +{ + "exit_code": 33 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_busy.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_busy.c new file mode 100644 index 0000000..b3e3af7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_busy.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(true, false, BLOCKING_TASK_BUSY_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_busy.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_busy.json new file mode 100644 index 0000000..07689a1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_busy.json @@ -0,0 +1,3 @@ +{ + "exit_code": 1 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_sleep.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_sleep.c new file mode 100644 index 0000000..a68ae8b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_sleep.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(true, false, BLOCKING_TASK_POLL_ONEOFF); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_sleep.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_sleep.json new file mode 100644 index 0000000..07689a1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_sleep.json @@ -0,0 +1,3 @@ +{ + "exit_code": 1 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_wait.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_wait.c new file mode 100644 index 0000000..52c684a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_wait.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include "common.h" + +int +main(int argc, char **argv) +{ + test_termination(true, false, BLOCKING_TASK_ATOMIC_WAIT); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_wait.json b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_wait.json new file mode 100644 index 0000000..07689a1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/nonmain_trap_wait.json @@ -0,0 +1,3 @@ +{ + "exit_code": 1 +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/spawn_multiple_times.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/spawn_multiple_times.c new file mode 100644 index 0000000..24664c4 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/spawn_multiple_times.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include +#include + +#include "wasi_thread_start.h" + +enum CONSTANTS { + NUM_ITER = 50, + NUM_RETRY = 5, + SECOND = 1000 * 1000 * 1000, /* 1 second */ + TIMEOUT = 5LL * SECOND +}; + +typedef struct { + start_args_t base; + int th_done; +} shared_t; + +int g_count = 0; + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + shared_t *data = (shared_t *)start_arg; + + g_count++; + + __atomic_store_n(&data->th_done, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data->th_done, 1); +} + +int +main(int argc, char **argv) +{ + shared_t data = { 0 }; + assert(start_args_init(&data.base) && "Stack allocation for thread failed"); + + for (int i = 0; i < NUM_ITER; i++) { + data.th_done = 0; + + printf("Creating thread\n"); + int thread_id = -1; + for (int j = 0; j < NUM_RETRY && thread_id < 0; j++) { + thread_id = __wasi_thread_spawn(&data); + if (thread_id < 0) + __builtin_wasm_memory_atomic_wait32(NULL, 0, SECOND); + } + assert(thread_id > 0 && "Thread creation should succeed"); + + printf("Waiting for thread to finish\n"); + if (__builtin_wasm_memory_atomic_wait32(&data.th_done, 0, TIMEOUT) + == 2) { + assert(false && "Wait should not time out"); + } + printf("Thread has finished\n"); + } + + assert(g_count == NUM_ITER && "Count has not been updated correctly"); + + start_args_deinit(&data.base); + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/sync_primitives.h b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/sync_primitives.h new file mode 100644 index 0000000..4b7dac8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/sync_primitives.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +/* Mutex */ + +typedef int pthread_mutex_t; + +int +pthread_mutex_init(pthread_mutex_t *mutex, void *unused) +{ + *mutex = 0; + return 0; +} + +int +pthread_mutex_destroy(pthread_mutex_t *mutex) +{ + return 0; +} + +static bool +try_pthread_mutex_lock(pthread_mutex_t *mutex) +{ + int expected = 0; + return __atomic_compare_exchange_n(mutex, &expected, 1, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} + +int +pthread_mutex_lock(pthread_mutex_t *mutex) +{ + while (!try_pthread_mutex_lock(mutex)) + __builtin_wasm_memory_atomic_wait32(mutex, 1, -1); + return 0; +} + +int +pthread_mutex_unlock(pthread_mutex_t *mutex) +{ + __atomic_store_n(mutex, 0, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(mutex, 1); + return 0; +} + +/* Barrier */ + +typedef struct { + int count; + int num_threads; + int mutex; + int ready; +} pthread_barrier_t; + +int +pthread_barrier_init(pthread_barrier_t *barrier, void *unused, int num_threads) +{ + barrier->count = 0; + barrier->num_threads = num_threads; + barrier->ready = 0; + pthread_mutex_init(&barrier->mutex, NULL); + + return 0; +} + +int +pthread_barrier_wait(pthread_barrier_t *barrier) +{ + bool no_wait = false; + int count; + + pthread_mutex_lock(&barrier->mutex); + count = barrier->count++; + if (barrier->count >= barrier->num_threads) { + no_wait = true; + barrier->count = 0; + } + pthread_mutex_unlock(&barrier->mutex); + + if (no_wait) { + __atomic_store_n(&barrier->ready, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&barrier->ready, count); + return 0; + } + + __builtin_wasm_memory_atomic_wait32(&barrier->ready, 0, -1); + return 0; +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/trap_after_main_thread_finishes.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/trap_after_main_thread_finishes.c new file mode 100644 index 0000000..5cf6133 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/trap_after_main_thread_finishes.c @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include + +#include "wasi_thread_start.h" + +enum CONSTANTS { + SECOND = 1000 * 1000 * 1000, /* 1 second */ + TIMEOUT = 1LL * SECOND +}; + +typedef struct { + start_args_t base; +} shared_t; + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + /* Wait so that the exception is raised after the main thread has finished + * already */ + __builtin_wasm_memory_atomic_wait32(NULL, 0, TIMEOUT); + __builtin_trap(); +} + +int +main(int argc, char **argv) +{ + shared_t data = { 0 }; + + assert(start_args_init(&data.base)); + int thread_id = __wasi_thread_spawn(&data); + ASSERT_VALID_TID(thread_id); + + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/update_shared_data_and_alloc_heap.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/update_shared_data_and_alloc_heap.c new file mode 100644 index 0000000..d6e3453 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/test/update_shared_data_and_alloc_heap.c @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include +#include +#include + +#include "wasi_thread_start.h" + +enum CONSTANTS { + NUM_THREADS = 4, + NUM_ITER = 30, + SECOND = 1000 * 1000 * 1000, /* 1 second */ + TIMEOUT = 10LL * SECOND +}; + +typedef struct { + start_args_t base; + int th_done; + int *count; + int iteration; + int *pval; +} shared_t; + +pthread_mutex_t mutex; +int *vals[NUM_THREADS]; + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + shared_t *data = (shared_t *)start_arg; + + for (int i = 0; i < NUM_ITER; i++) + __atomic_fetch_add(data->count, 1, __ATOMIC_SEQ_CST); + + *vals[data->iteration] = data->iteration; + + __atomic_store_n(&data->th_done, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data->th_done, 1); +} + +int +main(int argc, char **argv) +{ + shared_t data[NUM_THREADS] = { 0 }; + int thread_ids[NUM_THREADS]; + int *count = calloc(1, sizeof(int)); + + assert(count != NULL && "Failed to call calloc"); + assert(pthread_mutex_init(&mutex, NULL) == 0 && "Failed to init mutex"); + + for (int i = 0; i < NUM_THREADS; i++) { + vals[i] = malloc(sizeof(int)); + assert(vals[i] != NULL && "Failed to call calloc"); + } + + for (int i = 0; i < NUM_THREADS; i++) { + assert(start_args_init(&data[i].base) + && "Stack allocation for thread failed"); + __atomic_store_n(&data[i].count, count, __ATOMIC_SEQ_CST); + data[i].iteration = i; + + thread_ids[i] = __wasi_thread_spawn(&data[i]); + ASSERT_VALID_TID(thread_ids[i]); + } + + printf("Wait for threads to finish\n"); + for (int i = 0; i < NUM_THREADS; i++) { + if (__builtin_wasm_memory_atomic_wait32(&data[i].th_done, 0, TIMEOUT) + == 2) { + assert(false && "Wait should not time out"); + } + + start_args_deinit(&data[i].base); + } + + assert(*count == (NUM_THREADS * NUM_ITER) && "Count not updated correctly"); + + for (int i = 0; i < NUM_THREADS; i++) { + printf("val=%d\n", *vals[i]); + assert(*vals[i] == i && "Value not updated correctly"); + free(vals[i]); + } + + free(count); + assert(pthread_mutex_destroy(&mutex) == 0 && "Failed to destroy mutex"); + + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/tid_allocator.c b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/tid_allocator.c new file mode 100644 index 0000000..dc2d4f1 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/tid_allocator.c @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "tid_allocator.h" +#include "wasm_export.h" +#include "bh_log.h" + +bh_static_assert(TID_MIN <= TID_MAX); +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) + +bool +tid_allocator_init(TidAllocator *tid_allocator) +{ + tid_allocator->size = MIN(TID_ALLOCATOR_INIT_SIZE, TID_MAX - TID_MIN + 1); + tid_allocator->pos = tid_allocator->size; + tid_allocator->ids = + wasm_runtime_malloc(tid_allocator->size * sizeof(int32)); + if (tid_allocator->ids == NULL) + return false; + + for (int64 i = tid_allocator->pos - 1; i >= 0; i--) + tid_allocator->ids[i] = + (uint32)(TID_MIN + (tid_allocator->pos - 1 - i)); + + return true; +} + +void +tid_allocator_deinit(TidAllocator *tid_allocator) +{ + wasm_runtime_free(tid_allocator->ids); +} + +int32 +tid_allocator_get_tid(TidAllocator *tid_allocator) +{ + if (tid_allocator->pos == 0) { // Resize stack and push new thread ids + if (tid_allocator->size == TID_MAX - TID_MIN + 1) { + LOG_ERROR("Maximum thread identifier reached"); + return -1; + } + + uint32 old_size = tid_allocator->size; + uint32 new_size = MIN(tid_allocator->size * 2, TID_MAX - TID_MIN + 1); + if (new_size != TID_MAX - TID_MIN + 1 + && new_size / 2 != tid_allocator->size) { + LOG_ERROR("Overflow detected during new size calculation"); + return -1; + } + + size_t realloc_size = new_size * sizeof(int32); + if (realloc_size / sizeof(int32) != new_size) { + LOG_ERROR("Overflow detected during realloc"); + return -1; + } + int32 *tmp = + wasm_runtime_realloc(tid_allocator->ids, (uint32)realloc_size); + if (tmp == NULL) { + LOG_ERROR("Thread ID allocator realloc failed"); + return -1; + } + + tid_allocator->size = new_size; + tid_allocator->pos = new_size - old_size; + tid_allocator->ids = tmp; + for (int64 i = tid_allocator->pos - 1; i >= 0; i--) + tid_allocator->ids[i] = + (uint32)(TID_MIN + (tid_allocator->size - 1 - i)); + } + + // Pop available thread identifier from the stack + return tid_allocator->ids[--tid_allocator->pos]; +} + +void +tid_allocator_release_tid(TidAllocator *tid_allocator, int32 thread_id) +{ + // Release thread identifier by pushing it into the stack + bh_assert(tid_allocator->pos < tid_allocator->size); + tid_allocator->ids[tid_allocator->pos++] = thread_id; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/tid_allocator.h b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/tid_allocator.h new file mode 100644 index 0000000..6e25f77 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/tid_allocator.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _TID_ALLOCATOR_H +#define _TID_ALLOCATOR_H + +#include "platform_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define TID_ALLOCATOR_INIT_SIZE CLUSTER_MAX_THREAD_NUM +enum { + /* Keep it in sync with + https://github.com/WebAssembly/wasi-threads#design-choice-thread-ids */ + TID_MIN = 1, + TID_MAX = 0x1FFFFFFF +}; // Reserved TIDs (WASI specification) + +/* Stack data structure to track available thread identifiers */ +typedef struct { + int32 *ids; // Array used to store the stack + uint32 size; // Stack capacity + uint32 pos; // Index of the element after the stack top +} TidAllocator; + +bool +tid_allocator_init(TidAllocator *tid_allocator); + +void +tid_allocator_deinit(TidAllocator *tid_allocator); + +int32 +tid_allocator_get_tid(TidAllocator *tid_allocator); + +void +tid_allocator_release_tid(TidAllocator *tid_allocator, int32 thread_id); + +#ifdef __cplusplus +} +#endif + +#endif /* _TID_ALLOCATOR_H */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/unit-test/lib_wasi_threads_unit_tests.cmake b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/unit-test/lib_wasi_threads_unit_tests.cmake new file mode 100644 index 0000000..75d8f4e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/unit-test/lib_wasi_threads_unit_tests.cmake @@ -0,0 +1,6 @@ +# Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +create_wamr_unit_test(wasi_threads + ${CMAKE_CURRENT_LIST_DIR}/test_tid_allocator.cpp +) diff --git a/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/unit-test/test_tid_allocator.cpp b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/unit-test/test_tid_allocator.cpp new file mode 100644 index 0000000..6fa7300 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/lib-wasi-threads/unit-test/test_tid_allocator.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include "tid_allocator.h" + +#include + +class TidAllocatorTest : public ::testing::Test +{ + protected: + void SetUp() override { ASSERT_TRUE(tid_allocator_init(&_allocator)); } + + void TearDown() override { tid_allocator_deinit(&_allocator); } + + TidAllocator _allocator; +}; + +static bool +is_tid_valid(int32 tid) +{ + /* See: https://github.com/WebAssembly/wasi-threads#design-choice-thread-ids + */ + return tid >= TID_MIN && tid <= TID_MAX; +} + +TEST_F(TidAllocatorTest, BasicTest) +{ + int32 tid = tid_allocator_get_tid(&_allocator); + + ASSERT_TRUE(is_tid_valid(tid)); +} + +TEST_F(TidAllocatorTest, ShouldFailOnAllocatingMoreThanAllowedThreadIDs) +{ + int32 last_tid = 0; + for (int32 i = 0; i < TID_MAX + 1; i++) { + last_tid = tid_allocator_get_tid(&_allocator); + if (last_tid < 0) { + break; + } + ASSERT_TRUE(is_tid_valid(last_tid)); + } + + ASSERT_LT(last_tid, 0); +} + +TEST_F(TidAllocatorTest, ShouldAllocateMoreThanAllowedTIDsIfOldTIDsAreReleased) +{ + int32 last_tid = 0; + for (int32 i = 0; i < TID_MAX + 1; i++) { + if (last_tid != 0) { + tid_allocator_release_tid(&_allocator, last_tid); + } + + last_tid = tid_allocator_get_tid(&_allocator); + ASSERT_TRUE(is_tid_valid(last_tid)); + } +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/SConscript b/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/SConscript new file mode 100644 index 0000000..8dd0043 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/SConscript @@ -0,0 +1,24 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * + +cwd = GetCurrentDir() + +#src = Split(''' +#libc_builtin_wrapper.c +#''') + +src = Glob('*.c') + +CPPDEFINES = ['WASM_ENABLE_LIBC_BUILTIN=1'] + +CPPPATH = [cwd] + + +group = DefineGroup('iwasm_libc_builtin', src, depend = [''], CPPPATH = CPPPATH, CPPDEFINES = CPPDEFINES) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/libc_builtin.cmake b/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/libc_builtin.cmake new file mode 100644 index 0000000..0838712 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/libc_builtin.cmake @@ -0,0 +1,13 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIBC_BUILTIN_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_LIBC_BUILTIN=1) + +include_directories(${LIBC_BUILTIN_DIR}) + +file (GLOB source_all ${LIBC_BUILTIN_DIR}/*.c) + +set (LIBC_BUILTIN_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c b/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c new file mode 100644 index 0000000..2ac3810 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c @@ -0,0 +1,1173 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "../interpreter/wasm.h" + +#if defined(_WIN32) || defined(_WIN32_) +#define strncasecmp _strnicmp +#define strcasecmp _stricmp +#endif + +void +wasm_runtime_set_exception(wasm_module_inst_t module, const char *exception); + +uint32 +wasm_runtime_module_realloc(wasm_module_inst_t module, uint64 ptr, uint64 size, + void **p_native_addr); + +/* clang-format off */ +#define get_module_inst(exec_env) \ + wasm_runtime_get_module_inst(exec_env) + +#define validate_app_addr(offset, size) \ + wasm_runtime_validate_app_addr(module_inst, offset, size) + +#define validate_app_str_addr(offset) \ + wasm_runtime_validate_app_str_addr(module_inst, offset) + +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define addr_app_to_native(offset) \ + wasm_runtime_addr_app_to_native(module_inst, offset) + +#define addr_native_to_app(ptr) \ + wasm_runtime_addr_native_to_app(module_inst, ptr) + +#define module_malloc(size, p_native_addr) \ + wasm_runtime_module_malloc(module_inst, size, p_native_addr) + +#define module_free(offset) \ + wasm_runtime_module_free(module_inst, offset) +/* clang-format on */ + +typedef int (*out_func_t)(int c, void *ctx); + +typedef char *_va_list; +#define _INTSIZEOF(n) (((uint32)sizeof(n) + 3) & (uint32)~3) +#define _va_arg(ap, t) (*(t *)((ap += _INTSIZEOF(t)) - _INTSIZEOF(t))) + +#define CHECK_VA_ARG(ap, t) \ + do { \ + if ((uint8 *)ap + _INTSIZEOF(t) > native_end_addr) { \ + if (fmt_buf != temp_fmt) { \ + wasm_runtime_free(fmt_buf); \ + } \ + goto fail; \ + } \ + } while (0) + +/* clang-format off */ +#define PREPARE_TEMP_FORMAT() \ + char temp_fmt[32], *s, *fmt_buf = temp_fmt; \ + uint32 fmt_buf_len = (uint32)sizeof(temp_fmt); \ + int32 n; \ + \ + /* additional 2 bytes: one is the format char, \ + the other is `\0` */ \ + if ((uint32)(fmt - fmt_start_addr + 2) >= fmt_buf_len) { \ + bh_assert((uint32)(fmt - fmt_start_addr) <= \ + UINT32_MAX - 2); \ + fmt_buf_len = (uint32)(fmt - fmt_start_addr + 2); \ + if (!(fmt_buf = wasm_runtime_malloc(fmt_buf_len))) { \ + print_err(out, ctx); \ + break; \ + } \ + } \ + \ + memset(fmt_buf, 0, fmt_buf_len); \ + bh_memcpy_s(fmt_buf, fmt_buf_len, fmt_start_addr, \ + (uint32)(fmt - fmt_start_addr + 1)); +/* clang-format on */ + +#define OUTPUT_TEMP_FORMAT() \ + do { \ + if (n > 0) { \ + s = buf; \ + while (*s) \ + out((int)(*s++), ctx); \ + } \ + \ + if (fmt_buf != temp_fmt) { \ + wasm_runtime_free(fmt_buf); \ + } \ + } while (0) + +static void +print_err(out_func_t out, void *ctx) +{ + out('E', ctx); + out('R', ctx); + out('R', ctx); +} + +static bool +_vprintf_wa(out_func_t out, void *ctx, const char *fmt, _va_list ap, + wasm_module_inst_t module_inst) +{ + int might_format = 0; /* 1 if encountered a '%' */ + int long_ctr = 0; + uint8 *native_end_addr; + const char *fmt_start_addr = NULL; + + if (!wasm_runtime_get_native_addr_range(module_inst, (uint8 *)ap, NULL, + &native_end_addr)) + goto fail; + + /* fmt has already been adjusted if needed */ + + while (*fmt) { + if (!might_format) { + if (*fmt != '%') { + out((int)*fmt, ctx); + } + else { + might_format = 1; + long_ctr = 0; + fmt_start_addr = fmt; + } + } + else { + switch (*fmt) { + case '.': + case '+': + case '-': + case ' ': + case '#': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + goto still_might_format; + + case 't': /* ptrdiff_t */ + case 'z': /* size_t (32bit on wasm) */ + long_ctr = 1; + goto still_might_format; + + case 'j': + /* intmax_t/uintmax_t */ + long_ctr = 2; + goto still_might_format; + + case 'l': + long_ctr++; + /* Fall through */ + case 'h': + /* FIXME: do nothing for these modifiers */ + goto still_might_format; + + case 'o': + case 'd': + case 'i': + case 'u': + case 'p': + case 'x': + case 'X': + case 'c': + { + char buf[64]; + PREPARE_TEMP_FORMAT(); + + if (long_ctr < 2) { + int32 d; + + CHECK_VA_ARG(ap, uint32); + d = _va_arg(ap, int32); + + if (long_ctr == 1) { + uint32 fmt_end_idx = (uint32)(fmt - fmt_start_addr); + + if (fmt_buf[fmt_end_idx - 1] == 'l' + || fmt_buf[fmt_end_idx - 1] == 'z' + || fmt_buf[fmt_end_idx - 1] == 't') { + /* The %ld, %zd and %td should be treated as + * 32bit integer in wasm */ + fmt_buf[fmt_end_idx - 1] = fmt_buf[fmt_end_idx]; + fmt_buf[fmt_end_idx] = '\0'; + } + } + + n = snprintf(buf, sizeof(buf), fmt_buf, d); + } + else { + int64 lld; + + /* Make 8-byte aligned */ + ap = (_va_list)(((uintptr_t)ap + 7) & ~(uintptr_t)7); + CHECK_VA_ARG(ap, uint64); + lld = _va_arg(ap, int64); + n = snprintf(buf, sizeof(buf), fmt_buf, lld); + } + + OUTPUT_TEMP_FORMAT(); + break; + } + + case 's': + { + char buf_tmp[128], *buf = buf_tmp; + char *start; + uint32 s_offset, str_len, buf_len; + + PREPARE_TEMP_FORMAT(); + + CHECK_VA_ARG(ap, int32); + s_offset = _va_arg(ap, uint32); + + if (!validate_app_str_addr(s_offset)) { + if (fmt_buf != temp_fmt) { + wasm_runtime_free(fmt_buf); + } + return false; + } + + s = start = addr_app_to_native((uint64)s_offset); + + str_len = (uint32)strlen(start); + if (str_len >= UINT32_MAX - 64) { + print_err(out, ctx); + if (fmt_buf != temp_fmt) { + wasm_runtime_free(fmt_buf); + } + break; + } + + /* reserve 64 more bytes as there may be width description + * in the fmt */ + buf_len = str_len + 64; + + if (buf_len > (uint32)sizeof(buf_tmp)) { + if (!(buf = wasm_runtime_malloc(buf_len))) { + print_err(out, ctx); + if (fmt_buf != temp_fmt) { + wasm_runtime_free(fmt_buf); + } + break; + } + } + + n = snprintf(buf, buf_len, fmt_buf, + (s_offset == 0 && str_len == 0) ? NULL + : start); + + OUTPUT_TEMP_FORMAT(); + + if (buf != buf_tmp) { + wasm_runtime_free(buf); + } + + break; + } + + case '%': + { + out((int)'%', ctx); + break; + } + + case 'e': + case 'E': + case 'g': + case 'G': + case 'f': + case 'F': + { + float64 f64; + char buf[64]; + PREPARE_TEMP_FORMAT(); + + /* Make 8-byte aligned */ + ap = (_va_list)(((uintptr_t)ap + 7) & ~(uintptr_t)7); + CHECK_VA_ARG(ap, float64); + f64 = _va_arg(ap, float64); + n = snprintf(buf, sizeof(buf), fmt_buf, f64); + + OUTPUT_TEMP_FORMAT(); + break; + } + + case 'n': + /* print nothing */ + break; + + default: + out((int)'%', ctx); + out((int)*fmt, ctx); + break; + } + + might_format = 0; + } + + still_might_format: + ++fmt; + } + return true; + +fail: + wasm_runtime_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +struct str_context { + char *str; + uint32 max; + uint32 count; +}; + +static int +sprintf_out(int c, struct str_context *ctx) +{ + if (!ctx->str || ctx->count >= ctx->max) { + ctx->count++; + return c; + } + + if (ctx->count == ctx->max - 1) { + ctx->str[ctx->count++] = '\0'; + } + else { + ctx->str[ctx->count++] = (char)c; + } + + return c; +} + +#ifndef BUILTIN_LIBC_BUFFERED_PRINTF +#define BUILTIN_LIBC_BUFFERED_PRINTF 0 +#endif + +#ifndef BUILTIN_LIBC_BUFFERED_PRINT_SIZE +#define BUILTIN_LIBC_BUFFERED_PRINT_SIZE 128 +#endif +#ifndef BUILTIN_LIBC_BUFFERED_PRINT_PREFIX +#define BUILTIN_LIBC_BUFFERED_PRINT_PREFIX +#endif + +#if BUILTIN_LIBC_BUFFERED_PRINTF != 0 + +BUILTIN_LIBC_BUFFERED_PRINT_PREFIX +static char print_buf[BUILTIN_LIBC_BUFFERED_PRINT_SIZE] = { 0 }; + +BUILTIN_LIBC_BUFFERED_PRINT_PREFIX +static int print_buf_size = 0; + +static int +printf_out(int c, struct str_context *ctx) +{ + if (c == '\n') { + print_buf[print_buf_size] = '\0'; + os_printf("%s\n", print_buf); + print_buf_size = 0; + } + else if (print_buf_size >= sizeof(print_buf) - 2) { + print_buf[print_buf_size++] = (char)c; + print_buf[print_buf_size] = '\0'; + os_printf("%s\n", print_buf); + print_buf_size = 0; + } + else { + print_buf[print_buf_size++] = (char)c; + } + ctx->count++; + return c; +} +#else +static int +printf_out(int c, struct str_context *ctx) +{ + os_printf("%c", c); + ctx->count++; + return c; +} +#endif + +static int +printf_wrapper(wasm_exec_env_t exec_env, const char *format, _va_list va_args) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + struct str_context ctx = { NULL, 0, 0 }; + + /* format has been checked by runtime */ + if (!validate_native_addr(va_args, (uint64)sizeof(int32))) + return 0; + + if (!_vprintf_wa((out_func_t)printf_out, &ctx, format, va_args, + module_inst)) + return 0; + + return (int)ctx.count; +} + +static int +sprintf_wrapper(wasm_exec_env_t exec_env, char *str, const char *format, + _va_list va_args) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint8 *native_end_offset; + struct str_context ctx; + + /* str and format have been checked by runtime */ + if (!validate_native_addr(va_args, (uint64)sizeof(uint32))) + return 0; + + if (!wasm_runtime_get_native_addr_range(module_inst, (uint8 *)str, NULL, + &native_end_offset)) { + wasm_runtime_set_exception(module_inst, "out of bounds memory access"); + return 0; + } + + ctx.str = str; + ctx.max = (uint32)(native_end_offset - (uint8 *)str); + ctx.count = 0; + + if (!_vprintf_wa((out_func_t)sprintf_out, &ctx, format, va_args, + module_inst)) + return 0; + + if (ctx.count < ctx.max) { + str[ctx.count] = '\0'; + } + + return (int)ctx.count; +} + +static int +snprintf_wrapper(wasm_exec_env_t exec_env, char *str, uint32 size, + const char *format, _va_list va_args) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + struct str_context ctx; + + /* str and format have been checked by runtime */ + if (!validate_native_addr(va_args, (uint64)sizeof(uint32))) + return 0; + + ctx.str = str; + ctx.max = size; + ctx.count = 0; + + if (!_vprintf_wa((out_func_t)sprintf_out, &ctx, format, va_args, + module_inst)) + return 0; + + if (ctx.count < ctx.max) { + str[ctx.count] = '\0'; + } + + return (int)ctx.count; +} + +static int +puts_wrapper(wasm_exec_env_t exec_env, const char *str) +{ + (void)exec_env; + + return os_printf("%s\n", str); +} + +static int +putchar_wrapper(wasm_exec_env_t exec_env, int c) +{ + (void)exec_env; + + os_printf("%c", c); + return 1; +} + +static uint32 +strdup_wrapper(wasm_exec_env_t exec_env, const char *str) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char *str_ret; + uint32 len; + uint32 str_ret_offset = 0; + + /* str has been checked by runtime */ + if (str) { + len = (uint32)strlen(str) + 1; + + str_ret_offset = (uint32)module_malloc((uint64)len, (void **)&str_ret); + if (str_ret_offset) { + bh_memcpy_s(str_ret, len, str, len); + } + } + + return str_ret_offset; +} + +static uint32 +_strdup_wrapper(wasm_exec_env_t exec_env, const char *str) +{ + return strdup_wrapper(exec_env, str); +} + +static int32 +memcmp_wrapper(wasm_exec_env_t exec_env, const void *s1, const void *s2, + uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + /* s2 has been checked by runtime */ + if (!validate_native_addr((void *)s1, (uint64)size)) + return 0; + + return memcmp(s1, s2, size); +} + +static uint32 +memcpy_wrapper(wasm_exec_env_t exec_env, void *dst, const void *src, + uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 dst_offset = (uint32)addr_native_to_app(dst); + + if (size == 0) + return dst_offset; + + /* src has been checked by runtime */ + if (!validate_native_addr(dst, (uint64)size)) + return dst_offset; + + bh_memcpy_s(dst, size, src, size); + return dst_offset; +} + +static uint32 +memmove_wrapper(wasm_exec_env_t exec_env, void *dst, void *src, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 dst_offset = (uint32)addr_native_to_app(dst); + + if (size == 0) + return dst_offset; + + /* src has been checked by runtime */ + if (!validate_native_addr(dst, (uint64)size)) + return dst_offset; + + memmove(dst, src, size); + return dst_offset; +} + +static uint32 +memset_wrapper(wasm_exec_env_t exec_env, void *s, int32 c, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 s_offset = (uint32)addr_native_to_app(s); + + if (!validate_native_addr(s, (uint64)size)) + return s_offset; + + memset(s, c, size); + return s_offset; +} + +static uint32 +strchr_wrapper(wasm_exec_env_t exec_env, const char *s, int32 c) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char *ret; + + /* s has been checked by runtime */ + ret = strchr(s, c); + return ret ? (uint32)addr_native_to_app(ret) : 0; +} + +static int32 +strcmp_wrapper(wasm_exec_env_t exec_env, const char *s1, const char *s2) +{ + (void)exec_env; + + /* s1 and s2 have been checked by runtime */ + return strcmp(s1, s2); +} + +static int32 +strncmp_wrapper(wasm_exec_env_t exec_env, const char *s1, const char *s2, + uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + /* s2 has been checked by runtime */ + if (!validate_native_addr((void *)s1, (uint64)size)) + return 0; + + return strncmp(s1, s2, size); +} + +static uint32 +strcpy_wrapper(wasm_exec_env_t exec_env, char *dst, const char *src) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 len = (uint32)strlen(src) + 1; + + /* src has been checked by runtime */ + if (!validate_native_addr(dst, (uint64)len)) + return 0; + +#ifndef BH_PLATFORM_WINDOWS + strncpy(dst, src, len); +#else + strncpy_s(dst, len, src, len); +#endif + return (uint32)addr_native_to_app(dst); +} + +static uint32 +strncpy_wrapper(wasm_exec_env_t exec_env, char *dst, const char *src, + uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + /* src has been checked by runtime */ + if (!validate_native_addr(dst, (uint64)size)) + return 0; + +#ifndef BH_PLATFORM_WINDOWS + strncpy(dst, src, size); +#else + strncpy_s(dst, size, src, size); +#endif + return (uint32)addr_native_to_app(dst); +} + +static uint32 +strlen_wrapper(wasm_exec_env_t exec_env, const char *s) +{ + (void)exec_env; + + /* s has been checked by runtime */ + return (uint32)strlen(s); +} + +static uint32 +malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + return (uint32)module_malloc((uint64)size, NULL); +} + +static uint32 +calloc_wrapper(wasm_exec_env_t exec_env, uint32 nmemb, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint64 total_size = (uint64)nmemb * (uint64)size; + uint32 ret_offset = 0; + uint8 *ret_ptr; + + if (total_size >= UINT32_MAX) + return 0; + + ret_offset = (uint32)module_malloc(total_size, (void **)&ret_ptr); + if (ret_offset) { + memset(ret_ptr, 0, (uint32)total_size); + } + + return ret_offset; +} + +static uint32 +realloc_wrapper(wasm_exec_env_t exec_env, uint32 ptr, uint32 new_size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + return wasm_runtime_module_realloc(module_inst, ptr, new_size, NULL); +} + +static void +free_wrapper(wasm_exec_env_t exec_env, void *ptr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + if (!validate_native_addr(ptr, (uint64)sizeof(uint32))) + return; + + module_free(addr_native_to_app(ptr)); +} + +static int32 +atoi_wrapper(wasm_exec_env_t exec_env, const char *s) +{ + (void)exec_env; + /* s has been checked by runtime */ + return atoi(s); +} + +static void +exit_wrapper(wasm_exec_env_t exec_env, int32 status) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char buf[32]; + snprintf(buf, sizeof(buf), "env.exit(%" PRId32 ")", status); + wasm_runtime_set_exception(module_inst, buf); +} + +static int32 +strtol_wrapper(wasm_exec_env_t exec_env, const char *nptr, char **endptr, + int32 base) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + int32 num = 0; + + /* nptr has been checked by runtime */ + if (!validate_native_addr(endptr, (uint64)sizeof(uint32))) + return 0; + + num = (int32)strtol(nptr, endptr, base); + *(uint32 *)endptr = (uint32)addr_native_to_app(*endptr); + + return num; +} + +static uint32 +strtoul_wrapper(wasm_exec_env_t exec_env, const char *nptr, char **endptr, + int32 base) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 num = 0; + + /* nptr has been checked by runtime */ + if (!validate_native_addr(endptr, (uint64)sizeof(uint32))) + return 0; + + num = (uint32)strtoul(nptr, endptr, base); + *(uint32 *)endptr = (uint32)addr_native_to_app(*endptr); + + return num; +} + +static uint32 +memchr_wrapper(wasm_exec_env_t exec_env, const void *s, int32 c, uint32 n) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + void *res; + + if (!validate_native_addr((void *)s, (uint64)n)) + return 0; + + res = memchr(s, c, n); + return (uint32)addr_native_to_app(res); +} + +static int32 +strncasecmp_wrapper(wasm_exec_env_t exec_env, const char *s1, const char *s2, + uint32 n) +{ + (void)exec_env; + + /* s1 and s2 have been checked by runtime */ + return strncasecmp(s1, s2, n); +} + +static uint32 +strspn_wrapper(wasm_exec_env_t exec_env, const char *s, const char *accept) +{ + (void)exec_env; + + /* s and accept have been checked by runtime */ + return (uint32)strspn(s, accept); +} + +static uint32 +strcspn_wrapper(wasm_exec_env_t exec_env, const char *s, const char *reject) +{ + (void)exec_env; + + /* s and reject have been checked by runtime */ + return (uint32)strcspn(s, reject); +} + +static uint32 +strstr_wrapper(wasm_exec_env_t exec_env, const char *s, const char *find) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + /* s and find have been checked by runtime */ + char *res = strstr(s, find); + return (uint32)addr_native_to_app(res); +} + +static int32 +isupper_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isupper(c); +} + +static int32 +isalpha_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isalpha(c); +} + +static int32 +isspace_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isspace(c); +} + +static int32 +isgraph_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isgraph(c); +} + +static int32 +isprint_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isprint(c); +} + +static int32 +isdigit_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isdigit(c); +} + +static int32 +isxdigit_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isxdigit(c); +} + +static int32 +tolower_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return tolower(c); +} + +static int32 +toupper_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return toupper(c); +} + +static int32 +isalnum_wrapper(wasm_exec_env_t exec_env, int32 c) +{ + (void)exec_env; + + return isalnum(c); +} + +static uint32 +emscripten_memcpy_big_wrapper(wasm_exec_env_t exec_env, void *dst, + const void *src, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 dst_offset = (uint32)addr_native_to_app(dst); + + /* src has been checked by runtime */ + if (!validate_native_addr(dst, (uint64)size)) + return dst_offset; + + bh_memcpy_s(dst, size, src, size); + return dst_offset; +} + +static void +abort_wrapper(wasm_exec_env_t exec_env, int32 code) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char buf[32]; + snprintf(buf, sizeof(buf), "env.abort(%" PRId32 ")", code); + wasm_runtime_set_exception(module_inst, buf); +} + +static void +abortStackOverflow_wrapper(wasm_exec_env_t exec_env, int32 code) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char buf[32]; + snprintf(buf, sizeof(buf), "env.abortStackOverflow(%" PRId32 ")", code); + wasm_runtime_set_exception(module_inst, buf); +} + +static void +nullFunc_X_wrapper(wasm_exec_env_t exec_env, int32 code) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char buf[32]; + snprintf(buf, sizeof(buf), "env.nullFunc_X(%" PRId32 ")", code); + wasm_runtime_set_exception(module_inst, buf); +} + +static uint32 +__cxa_allocate_exception_wrapper(wasm_exec_env_t exec_env, uint32 thrown_size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 exception = (uint32)module_malloc((uint64)thrown_size, NULL); + if (!exception) + return 0; + + return exception; +} + +static void +__cxa_begin_catch_wrapper(wasm_exec_env_t exec_env, void *exception_object) +{ + (void)exec_env; + (void)exception_object; +} + +static void +__cxa_throw_wrapper(wasm_exec_env_t exec_env, void *thrown_exception, + void *tinfo, uint32 table_elem_idx) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char buf[32]; + + (void)thrown_exception; + (void)tinfo; + (void)table_elem_idx; + + snprintf(buf, sizeof(buf), "%s", "exception thrown by stdc++"); + wasm_runtime_set_exception(module_inst, buf); +} + +struct timespec_app { + int64 tv_sec; + int32 tv_nsec; +}; + +static uint32 +clock_gettime_wrapper(wasm_exec_env_t exec_env, uint32 clk_id, + struct timespec_app *ts_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint64 time; + + (void)clk_id; + + if (!validate_native_addr(ts_app, (uint64)sizeof(struct timespec_app))) + return (uint32)-1; + + time = os_time_get_boot_us(); + ts_app->tv_sec = time / 1000000; + ts_app->tv_nsec = (time % 1000000) * 1000; + + return (uint32)0; +} + +static uint64 +clock_wrapper(wasm_exec_env_t exec_env) +{ + (void)exec_env; + + /* Convert to nano seconds as CLOCKS_PER_SEC in wasi-sdk */ + + return os_time_get_boot_us() * 1000; +} + +#if WASM_ENABLE_SPEC_TEST != 0 +static void +print_wrapper(wasm_exec_env_t exec_env) +{ + os_printf("in specttest.print()\n"); +} + +static void +print_i32_wrapper(wasm_exec_env_t exec_env, int32 i32) +{ + os_printf("in specttest.print_i32(%" PRId32 ")\n", i32); +} + +static void +print_i32_f32_wrapper(wasm_exec_env_t exec_env, int32 i32, float f32) +{ + os_printf("in specttest.print_i32_f32(%" PRId32 ", %f)\n", i32, f32); +} + +static void +print_f64_f64_wrapper(wasm_exec_env_t exec_env, double f64_1, double f64_2) +{ + os_printf("in specttest.print_f64_f64(%f, %f)\n", f64_1, f64_2); +} + +static void +print_f32_wrapper(wasm_exec_env_t exec_env, float f32) +{ + os_printf("in specttest.print_f32(%f)\n", f32); +} + +static void +print_f64_wrapper(wasm_exec_env_t exec_env, double f64) +{ + os_printf("in specttest.print_f64(%f)\n", f64); +} +#endif /* WASM_ENABLE_SPEC_TEST */ + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_libc_builtin[] = { + REG_NATIVE_FUNC(printf, "($*)i"), + REG_NATIVE_FUNC(sprintf, "($$*)i"), + REG_NATIVE_FUNC(snprintf, "(*~$*)i"), + { "vprintf", printf_wrapper, "($*)i", NULL }, + { "vsprintf", sprintf_wrapper, "($$*)i", NULL }, + { "vsnprintf", snprintf_wrapper, "(*~$*)i", NULL }, + REG_NATIVE_FUNC(puts, "($)i"), + REG_NATIVE_FUNC(putchar, "(i)i"), + REG_NATIVE_FUNC(memcmp, "(**~)i"), + REG_NATIVE_FUNC(memcpy, "(**~)i"), + REG_NATIVE_FUNC(memmove, "(**~)i"), + REG_NATIVE_FUNC(memset, "(*ii)i"), + REG_NATIVE_FUNC(strchr, "($i)i"), + REG_NATIVE_FUNC(strcmp, "($$)i"), + REG_NATIVE_FUNC(strcpy, "(*$)i"), + REG_NATIVE_FUNC(strlen, "($)i"), + REG_NATIVE_FUNC(strncmp, "(**~)i"), + REG_NATIVE_FUNC(strncpy, "(**~)i"), + REG_NATIVE_FUNC(malloc, "(i)i"), + REG_NATIVE_FUNC(realloc, "(ii)i"), + REG_NATIVE_FUNC(calloc, "(ii)i"), + REG_NATIVE_FUNC(strdup, "($)i"), + /* clang may introduce __strdup */ + REG_NATIVE_FUNC(_strdup, "($)i"), + REG_NATIVE_FUNC(free, "(*)"), + REG_NATIVE_FUNC(atoi, "($)i"), + REG_NATIVE_FUNC(exit, "(i)"), + REG_NATIVE_FUNC(strtol, "($*i)i"), + REG_NATIVE_FUNC(strtoul, "($*i)i"), + REG_NATIVE_FUNC(memchr, "(*ii)i"), + REG_NATIVE_FUNC(strncasecmp, "($$i)i"), + REG_NATIVE_FUNC(strspn, "($$)i"), + REG_NATIVE_FUNC(strcspn, "($$)i"), + REG_NATIVE_FUNC(strstr, "($$)i"), + REG_NATIVE_FUNC(isupper, "(i)i"), + REG_NATIVE_FUNC(isalpha, "(i)i"), + REG_NATIVE_FUNC(isspace, "(i)i"), + REG_NATIVE_FUNC(isgraph, "(i)i"), + REG_NATIVE_FUNC(isprint, "(i)i"), + REG_NATIVE_FUNC(isdigit, "(i)i"), + REG_NATIVE_FUNC(isxdigit, "(i)i"), + REG_NATIVE_FUNC(tolower, "(i)i"), + REG_NATIVE_FUNC(toupper, "(i)i"), + REG_NATIVE_FUNC(isalnum, "(i)i"), + REG_NATIVE_FUNC(emscripten_memcpy_big, "(**~)i"), + REG_NATIVE_FUNC(abort, "(i)"), + REG_NATIVE_FUNC(abortStackOverflow, "(i)"), + REG_NATIVE_FUNC(nullFunc_X, "(i)"), + REG_NATIVE_FUNC(__cxa_allocate_exception, "(i)i"), + REG_NATIVE_FUNC(__cxa_begin_catch, "(*)"), + REG_NATIVE_FUNC(__cxa_throw, "(**i)"), + REG_NATIVE_FUNC(clock_gettime, "(i*)i"), + REG_NATIVE_FUNC(clock, "()I"), +}; + +#if WASM_ENABLE_SPEC_TEST != 0 +static NativeSymbol native_symbols_spectest[] = { + REG_NATIVE_FUNC(print, "()"), + REG_NATIVE_FUNC(print_i32, "(i)"), + REG_NATIVE_FUNC(print_i32_f32, "(if)"), + REG_NATIVE_FUNC(print_f64_f64, "(FF)"), + REG_NATIVE_FUNC(print_f32, "(f)"), + REG_NATIVE_FUNC(print_f64, "(F)") +}; +#endif + +uint32 +get_libc_builtin_export_apis(NativeSymbol **p_libc_builtin_apis) +{ + *p_libc_builtin_apis = native_symbols_libc_builtin; + return sizeof(native_symbols_libc_builtin) / sizeof(NativeSymbol); +} + +#if WASM_ENABLE_SPEC_TEST != 0 +uint32 +get_spectest_export_apis(NativeSymbol **p_libc_builtin_apis) +{ + *p_libc_builtin_apis = native_symbols_spectest; + return sizeof(native_symbols_spectest) / sizeof(NativeSymbol); +} +#endif + +/************************************* + * Global Variables * + *************************************/ + +typedef struct WASMNativeGlobalDef { + const char *module_name; + const char *global_name; + uint8 type; + bool is_mutable; + WASMValue value; +} WASMNativeGlobalDef; + +static WASMNativeGlobalDef native_global_defs[] = { +#if WASM_ENABLE_SPEC_TEST != 0 + { "spectest", "global_i32", VALUE_TYPE_I32, false, .value.i32 = 666 }, + { "spectest", "global_i64", VALUE_TYPE_I64, false, .value.i64 = 666 }, + { "spectest", "global_f32", VALUE_TYPE_F32, false, .value.f32 = 666.6 }, + { "spectest", "global_f64", VALUE_TYPE_F64, false, .value.f64 = 666.6 }, + { "test", "global-i32", VALUE_TYPE_I32, false, .value.i32 = 0 }, + { "test", "global-f32", VALUE_TYPE_F32, false, .value.f32 = 0 }, + { "test", "global-mut-i32", VALUE_TYPE_I32, true, .value.i32 = 0 }, + { "test", "global-mut-i64", VALUE_TYPE_I64, true, .value.i64 = 0 }, +#if WASM_ENABLE_GC != 0 + { "G", "g", VALUE_TYPE_I32, false, .value.i32 = 4 }, + { "M", "g", REF_TYPE_HT_NON_NULLABLE, false, .value.gc_obj = 0 }, +#endif +#endif + { "global", "NaN", VALUE_TYPE_F64, .value.u64 = 0x7FF8000000000000LL }, + { "global", "Infinity", VALUE_TYPE_F64, .value.u64 = 0x7FF0000000000000LL } +}; + +bool +wasm_native_lookup_libc_builtin_global(const char *module_name, + const char *global_name, + WASMGlobalImport *global) +{ + uint32 size = sizeof(native_global_defs) / sizeof(WASMNativeGlobalDef); + WASMNativeGlobalDef *global_def = native_global_defs; + WASMNativeGlobalDef *global_def_end = global_def + size; + + if (!module_name || !global_name || !global) + return false; + + /* Lookup constant globals which can be defined by table */ + while (global_def < global_def_end) { + if (!strcmp(global_def->module_name, module_name) + && !strcmp(global_def->global_name, global_name)) { + global->type = global_def->type; + global->is_mutable = global_def->is_mutable; + global->global_data_linked = global_def->value; + return true; + } + global_def++; + } + + return false; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/SConscript b/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/SConscript new file mode 100644 index 0000000..432ed4a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/SConscript @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * + +cwd = GetCurrentDir() + +src = Split(''' +libc_emcc_wrapper.c +''') + +CPPPATH = [cwd] + +group = DefineGroup('iwasm_libc_emcc', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/libc_emcc.cmake b/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/libc_emcc.cmake new file mode 100644 index 0000000..d237a16 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/libc_emcc.cmake @@ -0,0 +1,12 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIBC_EMCC_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_LIBC_EMCC=1) + +include_directories(${LIBC_EMCC_DIR}) + +file (GLOB source_all ${LIBC_EMCC_DIR}/*.c) + +set (LIBC_EMCC_SOURCE ${source_all}) \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/libc_emcc_wrapper.c b/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/libc_emcc_wrapper.c new file mode 100644 index 0000000..9699554 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-emcc/libc_emcc_wrapper.c @@ -0,0 +1,556 @@ +/* + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "../interpreter/wasm.h" +#if !defined(_DEFAULT_SOURCE) && !defined(BH_PLATFORM_LINUX_SGX) +#include "sys/syscall.h" +#endif + +/* clang-format off */ +#define get_module_inst(exec_env) \ + wasm_runtime_get_module_inst(exec_env) + +#define validate_app_addr(offset, size) \ + wasm_runtime_validate_app_addr(module_inst, offset, size) + +#define validate_app_str_addr(offset) \ + wasm_runtime_validate_app_str_addr(module_inst, offset) + +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define addr_app_to_native(offset) \ + wasm_runtime_addr_app_to_native(module_inst, offset) + +#define addr_native_to_app(ptr) \ + wasm_runtime_addr_native_to_app(module_inst, ptr) + +#define module_malloc(size, p_native_addr) \ + wasm_runtime_module_malloc(module_inst, size, p_native_addr) + +#define module_free(offset) \ + wasm_runtime_module_free(module_inst, offset) +/* clang-format on */ + +static void +invoke_viiii_wrapper(wasm_exec_env_t exec_env, uint32 elem_idx, int arg0, + int arg1, int arg2, int arg3) +{ + uint32 argv[4]; + bool ret; + + argv[0] = arg0; + argv[1] = arg1; + argv[2] = arg2; + argv[3] = arg3; + ret = wasm_runtime_call_indirect(exec_env, elem_idx, 4, argv); + (void)ret; +} + +static void +invoke_viii_wrapper(wasm_exec_env_t exec_env, uint32 elem_idx, int arg0, + int arg1, int arg2) +{ + uint32 argv[4]; + bool ret; + + argv[0] = arg0; + argv[1] = arg1; + argv[2] = arg2; + ret = wasm_runtime_call_indirect(exec_env, elem_idx, 3, argv); + (void)ret; +} + +static void +invoke_vii_wrapper(wasm_exec_env_t exec_env, uint32 elem_idx, int arg0, + int arg1) +{ + uint32 argv[4]; + bool ret; + + argv[0] = arg0; + argv[1] = arg1; + ret = wasm_runtime_call_indirect(exec_env, elem_idx, 2, argv); + (void)ret; +} + +static void +invoke_vi_wrapper(wasm_exec_env_t exec_env, uint32 elem_idx, int arg0) +{ + uint32 argv[4]; + bool ret; + + argv[0] = arg0; + ret = wasm_runtime_call_indirect(exec_env, elem_idx, 1, argv); + (void)ret; +} + +static int +invoke_iii_wrapper(wasm_exec_env_t exec_env, uint32 elem_idx, int arg0, + int arg1) +{ + uint32 argv[4]; + bool ret; + + argv[0] = arg0; + argv[1] = arg1; + ret = wasm_runtime_call_indirect(exec_env, elem_idx, 2, argv); + return ret ? argv[0] : 0; +} + +static int +invoke_ii_wrapper(wasm_exec_env_t exec_env, uint32 elem_idx, int arg0) +{ + uint32 argv[4]; + bool ret; + + argv[0] = arg0; + ret = wasm_runtime_call_indirect(exec_env, elem_idx, 1, argv); + return ret ? argv[0] : 0; +} + +struct timespec_emcc { + int tv_sec; + int tv_nsec; +}; + +struct stat_emcc { + unsigned st_dev; + int __st_dev_padding; + unsigned __st_ino_truncated; + unsigned st_mode; + unsigned st_nlink; + unsigned st_uid; + unsigned st_gid; + unsigned st_rdev; + int __st_rdev_padding; + int64 st_size; + int st_blksize; + int st_blocks; + struct timespec_emcc st_atim; + struct timespec_emcc st_mtim; + struct timespec_emcc st_ctim; + int64 st_ino; +}; + +static int +open_wrapper(wasm_exec_env_t exec_env, const char *pathname, int flags, + int mode) +{ + if (pathname == NULL) + return -1; + return open(pathname, flags, mode); +} + +static int +__sys_read_wrapper(wasm_exec_env_t exec_env, int fd, void *buf, uint32 count) +{ + return read(fd, buf, count); +} + +static void +statbuf_native2app(const struct stat *statbuf_native, + struct stat_emcc *statbuf_app) +{ + statbuf_app->st_dev = (unsigned)statbuf_native->st_dev; + statbuf_app->__st_ino_truncated = (unsigned)statbuf_native->st_ino; + statbuf_app->st_mode = (unsigned)statbuf_native->st_mode; + statbuf_app->st_nlink = (unsigned)statbuf_native->st_nlink; + statbuf_app->st_uid = (unsigned)statbuf_native->st_uid; + statbuf_app->st_gid = (unsigned)statbuf_native->st_gid; + statbuf_app->st_rdev = (unsigned)statbuf_native->st_rdev; + statbuf_app->st_size = (int64)statbuf_native->st_size; + statbuf_app->st_blksize = (unsigned)statbuf_native->st_blksize; + statbuf_app->st_blocks = (unsigned)statbuf_native->st_blocks; + statbuf_app->st_ino = (int64)statbuf_native->st_ino; + statbuf_app->st_atim.tv_sec = (int)statbuf_native->st_atim.tv_sec; + statbuf_app->st_atim.tv_nsec = (int)statbuf_native->st_atim.tv_nsec; + statbuf_app->st_mtim.tv_sec = (int)statbuf_native->st_mtim.tv_sec; + statbuf_app->st_mtim.tv_nsec = (int)statbuf_native->st_mtim.tv_nsec; + statbuf_app->st_ctim.tv_sec = (int)statbuf_native->st_ctim.tv_sec; + statbuf_app->st_ctim.tv_nsec = (int)statbuf_native->st_ctim.tv_nsec; +} + +static int +__sys_stat64_wrapper(wasm_exec_env_t exec_env, const char *pathname, + struct stat_emcc *statbuf_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + int ret; + struct stat statbuf; + + if (!validate_native_addr((void *)statbuf_app, + (uint64)sizeof(struct stat_emcc))) + return -1; + + if (pathname == NULL) + return -1; + + ret = stat(pathname, &statbuf); + if (ret == 0) + statbuf_native2app(&statbuf, statbuf_app); + return ret; +} + +static int +__sys_fstat64_wrapper(wasm_exec_env_t exec_env, int fd, + struct stat_emcc *statbuf_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + int ret; + struct stat statbuf; + + if (!validate_native_addr((void *)statbuf_app, + (uint64)sizeof(struct stat_emcc))) + return -1; + + if (fd <= 0) + return -1; + + ret = fstat(fd, &statbuf); + if (ret == 0) + statbuf_native2app(&statbuf, statbuf_app); + return ret; +} + +static int +mmap_wrapper(wasm_exec_env_t exec_env, void *addr, int length, int prot, + int flags, int fd, int64 offset) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uint32 buf_offset; + char *buf; + int size_read; + + buf_offset = module_malloc((uint64)length, (void **)&buf); + if (buf_offset == 0) + return -1; + + if (fd <= 0) + return -1; + + if (lseek(fd, offset, SEEK_SET) == -1) + return -1; + + size_read = read(fd, buf, length); + (void)size_read; + return buf_offset; +} + +static int +munmap_wrapper(wasm_exec_env_t exec_env, uint32 buf_offset, int length) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + module_free((uint64)buf_offset); + return 0; +} + +static int +__munmap_wrapper(wasm_exec_env_t exec_env, uint32 buf_offset, int length) +{ + return munmap_wrapper(exec_env, buf_offset, length); +} + +static int +getentropy_wrapper(wasm_exec_env_t exec_env, void *buffer, uint32 length) +{ + if (buffer == NULL) + return -1; +#if defined(_DEFAULT_SOURCE) || defined(BH_PLATFORM_LINUX_SGX) + return getentropy(buffer, length); +#else + return syscall(SYS_getrandom, buffer, length, 0); +#endif +} + +static int +setjmp_wrapper(wasm_exec_env_t exec_env, void *jmp_buf) +{ + LOG_DEBUG("setjmp() called\n"); + return 0; +} + +static void +longjmp_wrapper(wasm_exec_env_t exec_env, void *jmp_buf, int val) +{ + LOG_DEBUG("longjmp() called\n"); +} + +#if !defined(BH_PLATFORM_LINUX_SGX) +static FILE *file_list[32] = { 0 }; + +static int +get_free_file_slot() +{ + unsigned int i; + + for (i = 0; i < sizeof(file_list) / sizeof(FILE *); i++) { + if (file_list[i] == NULL) + return (int)i; + } + return -1; +} + +static int +fopen_wrapper(wasm_exec_env_t exec_env, const char *pathname, const char *mode) +{ + FILE *file; + int file_id; + + if (pathname == NULL || mode == NULL) + return 0; + + if ((file_id = get_free_file_slot()) == -1) + return 0; + + file = fopen(pathname, mode); + if (!file) + return 0; + + file_list[file_id] = file; + return file_id + 1; +} + +static uint32 +fread_wrapper(wasm_exec_env_t exec_env, void *ptr, uint32 size, uint32 nmemb, + int file_id) +{ + FILE *file; + + file_id = file_id - 1; + if ((unsigned)file_id >= sizeof(file_list) / sizeof(FILE *)) { + return 0; + } + if ((file = file_list[file_id]) == NULL) { + return 0; + } + return (uint32)fread(ptr, size, nmemb, file); +} + +static int +fseeko_wrapper(wasm_exec_env_t exec_env, int file_id, int64 offset, int whence) +{ + FILE *file; + + file_id = file_id - 1; + if ((unsigned)file_id >= sizeof(file_list) / sizeof(FILE *)) { + return -1; + } + if ((file = file_list[file_id]) == NULL) { + return -1; + } + return (uint32)fseek(file, offset, whence); +} + +static uint32 +emcc_fwrite_wrapper(wasm_exec_env_t exec_env, const void *ptr, uint32 size, + uint32 nmemb, int file_id) +{ + FILE *file; + + file_id = file_id - 1; + if ((unsigned)file_id >= sizeof(file_list) / sizeof(FILE *)) { + return 0; + } + if ((file = file_list[file_id]) == NULL) { + return 0; + } + return (uint32)fwrite(ptr, size, nmemb, file); +} + +static int +feof_wrapper(wasm_exec_env_t exec_env, int file_id) +{ + FILE *file; + + file_id = file_id - 1; + if ((unsigned)file_id >= sizeof(file_list) / sizeof(FILE *)) + return 1; + if ((file = file_list[file_id]) == NULL) + return 1; + return feof(file); +} + +static int +fclose_wrapper(wasm_exec_env_t exec_env, int file_id) +{ + FILE *file; + + file_id = file_id - 1; + if ((unsigned)file_id >= sizeof(file_list) / sizeof(FILE *)) + return -1; + if ((file = file_list[file_id]) == NULL) + return -1; + file_list[file_id] = NULL; + return fclose(file); +} + +static int +__sys_mkdir_wrapper(wasm_exec_env_t exec_env, const char *pathname, int mode) +{ + if (!pathname) + return -1; + return mkdir(pathname, mode); +} + +static int +__sys_rmdir_wrapper(wasm_exec_env_t exec_env, const char *pathname) +{ + if (!pathname) + return -1; + return rmdir(pathname); +} + +static int +__sys_unlink_wrapper(wasm_exec_env_t exec_env, const char *pathname) +{ + if (!pathname) + return -1; + return unlink(pathname); +} + +static uint32 +__sys_getcwd_wrapper(wasm_exec_env_t exec_env, char *buf, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char *ret; + + if (!buf) + return -1; + + ret = getcwd(buf, size); + return ret ? (uint32)addr_native_to_app(ret) : 0; +} + +#include + +struct utsname_app { + char sysname[64]; + char nodename[64]; + char release[64]; + char version[64]; + char machine[64]; + char domainname[64]; +}; + +static int +__sys_uname_wrapper(wasm_exec_env_t exec_env, struct utsname_app *uname_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + struct utsname uname_native = { 0 }; + uint32 length; + + if (!validate_native_addr(uname_app, (uint64)sizeof(struct utsname_app))) + return -1; + + if (uname(&uname_native) != 0) { + return -1; + } + + memset(uname_app, 0, sizeof(struct utsname_app)); + + length = strlen(uname_native.sysname); + if (length > sizeof(uname_app->sysname) - 1) + length = sizeof(uname_app->sysname) - 1; + bh_memcpy_s(uname_app->sysname, sizeof(uname_app->sysname), + uname_native.sysname, length); + + length = strlen(uname_native.nodename); + if (length > sizeof(uname_app->nodename) - 1) + length = sizeof(uname_app->nodename) - 1; + bh_memcpy_s(uname_app->nodename, sizeof(uname_app->nodename), + uname_native.nodename, length); + + length = strlen(uname_native.release); + if (length > sizeof(uname_app->release) - 1) + length = sizeof(uname_app->release) - 1; + bh_memcpy_s(uname_app->release, sizeof(uname_app->release), + uname_native.release, length); + + length = strlen(uname_native.version); + if (length > sizeof(uname_app->version) - 1) + length = sizeof(uname_app->version) - 1; + bh_memcpy_s(uname_app->version, sizeof(uname_app->version), + uname_native.version, length); + +#ifdef _GNU_SOURCE + length = strlen(uname_native.domainname); + if (length > sizeof(uname_app->domainname) - 1) + length = sizeof(uname_app->domainname) - 1; + bh_memcpy_s(uname_app->domainname, sizeof(uname_app->domainname), + uname_native.domainname, length); +#endif + + return 0; +} + +static void +emscripten_notify_memory_growth_wrapper(wasm_exec_env_t exec_env, int i) +{ + (void)i; +} + +static void +emscripten_thread_sleep_wrapper(wasm_exec_env_t exec_env, double timeout_ms) +{ + uint64 ms = (uint64)timeout_ms; + uint64 sec = ms / 1000, us = (ms % 1000) * 1000; + + if (sec > 0) + sleep(sec); + if (us > 0) + usleep(us); +} + +#endif /* end of BH_PLATFORM_LINUX_SGX */ + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } +/* clang-format off */ + +static NativeSymbol native_symbols_libc_emcc[] = { + REG_NATIVE_FUNC(invoke_viiii, "(iiiii)"), + REG_NATIVE_FUNC(invoke_viii, "(iiii)"), + REG_NATIVE_FUNC(invoke_vii, "(iii)"), + REG_NATIVE_FUNC(invoke_vi, "(ii)"), + REG_NATIVE_FUNC(invoke_iii, "(iii)i"), + REG_NATIVE_FUNC(invoke_ii, "(ii)i"), + REG_NATIVE_FUNC(open, "($ii)i"), + REG_NATIVE_FUNC(__sys_read, "(i*~)i"), + REG_NATIVE_FUNC(__sys_stat64, "($*)i"), + REG_NATIVE_FUNC(__sys_fstat64, "(i*)i"), + REG_NATIVE_FUNC(mmap, "(*iiiiI)i"), + REG_NATIVE_FUNC(munmap, "(ii)i"), + REG_NATIVE_FUNC(__munmap, "(ii)i"), + REG_NATIVE_FUNC(getentropy, "(*~)i"), + REG_NATIVE_FUNC(setjmp, "(*)i"), + REG_NATIVE_FUNC(longjmp, "(*i)"), +#if !defined(BH_PLATFORM_LINUX_SGX) + REG_NATIVE_FUNC(fopen, "($$)i"), + REG_NATIVE_FUNC(fread, "(*iii)i"), + REG_NATIVE_FUNC(fseeko, "(iIi)i"), + REG_NATIVE_FUNC(emcc_fwrite, "(*iii)i"), + REG_NATIVE_FUNC(feof, "(i)i"), + REG_NATIVE_FUNC(fclose, "(i)i"), + REG_NATIVE_FUNC(__sys_mkdir, "($i)i"), + REG_NATIVE_FUNC(__sys_rmdir, "($)i"), + REG_NATIVE_FUNC(__sys_unlink, "($)i"), + REG_NATIVE_FUNC(__sys_getcwd, "(*~)i"), + REG_NATIVE_FUNC(__sys_uname, "(*)i"), + REG_NATIVE_FUNC(emscripten_notify_memory_growth, "(i)"), + REG_NATIVE_FUNC(emscripten_thread_sleep, "(F)"), +#endif /* end of BH_PLATFORM_LINUX_SGX */ +}; + +uint32 +get_libc_emcc_export_apis(NativeSymbol **p_libc_emcc_apis) +{ + *p_libc_emcc_apis = native_symbols_libc_emcc; + return sizeof(native_symbols_libc_emcc) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/FindLIBUV.cmake b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/FindLIBUV.cmake new file mode 100644 index 0000000..7949f06 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/FindLIBUV.cmake @@ -0,0 +1,28 @@ +# Copyright (C) 2023 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Find libuv library +# This module defines +# LIBUV_FOUND, if false, do not try to link to libuv +# LIBUV_LIBRARIES +# LIBUV_INCLUDE_DIR, where to find uv.h + +find_path(LIBUV_INCLUDE_DIR NAMES uv.h) +find_library(LIBUV_LIBRARIES NAMES uv libuv) + +include(FindPackageHandleStandardArgs) + +find_package_handle_standard_args( + LIBUV + FOUND_VAR LIBUV_FOUND + REQUIRED_VARS + LIBUV_LIBRARIES + LIBUV_INCLUDE_DIR +) + +if(WIN32) + list(APPEND LIBUV_LIBRARIES iphlpapi) + list(APPEND LIBUV_LIBRARIES psapi) + list(APPEND LIBUV_LIBRARIES userenv) + list(APPEND LIBUV_LIBRARIES ws2_32) +endif() diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/FindUVWASI.cmake b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/FindUVWASI.cmake new file mode 100644 index 0000000..88499ea --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/FindUVWASI.cmake @@ -0,0 +1,25 @@ +# Copyright (C) 2023 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Find libuvwasi library +# This module defines +# UVWASI_FOUND, if false, do not try to link to libuvwasi +# UVWASI_LIBRARIES +# UVWASI_INCLUDE_DIR, where to find headers + +find_path(UVWASI_INCLUDE_DIR NAMES uvwasi.h wasi_serdes.h wasi_types.h PATH_SUFFIXES uvwasi) +find_library(UVWASI_LIBRARIES NAMES uvwasi_a) + +include(FindPackageHandleStandardArgs) + +find_package_handle_standard_args( + UVWASI + FOUND_VAR UVWASI_FOUND + REQUIRED_VARS + UVWASI_LIBRARIES + UVWASI_INCLUDE_DIR +) + +if(UVWASI_FOUND) + set(UVWASI_INCLUDE_DIR ${UVWASI_INCLUDE_DIR}/uvwasi) +endif() diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/LICENSE_LIBUV b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/LICENSE_LIBUV new file mode 100644 index 0000000..eb126da --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/LICENSE_LIBUV @@ -0,0 +1,66 @@ +libuv is licensed for use as follows: + +==== +Copyright (c) 2015-present libuv project contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. +==== + +This license applies to parts of libuv originating from the +https://github.com/joyent/libuv repository: + +==== + +Copyright Joyent, Inc. and other Node contributors. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +==== + +This license applies to all parts of libuv that are not externally +maintained libraries. + +The externally maintained libraries used by libuv are: + + - tree.h (from FreeBSD), copyright Niels Provos. Two clause BSD license. + + - inet_pton and inet_ntop implementations, contained in src/inet.c, are + copyright the Internet Systems Consortium, Inc., and licensed under the ISC + license. + + - stdint-msvc2008.h (from msinttypes), copyright Alexander Chemeris. Three + clause BSD license. + + - pthread-fixes.c, copyright Google Inc. and Sony Mobile Communications AB. + Three clause BSD license. diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/LICENSE_UVWASI b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/LICENSE_UVWASI new file mode 100644 index 0000000..dfb8546 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/LICENSE_UVWASI @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Colin Ihrig and Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/libc_uvwasi.cmake b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/libc_uvwasi.cmake new file mode 100644 index 0000000..7a3bfbd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/libc_uvwasi.cmake @@ -0,0 +1,61 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIBC_WASI_DIR ${CMAKE_CURRENT_LIST_DIR}) + +set (LIBUV_VERSION v1.46.0) + +add_definitions (-DWASM_ENABLE_LIBC_WASI=1 -DWASM_ENABLE_UVWASI=1) + +include(FetchContent) + +# Point CMake at the custom modules to find libuv and uvwasi +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + +## libuv +find_package(LIBUV QUIET) +if (LIBUV_FOUND) + include_directories(${LIBUV_INCLUDE_DIR}) +else() + FetchContent_Declare( + libuv + GIT_REPOSITORY https://github.com/libuv/libuv.git + GIT_TAG ${LIBUV_VERSION} + ) + FetchContent_GetProperties(libuv) + if (NOT libuv_POPULATED) + message("-- Fetching libuv ..") + FetchContent_Populate(libuv) + include_directories("${libuv_SOURCE_DIR}/include") + add_subdirectory(${libuv_SOURCE_DIR} ${libuv_BINARY_DIR} EXCLUDE_FROM_ALL) + set (LIBUV_LIBRARIES uv_a) + set_target_properties(uv_a PROPERTIES POSITION_INDEPENDENT_CODE 1) + endif() +endif() + +## uvwasi +find_package(UVWASI QUIET) +if (UVWASI_FOUND) + include_directories(${UVWASI_INCLUDE_DIR}) +else() + FetchContent_Declare( + uvwasi + GIT_REPOSITORY https://github.com/nodejs/uvwasi.git + GIT_TAG main + ) + FetchContent_GetProperties(uvwasi) + if (NOT uvwasi_POPULATED) + message("-- Fetching uvwasi ..") + FetchContent_Populate(uvwasi) + include_directories("${uvwasi_SOURCE_DIR}/include") + add_subdirectory(${uvwasi_SOURCE_DIR} ${uvwasi_BINARY_DIR} EXCLUDE_FROM_ALL) + set (UVWASI_LIBRARIES uvwasi_a) + set_target_properties(uvwasi_a PROPERTIES POSITION_INDEPENDENT_CODE 1) + endif() +endif() + +set (UV_A_LIBS ${LIBUV_LIBRARIES} ${UVWASI_LIBRARIES}) + +file (GLOB_RECURSE source_all ${LIBC_WASI_DIR}/*.c) + +set (LIBC_WASI_SOURCE ${source_all}) diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/libc_uvwasi_wrapper.c b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/libc_uvwasi_wrapper.c new file mode 100644 index 0000000..35d091e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-uvwasi/libc_uvwasi_wrapper.c @@ -0,0 +1,1165 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "uvwasi.h" +#include "bh_platform.h" +#include "wasm_export.h" + +/* clang-format off */ +#define get_module_inst(exec_env) \ + wasm_runtime_get_module_inst(exec_env) + +#define validate_app_addr(offset, size) \ + wasm_runtime_validate_app_addr(module_inst, offset, size) + +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define addr_app_to_native(offset) \ + wasm_runtime_addr_app_to_native(module_inst, offset) + +#define addr_native_to_app(ptr) \ + wasm_runtime_addr_native_to_app(module_inst, ptr) + +#define module_malloc(size, p_native_addr) \ + wasm_runtime_module_malloc(module_inst, size, p_native_addr) + +#define module_free(offset) \ + wasm_runtime_module_free(module_inst, offset) +/* clang-format on */ + +// uvwasi_errno_t is typedef'd to uint16 which is correct according to the ABI +// specification. However, in WASM, the smallest integer type is int32. If we +// return uint16, we would rely on language SDKs to implement the correct +// behaviour of casting to uint16 before checking the value or using it any way. +// Failure to do so can cause tricky bugs as the upper 16 bits of the error +// result are not guaranteed to be zero'ed by us so the result essentially +// contains garbage from the WASM app perspective. To prevent this, we return +// uint32 directly instead so as not to be reliant on the correct behaviour of +// any current/future SDK implementations. +#define wasi_errno_t uint32_t +#define wasi_fd_t uvwasi_fd_t +#define wasi_clockid_t uvwasi_clockid_t +#define wasi_timestamp_t uvwasi_timestamp_t +#define wasi_filesize_t uvwasi_filesize_t +#define wasi_prestat_app_t uvwasi_prestat_app_t +#define wasi_filedelta_t uvwasi_filedelta_t +#define wasi_whence_t uvwasi_whence_t +#define wasi_fdflags_t uvwasi_fdflags_t +#define wasi_rights_t uvwasi_rights_t +#define wasi_advice_t uvwasi_advice_t +#define wasi_lookupflags_t uvwasi_lookupflags_t +#define wasi_preopentype_t uvwasi_preopentype_t +#define wasi_fdstat_t uvwasi_fdstat_t +#define wasi_oflags_t uvwasi_oflags_t +#define wasi_dircookie_t uvwasi_dircookie_t +#define wasi_filestat_t uvwasi_filestat_t +#define wasi_fstflags_t uvwasi_fstflags_t +#define wasi_subscription_t uvwasi_subscription_t +#define wasi_event_t uvwasi_event_t +#define wasi_exitcode_t uvwasi_exitcode_t +#define wasi_signal_t uvwasi_signal_t +#define wasi_riflags_t uvwasi_riflags_t +#define wasi_roflags_t uvwasi_roflags_t +#define wasi_siflags_t uvwasi_siflags_t +#define wasi_sdflags_t uvwasi_sdflags_t +#define wasi_iovec_t uvwasi_iovec_t +#define wasi_ciovec_t uvwasi_ciovec_t + +typedef struct wasi_prestat_app { + wasi_preopentype_t pr_type; + uint32 pr_name_len; +} wasi_prestat_app_t; + +typedef struct iovec_app { + uint32 buf_offset; + uint32 buf_len; +} iovec_app_t; + +typedef struct WASIContext { + uvwasi_t uvwasi; + uint32_t exit_code; +} WASIContext; + +void * +wasm_runtime_get_wasi_ctx(wasm_module_inst_t module_inst); + +static uvwasi_t * +get_wasi_ctx(wasm_module_inst_t module_inst) +{ + WASIContext *ctx = wasm_runtime_get_wasi_ctx(module_inst); + if (ctx == NULL) { + return NULL; + } + return &ctx->uvwasi; +} + +static wasi_errno_t +wasi_args_get(wasm_exec_env_t exec_env, uint32 *argv_offsets, char *argv_buf) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_size_t argc, argv_buf_size, i; + char **argv; + uint64 total_size; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + err = uvwasi_args_sizes_get(uvwasi, &argc, &argv_buf_size); + if (err) + return err; + + total_size = sizeof(int32) * ((uint64)argc + 1); + if (total_size >= UINT32_MAX + || !validate_native_addr(argv_offsets, total_size) + || argv_buf_size >= UINT32_MAX + || !validate_native_addr(argv_buf, (uint64)argv_buf_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(char *) * ((uint64)argc + 1); + if (total_size >= UINT32_MAX + || !(argv = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + err = uvwasi_args_get(uvwasi, argv, argv_buf); + if (err) { + wasm_runtime_free(argv); + return err; + } + + for (i = 0; i < argc; i++) + argv_offsets[i] = (uint32)addr_native_to_app(argv[i]); + + wasm_runtime_free(argv); + return 0; +} + +static wasi_errno_t +wasi_args_sizes_get(wasm_exec_env_t exec_env, uint32 *argc_app, + uint32 *argv_buf_size_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_size_t argc, argv_buf_size; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(argc_app, (uint64)sizeof(uint32)) + || !validate_native_addr(argv_buf_size_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = uvwasi_args_sizes_get(uvwasi, &argc, &argv_buf_size); + if (err) + return err; + + *argc_app = (uint32)argc; + *argv_buf_size_app = (uint32)argv_buf_size; + return 0; +} + +static wasi_errno_t +wasi_clock_res_get(wasm_exec_env_t exec_env, wasi_clockid_t clock_id, + wasi_timestamp_t *resolution) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!validate_native_addr(resolution, (uint64)sizeof(wasi_timestamp_t))) + return (wasi_errno_t)-1; + + return uvwasi_clock_res_get(uvwasi, clock_id, resolution); +} + +static wasi_errno_t +wasi_clock_time_get(wasm_exec_env_t exec_env, wasi_clockid_t clock_id, + wasi_timestamp_t precision, wasi_timestamp_t *time) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!validate_native_addr(time, (uint64)sizeof(wasi_timestamp_t))) + return (wasi_errno_t)-1; + + return uvwasi_clock_time_get(uvwasi, clock_id, precision, time); +} + +static wasi_errno_t +wasi_environ_get(wasm_exec_env_t exec_env, uint32 *environ_offsets, + char *environ_buf) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_size_t environ_count, environ_buf_size, i; + uint64 total_size; + char **environs; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + err = uvwasi_environ_sizes_get(uvwasi, &environ_count, &environ_buf_size); + if (err) + return err; + + if (environ_count == 0) + return 0; + + total_size = sizeof(int32) * ((uint64)environ_count + 1); + if (total_size >= UINT32_MAX + || !validate_native_addr(environ_offsets, total_size) + || environ_buf_size >= UINT32_MAX + || !validate_native_addr(environ_buf, (uint64)environ_buf_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(char *) * (((uint64)environ_count + 1)); + + if (total_size >= UINT32_MAX + || !(environs = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + err = uvwasi_environ_get(uvwasi, environs, environ_buf); + if (err) { + wasm_runtime_free(environs); + return err; + } + + for (i = 0; i < environ_count; i++) + environ_offsets[i] = (uint32)addr_native_to_app(environs[i]); + + wasm_runtime_free(environs); + return 0; +} + +static wasi_errno_t +wasi_environ_sizes_get(wasm_exec_env_t exec_env, uint32 *environ_count_app, + uint32 *environ_buf_size_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_size_t environ_count, environ_buf_size; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(environ_count_app, (uint64)sizeof(uint32)) + || !validate_native_addr(environ_buf_size_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = uvwasi_environ_sizes_get(uvwasi, &environ_count, &environ_buf_size); + if (err) + return err; + + *environ_count_app = (uint32)environ_count; + *environ_buf_size_app = (uint32)environ_buf_size; + return 0; +} + +static wasi_errno_t +wasi_fd_prestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_prestat_app_t *prestat_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_prestat_t prestat; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(prestat_app, (uint64)sizeof(wasi_prestat_app_t))) + return (wasi_errno_t)-1; + + err = uvwasi_fd_prestat_get(uvwasi, fd, &prestat); + if (err) + return err; + + prestat_app->pr_type = prestat.pr_type; + prestat_app->pr_name_len = (uint32)prestat.u.dir.pr_name_len; + return 0; +} + +static wasi_errno_t +wasi_fd_prestat_dir_name(wasm_exec_env_t exec_env, wasi_fd_t fd, char *path, + uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_prestat_dir_name(uvwasi, fd, path, path_len); +} + +static wasi_errno_t +wasi_fd_close(wasm_exec_env_t exec_env, wasi_fd_t fd) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_close(uvwasi, fd); +} + +static wasi_errno_t +wasi_fd_datasync(wasm_exec_env_t exec_env, wasi_fd_t fd) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_datasync(uvwasi, fd); +} + +static wasi_errno_t +wasi_fd_pread(wasm_exec_env_t exec_env, wasi_fd_t fd, iovec_app_t *iovec_app, + uint32 iovs_len, wasi_filesize_t offset, uint32 *nread_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_iovec_t *iovec, *iovec_begin; + uint64 total_size; + uvwasi_size_t nread; + uint32 i; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nread_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr(iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_iovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(iovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + iovec = iovec_begin; + for (i = 0; i < iovs_len; i++, iovec_app++, iovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + iovec->buf = (void *)addr_app_to_native((uint64)iovec_app->buf_offset); + iovec->buf_len = iovec_app->buf_len; + } + + err = uvwasi_fd_pread(uvwasi, fd, iovec_begin, iovs_len, offset, &nread); + if (err) + goto fail; + + *nread_app = (uint32)nread; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(iovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_pwrite(wasm_exec_env_t exec_env, wasi_fd_t fd, + const iovec_app_t *iovec_app, uint32 iovs_len, + wasi_filesize_t offset, uint32 *nwritten_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_ciovec_t *ciovec, *ciovec_begin; + uint64 total_size; + uvwasi_size_t nwritten; + uint32 i; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nwritten_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr((void *)iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_ciovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(ciovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + ciovec = ciovec_begin; + for (i = 0; i < iovs_len; i++, iovec_app++, ciovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + ciovec->buf = (char *)addr_app_to_native((uint64)iovec_app->buf_offset); + ciovec->buf_len = iovec_app->buf_len; + } + + err = + uvwasi_fd_pwrite(uvwasi, fd, ciovec_begin, iovs_len, offset, &nwritten); + if (err) + goto fail; + + *nwritten_app = (uint32)nwritten; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(ciovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_read(wasm_exec_env_t exec_env, wasi_fd_t fd, + const iovec_app_t *iovec_app, uint32 iovs_len, uint32 *nread_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_iovec_t *iovec, *iovec_begin; + uint64 total_size; + uvwasi_size_t nread; + uint32 i; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nread_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr((void *)iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_iovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(iovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + iovec = iovec_begin; + for (i = 0; i < iovs_len; i++, iovec_app++, iovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + iovec->buf = (void *)addr_app_to_native((uint64)iovec_app->buf_offset); + iovec->buf_len = iovec_app->buf_len; + } + + err = uvwasi_fd_read(uvwasi, fd, iovec_begin, iovs_len, &nread); + if (err) + goto fail; + + *nread_app = (uint32)nread; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(iovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_renumber(wasm_exec_env_t exec_env, wasi_fd_t from, wasi_fd_t to) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_renumber(uvwasi, from, to); +} + +static wasi_errno_t +wasi_fd_seek(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filedelta_t offset, + wasi_whence_t whence, wasi_filesize_t *newoffset) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(newoffset, (uint64)sizeof(wasi_filesize_t))) + return (wasi_errno_t)-1; + + return uvwasi_fd_seek(uvwasi, fd, offset, whence, newoffset); +} + +static wasi_errno_t +wasi_fd_tell(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filesize_t *newoffset) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(newoffset, (uint64)sizeof(wasi_filesize_t))) + return (wasi_errno_t)-1; + + return uvwasi_fd_tell(uvwasi, fd, newoffset); +} + +static wasi_errno_t +wasi_fd_fdstat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_fdstat_t *fdstat_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_fdstat_t fdstat; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(fdstat_app, (uint64)sizeof(wasi_fdstat_t))) + return (wasi_errno_t)-1; + + err = uvwasi_fd_fdstat_get(uvwasi, fd, &fdstat); + if (err) + return err; + + memcpy(fdstat_app, &fdstat, sizeof(wasi_fdstat_t)); + return 0; +} + +static wasi_errno_t +wasi_fd_fdstat_set_flags(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_fdflags_t flags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_fdstat_set_flags(uvwasi, fd, flags); +} + +static wasi_errno_t +wasi_fd_fdstat_set_rights(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_rights_t fs_rights_base, + wasi_rights_t fs_rights_inheriting) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_fdstat_set_rights(uvwasi, fd, fs_rights_base, + fs_rights_inheriting); +} + +static wasi_errno_t +wasi_fd_sync(wasm_exec_env_t exec_env, wasi_fd_t fd) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_sync(uvwasi, fd); +} + +static wasi_errno_t +wasi_fd_write(wasm_exec_env_t exec_env, wasi_fd_t fd, + const iovec_app_t *iovec_app, uint32 iovs_len, + uint32 *nwritten_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_ciovec_t *ciovec, *ciovec_begin; + uint64 total_size; + uvwasi_size_t nwritten; + uint32 i; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nwritten_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr((void *)iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_ciovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(ciovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + ciovec = ciovec_begin; + for (i = 0; i < iovs_len; i++, iovec_app++, ciovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + ciovec->buf = (char *)addr_app_to_native((uint64)iovec_app->buf_offset); + ciovec->buf_len = iovec_app->buf_len; + } + +#ifndef BH_VPRINTF + err = uvwasi_fd_write(uvwasi, fd, ciovec_begin, iovs_len, &nwritten); +#else + /* redirect stdout/stderr output to BH_VPRINTF function */ + if (fd == 1 || fd == 2) { + int i; + const struct iovec *iov1 = (const struct iovec *)ciovec_begin; + + nwritten = 0; + for (i = 0; i < (int)iovs_len; i++, iov1++) { + if (iov1->iov_len > 0 && iov1->iov_base) { + char format[16]; + + /* make up format string "%.ns" */ + snprintf(format, sizeof(format), "%%.%ds", (int)iov1->iov_len); + nwritten += (uvwasi_size_t)os_printf(format, iov1->iov_base); + } + } + err = 0; + } + else { + err = uvwasi_fd_write(uvwasi, fd, ciovec_begin, iovs_len, &nwritten); + } +#endif /* end of BH_VPRINTF */ + + if (err) + goto fail; + + *nwritten_app = (uint32)nwritten; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(ciovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_advise(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filesize_t offset, + wasi_filesize_t len, wasi_advice_t advice) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_advise(uvwasi, fd, offset, len, advice); +} + +static wasi_errno_t +wasi_fd_allocate(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filesize_t offset, + wasi_filesize_t len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_allocate(uvwasi, fd, offset, len); +} + +static wasi_errno_t +wasi_path_create_directory(wasm_exec_env_t exec_env, wasi_fd_t fd, + const char *path, uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_path_create_directory(uvwasi, fd, path, path_len); +} + +static wasi_errno_t +wasi_path_link(wasm_exec_env_t exec_env, wasi_fd_t old_fd, + wasi_lookupflags_t old_flags, const char *old_path, + uint32 old_path_len, wasi_fd_t new_fd, const char *new_path, + uint32 new_path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_path_link(uvwasi, old_fd, old_flags, old_path, old_path_len, + new_fd, new_path, new_path_len); +} + +static wasi_errno_t +wasi_path_open(wasm_exec_env_t exec_env, wasi_fd_t dirfd, + wasi_lookupflags_t dirflags, const char *path, uint32 path_len, + wasi_oflags_t oflags, wasi_rights_t fs_rights_base, + wasi_rights_t fs_rights_inheriting, wasi_fdflags_t fs_flags, + wasi_fd_t *fd_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_fd_t fd = (wasi_fd_t)-1; /* set fd_app -1 if path open failed */ + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(fd_app, (uint64)sizeof(wasi_fd_t))) + return (wasi_errno_t)-1; + + err = uvwasi_path_open(uvwasi, dirfd, dirflags, path, path_len, oflags, + fs_rights_base, fs_rights_inheriting, fs_flags, &fd); + + *fd_app = fd; + return err; +} + +static wasi_errno_t +wasi_fd_readdir(wasm_exec_env_t exec_env, wasi_fd_t fd, void *buf, + uint32 buf_len, wasi_dircookie_t cookie, uint32 *bufused_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_size_t bufused; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(bufused_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = uvwasi_fd_readdir(uvwasi, fd, buf, buf_len, cookie, &bufused); + if (err) + return err; + + *bufused_app = (uint32)bufused; + return 0; +} + +static wasi_errno_t +wasi_path_readlink(wasm_exec_env_t exec_env, wasi_fd_t fd, const char *path, + uint32 path_len, char *buf, uint32 buf_len, + uint32 *bufused_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_size_t bufused; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(bufused_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = uvwasi_path_readlink(uvwasi, fd, path, path_len, buf, buf_len, + &bufused); + if (err) + return err; + + *bufused_app = (uint32)bufused; + return 0; +} + +static wasi_errno_t +wasi_path_rename(wasm_exec_env_t exec_env, wasi_fd_t old_fd, + const char *old_path, uint32 old_path_len, wasi_fd_t new_fd, + const char *new_path, uint32 new_path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_path_rename(uvwasi, old_fd, old_path, old_path_len, new_fd, + new_path, new_path_len); +} + +static wasi_errno_t +wasi_fd_filestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_filestat_t *filestat) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(filestat, (uint64)sizeof(wasi_filestat_t))) + return (wasi_errno_t)-1; + + return uvwasi_fd_filestat_get(uvwasi, fd, filestat); +} + +static wasi_errno_t +wasi_fd_filestat_set_times(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_timestamp_t st_atim, wasi_timestamp_t st_mtim, + wasi_fstflags_t fstflags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_filestat_set_times(uvwasi, fd, st_atim, st_mtim, fstflags); +} + +static wasi_errno_t +wasi_fd_filestat_set_size(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_filesize_t st_size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_fd_filestat_set_size(uvwasi, fd, st_size); +} + +static wasi_errno_t +wasi_path_filestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_lookupflags_t flags, const char *path, + uint32 path_len, wasi_filestat_t *filestat) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr(filestat, (uint64)sizeof(wasi_filestat_t))) + return (wasi_errno_t)-1; + + return uvwasi_path_filestat_get(uvwasi, fd, flags, path, path_len, + filestat); +} + +static wasi_errno_t +wasi_path_filestat_set_times(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_lookupflags_t flags, const char *path, + uint32 path_len, wasi_timestamp_t st_atim, + wasi_timestamp_t st_mtim, wasi_fstflags_t fstflags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_path_filestat_set_times(uvwasi, fd, flags, path, path_len, + st_atim, st_mtim, fstflags); +} + +static wasi_errno_t +wasi_path_symlink(wasm_exec_env_t exec_env, const char *old_path, + uint32 old_path_len, wasi_fd_t fd, const char *new_path, + uint32 new_path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_path_symlink(uvwasi, old_path, old_path_len, fd, new_path, + new_path_len); +} + +static wasi_errno_t +wasi_path_unlink_file(wasm_exec_env_t exec_env, wasi_fd_t fd, const char *path, + uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_path_unlink_file(uvwasi, fd, path, path_len); +} + +static wasi_errno_t +wasi_path_remove_directory(wasm_exec_env_t exec_env, wasi_fd_t fd, + const char *path, uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_path_remove_directory(uvwasi, fd, path, path_len); +} + +static wasi_errno_t +wasi_poll_oneoff(wasm_exec_env_t exec_env, const wasi_subscription_t *in, + wasi_event_t *out, uint32 nsubscriptions, uint32 *nevents_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + uvwasi_size_t nevents; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + if (!validate_native_addr((void *)in, (uint64)sizeof(wasi_subscription_t)) + || !validate_native_addr(out, (uint64)sizeof(wasi_event_t)) + || !validate_native_addr(nevents_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = uvwasi_poll_oneoff(uvwasi, in, out, nsubscriptions, &nevents); + if (err) + return err; + + *nevents_app = (uint32)nevents; + return 0; +} + +static void +wasi_proc_exit(wasm_exec_env_t exec_env, wasi_exitcode_t rval) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + WASIContext *wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst); + /* Here throwing exception is just to let wasm app exit, + the upper layer should clear the exception and return + as normal */ + wasm_runtime_set_exception(module_inst, "wasi proc exit"); + wasi_ctx->exit_code = rval; +} + +static wasi_errno_t +wasi_proc_raise(wasm_exec_env_t exec_env, wasi_signal_t sig) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char buf[32]; + + snprintf(buf, sizeof(buf), "%s%d", "wasi proc raise ", sig); + wasm_runtime_set_exception(module_inst, buf); + return 0; +} + +static wasi_errno_t +wasi_random_get(wasm_exec_env_t exec_env, void *buf, uint32 buf_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + return uvwasi_random_get(uvwasi, buf, buf_len); +} + +static wasi_errno_t +wasi_sock_recv(wasm_exec_env_t exec_env, wasi_fd_t sock, iovec_app_t *ri_data, + uint32 ri_data_len, wasi_riflags_t ri_flags, + uint32 *ro_datalen_app, wasi_roflags_t *ro_flags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_iovec_t *iovec, *iovec_begin; + uint64 total_size; + uvwasi_size_t ro_datalen; + uint32 i; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)ri_data_len; + if (!validate_native_addr(ro_datalen_app, (uint32)sizeof(uint32)) + || !validate_native_addr(ro_flags, (uint32)sizeof(wasi_roflags_t)) + || total_size >= UINT32_MAX + || !validate_native_addr(ri_data, (uint32)total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_iovec_t) * (uint64)ri_data_len; + if (total_size >= UINT32_MAX + || !(iovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + iovec = iovec_begin; + for (i = 0; i < ri_data_len; i++, ri_data++, iovec++) { + if (!validate_app_addr((uint64)ri_data->buf_offset, + (uint64)ri_data->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + iovec->buf = (void *)addr_app_to_native((uint64)ri_data->buf_offset); + iovec->buf_len = ri_data->buf_len; + } + + err = uvwasi_sock_recv(uvwasi, sock, iovec_begin, ri_data_len, ri_flags, + &ro_datalen, ro_flags); + if (err) + goto fail; + + *(uint32 *)ro_datalen_app = (uint32)ro_datalen; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(iovec_begin); + return err; +} + +static wasi_errno_t +wasi_sock_send(wasm_exec_env_t exec_env, wasi_fd_t sock, + const iovec_app_t *si_data, uint32 si_data_len, + wasi_siflags_t si_flags, uint32 *so_datalen_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + wasi_ciovec_t *ciovec, *ciovec_begin; + uint64 total_size; + uvwasi_size_t so_datalen; + uint32 i; + wasi_errno_t err; + + if (!uvwasi) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)si_data_len; + if (!validate_native_addr(so_datalen_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr((void *)si_data, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_ciovec_t) * (uint64)si_data_len; + if (total_size >= UINT32_MAX + || !(ciovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + ciovec = ciovec_begin; + for (i = 0; i < si_data_len; i++, si_data++, ciovec++) { + if (!validate_app_addr((uint64)si_data->buf_offset, + (uint64)si_data->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + ciovec->buf = (char *)addr_app_to_native((uint64)si_data->buf_offset); + ciovec->buf_len = si_data->buf_len; + } + + err = uvwasi_sock_send(uvwasi, sock, ciovec_begin, si_data_len, si_flags, + &so_datalen); + if (err) + goto fail; + + *so_datalen_app = (uint32)so_datalen; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(ciovec_begin); + return err; +} + +static wasi_errno_t +wasi_sock_shutdown(wasm_exec_env_t exec_env, wasi_fd_t sock, wasi_sdflags_t how) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + if (!uvwasi) + return (wasi_errno_t)-1; + + return uvwasi_sock_shutdown(uvwasi, sock, how); +} + +static wasi_errno_t +wasi_sched_yield(wasm_exec_env_t exec_env) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + uvwasi_t *uvwasi = get_wasi_ctx(module_inst); + + return uvwasi_sched_yield(uvwasi); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, wasi_##func_name, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_libc_wasi[] = { + REG_NATIVE_FUNC(args_get, "(**)i"), + REG_NATIVE_FUNC(args_sizes_get, "(**)i"), + REG_NATIVE_FUNC(clock_res_get, "(i*)i"), + REG_NATIVE_FUNC(clock_time_get, "(iI*)i"), + REG_NATIVE_FUNC(environ_get, "(**)i"), + REG_NATIVE_FUNC(environ_sizes_get, "(**)i"), + REG_NATIVE_FUNC(fd_prestat_get, "(i*)i"), + REG_NATIVE_FUNC(fd_prestat_dir_name, "(i*~)i"), + REG_NATIVE_FUNC(fd_close, "(i)i"), + REG_NATIVE_FUNC(fd_datasync, "(i)i"), + REG_NATIVE_FUNC(fd_pread, "(i*iI*)i"), + REG_NATIVE_FUNC(fd_pwrite, "(i*iI*)i"), + REG_NATIVE_FUNC(fd_read, "(i*i*)i"), + REG_NATIVE_FUNC(fd_renumber, "(ii)i"), + REG_NATIVE_FUNC(fd_seek, "(iIi*)i"), + REG_NATIVE_FUNC(fd_tell, "(i*)i"), + REG_NATIVE_FUNC(fd_fdstat_get, "(i*)i"), + REG_NATIVE_FUNC(fd_fdstat_set_flags, "(ii)i"), + REG_NATIVE_FUNC(fd_fdstat_set_rights, "(iII)i"), + REG_NATIVE_FUNC(fd_sync, "(i)i"), + REG_NATIVE_FUNC(fd_write, "(i*i*)i"), + REG_NATIVE_FUNC(fd_advise, "(iIIi)i"), + REG_NATIVE_FUNC(fd_allocate, "(iII)i"), + REG_NATIVE_FUNC(path_create_directory, "(i*~)i"), + REG_NATIVE_FUNC(path_link, "(ii*~i*~)i"), + REG_NATIVE_FUNC(path_open, "(ii*~iIIi*)i"), + REG_NATIVE_FUNC(fd_readdir, "(i*~I*)i"), + REG_NATIVE_FUNC(path_readlink, "(i*~*~*)i"), + REG_NATIVE_FUNC(path_rename, "(i*~i*~)i"), + REG_NATIVE_FUNC(fd_filestat_get, "(i*)i"), + REG_NATIVE_FUNC(fd_filestat_set_times, "(iIIi)i"), + REG_NATIVE_FUNC(fd_filestat_set_size, "(iI)i"), + REG_NATIVE_FUNC(path_filestat_get, "(ii*~*)i"), + REG_NATIVE_FUNC(path_filestat_set_times, "(ii*~IIi)i"), + REG_NATIVE_FUNC(path_symlink, "(*~i*~)i"), + REG_NATIVE_FUNC(path_unlink_file, "(i*~)i"), + REG_NATIVE_FUNC(path_remove_directory, "(i*~)i"), + REG_NATIVE_FUNC(poll_oneoff, "(**i*)i"), + REG_NATIVE_FUNC(proc_exit, "(i)"), + REG_NATIVE_FUNC(proc_raise, "(i)i"), + REG_NATIVE_FUNC(random_get, "(*~)i"), + REG_NATIVE_FUNC(sock_recv, "(i*ii**)i"), + REG_NATIVE_FUNC(sock_send, "(i*ii*)i"), + REG_NATIVE_FUNC(sock_shutdown, "(ii)i"), + REG_NATIVE_FUNC(sched_yield, "()i"), +}; + +uint32 +get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis) +{ + *p_libc_wasi_apis = native_symbols_libc_wasi; + return sizeof(native_symbols_libc_wasi) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/SConscript b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/SConscript new file mode 100644 index 0000000..6ed3799 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/SConscript @@ -0,0 +1,34 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * +import os + +cwd = GetCurrentDir() + +src = Split(''' +''') + +CPPPATH = [cwd, + cwd+'/sandboxed-system-primitives/include', + cwd+'/sandboxed-system-primitives/src'] + +def addSrcFiles(arr, path): + for f in os.listdir(path): + fpath = os.path.join(path, f); + if os.path.isfile(fpath): + ext = os.path.splitext(fpath)[-1] + if ext == '.c' or ext == '.cpp': + arr += [fpath] + elif os.path.isdir(fpath): + addSrcFiles(arr, fpath) + + +addSrcFiles(src, cwd) + +group = DefineGroup('iwasm_libc_wasi', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi.cmake b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi.cmake new file mode 100644 index 0000000..d72c42a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi.cmake @@ -0,0 +1,13 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIBC_WASI_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_LIBC_WASI=1) + +include_directories(${LIBC_WASI_DIR}/sandboxed-system-primitives/include + ${LIBC_WASI_DIR}/sandboxed-system-primitives/src) + +file (GLOB_RECURSE source_all ${LIBC_WASI_DIR}/*.c ) + +set (LIBC_WASI_SOURCE ${source_all}) diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c new file mode 100644 index 0000000..aef8f17 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c @@ -0,0 +1,2364 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "libc_wasi_wrapper.h" +#include "bh_platform.h" +#include "wasm_export.h" +#include "wasm_runtime_common.h" +#include "wasmtime_ssp.h" + +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../../../thread-mgr/thread_manager.h" +#endif + +void +wasm_runtime_set_exception(wasm_module_inst_t module, const char *exception); + +/* clang-format off */ +#define get_module_inst(exec_env) \ + wasm_runtime_get_module_inst(exec_env) + +#define get_wasi_ctx(module_inst) \ + wasm_runtime_get_wasi_ctx(module_inst) + +#define validate_app_addr(offset, size) \ + wasm_runtime_validate_app_addr(module_inst, offset, size) + +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define addr_app_to_native(offset) \ + wasm_runtime_addr_app_to_native(module_inst, offset) + +#define addr_native_to_app(ptr) \ + wasm_runtime_addr_native_to_app(module_inst, ptr) + +#define module_malloc(size, p_native_addr) \ + wasm_runtime_module_malloc(module_inst, size, p_native_addr) + +#define module_free(offset) \ + wasm_runtime_module_free(module_inst, offset) +/* clang-format on */ + +typedef struct wasi_prestat_app { + wasi_preopentype_t pr_type; + uint32 pr_name_len; +} wasi_prestat_app_t; + +typedef struct iovec_app { + uint32 buf_offset; + uint32 buf_len; +} iovec_app_t; + +typedef struct WASIContext *wasi_ctx_t; + +wasi_ctx_t +wasm_runtime_get_wasi_ctx(wasm_module_inst_t module_inst); + +#if WASM_ENABLE_THREAD_MGR != 0 +static inline uint64_t +min_uint64(uint64_t a, uint64_t b) +{ + return a > b ? b : a; +} +#endif + +static inline uint32_t +min_uint32(uint32_t a, uint32_t b) +{ + return a > b ? b : a; +} + +static inline struct fd_table * +wasi_ctx_get_curfds(wasi_ctx_t wasi_ctx) +{ + if (!wasi_ctx) + return NULL; + return wasi_ctx->curfds; +} + +static inline struct argv_environ_values * +wasi_ctx_get_argv_environ(wasm_module_inst_t module_inst, wasi_ctx_t wasi_ctx) +{ + if (!wasi_ctx) + return NULL; + return wasi_ctx->argv_environ; +} + +static inline struct fd_prestats * +wasi_ctx_get_prestats(wasi_ctx_t wasi_ctx) +{ + if (!wasi_ctx) + return NULL; + return wasi_ctx->prestats; +} + +static inline struct addr_pool * +wasi_ctx_get_addr_pool(wasi_ctx_t wasi_ctx) +{ + if (!wasi_ctx) + return NULL; + return wasi_ctx->addr_pool; +} + +static inline char ** +wasi_ctx_get_ns_lookup_list(wasi_ctx_t wasi_ctx) +{ + if (!wasi_ctx) + return NULL; + return wasi_ctx->ns_lookup_list; +} + +static wasi_errno_t +wasi_args_get(wasm_exec_env_t exec_env, uint32 *argv_offsets, char *argv_buf) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct argv_environ_values *argv_environ = + wasi_ctx_get_argv_environ(module_inst, wasi_ctx); + size_t argc, argv_buf_size, i; + char **argv; + uint64 total_size; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_args_sizes_get(argv_environ, &argc, &argv_buf_size); + if (err) + return err; + + total_size = sizeof(int32) * ((uint64)argc + 1); + if (total_size >= UINT32_MAX + || !validate_native_addr(argv_offsets, total_size) + || argv_buf_size >= UINT32_MAX + || !validate_native_addr(argv_buf, (uint64)argv_buf_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(char *) * ((uint64)argc + 1); + if (total_size >= UINT32_MAX + || !(argv = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_args_get(argv_environ, argv, argv_buf); + if (err) { + wasm_runtime_free(argv); + return err; + } + + for (i = 0; i < argc; i++) + argv_offsets[i] = (uint32)addr_native_to_app(argv[i]); + + wasm_runtime_free(argv); + return 0; +} + +static wasi_errno_t +wasi_args_sizes_get(wasm_exec_env_t exec_env, uint32 *argc_app, + uint32 *argv_buf_size_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct argv_environ_values *argv_environ; + size_t argc, argv_buf_size; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(argc_app, (uint64)sizeof(uint32)) + || !validate_native_addr(argv_buf_size_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + argv_environ = wasi_ctx->argv_environ; + + err = wasmtime_ssp_args_sizes_get(argv_environ, &argc, &argv_buf_size); + if (err) + return err; + + *argc_app = (uint32)argc; + *argv_buf_size_app = (uint32)argv_buf_size; + return 0; +} + +static wasi_errno_t +wasi_clock_res_get(wasm_exec_env_t exec_env, + wasi_clockid_t clock_id, /* uint32 clock_id */ + wasi_timestamp_t *resolution /* uint64 *resolution */) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + if (!validate_native_addr(resolution, (uint64)sizeof(wasi_timestamp_t))) + return (wasi_errno_t)-1; + + return os_clock_res_get(clock_id, resolution); +} + +static wasi_errno_t +wasi_clock_time_get(wasm_exec_env_t exec_env, + wasi_clockid_t clock_id, /* uint32 clock_id */ + wasi_timestamp_t precision, /* uint64 precision */ + wasi_timestamp_t *time /* uint64 *time */) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + if (!validate_native_addr(time, (uint64)sizeof(wasi_timestamp_t))) + return (wasi_errno_t)-1; + + return os_clock_time_get(clock_id, precision, time); +} + +static wasi_errno_t +wasi_environ_get(wasm_exec_env_t exec_env, uint32 *environ_offsets, + char *environ_buf) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct argv_environ_values *argv_environ = + wasi_ctx_get_argv_environ(module_inst, wasi_ctx); + size_t environ_count, environ_buf_size, i; + uint64 total_size; + char **environs; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_environ_sizes_get(argv_environ, &environ_count, + &environ_buf_size); + if (err) + return err; + + total_size = sizeof(int32) * ((uint64)environ_count + 1); + if (total_size >= UINT32_MAX + || !validate_native_addr(environ_offsets, total_size) + || environ_buf_size >= UINT32_MAX + || !validate_native_addr(environ_buf, (uint64)environ_buf_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(char *) * (((uint64)environ_count + 1)); + + if (total_size >= UINT32_MAX + || !(environs = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_environ_get(argv_environ, environs, environ_buf); + if (err) { + wasm_runtime_free(environs); + return err; + } + + for (i = 0; i < environ_count; i++) + environ_offsets[i] = (uint32)addr_native_to_app(environs[i]); + + wasm_runtime_free(environs); + return 0; +} + +static wasi_errno_t +wasi_environ_sizes_get(wasm_exec_env_t exec_env, uint32 *environ_count_app, + uint32 *environ_buf_size_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct argv_environ_values *argv_environ = + wasi_ctx_get_argv_environ(module_inst, wasi_ctx); + size_t environ_count, environ_buf_size; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(environ_count_app, (uint64)sizeof(uint32)) + || !validate_native_addr(environ_buf_size_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_environ_sizes_get(argv_environ, &environ_count, + &environ_buf_size); + if (err) + return err; + + *environ_count_app = (uint32)environ_count; + *environ_buf_size_app = (uint32)environ_buf_size; + + return 0; +} + +static wasi_errno_t +wasi_fd_prestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_prestat_app_t *prestat_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); + wasi_prestat_t prestat; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(prestat_app, (uint64)sizeof(wasi_prestat_app_t))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_fd_prestat_get(prestats, fd, &prestat); + if (err) + return err; + + prestat_app->pr_type = prestat.pr_type; + prestat_app->pr_name_len = (uint32)prestat.u.dir.pr_name_len; + return 0; +} + +static wasi_errno_t +wasi_fd_prestat_dir_name(wasm_exec_env_t exec_env, wasi_fd_t fd, char *path, + uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_prestat_dir_name(prestats, fd, path, path_len); +} + +static wasi_errno_t +wasi_fd_close(wasm_exec_env_t exec_env, wasi_fd_t fd) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_close(exec_env, curfds, prestats, fd); +} + +static wasi_errno_t +wasi_fd_datasync(wasm_exec_env_t exec_env, wasi_fd_t fd) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_datasync(exec_env, curfds, fd); +} + +static wasi_errno_t +wasi_fd_pread(wasm_exec_env_t exec_env, wasi_fd_t fd, iovec_app_t *iovec_app, + uint32 iovs_len, wasi_filesize_t offset, uint32 *nread_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + wasi_iovec_t *iovec, *iovec_begin; + uint64 total_size; + size_t nread; + uint32 i; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nread_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr(iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_iovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(iovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + iovec = iovec_begin; + + for (i = 0; i < iovs_len; i++, iovec_app++, iovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + iovec->buf = (void *)addr_app_to_native((uint64)iovec_app->buf_offset); + iovec->buf_len = iovec_app->buf_len; + } + + err = wasmtime_ssp_fd_pread(exec_env, curfds, fd, iovec_begin, iovs_len, + offset, &nread); + if (err) + goto fail; + + *nread_app = (uint32)nread; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(iovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_pwrite(wasm_exec_env_t exec_env, wasi_fd_t fd, + const iovec_app_t *iovec_app, uint32 iovs_len, + wasi_filesize_t offset, uint32 *nwritten_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + wasi_ciovec_t *ciovec, *ciovec_begin; + uint64 total_size; + size_t nwritten; + uint32 i; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nwritten_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr((void *)iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_ciovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(ciovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + ciovec = ciovec_begin; + + for (i = 0; i < iovs_len; i++, iovec_app++, ciovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + ciovec->buf = (char *)addr_app_to_native((uint64)iovec_app->buf_offset); + ciovec->buf_len = iovec_app->buf_len; + } + + err = wasmtime_ssp_fd_pwrite(exec_env, curfds, fd, ciovec_begin, iovs_len, + offset, &nwritten); + if (err) + goto fail; + + *nwritten_app = (uint32)nwritten; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(ciovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_read(wasm_exec_env_t exec_env, wasi_fd_t fd, + const iovec_app_t *iovec_app, uint32 iovs_len, uint32 *nread_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + wasi_iovec_t *iovec, *iovec_begin; + uint64 total_size; + size_t nread; + uint32 i; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nread_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr((void *)iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_iovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(iovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + iovec = iovec_begin; + + for (i = 0; i < iovs_len; i++, iovec_app++, iovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + iovec->buf = (void *)addr_app_to_native((uint64)iovec_app->buf_offset); + iovec->buf_len = iovec_app->buf_len; + } + + err = wasmtime_ssp_fd_read(exec_env, curfds, fd, iovec_begin, iovs_len, + &nread); + if (err) + goto fail; + + *nread_app = (uint32)nread; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(iovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_renumber(wasm_exec_env_t exec_env, wasi_fd_t from, wasi_fd_t to) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_renumber(exec_env, curfds, prestats, from, to); +} + +static wasi_errno_t +wasi_fd_seek(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filedelta_t offset, + wasi_whence_t whence, wasi_filesize_t *newoffset) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(newoffset, (uint64)sizeof(wasi_filesize_t))) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_seek(exec_env, curfds, fd, offset, whence, + newoffset); +} + +static wasi_errno_t +wasi_fd_tell(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filesize_t *newoffset) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(newoffset, (uint64)sizeof(wasi_filesize_t))) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_tell(exec_env, curfds, fd, newoffset); +} + +static wasi_errno_t +wasi_fd_fdstat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_fdstat_t *fdstat_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + wasi_fdstat_t fdstat; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(fdstat_app, (uint64)sizeof(wasi_fdstat_t))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_fd_fdstat_get(exec_env, curfds, fd, &fdstat); + if (err) + return err; + + memcpy(fdstat_app, &fdstat, sizeof(wasi_fdstat_t)); + return 0; +} + +static wasi_errno_t +wasi_fd_fdstat_set_flags(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_fdflags_t flags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_fdstat_set_flags(exec_env, curfds, fd, flags); +} + +static wasi_errno_t +wasi_fd_fdstat_set_rights(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_rights_t fs_rights_base, + wasi_rights_t fs_rights_inheriting) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_fdstat_set_rights( + exec_env, curfds, fd, fs_rights_base, fs_rights_inheriting); +} + +static wasi_errno_t +wasi_fd_sync(wasm_exec_env_t exec_env, wasi_fd_t fd) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_sync(exec_env, curfds, fd); +} + +static wasi_errno_t +wasi_fd_write(wasm_exec_env_t exec_env, wasi_fd_t fd, + const iovec_app_t *iovec_app, uint32 iovs_len, + uint32 *nwritten_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + wasi_ciovec_t *ciovec, *ciovec_begin; + uint64 total_size; + size_t nwritten; + uint32 i; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + total_size = sizeof(iovec_app_t) * (uint64)iovs_len; + if (!validate_native_addr(nwritten_app, (uint64)sizeof(uint32)) + || total_size >= UINT32_MAX + || !validate_native_addr((void *)iovec_app, total_size)) + return (wasi_errno_t)-1; + + total_size = sizeof(wasi_ciovec_t) * (uint64)iovs_len; + if (total_size >= UINT32_MAX + || !(ciovec_begin = wasm_runtime_malloc((uint32)total_size))) + return (wasi_errno_t)-1; + + ciovec = ciovec_begin; + + for (i = 0; i < iovs_len; i++, iovec_app++, ciovec++) { + if (!validate_app_addr((uint64)iovec_app->buf_offset, + (uint64)iovec_app->buf_len)) { + err = (wasi_errno_t)-1; + goto fail; + } + ciovec->buf = (char *)addr_app_to_native((uint64)iovec_app->buf_offset); + ciovec->buf_len = iovec_app->buf_len; + } + + err = wasmtime_ssp_fd_write(exec_env, curfds, fd, ciovec_begin, iovs_len, + &nwritten); + if (err) + goto fail; + + *nwritten_app = (uint32)nwritten; + + /* success */ + err = 0; + +fail: + wasm_runtime_free(ciovec_begin); + return err; +} + +static wasi_errno_t +wasi_fd_advise(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filesize_t offset, + wasi_filesize_t len, wasi_advice_t advice) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_advise(exec_env, curfds, fd, offset, len, advice); +} + +static wasi_errno_t +wasi_fd_allocate(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filesize_t offset, + wasi_filesize_t len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_allocate(exec_env, curfds, fd, offset, len); +} + +static wasi_errno_t +wasi_path_create_directory(wasm_exec_env_t exec_env, wasi_fd_t fd, + const char *path, uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_create_directory(exec_env, curfds, fd, path, + path_len); +} + +static wasi_errno_t +wasi_path_link(wasm_exec_env_t exec_env, wasi_fd_t old_fd, + wasi_lookupflags_t old_flags, const char *old_path, + uint32 old_path_len, wasi_fd_t new_fd, const char *new_path, + uint32 new_path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_link(exec_env, curfds, prestats, old_fd, old_flags, + old_path, old_path_len, new_fd, new_path, + new_path_len); +} + +static wasi_errno_t +wasi_path_open(wasm_exec_env_t exec_env, wasi_fd_t dirfd, + wasi_lookupflags_t dirflags, const char *path, uint32 path_len, + wasi_oflags_t oflags, wasi_rights_t fs_rights_base, + wasi_rights_t fs_rights_inheriting, wasi_fdflags_t fs_flags, + wasi_fd_t *fd_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + wasi_fd_t fd = (wasi_fd_t)-1; /* set fd_app -1 if path open failed */ + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(fd_app, (uint64)sizeof(wasi_fd_t))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_path_open(exec_env, curfds, dirfd, dirflags, path, + path_len, oflags, fs_rights_base, + fs_rights_inheriting, fs_flags, &fd); + + *fd_app = fd; + return err; +} + +static wasi_errno_t +wasi_fd_readdir(wasm_exec_env_t exec_env, wasi_fd_t fd, void *buf, + uint32 buf_len, wasi_dircookie_t cookie, uint32 *bufused_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + size_t bufused; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(bufused_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_fd_readdir(exec_env, curfds, fd, buf, buf_len, cookie, + &bufused); + if (err) + return err; + + *bufused_app = (uint32)bufused; + return 0; +} + +static wasi_errno_t +wasi_path_readlink(wasm_exec_env_t exec_env, wasi_fd_t fd, const char *path, + uint32 path_len, char *buf, uint32 buf_len, + uint32 *bufused_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + size_t bufused; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(bufused_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + + err = wasmtime_ssp_path_readlink(exec_env, curfds, fd, path, path_len, buf, + buf_len, &bufused); + if (err) + return err; + + *bufused_app = (uint32)bufused; + return 0; +} + +static wasi_errno_t +wasi_path_rename(wasm_exec_env_t exec_env, wasi_fd_t old_fd, + const char *old_path, uint32 old_path_len, wasi_fd_t new_fd, + const char *new_path, uint32 new_path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_rename(exec_env, curfds, old_fd, old_path, + old_path_len, new_fd, new_path, + new_path_len); +} + +static wasi_errno_t +wasi_fd_filestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_filestat_t *filestat) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(filestat, (uint64)sizeof(wasi_filestat_t))) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_filestat_get(exec_env, curfds, fd, filestat); +} + +static wasi_errno_t +wasi_fd_filestat_set_times(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_timestamp_t st_atim, wasi_timestamp_t st_mtim, + wasi_fstflags_t fstflags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_filestat_set_times(exec_env, curfds, fd, st_atim, + st_mtim, fstflags); +} + +static wasi_errno_t +wasi_fd_filestat_set_size(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_filesize_t st_size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_fd_filestat_set_size(exec_env, curfds, fd, st_size); +} + +static wasi_errno_t +wasi_path_filestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_lookupflags_t flags, const char *path, + uint32 path_len, wasi_filestat_t *filestat) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr(filestat, (uint64)sizeof(wasi_filestat_t))) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_filestat_get(exec_env, curfds, fd, flags, path, + path_len, filestat); +} + +static wasi_errno_t +wasi_path_filestat_set_times(wasm_exec_env_t exec_env, wasi_fd_t fd, + wasi_lookupflags_t flags, const char *path, + uint32 path_len, wasi_timestamp_t st_atim, + wasi_timestamp_t st_mtim, wasi_fstflags_t fstflags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_filestat_set_times(exec_env, curfds, fd, flags, + path, path_len, st_atim, + st_mtim, fstflags); +} + +static wasi_errno_t +wasi_path_symlink(wasm_exec_env_t exec_env, const char *old_path, + uint32 old_path_len, wasi_fd_t fd, const char *new_path, + uint32 new_path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_symlink(exec_env, curfds, prestats, old_path, + old_path_len, fd, new_path, new_path_len); +} + +static wasi_errno_t +wasi_path_unlink_file(wasm_exec_env_t exec_env, wasi_fd_t fd, const char *path, + uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_unlink_file(exec_env, curfds, fd, path, path_len); +} + +static wasi_errno_t +wasi_path_remove_directory(wasm_exec_env_t exec_env, wasi_fd_t fd, + const char *path, uint32 path_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + return wasmtime_ssp_path_remove_directory(exec_env, curfds, fd, path, + path_len); +} + +#if WASM_ENABLE_THREAD_MGR != 0 +static __wasi_timestamp_t +get_timeout_for_poll_oneoff(const wasi_subscription_t *in, + uint32 nsubscriptions) +{ + __wasi_timestamp_t timeout = (__wasi_timestamp_t)-1; + uint32 i = 0; + + for (i = 0; i < nsubscriptions; ++i) { + const __wasi_subscription_t *s = &in[i]; + if (s->u.type == __WASI_EVENTTYPE_CLOCK + && (s->u.u.clock.flags & __WASI_SUBSCRIPTION_CLOCK_ABSTIME) == 0) { + timeout = min_uint64(timeout, s->u.u.clock.timeout); + } + } + return timeout; +} + +static void +update_clock_subscription_data(wasi_subscription_t *in, uint32 nsubscriptions, + const wasi_timestamp_t new_timeout) +{ + uint32 i = 0; + for (i = 0; i < nsubscriptions; ++i) { + __wasi_subscription_t *s = &in[i]; + if (s->u.type == __WASI_EVENTTYPE_CLOCK) { + s->u.u.clock.timeout = new_timeout; + } + } +} + +static wasi_errno_t +execute_interruptible_poll_oneoff( +#if !defined(WASMTIME_SSP_STATIC_CURFDS) + struct fd_table *curfds, +#endif + const __wasi_subscription_t *in, __wasi_event_t *out, size_t nsubscriptions, + size_t *nevents, wasm_exec_env_t exec_env) +{ + if (nsubscriptions == 0) { + *nevents = 0; + return __WASI_ESUCCESS; + } + + wasi_errno_t err; + __wasi_timestamp_t elapsed = 0; + bool all_outs_are_type_clock; + uint32 i; + + const __wasi_timestamp_t timeout = get_timeout_for_poll_oneoff( + in, nsubscriptions), + time_quant = 1e9; + const uint64 size_to_copy = + nsubscriptions * (uint64)sizeof(wasi_subscription_t); + __wasi_subscription_t *in_copy = NULL; + + if (size_to_copy >= UINT32_MAX + || !(in_copy = (__wasi_subscription_t *)wasm_runtime_malloc( + (uint32)size_to_copy))) { + return __WASI_ENOMEM; + } + + bh_memcpy_s(in_copy, size_to_copy, in, size_to_copy); + + while (timeout == (__wasi_timestamp_t)-1 || elapsed <= timeout) { + /* update timeout for clock subscription events */ + update_clock_subscription_data( + in_copy, nsubscriptions, min_uint64(time_quant, timeout - elapsed)); + err = wasmtime_ssp_poll_oneoff(exec_env, curfds, in_copy, out, + nsubscriptions, nevents); + elapsed += time_quant; + + if (err) { + wasm_runtime_free(in_copy); + return err; + } + + if (wasm_cluster_is_thread_terminated(exec_env)) { + wasm_runtime_free(in_copy); + return __WASI_EINTR; + } + else if (*nevents > 0) { + all_outs_are_type_clock = true; + for (i = 0; i < *nevents; i++) { + if (out[i].type != __WASI_EVENTTYPE_CLOCK) { + all_outs_are_type_clock = false; + break; + } + } + + if (!all_outs_are_type_clock) { + wasm_runtime_free(in_copy); + return __WASI_ESUCCESS; + } + } + } + + wasm_runtime_free(in_copy); + return __WASI_ESUCCESS; +} +#endif + +static wasi_errno_t +wasi_poll_oneoff(wasm_exec_env_t exec_env, const wasi_subscription_t *in, + wasi_event_t *out, uint32 nsubscriptions, uint32 *nevents_app) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + size_t nevents = 0; + wasi_errno_t err; + + if (!wasi_ctx) + return (wasi_errno_t)-1; + + if (!validate_native_addr((void *)in, (uint64)sizeof(wasi_subscription_t)) + || !validate_native_addr(out, (uint64)sizeof(wasi_event_t)) + || !validate_native_addr(nevents_app, (uint64)sizeof(uint32))) + return (wasi_errno_t)-1; + +#if WASM_ENABLE_THREAD_MGR == 0 + err = wasmtime_ssp_poll_oneoff(exec_env, curfds, in, out, nsubscriptions, + &nevents); +#else + err = execute_interruptible_poll_oneoff(curfds, in, out, nsubscriptions, + &nevents, exec_env); +#endif + if (err) + return err; + + *nevents_app = (uint32)nevents; + return 0; +} + +static void +wasi_proc_exit(wasm_exec_env_t exec_env, wasi_exitcode_t rval) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + /* Here throwing exception is just to let wasm app exit, + the upper layer should clear the exception and return + as normal */ + wasm_runtime_set_exception(module_inst, "wasi proc exit"); + wasi_ctx->exit_code = rval; +} + +static wasi_errno_t +wasi_proc_raise(wasm_exec_env_t exec_env, wasi_signal_t sig) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char buf[32]; + snprintf(buf, sizeof(buf), "%s%d", "wasi proc raise ", sig); + wasm_runtime_set_exception(module_inst, buf); + + return 0; +} + +static wasi_errno_t +wasi_random_get(wasm_exec_env_t exec_env, void *buf, uint32 buf_len) +{ + (void)exec_env; + + return wasmtime_ssp_random_get(buf, buf_len); +} + +static wasi_errno_t +wasi_sock_accept(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_fdflags_t flags, + wasi_fd_t *fd_new) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasi_ssp_sock_accept(exec_env, curfds, fd, flags, fd_new); +} + +static wasi_errno_t +wasi_sock_addr_local(wasm_exec_env_t exec_env, wasi_fd_t fd, + __wasi_addr_t *addr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(addr, (uint64)sizeof(__wasi_addr_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasi_ssp_sock_addr_local(exec_env, curfds, fd, addr); +} + +static wasi_errno_t +wasi_sock_addr_remote(wasm_exec_env_t exec_env, wasi_fd_t fd, + __wasi_addr_t *addr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(addr, (uint64)sizeof(__wasi_addr_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasi_ssp_sock_addr_remote(exec_env, curfds, fd, addr); +} + +static wasi_errno_t +wasi_sock_addr_resolve(wasm_exec_env_t exec_env, const char *host, + const char *service, __wasi_addr_info_hints_t *hints, + __wasi_addr_info_t *addr_info, + __wasi_size_t addr_info_size, + __wasi_size_t *max_info_size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + char **ns_lookup_list = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + ns_lookup_list = wasi_ctx_get_ns_lookup_list(wasi_ctx); + + return wasi_ssp_sock_addr_resolve(exec_env, curfds, ns_lookup_list, host, + service, hints, addr_info, addr_info_size, + max_info_size); +} + +static wasi_errno_t +wasi_sock_bind(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_addr_t *addr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + struct addr_pool *addr_pool = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + addr_pool = wasi_ctx_get_addr_pool(wasi_ctx); + + return wasi_ssp_sock_bind(exec_env, curfds, addr_pool, fd, addr); +} + +static wasi_errno_t +wasi_sock_close(wasm_exec_env_t exec_env, wasi_fd_t fd) +{ + (void)exec_env; + (void)fd; + + return __WASI_ENOSYS; +} + +static wasi_errno_t +wasi_sock_connect(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_addr_t *addr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + struct addr_pool *addr_pool = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + addr_pool = wasi_ctx_get_addr_pool(wasi_ctx); + + return wasi_ssp_sock_connect(exec_env, curfds, addr_pool, fd, addr); +} + +static wasi_errno_t +wasi_sock_get_broadcast(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_broadcast(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_get_keep_alive(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_keep_alive(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_get_linger(wasm_exec_env_t exec_env, wasi_fd_t fd, bool *is_enabled, + int *linger_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool)) + || !validate_native_addr(linger_s, (uint64)sizeof(int))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_linger(exec_env, curfds, fd, is_enabled, + linger_s); +} + +static wasi_errno_t +wasi_sock_get_recv_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, + size_t *size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(size, (uint64)sizeof(wasi_size_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_recv_buf_size(exec_env, curfds, fd, size); +} + +static wasi_errno_t +wasi_sock_get_recv_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint64_t *timeout_us) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(timeout_us, (uint64)sizeof(uint64_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_recv_timeout(exec_env, curfds, fd, timeout_us); +} + +static wasi_errno_t +wasi_sock_get_reuse_addr(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_reuse_addr(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_get_reuse_port(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_reuse_port(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_get_send_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, + size_t *size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(size, (uint64)sizeof(__wasi_size_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_send_buf_size(exec_env, curfds, fd, size); +} + +static wasi_errno_t +wasi_sock_get_send_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint64_t *timeout_us) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(timeout_us, (uint64)sizeof(uint64_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_send_timeout(exec_env, curfds, fd, timeout_us); +} + +static wasi_errno_t +wasi_sock_get_tcp_fastopen_connect(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_tcp_fastopen_connect(exec_env, curfds, fd, + is_enabled); +} + +static wasi_errno_t +wasi_sock_get_tcp_no_delay(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_tcp_no_delay(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_get_tcp_quick_ack(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_tcp_quick_ack(exec_env, curfds, fd, + is_enabled); +} + +static wasi_errno_t +wasi_sock_get_tcp_keep_idle(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint32_t *time_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(time_s, (uint64)sizeof(uint32_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_tcp_keep_idle(exec_env, curfds, fd, time_s); +} + +static wasi_errno_t +wasi_sock_get_tcp_keep_intvl(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint32_t *time_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(time_s, (uint64)sizeof(uint32_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_tcp_keep_intvl(exec_env, curfds, fd, time_s); +} + +static wasi_errno_t +wasi_sock_get_ip_multicast_loop(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool ipv6, bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_ip_multicast_loop(exec_env, curfds, fd, ipv6, + is_enabled); +} + +static wasi_errno_t +wasi_sock_get_ip_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, uint8_t *ttl_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(ttl_s, (uint64)sizeof(uint8_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_ip_ttl(exec_env, curfds, fd, ttl_s); +} + +static wasi_errno_t +wasi_sock_get_ip_multicast_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint8_t *ttl_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(ttl_s, (uint64)sizeof(uint8_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_ip_multicast_ttl(exec_env, curfds, fd, ttl_s); +} + +static wasi_errno_t +wasi_sock_get_ipv6_only(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool *is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(is_enabled, (uint64)sizeof(bool))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_get_ipv6_only(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_listen(wasm_exec_env_t exec_env, wasi_fd_t fd, uint32 backlog) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasi_ssp_sock_listen(exec_env, curfds, fd, backlog); +} + +static wasi_errno_t +wasi_sock_open(wasm_exec_env_t exec_env, wasi_fd_t poolfd, + wasi_address_family_t af, wasi_sock_type_t socktype, + wasi_fd_t *sockfd) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasi_ssp_sock_open(exec_env, curfds, poolfd, af, socktype, sockfd); +} + +static wasi_errno_t +wasi_sock_set_broadcast(wasm_exec_env_t exec_env, wasi_fd_t fd, bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_broadcast(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_set_keep_alive(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_keep_alive(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_set_linger(wasm_exec_env_t exec_env, wasi_fd_t fd, bool is_enabled, + int linger_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_linger(exec_env, curfds, fd, is_enabled, + linger_s); +} + +static wasi_errno_t +wasi_sock_set_recv_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, size_t size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_recv_buf_size(exec_env, curfds, fd, size); +} + +static wasi_errno_t +wasi_sock_set_recv_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint64_t timeout_us) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_recv_timeout(exec_env, curfds, fd, timeout_us); +} + +static wasi_errno_t +wasi_sock_set_reuse_addr(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_reuse_addr(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_set_reuse_port(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_reuse_port(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_set_send_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, size_t size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_send_buf_size(exec_env, curfds, fd, size); +} + +static wasi_errno_t +wasi_sock_set_send_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint64_t timeout_us) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_send_timeout(exec_env, curfds, fd, timeout_us); +} + +static wasi_errno_t +wasi_sock_set_tcp_fastopen_connect(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_tcp_fastopen_connect(exec_env, curfds, fd, + is_enabled); +} + +static wasi_errno_t +wasi_sock_set_tcp_no_delay(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_tcp_no_delay(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +wasi_sock_set_tcp_quick_ack(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_tcp_quick_ack(exec_env, curfds, fd, + is_enabled); +} + +static wasi_errno_t +wasi_sock_set_tcp_keep_idle(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint32_t time_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_tcp_keep_idle(exec_env, curfds, fd, time_s); +} + +static wasi_errno_t +wasi_sock_set_tcp_keep_intvl(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint32_t time_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_tcp_keep_intvl(exec_env, curfds, fd, time_s); +} + +static wasi_errno_t +wasi_sock_set_ip_multicast_loop(wasm_exec_env_t exec_env, wasi_fd_t fd, + bool ipv6, bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_ip_multicast_loop(exec_env, curfds, fd, ipv6, + is_enabled); +} + +static wasi_errno_t +wasi_sock_set_ip_add_membership(wasm_exec_env_t exec_env, wasi_fd_t fd, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(imr_multiaddr, (uint64)sizeof(__wasi_addr_ip_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_ip_add_membership( + exec_env, curfds, fd, imr_multiaddr, imr_interface); +} + +static wasi_errno_t +wasi_sock_set_ip_drop_membership(wasm_exec_env_t exec_env, wasi_fd_t fd, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + if (!validate_native_addr(imr_multiaddr, (uint64)sizeof(__wasi_addr_ip_t))) + return __WASI_EINVAL; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_ip_drop_membership( + exec_env, curfds, fd, imr_multiaddr, imr_interface); +} + +static wasi_errno_t +wasi_sock_set_ip_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, uint8_t ttl_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_ip_ttl(exec_env, curfds, fd, ttl_s); +} + +static wasi_errno_t +wasi_sock_set_ip_multicast_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, + uint8_t ttl_s) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_ip_multicast_ttl(exec_env, curfds, fd, ttl_s); +} + +static wasi_errno_t +wasi_sock_set_ipv6_only(wasm_exec_env_t exec_env, wasi_fd_t fd, bool is_enabled) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = NULL; + + if (!wasi_ctx) + return __WASI_EACCES; + + curfds = wasi_ctx_get_curfds(wasi_ctx); + + return wasmtime_ssp_sock_set_ipv6_only(exec_env, curfds, fd, is_enabled); +} + +static wasi_errno_t +allocate_iovec_app_buffer(wasm_module_inst_t module_inst, + const iovec_app_t *data, uint32 data_len, + uint8 **buf_ptr, uint64 *buf_len) +{ + uint64 total_size = 0; + uint32 i; + uint8 *buf_begin = NULL; + + if (data_len == 0) { + return __WASI_EINVAL; + } + + total_size = sizeof(iovec_app_t) * (uint64)data_len; + if (total_size >= UINT32_MAX + || !validate_native_addr((void *)data, total_size)) + return __WASI_EINVAL; + + for (total_size = 0, i = 0; i < data_len; i++, data++) { + total_size += data->buf_len; + } + + if (total_size == 0) { + return __WASI_EINVAL; + } + + if (total_size >= UINT32_MAX + || !(buf_begin = wasm_runtime_malloc((uint32)total_size))) { + return __WASI_ENOMEM; + } + + *buf_len = total_size; + *buf_ptr = buf_begin; + + return __WASI_ESUCCESS; +} + +static wasi_errno_t +copy_buffer_to_iovec_app(wasm_module_inst_t module_inst, uint8 *buf_begin, + uint32 buf_size, iovec_app_t *data, uint32 data_len, + uint32 size_to_copy) +{ + uint8 *buf = buf_begin; + uint32 i; + uint32 size_to_copy_into_iovec; + + if (buf_size < size_to_copy) { + return __WASI_EINVAL; + } + + for (i = 0; i < data_len; data++, i++) { + char *native_addr; + + if (!validate_app_addr((uint64)data->buf_offset, + (uint64)data->buf_len)) { + return __WASI_EINVAL; + } + + if (buf >= buf_begin + buf_size + /* integer overflow */ + || data->buf_len > UINTPTR_MAX - (uintptr_t)buf + || buf + data->buf_len > buf_begin + buf_size + || size_to_copy == 0) { + break; + } + + /** + * If our app buffer size is smaller than the amount to be copied, + * only copy the amount in the app buffer. Otherwise, we fill the iovec + * buffer and reduce size to copy on the next iteration + */ + size_to_copy_into_iovec = min_uint32(data->buf_len, size_to_copy); + + native_addr = (void *)addr_app_to_native((uint64)data->buf_offset); + bh_memcpy_s(native_addr, size_to_copy_into_iovec, buf, + size_to_copy_into_iovec); + buf += size_to_copy_into_iovec; + size_to_copy -= size_to_copy_into_iovec; + } + + return __WASI_ESUCCESS; +} + +static wasi_errno_t +wasi_sock_recv_from(wasm_exec_env_t exec_env, wasi_fd_t sock, + iovec_app_t *ri_data, uint32 ri_data_len, + wasi_riflags_t ri_flags, __wasi_addr_t *src_addr, + uint32 *ro_data_len) +{ + /** + * ri_data_len is the length of a list of iovec_app_t, which head is + * ri_data. ro_data_len is the number of bytes received + **/ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + uint64 total_size; + uint8 *buf_begin = NULL; + wasi_errno_t err; + size_t recv_bytes = 0; + + if (!wasi_ctx) { + return __WASI_EINVAL; + } + + if (!validate_native_addr(ro_data_len, (uint64)sizeof(uint32))) + return __WASI_EINVAL; + + err = allocate_iovec_app_buffer(module_inst, ri_data, ri_data_len, + &buf_begin, &total_size); + if (err != __WASI_ESUCCESS) { + goto fail; + } + + memset(buf_begin, 0, total_size); + + *ro_data_len = 0; + err = wasmtime_ssp_sock_recv_from(exec_env, curfds, sock, buf_begin, + total_size, ri_flags, src_addr, + &recv_bytes); + if (err != __WASI_ESUCCESS) { + goto fail; + } + *ro_data_len = (uint32)recv_bytes; + + err = copy_buffer_to_iovec_app(module_inst, buf_begin, (uint32)total_size, + ri_data, ri_data_len, (uint32)recv_bytes); + +fail: + if (buf_begin) { + wasm_runtime_free(buf_begin); + } + return err; +} + +static wasi_errno_t +wasi_sock_recv(wasm_exec_env_t exec_env, wasi_fd_t sock, iovec_app_t *ri_data, + uint32 ri_data_len, wasi_riflags_t ri_flags, uint32 *ro_data_len, + wasi_roflags_t *ro_flags) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + __wasi_addr_t src_addr; + wasi_errno_t error; + + if (!validate_native_addr(ro_flags, (uint64)sizeof(wasi_roflags_t))) + return __WASI_EINVAL; + + error = wasi_sock_recv_from(exec_env, sock, ri_data, ri_data_len, ri_flags, + &src_addr, ro_data_len); + *ro_flags = ri_flags; + + return error; +} + +static wasi_errno_t +convert_iovec_app_to_buffer(wasm_module_inst_t module_inst, + const iovec_app_t *si_data, uint32 si_data_len, + uint8 **buf_ptr, uint64 *buf_len) +{ + uint32 i; + const iovec_app_t *si_data_orig = si_data; + uint8 *buf = NULL; + wasi_errno_t error; + + error = allocate_iovec_app_buffer(module_inst, si_data, si_data_len, + buf_ptr, buf_len); + if (error != __WASI_ESUCCESS) { + return error; + } + + buf = *buf_ptr; + si_data = si_data_orig; + for (i = 0; i < si_data_len; i++, si_data++) { + char *native_addr; + + if (!validate_app_addr((uint64)si_data->buf_offset, + (uint64)si_data->buf_len)) { + wasm_runtime_free(*buf_ptr); + return __WASI_EINVAL; + } + + native_addr = (char *)addr_app_to_native((uint64)si_data->buf_offset); + bh_memcpy_s(buf, si_data->buf_len, native_addr, si_data->buf_len); + buf += si_data->buf_len; + } + + return __WASI_ESUCCESS; +} + +static wasi_errno_t +wasi_sock_send(wasm_exec_env_t exec_env, wasi_fd_t sock, + const iovec_app_t *si_data, uint32 si_data_len, + wasi_siflags_t si_flags, uint32 *so_data_len) +{ + /** + * si_data_len is the length of a list of iovec_app_t, which head is + * si_data. so_data_len is the number of bytes sent + **/ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + uint64 buf_size = 0; + uint8 *buf = NULL; + wasi_errno_t err; + size_t send_bytes = 0; + + if (!wasi_ctx) { + return __WASI_EINVAL; + } + + if (!validate_native_addr(so_data_len, (uint64)sizeof(uint32))) + return __WASI_EINVAL; + + err = convert_iovec_app_to_buffer(module_inst, si_data, si_data_len, &buf, + &buf_size); + if (err != __WASI_ESUCCESS) + return err; + + *so_data_len = 0; + err = wasmtime_ssp_sock_send(exec_env, curfds, sock, buf, buf_size, + &send_bytes); + *so_data_len = (uint32)send_bytes; + + wasm_runtime_free(buf); + + return err; +} + +static wasi_errno_t +wasi_sock_send_to(wasm_exec_env_t exec_env, wasi_fd_t sock, + const iovec_app_t *si_data, uint32 si_data_len, + wasi_siflags_t si_flags, const __wasi_addr_t *dest_addr, + uint32 *so_data_len) +{ + /** + * si_data_len is the length of a list of iovec_app_t, which head is + * si_data. so_data_len is the number of bytes sent + **/ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + uint64 buf_size = 0; + uint8 *buf = NULL; + wasi_errno_t err; + size_t send_bytes = 0; + struct addr_pool *addr_pool = wasi_ctx_get_addr_pool(wasi_ctx); + + if (!wasi_ctx) { + return __WASI_EINVAL; + } + + if (!validate_native_addr(so_data_len, (uint64)sizeof(uint32))) + return __WASI_EINVAL; + + err = convert_iovec_app_to_buffer(module_inst, si_data, si_data_len, &buf, + &buf_size); + if (err != __WASI_ESUCCESS) + return err; + + *so_data_len = 0; + err = wasmtime_ssp_sock_send_to(exec_env, curfds, addr_pool, sock, buf, + buf_size, si_flags, dest_addr, &send_bytes); + *so_data_len = (uint32)send_bytes; + + wasm_runtime_free(buf); + + return err; +} + +static wasi_errno_t +wasi_sock_shutdown(wasm_exec_env_t exec_env, wasi_fd_t sock, wasi_sdflags_t how) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + + if (!wasi_ctx) + return __WASI_EINVAL; + + return wasmtime_ssp_sock_shutdown(exec_env, curfds, sock); +} + +static wasi_errno_t +wasi_sched_yield(wasm_exec_env_t exec_env) +{ + (void)exec_env; + + return wasmtime_ssp_sched_yield(); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, wasi_##func_name, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_libc_wasi[] = { + REG_NATIVE_FUNC(args_get, "(**)i"), + REG_NATIVE_FUNC(args_sizes_get, "(**)i"), + REG_NATIVE_FUNC(clock_res_get, "(i*)i"), + REG_NATIVE_FUNC(clock_time_get, "(iI*)i"), + REG_NATIVE_FUNC(environ_get, "(**)i"), + REG_NATIVE_FUNC(environ_sizes_get, "(**)i"), + REG_NATIVE_FUNC(fd_prestat_get, "(i*)i"), + REG_NATIVE_FUNC(fd_prestat_dir_name, "(i*~)i"), + REG_NATIVE_FUNC(fd_close, "(i)i"), + REG_NATIVE_FUNC(fd_datasync, "(i)i"), + REG_NATIVE_FUNC(fd_pread, "(i*iI*)i"), + REG_NATIVE_FUNC(fd_pwrite, "(i*iI*)i"), + REG_NATIVE_FUNC(fd_read, "(i*i*)i"), + REG_NATIVE_FUNC(fd_renumber, "(ii)i"), + REG_NATIVE_FUNC(fd_seek, "(iIi*)i"), + REG_NATIVE_FUNC(fd_tell, "(i*)i"), + REG_NATIVE_FUNC(fd_fdstat_get, "(i*)i"), + REG_NATIVE_FUNC(fd_fdstat_set_flags, "(ii)i"), + REG_NATIVE_FUNC(fd_fdstat_set_rights, "(iII)i"), + REG_NATIVE_FUNC(fd_sync, "(i)i"), + REG_NATIVE_FUNC(fd_write, "(i*i*)i"), + REG_NATIVE_FUNC(fd_advise, "(iIIi)i"), + REG_NATIVE_FUNC(fd_allocate, "(iII)i"), + REG_NATIVE_FUNC(path_create_directory, "(i*~)i"), + REG_NATIVE_FUNC(path_link, "(ii*~i*~)i"), + REG_NATIVE_FUNC(path_open, "(ii*~iIIi*)i"), + REG_NATIVE_FUNC(fd_readdir, "(i*~I*)i"), + REG_NATIVE_FUNC(path_readlink, "(i*~*~*)i"), + REG_NATIVE_FUNC(path_rename, "(i*~i*~)i"), + REG_NATIVE_FUNC(fd_filestat_get, "(i*)i"), + REG_NATIVE_FUNC(fd_filestat_set_times, "(iIIi)i"), + REG_NATIVE_FUNC(fd_filestat_set_size, "(iI)i"), + REG_NATIVE_FUNC(path_filestat_get, "(ii*~*)i"), + REG_NATIVE_FUNC(path_filestat_set_times, "(ii*~IIi)i"), + REG_NATIVE_FUNC(path_symlink, "(*~i*~)i"), + REG_NATIVE_FUNC(path_unlink_file, "(i*~)i"), + REG_NATIVE_FUNC(path_remove_directory, "(i*~)i"), + REG_NATIVE_FUNC(poll_oneoff, "(**i*)i"), + REG_NATIVE_FUNC(proc_exit, "(i)"), + REG_NATIVE_FUNC(proc_raise, "(i)i"), + REG_NATIVE_FUNC(random_get, "(*~)i"), + REG_NATIVE_FUNC(sock_accept, "(ii*)i"), + REG_NATIVE_FUNC(sock_addr_local, "(i*)i"), + REG_NATIVE_FUNC(sock_addr_remote, "(i*)i"), + REG_NATIVE_FUNC(sock_addr_resolve, "($$**i*)i"), + REG_NATIVE_FUNC(sock_bind, "(i*)i"), + REG_NATIVE_FUNC(sock_close, "(i)i"), + REG_NATIVE_FUNC(sock_connect, "(i*)i"), + REG_NATIVE_FUNC(sock_get_broadcast, "(i*)i"), + REG_NATIVE_FUNC(sock_get_keep_alive, "(i*)i"), + REG_NATIVE_FUNC(sock_get_linger, "(i**)i"), + REG_NATIVE_FUNC(sock_get_recv_buf_size, "(i*)i"), + REG_NATIVE_FUNC(sock_get_recv_timeout, "(i*)i"), + REG_NATIVE_FUNC(sock_get_reuse_addr, "(i*)i"), + REG_NATIVE_FUNC(sock_get_reuse_port, "(i*)i"), + REG_NATIVE_FUNC(sock_get_send_buf_size, "(i*)i"), + REG_NATIVE_FUNC(sock_get_send_timeout, "(i*)i"), + REG_NATIVE_FUNC(sock_get_tcp_fastopen_connect, "(i*)i"), + REG_NATIVE_FUNC(sock_get_tcp_keep_idle, "(i*)i"), + REG_NATIVE_FUNC(sock_get_tcp_keep_intvl, "(i*)i"), + REG_NATIVE_FUNC(sock_get_tcp_no_delay, "(i*)i"), + REG_NATIVE_FUNC(sock_get_tcp_quick_ack, "(i*)i"), + REG_NATIVE_FUNC(sock_get_ip_multicast_loop, "(ii*)i"), + REG_NATIVE_FUNC(sock_get_ip_multicast_ttl, "(i*)i"), + REG_NATIVE_FUNC(sock_get_ip_ttl, "(i*)i"), + REG_NATIVE_FUNC(sock_get_ipv6_only, "(i*)i"), + REG_NATIVE_FUNC(sock_listen, "(ii)i"), + REG_NATIVE_FUNC(sock_open, "(iii*)i"), + REG_NATIVE_FUNC(sock_recv, "(i*ii**)i"), + REG_NATIVE_FUNC(sock_recv_from, "(i*ii**)i"), + REG_NATIVE_FUNC(sock_send, "(i*ii*)i"), + REG_NATIVE_FUNC(sock_send_to, "(i*ii**)i"), + REG_NATIVE_FUNC(sock_set_broadcast, "(ii)i"), + REG_NATIVE_FUNC(sock_set_keep_alive, "(ii)i"), + REG_NATIVE_FUNC(sock_set_linger, "(iii)i"), + REG_NATIVE_FUNC(sock_set_recv_buf_size, "(ii)i"), + REG_NATIVE_FUNC(sock_set_recv_timeout, "(iI)i"), + REG_NATIVE_FUNC(sock_set_reuse_addr, "(ii)i"), + REG_NATIVE_FUNC(sock_set_reuse_port, "(ii)i"), + REG_NATIVE_FUNC(sock_set_send_buf_size, "(ii)i"), + REG_NATIVE_FUNC(sock_set_send_timeout, "(iI)i"), + REG_NATIVE_FUNC(sock_set_tcp_fastopen_connect, "(ii)i"), + REG_NATIVE_FUNC(sock_set_tcp_keep_idle, "(ii)i"), + REG_NATIVE_FUNC(sock_set_tcp_keep_intvl, "(ii)i"), + REG_NATIVE_FUNC(sock_set_tcp_no_delay, "(ii)i"), + REG_NATIVE_FUNC(sock_set_tcp_quick_ack, "(ii)i"), + REG_NATIVE_FUNC(sock_set_ip_multicast_loop, "(iii)i"), + REG_NATIVE_FUNC(sock_set_ip_multicast_ttl, "(ii)i"), + REG_NATIVE_FUNC(sock_set_ip_add_membership, "(i*i)i"), + REG_NATIVE_FUNC(sock_set_ip_drop_membership, "(i*i)i"), + REG_NATIVE_FUNC(sock_set_ip_ttl, "(ii)i"), + REG_NATIVE_FUNC(sock_set_ipv6_only, "(ii)i"), + REG_NATIVE_FUNC(sock_shutdown, "(ii)i"), + REG_NATIVE_FUNC(sched_yield, "()i"), +}; + +uint32 +get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis) +{ + *p_libc_wasi_apis = native_symbols_libc_wasi; + return sizeof(native_symbols_libc_wasi) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.h new file mode 100644 index 0000000..d958fa3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _LIBC_WASI_WRAPPER_H +#define _LIBC_WASI_WRAPPER_H + +#include "posix.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef __wasi_address_family_t wasi_address_family_t; +typedef __wasi_addr_t wasi_addr_t; +typedef __wasi_advice_t wasi_advice_t; +typedef __wasi_ciovec_t wasi_ciovec_t; +typedef __wasi_clockid_t wasi_clockid_t; +typedef __wasi_dircookie_t wasi_dircookie_t; +// __wasi_errno_t is typedef'd to uint16 which is correct according to the ABI +// specification. However, in WASM, the smallest integer type is int32. If we +// return uint16, we would rely on language SDKs to implement the correct +// behaviour of casting to uint16 before checking the value or using it any way. +// Failure to do so can cause tricky bugs as the upper 16 bits of the error +// result are not guaranteed to be zero'ed by us so the result essentially +// contains garbage from the WASM app perspective. To prevent this, we return +// uint32 directly instead so as not to be reliant on the correct behaviour of +// any current/future WASI SDK implemenations. +typedef uint32_t wasi_errno_t; +typedef __wasi_event_t wasi_event_t; +typedef __wasi_exitcode_t wasi_exitcode_t; +typedef __wasi_fdflags_t wasi_fdflags_t; +typedef __wasi_fdstat_t wasi_fdstat_t; +typedef __wasi_fd_t wasi_fd_t; +typedef __wasi_filedelta_t wasi_filedelta_t; +typedef __wasi_filesize_t wasi_filesize_t; +typedef __wasi_filestat_t wasi_filestat_t; +typedef __wasi_filetype_t wasi_filetype_t; +typedef __wasi_fstflags_t wasi_fstflags_t; +typedef __wasi_iovec_t wasi_iovec_t; +typedef __wasi_ip_port_t wasi_ip_port_t; +typedef __wasi_lookupflags_t wasi_lookupflags_t; +typedef __wasi_oflags_t wasi_oflags_t; +typedef __wasi_preopentype_t wasi_preopentype_t; +typedef __wasi_prestat_t wasi_prestat_t; +typedef __wasi_riflags_t wasi_riflags_t; +typedef __wasi_rights_t wasi_rights_t; +typedef __wasi_roflags_t wasi_roflags_t; +typedef __wasi_sdflags_t wasi_sdflags_t; +typedef __wasi_siflags_t wasi_siflags_t; +typedef __wasi_signal_t wasi_signal_t; +typedef __wasi_size_t wasi_size_t; +typedef __wasi_sock_type_t wasi_sock_type_t; +typedef __wasi_subscription_t wasi_subscription_t; +typedef __wasi_timestamp_t wasi_timestamp_t; +typedef __wasi_whence_t wasi_whence_t; + +#ifdef __cplusplus +} +#endif + +#endif /* end of _LIBC_WASI_WRAPPER_H */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/LICENSE b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/LICENSE new file mode 100644 index 0000000..83386f8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/LICENSE @@ -0,0 +1,7 @@ +Please see the LICENSE file in each top-level directory for the terms applicable to that directory and its relative sub-directories. + +The relevant directories and licenses are: + +src/ - BSD-2-Clause; see src/LICENSE for details +include/ - CC0 1.0 Universal (CC0 1.0) Public Domain Dedication +polyfill/clang/ - MIT; see the header of polyfill/clang/stdatomic.h for details diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/include/LICENSE b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/include/LICENSE new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/include/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/include/wasmtime_ssp.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/include/wasmtime_ssp.h new file mode 100644 index 0000000..ed84638 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/include/wasmtime_ssp.h @@ -0,0 +1,606 @@ +/* + * Part of the Wasmtime Project, under the Apache License v2.0 with + * LLVM Exceptions. See + * https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE + * for license information. + */ + +/** + * The defitions of type, macro and structure in this file should be + * consistent with those in wasi-libc: + * https://github.com/WebAssembly/wasi-libc/blob/main/libc-bottom-half/headers/public/wasi/api.h + */ + +#ifndef WASMTIME_SSP_H +#define WASMTIME_SSP_H + +#include +#include +#include + +#include "bh_platform.h" +#include "wasm_export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(WASMTIME_SSP_WASI_API) +#define WASMTIME_SSP_SYSCALL_NAME(name) asm("__wasi_" #name) +#else +#define WASMTIME_SSP_SYSCALL_NAME(name) +#endif + +__wasi_errno_t +wasmtime_ssp_args_get(struct argv_environ_values *arg_environ, char **argv, + char *argv_buf) + WASMTIME_SSP_SYSCALL_NAME(args_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_args_sizes_get(struct argv_environ_values *arg_environ, + size_t *argc, size_t *argv_buf_size) + WASMTIME_SSP_SYSCALL_NAME(args_sizes_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_environ_get(struct argv_environ_values *arg_environ, + char **environs, char *environ_buf) + WASMTIME_SSP_SYSCALL_NAME(environ_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_environ_sizes_get(struct argv_environ_values *arg_environ, + size_t *environ_count, size_t *environ_buf_size) + WASMTIME_SSP_SYSCALL_NAME(environ_sizes_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_prestat_get(struct fd_prestats *prestats, __wasi_fd_t fd, + __wasi_prestat_t *buf) + WASMTIME_SSP_SYSCALL_NAME(fd_prestat_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_prestat_dir_name(struct fd_prestats *prestats, __wasi_fd_t fd, + char *path, size_t path_len) + WASMTIME_SSP_SYSCALL_NAME(fd_prestat_dir_name) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_close(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, __wasi_fd_t fd) + WASMTIME_SSP_SYSCALL_NAME(fd_close) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_datasync(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd) + WASMTIME_SSP_SYSCALL_NAME(fd_datasync) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_pread(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_filesize_t offset, size_t *nread) + WASMTIME_SSP_SYSCALL_NAME(fd_pread) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_pwrite(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_ciovec_t *iovs, + size_t iovs_len, __wasi_filesize_t offset, + size_t *nwritten) + WASMTIME_SSP_SYSCALL_NAME(fd_pwrite) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_read(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, size_t *nread) + WASMTIME_SSP_SYSCALL_NAME(fd_read) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_renumber(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, __wasi_fd_t from, + __wasi_fd_t to) + WASMTIME_SSP_SYSCALL_NAME(fd_renumber) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_seek(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filedelta_t offset, + __wasi_whence_t whence, __wasi_filesize_t *newoffset) + WASMTIME_SSP_SYSCALL_NAME(fd_seek) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_tell(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filesize_t *newoffset) + WASMTIME_SSP_SYSCALL_NAME(fd_tell) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_fdstat_get(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_fdstat_t *buf) + WASMTIME_SSP_SYSCALL_NAME(fd_fdstat_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_fdstat_set_flags(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_fdflags_t flags) + WASMTIME_SSP_SYSCALL_NAME(fd_fdstat_set_flags) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_fdstat_set_rights(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_rights_t fs_rights_base, + __wasi_rights_t fs_rights_inheriting) + WASMTIME_SSP_SYSCALL_NAME(fd_fdstat_set_rights) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_sync(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd) + WASMTIME_SSP_SYSCALL_NAME(fd_sync) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_write(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_ciovec_t *iovs, + size_t iovs_len, size_t *nwritten) + WASMTIME_SSP_SYSCALL_NAME(fd_write) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_advise(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filesize_t offset, + __wasi_filesize_t len, __wasi_advice_t advice) + WASMTIME_SSP_SYSCALL_NAME(fd_advise) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_allocate(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filesize_t offset, + __wasi_filesize_t len) + WASMTIME_SSP_SYSCALL_NAME(fd_allocate) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_create_directory(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + const char *path, size_t path_len) + WASMTIME_SSP_SYSCALL_NAME(path_create_directory) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_link(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, __wasi_fd_t old_fd, + __wasi_lookupflags_t old_flags, const char *old_path, + size_t old_path_len, __wasi_fd_t new_fd, + const char *new_path, size_t new_path_len) + WASMTIME_SSP_SYSCALL_NAME(path_link) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_open(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t dirfd, __wasi_lookupflags_t dirflags, + const char *path, size_t path_len, + __wasi_oflags_t oflags, __wasi_rights_t fs_rights_base, + __wasi_rights_t fs_rights_inheriting, + __wasi_fdflags_t fs_flags, __wasi_fd_t *fd) + WASMTIME_SSP_SYSCALL_NAME(path_open) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_readdir(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, void *buf, size_t buf_len, + __wasi_dircookie_t cookie, size_t *bufused) + WASMTIME_SSP_SYSCALL_NAME(fd_readdir) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_readlink(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const char *path, size_t path_len, + char *buf, size_t buf_len, size_t *bufused) + WASMTIME_SSP_SYSCALL_NAME(path_readlink) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_rename(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t old_fd, const char *old_path, + size_t old_path_len, __wasi_fd_t new_fd, + const char *new_path, size_t new_path_len) + WASMTIME_SSP_SYSCALL_NAME(path_rename) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_filestat_get(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filestat_t *buf) + WASMTIME_SSP_SYSCALL_NAME(fd_filestat_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_filestat_set_times(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_timestamp_t st_atim, + __wasi_timestamp_t st_mtim, + __wasi_fstflags_t fstflags) + WASMTIME_SSP_SYSCALL_NAME(fd_filestat_set_times) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_fd_filestat_set_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_filesize_t st_size) + WASMTIME_SSP_SYSCALL_NAME(fd_filestat_set_size) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_filestat_get(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_lookupflags_t flags, const char *path, + size_t path_len, __wasi_filestat_t *buf) + WASMTIME_SSP_SYSCALL_NAME(path_filestat_get) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_filestat_set_times(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_lookupflags_t flags, + const char *path, size_t path_len, + __wasi_timestamp_t st_atim, + __wasi_timestamp_t st_mtim, + __wasi_fstflags_t fstflags) + WASMTIME_SSP_SYSCALL_NAME(path_filestat_set_times) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_symlink(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, const char *old_path, + size_t old_path_len, __wasi_fd_t fd, + const char *new_path, size_t new_path_len) + WASMTIME_SSP_SYSCALL_NAME(path_symlink) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_unlink_file(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const char *path, size_t path_len) + WASMTIME_SSP_SYSCALL_NAME(path_unlink_file) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_path_remove_directory(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + const char *path, size_t path_len) + WASMTIME_SSP_SYSCALL_NAME(path_remove_directory) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_poll_oneoff(wasm_exec_env_t exec_env, struct fd_table *curfds, + const __wasi_subscription_t *in, __wasi_event_t *out, + size_t nsubscriptions, size_t *nevents) + WASMTIME_SSP_SYSCALL_NAME(poll_oneoff) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_random_get(void *buf, size_t buf_len) + WASMTIME_SSP_SYSCALL_NAME(random_get) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_accept(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_fdflags_t flags, + __wasi_fd_t *fd_new) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_addr_local(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_addr_t *addr) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_addr_remote(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_addr_t *addr) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_open(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t poolfd, __wasi_address_family_t af, + __wasi_sock_type_t socktype, + __wasi_fd_t *sockfd) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_bind(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct addr_pool *addr_pool, __wasi_fd_t fd, + __wasi_addr_t *addr) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_addr_resolve(wasm_exec_env_t exec_env, struct fd_table *curfds, + char **ns_lookup_list, const char *host, + const char *service, __wasi_addr_info_hints_t *hints, + __wasi_addr_info_t *addr_info, + __wasi_size_t addr_info_size, + __wasi_size_t *max_info_size) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_connect(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct addr_pool *addr_pool, __wasi_fd_t fd, + __wasi_addr_t *addr) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_get_recv_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t *size) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_get_reuse_addr(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t *reuse) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_get_reuse_port(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t *reuse) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_get_send_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t *size) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_set_recv_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t size) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_set_reuse_addr(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t reuse) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_set_reuse_port(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t reuse) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_set_send_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t size) WARN_UNUSED; + +__wasi_errno_t +wasi_ssp_sock_listen(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_size_t backlog) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_recv(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, void *buf, size_t buf_len, + size_t *recv_len) + WASMTIME_SSP_SYSCALL_NAME(sock_recv) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_recv_from(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, void *buf, size_t buf_len, + __wasi_riflags_t ri_flags, __wasi_addr_t *src_addr, + size_t *recv_len) + WASMTIME_SSP_SYSCALL_NAME(sock_recv_from) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_send(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, const void *buf, size_t buf_len, + size_t *sent_len) + WASMTIME_SSP_SYSCALL_NAME(sock_send) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_send_to(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct addr_pool *addr_pool, __wasi_fd_t sock, + const void *buf, size_t buf_len, + __wasi_siflags_t si_flags, + const __wasi_addr_t *dest_addr, size_t *sent_len) + WASMTIME_SSP_SYSCALL_NAME(sock_send_to) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_shutdown(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock) + WASMTIME_SSP_SYSCALL_NAME(sock_shutdown) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_recv_timeout(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint64_t timeout_us) + WASMTIME_SSP_SYSCALL_NAME(sock_set_recv_timeout) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_recv_timeout(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint64_t *timeout_us) + WASMTIME_SSP_SYSCALL_NAME(sock_get_recv_timeout) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_send_timeout(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint64_t timeout_us) + WASMTIME_SSP_SYSCALL_NAME(sock_set_send_timeout) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_send_timeout(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint64_t *timeout_us) + WASMTIME_SSP_SYSCALL_NAME(sock_get_send_timeout) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_send_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + size_t bufsiz) + WASMTIME_SSP_SYSCALL_NAME(sock_set_send_buf_size) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_send_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + size_t *bufsiz) + WASMTIME_SSP_SYSCALL_NAME(sock_get_send_buf_size) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_recv_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + size_t bufsiz) + WASMTIME_SSP_SYSCALL_NAME(sock_set_recv_buf_size) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_recv_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + size_t *bufsiz) + WASMTIME_SSP_SYSCALL_NAME(sock_get_recv_buf_size) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_keep_alive(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_keep_alive) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_keep_alive(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_keep_alive) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_reuse_addr(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_reuse_addr) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_reuse_addr(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_reuse_addr) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_reuse_port(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_reuse_port) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_reuse_port(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_reuse_port) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_linger(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, bool is_enabled, int linger_s) + WASMTIME_SSP_SYSCALL_NAME(sock_set_linger) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_linger(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, bool *is_enabled, int *linger_s) + WASMTIME_SSP_SYSCALL_NAME(sock_get_linger) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_broadcast(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_broadcast) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_broadcast(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_broadcast) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_tcp_no_delay(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_tcp_no_delay) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_tcp_no_delay(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_tcp_no_delay) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_tcp_quick_ack(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_tcp_quick_ack) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_tcp_quick_ack(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_tcp_quick_ack) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_tcp_keep_idle(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint32_t time_s) + WASMTIME_SSP_SYSCALL_NAME(sock_set_tcp_keep_idle) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_tcp_keep_idle(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint32_t *time_s) + WASMTIME_SSP_SYSCALL_NAME(sock_get_tcp_keep_idle) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_tcp_keep_intvl(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint32_t time_s) + WASMTIME_SSP_SYSCALL_NAME(sock_set_tcp_keep_intvl) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_tcp_keep_intvl(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + uint32_t *time_s) + WASMTIME_SSP_SYSCALL_NAME(sock_get_tcp_keep_intvl) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_tcp_fastopen_connect(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_tcp_fastopen_connect) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_tcp_fastopen_connect(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_tcp_fastopen_connect) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_multicast_loop(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, bool ipv6, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_ip_multicast_loop) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_ip_multicast_loop(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, bool ipv6, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_ip_multicast_loop) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_add_membership(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) + WASMTIME_SSP_SYSCALL_NAME(sock_set_ip_add_membership) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_drop_membership(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) + WASMTIME_SSP_SYSCALL_NAME(sock_set_ip_drop_membership) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_ttl(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, uint8_t ttl_s) + WASMTIME_SSP_SYSCALL_NAME(sock_set_ip_ttl) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_ip_ttl(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, uint8_t *ttl_s) + WASMTIME_SSP_SYSCALL_NAME(sock_get_ip_ttl) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_multicast_ttl(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, uint8_t ttl_s) + WASMTIME_SSP_SYSCALL_NAME(sock_set_ip_multicast_ttl) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_ip_multicast_ttl(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, uint8_t *ttl_s) + WASMTIME_SSP_SYSCALL_NAME(sock_get_ip_multicast_ttl) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_set_ipv6_only(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_set_ipv6_only) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sock_get_ipv6_only(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t sock, + bool *is_enabled) + WASMTIME_SSP_SYSCALL_NAME(sock_get_ipv6_only) WARN_UNUSED; + +__wasi_errno_t +wasmtime_ssp_sched_yield(void) + WASMTIME_SSP_SYSCALL_NAME(sched_yield) WARN_UNUSED; + +#ifdef __cplusplus +} +#endif + +#undef WASMTIME_SSP_SYSCALL_NAME + +#endif /* end of WASMTIME_SSP_H */ diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/LICENSE b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/LICENSE new file mode 100644 index 0000000..04c6f48 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/LICENSE @@ -0,0 +1,24 @@ +All code is distributed under the following license: + + Copyright (c) 2015 Nuxi, https://nuxi.nl/ + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/README.md b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/README.md new file mode 100644 index 0000000..b4d55d8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/README.md @@ -0,0 +1,14 @@ +This directory consists of selected files copied from the [src/libemulator] +directory in the [cloudabi-utils] repository, with minor modifications, +along with the accompanying LICENSE file from that repository. + +The modifications are marked with `WASMTIME_*` preprocessor macros. + +The files were copied at git revision +223dadc53248552db43e012c67ed08cf416a2b12 +which is dated +Tue Jun 25 17:22:07 2019 -0700 +. + +[libemulator]: https://github.com/NuxiNL/cloudabi-utils/tree/223dadc53248552db43e012c67ed08cf416a2b12/src/libemulator +[cloudabi-utils]: https://github.com/NuxiNL/cloudabi-utils diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.c b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.c new file mode 100644 index 0000000..4dcc4f5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.c @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include "ssp_config.h" +#include "blocking_op.h" +#include "libc_errno.h" + +__wasi_errno_t +blocking_op_close(wasm_exec_env_t exec_env, os_file_handle handle, + bool is_stdio) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + return __WASI_EINTR; + } + __wasi_errno_t error = os_close(handle, is_stdio); + wasm_runtime_end_blocking_op(exec_env); + return error; +} + +__wasi_errno_t +blocking_op_readv(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_iovec_t *iov, int iovcnt, size_t *nread) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + return __WASI_EINTR; + } + __wasi_errno_t error = os_readv(handle, iov, iovcnt, nread); + wasm_runtime_end_blocking_op(exec_env); + return error; +} + +__wasi_errno_t +blocking_op_preadv(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_iovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nread) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + return __WASI_EINTR; + } + __wasi_errno_t ret = os_preadv(handle, iov, iovcnt, offset, nread); + wasm_runtime_end_blocking_op(exec_env); + return ret; +} + +__wasi_errno_t +blocking_op_writev(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_ciovec_t *iov, int iovcnt, + size_t *nwritten) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + return __WASI_EINTR; + } + __wasi_errno_t error = os_writev(handle, iov, iovcnt, nwritten); + wasm_runtime_end_blocking_op(exec_env); + return error; +} + +__wasi_errno_t +blocking_op_pwritev(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_ciovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nwritten) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + return __WASI_EINTR; + } + __wasi_errno_t error = os_pwritev(handle, iov, iovcnt, offset, nwritten); + wasm_runtime_end_blocking_op(exec_env); + return error; +} + +int +blocking_op_socket_accept(wasm_exec_env_t exec_env, bh_socket_t server_sock, + bh_socket_t *sockp, void *addr, + unsigned int *addrlenp) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + errno = EINTR; + return -1; + } + int ret = os_socket_accept(server_sock, sockp, addr, addrlenp); + wasm_runtime_end_blocking_op(exec_env); + return ret; +} + +int +blocking_op_socket_connect(wasm_exec_env_t exec_env, bh_socket_t sock, + const char *addr, int port) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + errno = EINTR; + return -1; + } + int ret = os_socket_connect(sock, addr, port); + wasm_runtime_end_blocking_op(exec_env); + return ret; +} + +int +blocking_op_socket_recv_from(wasm_exec_env_t exec_env, bh_socket_t sock, + void *buf, unsigned int len, int flags, + bh_sockaddr_t *src_addr) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + errno = EINTR; + return -1; + } + int ret = os_socket_recv_from(sock, buf, len, flags, src_addr); + wasm_runtime_end_blocking_op(exec_env); + return ret; +} + +int +blocking_op_socket_send_to(wasm_exec_env_t exec_env, bh_socket_t sock, + const void *buf, unsigned int len, int flags, + const bh_sockaddr_t *dest_addr) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + errno = EINTR; + return -1; + } + int ret = os_socket_send_to(sock, buf, len, flags, dest_addr); + wasm_runtime_end_blocking_op(exec_env); + return ret; +} + +int +blocking_op_socket_addr_resolve(wasm_exec_env_t exec_env, const char *host, + const char *service, uint8_t *hint_is_tcp, + uint8_t *hint_is_ipv4, + bh_addr_info_t *addr_info, + size_t addr_info_size, size_t *max_info_size) +{ + /* + * Note: Unlike others, os_socket_addr_resolve() is not a simple system + * call. It's likely backed by a complex libc function, getaddrinfo(). + * Depending on the implementation of getaddrinfo() and underlying + * DNS resolver, it might or might not be possible to make it return + * with os_wakeup_blocking_op(). + * + * Unfortunately, many of ISC/bind based resolvers just keep going on + * interrupted system calls. It includes macOS and glibc. + * + * On the other hand, NuttX as of writing this returns EAI_AGAIN + * on EINTR. + */ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + errno = EINTR; + return -1; + } + int ret = os_socket_addr_resolve(host, service, hint_is_tcp, hint_is_ipv4, + addr_info, addr_info_size, max_info_size); + wasm_runtime_end_blocking_op(exec_env); + return ret; +} + +__wasi_errno_t +blocking_op_openat(wasm_exec_env_t exec_env, os_file_handle handle, + const char *path, __wasi_oflags_t oflags, + __wasi_fdflags_t fd_flags, __wasi_lookupflags_t lookup_flags, + wasi_libc_file_access_mode access_mode, os_file_handle *out) +{ + if (!wasm_runtime_begin_blocking_op(exec_env)) { + return __WASI_EINTR; + } + __wasi_errno_t error = os_openat(handle, path, oflags, fd_flags, + lookup_flags, access_mode, out); + wasm_runtime_end_blocking_op(exec_env); + return error; +} + +#ifndef BH_PLATFORM_WINDOWS +/* REVISIT: apply the os_file_handle style abstraction for pollfd? */ +__wasi_errno_t +blocking_op_poll(wasm_exec_env_t exec_env, struct pollfd *pfds, nfds_t nfds, + int timeout_ms, int *retp) +{ + int ret; + if (!wasm_runtime_begin_blocking_op(exec_env)) { + return __WASI_EINTR; + } + ret = poll(pfds, nfds, timeout_ms); + wasm_runtime_end_blocking_op(exec_env); + if (ret == -1) { + return convert_errno(errno); + } + *retp = ret; + return 0; +} +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.h new file mode 100644 index 0000000..a32e5d6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BLOCKING_OP_H_ +#define _BLOCKING_OP_H_ + +#include "bh_platform.h" +#include "wasm_export.h" + +__wasi_errno_t +blocking_op_close(wasm_exec_env_t exec_env, os_file_handle handle, + bool is_stdio); +__wasi_errno_t +blocking_op_readv(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_iovec_t *iov, int iovcnt, size_t *nread); +__wasi_errno_t +blocking_op_preadv(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_iovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nread); +__wasi_errno_t +blocking_op_writev(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_ciovec_t *iov, int iovcnt, + size_t *nwritten); +__wasi_errno_t +blocking_op_pwritev(wasm_exec_env_t exec_env, os_file_handle handle, + const struct __wasi_ciovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nwritten); +int +blocking_op_socket_accept(wasm_exec_env_t exec_env, bh_socket_t server_sock, + bh_socket_t *sockp, void *addr, + unsigned int *addrlenp); +int +blocking_op_socket_connect(wasm_exec_env_t exec_env, bh_socket_t sock, + const char *addr, int port); +int +blocking_op_socket_recv_from(wasm_exec_env_t exec_env, bh_socket_t sock, + void *buf, unsigned int len, int flags, + bh_sockaddr_t *src_addr); +int +blocking_op_socket_send_to(wasm_exec_env_t exec_env, bh_socket_t sock, + const void *buf, unsigned int len, int flags, + const bh_sockaddr_t *dest_addr); +int +blocking_op_socket_addr_resolve(wasm_exec_env_t exec_env, const char *host, + const char *service, uint8_t *hint_is_tcp, + uint8_t *hint_is_ipv4, + bh_addr_info_t *addr_info, + size_t addr_info_size, size_t *max_info_size); + +__wasi_errno_t +blocking_op_openat(wasm_exec_env_t exec_env, os_file_handle handle, + const char *path, __wasi_oflags_t oflags, + __wasi_fdflags_t fd_flags, __wasi_lookupflags_t lookup_flags, + wasi_libc_file_access_mode access_mode, os_file_handle *out); + +#ifndef BH_PLATFORM_WINDOWS +__wasi_errno_t +blocking_op_poll(wasm_exec_env_t exec_env, struct pollfd *pfds, nfds_t nfds, + int timeout, int *retp); +#endif + +#endif /* end of _BLOCKING_OP_H_ */ diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/locking.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/locking.h new file mode 100644 index 0000000..0ad40ec --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/locking.h @@ -0,0 +1,273 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#ifndef LOCKING_H +#define LOCKING_H + +#include "ssp_config.h" + +#ifndef __has_extension +#define __has_extension(x) 0 +#endif + +#if __has_extension(c_thread_safety_attributes) +#define LOCK_ANNOTATE(x) __attribute__((x)) +#else +#define LOCK_ANNOTATE(x) +#endif + +/* Lock annotation macros. */ + +#define LOCKABLE LOCK_ANNOTATE(lockable) + +#define LOCKS_EXCLUSIVE(...) LOCK_ANNOTATE(exclusive_lock_function(__VA_ARGS__)) +#define LOCKS_SHARED(...) LOCK_ANNOTATE(shared_lock_function(__VA_ARGS__)) + +#define TRYLOCKS_EXCLUSIVE(...) \ + LOCK_ANNOTATE(exclusive_trylock_function(__VA_ARGS__)) +#define TRYLOCKS_SHARED(...) LOCK_ANNOTATE(shared_trylock_function(__VA_ARGS__)) + +#define UNLOCKS(...) LOCK_ANNOTATE(unlock_function(__VA_ARGS__)) + +#define REQUIRES_EXCLUSIVE(...) \ + LOCK_ANNOTATE(exclusive_locks_required(__VA_ARGS__)) +#define REQUIRES_SHARED(...) LOCK_ANNOTATE(shared_locks_required(__VA_ARGS__)) +#define REQUIRES_UNLOCKED(...) LOCK_ANNOTATE(locks_excluded(__VA_ARGS__)) + +#define NO_LOCK_ANALYSIS LOCK_ANNOTATE(no_thread_safety_analysis) + +/* Mutex that uses the lock annotations. */ + +struct LOCKABLE mutex { + korp_mutex object; +}; + +/* clang-format off */ +#define MUTEX_INITIALIZER \ + { PTHREAD_MUTEX_INITIALIZER } +/* clang-format on */ + +static inline bool +mutex_init(struct mutex *lock) REQUIRES_UNLOCKED(*lock) +{ + return os_mutex_init(&lock->object) == BHT_OK ? true : false; +} + +static inline void +mutex_destroy(struct mutex *lock) REQUIRES_UNLOCKED(*lock) +{ + os_mutex_destroy(&lock->object); +} + +static inline void +mutex_lock(struct mutex *lock) LOCKS_EXCLUSIVE(*lock) NO_LOCK_ANALYSIS +{ + os_mutex_lock(&lock->object); +} + +static inline void +mutex_unlock(struct mutex *lock) UNLOCKS(*lock) NO_LOCK_ANALYSIS +{ + os_mutex_unlock(&lock->object); +} + +/* Read-write lock that uses the lock annotations. */ + +struct LOCKABLE rwlock { + korp_rwlock object; +}; + +static inline bool +rwlock_initialize(struct rwlock *lock) REQUIRES_UNLOCKED(*lock) +{ + return os_rwlock_init(&lock->object) == 0 ? true : false; +} + +static inline void +rwlock_rdlock(struct rwlock *lock) LOCKS_SHARED(*lock) NO_LOCK_ANALYSIS +{ + os_rwlock_rdlock(&lock->object); +} + +static inline void +rwlock_wrlock(struct rwlock *lock) LOCKS_EXCLUSIVE(*lock) NO_LOCK_ANALYSIS +{ + os_rwlock_wrlock(&lock->object); +} + +static inline void +rwlock_unlock(struct rwlock *lock) UNLOCKS(*lock) NO_LOCK_ANALYSIS +{ + os_rwlock_unlock(&lock->object); +} + +static inline void +rwlock_destroy(struct rwlock *lock) UNLOCKS(*lock) NO_LOCK_ANALYSIS +{ + os_rwlock_destroy(&lock->object); +} + +/* Condition variable that uses the lock annotations. */ + +struct LOCKABLE cond { + korp_cond object; + +#if !CONFIG_HAS_CLOCK_NANOSLEEP \ + && (!CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK \ + || !CONFIG_HAS_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP) + clockid_t clock; +#endif +}; + +static inline bool +cond_init_monotonic(struct cond *cond) +{ + bool ret = false; +#if CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK + pthread_condattr_t attr; + + if (pthread_condattr_init(&attr) != 0) + return false; + + if (pthread_condattr_setclock(&attr, CLOCK_MONOTONIC) != 0) + goto fail; + + if (pthread_cond_init(&cond->object, &attr) != 0) + goto fail; + + ret = true; +fail: + pthread_condattr_destroy(&attr); +#else + if (os_cond_init(&cond->object) != 0) + return false; + ret = true; +#endif + +#if !CONFIG_HAS_CLOCK_NANOSLEEP \ + && (!CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK \ + || !CONFIG_HAS_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP) + cond->clock = CLOCK_MONOTONIC; +#endif + + return ret; +} + +static inline bool +cond_init_realtime(struct cond *cond) +{ + if (os_cond_init(&cond->object) != 0) + return false; + +#if !CONFIG_HAS_CLOCK_NANOSLEEP \ + && (!CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK \ + || !CONFIG_HAS_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP) + cond->clock = CLOCK_REALTIME; +#endif + + return true; +} + +static inline void +cond_destroy(struct cond *cond) +{ + os_cond_destroy(&cond->object); +} + +static inline void +cond_signal(struct cond *cond) +{ + os_cond_signal(&cond->object); +} + +#if !CONFIG_HAS_CLOCK_NANOSLEEP + +static inline bool +cond_timedwait(struct cond *cond, struct mutex *lock, uint64_t timeout, + bool abstime) REQUIRES_EXCLUSIVE(*lock) NO_LOCK_ANALYSIS +{ + int ret; + struct timespec ts = { + .tv_sec = (time_t)(timeout / 1000000000), + .tv_nsec = (long)(timeout % 1000000000), + }; + + if (abstime) { +#if !CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK + /** + * No native support for sleeping on monotonic clocks. Convert the + * timeout to a relative value and then to an absolute value for the + * realtime clock. + */ + if (cond->clock != CLOCK_REALTIME) { + struct timespec ts_monotonic; + struct timespec ts_realtime; + + clock_gettime(cond->clock, &ts_monotonic); + ts.tv_sec -= ts_monotonic.tv_sec; + ts.tv_nsec -= ts_monotonic.tv_nsec; + if (ts.tv_nsec < 0) { + ts.tv_nsec += 1000000000; + --ts.tv_sec; + } + + clock_gettime(CLOCK_REALTIME, &ts_realtime); + ts.tv_sec += ts_realtime.tv_sec; + ts.tv_nsec += ts_realtime.tv_nsec; + if (ts.tv_nsec >= 1000000000) { + ts.tv_nsec -= 1000000000; + ++ts.tv_sec; + } + } +#endif + } + else { +#if CONFIG_HAS_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP + /* Implementation supports relative timeouts. */ + ret = pthread_cond_timedwait_relative_np(&cond->object, &lock->object, + &ts); + bh_assert((ret == 0 || ret == ETIMEDOUT) + && "pthread_cond_timedwait_relative_np() failed"); + return ret == ETIMEDOUT; +#else + /* Convert to absolute timeout. */ + struct timespec ts_now; +#if CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK + clock_gettime(cond->clock, &ts_now); +#else + clock_gettime(CLOCK_REALTIME, &ts_now); +#endif + ts.tv_sec += ts_now.tv_sec; + ts.tv_nsec += ts_now.tv_nsec; + if (ts.tv_nsec >= 1000000000) { + ts.tv_nsec -= 1000000000; + ++ts.tv_sec; + } +#endif + } + + ret = pthread_cond_timedwait(&cond->object, &lock->object, &ts); + bh_assert((ret == 0 || ret == ETIMEDOUT) + && "pthread_cond_timedwait() failed"); + return ret == ETIMEDOUT; +} +#endif + +static inline void +cond_wait(struct cond *cond, struct mutex *lock) + REQUIRES_EXCLUSIVE(*lock) NO_LOCK_ANALYSIS +{ + os_cond_wait(&cond->object, &lock->object); +} + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c new file mode 100644 index 0000000..03ff50a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c @@ -0,0 +1,3370 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016-2018 Nuxi, https://nuxi.nl/ + +#include "ssp_config.h" +#include "bh_platform.h" +#include "blocking_op.h" +#include "wasmtime_ssp.h" +#include "libc_errno.h" +#include "locking.h" +#include "posix.h" +#include "random.h" +#include "refcount.h" +#include "rights.h" +#include "str.h" + +/* Some platforms (e.g. Windows) already define `min()` macro. + We're undefing it here to make sure the `min` call does exactly + what we want it to do. */ +#ifdef min +#undef min +#endif +static inline size_t +min(size_t a, size_t b) +{ + return a > b ? b : a; +} + +#if 0 /* TODO: -std=gnu99 causes compile error, comment them first */ +// struct iovec must have the same layout as __wasi_iovec_t. +static_assert(offsetof(struct iovec, iov_base) == + offsetof(__wasi_iovec_t, buf), + "Offset mismatch"); +static_assert(sizeof(((struct iovec *)0)->iov_base) == + sizeof(((__wasi_iovec_t *)0)->buf), + "Size mismatch"); +static_assert(offsetof(struct iovec, iov_len) == + offsetof(__wasi_iovec_t, buf_len), + "Offset mismatch"); +static_assert(sizeof(((struct iovec *)0)->iov_len) == + sizeof(((__wasi_iovec_t *)0)->buf_len), + "Size mismatch"); +static_assert(sizeof(struct iovec) == sizeof(__wasi_iovec_t), + "Size mismatch"); + +// struct iovec must have the same layout as __wasi_ciovec_t. +static_assert(offsetof(struct iovec, iov_base) == + offsetof(__wasi_ciovec_t, buf), + "Offset mismatch"); +static_assert(sizeof(((struct iovec *)0)->iov_base) == + sizeof(((__wasi_ciovec_t *)0)->buf), + "Size mismatch"); +static_assert(offsetof(struct iovec, iov_len) == + offsetof(__wasi_ciovec_t, buf_len), + "Offset mismatch"); +static_assert(sizeof(((struct iovec *)0)->iov_len) == + sizeof(((__wasi_ciovec_t *)0)->buf_len), + "Size mismatch"); +static_assert(sizeof(struct iovec) == sizeof(__wasi_ciovec_t), + "Size mismatch"); +#endif + +static bool +ns_lookup_list_search(char **list, const char *host) +{ + size_t host_len = strlen(host), suffix_len; + + while (*list) { + if (*list[0] == '*') { + suffix_len = strlen(*list) - 1; + if (suffix_len <= host_len + && strncmp(host + host_len - suffix_len, *list + 1, suffix_len) + == 0) { + return true; + } + } + else { + if (strcmp(*list, host) == 0) { + return true; + } + } + list++; + } + + return false; +} + +#if !defined(BH_PLATFORM_WINDOWS) && CONFIG_HAS_CLOCK_NANOSLEEP +static bool +wasi_clockid_to_clockid(__wasi_clockid_t in, clockid_t *out) +{ + switch (in) { + case __WASI_CLOCK_MONOTONIC: + *out = CLOCK_MONOTONIC; + return true; +#if defined(CLOCK_PROCESS_CPUTIME_ID) + case __WASI_CLOCK_PROCESS_CPUTIME_ID: + *out = CLOCK_PROCESS_CPUTIME_ID; + return true; +#endif + case __WASI_CLOCK_REALTIME: + *out = CLOCK_REALTIME; + return true; +#if defined(CLOCK_THREAD_CPUTIME_ID) + case __WASI_CLOCK_THREAD_CPUTIME_ID: + *out = CLOCK_THREAD_CPUTIME_ID; + return true; +#endif + default: + return false; + } +} +#endif + +static void +wasi_addr_to_bh_sockaddr(const __wasi_addr_t *wasi_addr, + bh_sockaddr_t *sockaddr) +{ + if (wasi_addr->kind == IPv4) { + sockaddr->addr_buffer.ipv4 = (wasi_addr->addr.ip4.addr.n0 << 24) + | (wasi_addr->addr.ip4.addr.n1 << 16) + | (wasi_addr->addr.ip4.addr.n2 << 8) + | wasi_addr->addr.ip4.addr.n3; + sockaddr->is_ipv4 = true; + sockaddr->port = wasi_addr->addr.ip4.port; + } + else { + sockaddr->addr_buffer.ipv6[0] = wasi_addr->addr.ip6.addr.n0; + sockaddr->addr_buffer.ipv6[1] = wasi_addr->addr.ip6.addr.n1; + sockaddr->addr_buffer.ipv6[2] = wasi_addr->addr.ip6.addr.n2; + sockaddr->addr_buffer.ipv6[3] = wasi_addr->addr.ip6.addr.n3; + sockaddr->addr_buffer.ipv6[4] = wasi_addr->addr.ip6.addr.h0; + sockaddr->addr_buffer.ipv6[5] = wasi_addr->addr.ip6.addr.h1; + sockaddr->addr_buffer.ipv6[6] = wasi_addr->addr.ip6.addr.h2; + sockaddr->addr_buffer.ipv6[7] = wasi_addr->addr.ip6.addr.h3; + sockaddr->is_ipv4 = false; + sockaddr->port = wasi_addr->addr.ip6.port; + } +} + +// Converts an IPv6 binary address object to WASI address object. +static void +bh_sockaddr_to_wasi_addr(const bh_sockaddr_t *sockaddr, + __wasi_addr_t *wasi_addr) +{ + if (sockaddr->is_ipv4) { + wasi_addr->kind = IPv4; + wasi_addr->addr.ip4.port = sockaddr->port; + wasi_addr->addr.ip4.addr.n0 = + (sockaddr->addr_buffer.ipv4 & 0xFF000000) >> 24; + wasi_addr->addr.ip4.addr.n1 = + (sockaddr->addr_buffer.ipv4 & 0x00FF0000) >> 16; + wasi_addr->addr.ip4.addr.n2 = + (sockaddr->addr_buffer.ipv4 & 0x0000FF00) >> 8; + wasi_addr->addr.ip4.addr.n3 = (sockaddr->addr_buffer.ipv4 & 0x000000FF); + } + else { + wasi_addr->kind = IPv6; + wasi_addr->addr.ip6.port = sockaddr->port; + wasi_addr->addr.ip6.addr.n0 = sockaddr->addr_buffer.ipv6[0]; + wasi_addr->addr.ip6.addr.n1 = sockaddr->addr_buffer.ipv6[1]; + wasi_addr->addr.ip6.addr.n2 = sockaddr->addr_buffer.ipv6[2]; + wasi_addr->addr.ip6.addr.n3 = sockaddr->addr_buffer.ipv6[3]; + wasi_addr->addr.ip6.addr.h0 = sockaddr->addr_buffer.ipv6[4]; + wasi_addr->addr.ip6.addr.h1 = sockaddr->addr_buffer.ipv6[5]; + wasi_addr->addr.ip6.addr.h2 = sockaddr->addr_buffer.ipv6[6]; + wasi_addr->addr.ip6.addr.h3 = sockaddr->addr_buffer.ipv6[7]; + } +} + +static void +wasi_addr_ip_to_bh_ip_addr_buffer(__wasi_addr_ip_t *addr, + bh_ip_addr_buffer_t *out) +{ + if (addr->kind == IPv4) { + out->ipv4 = htonl((addr->addr.ip4.n0 << 24) | (addr->addr.ip4.n1 << 16) + | (addr->addr.ip4.n2 << 8) | addr->addr.ip4.n3); + } + else { + out->ipv6[0] = htons(addr->addr.ip6.n0); + out->ipv6[1] = htons(addr->addr.ip6.n1); + out->ipv6[2] = htons(addr->addr.ip6.n2); + out->ipv6[3] = htons(addr->addr.ip6.n3); + out->ipv6[4] = htons(addr->addr.ip6.h0); + out->ipv6[5] = htons(addr->addr.ip6.h1); + out->ipv6[6] = htons(addr->addr.ip6.h2); + out->ipv6[7] = htons(addr->addr.ip6.h3); + } +} + +struct fd_prestat { + const char *dir; +}; + +bool +fd_prestats_init(struct fd_prestats *pt) +{ + if (!rwlock_initialize(&pt->lock)) + return false; + pt->prestats = NULL; + pt->size = 0; + pt->used = 0; + return true; +} + +// Grows the preopened resource table to a required lower bound and a +// minimum number of free preopened resource table entries. +static __wasi_errno_t +fd_prestats_grow(struct fd_prestats *pt, size_t min, size_t incr) + REQUIRES_EXCLUSIVE(pt->lock) +{ + if (pt->size <= min || pt->size < (pt->used + incr) * 2) { + // Keep on doubling the table size until we've met our constraints. + size_t size = pt->size == 0 ? 1 : pt->size; + while (size <= min || size < (pt->used + incr) * 2) + size *= 2; + + // Grow the file descriptor table's allocation. + struct fd_prestat *prestats = + wasm_runtime_malloc((uint32)(sizeof(*prestats) * size)); + if (prestats == NULL) + return __WASI_ENOMEM; + + if (pt->prestats && pt->size > 0) { + bh_memcpy_s(prestats, (uint32)(sizeof(*prestats) * size), + pt->prestats, (uint32)(sizeof(*prestats) * pt->size)); + } + + if (pt->prestats) + wasm_runtime_free(pt->prestats); + + // Mark all new file descriptors as unused. + for (size_t i = pt->size; i < size; ++i) + prestats[i].dir = NULL; + pt->prestats = prestats; + pt->size = size; + } + return __WASI_ESUCCESS; +} + +static __wasi_errno_t +fd_prestats_insert_locked(struct fd_prestats *pt, const char *dir, + __wasi_fd_t fd) +{ + // Grow the preopened resource table if needed. + __wasi_errno_t error = fd_prestats_grow(pt, fd, 1); + + if (error != __WASI_ESUCCESS) { + return error; + } + + pt->prestats[fd].dir = bh_strdup(dir); + + if (pt->prestats[fd].dir == NULL) + return __WASI_ENOMEM; + + return __WASI_ESUCCESS; +} + +// Inserts a preopened resource record into the preopened resource table. +bool +fd_prestats_insert(struct fd_prestats *pt, const char *dir, __wasi_fd_t fd) +{ + rwlock_wrlock(&pt->lock); + + __wasi_errno_t error = fd_prestats_insert_locked(pt, dir, fd); + + rwlock_unlock(&pt->lock); + + return error == __WASI_ESUCCESS; +} + +// Looks up a preopened resource table entry by number. +static __wasi_errno_t +fd_prestats_get_entry(struct fd_prestats *pt, __wasi_fd_t fd, + struct fd_prestat **ret) REQUIRES_SHARED(pt->lock) +{ + // Test for file descriptor existence. + if (fd >= pt->size) + return __WASI_EBADF; + struct fd_prestat *prestat = &pt->prestats[fd]; + if (prestat->dir == NULL) + return __WASI_EBADF; + + *ret = prestat; + return 0; +} + +// Remove a preopened resource record from the preopened resource table by +// number +static __wasi_errno_t +fd_prestats_remove_entry(struct fd_prestats *pt, __wasi_fd_t fd) +{ + // Test for file descriptor existence. + if (fd >= pt->size) + return __WASI_EBADF; + struct fd_prestat *prestat = &pt->prestats[fd]; + + if (prestat->dir != NULL) { + wasm_runtime_free((void *)prestat->dir); + prestat->dir = NULL; + } + + return __WASI_ESUCCESS; +} + +struct fd_object { + struct refcount refcount; + __wasi_filetype_t type; + os_file_handle file_handle; + + // Keep track of whether this fd object refers to a stdio stream so we know + // whether to close the underlying file handle when releasing the object. + bool is_stdio; + union { + // Data associated with directory file descriptors. + struct { + struct mutex lock; // Lock to protect members below. + os_dir_stream handle; // Directory handle. + __wasi_dircookie_t offset; // Offset of the directory. + } directory; + }; +}; + +struct fd_entry { + struct fd_object *object; + __wasi_rights_t rights_base; + __wasi_rights_t rights_inheriting; +}; + +bool +fd_table_init(struct fd_table *ft) +{ + if (!rwlock_initialize(&ft->lock)) + return false; + ft->entries = NULL; + ft->size = 0; + ft->used = 0; + return true; +} + +// Looks up a file descriptor table entry by number and required rights. +static __wasi_errno_t +fd_table_get_entry(struct fd_table *ft, __wasi_fd_t fd, + __wasi_rights_t rights_base, + __wasi_rights_t rights_inheriting, struct fd_entry **ret) + REQUIRES_SHARED(ft->lock) +{ + // Test for file descriptor existence. + if (fd >= ft->size) + return __WASI_EBADF; + struct fd_entry *fe = &ft->entries[fd]; + if (fe->object == NULL) + return __WASI_EBADF; + + // Validate rights. + if ((~fe->rights_base & rights_base) != 0 + || (~fe->rights_inheriting & rights_inheriting) != 0) + return __WASI_ENOTCAPABLE; + *ret = fe; + return 0; +} + +// Grows the file descriptor table to a required lower bound and a +// minimum number of free file descriptor table entries. +static bool +fd_table_grow(struct fd_table *ft, size_t min, size_t incr) + REQUIRES_EXCLUSIVE(ft->lock) +{ + if (ft->size <= min || ft->size < (ft->used + incr) * 2) { + // Keep on doubling the table size until we've met our constraints. + size_t size = ft->size == 0 ? 1 : ft->size; + while (size <= min || size < (ft->used + incr) * 2) + size *= 2; + + // Grow the file descriptor table's allocation. + struct fd_entry *entries = + wasm_runtime_malloc((uint32)(sizeof(*entries) * size)); + if (entries == NULL) + return false; + + if (ft->entries && ft->size > 0) { + bh_memcpy_s(entries, (uint32)(sizeof(*entries) * size), ft->entries, + (uint32)(sizeof(*entries) * ft->size)); + } + + if (ft->entries) + wasm_runtime_free(ft->entries); + + // Mark all new file descriptors as unused. + for (size_t i = ft->size; i < size; ++i) + entries[i].object = NULL; + ft->entries = entries; + ft->size = size; + } + return true; +} + +// Allocates a new file descriptor object. +static __wasi_errno_t +fd_object_new(__wasi_filetype_t type, bool is_stdio, struct fd_object **fo) + TRYLOCKS_SHARED(0, (*fo)->refcount) +{ + *fo = wasm_runtime_malloc(sizeof(**fo)); + if (*fo == NULL) + return __WASI_ENOMEM; + refcount_init(&(*fo)->refcount, 1); + (*fo)->type = type; + (*fo)->file_handle = os_get_invalid_handle(); + (*fo)->is_stdio = is_stdio; + return 0; +} + +// Attaches a file descriptor to the file descriptor table. +static void +fd_table_attach(struct fd_table *ft, __wasi_fd_t fd, struct fd_object *fo, + __wasi_rights_t rights_base, __wasi_rights_t rights_inheriting) + REQUIRES_EXCLUSIVE(ft->lock) CONSUMES(fo->refcount) +{ + assert(ft->size > fd && "File descriptor table too small"); + struct fd_entry *fe = &ft->entries[fd]; + assert(fe->object == NULL + && "Attempted to overwrite an existing descriptor"); + fe->object = fo; + fe->rights_base = rights_base; + fe->rights_inheriting = rights_inheriting; + ++ft->used; + assert(ft->size >= ft->used * 2 && "File descriptor too full"); +} + +// Detaches a file descriptor from the file descriptor table. +static void +fd_table_detach(struct fd_table *ft, __wasi_fd_t fd, struct fd_object **fo) + REQUIRES_EXCLUSIVE(ft->lock) PRODUCES((*fo)->refcount) +{ + assert(ft->size > fd && "File descriptor table too small"); + struct fd_entry *fe = &ft->entries[fd]; + *fo = fe->object; + assert(*fo != NULL && "Attempted to detach nonexistent descriptor"); + fe->object = NULL; + assert(ft->used > 0 && "Reference count mismatch"); + --ft->used; +} + +// Determines the type of a file descriptor and its maximum set of +// rights that should be attached to it. +static __wasi_errno_t +fd_determine_type_rights(os_file_handle fd, __wasi_filetype_t *type, + __wasi_rights_t *rights_base, + __wasi_rights_t *rights_inheriting) +{ + struct __wasi_filestat_t buf; + __wasi_errno_t error = os_fstat(fd, &buf); + + if (error != __WASI_ESUCCESS) + return error; + + *type = buf.st_filetype; + + switch (buf.st_filetype) { + case __WASI_FILETYPE_BLOCK_DEVICE: + *rights_base = RIGHTS_BLOCK_DEVICE_BASE; + *rights_inheriting = RIGHTS_BLOCK_DEVICE_INHERITING; + break; + case __WASI_FILETYPE_CHARACTER_DEVICE: + error = os_isatty(fd); + + if (error == __WASI_ESUCCESS) { + *rights_base = RIGHTS_TTY_BASE; + *rights_inheriting = RIGHTS_TTY_INHERITING; + } + else { + *rights_base = RIGHTS_CHARACTER_DEVICE_BASE; + *rights_inheriting = RIGHTS_CHARACTER_DEVICE_INHERITING; + } + break; + case __WASI_FILETYPE_DIRECTORY: + *rights_base = RIGHTS_DIRECTORY_BASE; + *rights_inheriting = RIGHTS_DIRECTORY_INHERITING; + break; + case __WASI_FILETYPE_REGULAR_FILE: + *rights_base = RIGHTS_REGULAR_FILE_BASE; + *rights_inheriting = RIGHTS_REGULAR_FILE_INHERITING; + break; + case __WASI_FILETYPE_SOCKET_DGRAM: + case __WASI_FILETYPE_SOCKET_STREAM: + *rights_base = RIGHTS_SOCKET_BASE; + *rights_inheriting = RIGHTS_SOCKET_INHERITING; + break; + case __WASI_FILETYPE_SYMBOLIC_LINK: + case __WASI_FILETYPE_UNKNOWN: + // If we don't know the type, allow for the maximum set of + // rights + *rights_base = RIGHTS_ALL; + *rights_inheriting = RIGHTS_ALL; + break; + default: + return __WASI_EINVAL; + } + + wasi_libc_file_access_mode access_mode; + error = os_file_get_access_mode(fd, &access_mode); + + if (error != __WASI_ESUCCESS) + return error; + + // Strip off read/write bits based on the access mode. + switch (access_mode) { + case WASI_LIBC_ACCESS_MODE_READ_ONLY: + *rights_base &= ~(__wasi_rights_t)__WASI_RIGHT_FD_WRITE; + break; + case WASI_LIBC_ACCESS_MODE_WRITE_ONLY: + *rights_base &= ~(__wasi_rights_t)__WASI_RIGHT_FD_READ; + break; + } + + return error; +} + +// Lowers the reference count on a file descriptor object. When the +// reference count reaches zero, its resources are cleaned up. +static __wasi_errno_t +fd_object_release(wasm_exec_env_t env, struct fd_object *fo) + UNLOCKS(fo->refcount) +{ + __wasi_errno_t error = __WASI_ESUCCESS; + + if (refcount_release(&fo->refcount)) { + int saved_errno = errno; + switch (fo->type) { + case __WASI_FILETYPE_DIRECTORY: + // For directories we may keep track of a DIR object. + // Calling os_closedir() on it also closes the underlying file + // descriptor. + mutex_destroy(&fo->directory.lock); + if (os_is_dir_stream_valid(&fo->directory.handle)) { + error = os_closedir(fo->directory.handle); + break; + } + // Fallthrough. + default: + // The env == NULL case is for + // fd_table_destroy, path_get, path_put, + // fd_table_insert_existing + error = (env == NULL) ? os_close(fo->file_handle, fo->is_stdio) + : blocking_op_close(env, fo->file_handle, + fo->is_stdio); + break; + } + wasm_runtime_free(fo); + errno = saved_errno; + } + return error; +} + +// Inserts an already existing file descriptor into the file descriptor +// table. +bool +fd_table_insert_existing(struct fd_table *ft, __wasi_fd_t in, + os_file_handle out, bool is_stdio) +{ + __wasi_filetype_t type = __WASI_FILETYPE_UNKNOWN; + __wasi_rights_t rights_base = 0, rights_inheriting = 0; + struct fd_object *fo; + __wasi_errno_t error; + + error = + fd_determine_type_rights(out, &type, &rights_base, &rights_inheriting); + if (error != 0) { +#ifdef BH_PLATFORM_EGO + /** + * since it is an already opened file and we can assume the opened + * file has all necessary rights no matter how to get + */ + if (error != __WASI_ENOTSUP) + return false; +#else + return false; +#endif + } + + error = fd_object_new(type, is_stdio, &fo); + if (error != 0) + return false; + fo->file_handle = out; + if (type == __WASI_FILETYPE_DIRECTORY) { + if (!mutex_init(&fo->directory.lock)) { + fd_object_release(NULL, fo); + return false; + } + fo->directory.handle = os_get_invalid_dir_stream(); + } + + // Grow the file descriptor table if needed. + rwlock_wrlock(&ft->lock); + if (!fd_table_grow(ft, in, 1)) { + rwlock_unlock(&ft->lock); + fd_object_release(NULL, fo); + return false; + } + + fd_table_attach(ft, in, fo, rights_base, rights_inheriting); + rwlock_unlock(&ft->lock); + return true; +} + +// Picks an unused slot from the file descriptor table. +static __wasi_errno_t +fd_table_unused(struct fd_table *ft, __wasi_fd_t *out) REQUIRES_SHARED(ft->lock) +{ + assert(ft->size > ft->used && "File descriptor table has no free slots"); + for (;;) { + uintmax_t random_fd = 0; + __wasi_errno_t error = random_uniform(ft->size, &random_fd); + + if (error != __WASI_ESUCCESS) + return error; + + if (ft->entries[(__wasi_fd_t)random_fd].object == NULL) { + *out = (__wasi_fd_t)random_fd; + return error; + } + } +} + +// Inserts a file descriptor object into an unused slot of the file +// descriptor table. +static __wasi_errno_t +fd_table_insert(wasm_exec_env_t exec_env, struct fd_table *ft, + struct fd_object *fo, __wasi_rights_t rights_base, + __wasi_rights_t rights_inheriting, __wasi_fd_t *out) + REQUIRES_UNLOCKED(ft->lock) UNLOCKS(fo->refcount) +{ + // Grow the file descriptor table if needed. + rwlock_wrlock(&ft->lock); + if (!fd_table_grow(ft, 0, 1)) { + rwlock_unlock(&ft->lock); + fd_object_release(exec_env, fo); + return convert_errno(errno); + } + + __wasi_errno_t error = fd_table_unused(ft, out); + + if (error != __WASI_ESUCCESS) + return error; + + fd_table_attach(ft, *out, fo, rights_base, rights_inheriting); + rwlock_unlock(&ft->lock); + return error; +} + +// Inserts a numerical file descriptor into the file descriptor table. +static __wasi_errno_t +fd_table_insert_fd(wasm_exec_env_t exec_env, struct fd_table *ft, + os_file_handle in, __wasi_filetype_t type, + __wasi_rights_t rights_base, + __wasi_rights_t rights_inheriting, __wasi_fd_t *out) + REQUIRES_UNLOCKED(ft->lock) +{ + struct fd_object *fo; + + __wasi_errno_t error = fd_object_new(type, false, &fo); + if (error != 0) { + os_close(in, false); + return error; + } + + fo->file_handle = in; + if (type == __WASI_FILETYPE_DIRECTORY) { + if (!mutex_init(&fo->directory.lock)) { + fd_object_release(exec_env, fo); + return (__wasi_errno_t)-1; + } + fo->directory.handle = os_get_invalid_dir_stream(); + } + return fd_table_insert(exec_env, ft, fo, rights_base, rights_inheriting, + out); +} + +__wasi_errno_t +wasmtime_ssp_fd_prestat_get(struct fd_prestats *prestats, __wasi_fd_t fd, + __wasi_prestat_t *buf) +{ + rwlock_rdlock(&prestats->lock); + struct fd_prestat *prestat; + __wasi_errno_t error = fd_prestats_get_entry(prestats, fd, &prestat); + if (error != 0) { + rwlock_unlock(&prestats->lock); + return error; + } + + *buf = (__wasi_prestat_t){ + .pr_type = __WASI_PREOPENTYPE_DIR, + }; + + buf->u.dir.pr_name_len = strlen(prestat->dir); + + rwlock_unlock(&prestats->lock); + + return 0; +} + +__wasi_errno_t +wasmtime_ssp_fd_prestat_dir_name(struct fd_prestats *prestats, __wasi_fd_t fd, + char *path, size_t path_len) +{ + rwlock_rdlock(&prestats->lock); + struct fd_prestat *prestat; + __wasi_errno_t error = fd_prestats_get_entry(prestats, fd, &prestat); + if (error != 0) { + rwlock_unlock(&prestats->lock); + return error; + } + + const size_t prestat_dir_len = strlen(prestat->dir); + if (path_len < prestat_dir_len) { + rwlock_unlock(&prestats->lock); + return __WASI_EINVAL; + } + + bh_memcpy_s(path, (uint32)path_len, prestat->dir, (uint32)prestat_dir_len); + + rwlock_unlock(&prestats->lock); + + return 0; +} + +__wasi_errno_t +wasmtime_ssp_fd_close(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, __wasi_fd_t fd) +{ + // Validate the file descriptor. + struct fd_table *ft = curfds; + rwlock_wrlock(&ft->lock); + rwlock_wrlock(&prestats->lock); + + struct fd_entry *fe; + __wasi_errno_t error = fd_table_get_entry(ft, fd, 0, 0, &fe); + if (error != 0) { + rwlock_unlock(&prestats->lock); + rwlock_unlock(&ft->lock); + return error; + } + + // Remove it from the file descriptor table. + struct fd_object *fo; + fd_table_detach(ft, fd, &fo); + + // Remove it from the preopened resource table if it exists + error = fd_prestats_remove_entry(prestats, fd); + + rwlock_unlock(&prestats->lock); + rwlock_unlock(&ft->lock); + fd_object_release(exec_env, fo); + + // Ignore the error if there is no preopen associated with this fd + if (error == __WASI_EBADF) { + return __WASI_ESUCCESS; + } + + return error; +} + +// Look up a file descriptor object in a locked file descriptor table +// and increases its reference count. +static __wasi_errno_t +fd_object_get_locked(struct fd_object **fo, struct fd_table *ft, __wasi_fd_t fd, + __wasi_rights_t rights_base, + __wasi_rights_t rights_inheriting) + TRYLOCKS_EXCLUSIVE(0, (*fo)->refcount) REQUIRES_EXCLUSIVE(ft->lock) +{ + // Test whether the file descriptor number is valid. + struct fd_entry *fe; + __wasi_errno_t error = + fd_table_get_entry(ft, fd, rights_base, rights_inheriting, &fe); + if (error != 0) + return error; + + // Increase the reference count on the file descriptor object. A copy + // of the rights are also stored, so callers can still access those if + // needed. + *fo = fe->object; + refcount_acquire(&(*fo)->refcount); + return 0; +} + +// Temporarily locks the file descriptor table to look up a file +// descriptor object, increases its reference count and drops the lock. +static __wasi_errno_t +fd_object_get(struct fd_table *curfds, struct fd_object **fo, __wasi_fd_t fd, + __wasi_rights_t rights_base, __wasi_rights_t rights_inheriting) + TRYLOCKS_EXCLUSIVE(0, (*fo)->refcount) +{ + struct fd_table *ft = curfds; + rwlock_rdlock(&ft->lock); + __wasi_errno_t error = + fd_object_get_locked(fo, ft, fd, rights_base, rights_inheriting); + rwlock_unlock(&ft->lock); + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_datasync(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_DATASYNC, 0); + if (error != 0) + return error; + + error = os_fdatasync(fo->file_handle); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_pread(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_iovec_t *iov, size_t iovcnt, + __wasi_filesize_t offset, size_t *nread) +{ + if (iovcnt == 0) + return __WASI_EINVAL; + + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_READ, 0); + + if (error != 0) + return error; + + error = blocking_op_preadv(exec_env, fo->file_handle, iov, (int)iovcnt, + offset, nread); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_pwrite(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_ciovec_t *iov, + size_t iovcnt, __wasi_filesize_t offset, + size_t *nwritten) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_WRITE, 0); + + if (error != 0) + return error; + + error = blocking_op_pwritev(exec_env, fo->file_handle, iov, (int)iovcnt, + offset, nwritten); + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_read(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_iovec_t *iov, size_t iovcnt, + size_t *nread) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_READ, 0); + + if (error != 0) + return error; + + error = + blocking_op_readv(exec_env, fo->file_handle, iov, (int)iovcnt, nread); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_renumber(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, __wasi_fd_t from, + __wasi_fd_t to) +{ + struct fd_table *ft = curfds; + rwlock_wrlock(&ft->lock); + rwlock_wrlock(&prestats->lock); + + struct fd_entry *fe_from; + __wasi_errno_t error = fd_table_get_entry(ft, from, 0, 0, &fe_from); + if (error != 0) { + rwlock_unlock(&prestats->lock); + rwlock_unlock(&ft->lock); + return error; + } + struct fd_entry *fe_to; + error = fd_table_get_entry(ft, to, 0, 0, &fe_to); + if (error != 0) { + rwlock_unlock(&prestats->lock); + rwlock_unlock(&ft->lock); + return error; + } + + struct fd_object *fo; + fd_table_detach(ft, to, &fo); + refcount_acquire(&fe_from->object->refcount); + fd_table_attach(ft, to, fe_from->object, fe_from->rights_base, + fe_from->rights_inheriting); + fd_object_release(exec_env, fo); + + // Remove the old fd from the file descriptor table. + fd_table_detach(ft, from, &fo); + fd_object_release(exec_env, fo); + --ft->used; + + // Handle renumbering of any preopened resources + struct fd_prestat *prestat_from; + __wasi_errno_t prestat_from_error = + fd_prestats_get_entry(prestats, from, &prestat_from); + + struct fd_prestat *prestat_to; + __wasi_errno_t prestat_to_error = + fd_prestats_get_entry(prestats, to, &prestat_to); + + // Renumbering over two preopened resources. + if (prestat_from_error == __WASI_ESUCCESS + && prestat_to_error == __WASI_ESUCCESS) { + (void)fd_prestats_remove_entry(prestats, to); + + error = fd_prestats_insert_locked(prestats, prestat_from->dir, to); + + if (error == __WASI_ESUCCESS) { + (void)fd_prestats_remove_entry(prestats, from); + } + else { + (void)fd_prestats_remove_entry(prestats, to); + } + } + // Renumbering from a non-preopened fd to a preopened fd. In this case, + // we can't a keep the destination fd entry in the preopened table so + // remove it entirely. + else if (prestat_from_error != __WASI_ESUCCESS + && prestat_to_error == __WASI_ESUCCESS) { + (void)fd_prestats_remove_entry(prestats, to); + } + // Renumbering from a preopened fd to a non-preopened fd + else if (prestat_from_error == __WASI_ESUCCESS + && prestat_to_error != __WASI_ESUCCESS) { + error = fd_prestats_insert_locked(prestats, prestat_from->dir, to); + + if (error == __WASI_ESUCCESS) { + (void)fd_prestats_remove_entry(prestats, from); + } + else { + (void)fd_prestats_remove_entry(prestats, to); + } + } + + rwlock_unlock(&prestats->lock); + rwlock_unlock(&ft->lock); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_seek(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filedelta_t offset, + __wasi_whence_t whence, __wasi_filesize_t *newoffset) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, + offset == 0 && whence == __WASI_WHENCE_CUR + ? __WASI_RIGHT_FD_TELL + : __WASI_RIGHT_FD_SEEK | __WASI_RIGHT_FD_TELL, + 0); + if (error != 0) + return error; + + error = os_lseek(fo->file_handle, offset, whence, newoffset); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_tell(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filesize_t *newoffset) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_TELL, 0); + if (error != 0) + return error; + + error = os_lseek(fo->file_handle, 0, __WASI_WHENCE_CUR, newoffset); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_fdstat_get(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_fdstat_t *buf) +{ + struct fd_table *ft = curfds; + struct fd_entry *fe; + __wasi_errno_t error; + + (void)exec_env; + + rwlock_rdlock(&ft->lock); + error = fd_table_get_entry(ft, fd, 0, 0, &fe); + if (error != __WASI_ESUCCESS) { + rwlock_unlock(&ft->lock); + return error; + } + + // Extract file descriptor type and rights. + struct fd_object *fo = fe->object; + + __wasi_fdflags_t flags; + error = os_file_get_fdflags(fo->file_handle, &flags); + + if (error != __WASI_ESUCCESS) { + rwlock_unlock(&ft->lock); + return error; + } + + *buf = (__wasi_fdstat_t){ .fs_filetype = fo->type, + .fs_rights_base = fe->rights_base, + .fs_rights_inheriting = fe->rights_inheriting, + .fs_flags = flags }; + + rwlock_unlock(&ft->lock); + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_fdstat_set_flags(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_fdflags_t fs_flags) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_FDSTAT_SET_FLAGS, 0); + + if (error != 0) + return error; + + error = os_file_set_fdflags(fo->file_handle, fs_flags); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_fdstat_set_rights(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_rights_t fs_rights_base, + __wasi_rights_t fs_rights_inheriting) +{ + struct fd_table *ft = curfds; + struct fd_entry *fe; + __wasi_errno_t error; + + (void)exec_env; + + rwlock_wrlock(&ft->lock); + error = + fd_table_get_entry(ft, fd, fs_rights_base, fs_rights_inheriting, &fe); + if (error != 0) { + rwlock_unlock(&ft->lock); + return error; + } + + // Restrict the rights on the file descriptor. + fe->rights_base = fs_rights_base; + fe->rights_inheriting = fs_rights_inheriting; + rwlock_unlock(&ft->lock); + return 0; +} + +__wasi_errno_t +wasmtime_ssp_fd_sync(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_SYNC, 0); + + if (error != 0) + return error; + + error = os_fsync(fo->file_handle); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_write(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const __wasi_ciovec_t *iov, size_t iovcnt, + size_t *nwritten) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_WRITE, 0); + if (error != 0) + return error; + +#ifndef BH_VPRINTF + error = blocking_op_writev(exec_env, fo->file_handle, iov, (int)iovcnt, + nwritten); +#else + /* redirect stdout/stderr output to BH_VPRINTF function */ + if (fo->is_stdio) { + int i; + *nwritten = 0; + for (i = 0; i < (int)iovcnt; i++) { + if (iov[i].buf_len > 0 && iov[i].buf != NULL) { + char format[16]; + + /* make up format string "%.ns" */ + snprintf(format, sizeof(format), "%%.%ds", (int)iov[i].buf_len); + *nwritten += (size_t)os_printf(format, iov[i].buf); + } + } + } + else { + error = blocking_op_writev(exec_env, fo->file_handle, iov, (int)iovcnt, + nwritten); + } +#endif /* end of BH_VPRINTF */ + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_advise(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filesize_t offset, + __wasi_filesize_t len, __wasi_advice_t advice) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_ADVISE, 0); + if (error != 0) + return error; + + if (fo->type == __WASI_FILETYPE_DIRECTORY) { + fd_object_release(exec_env, fo); + return __WASI_EBADF; + } + + error = os_fadvise(fo->file_handle, offset, len, advice); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_allocate(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filesize_t offset, + __wasi_filesize_t len) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_ALLOCATE, 0); + if (error != __WASI_ESUCCESS) + return error; + + error = os_fallocate(fo->file_handle, offset, len); + + fd_object_release(exec_env, fo); + + return error; +} + +// Reads the entire contents of a symbolic link, returning the contents +// in an allocated buffer. The allocated buffer is large enough to fit +// at least one extra byte, so the caller may append a trailing slash to +// it. This is needed by path_get(). +__wasi_errno_t +readlinkat_dup(os_file_handle handle, const char *path, size_t *p_len, + char **out_buf) +{ + char *buf = NULL; + size_t len = 32; + size_t len_org = len; + + for (;;) { + char *newbuf = wasm_runtime_malloc((uint32)len); + + if (newbuf == NULL) { + if (buf) + wasm_runtime_free(buf); + *out_buf = NULL; + return __WASI_ENOMEM; + } + + if (buf != NULL) { + bh_memcpy_s(newbuf, (uint32)len, buf, (uint32)len_org); + wasm_runtime_free(buf); + } + + buf = newbuf; + size_t bytes_read = 0; + __wasi_errno_t error = + os_readlinkat(handle, path, buf, len, &bytes_read); + if (error != __WASI_ESUCCESS) { + wasm_runtime_free(buf); + *out_buf = NULL; + return error; + } + if ((size_t)bytes_read + 1 < len) { + buf[bytes_read] = '\0'; + *p_len = len; + *out_buf = buf; + + return __WASI_ESUCCESS; + } + len_org = len; + len *= 2; + } +} + +// Lease to a directory, so a path underneath it can be accessed. +// +// This structure is used by system calls that operate on pathnames. In +// this environment, pathnames always consist of a pair of a file +// descriptor representing the directory where the lookup needs to start +// and the actual pathname string. +struct path_access { + os_file_handle fd; // Directory file descriptor. + const char *path; // Pathname. + bool follow; // Whether symbolic links should be followed. + char *path_start; // Internal: pathname to free. + struct fd_object *fd_object; // Internal: directory file descriptor object. +}; + +// Creates a lease to a file descriptor and pathname pair. If the +// operating system does not implement Capsicum, it also normalizes the +// pathname to ensure the target path is placed underneath the +// directory. +static __wasi_errno_t +path_get(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct path_access *pa, __wasi_fd_t fd, __wasi_lookupflags_t flags, + const char *upath, size_t upathlen, __wasi_rights_t rights_base, + __wasi_rights_t rights_inheriting, bool needs_final_component) + TRYLOCKS_EXCLUSIVE(0, pa->fd_object->refcount) +{ + char *path = str_nullterminate(upath, upathlen); + if (path == NULL) + return convert_errno(errno); + + // Fetch the directory file descriptor. + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, rights_base, rights_inheriting); + if (error != 0) { + wasm_runtime_free(path); + return error; + } + +#if CONFIG_HAS_CAP_ENTER + // Rely on the kernel to constrain access to automatically constrain + // access to files stored underneath this directory. + pa->fd = fo->file_handle; + pa->path = pa->path_start = path; + pa->follow = (flags & __WASI_LOOKUP_SYMLINK_FOLLOW) != 0; + pa->fd_object = fo; + return 0; +#else + // The implementation provides no mechanism to constrain lookups to a + // directory automatically. Emulate this logic by resolving the + // pathname manually. + + // Stack of directory file descriptors. Index 0 always corresponds + // with the directory provided to this function. Entering a directory + // causes a file descriptor to be pushed, while handling ".." entries + // causes an entry to be popped. Index 0 cannot be popped, as this + // would imply escaping the base directory. + os_file_handle fds[128]; + fds[0] = fo->file_handle; + size_t curfd = 0; + + // Stack of pathname strings used for symlink expansion. By using a + // stack, there is no need to concatenate any pathname strings while + // expanding symlinks. + char *paths[32]; + char *paths_start[32]; + paths[0] = paths_start[0] = path; + size_t curpath = 0; + size_t expansions = 0; + char *symlink = NULL; + size_t symlink_len; +#ifdef BH_PLATFORM_WINDOWS +#define PATH_SEPARATORS "/\\" +#else +#define PATH_SEPARATORS "/" +#endif + + for (;;) { + // Extract the next pathname component from 'paths[curpath]', null + // terminate it and store it in 'file'. 'ends_with_slashes' stores + // whether the pathname component is followed by one or more + // trailing slashes, as this requires it to be a directory. + char *file = paths[curpath]; + char *file_end = file + strcspn(file, PATH_SEPARATORS); + paths[curpath] = file_end + strspn(file_end, PATH_SEPARATORS); + bool ends_with_slashes = + (*file_end != '\0' && strchr(PATH_SEPARATORS, *file_end)); + *file_end = '\0'; + + // Test for empty pathname strings and absolute paths. + if (file == file_end) { + error = ends_with_slashes ? __WASI_ENOTCAPABLE : __WASI_ENOENT; + goto fail; + } + + if (strcmp(file, ".") == 0) { + // Skip component. + } + else if (strcmp(file, "..") == 0) { + // Pop a directory off the stack. + if (curfd == 0) { + // Attempted to go to parent directory of the directory file + // descriptor. + error = __WASI_ENOTCAPABLE; + goto fail; + } + error = os_close(fds[curfd--], false); + + if (error != __WASI_ESUCCESS) + goto fail; + } + else if (curpath > 0 || *paths[curpath] != '\0' + || (ends_with_slashes && !needs_final_component)) { + // A pathname component whose name we're not interested in that is + // followed by a slash or is followed by other pathname + // components. In other words, a pathname component that must be a + // directory. First attempt to obtain a directory file descriptor + // for it. + os_file_handle newdir; + error = blocking_op_openat( + exec_env, fds[curfd], file, __WASI_O_DIRECTORY, 0, 0, + WASI_LIBC_ACCESS_MODE_READ_ONLY, &newdir); + if (error == __WASI_ESUCCESS) { + // Success. Push it onto the directory stack. + if (curfd + 1 == sizeof(fds) / sizeof(fds[0])) { + os_close(newdir, false); + error = __WASI_ENAMETOOLONG; + goto fail; + } + fds[++curfd] = newdir; + } + else { + // Failed to open it. Attempt symlink expansion. + if (error != __WASI_ELOOP && error != __WASI_EMLINK + && error != __WASI_ENOTDIR) { + goto fail; + } + error = + readlinkat_dup(fds[curfd], file, &symlink_len, &symlink); + + if (error == __WASI_ESUCCESS) { + bh_assert(symlink != NULL); + goto push_symlink; + } + + // readlink returns EINVAL if the path isn't a symlink. In that + // case, it's more informative to return ENOTDIR. + if (error == __WASI_EINVAL) + error = __WASI_ENOTDIR; + + goto fail; + } + } + else { + // The final pathname component. Depending on whether it ends with + // a slash or the symlink-follow flag is set, perform symlink + // expansion. + if (ends_with_slashes + || (flags & __WASI_LOOKUP_SYMLINK_FOLLOW) != 0) { + error = + readlinkat_dup(fds[curfd], file, &symlink_len, &symlink); + if (error == __WASI_ESUCCESS) { + bh_assert(symlink != NULL); + goto push_symlink; + } + if (error != __WASI_EINVAL && error != __WASI_ENOENT) { + goto fail; + } + } + + // Not a symlink, meaning we're done. Return the filename, + // together with the directory containing this file. + // + // If the file was followed by a trailing slash, we must retain + // it, to ensure system calls properly return ENOTDIR. + // Unfortunately, this opens up a race condition, because this + // means that users of path_get() will perform symlink expansion a + // second time. There is nothing we can do to mitigate this, as + // far as I know. + if (ends_with_slashes) + *file_end = '/'; + pa->path = file; + pa->path_start = paths_start[0]; + goto success; + } + + if (*paths[curpath] == '\0') { + if (curpath == 0) { + // No further pathname components to process. We may end up here + // when called on paths like ".", "a/..", but also if the path + // had trailing slashes and the caller is not interested in the + // name of the pathname component. + wasm_runtime_free(paths_start[0]); + pa->path = "."; + pa->path_start = NULL; + goto success; + } + + // Finished expanding symlink. Continue processing along the + // original path. + wasm_runtime_free(paths_start[curpath--]); + } + continue; + + push_symlink: + // Prevent infinite loops by placing an upper limit on the number of + // symlink expansions. + if (++expansions == 128) { + wasm_runtime_free(symlink); + error = __WASI_ELOOP; + goto fail; + } + + if (*paths[curpath] == '\0') { + // The original path already finished processing. Replace it by + // this symlink entirely. + wasm_runtime_free(paths_start[curpath]); + } + else if (curpath + 1 == sizeof(paths) / sizeof(paths[0])) { + // Too many nested symlinks. Stop processing. + wasm_runtime_free(symlink); + error = __WASI_ELOOP; + goto fail; + } + else { + // The original path still has components left. Retain the + // components that remain, so we can process them afterwards. + ++curpath; + } + + // Append a trailing slash to the symlink if the path leading up to + // it also contained one. Otherwise we would not throw ENOTDIR if + // the target is not a directory. + if (ends_with_slashes) + bh_strcat_s(symlink, (uint32)symlink_len, "/"); + paths[curpath] = paths_start[curpath] = symlink; + } + +success: + // Return the lease. Close all directories, except the one the caller + // needs to use. + for (size_t i = 1; i < curfd; ++i) + os_close(fds[i], false); + pa->fd = fds[curfd]; + pa->follow = false; + pa->fd_object = fo; + return 0; + +fail: + // Failure. Free all resources. + for (size_t i = 1; i <= curfd; ++i) + os_close(fds[i], false); + for (size_t i = 0; i <= curpath; ++i) + wasm_runtime_free(paths_start[i]); + fd_object_release(NULL, fo); + return error; +#endif +} + +static __wasi_errno_t +path_get_nofollow(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct path_access *pa, __wasi_fd_t fd, const char *path, + size_t pathlen, __wasi_rights_t rights_base, + __wasi_rights_t rights_inheriting, bool needs_final_component) + TRYLOCKS_EXCLUSIVE(0, pa->fd_object->refcount) +{ + __wasi_lookupflags_t flags = 0; + return path_get(exec_env, curfds, pa, fd, flags, path, pathlen, rights_base, + rights_inheriting, needs_final_component); +} + +static void +path_put(struct path_access *pa) UNLOCKS(pa->fd_object->refcount) +{ + if (pa->path_start) + wasm_runtime_free(pa->path_start); + if (pa->fd_object->file_handle != pa->fd) + os_close(pa->fd, false); + fd_object_release(NULL, pa->fd_object); +} + +__wasi_errno_t +wasmtime_ssp_path_create_directory(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + const char *path, size_t pathlen) +{ + struct path_access pa; + __wasi_errno_t error = + path_get_nofollow(exec_env, curfds, &pa, fd, path, pathlen, + __WASI_RIGHT_PATH_CREATE_DIRECTORY, 0, true); + if (error != 0) + return error; + + error = os_mkdirat(pa.fd, pa.path); + path_put(&pa); + + return error; +} + +static bool +validate_path(const char *path, struct fd_prestats *pt) +{ + size_t i; + char path_resolved[PATH_MAX], prestat_dir_resolved[PATH_MAX]; + char *path_real, *prestat_dir_real; + + if (!(path_real = os_realpath(path, path_resolved))) + /* path doesn't exist, creating a link to this file + is allowed: if this file is to be created in + the future, WASI will strictly check whether it + can be created or not. */ + return true; + + for (i = 0; i < pt->size; i++) { + if (pt->prestats[i].dir) { + if (!(prestat_dir_real = + os_realpath(pt->prestats[i].dir, prestat_dir_resolved))) + return false; + if (!strncmp(path_real, prestat_dir_real, strlen(prestat_dir_real))) + return true; + } + } + + return false; +} + +__wasi_errno_t +wasmtime_ssp_path_link(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, __wasi_fd_t old_fd, + __wasi_lookupflags_t old_flags, const char *old_path, + size_t old_path_len, __wasi_fd_t new_fd, + const char *new_path, size_t new_path_len) +{ + struct path_access old_pa; + __wasi_errno_t error = + path_get(exec_env, curfds, &old_pa, old_fd, old_flags, old_path, + old_path_len, __WASI_RIGHT_PATH_LINK_SOURCE, 0, false); + if (error != 0) + return error; + + struct path_access new_pa; + error = + path_get_nofollow(exec_env, curfds, &new_pa, new_fd, new_path, + new_path_len, __WASI_RIGHT_PATH_LINK_TARGET, 0, true); + if (error != 0) { + path_put(&old_pa); + return error; + } + + rwlock_rdlock(&prestats->lock); + if (!validate_path(old_pa.path, prestats) + || !validate_path(new_pa.path, prestats)) { + rwlock_unlock(&prestats->lock); + return __WASI_EBADF; + } + rwlock_unlock(&prestats->lock); + + error = os_linkat(old_pa.fd, old_pa.path, new_pa.fd, new_pa.path, + old_pa.follow ? __WASI_LOOKUP_SYMLINK_FOLLOW : 0); + +#if defined(__APPLE__) + if (error == __WASI_ENOTSUP && !old_pa.follow) { + // OS X doesn't allow creating hardlinks to symbolic links. + // Duplicate the symbolic link instead. + size_t target_len; + char *target = NULL; + error = readlinkat_dup(old_pa.fd, old_pa.path, &target_len, &target); + if (error == __WASI_ESUCCESS) { + bh_assert(target != NULL); + bh_assert(target[target_len] == '\0'); + rwlock_rdlock(&prestats->lock); + if (!validate_path(target, prestats)) { + rwlock_unlock(&prestats->lock); + wasm_runtime_free(target); + return __WASI_EBADF; + } + rwlock_unlock(&prestats->lock); + error = os_symlinkat(target, new_pa.fd, new_pa.path); + wasm_runtime_free(target); + } + } +#endif + + path_put(&old_pa); + path_put(&new_pa); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_path_open(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t dirfd, __wasi_lookupflags_t dirflags, + const char *path, size_t pathlen, __wasi_oflags_t oflags, + __wasi_rights_t fs_rights_base, + __wasi_rights_t fs_rights_inheriting, + __wasi_fdflags_t fs_flags, __wasi_fd_t *fd) +{ + // Rights that should be installed on the new file descriptor. + __wasi_rights_t rights_base = fs_rights_base; + __wasi_rights_t rights_inheriting = fs_rights_inheriting; + + // Which open() mode should be used to satisfy the needed rights. + bool read = + (rights_base & (__WASI_RIGHT_FD_READ | __WASI_RIGHT_FD_READDIR)) != 0; + bool write = + (rights_base + & (__WASI_RIGHT_FD_DATASYNC | __WASI_RIGHT_FD_WRITE + | __WASI_RIGHT_FD_ALLOCATE | __WASI_RIGHT_FD_FILESTAT_SET_SIZE)) + != 0; + + wasi_libc_file_access_mode access_mode = + write ? read ? WASI_LIBC_ACCESS_MODE_READ_WRITE + : WASI_LIBC_ACCESS_MODE_WRITE_ONLY + : WASI_LIBC_ACCESS_MODE_READ_ONLY; + + // Which rights are needed on the directory file descriptor. + __wasi_rights_t needed_base = __WASI_RIGHT_PATH_OPEN; + __wasi_rights_t needed_inheriting = rights_base | rights_inheriting; + + // Convert open flags. + if ((oflags & __WASI_O_CREAT) != 0) { + needed_base |= __WASI_RIGHT_PATH_CREATE_FILE; + } + if ((oflags & __WASI_O_TRUNC) != 0) { + needed_base |= __WASI_RIGHT_PATH_FILESTAT_SET_SIZE; + } + + // Convert file descriptor flags. + if ((fs_flags & __WASI_FDFLAG_SYNC) != 0) { + needed_inheriting |= __WASI_RIGHT_FD_SYNC; + } + if ((fs_flags & __WASI_FDFLAG_RSYNC) != 0) { + needed_inheriting |= __WASI_RIGHT_FD_SYNC; + } + if ((fs_flags & __WASI_FDFLAG_DSYNC) != 0) { + needed_inheriting |= __WASI_RIGHT_FD_DATASYNC; + } + + if (write + && !((fs_flags & __WASI_FDFLAG_APPEND) || (__WASI_O_TRUNC & oflags))) + needed_inheriting |= __WASI_RIGHT_FD_SEEK; + + struct path_access pa; + __wasi_errno_t error = path_get( + exec_env, curfds, &pa, dirfd, dirflags, path, pathlen, needed_base, + needed_inheriting, (oflags & __WASI_O_CREAT) != 0); + + if (error != 0) + return error; + + os_file_handle handle; + error = blocking_op_openat(exec_env, pa.fd, pa.path, oflags, fs_flags, + dirflags, access_mode, &handle); + + path_put(&pa); + + if (error != __WASI_ESUCCESS) + return error; + + // Determine the type of the new file descriptor and which rights + // contradict with this type. + __wasi_filetype_t type; + __wasi_rights_t max_base, max_inheriting; + + error = fd_determine_type_rights(handle, &type, &max_base, &max_inheriting); + + if (error != __WASI_ESUCCESS) { + os_close(handle, false); + return error; + } + + return fd_table_insert_fd(exec_env, curfds, handle, type, + rights_base & max_base, + rights_inheriting & max_inheriting, fd); +} + +// Copies out directory entry metadata or filename, potentially +// truncating it in the process. +static void +fd_readdir_put(void *buf, size_t bufsize, size_t *bufused, const void *elem, + size_t elemsize) +{ + size_t bufavail = bufsize - *bufused; + if (elemsize > bufavail) + elemsize = bufavail; + bh_memcpy_s((char *)buf + *bufused, (uint32)bufavail, elem, + (uint32)elemsize); + *bufused += elemsize; +} + +__wasi_errno_t +wasmtime_ssp_fd_readdir(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, void *buf, size_t nbyte, + __wasi_dircookie_t cookie, size_t *bufused) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_READDIR, 0); + if (error != 0) { + return error; + } + + // Create a directory handle if none has been opened yet. + mutex_lock(&fo->directory.lock); + if (!os_is_dir_stream_valid(&fo->directory.handle)) { + error = os_fdopendir(fo->file_handle, &fo->directory.handle); + if (error != __WASI_ESUCCESS) { + mutex_unlock(&fo->directory.lock); + fd_object_release(exec_env, fo); + return error; + } + fo->directory.offset = __WASI_DIRCOOKIE_START; + } + + // Seek to the right position if the requested offset does not match + // the current offset. + if (fo->directory.offset != cookie) { + if (cookie == __WASI_DIRCOOKIE_START) + os_rewinddir(fo->directory.handle); + else + os_seekdir(fo->directory.handle, cookie); + fo->directory.offset = cookie; + } + + *bufused = 0; + while (*bufused < nbyte) { + // Read the next directory entry. + __wasi_dirent_t cde; + const char *d_name = NULL; + + error = os_readdir(fo->directory.handle, &cde, &d_name); + if (d_name == NULL) { + mutex_unlock(&fo->directory.lock); + fd_object_release(exec_env, fo); + + return *bufused > 0 ? __WASI_ESUCCESS : error; + } + + fo->directory.offset = cde.d_next; + + fd_readdir_put(buf, nbyte, bufused, &cde, sizeof(cde)); + fd_readdir_put(buf, nbyte, bufused, d_name, cde.d_namlen); + } + mutex_unlock(&fo->directory.lock); + fd_object_release(exec_env, fo); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_path_readlink(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const char *path, size_t pathlen, + char *buf, size_t bufsize, size_t *bufused) +{ + struct path_access pa; + __wasi_errno_t error = + path_get_nofollow(exec_env, curfds, &pa, fd, path, pathlen, + __WASI_RIGHT_PATH_READLINK, 0, false); + + if (error != 0) + return error; + + error = os_readlinkat(pa.fd, pa.path, buf, bufsize, bufused); + + path_put(&pa); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_path_rename(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t old_fd, const char *old_path, + size_t old_path_len, __wasi_fd_t new_fd, + const char *new_path, size_t new_path_len) +{ + struct path_access old_pa; + __wasi_errno_t error = path_get_nofollow( + exec_env, curfds, &old_pa, old_fd, old_path, old_path_len, + __WASI_RIGHT_PATH_RENAME_SOURCE, 0, true); + if (error != 0) + return error; + + struct path_access new_pa; + error = path_get_nofollow(exec_env, curfds, &new_pa, new_fd, new_path, + new_path_len, __WASI_RIGHT_PATH_RENAME_TARGET, 0, + true); + if (error != 0) { + path_put(&old_pa); + return error; + } + + error = os_renameat(old_pa.fd, old_pa.path, new_pa.fd, new_pa.path); + + path_put(&old_pa); + path_put(&new_pa); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_filestat_get(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_filestat_t *buf) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_FILESTAT_GET, 0); + + if (error != 0) + return error; + + error = os_fstat(fo->file_handle, buf); + + fd_object_release(exec_env, fo); + + return error; +} + +static void +convert_timestamp(__wasi_timestamp_t in, struct timespec *out) +{ + // Store sub-second remainder. +#if defined(__SYSCALL_SLONG_TYPE) + out->tv_nsec = (__SYSCALL_SLONG_TYPE)(in % 1000000000); +#else + out->tv_nsec = (long)(in % 1000000000); +#endif + in /= 1000000000; + + // Clamp to the maximum in case it would overflow our system's time_t. + out->tv_sec = (time_t)in < BH_TIME_T_MAX ? (time_t)in : BH_TIME_T_MAX; +} + +__wasi_errno_t +wasmtime_ssp_fd_filestat_set_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_filesize_t st_size) +{ + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_FILESTAT_SET_SIZE, 0); + + if (error != 0) + return error; + + error = os_ftruncate(fo->file_handle, st_size); + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_fd_filestat_set_times(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_timestamp_t st_atim, + __wasi_timestamp_t st_mtim, + __wasi_fstflags_t fstflags) +{ + if ((fstflags + & ~(__WASI_FILESTAT_SET_ATIM | __WASI_FILESTAT_SET_ATIM_NOW + | __WASI_FILESTAT_SET_MTIM | __WASI_FILESTAT_SET_MTIM_NOW)) + != 0 + || (fstflags + & (__WASI_FILESTAT_SET_ATIM | __WASI_FILESTAT_SET_ATIM_NOW)) + == (__WASI_FILESTAT_SET_ATIM | __WASI_FILESTAT_SET_ATIM_NOW) + || (fstflags + & (__WASI_FILESTAT_SET_MTIM | __WASI_FILESTAT_SET_MTIM_NOW)) + == (__WASI_FILESTAT_SET_MTIM | __WASI_FILESTAT_SET_MTIM_NOW)) + return __WASI_EINVAL; + + struct fd_object *fo; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_FD_FILESTAT_SET_TIMES, 0); + if (error != 0) + return error; + + error = os_futimens(fo->file_handle, st_atim, st_mtim, fstflags); + + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_path_filestat_get(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_lookupflags_t flags, const char *path, + size_t pathlen, __wasi_filestat_t *buf) +{ + struct path_access pa; + __wasi_errno_t error = + path_get(exec_env, curfds, &pa, fd, flags, path, pathlen, + __WASI_RIGHT_PATH_FILESTAT_GET, 0, false); + if (error != 0) + return error; + + error = os_fstatat(pa.fd, pa.path, buf, + pa.follow ? __WASI_LOOKUP_SYMLINK_FOLLOW : 0); + + path_put(&pa); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_path_filestat_set_times(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_lookupflags_t flags, + const char *path, size_t pathlen, + __wasi_timestamp_t st_atim, + __wasi_timestamp_t st_mtim, + __wasi_fstflags_t fstflags) +{ + if (((fstflags + & ~(__WASI_FILESTAT_SET_ATIM | __WASI_FILESTAT_SET_ATIM_NOW + | __WASI_FILESTAT_SET_MTIM | __WASI_FILESTAT_SET_MTIM_NOW)) + != 0) + /* ATIM & ATIM_NOW can't be set at the same time */ + || ((fstflags & __WASI_FILESTAT_SET_ATIM) != 0 + && (fstflags & __WASI_FILESTAT_SET_ATIM_NOW) != 0) + /* MTIM & MTIM_NOW can't be set at the same time */ + || ((fstflags & __WASI_FILESTAT_SET_MTIM) != 0 + && (fstflags & __WASI_FILESTAT_SET_MTIM_NOW) != 0)) + return __WASI_EINVAL; + + struct path_access pa; + __wasi_errno_t error = + path_get(exec_env, curfds, &pa, fd, flags, path, pathlen, + __WASI_RIGHT_PATH_FILESTAT_SET_TIMES, 0, false); + if (error != 0) + return error; + + error = os_utimensat(pa.fd, pa.path, st_atim, st_mtim, fstflags, + pa.follow ? __WASI_LOOKUP_SYMLINK_FOLLOW : 0); + + path_put(&pa); + return error; +} + +__wasi_errno_t +wasmtime_ssp_path_symlink(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct fd_prestats *prestats, const char *old_path, + size_t old_path_len, __wasi_fd_t fd, + const char *new_path, size_t new_path_len) +{ + char *target = str_nullterminate(old_path, old_path_len); + if (target == NULL) + return convert_errno(errno); + + struct path_access pa; + __wasi_errno_t error = + path_get_nofollow(exec_env, curfds, &pa, fd, new_path, new_path_len, + __WASI_RIGHT_PATH_SYMLINK, 0, true); + if (error != 0) { + wasm_runtime_free(target); + return error; + } + + rwlock_rdlock(&prestats->lock); + if (!validate_path(target, prestats)) { + rwlock_unlock(&prestats->lock); + wasm_runtime_free(target); + return __WASI_EBADF; + } + rwlock_unlock(&prestats->lock); + + error = os_symlinkat(target, pa.fd, pa.path); + + path_put(&pa); + wasm_runtime_free(target); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_path_unlink_file(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, const char *path, size_t pathlen) +{ + struct path_access pa; + __wasi_errno_t error = + path_get_nofollow(exec_env, curfds, &pa, fd, path, pathlen, + __WASI_RIGHT_PATH_UNLINK_FILE, 0, true); + if (error != __WASI_ESUCCESS) + return error; + + error = os_unlinkat(pa.fd, pa.path, false); + + path_put(&pa); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_path_remove_directory(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + const char *path, size_t pathlen) +{ + struct path_access pa; + __wasi_errno_t error = + path_get_nofollow(exec_env, curfds, &pa, fd, path, pathlen, + __WASI_RIGHT_PATH_REMOVE_DIRECTORY, 0, true); + if (error != 0) + return error; + + error = os_unlinkat(pa.fd, pa.path, true); + + path_put(&pa); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_poll_oneoff(wasm_exec_env_t exec_env, struct fd_table *curfds, + const __wasi_subscription_t *in, __wasi_event_t *out, + size_t nsubscriptions, + size_t *nevents) NO_LOCK_ANALYSIS +{ +#ifdef BH_PLATFORM_WINDOWS + return __WASI_ENOSYS; +#else + // Sleeping. + if (nsubscriptions == 1 && in[0].u.type == __WASI_EVENTTYPE_CLOCK) { + out[0] = (__wasi_event_t){ + .userdata = in[0].userdata, + .type = in[0].u.type, + }; +#if CONFIG_HAS_CLOCK_NANOSLEEP + clockid_t clock_id; + if (wasi_clockid_to_clockid(in[0].u.u.clock.clock_id, &clock_id)) { + struct timespec ts; + convert_timestamp(in[0].u.u.clock.timeout, &ts); + int ret = clock_nanosleep( + clock_id, + (in[0].u.u.clock.flags & __WASI_SUBSCRIPTION_CLOCK_ABSTIME) != 0 + ? TIMER_ABSTIME + : 0, + &ts, NULL); + if (ret != 0) + out[0].error = convert_errno(ret); + } + else { + out[0].error = __WASI_ENOTSUP; + } +#else + switch (in[0].u.u.clock.clock_id) { + case __WASI_CLOCK_MONOTONIC: + if ((in[0].u.u.clock.flags & __WASI_SUBSCRIPTION_CLOCK_ABSTIME) + != 0) { + // TODO(ed): Implement. + fputs("Unimplemented absolute sleep on monotonic clock\n", + stderr); + out[0].error = __WASI_ENOSYS; + } + else { + // Perform relative sleeps on the monotonic clock also using + // nanosleep(). This is incorrect, but good enough for now. + struct timespec ts; + convert_timestamp(in[0].u.u.clock.timeout, &ts); + nanosleep(&ts, NULL); + } + break; + case __WASI_CLOCK_REALTIME: + if ((in[0].u.u.clock.flags & __WASI_SUBSCRIPTION_CLOCK_ABSTIME) + != 0) { + // Sleeping to an absolute point in time can only be done + // by waiting on a condition variable. + struct mutex mutex; + struct cond cond; + + if (!mutex_init(&mutex)) + return -1; + if (!cond_init_realtime(&cond)) { + mutex_destroy(&mutex); + return -1; + } + mutex_lock(&mutex); + cond_timedwait(&cond, &mutex, in[0].u.u.clock.timeout, + true); + mutex_unlock(&mutex); + mutex_destroy(&mutex); + cond_destroy(&cond); + } + else { + // Relative sleeps can be done using nanosleep(). + struct timespec ts; + convert_timestamp(in[0].u.u.clock.timeout, &ts); + nanosleep(&ts, NULL); + } + break; + default: + out[0].error = __WASI_ENOTSUP; + break; + } +#endif + *nevents = 1; + if (out[0].error != 0) + return convert_errno(out[0].error); + return 0; + } + + // Last option: call into poll(). This can only be done in case all + // subscriptions consist of __WASI_EVENTTYPE_FD_READ and + // __WASI_EVENTTYPE_FD_WRITE entries. There may be up to one + // __WASI_EVENTTYPE_CLOCK entry to act as a timeout. These are also + // the subscriptions generate by cloudlibc's poll() and select(). + struct fd_object **fos = + wasm_runtime_malloc((uint32)(nsubscriptions * sizeof(*fos))); + if (fos == NULL) + return __WASI_ENOMEM; + struct pollfd *pfds = + wasm_runtime_malloc((uint32)(nsubscriptions * sizeof(*pfds))); + if (pfds == NULL) { + wasm_runtime_free(fos); + return __WASI_ENOMEM; + } + + // Convert subscriptions to pollfd entries. Increase the reference + // count on the file descriptors to ensure they remain valid across + // the call to poll(). + struct fd_table *ft = curfds; + rwlock_rdlock(&ft->lock); + *nevents = 0; + const __wasi_subscription_t *clock_subscription = NULL; + for (size_t i = 0; i < nsubscriptions; ++i) { + const __wasi_subscription_t *s = &in[i]; + switch (s->u.type) { + case __WASI_EVENTTYPE_FD_READ: + case __WASI_EVENTTYPE_FD_WRITE: + { + __wasi_errno_t error = + fd_object_get_locked(&fos[i], ft, s->u.u.fd_readwrite.fd, + __WASI_RIGHT_POLL_FD_READWRITE, 0); + if (error == 0) { + // Proper file descriptor on which we can poll(). + pfds[i] = (struct pollfd){ + .fd = fos[i]->file_handle, + .events = s->u.type == __WASI_EVENTTYPE_FD_READ + ? POLLIN + : POLLOUT, + }; + } + else { + // Invalid file descriptor or rights missing. + fos[i] = NULL; + pfds[i] = (struct pollfd){ .fd = -1 }; + out[(*nevents)++] = (__wasi_event_t){ + .userdata = s->userdata, + .error = error, + .type = s->u.type, + }; + } + break; + } + case __WASI_EVENTTYPE_CLOCK: + if (clock_subscription == NULL + && (s->u.u.clock.flags & __WASI_SUBSCRIPTION_CLOCK_ABSTIME) + == 0) { + // Relative timeout. + fos[i] = NULL; + pfds[i] = (struct pollfd){ .fd = -1 }; + clock_subscription = s; + break; + } + // Fallthrough. + default: + // Unsupported event. + fos[i] = NULL; + pfds[i] = (struct pollfd){ .fd = -1 }; + out[(*nevents)++] = (__wasi_event_t){ + .userdata = s->userdata, + .error = __WASI_ENOSYS, + .type = s->u.type, + }; + break; + } + } + rwlock_unlock(&ft->lock); + + // Use a zero-second timeout in case we've already generated events in + // the loop above. + int timeout; + if (*nevents != 0) { + timeout = 0; + } + else if (clock_subscription != NULL) { + __wasi_timestamp_t ts = clock_subscription->u.u.clock.timeout / 1000000; + timeout = ts > INT_MAX ? -1 : (int)ts; + } + else { + timeout = -1; + } + + int ret; + int error = blocking_op_poll(exec_env, pfds, nsubscriptions, timeout, &ret); + if (error != 0) { + /* got an error */ + } + else if (ret == 0 && *nevents == 0 && clock_subscription != NULL) { + // No events triggered. Trigger the clock event. + out[(*nevents)++] = (__wasi_event_t){ + .userdata = clock_subscription->userdata, + .type = __WASI_EVENTTYPE_CLOCK, + }; + } + else { + // Events got triggered. Don't trigger the clock event. + for (size_t i = 0; i < nsubscriptions; ++i) { + if (pfds[i].fd >= 0) { + __wasi_filesize_t nbytes = 0; + if (in[i].u.type == __WASI_EVENTTYPE_FD_READ) { + int l; + if (ioctl(fos[i]->file_handle, FIONREAD, &l) == 0) + nbytes = (__wasi_filesize_t)l; + } + if ((pfds[i].revents & POLLNVAL) != 0) { + // Bad file descriptor. This normally cannot occur, as + // referencing the file descriptor object will always ensure + // the descriptor is valid. Still, macOS may sometimes + // return this on FIFOs when reaching end-of-file. + out[(*nevents)++] = (__wasi_event_t){ + .userdata = in[i].userdata, +#ifdef __APPLE__ + .u.fd_readwrite.nbytes = nbytes, + .u.fd_readwrite.flags = + __WASI_EVENT_FD_READWRITE_HANGUP, +#else + .error = __WASI_EBADF, +#endif + .type = in[i].u.type, + }; + } + else if ((pfds[i].revents & POLLERR) != 0) { + // File descriptor is in an error state. + out[(*nevents)++] = (__wasi_event_t){ + .userdata = in[i].userdata, + .error = __WASI_EIO, + .type = in[i].u.type, + }; + } + else if ((pfds[i].revents & POLLHUP) != 0) { + // End-of-file. + out[(*nevents)++] = (__wasi_event_t){ + .userdata = in[i].userdata, + .type = in[i].u.type, + .u.fd_readwrite.nbytes = nbytes, + .u.fd_readwrite.flags = + __WASI_EVENT_FD_READWRITE_HANGUP, + }; + } + else if ((pfds[i].revents & (POLLIN | POLLOUT)) != 0) { + // Read or write possible. + out[(*nevents)++] = (__wasi_event_t){ + .userdata = in[i].userdata, + .type = in[i].u.type, + .u.fd_readwrite.nbytes = nbytes, + }; + } + } + } + } + + for (size_t i = 0; i < nsubscriptions; ++i) + if (fos[i] != NULL) + fd_object_release(exec_env, fos[i]); + wasm_runtime_free(fos); + wasm_runtime_free(pfds); + return error; +#endif +} + +__wasi_errno_t +wasmtime_ssp_random_get(void *buf, size_t nbyte) +{ + return random_buf(buf, nbyte); +} + +__wasi_errno_t +wasi_ssp_sock_accept(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_fdflags_t flags, + __wasi_fd_t *fd_new) +{ + __wasi_filetype_t wasi_type; + __wasi_rights_t max_base, max_inheriting; + struct fd_object *fo; + bh_socket_t new_sock = os_get_invalid_handle(); + int ret; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_SOCK_ACCEPT, 0); + if (error != __WASI_ESUCCESS) { + goto fail; + } + + ret = blocking_op_socket_accept(exec_env, fo->file_handle, &new_sock, NULL, + NULL); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + error = convert_errno(errno); + goto fail; + } + + error = fd_determine_type_rights(new_sock, &wasi_type, &max_base, + &max_inheriting); + if (error != __WASI_ESUCCESS) { + goto fail; + } + + error = fd_table_insert_fd(exec_env, curfds, new_sock, wasi_type, max_base, + max_inheriting, fd_new); + if (error != __WASI_ESUCCESS) { + /* released in fd_table_insert_fd() */ + new_sock = os_get_invalid_handle(); + goto fail; + } + + return __WASI_ESUCCESS; + +fail: + if (os_is_handle_valid(&new_sock)) { + os_socket_close(new_sock); + } + return error; +} + +__wasi_errno_t +wasi_ssp_sock_addr_local(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_addr_t *addr) +{ + struct fd_object *fo; + bh_sockaddr_t bh_addr; + int ret; + + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_SOCK_ADDR_LOCAL, 0); + if (error != __WASI_ESUCCESS) + return error; + + ret = os_socket_addr_local(fo->file_handle, &bh_addr); + fd_object_release(exec_env, fo); + if (ret != BHT_OK) { + return convert_errno(errno); + } + + bh_sockaddr_to_wasi_addr(&bh_addr, addr); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_addr_remote(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_addr_t *addr) +{ + struct fd_object *fo; + bh_sockaddr_t bh_addr; + int ret; + + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_SOCK_ADDR_LOCAL, 0); + if (error != __WASI_ESUCCESS) + return error; + + ret = os_socket_addr_remote(fo->file_handle, &bh_addr); + fd_object_release(exec_env, fo); + if (ret != BHT_OK) { + return convert_errno(errno); + } + + bh_sockaddr_to_wasi_addr(&bh_addr, addr); + + return __WASI_ESUCCESS; +} + +static bool +wasi_addr_to_string(const __wasi_addr_t *addr, char *buf, size_t buflen) +{ + if (addr->kind == IPv4) { + const char *format = "%u.%u.%u.%u"; + + assert(buflen >= 16); + + snprintf(buf, buflen, format, addr->addr.ip4.addr.n0, + addr->addr.ip4.addr.n1, addr->addr.ip4.addr.n2, + addr->addr.ip4.addr.n3); + + return true; + } + else if (addr->kind == IPv6) { + const char *format = "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x"; + __wasi_addr_ip6_t ipv6 = addr->addr.ip6.addr; + + assert(buflen >= 40); + + snprintf(buf, buflen, format, ipv6.n0, ipv6.n1, ipv6.n2, ipv6.n3, + ipv6.h0, ipv6.h1, ipv6.h2, ipv6.h3); + + return true; + } + + return false; +} + +__wasi_errno_t +wasi_ssp_sock_bind(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct addr_pool *addr_pool, __wasi_fd_t fd, + __wasi_addr_t *addr) +{ + char buf[48] = { 0 }; + struct fd_object *fo; + __wasi_errno_t error; + int port = addr->kind == IPv4 ? addr->addr.ip4.port : addr->addr.ip6.port; + int ret; + + if (!wasi_addr_to_string(addr, buf, sizeof(buf))) { + return __WASI_EPROTONOSUPPORT; + } + + if (!addr_pool_search(addr_pool, buf)) { + return __WASI_EACCES; + } + + error = fd_object_get(curfds, &fo, fd, __WASI_RIGHT_SOCK_BIND, 0); + if (error != __WASI_ESUCCESS) + return error; + + ret = os_socket_bind(fo->file_handle, buf, &port); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_addr_resolve(wasm_exec_env_t exec_env, struct fd_table *curfds, + char **ns_lookup_list, const char *host, + const char *service, __wasi_addr_info_hints_t *hints, + __wasi_addr_info_t *addr_info, + __wasi_size_t addr_info_size, + __wasi_size_t *max_info_size) +{ + bh_addr_info_t *wamr_addr_info = + wasm_runtime_malloc(addr_info_size * sizeof(bh_addr_info_t)); + uint8_t hints_is_ipv4 = hints->family == INET4; + uint8_t hints_is_tcp = hints->type == SOCKET_STREAM; + size_t _max_info_size; + size_t actual_info_size; + + if (!wamr_addr_info) { + return __WASI_ENOMEM; + } + + if (!ns_lookup_list_search(ns_lookup_list, host)) { + wasm_runtime_free(wamr_addr_info); + return __WASI_EACCES; + } + + int ret = blocking_op_socket_addr_resolve( + exec_env, host, service, + hints->hints_enabled && hints->type != SOCKET_ANY ? &hints_is_tcp + : NULL, + hints->hints_enabled && hints->family != INET_UNSPEC ? &hints_is_ipv4 + : NULL, + wamr_addr_info, addr_info_size, &_max_info_size); + + if (ret != BHT_OK) { + wasm_runtime_free(wamr_addr_info); + return convert_errno(errno); + } + + *max_info_size = _max_info_size; + actual_info_size = + addr_info_size < *max_info_size ? addr_info_size : *max_info_size; + + for (size_t i = 0; i < actual_info_size; i++) { + addr_info[i].type = + wamr_addr_info[i].is_tcp ? SOCKET_STREAM : SOCKET_DGRAM; + bh_sockaddr_to_wasi_addr(&wamr_addr_info[i].sockaddr, + &addr_info[i].addr); + } + + wasm_runtime_free(wamr_addr_info); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_connect(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct addr_pool *addr_pool, __wasi_fd_t fd, + __wasi_addr_t *addr) +{ + char buf[48] = { 0 }; + struct fd_object *fo; + __wasi_errno_t error; + int ret; + + if (!wasi_addr_to_string(addr, buf, sizeof(buf))) { + return __WASI_EPROTONOSUPPORT; + } + + if (!addr_pool_search(addr_pool, buf)) { + return __WASI_EACCES; + } + + error = fd_object_get(curfds, &fo, fd, __WASI_RIGHT_SOCK_BIND, 0); + if (error != __WASI_ESUCCESS) + return error; + + ret = blocking_op_socket_connect(exec_env, fo->file_handle, buf, + addr->kind == IPv4 ? addr->addr.ip4.port + : addr->addr.ip6.port); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_get_recv_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t *size) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + size_t bufsize = 0; + int ret = os_socket_get_recv_buf_size(fo->file_handle, &bufsize); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + *size = (__wasi_size_t)bufsize; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_get_reuse_addr(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t *reuse) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + bool enabled = false; + + int ret = os_socket_get_reuse_addr(fo->file_handle, &enabled); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + *reuse = (uint8_t)enabled; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_get_reuse_port(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t *reuse) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + bool enabled = false; + int ret = os_socket_get_reuse_port(fo->file_handle, &enabled); + + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + *reuse = (uint8_t)enabled; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_get_send_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t *size) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + size_t bufsize = 0; + int ret = os_socket_get_send_buf_size(fo->file_handle, &bufsize); + + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + *size = (__wasi_size_t)bufsize; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_listen(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, __wasi_size_t backlog) +{ + struct fd_object *fo; + int ret; + __wasi_errno_t error = + fd_object_get(curfds, &fo, fd, __WASI_RIGHT_SOCK_LISTEN, 0); + if (error != __WASI_ESUCCESS) + return error; + + ret = os_socket_listen(fo->file_handle, backlog); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_open(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t poolfd, __wasi_address_family_t af, + __wasi_sock_type_t socktype, __wasi_fd_t *sockfd) +{ + bh_socket_t sock; + bool is_tcp = SOCKET_DGRAM == socktype ? false : true; + bool is_ipv4 = INET6 == af ? false : true; + int ret; + __wasi_filetype_t wasi_type = __WASI_FILETYPE_UNKNOWN; + __wasi_rights_t max_base = 0, max_inheriting = 0; + __wasi_errno_t error; + + (void)poolfd; + + ret = os_socket_create(&sock, is_ipv4, is_tcp); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + error = + fd_determine_type_rights(sock, &wasi_type, &max_base, &max_inheriting); + if (error != __WASI_ESUCCESS) { + os_socket_close(sock); + return error; + } + + if (SOCKET_DGRAM == socktype) { + assert(wasi_type == __WASI_FILETYPE_SOCKET_DGRAM); + } + else { + assert(wasi_type == __WASI_FILETYPE_SOCKET_STREAM); + } + + // TODO: base rights and inheriting rights ? + error = fd_table_insert_fd(exec_env, curfds, sock, wasi_type, max_base, + max_inheriting, sockfd); + if (error != __WASI_ESUCCESS) { + return error; + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_set_recv_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t size) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + int ret = os_socket_set_recv_buf_size(fo->file_handle, size); + + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_set_reuse_addr(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t reuse) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + int ret = os_socket_set_reuse_addr(fo->file_handle, (bool)reuse); + + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_set_reuse_port(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t fd, uint8_t reuse) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + int ret = os_socket_set_reuse_port(fo->file_handle, (bool)reuse); + + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasi_ssp_sock_set_send_buf_size(wasm_exec_env_t exec_env, + struct fd_table *curfds, __wasi_fd_t fd, + __wasi_size_t size) +{ + struct fd_object *fo; + __wasi_errno_t error = fd_object_get(curfds, &fo, fd, 0, 0); + if (error != __WASI_ESUCCESS) + return error; + + int ret = os_socket_set_send_buf_size(fo->file_handle, size); + + fd_object_release(exec_env, fo); + if (BHT_OK != ret) { + return convert_errno(errno); + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_recv(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, void *buf, size_t buf_len, + size_t *recv_len) +{ + __wasi_addr_t src_addr; + + return wasmtime_ssp_sock_recv_from(exec_env, curfds, sock, buf, buf_len, 0, + &src_addr, recv_len); +} + +__wasi_errno_t +wasmtime_ssp_sock_recv_from(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, void *buf, size_t buf_len, + __wasi_riflags_t ri_flags, __wasi_addr_t *src_addr, + size_t *recv_len) +{ + struct fd_object *fo; + __wasi_errno_t error; + bh_sockaddr_t sockaddr; + int ret; + + error = fd_object_get(curfds, &fo, sock, __WASI_RIGHT_FD_READ, 0); + if (error != 0) { + return error; + } + + ret = blocking_op_socket_recv_from(exec_env, fo->file_handle, buf, buf_len, + 0, &sockaddr); + fd_object_release(exec_env, fo); + if (-1 == ret) { + return convert_errno(errno); + } + + bh_sockaddr_to_wasi_addr(&sockaddr, src_addr); + + *recv_len = (size_t)ret; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_send(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, const void *buf, size_t buf_len, + size_t *sent_len) +{ + struct fd_object *fo; + __wasi_errno_t error; + int ret; + + error = fd_object_get(curfds, &fo, sock, __WASI_RIGHT_FD_WRITE, 0); + if (error != 0) { + return error; + } + + ret = os_socket_send(fo->file_handle, buf, buf_len); + fd_object_release(exec_env, fo); + if (-1 == ret) { + return convert_errno(errno); + } + + *sent_len = (size_t)ret; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_send_to(wasm_exec_env_t exec_env, struct fd_table *curfds, + struct addr_pool *addr_pool, __wasi_fd_t sock, + const void *buf, size_t buf_len, + __wasi_siflags_t si_flags, + const __wasi_addr_t *dest_addr, size_t *sent_len) +{ + char addr_buf[48] = { 0 }; + struct fd_object *fo; + __wasi_errno_t error; + int ret; + bh_sockaddr_t sockaddr; + + if (!wasi_addr_to_string(dest_addr, addr_buf, sizeof(addr_buf))) { + return __WASI_EPROTONOSUPPORT; + } + + if (!addr_pool_search(addr_pool, addr_buf)) { + return __WASI_EACCES; + } + + error = fd_object_get(curfds, &fo, sock, __WASI_RIGHT_FD_WRITE, 0); + if (error != 0) { + return error; + } + + wasi_addr_to_bh_sockaddr(dest_addr, &sockaddr); + + ret = blocking_op_socket_send_to(exec_env, fo->file_handle, buf, buf_len, 0, + &sockaddr); + fd_object_release(exec_env, fo); + if (-1 == ret) { + return convert_errno(errno); + } + + *sent_len = (size_t)ret; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_shutdown(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock) +{ + struct fd_object *fo; + __wasi_errno_t error; + + error = fd_object_get(curfds, &fo, sock, 0, 0); + if (error != 0) + return error; + + error = os_socket_shutdown(fo->file_handle); + fd_object_release(exec_env, fo); + + return error; +} + +__wasi_errno_t +wasmtime_ssp_sched_yield(void) +{ +#ifdef BH_PLATFORM_WINDOWS + SwitchToThread(); +#else + if (sched_yield() < 0) + return convert_errno(errno); +#endif + return 0; +} + +__wasi_errno_t +wasmtime_ssp_args_get(struct argv_environ_values *argv_environ, char **argv, + char *argv_buf) +{ + for (size_t i = 0; i < argv_environ->argc; ++i) { + argv[i] = + argv_buf + (argv_environ->argv_list[i] - argv_environ->argv_buf); + } + argv[argv_environ->argc] = NULL; + bh_memcpy_s(argv_buf, (uint32)argv_environ->argv_buf_size, + argv_environ->argv_buf, (uint32)argv_environ->argv_buf_size); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_args_sizes_get(struct argv_environ_values *argv_environ, + size_t *argc, size_t *argv_buf_size) +{ + *argc = argv_environ->argc; + *argv_buf_size = argv_environ->argv_buf_size; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_environ_get(struct argv_environ_values *argv_environ, + char **environs, char *environ_buf) +{ + for (size_t i = 0; i < argv_environ->environ_count; ++i) { + environs[i] = + environ_buf + + (argv_environ->environ_list[i] - argv_environ->environ_buf); + } + environs[argv_environ->environ_count] = NULL; + bh_memcpy_s(environ_buf, (uint32)argv_environ->environ_buf_size, + argv_environ->environ_buf, + (uint32)argv_environ->environ_buf_size); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_environ_sizes_get(struct argv_environ_values *argv_environ, + size_t *environ_count, size_t *environ_buf_size) +{ + *environ_count = argv_environ->environ_count; + *environ_buf_size = argv_environ->environ_buf_size; + return __WASI_ESUCCESS; +} + +bool +argv_environ_init(struct argv_environ_values *argv_environ, char *argv_buf, + size_t argv_buf_size, char **argv_list, size_t argc, + char *environ_buf, size_t environ_buf_size, + char **environ_list, size_t environ_count) +{ + memset(argv_environ, 0, sizeof(struct argv_environ_values)); + + argv_environ->argv_buf = argv_buf; + argv_environ->argv_buf_size = argv_buf_size; + argv_environ->argv_list = argv_list; + argv_environ->argc = argc; + argv_environ->environ_buf = environ_buf; + argv_environ->environ_buf_size = environ_buf_size; + argv_environ->environ_list = environ_list; + argv_environ->environ_count = environ_count; + return true; +} + +void +argv_environ_destroy(struct argv_environ_values *argv_environ) +{ + (void)argv_environ; +} + +void +fd_table_destroy(struct fd_table *ft) +{ + if (ft->entries) { + for (uint32 i = 0; i < ft->size; i++) { + if (ft->entries[i].object != NULL) { + fd_object_release(NULL, ft->entries[i].object); + } + } + rwlock_destroy(&ft->lock); + wasm_runtime_free(ft->entries); + } +} + +void +fd_prestats_destroy(struct fd_prestats *pt) +{ + if (pt->prestats) { + for (uint32 i = 0; i < pt->size; i++) { + if (pt->prestats[i].dir != NULL) { + wasm_runtime_free((void *)pt->prestats[i].dir); + } + } + rwlock_destroy(&pt->lock); + wasm_runtime_free(pt->prestats); + } +} + +bool +addr_pool_init(struct addr_pool *addr_pool) +{ + memset(addr_pool, 0, sizeof(*addr_pool)); + + return true; +} + +bool +addr_pool_insert(struct addr_pool *addr_pool, const char *addr, uint8 mask) +{ + struct addr_pool *cur = addr_pool; + struct addr_pool *next; + bh_ip_addr_buffer_t target; + + if (!addr_pool) { + return false; + } + + if (!(next = wasm_runtime_malloc(sizeof(struct addr_pool)))) { + return false; + } + + next->next = NULL; + next->mask = mask; + + if (os_socket_inet_network(true, addr, &target) != BHT_OK) { + // If parsing IPv4 fails, try IPv6 + if (os_socket_inet_network(false, addr, &target) != BHT_OK) { + wasm_runtime_free(next); + return false; + } + next->type = IPv6; + bh_memcpy_s(next->addr.ip6, sizeof(next->addr.ip6), target.ipv6, + sizeof(target.ipv6)); + } + else { + next->type = IPv4; + next->addr.ip4 = target.ipv4; + } + + /* attach with */ + while (cur->next) { + cur = cur->next; + } + cur->next = next; + return true; +} + +static void +init_address_mask(uint8_t *buf, size_t buflen, size_t mask) +{ + size_t element_size = sizeof(uint8_t) * 8; + + for (size_t i = 0; i < buflen; i++) { + if (mask <= i * element_size) { + buf[i] = 0; + } + else { + size_t offset = min(mask - i * element_size, element_size); + buf[i] = (~0u) << (element_size - offset); + } + } +} + +/* target must be in network byte order */ +static bool +compare_address(const struct addr_pool *addr_pool_entry, + bh_ip_addr_buffer_t *target) +{ + uint8_t maskbuf[16] = { 0 }; + uint8_t basebuf[16] = { 0 }; + size_t addr_size; + uint8_t max_addr_mask; + + if (addr_pool_entry->type == IPv4) { + uint32_t addr_ip4 = htonl(addr_pool_entry->addr.ip4); + bh_memcpy_s(basebuf, sizeof(addr_ip4), &addr_ip4, sizeof(addr_ip4)); + addr_size = 4; + } + else { + uint16_t partial_addr_ip6; + for (int i = 0; i < 8; i++) { + partial_addr_ip6 = htons(addr_pool_entry->addr.ip6[i]); + bh_memcpy_s(&basebuf[i * sizeof(partial_addr_ip6)], + sizeof(partial_addr_ip6), &partial_addr_ip6, + sizeof(partial_addr_ip6)); + } + addr_size = 16; + } + max_addr_mask = addr_size * 8; + + /* IPv4 0.0.0.0 or IPv6 :: means any address */ + if (basebuf[0] == 0 && !memcmp(basebuf, basebuf + 1, addr_size - 1)) { + return true; + } + + /* No support for invalid mask value */ + if (addr_pool_entry->mask > max_addr_mask) { + return false; + } + + init_address_mask(maskbuf, addr_size, addr_pool_entry->mask); + + for (size_t i = 0; i < addr_size; i++) { + uint8_t addr_mask = target->data[i] & maskbuf[i]; + uint8_t range_mask = basebuf[i] & maskbuf[i]; + if (addr_mask != range_mask) { + return false; + } + } + + return true; +} + +bool +addr_pool_search(struct addr_pool *addr_pool, const char *addr) +{ + struct addr_pool *cur = addr_pool->next; + bh_ip_addr_buffer_t target; + __wasi_addr_type_t addr_type; + + if (os_socket_inet_network(true, addr, &target) != BHT_OK) { + size_t i; + + if (os_socket_inet_network(false, addr, &target) != BHT_OK) { + return false; + } + addr_type = IPv6; + for (i = 0; i < sizeof(target.ipv6) / sizeof(target.ipv6[0]); i++) { + target.ipv6[i] = htons(target.ipv6[i]); + } + } + else { + addr_type = IPv4; + target.ipv4 = htonl(target.ipv4); + } + + while (cur) { + if (cur->type == addr_type && compare_address(cur, &target)) { + return true; + } + + cur = cur->next; + } + + return false; +} + +void +addr_pool_destroy(struct addr_pool *addr_pool) +{ + struct addr_pool *cur = addr_pool->next; + + while (cur) { + struct addr_pool *next = cur->next; + wasm_runtime_free(cur); + cur = next; + } +} + +#define WASMTIME_SSP_PASSTHROUGH_FD_TABLE struct fd_table *curfds, + +// Defines a function that passes through the socket option to the OS +// implementation +#define WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(FUNC_NAME, OPTION_TYPE) \ + __wasi_errno_t wasmtime_ssp_sock_##FUNC_NAME( \ + wasm_exec_env_t exec_env, \ + WASMTIME_SSP_PASSTHROUGH_FD_TABLE __wasi_fd_t sock, \ + OPTION_TYPE option) \ + { \ + struct fd_object *fo; \ + __wasi_errno_t error; \ + int ret; \ + error = fd_object_get(curfds, &fo, sock, 0, 0); \ + if (error != 0) \ + return error; \ + ret = os_socket_##FUNC_NAME(fo->file_handle, option); \ + fd_object_release(exec_env, fo); \ + if (BHT_OK != ret) \ + return convert_errno(errno); \ + return __WASI_ESUCCESS; \ + } + +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_send_timeout, uint64) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_send_timeout, uint64 *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_recv_timeout, uint64) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_recv_timeout, uint64 *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_send_buf_size, size_t) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_send_buf_size, size_t *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_recv_buf_size, size_t) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_recv_buf_size, size_t *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_broadcast, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_broadcast, bool *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_keep_alive, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_keep_alive, bool *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_reuse_addr, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_reuse_addr, bool *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_reuse_port, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_reuse_port, bool *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_tcp_no_delay, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_tcp_no_delay, bool *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_tcp_quick_ack, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_tcp_quick_ack, bool *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_tcp_keep_idle, uint32) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_tcp_keep_idle, uint32 *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_tcp_keep_intvl, uint32) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_tcp_keep_intvl, uint32 *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_tcp_fastopen_connect, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_tcp_fastopen_connect, bool *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_ip_ttl, uint8_t) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_ip_ttl, uint8_t *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_ip_multicast_ttl, uint8_t) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_ip_multicast_ttl, uint8_t *) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(set_ipv6_only, bool) +WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION(get_ipv6_only, bool *) + +#undef WASMTIME_SSP_PASSTHROUGH_FD_TABLE +#undef WASMTIME_SSP_PASSTHROUGH_SOCKET_OPTION + +__wasi_errno_t +wasmtime_ssp_sock_set_linger(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, bool is_enabled, int linger_s) +{ + struct fd_object *fo; + __wasi_errno_t error; + int ret; + error = fd_object_get(curfds, &fo, sock, 0, 0); + if (error != 0) + return error; + + ret = os_socket_set_linger(fo->file_handle, is_enabled, linger_s); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) + return convert_errno(errno); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_get_linger(wasm_exec_env_t exec_env, struct fd_table *curfds, + __wasi_fd_t sock, bool *is_enabled, int *linger_s) +{ + struct fd_object *fo; + __wasi_errno_t error; + int ret; + error = fd_object_get(curfds, &fo, sock, 0, 0); + if (error != 0) + return error; + + ret = os_socket_get_linger(fo->file_handle, is_enabled, linger_s); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_add_membership(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) +{ + struct fd_object *fo; + __wasi_errno_t error; + int ret; + bh_ip_addr_buffer_t addr_info; + bool is_ipv6; + error = fd_object_get(curfds, &fo, sock, 0, 0); + if (error != 0) + return error; + + wasi_addr_ip_to_bh_ip_addr_buffer(imr_multiaddr, &addr_info); + is_ipv6 = imr_multiaddr->kind == IPv6; + ret = os_socket_set_ip_add_membership(fo->file_handle, &addr_info, + imr_interface, is_ipv6); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) + return convert_errno(errno); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_drop_membership(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, + __wasi_addr_ip_t *imr_multiaddr, + uint32_t imr_interface) +{ + struct fd_object *fo; + __wasi_errno_t error; + int ret; + bh_ip_addr_buffer_t addr_info; + bool is_ipv6; + error = fd_object_get(curfds, &fo, sock, 0, 0); + if (error != 0) + return error; + + wasi_addr_ip_to_bh_ip_addr_buffer(imr_multiaddr, &addr_info); + is_ipv6 = imr_multiaddr->kind == IPv6; + ret = os_socket_set_ip_drop_membership(fo->file_handle, &addr_info, + imr_interface, is_ipv6); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) + return convert_errno(errno); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_set_ip_multicast_loop(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, bool ipv6, + bool is_enabled) +{ + struct fd_object *fo; + __wasi_errno_t error; + int ret; + error = fd_object_get(curfds, &fo, sock, 0, 0); + if (error != 0) + return error; + + ret = os_socket_set_ip_multicast_loop(fo->file_handle, ipv6, is_enabled); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) + return convert_errno(errno); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +wasmtime_ssp_sock_get_ip_multicast_loop(wasm_exec_env_t exec_env, + struct fd_table *curfds, + __wasi_fd_t sock, bool ipv6, + bool *is_enabled) +{ + struct fd_object *fo; + __wasi_errno_t error; + int ret; + error = fd_object_get(curfds, &fo, sock, 0, 0); + if (error != 0) + return error; + + ret = os_socket_get_ip_multicast_loop(fo->file_handle, ipv6, is_enabled); + fd_object_release(exec_env, fo); + if (BHT_OK != ret) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.h new file mode 100644 index 0000000..75ed597 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.h @@ -0,0 +1,90 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016-2018 Nuxi, https://nuxi.nl/ + +#ifndef POSIX_H +#define POSIX_H + +#include "bh_platform.h" +#include "locking.h" + +struct fd_entry; +struct fd_prestat; +struct syscalls; + +struct fd_table { + struct rwlock lock; + struct fd_entry *entries; + size_t size; + size_t used; +}; + +struct fd_prestats { + struct rwlock lock; + struct fd_prestat *prestats; + size_t size; + size_t used; +}; + +struct argv_environ_values { + const char *argv_buf; + size_t argv_buf_size; + char **argv_list; + size_t argc; + char *environ_buf; + size_t environ_buf_size; + char **environ_list; + size_t environ_count; +}; + +struct addr_pool { + /* addr and mask in host order */ + union { + uint32 ip4; + uint16 ip6[8]; + } addr; + struct addr_pool *next; + __wasi_addr_type_t type; + uint8 mask; +}; + +bool +fd_table_init(struct fd_table *); +bool +fd_table_insert_existing(struct fd_table *, __wasi_fd_t, os_file_handle, + bool is_stdio); +bool +fd_prestats_init(struct fd_prestats *); +bool +fd_prestats_insert(struct fd_prestats *, const char *, __wasi_fd_t); +bool +argv_environ_init(struct argv_environ_values *argv_environ, char *argv_buf, + size_t argv_buf_size, char **argv_list, size_t argc, + char *environ_buf, size_t environ_buf_size, + char **environ_list, size_t environ_count); +void +argv_environ_destroy(struct argv_environ_values *argv_environ); +void +fd_table_destroy(struct fd_table *ft); +void +fd_prestats_destroy(struct fd_prestats *pt); + +bool +addr_pool_init(struct addr_pool *); +bool +addr_pool_insert(struct addr_pool *, const char *, uint8 mask); +bool +addr_pool_search(struct addr_pool *, const char *); +void +addr_pool_destroy(struct addr_pool *); + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/queue.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/queue.h new file mode 100644 index 0000000..2d40bc3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/queue.h @@ -0,0 +1,98 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#ifndef QUEUE_H +#define QUEUE_H + +// LIST: Double-linked list. + +#define LIST_HEAD(name, type) \ + struct name { \ + struct type *l_first; \ + } + +/* clang-format off */ +#define LIST_HEAD_INITIALIZER(head) \ + { NULL } +/* clang-format on */ + +#define LIST_ENTRY(type) \ + struct { \ + struct type *l_next; \ + struct type **l_prev; \ + } + +#define LIST_FOREACH(var, head, field) \ + for ((var) = (head)->l_first; (var) != NULL; (var) = (var)->field.l_next) + +#define LIST_INIT(head) \ + do { \ + (head)->l_first = NULL; \ + } while (0) + +#define LIST_INSERT_HEAD(head, element, field) \ + do { \ + (element)->field.l_next = (head)->l_first; \ + if ((head)->l_first != NULL) \ + (head)->l_first->field.l_prev = &(element)->field.l_next; \ + (head)->l_first = (element); \ + (element)->field.l_prev = &(head)->l_first; \ + } while (0) + +#define LIST_REMOVE(element, field) \ + do { \ + if ((element)->field.l_next != NULL) \ + (element)->field.l_next->field.l_prev = (element)->field.l_prev; \ + *(element)->field.l_prev = (element)->field.l_next; \ + } while (0) + +// TAILQ: Double-linked list with tail pointer. + +#define TAILQ_HEAD(name, type) \ + struct name { \ + struct type *t_first; \ + struct type **t_last; \ + } + +#define TAILQ_ENTRY(type) \ + struct { \ + struct type *t_next; \ + struct type **t_prev; \ + } + +#define TAILQ_EMPTY(head) ((head)->t_first == NULL) +#define TAILQ_FIRST(head) ((head)->t_first) +#define TAILQ_FOREACH(var, head, field) \ + for ((var) = (head)->t_first; (var) != NULL; (var) = (var)->field.t_next) +#define TAILQ_INIT(head) \ + do { \ + (head)->t_first = NULL; \ + (head)->t_last = &(head)->t_first; \ + } while (0) +#define TAILQ_INSERT_TAIL(head, elm, field) \ + do { \ + (elm)->field.t_next = NULL; \ + (elm)->field.t_prev = (head)->t_last; \ + *(head)->t_last = (elm); \ + (head)->t_last = &(elm)->field.t_next; \ + } while (0) +#define TAILQ_REMOVE(head, element, field) \ + do { \ + if ((element)->field.t_next != NULL) \ + (element)->field.t_next->field.t_prev = (element)->field.t_prev; \ + else \ + (head)->t_last = (element)->field.t_prev; \ + *(element)->field.t_prev = (element)->field.t_next; \ + } while (0) + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.c b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.c new file mode 100644 index 0000000..29c50dd --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.c @@ -0,0 +1,137 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#include "ssp_config.h" +#include "bh_platform.h" +#include "libc_errno.h" +#include "random.h" + +#if CONFIG_HAS_ARC4RANDOM_BUF + +__wasi_errno_t +random_buf(void *buf, size_t len) +{ + arc4random_buf(buf, len); + return __WASI_ESUCCESS; +} + +#elif CONFIG_HAS_GETRANDOM + +#ifndef BH_PLATFORM_LINUX_SGX +#include +#endif + +__wasi_errno_t +random_buf(void *buf, size_t len) +{ + for (;;) { + ssize_t x = getrandom(buf, len, 0); + if (x < 0) { + if (errno == EINTR) + continue; + return convert_errno(errno); + } + if ((size_t)x == len) + break; + buf = (void *)((unsigned char *)buf + x); + len -= (size_t)x; + } + return __WASI_ESUCCESS; +} + +#elif defined(BH_PLATFORM_WINDOWS) + +#include +#pragma comment(lib, "Bcrypt.lib") + +__wasi_errno_t +random_buf(void *buf, size_t len) +{ + NTSTATUS ret = + BCryptGenRandom(NULL, buf, (ULONG)len, BCRYPT_USE_SYSTEM_PREFERRED_RNG); + + // Since we pass NULL for the algorithm handle, the only way BCryptGenRandom + // can fail is if one of the parameters is invalid + // (STATUS_INVALID_PARAMETER). + return ret ? __WASI_EINVAL : __WASI_ESUCCESS; +} + +#else + +static int urandom = -1; +static __wasi_errno_t urandom_error = __WASI_ESUCCESS; + +static void +open_urandom(void) +{ + urandom = open("/dev/urandom", O_RDONLY); + if (urandom < 0) + urandom_error = convert_errno(errno); +} + +__wasi_errno_t +random_buf(void *buf, size_t len) +{ + static pthread_once_t open_once = PTHREAD_ONCE_INIT; + int pthread_ret = pthread_once(&open_once, open_urandom); + + if (pthread_ret != 0) + return convert_errno(pthread_ret); + + if (urandom < 0) + return urandom_error; + + size_t bytes_read = 0; + + while (bytes_read < len) { + ssize_t bytes_read_now = + read(urandom, buf + bytes_read, len - bytes_read); + + if (bytes_read_now < 0) + return convert_errno(errno); + + bytes_read += (size_t)bytes_read_now; + } + + return __WASI_ESUCCESS; +} + +#endif + +// Calculates a random number within the range [0, upper - 1] without +// any modulo bias. +// +// The function below repeatedly obtains a random number from +// arc4random() until it lies within the range [2^k % upper, 2^k). As +// this range has length k * upper, we can safely obtain a number +// without any modulo bias. +__wasi_errno_t +random_uniform(uintmax_t upper, uintmax_t *out) +{ + // Compute 2^k % upper + // == (2^k - upper) % upper + // == -upper % upper. + uintmax_t lower = -upper % upper; + for (;;) { + uintmax_t value; + __wasi_errno_t error = random_buf(&value, sizeof(value)); + + if (error != __WASI_ESUCCESS) + return error; + + if (value >= lower) { + *out = value % upper; + return error; + } + } +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.h new file mode 100644 index 0000000..7cd94d7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.h @@ -0,0 +1,25 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#ifndef RANDOM_H +#define RANDOM_H + +#include "bh_platform.h" + +__wasi_errno_t +random_buf(void *, size_t); + +__wasi_errno_t +random_uniform(uintmax_t upper, uintmax_t *out); + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/refcount.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/refcount.h new file mode 100644 index 0000000..03b4b87 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/refcount.h @@ -0,0 +1,139 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#ifndef REFCOUNT_H +#define REFCOUNT_H + +#include "bh_platform.h" +#include "locking.h" +#include "gnuc.h" + +#define PRODUCES(...) LOCKS_SHARED(__VA_ARGS__) NO_LOCK_ANALYSIS +#define CONSUMES(...) UNLOCKS(__VA_ARGS__) NO_LOCK_ANALYSIS + +#if CONFIG_HAS_STD_ATOMIC != 0 + +#include + +/* Simple reference counter. */ +struct LOCKABLE refcount { + atomic_uint count; +}; + +/* Initialize the reference counter. */ +static inline void +refcount_init(struct refcount *r, unsigned int count) PRODUCES(*r) +{ + atomic_init(&r->count, count); +} + +/* Increment the reference counter. */ +static inline void +refcount_acquire(struct refcount *r) PRODUCES(*r) +{ + atomic_fetch_add_explicit(&r->count, 1, memory_order_acquire); +} + +/* Decrement the reference counter, returning whether the reference + dropped to zero. */ +static inline bool +refcount_release(struct refcount *r) CONSUMES(*r) +{ + int old = + (int)atomic_fetch_sub_explicit(&r->count, 1, memory_order_release); + bh_assert(old != 0 && "Reference count becoming negative"); + return old == 1; +} + +#elif defined(BH_PLATFORM_LINUX_SGX) + +#include + +/* Simple reference counter. */ +struct refcount { + sgx_spinlock_t lock; + unsigned int count; +}; + +/* Initialize the reference counter. */ +static inline void +refcount_init(struct refcount *r, unsigned int count) +{ + r->lock = SGX_SPINLOCK_INITIALIZER; + r->count = count; +} + +/* Increment the reference counter. */ +static inline void +refcount_acquire(struct refcount *r) +{ + sgx_spin_lock(&r->lock); + r->count++; + sgx_spin_unlock(&r->lock); +} + +/* Decrement the reference counter, returning whether the reference + dropped to zero. */ +static inline bool +refcount_release(struct refcount *r) +{ + int old; + sgx_spin_lock(&r->lock); + old = (int)r->count; + r->count--; + sgx_spin_unlock(&r->lock); + bh_assert(old != 0 && "Reference count becoming negative"); + return old == 1; +} + +#elif defined(__GNUC_PREREQ) + +#if __GNUC_PREREQ(4, 7) + +struct refcount { + unsigned int count; +}; + +/* Initialize the reference counter. */ +static inline void +refcount_init(struct refcount *r, unsigned int count) +{ + __atomic_store_n(&r->count, count, __ATOMIC_SEQ_CST); +} + +/* Increment the reference counter. */ +static inline void +refcount_acquire(struct refcount *r) +{ + __atomic_fetch_add(&r->count, 1, __ATOMIC_ACQUIRE); +} + +/* Decrement the reference counter, returning whether the reference + dropped to zero. */ +static inline bool +refcount_release(struct refcount *r) +{ + int old = (int)__atomic_fetch_sub(&r->count, 1, __ATOMIC_RELEASE); + bh_assert(old != 0 && "Reference count becoming negative"); + return old == 1; +} + +#else /* else of __GNUC_PREREQ (4.7) */ +#error "Reference counter isn't implemented" +#endif /* end of __GNUC_PREREQ (4.7) */ + +#else /* else of CONFIG_HAS_STD_ATOMIC */ +#error "Reference counter isn't implemented" +#endif /* end of CONFIG_HAS_STD_ATOMIC */ + +#endif /* end of REFCOUNT_H */ diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/rights.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/rights.h new file mode 100644 index 0000000..4f58381 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/rights.h @@ -0,0 +1,100 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#ifndef RIGHTS_H +#define RIGHTS_H + +/* clang-format off */ + +#define RIGHTS_ALL \ + (__WASI_RIGHT_FD_DATASYNC | __WASI_RIGHT_FD_READ | \ + __WASI_RIGHT_FD_SEEK | __WASI_RIGHT_FD_FDSTAT_SET_FLAGS | \ + __WASI_RIGHT_FD_SYNC | __WASI_RIGHT_FD_TELL | __WASI_RIGHT_FD_WRITE | \ + __WASI_RIGHT_FD_ADVISE | __WASI_RIGHT_FD_ALLOCATE | \ + __WASI_RIGHT_PATH_CREATE_DIRECTORY | __WASI_RIGHT_PATH_CREATE_FILE | \ + __WASI_RIGHT_PATH_LINK_SOURCE | __WASI_RIGHT_PATH_LINK_TARGET | \ + __WASI_RIGHT_PATH_OPEN | __WASI_RIGHT_FD_READDIR | \ + __WASI_RIGHT_PATH_READLINK | __WASI_RIGHT_PATH_RENAME_SOURCE | \ + __WASI_RIGHT_PATH_RENAME_TARGET | __WASI_RIGHT_PATH_FILESTAT_GET | \ + __WASI_RIGHT_PATH_FILESTAT_SET_SIZE | \ + __WASI_RIGHT_PATH_FILESTAT_SET_TIMES | \ + __WASI_RIGHT_FD_FILESTAT_GET | __WASI_RIGHT_FD_FILESTAT_SET_TIMES | \ + __WASI_RIGHT_FD_FILESTAT_SET_SIZE | \ + __WASI_RIGHT_PATH_SYMLINK | __WASI_RIGHT_PATH_UNLINK_FILE | \ + __WASI_RIGHT_PATH_REMOVE_DIRECTORY | \ + __WASI_RIGHT_POLL_FD_READWRITE | __WASI_RIGHT_SOCK_CONNECT | \ + __WASI_RIGHT_SOCK_LISTEN | __WASI_RIGHT_SOCK_BIND | \ + __WASI_RIGHT_SOCK_ACCEPT | __WASI_RIGHT_SOCK_RECV | \ + __WASI_RIGHT_SOCK_SEND | __WASI_RIGHT_SOCK_ADDR_LOCAL | \ + __WASI_RIGHT_SOCK_ADDR_REMOTE | __WASI_RIGHT_SOCK_RECV_FROM | \ + __WASI_RIGHT_SOCK_SEND_TO) + + +// Block and character device interaction is outside the scope of +// CloudABI. Simply allow everything. +#define RIGHTS_BLOCK_DEVICE_BASE RIGHTS_ALL +#define RIGHTS_BLOCK_DEVICE_INHERITING RIGHTS_ALL +#define RIGHTS_CHARACTER_DEVICE_BASE RIGHTS_ALL +#define RIGHTS_CHARACTER_DEVICE_INHERITING RIGHTS_ALL + +// Only allow directory operations on directories. Directories can only +// yield file descriptors to other directories and files. +#define RIGHTS_DIRECTORY_BASE \ + (__WASI_RIGHT_FD_FDSTAT_SET_FLAGS | __WASI_RIGHT_FD_SYNC | \ + __WASI_RIGHT_FD_ADVISE | __WASI_RIGHT_PATH_CREATE_DIRECTORY | \ + __WASI_RIGHT_PATH_CREATE_FILE | __WASI_RIGHT_PATH_LINK_SOURCE | \ + __WASI_RIGHT_PATH_LINK_TARGET | __WASI_RIGHT_PATH_OPEN | \ + __WASI_RIGHT_FD_READDIR | __WASI_RIGHT_PATH_READLINK | \ + __WASI_RIGHT_PATH_RENAME_SOURCE | __WASI_RIGHT_PATH_RENAME_TARGET | \ + __WASI_RIGHT_PATH_FILESTAT_GET | \ + __WASI_RIGHT_PATH_FILESTAT_SET_SIZE | \ + __WASI_RIGHT_PATH_FILESTAT_SET_TIMES | \ + __WASI_RIGHT_FD_FILESTAT_GET | __WASI_RIGHT_FD_FILESTAT_SET_TIMES | \ + __WASI_RIGHT_PATH_SYMLINK | __WASI_RIGHT_PATH_UNLINK_FILE | \ + __WASI_RIGHT_PATH_REMOVE_DIRECTORY | \ + __WASI_RIGHT_POLL_FD_READWRITE) +#define RIGHTS_DIRECTORY_INHERITING \ + (RIGHTS_DIRECTORY_BASE | RIGHTS_REGULAR_FILE_BASE) + +// Operations that apply to regular files. +#define RIGHTS_REGULAR_FILE_BASE \ + (__WASI_RIGHT_FD_DATASYNC | __WASI_RIGHT_FD_READ | \ + __WASI_RIGHT_FD_SEEK | __WASI_RIGHT_FD_FDSTAT_SET_FLAGS | \ + __WASI_RIGHT_FD_SYNC | __WASI_RIGHT_FD_TELL | __WASI_RIGHT_FD_WRITE | \ + __WASI_RIGHT_FD_ADVISE | __WASI_RIGHT_FD_ALLOCATE | \ + __WASI_RIGHT_FD_FILESTAT_GET | __WASI_RIGHT_FD_FILESTAT_SET_SIZE | \ + __WASI_RIGHT_FD_FILESTAT_SET_TIMES | __WASI_RIGHT_POLL_FD_READWRITE) +#define RIGHTS_REGULAR_FILE_INHERITING 0 + +// Operations that apply to sockets and socket pairs. +#define RIGHTS_SOCKET_BASE \ + (__WASI_RIGHT_FD_READ | __WASI_RIGHT_FD_FDSTAT_SET_FLAGS | \ + __WASI_RIGHT_FD_WRITE | __WASI_RIGHT_FD_FILESTAT_GET | \ + __WASI_RIGHT_POLL_FD_READWRITE | __WASI_RIGHT_SOCK_CONNECT | \ + __WASI_RIGHT_SOCK_LISTEN | __WASI_RIGHT_SOCK_BIND | \ + __WASI_RIGHT_SOCK_ACCEPT | __WASI_RIGHT_SOCK_RECV | \ + __WASI_RIGHT_SOCK_SEND | __WASI_RIGHT_SOCK_ADDR_LOCAL | \ + __WASI_RIGHT_SOCK_ADDR_REMOTE | __WASI_RIGHT_SOCK_RECV_FROM | \ + __WASI_RIGHT_SOCK_SEND_TO) +#define RIGHTS_SOCKET_INHERITING RIGHTS_ALL + +// Operations that apply to TTYs. +#define RIGHTS_TTY_BASE \ + (__WASI_RIGHT_FD_READ | __WASI_RIGHT_FD_FDSTAT_SET_FLAGS | \ + __WASI_RIGHT_FD_WRITE | __WASI_RIGHT_FD_FILESTAT_GET | \ + __WASI_RIGHT_POLL_FD_READWRITE) +#define RIGHTS_TTY_INHERITING 0 + +/* clang-format on */ + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/ssp_config.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/ssp_config.h new file mode 100644 index 0000000..298dea6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/ssp_config.h @@ -0,0 +1,91 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#ifndef SSP_CONFIG_H +#define SSP_CONFIG_H + +#include "bh_platform.h" +#include "gnuc.h" + +#if defined(__FreeBSD__) || defined(__APPLE__) \ + || (defined(ANDROID) && __ANDROID_API__ < 28) +#define CONFIG_HAS_ARC4RANDOM_BUF 1 +#else +#define CONFIG_HAS_ARC4RANDOM_BUF 0 +#endif + +// On Linux, prefer to use getrandom, though it isn't available in +// GLIBC before 2.25. +#if (defined(__linux__) || defined(ESP_PLATFORM) || defined(__COSMOPOLITAN__)) \ + && (!defined(__GLIBC__) || __GLIBC__ > 2 \ + || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 25)) +#define CONFIG_HAS_GETRANDOM 1 +#else +#define CONFIG_HAS_GETRANDOM 0 +#endif + +#if defined(__CloudABI__) || defined(BH_PLATFORM_FREERTOS) +#define CONFIG_HAS_CAP_ENTER 1 +#else +#define CONFIG_HAS_CAP_ENTER 0 +#endif + +#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__EMSCRIPTEN__) \ + && !defined(ESP_PLATFORM) && !defined(DISABLE_CLOCK_NANOSLEEP) \ + && !defined(BH_PLATFORM_FREERTOS) +#define CONFIG_HAS_CLOCK_NANOSLEEP 1 +#else +#define CONFIG_HAS_CLOCK_NANOSLEEP 0 +#endif + +#if defined(__APPLE__) || defined(__CloudABI__) +#define CONFIG_HAS_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP 1 +#else +#define CONFIG_HAS_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP 0 +#endif + +#if !defined(__APPLE__) && !defined(BH_PLATFORM_LINUX_SGX) && !defined(_WIN32) \ + && !defined(__COSMOPOLITAN__) && !defined(BH_PLATFORM_FREERTOS) +#define CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK 1 +#else +#define CONFIG_HAS_PTHREAD_CONDATTR_SETCLOCK 0 +#endif + +#if !defined(BH_PLATFORM_LINUX_SGX) +/* Clang's __GNUC_PREREQ macro has a different meaning than GCC one, +so we have to handle this case specially */ +#if defined(__clang__) +/* Clang provides stdatomic.h since 3.6.0 +See https://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html */ +#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) +#define CONFIG_HAS_STD_ATOMIC 1 +#else +#define CONFIG_HAS_STD_ATOMIC 0 +#endif +#elif defined(__GNUC_PREREQ) +/* Even though older versions of GCC support C11, atomics were +not implemented until 4.9. See +https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58016 */ +#if __GNUC_PREREQ(4, 9) +#define CONFIG_HAS_STD_ATOMIC 1 +#else /* else of __GNUC_PREREQ(4, 9) */ +#define CONFIG_HAS_STD_ATOMIC 0 +#endif /* end of __GNUC_PREREQ(4, 9) */ +#else /* else of defined(__GNUC_PREREQ) */ +#define CONFIG_HAS_STD_ATOMIC 1 +#endif /* end of defined(__GNUC_PREREQ) */ +#else /* else of !defined(BH_PLATFORM_LINUX_SGX) */ +#define CONFIG_HAS_STD_ATOMIC 0 +#endif /* end of !defined(BH_PLATFORM_LINUX_SGX) */ + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/str.c b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/str.c new file mode 100644 index 0000000..858d8d5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/str.c @@ -0,0 +1,47 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#include "ssp_config.h" +#include "bh_platform.h" +#include "str.h" + +static char * +bh_strndup(const char *s, size_t n) +{ + size_t l = strnlen(s, n); + char *s1 = wasm_runtime_malloc((uint32)(l + 1)); + + if (!s1) + return NULL; + bh_memcpy_s(s1, (uint32)(l + 1), s, (uint32)l); + s1[l] = 0; + return s1; +} + +char * +str_nullterminate(const char *s, size_t len) +{ + /* Copy string */ + char *ret = bh_strndup(s, len); + + if (ret == NULL) + return NULL; + + /* Ensure that it contains no null bytes within */ + if (strlen(ret) != len) { + wasm_runtime_free(ret); + errno = EILSEQ; + return NULL; + } + return ret; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/str.h b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/str.h new file mode 100644 index 0000000..7d633e5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/str.h @@ -0,0 +1,22 @@ +// Part of the Wasmtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See +// https://github.com/bytecodealliance/wasmtime/blob/main/LICENSE for license +// information. +// +// Significant parts of this file are derived from cloudabi-utils. See +// https://github.com/bytecodealliance/wasmtime/blob/main/lib/wasi/sandboxed-system-primitives/src/LICENSE +// for license information. +// +// The upstream file contains the following copyright notice: +// +// Copyright (c) 2016 Nuxi, https://nuxi.nl/ + +#ifndef STR_H +#define STR_H + +#include "ssp_config.h" + +char * +str_nullterminate(const char *, size_t); + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/SConscript b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/SConscript new file mode 100644 index 0000000..65f561a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/SConscript @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * + +cwd = GetCurrentDir() + +src = Split(''' +thread_manager.c +''') + +CPPPATH = [cwd] + +group = DefineGroup('iwasm_lib_thread_mgr', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_manager.c b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_manager.c new file mode 100644 index 0000000..ac4deb9 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -0,0 +1,1492 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "thread_manager.h" +#include "../common/wasm_c_api_internal.h" + +#if WASM_ENABLE_INTERP != 0 +#include "../interpreter/wasm_runtime.h" +#endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "debug_engine.h" +#endif + +typedef struct { + bh_list_link l; + void (*destroy_cb)(WASMCluster *); +} DestroyCallBackNode; + +static bh_list destroy_callback_list_head; +static bh_list *const destroy_callback_list = &destroy_callback_list_head; + +static bh_list cluster_list_head; +static bh_list *const cluster_list = &cluster_list_head; +static korp_mutex cluster_list_lock; + +static korp_mutex _exception_lock; + +typedef void (*list_visitor)(void *, void *); + +static uint32 cluster_max_thread_num = CLUSTER_MAX_THREAD_NUM; + +/* Set the maximum thread number, if this function is not called, + the max thread num is defined by CLUSTER_MAX_THREAD_NUM */ +void +wasm_cluster_set_max_thread_num(uint32 num) +{ + if (num > 0) + cluster_max_thread_num = num; +} + +bool +thread_manager_init() +{ + if (bh_list_init(cluster_list) != 0) + return false; + if (os_mutex_init(&cluster_list_lock) != 0) + return false; + if (os_mutex_init(&_exception_lock) != 0) { + os_mutex_destroy(&cluster_list_lock); + return false; + } + return true; +} + +void +thread_manager_destroy() +{ + WASMCluster *cluster = bh_list_first_elem(cluster_list); + WASMCluster *next; + while (cluster) { + next = bh_list_elem_next(cluster); + wasm_cluster_destroy(cluster); + cluster = next; + } + wasm_cluster_cancel_all_callbacks(); + os_mutex_destroy(&_exception_lock); + os_mutex_destroy(&cluster_list_lock); +} + +static void +traverse_list(bh_list *l, list_visitor visitor, void *user_data) +{ + void *next, *node = bh_list_first_elem(l); + while (node) { + next = bh_list_elem_next(node); + visitor(node, user_data); + node = next; + } +} + +/* Assumes cluster->lock is locked */ +static bool +safe_traverse_exec_env_list(WASMCluster *cluster, list_visitor visitor, + void *user_data) +{ + Vector proc_nodes; + void *node; + bool ret = true; + + if (!bh_vector_init(&proc_nodes, cluster->exec_env_list.len, sizeof(void *), + false)) { + ret = false; + goto final; + } + + node = bh_list_first_elem(&cluster->exec_env_list); + + while (node) { + bool already_processed = false; + void *proc_node; + uint32 i; + for (i = 0; i < (uint32)bh_vector_size(&proc_nodes); i++) { + if (!bh_vector_get(&proc_nodes, i, &proc_node)) { + ret = false; + goto final; + } + if (proc_node == node) { + already_processed = true; + break; + } + } + if (already_processed) { + node = bh_list_elem_next(node); + continue; + } + + os_mutex_unlock(&cluster->lock); + visitor(node, user_data); + os_mutex_lock(&cluster->lock); + if (!bh_vector_append(&proc_nodes, &node)) { + ret = false; + goto final; + } + + node = bh_list_first_elem(&cluster->exec_env_list); + } + +final: + bh_vector_destroy(&proc_nodes); + + return ret; +} + +/* The caller must not have any locks */ +bool +wasm_cluster_allocate_aux_stack(WASMExecEnv *exec_env, uint64 *p_start, + uint32 *p_size) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION != 0 + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); + uint64 stack_end; + + stack_end = wasm_runtime_module_malloc_internal(module_inst, exec_env, + cluster->stack_size, NULL); + *p_start = stack_end + cluster->stack_size; + *p_size = cluster->stack_size; + + return stack_end != 0; +#else + uint32 i; + + /* If the module doesn't have aux stack info, + it can't create any threads */ + + os_mutex_lock(&cluster->lock); + if (!cluster->stack_segment_occupied) { + os_mutex_unlock(&cluster->lock); + return false; + } + + for (i = 0; i < cluster_max_thread_num; i++) { + if (!cluster->stack_segment_occupied[i]) { + if (p_start) + *p_start = cluster->stack_tops[i]; + if (p_size) + *p_size = cluster->stack_size; + cluster->stack_segment_occupied[i] = true; + os_mutex_unlock(&cluster->lock); + return true; + } + } + os_mutex_unlock(&cluster->lock); + + return false; +#endif +} + +/* The caller must not have any locks */ +bool +wasm_cluster_free_aux_stack(WASMExecEnv *exec_env, uint64 start) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION != 0 + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); + + if (!wasm_exec_env_is_aux_stack_managed_by_runtime(exec_env)) { + return true; + } + + bh_assert(start >= cluster->stack_size); + + wasm_runtime_module_free_internal(module_inst, exec_env, + start - cluster->stack_size); + + return true; +#else + uint32 i; + + os_mutex_lock(&cluster->lock); + for (i = 0; i < cluster_max_thread_num; i++) { + if (start == cluster->stack_tops[i]) { + cluster->stack_segment_occupied[i] = false; + os_mutex_unlock(&cluster->lock); + return true; + } + } + os_mutex_unlock(&cluster->lock); + return false; +#endif +} + +WASMCluster * +wasm_cluster_create(WASMExecEnv *exec_env) +{ + WASMCluster *cluster; + uint32 aux_stack_size; + uint64 aux_stack_start; + + bh_assert(exec_env->cluster == NULL); + if (!(cluster = wasm_runtime_malloc(sizeof(WASMCluster)))) { + LOG_ERROR("thread manager error: failed to allocate memory"); + return NULL; + } + memset(cluster, 0, sizeof(WASMCluster)); + + exec_env->cluster = cluster; + + bh_list_init(&cluster->exec_env_list); + bh_list_insert(&cluster->exec_env_list, exec_env); + if (os_mutex_init(&cluster->lock) != 0) { + wasm_runtime_free(cluster); + LOG_ERROR("thread manager error: failed to init mutex"); + return NULL; + } + + /* Prepare the aux stack top and size for every thread */ + if (!wasm_exec_env_get_aux_stack(exec_env, &aux_stack_start, + &aux_stack_size)) { +#if WASM_ENABLE_LIB_WASI_THREADS == 0 + LOG_VERBOSE("No aux stack info for this module, can't create thread"); +#endif + + /* If the module don't have aux stack info, don't throw error here, + but remain stack_tops and stack_segment_occupied as NULL */ + os_mutex_lock(&cluster_list_lock); + if (bh_list_insert(cluster_list, cluster) != 0) { + os_mutex_unlock(&cluster_list_lock); + goto fail; + } + os_mutex_unlock(&cluster_list_lock); + + return cluster; + } + +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION != 0 + cluster->stack_size = aux_stack_size; +#else + cluster->stack_size = aux_stack_size / (cluster_max_thread_num + 1); + if (cluster->stack_size < WASM_THREAD_AUX_STACK_SIZE_MIN) { + goto fail; + } + /* Make stack size 16-byte aligned */ + cluster->stack_size = cluster->stack_size & (~15); +#endif + + /* Set initial aux stack top to the instance and + aux stack boundary to the main exec_env */ + if (!wasm_exec_env_set_aux_stack(exec_env, aux_stack_start, + cluster->stack_size)) + goto fail; + +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0 + if (cluster_max_thread_num != 0) { + uint64 total_size = cluster_max_thread_num * sizeof(uint64); + uint32 i; + if (total_size >= UINT32_MAX + || !(cluster->stack_tops = + wasm_runtime_malloc((uint32)total_size))) { + goto fail; + } + memset(cluster->stack_tops, 0, (uint32)total_size); + + if (!(cluster->stack_segment_occupied = + wasm_runtime_malloc(cluster_max_thread_num * sizeof(bool)))) { + goto fail; + } + memset(cluster->stack_segment_occupied, 0, + cluster_max_thread_num * sizeof(bool)); + + /* Reserve space for main instance */ + aux_stack_start -= cluster->stack_size; + + for (i = 0; i < cluster_max_thread_num; i++) { + cluster->stack_tops[i] = aux_stack_start - cluster->stack_size * i; + } + } +#endif + + os_mutex_lock(&cluster_list_lock); + if (bh_list_insert(cluster_list, cluster) != 0) { + os_mutex_unlock(&cluster_list_lock); + goto fail; + } + os_mutex_unlock(&cluster_list_lock); + + return cluster; + +fail: + if (cluster) + wasm_cluster_destroy(cluster); + + return NULL; +} + +static void +destroy_cluster_visitor(void *node, void *user_data) +{ + DestroyCallBackNode *destroy_node = (DestroyCallBackNode *)node; + WASMCluster *cluster = (WASMCluster *)user_data; + + destroy_node->destroy_cb(cluster); +} + +void +wasm_cluster_destroy(WASMCluster *cluster) +{ + traverse_list(destroy_callback_list, destroy_cluster_visitor, + (void *)cluster); + + /* Remove the cluster from the cluster list */ + os_mutex_lock(&cluster_list_lock); + bh_list_remove(cluster_list, cluster); + os_mutex_unlock(&cluster_list_lock); + + os_mutex_destroy(&cluster->lock); + +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0 + if (cluster->stack_tops) + wasm_runtime_free(cluster->stack_tops); + if (cluster->stack_segment_occupied) + wasm_runtime_free(cluster->stack_segment_occupied); +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_debug_instance_destroy(cluster); +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + bh_vector_destroy(&cluster->exception_frames); +#endif + + wasm_runtime_free(cluster); +} + +static void +free_node_visitor(void *node, void *user_data) +{ + wasm_runtime_free(node); +} + +void +wasm_cluster_cancel_all_callbacks() +{ + traverse_list(destroy_callback_list, free_node_visitor, NULL); + bh_list_init(destroy_callback_list); +} + +WASMCluster * +wasm_exec_env_get_cluster(WASMExecEnv *exec_env) +{ + return exec_env->cluster; +} + +/* The caller must lock cluster->lock */ +static bool +wasm_cluster_add_exec_env(WASMCluster *cluster, WASMExecEnv *exec_env) +{ + bool ret = true; + + exec_env->cluster = cluster; + + if (cluster->exec_env_list.len == cluster_max_thread_num + 1) { + LOG_ERROR("thread manager error: " + "maximum number of threads exceeded"); + ret = false; + } + + if (ret && bh_list_insert(&cluster->exec_env_list, exec_env) != 0) + ret = false; + + return ret; +} + +static bool +wasm_cluster_del_exec_env_internal(WASMCluster *cluster, WASMExecEnv *exec_env, + bool can_destroy_cluster) +{ + bool ret = true; + bh_assert(exec_env->cluster == cluster); + +#if WASM_ENABLE_DEBUG_INTERP != 0 + /* Wait for debugger control thread to process the + stop event of this thread */ + if (cluster->debug_inst) { + /* lock the debug_inst->wait_lock so + other threads can't fire stop events */ + os_mutex_lock(&cluster->debug_inst->wait_lock); + while (cluster->debug_inst->stopped_thread == exec_env) { + /* either wakes up by signal or by 1-second timeout */ + os_cond_reltimedwait(&cluster->debug_inst->wait_cond, + &cluster->debug_inst->wait_lock, 1000000); + } + os_mutex_unlock(&cluster->debug_inst->wait_lock); + } +#endif + if (bh_list_remove(&cluster->exec_env_list, exec_env) != 0) + ret = false; + + if (can_destroy_cluster) { + if (cluster->exec_env_list.len == 0) { + /* exec_env_list empty, destroy the cluster */ + wasm_cluster_destroy(cluster); + } + } + else { + /* Don't destroy cluster as cluster->lock is being used */ + } + + return ret; +} + +/* The caller should lock cluster->lock for thread safety */ +bool +wasm_cluster_del_exec_env(WASMCluster *cluster, WASMExecEnv *exec_env) +{ + return wasm_cluster_del_exec_env_internal(cluster, exec_env, true); +} + +static WASMExecEnv * +wasm_cluster_search_exec_env(WASMCluster *cluster, + WASMModuleInstanceCommon *module_inst) +{ + WASMExecEnv *node = NULL; + + os_mutex_lock(&cluster->lock); + node = bh_list_first_elem(&cluster->exec_env_list); + while (node) { + if (node->module_inst == module_inst) { + os_mutex_unlock(&cluster->lock); + return node; + } + node = bh_list_elem_next(node); + } + + os_mutex_unlock(&cluster->lock); + return NULL; +} + +/* search the global cluster list to find if the given + module instance have a corresponding exec_env */ +WASMExecEnv * +wasm_clusters_search_exec_env(WASMModuleInstanceCommon *module_inst) +{ + WASMCluster *cluster = NULL; + WASMExecEnv *exec_env = NULL; + + os_mutex_lock(&cluster_list_lock); + cluster = bh_list_first_elem(cluster_list); + while (cluster) { + exec_env = wasm_cluster_search_exec_env(cluster, module_inst); + if (exec_env) { + os_mutex_unlock(&cluster_list_lock); + return exec_env; + } + cluster = bh_list_elem_next(cluster); + } + + os_mutex_unlock(&cluster_list_lock); + return NULL; +} + +WASMExecEnv * +wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasm_module_t module; + wasm_module_inst_t new_module_inst; + WASMExecEnv *new_exec_env; + uint32 aux_stack_size; + uint64 aux_stack_start; + uint32 stack_size = 8192; + + if (!module_inst || !(module = wasm_exec_env_get_module(exec_env))) { + return NULL; + } + + if (!(new_module_inst = wasm_runtime_instantiate_internal( + module, module_inst, exec_env, stack_size, 0, 0, NULL, 0))) { + return NULL; + } + + /* Set custom_data to new module instance */ + wasm_runtime_set_custom_data_internal( + new_module_inst, wasm_runtime_get_custom_data(module_inst)); + + wasm_native_inherit_contexts(new_module_inst, module_inst); + + if (!(wasm_cluster_dup_c_api_imports(new_module_inst, module_inst))) { + goto fail1; + } + + if (!wasm_cluster_allocate_aux_stack(exec_env, &aux_stack_start, + &aux_stack_size)) { + LOG_ERROR("thread manager error: " + "failed to allocate aux stack space for new thread"); + goto fail1; + } + + os_mutex_lock(&cluster->lock); + + if (cluster->has_exception || cluster->processing) { + goto fail2; + } + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + stack_size = + ((WASMModuleInstance *)module_inst)->default_wasm_stack_size; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + stack_size = + ((AOTModuleInstance *)module_inst)->default_wasm_stack_size; + } +#endif + + new_exec_env = wasm_exec_env_create_internal(new_module_inst, + exec_env->wasm_stack_size); + if (!new_exec_env) { + goto fail2; + } + + /* Set aux stack for current thread */ + if (!wasm_exec_env_set_aux_stack(new_exec_env, aux_stack_start, + aux_stack_size)) { + goto fail3; + } + new_exec_env->is_aux_stack_allocated = true; + + /* Inherit suspend_flags of parent thread */ + new_exec_env->suspend_flags.flags = + (exec_env->suspend_flags.flags & WASM_SUSPEND_FLAG_INHERIT_MASK); + + if (!wasm_cluster_add_exec_env(cluster, new_exec_env)) { + goto fail3; + } + + os_mutex_unlock(&cluster->lock); + + return new_exec_env; + +fail3: + wasm_exec_env_destroy_internal(new_exec_env); +fail2: + os_mutex_unlock(&cluster->lock); + /* free the allocated aux stack space */ + wasm_cluster_free_aux_stack(exec_env, aux_stack_start); +fail1: + wasm_runtime_deinstantiate_internal(new_module_inst, true); + + return NULL; +} + +void +wasm_cluster_destroy_spawned_exec_env(WASMExecEnv *exec_env) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + wasm_module_inst_t module_inst = wasm_runtime_get_module_inst(exec_env); + bh_assert(cluster != NULL); + WASMExecEnv *exec_env_tls = NULL; + +#ifdef OS_ENABLE_HW_BOUND_CHECK + /* Note: free_aux_stack can execute the module's "free" function + * using the specified exec_env. In case of OS_ENABLE_HW_BOUND_CHECK, + * it needs to match the TLS exec_env if available. (Consider a native + * function which calls wasm_cluster_destroy_spawned_exec_env.) + */ + exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + if (exec_env_tls == NULL) { + exec_env_tls = exec_env; + } + + /* Free aux stack space which was allocated in + wasm_cluster_spawn_exec_env */ + bh_assert(exec_env_tls->is_aux_stack_allocated); + wasm_cluster_free_aux_stack(exec_env_tls, + (uint64)exec_env->aux_stack_bottom); + + os_mutex_lock(&cluster->lock); + + /* Remove exec_env */ + wasm_cluster_del_exec_env_internal(cluster, exec_env, false); + /* Destroy exec_env */ + wasm_exec_env_destroy_internal(exec_env); + /* Routine exit, destroy instance */ + wasm_runtime_deinstantiate_internal(module_inst, true); + + os_mutex_unlock(&cluster->lock); +} + +/* start routine of thread manager */ +static void * +thread_manager_start_routine(void *arg) +{ + void *ret; + WASMExecEnv *exec_env = (WASMExecEnv *)arg; + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); + + bh_assert(cluster != NULL); + bh_assert(module_inst != NULL); + + os_mutex_lock(&exec_env->wait_lock); + exec_env->handle = os_self_thread(); + /* Notify the parent thread to continue running */ + os_cond_signal(&exec_env->wait_cond); + os_mutex_unlock(&exec_env->wait_lock); + + ret = exec_env->thread_start_routine(exec_env); + +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_mutex_lock(&exec_env->wait_lock); + if (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) + & WASM_SUSPEND_FLAG_EXIT) + ret = exec_env->thread_ret_value; + os_mutex_unlock(&exec_env->wait_lock); +#endif + + /* Routine exit */ + +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_thread_exited(exec_env); +#endif + + /* Free aux stack space */ + if (exec_env->is_aux_stack_allocated) + wasm_cluster_free_aux_stack(exec_env, + (uint64)exec_env->aux_stack_bottom); + + os_mutex_lock(&cluster_list_lock); + + os_mutex_lock(&cluster->lock); + + /* Detach the native thread here to ensure the resources are freed */ + if (exec_env->wait_count == 0 && !exec_env->thread_is_detached) { + /* Only detach current thread when there is no other thread + joining it, otherwise let the system resources for the + thread be released after joining */ + os_thread_detach(exec_env->handle); + /* No need to set exec_env->thread_is_detached to true here + since we will exit soon */ + } + +#if WASM_ENABLE_PERF_PROFILING != 0 + os_printf("============= Spawned thread ===========\n"); + wasm_runtime_dump_perf_profiling(module_inst); + os_printf("========================================\n"); +#endif + + /* Remove exec_env */ + wasm_cluster_del_exec_env_internal(cluster, exec_env, false); + /* Destroy exec_env */ + wasm_exec_env_destroy_internal(exec_env); + /* Routine exit, destroy instance */ + wasm_runtime_deinstantiate_internal(module_inst, true); + + os_mutex_unlock(&cluster->lock); + + os_mutex_unlock(&cluster_list_lock); + + os_thread_exit(ret); + return ret; +} + +int32 +wasm_cluster_create_thread(WASMExecEnv *exec_env, + wasm_module_inst_t module_inst, + bool is_aux_stack_allocated, uint64 aux_stack_start, + uint32 aux_stack_size, + void *(*thread_routine)(void *), void *arg) +{ + WASMCluster *cluster; + WASMExecEnv *new_exec_env; + korp_tid tid; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + + if (cluster->has_exception || cluster->processing) { + goto fail1; + } + + new_exec_env = + wasm_exec_env_create_internal(module_inst, exec_env->wasm_stack_size); + if (!new_exec_env) + goto fail1; + + if (is_aux_stack_allocated) { + /* Set aux stack for current thread */ + if (!wasm_exec_env_set_aux_stack(new_exec_env, aux_stack_start, + aux_stack_size)) { + goto fail2; + } + new_exec_env->is_aux_stack_allocated = true; + } + else { + /* Disable aux stack */ + new_exec_env->aux_stack_boundary = 0; + new_exec_env->aux_stack_bottom = UINTPTR_MAX; + new_exec_env->is_aux_stack_allocated = false; + } + + /* Inherit suspend_flags of parent thread */ + new_exec_env->suspend_flags.flags = + (exec_env->suspend_flags.flags & WASM_SUSPEND_FLAG_INHERIT_MASK); + + if (!wasm_cluster_add_exec_env(cluster, new_exec_env)) + goto fail2; + + new_exec_env->thread_start_routine = thread_routine; + new_exec_env->thread_arg = arg; + + os_mutex_lock(&new_exec_env->wait_lock); + + if (0 + != os_thread_create(&tid, thread_manager_start_routine, + (void *)new_exec_env, + APP_THREAD_STACK_SIZE_DEFAULT)) { + os_mutex_unlock(&new_exec_env->wait_lock); + goto fail3; + } + + /* Wait until the new_exec_env->handle is set to avoid it is + illegally accessed after unlocking cluster->lock */ + os_cond_wait(&new_exec_env->wait_cond, &new_exec_env->wait_lock); + os_mutex_unlock(&new_exec_env->wait_lock); + + os_mutex_unlock(&cluster->lock); + + return 0; + +fail3: + wasm_cluster_del_exec_env_internal(cluster, new_exec_env, false); +fail2: + wasm_exec_env_destroy_internal(new_exec_env); +fail1: + os_mutex_unlock(&cluster->lock); + + return -1; +} + +bool +wasm_cluster_dup_c_api_imports(WASMModuleInstanceCommon *module_inst_dst, + const WASMModuleInstanceCommon *module_inst_src) +{ + /* workaround about passing instantiate-linking information */ + CApiFuncImport **new_c_api_func_imports = NULL; + CApiFuncImport *c_api_func_imports = NULL; + uint32 import_func_count = 0; + uint32 size_in_bytes = 0; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst_src->module_type == Wasm_Module_Bytecode) { + new_c_api_func_imports = + &(((WASMModuleInstance *)module_inst_dst)->c_api_func_imports); + c_api_func_imports = + ((const WASMModuleInstance *)module_inst_src)->c_api_func_imports; + import_func_count = + ((WASMModule *)(((const WASMModuleInstance *)module_inst_src) + ->module)) + ->import_function_count; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_src->module_type == Wasm_Module_AoT) { + new_c_api_func_imports = + &(((AOTModuleInstance *)module_inst_dst)->c_api_func_imports); + c_api_func_imports = + ((const AOTModuleInstance *)module_inst_src)->c_api_func_imports; + import_func_count = + ((AOTModule *)(((AOTModuleInstance *)module_inst_src)->module)) + ->import_func_count; + } +#endif + + if (import_func_count != 0 && c_api_func_imports) { + size_in_bytes = sizeof(CApiFuncImport) * import_func_count; + *new_c_api_func_imports = wasm_runtime_malloc(size_in_bytes); + if (!(*new_c_api_func_imports)) + return false; + + bh_memcpy_s(*new_c_api_func_imports, size_in_bytes, c_api_func_imports, + size_in_bytes); + } + return true; +} + +#if WASM_ENABLE_DEBUG_INTERP != 0 +WASMCurrentEnvStatus * +wasm_cluster_create_exenv_status() +{ + WASMCurrentEnvStatus *status; + + if (!(status = wasm_runtime_malloc(sizeof(WASMCurrentEnvStatus)))) { + return NULL; + } + + status->step_count = 0; + status->signal_flag = 0; + status->running_status = 0; + return status; +} + +void +wasm_cluster_destroy_exenv_status(WASMCurrentEnvStatus *status) +{ + wasm_runtime_free(status); +} + +inline static bool +wasm_cluster_thread_is_running(WASMExecEnv *exec_env) +{ + return exec_env->current_status->running_status == STATUS_RUNNING + || exec_env->current_status->running_status == STATUS_STEP; +} + +void +wasm_cluster_clear_thread_signal(WASMExecEnv *exec_env) +{ + exec_env->current_status->signal_flag = 0; +} + +void +wasm_cluster_thread_send_signal(WASMExecEnv *exec_env, uint32 signo) +{ + exec_env->current_status->signal_flag = signo; +} + +static void +notify_debug_instance(WASMExecEnv *exec_env) +{ + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + if (!cluster->debug_inst) { + return; + } + + on_thread_stop_event(cluster->debug_inst, exec_env); +} + +static void +notify_debug_instance_exit(WASMExecEnv *exec_env) +{ + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + if (!cluster->debug_inst) { + return; + } + + on_thread_exit_event(cluster->debug_inst, exec_env); +} + +void +wasm_cluster_thread_waiting_run(WASMExecEnv *exec_env) +{ + exec_env->current_status->running_status = STATUS_STOP; + notify_debug_instance(exec_env); + + while (!wasm_cluster_thread_is_running(exec_env)) { + os_cond_wait(&exec_env->wait_cond, &exec_env->wait_lock); + } +} + +void +wasm_cluster_send_signal_all(WASMCluster *cluster, uint32 signo) +{ + WASMExecEnv *exec_env = bh_list_first_elem(&cluster->exec_env_list); + while (exec_env) { + wasm_cluster_thread_send_signal(exec_env, signo); + exec_env = bh_list_elem_next(exec_env); + } +} + +void +wasm_cluster_thread_exited(WASMExecEnv *exec_env) +{ + exec_env->current_status->running_status = STATUS_EXIT; + notify_debug_instance_exit(exec_env); +} + +void +wasm_cluster_thread_continue(WASMExecEnv *exec_env) +{ + os_mutex_lock(&exec_env->wait_lock); + wasm_cluster_clear_thread_signal(exec_env); + exec_env->current_status->running_status = STATUS_RUNNING; + os_cond_signal(&exec_env->wait_cond); + os_mutex_unlock(&exec_env->wait_lock); +} + +void +wasm_cluster_thread_step(WASMExecEnv *exec_env) +{ + os_mutex_lock(&exec_env->wait_lock); + exec_env->current_status->running_status = STATUS_STEP; + os_cond_signal(&exec_env->wait_cond); + os_mutex_unlock(&exec_env->wait_lock); +} + +void +wasm_cluster_set_debug_inst(WASMCluster *cluster, WASMDebugInstance *inst) +{ + cluster->debug_inst = inst; +} + +#endif /* end of WASM_ENABLE_DEBUG_INTERP */ + +/* Check whether the exec_env is in one of all clusters, the caller + should add lock to the cluster list before calling us */ +static bool +clusters_have_exec_env(WASMExecEnv *exec_env) +{ + WASMCluster *cluster = bh_list_first_elem(cluster_list); + WASMExecEnv *node; + + while (cluster) { + os_mutex_lock(&cluster->lock); + node = bh_list_first_elem(&cluster->exec_env_list); + + while (node) { + if (node == exec_env) { + bh_assert(exec_env->cluster == cluster); + os_mutex_unlock(&cluster->lock); + return true; + } + node = bh_list_elem_next(node); + } + os_mutex_unlock(&cluster->lock); + + cluster = bh_list_elem_next(cluster); + } + + return false; +} + +int32 +wasm_cluster_join_thread(WASMExecEnv *exec_env, void **ret_val) +{ + korp_tid handle; + + os_mutex_lock(&cluster_list_lock); + + if (!clusters_have_exec_env(exec_env) || exec_env->thread_is_detached) { + /* Invalid thread, thread has exited or thread has been detached */ + if (ret_val) + *ret_val = NULL; + os_mutex_unlock(&cluster_list_lock); + return 0; + } + + os_mutex_lock(&exec_env->wait_lock); + exec_env->wait_count++; + handle = exec_env->handle; + os_mutex_unlock(&exec_env->wait_lock); + + os_mutex_unlock(&cluster_list_lock); + + return os_thread_join(handle, ret_val); +} + +int32 +wasm_cluster_detach_thread(WASMExecEnv *exec_env) +{ + int32 ret = 0; + + os_mutex_lock(&cluster_list_lock); + if (!clusters_have_exec_env(exec_env)) { + /* Invalid thread or the thread has exited */ + os_mutex_unlock(&cluster_list_lock); + return 0; + } + if (exec_env->wait_count == 0 && !exec_env->thread_is_detached) { + /* Only detach current thread when there is no other thread + joining it, otherwise let the system resources for the + thread be released after joining */ + ret = os_thread_detach(exec_env->handle); + exec_env->thread_is_detached = true; + } + os_mutex_unlock(&cluster_list_lock); + return ret; +} + +void +wasm_cluster_exit_thread(WASMExecEnv *exec_env, void *retval) +{ + WASMCluster *cluster; + WASMModuleInstanceCommon *module_inst; + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env->jmpbuf_stack_top) { + /* Store the return value in exec_env */ + exec_env->thread_ret_value = retval; + + WASM_SUSPEND_FLAGS_FETCH_OR(exec_env->suspend_flags, + WASM_SUSPEND_FLAG_EXIT); + +#ifndef BH_PLATFORM_WINDOWS + /* Pop all jmpbuf_node except the last one */ + while (exec_env->jmpbuf_stack_top->prev) { + wasm_exec_env_pop_jmpbuf(exec_env); + } + os_longjmp(exec_env->jmpbuf_stack_top->jmpbuf, 1); + return; +#endif + } +#endif + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_clear_thread_signal(exec_env); + wasm_cluster_thread_exited(exec_env); +#endif + + /* Free aux stack space */ + if (exec_env->is_aux_stack_allocated) + wasm_cluster_free_aux_stack(exec_env, + (uint64)exec_env->aux_stack_bottom); + + /* App exit the thread, free the resources before exit native thread */ + + os_mutex_lock(&cluster_list_lock); + + os_mutex_lock(&cluster->lock); + + /* Detach the native thread here to ensure the resources are freed */ + if (exec_env->wait_count == 0 && !exec_env->thread_is_detached) { + /* Only detach current thread when there is no other thread + joining it, otherwise let the system resources for the + thread be released after joining */ + os_thread_detach(exec_env->handle); + /* No need to set exec_env->thread_is_detached to true here + since we will exit soon */ + } + + module_inst = exec_env->module_inst; + + /* Remove exec_env */ + wasm_cluster_del_exec_env_internal(cluster, exec_env, false); + /* Destroy exec_env */ + wasm_exec_env_destroy_internal(exec_env); + /* Routine exit, destroy instance */ + wasm_runtime_deinstantiate_internal(module_inst, true); + + os_mutex_unlock(&cluster->lock); + + os_mutex_unlock(&cluster_list_lock); + + os_thread_exit(retval); +} + +static void +set_thread_cancel_flags(WASMExecEnv *exec_env) +{ + os_mutex_lock(&exec_env->wait_lock); + +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TERM); +#endif + WASM_SUSPEND_FLAGS_FETCH_OR(exec_env->suspend_flags, + WASM_SUSPEND_FLAG_TERMINATE); + + os_mutex_unlock(&exec_env->wait_lock); + +#ifdef OS_ENABLE_WAKEUP_BLOCKING_OP + wasm_runtime_interrupt_blocking_op(exec_env); +#endif +} + +static void +clear_thread_cancel_flags(WASMExecEnv *exec_env) +{ + os_mutex_lock(&exec_env->wait_lock); + WASM_SUSPEND_FLAGS_FETCH_AND(exec_env->suspend_flags, + ~WASM_SUSPEND_FLAG_TERMINATE); + os_mutex_unlock(&exec_env->wait_lock); +} + +int32 +wasm_cluster_cancel_thread(WASMExecEnv *exec_env) +{ + os_mutex_lock(&cluster_list_lock); + + if (!exec_env->cluster) { + os_mutex_unlock(&cluster_list_lock); + return 0; + } + + if (!clusters_have_exec_env(exec_env)) { + /* Invalid thread or the thread has exited */ + goto final; + } + + set_thread_cancel_flags(exec_env); + +final: + os_mutex_unlock(&cluster_list_lock); + + return 0; +} + +static void +terminate_thread_visitor(void *node, void *user_data) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMExecEnv *exec_env = (WASMExecEnv *)user_data; + + if (curr_exec_env == exec_env) + return; + + wasm_cluster_cancel_thread(curr_exec_env); + wasm_cluster_join_thread(curr_exec_env, NULL); +} + +void +wasm_cluster_terminate_all(WASMCluster *cluster) +{ + os_mutex_lock(&cluster->lock); + cluster->processing = true; + + safe_traverse_exec_env_list(cluster, terminate_thread_visitor, NULL); + + cluster->processing = false; + os_mutex_unlock(&cluster->lock); +} + +void +wasm_cluster_terminate_all_except_self(WASMCluster *cluster, + WASMExecEnv *exec_env) +{ + os_mutex_lock(&cluster->lock); + cluster->processing = true; + + safe_traverse_exec_env_list(cluster, terminate_thread_visitor, + (void *)exec_env); + + cluster->processing = false; + os_mutex_unlock(&cluster->lock); +} + +static void +wait_for_thread_visitor(void *node, void *user_data) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMExecEnv *exec_env = (WASMExecEnv *)user_data; + + if (curr_exec_env == exec_env) + return; + + wasm_cluster_join_thread(curr_exec_env, NULL); +} + +void +wams_cluster_wait_for_all(WASMCluster *cluster) +{ + os_mutex_lock(&cluster->lock); + cluster->processing = true; + + safe_traverse_exec_env_list(cluster, wait_for_thread_visitor, NULL); + + cluster->processing = false; + os_mutex_unlock(&cluster->lock); +} + +void +wasm_cluster_wait_for_all_except_self(WASMCluster *cluster, + WASMExecEnv *exec_env) +{ + os_mutex_lock(&cluster->lock); + cluster->processing = true; + + safe_traverse_exec_env_list(cluster, wait_for_thread_visitor, + (void *)exec_env); + + cluster->processing = false; + os_mutex_unlock(&cluster->lock); +} + +bool +wasm_cluster_register_destroy_callback(void (*callback)(WASMCluster *)) +{ + DestroyCallBackNode *node; + + if (!(node = wasm_runtime_malloc(sizeof(DestroyCallBackNode)))) { + LOG_ERROR("thread manager error: failed to allocate memory"); + return false; + } + node->destroy_cb = callback; + bh_list_insert(destroy_callback_list, node); + return true; +} + +void +wasm_cluster_suspend_thread(WASMExecEnv *exec_env) +{ + /* Set the suspend flag */ + WASM_SUSPEND_FLAGS_FETCH_OR(exec_env->suspend_flags, + WASM_SUSPEND_FLAG_SUSPEND); +} + +static void +suspend_thread_visitor(void *node, void *user_data) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMExecEnv *exec_env = (WASMExecEnv *)user_data; + + if (curr_exec_env == exec_env) + return; + + wasm_cluster_suspend_thread(curr_exec_env); +} + +void +wasm_cluster_suspend_all(WASMCluster *cluster) +{ + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, suspend_thread_visitor, NULL); + os_mutex_unlock(&cluster->lock); +} + +void +wasm_cluster_suspend_all_except_self(WASMCluster *cluster, + WASMExecEnv *exec_env) +{ + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, suspend_thread_visitor, + (void *)exec_env); + os_mutex_unlock(&cluster->lock); +} + +void +wasm_cluster_resume_thread(WASMExecEnv *exec_env) +{ + WASM_SUSPEND_FLAGS_FETCH_AND(exec_env->suspend_flags, + ~WASM_SUSPEND_FLAG_SUSPEND); + os_cond_signal(&exec_env->wait_cond); +} + +static void +resume_thread_visitor(void *node, void *user_data) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + + wasm_cluster_resume_thread(curr_exec_env); +} + +void +wasm_cluster_resume_all(WASMCluster *cluster) +{ + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, resume_thread_visitor, NULL); + os_mutex_unlock(&cluster->lock); +} + +struct spread_exception_data { + WASMExecEnv *skip; + const char *exception; +}; + +static void +set_exception_visitor(void *node, void *user_data) +{ + const struct spread_exception_data *data = user_data; + WASMExecEnv *exec_env = (WASMExecEnv *)node; + + if (exec_env != data->skip) { + WASMModuleInstance *wasm_inst = + (WASMModuleInstance *)get_module_inst(exec_env); + + exception_lock(wasm_inst); + if (data->exception != NULL) { + snprintf(wasm_inst->cur_exception, sizeof(wasm_inst->cur_exception), + "Exception: %s", data->exception); + } + else { + wasm_inst->cur_exception[0] = '\0'; + } + exception_unlock(wasm_inst); + + /* Terminate the thread so it can exit from dead loops */ + if (data->exception != NULL) { + set_thread_cancel_flags(exec_env); + } + else { + clear_thread_cancel_flags(exec_env); + } + } +} + +void +wasm_cluster_set_exception(WASMExecEnv *exec_env, const char *exception) +{ + const bool has_exception = exception != NULL; + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + struct spread_exception_data data; + data.skip = NULL; + data.exception = exception; + + os_mutex_lock(&cluster->lock); +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (has_exception) { + /* Save the stack frames of the crashed thread into the cluster */ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)get_module_inst(exec_env); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode + && wasm_interp_create_call_stack(exec_env)) { + wasm_frame_vec_clone_internal(module_inst->frames, + &cluster->exception_frames); + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT + && aot_create_call_stack(exec_env)) { + wasm_frame_vec_clone_internal(module_inst->frames, + &cluster->exception_frames); + } +#endif + } +#endif /* WASM_ENABLE_DUMP_CALL_STACK != 0 */ + cluster->has_exception = has_exception; + traverse_list(&cluster->exec_env_list, set_exception_visitor, &data); + os_mutex_unlock(&cluster->lock); +} + +static void +set_custom_data_visitor(void *node, void *user_data) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + + wasm_runtime_set_custom_data_internal(module_inst, user_data); +} + +void +wasm_cluster_spread_custom_data(WASMModuleInstanceCommon *module_inst, + void *custom_data) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + wasm_runtime_set_custom_data_internal(module_inst, custom_data); + } + else { + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, set_custom_data_visitor, + custom_data); + os_mutex_unlock(&cluster->lock); + } +} + +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 +struct inst_set_context_data { + void *key; + void *ctx; +}; + +static void +set_context_visitor(void *node, void *user_data) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + const struct inst_set_context_data *data = user_data; + + wasm_runtime_set_context(module_inst, data->key, data->ctx); +} + +void +wasm_cluster_set_context(WASMModuleInstanceCommon *module_inst, void *key, + void *ctx) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + wasm_runtime_set_context(module_inst, key, ctx); + } + else { + WASMCluster *cluster; + struct inst_set_context_data data; + data.key = key; + data.ctx = ctx; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, set_context_visitor, &data); + os_mutex_unlock(&cluster->lock); + } +} +#endif /* WASM_ENABLE_MODULE_INST_CONTEXT != 0 */ + +bool +wasm_cluster_is_thread_terminated(WASMExecEnv *exec_env) +{ + os_mutex_lock(&exec_env->wait_lock); + bool is_thread_terminated = (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) + & WASM_SUSPEND_FLAG_TERMINATE) + ? true + : false; + os_mutex_unlock(&exec_env->wait_lock); + + return is_thread_terminated; +} + +void +exception_lock(WASMModuleInstance *module_inst) +{ + /* + * Note: this lock could be per module instance if desirable. + * We can revisit on AOT version bump. + * It probably doesn't matter though because the exception handling + * logic should not be executed too frequently anyway. + */ + os_mutex_lock(&_exception_lock); +} + +void +exception_unlock(WASMModuleInstance *module_inst) +{ + os_mutex_unlock(&_exception_lock); +} + +void +wasm_cluster_traverse_lock(WASMExecEnv *exec_env) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + os_mutex_lock(&cluster->lock); +} + +void +wasm_cluster_traverse_unlock(WASMExecEnv *exec_env) +{ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + os_mutex_unlock(&cluster->lock); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_manager.h b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_manager.h new file mode 100644 index 0000000..ee2383a --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_manager.h @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _THREAD_MANAGER_H +#define _THREAD_MANAGER_H + +#include "bh_common.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "../interpreter/wasm.h" +#include "../common/wasm_runtime_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 +typedef struct WASMDebugInstance WASMDebugInstance; +#endif + +struct WASMCluster { + struct WASMCluster *next; + + korp_mutex lock; + bh_list exec_env_list; + +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0 + /* The aux stack of a module with shared memory will be + divided into several segments. This array store the + stack top of different segments */ + uint64 *stack_tops; + /* Record which segments are occupied */ + bool *stack_segment_occupied; +#endif + /* Size of every stack segment */ + uint32 stack_size; + /* When has_exception == true, this cluster should refuse any spawn thread + * requests, this flag can be cleared by calling + * wasm_runtime_clear_exception on instances of any threads of this cluster + */ + bool has_exception; + /* When processing is true, this cluster should refuse any spawn thread + * requests. This is a short-lived state, must be cleared immediately once + * the processing finished. + * This is used to avoid dead lock when one thread waiting another thread + * with lock, see wams_cluster_wait_for_all and wasm_cluster_terminate_all + */ + bool processing; +#if WASM_ENABLE_DEBUG_INTERP != 0 + WASMDebugInstance *debug_inst; +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + /* When an exception occurs in a thread, the stack frames of that thread are + * saved into the cluster + */ + Vector exception_frames; +#endif +}; + +void +wasm_cluster_set_max_thread_num(uint32 num); + +bool +thread_manager_init(); + +void +thread_manager_destroy(); + +/* Create cluster */ +WASMCluster * +wasm_cluster_create(WASMExecEnv *exec_env); + +/* Destroy cluster */ +void +wasm_cluster_destroy(WASMCluster *cluster); + +/* Get the cluster of the current exec_env */ +WASMCluster * +wasm_exec_env_get_cluster(WASMExecEnv *exec_env); + +/* Forward registered functions to a new thread */ +bool +wasm_cluster_dup_c_api_imports(WASMModuleInstanceCommon *module_inst_dst, + const WASMModuleInstanceCommon *module_inst_src); + +int32 +wasm_cluster_create_thread(WASMExecEnv *exec_env, + wasm_module_inst_t module_inst, + bool is_aux_stack_allocated, uint64 aux_stack_start, + uint32 aux_stack_size, + void *(*thread_routine)(void *), void *arg); + +int32 +wasm_cluster_join_thread(WASMExecEnv *exec_env, void **ret_val); + +int32 +wasm_cluster_detach_thread(WASMExecEnv *exec_env); + +int32 +wasm_cluster_cancel_thread(WASMExecEnv *exec_env); + +void +wasm_cluster_exit_thread(WASMExecEnv *exec_env, void *retval); + +bool +wasm_cluster_register_destroy_callback(void (*callback)(WASMCluster *)); + +void +wasm_cluster_cancel_all_callbacks(); + +void +wasm_cluster_suspend_all(WASMCluster *cluster); + +void +wasm_cluster_suspend_all_except_self(WASMCluster *cluster, + WASMExecEnv *exec_env); + +void +wasm_cluster_suspend_thread(WASMExecEnv *exec_env); + +void +wasm_cluster_resume_thread(WASMExecEnv *exec_env); + +void +wasm_cluster_resume_all(WASMCluster *cluster); + +void +wasm_cluster_terminate_all(WASMCluster *cluster); + +void +wasm_cluster_terminate_all_except_self(WASMCluster *cluster, + WASMExecEnv *exec_env); + +void +wams_cluster_wait_for_all(WASMCluster *cluster); + +void +wasm_cluster_wait_for_all_except_self(WASMCluster *cluster, + WASMExecEnv *exec_env); + +bool +wasm_cluster_del_exec_env(WASMCluster *cluster, WASMExecEnv *exec_env); + +WASMExecEnv * +wasm_clusters_search_exec_env(WASMModuleInstanceCommon *module_inst); + +void +wasm_cluster_set_exception(WASMExecEnv *exec_env, const char *exception); + +WASMExecEnv * +wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env); + +void +wasm_cluster_destroy_spawned_exec_env(WASMExecEnv *exec_env); + +void +wasm_cluster_spread_custom_data(WASMModuleInstanceCommon *module_inst, + void *custom_data); + +void +wasm_cluster_set_context(WASMModuleInstanceCommon *module_inst, void *key, + void *ctx); + +bool +wasm_cluster_is_thread_terminated(WASMExecEnv *exec_env); + +#if WASM_ENABLE_DEBUG_INTERP != 0 +#define WAMR_SIG_TRAP (5) +#define WAMR_SIG_STOP (19) +#define WAMR_SIG_TERM (15) +#define WAMR_SIG_SINGSTEP (0x1ff) + +#define STATUS_RUNNING (0) +#define STATUS_STOP (1) +#define STATUS_EXIT (2) +#define STATUS_STEP (3) + +#define IS_WAMR_TERM_SIG(signo) ((signo) == WAMR_SIG_TERM) + +#define IS_WAMR_STOP_SIG(signo) \ + ((signo) == WAMR_SIG_STOP || (signo) == WAMR_SIG_TRAP) + +struct WASMCurrentEnvStatus { + uint64 signal_flag : 32; + uint64 step_count : 16; + uint64 running_status : 16; +}; + +WASMCurrentEnvStatus * +wasm_cluster_create_exenv_status(); + +void +wasm_cluster_destroy_exenv_status(WASMCurrentEnvStatus *status); + +void +wasm_cluster_send_signal_all(WASMCluster *cluster, uint32 signo); + +/* This function must be called with exec_env->wait_lock locked, otherwise we + * may miss the signal from debugger thread, see + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/1860 */ +void +wasm_cluster_thread_waiting_run(WASMExecEnv *exec_env); + +void +wasm_cluster_wait_thread_status(WASMExecEnv *exec_env, uint32 *status); + +void +wasm_cluster_thread_exited(WASMExecEnv *exec_env); + +void +wasm_cluster_thread_continue(WASMExecEnv *exec_env); + +void +wasm_cluster_thread_send_signal(WASMExecEnv *exec_env, uint32 signo); + +void +wasm_cluster_thread_step(WASMExecEnv *exec_env); + +void +wasm_cluster_set_debug_inst(WASMCluster *cluster, WASMDebugInstance *inst); + +#endif /* end of WASM_ENABLE_DEBUG_INTERP != 0 */ + +void +wasm_cluster_traverse_lock(WASMExecEnv *exec_env); + +void +wasm_cluster_traverse_unlock(WASMExecEnv *exec_env); + +bool +wasm_cluster_allocate_aux_stack(WASMExecEnv *exec_env, uint64 *p_start, + uint32 *p_size); + +bool +wasm_cluster_free_aux_stack(WASMExecEnv *exec_env, uint64 start); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _THREAD_MANAGER_H */ diff --git a/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_mgr.cmake b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_mgr.cmake new file mode 100644 index 0000000..c37a336 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/thread-mgr/thread_mgr.cmake @@ -0,0 +1,13 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (THREAD_MGR_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_THREAD_MGR=1) + +include_directories(${THREAD_MGR_DIR}) + +file (GLOB source_all ${THREAD_MGR_DIR}/*.c) + +set (THREAD_MGR_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/.gitignore b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/.gitignore new file mode 100644 index 0000000..81c0ff6 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/.gitignore @@ -0,0 +1,2 @@ +**/*.wasm +**/*.tflite diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/README.md b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/README.md new file mode 100644 index 0000000..fbc99cc --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/README.md @@ -0,0 +1,114 @@ +# WASI-NN + +## How to use + +Enable WASI-NN in the WAMR by spefiying it in the cmake building configuration as follows, + +``` +set (WAMR_BUILD_WASI_NN 1) +``` + +The definition of the functions provided by WASI-NN is in the header file `core/iwasm/libraries/wasi-nn/wasi_nn.h`. + +By only including this file in your WASM application you will bind WASI-NN into your module. + +## Tests + +To run the tests we assume that the current directory is the root of the repository. + + +### Build the runtime + +Build the runtime image for your execution target type. + +`EXECUTION_TYPE` can be: +* `cpu` +* `nvidia-gpu` +* `vx-delegate` +* `tpu` + +``` +EXECUTION_TYPE=cpu +docker build -t wasi-nn-${EXECUTION_TYPE} -f core/iwasm/libraries/wasi-nn/test/Dockerfile.${EXECUTION_TYPE} . +``` + + +### Build wasm app + +``` +docker build -t wasi-nn-compile -f core/iwasm/libraries/wasi-nn/test/Dockerfile.compile . +``` + +``` +docker run -v $PWD/core/iwasm/libraries/wasi-nn:/wasi-nn wasi-nn-compile +``` + + +### Run wasm app + +If all the tests have run properly you will the the following message in the terminal, + +``` +Tests: passed! +``` + +* CPU + +``` +docker run \ + -v $PWD/core/iwasm/libraries/wasi-nn/test:/assets \ + -v $PWD/core/iwasm/libraries/wasi-nn/test/models:/models \ + wasi-nn-cpu \ + --dir=/ \ + --env="TARGET=cpu" \ + /assets/test_tensorflow.wasm +``` + +* (NVIDIA) GPU + * Requirements: + * [NVIDIA docker](https://github.com/NVIDIA/nvidia-docker). + +``` +docker run \ + --runtime=nvidia \ + -v $PWD/core/iwasm/libraries/wasi-nn/test:/assets \ + -v $PWD/core/iwasm/libraries/wasi-nn/test/models:/models \ + wasi-nn-nvidia-gpu \ + --dir=/ \ + --env="TARGET=gpu" \ + /assets/test_tensorflow.wasm +``` + +* vx-delegate for NPU (x86 simulator) + +``` +docker run \ + -v $PWD/core/iwasm/libraries/wasi-nn/test:/assets \ + wasi-nn-vx-delegate \ + --dir=/ \ + --env="TARGET=gpu" \ + /assets/test_tensorflow_quantized.wasm +``` + +* (Coral) TPU + * Requirements: + * [Coral USB](https://coral.ai/products/accelerator/). + +``` +docker run \ + --privileged \ + --device=/dev/bus/usb:/dev/bus/usb \ + -v $PWD/core/iwasm/libraries/wasi-nn/test:/assets \ + wasi-nn-tpu \ + --dir=/ \ + --env="TARGET=tpu" \ + /assets/test_tensorflow_quantized.wasm +``` + +## What is missing + +Supported: + +* Graph encoding: `tensorflowlite`. +* Execution target: `cpu`, `gpu` and `tpu`. +* Tensor type: `fp32`. diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake new file mode 100644 index 0000000..59fdaf7 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake @@ -0,0 +1,42 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + + +find_library(TENSORFLOW_LITE + NAMES tensorflow-lite +) + +if(NOT EXISTS ${TENSORFLOW_LITE}) + if(NOT EXISTS "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") + execute_process( + COMMAND "${WAMR_ROOT_DIR}/core/deps/install_tensorflow.sh" + RESULT_VARIABLE TENSORFLOW_RESULT + ) + else() + message("Tensorflow is already downloaded.") + endif() + + set(TENSORFLOW_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") + + if(WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) + # Tensorflow specific: + # * https://www.tensorflow.org/lite/guide/build_cmake#available_options_to_build_tensorflow_lite + set (TFLITE_ENABLE_GPU ON) + endif() + + if (CMAKE_SIZEOF_VOID_P EQUAL 4) + set (TFLITE_ENABLE_XNNPACK OFF) + endif() + + add_subdirectory( + "${TENSORFLOW_SOURCE_DIR}/tensorflow/lite" + "${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite" + EXCLUDE_FROM_ALL + ) + + set(TENSORFLOW_LITE_INCLUDE_DIR "${TENSORFLOW_SOURCE_DIR}") + set(FLATBUFFER_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers/include") + + include_directories(${TENSORFLOW_LITE_INCLUDE_DIR}) + include_directories(${FLATBUFFER_INCLUDE_DIR}) +endif() diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/iwasm_helper.cmake b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/iwasm_helper.cmake new file mode 100644 index 0000000..670988e --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/iwasm_helper.cmake @@ -0,0 +1,46 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +if (NOT DEFINED WAMR_BUILD_PLATFORM) + string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +endif () + +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +set (CMAKE_C_STANDARD 99) + +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow -Wno-unused-parameter") + +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") + +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_WASI 1) +set (WAMR_BUILD_FAST_INTERP 1) + +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake new file mode 100644 index 0000000..48af9ba --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) + +# Find tensorflow-lite +find_package(tensorflow_lite REQUIRED) + +set(WASI_NN_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/..) + +include_directories (${WASI_NN_ROOT_DIR}/include) +include_directories (${WASI_NN_ROOT_DIR}/src) +include_directories (${WASI_NN_ROOT_DIR}/src/utils) + +set ( + WASI_NN_SOURCES + ${WASI_NN_ROOT_DIR}/src/wasi_nn.c + ${WASI_NN_ROOT_DIR}/src/wasi_nn_tensorflowlite.cpp + ${WASI_NN_ROOT_DIR}/src/utils/wasi_nn_app_native.c +) + +set (WASI_NN_LIBS tensorflow-lite) diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/external/CMakeLists.txt b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/external/CMakeLists.txt new file mode 100644 index 0000000..4157e19 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/external/CMakeLists.txt @@ -0,0 +1,58 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.16) +project(wasi-nn C CXX) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../../..) +set(WASI_NN_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/..) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif() + +#### libvmlib #### +# NOTE: we build vmlib as a shared library here so that it can be +# shared between iwasm and native libraries. +include(${WASI_NN_ROOT_DIR}/cmake/iwasm_helper.cmake) +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib SHARED ${WAMR_RUNTIME_LIB_SOURCE}) + +# iwasm +include(${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) +set(RUNTIME_SOURCE_ALL + ${WAMR_ROOT_DIR}/product-mini/platforms/${WAMR_BUILD_PLATFORM}/main.c + ${UNCOMMON_SHARED_SOURCE} +) + +add_executable(iwasm ${RUNTIME_SOURCE_ALL}) +target_link_libraries(iwasm vmlib -lpthread -lm -ldl) + +#### TensorFlow #### + +include(${WASI_NN_ROOT_DIR}/cmake/wasi_nn.cmake) + +#### WASI-NN #### + +include_directories( + ${WAMR_ROOT_DIR}/core/iwasm/include + ${WAMR_ROOT_DIR}/core/shared/utils + ${WAMR_ROOT_DIR}/core/shared/platform/linux +) + +add_library(wasi-nn SHARED + ${WASI_NN_SOURCES} +) + +# Add `get_native_lib` symbol +target_compile_definitions(wasi-nn PUBLIC + WASI_NN_SHARED +) + +target_link_libraries(wasi-nn + ${WASI_NN_LIBS} + vmlib +) diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/external/README.md b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/external/README.md new file mode 100644 index 0000000..b255185 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/external/README.md @@ -0,0 +1,13 @@ +# wasi-nn as shared library + +Example on how to create libwasi-nn (external library) instead of embedding wasi-nn inside iwasm + +From folder `core/iwasm/libraries/wasi-nn/test`, build the test and run + +```sh +../external/build/iwasm \ + --dir=. \ + --env="TARGET=cpu" \ + --native-lib=../external/build/libwasi-nn.so \ + test_tensorflow.wasm +``` diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/include/wasi_nn.h b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/include/wasi_nn.h new file mode 100644 index 0000000..2bf0a19 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/include/wasi_nn.h @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * Following definition from: + * [Oct 25th, 2022] + * https://github.com/WebAssembly/wasi-nn/blob/0f77c48ec195748990ff67928a4b3eef5f16c2de/wasi-nn.wit.md + */ + +#ifndef WASI_NN_H +#define WASI_NN_H + +#include +#include "wasi_nn_types.h" + +/** + * @brief Load an opaque sequence of bytes to use for inference. + * + * @param builder Model builder. + * @param encoding Model encoding. + * @param target Execution target. + * @param g Graph. + * @return error Execution status. + */ +error +load(graph_builder_array *builder, graph_encoding encoding, + execution_target target, graph *g) + __attribute__((import_module("wasi_nn"))); + +/** + * INFERENCE + * + */ + +// Bind a `graph` to the input and output tensors for an inference. +typedef uint32_t graph_execution_context; + +/** + * @brief Create an execution instance of a loaded graph. + * + * @param g Graph. + * @param ctx Execution context. + * @return error Execution status. + */ +error +init_execution_context(graph g, graph_execution_context *ctx) + __attribute__((import_module("wasi_nn"))); + +/** + * @brief Define the inputs to use for inference. + * + * @param ctx Execution context. + * @param index Input tensor index. + * @param tensor Input tensor. + * @return error Execution status. + */ +error +set_input(graph_execution_context ctx, uint32_t index, tensor *tensor) + __attribute__((import_module("wasi_nn"))); + +/** + * @brief Compute the inference on the given inputs. + * + * @param ctx Execution context. + * @return error Execution status. + */ +error +compute(graph_execution_context ctx) __attribute__((import_module("wasi_nn"))); + +/** + * @brief Extract the outputs after inference. + * + * @param ctx Execution context. + * @param index Output tensor index. + * @param output_tensor Buffer where output tensor with index `index` is + * copied. + * @param output_tensor_size Pointer to `output_tensor` maximum size. + * After the function call it is updated with the + * copied number of bytes. + * @return error Execution status. + */ +error +get_output(graph_execution_context ctx, uint32_t index, + tensor_data output_tensor, uint32_t *output_tensor_size) + __attribute__((import_module("wasi_nn"))); + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h new file mode 100644 index 0000000..7cfc70f --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_NN_TYPES_H +#define WASI_NN_TYPES_H + +#include +#include + +/** + * ERRORS + * + */ + +// Error codes returned by functions in this API. +typedef enum { + // No error occurred. + success = 0, + // Caller module passed an invalid argument. + invalid_argument, + // Invalid encoding. + invalid_encoding, + // Caller module is missing a memory export. + missing_memory, + // Device or resource busy. + busy, + // Runtime Error. + runtime_error, +} error; + +/** + * TENSOR + * + */ + +// The dimensions of a tensor. +// +// The array length matches the tensor rank and each element in the array +// describes the size of each dimension. +typedef struct { + uint32_t *buf; + uint32_t size; +} tensor_dimensions; + +// The type of the elements in a tensor. +typedef enum { fp16 = 0, fp32, up8, ip32 } tensor_type; + +// The tensor data. +// +// Initially conceived as a sparse representation, each empty cell would be +// filled with zeros and the array length must match the product of all of the +// dimensions and the number of bytes in the type (e.g., a 2x2 tensor with +// 4-byte f32 elements would have a data array of length 16). Naturally, this +// representation requires some knowledge of how to lay out data in +// memory--e.g., using row-major ordering--and could perhaps be improved. +typedef uint8_t *tensor_data; + +// A tensor. +typedef struct { + // Describe the size of the tensor (e.g., 2x2x2x2 -> [2, 2, 2, 2]). To + // represent a tensor containing a single value, use `[1]` for the tensor + // dimensions. + tensor_dimensions *dimensions; + // Describe the type of element in the tensor (e.g., f32). + tensor_type type; + // Contains the tensor data. + tensor_data data; +} tensor; + +/** + * GRAPH + * + */ + +// The graph initialization data. +// +// This consists of an array of buffers because implementing backends may encode +// their graph IR in parts (e.g., OpenVINO stores its IR and weights +// separately). +typedef struct { + uint8_t *buf; + uint32_t size; +} graph_builder; + +typedef struct { + graph_builder *buf; + uint32_t size; +} graph_builder_array; + +// An execution graph for performing inference (i.e., a model). +typedef uint32_t graph; + +// Describes the encoding of the graph. This allows the API to be implemented by +// various backends that encode (i.e., serialize) their graph IR with different +// formats. +typedef enum { + openvino = 0, + onnx, + tensorflow, + pytorch, + tensorflowlite +} graph_encoding; + +// Define where the graph should be executed. +typedef enum execution_target { cpu = 0, gpu, tpu } execution_target; + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/logger.h b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/logger.h new file mode 100644 index 0000000..a196429 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/logger.h @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_NN_LOGGER_H +#define WASI_NN_LOGGER_H + +#include +#include + +#define __FILENAME__ \ + (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) + +/* Disable a level by removing the define */ +#ifndef NN_LOG_LEVEL +/* + 0 -> debug, info, warn, err + 1 -> info, warn, err + 2 -> warn, err + 3 -> err + 4 -> NO LOGS +*/ +#define NN_LOG_LEVEL 0 +#endif + +// Definition of the levels +#if NN_LOG_LEVEL <= 3 +#define NN_ERR_PRINTF(fmt, ...) \ + do { \ + printf("[%s:%d ERROR] " fmt, __FILENAME__, __LINE__, ##__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } while (0) +#else +#define NN_ERR_PRINTF(fmt, ...) +#endif +#if NN_LOG_LEVEL <= 2 +#define NN_WARN_PRINTF(fmt, ...) \ + do { \ + printf("[%s:%d WARNING] " fmt, __FILENAME__, __LINE__, ##__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } while (0) +#else +#define NN_WARN_PRINTF(fmt, ...) +#endif +#if NN_LOG_LEVEL <= 1 +#define NN_INFO_PRINTF(fmt, ...) \ + do { \ + printf("[%s:%d INFO] " fmt, __FILENAME__, __LINE__, ##__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } while (0) +#else +#define NN_INFO_PRINTF(fmt, ...) +#endif +#if NN_LOG_LEVEL <= 0 +#define NN_DBG_PRINTF(fmt, ...) \ + do { \ + printf("[%s:%d DEBUG] " fmt, __FILENAME__, __LINE__, ##__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } while (0) +#else +#define NN_DBG_PRINTF(fmt, ...) +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c new file mode 100644 index 0000000..44aef53 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasi_nn_app_native.h" + +static error +graph_builder_app_native(wasm_module_inst_t instance, + graph_builder_wasm *builder_wasm, + graph_builder *builder) +{ + if (!wasm_runtime_validate_app_addr( + instance, (uint64)builder_wasm->buf_offset, + (uint64)builder_wasm->size * sizeof(uint8_t))) { + NN_ERR_PRINTF("builder_wasm->buf_offset is invalid"); + return invalid_argument; + } + + builder->buf = (uint8_t *)wasm_runtime_addr_app_to_native( + instance, (uint64)builder_wasm->buf_offset); + builder->size = builder_wasm->size; + return success; +} + +/** + * builder_array_wasm is consisted of {builder_wasm, size} + */ +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +graph_builder_array_app_native(wasm_module_inst_t instance, + graph_builder_wasm *builder_wasm, uint32_t size, + graph_builder_array *builder_array) +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +error +graph_builder_array_app_native(wasm_module_inst_t instance, + graph_builder_array_wasm *builder_array_wasm, + graph_builder_array *builder_array) +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ +{ +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +#define array_size size +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +#define array_size builder_array_wasm->size + + if (!wasm_runtime_validate_native_addr( + instance, builder_array_wasm, + (uint64)sizeof(graph_builder_array_wasm))) { + NN_ERR_PRINTF("builder_array_wasm is invalid"); + return invalid_argument; + } +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + + NN_DBG_PRINTF("Graph builder array contains %d elements", array_size); + +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + if (!wasm_runtime_validate_native_addr(instance, builder_wasm, + (uint64)array_size + * sizeof(graph_builder_wasm))) { + NN_ERR_PRINTF("builder_wasm is invalid"); + return invalid_argument; + } +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ + if (!wasm_runtime_validate_app_addr( + instance, (uint64)builder_array_wasm->buf_offset, + (uint64)array_size * sizeof(graph_builder_wasm))) { + NN_ERR_PRINTF("builder_array_wasm->buf_offset is invalid"); + return invalid_argument; + } + + graph_builder_wasm *builder_wasm = + (graph_builder_wasm *)wasm_runtime_addr_app_to_native( + instance, (uint64)builder_array_wasm->buf_offset); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + + graph_builder *builder = (graph_builder *)wasm_runtime_malloc( + array_size * sizeof(graph_builder)); + if (builder == NULL) + return missing_memory; + + for (uint32_t i = 0; i < array_size; ++i) { + error res; + if (success + != (res = graph_builder_app_native(instance, &builder_wasm[i], + &builder[i]))) { + wasm_runtime_free(builder); + return res; + } + + NN_DBG_PRINTF("Graph builder %d contains %d elements", i, + builder->size); + } + + builder_array->buf = builder; + builder_array->size = array_size; + return success; +#undef array_size +} + +static error +tensor_data_app_native(wasm_module_inst_t instance, uint32_t total_elements, + tensor_wasm *input_tensor_wasm, tensor_data *data) +{ +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +#define data_size input_tensor_wasm->data_size +#else +#define data_size total_elements +#endif + + if (!wasm_runtime_validate_app_addr(instance, + (uint64)input_tensor_wasm->data_offset, + (uint64)data_size)) { + NN_ERR_PRINTF("input_tensor_wasm->data_offset is invalid"); + return invalid_argument; + } + *data = (tensor_data)wasm_runtime_addr_app_to_native( + instance, (uint64)input_tensor_wasm->data_offset); + return success; +#undef data_size +} + +static error +tensor_dimensions_app_native(wasm_module_inst_t instance, + tensor_wasm *input_tensor_wasm, + tensor_dimensions **dimensions) +{ +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + tensor_dimensions_wasm *dimensions_wasm = &input_tensor_wasm->dimensions; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ + if (!wasm_runtime_validate_app_addr( + instance, (uint64)input_tensor_wasm->dimensions_offset, + (uint64)sizeof(tensor_dimensions_wasm))) { + NN_ERR_PRINTF("input_tensor_wasm->dimensions_offset is invalid"); + return invalid_argument; + } + + tensor_dimensions_wasm *dimensions_wasm = + (tensor_dimensions_wasm *)wasm_runtime_addr_app_to_native( + instance, (uint64)input_tensor_wasm->dimensions_offset); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + + if (!wasm_runtime_validate_app_addr(instance, + (uint64)dimensions_wasm->buf_offset, + (uint64)sizeof(tensor_dimensions))) { + NN_ERR_PRINTF("dimensions_wasm->buf_offset is invalid"); + return invalid_argument; + } + + *dimensions = + (tensor_dimensions *)wasm_runtime_malloc(sizeof(tensor_dimensions)); + if (dimensions == NULL) + return missing_memory; + + (*dimensions)->size = dimensions_wasm->size; + (*dimensions)->buf = (uint32_t *)wasm_runtime_addr_app_to_native( + instance, (uint64)dimensions_wasm->buf_offset); + + NN_DBG_PRINTF("Number of dimensions: %d", (*dimensions)->size); + return success; +} + +error +tensor_app_native(wasm_module_inst_t instance, tensor_wasm *input_tensor_wasm, + tensor *input_tensor) +{ + NN_DBG_PRINTF("Converting tensor_wasm to tensor"); + if (!wasm_runtime_validate_native_addr(instance, input_tensor_wasm, + (uint64)sizeof(tensor_wasm))) { + NN_ERR_PRINTF("input_tensor_wasm is invalid"); + return invalid_argument; + } + + error res; + + tensor_dimensions *dimensions = NULL; + if (success + != (res = tensor_dimensions_app_native(instance, input_tensor_wasm, + &dimensions))) { + NN_ERR_PRINTF("error when parsing dimensions"); + return res; + } + + uint32_t total_elements = 1; + for (uint32_t i = 0; i < dimensions->size; ++i) { + total_elements *= dimensions->buf[i]; + NN_DBG_PRINTF("Dimension %d: %d", i, dimensions->buf[i]); + } + NN_DBG_PRINTF("Tensor type: %d", input_tensor_wasm->type); + NN_DBG_PRINTF("Total number of elements: %d", total_elements); + + tensor_data data = NULL; + if (success + != (res = tensor_data_app_native(instance, total_elements, + input_tensor_wasm, &data))) { + wasm_runtime_free(dimensions); + return res; + } + + input_tensor->type = input_tensor_wasm->type; + input_tensor->dimensions = dimensions; + input_tensor->data = data; + return success; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.h b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.h new file mode 100644 index 0000000..f0930a8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_NN_APP_NATIVE +#define WASI_NN_APP_NATIVE + +#include +#include +#include +#include +#include + +#include "wasi_nn.h" +#include "logger.h" + +#include "bh_platform.h" +#include "wasm_export.h" + +typedef struct { + uint32_t buf_offset; + uint32_t size; +} graph_builder_wasm; + +typedef struct { + uint32_t buf_offset; + uint32_t size; +} graph_builder_array_wasm; + +typedef struct { + uint32_t buf_offset; + uint32_t size; +} tensor_dimensions_wasm; + +typedef struct { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + tensor_dimensions_wasm dimensions; + tensor_type type; + uint32_t data_offset; + uint32_t data_size; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ + uint32_t dimensions_offset; + tensor_type type; + uint32_t data_offset; +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ +} tensor_wasm; + +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +graph_builder_array_app_native(wasm_module_inst_t instance, + graph_builder_wasm *builder_wasm, uint32_t size, + graph_builder_array *builder_array); +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +error +graph_builder_array_app_native(wasm_module_inst_t instance, + graph_builder_array_wasm *builder, + graph_builder_array *builder_native); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + +error +tensor_app_native(wasm_module_inst_t instance, tensor_wasm *input_tensor, + tensor *input_tensor_native); + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn.c b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn.c new file mode 100644 index 0000000..1fbb944 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn.c @@ -0,0 +1,413 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "wasi_nn.h" +#include "wasi_nn_private.h" +#include "wasi_nn_app_native.h" +#include "wasi_nn_tensorflowlite.hpp" +#include "logger.h" + +#include "bh_platform.h" +#include "wasm_export.h" + +#define HASHMAP_INITIAL_SIZE 20 + +/* Definition of 'wasi_nn.h' structs in WASM app format (using offset) */ + +typedef error (*LOAD)(void *, graph_builder_array *, graph_encoding, + execution_target, graph *); +typedef error (*INIT_EXECUTION_CONTEXT)(void *, graph, + graph_execution_context *); +typedef error (*SET_INPUT)(void *, graph_execution_context, uint32_t, tensor *); +typedef error (*COMPUTE)(void *, graph_execution_context); +typedef error (*GET_OUTPUT)(void *, graph_execution_context, uint32_t, + tensor_data, uint32_t *); + +typedef struct { + LOAD load; + INIT_EXECUTION_CONTEXT init_execution_context; + SET_INPUT set_input; + COMPUTE compute; + GET_OUTPUT get_output; +} api_function; + +/* Global variables */ + +static api_function lookup[] = { + { NULL, NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL }, + { tensorflowlite_load, tensorflowlite_init_execution_context, + tensorflowlite_set_input, tensorflowlite_compute, + tensorflowlite_get_output } +}; + +static HashMap *hashmap; + +static void +wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx); + +/* Get wasi-nn context from module instance */ + +static uint32 +hash_func(const void *key) +{ + // fnv1a_hash + const uint32 FNV_PRIME = 16777619; + const uint32 FNV_OFFSET_BASIS = 2166136261U; + + uint32 hash = FNV_OFFSET_BASIS; + const unsigned char *bytes = (const unsigned char *)key; + + for (size_t i = 0; i < sizeof(uintptr_t); ++i) { + hash ^= bytes[i]; + hash *= FNV_PRIME; + } + + return hash; +} + +static bool +key_equal_func(void *key1, void *key2) +{ + return key1 == key2; +} + +static void +key_destroy_func(void *key1) +{} + +static void +value_destroy_func(void *value) +{ + wasi_nn_ctx_destroy((WASINNContext *)value); +} + +static WASINNContext * +wasi_nn_initialize_context() +{ + NN_DBG_PRINTF("Initializing wasi-nn context"); + WASINNContext *wasi_nn_ctx = + (WASINNContext *)wasm_runtime_malloc(sizeof(WASINNContext)); + if (wasi_nn_ctx == NULL) { + NN_ERR_PRINTF("Error when allocating memory for WASI-NN context"); + return NULL; + } + wasi_nn_ctx->is_model_loaded = false; + tensorflowlite_initialize(&wasi_nn_ctx->tflite_ctx); + return wasi_nn_ctx; +} + +static bool +wasi_nn_initialize() +{ + NN_DBG_PRINTF("Initializing wasi-nn"); + hashmap = bh_hash_map_create(HASHMAP_INITIAL_SIZE, true, hash_func, + key_equal_func, key_destroy_func, + value_destroy_func); + if (hashmap == NULL) { + NN_ERR_PRINTF("Error while initializing hashmap"); + return false; + } + return true; +} + +static WASINNContext * +wasm_runtime_get_wasi_nn_ctx(wasm_module_inst_t instance) +{ + WASINNContext *wasi_nn_ctx = + (WASINNContext *)bh_hash_map_find(hashmap, (void *)instance); + if (wasi_nn_ctx == NULL) { + wasi_nn_ctx = wasi_nn_initialize_context(); + if (wasi_nn_ctx == NULL) + return NULL; + bool ok = + bh_hash_map_insert(hashmap, (void *)instance, (void *)wasi_nn_ctx); + if (!ok) { + NN_ERR_PRINTF("Error while storing context"); + wasi_nn_ctx_destroy(wasi_nn_ctx); + return NULL; + } + } + NN_DBG_PRINTF("Returning ctx"); + return wasi_nn_ctx; +} + +static void +wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx) +{ + if (wasi_nn_ctx == NULL) { + NN_ERR_PRINTF( + "Error when deallocating memory. WASI-NN context is NULL"); + return; + } + NN_DBG_PRINTF("Freeing wasi-nn"); + NN_DBG_PRINTF("-> is_model_loaded: %d", wasi_nn_ctx->is_model_loaded); + NN_DBG_PRINTF("-> current_encoding: %d", wasi_nn_ctx->current_encoding); + tensorflowlite_destroy(wasi_nn_ctx->tflite_ctx); + wasm_runtime_free(wasi_nn_ctx); +} + +void +wasi_nn_destroy(wasm_module_inst_t instance) +{ + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + bh_hash_map_remove(hashmap, (void *)instance, NULL, NULL); + wasi_nn_ctx_destroy(wasi_nn_ctx); +} + +/* Utils */ + +static bool +is_encoding_implemented(graph_encoding encoding) +{ + return lookup[encoding].load && lookup[encoding].init_execution_context + && lookup[encoding].set_input && lookup[encoding].compute + && lookup[encoding].get_output; +} + +static error +is_model_initialized(WASINNContext *wasi_nn_ctx) +{ + if (!wasi_nn_ctx->is_model_loaded) { + NN_ERR_PRINTF("Model not initialized."); + return runtime_error; + } + return success; +} + +/* WASI-NN implementation */ + +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_wasm *builder, + uint32_t builder_wasm_size, graph_encoding encoding, + execution_target target, graph *g) +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +error +wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, + graph_encoding encoding, execution_target target, graph *g) +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ +{ + NN_DBG_PRINTF("Running wasi_nn_load [encoding=%d, target=%d]...", encoding, + target); + + if (!is_encoding_implemented(encoding)) { + NN_ERR_PRINTF("Encoding not supported."); + return invalid_encoding; + } + + wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); + bh_assert(instance); + + error res; + graph_builder_array builder_native = { 0 }; +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + if (success + != (res = graph_builder_array_app_native( + instance, builder, builder_wasm_size, &builder_native))) + return res; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ + if (success + != (res = graph_builder_array_app_native(instance, builder, + &builder_native))) + return res; +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + + if (!wasm_runtime_validate_native_addr(instance, g, + (uint64)sizeof(graph))) { + NN_ERR_PRINTF("graph is invalid"); + res = invalid_argument; + goto fail; + } + + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + res = lookup[encoding].load(wasi_nn_ctx->tflite_ctx, &builder_native, + encoding, target, g); + + NN_DBG_PRINTF("wasi_nn_load finished with status %d [graph=%d]", res, *g); + + wasi_nn_ctx->current_encoding = encoding; + wasi_nn_ctx->is_model_loaded = true; + +fail: + // XXX: Free intermediate structure pointers + if (builder_native.buf) + wasm_runtime_free(builder_native.buf); + + return res; +} + +error +wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g, + graph_execution_context *ctx) +{ + NN_DBG_PRINTF("Running wasi_nn_init_execution_context [graph=%d]...", g); + + wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); + bh_assert(instance); + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + + error res; + if (success != (res = is_model_initialized(wasi_nn_ctx))) + return res; + + if (!wasm_runtime_validate_native_addr( + instance, ctx, (uint64)sizeof(graph_execution_context))) { + NN_ERR_PRINTF("ctx is invalid"); + return invalid_argument; + } + + res = lookup[wasi_nn_ctx->current_encoding].init_execution_context( + wasi_nn_ctx->tflite_ctx, g, ctx); + + NN_DBG_PRINTF( + "wasi_nn_init_execution_context finished with status %d [ctx=%d]", res, + *ctx); + return res; +} + +error +wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx, + uint32_t index, tensor_wasm *input_tensor) +{ + NN_DBG_PRINTF("Running wasi_nn_set_input [ctx=%d, index=%d]...", ctx, + index); + + wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); + bh_assert(instance); + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + + error res; + if (success != (res = is_model_initialized(wasi_nn_ctx))) + return res; + + tensor input_tensor_native = { 0 }; + if (success + != (res = tensor_app_native(instance, input_tensor, + &input_tensor_native))) + return res; + + res = lookup[wasi_nn_ctx->current_encoding].set_input( + wasi_nn_ctx->tflite_ctx, ctx, index, &input_tensor_native); + + // XXX: Free intermediate structure pointers + if (input_tensor_native.dimensions) + wasm_runtime_free(input_tensor_native.dimensions); + + NN_DBG_PRINTF("wasi_nn_set_input finished with status %d", res); + return res; +} + +error +wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx) +{ + NN_DBG_PRINTF("Running wasi_nn_compute [ctx=%d]...", ctx); + + wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); + bh_assert(instance); + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + + error res; + if (success != (res = is_model_initialized(wasi_nn_ctx))) + return res; + + res = lookup[wasi_nn_ctx->current_encoding].compute(wasi_nn_ctx->tflite_ctx, + ctx); + NN_DBG_PRINTF("wasi_nn_compute finished with status %d", res); + return res; +} + +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, + uint32_t index, tensor_data output_tensor, + uint32_t output_tensor_len, uint32_t *output_tensor_size) +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +error +wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, + uint32_t index, tensor_data output_tensor, + uint32_t *output_tensor_size) +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ +{ + NN_DBG_PRINTF("Running wasi_nn_get_output [ctx=%d, index=%d]...", ctx, + index); + + wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); + bh_assert(instance); + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + + error res; + if (success != (res = is_model_initialized(wasi_nn_ctx))) + return res; + + if (!wasm_runtime_validate_native_addr(instance, output_tensor_size, + (uint64)sizeof(uint32_t))) { + NN_ERR_PRINTF("output_tensor_size is invalid"); + return invalid_argument; + } + +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + res = lookup[wasi_nn_ctx->current_encoding].get_output( + wasi_nn_ctx->tflite_ctx, ctx, index, output_tensor, &output_tensor_len); + *output_tensor_size = output_tensor_len; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ + res = lookup[wasi_nn_ctx->current_encoding].get_output( + wasi_nn_ctx->tflite_ctx, ctx, index, output_tensor, output_tensor_size); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + NN_DBG_PRINTF("wasi_nn_get_output finished with status %d [data_size=%d]", + res, *output_tensor_size); + return res; +} + +/* Register WASI-NN in WAMR */ + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, wasi_nn_##func_name, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_wasi_nn[] = { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + REG_NATIVE_FUNC(load, "(*iii*)i"), + REG_NATIVE_FUNC(init_execution_context, "(i*)i"), + REG_NATIVE_FUNC(set_input, "(ii*)i"), + REG_NATIVE_FUNC(compute, "(i)i"), + REG_NATIVE_FUNC(get_output, "(ii*i*)i"), +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ + REG_NATIVE_FUNC(load, "(*ii*)i"), + REG_NATIVE_FUNC(init_execution_context, "(i*)i"), + REG_NATIVE_FUNC(set_input, "(ii*)i"), + REG_NATIVE_FUNC(compute, "(i)i"), + REG_NATIVE_FUNC(get_output, "(ii**)i"), +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ +}; + +uint32_t +get_wasi_nn_export_apis(NativeSymbol **p_native_symbols) +{ + if (!wasi_nn_initialize()) + return 0; + *p_native_symbols = native_symbols_wasi_nn; + return sizeof(native_symbols_wasi_nn) / sizeof(NativeSymbol); +} + +#if defined(WASI_NN_SHARED) +uint32_t +get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) +{ + *p_module_name = "wasi_nn"; + return get_wasi_nn_export_apis(p_native_symbols); +} +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_private.h b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_private.h new file mode 100644 index 0000000..e66c274 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_private.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_NN_PRIVATE_H +#define WASI_NN_PRIVATE_H + +#include "wasi_nn_types.h" +#include "wasm_export.h" + +typedef struct { + bool is_model_loaded; + graph_encoding current_encoding; + void *tflite_ctx; +} WASINNContext; + +/** + * @brief Destroy wasi-nn on app exists + * + */ + +void +wasi_nn_destroy(wasm_module_inst_t instance); + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp new file mode 100644 index 0000000..65e38f8 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp @@ -0,0 +1,487 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasi_nn.h" +#include "wasi_nn_tensorflowlite.hpp" +#include "logger.h" + +#include "bh_platform.h" +#include "wasm_export.h" + +#include +#include +#include +#include +#include + +#if WASM_ENABLE_WASI_NN_GPU != 0 +#include +#endif + +#if WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE != 0 +#include +#endif + +/* Maximum number of graphs per WASM instance */ +#define MAX_GRAPHS_PER_INST 10 +/* Maximum number of graph execution context per WASM instance*/ +#define MAX_GRAPH_EXEC_CONTEXTS_PER_INST 10 + +typedef struct { + std::unique_ptr interpreter; +} Interpreter; + +typedef struct { + char *model_pointer; + std::unique_ptr model; + execution_target target; +} Model; + +typedef struct { + uint32_t current_models; + Model models[MAX_GRAPHS_PER_INST]; + uint32_t current_interpreters; + Interpreter interpreters[MAX_GRAPH_EXEC_CONTEXTS_PER_INST]; + korp_mutex g_lock; + TfLiteDelegate *delegate; +} TFLiteContext; + +/* Utils */ + +static error +initialize_g(TFLiteContext *tfl_ctx, graph *g) +{ + os_mutex_lock(&tfl_ctx->g_lock); + if (tfl_ctx->current_models == MAX_GRAPHS_PER_INST) { + os_mutex_unlock(&tfl_ctx->g_lock); + NN_ERR_PRINTF("Excedded max graphs per WASM instance"); + return runtime_error; + } + *g = tfl_ctx->current_models++; + os_mutex_unlock(&tfl_ctx->g_lock); + return success; +} +static error +initialize_graph_ctx(TFLiteContext *tfl_ctx, graph g, + graph_execution_context *ctx) +{ + os_mutex_lock(&tfl_ctx->g_lock); + if (tfl_ctx->current_interpreters == MAX_GRAPH_EXEC_CONTEXTS_PER_INST) { + os_mutex_unlock(&tfl_ctx->g_lock); + NN_ERR_PRINTF("Excedded max graph execution context per WASM instance"); + return runtime_error; + } + *ctx = tfl_ctx->current_interpreters++; + os_mutex_unlock(&tfl_ctx->g_lock); + return success; +} + +static error +is_valid_graph(TFLiteContext *tfl_ctx, graph g) +{ + if (g >= MAX_GRAPHS_PER_INST) { + NN_ERR_PRINTF("Invalid graph: %d >= %d.", g, MAX_GRAPHS_PER_INST); + return runtime_error; + } + if (tfl_ctx->models[g].model_pointer == NULL) { + NN_ERR_PRINTF("Context (model) non-initialized."); + return runtime_error; + } + if (tfl_ctx->models[g].model == NULL) { + NN_ERR_PRINTF("Context (tflite model) non-initialized."); + return runtime_error; + } + return success; +} + +static error +is_valid_graph_execution_context(TFLiteContext *tfl_ctx, + graph_execution_context ctx) +{ + if (ctx >= MAX_GRAPH_EXEC_CONTEXTS_PER_INST) { + NN_ERR_PRINTF("Invalid graph execution context: %d >= %d", ctx, + MAX_GRAPH_EXEC_CONTEXTS_PER_INST); + return runtime_error; + } + if (tfl_ctx->interpreters[ctx].interpreter == NULL) { + NN_ERR_PRINTF("Context (interpreter) non-initialized."); + return runtime_error; + } + return success; +} + +/* WASI-NN (tensorflow) implementation */ + +error +tensorflowlite_load(void *tflite_ctx, graph_builder_array *builder, + graph_encoding encoding, execution_target target, graph *g) +{ + TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; + + if (builder->size != 1) { + NN_ERR_PRINTF("Unexpected builder format."); + return invalid_argument; + } + + if (encoding != tensorflowlite) { + NN_ERR_PRINTF("Encoding is not tensorflowlite."); + return invalid_argument; + } + + if (target != cpu && target != gpu && target != tpu) { + NN_ERR_PRINTF("Only CPU, GPU and TPU target is supported."); + return invalid_argument; + } + + error res; + if (success != (res = initialize_g(tfl_ctx, g))) + return res; + + uint32_t size = builder->buf[0].size; + + // Save model + tfl_ctx->models[*g].model_pointer = (char *)wasm_runtime_malloc(size); + if (tfl_ctx->models[*g].model_pointer == NULL) { + NN_ERR_PRINTF("Error when allocating memory for model."); + return missing_memory; + } + + bh_memcpy_s(tfl_ctx->models[*g].model_pointer, size, builder->buf[0].buf, + size); + + // Save model flatbuffer + tfl_ctx->models[*g].model = + std::move(tflite::FlatBufferModel::BuildFromBuffer( + tfl_ctx->models[*g].model_pointer, size, NULL)); + + if (tfl_ctx->models[*g].model == NULL) { + NN_ERR_PRINTF("Loading model error."); + wasm_runtime_free(tfl_ctx->models[*g].model_pointer); + tfl_ctx->models[*g].model_pointer = NULL; + return missing_memory; + } + + // Save target + tfl_ctx->models[*g].target = target; + return success; +} + +error +tensorflowlite_init_execution_context(void *tflite_ctx, graph g, + graph_execution_context *ctx) +{ + TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; + + error res; + if (success != (res = is_valid_graph(tfl_ctx, g))) + return res; + + if (success != (res = initialize_graph_ctx(tfl_ctx, g, ctx))) + return res; + + // Build the interpreter with the InterpreterBuilder. + tflite::ops::builtin::BuiltinOpResolver resolver; + tflite::InterpreterBuilder tflite_builder(*tfl_ctx->models[g].model, + resolver); + tflite_builder(&tfl_ctx->interpreters[*ctx].interpreter); + if (tfl_ctx->interpreters[*ctx].interpreter == NULL) { + NN_ERR_PRINTF("Error when generating the interpreter."); + return missing_memory; + } + + bool use_default = false; + switch (tfl_ctx->models[g].target) { + case gpu: + { +#if WASM_ENABLE_WASI_NN_GPU != 0 + NN_WARN_PRINTF("GPU enabled."); + // https://www.tensorflow.org/lite/performance/gpu + TfLiteGpuDelegateOptionsV2 options = + TfLiteGpuDelegateOptionsV2Default(); + options.inference_preference = + TFLITE_GPU_INFERENCE_PREFERENCE_SUSTAINED_SPEED; + options.inference_priority1 = + TFLITE_GPU_INFERENCE_PRIORITY_MIN_LATENCY; + tfl_ctx->delegate = TfLiteGpuDelegateV2Create(&options); + if (tfl_ctx->delegate == NULL) { + NN_ERR_PRINTF("Error when generating GPU delegate."); + use_default = true; + return missing_memory; + } + if (tfl_ctx->interpreters[*ctx] + .interpreter->ModifyGraphWithDelegate(tfl_ctx->delegate) + != kTfLiteOk) { + NN_ERR_PRINTF("Error when enabling GPU delegate."); + use_default = true; + } +#else + NN_WARN_PRINTF("GPU not enabled."); + use_default = true; +#endif + break; + } + case tpu: + { +#if WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE != 0 + NN_WARN_PRINTF("external delegation enabled."); + TfLiteExternalDelegateOptions options = + TfLiteExternalDelegateOptionsDefault( + WASM_WASI_NN_EXTERNAL_DELEGATE_PATH); + tfl_ctx->delegate = TfLiteExternalDelegateCreate(&options); + if (tfl_ctx->delegate == NULL) { + NN_ERR_PRINTF("Error when generating External delegate."); + use_default = true; + return missing_memory; + } + if (tfl_ctx->interpreters[*ctx] + .interpreter->ModifyGraphWithDelegate(tfl_ctx->delegate) + != kTfLiteOk) { + NN_ERR_PRINTF("Error when enabling External delegate."); + use_default = true; + } +#else + NN_WARN_PRINTF("External delegate not enabled."); + use_default = true; +#endif + break; + } + default: + use_default = true; + } + if (use_default) + NN_WARN_PRINTF("Default encoding is CPU."); + + tfl_ctx->interpreters[*ctx].interpreter->AllocateTensors(); + return success; +} + +error +tensorflowlite_set_input(void *tflite_ctx, graph_execution_context ctx, + uint32_t index, tensor *input_tensor) +{ + TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; + + error res; + if (success != (res = is_valid_graph_execution_context(tfl_ctx, ctx))) + return res; + + uint32_t num_tensors = + tfl_ctx->interpreters[ctx].interpreter->inputs().size(); + NN_DBG_PRINTF("Number of tensors (%d)", num_tensors); + if (index + 1 > num_tensors) { + return runtime_error; + } + + auto tensor = tfl_ctx->interpreters[ctx].interpreter->input_tensor(index); + if (tensor == NULL) { + NN_ERR_PRINTF("Missing memory"); + return missing_memory; + } + + uint32_t model_tensor_size = 1; + for (int i = 0; i < tensor->dims->size; ++i) + model_tensor_size *= (uint32_t)tensor->dims->data[i]; + + uint32_t input_tensor_size = 1; + for (uint32_t i = 0; i < input_tensor->dimensions->size; i++) + input_tensor_size *= (uint32_t)input_tensor->dimensions->buf[i]; + + if (model_tensor_size != input_tensor_size) { + NN_ERR_PRINTF("Input tensor shape from the model is different than the " + "one provided"); + return invalid_argument; + } + + if (tensor->quantization.type == kTfLiteNoQuantization) { + NN_DBG_PRINTF("No quantization information. Using float as default"); + float *it = + tfl_ctx->interpreters[ctx].interpreter->typed_input_tensor( + index); + + int size = model_tensor_size * sizeof(float); + bh_memcpy_s(it, size, input_tensor->data, size); + } + else { // TODO: Assumming uint8 quantized networks. + TfLiteAffineQuantization *quant_info = + (TfLiteAffineQuantization *)tensor->quantization.params; + if (quant_info->scale->size != 1 || quant_info->zero_point->size != 1) { + NN_ERR_PRINTF("Quantization per channel is not supported"); + return runtime_error; + } + uint8_t *it = + tfl_ctx->interpreters[ctx].interpreter->typed_input_tensor( + index); + + float scale = quant_info->scale->data[0]; + float zero_point = (float)quant_info->zero_point->data[0]; + NN_DBG_PRINTF("input tensor: (scale, offset) = (%f, %f)", scale, + zero_point); + + float *input_tensor_f = (float *)input_tensor->data; + for (uint32_t i = 0; i < model_tensor_size; ++i) { + it[i] = (uint8_t)(input_tensor_f[i] / scale + zero_point); + } + } + + return success; +} + +error +tensorflowlite_compute(void *tflite_ctx, graph_execution_context ctx) +{ + TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; + + error res; + if (success != (res = is_valid_graph_execution_context(tfl_ctx, ctx))) + return res; + + tfl_ctx->interpreters[ctx].interpreter->Invoke(); + return success; +} + +error +tensorflowlite_get_output(void *tflite_ctx, graph_execution_context ctx, + uint32_t index, tensor_data output_tensor, + uint32_t *output_tensor_size) +{ + TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; + + error res; + if (success != (res = is_valid_graph_execution_context(tfl_ctx, ctx))) + return res; + + uint32_t num_output_tensors = + tfl_ctx->interpreters[ctx].interpreter->outputs().size(); + NN_DBG_PRINTF("Number of tensors (%d)", num_output_tensors); + + if (index + 1 > num_output_tensors) { + NN_ERR_PRINTF("Index %d is invalid.", index); + return runtime_error; + } + + auto tensor = tfl_ctx->interpreters[ctx].interpreter->output_tensor(index); + if (tensor == NULL) { + NN_ERR_PRINTF("Missing memory"); + return missing_memory; + } + + uint32_t model_tensor_size = 1; + for (int i = 0; i < (int)tensor->dims->size; ++i) + model_tensor_size *= (uint32_t)tensor->dims->data[i]; + + if (*output_tensor_size < model_tensor_size) { + NN_ERR_PRINTF("Insufficient memory to copy tensor %d", index); + return missing_memory; + } + + if (tensor->quantization.type == kTfLiteNoQuantization) { + NN_DBG_PRINTF("No quantization information"); + float *ot = + tfl_ctx->interpreters[ctx].interpreter->typed_output_tensor( + index); + + int size = model_tensor_size * sizeof(float); + bh_memcpy_s(output_tensor, size, ot, size); + } + else { // TODO: Assumming uint8 quantized networks. + TfLiteAffineQuantization *quant_info = + (TfLiteAffineQuantization *)tensor->quantization.params; + if (quant_info->scale->size != 1 || quant_info->zero_point->size != 1) { + NN_ERR_PRINTF("Quantization per channel is not supported"); + return runtime_error; + } + uint8_t *ot = tfl_ctx->interpreters[ctx] + .interpreter->typed_output_tensor(index); + + float scale = quant_info->scale->data[0]; + float zero_point = (float)quant_info->zero_point->data[0]; + NN_DBG_PRINTF("output tensor: (scale, offset) = (%f, %f)", scale, + zero_point); + + float *output_tensor_f = (float *)output_tensor; + for (uint32_t i = 0; i < model_tensor_size; ++i) { + output_tensor_f[i] = (ot[i] - zero_point) * scale; + } + } + + *output_tensor_size = model_tensor_size; + return success; +} + +void +tensorflowlite_initialize(void **tflite_ctx) +{ + TFLiteContext *tfl_ctx = new TFLiteContext(); + if (tfl_ctx == NULL) { + NN_ERR_PRINTF("Error when allocating memory for tensorflowlite."); + return; + } + + NN_DBG_PRINTF("Initializing models."); + tfl_ctx->current_models = 0; + for (int i = 0; i < MAX_GRAPHS_PER_INST; ++i) { + tfl_ctx->models[i].model_pointer = NULL; + } + NN_DBG_PRINTF("Initializing interpreters."); + tfl_ctx->current_interpreters = 0; + + if (os_mutex_init(&tfl_ctx->g_lock) != 0) { + NN_ERR_PRINTF("Error while initializing the lock"); + } + + tfl_ctx->delegate = NULL; + + *tflite_ctx = (void *)tfl_ctx; +} + +void +tensorflowlite_destroy(void *tflite_ctx) +{ + /* + TensorFlow Lite memory is internally managed by tensorflow + + Related issues: + * https://github.com/tensorflow/tensorflow/issues/15880 + */ + TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; + + NN_DBG_PRINTF("Freeing memory."); + for (int i = 0; i < MAX_GRAPHS_PER_INST; ++i) { + tfl_ctx->models[i].model.reset(); + if (tfl_ctx->models[i].model_pointer) { + if (tfl_ctx->delegate) { + switch (tfl_ctx->models[i].target) { + case gpu: + { +#if WASM_ENABLE_WASI_NN_GPU != 0 + TfLiteGpuDelegateV2Delete(tfl_ctx->delegate); +#else + NN_ERR_PRINTF("GPU delegate delete but not enabled."); +#endif + break; + } + case tpu: + { +#if WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE != 0 + TfLiteExternalDelegateDelete(tfl_ctx->delegate); +#else + NN_ERR_PRINTF( + "External delegate delete but not enabled."); +#endif + break; + } + } + } + wasm_runtime_free(tfl_ctx->models[i].model_pointer); + } + tfl_ctx->models[i].model_pointer = NULL; + } + for (int i = 0; i < MAX_GRAPH_EXEC_CONTEXTS_PER_INST; ++i) { + tfl_ctx->interpreters[i].interpreter.reset(); + } + os_mutex_destroy(&tfl_ctx->g_lock); + delete tfl_ctx; + NN_DBG_PRINTF("Memory free'd."); +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.hpp b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.hpp new file mode 100644 index 0000000..9605420 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_NN_TENSORFLOWLITE_HPP +#define WASI_NN_TENSORFLOWLITE_HPP + +#include "wasi_nn.h" + +#ifdef __cplusplus +extern "C" { +#endif + +error +tensorflowlite_load(void *tflite_ctx, graph_builder_array *builder, + graph_encoding encoding, execution_target target, graph *g); + +error +tensorflowlite_init_execution_context(void *tflite_ctx, graph g, + graph_execution_context *ctx); + +error +tensorflowlite_set_input(void *tflite_ctx, graph_execution_context ctx, + uint32_t index, tensor *input_tensor); + +error +tensorflowlite_compute(void *tflite_ctx, graph_execution_context ctx); + +error +tensorflowlite_get_output(void *tflite_ctx, graph_execution_context ctx, + uint32_t index, tensor_data output_tensor, + uint32_t *output_tensor_size); + +void +tensorflowlite_initialize(void **tflite_ctx); + +void +tensorflowlite_destroy(void *tflite_ctx); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.compile b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.compile new file mode 100644 index 0000000..b20d011 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.compile @@ -0,0 +1,26 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y \ + cmake build-essential git wget python3.10 python3-pip --no-install-recommends \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +ARG WASI_SDK_VER=19 +RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VER}/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -P /opt \ + && tar xf /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -C /opt \ + && ln -fs /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk \ + && rm /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz + +WORKDIR /wasi-nn/test + +COPY core/iwasm/libraries/wasi-nn/test/requirements.txt . + +RUN pip3 install --no-cache-dir -r requirements.txt && rm requirements.txt + +ENTRYPOINT [ "bash", "./build.sh" ] diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu new file mode 100644 index 0000000..6fe5656 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu @@ -0,0 +1,35 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +FROM ubuntu:20.04 AS base + +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y \ + cmake build-essential git --no-install-recommends + +WORKDIR /home/wamr + +COPY . . + +WORKDIR /home/wamr/product-mini/platforms/linux/build + +# hadolint ignore=DL3008 +RUN apt-get install -y wget ca-certificates --no-install-recommends \ + && mkdir /usr/local/share/ca-certificates/cacert.org \ + && wget -qP /usr/local/share/ca-certificates/cacert.org http://www.cacert.org/certs/root.crt http://www.cacert.org/certs/class3.crt \ + && update-ca-certificates \ + && git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + +RUN cmake \ + -DWAMR_BUILD_WASI_NN=1 \ + .. + +RUN make -j "$(grep -c ^processor /proc/cpuinfo)" + +FROM ubuntu:22.04 + +COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /iwasm + +ENTRYPOINT [ "/iwasm" ] diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu new file mode 100644 index 0000000..737b2d3 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu @@ -0,0 +1,49 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +FROM ubuntu:20.04 AS base + +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y \ + cmake build-essential git --no-install-recommends + +WORKDIR /home/wamr + +COPY . . + +WORKDIR /home/wamr/product-mini/platforms/linux/build + +# hadolint ignore=DL3008 +RUN apt-get install -y wget ca-certificates --no-install-recommends \ + && mkdir /usr/local/share/ca-certificates/cacert.org \ + && wget -qP /usr/local/share/ca-certificates/cacert.org http://www.cacert.org/certs/root.crt http://www.cacert.org/certs/class3.crt \ + && update-ca-certificates \ + && git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + +RUN cmake \ + -DWAMR_BUILD_WASI_NN=1 \ + -DWAMR_BUILD_WASI_NN_ENABLE_GPU=1 \ + .. + +RUN make -j "$(grep -c ^processor /proc/cpuinfo)" + +FROM nvidia/cuda:11.3.0-runtime-ubuntu20.04 + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ocl-icd-libopencl1 \ + ocl-icd-opencl-dev \ + clinfo && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /etc/OpenCL/vendors && \ + echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility + +COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /iwasm + +ENTRYPOINT [ "/iwasm" ] diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu new file mode 100644 index 0000000..742645b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu @@ -0,0 +1,37 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +FROM ubuntu:20.04 AS base + +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y \ + cmake build-essential git curl gnupg --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +# hadolint ignore=DL3008,DL4006 +RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update && apt-get install -y libedgetpu1-std --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /home/wamr + +COPY . . + +WORKDIR /home/wamr/product-mini/platforms/linux/build + +RUN cmake \ + -DWAMR_BUILD_WASI_NN=1 \ + -DWAMR_BUILD_WASI_NN_ENABLE_EXTERNAL_DELEGATE=1 \ + -DWAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH="libedgetpu.so.1.0" \ + -DWAMR_BUILD_WASI_NN_ENABLE_GPU=1 \ + .. + +RUN make -j "$(grep -c ^processor /proc/cpuinfo)" && \ + cp /home/wamr/product-mini/platforms/linux/build/iwasm /iwasm + +WORKDIR /assets + +ENTRYPOINT [ "/iwasm" ] diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.vx-delegate b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.vx-delegate new file mode 100644 index 0000000..f078045 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/Dockerfile.vx-delegate @@ -0,0 +1,113 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +FROM ubuntu:20.04 AS base + +ENV DEBIAN_FRONTEND=noninteractive + + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y \ + cmake build-essential git curl libssl-dev python3 --no-install-recommends \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y wget ca-certificates --no-install-recommends \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir /usr/local/share/ca-certificates/cacert.org \ + && wget -qP /usr/local/share/ca-certificates/cacert.org http://www.cacert.org/certs/root.crt http://www.cacert.org/certs/class3.crt \ + && update-ca-certificates \ + && git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + +# Build TensorFlow Lite VX delegate default built for x86-64 simulator +WORKDIR /tmp +RUN git clone https://github.com/VeriSilicon/TIM-VX.git tim-vx \ + && git clone https://github.com/VeriSilicon/tflite-vx-delegate.git \ + && git clone https://github.com/tensorflow/tensorflow.git + + +# Build TIM-VX +WORKDIR /tmp/tim-vx/host_build +RUN cmake -DCMAKE_INSTALL_PREFIX=/usr/local ../ \ + && make -j "$(grep -c ^processor /proc/cpuinfo)" \ + && make install + +WORKDIR /tmp/tim-vx +#RUN mkdir -p prebuilt-sdk/x86_64_linux/lib/include +#RUN cp prebuilt-sdk/x86_64_linux/include/CL prebuilt-sdk/x86_64_linux/lib/include -fr + + +# Build TensorFlow Lite +WORKDIR /tmp/tensorflow/build +RUN cmake \ + -DBUILD_SHARED_LIBS=ON=on \ + -DTFLITE_ENABLE_RUY=on \ + -DTFLITE_ENABLE_NNAPI=off \ + -DTFLITE_ENABLE_XNNPACK=on \ + -DTFLITE_ENABLE_EXTERNAL_DELEGATE=on \ + ../tensorflow/lite/ +RUN make -j "$(grep -c ^processor /proc/cpuinfo)" \ + && make install \ + && cp --no-preserve=ownership -d lib*.so* /usr/local/lib \ + && cp -r --no-preserve=ownership -d flatbuffers/include/flatbuffers /usr/local/include +# install header files +RUN install -d /usr/local/include/tensorflow/lite +WORKDIR /tmp/tensorflow/tensorflow/lite +# hadolint ignore=SC2046 +RUN cp --parents \ + $(find . -name "*.h*") \ + /usr/local/include/tensorflow/lite +# install version.h from core +RUN install -d /usr/local/include/tensorflow/core/public && \ + cp /tmp/tensorflow/tensorflow/core/public/version.h /usr/local/include/tensorflow/core/public + + +# Build Vx Delegate default built for x86-64 simulator +WORKDIR /tmp/tflite-vx-delegate/build +RUN cmake \ + -DBUILD_SHARED_LIBS=ON \ + -DFETCHCONTENT_SOURCE_DIR_TENSORFLOW=/tmp/tensorflow \ + -DTFLITE_LIB_LOC=/usr/local/lib/libtensorflow-lite.so \ + -DTIM_VX_INSTALL=/usr/local \ + -DCMAKE_INSTALL_PREFIX=/usr/ \ + ../ +RUN make vx_delegate -j "$(grep -c ^processor /proc/cpuinfo)" \ + && make install \ + && cp --no-preserve=ownership -d lib*.so* /usr/lib +# install header files +RUN install -d /usr/local/include/tensorflow-lite-vx-delegate +WORKDIR /tmp/tflite-vx-delegate/ +# hadolint ignore=SC2046 +RUN cp --parents \ + $(find . -name "*.h*") \ + /usr/local/include/tensorflow-lite-vx-delegate + +ENV VIVANTE_SDK_DIR=/tmp/tim-vx/prebuilt-sdk/x86_64_linux/ +ENV VSIMULATOR_CONFIG=czl + +ENV LD_LIBRARY_PATH=/tmp/tim-vx/prebuilt-sdk/x86_64_linux/lib:/usr/local/lib:/lib/x86_64-linux-gnu/:/lib64/:/usr/lib:$LD_LIBRARY_PATH + + +# Build WASI-NN +WORKDIR /home/wamr + +COPY . . + +WORKDIR /home/wamr/core/iwasm/libraries/wasi-nn/test/build + +# hadolint ignore=SC2086 +RUN cmake \ + -DCMAKE_LIBRARY_PATH=${CMAKE_LIBRARY_PATH}:/usr/local/lib/ \ + -DCMAKE_INCLUDE_PATH=${CMAKE_INCLUDE_PATH}:/usr/local/include/ \ + -DWAMR_BUILD_WASI_NN=1 \ + -DWAMR_BUILD_WASI_NN_ENABLE_EXT=1 \ + -DWASI_NN_EXT_DELEGATE_PATH="/usr/lib/libvx_delegate.so" \ + .. + +RUN make -j "$(grep -c ^processor /proc/cpuinfo)" + +RUN cp /home/wamr/core/iwasm/libraries/wasi-nn/test/build/iwasm /run/iwasm + +ENTRYPOINT [ "/run/iwasm" ] diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/build.sh b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/build.sh new file mode 100755 index 0000000..da6cfa5 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/build.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CURR_PATH=$(cd $(dirname $0) && pwd -P) + +# WASM application that uses WASI-NN + +/opt/wasi-sdk/bin/clang \ + -Wl,--allow-undefined \ + -Wl,--strip-all,--no-entry \ + --sysroot=/opt/wasi-sdk/share/wasi-sysroot \ + -I../include -I../src/utils \ + -o test_tensorflow.wasm \ + test_tensorflow.c utils.c + +# TFLite models to use in the tests + +cd ${CURR_PATH}/models +python3 average.py +python3 max.py +python3 mult_dimension.py +python3 mult_outputs.py +python3 sum.py + +# Specific tests for TPU + +cd ${CURR_PATH} +/opt/wasi-sdk/bin/clang \ + -Wl,--allow-undefined \ + -Wl,--strip-all,--no-entry \ + --sysroot=/opt/wasi-sdk/share/wasi-sysroot \ + -I../include -I../src/utils \ + -o test_tensorflow_quantized.wasm \ + test_tensorflow_quantized.c utils.c + +cd ${CURR_PATH}/models +python3 quantized.py + +cd ${CURR_PATH} diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/average.py b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/average.py new file mode 100755 index 0000000..a21fe75 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/average.py @@ -0,0 +1,16 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf +from utils import save_model + +model = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=[5, 5, 1]), + tf.keras.layers.AveragePooling2D( + pool_size=(5, 5), strides=None, padding="valid", data_format=None) + +]) + +# Export model to tflite + +save_model(model, "average.tflite") diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/max.py b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/max.py new file mode 100755 index 0000000..a3ec456 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/max.py @@ -0,0 +1,17 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf + +from utils import save_model + +model = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=[5, 5, 1]), + tf.keras.layers.MaxPooling2D( + pool_size=(5, 5), strides=None, padding="valid", data_format=None) + +]) + +# Export model to tflite + +save_model(model, "max.tflite") diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/mult_dimension.py b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/mult_dimension.py new file mode 100644 index 0000000..f521a93 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/mult_dimension.py @@ -0,0 +1,15 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf +from utils import save_model + +model = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=[3, 3, 1]), + tf.keras.layers.Conv2D(1, (1, 1), kernel_initializer=tf.keras.initializers.Constant( + value=1), bias_initializer='zeros' + ) +]) +# Export model to tflite + +save_model(model, "mult_dim.tflite") diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/mult_outputs.py b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/mult_outputs.py new file mode 100755 index 0000000..98a5012 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/mult_outputs.py @@ -0,0 +1,33 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf +import numpy as np +from keras.layers import AveragePooling2D, Conv2D + +from tensorflow.keras import Input, Model + +from utils import save_model + + +inputs = Input(shape=(4, 4, 1)) + +output1 = Conv2D(1, (4, 1), kernel_initializer=tf.keras.initializers.Constant( + value=1), bias_initializer='zeros' +)(inputs) +output2 = AveragePooling2D(pool_size=( + 4, 1), strides=None, padding="valid", data_format=None)(inputs) + +model = Model(inputs=inputs, outputs=[output1, output2]) + +inp = np.arange(16).reshape((1, 4, 4, 1)) + +print(inp) + +res = model.predict(inp) + +print(res) +print(res[0].shape) +print(res[1].shape) + +save_model(model, "mult_out.tflite") diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/quantized.py b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/quantized.py new file mode 100644 index 0000000..b195b31 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/quantized.py @@ -0,0 +1,30 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf +import numpy as np +import pathlib + +model = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=[5, 5, 1]), + tf.keras.layers.AveragePooling2D( + pool_size=(5, 5), strides=None, padding="valid", data_format=None) + +]) + +def representative_dataset(): + for _ in range(1000): + data = np.random.randint(0, 25, (1, 5, 5, 1)) + yield [data.astype(np.float32)] + +converter = tf.lite.TFLiteConverter.from_keras_model(model) +converter.optimizations = [tf.lite.Optimize.DEFAULT] +converter.representative_dataset = representative_dataset +converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] +converter.inference_input_type = tf.uint8 # or tf.int8 +converter.inference_output_type = tf.uint8 # or tf.int8 +tflite_model = converter.convert() + +tflite_models_dir = pathlib.Path("./") +tflite_model_file = tflite_models_dir / "quantized_model.tflite" +tflite_model_file.write_bytes(tflite_model) diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/sum.py b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/sum.py new file mode 100755 index 0000000..503125b --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/sum.py @@ -0,0 +1,17 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf + +from utils import save_model + +model = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=[5, 5, 1]), + tf.keras.layers.Conv2D(1, (5, 5), kernel_initializer=tf.keras.initializers.Constant( + value=1), bias_initializer='zeros' + ) +]) + +# Export model to tflite + +save_model(model, "sum.tflite") diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/utils.py b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/utils.py new file mode 100644 index 0000000..8335f05 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/models/utils.py @@ -0,0 +1,13 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf +import pathlib + + +def save_model(model, filename): + converter = tf.lite.TFLiteConverter.from_keras_model(model) + tflite_model = converter.convert() + tflite_models_dir = pathlib.Path("./") + tflite_model_file = tflite_models_dir/filename + tflite_model_file.write_bytes(tflite_model) diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/requirements.txt b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/requirements.txt new file mode 100644 index 0000000..4cf2910 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.11.1 \ No newline at end of file diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/test_tensorflow.c b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/test_tensorflow.c new file mode 100644 index 0000000..6a9e207 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/test_tensorflow.c @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +#include "utils.h" +#include "logger.h" + +void +test_sum(execution_target target) +{ + int dims[] = { 1, 5, 5, 1 }; + input_info input = create_input(dims); + + uint32_t output_size = 0; + float *output = run_inference(target, input.input_tensor, input.dim, + &output_size, "./models/sum.tflite", 1); + + assert(output_size == 1); + assert(fabs(output[0] - 300.0) < EPSILON); + + free(input.dim); + free(input.input_tensor); + free(output); +} + +void +test_max(execution_target target) +{ + int dims[] = { 1, 5, 5, 1 }; + input_info input = create_input(dims); + + uint32_t output_size = 0; + float *output = run_inference(target, input.input_tensor, input.dim, + &output_size, "./models/max.tflite", 1); + + assert(output_size == 1); + assert(fabs(output[0] - 24.0) < EPSILON); + NN_INFO_PRINTF("Result: max is %f", output[0]); + + free(input.dim); + free(input.input_tensor); + free(output); +} + +void +test_average(execution_target target) +{ + int dims[] = { 1, 5, 5, 1 }; + input_info input = create_input(dims); + + uint32_t output_size = 0; + float *output = run_inference(target, input.input_tensor, input.dim, + &output_size, "./models/average.tflite", 1); + + assert(output_size == 1); + assert(fabs(output[0] - 12.0) < EPSILON); + NN_INFO_PRINTF("Result: average is %f", output[0]); + + free(input.dim); + free(input.input_tensor); + free(output); +} + +void +test_mult_dimensions(execution_target target) +{ + int dims[] = { 1, 3, 3, 1 }; + input_info input = create_input(dims); + + uint32_t output_size = 0; + float *output = run_inference(target, input.input_tensor, input.dim, + &output_size, "./models/mult_dim.tflite", 1); + + assert(output_size == 9); + for (int i = 0; i < 9; i++) + assert(fabs(output[i] - i) < EPSILON); + + free(input.dim); + free(input.input_tensor); + free(output); +} + +void +test_mult_outputs(execution_target target) +{ + int dims[] = { 1, 4, 4, 1 }; + input_info input = create_input(dims); + + uint32_t output_size = 0; + float *output = run_inference(target, input.input_tensor, input.dim, + &output_size, "./models/mult_out.tflite", 2); + + assert(output_size == 8); + // first tensor check + for (int i = 0; i < 4; i++) + assert(fabs(output[i] - (i * 4 + 24)) < EPSILON); + // second tensor check + for (int i = 0; i < 4; i++) + assert(fabs(output[i + 4] - (i + 6)) < EPSILON); + + free(input.dim); + free(input.input_tensor); + free(output); +} + +int +main() +{ + char *env = getenv("TARGET"); + if (env == NULL) { + NN_INFO_PRINTF("Usage:\n--env=\"TARGET=[cpu|gpu]\""); + return 1; + } + execution_target target; + if (strcmp(env, "cpu") == 0) + target = cpu; + else if (strcmp(env, "gpu") == 0) + target = gpu; + else { + NN_ERR_PRINTF("Wrong target!"); + return 1; + } + NN_INFO_PRINTF("################### Testing sum..."); + test_sum(target); + NN_INFO_PRINTF("################### Testing max..."); + test_max(target); + NN_INFO_PRINTF("################### Testing average..."); + test_average(target); + NN_INFO_PRINTF("################### Testing multiple dimensions..."); + test_mult_dimensions(target); + NN_INFO_PRINTF("################### Testing multiple outputs..."); + test_mult_outputs(target); + + NN_INFO_PRINTF("Tests: passed!"); + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/test_tensorflow_quantized.c b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/test_tensorflow_quantized.c new file mode 100644 index 0000000..3ed7c75 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/test_tensorflow_quantized.c @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +#include "utils.h" +#include "logger.h" + +#undef EPSILON +#define EPSILON 1e-2 + +void +test_average_quantized(execution_target target) +{ + int dims[] = { 1, 5, 5, 1 }; + input_info input = create_input(dims); + + uint32_t output_size = 0; + float *output = + run_inference(target, input.input_tensor, input.dim, &output_size, + "./models/quantized_model.tflite", 1); + + NN_INFO_PRINTF("Output size: %d", output_size); + NN_INFO_PRINTF("Result: average is %f", output[0]); + // NOTE: 11.95 instead of 12 because of errors due quantization + assert(fabs(output[0] - 11.95) < EPSILON); + + free(input.dim); + free(input.input_tensor); + free(output); +} + +int +main() +{ + char *env = getenv("TARGET"); + if (env == NULL) { + NN_INFO_PRINTF("Usage:\n--env=\"TARGET=[cpu|gpu|tpu]\""); + return 1; + } + execution_target target; + if (strcmp(env, "cpu") == 0) + target = cpu; + else if (strcmp(env, "gpu") == 0) + target = gpu; + else if (strcmp(env, "tpu") == 0) + target = tpu; + else { + NN_ERR_PRINTF("Wrong target!"); + return 1; + } + NN_INFO_PRINTF("################### Testing quantized model..."); + test_average_quantized(target); + + NN_INFO_PRINTF("Tests: passed!"); + return 0; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/utils.c b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/utils.c new file mode 100644 index 0000000..7b4f65d --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/utils.c @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "utils.h" +#include "logger.h" + +#include +#include + +error +wasm_load(char *model_name, graph *g, execution_target target) +{ + FILE *pFile = fopen(model_name, "r"); + if (pFile == NULL) + return invalid_argument; + + uint8_t *buffer; + size_t result; + + // allocate memory to contain the whole file: + buffer = (uint8_t *)malloc(sizeof(uint8_t) * MAX_MODEL_SIZE); + if (buffer == NULL) { + fclose(pFile); + return missing_memory; + } + + result = fread(buffer, 1, MAX_MODEL_SIZE, pFile); + if (result <= 0) { + fclose(pFile); + free(buffer); + return missing_memory; + } + + graph_builder_array arr; + + arr.size = 1; + arr.buf = (graph_builder *)malloc(sizeof(graph_builder)); + if (arr.buf == NULL) { + fclose(pFile); + free(buffer); + return missing_memory; + } + + arr.buf[0].size = result; + arr.buf[0].buf = buffer; + + error res = load(&arr, tensorflowlite, target, g); + + fclose(pFile); + free(buffer); + free(arr.buf); + return res; +} + +error +wasm_init_execution_context(graph g, graph_execution_context *ctx) +{ + return init_execution_context(g, ctx); +} + +error +wasm_set_input(graph_execution_context ctx, float *input_tensor, uint32_t *dim) +{ + tensor_dimensions dims; + dims.size = INPUT_TENSOR_DIMS; + dims.buf = (uint32_t *)malloc(dims.size * sizeof(uint32_t)); + if (dims.buf == NULL) + return missing_memory; + + tensor tensor; + tensor.dimensions = &dims; + for (int i = 0; i < tensor.dimensions->size; ++i) + tensor.dimensions->buf[i] = dim[i]; + tensor.type = fp32; + tensor.data = (uint8_t *)input_tensor; + error err = set_input(ctx, 0, &tensor); + + free(dims.buf); + return err; +} + +error +wasm_compute(graph_execution_context ctx) +{ + return compute(ctx); +} + +error +wasm_get_output(graph_execution_context ctx, uint32_t index, float *out_tensor, + uint32_t *out_size) +{ + return get_output(ctx, index, (uint8_t *)out_tensor, out_size); +} + +float * +run_inference(execution_target target, float *input, uint32_t *input_size, + uint32_t *output_size, char *model_name, + uint32_t num_output_tensors) +{ + graph graph; + if (wasm_load(model_name, &graph, target) != success) { + NN_ERR_PRINTF("Error when loading model."); + exit(1); + } + + graph_execution_context ctx; + if (wasm_init_execution_context(graph, &ctx) != success) { + NN_ERR_PRINTF("Error when initialixing execution context."); + exit(1); + } + + if (wasm_set_input(ctx, input, input_size) != success) { + NN_ERR_PRINTF("Error when setting input tensor."); + exit(1); + } + + if (wasm_compute(ctx) != success) { + NN_ERR_PRINTF("Error when running inference."); + exit(1); + } + + float *out_tensor = (float *)malloc(sizeof(float) * MAX_OUTPUT_TENSOR_SIZE); + if (out_tensor == NULL) { + NN_ERR_PRINTF("Error when allocating memory for output tensor."); + exit(1); + } + + uint32_t offset = 0; + for (int i = 0; i < num_output_tensors; ++i) { + *output_size = MAX_OUTPUT_TENSOR_SIZE - *output_size; + if (wasm_get_output(ctx, i, &out_tensor[offset], output_size) + != success) { + NN_ERR_PRINTF("Error when getting index %d.", i); + break; + } + + offset += *output_size; + } + *output_size = offset; + return out_tensor; +} + +input_info +create_input(int *dims) +{ + input_info input = { .dim = NULL, .input_tensor = NULL, .elements = 1 }; + + input.dim = malloc(INPUT_TENSOR_DIMS * sizeof(uint32_t)); + if (input.dim) + for (int i = 0; i < INPUT_TENSOR_DIMS; ++i) { + input.dim[i] = dims[i]; + input.elements *= dims[i]; + } + + input.input_tensor = malloc(input.elements * sizeof(float)); + for (int i = 0; i < input.elements; ++i) + input.input_tensor[i] = i; + + return input; +} diff --git a/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/utils.h b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/utils.h new file mode 100644 index 0000000..0b23284 --- /dev/null +++ b/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test/utils.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_NN_UTILS +#define WASI_NN_UTILS + +#include + +#include "wasi_nn.h" + +#define MAX_MODEL_SIZE 85000000 +#define MAX_OUTPUT_TENSOR_SIZE 1000000 +#define INPUT_TENSOR_DIMS 4 +#define EPSILON 1e-8 + +typedef struct { + float *input_tensor; + uint32_t *dim; + uint32_t elements; +} input_info; + +/* wasi-nn wrappers */ + +error +wasm_load(char *model_name, graph *g, execution_target target); + +error +wasm_init_execution_context(graph g, graph_execution_context *ctx); + +error +wasm_set_input(graph_execution_context ctx, float *input_tensor, uint32_t *dim); + +error +wasm_compute(graph_execution_context ctx); + +error +wasm_get_output(graph_execution_context ctx, uint32_t index, float *out_tensor, + uint32_t *out_size); + +/* Utils */ + +float * +run_inference(execution_target target, float *input, uint32_t *input_size, + uint32_t *output_size, char *model_name, + uint32_t num_output_tensors); + +input_info +create_input(int *dims); + +#endif diff --git a/wasm-micro-runtime/core/shared/coap/er-coap/LICENSE.md b/wasm-micro-runtime/core/shared/coap/er-coap/LICENSE.md new file mode 100644 index 0000000..f4b1a05 --- /dev/null +++ b/wasm-micro-runtime/core/shared/coap/er-coap/LICENSE.md @@ -0,0 +1,30 @@ +Copyright (c) (Year), (Name of copyright holder) +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/wasm-micro-runtime/core/shared/coap/er-coap/coap-constants.h b/wasm-micro-runtime/core/shared/coap/er-coap/coap-constants.h new file mode 100644 index 0000000..1de2ed9 --- /dev/null +++ b/wasm-micro-runtime/core/shared/coap/er-coap/coap-constants.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2013, Institute for Pervasive Computing, ETH Zurich + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file is part of the Contiki operating system. + */ + +/** + * \file + * Collection of constants specified in the CoAP standard. + * \author + * Matthias Kovatsch + */ + +/** + * \addtogroup coap + * @{ + */ + +#ifndef COAP_CONSTANTS_H_ +#define COAP_CONSTANTS_H_ + +/* clang-format off */ +#define COAP_DEFAULT_PORT 5683 +#define COAP_DEFAULT_SECURE_PORT 5684 + +#define COAP_DEFAULT_MAX_AGE 60 +#define COAP_RESPONSE_TIMEOUT 3 +#define COAP_RESPONSE_RANDOM_FACTOR 1.5 +#define COAP_MAX_RETRANSMIT 4 + +#define COAP_HEADER_LEN 4 /* | version:0x03 type:0x0C tkl:0xF0 | code | mid:0x00FF | mid:0xFF00 | */ +#define COAP_TOKEN_LEN 8 /* The maximum number of bytes for the Token */ +#define COAP_ETAG_LEN 8 /* The maximum number of bytes for the ETag */ + +#define COAP_HEADER_VERSION_MASK 0xC0 +#define COAP_HEADER_VERSION_POSITION 6 +#define COAP_HEADER_TYPE_MASK 0x30 +#define COAP_HEADER_TYPE_POSITION 4 +#define COAP_HEADER_TOKEN_LEN_MASK 0x0F +#define COAP_HEADER_TOKEN_LEN_POSITION 0 + +#define COAP_HEADER_OPTION_DELTA_MASK 0xF0 +#define COAP_HEADER_OPTION_SHORT_LENGTH_MASK 0x0F +/* clang-format on */ + +/* CoAP message types */ +typedef enum { + COAP_TYPE_CON, /* confirmables */ + COAP_TYPE_NON, /* non-confirmables */ + COAP_TYPE_ACK, /* acknowledgements */ + COAP_TYPE_RST /* reset */ +} coap_message_type_t; + +/* clang-format off */ +/* CoAP request method codes */ +typedef enum { + COAP_GET = 1, + COAP_POST, COAP_PUT, + COAP_DELETE +} coap_method_t; +/* clang-format on */ + +/* CoAP response codes */ +typedef enum { + COAP_NO_ERROR = 0, + + CREATED_2_01 = 65, /* CREATED */ + DELETED_2_02 = 66, /* DELETED */ + VALID_2_03 = 67, /* NOT_MODIFIED */ + CHANGED_2_04 = 68, /* CHANGED */ + CONTENT_2_05 = 69, /* OK */ + CONTINUE_2_31 = 95, /* CONTINUE */ + + BAD_REQUEST_4_00 = 128, /* BAD_REQUEST */ + UNAUTHORIZED_4_01 = 129, /* UNAUTHORIZED */ + BAD_OPTION_4_02 = 130, /* BAD_OPTION */ + FORBIDDEN_4_03 = 131, /* FORBIDDEN */ + NOT_FOUND_4_04 = 132, /* NOT_FOUND */ + METHOD_NOT_ALLOWED_4_05 = 133, /* METHOD_NOT_ALLOWED */ + NOT_ACCEPTABLE_4_06 = 134, /* NOT_ACCEPTABLE */ + PRECONDITION_FAILED_4_12 = 140, /* BAD_REQUEST */ + REQUEST_ENTITY_TOO_LARGE_4_13 = 141, /* REQUEST_ENTITY_TOO_LARGE */ + UNSUPPORTED_MEDIA_TYPE_4_15 = 143, /* UNSUPPORTED_MEDIA_TYPE */ + + INTERNAL_SERVER_ERROR_5_00 = 160, /* INTERNAL_SERVER_ERROR */ + NOT_IMPLEMENTED_5_01 = 161, /* NOT_IMPLEMENTED */ + BAD_GATEWAY_5_02 = 162, /* BAD_GATEWAY */ + SERVICE_UNAVAILABLE_5_03 = 163, /* SERVICE_UNAVAILABLE */ + GATEWAY_TIMEOUT_5_04 = 164, /* GATEWAY_TIMEOUT */ + PROXYING_NOT_SUPPORTED_5_05 = 165, /* PROXYING_NOT_SUPPORTED */ + + /* Erbium errors */ + MEMORY_ALLOCATION_ERROR = 192, + PACKET_SERIALIZATION_ERROR, + + /* Erbium hooks */ + MANUAL_RESPONSE, + PING_RESPONSE +} coap_status_t; + +/* CoAP header option numbers */ +typedef enum { + COAP_OPTION_IF_MATCH = 1, /* 0-8 B */ + COAP_OPTION_URI_HOST = 3, /* 1-255 B */ + COAP_OPTION_ETAG = 4, /* 1-8 B */ + COAP_OPTION_IF_NONE_MATCH = 5, /* 0 B */ + COAP_OPTION_OBSERVE = 6, /* 0-3 B */ + COAP_OPTION_URI_PORT = 7, /* 0-2 B */ + COAP_OPTION_LOCATION_PATH = 8, /* 0-255 B */ + COAP_OPTION_URI_PATH = 11, /* 0-255 B */ + COAP_OPTION_CONTENT_FORMAT = 12, /* 0-2 B */ + COAP_OPTION_MAX_AGE = 14, /* 0-4 B */ + COAP_OPTION_URI_QUERY = 15, /* 0-255 B */ + COAP_OPTION_ACCEPT = 17, /* 0-2 B */ + COAP_OPTION_LOCATION_QUERY = 20, /* 0-255 B */ + COAP_OPTION_BLOCK2 = 23, /* 1-3 B */ + COAP_OPTION_BLOCK1 = 27, /* 1-3 B */ + COAP_OPTION_SIZE2 = 28, /* 0-4 B */ + COAP_OPTION_PROXY_URI = 35, /* 1-1034 B */ + COAP_OPTION_PROXY_SCHEME = 39, /* 1-255 B */ + COAP_OPTION_SIZE1 = 60, /* 0-4 B */ +} coap_option_t; + +/* CoAP Content-Formats */ +typedef enum { + TEXT_PLAIN = 0, + TEXT_XML = 1, + TEXT_CSV = 2, + TEXT_HTML = 3, + IMAGE_GIF = 21, + IMAGE_JPEG = 22, + IMAGE_PNG = 23, + IMAGE_TIFF = 24, + AUDIO_RAW = 25, + VIDEO_RAW = 26, + APPLICATION_LINK_FORMAT = 40, + APPLICATION_XML = 41, + APPLICATION_OCTET_STREAM = 42, + APPLICATION_RDF_XML = 43, + APPLICATION_SOAP_XML = 44, + APPLICATION_ATOM_XML = 45, + APPLICATION_XMPP_XML = 46, + APPLICATION_EXI = 47, + APPLICATION_FASTINFOSET = 48, + APPLICATION_SOAP_FASTINFOSET = 49, + APPLICATION_JSON = 50, + APPLICATION_X_OBIX_BINARY = 51 +} coap_content_format_t; + +/** + * Resource flags for allowed methods and special functionalities. + */ +typedef enum { + NO_FLAGS = 0, + + /* methods to handle */ + METHOD_GET = (1 << 0), + METHOD_POST = (1 << 1), + METHOD_PUT = (1 << 2), + METHOD_DELETE = (1 << 3), + + /* special flags */ + HAS_SUB_RESOURCES = (1 << 4), + IS_SEPARATE = (1 << 5), + IS_OBSERVABLE = (1 << 6), + IS_PERIODIC = (1 << 7) +} coap_resource_flags_t; + +#endif /* COAP_CONSTANTS_H_ */ diff --git a/wasm-micro-runtime/core/shared/coap/extension/coap_ext.h b/wasm-micro-runtime/core/shared/coap/extension/coap_ext.h new file mode 100644 index 0000000..f61deac --- /dev/null +++ b/wasm-micro-runtime/core/shared/coap/extension/coap_ext.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef COAP_EXTENSION_COAP_EXT_H_ +#define COAP_EXTENSION_COAP_EXT_H_ + +#include "coap-constants.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define COAP_EVENT (COAP_DELETE + 2) + +#ifdef __cplusplus +} +#endif +#endif /* COAP_EXTENSION_COAP_EXT_H_ */ diff --git a/wasm-micro-runtime/core/shared/coap/lib_coap.cmake b/wasm-micro-runtime/core/shared/coap/lib_coap.cmake new file mode 100644 index 0000000..8970e5d --- /dev/null +++ b/wasm-micro-runtime/core/shared/coap/lib_coap.cmake @@ -0,0 +1,12 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIB_COAP_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include_directories(${LIB_COAP_DIR}/er-coap) +include_directories(${LIB_COAP_DIR}/extension) + +file (GLOB_RECURSE source_all ${LIB_COAP_DIR}/*.c) + +set (LIB_COAP_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/shared/mem-alloc/SConscript b/wasm-micro-runtime/core/shared/mem-alloc/SConscript new file mode 100644 index 0000000..602d871 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/SConscript @@ -0,0 +1,33 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * +import os + +cwd = GetCurrentDir() + +src = Split(''' +''') + + +def addSrcFiles(arr, path): + for f in os.listdir(path): + fpath = os.path.join(path, f); + if os.path.isfile(fpath): + ext = os.path.splitext(fpath)[-1] + if ext == '.c' or ext == '.cpp': + arr += [fpath] + elif os.path.isdir(fpath): + addSrcFiles(arr, fpath) + + + +addSrcFiles(src, cwd); +CPPPATH = [cwd, cwd+'/../include'] + +group = DefineGroup('iwasm_platform_core', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_alloc.c b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_alloc.c new file mode 100644 index 0000000..4863527 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_alloc.c @@ -0,0 +1,1169 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" + +#if WASM_ENABLE_GC != 0 +#define LOCK_HEAP(heap) \ + do { \ + if (!heap->is_doing_reclaim) \ + /* If the heap is doing reclaim, it must have been locked, \ + we should not lock the heap again. */ \ + os_mutex_lock(&heap->lock); \ + } while (0) +#define UNLOCK_HEAP(heap) \ + do { \ + if (!heap->is_doing_reclaim) \ + /* If the heap is doing reclaim, it must have been locked, \ + and will be unlocked after reclaim, we should not \ + unlock the heap again. */ \ + os_mutex_unlock(&heap->lock); \ + } while (0) +#else +#define LOCK_HEAP(heap) os_mutex_lock(&heap->lock) +#define UNLOCK_HEAP(heap) os_mutex_unlock(&heap->lock) +#endif + +static inline bool +hmu_is_in_heap(void *hmu, gc_uint8 *heap_base_addr, gc_uint8 *heap_end_addr) +{ + gc_uint8 *addr = (gc_uint8 *)hmu; + return (addr >= heap_base_addr && addr < heap_end_addr) ? true : false; +} + +/** + * Remove a node from the tree it belongs to + * + * @param p the node to remove, can not be NULL, can not be the ROOT node + * the node will be removed from the tree, and the left, right and + * parent pointers of the node @p will be set to be NULL. Other fields + * won't be touched. The tree will be re-organized so that the order + * conditions are still satisified. + */ +static bool +remove_tree_node(gc_heap_t *heap, hmu_tree_node_t *p) +{ + hmu_tree_node_t *q = NULL, **slot = NULL; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + hmu_tree_node_t *root = heap->kfc_tree_root, *parent; + gc_uint8 *base_addr = heap->base_addr; + gc_uint8 *end_addr = base_addr + heap->current_size; +#endif + + bh_assert(p); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + parent = p->parent; + if (!parent || p == root /* p can not be the ROOT node */ + || !hmu_is_in_heap(p, base_addr, end_addr) + || (parent != root && !hmu_is_in_heap(parent, base_addr, end_addr))) { + goto fail; + } +#endif + + /* get the slot which holds pointer to node p */ + if (p == p->parent->right) { + /* Don't use `slot = &p->parent->right` to avoid compiler warning */ + slot = (hmu_tree_node_t **)((uint8 *)p->parent + + offsetof(hmu_tree_node_t, right)); + } + else if (p == p->parent->left) { + /* p should be a child of its parent */ + /* Don't use `slot = &p->parent->left` to avoid compiler warning */ + slot = (hmu_tree_node_t **)((uint8 *)p->parent + + offsetof(hmu_tree_node_t, left)); + } + else { + goto fail; + } + + /** + * algorithms used to remove node p + * case 1: if p has no left child, replace p with its right child + * case 2: if p has no right child, replace p with its left child + * case 3: otherwise, find p's predecessor, remove it from the tree + * and replace p with it. + * use predecessor can keep the left <= root < right condition. + */ + + if (!p->left) { + /* move right child up*/ + *slot = p->right; + if (p->right) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(p->right, base_addr, end_addr)) { + goto fail; + } +#endif + p->right->parent = p->parent; + } + + p->left = p->right = p->parent = NULL; + return true; + } + + if (!p->right) { + /* move left child up*/ + *slot = p->left; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(p->left, base_addr, end_addr)) { + goto fail; + } +#endif + /* p->left can never be NULL unless it is corrupted. */ + p->left->parent = p->parent; + + p->left = p->right = p->parent = NULL; + return true; + } + + /* both left & right exist, find p's predecessor at first*/ + q = p->left; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } +#endif + while (q->right) { + q = q->right; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } +#endif + } + + /* remove from the tree*/ + if (!remove_tree_node(heap, q)) + return false; + + *slot = q; + q->parent = p->parent; + q->left = p->left; + q->right = p->right; + if (q->left) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q->left, base_addr, end_addr)) { + goto fail; + } +#endif + q->left->parent = q; + } + if (q->right) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q->right, base_addr, end_addr)) { + goto fail; + } +#endif + q->right->parent = q; + } + + p->left = p->right = p->parent = NULL; + + return true; +fail: +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + heap->is_heap_corrupted = true; +#endif + return false; +} + +static bool +unlink_hmu(gc_heap_t *heap, hmu_t *hmu) +{ +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + gc_uint8 *base_addr, *end_addr; +#endif + gc_size_t size; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(hmu && (gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (hmu_get_ut(hmu) != HMU_FC) { + heap->is_heap_corrupted = true; + return false; + } +#endif + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; +#endif + size = hmu_get_size(hmu); + + if (HMU_IS_FC_NORMAL(size)) { + uint32 node_idx = size >> 3; + hmu_normal_node_t *node_prev = NULL, *node_next; + hmu_normal_node_t *node = heap->kfc_normal_list[node_idx].next; + + while (node) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(node, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } +#endif + node_next = get_hmu_normal_node_next(node); + if ((hmu_t *)node == hmu) { + if (!node_prev) /* list head */ + heap->kfc_normal_list[node_idx].next = node_next; + else + set_hmu_normal_node_next(node_prev, node_next); + break; + } + node_prev = node; + node = node_next; + } + + if (!node) { + LOG_ERROR("[GC_ERROR]couldn't find the node in the normal list\n"); + } + } + else { + if (!remove_tree_node(heap, (hmu_tree_node_t *)hmu)) + return false; + } + return true; +} + +static void +hmu_set_free_size(hmu_t *hmu) +{ + gc_size_t size; + bh_assert(hmu && hmu_get_ut(hmu) == HMU_FC); + + size = hmu_get_size(hmu); + *((uint32 *)((char *)hmu + size) - 1) = size; +} + +/** + * Add free chunk back to KFC + * + * @param heap should not be NULL and it should be a valid heap + * @param hmu should not be NULL and it should be a HMU of length @size inside + * @heap hmu should be 8-bytes aligned + * @param size should be positive and multiple of 8 + * hmu with size @size will be added into KFC as a new FC. + */ +bool +gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size) +{ +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + gc_uint8 *base_addr, *end_addr; +#endif + hmu_normal_node_t *np = NULL; + hmu_tree_node_t *root = NULL, *tp = NULL, *node = NULL; + uint32 node_idx; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(hmu && (gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + bh_assert(((gc_uint32)(uintptr_t)hmu_to_obj(hmu) & 7) == 0); + bh_assert(size > 0 + && ((gc_uint8 *)hmu) + size + <= heap->base_addr + heap->current_size); + bh_assert(!(size & 7)); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; +#endif + + hmu_set_ut(hmu, HMU_FC); + hmu_set_size(hmu, size); + hmu_set_free_size(hmu); + + if (HMU_IS_FC_NORMAL(size)) { + np = (hmu_normal_node_t *)hmu; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(np, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } +#endif + + node_idx = size >> 3; + set_hmu_normal_node_next(np, heap->kfc_normal_list[node_idx].next); + heap->kfc_normal_list[node_idx].next = np; + return true; + } + + /* big block */ + node = (hmu_tree_node_t *)hmu; + node->size = size; + node->left = node->right = node->parent = NULL; + + /* find proper node to link this new node to */ + root = heap->kfc_tree_root; + tp = root; + bh_assert(tp->size < size); + while (1) { + if (tp->size < size) { + if (!tp->right) { + tp->right = node; + node->parent = tp; + break; + } + tp = tp->right; + } + else { /* tp->size >= size */ + if (!tp->left) { + tp->left = node; + node->parent = tp; + break; + } + tp = tp->left; + } +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } +#endif + } + return true; +} + +/** + * Find a proper hmu for required memory size + * + * @param heap should not be NULL and should be a valid heap + * @param size should cover the header and should be 8 bytes aligned + * GC will not be performed here. + * Heap extension will not be performed here. + * + * @return hmu allocated if success, which will be aligned to 8 bytes, + * NULL otherwise + */ +static hmu_t * +alloc_hmu(gc_heap_t *heap, gc_size_t size) +{ + gc_uint8 *base_addr, *end_addr; + hmu_normal_list_t *normal_head = NULL; + hmu_normal_node_t *p = NULL; + uint32 node_idx = 0, init_node_idx = 0; + hmu_tree_node_t *root = NULL, *tp = NULL, *last_tp = NULL; + hmu_t *next, *rest; + uintptr_t tp_ret; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(size > 0 && !(size & 7)); + +#if WASM_ENABLE_GC != 0 + /* In doing reclaim, gc must not alloc memory again. */ + bh_assert(!heap->is_doing_reclaim); +#endif + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + if (size < GC_SMALLEST_SIZE) + size = GC_SMALLEST_SIZE; + + /* check normal list at first*/ + if (HMU_IS_FC_NORMAL(size)) { + /* find a non-empty slot in normal_node_list with good size*/ + init_node_idx = (size >> 3); + for (node_idx = init_node_idx; node_idx < HMU_NORMAL_NODE_CNT; + node_idx++) { + normal_head = heap->kfc_normal_list + node_idx; + if (normal_head->next) + break; + normal_head = NULL; + } + + /* found in normal list*/ + if (normal_head) { + bh_assert(node_idx >= init_node_idx); + + p = normal_head->next; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(p, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } +#endif + normal_head->next = get_hmu_normal_node_next(p); +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (((gc_int32)(uintptr_t)hmu_to_obj(p) & 7) != 0) { + heap->is_heap_corrupted = true; + return NULL; + } +#endif + + if ((gc_size_t)node_idx != (uint32)init_node_idx + /* with bigger size*/ + && ((gc_size_t)node_idx << 3) >= size + GC_SMALLEST_SIZE) { + rest = (hmu_t *)(((char *)p) + size); + if (!gci_add_fc(heap, rest, (node_idx << 3) - size)) { + return NULL; + } + hmu_mark_pinuse(rest); + } + else { + size = node_idx << 3; + next = (hmu_t *)((char *)p + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) + hmu_mark_pinuse(next); + } + + heap->total_free_size -= size; + if ((heap->current_size - heap->total_free_size) + > heap->highmark_size) + heap->highmark_size = + heap->current_size - heap->total_free_size; + + hmu_set_size((hmu_t *)p, size); + return (hmu_t *)p; + } + } + + /* need to find a node in tree*/ + root = heap->kfc_tree_root; + + /* find the best node*/ + bh_assert(root); + tp = root->right; + while (tp) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } +#endif + + if (tp->size < size) { + tp = tp->right; + continue; + } + + /* record the last node with size equal to or bigger than given size*/ + last_tp = tp; + tp = tp->left; + } + + if (last_tp) { + bh_assert(last_tp->size >= size); + + /* alloc in last_p*/ + + /* remove node last_p from tree*/ + if (!remove_tree_node(heap, last_tp)) + return NULL; + + if (last_tp->size >= size + GC_SMALLEST_SIZE) { + rest = (hmu_t *)((char *)last_tp + size); + if (!gci_add_fc(heap, rest, last_tp->size - size)) + return NULL; + hmu_mark_pinuse(rest); + } + else { + size = last_tp->size; + next = (hmu_t *)((char *)last_tp + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) + hmu_mark_pinuse(next); + } + + heap->total_free_size -= size; + if ((heap->current_size - heap->total_free_size) > heap->highmark_size) + heap->highmark_size = heap->current_size - heap->total_free_size; + + hmu_set_size((hmu_t *)last_tp, size); + tp_ret = (uintptr_t)last_tp; + return (hmu_t *)tp_ret; + } + + return NULL; +} + +#if WASM_ENABLE_GC != 0 +static int +do_gc_heap(gc_heap_t *heap) +{ + int ret = GC_SUCCESS; +#if WASM_ENABLE_GC_PERF_PROFILING != 0 + uint64 start = 0, end = 0, time = 0; + + start = os_time_get_boot_microsecond(); +#endif + if (heap->is_reclaim_enabled) { + UNLOCK_HEAP(heap); + ret = gci_gc_heap(heap); + LOCK_HEAP(heap); + } +#if WASM_ENABLE_GC_PERF_PROFILING != 0 + end = os_time_get_boot_microsecond(); + time = end - start; + heap->total_gc_time += time; + if (time > heap->max_gc_time) { + heap->max_gc_time = time; + } + heap->total_gc_count += 1; +#endif + return ret; +} +#endif + +/** + * Find a proper HMU with given size + * + * @param heap should not be NULL and should be a valid heap + * @param size should cover the header and should be 8 bytes aligned + * + * Note: This function will try several ways to satisfy the allocation request: + * 1. Find a proper on available HMUs. + * 2. GC will be triggered if 1 failed. + * 3. Find a proper on available HMUS. + * 4. Return NULL if 3 failed + * + * @return hmu allocated if success, which will be aligned to 8 bytes, + * NULL otherwise + */ +static hmu_t * +alloc_hmu_ex(gc_heap_t *heap, gc_size_t size) +{ + bh_assert(gci_is_heap_valid(heap)); + bh_assert(size > 0 && !(size & 7)); + +#if WASM_ENABLE_GC != 0 +#if GC_IN_EVERY_ALLOCATION != 0 + if (GC_SUCCESS != do_gc_heap(heap)) + return NULL; +#else + if (heap->total_free_size < heap->gc_threshold) { + if (GC_SUCCESS != do_gc_heap(heap)) + return NULL; + } + else { + hmu_t *ret = NULL; + if ((ret = alloc_hmu(heap, size))) { + return ret; + } + if (GC_SUCCESS != do_gc_heap(heap)) + return NULL; + } +#endif +#endif + + return alloc_hmu(heap, size); +} + +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_alloc_vo(void *vheap, gc_size_t size) +#else +gc_object_t +gc_alloc_vo_internal(void *vheap, gc_size_t size, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL; + gc_object_t ret = (gc_object_t)NULL; + gc_size_t tot_size = 0, tot_size_unaligned; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } +#endif + + LOCK_HEAP(heap); + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + +#if GC_STAT_DATA != 0 + heap->total_size_allocated += tot_size; +#endif + + hmu_set_ut(hmu, HMU_VO); + hmu_unfree_vo(hmu); + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + if (tot_size > tot_size_unaligned) + /* clear buffer appended by GC_ALIGN_8() */ + memset((uint8 *)ret + size, 0, tot_size - tot_size_unaligned); + +finish: + UNLOCK_HEAP(heap); + return ret; +} + +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_realloc_vo(void *vheap, void *ptr, gc_size_t size) +#else +gc_object_t +gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, const char *file, + int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL, *hmu_old = NULL, *hmu_next; + gc_object_t ret = (gc_object_t)NULL, obj_old = (gc_object_t)ptr; + gc_size_t tot_size, tot_size_unaligned, tot_size_old = 0, tot_size_next; + gc_size_t obj_size, obj_size_old; + gc_uint8 *base_addr, *end_addr; + hmu_type_t ut; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } +#endif + + if (obj_old) { + hmu_old = obj_to_hmu(obj_old); + tot_size_old = hmu_get_size(hmu_old); + if (tot_size <= tot_size_old) + /* current node alreay meets requirement */ + return obj_old; + } + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + LOCK_HEAP(heap); + + if (hmu_old) { + hmu_next = (hmu_t *)((char *)hmu_old + tot_size_old); + if (hmu_is_in_heap(hmu_next, base_addr, end_addr)) { + ut = hmu_get_ut(hmu_next); + tot_size_next = hmu_get_size(hmu_next); + if (ut == HMU_FC && tot_size <= tot_size_old + tot_size_next) { + /* current node and next node meets requirement */ + if (!unlink_hmu(heap, hmu_next)) { + UNLOCK_HEAP(heap); + return NULL; + } + hmu_set_size(hmu_old, tot_size); + memset((char *)hmu_old + tot_size_old, 0, + tot_size - tot_size_old); +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu_old, tot_size, file, line); +#endif + if (tot_size < tot_size_old + tot_size_next) { + hmu_next = (hmu_t *)((char *)hmu_old + tot_size); + tot_size_next = tot_size_old + tot_size_next - tot_size; + if (!gci_add_fc(heap, hmu_next, tot_size_next)) { + UNLOCK_HEAP(heap); + return NULL; + } + hmu_mark_pinuse(hmu_next); + } + UNLOCK_HEAP(heap); + return obj_old; + } + } + } + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + +#if GC_STAT_DATA != 0 + heap->total_size_allocated += tot_size; +#endif + + hmu_set_ut(hmu, HMU_VO); + hmu_unfree_vo(hmu); + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + +finish: + + if (ret) { + obj_size = tot_size - HMU_SIZE - OBJ_PREFIX_SIZE - OBJ_SUFFIX_SIZE; + memset(ret, 0, obj_size); + if (obj_old) { + obj_size_old = + tot_size_old - HMU_SIZE - OBJ_PREFIX_SIZE - OBJ_SUFFIX_SIZE; + bh_memcpy_s(ret, obj_size, obj_old, obj_size_old); + } + } + + UNLOCK_HEAP(heap); + + if (ret && obj_old) + gc_free_vo(vheap, obj_old); + + return ret; +} + +#if GC_MANUALLY != 0 +void +gc_free_wo(void *vheap, void *ptr) +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + gc_object_t *obj = (gc_object_t *)ptr; + hmu_t *hmu = obj_to_hmu(obj); + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(obj); + bh_assert((gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + bh_assert(hmu_get_ut(hmu) == HMU_WO); + + hmu_unmark_wo(hmu); + (void)heap; +} +#endif + +/* see ems_gc.h for description*/ +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_alloc_wo(void *vheap, gc_size_t size) +#else +gc_object_t +gc_alloc_wo_internal(void *vheap, gc_size_t size, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL; + gc_object_t ret = (gc_object_t)NULL; + gc_size_t tot_size = 0, tot_size_unaligned; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } +#endif + + LOCK_HEAP(heap); + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + /* Don't memset the memory to improve performance, the caller should + decide whether to memset it or not */ + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + +#if GC_STAT_DATA != 0 + heap->total_size_allocated += tot_size; +#endif + + hmu_set_ut(hmu, HMU_WO); +#if GC_MANUALLY != 0 + hmu_mark_wo(hmu); +#else + hmu_unmark_wo(hmu); +#endif + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + if (tot_size > tot_size_unaligned) + /* clear buffer appended by GC_ALIGN_8() */ + memset((uint8 *)ret + size, 0, tot_size - tot_size_unaligned); + +finish: + UNLOCK_HEAP(heap); + return ret; +} + +/** + * Do some checking to see if given pointer is a possible valid heap + * @return GC_TRUE if all checking passed, GC_FALSE otherwise + */ +int +gci_is_heap_valid(gc_heap_t *heap) +{ + if (!heap) + return GC_FALSE; + if (heap->heap_id != (gc_handle_t)heap) + return GC_FALSE; + + return GC_TRUE; +} + +#if BH_ENABLE_GC_VERIFY == 0 +int +gc_free_vo(void *vheap, gc_object_t obj) +#else +int +gc_free_vo_internal(void *vheap, gc_object_t obj, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + gc_uint8 *base_addr, *end_addr; + hmu_t *hmu = NULL; + hmu_t *prev = NULL; + hmu_t *next = NULL; + gc_size_t size = 0; + hmu_type_t ut; + int ret = GC_SUCCESS; + + if (!obj) { + return GC_SUCCESS; + } + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, free memory failed.\n"); + return GC_ERROR; + } +#endif + + hmu = obj_to_hmu(obj); + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + LOCK_HEAP(heap); + + if (hmu_is_in_heap(hmu, base_addr, end_addr)) { +#if BH_ENABLE_GC_VERIFY != 0 + hmu_verify(heap, hmu); +#endif + ut = hmu_get_ut(hmu); + if (ut == HMU_VO) { + if (hmu_is_vo_freed(hmu)) { + bh_assert(0); + ret = GC_ERROR; + goto out; + } + + size = hmu_get_size(hmu); + + heap->total_free_size += size; + +#if GC_STAT_DATA != 0 + heap->total_size_freed += size; +#endif + + if (!hmu_get_pinuse(hmu)) { + prev = (hmu_t *)((char *)hmu - *((int *)hmu - 1)); + + if (hmu_is_in_heap(prev, base_addr, end_addr) + && hmu_get_ut(prev) == HMU_FC) { + size += hmu_get_size(prev); + hmu = prev; + if (!unlink_hmu(heap, prev)) { + ret = GC_ERROR; + goto out; + } + } + } + + next = (hmu_t *)((char *)hmu + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) { + if (hmu_get_ut(next) == HMU_FC) { + size += hmu_get_size(next); + if (!unlink_hmu(heap, next)) { + ret = GC_ERROR; + goto out; + } + next = (hmu_t *)((char *)hmu + size); + } + } + + if (!gci_add_fc(heap, hmu, size)) { + ret = GC_ERROR; + goto out; + } + + if (hmu_is_in_heap(next, base_addr, end_addr)) { + hmu_unmark_pinuse(next); + } + } + else { + ret = GC_ERROR; + goto out; + } + ret = GC_SUCCESS; + goto out; + } + +out: + UNLOCK_HEAP(heap); + return ret; +} + +void +gc_dump_heap_stats(gc_heap_t *heap) +{ + os_printf("heap: %p, heap start: %p\n", heap, heap->base_addr); + os_printf("total free: %" PRIu32 ", current: %" PRIu32 + ", highmark: %" PRIu32 "\n", + heap->total_free_size, heap->current_size, heap->highmark_size); +#if GC_STAT_DATA != 0 + os_printf("total size allocated: %" PRIu64 ", total size freed: %" PRIu64 + ", total occupied: %" PRIu64 "\n", + heap->total_size_allocated, heap->total_size_freed, + heap->total_size_allocated - heap->total_size_freed); +#endif +} + +uint32 +gc_get_heap_highmark_size(gc_heap_t *heap) +{ + return heap->highmark_size; +} + +void +gci_dump(gc_heap_t *heap) +{ + hmu_t *cur = NULL, *end = NULL; + hmu_type_t ut; + gc_size_t size; + int i = 0, p, mark; + char inuse = 'U'; + + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + ut = hmu_get_ut(cur); + size = hmu_get_size(cur); + p = hmu_get_pinuse(cur); + mark = hmu_is_wo_marked(cur); + + if (ut == HMU_VO) + inuse = 'V'; + else if (ut == HMU_WO) + inuse = hmu_is_wo_marked(cur) ? 'W' : 'w'; + else if (ut == HMU_FC) + inuse = 'F'; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (size == 0 || size > (uint32)((uint8 *)end - (uint8 *)cur)) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap dump failed.\n"); + heap->is_heap_corrupted = true; + return; + } +#endif + + os_printf("#%d %08" PRIx32 " %" PRIx32 " %d %d" + " %c %" PRId32 "\n", + i, (int32)((char *)cur - (char *)heap->base_addr), (int32)ut, + p, mark, inuse, (int32)hmu_obj_size(size)); +#if BH_ENABLE_GC_VERIFY != 0 + if (inuse == 'V') { + gc_object_prefix_t *prefix = (gc_object_prefix_t *)(cur + 1); + os_printf("#%s:%d\n", prefix->file_name, prefix->line_no); + } +#endif + + cur = (hmu_t *)((char *)cur + size); + i++; + } + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (cur != end) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap dump failed.\n"); + heap->is_heap_corrupted = true; + } +#else + bh_assert(cur == end); +#endif +} + +#if WASM_ENABLE_GC != 0 +extra_info_node_t * +gc_search_extra_info_node(gc_handle_t handle, gc_object_t obj, + gc_size_t *p_index) +{ + gc_heap_t *vheap = (gc_heap_t *)handle; + int32 low = 0, high = vheap->extra_info_node_cnt - 1; + int32 mid; + extra_info_node_t *node; + + if (!vheap->extra_info_nodes) + return NULL; + + while (low <= high) { + mid = (low + high) / 2; + node = vheap->extra_info_nodes[mid]; + + if (obj == node->obj) { + if (p_index) { + *p_index = mid; + } + return node; + } + else if (obj < node->obj) { + high = mid - 1; + } + else { + low = mid + 1; + } + } + + if (p_index) { + *p_index = low; + } + return NULL; +} + +static bool +insert_extra_info_node(gc_heap_t *vheap, extra_info_node_t *node) +{ + gc_size_t index; + extra_info_node_t *orig_node; + + if (!vheap->extra_info_nodes) { + vheap->extra_info_nodes = vheap->extra_info_normal_nodes; + vheap->extra_info_node_capacity = sizeof(vheap->extra_info_normal_nodes) + / sizeof(extra_info_node_t *); + vheap->extra_info_nodes[0] = node; + vheap->extra_info_node_cnt = 1; + return true; + } + + /* extend array */ + if (vheap->extra_info_node_cnt == vheap->extra_info_node_capacity) { + extra_info_node_t **new_nodes = NULL; + gc_size_t new_capacity = vheap->extra_info_node_capacity * 3 / 2; + gc_size_t total_size = sizeof(extra_info_node_t *) * new_capacity; + + new_nodes = (extra_info_node_t **)BH_MALLOC(total_size); + if (!new_nodes) { + LOG_ERROR("alloc extra info nodes failed"); + return false; + } + + bh_memcpy_s(new_nodes, total_size, vheap->extra_info_nodes, + sizeof(extra_info_node_t *) * vheap->extra_info_node_cnt); + if (vheap->extra_info_nodes != vheap->extra_info_normal_nodes) { + BH_FREE(vheap->extra_info_nodes); + } + + vheap->extra_info_nodes = new_nodes; + vheap->extra_info_node_capacity = new_capacity; + } + + orig_node = gc_search_extra_info_node(vheap, node->obj, &index); + if (orig_node) { + /* replace the old node */ + vheap->extra_info_nodes[index] = node; + BH_FREE(orig_node); + } + else { + bh_memmove_s(vheap->extra_info_nodes + index + 1, + (vheap->extra_info_node_capacity - index - 1) + * sizeof(extra_info_node_t *), + vheap->extra_info_nodes + index, + (vheap->extra_info_node_cnt - index) + * sizeof(extra_info_node_t *)); + vheap->extra_info_nodes[index] = node; + vheap->extra_info_node_cnt += 1; + } + + return true; +} + +bool +gc_set_finalizer(gc_handle_t handle, gc_object_t obj, gc_finalizer_t cb, + void *data) +{ + extra_info_node_t *node = NULL; + gc_heap_t *vheap = (gc_heap_t *)handle; + + node = (extra_info_node_t *)BH_MALLOC(sizeof(extra_info_node_t)); + + if (!node) { + LOG_ERROR("alloc a new extra info node failed"); + return GC_FALSE; + } + memset(node, 0, sizeof(extra_info_node_t)); + + node->finalizer = cb; + node->obj = obj; + node->data = data; + + LOCK_HEAP(vheap); + if (!insert_extra_info_node(vheap, node)) { + BH_FREE(node); + UNLOCK_HEAP(vheap); + return GC_FALSE; + } + UNLOCK_HEAP(vheap); + + gct_vm_set_extra_info_flag(obj, true); + return GC_TRUE; +} + +void +gc_unset_finalizer(gc_handle_t handle, gc_object_t obj) +{ + gc_size_t index; + gc_heap_t *vheap = (gc_heap_t *)handle; + extra_info_node_t *node; + + LOCK_HEAP(vheap); + node = gc_search_extra_info_node(vheap, obj, &index); + + if (!node) { + UNLOCK_HEAP(vheap); + return; + } + + BH_FREE(node); + bh_memmove_s( + vheap->extra_info_nodes + index, + (vheap->extra_info_node_capacity - index) * sizeof(extra_info_node_t *), + vheap->extra_info_nodes + index + 1, + (vheap->extra_info_node_cnt - index - 1) * sizeof(extra_info_node_t *)); + vheap->extra_info_node_cnt -= 1; + UNLOCK_HEAP(vheap); + + gct_vm_set_extra_info_flag(obj, false); +} +#endif diff --git a/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc.c b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc.c new file mode 100644 index 0000000..26e83a9 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc.c @@ -0,0 +1,497 @@ +/* + * Copyright (C) 2022 Tencent Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc.h" +#include "ems_gc_internal.h" + +#define GB (1 << 30UL) + +#define MARK_NODE_OBJ_CNT 256 + +#if WASM_ENABLE_GC != 0 + +/* mark node is used for gc marker*/ +typedef struct mark_node_struct { + /* number of to-expand objects can be saved in this node */ + gc_size_t cnt; + + /* the first unused index */ + uint32 idx; + + /* next node on the node list */ + struct mark_node_struct *next; + + /* the actual to-expand objects list */ + gc_object_t set[MARK_NODE_OBJ_CNT]; +} mark_node_t; + +/** + * Alloc a mark node from the native heap + * + * @return a valid mark node if success, NULL otherwise + */ +static mark_node_t * +alloc_mark_node(void) +{ + mark_node_t *ret = (mark_node_t *)BH_MALLOC(sizeof(mark_node_t)); + + if (!ret) { + LOG_ERROR("alloc a new mark node failed"); + return NULL; + } + ret->cnt = sizeof(ret->set) / sizeof(ret->set[0]); + ret->idx = 0; + ret->next = NULL; + return ret; +} + +/* Free a mark node to the native heap + * + * @param node the mark node to free, should not be NULL + */ +static void +free_mark_node(mark_node_t *node) +{ + bh_assert(node); + BH_FREE((gc_object_t)node); +} + +/** + * Sweep phase of mark_sweep algorithm + * @param heap the heap to sweep, should be a valid instance heap + * which has already been marked + */ +static void +sweep_instance_heap(gc_heap_t *heap) +{ + hmu_t *cur = NULL, *end = NULL, *last = NULL; + hmu_type_t ut; + gc_size_t size; + int i, lsize; + gc_size_t tot_free = 0; + + bh_assert(gci_is_heap_valid(heap)); + + cur = (hmu_t *)heap->base_addr; + last = NULL; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + /* reset KFC */ + lsize = + (int)(sizeof(heap->kfc_normal_list) / sizeof(heap->kfc_normal_list[0])); + for (i = 0; i < lsize; i++) { + heap->kfc_normal_list[i].next = NULL; + } + heap->kfc_tree_root->right = NULL; + heap->root_set = NULL; + + while (cur < end) { + ut = hmu_get_ut(cur); + size = hmu_get_size(cur); + bh_assert(size > 0); + + if (ut == HMU_FC || ut == HMU_FM + || (ut == HMU_VO && hmu_is_vo_freed(cur)) + || (ut == HMU_WO && !hmu_is_wo_marked(cur))) { + /* merge previous free areas with current one */ + if (!last) + last = cur; + + if (ut == HMU_WO) { + /* Invoke registered finalizer */ + gc_object_t cur_obj = hmu_to_obj(cur); + if (gct_vm_get_extra_info_flag(cur_obj)) { + extra_info_node_t *node = gc_search_extra_info_node( + (gc_handle_t)heap, cur_obj, NULL); + bh_assert(node); + node->finalizer(node->obj, node->data); + gc_unset_finalizer((gc_handle_t)heap, cur_obj); + } + } + } + else { + /* current block is still live */ + if (last) { + tot_free += (gc_size_t)((char *)cur - (char *)last); + gci_add_fc(heap, last, (gc_size_t)((char *)cur - (char *)last)); + hmu_mark_pinuse(last); + last = NULL; + } + + if (ut == HMU_WO) { + /* unmark it */ + hmu_unmark_wo(cur); + } + } + + cur = (hmu_t *)((char *)cur + size); + } + + bh_assert(cur == end); + + if (last) { + tot_free += (gc_size_t)((char *)cur - (char *)last); + gci_add_fc(heap, last, (gc_size_t)((char *)cur - (char *)last)); + hmu_mark_pinuse(last); + } + + heap->total_free_size = tot_free; + +#if GC_STAT_DATA != 0 + heap->total_gc_count++; + if ((heap->current_size - tot_free) > heap->highmark_size) + heap->highmark_size = heap->current_size - tot_free; + +#endif + gc_update_threshold(heap); +} + +/** + * Add a to-expand node to the to-expand list + * + * @param heap should be a valid instance heap + * @param obj should be a valid wo inside @heap + * + * @return GC_ERROR if there is no more resource for marking, + * GC_SUCCESS if success + */ +static int +add_wo_to_expand(gc_heap_t *heap, gc_object_t obj) +{ + mark_node_t *mark_node = NULL, *new_node = NULL; + hmu_t *hmu = NULL; + + bh_assert(obj); + + hmu = obj_to_hmu(obj); + + bh_assert(gci_is_heap_valid(heap)); + bh_assert((gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + bh_assert(hmu_get_ut(hmu) == HMU_WO); + + if (hmu_is_wo_marked(hmu)) + return GC_SUCCESS; /* already marked*/ + + mark_node = (mark_node_t *)heap->root_set; + if (!mark_node || mark_node->idx == mark_node->cnt) { + new_node = alloc_mark_node(); + if (!new_node) { + LOG_ERROR("can not add obj to mark node because of mark node " + "allocation failed"); + return GC_ERROR; + } + new_node->next = mark_node; + heap->root_set = new_node; + mark_node = new_node; + } + + mark_node->set[mark_node->idx++] = obj; + hmu_mark_wo(hmu); + return GC_SUCCESS; +} + +/* Check ems_gc.h for description*/ +int +gc_add_root(void *heap_p, gc_object_t obj) +{ + gc_heap_t *heap = (gc_heap_t *)heap_p; + hmu_t *hmu = NULL; + + if (!obj) { + LOG_ERROR("gc_add_root with NULL obj"); + return GC_ERROR; + } + + hmu = obj_to_hmu(obj); + + if (!gci_is_heap_valid(heap)) { + LOG_ERROR("vm_get_gc_handle_for_current_instance returns invalid heap"); + return GC_ERROR; + } + + if (!((gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size)) { + LOG_ERROR("Obj is not a object in current instance heap"); + return GC_ERROR; + } + + if (hmu_get_ut(hmu) != HMU_WO) { + LOG_ERROR("Given object is not wo"); + return GC_ERROR; + } + + if (add_wo_to_expand(heap, obj) != GC_SUCCESS) { + heap->is_fast_marking_failed = 1; + return GC_ERROR; + } + + return GC_SUCCESS; +} + +/** + * Unmark all marked objects to do rollback + * + * @param heap the heap to do rollback, should be a valid instance heap + */ +static void +rollback_mark(gc_heap_t *heap) +{ + mark_node_t *mark_node = NULL, *next_mark_node = NULL; + hmu_t *cur = NULL, *end = NULL; + hmu_type_t ut; + gc_size_t size; + + bh_assert(gci_is_heap_valid(heap)); + + /* roll back*/ + mark_node = (mark_node_t *)heap->root_set; + while (mark_node) { + next_mark_node = mark_node->next; + free_mark_node(mark_node); + mark_node = next_mark_node; + } + + heap->root_set = NULL; + + /* then traverse the heap to unmark all marked wos*/ + + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + ut = hmu_get_ut(cur); + size = hmu_get_size(cur); + + if (ut == HMU_WO && hmu_is_wo_marked(cur)) { + hmu_unmark_wo(cur); + } + + cur = (hmu_t *)((char *)cur + size); + } + + bh_assert(cur == end); +} + +/** + * Reclaim GC instance heap + * + * @param heap the heap to reclaim, should be a valid instance heap + * + * @return GC_SUCCESS if success, GC_ERROR otherwise + */ +static int +reclaim_instance_heap(gc_heap_t *heap) +{ + mark_node_t *mark_node = NULL; + int idx = 0, j = 0; + bool ret, is_compact_mode = false; + gc_object_t obj = NULL, ref = NULL; + hmu_t *hmu = NULL; + gc_uint32 ref_num = 0, ref_start_offset = 0, size = 0, offset = 0; + gc_uint16 *ref_list = NULL; + + bh_assert(gci_is_heap_valid(heap)); + + heap->root_set = NULL; + +#if WASM_ENABLE_THREAD_MGR == 0 + if (!heap->exec_env) + return GC_SUCCESS; + ret = gct_vm_begin_rootset_enumeration(heap->exec_env, heap); +#else + if (!heap->cluster) + return GC_SUCCESS; + ret = gct_vm_begin_rootset_enumeration(heap->cluster, heap); +#endif + if (!ret) + return GC_ERROR; + +#if BH_ENABLE_GC_VERIFY != 0 + /* no matter whether the enumeration is successful or not, the data + collected should be checked at first */ + mark_node = (mark_node_t *)heap->root_set; + while (mark_node) { + /* all nodes except first should be full filled */ + bh_assert(mark_node == (mark_node_t *)heap->root_set + || mark_node->idx == mark_node->cnt); + + /* all nodes should be non-empty */ + bh_assert(mark_node->idx > 0); + + for (idx = 0; idx < (int)mark_node->idx; idx++) { + obj = mark_node->set[idx]; + hmu = obj_to_hmu(obj); + bh_assert(hmu_is_wo_marked(hmu)); + bh_assert((gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu + < heap->base_addr + heap->current_size); + } + + mark_node = mark_node->next; + } +#endif + + /* TODO: when fast marking failed, we can still do slow + marking, currently just simply roll it back. */ + if (heap->is_fast_marking_failed) { + LOG_ERROR("enumerate rootset failed"); + LOG_ERROR("all marked wos will be unmarked to keep heap consistency"); + + rollback_mark(heap); + heap->is_fast_marking_failed = 0; + return GC_ERROR; + } + + /* the algorithm we use to mark all objects */ + /* 1. mark rootset and organize them into a mark_node list (last marked + * roots at list header, i.e. stack top) */ + /* 2. in every iteration, we use the top node to expand*/ + /* 3. execute step 2 till no expanding */ + /* this is a BFS & DFS mixed algorithm, but more like DFS */ + mark_node = (mark_node_t *)heap->root_set; + while (mark_node) { + heap->root_set = mark_node->next; + + /* note that mark_node->idx may change in each loop */ + for (idx = 0; idx < (int)mark_node->idx; idx++) { + obj = mark_node->set[idx]; + hmu = obj_to_hmu(obj); + size = hmu_get_size(hmu); + + if (!gct_vm_get_wasm_object_ref_list(obj, &is_compact_mode, + &ref_num, &ref_list, + &ref_start_offset)) { + LOG_ERROR("mark process failed because failed " + "vm_get_wasm_object_ref_list"); + break; + } + + if (ref_num >= 2U * GB) { + LOG_ERROR("Invalid ref_num returned"); + break; + } + + if (is_compact_mode) { + for (j = 0; j < (int)ref_num; j++) { + offset = ref_start_offset + j * sizeof(void *); + bh_assert(offset + sizeof(void *) < size); + ref = *(gc_object_t *)(((gc_uint8 *)obj) + offset); + if (ref == NULL_REF || ((uintptr_t)ref & 1)) + continue; /* null object or i31 object */ + if (add_wo_to_expand(heap, ref) == GC_ERROR) { + LOG_ERROR("add_wo_to_expand failed"); + break; + } + } + if (j < (int)ref_num) + break; + } + else { + for (j = 0; j < (int)ref_num; j++) { + offset = ref_list[j]; + bh_assert(offset + sizeof(void *) < size); + + ref = *(gc_object_t *)(((gc_uint8 *)obj) + offset); + if (ref == NULL_REF || ((uintptr_t)ref & 1)) + continue; /* null object or i31 object */ + if (add_wo_to_expand(heap, ref) == GC_ERROR) { + LOG_ERROR("mark process failed"); + break; + } + } + if (j < (int)ref_num) + break; + } + } + if (idx < (int)mark_node->idx) + break; /* not yet done */ + + /* obj's in mark_node are all expanded */ + free_mark_node(mark_node); + mark_node = heap->root_set; + } + + if (mark_node) { + LOG_ERROR("mark process is not successfully finished"); + + free_mark_node(mark_node); + /* roll back is required */ + rollback_mark(heap); + + return GC_ERROR; + } + + /* now sweep */ + sweep_instance_heap(heap); + + (void)size; + + return GC_SUCCESS; +} + +/** + * Do GC on given heap + * + * @param the heap to do GC, should be a valid heap + * + * @return GC_SUCCESS if success, GC_ERROR otherwise + */ +int +gci_gc_heap(void *h) +{ + int ret = GC_ERROR; + gc_heap_t *heap = (gc_heap_t *)h; + + bh_assert(gci_is_heap_valid(heap)); + + LOG_VERBOSE("#reclaim instance heap %p", heap); + + /* TODO: get exec_env of current thread when GC multi-threading + is enabled, and pass it to runtime */ + gct_vm_gc_prepare(NULL); + + gct_vm_mutex_lock(&heap->lock); + heap->is_doing_reclaim = 1; + + ret = reclaim_instance_heap(heap); + + heap->is_doing_reclaim = 0; + gct_vm_mutex_unlock(&heap->lock); + + /* TODO: get exec_env of current thread when GC multi-threading + is enabled, and pass it to runtime */ + gct_vm_gc_finished(NULL); + + LOG_VERBOSE("#reclaim instance heap %p done", heap); + +#if BH_ENABLE_GC_VERIFY != 0 + gci_verify_heap(heap); +#endif + +#if GC_STAT_SHOW != 0 + gc_show_stat(heap); + gc_show_fragment(heap); +#endif + + return ret; +} + +int +gc_is_dead_object(void *obj) +{ + return !hmu_is_wo_marked(obj_to_hmu(obj)); +} + +#else + +int +gci_gc_heap(void *h) +{ + (void)h; + return GC_ERROR; +} + +#endif /* end of WASM_ENABLE_GC != 0 */ diff --git a/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc.h b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc.h new file mode 100644 index 0000000..293ad18 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc.h @@ -0,0 +1,347 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file ems_gc.h + * @date Wed Aug 3 10:46:38 2011 + * + * @brief This file defines GC modules types and interfaces. + */ + +#ifndef _EMS_GC_H +#define _EMS_GC_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef GC_STAT_DATA +#define GC_STAT_DATA 0 +#endif + +#ifndef GC_STAT_SHOW +#define GC_STAT_SHOW 0 +#endif + +#ifndef GC_IN_EVERY_ALLOCATION +#define GC_IN_EVERY_ALLOCATION 0 +#endif + +#ifndef GC_MANUALLY +#define GC_MANUALLY 0 +#endif + +#define GC_HEAD_PADDING 4 + +#ifndef NULL_REF +#define NULL_REF ((gc_object_t)NULL) +#endif + +#define GC_SUCCESS (0) +#define GC_ERROR (-1) + +#define GC_TRUE (1) +#define GC_FALSE (0) + +#define GC_MAX_HEAP_SIZE (256 * BH_KB) + +typedef void *gc_handle_t; +typedef void *gc_object_t; +typedef uint64 gc_uint64; +typedef int64 gc_int64; +typedef uint32 gc_uint32; +typedef int32 gc_int32; +typedef uint16 gc_uint16; +typedef int16 gc_int16; +typedef uint8 gc_uint8; +typedef int8 gc_int8; +typedef uint32 gc_size_t; + +typedef enum { + GC_STAT_TOTAL = 0, + GC_STAT_FREE, + GC_STAT_HIGHMARK, + GC_STAT_COUNT, + GC_STAT_TIME, + GC_STAT_MAX +} GC_STAT_INDEX; + +typedef void (*gc_finalizer_t)(void *obj, void *data); + +#ifndef EXTRA_INFO_NORMAL_NODE_CNT +#define EXTRA_INFO_NORMAL_NODE_CNT 32 +#endif + +/* extra information attached to specific object */ +typedef struct extra_info_node { + gc_object_t obj; + gc_finalizer_t finalizer; + void *data; +} extra_info_node_t; + +/** + * GC initialization from a buffer, which is separated into + * two parts: the beginning of the buffer is used to create + * the heap structure, and the left is used to create the + * actual pool data + * + * @param buf the buffer to be initialized to a heap + * @param buf_size the size of buffer + * + * @return gc handle if success, NULL otherwise + */ +gc_handle_t +gc_init_with_pool(char *buf, gc_size_t buf_size); + +/** + * GC initialization from heap struct buffer and pool buffer + * + * @param struct_buf the struct buffer to create the heap structure + * @param struct_buf_size the size of struct buffer + * @param pool_buf the pool buffer to create pool data + * @param pool_buf_size the size of poll buffer + * + * @return gc handle if success, NULL otherwise + */ +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size); + +/** + * Destroy heap which is initilized from a buffer + * + * @param handle handle to heap needed destroy + * + * @return GC_SUCCESS if success + * GC_ERROR for bad parameters or failed system resource freeing. + */ +int +gc_destroy_with_pool(gc_handle_t handle); + +#if WASM_ENABLE_GC != 0 +/** + * Enable or disable GC reclaim for a heap + * + * @param handle handle of the heap + * @param exec_env the exec_env of current module instance + */ +#if WASM_ENABLE_THREAD_MGR == 0 +void +gc_enable_gc_reclaim(gc_handle_t handle, void *exec_env); +#else +/** + * Enable or disable GC reclaim for a heap + * + * @param handle handle of the heap + * @param cluster the tread cluster of current module instance + */ +void +gc_enable_gc_reclaim(gc_handle_t handle, void *cluster); +#endif +#endif + +/** + * Return heap struct size + */ +uint32 +gc_get_heap_struct_size(void); + +/** + * Migrate heap from one pool buf to another pool buf + * + * @param handle handle of the new heap + * @param pool_buf_new the new pool buffer + * @param pool_buf_size the size of new pool buffer + * + * @return GC_SUCCESS if success, GC_ERROR otherwise + */ +int +gc_migrate(gc_handle_t handle, char *pool_buf_new, gc_size_t pool_buf_size); + +/** + * Check whether the heap is corrupted + * + * @param handle handle of the heap + * + * @return true if success, false otherwise + */ +bool +gc_is_heap_corrupted(gc_handle_t handle); + +/** + * Get Heap Stats + * + * @param stats [out] integer array to save heap stats + * @param size [in] the size of stats + * @param mmt [in] type of heap, MMT_SHARED or MMT_INSTANCE + */ +void * +gc_heap_stats(void *heap, uint32 *stats, int size); + +#if BH_ENABLE_GC_VERIFY == 0 + +gc_object_t +gc_alloc_vo(void *heap, gc_size_t size); + +gc_object_t +gc_realloc_vo(void *heap, void *ptr, gc_size_t size); + +int +gc_free_vo(void *heap, gc_object_t obj); + +#if WASM_ENABLE_GC != 0 +gc_object_t +gc_alloc_wo(void *heap, gc_size_t size); + +void +gc_free_wo(void *vheap, void *ptr); +#endif + +#else /* else of BH_ENABLE_GC_VERIFY */ + +gc_object_t +gc_alloc_vo_internal(void *heap, gc_size_t size, const char *file, int line); + +gc_object_t +gc_realloc_vo_internal(void *heap, void *ptr, gc_size_t size, const char *file, + int line); + +int +gc_free_vo_internal(void *heap, gc_object_t obj, const char *file, int line); + +#if WASM_ENABLE_GC != 0 +gc_object_t +gc_alloc_wo_internal(void *heap, gc_size_t size, const char *file, int line); + +void +gc_free_wo_internal(void *vheap, void *ptr, const char *file, int line); +#endif + +/* clang-format off */ +#define gc_alloc_vo(heap, size) \ + gc_alloc_vo_internal(heap, size, __FILE__, __LINE__) + +#define gc_realloc_vo(heap, ptr, size) \ + gc_realloc_vo_internal(heap, ptr, size, __FILE__, __LINE__) + +#define gc_free_vo(heap, obj) \ + gc_free_vo_internal(heap, obj, __FILE__, __LINE__) + +#if WASM_ENABLE_GC != 0 +#define gc_alloc_wo(heap, size) \ + gc_alloc_wo_internal(heap, size, __FILE__, __LINE__) + +#define gc_free_wo(heap, obj) \ + gc_free_wo_internal(heap, obj, __FILE__, __LINE__) +#endif +/* clang-format on */ + +#endif /* end of BH_ENABLE_GC_VERIFY */ + +#if WASM_ENABLE_GC != 0 +/** + * Add gc object ref to the rootset of a gc heap. + * + * @param heap the heap to add the gc object to its rootset + * @param obj pointer to a valid WASM object managed by the gc heap. + * + * @return GC_SUCCESS if success, GC_ERROR otherwise + */ +int +gc_add_root(void *heap, gc_object_t obj); + +int +gci_gc_heap(void *heap); + +extra_info_node_t * +gc_search_extra_info_node(gc_handle_t handle, gc_object_t obj, + gc_size_t *p_index); + +/** + * Set finalizer to the given object, if another finalizer is set to the same + * object, the previous one will be cancelled + * + * @param handle handle of the heap + * @param obj object to set finalizer + * @param cb finalizer function to be called before this object is freed + * @param data custom data to be passed to finalizer function + * + * @return true if success, false otherwise + */ +bool +gc_set_finalizer(gc_handle_t handle, gc_object_t obj, gc_finalizer_t cb, + void *data); + +/** + * Unset finalizer to the given object + * + * @param handle handle of the heap + * @param obj object to unset finalizer + */ +void +gc_unset_finalizer(gc_handle_t handle, gc_object_t obj); + +#if WASM_ENABLE_THREAD_MGR == 0 +bool +wasm_runtime_traverse_gc_rootset(void *exec_env, void *heap); +#else +bool +wasm_runtime_traverse_gc_rootset(void *cluster, void *heap); +#endif + +bool +wasm_runtime_get_wasm_object_ref_list(gc_object_t obj, bool *p_is_compact_mode, + gc_uint32 *p_ref_num, + gc_uint16 **p_ref_list, + gc_uint32 *p_ref_start_offset); + +bool +wasm_runtime_get_wasm_object_extra_info_flag(gc_object_t obj); + +void +wasm_runtime_set_wasm_object_extra_info_flag(gc_object_t obj, bool set); + +void +wasm_runtime_gc_prepare(); + +void +wasm_runtime_gc_finalize(); +#endif /* end of WASM_ENABLE_GC != 0 */ + +#define GC_HEAP_STAT_SIZE (128 / 4) + +typedef struct { + int usage; + int usage_block; + int vo_usage; + int wo_usage; + int free; + int free_block; + int vo_free; + int wo_free; + int usage_sizes[GC_HEAP_STAT_SIZE]; + int free_sizes[GC_HEAP_STAT_SIZE]; +} gc_stat_t; + +void +gc_show_stat(gc_handle_t handle); + +#if WASM_ENABLE_GC != 0 +void +gc_show_fragment(gc_handle_t handle); + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +gc_dump_perf_profiling(gc_handle_t *handle); +#endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc_internal.h b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc_internal.h new file mode 100644 index 0000000..c902d57 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_gc_internal.h @@ -0,0 +1,384 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _EMS_GC_INTERNAL_H +#define _EMS_GC_INTERNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "bh_platform.h" +#include "ems_gc.h" + +/* HMU (heap memory unit) basic block type */ +typedef enum hmu_type_enum { + HMU_TYPE_MIN = 0, + HMU_TYPE_MAX = 3, + HMU_WO = 3, /* WASM Object */ + HMU_VO = 2, /* VM Object */ + HMU_FC = 1, + HMU_FM = 0 +} hmu_type_t; + +typedef struct hmu_struct { + gc_uint32 header; +} hmu_t; + +#if BH_ENABLE_GC_VERIFY != 0 + +#if UINTPTR_MAX > UINT32_MAX +/* 2 prefix paddings for 64-bit pointer */ +#define GC_OBJECT_PREFIX_PADDING_CNT 2 +#else +/* 3 prefix paddings for 32-bit pointer */ +#define GC_OBJECT_PREFIX_PADDING_CNT 3 +#endif +#define GC_OBJECT_SUFFIX_PADDING_CNT 4 +#define GC_OBJECT_PADDING_VALUE (0x12345678) + +typedef struct gc_object_prefix { + const char *file_name; + gc_int32 line_no; + gc_int32 size; + gc_uint32 padding[GC_OBJECT_PREFIX_PADDING_CNT]; +} gc_object_prefix_t; + +typedef struct gc_object_suffix { + gc_uint32 padding[GC_OBJECT_SUFFIX_PADDING_CNT]; +} gc_object_suffix_t; + +#define OBJ_PREFIX_SIZE (sizeof(gc_object_prefix_t)) +#define OBJ_SUFFIX_SIZE (sizeof(gc_object_suffix_t)) + +void +hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size, + const char *file_name, int line_no); + +void +hmu_verify(void *vheap, hmu_t *hmu); + +#define SKIP_OBJ_PREFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_PREFIX_SIZE)) +#define SKIP_OBJ_SUFFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_SUFFIX_SIZE)) + +#define OBJ_EXTRA_SIZE (HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE) + +#else /* else of BH_ENABLE_GC_VERIFY */ + +#define OBJ_PREFIX_SIZE 0 +#define OBJ_SUFFIX_SIZE 0 + +#define SKIP_OBJ_PREFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_PREFIX_SIZE)) +#define SKIP_OBJ_SUFFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_SUFFIX_SIZE)) + +#define OBJ_EXTRA_SIZE (HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE) + +#endif /* end of BH_ENABLE_GC_VERIFY */ + +#define hmu_obj_size(s) ((s)-OBJ_EXTRA_SIZE) + +#define GC_ALIGN_8(s) (((uint32)(s) + 7) & (uint32)~7) + +#define GC_SMALLEST_SIZE \ + GC_ALIGN_8(HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE + 8) +#define GC_GET_REAL_SIZE(x) \ + GC_ALIGN_8(HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE \ + + (((x) > 8) ? (x) : 8)) + +/** + * hmu bit operation + */ + +#define SETBIT(v, offset) (v) |= ((uint32)1 << (offset)) +#define GETBIT(v, offset) ((v) & ((uint32)1 << (offset)) ? 1 : 0) +#define CLRBIT(v, offset) (v) &= (~((uint32)1 << (offset))) + +/* clang-format off */ +#define SETBITS(v, offset, size, value) \ + do { \ + (v) &= ~((((uint32)1 << size) - 1) << offset); \ + (v) |= ((uint32)value << offset); \ + } while (0) +#define CLRBITS(v, offset, size) \ + (v) &= ~((((uint32)1 << size) - 1) << offset) +#define GETBITS(v, offset, size) \ + (((v) & (((((uint32)1 << size) - 1) << offset))) >> offset) +/* clang-format on */ + +/** + * gc object layout definition + */ + +#define HMU_SIZE (sizeof(hmu_t)) + +#define hmu_to_obj(hmu) (gc_object_t)(SKIP_OBJ_PREFIX((hmu_t *)(hmu) + 1)) +#define obj_to_hmu(obj) ((hmu_t *)((gc_uint8 *)(obj)-OBJ_PREFIX_SIZE) - 1) + +#define HMU_UT_SIZE 2 +#define HMU_UT_OFFSET 30 + +/* clang-format off */ +#define hmu_get_ut(hmu) \ + GETBITS((hmu)->header, HMU_UT_OFFSET, HMU_UT_SIZE) +#define hmu_set_ut(hmu, type) \ + SETBITS((hmu)->header, HMU_UT_OFFSET, HMU_UT_SIZE, type) +#define hmu_is_ut_valid(tp) \ + (tp >= HMU_TYPE_MIN && tp <= HMU_TYPE_MAX) +/* clang-format on */ + +/* P in use bit means the previous chunk is in use */ +#define HMU_P_OFFSET 29 + +#define hmu_mark_pinuse(hmu) SETBIT((hmu)->header, HMU_P_OFFSET) +#define hmu_unmark_pinuse(hmu) CLRBIT((hmu)->header, HMU_P_OFFSET) +#define hmu_get_pinuse(hmu) GETBIT((hmu)->header, HMU_P_OFFSET) + +#define HMU_WO_VT_SIZE 27 +#define HMU_WO_VT_OFFSET 0 +#define HMU_WO_MB_OFFSET 28 + +#define hmu_mark_wo(hmu) SETBIT((hmu)->header, HMU_WO_MB_OFFSET) +#define hmu_unmark_wo(hmu) CLRBIT((hmu)->header, HMU_WO_MB_OFFSET) +#define hmu_is_wo_marked(hmu) GETBIT((hmu)->header, HMU_WO_MB_OFFSET) + +/** + * The hmu size is divisible by 8, its lowest 3 bits are 0, so we only + * store its higher bits of bit [29..3], and bit [2..0] are not stored. + * After that, the maximal heap size can be enlarged from (1<<27) = 128MB + * to (1<<27) * 8 = 1GB. + */ +#define HMU_SIZE_SIZE 27 +#define HMU_SIZE_OFFSET 0 + +#define HMU_VO_FB_OFFSET 28 + +#define hmu_is_vo_freed(hmu) GETBIT((hmu)->header, HMU_VO_FB_OFFSET) +#define hmu_unfree_vo(hmu) CLRBIT((hmu)->header, HMU_VO_FB_OFFSET) + +#define hmu_get_size(hmu) \ + (GETBITS((hmu)->header, HMU_SIZE_OFFSET, HMU_SIZE_SIZE) << 3) +#define hmu_set_size(hmu, size) \ + SETBITS((hmu)->header, HMU_SIZE_OFFSET, HMU_SIZE_SIZE, ((size) >> 3)) + +/** + * HMU free chunk management + */ + +#ifndef HMU_NORMAL_NODE_CNT +#define HMU_NORMAL_NODE_CNT 32 +#endif +#define HMU_FC_NORMAL_MAX_SIZE ((HMU_NORMAL_NODE_CNT - 1) << 3) +#define HMU_IS_FC_NORMAL(size) ((size) < HMU_FC_NORMAL_MAX_SIZE) +#if HMU_FC_NORMAL_MAX_SIZE >= GC_MAX_HEAP_SIZE +#error "Too small GC_MAX_HEAP_SIZE" +#endif + +typedef struct hmu_normal_node { + hmu_t hmu_header; + gc_int32 next_offset; +} hmu_normal_node_t; + +typedef struct hmu_normal_list { + hmu_normal_node_t *next; +} hmu_normal_list_t; + +static inline hmu_normal_node_t * +get_hmu_normal_node_next(hmu_normal_node_t *node) +{ + return node->next_offset + ? (hmu_normal_node_t *)((uint8 *)node + node->next_offset) + : NULL; +} + +static inline void +set_hmu_normal_node_next(hmu_normal_node_t *node, hmu_normal_node_t *next) +{ + if (next) { + bh_assert((uint8 *)next - (uint8 *)node < INT32_MAX); + node->next_offset = (gc_int32)(intptr_t)((uint8 *)next - (uint8 *)node); + } + else { + node->next_offset = 0; + } +} + +/** + * Define hmu_tree_node as a packed struct, since it is at the 4-byte + * aligned address and the size of hmu_head is 4, so in 64-bit target, + * the left/right/parent fields will be at 8-byte aligned address, + * we can access them directly. + */ +#if UINTPTR_MAX == UINT64_MAX +#if defined(_MSC_VER) +__pragma(pack(push, 1)); +#define __attr_packed +#define __attr_aligned(a) +#elif defined(__GNUC__) || defined(__clang__) +#define __attr_packed __attribute__((packed)) +#define __attr_aligned(a) __attribute__((aligned(a))) +#else +#error "packed attribute isn't used to define struct hmu_tree_node" +#endif +#else /* else of UINTPTR_MAX == UINT64_MAX */ +#define __attr_packed +#define __attr_aligned(a) +#endif + +typedef struct hmu_tree_node { + hmu_t hmu_header; + struct hmu_tree_node *left; + struct hmu_tree_node *right; + struct hmu_tree_node *parent; + gc_size_t size; +} __attr_packed __attr_aligned(4) hmu_tree_node_t; + +#if UINTPTR_MAX == UINT64_MAX +#if defined(_MSC_VER) +__pragma(pack(pop)); +#endif +#endif + +bh_static_assert(sizeof(hmu_tree_node_t) == 8 + 3 * sizeof(void *)); +bh_static_assert(offsetof(hmu_tree_node_t, left) == 4); + +#define ASSERT_TREE_NODE_ALIGNED_ACCESS(tree_node) \ + do { \ + bh_assert((((uintptr_t)&tree_node->left) & (sizeof(uintptr_t) - 1)) \ + == 0); \ + } while (0) + +typedef struct gc_heap_struct { + /* for double checking*/ + gc_handle_t heap_id; + + gc_uint8 *base_addr; + gc_size_t current_size; + + korp_mutex lock; + + hmu_normal_list_t kfc_normal_list[HMU_NORMAL_NODE_CNT]; + +#if UINTPTR_MAX == UINT64_MAX + /* make kfc_tree_root_buf 4-byte aligned and not 8-byte aligned, + so kfc_tree_root's left/right/parent fields are 8-byte aligned + and we can access them directly */ + uint32 __padding; +#endif + uint8 kfc_tree_root_buf[sizeof(hmu_tree_node_t)]; + /* point to kfc_tree_root_buf, the order in kfc_tree is: + size[left] <= size[cur] < size[right] */ + hmu_tree_node_t *kfc_tree_root; + +#if WASM_ENABLE_GC != 0 + /* for rootset enumeration of private heap*/ + void *root_set; + +#if WASM_ENABLE_THREAD_MGR == 0 + /* exec_env of current wasm module instance */ + void *exec_env; +#else + /* thread cluster of current module instances */ + void *cluster; +#endif + + /* whether the fast mode of marking process that requires + additional memory fails. When the fast mode fails, the + marking process can still be done in the slow mode, which + doesn't need additional memory (by walking through all + blocks and marking sucessors of marked nodes until no new + node is marked). TODO: slow mode is not implemented. */ + unsigned is_fast_marking_failed : 1; + + /* whether the heap is doing reclaim */ + unsigned is_doing_reclaim : 1; + + /* Whether the heap can do reclaim */ + unsigned is_reclaim_enabled : 1; +#endif + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + /* whether heap is corrupted, e.g. the hmu nodes are modified + by user */ + bool is_heap_corrupted; +#endif + + gc_size_t init_size; + gc_size_t highmark_size; + gc_size_t total_free_size; + +#if WASM_ENABLE_GC != 0 + gc_size_t gc_threshold; + gc_size_t gc_threshold_factor; + gc_size_t total_gc_count; + gc_size_t total_gc_time; + gc_size_t max_gc_time; + /* Usually there won't be too many extra info node, so we try to use a fixed + * array to store them, if the fixed array don't have enough space to store + * the nodes, a new space will be allocated from heap */ + extra_info_node_t *extra_info_normal_nodes[EXTRA_INFO_NORMAL_NODE_CNT]; + /* Used to store extra information such as finalizer for specified nodes, we + * introduce a seperate space to store these information so only nodes who + * really require extra information will occupy additional memory spaces. */ + extra_info_node_t **extra_info_nodes; + gc_size_t extra_info_node_cnt; + gc_size_t extra_info_node_capacity; +#endif +#if GC_STAT_DATA != 0 + gc_uint64 total_size_allocated; + gc_uint64 total_size_freed; +#endif +} gc_heap_t; + +#if WASM_ENABLE_GC != 0 + +#define GC_DEFAULT_THRESHOLD_FACTOR 300 + +static inline void +gc_update_threshold(gc_heap_t *heap) +{ + heap->gc_threshold = + heap->total_free_size * heap->gc_threshold_factor / 1000; +} + +#define gct_vm_mutex_init os_mutex_init +#define gct_vm_mutex_destroy os_mutex_destroy +#define gct_vm_mutex_lock os_mutex_lock +#define gct_vm_mutex_unlock os_mutex_unlock +#define gct_vm_gc_prepare wasm_runtime_gc_prepare +#define gct_vm_gc_finished wasm_runtime_gc_finalize +#define gct_vm_begin_rootset_enumeration wasm_runtime_traverse_gc_rootset +#define gct_vm_get_wasm_object_ref_list wasm_runtime_get_wasm_object_ref_list +#define gct_vm_get_extra_info_flag wasm_runtime_get_wasm_object_extra_info_flag +#define gct_vm_set_extra_info_flag wasm_runtime_set_wasm_object_extra_info_flag + +#endif /* end of WAMS_ENABLE_GC != 0 */ + +/** + * MISC internal used APIs + */ + +bool +gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size); + +int +gci_is_heap_valid(gc_heap_t *heap); + +/** + * Verify heap integrity + */ +void +gci_verify_heap(gc_heap_t *heap); + +/** + * Dump heap nodes + */ +void +gci_dump(gc_heap_t *heap); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _EMS_GC_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_hmu.c b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_hmu.c new file mode 100644 index 0000000..e4c79e3 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_hmu.c @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" + +#if BH_ENABLE_GC_VERIFY != 0 + +/** + * Set default value to prefix and suffix + * @param hmu should not be NULL and should have been correctly initilized + * (except prefix and suffix part) + * @param tot_size is offered here because hmu_get_size can not be used + * till now. tot_size should not be smaller than OBJ_EXTRA_SIZE. + * For VO, tot_size should be equal to object total size. + */ +void +hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size, + const char *file_name, int line_no) +{ + gc_object_prefix_t *prefix = NULL; + gc_object_suffix_t *suffix = NULL; + gc_uint32 i = 0; + + bh_assert(hmu); + bh_assert(hmu_get_ut(hmu) == HMU_WO || hmu_get_ut(hmu) == HMU_VO); + bh_assert(tot_size >= OBJ_EXTRA_SIZE); + bh_assert(!(tot_size & 7)); + bh_assert(hmu_get_ut(hmu) != HMU_VO || hmu_get_size(hmu) >= tot_size); + + prefix = (gc_object_prefix_t *)(hmu + 1); + suffix = + (gc_object_suffix_t *)((gc_uint8 *)hmu + tot_size - OBJ_SUFFIX_SIZE); + prefix->file_name = file_name; + prefix->line_no = line_no; + prefix->size = tot_size; + + for (i = 0; i < GC_OBJECT_PREFIX_PADDING_CNT; i++) { + prefix->padding[i] = GC_OBJECT_PADDING_VALUE; + } + + for (i = 0; i < GC_OBJECT_SUFFIX_PADDING_CNT; i++) { + suffix->padding[i] = GC_OBJECT_PADDING_VALUE; + } +} + +void +hmu_verify(void *vheap, hmu_t *hmu) +{ +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + gc_heap_t *heap = (gc_heap_t *)vheap; +#endif + gc_object_prefix_t *prefix = NULL; + gc_object_suffix_t *suffix = NULL; + gc_uint32 i = 0; + hmu_type_t ut; + gc_size_t size = 0; + int is_padding_ok = 1; + + bh_assert(hmu); + ut = hmu_get_ut(hmu); + bh_assert(hmu_is_ut_valid(ut)); + + prefix = (gc_object_prefix_t *)(hmu + 1); + size = prefix->size; + suffix = (gc_object_suffix_t *)((gc_uint8 *)hmu + size - OBJ_SUFFIX_SIZE); + + if (ut == HMU_VO || ut == HMU_WO) { + /* check padding*/ + for (i = 0; i < GC_OBJECT_PREFIX_PADDING_CNT; i++) { + if (prefix->padding[i] != GC_OBJECT_PADDING_VALUE) { + is_padding_ok = 0; + break; + } + } + for (i = 0; i < GC_OBJECT_SUFFIX_PADDING_CNT; i++) { + if (suffix->padding[i] != GC_OBJECT_PADDING_VALUE) { + is_padding_ok = 0; + break; + } + } + + if (!is_padding_ok) { + LOG_ERROR("Invalid padding for object created at %s:%d\n", + (prefix->file_name ? prefix->file_name : ""), + prefix->line_no); +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + heap->is_heap_corrupted = true; +#endif + } + } +} + +#endif /* end of BH_ENABLE_GC_VERIFY */ diff --git a/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_kfc.c b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_kfc.c new file mode 100644 index 0000000..8ab2df5 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/ems/ems_kfc.c @@ -0,0 +1,527 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" + +static gc_handle_t +gc_init_internal(gc_heap_t *heap, char *base_addr, gc_size_t heap_max_size) +{ + hmu_tree_node_t *root = NULL, *q = NULL; + int ret; + + memset(heap, 0, sizeof *heap); + memset(base_addr, 0, heap_max_size); + + ret = os_mutex_init(&heap->lock); + if (ret != BHT_OK) { + LOG_ERROR("[GC_ERROR]failed to init lock\n"); + return NULL; + } + + /* init all data structures*/ + heap->current_size = heap_max_size; + heap->base_addr = (gc_uint8 *)base_addr; + heap->heap_id = (gc_handle_t)heap; + + heap->total_free_size = heap->current_size; + heap->highmark_size = 0; +#if WASM_ENABLE_GC != 0 + heap->gc_threshold_factor = GC_DEFAULT_THRESHOLD_FACTOR; + gc_update_threshold(heap); +#endif + + root = heap->kfc_tree_root = (hmu_tree_node_t *)heap->kfc_tree_root_buf; + memset(root, 0, sizeof *root); + root->size = sizeof *root; + hmu_set_ut(&root->hmu_header, HMU_FC); + hmu_set_size(&root->hmu_header, sizeof *root); + + q = (hmu_tree_node_t *)heap->base_addr; + memset(q, 0, sizeof *q); + hmu_set_ut(&q->hmu_header, HMU_FC); + hmu_set_size(&q->hmu_header, heap->current_size); + + ASSERT_TREE_NODE_ALIGNED_ACCESS(q); + ASSERT_TREE_NODE_ALIGNED_ACCESS(root); + + hmu_mark_pinuse(&q->hmu_header); + root->right = q; + q->parent = root; + q->size = heap->current_size; + + bh_assert(root->size <= HMU_FC_NORMAL_MAX_SIZE); + + return heap; +} + +gc_handle_t +gc_init_with_pool(char *buf, gc_size_t buf_size) +{ + char *buf_end = buf + buf_size; + char *buf_aligned = (char *)(((uintptr_t)buf + 7) & (uintptr_t)~7); + char *base_addr = buf_aligned + sizeof(gc_heap_t); + gc_heap_t *heap = (gc_heap_t *)buf_aligned; + gc_size_t heap_max_size; + + if (buf_size < APP_HEAP_SIZE_MIN) { + LOG_ERROR("[GC_ERROR]heap init buf size (%" PRIu32 ") < %" PRIu32 "\n", + buf_size, (uint32)APP_HEAP_SIZE_MIN); + return NULL; + } + + base_addr = + (char *)(((uintptr_t)base_addr + 7) & (uintptr_t)~7) + GC_HEAD_PADDING; + heap_max_size = (uint32)(buf_end - base_addr) & (uint32)~7; + +#if WASM_ENABLE_MEMORY_TRACING != 0 + os_printf("Heap created, total size: %u\n", buf_size); + os_printf(" heap struct size: %u\n", sizeof(gc_heap_t)); + os_printf(" actual heap size: %u\n", heap_max_size); + os_printf(" padding bytes: %u\n", + buf_size - sizeof(gc_heap_t) - heap_max_size); +#endif + return gc_init_internal(heap, base_addr, heap_max_size); +} + +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size) +{ + gc_heap_t *heap = (gc_heap_t *)struct_buf; + char *base_addr = pool_buf + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf + pool_buf_size; + gc_size_t heap_max_size; + + if ((((uintptr_t)struct_buf) & 7) != 0) { + LOG_ERROR("[GC_ERROR]heap init struct buf not 8-byte aligned\n"); + return NULL; + } + + if (struct_buf_size < sizeof(gc_handle_t)) { + LOG_ERROR("[GC_ERROR]heap init struct buf size (%" PRIu32 ") < %zu\n", + struct_buf_size, sizeof(gc_handle_t)); + return NULL; + } + + if ((((uintptr_t)pool_buf) & 7) != 0) { + LOG_ERROR("[GC_ERROR]heap init pool buf not 8-byte aligned\n"); + return NULL; + } + + if (pool_buf_size < APP_HEAP_SIZE_MIN) { + LOG_ERROR("[GC_ERROR]heap init buf size (%" PRIu32 ") < %u\n", + pool_buf_size, APP_HEAP_SIZE_MIN); + return NULL; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr) & (uint32)~7; + +#if WASM_ENABLE_MEMORY_TRACING != 0 + os_printf("Heap created, total size: %u\n", + struct_buf_size + pool_buf_size); + os_printf(" heap struct size: %u\n", sizeof(gc_heap_t)); + os_printf(" actual heap size: %u\n", heap_max_size); + os_printf(" padding bytes: %u\n", pool_buf_size - heap_max_size); +#endif + return gc_init_internal(heap, base_addr, heap_max_size); +} + +int +gc_destroy_with_pool(gc_handle_t handle) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + int ret = GC_SUCCESS; + +#if WASM_ENABLE_GC != 0 + gc_size_t i = 0; + + if (heap->extra_info_node_cnt > 0) { + for (i = 0; i < heap->extra_info_node_cnt; i++) { + extra_info_node_t *node = heap->extra_info_nodes[i]; +#if BH_ENABLE_GC_VERIFY != 0 + os_printf("Memory leak detected: gc object [%p] not claimed\n", + node->obj); +#endif + bh_assert(heap->is_reclaim_enabled); + node->finalizer(node->obj, node->data); + + BH_FREE(heap->extra_info_nodes[i]); + } + + if (heap->extra_info_nodes != heap->extra_info_normal_nodes) { + BH_FREE(heap->extra_info_nodes); + } + } +#endif + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_t *cur = (hmu_t *)heap->base_addr; + hmu_t *end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + if ( +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + !heap->is_heap_corrupted && +#endif + (hmu_t *)((char *)cur + hmu_get_size(cur)) != end) { + LOG_WARNING("Memory leak detected:\n"); + gci_dump(heap); + ret = GC_ERROR; + } +#endif + + os_mutex_destroy(&heap->lock); + memset(heap->base_addr, 0, heap->current_size); + memset(heap, 0, sizeof(gc_heap_t)); + return ret; +} + +#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_THREAD_MGR == 0 +void +gc_enable_gc_reclaim(gc_handle_t handle, void *exec_env) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + + heap->is_reclaim_enabled = 1; + heap->exec_env = exec_env; +} +#else +void +gc_enable_gc_reclaim(gc_handle_t handle, void *cluster) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + + heap->is_reclaim_enabled = 1; + heap->cluster = cluster; +} +#endif +#endif + +uint32 +gc_get_heap_struct_size() +{ + return sizeof(gc_heap_t); +} + +static void +adjust_ptr(uint8 **p_ptr, intptr_t offset) +{ + if (*p_ptr) + *p_ptr = (uint8 *)((intptr_t)(*p_ptr) + offset); +} + +int +gc_migrate(gc_handle_t handle, char *pool_buf_new, gc_size_t pool_buf_size) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + char *base_addr_new = pool_buf_new + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf_new + pool_buf_size; + intptr_t offset = (uint8 *)base_addr_new - (uint8 *)heap->base_addr; + hmu_t *cur = NULL, *end = NULL; + hmu_tree_node_t *tree_node; + uint8 **p_left, **p_right, **p_parent; + gc_size_t heap_max_size, size; + + if ((((uintptr_t)pool_buf_new) & 7) != 0) { + LOG_ERROR("[GC_ERROR]heap migrate pool buf not 8-byte aligned\n"); + return GC_ERROR; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr_new) & (uint32)~7; + + if (pool_buf_end < base_addr_new || heap_max_size < heap->current_size) { + LOG_ERROR("[GC_ERROR]heap migrate invlaid pool buf size\n"); + return GC_ERROR; + } + + if (offset == 0) + return 0; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + return GC_ERROR; + } +#endif + + heap->base_addr = (uint8 *)base_addr_new; + + ASSERT_TREE_NODE_ALIGNED_ACCESS(heap->kfc_tree_root); + + p_left = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, left)); + p_right = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, right)); + p_parent = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, parent)); + adjust_ptr(p_left, offset); + adjust_ptr(p_right, offset); + adjust_ptr(p_parent, offset); + + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + size = hmu_get_size(cur); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (size <= 0 || size > (uint32)((uint8 *)end - (uint8 *)cur)) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + heap->is_heap_corrupted = true; + return GC_ERROR; + } +#endif + + if (hmu_get_ut(cur) == HMU_FC && !HMU_IS_FC_NORMAL(size)) { + tree_node = (hmu_tree_node_t *)cur; + + ASSERT_TREE_NODE_ALIGNED_ACCESS(tree_node); + + p_left = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, left)); + p_right = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, right)); + p_parent = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, parent)); + adjust_ptr(p_left, offset); + adjust_ptr(p_right, offset); + if (tree_node->parent != heap->kfc_tree_root) + /* The root node belongs to heap structure, + it is fixed part and isn't changed. */ + adjust_ptr(p_parent, offset); + } + cur = (hmu_t *)((char *)cur + size); + } + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (cur != end) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + heap->is_heap_corrupted = true; + return GC_ERROR; + } +#else + bh_assert(cur == end); +#endif + + return 0; +} + +bool +gc_is_heap_corrupted(gc_handle_t handle) +{ +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + gc_heap_t *heap = (gc_heap_t *)handle; + + return heap->is_heap_corrupted ? true : false; +#else + return false; +#endif +} + +#if BH_ENABLE_GC_VERIFY != 0 +void +gci_verify_heap(gc_heap_t *heap) +{ + hmu_t *cur = NULL, *end = NULL; + + bh_assert(heap && gci_is_heap_valid(heap)); + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)(heap->base_addr + heap->current_size); + while (cur < end) { + hmu_verify(heap, cur); + cur = (hmu_t *)((gc_uint8 *)cur + hmu_get_size(cur)); + } + bh_assert(cur == end); +} +#endif + +void +gc_heap_stat(void *heap_ptr, gc_stat_t *stat) +{ + hmu_t *cur = NULL, *end = NULL; + hmu_type_t ut; + gc_size_t size; + gc_heap_t *heap = (gc_heap_t *)heap_ptr; + + memset(stat, 0, sizeof(gc_stat_t)); + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + ut = hmu_get_ut(cur); + size = hmu_get_size(cur); + bh_assert(size > 0); + + if (ut == HMU_FC || ut == HMU_FM + || (ut == HMU_VO && hmu_is_vo_freed(cur)) + || (ut == HMU_WO && !hmu_is_wo_marked(cur))) { + if (ut == HMU_VO) + stat->vo_free += size; + if (ut == HMU_WO) + stat->wo_free += size; + stat->free += size; + stat->free_block++; + if (size / sizeof(int) < GC_HEAP_STAT_SIZE - 1) + stat->free_sizes[size / sizeof(int)] += 1; + else + stat->free_sizes[GC_HEAP_STAT_SIZE - 1] += 1; + } + else { + if (ut == HMU_VO) + stat->vo_usage += size; + if (ut == HMU_WO) + stat->wo_usage += size; + stat->usage += size; + stat->usage_block++; + if (size / sizeof(int) < GC_HEAP_STAT_SIZE - 1) + stat->usage_sizes[size / sizeof(int)] += 1; + else + stat->usage_sizes[GC_HEAP_STAT_SIZE - 1] += 1; + } + + cur = (hmu_t *)((char *)cur + size); + } +} + +void +gc_print_stat(void *heap_ptr, int verbose) +{ + gc_stat_t stat; + int i; + + bh_assert(heap_ptr != NULL); + gc_heap_t *heap = (gc_heap_t *)(heap_ptr); + + gc_heap_stat(heap, &stat); + + os_printf("# stat %s %p use %d free %d \n", "instance", heap, stat.usage, + stat.free); + os_printf("# stat %s %p wo_usage %d vo_usage %d \n", "instance", heap, + stat.wo_usage, stat.vo_usage); + os_printf("# stat %s %p wo_free %d vo_free %d \n", "instance", heap, + stat.wo_free, stat.vo_free); +#if WASM_ENABLE_GC == 0 + os_printf("# stat free size %" PRIu32 " high %" PRIu32 "\n", + heap->total_free_size, heap->highmark_size); +#else + os_printf("# stat gc %" PRIu32 " free size %" PRIu32 " high %" PRIu32 "\n", + heap->total_gc_count, heap->total_free_size, heap->highmark_size); +#endif + if (verbose) { + os_printf("usage sizes: \n"); + for (i = 0; i < GC_HEAP_STAT_SIZE; i++) + if (stat.usage_sizes[i]) + os_printf(" %d: %d; ", i * 4, stat.usage_sizes[i]); + os_printf(" \n"); + os_printf("free sizes: \n"); + for (i = 0; i < GC_HEAP_STAT_SIZE; i++) + if (stat.free_sizes[i]) + os_printf(" %d: %d; ", i * 4, stat.free_sizes[i]); + } +} + +void * +gc_heap_stats(void *heap_arg, uint32 *stats, int size) +{ + int i; + gc_heap_t *heap = (gc_heap_t *)heap_arg; + + if (!gci_is_heap_valid(heap)) { + for (i = 0; i < size; i++) + stats[i] = 0; + return NULL; + } + + for (i = 0; i < size; i++) { + switch (i) { + case GC_STAT_TOTAL: + stats[i] = heap->current_size; + break; + case GC_STAT_FREE: + stats[i] = heap->total_free_size; + break; + case GC_STAT_HIGHMARK: + stats[i] = heap->highmark_size; + break; +#if WASM_ENABLE_GC != 0 + case GC_STAT_COUNT: + stats[i] = heap->total_gc_count; + break; + case GC_STAT_TIME: + stats[i] = heap->total_gc_time; + break; +#endif + default: + break; + } + } + + return heap; +} + +void +gc_traverse_tree(hmu_tree_node_t *node, gc_size_t *stats, int *n) +{ + if (!node) + return; + + if (*n > 0) + gc_traverse_tree(node->right, stats, n); + + if (*n > 0) { + (*n)--; + stats[*n] = node->size; + } + + if (*n > 0) + gc_traverse_tree(node->left, stats, n); +} + +void +gc_show_stat(void *heap) +{ + + uint32 stats[GC_STAT_MAX]; + + heap = gc_heap_stats(heap, stats, GC_STAT_MAX); + + os_printf("\n[GC stats %p] %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32 + " %" PRIu32 "\n", + heap, stats[0], stats[1], stats[2], stats[3], stats[4]); +} + +#if WASM_ENABLE_GC != 0 +void +gc_show_fragment(void *heap_arg) +{ + uint32 stats[3]; + int n = 3; + gc_heap_t *heap = (gc_heap_t *)heap_arg; + + memset(stats, 0, n * sizeof(int)); + gct_vm_mutex_lock(&heap->lock); + gc_traverse_tree(heap->kfc_tree_root, (gc_size_t *)stats, &n); + gct_vm_mutex_unlock(&heap->lock); + os_printf("\n[GC %p top sizes] %" PRIu32 " %" PRIu32 " %" PRIu32 "\n", heap, + stats[0], stats[1], stats[2]); +} + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +gc_dump_perf_profiling(gc_handle_t *handle) +{ + gc_heap_t *gc_heap_handle = (void *)handle; + if (gc_heap_handle) { + os_printf("\nGC performance summary\n"); + os_printf(" Total GC time (ms): %u\n", + gc_heap_handle->total_gc_time); + os_printf(" Max GC time (ms): %u\n", gc_heap_handle->max_gc_time); + } + else { + os_printf("Failed to dump GC performance\n"); + } +} +#endif +#endif diff --git a/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.c b/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.c new file mode 100644 index 0000000..df1a4de --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.c @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "mem_alloc.h" +#include + +#if DEFAULT_MEM_ALLOCATOR == MEM_ALLOCATOR_EMS + +#include "ems/ems_gc.h" + +mem_allocator_t +mem_allocator_create(void *mem, uint32_t size) +{ + return gc_init_with_pool((char *)mem, size); +} + +mem_allocator_t +mem_allocator_create_with_struct_and_pool(void *struct_buf, + uint32_t struct_buf_size, + void *pool_buf, + uint32_t pool_buf_size) +{ + return gc_init_with_struct_and_pool((char *)struct_buf, struct_buf_size, + pool_buf, pool_buf_size); +} + +int +mem_allocator_destroy(mem_allocator_t allocator) +{ + return gc_destroy_with_pool((gc_handle_t)allocator); +} + +uint32 +mem_allocator_get_heap_struct_size() +{ + return gc_get_heap_struct_size(); +} + +void * +mem_allocator_malloc(mem_allocator_t allocator, uint32_t size) +{ + return gc_alloc_vo((gc_handle_t)allocator, size); +} + +void * +mem_allocator_realloc(mem_allocator_t allocator, void *ptr, uint32_t size) +{ + return gc_realloc_vo((gc_handle_t)allocator, ptr, size); +} + +void +mem_allocator_free(mem_allocator_t allocator, void *ptr) +{ + if (ptr) + gc_free_vo((gc_handle_t)allocator, ptr); +} + +#if WASM_ENABLE_GC != 0 +void * +mem_allocator_malloc_with_gc(mem_allocator_t allocator, uint32_t size) +{ + return gc_alloc_wo((gc_handle_t)allocator, size); +} + +#if WASM_GC_MANUALLY != 0 +void +mem_allocator_free_with_gc(mem_allocator_t allocator, void *ptr) +{ + if (ptr) + gc_free_wo((gc_handle_t)allocator, ptr); +} +#endif + +#if WASM_ENABLE_THREAD_MGR == 0 +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *exec_env) +{ + gc_enable_gc_reclaim((gc_handle_t)allocator, exec_env); +} +#else +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *cluster) +{ + gc_enable_gc_reclaim((gc_handle_t)allocator, cluster); +} +#endif + +int +mem_allocator_add_root(mem_allocator_t allocator, WASMObjectRef obj) +{ + return gc_add_root((gc_handle_t)allocator, (gc_object_t)obj); +} +#endif + +int +mem_allocator_migrate(mem_allocator_t allocator, char *pool_buf_new, + uint32 pool_buf_size) +{ + return gc_migrate((gc_handle_t)allocator, pool_buf_new, pool_buf_size); +} + +bool +mem_allocator_is_heap_corrupted(mem_allocator_t allocator) +{ + return gc_is_heap_corrupted((gc_handle_t)allocator); +} + +bool +mem_allocator_get_alloc_info(mem_allocator_t allocator, void *mem_alloc_info) +{ + gc_heap_stats((gc_handle_t)allocator, mem_alloc_info, 3); + return true; +} + +#if WASM_ENABLE_GC != 0 +bool +mem_allocator_set_gc_finalizer(mem_allocator_t allocator, void *obj, + gc_finalizer_t cb, void *data) +{ + return gc_set_finalizer((gc_handle_t)allocator, (gc_object_t)obj, cb, data); +} + +void +mem_allocator_unset_gc_finalizer(mem_allocator_t allocator, void *obj) +{ + gc_unset_finalizer((gc_handle_t)allocator, (gc_object_t)obj); +} + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +mem_allocator_dump_perf_profiling(mem_allocator_t allocator) +{ + gc_dump_perf_profiling((gc_handle_t)allocator); +} +#endif + +#endif + +#else /* else of DEFAULT_MEM_ALLOCATOR */ + +#include "tlsf/tlsf.h" + +typedef struct mem_allocator_tlsf { + tlsf_t tlsf; + korp_mutex lock; +} mem_allocator_tlsf; + +mem_allocator_t +mem_allocator_create(void *mem, uint32_t size) +{ + mem_allocator_tlsf *allocator_tlsf; + tlsf_t tlsf; + char *mem_aligned = (char *)(((uintptr_t)mem + 3) & ~3); + + if (size < 1024) { + printf("Create mem allocator failed: pool size must be " + "at least 1024 bytes.\n"); + return NULL; + } + + size -= mem_aligned - (char *)mem; + mem = (void *)mem_aligned; + + tlsf = tlsf_create_with_pool(mem, size); + if (!tlsf) { + printf("Create mem allocator failed: tlsf_create_with_pool failed.\n"); + return NULL; + } + + allocator_tlsf = tlsf_malloc(tlsf, sizeof(mem_allocator_tlsf)); + if (!allocator_tlsf) { + printf("Create mem allocator failed: tlsf_malloc failed.\n"); + tlsf_destroy(tlsf); + return NULL; + } + + allocator_tlsf->tlsf = tlsf; + + if (os_mutex_init(&allocator_tlsf->lock)) { + printf("Create mem allocator failed: tlsf_malloc failed.\n"); + tlsf_free(tlsf, allocator_tlsf); + tlsf_destroy(tlsf); + return NULL; + } + + return allocator_tlsf; +} + +void +mem_allocator_destroy(mem_allocator_t allocator) +{ + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + tlsf_t tlsf = allocator_tlsf->tlsf; + + os_mutex_destroy(&allocator_tlsf->lock); + tlsf_free(tlsf, allocator_tlsf); + tlsf_destroy(tlsf); +} + +void * +mem_allocator_malloc(mem_allocator_t allocator, uint32_t size) +{ + void *ret; + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + + if (size == 0) + /* tlsf doesn't allow to allocate 0 byte */ + size = 1; + + os_mutex_lock(&allocator_tlsf->lock); + ret = tlsf_malloc(allocator_tlsf->tlsf, size); + os_mutex_unlock(&allocator_tlsf->lock); + return ret; +} + +void * +mem_allocator_realloc(mem_allocator_t allocator, void *ptr, uint32_t size) +{ + void *ret; + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + + if (size == 0) + /* tlsf doesn't allow to allocate 0 byte */ + size = 1; + + os_mutex_lock(&allocator_tlsf->lock); + ret = tlsf_realloc(allocator_tlsf->tlsf, ptr, size); + os_mutex_unlock(&allocator_tlsf->lock); + return ret; +} + +void +mem_allocator_free(mem_allocator_t allocator, void *ptr) +{ + if (ptr) { + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + os_mutex_lock(&allocator_tlsf->lock); + tlsf_free(allocator_tlsf->tlsf, ptr); + os_mutex_unlock(&allocator_tlsf->lock); + } +} + +int +mem_allocator_migrate(mem_allocator_t allocator, mem_allocator_t allocator_old) +{ + return tlsf_migrate((mem_allocator_tlsf *)allocator, + (mem_allocator_tlsf *)allocator_old); +} + +#endif /* end of DEFAULT_MEM_ALLOCATOR */ diff --git a/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.cmake b/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.cmake new file mode 100644 index 0000000..5f47cce --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.cmake @@ -0,0 +1,33 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + + +set (MEM_ALLOC_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include_directories(${MEM_ALLOC_DIR}) + +if (WAMR_BUILD_GC_VERIFY EQUAL 1) + add_definitions (-DBH_ENABLE_GC_VERIFY=1) +endif () + +if (NOT DEFINED WAMR_BUILD_GC_CORRUPTION_CHECK) + # Disable memory allocator heap corruption check + # when GC is enabled + if (WAMR_BUILD_GC EQUAL 1) + set (WAMR_BUILD_GC_CORRUPTION_CHECK 0) + else () + set (WAMR_BUILD_GC_CORRUPTION_CHECK 1) + endif () +endif () + +if (WAMR_BUILD_GC_CORRUPTION_CHECK EQUAL 0) + add_definitions (-DBH_ENABLE_GC_CORRUPTION_CHECK=0) +endif () + +file (GLOB_RECURSE source_all + ${MEM_ALLOC_DIR}/ems/*.c + ${MEM_ALLOC_DIR}/tlsf/*.c + ${MEM_ALLOC_DIR}/mem_alloc.c) + +set (MEM_ALLOC_SHARED_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.h b/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.h new file mode 100644 index 0000000..ca683d1 --- /dev/null +++ b/wasm-micro-runtime/core/shared/mem-alloc/mem_alloc.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __MEM_ALLOC_H +#define __MEM_ALLOC_H + +#include "bh_platform.h" +#if WASM_ENABLE_GC != 0 +#include "../../common/gc/gc_object.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *mem_allocator_t; + +typedef void (*gc_finalizer_t)(void *obj, void *data); + +mem_allocator_t +mem_allocator_create(void *mem, uint32_t size); + +mem_allocator_t +mem_allocator_create_with_struct_and_pool(void *struct_buf, + uint32_t struct_buf_size, + void *pool_buf, + uint32_t pool_buf_size); + +int +mem_allocator_destroy(mem_allocator_t allocator); + +uint32 +mem_allocator_get_heap_struct_size(void); + +void * +mem_allocator_malloc(mem_allocator_t allocator, uint32_t size); + +void * +mem_allocator_realloc(mem_allocator_t allocator, void *ptr, uint32_t size); + +void +mem_allocator_free(mem_allocator_t allocator, void *ptr); + +int +mem_allocator_migrate(mem_allocator_t allocator, char *pool_buf_new, + uint32 pool_buf_size); + +bool +mem_allocator_is_heap_corrupted(mem_allocator_t allocator); + +#if WASM_ENABLE_GC != 0 +void * +mem_allocator_malloc_with_gc(mem_allocator_t allocator, uint32_t size); + +#if WASM_GC_MANUALLY != 0 +void +mem_allocator_free_with_gc(mem_allocator_t allocator, void *ptr); +#endif + +#if WASM_ENABLE_THREAD_MGR == 0 +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *exec_env); +#else +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *cluster); +#endif + +int +mem_allocator_add_root(mem_allocator_t allocator, WASMObjectRef obj); + +bool +mem_allocator_set_gc_finalizer(mem_allocator_t allocator, void *obj, + gc_finalizer_t cb, void *data); + +void +mem_allocator_unset_gc_finalizer(mem_allocator_t allocator, void *obj); + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +mem_allocator_dump_perf_profiling(mem_allocator_t allocator); +#endif +#endif /* end of WASM_ENABLE_GC != 0 */ + +bool +mem_allocator_get_alloc_info(mem_allocator_t allocator, void *mem_alloc_info); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef __MEM_ALLOC_H */ diff --git a/wasm-micro-runtime/core/shared/platform/README.md b/wasm-micro-runtime/core/shared/platform/README.md new file mode 100644 index 0000000..de6f1cc --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/README.md @@ -0,0 +1,10 @@ +This folder contains the platform abstract layer for multiple platforms. To support a new platform, you can simply create a new folder here and implement all the APIs defined in [`include`](./include) folder. + + + +Refer to [port_wamr.md](../../../doc/port_wamr.md) for how to port WAMR to a target platform. + + + + + diff --git a/wasm-micro-runtime/core/shared/platform/alios/alios_platform.c b/wasm-micro-runtime/core/shared/platform/alios/alios_platform.c new file mode 100644 index 0000000..24bf9c7 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/alios/alios_platform.c @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +int +os_thread_sys_init(); + +void +os_thread_sys_destroy(); + +int +bh_platform_init() +{ + return os_thread_sys_init(); +} + +void +bh_platform_destroy() +{ + os_thread_sys_destroy(); +} + +void * +os_malloc(unsigned size) +{ + return NULL; +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return NULL; +} + +void +os_free(void *ptr) +{} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + if ((uint64)size >= UINT32_MAX) + return NULL; + return BH_MALLOC((uint32)size); +} + +void +os_munmap(void *addr, size_t size) +{ + return BH_FREE(addr); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + return 0; +} + +void +os_dcache_flush() +{} + +void +os_icache_flush(void *start, size_t len) +{} diff --git a/wasm-micro-runtime/core/shared/platform/alios/alios_thread.c b/wasm-micro-runtime/core/shared/platform/alios/alios_thread.c new file mode 100644 index 0000000..9fe927d --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/alios/alios_thread.c @@ -0,0 +1,365 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +/* clang-format off */ +#define bh_assert(v) do { \ + if (!(v)) { \ + printf("\nASSERTION FAILED: %s, at %s, line %d\n", \ + #v, __FILE__, __LINE__); \ + aos_reboot(); \ + while (1); \ + } \ +} while (0) +/* clang-format on */ + +struct os_thread_data; +typedef struct os_thread_wait_node { + aos_sem_t sem; + os_thread_wait_list next; +} os_thread_wait_node; + +typedef struct os_thread_data { + /* Thread body */ + aos_task_t thread; + /* Thread start routine */ + thread_start_routine_t start_routine; + /* Thread start routine argument */ + void *arg; + /* Thread local root */ + void *tlr; + /* Wait node of current thread */ + os_thread_wait_node wait_node; + /* Lock for waiting list */ + aos_mutex_t wait_list_lock; + /* Waiting list of other threads who are joining this thread */ + os_thread_wait_list thread_wait_list; +} os_thread_data; + +static bool is_thread_sys_inited = false; + +/* Thread data of supervisor thread */ +static os_thread_data supervisor_thread_data; + +/* Thread data key */ +static aos_task_key_t thread_data_key; + +/* Thread name index */ +static int thread_name_index; + +int +os_thread_sys_init() +{ + if (is_thread_sys_inited) + return BHT_OK; + + if (aos_task_key_create(&thread_data_key) != 0) + return BHT_ERROR; + + /* Initialize supervisor thread data */ + memset(&supervisor_thread_data, 0, sizeof(supervisor_thread_data)); + + if (aos_sem_new(&supervisor_thread_data.wait_node.sem, 1) != 0) { + aos_task_key_delete(thread_data_key); + return BHT_ERROR; + } + + if (aos_task_setspecific(thread_data_key, &supervisor_thread_data)) { + aos_sem_free(&supervisor_thread_data.wait_node.sem); + aos_task_key_delete(thread_data_key); + return BHT_ERROR; + } + + is_thread_sys_inited = true; + return BHT_OK; +} + +void +os_thread_sys_destroy() +{ + if (is_thread_sys_inited) { + aos_task_key_delete(thread_data_key); + aos_sem_free(&supervisor_thread_data.wait_node.sem); + is_thread_sys_inited = false; + } +} + +static os_thread_data * +thread_data_current() +{ + return aos_task_getspecific(thread_data_key); +} + +static void +os_thread_cleanup(void) +{ + os_thread_data *thread_data = thread_data_current(); + os_thread_wait_list thread_wait_list; + aos_mutex_t *wait_list_lock; + aos_sem_t *wait_node_sem; + + bh_assert(thread_data != NULL); + wait_list_lock = &thread_data->wait_list_lock; + thread_wait_list = thread_data->thread_wait_list; + wait_node_sem = &thread_data->wait_node.sem; + + /* Free thread data firstly */ + BH_FREE(thread_data); + + aos_mutex_lock(wait_list_lock, AOS_WAIT_FOREVER); + if (thread_wait_list) { + /* Signal each joining thread */ + os_thread_wait_list head = thread_wait_list; + while (head) { + os_thread_wait_list next = head->next; + aos_sem_signal(&head->sem); + head = next; + } + } + aos_mutex_unlock(wait_list_lock); + + /* Free sem and lock */ + aos_sem_free(wait_node_sem); + aos_mutex_free(wait_list_lock); +} + +static void +os_thread_wrapper(void *arg) +{ + os_thread_data *thread_data = arg; + + /* Set thread custom data */ + if (!aos_task_setspecific(thread_data_key, thread_data)) + thread_data->start_routine(thread_data->arg); + + os_thread_cleanup(); +} + +int +os_thread_create(korp_tid *p_tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(p_tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +int +os_thread_create_with_prio(korp_tid *p_tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + os_thread_data *thread_data; + char thread_name[32]; + + if (!p_tid || !stack_size) + return BHT_ERROR; + + /* Create and initialize thread data */ + if (!(thread_data = BH_MALLOC(sizeof(os_thread_data)))) + return BHT_ERROR; + + memset(thread_data, 0, sizeof(os_thread_data)); + + thread_data->start_routine = start; + thread_data->arg = arg; + + if (aos_sem_new(&thread_data->wait_node.sem, 1) != 0) + goto fail1; + + if (aos_mutex_new(&thread_data->wait_list_lock)) + goto fail2; + + snprintf(thread_name, sizeof(thread_name), "%s%d", "wasm-thread-", + ++thread_name_index); + + /* Create the thread */ + if (aos_task_new_ext((aos_task_t *)thread_data, thread_name, + os_thread_wrapper, thread_data, stack_size, prio)) + goto fail3; + + aos_msleep(10); + *p_tid = (korp_tid)thread_data; + return BHT_OK; + +fail3: + aos_mutex_free(&thread_data->wait_list_lock); +fail2: + aos_sem_free(&thread_data->wait_node.sem); +fail1: + BH_FREE(thread_data); + return BHT_ERROR; +} + +korp_tid +os_self_thread() +{ + return (korp_tid)aos_task_getspecific(thread_data_key); +} + +int +os_thread_join(korp_tid thread, void **value_ptr) +{ + (void)value_ptr; + os_thread_data *thread_data, *curr_thread_data; + + /* Get thread data of current thread */ + curr_thread_data = thread_data_current(); + curr_thread_data->wait_node.next = NULL; + + /* Get thread data */ + thread_data = (os_thread_data *)thread; + + aos_mutex_lock(&thread_data->wait_list_lock, AOS_WAIT_FOREVER); + if (!thread_data->thread_wait_list) + thread_data->thread_wait_list = &curr_thread_data->wait_node; + else { + /* Add to end of waiting list */ + os_thread_wait_node *p = thread_data->thread_wait_list; + while (p->next) + p = p->next; + p->next = &curr_thread_data->wait_node; + } + aos_mutex_unlock(&thread_data->wait_list_lock); + + /* Wait the sem */ + aos_sem_wait(&curr_thread_data->wait_node.sem, AOS_WAIT_FOREVER); + + return BHT_OK; +} + +int +os_mutex_init(korp_mutex *mutex) +{ + return aos_mutex_new(mutex) == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + aos_mutex_free(mutex); + return BHT_OK; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + return aos_mutex_lock(mutex, AOS_WAIT_FOREVER); +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + return aos_mutex_unlock(mutex); +} + +int +os_cond_init(korp_cond *cond) +{ + if (aos_mutex_new(&cond->wait_list_lock) != 0) + return BHT_ERROR; + + cond->thread_wait_list = NULL; + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ + aos_mutex_free(&cond->wait_list_lock); + return BHT_OK; +} + +static int +os_cond_wait_internal(korp_cond *cond, korp_mutex *mutex, bool timed, + uint32 mills) +{ + os_thread_wait_node *node = &thread_data_current()->wait_node; + + node->next = NULL; + + aos_mutex_lock(&cond->wait_list_lock, AOS_WAIT_FOREVER); + if (!cond->thread_wait_list) + cond->thread_wait_list = node; + else { + /* Add to end of wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next) + p = p->next; + p->next = node; + } + aos_mutex_unlock(&cond->wait_list_lock); + + /* Unlock mutex, wait sem and lock mutex again */ + aos_mutex_unlock(mutex); + aos_sem_wait(&node->sem, timed ? mills : AOS_WAIT_FOREVER); + aos_mutex_lock(mutex, AOS_WAIT_FOREVER); + + /* Remove wait node from wait list */ + aos_mutex_lock(&cond->wait_list_lock, AOS_WAIT_FOREVER); + if (cond->thread_wait_list == node) + cond->thread_wait_list = node->next; + else { + /* Remove from the wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next != node) + p = p->next; + p->next = node->next; + } + aos_mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + return os_cond_wait_internal(cond, mutex, false, 0); +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + if (useconds == BHT_WAIT_FOREVER) { + return os_cond_wait_internal(cond, mutex, false, 0); + } + else { + uint64 mills_64 = useconds / 1000; + uint32 mills; + + if (mills_64 < (uint64)(UINT32_MAX - 1)) { + mills = (uint64)mills_64; + } + else { + mills = UINT32_MAX - 1; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + return os_cond_wait_internal(cond, mutex, true, mills); + } +} + +int +os_cond_signal(korp_cond *cond) +{ + /* Signal the head wait node of wait list */ + aos_mutex_lock(&cond->wait_list_lock, AOS_WAIT_FOREVER); + if (cond->thread_wait_list) + aos_sem_signal(&cond->thread_wait_list->sem); + aos_mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +uint8 * +os_thread_get_stack_boundary() +{ + /* TODO: get alios stack boundary */ + return NULL; +} + +void +os_thread_jit_write_protect_np(bool enabled) +{} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/alios/alios_time.c b/wasm-micro-runtime/core/shared/platform/alios/alios_time.c new file mode 100644 index 0000000..fb09623 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/alios/alios_time.c @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +uint64 +os_time_get_boot_us() +{ + return (uint64)aos_now_ms() * 1000; +} + +uint64 +os_time_thread_cputime_us(void) +{ + /* FIXME if u know the right api */ + return os_time_get_boot_us(); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/alios/platform_internal.h b/wasm-micro-runtime/core/shared/platform/alios/platform_internal.h new file mode 100644 index 0000000..d2897a6 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/alios/platform_internal.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef BH_PLATFORM_ALIOS_THINGS +#define BH_PLATFORM_ALIOS_THINGS +#endif + +#define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 30 + +typedef aos_task_t korp_thread; +typedef korp_thread *korp_tid; +typedef aos_task_t *aos_tid_t; +typedef aos_mutex_t korp_mutex; +typedef aos_sem_t korp_sem; + +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + +struct os_thread_wait_node; +typedef struct os_thread_wait_node *os_thread_wait_list; +typedef struct korp_cond { + aos_mutex_t wait_list_lock; + os_thread_wait_list thread_wait_list; +} korp_cond; + +#define os_printf printf +#define os_vprintf vprintf + +/* clang-format off */ +/* math functions which are not provided by os*/ +double sqrt(double x); +double floor(double x); +double ceil(double x); +double fmin(double x, double y); +double fmax(double x, double y); +double rint(double x); +double fabs(double x); +double trunc(double x); +float sqrtf(float x); +float floorf(float x); +float ceilf(float x); +float fminf(float x, float y); +float fmaxf(float x, float y); +float rintf(float x); +float fabsf(float x); +float truncf(float x); +int signbit(double x); +int isnan(double x); +/* clang-format on */ + +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ +typedef int os_file_handle; +typedef void *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#endif /* end of _BH_PLATFORM_H */ diff --git a/wasm-micro-runtime/core/shared/platform/alios/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/alios/shared_platform.cmake new file mode 100644 index 0000000..a3aaddd --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/alios/shared_platform.cmake @@ -0,0 +1,16 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_ALIOS_THINGS) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/math/platform_api_math.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE}) + diff --git a/wasm-micro-runtime/core/shared/platform/android/platform_init.c b/wasm-micro-runtime/core/shared/platform/android/platform_init.c new file mode 100644 index 0000000..ad206af --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/android/platform_init.c @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#define API_NOT_SUPPORT_ERROR(API, VERSION) \ + __android_log_print(ANDROID_LOG_ERROR, "wasm_runtime::", \ + "%s() is only supported when __ANDROID_API__ >= %s.", \ + #API, #VERSION); + +int +bh_platform_init() +{ + return 0; +} + +void +bh_platform_destroy() +{} + +int +os_printf(const char *fmt, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, fmt); +#ifndef BH_VPRINTF + ret += __android_log_vprint(ANDROID_LOG_INFO, "wasm_runtime::", fmt, ap); +#else + ret += BH_VPRINTF(fmt, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *fmt, va_list ap) +{ +#ifndef BH_VPRINTF + return __android_log_vprint(ANDROID_LOG_INFO, "wasm_runtime::", fmt, ap); +#else + return BH_VPRINTF(fmt, ap); +#endif +} + +#if __ANDROID_API__ < 19 + +int +futimens(int __dir_fd, const struct timespec __times[2]) +{ + API_NOT_SUPPORT_ERROR(futimens, 19); + return -1; +} + +#endif + +#if __ANDROID_API__ < 21 + +int +posix_fallocate(int __fd, off_t __offset, off_t __length) +{ + API_NOT_SUPPORT_ERROR(posix_fallocate, 21); + return -1; +} + +int +posix_fadvise(int fd, off_t offset, off_t len, int advice) +{ + API_NOT_SUPPORT_ERROR(posix_fadvise, 21); + return -1; +} + +int +linkat(int __old_dir_fd, const char *__old_path, int __new_dir_fd, + const char *__new_path, int __flags) +{ + API_NOT_SUPPORT_ERROR(linkat, 21); + return -1; +} + +int +symlinkat(const char *__old_path, int __new_dir_fd, const char *__new_path) +{ + API_NOT_SUPPORT_ERROR(symlinkat, 21); + return -1; +} + +ssize_t +readlinkat(int __dir_fd, const char *__path, char *__buf, size_t __buf_size) +{ + API_NOT_SUPPORT_ERROR(readlinkat, 21); + return -1; +} + +int +accept4(int __fd, struct sockaddr *__addr, socklen_t *__addr_length, + int __flags) +{ + API_NOT_SUPPORT_ERROR(accept4, 21); + return -1; +} + +int +dup3(int oldfd, int newfd, int cloexec) +{ + API_NOT_SUPPORT_ERROR(dup3, 21); + return -1; +} + +int +pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock_id) +{ + API_NOT_SUPPORT_ERROR(pthread_condattr_setclock, 21); + return -1; +} + +int +epoll_create1(int flags) +{ + API_NOT_SUPPORT_ERROR(epoll_create1, 21); + return -1; +} + +int +epoll_pwait(int epfd, struct epoll_event *events, int maxevents, int timeout, + const sigset_t *sigmask) +{ + API_NOT_SUPPORT_ERROR(epoll_pwait, 21); + return -1; +} + +int +inotify_init1(int flags) +{ + API_NOT_SUPPORT_ERROR(inotify_init1, 21); + return -1; +} + +#endif + +#if __ANDROID_API__ < 23 + +long +telldir(DIR *__dir) +{ + API_NOT_SUPPORT_ERROR(telldir, 23); + return -1; +} + +void +seekdir(DIR *__dir, long __location) +{ + API_NOT_SUPPORT_ERROR(seekdir, 23); +} + +#endif + +#if __ANDROID_API__ < 24 + +ssize_t +preadv(int __fd, const struct iovec *__iov, int __count, off_t __offset) +{ + API_NOT_SUPPORT_ERROR(preadv, 24); + return -1; +} + +ssize_t +pwritev(int __fd, const struct iovec *__iov, int __count, off_t __offset) +{ + API_NOT_SUPPORT_ERROR(pwritev, 24); + return -1; +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/android/platform_internal.h b/wasm-micro-runtime/core/shared/platform/android/platform_internal.h new file mode 100644 index 0000000..4449f21 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/android/platform_internal.h @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_ANDROID +#define BH_PLATFORM_ANDROID +#endif + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (32 * 1024) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef sem_t korp_sem; + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define os_thread_local_attribute __thread + +#define bh_socket_t int + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + +#include + +#define OS_ENABLE_HW_BOUND_CHECK + +typedef jmp_buf korp_jmpbuf; + +#define os_setjmp setjmp +#define os_longjmp longjmp +#define os_alloca alloca + +typedef void (*os_signal_handler)(void *sig_addr); + +int +os_thread_signal_init(os_signal_handler handler); + +void +os_thread_signal_destroy(); + +bool +os_thread_signal_inited(); + +void +os_signal_unmask(); + +void +os_sigreturn(); +#endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ +#endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ + +#define os_getpagesize getpagesize + +typedef long int __syscall_slong_t; + +#if __ANDROID_API__ < 19 + +int +futimens(int __dir_fd, const struct timespec __times[2]); + +#endif + +#if __ANDROID_API__ < 21 + +int +posix_fallocate(int __fd, off_t __offset, off_t __length); + +int +posix_fadvise(int fd, off_t offset, off_t len, int advice); + +int +linkat(int __old_dir_fd, const char *__old_path, int __new_dir_fd, + const char *__new_path, int __flags); + +int +symlinkat(const char *__old_path, int __new_dir_fd, const char *__new_path); + +ssize_t +readlinkat(int __dir_fd, const char *__path, char *__buf, size_t __buf_size); + +#endif + +#if __ANDROID_API__ < 23 + +long +telldir(DIR *__dir); + +void +seekdir(DIR *__dir, long __location); + +#endif + +ssize_t +preadv(int __fd, const struct iovec *__iov, int __count, off_t __offset); + +ssize_t +pwritev(int __fd, const struct iovec *__iov, int __count, off_t __offset); + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/android/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/android/shared_platform.cmake new file mode 100644 index 0000000..13beb8e --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/android/shared_platform.cmake @@ -0,0 +1,18 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_ANDROID) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE}) + +file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) +LIST (APPEND RUNTIME_LIB_HEADER_LIST ${header}) diff --git a/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_malloc.c b/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_malloc.c new file mode 100644 index 0000000..e47a8cc --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_malloc.c @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +void * +os_malloc(unsigned size) +{ + return NULL; +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return NULL; +} + +void +os_free(void *ptr) +{} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} diff --git a/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_thread.c b/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_thread.c new file mode 100644 index 0000000..8d57fda --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_thread.c @@ -0,0 +1,499 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +/* clang-format off */ +#define bh_assert(v) do { \ + if (!(v)) { \ + int _count = 1; \ + os_printf("\nASSERTION FAILED: %s, at %s, line %d\n",\ + #v, __FILE__, __LINE__); \ + /* divived by 0 to make it abort */ \ + os_printf("%d\n", _count / (_count - 1)); \ + while (1); \ + } \ +} while (0) +/* clang-format on */ + +struct os_thread_data; +typedef struct os_thread_wait_node { + /* Binary semaphore */ + SemaphoreHandle_t sem; + os_thread_wait_list next; +} os_thread_wait_node; + +typedef struct os_thread_data { + /* Next thread data */ + struct os_thread_data *next; + /* Thread handle */ + TaskHandle_t handle; + /* Thread start routine */ + thread_start_routine_t start_routine; + /* Thread start routine argument */ + void *arg; + /* Thread local root */ + void *tlr; + /* Wait node of current thread */ + os_thread_wait_node wait_node; + /* Lock for waiting list */ + SemaphoreHandle_t wait_list_lock; + /* Waiting list of other threads who are joining this thread */ + os_thread_wait_list thread_wait_list; +} os_thread_data; + +static bool is_thread_sys_inited = false; + +/* Lock for thread data list */ +static SemaphoreHandle_t thread_data_lock; + +/* Thread data list */ +static os_thread_data *thread_data_list = NULL; +/* Thread data of supervisor thread */ +static os_thread_data supervisor_thread_data; + +/* Thread name index */ +static int thread_name_index; + +static void +thread_data_list_add(os_thread_data *thread_data) +{ + xSemaphoreTake(thread_data_lock, portMAX_DELAY); + if (!thread_data_list) + thread_data_list = thread_data; + else { + /* If already in list, just return */ + os_thread_data *p = thread_data_list; + while (p) { + if (p == thread_data) { + xSemaphoreGive(thread_data_lock); + return; + } + p = p->next; + } + + /* Set as head of list */ + thread_data->next = thread_data_list; + thread_data_list = thread_data; + } + xSemaphoreGive(thread_data_lock); +} + +static void +thread_data_list_remove(os_thread_data *thread_data) +{ + xSemaphoreTake(thread_data_lock, portMAX_DELAY); + if (thread_data_list) { + if (thread_data_list == thread_data) + thread_data_list = thread_data_list->next; + else { + /* Search and remove it from list */ + os_thread_data *p = thread_data_list; + while (p && p->next != thread_data) + p = p->next; + if (p && p->next == thread_data) + p->next = p->next->next; + } + } + xSemaphoreGive(thread_data_lock); +} + +static os_thread_data * +thread_data_list_lookup(TaskHandle_t handle) +{ + xSemaphoreTake(thread_data_lock, portMAX_DELAY); + if (thread_data_list) { + os_thread_data *p = thread_data_list; + while (p) { + if (p->handle == handle) { + /* Found */ + xSemaphoreGive(thread_data_lock); + return p; + } + p = p->next; + } + } + xSemaphoreGive(thread_data_lock); + return NULL; +} + +int +os_thread_sys_init() +{ + if (is_thread_sys_inited) + return BHT_OK; + + if (!(thread_data_lock = xSemaphoreCreateMutex())) + return BHT_ERROR; + + /* Initialize supervisor thread data */ + memset(&supervisor_thread_data, 0, sizeof(supervisor_thread_data)); + + if (!(supervisor_thread_data.wait_node.sem = xSemaphoreCreateBinary())) { + vSemaphoreDelete(thread_data_lock); + return BHT_ERROR; + } + + supervisor_thread_data.handle = xTaskGetCurrentTaskHandle(); + /* Set as head of thread data list */ + thread_data_list = &supervisor_thread_data; + + is_thread_sys_inited = true; + return BHT_OK; +} + +void +os_thread_sys_destroy() +{ + if (is_thread_sys_inited) { + vSemaphoreDelete(supervisor_thread_data.wait_node.sem); + vSemaphoreDelete(thread_data_lock); + is_thread_sys_inited = false; + } +} + +static os_thread_data * +thread_data_current() +{ + TaskHandle_t handle = xTaskGetCurrentTaskHandle(); + return thread_data_list_lookup(handle); +} + +static void +os_thread_cleanup(void) +{ + os_thread_data *thread_data = thread_data_current(); + os_thread_wait_list thread_wait_list; + SemaphoreHandle_t wait_list_lock; + SemaphoreHandle_t wait_node_sem; + + bh_assert(thread_data != NULL); + wait_list_lock = thread_data->wait_list_lock; + thread_wait_list = thread_data->thread_wait_list; + wait_node_sem = thread_data->wait_node.sem; + + xSemaphoreTake(wait_list_lock, portMAX_DELAY); + if (thread_wait_list) { + /* Signal each joining thread */ + os_thread_wait_list head = thread_wait_list; + while (head) { + os_thread_wait_list next = head->next; + xSemaphoreGive(head->sem); + head = next; + } + } + xSemaphoreGive(wait_list_lock); + + /* Free sem and lock */ + vSemaphoreDelete(wait_node_sem); + vSemaphoreDelete(wait_list_lock); + + thread_data_list_remove(thread_data); + BH_FREE(thread_data); +} + +static void +os_thread_wrapper(void *arg) +{ + os_thread_data *thread_data = arg; + + thread_data->handle = xTaskGetCurrentTaskHandle(); + thread_data_list_add(thread_data); + + thread_data->start_routine(thread_data->arg); + os_thread_exit(NULL); +} + +int +os_thread_create(korp_tid *p_tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(p_tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +int +os_thread_create_with_prio(korp_tid *p_tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + os_thread_data *thread_data; + char thread_name[32]; + + if (!p_tid || !stack_size) + return BHT_ERROR; + + /* Create and initialize thread data */ + if (!(thread_data = BH_MALLOC(sizeof(os_thread_data)))) + return BHT_ERROR; + + memset(thread_data, 0, sizeof(os_thread_data)); + + thread_data->start_routine = start; + thread_data->arg = arg; + + if (!(thread_data->wait_node.sem = xSemaphoreCreateBinary())) + goto fail1; + + if (!(thread_data->wait_list_lock = xSemaphoreCreateMutex())) + goto fail2; + + snprintf(thread_name, sizeof(thread_name), "%s%d", "wasm-thread-", + ++thread_name_index); + + /* Create the thread */ + if (pdPASS + != xTaskCreate(os_thread_wrapper, thread_name, stack_size / 4, + thread_data, prio, &thread_data->handle)) + goto fail3; + + thread_data_list_add(thread_data); + *p_tid = thread_data->handle; + return BHT_OK; + +fail3: + vSemaphoreDelete(thread_data->wait_list_lock); +fail2: + vSemaphoreDelete(thread_data->wait_node.sem); +fail1: + BH_FREE(thread_data); + return BHT_ERROR; +} + +korp_tid +os_self_thread() +{ + return xTaskGetCurrentTaskHandle(); +} + +int +os_thread_join(korp_tid thread, void **value_ptr) +{ + os_thread_data *thread_data, *curr_thread_data; + TaskHandle_t handle = thread; + + (void)value_ptr; + + /* Get thread data of current thread */ + curr_thread_data = thread_data_current(); + curr_thread_data->wait_node.next = NULL; + + /* Get thread data */ + thread_data = thread_data_list_lookup(handle); + + xSemaphoreTake(thread_data->wait_list_lock, portMAX_DELAY); + if (!thread_data->thread_wait_list) + thread_data->thread_wait_list = &curr_thread_data->wait_node; + else { + /* Add to end of waiting list */ + os_thread_wait_node *p = thread_data->thread_wait_list; + while (p->next) + p = p->next; + p->next = &curr_thread_data->wait_node; + } + xSemaphoreGive(thread_data->wait_list_lock); + + /* Wait the sem */ + xSemaphoreTake(curr_thread_data->wait_node.sem, portMAX_DELAY); + return BHT_OK; +} + +int +os_thread_detach(korp_tid thread) +{ + /* Do nothing */ + (void)thread; + return BHT_OK; +} + +void +os_thread_exit(void *retval) +{ + (void)retval; + os_thread_cleanup(); + vTaskDelete(NULL); +} + +int +os_mutex_init(korp_mutex *mutex) +{ + SemaphoreHandle_t semaphore; + + if (!(semaphore = xSemaphoreCreateMutex())) + return BHT_ERROR; + mutex->sem = semaphore; + mutex->is_recursive = false; + return BHT_OK; +} + +int +os_recursive_mutex_init(korp_mutex *mutex) +{ + SemaphoreHandle_t semaphore; + + if (!(semaphore = xSemaphoreCreateRecursiveMutex())) + return BHT_ERROR; + mutex->sem = semaphore; + mutex->is_recursive = true; + return BHT_OK; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + vSemaphoreDelete(mutex->sem); + return BHT_OK; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + int ret = -1; + + if (!mutex->is_recursive) + ret = xSemaphoreTake(mutex->sem, portMAX_DELAY); + else + ret = xSemaphoreTakeRecursive(mutex->sem, portMAX_DELAY); + return ret == pdPASS ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + int ret = -1; + + if (!mutex->is_recursive) + ret = xSemaphoreGive(mutex->sem); + else + ret = xSemaphoreGiveRecursive(mutex->sem); + return ret == pdPASS ? BHT_OK : BHT_ERROR; +} + +int +os_cond_init(korp_cond *cond) +{ + if (!(cond->wait_list_lock = xSemaphoreCreateMutex())) + return BHT_ERROR; + + cond->thread_wait_list = NULL; + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ + vSemaphoreDelete(cond->wait_list_lock); + return BHT_OK; +} + +static int +os_cond_wait_internal(korp_cond *cond, korp_mutex *mutex, bool timed, int mills) +{ + os_thread_wait_node *node = &thread_data_current()->wait_node; + + node->next = NULL; + + xSemaphoreTake(cond->wait_list_lock, portMAX_DELAY); + if (!cond->thread_wait_list) + cond->thread_wait_list = node; + else { + /* Add to end of wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next) + p = p->next; + p->next = node; + } + xSemaphoreGive(cond->wait_list_lock); + + /* Unlock mutex, wait sem and lock mutex again */ + os_mutex_unlock(mutex); + xSemaphoreTake(node->sem, timed ? mills / portTICK_RATE_MS : portMAX_DELAY); + os_mutex_lock(mutex); + + /* Remove wait node from wait list */ + xSemaphoreTake(cond->wait_list_lock, portMAX_DELAY); + if (cond->thread_wait_list == node) + cond->thread_wait_list = node->next; + else { + /* Remove from the wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next != node) + p = p->next; + p->next = node->next; + } + xSemaphoreGive(cond->wait_list_lock); + + return BHT_OK; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + return os_cond_wait_internal(cond, mutex, false, 0); +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + if (useconds == BHT_WAIT_FOREVER) { + return os_cond_wait_internal(cond, mutex, false, 0); + } + else { + uint64 mills_64 = useconds / 1000; + int32 mills; + + if (mills_64 < (uint64)INT32_MAX) { + mills = (int32)mills_64; + } + else { + mills = INT32_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + return os_cond_wait_internal(cond, mutex, true, mills); + } +} + +int +os_cond_signal(korp_cond *cond) +{ + /* Signal the head wait node of wait list */ + xSemaphoreTake(cond->wait_list_lock, portMAX_DELAY); + if (cond->thread_wait_list) + xSemaphoreGive(cond->thread_wait_list->sem); + xSemaphoreGive(cond->wait_list_lock); + + return BHT_OK; +} + +int +os_cond_broadcast(korp_cond *cond) +{ + /* Signal all of the wait node of wait list */ + xSemaphoreTake(cond->wait_list_lock, portMAX_DELAY); + if (cond->thread_wait_list) { + os_thread_wait_node *p = cond->thread_wait_list; + while (p) { + xSemaphoreGive(p->sem); + p = p->next; + } + } + xSemaphoreGive(cond->wait_list_lock); + + return BHT_OK; +} + +uint8 * +os_thread_get_stack_boundary() +{ + /* TODO: get freertos stack boundary */ + return NULL; +} + +void +os_thread_jit_write_protect_np(bool enabled) +{ + (void)enabled; +} diff --git a/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_time.c b/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_time.c new file mode 100644 index 0000000..e8249fe --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/freertos/freertos_time.c @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +uint64 +os_time_get_boot_us() +{ + TickType_t ticks = xTaskGetTickCount(); + return (uint64)1000 * 1000 / configTICK_RATE_HZ * ticks; +} + +uint64 +os_time_thread_cputime_us(void) +{ + /* FIXME if u know the right api */ + return os_time_get_boot_us(); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/common/freertos/platform_api_freertos.cmake b/wasm-micro-runtime/core/shared/platform/common/freertos/platform_api_freertos.cmake new file mode 100644 index 0000000..ebfc19d --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/freertos/platform_api_freertos.cmake @@ -0,0 +1,8 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_COMMON_FREERTOS_DIR ${CMAKE_CURRENT_LIST_DIR}) + +file (GLOB_RECURSE source_all ${PLATFORM_COMMON_FREERTOS_DIR}/*.c) + +set (PLATFORM_COMMON_FREERTOS_SOURCE ${source_all} ) diff --git a/wasm-micro-runtime/core/shared/platform/common/libc-util/libc_errno.c b/wasm-micro-runtime/core/shared/platform/common/libc-util/libc_errno.c new file mode 100644 index 0000000..e6c26c8 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/libc-util/libc_errno.c @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "errno.h" +#include "libc_errno.h" + +__wasi_errno_t +convert_errno(int error) +{ + // The C standard library only requires EDOM, EILSEQ and ERANGE to be + // defined. Other error codes are POSIX-specific and hence may or may + // not be available on non-POSIX platforms. + __wasi_errno_t code = __WASI_ENOSYS; +#define X(v) \ + case v: \ + code = __WASI_##v; \ + break; + switch (error) { + X(EDOM) + X(EILSEQ) + X(ERANGE) +#ifdef E2BIG + X(E2BIG) +#endif +#ifdef EACCES + X(EACCES) +#endif +#ifdef EADDRINUSE + X(EADDRINUSE) +#endif +#ifdef EADDRNOTAVAIL + X(EADDRNOTAVAIL) +#endif +#ifdef EAFNOSUPPORT + X(EAFNOSUPPORT) +#endif +#ifdef EAGAIN + X(EAGAIN) +#endif +#ifdef EALREADY + X(EALREADY) +#endif +#ifdef EBADF + X(EBADF) +#endif +#ifdef EBADMSG + X(EBADMSG) +#endif +#ifdef EBUSY + X(EBUSY) +#endif +#ifdef ECANCELED + X(ECANCELED) +#endif +#ifdef ECHILD + X(ECHILD) +#endif +#ifdef ECONNABORTED + X(ECONNABORTED) +#endif +#ifdef ECONNREFUSED + X(ECONNREFUSED) +#endif +#ifdef ECONNRESET + X(ECONNRESET) +#endif +#ifdef EDEADLK + X(EDEADLK) +#endif +#ifdef EDESTADDRREQ + X(EDESTADDRREQ) +#endif +#ifdef EDQUOT + X(EDQUOT) +#endif +#ifdef EEXIST + X(EEXIST) +#endif +#ifdef EFAULT + X(EFAULT) +#endif +#ifdef EFBIG + X(EFBIG) +#endif +#ifdef EHOSTUNREACH + X(EHOSTUNREACH) +#endif +#ifdef EIDRM + X(EIDRM) +#endif +#ifdef EINPROGRESS + X(EINPROGRESS) +#endif +#ifdef EINTR + X(EINTR) +#endif +#ifdef EINVAL + X(EINVAL) +#endif +#ifdef EIO + X(EIO) +#endif +#ifdef EISCONN + X(EISCONN) +#endif +#ifdef EISDIR + X(EISDIR) +#endif +#ifdef ELOOP + X(ELOOP) +#endif +#ifdef EMFILE + X(EMFILE) +#endif +#ifdef EMLINK + X(EMLINK) +#endif +#ifdef EMSGSIZE + X(EMSGSIZE) +#endif +#ifdef EMULTIHOP + X(EMULTIHOP) +#endif +#ifdef ENAMETOOLONG + X(ENAMETOOLONG) +#endif +#ifdef ENETDOWN + X(ENETDOWN) +#endif +#ifdef ENETRESET + X(ENETRESET) +#endif +#ifdef ENETUNREACH + X(ENETUNREACH) +#endif +#ifdef ENFILE + X(ENFILE) +#endif +#ifdef ENOBUFS + X(ENOBUFS) +#endif +#ifdef ENODEV + X(ENODEV) +#endif +#ifdef ENOENT + X(ENOENT) +#endif +#ifdef ENOEXEC + X(ENOEXEC) +#endif +#ifdef ENOLCK + X(ENOLCK) +#endif +#ifdef ENOLINK + X(ENOLINK) +#endif +#ifdef ENOMEM + X(ENOMEM) +#endif +#ifdef ENOMSG + X(ENOMSG) +#endif +#ifdef ENOPROTOOPT + X(ENOPROTOOPT) +#endif +#ifdef ENOSPC + X(ENOSPC) +#endif +#ifdef ENOSYS + X(ENOSYS) +#endif +#ifdef ENOTCAPABLE + X(ENOTCAPABLE) +#endif +#ifdef ENOTCONN + X(ENOTCONN) +#endif +#ifdef ENOTDIR + X(ENOTDIR) +#endif +#ifdef ENOTEMPTY + X(ENOTEMPTY) +#endif +#ifdef ENOTRECOVERABLE + X(ENOTRECOVERABLE) +#endif +#ifdef ENOTSOCK + X(ENOTSOCK) +#endif +#ifdef ENOTSUP + X(ENOTSUP) +#endif +#ifdef ENOTTY + X(ENOTTY) +#endif +#ifdef ENXIO + X(ENXIO) +#endif +#ifdef EOVERFLOW + X(EOVERFLOW) +#endif +#ifdef EOWNERDEAD + X(EOWNERDEAD) +#endif +#ifdef EPERM + X(EPERM) +#endif +#ifdef EPIPE + X(EPIPE) +#endif +#ifdef EPROTO + X(EPROTO) +#endif +#ifdef EPROTONOSUPPORT + X(EPROTONOSUPPORT) +#endif +#ifdef EPROTOTYPE + X(EPROTOTYPE) +#endif +#ifdef EROFS + X(EROFS) +#endif +#ifdef ESPIPE + X(ESPIPE) +#endif +#ifdef ESRCH + X(ESRCH) +#endif +#ifdef ESTALE + X(ESTALE) +#endif +#ifdef ETIMEDOUT + X(ETIMEDOUT) +#endif +#ifdef ETXTBSY + X(ETXTBSY) +#endif +#ifdef EXDEV + X(EXDEV) +#endif + default: +#ifdef EOPNOTSUPP + if (error == EOPNOTSUPP) + code = __WASI_ENOTSUP; +#endif +#ifdef EWOULDBLOCK + if (error == EWOULDBLOCK) + code = __WASI_EAGAIN; +#endif + break; + } +#undef X + return code; +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/common/libc-util/libc_errno.h b/wasm-micro-runtime/core/shared/platform/common/libc-util/libc_errno.h new file mode 100644 index 0000000..b0a8b78 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/libc-util/libc_errno.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_ERRNO_H +#define WASI_ERRNO_H + +#include "platform_wasi_types.h" + +// Converts an errno error code to a WASI error code. +__wasi_errno_t +convert_errno(int error); + +#endif /* end of WASI_ERRNO_H */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/common/libc-util/platform_common_libc_util.cmake b/wasm-micro-runtime/core/shared/platform/common/libc-util/platform_common_libc_util.cmake new file mode 100644 index 0000000..a7c7645 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/libc-util/platform_common_libc_util.cmake @@ -0,0 +1,8 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_COMMON_LIBC_UTIL_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include_directories(${PLATFORM_COMMON_LIBC_UTIL_DIR}) + +file (GLOB_RECURSE PLATFORM_COMMON_LIBC_UTIL_SOURCE ${PLATFORM_COMMON_LIBC_UTIL_DIR}/*.c) \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/common/math/COPYRIGHT b/wasm-micro-runtime/core/shared/platform/common/math/COPYRIGHT new file mode 100644 index 0000000..a0e1c83 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/math/COPYRIGHT @@ -0,0 +1,126 @@ +# $FreeBSD$ +# @(#)COPYRIGHT 8.2 (Berkeley) 3/21/94 + +The compilation of software known as FreeBSD is distributed under the +following terms: + +Copyright (c) 1992-2019 The FreeBSD Project. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The 4.4BSD and 4.4BSD-Lite software is distributed under the following +terms: + +All of the documentation and software included in the 4.4BSD and 4.4BSD-Lite +Releases is copyrighted by The Regents of the University of California. + +Copyright 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. All advertising materials mentioning features or use of this software + must display the following acknowledgement: +This product includes software developed by the University of +California, Berkeley and its contributors. +4. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The Institute of Electrical and Electronics Engineers and the American +National Standards Committee X3, on Information Processing Systems have +given us permission to reprint portions of their documentation. + +In the following statement, the phrase ``this text'' refers to portions +of the system documentation. + +Portions of this text are reprinted and reproduced in electronic form in +the second BSD Networking Software Release, from IEEE Std 1003.1-1988, IEEE +Standard Portable Operating System Interface for Computer Environments +(POSIX), copyright C 1988 by the Institute of Electrical and Electronics +Engineers, Inc. In the event of any discrepancy between these versions +and the original IEEE Standard, the original IEEE Standard is the referee +document. + +In the following statement, the phrase ``This material'' refers to portions +of the system documentation. + +This material is reproduced with permission from American National +Standards Committee X3, on Information Processing Systems. Computer and +Business Equipment Manufacturers Association (CBEMA), 311 First St., NW, +Suite 500, Washington, DC 20001-2178. The developmental work of +Programming Language C was completed by the X3J11 Technical Committee. + +The views and conclusions contained in the software and documentation are +those of the authors and should not be interpreted as representing official +policies, either expressed or implied, of the Regents of the University +of California. + + +NOTE: The copyright of UC Berkeley's Berkeley Software Distribution ("BSD") +source has been updated. The copyright addendum may be found at +ftp://ftp.cs.berkeley.edu/pub/4bsd/README.Impt.License.Change and is +included below. + +July 22, 1999 + +To All Licensees, Distributors of Any Version of BSD: + +As you know, certain of the Berkeley Software Distribution ("BSD") source +code files require that further distributions of products containing all or +portions of the software, acknowledge within their advertising materials +that such products contain software developed by UC Berkeley and its +contributors. + +Specifically, the provision reads: + +" * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors." + +Effective immediately, licensees and distributors are no longer required to +include the acknowledgement within advertising materials. Accordingly, the +foregoing paragraph of those BSD Unix files containing it is hereby deleted +in its entirety. + +William Hoskins +Director, Office of Technology Licensing +University of California, Berkeley diff --git a/wasm-micro-runtime/core/shared/platform/common/math/math.c b/wasm-micro-runtime/core/shared/platform/common/math/math.c new file mode 100644 index 0000000..2ba9f4d --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/math/math.c @@ -0,0 +1,1681 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2004 David Schultz + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "platform_common.h" + +#define __FDLIBM_STDC__ + +#ifndef FLT_EVAL_METHOD +#define FLT_EVAL_METHOD 0 +#endif + +typedef uint32_t u_int32_t; +typedef uint64_t u_int64_t; + +typedef union u32double_tag { + int *pint; + double *pdouble; +} U32DOUBLE; + +static inline int * +pdouble2pint(double *pdouble) +{ + U32DOUBLE u; + u.pdouble = pdouble; + return u.pint; +} + +typedef union { + double value; + struct { + u_int32_t lsw; + u_int32_t msw; + } parts; + struct { + u_int64_t w; + } xparts; +} ieee_double_shape_type_little; + +typedef union { + double value; + struct { + u_int32_t msw; + u_int32_t lsw; + } parts; + struct { + u_int64_t w; + } xparts; +} ieee_double_shape_type_big; + +typedef union { + double d; + struct { + unsigned int manl : 32; + unsigned int manh : 20; + unsigned int exp : 11; + unsigned int sign : 1; + } bits; +} IEEEd2bits_L; + +typedef union { + double d; + struct { + unsigned int sign : 1; + unsigned int exp : 11; + unsigned int manh : 20; + unsigned int manl : 32; + } bits; +} IEEEd2bits_B; + +typedef union { + float f; + struct { + unsigned int man : 23; + unsigned int exp : 8; + unsigned int sign : 1; + } bits; +} IEEEf2bits_L; + +typedef union { + float f; + struct { + unsigned int sign : 1; + unsigned int exp : 8; + unsigned int man : 23; + } bits; +} IEEEf2bits_B; + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +#define __HIL(x) *(1 + pdouble2pint(&x)) +#define __LOL(x) *(pdouble2pint(&x)) +#define __HIB(x) *(pdouble2pint(&x)) +#define __LOB(x) *(1 + pdouble2pint(&x)) + +/* Get two 32 bit ints from a double. */ + +#define EXTRACT_WORDS_L(ix0, ix1, d) \ + do { \ + ieee_double_shape_type_little ew_u; \ + ew_u.value = (d); \ + (ix0) = ew_u.parts.msw; \ + (ix1) = ew_u.parts.lsw; \ + } while (0) + +/* Set a double from two 32 bit ints. */ + +#define INSERT_WORDS_L(d, ix0, ix1) \ + do { \ + ieee_double_shape_type_little iw_u; \ + iw_u.parts.msw = (ix0); \ + iw_u.parts.lsw = (ix1); \ + (d) = iw_u.value; \ + } while (0) + +/* Get two 32 bit ints from a double. */ + +#define EXTRACT_WORDS_B(ix0, ix1, d) \ + do { \ + ieee_double_shape_type_big ew_u; \ + ew_u.value = (d); \ + (ix0) = ew_u.parts.msw; \ + (ix1) = ew_u.parts.lsw; \ + } while (0) + +/* Set a double from two 32 bit ints. */ + +#define INSERT_WORDS_B(d, ix0, ix1) \ + do { \ + ieee_double_shape_type_big iw_u; \ + iw_u.parts.msw = (ix0); \ + iw_u.parts.lsw = (ix1); \ + (d) = iw_u.value; \ + } while (0) + +/* Get the more significant 32 bit int from a double. */ +#define GET_HIGH_WORD_L(i, d) \ + do { \ + ieee_double_shape_type_little gh_u; \ + gh_u.value = (d); \ + (i) = gh_u.parts.msw; \ + } while (0) + +/* Get the more significant 32 bit int from a double. */ +#define GET_HIGH_WORD_B(i, d) \ + do { \ + ieee_double_shape_type_big gh_u; \ + gh_u.value = (d); \ + (i) = gh_u.parts.msw; \ + } while (0) + +/* Set the more significant 32 bits of a double from an int. */ +#define SET_HIGH_WORD_L(d, v) \ + do { \ + ieee_double_shape_type_little sh_u; \ + sh_u.value = (d); \ + sh_u.parts.msw = (v); \ + (d) = sh_u.value; \ + } while (0) + +/* Set the more significant 32 bits of a double from an int. */ +#define SET_HIGH_WORD_B(d, v) \ + do { \ + ieee_double_shape_type_big sh_u; \ + sh_u.value = (d); \ + sh_u.parts.msw = (v); \ + (d) = sh_u.value; \ + } while (0) + +/* Set the less significant 32 bits of a double from an int. */ +#define SET_LOW_WORD_L(d, v) \ + do { \ + ieee_double_shape_type_little sh_u; \ + sh_u.value = (d); \ + sh_u.parts.lsw = (v); \ + (d) = sh_u.value; \ + } while (0) + +/* Set the more significant 32 bits of a double from an int. */ +#define SET_LOW_WORD_B(d, v) \ + do { \ + ieee_double_shape_type_big sh_u; \ + sh_u.value = (d); \ + sh_u.parts.lsw = (v); \ + (d) = sh_u.value; \ + } while (0) + +/* Get the less significant 32 bit int from a double. */ +#define GET_LOW_WORD_L(i, d) \ + do { \ + ieee_double_shape_type_little gl_u; \ + gl_u.value = (d); \ + (i) = gl_u.parts.lsw; \ + } while (0) + +/* Get the less significant 32 bit int from a double. */ +#define GET_LOW_WORD_B(i, d) \ + do { \ + ieee_double_shape_type_big gl_u; \ + gl_u.value = (d); \ + (i) = gl_u.parts.lsw; \ + } while (0) + +/* + * A union which permits us to convert between a float and a 32 bit + * int. + */ +typedef union { + float value; + /* FIXME: Assumes 32 bit int. */ + unsigned int word; +} ieee_float_shape_type; + +/* Get a 32 bit int from a float. */ +#define GET_FLOAT_WORD(i, d) \ + do { \ + ieee_float_shape_type gf_u; \ + gf_u.value = (d); \ + (i) = gf_u.word; \ + } while (0) + +/* Set a float from a 32 bit int. */ +#define SET_FLOAT_WORD(d, i) \ + do { \ + ieee_float_shape_type sf_u; \ + sf_u.word = (i); \ + (d) = sf_u.value; \ + } while (0) + +/* Macro wrappers. */ +#define EXTRACT_WORDS(ix0, ix1, d) \ + do { \ + if (is_little_endian()) \ + EXTRACT_WORDS_L(ix0, ix1, d); \ + else \ + EXTRACT_WORDS_B(ix0, ix1, d); \ + } while (0) + +#define INSERT_WORDS(d, ix0, ix1) \ + do { \ + if (is_little_endian()) \ + INSERT_WORDS_L(d, ix0, ix1); \ + else \ + INSERT_WORDS_B(d, ix0, ix1); \ + } while (0) + +#define GET_HIGH_WORD(i, d) \ + do { \ + if (is_little_endian()) \ + GET_HIGH_WORD_L(i, d); \ + else \ + GET_HIGH_WORD_B(i, d); \ + } while (0) + +#define SET_HIGH_WORD(d, v) \ + do { \ + if (is_little_endian()) \ + SET_HIGH_WORD_L(d, v); \ + else \ + SET_HIGH_WORD_B(d, v); \ + } while (0) + +#define GET_LOW_WORD(d, v) \ + do { \ + if (is_little_endian()) \ + GET_LOW_WORD_L(d, v); \ + else \ + GET_LOW_WORD_B(d, v); \ + } while (0) + +#define SET_LOW_WORD(d, v) \ + do { \ + if (is_little_endian()) \ + SET_LOW_WORD_L(d, v); \ + else \ + SET_LOW_WORD_B(d, v); \ + } while (0) + +#define __HI(x) (is_little_endian() ? __HIL(x) : __HIB(x)) + +#define __LO(x) (is_little_endian() ? __LOL(x) : __LOB(x)) + +/* + * Attempt to get strict C99 semantics for assignment with non-C99 compilers. + */ +#if FLT_EVAL_METHOD == 0 || __GNUC__ == 0 +#define STRICT_ASSIGN(type, lval, rval) ((lval) = (rval)) +#else +#define STRICT_ASSIGN(type, lval, rval) \ + do { \ + volatile type __lval; \ + \ + if (sizeof(type) >= sizeof(long double)) \ + (lval) = (rval); \ + else { \ + __lval = (rval); \ + (lval) = __lval; \ + } \ + } while (0) +#endif + +#ifdef __FDLIBM_STDC__ +static const double huge = 1.0e300; +#else +static double huge = 1.0e300; +#endif + +#ifdef __STDC__ +static const double +#else +static double +#endif + tiny = 1.0e-300; + +#ifdef __STDC__ +static const double +#else +static double +#endif + one = 1.00000000000000000000e+00; /* 0x3FF00000, 0x00000000 */ + +#ifdef __STDC__ +static const double +#else +static double +#endif + TWO52[2] = { + 4.50359962737049600000e+15, /* 0x43300000, 0x00000000 */ + -4.50359962737049600000e+15, /* 0xC3300000, 0x00000000 */ + }; + +#ifdef __STDC__ +static const double +#else +static double +#endif + atanhi[] = { + 4.63647609000806093515e-01, /* atan(0.5)hi 0x3FDDAC67, 0x0561BB4F */ + 7.85398163397448278999e-01, /* atan(1.0)hi 0x3FE921FB, 0x54442D18 */ + 9.82793723247329054082e-01, /* atan(1.5)hi 0x3FEF730B, 0xD281F69B */ + 1.57079632679489655800e+00, /* atan(inf)hi 0x3FF921FB, 0x54442D18 */ + }; + +#ifdef __STDC__ +static const double +#else +static double +#endif + atanlo[] = { + 2.26987774529616870924e-17, /* atan(0.5)lo 0x3C7A2B7F, 0x222F65E2 */ + 3.06161699786838301793e-17, /* atan(1.0)lo 0x3C81A626, 0x33145C07 */ + 1.39033110312309984516e-17, /* atan(1.5)lo 0x3C700788, 0x7AF0CBBD */ + 6.12323399573676603587e-17, /* atan(inf)lo 0x3C91A626, 0x33145C07 */ + }; + +#ifdef __STDC__ +static const double +#else +static double +#endif + aT[] = { + 3.33333333333329318027e-01, /* 0x3FD55555, 0x5555550D */ + -1.99999999998764832476e-01, /* 0xBFC99999, 0x9998EBC4 */ + 1.42857142725034663711e-01, /* 0x3FC24924, 0x920083FF */ + -1.11111104054623557880e-01, /* 0xBFBC71C6, 0xFE231671 */ + 9.09088713343650656196e-02, /* 0x3FB745CD, 0xC54C206E */ + -7.69187620504482999495e-02, /* 0xBFB3B0F2, 0xAF749A6D */ + 6.66107313738753120669e-02, /* 0x3FB10D66, 0xA0D03D51 */ + -5.83357013379057348645e-02, /* 0xBFADDE2D, 0x52DEFD9A */ + 4.97687799461593236017e-02, /* 0x3FA97B4B, 0x24760DEB */ + -3.65315727442169155270e-02, /* 0xBFA2B444, 0x2C6A6C2F */ + 1.62858201153657823623e-02, /* 0x3F90AD3A, 0xE322DA11 */ + }; + +#ifdef __STDC__ +static const double +#else +static double +#endif + zero = 0.0, + pi_o_4 = 7.8539816339744827900E-01, /* 0x3FE921FB, 0x54442D18 */ + pi_o_2 = 1.5707963267948965580E+00, /* 0x3FF921FB, 0x54442D18 */ + pi = 3.1415926535897931160E+00, /* 0x400921FB, 0x54442D18 */ + pi_lo = 1.2246467991473531772E-16; /* 0x3CA1A626, 0x33145C07 */ + +#ifdef __STDC__ +static const double +#else +static double +#endif +bp[] = {1.0, 1.5,}, +dp_h[] = { 0.0, 5.84962487220764160156e-01,}, /* 0x3FE2B803, 0x40000000 */ +dp_l[] = { 0.0, 1.35003920212974897128e-08,}, /* 0x3E4CFDEB, 0x43CFD006 */ +two = 2.0, +two53 = 9007199254740992.0, /* 0x43400000, 0x00000000 */ +two54 = 1.80143985094819840000e+16, /* 0x43500000, 0x00000000 */ +twom54 = 5.55111512312578270212e-17, /* 0x3C900000, 0x00000000 */ + /* poly coefs for (3/2)*(log(x)-2s-2/3*s**3 */ +L1 = 5.99999999999994648725e-01, /* 0x3FE33333, 0x33333303 */ +L2 = 4.28571428578550184252e-01, /* 0x3FDB6DB6, 0xDB6FABFF */ +L3 = 3.33333329818377432918e-01, /* 0x3FD55555, 0x518F264D */ +L4 = 2.72728123808534006489e-01, /* 0x3FD17460, 0xA91D4101 */ +L5 = 2.30660745775561754067e-01, /* 0x3FCD864A, 0x93C9DB65 */ +L6 = 2.06975017800338417784e-01, /* 0x3FCA7E28, 0x4A454EEF */ +P1 = 1.66666666666666019037e-01, /* 0x3FC55555, 0x5555553E */ +P2 = -2.77777777770155933842e-03, /* 0xBF66C16C, 0x16BEBD93 */ +P3 = 6.61375632143793436117e-05, /* 0x3F11566A, 0xAF25DE2C */ +P4 = -1.65339022054652515390e-06, /* 0xBEBBBD41, 0xC5D26BF1 */ +P5 = 4.13813679705723846039e-08, /* 0x3E663769, 0x72BEA4D0 */ +lg2 = 6.93147180559945286227e-01, /* 0x3FE62E42, 0xFEFA39EF */ +lg2_h = 6.93147182464599609375e-01, /* 0x3FE62E43, 0x00000000 */ +lg2_l = -1.90465429995776804525e-09, /* 0xBE205C61, 0x0CA86C39 */ +ovt = 8.0085662595372944372e-0017, /* -(1024-log2(ovfl+.5ulp)) */ +cp = 9.61796693925975554329e-01, /* 0x3FEEC709, 0xDC3A03FD =2/(3ln2) */ +cp_h = 9.61796700954437255859e-01, /* 0x3FEEC709, 0xE0000000 =(float)cp */ +cp_l = -7.02846165095275826516e-09, /* 0xBE3E2FE0, 0x145B01F5 =tail of cp_h*/ +ivln2 = 1.44269504088896338700e+00, /* 0x3FF71547, 0x652B82FE =1/ln2 */ +ivln2_h = 1.44269502162933349609e+00, /* 0x3FF71547, 0x60000000 =24b 1/ln2*/ +ivln2_l = 1.92596299112661746887e-08; /* 0x3E54AE0B, 0xF85DDF44 =1/ln2 tail*/ + +static double +freebsd_floor(double x); +static double +freebsd_ceil(double x); +static double +freebsd_fabs(double x); +static double +freebsd_rint(double x); +static int +freebsd_isnan(double x); +static double +freebsd_atan(double x); +static double +freebsd_atan2(double y, double x); + +static double +freebsd_atan(double x) +{ + double w, s1, s2, z; + int32_t ix, hx, id; + + GET_HIGH_WORD(hx, x); + ix = hx & 0x7fffffff; + if (ix >= 0x44100000) { /* if |x| >= 2^66 */ + u_int32_t low; + GET_LOW_WORD(low, x); + if (ix > 0x7ff00000 || (ix == 0x7ff00000 && (low != 0))) + return x + x; /* NaN */ + if (hx > 0) + return atanhi[3] + *(volatile double *)&atanlo[3]; + else + return -atanhi[3] - *(volatile double *)&atanlo[3]; + } + if (ix < 0x3fdc0000) { /* |x| < 0.4375 */ + if (ix < 0x3e400000) { /* |x| < 2^-27 */ + if (huge + x > one) + return x; /* raise inexact */ + } + id = -1; + } + else { + x = freebsd_fabs(x); + if (ix < 0x3ff30000) { /* |x| < 1.1875 */ + if (ix < 0x3fe60000) { /* 7/16 <=|x|<11/16 */ + id = 0; + x = (2.0 * x - one) / (2.0 + x); + } + else { /* 11/16<=|x|< 19/16 */ + id = 1; + x = (x - one) / (x + one); + } + } + else { + if (ix < 0x40038000) { /* |x| < 2.4375 */ + id = 2; + x = (x - 1.5) / (one + 1.5 * x); + } + else { /* 2.4375 <= |x| < 2^66 */ + id = 3; + x = -1.0 / x; + } + } + } + /* end of argument reduction */ + z = x * x; + w = z * z; + /* break sum from i=0 to 10 aT[i]z**(i+1) into odd and even poly */ + s1 = z + * (aT[0] + + w + * (aT[2] + + w * (aT[4] + w * (aT[6] + w * (aT[8] + w * aT[10]))))); + s2 = w * (aT[1] + w * (aT[3] + w * (aT[5] + w * (aT[7] + w * aT[9])))); + if (id < 0) + return x - x * (s1 + s2); + else { + z = atanhi[id] - ((x * (s1 + s2) - atanlo[id]) - x); + return (hx < 0) ? -z : z; + } +} + +static double +freebsd_atan2(double y, double x) +{ + double z; + int32_t k, m, hx, hy, ix, iy; + u_int32_t lx, ly; + + EXTRACT_WORDS(hx, lx, x); + ix = hx & 0x7fffffff; + EXTRACT_WORDS(hy, ly, y); + iy = hy & 0x7fffffff; + if (((ix | ((lx | -lx) >> 31)) > 0x7ff00000) + || ((iy | ((ly | -ly) >> 31)) > 0x7ff00000)) /* x or y is NaN */ + return x + y; + if (hx == 0x3ff00000 && lx == 0) + return freebsd_atan(y); /* x=1.0 */ + m = ((hy >> 31) & 1) | ((hx >> 30) & 2); /* 2*sign(x)+sign(y) */ + + /* when y = 0 */ + if ((iy | ly) == 0) { + switch (m) { + case 0: + case 1: + return y; /* atan(+-0,+anything)=+-0 */ + case 2: + return pi + tiny; /* atan(+0,-anything) = pi */ + case 3: + default: + return -pi - tiny; /* atan(-0,-anything) =-pi */ + } + } + /* when x = 0 */ + if ((ix | lx) == 0) + return (hy < 0) ? -pi_o_2 - tiny : pi_o_2 + tiny; + + /* when x is INF */ + if (ix == 0x7ff00000) { + if (iy == 0x7ff00000) { + switch (m) { + case 0: + return pi_o_4 + tiny; /* atan(+INF,+INF) */ + case 1: + return -pi_o_4 - tiny; /* atan(-INF,+INF) */ + case 2: + return 3.0 * pi_o_4 + tiny; /*atan(+INF,-INF)*/ + case 3: + default: + return -3.0 * pi_o_4 - tiny; /*atan(-INF,-INF)*/ + } + } + else { + switch (m) { + case 0: + return zero; /* atan(+...,+INF) */ + case 1: + return -zero; /* atan(-...,+INF) */ + case 2: + return pi + tiny; /* atan(+...,-INF) */ + case 3: + default: + return -pi - tiny; /* atan(-...,-INF) */ + } + } + } + /* when y is INF */ + if (iy == 0x7ff00000) + return (hy < 0) ? -pi_o_2 - tiny : pi_o_2 + tiny; + + /* compute y/x */ + k = (iy - ix) >> 20; + if (k > 60) { /* |y/x| > 2**60 */ + z = pi_o_2 + 0.5 * pi_lo; + m &= 1; + } + else if (hx < 0 && k < -60) + z = 0.0; /* 0 > |y|/x > -2**-60 */ + else + z = freebsd_atan(fabs(y / x)); /* safe to do y/x */ + switch (m) { + case 0: + return z; /* atan(+,+) */ + case 1: + return -z; /* atan(-,+) */ + case 2: + return pi - (z - pi_lo); /* atan(+,-) */ + default: /* case 3 */ + return (z - pi_lo) - pi; /* atan(-,-) */ + } +} + +#ifndef BH_HAS_SQRTF +static float +freebsd_sqrtf(float x) +{ + float z; + int32_t sign = (int)0x80000000; + int32_t ix, s, q, m, t, i; + u_int32_t r; + + GET_FLOAT_WORD(ix, x); + + /* take care of Inf and NaN */ + if ((ix & 0x7f800000) == 0x7f800000) { + return x * x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf + sqrt(-inf)=sNaN */ + } + /* take care of zero */ + if (ix <= 0) { + if ((ix & (~sign)) == 0) + return x; /* sqrt(+-0) = +-0 */ + else if (ix < 0) + return (x - x) / (x - x); /* sqrt(-ve) = sNaN */ + } + /* normalize x */ + m = (ix >> 23); + if (m == 0) { /* subnormal x */ + for (i = 0; (ix & 0x00800000) == 0; i++) + ix <<= 1; + m -= i - 1; + } + m -= 127; /* unbias exponent */ + ix = (ix & 0x007fffff) | 0x00800000; + if (m & 1) /* odd m, double x to make it even */ + ix += ix; + m >>= 1; /* m = [m/2] */ + + /* generate sqrt(x) bit by bit */ + ix += ix; + q = s = 0; /* q = sqrt(x) */ + r = 0x01000000; /* r = moving bit from right to left */ + + while (r != 0) { + t = s + r; + if (t <= ix) { + s = t + r; + ix -= t; + q += r; + } + ix += ix; + r >>= 1; + } + + /* use floating add to find out rounding direction */ + if (ix != 0) { + z = one - tiny; /* trigger inexact flag */ + if (z >= one) { + z = one + tiny; + if (z > one) + q += 2; + else + q += (q & 1); + } + } + ix = (q >> 1) + 0x3f000000; + ix += (m << 23); + SET_FLOAT_WORD(z, ix); + return z; +} +#endif /* end of BH_HAS_SQRTF */ + +#ifndef BH_HAS_SQRT +static double +freebsd_sqrt(double x) /* wrapper sqrt */ +{ + double z; + int32_t sign = (int)0x80000000; + int32_t ix0, s0, q, m, t, i; + u_int32_t r, t1, s1, ix1, q1; + + EXTRACT_WORDS(ix0, ix1, x); + + /* take care of Inf and NaN */ + if ((ix0 & 0x7ff00000) == 0x7ff00000) { + return x * x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf + sqrt(-inf)=sNaN */ + } + /* take care of zero */ + if (ix0 <= 0) { + if (((ix0 & (~sign)) | ix1) == 0) + return x; /* sqrt(+-0) = +-0 */ + else if (ix0 < 0) + return (x - x) / (x - x); /* sqrt(-ve) = sNaN */ + } + /* normalize x */ + m = (ix0 >> 20); + if (m == 0) { /* subnormal x */ + while (ix0 == 0) { + m -= 21; + ix0 |= (ix1 >> 11); + ix1 <<= 21; + } + for (i = 0; (ix0 & 0x00100000) == 0; i++) + ix0 <<= 1; + m -= i - 1; + ix0 |= (ix1 >> (32 - i)); + ix1 <<= i; + } + m -= 1023; /* unbias exponent */ + ix0 = (ix0 & 0x000fffff) | 0x00100000; + if (m & 1) { /* odd m, double x to make it even */ + ix0 += ix0 + ((ix1 & sign) >> 31); + ix1 += ix1; + } + m >>= 1; /* m = [m/2] */ + + /* generate sqrt(x) bit by bit */ + ix0 += ix0 + ((ix1 & sign) >> 31); + ix1 += ix1; + q = q1 = s0 = s1 = 0; /* [q,q1] = sqrt(x) */ + r = 0x00200000; /* r = moving bit from right to left */ + + while (r != 0) { + t = s0 + r; + if (t <= ix0) { + s0 = t + r; + ix0 -= t; + q += r; + } + ix0 += ix0 + ((ix1 & sign) >> 31); + ix1 += ix1; + r >>= 1; + } + + r = sign; + while (r != 0) { + t1 = s1 + r; + t = s0; + if ((t < ix0) || ((t == ix0) && (t1 <= ix1))) { + s1 = t1 + r; + if (((t1 & sign) == sign) && (s1 & sign) == 0) + s0 += 1; + ix0 -= t; + if (ix1 < t1) + ix0 -= 1; + ix1 -= t1; + q1 += r; + } + ix0 += ix0 + ((ix1 & sign) >> 31); + ix1 += ix1; + r >>= 1; + } + + /* use floating add to find out rounding direction */ + if ((ix0 | ix1) != 0) { + z = one - tiny; /* trigger inexact flag */ + if (z >= one) { + z = one + tiny; + if (q1 == (u_int32_t)0xffffffff) { + q1 = 0; + q += 1; + } + else if (z > one) { + if (q1 == (u_int32_t)0xfffffffe) + q += 1; + q1 += 2; + } + else + q1 += (q1 & 1); + } + } + ix0 = (q >> 1) + 0x3fe00000; + ix1 = q1 >> 1; + if ((q & 1) == 1) + ix1 |= sign; + ix0 += (m << 20); + + INSERT_WORDS(z, ix0, ix1); + + return z; +} +#endif /* end of BH_HAS_SQRT */ + +static double +freebsd_floor(double x) +{ + int32_t i0, i1, j0; + u_int32_t i, j; + + EXTRACT_WORDS(i0, i1, x); + + j0 = ((i0 >> 20) & 0x7ff) - 0x3ff; + if (j0 < 20) { + if (j0 < 0) { /* raise inexact if x != 0 */ + if (huge + x > 0.0) { /* return 0*sign(x) if |x|<1 */ + if (i0 >= 0) { + i0 = i1 = 0; + } + else if (((i0 & 0x7fffffff) | i1) != 0) { + i0 = 0xbff00000; + i1 = 0; + } + } + } + else { + i = (0x000fffff) >> j0; + if (((i0 & i) | i1) == 0) + return x; /* x is integral */ + if (huge + x > 0.0) { /* raise inexact flag */ + if (i0 < 0) + i0 += (0x00100000) >> j0; + i0 &= (~i); + i1 = 0; + } + } + } + else if (j0 > 51) { + if (j0 == 0x400) + return x + x; /* inf or NaN */ + else + return x; /* x is integral */ + } + else { + i = ((u_int32_t)(0xffffffff)) >> (j0 - 20); + if ((i1 & i) == 0) + return x; /* x is integral */ + if (huge + x > 0.0) { /* raise inexact flag */ + if (i0 < 0) { + if (j0 == 20) + i0 += 1; + else { + j = i1 + (1 << (52 - j0)); + if (j < i1) + i0 += 1; /* got a carry */ + i1 = j; + } + } + i1 &= (~i); + } + } + + INSERT_WORDS(x, i0, i1); + + return x; +} + +static double +freebsd_ceil(double x) +{ + int32_t i0, i1, j0; + u_int32_t i, j; + EXTRACT_WORDS(i0, i1, x); + j0 = ((i0 >> 20) & 0x7ff) - 0x3ff; + if (j0 < 20) { + if (j0 < 0) { /* raise inexact if x != 0 */ + if (huge + x > 0.0) { /* return 0*sign(x) if |x|<1 */ + if (i0 < 0) { + i0 = 0x80000000; + i1 = 0; + } + else if ((i0 | i1) != 0) { + i0 = 0x3ff00000; + i1 = 0; + } + } + } + else { + i = (0x000fffff) >> j0; + if (((i0 & i) | i1) == 0) + return x; /* x is integral */ + if (huge + x > 0.0) { /* raise inexact flag */ + if (i0 > 0) + i0 += (0x00100000) >> j0; + i0 &= (~i); + i1 = 0; + } + } + } + else if (j0 > 51) { + if (j0 == 0x400) + return x + x; /* inf or NaN */ + else + return x; /* x is integral */ + } + else { + i = ((u_int32_t)(0xffffffff)) >> (j0 - 20); + if ((i1 & i) == 0) + return x; /* x is integral */ + if (huge + x > 0.0) { /* raise inexact flag */ + if (i0 > 0) { + if (j0 == 20) + i0 += 1; + else { + j = i1 + (1 << (52 - j0)); + if (j < i1) + i0 += 1; /* got a carry */ + i1 = j; + } + } + i1 &= (~i); + } + } + INSERT_WORDS(x, i0, i1); + return x; +} + +static double +freebsd_rint(double x) +{ + int32_t i0, j0, sx; + u_int32_t i, i1; + double w, t; + EXTRACT_WORDS(i0, i1, x); + sx = (i0 >> 31) & 1; + j0 = ((i0 >> 20) & 0x7ff) - 0x3ff; + if (j0 < 20) { + if (j0 < 0) { + if (((i0 & 0x7fffffff) | i1) == 0) + return x; + i1 |= (i0 & 0x0fffff); + i0 &= 0xfffe0000; + i0 |= ((i1 | -i1) >> 12) & 0x80000; + SET_HIGH_WORD(x, i0); + STRICT_ASSIGN(double, w, TWO52[sx] + x); + t = w - TWO52[sx]; + GET_HIGH_WORD(i0, t); + SET_HIGH_WORD(t, (i0 & 0x7fffffff) | (sx << 31)); + return t; + } + else { + i = (0x000fffff) >> j0; + if (((i0 & i) | i1) == 0) + return x; /* x is integral */ + i >>= 1; + if (((i0 & i) | i1) != 0) { + /* + * Some bit is set after the 0.5 bit. To avoid the + * possibility of errors from double rounding in + * w = TWO52[sx]+x, adjust the 0.25 bit to a lower + * guard bit. We do this for all j0<=51. The + * adjustment is trickiest for j0==18 and j0==19 + * since then it spans the word boundary. + */ + if (j0 == 19) + i1 = 0x40000000; + else if (j0 == 18) + i1 = 0x80000000; + else + i0 = (i0 & (~i)) | ((0x20000) >> j0); + } + } + } + else if (j0 > 51) { + if (j0 == 0x400) + return x + x; /* inf or NaN */ + else + return x; /* x is integral */ + } + else { + i = ((u_int32_t)(0xffffffff)) >> (j0 - 20); + if ((i1 & i) == 0) + return x; /* x is integral */ + i >>= 1; + if ((i1 & i) != 0) + i1 = (i1 & (~i)) | ((0x40000000) >> (j0 - 20)); + } + INSERT_WORDS(x, i0, i1); + STRICT_ASSIGN(double, w, TWO52[sx] + x); + return w - TWO52[sx]; +} + +static int +freebsd_isnan(double d) +{ + if (is_little_endian()) { + IEEEd2bits_L u; + u.d = d; + return (u.bits.exp == 2047 && (u.bits.manl != 0 || u.bits.manh != 0)); + } + else { + IEEEd2bits_B u; + u.d = d; + return (u.bits.exp == 2047 && (u.bits.manl != 0 || u.bits.manh != 0)); + } +} + +static float +freebsd_fabsf(float x) +{ + u_int32_t ix; + GET_FLOAT_WORD(ix, x); + SET_FLOAT_WORD(x, ix & 0x7fffffff); + return x; +} + +static double +freebsd_fabs(double x) +{ + u_int32_t high; + GET_HIGH_WORD(high, x); + SET_HIGH_WORD(x, high & 0x7fffffff); + return x; +} + +static const float huge_f = 1.0e30F; + +static const float TWO23[2] = { + 8.3886080000e+06, /* 0x4b000000 */ + -8.3886080000e+06, /* 0xcb000000 */ +}; + +static float +freebsd_truncf(float x) +{ + int32_t i0, j0; + u_int32_t i; + GET_FLOAT_WORD(i0, x); + j0 = ((i0 >> 23) & 0xff) - 0x7f; + if (j0 < 23) { + if (j0 < 0) { /* raise inexact if x != 0 */ + if (huge_f + x > 0.0F) /* |x|<1, so return 0*sign(x) */ + i0 &= 0x80000000; + } + else { + i = (0x007fffff) >> j0; + if ((i0 & i) == 0) + return x; /* x is integral */ + if (huge_f + x > 0.0F) /* raise inexact flag */ + i0 &= (~i); + } + } + else { + if (j0 == 0x80) + return x + x; /* inf or NaN */ + else + return x; /* x is integral */ + } + SET_FLOAT_WORD(x, i0); + return x; +} + +static float +freebsd_rintf(float x) +{ + int32_t i0, j0, sx; + float w, t; + GET_FLOAT_WORD(i0, x); + sx = (i0 >> 31) & 1; + j0 = ((i0 >> 23) & 0xff) - 0x7f; + if (j0 < 23) { + if (j0 < 0) { + if ((i0 & 0x7fffffff) == 0) + return x; + STRICT_ASSIGN(float, w, TWO23[sx] + x); + t = w - TWO23[sx]; + GET_FLOAT_WORD(i0, t); + SET_FLOAT_WORD(t, (i0 & 0x7fffffff) | (sx << 31)); + return t; + } + STRICT_ASSIGN(float, w, TWO23[sx] + x); + return w - TWO23[sx]; + } + if (j0 == 0x80) + return x + x; /* inf or NaN */ + else + return x; /* x is integral */ +} + +static float +freebsd_ceilf(float x) +{ + int32_t i0, j0; + u_int32_t i; + + GET_FLOAT_WORD(i0, x); + j0 = ((i0 >> 23) & 0xff) - 0x7f; + if (j0 < 23) { + if (j0 < 0) { /* raise inexact if x != 0 */ + if (huge_f + x > (float)0.0) { /* return 0*sign(x) if |x|<1 */ + if (i0 < 0) { + i0 = 0x80000000; + } + else if (i0 != 0) { + i0 = 0x3f800000; + } + } + } + else { + i = (0x007fffff) >> j0; + if ((i0 & i) == 0) + return x; /* x is integral */ + if (huge_f + x > (float)0.0) { /* raise inexact flag */ + if (i0 > 0) + i0 += (0x00800000) >> j0; + i0 &= (~i); + } + } + } + else { + if (j0 == 0x80) + return x + x; /* inf or NaN */ + else + return x; /* x is integral */ + } + SET_FLOAT_WORD(x, i0); + return x; +} + +static float +freebsd_floorf(float x) +{ + int32_t i0, j0; + u_int32_t i; + GET_FLOAT_WORD(i0, x); + j0 = ((i0 >> 23) & 0xff) - 0x7f; + if (j0 < 23) { + if (j0 < 0) { /* raise inexact if x != 0 */ + if (huge_f + x > (float)0.0) { /* return 0*sign(x) if |x|<1 */ + if (i0 >= 0) { + i0 = 0; + } + else if ((i0 & 0x7fffffff) != 0) { + i0 = 0xbf800000; + } + } + } + else { + i = (0x007fffff) >> j0; + if ((i0 & i) == 0) + return x; /* x is integral */ + if (huge_f + x > (float)0.0) { /* raise inexact flag */ + if (i0 < 0) + i0 += (0x00800000) >> j0; + i0 &= (~i); + } + } + } + else { + if (j0 == 0x80) + return x + x; /* inf or NaN */ + else + return x; /* x is integral */ + } + SET_FLOAT_WORD(x, i0); + return x; +} + +static float +freebsd_fminf(float x, float y) +{ + if (is_little_endian()) { + IEEEf2bits_L u[2] = { 0 }; + + u[0].f = x; + u[1].f = y; + + /* Check for NaNs to avoid raising spurious exceptions. */ + if (u[0].bits.exp == 255 && u[0].bits.man != 0) + return (y); + if (u[1].bits.exp == 255 && u[1].bits.man != 0) + return (x); + + /* Handle comparisons of signed zeroes. */ + if (u[0].bits.sign != u[1].bits.sign) + return (u[u[1].bits.sign].f); + } + else { + IEEEf2bits_B u[2] = { 0 }; + + u[0].f = x; + u[1].f = y; + + /* Check for NaNs to avoid raising spurious exceptions. */ + if (u[0].bits.exp == 255 && u[0].bits.man != 0) + return (y); + if (u[1].bits.exp == 255 && u[1].bits.man != 0) + return (x); + + /* Handle comparisons of signed zeroes. */ + if (u[0].bits.sign != u[1].bits.sign) + return (u[u[1].bits.sign].f); + } + + return (x < y ? x : y); +} + +static float +freebsd_fmaxf(float x, float y) +{ + if (is_little_endian()) { + IEEEf2bits_L u[2] = { 0 }; + + u[0].f = x; + u[1].f = y; + + /* Check for NaNs to avoid raising spurious exceptions. */ + if (u[0].bits.exp == 255 && u[0].bits.man != 0) + return (y); + if (u[1].bits.exp == 255 && u[1].bits.man != 0) + return (x); + + /* Handle comparisons of signed zeroes. */ + if (u[0].bits.sign != u[1].bits.sign) + return (u[u[0].bits.sign].f); + } + else { + IEEEf2bits_B u[2] = { 0 }; + + u[0].f = x; + u[1].f = y; + + /* Check for NaNs to avoid raising spurious exceptions. */ + if (u[0].bits.exp == 255 && u[0].bits.man != 0) + return (y); + if (u[1].bits.exp == 255 && u[1].bits.man != 0) + return (x); + + /* Handle comparisons of signed zeroes. */ + if (u[0].bits.sign != u[1].bits.sign) + return (u[u[0].bits.sign].f); + } + + return (x > y ? x : y); +} + +static double +freebsd_copysign(double x, double y) +{ + u_int32_t hx, hy; + GET_HIGH_WORD(hx, x); + GET_HIGH_WORD(hy, y); + SET_HIGH_WORD(x, (hx & 0x7fffffff) | (hy & 0x80000000)); + return x; +} + +static double +freebsd_scalbn(double x, int n) +{ + int32_t k, hx, lx; + EXTRACT_WORDS(hx, lx, x); + k = (hx & 0x7ff00000) >> 20; /* extract exponent */ + if (k == 0) { /* 0 or subnormal x */ + if ((lx | (hx & 0x7fffffff)) == 0) + return x; /* +-0 */ + x *= two54; + GET_HIGH_WORD(hx, x); + k = ((hx & 0x7ff00000) >> 20) - 54; + if (n < -50000) + return tiny * x; /*underflow*/ + } + if (k == 0x7ff) + return x + x; /* NaN or Inf */ + k = k + n; + if (k > 0x7fe) + return huge * freebsd_copysign(huge, x); /* overflow */ + if (k > 0) /* normal result */ + { + SET_HIGH_WORD(x, (hx & 0x800fffff) | (k << 20)); + return x; + } + if (k <= -54) { + if (n > 50000) /* in case integer overflow in n+k */ + return huge * freebsd_copysign(huge, x); /*overflow*/ + else + return tiny * freebsd_copysign(tiny, x); /*underflow*/ + } + k += 54; /* subnormal result */ + SET_HIGH_WORD(x, (hx & 0x800fffff) | (k << 20)); + return x * twom54; +} + +static double +freebsd_pow(double x, double y) +{ + double z, ax, z_h, z_l, p_h, p_l; + double y1, t1, t2, r, s, t, u, v, w; + int32_t i, j, k, yisint, n; + int32_t hx, hy, ix, iy; + u_int32_t lx, ly; + + EXTRACT_WORDS(hx, lx, x); + EXTRACT_WORDS(hy, ly, y); + ix = hx & 0x7fffffff; + iy = hy & 0x7fffffff; + + /* y==zero: x**0 = 1 */ + if ((iy | ly) == 0) + return one; + + /* x==1: 1**y = 1, even if y is NaN */ + if (hx == 0x3ff00000 && lx == 0) + return one; + + /* y!=zero: result is NaN if either arg is NaN */ + if (ix > 0x7ff00000 || ((ix == 0x7ff00000) && (lx != 0)) || iy > 0x7ff00000 + || ((iy == 0x7ff00000) && (ly != 0))) + return (x + 0.0) + (y + 0.0); + + /* determine if y is an odd int when x < 0 + * yisint = 0 ... y is not an integer + * yisint = 1 ... y is an odd int + * yisint = 2 ... y is an even int + */ + yisint = 0; + if (hx < 0) { + if (iy >= 0x43400000) + yisint = 2; /* even integer y */ + else if (iy >= 0x3ff00000) { + k = (iy >> 20) - 0x3ff; /* exponent */ + if (k > 20) { + j = ly >> (52 - k); + if ((j << (52 - k)) == ly) + yisint = 2 - (j & 1); + } + else if (ly == 0) { + j = iy >> (20 - k); + if ((j << (20 - k)) == iy) + yisint = 2 - (j & 1); + } + } + } + + /* special value of y */ + if (ly == 0) { + if (iy == 0x7ff00000) { /* y is +-inf */ + if (((ix - 0x3ff00000) | lx) == 0) + return one; /* (-1)**+-inf is NaN */ + else if (ix >= 0x3ff00000) /* (|x|>1)**+-inf = inf,0 */ + return (hy >= 0) ? y : zero; + else /* (|x|<1)**-,+inf = inf,0 */ + return (hy < 0) ? -y : zero; + } + if (iy == 0x3ff00000) { /* y is +-1 */ + if (hy < 0) + return one / x; + else + return x; + } + if (hy == 0x40000000) + return x * x; /* y is 2 */ + if (hy == 0x40080000) + return x * x * x; /* y is 3 */ + if (hy == 0x40100000) { /* y is 4 */ + u = x * x; + return u * u; + } + if (hy == 0x3fe00000) { /* y is 0.5 */ + if (hx >= 0) /* x >= +0 */ + return sqrt(x); + } + } + + ax = fabs(x); + /* special value of x */ + if (lx == 0) { + if (ix == 0x7ff00000 || ix == 0 || ix == 0x3ff00000) { + z = ax; /*x is +-0,+-inf,+-1*/ + if (hy < 0) + z = one / z; /* z = (1/|x|) */ + if (hx < 0) { + if (((ix - 0x3ff00000) | yisint) == 0) { + z = (z - z) / (z - z); /* (-1)**non-int is NaN */ + } + else if (yisint == 1) + z = -z; /* (x<0)**odd = -(|x|**odd) */ + } + return z; + } + } + + /* CYGNUS LOCAL + fdlibm-5.3 fix: This used to be + n = (hx>>31)+1; + but ANSI C says a right shift of a signed negative quantity is + implementation defined. */ + n = ((u_int32_t)hx >> 31) - 1; + + /* (x<0)**(non-int) is NaN */ + if ((n | yisint) == 0) + return (x - x) / (x - x); + + s = one; /* s (sign of result -ve**odd) = -1 else = 1 */ + if ((n | (yisint - 1)) == 0) + s = -one; /* (-ve)**(odd int) */ + + /* |y| is huge */ + if (iy > 0x41e00000) { /* if |y| > 2**31 */ + if (iy > 0x43f00000) { /* if |y| > 2**64, must o/uflow */ + if (ix <= 0x3fefffff) + return (hy < 0) ? huge * huge : tiny * tiny; + if (ix >= 0x3ff00000) + return (hy > 0) ? huge * huge : tiny * tiny; + } + /* over/underflow if x is not close to one */ + if (ix < 0x3fefffff) + return (hy < 0) ? s * huge * huge : s * tiny * tiny; + if (ix > 0x3ff00000) + return (hy > 0) ? s * huge * huge : s * tiny * tiny; + /* now |1-x| is tiny <= 2**-20, suffice to compute + log(x) by x-x^2/2+x^3/3-x^4/4 */ + t = ax - one; /* t has 20 trailing zeros */ + w = (t * t) * (0.5 - t * (0.3333333333333333333333 - t * 0.25)); + u = ivln2_h * t; /* ivln2_h has 21 sig. bits */ + v = t * ivln2_l - w * ivln2; + t1 = u + v; + SET_LOW_WORD(t1, 0); + t2 = v - (t1 - u); + } + else { + double ss, s2, s_h, s_l, t_h, t_l; + n = 0; + /* take care subnormal number */ + if (ix < 0x00100000) { + ax *= two53; + n -= 53; + GET_HIGH_WORD(ix, ax); + } + n += ((ix) >> 20) - 0x3ff; + j = ix & 0x000fffff; + /* determine interval */ + ix = j | 0x3ff00000; /* normalize ix */ + if (j <= 0x3988E) + k = 0; /* |x|> 1) | 0x20000000) + 0x00080000 + (k << 18)); + t_l = ax - (t_h - bp[k]); + s_l = v * ((u - s_h * t_h) - s_h * t_l); + /* compute log(ax) */ + s2 = ss * ss; + r = s2 * s2 + * (L1 + s2 * (L2 + s2 * (L3 + s2 * (L4 + s2 * (L5 + s2 * L6))))); + r += s_l * (s_h + ss); + s2 = s_h * s_h; + t_h = 3.0 + s2 + r; + SET_LOW_WORD(t_h, 0); + t_l = r - ((t_h - 3.0) - s2); + /* u+v = ss*(1+...) */ + u = s_h * t_h; + v = s_l * t_h + t_l * ss; + /* 2/(3log2)*(ss+...) */ + p_h = u + v; + SET_LOW_WORD(p_h, 0); + p_l = v - (p_h - u); + z_h = cp_h * p_h; /* cp_h+cp_l = 2/(3*log2) */ + z_l = cp_l * p_h + p_l * cp + dp_l[k]; + /* log2(ax) = (ss+..)*2/(3*log2) = n + dp_h + z_h + z_l */ + t = (double)n; + t1 = (((z_h + z_l) + dp_h[k]) + t); + SET_LOW_WORD(t1, 0); + t2 = z_l - (((t1 - t) - dp_h[k]) - z_h); + } + + /* split up y into y1+y2 and compute (y1+y2)*(t1+t2) */ + y1 = y; + SET_LOW_WORD(y1, 0); + p_l = (y - y1) * t1 + y * t2; + p_h = y1 * t1; + z = p_l + p_h; + EXTRACT_WORDS(j, i, z); + if (j >= 0x40900000) { /* z >= 1024 */ + if (((j - 0x40900000) | i) != 0) /* if z > 1024 */ + return s * huge * huge; /* overflow */ + else { + if (p_l + ovt > z - p_h) + return s * huge * huge; /* overflow */ + } + } + else if ((j & 0x7fffffff) >= 0x4090cc00) { /* z <= -1075 */ + if (((j - 0xc090cc00) | i) != 0) /* z < -1075 */ + return s * tiny * tiny; /* underflow */ + else { + if (p_l <= z - p_h) + return s * tiny * tiny; /* underflow */ + } + } + /* + * compute 2**(p_h+p_l) + */ + i = j & 0x7fffffff; + k = (i >> 20) - 0x3ff; + n = 0; + if (i > 0x3fe00000) { /* if |z| > 0.5, set n = [z+0.5] */ + n = j + (0x00100000 >> (k + 1)); + k = ((n & 0x7fffffff) >> 20) - 0x3ff; /* new k for n */ + t = zero; + SET_HIGH_WORD(t, n & ~(0x000fffff >> k)); + n = ((n & 0x000fffff) | 0x00100000) >> (20 - k); + if (j < 0) + n = -n; + p_h -= t; + } + t = p_l + p_h; + SET_LOW_WORD(t, 0); + u = t * lg2_h; + v = (p_l - (t - p_h)) * lg2 + t * lg2_l; + z = u + v; + w = v - (z - u); + t = z * z; + t1 = z - t * (P1 + t * (P2 + t * (P3 + t * (P4 + t * P5)))); + r = (z * t1) / (t1 - two) - (w + z * w); + z = one - (r - z); + GET_HIGH_WORD(j, z); + j += (n << 20); + if ((j >> 20) <= 0) + z = freebsd_scalbn(z, n); /* subnormal output */ + else + SET_HIGH_WORD(z, j); + return s * z; +} + +double +atan(double x) +{ + return freebsd_atan(x); +} + +double +atan2(double y, double x) +{ + return freebsd_atan2(y, x); +} + +#ifndef BH_HAS_SQRT +double +sqrt(double x) +{ + return freebsd_sqrt(x); +} +#endif + +double +floor(double x) +{ + return freebsd_floor(x); +} + +double +ceil(double x) +{ + return freebsd_ceil(x); +} + +double +fmin(double x, double y) +{ + return x < y ? x : y; +} + +double +fmax(double x, double y) +{ + return x > y ? x : y; +} + +double +rint(double x) +{ + return freebsd_rint(x); +} + +double +fabs(double x) +{ + return freebsd_fabs(x); +} + +int +isnan(double x) +{ + return freebsd_isnan(x); +} + +double +trunc(double x) +{ + return (x > 0) ? freebsd_floor(x) : freebsd_ceil(x); +} + +int +signbit(double x) +{ + return ((__HI(x) & 0x80000000) >> 31); +} + +float +fabsf(float x) +{ + return freebsd_fabsf(x); +} + +float +truncf(float x) +{ + return freebsd_truncf(x); +} + +float +rintf(float x) +{ + return freebsd_rintf(x); +} + +float +ceilf(float x) +{ + return freebsd_ceilf(x); +} + +float +floorf(float x) +{ + return freebsd_floorf(x); +} + +float +fminf(float x, float y) +{ + return freebsd_fminf(x, y); +} + +float +fmaxf(float x, float y) +{ + return freebsd_fmaxf(x, y); +} + +#ifndef BH_HAS_SQRTF +float +sqrtf(float x) +{ + return freebsd_sqrtf(x); +} +#endif + +double +pow(double x, double y) +{ + return freebsd_pow(x, y); +} + +double +scalbn(double x, int n) +{ + return freebsd_scalbn(x, n); +} diff --git a/wasm-micro-runtime/core/shared/platform/common/math/platform_api_math.cmake b/wasm-micro-runtime/core/shared/platform/common/math/platform_api_math.cmake new file mode 100644 index 0000000..09c74bf --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/math/platform_api_math.cmake @@ -0,0 +1,8 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_COMMON_MATH_DIR ${CMAKE_CURRENT_LIST_DIR}) + +file (GLOB_RECURSE source_all ${PLATFORM_COMMON_MATH_DIR}/*.c) + +set (PLATFORM_COMMON_MATH_SOURCE ${source_all} ) diff --git a/wasm-micro-runtime/core/shared/platform/common/memory/mremap.c b/wasm-micro-runtime/core/shared/platform/common/memory/mremap.c new file mode 100644 index 0000000..dca10a3 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/memory/mremap.c @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size) +{ + return os_mremap_slow(old_addr, old_size, new_size); +} diff --git a/wasm-micro-runtime/core/shared/platform/common/memory/platform_api_memory.cmake b/wasm-micro-runtime/core/shared/platform/common/memory/platform_api_memory.cmake new file mode 100644 index 0000000..9f06c13 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/memory/platform_api_memory.cmake @@ -0,0 +1,4 @@ +# Copyright (C) 2024 Amazon Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +file (GLOB_RECURSE PLATFORM_COMMON_MEMORY_SOURCE ${CMAKE_CURRENT_LIST_DIR}/*.c) diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/platform_api_posix.cmake b/wasm-micro-runtime/core/shared/platform/common/posix/platform_api_posix.cmake new file mode 100644 index 0000000..15d6daf --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/platform_api_posix.cmake @@ -0,0 +1,35 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_COMMON_POSIX_DIR ${CMAKE_CURRENT_LIST_DIR}) + +file (GLOB_RECURSE source_all ${PLATFORM_COMMON_POSIX_DIR}/*.c) + +if (NOT WAMR_BUILD_LIBC_WASI EQUAL 1) + list(REMOVE_ITEM source_all + ${PLATFORM_COMMON_POSIX_DIR}/posix_file.c + ${PLATFORM_COMMON_POSIX_DIR}/posix_clock.c + ${PLATFORM_COMMON_POSIX_DIR}/posix_socket.c + ) +else() + include (${CMAKE_CURRENT_LIST_DIR}/../libc-util/platform_common_libc_util.cmake) + set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) +endif() + +# This is to support old CMake version. Newer version of CMake could use +# list APPEND/POP_BACK methods. +include(CheckSymbolExists) +set (CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE ${CMAKE_REQUIRED_DEFINITIONS}) +check_symbol_exists (mremap "sys/mman.h" MREMAP_EXISTS) +list (REMOVE_AT CMAKE_REQUIRED_DEFINITIONS 0) + +if(MREMAP_EXISTS) + add_definitions (-DWASM_HAVE_MREMAP=1) + add_definitions (-D_GNU_SOURCE) +else() + add_definitions (-DWASM_HAVE_MREMAP=0) + include (${CMAKE_CURRENT_LIST_DIR}/../memory/platform_api_memory.cmake) + set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) +endif() + +set (PLATFORM_COMMON_POSIX_SOURCE ${source_all} ) diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_blocking_op.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_blocking_op.c new file mode 100644 index 0000000..e56f84c --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_blocking_op.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_extension.h" + +#ifdef OS_ENABLE_WAKEUP_BLOCKING_OP + +static bool g_blocking_op_inited = false; +static int g_blocking_op_signo = SIGUSR1; +static sigset_t g_blocking_op_sigmask; + +static void +blocking_op_sighandler(int signo) +{ + /* nothing */ + (void)signo; +} + +void +os_set_signal_number_for_blocking_op(int signo) +{ + g_blocking_op_signo = signo; +} + +int +os_blocking_op_init() +{ + if (g_blocking_op_inited) { + return BHT_OK; + } + + sigemptyset(&g_blocking_op_sigmask); + sigaddset(&g_blocking_op_sigmask, g_blocking_op_signo); + + struct sigaction sa; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sa.sa_handler = blocking_op_sighandler; + if (sigaction(g_blocking_op_signo, &sa, NULL)) { + return BHT_ERROR; + } + g_blocking_op_inited = true; + return BHT_OK; +} + +void +os_begin_blocking_op() +{ + pthread_sigmask(SIG_UNBLOCK, &g_blocking_op_sigmask, NULL); +} + +void +os_end_blocking_op() +{ + pthread_sigmask(SIG_BLOCK, &g_blocking_op_sigmask, NULL); +} + +int +os_wakeup_blocking_op(korp_tid tid) +{ + int ret = pthread_kill(tid, g_blocking_op_signo); + if (ret != 0) { + return BHT_ERROR; + } + return BHT_OK; +} + +#endif /* OS_ENABLE_WAKEUP_BLOCKING_OP */ diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_clock.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_clock.c new file mode 100644 index 0000000..4141321 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_clock.c @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "libc_errno.h" +#include "platform_api_extension.h" + +#define NANOSECONDS_PER_SECOND 1000000000ULL + +static __wasi_errno_t +wasi_clockid_to_clockid(__wasi_clockid_t in, clockid_t *out) +{ + switch (in) { + case __WASI_CLOCK_MONOTONIC: + *out = CLOCK_MONOTONIC; + return __WASI_ESUCCESS; + case __WASI_CLOCK_REALTIME: + *out = CLOCK_REALTIME; + return __WASI_ESUCCESS; + case __WASI_CLOCK_PROCESS_CPUTIME_ID: +#if defined(CLOCK_PROCESS_CPUTIME_ID) + *out = CLOCK_PROCESS_CPUTIME_ID; + return __WASI_ESUCCESS; +#else + return __WASI_ENOTSUP; +#endif + case __WASI_CLOCK_THREAD_CPUTIME_ID: +#if defined(CLOCK_THREAD_CPUTIME_ID) + *out = CLOCK_THREAD_CPUTIME_ID; + return __WASI_ESUCCESS; +#else + return __WASI_ENOTSUP; +#endif + default: + return __WASI_EINVAL; + } +} + +static __wasi_timestamp_t +timespec_to_nanoseconds(const struct timespec *ts) +{ + if (ts->tv_sec < 0) + return 0; + if ((__wasi_timestamp_t)ts->tv_sec >= UINT64_MAX / NANOSECONDS_PER_SECOND) + return UINT64_MAX; + return (__wasi_timestamp_t)ts->tv_sec * NANOSECONDS_PER_SECOND + + (__wasi_timestamp_t)ts->tv_nsec; +} + +__wasi_errno_t +os_clock_res_get(__wasi_clockid_t clock_id, __wasi_timestamp_t *resolution) +{ + clockid_t nclock_id; + __wasi_errno_t error = wasi_clockid_to_clockid(clock_id, &nclock_id); + + if (error != __WASI_ESUCCESS) + return error; + + struct timespec ts; + if (clock_getres(nclock_id, &ts) < 0) + return convert_errno(errno); + + *resolution = timespec_to_nanoseconds(&ts); + + return error; +} + +__wasi_errno_t +os_clock_time_get(__wasi_clockid_t clock_id, __wasi_timestamp_t precision, + __wasi_timestamp_t *time) +{ + clockid_t nclock_id; + __wasi_errno_t error = wasi_clockid_to_clockid(clock_id, &nclock_id); + + (void)precision; + + if (error != __WASI_ESUCCESS) + return error; + + struct timespec ts; + if (clock_gettime(nclock_id, &ts) < 0) + return convert_errno(errno); + + *time = timespec_to_nanoseconds(&ts); + + return error; +} diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_file.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_file.c new file mode 100644 index 0000000..ac7e585 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_file.c @@ -0,0 +1,1014 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_extension.h" +#include "libc_errno.h" +#include + +#if !defined(__APPLE__) && !defined(ESP_PLATFORM) +#define CONFIG_HAS_PWRITEV 1 +#define CONFIG_HAS_PREADV 1 +#else +#define CONFIG_HAS_PWRITEV 0 +#define CONFIG_HAS_PREADV 0 +#endif + +#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(ESP_PLATFORM) +#define CONFIG_HAS_FDATASYNC 1 +#else +#define CONFIG_HAS_FDATASYNC 0 +#endif + +/* + * For NuttX, CONFIG_HAS_ISATTY is provided by its platform header. + * (platform_internal.h) + */ +#if !defined(CONFIG_HAS_D_INO) +#if !defined(__NuttX__) +#define CONFIG_HAS_D_INO 1 +#define CONFIG_HAS_ISATTY 1 +#else +#define CONFIG_HAS_D_INO 0 +#endif +#endif + +#if !defined(__APPLE__) && !defined(ESP_PLATFORM) && !defined(__COSMOPOLITAN__) +#define CONFIG_HAS_POSIX_FALLOCATE 1 +#else +#define CONFIG_HAS_POSIX_FALLOCATE 0 +#endif + +#if defined(O_DSYNC) +#define CONFIG_HAS_O_DSYNC +#endif + +// POSIX requires O_RSYNC to be defined, but Linux explicitly doesn't support +// it. +#if defined(O_RSYNC) && !defined(__linux__) +#define CONFIG_HAS_O_RSYNC +#endif + +#if defined(O_SYNC) +#define CONFIG_HAS_O_SYNC +#endif + +// Converts a POSIX timespec to a WASI timestamp. +static __wasi_timestamp_t +convert_timespec(const struct timespec *ts) +{ + if (ts->tv_sec < 0) + return 0; + if ((__wasi_timestamp_t)ts->tv_sec >= UINT64_MAX / 1000000000) + return UINT64_MAX; + return (__wasi_timestamp_t)ts->tv_sec * 1000000000 + + (__wasi_timestamp_t)ts->tv_nsec; +} + +// Converts a POSIX stat structure to a WASI filestat structure +static void +convert_stat(os_file_handle handle, const struct stat *in, + __wasi_filestat_t *out) +{ + out->st_dev = in->st_dev; + out->st_ino = in->st_ino; + out->st_nlink = (__wasi_linkcount_t)in->st_nlink; + out->st_size = (__wasi_filesize_t)in->st_size; +#ifdef __APPLE__ + out->st_atim = convert_timespec(&in->st_atimespec); + out->st_mtim = convert_timespec(&in->st_mtimespec); + out->st_ctim = convert_timespec(&in->st_ctimespec); +#else + out->st_atim = convert_timespec(&in->st_atim); + out->st_mtim = convert_timespec(&in->st_mtim); + out->st_ctim = convert_timespec(&in->st_ctim); +#endif + + // Convert the file type. In the case of sockets there is no way we + // can easily determine the exact socket type. + if (S_ISBLK(in->st_mode)) { + out->st_filetype = __WASI_FILETYPE_BLOCK_DEVICE; + } + else if (S_ISCHR(in->st_mode)) { + out->st_filetype = __WASI_FILETYPE_CHARACTER_DEVICE; + } + else if (S_ISDIR(in->st_mode)) { + out->st_filetype = __WASI_FILETYPE_DIRECTORY; + } + else if (S_ISFIFO(in->st_mode)) { + out->st_filetype = __WASI_FILETYPE_SOCKET_STREAM; + } + else if (S_ISLNK(in->st_mode)) { + out->st_filetype = __WASI_FILETYPE_SYMBOLIC_LINK; + } + else if (S_ISREG(in->st_mode)) { + out->st_filetype = __WASI_FILETYPE_REGULAR_FILE; + } + else if (S_ISSOCK(in->st_mode)) { + int socktype; + socklen_t socktypelen = sizeof(socktype); + + if (getsockopt(handle, SOL_SOCKET, SO_TYPE, &socktype, &socktypelen) + < 0) { + out->st_filetype = __WASI_FILETYPE_UNKNOWN; + return; + } + + switch (socktype) { + case SOCK_DGRAM: + out->st_filetype = __WASI_FILETYPE_SOCKET_DGRAM; + break; + case SOCK_STREAM: + out->st_filetype = __WASI_FILETYPE_SOCKET_STREAM; + break; + default: + out->st_filetype = __WASI_FILETYPE_UNKNOWN; + return; + } + } + else { + out->st_filetype = __WASI_FILETYPE_UNKNOWN; + } +} + +static void +convert_timestamp(__wasi_timestamp_t in, struct timespec *out) +{ + // Store sub-second remainder. +#if defined(__SYSCALL_SLONG_TYPE) + out->tv_nsec = (__SYSCALL_SLONG_TYPE)(in % 1000000000); +#else + out->tv_nsec = (long)(in % 1000000000); +#endif + in /= 1000000000; + + // Clamp to the maximum in case it would overflow our system's time_t. + out->tv_sec = (time_t)in < BH_TIME_T_MAX ? (time_t)in : BH_TIME_T_MAX; +} + +// Converts the provided timestamps and flags to a set of arguments for +// futimens() and utimensat(). +static void +convert_utimens_arguments(__wasi_timestamp_t st_atim, + __wasi_timestamp_t st_mtim, + __wasi_fstflags_t fstflags, struct timespec *ts) +{ + if ((fstflags & __WASI_FILESTAT_SET_ATIM_NOW) != 0) { + ts[0].tv_nsec = UTIME_NOW; + } + else if ((fstflags & __WASI_FILESTAT_SET_ATIM) != 0) { + convert_timestamp(st_atim, &ts[0]); + } + else { + ts[0].tv_nsec = UTIME_OMIT; + } + + if ((fstflags & __WASI_FILESTAT_SET_MTIM_NOW) != 0) { + ts[1].tv_nsec = UTIME_NOW; + } + else if ((fstflags & __WASI_FILESTAT_SET_MTIM) != 0) { + convert_timestamp(st_mtim, &ts[1]); + } + else { + ts[1].tv_nsec = UTIME_OMIT; + } +} + +__wasi_errno_t +os_fstat(os_file_handle handle, struct __wasi_filestat_t *buf) +{ + struct stat stat_buf; + int ret = fstat(handle, &stat_buf); + + if (ret < 0) + return convert_errno(errno); + + convert_stat(handle, &stat_buf, buf); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_fstatat(os_file_handle handle, const char *path, + struct __wasi_filestat_t *buf, __wasi_lookupflags_t lookup_flags) +{ + struct stat stat_buf; + int ret = fstatat(handle, path, &stat_buf, + (lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) + ? AT_SYMLINK_FOLLOW + : AT_SYMLINK_NOFOLLOW); + + if (ret < 0) + return convert_errno(errno); + + convert_stat(handle, &stat_buf, buf); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_file_get_fdflags(os_file_handle handle, __wasi_fdflags_t *flags) +{ + int ret = fcntl(handle, F_GETFL); + + if (ret < 0) + return convert_errno(errno); + + *flags = 0; + + if ((ret & O_APPEND) != 0) + *flags |= __WASI_FDFLAG_APPEND; +#ifdef CONFIG_HAS_O_DSYNC + if ((ret & O_DSYNC) != 0) + *flags |= __WASI_FDFLAG_DSYNC; +#endif + if ((ret & O_NONBLOCK) != 0) + *flags |= __WASI_FDFLAG_NONBLOCK; +#ifdef CONFIG_HAS_O_RSYNC + if ((ret & O_RSYNC) != 0) + *flags |= __WASI_FDFLAG_RSYNC; +#endif +#ifdef CONFIG_HAS_O_SYNC + if ((ret & O_SYNC) != 0) + *flags |= __WASI_FDFLAG_SYNC; +#endif + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_file_set_fdflags(os_file_handle handle, __wasi_fdflags_t flags) +{ + int fcntl_flags = 0; + + if ((flags & __WASI_FDFLAG_APPEND) != 0) + fcntl_flags |= O_APPEND; + if ((flags & __WASI_FDFLAG_DSYNC) != 0) +#ifdef CONFIG_HAS_O_DSYNC + fcntl_flags |= O_DSYNC; +#else + return __WASI_ENOTSUP; +#endif + if ((flags & __WASI_FDFLAG_NONBLOCK) != 0) + fcntl_flags |= O_NONBLOCK; + if ((flags & __WASI_FDFLAG_RSYNC) != 0) +#ifdef CONFIG_HAS_O_RSYNC + fcntl_flags |= O_RSYNC; +#else + return __WASI_ENOTSUP; +#endif + if ((flags & __WASI_FDFLAG_SYNC) != 0) +#ifdef CONFIG_HAS_O_SYNC + fcntl_flags |= O_SYNC; +#else + return __WASI_ENOTSUP; +#endif + + int ret = fcntl(handle, F_SETFL, fcntl_flags); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_fdatasync(os_file_handle handle) +{ +#if CONFIG_HAS_FDATASYNC + int ret = fdatasync(handle); +#else + int ret = fsync(handle); +#endif + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_fsync(os_file_handle handle) +{ + int ret = fsync(handle); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_open_preopendir(const char *path, os_file_handle *out) +{ + + int fd = open(path, O_RDONLY | O_DIRECTORY, 0); + + if (fd < 0) + return convert_errno(errno); + + *out = fd; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_openat(os_file_handle handle, const char *path, __wasi_oflags_t oflags, + __wasi_fdflags_t fs_flags, __wasi_lookupflags_t lookup_flags, + wasi_libc_file_access_mode read_write_mode, os_file_handle *out) +{ + int open_flags = 0; + + // Convert open flags. + if ((oflags & __WASI_O_CREAT) != 0) { + open_flags |= O_CREAT; + } + if ((oflags & __WASI_O_DIRECTORY) != 0) + open_flags |= O_DIRECTORY; + if ((oflags & __WASI_O_EXCL) != 0) + open_flags |= O_EXCL; + if ((oflags & __WASI_O_TRUNC) != 0) { + open_flags |= O_TRUNC; + } + + // Convert file descriptor flags. + if ((fs_flags & __WASI_FDFLAG_APPEND) != 0) + open_flags |= O_APPEND; + if ((fs_flags & __WASI_FDFLAG_DSYNC) != 0) { +#ifdef CONFIG_HAS_O_DSYNC + open_flags |= O_DSYNC; +#else + return __WASI_ENOTSUP; +#endif + } + if ((fs_flags & __WASI_FDFLAG_NONBLOCK) != 0) + open_flags |= O_NONBLOCK; + if ((fs_flags & __WASI_FDFLAG_RSYNC) != 0) { +#ifdef CONFIG_HAS_O_RSYNC + open_flags |= O_RSYNC; +#else + return __WASI_ENOTSUP; +#endif + } + if ((fs_flags & __WASI_FDFLAG_SYNC) != 0) { +#ifdef CONFIG_HAS_O_SYNC + open_flags |= O_SYNC; +#else + return __WASI_ENOTSUP; +#endif + } + + if ((lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) == 0) { + open_flags |= O_NOFOLLOW; + } + + switch (read_write_mode) { + case WASI_LIBC_ACCESS_MODE_READ_WRITE: + open_flags |= O_RDWR; + break; + case WASI_LIBC_ACCESS_MODE_READ_ONLY: + open_flags |= O_RDONLY; + break; + case WASI_LIBC_ACCESS_MODE_WRITE_ONLY: + open_flags |= O_WRONLY; + break; + default: + return __WASI_EINVAL; + } + + int fd = openat(handle, path, open_flags, 0666); + + if (fd < 0) { + int openat_errno = errno; + // Linux returns ENXIO instead of EOPNOTSUPP when opening a socket. + if (openat_errno == ENXIO) { + struct stat sb; + int ret = fstatat(handle, path, &sb, + (lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) + ? 0 + : AT_SYMLINK_NOFOLLOW); + return ret == 0 && S_ISSOCK(sb.st_mode) ? __WASI_ENOTSUP + : __WASI_ENXIO; + } + // Linux returns ENOTDIR instead of ELOOP when using + // O_NOFOLLOW|O_DIRECTORY on a symlink. + if (openat_errno == ENOTDIR + && (open_flags & (O_NOFOLLOW | O_DIRECTORY)) != 0) { + struct stat sb; + int ret = fstatat(handle, path, &sb, AT_SYMLINK_NOFOLLOW); + if (S_ISLNK(sb.st_mode)) { + return __WASI_ELOOP; + } + (void)ret; + } + // FreeBSD returns EMLINK instead of ELOOP when using O_NOFOLLOW on + // a symlink. + if ((lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) == 0 + && openat_errno == EMLINK) + return __WASI_ELOOP; + + return convert_errno(openat_errno); + } + + *out = fd; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_file_get_access_mode(os_file_handle handle, + wasi_libc_file_access_mode *access_mode) +{ + int ret = fcntl(handle, F_GETFL, 0); + + if (ret < 0) + return convert_errno(errno); + + switch (ret & O_ACCMODE) { + case O_RDONLY: + *access_mode = WASI_LIBC_ACCESS_MODE_READ_ONLY; + break; + case O_WRONLY: + *access_mode = WASI_LIBC_ACCESS_MODE_WRITE_ONLY; + break; + case O_RDWR: + *access_mode = WASI_LIBC_ACCESS_MODE_READ_WRITE; + break; + default: + return __WASI_EINVAL; + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_close(os_file_handle handle, bool is_stdio) +{ + if (is_stdio) + return __WASI_ESUCCESS; + + int ret = close(handle); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_preadv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nread) +{ +#if CONFIG_HAS_PREADV + ssize_t len = + preadv(handle, (const struct iovec *)iov, (int)iovcnt, (off_t)offset); + if (len < 0) + return convert_errno(errno); + + *nread = (size_t)len; + return __WASI_ESUCCESS; +#else + if (iovcnt == 1) { + ssize_t len = pread(handle, iov->buf, iov->buf_len, offset); + + if (len < 0) + return convert_errno(errno); + + *nread = len; + return __WASI_ESUCCESS; + } + + // Allocate a single buffer to fit all data. + size_t totalsize = 0; + for (int i = 0; i < iovcnt; ++i) + totalsize += iov[i].buf_len; + + char *buf = BH_MALLOC(totalsize); + + if (buf == NULL) { + return __WASI_ENOMEM; + } + + // Perform a single read operation. + ssize_t len = pread(handle, buf, totalsize, offset); + + if (len < 0) { + BH_FREE(buf); + return convert_errno(errno); + } + + // Copy data back to vectors. + size_t bufoff = 0; + for (int i = 0; i < iovcnt; ++i) { + if (bufoff + iov[i].buf_len < (size_t)len) { + memcpy(iov[i].buf, buf + bufoff, iov[i].buf_len); + bufoff += iov[i].buf_len; + } + else { + memcpy(iov[i].buf, buf + bufoff, len - bufoff); + break; + } + } + BH_FREE(buf); + *nread = len; + + return __WASI_ESUCCESS; +#endif +} + +__wasi_errno_t +os_pwritev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nwritten) +{ + if (iovcnt == 0) + return __WASI_EINVAL; + + ssize_t len = 0; +#if CONFIG_HAS_PWRITEV + len = + pwritev(handle, (const struct iovec *)iov, (int)iovcnt, (off_t)offset); +#else + if (iovcnt == 1) { + len = pwrite(handle, iov->buf, iov->buf_len, offset); + } + else { + // Allocate a single buffer to fit all data. + size_t totalsize = 0; + for (int i = 0; i < iovcnt; ++i) + totalsize += iov[i].buf_len; + char *buf = BH_MALLOC(totalsize); + if (buf == NULL) { + return __WASI_ENOMEM; + } + size_t bufoff = 0; + for (int i = 0; i < iovcnt; ++i) { + memcpy(buf + bufoff, iov[i].buf, iov[i].buf_len); + bufoff += iov[i].buf_len; + } + + // Perform a single write operation. + len = pwrite(handle, buf, totalsize, offset); + BH_FREE(buf); + } +#endif + if (len < 0) + return convert_errno(errno); + + *nwritten = (size_t)len; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_readv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + size_t *nread) +{ + ssize_t len = readv(handle, (const struct iovec *)iov, (int)iovcnt); + + if (len < 0) + return convert_errno(errno); + + *nread = (size_t)len; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_writev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + size_t *nwritten) +{ + ssize_t len = writev(handle, (const struct iovec *)iov, (int)iovcnt); + + if (len < 0) + return convert_errno(errno); + + *nwritten = (size_t)len; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_fallocate(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length) +{ +#if CONFIG_HAS_POSIX_FALLOCATE + int ret = posix_fallocate(handle, (off_t)offset, (off_t)length); +#else + // At least ensure that the file is grown to the right size. + // TODO(ed): See if this can somehow be implemented without any race + // conditions. We may end up shrinking the file right now. + struct stat sb; + int ret = fstat(handle, &sb); + off_t newsize = (off_t)(offset + length); + + if (ret == 0 && sb.st_size < newsize) + ret = ftruncate(handle, newsize); +#endif + + if (ret != 0) + return convert_errno(ret); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_ftruncate(os_file_handle handle, __wasi_filesize_t size) +{ + int ret = ftruncate(handle, (off_t)size); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_futimens(os_file_handle handle, __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags) +{ + struct timespec ts[2]; + convert_utimens_arguments(access_time, modification_time, fstflags, ts); + + int ret = futimens(handle, ts); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_utimensat(os_file_handle handle, const char *path, + __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags, + __wasi_lookupflags_t lookup_flags) +{ + struct timespec ts[2]; + convert_utimens_arguments(access_time, modification_time, fstflags, ts); + + int ret = utimensat(handle, path, ts, + (lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) + ? 0 + : AT_SYMLINK_NOFOLLOW); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_readlinkat(os_file_handle handle, const char *path, char *buf, + size_t bufsize, size_t *nread) +{ + // Linux requires that the buffer size is positive. whereas POSIX does + // not. Use a fake buffer to store the results if the size is zero. + char fakebuf[1]; + ssize_t len = readlinkat(handle, path, bufsize == 0 ? fakebuf : buf, + bufsize == 0 ? sizeof(fakebuf) : bufsize); + + if (len < 0) + return convert_errno(errno); + + *nread = (size_t)len < bufsize ? (size_t)len : bufsize; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_linkat(os_file_handle from_handle, const char *from_path, + os_file_handle to_handle, const char *to_path, + __wasi_lookupflags_t lookup_flags) +{ + int ret = linkat( + from_handle, from_path, to_handle, to_path, + (lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) ? AT_SYMLINK_FOLLOW : 0); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_symlinkat(const char *old_path, os_file_handle handle, const char *new_path) +{ + int ret = symlinkat(old_path, handle, new_path); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_mkdirat(os_file_handle handle, const char *path) +{ + int ret = mkdirat(handle, path, 0777); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_renameat(os_file_handle old_handle, const char *old_path, + os_file_handle new_handle, const char *new_path) +{ + + int ret = renameat(old_handle, old_path, new_handle, new_path); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_unlinkat(os_file_handle handle, const char *path, bool is_dir) +{ + int ret = unlinkat(handle, path, is_dir ? AT_REMOVEDIR : 0); + +#ifndef __linux__ + if (ret < 0) { + // Non-Linux implementations may return EPERM when attempting to remove + // a directory without REMOVEDIR. While that's what POSIX specifies, + // it's less useful. Adjust this to EISDIR. It doesn't matter that this + // is not atomic with the unlinkat, because if the file is removed and a + // directory is created before fstatat sees it, we're racing with that + // change anyway and unlinkat could have legitimately seen the directory + // if the race had turned out differently. + if (errno == EPERM) { + struct stat statbuf; + if (fstatat(handle, path, &statbuf, AT_SYMLINK_NOFOLLOW) == 0 + && S_ISDIR(statbuf.st_mode)) { + errno = EISDIR; + } + } + // POSIX permits either EEXIST or ENOTEMPTY when the directory is not + // empty. Map it to ENOTEMPTY. + else if (errno == EEXIST) { + errno = ENOTEMPTY; + } + + return convert_errno(errno); + } +#endif + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_lseek(os_file_handle handle, __wasi_filedelta_t offset, + __wasi_whence_t whence, __wasi_filesize_t *new_offset) +{ + int nwhence; + + switch (whence) { + case __WASI_WHENCE_CUR: + nwhence = SEEK_CUR; + break; + case __WASI_WHENCE_END: + nwhence = SEEK_END; + break; + case __WASI_WHENCE_SET: + nwhence = SEEK_SET; + break; + default: + return __WASI_EINVAL; + } + + off_t ret = lseek(handle, offset, nwhence); + + if (ret < 0) + return convert_errno(errno); + + *new_offset = (__wasi_filesize_t)ret; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_fadvise(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length, __wasi_advice_t advice) +{ +#ifdef POSIX_FADV_NORMAL + int nadvice; + switch (advice) { + case __WASI_ADVICE_DONTNEED: + nadvice = POSIX_FADV_DONTNEED; + break; + case __WASI_ADVICE_NOREUSE: + nadvice = POSIX_FADV_NOREUSE; + break; + case __WASI_ADVICE_NORMAL: + nadvice = POSIX_FADV_NORMAL; + break; + case __WASI_ADVICE_RANDOM: + nadvice = POSIX_FADV_RANDOM; + break; + case __WASI_ADVICE_SEQUENTIAL: + nadvice = POSIX_FADV_SEQUENTIAL; + break; + case __WASI_ADVICE_WILLNEED: + nadvice = POSIX_FADV_WILLNEED; + break; + default: + return __WASI_EINVAL; + } + + int ret = posix_fadvise(handle, (off_t)offset, (off_t)length, nadvice); + + if (ret != 0) + return convert_errno(ret); + + return __WASI_ESUCCESS; +#else + // Advisory information can be safely ignored if not supported + switch (advice) { + case __WASI_ADVICE_DONTNEED: + case __WASI_ADVICE_NOREUSE: + case __WASI_ADVICE_NORMAL: + case __WASI_ADVICE_RANDOM: + case __WASI_ADVICE_SEQUENTIAL: + case __WASI_ADVICE_WILLNEED: + return __WASI_ESUCCESS; + default: + return __WASI_EINVAL; + } +#endif +} + +__wasi_errno_t +os_isatty(os_file_handle handle) +{ +#if CONFIG_HAS_ISATTY + int ret = isatty(handle); + + if (ret == 1) + return __WASI_ESUCCESS; + + return __WASI_ENOTTY; +#else + return __WASI_ENOTSUP; +#endif +} + +os_file_handle +os_convert_stdin_handle(os_raw_file_handle raw_stdin) +{ +#ifndef STDIN_FILENO +#define STDIN_FILENO 0 +#endif + return raw_stdin >= 0 ? raw_stdin : STDIN_FILENO; +} + +os_file_handle +os_convert_stdout_handle(os_raw_file_handle raw_stdout) +{ +#ifndef STDOUT_FILENO +#define STDOUT_FILENO 1 +#endif + return raw_stdout >= 0 ? raw_stdout : STDOUT_FILENO; +} + +os_file_handle +os_convert_stderr_handle(os_raw_file_handle raw_stderr) +{ +#ifndef STDERR_FILENO +#define STDERR_FILENO 2 +#endif + return raw_stderr >= 0 ? raw_stderr : STDERR_FILENO; +} + +__wasi_errno_t +os_fdopendir(os_file_handle handle, os_dir_stream *dir_stream) +{ + *dir_stream = fdopendir(handle); + + if (*dir_stream == NULL) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_rewinddir(os_dir_stream dir_stream) +{ + rewinddir(dir_stream); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_seekdir(os_dir_stream dir_stream, __wasi_dircookie_t position) +{ + seekdir(dir_stream, (long)position); + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_readdir(os_dir_stream dir_stream, __wasi_dirent_t *entry, + const char **d_name) +{ + errno = 0; + + struct dirent *dent = readdir(dir_stream); + + if (dent == NULL) { + *d_name = NULL; + if (errno != 0) { + return convert_errno(errno); + } + else { + return 0; + } + } + + long offset = (__wasi_dircookie_t)telldir(dir_stream); + + size_t namlen = strlen(dent->d_name); + + *d_name = dent->d_name; + entry->d_next = offset; + entry->d_namlen = (__wasi_dirnamlen_t)namlen; +#if CONFIG_HAS_D_INO + entry->d_ino = dent->d_ino; +#else + entry->d_ino = 0; +#endif + + switch (dent->d_type) { + case DT_BLK: + entry->d_type = __WASI_FILETYPE_BLOCK_DEVICE; + break; + case DT_CHR: + entry->d_type = __WASI_FILETYPE_CHARACTER_DEVICE; + break; + case DT_DIR: + entry->d_type = __WASI_FILETYPE_DIRECTORY; + break; + case DT_FIFO: + entry->d_type = __WASI_FILETYPE_SOCKET_STREAM; + break; + case DT_LNK: + entry->d_type = __WASI_FILETYPE_SYMBOLIC_LINK; + break; + case DT_REG: + entry->d_type = __WASI_FILETYPE_REGULAR_FILE; + break; +#ifdef DT_SOCK + case DT_SOCK: + // Technically not correct, but good enough. + entry->d_type = __WASI_FILETYPE_SOCKET_STREAM; + break; +#endif + default: + entry->d_type = __WASI_FILETYPE_UNKNOWN; + break; + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_closedir(os_dir_stream dir_stream) +{ + int ret = closedir(dir_stream); + + if (ret < 0) + return convert_errno(errno); + + return __WASI_ESUCCESS; +} + +os_dir_stream +os_get_invalid_dir_stream() +{ + return NULL; +} + +bool +os_is_dir_stream_valid(os_dir_stream *dir_stream) +{ + assert(dir_stream != NULL); + + return *dir_stream != NULL; +} + +bool +os_is_handle_valid(os_file_handle *handle) +{ + assert(handle != NULL); + + return *handle > -1; +} + +char * +os_realpath(const char *path, char *resolved_path) +{ + return realpath(path, resolved_path); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_malloc.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_malloc.c new file mode 100644 index 0000000..912998e --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_malloc.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +void * +os_malloc(unsigned size) +{ + return malloc(size); +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return realloc(ptr, size); +} + +void +os_free(void *ptr) +{ + free(ptr); +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + int ret = -1; + FILE *f; + char line[128] = { 0 }; + unsigned int out_idx = 0; + + if (!out || !size) + goto quit; + + f = fopen("/proc/self/status", "r"); + if (!f) { + perror("fopen failed: "); + goto quit; + } + + memset(out, 0, size); + + while (fgets(line, sizeof(line), f)) { +#if WASM_ENABLE_MEMORY_PROFILING != 0 + if (strncmp(line, "Vm", 2) == 0 || strncmp(line, "Rss", 3) == 0) { +#else + if (strncmp(line, "VmRSS", 5) == 0 + || strncmp(line, "RssAnon", 7) == 0) { +#endif + size_t line_len = strlen(line); + if (line_len >= size - 1 - out_idx) + goto close_file; + + /* copying without null-terminated byte */ + memcpy(out + out_idx, line, line_len); + out_idx += line_len; + } + } + + if (ferror(f)) { + perror("fgets failed: "); + goto close_file; + } + + ret = 0; +close_file: + fclose(f); +quit: + return ret; +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_memmap.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_memmap.c new file mode 100644 index 0000000..c76abf1 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_memmap.c @@ -0,0 +1,293 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +#if defined(__APPLE__) || defined(__MACH__) +#include +#include +#endif + +#ifndef BH_ENABLE_TRACE_MMAP +#define BH_ENABLE_TRACE_MMAP 0 +#endif + +#if BH_ENABLE_TRACE_MMAP != 0 +static size_t total_size_mmapped = 0; +static size_t total_size_munmapped = 0; +#endif + +#define HUGE_PAGE_SIZE (2 * 1024 * 1024) + +#if !defined(__APPLE__) && !defined(__NuttX__) && defined(MADV_HUGEPAGE) +static inline uintptr_t +round_up(uintptr_t v, uintptr_t b) +{ + uintptr_t m = b - 1; + return (v + m) & ~m; +} + +static inline uintptr_t +round_down(uintptr_t v, uintptr_t b) +{ + uintptr_t m = b - 1; + return v & ~m; +} +#endif + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + int map_prot = PROT_NONE; +#if (defined(__APPLE__) || defined(__MACH__)) && defined(__arm64__) \ + && defined(TARGET_OS_OSX) && TARGET_OS_OSX != 0 + int map_flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_JIT; +#else + int map_flags = MAP_ANONYMOUS | MAP_PRIVATE; +#endif + uint64 request_size, page_size; + uint8 *addr = MAP_FAILED; + uint32 i; + + page_size = (uint64)getpagesize(); + request_size = (size + page_size - 1) & ~(page_size - 1); + +#if !defined(__APPLE__) && !defined(__NuttX__) && defined(MADV_HUGEPAGE) + /* huge page isn't supported on MacOS and NuttX */ + if (request_size >= HUGE_PAGE_SIZE) + /* apply one extra huge page */ + request_size += HUGE_PAGE_SIZE; +#endif + + if ((size_t)request_size < size) + /* integer overflow */ + return NULL; + +#if WASM_ENABLE_MEMORY64 == 0 + if (request_size > 16 * (uint64)UINT32_MAX) + /* at most 64 G is allowed */ + return NULL; +#endif + + if (prot & MMAP_PROT_READ) + map_prot |= PROT_READ; + + if (prot & MMAP_PROT_WRITE) + map_prot |= PROT_WRITE; + + if (prot & MMAP_PROT_EXEC) + map_prot |= PROT_EXEC; + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +#ifndef __APPLE__ + if (flags & MMAP_MAP_32BIT) + map_flags |= MAP_32BIT; +#endif +#endif + + if (flags & MMAP_MAP_FIXED) + map_flags |= MAP_FIXED; + +#if defined(BUILD_TARGET_RISCV64_LP64D) || defined(BUILD_TARGET_RISCV64_LP64) + /* As AOT relocation in RISCV64 may require that the code/data mapped + * is in range 0 to 2GB, we try to map the memory with hint address + * (mmap's first argument) to meet the requirement. + */ + if (!hint && !(flags & MMAP_MAP_FIXED) && (flags & MMAP_MAP_32BIT)) { + uint8 *stack_addr = (uint8 *)&map_prot; + uint8 *text_addr = (uint8 *)os_mmap; + /* hint address begins with 1MB */ + static uint8 *hint_addr = (uint8 *)(uintptr_t)BH_MB; + + if ((hint_addr - text_addr >= 0 && hint_addr - text_addr < 100 * BH_MB) + || (text_addr - hint_addr >= 0 + && text_addr - hint_addr < 100 * BH_MB)) { + /* hint address is possibly in text section, skip it */ + hint_addr += 100 * BH_MB; + } + + if ((hint_addr - stack_addr >= 0 && hint_addr - stack_addr < 8 * BH_MB) + || (stack_addr - hint_addr >= 0 + && stack_addr - hint_addr < 8 * BH_MB)) { + /* hint address is possibly in native stack area, skip it */ + hint_addr += 8 * BH_MB; + } + + /* try 10 times, step with 1MB each time */ + for (i = 0; i < 10 && hint_addr < (uint8 *)(uintptr_t)(2ULL * BH_GB); + i++) { + addr = mmap(hint_addr, request_size, map_prot, map_flags, file, 0); + if (addr != MAP_FAILED) { + if (addr > (uint8 *)(uintptr_t)(2ULL * BH_GB)) { + /* unmap and try again if the mapped address doesn't + * meet the requirement */ + os_munmap(addr, request_size); + } + else { + /* success, reset next hint address */ + hint_addr += request_size; + break; + } + } + hint_addr += BH_MB; + } + } +#endif /* end of BUILD_TARGET_RISCV64_LP64D || BUILD_TARGET_RISCV64_LP64 */ + + /* memory hasn't been mapped or was mapped failed previously */ + if (addr == MAP_FAILED) { + /* try 5 times */ + for (i = 0; i < 5; i++) { + addr = mmap(hint, request_size, map_prot, map_flags, file, 0); + if (addr != MAP_FAILED) + break; + } + } + + if (addr == MAP_FAILED) { +#if BH_ENABLE_TRACE_MMAP != 0 + os_printf("mmap failed\n"); +#endif + return NULL; + } + +#if BH_ENABLE_TRACE_MMAP != 0 + total_size_mmapped += request_size; + os_printf("mmap return: %p with size: %zu, total_size_mmapped: %zu, " + "total_size_munmapped: %zu\n", + addr, request_size, total_size_mmapped, total_size_munmapped); +#endif + +#if !defined(__APPLE__) && !defined(__NuttX__) && defined(MADV_HUGEPAGE) + /* huge page isn't supported on MacOS and NuttX */ + if (request_size > HUGE_PAGE_SIZE) { + uintptr_t huge_start, huge_end; + size_t prefix_size = 0, suffix_size = HUGE_PAGE_SIZE; + + huge_start = round_up((uintptr_t)addr, HUGE_PAGE_SIZE); + + if (huge_start > (uintptr_t)addr) { + prefix_size += huge_start - (uintptr_t)addr; + suffix_size -= huge_start - (uintptr_t)addr; + } + + /* unmap one extra huge page */ + + if (prefix_size > 0) { + munmap(addr, prefix_size); +#if BH_ENABLE_TRACE_MMAP != 0 + total_size_munmapped += prefix_size; + os_printf("munmap %p with size: %zu, total_size_mmapped: %zu, " + "total_size_munmapped: %zu\n", + addr, prefix_size, total_size_mmapped, + total_size_munmapped); +#endif + } + if (suffix_size > 0) { + munmap(addr + request_size - suffix_size, suffix_size); +#if BH_ENABLE_TRACE_MMAP != 0 + total_size_munmapped += suffix_size; + os_printf("munmap %p with size: %zu, total_size_mmapped: %zu, " + "total_size_munmapped: %zu\n", + addr + request_size - suffix_size, suffix_size, + total_size_mmapped, total_size_munmapped); +#endif + } + + addr = (uint8 *)huge_start; + request_size -= HUGE_PAGE_SIZE; + + huge_end = round_down(huge_start + request_size, HUGE_PAGE_SIZE); + if (huge_end > huge_start) { + int ret = madvise((void *)huge_start, huge_end - huge_start, + MADV_HUGEPAGE); + if (ret) { +#if BH_ENABLE_TRACE_MMAP != 0 + os_printf( + "warning: madvise(%p, %lu) huge page failed, return %d\n", + (void *)huge_start, huge_end - huge_start, ret); +#endif + } + } + } +#endif /* end of __APPLE__ || __NuttX__ || !MADV_HUGEPAGE */ + + return addr; +} + +void +os_munmap(void *addr, size_t size) +{ + uint64 page_size = (uint64)getpagesize(); + uint64 request_size = (size + page_size - 1) & ~(page_size - 1); + + if (addr) { + if (munmap(addr, request_size)) { + os_printf("os_munmap error addr:%p, size:0x%" PRIx64 ", errno:%d\n", + addr, request_size, errno); + return; + } +#if BH_ENABLE_TRACE_MMAP != 0 + total_size_munmapped += request_size; + os_printf("munmap %p with size: %zu, total_size_mmapped: %zu, " + "total_size_munmapped: %zu\n", + addr, request_size, total_size_mmapped, total_size_munmapped); +#endif + } +} + +#if WASM_HAVE_MREMAP != 0 +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size) +{ + void *ptr = mremap(old_addr, old_size, new_size, MREMAP_MAYMOVE); + + if (ptr == MAP_FAILED) { +#if BH_ENABLE_TRACE_MMAP != 0 + os_printf("mremap failed: %d\n", errno); +#endif + return os_mremap_slow(old_addr, old_size, new_size); + } + + return ptr; +} +#endif + +int +os_mprotect(void *addr, size_t size, int prot) +{ + int map_prot = PROT_NONE; + uint64 page_size = (uint64)getpagesize(); + uint64 request_size = (size + page_size - 1) & ~(page_size - 1); + + if (!addr) + return 0; + + if (prot & MMAP_PROT_READ) + map_prot |= PROT_READ; + + if (prot & MMAP_PROT_WRITE) + map_prot |= PROT_WRITE; + + if (prot & MMAP_PROT_EXEC) + map_prot |= PROT_EXEC; + + return mprotect(addr, request_size, map_prot); +} + +void +os_dcache_flush(void) +{} + +void +os_icache_flush(void *start, size_t len) +{ +#if defined(__APPLE__) || defined(__MACH__) + sys_icache_invalidate(start, len); +#else + (void)start; + (void)len; +#endif +} diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_sleep.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_sleep.c new file mode 100644 index 0000000..fa06450 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_sleep.c @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include "platform_api_extension.h" + +int +os_usleep(uint32 usec) +{ + struct timespec ts; + int ret; + + ts.tv_sec = usec / 1000000; + ts.tv_nsec = (usec % 1000000) * 1000; + ret = nanosleep(&ts, NULL); + return ret == 0 ? 0 : -1; +} diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_socket.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_socket.c new file mode 100644 index 0000000..0293b08 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_socket.c @@ -0,0 +1,1034 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" +#include "libc_errno.h" + +#include +#include +#include +#include + +static bool +textual_addr_to_sockaddr(const char *textual, int port, struct sockaddr *out, + socklen_t *out_len) +{ + struct sockaddr_in *v4; +#ifdef IPPROTO_IPV6 + struct sockaddr_in6 *v6; +#endif + + assert(textual); + + v4 = (struct sockaddr_in *)out; + if (inet_pton(AF_INET, textual, &v4->sin_addr.s_addr) == 1) { + v4->sin_family = AF_INET; + v4->sin_port = htons(port); + *out_len = sizeof(struct sockaddr_in); + return true; + } + +#ifdef IPPROTO_IPV6 + v6 = (struct sockaddr_in6 *)out; + if (inet_pton(AF_INET6, textual, &v6->sin6_addr.s6_addr) == 1) { + v6->sin6_family = AF_INET6; + v6->sin6_port = htons(port); + *out_len = sizeof(struct sockaddr_in6); + return true; + } +#endif + + return false; +} + +static int +sockaddr_to_bh_sockaddr(const struct sockaddr *sockaddr, + bh_sockaddr_t *bh_sockaddr) +{ + switch (sockaddr->sa_family) { + case AF_INET: + { + struct sockaddr_in *addr = (struct sockaddr_in *)sockaddr; + + bh_sockaddr->port = ntohs(addr->sin_port); + bh_sockaddr->addr_buffer.ipv4 = ntohl(addr->sin_addr.s_addr); + bh_sockaddr->is_ipv4 = true; + return BHT_OK; + } +#ifdef IPPROTO_IPV6 + case AF_INET6: + { + struct sockaddr_in6 *addr = (struct sockaddr_in6 *)sockaddr; + size_t i; + + bh_sockaddr->port = ntohs(addr->sin6_port); + + for (i = 0; i < sizeof(bh_sockaddr->addr_buffer.ipv6) + / sizeof(bh_sockaddr->addr_buffer.ipv6[0]); + i++) { + uint16 part_addr = addr->sin6_addr.s6_addr[i * 2] + | (addr->sin6_addr.s6_addr[i * 2 + 1] << 8); + bh_sockaddr->addr_buffer.ipv6[i] = ntohs(part_addr); + } + + bh_sockaddr->is_ipv4 = false; + return BHT_OK; + } +#endif + default: + errno = EAFNOSUPPORT; + return BHT_ERROR; + } +} + +static void +bh_sockaddr_to_sockaddr(const bh_sockaddr_t *bh_sockaddr, + struct sockaddr_storage *sockaddr, socklen_t *socklen) +{ + if (bh_sockaddr->is_ipv4) { + struct sockaddr_in *addr = (struct sockaddr_in *)sockaddr; + addr->sin_port = htons(bh_sockaddr->port); + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = htonl(bh_sockaddr->addr_buffer.ipv4); + *socklen = sizeof(*addr); + } +#ifdef IPPROTO_IPV6 + else { + struct sockaddr_in6 *addr = (struct sockaddr_in6 *)sockaddr; + size_t i; + addr->sin6_port = htons(bh_sockaddr->port); + addr->sin6_family = AF_INET6; + + for (i = 0; i < sizeof(bh_sockaddr->addr_buffer.ipv6) + / sizeof(bh_sockaddr->addr_buffer.ipv6[0]); + i++) { + uint16 part_addr = htons(bh_sockaddr->addr_buffer.ipv6[i]); + addr->sin6_addr.s6_addr[i * 2] = 0xff & part_addr; + addr->sin6_addr.s6_addr[i * 2 + 1] = (0xff00 & part_addr) >> 8; + } + + *socklen = sizeof(*addr); + } +#endif +} + +int +os_socket_create(bh_socket_t *sock, bool is_ipv4, bool is_tcp) +{ + int af = is_ipv4 ? AF_INET : AF_INET6; + + if (!sock) { + return BHT_ERROR; + } + + if (is_tcp) { + *sock = socket(af, SOCK_STREAM, IPPROTO_TCP); + } + else { + *sock = socket(af, SOCK_DGRAM, 0); + } + + return (*sock == -1) ? BHT_ERROR : BHT_OK; +} + +int +os_socket_bind(bh_socket_t socket, const char *host, int *port) +{ + struct sockaddr_storage addr = { 0 }; + struct linger ling; + socklen_t socklen; + int ret; + + assert(host); + assert(port); + + ling.l_onoff = 1; + ling.l_linger = 0; + + if (!textual_addr_to_sockaddr(host, *port, (struct sockaddr *)&addr, + &socklen)) { + goto fail; + } + + ret = fcntl(socket, F_SETFD, FD_CLOEXEC); + if (ret < 0) { + goto fail; + } + + ret = setsockopt(socket, SOL_SOCKET, SO_LINGER, &ling, sizeof(ling)); + if (ret < 0) { + goto fail; + } + + ret = bind(socket, (struct sockaddr *)&addr, socklen); + if (ret < 0) { + goto fail; + } + + socklen = sizeof(addr); + if (getsockname(socket, (void *)&addr, &socklen) == -1) { + goto fail; + } + + if (addr.ss_family == AF_INET) { + *port = ntohs(((struct sockaddr_in *)&addr)->sin_port); + } + else { +#ifdef IPPROTO_IPV6 + *port = ntohs(((struct sockaddr_in6 *)&addr)->sin6_port); +#else + goto fail; +#endif + } + + return BHT_OK; + +fail: + return BHT_ERROR; +} + +int +os_socket_settimeout(bh_socket_t socket, uint64 timeout_us) +{ + struct timeval tv; + tv.tv_sec = timeout_us / 1000000UL; + tv.tv_usec = timeout_us % 1000000UL; + + if (setsockopt(socket, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, + sizeof(tv)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_listen(bh_socket_t socket, int max_client) +{ + if (listen(socket, max_client) != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, + unsigned int *addrlen) +{ + *sock = accept(server_sock, addr, addrlen); + + if (*sock < 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_connect(bh_socket_t socket, const char *addr, int port) +{ + struct sockaddr_storage addr_in = { 0 }; + socklen_t addr_len; + int ret = 0; + + if (!textual_addr_to_sockaddr(addr, port, (struct sockaddr *)&addr_in, + &addr_len)) { + return BHT_ERROR; + } + + ret = connect(socket, (struct sockaddr *)&addr_in, addr_len); + if (ret == -1) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_recv(bh_socket_t socket, void *buf, unsigned int len) +{ + return recv(socket, buf, len, 0); +} + +int +os_socket_recv_from(bh_socket_t socket, void *buf, unsigned int len, int flags, + bh_sockaddr_t *src_addr) +{ + struct sockaddr_storage sock_addr = { 0 }; + socklen_t socklen = sizeof(sock_addr); + int ret; + + ret = recvfrom(socket, buf, len, flags, (struct sockaddr *)&sock_addr, + &socklen); + + if (ret < 0) { + return ret; + } + + if (src_addr && socklen > 0) { + if (sockaddr_to_bh_sockaddr((struct sockaddr *)&sock_addr, src_addr) + == BHT_ERROR) { + return -1; + } + } + else if (src_addr) { + memset(src_addr, 0, sizeof(*src_addr)); + } + + return ret; +} + +int +os_socket_send(bh_socket_t socket, const void *buf, unsigned int len) +{ + return send(socket, buf, len, 0); +} + +int +os_socket_send_to(bh_socket_t socket, const void *buf, unsigned int len, + int flags, const bh_sockaddr_t *dest_addr) +{ + struct sockaddr_storage sock_addr = { 0 }; + socklen_t socklen = 0; + + bh_sockaddr_to_sockaddr(dest_addr, &sock_addr, &socklen); + + return sendto(socket, buf, len, flags, (const struct sockaddr *)&sock_addr, + socklen); +} + +int +os_socket_close(bh_socket_t socket) +{ + close(socket); + return BHT_OK; +} + +__wasi_errno_t +os_socket_shutdown(bh_socket_t socket) +{ + if (shutdown(socket, O_RDWR) != 0) { + return convert_errno(errno); + } + return __WASI_ESUCCESS; +} + +int +os_socket_inet_network(bool is_ipv4, const char *cp, bh_ip_addr_buffer_t *out) +{ + if (!cp) + return BHT_ERROR; + + if (is_ipv4) { + if (inet_pton(AF_INET, cp, &out->ipv4) != 1) { + return BHT_ERROR; + } + /* Note: ntohl(INADDR_NONE) == INADDR_NONE */ + out->ipv4 = ntohl(out->ipv4); + } + else { +#ifdef IPPROTO_IPV6 + if (inet_pton(AF_INET6, cp, out->ipv6) != 1) { + return BHT_ERROR; + } + for (int i = 0; i < 8; i++) { + out->ipv6[i] = ntohs(out->ipv6[i]); + } +#else + errno = EAFNOSUPPORT; + return BHT_ERROR; +#endif + } + + return BHT_OK; +} + +static int +getaddrinfo_error_to_errno(int error) +{ + switch (error) { + case EAI_AGAIN: + return EAGAIN; + case EAI_FAIL: + return EFAULT; + case EAI_MEMORY: + return ENOMEM; + case EAI_SYSTEM: + return errno; + default: + return EINVAL; + } +} + +static int +is_addrinfo_supported(struct addrinfo *info) +{ + return + // Allow only IPv4 and IPv6 + (info->ai_family == AF_INET || info->ai_family == AF_INET6) + // Allow only UDP and TCP + && (info->ai_socktype == SOCK_DGRAM || info->ai_socktype == SOCK_STREAM) + && (info->ai_protocol == IPPROTO_TCP + || info->ai_protocol == IPPROTO_UDP); +} + +int +os_socket_addr_resolve(const char *host, const char *service, + uint8_t *hint_is_tcp, uint8_t *hint_is_ipv4, + bh_addr_info_t *addr_info, size_t addr_info_size, + size_t *max_info_size) +{ + struct addrinfo hints = { 0 }, *res, *result; + int hints_enabled = hint_is_tcp || hint_is_ipv4; + int ret; + size_t pos = 0; + + if (hints_enabled) { + if (hint_is_ipv4) { + hints.ai_family = *hint_is_ipv4 ? AF_INET : AF_INET6; + } + if (hint_is_tcp) { + hints.ai_socktype = *hint_is_tcp ? SOCK_STREAM : SOCK_DGRAM; + } + } + + ret = getaddrinfo(host, strlen(service) == 0 ? NULL : service, + hints_enabled ? &hints : NULL, &result); + if (ret != BHT_OK) { + errno = getaddrinfo_error_to_errno(ret); + return BHT_ERROR; + } + + res = result; + while (res) { + if (addr_info_size > pos) { + if (!is_addrinfo_supported(res)) { + res = res->ai_next; + continue; + } + + ret = + sockaddr_to_bh_sockaddr(res->ai_addr, &addr_info[pos].sockaddr); + + if (ret == BHT_ERROR) { + freeaddrinfo(result); + return BHT_ERROR; + } + + addr_info[pos].is_tcp = res->ai_socktype == SOCK_STREAM; + } + + pos++; + res = res->ai_next; + } + + *max_info_size = pos; + freeaddrinfo(result); + + return BHT_OK; +} + +static int +os_socket_setbooloption(bh_socket_t socket, int level, int optname, + bool is_enabled) +{ + int option = (int)is_enabled; + if (setsockopt(socket, level, optname, &option, sizeof(option)) != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +static int +os_socket_getbooloption(bh_socket_t socket, int level, int optname, + bool *is_enabled) +{ + assert(is_enabled); + + int optval; + socklen_t optval_size = sizeof(optval); + if (getsockopt(socket, level, optname, &optval, &optval_size) != 0) { + return BHT_ERROR; + } + *is_enabled = (bool)optval; + return BHT_OK; +} + +int +os_socket_set_send_buf_size(bh_socket_t socket, size_t bufsiz) +{ + int buf_size_int = (int)bufsiz; + if (setsockopt(socket, SOL_SOCKET, SO_SNDBUF, &buf_size_int, + sizeof(buf_size_int)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_get_send_buf_size(bh_socket_t socket, size_t *bufsiz) +{ + assert(bufsiz); + + int buf_size_int; + socklen_t bufsiz_len = sizeof(buf_size_int); + if (getsockopt(socket, SOL_SOCKET, SO_SNDBUF, &buf_size_int, &bufsiz_len) + != 0) { + return BHT_ERROR; + } + *bufsiz = (size_t)buf_size_int; + + return BHT_OK; +} + +int +os_socket_set_recv_buf_size(bh_socket_t socket, size_t bufsiz) +{ + int buf_size_int = (int)bufsiz; + if (setsockopt(socket, SOL_SOCKET, SO_RCVBUF, &buf_size_int, + sizeof(buf_size_int)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_get_recv_buf_size(bh_socket_t socket, size_t *bufsiz) +{ + assert(bufsiz); + + int buf_size_int; + socklen_t bufsiz_len = sizeof(buf_size_int); + if (getsockopt(socket, SOL_SOCKET, SO_RCVBUF, &buf_size_int, &bufsiz_len) + != 0) { + return BHT_ERROR; + } + *bufsiz = (size_t)buf_size_int; + + return BHT_OK; +} + +int +os_socket_set_keep_alive(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_KEEPALIVE, + is_enabled); +} + +int +os_socket_get_keep_alive(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_KEEPALIVE, + is_enabled); +} + +int +os_socket_set_reuse_addr(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_REUSEADDR, + is_enabled); +} + +int +os_socket_get_reuse_addr(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_REUSEADDR, + is_enabled); +} + +int +os_socket_set_reuse_port(bh_socket_t socket, bool is_enabled) +{ +#if defined(SO_REUSEPORT) /* NuttX doesn't have SO_REUSEPORT */ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_REUSEPORT, + is_enabled); +#else + errno = ENOTSUP; + return BHT_ERROR; +#endif /* defined(SO_REUSEPORT) */ +} + +int +os_socket_get_reuse_port(bh_socket_t socket, bool *is_enabled) +{ +#if defined(SO_REUSEPORT) /* NuttX doesn't have SO_REUSEPORT */ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_REUSEPORT, + is_enabled); +#else + errno = ENOTSUP; + return BHT_ERROR; +#endif /* defined(SO_REUSEPORT) */ +} + +int +os_socket_set_linger(bh_socket_t socket, bool is_enabled, int linger_s) +{ + struct linger linger_opts = { .l_onoff = (int)is_enabled, + .l_linger = linger_s }; + if (setsockopt(socket, SOL_SOCKET, SO_LINGER, &linger_opts, + sizeof(linger_opts)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_get_linger(bh_socket_t socket, bool *is_enabled, int *linger_s) +{ + assert(is_enabled); + assert(linger_s); + + struct linger linger_opts; + socklen_t linger_opts_len = sizeof(linger_opts); + if (getsockopt(socket, SOL_SOCKET, SO_LINGER, &linger_opts, + &linger_opts_len) + != 0) { + return BHT_ERROR; + } + *linger_s = linger_opts.l_linger; + *is_enabled = (bool)linger_opts.l_onoff; + return BHT_OK; +} + +int +os_socket_set_tcp_no_delay(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, IPPROTO_TCP, TCP_NODELAY, + is_enabled); +} + +int +os_socket_get_tcp_no_delay(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, IPPROTO_TCP, TCP_NODELAY, + is_enabled); +} + +int +os_socket_set_tcp_quick_ack(bh_socket_t socket, bool is_enabled) +{ +#ifdef TCP_QUICKACK + return os_socket_setbooloption(socket, IPPROTO_TCP, TCP_QUICKACK, + is_enabled); +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_get_tcp_quick_ack(bh_socket_t socket, bool *is_enabled) +{ +#ifdef TCP_QUICKACK + return os_socket_getbooloption(socket, IPPROTO_TCP, TCP_QUICKACK, + is_enabled); +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_set_tcp_keep_idle(bh_socket_t socket, uint32 time_s) +{ + int time_s_int = (int)time_s; +#ifdef TCP_KEEPIDLE + if (setsockopt(socket, IPPROTO_TCP, TCP_KEEPIDLE, &time_s_int, + sizeof(time_s_int)) + != 0) { + return BHT_ERROR; + } + return BHT_OK; +#elif defined(TCP_KEEPALIVE) + if (setsockopt(socket, IPPROTO_TCP, TCP_KEEPALIVE, &time_s_int, + sizeof(time_s_int)) + != 0) { + return BHT_ERROR; + } + return BHT_OK; +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_get_tcp_keep_idle(bh_socket_t socket, uint32 *time_s) +{ + assert(time_s); + int time_s_int; + socklen_t time_s_len = sizeof(time_s_int); +#ifdef TCP_KEEPIDLE + if (getsockopt(socket, IPPROTO_TCP, TCP_KEEPIDLE, &time_s_int, &time_s_len) + != 0) { + return BHT_ERROR; + } + *time_s = (uint32)time_s_int; + return BHT_OK; +#elif defined(TCP_KEEPALIVE) + if (getsockopt(socket, IPPROTO_TCP, TCP_KEEPALIVE, &time_s_int, &time_s_len) + != 0) { + return BHT_ERROR; + } + *time_s = (uint32)time_s_int; + return BHT_OK; +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_set_tcp_keep_intvl(bh_socket_t socket, uint32 time_s) +{ + int time_s_int = (int)time_s; +#ifdef TCP_KEEPINTVL + if (setsockopt(socket, IPPROTO_TCP, TCP_KEEPINTVL, &time_s_int, + sizeof(time_s_int)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_get_tcp_keep_intvl(bh_socket_t socket, uint32 *time_s) +{ +#ifdef TCP_KEEPINTVL + assert(time_s); + int time_s_int; + socklen_t time_s_len = sizeof(time_s_int); + if (getsockopt(socket, IPPROTO_TCP, TCP_KEEPINTVL, &time_s_int, &time_s_len) + != 0) { + return BHT_ERROR; + } + *time_s = (uint32)time_s_int; + return BHT_OK; +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_set_tcp_fastopen_connect(bh_socket_t socket, bool is_enabled) +{ +#ifdef TCP_FASTOPEN_CONNECT + return os_socket_setbooloption(socket, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, + is_enabled); +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_get_tcp_fastopen_connect(bh_socket_t socket, bool *is_enabled) +{ +#ifdef TCP_FASTOPEN_CONNECT + return os_socket_getbooloption(socket, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, + is_enabled); +#else + errno = ENOSYS; + + return BHT_ERROR; +#endif +} + +int +os_socket_set_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool is_enabled) +{ + if (ipv6) { +#ifdef IPPROTO_IPV6 + return os_socket_setbooloption(socket, IPPROTO_IPV6, + IPV6_MULTICAST_LOOP, is_enabled); +#else + errno = EAFNOSUPPORT; + return BHT_ERROR; +#endif + } + else { + return os_socket_setbooloption(socket, IPPROTO_IP, IP_MULTICAST_LOOP, + is_enabled); + } +} + +int +os_socket_get_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool *is_enabled) +{ + if (ipv6) { +#ifdef IPPROTO_IPV6 + return os_socket_getbooloption(socket, IPPROTO_IPV6, + IPV6_MULTICAST_LOOP, is_enabled); +#else + errno = EAFNOSUPPORT; + return BHT_ERROR; +#endif + } + else { + return os_socket_getbooloption(socket, IPPROTO_IP, IP_MULTICAST_LOOP, + is_enabled); + } +} + +int +os_socket_set_ip_add_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6) +{ + assert(imr_multiaddr); + if (is_ipv6) { +#if defined(IPPROTO_IPV6) && !defined(BH_PLATFORM_COSMOPOLITAN) + struct ipv6_mreq mreq; + for (int i = 0; i < 8; i++) { + ((uint16_t *)mreq.ipv6mr_multiaddr.s6_addr)[i] = + imr_multiaddr->ipv6[i]; + } + mreq.ipv6mr_interface = imr_interface; + if (setsockopt(socket, IPPROTO_IPV6, IPV6_JOIN_GROUP, &mreq, + sizeof(mreq)) + != 0) { + return BHT_ERROR; + } +#else + errno = EAFNOSUPPORT; + return BHT_ERROR; +#endif + } + else { + struct ip_mreq mreq; + mreq.imr_multiaddr.s_addr = imr_multiaddr->ipv4; + mreq.imr_interface.s_addr = imr_interface; + if (setsockopt(socket, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, + sizeof(mreq)) + != 0) { + return BHT_ERROR; + } + } + + return BHT_OK; +} + +int +os_socket_set_ip_drop_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6) +{ + assert(imr_multiaddr); + if (is_ipv6) { +#if defined(IPPROTO_IPV6) && !defined(BH_PLATFORM_COSMOPOLITAN) + struct ipv6_mreq mreq; + for (int i = 0; i < 8; i++) { + ((uint16_t *)mreq.ipv6mr_multiaddr.s6_addr)[i] = + imr_multiaddr->ipv6[i]; + } + mreq.ipv6mr_interface = imr_interface; + if (setsockopt(socket, IPPROTO_IPV6, IPV6_LEAVE_GROUP, &mreq, + sizeof(mreq)) + != 0) { + return BHT_ERROR; + } +#else + errno = EAFNOSUPPORT; + return BHT_ERROR; +#endif + } + else { + struct ip_mreq mreq; + mreq.imr_multiaddr.s_addr = imr_multiaddr->ipv4; + mreq.imr_interface.s_addr = imr_interface; + if (setsockopt(socket, IPPROTO_IP, IP_DROP_MEMBERSHIP, &mreq, + sizeof(mreq)) + != 0) { + return BHT_ERROR; + } + } + + return BHT_OK; +} + +int +os_socket_set_ip_ttl(bh_socket_t socket, uint8_t ttl_s) +{ + if (setsockopt(socket, IPPROTO_IP, IP_TTL, &ttl_s, sizeof(ttl_s)) != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_get_ip_ttl(bh_socket_t socket, uint8_t *ttl_s) +{ + socklen_t opt_len = sizeof(*ttl_s); + if (getsockopt(socket, IPPROTO_IP, IP_TTL, ttl_s, &opt_len) != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_set_ip_multicast_ttl(bh_socket_t socket, uint8_t ttl_s) +{ + if (setsockopt(socket, IPPROTO_IP, IP_MULTICAST_TTL, &ttl_s, sizeof(ttl_s)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_get_ip_multicast_ttl(bh_socket_t socket, uint8_t *ttl_s) +{ + socklen_t opt_len = sizeof(*ttl_s); + if (getsockopt(socket, IPPROTO_IP, IP_MULTICAST_TTL, ttl_s, &opt_len) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_set_ipv6_only(bh_socket_t socket, bool is_enabled) +{ +#ifdef IPPROTO_IPV6 + return os_socket_setbooloption(socket, IPPROTO_IPV6, IPV6_V6ONLY, + is_enabled); +#else + errno = EAFNOSUPPORT; + return BHT_ERROR; +#endif +} + +int +os_socket_get_ipv6_only(bh_socket_t socket, bool *is_enabled) +{ +#ifdef IPPROTO_IPV6 + return os_socket_getbooloption(socket, IPPROTO_IPV6, IPV6_V6ONLY, + is_enabled); +#else + errno = EAFNOSUPPORT; + return BHT_ERROR; +#endif +} + +int +os_socket_set_broadcast(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_BROADCAST, + is_enabled); +} + +int +os_socket_get_broadcast(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_BROADCAST, + is_enabled); +} + +int +os_socket_set_send_timeout(bh_socket_t socket, uint64 timeout_us) +{ + struct timeval tv; + tv.tv_sec = timeout_us / 1000000UL; + tv.tv_usec = timeout_us % 1000000UL; + if (setsockopt(socket, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) != 0) { + return BHT_ERROR; + } + return BHT_OK; +} + +int +os_socket_get_send_timeout(bh_socket_t socket, uint64 *timeout_us) +{ + struct timeval tv; + socklen_t tv_len = sizeof(tv); + if (getsockopt(socket, SOL_SOCKET, SO_SNDTIMEO, &tv, &tv_len) != 0) { + return BHT_ERROR; + } + *timeout_us = (tv.tv_sec * 1000000UL) + tv.tv_usec; + return BHT_OK; +} + +int +os_socket_set_recv_timeout(bh_socket_t socket, uint64 timeout_us) +{ + struct timeval tv; + tv.tv_sec = timeout_us / 1000000UL; + tv.tv_usec = timeout_us % 1000000UL; + if (setsockopt(socket, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) { + return BHT_ERROR; + } + return BHT_OK; +} + +int +os_socket_get_recv_timeout(bh_socket_t socket, uint64 *timeout_us) +{ + struct timeval tv; + socklen_t tv_len = sizeof(tv); + if (getsockopt(socket, SOL_SOCKET, SO_RCVTIMEO, &tv, &tv_len) != 0) { + return BHT_ERROR; + } + *timeout_us = (tv.tv_sec * 1000000UL) + tv.tv_usec; + return BHT_OK; +} + +int +os_socket_addr_local(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + struct sockaddr_storage addr_storage = { 0 }; + socklen_t addr_len = sizeof(addr_storage); + int ret; + + ret = getsockname(socket, (struct sockaddr *)&addr_storage, &addr_len); + + if (ret != BHT_OK) { + return BHT_ERROR; + } + + return sockaddr_to_bh_sockaddr((struct sockaddr *)&addr_storage, sockaddr); +} + +int +os_socket_addr_remote(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + struct sockaddr_storage addr_storage = { 0 }; + socklen_t addr_len = sizeof(addr_storage); + int ret; + + ret = getpeername(socket, (struct sockaddr *)&addr_storage, &addr_len); + + if (ret != BHT_OK) { + return BHT_ERROR; + } + + return sockaddr_to_bh_sockaddr((struct sockaddr *)&addr_storage, sockaddr); +} diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_thread.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_thread.c new file mode 100644 index 0000000..1195d80 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_thread.c @@ -0,0 +1,776 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#if defined(__APPLE__) || defined(__MACH__) +#include +#endif + +typedef struct { + thread_start_routine_t start; + void *arg; +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_signal_handler signal_handler; +#endif +} thread_wrapper_arg; + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* The signal handler passed to os_thread_signal_init() */ +static os_thread_local_attribute os_signal_handler signal_handler; +#endif + +static void * +os_thread_wrapper(void *arg) +{ + thread_wrapper_arg *targ = arg; + thread_start_routine_t start_func = targ->start; + void *thread_arg = targ->arg; +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_signal_handler handler = targ->signal_handler; +#endif + +#if 0 + os_printf("THREAD CREATED %jx\n", (uintmax_t)(uintptr_t)pthread_self()); +#endif + BH_FREE(targ); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (os_thread_signal_init(handler) != 0) + return NULL; +#endif +#ifdef OS_ENABLE_WAKEUP_BLOCKING_OP + os_end_blocking_op(); +#endif + start_func(thread_arg); +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_signal_destroy(); +#endif + return NULL; +} + +int +os_thread_create_with_prio(korp_tid *tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + pthread_attr_t tattr; + thread_wrapper_arg *targ; + + assert(stack_size > 0); + assert(tid); + assert(start); + + pthread_attr_init(&tattr); + pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE); + if (pthread_attr_setstacksize(&tattr, stack_size) != 0) { + os_printf("Invalid thread stack size %u. " + "Min stack size on Linux = %u\n", + stack_size, (unsigned int)PTHREAD_STACK_MIN); + pthread_attr_destroy(&tattr); + return BHT_ERROR; + } + + targ = (thread_wrapper_arg *)BH_MALLOC(sizeof(*targ)); + if (!targ) { + pthread_attr_destroy(&tattr); + return BHT_ERROR; + } + + targ->start = start; + targ->arg = arg; +#ifdef OS_ENABLE_HW_BOUND_CHECK + targ->signal_handler = signal_handler; +#endif + + if (pthread_create(tid, &tattr, os_thread_wrapper, targ) != 0) { + pthread_attr_destroy(&tattr); + BH_FREE(targ); + return BHT_ERROR; + } + + pthread_attr_destroy(&tattr); + return BHT_OK; +} + +int +os_thread_create(korp_tid *tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +korp_tid +os_self_thread() +{ + return (korp_tid)pthread_self(); +} + +int +os_mutex_init(korp_mutex *mutex) +{ + return pthread_mutex_init(mutex, NULL) == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_recursive_mutex_init(korp_mutex *mutex) +{ + int ret; + + pthread_mutexattr_t mattr; + + assert(mutex); + ret = pthread_mutexattr_init(&mattr); + if (ret) + return BHT_ERROR; + + pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_RECURSIVE); + ret = pthread_mutex_init(mutex, &mattr); + pthread_mutexattr_destroy(&mattr); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + int ret; + + assert(mutex); + ret = pthread_mutex_destroy(mutex); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + int ret; + + assert(mutex); + ret = pthread_mutex_lock(mutex); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + int ret; + + assert(mutex); + ret = pthread_mutex_unlock(mutex); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_cond_init(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_init(cond, NULL) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_destroy(cond) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + assert(cond); + assert(mutex); + + if (pthread_cond_wait(cond, mutex) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +korp_sem * +os_sem_open(const char *name, int oflags, int mode, int val) +{ + return sem_open(name, oflags, mode, val); +} + +int +os_sem_close(korp_sem *sem) +{ + return sem_close(sem); +} + +int +os_sem_wait(korp_sem *sem) +{ + return sem_wait(sem); +} + +int +os_sem_trywait(korp_sem *sem) +{ + return sem_trywait(sem); +} + +int +os_sem_post(korp_sem *sem) +{ + return sem_post(sem); +} + +int +os_sem_getvalue(korp_sem *sem, int *sval) +{ +#if defined(__APPLE__) + /* + * macOS doesn't have working sem_getvalue. + * It's marked as deprecated in the system header. + * Mock it up here to avoid compile-time deprecation warnings. + */ + errno = ENOSYS; + return -1; +#else + return sem_getvalue(sem, sval); +#endif +} + +int +os_sem_unlink(const char *name) +{ + return sem_unlink(name); +} + +static void +msec_nsec_to_abstime(struct timespec *ts, uint64 usec) +{ + struct timeval tv; + time_t tv_sec_new; + long int tv_nsec_new; + + gettimeofday(&tv, NULL); + + tv_sec_new = (time_t)(tv.tv_sec + usec / 1000000); + if (tv_sec_new >= tv.tv_sec) { + ts->tv_sec = tv_sec_new; + } + else { + /* integer overflow */ + ts->tv_sec = BH_TIME_T_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + + tv_nsec_new = (long int)(tv.tv_usec * 1000 + (usec % 1000000) * 1000); + if (tv.tv_usec * 1000 >= tv.tv_usec && tv_nsec_new >= tv.tv_usec * 1000) { + ts->tv_nsec = tv_nsec_new; + } + else { + /* integer overflow */ + ts->tv_nsec = LONG_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + + if (ts->tv_nsec >= 1000000000L && ts->tv_sec < BH_TIME_T_MAX) { + ts->tv_sec++; + ts->tv_nsec -= 1000000000L; + } +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + int ret; + struct timespec abstime; + + if (useconds == BHT_WAIT_FOREVER) + ret = pthread_cond_wait(cond, mutex); + else { + msec_nsec_to_abstime(&abstime, useconds); + ret = pthread_cond_timedwait(cond, mutex, &abstime); + } + + if (ret != BHT_OK && ret != ETIMEDOUT) + return BHT_ERROR; + + return ret; +} + +int +os_cond_signal(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_signal(cond) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_cond_broadcast(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_broadcast(cond) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_init(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_init(lock, NULL) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_rdlock(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_rdlock(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_wrlock(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_wrlock(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_unlock(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_unlock(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_destroy(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_destroy(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_thread_join(korp_tid thread, void **value_ptr) +{ + return pthread_join(thread, value_ptr); +} + +int +os_thread_detach(korp_tid thread) +{ + return pthread_detach(thread); +} + +void +os_thread_exit(void *retval) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_signal_destroy(); +#endif + return pthread_exit(retval); +} + +#if defined(os_thread_local_attribute) +static os_thread_local_attribute uint8 *thread_stack_boundary = NULL; +#endif + +uint8 * +os_thread_get_stack_boundary() +{ + pthread_t self; +#ifdef __linux__ + pthread_attr_t attr; + size_t guard_size; +#endif + uint8 *addr = NULL; + size_t stack_size, max_stack_size; + int page_size; + +#if defined(os_thread_local_attribute) + if (thread_stack_boundary) + return thread_stack_boundary; +#endif + + page_size = getpagesize(); + self = pthread_self(); + max_stack_size = + (size_t)(APP_THREAD_STACK_SIZE_MAX + page_size - 1) & ~(page_size - 1); + + if (max_stack_size < APP_THREAD_STACK_SIZE_DEFAULT) + max_stack_size = APP_THREAD_STACK_SIZE_DEFAULT; + +#ifdef __linux__ + if (pthread_getattr_np(self, &attr) == 0) { + pthread_attr_getstack(&attr, (void **)&addr, &stack_size); + pthread_attr_getguardsize(&attr, &guard_size); + pthread_attr_destroy(&attr); + if (stack_size > max_stack_size) + addr = addr + stack_size - max_stack_size; + if (guard_size < (size_t)page_size) + /* Reserved 1 guard page at least for safety */ + guard_size = (size_t)page_size; + addr += guard_size; + } + (void)stack_size; +#elif defined(__APPLE__) || defined(__NuttX__) + if ((addr = (uint8 *)pthread_get_stackaddr_np(self))) { + stack_size = pthread_get_stacksize_np(self); + + /** + * Check whether stack_addr is the base or end of the stack, + * change it to the base if it is the end of stack. + */ + if (addr <= (uint8 *)&stack_size) + addr = addr + stack_size; + + if (stack_size > max_stack_size) + stack_size = max_stack_size; + + addr -= stack_size; + /* Reserved 1 guard page at least for safety */ + addr += page_size; + } +#endif + +#if defined(os_thread_local_attribute) + thread_stack_boundary = addr; +#endif + return addr; +} + +void +os_thread_jit_write_protect_np(bool enabled) +{ +#if (defined(__APPLE__) || defined(__MACH__)) && defined(__arm64__) \ + && defined(TARGET_OS_OSX) && TARGET_OS_OSX != 0 + pthread_jit_write_protect_np(enabled); +#endif +} + +#ifdef OS_ENABLE_HW_BOUND_CHECK + +#define SIG_ALT_STACK_SIZE (32 * 1024) + +/** + * Whether thread signal enviornment is initialized: + * the signal handler is registered, the stack pages are touched, + * the stack guard pages are set and signal alternate stack are set. + */ +static os_thread_local_attribute bool thread_signal_inited = false; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 +/* The signal alternate stack base addr */ +static os_thread_local_attribute uint8 *sigalt_stack_base_addr; + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__clang__) +#pragma clang optimize off +__attribute__((no_sanitize_address)) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC optimize("O0") +__attribute__((no_sanitize_address)) +#endif +static uint32 +touch_pages(uint8 *stack_min_addr, uint32 page_size) +{ + uint8 sum = 0; + while (1) { + volatile uint8 *touch_addr = (volatile uint8 *)os_alloca(page_size / 2); + if (touch_addr < stack_min_addr + page_size) { + sum += *(stack_min_addr + page_size - 1); + break; + } + *touch_addr = 0; + sum += *touch_addr; + } + return sum; +} +#if defined(__clang__) +#pragma clang optimize on +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + +static bool +init_stack_guard_pages() +{ + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + uint8 *stack_min_addr = os_thread_get_stack_boundary(); + + if (stack_min_addr == NULL) + return false; + + /* Touch each stack page to ensure that it has been mapped: the OS + may lazily grow the stack mapping as a guard page is hit. */ + (void)touch_pages(stack_min_addr, page_size); + /* First time to call aot function, protect guard pages */ + if (os_mprotect(stack_min_addr, page_size * guard_page_count, + MMAP_PROT_NONE) + != 0) { + return false; + } + return true; +} + +static void +destroy_stack_guard_pages() +{ + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + uint8 *stack_min_addr = os_thread_get_stack_boundary(); + + os_mprotect(stack_min_addr, page_size * guard_page_count, + MMAP_PROT_READ | MMAP_PROT_WRITE); +} +#endif /* end of WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 */ + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +static void +mask_signals(int how) +{ + sigset_t set; + + sigemptyset(&set); + sigaddset(&set, SIGSEGV); + sigaddset(&set, SIGBUS); + pthread_sigmask(how, &set, NULL); +} + +static struct sigaction prev_sig_act_SIGSEGV; +static struct sigaction prev_sig_act_SIGBUS; + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +static void +signal_callback(int sig_num, siginfo_t *sig_info, void *sig_ucontext) +{ + void *sig_addr = sig_info->si_addr; + struct sigaction *prev_sig_act = NULL; + + mask_signals(SIG_BLOCK); + + /* Try to handle signal with the registered signal handler */ + if (signal_handler && (sig_num == SIGSEGV || sig_num == SIGBUS)) { + signal_handler(sig_addr); + } + + if (sig_num == SIGSEGV) + prev_sig_act = &prev_sig_act_SIGSEGV; + else if (sig_num == SIGBUS) + prev_sig_act = &prev_sig_act_SIGBUS; + + /* Forward the signal to next handler if found */ + if (prev_sig_act && (prev_sig_act->sa_flags & SA_SIGINFO)) { + prev_sig_act->sa_sigaction(sig_num, sig_info, sig_ucontext); + } + else if (prev_sig_act + && prev_sig_act->sa_handler + /* Filter out SIG_DFL and SIG_IGN here, they will + run into the else branch below */ + && (void *)prev_sig_act->sa_handler != SIG_DFL + && (void *)prev_sig_act->sa_handler != SIG_IGN) { + prev_sig_act->sa_handler(sig_num); + } + /* Output signal info and then crash if signal is unhandled */ + else { + switch (sig_num) { + case SIGSEGV: + os_printf("unhandled SIGSEGV, si_addr: %p\n", sig_addr); + break; + case SIGBUS: + os_printf("unhandled SIGBUS, si_addr: %p\n", sig_addr); + break; + default: + os_printf("unhandle signal %d, si_addr: %p\n", sig_num, + sig_addr); + break; + } + + abort(); + } +} + +int +os_thread_signal_init(os_signal_handler handler) +{ + struct sigaction sig_act; +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + stack_t sigalt_stack_info; + uint32 map_size = SIG_ALT_STACK_SIZE; + uint8 *map_addr; +#endif + + if (thread_signal_inited) + return 0; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + if (!init_stack_guard_pages()) { + os_printf("Failed to init stack guard pages\n"); + return -1; + } + + /* Initialize memory for signal alternate stack of current thread */ + if (!(map_addr = os_mmap(NULL, map_size, MMAP_PROT_READ | MMAP_PROT_WRITE, + MMAP_MAP_NONE, os_get_invalid_handle()))) { + os_printf("Failed to mmap memory for alternate stack\n"); + goto fail1; + } + + /* Initialize signal alternate stack */ + memset(map_addr, 0, map_size); + sigalt_stack_info.ss_sp = map_addr; + sigalt_stack_info.ss_size = map_size; + sigalt_stack_info.ss_flags = 0; + if (sigaltstack(&sigalt_stack_info, NULL) != 0) { + os_printf("Failed to init signal alternate stack\n"); + goto fail2; + } +#endif + + memset(&prev_sig_act_SIGSEGV, 0, sizeof(struct sigaction)); + memset(&prev_sig_act_SIGBUS, 0, sizeof(struct sigaction)); + + /* Install signal hanlder */ + sig_act.sa_sigaction = signal_callback; + sig_act.sa_flags = SA_SIGINFO | SA_NODEFER; +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + sig_act.sa_flags |= SA_ONSTACK; +#endif + sigemptyset(&sig_act.sa_mask); + if (sigaction(SIGSEGV, &sig_act, &prev_sig_act_SIGSEGV) != 0 + || sigaction(SIGBUS, &sig_act, &prev_sig_act_SIGBUS) != 0) { + os_printf("Failed to register signal handler\n"); + goto fail3; + } + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + sigalt_stack_base_addr = map_addr; +#endif + signal_handler = handler; + thread_signal_inited = true; + return 0; + +fail3: +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + memset(&sigalt_stack_info, 0, sizeof(stack_t)); + sigalt_stack_info.ss_flags = SS_DISABLE; + sigalt_stack_info.ss_size = map_size; + sigaltstack(&sigalt_stack_info, NULL); +fail2: + os_munmap(map_addr, map_size); +fail1: + destroy_stack_guard_pages(); +#endif + return -1; +} + +void +os_thread_signal_destroy() +{ +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + stack_t sigalt_stack_info; +#endif + + if (!thread_signal_inited) + return; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + /* Disable signal alternate stack */ + memset(&sigalt_stack_info, 0, sizeof(stack_t)); + sigalt_stack_info.ss_flags = SS_DISABLE; + sigalt_stack_info.ss_size = SIG_ALT_STACK_SIZE; + sigaltstack(&sigalt_stack_info, NULL); + + os_munmap(sigalt_stack_base_addr, SIG_ALT_STACK_SIZE); + + destroy_stack_guard_pages(); +#endif + + thread_signal_inited = false; +} + +bool +os_thread_signal_inited() +{ + return thread_signal_inited; +} + +void +os_signal_unmask() +{ + mask_signals(SIG_UNBLOCK); +} + +void +os_sigreturn() +{ +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 +#if defined(__APPLE__) +#define UC_RESET_ALT_STACK 0x80000000 + extern int __sigreturn(void *, int); + + /* It's necessary to call __sigreturn to restore the sigaltstack state + after exiting the signal handler. */ + __sigreturn(NULL, UC_RESET_ALT_STACK); +#endif +#endif +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ diff --git a/wasm-micro-runtime/core/shared/platform/common/posix/posix_time.c b/wasm-micro-runtime/core/shared/platform/common/posix/posix_time.c new file mode 100644 index 0000000..8c339ab --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/common/posix/posix_time.c @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +uint64 +os_time_get_boot_us() +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + + return ((uint64)ts.tv_sec) * 1000 * 1000 + ((uint64)ts.tv_nsec) / 1000; +} + +uint64 +os_time_thread_cputime_us() +{ + struct timespec ts; + if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) { + return 0; + } + + return ((uint64)ts.tv_sec) * 1000 * 1000 + ((uint64)ts.tv_nsec) / 1000; +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/cosmopolitan/platform_init.c b/wasm-micro-runtime/core/shared/platform/cosmopolitan/platform_init.c new file mode 100644 index 0000000..2aae13f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/cosmopolitan/platform_init.c @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +int +bh_platform_init() +{ + return 0; +} + +void +bh_platform_destroy() +{} + +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} diff --git a/wasm-micro-runtime/core/shared/platform/cosmopolitan/platform_internal.h b/wasm-micro-runtime/core/shared/platform/cosmopolitan/platform_internal.h new file mode 100644 index 0000000..7260211 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/cosmopolitan/platform_internal.h @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * Copyright (C) 2023 Dylibso. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_COSMOPOLITAN +#define BH_PLATFORM_COSMOPOLITAN +#endif + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (32 * 1024) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef sem_t korp_sem; + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define os_thread_local_attribute __thread + +#define bh_socket_t int + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#if WASM_DISABLE_WRITE_GS_BASE == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +#define os_writegsbase(base_addr) \ + do { \ + uint64 __gs_value = (uint64)(uintptr_t)base_addr; \ + asm volatile("wrgsbase %0" ::"r"(__gs_value) : "memory"); \ + } while (0) +#if 0 +/* _writegsbase_u64 also works, but need to add -mfsgsbase flag for gcc */ +#include +#define os_writegsbase(base_addr) \ + _writegsbase_u64(((uint64)(uintptr_t)base_addr)) +#endif +#endif +#endif + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + +#include + +#define OS_ENABLE_HW_BOUND_CHECK + +typedef jmp_buf korp_jmpbuf; + +#define os_setjmp setjmp +#define os_longjmp longjmp +#define os_alloca alloca + +typedef void (*os_signal_handler)(void *sig_addr); + +int +os_thread_signal_init(os_signal_handler handler); + +void +os_thread_signal_destroy(); + +bool +os_thread_signal_inited(); + +void +os_signal_unmask(); + +void +os_sigreturn(); +#endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ +#endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ + +#define os_getpagesize getpagesize + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/cosmopolitan/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/cosmopolitan/shared_platform.cmake new file mode 100644 index 0000000..929ecb9 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/cosmopolitan/shared_platform.cmake @@ -0,0 +1,19 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# Copyright (C) 2023 Dylibso. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_COSMOPOLITAN) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE}) + +file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) +LIST (APPEND RUNTIME_LIB_HEADER_LIST ${header}) diff --git a/wasm-micro-runtime/core/shared/platform/darwin/platform_init.c b/wasm-micro-runtime/core/shared/platform/darwin/platform_init.c new file mode 100644 index 0000000..2aae13f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/darwin/platform_init.c @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +int +bh_platform_init() +{ + return 0; +} + +void +bh_platform_destroy() +{} + +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} diff --git a/wasm-micro-runtime/core/shared/platform/darwin/platform_internal.h b/wasm-micro-runtime/core/shared/platform/darwin/platform_internal.h new file mode 100644 index 0000000..1cbecdc --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/darwin/platform_internal.h @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_DARWIN +#define BH_PLATFORM_DARWIN +#endif + +#define BH_HAS_DLFCN 1 + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (32 * 1024) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef sem_t korp_sem; + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define os_thread_local_attribute __thread + +#define bh_socket_t int + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + +#include + +#define OS_ENABLE_HW_BOUND_CHECK + +typedef jmp_buf korp_jmpbuf; + +#define os_setjmp setjmp +#define os_longjmp longjmp +#define os_alloca alloca + +typedef void (*os_signal_handler)(void *sig_addr); + +int +os_thread_signal_init(os_signal_handler handler); + +void +os_thread_signal_destroy(); + +bool +os_thread_signal_inited(); + +void +os_signal_unmask(); + +void +os_sigreturn(); +#endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ +#endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ + +#define os_getpagesize getpagesize + +#if WASM_DISABLE_WAKEUP_BLOCKING_OP == 0 +#define OS_ENABLE_WAKEUP_BLOCKING_OP +#endif +void +os_set_signal_number_for_blocking_op(int signo); + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/darwin/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/darwin/shared_platform.cmake new file mode 100644 index 0000000..3680f3d --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/darwin/shared_platform.cmake @@ -0,0 +1,21 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_DARWIN) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) +set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE}) + +file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) +LIST (APPEND RUNTIME_LIB_HEADER_LIST ${header}) diff --git a/wasm-micro-runtime/core/shared/platform/ego/platform_init.c b/wasm-micro-runtime/core/shared/platform/ego/platform_init.c new file mode 100644 index 0000000..38a0e80 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/ego/platform_init.c @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../linux/platform_init.c" \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/ego/platform_internal.h b/wasm-micro-runtime/core/shared/platform/ego/platform_internal.h new file mode 100644 index 0000000..1ece346 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/ego/platform_internal.h @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../linux/platform_internal.h" diff --git a/wasm-micro-runtime/core/shared/platform/ego/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/ego/shared_platform.cmake new file mode 100644 index 0000000..9b84c58 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/ego/shared_platform.cmake @@ -0,0 +1,20 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_EGO) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +set (PLATFORM_SHARED_SOURCE + ${PLATFORM_COMMON_POSIX_SOURCE} + ${CMAKE_CURRENT_LIST_DIR}/platform_init.c +) + +LIST (APPEND RUNTIME_LIB_HEADER_LIST + ${CMAKE_CURRENT_LIST_DIR}/platform_internal.h +) \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_malloc.c b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_malloc.c new file mode 100644 index 0000000..08ec883 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_malloc.c @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +void * +os_malloc(unsigned size) +{ + void *buf_origin; + void *buf_fixed; + uintptr_t *addr_field; + + buf_origin = malloc(size + 8 + sizeof(uintptr_t)); + if (!buf_origin) { + return NULL; + } + buf_fixed = buf_origin + sizeof(void *); + if ((uintptr_t)buf_fixed & (uintptr_t)0x7) { + buf_fixed = (void *)((uintptr_t)(buf_fixed + 8) & (~(uintptr_t)7)); + } + + addr_field = buf_fixed - sizeof(uintptr_t); + *addr_field = (uintptr_t)buf_origin; + + return buf_fixed; +} + +void * +os_realloc(void *ptr, unsigned size) +{ + void *mem_origin; + void *mem_new; + void *mem_new_fixed; + uintptr_t *addr_field; + + if (!ptr) { + return os_malloc(size); + } + + addr_field = ptr - sizeof(uintptr_t); + mem_origin = (void *)(*addr_field); + mem_new = realloc(mem_origin, size + 8 + sizeof(uintptr_t)); + if (!mem_new) { + return NULL; + } + + if (mem_origin != mem_new) { + mem_new_fixed = mem_new + sizeof(uintptr_t); + if ((uint32)mem_new_fixed & 0x7) { + mem_new_fixed = + (void *)((uintptr_t)(mem_new + 8) & (~(uintptr_t)7)); + } + + addr_field = mem_new_fixed - sizeof(uintptr_t); + *addr_field = (uintptr_t)mem_new; + + return mem_new_fixed; + } + + return ptr; +} + +void +os_free(void *ptr) +{ + void *mem_origin; + uintptr_t *addr_field; + + if (ptr) { + addr_field = ptr - sizeof(uintptr_t); + mem_origin = (void *)(*addr_field); + + free(mem_origin); + } +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} diff --git a/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_memmap.c b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_memmap.c new file mode 100644 index 0000000..21f186b --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_memmap.c @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) +#include "soc/mmu.h" +#include "rom/cache.h" + +#define MEM_DUAL_BUS_OFFSET (IRAM0_CACHE_ADDRESS_LOW - DRAM0_CACHE_ADDRESS_LOW) + +#define in_ibus_ext(addr) \ + (((uint32)addr >= IRAM0_CACHE_ADDRESS_LOW) \ + && ((uint32)addr < IRAM0_CACHE_ADDRESS_HIGH)) + +static portMUX_TYPE s_spinlock = portMUX_INITIALIZER_UNLOCKED; +#endif + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + if (prot & MMAP_PROT_EXEC) { +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + uint32_t mem_caps = MALLOC_CAP_SPIRAM; +#else + uint32_t mem_caps = MALLOC_CAP_EXEC; +#endif + + // Memory allocation with MALLOC_CAP_EXEC will return 4-byte aligned + // Reserve extra 4 byte to fixup alignment and size for the pointer to + // the originally allocated address + void *buf_origin = + heap_caps_malloc(size + 4 + sizeof(uintptr_t), mem_caps); + if (!buf_origin) { + return NULL; + } + void *buf_fixed = buf_origin + sizeof(void *); + if ((uintptr_t)buf_fixed & (uintptr_t)0x7) { + buf_fixed = (void *)((uintptr_t)(buf_fixed + 4) & (~(uintptr_t)7)); + } + + uintptr_t *addr_field = buf_fixed - sizeof(uintptr_t); + *addr_field = (uintptr_t)buf_origin; +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + return buf_fixed + MEM_DUAL_BUS_OFFSET; +#else + return buf_fixed; +#endif + } + else { +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + uint32_t mem_caps = MALLOC_CAP_SPIRAM; +#else + uint32_t mem_caps = MALLOC_CAP_8BIT; +#endif + void *buf_origin = + heap_caps_malloc(size + 4 + sizeof(uintptr_t), mem_caps); + if (!buf_origin) { + return NULL; + } + + // Memory allocation with MALLOC_CAP_SPIRAM or MALLOC_CAP_8BIT will + // return 4-byte aligned Reserve extra 4 byte to fixup alignment and + // size for the pointer to the originally allocated address + void *buf_fixed = buf_origin + sizeof(void *); + if ((uintptr_t)buf_fixed & (uintptr_t)0x7) { + buf_fixed = (void *)((uintptr_t)(buf_fixed + 4) & (~(uintptr_t)7)); + } + + uintptr_t *addr_field = buf_fixed - sizeof(uintptr_t); + *addr_field = (uintptr_t)buf_origin; + + return buf_fixed; + } +} + +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size) +{ + return os_mremap_slow(old_addr, old_size, new_size); +} + +void +os_munmap(void *addr, size_t size) +{ + char *ptr = (char *)addr; + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + if (in_ibus_ext(ptr)) { + ptr -= MEM_DUAL_BUS_OFFSET; + } +#endif + // We don't need special handling of the executable allocations + // here, free() of esp-idf handles it properly + return os_free(ptr); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + return 0; +} + +void +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + IRAM_ATTR +#endif + os_dcache_flush() +{ +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + uint32_t preload; + extern void Cache_WriteBack_All(void); + + portENTER_CRITICAL(&s_spinlock); + + Cache_WriteBack_All(); + preload = Cache_Disable_ICache(); + Cache_Enable_ICache(preload); + + portEXIT_CRITICAL(&s_spinlock); +#endif +} + +void +os_icache_flush(void *start, size_t len) +{} + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) +void * +os_get_dbus_mirror(void *ibus) +{ + if (in_ibus_ext(ibus)) { + return (void *)((char *)ibus - MEM_DUAL_BUS_OFFSET); + } + else { + return ibus; + } +} +#endif diff --git a/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_platform.c b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_platform.c new file mode 100644 index 0000000..8fea325 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_platform.c @@ -0,0 +1,289 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)) \ + && (ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 2, 0)) +#define UTIMENSAT_TIMESPEC_POINTER 1 +#define FUTIMENS_TIMESPEC_POINTER 1 +#endif + +int +bh_platform_init() +{ + return 0; +} + +void +bh_platform_destroy() +{} + +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} + +uint64 +os_time_get_boot_us(void) +{ + return (uint64)esp_timer_get_time(); +} + +uint64 +os_time_thread_cputime_us(void) +{ + /* FIXME if u know the right api */ + return os_time_get_boot_us(); +} + +uint8 * +os_thread_get_stack_boundary(void) +{ +#if defined(CONFIG_FREERTOS_USE_TRACE_FACILITY) + TaskStatus_t pxTaskStatus; + vTaskGetInfo(xTaskGetCurrentTaskHandle(), &pxTaskStatus, pdTRUE, eInvalid); + return pxTaskStatus.pxStackBase; +#else // !defined(CONFIG_FREERTOS_USE_TRACE_FACILITY) + return NULL; +#endif +} + +void +os_thread_jit_write_protect_np(bool enabled) +{} + +int +os_usleep(uint32 usec) +{ + return usleep(usec); +} + +/* Below parts of readv & writev are ported from Nuttx, under Apache License + * v2.0 */ + +ssize_t +readv(int fildes, const struct iovec *iov, int iovcnt) +{ + ssize_t ntotal; + ssize_t nread; + size_t remaining; + uint8_t *buffer; + int i; + + /* Process each entry in the struct iovec array */ + + for (i = 0, ntotal = 0; i < iovcnt; i++) { + /* Ignore zero-length reads */ + + if (iov[i].iov_len > 0) { + buffer = iov[i].iov_base; + remaining = iov[i].iov_len; + + /* Read repeatedly as necessary to fill buffer */ + + do { + /* NOTE: read() is a cancellation point */ + + nread = read(fildes, buffer, remaining); + + /* Check for a read error */ + + if (nread < 0) { + return nread; + } + + /* Check for an end-of-file condition */ + + else if (nread == 0) { + return ntotal; + } + + /* Update pointers and counts in order to handle partial + * buffer reads. + */ + + buffer += nread; + remaining -= nread; + ntotal += nread; + } while (remaining > 0); + } + } + + return ntotal; +} + +ssize_t +writev(int fildes, const struct iovec *iov, int iovcnt) +{ + ssize_t ntotal; + ssize_t nwritten; + size_t remaining; + uint8_t *buffer; + int i; + + /* Process each entry in the struct iovec array */ + + for (i = 0, ntotal = 0; i < iovcnt; i++) { + /* Ignore zero-length writes */ + + if (iov[i].iov_len > 0) { + buffer = iov[i].iov_base; + remaining = iov[i].iov_len; + + /* Write repeatedly as necessary to write the entire buffer */ + + do { + /* NOTE: write() is a cancellation point */ + + nwritten = write(fildes, buffer, remaining); + + /* Check for a write error */ + + if (nwritten < 0) { + return ntotal ? ntotal : -1; + } + + /* Update pointers and counts in order to handle partial + * buffer writes. + */ + + buffer += nwritten; + remaining -= nwritten; + ntotal += nwritten; + } while (remaining > 0); + } + } + + return ntotal; +} + +int +openat(int fd, const char *path, int oflags, ...) +{ + errno = ENOSYS; + return -1; +} + +int +fstatat(int fd, const char *path, struct stat *buf, int flag) +{ + errno = ENOSYS; + return -1; +} + +int +mkdirat(int fd, const char *path, mode_t mode) +{ + errno = ENOSYS; + return -1; +} + +ssize_t +readlinkat(int fd, const char *path, char *buf, size_t bufsize) +{ + errno = EINVAL; + return -1; +} + +int +linkat(int fd1, const char *path1, int fd2, const char *path2, int flag) +{ + errno = ENOSYS; + return -1; +} + +int +renameat(int fromfd, const char *from, int tofd, const char *to) +{ + errno = ENOSYS; + return -1; +} + +int +symlinkat(const char *target, int fd, const char *path) +{ + errno = ENOSYS; + return -1; +} + +int +unlinkat(int fd, const char *path, int flag) +{ + errno = ENOSYS; + return -1; +} + +int +utimensat(int fd, const char *path, +#if UTIMENSAT_TIMESPEC_POINTER + const struct timespec *ts, +#else + const struct timespec ts[2], +#endif + int flag) +{ + errno = ENOSYS; + return -1; +} + +DIR * +fdopendir(int fd) +{ + errno = ENOSYS; + return NULL; +} + +#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(4, 4, 2) +int +ftruncate(int fd, off_t length) +{ + errno = ENOSYS; + return -1; +} +#endif + +int +futimens(int fd, +#if FUTIMENS_TIMESPEC_POINTER + const struct timespec *times +#else + const struct timespec times[2] +#endif +) +{ + errno = ENOSYS; + return -1; +} + +int +nanosleep(const struct timespec *req, struct timespec *rem) +{ + errno = ENOSYS; + return -1; +} diff --git a/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_socket.c b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_socket.c new file mode 100644 index 0000000..a75d829 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_socket.c @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" +#include "libc_errno.h" + +#include + +static void +textual_addr_to_sockaddr(const char *textual, int port, struct sockaddr_in *out) +{ + assert(textual); + + out->sin_family = AF_INET; + out->sin_port = htons(port); + out->sin_addr.s_addr = inet_addr(textual); +} + +static int +sockaddr_to_bh_sockaddr(const struct sockaddr *sockaddr, socklen_t socklen, + bh_sockaddr_t *bh_sockaddr) +{ + switch (sockaddr->sa_family) { + case AF_INET: + { + struct sockaddr_in *addr = (struct sockaddr_in *)sockaddr; + + assert(socklen >= sizeof(struct sockaddr_in)); + + bh_sockaddr->port = ntohs(addr->sin_port); + bh_sockaddr->addr_buffer.ipv4 = ntohl(addr->sin_addr.s_addr); + bh_sockaddr->is_ipv4 = true; + return BHT_OK; + } + default: + errno = EAFNOSUPPORT; + return BHT_ERROR; + } +} + +int +os_socket_create(bh_socket_t *sock, bool is_ipv4, bool is_tcp) +{ + if (!sock) { + return BHT_ERROR; + } + + if (is_tcp) { + *sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + } + else { + *sock = socket(AF_INET, SOCK_DGRAM, 0); + } + + return (*sock == -1) ? BHT_ERROR : BHT_OK; +} + +int +os_socket_bind(bh_socket_t socket, const char *host, int *port) +{ + struct sockaddr_in addr; + socklen_t socklen; + int ret; + + assert(host); + assert(port); + + addr.sin_addr.s_addr = inet_addr(host); + addr.sin_port = htons(*port); + addr.sin_family = AF_INET; + + ret = bind(socket, (struct sockaddr *)&addr, sizeof(addr)); + if (ret < 0) { + goto fail; + } + + socklen = sizeof(addr); + if (getsockname(socket, (struct sockaddr *)&addr, &socklen) == -1) { + goto fail; + } + + *port = ntohs(addr.sin_port); + + return BHT_OK; + +fail: + return BHT_ERROR; +} + +int +os_socket_settimeout(bh_socket_t socket, uint64 timeout_us) +{ + struct timeval tv; + tv.tv_sec = timeout_us / 1000000UL; + tv.tv_usec = timeout_us % 1000000UL; + + if (setsockopt(socket, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, + sizeof(tv)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_listen(bh_socket_t socket, int max_client) +{ + if (listen(socket, max_client) != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, + unsigned int *addrlen) +{ + struct sockaddr addr_tmp; + socklen_t len = sizeof(struct sockaddr); + + *sock = accept(server_sock, (struct sockaddr *)&addr_tmp, &len); + + if (*sock < 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_connect(bh_socket_t socket, const char *addr, int port) +{ + struct sockaddr_in addr_in = { 0 }; + socklen_t addr_len = sizeof(struct sockaddr_in); + int ret = 0; + + textual_addr_to_sockaddr(addr, port, &addr_in); + + ret = connect(socket, (struct sockaddr *)&addr_in, addr_len); + if (ret == -1) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_recv(bh_socket_t socket, void *buf, unsigned int len) +{ + return recv(socket, buf, len, 0); +} + +int +os_socket_send(bh_socket_t socket, const void *buf, unsigned int len) +{ + return send(socket, buf, len, 0); +} + +int +os_socket_close(bh_socket_t socket) +{ + close(socket); + return BHT_OK; +} + +__wasi_errno_t +os_socket_shutdown(bh_socket_t socket) +{ + if (shutdown(socket, O_RDWR) != 0) { + return convert_errno(errno); + } + return __WASI_ESUCCESS; +} + +int +os_socket_inet_network(bool is_ipv4, const char *cp, bh_ip_addr_buffer_t *out) +{ + if (!cp) + return BHT_ERROR; + + if (is_ipv4) { + if (inet_pton(AF_INET, cp, &out->ipv4) != 1) { + return BHT_ERROR; + } + /* Note: ntohl(INADDR_NONE) == INADDR_NONE */ + out->ipv4 = ntohl(out->ipv4); + } + else { + if (inet_pton(AF_INET6, cp, out->ipv6) != 1) { + return BHT_ERROR; + } + for (int i = 0; i < 8; i++) { + out->ipv6[i] = ntohs(out->ipv6[i]); + } + } + + return BHT_OK; +} + +int +os_socket_addr_remote(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + struct sockaddr_in addr; + socklen_t addr_len = sizeof(addr); + + if (getpeername(socket, (struct sockaddr *)&addr, &addr_len) == -1) { + return BHT_ERROR; + } + + return sockaddr_to_bh_sockaddr((struct sockaddr *)&addr, addr_len, + sockaddr); +} + +int +os_socket_addr_local(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + struct sockaddr_in addr; + socklen_t addr_len = sizeof(addr); + + if (getsockname(socket, (struct sockaddr *)&addr, &addr_len) == -1) { + return BHT_ERROR; + } + + return sockaddr_to_bh_sockaddr((struct sockaddr *)&addr, addr_len, + sockaddr); +} diff --git a/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_thread.c b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_thread.c new file mode 100644 index 0000000..637cd41 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/esp-idf/espidf_thread.c @@ -0,0 +1,233 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +typedef struct { + thread_start_routine_t start; + void *arg; +} thread_wrapper_arg; + +static void * +os_thread_wrapper(void *arg) +{ + thread_wrapper_arg *targ = arg; + thread_start_routine_t start_func = targ->start; + void *thread_arg = targ->arg; + +#if 0 + os_printf("THREAD CREATED %jx\n", (uintmax_t)(uintptr_t)pthread_self()); +#endif + BH_FREE(targ); + start_func(thread_arg); + return NULL; +} + +korp_tid +os_self_thread(void) +{ + /* only allowed if this is a thread, xTaskCreate is not enough look at + * product_mini for how to use this*/ + return pthread_self(); +} + +int +os_mutex_init(korp_mutex *mutex) +{ + return pthread_mutex_init(mutex, NULL); +} + +int +os_recursive_mutex_init(korp_mutex *mutex) +{ + int ret; + + pthread_mutexattr_t mattr; + + assert(mutex); + ret = pthread_mutexattr_init(&mattr); + if (ret) + return BHT_ERROR; + + pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_RECURSIVE); + ret = pthread_mutex_init(mutex, &mattr); + pthread_mutexattr_destroy(&mattr); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + return pthread_mutex_destroy(mutex); +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + return pthread_mutex_lock(mutex); +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + return pthread_mutex_unlock(mutex); +} + +int +os_thread_create_with_prio(korp_tid *tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + pthread_attr_t tattr; + thread_wrapper_arg *targ; + + assert(stack_size > 0); + assert(tid); + assert(start); + + pthread_attr_init(&tattr); + pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE); + if (pthread_attr_setstacksize(&tattr, stack_size) != 0) { + os_printf("Invalid thread stack size %u. Min stack size = %u", + stack_size, PTHREAD_STACK_MIN); + pthread_attr_destroy(&tattr); + return BHT_ERROR; + } + + targ = (thread_wrapper_arg *)BH_MALLOC(sizeof(*targ)); + if (!targ) { + pthread_attr_destroy(&tattr); + return BHT_ERROR; + } + + targ->start = start; + targ->arg = arg; + + if (pthread_create(tid, &tattr, os_thread_wrapper, targ) != 0) { + pthread_attr_destroy(&tattr); + os_free(targ); + return BHT_ERROR; + } + + pthread_attr_destroy(&tattr); + return BHT_OK; +} + +int +os_thread_create(korp_tid *tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +int +os_thread_join(korp_tid thread, void **retval) +{ + return pthread_join(thread, retval); +} + +int +os_thread_detach(korp_tid tid) +{ + return pthread_detach(tid); +} + +void +os_thread_exit(void *retval) +{ + pthread_exit(retval); +} + +int +os_cond_init(korp_cond *cond) +{ + return pthread_cond_init(cond, NULL); +} + +int +os_cond_destroy(korp_cond *cond) +{ + return pthread_cond_destroy(cond); +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + return pthread_cond_wait(cond, mutex); +} + +static void +msec_nsec_to_abstime(struct timespec *ts, uint64 usec) +{ + struct timeval tv; + time_t tv_sec_new; + long int tv_nsec_new; + + gettimeofday(&tv, NULL); + + tv_sec_new = (time_t)(tv.tv_sec + usec / 1000000); + if (tv_sec_new >= tv.tv_sec) { + ts->tv_sec = tv_sec_new; + } + else { + /* integer overflow */ + ts->tv_sec = BH_TIME_T_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + + tv_nsec_new = (long int)(tv.tv_usec * 1000 + (usec % 1000000) * 1000); + if (tv.tv_usec * 1000 >= tv.tv_usec && tv_nsec_new >= tv.tv_usec * 1000) { + ts->tv_nsec = tv_nsec_new; + } + else { + /* integer overflow */ + ts->tv_nsec = LONG_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + + if (ts->tv_nsec >= 1000000000L && ts->tv_sec < BH_TIME_T_MAX) { + ts->tv_sec++; + ts->tv_nsec -= 1000000000L; + } +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + int ret; + struct timespec abstime; + + if (useconds == BHT_WAIT_FOREVER) + ret = pthread_cond_wait(cond, mutex); + else { + msec_nsec_to_abstime(&abstime, useconds); + ret = pthread_cond_timedwait(cond, mutex, &abstime); + } + + if (ret != BHT_OK && ret != ETIMEDOUT) + return BHT_ERROR; + + return ret; +} + +int +os_cond_signal(korp_cond *cond) +{ + return pthread_cond_signal(cond); +} + +int +os_cond_broadcast(korp_cond *cond) +{ + return pthread_cond_broadcast(cond); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/esp-idf/platform_internal.h b/wasm-micro-runtime/core/shared/platform/esp-idf/platform_internal.h new file mode 100644 index 0000000..0f87381 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/esp-idf/platform_internal.h @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "esp_pthread.h" +#include "esp_timer.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_ESP_IDF +#define BH_PLATFORM_ESP_IDF +#endif + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef unsigned int korp_sem; + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 5 + +/* Special value for tv_nsec field of timespec */ + +#define UTIME_NOW ((1l << 30) - 1l) +#ifndef __cplusplus +#define UTIME_OMIT ((1l << 30) - 2l) +#endif + +#ifdef DT_UNKNOWN +#undef DT_UNKNOWN +#endif + +#ifdef DT_REG +#undef DT_REG +#endif + +#ifdef DT_DIR +#undef DT_DIR +#endif + +/* Below parts of d_type define are ported from Nuttx, under Apache License v2.0 + */ + +/* File type code for the d_type field in dirent structure. + * Note that because of the simplified filesystem organization of the NuttX, + * top-level, pseudo-file system, an inode can be BOTH a file and a directory + */ + +#define DTYPE_UNKNOWN 0 +#define DTYPE_FIFO 1 +#define DTYPE_CHR 2 +#define DTYPE_SEM 3 +#define DTYPE_DIRECTORY 4 +#define DTYPE_MQ 5 +#define DTYPE_BLK 6 +#define DTYPE_SHM 7 +#define DTYPE_FILE 8 +#define DTYPE_MTD 9 +#define DTYPE_LINK 10 +#define DTYPE_SOCK 12 + +/* The d_type field of the dirent structure is not specified by POSIX. It + * is a non-standard, 4.5BSD extension that is implemented by most OSs. A + * POSIX compliant OS may not implement the d_type field at all. Many OS's + * (including glibc) may use the following alternative naming for the file + * type names: + */ + +#define DT_UNKNOWN DTYPE_UNKNOWN +#define DT_FIFO DTYPE_FIFO +#define DT_CHR DTYPE_CHR +#define DT_SEM DTYPE_SEM +#define DT_DIR DTYPE_DIRECTORY +#define DT_MQ DTYPE_MQ +#define DT_BLK DTYPE_BLK +#define DT_SHM DTYPE_SHM +#define DT_REG DTYPE_FILE +#define DT_MTD DTYPE_MTD +#define DT_LNK DTYPE_LINK +#define DT_SOCK DTYPE_SOCK + +static inline int +os_getpagesize() +{ + return 4096; +} + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/esp-idf/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/esp-idf/shared_platform.cmake new file mode 100644 index 0000000..d254b08 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/esp-idf/shared_platform.cmake @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_ESP_IDF) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/libc-util/platform_common_libc_util.cmake) +set (source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE}) + +# If enable PSRAM of ESP32-S3, it had better to put AOT into PSRAM, so that +# users can use SRAM to for Wi-Fi/BLE and peripheral driver. +if(CONFIG_ESP32S3_SPIRAM_SUPPORT) + add_definitions(-DWASM_MEM_DUAL_BUS_MIRROR=1) +endif() diff --git a/wasm-micro-runtime/core/shared/platform/freebsd/platform_init.c b/wasm-micro-runtime/core/shared/platform/freebsd/platform_init.c new file mode 100644 index 0000000..2aae13f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/freebsd/platform_init.c @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +int +bh_platform_init() +{ + return 0; +} + +void +bh_platform_destroy() +{} + +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} diff --git a/wasm-micro-runtime/core/shared/platform/freebsd/platform_internal.h b/wasm-micro-runtime/core/shared/platform/freebsd/platform_internal.h new file mode 100644 index 0000000..bfdfe14 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/freebsd/platform_internal.h @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_FREEBSD +#define BH_PLATFORM_FREEBSD +#endif + +#define BH_HAS_DLFCN 1 + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (32 * 1024) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef sem_t korp_sem; + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define os_thread_local_attribute __thread + +#define bh_socket_t int + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + +#include + +#define OS_ENABLE_HW_BOUND_CHECK + +typedef jmp_buf korp_jmpbuf; + +#define os_setjmp setjmp +#define os_longjmp longjmp +#define os_alloca alloca + +typedef void (*os_signal_handler)(void *sig_addr); + +int +os_thread_signal_init(os_signal_handler handler); + +void +os_thread_signal_destroy(); + +bool +os_thread_signal_inited(); + +void +os_signal_unmask(); + +void +os_sigreturn(); +#endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ +#endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ + +#define os_getpagesize getpagesize + +#if WASM_DISABLE_WAKEUP_BLOCKING_OP == 0 +#define OS_ENABLE_WAKEUP_BLOCKING_OP +#endif +void +os_set_signal_number_for_blocking_op(int signo); + +typedef int os_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/freebsd/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/freebsd/shared_platform.cmake new file mode 100644 index 0000000..12583fc --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/freebsd/shared_platform.cmake @@ -0,0 +1,18 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_FREEBSD) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE}) + +file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) +LIST (APPEND RUNTIME_LIB_HEADER_LIST ${header}) diff --git a/wasm-micro-runtime/core/shared/platform/include/platform_api_extension.h b/wasm-micro-runtime/core/shared/platform/include/platform_api_extension.h new file mode 100644 index 0000000..7c6120b --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/include/platform_api_extension.h @@ -0,0 +1,1639 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef PLATFORM_API_EXTENSION_H +#define PLATFORM_API_EXTENSION_H + +#include "platform_common.h" +#include "platform_wasi_types.h" +/** + * The related data structures should be defined + * in platform_internal.h + **/ +#include "platform_internal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*************************************************** + * * + * Extension interface * + * * + ***************************************************/ + +/**************************************************** + * Section 1 * + * Multi thread support * + ****************************************************/ + +/** + * NOTES: + * 1. If you are building VM core only, it must be implemented to + * enable multi-thread support, otherwise no need to implement it + * 2. To build the app-mgr and app-framework, you must implement it + */ + +/** + * Creates a thread + * + * @param p_tid [OUTPUT] the pointer of tid + * @param start main routine of the thread + * @param arg argument passed to main routine + * @param stack_size bytes of stack size + * + * @return 0 if success. + */ +int +os_thread_create(korp_tid *p_tid, thread_start_routine_t start, void *arg, + unsigned int stack_size); + +/** + * Creates a thread with priority + * + * @param p_tid [OUTPUT] the pointer of tid + * @param start main routine of the thread + * @param arg argument passed to main routine + * @param stack_size bytes of stack size + * @param prio the priority + * + * @return 0 if success. + */ +int +os_thread_create_with_prio(korp_tid *p_tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio); + +/** + * Waits for the thread specified by thread to terminate + * + * @param thread the thread to wait + * @param retval if not NULL, output the exit status of the terminated thread + * + * @return return 0 if success + */ +int +os_thread_join(korp_tid thread, void **retval); + +/** + * Detach the thread specified by thread + * + * @param thread the thread to detach + * + * @return return 0 if success + */ +int os_thread_detach(korp_tid); + +/** + * Exit current thread + * + * @param retval the return value of the current thread + */ +void +os_thread_exit(void *retval); + +/* Try to define os_atomic_thread_fence if it isn't defined in + platform's platform_internal.h */ +#ifndef os_atomic_thread_fence + +#if !defined(__GNUC_PREREQ) && (defined(__GNUC__) || defined(__GNUG__)) \ + && !defined(__clang__) && defined(__GNUC_MINOR__) +#define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#endif + +/* Clang's __GNUC_PREREQ macro has a different meaning than GCC one, + so we have to handle this case specially */ +#if defined(__clang__) +/* Clang provides stdatomic.h since 3.6.0 + See https://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html */ +#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) +#define BH_HAS_STD_ATOMIC +#endif +#elif defined(__GNUC_PREREQ) +/* Even though older versions of GCC support C11, atomics were + not implemented until 4.9. See + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58016 */ +#if __GNUC_PREREQ(4, 9) +#define BH_HAS_STD_ATOMIC +#elif __GNUC_PREREQ(4, 7) +#define os_memory_order_acquire __ATOMIC_ACQUIRE +#define os_memory_order_release __ATOMIC_RELEASE +#define os_memory_order_seq_cst __ATOMIC_SEQ_CST +#define os_atomic_thread_fence __atomic_thread_fence +#endif /* end of __GNUC_PREREQ(4, 9) */ +#endif /* end of defined(__GNUC_PREREQ) */ + +#if defined(BH_HAS_STD_ATOMIC) && !defined(__cplusplus) +#include +#define os_memory_order_acquire memory_order_acquire +#define os_memory_order_release memory_order_release +#define os_memory_order_seq_cst memory_order_seq_cst +#define os_atomic_thread_fence atomic_thread_fence +#define os_atomic_cmpxchg atomic_compare_exchange_strong +#endif + +#endif /* end of os_atomic_thread_fence */ + +/** + * Initialize current thread environment if current thread + * is created by developer but not runtime + * + * @return 0 if success, -1 otherwise + */ +int +os_thread_env_init(void); + +/** + * Destroy current thread environment + */ +void +os_thread_env_destroy(void); + +/** + * Whether the thread environment is initialized + */ +bool +os_thread_env_inited(void); + +/** + * Suspend execution of the calling thread for (at least) + * usec microseconds + * + * @return 0 if success, -1 otherwise + */ +int +os_usleep(uint32 usec); + +/** + * Creates a recursive mutex + * + * @param mutex [OUTPUT] pointer to mutex initialized. + * + * @return 0 if success + */ +int +os_recursive_mutex_init(korp_mutex *mutex); + +/** + * This function creates a condition variable + * + * @param cond [OUTPUT] pointer to condition variable + * + * @return 0 if success + */ +int +os_cond_init(korp_cond *cond); + +/** + * This function destroys condition variable + * + * @param cond pointer to condition variable + * + * @return 0 if success + */ +int +os_cond_destroy(korp_cond *cond); + +/** + * Wait a condition variable. + * + * @param cond pointer to condition variable + * @param mutex pointer to mutex to protect the condition variable + * + * @return 0 if success + */ +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex); + +/** + * Wait a condition varible or return if time specified passes. + * + * @param cond pointer to condition variable + * @param mutex pointer to mutex to protect the condition variable + * @param useconds microseconds to wait + * + * @return 0 if success + */ +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds); + +/** + * Signals the condition variable + * + * @param cond condition variable + * + * @return 0 if success + */ +int +os_cond_signal(korp_cond *cond); + +/** + * Broadcast the condition variable + * + * @param cond condition variable + * + * @return 0 if success + */ +int +os_cond_broadcast(korp_cond *cond); + +/** + * Initialize readwrite lock object + * + * @param cond [OUTPUT] pointer to a lock object variable + * + * @return 0 if success + */ +int +os_rwlock_init(korp_rwlock *lock); + +/** + * Acquire the read lock + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_rdlock(korp_rwlock *lock); + +/** + * Acquire the write lock + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_wrlock(korp_rwlock *lock); + +/** + * Unlocks the lock object + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_unlock(korp_rwlock *lock); + +/** + * Destroy a lock object + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_destroy(korp_rwlock *lock); + +/** + * Creates a new POSIX-like semaphore or opens an existing + * semaphore. The semaphore is identified by name. For details of + * the construction of name, please refer to + * https://man7.org/linux/man-pages/man3/sem_open.3.html. + * + * @param name semaphore name + * @param oflasg specifies flags that control the operation of the call + * @param mode permission flags + * @param val initial value of the named semaphore. + * + * @return korp_sem * if success, NULL otherwise + */ +korp_sem * +os_sem_open(const char *name, int oflags, int mode, int val); + +/** + * Closes the named semaphore referred to by sem, + * allowing any resources that the system has allocated to the + * calling process for this semaphore to be freed. + * + * @param sem + * + * @return 0 if success + */ +int +os_sem_close(korp_sem *sem); + +/** + * Decrements (locks) the semaphore pointed to by sem. + * If the semaphore's value is greater than zero, then the decrement + * proceeds, and the function returns, immediately. If the + * semaphore currently has the value zero, then the call blocks + * until either it becomes possible to perform the decrement (i.e., + * the semaphore value rises above zero), or a signal handler + * interrupts the call. + * + * @return 0 if success + */ +int +os_sem_wait(korp_sem *sem); + +/** + * Is the same as sem_wait(), except that if the + * decrement cannot be immediately performed, then call returns an + * error (errno set to EAGAIN) instead of blocking. + * + * @return 0 if success + */ +int +os_sem_trywait(korp_sem *sem); + +/** + * Increments (unlocks) the semaphore pointed to by sem. + * If the semaphore's value consequently becomes greater than zero, + * then another process or thread blocked in a sem_wait(3) call will + * be woken up and proceed to lock the semaphore. + * + * @return 0 if success + */ +int +os_sem_post(korp_sem *sem); + +/** + * Places the current value of the semaphore pointed + * to sem into the integer pointed to by sval. + * + * @return 0 if success + */ +int +os_sem_getvalue(korp_sem *sem, int *sval); + +/** + * Remove the named semaphore referred to by name. + * The semaphore name is removed immediately. The semaphore is + * destroyed once all other processes that have the semaphore open + * close it. + * + * @param name semaphore name + * + * @return 0 if success + */ +int +os_sem_unlink(const char *name); + +/** + * Initialize process-global state for os_wakeup_blocking_op. + */ +int +os_blocking_op_init(); + +/** + * Start accepting os_wakeup_blocking_op requests for the calling thread. + */ +void +os_begin_blocking_op(); + +/** + * Stop accepting os_wakeup_blocking_op requests for the calling thread. + */ +void +os_end_blocking_op(); + +/** + * Wake up the specified thread. + * + * For example, on posix-like platforms, this can be implemented by + * sending a signal (w/o SA_RESTART) which interrupts a blocking + * system call. + */ +int +os_wakeup_blocking_op(korp_tid tid); + +/**************************************************** + * Section 2 * + * Socket support * + ****************************************************/ + +/** + * NOTES: + * Socket APIs are required by source debugging feature. + * If you don't need source debugging feature, then no + * need to implement these APIs + */ + +typedef union { + uint32 ipv4; + uint16 ipv6[8]; + uint8 data[1]; +} bh_ip_addr_buffer_t; + +typedef struct { + bh_ip_addr_buffer_t addr_buffer; + uint16 port; + bool is_ipv4; +} bh_sockaddr_t; + +/** + * Create a socket + * + * @param sock [OUTPUT] the pointer of socket + * @param is_ipv4 true for IPv4, false for IPv6 + * @param is_tcp true for tcp, false for udp + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_create(bh_socket_t *sock, bool is_ipv4, bool is_tcp); + +/** + * Assign the address and port to the socket + * + * @param socket the socket to bind + * @param addr the ip address, only IPv4 supported currently + * @param port [INPUT/OUTPUT] the port number, if the value is 0, + * it will use a port assigned by OS. On return it will + * contain the actual bound port number + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_bind(bh_socket_t socket, const char *addr, int *port); + +/** + * Set timeout for the given socket + * + * @param socket the socket to set timeout + * @param timeout_us timeout in microseconds + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_settimeout(bh_socket_t socket, uint64 timeout_us); + +/** + * Make the socket as a passive socket to accept incoming connection requests + * + * @param socket the socket to listen + * @param max_client maximum clients + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_listen(bh_socket_t socket, int max_client); + +/** + * Accept an incoming connection + * + * @param server_sock the socket to accept new connections + * @param sock [OUTPUT] the connected socket + * @param addr [OUTPUT] the address of the peer socket. If addr is NULL, + * nothing is filled in, and addrlen will not be used + * @param addrlen [INPUT/OUTPUT] the size (in bytes) of the structure + * pointed to by addr, on return it will contain the actual + * size of the peer address + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, + unsigned int *addrlen); + +/** + * initiate a connection on a socket + * + * @param socket the socket to connect with + * @param addr the ip address, only IPv4 supported currently + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_connect(bh_socket_t socket, const char *addr, int port); + +/** + * Blocking receive message from a socket. + * + * @param socket the socket to receive message from + * @param buf the buffer to store the data + * @param len length of the buffer, this API does not guarantee that + * [len] bytes are received + * + * @return number of bytes received if success, -1 otherwise + */ +int +os_socket_recv(bh_socket_t socket, void *buf, unsigned int len); + +/** + * Blocking receive message from a socket. + * + * @param socket the socket to send message + * @param buf the buffer to store the data + * @param len length of the buffer, this API does not guarantee that + * [len] bytes are received + * @param flags control the operation + * @param src_addr source address + * + * @return number of bytes sent if success, -1 otherwise + */ +int +os_socket_recv_from(bh_socket_t socket, void *buf, unsigned int len, int flags, + bh_sockaddr_t *src_addr); + +/** + * Blocking send message on a socket + * + * @param socket the socket to send message + * @param buf the buffer of data to be sent + * @param len length of the buffer + * + * @return number of bytes sent if success, -1 otherwise + */ +int +os_socket_send(bh_socket_t socket, const void *buf, unsigned int len); + +/** + * Blocking send message on a socket to the target address + * + * @param socket the socket to send message + * @param buf the buffer of data to be sent + * @param len length of the buffer + * @param flags control the operation + * @param dest_addr target address + * + * @return number of bytes sent if success, -1 otherwise + */ +int +os_socket_send_to(bh_socket_t socket, const void *buf, unsigned int len, + int flags, const bh_sockaddr_t *dest_addr); + +/** + * Close a socket + * + * @param socket the socket to be closed + * + * @return always return 0 + */ +int +os_socket_close(bh_socket_t socket); + +/** + * Shutdown a socket + * + * @param socket the socket to be shutdown + * + * @return returns corresponding error code + */ +__wasi_errno_t +os_socket_shutdown(bh_socket_t socket); + +/** + * converts cp into a number in host byte order suitable for use as + * an Internet network address + * + * @param is_ipv4 a flag that indicates whether the string is an IPv4 or + * IPv6 address + * + * @param cp a string in IPv4 numbers-and-dots notation or IPv6 + * numbers-and-colons notation + * + * @param out an output buffer to store binary address + * + * @return On success, the function returns 0. + * If the input is invalid, -1 is returned + */ +int +os_socket_inet_network(bool is_ipv4, const char *cp, bh_ip_addr_buffer_t *out); + +typedef struct { + bh_sockaddr_t sockaddr; + uint8_t is_tcp; +} bh_addr_info_t; + +/** + * Resolve a host a hostname and a service to one or more IP addresses + * + * @param host a host to resolve + * + * @param service a service to find a port for + * + * @param hint_is_tcp an optional flag that determines a preferred socket type + (TCP or UDP). + * + * @param hint_is_ipv4 an optional flag that determines a preferred address + family (IPv4 or IPv6) + * + * @param addr_info a buffer for resolved addresses + * + * @param addr_info_size a size of the buffer for resolved addresses + + * @param max_info_size a maximum number of addresses available (can be bigger + or smaller than buffer size) + + * @return On success, the function returns 0; otherwise, it returns -1 + */ +int +os_socket_addr_resolve(const char *host, const char *service, + uint8_t *hint_is_tcp, uint8_t *hint_is_ipv4, + bh_addr_info_t *addr_info, size_t addr_info_size, + size_t *max_info_size); + +/** + * Returns an binary address and a port of the local socket + * + * @param socket the local socket + * + * @param sockaddr a buffer for storing the address + * + * @return On success, returns 0; otherwise, it returns -1. + */ +int +os_socket_addr_local(bh_socket_t socket, bh_sockaddr_t *sockaddr); + +/** + * Returns an binary address and a port of the remote socket + * + * @param socket the remote socket + * + * @param sockaddr a buffer for storing the address + * + * @return On success, returns 0; otherwise, it returns -1. + */ +int +os_socket_addr_remote(bh_socket_t socket, bh_sockaddr_t *sockaddr); + +/** + * Set the maximum send buffer size. + * + * @param socket the socket to set + * @param bufsiz requested kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_send_buf_size(bh_socket_t socket, size_t bufsiz); + +/** + * Get the maximum send buffer size. + * + * @param socket the socket to set + * @param bufsiz the returned kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_send_buf_size(bh_socket_t socket, size_t *bufsiz); + +/** + * Set the maximum receive buffer size. + * + * @param socket the socket to set + * @param bufsiz requested kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_recv_buf_size(bh_socket_t socket, size_t bufsiz); + +/** + * Get the maximum receive buffer size. + * + * @param socket the socket to set + * @param bufsiz the returned kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_recv_buf_size(bh_socket_t socket, size_t *bufsiz); + +/** + * Enable sending of keep-alive messages on connection-oriented sockets + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_keep_alive(bh_socket_t socket, bool is_enabled); + +/** + * Get if sending of keep-alive messages on connection-oriented sockets is + * enabled + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_keep_alive(bh_socket_t socket, bool *is_enabled); + +/** + * Set the send timeout until reporting an error + * + * @param socket the socket to set + * @param time_us microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_send_timeout(bh_socket_t socket, uint64 timeout_us); + +/** + * Get the send timeout until reporting an error + * + * @param socket the socket to set + * @param time_us the returned microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_send_timeout(bh_socket_t socket, uint64 *timeout_us); + +/** + * Set the recv timeout until reporting an error + * + * @param socket the socket to set + * @param time_us microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_recv_timeout(bh_socket_t socket, uint64 timeout_us); + +/** + * Get the recv timeout until reporting an error + * + * @param socket the socket to set + * @param time_us the returned microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_recv_timeout(bh_socket_t socket, uint64 *timeout_us); + +/** + * Enable re-use of local addresses + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_reuse_addr(bh_socket_t socket, bool is_enabled); + +/** + * Get whether re-use of local addresses is enabled + * + * @param socket the socket to set + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_reuse_addr(bh_socket_t socket, bool *is_enabled); + +/** + * Enable re-use of local ports + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_reuse_port(bh_socket_t socket, bool is_enabled); + +/** + * Get whether re-use of local ports is enabled + * + * @param socket the socket to set + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_reuse_port(bh_socket_t socket, bool *is_enabled); + +/** + * Set the linger options for the given socket + * + * @param socket the socket to set + * @param is_enabled whether linger is enabled + * @param linger_s linger time (seconds) + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_linger(bh_socket_t socket, bool is_enabled, int linger_s); + +/** + * Get the linger options for the given socket + * + * @param socket the socket to get + * @param is_enabled whether linger is enabled + * @param linger_s linger time (seconds) + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_linger(bh_socket_t socket, bool *is_enabled, int *linger_s); + +/** + * Set no delay TCP + * If set, disable the Nagle algorithm. + * This means that segments are always sent as soon as possible, + * even if there is only a small amount of data + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_no_delay(bh_socket_t socket, bool is_enabled); + +/** + * Get no delay TCP + * If set, disable the Nagle algorithm. + * This means that segments are always sent as soon as possible, + * even if there is only a small amount of data + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_no_delay(bh_socket_t socket, bool *is_enabled); + +/** + * Enable/Disable tcp quickack mode + * In quickack mode, acks are sent immediately, rather than delayed if needed in + * accordance to normal TCP operation + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_quick_ack(bh_socket_t socket, bool is_enabled); + +/** + * Enable/Disable tcp quickack mode + * In quickack mode, acks are sent immediately, rather than delayed if needed in + * accordance to normal TCP operation + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_quick_ack(bh_socket_t socket, bool *is_enabled); + +/** + * Set the time the connection needs to remain idle before sending keepalive + * probes + * + * @param socket the socket to set + * @param time_s seconds until keepalive probes are sent + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_keep_idle(bh_socket_t socket, uint32_t time_s); + +/** + * Gets the time the connection needs to remain idle before sending keepalive + * probes + * + * @param socket the socket to check + * @param time_s seconds until keepalive probes are sent + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_keep_idle(bh_socket_t socket, uint32_t *time_s); + +/** + * Set the time between individual keepalive probes + * + * @param socket the socket to set + * @param time_us seconds between individual keepalive probes + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_keep_intvl(bh_socket_t socket, uint32_t time_s); + +/** + * Get the time between individual keepalive probes + * + * @param socket the socket to get + * @param time_s seconds between individual keepalive probes + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_keep_intvl(bh_socket_t socket, uint32_t *time_s); + +/** + * Set use of TCP Fast Open + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_fastopen_connect(bh_socket_t socket, bool is_enabled); + +/** + * Get whether use of TCP Fast Open is enabled + * + * @param socket the socket to get + * @param is_enabled 1 to enabled or 0 to disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_fastopen_connect(bh_socket_t socket, bool *is_enabled); + +/** + * Set enable or disable IPv4 or IPv6 multicast loopback. + * + * @param socket the socket to set + * @param ipv6 true to set ipv6 loopback or false for ipv4 + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool is_enabled); + +/** + * Get enable or disable IPv4 or IPv6 multicast loopback. + * + * @param socket the socket to check + * @param ipv6 true to set ipv6 loopback or false for ipv4 + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ip_multicast_loop(bh_socket_t socket, bool ipv6, + bool *is_enabled); + +/** + * Add membership to a group + * + * @param socket the socket to add membership to + * @param imr_multiaddr the group multicast address (IPv4 or IPv6) + * @param imr_interface the interface to join on + * @param is_ipv6 whether the imr_multiaddr is IPv4 or IPv6 (true for IPv6) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_add_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6); + +/** + * Drop membership of a group + * + * @param socket the socket to drop membership to + * @param imr_multiaddr the group multicast address (IPv4 or IPv6) + * @param imr_interface the interface to join on + * @param is_ipv6 whether the imr_multiaddr is IPv4 or IPv6 (true for IPv6) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_drop_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6); + +/** + * Set the current time-to-live field that is + * used in every packet sent from this socket. + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_ttl(bh_socket_t socket, uint8_t ttl_s); + +/** + * Retrieve the current time-to-live field that is + * used in every packet sent from this socket. + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ip_ttl(bh_socket_t socket, uint8_t *ttl_s); + +/** + * Set the time-to-live value of outgoing multicast + * packets for this socket + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_multicast_ttl(bh_socket_t socket, uint8_t ttl_s); + +/** + * Read the time-to-live value of outgoing multicast + * packets for this socket + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ip_multicast_ttl(bh_socket_t socket, uint8_t *ttl_s); + +/** + * Restrict to sending and receiving IPv6 packets only + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ipv6_only(bh_socket_t socket, bool is_enabled); + +/** + * Get whether only sending and receiving IPv6 packets + * + * @param socket the socket to check + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ipv6_only(bh_socket_t socket, bool *is_enabled); + +/** + * Set whether broadcast is enabled + * When enabled, datagram sockets are allowed + * to send packets to a broadcast address. + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_broadcast(bh_socket_t socket, bool is_enabled); + +/** + * Get whether broadcast is enabled + * When enabled, datagram sockets are allowed + * to send packets to a broadcast address. + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_broadcast(bh_socket_t socket, bool *is_enabled); + +/** + * Dump memory information of the current process + * It may have variant implementations in different platforms + * + * @param out the output buffer. It is for sure the return content + * is a c-string which ends up with '\0' + * @param size the size of the output buffer + * + * @return 0 if success, -1 otherwise + */ +int +os_dumps_proc_mem_info(char *out, unsigned int size); + +/**************************************************** + * Section 3 * + * Filesystem support * + ****************************************************/ + +/** + * NOTES: + * Fileystem APIs are required for WASI libc support. If you don't need to + * support WASI libc, there is no need to implement these APIs. With a + * few exceptions, each filesystem function has been named after the equivalent + * POSIX filesystem function with an os_ prefix. + * + * Filesystem types + * + * os_raw_file_handle: the underlying OS file handle type e.g. int on POSIX + * systems and HANDLE on Windows. This type exists to allow embedders to provide + * custom file handles for stdout/stdin/stderr. + * + * os_file_handle: the file handle type used in the WASI libc fd + * table. Filesystem implementations can use it as a means to store any + * necessary platform-specific information which may not be directly available + * through the raw OS file handle. Similiar to POSIX file descriptors, file + * handles may also refer to sockets, directories, symbolic links or character + * devices and any of the filesystem operations which make sense for these + * resource types should be supported as far as possible. + * + * os_dir_stream: a directory stream type in which fileystem implementations + * can store any necessary state to iterate over the entries in a directory. + */ + +/** + * Obtain information about an open file associated with the given handle. + * + * @param handle the handle for which to obtain file information + * @param buf a buffer in which to store the information + */ +__wasi_errno_t +os_fstat(os_file_handle handle, struct __wasi_filestat_t *buf); + +/** + * Obtain information about an open file or directory. + * @param handle the directory handle from which to resolve the file/directory + * path + * @param path the relative path of the file or directory for which to obtain + * information + * @param buf a buffer in which to store the information + * @param follow_symlink whether to follow symlinks when resolving the path + */ +__wasi_errno_t +os_fstatat(os_file_handle handle, const char *path, + struct __wasi_filestat_t *buf, __wasi_lookupflags_t lookup_flags); + +/** + * Obtain the file status flags for the provided handle. This is similiar to the + * POSIX function fcntl called with the F_GETFL command. + * + * @param handle the handle for which to obtain the file status flags + * @param flags a pointer in which to store the output + */ +__wasi_errno_t +os_file_get_fdflags(os_file_handle handle, __wasi_fdflags_t *flags); + +/** + * Set the file status flags for the provided handle. This is similiar to the + * POSIX function fcntl called with the F_SETFL command. + * + * @param handle the handle for which to set the file status flags + * @param flags the flags to set + */ +__wasi_errno_t +os_file_set_fdflags(os_file_handle handle, __wasi_fdflags_t flags); + +/** + * Synchronize the data of a file to disk. + * + * @param handle + */ +__wasi_errno_t +os_fdatasync(os_file_handle handle); + +/** + * Synchronize the data and metadata of a file to disk. + * + * @param handle + */ +__wasi_errno_t +os_fsync(os_file_handle handle); + +/** + * Open a preopen directory. The path provided must refer to a directory and the + * returned handle will allow only readonly operations. + * + * @param path the path of the preopen directory to open + * @param out a pointer in which to store the newly opened handle + */ +__wasi_errno_t +os_open_preopendir(const char *path, os_file_handle *out); + +typedef uint8 wasi_libc_file_access_mode; +#define WASI_LIBC_ACCESS_MODE_READ_ONLY 0 +#define WASI_LIBC_ACCESS_MODE_WRITE_ONLY 1 +#define WASI_LIBC_ACCESS_MODE_READ_WRITE 2 + +/** + * Open a file or directory at the given path. + * + * @param handle a handle to the directory in which to open the new file or + * directory + * @param path the relative path of the file or directory to open + * @param oflags the flags to determine how the file or directory is opened + * @param fd_flags the flags to set on the returned handle + * @param lookup_flags whether to follow symlinks when resolving the path + * @param access_mode whether the file is opened as read only, write only or + * both + * @param out a pointer in which to store the newly opened handle + */ +__wasi_errno_t +os_openat(os_file_handle handle, const char *path, __wasi_oflags_t oflags, + __wasi_fdflags_t fd_flags, __wasi_lookupflags_t lookup_flags, + wasi_libc_file_access_mode access_mode, os_file_handle *out); + +/** + * Obtain the file access mode for the provided handle. This is similiar to the + * POSIX function fcntl called with the F_GETFL command combined with the + * O_ACCMODE mask. + * + * @param handle the handle for which to obtain the access mode + * @param access_mode a pointer in which to store the access mode + */ +__wasi_errno_t +os_file_get_access_mode(os_file_handle handle, + wasi_libc_file_access_mode *access_mode); + +/** + * Close the provided handle. If is_stdio is true, the raw file handle + * associated with the given file handle will not be closed. + * + * @param handle the handle to close + * @param is_stdio whether the provided handle refers to a stdio device + */ +__wasi_errno_t +os_close(os_file_handle handle, bool is_stdio); + +/** + * Read data from the provided handle at the given offset into multiple buffers. + * + * @param handle the handle to read from + * @param iov the buffers to read into + * @param iovcnt the number of buffers to read into + * @param offset the offset to read from + * @param nread a pointer in which to store the number of bytes read + */ +__wasi_errno_t +os_preadv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nread); + +/** + * Write data from multiple buffers at the given offset to the provided handle. + * + * @param handle the handle to write to + * @param iov the buffers to write from + * @param iovcnt the number of buffers to write from + * @param offset the offset to write from + * @param nwritten a pointer in which to store the number of bytes written + */ +__wasi_errno_t +os_pwritev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nwritten); + +/** + * Read data from the provided handle into multiple buffers. + * + * @param handle the handle to read from + * @param iov the buffers to read into + * @param iovcnt the number of buffers to read into + * @param nread a pointer in which to store the number of bytes read + */ +__wasi_errno_t +os_readv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + size_t *nread); + +/** + * Write data from multiple buffers to the provided handle. + * + * @param handle the handle to write to + * @param iov the buffers to write from + * @param iovcnt the number of buffers to write from + * @param nwritten a pointer in which to store the number of bytes written + */ +__wasi_errno_t +os_writev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + size_t *nwritten); + +/** + * Allocate storage space for the file associated with the provided handle. This + * is similar to the POSIX function posix_fallocate. + * + * @param handle the handle to allocate space for + * @param offset the offset to allocate space at + * @param length the amount of space to allocate + */ +__wasi_errno_t +os_fallocate(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length); + +/** + * Adjust the size of an open file. + * + * @param handle the associated file handle for which to adjust the size + * @param size the new size of the file + */ +__wasi_errno_t +os_ftruncate(os_file_handle handle, __wasi_filesize_t size); + +/** + * Set file access and modification times on an open file or directory. + * + * @param handle the associated file handle for which to adjust the + * access/modification times + * @param access_time the timestamp for the new access time + * @param modification_time the timestamp for the new modification time + * @param fstflags a bitmask to indicate which timestamps to adjust + */ +__wasi_errno_t +os_futimens(os_file_handle handle, __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags); + +/** + * Set file access and modification times on an open file or directory. + * + * @param handle the directory handle from which to resolve the path + * @param path the relative path of the file or directory for which to adjust + * the access/modification times + * @param access_time the timestamp for the new access time + * @param modification_time the timestamp for the new modification time + * @param fstflags a bitmask to indicate which timestamps to adjust + * @param lookup_flags whether to follow symlinks when resolving the path + */ +__wasi_errno_t +os_utimensat(os_file_handle handle, const char *path, + __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags, + __wasi_lookupflags_t lookup_flags); + +/** + * Read the contents of a symbolic link relative to the provided directory + * handle. + * + * @param handle the directory handle + * @param path the relative path of the symbolic link from which to read + * @param buf the buffer to read the link contents into + * @param bufsize the size of the provided buffer + * @param nread a pointer in which to store the number of bytes read into the + * buffer + */ +__wasi_errno_t +os_readlinkat(os_file_handle handle, const char *path, char *buf, + size_t bufsize, size_t *nread); + +/** + * Create a link from one path to another path. + * + * @param from_handle the directory handle from which to resolve the origin path + * @param from_path the origin path to link from + * @param to_handle the directory handle from which to resolve the destination + * path + * @param to_path the destination path at which to create the link + * @param lookup_flags whether to follow symlinks when resolving the origin path + */ +__wasi_errno_t +os_linkat(os_file_handle from_handle, const char *from_path, + os_file_handle to_handle, const char *to_path, + __wasi_lookupflags_t lookup_flags); + +/** + * Create a symbolic link from one path to another path. + * + * @param old_path the symbolic link contents + * @param handle the directory handle from which to resolve the destination path + * @param new_path the destination path at which to create the symbolic link + */ +__wasi_errno_t +os_symlinkat(const char *old_path, os_file_handle handle, const char *new_path); + +/** + * Create a directory relative to the provided directory handle. + * + * @param handle the directory handle + * @param path the relative path of the directory to create + */ +__wasi_errno_t +os_mkdirat(os_file_handle handle, const char *path); + +/** + * Rename a file or directory. + * + * @param old_handle the directory handle from which to resolve the old path + * @param old_path the source path to rename + * @param new_handle the directory handle from which to resolve the destination + * path + * @param new_path the destination path to which to rename the file or directory + */ +__wasi_errno_t +os_renameat(os_file_handle old_handle, const char *old_path, + os_file_handle new_handle, const char *new_path); + +/** + * Unlink a file or directory. + * + * @param handle the directory handle from which to resolve the path + * @param path the relative path of the file or directory to unlink + * @param is_dir whether the provided handle refers to a directory or file + */ +__wasi_errno_t +os_unlinkat(os_file_handle handle, const char *path, bool is_dir); + +/** + * Move the read/write offset of an open file. + * + * @param handle the associated file handle for which to adjust the offset + * @param offset the number of bytes to adjust the offset by + * @param whence the position whence to adjust the offset + * @param new_offset a pointer in which to store the new offset + */ +__wasi_errno_t +os_lseek(os_file_handle handle, __wasi_filedelta_t offset, + __wasi_whence_t whence, __wasi_filesize_t *new_offset); + +/** + * Provide file advisory information for the given handle. This is similar to + * the POSIX function posix_fadvise. + * + * @param handle the associated file handle for which to provide advisory + * information + * @param offset the offset within the file to which the advisory + * information applies + * @param length the length of the region for which the advisory information + * applies + * @param advice the advice to provide + */ +__wasi_errno_t +os_fadvise(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length, __wasi_advice_t advice); + +/** + * Determine if the given handle refers to a terminal device. __WASI_ESUCCESS + * will be returned if the handle is associated with a terminal device, + * otherwise an appropriate error code will be returned. + * + * @param handle + */ +__wasi_errno_t +os_isatty(os_file_handle handle); + +/** + * Converts a raw file handle to STDIN to a corresponding file handle to STDIN. + * If the provided raw file handle is invalid, the platform-default raw handle + * for STDIN will be used. + * + * @param raw_stdin a raw file handle to STDIN + * + * @return a handle to STDIN + */ +os_file_handle +os_convert_stdin_handle(os_raw_file_handle raw_stdin); + +/** + * Converts a raw file handle to STDOUT to a correponding file handle to STDOUT. + * If the provided raw file handle is invalid, the platform-default raw handle + * for STDOUT will be used. + * + * @param raw_stdout a raw file handle to STDOUT + * + * @return a handle to STDOUT + */ +os_file_handle +os_convert_stdout_handle(os_raw_file_handle raw_stdout); + +/** + * Converts a raw file handle to STDERR to a correponding file handle to STDERR. + * If the provided raw file handle is invalid, the platform-default raw handle + * for STDERR will be used. + * + * @param raw_stderr a raw file handle to STDERR + * + * @return a handle to STDERR + */ +os_file_handle +os_convert_stderr_handle(os_raw_file_handle raw_stderr); + +/** + * Open a directory stream for the provided directory handle. The returned + * directory stream will be positioned at the first entry in the directory. + * + * @param handle the directory handle + * @param dir_stream a pointer in which to store the new directory stream + */ +__wasi_errno_t +os_fdopendir(os_file_handle handle, os_dir_stream *dir_stream); + +/** + * Reset the position of a directory stream to the beginning of the directory. + * + * @param dir_stream the directory stream for which to reset the position + */ +__wasi_errno_t +os_rewinddir(os_dir_stream dir_stream); + +/** + * Set the position of the given directory stream. + * + * @param dir_stream the directory stream for which to set the position + * @param position the position to set + */ +__wasi_errno_t +os_seekdir(os_dir_stream dir_stream, __wasi_dircookie_t position); + +/** + * Read a directory entry from the given directory stream. The directory name + * will be NULL if the end of the directory is reached or an error is + * encountered. + * + * @param dir_stream the directory stream from which to read the entry + * @param entry a pointer in which to store the directory entry + * @param d_name a pointer in which to store the directory entry name + */ +__wasi_errno_t +os_readdir(os_dir_stream dir_stream, __wasi_dirent_t *entry, + const char **d_name); + +/** + * Close the given directory stream. The handle associated with the directory + * stream will also be closed. + * + * @param dir_stream the directory stream to close + */ +__wasi_errno_t +os_closedir(os_dir_stream dir_stream); + +/** + * Returns an invalid directory stream that is guaranteed to cause failure when + * called with any directory filesystem operation. + * + * @return the invalid directory stream + */ +os_dir_stream +os_get_invalid_dir_stream(); + +/** + * Checks whether the given directory stream is valid. An invalid directory + * stream is guaranteed to cause failure when called with any directory + * filesystem operation. + * + * @param dir_stream a pointer to a directory stream + */ +bool +os_is_dir_stream_valid(os_dir_stream *dir_stream); + +/** + * Returns an invalid handle that is guaranteed to cause failure when + * called with any filesystem operation. + * + * @return the invalid handle + */ +os_file_handle +os_get_invalid_handle(); + +/** + * Checks whether the given file handle is valid. An invalid handle is + * guaranteed to cause failure when called with any filesystem operation. + * + * @param handle a pointer to a file handle + */ +bool +os_is_handle_valid(os_file_handle *handle); + +/** + * Resolve a pathname. The generated pathname will be stored as a + * null-terminated string, with a maximum length of PATH_MAX bytes. + * + * @param path the path to resolve + * @param resolved_path the buffer to store the resolved path in + * + * @return the resolved path if success, NULL otherwise + */ +char * +os_realpath(const char *path, char *resolved_path); + +/**************************************************** + * Section 4 * + * Clock functions * + ****************************************************/ + +/** + * NOTES: + * Clock functions are required for WASI libc support. If you don't need to + * support WASI libc, there is no need to implement these APIs. + */ + +/** + * Get the resolution of the specified clock. + * + * @param clock_id clock identifier + * @param resolution output variable to store the clock resolution + */ +__wasi_errno_t +os_clock_res_get(__wasi_clockid_t clock_id, __wasi_timestamp_t *resolution); + +/** + * Get the current time of the specified clock. + * + * @param clock_id clock identifier + * @param precision the maximum lag that the returned time value may have, + * compared to its actual value. + * @param time output variable to store the clock time + */ +__wasi_errno_t +os_clock_time_get(__wasi_clockid_t clock_id, __wasi_timestamp_t precision, + __wasi_timestamp_t *time); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef PLATFORM_API_EXTENSION_H */ diff --git a/wasm-micro-runtime/core/shared/platform/include/platform_api_vmcore.h b/wasm-micro-runtime/core/shared/platform/include/platform_api_vmcore.h new file mode 100644 index 0000000..1fa524f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/include/platform_api_vmcore.h @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_API_VMCORE_H +#define _PLATFORM_API_VMCORE_H + +#include "platform_common.h" +#include "platform_internal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************** + * Section 1 * + * Interfaces required by the runtime * + ****************************************************/ + +/** + * Initialize the platform internal resources if needed, + * this function is called by wasm_runtime_init() and + * wasm_runtime_full_init() + * + * @return 0 if success + */ +int +bh_platform_init(void); + +/** + * Destroy the platform internal resources if needed, + * this function is called by wasm_runtime_destroy() + */ +void +bh_platform_destroy(void); + +/** + ******** memory allocator APIs ********** + */ + +void * +os_malloc(unsigned size); + +void * +os_realloc(void *ptr, unsigned size); + +void +os_free(void *ptr); + +/** + * Note: the above APIs can simply return NULL if wasm runtime + * isn't initialized with Alloc_With_System_Allocator. + * Refer to wasm_runtime_full_init(). + */ + +int +os_printf(const char *format, ...); + +int +os_vprintf(const char *format, va_list ap); + +/** + * Get microseconds after boot. + */ +uint64 +os_time_get_boot_us(void); + +/** + * Get thread-specific CPU-time clock in microseconds + */ +uint64 +os_time_thread_cputime_us(void); + +/** + * Get current thread id. + * Implementation optional: Used by runtime for logging only. + */ +korp_tid +os_self_thread(void); + +/** + * Get current thread's stack boundary address, used for runtime + * to check the native stack overflow. Return NULL if it is not + * easy to implement, but may have potential issue. + */ +uint8 * +os_thread_get_stack_boundary(void); + +/** + * Set whether the MAP_JIT region write protection is enabled for this thread. + * Pass true to make the region executable, false to make it writable. + */ +void +os_thread_jit_write_protect_np(bool enabled); + +/** + ************** mutext APIs *********** + * vmcore: Not required until pthread is supported by runtime + * app-mgr: Must be implemented + */ + +int +os_mutex_init(korp_mutex *mutex); + +int +os_mutex_destroy(korp_mutex *mutex); + +int +os_mutex_lock(korp_mutex *mutex); + +int +os_mutex_unlock(korp_mutex *mutex); + +/************************************************** + * Section 2 * + * APIs required by WAMR AOT * + **************************************************/ + +/* Memory map modes */ +enum { + MMAP_PROT_NONE = 0, + MMAP_PROT_READ = 1, + MMAP_PROT_WRITE = 2, + MMAP_PROT_EXEC = 4 +}; + +/* Memory map flags */ +enum { + MMAP_MAP_NONE = 0, + /* Put the mapping into 0 to 2 G, supported only on x86_64 */ + MMAP_MAP_32BIT = 1, + /* Don't interpret addr as a hint: place the mapping at exactly + that address. */ + MMAP_MAP_FIXED = 2, +}; + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file); +void +os_munmap(void *addr, size_t size); +int +os_mprotect(void *addr, size_t size, int prot); + +static inline void * +os_mremap_slow(void *old_addr, size_t old_size, size_t new_size) +{ + void *new_memory = os_mmap(NULL, new_size, MMAP_PROT_WRITE | MMAP_PROT_READ, + 0, os_get_invalid_handle()); + if (!new_memory) { + return NULL; + } + /* + * bh_memcpy_s can't be used as it doesn't support values bigger than + * UINT32_MAX + */ + memcpy(new_memory, old_addr, new_size < old_size ? new_size : old_size); + os_munmap(old_addr, old_size); + + return new_memory; +} + +/* Doesn't guarantee that protection flags will be preserved. + os_mprotect() must be called after remapping. */ +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size); + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) +void * +os_get_dbus_mirror(void *ibus); +#endif + +/** + * Flush cpu data cache, in some CPUs, after applying relocation to the + * AOT code, the code may haven't been written back to the cpu data cache, + * which may cause unexpected behaviour when executing the AOT code. + * Implement this function if required, or just leave it empty. + */ +void +os_dcache_flush(void); + +/** + * Flush instruction cache. + */ +void +os_icache_flush(void *start, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _PLATFORM_API_VMCORE_H */ diff --git a/wasm-micro-runtime/core/shared/platform/include/platform_common.h b/wasm-micro-runtime/core/shared/platform/include/platform_common.h new file mode 100644 index 0000000..28001af --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/include/platform_common.h @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_COMMON_H +#define _PLATFORM_COMMON_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "platform_internal.h" +#include "../../../config.h" + +#define BH_MAX_THREAD 32 + +#define BHT_ERROR (-1) +#define BHT_TIMED_OUT (1) +#define BHT_OK (0) + +#define BHT_WAIT_FOREVER ((uint64)-1LL) + +#define BH_KB (1024) +#define BH_MB ((BH_KB)*1024) +#define BH_GB ((BH_MB)*1024) + +#ifndef BH_MALLOC +#define BH_MALLOC os_malloc +#endif + +#ifndef BH_FREE +#define BH_FREE os_free +#endif + +#ifndef BH_TIME_T_MAX +#define BH_TIME_T_MAX LONG_MAX +#endif + +#if defined(_MSC_BUILD) +#if defined(COMPILING_WASM_RUNTIME_API) +__declspec(dllexport) void *BH_MALLOC(unsigned int size); +__declspec(dllexport) void BH_FREE(void *ptr); +#else +__declspec(dllimport) void *BH_MALLOC(unsigned int size); +__declspec(dllimport) void BH_FREE(void *ptr); +#endif +#else +void * +BH_MALLOC(unsigned int size); +void +BH_FREE(void *ptr); +#endif + +#if defined(BH_VPRINTF) +#if defined(MSVC) +__declspec(dllimport) int BH_VPRINTF(const char *format, va_list ap); +#else +int +BH_VPRINTF(const char *format, va_list ap); +#endif +#endif + +#ifndef NULL +#define NULL (void *)0 +#endif + +#if !defined(BH_HAS_DLFCN) +#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) +#define BH_HAS_DLFCN 1 +#else +#define BH_HAS_DLFCN 0 +#endif +#endif + +#ifndef __cplusplus + +#ifndef true +#define true 1 +#endif + +#ifndef false +#define false 0 +#endif + +#ifndef inline +#define inline __inline +#endif + +#endif + +/* Return the offset of the given field in the given type */ +#ifndef offsetof +/* GCC 4.0 and later has the builtin. */ +#if defined(__GNUC__) && __GNUC__ >= 4 +#define offsetof(Type, field) __builtin_offsetof(Type, field) +#else +#define offsetof(Type, field) ((size_t)(&((Type *)0)->field)) +#endif +#endif + +typedef uint8_t uint8; +typedef int8_t int8; +typedef uint16_t uint16; +typedef int16_t int16; +typedef uint32_t uint32; +typedef int32_t int32; +typedef float float32; +typedef double float64; +typedef uint64_t uint64; +typedef int64_t int64; + +typedef void *(*thread_start_routine_t)(void *); + +#ifndef bh_socket_t +/* If no socket defined on current platform, + give a fake definition to make the compiler happy */ +#define bh_socket_t int +#endif + +/* Format specifiers macros in case + they are not provided by compiler */ +#ifndef __PRI64_PREFIX +#if UINTPTR_MAX == UINT64_MAX +#define __PRI64_PREFIX "l" +#define __PRIPTR_PREFIX "l" +#else +#define __PRI64_PREFIX "ll" +#define __PRIPTR_PREFIX +#endif +#endif /* #ifndef __PRI64_PREFIX */ + +/* Macros for printing format specifiers */ +#ifndef PRId32 +#define PRId32 "d" +#endif +#ifndef PRIi32 +#define PRIi32 "i" +#endif +#ifndef PRIu32 +#define PRIu32 "u" +#endif +#ifndef PRIx32 +#define PRIx32 "x" +#endif +#ifndef PRIX32 +#define PRIX32 "X" +#endif + +#ifndef PRId64 +#define PRId64 __PRI64_PREFIX "d" +#endif +#ifndef PRIu64 +#define PRIu64 __PRI64_PREFIX "u" +#endif +#ifndef PRIx64 +#define PRIx64 __PRI64_PREFIX "x" +#endif +#ifndef PRIX64 +#define PRIX64 __PRI64_PREFIX "X" +#endif +#ifndef PRIxPTR +#define PRIxPTR __PRIPTR_PREFIX "x" +#endif +#ifndef PRIXPTR +#define PRIXPTR __PRIPTR_PREFIX "X" +#endif + +/* Macros for scanning format specifiers */ +#ifndef SCNd32 +#define SCNd32 "d" +#endif +#ifndef SCNi32 +#define SCNi32 "i" +#endif +#ifndef SCNu32 +#define SCNu32 "u" +#endif +#ifndef SCNx32 +#define SCNx32 "x" +#endif + +#ifndef SCNd64 +#define SCNd64 __PRI64_PREFIX "d" +#endif +#ifndef SCNu64 +#define SCNu64 __PRI64_PREFIX "u" +#endif +#ifndef SCNx64 +#define SCNx64 __PRI64_PREFIX "x" +#endif +#ifndef SCNxPTR +#define SCNxPTR __PRIPTR_PREFIX "x" +#endif + +#ifndef NAN +#define NAN (0.0 / 0.0) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _PLATFORM_COMMON_H */ diff --git a/wasm-micro-runtime/core/shared/platform/include/platform_wasi_types.h b/wasm-micro-runtime/core/shared/platform/include/platform_wasi_types.h new file mode 100644 index 0000000..ac1a95e --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/include/platform_wasi_types.h @@ -0,0 +1,610 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* + * This file declares the WASI interface. The definitions of types, macros and + * structures in this file should be consistent with those in wasi-libc: + * https://github.com/WebAssembly/wasi-libc/blob/main/libc-bottom-half/headers/public/wasi/api.h + */ + +#ifndef _PLATFORM_WASI_TYPES_H +#define _PLATFORM_WASI_TYPES_H + +#include "../../../config.h" + +#include +#include + +/* clang-format off */ + +#ifdef __cplusplus +#ifndef _Static_assert +#define _Static_assert static_assert +#endif /* _Static_assert */ + +#ifndef _Alignof +#define _Alignof alignof +#endif /* _Alignof */ + +extern "C" { +#endif + +/* There is no need to check the WASI layout if we're using uvwasi or libc-wasi + * is not enabled at all. */ +#if WASM_ENABLE_UVWASI != 0 || WASM_ENABLE_LIBC_WASI == 0 +#define assert_wasi_layout(expr, message) /* nothing */ +#else +#define assert_wasi_layout(expr, message) _Static_assert(expr, message) +#endif + +assert_wasi_layout(_Alignof(int8_t) == 1, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint8_t) == 1, "non-wasi data layout"); +assert_wasi_layout(_Alignof(int16_t) == 2, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint16_t) == 2, "non-wasi data layout"); +assert_wasi_layout(_Alignof(int32_t) == 4, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint32_t) == 4, "non-wasi data layout"); +#if 0 +assert_wasi_layout(_Alignof(int64_t) == 8, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint64_t) == 8, "non-wasi data layout"); +#endif + +typedef uint32_t __wasi_size_t; +assert_wasi_layout(_Alignof(__wasi_size_t) == 4, "non-wasi data layout"); + +typedef uint8_t __wasi_advice_t; +#define __WASI_ADVICE_NORMAL (0) +#define __WASI_ADVICE_SEQUENTIAL (1) +#define __WASI_ADVICE_RANDOM (2) +#define __WASI_ADVICE_WILLNEED (3) +#define __WASI_ADVICE_DONTNEED (4) +#define __WASI_ADVICE_NOREUSE (5) + +typedef uint32_t __wasi_clockid_t; +#define __WASI_CLOCK_REALTIME (0) +#define __WASI_CLOCK_MONOTONIC (1) +#define __WASI_CLOCK_PROCESS_CPUTIME_ID (2) +#define __WASI_CLOCK_THREAD_CPUTIME_ID (3) + +typedef uint64_t __wasi_device_t; + +typedef uint64_t __wasi_dircookie_t; +#define __WASI_DIRCOOKIE_START (0) + +typedef uint32_t __wasi_dirnamlen_t; + +typedef uint16_t __wasi_errno_t; +#define __WASI_ESUCCESS (0) +#define __WASI_E2BIG (1) +#define __WASI_EACCES (2) +#define __WASI_EADDRINUSE (3) +#define __WASI_EADDRNOTAVAIL (4) +#define __WASI_EAFNOSUPPORT (5) +#define __WASI_EAGAIN (6) +#define __WASI_EALREADY (7) +#define __WASI_EBADF (8) +#define __WASI_EBADMSG (9) +#define __WASI_EBUSY (10) +#define __WASI_ECANCELED (11) +#define __WASI_ECHILD (12) +#define __WASI_ECONNABORTED (13) +#define __WASI_ECONNREFUSED (14) +#define __WASI_ECONNRESET (15) +#define __WASI_EDEADLK (16) +#define __WASI_EDESTADDRREQ (17) +#define __WASI_EDOM (18) +#define __WASI_EDQUOT (19) +#define __WASI_EEXIST (20) +#define __WASI_EFAULT (21) +#define __WASI_EFBIG (22) +#define __WASI_EHOSTUNREACH (23) +#define __WASI_EIDRM (24) +#define __WASI_EILSEQ (25) +#define __WASI_EINPROGRESS (26) +#define __WASI_EINTR (27) +#define __WASI_EINVAL (28) +#define __WASI_EIO (29) +#define __WASI_EISCONN (30) +#define __WASI_EISDIR (31) +#define __WASI_ELOOP (32) +#define __WASI_EMFILE (33) +#define __WASI_EMLINK (34) +#define __WASI_EMSGSIZE (35) +#define __WASI_EMULTIHOP (36) +#define __WASI_ENAMETOOLONG (37) +#define __WASI_ENETDOWN (38) +#define __WASI_ENETRESET (39) +#define __WASI_ENETUNREACH (40) +#define __WASI_ENFILE (41) +#define __WASI_ENOBUFS (42) +#define __WASI_ENODEV (43) +#define __WASI_ENOENT (44) +#define __WASI_ENOEXEC (45) +#define __WASI_ENOLCK (46) +#define __WASI_ENOLINK (47) +#define __WASI_ENOMEM (48) +#define __WASI_ENOMSG (49) +#define __WASI_ENOPROTOOPT (50) +#define __WASI_ENOSPC (51) +#define __WASI_ENOSYS (52) +#define __WASI_ENOTCONN (53) +#define __WASI_ENOTDIR (54) +#define __WASI_ENOTEMPTY (55) +#define __WASI_ENOTRECOVERABLE (56) +#define __WASI_ENOTSOCK (57) +#define __WASI_ENOTSUP (58) +#define __WASI_ENOTTY (59) +#define __WASI_ENXIO (60) +#define __WASI_EOVERFLOW (61) +#define __WASI_EOWNERDEAD (62) +#define __WASI_EPERM (63) +#define __WASI_EPIPE (64) +#define __WASI_EPROTO (65) +#define __WASI_EPROTONOSUPPORT (66) +#define __WASI_EPROTOTYPE (67) +#define __WASI_ERANGE (68) +#define __WASI_EROFS (69) +#define __WASI_ESPIPE (70) +#define __WASI_ESRCH (71) +#define __WASI_ESTALE (72) +#define __WASI_ETIMEDOUT (73) +#define __WASI_ETXTBSY (74) +#define __WASI_EXDEV (75) +#define __WASI_ENOTCAPABLE (76) + +#if defined(_MSC_VER) +#define ALIGNED_(x) __declspec(align(x)) +#define WARN_UNUSED _Check_return_ +#elif defined(__GNUC__) +#define ALIGNED_(x) __attribute__ ((aligned(x))) +#define WARN_UNUSED __attribute__((__warn_unused_result__)) +#endif + +#define ALIGNED_TYPE(t,x) typedef t ALIGNED_(x) + +typedef uint16_t __wasi_eventrwflags_t; +#define __WASI_EVENT_FD_READWRITE_HANGUP (0x0001) + +typedef uint8_t __wasi_eventtype_t; +#define __WASI_EVENTTYPE_CLOCK (0) +#define __WASI_EVENTTYPE_FD_READ (1) +#define __WASI_EVENTTYPE_FD_WRITE (2) + +typedef uint32_t __wasi_exitcode_t; + +typedef uint32_t __wasi_fd_t; + +typedef uint16_t __wasi_fdflags_t; +#define __WASI_FDFLAG_APPEND (0x0001) +#define __WASI_FDFLAG_DSYNC (0x0002) +#define __WASI_FDFLAG_NONBLOCK (0x0004) +#define __WASI_FDFLAG_RSYNC (0x0008) +#define __WASI_FDFLAG_SYNC (0x0010) + +typedef int64_t __wasi_filedelta_t; + +typedef uint64_t __wasi_filesize_t; + +typedef uint8_t __wasi_filetype_t; +#define __WASI_FILETYPE_UNKNOWN (0) +#define __WASI_FILETYPE_BLOCK_DEVICE (1) +#define __WASI_FILETYPE_CHARACTER_DEVICE (2) +#define __WASI_FILETYPE_DIRECTORY (3) +#define __WASI_FILETYPE_REGULAR_FILE (4) +#define __WASI_FILETYPE_SOCKET_DGRAM (5) +#define __WASI_FILETYPE_SOCKET_STREAM (6) +#define __WASI_FILETYPE_SYMBOLIC_LINK (7) + +typedef uint16_t __wasi_fstflags_t; +#define __WASI_FILESTAT_SET_ATIM (0x0001) +#define __WASI_FILESTAT_SET_ATIM_NOW (0x0002) +#define __WASI_FILESTAT_SET_MTIM (0x0004) +#define __WASI_FILESTAT_SET_MTIM_NOW (0x0008) + +typedef uint64_t __wasi_inode_t; + +ALIGNED_TYPE(uint64_t, 8) __wasi_linkcount_t; + +typedef uint32_t __wasi_lookupflags_t; +#define __WASI_LOOKUP_SYMLINK_FOLLOW (0x00000001) + +typedef uint16_t __wasi_oflags_t; +#define __WASI_O_CREAT (0x0001) +#define __WASI_O_DIRECTORY (0x0002) +#define __WASI_O_EXCL (0x0004) +#define __WASI_O_TRUNC (0x0008) + +typedef uint16_t __wasi_riflags_t; +#define __WASI_SOCK_RECV_PEEK (0x0001) +#define __WASI_SOCK_RECV_WAITALL (0x0002) + +typedef uint64_t __wasi_rights_t; + +/** + * Observe that WASI defines rights in the plural form + * TODO: refactor to use RIGHTS instead of RIGHT + */ +#define __WASI_RIGHT_FD_DATASYNC ((__wasi_rights_t)(UINT64_C(1) << 0)) +#define __WASI_RIGHT_FD_READ ((__wasi_rights_t)(UINT64_C(1) << 1)) +#define __WASI_RIGHT_FD_SEEK ((__wasi_rights_t)(UINT64_C(1) << 2)) +#define __WASI_RIGHT_FD_FDSTAT_SET_FLAGS ((__wasi_rights_t)(UINT64_C(1) << 3)) +#define __WASI_RIGHT_FD_SYNC ((__wasi_rights_t)(UINT64_C(1) << 4)) +#define __WASI_RIGHT_FD_TELL ((__wasi_rights_t)(UINT64_C(1) << 5)) +#define __WASI_RIGHT_FD_WRITE ((__wasi_rights_t)(UINT64_C(1) << 6)) +#define __WASI_RIGHT_FD_ADVISE ((__wasi_rights_t)(UINT64_C(1) << 7)) +#define __WASI_RIGHT_FD_ALLOCATE ((__wasi_rights_t)(UINT64_C(1) << 8)) +#define __WASI_RIGHT_PATH_CREATE_DIRECTORY ((__wasi_rights_t)(UINT64_C(1) << 9)) +#define __WASI_RIGHT_PATH_CREATE_FILE ((__wasi_rights_t)(UINT64_C(1) << 10)) +#define __WASI_RIGHT_PATH_LINK_SOURCE ((__wasi_rights_t)(UINT64_C(1) << 11)) +#define __WASI_RIGHT_PATH_LINK_TARGET ((__wasi_rights_t)(UINT64_C(1) << 12)) +#define __WASI_RIGHT_PATH_OPEN ((__wasi_rights_t)(UINT64_C(1) << 13)) +#define __WASI_RIGHT_FD_READDIR ((__wasi_rights_t)(UINT64_C(1) << 14)) +#define __WASI_RIGHT_PATH_READLINK ((__wasi_rights_t)(UINT64_C(1) << 15)) +#define __WASI_RIGHT_PATH_RENAME_SOURCE ((__wasi_rights_t)(UINT64_C(1) << 16)) +#define __WASI_RIGHT_PATH_RENAME_TARGET ((__wasi_rights_t)(UINT64_C(1) << 17)) +#define __WASI_RIGHT_PATH_FILESTAT_GET ((__wasi_rights_t)(UINT64_C(1) << 18)) +#define __WASI_RIGHT_PATH_FILESTAT_SET_SIZE ((__wasi_rights_t)(UINT64_C(1) << 19)) +#define __WASI_RIGHT_PATH_FILESTAT_SET_TIMES ((__wasi_rights_t)(UINT64_C(1) << 20)) +#define __WASI_RIGHT_FD_FILESTAT_GET ((__wasi_rights_t)(UINT64_C(1) << 21)) +#define __WASI_RIGHT_FD_FILESTAT_SET_SIZE ((__wasi_rights_t)(UINT64_C(1) << 22)) +#define __WASI_RIGHT_FD_FILESTAT_SET_TIMES ((__wasi_rights_t)(UINT64_C(1) << 23)) +#define __WASI_RIGHT_PATH_SYMLINK ((__wasi_rights_t)(UINT64_C(1) << 24)) +#define __WASI_RIGHT_PATH_REMOVE_DIRECTORY ((__wasi_rights_t)(UINT64_C(1) << 25)) +#define __WASI_RIGHT_PATH_UNLINK_FILE ((__wasi_rights_t)(UINT64_C(1) << 26)) +#define __WASI_RIGHT_POLL_FD_READWRITE ((__wasi_rights_t)(UINT64_C(1) << 27)) +#define __WASI_RIGHT_SOCK_CONNECT ((__wasi_rights_t)(UINT64_C(1) << 28)) +#define __WASI_RIGHT_SOCK_LISTEN ((__wasi_rights_t)(UINT64_C(1) << 29)) +#define __WASI_RIGHT_SOCK_BIND ((__wasi_rights_t)(UINT64_C(1) << 30)) +#define __WASI_RIGHT_SOCK_ACCEPT ((__wasi_rights_t)(UINT64_C(1) << 31)) +#define __WASI_RIGHT_SOCK_RECV ((__wasi_rights_t)(UINT64_C(1) << 32)) +#define __WASI_RIGHT_SOCK_SEND ((__wasi_rights_t)(UINT64_C(1) << 33)) +#define __WASI_RIGHT_SOCK_ADDR_LOCAL ((__wasi_rights_t)(UINT64_C(1) << 34)) +#define __WASI_RIGHT_SOCK_ADDR_REMOTE ((__wasi_rights_t)(UINT64_C(1) << 35)) +#define __WASI_RIGHT_SOCK_RECV_FROM ((__wasi_rights_t)(UINT64_C(1) << 36)) +#define __WASI_RIGHT_SOCK_SEND_TO ((__wasi_rights_t)(UINT64_C(1) << 37)) + +typedef uint16_t __wasi_roflags_t; +#define __WASI_SOCK_RECV_DATA_TRUNCATED (0x0001) + +typedef uint8_t __wasi_sdflags_t; +#define __WASI_SHUT_RD (0x01) +#define __WASI_SHUT_WR (0x02) + +typedef uint16_t __wasi_siflags_t; + +typedef uint8_t __wasi_signal_t; + +typedef uint16_t __wasi_subclockflags_t; +#define __WASI_SUBSCRIPTION_CLOCK_ABSTIME (0x0001) + +typedef uint64_t __wasi_timestamp_t; + +typedef uint64_t __wasi_userdata_t; + +typedef uint8_t __wasi_whence_t; +#define __WASI_WHENCE_SET (0) +#define __WASI_WHENCE_CUR (1) +#define __WASI_WHENCE_END (2) + +typedef uint8_t __wasi_preopentype_t; +#define __WASI_PREOPENTYPE_DIR (0) + +struct fd_table; +struct fd_prestats; +struct argv_environ_values; +struct addr_pool; + +typedef struct ALIGNED_(8) __wasi_dirent_t { + __wasi_dircookie_t d_next; + __wasi_inode_t d_ino; + __wasi_dirnamlen_t d_namlen; + __wasi_filetype_t d_type; +} __wasi_dirent_t; +assert_wasi_layout(offsetof(__wasi_dirent_t, d_next) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_dirent_t, d_ino) == 8, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_dirent_t, d_namlen) == 16, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_dirent_t, d_type) == 20, "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_dirent_t) == 24, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_dirent_t) == 8, "non-wasi data layout"); + +typedef struct ALIGNED_(8) __wasi_event_t { + __wasi_userdata_t userdata; + __wasi_errno_t error; + __wasi_eventtype_t type; + uint8_t __paddings[5]; + union __wasi_event_u { + struct __wasi_event_u_fd_readwrite_t { + __wasi_filesize_t nbytes; + __wasi_eventrwflags_t flags; + uint8_t __paddings[6]; + } fd_readwrite; + } u; +} __wasi_event_t; +assert_wasi_layout(offsetof(__wasi_event_t, userdata) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_event_t, error) == 8, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_event_t, type) == 10, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_event_t, u.fd_readwrite.nbytes) == 16, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_event_t, u.fd_readwrite.flags) == 24, "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_event_t) == 32, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_event_t) == 8, "non-wasi data layout"); + +typedef struct __wasi_prestat_t { + __wasi_preopentype_t pr_type; + union __wasi_prestat_u { + struct __wasi_prestat_u_dir_t { + size_t pr_name_len; + } dir; + } u; +} __wasi_prestat_t; +assert_wasi_layout(offsetof(__wasi_prestat_t, pr_type) == 0, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + offsetof(__wasi_prestat_t, u.dir.pr_name_len) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + offsetof(__wasi_prestat_t, u.dir.pr_name_len) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + sizeof(__wasi_prestat_t) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + sizeof(__wasi_prestat_t) == 16, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + _Alignof(__wasi_prestat_t) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + _Alignof(__wasi_prestat_t) == 8, "non-wasi data layout"); + +typedef struct ALIGNED_(8) __wasi_fdstat_t { + __wasi_filetype_t fs_filetype; + __wasi_fdflags_t fs_flags; + uint8_t __paddings[4]; + __wasi_rights_t fs_rights_base; + __wasi_rights_t fs_rights_inheriting; +} __wasi_fdstat_t; +assert_wasi_layout( + offsetof(__wasi_fdstat_t, fs_filetype) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_fdstat_t, fs_flags) == 2, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_fdstat_t, fs_rights_base) == 8, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_fdstat_t, fs_rights_inheriting) == 16, + "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_fdstat_t) == 24, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_fdstat_t) == 8, "non-wasi data layout"); + +typedef struct ALIGNED_(8) __wasi_filestat_t { + __wasi_device_t st_dev; + __wasi_inode_t st_ino; + __wasi_filetype_t st_filetype; + __wasi_linkcount_t st_nlink; + __wasi_filesize_t st_size; + __wasi_timestamp_t st_atim; + __wasi_timestamp_t st_mtim; + __wasi_timestamp_t st_ctim; +} __wasi_filestat_t; +assert_wasi_layout(offsetof(__wasi_filestat_t, st_dev) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_filestat_t, st_ino) == 8, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_filetype) == 16, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_nlink) == 24, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_size) == 32, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_atim) == 40, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_mtim) == 48, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_ctim) == 56, "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_filestat_t) == 64, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_filestat_t) == 8, "non-wasi data layout"); + +typedef struct __wasi_ciovec_t { + const void *buf; + size_t buf_len; +} __wasi_ciovec_t; +assert_wasi_layout(offsetof(__wasi_ciovec_t, buf) == 0, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + offsetof(__wasi_ciovec_t, buf_len) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + offsetof(__wasi_ciovec_t, buf_len) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + sizeof(__wasi_ciovec_t) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + sizeof(__wasi_ciovec_t) == 16, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + _Alignof(__wasi_ciovec_t) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + _Alignof(__wasi_ciovec_t) == 8, "non-wasi data layout"); + +typedef struct __wasi_iovec_t { + void *buf; + size_t buf_len; +} __wasi_iovec_t; +assert_wasi_layout(offsetof(__wasi_iovec_t, buf) == 0, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + offsetof(__wasi_iovec_t, buf_len) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + offsetof(__wasi_iovec_t, buf_len) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + sizeof(__wasi_iovec_t) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + sizeof(__wasi_iovec_t) == 16, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + _Alignof(__wasi_iovec_t) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + _Alignof(__wasi_iovec_t) == 8, "non-wasi data layout"); + +/** + * The contents of a `subscription` when type is `eventtype::clock`. + */ +typedef struct ALIGNED_(8) __wasi_subscription_clock_t { + /** + * The clock against which to compare the timestamp. + */ + __wasi_clockid_t clock_id; + + uint8_t __paddings1[4]; + + /** + * The absolute or relative timestamp. + */ + __wasi_timestamp_t timeout; + + /** + * The amount of time that the implementation may wait additionally + * to coalesce with other events. + */ + __wasi_timestamp_t precision; + + /** + * Flags specifying whether the timeout is absolute or relative + */ + __wasi_subclockflags_t flags; + + uint8_t __paddings2[4]; + +} __wasi_subscription_clock_t; + +assert_wasi_layout(sizeof(__wasi_subscription_clock_t) == 32, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_clock_t) == 8, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, clock_id) == 0, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, timeout) == 8, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, precision) == 16, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, flags) == 24, "witx calculated offset"); + +/** + * The contents of a `subscription` when type is type is + * `eventtype::fd_read` or `eventtype::fd_write`. + */ +typedef struct __wasi_subscription_fd_readwrite_t { + /** + * The file descriptor on which to wait for it to become ready for reading or writing. + */ + __wasi_fd_t fd; + +} __wasi_subscription_fd_readwrite_t; + +assert_wasi_layout(sizeof(__wasi_subscription_fd_readwrite_t) == 4, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_fd_readwrite_t) == 4, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_fd_readwrite_t, fd) == 0, "witx calculated offset"); + +/** + * The contents of a `subscription`. + */ +typedef union __wasi_subscription_u_u_t { + __wasi_subscription_clock_t clock; + __wasi_subscription_fd_readwrite_t fd_readwrite; +} __wasi_subscription_u_u_t ; + +typedef struct ALIGNED_(8) __wasi_subscription_u_t { + __wasi_eventtype_t type; + __wasi_subscription_u_u_t u; +} __wasi_subscription_u_t; + +assert_wasi_layout(sizeof(__wasi_subscription_u_t) == 40, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_u_t) == 8, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_u_t, u) == 8, "witx calculated union offset"); +assert_wasi_layout(sizeof(__wasi_subscription_u_u_t) == 32, "witx calculated union size"); +assert_wasi_layout(_Alignof(__wasi_subscription_u_u_t) == 8, "witx calculated union align"); + +/** + * Subscription to an event. + */ +typedef struct __wasi_subscription_t { + /** + * User-provided value that is attached to the subscription in the + * implementation and returned through `event::userdata`. + */ + __wasi_userdata_t userdata; + + /** + * The type of the event to which to subscribe, and its contents + */ + __wasi_subscription_u_t u; + +} __wasi_subscription_t; + +assert_wasi_layout(sizeof(__wasi_subscription_t) == 48, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_t) == 8, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_t, userdata) == 0, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_t, u) == 8, "witx calculated offset"); + +/* keep syncing with wasi_socket_ext.h */ +typedef enum { + /* Used only for sock_addr_resolve hints */ + SOCKET_ANY = -1, + SOCKET_DGRAM = 0, + SOCKET_STREAM, +} __wasi_sock_type_t; + +typedef uint16_t __wasi_ip_port_t; + +typedef enum { IPv4 = 0, IPv6 } __wasi_addr_type_t; + +/* n0.n1.n2.n3 */ +typedef struct __wasi_addr_ip4_t { + uint8_t n0; + uint8_t n1; + uint8_t n2; + uint8_t n3; +} __wasi_addr_ip4_t; + +typedef struct __wasi_addr_ip4_port_t { + __wasi_addr_ip4_t addr; + __wasi_ip_port_t port; +} __wasi_addr_ip4_port_t; + +typedef struct __wasi_addr_ip6_t { + uint16_t n0; + uint16_t n1; + uint16_t n2; + uint16_t n3; + uint16_t h0; + uint16_t h1; + uint16_t h2; + uint16_t h3; +} __wasi_addr_ip6_t; + +typedef struct __wasi_addr_ip6_port_t { + __wasi_addr_ip6_t addr; + __wasi_ip_port_t port; +} __wasi_addr_ip6_port_t; + +typedef struct __wasi_addr_ip_t { + __wasi_addr_type_t kind; + union { + __wasi_addr_ip4_t ip4; + __wasi_addr_ip6_t ip6; + } addr; +} __wasi_addr_ip_t; + +typedef struct __wasi_addr_t { + __wasi_addr_type_t kind; + union { + __wasi_addr_ip4_port_t ip4; + __wasi_addr_ip6_port_t ip6; + } addr; +} __wasi_addr_t; + +typedef enum { INET4 = 0, INET6, INET_UNSPEC } __wasi_address_family_t; + +typedef struct __wasi_addr_info_t { + __wasi_addr_t addr; + __wasi_sock_type_t type; +} __wasi_addr_info_t; + +typedef struct __wasi_addr_info_hints_t { + __wasi_sock_type_t type; + __wasi_address_family_t family; + // this is to workaround lack of optional parameters + uint8_t hints_enabled; +} __wasi_addr_info_hints_t; + +#undef assert_wasi_layout + +/* clang-format on */ +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_WASI_TYPES_H */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/platform_internal.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/platform_internal.h new file mode 100644 index 0000000..2cc34df --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/platform_internal.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sgx_error.h" +#include "sgx_file.h" +#include "sgx_pthread.h" +#include "sgx_time.h" +#include "sgx_socket.h" +#include "sgx_signal.h" +#include "sgx_trts.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_LINUX_SGX +#define BH_PLATFORM_LINUX_SGX +#endif + +#define _STACK_SIZE_ADJUSTMENT (32 * 1024) + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (8 * 1024 + _STACK_SIZE_ADJUSTMENT) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef pthread_t korp_thread; +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_rwlock_t korp_rwlock; +typedef unsigned int korp_sem; + +#ifndef SGX_DISABLE_PTHREAD +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +#endif + +typedef int (*os_print_function_t)(const char *message); +void +os_set_print_function(os_print_function_t pf); + +char * +strcpy(char *dest, const char *src); + +#define os_memory_order_acquire __ATOMIC_ACQUIRE +#define os_memory_order_release __ATOMIC_RELEASE +#define os_memory_order_seq_cst __ATOMIC_SEQ_CST +#define os_atomic_thread_fence __atomic_thread_fence + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#define os_getpagesize getpagesize + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_file.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_file.c new file mode 100644 index 0000000..a8ae8d2 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_file.c @@ -0,0 +1,1117 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "sgx_error.h" +#include "sgx_file.h" + +#if WASM_ENABLE_SGX_IPFS != 0 +#include "sgx_ipfs.h" +#endif + +#ifndef SGX_DISABLE_WASI + +#define TRACE_FUNC() os_printf("undefined %s\n", __FUNCTION__) +#define TRACE_OCALL_FAIL() os_printf("ocall %s failed!\n", __FUNCTION__) + +/** fd **/ +int +ocall_open(int *p_fd, const char *pathname, int flags, bool has_mode, + unsigned mode); + +int +ocall_openat(int *p_fd, int dirfd, const char *pathname, int flags, + bool has_mode, unsigned mode); + +int +ocall_read(ssize_t *p_ret, int fd, void *buf, size_t read_size); + +int +ocall_close(int *p_ret, int fd); + +int +ocall_lseek(off_t *p_ret, int fd, off_t offset, int whence); + +int +ocall_ftruncate(int *p_ret, int fd, off_t length); + +int +ocall_fsync(int *p_ret, int fd); + +int +ocall_fdatasync(int *p_ret, int fd); + +int +ocall_isatty(int *p_ret, int fd); +/** fd end **/ + +/** DIR **/ +int +ocall_fdopendir(int fd, void **p_dirp); + +int +ocall_readdir(void **p_dirent, void *dirp); + +int +ocall_rewinddir(void *dirp); + +int +ocall_seekdir(void *dirp, long loc); + +int +ocall_telldir(long *p_dir, void *dirp); + +int +ocall_closedir(int *p_ret, void *dirp); +/** DIR end **/ + +/** stat **/ +int +ocall_stat(int *p_ret, const char *pathname, void *buf, unsigned int buf_len); +int +ocall_fstat(int *p_ret, int fd, void *buf, unsigned int buf_len); +int +ocall_fstatat(int *p_ret, int dirfd, const char *pathname, void *buf, + unsigned int buf_len, int flags); +/** stat end **/ + +/** link **/ +int +ocall_mkdirat(int *p_ret, int dirfd, const char *pathname, unsigned mode); +int +ocall_link(int *p_ret, const char *oldpath, const char *newpath); +int +ocall_linkat(int *p_ret, int olddirfd, const char *oldpath, int newdirfd, + const char *newpath, int flags); +int +ocall_unlinkat(int *p_ret, int dirfd, const char *pathname, int flags); +int +ocall_readlink(ssize_t *p_ret, const char *pathname, char *buf, size_t bufsiz); +int +ocall_readlinkat(ssize_t *p_ret, int dirfd, const char *pathname, char *buf, + size_t bufsiz); +int +ocall_renameat(int *p_ret, int olddirfd, const char *oldpath, int newdirfd, + const char *newpath); +int +ocall_symlinkat(int *p_ret, const char *target, int newdirfd, + const char *linkpath); +/** link end **/ + +/** control **/ +int +ocall_ioctl(int *p_ret, int fd, unsigned long request, void *arg, + unsigned int arg_len); +int +ocall_fcntl(int *p_ret, int fd, int cmd); +int +ocall_fcntl_long(int *p_ret, int fd, int cmd, long arg); +/** control end **/ + +/** **/ +int +ocall_realpath(int *p_ret, const char *path, char *buf, unsigned int buf_len); +int +ocall_posix_fallocate(int *p_ret, int fd, off_t offset, off_t len); +int +ocall_poll(int *p_ret, void *fds, unsigned nfds, int timeout, + unsigned int fds_len); +int +ocall_getopt(int *p_ret, int argc, char *argv_buf, unsigned int argv_buf_len, + const char *optstring); +int +ocall_sched_yield(int *p_ret); + +/** struct iovec **/ +ssize_t +ocall_readv(ssize_t *p_ret, int fd, char *iov_buf, unsigned int buf_size, + int iovcnt, bool has_offset, off_t offset); +ssize_t +ocall_writev(ssize_t *p_ret, int fd, char *iov_buf, unsigned int buf_size, + int iovcnt, bool has_offset, off_t offset); +/** iovec end **/ + +int +ocall_get_errno(int *p_ret); + +int +open(const char *pathname, int flags, ...) +{ + int fd; + bool has_mode = false; + mode_t mode = 0; + + if ((flags & O_CREAT) || (flags & O_TMPFILE) == O_TMPFILE) { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, mode_t); + va_end(ap); + has_mode = true; + } + + if (SGX_SUCCESS != ocall_open(&fd, pathname, flags, has_mode, mode)) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (fd >= 0 && (flags & O_CLOEXEC)) + fcntl(fd, F_SETFD, FD_CLOEXEC); + + if (fd == -1) + errno = get_errno(); + return fd; +} + +int +openat(int dirfd, const char *pathname, int flags, ...) +{ + int fd; + bool has_mode = false; + mode_t mode = 0; + + if ((flags & O_CREAT) || (flags & O_TMPFILE) == O_TMPFILE) { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, mode_t); + va_end(ap); + has_mode = true; + } + + if (SGX_SUCCESS + != ocall_openat(&fd, dirfd, pathname, flags, has_mode, mode)) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (fd >= 0 && (flags & O_CLOEXEC)) + fcntl(fd, F_SETFD, FD_CLOEXEC); + + if (fd == -1) + errno = get_errno(); + +#if WASM_ENABLE_SGX_IPFS != 0 + struct stat sb; + int ret = fstatat(dirfd, pathname, &sb, 0); + if (ret < 0) { + if (ocall_close(&ret, fd) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } + return -1; + } + + // Ony files are managed by SGX IPFS + if (S_ISREG(sb.st_mode)) { + // When WAMR uses Intel SGX IPFS to enabled, it opens a second + // file descriptor to interact with the secure file. + // The first file descriptor opened earlier is used to interact + // with the metadata of the file (e.g., time, flags, etc.). + void *file_ptr = ipfs_fopen(fd, flags); + if (file_ptr == NULL) { + if (ocall_close(&ret, fd) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } + return -1; + } + } +#endif + + return fd; +} + +int +close(int fd) +{ + int ret; + +#if WASM_ENABLE_SGX_IPFS != 0 + // Close the IPFS file pointer in addition of the file descriptor + ret = ipfs_close(fd); + if (ret == -1) + errno = get_errno(); +#endif + + if (ocall_close(&ret, fd) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) + errno = get_errno(); + return ret; +} + +ssize_t +read(int fd, void *buf, size_t size) +{ + ssize_t ret; + int size_read_max = 2048, size_read, total_size_read = 0, count, i; + char *p = buf; + + if (buf == NULL) { + TRACE_FUNC(); + return -1; + } + + count = (size + size_read_max - 1) / size_read_max; + for (i = 0; i < count; i++) { + size_read = (i < count - 1) ? size_read_max : size - size_read_max * i; + + if (ocall_read(&ret, fd, p, size_read) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) { + /* read failed */ + errno = get_errno(); + return -1; + } + + p += ret; + total_size_read += ret; + + if (ret < size_read) + /* end of file */ + break; + } + return total_size_read; +} + +DIR * +fdopendir(int fd) +{ + DIR *result = NULL; + + result = (DIR *)BH_MALLOC(sizeof(DIR)); + if (!result) + return NULL; + + if (ocall_fdopendir(fd, (void **)result) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + BH_FREE(result); + return NULL; + } + + if ((void *)*result == NULL) { /* opendir failed */ + TRACE_FUNC(); + BH_FREE(result); + errno = get_errno(); + return NULL; + } + + return result; +} + +struct dirent * +readdir(DIR *dirp) +{ + struct dirent *result; + + if (dirp == NULL) + return NULL; + + if (ocall_readdir((void **)&result, (void *)*dirp) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return NULL; + } + + if (!result) + errno = get_errno(); + return result; +} + +void +rewinddir(DIR *dirp) +{ + if (ocall_rewinddir((void *)*dirp) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } +} + +void +seekdir(DIR *dirp, long loc) +{ + if (ocall_seekdir((void *)*dirp, loc) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } +} + +long +telldir(DIR *dirp) +{ + long ret; + + if (ocall_telldir(&ret, (void *)*dirp) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +closedir(DIR *dirp) +{ + int ret; + + if (ocall_closedir(&ret, (void *)*dirp) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + BH_FREE(dirp); + if (ret == -1) + errno = get_errno(); + return ret; +} + +static ssize_t +readv_internal(int fd, const struct iovec *iov, int iovcnt, bool has_offset, + off_t offset) +{ + ssize_t ret, size_left; + struct iovec *iov1; + int i; + char *p; + uint64 total_size = sizeof(struct iovec) * (uint64)iovcnt; + + if (iov == NULL || iovcnt < 1) + return -1; + + for (i = 0; i < iovcnt; i++) { + total_size += iov[i].iov_len; + } + + if (total_size >= UINT32_MAX) + return -1; + +#if WASM_ENABLE_SGX_IPFS != 0 + if (fd > 2) { + return ipfs_read(fd, iov, iovcnt, has_offset, offset); + } +#endif + + iov1 = BH_MALLOC((uint32)total_size); + + if (iov1 == NULL) + return -1; + + memset(iov1, 0, (uint32)total_size); + + p = (char *)(uintptr_t)(sizeof(struct iovec) * iovcnt); + + for (i = 0; i < iovcnt; i++) { + iov1[i].iov_len = iov[i].iov_len; + iov1[i].iov_base = p; + p += iov[i].iov_len; + } + + if (ocall_readv(&ret, fd, (char *)iov1, (uint32)total_size, iovcnt, + has_offset, offset) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + BH_FREE(iov1); + return -1; + } + + p = (char *)(uintptr_t)(sizeof(struct iovec) * iovcnt); + + size_left = ret; + for (i = 0; i < iovcnt; i++) { + if (size_left > iov[i].iov_len) { + memcpy(iov[i].iov_base, (uintptr_t)p + (char *)iov1, + iov[i].iov_len); + p += iov[i].iov_len; + size_left -= iov[i].iov_len; + } + else { + memcpy(iov[i].iov_base, (uintptr_t)p + (char *)iov1, size_left); + break; + } + } + + BH_FREE(iov1); + if (ret == -1) + errno = get_errno(); + return ret; +} + +static ssize_t +writev_internal(int fd, const struct iovec *iov, int iovcnt, bool has_offset, + off_t offset) +{ + ssize_t ret; + struct iovec *iov1; + int i; + char *p; + uint64 total_size = sizeof(struct iovec) * (uint64)iovcnt; + + if (iov == NULL || iovcnt < 1) + return -1; + + for (i = 0; i < iovcnt; i++) { + total_size += iov[i].iov_len; + } + + if (total_size >= UINT32_MAX) + return -1; + +#if WASM_ENABLE_SGX_IPFS != 0 + if (fd > 2) { + return ipfs_write(fd, iov, iovcnt, has_offset, offset); + } +#endif + + iov1 = BH_MALLOC((uint32)total_size); + + if (iov1 == NULL) + return -1; + + memset(iov1, 0, (uint32)total_size); + + p = (char *)(uintptr_t)(sizeof(struct iovec) * iovcnt); + + for (i = 0; i < iovcnt; i++) { + iov1[i].iov_len = iov[i].iov_len; + iov1[i].iov_base = p; + memcpy((uintptr_t)p + (char *)iov1, iov[i].iov_base, iov[i].iov_len); + p += iov[i].iov_len; + } + + if (ocall_writev(&ret, fd, (char *)iov1, (uint32)total_size, iovcnt, + has_offset, offset) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + BH_FREE(iov1); + return -1; + } + + BH_FREE(iov1); + if (ret == -1) + errno = get_errno(); + return ret; +} + +ssize_t +readv(int fd, const struct iovec *iov, int iovcnt) +{ + return readv_internal(fd, iov, iovcnt, false, 0); +} + +ssize_t +writev(int fd, const struct iovec *iov, int iovcnt) +{ + return writev_internal(fd, iov, iovcnt, false, 0); +} + +ssize_t +preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset) +{ + return readv_internal(fd, iov, iovcnt, true, offset); +} + +ssize_t +pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset) +{ + return writev_internal(fd, iov, iovcnt, true, offset); +} + +off_t +lseek(int fd, off_t offset, int whence) +{ + off_t ret; + +#if WASM_ENABLE_SGX_IPFS != 0 + ret = ipfs_lseek(fd, offset, whence); +#else + if (ocall_lseek(&ret, fd, (long)offset, whence) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) + errno = get_errno(); +#endif + + return ret; +} + +int +ftruncate(int fd, off_t length) +{ + int ret; + +#if WASM_ENABLE_SGX_IPFS != 0 + ret = ipfs_ftruncate(fd, length); +#else + if (ocall_ftruncate(&ret, fd, length) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) + errno = get_errno(); +#endif + + return ret; +} + +int +stat(const char *pathname, struct stat *statbuf) +{ + int ret; + + if (statbuf == NULL) + return -1; + + if (ocall_stat(&ret, pathname, (void *)statbuf, sizeof(struct stat)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +fstat(int fd, struct stat *statbuf) +{ + int ret; + + if (statbuf == NULL) + return -1; + + if (ocall_fstat(&ret, fd, (void *)statbuf, sizeof(struct stat)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +fstatat(int dirfd, const char *pathname, struct stat *statbuf, int flags) +{ + int ret; + + if (statbuf == NULL) + return -1; + + if (ocall_fstatat(&ret, dirfd, pathname, (void *)statbuf, + sizeof(struct stat), flags) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +fsync(int fd) +{ + int ret; + +#if WASM_ENABLE_SGX_IPFS != 0 + ret = ipfs_fflush(fd); +#else + if (ocall_fsync(&ret, fd) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) + errno = get_errno(); +#endif + + return ret; +} + +int +fdatasync(int fd) +{ + int ret; + +#if WASM_ENABLE_SGX_IPFS != 0 + ret = ipfs_fflush(fd); +#else + if (ocall_fdatasync(&ret, fd) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) + errno = get_errno(); +#endif + + return ret; +} + +int +mkdirat(int dirfd, const char *pathname, mode_t mode) +{ + int ret; + + if (ocall_mkdirat(&ret, dirfd, pathname, mode) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +link(const char *oldpath, const char *newpath) +{ + int ret; + + if (ocall_link(&ret, oldpath, newpath) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, + int flags) +{ + int ret; + + if (ocall_linkat(&ret, olddirfd, oldpath, newdirfd, newpath, flags) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +unlinkat(int dirfd, const char *pathname, int flags) +{ + int ret; + + if (ocall_unlinkat(&ret, dirfd, pathname, flags) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +ssize_t +readlink(const char *pathname, char *buf, size_t bufsiz) +{ + ssize_t ret; + + if (buf == NULL) + return -1; + + if (ocall_readlink(&ret, pathname, buf, bufsiz) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +ssize_t +readlinkat(int dirfd, const char *pathname, char *buf, size_t bufsiz) +{ + ssize_t ret; + + if (buf == NULL) + return -1; + + if (ocall_readlinkat(&ret, dirfd, pathname, buf, bufsiz) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +symlinkat(const char *target, int newdirfd, const char *linkpath) +{ + int ret; + + if (ocall_symlinkat(&ret, target, newdirfd, linkpath) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +renameat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath) +{ + int ret; + + if (ocall_renameat(&ret, olddirfd, oldpath, newdirfd, newpath) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +ioctl(int fd, unsigned long request, ...) +{ + int ret; + va_list args; + + switch (request) { + case FIONREAD: + va_start(args, request); + int *arg = (int *)va_arg(args, int *); + if (ocall_ioctl(&ret, fd, request, arg, sizeof(*arg)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + va_end(args); + return -1; + } + va_end(args); + break; + + default: + os_printf("ioctl failed: unknown request", request); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +fcntl(int fd, int cmd, ... /* arg */) +{ + int ret; + va_list args; + + switch (cmd) { + case F_GETFD: + case F_GETFL: + if (ocall_fcntl(&ret, fd, cmd) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + break; + + case F_DUPFD: + case F_SETFD: + case F_SETFL: + va_start(args, cmd); + long arg_1 = (long)va_arg(args, long); + if (ocall_fcntl_long(&ret, fd, cmd, arg_1) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + va_end(args); + return -1; + } + va_end(args); + break; + + default: + os_printf("fcntl failed: unknown cmd %d.\n", cmd); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +isatty(int fd) +{ + int ret; + + if (ocall_isatty(&ret, fd) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == 0) + errno = get_errno(); + return ret; +} + +char * +realpath(const char *path, char *resolved_path) +{ + int ret; + char buf[PATH_MAX] = { 0 }; + + if (ocall_realpath(&ret, path, buf, PATH_MAX) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return (char *)NULL; + } + + if (ret != 0) + return (char *)NULL; + + if (resolved_path) { + strcpy(resolved_path, buf); + } + else { + resolved_path = BH_MALLOC(strlen(buf) + 1); + if (resolved_path == NULL) + return NULL; + strcpy(resolved_path, buf); + } + + return resolved_path; +} + +int +posix_fallocate(int fd, off_t offset, off_t len) +{ + int ret; + +#if WASM_ENABLE_SGX_IPFS != 0 + ret = ipfs_posix_fallocate(fd, offset, len); +#else + if (ocall_posix_fallocate(&ret, fd, offset, len) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } +#endif + + return ret; +} + +int +poll(struct pollfd *fds, nfds_t nfds, int timeout) +{ + int ret; + + if (fds == NULL) + return -1; + + if (ocall_poll(&ret, fds, nfds, timeout, sizeof(*fds) * nfds) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +getopt(int argc, char *const argv[], const char *optstring) +{ + int ret; + char **argv1; + char *p; + int i; + uint64 total_size = sizeof(char *) * (uint64)argc; + + for (i = 0; i < argc; i++) { + total_size += strlen(argv[i]) + 1; + } + + if (total_size >= UINT32_MAX) + return -1; + + argv1 = BH_MALLOC((uint32)total_size); + + if (argv1 == NULL) + return -1; + + p = (char *)(uintptr_t)(sizeof(char *) * argc); + + for (i = 0; i < argc; i++) { + argv1[i] = p; + strcpy((char *)argv1 + (uintptr_t)p, argv[i]); + p += ((uintptr_t)strlen(argv[i]) + 1); + } + + if (ocall_getopt(&ret, argc, (char *)argv1, total_size, optstring) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + BH_FREE(argv1); + return -1; + } + + BH_FREE(argv1); + if (ret == -1) + errno = get_errno(); + return ret; +} + +int +sched_yield(void) +{ + int ret; + + if (ocall_sched_yield(&ret) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + if (ret == -1) + errno = get_errno(); + return ret; +} + +ssize_t +getrandom(void *buf, size_t buflen, unsigned int flags) +{ + sgx_status_t ret; + + if (!buf || buflen > INT32_MAX || flags != 0) { + errno = EINVAL; + return -1; + } + + ret = sgx_read_rand(buf, buflen); + if (ret != SGX_SUCCESS) { + errno = EFAULT; + return -1; + } + + return (ssize_t)buflen; +} + +#define RDRAND_RETRIES 3 + +static int +rdrand64_step(uint64 *seed) +{ + uint8 ok; + __asm__ volatile("rdseed %0; setc %1" : "=r"(*seed), "=qm"(ok)); + return (int)ok; +} + +static int +rdrand64_retry(uint64 *rand, uint32 retries) +{ + uint32 count = 0; + + while (count++ <= retries) { + if (rdrand64_step(rand)) { + return -1; + } + } + return 0; +} + +static uint32 +rdrand_get_bytes(uint8 *dest, uint32 n) +{ + uint8 *head_start = dest, *tail_start = NULL; + uint64 *block_start; + uint32 count, ltail, lhead, lblock; + uint64 i, temp_rand; + + /* Get the address of the first 64-bit aligned block in the + destination buffer. */ + if (((uintptr_t)head_start & (uintptr_t)7) == 0) { + /* already 8-byte aligned */ + block_start = (uint64 *)head_start; + lhead = 0; + lblock = n & ~7; + } + else { + /* next 8-byte aligned */ + block_start = (uint64 *)(((uintptr_t)head_start + 7) & ~(uintptr_t)7); + lhead = (uint32)((uintptr_t)block_start - (uintptr_t)head_start); + lblock = (n - lhead) & ~7; + } + + /* Compute the number of 64-bit blocks and the remaining number + of bytes (the tail) */ + ltail = n - lblock - lhead; + if (ltail > 0) { + tail_start = (uint8 *)block_start + lblock; + } + + /* Populate the starting, mis-aligned section (the head) */ + if (lhead > 0) { + if (!rdrand64_retry(&temp_rand, RDRAND_RETRIES)) { + return 0; + } + memcpy(head_start, &temp_rand, lhead); + } + + /* Populate the central, aligned blocks */ + count = lblock / 8; + for (i = 0; i < count; i++, block_start++) { + if (!rdrand64_retry(block_start, RDRAND_RETRIES)) { + return i * 8 + lhead; + } + } + + /* Populate the tail */ + if (ltail > 0) { + if (!rdrand64_retry(&temp_rand, RDRAND_RETRIES)) { + return count * 8 + lhead; + } + + memcpy(tail_start, &temp_rand, ltail); + } + + return n; +} + +int +getentropy(void *buffer, size_t length) +{ + uint32 size; + + if (!buffer || length > INT32_MAX) { + errno = EINVAL; + return -1; + } + + if (length == 0) { + return 0; + } + + size = rdrand_get_bytes(buffer, (uint32)length); + if (size != length) { + errno = EFAULT; + return -1; + } + + return 0; +} + +int +get_errno(void) +{ + int ret; + + if (ocall_get_errno(&ret) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + return ret; +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_file.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_file.h new file mode 100644 index 0000000..8690e1f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_file.h @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SGX_FILE_H +#define _SGX_FILE_H + +#include "sgx_time.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define F_DUPFD 0 +#define F_GETFD 1 +#define F_SETFD 2 +#define F_GETFL 3 +#define F_SETFL 4 + +#define FD_CLOEXEC 1 + +#define O_PATH 010000000 +#define O_SEARCH O_PATH +#define O_EXEC O_PATH + +#define O_ACCMODE (03 | O_SEARCH) +#define O_RDONLY 00 +#define O_WRONLY 01 +#define O_RDWR 02 + +#define O_CREAT 0100 +#define O_EXCL 0200 +#define O_NOCTTY 0400 +#define O_TRUNC 01000 +#define O_APPEND 02000 +#define O_NONBLOCK 04000 +#define O_DSYNC 010000 +#define O_SYNC 04010000 +#define O_RSYNC 04010000 +#define O_DIRECTORY 0200000 +#define O_NOFOLLOW 0400000 +#define O_CLOEXEC 02000000 + +#define O_ASYNC 020000 +#define O_DIRECT 040000 +#define O_LARGEFILE 0 +#define O_NOATIME 01000000 +#define O_PATH 010000000 +#define O_TMPFILE 020200000 +#define O_NDELAY O_NONBLOCK + +#define S_IFMT 0170000 +#define S_IFDIR 0040000 +#define S_IFCHR 0020000 +#define S_IFBLK 0060000 +#define S_IFREG 0100000 +#define S_IFIFO 0010000 +#define S_IFLNK 0120000 +#define S_IFSOCK 0140000 + +#define SEEK_SET 0 +#define SEEK_CUR 1 +#define SEEK_END 2 + +#define S_ISDIR(mode) (((mode)&S_IFMT) == S_IFDIR) +#define S_ISCHR(mode) (((mode)&S_IFMT) == S_IFCHR) +#define S_ISBLK(mode) (((mode)&S_IFMT) == S_IFBLK) +#define S_ISREG(mode) (((mode)&S_IFMT) == S_IFREG) +#define S_ISFIFO(mode) (((mode)&S_IFMT) == S_IFIFO) +#define S_ISLNK(mode) (((mode)&S_IFMT) == S_IFLNK) +#define S_ISSOCK(mode) (((mode)&S_IFMT) == S_IFSOCK) + +#define DT_UNKNOWN 0 +#define DT_FIFO 1 +#define DT_CHR 2 +#define DT_DIR 4 +#define DT_BLK 6 +#define DT_REG 8 +#define DT_LNK 10 +#define DT_SOCK 12 +#define DT_WHT 14 + +#define AT_SYMLINK_NOFOLLOW 0x100 +#define AT_REMOVEDIR 0x200 +#define AT_SYMLINK_FOLLOW 0x400 + +#define POLLIN 0x001 +#define POLLPRI 0x002 +#define POLLOUT 0x004 +#define POLLERR 0x008 +#define POLLHUP 0x010 +#define POLLNVAL 0x020 +#define POLLRDNORM 0x040 +#define POLLRDBAND 0x080 +#define POLLWRNORM 0x100 +#define POLLWRBAND 0x200 + +#define FIONREAD 0x541B + +#define PATH_MAX 4096 + +/* Special value used to indicate openat should use the current + working directory. */ +#define AT_FDCWD -100 + +typedef long __syscall_slong_t; + +typedef unsigned long dev_t; +typedef unsigned long ino_t; +typedef unsigned mode_t; +typedef unsigned long nlink_t; +typedef unsigned socklen_t; +typedef long blksize_t; +typedef long blkcnt_t; + +typedef int pid_t; +typedef unsigned gid_t; +typedef unsigned uid_t; + +typedef unsigned long nfds_t; + +typedef uintptr_t DIR; + +struct dirent { + ino_t d_ino; + off_t d_off; + unsigned short d_reclen; + unsigned char d_type; + char d_name[256]; +}; + +struct stat { + dev_t st_dev; + ino_t st_ino; + nlink_t st_nlink; + + mode_t st_mode; + uid_t st_uid; + gid_t st_gid; + unsigned int __pad0; + dev_t st_rdev; + off_t st_size; + blksize_t st_blksize; + blkcnt_t st_blocks; + + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; + long __unused[3]; +}; + +struct iovec { + void *iov_base; + size_t iov_len; +}; + +struct pollfd { + int fd; + short events; + short revents; +}; + +int +open(const char *pathname, int flags, ...); +int +openat(int dirfd, const char *pathname, int flags, ...); +int +close(int fd); + +DIR * +fdopendir(int fd); +int +closedir(DIR *dirp); +void +rewinddir(DIR *dirp); +void +seekdir(DIR *dirp, long loc); +struct dirent * +readdir(DIR *dirp); +long +telldir(DIR *dirp); + +ssize_t +read(int fd, void *buf, size_t count); +ssize_t +readv(int fd, const struct iovec *iov, int iovcnt); +ssize_t +writev(int fd, const struct iovec *iov, int iovcnt); +ssize_t +preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset); +ssize_t +pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset); + +off_t +lseek(int fd, off_t offset, int whence); +int +ftruncate(int fd, off_t length); + +int +stat(const char *pathname, struct stat *statbuf); +int +fstat(int fd, struct stat *statbuf); +int +fstatat(int dirfd, const char *pathname, struct stat *statbuf, int flags); + +int +fsync(int fd); +int +fdatasync(int fd); + +int +mkdirat(int dirfd, const char *pathname, mode_t mode); +int +link(const char *oldpath, const char *newpath); +int +linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, + int flags); +int +unlinkat(int dirfd, const char *pathname, int flags); +ssize_t +readlink(const char *pathname, char *buf, size_t bufsiz); +ssize_t +readlinkat(int dirfd, const char *pathname, char *buf, size_t bufsiz); +int +symlinkat(const char *target, int newdirfd, const char *linkpath); +int +renameat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath); + +int +ioctl(int fd, unsigned long request, ...); +int +fcntl(int fd, int cmd, ... /* arg */); + +int +isatty(int fd); + +char * +realpath(const char *path, char *resolved_path); + +int +posix_fallocate(int fd, off_t offset, off_t len); + +int +poll(struct pollfd *fds, nfds_t nfds, int timeout); + +int +getopt(int argc, char *const argv[], const char *optstring); + +int +sched_yield(void); + +ssize_t +getrandom(void *buf, size_t buflen, unsigned int flags); + +int +getentropy(void *buffer, size_t length); + +int +get_errno(void); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _SGX_FILE_H */ diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_ipfs.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_ipfs.c new file mode 100644 index 0000000..4f4bbef --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_ipfs.c @@ -0,0 +1,532 @@ +/* + * Copyright (C) 2022 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#if WASM_ENABLE_SGX_IPFS != 0 + +#include "ssp_config.h" +#include "bh_platform.h" +#include "sgx_ipfs.h" + +#include + +#include "sgx_tprotected_fs.h" + +#define SGX_ERROR_FILE_LOWEST_ERROR_ID SGX_ERROR_FILE_BAD_STATUS +#define SGX_ERROR_FILE_HIGHEST_ERROR_ID SGX_ERROR_FILE_CLOSE_FAILED + +// Internal buffer filled with zeroes and used when extending the size of +// protected files. +#define ZEROES_PADDING_LENGTH 32 * 1024 +char zeroes_padding[ZEROES_PADDING_LENGTH] = { 0 }; + +// The mapping between file descriptors and IPFS file pointers. +static HashMap *ipfs_file_list; + +// Converts an SGX error code to a POSIX error code. +static __wasi_errno_t +convert_sgx_errno(int error) +{ + if (error >= SGX_ERROR_FILE_LOWEST_ERROR_ID + && error <= SGX_ERROR_FILE_HIGHEST_ERROR_ID) { + switch (error) { + /* The file is in bad status */ + case SGX_ERROR_FILE_BAD_STATUS: + return ENOTRECOVERABLE; + /* The Key ID field is all zeros, can't re-generate the encryption + * key */ + case SGX_ERROR_FILE_NO_KEY_ID: + return EKEYREJECTED; + /* The current file name is different then the original file name + * (not allowed, substitution attack) */ + case SGX_ERROR_FILE_NAME_MISMATCH: + return EIO; + /* The file is not an SGX file */ + case SGX_ERROR_FILE_NOT_SGX_FILE: + return EEXIST; + /* A recovery file can't be opened, so flush operation can't + * continue (only used when no EXXX is returned) */ + case SGX_ERROR_FILE_CANT_OPEN_RECOVERY_FILE: + return EIO; + /* A recovery file can't be written, so flush operation can't + * continue (only used when no EXXX is returned) */ + case SGX_ERROR_FILE_CANT_WRITE_RECOVERY_FILE: + return EIO; + /* When openeing the file, recovery is needed, but the recovery + * process failed */ + case SGX_ERROR_FILE_RECOVERY_NEEDED: + return EIO; + /* fflush operation (to disk) failed (only used when no EXXX is + * returned) */ + case SGX_ERROR_FILE_FLUSH_FAILED: + return EIO; + /* fclose operation (to disk) failed (only used when no EXXX is + * returned) */ + case SGX_ERROR_FILE_CLOSE_FAILED: + return EIO; + } + } + + return error; +} + +static void * +fd2file(int fd) +{ + return bh_hash_map_find(ipfs_file_list, (void *)(intptr_t)fd); +} + +static void +ipfs_file_destroy(void *sgx_file) +{ + sgx_fclose(sgx_file); +} + +// Writes a given number of zeroes in file at the current offset. +// The return value is zero if successful; otherwise non-zero. +static int +ipfs_write_zeroes(void *sgx_file, size_t len) +{ + int min_count; + + while (len > 0) { + min_count = len < ZEROES_PADDING_LENGTH ? len : ZEROES_PADDING_LENGTH; + + if (sgx_fwrite(zeroes_padding, 1, min_count, sgx_file) == 0) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + len -= min_count; + } + + return 0; +} + +int +ipfs_init() +{ + ipfs_file_list = + bh_hash_map_create(32, true, (HashFunc)fd_hash, (KeyEqualFunc)fd_equal, + NULL, (ValueDestroyFunc)ipfs_file_destroy); + + return ipfs_file_list != NULL ? BHT_OK : BHT_ERROR; +} + +void +ipfs_destroy() +{ + bh_hash_map_destroy(ipfs_file_list); +} + +int +ipfs_posix_fallocate(int fd, off_t offset, size_t len) +{ + void *sgx_file = fd2file(fd); + if (!sgx_file) { + return EBADF; + } + + // The wrapper for fseek takes care of extending the file if sought beyond + // the end + if (ipfs_lseek(fd, offset + len, SEEK_SET) == -1) { + return errno; + } + + // Make sure the file is allocated by flushing it + if (sgx_fflush(sgx_file) != 0) { + return errno; + } + + return 0; +} + +size_t +ipfs_read(int fd, const struct iovec *iov, int iovcnt, bool has_offset, + off_t offset) +{ + int i; + off_t original_offset = 0; + void *sgx_file = fd2file(fd); + size_t read_result, number_of_read_bytes = 0; + + if (!sgx_file) { + errno = EBADF; + return -1; + } + + if (has_offset) { + // Save the current offset, to restore it after the read operation + original_offset = (off_t)sgx_ftell(sgx_file); + + if (original_offset == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + // Move to the desired location + if (sgx_fseek(sgx_file, offset, SEEK_SET) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + } + + // For each element in the vector + for (i = 0; i < iovcnt; i++) { + if (iov[i].iov_len == 0) + continue; + + read_result = sgx_fread(iov[i].iov_base, 1, iov[i].iov_len, sgx_file); + number_of_read_bytes += read_result; + + if (read_result != iov[i].iov_len) { + if (!sgx_feof(sgx_file)) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + } + } + + if (has_offset) { + // Restore the position of the cursor + if (sgx_fseek(sgx_file, original_offset, SEEK_SET) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + } + + return number_of_read_bytes; +} + +size_t +ipfs_write(int fd, const struct iovec *iov, int iovcnt, bool has_offset, + off_t offset) +{ + int i; + off_t original_offset = 0; + void *sgx_file = fd2file(fd); + size_t write_result, number_of_written_bytes = 0; + + if (!sgx_file) { + errno = EBADF; + return -1; + } + + if (has_offset) { + // Save the current offset, to restore it after the read operation + original_offset = (off_t)sgx_ftell(sgx_file); + + if (original_offset == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + // Move to the desired location + if (sgx_fseek(sgx_file, offset, SEEK_SET) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + } + + // For each element in the vector + for (i = 0; i < iovcnt; i++) { + if (iov[i].iov_len == 0) + continue; + + write_result = sgx_fwrite(iov[i].iov_base, 1, iov[i].iov_len, sgx_file); + number_of_written_bytes += write_result; + + if (write_result != iov[i].iov_len) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + } + + if (has_offset) { + // Restore the position of the cursor + if (sgx_fseek(sgx_file, original_offset, SEEK_SET) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + } + + return number_of_written_bytes; +} + +int +ipfs_close(int fd) +{ + void *sgx_file; + + if (!bh_hash_map_remove(ipfs_file_list, (void *)(intptr_t)fd, NULL, + &sgx_file)) { + errno = EBADF; + return -1; + } + + if (sgx_fclose(sgx_file)) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + return 0; +} + +void * +ipfs_fopen(int fd, int flags) +{ + // Mapping back the mode + const char *mode; + + bool must_create = (flags & O_CREAT) != 0; + bool must_truncate = (flags & O_TRUNC) != 0; + bool must_append = (flags & O_APPEND) != 0; + bool read_only = (flags & O_ACCMODE) == O_RDONLY; + bool write_only = (flags & O_ACCMODE) == O_WRONLY; + bool read_write = (flags & O_ACCMODE) == O_RDWR; + + // The mapping of the mode is similar to the table in the official + // specifications: + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/fopen.html + // Note that POSIX has obtained a file descriptor beforehand. + // If opened with a destructive mode ("w" or "w+"), the truncate operation + // already occurred and must not be repeated because this will invalidate + // the file descriptor obtained by POSIX. Therefore, we do NOT map to the + // modes that truncate the file ("w" and "w+"). Instead, we map to a + // non-destructive mode ("r+"). + + if (read_only) + mode = "r"; + else if (write_only && must_create && must_truncate) + // Rather than "w", we map to a non-destructive mode + mode = "r+"; + else if (write_only && must_create && must_append) + mode = "a"; + else if (read_write && must_create && must_append) + mode = "a+"; + else if (read_write) + // Rather than "w+", we map to a non-destructive mode + mode = "r+"; + else + mode = NULL; + + // Cannot map the requested access to the SGX IPFS + if (mode == NULL) { + errno = __WASI_ENOTCAPABLE; + return NULL; + } + + // Determine the symbolic link of the file descriptor, because IPFS does not + // support opening a relative path to a file descriptor (i.e., openat). + // Using the symbolic link in /proc/self allows to retrieve the same path as + // opened by the initial openat and respects the chroot of WAMR. + size_t ret; + char symbolic_path[32]; + ret = + snprintf(symbolic_path, sizeof(symbolic_path), "/proc/self/fd/%d", fd); + if (ret >= sizeof(symbolic_path)) { + errno = ENAMETOOLONG; + return NULL; + } + + // Resolve the symbolic link to real absolute path, because IPFS can only + // open a file with a same file name it was initially created. Otherwise, + // IPFS throws SGX_ERROR_FILE_NAME_MISMATCH. + char real_path[PATH_MAX] = { 0 }; + ret = readlink(symbolic_path, real_path, PATH_MAX - 1); + if (ret == -1) + return NULL; + + // Opening the file using the real path + void *sgx_file = sgx_fopen_auto_key(real_path, mode); + + if (sgx_file == NULL) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return NULL; + } + + if (!bh_hash_map_insert(ipfs_file_list, (void *)(intptr_t)fd, sgx_file)) { + errno = __WASI_ECANCELED; + sgx_fclose(sgx_file); + os_printf("An error occurred while inserting the IPFS file pointer in " + "the map.\n"); + return NULL; + } + + return sgx_file; +} + +int +ipfs_fflush(int fd) +{ + void *sgx_file = fd2file(fd); + + if (!sgx_file) { + errno = EBADF; + return EOF; + } + + int ret = sgx_fflush(sgx_file); + + if (ret == 1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return EOF; + } + + return ret; +} + +off_t +ipfs_lseek(int fd, off_t offset, int nwhence) +{ + off_t cursor_current_location; + void *sgx_file = fd2file(fd); + if (!sgx_file) { + errno = EBADF; + return -1; + } + + // Optimization: if the offset is 0 and the whence is SEEK_CUR, + // this is equivalent of a call to ftell. + if (offset == 0 && nwhence == SEEK_CUR) { + cursor_current_location = (off_t)sgx_ftell(sgx_file); + + if (cursor_current_location == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + return cursor_current_location; + } + + int fseek_result = sgx_fseek(sgx_file, offset, nwhence); + + if (fseek_result == 0) { + off_t new_offset = (off_t)sgx_ftell(sgx_file); + + if (new_offset == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + return new_offset; + } + else { + // In the case fseek returned an error + int sgx_error = sgx_ferror(sgx_file); + if (sgx_error != EINVAL) { + errno = convert_sgx_errno(sgx_error); + return -1; + } + + // We must consider a difference in behavior of sgx_fseek and the POSIX + // fseek. If the cursor is moved beyond the end of the file, sgx_fseek + // returns an error, whereas POSIX fseek accepts the cursor move and + // fill with zeroes the difference for the next write. This + // implementation handle zeroes completion and moving the cursor forward + // the end of the file, but does it now (during the fseek), which is + // different compared to POSIX implementation, that writes zeroes on the + // next write. This avoids the runtime to keep track of the cursor + // manually. + + // Assume the error is raised because the cursor is moved beyond the end + // of the file. + + // If the whence is the current cursor location, retrieve it + if (nwhence == SEEK_CUR) { + cursor_current_location = (off_t)sgx_ftell(sgx_file); + } + + // Move the cursor at the end of the file + if (sgx_fseek(sgx_file, 0, SEEK_END) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + // Compute the number of zeroes to append. + int64_t number_of_zeroes; + switch (nwhence) { + case SEEK_SET: + number_of_zeroes = offset - sgx_ftell(sgx_file); + break; + case SEEK_END: + number_of_zeroes = offset; + break; + case SEEK_CUR: + number_of_zeroes = + cursor_current_location + offset - sgx_ftell(sgx_file); + break; + default: + errno = EINVAL; + return -1; + } + + // Write the missing zeroes + if (ipfs_write_zeroes(sgx_file, number_of_zeroes) != 0) { + return -1; + } + + // Move again at the end of the file + if (sgx_fseek(sgx_file, 0, SEEK_END) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + return offset; + } +} + +// The official API does not provide a way to truncate files. +// Only files extension is supported. +int +ipfs_ftruncate(int fd, off_t len) +{ + void *sgx_file = fd2file(fd); + if (!sgx_file) { + errno = EBADF; + return -1; + } + + off_t original_offset = sgx_ftell(sgx_file); + + // Optimization path: if the length is smaller than the offset, + // IPFS does not support truncate to a smaller size. + if (len < original_offset) { + os_printf( + "SGX IPFS does not support truncate files to smaller sizes.\n"); + return __WASI_ECANCELED; + } + + // Move to the end of the file to determine whether this is + // a file extension or reduction. + if (sgx_fseek(sgx_file, 0, SEEK_END) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + off_t file_size = sgx_ftell(sgx_file); + + // Reducing the file space is not supported by IPFS. + if (len < file_size) { + os_printf( + "SGX IPFS does not support truncate files to smaller sizes.\n"); + return __WASI_ECANCELED; + } + + // Increasing the size is equal to writing from the end of the file + // with null bytes. + if (ipfs_write_zeroes(sgx_file, len - file_size) != 0) { + return -1; + } + + // Restore the position of the cursor + if (sgx_fseek(sgx_file, original_offset, SEEK_SET) == -1) { + errno = convert_sgx_errno(sgx_ferror(sgx_file)); + return -1; + } + + return 0; +} + +#endif /* end of WASM_ENABLE_SGX_IPFS */ diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_ipfs.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_ipfs.h new file mode 100644 index 0000000..3a911d2 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_ipfs.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _LIBC_WASI_SGX_PFS_H +#define _LIBC_WASI_SGX_PFS_H + +#include "bh_hashmap.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int +ipfs_init(); +void +ipfs_destroy(); +int +ipfs_posix_fallocate(int fd, off_t offset, size_t len); +size_t +ipfs_read(int fd, const struct iovec *iov, int iovcnt, bool has_offset, + off_t offset); +size_t +ipfs_write(int fd, const struct iovec *iov, int iovcnt, bool has_offset, + off_t offset); +int +ipfs_close(int fd); +void * +ipfs_fopen(int fd, int flags); +int +ipfs_fflush(int fd); +off_t +ipfs_lseek(int fd, off_t offset, int nwhence); +int +ipfs_ftruncate(int fd, off_t len); + +/** + * Whether two file descriptors are equal. + */ +inline static bool +fd_equal(int left, int right) +{ + return left == right ? true : false; +} + +/** + * Returns the file descriptor as a hash value. + */ +inline static uint32 +fd_hash(int fd) +{ + return (uint32)fd; +} + +#ifdef __cplusplus +} +#endif + +#endif /* end of _LIBC_WASI_SGX_PFS_H */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_platform.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_platform.c new file mode 100644 index 0000000..d97883a --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_platform.c @@ -0,0 +1,221 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" +#include "sgx_rsrv_mem_mngr.h" + +#if WASM_ENABLE_SGX_IPFS != 0 +#include "sgx_ipfs.h" +#endif + +static os_print_function_t print_function = NULL; + +int +bh_platform_init() +{ + int ret = BHT_OK; + +#if WASM_ENABLE_SGX_IPFS != 0 + ret = ipfs_init(); +#endif + + return ret; +} + +void +bh_platform_destroy() +{ +#if WASM_ENABLE_SGX_IPFS != 0 + ipfs_destroy(); +#endif +} + +void * +os_malloc(unsigned size) +{ + return malloc(size); +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return realloc(ptr, size); +} + +void +os_free(void *ptr) +{ + free(ptr); +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} + +int +putchar(int c) +{ + return 0; +} + +int +puts(const char *s) +{ + return 0; +} + +void +os_set_print_function(os_print_function_t pf) +{ + print_function = pf; +} + +#define FIXED_BUFFER_SIZE 4096 + +int +os_printf(const char *message, ...) +{ + int bytes_written = 0; + + if (print_function != NULL) { + char msg[FIXED_BUFFER_SIZE] = { '\0' }; + va_list ap; + va_start(ap, message); + vsnprintf(msg, FIXED_BUFFER_SIZE, message, ap); + va_end(ap); + bytes_written += print_function(msg); + } + + return bytes_written; +} + +int +os_vprintf(const char *format, va_list arg) +{ + int bytes_written = 0; + + if (print_function != NULL) { + char msg[FIXED_BUFFER_SIZE] = { '\0' }; + vsnprintf(msg, FIXED_BUFFER_SIZE, format, arg); + bytes_written += print_function(msg); + } + + return bytes_written; +} + +char * +strcpy(char *dest, const char *src) +{ + const unsigned char *s = src; + unsigned char *d = dest; + + while ((*d++ = *s++)) { + } + return dest; +} + +#if WASM_ENABLE_LIBC_WASI == 0 +bool +os_is_handle_valid(os_file_handle *handle) +{ + assert(handle != NULL); + + return *handle > -1; +} +#else +/* implemented in posix_file.c */ +#endif + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + int mprot = 0; + uint64 aligned_size, page_size; + void *ret = NULL; + sgx_status_t st = 0; + + if (os_is_handle_valid(&file)) { + os_printf("os_mmap(size=%u, prot=0x%x, file=%x) failed: file is not " + "supported.\n", + size, prot, file); + return NULL; + } + + page_size = getpagesize(); + aligned_size = (size + page_size - 1) & ~(page_size - 1); + + if (aligned_size >= UINT32_MAX) + return NULL; + + ret = sgx_alloc_rsrv_mem(aligned_size); + if (ret == NULL) { + os_printf("os_mmap(size=%u, aligned size=%lu, prot=0x%x) failed.\n", + size, aligned_size, prot); + return NULL; + } + + if (prot & MMAP_PROT_READ) + mprot |= SGX_PROT_READ; + if (prot & MMAP_PROT_WRITE) + mprot |= SGX_PROT_WRITE; + if (prot & MMAP_PROT_EXEC) + mprot |= SGX_PROT_EXEC; + + st = sgx_tprotect_rsrv_mem(ret, aligned_size, mprot); + if (st != SGX_SUCCESS) { + os_printf("os_mmap(size=%u, prot=0x%x) failed to set protect.\n", size, + prot); + sgx_free_rsrv_mem(ret, aligned_size); + return NULL; + } + + return ret; +} + +void +os_munmap(void *addr, size_t size) +{ + uint64 aligned_size, page_size; + + page_size = getpagesize(); + aligned_size = (size + page_size - 1) & ~(page_size - 1); + sgx_free_rsrv_mem(addr, aligned_size); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + int mprot = 0; + sgx_status_t st = 0; + uint64 aligned_size, page_size; + + page_size = getpagesize(); + aligned_size = (size + page_size - 1) & ~(page_size - 1); + + if (prot & MMAP_PROT_READ) + mprot |= SGX_PROT_READ; + if (prot & MMAP_PROT_WRITE) + mprot |= SGX_PROT_WRITE; + if (prot & MMAP_PROT_EXEC) + mprot |= SGX_PROT_EXEC; + st = sgx_tprotect_rsrv_mem(addr, aligned_size, mprot); + if (st != SGX_SUCCESS) + os_printf("os_mprotect(addr=0x%" PRIx64 + ", size=%u, prot=0x%x) failed.\n", + (uintptr_t)addr, size, prot); + + return (st == SGX_SUCCESS ? 0 : -1); +} + +void +os_dcache_flush(void) +{} + +void +os_icache_flush(void *start, size_t len) +{} diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_pthread.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_pthread.c new file mode 100644 index 0000000..7801e35 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_pthread.c @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "sgx_pthread.h" +#include "sgx_error.h" + +#ifndef SGX_DISABLE_WASI + +#define TRACE_FUNC() os_printf("undefined %s\n", __FUNCTION__) +#define TRACE_OCALL_FAIL() os_printf("ocall %s failed!\n", __FUNCTION__) + +#ifndef SGX_THREAD_LOCK_INITIALIZER /* defined since sgxsdk-2.11 */ +/* sgxsdk doesn't support pthread_rwlock related APIs until + version 2.11, we implement them by ourselves. */ +int +ocall_pthread_rwlock_init(int *p_ret, void **rwlock, void *attr); + +int +ocall_pthread_rwlock_destroy(int *p_ret, void **rwlock); + +int +ocall_pthread_rwlock_rdlock(int *p_ret, void **rwlock); + +int +ocall_pthread_rwlock_wrlock(int *p_ret, void **rwlock); + +int +ocall_pthread_rwlock_unlock(int *p_ret, void **rwlock); + +int +pthread_rwlock_init(pthread_rwlock_t *rwlock, void *attr) +{ + int ret = -1; + + if (ocall_pthread_rwlock_init(&ret, (void **)rwlock, NULL) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + (void)attr; + return ret; +} + +int +pthread_rwlock_destroy(pthread_rwlock_t *rwlock) +{ + int ret = -1; + + if (ocall_pthread_rwlock_destroy(&ret, (void *)*rwlock) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } + return ret; +} + +int +pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) +{ + int ret = -1; + + if (ocall_pthread_rwlock_rdlock(&ret, (void *)*rwlock) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } + return ret; +} + +int +pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) +{ + int ret = -1; + + if (ocall_pthread_rwlock_wrlock(&ret, (void *)*rwlock) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } + return ret; +} + +int +pthread_rwlock_unlock(pthread_rwlock_t *rwlock) +{ + int ret = -1; + + if (ocall_pthread_rwlock_unlock(&ret, (void *)*rwlock) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + } + return ret; +} +#endif /* end of SGX_THREAD_LOCK_INITIALIZER */ + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_pthread.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_pthread.h new file mode 100644 index 0000000..01a3ae0 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_pthread.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SGX_PTHREAD_H +#define _SGX_PTHREAD_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SGX_THREAD_LOCK_INITIALIZER /* defined since sgxsdk-2.11 */ +/* sgxsdk doesn't support pthread_rwlock related APIs until + version 2.11, we implement them by ourselves. */ +typedef uintptr_t pthread_rwlock_t; + +int +pthread_rwlock_init(pthread_rwlock_t *rwlock, void *attr); +int +pthread_rwlock_destroy(pthread_rwlock_t *rwlock); + +int +pthread_rwlock_wrlock(pthread_rwlock_t *rwlock); +int +pthread_rwlock_rdlock(pthread_rwlock_t *rwlock); +int +pthread_rwlock_unlock(pthread_rwlock_t *rwlock); +#endif /* end of SGX_THREAD_LOCK_INITIALIZER */ + +#ifdef __cplusplus +} +#endif + +#endif /* end of _SGX_PTHREAD_H */ diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_rsrv_mem_mngr.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_rsrv_mem_mngr.h new file mode 100644 index 0000000..5555d4d --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_rsrv_mem_mngr.h @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2011-2019 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* + * This file is copied from + * https://github.com/intel/linux-sgx/blob/4589daddd58bec7367a6a9de3fe301e6de17671a/common/inc/internal/sgx_rsrv_mem_mngr.h + * The reason we copied here is that the official SGX SDK release has + * not included this header file yet. + */ + +#pragma once + +#ifndef _SGX_RSRV_MEM_MNGR_H_ +#define _SGX_RSRV_MEM_MNGR_H_ + +#include "stdint.h" +#include "sgx_error.h" + +#define SGX_PROT_READ 0x1 /* page can be read */ +#define SGX_PROT_WRITE 0x2 /* page can be written */ +#define SGX_PROT_EXEC 0x4 /* page can be executed */ +#define SGX_PROT_NONE 0x0 /* page can not be accessed */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Allocate a range of EPC memory from the reserved memory area with RW + * permission + * + * Parameters: + * Inputs: length [in]: Size of region to be allocated in bytes. Page aligned. + * Return: Starting address of the new allocated memory area on success; + * otherwise NULL + */ +void * +sgx_alloc_rsrv_mem(size_t length); + +/* Free a range of EPC memory from the reserved memory area + * + * Parameters: + * Inputs: addr[in]: Starting address of region to be freed. Page aligned. + * length[in]: The length of the memory to be freed in bytes. + * Page aligned. + * Return: 0 on success; otherwise -1 + */ +int +sgx_free_rsrv_mem(void *addr, size_t length); + +/* Modify the access permissions of the pages in the reserved memory area. + * + * Parameters: + * Inputs: addr[in]: Starting address of region which needs to change access + * permission. Page aligned. + * length[in]: The length of the memory to be manipulated in bytes. + * Page aligned. + * prot[in]: The target memory protection. + * Return: sgx_status_t - SGX_SUCCESS or failure as defined in sgx_error.h + */ +sgx_status_t +sgx_tprotect_rsrv_mem(void *addr, size_t len, int prot); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_signal.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_signal.c new file mode 100644 index 0000000..b52c188 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_signal.c @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +#ifndef SGX_DISABLE_WASI + +#define TRACE_OCALL_FAIL() os_printf("ocall %s failed!\n", __FUNCTION__) + +int +ocall_raise(int *p_ret, int sig); + +int +raise(int sig) +{ + int ret; + + if (ocall_raise(&ret, sig) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_signal.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_signal.h new file mode 100644 index 0000000..494342b --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_signal.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SGX_SIGNAL_H +#define _SGX_SIGNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Signals. */ +#define SIGHUP 1 /* Hangup (POSIX). */ +#define SIGINT 2 /* Interrupt (ANSI). */ +#define SIGQUIT 3 /* Quit (POSIX). */ +#define SIGILL 4 /* Illegal instruction (ANSI). */ +#define SIGTRAP 5 /* Trace trap (POSIX). */ +#define SIGABRT 6 /* Abort (ANSI). */ +#define SIGIOT 6 /* IOT trap (4.2 BSD). */ +#define SIGBUS 7 /* BUS error (4.2 BSD). */ +#define SIGFPE 8 /* Floating-point exception (ANSI). */ +#define SIGKILL 9 /* Kill, unblockable (POSIX). */ +#define SIGUSR1 10 /* User-defined signal 1 (POSIX). */ +#define SIGSEGV 11 /* Segmentation violation (ANSI). */ +#define SIGUSR2 12 /* User-defined signal 2 (POSIX). */ +#define SIGPIPE 13 /* Broken pipe (POSIX). */ +#define SIGALRM 14 /* Alarm clock (POSIX). */ +#define SIGTERM 15 /* Termination (ANSI). */ +#define SIGSTKFLT 16 /* Stack fault. */ +#define SIGCLD SIGCHLD /* Same as SIGCHLD (System V). */ +#define SIGCHLD 17 /* Child status has changed (POSIX). */ +#define SIGCONT 18 /* Continue (POSIX). */ +#define SIGSTOP 19 /* Stop, unblockable (POSIX). */ +#define SIGTSTP 20 /* Keyboard stop (POSIX). */ +#define SIGTTIN 21 /* Background read from tty (POSIX). */ +#define SIGTTOU 22 /* Background write to tty (POSIX). */ +#define SIGURG 23 /* Urgent condition on socket (4.2 BSD). */ +#define SIGXCPU 24 /* CPU limit exceeded (4.2 BSD). */ +#define SIGXFSZ 25 /* File size limit exceeded (4.2 BSD). */ +#define SIGVTALRM 26 /* Virtual alarm clock (4.2 BSD). */ +#define SIGPROF 27 /* Profiling alarm clock (4.2 BSD). */ +#define SIGWINCH 28 /* Window size change (4.3 BSD, Sun). */ +#define SIGPOLL SIGIO /* Pollable event occurred (System V). */ +#define SIGIO 29 /* I/O now possible (4.2 BSD). */ +#define SIGPWR 30 /* Power failure restart (System V). */ +#define SIGSYS 31 /* Bad system call. */ +#define SIGUNUSED 31 + +int +raise(int sig); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _SGX_SIGNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_socket.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_socket.c new file mode 100644 index 0000000..458bb1e --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_socket.c @@ -0,0 +1,1227 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#ifndef SGX_DISABLE_WASI + +#include "libc_errno.h" + +#define TRACE_OCALL_FAIL() os_printf("ocall %s failed!\n", __FUNCTION__) + +/** OCALLs prototypes **/ +int +ocall_accept(int *p_ret, int sockfd, void *addr, uint32_t *addrlen, + uint32_t addr_size); + +int +ocall_bind(int *p_ret, int sockfd, const void *addr, uint32_t addrlen); + +int +ocall_close(int *p_ret, int fd); + +int +ocall_connect(int *p_ret, int sockfd, void *addr, uint32_t addrlen); + +int +ocall_fcntl_long(int *p_ret, int fd, int cmd, long arg); + +int +ocall_getsockname(int *p_ret, int sockfd, void *addr, uint32_t *addrlen, + uint32_t addr_size); + +int +ocall_getpeername(int *p_ret, int sockfd, void *addr, uint32_t *addrlen, + uint32_t addr_size); + +int +ocall_getsockopt(int *p_ret, int sockfd, int level, int optname, void *val_buf, + unsigned int val_buf_size, void *len_buf); + +int +ocall_listen(int *p_ret, int sockfd, int backlog); + +int +ocall_recv(int *p_ret, int sockfd, void *buf, size_t len, int flags); + +int +ocall_recvfrom(ssize_t *p_ret, int sockfd, void *buf, size_t len, int flags, + void *src_addr, uint32_t *addrlen, uint32_t addr_size); + +int +ocall_recvmsg(ssize_t *p_ret, int sockfd, void *msg_buf, + unsigned int msg_buf_size, int flags); + +int +ocall_send(int *p_ret, int sockfd, const void *buf, size_t len, int flags); + +int +ocall_sendto(ssize_t *p_ret, int sockfd, const void *buf, size_t len, int flags, + void *dest_addr, uint32_t addrlen); + +int +ocall_sendmsg(ssize_t *p_ret, int sockfd, void *msg_buf, + unsigned int msg_buf_size, int flags); + +int +ocall_setsockopt(int *p_ret, int sockfd, int level, int optname, void *optval, + unsigned int optlen); + +int +ocall_shutdown(int *p_ret, int sockfd, int how); + +int +ocall_socket(int *p_ret, int domain, int type, int protocol); +/** OCALLs prototypes end **/ + +/** In-enclave implementation of POSIX functions **/ +static bool +is_little_endian() +{ + long i = 0x01020304; + unsigned char *c = (unsigned char *)&i; + return (*c == 0x04) ? true : false; +} + +static void +swap32(uint8 *pData) +{ + uint8 value = *pData; + *pData = *(pData + 3); + *(pData + 3) = value; + + value = *(pData + 1); + *(pData + 1) = *(pData + 2); + *(pData + 2) = value; +} + +static void +swap16(uint8 *pData) +{ + uint8 value = *pData; + *(pData) = *(pData + 1); + *(pData + 1) = value; +} + +uint32 +htonl(uint32 value) +{ + uint32 ret; + if (is_little_endian()) { + ret = value; + swap32((uint8 *)&ret); + return ret; + } + + return value; +} + +uint32 +ntohl(uint32 value) +{ + return htonl(value); +} + +uint16 +htons(uint16 value) +{ + uint16 ret; + if (is_little_endian()) { + ret = value; + swap16((uint8 *)&ret); + return ret; + } + + return value; +} + +static uint16 +ntohs(uint16 value) +{ + return htons(value); +} + +/* Coming from musl, under MIT license */ +static int +hexval(unsigned c) +{ + if (c - '0' < 10) + return c - '0'; + c |= 32; + if (c - 'a' < 6) + return c - 'a' + 10; + return -1; +} + +/* Coming from musl, under MIT license */ +static int +inet_pton(int af, const char *restrict s, void *restrict a0) +{ + uint16_t ip[8]; + unsigned char *a = a0; + int i, j, v, d, brk = -1, need_v4 = 0; + + if (af == AF_INET) { + for (i = 0; i < 4; i++) { + for (v = j = 0; j < 3 && isdigit(s[j]); j++) + v = 10 * v + s[j] - '0'; + if (j == 0 || (j > 1 && s[0] == '0') || v > 255) + return 0; + a[i] = v; + if (s[j] == 0 && i == 3) + return 1; + if (s[j] != '.') + return 0; + s += j + 1; + } + return 0; + } + else if (af != AF_INET6) { + errno = EAFNOSUPPORT; + return -1; + } + + if (*s == ':' && *++s != ':') + return 0; + + for (i = 0;; i++) { + if (s[0] == ':' && brk < 0) { + brk = i; + ip[i & 7] = 0; + if (!*++s) + break; + if (i == 7) + return 0; + continue; + } + for (v = j = 0; j < 4 && (d = hexval(s[j])) >= 0; j++) + v = 16 * v + d; + if (j == 0) + return 0; + ip[i & 7] = v; + if (!s[j] && (brk >= 0 || i == 7)) + break; + if (i == 7) + return 0; + if (s[j] != ':') { + if (s[j] != '.' || (i < 6 && brk < 0)) + return 0; + need_v4 = 1; + i++; + break; + } + s += j + 1; + } + if (brk >= 0) { + memmove(ip + brk + 7 - i, ip + brk, 2 * (i + 1 - brk)); + for (j = 0; j < 7 - i; j++) + ip[brk + j] = 0; + } + for (j = 0; j < 8; j++) { + *a++ = ip[j] >> 8; + *a++ = ip[j]; + } + if (need_v4 && inet_pton(AF_INET, (void *)s, a - 4) <= 0) + return 0; + return 1; +} + +static int +inet_addr(const char *p) +{ + struct in_addr a; + if (!inet_pton(AF_INET, p, &a)) + return -1; + return a.s_addr; +} +/** In-enclave implementation of POSIX functions end **/ + +static int +textual_addr_to_sockaddr(const char *textual, int port, struct sockaddr_in *out) +{ + assert(textual); + + out->sin_family = AF_INET; + out->sin_port = htons(port); + out->sin_addr.s_addr = inet_addr(textual); + + return BHT_OK; +} + +static int +sockaddr_to_bh_sockaddr(const struct sockaddr *sockaddr, socklen_t socklen, + bh_sockaddr_t *bh_sockaddr) +{ + switch (sockaddr->sa_family) { + case AF_INET: + { + struct sockaddr_in *addr = (struct sockaddr_in *)sockaddr; + + assert(socklen >= sizeof(struct sockaddr_in)); + + bh_sockaddr->port = ntohs(addr->sin_port); + bh_sockaddr->addr_buffer.ipv4 = ntohl(addr->sin_addr.s_addr); + bh_sockaddr->is_ipv4 = true; + return BHT_OK; + } + default: + errno = EAFNOSUPPORT; + return BHT_ERROR; + } +} + +static int +bh_sockaddr_to_sockaddr(const bh_sockaddr_t *bh_sockaddr, + struct sockaddr *sockaddr, socklen_t *socklen) +{ + if (bh_sockaddr->is_ipv4) { + struct sockaddr_in *addr = (struct sockaddr_in *)sockaddr; + addr->sin_port = htons(bh_sockaddr->port); + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = htonl(bh_sockaddr->addr_buffer.ipv4); + *socklen = sizeof(*addr); + return BHT_OK; + } + else { + errno = EAFNOSUPPORT; + return BHT_ERROR; + } +} + +static int +os_socket_setbooloption(bh_socket_t socket, int level, int optname, + bool is_enabled) +{ + int option = (int)is_enabled; + int ret; + + if (ocall_setsockopt(&ret, socket, level, optname, &option, sizeof(option)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return BHT_ERROR; + } + + if (ret != 0) { + errno = get_errno(); + return BHT_ERROR; + } + + return BHT_OK; +} + +static int +os_socket_getbooloption(bh_socket_t socket, int level, int optname, + bool *is_enabled) +{ + assert(is_enabled); + + int optval; + socklen_t optval_size = sizeof(optval); + int ret; + if (ocall_getsockopt(&ret, socket, level, optname, &optval, optval_size, + &optval_size) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return BHT_ERROR; + } + + if (ret != 0) { + errno = get_errno(); + return BHT_ERROR; + } + + *is_enabled = (bool)optval; + return BHT_OK; +} + +int +socket(int domain, int type, int protocol) +{ + int ret; + + if (ocall_socket(&ret, domain, type, protocol) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +getsockopt(int sockfd, int level, int optname, void *optval, socklen_t *optlen) +{ + int ret; + unsigned int val_buf_size = *optlen; + + if (ocall_getsockopt(&ret, sockfd, level, optname, optval, val_buf_size, + (void *)optlen) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +setsockopt(int sockfd, int level, int optname, const void *optval, + socklen_t optlen) +{ + int ret; + + if (ocall_setsockopt(&ret, sockfd, level, optname, (void *)optval, optlen) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +ssize_t +sendmsg(int sockfd, const struct msghdr *msg, int flags) +{ + ssize_t ret; + int i; + char *p; + struct msghdr *msg1; + + uint64 total_size = sizeof(struct msghdr) + (uint64)msg->msg_namelen + + (uint64)msg->msg_controllen; + + total_size += sizeof(struct iovec) * (msg->msg_iovlen); + + for (i = 0; i < msg->msg_iovlen; i++) { + total_size += msg->msg_iov[i].iov_len; + } + + if (total_size >= UINT32_MAX) + return -1; + + msg1 = BH_MALLOC((uint32)total_size); + + if (msg1 == NULL) + return -1; + + p = (char *)(uintptr_t)sizeof(struct msghdr); + + if (msg->msg_name != NULL) { + msg1->msg_name = p; + memcpy((uintptr_t)p + (char *)msg1, msg->msg_name, + (size_t)msg->msg_namelen); + p += msg->msg_namelen; + } + + if (msg->msg_control != NULL) { + msg1->msg_control = p; + memcpy((uintptr_t)p + (char *)msg1, msg->msg_control, + (size_t)msg->msg_control); + p += msg->msg_controllen; + } + + if (msg->msg_iov != NULL) { + msg1->msg_iov = (struct iovec *)p; + p += (uintptr_t)(sizeof(struct iovec) * (msg->msg_iovlen)); + + for (i = 0; i < msg->msg_iovlen; i++) { + msg1->msg_iov[i].iov_base = p; + msg1->msg_iov[i].iov_len = msg->msg_iov[i].iov_len; + memcpy((uintptr_t)p + (char *)msg1, msg->msg_iov[i].iov_base, + (size_t)(msg->msg_iov[i].iov_len)); + p += msg->msg_iov[i].iov_len; + } + } + + if (ocall_sendmsg(&ret, sockfd, (void *)msg1, (uint32)total_size, flags) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +ssize_t +recvmsg(int sockfd, struct msghdr *msg, int flags) +{ + ssize_t ret; + int i; + char *p; + struct msghdr *msg1; + + uint64 total_size = sizeof(struct msghdr) + (uint64)msg->msg_namelen + + (uint64)msg->msg_controllen; + + total_size += sizeof(struct iovec) * (msg->msg_iovlen); + + for (i = 0; i < msg->msg_iovlen; i++) { + total_size += msg->msg_iov[i].iov_len; + } + + if (total_size >= UINT32_MAX) + return -1; + + msg1 = BH_MALLOC((uint32)total_size); + + if (msg1 == NULL) + return -1; + + memset(msg1, 0, total_size); + + p = (char *)(uintptr_t)sizeof(struct msghdr); + + if (msg->msg_name != NULL) { + msg1->msg_name = p; + p += msg->msg_namelen; + } + + if (msg->msg_control != NULL) { + msg1->msg_control = p; + p += msg->msg_controllen; + } + + if (msg->msg_iov != NULL) { + msg1->msg_iov = (struct iovec *)p; + p += (uintptr_t)(sizeof(struct iovec) * (msg->msg_iovlen)); + + for (i = 0; i < msg->msg_iovlen; i++) { + msg1->msg_iov[i].iov_base = p; + msg1->msg_iov[i].iov_len = msg->msg_iov[i].iov_len; + p += msg->msg_iov[i].iov_len; + } + } + + if (ocall_recvmsg(&ret, sockfd, (void *)msg1, (uint32)total_size, flags) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + p = (char *)(uintptr_t)(sizeof(struct msghdr)); + + if (msg1->msg_name != NULL) { + memcpy(msg->msg_name, (uintptr_t)p + (char *)msg1, + (size_t)msg1->msg_namelen); + p += msg1->msg_namelen; + } + + if (msg1->msg_control != NULL) { + memcpy(msg->msg_control, (uintptr_t)p + (char *)msg1, + (size_t)msg1->msg_control); + p += msg->msg_controllen; + } + + if (msg1->msg_iov != NULL) { + p += (uintptr_t)(sizeof(struct iovec) * (msg1->msg_iovlen)); + + for (i = 0; i < msg1->msg_iovlen; i++) { + memcpy(msg->msg_iov[i].iov_base, (uintptr_t)p + (char *)msg1, + (size_t)(msg1->msg_iov[i].iov_len)); + p += msg1->msg_iov[i].iov_len; + } + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +shutdown(int sockfd, int how) +{ + int ret; + + if (ocall_shutdown(&ret, sockfd, how) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, + unsigned int *addrlen) + +{ + struct sockaddr addr_tmp; + unsigned int len = sizeof(struct sockaddr); + + if (ocall_accept(sock, server_sock, &addr_tmp, &len, len) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (*sock < 0) { + errno = get_errno(); + return BHT_ERROR; + } + + return BHT_OK; +} +int +os_socket_bind(bh_socket_t socket, const char *host, int *port) +{ + struct sockaddr_in addr; + struct linger ling; + unsigned int socklen; + int ret; + + assert(host); + assert(port); + + ling.l_onoff = 1; + ling.l_linger = 0; + + if (ocall_fcntl_long(&ret, socket, F_SETFD, FD_CLOEXEC) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret < 0) { + goto fail; + } + + if (ocall_setsockopt(&ret, socket, SOL_SOCKET, SO_LINGER, &ling, + sizeof(ling)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret < 0) { + goto fail; + } + + addr.sin_addr.s_addr = inet_addr(host); + addr.sin_port = htons(*port); + addr.sin_family = AF_INET; + + if (ocall_bind(&ret, socket, &addr, sizeof(addr)) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret < 0) { + goto fail; + } + + socklen = sizeof(addr); + + if (ocall_getsockname(&ret, socket, (void *)&addr, &socklen, socklen) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) { + goto fail; + } + + *port = ntohs(addr.sin_port); + + return BHT_OK; + +fail: + errno = get_errno(); + return BHT_ERROR; +} + +int +os_socket_close(bh_socket_t socket) +{ + int ret; + + if (ocall_close(&ret, socket) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +os_socket_connect(bh_socket_t socket, const char *addr, int port) +{ + struct sockaddr_in addr_in = { 0 }; + socklen_t addr_len = sizeof(struct sockaddr_in); + int ret = 0; + + if ((ret = textual_addr_to_sockaddr(addr, port, &addr_in)) < 0) { + return ret; + } + + if (ocall_connect(&ret, socket, &addr_in, addr_len) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +os_socket_create(bh_socket_t *sock, bool is_ipv4, bool is_tcp) +{ + int af; + + if (!sock) { + return BHT_ERROR; + } + + if (is_ipv4) { + af = AF_INET; + } + else { + errno = ENOSYS; + return BHT_ERROR; + } + + if (is_tcp) { + if (ocall_socket(sock, af, SOCK_STREAM, IPPROTO_TCP) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + } + else { + if (ocall_socket(sock, af, SOCK_DGRAM, 0) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + } + + if (*sock == -1) { + errno = get_errno(); + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_inet_network(bool is_ipv4, const char *cp, bh_ip_addr_buffer_t *out) +{ + if (!cp) + return BHT_ERROR; + + if (is_ipv4) { + if (inet_pton(AF_INET, cp, &out->ipv4) != 1) { + return BHT_ERROR; + } + /* Note: ntohl(INADDR_NONE) == INADDR_NONE */ + out->ipv4 = ntohl(out->ipv4); + } + else { + if (inet_pton(AF_INET6, cp, out->ipv6) != 1) { + return BHT_ERROR; + } + for (int i = 0; i < 8; i++) { + out->ipv6[i] = ntohs(out->ipv6[i]); + } + } + + return BHT_OK; +} + +int +os_socket_listen(bh_socket_t socket, int max_client) +{ + int ret; + + if (ocall_listen(&ret, socket, max_client) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +os_socket_recv(bh_socket_t socket, void *buf, unsigned int len) +{ + int ret; + + if (ocall_recv(&ret, socket, buf, len, 0) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + errno = ENOSYS; + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +os_socket_recv_from(bh_socket_t socket, void *buf, unsigned int len, int flags, + bh_sockaddr_t *src_addr) +{ + struct sockaddr_in addr; + socklen_t addr_len = sizeof(addr); + ssize_t ret; + + if (ocall_recvfrom(&ret, socket, buf, len, flags, &addr, &addr_len, + addr_len) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + errno = ENOSYS; + return -1; + } + + if (ret < 0) { + errno = get_errno(); + return ret; + } + + if (src_addr && addr_len > 0) { + if (sockaddr_to_bh_sockaddr((struct sockaddr *)&addr, addr_len, + src_addr) + == BHT_ERROR) { + return -1; + } + } + + return ret; +} + +int +os_socket_send(bh_socket_t socket, const void *buf, unsigned int len) +{ + int ret; + + if (ocall_send(&ret, socket, buf, len, 0) != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + errno = ENOSYS; + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +os_socket_send_to(bh_socket_t socket, const void *buf, unsigned int len, + int flags, const bh_sockaddr_t *dest_addr) +{ + struct sockaddr_in addr; + socklen_t addr_len; + ssize_t ret; + + if (bh_sockaddr_to_sockaddr(dest_addr, (struct sockaddr *)&addr, &addr_len) + == BHT_ERROR) { + return -1; + } + + if (ocall_sendto(&ret, socket, buf, len, flags, (struct sockaddr *)&addr, + addr_len) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + errno = ENOSYS; + return -1; + } + + if (ret == -1) { + errno = get_errno(); + } + + return ret; +} + +__wasi_errno_t +os_socket_shutdown(bh_socket_t socket) +{ + if (shutdown(socket, O_RDWR) != 0) { + return convert_errno(errno); + } + return __WASI_ESUCCESS; +} + +int +os_socket_addr_resolve(const char *host, const char *service, + uint8_t *hint_is_tcp, uint8_t *hint_is_ipv4, + bh_addr_info_t *addr_info, size_t addr_info_size, + size_t *max_info_size) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_addr_local(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + struct sockaddr_in addr; + socklen_t addr_len = sizeof(addr); + int ret; + + if (ocall_getsockname(&ret, socket, (struct sockaddr *)&addr, &addr_len, + addr_len) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return BHT_ERROR; + } + + if (ret != BHT_OK) { + errno = get_errno(); + return BHT_ERROR; + } + + return sockaddr_to_bh_sockaddr((struct sockaddr *)&addr, addr_len, + sockaddr); +} + +int +os_socket_addr_remote(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + struct sockaddr_in addr; + socklen_t addr_len = sizeof(addr); + int ret; + + if (ocall_getpeername(&ret, socket, (void *)&addr, &addr_len, addr_len) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret != BHT_OK) { + errno = get_errno(); + return BHT_ERROR; + } + + return sockaddr_to_bh_sockaddr((struct sockaddr *)&addr, addr_len, + sockaddr); +} + +int +os_socket_set_send_timeout(bh_socket_t socket, uint64 timeout_us) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_send_timeout(bh_socket_t socket, uint64 *timeout_us) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_recv_timeout(bh_socket_t socket, uint64 timeout_us) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_recv_timeout(bh_socket_t socket, uint64 *timeout_us) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_send_buf_size(bh_socket_t socket, size_t bufsiz) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_send_buf_size(bh_socket_t socket, size_t *bufsiz) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_recv_buf_size(bh_socket_t socket, size_t bufsiz) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_recv_buf_size(bh_socket_t socket, size_t *bufsiz) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_keep_alive(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_KEEPALIVE, + is_enabled); +} + +int +os_socket_get_keep_alive(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_KEEPALIVE, + is_enabled); +} + +int +os_socket_set_reuse_addr(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_REUSEADDR, + is_enabled); +} + +int +os_socket_get_reuse_addr(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_REUSEADDR, + is_enabled); +} + +int +os_socket_set_reuse_port(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_REUSEPORT, + is_enabled); +} + +int +os_socket_get_reuse_port(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_REUSEPORT, + is_enabled); +} + +int +os_socket_set_linger(bh_socket_t socket, bool is_enabled, int linger_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_linger(bh_socket_t socket, bool *is_enabled, int *linger_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_no_delay(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, IPPROTO_TCP, TCP_NODELAY, + is_enabled); +} + +int +os_socket_get_tcp_no_delay(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, IPPROTO_TCP, TCP_NODELAY, + is_enabled); +} + +int +os_socket_set_tcp_quick_ack(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, IPPROTO_TCP, TCP_QUICKACK, + is_enabled); +} + +int +os_socket_get_tcp_quick_ack(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, IPPROTO_TCP, TCP_QUICKACK, + is_enabled); +} + +int +os_socket_set_tcp_keep_idle(bh_socket_t socket, uint32 time_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_tcp_keep_idle(bh_socket_t socket, uint32 *time_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_keep_intvl(bh_socket_t socket, uint32 time_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_tcp_keep_intvl(bh_socket_t socket, uint32 *time_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_fastopen_connect(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, + is_enabled); +} + +int +os_socket_get_tcp_fastopen_connect(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, + is_enabled); +} + +int +os_socket_set_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool is_enabled) +{ + if (ipv6) { + return os_socket_setbooloption(socket, IPPROTO_IPV6, + IPV6_MULTICAST_LOOP, is_enabled); + } + else { + return os_socket_setbooloption(socket, IPPROTO_IP, IP_MULTICAST_LOOP, + is_enabled); + } +} + +int +os_socket_get_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool *is_enabled) +{ + if (ipv6) { + return os_socket_getbooloption(socket, IPPROTO_IPV6, + IPV6_MULTICAST_LOOP, is_enabled); + } + else { + return os_socket_getbooloption(socket, IPPROTO_IP, IP_MULTICAST_LOOP, + is_enabled); + } +} + +int +os_socket_set_ip_add_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_drop_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_ttl(bh_socket_t socket, uint8_t ttl_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_ip_ttl(bh_socket_t socket, uint8_t *ttl_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_multicast_ttl(bh_socket_t socket, uint8_t ttl_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_ip_multicast_ttl(bh_socket_t socket, uint8_t *ttl_s) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ipv6_only(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, IPPROTO_IPV6, IPV6_V6ONLY, + is_enabled); +} + +int +os_socket_get_ipv6_only(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, IPPROTO_IPV6, IPV6_V6ONLY, + is_enabled); +} + +int +os_socket_set_broadcast(bh_socket_t socket, bool is_enabled) +{ + return os_socket_setbooloption(socket, SOL_SOCKET, SO_BROADCAST, + is_enabled); +} + +int +os_socket_get_broadcast(bh_socket_t socket, bool *is_enabled) +{ + return os_socket_getbooloption(socket, SOL_SOCKET, SO_BROADCAST, + is_enabled); +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_socket.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_socket.h new file mode 100644 index 0000000..edf977d --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_socket.h @@ -0,0 +1,332 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SGX_SOCKET_H +#define _SGX_SOCKET_H + +#include "sgx_file.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* For setsockopt(2) */ +#define SOL_SOCKET 1 + +#define SO_DEBUG 1 +#define SO_REUSEADDR 2 +#define SO_TYPE 3 +#define SO_ERROR 4 +#define SO_DONTROUTE 5 +#define SO_BROADCAST 6 +#define SO_SNDBUF 7 +#define SO_RCVBUF 8 +#define SO_SNDBUFFORCE 32 +#define SO_RCVBUFFORCE 33 +#define SO_KEEPALIVE 9 +#define SO_OOBINLINE 10 +#define SO_NO_CHECK 11 +#define SO_PRIORITY 12 +#define SO_LINGER 13 +#define SO_BSDCOMPAT 14 +#define SO_REUSEPORT 15 +#define SO_PASSCRED 16 +#define SO_PEERCRED 17 +#define SO_RCVLOWAT 18 +#define SO_SNDLOWAT 19 +#define SO_RCVTIMEO_OLD 20 +#define SO_SNDTIMEO_OLD 21 + +/* User-settable options (used with setsockopt) */ +#define TCP_NODELAY 1 /* Don't delay send to coalesce packets */ +#define TCP_MAXSEG 2 /* Set maximum segment size */ +#define TCP_CORK 3 /* Control sending of partial frames */ +#define TCP_KEEPIDLE 4 /* Start keeplives after this period */ +#define TCP_KEEPINTVL 5 /* Interval between keepalives */ +#define TCP_KEEPCNT 6 /* Number of keepalives before death */ +#define TCP_SYNCNT 7 /* Number of SYN retransmits */ +#define TCP_LINGER2 8 /* Life time of orphaned FIN-WAIT-2 state */ +#define TCP_DEFER_ACCEPT 9 /* Wake up listener only when data arrive */ +#define TCP_WINDOW_CLAMP 10 /* Bound advertised window */ +#define TCP_INFO 11 /* Information about this connection. */ +#define TCP_QUICKACK 12 /* Bock/reenable quick ACKs. */ +#define TCP_CONGESTION 13 /* Congestion control algorithm. */ +#define TCP_MD5SIG 14 /* TCP MD5 Signature (RFC2385) */ +#define TCP_COOKIE_TRANSACTIONS 15 /* TCP Cookie Transactions */ +#define TCP_THIN_LINEAR_TIMEOUTS 16 /* Use linear timeouts for thin streams*/ +#define TCP_THIN_DUPACK 17 /* Fast retrans. after 1 dupack */ +#define TCP_USER_TIMEOUT 18 /* How long for loss retry before timeout */ +#define TCP_REPAIR 19 /* TCP sock is under repair right now */ +#define TCP_REPAIR_QUEUE 20 /* Set TCP queue to repair */ +#define TCP_QUEUE_SEQ 21 /* Set sequence number of repaired queue. */ +#define TCP_REPAIR_OPTIONS 22 /* Repair TCP connection options */ +#define TCP_FASTOPEN 23 /* Enable FastOpen on listeners */ +#define TCP_TIMESTAMP 24 /* TCP time stamp */ +#define TCP_NOTSENT_LOWAT \ + 25 /* Limit number of unsent bytes in write queue. \ + */ +#define TCP_CC_INFO 26 /* Get Congestion Control (optional) info. */ +#define TCP_SAVE_SYN 27 /* Record SYN headers for new connections. */ +#define TCP_SAVED_SYN 28 /* Get SYN headers recorded for connection. */ +#define TCP_REPAIR_WINDOW 29 /* Get/set window parameters. */ +#define TCP_FASTOPEN_CONNECT 30 /* Attempt FastOpen with connect. */ +#define TCP_ULP 31 /* Attach a ULP to a TCP connection. */ +#define TCP_MD5SIG_EXT 32 /* TCP MD5 Signature with extensions. */ +#define TCP_FASTOPEN_KEY 33 /* Set the key for Fast Open (cookie). */ +#define TCP_FASTOPEN_NO_COOKIE 34 /* Enable TFO without a TFO cookie. */ +#define TCP_ZEROCOPY_RECEIVE 35 +#define TCP_INQ 36 /* Notify bytes available to read as a cmsg on read. */ +#define TCP_CM_INQ TCP_INQ +#define TCP_TX_DELAY 37 /* Delay outgoing packets by XX usec. */ + +/* Standard well-defined IP protocols. */ +#define IPPROTO_IP 0 /* Dummy protocol for TCP. */ +#define IPPROTO_ICMP 1 /* Internet Control Message Protocol. */ +#define IPPROTO_IGMP 2 /* Internet Group Management Protocol. */ +#define IPPROTO_IPIP 4 /* IPIP tunnels (older KA9Q tunnels use 94). */ +#define IPPROTO_TCP 6 /* Transmission Control Protocol. */ +#define IPPROTO_EGP 8 /* Exterior Gateway Protocol. */ +#define IPPROTO_PUP 12 /* PUP protocol. */ +#define IPPROTO_UDP 17 /* User Datagram Protocol. */ +#define IPPROTO_IDP 22 /* XNS IDP protocol. */ +#define IPPROTO_TP 29 /* SO Transport Protocol Class 4. */ +#define IPPROTO_DCCP 33 /* Datagram Congestion Control Protocol. */ +#define IPPROTO_IPV6 41 /* IPv6 header. */ +#define IPPROTO_RSVP 46 /* Reservation Protocol. */ +#define IPPROTO_GRE 47 /* General Routing Encapsulation. */ +#define IPPROTO_ESP 50 /* encapsulating security payload. */ +#define IPPROTO_AH 51 /* authentication header. */ +#define IPPROTO_MTP 92 /* Multicast Transport Protocol. */ +#define IPPROTO_BEETPH 94 /* IP option pseudo header for BEET. */ +#define IPPROTO_ENCAP 98 /* Encapsulation Header. */ +#define IPPROTO_PIM 103 /* Protocol Independent Multicast. */ +#define IPPROTO_COMP 108 /* Compression Header Protocol. */ +#define IPPROTO_SCTP 132 /* Stream Control Transmission Protocol. */ +#define IPPROTO_UDPLITE 136 /* UDP-Lite protocol. */ +#define IPPROTO_MPLS 137 /* MPLS in IP. */ +#define IPPROTO_RAW 255 /* Raw IP packets. */ + +#define IP_ROUTER_ALERT 5 /* bool */ +#define IP_PKTINFO 8 /* bool */ +#define IP_PKTOPTIONS 9 +#define IP_PMTUDISC 10 /* obsolete name? */ +#define IP_MTU_DISCOVER 10 /* int; see below */ +#define IP_RECVERR 11 /* bool */ +#define IP_RECVTTL 12 /* bool */ +#define IP_RECVTOS 13 /* bool */ +#define IP_MTU 14 /* int */ +#define IP_FREEBIND 15 +#define IP_IPSEC_POLICY 16 +#define IP_XFRM_POLICY 17 +#define IP_PASSSEC 18 +#define IP_TRANSPARENT 19 +#define IP_MULTICAST_ALL 49 /* bool */ + +/* TProxy original addresses */ +#define IP_ORIGDSTADDR 20 +#define IP_RECVORIGDSTADDR IP_ORIGDSTADDR +#define IP_MINTTL 21 +#define IP_NODEFRAG 22 +#define IP_CHECKSUM 23 +#define IP_BIND_ADDRESS_NO_PORT 24 +#define IP_RECVFRAGSIZE 25 +#define IP_PMTUDISC_DONT 0 +#define IP_PMTUDISC_WANT 1 +#define IP_PMTUDISC_DO 2 +#define IP_PMTUDISC_PROBE 3 +#define IP_PMTUDISC_INTERFACE 4 +#define IP_PMTUDISC_OMIT 5 +#define IP_MULTICAST_IF 32 +#define IP_MULTICAST_TTL 33 +#define IP_MULTICAST_LOOP 34 +#define IP_ADD_MEMBERSHIP 35 +#define IP_DROP_MEMBERSHIP 36 +#define IP_UNBLOCK_SOURCE 37 +#define IP_BLOCK_SOURCE 38 +#define IP_ADD_SOURCE_MEMBERSHIP 39 +#define IP_DROP_SOURCE_MEMBERSHIP 40 +#define IP_MSFILTER 41 +#define IP_MULTICAST_ALL 49 +#define IP_UNICAST_IF 50 + +#define IPV6_ADDRFORM 1 +#define IPV6_2292PKTINFO 2 +#define IPV6_2292HOPOPTS 3 +#define IPV6_2292DSTOPTS 4 +#define IPV6_2292RTHDR 5 +#define IPV6_2292PKTOPTIONS 6 +#define IPV6_CHECKSUM 7 +#define IPV6_2292HOPLIMIT 8 + +#define SCM_SRCRT IPV6_RXSRCRT + +#define IPV6_NEXTHOP 9 +#define IPV6_AUTHHDR 10 +#define IPV6_UNICAST_HOPS 16 +#define IPV6_MULTICAST_IF 17 +#define IPV6_MULTICAST_HOPS 18 +#define IPV6_MULTICAST_LOOP 19 +#define IPV6_JOIN_GROUP 20 +#define IPV6_LEAVE_GROUP 21 +#define IPV6_ROUTER_ALERT 22 +#define IPV6_MTU_DISCOVER 23 +#define IPV6_MTU 24 +#define IPV6_RECVERR 25 +#define IPV6_V6ONLY 26 +#define IPV6_JOIN_ANYCAST 27 +#define IPV6_LEAVE_ANYCAST 28 +#define IPV6_MULTICAST_ALL 29 +#define IPV6_ROUTER_ALERT_ISOLATE 30 +#define IPV6_IPSEC_POLICY 34 +#define IPV6_XFRM_POLICY 35 +#define IPV6_HDRINCL 36 + +/* Advanced API (RFC3542) (1). */ +#define IPV6_RECVPKTINFO 49 +#define IPV6_PKTINFO 50 +#define IPV6_RECVHOPLIMIT 51 +#define IPV6_HOPLIMIT 52 +#define IPV6_RECVHOPOPTS 53 +#define IPV6_HOPOPTS 54 +#define IPV6_RTHDRDSTOPTS 55 +#define IPV6_RECVRTHDR 56 +#define IPV6_RTHDR 57 +#define IPV6_RECVDSTOPTS 58 +#define IPV6_DSTOPTS 59 +#define IPV6_RECVPATHMTU 60 +#define IPV6_PATHMTU 61 +#define IPV6_DONTFRAG 62 + +/* Advanced API (RFC3542) (2). */ +#define IPV6_RECVTCLASS 66 +#define IPV6_TCLASS 67 + +#define IPV6_AUTOFLOWLABEL 70 + +/* RFC5014. */ +#define IPV6_ADDR_PREFERENCES 72 + +/* RFC5082. */ +#define IPV6_MINHOPCOUNT 73 + +#define IPV6_ORIGDSTADDR 74 +#define IPV6_RECVORIGDSTADDR IPV6_ORIGDSTADDR +#define IPV6_TRANSPARENT 75 +#define IPV6_UNICAST_IF 76 +#define IPV6_RECVFRAGSIZE 77 +#define IPV6_FREEBIND 78 + +#define SOCK_STREAM 1 +#define SOCK_DGRAM 2 + +#define MSG_OOB 0x0001 +#define MSG_PEEK 0x0002 +#define MSG_DONTROUTE 0x0004 +#define MSG_CTRUNC 0x0008 +#define MSG_PROXY 0x0010 +#define MSG_TRUNC 0x0020 +#define MSG_DONTWAIT 0x0040 +#define MSG_EOR 0x0080 +#define MSG_WAITALL 0x0100 +#define MSG_FIN 0x0200 +#define MSG_SYN 0x0400 +#define MSG_CONFIRM 0x0800 +#define MSG_RST 0x1000 +#define MSG_ERRQUEUE 0x2000 +#define MSG_NOSIGNAL 0x4000 +#define MSG_MORE 0x8000 +#define MSG_WAITFORONE 0x10000 +#define MSG_BATCH 0x40000 +#define MSG_FASTOPEN 0x20000000 +#define MSG_CMSG_CLOEXEC 0x40000000 + +#define SHUT_RD 0 +#define SHUT_WR 1 +#define SHUT_RDWR 2 + +/* Address families. */ +#define AF_INET 2 /* IP protocol family. */ +#define AF_INET6 10 /* IP version 6. */ + +/* Standard well-defined IP protocols. */ +#define IPPROTO_TCP 6 /* Transmission Control Protocol. */ + +/* Types of sockets. */ +#define SOCK_DGRAM \ + 2 /* Connectionless, unreliable datagrams of fixed maximum length. */ + +struct msghdr { + void *msg_name; + socklen_t msg_namelen; + struct iovec *msg_iov; + int msg_iovlen; + void *msg_control; + socklen_t msg_controllen; + int msg_flags; +}; + +/* Internet address. */ +struct in_addr { + uint32_t s_addr; +}; +typedef struct in_addr in_addr_t; + +/* Structure describing an Internet socket address. */ +#define __SOCK_SIZE__ 16 /* sizeof(struct sockaddr) */ +struct sockaddr_in { + uint16_t sin_family; + uint16_t sin_port; /* Port number. */ + struct in_addr sin_addr; /* Internet address. */ + + /* Pad to size of `struct sockaddr'. */ + unsigned char__pad[__SOCK_SIZE__ - sizeof(uint16_t) - sizeof(uint16_t) + - sizeof(struct in_addr)]; +}; + +/* Structure used to manipulate the SO_LINGER option. */ +struct linger { + int l_onoff; /* Nonzero to linger on close. */ + int l_linger; /* Time to linger. */ +}; + +/* Structure describing a generic socket address. */ +struct sockaddr { + unsigned short int sa_family; /* Common data: address family and length. */ + char sa_data[14]; /* Address data. */ +}; + +uint32_t +ntohl(uint32_t value); + +uint32_t +htonl(uint32_t value); + +uint16_t +htons(uint16_t value); + +int +socket(int domain, int type, int protocol); + +int +getsockopt(int sockfd, int level, int optname, void *optval, socklen_t *optlen); + +int +setsockopt(int sockfd, int level, int optname, const void *optval, + socklen_t optlen); + +ssize_t +sendmsg(int sockfd, const struct msghdr *msg, int flags); + +ssize_t +recvmsg(int sockfd, struct msghdr *msg, int flags); + +int +shutdown(int sockfd, int how); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _SGX_SOCKET_H */ diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_thread.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_thread.c new file mode 100644 index 0000000..73f3005 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_thread.c @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#ifndef SGX_DISABLE_PTHREAD +typedef struct { + thread_start_routine_t start; + void *arg; +} thread_wrapper_arg; + +static void * +os_thread_wrapper(void *arg) +{ + thread_wrapper_arg *targ = arg; + thread_start_routine_t start_func = targ->start; + void *thread_arg = targ->arg; + +#if 0 + os_printf("THREAD CREATED %p\n", &targ); +#endif + BH_FREE(targ); + start_func(thread_arg); + return NULL; +} + +int +os_thread_create_with_prio(korp_tid *tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + thread_wrapper_arg *targ; + + assert(tid); + assert(start); + + targ = (thread_wrapper_arg *)BH_MALLOC(sizeof(*targ)); + if (!targ) { + return BHT_ERROR; + } + + targ->start = start; + targ->arg = arg; + + if (pthread_create(tid, NULL, os_thread_wrapper, targ) != 0) { + BH_FREE(targ); + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_thread_create(korp_tid *tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} +#endif + +korp_tid +os_self_thread() +{ +#ifndef SGX_DISABLE_PTHREAD + return pthread_self(); +#else + return 0; +#endif +} + +int +os_mutex_init(korp_mutex *mutex) +{ +#ifndef SGX_DISABLE_PTHREAD + pthread_mutex_t m = PTHREAD_MUTEX_INITIALIZER; + *mutex = m; +#endif + return BHT_OK; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ +#ifndef SGX_DISABLE_PTHREAD + pthread_mutex_destroy(mutex); +#endif + return BHT_OK; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ +#ifndef SGX_DISABLE_PTHREAD + return pthread_mutex_lock(mutex); +#else + return 0; +#endif +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ +#ifndef SGX_DISABLE_PTHREAD + return pthread_mutex_unlock(mutex); +#else + return 0; +#endif +} + +int +os_cond_init(korp_cond *cond) +{ +#ifndef SGX_DISABLE_PTHREAD + pthread_cond_t c = PTHREAD_COND_INITIALIZER; + *cond = c; +#endif + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ +#ifndef SGX_DISABLE_PTHREAD + pthread_cond_destroy(cond); +#endif + return BHT_OK; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(cond); + assert(mutex); + + if (pthread_cond_wait(cond, mutex) != BHT_OK) + return BHT_ERROR; + +#endif + return BHT_OK; +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + os_printf("warning: SGX pthread_cond_timedwait isn't supported, " + "calling pthread_cond_wait instead!\n"); + return BHT_ERROR; +} + +int +os_cond_signal(korp_cond *cond) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(cond); + + if (pthread_cond_signal(cond) != BHT_OK) + return BHT_ERROR; + +#endif + return BHT_OK; +} + +int +os_cond_broadcast(korp_cond *cond) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(cond); + + if (pthread_cond_broadcast(cond) != BHT_OK) + return BHT_ERROR; + +#endif + return BHT_OK; +} + +int +os_thread_join(korp_tid thread, void **value_ptr) +{ +#ifndef SGX_DISABLE_PTHREAD + return pthread_join(thread, value_ptr); +#else + return 0; +#endif +} + +int +os_thread_detach(korp_tid thread) +{ + /* SGX pthread_detach isn't provided, return directly. */ + return 0; +} + +void +os_thread_exit(void *retval) +{ +#ifndef SGX_DISABLE_PTHREAD + pthread_exit(retval); +#else + return; +#endif +} + +uint8 * +os_thread_get_stack_boundary() +{ + /* TODO: get sgx stack boundary */ + return NULL; +} + +void +os_thread_jit_write_protect_np(bool enabled) +{} + +int +os_rwlock_init(korp_rwlock *lock) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(lock); + + if (pthread_rwlock_init(lock, NULL) != BHT_OK) + return BHT_ERROR; +#endif + + return BHT_OK; +} + +int +os_rwlock_rdlock(korp_rwlock *lock) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(lock); + + if (pthread_rwlock_rdlock(lock) != BHT_OK) + return BHT_ERROR; +#endif + + return BHT_OK; +} + +int +os_rwlock_wrlock(korp_rwlock *lock) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(lock); + + if (pthread_rwlock_wrlock(lock) != BHT_OK) + return BHT_ERROR; +#endif + + return BHT_OK; +} + +int +os_rwlock_unlock(korp_rwlock *lock) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(lock); + + if (pthread_rwlock_unlock(lock) != BHT_OK) + return BHT_ERROR; +#endif + + return BHT_OK; +} + +int +os_rwlock_destroy(korp_rwlock *lock) +{ +#ifndef SGX_DISABLE_PTHREAD + assert(lock); + + if (pthread_rwlock_destroy(lock) != BHT_OK) + return BHT_ERROR; +#endif + + return BHT_OK; +} diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_time.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_time.c new file mode 100644 index 0000000..d39db22 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_time.c @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +#define TRACE_FUNC() os_printf("undefined %s\n", __FUNCTION__) +#define TRACE_OCALL_FAIL() os_printf("ocall %s failed!\n", __FUNCTION__) + +int +ocall_clock_gettime(int *p_ret, unsigned clock_id, void *tp_buf, + unsigned int tp_buf_size); +int +ocall_clock_getres(int *p_ret, int clock_id, void *res_buf, + unsigned int res_buf_size); +int +ocall_utimensat(int *p_ret, int dirfd, const char *pathname, + const void *times_buf, unsigned int times_buf_size, int flags); +int +ocall_futimens(int *p_ret, int fd, const void *times_buf, + unsigned int times_buf_size); +int +ocall_clock_nanosleep(int *p_ret, unsigned clock_id, int flags, + const void *req_buf, unsigned int req_buf_size, + const void *rem_buf, unsigned int rem_buf_size); + +uint64 +os_time_get_boot_us() +{ +#ifndef SGX_DISABLE_WASI + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + + return ((uint64)ts.tv_sec) * 1000 * 1000 + ((uint64)ts.tv_nsec) / 1000; +#else + return 0; +#endif +} + +uint64 +os_time_thread_cputime_us(void) +{ +#ifndef SGX_DISABLE_WASI + struct timespec ts; + if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) { + return 0; + } + + return ((uint64)ts.tv_sec) * 1000 * 1000 + ((uint64)ts.tv_nsec) / 1000; +#else + return 0; +#endif +} + +#ifndef SGX_DISABLE_WASI + +int +clock_getres(int clock_id, struct timespec *res) +{ + int ret; + + if (ocall_clock_getres(&ret, clock_id, (void *)res, sizeof(struct timespec)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +clock_gettime(clockid_t clock_id, struct timespec *tp) +{ + int ret; + + if (ocall_clock_gettime(&ret, clock_id, (void *)tp, sizeof(struct timespec)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +utimensat(int dirfd, const char *pathname, const struct timespec times[2], + int flags) +{ + int ret; + + if (ocall_utimensat(&ret, dirfd, pathname, (void *)times, + sizeof(struct timespec) * 2, flags) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +futimens(int fd, const struct timespec times[2]) +{ + int ret; + + if (ocall_futimens(&ret, fd, (void *)times, sizeof(struct timespec) * 2) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +int +clock_nanosleep(clockid_t clock_id, int flags, const struct timespec *request, + struct timespec *remain) +{ + int ret; + + if (ocall_clock_nanosleep(&ret, clock_id, flags, (void *)request, + sizeof(struct timespec), (void *)remain, + sizeof(struct timespec)) + != SGX_SUCCESS) { + TRACE_OCALL_FAIL(); + return -1; + } + + if (ret == -1) + errno = get_errno(); + + return ret; +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_time.h b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_time.h new file mode 100644 index 0000000..8267f1f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_time.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _SGX_TIME_H +#define _SGX_TIME_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define CLOCK_REALTIME 0 +#define CLOCK_MONOTONIC 1 +#define CLOCK_PROCESS_CPUTIME_ID 2 +#define CLOCK_THREAD_CPUTIME_ID 3 + +#define UTIME_NOW 0x3fffffff +#define UTIME_OMIT 0x3ffffffe +#define TIMER_ABSTIME 1 + +typedef long int time_t; + +typedef int clockid_t; + +struct timespec { + time_t tv_sec; + long tv_nsec; +}; + +int +clock_getres(int clock_id, struct timespec *res); + +int +clock_gettime(clockid_t clock_id, struct timespec *tp); + +int +utimensat(int dirfd, const char *pathname, const struct timespec times[2], + int flags); +int +futimens(int fd, const struct timespec times[2]); +int +clock_nanosleep(clockid_t clock_id, int flags, const struct timespec *request, + struct timespec *remain); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _SGX_TIME_H */ diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_wamr.edl b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_wamr.edl new file mode 100644 index 0000000..7cb4817 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/sgx_wamr.edl @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +enclave { + include "stdint.h" + include "stdbool.h" + include "unistd.h" + + untrusted { + int ocall_open([in, string]const char *pathname, int flags, + bool has_mode, unsigned mode); + int ocall_openat(int dirfd, + [in, string]const char *pathname, int flags, + bool has_mode, unsigned mode); + int ocall_close(int fd); + ssize_t ocall_read(int fd, [out, size=read_size]void *buf, + size_t read_size); + off_t ocall_lseek(int fd, off_t offset, int whence); + int ocall_ftruncate(int fd, off_t length); + int ocall_fsync(int fd); + int ocall_fdatasync(int fd); + int ocall_isatty(int fd); + void ocall_fdopendir(int fd, [out]void **p_dirp); + /* implementation related to multiple thread */ + void *ocall_readdir([user_check]void *dirp); + void ocall_rewinddir([user_check]void *dirp); + void ocall_seekdir([user_check]void *dirp, long loc); + long ocall_telldir([user_check]void *dirp); + int ocall_closedir([user_check]void *dirp); + + int ocall_stat([in, string]const char *pathname, + [out, size=buf_len]void *buf, + unsigned int buf_len); + int ocall_fstat(int fd, [out, size=buf_len]void *buf, + unsigned int buf_len); + int ocall_fstatat(int dirfd, [in, string]const char *pathname, + [out, size=buf_len]void *buf, + unsigned int buf_len, int flags); + + int ocall_mkdirat(int dirfd, [in, string]const char *pathname, + unsigned mode); + int ocall_link([in, string] const char *oldpath, + [in, string] const char *newpath); + int ocall_linkat(int olddirfd, [in, string]const char *oldpath, + int newdirfd, [in, string]const char *newpath, + int flags); + int ocall_unlinkat(int dirfd, [in, string]const char *pathname, + int flags); + ssize_t ocall_readlink([in, string]const char *pathname, + [out, size=bufsiz]char *buf, + size_t bufsiz); + ssize_t ocall_readlinkat(int dirfd, + [in, string]const char *pathname, + [out, size=bufsiz]char *buf, + size_t bufsiz); + int ocall_renameat(int olddirfd, + [in, string]const char *oldpath, + int newdirfd, + [in, string]const char *newpath); + int ocall_symlinkat([in ,string]const char *target, + int newdirfd, + [in, string]const char *linkpath); + + int ocall_ioctl(int fd, unsigned long request, + [out, size=arg_len]void *arg, + unsigned int arg_len); + int ocall_fcntl(int fd, int cmd); + int ocall_fcntl_long(int fd, int cmd, long arg); + + int ocall_realpath([in, string]const char *path, + [out, size=buf_len]char *buf, + unsigned int buf_len); + int ocall_posix_fallocate(int fd, off_t offset, off_t len); + int ocall_poll([in, out, size=fds_len]void *fds, unsigned nfds, + int timeout, unsigned int fds_len); + + int ocall_getopt(int argc, + [in, size=argv_buf_len]char *argv_buf, + unsigned int argv_buf_len, + [in, string]const char *optstring); + ssize_t ocall_readv(int fd, + [in, out, size=buf_size]char *iov_buf, + unsigned int buf_size, int iovcnt, + bool has_offset, off_t offset); + ssize_t ocall_writev(int fd, + [in, size=buf_size]char *iov_buf, + unsigned int buf_size, int iovcnt, + bool has_offset, off_t offset); + + /* time clock */ + int ocall_clock_gettime(unsigned clock_id, + [out, size=tp_buf_size]void *tp_buf, + unsigned int tp_buf_size); + int ocall_clock_getres(int clock_id, + [out, size=res_buf_size]void *res_buf, + unsigned int res_buf_size); + int ocall_utimensat(int dirfd, [in, string]const char *pathname, + [in, size=times_buf_size]const void *times_buf, + unsigned int times_buf_size, int flags); + int ocall_futimens(int fd, [in, size=times_buf_size]const void *times_buf, + unsigned int times_buf_size); + int ocall_clock_nanosleep(unsigned clock_id, int flags, + [in, size=req_buf_size]const void *req_buf, + unsigned int req_buf_size, + [out, size=rem_buf_size]void *rem_buf, + unsigned int rem_buf_size); + + int ocall_raise(int sig); + + int ocall_sched_yield(); + + int ocall_pthread_rwlock_init([out]void **rwlock, [user_check]void *attr); + int ocall_pthread_rwlock_destroy([user_check]void *rwlock); + int ocall_pthread_rwlock_rdlock([user_check]void *rwlock); + int ocall_pthread_rwlock_wrlock([user_check]void *rwlock); + int ocall_pthread_rwlock_unlock([user_check]void *rwlock); + + int ocall_get_errno(); + + /* sockets */ + int ocall_accept(int sockfd, [in, size=addr_size]void *addr, + [in, size=4] uint32_t *addrlen, uint32_t addr_size); + int ocall_bind(int sockfd, [in, size=addrlen]const void *addr, + uint32_t addrlen); + int ocall_connect(int sockfd, [in, size=addrlen]void *addr, uint32_t addrlen); + int ocall_getsockname(int sockfd, [out, size=addr_size]void *addr, + [in, out, size=4]uint32_t *addrlen, uint32_t addr_size); + int ocall_getpeername(int sockfd, [out, size=addr_size]void *addr, + [in, out, size=4]uint32_t *addrlen, uint32_t addr_size); + int ocall_getsockopt(int sockfd, int level, int optname, + [out, size=val_buf_size]void *val_buf, + unsigned int val_buf_size, + [in, out, size=4]void *len_buf); + int ocall_listen(int sockfd, int backlog); + int ocall_recv(int sockfd, [out, size=len]void *buf, size_t len, int flags); + ssize_t ocall_recvfrom(int sockfd, [out, size=len]void *buf, size_t len, int flags, + [out, size=addr_size]void *src_addr, + [in, out, size=4]uint32_t *addrlen, uint32_t addr_size); + ssize_t ocall_recvmsg(int sockfd, + [in, out, size=msg_buf_size]void *msg_buf, + unsigned int msg_buf_size, + int flags); + int ocall_send(int sockfd, [in, size=len]const void *buf, size_t len, int flags); + ssize_t ocall_sendto(int sockfd, [in, size=len]const void *buf, size_t len, int flags, + [in, size=addrlen]void *dest_addr, uint32_t addrlen); + ssize_t ocall_sendmsg(int sockfd, + [in, size=msg_buf_size]void *msg_buf, + unsigned int msg_buf_size, + int flags); + int ocall_setsockopt(int sockfd, int level, int optname, + [in, size=optlen]void *optval, + unsigned int optlen); + int ocall_shutdown(int sockfd, int how); + int ocall_socket(int domain, int type, int protocol); + }; +}; diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/linux-sgx/shared_platform.cmake new file mode 100644 index 0000000..9cd765b --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/shared_platform.cmake @@ -0,0 +1,48 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_LINUX_SGX) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +if ("$ENV{SGX_SDK}" STREQUAL "") + set (SGX_SDK_DIR "/opt/intel/sgxsdk") +else() + set (SGX_SDK_DIR $ENV{SGX_SDK}) +endif() + +include_directories (${SGX_SDK_DIR}/include) +if (NOT BUILD_UNTRUST_PART EQUAL 1) + include_directories (${SGX_SDK_DIR}/include/tlibc + ${SGX_SDK_DIR}/include/libcxx) +endif () + +if (NOT WAMR_BUILD_THREAD_MGR EQUAL 1) + add_definitions(-DSGX_DISABLE_PTHREAD) +endif () + +file (GLOB source_all ${PLATFORM_SHARED_DIR}/*.c) + +if (NOT WAMR_BUILD_LIBC_WASI EQUAL 1) + add_definitions(-DSGX_DISABLE_WASI) +else() + list(APPEND source_all + ${PLATFORM_SHARED_DIR}/../common/posix/posix_file.c + ${PLATFORM_SHARED_DIR}/../common/posix/posix_clock.c + ) + include (${CMAKE_CURRENT_LIST_DIR}/../common/libc-util/platform_common_libc_util.cmake) + set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) +endif() + +include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) +set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) + +file (GLOB source_all_untrusted ${PLATFORM_SHARED_DIR}/untrusted/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all}) + +set (PLATFORM_SHARED_SOURCE_UNTRUSTED ${source_all_untrusted}) + diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/file.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/file.c new file mode 100644 index 0000000..cb9bf6a --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/file.c @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int +ocall_open(const char *pathname, int flags, bool has_mode, unsigned mode) +{ + if (has_mode) { + return open(pathname, flags, (mode_t)mode); + } + else { + return open(pathname, flags); + } +} + +int +ocall_openat(int dirfd, const char *pathname, int flags, bool has_mode, + unsigned mode) +{ + if (has_mode) { + return openat(dirfd, pathname, flags, (mode_t)mode); + } + else { + return openat(dirfd, pathname, flags); + } +} + +int +ocall_close(int fd) +{ + return close(fd); +} + +ssize_t +ocall_read(int fd, void *buf, size_t read_size) +{ + if (buf != NULL) { + return read(fd, buf, read_size); + } + else { + return -1; + } +} + +off_t +ocall_lseek(int fd, off_t offset, int whence) +{ + return lseek(fd, offset, whence); +} + +int +ocall_ftruncate(int fd, off_t length) +{ + return ftruncate(fd, length); +} + +int +ocall_fsync(int fd) +{ + return fsync(fd); +} + +int +ocall_fdatasync(int fd) +{ + return fdatasync(fd); +} + +int +ocall_isatty(int fd) +{ + return isatty(fd); +} + +void +ocall_fdopendir(int fd, void **dirp) +{ + if (dirp) { + *(DIR **)dirp = fdopendir(fd); + } +} + +void * +ocall_readdir(void *dirp) +{ + DIR *p_dirp = (DIR *)dirp; + return readdir(p_dirp); +} + +void +ocall_rewinddir(void *dirp) +{ + DIR *p_dirp = (DIR *)dirp; + if (p_dirp) { + rewinddir(p_dirp); + } +} + +void +ocall_seekdir(void *dirp, long loc) +{ + DIR *p_dirp = (DIR *)dirp; + + if (p_dirp) { + seekdir(p_dirp, loc); + } +} + +long +ocall_telldir(void *dirp) +{ + DIR *p_dirp = (DIR *)dirp; + if (p_dirp) { + return telldir(p_dirp); + } + return -1; +} + +int +ocall_closedir(void *dirp) +{ + DIR *p_dirp = (DIR *)dirp; + if (p_dirp) { + return closedir(p_dirp); + } + return -1; +} + +int +ocall_stat(const char *pathname, void *buf, unsigned int buf_len) +{ + return stat(pathname, (struct stat *)buf); +} + +int +ocall_fstat(int fd, void *buf, unsigned int buf_len) +{ + return fstat(fd, (struct stat *)buf); +} + +int +ocall_fstatat(int dirfd, const char *pathname, void *buf, unsigned int buf_len, + int flags) +{ + return fstatat(dirfd, pathname, (struct stat *)buf, flags); +} + +int +ocall_mkdirat(int dirfd, const char *pathname, unsigned mode) +{ + return mkdirat(dirfd, pathname, (mode_t)mode); +} + +int +ocall_link(const char *oldpath, const char *newpath) +{ + return link(oldpath, newpath); +} + +int +ocall_linkat(int olddirfd, const char *oldpath, int newdirfd, + const char *newpath, int flags) +{ + return linkat(olddirfd, oldpath, newdirfd, newpath, flags); +} + +int +ocall_unlinkat(int dirfd, const char *pathname, int flags) +{ + return unlinkat(dirfd, pathname, flags); +} + +ssize_t +ocall_readlink(const char *pathname, char *buf, size_t bufsiz) +{ + return readlink(pathname, buf, bufsiz); +} + +ssize_t +ocall_readlinkat(int dirfd, const char *pathname, char *buf, size_t bufsiz) +{ + return readlinkat(dirfd, pathname, buf, bufsiz); +} + +int +ocall_renameat(int olddirfd, const char *oldpath, int newdirfd, + const char *newpath) +{ + return renameat(olddirfd, oldpath, newdirfd, newpath); +} + +int +ocall_symlinkat(const char *target, int newdirfd, const char *linkpath) +{ + return symlinkat(target, newdirfd, linkpath); +} + +int +ocall_ioctl(int fd, unsigned long request, void *arg, unsigned int arg_len) +{ + /* support just int *arg temporally */ + return ioctl(fd, request, (int *)arg); +} + +int +ocall_fcntl(int fd, int cmd) +{ + return fcntl(fd, cmd); +} + +int +ocall_fcntl_long(int fd, int cmd, long arg) +{ + return fcntl(fd, cmd, arg); +} + +ssize_t +ocall_readv(int fd, char *iov_buf, unsigned int buf_size, int iovcnt, + bool has_offset, off_t offset) +{ + struct iovec *iov = (struct iovec *)iov_buf; + ssize_t ret; + int i; + + for (i = 0; i < iovcnt; i++) { + iov[i].iov_base = iov_buf + (unsigned)(uintptr_t)iov[i].iov_base; + } + + if (has_offset) + ret = preadv(fd, iov, iovcnt, offset); + else + ret = readv(fd, iov, iovcnt); + + return ret; +} + +ssize_t +ocall_writev(int fd, char *iov_buf, unsigned int buf_size, int iovcnt, + bool has_offset, off_t offset) +{ + struct iovec *iov = (struct iovec *)iov_buf; + int i; + ssize_t ret; + + for (i = 0; i < iovcnt; i++) { + iov[i].iov_base = iov_buf + (unsigned)(uintptr_t)iov[i].iov_base; + } + + if (has_offset) + ret = pwritev(fd, iov, iovcnt, offset); + else + ret = writev(fd, iov, iovcnt); + + return ret; +} + +int +ocall_realpath(const char *path, char *buf, unsigned int buf_len) +{ + char *val = NULL; + val = realpath(path, buf); + if (val != NULL) { + return 0; + } + return -1; +} + +int +ocall_posix_fallocate(int fd, off_t offset, off_t len) +{ + return posix_fallocate(fd, offset, len); +} + +int +ocall_poll(void *fds, unsigned nfds, int timeout, unsigned int fds_len) +{ + return poll((struct pollfd *)fds, (nfds_t)nfds, timeout); +} + +int +ocall_getopt(int argc, char *argv_buf, unsigned int argv_buf_len, + const char *optstring) +{ + int ret; + int i; + char **argv = (char **)argv_buf; + + for (i = 0; i < argc; i++) { + argv[i] = argv_buf + (uintptr_t)argv[i]; + } + + return getopt(argc, argv, optstring); +} + +int +ocall_sched_yield() +{ + return sched_yield(); +} + +int +ocall_get_errno() +{ + return errno; +} diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/pthread.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/pthread.c new file mode 100644 index 0000000..890ef75 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/pthread.c @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +int +ocall_pthread_rwlock_init(void **rwlock, void *attr) +{ + int ret = 0; + + *rwlock = malloc(sizeof(pthread_rwlock_t)); + if (*rwlock == NULL) + return -1; + + ret = pthread_rwlock_init((pthread_rwlock_t *)*rwlock, NULL); + if (ret != 0) { + free(*rwlock); + *rwlock = NULL; + } + (void)attr; + return ret; +} + +int +ocall_pthread_rwlock_destroy(void *rwlock) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)rwlock; + int ret; + + ret = pthread_rwlock_destroy(lock); + free(lock); + return ret; +} + +int +ocall_pthread_rwlock_rdlock(void *rwlock) +{ + return pthread_rwlock_rdlock((pthread_rwlock_t *)rwlock); +} + +int +ocall_pthread_rwlock_wrlock(void *rwlock) +{ + return pthread_rwlock_wrlock((pthread_rwlock_t *)rwlock); +} + +int +ocall_pthread_rwlock_unlock(void *rwlock) +{ + return pthread_rwlock_unlock((pthread_rwlock_t *)rwlock); +} diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/signal.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/signal.c new file mode 100644 index 0000000..b2eecfb --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/signal.c @@ -0,0 +1,11 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include + +int +ocall_raise(int sig) +{ + return raise(sig); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/socket.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/socket.c new file mode 100644 index 0000000..6f598ab --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/socket.c @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include +#include +#include +#include +#include +#include + +int +ocall_socket(int domain, int type, int protocol) +{ + return socket(domain, type, protocol); +} + +int +ocall_getsockopt(int sockfd, int level, int optname, void *val_buf, + unsigned int val_buf_size, void *len_buf) +{ + return getsockopt(sockfd, level, optname, val_buf, (socklen_t *)len_buf); +} + +ssize_t +ocall_sendmsg(int sockfd, void *msg_buf, unsigned int msg_buf_size, int flags) +{ + struct msghdr *msg = (struct msghdr *)msg_buf; + int i; + ssize_t ret; + + if (msg->msg_name != NULL) + msg->msg_name = msg_buf + (unsigned)(uintptr_t)msg->msg_name; + + if (msg->msg_control != NULL) + msg->msg_control = msg_buf + (unsigned)(uintptr_t)msg->msg_control; + + if (msg->msg_iov != NULL) { + msg->msg_iov = msg_buf + (unsigned)(uintptr_t)msg->msg_iov; + for (i = 0; i < msg->msg_iovlen; i++) { + msg->msg_iov[i].iov_base = + msg_buf + (unsigned)(uintptr_t)msg->msg_iov[i].iov_base; + } + } + + return sendmsg(sockfd, msg, flags); +} + +ssize_t +ocall_recvmsg(int sockfd, void *msg_buf, unsigned int msg_buf_size, int flags) +{ + struct msghdr *msg = (struct msghdr *)msg_buf; + int i; + ssize_t ret; + + if (msg->msg_name != NULL) + msg->msg_name = msg_buf + (unsigned)(uintptr_t)msg->msg_name; + + if (msg->msg_control != NULL) + msg->msg_control = msg_buf + (unsigned)(uintptr_t)msg->msg_control; + + if (msg->msg_iov != NULL) { + msg->msg_iov = msg_buf + (unsigned)(uintptr_t)msg->msg_iov; + for (i = 0; i < msg->msg_iovlen; i++) { + msg->msg_iov[i].iov_base = + msg_buf + (unsigned)(uintptr_t)msg->msg_iov[i].iov_base; + } + } + + return recvmsg(sockfd, msg, flags); +} + +int +ocall_shutdown(int sockfd, int how) +{ + return shutdown(sockfd, how); +} + +int +ocall_setsockopt(int sockfd, int level, int optname, void *optval, + unsigned int optlen) +{ + return setsockopt(sockfd, level, optname, optval, optlen); +} + +int +ocall_bind(int sockfd, const void *addr, uint32_t addrlen) +{ + return bind(sockfd, (const struct sockaddr *)addr, addrlen); +} + +int +ocall_getsockname(int sockfd, void *addr, uint32_t *addrlen, uint32_t addr_size) +{ + return getsockname(sockfd, (struct sockaddr *)addr, addrlen); +} + +int +ocall_getpeername(int sockfd, void *addr, uint32_t *addrlen, uint32_t addr_size) +{ + return getpeername(sockfd, (struct sockaddr *)addr, addrlen); +} + +int +ocall_listen(int sockfd, int backlog) +{ + return listen(sockfd, backlog); +} + +int +ocall_accept(int sockfd, void *addr, uint32_t *addrlen, uint32_t addr_size) +{ + return accept(sockfd, (struct sockaddr *)addr, addrlen); +} + +int +ocall_recv(int sockfd, void *buf, size_t len, int flags) +{ + return recv(sockfd, buf, len, flags); +} + +ssize_t +ocall_recvfrom(int sockfd, void *buf, size_t len, int flags, void *src_addr, + uint32_t *addrlen, uint32_t addr_size) +{ + return recvfrom(sockfd, buf, len, flags, (struct sockaddr *)src_addr, + addrlen); +} + +int +ocall_send(int sockfd, const void *buf, size_t len, int flags) +{ + return send(sockfd, buf, len, flags); +} + +ssize_t +ocall_sendto(int sockfd, const void *buf, size_t len, int flags, + void *dest_addr, uint32_t addrlen) +{ + return sendto(sockfd, buf, len, flags, (struct sockaddr *)dest_addr, + addrlen); +} + +int +ocall_connect(int sockfd, void *addr, uint32_t addrlen) +{ + return connect(sockfd, (const struct sockaddr *)addr, addrlen); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/time.c b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/time.c new file mode 100644 index 0000000..5fa387b --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/linux-sgx/untrusted/time.c @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include +#include +#include +#include + +/** time clock **/ +int +ocall_clock_gettime(unsigned clock_id, void *tp_buf, unsigned int tp_buf_size) +{ + return clock_gettime((clockid_t)clock_id, (struct timespec *)tp_buf); +} + +int +ocall_clock_getres(int clock_id, void *res_buf, unsigned int res_buf_size) +{ + return clock_getres(clock_id, (struct timespec *)res_buf); +} + +int +ocall_utimensat(int dirfd, const char *pathname, const void *times_buf, + unsigned int times_buf_size, int flags) +{ + return utimensat(dirfd, pathname, (struct timespec *)times_buf, flags); +} + +int +ocall_futimens(int fd, const void *times_buf, unsigned int times_buf_size) +{ + return futimens(fd, (struct timespec *)times_buf); +} + +int +ocall_clock_nanosleep(unsigned clock_id, int flags, const void *req_buf, + unsigned int req_buf_size, const void *rem_buf, + unsigned int rem_buf_size) +{ + return clock_nanosleep((clockid_t)clock_id, flags, + (struct timespec *)req_buf, + (struct timespec *)rem_buf); +} diff --git a/wasm-micro-runtime/core/shared/platform/nuttx/nuttx_platform.c b/wasm-micro-runtime/core/shared/platform/nuttx/nuttx_platform.c new file mode 100644 index 0000000..0077c9e --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/nuttx/nuttx_platform.c @@ -0,0 +1,367 @@ +/* + * Copyright (C) 2020 XiaoMi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_extension.h" +#include "platform_api_vmcore.h" + +#if defined(CONFIG_ARCH_USE_TEXT_HEAP) +#include +#endif + +#if defined(CONFIG_ARCH_CHIP_ESP32S3) +/* + * TODO: Move these methods below the operating system level + */ +#define MEM_DUAL_BUS_OFFSET (0x42000000 - 0x3C000000) +#define IRAM0_CACHE_ADDRESS_LOW 0x42000000 +#define IRAM0_CACHE_ADDRESS_HIGH 0x44000000 +#define IRAM_ATTR locate_data(".iram1") + +#define in_ibus_ext(addr) \ + (((uint32)addr >= IRAM0_CACHE_ADDRESS_LOW) \ + && ((uint32)addr < IRAM0_CACHE_ADDRESS_HIGH)) +void IRAM_ATTR +bus_sync(void) +{ + extern void cache_writeback_all(void); + extern uint32_t Cache_Disable_ICache(void); + extern void Cache_Enable_ICache(uint32_t autoload); + + irqstate_t flags; + uint32_t preload; + + flags = enter_critical_section(); + + cache_writeback_all(); + preload = Cache_Disable_ICache(); + Cache_Enable_ICache(preload); + + leave_critical_section(flags); +} +#else +#define MEM_DUAL_BUS_OFFSET (0) +#define IRAM0_CACHE_ADDRESS_LOW (0) +#define IRAM0_CACHE_ADDRESS_HIGH (0) +#define in_ibus_ext(addr) (0) +static void +bus_sync(void) +{} +#endif + +int +bh_platform_init() +{ + return 0; +} + +void +bh_platform_destroy() +{} + +void * +os_malloc(unsigned size) +{ + return malloc(size); +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return realloc(ptr, size); +} + +void +os_free(void *ptr) +{ + free(ptr); +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + void *p; +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + void *i_addr, *d_addr; +#endif + +#if defined(CONFIG_ARCH_USE_TEXT_HEAP) + if ((prot & MMAP_PROT_EXEC) != 0) { + p = up_textheap_memalign(sizeof(void *), size); + if (p) { + memset(p, 0, size); + } + return p; + } +#endif + + if ((uint64)size >= UINT32_MAX) + return NULL; + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + if ((prot & MMAP_PROT_EXEC) != 0) { + d_addr = malloc((uint32)size); + if (d_addr == NULL) { + return NULL; + } + i_addr = (void *)((uint8 *)d_addr + MEM_DUAL_BUS_OFFSET); + p = in_ibus_ext(i_addr) ? i_addr : d_addr; + if (p) { + memset(p, 0, size); + } + return p; + } +#endif + /* Note: aot_loader.c assumes that os_mmap provides large enough + * alignment for any data sections. Some sections like rodata.cst32 + * actually require alignment larger than the natural alignment + * provided by malloc. + * + * Probably it's cleaner to add an explicit alignment argument to + * os_mmap. However, it only makes sense if we change our aot format + * to keep the necessary alignment. + * + * For now, let's assume 32 byte alignment is enough. + */ + if (posix_memalign(&p, 32, size)) { + return NULL; + } + + /* Zero the memory which is required by os_mmap */ + memset(p, 0, size); + + return p; +} + +void +os_munmap(void *addr, size_t size) +{ +#if defined(CONFIG_ARCH_USE_TEXT_HEAP) + if (up_textheap_heapmember(addr)) { + up_textheap_free(addr); + return; + } +#endif + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) + if (in_ibus_ext(addr)) { + free((void *)((uint8 *)addr - MEM_DUAL_BUS_OFFSET)); + return; + } +#endif + free(addr); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + return 0; +} + +void +os_dcache_flush() +{ + bus_sync(); +} + +void +os_icache_flush(void *start, size_t len) +{} + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) +void * +os_get_dbus_mirror(void *ibus) +{ + if (in_ibus_ext(ibus)) { + return (void *)((uint8 *)ibus - MEM_DUAL_BUS_OFFSET); + } + else { + return ibus; + } +} +#endif + +/* If AT_FDCWD is provided, maybe we have openat family */ +#if !defined(AT_FDCWD) + +int +openat(int fd, const char *path, int oflags, ...) +{ + errno = ENOSYS; + return -1; +} + +int +fstatat(int fd, const char *path, struct stat *buf, int flag) +{ + errno = ENOSYS; + return -1; +} + +int +mkdirat(int fd, const char *path, mode_t mode) +{ + errno = ENOSYS; + return -1; +} + +ssize_t +readlinkat(int fd, const char *path, char *buf, size_t bufsize) +{ + errno = ENOSYS; + return -1; +} + +int +linkat(int fd1, const char *path1, int fd2, const char *path2, int flag) +{ + errno = ENOSYS; + return -1; +} + +int +renameat(int fromfd, const char *from, int tofd, const char *to) +{ + errno = ENOSYS; + return -1; +} +int +symlinkat(const char *target, int fd, const char *path) +{ + errno = ENOSYS; + return -1; +} +int +unlinkat(int fd, const char *path, int flag) +{ + errno = ENOSYS; + return -1; +} +int +utimensat(int fd, const char *path, const struct timespec ts[2], int flag) +{ + errno = ENOSYS; + return -1; +} + +#endif /* !defined(AT_FDCWD) */ + +#ifndef CONFIG_NET + +#include + +int +accept(int sockfd, FAR struct sockaddr *addr, FAR socklen_t *addrlen) +{ + errno = ENOTSUP; + return -1; +} + +int +bind(int sockfd, FAR const struct sockaddr *addr, socklen_t addrlen) +{ + errno = ENOTSUP; + return -1; +} + +int +listen(int sockfd, int backlog) +{ + errno = ENOTSUP; + return -1; +} + +int +connect(int sockfd, FAR const struct sockaddr *addr, socklen_t addrlen) +{ + errno = ENOTSUP; + return -1; +} + +ssize_t +recvfrom(int sockfd, FAR void *buf, size_t len, int flags, + FAR struct sockaddr *from, FAR socklen_t *fromlen) +{ + errno = ENOTSUP; + return -1; +} + +ssize_t +send(int sockfd, FAR const void *buf, size_t len, int flags) +{ + errno = ENOTSUP; + return -1; +} + +ssize_t +sendto(int sockfd, FAR const void *buf, size_t len, int flags, + FAR const struct sockaddr *to, socklen_t tolen) +{ + errno = ENOTSUP; + return -1; +} + +int +socket(int domain, int type, int protocol) +{ + errno = ENOTSUP; + return -1; +} + +int +shutdown(int sockfd, int how) +{ + errno = ENOTSUP; + return -1; +} + +int +getaddrinfo(FAR const char *nodename, FAR const char *servname, + FAR const struct addrinfo *hints, FAR struct addrinfo **res) +{ + errno = ENOTSUP; + return -1; +} + +void +freeaddrinfo(FAR struct addrinfo *ai) +{} + +int +setsockopt(int sockfd, int level, int option, FAR const void *value, + socklen_t value_len) +{ + errno = ENOTSUP; + return -1; +} + +int +getsockopt(int sockfd, int level, int option, FAR void *value, + FAR socklen_t *value_len) +{ + errno = ENOTSUP; + return -1; +} + +int +getpeername(int sockfd, FAR struct sockaddr *addr, FAR socklen_t *addrlen) +{ + errno = ENOTSUP; + return -1; +} + +int +getsockname(int sockfd, FAR struct sockaddr *addr, FAR socklen_t *addrlen) +{ + errno = ENOTSUP; + return -1; +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/nuttx/platform_internal.h b/wasm-micro-runtime/core/shared/platform/nuttx/platform_internal.h new file mode 100644 index 0000000..0b54d85 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/nuttx/platform_internal.h @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2020 XiaoMi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_NUTTX +#define BH_PLATFORM_NUTTX +#endif + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef sem_t korp_sem; + +#define os_getpagesize getpagesize + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 100 + +#define os_printf printf +#define os_vprintf vprintf + +#if defined(CONFIG_LIBC_DLFCN) +#define BH_HAS_DLFCN 1 +#else +#define BH_HAS_DLFCN 0 +#endif + +/* On NuttX, time_t is uint32_t */ +#define BH_TIME_T_MAX 0xffffffff + +/* + * NuttX doesn't have O_DIRECTORY or directory open. + * REVISIT: maybe this is safer to be disabled at higher level. + */ +#if !defined(O_DIRECTORY) +#define O_DIRECTORY 0 +#endif + +#if !defined(O_NOFOLLOW) +#define O_NOFOLLOW 0 +#endif + +#undef CONFIG_HAS_ISATTY +#ifdef CONFIG_SERIAL_TERMIOS +#define CONFIG_HAS_ISATTY 1 +#else +#define CONFIG_HAS_ISATTY 0 +#endif + +#define BUILTIN_LIBC_BUFFERED_PRINTF 1 +#define BUILTIN_LIBC_BUFFERED_PRINT_SIZE 128 +#define BUILTIN_LIBC_BUFFERED_PRINT_PREFIX + +/* + * NuttX doesn't have openat family. + */ + +/* If AT_FDCWD is provided, maybe we have openat family */ +#if !defined(AT_FDCWD) + +int +openat(int fd, const char *path, int oflags, ...); +int +fstatat(int fd, const char *path, struct stat *buf, int flag); +int +mkdirat(int fd, const char *path, mode_t mode); +ssize_t +readlinkat(int fd, const char *path, char *buf, size_t bufsize); +int +linkat(int fd1, const char *path1, int fd2, const char *path2, int flag); +int +renameat(int fromfd, const char *from, int tofd, const char *to); +int +symlinkat(const char *target, int fd, const char *path); +int +unlinkat(int fd, const char *path, int flag); +int +utimensat(int fd, const char *path, const struct timespec ts[2], int flag); +#define AT_SYMLINK_NOFOLLOW 0 +#define AT_SYMLINK_FOLLOW 0 +#define AT_REMOVEDIR 0 + +#endif /* !defined(AT_FDCWD) */ + +/* + * NuttX doesn't have fdopendir. + */ + +DIR * +fdopendir(int fd); + +#if WASM_DISABLE_WAKEUP_BLOCKING_OP == 0 +#define OS_ENABLE_WAKEUP_BLOCKING_OP +#endif +void +os_set_signal_number_for_blocking_op(int signo); + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* end of _BH_PLATFORM_H */ diff --git a/wasm-micro-runtime/core/shared/platform/nuttx/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/nuttx/shared_platform.cmake new file mode 100644 index 0000000..d691068 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/nuttx/shared_platform.cmake @@ -0,0 +1,26 @@ +# Copyright (C) 2020 XiaoMi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_NUTTX) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +if (WAMR_BUILD_LIBC_WASI EQUAL 1) + list(APPEND source_all ${PLATFORM_SHARED_DIR}/../common/posix/posix_file.c) + include (${CMAKE_CURRENT_LIST_DIR}/../common/libc-util/platform_common_libc_util.cmake) + set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) +endif () + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE} ${PLATFORM_COMMON_POSIX_SOURCE} ${UNCOMMON_SHARED_SOURCE}) +# remove posix_memmap.c for NuttX +list(REMOVE_ITEM ${PLATFORM_SHARED_SOURCE} ${PLATFORM_COMMON_POSIX_DIR}/posix_memmap.c) + diff --git a/wasm-micro-runtime/core/shared/platform/riot/platform_internal.h b/wasm-micro-runtime/core/shared/platform/riot/platform_internal.h new file mode 100644 index 0000000..e88b25d --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/riot/platform_internal.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * Copyright (C) 2020 TU Bergakademie Freiberg Karl Fessel + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +/* Riot includes core */ +#include +#include +#include + +/* Riot includes sys */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef BH_PLATFORM_RIOT +#define BH_PLATFORM_RIOT +#endif + +#define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 7 + +typedef thread_t korp_thread; +typedef kernel_pid_t korp_tid; +typedef mutex_t korp_mutex; +typedef unsigned int korp_sem; + +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + +/* typedef sema_t korp_sem; */ + +struct os_thread_wait_node; +typedef struct os_thread_wait_node *os_thread_wait_list; +typedef struct korp_cond { + mutex_t wait_list_lock; + os_thread_wait_list thread_wait_list; +} korp_cond; + +#define os_printf printf +#define os_vprintf vprintf + +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ +typedef int os_file_handle; +typedef void *os_dir_stream; +typedef int os_raw_file_handle; + +#if WA_MATH +/* clang-format off */ +/* math functions which are not provided by os*/ +double sqrt(double x); +double floor(double x); +double ceil(double x); +double fmin(double x, double y); +double fmax(double x, double y); +double rint(double x); +double fabs(double x); +double trunc(double x); +float sqrtf(float x); +float floorf(float x); +float ceilf(float x); +float fminf(float x, float y); +float fmaxf(float x, float y); +float rintf(float x); +float fabsf(float x); +float truncf(float x); +int signbit(double x); +int isnan(double x); +/* clang-format on */ +#endif + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#endif /* end of _BH_PLATFORM_H */ diff --git a/wasm-micro-runtime/core/shared/platform/riot/riot_platform.c b/wasm-micro-runtime/core/shared/platform/riot/riot_platform.c new file mode 100644 index 0000000..ad5927e --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/riot/riot_platform.c @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * Copyright (C) 2020 TU Bergakademie Freiberg Karl Fessel + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +int +os_thread_sys_init(void); + +void +os_thread_sys_destroy(void); + +int +bh_platform_init(void) +{ + return os_thread_sys_init(); +} + +void +bh_platform_destroy(void) +{ + os_thread_sys_destroy(); +} + +void * +os_malloc(unsigned size) +{ + return malloc(size); +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return realloc(ptr, size); +} + +void +os_free(void *ptr) +{ + free(ptr); +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + if (size > ((unsigned)~0)) + return NULL; + return BH_MALLOC((unsigned)size); +} + +void +os_munmap(void *addr, size_t size) +{ + return BH_FREE(addr); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + return 0; +} + +void +os_dcache_flush(void) +{ +#if defined(CONFIG_CPU_CORTEX_M7) && defined(CONFIG_ARM_MPU) + uint32 key; + key = irq_lock(); + SCB_CleanDCache(); + irq_unlock(key); +#endif +} + +void +os_icache_flush(void *start, size_t len) +{} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/riot/riot_thread.c b/wasm-micro-runtime/core/shared/platform/riot/riot_thread.c new file mode 100644 index 0000000..a9062bf --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/riot/riot_thread.c @@ -0,0 +1,436 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * Copyright (C) 2020 TU Bergakademie Freiberg Karl Fessel + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#include +#include +#include + +/* clang-format off */ +#define bh_assert(v) do { \ + if (!(v)) { \ + printf("\nASSERTION FAILED: %s, at %s, line %d\n", \ + #v, __FILE__, __LINE__); \ + core_panic(0, 0/*expr_string*/); \ + while (1); \ + } \ +} while (0) +/* clang-format on */ + +struct os_thread_data; +typedef struct os_thread_wait_node { + sema_t sem; + void *ret; + os_thread_wait_list next; +} os_thread_wait_node; + +// all information for thread to cleanup it self +typedef struct os_thread_data { + /* Next thread data */ + struct os_thread_data *next; + /* thread handle */ + kernel_pid_t tid; + /* Thread start routine */ + thread_start_routine_t start_routine; + /* Thread start routine argument */ + void *arg; + /* thread local root */ + void *tlr; + /* Lock for waiting list */ + mutex_t wait_list_lock; + /* Waiting list of other threads who are joining this thread */ + os_thread_wait_list thread_wait_list; + /* Thread stack size */ + unsigned stack_size; + /* Thread stack */ + char stack[1]; +} os_thread_data; + +typedef struct os_thread_obj { + korp_tid thread; + /* Whether the thread is terminated and this thread object is to + be freed in the future. */ + bool to_be_freed; + struct os_thread_obj *next; +} os_thread_obj; + +static bool is_thread_sys_inited = false; + +/* Lock for thread data list */ +static mutex_t thread_data_lock; + +/* Thread data list */ +static os_thread_data *thread_data_list = NULL; + +static void +thread_data_list_add(os_thread_data *thread_data) +{ + mutex_lock(&thread_data_lock); + if (!thread_data_list) + thread_data_list = thread_data; + else { + /* If already in list, just return */ + os_thread_data *p = thread_data_list; + while (p) { + if (p == thread_data) { + mutex_unlock(&thread_data_lock); + return; + } + p = p->next; + } + + /* Set as head of list */ + thread_data->next = thread_data_list; + thread_data_list = thread_data; + } + mutex_unlock(&thread_data_lock); +} + +static void +thread_data_list_remove(os_thread_data *thread_data) +{ + mutex_lock(&thread_data_lock); + if (thread_data_list) { + if (thread_data_list == thread_data) + thread_data_list = thread_data_list->next; + else { + /* Search and remove it from list */ + os_thread_data *p = thread_data_list; + while (p && p->next != thread_data) + p = p->next; + if (p && p->next == thread_data) + p->next = p->next->next; + } + } + mutex_unlock(&thread_data_lock); +} + +static os_thread_data * +thread_data_list_lookup(korp_tid tid) +{ + mutex_lock(&thread_data_lock); + if (thread_data_list) { + os_thread_data *p = thread_data_list; + while (p) { + if (p->tid == tid) { + /* Found */ + mutex_unlock(&thread_data_lock); + return p; + } + p = p->next; + } + } + mutex_unlock(&thread_data_lock); + return NULL; +} + +int +os_thread_sys_init() +{ + if (is_thread_sys_inited) + return BHT_OK; + + mutex_init(&thread_data_lock); + + is_thread_sys_inited = true; + return BHT_OK; +} + +void +os_thread_sys_destroy() +{ + if (is_thread_sys_inited) { + is_thread_sys_inited = false; + } +} + +static os_thread_data * +thread_data_current() +{ + kernel_pid_t tid = thread_getpid(); + return thread_data_list_lookup(tid); +} + +static void +os_thread_cleanup(void) +{ + // TODO Check this (Join sema trigger, cleanup of thread_data) + os_thread_data *thread_data = thread_data_current(); + bh_assert(thread_data != NULL); + mutex_lock(&thread_data->wait_list_lock); + if (thread_data->thread_wait_list) { + /* Signal each joining thread */ + os_thread_wait_list head = thread_data->thread_wait_list; + while (head) { + os_thread_wait_list next = head->next; + head->ret = thread_data->arg; + sema_post(&head->sem); + head = next; + } + thread_data->thread_wait_list = NULL; + } + mutex_unlock(&thread_data->wait_list_lock); + + thread_data_list_remove(thread_data); +} + +static void * +os_thread_wrapper(void *thread_data) +{ + /* Set thread custom data */ + os_thread_data *t = (os_thread_data *)thread_data; + t->tid = thread_getpid(); + thread_data_list_add(t); + + // save the return value to arg since it is not need after the call + t->arg = (t->start_routine)(t->arg); + + os_thread_cleanup(); // internal structures and joiners + + BH_FREE(thread_data); + sched_task_exit(); // stop thread //clean + return NULL; // never reached +} + +int +os_thread_create(korp_tid *p_tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(p_tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +int +os_thread_create_with_prio(korp_tid *p_tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + kernel_pid_t tid; + os_thread_data *thread_data; + unsigned thread_data_size; + + if (!p_tid || !stack_size) + return BHT_ERROR; + + /* Create and initialize thread data */ + thread_data_size = offsetof(os_thread_data, stack) + stack_size; + if (!(thread_data = BH_MALLOC(thread_data_size))) { + return BHT_ERROR; + } + + memset(thread_data, 0, thread_data_size); + mutex_init(&thread_data->wait_list_lock); + thread_data->stack_size = stack_size; + thread_data->start_routine = start; + thread_data->arg = arg; + + /* Create the thread &*/ + if (!((tid = thread_create(thread_data->stack, stack_size, prio, 0, + os_thread_wrapper, thread_data, "WASM")))) { + BH_FREE(thread_data); + return BHT_ERROR; + } + + thread_data->tid = tid; + + /* Set thread custom data */ + thread_data_list_add(thread_data); + *p_tid = tid; + return BHT_OK; +} + +korp_tid +os_self_thread() +{ + return (korp_tid)thread_getpid(); +} + +int +os_thread_join(korp_tid thread, void **value_ptr) +{ + // will test if thread is still working, + // wait if it is + os_thread_data *thread_data; + os_thread_wait_node node; + + sema_create(&node.sem, 0); + node.next = NULL; + + /* Get thread data */ + thread_data = thread_data_list_lookup(thread); + if (thread_data == NULL) { + // thread not found + sema_destroy(&node.sem); + return BHT_ERROR; + } + bh_assert(thread_data != NULL); + + mutex_lock(&thread_data->wait_list_lock); + if (!thread_data->thread_wait_list) + thread_data->thread_wait_list = &node; + else { + /* Add to end of waiting list */ + os_thread_wait_node *p = thread_data->thread_wait_list; + while (p->next) + p = p->next; + p->next = &node; + } + mutex_unlock(&thread_data->wait_list_lock); + + sema_wait(&node.sem); + // get the return value pointer conted may not be availible after return + if (value_ptr) + (*value_ptr) = node.ret; + /* Wait some time for the thread to be actually terminated */ + // TODO: k_sleep(100); + + // TODO: bump target prio to make it finish and free its resources + thread_yield(); + + // node has done its job + sema_destroy(&node.sem); + + return BHT_OK; +} + +// int vm_mutex_trylock(korp_mutex *mutex) +// { +// return mutex_trylock(mutex); +// } + +int +os_mutex_init(korp_mutex *mutex) +{ + mutex_init(mutex); + return BHT_OK; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + (void)mutex; + return BHT_OK; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + mutex_lock(mutex); + return 0; // Riot mutexes do not return until success +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + mutex_unlock(mutex); + return 0; // Riot mutexes do not return until success +} + +int +os_cond_init(korp_cond *cond) +{ + mutex_init(&cond->wait_list_lock); + cond->thread_wait_list = NULL; + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ + (void)cond; + return BHT_OK; +} + +static int +os_cond_wait_internal(korp_cond *cond, korp_mutex *mutex, bool timed, + uint64 useconds) +{ + os_thread_wait_node *node; + + /* Create wait node and append it to wait list */ + if (!(node = BH_MALLOC(sizeof(os_thread_wait_node)))) + return BHT_ERROR; + + sema_create(&node->sem, 0); + node->next = NULL; + + mutex_lock(&cond->wait_list_lock); + if (!cond->thread_wait_list) + cond->thread_wait_list = node; + else { + /* Add to end of wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next) + p = p->next; + p->next = node; + } + mutex_unlock(&cond->wait_list_lock); + + /* Unlock mutex, wait sem and lock mutex again */ + mutex_unlock(mutex); + if (timed) + sema_wait(&node->sem); + else + sema_wait_timed_ztimer(&node->sem, ZTIMER_USEC, useconds); + mutex_lock(mutex); + + /* Remove wait node from wait list */ + mutex_lock(&cond->wait_list_lock); + if (cond->thread_wait_list == node) + cond->thread_wait_list = node->next; + else { + /* Remove from the wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next != node) + p = p->next; + p->next = node->next; + } + BH_FREE(node); + mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + return os_cond_wait_internal(cond, mutex, false, 0); +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + return os_cond_wait_internal(cond, mutex, (useconds != BHT_WAIT_FOREVER), + useconds); +} + +int +os_cond_signal(korp_cond *cond) +{ + /* Signal the head wait node of wait list */ + mutex_lock(&cond->wait_list_lock); + if (cond->thread_wait_list) + sema_post(&cond->thread_wait_list->sem); + mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +uint8 * +os_thread_get_stack_boundary() +{ +#if defined(DEVELHELP) || defined(SCHED_TEST_STACK) \ + || defined(MODULE_MPU_STACK_GUARD) + return (uint8 *)thread_get_active()->stack_start; +#else + return NULL; +#endif +} + +void +os_thread_jit_write_protect_np(bool enabled) +{} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/riot/riot_time.c b/wasm-micro-runtime/core/shared/platform/riot/riot_time.c new file mode 100644 index 0000000..ce73777 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/riot/riot_time.c @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * Copyright (C) 2020 TU Bergakademie Freiberg Karl Fessel + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include +#include + +#if IS_USED(MODULE_ZTIMER64_USEC) +uint64 +os_time_get_boot_us() +{ + return ztimer64_now(ZTIMER64_USEC); +} +#elif IS_USED(MODULE_ZTIMER64_MSEC) +uint64 +os_time_get_boot_us() +{ + return ztimer64_now(ZTIMER64_MSEC) * 1000; +} +#else +#ifdef __GNUC__ +__attribute__((weak)) uint64 +os_time_get_boot_us(); +#endif +uint64 +os_time_get_boot_us() +{ + static uint64_t times; + return ++times; +} +#endif + +uint64 +os_time_thread_cputime_us(void) +{ + /* FIXME if u know the right api */ + return os_time_get_boot_us(); +} diff --git a/wasm-micro-runtime/core/shared/platform/riot/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/riot/shared_platform.cmake new file mode 100644 index 0000000..52cf904 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/riot/shared_platform.cmake @@ -0,0 +1,17 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# Copyright (C) 2020 TU Bergakademie Freiberg Karl Fessel +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_RIOT) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +# include (${CMAKE_CURRENT_LIST_DIR}/../common/math/platform_api_math.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE}) + diff --git a/wasm-micro-runtime/core/shared/platform/rt-thread/SConscript b/wasm-micro-runtime/core/shared/platform/rt-thread/SConscript new file mode 100644 index 0000000..1e93f47 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/rt-thread/SConscript @@ -0,0 +1,34 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + + +from building import * +import os + +cwd = GetCurrentDir() + +src = Split(''' +''') + + +def addSrcFiles(arr, path): + for f in os.listdir(path): + fpath = os.path.join(path, f); + if os.path.isfile(fpath): + ext = os.path.splitext(fpath)[-1] + if ext == '.c' or ext == '.cpp': + arr += [fpath] + elif os.path.isdir(fpath): + addSrcFiles(arr, fpath) + + + +addSrcFiles(src, cwd); +CPPPATH = [cwd, cwd+'/../include'] + +group = DefineGroup('iwasm_platform_core', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/shared/platform/rt-thread/platform_internal.h b/wasm-micro-runtime/core/shared/platform/rt-thread/platform_internal.h new file mode 100644 index 0000000..4ebdabb --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/rt-thread/platform_internal.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef RTTHREAD_PLATFORM_INTERNAL_H +#define RTTHREAD_PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(WASM_ENABLE_AOT) +#if defined(RTT_WAMR_BUILD_TARGET_THUMB) +#define BUILD_TARGET "thumbv4t" +#elif defined(RTT_WAMR_BUILD_TARGET_ARMV7) +#define BUILD_TARGET "armv7" +#elif defined(RTT_WAMR_BUILD_TARGET_ARMV6) +#define BUILD_TARGET "armv6" +#elif defined(RTT_WAMR_BUILD_TARGET_ARMV4) +#define BUILD_TARGET "armv4" +#elif defined(RTT_WAMR_BUILD_TARGET_X86_32) +#define BUILD_TARGET "X86_32" +#else +#error "unsupported aot platform." +#endif +#endif /* WASM_ENABLE_AOT */ + +typedef rt_thread_t korp_tid; +typedef struct rt_mutex korp_mutex; +typedef struct rt_thread korp_cond; +typedef struct rt_thread korp_thread; +typedef unsigned int korp_sem; + +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + +typedef rt_uint8_t uint8_t; +typedef rt_int8_t int8_t; +typedef rt_uint16_t uint16_t; +typedef rt_int16_t int16_t; +typedef rt_uint64_t uint64_t; +typedef rt_int64_t int64_t; + +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ +typedef int os_file_handle; +typedef void *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#endif /* RTTHREAD_PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/rt-thread/rtt_platform.c b/wasm-micro-runtime/core/shared/platform/rt-thread/rtt_platform.c new file mode 100644 index 0000000..31fb316 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/rt-thread/rtt_platform.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +typedef struct os_malloc_list { + void *real; + void *used; + rt_list_t node; +} os_malloc_list_t; + +int +bh_platform_init(void) +{ + return 0; +} + +void +bh_platform_destroy(void) +{} + +void * +os_malloc(unsigned size) +{ + void *buf_origin; + void *buf_fixed; + rt_ubase_t *addr_field; + + buf_origin = rt_malloc(size + 8 + sizeof(rt_ubase_t)); + buf_fixed = buf_origin + sizeof(void *); + if ((rt_ubase_t)buf_fixed & 0x7) { + buf_fixed = (void *)((rt_ubase_t)(buf_fixed + 8) & (~7)); + } + + addr_field = buf_fixed - sizeof(rt_ubase_t); + *addr_field = (rt_ubase_t)buf_origin; + + return buf_fixed; +} + +void * +os_realloc(void *ptr, unsigned size) +{ + + void *mem_origin; + void *mem_new; + void *mem_new_fixed; + rt_ubase_t *addr_field; + + if (!ptr) { + return RT_NULL; + } + + addr_field = ptr - sizeof(rt_ubase_t); + mem_origin = (void *)(*addr_field); + mem_new = rt_realloc(mem_origin, size + 8 + sizeof(rt_ubase_t)); + + if (mem_origin != mem_new) { + mem_new_fixed = mem_new + sizeof(rt_ubase_t); + if ((rt_ubase_t)mem_new_fixed & 0x7) { + mem_new_fixed = (void *)((rt_ubase_t)(mem_new_fixed + 8) & (~7)); + } + + addr_field = mem_new_fixed - sizeof(rt_ubase_t); + *addr_field = (rt_ubase_t)mem_new; + + return mem_new_fixed; + } + + return ptr; +} + +void +os_free(void *ptr) +{ + void *mem_origin; + rt_ubase_t *addr_field; + + if (ptr) { + addr_field = ptr - sizeof(rt_ubase_t); + mem_origin = (void *)(*addr_field); + + rt_free(mem_origin); + } +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} + +static char wamr_vprint_buf[RT_CONSOLEBUF_SIZE * 2]; + +int +os_printf(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + rt_size_t len = + vsnprintf(wamr_vprint_buf, sizeof(wamr_vprint_buf) - 1, format, ap); + wamr_vprint_buf[len] = 0x00; + rt_kputs(wamr_vprint_buf); + va_end(ap); + return 0; +} + +int +os_vprintf(const char *format, va_list ap) +{ + rt_size_t len = + vsnprintf(wamr_vprint_buf, sizeof(wamr_vprint_buf) - 1, format, ap); + wamr_vprint_buf[len] = 0; + rt_kputs(wamr_vprint_buf); + return 0; +} + +uint64 +os_time_get_boot_us(void) +{ + uint64 ret = rt_tick_get() * 1000; + ret /= RT_TICK_PER_SECOND; + return ret; +} + +uint64 +os_time_thread_cputime_us(void) +{ + /* FIXME if u know the right api */ + return os_time_get_boot_us(); +} + +korp_tid +os_self_thread(void) +{ + return rt_thread_self(); +} + +uint8 * +os_thread_get_stack_boundary(void) +{ + rt_thread_t tid = rt_thread_self(); + return tid->stack_addr; +} + +void +os_thread_jit_write_protect_np(bool enabled) +{} + +int +os_mutex_init(korp_mutex *mutex) +{ + return rt_mutex_init(mutex, "wamr0", RT_IPC_FLAG_FIFO); +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + return rt_mutex_detach(mutex); +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + return rt_mutex_take(mutex, RT_WAITING_FOREVER); +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + return rt_mutex_release(mutex); +} + +/* + * functions below was not implement + */ + +int +os_cond_init(korp_cond *cond) +{ + return 0; +} + +int +os_cond_destroy(korp_cond *cond) +{ + return 0; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + return 0; +} + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + return rt_malloc(size); +} + +void +os_munmap(void *addr, size_t size) +{ + rt_free(addr); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + return 0; +} + +void +os_dcache_flush(void) +{} + +void +os_icache_flush(void *start, size_t len) +{} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/rt-thread/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/rt-thread/shared_platform.cmake new file mode 100644 index 0000000..fce9bff --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/rt-thread/shared_platform.cmake @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_RTT) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +# include (${CMAKE_CURRENT_LIST_DIR}/../common/math/platform_api_math.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE}) + diff --git a/wasm-micro-runtime/core/shared/platform/vxworks/platform_init.c b/wasm-micro-runtime/core/shared/platform/vxworks/platform_init.c new file mode 100644 index 0000000..2aae13f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/vxworks/platform_init.c @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +int +bh_platform_init() +{ + return 0; +} + +void +bh_platform_destroy() +{} + +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} diff --git a/wasm-micro-runtime/core/shared/platform/vxworks/platform_internal.h b/wasm-micro-runtime/core/shared/platform/vxworks/platform_internal.h new file mode 100644 index 0000000..1b870c7 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/vxworks/platform_internal.h @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_VXWORKS +#define BH_PLATFORM_VXWORKS +#endif + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (32 * 1024) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef sem_t korp_sem; + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define os_thread_local_attribute __thread + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) + +#include + +#define OS_ENABLE_HW_BOUND_CHECK + +typedef jmp_buf korp_jmpbuf; + +#define os_setjmp setjmp +#define os_longjmp longjmp +#define os_alloca alloca + +typedef void (*os_signal_handler)(void *sig_addr); + +int +os_thread_signal_init(os_signal_handler handler); + +void +os_thread_signal_destroy(); + +bool +os_thread_signal_inited(); + +void +os_signal_unmask(); + +void +os_sigreturn(); +#endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64 */ +#endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ + +#define os_getpagesize getpagesize + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/vxworks/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/vxworks/shared_platform.cmake new file mode 100644 index 0000000..6979ce2 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/vxworks/shared_platform.cmake @@ -0,0 +1,18 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_VXWORKS) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE}) + +file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) +LIST (APPEND RUNTIME_LIB_HEADER_LIST ${header}) diff --git a/wasm-micro-runtime/core/shared/platform/windows/platform_init.c b/wasm-micro-runtime/core/shared/platform/windows/platform_init.c new file mode 100644 index 0000000..96bcf9a --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/platform_init.c @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +int +os_thread_sys_init(); + +void +os_thread_sys_destroy(); + +int +init_winsock(); + +void +deinit_winsock(); + +int +bh_platform_init() +{ + if (init_winsock() != 0) { + return -1; + } + + return os_thread_sys_init(); +} + +void +bh_platform_destroy() +{ + deinit_winsock(); + + os_thread_sys_destroy(); +} + +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} + +unsigned +os_getpagesize() +{ + SYSTEM_INFO sys_info; + GetNativeSystemInfo(&sys_info); + return (unsigned)sys_info.dwPageSize; +} + +void +os_dcache_flush(void) +{} + +void +os_icache_flush(void *start, size_t len) +{} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/windows/platform_internal.h b/wasm-micro-runtime/core/shared/platform/windows/platform_internal.h new file mode 100644 index 0000000..8bb77e7 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/platform_internal.h @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "platform_wasi_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_WINDOWS +#define BH_PLATFORM_WINDOWS +#endif + +#ifdef _MSC_VER +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif +#endif /* #ifdef _MSC_VER */ + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (32 * 1024) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef SSIZE_T ssize_t; + +typedef void *korp_thread; +typedef void *korp_tid; +typedef void *korp_mutex; +typedef void *korp_sem; + +typedef struct { + SRWLOCK lock; + bool exclusive; +} korp_rwlock; + +/** + * Create the mutex when os_mutex_lock is called, and no need to + * CloseHandle() for the static lock's lifetime, since + * "The system closes the handle automatically when the process + * terminates. The mutex object is destroyed when its last + * handle has been closed." + * Refer to: + * https://learn.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-createmutexa + */ +#define OS_THREAD_MUTEX_INITIALIZER NULL + +struct os_thread_wait_node; +typedef struct os_thread_wait_node *os_thread_wait_list; +typedef struct korp_cond { + korp_mutex wait_list_lock; + os_thread_wait_list thread_wait_list; + struct os_thread_wait_node *thread_wait_list_end; +} korp_cond; + +unsigned +os_getpagesize(); +void * +os_mem_commit(void *ptr, size_t size, int flags); +void +os_mem_decommit(void *ptr, size_t size); + +#define os_thread_local_attribute __declspec(thread) + +#define strncasecmp _strnicmp +#define strcasecmp _stricmp + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + +#include + +#define OS_ENABLE_HW_BOUND_CHECK + +typedef jmp_buf korp_jmpbuf; + +#define os_setjmp setjmp +#define os_longjmp longjmp + +int +os_thread_signal_init(); + +void +os_thread_signal_destroy(); + +bool +os_thread_signal_inited(); + +#define os_signal_unmask() (void)0 +#define os_sigreturn() (void)0 + +#endif /* end of BUILD_TARGET_X86_64/AMD_64 */ +#endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ + +typedef enum os_memory_order { + os_memory_order_relaxed, + os_memory_order_consume, + os_memory_order_acquire, + os_memory_order_release, + os_memory_order_acq_rel, + os_memory_order_seq_cst, +} os_memory_order; + +void +bh_atomic_thread_fence(int mem_order); + +#define os_atomic_thread_fence bh_atomic_thread_fence + +typedef enum windows_handle_type { + windows_handle_type_socket, + windows_handle_type_file +} windows_handle_type; + +typedef enum windows_access_mode { + windows_access_mode_read = 1 << 0, + windows_access_mode_write = 1 << 1 +} windows_access_mode; + +typedef struct windows_handle { + windows_handle_type type; + __wasi_fdflags_t fdflags; + windows_access_mode access_mode; + union { + HANDLE handle; + SOCKET socket; + } raw; +} windows_handle; + +typedef struct windows_dir_stream { + // Enough space for the wide filename and the info struct itself + char info_buf[PATH_MAX * sizeof(wchar_t) + sizeof(FILE_ID_BOTH_DIR_INFO)]; + char current_entry_name[PATH_MAX]; + // An offset into info_buf to read the next entry from + DWORD cursor; + int cookie; + windows_handle *handle; +} windows_dir_stream; + +typedef windows_handle *os_file_handle; +typedef windows_dir_stream *os_dir_stream; + +#if WASM_ENABLE_UVWASI != 1 +typedef HANDLE os_raw_file_handle; +#else +typedef uint32_t os_raw_file_handle; +#endif + +#define bh_socket_t windows_handle * + +// UWP apps do not have stdout/stderr handles so provide a default +// implementation of vprintf on debug builds so output from WASI libc is sent to +// the debugger and not lost completely. +#if !defined(BH_VPRINTF) && !defined(NDEBUG) && WINAPI_PARTITION_DESKTOP == 0 +#define BH_VPRINTF uwp_print_to_debugger +#define UWP_DEFAULT_VPRINTF +#endif + +static inline os_file_handle +os_get_invalid_handle() +{ + return NULL; +} + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wasm-micro-runtime/core/shared/platform/windows/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/windows/shared_platform.cmake new file mode 100644 index 0000000..502a8a2 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/shared_platform.cmake @@ -0,0 +1,29 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_WINDOWS) +add_definitions(-DHAVE_STRUCT_TIMESPEC) +add_definitions(-D_WINSOCK_DEPRECATED_NO_WARNINGS) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c + ${PLATFORM_SHARED_DIR}/*.cpp) + +if (NOT WAMR_BUILD_LIBC_WASI EQUAL 1) + list(REMOVE_ITEM source_all ${PLATFORM_SHARED_DIR}/win_file.c) +else() + include (${CMAKE_CURRENT_LIST_DIR}/../common/libc-util/platform_common_libc_util.cmake) + set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) +endif() + +include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) +set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) + +set (PLATFORM_SHARED_SOURCE ${source_all}) + +file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) +LIST (APPEND RUNTIME_LIB_HEADER_LIST ${header}) diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_atomic.cpp b/wasm-micro-runtime/core/shared/platform/windows/win_atomic.cpp new file mode 100644 index 0000000..4e09405 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_atomic.cpp @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#if WASM_ENABLE_SHARED_MEMORY != 0 + +#include + +void +bh_atomic_thread_fence(int mem_order) +{ + std::memory_order order = + (std::memory_order)((int)std::memory_order::memory_order_relaxed + + mem_order - os_memory_order_relaxed); + std::atomic_thread_fence(order); +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_clock.c b/wasm-micro-runtime/core/shared/platform/windows/win_clock.c new file mode 100644 index 0000000..c96bdfb --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_clock.c @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_extension.h" +#include +#include "win_util.h" + +#define NANOSECONDS_PER_SECOND 1000000000ULL +#define NANOSECONDS_PER_TICK 100 + +static __wasi_errno_t +calculate_monotonic_clock_frequency(uint64 *out_frequency) +{ + LARGE_INTEGER frequency; + if (!QueryPerformanceFrequency(&frequency)) + return convert_windows_error_code(GetLastError()); + + *out_frequency = (uint64)frequency.QuadPart; + return __WASI_ESUCCESS; +} + +static __wasi_errno_t +get_performance_counter_value(uint64 *out_counter) +{ + LARGE_INTEGER counter; + if (!QueryPerformanceCounter(&counter)) + return convert_windows_error_code(GetLastError()); + + *out_counter = counter.QuadPart; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_clock_res_get(__wasi_clockid_t clock_id, __wasi_timestamp_t *resolution) +{ + __wasi_errno_t error = __WASI_ESUCCESS; + + switch (clock_id) { + case __WASI_CLOCK_MONOTONIC: + { + uint64 frequency; + error = calculate_monotonic_clock_frequency(&frequency); + + if (error != __WASI_ESUCCESS) + return error; + + const uint64 result = (uint64)NANOSECONDS_PER_SECOND / frequency; + *resolution = result; + return error; + } + case __WASI_CLOCK_REALTIME: + case __WASI_CLOCK_PROCESS_CPUTIME_ID: + case __WASI_CLOCK_THREAD_CPUTIME_ID: + { +#if WINAPI_PARTITION_DESKTOP + ULONG maximum_time; + ULONG minimum_time; + ULONG current_time; + NTSTATUS + status = NtQueryTimerResolution(&maximum_time, &minimum_time, + ¤t_time); + uint64 result = (uint64)current_time * NANOSECONDS_PER_TICK; + *resolution = result / (uint64)NANOSECONDS_PER_SECOND; + return error; +#else + return __WASI_ENOTSUP; +#endif + } + default: + return __WASI_EINVAL; + } +} + +__wasi_errno_t +os_clock_time_get(__wasi_clockid_t clock_id, __wasi_timestamp_t precision, + __wasi_timestamp_t *time) +{ + __wasi_errno_t error = __WASI_ESUCCESS; + + switch (clock_id) { + case __WASI_CLOCK_REALTIME: + { + FILETIME sys_now; +#if NTDDI_VERSION >= NTDDI_WIN8 + GetSystemTimePreciseAsFileTime(&sys_now); +#else + GetSystemTimeAsFileTime(&sys_now); +#endif + *time = convert_filetime_to_wasi_timestamp(&sys_now); + return BHT_OK; + } + case __WASI_CLOCK_MONOTONIC: + { + uint64 frequency; + error = calculate_monotonic_clock_frequency(&frequency); + + if (error != __WASI_ESUCCESS) + return error; + + uint64 counter; + error = get_performance_counter_value(&counter); + + if (error != __WASI_ESUCCESS) + return error; + + if (NANOSECONDS_PER_SECOND % frequency == 0) { + *time = counter * NANOSECONDS_PER_SECOND / frequency; + } + else { + uint64 seconds = counter / frequency; + uint64 fractions = counter % frequency; + *time = seconds * NANOSECONDS_PER_SECOND + + (fractions * NANOSECONDS_PER_SECOND) / frequency; + } + return error; + } + case __WASI_CLOCK_PROCESS_CPUTIME_ID: + case __WASI_CLOCK_THREAD_CPUTIME_ID: + { + FILETIME creation_time; + FILETIME exit_time; + FILETIME kernel_time; + FILETIME user_time; + + HANDLE handle = (clock_id == __WASI_CLOCK_PROCESS_CPUTIME_ID) + ? GetCurrentProcess() + : GetCurrentThread(); + + if (!GetProcessTimes(handle, &creation_time, &exit_time, + &kernel_time, &user_time)) + return convert_windows_error_code(GetLastError()); + + *time = convert_filetime_to_wasi_timestamp(&kernel_time) + + convert_filetime_to_wasi_timestamp(&user_time); + + return error; + } + default: + return __WASI_EINVAL; + } +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_file.c b/wasm-micro-runtime/core/shared/platform/windows/win_file.c new file mode 100644 index 0000000..63dfb5b --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_file.c @@ -0,0 +1,1794 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_extension.h" +#include "libc_errno.h" +#include "win_util.h" + +#include "PathCch.h" + +#pragma comment(lib, "Pathcch.lib") + +#define CHECK_VALID_HANDLE_WITH_RETURN_VALUE(win_handle, ret) \ + do { \ + if ((win_handle) == NULL \ + || ((win_handle)->type == windows_handle_type_socket \ + && (win_handle)->raw.socket == INVALID_SOCKET) \ + || ((win_handle)->type == windows_handle_type_file \ + && (win_handle)->raw.handle == INVALID_HANDLE_VALUE)) \ + return (ret); \ + \ + } while (0) + +#define CHECK_VALID_HANDLE(win_handle) \ + CHECK_VALID_HANDLE_WITH_RETURN_VALUE(win_handle, __WASI_EBADF) + +#define CHECK_VALID_FILE_HANDLE(win_handle) \ + do { \ + if ((win_handle) == NULL) \ + return __WASI_EBADF; \ + \ + if ((win_handle)->type == windows_handle_type_socket) \ + return __WASI_EINVAL; \ + \ + if (((win_handle)->type == windows_handle_type_file \ + && (win_handle)->raw.handle == INVALID_HANDLE_VALUE)) \ + return __WASI_EBADF; \ + \ + } while (0) + +#define CHECK_VALID_WIN_DIR_STREAM(win_dir_stream) \ + do { \ + if ((win_dir_stream) == NULL) \ + return __WASI_EINVAL; \ + CHECK_VALID_FILE_HANDLE((win_dir_stream)->handle); \ + } while (0) + +static __wasi_filetype_t +get_disk_filetype(DWORD attribute) +{ + if (attribute == INVALID_FILE_ATTRIBUTES) + return __WASI_FILETYPE_UNKNOWN; + if (attribute & FILE_ATTRIBUTE_REPARSE_POINT) + return __WASI_FILETYPE_SYMBOLIC_LINK; + if (attribute & FILE_ATTRIBUTE_DIRECTORY) + return __WASI_FILETYPE_DIRECTORY; + + return __WASI_FILETYPE_REGULAR_FILE; +} + +static __wasi_filetype_t +get_socket_filetype(SOCKET socket) +{ + char socket_type = 0; + int size = sizeof(socket_type); + + if (getsockopt(socket, SOL_SOCKET, SO_TYPE, &socket_type, &size) == 0) { + switch (socket_type) { + case SOCK_STREAM: + return __WASI_FILETYPE_SOCKET_STREAM; + case SOCK_DGRAM: + return __WASI_FILETYPE_SOCKET_DGRAM; + } + } + return __WASI_FILETYPE_UNKNOWN; +} + +static __wasi_errno_t +convert_windows_filetype(os_file_handle handle, DWORD filetype, + __wasi_filetype_t *out_filetype) +{ + __wasi_errno_t error = __WASI_ESUCCESS; + + switch (filetype) { + case FILE_TYPE_DISK: + FILE_ATTRIBUTE_TAG_INFO file_info; + + bool success = GetFileInformationByHandleEx( + handle->raw.handle, FileAttributeTagInfo, &file_info, + sizeof(file_info)); + + if (!success + || file_info.FileAttributes == INVALID_FILE_ATTRIBUTES) { + error = convert_windows_error_code(GetLastError()); + break; + } + + *out_filetype = get_disk_filetype(file_info.FileAttributes); + break; + case FILE_TYPE_CHAR: + *out_filetype = __WASI_FILETYPE_CHARACTER_DEVICE; + break; + case FILE_TYPE_PIPE: + if (handle->type == windows_handle_type_socket) + *out_filetype = get_socket_filetype(handle->raw.socket); + else + *out_filetype = __WASI_FILETYPE_BLOCK_DEVICE; + + break; + case FILE_TYPE_REMOTE: + case FILE_TYPE_UNKNOWN: + default: + *out_filetype = __WASI_FILETYPE_UNKNOWN; + } + + return error; +} + +// Converts the input string to a wchar string. +static __wasi_errno_t +convert_to_wchar(const char *str, wchar_t *buf, size_t buf_size) +{ + int converted_chars = + MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, (int)buf_size); + + if (converted_chars == 0) + return convert_windows_error_code(GetLastError()); + + return __WASI_ESUCCESS; +} + +// Get the filepath for a handle. The size of the buffer should be specified in +// terms of wchar. +static __wasi_errno_t +get_handle_filepath(HANDLE handle, wchar_t *buf, DWORD buf_size) +{ + DWORD bufsize_in_chars = buf_size * (sizeof(wchar_t) / sizeof(char)); + DWORD size = GetFinalPathNameByHandleW( + handle, buf, bufsize_in_chars, FILE_NAME_NORMALIZED | VOLUME_NAME_NONE); + + if (size > bufsize_in_chars) + return __WASI_ENAMETOOLONG; + + if (size == 0) + return convert_windows_error_code(GetLastError()); + + return __WASI_ESUCCESS; +} + +static __wasi_errno_t +convert_hresult_error_code(HRESULT error_code) +{ + switch (error_code) { + case E_OUTOFMEMORY: + return __WASI_ENOMEM; + case E_INVALIDARG: + default: + return __WASI_EINVAL; + } +} + +// Returns the absolute filepath from the relative path to the directory +// associated with the provided handle. +static __wasi_errno_t +get_absolute_filepath(HANDLE handle, const char *relative_path, + wchar_t *absolute_path, size_t buf_len) +{ + wchar_t handle_path[PATH_MAX]; + + __wasi_errno_t error = get_handle_filepath(handle, handle_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + wchar_t relative_wpath[PATH_MAX]; + error = convert_to_wchar(relative_path, relative_wpath, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + HRESULT ret = + PathCchCombine(absolute_path, buf_len, handle_path, relative_wpath); + if (ret != S_OK) + error = convert_hresult_error_code(ret); + + return error; +} + +static bool +has_directory_attribute(DWORD attributes) +{ + if (attributes == INVALID_FILE_ATTRIBUTES) + return false; + + return (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0; +} + +static bool +is_directory(const wchar_t *path) +{ + DWORD attributes = GetFileAttributesW(path); + + return has_directory_attribute(attributes); +} + +static bool +has_symlink_attribute(DWORD attributes) +{ + if (attributes == INVALID_FILE_ATTRIBUTES) + return false; + + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; +} + +static void +init_dir_stream(os_dir_stream dir_stream, os_file_handle handle) +{ + dir_stream->cursor = 0; + dir_stream->handle = handle; + dir_stream->cookie = 0; +} + +static void +reset_dir_stream(os_dir_stream dir_stream) +{ + dir_stream->cursor = 0; + dir_stream->cookie = 0; +} + +// Advances to the next directory entry and optionally reads into to the +// provided buffer if not NULL. +static __wasi_errno_t +read_next_dir_entry(os_dir_stream dir_stream, FILE_ID_BOTH_DIR_INFO **out_entry) +{ + FILE_INFO_BY_HANDLE_CLASS file_info_class; + + if (dir_stream->cookie == 0) + file_info_class = FileIdBothDirectoryRestartInfo; + else + file_info_class = FileIdBothDirectoryInfo; + + if (dir_stream->cursor == 0 + && !GetFileInformationByHandleEx(dir_stream->handle->raw.handle, + file_info_class, dir_stream->info_buf, + sizeof(dir_stream->info_buf))) { + if (out_entry != NULL) + *out_entry = NULL; + DWORD win_error = GetLastError(); + // We've reached the end of the directory - return success + if (win_error == ERROR_NO_MORE_FILES) { + dir_stream->cookie = 0; + dir_stream->cursor = 0; + return __WASI_ESUCCESS; + } + + return convert_windows_error_code(win_error); + } + + FILE_ID_BOTH_DIR_INFO *current_info = + (FILE_ID_BOTH_DIR_INFO *)(dir_stream->info_buf + dir_stream->cursor); + + if (current_info->NextEntryOffset == 0) + dir_stream->cursor = 0; + else + dir_stream->cursor += current_info->NextEntryOffset; + + ++dir_stream->cookie; + + if (out_entry != NULL) + *out_entry = current_info; + else + return __WASI_ESUCCESS; + + // Convert and copy over the wchar filename into the entry_name buf + int ret = WideCharToMultiByte( + CP_UTF8, 0, current_info->FileName, + current_info->FileNameLength / (sizeof(wchar_t) / sizeof(char)), + dir_stream->current_entry_name, sizeof(dir_stream->current_entry_name), + NULL, NULL); + + if (ret == 0) + return convert_windows_error_code(GetLastError()); + + return __WASI_ESUCCESS; +} + +static HANDLE +create_handle(wchar_t *path, bool is_dir, bool follow_symlink, bool readonly) +{ + CREATEFILE2_EXTENDED_PARAMETERS create_params; + + create_params.dwSize = sizeof(create_params); + create_params.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; + create_params.dwSecurityQosFlags = 0; + create_params.dwFileFlags = 0; + create_params.lpSecurityAttributes = NULL; + create_params.hTemplateFile = NULL; + + if (is_dir) { + create_params.dwFileAttributes |= FILE_ATTRIBUTE_DIRECTORY; + create_params.dwFileFlags |= FILE_FLAG_BACKUP_SEMANTICS; + } + + if (!follow_symlink) + create_params.dwFileFlags |= FILE_FLAG_OPEN_REPARSE_POINT; + + DWORD desired_access = GENERIC_READ; + + if (!readonly) + desired_access |= GENERIC_WRITE; + else + create_params.dwFileAttributes |= FILE_ATTRIBUTE_READONLY; + + return CreateFile2(path, desired_access, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, &create_params); +} + +#if WINAPI_PARTITION_DESKTOP == 0 +// Modifies the given path in place and replaces it with the filename component +// (including the extension) of the path. +static __wasi_errno_t +extract_filename_from_path(wchar_t *path, size_t buf_size) +{ + wchar_t extension[256]; + wchar_t filename[256]; + __wasi_errno_t error = __WASI_ESUCCESS; + + // Get the filename from the fullpath. + errno_t ret = + _wsplitpath_s(path, NULL, 0, NULL, 0, filename, 256, extension, 256); + if (ret != 0) { + error = convert_errno(ret); + return error; + } + + ret = wcscat_s(filename, 256, extension); + + if (ret != 0) { + error = convert_errno(ret); + return error; + } + + ret = wcscpy_s(path, buf_size, filename); + + if (ret != 0) + error = convert_errno(ret); + + return error; +} + +static __wasi_errno_t +get_handle_to_parent_directory(HANDLE handle, HANDLE *out_dir_handle) +{ + wchar_t path[PATH_MAX]; + __wasi_errno_t error = get_handle_filepath(handle, path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + wchar_t parent_dir_path[PATH_MAX]; + errno_t ret = wcscpy_s(parent_dir_path, PATH_MAX, path); + + if (ret != 0) { + error = convert_errno(ret); + return error; + } + + ret = wcscat_s(parent_dir_path, PATH_MAX, L"/.."); + + if (ret != 0) { + error = convert_errno(ret); + return error; + } + + HANDLE dir_handle = create_handle(parent_dir_path, true, true, true); + + if (dir_handle == INVALID_HANDLE_VALUE) { + error = convert_windows_error_code(GetLastError()); + return error; + } + + *out_dir_handle = dir_handle; + return error; +} + +// The easiest way to get all the necessary file information for files is to +// open a handle to the parent directory and iterate through the entries via +// FileIdBothDirectoryInfo. Other file information classes are only +// available on desktop. +static __wasi_errno_t +get_disk_file_information(HANDLE handle, __wasi_filestat_t *buf) +{ + __wasi_errno_t error = __WASI_ESUCCESS; + HANDLE raw_dir_handle = INVALID_HANDLE_VALUE; + + wchar_t path[PATH_MAX] = L"."; + + if (buf->st_filetype != __WASI_FILETYPE_DIRECTORY) { + error = get_handle_filepath(handle, path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + goto fail; + + error = get_handle_to_parent_directory(handle, &raw_dir_handle); + + if (error != __WASI_ESUCCESS) + goto fail; + + error = extract_filename_from_path(path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + goto fail; + } + else { + raw_dir_handle = handle; + } + + windows_handle dir_handle = { .access_mode = windows_access_mode_read, + .raw = { .handle = raw_dir_handle }, + .fdflags = 0, + .type = windows_handle_type_file }; + windows_dir_stream dir_stream; + init_dir_stream(&dir_stream, &dir_handle); + + do { + FILE_ID_BOTH_DIR_INFO *file_id_both_dir_info = NULL; + __wasi_errno_t error = + read_next_dir_entry(&dir_stream, &file_id_both_dir_info); + + if (error != __WASI_ESUCCESS || file_id_both_dir_info == NULL) + goto fail; + + const DWORD filename_length = file_id_both_dir_info->FileNameLength + / (sizeof(wchar_t) / sizeof(char)); + + if (wcsncmp(file_id_both_dir_info->FileName, path, filename_length) + == 0) { + buf->st_ino = + (__wasi_inode_t)(file_id_both_dir_info->FileId.QuadPart); + buf->st_atim = convert_filetime_to_wasi_timestamp( + (LPFILETIME)&file_id_both_dir_info->LastAccessTime.QuadPart); + buf->st_mtim = convert_filetime_to_wasi_timestamp( + (LPFILETIME)&file_id_both_dir_info->LastWriteTime.QuadPart); + buf->st_ctim = convert_filetime_to_wasi_timestamp( + (LPFILETIME)&file_id_both_dir_info->ChangeTime.QuadPart); + buf->st_size = + (__wasi_filesize_t)(file_id_both_dir_info->EndOfFile.QuadPart); + + break; + } + } while (dir_stream.cookie != 0); + + FILE_STANDARD_INFO file_standard_info; + + bool success = GetFileInformationByHandleEx(handle, FileStandardInfo, + &file_standard_info, + sizeof(file_standard_info)); + + if (!success) { + error = convert_windows_error_code(GetLastError()); + goto fail; + } + + buf->st_nlink = (__wasi_linkcount_t)file_standard_info.NumberOfLinks; +fail: + if (buf->st_filetype != __WASI_FILETYPE_DIRECTORY + && raw_dir_handle != INVALID_HANDLE_VALUE) + CloseHandle(raw_dir_handle); + + return error; +} + +#else + +static __wasi_errno_t +get_disk_file_information(HANDLE handle, __wasi_filestat_t *buf) +{ + __wasi_errno_t error = __WASI_ESUCCESS; + FILE_BASIC_INFO file_basic_info; + + int ret = GetFileInformationByHandleEx( + handle, FileBasicInfo, &file_basic_info, sizeof(file_basic_info)); + + if (ret == 0) { + error = convert_windows_error_code(GetLastError()); + return error; + } + + buf->st_atim = convert_filetime_to_wasi_timestamp( + (LPFILETIME)&file_basic_info.LastAccessTime.QuadPart); + buf->st_mtim = convert_filetime_to_wasi_timestamp( + (LPFILETIME)&file_basic_info.LastWriteTime.QuadPart); + buf->st_ctim = convert_filetime_to_wasi_timestamp( + (LPFILETIME)&file_basic_info.ChangeTime.QuadPart); + + BY_HANDLE_FILE_INFORMATION file_info; + ret = GetFileInformationByHandle(handle, &file_info); + + if (ret == 0) { + error = convert_windows_error_code(GetLastError()); + return error; + } + + ULARGE_INTEGER file_size = { .LowPart = file_info.nFileSizeLow, + .HighPart = file_info.nFileSizeHigh }; + buf->st_size = (__wasi_filesize_t)(file_size.QuadPart); + + ULARGE_INTEGER file_id = { .LowPart = file_info.nFileIndexLow, + .HighPart = file_info.nFileIndexHigh }; + buf->st_ino = (__wasi_inode_t)(file_id.QuadPart); + + buf->st_dev = (__wasi_device_t)file_info.dwVolumeSerialNumber; + buf->st_nlink = (__wasi_linkcount_t)file_info.nNumberOfLinks; + + return error; +} + +#endif /* end of WINAPI_PARTITION_DESKTOP == 0 */ + +static __wasi_errno_t +get_file_information(os_file_handle handle, __wasi_filestat_t *buf) +{ + __wasi_errno_t error = __WASI_ESUCCESS; + + DWORD windows_filetype = GetFileType(handle->raw.handle); + error = + convert_windows_filetype(handle, windows_filetype, &buf->st_filetype); + + if (error != __WASI_ESUCCESS) + return error; + + buf->st_dev = 0; + + if (windows_filetype != FILE_TYPE_DISK) { + buf->st_atim = 0; + buf->st_ctim = 0; + buf->st_mtim = 0; + buf->st_nlink = 0; + buf->st_size = 0; + buf->st_ino = 0; + + return error; + } + + return get_disk_file_information(handle->raw.handle, buf); +} + +__wasi_errno_t +os_fstat(os_file_handle handle, struct __wasi_filestat_t *buf) +{ + CHECK_VALID_HANDLE(handle); + + return get_file_information(handle, buf); +} + +__wasi_errno_t +os_fstatat(os_file_handle handle, const char *path, + struct __wasi_filestat_t *buf, __wasi_lookupflags_t lookup_flags) +{ + CHECK_VALID_FILE_HANDLE(handle); + + wchar_t absolute_path[PATH_MAX]; + + __wasi_errno_t error = get_absolute_filepath(handle->raw.handle, path, + absolute_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + windows_handle resolved_handle = { + .type = windows_handle_type_file, + .fdflags = 0, + .raw = { .handle = create_handle( + absolute_path, is_directory(absolute_path), + ((lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) != 0), + true) }, + .access_mode = windows_access_mode_read + }; + + if (resolved_handle.raw.handle == INVALID_HANDLE_VALUE) + return convert_windows_error_code(GetLastError()); + + error = get_file_information(&resolved_handle, buf); + + CloseHandle(resolved_handle.raw.handle); + + return error; +} + +__wasi_errno_t +os_file_get_fdflags(os_file_handle handle, __wasi_fdflags_t *flags) +{ + CHECK_VALID_HANDLE(handle); + + *flags = handle->fdflags; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_file_set_fdflags(os_file_handle handle, __wasi_fdflags_t flags) +{ + CHECK_VALID_HANDLE(handle); + + if (handle->type == windows_handle_type_socket + && (((handle->fdflags ^ flags) & __WASI_FDFLAG_NONBLOCK) != 0)) { + u_long non_block = flags & __WASI_FDFLAG_NONBLOCK; + + int ret = ioctlsocket(handle->raw.socket, (long)FIONBIO, &non_block); + + if (ret != 0) + return convert_winsock_error_code(WSAGetLastError()); + + if (non_block) + handle->fdflags |= __WASI_FDFLAG_NONBLOCK; + else + handle->fdflags &= ~__WASI_FDFLAG_NONBLOCK; + return __WASI_ESUCCESS; + } + + // It's not supported setting FILE_FLAG_WRITE_THROUGH or + // FILE_FLAG_NO_BUFFERING via SetFileAttributes so __WASI_FDFLAG_APPEND is + // the only flags we can do anything with. + if (((handle->fdflags ^ flags) & __WASI_FDFLAG_APPEND) != 0) { + if ((flags & __WASI_FDFLAG_APPEND) != 0) + handle->fdflags |= __WASI_FDFLAG_APPEND; + else + handle->fdflags &= ~__WASI_FDFLAG_APPEND; + } + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_file_get_access_mode(os_file_handle handle, + wasi_libc_file_access_mode *access_mode) +{ + CHECK_VALID_HANDLE(handle); + + if ((handle->access_mode & windows_access_mode_read) != 0 + && (handle->access_mode & windows_access_mode_write) != 0) + *access_mode = WASI_LIBC_ACCESS_MODE_READ_WRITE; + else if ((handle->access_mode & windows_access_mode_write) != 0) + *access_mode = WASI_LIBC_ACCESS_MODE_WRITE_ONLY; + else + *access_mode = WASI_LIBC_ACCESS_MODE_READ_ONLY; + + return __WASI_ESUCCESS; +} + +static __wasi_errno_t +flush_file_buffers_on_handle(HANDLE handle) +{ + bool success = FlushFileBuffers(handle); + + return success ? __WASI_ESUCCESS + : convert_windows_error_code(GetLastError()); +} + +__wasi_errno_t +os_fdatasync(os_file_handle handle) +{ + CHECK_VALID_FILE_HANDLE(handle); + + return flush_file_buffers_on_handle(handle->raw.handle); +} + +__wasi_errno_t +os_fsync(os_file_handle handle) +{ + CHECK_VALID_FILE_HANDLE(handle); + + return flush_file_buffers_on_handle(handle->raw.handle); +} + +__wasi_errno_t +os_open_preopendir(const char *path, os_file_handle *out) +{ + *out = NULL; + + wchar_t wpath[PATH_MAX]; + __wasi_errno_t error = convert_to_wchar(path, wpath, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + HANDLE dir_handle = create_handle(wpath, true, true, true); + + if (dir_handle == INVALID_HANDLE_VALUE) + return convert_windows_error_code(GetLastError()); + + *out = BH_MALLOC(sizeof(windows_handle)); + + if (*out == NULL) { + CloseHandle(dir_handle); + return __WASI_ENOMEM; + } + + (*out)->type = windows_handle_type_file; + (*out)->raw.handle = dir_handle; + (*out)->fdflags = 0; + (*out)->access_mode = windows_access_mode_read; + + return error; +} + +__wasi_errno_t +os_openat(os_file_handle handle, const char *path, __wasi_oflags_t oflags, + __wasi_fdflags_t fs_flags, __wasi_lookupflags_t lookup_flags, + wasi_libc_file_access_mode access_mode, os_file_handle *out) +{ + CHECK_VALID_FILE_HANDLE(handle); + *out = BH_MALLOC(sizeof(windows_handle)); + + if (*out == NULL) + return __WASI_ENOMEM; + + (*out)->type = windows_handle_type_file; + (*out)->fdflags = fs_flags; + (*out)->raw.handle = INVALID_HANDLE_VALUE; + + DWORD attributes = FILE_FLAG_BACKUP_SEMANTICS; + + if ((fs_flags & (__WASI_FDFLAG_SYNC | __WASI_FDFLAG_RSYNC)) != 0) + attributes |= (FILE_FLAG_WRITE_THROUGH | FILE_FLAG_NO_BUFFERING); + if ((fs_flags & __WASI_FDFLAG_DSYNC) != 0) + attributes |= FILE_FLAG_WRITE_THROUGH; + + if ((oflags & __WASI_O_DIRECTORY) != 0) { + attributes |= FILE_ATTRIBUTE_DIRECTORY; + oflags &= ~(__WASI_O_DIRECTORY); + } + // Use async operations on the handle if it's not a directory + else { + attributes |= FILE_FLAG_OVERLAPPED; + } + + __wasi_errno_t error = __WASI_ESUCCESS; + + DWORD access_flags = 0; + + switch (access_mode) { + case WASI_LIBC_ACCESS_MODE_READ_ONLY: + access_flags |= GENERIC_READ; + (*out)->access_mode = windows_access_mode_read; + break; + case WASI_LIBC_ACCESS_MODE_WRITE_ONLY: + access_flags |= GENERIC_WRITE; + (*out)->access_mode = windows_access_mode_write; + break; + case WASI_LIBC_ACCESS_MODE_READ_WRITE: + access_flags |= GENERIC_WRITE | GENERIC_READ; + (*out)->access_mode = + windows_access_mode_read | windows_access_mode_write; + break; + } + + DWORD creation_disposition = 0; + + switch (oflags) { + case __WASI_O_CREAT | __WASI_O_EXCL: + case __WASI_O_CREAT | __WASI_O_EXCL | __WASI_O_TRUNC: + creation_disposition = CREATE_NEW; + break; + case __WASI_O_CREAT | __WASI_O_TRUNC: + creation_disposition = CREATE_ALWAYS; + break; + case __WASI_O_CREAT: + creation_disposition = OPEN_ALWAYS; + break; + case 0: + case __WASI_O_EXCL: + creation_disposition = OPEN_EXISTING; + break; + case __WASI_O_TRUNC: + case __WASI_O_EXCL | __WASI_O_TRUNC: + creation_disposition = TRUNCATE_EXISTING; + // CreateFile2 requires write access if we truncate the file upon + // opening + access_flags |= GENERIC_WRITE; + break; + } + + wchar_t absolute_path[PATH_MAX]; + error = get_absolute_filepath(handle->raw.handle, path, absolute_path, + PATH_MAX); + + if (error != __WASI_ESUCCESS) + goto fail; + + if ((lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) == 0) + attributes |= FILE_FLAG_OPEN_REPARSE_POINT; + + // Windows doesn't seem to throw an error for the following cases where the + // file/directory already exists so add explicit checks. + if (creation_disposition == OPEN_EXISTING) { + DWORD file_attributes = GetFileAttributesW(absolute_path); + + if (file_attributes != INVALID_FILE_ATTRIBUTES) { + bool is_dir = file_attributes & FILE_ATTRIBUTE_DIRECTORY; + bool is_symlink = file_attributes & FILE_ATTRIBUTE_REPARSE_POINT; + // Check that we're not trying to open an existing file/symlink as a + // directory. + if ((attributes & FILE_ATTRIBUTE_DIRECTORY) != 0 + && (!is_dir || is_symlink)) { + error = __WASI_ENOTDIR; + goto fail; + } + + // Check that we're not trying to open an existing symlink with + // O_NOFOLLOW. + if ((file_attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 + && (lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) == 0) { + error = __WASI_ELOOP; + goto fail; + } + } + } + + CREATEFILE2_EXTENDED_PARAMETERS create_params; + create_params.dwSize = sizeof(create_params); + create_params.dwFileAttributes = attributes & 0xFFF; + create_params.dwFileFlags = attributes & 0xFFF00000; + create_params.dwSecurityQosFlags = 0; + create_params.lpSecurityAttributes = NULL; + create_params.hTemplateFile = NULL; + + (*out)->raw.handle = + CreateFile2(absolute_path, access_flags, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + creation_disposition, &create_params); + + if ((*out)->raw.handle == INVALID_HANDLE_VALUE) { + error = convert_windows_error_code(GetLastError()); + goto fail; + } + + return error; +fail: + if (*out != NULL) { + if ((*out)->raw.handle != INVALID_HANDLE_VALUE) + CloseHandle((*out)->raw.handle); + + BH_FREE(*out); + } + + return error; +} + +__wasi_errno_t +os_close(os_file_handle handle, bool is_stdio) +{ + CHECK_VALID_HANDLE(handle); + + // We don't own the underlying raw handle so just free the handle and return + // success. + if (is_stdio) { + BH_FREE(handle); + return __WASI_ESUCCESS; + } + + switch (handle->type) { + case windows_handle_type_file: + bool success = CloseHandle(handle->raw.handle); + + if (!success) + return convert_windows_error_code(GetLastError()); + + break; + case windows_handle_type_socket: + int ret = closesocket(handle->raw.socket); + + if (ret != 0) + return convert_winsock_error_code(WSAGetLastError()); + + break; + default: + assert(false && "unreachable"); + } + + BH_FREE(handle); + + return __WASI_ESUCCESS; +} + +static __wasi_errno_t +read_data_at_offset(HANDLE handle, const struct __wasi_iovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nwritten) +{ + OVERLAPPED *read_operations = + BH_MALLOC((uint32_t)(sizeof(OVERLAPPED) * (uint32_t)iovcnt)); + + if (read_operations == NULL) + return __WASI_ENOMEM; + + ULARGE_INTEGER query_offset = { .QuadPart = offset }; + __wasi_errno_t error = __WASI_ESUCCESS; + size_t total_bytes_read = 0; + + const __wasi_iovec_t *current = iov; + int successful_read_count = 0; + + for (int i = 0; i < iovcnt; ++i, ++current) { + read_operations[i].Internal = 0; + read_operations[i].InternalHigh = 0; + read_operations[i].Offset = query_offset.LowPart; + read_operations[i].OffsetHigh = query_offset.HighPart; + read_operations[i].hEvent = NULL; + + if (!ReadFileEx(handle, current->buf, (DWORD)current->buf_len, + &read_operations[i], NULL)) { + DWORD win_error = GetLastError(); + if (win_error != ERROR_IO_PENDING) { + error = convert_windows_error_code(win_error); + break; + } + } + ++successful_read_count; + query_offset.QuadPart += (DWORD)current->buf_len; + } + + // Get the result of all the asynchronous read operations + for (int i = 0; i < successful_read_count; ++i) { + DWORD bytes_transferred = 0; + if (!GetOverlappedResult(handle, &read_operations[i], + &bytes_transferred, true)) { + DWORD win_error = GetLastError(); + + if (win_error != ERROR_HANDLE_EOF) + error = convert_windows_error_code(win_error); + else + total_bytes_read += (size_t)bytes_transferred; + + CancelIo(handle); + + for (int j = i + 1; j < iovcnt; ++j) { + GetOverlappedResult(handle, &read_operations[j], + &bytes_transferred, true); + } + break; + } + + total_bytes_read += (size_t)bytes_transferred; + } + + *nwritten = total_bytes_read; + + BH_FREE(read_operations); + return error; +} + +__wasi_errno_t +os_preadv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nread) +{ + CHECK_VALID_FILE_HANDLE(handle); + + return read_data_at_offset(handle->raw.handle, iov, iovcnt, offset, nread); +} + +__wasi_errno_t +os_readv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + size_t *nread) +{ + CHECK_VALID_HANDLE(handle); + + LARGE_INTEGER current_offset = { .QuadPart = 0 }; + + // Seek to the current offset before reading + int ret = SetFilePointerEx(handle->raw.handle, current_offset, + ¤t_offset, FILE_CURRENT); + if (ret == 0) + return convert_windows_error_code(GetLastError()); + + __wasi_errno_t error = + read_data_at_offset(handle->raw.handle, iov, iovcnt, + (__wasi_filesize_t)current_offset.QuadPart, nread); + + if (error != __WASI_ESUCCESS) + return error; + + current_offset.QuadPart += (LONGLONG)(*nread); + + // Update the current offset to match how many bytes we've read + ret = + SetFilePointerEx(handle->raw.handle, current_offset, NULL, FILE_BEGIN); + + if (ret == 0) + error = convert_windows_error_code(GetLastError()); + + return error; +} + +static __wasi_errno_t +write_data_at_offset(HANDLE handle, const struct __wasi_ciovec_t *iov, + int iovcnt, __wasi_filesize_t offset, size_t *nwritten) +{ + OVERLAPPED *write_operations = + BH_MALLOC((uint32_t)(sizeof(OVERLAPPED) * (uint32_t)iovcnt)); + + if (write_operations == NULL) + return __WASI_ENOMEM; + + ULARGE_INTEGER query_offset = { .QuadPart = offset }; + __wasi_errno_t error = __WASI_ESUCCESS; + size_t total_bytes_written = 0; + + const __wasi_ciovec_t *current = iov; + int successful_write_count = 0; + for (int i = 0; i < iovcnt; ++i, ++current) { + write_operations[i].Internal = 0; + write_operations[i].InternalHigh = 0; + write_operations[i].Offset = query_offset.LowPart; + write_operations[i].OffsetHigh = query_offset.HighPart; + write_operations[i].hEvent = NULL; + + if (!WriteFileEx(handle, current->buf, (DWORD)current->buf_len, + &write_operations[i], NULL)) { + DWORD win_error = GetLastError(); + if (win_error != ERROR_IO_PENDING) { + error = convert_windows_error_code(win_error); + break; + } + } + ++successful_write_count; + query_offset.QuadPart += (DWORD)current->buf_len; + } + + // Get the result of all the asynchronous writes + for (int i = 0; i < successful_write_count; ++i) { + DWORD bytes_transferred = 0; + if (!GetOverlappedResult(handle, &write_operations[i], + &bytes_transferred, true)) { + error = convert_windows_error_code(GetLastError()); + CancelIo(handle); + + for (int j = i + 1; j < iovcnt; ++j) { + GetOverlappedResult(handle, &write_operations[j], + &bytes_transferred, true); + } + break; + } + + total_bytes_written += (size_t)bytes_transferred; + } + + *nwritten = total_bytes_written; + + BH_FREE(write_operations); + return error; +} + +__wasi_errno_t +os_pwritev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nwritten) +{ + CHECK_VALID_FILE_HANDLE(handle); + + return write_data_at_offset(handle->raw.handle, iov, iovcnt, offset, + nwritten); +} + +__wasi_errno_t +os_writev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + size_t *nwritten) +{ + CHECK_VALID_HANDLE(handle); + + bool append = (handle->fdflags & __WASI_FDFLAG_APPEND) != 0; + LARGE_INTEGER write_offset = { .QuadPart = 0 }; + DWORD move_method = append ? FILE_END : FILE_CURRENT; + + int ret = SetFilePointerEx(handle->raw.handle, write_offset, &write_offset, + move_method); + if (ret == 0) + return convert_windows_error_code(GetLastError()); + + __wasi_errno_t error = write_data_at_offset( + handle->raw.handle, iov, iovcnt, + (__wasi_filesize_t)write_offset.QuadPart, nwritten); + + if (error != __WASI_ESUCCESS) + return error; + + write_offset.QuadPart += (LONGLONG)(*nwritten); + + // Update the write offset to match how many bytes we've written + ret = SetFilePointerEx(handle->raw.handle, write_offset, NULL, FILE_BEGIN); + + if (ret == 0) + error = convert_windows_error_code(GetLastError()); + + return error; +} + +__wasi_errno_t +os_fallocate(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length) +{ + CHECK_VALID_FILE_HANDLE(handle); + + LARGE_INTEGER current_file_size; + int ret = GetFileSizeEx(handle->raw.handle, ¤t_file_size); + + if (ret == 0) + return convert_windows_error_code(GetLastError()); + + if (offset > INT64_MAX || length > INT64_MAX || offset + length > INT64_MAX) + return __WASI_EINVAL; + + // The best we can do here is to increase the size of the file if it's less + // than the offset + length. + const LONGLONG requested_size = (LONGLONG)(offset + length); + + FILE_END_OF_FILE_INFO end_of_file_info; + end_of_file_info.EndOfFile.QuadPart = requested_size; + + if (requested_size <= current_file_size.QuadPart) + return __WASI_ESUCCESS; + + bool success = + SetFileInformationByHandle(handle->raw.handle, FileEndOfFileInfo, + &end_of_file_info, sizeof(end_of_file_info)); + + return success ? __WASI_ESUCCESS + : convert_windows_error_code(GetLastError()); +} + +__wasi_errno_t +os_ftruncate(os_file_handle handle, __wasi_filesize_t size) +{ + CHECK_VALID_FILE_HANDLE(handle); + + FILE_END_OF_FILE_INFO end_of_file_info; + end_of_file_info.EndOfFile.QuadPart = (LONGLONG)size; + + bool success = + SetFileInformationByHandle(handle->raw.handle, FileEndOfFileInfo, + &end_of_file_info, sizeof(end_of_file_info)); + + return success ? __WASI_ESUCCESS + : convert_windows_error_code(GetLastError()); +} + +static __wasi_errno_t +set_file_times(HANDLE handle, __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags) +{ + FILETIME atim = { 0, 0 }; + FILETIME mtim = { 0, 0 }; + + if ((fstflags & __WASI_FILESTAT_SET_ATIM) != 0) { + atim = convert_wasi_timestamp_to_filetime(access_time); + } + else if ((fstflags & __WASI_FILESTAT_SET_ATIM_NOW) != 0) { + GetSystemTimePreciseAsFileTime(&atim); + } + + if ((fstflags & __WASI_FILESTAT_SET_MTIM) != 0) { + mtim = convert_wasi_timestamp_to_filetime(modification_time); + } + else if ((fstflags & __WASI_FILESTAT_SET_MTIM_NOW) != 0) { + GetSystemTimePreciseAsFileTime(&mtim); + } + + bool success = SetFileTime(handle, NULL, &atim, &mtim); + + return success ? __WASI_ESUCCESS + : convert_windows_error_code(GetLastError()); +} + +__wasi_errno_t +os_futimens(os_file_handle handle, __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags) +{ + CHECK_VALID_FILE_HANDLE(handle); + + return set_file_times(handle->raw.handle, access_time, modification_time, + fstflags); +} + +__wasi_errno_t +os_utimensat(os_file_handle handle, const char *path, + __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags, + __wasi_lookupflags_t lookup_flags) +{ + CHECK_VALID_FILE_HANDLE(handle); + + wchar_t absolute_path[PATH_MAX]; + __wasi_errno_t error = get_absolute_filepath(handle->raw.handle, path, + absolute_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + HANDLE resolved_handle = create_handle( + absolute_path, is_directory(absolute_path), + (lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) != 0, false); + + if (resolved_handle == INVALID_HANDLE_VALUE) + return convert_windows_error_code(GetLastError()); + + error = set_file_times(resolved_handle, access_time, modification_time, + fstflags); + + CloseHandle(resolved_handle); + + return error; +} + +__wasi_errno_t +os_readlinkat(os_file_handle handle, const char *path, char *buf, + size_t bufsize, size_t *nread) +{ + CHECK_VALID_FILE_HANDLE(handle); + + wchar_t symlink_path[PATH_MAX]; + __wasi_errno_t error = + get_absolute_filepath(handle->raw.handle, path, symlink_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + DWORD symlink_attributes = GetFileAttributesW(symlink_path); + + if (!has_symlink_attribute(symlink_attributes)) + return __WASI_EINVAL; + + HANDLE link_handle = create_handle( + symlink_path, has_directory_attribute(symlink_attributes), false, true); + + if (link_handle == INVALID_HANDLE_VALUE) + return convert_windows_error_code(GetLastError()); + +#if WINAPI_PARTITION_DESKTOP != 0 +// MinGW32 already has a definition for REPARSE_DATA_BUFFER +#if defined(_MSC_VER) || defined(__MINGW64_VERSION_MAJOR) + // See + // https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_reparse_data_buffer + // for more details. + typedef struct _REPARSE_DATA_BUFFER { + ULONG ReparseTag; + USHORT ReparseDataLength; + USHORT Reserved; + union { + struct { + USHORT SubstituteNameOffset; + USHORT SubstituteNameLength; + USHORT PrintNameOffset; + USHORT PrintNameLength; + ULONG Flags; + WCHAR PathBuffer[1]; + } SymbolicLinkReparseBuffer; + struct { + USHORT SubstituteNameOffset; + USHORT SubstituteNameLength; + USHORT PrintNameOffset; + USHORT PrintNameLength; + WCHAR PathBuffer[1]; + } MountPointReparseBuffer; + struct { + UCHAR DataBuffer[1]; + } GenericReparseBuffer; + } DUMMYUNIONNAME; + } REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; +#endif + + char buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; + + REPARSE_DATA_BUFFER *reparse_data = (REPARSE_DATA_BUFFER *)buffer; + + if (!DeviceIoControl(link_handle, FSCTL_GET_REPARSE_POINT, NULL, 0, &buffer, + sizeof(buffer), NULL, NULL)) { + error = convert_windows_error_code(GetLastError()); + goto fail; + } + + int wbufsize = 0; + wchar_t *wbuf = NULL; + + // The following checks are taken from the libuv windows filesystem + // implementation, + // https://github.com/libuv/libuv/blob/v1.x/src/win/fs.c#L181-L244. Real + // symlinks can contain pretty much anything, but the only thing we really + // care about is undoing the implicit conversion to an NT namespaced path + // that CreateSymbolicLink will perform on absolute paths. + if (reparse_data->ReparseTag == IO_REPARSE_TAG_SYMLINK) { + wbuf = reparse_data->SymbolicLinkReparseBuffer.PathBuffer + + (reparse_data->SymbolicLinkReparseBuffer.SubstituteNameOffset + / sizeof(wchar_t)); + wbufsize = reparse_data->SymbolicLinkReparseBuffer.SubstituteNameLength + / sizeof(wchar_t); + + if (wbufsize >= 4 && wbuf[0] == L'\\' && wbuf[1] == L'?' + && wbuf[2] == L'?' && wbuf[3] == L'\\') { + // Starts with \??\ + if (wbufsize >= 6 + && ((wbuf[4] >= L'A' && wbuf[4] <= L'Z') + || (wbuf[4] >= L'a' && wbuf[4] <= L'z')) + && wbuf[5] == L':' && (wbufsize == 6 || wbuf[6] == L'\\')) + { + // \??\:\ + wbuf += 4; + wbufsize -= 4; + } + else if (wbufsize >= 8 && (wbuf[4] == L'U' || wbuf[4] == L'u') + && (wbuf[5] == L'N' || wbuf[5] == L'n') + && (wbuf[6] == L'C' || wbuf[6] == L'c') + && wbuf[7] == L'\\') + { + // \??\UNC\\\ - make sure the final path looks like \\\\ + wbuf += 6; + wbuf[0] = L'\\'; + wbufsize -= 6; + } + } + } + else if (reparse_data->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) { + // Junction + wbuf = reparse_data->MountPointReparseBuffer.PathBuffer + + (reparse_data->MountPointReparseBuffer.SubstituteNameOffset + / sizeof(wchar_t)); + wbufsize = reparse_data->MountPointReparseBuffer.SubstituteNameLength + / sizeof(wchar_t); + + // Only treat junctions that look like \??\:\ as a symlink. + if (!(wbufsize >= 6 && wbuf[0] == L'\\' && wbuf[1] == L'?' + && wbuf[2] == L'?' && wbuf[3] == L'\\' + && ((wbuf[4] >= L'A' && wbuf[4] <= L'Z') + || (wbuf[4] >= L'a' && wbuf[4] <= L'z')) + && wbuf[5] == L':' && (wbufsize == 6 || wbuf[6] == L'\\'))) { + error = __WASI_EINVAL; + goto fail; + } + + /* Remove leading \??\ */ + wbuf += 4; + wbufsize -= 4; + } + else { + error = __WASI_EINVAL; + goto fail; + } + + if (wbuf != NULL) + *nread = (size_t)WideCharToMultiByte(CP_UTF8, 0, wbuf, wbufsize, buf, + (int)bufsize, NULL, NULL); + + if (*nread == 0 && wbuf != NULL) { + DWORD win_error = GetLastError(); + if (win_error == ERROR_INSUFFICIENT_BUFFER) + *nread = bufsize; + else + error = convert_windows_error_code(win_error); + } +#else + error = __WASI_ENOTSUP; +#endif /* end of WINAPI_PARTITION_DESKTOP == 0 */ +fail: + CloseHandle(link_handle); + return error; +} + +__wasi_errno_t +os_linkat(os_file_handle from_handle, const char *from_path, + os_file_handle to_handle, const char *to_path, + __wasi_lookupflags_t lookup_flags) +{ +#if WINAPI_PARTITION_DESKTOP == 0 + return __WASI_ENOSYS; +#else + CHECK_VALID_FILE_HANDLE(from_handle); + CHECK_VALID_FILE_HANDLE(to_handle); + + wchar_t absolute_from_path[PATH_MAX]; + __wasi_errno_t error = get_absolute_filepath( + from_handle->raw.handle, from_path, absolute_from_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + wchar_t absolute_to_path[PATH_MAX]; + error = get_absolute_filepath(to_handle->raw.handle, to_path, + absolute_to_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + size_t to_path_len = strlen(to_path); + + // Windows doesn't throw an error in the case that the new path has a + // trailing slash but the target to link to is a file. + if (to_path[to_path_len - 1] == '/' + || to_path[to_path_len - 1] == '\\' + && !is_directory(absolute_from_path)) { + return __WASI_ENOENT; + } + + int ret = CreateHardLinkW(absolute_to_path, absolute_from_path, NULL); + + if (ret == 0) + error = convert_windows_error_code(GetLastError()); + + return error; +#endif /* end of WINAPI_PARTITION_DESKTOP == 0 */ +} + +__wasi_errno_t +os_symlinkat(const char *old_path, os_file_handle handle, const char *new_path) +{ +#if WINAPI_PARTITION_DESKTOP == 0 + return __WASI_ENOSYS; +#else + CHECK_VALID_FILE_HANDLE(handle); + + wchar_t absolute_new_path[PATH_MAX]; + __wasi_errno_t error = get_absolute_filepath(handle->raw.handle, new_path, + absolute_new_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + DWORD target_type = SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; + + wchar_t old_wpath[PATH_MAX]; + size_t old_path_len = 0; + + error = convert_to_wchar(old_path, old_wpath, PATH_MAX); + + if (error != __WASI_ESUCCESS) + goto fail; + + wchar_t absolute_old_path[PATH_MAX]; + error = get_absolute_filepath(handle->raw.handle, old_path, + absolute_old_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + goto fail; + + if (is_directory(absolute_old_path)) + target_type |= SYMBOLIC_LINK_FLAG_DIRECTORY; + + bool success = + CreateSymbolicLinkW(absolute_new_path, old_wpath, target_type); + + if (!success) { + DWORD win_error = GetLastError(); + + // Return a more useful error code if a file/directory already exists at + // the symlink location. + if (win_error == ERROR_ACCESS_DENIED || win_error == ERROR_INVALID_NAME) + error = __WASI_ENOENT; + else + error = convert_windows_error_code(GetLastError()); + } +fail: + return error; +#endif /* end of WINAPI_PARTITION_DESKTOP == 0 */ +} + +__wasi_errno_t +os_mkdirat(os_file_handle handle, const char *path) +{ + CHECK_VALID_FILE_HANDLE(handle); + + wchar_t absolute_path[PATH_MAX]; + __wasi_errno_t error = get_absolute_filepath(handle->raw.handle, path, + absolute_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + bool success = CreateDirectoryW(absolute_path, NULL); + + if (!success) + error = convert_windows_error_code(GetLastError()); + + return error; +} + +__wasi_errno_t +os_renameat(os_file_handle old_handle, const char *old_path, + os_file_handle new_handle, const char *new_path) +{ + CHECK_VALID_FILE_HANDLE(old_handle); + CHECK_VALID_FILE_HANDLE(new_handle); + + wchar_t old_absolute_path[PATH_MAX]; + __wasi_errno_t error = get_absolute_filepath( + old_handle->raw.handle, old_path, old_absolute_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + wchar_t new_absolute_path[PATH_MAX]; + error = get_absolute_filepath(new_handle->raw.handle, new_path, + new_absolute_path, PATH_MAX); + + if (error != __WASI_ESUCCESS) + return error; + + int ret = MoveFileExW(old_absolute_path, new_absolute_path, + MOVEFILE_REPLACE_EXISTING); + if (ret == 0) + error = convert_windows_error_code(GetLastError()); + + return error; +} + +__wasi_errno_t +os_isatty(os_file_handle handle) +{ + CHECK_VALID_HANDLE(handle); + + DWORD console_mode; + return GetConsoleMode(handle->raw.handle, &console_mode) ? __WASI_ESUCCESS + : __WASI_ENOTTY; +} + +static os_file_handle +create_stdio_handle(HANDLE raw_stdio_handle, DWORD stdio) +{ + os_file_handle stdio_handle = BH_MALLOC(sizeof(windows_handle)); + + if (stdio_handle == NULL) + return NULL; + + stdio_handle->type = windows_handle_type_file; + stdio_handle->access_mode = + windows_access_mode_read | windows_access_mode_write; + stdio_handle->fdflags = 0; + + if (raw_stdio_handle == INVALID_HANDLE_VALUE) + raw_stdio_handle = GetStdHandle(stdio); + + stdio_handle->raw.handle = raw_stdio_handle; + + return stdio_handle; +} + +os_file_handle +os_convert_stdin_handle(os_raw_file_handle raw_stdin) +{ + return create_stdio_handle(raw_stdin, STD_INPUT_HANDLE); +} + +os_file_handle +os_convert_stdout_handle(os_raw_file_handle raw_stdout) +{ + return create_stdio_handle(raw_stdout, STD_OUTPUT_HANDLE); +} + +os_file_handle +os_convert_stderr_handle(os_raw_file_handle raw_stderr) +{ + return create_stdio_handle(raw_stderr, STD_ERROR_HANDLE); +} + +__wasi_errno_t +os_unlinkat(os_file_handle handle, const char *path, bool is_dir) +{ + CHECK_VALID_FILE_HANDLE(handle); + + wchar_t absolute_path[PATH_MAX]; + __wasi_errno_t error = get_absolute_filepath(handle->raw.handle, path, + absolute_path, PATH_MAX); + + DWORD attributes = GetFileAttributesW(absolute_path); + + if (attributes != INVALID_FILE_ATTRIBUTES + && (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { + // Override is_dir for symlinks. A symlink to a directory counts as a + // directory itself in Windows. + is_dir = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + } + + if (error != __WASI_ESUCCESS) + return error; + + int ret = + is_dir ? RemoveDirectoryW(absolute_path) : DeleteFileW(absolute_path); + + if (ret == 0) + error = convert_windows_error_code(GetLastError()); + + return error; +} + +__wasi_errno_t +os_lseek(os_file_handle handle, __wasi_filedelta_t offset, + __wasi_whence_t whence, __wasi_filesize_t *new_offset) +{ + CHECK_VALID_FILE_HANDLE(handle); + DWORD sys_whence = 0; + + switch (whence) { + case __WASI_WHENCE_SET: + sys_whence = FILE_BEGIN; + break; + case __WASI_WHENCE_END: + sys_whence = FILE_END; + break; + case __WASI_WHENCE_CUR: + sys_whence = FILE_CURRENT; + break; + default: + return __WASI_EINVAL; + } + + LARGE_INTEGER distance_to_move = { .QuadPart = offset }; + LARGE_INTEGER updated_offset = { .QuadPart = 0 }; + + int ret = SetFilePointerEx(handle->raw.handle, distance_to_move, + &updated_offset, sys_whence); + + if (ret == 0) + return convert_windows_error_code(GetLastError()); + + *new_offset = (__wasi_filesize_t)updated_offset.QuadPart; + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_fadvise(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length, __wasi_advice_t advice) +{ + CHECK_VALID_FILE_HANDLE(handle); + // Advisory information can be safely ignored if not supported + switch (advice) { + case __WASI_ADVICE_DONTNEED: + case __WASI_ADVICE_NOREUSE: + case __WASI_ADVICE_NORMAL: + case __WASI_ADVICE_RANDOM: + case __WASI_ADVICE_SEQUENTIAL: + case __WASI_ADVICE_WILLNEED: + return __WASI_ESUCCESS; + default: + return __WASI_EINVAL; + } +} + +__wasi_errno_t +os_fdopendir(os_file_handle handle, os_dir_stream *dir_stream) +{ + CHECK_VALID_FILE_HANDLE(handle); + + // Check the handle is a directory handle first + DWORD windows_filetype = GetFileType(handle->raw.handle); + + __wasi_filetype_t filetype = __WASI_FILETYPE_UNKNOWN; + __wasi_errno_t error = + convert_windows_filetype(handle, windows_filetype, &filetype); + + if (error != __WASI_ESUCCESS) + return error; + + if (filetype != __WASI_FILETYPE_DIRECTORY) + return __WASI_ENOTDIR; + + *dir_stream = BH_MALLOC(sizeof(windows_dir_stream)); + + if (*dir_stream == NULL) + return __WASI_ENOMEM; + + init_dir_stream(*dir_stream, handle); + + return error; +} + +__wasi_errno_t +os_rewinddir(os_dir_stream dir_stream) +{ + CHECK_VALID_WIN_DIR_STREAM(dir_stream); + + reset_dir_stream(dir_stream); + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_seekdir(os_dir_stream dir_stream, __wasi_dircookie_t position) +{ + CHECK_VALID_WIN_DIR_STREAM(dir_stream); + + if (dir_stream->cookie == position) + return __WASI_ESUCCESS; + + if (dir_stream->cookie > position) { + reset_dir_stream(dir_stream); + } + + while (dir_stream->cookie < position) { + __wasi_errno_t error = read_next_dir_entry(dir_stream, NULL); + + if (error != __WASI_ESUCCESS) + return error; + + // We've reached the end of the directory. + if (dir_stream->cookie == 0) { + break; + } + } + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_readdir(os_dir_stream dir_stream, __wasi_dirent_t *entry, + const char **d_name) +{ + CHECK_VALID_WIN_DIR_STREAM(dir_stream); + + FILE_ID_BOTH_DIR_INFO *file_id_both_dir_info = NULL; + + __wasi_errno_t error = + read_next_dir_entry(dir_stream, &file_id_both_dir_info); + + if (error != __WASI_ESUCCESS || file_id_both_dir_info == NULL) + return error; + + entry->d_ino = (__wasi_inode_t)file_id_both_dir_info->FileId.QuadPart; + entry->d_namlen = (__wasi_dirnamlen_t)(file_id_both_dir_info->FileNameLength + / (sizeof(wchar_t) / sizeof(char))); + entry->d_next = (__wasi_dircookie_t)dir_stream->cookie; + entry->d_type = get_disk_filetype(file_id_both_dir_info->FileAttributes); + + *d_name = dir_stream->current_entry_name; + + return __WASI_ESUCCESS; +} + +__wasi_errno_t +os_closedir(os_dir_stream dir_stream) +{ + CHECK_VALID_WIN_DIR_STREAM(dir_stream); + + bool success = CloseHandle(dir_stream->handle->raw.handle); + + if (!success) { + DWORD win_error = GetLastError(); + + if (win_error = ERROR_INVALID_HANDLE) + BH_FREE(dir_stream); + return convert_windows_error_code(win_error); + } + + BH_FREE(dir_stream); + + return __WASI_ESUCCESS; +} + +os_dir_stream +os_get_invalid_dir_stream() +{ + return NULL; +} + +bool +os_is_dir_stream_valid(os_dir_stream *dir_stream) +{ + assert(dir_stream != NULL); + + if (((*dir_stream) == NULL) || ((*dir_stream)->handle == NULL) + || ((*dir_stream)->handle->type != windows_handle_type_file) + || ((*dir_stream)->handle->raw.handle == INVALID_HANDLE_VALUE)) + return false; + + return true; +} + +bool +os_is_handle_valid(os_file_handle *handle) +{ + assert(handle != NULL); + + CHECK_VALID_HANDLE_WITH_RETURN_VALUE(*handle, false); + + return true; +} + +char * +os_realpath(const char *path, char *resolved_path) +{ + resolved_path = _fullpath(resolved_path, path, PATH_MAX); + + // Check the file/directory actually exists + DWORD attributes = GetFileAttributesA(resolved_path); + + if (attributes == INVALID_FILE_ATTRIBUTES) + return NULL; + + return resolved_path; +} diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_malloc.c b/wasm-micro-runtime/core/shared/platform/windows/win_malloc.c new file mode 100644 index 0000000..56aaf9c --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_malloc.c @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +void * +os_malloc(unsigned size) +{ + return malloc(size); +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return realloc(ptr, size); +} + +void +os_free(void *ptr) +{ + free(ptr); +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_memmap.c b/wasm-micro-runtime/core/shared/platform/windows/win_memmap.c new file mode 100644 index 0000000..db0f38a --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_memmap.c @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +#define TRACE_MEMMAP 0 + +static DWORD +access_to_win32_flags(int prot) +{ + DWORD protect = PAGE_NOACCESS; + + if (prot & MMAP_PROT_EXEC) { + if (prot & MMAP_PROT_WRITE) + protect = PAGE_EXECUTE_READWRITE; + else + protect = PAGE_EXECUTE_READ; + } + else if (prot & MMAP_PROT_WRITE) { + protect = PAGE_READWRITE; + } + else if (prot & MMAP_PROT_READ) { + protect = PAGE_READONLY; + } + + return protect; +} + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + DWORD alloc_type = MEM_RESERVE; + DWORD protect; + size_t request_size, page_size; + void *addr; + + page_size = os_getpagesize(); + request_size = (size + page_size - 1) & ~(page_size - 1); + + if (request_size < size) + /* integer overflow */ + return NULL; + +#if WASM_ENABLE_JIT != 0 + /** + * Allocate memory at the highest possible address if the + * request size is large, or LLVM JIT might report error: + * IMAGE_REL_AMD64_ADDR32NB relocation requires an ordered + * section layout. + */ + if (request_size > 10 * BH_MB) + alloc_type |= MEM_TOP_DOWN; +#endif + + protect = access_to_win32_flags(prot); + if (protect != PAGE_NOACCESS) { + alloc_type |= MEM_COMMIT; + } + + addr = VirtualAlloc((LPVOID)hint, request_size, alloc_type, protect); + +#if TRACE_MEMMAP != 0 + printf("Map memory, request_size: %zu, alloc_type: 0x%x, " + "protect: 0x%x, ret: %p\n", + request_size, alloc_type, protect, addr); +#endif + return addr; +} + +void +os_munmap(void *addr, size_t size) +{ + size_t page_size = os_getpagesize(); + size_t request_size = (size + page_size - 1) & ~(page_size - 1); + + if (addr) { + if (!VirtualFree(addr, request_size, MEM_DECOMMIT)) { + printf("warning: os_munmap decommit pages failed, " + "addr: %p, request_size: %zu, errno: %d\n", + addr, request_size, errno); + return; + } + + if (!VirtualFree(addr, 0, MEM_RELEASE)) { + printf("warning: os_munmap release pages failed, " + "addr: %p, size: %zu, errno:%d\n", + addr, request_size, errno); + } + } +#if TRACE_MEMMAP != 0 + printf("Unmap memory, addr: %p, request_size: %zu\n", addr, request_size); +#endif +} + +void * +os_mem_commit(void *addr, size_t size, int flags) +{ + DWORD protect = access_to_win32_flags(flags); + size_t page_size = os_getpagesize(); + size_t request_size = (size + page_size - 1) & ~(page_size - 1); + + if (!addr) + return NULL; + +#if TRACE_MEMMAP != 0 + printf("Commit memory, addr: %p, request_size: %zu, protect: 0x%x\n", addr, + request_size, protect); +#endif + return VirtualAlloc((LPVOID)addr, request_size, MEM_COMMIT, protect); +} + +void +os_mem_decommit(void *addr, size_t size) +{ + size_t page_size = os_getpagesize(); + size_t request_size = (size + page_size - 1) & ~(page_size - 1); + + if (!addr) + return; + +#if TRACE_MEMMAP != 0 + printf("Decommit memory, addr: %p, request_size: %zu\n", addr, + request_size); +#endif + VirtualFree((LPVOID)addr, request_size, MEM_DECOMMIT); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + DWORD protect; + size_t page_size = os_getpagesize(); + size_t request_size = (size + page_size - 1) & ~(page_size - 1); + + if (!addr) + return 0; + + protect = access_to_win32_flags(prot); +#if TRACE_MEMMAP != 0 + printf("Mprotect memory, addr: %p, request_size: %zu, protect: 0x%x\n", + addr, request_size, protect); +#endif + return VirtualProtect((LPVOID)addr, request_size, protect, NULL); +} diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_socket.c b/wasm-micro-runtime/core/shared/platform/windows/win_socket.c new file mode 100644 index 0000000..b19e5b0 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_socket.c @@ -0,0 +1,705 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" +#include "platform_wasi_types.h" +#include "win_util.h" + +/* link with Ws2_32.lib */ +#pragma comment(lib, "ws2_32.lib") + +static bool is_winsock_inited = false; + +#define CHECK_VALID_SOCKET_HANDLE(win_handle) \ + do { \ + if ((win_handle) == NULL) { \ + errno = EBADF; \ + return BHT_ERROR; \ + } \ + if ((win_handle)->type != windows_handle_type_socket) { \ + errno = ENOTSOCK; \ + return BHT_ERROR; \ + } \ + if ((win_handle)->raw.socket == INVALID_SOCKET) { \ + errno = EBADF; \ + return BHT_ERROR; \ + } \ + } while (0) + +int +init_winsock() +{ +#if WASM_ENABLE_HOST_SOCKET_INIT == 0 + WSADATA wsaData; + + if (!is_winsock_inited) { + if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) { + os_printf("winsock init failed"); + return BHT_ERROR; + } + + is_winsock_inited = true; + } +#endif + + return BHT_OK; +} + +void +deinit_winsock() +{ +#if WASM_ENABLE_HOST_SOCKET_INIT == 0 + if (is_winsock_inited) { + WSACleanup(); + } +#endif +} + +int +os_socket_create(bh_socket_t *sock, bool is_ipv4, bool is_tcp) +{ + int af; + + if (!sock) { + return BHT_ERROR; + } + + *(sock) = BH_MALLOC(sizeof(windows_handle)); + + if ((*sock) == NULL) { + errno = ENOMEM; + return BHT_ERROR; + } + + (*sock)->type = windows_handle_type_socket; + (*sock)->access_mode = windows_access_mode_read | windows_access_mode_write; + (*sock)->fdflags = 0; + + if (is_ipv4) { + af = AF_INET; + } + else { + errno = ENOSYS; + return BHT_ERROR; + } + + if (is_tcp) { + (*sock)->raw.socket = socket(af, SOCK_STREAM, IPPROTO_TCP); + } + else { + (*sock)->raw.socket = socket(af, SOCK_DGRAM, 0); + } + + if ((*sock)->raw.socket == INVALID_SOCKET) { + BH_FREE(*sock); + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_bind(bh_socket_t socket, const char *host, int *port) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + struct sockaddr_in addr; + int socklen, ret; + + assert(host); + assert(port); + + addr.sin_addr.s_addr = inet_addr(host); + addr.sin_port = htons(*port); + addr.sin_family = AF_INET; + + ret = bind(socket->raw.socket, (struct sockaddr *)&addr, sizeof(addr)); + if (ret < 0) { + goto fail; + } + + socklen = sizeof(addr); + if (getsockname(socket->raw.socket, (void *)&addr, &socklen) == -1) { + os_printf("getsockname failed with error %d\n", WSAGetLastError()); + goto fail; + } + + *port = ntohs(addr.sin_port); + + return BHT_OK; + +fail: + return BHT_ERROR; +} + +int +os_socket_settimeout(bh_socket_t socket, uint64 timeout_us) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + DWORD tv = (DWORD)(timeout_us / 1000UL); + + if (setsockopt(socket->raw.socket, SOL_SOCKET, SO_RCVTIMEO, + (const char *)&tv, sizeof(tv)) + != 0) { + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_listen(bh_socket_t socket, int max_client) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + if (listen(socket->raw.socket, max_client) != 0) { + os_printf("socket listen failed with error %d\n", WSAGetLastError()); + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, + unsigned int *addrlen) +{ + CHECK_VALID_SOCKET_HANDLE(server_sock); + + struct sockaddr addr_tmp; + unsigned int len = sizeof(struct sockaddr); + + *sock = BH_MALLOC(sizeof(windows_handle)); + + if (*sock == NULL) { + errno = ENOMEM; + return BHT_ERROR; + } + + (*sock)->type = windows_handle_type_socket; + (*sock)->access_mode = windows_access_mode_read | windows_access_mode_write; + (*sock)->fdflags = 0; + (*sock)->raw.socket = + accept(server_sock->raw.socket, (struct sockaddr *)&addr_tmp, &len); + + if ((*sock)->raw.socket == INVALID_SOCKET) { + BH_FREE(*sock); + os_printf("socket accept failed with error %d\n", WSAGetLastError()); + return BHT_ERROR; + } + + return BHT_OK; +} + +int +os_socket_recv(bh_socket_t socket, void *buf, unsigned int len) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + return recv(socket->raw.socket, buf, len, 0); +} + +int +os_socket_recv_from(bh_socket_t socket, void *buf, unsigned int len, int flags, + bh_sockaddr_t *src_addr) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_send(bh_socket_t socket, const void *buf, unsigned int len) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + return send(socket->raw.socket, buf, len, 0); +} + +int +os_socket_send_to(bh_socket_t socket, const void *buf, unsigned int len, + int flags, const bh_sockaddr_t *dest_addr) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_close(bh_socket_t socket) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + closesocket(socket->raw.socket); + + BH_FREE(socket); + + return BHT_OK; +} + +__wasi_errno_t +os_socket_shutdown(bh_socket_t socket) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + if (shutdown(socket->raw.socket, SD_BOTH) != 0) { + return convert_winsock_error_code(WSAGetLastError()); + } + return __WASI_ESUCCESS; +} + +int +os_socket_inet_network(bool is_ipv4, const char *cp, bh_ip_addr_buffer_t *out) +{ + if (!cp) + return BHT_ERROR; + + if (is_ipv4) { + if (inet_pton(AF_INET, cp, &out->ipv4) != 1) { + return BHT_ERROR; + } + /* Note: ntohl(INADDR_NONE) == INADDR_NONE */ + out->ipv4 = ntohl(out->ipv4); + } + else { + if (inet_pton(AF_INET6, cp, out->ipv6) != 1) { + return BHT_ERROR; + } + for (int i = 0; i < 8; i++) { + out->ipv6[i] = ntohs(out->ipv6[i]); + } + } + + return BHT_OK; +} + +int +os_socket_connect(bh_socket_t socket, const char *addr, int port) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_addr_resolve(const char *host, const char *service, + uint8_t *hint_is_tcp, uint8_t *hint_is_ipv4, + bh_addr_info_t *addr_info, size_t addr_info_size, + size_t *max_info_size) +{ + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_addr_local(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_send_timeout(bh_socket_t socket, uint64 timeout_us) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_send_timeout(bh_socket_t socket, uint64 *timeout_us) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_recv_timeout(bh_socket_t socket, uint64 timeout_us) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_recv_timeout(bh_socket_t socket, uint64 *timeout_us) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_addr_remote(bh_socket_t socket, bh_sockaddr_t *sockaddr) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_send_buf_size(bh_socket_t socket, size_t bufsiz) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_send_buf_size(bh_socket_t socket, size_t *bufsiz) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_recv_buf_size(bh_socket_t socket, size_t bufsiz) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_recv_buf_size(bh_socket_t socket, size_t *bufsiz) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_keep_alive(bh_socket_t socket, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_keep_alive(bh_socket_t socket, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_reuse_addr(bh_socket_t socket, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_reuse_addr(bh_socket_t socket, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_reuse_port(bh_socket_t socket, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_reuse_port(bh_socket_t socket, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_linger(bh_socket_t socket, bool is_enabled, int linger_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_linger(bh_socket_t socket, bool *is_enabled, int *linger_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_no_delay(bh_socket_t socket, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_tcp_no_delay(bh_socket_t socket, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_quick_ack(bh_socket_t socket, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_tcp_quick_ack(bh_socket_t socket, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_keep_idle(bh_socket_t socket, uint32 time_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_tcp_keep_idle(bh_socket_t socket, uint32 *time_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_keep_intvl(bh_socket_t socket, uint32 time_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_tcp_keep_intvl(bh_socket_t socket, uint32 *time_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_tcp_fastopen_connect(bh_socket_t socket, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_tcp_fastopen_connect(bh_socket_t socket, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_add_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_drop_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_ttl(bh_socket_t socket, uint8_t ttl_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_ip_ttl(bh_socket_t socket, uint8_t *ttl_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ip_multicast_ttl(bh_socket_t socket, uint8_t ttl_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_ip_multicast_ttl(bh_socket_t socket, uint8_t *ttl_s) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_ipv6_only(bh_socket_t socket, bool option) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_ipv6_only(bh_socket_t socket, bool *option) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_set_broadcast(bh_socket_t socket, bool is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + + return BHT_ERROR; +} + +int +os_socket_get_broadcast(bh_socket_t socket, bool *is_enabled) +{ + CHECK_VALID_SOCKET_HANDLE(socket); + + errno = ENOSYS; + return BHT_ERROR; +} diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_thread.c b/wasm-micro-runtime/core/shared/platform/windows/win_thread.c new file mode 100644 index 0000000..f37250f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_thread.c @@ -0,0 +1,810 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +#define bh_assert(v) assert(v) + +#define BH_SEM_COUNT_MAX 0xFFFF + +struct os_thread_data; + +typedef struct os_thread_wait_node { + korp_sem sem; + void *retval; + os_thread_wait_list next; +} os_thread_wait_node; + +typedef struct os_thread_data { + /* Next thread data */ + struct os_thread_data *next; + /* Thread data of parent thread */ + struct os_thread_data *parent; + /* Thread Id */ + DWORD thread_id; + /* Thread start routine */ + thread_start_routine_t start_routine; + /* Thread start routine argument */ + void *arg; + /* Wait node of current thread */ + os_thread_wait_node wait_node; + /* Wait cond */ + korp_cond wait_cond; + /* Wait lock */ + korp_mutex wait_lock; + /* Waiting list of other threads who are joining this thread */ + os_thread_wait_list thread_wait_list; + /* End node of the waiting list */ + os_thread_wait_node *thread_wait_list_end; + /* Whether the thread has exited */ + bool thread_exited; + /* Thread return value */ + void *thread_retval; +} os_thread_data; + +static bool is_thread_sys_inited = false; + +/* Thread data of supervisor thread */ +static os_thread_data supervisor_thread_data; + +/* Thread data list lock */ +static korp_mutex thread_data_list_lock; + +/* Thread data key */ +static DWORD thread_data_key; + +/* The GetCurrentThreadStackLimits API from "kernel32" */ +static void(WINAPI *GetCurrentThreadStackLimits_Kernel32)(PULONG_PTR, + PULONG_PTR) = NULL; + +int +os_sem_init(korp_sem *sem); +int +os_sem_destroy(korp_sem *sem); +int +os_sem_wait(korp_sem *sem); +int +os_sem_reltimed_wait(korp_sem *sem, uint64 useconds); +int +os_sem_signal(korp_sem *sem); + +int +os_thread_sys_init() +{ + HMODULE module; + + if (is_thread_sys_inited) + return BHT_OK; + + if ((thread_data_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) + return BHT_ERROR; + + /* Initialize supervisor thread data */ + memset(&supervisor_thread_data, 0, sizeof(os_thread_data)); + + supervisor_thread_data.thread_id = GetCurrentThreadId(); + + if (os_sem_init(&supervisor_thread_data.wait_node.sem) != BHT_OK) + goto fail1; + + if (os_mutex_init(&supervisor_thread_data.wait_lock) != BHT_OK) + goto fail2; + + if (os_cond_init(&supervisor_thread_data.wait_cond) != BHT_OK) + goto fail3; + + if (!TlsSetValue(thread_data_key, &supervisor_thread_data)) + goto fail4; + + if (os_mutex_init(&thread_data_list_lock) != BHT_OK) + goto fail5; + + if ((module = GetModuleHandle((LPCTSTR) "kernel32"))) { + *(void **)&GetCurrentThreadStackLimits_Kernel32 = + GetProcAddress(module, "GetCurrentThreadStackLimits"); + } + + is_thread_sys_inited = true; + return BHT_OK; + +fail5: + TlsSetValue(thread_data_key, NULL); +fail4: + os_cond_destroy(&supervisor_thread_data.wait_cond); +fail3: + os_mutex_destroy(&supervisor_thread_data.wait_lock); +fail2: + os_sem_destroy(&supervisor_thread_data.wait_node.sem); +fail1: + TlsFree(thread_data_key); + return BHT_ERROR; +} + +void +os_thread_sys_destroy() +{ + if (is_thread_sys_inited) { + os_thread_data *thread_data, *thread_data_next; + + thread_data = supervisor_thread_data.next; + while (thread_data) { + thread_data_next = thread_data->next; + + /* Destroy resources of thread data */ + os_cond_destroy(&thread_data->wait_cond); + os_sem_destroy(&thread_data->wait_node.sem); + os_mutex_destroy(&thread_data->wait_lock); + BH_FREE(thread_data); + + thread_data = thread_data_next; + } + + os_mutex_destroy(&thread_data_list_lock); + os_cond_destroy(&supervisor_thread_data.wait_cond); + os_mutex_destroy(&supervisor_thread_data.wait_lock); + os_sem_destroy(&supervisor_thread_data.wait_node.sem); + memset(&supervisor_thread_data, 0, sizeof(os_thread_data)); + TlsFree(thread_data_key); + thread_data_key = 0; + is_thread_sys_inited = false; + } +} + +static os_thread_data * +thread_data_current() +{ + return (os_thread_data *)TlsGetValue(thread_data_key); +} + +static void +os_thread_cleanup(void *retval) +{ + os_thread_data *thread_data = thread_data_current(); + + bh_assert(thread_data != NULL); + + os_mutex_lock(&thread_data->wait_lock); + if (thread_data->thread_wait_list) { + /* Signal each joining thread */ + os_thread_wait_list head = thread_data->thread_wait_list; + while (head) { + os_thread_wait_list next = head->next; + head->retval = retval; + os_sem_signal(&head->sem); + head = next; + } + thread_data->thread_wait_list = thread_data->thread_wait_list_end = + NULL; + } + /* Set thread status and thread return value */ + thread_data->thread_exited = true; + thread_data->thread_retval = retval; + os_mutex_unlock(&thread_data->wait_lock); +} + +static unsigned __stdcall os_thread_wrapper(void *arg) +{ + os_thread_data *thread_data = arg; + os_thread_data *parent = thread_data->parent; + void *retval; + bool result; + +#if 0 + os_printf("THREAD CREATED %p\n", thread_data); +#endif + + os_mutex_lock(&parent->wait_lock); + thread_data->thread_id = GetCurrentThreadId(); + result = TlsSetValue(thread_data_key, thread_data); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (result) + result = os_thread_signal_init() == 0 ? true : false; +#endif + /* Notify parent thread */ + os_cond_signal(&parent->wait_cond); + os_mutex_unlock(&parent->wait_lock); + + if (!result) + return -1; + + retval = thread_data->start_routine(thread_data->arg); + + os_thread_cleanup(retval); + return 0; +} + +int +os_thread_create_with_prio(korp_tid *p_tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + os_thread_data *parent = thread_data_current(); + os_thread_data *thread_data; + + if (!p_tid || !start) + return BHT_ERROR; + + if (stack_size < BH_APPLET_PRESERVED_STACK_SIZE) + stack_size = BH_APPLET_PRESERVED_STACK_SIZE; + + if (!(thread_data = BH_MALLOC(sizeof(os_thread_data)))) + return BHT_ERROR; + + memset(thread_data, 0, sizeof(os_thread_data)); + thread_data->parent = parent; + thread_data->start_routine = start; + thread_data->arg = arg; + + if (os_sem_init(&thread_data->wait_node.sem) != BHT_OK) + goto fail1; + + if (os_mutex_init(&thread_data->wait_lock) != BHT_OK) + goto fail2; + + if (os_cond_init(&thread_data->wait_cond) != BHT_OK) + goto fail3; + + os_mutex_lock(&parent->wait_lock); + if (!_beginthreadex(NULL, stack_size, os_thread_wrapper, thread_data, 0, + NULL)) { + os_mutex_unlock(&parent->wait_lock); + goto fail4; + } + + /* Add thread data into thread data list */ + os_mutex_lock(&thread_data_list_lock); + thread_data->next = supervisor_thread_data.next; + supervisor_thread_data.next = thread_data; + os_mutex_unlock(&thread_data_list_lock); + + /* Wait for the thread routine to set thread_data's tid + and add thread_data to thread data list */ + os_cond_wait(&parent->wait_cond, &parent->wait_lock); + os_mutex_unlock(&parent->wait_lock); + + *p_tid = (korp_tid)thread_data; + return BHT_OK; + +fail4: + os_cond_destroy(&thread_data->wait_cond); +fail3: + os_mutex_destroy(&thread_data->wait_lock); +fail2: + os_sem_destroy(&thread_data->wait_node.sem); +fail1: + BH_FREE(thread_data); + return BHT_ERROR; +} + +int +os_thread_create(korp_tid *tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +korp_tid +os_self_thread() +{ + return (korp_tid)TlsGetValue(thread_data_key); +} + +int +os_thread_join(korp_tid thread, void **p_retval) +{ + os_thread_data *thread_data, *curr_thread_data; + + /* Get thread data of current thread */ + curr_thread_data = thread_data_current(); + curr_thread_data->wait_node.next = NULL; + + /* Get thread data of thread to join */ + thread_data = (os_thread_data *)thread; + bh_assert(thread_data); + + os_mutex_lock(&thread_data->wait_lock); + + if (thread_data->thread_exited) { + /* Thread has exited */ + if (p_retval) + *p_retval = thread_data->thread_retval; + os_mutex_unlock(&thread_data->wait_lock); + return BHT_OK; + } + + /* Thread is running */ + if (!thread_data->thread_wait_list) { /* Waiting list is empty */ + thread_data->thread_wait_list = thread_data->thread_wait_list_end = + &curr_thread_data->wait_node; + } + else { /* Waiting list isn't empty */ + /* Add to end of waiting list */ + thread_data->thread_wait_list_end->next = &curr_thread_data->wait_node; + thread_data->thread_wait_list_end = &curr_thread_data->wait_node; + } + + os_mutex_unlock(&thread_data->wait_lock); + + /* Wait the sem */ + os_sem_wait(&curr_thread_data->wait_node.sem); + if (p_retval) + *p_retval = curr_thread_data->wait_node.retval; + return BHT_OK; +} + +int +os_thread_detach(korp_tid thread) +{ + /* Do nothing */ + return BHT_OK; + (void)thread; +} + +void +os_thread_exit(void *retval) +{ + os_thread_cleanup(retval); + _endthreadex(0); +} + +int +os_thread_env_init() +{ + os_thread_data *thread_data = TlsGetValue(thread_data_key); + + if (thread_data) + /* Already created */ + return BHT_OK; + + if (!(thread_data = BH_MALLOC(sizeof(os_thread_data)))) + return BHT_ERROR; + + memset(thread_data, 0, sizeof(os_thread_data)); + thread_data->thread_id = GetCurrentThreadId(); + + if (os_sem_init(&thread_data->wait_node.sem) != BHT_OK) + goto fail1; + + if (os_mutex_init(&thread_data->wait_lock) != BHT_OK) + goto fail2; + + if (os_cond_init(&thread_data->wait_cond) != BHT_OK) + goto fail3; + + if (!TlsSetValue(thread_data_key, thread_data)) + goto fail4; + + return BHT_OK; + +fail4: + os_cond_destroy(&thread_data->wait_cond); +fail3: + os_mutex_destroy(&thread_data->wait_lock); +fail2: + os_sem_destroy(&thread_data->wait_node.sem); +fail1: + BH_FREE(thread_data); + return BHT_ERROR; +} + +void +os_thread_env_destroy() +{ + os_thread_data *thread_data = TlsGetValue(thread_data_key); + + /* Note that supervisor_thread_data's resources will be destroyed + by os_thread_sys_destroy() */ + if (thread_data && thread_data != &supervisor_thread_data) { + TlsSetValue(thread_data_key, NULL); + os_cond_destroy(&thread_data->wait_cond); + os_mutex_destroy(&thread_data->wait_lock); + os_sem_destroy(&thread_data->wait_node.sem); + BH_FREE(thread_data); + } +} + +bool +os_thread_env_inited() +{ + os_thread_data *thread_data = TlsGetValue(thread_data_key); + return thread_data ? true : false; +} + +int +os_sem_init(korp_sem *sem) +{ + bh_assert(sem); + *sem = CreateSemaphore(NULL, 0, BH_SEM_COUNT_MAX, NULL); + return (*sem != NULL) ? BHT_OK : BHT_ERROR; +} + +int +os_sem_destroy(korp_sem *sem) +{ + bh_assert(sem); + CloseHandle(*sem); + return BHT_OK; +} + +int +os_sem_wait(korp_sem *sem) +{ + DWORD ret; + + bh_assert(sem); + + ret = WaitForSingleObject(*sem, INFINITE); + + if (ret == WAIT_OBJECT_0) + return BHT_OK; + else if (ret == WAIT_TIMEOUT) + return (int)WAIT_TIMEOUT; + else /* WAIT_FAILED or others */ + return BHT_ERROR; +} + +int +os_sem_reltimed_wait(korp_sem *sem, uint64 useconds) +{ + uint64 mseconds_64; + DWORD ret, mseconds; + + bh_assert(sem); + + if (useconds == BHT_WAIT_FOREVER) + mseconds = INFINITE; + else { + mseconds_64 = useconds / 1000; + + if (mseconds_64 < (uint64)(UINT32_MAX - 1)) { + mseconds = (uint32)mseconds_64; + } + else { + mseconds = UINT32_MAX - 1; + os_printf("Warning: os_sem_reltimed_wait exceeds limit, " + "set to max timeout instead\n"); + } + } + + ret = WaitForSingleObject(*sem, mseconds); + + if (ret == WAIT_OBJECT_0) + return BHT_OK; + else if (ret == WAIT_TIMEOUT) + return (int)WAIT_TIMEOUT; + else /* WAIT_FAILED or others */ + return BHT_ERROR; +} + +int +os_sem_signal(korp_sem *sem) +{ + bh_assert(sem); + return ReleaseSemaphore(*sem, 1, NULL) != FALSE ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_init(korp_mutex *mutex) +{ + bh_assert(mutex); + *mutex = CreateMutex(NULL, FALSE, NULL); + return (*mutex != NULL) ? BHT_OK : BHT_ERROR; +} + +int +os_recursive_mutex_init(korp_mutex *mutex) +{ + bh_assert(mutex); + *mutex = CreateMutex(NULL, FALSE, NULL); + return (*mutex != NULL) ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + assert(mutex); + return CloseHandle(*mutex) ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + int ret; + + assert(mutex); + + if (*mutex == NULL) { /* static initializer? */ + HANDLE p = CreateMutex(NULL, FALSE, NULL); + + if (!p) { + return BHT_ERROR; + } + + if (InterlockedCompareExchangePointer((PVOID *)mutex, (PVOID)p, NULL) + != NULL) { + /* lock has been created by other threads */ + CloseHandle(p); + } + } + + ret = WaitForSingleObject(*mutex, INFINITE); + return ret != WAIT_FAILED ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + bh_assert(mutex); + return ReleaseMutex(*mutex) ? BHT_OK : BHT_ERROR; +} + +int +os_rwlock_init(korp_rwlock *lock) +{ + bh_assert(lock); + + InitializeSRWLock(&(lock->lock)); + lock->exclusive = false; + + return BHT_OK; +} + +int +os_rwlock_rdlock(korp_rwlock *lock) +{ + bh_assert(lock); + + AcquireSRWLockShared(&(lock->lock)); + + return BHT_OK; +} + +int +os_rwlock_wrlock(korp_rwlock *lock) +{ + bh_assert(lock); + + AcquireSRWLockExclusive(&(lock->lock)); + lock->exclusive = true; + + return BHT_OK; +} + +int +os_rwlock_unlock(korp_rwlock *lock) +{ + bh_assert(lock); + + if (lock->exclusive) { + lock->exclusive = false; + ReleaseSRWLockExclusive(&(lock->lock)); + } + else { + ReleaseSRWLockShared(&(lock->lock)); + } + + return BHT_OK; +} + +int +os_rwlock_destroy(korp_rwlock *lock) +{ + (void)lock; + + return BHT_OK; +} + +int +os_cond_init(korp_cond *cond) +{ + bh_assert(cond); + if (os_mutex_init(&cond->wait_list_lock) != BHT_OK) + return BHT_ERROR; + + cond->thread_wait_list = cond->thread_wait_list_end = NULL; + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ + bh_assert(cond); + os_mutex_destroy(&cond->wait_list_lock); + return BHT_OK; +} + +static int +os_cond_wait_internal(korp_cond *cond, korp_mutex *mutex, bool timed, + uint64 useconds) +{ + os_thread_wait_node *node = &thread_data_current()->wait_node; + + node->next = NULL; + + bh_assert(cond); + bh_assert(mutex); + os_mutex_lock(&cond->wait_list_lock); + if (!cond->thread_wait_list) { /* Waiting list is empty */ + cond->thread_wait_list = cond->thread_wait_list_end = node; + } + else { /* Waiting list isn't empty */ + /* Add to end of wait list */ + cond->thread_wait_list_end->next = node; + cond->thread_wait_list_end = node; + } + os_mutex_unlock(&cond->wait_list_lock); + + /* Unlock mutex, wait sem and lock mutex again */ + os_mutex_unlock(mutex); + int wait_result; + if (timed) + wait_result = os_sem_reltimed_wait(&node->sem, useconds); + else + wait_result = os_sem_wait(&node->sem); + os_mutex_lock(mutex); + + /* Remove wait node from wait list */ + os_mutex_lock(&cond->wait_list_lock); + if (cond->thread_wait_list == node) { + cond->thread_wait_list = node->next; + + if (cond->thread_wait_list_end == node) { + bh_assert(node->next == NULL); + cond->thread_wait_list_end = NULL; + } + } + else { + /* Remove from the wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next != node) + p = p->next; + p->next = node->next; + + if (cond->thread_wait_list_end == node) { + cond->thread_wait_list_end = p; + } + } + os_mutex_unlock(&cond->wait_list_lock); + + return wait_result; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + return os_cond_wait_internal(cond, mutex, false, 0); +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + if (useconds == BHT_WAIT_FOREVER) { + return os_cond_wait_internal(cond, mutex, false, 0); + } + else { + return os_cond_wait_internal(cond, mutex, true, useconds); + } +} + +int +os_cond_signal(korp_cond *cond) +{ + /* Signal the head wait node of wait list */ + os_mutex_lock(&cond->wait_list_lock); + if (cond->thread_wait_list) + os_sem_signal(&cond->thread_wait_list->sem); + os_mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +int +os_cond_broadcast(korp_cond *cond) +{ + /* Signal all of the wait node of wait list */ + os_mutex_lock(&cond->wait_list_lock); + if (cond->thread_wait_list) { + os_thread_wait_node *p = cond->thread_wait_list; + while (p) { + os_sem_signal(&p->sem); + p = p->next; + } + } + + os_mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +static os_thread_local_attribute uint8 *thread_stack_boundary = NULL; + +static ULONG +GetCurrentThreadStackLimits_Win7(PULONG_PTR p_low_limit, + PULONG_PTR p_high_limit) +{ + MEMORY_BASIC_INFORMATION mbi; + NT_TIB *tib = (NT_TIB *)NtCurrentTeb(); + + if (!tib) { + os_printf("warning: NtCurrentTeb() failed\n"); + return -1; + } + + *p_high_limit = (ULONG_PTR)tib->StackBase; + + if (VirtualQuery(tib->StackLimit, &mbi, sizeof(mbi))) { + *p_low_limit = (ULONG_PTR)mbi.AllocationBase; + return 0; + } + + os_printf("warning: VirtualQuery() failed\n"); + return GetLastError(); +} + +uint8 * +os_thread_get_stack_boundary() +{ + ULONG_PTR low_limit = 0, high_limit = 0; + uint32 page_size; + + if (thread_stack_boundary) + return thread_stack_boundary; + + page_size = os_getpagesize(); + if (GetCurrentThreadStackLimits_Kernel32) { + GetCurrentThreadStackLimits_Kernel32(&low_limit, &high_limit); + } + else { + if (0 != GetCurrentThreadStackLimits_Win7(&low_limit, &high_limit)) + return NULL; + } + + /* 4 pages are set unaccessible by system, we reserved + one more page at least for safety */ + thread_stack_boundary = (uint8 *)(uintptr_t)low_limit + page_size * 5; + return thread_stack_boundary; +} + +void +os_thread_jit_write_protect_np(bool enabled) +{} + +#ifdef OS_ENABLE_HW_BOUND_CHECK +static os_thread_local_attribute bool thread_signal_inited = false; + +int +os_thread_signal_init() +{ +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + ULONG StackSizeInBytes = 16 * 1024; +#endif + bool ret; + + if (thread_signal_inited) + return 0; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + ret = SetThreadStackGuarantee(&StackSizeInBytes); +#else + ret = true; +#endif + if (ret) + thread_signal_inited = true; + return ret ? 0 : -1; +} + +void +os_thread_signal_destroy() +{ + /* Do nothing */ +} + +bool +os_thread_signal_inited() +{ + return thread_signal_inited; +} +#endif diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_time.c b/wasm-micro-runtime/core/shared/platform/windows/win_time.c new file mode 100644 index 0000000..7b2cd4f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_time.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +uint64 +os_time_get_boot_us() +{ + struct timespec ts; +#if defined(__MINGW32__) + // https://www.mail-archive.com/mingw-w64-public@lists.sourceforge.net/msg18361.html + clock_gettime(CLOCK_REALTIME, &ts); +#else + timespec_get(&ts, TIME_UTC); +#endif + + return ((uint64)ts.tv_sec) * 1000 * 1000 + ((uint64)ts.tv_nsec) / 1000; +} + +uint64 +os_time_thread_cputime_us(void) +{ + /* FIXME if u know the right api */ + return os_time_get_boot_us(); +} \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_util.c b/wasm-micro-runtime/core/shared/platform/windows/win_util.c new file mode 100644 index 0000000..58987fa --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_util.c @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_common.h" +#include "win_util.h" + +// From 1601-01-01 to 1970-01-01 there are 134774 days. +static const uint64_t NT_to_UNIX_epoch_in_ns = + 134774ull * 86400ull * 1000ull * 1000ull * 1000ull; + +__wasi_timestamp_t +convert_filetime_to_wasi_timestamp(LPFILETIME filetime) +{ + ULARGE_INTEGER temp = { .HighPart = filetime->dwHighDateTime, + .LowPart = filetime->dwLowDateTime }; + + // WASI timestamps are measured in nanoseconds whereas FILETIME structs are + // represented in terms 100-nanosecond intervals. + return (temp.QuadPart * 100ull) - NT_to_UNIX_epoch_in_ns; +} + +FILETIME +convert_wasi_timestamp_to_filetime(__wasi_timestamp_t timestamp) +{ + ULARGE_INTEGER temp = { .QuadPart = + (timestamp + NT_to_UNIX_epoch_in_ns) / 100ull }; + + FILETIME ret = { .dwLowDateTime = temp.LowPart, + .dwHighDateTime = temp.HighPart }; + + return ret; +} + +__wasi_errno_t +convert_windows_error_code(DWORD windows_error_code) +{ + switch (windows_error_code) { + case ERROR_INVALID_PARAMETER: + case ERROR_INVALID_HANDLE: + case ERROR_NEGATIVE_SEEK: + return __WASI_EINVAL; + case ERROR_SHARING_VIOLATION: + case ERROR_PIPE_BUSY: + return __WASI_EBUSY; + case ERROR_ACCESS_DENIED: + return __WASI_EACCES; + case ERROR_ALREADY_EXISTS: + case ERROR_FILE_EXISTS: + return __WASI_EEXIST; + case ERROR_NO_MORE_FILES: + case ERROR_FILE_NOT_FOUND: + case ERROR_INVALID_NAME: + return __WASI_ENOENT; + case ERROR_PRIVILEGE_NOT_HELD: + return __WASI_EPERM; + case ERROR_NOT_ENOUGH_MEMORY: + return __WASI_ENOMEM; + case ERROR_NOACCESS: + return __WASI_EFAULT; + case ERROR_DIR_NOT_EMPTY: + return __WASI_ENOTEMPTY; + case ERROR_DIRECTORY: + return __WASI_ENOTDIR; + case ERROR_IO_PENDING: + case ERROR_INSUFFICIENT_BUFFER: + case ERROR_INVALID_FLAGS: + case ERROR_NO_UNICODE_TRANSLATION: + default: + return __WASI_EINVAL; + } +} + +#ifdef UWP_DEFAULT_VPRINTF +int +uwp_print_to_debugger(const char *format, va_list ap) +{ + // Provide a stack buffer which should be large enough for any realistic + // string so we avoid making an allocation on every printf call. + char stack_buf[2048]; + char *buf = stack_buf; + int ret = vsnprintf(stack_buf, sizeof(stack_buf), format, ap); + + if ((size_t)ret >= sizeof(stack_buf)) { + // Allocate an extra byte for the null terminator. + char *heap_buf = BH_MALLOC((unsigned int)(ret) + 1); + buf = heap_buf; + + if (heap_buf == NULL) { + // Output as much as we can to the debugger if allocating a buffer + // fails. + OutputDebugStringA(stack_buf); + return ret; + } + + ret = vsnprintf(heap_buf, (size_t)ret + 1, format, ap); + } + + if (ret >= 0) + OutputDebugStringA(buf); + + if (buf != stack_buf) + BH_FREE(buf); + + return ret; +} +#endif + +__wasi_errno_t +convert_winsock_error_code(int error_code) +{ + switch (error_code) { + case WSASYSNOTREADY: + case WSAEWOULDBLOCK: + return __WASI_EAGAIN; + case WSAVERNOTSUPPORTED: + return __WASI_ENOTSUP; + case WSAEINPROGRESS: + return __WASI_EINPROGRESS; + case WSAEPROCLIM: + return __WASI_EBUSY; + case WSAEFAULT: + return __WASI_EFAULT; + case WSAENETDOWN: + return __WASI_ENETDOWN; + case WSAENOTSOCK: + return __WASI_ENOTSOCK; + case WSAEINTR: + return __WASI_EINTR; + case WSAEAFNOSUPPORT: + return __WASI_EAFNOSUPPORT; + case WSAEMFILE: + return __WASI_ENFILE; + case WSAEINVAL: + return __WASI_EINVAL; + case WSAENOBUFS: + return __WASI_ENOBUFS; + case WSAEPROTONOSUPPORT: + return __WASI_EPROTONOSUPPORT; + case WSAEPROTOTYPE: + return __WASI_EPROTOTYPE; + case WSAESOCKTNOSUPPORT: + return __WASI_ENOTSUP; + case WSAECONNABORTED: + return __WASI_ECONNABORTED; + case WSAECONNRESET: + return __WASI_ECONNRESET; + case WSAENOTCONN: + return __WASI_ENOTCONN; + case WSAEINVALIDPROCTABLE: + case WSAEINVALIDPROVIDER: + case WSAEPROVIDERFAILEDINIT: + case WSANOTINITIALISED: + default: + return __WASI_EINVAL; + } +} diff --git a/wasm-micro-runtime/core/shared/platform/windows/win_util.h b/wasm-micro-runtime/core/shared/platform/windows/win_util.h new file mode 100644 index 0000000..033c393 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/windows/win_util.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WIN_UTIL_H +#define _WIN_UTIL_H + +#include "platform_wasi_types.h" +#include "windows.h" + +__wasi_timestamp_t +convert_filetime_to_wasi_timestamp(LPFILETIME filetime); + +FILETIME +convert_wasi_timestamp_to_filetime(__wasi_timestamp_t timestamp); + +/* Convert a Windows error code to a WASI error code */ +__wasi_errno_t +convert_windows_error_code(DWORD windows_error_code); + +/* Convert a Winsock error code to a WASI error code */ +__wasi_errno_t +convert_winsock_error_code(int error_code); + +#endif /* end of _WIN_UTIL_H */ \ No newline at end of file diff --git a/wasm-micro-runtime/core/shared/platform/zephyr/platform_internal.h b/wasm-micro-runtime/core/shared/platform/zephyr/platform_internal.h new file mode 100644 index 0000000..3c0f552 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/zephyr/platform_internal.h @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#include +#if KERNEL_VERSION_NUMBER >= 0x020200 /* version 2.2.0 */ +#include +#else +#include +#endif +#else /* else of KERNEL_VERSION_NUMBER < 0x030200 */ +#include +#include +#endif /* end of KERNEL_VERSION_NUMBER < 0x030200 */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef CONFIG_NET_BUF_USER_DATA_SIZE +#define CONFIG_NET_BUF_USER_DATA_SIZE 0 +#endif + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#include +#include +#include +#include +#else /* else of KERNEL_VERSION_NUMBER < 0x030200 */ +#include +#include +#include +#include +#include +#endif /* end of KERNEL_VERSION_NUMBER < 0x030200 */ + +#if KERNEL_VERSION_NUMBER >= 0x030300 /* version 3.3.0 */ +#include +#endif /* end of KERNEL_VERSION_NUMBER > 0x030300 */ + +#ifdef CONFIG_ARM_MPU +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#elif KERNEL_VERSION_NUMBER < 0x030400 /* version 3.4.0 */ +#include +#else /* > 3.4.0 */ +#include +#endif +#endif + +#ifndef BH_PLATFORM_ZEPHYR +#define BH_PLATFORM_ZEPHYR +#endif + +#define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 7 + +typedef struct k_thread korp_thread; +typedef korp_thread *korp_tid; +typedef struct k_mutex korp_mutex; +typedef unsigned int korp_sem; + +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + +struct os_thread_wait_node; +typedef struct os_thread_wait_node *os_thread_wait_list; +typedef struct korp_cond { + struct k_mutex wait_list_lock; + os_thread_wait_list thread_wait_list; +} korp_cond; + +#ifndef Z_TIMEOUT_MS +#define Z_TIMEOUT_MS(ms) ms +#endif + +/* clang-format off */ +void abort(void); +size_t strspn(const char *s, const char *accept); +size_t strcspn(const char *s, const char *reject); + +/* math functions which are not provided by os with minimal libc */ +#if defined(CONFIG_MINIMAL_LIBC) +double atan(double x); +double atan2(double y, double x); +double sqrt(double x); +double floor(double x); +double ceil(double x); +double fmin(double x, double y); +double fmax(double x, double y); +double rint(double x); +double fabs(double x); +double trunc(double x); +float sqrtf(float x); +float floorf(float x); +float ceilf(float x); +float fminf(float x, float y); +float fmaxf(float x, float y); +float rintf(float x); +float fabsf(float x); +float truncf(float x); +int signbit(double x); +int isnan(double x); +double pow(double x, double y); +double scalbn(double x, int n); + +unsigned long long int strtoull(const char *nptr, char **endptr, int base); +double strtod(const char *nptr, char **endptr); +float strtof(const char *nptr, char **endptr); +#else +#include +#endif /* CONFIG_MINIMAL_LIBC */ + +/* clang-format on */ + +#if KERNEL_VERSION_NUMBER >= 0x030100 /* version 3.1.0 */ +#define BH_HAS_SQRT +#define BH_HAS_SQRTF +#endif + +/** + * @brief Allocate executable memory + * + * @param size size of the memory to be allocated + * + * @return the address of the allocated memory if not NULL + */ +typedef void *(*exec_mem_alloc_func_t)(unsigned int size); + +/** + * @brief Release executable memory + * + * @param the address of the executable memory to be released + */ +typedef void (*exec_mem_free_func_t)(void *addr); + +/* Below function are called by external project to set related function + * pointers that will be used to malloc/free executable memory. Otherwise + * default mechanise will be used. + */ +void +set_exec_mem_alloc_func(exec_mem_alloc_func_t alloc_func, + exec_mem_free_func_t free_func); + +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ +typedef int os_file_handle; +typedef void *os_dir_stream; +typedef int os_raw_file_handle; + +static inline os_file_handle +os_get_invalid_handle() +{ + return -1; +} + +static inline int +os_getpagesize() +{ +#ifdef CONFIG_MMU + return CONFIG_MMU_PAGE_SIZE; +#else + /* Return a default page size if the MMU is not enabled */ + return 4096; /* 4KB */ +#endif +} + +#endif diff --git a/wasm-micro-runtime/core/shared/platform/zephyr/shared_platform.cmake b/wasm-micro-runtime/core/shared/platform/zephyr/shared_platform.cmake new file mode 100644 index 0000000..dfd45a4 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/zephyr/shared_platform.cmake @@ -0,0 +1,18 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (PLATFORM_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions(-DBH_PLATFORM_ZEPHYR) + +include_directories(${PLATFORM_SHARED_DIR}) +include_directories(${PLATFORM_SHARED_DIR}/../include) + +if(${CONFIG_MINIMAL_LIBC}) + include (${CMAKE_CURRENT_LIST_DIR}/../common/math/platform_api_math.cmake) +endif() + +file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) + +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE}) + diff --git a/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_platform.c b/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_platform.c new file mode 100644 index 0000000..fc54ba5 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_platform.c @@ -0,0 +1,245 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +/* function pointers for executable memory management */ +static exec_mem_alloc_func_t exec_mem_alloc_func = NULL; +static exec_mem_free_func_t exec_mem_free_func = NULL; + +#if WASM_ENABLE_AOT != 0 +#ifdef CONFIG_ARM_MPU +/** + * This function will allow execute from sram region. + * This is needed for AOT code because by default all soc will + * disable the execute from SRAM. + */ +static void +disable_mpu_rasr_xn(void) +{ + uint32 index; + /* Kept the max index as 8 (irrespective of soc) because the sram + would most likely be set at index 2. */ + for (index = 0U; index < 8; index++) { + MPU->RNR = index; +#ifdef MPU_RASR_XN_Msk + if (MPU->RASR & MPU_RASR_XN_Msk) { + MPU->RASR |= ~MPU_RASR_XN_Msk; + } +#endif + } +} +#endif /* end of CONFIG_ARM_MPU */ +#endif + +static int +_stdout_hook_iwasm(int c) +{ + printk("%c", (char)c); + return 1; +} + +int +os_thread_sys_init(); + +void +os_thread_sys_destroy(); + +int +bh_platform_init() +{ + extern void __stdout_hook_install(int (*hook)(int)); + /* Enable printf() in Zephyr */ + __stdout_hook_install(_stdout_hook_iwasm); + +#if WASM_ENABLE_AOT != 0 +#ifdef CONFIG_ARM_MPU + /* Enable executable memory support */ + disable_mpu_rasr_xn(); +#endif +#endif + + return os_thread_sys_init(); +} + +void +bh_platform_destroy() +{ + os_thread_sys_destroy(); +} + +void * +os_malloc(unsigned size) +{ + return malloc(size); +} + +void * +os_realloc(void *ptr, unsigned size) +{ + return realloc(ptr, size); +} + +void +os_free(void *ptr) +{ + free(ptr); +} + +int +os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} + +#if 0 +struct out_context { + int count; +}; + +typedef int (*out_func_t)(int c, void *ctx); + +static int +char_out(int c, void *ctx) +{ + struct out_context *out_ctx = (struct out_context*)ctx; + out_ctx->count++; + return _stdout_hook_iwasm(c); +} + +int +os_vprintf(const char *fmt, va_list ap) +{ +#if 0 + struct out_context ctx = { 0 }; + cbvprintf(char_out, &ctx, fmt, ap); + return ctx.count; +#else + vprintk(fmt, ap); + return 0; +#endif +} +#endif + +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} + +#if KERNEL_VERSION_NUMBER <= 0x020400 /* version 2.4.0 */ +void +abort(void) +{ + int i = 0; + os_printf("%d\n", 1 / i); +} +#endif + +#if KERNEL_VERSION_NUMBER <= 0x010E01 /* version 1.14.1 */ +size_t +strspn(const char *s, const char *accept) +{ + os_printf("## unimplemented function %s called", __FUNCTION__); + return 0; +} + +size_t +strcspn(const char *s, const char *reject) +{ + os_printf("## unimplemented function %s called", __FUNCTION__); + return 0; +} +#endif + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + if ((uint64)size >= UINT32_MAX) + return NULL; + if (exec_mem_alloc_func) + return exec_mem_alloc_func((uint32)size); + else + return BH_MALLOC(size); +} + +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size) +{ + return os_mremap_slow(old_addr, old_size, new_size); +} + +void +os_munmap(void *addr, size_t size) +{ + if (exec_mem_free_func) + exec_mem_free_func(addr); + else + BH_FREE(addr); +} + +int +os_mprotect(void *addr, size_t size, int prot) +{ + return 0; +} + +void +os_dcache_flush() +{ +#if defined(CONFIG_CPU_CORTEX_M7) && defined(CONFIG_ARM_MPU) +#if KERNEL_VERSION_NUMBER < 0x030300 /* version 3.3.0 */ + uint32 key; + key = irq_lock(); + SCB_CleanDCache(); + irq_unlock(key); +#else + sys_cache_data_flush_all(); +#endif +#elif defined(CONFIG_SOC_CVF_EM7D) && defined(CONFIG_ARC_MPU) \ + && defined(CONFIG_CACHE_FLUSHING) + __asm__ __volatile__("sync"); + z_arc_v2_aux_reg_write(_ARC_V2_DC_FLSH, BIT(0)); + __asm__ __volatile__("sync"); +#endif +} + +void +os_icache_flush(void *start, size_t len) +{ +#if KERNEL_VERSION_NUMBER >= 0x030300 /* version 3.3.0 */ + sys_cache_instr_flush_range(start, len); +#endif +} + +void +set_exec_mem_alloc_func(exec_mem_alloc_func_t alloc_func, + exec_mem_free_func_t free_func) +{ + exec_mem_alloc_func = alloc_func; + exec_mem_free_func = free_func; +} diff --git a/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_thread.c b/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_thread.c new file mode 100644 index 0000000..53ca71f --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_thread.c @@ -0,0 +1,610 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" +#include "platform_api_extension.h" + +/* clang-format off */ +#define bh_assert(v) do { \ + if (!(v)) { \ + printf("\nASSERTION FAILED: %s, at %s, line %d\n", \ + #v, __FILE__, __LINE__); \ + abort(); \ + } \ +} while (0) +/* clang-format on */ + +#if defined(CONFIG_ARM_MPU) || defined(CONFIG_ARC_MPU) \ + || KERNEL_VERSION_NUMBER > 0x020300 /* version 2.3.0 */ +#define BH_ENABLE_ZEPHYR_MPU_STACK 1 +#elif !defined(BH_ENABLE_ZEPHYR_MPU_STACK) +#define BH_ENABLE_ZEPHYR_MPU_STACK 0 +#endif +#if !defined(BH_ZEPHYR_MPU_STACK_SIZE) +#define BH_ZEPHYR_MPU_STACK_SIZE APP_THREAD_STACK_SIZE_MIN +#endif +#if !defined(BH_ZEPHYR_MPU_STACK_COUNT) +#define BH_ZEPHYR_MPU_STACK_COUNT 4 +#endif + +#if BH_ENABLE_ZEPHYR_MPU_STACK != 0 +static K_THREAD_STACK_ARRAY_DEFINE(mpu_stacks, BH_ZEPHYR_MPU_STACK_COUNT, + BH_ZEPHYR_MPU_STACK_SIZE); +static bool mpu_stack_allocated[BH_ZEPHYR_MPU_STACK_COUNT]; +static struct k_mutex mpu_stack_lock; + +static char * +mpu_stack_alloc() +{ + int i; + + k_mutex_lock(&mpu_stack_lock, K_FOREVER); + for (i = 0; i < BH_ZEPHYR_MPU_STACK_COUNT; i++) { + if (!mpu_stack_allocated[i]) { + mpu_stack_allocated[i] = true; + k_mutex_unlock(&mpu_stack_lock); + return (char *)mpu_stacks[i]; + } + } + k_mutex_unlock(&mpu_stack_lock); + return NULL; +} + +static void +mpu_stack_free(char *stack) +{ + int i; + + k_mutex_lock(&mpu_stack_lock, K_FOREVER); + for (i = 0; i < BH_ZEPHYR_MPU_STACK_COUNT; i++) { + if ((char *)mpu_stacks[i] == stack) + mpu_stack_allocated[i] = false; + } + k_mutex_unlock(&mpu_stack_lock); +} +#endif + +typedef struct os_thread_wait_node { + struct k_sem sem; + os_thread_wait_list next; +} os_thread_wait_node; + +typedef struct os_thread_data { + /* Next thread data */ + struct os_thread_data *next; + /* Zephyr thread handle */ + korp_tid tid; + /* Jeff thread local root */ + void *tlr; + /* Lock for waiting list */ + struct k_mutex wait_list_lock; + /* Waiting list of other threads who are joining this thread */ + os_thread_wait_list thread_wait_list; + /* Thread stack size */ + unsigned stack_size; +#if BH_ENABLE_ZEPHYR_MPU_STACK == 0 + /* Thread stack */ + char stack[1]; +#else + char *stack; +#endif +} os_thread_data; + +typedef struct os_thread_obj { + struct k_thread thread; + /* Whether the thread is terminated and this thread object is to + be freed in the future. */ + bool to_be_freed; + struct os_thread_obj *next; +} os_thread_obj; + +static bool is_thread_sys_inited = false; + +/* Thread data of supervisor thread */ +static os_thread_data supervisor_thread_data; + +/* Lock for thread data list */ +static struct k_mutex thread_data_lock; + +/* Thread data list */ +static os_thread_data *thread_data_list = NULL; + +/* Lock for thread object list */ +static struct k_mutex thread_obj_lock; + +/* Thread object list */ +static os_thread_obj *thread_obj_list = NULL; + +static void +thread_data_list_add(os_thread_data *thread_data) +{ + k_mutex_lock(&thread_data_lock, K_FOREVER); + if (!thread_data_list) + thread_data_list = thread_data; + else { + /* If already in list, just return */ + os_thread_data *p = thread_data_list; + while (p) { + if (p == thread_data) { + k_mutex_unlock(&thread_data_lock); + return; + } + p = p->next; + } + + /* Set as head of list */ + thread_data->next = thread_data_list; + thread_data_list = thread_data; + } + k_mutex_unlock(&thread_data_lock); +} + +static void +thread_data_list_remove(os_thread_data *thread_data) +{ + k_mutex_lock(&thread_data_lock, K_FOREVER); + if (thread_data_list) { + if (thread_data_list == thread_data) + thread_data_list = thread_data_list->next; + else { + /* Search and remove it from list */ + os_thread_data *p = thread_data_list; + while (p && p->next != thread_data) + p = p->next; + if (p && p->next == thread_data) + p->next = p->next->next; + } + } + k_mutex_unlock(&thread_data_lock); +} + +static os_thread_data * +thread_data_list_lookup(k_tid_t tid) +{ + k_mutex_lock(&thread_data_lock, K_FOREVER); + if (thread_data_list) { + os_thread_data *p = thread_data_list; + while (p) { + if (p->tid == tid) { + /* Found */ + k_mutex_unlock(&thread_data_lock); + return p; + } + p = p->next; + } + } + k_mutex_unlock(&thread_data_lock); + return NULL; +} + +static void +thread_obj_list_add(os_thread_obj *thread_obj) +{ + k_mutex_lock(&thread_obj_lock, K_FOREVER); + if (!thread_obj_list) + thread_obj_list = thread_obj; + else { + /* Set as head of list */ + thread_obj->next = thread_obj_list; + thread_obj_list = thread_obj; + } + k_mutex_unlock(&thread_obj_lock); +} + +static void +thread_obj_list_reclaim() +{ + os_thread_obj *p, *p_prev; + k_mutex_lock(&thread_obj_lock, K_FOREVER); + p_prev = NULL; + p = thread_obj_list; + while (p) { + if (p->to_be_freed) { + if (p_prev == NULL) { /* p is the head of list */ + thread_obj_list = p->next; + BH_FREE(p); + p = thread_obj_list; + } + else { /* p is not the head of list */ + p_prev->next = p->next; + BH_FREE(p); + p = p_prev->next; + } + } + else { + p_prev = p; + p = p->next; + } + } + k_mutex_unlock(&thread_obj_lock); +} + +int +os_thread_sys_init() +{ + if (is_thread_sys_inited) + return BHT_OK; + +#if BH_ENABLE_ZEPHYR_MPU_STACK != 0 + k_mutex_init(&mpu_stack_lock); +#endif + k_mutex_init(&thread_data_lock); + k_mutex_init(&thread_obj_lock); + + /* Initialize supervisor thread data */ + memset(&supervisor_thread_data, 0, sizeof(supervisor_thread_data)); + supervisor_thread_data.tid = k_current_get(); + /* Set as head of thread data list */ + thread_data_list = &supervisor_thread_data; + + is_thread_sys_inited = true; + return BHT_OK; +} + +void +os_thread_sys_destroy(void) +{ + if (is_thread_sys_inited) { + is_thread_sys_inited = false; + } +} + +static os_thread_data * +thread_data_current() +{ + k_tid_t tid = k_current_get(); + return thread_data_list_lookup(tid); +} + +static void +os_thread_cleanup(void) +{ + os_thread_data *thread_data = thread_data_current(); + + bh_assert(thread_data != NULL); + k_mutex_lock(&thread_data->wait_list_lock, K_FOREVER); + if (thread_data->thread_wait_list) { + /* Signal each joining thread */ + os_thread_wait_list head = thread_data->thread_wait_list; + while (head) { + os_thread_wait_list next = head->next; + k_sem_give(&head->sem); + /* head will be freed by joining thread */ + head = next; + } + thread_data->thread_wait_list = NULL; + } + k_mutex_unlock(&thread_data->wait_list_lock); + + thread_data_list_remove(thread_data); + /* Set flag to true for the next thread creating to + free the thread object */ + ((os_thread_obj *)thread_data->tid)->to_be_freed = true; +#if BH_ENABLE_ZEPHYR_MPU_STACK != 0 + mpu_stack_free(thread_data->stack); +#endif + BH_FREE(thread_data); +} + +static void +os_thread_wrapper(void *start, void *arg, void *thread_data) +{ + /* Set thread custom data */ + ((os_thread_data *)thread_data)->tid = k_current_get(); + thread_data_list_add(thread_data); + + ((thread_start_routine_t)start)(arg); + os_thread_cleanup(); +} + +int +os_thread_create(korp_tid *p_tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(p_tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +int +os_thread_create_with_prio(korp_tid *p_tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + korp_tid tid; + os_thread_data *thread_data; + unsigned thread_data_size; + + if (!p_tid || !stack_size) + return BHT_ERROR; + + /* Free the thread objects of terminated threads */ + thread_obj_list_reclaim(); + + /* Create and initialize thread object */ + if (!(tid = BH_MALLOC(sizeof(os_thread_obj)))) + return BHT_ERROR; + + memset(tid, 0, sizeof(os_thread_obj)); + + /* Create and initialize thread data */ +#if BH_ENABLE_ZEPHYR_MPU_STACK == 0 + if (stack_size < APP_THREAD_STACK_SIZE_MIN) + stack_size = APP_THREAD_STACK_SIZE_MIN; + thread_data_size = offsetof(os_thread_data, stack) + stack_size; +#else + stack_size = BH_ZEPHYR_MPU_STACK_SIZE; + thread_data_size = sizeof(os_thread_data); +#endif + if (!(thread_data = BH_MALLOC(thread_data_size))) { + goto fail1; + } + + memset(thread_data, 0, thread_data_size); + k_mutex_init(&thread_data->wait_list_lock); + thread_data->stack_size = stack_size; + thread_data->tid = tid; + +#if BH_ENABLE_ZEPHYR_MPU_STACK != 0 + if (!(thread_data->stack = mpu_stack_alloc())) { + goto fail2; + } +#endif + + /* Create the thread */ + if (!((tid = k_thread_create(tid, (k_thread_stack_t *)thread_data->stack, + stack_size, os_thread_wrapper, start, arg, + thread_data, prio, 0, K_NO_WAIT)))) { + goto fail3; + } + + bh_assert(tid == thread_data->tid); + + /* Set thread custom data */ + thread_data_list_add(thread_data); + thread_obj_list_add((os_thread_obj *)tid); + *p_tid = tid; + return BHT_OK; + +fail3: +#if BH_ENABLE_ZEPHYR_MPU_STACK != 0 + mpu_stack_free(thread_data->stack); +fail2: +#endif + BH_FREE(thread_data); +fail1: + BH_FREE(tid); + return BHT_ERROR; +} + +korp_tid +os_self_thread() +{ + return (korp_tid)k_current_get(); +} + +int +os_thread_join(korp_tid thread, void **value_ptr) +{ + (void)value_ptr; + os_thread_data *thread_data; + os_thread_wait_node *node; + + /* Create wait node and append it to wait list */ + if (!(node = BH_MALLOC(sizeof(os_thread_wait_node)))) + return BHT_ERROR; + + k_sem_init(&node->sem, 0, 1); + node->next = NULL; + + /* Get thread data */ + thread_data = thread_data_list_lookup(thread); + bh_assert(thread_data != NULL); + + k_mutex_lock(&thread_data->wait_list_lock, K_FOREVER); + if (!thread_data->thread_wait_list) + thread_data->thread_wait_list = node; + else { + /* Add to end of waiting list */ + os_thread_wait_node *p = thread_data->thread_wait_list; + while (p->next) + p = p->next; + p->next = node; + } + k_mutex_unlock(&thread_data->wait_list_lock); + + /* Wait the sem */ + k_sem_take(&node->sem, K_FOREVER); + + /* Wait some time for the thread to be actually terminated */ + k_sleep(Z_TIMEOUT_MS(100)); + + /* Destroy resource */ + BH_FREE(node); + return BHT_OK; +} + +int +os_mutex_init(korp_mutex *mutex) +{ + k_mutex_init(mutex); + return BHT_OK; +} + +int +os_recursive_mutex_init(korp_mutex *mutex) +{ + k_mutex_init(mutex); + return BHT_OK; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + (void)mutex; + return BHT_OK; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + return k_mutex_lock(mutex, K_FOREVER); +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ +#if KERNEL_VERSION_NUMBER >= 0x020200 /* version 2.2.0 */ + return k_mutex_unlock(mutex); +#else + k_mutex_unlock(mutex); + return 0; +#endif +} + +int +os_cond_init(korp_cond *cond) +{ + k_mutex_init(&cond->wait_list_lock); + cond->thread_wait_list = NULL; + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ + (void)cond; + return BHT_OK; +} + +static int +os_cond_wait_internal(korp_cond *cond, korp_mutex *mutex, bool timed, int mills) +{ + os_thread_wait_node *node; + + /* Create wait node and append it to wait list */ + if (!(node = BH_MALLOC(sizeof(os_thread_wait_node)))) + return BHT_ERROR; + + k_sem_init(&node->sem, 0, 1); + node->next = NULL; + + k_mutex_lock(&cond->wait_list_lock, K_FOREVER); + if (!cond->thread_wait_list) + cond->thread_wait_list = node; + else { + /* Add to end of wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next) + p = p->next; + p->next = node; + } + k_mutex_unlock(&cond->wait_list_lock); + + /* Unlock mutex, wait sem and lock mutex again */ + k_mutex_unlock(mutex); + k_sem_take(&node->sem, timed ? Z_TIMEOUT_MS(mills) : K_FOREVER); + k_mutex_lock(mutex, K_FOREVER); + + /* Remove wait node from wait list */ + k_mutex_lock(&cond->wait_list_lock, K_FOREVER); + if (cond->thread_wait_list == node) + cond->thread_wait_list = node->next; + else { + /* Remove from the wait list */ + os_thread_wait_node *p = cond->thread_wait_list; + while (p->next != node) + p = p->next; + p->next = node->next; + } + BH_FREE(node); + k_mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + return os_cond_wait_internal(cond, mutex, false, 0); +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + + if (useconds == BHT_WAIT_FOREVER) { + return os_cond_wait_internal(cond, mutex, false, 0); + } + else { + uint64 mills_64 = useconds / 1000; + int32 mills; + + if (mills_64 < (uint64)INT32_MAX) { + mills = (int32)mills_64; + } + else { + mills = INT32_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + return os_cond_wait_internal(cond, mutex, true, mills); + } +} + +int +os_cond_signal(korp_cond *cond) +{ + /* Signal the head wait node of wait list */ + k_mutex_lock(&cond->wait_list_lock, K_FOREVER); + if (cond->thread_wait_list) + k_sem_give(&cond->thread_wait_list->sem); + k_mutex_unlock(&cond->wait_list_lock); + + return BHT_OK; +} + +uint8 * +os_thread_get_stack_boundary() +{ +#if defined(CONFIG_THREAD_STACK_INFO) + korp_tid thread = k_current_get(); + return (uint8 *)thread->stack_info.start; +#else + return NULL; +#endif +} + +void +os_thread_jit_write_protect_np(bool enabled) +{} + +int +os_thread_detach(korp_tid thread) +{ + (void)thread; + return BHT_OK; +} + +void +os_thread_exit(void *retval) +{ + (void)retval; + os_thread_cleanup(); + k_thread_abort(k_current_get()); +} + +int +os_cond_broadcast(korp_cond *cond) +{ + os_thread_wait_node *node; + k_mutex_lock(&cond->wait_list_lock, K_FOREVER); + node = cond->thread_wait_list; + while (node) { + os_thread_wait_node *next = node->next; + k_sem_give(&node->sem); + node = next; + } + k_mutex_unlock(&cond->wait_list_lock); + return BHT_OK; +} diff --git a/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_time.c b/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_time.c new file mode 100644 index 0000000..78bc3e0 --- /dev/null +++ b/wasm-micro-runtime/core/shared/platform/zephyr/zephyr_time.c @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "platform_api_vmcore.h" + +uint64 +os_time_get_boot_us() +{ + return k_uptime_get() * 1000; +} + +uint64 +os_time_thread_cputime_us(void) +{ + /* FIXME if u know the right api */ + return os_time_get_boot_us(); +} diff --git a/wasm-micro-runtime/core/shared/utils/SConscript b/wasm-micro-runtime/core/shared/utils/SConscript new file mode 100644 index 0000000..358f2ff --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/SConscript @@ -0,0 +1,17 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * +import os + +cwd = GetCurrentDir() + +src = Glob('*.c') +CPPPATH = [cwd] + +group = DefineGroup('iwasm_shared_utils', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/shared/utils/bh_assert.c b/wasm-micro-runtime/core/shared/utils/bh_assert.c new file mode 100644 index 0000000..5e62baa --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_assert.c @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_assert.h" + +void +bh_assert_internal(int64 v, const char *file_name, int line_number, + const char *expr_string) +{ + if (v) + return; + + if (!file_name) + file_name = "NULL FILENAME"; + + if (!expr_string) + expr_string = "NULL EXPR_STRING"; + + LOG_ERROR("\nASSERTION FAILED: %s, at file %s, line %d\n", expr_string, + file_name, line_number); + + abort(); +} diff --git a/wasm-micro-runtime/core/shared/utils/bh_assert.h b/wasm-micro-runtime/core/shared/utils/bh_assert.h new file mode 100644 index 0000000..b7c995a --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_assert.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_ASSERT_H +#define _BH_ASSERT_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if BH_DEBUG != 0 +void +bh_assert_internal(int64 v, const char *file_name, int line_number, + const char *expr_string); +#define bh_assert(expr) \ + bh_assert_internal((int64)(uintptr_t)(expr), __FILE__, __LINE__, #expr) +#else +#define bh_assert(expr) (void)0 +#endif /* end of BH_DEBUG */ + +#if !defined(__has_extension) +#define __has_extension(a) 0 +#endif + +#if __STDC_VERSION__ >= 201112L \ + || (defined(__GNUC__) && __GNUC__ * 0x100 + __GNUC_MINOR__ >= 0x406) \ + || __has_extension(c_static_assert) + +#define bh_static_assert(expr) _Static_assert(expr, #expr) +#else +#define bh_static_assert(expr) /* nothing */ +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _BH_ASSERT_H */ diff --git a/wasm-micro-runtime/core/shared/utils/bh_atomic.h b/wasm-micro-runtime/core/shared/utils/bh_atomic.h new file mode 100644 index 0000000..4f7d9bc --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_atomic.h @@ -0,0 +1,298 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_ATOMIC_H +#define _BH_ATOMIC_H + +#include "gnuc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Why don't we use C11 stdatomics here? + * + * Unlike C11 stdatomics, + * + * - bh_atomic_xxx_t is guaranteed to have the same size as the base type. + * Thus more friendly to our AOT conventions. + * + * - It's available for C++. + * Although C++23 will have C-compatible stdatomics.h, it isn't widely + * available yet. + */ + +/* + * Note about BH_ATOMIC_32_IS_ATOMIC + * + * If BH_ATOMIC_32_IS_ATOMIC == 0, BH_ATOMIC_xxx operations defined below + * are not really atomic and require an external lock. + * + * Expected usage is: + * + * bh_atomic_32_t var = 0; + * uint32 old; + * #if BH_ATOMIC_32_IS_ATOMIC == 0 + * lock(&some_lock); + * #endif + * old = BH_ATOMIC_32_FETCH_AND(var, 1); + * #if BH_ATOMIC_32_IS_ATOMIC == 0 + * unlock(&some_lock); + * #endif + */ + +typedef uint64 bh_atomic_64_t; +typedef uint32 bh_atomic_32_t; +typedef uint16 bh_atomic_16_t; + +/* The flag can be defined by the user if the platform + * supports atomic 32-bit operations. + * If left undefined, it will be automatically defined + * according to the platform. + */ +#ifdef WASM_UINT64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC WASM_UINT64_IS_ATOMIC +#endif /* WASM_UINT64_IS_ATOMIC */ + +#ifdef WASM_UINT32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC WASM_UINT32_IS_ATOMIC +#endif /* WASM_UINT32_IS_ATOMIC */ + +#ifdef WASM_UINT16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC WASM_UINT16_IS_ATOMIC +#endif /* WASM_UINT16_IS_ATOMIC */ + +#if defined(__GNUC_PREREQ) +#if __GNUC_PREREQ(4, 7) +#define CLANG_GCC_HAS_ATOMIC_BUILTIN +#endif +#elif defined(__clang__) +#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 0) +#define CLANG_GCC_HAS_ATOMIC_BUILTIN +#endif +#endif + +#if defined(CLANG_GCC_HAS_ATOMIC_BUILTIN) +#ifndef BH_ATOMIC_64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC 1 +#endif +#ifndef BH_ATOMIC_32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC 1 +#endif +#ifndef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 1 +#endif +#else +#ifndef BH_ATOMIC_64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC 0 +#endif +#ifndef BH_ATOMIC_32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC 0 +#endif +#ifndef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 0 +#endif +#endif + +/* Force disable atomic 16-bit operations on bare-metal RISC-V + * because the 16-bit atomic operations is emulated by 32-bit + * atomic operations, which has linkage problem on current toolchain: + * in function `shared_memory_inc_reference': + * wasm_shared_memory.c:85:(.text.shared_memory_inc_reference+0x10): undefined + * reference to `__atomic_fetch_add_2' + */ +#ifndef WASM_UINT16_IS_ATOMIC +#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) \ + && !defined(__OpenBSD__) && defined(__riscv) +#undef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 0 +#endif +#endif + +/* On some 32-bit platform, disable 64-bit atomic operations, otherwise + * undefined reference to `__atomic_load_8' */ +#ifndef WASM_UINT64_IS_ATOMIC +#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) \ + && !defined(__OpenBSD__) && (defined(__riscv) || defined(__arm__)) \ + && UINT32_MAX == UINTPTR_MAX +#undef BH_ATOMIC_64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC 0 +#endif +#endif + +#if BH_ATOMIC_64_IS_ATOMIC != 0 + +#define BH_ATOMIC_64_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_OR(v, val) \ + __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_AND(v, val) \ + __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_ADD(v, val) \ + __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_SUB(v, val) \ + __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) + +#else /* else of BH_ATOMIC_64_IS_ATOMIC != 0 */ + +#define BH_ATOMIC_64_LOAD(v) (v) +#define BH_ATOMIC_64_STORE(v, val) (v) = val +#define BH_ATOMIC_64_FETCH_OR(v, val) nonatomic_64_fetch_or(&(v), val) +#define BH_ATOMIC_64_FETCH_AND(v, val) nonatomic_64_fetch_and(&(v), val) +#define BH_ATOMIC_64_FETCH_ADD(v, val) nonatomic_64_fetch_add(&(v), val) +#define BH_ATOMIC_64_FETCH_SUB(v, val) nonatomic_64_fetch_sub(&(v), val) + +static inline uint64 +nonatomic_64_fetch_or(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p |= val; + return old; +} + +static inline uint64 +nonatomic_64_fetch_and(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p &= val; + return old; +} + +static inline uint64 +nonatomic_64_fetch_add(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p += val; + return old; +} + +static inline uint64 +nonatomic_64_fetch_sub(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p -= val; + return old; +} +#endif + +#if BH_ATOMIC_32_IS_ATOMIC != 0 + +#define BH_ATOMIC_32_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_OR(v, val) \ + __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_AND(v, val) \ + __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_ADD(v, val) \ + __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_SUB(v, val) \ + __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) + +#else /* else of BH_ATOMIC_32_IS_ATOMIC != 0 */ + +#define BH_ATOMIC_32_LOAD(v) (v) +#define BH_ATOMIC_32_STORE(v, val) (v) = val +#define BH_ATOMIC_32_FETCH_OR(v, val) nonatomic_32_fetch_or(&(v), val) +#define BH_ATOMIC_32_FETCH_AND(v, val) nonatomic_32_fetch_and(&(v), val) +#define BH_ATOMIC_32_FETCH_ADD(v, val) nonatomic_32_fetch_add(&(v), val) +#define BH_ATOMIC_32_FETCH_SUB(v, val) nonatomic_32_fetch_sub(&(v), val) + +static inline uint32 +nonatomic_32_fetch_or(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p |= val; + return old; +} + +static inline uint32 +nonatomic_32_fetch_and(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p &= val; + return old; +} + +static inline uint32 +nonatomic_32_fetch_add(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p += val; + return old; +} + +static inline uint32 +nonatomic_32_fetch_sub(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p -= val; + return old; +} + +#endif + +#if BH_ATOMIC_16_IS_ATOMIC != 0 + +#define BH_ATOMIC_16_IS_ATOMIC 1 +#define BH_ATOMIC_16_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_OR(v, val) \ + __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_AND(v, val) \ + __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_ADD(v, val) \ + __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_SUB(v, val) \ + __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) + +#else /* else of BH_ATOMIC_16_IS_ATOMIC != 0 */ + +#define BH_ATOMIC_16_LOAD(v) (v) +#define BH_ATOMIC_16_STORE(v) (v) = val +#define BH_ATOMIC_16_FETCH_OR(v, val) nonatomic_16_fetch_or(&(v), val) +#define BH_ATOMIC_16_FETCH_AND(v, val) nonatomic_16_fetch_and(&(v), val) +#define BH_ATOMIC_16_FETCH_ADD(v, val) nonatomic_16_fetch_add(&(v), val) +#define BH_ATOMIC_16_FETCH_SUB(v, val) nonatomic_16_fetch_sub(&(v), val) + +static inline uint16 +nonatomic_16_fetch_or(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p |= val; + return old; +} + +static inline uint16 +nonatomic_16_fetch_and(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p &= val; + return old; +} + +static inline uint16 +nonatomic_16_fetch_add(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p += val; + return old; +} + +static inline uint16 +nonatomic_16_fetch_sub(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p -= val; + return old; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _BH_ATOMIC_H */ diff --git a/wasm-micro-runtime/core/shared/utils/bh_bitmap.c b/wasm-micro-runtime/core/shared/utils/bh_bitmap.c new file mode 100644 index 0000000..2ee9180 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_bitmap.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_bitmap.h" + +bh_bitmap * +bh_bitmap_new(uintptr_t begin_index, unsigned bitnum) +{ + bh_bitmap *bitmap; + uint32 bitmap_size = (bitnum + 7) / 8; + uint32 total_size = offsetof(bh_bitmap, map) + bitmap_size; + + if (bitnum > UINT32_MAX - 7 || total_size < offsetof(bh_bitmap, map) + || (total_size - offsetof(bh_bitmap, map)) != bitmap_size) { + return NULL; /* integer overflow */ + } + + if ((bitmap = BH_MALLOC(total_size)) != NULL) { + memset(bitmap, 0, total_size); + bitmap->begin_index = begin_index; + bitmap->end_index = begin_index + bitnum; + } + + return bitmap; +} diff --git a/wasm-micro-runtime/core/shared/utils/bh_bitmap.h b/wasm-micro-runtime/core/shared/utils/bh_bitmap.h new file mode 100644 index 0000000..c0e56cb --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_bitmap.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_BITMAP_H +#define _BH_BITMAP_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * A simple fixed size bitmap. + */ +typedef struct bh_bitmap { + /* The first valid bit index. */ + uintptr_t begin_index; + + /* The last valid bit index plus one. */ + uintptr_t end_index; + + /* The bitmap. */ + uint8 map[1]; +} bh_bitmap; + +/** + * Create a new bitmap. + * + * @param begin_index the first valid bit index + * @param bitnum maximal bit number of the bitmap. + * + * @return the new bitmap if succeeds, NULL otherwise. + */ +bh_bitmap * +bh_bitmap_new(uintptr_t begin_index, unsigned bitnum); + +/** + * Delete a bitmap. + * + * @param bitmap the bitmap to be deleted + */ +static inline void +bh_bitmap_delete(bh_bitmap *bitmap) +{ + if (bitmap != NULL) + BH_FREE(bitmap); +} + +/** + * Check whether the given index is in the range of the bitmap. + * + * @param bitmap the bitmap + * @param n the bit index + * + * @return true if the index is in range, false otherwise + */ +static inline bool +bh_bitmap_is_in_range(bh_bitmap *bitmap, uintptr_t n) +{ + return n >= bitmap->begin_index && n < bitmap->end_index; +} + +/** + * Get a bit in the bitmap + * + * @param bitmap the bitmap + * @param n the n-th bit to be get + * + * @return value of the bit + */ +static inline int +bh_bitmap_get_bit(bh_bitmap *bitmap, uintptr_t n) +{ + uintptr_t idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + return (bitmap->map[idx / 8] >> (idx % 8)) & 1; +} + +/** + * Set a bit in the bitmap. + * + * @param bitmap the bitmap + * @param n the n-th bit to be set + */ +static inline void +bh_bitmap_set_bit(bh_bitmap *bitmap, uintptr_t n) +{ + uintptr_t idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + bitmap->map[idx / 8] |= 1 << (idx % 8); +} + +/** + * Clear a bit in the bitmap. + * + * @param bitmap the bitmap + * @param n the n-th bit to be cleared + */ +static inline void +bh_bitmap_clear_bit(bh_bitmap *bitmap, uintptr_t n) +{ + uintptr_t idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + bitmap->map[idx / 8] &= ~(1 << (idx % 8)); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/shared/utils/bh_common.c b/wasm-micro-runtime/core/shared/utils/bh_common.c new file mode 100644 index 0000000..7fe123c --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_common.c @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" + +static char * +align_ptr(char *src, unsigned int b) +{ + uintptr_t v = (uintptr_t)src; + uintptr_t m = b - 1; + return (char *)((v + m) & ~m); +} + +/* +Memory copy, with word alignment +*/ +int +b_memcpy_wa(void *s1, unsigned int s1max, const void *s2, unsigned int n) +{ + char *dest = (char *)s1; + char *src = (char *)s2; + + char *pa = align_ptr(src, 4); + char *pb = align_ptr((src + n), 4); + + unsigned int buff; + const char *p_byte_read; + + unsigned int *p; + char *ps; + + if (n == 0) { + return 0; + } + + if (pa > src) { + pa -= 4; + } + + for (p = (unsigned int *)pa; p < (unsigned int *)pb; p++) { + buff = *(p); + p_byte_read = ((char *)&buff); + + /* read leading word */ + if ((char *)p <= src) { + for (ps = src; ps < ((char *)p + 4); ps++) { + if (ps >= src + n) { + break; + } + p_byte_read = ((char *)&buff) + (ps - (char *)p); + *dest++ = *p_byte_read; + } + } + /* read trailing word */ + else if ((char *)p >= pb - 4) { + for (ps = (char *)p; ps < src + n; ps++) { + *dest++ = *p_byte_read++; + } + } + /* read meaning word(s) */ + else { + if ((char *)p + 4 >= src + n) { + for (ps = (char *)p; ps < src + n; ps++) { + *dest++ = *p_byte_read++; + } + } + else { + *(unsigned int *)dest = buff; + dest += 4; + } + } + } + + return 0; +} + +int +b_memcpy_s(void *s1, unsigned int s1max, const void *s2, unsigned int n) +{ + char *dest = (char *)s1; + char *src = (char *)s2; + if (n == 0) { + return 0; + } + + if (s1 == NULL) { + return -1; + } + if (s2 == NULL || n > s1max) { + memset(dest, 0, s1max); + return -1; + } + memcpy(dest, src, n); + return 0; +} + +int +b_memmove_s(void *s1, unsigned int s1max, const void *s2, unsigned int n) +{ + char *dest = (char *)s1; + char *src = (char *)s2; + if (n == 0) { + return 0; + } + + if (s1 == NULL) { + return -1; + } + if (s2 == NULL || n > s1max) { + memset(dest, 0, s1max); + return -1; + } + memmove(dest, src, n); + return 0; +} + +int +b_strcat_s(char *s1, unsigned int s1max, const char *s2) +{ + if (NULL == s1 || NULL == s2 || s1max < (strlen(s1) + strlen(s2) + 1)) { + return -1; + } + + memcpy(s1 + strlen(s1), s2, strlen(s2) + 1); + return 0; +} + +int +b_strcpy_s(char *s1, unsigned int s1max, const char *s2) +{ + if (NULL == s1 || NULL == s2 || s1max < (strlen(s2) + 1)) { + return -1; + } + + memcpy(s1, s2, strlen(s2) + 1); + return 0; +} + +char * +bh_strdup(const char *s) +{ + uint32 size; + char *s1 = NULL; + + if (s) { + size = (uint32)(strlen(s) + 1); + if ((s1 = BH_MALLOC(size))) + bh_memcpy_s(s1, size, s, size); + } + return s1; +} + +char * +wa_strdup(const char *s) +{ + uint32 size; + char *s1 = NULL; + + if (s) { + size = (uint32)(strlen(s) + 1); + if ((s1 = WA_MALLOC(size))) + bh_memcpy_s(s1, size, s, size); + } + return s1; +} diff --git a/wasm-micro-runtime/core/shared/utils/bh_common.h b/wasm-micro-runtime/core/shared/utils/bh_common.h new file mode 100644 index 0000000..adae722 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_common.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_COMMON_H +#define _BH_COMMON_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define bh_memcpy_s(dest, dlen, src, slen) \ + do { \ + int _ret = b_memcpy_s(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_memcpy_wa(dest, dlen, src, slen) \ + do { \ + int _ret = b_memcpy_wa(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_memmove_s(dest, dlen, src, slen) \ + do { \ + int _ret = b_memmove_s(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_strcat_s(dest, dlen, src) \ + do { \ + int _ret = b_strcat_s(dest, dlen, src); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_strcpy_s(dest, dlen, src) \ + do { \ + int _ret = b_strcpy_s(dest, dlen, src); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +int +b_memcpy_s(void *s1, unsigned int s1max, const void *s2, unsigned int n); +int +b_memcpy_wa(void *s1, unsigned int s1max, const void *s2, unsigned int n); +int +b_memmove_s(void *s1, unsigned int s1max, const void *s2, unsigned int n); +int +b_strcat_s(char *s1, unsigned int s1max, const char *s2); +int +b_strcpy_s(char *s1, unsigned int s1max, const char *s2); + +/* strdup with string allocated by BH_MALLOC */ +char * +bh_strdup(const char *s); + +/* strdup with string allocated by WA_MALLOC */ +char * +wa_strdup(const char *s); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wasm-micro-runtime/core/shared/utils/bh_hashmap.c b/wasm-micro-runtime/core/shared/utils/bh_hashmap.c new file mode 100644 index 0000000..794b7a7 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_hashmap.c @@ -0,0 +1,339 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_hashmap.h" + +typedef struct HashMapElem { + void *key; + void *value; + struct HashMapElem *next; +} HashMapElem; + +struct HashMap { + /* size of element array */ + uint32 size; + /* lock for elements */ + korp_mutex *lock; + /* hash function of key */ + HashFunc hash_func; + /* key equal function */ + KeyEqualFunc key_equal_func; + KeyDestroyFunc key_destroy_func; + ValueDestroyFunc value_destroy_func; + HashMapElem *elements[1]; +}; + +HashMap * +bh_hash_map_create(uint32 size, bool use_lock, HashFunc hash_func, + KeyEqualFunc key_equal_func, KeyDestroyFunc key_destroy_func, + ValueDestroyFunc value_destroy_func) +{ + HashMap *map; + uint64 total_size; + + if (size < HASH_MAP_MIN_SIZE) + size = HASH_MAP_MIN_SIZE; + + if (size > HASH_MAP_MAX_SIZE) { + LOG_ERROR("HashMap create failed: size is too large.\n"); + return NULL; + } + + if (!hash_func || !key_equal_func) { + LOG_ERROR("HashMap create failed: hash function or key equal function " + " is NULL.\n"); + return NULL; + } + + total_size = offsetof(HashMap, elements) + + sizeof(HashMapElem *) * (uint64)size + + (use_lock ? sizeof(korp_mutex) : 0); + + /* size <= HASH_MAP_MAX_SIZE, so total_size won't be larger than + UINT32_MAX, no need to check integer overflow */ + if (!(map = BH_MALLOC((uint32)total_size))) { + LOG_ERROR("HashMap create failed: alloc memory failed.\n"); + return NULL; + } + + memset(map, 0, (uint32)total_size); + + if (use_lock) { + map->lock = (korp_mutex *)((uint8 *)map + offsetof(HashMap, elements) + + sizeof(HashMapElem *) * size); + if (os_mutex_init(map->lock)) { + LOG_ERROR("HashMap create failed: init map lock failed.\n"); + BH_FREE(map); + return NULL; + } + } + + map->size = size; + map->hash_func = hash_func; + map->key_equal_func = key_equal_func; + map->key_destroy_func = key_destroy_func; + map->value_destroy_func = value_destroy_func; + return map; +} + +bool +bh_hash_map_insert(HashMap *map, void *key, void *value) +{ + uint32 index; + HashMapElem *elem; + + if (!map || !key) { + LOG_ERROR("HashMap insert elem failed: map or key is NULL.\n"); + return false; + } + + if (map->lock) { + os_mutex_lock(map->lock); + } + + index = map->hash_func(key) % map->size; + elem = map->elements[index]; + while (elem) { + if (map->key_equal_func(elem->key, key)) { + LOG_ERROR("HashMap insert elem failed: duplicated key found.\n"); + goto fail; + } + elem = elem->next; + } + + if (!(elem = BH_MALLOC(sizeof(HashMapElem)))) { + LOG_ERROR("HashMap insert elem failed: alloc memory failed.\n"); + goto fail; + } + + elem->key = key; + elem->value = value; + elem->next = map->elements[index]; + map->elements[index] = elem; + + if (map->lock) { + os_mutex_unlock(map->lock); + } + return true; + +fail: + if (map->lock) { + os_mutex_unlock(map->lock); + } + return false; +} + +void * +bh_hash_map_find(HashMap *map, void *key) +{ + uint32 index; + HashMapElem *elem; + void *value; + + if (!map || !key) { + LOG_ERROR("HashMap find elem failed: map or key is NULL.\n"); + return NULL; + } + + if (map->lock) { + os_mutex_lock(map->lock); + } + + index = map->hash_func(key) % map->size; + elem = map->elements[index]; + + while (elem) { + if (map->key_equal_func(elem->key, key)) { + value = elem->value; + if (map->lock) { + os_mutex_unlock(map->lock); + } + return value; + } + elem = elem->next; + } + + if (map->lock) { + os_mutex_unlock(map->lock); + } + return NULL; +} + +bool +bh_hash_map_update(HashMap *map, void *key, void *value, void **p_old_value) +{ + uint32 index; + HashMapElem *elem; + + if (!map || !key) { + LOG_ERROR("HashMap update elem failed: map or key is NULL.\n"); + return false; + } + + if (map->lock) { + os_mutex_lock(map->lock); + } + + index = map->hash_func(key) % map->size; + elem = map->elements[index]; + + while (elem) { + if (map->key_equal_func(elem->key, key)) { + if (p_old_value) + *p_old_value = elem->value; + elem->value = value; + if (map->lock) { + os_mutex_unlock(map->lock); + } + return true; + } + elem = elem->next; + } + + if (map->lock) { + os_mutex_unlock(map->lock); + } + return false; +} + +bool +bh_hash_map_remove(HashMap *map, void *key, void **p_old_key, + void **p_old_value) +{ + uint32 index; + HashMapElem *elem, *prev; + + if (!map || !key) { + LOG_ERROR("HashMap remove elem failed: map or key is NULL.\n"); + return false; + } + + if (map->lock) { + os_mutex_lock(map->lock); + } + + index = map->hash_func(key) % map->size; + prev = elem = map->elements[index]; + + while (elem) { + if (map->key_equal_func(elem->key, key)) { + if (p_old_key) + *p_old_key = elem->key; + if (p_old_value) + *p_old_value = elem->value; + + if (elem == map->elements[index]) + map->elements[index] = elem->next; + else + prev->next = elem->next; + + BH_FREE(elem); + + if (map->lock) { + os_mutex_unlock(map->lock); + } + return true; + } + + prev = elem; + elem = elem->next; + } + + if (map->lock) { + os_mutex_unlock(map->lock); + } + return false; +} + +bool +bh_hash_map_destroy(HashMap *map) +{ + uint32 index; + HashMapElem *elem, *next; + + if (!map) { + LOG_ERROR("HashMap destroy failed: map is NULL.\n"); + return false; + } + + if (map->lock) { + os_mutex_lock(map->lock); + } + + for (index = 0; index < map->size; index++) { + elem = map->elements[index]; + while (elem) { + next = elem->next; + + if (map->key_destroy_func) { + map->key_destroy_func(elem->key); + } + if (map->value_destroy_func) { + map->value_destroy_func(elem->value); + } + BH_FREE(elem); + + elem = next; + } + } + + if (map->lock) { + os_mutex_unlock(map->lock); + os_mutex_destroy(map->lock); + } + BH_FREE(map); + return true; +} + +uint32 +bh_hash_map_get_struct_size(HashMap *hashmap) +{ + uint32 size = (uint32)(uintptr_t)offsetof(HashMap, elements) + + (uint32)sizeof(HashMapElem *) * hashmap->size; + + if (hashmap->lock) { + size += (uint32)sizeof(korp_mutex); + } + + return size; +} + +uint32 +bh_hash_map_get_elem_struct_size() +{ + return (uint32)sizeof(HashMapElem); +} + +bool +bh_hash_map_traverse(HashMap *map, TraverseCallbackFunc callback, + void *user_data) +{ + uint32 index; + HashMapElem *elem, *next; + + if (!map || !callback) { + LOG_ERROR("HashMap traverse failed: map or callback is NULL.\n"); + return false; + } + + if (map->lock) { + os_mutex_lock(map->lock); + } + + for (index = 0; index < map->size; index++) { + elem = map->elements[index]; + while (elem) { + next = elem->next; + callback(elem->key, elem->value, user_data); + elem = next; + } + } + + if (map->lock) { + os_mutex_unlock(map->lock); + } + + return true; +} diff --git a/wasm-micro-runtime/core/shared/utils/bh_hashmap.h b/wasm-micro-runtime/core/shared/utils/bh_hashmap.h new file mode 100644 index 0000000..38aa2c6 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_hashmap.h @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASM_HASHMAP_H +#define WASM_HASHMAP_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Minimum initial size of hash map */ +#define HASH_MAP_MIN_SIZE 4 + +/* Maximum initial size of hash map */ +#define HASH_MAP_MAX_SIZE 65536 + +struct HashMap; +typedef struct HashMap HashMap; + +/* Hash function: to get the hash value of key. */ +typedef uint32 (*HashFunc)(const void *key); + +/* Key equal function: to check whether two keys are equal. */ +typedef bool (*KeyEqualFunc)(void *key1, void *key2); + +/* Key destroy function: to destroy the key, auto called + for each key when the hash map is destroyed. */ +typedef void (*KeyDestroyFunc)(void *key); + +/* Value destroy function: to destroy the value, auto called + for each value when the hash map is destroyed. */ +typedef void (*ValueDestroyFunc)(void *value); + +/* traverse callback function: + auto called when traverse every hash element */ +typedef void (*TraverseCallbackFunc)(void *key, void *value, void *user_data); + +/** + * Create a hash map. + * + * @param size: the initial size of the hash map + * @param use_lock whether to lock the hash map when operating on it + * @param hash_func hash function of the key, must be specified + * @param key_equal_func key equal function, check whether two keys + * are equal, must be specified + * @param key_destroy_func key destroy function, called for each key if not NULL + * when the hash map is destroyed + * @param value_destroy_func value destroy function, called for each value if + * not NULL when the hash map is destroyed + * + * @return the hash map created, NULL if failed + */ +HashMap * +bh_hash_map_create(uint32 size, bool use_lock, HashFunc hash_func, + KeyEqualFunc key_equal_func, KeyDestroyFunc key_destroy_func, + ValueDestroyFunc value_destroy_func); + +/** + * Insert an element to the hash map + * + * @param map the hash map to insert element + * @key the key of the element + * @value the value of the element + * + * @return true if success, false otherwise + * Note: fail if key is NULL or duplicated key exists in the hash map, + */ +bool +bh_hash_map_insert(HashMap *map, void *key, void *value); + +/** + * Find an element in the hash map + * + * @param map the hash map to find element + * @key the key of the element + * + * @return the value of the found element if success, NULL otherwise + */ +void * +bh_hash_map_find(HashMap *map, void *key); + +/** + * Update an element in the hash map with new value + * + * @param map the hash map to update element + * @key the key of the element + * @value the new value of the element + * @p_old_value if not NULL, copies the old value to it + * + * @return true if success, false otherwise + * Note: the old value won't be destroyed by value destroy function, + * it will be copied to p_old_value for user to process. + */ +bool +bh_hash_map_update(HashMap *map, void *key, void *value, void **p_old_value); + +/** + * Remove an element from the hash map + * + * @param map the hash map to remove element + * @key the key of the element + * @p_old_key if not NULL, copies the old key to it + * @p_old_value if not NULL, copies the old value to it + * + * @return true if success, false otherwise + * Note: the old key and old value won't be destroyed by key destroy + * function and value destroy function, they will be copied to + * p_old_key and p_old_value for user to process. + */ +bool +bh_hash_map_remove(HashMap *map, void *key, void **p_old_key, + void **p_old_value); + +/** + * Destroy the hashmap + * + * @param map the hash map to destroy + * + * @return true if success, false otherwise + * Note: the key destroy function and value destroy function will be + * called to destroy each element's key and value if they are + * not NULL. + */ +bool +bh_hash_map_destroy(HashMap *map); + +/** + * Get the structure size of HashMap + * + * @param map the hash map to calculate + * + * @return the memory space occupied by HashMap structure + */ +uint32 +bh_hash_map_get_struct_size(HashMap *hashmap); + +/** + * Get the structure size of HashMap Element + * + * @return the memory space occupied by HashMapElem structure + */ +uint32 +bh_hash_map_get_elem_struct_size(void); + +/** + * Traverse the hash map and call the callback function + * + * @param map the hash map to traverse + * @param callback the function to be called for every element + * @param user_data the argument to be passed to the callback function + * + * @return true if success, false otherwise + * Note: if the hash map has lock, the map will be locked during traverse, + * keep the callback function as simple as possible. + */ +bool +bh_hash_map_traverse(HashMap *map, TraverseCallbackFunc callback, + void *user_data); + +#ifdef __cplusplus +} +#endif + +#endif /* endof WASM_HASHMAP_H */ diff --git a/wasm-micro-runtime/core/shared/utils/bh_list.c b/wasm-micro-runtime/core/shared/utils/bh_list.c new file mode 100644 index 0000000..7102d42 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_list.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_list.h" + +#if BH_DEBUG != 0 +/** + * Test whehter a pointer value has exist in given list. + * + * @param list pointer to list. + * @param elem pointer to elem that will be inserted into list. + * @return true if the pointer has been in the list; + * false otherwise. + */ +static bool +bh_list_is_elem_exist(bh_list *list, void *elem); +#endif + +bh_list_status +bh_list_init(bh_list *list) +{ + if (!list) + return BH_LIST_ERROR; + + (list->head).next = NULL; + list->len = 0; + return BH_LIST_SUCCESS; +} + +bh_list_status +bh_list_insert(bh_list *list, void *elem) +{ + bh_list_link *p = NULL; + + if (!list || !elem) + return BH_LIST_ERROR; +#if BH_DEBUG != 0 + bh_assert(!bh_list_is_elem_exist(list, elem)); +#endif + p = (bh_list_link *)elem; + p->next = (list->head).next; + (list->head).next = p; + list->len++; + return BH_LIST_SUCCESS; +} + +bh_list_status +bh_list_remove(bh_list *list, void *elem) +{ + bh_list_link *cur = NULL; + bh_list_link *prev = NULL; + + if (!list || !elem) + return BH_LIST_ERROR; + + cur = (list->head).next; + + while (cur) { + if (cur == elem) { + if (prev) + prev->next = cur->next; + else + (list->head).next = cur->next; + + list->len--; + return BH_LIST_SUCCESS; + } + + prev = cur; + cur = cur->next; + } + + return BH_LIST_ERROR; +} + +uint32 +bh_list_length(bh_list *list) +{ + return (list ? list->len : 0); +} + +void * +bh_list_first_elem(bh_list *list) +{ + return (list ? (list->head).next : NULL); +} + +void * +bh_list_elem_next(void *node) +{ + return (node ? ((bh_list_link *)node)->next : NULL); +} + +#if BH_DEBUG != 0 +static bool +bh_list_is_elem_exist(bh_list *list, void *elem) +{ + bh_list_link *p = NULL; + + if (!list || !elem) + return false; + + p = (list->head).next; + while (p && p != elem) + p = p->next; + + return (p != NULL); +} +#endif diff --git a/wasm-micro-runtime/core/shared/utils/bh_list.h b/wasm-micro-runtime/core/shared/utils/bh_list.h new file mode 100644 index 0000000..f102153 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_list.h @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_LIST_H +#define _BH_LIST_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "bh_platform.h" + +/* List user should embedded bh_list_link into list elem data structure + * definition. And bh_list_link data field should be the first field. + * For example, if we would like to use bh_list for our own data type A, + * A must be defined as a structure like below: + * struct A { + * bh_list_link l; + * ... + * }; + * + * bh_list_link is defined as a structure (not typedef void*). + * It will make extend list into bi-direction easy. + */ +typedef struct bh_list_link { + struct bh_list_link *next; +} bh_list_link; + +typedef struct bh_list { + bh_list_link head; + uint32 len; +} bh_list; + +/* list operation return value */ +typedef enum bh_list_status { + BH_LIST_SUCCESS = 0, + BH_LIST_ERROR = -1 +} bh_list_status; + +/** + * Initialize a list. + * + * @param list pointer to list. + * @return BH_LIST_ERROR if OK; + * BH_LIST_ERROR if list pointer is NULL. + */ +bh_list_status +bh_list_init(bh_list *list); + +/** + * Insert an elem pointer into list. The list node memory is maintained by list + * while elem memory is the responsibility of list user. + * + * @param list pointer to list. + * @param elem pointer to elem that will be inserted into list. + * @return BH_LIST_ERROR if OK; + * BH_LIST_ERROR if input is invalid or no memory + * available. + */ +bh_list_status +bh_list_insert(bh_list *list, void *elem); + +/** + * Remove an elem pointer from list. The list node memory is maintained by list + * while elem memory is the responsibility of list user. + * + * @param list pointer to list. + * @param elem pointer to elem that will be inserted into list. + * @return BH_LIST_ERROR if OK; + * BH_LIST_ERROR if element does not exist in given + * list. + */ +bh_list_status +bh_list_remove(bh_list *list, void *elem); + +/** + * Get the list length. + * + * @param list pointer to list. + * @return the length of the list. + */ +uint32 +bh_list_length(bh_list *list); + +/** + * Get the first elem in the list. + * + * @param list pointer to list. + * @return pointer to the first node. + */ +void * +bh_list_first_elem(bh_list *list); + +/** + * Get the next elem of given list input elem. + * + * @param node pointer to list node. + * @return pointer to next list node. + */ +void * +bh_list_elem_next(void *node); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _BH_LIST_H */ diff --git a/wasm-micro-runtime/core/shared/utils/bh_log.c b/wasm-micro-runtime/core/shared/utils/bh_log.c new file mode 100644 index 0000000..1ffd9b7 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_log.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_log.h" + +/** + * The verbose level of the log system. Only those verbose logs whose + * levels are less than or equal to this value are outputed. + */ +static uint32 log_verbose_level = BH_LOG_LEVEL_WARNING; + +void +bh_log_set_verbose_level(uint32 level) +{ + log_verbose_level = level; +} + +#ifndef BH_LOG +void +bh_log(LogLevel log_level, const char *file, int line, const char *fmt, ...) +{ + va_list ap; + korp_tid self; + char buf[32] = { 0 }; + uint64 usec; + uint32 t, h, m, s, mills; + + if ((uint32)log_level > log_verbose_level) + return; + + self = os_self_thread(); + + usec = os_time_get_boot_us(); + t = (uint32)(usec / 1000000) % (24 * 60 * 60); + h = t / (60 * 60); + t = t % (60 * 60); + m = t / 60; + s = t % 60; + mills = (uint32)(usec % 1000); + + snprintf(buf, sizeof(buf), + "%02" PRIu32 ":%02" PRIu32 ":%02" PRIu32 ":%03" PRIu32, h, m, s, + mills); + +#ifndef BH_VPRINTF + os_printf("[%s - %" PRIXPTR "]: ", buf, (uintptr_t)self); +#endif + + if (file) + os_printf("%s, line %d, ", file, line); + + va_start(ap, fmt); + os_vprintf(fmt, ap); + va_end(ap); + + os_printf("\n"); +} +#endif + +static uint32 last_time_ms = 0; +static uint32 total_time_ms = 0; + +void +bh_print_time(const char *prompt) +{ + uint32 curr_time_ms; + + if (log_verbose_level < 3) + return; + + curr_time_ms = (uint32)bh_get_tick_ms(); + + if (last_time_ms == 0) + last_time_ms = curr_time_ms; + + total_time_ms += curr_time_ms - last_time_ms; + + os_printf("%-48s time of last stage: %" PRIu32 " ms, total time: %" PRIu32 + " ms\n", + prompt, curr_time_ms - last_time_ms, total_time_ms); + + last_time_ms = curr_time_ms; +} + +void +bh_print_proc_mem(const char *prompt) +{ + char buf[1024] = { 0 }; + + if (log_verbose_level < BH_LOG_LEVEL_DEBUG) + return; + + if (os_dumps_proc_mem_info(buf, sizeof(buf)) != 0) + return; + + os_printf("%s\n", prompt); + os_printf("===== memory usage =====\n"); + os_printf("%s", buf); + os_printf("==========\n"); + return; +} + +void +bh_log_proc_mem(const char *function, uint32 line) +{ + char prompt[128] = { 0 }; + snprintf(prompt, sizeof(prompt), "[MEM] %s(...) L%" PRIu32, function, line); + bh_print_proc_mem(prompt); +} diff --git a/wasm-micro-runtime/core/shared/utils/bh_log.h b/wasm-micro-runtime/core/shared/utils/bh_log.h new file mode 100644 index 0000000..53921b2 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_log.h @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +/** + * @file bh_log.h + * @date Tue Nov 8 18:19:10 2011 + * + * @brief This log system supports wrapping multiple outputs into one + * log message. This is useful for outputting variable-length logs + * without additional memory overhead (the buffer for concatenating + * the message), e.g. exception stack trace, which cannot be printed + * by a single log calling without the help of an additional buffer. + * Avoiding additional memory buffer is useful for resource-constraint + * systems. It can minimize the impact of log system on applications + * and logs can be printed even when no enough memory is available. + * Functions with prefix "_" are private functions. Only macros that + * are not start with "_" are exposed and can be used. + */ + +#ifndef _BH_LOG_H +#define _BH_LOG_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + BH_LOG_LEVEL_FATAL = 0, + BH_LOG_LEVEL_ERROR = 1, + BH_LOG_LEVEL_WARNING = 2, + BH_LOG_LEVEL_DEBUG = 3, + BH_LOG_LEVEL_VERBOSE = 4 +} LogLevel; + +void +bh_log_set_verbose_level(uint32 level); + +#ifndef BH_LOG +void +bh_log(LogLevel log_level, const char *file, int line, const char *fmt, ...); +#else +void +BH_LOG(uint32 log_level, const char *file, int line, const char *fmt, ...); +#define bh_log BH_LOG +#endif + +#ifdef BH_PLATFORM_NUTTX + +#undef LOG_FATAL +#undef LOG_ERROR +#undef LOG_WARNING +#undef LOG_VERBOSE +#undef LOG_DEBUG + +#endif + +#if BH_DEBUG != 0 +#define LOG_FATAL(...) \ + bh_log(BH_LOG_LEVEL_FATAL, __FILE__, __LINE__, __VA_ARGS__) +#else +#define LOG_FATAL(...) \ + bh_log(BH_LOG_LEVEL_FATAL, __FUNCTION__, __LINE__, __VA_ARGS__) +#endif + +#define LOG_ERROR(...) bh_log(BH_LOG_LEVEL_ERROR, NULL, 0, __VA_ARGS__) +#define LOG_WARNING(...) bh_log(BH_LOG_LEVEL_WARNING, NULL, 0, __VA_ARGS__) +#define LOG_VERBOSE(...) bh_log(BH_LOG_LEVEL_VERBOSE, NULL, 0, __VA_ARGS__) + +#if BH_DEBUG != 0 +#define LOG_DEBUG(...) \ + bh_log(BH_LOG_LEVEL_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#else +#define LOG_DEBUG(...) (void)0 +#endif + +void +bh_print_time(const char *prompt); + +void +bh_print_proc_mem(const char *prompt); + +void +bh_log_proc_mem(const char *function, uint32 line); + +#define LOG_PROC_MEM(...) bh_log_proc_mem(__FUNCTION__, __LINE__) + +#ifdef __cplusplus +} +#endif + +#endif /* _BH_LOG_H */ diff --git a/wasm-micro-runtime/core/shared/utils/bh_platform.h b/wasm-micro-runtime/core/shared/utils/bh_platform.h new file mode 100644 index 0000000..86aef83 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_platform.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_PLATFORM_H +#define _BH_PLATFORM_H + +#include "../platform/include/platform_common.h" +#include "../platform/include/platform_api_vmcore.h" +#include "../platform/include/platform_api_extension.h" +#include "bh_assert.h" +#include "bh_common.h" +#include "bh_hashmap.h" +#include "bh_list.h" +#include "bh_log.h" +#include "bh_queue.h" +#include "bh_vector.h" +#include "runtime_timer.h" + +/** + * WA_MALLOC/WA_FREE need to be redefined for both + * runtime native and WASM app respectively. + * + * Some source files are shared for building native and WASM, + * and this the mem allocator API for these files. + * + * Here we define it for the native world + */ +#ifndef WA_MALLOC +#define WA_MALLOC wasm_runtime_malloc +#endif + +#ifndef WA_FREE +#define WA_FREE wasm_runtime_free +#endif + +#endif /* #ifndef _BH_PLATFORM_H */ diff --git a/wasm-micro-runtime/core/shared/utils/bh_queue.c b/wasm-micro-runtime/core/shared/utils/bh_queue.c new file mode 100644 index 0000000..7c860d1 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_queue.c @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_queue.h" + +typedef struct bh_queue_node { + struct bh_queue_node *next; + struct bh_queue_node *prev; + unsigned short tag; + unsigned int len; + void *body; + bh_msg_cleaner msg_cleaner; +} bh_queue_node; + +struct bh_queue { + bh_queue_mutex queue_lock; + bh_queue_cond queue_wait_cond; + unsigned int cnt; + unsigned int max; + unsigned int drops; + bh_queue_node *head; + bh_queue_node *tail; + + bool exit_loop_run; +}; + +char * +bh_message_payload(bh_message_t message) +{ + return message->body; +} + +uint32 +bh_message_payload_len(bh_message_t message) +{ + return message->len; +} + +int +bh_message_type(bh_message_t message) +{ + return message->tag; +} + +bh_queue * +bh_queue_create() +{ + int ret; + bh_queue *queue = bh_queue_malloc(sizeof(bh_queue)); + + if (queue) { + memset(queue, 0, sizeof(bh_queue)); + queue->max = DEFAULT_QUEUE_LENGTH; + + ret = bh_queue_mutex_init(&queue->queue_lock); + if (ret != 0) { + bh_queue_free(queue); + return NULL; + } + + ret = bh_queue_cond_init(&queue->queue_wait_cond); + if (ret != 0) { + bh_queue_mutex_destroy(&queue->queue_lock); + bh_queue_free(queue); + return NULL; + } + } + + return queue; +} + +void +bh_queue_destroy(bh_queue *queue) +{ + bh_queue_node *node; + + if (!queue) + return; + + bh_queue_mutex_lock(&queue->queue_lock); + while (queue->head) { + node = queue->head; + queue->head = node->next; + + bh_free_msg(node); + } + bh_queue_mutex_unlock(&queue->queue_lock); + + bh_queue_cond_destroy(&queue->queue_wait_cond); + bh_queue_mutex_destroy(&queue->queue_lock); + bh_queue_free(queue); +} + +bool +bh_post_msg2(bh_queue *queue, bh_queue_node *msg) +{ + if (queue->cnt >= queue->max) { + queue->drops++; + bh_free_msg(msg); + return false; + } + + bh_queue_mutex_lock(&queue->queue_lock); + + if (queue->cnt == 0) { + bh_assert(queue->head == NULL); + bh_assert(queue->tail == NULL); + queue->head = queue->tail = msg; + msg->next = msg->prev = NULL; + queue->cnt = 1; + + bh_queue_cond_signal(&queue->queue_wait_cond); + } + else { + msg->next = NULL; + msg->prev = queue->tail; + queue->tail->next = msg; + queue->tail = msg; + queue->cnt++; + } + + bh_queue_mutex_unlock(&queue->queue_lock); + + return true; +} + +bool +bh_post_msg(bh_queue *queue, unsigned short tag, void *body, unsigned int len) +{ + bh_queue_node *msg = bh_new_msg(tag, body, len, NULL); + if (msg == NULL) { + queue->drops++; + if (len != 0 && body) + BH_FREE(body); + return false; + } + + if (!bh_post_msg2(queue, msg)) { + // bh_post_msg2 already freed the msg for failure + return false; + } + + return true; +} + +bh_queue_node * +bh_new_msg(unsigned short tag, void *body, unsigned int len, void *handler) +{ + bh_queue_node *msg = + (bh_queue_node *)bh_queue_malloc(sizeof(bh_queue_node)); + if (msg == NULL) + return NULL; + memset(msg, 0, sizeof(bh_queue_node)); + msg->len = len; + msg->body = body; + msg->tag = tag; + msg->msg_cleaner = (bh_msg_cleaner)handler; + + return msg; +} + +void +bh_free_msg(bh_queue_node *msg) +{ + if (msg->msg_cleaner) { + msg->msg_cleaner(msg->body); + bh_queue_free(msg); + return; + } + + // note: sometime we just use the payload pointer for a integer value + // len!=0 is the only indicator about the body is an allocated buffer. + if (msg->body && msg->len) + bh_queue_free(msg->body); + + bh_queue_free(msg); +} + +bh_message_t +bh_get_msg(bh_queue *queue, uint64 timeout_us) +{ + bh_queue_node *msg = NULL; + bh_queue_mutex_lock(&queue->queue_lock); + + if (queue->cnt == 0) { + bh_assert(queue->head == NULL); + bh_assert(queue->tail == NULL); + + if (timeout_us == 0) { + bh_queue_mutex_unlock(&queue->queue_lock); + return NULL; + } + + bh_queue_cond_timedwait(&queue->queue_wait_cond, &queue->queue_lock, + timeout_us); + } + + if (queue->cnt == 0) { + bh_assert(queue->head == NULL); + bh_assert(queue->tail == NULL); + } + else if (queue->cnt == 1) { + bh_assert(queue->head == queue->tail); + + msg = queue->head; + queue->head = queue->tail = NULL; + queue->cnt = 0; + } + else { + msg = queue->head; + queue->head = queue->head->next; + queue->head->prev = NULL; + queue->cnt--; + } + + bh_queue_mutex_unlock(&queue->queue_lock); + + return msg; +} + +unsigned +bh_queue_get_message_count(bh_queue *queue) +{ + if (!queue) + return 0; + + return queue->cnt; +} + +void +bh_queue_enter_loop_run(bh_queue *queue, bh_queue_handle_msg_callback handle_cb, + void *arg) +{ + if (!queue) + return; + + while (!queue->exit_loop_run) { + bh_queue_node *message = bh_get_msg(queue, BHT_WAIT_FOREVER); + + if (message) { + handle_cb(message, arg); + bh_free_msg(message); + } + } +} + +void +bh_queue_exit_loop_run(bh_queue *queue) +{ + if (queue) { + queue->exit_loop_run = true; + bh_queue_cond_signal(&queue->queue_wait_cond); + } +} diff --git a/wasm-micro-runtime/core/shared/utils/bh_queue.h b/wasm-micro-runtime/core/shared/utils/bh_queue.h new file mode 100644 index 0000000..c15f435 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_queue.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_QUEUE_H +#define _BH_QUEUE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "bh_platform.h" + +struct bh_queue_node; +typedef struct bh_queue_node *bh_message_t; +struct bh_queue; +typedef struct bh_queue bh_queue; + +typedef void (*bh_queue_handle_msg_callback)(void *message, void *arg); + +#define bh_queue_malloc BH_MALLOC +#define bh_queue_free BH_FREE + +#define bh_queue_mutex korp_mutex +#define bh_queue_cond korp_cond + +#define bh_queue_mutex_init os_mutex_init +#define bh_queue_mutex_destroy os_mutex_destroy +#define bh_queue_mutex_lock os_mutex_lock +#define bh_queue_mutex_unlock os_mutex_unlock + +#define bh_queue_cond_init os_cond_init +#define bh_queue_cond_destroy os_cond_destroy +#define bh_queue_cond_wait os_cond_wait +#define bh_queue_cond_timedwait os_cond_reltimedwait +#define bh_queue_cond_signal os_cond_signal +#define bh_queue_cond_broadcast os_cond_broadcast + +typedef void (*bh_msg_cleaner)(void *msg); + +bh_queue * +bh_queue_create(void); + +void +bh_queue_destroy(bh_queue *queue); + +char * +bh_message_payload(bh_message_t message); +uint32 +bh_message_payload_len(bh_message_t message); +int +bh_message_type(bh_message_t message); + +bh_message_t +bh_new_msg(unsigned short tag, void *body, unsigned int len, void *handler); +void +bh_free_msg(bh_message_t msg); +bool +bh_post_msg(bh_queue *queue, unsigned short tag, void *body, unsigned int len); +bool +bh_post_msg2(bh_queue *queue, bh_message_t msg); + +bh_message_t +bh_get_msg(bh_queue *queue, uint64 timeout_us); + +unsigned +bh_queue_get_message_count(bh_queue *queue); + +void +bh_queue_enter_loop_run(bh_queue *queue, bh_queue_handle_msg_callback handle_cb, + void *arg); +void +bh_queue_exit_loop_run(bh_queue *queue); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _BH_QUEUE_H */ diff --git a/wasm-micro-runtime/core/shared/utils/bh_vector.c b/wasm-micro-runtime/core/shared/utils/bh_vector.c new file mode 100644 index 0000000..352ce71 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_vector.c @@ -0,0 +1,279 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_vector.h" + +static uint8 * +alloc_vector_data(size_t length, size_t size_elem) +{ + uint64 total_size = ((uint64)size_elem) * length; + uint8 *data; + + if (length > UINT32_MAX || size_elem > UINT32_MAX + || total_size > UINT32_MAX) { + return NULL; + } + + if ((data = BH_MALLOC((uint32)total_size))) { + memset(data, 0, (uint32)total_size); + } + + return data; +} + +/** + * every caller of `extend_vector` must provide + * a thread-safe environment. + */ +static bool +extend_vector(Vector *vector, size_t length) +{ + uint8 *data; + + if (length <= vector->max_elems) + return true; + + if (length < vector->size_elem * 3 / 2) + length = vector->size_elem * 3 / 2; + + if (!(data = alloc_vector_data(length, vector->size_elem))) { + return false; + } + + bh_memcpy_s(data, (uint32)(vector->size_elem * length), vector->data, + (uint32)(vector->size_elem * vector->max_elems)); + BH_FREE(vector->data); + + vector->data = data; + vector->max_elems = length; + return true; +} + +bool +bh_vector_init(Vector *vector, size_t init_length, size_t size_elem, + bool use_lock) +{ + if (!vector) { + LOG_ERROR("Init vector failed: vector is NULL.\n"); + return false; + } + + if (init_length == 0) { + init_length = 4; + } + + if (!(vector->data = alloc_vector_data(init_length, size_elem))) { + LOG_ERROR("Init vector failed: alloc memory failed.\n"); + return false; + } + + vector->size_elem = size_elem; + vector->max_elems = init_length; + vector->num_elems = 0; + vector->lock = NULL; + + if (use_lock) { + if (!(vector->lock = BH_MALLOC(sizeof(korp_mutex)))) { + LOG_ERROR("Init vector failed: alloc locker failed.\n"); + bh_vector_destroy(vector); + return false; + } + + if (BHT_OK != os_mutex_init(vector->lock)) { + LOG_ERROR("Init vector failed: init locker failed.\n"); + + BH_FREE(vector->lock); + vector->lock = NULL; + + bh_vector_destroy(vector); + return false; + } + } + + return true; +} + +bool +bh_vector_set(Vector *vector, uint32 index, const void *elem_buf) +{ + if (!vector || !elem_buf) { + LOG_ERROR("Set vector elem failed: vector or elem buf is NULL.\n"); + return false; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Set vector elem failed: invalid elem index.\n"); + return false; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + bh_memcpy_s(vector->data + vector->size_elem * index, + (uint32)vector->size_elem, elem_buf, (uint32)vector->size_elem); + if (vector->lock) + os_mutex_unlock(vector->lock); + return true; +} + +bool +bh_vector_get(Vector *vector, uint32 index, void *elem_buf) +{ + if (!vector || !elem_buf) { + LOG_ERROR("Get vector elem failed: vector or elem buf is NULL.\n"); + return false; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Get vector elem failed: invalid elem index.\n"); + return false; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + bh_memcpy_s(elem_buf, (uint32)vector->size_elem, + vector->data + vector->size_elem * index, + (uint32)vector->size_elem); + if (vector->lock) + os_mutex_unlock(vector->lock); + return true; +} + +bool +bh_vector_insert(Vector *vector, uint32 index, const void *elem_buf) +{ + size_t i; + uint8 *p; + bool ret = false; + + if (!vector || !elem_buf) { + LOG_ERROR("Insert vector elem failed: vector or elem buf is NULL.\n"); + goto just_return; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Insert vector elem failed: invalid elem index.\n"); + goto just_return; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + + if (!extend_vector(vector, vector->num_elems + 1)) { + LOG_ERROR("Insert vector elem failed: extend vector failed.\n"); + goto unlock_return; + } + + p = vector->data + vector->size_elem * vector->num_elems; + for (i = vector->num_elems - 1; i > index; i--) { + bh_memcpy_s(p, (uint32)vector->size_elem, p - vector->size_elem, + (uint32)vector->size_elem); + p -= vector->size_elem; + } + + bh_memcpy_s(p, (uint32)vector->size_elem, elem_buf, + (uint32)vector->size_elem); + vector->num_elems++; + ret = true; + +unlock_return: + if (vector->lock) + os_mutex_unlock(vector->lock); +just_return: + return ret; +} + +bool +bh_vector_append(Vector *vector, const void *elem_buf) +{ + bool ret = false; + + if (!vector || !elem_buf) { + LOG_ERROR("Append vector elem failed: vector or elem buf is NULL.\n"); + goto just_return; + } + + /* make sure one more slot is used by the thread who allocas it */ + if (vector->lock) + os_mutex_lock(vector->lock); + + if (!extend_vector(vector, vector->num_elems + 1)) { + LOG_ERROR("Append ector elem failed: extend vector failed.\n"); + goto unlock_return; + } + + bh_memcpy_s(vector->data + vector->size_elem * vector->num_elems, + (uint32)vector->size_elem, elem_buf, (uint32)vector->size_elem); + vector->num_elems++; + ret = true; + +unlock_return: + if (vector->lock) + os_mutex_unlock(vector->lock); +just_return: + return ret; +} + +bool +bh_vector_remove(Vector *vector, uint32 index, void *old_elem_buf) +{ + uint32 i; + uint8 *p; + + if (!vector) { + LOG_ERROR("Remove vector elem failed: vector is NULL.\n"); + return false; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Remove vector elem failed: invalid elem index.\n"); + return false; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + p = vector->data + vector->size_elem * index; + + if (old_elem_buf) { + bh_memcpy_s(old_elem_buf, (uint32)vector->size_elem, p, + (uint32)vector->size_elem); + } + + for (i = index; i < vector->num_elems - 1; i++) { + bh_memcpy_s(p, (uint32)vector->size_elem, p + vector->size_elem, + (uint32)vector->size_elem); + p += vector->size_elem; + } + + vector->num_elems--; + if (vector->lock) + os_mutex_unlock(vector->lock); + return true; +} + +size_t +bh_vector_size(const Vector *vector) +{ + return vector ? vector->num_elems : 0; +} + +bool +bh_vector_destroy(Vector *vector) +{ + if (!vector) { + LOG_ERROR("Destroy vector elem failed: vector is NULL.\n"); + return false; + } + + if (vector->data) + BH_FREE(vector->data); + + if (vector->lock) { + os_mutex_destroy(vector->lock); + BH_FREE(vector->lock); + } + + memset(vector, 0, sizeof(Vector)); + return true; +} diff --git a/wasm-micro-runtime/core/shared/utils/bh_vector.h b/wasm-micro-runtime/core/shared/utils/bh_vector.h new file mode 100644 index 0000000..d0aaaf1 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/bh_vector.h @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_VECTOR_H +#define _WASM_VECTOR_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define DEFAULT_VECTOR_INIT_SIZE 8 + +typedef struct Vector { + /* max element number */ + size_t max_elems; + /* vector data allocated */ + uint8 *data; + /* current element num */ + size_t num_elems; + /* size of each element */ + size_t size_elem; + void *lock; +} Vector; + +/** + * Initialize vector + * + * @param vector the vector to init + * @param init_length the initial length of the vector + * @param size_elem size of each element + * + * @return true if success, false otherwise + */ +bool +bh_vector_init(Vector *vector, size_t init_length, size_t size_elem, + bool use_lock); + +/** + * Set element of vector + * + * @param vector the vector to set + * @param index the index of the element to set + * @param elem_buf the element buffer which stores the element data + * + * @return true if success, false otherwise + */ +bool +bh_vector_set(Vector *vector, uint32 index, const void *elem_buf); + +/** + * Get element of vector + * + * @param vector the vector to get + * @param index the index of the element to get + * @param elem_buf the element buffer to store the element data, + * whose length must be no less than element size + * + * @return true if success, false otherwise + */ +bool +bh_vector_get(Vector *vector, uint32 index, void *elem_buf); + +/** + * Insert element of vector + * + * @param vector the vector to insert + * @param index the index of the element to insert + * @param elem_buf the element buffer which stores the element data + * + * @return true if success, false otherwise + */ +bool +bh_vector_insert(Vector *vector, uint32 index, const void *elem_buf); + +/** + * Append element to the end of vector + * + * @param vector the vector to append + * @param elem_buf the element buffer which stores the element data + * + * @return true if success, false otherwise + */ +bool +bh_vector_append(Vector *vector, const void *elem_buf); + +/** + * Remove element from vector + * + * @param vector the vector to remove element + * @param index the index of the element to remove + * @param old_elem_buf if not NULL, copies the element data to the buffer + * + * @return true if success, false otherwise + */ +bool +bh_vector_remove(Vector *vector, uint32 index, void *old_elem_buf); + +/** + * Return the size of the vector + * + * @param vector the vector to get size + * + * @return return the size of the vector + */ +size_t +bh_vector_size(const Vector *vector); + +/** + * Destroy the vector + * + * @param vector the vector to destroy + * + * @return true if success, false otherwise + */ +bool +bh_vector_destroy(Vector *vector); + +#ifdef __cplusplus +} +#endif + +#endif /* endof _WASM_VECTOR_H */ diff --git a/wasm-micro-runtime/core/shared/utils/gnuc.h b/wasm-micro-runtime/core/shared/utils/gnuc.h new file mode 100644 index 0000000..70000ae --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/gnuc.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#if !defined(__GNUC_PREREQ) && (defined(__GNUC__) || defined(__GNUG__)) \ + && !defined(__clang__) && defined(__GNUC_MINOR__) +/* Depending on the platform the macro is defined in sys/features.h or + features.h Given the macro is simple, we re-implement it here instead of + dealing with two different paths. + */ +#define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#endif diff --git a/wasm-micro-runtime/core/shared/utils/runtime_timer.c b/wasm-micro-runtime/core/shared/utils/runtime_timer.c new file mode 100644 index 0000000..9d390c2 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/runtime_timer.c @@ -0,0 +1,469 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "runtime_timer.h" + +#if 1 +#define PRINT(...) (void)0 +#else +#define PRINT printf +#endif + +typedef struct _app_timer { + struct _app_timer *next; + uint32 id; + uint32 interval; + uint64 expiry; + bool is_periodic; +} app_timer_t; + +struct _timer_ctx { + app_timer_t *app_timers; + app_timer_t *idle_timers; + app_timer_t *free_timers; + uint32 max_timer_id; + int pre_allocated; + uint32 owner; + + /* mutex and condition */ + korp_cond cond; + korp_mutex mutex; + + timer_callback_f timer_callback; + check_timer_expiry_f refresh_checker; +}; + +uint64 +bh_get_tick_ms() +{ + return os_time_get_boot_us() / 1000; +} + +uint32 +bh_get_elpased_ms(uint32 *last_system_clock) +{ + uint32 elpased_ms; + /* attention: the bh_get_tick_ms() return 64 bits integer, but + the bh_get_elpased_ms() is designed to use 32 bits clock count */ + uint32 now = (uint32)bh_get_tick_ms(); + + /* system clock overrun */ + if (now < *last_system_clock) { + PRINT("system clock overrun!\n"); + elpased_ms = now + (UINT32_MAX - *last_system_clock) + 1; + } + else { + elpased_ms = now - *last_system_clock; + } + + *last_system_clock = now; + return elpased_ms; +} + +static app_timer_t * +remove_timer_from(timer_ctx_t ctx, uint32 timer_id, bool active_list) +{ + app_timer_t **head, *prev, *t; + + os_mutex_lock(&ctx->mutex); + + if (active_list) + head = &ctx->app_timers; + else + head = &ctx->idle_timers; + + t = *head; + prev = NULL; + + while (t) { + if (t->id == timer_id) { + if (prev == NULL) { + *head = t->next; + PRINT("removed timer [%d] at head from list %d\n", t->id, + active_list); + } + else { + prev->next = t->next; + PRINT("removed timer [%d] after [%d] from list %d\n", t->id, + prev->id, active_list); + } + os_mutex_unlock(&ctx->mutex); + + if (active_list && prev == NULL && ctx->refresh_checker) + ctx->refresh_checker(ctx); + return t; + } + else { + prev = t; + t = t->next; + } + } + + os_mutex_unlock(&ctx->mutex); + return NULL; +} + +static app_timer_t * +remove_timer(timer_ctx_t ctx, uint32 timer_id, bool *active) +{ + app_timer_t *t = remove_timer_from(ctx, timer_id, true); + + if (t) { + if (active) + *active = true; + return t; + } + + if (active) + *active = false; + return remove_timer_from(ctx, timer_id, false); +} + +static void +reschedule_timer(timer_ctx_t ctx, app_timer_t *timer) +{ + app_timer_t *t; + app_timer_t *prev = NULL; + + os_mutex_lock(&ctx->mutex); + + t = ctx->app_timers; + timer->next = NULL; + timer->expiry = bh_get_tick_ms() + timer->interval; + + while (t) { + if (timer->expiry < t->expiry) { + if (prev == NULL) { + timer->next = ctx->app_timers; + ctx->app_timers = timer; + PRINT("rescheduled timer [%d] at head\n", timer->id); + } + else { + timer->next = t; + prev->next = timer; + PRINT("rescheduled timer [%d] after [%d]\n", timer->id, + prev->id); + } + + goto out; + } + else { + prev = t; + t = t->next; + } + } + + if (prev) { + /* insert to the list end */ + prev->next = timer; + PRINT("rescheduled timer [%d] at end, after [%d]\n", timer->id, + prev->id); + } + else { + /* insert at the begin */ + bh_assert(ctx->app_timers == NULL); + ctx->app_timers = timer; + PRINT("rescheduled timer [%d] as first\n", timer->id); + } + +out: + os_mutex_unlock(&ctx->mutex); + + /* ensure the refresh_checker() is called out of the lock */ + if (prev == NULL && ctx->refresh_checker) + ctx->refresh_checker(ctx); +} + +static void +release_timer(timer_ctx_t ctx, app_timer_t *t) +{ + if (ctx->pre_allocated) { + os_mutex_lock(&ctx->mutex); + t->next = ctx->free_timers; + ctx->free_timers = t; + PRINT("recycle timer :%d\n", t->id); + os_mutex_unlock(&ctx->mutex); + } + else { + PRINT("destroy timer :%d\n", t->id); + BH_FREE(t); + } +} + +void +release_timer_list(app_timer_t **p_list) +{ + app_timer_t *t = *p_list; + + while (t) { + app_timer_t *next = t->next; + PRINT("destroy timer list:%d\n", t->id); + BH_FREE(t); + t = next; + } + + *p_list = NULL; +} + +/* + * API exposed + */ + +timer_ctx_t +create_timer_ctx(timer_callback_f timer_handler, + check_timer_expiry_f expiery_checker, int prealloc_num, + unsigned int owner) +{ + timer_ctx_t ctx = (timer_ctx_t)BH_MALLOC(sizeof(struct _timer_ctx)); + + if (ctx == NULL) + return NULL; + + memset(ctx, 0, sizeof(struct _timer_ctx)); + + ctx->timer_callback = timer_handler; + ctx->pre_allocated = prealloc_num; + ctx->refresh_checker = expiery_checker; + ctx->owner = owner; + + while (prealloc_num > 0) { + app_timer_t *timer = (app_timer_t *)BH_MALLOC(sizeof(app_timer_t)); + + if (timer == NULL) + goto cleanup; + + memset(timer, 0, sizeof(*timer)); + timer->next = ctx->free_timers; + ctx->free_timers = timer; + prealloc_num--; + } + + if (os_cond_init(&ctx->cond) != 0) + goto cleanup; + + if (os_mutex_init(&ctx->mutex) != 0) { + os_cond_destroy(&ctx->cond); + goto cleanup; + } + + PRINT("timer ctx created. pre-alloc: %d\n", ctx->pre_allocated); + return ctx; + +cleanup: + if (ctx) { + release_timer_list(&ctx->free_timers); + BH_FREE(ctx); + } + PRINT("timer ctx create failed\n"); + return NULL; +} + +void +destroy_timer_ctx(timer_ctx_t ctx) +{ + while (ctx->free_timers) { + void *tmp = ctx->free_timers; + ctx->free_timers = ctx->free_timers->next; + BH_FREE(tmp); + } + + cleanup_app_timers(ctx); + + os_cond_destroy(&ctx->cond); + os_mutex_destroy(&ctx->mutex); + BH_FREE(ctx); +} + +unsigned int +timer_ctx_get_owner(timer_ctx_t ctx) +{ + return ctx->owner; +} + +void +add_idle_timer(timer_ctx_t ctx, app_timer_t *timer) +{ + os_mutex_lock(&ctx->mutex); + timer->next = ctx->idle_timers; + ctx->idle_timers = timer; + os_mutex_unlock(&ctx->mutex); +} + +uint32 +sys_create_timer(timer_ctx_t ctx, int interval, bool is_period, bool auto_start) +{ + app_timer_t *timer; + + if (ctx->pre_allocated) { + if (ctx->free_timers == NULL) { + return (uint32)-1; + } + else { + timer = ctx->free_timers; + ctx->free_timers = timer->next; + } + } + else { + timer = (app_timer_t *)BH_MALLOC(sizeof(app_timer_t)); + if (timer == NULL) + return (uint32)-1; + } + + memset(timer, 0, sizeof(*timer)); + + ctx->max_timer_id++; + if (ctx->max_timer_id == (uint32)-1) + ctx->max_timer_id++; + timer->id = ctx->max_timer_id; + timer->interval = (uint32)interval; + timer->is_periodic = is_period; + + if (auto_start) + reschedule_timer(ctx, timer); + else + add_idle_timer(ctx, timer); + + return timer->id; +} + +bool +sys_timer_cancel(timer_ctx_t ctx, uint32 timer_id) +{ + bool from_active; + app_timer_t *t = remove_timer(ctx, timer_id, &from_active); + + if (t == NULL) + return false; + + add_idle_timer(ctx, t); + + PRINT("sys_timer_stop called\n"); + return from_active; +} + +bool +sys_timer_destroy(timer_ctx_t ctx, uint32 timer_id) +{ + bool from_active; + app_timer_t *t = remove_timer(ctx, timer_id, &from_active); + + if (t == NULL) + return false; + + release_timer(ctx, t); + + PRINT("sys_timer_destroy called\n"); + return true; +} + +bool +sys_timer_restart(timer_ctx_t ctx, uint32 timer_id, int interval) +{ + app_timer_t *t = remove_timer(ctx, timer_id, NULL); + + if (t == NULL) + return false; + + t->interval = (uint32)interval; + + reschedule_timer(ctx, t); + + PRINT("sys_timer_restart called\n"); + return true; +} + +/* + * API called by the timer manager from another thread or the kernel timer + * handler + */ + +/** + * lookup the app queue by the module name + * post a timeout message to the app queue + */ +static void +handle_expired_timers(timer_ctx_t ctx, app_timer_t *expired) +{ + while (expired) { + app_timer_t *t = expired; + ctx->timer_callback(t->id, ctx->owner); + + /* get next expired timer first, since the following + operation may change expired->next */ + expired = expired->next; + if (t->is_periodic) { + /* if it is repeating, then reschedule it */ + reschedule_timer(ctx, t); + } + else { + /* else move it to idle list */ + add_idle_timer(ctx, t); + } + } +} + +uint32 +get_expiry_ms(timer_ctx_t ctx) +{ + uint32 ms_to_next_expiry; + uint64 now = bh_get_tick_ms(); + + os_mutex_lock(&ctx->mutex); + if (ctx->app_timers == NULL) + ms_to_next_expiry = (uint32)-1; + else if (ctx->app_timers->expiry >= now) + ms_to_next_expiry = (uint32)(ctx->app_timers->expiry - now); + else + ms_to_next_expiry = 0; + os_mutex_unlock(&ctx->mutex); + + return ms_to_next_expiry; +} + +uint32 +check_app_timers(timer_ctx_t ctx) +{ + app_timer_t *t, *expired = NULL, *expired_end = NULL; + uint64 now = bh_get_tick_ms(); + + os_mutex_lock(&ctx->mutex); + + t = ctx->app_timers; + while (t) { + if (now >= t->expiry) { + ctx->app_timers = t->next; + + /* append t to the end of expired list */ + t->next = NULL; + if (!expired_end) { + expired = expired_end = t; + } + else { + expired_end->next = t; + expired_end = t; + } + + t = ctx->app_timers; + } + else { + break; + } + } + os_mutex_unlock(&ctx->mutex); + + handle_expired_timers(ctx, expired); + return get_expiry_ms(ctx); +} + +void +cleanup_app_timers(timer_ctx_t ctx) +{ + os_mutex_lock(&ctx->mutex); + + release_timer_list(&ctx->app_timers); + release_timer_list(&ctx->idle_timers); + + os_mutex_unlock(&ctx->mutex); +} diff --git a/wasm-micro-runtime/core/shared/utils/runtime_timer.h b/wasm-micro-runtime/core/shared/utils/runtime_timer.h new file mode 100644 index 0000000..b8d90c5 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/runtime_timer.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef LIB_BASE_RUNTIME_TIMER_H_ +#define LIB_BASE_RUNTIME_TIMER_H_ + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +uint64 +bh_get_tick_ms(void); +uint32 +bh_get_elpased_ms(uint32 *last_system_clock); + +struct _timer_ctx; +typedef struct _timer_ctx *timer_ctx_t; +typedef void (*timer_callback_f)(unsigned int id, unsigned int owner); +typedef void (*check_timer_expiry_f)(timer_ctx_t ctx); + +timer_ctx_t +create_timer_ctx(timer_callback_f timer_handler, check_timer_expiry_f, + int prealloc_num, unsigned int owner); +void destroy_timer_ctx(timer_ctx_t); +unsigned int +timer_ctx_get_owner(timer_ctx_t ctx); + +uint32 +sys_create_timer(timer_ctx_t ctx, int interval, bool is_period, + bool auto_start); +bool +sys_timer_destroy(timer_ctx_t ctx, uint32 timer_id); +bool +sys_timer_cancel(timer_ctx_t ctx, uint32 timer_id); +bool +sys_timer_restart(timer_ctx_t ctx, uint32 timer_id, int interval); +void +cleanup_app_timers(timer_ctx_t ctx); +uint32 +check_app_timers(timer_ctx_t ctx); +uint32 +get_expiry_ms(timer_ctx_t ctx); + +#ifdef __cplusplus +} +#endif +#endif /* LIB_BASE_RUNTIME_TIMER_H_ */ diff --git a/wasm-micro-runtime/core/shared/utils/shared_utils.cmake b/wasm-micro-runtime/core/shared/utils/shared_utils.cmake new file mode 100644 index 0000000..5b7d02d --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/shared_utils.cmake @@ -0,0 +1,12 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (UTILS_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include_directories(${UTILS_SHARED_DIR}) + +file (GLOB source_all ${UTILS_SHARED_DIR}/*.c) + +set (UTILS_SHARED_SOURCE ${source_all}) + +LIST (APPEND RUNTIME_LIB_HEADER_LIST "${UTILS_SHARED_DIR}/runtime_timer.h") diff --git a/wasm-micro-runtime/core/shared/utils/uncommon/SConscript b/wasm-micro-runtime/core/shared/utils/uncommon/SConscript new file mode 100644 index 0000000..f608645 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/uncommon/SConscript @@ -0,0 +1,32 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * +import os + +cwd = GetCurrentDir() + +# src = Split(''' +# ''') + + +def addSrcFiles(arr, path): + for f in os.listdir(path): + fpath = os.path.join(path, f); + if os.path.isfile(fpath): + ext = os.path.splitext(fpath)[-1] + if ext == '.c' or ext == '.cpp': + arr += [fpath] + #elif os.path.isdir(fpath): + # addSrcFiles(arr, fpath) + +src = Glob('*.c') +src += Glob('*.cpp') +CPPPATH = [cwd] + +group = DefineGroup('iwasm_shared_utils_uncommon', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/wasm-micro-runtime/core/shared/utils/uncommon/bh_getopt.c b/wasm-micro-runtime/core/shared/utils/uncommon/bh_getopt.c new file mode 100644 index 0000000..19e23a7 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/uncommon/bh_getopt.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2020 Ant Financial Services Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __GNUC__ + +#include "bh_getopt.h" +#include +#include + +char *optarg = NULL; +int optind = 1; + +int +getopt(int argc, char *const argv[], const char *optstring) +{ + static int sp = 1; + int opt; + char *p; + + if (sp == 1) { + if ((optind >= argc) || (argv[optind][0] != '-') + || (argv[optind][1] == 0)) { + return -1; + } + else if (!strcmp(argv[optind], "--")) { + optind++; + return -1; + } + } + + opt = argv[optind][sp]; + p = strchr(optstring, opt); + if (opt == ':' || p == NULL) { + printf("illegal option : '-%c'\n", opt); + if (argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return ('?'); + } + if (p[1] == ':') { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) { + printf("option '-%c' requires an argument :\n", opt); + sp = 1; + return ('?'); + } + else { + optarg = argv[optind++]; + } + sp = 1; + } + else { + if (argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return (opt); +} +#endif diff --git a/wasm-micro-runtime/core/shared/utils/uncommon/bh_getopt.h b/wasm-micro-runtime/core/shared/utils/uncommon/bh_getopt.h new file mode 100644 index 0000000..efd3ab4 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/uncommon/bh_getopt.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2020 Ant Financial Services Group. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifdef __GNUC__ +#include +#endif +#ifndef __GNUC__ +#ifndef GETOPT_H__ +#define GETOPT_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +extern char *optarg; +extern int optind; + +int +getopt(int argc, char *const argv[], const char *optstring); + +#ifdef __cplusplus +} +#endif + +#endif /* end of GETOPT_H__ */ +#endif /* end of __GNUC__ */ diff --git a/wasm-micro-runtime/core/shared/utils/uncommon/bh_read_file.c b/wasm-micro-runtime/core/shared/utils/uncommon/bh_read_file.c new file mode 100644 index 0000000..5ddf1b6 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/uncommon/bh_read_file.c @@ -0,0 +1,117 @@ +#include "bh_read_file.h" + +#include +#include +#if defined(_WIN32) || defined(_WIN32_) +#include +#else +#include +#endif + +#if defined(_WIN32) || defined(_WIN32_) + +#if defined(__MINGW32__) && !defined(_SH_DENYNO) +#define _SH_DENYNO 0x40 +#endif + +char * +bh_read_file_to_buffer(const char *filename, uint32 *ret_size) +{ + char *buffer; + int file; + uint32 file_size, buf_size, read_size; + struct stat stat_buf; + + if (!filename || !ret_size) { + printf("Read file to buffer failed: invalid filename or ret size.\n"); + return NULL; + } + + if (_sopen_s(&file, filename, _O_RDONLY | _O_BINARY, _SH_DENYNO, 0)) { + printf("Read file to buffer failed: open file %s failed.\n", filename); + return NULL; + } + + if (fstat(file, &stat_buf) != 0) { + printf("Read file to buffer failed: fstat file %s failed.\n", filename); + _close(file); + return NULL; + } + file_size = (uint32)stat_buf.st_size; + + /* At lease alloc 1 byte to avoid malloc failed */ + buf_size = file_size > 0 ? file_size : 1; + + if (!(buffer = (char *)BH_MALLOC(buf_size))) { + printf("Read file to buffer failed: alloc memory failed.\n"); + _close(file); + return NULL; + } +#if WASM_ENABLE_MEMORY_TRACING != 0 + printf("Read file, total size: %u\n", file_size); +#endif + + read_size = _read(file, buffer, file_size); + _close(file); + + if (read_size < file_size) { + printf("Read file to buffer failed: read file content failed.\n"); + BH_FREE(buffer); + return NULL; + } + + *ret_size = file_size; + return buffer; +} +#else /* else of defined(_WIN32) || defined(_WIN32_) */ +char * +bh_read_file_to_buffer(const char *filename, uint32 *ret_size) +{ + char *buffer; + int file; + uint32 file_size, buf_size, read_size; + struct stat stat_buf; + + if (!filename || !ret_size) { + printf("Read file to buffer failed: invalid filename or ret size.\n"); + return NULL; + } + + if ((file = open(filename, O_RDONLY, 0)) == -1) { + printf("Read file to buffer failed: open file %s failed.\n", filename); + return NULL; + } + + if (fstat(file, &stat_buf) != 0) { + printf("Read file to buffer failed: fstat file %s failed.\n", filename); + close(file); + return NULL; + } + + file_size = (uint32)stat_buf.st_size; + + /* At lease alloc 1 byte to avoid malloc failed */ + buf_size = file_size > 0 ? file_size : 1; + + if (!(buffer = BH_MALLOC(buf_size))) { + printf("Read file to buffer failed: alloc memory failed.\n"); + close(file); + return NULL; + } +#if WASM_ENABLE_MEMORY_TRACING != 0 + printf("Read file, total size: %u\n", file_size); +#endif + + read_size = (uint32)read(file, buffer, file_size); + close(file); + + if (read_size < file_size) { + printf("Read file to buffer failed: read file content failed.\n"); + BH_FREE(buffer); + return NULL; + } + + *ret_size = file_size; + return buffer; +} +#endif /* end of defined(_WIN32) || defined(_WIN32_) */ diff --git a/wasm-micro-runtime/core/shared/utils/uncommon/bh_read_file.h b/wasm-micro-runtime/core/shared/utils/uncommon/bh_read_file.h new file mode 100644 index 0000000..bbebf84 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/uncommon/bh_read_file.h @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_FILE_H +#define _BH_FILE_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +char * +bh_read_file_to_buffer(const char *filename, uint32 *ret_size); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _BH_FILE_H */ diff --git a/wasm-micro-runtime/core/shared/utils/uncommon/shared_uncommon.cmake b/wasm-micro-runtime/core/shared/utils/uncommon/shared_uncommon.cmake new file mode 100644 index 0000000..0a15b87 --- /dev/null +++ b/wasm-micro-runtime/core/shared/utils/uncommon/shared_uncommon.cmake @@ -0,0 +1,11 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (UNCOMMON_SHARED_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include_directories(${UNCOMMON_SHARED_DIR}) + +file (GLOB_RECURSE source_all ${UNCOMMON_SHARED_DIR}/*.c) + +set (UNCOMMON_SHARED_SOURCE ${source_all}) + diff --git a/wasm-micro-runtime/core/version.h b/wasm-micro-runtime/core/version.h new file mode 100644 index 0000000..de3f56a --- /dev/null +++ b/wasm-micro-runtime/core/version.h @@ -0,0 +1,11 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_VERSION_H_ +#define _WAMR_VERSION_H_ +#define WAMR_VERSION_MAJOR 2 +#define WAMR_VERSION_MINOR 0 +#define WAMR_VERSION_PATCH 0 +#endif diff --git a/wasm-micro-runtime/doc/build_wamr.md b/wasm-micro-runtime/doc/build_wamr.md new file mode 100644 index 0000000..5598ea3 --- /dev/null +++ b/wasm-micro-runtime/doc/build_wamr.md @@ -0,0 +1,315 @@ + +# Build WAMR vmcore + +WAMR vmcore is a set of runtime libraries for loading and running Wasm modules. This document introduces how to build the WAMR vmcore. + +References: +- [how to build iwasm](../product-mini/README.md): building different target platforms such as Linux, Windows, Mac etc +- [Blog: Introduction to WAMR running modes](https://bytecodealliance.github.io/wamr.dev/blog/introduction-to-wamr-running-modes/) + + +## WAMR vmcore cmake building configurations + +By including the script `runtime_lib.cmake` under folder [build-scripts](../build-scripts) in CMakeList.txt, it is easy to use vmcore to build host software with cmake. + +```cmake +# add this into your CMakeList.txt +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +``` + +The script `runtime_lib.cmake` defines a number of variables for configuring the WAMR runtime features. You can set these variables in your CMakeList.txt or pass the configurations from cmake command line. + +#### **Configure platform and architecture** + +- **WAMR_BUILD_PLATFORM**: set the target platform. It can be set to any platform name (folder name) under folder [core/shared/platform](../core/shared/platform). + +- **WAMR_BUILD_TARGET**: set the target CPU architecture. Current supported targets are: X86_64, X86_32, AARCH64, ARM, THUMB, XTENSA, ARC, RISCV32, RISCV64 and MIPS. + - For ARM and THUMB, the format is \\[\]\[_VFP], where \ is the ARM sub-architecture and the "_VFP" suffix means using VFP coprocessor registers s0-s15 (d0-d7) for passing arguments or returning results in standard procedure-call. Both \ and "_VFP" are optional, e.g. ARMV7, ARMV7_VFP, THUMBV7, THUMBV7_VFP and so on. + - For AARCH64, the format is\[\], VFP is enabled by default. \ is optional, e.g. AARCH64, AARCH64V8, AARCH64V8.1 and so on. + - For RISCV64, the format is \[_abi], where "_abi" is optional, currently the supported formats are RISCV64, RISCV64_LP64D and RISCV64_LP64: RISCV64 and RISCV64_LP64D are identical, using [LP64D](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (LP64 with hardware floating-point calling convention for FLEN=64). And RISCV64_LP64 uses [LP64](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (Integer calling-convention only, and hardware floating-point calling convention is not used). + - For RISCV32, the format is \[_abi], where "_abi" is optional, currently the supported formats are RISCV32, RISCV32_ILP32D and RISCV32_ILP32: RISCV32 and RISCV32_ILP32D are identical, using [ILP32D](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (ILP32 with hardware floating-point calling convention for FLEN=64). And RISCV32_ILP32 uses [ILP32](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (Integer calling-convention only, and hardware floating-point calling convention is not used). + +```bash +cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM +``` + +#### **Configure interpreters** + +- **WAMR_BUILD_INTERP**=1/0: enable or disable WASM interpreter + +- **WAMR_BUILD_FAST_INTERP**=1/0: build fast (default) or classic WASM interpreter. + + NOTE: the fast interpreter runs ~2X faster than classic interpreter, but consumes about 2X memory to hold the pre-compiled code. + +#### **Configure AOT and JITs** + +- **WAMR_BUILD_AOT**=1/0, enable AOT or not, default to enable if not set +- **WAMR_BUILD_JIT**=1/0, enable LLVM JIT or not, default to disable if not set +- **WAMR_BUILD_FAST_JIT**=1/0, enable Fast JIT or not, default to disable if not set +- **WAMR_BUILD_FAST_JIT**=1 and **WAMR_BUILD_JIT**=1, enable Multi-tier JIT, default to disable if not set + +#### **Configure LIBC** + +- **WAMR_BUILD_LIBC_BUILTIN**=1/0, build the built-in libc subset for WASM app, default to enable if not set + +- **WAMR_BUILD_LIBC_WASI**=1/0, build the [WASI](https://github.com/WebAssembly/WASI) libc subset for WASM app, default to enable if not set + +- **WAMR_BUILD_LIBC_UVWASI**=1/0 (Experiment), build the [WASI](https://github.com/WebAssembly/WASI) libc subset for WASM app based on [uvwasi](https://github.com/nodejs/uvwasi) implementation, default to disable if not set + +> Note: for platform which doesn't support **WAMR_BUILD_LIBC_WASI**, e.g. Windows, developer can try using **WAMR_BUILD_LIBC_UVWASI**. + +#### **Enable Multi-Module feature** + +- **WAMR_BUILD_MULTI_MODULE**=1/0, default to disable if not set +> Note: See [Multiple Modules as Dependencies](./multi_module.md) for more details. + +#### **Enable WASM mini loader** + +- **WAMR_BUILD_MINI_LOADER**=1/0, default to disable if not set + +> Note: the mini loader doesn't check the integrity of the WASM binary file, developer must ensure that the WASM file is well-formed. + +#### **Enable shared memory feature** +- **WAMR_BUILD_SHARED_MEMORY**=1/0, default to disable if not set + +#### **Enable bulk memory feature** +- **WAMR_BUILD_BULK_MEMORY**=1/0, default to disable if not set + +#### **Enable memory64 feature** +- **WAMR_BUILD_MEMORY64**=1/0, default to disable if not set + +> Note: Currently, the memory64 feature is only supported in classic interpreter running mode. + +#### **Enable thread manager** +- **WAMR_BUILD_THREAD_MGR**=1/0, default to disable if not set + +#### **Enable lib-pthread** +- **WAMR_BUILD_LIB_PTHREAD**=1/0, default to disable if not set +> Note: The dependent feature of lib pthread such as the `shared memory` and `thread manager` will be enabled automatically. + +> See [WAMR pthread library](./pthread_library.md) for more details. + +#### **Enable lib-pthread-semaphore** +- **WAMR_BUILD_LIB_PTHREAD_SEMAPHORE**=1/0, default to disable if not set +> Note: This feature depends on `lib-pthread`, it will be enabled automatically if this feature is enabled. + +#### **Enable lib wasi-threads** +- **WAMR_BUILD_LIB_WASI_THREADS**=1/0, default to disable if not set +> Note: The dependent feature of lib wasi-threads such as the `shared memory` and `thread manager` will be enabled automatically. + +> See [wasi-threads](./pthread_impls.md#wasi-threads-new) and [Introduction to WAMR WASI threads](https://bytecodealliance.github.io/wamr.dev/blog/introduction-to-wamr-wasi-threads) for more details. + +#### **Enable lib wasi-nn** +- **WAMR_BUILD_WASI_NN**=1/0, default to disable if not set +> Note: See [WASI-NN](../core/iwasm/libraries/wasi-nn) for more details. + +#### **Enable lib wasi-nn GPU mode** +- **WAMR_BUILD_WASI_NN_ENABLE_GPU**=1/0, default to disable if not set + +#### **Enable lib wasi-nn external delegate mode** +- **WAMR_BUILD_WASI_NN_ENABLE_EXTERNAL_DELEGATE**=1/0, default to disable if not set + +- **WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH**=Path to the external delegate shared library (e.g. `libedgetpu.so.1.0` for Coral USB) + +#### **Enable lib wasi-nn with `wasi_ephemeral_nn` module support** +- **WAMR_BUILD_WASI_EPHEMERAL_NN**=1/0, default to disable if not set + +#### **Disable boundary check with hardware trap** +- **WAMR_DISABLE_HW_BOUND_CHECK**=1/0, default to enable if not set and supported by platform +> Note: by default only platform [linux/darwin/android/windows/vxworks 64-bit](https://github.com/bytecodealliance/wasm-micro-runtime/blob/5fb5119239220b0803e7045ca49b0a29fe65e70e/core/shared/platform/linux/platform_internal.h#L81) will enable the boundary check with hardware trap feature, for 32-bit platforms it's automatically disabled even when the flag is set to 0, and the wamrc tool will generate AOT code without boundary check instructions in all 64-bit targets except SGX to improve performance. The boundary check includes linear memory access boundary and native stack access boundary, if `WAMR_DISABLE_STACK_HW_BOUND_CHECK` below isn't set. + +#### **Disable native stack boundary check with hardware trap** +- **WAMR_DISABLE_STACK_HW_BOUND_CHECK**=1/0, default to enable if not set and supported by platform, same as `WAMR_DISABLE_HW_BOUND_CHECK`. +> Note: When boundary check with hardware trap is disabled, or `WAMR_DISABLE_HW_BOUND_CHECK` is set to 1, the native stack boundary check with hardware trap will be disabled too, no matter what value is set to `WAMR_DISABLE_STACK_HW_BOUND_CHECK`. And when boundary check with hardware trap is enabled, the status of this feature is set according to the value of `WAMR_DISABLE_STACK_HW_BOUND_CHECK`. + +#### **Disable async wakeup of blocking operation** +- **WAMR_DISABLE_WAKEUP_BLOCKING_OP**=1/0, default to enable if supported by the platform +> Note: The feature helps async termination of blocking threads. If you disable it, the runtime can wait for termination of blocking threads possibly forever. + +#### **Enable tail call feature** +- **WAMR_BUILD_TAIL_CALL**=1/0, default to disable if not set + +#### **Enable 128-bit SIMD feature** +- **WAMR_BUILD_SIMD**=1/0, default to enable if not set +> Note: only supported in AOT mode x86-64 target. + +#### **Enable Exception Handling** +- **WAMR_BUILD_EXCE_HANDLING**=1/0, default to disable if not set + +> Note: Currently, the exception handling feature is only supported in classic interpreter running mode. + +#### **Enable Garbage Collection** +- **WAMR_BUILD_GC**=1/0, default to disable if not set + +#### **Configure Debug** + +- **WAMR_BUILD_CUSTOM_NAME_SECTION**=1/0, load the function name from custom name section, default to disable if not set + +#### **Enable AOT stack frame feature** +- **WAMR_BUILD_AOT_STACK_FRAME**=1/0, default to disable if not set +> Note: if it is enabled, the AOT or JIT stack frames (like stack frame of classic interpreter but only necessary data is committed) will be created for AOT or JIT mode in function calls. And please add `--enable-dump-call-stack` option to wamrc during compiling AOT module. + +#### **Enable dump call stack feature** +- **WAMR_BUILD_DUMP_CALL_STACK**=1/0, default to disable if not set + +> Note: if it is enabled, the call stack will be dumped when exception occurs. + +> - For interpreter mode, the function names are firstly extracted from *custom name section*, if this section doesn't exist or the feature is not enabled, then the name will be extracted from the import/export sections +> - For AOT/JIT mode, the function names are extracted from import/export section, please export as many functions as possible (for `wasi-sdk` you can use `-Wl,--export-all`) when compiling wasm module, and add `--enable-dump-call-stack --emit-custom-sections=name` option to wamrc during compiling AOT module. + +#### **Enable memory profiling (Experiment)** +- **WAMR_BUILD_MEMORY_PROFILING**=1/0, default to disable if not set +> Note: if it is enabled, developer can use API `void wasm_runtime_dump_mem_consumption(wasm_exec_env_t exec_env)` to dump the memory consumption info. +Currently we only profile the memory consumption of module, module_instance and exec_env, the memory consumed by other components such as `wasi-ctx`, `multi-module` and `thread-manager` are not included. + +> Also refer to [Memory usage estimation for a module](./memory_usage.md). + +#### **Enable performance profiling (Experiment)** +- **WAMR_BUILD_PERF_PROFILING**=1/0, default to disable if not set +> Note: if it is enabled, developer can use API `void wasm_runtime_dump_perf_profiling(wasm_module_inst_t module_inst)` to dump the performance consumption info. Currently we only profile the performance consumption of each WASM function. + +> The function name searching sequence is the same with dump call stack feature. + +> Also refer to [Tune the performance of running wasm/aot file](./perf_tune.md). + +#### **Enable the global heap** +- **WAMR_BUILD_GLOBAL_HEAP_POOL**=1/0, default to disable if not set for all *iwasm* applications, except for the platforms Alios and Zephyr. + +> **WAMR_BUILD_GLOBAL_HEAP_POOL** is used in the *iwasm* applications provided in the directory `product-mini`. When writing your own host application using WAMR, if you want to use a global heap and allocate memory from it, you must set the initialization argument `mem_alloc_type` to `Alloc_With_Pool`. +> The global heap is defined in the documentation [Memory model and memory usage tunning](memory_tune.md). + +#### **Set the global heap size** +- **WAMR_BUILD_GLOBAL_HEAP_SIZE**=n, default to 10 MB (10485760) if not set for all *iwasm* applications, except for the platforms Alios (256 kB), Riot (256 kB) and Zephyr (128 kB). + +> **WAMR_BUILD_GLOBAL_HEAP_SIZE** is used in the *iwasm* applications provided in the directory `product-mini`. When writing your own host application using WAMR, if you want to set the amount of memory dedicated to the global heap pool, you must set the initialization argument `mem_alloc_option.pool` with the appropriate values. +> The global heap is defined in the documentation [Memory model and memory usage tunning](memory_tune.md). +> Note: if `WAMR_BUILD_GLOBAL_HEAP_SIZE` is not set and the flag `WAMR_BUILD_SPEC_TEST` is set, the global heap size is equal to 300 MB (314572800), or 100 MB (104857600) when compiled for Intel SGX (Linux). + +#### **Set maximum app thread stack size** +- **WAMR_APP_THREAD_STACK_SIZE_MAX**=n, default to 8 MB (8388608) if not set +> Note: the AOT boundary check with hardware trap mechanism might consume large stack since the OS may lazily grow the stack mapping as a guard page is hit, we may use this configuration to reduce the total stack usage, e.g. -DWAMR_APP_THREAD_STACK_SIZE_MAX=131072 (128 KB). + +#### **Set vprintf callback** +- **WAMR_BH_VPRINTF**=, default to disable if not set +> Note: if the vprintf_callback function is provided by developer, the os_printf() and os_vprintf() in Linux, Darwin, Windows, VxWorks, Android and esp-idf platforms, besides WASI Libc output will call the callback function instead of libc vprintf() function to redirect the stdout output. For example, developer can define the callback function like below outside runtime lib: +> +> ```C +> int my_vprintf(const char *format, va_list ap) +> { +> /* output to pre-opened file stream */ +> FILE *my_file = ...; +> return vfprintf(my_file, format, ap); +> /* or output to pre-opened file descriptor */ +> int my_fd = ...; +> return vdprintf(my_fd, format, ap); +> /* or output to string buffer and print the string */ +> char buf[128]; +> vsnprintf(buf, sizeof(buf), format, ap); +> return my_printf("%s", buf); +> } +> ``` +> +> and then use `cmake -DWAMR_BH_VPRINTF=my_vprintf ..` to pass the callback function, or add `BH_VPRINTF=my_vprintf` macro for the compiler, e.g. add line `add_defintions(-DBH_VPRINTF=my_vprintf)` in CMakeListst.txt. See [basic sample](../samples/basic/src/main.c) for a usage example. + +#### **WAMR_BH_LOG**=, default to disable if not set +> Note: if the log_callback function is provided by the developer, WAMR logs are redirected to such callback. For example: +> ```C +> void my_log(uint32 log_level, const char *file, int line, const char *fmt, ...) +> { +> /* Usage of custom logger */ +> } +> ``` +> See [basic sample](../samples/basic/src/main.c) for a usage example. + +#### **Enable reference types feature** +- **WAMR_BUILD_REF_TYPES**=1/0, default to disable if not set + +#### **Exclude WAMR application entry functions** +- **WAMR_DISABLE_APP_ENTRY**=1/0, default to disable if not set + +> Note: The WAMR application entry (`core/iwasm/common/wasm_application.c`) encapsulate some common process to instantiate, execute the wasm functions and print the results. Some platform related APIs are used in these functions, so you can enable this flag to exclude this file if your platform doesn't support those APIs. +> *Don't enable this flag if you are building `product-mini`* + +#### **Enable source debugging features** +- **WAMR_BUILD_DEBUG_INTERP**=1/0, default to 0 if not set +> Note: There are some other setup required by source debugging, please refer to [source_debugging.md](./source_debugging.md) and [WAMR source debugging basic](https://bytecodealliance.github.io/wamr.dev/blog/wamr-source-debugging-basic) for more details. + +#### **Enable load wasm custom sections** +- **WAMR_BUILD_LOAD_CUSTOM_SECTION**=1/0, default to disable if not set + +> Note: By default, the custom sections are ignored. If the embedder wants to get custom sections from `wasm_module_t`, then `WAMR_BUILD_LOAD_CUSTOM_SECTION` should be enabled, and then `wasm_runtime_get_custom_section` can be used to get a custom section by name. + +> Note: If `WAMR_BUILD_CUSTOM_NAME_SECTION` is enabled, then the `custom name section` will be treated as a special section and consumed by the runtime, not available to the embedder. + +> For AoT file, must use `--emit-custom-sections` to specify which sections need to be emitted into AoT file, otherwise all custom sections will be ignored. + +#### **Stack guard size** +- **WAMR_BUILD_STACK_GUARD_SIZE**=n, default to N/A if not set. +> Note: By default, the stack guard size is 1K (1024) or 24K (if uvwasi enabled). + +#### **Disable writing the linear memory base address to x86 GS segment register** +- **WAMR_DISABLE_WRITE_GS_BASE**=1/0, default to enable if not set and supported by platform +> Note: by default only platform [linux x86-64](https://github.com/bytecodealliance/wasm-micro-runtime/blob/5fb5119239220b0803e7045ca49b0a29fe65e70e/core/shared/platform/linux/platform_internal.h#L67) will enable this feature, for 32-bit platforms it's automatically disabled even when the flag is set to 0. In linux x86-64, writing the linear memory base address to x86 GS segment register may be used to speedup the linear memory access for LLVM AOT/JIT, when `--enable-segue=[]` option is added for `wamrc` or `iwasm`. + +> See [Enable segue optimization for wamrc when generating the aot file](./perf_tune.md#3-enable-segue-optimization-for-wamrc-when-generating-the-aot-file) for more details. + +#### **User defined linear memory allocator** +- **WAMR_BUILD_ALLOC_WITH_USAGE**=1/0, default to disable if not set +> Notes: by default, the linear memory is allocated by system. when it's set to 1 and Alloc_With_Allocator is selected, it will be allocated by customer. + +#### **Enable running PGO(Profile-Guided Optimization) instrumented AOT file** +- **WAMR_BUILD_STATIC_PGO**=1/0, default to disable if not set +> Note: See [Use the AOT static PGO method](./perf_tune.md#5-use-the-aot-static-pgo-method) for more details. + +#### **Enable linux perf support** +- **WAMR_BUILD_LINUX_PERF**=1/0, enable linux perf support to generate the flamegraph to analyze the performance of a wasm application, default to disable if not set +> Note: See [Use linux-perf](./perf_tune.md#7-use-linux-perf) for more details. + +#### **Enable module instance context APIs** +- **WAMR_BUILD_MODULE_INST_CONTEXT**=1/0, enable module instance context APIs which can set one or more contexts created by the embedder for a wasm module instance, default to enable if not set: +```C + wasm_runtime_create_context_key + wasm_runtime_destroy_context_key + wasm_runtime_set_context + wasm_runtime_set_context_spread + wasm_runtime_get_context +``` +> Note: See [wasm_export.h](../core/iwasm/include/wasm_export.h) for more details. + +#### **Enable quick AOT/JTI entries** +- **WAMR_BUILD_QUICK_AOT_ENTRY**=1/0, enable registering quick call entries to speedup the aot/jit func call process, default to enable if not set +> Note: See [Refine callings to AOT/JIT functions from host native](./perf_tune.md#83-refine-callings-to-aotjit-functions-from-host-native) for more details. + +#### **Enable AOT intrinsics** +- **WAMR_BUILD_AOT_INTRINSICS**=1/0, enable the AOT intrinsic functions, default to enable if not set. These functions can be called from the AOT code when `--disable-llvm-intrinsics` flag or `--enable-builtin-intrinsics=` flag is used by wamrc to generate the AOT file. +> Note: See [Tuning the XIP intrinsic functions](./xip.md#tuning-the-xip-intrinsic-functions) for more details. + +#### **Configurale memory access boundary check** +- **WAMR_CONFIGUABLE_BOUNDS_CHECKS**=1/0, default to disable if not set +> Note: If it is enabled, allow to run `iwasm --disable-bounds-checks` to disable the memory access boundary checks for interpreter mode. + +#### **Module instance context APIs** +- **WAMR_BUILD_MODULE_INST_CONTEXT**=1/0, default to disable if not set +> Note: If it is enabled, allow to set one or more contexts created by embedder for a module instance, the below APIs are provided: +```C + wasm_runtime_create_context_key + wasm_runtime_destroy_context_key + wasm_runtime_set_context + wasm_runtime_set_context_spread + wasm_runtime_get_context +``` + +**Combination of configurations:** + +We can combine the configurations. For example, if we want to disable interpreter, enable AOT and WASI, we can run command: + +``` Bash +cmake .. -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_LIBC_WASI=0 -DWAMR_BUILD_PLATFORM=linux +``` + +Or if we want to enable interpreter, disable AOT and WASI, and build as X86_32, we can run command: + +``` Bash +cmake .. -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_AOT=0 -DWAMR_BUILD_LIBC_WASI=0 -DWAMR_BUILD_TARGET=X86_32 +``` diff --git a/wasm-micro-runtime/doc/build_wasm_app.md b/wasm-micro-runtime/doc/build_wasm_app.md new file mode 100644 index 0000000..7747d9a --- /dev/null +++ b/wasm-micro-runtime/doc/build_wasm_app.md @@ -0,0 +1,439 @@ +# Build WASM applications + +Prepare WASM building environments +================================== + +For C and C++, WASI-SDK version 19.0+ is the major tool supported by WAMR to build WASM applications. Also, we can use [Emscripten SDK (EMSDK)](https://github.com/emscripten-core/emsdk), but it is not recommended. And there are some other compilers such as the standard clang compiler, which might also work [here](./other_wasm_compilers.md). + +To install WASI SDK, please download the [wasi-sdk release](https://github.com/WebAssembly/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. + +The official *wasi-sdk release* doesn't fully support *latest 128-bit SIMD spec* yet. WAMR provides a script in [build-wasi-sdk](../test-tools/build-wasi-sdk/) to generate +another wasi-sdk with *llvm-15* from source code and installs it at *../test-tools/wasi-sdk*. If you plan to build WASM applications with *latest 128-bit SIMD*, please use it instead of the official release. + +And [sample workloads](../samples/workload) are using the self-compiled wasi-sdk. + +For [AssemblyScript](https://github.com/AssemblyScript/assemblyscript), please refer to [AssemblyScript quick start](https://www.assemblyscript.org/quick-start.html) and [AssemblyScript compiler](https://www.assemblyscript.org/compiler.html#command-line-options) for how to install `asc` compiler and build WASM applications. + +For Rust, please refer to [Install Rust and Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) to install *cargo*, *rustc* and *rustup*. By default they are under ~/.cargo/bin. + +And then run such a command to install `wasm32-wasi` target. + +``` bash +$ rustup target add wasm32-wasi +``` + +To build WASM applications, run + +``` bash +$ cargo build --target wasm32-wasi +``` + +The output files are under `target/wasm32-wasi`. + +To build a release version + +``` bash +$ cargo build --release --target wasm32-wasi +``` + + +Build WASM applications with wasi-sdk +===================================== + +You can write a simple ```test.c``` as the first sample. + +``` C +#include +#include + +int main(int argc, char **argv) +{ + char *buf; + + printf("Hello world!\n"); + + buf = malloc(1024); + if (!buf) { + printf("malloc buf failed\n"); + return -1; + } + + printf("buf ptr: %p\n", buf); + + sprintf(buf, "%s", "1234\n"); + printf("buf: %s", buf); + + free(buf); + return 0; +} +``` + +To build the source file to WASM bytecode, we can input the following command: + +``` Bash +/opt/wasi-sdk/bin/clang -O3 -o test.wasm test.c +``` + +## 1. wasi-sdk options + +There are some useful options that are used to compile C/C++ to Wasm (for a full introduction, please refer to [clang command line argument reference](https://clang.llvm.org/docs/ClangCommandLineReference.html) and [wasm-ld command line argument manual](https://lld.llvm.org/WebAssembly.html)): + +- **-nostdlib** Do not use the standard system startup files or libraries when linking. In this mode, the **libc-builtin** library of WAMR must be built to run the wasm app, otherwise, the **libc-wasi** library must be built. You can specify **-DWAMR_BUILD_LIBC_BUILTIN=1** or **-DWAMR_BUILD_LIBC_WASI=1** for CMake to build WAMR with libc-builtin support or libc-wasi support. + +- **-Wl,--no-entry** Do not output any entry point + +- **-Wl,--export=\** Force a symbol to be exported, e.g. **-Wl,--export=foo** to export foo function + +- **-Wl,--export-all** Export all symbols (normally combined with --no-gc-sections) + +- **-Wl,--initial-memory=\** Initial size of the linear memory, which must be a multiple of 65536 + +- **-Wl,--max-memory=\** Maximum size of the linear memory, which must be a multiple of 65536 + +- **-z stack-size=\** The auxiliary stack size, which is an area of linear memory, must be smaller than the initial memory size. + +- **-Wl,--strip-all** Strip all symbols + +- **-Wl,--shared-memory** Use shared linear memory + +- **-Wl,--allow-undefined** Allow undefined symbols in linked binary + +- **-Wl,--allow-undefined-file=\** Allow symbols listed in \ to be undefined in linked binary + +- **-pthread** Support POSIX threads in generated code + +For example, we can build the wasm app with the command: + +``` Bash +/opt/wasi-sdk/bin/clang -O3 -nostdlib \ + -z stack-size=8192 -Wl,--initial-memory=65536 \ + -o test.wasm test.c \ + -Wl,--export=main -Wl,--export=__main_argc_argv \ + -Wl,--export=__heap_base -Wl,--export=__data_end \ + -Wl,--no-entry -Wl,--strip-all -Wl,--allow-undefined +``` +to generate a wasm binary with nostdlib mode, the auxiliary stack size is 8192 bytes, initial memory size is 64 KB, main function, heap base global and data end global are exported, no entry function is generated (no _start function is exported), and all symbols are stripped. Note that it is nostdlib mode, so libc-builtin should be enabled by runtime embedder or iwasm (with `cmake -DWAMR_BUILD_LIBC_BUILT=1`, enabled by iwasm in Linux by default). + +If we want to build the wasm app with wasi mode, we may build the wasm app with the command: + +```bash +/opt/wasi-sdk/bin/clang -O3 \ + -z stack-size=8192 -Wl,--initial-memory=65536 \ + -o test.wasm test.c \ + -Wl,--export=__heap_base -Wl,--export=__data_end \ + -Wl,--strip-all +``` + +to generate a wasm binary with wasi mode, the auxiliary stack size is 8192 bytes, initial memory size is 64 KB, heap base global and data end global are exported, wasi entry function exported (_start function), and all symbols are stripped. Note that it is wasi mode, so libc-wasi should be enabled by runtime embedder or iwasm (with `cmake -DWAMR_BUILD_LIBC_WASI=1`, enabled by iwasm in Linux by default), and normally no need to export main function, by default _start function is executed by iwasm. + +> Note: for the Rust project, we can set these flags by setting the `rustflags` in the Cargo configuration file, e.g. `/.cargo/config.toml` or `$CARGO_HOME/config.toml`, for example: +> ```toml +> [build] +> rustflags = [ +> "-C", "link-arg=--initial-memory=65536", +> "-C", "link-arg=-zstack-size=8192", +> "-C", "link-arg=--export=__heap_base", +> "-C", "link-arg=--export=__data_end", +> "-C", "link-arg=--strip-all", +> ] +> ``` + +## 2. How to reduce the footprint? + +Firstly if libc-builtin (-nostdlib) mode meets the requirements, e.g. there are no file io operations in wasm app, we should build the wasm app with -nostdlib option as possible as we can, since the compiler doesn't build the libc source code into wasm bytecodes, which greatly reduces the binary size. + +### (1) Methods to reduce the libc-builtin (-nostdlib) mode footprint + +- export \_\_heap_base global and \_\_data_end global + ```bash + -Wl,--export=__heap_base -Wl,--export=__data_end + ``` + If the two globals are exported, and there are no memory.grow and memory.size opcodes (normally nostdlib mode doesn't introduce these opcodes since the libc malloc function isn't built into wasm bytecode), WAMR runtime will truncate the linear memory at the place of \__heap_base and append app heap to the end, so we don't need to allocate the memory specified by `-Wl,--initial-memory=n` which must be at least 64 KB. This is helpful for some embedded devices whose memory resource might be limited. + +> For the Rust project, please set the flags in the Cargo configuration file, for example: +> ```toml +> [build] +> rustflags = [ +> "-C", "link-arg=--export=__heap_base", +> "-C", "link-arg=--export=__data_end", +> "-C", "link-arg=--initial-memory=65536", +> ] +> ``` + +- reduce auxiliary stack size + + The auxiliary stack is an area of linear memory, normally the size is 64 KB by default which might be a little large for embedded devices and partly used, we can use `-z stack-size=n` to set its size. + +> For the Rust project, please set the flag in the Cargo configuration file, for example: +> ```toml +> [build] +> rustflags = [ +> "-C", "link-arg=-zstack-size=8192" +> ] +> ``` + +- use -O3 and -Wl,--strip-all + +> For the Rust project, please set the flag in the Cargo configuration file, for example: +> ```toml +> [build] +> rustflags = [ +> "-C", "link-arg=--strip-all" +> ] +> ``` + +- reduce app heap size when running iwasm + + We can pass `--heap-size=n` option to set the maximum app heap size for iwasm, by default it is 16 KB. For the runtime embedder, we can set the `uint32_t heap_size` argument when calling API ` wasm_runtime_instantiate`. + +- reduce wasm operand stack size when running iwasm + + WebAssembly is a binary instruction format for a stack-based virtual machine, which requires a stack to execute the bytecodes. We can pass `--stack-size=n` option to set the maximum stack size for iwasm, by default it is 16 KB. For the runtime embedder, we can set the `uint32_t stack_size` argument when calling API ` wasm_runtime_instantiate` and `wasm_runtime_create_exec_env`. + +- decrease block_addr_cache size for classic interpreter + + The block_addr_cache is a hash cache to store the else/end addresses for WebAssembly blocks (BLOCK/IF/LOOP) to speed up address lookup. This is only available in the classic interpreter. We can set it by defineing macro `-DBLOCK_ADDR_CACHE_SIZE=n`, e.g. add `add_defintion (-DBLOCK_ADDR_CACHE_SIZE=n)` in CMakeLists.txt, by default it is 64, and the total block_addr_cache size is 3072 bytes in 64-bit platform and 1536 bytes in 32-bit platform. + +### (2) Methods to reduce the libc-wasi (without -nostdlib) mode footprint + +Most of the above methods are also available for libc-wasi mode, besides them, we can export malloc and free functions with `-Wl,--export=malloc -Wl,--export=free` option, so WAMR runtime will disable its app heap and call the malloc/free function exported to allocate/free the memory from/to the heap space managed by libc. + +Note: wasm-ld from LLVM 13 and later automatically inserts ctor/dtor calls +for all exported functions for a command. (vs reactor) +It breaks the malloc/free exports mentioned above. + +## 3. Build wasm app with pthread support + +Please ref to [pthread library](./pthread_library.md) for more details. + +## 4. Build wasm app with SIMD support + +The official *wasi-sdk release* doesn't fully support *latest 128-bit SIMD spec* yet. WARM provides a script in [build-wasi-sdk](../test-tools/build-wasi-sdk/) to generate +another wasi-sdk with *llvm-13* from source code and installs it at *../test-tools/wasi-sdk*. If you plan to build WASM applications with *latest 128-bit SIMD*, please use it instead of the official release. + +And also you can install emsdk and use its SSE header files, please ref to workload samples, e.g. [bwa CMakeLists.txt](../samples/workload/bwa/CMakeLists.txt) and [wasm-av1 CMakeLists.txt](../samples/workload/wasm-av1/CMakeLists.txt) for more details. + +For both wasi-sdk and emsdk, please add the option `-msimd128` for clang or emcc to generate WASM application with SIMD bytecodes. + +# Build WASM applications with emsdk + +## 1. Install emsdk + +Assuming you are using Linux, you may install emcc and em++ from Emscripten EMSDK following the steps below: + +``` +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install latest +./emsdk activate latest +# And then source the emsdk_env.sh script before build wasm app +source emsdk_env.sh (or add it to ~/.bashrc if you don't want to run it each time) +``` + +The Emscripten website provides other installation methods beyond Linux. + +## 2. emsdk options + +To build the wasm C source code into wasm binary, we can use the following command: + +```bash +EMCC_ONLY_FORCED_STDLIBS=1 emcc -O3 -s STANDALONE_WASM=1 \ + -o test.wasm test.c \ + -s TOTAL_STACK=4096 -s TOTAL_MEMORY=65536 \ + -s "EXPORTED_FUNCTIONS=['_main']" \ + -s ERROR_ON_UNDEFINED_SYMBOLS=0 +``` + +There are some useful options: + +- **EMCC_ONLY_FORCED_STDLIBS=1** whether to link libc library into the output binary or not, similar to `-nostdlib` option of wasi-sdk clang. If specified, then no libc library is linked and the **libc-builtin** library of WAMR must be built to run the wasm app, otherwise, the **libc-wasi** library must be built. You can specify **-DWAMR_BUILD_LIBC_BUILTIN=1** or **-DWAMR_BUILD_LIBC_WASI=1** for CMake to build WAMR with libc-builtin support or libc-wasi support. + + The emsdk's wasi implementation is incomplete, e.g. open a file might just return fail, so it is strongly not recommended to use this mode, especially when there are file io operations in wasm app, please use wasi-sdk instead. + +- **-s STANDALONE_WASM=1** build wasm app in standalone mode (non-web mode), if the output file has suffix ".wasm", then only wasm file is generated (without html file and JavaScript file). + +- **-s TOTAL_STACK=\** the auxiliary stack size, same as `-z stack-size=\` of wasi-sdk + +- **-s TOTAL_MEMORY=\** or **-s INITIAL_MEORY=\** the initial linear memory size + +- **-s MAXIMUM_MEMORY=\** the maximum linear memory size, only take effect if **-s ALLOW_MEMORY_GROWTH=1** is set + +- **-s ALLOW_MEMORY_GROWTH=1/0** whether the linear memory is allowed to grow or not + +- **-s "EXPORTED_FUNCTIONS=['func name1', 'func name2']"** to export functions + +- **-s ERROR_ON_UNDEFINED_SYMBOLS=0** disable the errors when there are undefined symbols + +For more options, please ref to /upstream/emscripten/src/settings.js, or [Emscripten document](https://emscripten.org/docs/compiling/Building-Projects.html). + +# Build a project with cmake + +If you have a complex WASM application project which contains dozens of source files, you can consider using cmake for project building. + +You can cross compile your project by using the toolchain provided by WAMR. + +Assume the original `CMakeLists.txt` for `test.c` likes below: + +``` cmake +cmake_minimum_required (VERSION 3.5) +project(hello_world) +add_executable(hello_world test.c) +``` + +It is easy to use *wasi-sdk* in CMake by setting *CMAKE_TOOLCHAIN_FILE* without any modification on the original *CMakeLists.txt*. + +``` +$ cmake -DWASI_SDK_PREFIX=${WASI_SDK_INSTALLTION_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_INSTALLTION_DIR}/share/cmake/wasi-sdk.cmake + -DCMAKE_SYSROOT= + .. +``` + +`WASI_SDK_INSTALLTION_DIR` is the directory in where you install the *wasi-sdk*. like */opt/wasi-sdk* + +If you prefer WASI, set *CMAKE_SYSROOT* to *wasi-sysroot* + +``` +$ cmake + -DCMAKE_SYSROOT=${WASI_SDK_INSTALLTION_DIR}/share/wasi-sysroot + .. +``` + +If you prefer *WAMR builtin libc*, set *CMAKE_SYSROOT* to *libc-builtin-sysroot* + +> Note: If you have already built a SDK profile + +``` +$ cmake + -DCMAKE_SYSROOT=${WAMR_SOURCE_ROOT}/wamr-sdk/app/libc-builtin-sysroot + .. +``` + +You will get ```hello_world``` which is the WASM app binary. + + +# Compile WASM to AOT module + +Please ensure the wamrc was already generated and available in your shell PATH. Then we can use wamrc to compile WASM app binary to WAMR AOT binary. + +``` Bash +wamrc -o test.aot test.wasm +``` + +wamrc supports a number of compilation options through the command line arguments: + +``` Bash +wamrc --help +Usage: wamrc [options] -o output_file wasm_file + --target= Set the target arch, which has the general format: + = x86_64, i386, aarch64, arm, thumb, xtensa, mips, + riscv64, riscv32. + Default is host arch, e.g. x86_64 + = for ex. on arm or thumb: v5, v6m, v7a, v7m, etc. + Use --target=help to list supported targets + --target-abi= Set the target ABI, e.g. gnu, eabi, gnueabihf, msvc, etc. + Default is gnu if target isn't riscv64 or riscv32 + For target riscv64 and riscv32, default is lp64d and ilp32d + Use --target-abi=help to list all the ABI supported + --cpu= Set the target CPU (default: host CPU, e.g. skylake) + Use --cpu=help to list all the CPU supported + --cpu-features= Enable or disable the CPU features + Use +feature to enable a feature, or -feature to disable it + For example, --cpu-features=+feature1,-feature2 + Use --cpu-features=+help to list all the features supported + --opt-level=n Set the optimization level (0 to 3, default is 3) + --size-level=n Set the code size level (0 to 3, default is 3) + -sgx Generate code for SGX platform (Intel Software Guard Extention) + --bounds-checks=1/0 Enable or disable the bounds checks for memory access: + by default it is disabled in all 64-bit platforms except SGX and + in these platforms runtime does bounds checks with hardware trap, + and by default it is enabled in all 32-bit platforms + --format= Specifies the format of the output file + The format supported: + aot (default) AoT file + object Native object file + llvmir-unopt Unoptimized LLVM IR + llvmir-opt Optimized LLVM IR + --disable-bulk-memory Disable the MVP bulk memory feature + --enable-multi-thread Enable multi-thread feature, the dependent features bulk-memory and + thread-mgr will be enabled automatically + --enable-tail-call Enable the post-MVP tail call feature + --disable-simd Disable the post-MVP 128-bit SIMD feature: + currently 128-bit SIMD is only supported for x86-64 target, + and by default it is enabled in x86-64 target and disabled + in other targets + --disable-ref-types Disable the MVP reference types feature + --disable-aux-stack-check Disable auxiliary stack overflow/underflow check + --enable-dump-call-stack Enable stack trace feature + --enable-perf-profiling Enable function performance profiling + -v=n Set log verbose level (0 to 5, default is 2), larger with more log +Examples: wamrc -o test.aot test.wasm + wamrc --target=i386 -o test.aot test.wasm + wamrc --target=i386 --format=object -o test.o test.wasm +``` + +## AoT-compiled module compatibility among WAMR versions + +When making major ABI changes for AoT-compiled modules, we bump +`AOT_CURRENT_VERSION` constant in `core/config.h` header. +The runtime rejects to load a module AoT-compiled with wamrc with +a different `AOT_CURRENT_VERSION`. + +We try our best to maintain our runtime ABI for AoT-compiled modules +compatible among WAMR versions with the same `AOT_CURRENT_VERSION` +so that combinations of older wamrc and newer runtime usually work. +However, there might be minor incompatibilities time to time. +For productions, we recommend to use the exactly same version of +wamrc and the runtime. + +## AoT compilation with 3rd-party toolchains + +`wamrc` uses LLVM to compile wasm bytecode to AoT file, this works for most of the architectures, but there may be circumstances where you want to use 3rd-party toolchains to take over some steps of the compilation pipeline, e.g. + +1. The upstream LLVM doesn't support generating object file for your CPU architecture (such as ARC), then we may need some other assembler to do such things. +2. You may get some other LLVM-based toolchains which may have better optimizations for the specific target, then you may want your toolchain to take over all optimization steps. + +`wamrc` provides two environment variables to achieve these: +- `WAMRC_LLC_COMPILER` + + When specified, `wamrc` will emit the optimized LLVM-IR (.bc) to a file, and invoke `$WAMRC_LLC_COMPILER` with ` -c -O3 ` to generate the object file. + + Optionally, you can use environment variable `WAMRC_LLC_FLAGS` to overwrite the default flags. + +- `WAMRC_ASM_COMPILER` + + When specified, `wamrc` will emit the text based assembly file (.s), and invoke `$WAMRC_ASM_COMPILER` with ` -c -O3 ` to generate the object file. + + Optionally, you can use environment variable `WAMRC_ASM_FLAGS` to overwrite the default flags. + +### Usage example +``` bash +WAMRC_LLC_COMPILER=/usr/local/opt/llvm@14/bin/clang WAMRC_LLC_FLAGS="--target=x86_64-pc-linux-gnu -mcmodel=medium -c -O3" ./wamrc -o test.aot test.wasm +``` + +> Note: `wamrc` will verify whether the specified file exists and executable. If verification failed, `wamrc` will report a warning and fallback to normal pipeline. Since the verification is based on file, you **must specify the absolute path to the binary** even if it's in `$PATH` + +> Note: `WAMRC_LLC_COMPILER` has higher priority than `WAMRC_ASM_COMPILER`, if `WAMRC_LLC_COMPILER` is set and verified, then `WAMRC_ASM_COMPILER` will be ignored. + +> Note: the `LLC` and `ASM` in the env name just means this compiler will be used to compile the `LLVM IR file`/`assembly file` to object file, usually passing the compiler driver is the simplest way. (e.g. for LLVM toolchain, you don't need to pass `/usr/bin/llc`, using `/usr/bin/clang` is OK) + +> Note: You might need to set `WAMRC_LLC_FLAGS`/`WAMRC_ASM_FLAGS` to match whatever the `wamrc` command would automatically do. In the above example, `-mcmodel=medium` corresponds to `wamrc --size-level=1`, which is the default of `wamrc` on macOS. + +Run WASM app in WAMR mini product build +======================================= + +Run the test.wasm or test.aot with WAMR mini product build: +``` Bash +./iwasm test.wasm or +./iwasm test.aot +``` +You will get the following output: +``` +Hello world! +buf ptr: 0xffffc2c8 +buf: 1234 +``` +If you would like to run the test app on Zephyr, we have embedded a test sample into its OS image. You will need to execute: +``` +ninja run +``` diff --git a/wasm-micro-runtime/doc/devcontainer.md b/wasm-micro-runtime/doc/devcontainer.md new file mode 100644 index 0000000..b75f13e --- /dev/null +++ b/wasm-micro-runtime/doc/devcontainer.md @@ -0,0 +1,25 @@ +# Visual Studio Code development container + +We all know Docker containers and may use them a lot in school or work. It resolves dependency management for our projects/applications, prevents package version confusion and conflict, and contamination of the local environment. + +Now WAMR has a Dockerfile under path `.devontainer` to create a container image, dev container images that you could easily use in VS Code. In case you prefer other IDE like Clion, you can also build it and use for the IDE you like. + +## How to use it + +It's straightforward to use Docker in VS Code! First, you have VS Code and Docker installed(if not yet, check [next section](#learn-more-about-docker-and-vs-code) for howto). Then you need to download Docker in VS Code extensions marketplace. + +And that's it, and you are good to go! When you open the root folder of WAMR, in the bottom right corner, the Docker extension will pop a notification and ask if you like to reopen the folder in a container. + +If you encounter any problems or get stuck somewhere, may this video [demo](https://youtu.be/Uvf2FVS1F8k) for docker usage in VS Code will help. + +## Learn more about Docker and VS Code + +[Install Docker](https://docs.docker.com/get-docker/) + +[Install VS Code](https://code.visualstudio.com/) + +[Docker extension for VS Code](https://code.visualstudio.com/docs/containers/overview) + +[Remote development with Docker in VS Code](https://code.visualstudio.com/docs/remote/containers#_getting-started) + +[What is dev container image in VS Code](https://code.visualstudio.com/docs/remote/containers#_prebuilding-dev-container-images) \ No newline at end of file diff --git a/wasm-micro-runtime/doc/embed_wamr.md b/wasm-micro-runtime/doc/embed_wamr.md new file mode 100644 index 0000000..5e4e3a5 --- /dev/null +++ b/wasm-micro-runtime/doc/embed_wamr.md @@ -0,0 +1,337 @@ +Embedding WAMR guideline +===================================== + +**Note**: This document is about how to embed WAMR into C/C++ host applications, for other languages, please refer to: [Embed WAMR into Python](../language-bindings/python), [Embed WAMR into Go](../language-bindings/go). + +All the embedding APIs supported by the runtime are defined under folder [core/iwasm/include](../core/iwasm/include). The API details are available in the header files. + +## Embed WAMR into developer's project + +WAMR is designed to be easy embeddable in any project, a typical way of building WAMR is to use cmake, developer can configure features by setting cmake variables and then include the script `runtime_lib.cmake` under folder [build-scripts](../build-scripts) in his CMakeList.txt, for example: +``` cmake +set (WAMR_BUILD_PLATFORM "linux") +set (WAMR_BUILD_TARGET "X86_64") +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_FAST_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_LIBC_WASI 1) +set (WAMR_BUILD_SIMD 1) +set (WAMR_ROOT_DIR path/to/wamr/root) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +target_link_libraries (your_project vmlib) +``` +Examples can be found in [CMakeLists.txt of linux platform](../product-mini/platforms/linux/CMakeLists.txt) and [other platforms](../product-mini/platforms). The available features to configure can be found in [Build WAMR vmcore](./build_wamr.md#wamr-vmcore-cmake-building-configurations). + +Developer can also use Makefile to embed WAMR, by defining macros and including directories, and adding the source files, examples can be found in [makefile of alios-things platform](../product-mini/platforms/alios-things/aos.mk) and [makefile of nuttx platform](../product-mini/platforms/nuttx/wamr.mk). + +## The runtime initialization + +``` C + char *buffer, error_buf[128]; + wasm_module_t module; + wasm_module_inst_t module_inst; + wasm_function_inst_t func; + wasm_exec_env_t exec_env; + uint32 size, stack_size = 8092, heap_size = 8092; + + /* initialize the wasm runtime by default configurations */ + wasm_runtime_init(); + + /* read WASM file into a memory buffer */ + buffer = read_wasm_binary_to_buffer(…, &size); + + /* add line below if we want to export native functions to WASM app */ + wasm_runtime_register_natives(...); + + /* parse the WASM file from buffer and create a WASM module */ + module = wasm_runtime_load(buffer, size, error_buf, sizeof(error_buf)); + + /* create an instance of the WASM module (WASM linear memory is ready) */ + module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); +``` + +The `wasm_runtime_init()` uses the default memory allocator os_malloc/os_free function from the [`core/shared/platform`](../core/shared/platform) for the runtime memory management. + +WAMR supports to restrict its all memory allocations in a raw buffer. It ensures the dynamic memories used by the WASM applications won't harm the system availability, which is extremely important for embedded systems. This can be done by using `wasm_runtime_full_init()`. This function also allows you to configure the native API's for exporting to WASM app, and set the maximum thread number when multi-thread feature is enabled. + +Refer to the following sample: + +```c +/* the native functions that will be exported to WASM app */ +static NativeSymbol native_symbols[] = { + EXPORT_WASM_API_WITH_SIG(display_input_read, "(*)i"), + EXPORT_WASM_API_WITH_SIG(display_flush, "(iiii*)") +}; + +/* all the runtime memory allocations are retricted in the global_heap_buf array */ +static char global_heap_buf[512 * 1024]; +RuntimeInitArgs init_args; +memset(&init_args, 0, sizeof(RuntimeInitArgs)); + +/* configure the memory allocator for the runtime */ +init_args.mem_alloc_type = Alloc_With_Pool; +init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; +init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + +/* configure the native functions being exported to WASM app */ +init_args.native_module_name = "env"; +init_args.n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); +init_args.native_symbols = native_symbols; + +/* set maximum thread number if needed when multi-thread is enabled, + the default value is 4 */ +init_args.max_thread_num = max_thread_num; + +/* initialize runtime environment with user configurations*/ +if (!wasm_runtime_full_init(&init_args)) { + return -1; +} +``` + +## Native calls WASM functions and passes parameters + +After a module is instantiated, the runtime embedder can lookup the target WASM function by name, and create execution environment to call the function. + +```c + /* lookup a WASM function by its name + The function signature can NULL here */ + func = wasm_runtime_lookup_function(module_inst, "fib"); + + /* creat an execution environment to execute the WASM functions */ + exec_env = wasm_runtime_create_exec_env(module_inst, stack_size); +``` + +There are several ways to call WASM function: + +1. Function call with parameters in an array of 32 bits elements and size: + +```c + uint32 argv[2]; + + /* arguments are always transferred in 32-bit element */ + argv[0] = 8; + + /* call the WASM function */ + if (wasm_runtime_call_wasm(exec_env, func, 1, argv) ) { + /* the return value is stored in argv[0] */ + printf("fib function return: %d\n", argv[0]); + } + else { + /* exception is thrown if call fails */ + printf("%s\n", wasm_runtime_get_exception(module_inst)); + } +``` + +The parameters are transferred in an array of 32 bits elements. For parameters that occupy 4 or fewer bytes, each parameter can be a single array element. For parameters in types like double or int64, each parameter will take two array elements. The function return value will be sent back in the first one or two elements of the array according to the value type. See the sample code below: + +```c + uint32 argv[6]; + char arg1 = 'a'; + int arg2 = 10; + double arg3 = 1.0; + int64 arg4 = 100; + double ret; + + argv[0] = arg1; + argv[1] = arg2; + + /** + * use memory copy for 8-byte parameters rather than + * *(double*)(&argv[2]) = arg3 here because some archs + * like ARM, MIPS require the address must be 8-byte aligned. + * Or use the aligned malloc or compiler align attribute + * to ensure the array address is 8-byte aligned + */ + memcpy(&argv[2], &arg3, sizeof(arg3)); + memcpy(&argv[4], &arg4, sizeof(arg4)); + + /* attention: the arg number is 6 here since both + arg3 and arg4 each takes 2 elements */ + wasm_runtime_call_wasm(exec_env, func, 6, argv); + + /* if the return value is type of 8 bytes, it takes + the first two array elements */ + memcpy(&ret, &argv[0], sizeof(ret)); +``` + +2. Function call with results and arguments both in `wasm_val_t` struct and size: + +```c + uint32 num_args = 1, num_results = 1; + wasm_val_t args[1], results[1]; + + /* set the argument type and value */ + args[0].kind = WASM_I32; + args[0].of.i32 = 8; + + /* call the WASM function */ + if (wasm_runtime_call_wasm_a(exec_env, func, num_results, results, num_args, args)) { + /* the return value is stored in results */ + printf("fib function return: %d\n", results[0].of.i32); + } + else { + /* exception is thrown if call fails */ + printf("%s\n", wasm_runtime_get_exception(module_inst)); + } +``` + +3. Function call with variant argument support: + +```c + uint32 num_args = 1, num_results = 1; + wasm_val_t results[1]; + + /* call the WASM function */ + if (wasm_runtime_call_wasm_v(exec_env, func, 1, results, 1, 8)) { + /* the return value is stored in results */ + printf("fib function return: %d\n", results[0].of.i32); + } + else { + /* exception is thrown if call fails */ + printf("%s\n", wasm_runtime_get_exception(module_inst)); + } +``` + +## Pass buffer to WASM function + +If we need to transfer a buffer to WASM function, we can pass the buffer address through a parameter. **Attention**: The sandbox will forbid the WASM code to access outside memory, we must **allocate the buffer from WASM instance's own memory space and pass the buffer address in instance's space (not the runtime native address)**. + +There are two runtime APIs available for this purpose. + +```c +/** + * malloc a buffer from instance's private memory space. + * + * return: the buffer address in instance's memory space (pass to the WASM funciton) + * p_native_addr: return the native address of allocated memory + * size: the buffer size to allocate + */ +uint64_t +wasm_runtime_module_malloc(wasm_module_inst_t module_inst, + uint64_t size, void **p_native_addr); + +/** + * malloc a buffer from instance's private memory space, + * and copy the data from another native buffer to it. + * + * return: the buffer address in instance's memory space (pass to the WASM funciton) + * src: the native buffer address + * size: the size of buffer to be allocated and copy data + */ +uint64_t +wasm_runtime_module_dup_data(wasm_module_inst_t module_inst, + const char *src, uint64_t size); + +/* free the memory allocated from module memory space */ +void +wasm_runtime_module_free(wasm_module_inst_t module_inst, uint64_t ptr); +``` + +Usage sample: + +```c +char * buffer = NULL; +uint64_t buffer_for_wasm; + +buffer_for_wasm = wasm_runtime_module_malloc(module_inst, 100, &buffer); +if (buffer_for_wasm != 0) { + uint32 argv[3]; + strncpy(buffer, "hello", 100); /* use native address for accessing in runtime */ + argv[0] = buffer_for_wasm; /* pass the buffer address for WASM space */ + argv[2] = 100; /* the size of buffer */ + wasm_runtime_call_wasm(exec_env, func, 3, argv); + + /* it is runtime embedder's responsibility to release the memory, + unless the WASM app will free the passed pointer in its code */ + wasm_runtime_module_free(module_inst, buffer_for_wasm); +} +``` + +## Pass structured data to WASM function + +We can't pass structure data or class objects through the pointer since the memory layout can different in two worlds. The way to do it is serialization. Refer to [export_native_api.md](./export_native_api.md) for the details. + +## Execute wasm functions in multiple threads + +It isn't safe to use an `exec_env` object in multiple threads concurrently. +To run a multi-threaded application, you basically need a separate `exec_env` +for each threads. + +### Approaches to manage `exec_env` objects and threads + +WAMR supports two approaches to manage `exec_env` and threads as described +below. While they are not exclusive, you usually only need to use one of +them. + +#### Make your WASM application manage threads + + You can make your WASM application spawn threads by itself, + typically using `pthread` APIs like `pthread_create`. + See [pthread library](./pthread_library.md) and + [pthread implementations](./pthread_impls.md) for more details. + In this case, WAMR manages `exec_env` for the spawned threads. + +#### Make your embedder manage threads + + The `spawn exec_env` and `spawn thread` APIs are available for the embedder. + You can use these APIs to manage the threads. + See [Thread related embedder API](./embed_wamr_spawn_api.md) for details. + +### Other notes about threads + +* You can manage the maximum number of threads + + ```C + init_args.max_thread_num = THREAD_NUM; + /* If this init argument is not set, the default maximum thread number is 4 */ + ``` + +* To share memory among threads, you need to build your WASM application with shared memory + + For example, it can be done with `--shared-memory` and `-pthread`. + + ```bash + /opt/wasi-sdk/bin/clang -o test.wasm test.c -nostdlib -pthread \ + -Wl,--shared-memory,--max-memory=131072 \ + -Wl,--no-entry,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors,--export=${your_func_name} + ``` + +* The corresponding threading feature should be enabled while building the runtime + + - WAMR lib-pthread (legacy) + + ```bash + cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 + ``` + + - wasi-threads + + ```bash + cmake .. -DWAMR_BUILD_LIB_WASI_THREADS=1 + ``` + + - `wasm_runtime_spawn_exec_env` and `wasm_runtime_spawn_thread` + + ```bash + cmake .. -DWAMR_BUILD_THREAD_MGR=1 -DWAMR_BUILD_SHARED_MEMORY=1 + ``` + +## The deinitialization procedure + +``` + wasm_runtime_destroy_exec_env(exec_env); + wasm_runtime_deinstantiate(module_inst); + wasm_runtime_unload(module); + wasm_runtime_destroy(); + +``` + +## Native calling WASM function working flow + +![WAMR embed diagram](./pics/embed.PNG "WAMR embed architecture diagram") diff --git a/wasm-micro-runtime/doc/embed_wamr_spawn_api.md b/wasm-micro-runtime/doc/embed_wamr_spawn_api.md new file mode 100644 index 0000000..f0e637f --- /dev/null +++ b/wasm-micro-runtime/doc/embed_wamr_spawn_api.md @@ -0,0 +1,38 @@ +# Thread related embedder API + +This document explains `wasm_runtime_spawn_exec_env` and +`wasm_runtime_spawn_thread`. +[Here](../samples/spawn-thread) is a sample to show how to use these APIs. + + * spawn exec_env + + `spawn exec_env` API creates a new `exec_env` based on the original `exec_env`. You can use it in other threads. It's up to the embedder how to manage host threads to run the new `exec_env`. + + ```C + new_exec_env = wasm_runtime_spawn_exec_env(exec_env); + + /* Then you can use new_exec_env in your new thread */ + module_inst = wasm_runtime_get_module_inst(new_exec_env); + func_inst = wasm_runtime_lookup_function(module_inst, ...); + wasm_runtime_call_wasm(new_exec_env, func_inst, ...); + + /* you need to use this API to manually destroy the spawned exec_env */ + wasm_runtime_destroy_spawned_exec_env(new_exec_env); + ``` + + * spawn thread + + Alternatively, you can use `spawn thread` API to avoid managing the extra exec_env and the corresponding host thread manually: + + ```C + wasm_thread_t wasm_tid; + void *wamr_thread_cb(wasm_exec_env_t exec_env, void *arg) + { + module_inst = wasm_runtime_get_module_inst(exec_env); + func_inst = wasm_runtime_lookup_function(module_inst, ...); + wasm_runtime_call_wasm(exec_env, func_inst, ...); + } + wasm_runtime_spawn_thread(exec_env, &wasm_tid, wamr_thread_cb, NULL); + /* Use wasm_runtime_join_thread to join the spawned thread */ + wasm_runtime_join_thread(wasm_tid, NULL); + ``` diff --git a/wasm-micro-runtime/doc/export_native_api.md b/wasm-micro-runtime/doc/export_native_api.md new file mode 100644 index 0000000..0f7f166 --- /dev/null +++ b/wasm-micro-runtime/doc/export_native_api.md @@ -0,0 +1,292 @@ + +Export native API to WASM application +======================================================= + + + +Exporting native API steps +-------------------------- + +#### Step 1: Declare the function interface in WASM app + +Create a header file in a WASM app and declare the functions that are exported from native. In this example, we declare foo and foo2 as below in the header file `example.h` + +```c +/*** file name: example.h ***/ + +int foo(int a, int b); +void foo2(char * msg, char * buffer, int buf_len); +``` + + + +#### Step 2: Define the native API + +Then we should define the native functions in runtime source tree for handling the calls from the WASM app. The native function can be any name, for example **foo_native** and **foo2** here: + +``` C +int foo_native(wasm_exec_env_t exec_env , int a, int b) +{ + return a+b; +} + +void foo2(wasm_exec_env_t exec_env, char * msg, uint8 * buffer, int buf_len) +{ + strncpy(buffer, msg, buf_len); +} +``` + +The first parameter exec_env must be defined using type **wasm_exec_env_t** which is the calling convention by WAMR. + +The rest parameters should be in the same types as the parameters of WASM function foo(), but there are a few special cases that are explained in section "Buffer address conversion and boundary check". Regarding the parameter names, they don't have to be the same, but we would suggest using the same names for easy maintenance. + + + +#### Step 3: Register the native APIs + +Register the native APIs in the runtime, then everything is fine. It is ready to build the runtime software. + +``` C +// Define an array of NativeSymbol for the APIs to be exported. +// Note: the array must be static defined since runtime +// will keep it after registration +static NativeSymbol native_symbols[] = +{ + { + "foo", // the name of WASM function name + foo_native, // the native function pointer + "(ii)i" // the function prototype signature + }, + { + "foo2", // the name of WASM function name + foo2, // the native function pointer + "($*~)" // the function prototype signature + } +}; + +// initialize the runtime before registering the native functions +wasm_runtime_init(); + +int n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); +if (!wasm_runtime_register_natives("env", + native_symbols, + n_native_symbols)) { + goto fail1; +} + +// natives registeration must be done before loading WASM modules +module = wasm_runtime_load(buffer, size, error_buf, sizeof(error_buf)); + +``` + +**Function signature**: + +The function signature field in **NativeSymbol** structure is a string for describing the function prototype. It is critical to ensure the function signature is correctly mapping the native function interface. + +Each letter in the "()" represents a parameter type, and the one following after ")" represents the return value type. The meaning of each letter: + +- '**i**': i32 +- '**I**': i64 +- '**f**': f32 +- '**F**': f64 +- '**r**': externref (has to be the value of a `uintptr_t` variable), or all kinds of GC reference types when GC feature is enabled +- '**\***': the parameter is a buffer address in WASM application +- '**~**': the parameter is the byte length of WASM buffer as referred by preceding argument "\*". It must follow after '*', otherwise, registration will fail +- '**$**': the parameter is a string in WASM application + +The signature can defined as NULL, then all function parameters are assumed as i32 data type. + +**Use EXPORT_WASM_API_WITH_SIG** + +The `NativeSymbol` element for `foo2 ` above can be also defined with macro EXPORT_WASM_API_WITH_SIG. This macro can be used when the native function name is the same as the WASM symbol name. + +```c +static NativeSymbol native_symbols[] = +{ + EXPORT_WASM_API_WITH_SIG(foo2, "($*~)") // wasm symbol name will be "foo2" +}; +``` + +​ + +## Call exported API in WASM application + +Now we can call the exported native API in wasm application like this: +``` C +#include +#include "example.h" // where the APIs are declared + +int main(int argc, char **argv) +{ + int a = 0, b = 1; + char * msg = "hello"; + char buffer[100]; + + int c = foo(a, b); // call into native foo_native() + foo2(msg, buffer, sizeof(buffer)); // call into native foo2() + + return 0; +} +``` + +## Build native lib into shared library and register it with `iwasm` application + +Developer can also build the native library into a shared library and register it with iwasm application: +```bash +iwasm --native-lib= +``` + +Refer to [native lib sample](../samples/native-lib) for more details. + + +## Buffer address conversion and boundary check + +A WebAssembly sandbox ensures applications only access to its own memory with a private address space. When passing a pointer address from WASM to native, the address value must be converted to native address before the native function can access it. It is also the native world's responsibility to check the buffer length is not over its sandbox boundary. + + + +The signature letter '$', '\*' and '\~' help the runtime do automatic address conversion and buffer boundary check, so the native function directly uses the string and buffer address. **Notes**: if '\*' is not followed by '\~', the native function should not assume the length of the buffer is more than 1 byte. + + + +As function parameters are always passed in 32 bits numbers, you can also use 'i' for the pointer type argument, then you must do all the address conversion and boundary checking in your native function. For example, if you change the foo2 signature to "(iii)", then you will implement the native part as the following sample: + +```c +// +// If the function signature used i32 data type ("i") +// for buffer address or string parameters, here +// is how to do address conversion and boundary check manually +// +void foo2(wasm_exec_env_t exec_env, + uint32 msg_offset, + uint32 buffer_offset, + int32 buf_len) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + char *buffer; + char * msg ; + + // do boundary check + if (!wasm_runtime_validate_app_str_add(msg_offset)) + return 0; + + if (!wasm_runtime_validate_app_addr((uint64)buffer_offset, (uint64)buf_len)) + return; + + // do address conversion + buffer = wasm_runtime_addr_app_to_native((uint64)buffer_offset); + msg = wasm_runtime_addr_app_to_native((uint64)msg_offset); + + strncpy(buffer, msg, buf_len); +} +``` + + + + + +## Sandbox security attention + +The runtime builder should ensure not broking the memory sandbox when exporting the native function to WASM. + +A ground rule: + +- Do the pointer address conversion in the native API if "$\*" is not used for the pointer in the function signature + +A few recommendations: + +- Never pass any structure/class object pointer to native (do data serialization instead) +- Never pass a function pointer to the native + +Note: while not recommended here, nothing prevents you from passing +structure/function pointers as far as the native API is aware of +and careful about the ABI used in the wasm module. For example, +C function pointers are usually represented as table indexes which +the native API can call with wasm_runtime_call_indirect() or similar. +However, in this document, we don't recommend to implement your native +API that way unless necessary because it needs extra carefulness. + + +## Pass structured data or class object + +We must do data serialization for passing structured data or class objects between the two worlds of WASM and native. There are two serialization methods available in WASM as below, and yet you can introduce more like json, cbor etc. + +- [attributes container](https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-framework/app-native-shared/attr_container.c) +- [restful request/response](https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-framework/app-native-shared/restful_utils.c) + +Note the serialization library is separately compiled into WASM and runtime. And the source files are located in the folder "[wamr-app-framework/app-framework/app-native-shared](https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-framework/app-native-shared)“ where all source files will be compiled into both worlds. + + + +The following sample code demonstrates WASM app packs a response structure to buffer, then pass the buffer pointer to the native: + +```c +/*** file name: https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-framework/base/app/request.c ***/ + +void api_response_send(response_t *response) +{ + int size; + char * buffer = pack_response(response, &size); + if (buffer == NULL) + return; + + wasm_response_send(buffer, size); // calling exported native API + free_req_resp_packet(buffer); +} +``` + + + +The following code demonstrates the native API unpack the WASM buffer to local native data structure: + +```c +/*** file name: https://github.com/bytecodealliance/wamr-app-framework/blob/main/app-framework/base/native/request_response.c ***/ + +bool +wasm_response_send(wasm_exec_env_t exec_env, char *buffer, int size) +{ + if (buffer != NULL) { + response_t response[1]; + + if (NULL == unpack_response(buffer, size, response)) + return false; + + am_send_response(response); + + return true; + } + + return false; +} +``` + + +## Native API implementation notes + +### Async thread termination + +When threading support is enabled, a thread can be requested to terminate +itself either explicitly by the `wasm_runtime_terminate` API or implicitly +by cluster-wide activities like WASI `proc_exit`. +If a call to your native function can take long or even forever, +you should make it to respond to termination requests in a timely manner. +There are a few implementation approaches for that: + +* Design your API so that it always returns in a timely manner. + This is the most simple approach when possible. + +* Make your native function check the cluster state by calling + `wasm_cluster_is_thread_terminated` from time to time. + If `wasm_cluster_is_thread_terminated` returns true, make your + native function return. + Note: as of writing this, `wasm_cluster_is_thread_terminated` is not + exported as a runtime API. + +* If your native function is a simple wrapper of blocking system call, + you can probably use the `wasm_runtime_begin_blocking_op` and + `wasm_runtime_end_blocking_op` runtime API. + See the comment in `wamr_export.h` for details. + +* If your native function is very complex, it might be simpler to put the + guts of it into a separate worker process so that it can be cleaned up + by simply killing it. diff --git a/wasm-micro-runtime/doc/linux_sgx.md b/wasm-micro-runtime/doc/linux_sgx.md new file mode 100644 index 0000000..e7a3275 --- /dev/null +++ b/wasm-micro-runtime/doc/linux_sgx.md @@ -0,0 +1,271 @@ + +Build and Port WAMR vmcore (iwasm) for Linux SGX +================================================ + +Build WAMR vmcore (iwasm) for Linux SGX +--------------------------------------- + +First of all please install the [Intel SGX SDK](https://software.intel.com/en-us/sgx/sdk), v2.8 or later is required, and it is recommended to install the SDK to /opt/intel/sgxsdk. + +After installing the dependencies, build the source code: +``` Bash +source /environment +cd product-mini/platforms/linux-sgx/ +mkdir build && cd build +cmake .. +make +``` + +By default the `fast interpreter` and `AOT` is enabled. If to enable `Fast JIT`, run: +```Bash +mkdir build && cd build +cmake .. -DWAMR_BUILD_FAST_JIT=1 +make +``` + +This builds two libraries required by SGX application: + - libvmlib.a for Enclave part + - libvmlib_untrusted.a for App part + +**Note:** WAMR provides some features which can be easily configured by passing options to cmake, please see [WAMR vmcore cmake building configurations](./build_wamr.md#wamr-vmcore-cmake-building-configurations) for the details. Currently in Linux SGX, fast interpreter, AOT, libc-builtin, libc-WASI and lib-pthread are enabled by default. + +Then build the enclave sample: +``` Bash +source /environment +cd enclave-sample +make +``` + +**Note:** By default, the generated SGX application assumes it is signed with production key and running on simulation mode. The user can explicitly specify the relative variables in commandline to overwrite the default settings. For example, to build a debug enclave, please build the enclave with `make SGX_DEBUG=1`. To build the enclave running on a hardware-based SGX platform, execute `make SGX_MODE=HW`. + +The binary file iwasm will be generated. To run the sample: + +``` Bash +source /environment +iwasm [-options] wasm_file [args...] +or: +iwasm [-options] aot_file [args...] +``` + +### Minimal build +The libc-WASI and lib-pthread features require a lot of ocalls, if you don't need so much ocalls in your application, you can use the `minimal` version + +``` Bash +# replace the build files with minimal version +cd product-mini/platforms/linux-sgx/ +cp CMakeLists_minimal.txt CMakeLists.txt +cp enclave-sample/Makefile_minimal enclave-sample/Makefile +cp enclave-sample/Enclave/Enclave_minimal.edl enclave-sample/Enclave/Enclave.edl +# follow the building process above +``` + +Port WAMR vmcore for Linux SGX +------------------------------ + +The enclave-sample creates a sample to embed wamr vmlib of Enclave part and App part to an SGX application. To port WAMR vmcore lib to SGX application, there are some steps to do: + +**Step 1: Add "sgx_wamr.edl" and "sgx_pthread.edl" into EDL file, e.g. Enclave.edl:** +> This step is not required in minimal version + +```bash +from "sgx_pthread.edl" import *; +from "sgx_wamr.edl" import *; +``` + +The sgx_wamr.edl is under ${WAMR_ROOT}/core/shared/platform/linux-sgx, so please **add it to the search path list** when generating Enclave_u.c and Enclave_t.c from Enclave.edl: + +```bash +@cd App && $(SGX_EDGER8R) --untrusted ../Enclave/Enclave.edl \ + --search-path ../Enclave \ + --search-path $(SGX_SDK)/include \ + --search-path $(WAMR_ROOT)/core/shared/platform/linux-sgx +``` + +```bash +@cd Enclave && $(SGX_EDGER8R) --trusted ../Enclave/Enclave.edl \ + --search-path ../Enclave \ + --search-path $(SGX_SDK)/include \ + --search-path $(WAMR_ROOT)/core/shared/platform/linux-sgx +``` + +**Step 2: Link libvmlib.a to Enclave part and link libvmlib_untrusted.a to App part:** +> libvmlib_untrusted.a is not required in minimal version + +```makefile +Enclave_Link_Flags := ... libvmlib.a ... +``` + +```makefile +App_Link_Flags := ... libvmlib_untrusted.a ... +``` + +**And link SGX pthread lib to Enclave part:** +> SGX pthread lib is not required in minimal version + +```makefile +Enclave_Link_Flags := ... -lsgx_pthread ... +``` + +**Step 3: Add WAMR folders and SGX SDK folders to Enclave include path:** + +```makefile +Enclave_Include_Paths := ... -I$(WAMR_ROOT)/core/iwasm/include \ + -I$(WAMR_ROOT)/core/shared/utils \ + -I$(WAMR_ROOT)/core/shared/platform/linux-sgx \ + -I$(SGX_SDK)/include \ + -I$(SGX_SDK)/include/tlibc \ + -I$(SGX_SDK)/include/stlport +``` + +**Step 4: Configure reserved memory and thread info in file Enclave config file (e.g. Enclave.config.xml) to support WAMR AOT and multi-thread, e.g:** + +```xml +0x400000 +1 +10 +``` + +**Step 5: To support log output and os_printf() function in Enclave, please implement an ocall_print function, e.g. in Enclave.edl, add:** + +```cpp +untrusted { + void ocall_print([in, string]const char* str); +}; +``` + +In App part, add: + +```cpp +void +ocall_print(const char* str) +{ + printf("%s", str); +} +``` + +And in Enclave part, set the print function: + +```cpp +#include "wasm_export.h" +#include "bh_platform.h" + +extern "C" { + typedef void (*os_print_function_t)(const char* message); + extern void os_set_print_function(os_print_function_t pf); + + void + enclave_print(const char *message) + { + ocall_print(message); + } +} + +// In the beginning of Enclave initialization, add: +os_set_print_function(enclave_print); +``` + +Embed WAMR vmcore in Linux SGX +------------------------------ + +Normally we can embed WAMR vmcore in Linux SGX by calling the vmcore exported API's, see [Embed WAMR guide](./embed_wamr.md) for the details. And the the ecall_iwasm_main() function in file Enclave.cpp of enclave-sample also provides sample to invoke wasm app main function with wasm file buffer: + +```cpp +void +ecall_iwasm_main(uint8_t *wasm_file_buf, uint32_t wasm_file_size); +``` + +The enclave-sample also wraps an ecall function to receive commands from App to Enclave, and handle the commands in Enclave by calling the related WAMR vmcore API. The commands and related API's are: + +```cpp +typedef enum EcallCmd { + CMD_INIT_RUNTIME = 0, /* wasm_runtime_init/full_init() */ + CMD_LOAD_MODULE, /* wasm_runtime_load() */ + CMD_INSTANTIATE_MODULE, /* wasm_runtime_instantiate() */ + CMD_LOOKUP_FUNCTION, /* wasm_runtime_lookup_function() */ + CMD_CREATE_EXEC_ENV, /* wasm_runtime_create_exec_env() */ + CMD_CALL_WASM, /* wasm_runtime_call_wasm */ + CMD_EXEC_APP_FUNC, /* wasm_application_execute_func() */ + CMD_EXEC_APP_MAIN, /* wasm_application_execute_main() */ + CMD_GET_EXCEPTION, /* wasm_runtime_get_exception() */ + CMD_DEINSTANTIATE_MODULE, /* wasm_runtime_deinstantiate() */ + CMD_UNLOAD_MODULE, /* wasm_runtime_unload() */ + CMD_DESTROY_RUNTIME, /* wasm_runtime_destroy() */ + CMD_SET_WASI_ARGS, /* wasm_runtime_set_wasi_args() */ + CMD_SET_LOG_LEVEL, /* bh_log_set_verbose_level() */ +}; +``` + +SGX Intel Protected File System +------------------------------- +Intel SGX introduced a feature called [Intel Protection File System Library (IPFS)](https://www.intel.com/content/www/us/en/developer/articles/technical/overview-of-intel-protected-file-system-library-using-software-guard-extensions.html) to create, operate and delete files inside the enclave. +WAMR supports the mapping of IPFS on WASI functions related to file interactions, providing seamless persistence with confidentiality and integrity to the hosted WebAssembly applications in the enclave. + +The usage of SGX IPFS is an optional feature. +To opt-in, the support of IPFS requires the following changes: + - set the flag `WAMR_BUILD_SGX_IPFS=1` when running `cmake`, + - the enclave must be linked with the trusted IPFS library (`-lsgx_tprotected_fs`), + - the application outside of the enclave must be linked with the untrusted IPFS library (`-lsgx_uprotected_fs`), + - the EDL file must include the following import statement: + +```edl +from "sgx_tprotected_fs.edl" import *; +``` + +When using the [enclave-sample](../product-mini/platforms/linux-sgx/enclave-sample/) project, setting the flag `WAMR_BUILD_SGX_IPFS=1` when running `cmake` enables these changes automatically. + + +### Verification of SGX IPFS +One can observe the usage of IPFS by running the [file sample](../samples/file/) WebAssembly application. +Enabling the SGX IPFS on this sample project leads to the generation of an encrypted text file. + + +### Mapping of WASI/POSIX to IPFS +This table summarizes how WASI is mapped to POSIX and IPFS. +Since IPFS is a subset of the WASI/POSIX, emulation is performed to fill the missing implementation. + +| WASI | POSIX | IPFS | +|------------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------| +| `fd_read` | `readv` | `sgx_fread` | +| `fd_write` | `writev` | `sgx_fwrite` | +| `fd_close` | `close` | `sgx_fclose` | +| `path_open` | `openat` | `sgx_fopen` | +| `fd_datasync` | `fsync` | `sgx_fflush` | +| `fd_tell` | `lseek` | `sgx_ftell` | +| `fd_filestat_set_size` | `ftruncate` | Shrinking files is not supported, nor emulated. Extending files is emulated using `sgx_fseek`/`sgx_ftell`/`sgx_fwrite`. | +| `fd_seek` | `lseek` | The POSIX and IPFS behaviors differ. Emulated using `sgx_fseek`/`sgx_ftell`/`sgx_fwrite`. | +| `fd_pwrite` | `pwrite` | Not supported. Emulated using `sgx_fseek`/`sgx_ftell`/`sgx_fwrite`. | +| `fd_pread` | `pread` | Not supported. Emulated using `sgx_fseek`/`sgx_ftell`/`sgx_fread`. | +| `fd_allocate` | `posix_fallocate` | Not supported. Emulated using `sgx_fseek`/`sgx_ftell`/`sgx_fwrite`/`sgx_fflush`. | + + +### Performance overheads +Many benchmarks have assessed the overheads caused by IPFS through WASI functions using Twine, an early and academic adaptation of WAMR in Intel SGX with WASI support. +The results can be found in [this paper](https://arxiv.org/abs/2103.15860). + +### Limitations +The threat model and the limitations of SGX IPFS can be found in [the official documentation](https://www.intel.com/content/dam/develop/external/us/en/documents/overviewofintelprotectedfilesystemlibrary.pdf). + + +Others +------ + +- Please add "-sgx" option when generating AoT file for SGX platform, e.g.: + + ```bash + wamrc -sgx -o test.aot test.wasm + ``` + +- The default max heap size of Enclave is 16 MB, it might be not enough when executing some workloads, please modify it in Enclave/Enclave.config.xml with a larger size when exception was thrown: + + ```bash + Exception: fail to enlarge memory. + or + Exception: allocate memory failed. + ``` + + Enclave/Enclave.config.xml, default max heap size is 16 MB: + + ```xml + 0x1000000 + ``` + diff --git a/wasm-micro-runtime/doc/memory_tune.md b/wasm-micro-runtime/doc/memory_tune.md new file mode 100644 index 0000000..05a1a63 --- /dev/null +++ b/wasm-micro-runtime/doc/memory_tune.md @@ -0,0 +1,32 @@ +# Memory model and memory usage tunning + +References: +- [Blog: Understand WAMR heap](https://bytecodealliance.github.io/wamr.dev/blog/understand-the-wamr-heaps/) +- [Blog: Understand WAMR stacks](https://bytecodealliance.github.io/wamr.dev/blog/understand-the-wamr-stacks/) + +## The memory model + +The memory model of WAMR can be basically described as below: + +
+ +Note: +- **global heap**: the heap to allocate memory for runtime data structures, including wasm module, wasm module instance, exec env, wasm operand stack and so on. It is initialized by `wasm_runtime_init` or `wasm_runtime_full_init`. And for `wasm_runtime_full_init`, developer can specify the memory allocation mode with `RuntimeInitArgs *init_args`: allocate memory from a user defined byte buffer, from user defined allocation function, or from the platform's os_malloc function. Refer to [wasm_export.h](../core/iwasm/include/wasm_export.h#L98-L141) and [Embedding WAMR guideline](./embed_wamr.md#the-runtime-initialization) for more details. And developer can use `wasm_runtime_malloc/wasm_runtime_free` to allocate/free memory from/to the global heap. +- **wasm operand stack**: the stack to store the operands required by wasm bytecodes as WebAssembly is based on a stack machine. If the exec_env is created by developer with `wasm_runtime_create_exec_env`, then its size is specified by `wasm_runtime_create_exec_env`, otherwise if the exec_env is created by runtime internally, e.g. by `wasm_application_execute_main` or `wasm_application_execute_func`, then the size is specified by `wasm_runtime_instantiate`. +- **linear memory**: a contiguous, mutable array of raw bytes. It is created with an initial size but might be grown dynamically. For most compilers, e.g. wasi-sdk, emsdk, rustc or asc, normally it includes three parts, data area, auxiliary stack area and heap area. For wasi-sdk, the initial/max size can be specified with `-Wl,--initial-memory=n1,--max-memory=n2`, for emsdk, the initial/max size can be specified with `-s INITIAL_MEMORY=n1 -s MAXIMUM_MEMORY=n2 -s ALLOW_MEMORY_GROWTH=1` or `-s TOTAL_MEMORY=n`, and for asc, they can be specified with `--initialMemory` and `--maximumMemory` flags. + - If the memory access boundary check with hardware trap feature is enabled, e.g. in Linux/MacOS/Windows x86-64 by default, the linear memory is allocated by `os_mmap` from virtual address space instead of global heap. +- **aux stack**: the auxiliary stack resides in linear memory to store some temporary data when calling wasm functions, for example, calling a wasm function with complex struct arguments. For wasi-sdk, the size can be specified with `-z stack-size=n`, for emsdk, the size can be specified with `-s TOTAL_STACK=n`. +- **app heap and libc heap**: the heap to allocate memory for wasm app, note that app heap is created only when the malloc/free functions (or __new/__release functions for AssemblyScript) are not exported and runtime can not detect the libc heap. To export the malloc/free functions, for wasi-sdk and emsdk, developer can use `-Wl,--export=malloc -Wl,--export=free` options, for asc, developer can use `--exportRuntime` option. For app heap, the size is specified by `wasm_runtime_instantiate`. It is recommended to export the malloc/free functions and disable app heap. However, if you are using [the old pthread implementation](./pthread_impls.md), you might need some workaround to avoid the libc heap as mentioned in [WAMR pthread library](./pthread_library.md). And developer can use `wasm_runtime_module_malloc/wasm_runtime_module_free` to allocate/free memory from/to app heap (or libc heap if malloc/free functions are exported). +- **__data_end global and __heap_base global**: two globals exported by wasm application to indicate the end of data area and the base address of libc heap. For WAMR, it is recommended to export them as when there are no possible memory grow operations, runtime will truncate the linear memory into the size indicated by `__heap_base`, so as to reduce the footprint, or at least one page (64KB) is required by linear memory. + +## Tune the memory usage + +Normally there are some methods to tune the memory usage: +- set the global heap size with `wasm_runtime_full_init` +- set the wasm operand stack size with `wasm_runtime_create_exec_env` or `wasm_runtime_instantiate` +- set the linear memory size +- set the auxiliary stack size +- export `malloc/free` functions to use libc heap and disable app heap +- set the app heap size with `wasm_runtime_instantiate` +- use `nostdlib` mode, add `-Wl,--strip-all`: refer to [How to reduce the footprint](./build_wasm_app.md#2-how-to-reduce-the-footprint) of building wasm app for more details +- use XIP mode, refer to [WAMR XIP (Execution In Place) feature introduction](./xip.md) for more details diff --git a/wasm-micro-runtime/doc/memory_usage.md b/wasm-micro-runtime/doc/memory_usage.md new file mode 100644 index 0000000..88b792f --- /dev/null +++ b/wasm-micro-runtime/doc/memory_usage.md @@ -0,0 +1,129 @@ +Memory usage estimation for a module +==================================== + +This document aims to provide information useful to make a rough estimation +of necessary memory to execute a WASM module. + +Instead of trying to cover every possible configurations, +the following configuration is assumed in this document: + +* Module is built with `wasi-sdk` +* Module is loaded with `wasm_runtime_load` +* AOT is used +* WASI is used +* libc heap is used +* app heap is not used +* The pthread implementation in `wasi-libc`, which is based on `wasi-threads` + (`WASM_ENABLE_LIB_WASI_THREADS`) might be used +* The another pthread implementation (`WASM_ENABLE_LIB_PTHREAD`) is not used + +Module +------ + +The memory to store the module binary is allocated by the embedder and +passed to `wasm_runtime_load`. +While WAMR owns the buffer, WAMR might make in-place modifications to +its contents. + +Loaded module and its instances +------------------------------- + +Many of data structures for module and instances are allocated from +the global heap. (aka. `wasm_runtime_malloc`) + +AOT code section +---------------- + +Memory to load AOT machine code section. + +Because this memory needs to be executable, depending on platforms, +it's allocated from a separate allocator. +For example, `mmap` and `mprotect` are used on POSIX-like platforms. + +Linear memory +------------- + +A WASM linear memory is either shared or non-shared. + +A WASM linear memory has `min` and `max` sizes. +(They correspond to `wasm-ld`'s `--init-memory` and `--max-memory` options.) +They are in the number of WASM pages, each of which is of 65536 bytes. +The `max` is optional for non-shared memory. When omitted, it effectivily +means unlimited. + +The linear memory is allocated via `os_mmap` and `os_mem_commit`/`os_mprotect`. + +The `min` size of memory is allocated on instantiation. +It can later grow up to the `max` size via the `memory.grow` instruction. + +Libc heap +--------- + +The libc heap is the last (highest address) part of linear memory, +which might be dynamically grown with `memory.grow` instruction, when +necessary to serve memory allocations within the module. + +App heap +-------- + +Not used for the above mentioned configuration. + +You can safely disable the app heap creation by specifying `0` for +the `heap_size` argument of `wasm_runtime_instantiate`. +(It's automatically disabled if malloc/free are exported from the module.) + +WASM stack +---------- + +Operand stack is not used for AOT. + +However, a small amount of WASM stack is used for call frames when +certain features are enabled. +(`WASM_ENABLE_DUMP_CALL_STACK` or `WASM_ENABLE_PERF_PROFILING`) + +It's allocated from the global heap. + +You can specify its size with the `stack_size` argument of +`wasm_runtime_instantiate` and `wasm_runtime_create_exec_env`. +(1 is the minimum because 0 means the default.) + +AUX stack (aka. C shadow stack) +------------------------------- + +For the main thread, it's a part of the linear memory, +between `__data_end` and `__heap_base` symbols. +You can control the size of this stack with `wasm-ld`'s +`-z stack-size` option. + +For threads created by `pthread_create`, libc allocates the stack for +them dynamically from the libc heap. +The size of this stack is inherited from the main thread's one +unless overwritten with `pthread_attr_setstacksize` etc. + +WAMR tries to detect overflow/underflow when updating the stack pointer +global. For threads created by `pthread_create`, the detection mechanism +is disabled as of writing this. + +Native stack +------------ + +The stack of the host environment thread which runs WAMR. + +For threads created by `pthread_create`, WAMR automatically creates +host threads to run those WASM threads. The stack size of these host +threads are controlled by a build-time configuration. +(`APP_THREAD_STACK_SIZE_DEFAULT`) + +In some configurations, runtime overflow can be detected using hardware traps. +(`OS_ENABLE_HW_BOUND_CHECK`) + +In some configurations, explicit overflow detection logic can be emitted +into AOT modules themselves. (cf. `os_thread_get_stack_boundary`, +`check_stack_boundary`, `wamrc --stack-bounds-checks=1/0`) + +Memory profiling +================ + +You can collect and dump detailed information about memory usage +by actually running a module with the `WASM_ENABLE_MEMORY_PROFILING` +build-time option. diff --git a/wasm-micro-runtime/doc/multi_module.md b/wasm-micro-runtime/doc/multi_module.md new file mode 100644 index 0000000..ac9e3bb --- /dev/null +++ b/wasm-micro-runtime/doc/multi_module.md @@ -0,0 +1,160 @@ +# Multiple Modules as Dependencies + +A WASM module can _import_ _functions_, _globals_, _memories_ and _tables_ from other modules as dependencies. A module can also _export_ those entities for other modules like a library. + +WAMR loads all dependencies recursively according to the _import section_ of a module. + +> WAMR only implements the load-time dynamic linking. Please refer to [dynamic linking](https://webassembly.org/docs/dynamic-linking/) for more details. + +WAMR follows [WASI Command/Reactor Model](https://github.com/WebAssembly/WASI/blob/main/legacy/application-abi.md#current-unstable-abi). The WASI model separates modules into commands and reactors. A Command is the main module that requires exports of reactors(submodules). + +if `WASM_ENABLE_LIBC_WASI` is enabled, any module imports a WASI APIs, like `(import "wasi_snapshot_preview1" "XXX")`, should follow restrictions of the _WASI application ABI_: + +- a main module(a command) should include `_start()` +- a submodule(a reactor) should include `_initialize()` +- both a command and a reactor should include an exported `memory` + +## Multi-Module Related APIs + +### Register a module + +```c +bool +wasm_runtime_register_module(const char *module_name, + wasm_module_t module, + char *error_buf, + uint32_t error_buf_size); +``` + +It is used to register a _module_ with a _module_name_ to WASM runtime, especially for the main module, which is loaded by `wasm_runtime_load()` and doesn't have a chance to tell runtime its _module name_. + +WAMR will get submodules' names(according to the _import section_ of the main module) and load .wasm files from the filesystem or stream and then register them internally. + +### Find a registered module + +```c +wasm_module_t +wasm_runtime_find_module_registered( + const char *module_name); +``` + +It is used to check whether a module with a given _module_name_ has been registered before or not. Return the module if yes. + +### Module reader and destroyer + +```c +typedef bool (*module_reader)(const char *module_name, + uint8_t **p_buffer, + uint32_t *p_size); + +typedef void (*module_destroyer)(uint8_t *buffer, + uint32_t size); + +void +wasm_runtime_set_module_reader(const module_reader reader, + const module_destroyer destroyer); +``` + +WAMR hopes that the native host or embedding environment loads/unloads the module WASM files by themselves and only passes runtime the binary content without worrying about filesystem or storage issues. `module_reader` and `module_destroyer` are two callbacks called when dynamic-loading/unloading submodules. Developers must implement the two callbacks by themselves. + +### Call function of a submodule + +```c +wasm_function_inst_t +wasm_runtime_lookup_function(wasm_module_inst_t const module_inst, + const char *name); +``` + +Multi-module allows one to look up an exported function of a submodule. There are two ways to indicate the function _name_: + +- parent function name only by default, used to look up the function of the parent module +- submodule name, function name and two $ symbols, e.g. `$submodule_name$function_name`, used to lookup function of submodule +- `signature` can be NULL + +## Example + +### Attributes in C/C++ + +Suppose there are three C files, _mA.c_, _mB.c_ and _mC.c_. Each of them exports functions and imports from others except mA. + +import/export with two kinds of `__attribute__`: + +- `__attribute__((import_module("MODULE_NAME"))) __attribute__((import_name("FUNCTION_NAME")))`. to indicate dependencies of the current module. + +- `__attribute__((export_name("FUNCTION_NAME")))`. to expose functions. + +```C +// mA.c +__attribute__((export_name("A1"))) int +A1() +{ + return 11; +} +``` + +```C +// mB.c +__attribute__((import_module("mA"))) +__attribute__((import_name("A1"))) extern int +A1(); + +__attribute__((export_name("B1"))) int +B1() +{ + return 21; +} +``` + +### Compile Options + +to generate a wasm module as a command + +``` +$ /path/to/wasi-sdk/bin/clang -o command.wasm main_module.c +``` + +to generate a wasm module as a reactor + +``` +$ /path/to/wasi-sdk/bin/clang -mexec-model=reactor -o reactor.wasm submodule.c +``` + +In the above case, _mA_ and _mB_ are reactors(submodules), _mC_ is the command(main module). Their _import relationships_ will be like: + +![import relationships](./pics/multi_module_pic1.png) + +### libvmlib + +We need to enable _WAMR_BUILD_MULTI_MODULE_ option when building WAMR vmlib. Please ref to [Build WAMR core](./build_wamr.md) for a thoughtful guide. + +### code + +After all the preparation, we can call some functions from native code with APIs + +First, create two callbacks to load WASM module files into memory and unload them later + +```c +static bool +module_reader_cb(const char *module_name, uint8 **p_buffer, uint32 *p_size) +{ + // ... + *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_path, p_size); + // ... +} + +static void +module_destroyer_cb(uint8 *buffer, uint32 size) +{ + BH_FREE(buffer); +} +``` + +Second, create a large buffer and tell WAMR malloc any resource only from this buffer later. + +More details + +```c +static char sandbox_memory_space[10 * 1024 * 1024] = { 0 }; +``` + +Third, put all together. Please refer to [main.c](../samples/multi-module/src/main.c) diff --git a/wasm-micro-runtime/doc/other_wasm_compilers.md b/wasm-micro-runtime/doc/other_wasm_compilers.md new file mode 100644 index 0000000..5aa505a --- /dev/null +++ b/wasm-micro-runtime/doc/other_wasm_compilers.md @@ -0,0 +1,78 @@ + +## Use clang compiler + +The recommended method to build a WASM binary is to use clang compiler ```clang-8```. You can refer to [apt.llvm.org](https://apt.llvm.org) for the detailed instructions. Here are referenced steps to install clang-8 in Ubuntu 16.04 and Ubuntu 18.04. + +(1) Add source to your system source list from llvm website + +For Ubuntu 16.04, add the following lines to /etc/apt/sources.list: + +``` Bash +deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main +deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main +# 8 +deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main +deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main +# 9 +deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main +deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main +``` + +For Ubuntu 18.04, add the following lines to /etc/apt/sources.list: + +``` Bash +# i386 not available +deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main +deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main +# 8 +deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main +deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main +# 9 +deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main +deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main +``` + +(2) Download and install clang-8 tool-chain using following commands: + +``` Bash +sudo wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - +# Fingerprint: 6084 F3CF 814B 57C1 CF12 EFD5 15CF 4D18 AF4F 7421 +sudo apt-get update +sudo apt-get install llvm-8 lld-8 clang-8 +``` + +(3) Create a soft link under /usr/bin: + +``` Bash +cd /usr/bin +sudo ln -s wasm-ld-8 wasm-ld +``` + +(4) Use the clang-8 command below to build the WASM C source code into the WASM binary. + +``` Bash +clang-8 --target=wasm32 -O3 \ + -z stack-size=4096 -Wl,--initial-memory=65536 \ + -Wl,--allow-undefined,--export=main \ + -Wl,--strip-all,--no-entry -nostdlib \ + -o test.wasm test.c +``` + +You will get ```test.wasm``` which is the WASM app binary. + +## Using Docker + +Another method availble is using [Docker](https://www.docker.com/). We assume you've already configured Docker (see Platform section above) and have a running interactive shell. Currently the Dockerfile only supports compiling apps with clang, with Emscripten planned for the future. + +Use the clang-8 command below to build the WASM C source code into the WASM binary. + +``` Bash +clang-8 --target=wasm32 -O3 \ + -z stack-size=4096 -Wl,--initial-memory=65536 \ + -Wl,--allow-undefined,--export=main \ + -Wl,--strip-all,--no-entry -nostdlib \ + -o test.wasm test.c +``` + +You will get ```test.wasm``` which is the WASM app binary. + diff --git a/wasm-micro-runtime/doc/perf_tune.md b/wasm-micro-runtime/doc/perf_tune.md new file mode 100644 index 0000000..8932797 --- /dev/null +++ b/wasm-micro-runtime/doc/perf_tune.md @@ -0,0 +1,308 @@ +# Tune the performance of running wasm/aot file + +Normally there are some methods to tune the performance: + +## 1. Use `wasm-opt` tool + +Download the [binaryen release](https://github.com/WebAssembly/binaryen/releases), and use the `wasm-opt` tool in it to optimize the wasm file, for example: + +```bash +wasm-opt -O4 -o test_opt.wasm test.wasm +``` + +## 2. Enable `simd128` option when compiling wasm source files + +WebAssembly [128-bit SIMD](https://github.com/WebAssembly/simd) is supported by WAMR on x86-64 and aarch64 targets, enabling it when compiling wasm source files may greatly improve the performance. For [wasi-sdk](https://github.com/WebAssembly/wasi-sdk) and [emsdk](https://github.com/emscripten-core/emsdk), please add `-msimd128` flag for `clang` and `emcc/em++`: + +```bash +/opt/wasi-sdk/bin/clang -msimd128 -O3 -o + +emcc -msimd128 -O3 -o +``` + +## 3. Enable segue optimization for wamrc when generating the aot file + +[Segue](https://plas2022.github.io/files/pdf/SegueColorGuard.pdf) is an optimization technology which uses x86 segment register to store the WebAssembly linear memory base address, so as to remove most of the cost of SFI (Software-based Fault Isolation) base addition and free up a general purpose register, by this way it may: + +- Improve the performance of JIT/AOT +- Reduce the footprint of JIT/AOT, the JIT/AOT code generated is smaller +- Reduce the compilation time of JIT/AOT + +Currently it is only supported on linux x86-64, developer can use `--enable-segue=[]` for wamrc: + +```bash +wamrc --enable-segue -o aot_file wasm_file +# or +wamrc --enable-segue=[] -o aot_file wasm_file +``` + +`flags` can be: i32.load, i64.load, f32.load, f64.load, v128.load, i32.store, i64.store, f32.store, f64.store and v128.store, use comma to separate them, e.g. `--enable-segue=i32.load,i64.store`, and `--enable-segue` means all flags are added. + +> Note: Normally for most cases, using `--enable-segue` is enough, but for some cases, using `--enable-segue=` may be better, for example for CoreMark benchmark, `--enable-segue=i32.store` may lead to better performance than `--enable-segue`. + +## 4. Enable segue optimization for iwasm when running wasm file + +Similar to segue optimization for wamrc, run: + +```bash +iwasm --enable-segue wasm_file (iwasm is built with llvm-jit enabled) +# or +iwasm --enable-segue=[] wasm_file +``` + +> Note: Currently it is only supported on linux x86-64. + +## 5. Use the AOT static PGO method + +LLVM PGO (Profile-Guided Optimization) allows the compiler to better optimize code for how it actually runs. WAMR supports AOT static PGO, currently it is tested on Linux x86-64 and x86-32. The basic steps are: + +1. Use `wamrc --enable-llvm-pgo -o ` to generate an instrumented aot file. + +2. Compile iwasm with `cmake -DWAMR_BUILD_STATIC_PGO=1` and run `iwasm --gen-prof-file= ` to generate the raw profile file. + +> Note: Directly dumping raw profile data to file system may be unsupported in some environments, developer can dump the profile data into memory buffer instead and try outputting it through network (e.g. uart or socket): + +```C +uint32_t +wasm_runtime_get_pgo_prof_data_size(wasm_module_inst_t module_inst); + +uint32_t +wasm_runtime_dump_pgo_prof_data_to_buf(wasm_module_inst_t module_inst, char *buf, uint32_t len); +``` + +3. Install or compile `llvm-profdata` tool，refer to [here](../tests/benchmarks/README.md#install-llvm-profdata) for the details. + +4. Run `llvm-profdata merge -output= ` to merge the raw profile file into the profile file. + +5. Run `wamrc --use-prof-file= -o ` to generate the optimized aot file. + +6. Run the optimized aot_file: `iwasm `. + +Developer can refer to the `test_pgo.sh` files under each benchmark folder for more details, e.g. [test_pgo.sh](../tests/benchmarks/coremark/test_pgo.sh) of CoreMark benchmark. + +## 6. Disable the memory boundary check + +Please notice that this method is not a general solution since it may lead to security issues. And only boost the performance for some platforms in AOT mode and don't support hardware trap for memory boundary check. + +1. Build WAMR with `-DWAMR_CONFIGUABLE_BOUNDS_CHECKS=1` option. + +2. Compile AOT module by wamrc with `--bounds-check=0` option. + +3. Run the AOT module by iwasm with `--disable-bounds-checks` option. + +> Note: The size of AOT file will be much smaller than the default, and some tricks are possible such as let the wasm application access the memory of host os directly. +> Please notice that if this option is enabled, the wasm spec test will fail since it requires the memory boundary check. For example, the runtime will crash when accessing the memory out of the boundary in some cases instead of throwing an exception as the spec requires. + +You should only use this method for well tested wasm applications and make sure the memory access is safe. + +## 7. Use linux-perf + +Linux perf is a powerful tool to analyze the performance of a program, developer can use it to find the hot functions and optimize them. It is one profiler supported by WAMR. In order to use it, you need to add `--perf-profile` while running _iwasm_. By default, it is disabled. + +> [!CAUTION] +> For now, only llvm-jit mode and aot mode supports linux-perf. + +Here is a basic example, if there is a Wasm application _foo.wasm_, you'll execute. + +``` +$ perf record --output=perf.data.raw -- iwasm --enable-linux-perf foo.wasm +``` + +This will create a _perf.data_ and +- a _jit-xxx.dump_ under _~/.debug/jit/_ folder if running llvm-jit mode +- or _/tmp/perf-.map_ if running AOT mode + + +This file is WAMR generated. It contains information which includes jitted(precompiled) code addresses in memory, names of jitted (precompiled) functions which are named as *aot_func#N* and so on. + +If running with llvm-jit mode, the next thing is to merge _jit-xxx.dump_ file into the _perf.data_. + +``` +$ perf inject --jit --input=perf.data.raw --output=perf.data +``` + +This step will create a lot of _jitted-xxxx-N.so_ which are ELF images for all JIT functions created at runtime. + +> [!TIP] +> add `-v` and check if there is output likes _write ELF image ..._. If yes, it means above merge is successful. + +Finally, you can use _perf report_ to analyze the performance. + +``` +$ perf report --input=perf.data +``` + +> [!CAUTION] +> Using release builds of llvm and iwasm will produce "[unknown]" functions in the call graph. It is not only because +> of the missing debug information, but also because of removing frame pointers. To get the complete result, please +> use debug builds of both llvm and iwasm. +> +> Wasm functions names are stored in _the custom name section_. Toolchains always generate the custom name section in both debug and release builds. However, the custom name section is stripped to pursue smallest size in release build. So, if you want to get a understandable result, please search the manual of toolchain to look for a way to keep the custom name section. +> +> For example, with EMCC, you can add `-g2`. +> +> If not able to get the context of the custom name section, WAMR will use `aot_func#N` to represent the function name. `N` is from 0. `aot_func#0` represents the first _not imported wasm function_. + +### 7.1 Flamegraph + +[Flamegraph](https://www.brendangregg.com/flamegraphs.html) is a powerful tool to visualize stack traces of profiled software so that the most frequent code-paths can be identified quickly and accurately. In order to use it, you need to [capture graphs](https://github.com/brendangregg/FlameGraph#1-capture-stacks) when running `perf record` + +``` +$ perf record -k mono --call-graph=fp --output=perf.data.raw -- iwasm --enable-linux-perf foo.wasm +``` + +If running with llvm-jit mode, merge the _jit-xxx.dump_ file into the _perf.data.raw_. + +``` +$ perf inject --jit --input=perf.data.raw --output=perf.data +``` + +Generate the stack trace file. + +``` +$ perf script > out.perf +``` + +[Fold stacks](https://github.com/brendangregg/FlameGraph#2-fold-stacks). + +``` +$ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded +``` + +[Render a flamegraph](https://github.com/brendangregg/FlameGraph#3-flamegraphpl) + +``` +$ ./FlameGraph/flamegraph.pl out.folded > perf.foo.wasm.svg +``` + +> [!TIP] +> use `grep` to pick up folded stacks you are interested in or filter out something. +> +> For example, if just want to see stacks of wasm functions, you can use +> +> ```bash +> # only jitted functions +> $ grep "wasm_runtime_invoke_native" out.folded | ./FlameGraph/flamegraph.pl > perf.foo.wasm.only.svg +> ``` + +> [!TIP] +> use [trans_wasm_func_name.py](../test-tools/trans-jitted-func-name/trans_wasm_func_name.py) to translate jitted function +> names to its original wasm function names. It requires _wasm-objdump_ in _wabt_ and _name section_ in the .wasm file +> +> The input file is the output of `./FlameGraph/stackcollapse-perf.pl`. +> +> ```bash +> python trans_wasm_func_name.py --wabt_home --folded out.folded <.wasm> +> ``` +> +> Then you will see a new file named _out.folded.translated_ which contains the translated folded stacks. +> All wasm functions are translated to its original names with a prefix like "[Wasm]" + +## 8. Refine the calling processes between host native and wasm application + +In some scenarios, there may be lots of callings between host native and wasm application, e.g. frequent callings to AOT/JIT functions from host native or frequent callings to host native from AOT/JIT functions. It is important to refine these calling processes to speedup them, WAMR provides several methods: + +### 8.1 Refine callings to native APIs registered by `wasm_runtime_register_natives` from AOT code + +When wamrc compiles the wasm file to AOT code, it may generate LLVM IR to call the native API from an AOT function, and if it doesn't know the native API's signature, the generated LLVM IR has to call the runtime API `aot_invoke_native` to invoke the native API, which is a relatively slow way. If developer registers native APIs during execution by calling `wasm_runtime_register_natives` or by `iwasm --native-lib=`, then developer can also register native APIs with the same signatures to the AOT compiler by `wamrc --native-lib=`, so as to let the AOT compiler pre-know the native API's signature, and generate optimized LLVM IR to quickly call to the native API. + +The below sample registers an API `int test_add(int, int)` to the AOT compiler: + +```C +/* test_add.c */ + +#include "wasm_export.h" + +static int +test_add_wrapper(wasm_exec_env_t exec_env, int x, int y) { + return 0; /* empty function is enough */ +} + +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } + +static NativeSymbol native_symbols[] = { + REG_NATIVE_FUNC(test_add, "(ii)i") +}; + +uint32_t +get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) +{ + *p_module_name = "env"; + *p_native_symbols = native_symbols; + return sizeof(native_symbols) / sizeof(NativeSymbol); +} +``` +```bash +# build native lib +gcc -O3 -fPIC -shared -I /core/iwasm/include -o libtest_add.so test_add.c +# register native lib to aot compiler +wamrc --native-lib=./libtest_add.so -o +``` + +> Note: no need to do anything for LLVM JIT since the native APIs must have been registered before execution and JIT compiler already knows the native APIs' signatures. + +### 8.2 Refine callings to native APIs registered by wasm-c-api `wasm_instance_new` from AOT code + +In wasm-c-api mode, when the native APIs are registered by `wasm_instance_new(..., imports, ...)`, developer can use `wamrc --invoke-c-api-import` option to generate the AOT file, which treats the unknown import function as wasm-c-api import function and generates optimized LLVM IR to speedup the calling process. + +> Note: no need to do anything for LLVM JIT since the similar flag has been set to JIT compiler in wasm-c-api `wasm_engine_new` when LLVM JIT is enabled. + +### 8.3 Refine callings to AOT/JIT functions from host native + +Currently by default WAMR runtime has registered many quick AOT/JIT entries to speedup the calling processes to call AOT/JIT functions from host native, as long as developer doesn't disable it by using `cmake -DWAMR_BUILD_QUICK_AOT_ENTRY=0` or setting the compiler macro `WASM_ENABLE_QUICK_AOT_ENTRY` to 0 in the makefile. These quick AOT/JIT entries include: + +1. wasm function contains 0 to 4 arguments and 0 to 1 results, with the type of each argument is i32 or i64 and the type of result is i32, i64 or void. These functions are like: + +```C +// no argument +i32 foo(), i64 foo(), void foo() +// one argument, each argument is i32 or i64 +i32 foo(i32/i64), i64 foo(i32/i64), void(i32/i64) +// two arguments, each argument is i32 or i64 +i32 foo(i32/i64, i32/i64), i64 foo(i32/i64, i32/i64), void(i32/i64, i32/i64) +// three arguments, each argument is i32 or i64 +i32 foo(i32/i64, i32/i64, i32/i64), i64 foo(i32/i64, i32/i64, i32/i64), void(i32/i64, i32/i64, i32/i64) +// four arguments, each argument is i32 or i64 +i32 foo(i32/i64, i32/i64, i32/i64, i32/i64) +i64 foo(i32/i64, i32/i64, i32/i64, i32/i64) +void(i32/i64, i32/i64, i32/i64, i32/i64) +``` + +2. wasm function contains 5 arguments and 0 to 1 results, with the type of each argument is i32 and the type of result is i32, i64 or void. These functions are like: + +```C +i32 foo(i32, i32, i32, i32, i32) +i64 foo(i32, i32, i32, i32, i32) +void foo(i32, i32, i32, i32, i32) +``` + +To speedup the calling processes, developer had better ensure that the signatures of the wasm functions to expose are like above, or add some conversions to achieve it. For example, if a wasm function to call is `f32 foo(f32)`, developer can define a new function `i32 foo1(i32)` like below and export it: +```C +int32 foo1(int32 arg_i32) +{ + float arg_f32 = *(float *)&arg_i32; + float res_f32 = foo(f32); + int32 res_i32 = *(int32 *)&res_i32; + return res_i32; +} +``` +And in the host embedder: +``` + uint32 argv[2]; + float arg_f32 = ...; /* argument to foo */ + float res_f32; + bool ret; + + argv[0] = *(uint32 *)&arg_f32; + func = wasm_runtime_lookup_function(module_inst, "foo1"); + ret = wasm_runtime_call_wasm(exec_env, func, 1, argv); + if (!ret) { + /* handle exception */ + printf("%s\n", wasm_runtime_get_exception(module_inst)); + } + else { + /* the return value is stored in argv[0] */ + res_f32 = *(float *)&argv[0]; + } +``` diff --git a/wasm-micro-runtime/doc/pics/app_framework.PNG b/wasm-micro-runtime/doc/pics/app_framework.PNG new file mode 100644 index 0000000..802c45b Binary files /dev/null and b/wasm-micro-runtime/doc/pics/app_framework.PNG differ diff --git a/wasm-micro-runtime/doc/pics/embed.PNG b/wasm-micro-runtime/doc/pics/embed.PNG new file mode 100644 index 0000000..68340ee Binary files /dev/null and b/wasm-micro-runtime/doc/pics/embed.PNG differ diff --git a/wasm-micro-runtime/doc/pics/extend_library.PNG b/wasm-micro-runtime/doc/pics/extend_library.PNG new file mode 100644 index 0000000..abf10ca Binary files /dev/null and b/wasm-micro-runtime/doc/pics/extend_library.PNG differ diff --git a/wasm-micro-runtime/doc/pics/multi_module_pic1.png b/wasm-micro-runtime/doc/pics/multi_module_pic1.png new file mode 100644 index 0000000..f0c8e33 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/multi_module_pic1.png differ diff --git a/wasm-micro-runtime/doc/pics/native_call_wasm.PNG b/wasm-micro-runtime/doc/pics/native_call_wasm.PNG new file mode 100644 index 0000000..934af72 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/native_call_wasm.PNG differ diff --git a/wasm-micro-runtime/doc/pics/request.PNG b/wasm-micro-runtime/doc/pics/request.PNG new file mode 100644 index 0000000..281115b Binary files /dev/null and b/wasm-micro-runtime/doc/pics/request.PNG differ diff --git a/wasm-micro-runtime/doc/pics/safe.PNG b/wasm-micro-runtime/doc/pics/safe.PNG new file mode 100644 index 0000000..ee3ed21 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/safe.PNG differ diff --git a/wasm-micro-runtime/doc/pics/sensor_callflow.PNG b/wasm-micro-runtime/doc/pics/sensor_callflow.PNG new file mode 100644 index 0000000..12839aa Binary files /dev/null and b/wasm-micro-runtime/doc/pics/sensor_callflow.PNG differ diff --git a/wasm-micro-runtime/doc/pics/sub.PNG b/wasm-micro-runtime/doc/pics/sub.PNG new file mode 100644 index 0000000..25607a5 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/sub.PNG differ diff --git a/wasm-micro-runtime/doc/pics/vgl_demo.png b/wasm-micro-runtime/doc/pics/vgl_demo.png new file mode 100644 index 0000000..3cb7eb4 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/vgl_demo.png differ diff --git a/wasm-micro-runtime/doc/pics/vgl_demo2.png b/wasm-micro-runtime/doc/pics/vgl_demo2.png new file mode 100644 index 0000000..8b07e62 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/vgl_demo2.png differ diff --git a/wasm-micro-runtime/doc/pics/vgl_demo_linux.png b/wasm-micro-runtime/doc/pics/vgl_demo_linux.png new file mode 100644 index 0000000..8485373 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/vgl_demo_linux.png differ diff --git a/wasm-micro-runtime/doc/pics/vgl_linux.PNG b/wasm-micro-runtime/doc/pics/vgl_linux.PNG new file mode 100644 index 0000000..65ce063 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/vgl_linux.PNG differ diff --git a/wasm-micro-runtime/doc/pics/wamr-arch.JPG b/wasm-micro-runtime/doc/pics/wamr-arch.JPG new file mode 100644 index 0000000..759ed38 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/wamr-arch.JPG differ diff --git a/wasm-micro-runtime/doc/pics/wamr_memory_model.png b/wasm-micro-runtime/doc/pics/wamr_memory_model.png new file mode 100644 index 0000000..d01d378 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/wamr_memory_model.png differ diff --git a/wasm-micro-runtime/doc/pics/wamr_menu_config.png b/wasm-micro-runtime/doc/pics/wamr_menu_config.png new file mode 100644 index 0000000..c854095 Binary files /dev/null and b/wasm-micro-runtime/doc/pics/wamr_menu_config.png differ diff --git a/wasm-micro-runtime/doc/port_wamr.md b/wasm-micro-runtime/doc/port_wamr.md new file mode 100644 index 0000000..7899ff3 --- /dev/null +++ b/wasm-micro-runtime/doc/port_wamr.md @@ -0,0 +1,75 @@ + +WAMR porting guide +========================= + + +This document describes how to port WAMR to a new platform "**new-os**" + + + +# Step 1: Implement platform API layer + +------------------------- +Firstly create the folder for platform API layer implementations: +* for common platforms, create a folder in **`core/shared/platform/new-os`** in WAMR repository folder, so the implementation can be upstreamed +* for platforms that are internal and its implementation shouldn't be published, it's recommended to create a folder outside of the WAMR repository folder (e.g. have separate repository for platform API layer implementation) + +In the folder you just created, you must provide the following files: + +- `platform_internal.h`: It can be used for any platform-specific definitions such as macros, data types and internal APIs. + +- `shared_platform.cmake`: the cmake file will be included by the building script. It is recommended to add a definition for your platform: + + - ```cmake + add_definitions(-DBH_PLATFORM_YOUR_NAME) + ``` + +Then go to implement the APIs defined in the following header files for the platform abstraction layer: + +- [`platform_api_vmcore.h`](../core/shared/platform/include/platform_api_vmcore.h): mandatory for building mini-product (vmcore only). Part of the APIs is needed only for Ahead of Time compilation support. +- [`platform_api_extension.h`](../core/shared/platform/include/platform_api_extension.h): mandatory for app-mgr and app-framework. Given that the app-mgr and app-framework are not required for your target platform, you won't have to implement the API defined in the `platform_api_extension.h`. + + + +**common/posix:** + +There is posix based implementation of the platform API located in the `platform/common/posix` folder. You can include it if your platform supports posix API. refer to platform linux implementation. + + + +**common/math:** + +Some platforms such as ZephyrOS don't provide math functions e.g. sqrt, fabs and isnan, then you should include source files under the folder `platform/common/math`. + + + +# Step 2: Create the mini product for the platform + +------------------------- +You can build a mini WAMR product which is only the vmcore for your platform. Normally you need to implement the main function which loads a WASM file and run it with the WASM runtime. You don't have to do this step if there is no mini-product need for your platform porting. + + + +Firstly create folder **product-mini/platforms/new-os** for the platform mini product build, then refer to the linux platform mini-product for creating the CMakeList.txt and the C implementations. + + + +You should set cmake variable `WAMR_BUILD_PLATFORM` to your platform name while building the mini product. It can be done in the mini product CMakeList.txt file, or pass arguments to cmake command line like: + +``` +mkdir build +cd build +cmake .. -DWAMR_BUILD_PLATFORM=new-os +``` + +For platform implementations that are outside of the WAMR repository (e.g. internal platforms), you also need to provide `SHARED_PLATFORM_CONFIG` path: + +``` +cmake .. -DWAMR_BUILD_PLATFORM=new-os -DSHARED_PLATFORM_CONFIG=/path/to/new-os/shared_platform.cmake +``` + + +Refer to [build_wamr.md](./build_wamr.md) for the building configurations and parameters. + + + diff --git a/wasm-micro-runtime/doc/pthread_impls.md b/wasm-micro-runtime/doc/pthread_impls.md new file mode 100644 index 0000000..92b51cb --- /dev/null +++ b/wasm-micro-runtime/doc/pthread_impls.md @@ -0,0 +1,59 @@ +# Pthread implementations + +WAMR has two pthread implementations available as of writing this. + +These implementations are not ABI-compatible. You at least need to rebuild +your wasm modules when migrating from one pthread implementation to another. + +For new users, we recommend to use (or at least experiment) +the new wasi-threads based implementation. +In future, we might remove the old implementation. + +## WAMR lib-pthread (old) + + * The pthread API is directly implemented as host functions in WAMR. + (`WAMR_BUILD_LIB_PTHREAD`) + + * Only minimum API is implemented as of writing this. + (eg. no pthread barriers) + + * WAMR-specific ABI + + * [Known limitations](pthread_library.md#known-limits) + +## wasi-threads (new) + + * The pthread API is implemented in wasi-libc, based on + [wasi-threads](https://github.com/WebAssembly/wasi-threads) + and [WASM threads](https://github.com/WebAssembly/threads) proposals. + + * It requires a recent-enough version of wasi-libc. The experimental support + is included in + [wasi-sdk 20.0](https://github.com/WebAssembly/wasi-sdk/releases/tag/wasi-sdk-20) + or later. + To build your application, cmake users can use the + [cmake toolchain file](https://github.com/WebAssembly/wasi-sdk/blob/main/wasi-sdk-pthread.cmake) + provided by wasi-sdk. + + * wasi-threads is implemented as a host function in WAMR. + (`WAMR_BUILD_LIB_WASI_THREADS`) + + * The ABI is specified in wasi-threads proposal. + You can run the same wasm modules on other runtimes which implement + the proposal. (wasmtime, toywasm, ...) + + * Basically more feature-rich and complete than WAMR lib-pthread. + + **EXCEPTION**: `pthread_exit` is not available as of writing this. + If `pthread_exit` is important for your use cases, please speak up in + the [GitHub issue](https://github.com/WebAssembly/wasi-threads/issues/7). + + **EXCEPTION**: For threads created by `pthread_create`, the AUX stack + (aka C shadow stack) overflow detection mechanism is disabled as of + writing this. + If it's important for your use cases, please speak up in the + [GitHub issue](https://github.com/WebAssembly/wasi-threads/issues/12). + +# References + +* https://github.com/bytecodealliance/wasm-micro-runtime/issues/1790 diff --git a/wasm-micro-runtime/doc/pthread_library.md b/wasm-micro-runtime/doc/pthread_library.md new file mode 100644 index 0000000..ba43ee1 --- /dev/null +++ b/wasm-micro-runtime/doc/pthread_library.md @@ -0,0 +1,198 @@ +# WAMR pthread library + +**Note**: This document describes the old pthread implementation. +See [Pthread implementations](pthread_impls.md). + +WAMR provides a built-in library to support pthread APIs. You can call pthread APIs in your application source code. + +## Build and run +Suppose you have written a C program calling pthread_create() to create a thread, and the file name is main.c +``` C +#include +#include + +void *thread_routine(void *arg) +{ + printf("Enter thread\n"); + pthread_exit(NULL); + return NULL; +} + +int main(int argc, char** argv) +{ + pthread_t tid; + + if (0 != pthread_create(&tid, NULL, thread_routine, NULL)) { + printf("Failed to create thread\n"); + } + + if (0 != pthread_join(tid, NULL)) { + printf("Failed to join thread %d.\n", tid); + } + + printf("Exit\n"); + + return 0; +} +``` +**Build with libc-builtin** + +To build this C program into WebAssembly app with libc-builtin, you can use this command: +``` bash +/opt/wasi-sdk/bin/clang --target=wasm32 \ + --sysroot=${WAMR_ROOT}/wamr-sdk/app/libc-builtin-sysroot \ + -O3 -pthread -nostdlib -z stack-size=32768 \ + -Wl,--shared-memory \ + -Wl,--initial-memory=131072,--max-memory=131072 \ + -Wl,--allow-undefined-file=${WAMR_ROOT}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt \ + -Wl,--no-entry -Wl,--export=main \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors \ + main.c -o test.wasm +# -pthread: it will enable some dependent WebAssembly features for thread +# -nostdlib: disable the WASI standard library as we are using WAMR builtin-libc +# -z stack-size=: specify the total aux stack size +# -Wl,--export=__heap_base,--export=__data_end: export these globals so the runtime can resolve the total aux stack size and the start offset of the stack top +# -Wl,--export=__wasm_call_ctors: export the init function to initialize the passive data segments +``` + +**Build with libc-WASI** + +You can also build this program with WASI, but we need to make some changes to wasi-sysroot: + +1. disable malloc/free of wasi, as they are not atomic operations: + ``` bash + /opt/wasi-sdk/bin/llvm-ar -d /opt/wasi-sdk/share/wasi-sysroot/lib/wasm32-wasi/libc.a dlmalloc.o + ``` +2. copy the pthread.h to wasi-sysroot so the compiler can find it: + ``` bash + cp ${WAMR_ROOT}/wamr-sdk/app/libc-builtin-sysroot/include/pthread.h /opt/wasi-sdk/share/wasi-sysroot/include + ``` +> Note:
+>1. Remember to back up the original sysroot files + +Then build the program with this command: +``` bash +/opt/wasi-sdk/bin/clang -pthread -O3 \ + -Wl,--shared-memory,--max-memory=196608 \ + -Wl,--allow-undefined,--no-check-features \ + -Wl,--export=__heap_base,--export=__data_end \ + main.c -o test.wasm +# -Wl,--no-check-features: the errno.o in wasi-sysroot is not compatible with pthread feature, pass this option to avoid errors +``` + +**Build with EMCC** + +> Note: This document is based on `emcc 2.0.26`, other version may not work with these commands + +EMCC's `-pthread` option is not compatible with standalone mode, we need to pass `-mbulk-memory -matomics` to the compiler and `--shared-memory,--no-check-features` to linker manually + +EMCC provides some empty implementation for pthread related APIs, we need to remove them from emcc's libc. +``` bash +cd ${emsdk_dir}/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten +emar d libc.a library_pthread_stub.o +emranlib libc.a +``` + +``` bash +emcc -O3 -mbulk-memory -matomics -s MALLOC="none" \ + -Wl,--export=__data_end,--export=__heap_base \ + -Wl,--shared-memory,--no-check-features \ + -s ERROR_ON_UNDEFINED_SYMBOLS=0 \ + main.c -o test.wasm +``` + +**Build AOT module** + +You can build the wasm module into AOT module with pthread support, please pass option `--enable-multi-thread` to wamrc: +``` bash +wamrc --enable-multi-thread -o test.aot test.wasm +``` + +Currently WAMR disables pthread library by default. To run the module with pthread support, please build the runtime with `-DWAMR_BUILD_LIB_PTHREAD=1` +``` bash +cd ${WAMR_ROOT}/product-mini/platforms/linux +mkdir build && cd build +cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 +make +# Then you can run the wasm module above: +./iwasm test.wasm +# Or the AOT module: +# ./iwasm test.aot +``` + +[Here](../samples/multi-thread) is also a sample to show how wasm-apps use pthread APIs to create threads, and how to build it with cmake. You can build this sample and have a try: +``` bash +cd ${WAMR_ROOT}/samples/multi-thread +mkdir build && cd build +cmake .. +make +# Run wasm application +./iwasm wasm-apps/test.wasm +``` + + +## Aux stack seperation +The compiler may use some spaces in the linear memory as an auxiliary stack. When pthread is enabled, every thread should have its own aux stack space, so the total aux stack space reserved by the compiler will be divided into N + 1 parts, where N is the maximum number of threads that can be created by the user code. + +The default value of N is 4, which means you can create 4 threads at most. This value can be changed by an option if you are using product-mini: +``` bash +./iwasm --max-threads=n test.wasm +``` +If you are going to develop your own runtime product, you can use the API `wasm_runtime_set_max_thread_num` or init arg `init_args.max_thread_num` to set the value, or you can change the macro `CLUSTER_MAX_THREAD_NUM` in [config.h](../core/config.h). + +> Note: the total size of aux stack reserved by compiler can be set with `-z stack-size` option during compilation. If you need to create more threads, please set a larger value, otherwise it is easy to cause aux stack overflow. + +## Supported APIs +``` C +/* Thread APIs */ +int pthread_create(pthread_t *thread, const void *attr, + void *(*start_routine) (void *), void *arg); + +int pthread_join(pthread_t thread, void **retval); + +int pthread_detach(pthread_t thread); + +int pthread_cancel(pthread_t thread); + +pthread_t pthread_self(void); + +void pthread_exit(void *retval); + +/* Mutex APIs */ +int pthread_mutex_init(pthread_mutex_t *mutex, const void *attr); + +int pthread_mutex_lock(pthread_mutex_t *mutex); + +int pthread_mutex_unlock(pthread_mutex_t *mutex); + +int pthread_mutex_destroy(pthread_mutex_t *mutex); + +/* Cond APIs */ +int pthread_cond_init(pthread_cond_t *cond, const void *attr); + +int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex); + +int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, + unsigned int useconds); + +int pthread_cond_signal(pthread_cond_t *cond); + +int pthread_cond_broadcast(pthread_cond_t *cond); + +int pthread_cond_destroy(pthread_cond_t *cond); + +/* Pthread key APIs */ +int pthread_key_create(pthread_key_t *key, void (*destructor)(void *)); + +int pthread_setspecific(pthread_key_t key, const void *value); + +void *pthread_getspecific(pthread_key_t key); + +int pthread_key_delete(pthread_key_t key); +``` + +## Known limits +- `pthread_attr_t`, `pthread_mutexattr_t` and `pthread_condattr_t` are not supported yet, so please pass `NULL` as the second argument of `pthread_create`, `pthread_mutex_init` and `pthread_cond_init`. +- The `errno.o` in wasi-sysroot is not compatible with this feature, so using errno in multi-thread may cause unexpected behavior. +- Currently `struct timespec` is not supported, so the prototype of `pthread_cond_timedwait` is different from the native one, it takes an unsigned int argument `useconds` to indicate the waiting time. diff --git a/wasm-micro-runtime/doc/ref_types.md b/wasm-micro-runtime/doc/ref_types.md new file mode 100644 index 0000000..7fefa99 --- /dev/null +++ b/wasm-micro-runtime/doc/ref_types.md @@ -0,0 +1,9 @@ +# WAMR reference-types introduction + +WebAssembly [reference-types](https://github.com/WebAssembly/reference-types) proposal introduces two new types `funcref` and `externref`. With `externref`, It is easier and more efficient to interoperate with host environment. Host references are able to be represented directly by type `externref`. + +WAMR has implemented the reference-types proposal. WAMR allows a native method to pass a host object to a WASM application as an `externref` parameter or receives a host object from a WASM application as an `externref` result. Internally, WAMR won't try to parse or dereference `externref`. It is an opaque type. + +The restriction of using `externref` in a native method is the host object has to be the value of a `unintptr_t` variable. In other words, it takes **8 bytes** on 64-bit machine and **4 bytes** on 32-bit machines. Please keep that in mind especially when calling `wasm_runtime_call_wasm`. + +Please ref to the [sample](../samples/ref-types) for more details. diff --git a/wasm-micro-runtime/doc/release_ack.md b/wasm-micro-runtime/doc/release_ack.md new file mode 100644 index 0000000..a0adfff --- /dev/null +++ b/wasm-micro-runtime/doc/release_ack.md @@ -0,0 +1,61 @@ +Major feature releases and contributors +========================================= + + +**May 07, 2019: WAMR first GitHub release** + +- Contributors: Wenyong Huang, Weining Lu, Lei Shi, Li Tian, Jizhao Zhang, Yi Zhang, Daoming Qiu, Xin Wang (Intel) + +**May 17, 2019: Application manager, WASM APP API, samples and test tools** + +- Contributors: Wenyong Huang, Weining Lu, Lei Shi, Li Tian, Jizhao Zhang, Yi Zhang, Daoming Qiu, Xin Wang (Intel) + + +**May 23, 2019: Support AliOS Things** + +- Contributor: JinZhou Zhu (Alibaba) + +**May 24, 2019: Support memory usage profiler** + +- Contributors Wenyong Huang (Intel) + +**Jun 11, 2019: Add WASM APP API connection** + + +- Contributor: Weining Lu (Intel) + +**Jun 10, 2019: Support VxWorks** + +- Contributor: Yiting Wang (WindRiver) + +**Aug 1, 2019: Add WGL graphic user interface API** + +- Contributor: Weining Lu + +**Aug 14, 2019: Add Docker support** + + +- Contributor: beriberikix + + +**Aug 14, 2019: WASM IoT app store demo** + + +- Contributor: Luhanzhi Li, Jun Xu (Intel) + + +**Aug 28, 2019: SGX support** + + +- Contributor: Mic Bowman (Intel) + + +**Sep 6, 2019: Mac platform support** + + +- Contributor: Jonathan Dong (Alibaba) + +**Nov 2019: WASI support** (Intel) + +**Jan 2020: Ahead of time and Just-in-Time compilation support** (Intel) + diff --git a/wasm-micro-runtime/doc/roadmap.md b/wasm-micro-runtime/doc/roadmap.md new file mode 100644 index 0000000..c5c7b84 --- /dev/null +++ b/wasm-micro-runtime/doc/roadmap.md @@ -0,0 +1,23 @@ + +# WebAssembly Micro Runtime Roadmap + + + +## Data serialization +Evaluating using cbor as the default data serialization + +No plan yet. + + + +## Threading +Plan: 2020 Q1 + + + +## AssemblyScript Support and API + +Currently under evaluation + + + diff --git a/wasm-micro-runtime/doc/semantic_version.md b/wasm-micro-runtime/doc/semantic_version.md new file mode 100644 index 0000000..d0d46a5 --- /dev/null +++ b/wasm-micro-runtime/doc/semantic_version.md @@ -0,0 +1,21 @@ +# WAMR uses semantic versioning + +WAMR uses the _semantic versioning_ to replace the current _date versioning_ system. + +There are three parts in the new version string: + +- _major_. Any incompatible modification, on both ABI and APIs, will lead an increment + in the value of _major_. APIs includes: `wasm_export.h`, `wasm_c_api.h`, + _sections in AOT files_, and so on. +- _minor_. It represents new features. It includes not just MVP or POST-MVP features + but also WASI features and WAMR private ones. +- _patch_. It represents patches. + +## Legacy versions + +All legacy versions(tags) will keep their current status. No existed releasings names +and links will be changed. + +## Reference + +- [Semantic Versioning 2.0.0](https://semver.org/) diff --git a/wasm-micro-runtime/doc/socket_api.md b/wasm-micro-runtime/doc/socket_api.md new file mode 100644 index 0000000..eff9376 --- /dev/null +++ b/wasm-micro-runtime/doc/socket_api.md @@ -0,0 +1,89 @@ +# How to use Berkeley/Posix Socket APIs in WebAssembly + +**_Berkeley sockets_** usually means an API for Internet sockets and Unix domain +sockets. A socket is an abstract representation of the local endpoint of a +network communication path. + +Currently, WAMR supports some Socket API features: +- Support TCP and UDP +- Support IPv4 and IPv6 +- Support get/set socket options +- Support access control + +This document introduces a way to support the _Berkeley/POSIX Socket API_ in +WebAssembly code. + +## Patch the native code + +The first step is to include a header file of the WAMR socket extension in the +native source code. + +```c +#ifdef __wasi__ +#include +#endif +``` + +`__wasi__` is a macro defined by WASI. The host compiler will not enable it. + +## CMake files + +It is recommended that the project should use CMake as its build system. Use +[_wasi-sdk_](https://github.com/WebAssembly/wasi-sdk) +as a toolchain to compile C/C++ to WebAssembly + +```bash +$ cmake -DWASI_SDK_PREFIX=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_TOOLCHAIN_FILE} + -DCMAKE_SYSROOT=${WASI_SYS_ROOT} + .. +``` + +In the *CMakeLists.txt*, include an extension of socket support and link with it. + +```cmake +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../core/iwasm/libraries/lib-socket/lib_socket_wasi.cmake) +add_executable(socket_example tcp_server.c) +target_link_libraries(socket_example socket_wasi_ext) +``` + +Now, the native code with socket APIs is ready for compilation. + +## Run with iwasm + +If having the _.wasm_, the last step is to run it with _iwasm_. + +The _iwasm_ should be compiled with `WAMR_BUILD_LIBC_WASI=1`. By default, it is +enabled. + +_iwasm_ accepts address ranges via an option, `--addr-pool`, to implement +the capability control. All IP address the WebAssembly application may need to `bind()` or `connect()` +should be announced first. Every IP address should be in CIDR notation. + +```bash +$ iwasm --addr-pool=1.2.3.4/15,2.3.4.6/16 socket_example.wasm +``` + +_iwasm_ also accepts list of domain names and domain name patterns for the address resolution via an option, `--allow-resolve`, to implement the capability control. Every domain that will be resolved using `sock_addr_resolve` needs to be added to the allowlist first. + +```bash +$ iwasm --allow-resolve=*.example.com --allow-resolve=domain.com +``` + +The example above shows how to allow for resolving all `example.com`'s subdomains (e.g. `x.example.com`, `a.b.c.example.com`) and `domain.com` domain. + +Refer to [socket api sample](../samples/socket-api) for more details. + +## Intel SGX support + +WAMR also supports the socket API within Intel SGX enclaves. + +The _iwasm_ should be compiled with `WAMR_BUILD_LIBC_WASI=1` and `WAMR_BUILD_LIB_PTHREAD=1`, which are enabled by default. + +Similarly to running _iwasm_ outside of an enclave, the allowed address ranges are given via the option `--addr-pool`. + +```bash +$ iwasm --addr-pool=1.2.3.4/15,2.3.4.6/16 socket_example.wasm +``` + +Refer to [socket api sample](../samples/socket-api) for the compilation of the Wasm applications and [_iwasm_ for Intel SGX](../product-mini/platforms/linux-sgx) for the Wasm runtime. diff --git a/wasm-micro-runtime/doc/source_debugging.md b/wasm-micro-runtime/doc/source_debugging.md new file mode 100644 index 0000000..bfa62f6 --- /dev/null +++ b/wasm-micro-runtime/doc/source_debugging.md @@ -0,0 +1,20 @@ +# WAMR source debugging + +## Build wasm application with debug information +To debug your application, you need to compile them with debug information. You can use `-g` option when compiling the source code if you are using wasi-sdk (also work for emcc and rustc): +``` bash +/opt/wasi-sdk/bin/clang -g test.c -o test.wasm +``` + +Then you will get `test.wasm` which is a WebAssembly module with embedded DWARF sections. Further, you can use `llvm-dwarfdump` to check if the generated wasm file contains DWARF information: +``` bash +llvm-dwarfdump-12 test.wasm +``` + +## Debugging with interpreter + +See [Debuggging with interpreter](source_debugging_interpreter.md). + +## Debugging with AOT + +See [Debuggging with AOT](source_debugging_aot.md). diff --git a/wasm-micro-runtime/doc/source_debugging_aot.md b/wasm-micro-runtime/doc/source_debugging_aot.md new file mode 100644 index 0000000..129287b --- /dev/null +++ b/wasm-micro-runtime/doc/source_debugging_aot.md @@ -0,0 +1,100 @@ +# WAMR source debugging (AOT) + +## Debugging with AOT + +> Note: AOT debugging is experimental and only a few debugging capabilities are supported. + +1. Build lldb (assume you have already built llvm) +``` bash +cd ${WAMR_ROOT}/core/deps/llvm/build +cmake ../llvm -DLLVM_ENABLE_PROJECTS="clang;lldb" -DLLDB_INCLUDE_TESTS=OFF +make -j $(nproc) +``` + +2. Build wamrc with debugging feature +``` bash +cd ${WAMR_ROOT}/wamr-compiler +mkdir build && cd build +cmake .. -DWAMR_BUILD_DEBUG_AOT=1 +make -j $(nproc) +``` + +3. Build iwasm with debugging feature +``` bash +cd ${WAMR_ROOT}/product-mini/platforms/linux +mkdir build && cd build +cmake .. -DWAMR_BUILD_DEBUG_AOT=1 +make +``` + +4. Compile wasm module to AOT module +``` bash +wamrc -o test.aot test.wasm +``` + +5. Execute iwasm using lldb + + Then you can use lldb commands to debug both wamr runtime and your wasm application in ***current terminal***. + + ``` bash + % lldb iwasm -- test.aot + (lldb) target create "iwasm" + Current executable set to 'iwasm' (x86_64). + (lldb) settings set -- target.run-args "test.aot" + (lldb) settings set plugin.jit-loader.gdb.enable on + (lldb) b main + Breakpoint 1: where = iwasm`main + 48 at main.c:294:11, address = 0x0000000100001020 + (lldb) run + Process 27954 launched: '/tmp/bin/iwasm' (x86_64) + Process 27954 stopped + * thread #1, queue = 'com.apple.main-thread', stop reason = breakpoint 1.1 + frame #0: 0x0000000100001020 iwasm`main(argc=2, argv=0x00007ff7bfeff678) at main.c:294:11 + 291 int + 292 main(int argc, char *argv[]) + 293 { + -> 294 int32 ret = -1; + 295 char *wasm_file = NULL; + 296 const char *func_name = NULL; + 297 uint8 *wasm_file_buf = NULL; + Target 0: (iwasm) stopped. + (lldb) c + Process 27954 resuming + 1 location added to breakpoint 1 + error: need to add support for DW_TAG_base_type 'void' encoded with DW_ATE = 0x0, bit_size = 0 + Process 27954 stopped + * thread #1, queue = 'com.apple.main-thread', stop reason = breakpoint 1.2 + frame #0: 0x00000001002980a0 JIT(0x100298004)`main(exenv=0x0000000301808200) at hello.c:6:9 + 3 int + 4 main(void) + 5 { + -> 6 printf("hello\n"); + 7 + 8 return 0; + 9 } + Target 0: (iwasm) stopped. + (lldb) br l + Current breakpoints: + 1: name = 'main', locations = 2, resolved = 2, hit count = 2 + 1.1: where = iwasm`main + 48 at main.c:294:11, address = 0x0000000100001020, resolved, hit count = 1 + 1.2: where = JIT(0x100298004)`main + 12 at hello.c:6:9, address = 0x00000001002980a0, resolved, hit count = 1 + + (lldb) + ``` + + * In the above example, + + * The first `main` function, which is in `main.c`, is the main + function of the iwasm command. + + * The second `main` function, which is in `hello.c`, is the main + function of the AOT-compiled wasm module. + + * WAMR AOT debugging uses the GDB JIT loader mechanism to load + the debug info of the debugee module. + On some platforms including macOS, you need to enable it explicitly. + (`settings set plugin.jit-loader.gdb.enable on`) + + References: + + * https://github.com/llvm/llvm-project/blob/main/llvm/docs/DebuggingJITedCode.rst + * https://sourceware.org/gdb/current/onlinedocs/gdb/JIT-Interface.html diff --git a/wasm-micro-runtime/doc/source_debugging_interpreter.md b/wasm-micro-runtime/doc/source_debugging_interpreter.md new file mode 100644 index 0000000..577b9bf --- /dev/null +++ b/wasm-micro-runtime/doc/source_debugging_interpreter.md @@ -0,0 +1,115 @@ +# WAMR source debugging (interpreter) + +References: +- [Blog: WAMR source debugging basic](https://bytecodealliance.github.io/wamr.dev/blog/wamr-source-debugging-basic/) +- [Blog: Debugging wasm with VSCode](https://bytecodealliance.github.io/wamr.dev/blog/debugging-wasm-with-vscode/) + +WAMR supports source level debugging based on DWARF (normally used in C/C++/Rust), source map (normally used in AssemblyScript) is not supported. + +**The lldb's ability to debug wasm application is based on the patch [Add class WasmProcess for WebAssembly debugging](https://reviews.llvm.org/D78801). Thanks very much to the author @paolosev for such a great work!** + +## Debugging with interpreter + +1. Install dependent libraries +``` bash +apt update && apt install cmake make g++ libxml2-dev -y +``` + +2. Build iwasm with source debugging feature +``` bash +cd ${WAMR_ROOT}/product-mini/platforms/linux +mkdir build && cd build +cmake .. -DWAMR_BUILD_DEBUG_INTERP=1 +make +``` +> Note: On MacOS M1 environment, pass the additional `-DWAMR_DISABLE_HW_BOUND_CHECK=1` cmake configuration. + +3. Execute iwasm with debug engine enabled +``` bash +iwasm -g=127.0.0.1:1234 test.wasm +# Use port = 0 to allow a random assigned debug port +``` + +4. Build customized lldb +``` bash +git clone --branch release/13.x --depth=1 https://github.com/llvm/llvm-project +cd llvm-project +git apply ${WAMR_ROOT}/build-scripts/lldb_wasm.patch +mkdir build-lldb +cmake -S ./llvm -B build-lldb \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;lldb" \ + -DLLVM_TARGETS_TO_BUILD:STRING="X86;WebAssembly" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DLLVM_BUILD_BENCHMARKS:BOOL=OFF \ + -DLLVM_BUILD_DOCS:BOOL=OFF -DLLVM_BUILD_EXAMPLES:BOOL=OFF \ + -DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF -DLLVM_BUILD_TESTS:BOOL=OFF \ + -DLLVM_ENABLE_BINDINGS:BOOL=OFF -DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF \ + -DLLVM_INCLUDE_DOCS:BOOL=OFF -DLLVM_INCLUDE_EXAMPLES:BOOL=OFF \ + -DLLVM_INCLUDE_TESTS:BOOL=OFF -DLLVM_ENABLE_LIBXML2:BOOL=ON +cmake --build build-lldb --target lldb --parallel $(nproc) +# The lldb is generated under build-lldb/bin/lldb +``` +> Note: If using `CommandLineTools` on MacOS, make sure only one SDK is present in `/Library/Developer/CommandLineTools/SDKs`. + +> You can download pre-built `wamr-lldb` binaries from [here](https://github.com/bytecodealliance/wasm-micro-runtime/releases). + +5. Launch customized lldb and connect to iwasm +``` bash +lldb +(lldb) process connect -p wasm connect://127.0.0.1:1234 +``` +Then you can use lldb commands to debug your applications. Please refer to [lldb document](https://lldb.llvm.org/use/tutorial.html) for command usage. + +## Enable debugging in embedders (for interpreter) + +There are three steps to enable debugging in embedders + +1. Set the debug parameters when initializing the runtime environment: + ``` c + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + /* ... */ + strcpy(init_args.ip_addr, "127.0.0.1"); + init_args.instance_port = 1234; + /* + * Or set port to 0 to use a port assigned by os + * init_args.instance_port = 0; + */ + + if (!wasm_runtime_full_init(&init_args)) { + return false; + } + ``` + +2. Use `wasm_runtime_start_debug_instance` to create the debug instance: + ``` c + /* + initialization, loading and instantiating + ... + */ + exec_env = wasm_runtime_create_exec_env(module_inst, stack_size); + uint32_t debug_port = wasm_runtime_start_debug_instance(exec_env); + ``` + +3. Enable source debugging features during building + + You can use `-DWAMR_BUILD_DEBUG_INTERP=1` during cmake configuration + + Or you can set it directly in `cmake` files: + ``` cmake + set (WAMR_BUILD_DEBUG_INTERP 1) + ``` + +### Attentions +- Debugging `multi-thread wasm module` is not supported, if your wasm module use pthread APIs (see [pthread_library.md](./pthread_library.md)), or the embedder use `wasm_runtime_spawn_thread` to create new wasm threads, then there may be **unexpected behaviour** during debugging. + + > Note: This attention is about "wasm thread" rather than native threads. Executing wasm functions in several different native threads will **not** affect the normal behaviour of debugging feature. + +- When using source debugging features, **don't** create multiple `wasm_instance` from the same `wasm_module`, because the debugger may change the bytecode (set/unset breakpoints) of the `wasm_module`. If you do need several instance from the same bytecode, you need to copy the bytecode to a new butter, then load a new `wasm_module`, and then instantiate the new wasm module to get the new instance. + +- If you are running `lldb` on non-linux platforms, please use `platform select remote-linux` command in lldb before connecting to the runtime: + ``` + (lldb) platform select remote-linux + (lldb) process connect -p wasm connect://127.0.0.1:1234 + ``` diff --git a/wasm-micro-runtime/doc/wasm_c_api.md b/wasm-micro-runtime/doc/wasm_c_api.md new file mode 100644 index 0000000..131adad --- /dev/null +++ b/wasm-micro-runtime/doc/wasm_c_api.md @@ -0,0 +1,65 @@ +# wasm-c-api introduction + +wasm-c-api is an engine-agnostic API to embed a WASM engine. +In wasm-micro-runtime, it's provided by the header file `wasm_c_api.h`. +Its functionalities are overlapping with `wasm_export.h`, which is +a native API of wasm-micro-runtime. An embedder is supposed to pick +one of these APIs, rather than mixing both of them. + +All samples come from the commit 340fd9528cc3b26d22fe30ee1628c8c3f2b8c53b +of [wasm-c-api](https://github.com/WebAssembly/wasm-c-api). + +Developer can learn these _APIs_ from +[wasm.h](https://github.com/WebAssembly/wasm-c-api/blob/master/include/wasm.h). + +And here are [examples](https://github.com/WebAssembly/wasm-c-api/tree/master/example) which +are helpful. + +## FYI + +- The thread model of _wasm_c_api_ is + + - An `wasm_engine_t` instance may only be created once per process + - Every `wasm_store_t` and its objects may only be accessed in a single thread + +- `wasm_engine_new`, `wasm_engine_new_with_config`, `wasm_engine_new_with_args`, + `wasm_engine_delete`should be called in a thread-safe environment. Such + behaviors are not recommended, and please make sure an appropriate calling + sequence if it has to be. + + - call `wasm_engine_new` and `wasm_engine_delete` in different threads + - call `wasm_engine_new` or `wasm_engine_delete` multiple times in + different threads + +## unspported list + +Currently WAMR supports most of the APIs, the unsupported APIs are listed as below: + +- References + +```c +WASM_API_EXTERN own wasm_shared_##name##_t* wasm_##name##_share(const wasm_##name##_t*); +WASM_API_EXTERN own wasm_##name##_t* wasm_##name##_obtain(wasm_store_t*, const wasm_shared_##name##_t*); +``` + +- Several Module APIs + +```c +WASM_API_EXTERN void wasm_module_serialize(const wasm_module_t*, own wasm_byte_vec_t* out); +WASM_API_EXTERN own wasm_module_t* wasm_module_deserialize(wasm_store_t*, const wasm_byte_vec_t*); +``` + +Currently growing a table or memory by wasm opcode is supported and it is not supported to grow them +by host-side function callings. + +- Table Grow APIs + +```c +WASM_API_EXTERN bool wasm_table_grow(wasm_table_t*, wasm_table_size_t delta, wasm_ref_t* init); +``` + +- Memory Grow APIs + +```c +WASM_API_EXTERN bool wasm_memory_grow(wasm_memory_t*, wasm_memory_pages_t delta); +``` diff --git a/wasm-micro-runtime/doc/xip.md b/wasm-micro-runtime/doc/xip.md new file mode 100644 index 0000000..a76c761 --- /dev/null +++ b/wasm-micro-runtime/doc/xip.md @@ -0,0 +1,131 @@ +# WAMR XIP (Execution In Place) feature introduction + +Some IoT devices may require to run the AOT file from flash or ROM which is read-only, so as to reduce the memory consumption, or resolve the issue that there is no executable memory available to run AOT code. In such case, the AOT code inside the AOT file shouldn't be duplicated into memory and shouldn't be modified (or patched) by the AOT relocations. To address this, WAMR implements the XIP (Execution In Place) feature, which generates the AOT relocations as few as possible: +- In the AOT code, an AOT function calls other functions with indirect mode: it doesn't call other functions directly, but looks up their pointers from the function pointer table passed by its first argument exec_env, and then calls the function pointer found. By this way the relocations to other functions are eliminated. +- Eliminate the calls to the LLVM intrinsic functions, or, replace calling them with calling runtime self implemented functions instead, e.g. the calling to `llvm.experimental.constrained.fadd.f32` is replaced by the calling to `aot_intrinsic_fadd_f32`. + +The XIP file is an AOT file without (or with few) relocations to patch the AOT code (or text section). Developer can use the option `--enable-indirect-mode --disable-llvm-intrinsics` for wamrc to generate the AOT file, e.g.: +```bash +wamrc --enable-indirect-mode --disable-llvm-intrinsics -o +or +wamrc --xip -o +``` + +Note: --xip is a short option for --enable-indirect-mode --disable-llvm-intrinsics + +## Known issues + +There may be some relocations to the ".rodata" like sections which require to patch the AOT code. More work will be done to resolve it in the future. + +## Tuning the XIP intrinsic functions + +WAMR provides a default mapping table for some targets, but it may not be the best one for your target. And it doesn't cover all the supported targets. + +So, wamrc provides the option `--enable-builtin-intrinsics=` to make it possible to tune the intrinsic functions for your target. + +Firstly, you should understand why we don't use the LLVM intrinsic functions directly. The reason is that the LLVM intrinsic functions can't map to the native instructions directly, e.g. the LLVM intrinsic function `i32.div_s` can't map to the native instruction if the target doesn't support the division instruction, it will be translated to a function call to the runtime function from libgcc/compiler-rt. This will cause the AOT code to have the relocations to the libgcc/compiler-rt, which is not acceptable for the XIP feature. + +So, we need to replace the LLVM intrinsic functions with the runtime self implemented functions, which can be called through the function pointer table (--enable-indirect-mode) and don't have the relocations to the libgcc/compiler-rt (--disable-llvm-intrinsics). + +Available intrinsic functions for tuning: + +| LLVM intrinsic function | Explanation | +| --- | --- | +| llvm.experimental.constrained.fadd.f32 | float32 add | +| llvm.experimental.constrained.fadd.f64 | float64 add | +| llvm.experimental.constrained.fsub.f32 | float32 sub | +| llvm.experimental.constrained.fsub.f64 | float64 sub | +| llvm.experimental.constrained.fmul.f32 | float32 mul | +| llvm.experimental.constrained.fmul.f64 | float64 mul | +| llvm.experimental.constrained.fdiv.f32 | float32 div | +| llvm.experimental.constrained.fdiv.f64 | float64 div | +| llvm.fabs.f32 | float32 abs | +| llvm.fabs.f64 | float64 abs | +| llvm.ceil.f32 | float32 ceil | +| llvm.ceil.f64 | float64 ceil | +| llvm.floor.f32 | float32 floor | +| llvm.floor.f64 | float64 floor | +| llvm.trunc.f32 | float32 trunc | +| llvm.trunc.f64 | float64 trunc | +| llvm.rint.f32 | float32 rint | +| llvm.rint.f64 | float64 rint | +| llvm.sqrt.f32 | float32 sqrt | +| llvm.sqrt.f64 | float64 sqrt | +| llvm.copysign.f32 | float32 copysign | +| llvm.copysign.f64 | float64 copysign | +| llvm.minnum.f32 | float32 minnum | +| llvm.minnum.f64 | float64 minnum | +| llvm.maxnum.f32 | float32 maxnum | +| llvm.maxnum.f64 | float64 maxnum | +| llvm.ctlz.i32 | int32 count leading zeros | +| llvm.ctlz.i64 | int64 count leading zeros | +| llvm.cttz.i32 | int32 count trailing zeros | +| llvm.cttz.i64 | int64 count trailing zeros | +| llvm.ctpop.i32 | int32 count population | +| llvm.ctpop.i64 | int64 count population | +| f64_convert_i32_s | int32 to float64 | +| f64_convert_i32_u | uint32 to float64 | +| f32_convert_i32_s | int32 to float32 | +| f32_convert_i32_u | uint32 to float32 | +| f64_convert_i64_s | int64 to float64 | +| f64_convert_i64_u | uint64 to float64 | +| f32_convert_i64_s | int64 to float32 | +| f32_convert_i64_u | uint64 to float32 | +| i32_trunc_f32_s | float32 to int32 | +| i32_trunc_f32_u | float32 to uint32 | +| i32_trunc_f64_s | float64 to int32 | +| i32_trunc_f64_u | float64 to uint32 | +| i64_trunc_f64_s | float64 to int64 | +| i64_trunc_f64_u | float64 to uint64 | +| i64_trunc_f32_s | float32 to int64 | +| i64_trunc_f32_u | float32 to uint64 | +| f32_demote_f64 | float64 to float32 | +| f64_promote_f32 | float32 to float64 | +| f32_cmp | float32 compare | +| f64_cmp | float64 compare | +| i64.div_s | int64 div | +| i64.div_u | uint64 div | +| i32.div_s | int32 div | +| i32.div_u | uint32 div | +| i64.rem_s | int64 rem | +| i64.rem_u | uint64 rem | +| i32.rem_s | int32 rem | +| i32.rem_u | uint32 rem | +| i64.or | int64 or | +| i64.and | int64 and | +| i32.const | emit i32 const into constant table | +| i64.const | emit i64 const into constant table | +| f32.const | emit f32 const into constant table | +| f64.const | emit f64 const into constant table | + +And also provide combined intrinsic functions to simplify the tuning: + +* all: all the above intrinsic functions +* i32.common: i32.div_s, i32.div_u, i32.rem_s, i32.rem_u +* i64.common: i64.div_s, i64.div_u, i64.rem_s, i64.rem_u, i64.or, i64.and +* f32.common: f32_cmp, llvm.experimental.constrained.fadd.f32, llvm.experimental.constrained.fsub.f32, llvm.experimental.constrained.fmul.f32, llvm.experimental.constrained.fdiv.f32, llvm.fabs.f32, llvm.ceil.f32, llvm.floor.f32, llvm.trunc.f32, llvm.rint.f32, llvm.sqrt.f32, llvm.copysign.f32, llvm.minnum.f32, llvm.maxnum.f32 +* f64.common: f32_demote_f64, f64_promote_f32, f64_cmp, llvm.experimental.constrained.fadd.f64, llvm.experimental.constrained.fsub.f64, llvm.experimental.constrained.fmul.f64, llvm.experimental.constrained.fdiv.f64, llvm.fabs.f64, llvm.ceil.f64, llvm.floor.f64, llvm.trunc.f64, llvm.rint.f64, llvm.sqrt.f64, llvm.copysign.f64, llvm.minnum.f64, llvm.maxnum.f64 +* f32xi32: i32_trunc_f32_s, i32_trunc_f32_u, f32_convert_i32_s, f32_convert_i32_u +* f64xi32: i32_trunc_f64_s, i32_trunc_f64_u, f64_convert_i32_s, f64_convert_i32_u +* f32xi64: i64_trunc_f32_s, i64_trunc_f32_u, f32_convert_i64_s, f32_convert_i64_u +* f64xi64: i64_trunc_f64_s, i64_trunc_f64_u, f64_convert_i64_s, f64_convert_i64_u +* constop: i32.const, i64.const, f32.const, f64.const +* fpxint: f32xi32, f64xi32, f32xi64, f64xi64 +* fp.common: f32.common, f64.common + + +### Example + +For ARM Cortex-M55, since it has double precision floating point unit, so it can support f32/f64 operations. But as a 32-bit MCU, it can only support 32-bit integer operations. So we can use the following command to generate the XIP binary: + +``` +wamrc --target=thumbv8m.main --cpu=cortex-m55 --xip --enable-builtin-intrinsics=i64.common -o hello.aot hello.wasm +``` + +For ARM Cortex-M3, since it has no floating point unit, and it can only support 32-bit integer operations. So we can use the following command to generate the XIP binary: + +``` +wamrc --target=thumbv7m --cpu=cortex-m3 --xip --enable-builtin-intrinsics=i64.common,fp.common,fpxint -o hello.aot hello.wasm +``` + +Other platforms can be tuned in the same way, which intrinsic should be enabled depends on the target platform's hardware capability. diff --git a/wasm-micro-runtime/idf_component.yml b/wasm-micro-runtime/idf_component.yml new file mode 100644 index 0000000..ff25b32 --- /dev/null +++ b/wasm-micro-runtime/idf_component.yml @@ -0,0 +1,15 @@ +version: "2.0.0" +description: WebAssembly Micro Runtime - A lightweight standalone WebAssembly (Wasm) runtime with small footprint, high performance and highly configurable features +url: https://bytecodealliance.org/ +repository: https://github.com/bytecodealliance/wasm-micro-runtime.git +documentation: https://wamr.gitbook.io/ +issues: https://github.com/bytecodealliance/wasm-micro-runtime/issues +dependencies: + idf: ">=4.4" +targets: + - esp32 + - esp32s3 + - esp32c3 + - esp32c6 +examples: + - path: product-mini/platforms/esp-idf \ No newline at end of file diff --git a/wasm-micro-runtime/language-bindings/go/README.md b/wasm-micro-runtime/language-bindings/go/README.md new file mode 100644 index 0000000..34baf21 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/README.md @@ -0,0 +1,104 @@ +WAMR Go binding: Embedding WAMR in Go guideline +=============================================== + +This Go library uses CGO to consume the runtime APIs of the WAMR project which are defined in [core/iwasm/include/wasm_export.h](../../core/iwasm/include/wasm_export.h). The API details are available in the header files. + +## Installation + +### Installing from the source code + +Installing from local source tree is in _development mode_. + +Run `./build.sh` in this folder to build the package, which builds the WAMR runtime library firstly and then builds the Go binding library. + +Run `./build.sh` under `samples` folder to build and test the sample. + +```bash +cd samples +./build.sh +``` + +## Supported APIs + +All the embedding APIs supported are defined under folder [wamr](./wamr). + +### Runtime APIs + +```Go +func Runtime() *_Runtime +func (self *_Runtime) FullInit(alloc_with_pool bool, heap_buf []byte, + max_thread_num uint) error +func (self *_Runtime) Init() error +func (self *_Runtime) Destroy() +func (self *_Runtime) SetLogLevel(level LogLevel) +func (self *_Runtime) Malloc(size uint32) *uint8 +func (self *_Runtime) Free(ptr *uint8) +``` + +### Module APIs + +```Go +func NewModule(wasmBytes []byte) (*Module, error) +func (self *Module) Destroy() +func (self *Module) SetWasiArgs(dirList [][]byte, mapDirList [][]byte, + env [][]byte, argv[][]byte) +func (self *Module) SetWasiArgsEx(dirList [][]byte, mapDirList [][]byte, + env [][]byte, argv[][]byte, + stdinfd int, stdoutfd int, stderrfd int) +func (self *Module) SetWasiAddrPool(addrPool [][]byte) +``` + +### Instance APIs + +```Go +func NewInstance(module *Module, + stackSize uint, heapSize uint) (*Instance, error) +func (self *Instance) Destroy() +func (self *Instance) CallFunc(funcName string, + argc uint32, args []uint32) error +func (self *Instance) CallFuncV(funcName string, + num_results uint32, results []interface{}, + args ... interface{}) error +func (self *Instance) GetException() string +func (self Instance) ModuleMalloc(size uint32) (uint32, *uint8) +func (self Instance) ModuleFree(offset uint32) +func (self Instance) ValidateAppAddr(app_offset uint32, size uint32) bool +func (self Instance) ValidateNativeAddr(native_ptr *uint8, size uint32) bool +func (self Instance) AddrAppToNative(app_offset uint32) *uint8 +func (self Instance) AddrNativeToApp(native_ptr *uint8) uint32 +func (self Instance) GetAppAddrRange(app_offset uint32) (bool, uint32, uint32) +func (self Instance) GetNativeAddrRange(native_ptr *uint8) (bool, *uint8, *uint8) +func (self Instance) DumpMemoryConsumption() +func (self Instance) DumpCallStack() +``` + +## Sample codes + +```Go + var module *wamr.Module + var instance *wamr.Instance + var results []interface{} + var err error + + /* Runtime initialization */ + err = wamr.Runtime().FullInit(false, nil, 1) + + /* Read WASM/AOT file into a memory buffer */ + wasmBytes := read_wasm_binary_to_buffer(...) + + /* Load WASM/AOT module from the memory buffer */ + module, err = wamr.NewModule(wasmBytes) + + /* Create WASM/AOT instance from the module */ + instance, err = wamr.NewInstance(module, 16384, 16384) + + /* Call the `fib` function */ + results = make([]interface{}, 1, 1) + err = instance.CallFuncV("fib", 1, results, (int32)32) + fmt.Printf("fib(32) return: %d\n", results[0].(int32)); + + /* Destroy runtime */ + wamr.Runtime().Destroy() +``` + +More samples can be found in [test.go](./samples/test.go) diff --git a/wasm-micro-runtime/language-bindings/go/build.sh b/wasm-micro-runtime/language-bindings/go/build.sh new file mode 100755 index 0000000..fe46a9a --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) +CUR_DIR=$PWD +WAMR_DIR=$PWD/../.. +WAMR_GO_DIR=$PWD/wamr +ARCH=$(uname -m) +if [ ${ARCH} = "arm64" ]; then + ARCH="aarch64" +elif [ ${ARCH} = "x86_64" ]; then + ARCH="amd64" +fi + +cp -a ${WAMR_DIR}/core/iwasm/include/*.h ${WAMR_GO_DIR}/packaged/include + +mkdir -p build && cd build +cmake ${WAMR_DIR}/product-mini/platforms/${PLATFORM} \ + -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_DUMP_CALL_STACK=1 \ + -DWAMR_BUILD_MEMORY_PROFILING=1 +make -j ${nproc} +cp -a libvmlib.a ${WAMR_GO_DIR}/packaged/lib/${PLATFORM}-${ARCH} + +cd ${WAMR_GO_DIR} +go test diff --git a/wasm-micro-runtime/language-bindings/go/go.mod b/wasm-micro-runtime/language-bindings/go/go.mod new file mode 100644 index 0000000..b7e4286 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/go.mod @@ -0,0 +1,5 @@ +module github.com/bytecodealliance/wasm-micro-runtime/language-bindings/go + +go 1.15 + +require github.com/stretchr/testify v1.7.0 diff --git a/wasm-micro-runtime/language-bindings/go/go.sum b/wasm-micro-runtime/language-bindings/go/go.sum new file mode 100644 index 0000000..acb88a4 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/wasm-micro-runtime/language-bindings/go/samples/build.sh b/wasm-micro-runtime/language-bindings/go/samples/build.sh new file mode 100755 index 0000000..0799fe5 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/samples/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD + +pushd ${CUR_DIR}/.. > /dev/null 2>&1 +./build.sh +popd > /dev/null 2>& 1 + +cd ${CUR_DIR} +rm -f test +go build test.go +./test diff --git a/wasm-micro-runtime/language-bindings/go/samples/run.sh b/wasm-micro-runtime/language-bindings/go/samples/run.sh new file mode 100755 index 0000000..a57da7f --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/samples/run.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +go build test.go +./test diff --git a/wasm-micro-runtime/language-bindings/go/samples/test.go b/wasm-micro-runtime/language-bindings/go/samples/test.go new file mode 100644 index 0000000..19b2814 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/samples/test.go @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package main + +import ( + "github.com/bytecodealliance/wasm-micro-runtime/language-bindings/go/wamr" + "fmt" +) + +var wasmBytes = []byte { + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x29, 0x07, 0x60, + 0x01, 0x7F, 0x01, 0x7F, 0x60, 0x02, 0x7F, 0x7F, 0x01, 0x7F, 0x60, 0x01, + 0x7F, 0x00, 0x60, 0x04, 0x7F, 0x7E, 0x7D, 0x7C, 0x00, 0x60, 0x02, 0x7E, + 0x7E, 0x01, 0x7E, 0x60, 0x02, 0x7C, 0x7F, 0x01, 0x7D, 0x60, 0x02, 0x7D, + 0x7C, 0x01, 0x7C, 0x02, 0x31, 0x04, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x70, + 0x72, 0x69, 0x6E, 0x74, 0x66, 0x00, 0x01, 0x03, 0x65, 0x6E, 0x76, 0x04, + 0x70, 0x75, 0x74, 0x73, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x6D, + 0x61, 0x6C, 0x6C, 0x6F, 0x63, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x04, + 0x66, 0x72, 0x65, 0x65, 0x00, 0x02, 0x03, 0x07, 0x06, 0x00, 0x03, 0x04, + 0x06, 0x05, 0x01, 0x05, 0x03, 0x01, 0x00, 0x01, 0x06, 0x13, 0x03, 0x7F, + 0x01, 0x41, 0x90, 0x29, 0x0B, 0x7F, 0x00, 0x41, 0x90, 0x09, 0x0B, 0x7F, + 0x00, 0x41, 0x90, 0x29, 0x0B, 0x07, 0x5F, 0x09, 0x06, 0x6D, 0x65, 0x6D, + 0x6F, 0x72, 0x79, 0x02, 0x00, 0x04, 0x66, 0x69, 0x62, 0x32, 0x00, 0x04, + 0x05, 0x74, 0x65, 0x73, 0x74, 0x31, 0x00, 0x05, 0x05, 0x74, 0x65, 0x73, + 0x74, 0x32, 0x00, 0x06, 0x05, 0x74, 0x65, 0x73, 0x74, 0x33, 0x00, 0x07, + 0x05, 0x74, 0x65, 0x73, 0x74, 0x34, 0x00, 0x08, 0x10, 0x5F, 0x5F, 0x6D, + 0x61, 0x69, 0x6E, 0x5F, 0x61, 0x72, 0x67, 0x63, 0x5F, 0x61, 0x72, 0x67, + 0x76, 0x00, 0x09, 0x0A, 0x5F, 0x5F, 0x64, 0x61, 0x74, 0x61, 0x5F, 0x65, + 0x6E, 0x64, 0x03, 0x01, 0x0B, 0x5F, 0x5F, 0x68, 0x65, 0x61, 0x70, 0x5F, + 0x62, 0x61, 0x73, 0x65, 0x03, 0x02, 0x0A, 0xA5, 0x03, 0x06, 0x37, 0x01, + 0x01, 0x7F, 0x41, 0x01, 0x21, 0x01, 0x20, 0x00, 0x41, 0x02, 0x4F, 0x04, + 0x7F, 0x41, 0x00, 0x21, 0x01, 0x03, 0x40, 0x20, 0x00, 0x41, 0x02, 0x6B, + 0x10, 0x04, 0x20, 0x01, 0x6A, 0x21, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6B, + 0x22, 0x00, 0x41, 0x01, 0x4B, 0x0D, 0x00, 0x0B, 0x20, 0x01, 0x41, 0x01, + 0x6A, 0x05, 0x41, 0x01, 0x0B, 0x0B, 0x3F, 0x01, 0x01, 0x7F, 0x23, 0x00, + 0x41, 0x20, 0x6B, 0x22, 0x04, 0x24, 0x00, 0x20, 0x04, 0x41, 0x18, 0x6A, + 0x20, 0x03, 0x39, 0x03, 0x00, 0x20, 0x04, 0x41, 0x10, 0x6A, 0x20, 0x02, + 0xBB, 0x39, 0x03, 0x00, 0x20, 0x04, 0x20, 0x01, 0x37, 0x03, 0x08, 0x20, + 0x04, 0x20, 0x00, 0x36, 0x02, 0x00, 0x41, 0xD0, 0x08, 0x20, 0x04, 0x10, + 0x00, 0x1A, 0x20, 0x04, 0x41, 0x20, 0x6A, 0x24, 0x00, 0x0B, 0x3B, 0x01, + 0x01, 0x7F, 0x23, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, 0x24, 0x00, 0x20, + 0x02, 0x20, 0x00, 0x37, 0x03, 0x00, 0x20, 0x02, 0x20, 0x01, 0x37, 0x03, + 0x08, 0x20, 0x02, 0x41, 0x10, 0x6A, 0x20, 0x00, 0x20, 0x01, 0x7C, 0x22, + 0x00, 0x37, 0x03, 0x00, 0x41, 0xF6, 0x08, 0x20, 0x02, 0x10, 0x00, 0x1A, + 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x00, 0x20, 0x00, 0x0B, 0x40, 0x02, + 0x01, 0x7F, 0x01, 0x7C, 0x23, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, 0x24, + 0x00, 0x20, 0x02, 0x20, 0x01, 0x39, 0x03, 0x08, 0x20, 0x02, 0x20, 0x00, + 0xBB, 0x22, 0x03, 0x39, 0x03, 0x00, 0x20, 0x02, 0x41, 0x10, 0x6A, 0x20, + 0x03, 0x20, 0x01, 0xA2, 0x22, 0x01, 0x39, 0x03, 0x00, 0x41, 0xB4, 0x08, + 0x20, 0x02, 0x10, 0x00, 0x1A, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x00, + 0x20, 0x01, 0x0B, 0x3D, 0x01, 0x01, 0x7F, 0x23, 0x00, 0x41, 0x20, 0x6B, + 0x22, 0x02, 0x24, 0x00, 0x20, 0x02, 0x20, 0x00, 0x39, 0x03, 0x00, 0x20, + 0x02, 0x20, 0x01, 0x36, 0x02, 0x08, 0x20, 0x02, 0x41, 0x10, 0x6A, 0x20, + 0x00, 0x20, 0x01, 0xB7, 0xA3, 0x22, 0x00, 0x39, 0x03, 0x00, 0x41, 0xC2, + 0x08, 0x20, 0x02, 0x10, 0x00, 0x1A, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, + 0x00, 0x20, 0x00, 0xB6, 0x0B, 0x70, 0x00, 0x23, 0x00, 0x41, 0x20, 0x6B, + 0x22, 0x00, 0x24, 0x00, 0x41, 0x9A, 0x08, 0x10, 0x01, 0x1A, 0x02, 0x7F, + 0x41, 0x80, 0x08, 0x10, 0x02, 0x22, 0x01, 0x45, 0x04, 0x40, 0x41, 0x88, + 0x08, 0x10, 0x01, 0x1A, 0x41, 0x7F, 0x0C, 0x01, 0x0B, 0x20, 0x00, 0x20, + 0x01, 0x36, 0x02, 0x10, 0x41, 0xA7, 0x08, 0x20, 0x00, 0x41, 0x10, 0x6A, + 0x10, 0x00, 0x1A, 0x20, 0x01, 0x41, 0x04, 0x6A, 0x41, 0x8E, 0x09, 0x2F, + 0x00, 0x00, 0x3B, 0x00, 0x00, 0x20, 0x01, 0x41, 0x8A, 0x09, 0x28, 0x00, + 0x00, 0x36, 0x00, 0x00, 0x20, 0x00, 0x20, 0x01, 0x36, 0x02, 0x00, 0x41, + 0x80, 0x08, 0x20, 0x00, 0x10, 0x00, 0x1A, 0x20, 0x01, 0x10, 0x03, 0x41, + 0x00, 0x0B, 0x20, 0x00, 0x41, 0x20, 0x6A, 0x24, 0x00, 0x0B, 0x0B, 0x97, + 0x01, 0x01, 0x00, 0x41, 0x80, 0x08, 0x0B, 0x8F, 0x01, 0x62, 0x75, 0x66, + 0x3A, 0x20, 0x25, 0x73, 0x00, 0x6D, 0x61, 0x6C, 0x6C, 0x6F, 0x63, 0x20, + 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, 0x64, 0x00, 0x48, + 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, + 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, 0x20, 0x25, 0x70, 0x0A, + 0x00, 0x25, 0x66, 0x20, 0x2A, 0x20, 0x25, 0x66, 0x20, 0x3D, 0x20, 0x25, + 0x66, 0x0A, 0x00, 0x25, 0x66, 0x20, 0x2F, 0x20, 0x25, 0x64, 0x20, 0x3D, + 0x20, 0x25, 0x66, 0x0A, 0x00, 0x69, 0x33, 0x32, 0x3A, 0x20, 0x25, 0x64, + 0x2C, 0x20, 0x69, 0x36, 0x34, 0x3A, 0x20, 0x25, 0x6C, 0x6C, 0x64, 0x2C, + 0x20, 0x66, 0x33, 0x32, 0x3A, 0x20, 0x25, 0x66, 0x2C, 0x20, 0x66, 0x36, + 0x34, 0x3A, 0x20, 0x25, 0x66, 0x0A, 0x00, 0x25, 0x6C, 0x6C, 0x64, 0x20, + 0x2B, 0x20, 0x25, 0x6C, 0x6C, 0x64, 0x20, 0x3D, 0x20, 0x25, 0x6C, 0x6C, + 0x64, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A } + +var global_heap []byte = make([]byte, 128 * 1024, 128 * 1024) + +func main() { + var module *wamr.Module + var instance *wamr.Instance + var argv []uint32 + var results []interface{} + var offset uint64 + var native_addr *uint8 + var err error + + fmt.Print("Init wasm runtime with global heap buf\n"); + err = wamr.Runtime().FullInit(true, global_heap, 1) + if err != nil { + return + } + fmt.Print("Destroy runtime\n"); + wamr.Runtime().Destroy() + + fmt.Print("Init wasm runtime without global heap buf\n"); + err = wamr.Runtime().FullInit(false, nil, 1) + if err != nil { + return + } + + wamr.Runtime().SetLogLevel(wamr.LOG_LEVEL_WARNING) + + fmt.Print("Load wasm module\n"); + module, err = wamr.NewModule(wasmBytes) + if err != nil { + fmt.Println(err) + goto fail + } + + fmt.Print("Instantiate wasm module\n"); + instance, err = wamr.NewInstance(module, 16384, 16384) + if err != nil { + fmt.Println(err) + goto fail + } + + results = make([]interface{}, 8, 8) + argv = make([]uint32, 8) + + fmt.Print("\nCall func __main_argc_argv with CallFunc:\n"); + err = instance.CallFunc("__main_argc_argv", 2, argv) + if err != nil { + fmt.Println(err) + goto fail + } + + fmt.Print("\nCall func __main_argc_argv with CallFuncV:\n"); + err = instance.CallFuncV("__main_argc_argv", 2, results, + (int32)(0), (int32)(0)) + if err != nil { + fmt.Println(err) + goto fail + } + + fmt.Print("\nCall func `i32 fib2(i32)` with CallFunc:\n"); + argv[0] = 32 + err = instance.CallFunc("fib2", 1, argv) + if err != nil { + fmt.Println(err) + goto fail + } + fmt.Printf("fib2(32) return: %d\n", argv[0]); + + fmt.Print("\nCall func `void test1(i32, i64, f32, f64)` with CallFuncV:\n"); + err = instance.CallFuncV("test1", 0, nil, + (int32)(12345678), + (int64)(3344556677889900), + (float32)(5678.1234), + (float64)(987654321.5678)) + if err != nil { + fmt.Println(err) + goto fail + } + + fmt.Print("\nCall func `i64 test2(i64, i64)` with CallFuncV:\n"); + err = instance.CallFuncV("test2", 1, results, + (int64)(3344556677889900), + (int64)(1122331122110099)) + if err != nil { + fmt.Println(err) + goto fail + } + fmt.Printf("test2(3344556677889900, 1122331122110099) return: %d\n", + results[0].(int64)) + + fmt.Print("\nCall func `f64 test3(f32, f64)` with CallFuncV:\n"); + err = instance.CallFuncV("test3", 1, results, + (float32)(3456.1234), + (float64)(7890.4567)) + if err != nil { + fmt.Println(err) + goto fail + } + fmt.Printf("test3(3456.1234, 7890.4567) return: %f\n", + results[0].(float64)) + + fmt.Print("\nCall func `f32 test4(f64, i32)` with CallFuncV:\n"); + err = instance.CallFuncV("test4", 1, results, + (float64)(8912.3456), + (int32)(123)) + if err != nil { + fmt.Println(err) + goto fail + } + fmt.Printf("test4(8912.3456, 123) return: %f\n", + results[0].(float32)) + + fmt.Print("\nTest ModuleMalloc") + offset, native_addr = instance.ModuleMalloc(1024) + fmt.Printf("ModuleMalloc(%d) return offset: %d, native addr: %p\n", + 1024, offset, native_addr) + + if (!instance.ValidateAppAddr(offset, 1024)) { + fmt.Print("Validate app addr failed\n") + } + if (!instance.ValidateNativeAddr(native_addr, 1024)) { + fmt.Print("Validate native addr failed\n") + } + if (native_addr != instance.AddrAppToNative(offset)) { + fmt.Print("Convert app addr to native addr failed\n") + } + if (offset != instance.AddrNativeToApp(native_addr)) { + fmt.Print("Convert app addr to native addr failed\n") + } + + instance.ModuleFree(offset) + + /* + instance.DumpMemoryConsumption() + instance.DumpCallStack() + */ + + fmt.Print("\n"); + +fail: + if (instance != nil) { + fmt.Print("Destroy instance\n"); + instance.Destroy() + } + + if (module != nil) { + fmt.Print("Destroy module\n"); + module.Destroy() + } + + fmt.Print("Destroy wasm runtime\n"); + wamr.Runtime().Destroy() +} diff --git a/wasm-micro-runtime/language-bindings/go/samples/wasm-app/build.sh b/wasm-micro-runtime/language-bindings/go/samples/wasm-app/build.sh new file mode 100755 index 0000000..1d0cc83 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/samples/wasm-app/build.sh @@ -0,0 +1,32 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +WAMR_DIR=${PWD}/../../.. + +echo "Build wasm app .." +/opt/wasi-sdk/bin/clang -O3 \ + -z stack-size=4096 -Wl,--initial-memory=65536 \ + -o test.wasm main.c \ + -Wl,--export=main -Wl,--export=__main_argc_argv \ + -Wl,--export=fib2 \ + -Wl,--export=test1 \ + -Wl,--export=test2 \ + -Wl,--export=test3 \ + -Wl,--export=test4 \ + -Wl,--export=__data_end -Wl,--export=__heap_base \ + -Wl,--strip-all,--no-entry \ + -Wl,--allow-undefined \ + -nostdlib \ + +echo "Build binarydump tool .." +rm -fr build && mkdir build && cd build +cmake ../../../../../test-tools/binarydump-tool +make +cd .. + +echo "Generate test_wasm.h .." +./build/binarydump -o test_wasm.h -n wasm_test_file test.wasm + +rm -fr build + +echo "Done" diff --git a/wasm-micro-runtime/language-bindings/go/samples/wasm-app/main.c b/wasm-micro-runtime/language-bindings/go/samples/wasm-app/main.c new file mode 100644 index 0000000..8d1f912 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/samples/wasm-app/main.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +unsigned +fib2(unsigned n) +{ + if (n < 2) { + return 1; + } + return fib2(n - 2) + fib2(n - 1); +} + +void +test1(int32_t i32, int64_t i64, float f32, double f64) +{ + printf("i32: %d, i64: %lld, f32: %f, f64: %f\n", i32, i64, f32, f64); +} + +int64_t +test2(int64_t x, int64_t y) +{ + printf("%lld + %lld = %lld\n", x, y, x + y); + return x + y; +} + +double +test3(float x, double y) +{ + printf("%f * %f = %f\n", x, y, x * y); + return x * y; +} + +float +test4(double x, int32_t y) +{ + printf("%f / %d = %f\n", x, y, x / y); + return x / y; +} + +int +main(int argc, char **argv) +{ + char *buf; + + printf("Hello world!\n"); + + buf = malloc(1024); + if (!buf) { + printf("malloc buf failed\n"); + return -1; + } + + printf("buf ptr: %p\n", buf); + + snprintf(buf, 1024, "%s", "1234\n"); + printf("buf: %s", buf); + + free(buf); + return 0; +} diff --git a/wasm-micro-runtime/language-bindings/go/wamr/cgo.go b/wasm-micro-runtime/language-bindings/go/wamr/cgo.go new file mode 100644 index 0000000..74766c3 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/cgo.go @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package wamr + +// #cgo CFLAGS: -I${SRCDIR}/packaged/include +// #cgo LDFLAGS: -lvmlib -lm +// +// #cgo linux,amd64 LDFLAGS: -Wl,-rpath,${SRCDIR}/packaged/lib/linux-amd64 -L${SRCDIR}/packaged/lib/linux-amd64 +// #cgo linux,arm64 LDFLAGS: -Wl,-rpath,${SRCDIR}/packaged/lib/linux-aarch64 -L${SRCDIR}/packaged/lib/linux-aarch64 +// #cgo darwin,amd64 LDFLAGS: -Wl,-rpath,${SRCDIR}/packaged/lib/darwin-amd64 -L${SRCDIR}/packaged/lib/darwin-amd64 +// #cgo darwin,arm64 LDFLAGS: -Wl,-rpath,${SRCDIR}/packaged/lib/darwin-aarch64 -L${SRCDIR}/packaged/lib/darwin-aarch64 +// +// #include +import "C" + +import ( +) diff --git a/wasm-micro-runtime/language-bindings/go/wamr/instance.go b/wasm-micro-runtime/language-bindings/go/wamr/instance.go new file mode 100644 index 0000000..111d795 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/instance.go @@ -0,0 +1,385 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package wamr + +/* +#include +#include + +static inline void +PUT_I64_TO_ADDR(uint32_t *addr, int64_t value) +{ + union { + int64_t val; + uint32_t parts[2]; + } u; + u.val = value; + addr[0] = u.parts[0]; + addr[1] = u.parts[1]; +} + +static inline void +PUT_F64_TO_ADDR(uint32_t *addr, double value) +{ + union { + double val; + uint32_t parts[2]; + } u; + u.val = value; + addr[0] = u.parts[0]; + addr[1] = u.parts[1]; +} + +static inline int64_t +GET_I64_FROM_ADDR(uint32_t *addr) +{ + union { + int64_t val; + uint32_t parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} + +static inline double +GET_F64_FROM_ADDR(uint32_t *addr) +{ + union { + double val; + uint32_t parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} +*/ +import "C" + +import ( + "runtime" + "unsafe" + "fmt" +) + +type Instance struct { + _instance C.wasm_module_inst_t + _exec_env C.wasm_exec_env_t + _module *Module + _exportsCache map[string]C.wasm_function_inst_t +} + +/* Create instance from the module */ +func NewInstance(module *Module, + stackSize uint, heapSize uint) (*Instance, error) { + if (module == nil) { + return nil, fmt.Errorf("NewInstance error: invalid input") + } + + errorBytes := make([]byte, 128) + errorPtr := (*C.char)(unsafe.Pointer(&errorBytes[0])) + errorLen := C.uint(len(errorBytes)) + + instance := C.wasm_runtime_instantiate(module.module, C.uint(stackSize), + C.uint(heapSize), errorPtr, errorLen) + if (instance == nil) { + return nil, fmt.Errorf("NewInstance Error: %s", string(errorBytes)) + } + + exec_env := C.wasm_runtime_create_exec_env(instance, C.uint(stackSize)) + if (exec_env == nil) { + C.wasm_runtime_deinstantiate(instance) + return nil, fmt.Errorf("NewInstance Error: create exec_env failed") + } + + self := &Instance{ + _instance: instance, + _exec_env: exec_env, + _module: module, + _exportsCache: make(map[string]C.wasm_function_inst_t), + } + + runtime.SetFinalizer(self, func(self *Instance) { + self.Destroy() + }) + + return self, nil +} + +/* Destroy the instance */ +func (self *Instance) Destroy() { + runtime.SetFinalizer(self, nil) + if (self._instance != nil) { + C.wasm_runtime_deinstantiate(self._instance) + } + if (self._exec_env != nil) { + C.wasm_runtime_destroy_exec_env(self._exec_env) + } +} + +/* Call the wasm function with argument in the uint32 array, and store + the return values back into the array */ +func (self *Instance) CallFunc(funcName string, + argc uint32, args []uint32) error { + _func := self._exportsCache[funcName] + if _func == nil { + cName := C.CString(funcName) + defer C.free(unsafe.Pointer(cName)) + + _func = C.wasm_runtime_lookup_function(self._instance, cName) + if _func == nil { + return fmt.Errorf("CallFunc error: lookup function failed") + } + self._exportsCache[funcName] = _func + } + + thread_env_inited := Runtime().ThreadEnvInited() + if (!thread_env_inited) { + Runtime().InitThreadEnv() + } + + var args_C *C.uint32_t + if (argc > 0) { + args_C = (*C.uint32_t)(unsafe.Pointer(&args[0])) + } + if (!C.wasm_runtime_call_wasm(self._exec_env, _func, + C.uint(argc), args_C)) { + if (!thread_env_inited) { + Runtime().DestroyThreadEnv() + } + return fmt.Errorf("CallFunc error: %s", string(self.GetException())) + } + + if (!thread_env_inited) { + Runtime().DestroyThreadEnv() + } + return nil +} + +/* Call the wasm function with variant arguments, and store the return + values back into the results array */ +func (self *Instance) CallFuncV(funcName string, + num_results uint32, results []interface{}, + args ... interface{}) error { + _func := self._exportsCache[funcName] + if _func == nil { + cName := C.CString(funcName) + defer C.free(unsafe.Pointer(cName)) + + _func = C.wasm_runtime_lookup_function(self._instance, cName) + if _func == nil { + return fmt.Errorf("CallFunc error: lookup function failed") + } + self._exportsCache[funcName] = _func + } + + param_count := uint32(C.wasm_func_get_param_count(_func, self._instance)) + result_count := uint32(C.wasm_func_get_result_count(_func, self._instance)) + + if (num_results < result_count) { + str := "CallFunc error: invalid result count %d, " + + "must be no smaller than %d" + return fmt.Errorf(str, num_results, result_count) + } + + param_types := make([]C.uchar, param_count, param_count) + result_types := make([]C.uchar, result_count, result_count) + if (param_count > 0) { + C.wasm_func_get_param_types(_func, self._instance, + (*C.uchar)(unsafe.Pointer(¶m_types[0]))) + } + if (result_count > 0) { + C.wasm_func_get_result_types(_func, self._instance, + (*C.uchar)(unsafe.Pointer(&result_types[0]))) + } + + argv_size := param_count * 2 + if (result_count > param_count) { + argv_size = result_count * 2 + } + argv := make([]uint32, argv_size, argv_size) + + var i, argc uint32 + for _, arg := range args { + if (i >= param_count) { + break; + } + switch arg.(type) { + case int32: + if (param_types[i] != C.WASM_I32 && + param_types[i] != C.WASM_FUNCREF && + param_types[i] != C.WASM_ANYREF) { + str := "CallFunc error: invalid param type %d, " + + "expect i32 but got other" + return fmt.Errorf(str, param_types[i]) + } + argv[argc] = (uint32)(arg.(int32)) + argc++ + break + case int64: + if (param_types[i] != C.WASM_I64) { + str := "CallFunc error: invalid param type %d, " + + "expect i64 but got other" + return fmt.Errorf(str, param_types[i]) + } + addr := (*C.uint32_t)(unsafe.Pointer(&argv[argc])) + C.PUT_I64_TO_ADDR(addr, (C.int64_t)(arg.(int64))) + argc += 2 + break + case float32: + if (param_types[i] != C.WASM_F32) { + str := "CallFunc error: invalid param type %d, " + + "expect f32 but got other" + return fmt.Errorf(str, param_types[i]) + } + *(*C.float)(unsafe.Pointer(&argv[argc])) = (C.float)(arg.(float32)) + argc++ + break + case float64: + if (param_types[i] != C.WASM_F64) { + str := "CallFunc error: invalid param type %d, " + + "expect f64 but got other" + return fmt.Errorf(str, param_types[i]) + } + addr := (*C.uint32_t)(unsafe.Pointer(&argv[argc])) + C.PUT_F64_TO_ADDR(addr, (C.double)(arg.(float64))) + argc += 2 + break + default: + return fmt.Errorf("CallFunc error: unknown param type %d", + param_types[i]) + } + i++ + } + + if (i < param_count) { + str := "CallFunc error: invalid param count, " + + "must be no smaller than %d" + return fmt.Errorf(str, param_count) + } + + err := self.CallFunc(funcName, argc, argv) + if (err != nil) { + return err + } + + argc = 0 + for i = 0; i < result_count; i++ { + switch result_types[i] { + case C.WASM_I32: + fallthrough + case C.WASM_FUNCREF: + fallthrough + case C.WASM_ANYREF: + i32 := (int32)(argv[argc]) + results[i] = i32 + argc++ + break + case C.WASM_I64: + addr := (*C.uint32_t)(unsafe.Pointer(&argv[argc])) + results[i] = (int64)(C.GET_I64_FROM_ADDR(addr)) + argc += 2 + break + case C.WASM_F32: + addr := (*C.float)(unsafe.Pointer(&argv[argc])) + results[i] = (float32)(*addr) + argc++ + break + case C.WASM_F64: + addr := (*C.uint32_t)(unsafe.Pointer(&argv[argc])) + results[i] = (float64)(C.GET_F64_FROM_ADDR(addr)) + argc += 2 + break + } + } + + return nil +} + +/* Get exception info of the instance */ +func (self *Instance) GetException() string { + cStr := C.wasm_runtime_get_exception(self._instance) + goStr := C.GoString(cStr) + return goStr +} + +/* Allocate memory from the heap of the instance */ +func (self Instance) ModuleMalloc(size uint64) (uint64, *uint8) { + var offset C.uint64_t + native_addrs := make([]*uint8, 1, 1) + ptr := unsafe.Pointer(&native_addrs[0]) + offset = C.wasm_runtime_module_malloc(self._instance, (C.uint64_t)(size), + (*unsafe.Pointer)(ptr)) + return (uint64)(offset), native_addrs[0] +} + +/* Free memory to the heap of the instance */ +func (self Instance) ModuleFree(offset uint64) { + C.wasm_runtime_module_free(self._instance, (C.uint64_t)(offset)) +} + +func (self Instance) ValidateAppAddr(app_offset uint64, size uint64) bool { + ret := C.wasm_runtime_validate_app_addr(self._instance, + (C.uint64_t)(app_offset), + (C.uint64_t)(size)) + return (bool)(ret) +} + +func (self Instance) ValidateStrAddr(app_str_offset uint64) bool { + ret := C.wasm_runtime_validate_app_str_addr(self._instance, + (C.uint64_t)(app_str_offset)) + return (bool)(ret) +} + +func (self Instance) ValidateNativeAddr(native_ptr *uint8, size uint64) bool { + native_ptr_C := (unsafe.Pointer)(native_ptr) + ret := C.wasm_runtime_validate_native_addr(self._instance, + native_ptr_C, + (C.uint64_t)(size)) + return (bool)(ret) +} + +func (self Instance) AddrAppToNative(app_offset uint64) *uint8 { + native_ptr := C.wasm_runtime_addr_app_to_native(self._instance, + (C.uint64_t)(app_offset)) + return (*uint8)(native_ptr) +} + +func (self Instance) AddrNativeToApp(native_ptr *uint8) uint64 { + native_ptr_C := (unsafe.Pointer)(native_ptr) + offset := C.wasm_runtime_addr_native_to_app(self._instance, + native_ptr_C) + return (uint64)(offset) +} + +func (self Instance) GetAppAddrRange(app_offset uint64) (bool, + uint64, + uint64) { + var start_offset, end_offset C.uint64_t + ret := C.wasm_runtime_get_app_addr_range(self._instance, + (C.uint64_t)(app_offset), + &start_offset, &end_offset) + return (bool)(ret), (uint64)(start_offset), (uint64)(end_offset) +} + +func (self Instance) GetNativeAddrRange(native_ptr *uint8) (bool, + *uint8, + *uint8) { + var start_addr, end_addr *C.uint8_t + native_ptr_C := (*C.uint8_t)((unsafe.Pointer)(native_ptr)) + ret := C.wasm_runtime_get_native_addr_range(self._instance, + native_ptr_C, + &start_addr, &end_addr) + return (bool)(ret), (*uint8)(start_addr), (*uint8)(end_addr) +} + +func (self Instance) DumpMemoryConsumption() { + C.wasm_runtime_dump_mem_consumption(self._exec_env) +} + +func (self Instance) DumpCallStack() { + C.wasm_runtime_dump_call_stack(self._exec_env) +} diff --git a/wasm-micro-runtime/language-bindings/go/wamr/instance_test.go b/wasm-micro-runtime/language-bindings/go/wamr/instance_test.go new file mode 100644 index 0000000..ad679e2 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/instance_test.go @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package wamr + +import ( + //"github.com/stretchr/testify/assert" + "testing" +) + +func TestInstance(t *testing.T) { + /* TODO */ +} + +func TestCallFunc(t *testing.T) { + /* TODO */ +} diff --git a/wasm-micro-runtime/language-bindings/go/wamr/module.go b/wasm-micro-runtime/language-bindings/go/wamr/module.go new file mode 100644 index 0000000..7fd131e --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/module.go @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package wamr + +// #include +import "C" +import ( + "unsafe" + "runtime" + "fmt" +) + +type Module struct { + module C.wasm_module_t +} + +/* Create WASM/AOT module from the memory buffer */ +func NewModule(wasmBytes []byte) (*Module, error) { + if (wasmBytes == nil || len(wasmBytes) == 0) { + return nil, fmt.Errorf("NewModule error: invalid input") + } + + wasmPtr := (*C.uint8_t)(unsafe.Pointer(&wasmBytes[0])) + wasmLen := C.uint(len(wasmBytes)) + + errorBytes := make([]byte, 128) + errorPtr := (*C.char)(unsafe.Pointer(&errorBytes[0])) + errorLen := C.uint(len(errorBytes)) + + m := C.wasm_runtime_load(wasmPtr, wasmLen, errorPtr, errorLen) + if (m == nil) { + return nil, fmt.Errorf("NewModule error: %s", string(errorBytes)) + } + + self := &Module{ + module: m, + } + + runtime.SetFinalizer(self, func(self *Module) { + self.Destroy() + }) + + return self, nil +} + +/* Destroy the module */ +func (self *Module) Destroy() { + runtime.SetFinalizer(self, nil) + if (self.module != nil) { + C.wasm_runtime_unload(self.module) + } +} + +/* Set module's wasi arguments */ +func (self *Module) SetWasiArgs(dirList [][]byte, mapDirList [][]byte, + env [][]byte, argv[][]byte) { + var dirPtr, mapDirPtr, envPtr, argvPtr **C.char + var dirCount, mapDirCount, envCount C.uint + var argc C.int + + if (dirList != nil) { + dirPtr = (**C.char)(unsafe.Pointer(&dirList[0])) + dirCount = C.uint(len(dirList)) + } + + if (mapDirList != nil) { + mapDirPtr = (**C.char)(unsafe.Pointer(&mapDirList[0])) + mapDirCount = C.uint(len(mapDirList)) + } + + if (env != nil) { + envPtr = (**C.char)(unsafe.Pointer(&env[0])) + envCount = C.uint(len(env)) + } + + if (argv != nil) { + argvPtr = (**C.char)(unsafe.Pointer(&argv[0])) + argc = C.int(len(argv)) + } + + C.wasm_runtime_set_wasi_args(self.module, dirPtr, dirCount, + mapDirPtr, mapDirCount, + envPtr, envCount, argvPtr, argc) +} + +/* Set module's wasi arguments */ +func (self *Module) SetWasiArgsEx(dirList [][]byte, mapDirList [][]byte, + env [][]byte, argv[][]byte, + stdinfd int, stdoutfd int, stderrfd int) { + var dirPtr, mapDirPtr, envPtr, argvPtr **C.char + var dirCount, mapDirCount, envCount C.uint + var argc C.int + + if (dirList != nil) { + dirPtr = (**C.char)(unsafe.Pointer(&dirList[0])) + dirCount = C.uint(len(dirList)) + } + + if (mapDirList != nil) { + mapDirPtr = (**C.char)(unsafe.Pointer(&mapDirList[0])) + mapDirCount = C.uint(len(mapDirList)) + } + + if (env != nil) { + envPtr = (**C.char)(unsafe.Pointer(&env[0])) + envCount = C.uint(len(env)) + } + + if (argv != nil) { + argvPtr = (**C.char)(unsafe.Pointer(&argv[0])) + argc = C.int(len(argv)) + } + + C.wasm_runtime_set_wasi_args_ex(self.module, dirPtr, dirCount, + mapDirPtr, mapDirCount, + envPtr, envCount, argvPtr, argc, + C.int64_t(stdinfd), C.int64_t(stdoutfd), + C.int64_t(stderrfd)) +} + +/* Set module's wasi network address pool */ +func (self *Module) SetWasiAddrPool(addrPool [][]byte) { + var addrPoolPtr **C.char + var addrPoolSize C.uint + + if (addrPool != nil) { + addrPoolPtr = (**C.char)(unsafe.Pointer(&addrPool[0])) + addrPoolSize = C.uint(len(addrPool)) + } + C.wasm_runtime_set_wasi_addr_pool(self.module, addrPoolPtr, addrPoolSize) +} + +/* Set module's wasi domain lookup pool */ +func(self *Module) SetWasiNsLookupPool(nsLookupPool [][]byte) { + var nsLookupPoolPtr **C.char + var nsLookupPoolSize C.uint + + if (nsLookupPool != nil) { + nsLookupPoolPtr = (**C.char)(unsafe.Pointer(&nsLookupPool[0])) + nsLookupPoolSize = C.uint(len(nsLookupPool)) + } + C.wasm_runtime_set_wasi_ns_lookup_pool(self.module, nsLookupPoolPtr, nsLookupPoolSize) +} diff --git a/wasm-micro-runtime/language-bindings/go/wamr/module_test.go b/wasm-micro-runtime/language-bindings/go/wamr/module_test.go new file mode 100644 index 0000000..8abeccb --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/module_test.go @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package wamr + +import ( + //"github.com/stretchr/testify/assert" + "testing" +) + +func TestModule(t *testing.T) { + /* TODO */ +} diff --git a/wasm-micro-runtime/language-bindings/go/wamr/packaged/include/dummy.go b/wasm-micro-runtime/language-bindings/go/wamr/packaged/include/dummy.go new file mode 100644 index 0000000..271e5c1 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/packaged/include/dummy.go @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package include diff --git a/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/darwin-aarch64/dummy.go b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/darwin-aarch64/dummy.go new file mode 100644 index 0000000..35f3c70 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/darwin-aarch64/dummy.go @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package darwin_aarch64 diff --git a/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/darwin-amd64/dummy.go b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/darwin-amd64/dummy.go new file mode 100644 index 0000000..fa8096a --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/darwin-amd64/dummy.go @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package darwin_amd64 diff --git a/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/dummy.go b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/dummy.go new file mode 100644 index 0000000..20dfca8 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/dummy.go @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package lib diff --git a/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/linux-amd64/dummy.go b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/linux-amd64/dummy.go new file mode 100644 index 0000000..8a7c15a --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/packaged/lib/linux-amd64/dummy.go @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package linux_amd64 diff --git a/wasm-micro-runtime/language-bindings/go/wamr/runtime.go b/wasm-micro-runtime/language-bindings/go/wamr/runtime.go new file mode 100644 index 0000000..2c48a92 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/runtime.go @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package wamr + +/* +#include +#include + +#include + +void +bh_log_set_verbose_level(uint32_t level); + +bool +init_wamr_runtime(bool alloc_with_pool, uint8_t *heap_buf, + uint32_t heap_size, uint32_t max_thread_num) +{ + RuntimeInitArgs init_args; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + if (alloc_with_pool) { + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = heap_buf; + init_args.mem_alloc_option.pool.heap_size = heap_size; + } + else { + init_args.mem_alloc_type = Alloc_With_System_Allocator; + } + + return wasm_runtime_full_init(&init_args); +} +*/ +import "C" +import ( + "fmt" + "unsafe" +) + +type LogLevel uint32 +const ( + LOG_LEVEL_FATAL LogLevel = 0 + LOG_LEVEL_ERROR LogLevel = 1 + LOG_LEVEL_WARNING LogLevel = 2 + LOG_LEVEL_DEBUG LogLevel = 3 + LOG_LEVEL_VERBOSE LogLevel = 4 +) + +/* +type NativeSymbol struct { + symbol string + func_ptr *uint8 + signature string +} +*/ + +type _Runtime struct { + initialized bool +} + +var _runtime_singleton *_Runtime + +/* Return the runtime singleton */ +func Runtime() *_Runtime { + if (_runtime_singleton == nil) { + self := &_Runtime{} + _runtime_singleton = self + } + return _runtime_singleton; +} + +/* Initialize the WASM runtime environment */ +func (self *_Runtime) FullInit(alloc_with_pool bool, heap_buf []byte, + max_thread_num uint) error { + var heap_buf_C *C.uchar + + if (self.initialized) { + return nil + } + + if (alloc_with_pool) { + if (heap_buf == nil) { + return fmt.Errorf("Failed to init WAMR runtime") + } + heap_buf_C = (*C.uchar)(unsafe.Pointer(&heap_buf[0])) + } + + if (!C.init_wamr_runtime((C.bool)(alloc_with_pool), heap_buf_C, + (C.uint)(len(heap_buf)), + (C.uint)(max_thread_num))) { + return fmt.Errorf("Failed to init WAMR runtime") + } + + self.initialized = true + return nil +} + +/* Initialize the WASM runtime environment */ +func (self *_Runtime) Init() error { + return self.FullInit(false, nil, 1) +} + +/* Destroy the WASM runtime environment */ +func (self *_Runtime) Destroy() { + if (self.initialized) { + C.wasm_runtime_destroy() + self.initialized = false + } +} + +/* Set log verbose level (0 to 5, default is 2), + larger level with more log */ +func (self *_Runtime) SetLogLevel(level LogLevel) { + C.bh_log_set_verbose_level(C.uint32_t(level)) +} + +/* +func (self *_Runtime) RegisterNatives(moduleName string, + nativeSymbols []NativeSymbol) { +} +*/ /* TODO */ + +func (self *_Runtime) InitThreadEnv() bool { + if (!C.wasm_runtime_init_thread_env()) { + return false + } + return true +} + +func (self *_Runtime) DestroyThreadEnv() { + C.wasm_runtime_destroy_thread_env(); +} + +func (self *_Runtime) ThreadEnvInited() bool { + if (!C.wasm_runtime_thread_env_inited()) { + return false + } + return true +} + +/* Allocate memory from runtime memory environment */ +func (self *_Runtime) Malloc(size uint32) *uint8 { + ptr := C.wasm_runtime_malloc((C.uint32_t)(size)) + return (*uint8)(ptr) +} + +/* Free memory to runtime memory environment */ +func (self *_Runtime) Free(ptr *uint8) { + C.wasm_runtime_free((unsafe.Pointer)(ptr)) +} diff --git a/wasm-micro-runtime/language-bindings/go/wamr/runtime_test.go b/wasm-micro-runtime/language-bindings/go/wamr/runtime_test.go new file mode 100644 index 0000000..66fdf65 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/go/wamr/runtime_test.go @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +package wamr + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRuntime(t *testing.T) { + res := false + if (Runtime() != nil) { + res = true; + } + assert.Equal(t, res, true) + + err := Runtime().Init() + assert.NoError(t, err) + Runtime().Destroy() + + err = Runtime().FullInit(false, nil, 6) + assert.NoError(t, err) + Runtime().Destroy() + + err = Runtime().FullInit(false, nil, 0) + assert.NoError(t, err) + Runtime().Destroy() + + heap_buf := make([]byte, 128 * 1024) + err = Runtime().FullInit(true, heap_buf, 4) + assert.NoError(t, err) + Runtime().Destroy() + + Runtime().FullInit(false, nil, 0) + err = Runtime().FullInit(false, nil, 0) + assert.NoError(t, err) + Runtime().Destroy() + Runtime().Destroy() +} diff --git a/wasm-micro-runtime/language-bindings/python/.gitignore b/wasm-micro-runtime/language-bindings/python/.gitignore new file mode 100644 index 0000000..65efaed --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/.gitignore @@ -0,0 +1,160 @@ +# Refer to https://github.com/github/gitignore/blob/main/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# virtual environment +Pipfile + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# VSCode settings +.vscode/ + diff --git a/wasm-micro-runtime/language-bindings/python/LICENSE b/wasm-micro-runtime/language-bindings/python/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/wasm-micro-runtime/language-bindings/python/MANIFEST.in b/wasm-micro-runtime/language-bindings/python/MANIFEST.in new file mode 100644 index 0000000..9b0c089 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/MANIFEST.in @@ -0,0 +1 @@ +include src/wamr/libs/* diff --git a/wasm-micro-runtime/language-bindings/python/README.md b/wasm-micro-runtime/language-bindings/python/README.md new file mode 100644 index 0000000..45604af --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/README.md @@ -0,0 +1,36 @@ +# wamr-python + +The WAMR Python package contains a set of high-level bindings for WAMR API and WASM-C-API. + +## Installation + +* **Notice**: This python package need python >= `3.10`. + +To Install from local source tree in _development mode_ run the following command, + +```bash +python -m pip install -e . +``` + +In this mode the package appears to be installed but still is editable from the source tree. + +## Usage + +From the same package you can use two set of APIs. + +To use the WAMR API you can import the symbols as follows, + +```py +from wamr.wamrapi.wamr import Engine, Module, Instance, ExecEnv +``` + +In the order hand, to use the WASM-C-API, + +```py +import wamr.wasmcapi.ffi as ffi +``` + +For more information: + +* [WAMR API](./wamr-api) +* [WASM-C-API](./wasm-c-api) diff --git a/wasm-micro-runtime/language-bindings/python/pyproject.toml b/wasm-micro-runtime/language-bindings/python/pyproject.toml new file mode 100644 index 0000000..a480ac6 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "ctypesgen==1.1.1"] +build-backend = "setuptools.build_meta" diff --git a/wasm-micro-runtime/language-bindings/python/setup.py b/wasm-micro-runtime/language-bindings/python/setup.py new file mode 100755 index 0000000..1087c64 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/setup.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +import pathlib +from setuptools import setup, find_packages +from setuptools.command.develop import develop +from setuptools.command.install import install +from setuptools.command.egg_info import egg_info +from subprocess import check_call + + +def build_library(): + cur_path = pathlib.Path(__file__).parent + check_call(f"{cur_path}/utils/create_lib.sh".split()) + + +class PreDevelopCommand(develop): + def run(self): + build_library() + develop.run(self) + + +class PreInstallCommand(install): + def run(self): + build_library() + install.run(self) + + +class PreEggInfoCommand(egg_info): + def run(self): + build_library() + egg_info.run(self) + + +with open("README.md") as f: + readme = f.read() + +with open("LICENSE") as f: + license = f.read() + +setup( + name="wamr-python", + version="0.1.0", + description="A WebAssembly runtime powered by WAMR", + long_description=readme, + packages=find_packages(where="src"), + package_dir={"": "src"}, + author="The WAMR Project Developers", + author_email="hello@bytecodealliance.org", + url="https://github.com/bytecodealliance/wasm-micro-runtime", + license=license, + include_package_data=True, + cmdclass={ + 'develop': PreDevelopCommand, + 'install': PreInstallCommand, + 'egg_info': PreEggInfoCommand, + }, + python_requires='>=3.10' +) diff --git a/wasm-micro-runtime/language-bindings/python/src/wamr/__init__.py b/wasm-micro-runtime/language-bindings/python/src/wamr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wasm-micro-runtime/language-bindings/python/src/wamr/libs/.placeholder b/wasm-micro-runtime/language-bindings/python/src/wamr/libs/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/wasm-micro-runtime/language-bindings/python/src/wamr/wamrapi/__init__.py b/wasm-micro-runtime/language-bindings/python/src/wamr/wamrapi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wasm-micro-runtime/language-bindings/python/src/wamr/wamrapi/wamr.py b/wasm-micro-runtime/language-bindings/python/src/wamr/wamrapi/wamr.py new file mode 100644 index 0000000..9727faa --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/src/wamr/wamrapi/wamr.py @@ -0,0 +1,249 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from ctypes import Array +from ctypes import addressof +from ctypes import c_char +from ctypes import c_uint +from ctypes import c_uint8 +from ctypes import c_uint64 +from ctypes import c_void_p +from ctypes import cast +from ctypes import create_string_buffer +from ctypes import POINTER +from ctypes import pointer +from typing import List +from typing import Tuple +from wamr.wamrapi.iwasm import String +from wamr.wamrapi.iwasm import Alloc_With_Pool +from wamr.wamrapi.iwasm import RuntimeInitArgs +from wamr.wamrapi.iwasm import wasm_exec_env_t +from wamr.wamrapi.iwasm import wasm_function_inst_t +from wamr.wamrapi.iwasm import wasm_module_inst_t +from wamr.wamrapi.iwasm import wasm_module_t +from wamr.wamrapi.iwasm import wasm_runtime_call_wasm +from wamr.wamrapi.iwasm import wasm_runtime_create_exec_env +from wamr.wamrapi.iwasm import wasm_runtime_deinstantiate +from wamr.wamrapi.iwasm import wasm_runtime_destroy +from wamr.wamrapi.iwasm import wasm_runtime_destroy_exec_env +from wamr.wamrapi.iwasm import wasm_runtime_full_init +from wamr.wamrapi.iwasm import wasm_runtime_instantiate +from wamr.wamrapi.iwasm import wasm_runtime_load +from wamr.wamrapi.iwasm import wasm_runtime_lookup_function +from wamr.wamrapi.iwasm import wasm_runtime_unload +from wamr.wamrapi.iwasm import wasm_runtime_module_malloc +from wamr.wamrapi.iwasm import wasm_runtime_module_free +from wamr.wamrapi.iwasm import wasm_runtime_register_natives +from wamr.wamrapi.iwasm import NativeSymbol +from wamr.wamrapi.iwasm import wasm_runtime_start_debug_instance +from wamr.wamrapi.iwasm import wasm_runtime_call_indirect +from wamr.wamrapi.iwasm import wasm_runtime_get_module_inst +from wamr.wamrapi.iwasm import wasm_runtime_addr_app_to_native +from wamr.wamrapi.iwasm import wasm_runtime_addr_native_to_app +from wamr.wamrapi.iwasm import wasm_runtime_set_wasi_args + +ID_TO_EXEC_ENV_MAPPING = {} + + +class Engine: + def __init__(self): + self._native_symbols = dict() + self.init_args = self._get_init_args() + wasm_runtime_full_init(pointer(self.init_args)) + + def __del__(self): + print("deleting Engine") + wasm_runtime_destroy() + + def _get_init_args( + self, + heap_size: int = 1024 * 1024 * 2, + ip_addr: str = "127.0.0.1", + instance_port: int = 1234, + ) -> RuntimeInitArgs: + init_args = RuntimeInitArgs() + init_args.mem_alloc_type = Alloc_With_Pool + init_args.mem_alloc_option.pool.heap_buf = cast( + (c_char * heap_size)(), c_void_p + ) + init_args.mem_alloc_option.pool.heap_size = heap_size + # Debug port setting + init_args.ip_addr = bytes(ip_addr, "utf-8") + init_args.instance_port = instance_port + return init_args + + def register_natives( + self, module_name: str, native_symbols: List[NativeSymbol] + ) -> None: + module_name = String.from_param(module_name) + # WAMR does not copy the symbols. We must store them. + for native in native_symbols: + self._native_symbols[str(native.symbol)] = (module_name, native) + + if not wasm_runtime_register_natives( + module_name, + cast( + (NativeSymbol * len(native_symbols))(*native_symbols), + POINTER(NativeSymbol), + ), + len(native_symbols), + ): + raise Exception("Error while registering symbols") + + +class Module: + __create_key = object() + + @classmethod + def from_file(cls, engine: Engine, fp: str) -> "Module": + return Module(cls.__create_key, engine, fp) + + def __init__(self, create_key: object, engine: Engine, fp: str) -> None: + assert ( + create_key == Module.__create_key + ), "Module objects must be created using Module.from_file" + self.engine = engine + self.module, self.file_data = self._create_module(fp) + + def __del__(self): + print("deleting Module") + wasm_runtime_unload(self.module) + + def _create_module(self, fp: str) -> Tuple[wasm_module_t, "Array[c_uint]"]: + with open(fp, "rb") as f: + data = f.read() + data = (c_uint8 * len(data))(*data) + + error_buf = create_string_buffer(128) + module = wasm_runtime_load(data, len(data), error_buf, len(error_buf)) + if not module: + raise Exception("Error while creating module") + return module, data + + +class Instance: + def __init__( + self, + module: Module, + stack_size: int = 65536, + heap_size: int = 16384, + dir_list: List[str] | None = None, + preinitialized_module_inst: wasm_module_inst_t | None = None, + ): + # Store module ensures GC does not remove it + self.module = module + if dir_list: + self._set_wasi_args(module, dir_list) + if preinitialized_module_inst is None: + self.module_inst = self._create_module_inst(module, stack_size, heap_size) + else: + self.module_inst = preinitialized_module_inst + + def __del__(self): + print("deleting Instance") + wasm_runtime_deinstantiate(self.module_inst) + + def _set_wasi_args(self, module: Module, dir_list: List[str]) -> None: + LP_c_char = POINTER(c_char) + LP_LP_c_char = POINTER(LP_c_char) + + p = (LP_c_char * len(dir_list))() + for i, dir in enumerate(dir_list): + enc_dir = dir.encode("utf-8") + p[i] = create_string_buffer(enc_dir) + + na = cast(p, LP_LP_c_char) + wasm_runtime_set_wasi_args( + module.module, na, len(dir_list), None, 0, None, 0, None, 0 + ) + + def _create_module_inst( + self, module: Module, stack_size: int, heap_size: int + ) -> wasm_module_inst_t: + error_buf = create_string_buffer(128) + module_inst = wasm_runtime_instantiate( + module.module, stack_size, heap_size, error_buf, len(error_buf) + ) + if not module_inst: + raise Exception("Error while creating module instance") + return module_inst + + def malloc(self, nbytes: int, native_handler) -> c_uint64: + return wasm_runtime_module_malloc(self.module_inst, nbytes, native_handler) + + def free(self, wasm_handler) -> None: + wasm_runtime_module_free(self.module_inst, wasm_handler) + + def lookup_function(self, name: str) -> wasm_function_inst_t: + func = wasm_runtime_lookup_function(self.module_inst, name) + if not func: + raise Exception("Error while looking-up function") + return func + + def native_addr_to_app_addr(self, native_addr) -> c_void_p: + return wasm_runtime_addr_native_to_app(self.module_inst, native_addr) + + def app_addr_to_native_addr(self, app_addr) -> c_void_p: + return wasm_runtime_addr_app_to_native(self.module_inst, app_addr) + + +class ExecEnv: + def __init__(self, module_inst: Instance, stack_size: int = 65536): + self.module_inst = module_inst + self.exec_env = self._create_exec_env(module_inst, stack_size) + self.env = addressof(self.exec_env.contents) + self.own_c = True + + ID_TO_EXEC_ENV_MAPPING[str(self.env)] = self + + def __del__(self): + if self.own_c: + print("deleting ExecEnv") + wasm_runtime_destroy_exec_env(self.exec_env) + del ID_TO_EXEC_ENV_MAPPING[str(self.env)] + + def _create_exec_env( + self, module_inst: Instance, stack_size: int + ) -> wasm_exec_env_t: + exec_env = wasm_runtime_create_exec_env(module_inst.module_inst, stack_size) + if not exec_env: + raise Exception("Error while creating execution environment") + return exec_env + + def call(self, func: wasm_function_inst_t, argc: int, argv: "POINTER[c_uint]"): + if not wasm_runtime_call_wasm(self.exec_env, func, argc, argv): + raise Exception("Error while calling function") + + def get_module_inst(self) -> Instance: + return self.module_inst + + def start_debugging(self) -> int: + return wasm_runtime_start_debug_instance(self.exec_env) + + def call_indirect(self, element_index: int, argc: int, argv: "POINTER[c_uint]"): + if not wasm_runtime_call_indirect(self.exec_env, element_index, argc, argv): + raise Exception("Error while calling function") + + @staticmethod + def wrap(env: int) -> "ExecEnv": + if str(env) in ID_TO_EXEC_ENV_MAPPING: + return ID_TO_EXEC_ENV_MAPPING[str(env)] + return InternalExecEnv(env) + + +class InternalExecEnv(ExecEnv): + """ + Generate Python ExecEnv-like object from a `wasm_exec_env_t` index. + """ + + def __init__(self, env: int): + self.env = env + self.exec_env = cast(env, wasm_exec_env_t) + self.module_inst = Instance( + module=object(), + preinitialized_module_inst=wasm_runtime_get_module_inst(self.exec_env), + ) + ID_TO_EXEC_ENV_MAPPING[str(env)] = self + + def __del__(self): + del ID_TO_EXEC_ENV_MAPPING[str(self.env)] diff --git a/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/__init__.py b/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/__init__.py new file mode 100644 index 0000000..8d7404a --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +__all__ = ["ffi"] diff --git a/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/binding.py b/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/binding.py new file mode 100644 index 0000000..1f4e0cf --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/binding.py @@ -0,0 +1,2029 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#It is a generated file. DO NOT EDIT. +# +from ctypes import * + +from .ffi import dereference, libiwasm, wasm_ref_t, wasm_val_t + + +wasm_byte_t = c_ubyte + +class wasm_byte_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(wasm_byte_t)), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_byte_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(self.data[i]) + ret += " " + return ret + + + +def wasm_byte_vec_new_empty(arg0): + _wasm_byte_vec_new_empty = libiwasm.wasm_byte_vec_new_empty + _wasm_byte_vec_new_empty.restype = None + _wasm_byte_vec_new_empty.argtypes = [POINTER(wasm_byte_vec_t)] + return _wasm_byte_vec_new_empty(arg0) + +def wasm_byte_vec_new_uninitialized(arg0,arg1): + _wasm_byte_vec_new_uninitialized = libiwasm.wasm_byte_vec_new_uninitialized + _wasm_byte_vec_new_uninitialized.restype = None + _wasm_byte_vec_new_uninitialized.argtypes = [POINTER(wasm_byte_vec_t),c_size_t] + return _wasm_byte_vec_new_uninitialized(arg0,arg1) + +def wasm_byte_vec_new(arg0,arg1,arg2): + _wasm_byte_vec_new = libiwasm.wasm_byte_vec_new + _wasm_byte_vec_new.restype = None + _wasm_byte_vec_new.argtypes = [POINTER(wasm_byte_vec_t),c_size_t,POINTER(wasm_byte_t)] + return _wasm_byte_vec_new(arg0,arg1,arg2) + +def wasm_byte_vec_copy(arg0,arg1): + _wasm_byte_vec_copy = libiwasm.wasm_byte_vec_copy + _wasm_byte_vec_copy.restype = None + _wasm_byte_vec_copy.argtypes = [POINTER(wasm_byte_vec_t),POINTER(wasm_byte_vec_t)] + return _wasm_byte_vec_copy(arg0,arg1) + +def wasm_byte_vec_delete(arg0): + _wasm_byte_vec_delete = libiwasm.wasm_byte_vec_delete + _wasm_byte_vec_delete.restype = None + _wasm_byte_vec_delete.argtypes = [POINTER(wasm_byte_vec_t)] + return _wasm_byte_vec_delete(arg0) + +wasm_name_t = wasm_byte_vec_t + +class wasm_config_t(Structure): + pass + +def wasm_config_delete(arg0): + _wasm_config_delete = libiwasm.wasm_config_delete + _wasm_config_delete.restype = None + _wasm_config_delete.argtypes = [POINTER(wasm_config_t)] + return _wasm_config_delete(arg0) + +def wasm_config_new(): + _wasm_config_new = libiwasm.wasm_config_new + _wasm_config_new.restype = POINTER(wasm_config_t) + _wasm_config_new.argtypes = None + return _wasm_config_new() + +class wasm_engine_t(Structure): + pass + +def wasm_engine_delete(arg0): + _wasm_engine_delete = libiwasm.wasm_engine_delete + _wasm_engine_delete.restype = None + _wasm_engine_delete.argtypes = [POINTER(wasm_engine_t)] + return _wasm_engine_delete(arg0) + +def wasm_engine_new(): + _wasm_engine_new = libiwasm.wasm_engine_new + _wasm_engine_new.restype = POINTER(wasm_engine_t) + _wasm_engine_new.argtypes = None + return _wasm_engine_new() + +def wasm_engine_new_with_config(arg0): + _wasm_engine_new_with_config = libiwasm.wasm_engine_new_with_config + _wasm_engine_new_with_config.restype = POINTER(wasm_engine_t) + _wasm_engine_new_with_config.argtypes = [POINTER(wasm_config_t)] + return _wasm_engine_new_with_config(arg0) + +class wasm_store_t(Structure): + pass + +def wasm_store_delete(arg0): + _wasm_store_delete = libiwasm.wasm_store_delete + _wasm_store_delete.restype = None + _wasm_store_delete.argtypes = [POINTER(wasm_store_t)] + return _wasm_store_delete(arg0) + +def wasm_store_new(arg0): + _wasm_store_new = libiwasm.wasm_store_new + _wasm_store_new.restype = POINTER(wasm_store_t) + _wasm_store_new.argtypes = [POINTER(wasm_engine_t)] + return _wasm_store_new(arg0) + +wasm_mutability_t = c_uint8 + +WASM_CONST = 0 +WASM_VAR = 1 + +class wasm_limits_t(Structure): + _fields_ = [ + ("min", c_uint32), + ("max", c_uint32), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_limits_t): + return False + return self.min == other.min and self.max == other.max + + def __repr__(self): + return f"{{min={self.min}, max={self.max}}}" + + +class wasm_valtype_t(Structure): + pass + +def wasm_valtype_delete(arg0): + _wasm_valtype_delete = libiwasm.wasm_valtype_delete + _wasm_valtype_delete.restype = None + _wasm_valtype_delete.argtypes = [POINTER(wasm_valtype_t)] + return _wasm_valtype_delete(arg0) + +class wasm_valtype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_valtype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_valtype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_valtype_vec_new_empty(arg0): + _wasm_valtype_vec_new_empty = libiwasm.wasm_valtype_vec_new_empty + _wasm_valtype_vec_new_empty.restype = None + _wasm_valtype_vec_new_empty.argtypes = [POINTER(wasm_valtype_vec_t)] + return _wasm_valtype_vec_new_empty(arg0) + +def wasm_valtype_vec_new_uninitialized(arg0,arg1): + _wasm_valtype_vec_new_uninitialized = libiwasm.wasm_valtype_vec_new_uninitialized + _wasm_valtype_vec_new_uninitialized.restype = None + _wasm_valtype_vec_new_uninitialized.argtypes = [POINTER(wasm_valtype_vec_t),c_size_t] + return _wasm_valtype_vec_new_uninitialized(arg0,arg1) + +def wasm_valtype_vec_new(arg0,arg1,arg2): + _wasm_valtype_vec_new = libiwasm.wasm_valtype_vec_new + _wasm_valtype_vec_new.restype = None + _wasm_valtype_vec_new.argtypes = [POINTER(wasm_valtype_vec_t),c_size_t,POINTER(POINTER(wasm_valtype_t))] + return _wasm_valtype_vec_new(arg0,arg1,arg2) + +def wasm_valtype_vec_copy(arg0,arg1): + _wasm_valtype_vec_copy = libiwasm.wasm_valtype_vec_copy + _wasm_valtype_vec_copy.restype = None + _wasm_valtype_vec_copy.argtypes = [POINTER(wasm_valtype_vec_t),POINTER(wasm_valtype_vec_t)] + return _wasm_valtype_vec_copy(arg0,arg1) + +def wasm_valtype_vec_delete(arg0): + _wasm_valtype_vec_delete = libiwasm.wasm_valtype_vec_delete + _wasm_valtype_vec_delete.restype = None + _wasm_valtype_vec_delete.argtypes = [POINTER(wasm_valtype_vec_t)] + return _wasm_valtype_vec_delete(arg0) + +def wasm_valtype_copy(arg0): + _wasm_valtype_copy = libiwasm.wasm_valtype_copy + _wasm_valtype_copy.restype = POINTER(wasm_valtype_t) + _wasm_valtype_copy.argtypes = [POINTER(wasm_valtype_t)] + return _wasm_valtype_copy(arg0) + +wasm_valkind_t = c_uint8 + +WASM_I32 = 0 +WASM_I64 = 1 +WASM_F32 = 2 +WASM_F64 = 3 +WASM_ANYREF = 128 +WASM_FUNCREF = 129 + +def wasm_valtype_new(arg0): + _wasm_valtype_new = libiwasm.wasm_valtype_new + _wasm_valtype_new.restype = POINTER(wasm_valtype_t) + _wasm_valtype_new.argtypes = [wasm_valkind_t] + return _wasm_valtype_new(arg0) + +def wasm_valtype_kind(arg0): + _wasm_valtype_kind = libiwasm.wasm_valtype_kind + _wasm_valtype_kind.restype = wasm_valkind_t + _wasm_valtype_kind.argtypes = [POINTER(wasm_valtype_t)] + return _wasm_valtype_kind(arg0) + +class wasm_functype_t(Structure): + pass + +def wasm_functype_delete(arg0): + _wasm_functype_delete = libiwasm.wasm_functype_delete + _wasm_functype_delete.restype = None + _wasm_functype_delete.argtypes = [POINTER(wasm_functype_t)] + return _wasm_functype_delete(arg0) + +class wasm_functype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_functype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_functype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_functype_vec_new_empty(arg0): + _wasm_functype_vec_new_empty = libiwasm.wasm_functype_vec_new_empty + _wasm_functype_vec_new_empty.restype = None + _wasm_functype_vec_new_empty.argtypes = [POINTER(wasm_functype_vec_t)] + return _wasm_functype_vec_new_empty(arg0) + +def wasm_functype_vec_new_uninitialized(arg0,arg1): + _wasm_functype_vec_new_uninitialized = libiwasm.wasm_functype_vec_new_uninitialized + _wasm_functype_vec_new_uninitialized.restype = None + _wasm_functype_vec_new_uninitialized.argtypes = [POINTER(wasm_functype_vec_t),c_size_t] + return _wasm_functype_vec_new_uninitialized(arg0,arg1) + +def wasm_functype_vec_new(arg0,arg1,arg2): + _wasm_functype_vec_new = libiwasm.wasm_functype_vec_new + _wasm_functype_vec_new.restype = None + _wasm_functype_vec_new.argtypes = [POINTER(wasm_functype_vec_t),c_size_t,POINTER(POINTER(wasm_functype_t))] + return _wasm_functype_vec_new(arg0,arg1,arg2) + +def wasm_functype_vec_copy(arg0,arg1): + _wasm_functype_vec_copy = libiwasm.wasm_functype_vec_copy + _wasm_functype_vec_copy.restype = None + _wasm_functype_vec_copy.argtypes = [POINTER(wasm_functype_vec_t),POINTER(wasm_functype_vec_t)] + return _wasm_functype_vec_copy(arg0,arg1) + +def wasm_functype_vec_delete(arg0): + _wasm_functype_vec_delete = libiwasm.wasm_functype_vec_delete + _wasm_functype_vec_delete.restype = None + _wasm_functype_vec_delete.argtypes = [POINTER(wasm_functype_vec_t)] + return _wasm_functype_vec_delete(arg0) + +def wasm_functype_copy(arg0): + _wasm_functype_copy = libiwasm.wasm_functype_copy + _wasm_functype_copy.restype = POINTER(wasm_functype_t) + _wasm_functype_copy.argtypes = [POINTER(wasm_functype_t)] + return _wasm_functype_copy(arg0) + +def wasm_functype_new(arg0,arg1): + _wasm_functype_new = libiwasm.wasm_functype_new + _wasm_functype_new.restype = POINTER(wasm_functype_t) + _wasm_functype_new.argtypes = [POINTER(wasm_valtype_vec_t),POINTER(wasm_valtype_vec_t)] + return _wasm_functype_new(arg0,arg1) + +def wasm_functype_params(arg0): + _wasm_functype_params = libiwasm.wasm_functype_params + _wasm_functype_params.restype = POINTER(wasm_valtype_vec_t) + _wasm_functype_params.argtypes = [POINTER(wasm_functype_t)] + return _wasm_functype_params(arg0) + +def wasm_functype_results(arg0): + _wasm_functype_results = libiwasm.wasm_functype_results + _wasm_functype_results.restype = POINTER(wasm_valtype_vec_t) + _wasm_functype_results.argtypes = [POINTER(wasm_functype_t)] + return _wasm_functype_results(arg0) + +class wasm_globaltype_t(Structure): + pass + +def wasm_globaltype_delete(arg0): + _wasm_globaltype_delete = libiwasm.wasm_globaltype_delete + _wasm_globaltype_delete.restype = None + _wasm_globaltype_delete.argtypes = [POINTER(wasm_globaltype_t)] + return _wasm_globaltype_delete(arg0) + +class wasm_globaltype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_globaltype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_globaltype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_globaltype_vec_new_empty(arg0): + _wasm_globaltype_vec_new_empty = libiwasm.wasm_globaltype_vec_new_empty + _wasm_globaltype_vec_new_empty.restype = None + _wasm_globaltype_vec_new_empty.argtypes = [POINTER(wasm_globaltype_vec_t)] + return _wasm_globaltype_vec_new_empty(arg0) + +def wasm_globaltype_vec_new_uninitialized(arg0,arg1): + _wasm_globaltype_vec_new_uninitialized = libiwasm.wasm_globaltype_vec_new_uninitialized + _wasm_globaltype_vec_new_uninitialized.restype = None + _wasm_globaltype_vec_new_uninitialized.argtypes = [POINTER(wasm_globaltype_vec_t),c_size_t] + return _wasm_globaltype_vec_new_uninitialized(arg0,arg1) + +def wasm_globaltype_vec_new(arg0,arg1,arg2): + _wasm_globaltype_vec_new = libiwasm.wasm_globaltype_vec_new + _wasm_globaltype_vec_new.restype = None + _wasm_globaltype_vec_new.argtypes = [POINTER(wasm_globaltype_vec_t),c_size_t,POINTER(POINTER(wasm_globaltype_t))] + return _wasm_globaltype_vec_new(arg0,arg1,arg2) + +def wasm_globaltype_vec_copy(arg0,arg1): + _wasm_globaltype_vec_copy = libiwasm.wasm_globaltype_vec_copy + _wasm_globaltype_vec_copy.restype = None + _wasm_globaltype_vec_copy.argtypes = [POINTER(wasm_globaltype_vec_t),POINTER(wasm_globaltype_vec_t)] + return _wasm_globaltype_vec_copy(arg0,arg1) + +def wasm_globaltype_vec_delete(arg0): + _wasm_globaltype_vec_delete = libiwasm.wasm_globaltype_vec_delete + _wasm_globaltype_vec_delete.restype = None + _wasm_globaltype_vec_delete.argtypes = [POINTER(wasm_globaltype_vec_t)] + return _wasm_globaltype_vec_delete(arg0) + +def wasm_globaltype_copy(arg0): + _wasm_globaltype_copy = libiwasm.wasm_globaltype_copy + _wasm_globaltype_copy.restype = POINTER(wasm_globaltype_t) + _wasm_globaltype_copy.argtypes = [POINTER(wasm_globaltype_t)] + return _wasm_globaltype_copy(arg0) + +def wasm_globaltype_new(arg0,arg1): + _wasm_globaltype_new = libiwasm.wasm_globaltype_new + _wasm_globaltype_new.restype = POINTER(wasm_globaltype_t) + _wasm_globaltype_new.argtypes = [POINTER(wasm_valtype_t),wasm_mutability_t] + return _wasm_globaltype_new(arg0,arg1) + +def wasm_globaltype_content(arg0): + _wasm_globaltype_content = libiwasm.wasm_globaltype_content + _wasm_globaltype_content.restype = POINTER(wasm_valtype_t) + _wasm_globaltype_content.argtypes = [POINTER(wasm_globaltype_t)] + return _wasm_globaltype_content(arg0) + +def wasm_globaltype_mutability(arg0): + _wasm_globaltype_mutability = libiwasm.wasm_globaltype_mutability + _wasm_globaltype_mutability.restype = wasm_mutability_t + _wasm_globaltype_mutability.argtypes = [POINTER(wasm_globaltype_t)] + return _wasm_globaltype_mutability(arg0) + +class wasm_tabletype_t(Structure): + pass + +def wasm_tabletype_delete(arg0): + _wasm_tabletype_delete = libiwasm.wasm_tabletype_delete + _wasm_tabletype_delete.restype = None + _wasm_tabletype_delete.argtypes = [POINTER(wasm_tabletype_t)] + return _wasm_tabletype_delete(arg0) + +class wasm_tabletype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_tabletype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_tabletype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_tabletype_vec_new_empty(arg0): + _wasm_tabletype_vec_new_empty = libiwasm.wasm_tabletype_vec_new_empty + _wasm_tabletype_vec_new_empty.restype = None + _wasm_tabletype_vec_new_empty.argtypes = [POINTER(wasm_tabletype_vec_t)] + return _wasm_tabletype_vec_new_empty(arg0) + +def wasm_tabletype_vec_new_uninitialized(arg0,arg1): + _wasm_tabletype_vec_new_uninitialized = libiwasm.wasm_tabletype_vec_new_uninitialized + _wasm_tabletype_vec_new_uninitialized.restype = None + _wasm_tabletype_vec_new_uninitialized.argtypes = [POINTER(wasm_tabletype_vec_t),c_size_t] + return _wasm_tabletype_vec_new_uninitialized(arg0,arg1) + +def wasm_tabletype_vec_new(arg0,arg1,arg2): + _wasm_tabletype_vec_new = libiwasm.wasm_tabletype_vec_new + _wasm_tabletype_vec_new.restype = None + _wasm_tabletype_vec_new.argtypes = [POINTER(wasm_tabletype_vec_t),c_size_t,POINTER(POINTER(wasm_tabletype_t))] + return _wasm_tabletype_vec_new(arg0,arg1,arg2) + +def wasm_tabletype_vec_copy(arg0,arg1): + _wasm_tabletype_vec_copy = libiwasm.wasm_tabletype_vec_copy + _wasm_tabletype_vec_copy.restype = None + _wasm_tabletype_vec_copy.argtypes = [POINTER(wasm_tabletype_vec_t),POINTER(wasm_tabletype_vec_t)] + return _wasm_tabletype_vec_copy(arg0,arg1) + +def wasm_tabletype_vec_delete(arg0): + _wasm_tabletype_vec_delete = libiwasm.wasm_tabletype_vec_delete + _wasm_tabletype_vec_delete.restype = None + _wasm_tabletype_vec_delete.argtypes = [POINTER(wasm_tabletype_vec_t)] + return _wasm_tabletype_vec_delete(arg0) + +def wasm_tabletype_copy(arg0): + _wasm_tabletype_copy = libiwasm.wasm_tabletype_copy + _wasm_tabletype_copy.restype = POINTER(wasm_tabletype_t) + _wasm_tabletype_copy.argtypes = [POINTER(wasm_tabletype_t)] + return _wasm_tabletype_copy(arg0) + +def wasm_tabletype_new(arg0,arg1): + _wasm_tabletype_new = libiwasm.wasm_tabletype_new + _wasm_tabletype_new.restype = POINTER(wasm_tabletype_t) + _wasm_tabletype_new.argtypes = [POINTER(wasm_valtype_t),POINTER(wasm_limits_t)] + return _wasm_tabletype_new(arg0,arg1) + +def wasm_tabletype_element(arg0): + _wasm_tabletype_element = libiwasm.wasm_tabletype_element + _wasm_tabletype_element.restype = POINTER(wasm_valtype_t) + _wasm_tabletype_element.argtypes = [POINTER(wasm_tabletype_t)] + return _wasm_tabletype_element(arg0) + +def wasm_tabletype_limits(arg0): + _wasm_tabletype_limits = libiwasm.wasm_tabletype_limits + _wasm_tabletype_limits.restype = POINTER(wasm_limits_t) + _wasm_tabletype_limits.argtypes = [POINTER(wasm_tabletype_t)] + return _wasm_tabletype_limits(arg0) + +class wasm_memorytype_t(Structure): + pass + +def wasm_memorytype_delete(arg0): + _wasm_memorytype_delete = libiwasm.wasm_memorytype_delete + _wasm_memorytype_delete.restype = None + _wasm_memorytype_delete.argtypes = [POINTER(wasm_memorytype_t)] + return _wasm_memorytype_delete(arg0) + +class wasm_memorytype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_memorytype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_memorytype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_memorytype_vec_new_empty(arg0): + _wasm_memorytype_vec_new_empty = libiwasm.wasm_memorytype_vec_new_empty + _wasm_memorytype_vec_new_empty.restype = None + _wasm_memorytype_vec_new_empty.argtypes = [POINTER(wasm_memorytype_vec_t)] + return _wasm_memorytype_vec_new_empty(arg0) + +def wasm_memorytype_vec_new_uninitialized(arg0,arg1): + _wasm_memorytype_vec_new_uninitialized = libiwasm.wasm_memorytype_vec_new_uninitialized + _wasm_memorytype_vec_new_uninitialized.restype = None + _wasm_memorytype_vec_new_uninitialized.argtypes = [POINTER(wasm_memorytype_vec_t),c_size_t] + return _wasm_memorytype_vec_new_uninitialized(arg0,arg1) + +def wasm_memorytype_vec_new(arg0,arg1,arg2): + _wasm_memorytype_vec_new = libiwasm.wasm_memorytype_vec_new + _wasm_memorytype_vec_new.restype = None + _wasm_memorytype_vec_new.argtypes = [POINTER(wasm_memorytype_vec_t),c_size_t,POINTER(POINTER(wasm_memorytype_t))] + return _wasm_memorytype_vec_new(arg0,arg1,arg2) + +def wasm_memorytype_vec_copy(arg0,arg1): + _wasm_memorytype_vec_copy = libiwasm.wasm_memorytype_vec_copy + _wasm_memorytype_vec_copy.restype = None + _wasm_memorytype_vec_copy.argtypes = [POINTER(wasm_memorytype_vec_t),POINTER(wasm_memorytype_vec_t)] + return _wasm_memorytype_vec_copy(arg0,arg1) + +def wasm_memorytype_vec_delete(arg0): + _wasm_memorytype_vec_delete = libiwasm.wasm_memorytype_vec_delete + _wasm_memorytype_vec_delete.restype = None + _wasm_memorytype_vec_delete.argtypes = [POINTER(wasm_memorytype_vec_t)] + return _wasm_memorytype_vec_delete(arg0) + +def wasm_memorytype_copy(arg0): + _wasm_memorytype_copy = libiwasm.wasm_memorytype_copy + _wasm_memorytype_copy.restype = POINTER(wasm_memorytype_t) + _wasm_memorytype_copy.argtypes = [POINTER(wasm_memorytype_t)] + return _wasm_memorytype_copy(arg0) + +def wasm_memorytype_new(arg0): + _wasm_memorytype_new = libiwasm.wasm_memorytype_new + _wasm_memorytype_new.restype = POINTER(wasm_memorytype_t) + _wasm_memorytype_new.argtypes = [POINTER(wasm_limits_t)] + return _wasm_memorytype_new(arg0) + +def wasm_memorytype_limits(arg0): + _wasm_memorytype_limits = libiwasm.wasm_memorytype_limits + _wasm_memorytype_limits.restype = POINTER(wasm_limits_t) + _wasm_memorytype_limits.argtypes = [POINTER(wasm_memorytype_t)] + return _wasm_memorytype_limits(arg0) + +class wasm_externtype_t(Structure): + pass + +def wasm_externtype_delete(arg0): + _wasm_externtype_delete = libiwasm.wasm_externtype_delete + _wasm_externtype_delete.restype = None + _wasm_externtype_delete.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_delete(arg0) + +class wasm_externtype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_externtype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_externtype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_externtype_vec_new_empty(arg0): + _wasm_externtype_vec_new_empty = libiwasm.wasm_externtype_vec_new_empty + _wasm_externtype_vec_new_empty.restype = None + _wasm_externtype_vec_new_empty.argtypes = [POINTER(wasm_externtype_vec_t)] + return _wasm_externtype_vec_new_empty(arg0) + +def wasm_externtype_vec_new_uninitialized(arg0,arg1): + _wasm_externtype_vec_new_uninitialized = libiwasm.wasm_externtype_vec_new_uninitialized + _wasm_externtype_vec_new_uninitialized.restype = None + _wasm_externtype_vec_new_uninitialized.argtypes = [POINTER(wasm_externtype_vec_t),c_size_t] + return _wasm_externtype_vec_new_uninitialized(arg0,arg1) + +def wasm_externtype_vec_new(arg0,arg1,arg2): + _wasm_externtype_vec_new = libiwasm.wasm_externtype_vec_new + _wasm_externtype_vec_new.restype = None + _wasm_externtype_vec_new.argtypes = [POINTER(wasm_externtype_vec_t),c_size_t,POINTER(POINTER(wasm_externtype_t))] + return _wasm_externtype_vec_new(arg0,arg1,arg2) + +def wasm_externtype_vec_copy(arg0,arg1): + _wasm_externtype_vec_copy = libiwasm.wasm_externtype_vec_copy + _wasm_externtype_vec_copy.restype = None + _wasm_externtype_vec_copy.argtypes = [POINTER(wasm_externtype_vec_t),POINTER(wasm_externtype_vec_t)] + return _wasm_externtype_vec_copy(arg0,arg1) + +def wasm_externtype_vec_delete(arg0): + _wasm_externtype_vec_delete = libiwasm.wasm_externtype_vec_delete + _wasm_externtype_vec_delete.restype = None + _wasm_externtype_vec_delete.argtypes = [POINTER(wasm_externtype_vec_t)] + return _wasm_externtype_vec_delete(arg0) + +def wasm_externtype_copy(arg0): + _wasm_externtype_copy = libiwasm.wasm_externtype_copy + _wasm_externtype_copy.restype = POINTER(wasm_externtype_t) + _wasm_externtype_copy.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_copy(arg0) + +wasm_externkind_t = c_uint8 + +WASM_EXTERN_FUNC = 0 +WASM_EXTERN_GLOBAL = 1 +WASM_EXTERN_TABLE = 2 +WASM_EXTERN_MEMORY = 3 + +def wasm_externtype_kind(arg0): + _wasm_externtype_kind = libiwasm.wasm_externtype_kind + _wasm_externtype_kind.restype = wasm_externkind_t + _wasm_externtype_kind.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_kind(arg0) + +def wasm_functype_as_externtype(arg0): + _wasm_functype_as_externtype = libiwasm.wasm_functype_as_externtype + _wasm_functype_as_externtype.restype = POINTER(wasm_externtype_t) + _wasm_functype_as_externtype.argtypes = [POINTER(wasm_functype_t)] + return _wasm_functype_as_externtype(arg0) + +def wasm_globaltype_as_externtype(arg0): + _wasm_globaltype_as_externtype = libiwasm.wasm_globaltype_as_externtype + _wasm_globaltype_as_externtype.restype = POINTER(wasm_externtype_t) + _wasm_globaltype_as_externtype.argtypes = [POINTER(wasm_globaltype_t)] + return _wasm_globaltype_as_externtype(arg0) + +def wasm_tabletype_as_externtype(arg0): + _wasm_tabletype_as_externtype = libiwasm.wasm_tabletype_as_externtype + _wasm_tabletype_as_externtype.restype = POINTER(wasm_externtype_t) + _wasm_tabletype_as_externtype.argtypes = [POINTER(wasm_tabletype_t)] + return _wasm_tabletype_as_externtype(arg0) + +def wasm_memorytype_as_externtype(arg0): + _wasm_memorytype_as_externtype = libiwasm.wasm_memorytype_as_externtype + _wasm_memorytype_as_externtype.restype = POINTER(wasm_externtype_t) + _wasm_memorytype_as_externtype.argtypes = [POINTER(wasm_memorytype_t)] + return _wasm_memorytype_as_externtype(arg0) + +def wasm_externtype_as_functype(arg0): + _wasm_externtype_as_functype = libiwasm.wasm_externtype_as_functype + _wasm_externtype_as_functype.restype = POINTER(wasm_functype_t) + _wasm_externtype_as_functype.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_functype(arg0) + +def wasm_externtype_as_globaltype(arg0): + _wasm_externtype_as_globaltype = libiwasm.wasm_externtype_as_globaltype + _wasm_externtype_as_globaltype.restype = POINTER(wasm_globaltype_t) + _wasm_externtype_as_globaltype.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_globaltype(arg0) + +def wasm_externtype_as_tabletype(arg0): + _wasm_externtype_as_tabletype = libiwasm.wasm_externtype_as_tabletype + _wasm_externtype_as_tabletype.restype = POINTER(wasm_tabletype_t) + _wasm_externtype_as_tabletype.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_tabletype(arg0) + +def wasm_externtype_as_memorytype(arg0): + _wasm_externtype_as_memorytype = libiwasm.wasm_externtype_as_memorytype + _wasm_externtype_as_memorytype.restype = POINTER(wasm_memorytype_t) + _wasm_externtype_as_memorytype.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_memorytype(arg0) + +def wasm_functype_as_externtype_const(arg0): + _wasm_functype_as_externtype_const = libiwasm.wasm_functype_as_externtype_const + _wasm_functype_as_externtype_const.restype = POINTER(wasm_externtype_t) + _wasm_functype_as_externtype_const.argtypes = [POINTER(wasm_functype_t)] + return _wasm_functype_as_externtype_const(arg0) + +def wasm_globaltype_as_externtype_const(arg0): + _wasm_globaltype_as_externtype_const = libiwasm.wasm_globaltype_as_externtype_const + _wasm_globaltype_as_externtype_const.restype = POINTER(wasm_externtype_t) + _wasm_globaltype_as_externtype_const.argtypes = [POINTER(wasm_globaltype_t)] + return _wasm_globaltype_as_externtype_const(arg0) + +def wasm_tabletype_as_externtype_const(arg0): + _wasm_tabletype_as_externtype_const = libiwasm.wasm_tabletype_as_externtype_const + _wasm_tabletype_as_externtype_const.restype = POINTER(wasm_externtype_t) + _wasm_tabletype_as_externtype_const.argtypes = [POINTER(wasm_tabletype_t)] + return _wasm_tabletype_as_externtype_const(arg0) + +def wasm_memorytype_as_externtype_const(arg0): + _wasm_memorytype_as_externtype_const = libiwasm.wasm_memorytype_as_externtype_const + _wasm_memorytype_as_externtype_const.restype = POINTER(wasm_externtype_t) + _wasm_memorytype_as_externtype_const.argtypes = [POINTER(wasm_memorytype_t)] + return _wasm_memorytype_as_externtype_const(arg0) + +def wasm_externtype_as_functype_const(arg0): + _wasm_externtype_as_functype_const = libiwasm.wasm_externtype_as_functype_const + _wasm_externtype_as_functype_const.restype = POINTER(wasm_functype_t) + _wasm_externtype_as_functype_const.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_functype_const(arg0) + +def wasm_externtype_as_globaltype_const(arg0): + _wasm_externtype_as_globaltype_const = libiwasm.wasm_externtype_as_globaltype_const + _wasm_externtype_as_globaltype_const.restype = POINTER(wasm_globaltype_t) + _wasm_externtype_as_globaltype_const.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_globaltype_const(arg0) + +def wasm_externtype_as_tabletype_const(arg0): + _wasm_externtype_as_tabletype_const = libiwasm.wasm_externtype_as_tabletype_const + _wasm_externtype_as_tabletype_const.restype = POINTER(wasm_tabletype_t) + _wasm_externtype_as_tabletype_const.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_tabletype_const(arg0) + +def wasm_externtype_as_memorytype_const(arg0): + _wasm_externtype_as_memorytype_const = libiwasm.wasm_externtype_as_memorytype_const + _wasm_externtype_as_memorytype_const.restype = POINTER(wasm_memorytype_t) + _wasm_externtype_as_memorytype_const.argtypes = [POINTER(wasm_externtype_t)] + return _wasm_externtype_as_memorytype_const(arg0) + +class wasm_importtype_t(Structure): + pass + +def wasm_importtype_delete(arg0): + _wasm_importtype_delete = libiwasm.wasm_importtype_delete + _wasm_importtype_delete.restype = None + _wasm_importtype_delete.argtypes = [POINTER(wasm_importtype_t)] + return _wasm_importtype_delete(arg0) + +class wasm_importtype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_importtype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_importtype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_importtype_vec_new_empty(arg0): + _wasm_importtype_vec_new_empty = libiwasm.wasm_importtype_vec_new_empty + _wasm_importtype_vec_new_empty.restype = None + _wasm_importtype_vec_new_empty.argtypes = [POINTER(wasm_importtype_vec_t)] + return _wasm_importtype_vec_new_empty(arg0) + +def wasm_importtype_vec_new_uninitialized(arg0,arg1): + _wasm_importtype_vec_new_uninitialized = libiwasm.wasm_importtype_vec_new_uninitialized + _wasm_importtype_vec_new_uninitialized.restype = None + _wasm_importtype_vec_new_uninitialized.argtypes = [POINTER(wasm_importtype_vec_t),c_size_t] + return _wasm_importtype_vec_new_uninitialized(arg0,arg1) + +def wasm_importtype_vec_new(arg0,arg1,arg2): + _wasm_importtype_vec_new = libiwasm.wasm_importtype_vec_new + _wasm_importtype_vec_new.restype = None + _wasm_importtype_vec_new.argtypes = [POINTER(wasm_importtype_vec_t),c_size_t,POINTER(POINTER(wasm_importtype_t))] + return _wasm_importtype_vec_new(arg0,arg1,arg2) + +def wasm_importtype_vec_copy(arg0,arg1): + _wasm_importtype_vec_copy = libiwasm.wasm_importtype_vec_copy + _wasm_importtype_vec_copy.restype = None + _wasm_importtype_vec_copy.argtypes = [POINTER(wasm_importtype_vec_t),POINTER(wasm_importtype_vec_t)] + return _wasm_importtype_vec_copy(arg0,arg1) + +def wasm_importtype_vec_delete(arg0): + _wasm_importtype_vec_delete = libiwasm.wasm_importtype_vec_delete + _wasm_importtype_vec_delete.restype = None + _wasm_importtype_vec_delete.argtypes = [POINTER(wasm_importtype_vec_t)] + return _wasm_importtype_vec_delete(arg0) + +def wasm_importtype_copy(arg0): + _wasm_importtype_copy = libiwasm.wasm_importtype_copy + _wasm_importtype_copy.restype = POINTER(wasm_importtype_t) + _wasm_importtype_copy.argtypes = [POINTER(wasm_importtype_t)] + return _wasm_importtype_copy(arg0) + +def wasm_importtype_new(arg0,arg1,arg2): + _wasm_importtype_new = libiwasm.wasm_importtype_new + _wasm_importtype_new.restype = POINTER(wasm_importtype_t) + _wasm_importtype_new.argtypes = [POINTER(wasm_name_t),POINTER(wasm_name_t),POINTER(wasm_externtype_t)] + return _wasm_importtype_new(arg0,arg1,arg2) + +def wasm_importtype_module(arg0): + _wasm_importtype_module = libiwasm.wasm_importtype_module + _wasm_importtype_module.restype = POINTER(wasm_name_t) + _wasm_importtype_module.argtypes = [POINTER(wasm_importtype_t)] + return _wasm_importtype_module(arg0) + +def wasm_importtype_name(arg0): + _wasm_importtype_name = libiwasm.wasm_importtype_name + _wasm_importtype_name.restype = POINTER(wasm_name_t) + _wasm_importtype_name.argtypes = [POINTER(wasm_importtype_t)] + return _wasm_importtype_name(arg0) + +def wasm_importtype_type(arg0): + _wasm_importtype_type = libiwasm.wasm_importtype_type + _wasm_importtype_type.restype = POINTER(wasm_externtype_t) + _wasm_importtype_type.argtypes = [POINTER(wasm_importtype_t)] + return _wasm_importtype_type(arg0) + +class wasm_exporttype_t(Structure): + pass + +def wasm_exporttype_delete(arg0): + _wasm_exporttype_delete = libiwasm.wasm_exporttype_delete + _wasm_exporttype_delete.restype = None + _wasm_exporttype_delete.argtypes = [POINTER(wasm_exporttype_t)] + return _wasm_exporttype_delete(arg0) + +class wasm_exporttype_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_exporttype_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_exporttype_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_exporttype_vec_new_empty(arg0): + _wasm_exporttype_vec_new_empty = libiwasm.wasm_exporttype_vec_new_empty + _wasm_exporttype_vec_new_empty.restype = None + _wasm_exporttype_vec_new_empty.argtypes = [POINTER(wasm_exporttype_vec_t)] + return _wasm_exporttype_vec_new_empty(arg0) + +def wasm_exporttype_vec_new_uninitialized(arg0,arg1): + _wasm_exporttype_vec_new_uninitialized = libiwasm.wasm_exporttype_vec_new_uninitialized + _wasm_exporttype_vec_new_uninitialized.restype = None + _wasm_exporttype_vec_new_uninitialized.argtypes = [POINTER(wasm_exporttype_vec_t),c_size_t] + return _wasm_exporttype_vec_new_uninitialized(arg0,arg1) + +def wasm_exporttype_vec_new(arg0,arg1,arg2): + _wasm_exporttype_vec_new = libiwasm.wasm_exporttype_vec_new + _wasm_exporttype_vec_new.restype = None + _wasm_exporttype_vec_new.argtypes = [POINTER(wasm_exporttype_vec_t),c_size_t,POINTER(POINTER(wasm_exporttype_t))] + return _wasm_exporttype_vec_new(arg0,arg1,arg2) + +def wasm_exporttype_vec_copy(arg0,arg1): + _wasm_exporttype_vec_copy = libiwasm.wasm_exporttype_vec_copy + _wasm_exporttype_vec_copy.restype = None + _wasm_exporttype_vec_copy.argtypes = [POINTER(wasm_exporttype_vec_t),POINTER(wasm_exporttype_vec_t)] + return _wasm_exporttype_vec_copy(arg0,arg1) + +def wasm_exporttype_vec_delete(arg0): + _wasm_exporttype_vec_delete = libiwasm.wasm_exporttype_vec_delete + _wasm_exporttype_vec_delete.restype = None + _wasm_exporttype_vec_delete.argtypes = [POINTER(wasm_exporttype_vec_t)] + return _wasm_exporttype_vec_delete(arg0) + +def wasm_exporttype_copy(arg0): + _wasm_exporttype_copy = libiwasm.wasm_exporttype_copy + _wasm_exporttype_copy.restype = POINTER(wasm_exporttype_t) + _wasm_exporttype_copy.argtypes = [POINTER(wasm_exporttype_t)] + return _wasm_exporttype_copy(arg0) + +def wasm_exporttype_new(arg0,arg1): + _wasm_exporttype_new = libiwasm.wasm_exporttype_new + _wasm_exporttype_new.restype = POINTER(wasm_exporttype_t) + _wasm_exporttype_new.argtypes = [POINTER(wasm_name_t),POINTER(wasm_externtype_t)] + return _wasm_exporttype_new(arg0,arg1) + +def wasm_exporttype_name(arg0): + _wasm_exporttype_name = libiwasm.wasm_exporttype_name + _wasm_exporttype_name.restype = POINTER(wasm_name_t) + _wasm_exporttype_name.argtypes = [POINTER(wasm_exporttype_t)] + return _wasm_exporttype_name(arg0) + +def wasm_exporttype_type(arg0): + _wasm_exporttype_type = libiwasm.wasm_exporttype_type + _wasm_exporttype_type.restype = POINTER(wasm_externtype_t) + _wasm_exporttype_type.argtypes = [POINTER(wasm_exporttype_t)] + return _wasm_exporttype_type(arg0) + +def wasm_val_delete(arg0): + _wasm_val_delete = libiwasm.wasm_val_delete + _wasm_val_delete.restype = None + _wasm_val_delete.argtypes = [POINTER(wasm_val_t)] + return _wasm_val_delete(arg0) + +def wasm_val_copy(arg0,arg1): + _wasm_val_copy = libiwasm.wasm_val_copy + _wasm_val_copy.restype = None + _wasm_val_copy.argtypes = [POINTER(wasm_val_t),POINTER(wasm_val_t)] + return _wasm_val_copy(arg0,arg1) + +class wasm_val_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(wasm_val_t)), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_val_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(self.data[i]) + ret += " " + return ret + + + +def wasm_val_vec_new_empty(arg0): + _wasm_val_vec_new_empty = libiwasm.wasm_val_vec_new_empty + _wasm_val_vec_new_empty.restype = None + _wasm_val_vec_new_empty.argtypes = [POINTER(wasm_val_vec_t)] + return _wasm_val_vec_new_empty(arg0) + +def wasm_val_vec_new_uninitialized(arg0,arg1): + _wasm_val_vec_new_uninitialized = libiwasm.wasm_val_vec_new_uninitialized + _wasm_val_vec_new_uninitialized.restype = None + _wasm_val_vec_new_uninitialized.argtypes = [POINTER(wasm_val_vec_t),c_size_t] + return _wasm_val_vec_new_uninitialized(arg0,arg1) + +def wasm_val_vec_new(arg0,arg1,arg2): + _wasm_val_vec_new = libiwasm.wasm_val_vec_new + _wasm_val_vec_new.restype = None + _wasm_val_vec_new.argtypes = [POINTER(wasm_val_vec_t),c_size_t,POINTER(wasm_val_t)] + return _wasm_val_vec_new(arg0,arg1,arg2) + +def wasm_val_vec_copy(arg0,arg1): + _wasm_val_vec_copy = libiwasm.wasm_val_vec_copy + _wasm_val_vec_copy.restype = None + _wasm_val_vec_copy.argtypes = [POINTER(wasm_val_vec_t),POINTER(wasm_val_vec_t)] + return _wasm_val_vec_copy(arg0,arg1) + +def wasm_val_vec_delete(arg0): + _wasm_val_vec_delete = libiwasm.wasm_val_vec_delete + _wasm_val_vec_delete.restype = None + _wasm_val_vec_delete.argtypes = [POINTER(wasm_val_vec_t)] + return _wasm_val_vec_delete(arg0) + +def wasm_ref_delete(arg0): + _wasm_ref_delete = libiwasm.wasm_ref_delete + _wasm_ref_delete.restype = None + _wasm_ref_delete.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_delete(arg0) + +def wasm_ref_copy(arg0): + _wasm_ref_copy = libiwasm.wasm_ref_copy + _wasm_ref_copy.restype = POINTER(wasm_ref_t) + _wasm_ref_copy.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_copy(arg0) + +def wasm_ref_same(arg0,arg1): + _wasm_ref_same = libiwasm.wasm_ref_same + _wasm_ref_same.restype = c_bool + _wasm_ref_same.argtypes = [POINTER(wasm_ref_t),POINTER(wasm_ref_t)] + return _wasm_ref_same(arg0,arg1) + +def wasm_ref_get_host_info(arg0): + _wasm_ref_get_host_info = libiwasm.wasm_ref_get_host_info + _wasm_ref_get_host_info.restype = c_void_p + _wasm_ref_get_host_info.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_get_host_info(arg0) + +def wasm_ref_set_host_info(arg0,arg1): + _wasm_ref_set_host_info = libiwasm.wasm_ref_set_host_info + _wasm_ref_set_host_info.restype = None + _wasm_ref_set_host_info.argtypes = [POINTER(wasm_ref_t),c_void_p] + return _wasm_ref_set_host_info(arg0,arg1) + +def wasm_ref_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_ref_set_host_info_with_finalizer = libiwasm.wasm_ref_set_host_info_with_finalizer + _wasm_ref_set_host_info_with_finalizer.restype = None + _wasm_ref_set_host_info_with_finalizer.argtypes = [POINTER(wasm_ref_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_ref_set_host_info_with_finalizer(arg0,arg1,arg2) + +class wasm_frame_t(Structure): + pass + +def wasm_frame_delete(arg0): + _wasm_frame_delete = libiwasm.wasm_frame_delete + _wasm_frame_delete.restype = None + _wasm_frame_delete.argtypes = [POINTER(wasm_frame_t)] + return _wasm_frame_delete(arg0) + +class wasm_frame_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_frame_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_frame_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_frame_vec_new_empty(arg0): + _wasm_frame_vec_new_empty = libiwasm.wasm_frame_vec_new_empty + _wasm_frame_vec_new_empty.restype = None + _wasm_frame_vec_new_empty.argtypes = [POINTER(wasm_frame_vec_t)] + return _wasm_frame_vec_new_empty(arg0) + +def wasm_frame_vec_new_uninitialized(arg0,arg1): + _wasm_frame_vec_new_uninitialized = libiwasm.wasm_frame_vec_new_uninitialized + _wasm_frame_vec_new_uninitialized.restype = None + _wasm_frame_vec_new_uninitialized.argtypes = [POINTER(wasm_frame_vec_t),c_size_t] + return _wasm_frame_vec_new_uninitialized(arg0,arg1) + +def wasm_frame_vec_new(arg0,arg1,arg2): + _wasm_frame_vec_new = libiwasm.wasm_frame_vec_new + _wasm_frame_vec_new.restype = None + _wasm_frame_vec_new.argtypes = [POINTER(wasm_frame_vec_t),c_size_t,POINTER(POINTER(wasm_frame_t))] + return _wasm_frame_vec_new(arg0,arg1,arg2) + +def wasm_frame_vec_copy(arg0,arg1): + _wasm_frame_vec_copy = libiwasm.wasm_frame_vec_copy + _wasm_frame_vec_copy.restype = None + _wasm_frame_vec_copy.argtypes = [POINTER(wasm_frame_vec_t),POINTER(wasm_frame_vec_t)] + return _wasm_frame_vec_copy(arg0,arg1) + +def wasm_frame_vec_delete(arg0): + _wasm_frame_vec_delete = libiwasm.wasm_frame_vec_delete + _wasm_frame_vec_delete.restype = None + _wasm_frame_vec_delete.argtypes = [POINTER(wasm_frame_vec_t)] + return _wasm_frame_vec_delete(arg0) + +def wasm_frame_copy(arg0): + _wasm_frame_copy = libiwasm.wasm_frame_copy + _wasm_frame_copy.restype = POINTER(wasm_frame_t) + _wasm_frame_copy.argtypes = [POINTER(wasm_frame_t)] + return _wasm_frame_copy(arg0) + +def wasm_frame_instance(arg0): + _wasm_frame_instance = libiwasm.wasm_frame_instance + _wasm_frame_instance.restype = POINTER(wasm_instance_t) + _wasm_frame_instance.argtypes = [POINTER(wasm_frame_t)] + return _wasm_frame_instance(arg0) + +def wasm_frame_func_index(arg0): + _wasm_frame_func_index = libiwasm.wasm_frame_func_index + _wasm_frame_func_index.restype = c_uint32 + _wasm_frame_func_index.argtypes = [POINTER(wasm_frame_t)] + return _wasm_frame_func_index(arg0) + +def wasm_frame_func_offset(arg0): + _wasm_frame_func_offset = libiwasm.wasm_frame_func_offset + _wasm_frame_func_offset.restype = c_size_t + _wasm_frame_func_offset.argtypes = [POINTER(wasm_frame_t)] + return _wasm_frame_func_offset(arg0) + +def wasm_frame_module_offset(arg0): + _wasm_frame_module_offset = libiwasm.wasm_frame_module_offset + _wasm_frame_module_offset.restype = c_size_t + _wasm_frame_module_offset.argtypes = [POINTER(wasm_frame_t)] + return _wasm_frame_module_offset(arg0) + +wasm_message_t = wasm_name_t + +class wasm_trap_t(Structure): + pass + +def wasm_trap_delete(arg0): + _wasm_trap_delete = libiwasm.wasm_trap_delete + _wasm_trap_delete.restype = None + _wasm_trap_delete.argtypes = [POINTER(wasm_trap_t)] + return _wasm_trap_delete(arg0) + +def wasm_trap_copy(arg0): + _wasm_trap_copy = libiwasm.wasm_trap_copy + _wasm_trap_copy.restype = POINTER(wasm_trap_t) + _wasm_trap_copy.argtypes = [POINTER(wasm_trap_t)] + return _wasm_trap_copy(arg0) + +def wasm_trap_same(arg0,arg1): + _wasm_trap_same = libiwasm.wasm_trap_same + _wasm_trap_same.restype = c_bool + _wasm_trap_same.argtypes = [POINTER(wasm_trap_t),POINTER(wasm_trap_t)] + return _wasm_trap_same(arg0,arg1) + +def wasm_trap_get_host_info(arg0): + _wasm_trap_get_host_info = libiwasm.wasm_trap_get_host_info + _wasm_trap_get_host_info.restype = c_void_p + _wasm_trap_get_host_info.argtypes = [POINTER(wasm_trap_t)] + return _wasm_trap_get_host_info(arg0) + +def wasm_trap_set_host_info(arg0,arg1): + _wasm_trap_set_host_info = libiwasm.wasm_trap_set_host_info + _wasm_trap_set_host_info.restype = None + _wasm_trap_set_host_info.argtypes = [POINTER(wasm_trap_t),c_void_p] + return _wasm_trap_set_host_info(arg0,arg1) + +def wasm_trap_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_trap_set_host_info_with_finalizer = libiwasm.wasm_trap_set_host_info_with_finalizer + _wasm_trap_set_host_info_with_finalizer.restype = None + _wasm_trap_set_host_info_with_finalizer.argtypes = [POINTER(wasm_trap_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_trap_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_trap_as_ref(arg0): + _wasm_trap_as_ref = libiwasm.wasm_trap_as_ref + _wasm_trap_as_ref.restype = POINTER(wasm_ref_t) + _wasm_trap_as_ref.argtypes = [POINTER(wasm_trap_t)] + return _wasm_trap_as_ref(arg0) + +def wasm_ref_as_trap(arg0): + _wasm_ref_as_trap = libiwasm.wasm_ref_as_trap + _wasm_ref_as_trap.restype = POINTER(wasm_trap_t) + _wasm_ref_as_trap.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_trap(arg0) + +def wasm_trap_as_ref_const(arg0): + _wasm_trap_as_ref_const = libiwasm.wasm_trap_as_ref_const + _wasm_trap_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_trap_as_ref_const.argtypes = [POINTER(wasm_trap_t)] + return _wasm_trap_as_ref_const(arg0) + +def wasm_ref_as_trap_const(arg0): + _wasm_ref_as_trap_const = libiwasm.wasm_ref_as_trap_const + _wasm_ref_as_trap_const.restype = POINTER(wasm_trap_t) + _wasm_ref_as_trap_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_trap_const(arg0) + +def wasm_trap_new(arg0,arg1): + _wasm_trap_new = libiwasm.wasm_trap_new + _wasm_trap_new.restype = POINTER(wasm_trap_t) + _wasm_trap_new.argtypes = [POINTER(wasm_store_t),POINTER(wasm_message_t)] + return _wasm_trap_new(arg0,arg1) + +def wasm_trap_message(arg0,arg1): + _wasm_trap_message = libiwasm.wasm_trap_message + _wasm_trap_message.restype = None + _wasm_trap_message.argtypes = [POINTER(wasm_trap_t),POINTER(wasm_message_t)] + return _wasm_trap_message(arg0,arg1) + +def wasm_trap_origin(arg0): + _wasm_trap_origin = libiwasm.wasm_trap_origin + _wasm_trap_origin.restype = POINTER(wasm_frame_t) + _wasm_trap_origin.argtypes = [POINTER(wasm_trap_t)] + return _wasm_trap_origin(arg0) + +def wasm_trap_trace(arg0,arg1): + _wasm_trap_trace = libiwasm.wasm_trap_trace + _wasm_trap_trace.restype = None + _wasm_trap_trace.argtypes = [POINTER(wasm_trap_t),POINTER(wasm_frame_vec_t)] + return _wasm_trap_trace(arg0,arg1) + +class wasm_foreign_t(Structure): + pass + +def wasm_foreign_delete(arg0): + _wasm_foreign_delete = libiwasm.wasm_foreign_delete + _wasm_foreign_delete.restype = None + _wasm_foreign_delete.argtypes = [POINTER(wasm_foreign_t)] + return _wasm_foreign_delete(arg0) + +def wasm_foreign_copy(arg0): + _wasm_foreign_copy = libiwasm.wasm_foreign_copy + _wasm_foreign_copy.restype = POINTER(wasm_foreign_t) + _wasm_foreign_copy.argtypes = [POINTER(wasm_foreign_t)] + return _wasm_foreign_copy(arg0) + +def wasm_foreign_same(arg0,arg1): + _wasm_foreign_same = libiwasm.wasm_foreign_same + _wasm_foreign_same.restype = c_bool + _wasm_foreign_same.argtypes = [POINTER(wasm_foreign_t),POINTER(wasm_foreign_t)] + return _wasm_foreign_same(arg0,arg1) + +def wasm_foreign_get_host_info(arg0): + _wasm_foreign_get_host_info = libiwasm.wasm_foreign_get_host_info + _wasm_foreign_get_host_info.restype = c_void_p + _wasm_foreign_get_host_info.argtypes = [POINTER(wasm_foreign_t)] + return _wasm_foreign_get_host_info(arg0) + +def wasm_foreign_set_host_info(arg0,arg1): + _wasm_foreign_set_host_info = libiwasm.wasm_foreign_set_host_info + _wasm_foreign_set_host_info.restype = None + _wasm_foreign_set_host_info.argtypes = [POINTER(wasm_foreign_t),c_void_p] + return _wasm_foreign_set_host_info(arg0,arg1) + +def wasm_foreign_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_foreign_set_host_info_with_finalizer = libiwasm.wasm_foreign_set_host_info_with_finalizer + _wasm_foreign_set_host_info_with_finalizer.restype = None + _wasm_foreign_set_host_info_with_finalizer.argtypes = [POINTER(wasm_foreign_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_foreign_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_foreign_as_ref(arg0): + _wasm_foreign_as_ref = libiwasm.wasm_foreign_as_ref + _wasm_foreign_as_ref.restype = POINTER(wasm_ref_t) + _wasm_foreign_as_ref.argtypes = [POINTER(wasm_foreign_t)] + return _wasm_foreign_as_ref(arg0) + +def wasm_ref_as_foreign(arg0): + _wasm_ref_as_foreign = libiwasm.wasm_ref_as_foreign + _wasm_ref_as_foreign.restype = POINTER(wasm_foreign_t) + _wasm_ref_as_foreign.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_foreign(arg0) + +def wasm_foreign_as_ref_const(arg0): + _wasm_foreign_as_ref_const = libiwasm.wasm_foreign_as_ref_const + _wasm_foreign_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_foreign_as_ref_const.argtypes = [POINTER(wasm_foreign_t)] + return _wasm_foreign_as_ref_const(arg0) + +def wasm_ref_as_foreign_const(arg0): + _wasm_ref_as_foreign_const = libiwasm.wasm_ref_as_foreign_const + _wasm_ref_as_foreign_const.restype = POINTER(wasm_foreign_t) + _wasm_ref_as_foreign_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_foreign_const(arg0) + +def wasm_foreign_new(arg0): + _wasm_foreign_new = libiwasm.wasm_foreign_new + _wasm_foreign_new.restype = POINTER(wasm_foreign_t) + _wasm_foreign_new.argtypes = [POINTER(wasm_store_t)] + return _wasm_foreign_new(arg0) + +class WASMModuleCommon(Structure): + pass + +class WASMModuleCommon(Structure): + pass + +wasm_module_t = POINTER(WASMModuleCommon) + +def wasm_module_new(arg0,arg1): + _wasm_module_new = libiwasm.wasm_module_new + _wasm_module_new.restype = POINTER(wasm_module_t) + _wasm_module_new.argtypes = [POINTER(wasm_store_t),POINTER(wasm_byte_vec_t)] + return _wasm_module_new(arg0,arg1) + +def wasm_module_delete(arg0): + _wasm_module_delete = libiwasm.wasm_module_delete + _wasm_module_delete.restype = None + _wasm_module_delete.argtypes = [POINTER(wasm_module_t)] + return _wasm_module_delete(arg0) + +def wasm_module_validate(arg0,arg1): + _wasm_module_validate = libiwasm.wasm_module_validate + _wasm_module_validate.restype = c_bool + _wasm_module_validate.argtypes = [POINTER(wasm_store_t),POINTER(wasm_byte_vec_t)] + return _wasm_module_validate(arg0,arg1) + +def wasm_module_imports(arg0,arg1): + _wasm_module_imports = libiwasm.wasm_module_imports + _wasm_module_imports.restype = None + _wasm_module_imports.argtypes = [POINTER(wasm_module_t),POINTER(wasm_importtype_vec_t)] + return _wasm_module_imports(arg0,arg1) + +def wasm_module_exports(arg0,arg1): + _wasm_module_exports = libiwasm.wasm_module_exports + _wasm_module_exports.restype = None + _wasm_module_exports.argtypes = [POINTER(wasm_module_t),POINTER(wasm_exporttype_vec_t)] + return _wasm_module_exports(arg0,arg1) + +def wasm_module_serialize(arg0,arg1): + _wasm_module_serialize = libiwasm.wasm_module_serialize + _wasm_module_serialize.restype = None + _wasm_module_serialize.argtypes = [POINTER(wasm_module_t),POINTER(wasm_byte_vec_t)] + return _wasm_module_serialize(arg0,arg1) + +def wasm_module_deserialize(arg0,arg1): + _wasm_module_deserialize = libiwasm.wasm_module_deserialize + _wasm_module_deserialize.restype = POINTER(wasm_module_t) + _wasm_module_deserialize.argtypes = [POINTER(wasm_store_t),POINTER(wasm_byte_vec_t)] + return _wasm_module_deserialize(arg0,arg1) + +class wasm_func_t(Structure): + pass + +def wasm_func_delete(arg0): + _wasm_func_delete = libiwasm.wasm_func_delete + _wasm_func_delete.restype = None + _wasm_func_delete.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_delete(arg0) + +def wasm_func_copy(arg0): + _wasm_func_copy = libiwasm.wasm_func_copy + _wasm_func_copy.restype = POINTER(wasm_func_t) + _wasm_func_copy.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_copy(arg0) + +def wasm_func_same(arg0,arg1): + _wasm_func_same = libiwasm.wasm_func_same + _wasm_func_same.restype = c_bool + _wasm_func_same.argtypes = [POINTER(wasm_func_t),POINTER(wasm_func_t)] + return _wasm_func_same(arg0,arg1) + +def wasm_func_get_host_info(arg0): + _wasm_func_get_host_info = libiwasm.wasm_func_get_host_info + _wasm_func_get_host_info.restype = c_void_p + _wasm_func_get_host_info.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_get_host_info(arg0) + +def wasm_func_set_host_info(arg0,arg1): + _wasm_func_set_host_info = libiwasm.wasm_func_set_host_info + _wasm_func_set_host_info.restype = None + _wasm_func_set_host_info.argtypes = [POINTER(wasm_func_t),c_void_p] + return _wasm_func_set_host_info(arg0,arg1) + +def wasm_func_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_func_set_host_info_with_finalizer = libiwasm.wasm_func_set_host_info_with_finalizer + _wasm_func_set_host_info_with_finalizer.restype = None + _wasm_func_set_host_info_with_finalizer.argtypes = [POINTER(wasm_func_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_func_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_func_as_ref(arg0): + _wasm_func_as_ref = libiwasm.wasm_func_as_ref + _wasm_func_as_ref.restype = POINTER(wasm_ref_t) + _wasm_func_as_ref.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_as_ref(arg0) + +def wasm_ref_as_func(arg0): + _wasm_ref_as_func = libiwasm.wasm_ref_as_func + _wasm_ref_as_func.restype = POINTER(wasm_func_t) + _wasm_ref_as_func.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_func(arg0) + +def wasm_func_as_ref_const(arg0): + _wasm_func_as_ref_const = libiwasm.wasm_func_as_ref_const + _wasm_func_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_func_as_ref_const.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_as_ref_const(arg0) + +def wasm_ref_as_func_const(arg0): + _wasm_ref_as_func_const = libiwasm.wasm_ref_as_func_const + _wasm_ref_as_func_const.restype = POINTER(wasm_func_t) + _wasm_ref_as_func_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_func_const(arg0) + +wasm_func_callback_t = CFUNCTYPE(c_void_p,POINTER(wasm_val_vec_t),POINTER(wasm_val_vec_t)) + +wasm_func_callback_with_env_t = CFUNCTYPE(c_void_p,c_void_p,POINTER(wasm_val_vec_t),POINTER(wasm_val_vec_t)) + +def wasm_func_new(arg0,arg1,arg2): + _wasm_func_new = libiwasm.wasm_func_new + _wasm_func_new.restype = POINTER(wasm_func_t) + _wasm_func_new.argtypes = [POINTER(wasm_store_t),POINTER(wasm_functype_t),wasm_func_callback_t] + return _wasm_func_new(arg0,arg1,arg2) + +def wasm_func_new_with_env(arg0,arg1,arg2,arg3,arg4): + _wasm_func_new_with_env = libiwasm.wasm_func_new_with_env + _wasm_func_new_with_env.restype = POINTER(wasm_func_t) + _wasm_func_new_with_env.argtypes = [POINTER(wasm_store_t),POINTER(wasm_functype_t),wasm_func_callback_with_env_t,c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_func_new_with_env(arg0,arg1,arg2,arg3,arg4) + +def wasm_func_type(arg0): + _wasm_func_type = libiwasm.wasm_func_type + _wasm_func_type.restype = POINTER(wasm_functype_t) + _wasm_func_type.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_type(arg0) + +def wasm_func_param_arity(arg0): + _wasm_func_param_arity = libiwasm.wasm_func_param_arity + _wasm_func_param_arity.restype = c_size_t + _wasm_func_param_arity.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_param_arity(arg0) + +def wasm_func_result_arity(arg0): + _wasm_func_result_arity = libiwasm.wasm_func_result_arity + _wasm_func_result_arity.restype = c_size_t + _wasm_func_result_arity.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_result_arity(arg0) + +def wasm_func_call(arg0,arg1,arg2): + _wasm_func_call = libiwasm.wasm_func_call + _wasm_func_call.restype = POINTER(wasm_trap_t) + _wasm_func_call.argtypes = [POINTER(wasm_func_t),POINTER(wasm_val_vec_t),POINTER(wasm_val_vec_t)] + return _wasm_func_call(arg0,arg1,arg2) + +class wasm_global_t(Structure): + pass + +def wasm_global_delete(arg0): + _wasm_global_delete = libiwasm.wasm_global_delete + _wasm_global_delete.restype = None + _wasm_global_delete.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_delete(arg0) + +def wasm_global_copy(arg0): + _wasm_global_copy = libiwasm.wasm_global_copy + _wasm_global_copy.restype = POINTER(wasm_global_t) + _wasm_global_copy.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_copy(arg0) + +def wasm_global_same(arg0,arg1): + _wasm_global_same = libiwasm.wasm_global_same + _wasm_global_same.restype = c_bool + _wasm_global_same.argtypes = [POINTER(wasm_global_t),POINTER(wasm_global_t)] + return _wasm_global_same(arg0,arg1) + +def wasm_global_get_host_info(arg0): + _wasm_global_get_host_info = libiwasm.wasm_global_get_host_info + _wasm_global_get_host_info.restype = c_void_p + _wasm_global_get_host_info.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_get_host_info(arg0) + +def wasm_global_set_host_info(arg0,arg1): + _wasm_global_set_host_info = libiwasm.wasm_global_set_host_info + _wasm_global_set_host_info.restype = None + _wasm_global_set_host_info.argtypes = [POINTER(wasm_global_t),c_void_p] + return _wasm_global_set_host_info(arg0,arg1) + +def wasm_global_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_global_set_host_info_with_finalizer = libiwasm.wasm_global_set_host_info_with_finalizer + _wasm_global_set_host_info_with_finalizer.restype = None + _wasm_global_set_host_info_with_finalizer.argtypes = [POINTER(wasm_global_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_global_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_global_as_ref(arg0): + _wasm_global_as_ref = libiwasm.wasm_global_as_ref + _wasm_global_as_ref.restype = POINTER(wasm_ref_t) + _wasm_global_as_ref.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_as_ref(arg0) + +def wasm_ref_as_global(arg0): + _wasm_ref_as_global = libiwasm.wasm_ref_as_global + _wasm_ref_as_global.restype = POINTER(wasm_global_t) + _wasm_ref_as_global.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_global(arg0) + +def wasm_global_as_ref_const(arg0): + _wasm_global_as_ref_const = libiwasm.wasm_global_as_ref_const + _wasm_global_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_global_as_ref_const.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_as_ref_const(arg0) + +def wasm_ref_as_global_const(arg0): + _wasm_ref_as_global_const = libiwasm.wasm_ref_as_global_const + _wasm_ref_as_global_const.restype = POINTER(wasm_global_t) + _wasm_ref_as_global_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_global_const(arg0) + +def wasm_global_new(arg0,arg1,arg2): + _wasm_global_new = libiwasm.wasm_global_new + _wasm_global_new.restype = POINTER(wasm_global_t) + _wasm_global_new.argtypes = [POINTER(wasm_store_t),POINTER(wasm_globaltype_t),POINTER(wasm_val_t)] + return _wasm_global_new(arg0,arg1,arg2) + +def wasm_global_type(arg0): + _wasm_global_type = libiwasm.wasm_global_type + _wasm_global_type.restype = POINTER(wasm_globaltype_t) + _wasm_global_type.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_type(arg0) + +def wasm_global_get(arg0,arg1): + _wasm_global_get = libiwasm.wasm_global_get + _wasm_global_get.restype = None + _wasm_global_get.argtypes = [POINTER(wasm_global_t),POINTER(wasm_val_t)] + return _wasm_global_get(arg0,arg1) + +def wasm_global_set(arg0,arg1): + _wasm_global_set = libiwasm.wasm_global_set + _wasm_global_set.restype = None + _wasm_global_set.argtypes = [POINTER(wasm_global_t),POINTER(wasm_val_t)] + return _wasm_global_set(arg0,arg1) + +class wasm_table_t(Structure): + pass + +def wasm_table_delete(arg0): + _wasm_table_delete = libiwasm.wasm_table_delete + _wasm_table_delete.restype = None + _wasm_table_delete.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_delete(arg0) + +def wasm_table_copy(arg0): + _wasm_table_copy = libiwasm.wasm_table_copy + _wasm_table_copy.restype = POINTER(wasm_table_t) + _wasm_table_copy.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_copy(arg0) + +def wasm_table_same(arg0,arg1): + _wasm_table_same = libiwasm.wasm_table_same + _wasm_table_same.restype = c_bool + _wasm_table_same.argtypes = [POINTER(wasm_table_t),POINTER(wasm_table_t)] + return _wasm_table_same(arg0,arg1) + +def wasm_table_get_host_info(arg0): + _wasm_table_get_host_info = libiwasm.wasm_table_get_host_info + _wasm_table_get_host_info.restype = c_void_p + _wasm_table_get_host_info.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_get_host_info(arg0) + +def wasm_table_set_host_info(arg0,arg1): + _wasm_table_set_host_info = libiwasm.wasm_table_set_host_info + _wasm_table_set_host_info.restype = None + _wasm_table_set_host_info.argtypes = [POINTER(wasm_table_t),c_void_p] + return _wasm_table_set_host_info(arg0,arg1) + +def wasm_table_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_table_set_host_info_with_finalizer = libiwasm.wasm_table_set_host_info_with_finalizer + _wasm_table_set_host_info_with_finalizer.restype = None + _wasm_table_set_host_info_with_finalizer.argtypes = [POINTER(wasm_table_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_table_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_table_as_ref(arg0): + _wasm_table_as_ref = libiwasm.wasm_table_as_ref + _wasm_table_as_ref.restype = POINTER(wasm_ref_t) + _wasm_table_as_ref.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_as_ref(arg0) + +def wasm_ref_as_table(arg0): + _wasm_ref_as_table = libiwasm.wasm_ref_as_table + _wasm_ref_as_table.restype = POINTER(wasm_table_t) + _wasm_ref_as_table.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_table(arg0) + +def wasm_table_as_ref_const(arg0): + _wasm_table_as_ref_const = libiwasm.wasm_table_as_ref_const + _wasm_table_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_table_as_ref_const.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_as_ref_const(arg0) + +def wasm_ref_as_table_const(arg0): + _wasm_ref_as_table_const = libiwasm.wasm_ref_as_table_const + _wasm_ref_as_table_const.restype = POINTER(wasm_table_t) + _wasm_ref_as_table_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_table_const(arg0) + +wasm_table_size_t = c_uint32 + +def wasm_table_new(arg0,arg1,arg2): + _wasm_table_new = libiwasm.wasm_table_new + _wasm_table_new.restype = POINTER(wasm_table_t) + _wasm_table_new.argtypes = [POINTER(wasm_store_t),POINTER(wasm_tabletype_t),POINTER(wasm_ref_t)] + return _wasm_table_new(arg0,arg1,arg2) + +def wasm_table_type(arg0): + _wasm_table_type = libiwasm.wasm_table_type + _wasm_table_type.restype = POINTER(wasm_tabletype_t) + _wasm_table_type.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_type(arg0) + +def wasm_table_get(arg0,arg1): + _wasm_table_get = libiwasm.wasm_table_get + _wasm_table_get.restype = POINTER(wasm_ref_t) + _wasm_table_get.argtypes = [POINTER(wasm_table_t),wasm_table_size_t] + return _wasm_table_get(arg0,arg1) + +def wasm_table_set(arg0,arg1,arg2): + _wasm_table_set = libiwasm.wasm_table_set + _wasm_table_set.restype = c_bool + _wasm_table_set.argtypes = [POINTER(wasm_table_t),wasm_table_size_t,POINTER(wasm_ref_t)] + return _wasm_table_set(arg0,arg1,arg2) + +def wasm_table_size(arg0): + _wasm_table_size = libiwasm.wasm_table_size + _wasm_table_size.restype = wasm_table_size_t + _wasm_table_size.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_size(arg0) + +def wasm_table_grow(arg0,arg1,arg2): + _wasm_table_grow = libiwasm.wasm_table_grow + _wasm_table_grow.restype = c_bool + _wasm_table_grow.argtypes = [POINTER(wasm_table_t),wasm_table_size_t,POINTER(wasm_ref_t)] + return _wasm_table_grow(arg0,arg1,arg2) + +class wasm_memory_t(Structure): + pass + +def wasm_memory_delete(arg0): + _wasm_memory_delete = libiwasm.wasm_memory_delete + _wasm_memory_delete.restype = None + _wasm_memory_delete.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_delete(arg0) + +def wasm_memory_copy(arg0): + _wasm_memory_copy = libiwasm.wasm_memory_copy + _wasm_memory_copy.restype = POINTER(wasm_memory_t) + _wasm_memory_copy.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_copy(arg0) + +def wasm_memory_same(arg0,arg1): + _wasm_memory_same = libiwasm.wasm_memory_same + _wasm_memory_same.restype = c_bool + _wasm_memory_same.argtypes = [POINTER(wasm_memory_t),POINTER(wasm_memory_t)] + return _wasm_memory_same(arg0,arg1) + +def wasm_memory_get_host_info(arg0): + _wasm_memory_get_host_info = libiwasm.wasm_memory_get_host_info + _wasm_memory_get_host_info.restype = c_void_p + _wasm_memory_get_host_info.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_get_host_info(arg0) + +def wasm_memory_set_host_info(arg0,arg1): + _wasm_memory_set_host_info = libiwasm.wasm_memory_set_host_info + _wasm_memory_set_host_info.restype = None + _wasm_memory_set_host_info.argtypes = [POINTER(wasm_memory_t),c_void_p] + return _wasm_memory_set_host_info(arg0,arg1) + +def wasm_memory_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_memory_set_host_info_with_finalizer = libiwasm.wasm_memory_set_host_info_with_finalizer + _wasm_memory_set_host_info_with_finalizer.restype = None + _wasm_memory_set_host_info_with_finalizer.argtypes = [POINTER(wasm_memory_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_memory_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_memory_as_ref(arg0): + _wasm_memory_as_ref = libiwasm.wasm_memory_as_ref + _wasm_memory_as_ref.restype = POINTER(wasm_ref_t) + _wasm_memory_as_ref.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_as_ref(arg0) + +def wasm_ref_as_memory(arg0): + _wasm_ref_as_memory = libiwasm.wasm_ref_as_memory + _wasm_ref_as_memory.restype = POINTER(wasm_memory_t) + _wasm_ref_as_memory.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_memory(arg0) + +def wasm_memory_as_ref_const(arg0): + _wasm_memory_as_ref_const = libiwasm.wasm_memory_as_ref_const + _wasm_memory_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_memory_as_ref_const.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_as_ref_const(arg0) + +def wasm_ref_as_memory_const(arg0): + _wasm_ref_as_memory_const = libiwasm.wasm_ref_as_memory_const + _wasm_ref_as_memory_const.restype = POINTER(wasm_memory_t) + _wasm_ref_as_memory_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_memory_const(arg0) + +wasm_memory_pages_t = c_uint32 + +def wasm_memory_new(arg0,arg1): + _wasm_memory_new = libiwasm.wasm_memory_new + _wasm_memory_new.restype = POINTER(wasm_memory_t) + _wasm_memory_new.argtypes = [POINTER(wasm_store_t),POINTER(wasm_memorytype_t)] + return _wasm_memory_new(arg0,arg1) + +def wasm_memory_type(arg0): + _wasm_memory_type = libiwasm.wasm_memory_type + _wasm_memory_type.restype = POINTER(wasm_memorytype_t) + _wasm_memory_type.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_type(arg0) + +def wasm_memory_data(arg0): + _wasm_memory_data = libiwasm.wasm_memory_data + _wasm_memory_data.restype = POINTER(c_ubyte) + _wasm_memory_data.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_data(arg0) + +def wasm_memory_data_size(arg0): + _wasm_memory_data_size = libiwasm.wasm_memory_data_size + _wasm_memory_data_size.restype = c_size_t + _wasm_memory_data_size.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_data_size(arg0) + +def wasm_memory_size(arg0): + _wasm_memory_size = libiwasm.wasm_memory_size + _wasm_memory_size.restype = wasm_memory_pages_t + _wasm_memory_size.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_size(arg0) + +def wasm_memory_grow(arg0,arg1): + _wasm_memory_grow = libiwasm.wasm_memory_grow + _wasm_memory_grow.restype = c_bool + _wasm_memory_grow.argtypes = [POINTER(wasm_memory_t),wasm_memory_pages_t] + return _wasm_memory_grow(arg0,arg1) + +class wasm_extern_t(Structure): + pass + +def wasm_extern_delete(arg0): + _wasm_extern_delete = libiwasm.wasm_extern_delete + _wasm_extern_delete.restype = None + _wasm_extern_delete.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_delete(arg0) + +def wasm_extern_copy(arg0): + _wasm_extern_copy = libiwasm.wasm_extern_copy + _wasm_extern_copy.restype = POINTER(wasm_extern_t) + _wasm_extern_copy.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_copy(arg0) + +def wasm_extern_same(arg0,arg1): + _wasm_extern_same = libiwasm.wasm_extern_same + _wasm_extern_same.restype = c_bool + _wasm_extern_same.argtypes = [POINTER(wasm_extern_t),POINTER(wasm_extern_t)] + return _wasm_extern_same(arg0,arg1) + +def wasm_extern_get_host_info(arg0): + _wasm_extern_get_host_info = libiwasm.wasm_extern_get_host_info + _wasm_extern_get_host_info.restype = c_void_p + _wasm_extern_get_host_info.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_get_host_info(arg0) + +def wasm_extern_set_host_info(arg0,arg1): + _wasm_extern_set_host_info = libiwasm.wasm_extern_set_host_info + _wasm_extern_set_host_info.restype = None + _wasm_extern_set_host_info.argtypes = [POINTER(wasm_extern_t),c_void_p] + return _wasm_extern_set_host_info(arg0,arg1) + +def wasm_extern_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_extern_set_host_info_with_finalizer = libiwasm.wasm_extern_set_host_info_with_finalizer + _wasm_extern_set_host_info_with_finalizer.restype = None + _wasm_extern_set_host_info_with_finalizer.argtypes = [POINTER(wasm_extern_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_extern_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_extern_as_ref(arg0): + _wasm_extern_as_ref = libiwasm.wasm_extern_as_ref + _wasm_extern_as_ref.restype = POINTER(wasm_ref_t) + _wasm_extern_as_ref.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_ref(arg0) + +def wasm_ref_as_extern(arg0): + _wasm_ref_as_extern = libiwasm.wasm_ref_as_extern + _wasm_ref_as_extern.restype = POINTER(wasm_extern_t) + _wasm_ref_as_extern.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_extern(arg0) + +def wasm_extern_as_ref_const(arg0): + _wasm_extern_as_ref_const = libiwasm.wasm_extern_as_ref_const + _wasm_extern_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_extern_as_ref_const.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_ref_const(arg0) + +def wasm_ref_as_extern_const(arg0): + _wasm_ref_as_extern_const = libiwasm.wasm_ref_as_extern_const + _wasm_ref_as_extern_const.restype = POINTER(wasm_extern_t) + _wasm_ref_as_extern_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_extern_const(arg0) + +class wasm_extern_vec_t(Structure): + _fields_ = [ + ("size", c_size_t), + ("data", POINTER(POINTER(wasm_extern_t))), + ("num_elems", c_size_t), + ("size_of_elem", c_size_t), + ("lock", c_void_p), + ] + + def __eq__(self, other): + if not isinstance(other, wasm_extern_vec_t): + return False + return self.size == other.size and self.num_elems == other.num_elems and self.size_of_elem == other.size_of_elem + + def __repr__(self): + ret = "" + for i in range(self.num_elems): + ret += str(dereference(self.data[i])) + ret += " " + return ret + + + +def wasm_extern_vec_new_empty(arg0): + _wasm_extern_vec_new_empty = libiwasm.wasm_extern_vec_new_empty + _wasm_extern_vec_new_empty.restype = None + _wasm_extern_vec_new_empty.argtypes = [POINTER(wasm_extern_vec_t)] + return _wasm_extern_vec_new_empty(arg0) + +def wasm_extern_vec_new_uninitialized(arg0,arg1): + _wasm_extern_vec_new_uninitialized = libiwasm.wasm_extern_vec_new_uninitialized + _wasm_extern_vec_new_uninitialized.restype = None + _wasm_extern_vec_new_uninitialized.argtypes = [POINTER(wasm_extern_vec_t),c_size_t] + return _wasm_extern_vec_new_uninitialized(arg0,arg1) + +def wasm_extern_vec_new(arg0,arg1,arg2): + _wasm_extern_vec_new = libiwasm.wasm_extern_vec_new + _wasm_extern_vec_new.restype = None + _wasm_extern_vec_new.argtypes = [POINTER(wasm_extern_vec_t),c_size_t,POINTER(POINTER(wasm_extern_t))] + return _wasm_extern_vec_new(arg0,arg1,arg2) + +def wasm_extern_vec_copy(arg0,arg1): + _wasm_extern_vec_copy = libiwasm.wasm_extern_vec_copy + _wasm_extern_vec_copy.restype = None + _wasm_extern_vec_copy.argtypes = [POINTER(wasm_extern_vec_t),POINTER(wasm_extern_vec_t)] + return _wasm_extern_vec_copy(arg0,arg1) + +def wasm_extern_vec_delete(arg0): + _wasm_extern_vec_delete = libiwasm.wasm_extern_vec_delete + _wasm_extern_vec_delete.restype = None + _wasm_extern_vec_delete.argtypes = [POINTER(wasm_extern_vec_t)] + return _wasm_extern_vec_delete(arg0) + +def wasm_extern_kind(arg0): + _wasm_extern_kind = libiwasm.wasm_extern_kind + _wasm_extern_kind.restype = wasm_externkind_t + _wasm_extern_kind.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_kind(arg0) + +def wasm_extern_type(arg0): + _wasm_extern_type = libiwasm.wasm_extern_type + _wasm_extern_type.restype = POINTER(wasm_externtype_t) + _wasm_extern_type.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_type(arg0) + +def wasm_func_as_extern(arg0): + _wasm_func_as_extern = libiwasm.wasm_func_as_extern + _wasm_func_as_extern.restype = POINTER(wasm_extern_t) + _wasm_func_as_extern.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_as_extern(arg0) + +def wasm_global_as_extern(arg0): + _wasm_global_as_extern = libiwasm.wasm_global_as_extern + _wasm_global_as_extern.restype = POINTER(wasm_extern_t) + _wasm_global_as_extern.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_as_extern(arg0) + +def wasm_table_as_extern(arg0): + _wasm_table_as_extern = libiwasm.wasm_table_as_extern + _wasm_table_as_extern.restype = POINTER(wasm_extern_t) + _wasm_table_as_extern.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_as_extern(arg0) + +def wasm_memory_as_extern(arg0): + _wasm_memory_as_extern = libiwasm.wasm_memory_as_extern + _wasm_memory_as_extern.restype = POINTER(wasm_extern_t) + _wasm_memory_as_extern.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_as_extern(arg0) + +def wasm_extern_as_func(arg0): + _wasm_extern_as_func = libiwasm.wasm_extern_as_func + _wasm_extern_as_func.restype = POINTER(wasm_func_t) + _wasm_extern_as_func.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_func(arg0) + +def wasm_extern_as_global(arg0): + _wasm_extern_as_global = libiwasm.wasm_extern_as_global + _wasm_extern_as_global.restype = POINTER(wasm_global_t) + _wasm_extern_as_global.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_global(arg0) + +def wasm_extern_as_table(arg0): + _wasm_extern_as_table = libiwasm.wasm_extern_as_table + _wasm_extern_as_table.restype = POINTER(wasm_table_t) + _wasm_extern_as_table.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_table(arg0) + +def wasm_extern_as_memory(arg0): + _wasm_extern_as_memory = libiwasm.wasm_extern_as_memory + _wasm_extern_as_memory.restype = POINTER(wasm_memory_t) + _wasm_extern_as_memory.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_memory(arg0) + +def wasm_func_as_extern_const(arg0): + _wasm_func_as_extern_const = libiwasm.wasm_func_as_extern_const + _wasm_func_as_extern_const.restype = POINTER(wasm_extern_t) + _wasm_func_as_extern_const.argtypes = [POINTER(wasm_func_t)] + return _wasm_func_as_extern_const(arg0) + +def wasm_global_as_extern_const(arg0): + _wasm_global_as_extern_const = libiwasm.wasm_global_as_extern_const + _wasm_global_as_extern_const.restype = POINTER(wasm_extern_t) + _wasm_global_as_extern_const.argtypes = [POINTER(wasm_global_t)] + return _wasm_global_as_extern_const(arg0) + +def wasm_table_as_extern_const(arg0): + _wasm_table_as_extern_const = libiwasm.wasm_table_as_extern_const + _wasm_table_as_extern_const.restype = POINTER(wasm_extern_t) + _wasm_table_as_extern_const.argtypes = [POINTER(wasm_table_t)] + return _wasm_table_as_extern_const(arg0) + +def wasm_memory_as_extern_const(arg0): + _wasm_memory_as_extern_const = libiwasm.wasm_memory_as_extern_const + _wasm_memory_as_extern_const.restype = POINTER(wasm_extern_t) + _wasm_memory_as_extern_const.argtypes = [POINTER(wasm_memory_t)] + return _wasm_memory_as_extern_const(arg0) + +def wasm_extern_as_func_const(arg0): + _wasm_extern_as_func_const = libiwasm.wasm_extern_as_func_const + _wasm_extern_as_func_const.restype = POINTER(wasm_func_t) + _wasm_extern_as_func_const.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_func_const(arg0) + +def wasm_extern_as_global_const(arg0): + _wasm_extern_as_global_const = libiwasm.wasm_extern_as_global_const + _wasm_extern_as_global_const.restype = POINTER(wasm_global_t) + _wasm_extern_as_global_const.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_global_const(arg0) + +def wasm_extern_as_table_const(arg0): + _wasm_extern_as_table_const = libiwasm.wasm_extern_as_table_const + _wasm_extern_as_table_const.restype = POINTER(wasm_table_t) + _wasm_extern_as_table_const.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_table_const(arg0) + +def wasm_extern_as_memory_const(arg0): + _wasm_extern_as_memory_const = libiwasm.wasm_extern_as_memory_const + _wasm_extern_as_memory_const.restype = POINTER(wasm_memory_t) + _wasm_extern_as_memory_const.argtypes = [POINTER(wasm_extern_t)] + return _wasm_extern_as_memory_const(arg0) + +class wasm_instance_t(Structure): + pass + +def wasm_instance_delete(arg0): + _wasm_instance_delete = libiwasm.wasm_instance_delete + _wasm_instance_delete.restype = None + _wasm_instance_delete.argtypes = [POINTER(wasm_instance_t)] + return _wasm_instance_delete(arg0) + +def wasm_instance_copy(arg0): + _wasm_instance_copy = libiwasm.wasm_instance_copy + _wasm_instance_copy.restype = POINTER(wasm_instance_t) + _wasm_instance_copy.argtypes = [POINTER(wasm_instance_t)] + return _wasm_instance_copy(arg0) + +def wasm_instance_same(arg0,arg1): + _wasm_instance_same = libiwasm.wasm_instance_same + _wasm_instance_same.restype = c_bool + _wasm_instance_same.argtypes = [POINTER(wasm_instance_t),POINTER(wasm_instance_t)] + return _wasm_instance_same(arg0,arg1) + +def wasm_instance_get_host_info(arg0): + _wasm_instance_get_host_info = libiwasm.wasm_instance_get_host_info + _wasm_instance_get_host_info.restype = c_void_p + _wasm_instance_get_host_info.argtypes = [POINTER(wasm_instance_t)] + return _wasm_instance_get_host_info(arg0) + +def wasm_instance_set_host_info(arg0,arg1): + _wasm_instance_set_host_info = libiwasm.wasm_instance_set_host_info + _wasm_instance_set_host_info.restype = None + _wasm_instance_set_host_info.argtypes = [POINTER(wasm_instance_t),c_void_p] + return _wasm_instance_set_host_info(arg0,arg1) + +def wasm_instance_set_host_info_with_finalizer(arg0,arg1,arg2): + _wasm_instance_set_host_info_with_finalizer = libiwasm.wasm_instance_set_host_info_with_finalizer + _wasm_instance_set_host_info_with_finalizer.restype = None + _wasm_instance_set_host_info_with_finalizer.argtypes = [POINTER(wasm_instance_t),c_void_p,CFUNCTYPE(None,c_void_p)] + return _wasm_instance_set_host_info_with_finalizer(arg0,arg1,arg2) + +def wasm_instance_as_ref(arg0): + _wasm_instance_as_ref = libiwasm.wasm_instance_as_ref + _wasm_instance_as_ref.restype = POINTER(wasm_ref_t) + _wasm_instance_as_ref.argtypes = [POINTER(wasm_instance_t)] + return _wasm_instance_as_ref(arg0) + +def wasm_ref_as_instance(arg0): + _wasm_ref_as_instance = libiwasm.wasm_ref_as_instance + _wasm_ref_as_instance.restype = POINTER(wasm_instance_t) + _wasm_ref_as_instance.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_instance(arg0) + +def wasm_instance_as_ref_const(arg0): + _wasm_instance_as_ref_const = libiwasm.wasm_instance_as_ref_const + _wasm_instance_as_ref_const.restype = POINTER(wasm_ref_t) + _wasm_instance_as_ref_const.argtypes = [POINTER(wasm_instance_t)] + return _wasm_instance_as_ref_const(arg0) + +def wasm_ref_as_instance_const(arg0): + _wasm_ref_as_instance_const = libiwasm.wasm_ref_as_instance_const + _wasm_ref_as_instance_const.restype = POINTER(wasm_instance_t) + _wasm_ref_as_instance_const.argtypes = [POINTER(wasm_ref_t)] + return _wasm_ref_as_instance_const(arg0) + +def wasm_instance_new(arg0,arg1,arg2,arg3): + _wasm_instance_new = libiwasm.wasm_instance_new + _wasm_instance_new.restype = POINTER(wasm_instance_t) + _wasm_instance_new.argtypes = [POINTER(wasm_store_t),POINTER(wasm_module_t),POINTER(wasm_extern_vec_t),POINTER(POINTER(wasm_trap_t))] + return _wasm_instance_new(arg0,arg1,arg2,arg3) + +def wasm_instance_new_with_args(arg0,arg1,arg2,arg3,arg4,arg5): + _wasm_instance_new_with_args = libiwasm.wasm_instance_new_with_args + _wasm_instance_new_with_args.restype = POINTER(wasm_instance_t) + _wasm_instance_new_with_args.argtypes = [POINTER(wasm_store_t),POINTER(wasm_module_t),POINTER(wasm_extern_vec_t),POINTER(POINTER(wasm_trap_t)),c_uint32,c_uint32] + return _wasm_instance_new_with_args(arg0,arg1,arg2,arg3,arg4,arg5) + +class InstantiationArgs(Structure): + pass + +def wasm_instance_new_with_args_ex(arg0,arg1,arg2,arg3,arg4): + _wasm_instance_new_with_args_ex = libiwasm.wasm_instance_new_with_args_ex + _wasm_instance_new_with_args_ex.restype = POINTER(wasm_instance_t) + _wasm_instance_new_with_args_ex.argtypes = [POINTER(wasm_store_t),POINTER(wasm_module_t),POINTER(wasm_extern_vec_t),POINTER(POINTER(wasm_trap_t)),POINTER(InstantiationArgs)] + return _wasm_instance_new_with_args_ex(arg0,arg1,arg2,arg3,arg4) + +def wasm_instance_exports(arg0,arg1): + _wasm_instance_exports = libiwasm.wasm_instance_exports + _wasm_instance_exports.restype = None + _wasm_instance_exports.argtypes = [POINTER(wasm_instance_t),POINTER(wasm_extern_vec_t)] + return _wasm_instance_exports(arg0,arg1) diff --git a/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/ffi.py b/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/ffi.py new file mode 100644 index 0000000..18b6bc9 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/src/wamr/wasmcapi/ffi.py @@ -0,0 +1,642 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +import ctypes as c +import os +from pathlib import Path +import sys + +# +# Prologue. Dependencies of binding +# + +# how to open the library file of WAMR + +if sys.platform == "linux": + BUILDING_DIR = "product-mini/platforms/linux/build" + LIBRARY_NAME = "libiwasm.so" +elif sys.platform == "win32": + BUILDING_DIR = "product-mini/platforms/windows/build" + LIBRARY_NAME = "iwasm.dll" +elif sys.platform == "darwin": + BUILDING_DIR = "product-mini/platforms/darwin/build" + LIBRARY_NAME = "libiwasm.dylib" +else: + raise RuntimeError(f"unsupported platform `{sys.platform}`") + +# FIXME: should load libiwasm.so from current system library path +current_file = Path(__file__) +if current_file.is_symlink(): + current_file = Path(os.readlink(current_file)) +current_dir = current_file.parent.resolve() +root_dir = current_dir.parents[4].resolve() +wamr_dir = root_dir.resolve() +if not wamr_dir.exists(): + raise RuntimeError(f"not found the repo of wasm-micro-runtime under {root_dir}") + +libpath = wamr_dir.joinpath(BUILDING_DIR).joinpath(LIBRARY_NAME).resolve() +if not libpath.exists(): + raise RuntimeError(f"not found precompiled wamr library at {libpath}") + +print(f"loading WAMR library from {libpath} ...") +libiwasm = c.cdll.LoadLibrary(libpath) + + +class wasm_ref_t(c.Structure): + # pylint: disable=invalid-name + pass + + +class wasm_val_union(c.Union): + # pylint: disable=invalid-name + _fields_ = [ + ("i32", c.c_int32), + ("i64", c.c_int64), + ("f32", c.c_float), + ("f64", c.c_double), + ("ref", c.POINTER(wasm_ref_t)), + ] + + +class wasm_val_t(c.Structure): + # pylint: disable=invalid-name + _fields_ = [ + ("kind", c.c_uint8), + ("of", wasm_val_union), + ] + + +def dereference(p): + # pylint: disable=protected-access + if not isinstance(p, c._Pointer): + raise RuntimeError("not a pointer") + return p.contents + + +# HELPERs +def create_null_pointer(struct_type): + return c.POINTER(struct_type)() + + +def is_null_pointer(c_pointer): + # pylint: disable=protected-access + if isinstance(c_pointer, c._Pointer): + return False if c_pointer else True + else: + raise RuntimeError("not a pointer") + + +def wasm_vec_to_list(vec): + """ + Converts a vector or a POINTER(vector) to a list + vector of type pointers -> list of type pointers + """ + known_vec_type = [ + wasm_byte_vec_t, + wasm_valtype_vec_t, + wasm_functype_vec_t, + wasm_globaltype_vec_t, + wasm_tabletype_vec_t, + wasm_memorytype_vec_t, + wasm_externtype_vec_t, + wasm_importtype_vec_t, + wasm_exporttype_vec_t, + wasm_val_vec_t, + wasm_frame_vec_t, + wasm_extern_vec_t, + ] + known_vec_pointer_type = [POINTER(type) for type in known_vec_type] + + if any([isinstance(vec, type) for type in known_vec_pointer_type]): + vec = dereference(vec) + return [vec.data[i] for i in range(vec.num_elems)] + elif any([isinstance(vec, type) for type in known_vec_type]): + return [vec.data[i] for i in range(vec.num_elems)] + else: + raise RuntimeError("not a known vector type") + + +def list_to_carray(elem_type, *args): + """ + Converts a python list into a C array + """ + data = (elem_type * len(args))(*args) + return data + + +def load_module_file(wasm_content): + binary = wasm_byte_vec_t() + wasm_byte_vec_new_uninitialized(binary, len(wasm_content)) + # has to use malloced memory. + c.memmove(binary.data, wasm_content, len(wasm_content)) + binary.num_elems = len(wasm_content) + return binary + + +# +# Enhancment of binding +# + +from .binding import * + +# Built-in functions for Structure + + +wasm_finalizer = CFUNCTYPE(None, c_void_p) + + +def __repr_wasm_limits_t(self): + return f"{self.min:#x} {self.max:#x}" + + +# overwrite +wasm_limits_t.__repr__ = __repr_wasm_limits_t + + +def __compare_wasm_valtype_t(self, other): + if not isinstance(other, wasm_valtype_t): + return False + + return wasm_valtype_kind(byref(self)) == wasm_valtype_kind(byref(other)) + + +def __repr_wasm_valtype_t(self): + val_kind = wasm_valtype_kind(byref(self)) + if WASM_I32 == val_kind: + return "i32" + elif WASM_I64 == val_kind: + return "i64" + elif WASM_F32 == val_kind: + return "f32" + elif WASM_F64 == val_kind: + return "f64" + elif WASM_FUNCREF == val_kind: + return "funcref" + else: + return "anyref" + + +wasm_valtype_t.__eq__ = __compare_wasm_valtype_t +wasm_valtype_t.__repr__ = __repr_wasm_valtype_t + + +def __compare_wasm_byte_vec_t(self, other): + if not isinstance(other, wasm_byte_vec_t): + return False + + if self.num_elems != other.num_elems: + return False + + self_data = bytes(self.data[: self.num_elems]) + other_data = bytes(other.data[: other.num_elems]) + return self_data.decode() == other_data.decode() + + +def __repr_wasm_byte_vec_t(self): + data = bytes(self.data[: self.num_elems]) + return data.decode() if self.size else "" + + +wasm_byte_vec_t.__eq__ = __compare_wasm_byte_vec_t +wasm_byte_vec_t.__repr__ = __repr_wasm_byte_vec_t + + +def __compare_wasm_functype_t(self, other): + if not isinstance(other, wasm_functype_t): + return False + + params1 = dereference(wasm_functype_params(byref(self))) + params2 = dereference(wasm_functype_params(byref(other))) + results1 = dereference(wasm_functype_results(byref(self))) + results2 = dereference(wasm_functype_results(byref(other))) + return params1 == params2 and results1 == results2 + + +def __repr_wasm_functype_t(self): + params = dereference(wasm_functype_params(byref(self))) + results = dereference(wasm_functype_results(byref(self))) + params = f" (params {params})" if params.size else "" + results = f" (results {results})" if results.size else "" + return f"(func{params}{results})" + + +wasm_functype_t.__eq__ = __compare_wasm_functype_t +wasm_functype_t.__repr__ = __repr_wasm_functype_t + + +def __compare_wasm_globaltype_t(self, other): + if not isinstance(other, wasm_globaltype_t): + return False + + content1 = dereference(wasm_globaltype_content(byref(self))) + content2 = dereference(wasm_globaltype_content(byref(other))) + mutability1 = wasm_globaltype_mutability(byref(self)) + mutability2 = wasm_globaltype_mutability(byref(other)) + return content1 == content2 and mutability1 == mutability2 + + +def __repr_wasm_globaltype_t(self): + mutability = f"{wasm_globaltype_mutability(byref(self))}" + content = f"{dereference(wasm_globaltype_content(byref(self)))}" + return f"(global{' mut ' if mutability else ' '}{content})" + + +wasm_globaltype_t.__eq__ = __compare_wasm_globaltype_t +wasm_globaltype_t.__repr__ = __repr_wasm_globaltype_t + + +def __compare_wasm_tabletype_t(self, other): + if not isinstance(other, wasm_tabletype_t): + return False + + element1 = dereference(wasm_tabletype_element(byref(self))) + element2 = dereference(wasm_tabletype_element(byref(other))) + limits1 = dereference(wasm_tabletype_limits(byref(self))) + limits2 = dereference(wasm_tabletype_limits(byref(other))) + return element1 == element2 and limits1 == limits2 + + +def __repr_wasm_tabletype_t(self): + element = dereference(wasm_tabletype_element(byref(self))) + limit = dereference(wasm_tabletype_limits(byref(self))) + return f"(table {limit} {element})" + + +wasm_tabletype_t.__eq__ = __compare_wasm_tabletype_t +wasm_tabletype_t.__repr__ = __repr_wasm_tabletype_t + + +def __compare_wasm_memorytype_t(self, other): + if not isinstance(other, wasm_memorytype_t): + return False + + limits1 = dereference(wasm_memorytype_limits(byref(self))) + limits2 = dereference(wasm_memorytype_limits(byref(other))) + return limits1 == limits2 + + +def __repr_wasm_memorytype_t(self): + limit = dereference(wasm_memorytype_limits(byref(self))) + return f"(memory {limit})" + + +wasm_memorytype_t.__eq__ = __compare_wasm_memorytype_t +wasm_memorytype_t.__repr__ = __repr_wasm_memorytype_t + + +def __compare_wasm_externtype_t(self, other): + if not isinstance(other, wasm_externtype_t): + return False + + if wasm_externtype_kind(byref(self)) != wasm_externtype_kind(byref(other)): + return False + + extern_kind = wasm_externtype_kind(byref(self)) + if WASM_EXTERN_FUNC == extern_kind: + return dereference(wasm_externtype_as_functype(self)) == dereference( + wasm_externtype_as_functype(other) + ) + elif WASM_EXTERN_GLOBAL == extern_kind: + return dereference(wasm_externtype_as_globaltype(self)) == dereference( + wasm_externtype_as_globaltype(other) + ) + elif WASM_EXTERN_MEMORY == extern_kind: + return dereference(wasm_externtype_as_memorytype(self)) == dereference( + wasm_externtype_as_memorytype(other) + ) + elif WASM_EXTERN_TABLE == extern_kind: + return dereference(wasm_externtype_as_tabletype(self)) == dereference( + wasm_externtype_as_tabletype(other) + ) + else: + raise RuntimeError("not a valid wasm_externtype_t") + + +def __repr_wasm_externtype_t(self): + extern_kind = wasm_externtype_kind(byref(self)) + if WASM_EXTERN_FUNC == extern_kind: + return str(dereference(wasm_externtype_as_functype(byref(self)))) + elif WASM_EXTERN_GLOBAL == extern_kind: + return str(dereference(wasm_externtype_as_globaltype(byref(self)))) + elif WASM_EXTERN_MEMORY == extern_kind: + return str(dereference(wasm_externtype_as_memorytype(byref(self)))) + elif WASM_EXTERN_TABLE == extern_kind: + return str(dereference(wasm_externtype_as_tabletype(byref(self)))) + else: + raise RuntimeError("not a valid wasm_externtype_t") + + +wasm_externtype_t.__eq__ = __compare_wasm_externtype_t +wasm_externtype_t.__repr__ = __repr_wasm_externtype_t + + +def __compare_wasm_importtype_t(self, other): + if not isinstance(other, wasm_importtype_t): + return False + + if dereference(wasm_importtype_module(self)) != dereference( + wasm_importtype_module(other) + ): + return False + + if dereference(wasm_importtype_name(self)) != dereference( + wasm_importtype_name(other) + ): + return False + + self_type = dereference(wasm_importtype_type(byref(self))) + other_type = dereference(wasm_importtype_type(byref(other))) + return self_type == other_type + + +def __repr_wasm_importtype_t(self): + module = wasm_importtype_module(byref(self)) + name = wasm_importtype_name(byref(self)) + extern_type = wasm_importtype_type(byref(self)) + return f'(import "{dereference(module)}" "{dereference(name)}" {dereference(extern_type)})' + + +wasm_importtype_t.__eq__ = __compare_wasm_importtype_t +wasm_importtype_t.__repr__ = __repr_wasm_importtype_t + + +def __compare_wasm_exporttype_t(self, other): + if not isinstance(other, wasm_exporttype_t): + return False + + self_name = dereference(wasm_exporttype_name(byref(self))) + other_name = dereference(wasm_exporttype_name(byref(other))) + if self_name != other_name: + return False + + self_type = dereference(wasm_exporttype_type(byref(self))) + other_type = dereference(wasm_exporttype_type(byref(other))) + return self_type == other_type + + +def __repr_wasm_exporttype_t(self): + name = wasm_exporttype_name(byref(self)) + extern_type = wasm_exporttype_type(byref(self)) + return f'(export "{dereference(name)}" {dereference(extern_type)})' + + +wasm_exporttype_t.__eq__ = __compare_wasm_exporttype_t +wasm_exporttype_t.__repr__ = __repr_wasm_exporttype_t + + +def __compare_wasm_val_t(self, other): + if not isinstance(other, wasm_val_t): + return False + + if self.kind != other.kind: + return False + + if WASM_I32 == self.kind: + return self.of.i32 == other.of.i32 + elif WASM_I64 == self.kind: + return self.of.i64 == other.of.i64 + elif WASM_F32 == self.kind: + return self.of.f32 == other.of.f32 + elif WASM_F64 == self.kind: + return self.of.f64 == other.of.f63 + elif WASM_ANYREF == self.kind: + raise RuntimeError("FIXME") + else: + raise RuntimeError("not a valid val kind") + + +def __repr_wasm_val_t(self): + if WASM_I32 == self.kind: + return f"i32 {self.of.i32}" + elif WASM_I64 == self.kind: + return f"i64 {self.of.i64}" + elif WASM_F32 == self.kind: + return f"f32 {self.of.f32}" + elif WASM_F64 == self.kind: + return f"f64 {self.of.f64}" + elif WASM_ANYREF == self.kind: + return f"anyref {self.of.ref}" + else: + raise RuntimeError("not a valid val kind") + + +wasm_val_t.__repr__ = __repr_wasm_val_t +wasm_val_t.__eq__ = __compare_wasm_val_t + + +def __repr_wasm_trap_t(self): + message = wasm_message_t() + wasm_trap_message(self, message) + return f'(trap "{str(message)}")' + + +wasm_trap_t.__repr__ = __repr_wasm_trap_t + + +def __repr_wasm_frame_t(self): + instance = wasm_frame_instance(self) + module_offset = wasm_frame_module_offset(self) + func_index = wasm_frame_func_index(self) + func_offset = wasm_frame_func_offset(self) + return f"> module:{module_offset:#x} => func#{func_index:#x}.{func_offset:#x}" + + +wasm_frame_t.__repr__ = __repr_wasm_frame_t + + +def __repr_wasm_module_t(self): + imports = wasm_importtype_vec_t() + wasm_module_imports(self, imports) + + exports = wasm_exporttype_vec_t() + wasm_module_exports(self, exports) + + ret = "(module" + ret += str(imports).replace("(import", "\n (import") + ret += str(exports).replace("(export", "\n (export") + ret += "\n)" + return ret + + +wasm_module_t.__repr__ = __repr_wasm_module_t + + +def __repr_wasm_instance_t(self): + exports = wasm_extern_vec_t() + wasm_instance_exports(self, exports) + + ret = "(instance" + ret += str(exports).replace("(export", "\n (export") + ret += "\n)" + return ret + + +wasm_instance_t.__repr__ = __repr_wasm_instance_t + + +def __repr_wasm_func_t(self): + ft = wasm_func_type(self) + return f"{str(dereference(ft))[:-1]} ... )" + + +wasm_func_t.__repr__ = __repr_wasm_func_t + + +def __repr_wasm_global_t(self): + gt = wasm_global_type(self) + return f"{str(dereference(gt))[:-1]} ... )" + + +wasm_global_t.__repr__ = __repr_wasm_global_t + + +def __repr_wasm_table_t(self): + tt = wasm_table_type(self) + return f"{str(dereference(tt))[:-1]} ... )" + + +wasm_table_t.__repr__ = __repr_wasm_table_t + + +def __repr_wasm_memory_t(self): + mt = wasm_memory_type(self) + return f"{str(dereference(mt))[:-1]} ... )" + + +wasm_memory_t.__repr__ = __repr_wasm_memory_t + + +def __repr_wasm_extern_t(self): + ext_type = wasm_extern_type(self) + ext_kind = wasm_extern_kind(self) + + ret = "(export " + if WASM_EXTERN_FUNC == ext_kind: + ft = wasm_externtype_as_functype(ext_type) + ret += str(dereference(ft)) + elif WASM_EXTERN_GLOBAL == ext_kind: + gt = wasm_externtype_as_globaltype(ext_type) + ret += str(dereference(gt)) + elif WASM_EXTERN_MEMORY == ext_kind: + mt = wasm_externtype_as_memorytype(ext_type) + ret += str(dereference(mt)) + elif WASM_EXTERN_TABLE == ext_kind: + tt = wasm_externtype_as_tabletype(ext_type) + ret += str(dereference(tt)) + else: + raise RuntimeError("not a valid extern kind") + ret += ")" + return ret + + +wasm_extern_t.__repr__ = __repr_wasm_extern_t + + +# Function Types construction short-hands +def wasm_name_new_from_string(s): + name = wasm_name_t() + data = ((c.c_ubyte) * len(s)).from_buffer_copy(s.encode()) + wasm_byte_vec_new(byref(name), len(s), data) + return name + + +def __wasm_functype_new(param_list, result_list): + def __list_to_wasm_valtype_vec(l): + vec = wasm_valtype_vec_t() + + if not l: + wasm_valtype_vec_new_empty(byref(vec)) + else: + data_type = POINTER(wasm_valtype_t) * len(l) + data = data_type() + for i in range(len(l)): + data[i] = l[i] + wasm_valtype_vec_new(byref(vec), len(l), data) + + return vec + + params = __list_to_wasm_valtype_vec(param_list) + results = __list_to_wasm_valtype_vec(result_list) + return wasm_functype_new(byref(params), byref(results)) + + +def wasm_functype_new_0_0(): + return __wasm_functype_new([], []) + + +def wasm_functype_new_1_0(p1): + return __wasm_functype_new([p1], []) + + +def wasm_functype_new_2_0(p1, p2): + return __wasm_functype_new([p1, p2], []) + + +def wasm_functype_new_3_0(p1, p2, p3): + return __wasm_functype_new([p1, p2, p3], []) + + +def wasm_functype_new_0_1(r1): + return __wasm_functype_new([], [r1]) + + +def wasm_functype_new_1_1(p1, r1): + return __wasm_functype_new([p1], [r1]) + + +def wasm_functype_new_2_1(p1, p2, r1): + return __wasm_functype_new([p1, p2], [r1]) + + +def wasm_functype_new_3_1(p1, p2, p3, r1): + return __wasm_functype_new([p1, p2, p3], [r1]) + + +def wasm_limits_new(min, max): + limit = wasm_limits_t() + limit.min = min + limit.max = max + return c.pointer(limit) + + +def wasm_i32_val(i): + v = wasm_val_t() + v.kind = WASM_I32 + v.of.i32 = i + return v + + +def wasm_i64_val(i): + v = wasm_val_t() + v.kind = WASM_I64 + v.of.i64 = i + return v + + +def wasm_f32_val(z): + v = wasm_val_t() + v.kind = WASM_F32 + v.of.f32 = z + return v + + +def wasm_f64_val(z): + v = wasm_val_t() + v.kind = WASM_F64 + v.of.f64 = z + return v + + +def wasm_func_cb_decl(func): + return wasm_func_callback_t(func) + + +def wasm_func_with_env_cb_decl(func): + return wasm_func_callback_with_env_t(func) diff --git a/wasm-micro-runtime/language-bindings/python/utils/create_lib.sh b/wasm-micro-runtime/language-bindings/python/utils/create_lib.sh new file mode 100755 index 0000000..801186e --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/utils/create_lib.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$(cd $(dirname $0) && pwd -P) +ROOT_DIR=${CUR_DIR}/../../.. + +UNAME=$(uname -s|tr A-Z a-z) +WAMR_BUILD_PLATFORM=${WAMR_BUILD_PLATFORM:-${UNAME}} + +cd ${ROOT_DIR}/product-mini/platforms/${WAMR_BUILD_PLATFORM} + +mkdir -p build && cd build +cmake \ + -DWAMR_BUILD_DEBUG_INTERP=1 \ + -DWAMR_BUILD_LIB_PTHREAD=1 \ + -DWAMR_BUILD_LIB_WASI_THREADS=1 \ + -DWAMR_BUILD_LIB_WASI=1 \ + .. +make -j + +case ${UNAME} in +darwin) + LIBNAME=libiwasm.dylib + ;; +*) + LIBNAME=libiwasm.so + ;; +esac +cp ${LIBNAME} ${CUR_DIR}/../src/wamr/libs + +cd ${ROOT_DIR}/language-bindings/python/src/wamr/wamrapi +ctypesgen \ +${ROOT_DIR}/core/iwasm/include/wasm_export.h \ +-l ../libs/${LIBNAME} \ +-o iwasm.py diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/README.md b/wasm-micro-runtime/language-bindings/python/wamr-api/README.md new file mode 100644 index 0000000..38a4401 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/README.md @@ -0,0 +1,29 @@ +# WARM API + +* **Notice**: The python package `wamr.wamrapi.wamr` need python >= `3.10`. + +## Setup + +### Pre-requisites + +Install requirements, + +```shell +pip install -r requirements.txt +``` + +### Build native lib and update bindings + +The following command builds the iwasm library and generates the Python bindings, + +```sh +# In WAMR root directory +bash language-bindings/python/utils/create_lib.sh +``` + +This will build and copy libiwasm into the package. + +## Samples + +- **[basic](./samples/basic)**: Demonstrating how to use basic python bindings. +- **[native-symbol](./samples/native-symbol)**: Desmostrate how to call WASM from Python and how to export Python functions into WASM. diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/requirements.txt b/wasm-micro-runtime/language-bindings/python/wamr-api/requirements.txt new file mode 100644 index 0000000..923575a --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/requirements.txt @@ -0,0 +1 @@ +ctypesgen==1.1.1 \ No newline at end of file diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/compile.sh b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/compile.sh new file mode 100755 index 0000000..4745ef1 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/compile.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/opt/wasi-sdk/bin/clang \ + -O0 -z stack-size=4096 -Wl,--initial-memory=65536 \ + -Wl,--strip-all,--no-entry -nostdlib \ + -Wl,--export=sum\ + -Wl,--allow-undefined \ + -o sum.wasm sum.c diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/main.py b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/main.py new file mode 100644 index 0000000..525aab8 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/main.py @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from wamr.wamrapi.wamr import Engine, Module, Instance, ExecEnv +from ctypes import c_uint +import pathlib + +def main(): + engine = Engine() + module = Module.from_file(engine, pathlib.Path(__file__).parent / "sum.wasm") + module_inst = Instance(module) + exec_env = ExecEnv(module_inst) + + func = module_inst.lookup_function("sum") + + argv = (c_uint * 2)(*[10, 11]) + exec_env.call(func, len(argv), argv) + print(argv[0]) + + +if __name__ == "__main__": + main() diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/sum.c b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/sum.c new file mode 100644 index 0000000..586c5bc --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/basic/sum.c @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +int +sum(int a, int b) +{ + return a + b; +} diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/README.md b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/README.md new file mode 100644 index 0000000..75d4fab --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/README.md @@ -0,0 +1,44 @@ +# Native Symbol + +This sample demonstrates how to declare a Python function as `NativeSymbol`. + +Steps of the example: +1. Load WASM from Python +2. Call `c_func` from WASM. +3. `c_func` calls `python_func` from Python. +4. `python_func` calls `add` from WASM. +5. Result shown by Python. + +## Build + +Follow instructions [build wamr Python package](../../README.md). + +Compile WASM app example, + +```sh +./compile.sh +``` + +## Run sample + +```sh +python main.py +``` + +Output: + +``` +python: calling c_func(10) +c: in c_func with input: 10 +c: calling python_func(11) +python: in python_func with input: 11 +python: calling add(11, 1000) +python: result from add: 1011 +c: result from python_func: 1012 +c: returning 1013 +python: result from c_func: 1013 +deleting ExecEnv +deleting Instance +deleting Module +deleting Engine +``` diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/compile.sh b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/compile.sh new file mode 100755 index 0000000..4e504b0 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/compile.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/opt/wasi-sdk/bin/clang \ + -O0 -z stack-size=4096 -Wl,--initial-memory=65536 \ + -Wl,--export=main -Wl,--export=__main_argc_argv \ + -Wl,--export=__data_end -Wl,--export=__heap_base \ + -Wl,--strip-all,--no-entry \ + -Wl,--allow-undefined \ + -Wl,--export=c_func\ + -Wl,--export=add\ + -o func.wasm func.c diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/func.c b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/func.c new file mode 100644 index 0000000..4411a6b --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/func.c @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +int +python_func(int val); + +int +add(int val1, int val2) +{ + return val1 + val2; +} + +int +c_func(int val) +{ + printf("c: in c_func with input: %d\n", val); + printf("c: calling python_func(%d)\n", val + 1); + int res = python_func(val + 1); + printf("c: result from python_func: %d\n", res); + printf("c: returning %d\n", res + 1); + return res + 1; +} + +int +main() +{} diff --git a/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/main.py b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/main.py new file mode 100644 index 0000000..3871ef1 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wamr-api/samples/native-symbol/main.py @@ -0,0 +1,59 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from wamr.wamrapi.wamr import Engine, Module, Instance, ExecEnv +from ctypes import c_uint +import pathlib +from ctypes import c_int32 +from ctypes import c_uint +from ctypes import c_void_p +from ctypes import cast +from ctypes import CFUNCTYPE + +from wamr.wamrapi.iwasm import NativeSymbol +from wamr.wamrapi.iwasm import String +from wamr.wamrapi.wamr import ExecEnv + +def python_func(env: int, value: int) -> int: + print("python: in python_func with input:", value) + # Example of generating ExecEnv from `wasm_exec_env_t`` + exec_env = ExecEnv.wrap(env) + add = exec_env.get_module_inst().lookup_function("add") + const = 1000 + argv = (c_uint * 2)(value, const) + print(f"python: calling add({value}, {const})") + exec_env.call(add, 2, argv) + res = argv[0] + print("python: result from add:", res) + return res + 1 + + +native_symbols = (NativeSymbol * 1)( + *[ + NativeSymbol( + symbol=String.from_param("python_func"), + func_ptr=cast( + CFUNCTYPE(c_int32, c_void_p, c_int32)(python_func), c_void_p + ), + signature=String.from_param("(i)i"), + ) + ] +) + +def main(): + engine = Engine() + engine.register_natives("env", native_symbols) + module = Module.from_file(engine, pathlib.Path(__file__).parent / "func.wasm") + module_inst = Instance(module) + exec_env = ExecEnv(module_inst) + + func = module_inst.lookup_function("c_func") + + inp = 10 + print(f"python: calling c_func({inp})") + argv = (c_uint)(inp) + exec_env.call(func, 1, argv) + print("python: result from c_func:", argv.value) + +if __name__ == "__main__": + main() diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/README.md b/wasm-micro-runtime/language-bindings/python/wasm-c-api/README.md new file mode 100644 index 0000000..1684afa --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/README.md @@ -0,0 +1,7 @@ +# WASM-C-API + +## Examples + +There is a [simple example](./samples/hello_procedural.py) to show how to use bindings. Actually, the python binding follows C-APIs. There it should be easy if be familiar with _programming with wasm-c-api_. + +Unit test cases under _./tests_ could be another but more complete references. diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/design.md b/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/design.md new file mode 100644 index 0000000..78bf56d --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/design.md @@ -0,0 +1,709 @@ +# how to implement a python binding of WAMR + +A python language binding of Wasm runtime allows its users to call a set of APIs of +the runtime from the python world. Those APIs maybe implemented in C, C++, or Rust. + +In the WAMR case, a python binding allows APIs in `core/iwasm/include/wasm_c_api.h` +to be used in the python scripts. To achieve that, we will create two kinds +of stuff: wrappers of structured data types and wrappers of functions under the +help of _ctypes_. + +Cyptes is a tool in the standard library for creating Python bindings. It +provides a low-level toolset for loading shared libraries and marshaling +data between Python and C. Other options include _cffi_, _pybind11_, +_cpython_ and so on. Because we tend to make the binding depending on least +items. The built-in module, _ctypes_, is a good choice. + +## General rules to marshal + +The core of the idea of a language binding is how to translate different +representations of types in different language. + +### load libraries + +The `ctypes` supports locating a dynamic link library in a way similar to the +compiler does. + +Currently, `ctypes.LoadLibrary` supports: + +- `CDLL`. Those libraries use the standard C calling conversion. +- `OleDLL` and `WinDLL`. Those libraries use the `stdcall` calling conversion on + Windows only + +### fundamental datatypes + +_ctypes_ provides [primitive C compatiable data types](https://docs.python.org/3/library/ctypes.html#fundamental-data-types). +Like `c_bool`, `c_byte`, `c_int`, `c_long` and so on. + +> `c_int` represents the _C_ `signed int` datatype. On platforms where +> `sizeof(int) == sizeof(long)` it is an alias to `c_long`. + +| c datatypes | ctypes | +| ------------------- | ----------------------- | +| bool | c_bool | +| byte_t | c_ubyte | +| char | c_char | +| float32_t | c_float | +| float64_t | c_double | +| int32_t | c_int32 | +| int64_t | c_int64 | +| intptr_t | c_void_p | +| size_t | c_size_t | +| uint8_t | c_uint8 | +| uint32_t | c_uint32 | +| void | None | +| wasm_byte_t | c_ubyte | +| wasm_externkind_t | c_uint8 | +| wasm_memory_pages_t | c_uint32 | +| wasm_mutability_t | c_bool | +| wasm_table_size_t | c_uint32 | +| wasm_valkind_t | c_uint8 | +| wasm_data_type\* | POINTER(wasm_data_type) | + +- `c_void_p` only represents `void *` only +- `None` represents `void` in function parameter lists and return lists + +### structured datatypes + +Create a corresponding concept for every native structured data type includes +`enum`, `struct` and `union`, in the python world. + +#### Enum types + +For example, if there is a `enum wams_mutability_enum` in native. + +```c +typedef uint8_t wams_mutability_t; +enum wams_mutability_enum { + WASM_CONST, + WASM_VAR +}; +``` + +Use `ctypes.int`(or any integer types in ctypes) to represents its value directly. + +```python +# represents enum wams_mutability_enum +wasm_mutability_t = c_uint8 + +WASM_CONST = 0 +WASM_VAR = 1 +``` + +> C standard only requires "Each enumerated type shall be compatible with char, +> a signed integer type, or an unsigned integer type. The choice of the integer +> type is implementation-defined, but shall be capable of representing the +> values of all the members of the enumeration. + +#### Struct types + +If there is a `struct wasm_byte_vec_t` in native(in C). + +```c +typedef struct wasm_byte_vec_t { + size_t size; + wasm_byte_t *data; + size_t num_elems; + size_t size_of_elem; +} wasm_byte_vec_t; +``` + +Use `ctypes.Structure` to create its corresponding data type in python. + +```python +class wasm_byte_vec_t(ctypes.Structure): + _fileds_ = [ + ("size", ctypes.c_size_t), + ("data", ctypes.POINTER(c_ubyte)), + ("num_elems", ctypes.c_size_t), + ("size_of_elem", ctypes.c_size_t), + ] +``` + +a list of `Structures` + +| name | +| ----------------- | +| wasm_engine_t | +| wasm_store_t | +| wasm_limits_t | +| wasm_valtype_t | +| wasm_functype_t | +| wasm_globaltype_t | +| wasm_tabletype_t | +| wasm_memorytype_t | +| wasm_externtype_t | +| wasm_importtype_t | +| wasm_exporttype_t | +| wasm_ref_t | +| wasm_ref_t | +| wasm_frame_t | +| wasm_trap_t | +| wasm_foreign_t | +| WASMModuleCommon | +| WASMModuleCommon | +| wasm_func_t | +| wasm_global_t | +| wasm_table_t | +| wasm_memory_t | +| wasm_extern_t | +| wasm_instance_t | + +not supported `struct` + +- wasm_config_t + +If there is an anonymous `union` in native. + +```c +typedef struct wasm_val_t { + wasm_valkind_t kind; + union { + int32_t i32; + int64_t i64; + float32_t f32; + float64_t f64; + } of; +} wasm_val_t; +``` + +Use `ctypes.Union` to create its corresponding data type in python. + +```python +class _OF(ctypes.Union): + _fields_ = [ + ("i32", ctypes.c_int32), + ("i64", ctypes.c_int64), + ("f32", ctypes.c_float), + ("f64", ctypes.c_double), + ] + +class wasm_val_t(ctypes.Structure): + _anonymous_ = ("of",) + _fields_ = [ + ("kind", ctypes.c_uint8) + ("of", _OF) + ] +``` + +### wrappers of functions + +Foreign functions (C functions) can be accessed as attributes of loaded shared +libraries or an instance of function prototypes. Callback functions(python +functions) can only be accessed by instantiating function prototypes. + +For example, + +```c +void wasm_name_new(wasm_name_t* out, size_t len, wasm_byte_t [] data); +``` + +Assume there are: + +- `class wasm_name_t` of python represents `wasm_name_t` of C +- `libiwasm` represents loaded _libiwasm.so_ + +If to access a c function like an attribute, + +```python +def wasm_name_new(out, len, data): + _wasm_name_new = libiwasm.wasm_name_new + _wasm_name_new.argtypes = (ctypes.POINTER(wasm_name_t), ctypes.c_size_t, ctypes.POINTER(ctypes.c_ubyte)) + _wasm_name_new.restype = None + return _wasm_name_new(out, len, data) +``` + +Or to instantiate a function prototype, + +```python +def wasm_name_new(out, len, data): + return ctypes.CFUNCTYPE(None, (ctypes.POINTER(wasm_name_t), ctypes.c_size_t, ctypes.POINTER(ctypes.c_ubyte)))( + ("wasm_name_new", libiwasm), out, len, data) +``` + +Now it is able to create a `wasm_name_t` with `wasm_name_new()` in python. + +Sometimes, need to create a python function as a callback of c. + +```c +wasm_trap_t* (*wasm_func_callback_t)(wasm_val_vec_t* args, wasm_val_vec_t *results); +``` + +Use `cyptes.CFUNCTYPE` to create a _pointer of function_ + +```python +def hello(args, results): + print("hello from a callback") + +wasm_func_callback_t = ctypes.CFUNCTYPE(c_size_t, POINTER(wasm_val_vec_t), POINTER(wasm_val_vec_t)) +hello_callback = wasm_func_callback_t(hello) +``` + +or with a decorator + +```python +def wasm_func_cb_decl(func): + return @ctypes.CFUNCTYPE(ctypes.POINTER(wasm_trap_t), (ctypes.POINTER(wasm_val_vec_t), ctypes.POINTER(wasm_val_vec_t)))(func) + +@wasm_func_cb_decl +def hello(args, results): + print("hello from a callback") +``` + +### programming tips + +#### `struct` and `ctypes.Structure` + +There are two kinds of `cytes.Structure` in `binding.py`. + +- has `__field__` definition. like `class wasm_byte_vec_t(Structure)` +- doesn't have `__field__` definition. like `class wasm_config_t(Structure)` + +Since, `ctypes` will create its C world _mirror_ variable according to `__field__` +information, `wasm_config_t()` will only create a python instance without binding +to any C variable. `wasm_byte_vec_t()` will return a python instance with an internal +C variable. + +That is why `pointer(wasm_config_t())` is a NULL pointer which can not be dereferenced. + +#### deal with pointers + +`byref()` and `pointer()` are two functions can return a pointer. + +```python +x = ctypes.c_int(2) + +# use pointer() to creates a new pointer instance which would later be used in Python +x_ptr = ctypes.pointer(x) +... +struct_use_pointer = Mystruct() +struct_use_pointer.ptr = x_ptr + +# use byref() pass a pointer to an object to a foreign function call +func(ctypes.byref(x)) +``` + +The main difference is that `pointer()` does a lot more work since it +constructs a real pointer object. It is faster to use `byref(`) if don't need +the pointer object in Python itself(e.g. only use it as an argument to pass +to a function). + +There is no doubt that `wasm_xxx_new()` which return type is `ctypes.POINTER` +can return a pointer. Plus, the return value of `wasm_xxx_t()` can also be +used as a pointer without casting by `byref` or `pointer`. + +#### array + +In [ctypes document](https://docs.python.org/3/library/ctypes.html#arrays), +it states that "The recommended way to create array types is by multiplying a +data type with a positive integer". So _multiplying a data type_ should be a +better way to create arrays + +```python +from ctypes import * + +class POINT(Structure): + _fields_ = ("x", c_int), ("y", c_int) + +# multiplying a data type +# type(TenPointsArrayType) is +TenPointsArrayType = POINT * 10 + +# Instances are created in the usual way, by calling the class: +arr = TenPointsArrayType() +arr[0] = POINT(3,2) +for pt in arr: + print(pt.x, pt.y) +``` + +On both sides, it is OK to assign an array to a pointer. + +```c +char buf[128] = {0}; +char *ptr = buf; +``` + +```python +binary = wasm_byte_vec_t() +binary.data = (ctypes.c_ubyte * len(wasm)).from_buffer_copy(wasm) +``` + +#### exceptions and traps + +Interfaces of _wasm-c-api_ have their return values to represent failures. +The python binding should just keep and transfer them to callers instead of +raising any additional exception. + +The python binding should raise exceptions when the python partial is failed. + +#### readonly buffer + +```python +with open("hello.wasm", "rb") as f: + wasm = f.read() + binary = wasm_byte_vec_t() + wasm_byte_vec_new_uninitialized(byref(binary), len(wasm)) + # create a ctypes instance (byte[] in c) and copy the content + # from wasm(bytearray in python) + binary.data = (ctypes.c_ubyte * len(wasm)).from_buffer_copy(wasm) +``` + +in the above example, `wasm` is a python-created readable buffer. It is not +writable and needs to be copied into a ctype array. + +#### variable arguments + +A function with _variable arugments_ makes it hard to specify the required +argument types for the function prototype. It leaves us one way to call it +directly without any arguments type checking. + +```python +libc.printf(b"Hello, an int %d, a float %f, a string %s\n", c_int(1), c_doulbe(3.14), "World!") +``` + +#### Use `c_bool` to represent `wasm_mutability_t ` + +- `True` for `WASM_CONST` +- `False` for `WASM_VALUE` + +#### customize class builtins + +- `__eq__` for comparation. +- `__repr__` for printing. + +### bindgen.py + +`bindge.py` is a tool to create WAMR python binding automatically. `binding.py` +is generated. We should avoid modification on it. Additional helpers should go +to `ffi.py`. + +`bindgen.py` uses _pycparser_. Visit the AST of `core/iwasm/include/wasm_c_api.h` +created by _gcc_ and generate necessary wrappers. + +```python +from pycparser import c_ast + +class Visitor(c_ast.NodeVisitor): + def visit_Struct(self, node): + pass + + def visit_Union(self, node): + pass + + def visit_TypeDef(self, node): + pass + + def visit_FuncDecl(self, node): + pass + +ast = parse_file(...) +v = Visitor() +v.visit(ast) +``` + +Before running _bindgen.py_, the shared library _libiwasm.so_ should be generated. + +```bash +$ cd /path/to/wamr/repo +$ # if it is in linux +$ pushd product-mini/platforms/linux/ +$ cmake -S . -B build .. +$ cmake --build build --target iwasm +$ popd +$ cd binding/python +$ python utils/bindgen.py +``` + +`wasm_frame_xxx` and `wasm_trap_xxx` only work well when enabling `WAMR_BUILD_DUMP_CALL_STACK`. + +```bash +$ cmake -S . -B build -DWAMR_BUILD_DUMP_CALL_STACK=1 .. +``` + +## OOP wrappers + +Based on the above general rules, there will be corresponding python +APIs for every C API in `wasm_c_api.h` with same name. Users can do procedural +programming with those. + +In next phase, we will create OOP APIs. Almost follow the +[C++ version of wasm_c_api](https://github.com/WebAssembly/wasm-c-api/blob/master/include/wasm.hh) + +## A big list + +| WASM Concept | Procedural APIs | OOP APIs | OOP APIs methods | +| ------------ | -------------------------------- | ---------- | ---------------- | +| XXX_vec | wasm_xxx_vec_new | | list | +| | wasm_xxx_vec_new_uninitialized | | | +| | wasm_xxx_vec_new_empty | | | +| | wasm_xxx_vec_copy | | | +| | wasm_xxx_vec_delete | | | +| valtype | wasm_valtype_new | valtype | \_\_init\_\_ | +| | wasm_valtype_delete | | \_\_del\_\_ | +| | wasm_valtype_kind | | \_\_eq\_\_ | +| | wasm_valtype_copy | | | +| | _vector methods_ | | | +| functype | wasm_functype_new | functype | | +| | wasm_functype_delete | | | +| | wasm_functype_params | | | +| | wasm_functype_results | | | +| | wasm_functype_copy | | | +| | _vector methods_ | | | +| globaltype | wasm_globaltype_new | globaltype | \_\_init\_\_ | +| | wasm_globaltype_delete | | \_\_del\_\_ | +| | wasm_globaltype_content | | \_\_eq\_\_ | +| | wasm_globaltype_mutability | | | +| | wasm_globaltype_copy | | | +| | _vector methods_ | | | +| tabletype | wasm_tabletype_new | tabletype | \_\_init\_\_ | +| | wasm_tabletype_delete | | \_\_del\_\_ | +| | wasm_tabletype_element | | \_\_eq\_\_ | +| | wasm_tabletype_limits | | | +| | wasm_tabletype_copy | | | +| | _vector methods_ | | | +| memorytype | wasm_memorytype_new | memorytype | \_\_init\_\_ | +| | wasm_memorytype_delete | | \_\_del\_\_ | +| | wasm_memorytype_limits | | \_\_eq\_\_ | +| | wasm_memorytype_copy | | | +| | _vector methods_ | | | +| externtype | wasm_externtype_as_XXX | externtype | | +| | wasm_XXX_as_externtype | | | +| | wasm_externtype_copy | | | +| | wasm_externtype_delete | | | +| | wasm_externtype_kind | | | +| | _vector methods_ | | | +| importtype | wasm_importtype_new | importtype | | +| | wasm_importtype_delete | | | +| | wasm_importtype_module | | | +| | wasm_importtype_name | | | +| | wasm_importtype_type | | | +| | wasm_importtype_copy | | | +| | _vector methods_ | | | +| exportype | wasm_exporttype_new | exporttype | | +| | wasm_exporttype_delete | | | +| | wasm_exporttype_name | | | +| | wasm_exporttype_type | | | +| | wasm_exporttype_copy | | | +| | _vector methods_ | | | +| val | wasm_val_delete | val | | +| | wasm_val_copy | | | +| | _vector methods_ | | | +| frame | wasm_frame_delete | frame | | +| | wasm_frame_instance | | | +| | wasm_frame_func_index | | | +| | wasm_frame_func_offset | | | +| | wasm_frame_module_offset | | | +| | wasm_frame_copy | | | +| | _vector methods_ | | | +| trap | wasm_trap_new | trap | | +| | wasm_trap_delete | | | +| | wasm_trap_message | | | +| | wasm_trap_origin | | | +| | wasm_trap_trace | | | +| | _vector methods_ | | | +| foreign | wasm_foreign_new | foreign | | +| | wasm_foreign_delete | | | +| | _vector methods_ | | | +| engine | wasm_engine_new | engine | | +| | wasm_engine_new_with_args\* | | | +| | wasm_engine_new_with_config | | | +| | wasm_engine_delete | | | +| store | wasm_store_new | store | | +| | wasm_store_delete | | | +| | _vector methods_ | | | +| module | wasm_module_new | module | | +| | wasm_module_delete | | | +| | wasm_module_validate | | | +| | wasm_module_imports | | | +| | wasm_module_exports | | | +| instance | wasm_instance_new | instance | | +| | wasm_instance_delete | | | +| | wasm_instance_new_with_args\* | | | +| | wasm_instance_new_with_args_ex\* | | | +| | wasm_instance_exports | | | +| | _vector methods_ | | | +| func | wasm_func_new | func | | +| | wasm_func_new_with_env | | | +| | wasm_func_delete | | | +| | wasm_func_type | | | +| | wasm_func_call | | | +| | wasm_func_param_arity | | | +| | wasm_func_result_arity | | | +| | _vector methods_ | | | +| global | wasm_global_new | global | | +| | wasm_global_delete | | | +| | wasm_global_type | | | +| | wasm_global_get | | | +| | wasm_global_set | | | +| | _vector methods_ | | | +| table | wasm_table_new | table | | +| | wasm_table_delete | | | +| | wasm_table_type | | | +| | wasm_table_get | | | +| | wasm_table_set | | | +| | wasm_table_size | | | +| | _vector methods_ | | | +| memory | wasm_memory_new | memory | | +| | wasm_memory_delete | | | +| | wasm_memory_type | | | +| | wasm_memory_data | | | +| | wasm_memory_data_size | | | +| | wasm_memory_size | | | +| | _vector methods_ | | | +| extern | wasm_extern_delete | extern | | +| | wasm_extern_as_XXX | | | +| | wasm_XXX_as_extern | | | +| | wasm_extern_kind | | | +| | wasm_extern_type | | | +| | _vector methods_ | | | + +not supported _functions_ + +- wasm_config_XXX +- wasm_module_deserialize +- wasm_module_serialize +- wasm_ref_XXX +- wasm_XXX_as_ref +- wasm_XXX_as_ref_const +- wasm_XXX_copy +- wasm_XXX_get_host_info +- wasm_XXX_set_host_info + +## test + +there will be two kinds of tests in the project + +- unit test. located in `./tests`. driven by _unittest_. run by + `$ python -m unittest` or `$ make test`. +- integration test. located in `./samples`. + +The whole project is under test-driven development. Every wrapper function will +have two kinds of test cases. The first kind is a positive case. It checks a +wrapper function with expected and safe arguments combinations. Its goal is the +function should work well with expected inputs. Another kind is a negative +case. It feeds unexpected arguments combinations into a wrapper function. Arguments +should include but not be limited to `None`. It ensures that the function will +gracefully handle invalid input or unexpected behaviors. + +## distribution + +### package + +Create a python package named `wamr`. Users should import it after installation +just like any other python module. + +```python +from wamr import * +``` + +### PyPI + +Refer to [tutorial provided by PyPA](https://packaging.python.org/en/latest/tutorials/packaging-projects/). +Steps to publish WAMR Python library: + +1. Creating `pyproject.toml` tells build tools (like pip and build) what is + required to build a project. An example .toml file uses _setuptools_ + + ```toml + [build-system] + requires = [ + "setuptools>=42", + "wheel" + ] + build-backend = "setuptools.build_meta" + ``` + +2. Configuring metadata tells build tools about a package (such as the name + and the version), as well as which code files to include + + - Static metadata (`setup.cfg`): guaranteed to be the same every time. + It is simpler, easier to read, and avoids many common errors, like + encoding errors. + + - Dynamic metadata (`setup.py`): possibly non-deterministic. Any items that + are dynamic or determined at install-time, as well as extension modules + or extensions to setuptools, need to go into setup.py. + + **_Static metadata should be preferred_**. Dynamic metadata should be used + only as an escape hatch when necessary. setup.py used to be + required, but can be omitted with newer versions of setuptools and pip. + +3. Including other files in the distribution + + - For [source distribution](https://packaging.python.org/en/latest/glossary/#term-Source-Distribution-or-sdist): + + It's usually generated using `python setup.py sdist`, providing metadata + and the essential source files needed for installing by a tool like pip, + or for generating a Built Distribution. + + It includes our Python modules, pyproject.toml, metadata, README.md, + LICENSE. If you want to control what goes in this explicitly, + see [Including files in source distributions with MANIFEST.in](https://packaging.python.org/en/latest/guides/using-manifest-in/#using-manifest-in). + + - For [final built distribution](https://packaging.python.org/en/latest/glossary/#term-Built-Distribution) + + A Distribution format containing files and metadata that only need to be + moved to the correct location on the target system, to be installed. + e.g. `Wheel` + + It will have the Python files in the discovered or listed Python packages. + If you want to control what goes here, such as to add data files, + see [Including Data Files](https://setuptools.pypa.io/en/latest/userguide/datafiles.html) from the [setuptools docs](https://setuptools.pypa.io/en/latest/index.html). + +4. Generating distribution archives. These are archives that are uploaded to + the Python Package Index and can be installed by pip. + + example using `setuptools` + + ```shell + python3 -m pip install --upgrade build + python3 -m build + ``` + + generated files: + + ```shell + dist/ + WAMR-package-0.0.1-py3-none-any.whl + WAMR-package-0.0.1.tar.gz + ``` + + The `tar.gz` file is a _source archive_ whereas the `.whl file` is a + _built distribution_. Newer pip versions preferentially install built + distributions but will fall back to source archives if needed. You should + always upload a source archive and provide built archives for compatibility + reasons. + +5. Uploading the distribution archives + + - Register an account on https://pypi.org. + - To securely upload your project, you’ll need a + [PyPI API token](https://pypi.org/help/#apitoken). It can create at + [here](https://pypi.org/manage/account/#api-tokens), and the “Scope” + the setting needs to be “Entire account”. + - After registration, now twine can be used to upload the distribution packages. + + ```shell + # install twine + python3 -m pip install --upgrade twine + # --repository is https://pypi.org/ by default. + # You will be prompted for a username and password. For the username, use __token__. For the password, use the token value, including the pypi- prefix. + twine upload dist/* + ``` + +after all, the python binding will be installed with + +```shell +$ pip install wamr +``` + +PS: A example lifecycle of a python package +![python-package-lifecycle](images/python_package_life_cycle.png) + +## CI + +There are several parts: + +- code format check. +- test. include running all unit test cases and examples. +- publish built distribution. diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/images/python_package_life_cycle.png b/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/images/python_package_life_cycle.png new file mode 100644 index 0000000..90856e1 Binary files /dev/null and b/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/images/python_package_life_cycle.png differ diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/setup_dev_env.md b/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/setup_dev_env.md new file mode 100644 index 0000000..6612748 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/docs/setup_dev_env.md @@ -0,0 +1,12 @@ +Use a python virtual environment tool to create an environment for development. All necessary packages are in _../requirements.txt_. + +python code formatter is provided by _black_. + +python code linter is provided by _pylint_ and default configuration. + +Unit tests are driven by _unittest_. + +```bash +$ python -m unittest -v tests/test_basics.py +$ python -m unittest -v tests/test_advanced.py +``` diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/requirements.txt b/wasm-micro-runtime/language-bindings/python/wasm-c-api/requirements.txt new file mode 100644 index 0000000..7bb68ba --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/requirements.txt @@ -0,0 +1,5 @@ +black +nose +pycparser +pylint + diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello.wat b/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello.wat new file mode 100644 index 0000000..1c56c55 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello.wat @@ -0,0 +1,4 @@ +(module + (func $hello (import "" "hello")) + (func (export "run") (call $hello)) +) diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello_oop.py b/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello_oop.py new file mode 100644 index 0000000..666f63c --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello_oop.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +import ctypes +from wamr import * + + +def hello_callback(): + print("Calling back...") + print("> Hello World!") + + +def main(): + print("Initializing...") + engine = Engine() + store = Store(engine) + + print("Loading binary...") + print("Compiling module...") + module = Module.from_file(engine, "./hello.wasm") + + print("Creating callback...") + hello = Func(store, FuncType([], []), hello_callback) + + print("Instantiating module...") + instance = Instance(store, module, [hello]) + + print("Extracting export...") + run = instance.exports(store)["run"] + + print("Calling export...") + run(store) + + print("Shutting down...") + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello_procedural.py b/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello_procedural.py new file mode 100644 index 0000000..5924423 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/samples/hello_procedural.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +import ctypes +import wamr.wasmcapi.ffi as ffi + +WAMS_BINARY_CONTENT = ( + b"\x00asm\x01\x00\x00\x00\x01\x84\x80\x80\x80\x00\x01`\x00\x00\x02\x8a\x80" + b"\x80\x80\x00\x01\x00\x05hello\x00\x00\x03\x82\x80\x80\x80\x00\x01\x00" + b"\x07\x87\x80\x80\x80\x00\x01\x03run\x00\x01\n\x8a\x80\x80\x80\x00\x01" + b"\x84\x80\x80\x80\x00\x00\x10\x00\x0b" +) + + +@ffi.wasm_func_cb_decl +def hello_callback(args, results): + print("Calling back...") + print("> Hello World!") + + +def main(): + print("Initializing...") + engine = ffi.wasm_engine_new() + store = ffi.wasm_store_new(engine) + + print("Loading binary...") + + # for convenience, use binary content instead of open file + # with open("./hello.wasm", "rb") as f: + # wasm = f.read() + wasm = WAMS_BINARY_CONTENT + binary = ffi.wasm_byte_vec_t() + ffi.wasm_byte_vec_new_uninitialized(binary, len(wasm)) + # underlying buffer is not writable + binary.data = (ctypes.c_ubyte * len(wasm)).from_buffer_copy(wasm) + + print("Compiling module...") + module = ffi.wasm_module_new(store, binary) + if not module: + raise RuntimeError("Compiling module failed") + + binary.data = None + ffi.wasm_byte_vec_delete(binary) + + print("Creating callback...") + hello_type = ffi.wasm_functype_new_0_0() + hello_func = ffi.wasm_func_new( + store, + hello_type, + hello_callback, + ) + + ffi.wasm_functype_delete(hello_type) + + print("Instantiating module...") + + imports = ffi.wasm_extern_vec_t() + ffi.wasm_extern_vec_new((imports), 1, ffi.wasm_func_as_extern(hello_func)) + instance = ffi.wasm_instance_new(store, module, imports, None) + + ffi.wasm_func_delete(hello_func) + + print("Extracting export...") + exports = ffi.wasm_extern_vec_t() + ffi.wasm_instance_exports(instance, exports) + + run_func = ffi.wasm_extern_as_func(exports.data[0]) + if not run_func: + raise RuntimeError("can not extract exported function") + + ffi.wasm_instance_delete(instance) + ffi.wasm_module_delete(module) + + print("Calling export...") + args = ffi.wasm_val_vec_t() + results = ffi.wasm_val_vec_t() + + ffi.wasm_val_vec_new_empty(args) + ffi.wasm_val_vec_new_empty(results) + ffi.wasm_func_call(run_func, args, results) + + print("Shutting down...") + ffi.wasm_store_delete(store) + ffi.wasm_engine_delete(engine) + + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/__init__.py b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/__init__.py new file mode 100644 index 0000000..fd913c6 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +__all__ = ["test_basic", "test_advanced"] diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/context.py b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/context.py new file mode 100644 index 0000000..15c3008 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/context.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import sys +import os + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +import wamr diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/test_advanced.py b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/test_advanced.py new file mode 100644 index 0000000..706447a --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/test_advanced.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +import ctypes as c +import math +import unittest + +import wamr.wasmcapi.ffi as ffi + + +# It is a module likes: +# (module +# (import "mod" "g0" (global i32)) +# (import "mod" "f0" (func (param f32) (result f64))) +# +# (func (export "f1") (param i32 i64)) +# (global (export "g1") (mut f32) (f32.const 3.14)) +# (memory (export "m1") 1 2) +# (table (export "t1") 1 funcref) +# +# (func (export "f2") (unreachable)) +# ) +MODULE_BINARY = ( + b"\x00asm\x01\x00\x00\x00\x01\x0e\x03`\x01}\x01|`\x02\x7f~\x00`\x00" + b"\x00\x02\x14\x02\x03mod\x02g0\x03\x7f\x00\x03mod\x02f0\x00\x00\x03\x03" + b"\x02\x01\x02\x04\x04\x01p\x00\x01\x05\x04\x01\x01\x01\x02\x06\t\x01}\x01C" + b"\xc3\xf5H@\x0b\x07\x1a\x05\x02f1\x00\x01\x02g1\x03\x01\x02m1\x02\x00\x02t1" + b"\x01\x00\x02f2\x00\x02\n\x08\x02\x02\x00\x0b\x03\x00\x00\x0b" +) + +# False -> True when testing with a library enabling WAMR_BUILD_DUMP_CALL_STACK flag +TEST_WITH_WAMR_BUILD_DUMP_CALL_STACK = False + + +@ffi.wasm_func_cb_decl +def callback(args, results): + args = ffi.dereference(args) + results = ffi.dereference(results) + + arg_v = args.data[0] + + result_v = ffi.wasm_f64_val(arg_v.of.f32 * 2.0) + ffi.wasm_val_copy(results.data[0], result_v) + results.num_elems = 1 + + print(f"\nIn callback: {arg_v} --> {result_v}\n") + + +@ffi.wasm_func_with_env_cb_decl +def callback_with_env(env, args, results): + # pylint: disable=unused-argument + print("summer") + + +class AdvancedTestSuite(unittest.TestCase): + @classmethod + def setUpClass(cls): + print("Initializing...") + cls._wasm_engine = ffi.wasm_engine_new() + cls._wasm_store = ffi.wasm_store_new(cls._wasm_engine) + + def assertIsNullPointer(self, pointer): + # pylint: disable=invalid-name + if not ffi.is_null_pointer(pointer): + self.fail("not a non-null pointer") + + def assertIsNotNullPointer(self, pointer): + # pylint: disable=invalid-name + if ffi.is_null_pointer(pointer): + self.fail("not a non-null pointer") + + def load_binary(self, binary_string): + print("Load binary...") + binary = ffi.load_module_file(binary_string) + binary = c.pointer(binary) + self.assertIsNotNullPointer(binary) + return binary + + def compile(self, binary): + print("Compile...") + module = ffi.wasm_module_new(self._wasm_store, binary) + self.assertIsNotNullPointer(module) + return module + + def prepare_imports_local(self): + print("Prepare imports...") + func_type = ffi.wasm_functype_new_1_1( + ffi.wasm_valtype_new(ffi.WASM_F32), + ffi.wasm_valtype_new(ffi.WASM_F64), + ) + func = ffi.wasm_func_new(self._wasm_store, func_type, callback) + self.assertIsNotNullPointer(func) + ffi.wasm_functype_delete(func_type) + + glbl_type = ffi.wasm_globaltype_new(ffi.wasm_valtype_new(ffi.WASM_I32), True) + init = ffi.wasm_i32_val(1024) + glbl = ffi.wasm_global_new(self._wasm_store, glbl_type, init) + self.assertIsNotNullPointer(glbl) + ffi.wasm_globaltype_delete(glbl_type) + + imports = ffi.wasm_extern_vec_t() + data = ffi.list_to_carray( + c.POINTER(ffi.wasm_extern_t), + ffi.wasm_func_as_extern(func), + ffi.wasm_global_as_extern(glbl), + ) + ffi.wasm_extern_vec_new(imports, 2, data) + imports = c.pointer(imports) + self.assertIsNotNullPointer(imports) + return imports + + def instantiate(self, module, imports): + print("Instantiate module...") + instance = ffi.wasm_instance_new( + self._wasm_store, module, imports, ffi.create_null_pointer(ffi.wasm_trap_t) + ) + self.assertIsNotNone(instance) + self.assertIsNotNullPointer(instance) + return instance + + def extract_exports(self, instance): + print("Extracting exports...") + exports = ffi.wasm_extern_vec_t() + ffi.wasm_instance_exports(instance, exports) + exports = c.pointer(exports) + self.assertIsNotNullPointer(exports) + return exports + + def setUp(self): + binary = self.load_binary(MODULE_BINARY) + self.module = self.compile(binary) + self.imports = self.prepare_imports_local() + self.instance = self.instantiate(self.module, self.imports) + self.exports = self.extract_exports(self.instance) + + ffi.wasm_byte_vec_delete(binary) + + def tearDown(self): + if self.imports: + ffi.wasm_extern_vec_delete(self.imports) + + if self.exports: + ffi.wasm_extern_vec_delete(self.exports) + + ffi.wasm_instance_delete(self.instance) + ffi.wasm_module_delete(self.module) + + def test_wasm_func_call_wasm(self): + export_list = ffi.wasm_vec_to_list(self.exports) + print(export_list) + + func = ffi.wasm_extern_as_func(export_list[0]) + self.assertIsNotNullPointer(func) + + # make a call + params = ffi.wasm_val_vec_t() + data = ffi.list_to_carray( + ffi.wasm_val_t, + ffi.wasm_i32_val(1024), + ffi.wasm_i64_val(1024 * 1024), + ) + ffi.wasm_val_vec_new(params, 2, data) + + results = ffi.wasm_val_vec_t() + ffi.wasm_val_vec_new_empty(results) + + ffi.wasm_func_call(func, params, results) + + def test_wasm_func_call_native(self): + import_list = ffi.wasm_vec_to_list(self.imports) + + func = ffi.wasm_extern_as_func(import_list[0]) + self.assertIsNotNullPointer(func) + + params = ffi.wasm_val_vec_t() + ffi.wasm_val_vec_new( + params, 1, ffi.list_to_carray(ffi.wasm_val_t, ffi.wasm_f32_val(3.14)) + ) + results = ffi.wasm_val_vec_t() + ffi.wasm_val_vec_new_uninitialized(results, 1) + ffi.wasm_func_call(func, params, results) + self.assertEqual(params.data[0].of.f32 * 2, results.data[0].of.f64) + + def test_wasm_func_call_unlinked(self): + ft = ffi.wasm_functype_new_0_0() + func = ffi.wasm_func_new(self._wasm_store, ft, callback) + params = ffi.wasm_val_vec_t() + ffi.wasm_val_vec_new_empty(params) + results = ffi.wasm_val_vec_t() + ffi.wasm_val_vec_new_empty(results) + trap = ffi.wasm_func_call(func, params, results) + ffi.wasm_func_delete(func) + + def test_wasm_global_get_wasm(self): + export_list = ffi.wasm_vec_to_list(self.exports) + glb = ffi.wasm_extern_as_global(export_list[1]) + self.assertIsNotNullPointer(glb) + + # access the global + val = ffi.wasm_val_t() + ffi.wasm_global_get(glb, val) + self.assertAlmostEqual(val.of.f32, 3.14, places=3) + + def test_wasm_global_get_native(self): + import_list = ffi.wasm_vec_to_list(self.imports) + + glb = ffi.wasm_extern_as_global(import_list[1]) + self.assertIsNotNullPointer(glb) + + val = ffi.wasm_val_t() + ffi.wasm_global_get(glb, val) + self.assertEqual(val.of.i32, 1024) + + def test_wasm_global_get_unlinked(self): + gt = ffi.wasm_globaltype_new(ffi.wasm_valtype_new(ffi.WASM_I32), True) + init = ffi.wasm_i32_val(32) + glbl = ffi.wasm_global_new(self._wasm_store, gt, init) + val_ret = ffi.wasm_f32_val(3.14) + ffi.wasm_global_get(glbl, val_ret) + ffi.wasm_global_delete(glbl) + + # val_ret wasn't touched, keep the original value + self.assertAlmostEqual(val_ret.of.f32, 3.14, 3) + + def test_wasm_global_get_null_val(self): + export_list = ffi.wasm_vec_to_list(self.exports) + glb = ffi.wasm_extern_as_global(export_list[1]) + ffi.wasm_global_get(glb, ffi.create_null_pointer(ffi.wasm_val_t)) + + def test_wasm_global_get_null_global(self): + val = ffi.wasm_val_t() + ffi.wasm_global_get(ffi.create_null_pointer(ffi.wasm_global_t), val) + + def test_wasm_global_set_wasm(self): + export_list = ffi.wasm_vec_to_list(self.exports) + glb = ffi.wasm_extern_as_global(export_list[1]) + self.assertIsNotNullPointer(glb) + + # access the global + new_val = ffi.wasm_f32_val(math.e) + ffi.wasm_global_set(glb, new_val) + + val = ffi.wasm_val_t() + ffi.wasm_global_get(glb, val) + self.assertNotEqual(val.of.f32, 3.14) + + def test_wasm_global_set_native(self): + import_list = ffi.wasm_vec_to_list(self.imports) + + glb = ffi.wasm_extern_as_global(import_list[1]) + self.assertIsNotNullPointer(glb) + + new_val = ffi.wasm_i32_val(2048) + ffi.wasm_global_set(glb, new_val) + + val = ffi.wasm_val_t() + ffi.wasm_global_get(glb, val) + self.assertEqual(val, new_val) + + def test_wasm_global_set_unlinked(self): + gt = ffi.wasm_globaltype_new(ffi.wasm_valtype_new(ffi.WASM_I32), True) + init = ffi.wasm_i32_val(32) + glbl = ffi.wasm_global_new(self._wasm_store, gt, init) + val_ret = ffi.wasm_f32_val(3.14) + ffi.wasm_global_set(glbl, val_ret) + ffi.wasm_global_delete(glbl) + + def test_wasm_global_set_null_v(self): + export_list = ffi.wasm_vec_to_list(self.exports) + glb = ffi.wasm_extern_as_global(export_list[1]) + # access the global + ffi.wasm_global_set(glb, ffi.create_null_pointer(ffi.wasm_val_t)) + + def test_wasm_global_set_null_global(self): + # access the global + new_val = ffi.wasm_f32_val(math.e) + ffi.wasm_global_set(ffi.create_null_pointer(ffi.wasm_global_t), new_val) + + def test_wasm_table_size(self): + export_list = ffi.wasm_vec_to_list(self.exports) + tbl = ffi.wasm_extern_as_table(export_list[3]) + self.assertIsNotNullPointer(tbl) + + tbl_sz = ffi.wasm_table_size(tbl) + self.assertEqual(tbl_sz, 1) + + def test_wasm_table_size_unlink(self): + vt = ffi.wasm_valtype_new(ffi.WASM_FUNCREF) + limits = ffi.wasm_limits_new(10, 15) + tt = ffi.wasm_tabletype_new(vt, limits) + tbl = ffi.wasm_table_new( + self._wasm_store, tt, ffi.create_null_pointer(ffi.wasm_ref_t) + ) + tbl_sz = ffi.wasm_table_size(tbl) + ffi.wasm_table_delete(tbl) + + def test_wasm_table_size_null_table(self): + ffi.wasm_table_size(ffi.create_null_pointer(ffi.wasm_table_t)) + + def test_wasm_table_get(self): + export_list = ffi.wasm_vec_to_list(self.exports) + tbl = ffi.wasm_extern_as_table(export_list[3]) + self.assertIsNotNullPointer(tbl) + + ref = ffi.wasm_table_get(tbl, 0) + self.assertIsNullPointer(ref) + + ref = ffi.wasm_table_get(tbl, 4096) + self.assertIsNullPointer(ref) + + def test_wasm_table_get_unlinked(self): + vt = ffi.wasm_valtype_new(ffi.WASM_FUNCREF) + limits = ffi.wasm_limits_new(10, 15) + tt = ffi.wasm_tabletype_new(vt, limits) + tbl = ffi.wasm_table_new( + self._wasm_store, tt, ffi.create_null_pointer(ffi.wasm_ref_t) + ) + ffi.wasm_table_get(tbl, 0) + ffi.wasm_table_delete(tbl) + + def test_wasm_table_get_null_table(self): + ffi.wasm_table_get(ffi.create_null_pointer(ffi.wasm_table_t), 0) + + def test_wasm_table_get_out_of_bounds(self): + export_list = ffi.wasm_vec_to_list(self.exports) + tbl = ffi.wasm_extern_as_table(export_list[3]) + ffi.wasm_table_get(tbl, 1_000_000_000) + + def test_wasm_ref(self): + export_list = ffi.wasm_vec_to_list(self.exports) + func = ffi.wasm_extern_as_func(export_list[0]) + self.assertIsNotNullPointer(func) + + ref = ffi.wasm_func_as_ref(func) + self.assertIsNotNullPointer(ref) + + func_from_ref = ffi.wasm_ref_as_func(ref) + self.assertEqual( + ffi.dereference(ffi.wasm_func_type(func)), + ffi.dereference(ffi.wasm_func_type(func_from_ref)), + ) + + def test_wasm_table_set(self): + export_list = ffi.wasm_vec_to_list(self.exports) + tbl = ffi.wasm_extern_as_table(export_list[3]) + self.assertIsNotNullPointer(tbl) + + func = ffi.wasm_extern_as_func(export_list[0]) + ref = ffi.wasm_func_as_ref(func) + + ffi.wasm_table_set(tbl, 0, ref) + + ref_ret = ffi.wasm_table_get(tbl, 0) + self.assertIsNotNullPointer(ref_ret) + func_ret = ffi.wasm_ref_as_func(ref_ret) + self.assertEqual( + ffi.dereference(ffi.wasm_func_type(func)), + ffi.dereference(ffi.wasm_func_type(func_ret)), + ) + + def test_wasm_table_set_unlinked(self): + vt = ffi.wasm_valtype_new(ffi.WASM_FUNCREF) + limits = ffi.wasm_limits_new(10, 15) + tt = ffi.wasm_tabletype_new(vt, limits) + tbl = ffi.wasm_table_new( + self._wasm_store, tt, ffi.create_null_pointer(ffi.wasm_ref_t) + ) + export_list = ffi.wasm_vec_to_list(self.exports) + func = ffi.wasm_extern_as_func(export_list[0]) + ref = ffi.wasm_func_as_ref(func) + ffi.wasm_table_set(tbl, 0, ref) + ffi.wasm_table_delete(tbl) + + def test_wasm_table_set_null_table(self): + export_list = ffi.wasm_vec_to_list(self.exports) + func = ffi.wasm_extern_as_func(export_list[0]) + ref = ffi.wasm_func_as_ref(func) + ffi.wasm_table_set(ffi.create_null_pointer(ffi.wasm_table_t), 0, ref) + + def test_wasm_table_set_null_ref(self): + export_list = ffi.wasm_vec_to_list(self.exports) + tbl = ffi.wasm_extern_as_table(export_list[3]) + ffi.wasm_table_set(tbl, 0, ffi.create_null_pointer(ffi.wasm_ref_t)) + + def test_wasm_table_set_out_of_bounds(self): + export_list = ffi.wasm_vec_to_list(self.exports) + tbl = ffi.wasm_extern_as_table(export_list[3]) + func = ffi.wasm_extern_as_func(export_list[0]) + ref = ffi.wasm_func_as_ref(func) + ffi.wasm_table_set(tbl, 1_000_000_000, ref) + + def test_wasm_memory_size(self): + export_list = ffi.wasm_vec_to_list(self.exports) + mem = ffi.wasm_extern_as_memory(export_list[2]) + self.assertIsNotNullPointer(mem) + + pg_sz = ffi.wasm_memory_size(mem) + self.assertEqual(pg_sz, 1) + + def test_wasm_memory_size_unlinked(self): + limits = ffi.wasm_limits_new(10, 12) + mt = ffi.wasm_memorytype_new(limits) + mem = ffi.wasm_memory_new(self._wasm_store, mt) + ffi.wasm_memory_size(mem) + ffi.wasm_memory_delete(mem) + + def test_wasm_memory_data(self): + export_list = ffi.wasm_vec_to_list(self.exports) + mem = ffi.wasm_extern_as_memory(export_list[2]) + self.assertIsNotNullPointer(mem) + + data_base = ffi.wasm_memory_data(mem) + self.assertIsNotNone(data_base) + + def test_wasm_memory_data_unlinked(self): + limits = ffi.wasm_limits_new(10, 12) + mt = ffi.wasm_memorytype_new(limits) + mem = ffi.wasm_memory_new(self._wasm_store, mt) + ffi.wasm_memory_data(mem) + ffi.wasm_memory_delete(mem) + + def test_wasm_memory_data_size(self): + export_list = ffi.wasm_vec_to_list(self.exports) + mem = ffi.wasm_extern_as_memory(export_list[2]) + self.assertIsNotNullPointer(mem) + + mem_sz = ffi.wasm_memory_data_size(mem) + self.assertGreater(mem_sz, 0) + + def test_wasm_memory_data_size_unlinked(self): + limits = ffi.wasm_limits_new(10, 12) + mt = ffi.wasm_memorytype_new(limits) + mem = ffi.wasm_memory_new(self._wasm_store, mt) + ffi.wasm_memory_data_size(mem) + ffi.wasm_memory_delete(mem) + + @unittest.skipUnless( + TEST_WITH_WAMR_BUILD_DUMP_CALL_STACK, + "need to enable WAMR_BUILD_DUMP_CALL_STACK", + ) + # assertions only works if enabling WAMR_BUILD_DUMP_CALL_STACK + def test_wasm_frame(self): + export_list = ffi.wasm_vec_to_list(self.exports) + func = ffi.wasm_extern_as_func(export_list[4]) + # make a call + params = ffi.wasm_val_vec_t() + ffi.wasm_val_vec_new_empty(params) + results = ffi.wasm_val_vec_t() + ffi.wasm_val_vec_new_empty(results) + + print("Making a call...") + trap = ffi.wasm_func_call(func, params, results) + + message = ffi.wasm_message_t() + ffi.wasm_trap_message(trap, message) + self.assertIsNotNullPointer(c.pointer(message)) + print(message) + + frame = ffi.wasm_trap_origin(trap) + self.assertIsNotNullPointer(frame) + print(ffi.dereference(frame)) + + traces = ffi.wasm_frame_vec_t() + ffi.wasm_trap_trace(trap, traces) + self.assertIsNotNullPointer(c.pointer(frame)) + + instance = ffi.wasm_frame_instance(frame) + self.assertIsNotNullPointer(instance) + + module_offset = ffi.wasm_frame_module_offset(frame) + + func_index = ffi.wasm_frame_func_index(frame) + self.assertEqual(func_index, 2) + + func_offset = ffi.wasm_frame_func_offset(frame) + self.assertGreater(func_offset, 0) + + @classmethod + def tearDownClass(cls): + print("Shutting down...") + ffi.wasm_store_delete(cls._wasm_store) + ffi.wasm_engine_delete(cls._wasm_engine) + + +if __name__ == "__main__": + unittest.main() diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/test_basic.py b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/test_basic.py new file mode 100644 index 0000000..a2d3f28 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/tests/test_basic.py @@ -0,0 +1,1590 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +import ctypes as c +import unittest +from venv import create + +from wamr.wasmcapi.ffi import * + +# It is a module likes: +# (module +# (import "mod" "g0" (global i32)) +# (import "mod" "f0" (func (param f32) (result f64))) +# +# (func (export "f1") (param i32 i64)) +# (global (export "g1") (mut f32) (f32.const 3.14)) +# (memory 1 2) +# (table 1 funcref) +# ) +MODULE_BINARY = ( + b"\x00asm\x01\x00\x00\x00\x01\x0b\x02`\x01}\x01|`\x02\x7f~\x00" + b"\x02\x14\x02\x03mod\x02g0\x03\x7f\x00\x03mod\x02f0\x00\x00\x03" + b"\x02\x01\x01\x04\x04\x01p\x00\x01\x05\x04\x01\x01\x01\x02\x06\t" + b"\x01}\x01C\xc3\xf5H@\x0b\x07\x0b\x02\x02f1\x00\x01\x02g1\x03\x01\n" + b"\x04\x01\x02\x00\x0b" +) + + +@wasm_func_cb_decl +def callback(args, results): + # pylint: disable=unused-argument + print("summer") + + +@wasm_func_with_env_cb_decl +def callback_with_env(env, args, results): + # pylint: disable=unused-argument + print("summer") + + +class BasicTestSuite(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._wasm_engine = wasm_engine_new() + cls._wasm_store = wasm_store_new(cls._wasm_engine) + + def assertIsNullPointer(self, c_pointer): + if not is_null_pointer(c_pointer): + self.fail("not a null pointer") + + def assertIsNotNullPointer(self, c_pointer): + if is_null_pointer(c_pointer): + self.fail("not a non-null pointer") + + def test_wasm_valkind(self): + self.assertEqual( + [WASM_I32, WASM_I64, WASM_F32, WASM_F64, WASM_ANYREF, WASM_FUNCREF], + [0, 1, 2, 3, 128, 129], + ) + + def test_wasm_valtype_new_pos(self): + vt = wasm_valtype_new(WASM_I32) + self.assertIsNotNullPointer(vt) + wasm_valtype_delete(vt) + + def test_wasm_valtype_new_neg(self): + vt = wasm_valtype_new(37) + self.assertIsNullPointer(vt) + wasm_valtype_delete(vt) + + def test_wasm_valtype_kind_pos(self): + vt = wasm_valtype_new(WASM_I64) + self.assertEqual(wasm_valtype_kind(vt), WASM_I64) + wasm_valtype_delete(vt) + + def test_wasm_valtype_kind_neg(self): + wasm_valtype_kind(create_null_pointer(wasm_valtype_t)) + + def test_wasm_valtype_delete_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + wasm_valtype_delete(vt) + + def test_wasm_valtype_delete_neg(self): + wasm_valtype_delete(create_null_pointer(wasm_valtype_t)) + + def test_wasm_valtype_copy_pos(self): + vt1 = wasm_valtype_new(WASM_FUNCREF) + vt2 = wasm_valtype_copy(vt1) + + self.assertIsNotNone(vt1) + self.assertIsNotNullPointer(vt1) + self.assertEqual(dereference(vt1), dereference(vt2)) + + wasm_valtype_delete(vt1) + wasm_valtype_delete(vt2) + + def test_wasm_valtype_copy_neg(self): + vt = wasm_valtype_copy(create_null_pointer(wasm_valtype_t)) + self.assertIsNotNone(vt) + self.assertIsNullPointer(vt) + + def test_list_to_carray(self): + v1 = wasm_valtype_new(WASM_I64) + v2 = wasm_valtype_new(WASM_F32) + v3 = wasm_valtype_new(WASM_FUNCREF) + data = list_to_carray(c.POINTER(wasm_valtype_t), v1, v2, v3) + + self.assertIsNotNone(data) + self.assertTrue(isinstance(data, c.Array)) + self.assertEqual(data._length_, 3) + self.assertEqual(dereference(data[0]), dereference(v1)) + self.assertEqual(dereference(data[1]), dereference(v2)) + self.assertEqual(dereference(data[2]), dereference(v3)) + + wasm_valtype_delete(v1) + wasm_valtype_delete(v2) + wasm_valtype_delete(v3) + + def test_wasm_valtype_vec_new_pos(self): + def_vt_list = [ + wasm_valtype_new(WASM_I32), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_FUNCREF), + ] + data = list_to_carray(c.POINTER(wasm_valtype_t), *def_vt_list) + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new(vt_vec, 3, data) + + self.assertEqual(vt_vec.size, 3) + self.assertEqual(vt_vec.num_elems, 3) + self.assertIsNotNullPointer(vt_vec.data) + + ret_vt_list = wasm_vec_to_list(vt_vec) + ret_vt_list = [dereference(vt) for vt in ret_vt_list] + def_vt_list = [dereference(vt) for vt in def_vt_list] + self.assertEqual(ret_vt_list, def_vt_list) + + wasm_valtype_vec_delete(vt_vec) + + def test_wasm_valtype_vec_new_neg(self): + data = list_to_carray( + c.POINTER(wasm_valtype_t), + wasm_valtype_new(WASM_I32), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_FUNCREF), + ) + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new(vt_vec, 1_000_000_000, data) + + self.assertEqual(vt_vec.size, 0) + self.assertIsNullPointer(vt_vec.data) + + wasm_valtype_vec_delete(vt_vec) + + def test_wasm_valtype_vec_new_null_out(self): + data = list_to_carray( + c.POINTER(wasm_valtype_t), + wasm_valtype_new(WASM_I32), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_FUNCREF), + ) + wasm_valtype_vec_new(create_null_pointer(wasm_valtype_vec_t), 10, data) + + def test_wasm_valtype_vec_new_null_data(self): + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new(vt_vec, 3, create_null_pointer(wasm_valtype_t)) + self.assertIsNotNone(vt_vec) + self.assertIsNotNullPointer(c.pointer(vt_vec)) + + def test_wasm_valtype_vec_new_uninitialized_pos(self): + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new_uninitialized((vt_vec), 2) + self.assertEqual(2, vt_vec.size) + wasm_valtype_vec_delete(vt_vec) + + def test_wasm_valtype_vec_new_uninitialized_neg(self): + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new_uninitialized(vt_vec, 1_000_000_000) + self.assertEqual(vt_vec.size, 0) + self.assertIsNullPointer(vt_vec.data) + wasm_valtype_vec_delete(vt_vec) + + def test_wasm_valtype_vec_new_uninitialized_null_out(self): + wasm_valtype_vec_new_uninitialized(create_null_pointer(wasm_valtype_vec_t), 2) + + def test_wasm_valtype_vec_new_empty_pos(self): + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new_empty(vt_vec) + self.assertEqual(0, vt_vec.size) + self.assertIsNullPointer(vt_vec.data) + wasm_valtype_vec_delete(vt_vec) + + def test_wasm_valtype_vec_new_empty_neg(self): + wasm_valtype_vec_new_empty(create_null_pointer(wasm_valtype_vec_t)) + + def test_wasm_valtype_vec_copy_pos(self): + vt_vec1 = wasm_valtype_vec_t() + vt1 = wasm_valtype_new(WASM_F32) + vt2 = wasm_valtype_new(WASM_I32) + data = list_to_carray(c.POINTER(wasm_valtype_t), vt1, vt2) + wasm_valtype_vec_new(vt_vec1, 2, data) + + vt_vec2 = wasm_valtype_vec_t() + wasm_valtype_vec_copy(vt_vec2, vt_vec1) + + print(f"{vt_vec1} --> {vt_vec2}") + + self.assertEqual(vt_vec2.size, 2) + self.assertEqual(vt_vec2.num_elems, 2) + self.assertEqual(dereference(vt_vec2.data[0]), dereference(vt1)) + self.assertEqual(dereference(vt_vec2.data[1]), dereference(vt2)) + + wasm_valtype_vec_delete(vt_vec1) + wasm_valtype_vec_delete(vt_vec2) + + def test_wasm_valtype_vec_copy_null_src(self): + dst = wasm_valtype_vec_t() + wasm_valtype_vec_copy(dst, create_null_pointer(wasm_valtype_vec_t)) + self.assertIsNotNullPointer(c.pointer(dst)) + self.assertIsNullPointer(dst.data) + + def test_wasm_valtype_vec_copy_null_dst(self): + src = wasm_valtype_vec_t() + wasm_valtype_vec_new_empty(src) + wasm_valtype_vec_copy(create_null_pointer(wasm_valtype_vec_t), src) + wasm_valtype_vec_delete(src) + + def test_wasm_valtype_vec_delete_pos(self): + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new_uninitialized(vt_vec, 10) + wasm_valtype_vec_delete(vt_vec) + + vt_vec = wasm_valtype_vec_t() + wasm_valtype_vec_new_empty(vt_vec) + wasm_valtype_vec_delete(vt_vec) + + def test_wasm_valtype_vec_delete_neg(self): + wasm_valtype_vec_delete(create_null_pointer(wasm_valtype_vec_t)) + + def test_wasm_functype_new_0_0(self): + ft = wasm_functype_new_0_0() + + self.assertIsNotNullPointer(ft) + self.assertEqual(0, dereference(wasm_functype_params(ft)).size) + self.assertEqual(0, dereference(wasm_functype_results(ft)).size) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_1_0(self): + vt = wasm_valtype_new(WASM_I64) + ft = wasm_functype_new_1_0(vt) + + self.assertIsNotNullPointer(ft) + params = wasm_vec_to_list(wasm_functype_params(ft)) + self.assertEqual([dereference(p) for p in params], [dereference(vt)]) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_2_0(self): + vt1 = wasm_valtype_new(WASM_I64) + vt2 = wasm_valtype_new(WASM_F64) + ft = wasm_functype_new_2_0(vt1, vt2) + + self.assertIsNotNullPointer(ft) + self.assertEqual(2, dereference(wasm_functype_params(ft)).size) + self.assertEqual(0, dereference(wasm_functype_results(ft)).size) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_3_0(self): + vt_list = [ + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_I64), + ] + ft = wasm_functype_new_3_0(*vt_list) + + params = wasm_vec_to_list(wasm_functype_params(ft)) + self.assertEqual( + [dereference(p) for p in params], + [dereference(vt) for vt in vt_list], + ) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_0_1(self): + vt1 = wasm_valtype_new(WASM_I64) + ft = wasm_functype_new_0_1(vt1) + + self.assertIsNotNullPointer(ft) + self.assertEqual(0, dereference(wasm_functype_params(ft)).size) + self.assertEqual(1, dereference(wasm_functype_results(ft)).size) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_1_1(self): + vt1 = wasm_valtype_new(WASM_I64) + vt2 = wasm_valtype_new(WASM_F64) + ft = wasm_functype_new_1_1(vt1, vt2) + + params = wasm_vec_to_list(wasm_functype_params(ft)) + self.assertEqual(dereference(params[0]), dereference(vt1)) + + results = wasm_vec_to_list(wasm_functype_results(ft)) + self.assertEqual(dereference(results[0]), dereference(vt2)) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_2_1(self): + vt_list = [ + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_I64), + ] + ft = wasm_functype_new_2_1(*vt_list) + + self.assertIsNotNullPointer(ft) + self.assertEqual(2, dereference(wasm_functype_params(ft)).size) + self.assertEqual(1, dereference(wasm_functype_results(ft)).size) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_3_1(self): + vt_list = [ + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_I32), + ] + ft = wasm_functype_new_3_1(*vt_list) + + params = wasm_vec_to_list(wasm_functype_params(ft)) + self.assertEqual( + [dereference(p) for p in params], [dereference(vt) for vt in vt_list[:3]] + ) + + results = wasm_vec_to_list(wasm_functype_results(ft)) + self.assertEqual(dereference(results[0]), dereference(vt_list[-1])) + + wasm_functype_delete(ft) + + def test_wasm_functype_new_neg(self): + ft = wasm_functype_new( + create_null_pointer(wasm_valtype_vec_t), + create_null_pointer(wasm_valtype_vec_t), + ) + + self.assertIsNotNullPointer(ft) + + wasm_functype_delete(ft) + + def test_wasm_functype_delete_pos(self): + ft = wasm_functype_new_0_0() + wasm_functype_delete(ft) + + def test_wasm_functype_delete_neg(self): + wasm_functype_delete(create_null_pointer(wasm_functype_t)) + + def test_wasm_functype_params_pos(self): + vt_list = [ + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_I64), + ] + ft = wasm_functype_new_3_0(*vt_list) + params = wasm_vec_to_list(wasm_functype_params(ft)) + + self.assertEqual( + [dereference(p) for p in params], + [dereference(vt) for vt in vt_list], + ) + + wasm_functype_delete(ft) + + def test_wasm_functype_params_neg(self): + params = wasm_functype_params(create_null_pointer(wasm_functype_t)) + self.assertIsNullPointer(params) + + def test_wasm_functype_results_pos(self): + vt1 = wasm_valtype_new(WASM_I64) + ft = wasm_functype_new_0_1(vt1) + results = wasm_vec_to_list(wasm_functype_results(ft)) + + self.assertEqual(dereference(results[0]), dereference(vt1)) + + wasm_functype_delete(ft) + + def test_wasm_functype_results_neg(self): + results = wasm_functype_results(create_null_pointer(wasm_functype_t)) + self.assertIsNullPointer(results) + + def test_wasm_functype_copy_pos(self): + ft1 = wasm_functype_new_2_1( + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_F64), + wasm_valtype_new(WASM_I64), + ) + ft2 = wasm_functype_copy(ft1) + + self.assertIsNotNullPointer(ft2) + self.assertEqual(2, dereference(wasm_functype_params(ft1)).size) + self.assertEqual(1, dereference(wasm_functype_results(ft2)).size) + + wasm_functype_delete(ft1) + wasm_functype_delete(ft2) + + def test_wasm_functype_copy_neg(self): + ft2 = wasm_functype_copy(create_null_pointer(wasm_functype_t)) + self.assertIsNullPointer(ft2) + wasm_functype_delete(ft2) + + def test_wasm_globaltype_new_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + gt = wasm_globaltype_new(vt, True) + + self.assertIsNotNullPointer(gt) + + wasm_globaltype_delete(gt) + + def test_wasm_globaltype_new_neg(self): + gt = wasm_globaltype_new(create_null_pointer(wasm_valtype_t), True) + self.assertIsNullPointer(gt) + wasm_globaltype_delete(gt) + + def test_wasm_globaltype_delete_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + gt = wasm_globaltype_new(vt, False) + wasm_globaltype_delete(gt) + + def test_wasm_globaltype_delete_neg(self): + wasm_globaltype_delete(create_null_pointer(wasm_globaltype_t)) + + def test_wasm_globaltype_content_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + gt = wasm_globaltype_new(vt, True) + gt_ret = wasm_globaltype_content(gt) + + self.assertEqual(dereference(vt), dereference(gt_ret)) + + wasm_globaltype_delete(gt) + + def test_wasm_globaltype_content_neg(self): + gt_ret = wasm_globaltype_content(create_null_pointer(wasm_globaltype_t)) + self.assertIsNullPointer(gt_ret) + + def test_wasm_globaltype_mutability_pos(self): + vt1 = wasm_valtype_new(WASM_F32) + gt1 = wasm_globaltype_new(vt1, False) + vt2 = wasm_valtype_new(WASM_F32) + gt2 = wasm_globaltype_new(vt2, True) + + self.assertFalse(wasm_globaltype_mutability(gt1)) + self.assertTrue(wasm_globaltype_mutability(gt2)) + + wasm_globaltype_delete(gt1) + wasm_globaltype_delete(gt2) + + def test_wasm_globaltype_mutability_neg(self): + self.assertFalse( + wasm_globaltype_mutability(create_null_pointer(wasm_globaltype_t)) + ) + + def test_wasm_globaltype_copy_pos(self): + vt = wasm_valtype_new(WASM_I32) + gt1 = wasm_globaltype_new(vt, True) + gt2 = wasm_globaltype_copy(gt1) + + self.assertEqual(dereference(gt1), dereference(gt2)) + + wasm_globaltype_delete(gt1) + wasm_globaltype_delete(gt2) + + def test_wasm_globaltype_copy_neg(self): + gt2 = wasm_globaltype_copy(create_null_pointer(wasm_globaltype_t)) + + self.assertIsNullPointer(gt2) + wasm_globaltype_delete(gt2) + + def test_wasm_limit_new(self): + limit = wasm_limits_new(10, 20) + self.assertIsNotNullPointer(limit) + self.assertEqual(dereference(limit).min, 10) + self.assertEqual(dereference(limit).max, 20) + + def test_wasm_tabletype_new_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limit = wasm_limits_new(0, 0xFF) + tt = wasm_tabletype_new(vt, limit) + + self.assertIsNotNullPointer(tt) + wasm_tabletype_delete(tt) + + def test_wasm_tabletype_new_null_val_type(self): + limit = wasm_limits_new(0, 0xFFFFFFFF) + tt = wasm_tabletype_new(create_null_pointer(wasm_valtype_t), limit) + + self.assertIsNullPointer(tt) + wasm_tabletype_delete(tt) + + def test_wasm_tabletype_new_null_limits(self): + vt = wasm_valtype_new(WASM_FUNCREF) + tt = wasm_tabletype_new(vt, create_null_pointer(wasm_limits_t)) + + self.assertIsNullPointer(tt) + wasm_tabletype_delete(tt) + + def test_wasm_tabletype_delete_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limit = wasm_limits_new(0, 0xFFFFFFFF) + tt = wasm_tabletype_new(vt, limit) + wasm_tabletype_delete(tt) + + def test_wasm_tabletype_delete_neg(self): + wasm_tabletype_delete(create_null_pointer(wasm_tabletype_t)) + + def test_wasm_tabletype_element_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limit = wasm_limits_new(0, 0xFFFFFFFF) + tt = wasm_tabletype_new(vt, limit) + vt_ret = wasm_tabletype_element(tt) + + self.assertEqual(dereference(vt), dereference(vt_ret)) + + wasm_tabletype_delete(tt) + + def test_wasm_tabletype_element_neg(self): + vt_ret = wasm_tabletype_element(create_null_pointer(wasm_tabletype_t)) + self.assertIsNullPointer(vt_ret) + + def test_wasm_tabletype_limits_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limit = wasm_limits_new(100, 256) + tt = wasm_tabletype_new(vt, limit) + limit_ret = wasm_tabletype_limits(tt) + + self.assertEqual(dereference(limit), dereference(limit_ret)) + + wasm_tabletype_delete(tt) + + def test_wasm_tabletype_limits_neg(self): + limit_ret = wasm_tabletype_limits(create_null_pointer(wasm_tabletype_t)) + self.assertIsNullPointer(limit_ret) + + def test_wasm_tabletype_copy_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limit = wasm_limits_new(13, 19) + tt1 = wasm_tabletype_new(vt, limit) + tt2 = wasm_tabletype_copy(tt1) + + self.assertEqual(dereference(tt1), dereference(tt2)) + + wasm_tabletype_delete(tt1) + wasm_tabletype_delete(tt2) + + def test_wasm_tabletype_copy_neg(self): + tt2 = wasm_tabletype_copy(create_null_pointer(wasm_tabletype_t)) + self.assertIsNullPointer(tt2) + wasm_tabletype_delete(tt2) + + def test_wasm_memorytype_new_pos(self): + limit = wasm_limits_new(0, 3) + mt = wasm_memorytype_new(limit) + + self.assertIsNotNullPointer(mt) + + wasm_memorytype_delete(mt) + + def test_wasm_memorytype_new_neg(self): + mt = wasm_memorytype_new(None) + + self.assertIsNullPointer(mt) + + wasm_memorytype_delete(mt) + + def test_wasm_memorytype_delete_pos(self): + limit = wasm_limits_new(1, 2) + mt = wasm_memorytype_new(limit) + wasm_memorytype_delete(mt) + + def test_wasm_memorytype_delete_neg(self): + wasm_memorytype_delete(create_null_pointer(wasm_memorytype_t)) + + def test_wasm_memorytype_limits_pos(self): + limit = wasm_limits_new(3, 8) + mt = wasm_memorytype_new(limit) + limit_ret = wasm_memorytype_limits(mt) + + self.assertEqual(dereference(limit), dereference(limit_ret)) + + wasm_memorytype_delete(mt) + + def test_wasm_memorytype_limits_neg(self): + wasm_memorytype_limits(create_null_pointer(wasm_memorytype_t)) + + def test_wasm_memorytype_copy_pos(self): + limit = wasm_limits_new(7, 13) + mt1 = wasm_memorytype_new(limit) + mt2 = wasm_memorytype_copy(mt1) + + self.assertEqual( + dereference(mt1), + dereference(mt2), + ) + + wasm_memorytype_delete(mt1) + wasm_memorytype_delete(mt2) + + def test_wasm_memorytype_copy_neg(self): + mt2 = wasm_memorytype_copy(create_null_pointer(wasm_memorytype_t)) + + self.assertIsNullPointer(mt2) + + wasm_memorytype_delete(mt2) + + def test_wasm_externtype_kind_pos(self): + ft = wasm_functype_new_0_0() + gt = wasm_globaltype_new(wasm_valtype_new(WASM_FUNCREF), True) + mt = wasm_memorytype_new(wasm_limits_new(1, 2)) + tt = wasm_tabletype_new(wasm_valtype_new(WASM_FUNCREF), wasm_limits_new(10, 20)) + ets = [ + wasm_functype_as_externtype(ft), + wasm_globaltype_as_externtype(gt), + wasm_memorytype_as_externtype(mt), + wasm_tabletype_as_externtype(tt), + ] + type_kinds = [wasm_externtype_kind(et) for et in ets] + + self.assertEqual( + type_kinds, + [ + WASM_EXTERN_FUNC, + WASM_EXTERN_GLOBAL, + WASM_EXTERN_MEMORY, + WASM_EXTERN_TABLE, + ], + ) + + [wasm_externtype_delete(et) for et in ets] + + def test_wasm_externtype_kind_neg(self): + et = wasm_memorytype_as_externtype(create_null_pointer(wasm_memorytype_t)) + self.assertIsNullPointer(et) + + def test_wasm_externtype_delete_pos(self): + mt = wasm_memorytype_new(wasm_limits_new(10, 20)) + et = wasm_memorytype_as_externtype(mt) + wasm_externtype_delete(et) + + def test_wasm_externtype_delete_neg(self): + et = wasm_globaltype_as_externtype(create_null_pointer(wasm_globaltype_t)) + wasm_externtype_delete(et) + + def test_wasm_externtype_copy_pos(self): + tt1 = wasm_tabletype_new( + wasm_valtype_new(WASM_FUNCREF), wasm_limits_new(10, 20) + ) + et1 = wasm_tabletype_as_externtype(tt1) + et2 = wasm_externtype_copy(et1) + + tt2 = wasm_externtype_as_tabletype(et2) + self.assertEqual(dereference(tt1), dereference(tt2)) + + wasm_externtype_delete(et2) + wasm_externtype_delete(et1) + + def test_wasm_externtype_copy_neg(self): + et1 = create_null_pointer(wasm_externtype_t) + et2 = wasm_externtype_copy(et1) + wasm_externtype_delete(et2) + wasm_externtype_delete(et1) + + def test_wasm_name_new_from_string(self): + s = "let the stars shine upon you" + name = wasm_name_new_from_string(s) + + name_data = c.cast(name.data, c.c_char_p) + name_data = bytes.decode(name_data.value) + self.assertEqual(name_data, s) + + def test_wasm_importtype_new_pos(self): + module_name = "mA" + field_name = "func#1" + module_name = wasm_name_new_from_string(module_name) + field_name = wasm_name_new_from_string(field_name) + ft = wasm_functype_new_0_0() + et = wasm_functype_as_externtype(ft) + it = wasm_importtype_new(module_name, field_name, et) + + self.assertIsNotNullPointer(it) + self.assertEqual(dereference(wasm_importtype_module(it)), module_name) + self.assertEqual(dereference(wasm_importtype_name(it)), field_name) + self.assertEqual(dereference(wasm_importtype_type(it)), dereference(et)) + + wasm_importtype_delete(it) + + def test_wasm_importtype_new_null_ext_type(self): + module_name = "mA" + field_name = "func#1" + module_name = wasm_name_new_from_string(module_name) + field_name = wasm_name_new_from_string(field_name) + it = wasm_importtype_new( + module_name, + field_name, + create_null_pointer(wasm_externtype_t), + ) + + self.assertIsNullPointer(it) + + wasm_importtype_delete(it) + + def test_wasm_importtype_new_null_module(self): + field_name = "func#1" + field_name = wasm_name_new_from_string(field_name) + ft = wasm_functype_new_0_0() + et = wasm_functype_as_externtype(ft) + it = wasm_importtype_new(create_null_pointer(wasm_name_t), field_name, et) + + self.assertIsNullPointer(it) + + wasm_importtype_delete(it) + + def test_wasm_importtype_new_null_field(self): + module_name = "mA" + module_name = wasm_name_new_from_string(module_name) + ft = wasm_functype_new_0_0() + et = wasm_functype_as_externtype(ft) + it = wasm_importtype_new(module_name, create_null_pointer(wasm_name_t), et) + + self.assertIsNullPointer(it) + + wasm_importtype_delete(it) + + def test_wasm_importtype_copy_pos(self): + module_name = "mA" + field_name = "memory#1" + module_name = wasm_name_new_from_string(module_name) + field_name = wasm_name_new_from_string(field_name) + mt = wasm_memorytype_new(wasm_limits_new(10, 20)) + et = wasm_memorytype_as_externtype(mt) + it1 = wasm_importtype_new(module_name, field_name, et) + it2 = wasm_importtype_copy(it1) + + self.assertEqual(dereference(it1), dereference(it2)) + + wasm_importtype_delete(it1) + wasm_importtype_delete(it2) + + def test_wasm_importtype_copy_neg(self): + it1 = create_null_pointer(wasm_importtype_t) + it2 = wasm_importtype_copy(it1) + wasm_importtype_delete(it1) + wasm_importtype_delete(it2) + + def test_wasm_importtype_delete_pos(self): + module_name = "mA" + field_name = "memory#1" + module_name = wasm_name_new_from_string(module_name) + field_name = wasm_name_new_from_string(field_name) + tt = wasm_tabletype_new(wasm_valtype_new(WASM_FUNCREF), wasm_limits_new(10, 20)) + et = wasm_tabletype_as_externtype(tt) + it = wasm_importtype_new(module_name, field_name, et) + wasm_importtype_delete(it) + + def test_wasm_importtype_delete_neg(self): + wasm_importtype_delete(create_null_pointer(wasm_importtype_t)) + + def test_wasm_importtype_module_pos(self): + module_name = "mA" + field_name = "func#1" + module_name = wasm_name_new_from_string(module_name) + field_name = wasm_name_new_from_string(field_name) + ft = wasm_functype_new_0_0() + et = wasm_functype_as_externtype(ft) + it = wasm_importtype_new(module_name, field_name, et) + module_name_ret = wasm_importtype_module(it) + + self.assertEqual(dereference(module_name_ret), module_name) + + wasm_importtype_delete(it) + + def test_wasm_importtype_module_neg(self): + it = create_null_pointer(wasm_importtype_t) + wasm_importtype_module(it) + wasm_importtype_delete(it) + + def test_wasm_importtype_name_pos(self): + module_name = "mA" + field_name = "func#1" + module_name = wasm_name_new_from_string(module_name) + field_name = wasm_name_new_from_string(field_name) + ft = wasm_functype_new_0_0() + et = wasm_functype_as_externtype(ft) + it = wasm_importtype_new(module_name, field_name, et) + field_name_ret = wasm_importtype_name(it) + + self.assertEqual(dereference(field_name_ret), field_name) + + wasm_importtype_delete(it) + + def test_wasm_importtype_name_neg(self): + it = create_null_pointer(wasm_importtype_t) + wasm_importtype_name(it) + wasm_importtype_delete(it) + + def test_wasm_importtype_type_pos(self): + module_name = "mA" + field_name = "func#1" + module_name = wasm_name_new_from_string(module_name) + field_name = wasm_name_new_from_string(field_name) + ft = wasm_functype_new_0_0() + et = wasm_functype_as_externtype(ft) + it = wasm_importtype_new(module_name, field_name, et) + et_ret = wasm_importtype_type(it) + + self.assertEqual(dereference(et_ret), dereference(et)) + + wasm_importtype_delete(it) + + def test_wasm_importtype_type_neg(self): + it = create_null_pointer(wasm_importtype_t) + wasm_importtype_type(it) + wasm_importtype_delete(it) + + def test_wasm_exporttype_new_pos(self): + name = "hello" + name = wasm_name_new_from_string(name) + ft = wasm_functype_new_0_0() + ft = wasm_functype_as_externtype(ft) + et = wasm_exporttype_new(name, ft) + + self.assertIsNotNullPointer(et) + + wasm_exporttype_delete(et) + + def test_wasm_exporttype_new_null_name(self): + name = create_null_pointer(wasm_name_t) + ft = wasm_functype_new_0_0() + ft = wasm_functype_as_externtype(ft) + et = wasm_exporttype_new(name, ft) + + self.assertIsNullPointer(et) + + wasm_exporttype_delete(et) + + def test_wasm_exporttype_new_null_ext_type(self): + name = "hello" + name = wasm_name_new_from_string(name) + ext_type = create_null_pointer(wasm_externtype_t) + et = wasm_exporttype_new(name, ext_type) + + self.assertIsNullPointer(et) + + wasm_exporttype_delete(et) + + def test_wasm_exporttype_copy_pos(self): + name = "hello" + name = wasm_name_new_from_string(name) + gt = wasm_globaltype_new(wasm_valtype_new(WASM_F32), True) + gt = wasm_globaltype_as_externtype(gt) + et1 = wasm_exporttype_new(name, gt) + et2 = wasm_exporttype_copy(et1) + + self.assertEqual( + dereference(et1), + dereference(et2), + ) + + wasm_exporttype_delete(et1) + wasm_exporttype_delete(et2) + + def test_wasm_exporttype_copy_neg(self): + et1 = create_null_pointer(wasm_exporttype_t) + et2 = wasm_exporttype_copy(et1) + + wasm_exporttype_delete(et1) + wasm_exporttype_delete(et2) + + def test_wasm_exporttype_delete_pos(self): + name = "hello" + name = wasm_name_new_from_string(name) + mt = wasm_memorytype_new(wasm_limits_new(10, 20)) + mt = wasm_memorytype_as_externtype(mt) + et = wasm_exporttype_new(name, mt) + + wasm_exporttype_delete(et) + + def test_wasm_exporttype_delete_neg(self): + et = create_null_pointer(wasm_exporttype_t) + wasm_exporttype_delete(et) + + def test_wasm_exporttype_name_pos(self): + name = "hello" + name = wasm_name_new_from_string(name) + tt = wasm_tabletype_new(wasm_valtype_new(WASM_FUNCREF), wasm_limits_new(10, 20)) + tt = wasm_tabletype_as_externtype(tt) + et = wasm_exporttype_new(name, tt) + name_ret = wasm_exporttype_name(et) + + self.assertEqual(dereference(name_ret), name) + + wasm_exporttype_delete(et) + + def test_wasm_exporttype_name_neg(self): + et = create_null_pointer(wasm_exporttype_t) + wasm_exporttype_name(et) + wasm_exporttype_delete(et) + + def test_wasm_exporttype_type_pos(self): + name = "hello" + name = wasm_name_new_from_string(name) + tt = wasm_tabletype_new(wasm_valtype_new(WASM_FUNCREF), wasm_limits_new(10, 20)) + tt = wasm_tabletype_as_externtype(tt) + et = wasm_exporttype_new(name, tt) + tt_ret = wasm_exporttype_type(et) + + self.assertEqual(dereference(tt_ret), dereference(tt)) + + wasm_exporttype_delete(et) + + def test_wasm_exporttype_type_neg(self): + et = create_null_pointer(wasm_exporttype_t) + wasm_exporttype_type(et) + wasm_exporttype_delete(et) + + def test_wasm_i32_val(self): + val = wasm_i32_val(100) + + self.assertEqual(val.kind, WASM_I32) + self.assertEqual(val.of.i32, 100) + + # can not use wasm_val_delete() because it is not malloced + + def test_wasm_i64_val(self): + val = wasm_i64_val(-100) + + self.assertEqual(val.kind, WASM_I64) + self.assertEqual(val.of.i64, -100) + + # can not use wasm_val_delete() because it is not malloced + + def test_wasm_f32_val(self): + val = wasm_f32_val(100) + + self.assertEqual(val.kind, WASM_F32) + self.assertEqual(val.of.f32, 100.0) + + # can not use wasm_val_delete() because it is not malloced + + def test_wasm_f64_val(self): + val = wasm_f64_val(-100) + + self.assertEqual(val.kind, WASM_F64) + self.assertEqual(val.of.f64, -100.0) + + # can not use wasm_val_delete() because it is not malloced + + # there is no wasm_val_new() to malloc a wasm_val_t + def test_wasm_val_delete(self): + pass + + def test_wasm_val_copy(self): + v1 = wasm_f32_val(3.14) + v2 = wasm_val_t() + wasm_val_copy(v1, v2) + + self.assertEqual(v1, v2) + # can not use wasm_val_delete() because it is not malloced + + def test_wasm_ref_delete_neg(self): + ref = create_null_pointer(wasm_ref_t) + wasm_ref_delete(ref) + + ref = wasm_ref_t() + wasm_ref_delete(ref) + + def test_wasm_trap_new_pos(self): + # can't create a trap with traces(wasm_frame_vec_t) + msg = wasm_name_new_from_string("a fake trap") + trap = wasm_trap_new(self._wasm_store, msg) + + self.assertIsNotNone(trap) + + wasm_trap_delete(trap) + + def test_wasm_trap_new_null_msg(self): + trap = wasm_trap_new(self._wasm_store, create_null_pointer(wasm_name_t)) + + self.assertIsNotNone(trap) + self.assertIsNotNullPointer(trap) + + wasm_trap_delete(trap) + + def test_wasm_trap_message_pos(self): + msg = wasm_name_new_from_string("a fake trap") + trap = wasm_trap_new(self._wasm_store, msg) + msg_in_trap = wasm_message_t() + wasm_trap_message(trap, msg_in_trap) + + self.assertEqual( + msg, + msg_in_trap, + ) + + wasm_trap_delete(trap) + + def test_wasm_trap_message_null_trap(self): + msg = wasm_name_new_from_string("a fake trap") + wasm_trap_message(create_null_pointer(wasm_trap_t), msg) + + def test_wasm_trap_message_null_out(self): + msg = wasm_name_new_from_string("a fake trap") + trap = wasm_trap_new(self._wasm_store, msg) + wasm_trap_message(trap, create_null_pointer(wasm_message_t)) + wasm_trap_delete(trap) + + # test those APIs in advance: + # wasm_trap_origin + # wasm_trap_trace + # wasm_frame_delete + # wasm_frame_copy + # wasm_frame_module_offset + # wasm_frame_instance + # wasm_frame_func_index + # wasm_frame_func_offset + + def test_wasm_foreign_new_pos(self): + foreign = wasm_foreign_new(self._wasm_store) + + self.assertIsNotNone(foreign) + self.assertIsNotNullPointer(foreign) + + wasm_foreign_delete(foreign) + + def test_wasm_foreign_new_neg(self): + foreign = wasm_foreign_new(create_null_pointer(wasm_store_t)) + + self.assertIsNotNone(foreign) + self.assertIsNullPointer(foreign) + + wasm_foreign_delete(foreign) + + def test_wasm_foreign_delete_pos(self): + foreign = wasm_foreign_new(self._wasm_store) + wasm_foreign_delete(foreign) + + def test_wasm_foreign_delete_neg(self): + wasm_foreign_delete(create_null_pointer(wasm_foreign_t)) + + # wasm_egnine_new()/wasm_engine_delete() + # wasm_store_new()/wasm_store_delete() + # used in setUpClass() and tearDownClass + + def test_wasm_module_new_pos(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + + self.assertIsNotNone(module) + self.assertIsNotNullPointer(module) + + wasm_byte_vec_delete(binary) + wasm_module_delete(module) + + def test_wasm_module_new_neg(self): + module = wasm_module_new(self._wasm_store, create_null_pointer(wasm_byte_vec_t)) + + self.assertIsNotNone(module) + self.assertIsNullPointer(module) + + wasm_module_delete(module) + + def test_wasm_module_delete_pos(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + wasm_byte_vec_delete(binary) + wasm_module_delete(module) + + def test_wasm_module_delete_neg(self): + module = wasm_module_new(self._wasm_store, create_null_pointer(wasm_byte_vec_t)) + wasm_module_delete(module) + + def test_wasm_module_validate_pos(self): + binary = load_module_file(MODULE_BINARY) + validation = wasm_module_validate(self._wasm_store, binary) + + self.assertTrue(validation) + + wasm_byte_vec_delete(binary) + + def test_wasm_module_validate_neg(self): + tmp = (1024).to_bytes(2, byteorder="big") + binary = load_module_file(tmp) + validation = wasm_module_validate(self._wasm_store, binary) + + self.assertFalse(validation) + + wasm_byte_vec_delete(binary) + + def test_wasm_module_imports_pos(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + imports = wasm_importtype_vec_t() + wasm_module_imports(module, imports) + + imports_list = wasm_vec_to_list(imports) + self.assertEqual(len(imports_list), 2) + + func_type = wasm_functype_new_1_1( + wasm_valtype_new(WASM_F32), + wasm_valtype_new(WASM_F64), + ) + ext_type = wasm_functype_as_externtype(func_type) + self.assertEqual( + dereference(wasm_importtype_type(imports_list[0])), dereference(ext_type) + ) + + wasm_externtype_delete(ext_type) + wasm_importtype_vec_delete(imports) + wasm_byte_vec_delete(binary) + wasm_module_delete(module) + + def test_wasm_module_imports_null_module(self): + imports = wasm_importtype_vec_t() + wasm_module_imports(create_null_pointer(wasm_module_t), imports) + + self.assertEqual(imports.size, 0) + + wasm_importtype_vec_delete(imports) + + def test_wasm_module_imports_null_out(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + wasm_module_imports(module, create_null_pointer(wasm_importtype_vec_t)) + wasm_byte_vec_delete(binary) + wasm_module_delete(module) + + def test_wasm_module_exports_pos(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + exports = wasm_exporttype_vec_t() + wasm_module_exports(module, exports) + + exports_list = wasm_vec_to_list(exports) + self.assertEqual(len(exports_list), 2) + + glbl_type = wasm_globaltype_new(wasm_valtype_new(WASM_F32), True) + ext_type = wasm_globaltype_as_externtype(glbl_type) + self.assertEqual( + dereference(wasm_exporttype_type(exports_list[1])), dereference(ext_type) + ) + + wasm_exporttype_vec_delete(exports) + wasm_byte_vec_delete(binary) + wasm_module_delete(module) + + def test_wasm_module_exports_null_module(self): + exports = wasm_exporttype_vec_t() + wasm_module_exports(create_null_pointer(wasm_module_t), exports) + + self.assertEqual(exports.size, 0) + + wasm_exporttype_vec_delete(exports) + + def test_wasm_module_exports_null_out(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + wasm_module_exports(module, create_null_pointer(wasm_exporttype_vec_t)) + wasm_byte_vec_delete(binary) + wasm_module_delete(module) + + def test_wasm_instance_new_pos_empty_imports(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + imports = wasm_extern_vec_t() + wasm_extern_vec_new_empty(imports) + instance = wasm_instance_new( + self._wasm_store, module, imports, create_null_pointer(wasm_trap_t) + ) + + wasm_module_delete(module) + + self.assertIsNullPointer(instance) + + def test_wasm_instance_new_pos(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + + ft = wasm_functype_new_1_1( + wasm_valtype_new(WASM_F32), + wasm_valtype_new(WASM_F64), + ) + func = wasm_func_new(self._wasm_store, ft, callback) + + gt = wasm_globaltype_new(wasm_valtype_new(WASM_I32), True) + init = wasm_i32_val(100) + gb = wasm_global_new(self._wasm_store, gt, init) + + imports = wasm_extern_vec_t() + data = list_to_carray( + c.POINTER(wasm_extern_t), + wasm_func_as_extern(func), + wasm_global_as_extern(gb), + ) + wasm_extern_vec_new(imports, 2, data) + + instance = wasm_instance_new( + self._wasm_store, module, imports, create_null_pointer(wasm_trap_t) + ) + + self.assertIsNotNone(instance) + + wasm_instance_delete(instance) + wasm_module_delete(module) + + def test_wasm_instance_new_neg_null_imports(self): + binary = load_module_file(MODULE_BINARY) + module = wasm_module_new(self._wasm_store, binary) + instance = wasm_instance_new( + self._wasm_store, + module, + create_null_pointer(wasm_extern_vec_t), + create_null_pointer(wasm_trap_t), + ) + + wasm_module_delete(module) + + self.assertIsNullPointer(instance) + + # test those APIs in advanced: + # wasm_instance_delete + # wasm_instance_exports + + def test_wasm_func_new_pos(self): + vt1 = wasm_valtype_new(WASM_F32) + vt2 = wasm_valtype_new(WASM_FUNCREF) + ft = wasm_functype_new_1_1(vt1, vt2) + func = wasm_func_new(self._wasm_store, ft, callback) + + self.assertIsNotNone(func) + self.assertIsNotNullPointer(func) + + wasm_func_delete(func) + + def test_wasm_func_new_null_type(self): + func = wasm_func_new( + self._wasm_store, create_null_pointer(wasm_functype_t), callback + ) + + self.assertIsNotNone(func) + self.assertIsNullPointer(func) + + wasm_func_delete(func) + + def test_wasm_func_new_null_callback(self): + vt1 = wasm_valtype_new(WASM_F32) + vt2 = wasm_valtype_new(WASM_FUNCREF) + ft = wasm_functype_new_1_1(vt1, vt2) + func = wasm_func_new(self._wasm_store, ft, wasm_func_callback_t()) + + self.assertIsNotNone(func) + self.assertIsNullPointer(func) + + wasm_func_delete(func) + + def test_wasm_func_new_with_env_pos(self): + ft = wasm_functype_new_3_1( + wasm_valtype_new(WASM_I32), + wasm_valtype_new(WASM_F32), + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_I64), + ) + func = wasm_func_new_with_env( + self._wasm_store, + ft, + callback_with_env, + c.c_void_p(0), + wasm_finalizer(0), + ) + + self.assertIsNotNone(func) + self.assertIsNotNullPointer(func) + + wasm_func_delete(func) + + def test_wasm_func_new_with_env_null_type(self): + func = wasm_func_new_with_env( + self._wasm_store, + create_null_pointer(wasm_functype_t), + callback_with_env, + c.c_void_p(0), + wasm_finalizer(0), + ) + + self.assertIsNotNone(func) + self.assertIsNullPointer(func) + + wasm_func_delete(func) + + def test_wasm_func_new_with_env_null_callback(self): + ft = wasm_functype_new_3_1( + wasm_valtype_new(WASM_I32), + wasm_valtype_new(WASM_F32), + wasm_valtype_new(WASM_I64), + wasm_valtype_new(WASM_I64), + ) + func = wasm_func_new_with_env( + self._wasm_store, + ft, + wasm_func_callback_with_env_t(), + c.c_void_p(0), + wasm_finalizer(0), + ) + + self.assertIsNotNone(func) + self.assertIsNullPointer(func) + + wasm_func_delete(func) + + def test_wasm_func_delete_pos(self): + ft = wasm_functype_new_0_0() + func = wasm_func_new(self._wasm_store, ft, callback) + wasm_func_delete(func) + + def test_wasm_func_delete_neg(self): + wasm_func_delete(create_null_pointer(wasm_func_t)) + + def test_wasm_func_type_pos(self): + ft = wasm_functype_new_2_0( + wasm_valtype_new(WASM_F32), + wasm_valtype_new(WASM_FUNCREF), + ) + func = wasm_func_new(self._wasm_store, ft, callback) + ft_ret = wasm_func_type(func) + + self.assertEqual( + dereference(ft), + dereference(ft_ret), + ) + + wasm_functype_delete(ft_ret) + wasm_func_delete(func) + + def test_wasm_func_type_neg(self): + ft_ret = wasm_func_type(create_null_pointer(wasm_func_t)) + wasm_functype_delete(ft_ret) + + def test_wasm_func_copy_pos(self): + vt1 = wasm_valtype_new(WASM_F32) + ft = wasm_functype_new_0_1(vt1) + func1 = wasm_func_new(self._wasm_store, ft, callback) + func2 = wasm_func_copy(func1) + + self.assertEqual( + dereference(wasm_func_type(func1)), dereference(wasm_func_type(func2)) + ) + + wasm_func_delete(func2) + wasm_func_delete(func1) + + def test_wasm_func_copy_neg(self): + func1 = wasm_func_new( + self._wasm_store, create_null_pointer(wasm_functype_t), callback + ) + func2 = wasm_func_copy(func1) + + wasm_func_delete(func2) + wasm_func_delete(func1) + + # test wasm_func_call in advanced + + def test_wasm_global_new_pos(self): + vt = wasm_valtype_new(WASM_F32) + gt = wasm_globaltype_new(vt, False) + v = wasm_f32_val(3.14) + g = wasm_global_new(self._wasm_store, gt, v) + + self.assertIsNotNone(g) + self.assertIsNotNullPointer(g) + + wasm_globaltype_delete(gt) + wasm_global_delete(g) + + def test_wasm_global_new_null_type(self): + v = wasm_f32_val(3.14) + g = wasm_global_new(self._wasm_store, create_null_pointer(wasm_globaltype_t), v) + + self.assertIsNotNone(g) + self.assertIsNullPointer(g) + + wasm_global_delete(g) + + def test_wasm_global_new_null_init(self): + vt = wasm_valtype_new(WASM_F32) + gt = wasm_globaltype_new(vt, False) + g = wasm_global_new(self._wasm_store, gt, create_null_pointer(wasm_val_t)) + + self.assertIsNotNone(g) + self.assertIsNullPointer(g) + + wasm_globaltype_delete(gt) + wasm_global_delete(g) + + def test_wasm_global_delete_pos(self): + vt = wasm_valtype_new(WASM_I32) + gt = wasm_globaltype_new(vt, True) + v = wasm_i32_val(3) + g = wasm_global_new(self._wasm_store, gt, v) + wasm_globaltype_delete(gt) + wasm_global_delete(g) + + def test_wasm_global_delete_neg(self): + wasm_global_delete(create_null_pointer(wasm_global_t)) + + def test_wasm_global_type_pos(self): + vt = wasm_valtype_new(WASM_I64) + gt = wasm_globaltype_new(vt, False) + v = wasm_i32_val(3) + g = wasm_global_new(self._wasm_store, gt, v) + gt_ret = wasm_global_type(g) + + self.assertEqual(dereference(gt), dereference(gt_ret)) + + wasm_globaltype_delete(gt) + wasm_globaltype_delete(gt_ret) + wasm_global_delete(g) + + def test_wasm_global_type_neg(self): + gt = wasm_global_type(create_null_pointer(wasm_global_t)) + wasm_globaltype_delete(gt) + + # test wasm_global_get and wasm_global_set in advanced + + def test_wasm_table_new_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limits = wasm_limits_new(10, 15) + tt = wasm_tabletype_new(vt, limits) + t = wasm_table_new(self._wasm_store, tt, create_null_pointer(wasm_ref_t)) + + self.assertIsNotNone(t) + self.assertIsNotNullPointer(t) + + wasm_table_delete(t) + + def test_wasm_table_new_null_type(self): + t = wasm_table_new( + self._wasm_store, + create_null_pointer(wasm_tabletype_t), + create_null_pointer(wasm_ref_t), + ) + + self.assertIsNotNone(t) + self.assertIsNullPointer(t) + + wasm_table_delete(t) + + def test_wasm_table_delete_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limits = wasm_limits_new(10, 15) + tt = wasm_tabletype_new(vt, limits) + t = wasm_table_new(self._wasm_store, tt, create_null_pointer(wasm_ref_t)) + wasm_table_delete(t) + + def test_wasm_table_delete_neg(self): + wasm_table_delete(create_null_pointer(wasm_table_t)) + + def test_wasm_table_type_pos(self): + vt = wasm_valtype_new(WASM_FUNCREF) + limits = wasm_limits_new(1, 2) + tt = wasm_tabletype_new(vt, limits) + t = wasm_table_new(self._wasm_store, tt, create_null_pointer(wasm_ref_t)) + tt_ret = wasm_table_type(t) + + self.assertEqual( + dereference(tt), + dereference(tt_ret), + ) + + wasm_table_delete(t) + + def test_wasm_table_type_neg(self): + t = wasm_table_new( + self._wasm_store, + create_null_pointer(wasm_tabletype_t), + create_null_pointer(wasm_ref_t), + ) + tt_ret = wasm_table_type(t) + wasm_table_delete(t) + + # test wasm_table_size, wasm_table_get, wasm_table_set in advanced + + def test_wasm_memory_new_pos(self): + limits = wasm_limits_new(10, 12) + mt = wasm_memorytype_new(limits) + m = wasm_memory_new(self._wasm_store, mt) + + self.assertIsNotNullPointer(m) + + wasm_memory_delete(m) + + def test_wasm_memory_new_null_type(self): + m = wasm_memory_new(self._wasm_store, create_null_pointer(wasm_memorytype_t)) + + self.assertIsNullPointer(m) + + wasm_memory_delete(m) + + def test_wasm_memory_delete_pos(self): + limits = wasm_limits_new(10, 21) + mt = wasm_memorytype_new(limits) + m = wasm_memory_new(self._wasm_store, mt) + wasm_memory_delete(m) + + def test_wasm_memory_delete_neg(self): + wasm_memory_delete(create_null_pointer(wasm_memory_t)) + + def test_wasm_memory_type_pos(self): + limits = wasm_limits_new(10, 21) + mt = wasm_memorytype_new(limits) + m = wasm_memory_new(self._wasm_store, mt) + mt_ret = wasm_memory_type(m) + + self.assertEqual(dereference(mt), dereference(mt_ret)) + + wasm_memory_delete(m) + + def test_wasm_memory_type_neg(self): + mt = wasm_memory_type(create_null_pointer(wasm_memory_t)) + + self.assertIsNullPointer(mt) + wasm_memorytype_delete(mt) + + # test wasm_memory_size, wasm_memory_data, wasm_memory_data_size in advanced + + def test_wasm_extern_delete_pos(self): + vt = wasm_valtype_new(WASM_I64) + gt = wasm_globaltype_new(vt, False) + v = wasm_i64_val(128) + glb = wasm_global_new(self._wasm_store, gt, v) + etrn = wasm_global_as_extern(glb) + wasm_extern_delete(etrn) + + def test_wasm_extern_delete_neg(self): + etrn = wasm_global_as_extern(create_null_pointer(wasm_global_t)) + wasm_extern_delete(etrn) + + def test_wasm_extern_type_pos(self): + vt = wasm_valtype_new(WASM_I64) + gt = wasm_globaltype_new(vt, False) + v = wasm_i64_val(128) + glb = wasm_global_new(self._wasm_store, gt, v) + etrn = wasm_global_as_extern(glb) + + tp = wasm_extern_type(etrn) + gt_ret = wasm_externtype_as_globaltype(tp) + self.assertEqual( + dereference(gt), + dereference(gt_ret), + ) + wasm_extern_delete(etrn) + + def test_wasm_extern_type_neg(self): + wasm_extern_type(create_null_pointer(wasm_extern_t)) + + def test_wasm_extern_kind_pos(self): + ft = wasm_functype_new_0_0() + func = wasm_func_new(self._wasm_store, ft, callback) + etrn = wasm_func_as_extern(func) + kind = wasm_extern_kind(etrn) + + self.assertEqual(WASM_EXTERN_FUNC, kind) + + wasm_extern_delete(etrn) + + def test_wasm_extern_kind_neg(self): + wasm_extern_kind(create_null_pointer(wasm_extern_t)) + + @classmethod + def tearDownClass(cls): + wasm_store_delete(cls._wasm_store) + wasm_engine_delete(cls._wasm_engine) + + +if __name__ == "__main__": + unittest.main() diff --git a/wasm-micro-runtime/language-bindings/python/wasm-c-api/utils/bindgen.py b/wasm-micro-runtime/language-bindings/python/wasm-c-api/utils/bindgen.py new file mode 100644 index 0000000..a505404 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/python/wasm-c-api/utils/bindgen.py @@ -0,0 +1,386 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +""" +- Need to run *download_wamr.py* firstly. +- Parse *./wasm-micro-runtime/core/iwasm/include/wasm_c_api.h* and generate + *wamr/binding.py* +""" +import os +import pathlib +import shutil +import sys + +from pycparser import c_ast, parse_file + +WASM_C_API_HEADER = "core/iwasm/include/wasm_c_api.h" +BINDING_PATH = "language-bindings/python/wamr/wasmcapi/binding.py" +# 4 spaces as default indent +INDENT = " " + +IGNORE_SYMOLS = ( + "wasm_engine_new_with_args", + "wasm_valkind_is_num", + "wasm_valkind_is_ref", + "wasm_valtype_is_num", + "wasm_valtype_is_ref", + "wasm_valtype_new_i32", + "wasm_valtype_new_i64", + "wasm_valtype_new_f32", + "wasm_valtype_new_f64", + "wasm_valtype_new_anyref", + "wasm_valtype_new_funcref", + "wasm_functype_new_0_0", + "wasm_functype_new_0_0", + "wasm_functype_new_1_0", + "wasm_functype_new_2_0", + "wasm_functype_new_3_0", + "wasm_functype_new_0_1", + "wasm_functype_new_1_1", + "wasm_functype_new_2_1", + "wasm_functype_new_3_1", + "wasm_functype_new_0_2", + "wasm_functype_new_1_2", + "wasm_functype_new_2_2", + "wasm_functype_new_3_2", + "wasm_val_init_ptr", + "wasm_val_ptr", + "wasm_val_t", + "wasm_ref_t", + "wasm_name_new_from_string", + "wasm_name_new_from_string_nt", +) + + +class Visitor(c_ast.NodeVisitor): + def __init__(self): + self.type_map = { + "_Bool": "c_bool", + "byte_t": "c_ubyte", + "char": "c_char", + "errno_t": "c_int", + "int": "c_int", + "long": "c_long", + "size_t": "c_size_t", + "uint32_t": "c_uint32", + "uint8_t": "c_uint8", + "void": "None", + } + self.ret = ( + "# -*- coding: utf-8 -*-\n" + "#!/usr/bin/env python3\n" + "#\n" + "# Copyright (C) 2019 Intel Corporation. All rights reserved.\n" + "# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n" + "#\n" + "#It is a generated file. DO NOT EDIT.\n" + "#\n" + "from ctypes import *\n" + "\n" + "from .ffi import dereference, libiwasm, wasm_ref_t, wasm_val_t\n" + "\n" + "\n" + ) + + def get_type_name(self, c_type): + if isinstance(c_type, c_ast.TypeDecl): + return self.get_type_name(c_type.type) + elif isinstance(c_type, c_ast.PtrDecl): + pointed_type = self.get_type_name(c_type.type) + + if isinstance(c_type.type, c_ast.FuncDecl): + # CFUCNTYPE is a pointer of function + return pointed_type + + if "None" == pointed_type: + return "c_void_p" + + return f"POINTER({pointed_type})" + + elif isinstance(c_type, c_ast.ArrayDecl): + return f"POINTER({self.get_type_name(c_type.type)})" + elif isinstance(c_type, c_ast.IdentifierType): + if len(c_type.names) > 1: + raise RuntimeError(f"unexpected type with a long names: {c_type}") + + type_name = c_type.names[0] + + if type_name.startswith("wasm_"): + return type_name + + if not type_name in self.type_map: + raise RuntimeError(f"a new type should be in type_map: {type_name}") + + return self.type_map.get(type_name) + elif isinstance(c_type, c_ast.Union): + if not c_type.name: + raise RuntimeError(f"found an anonymous union {c_type}") + + return c_type.name + elif isinstance(c_type, c_ast.Struct): + if not c_type.name: + raise RuntimeError(f"found an anonymous union {c_type}") + + return c_type.name + elif isinstance(c_type, c_ast.FuncDecl): + content = "CFUNCTYPE(" + if isinstance(c_type.type, c_ast.PtrDecl): + # there is a bug in CFUNCTYPE if the result type is a pointer + content += "c_void_p" + else: + content += f"{self.get_type_name(c_type.type)}" + content += f",{self.get_type_name(c_type.args)}" if c_type.args else "" + content += ")" + return content + elif isinstance(c_type, c_ast.Decl): + return self.get_type_name(c_type.type) + elif isinstance(c_type, c_ast.ParamList): + content = ",".join( + [self.get_type_name(param.type) for param in c_type.params] + ) + return content + else: + raise RuntimeError(f"unexpected type: {c_type.show()}") + + def visit_Struct(self, node): + # pylint: disable=invalid-name + def gen_fields(info, indent): + content = "" + for k, v in info.items(): + content += f'{indent}("{k}", {v}),\n' + return content[:-1] + + def gen_equal(info, indent): + content = f"{indent}return" + for k, v in info.items(): + # not compare pointer value in __eq__ + if v.startswith("POINTER") or v.startswith("c_void_p"): + continue + + content += f" self.{k} == other.{k} and" + return content[:-4] + + def gen_repr(info, indent): + content = f'{indent}return f"{{{{' + for k, _ in info.items(): + content += f"{k}={{self.{k}}}, " + content = content[:-2] + '}}"' + return content + + def gen_vector_repr(info, indent): + content = f'{indent}ret = ""\n' + content += f"{indent}for i in range(self.num_elems):\n" + + if 1 == info["data"].count("POINTER"): + # pointer + content += f"{2*indent}ret += str(self.data[i])\n" + else: + # pointer of pointer + content += f"{2*indent}ret += str(dereference(self.data[i]))\n" + + content += f'{2*indent}ret += " "\n' + content += f"{indent}return ret\n" + return content + + if not node.name or not node.name.lower().startswith("wasm"): + return + + if node.name in IGNORE_SYMOLS: + return + + name = node.name + + info = {} + if node.decls: + for decl in node.decls: + info[decl.name] = self.get_type_name(decl.type) + + if info: + self.ret += ( + f"class {name}(Structure):\n" + f"{INDENT}_fields_ = [\n" + f"{gen_fields(info, INDENT*2)}\n" + f"{INDENT}]\n" + f"\n" + f"{INDENT}def __eq__(self, other):\n" + f"{INDENT*2}if not isinstance(other, {name}):\n" + f"{INDENT*3}return False\n" + f"{gen_equal(info, INDENT*2)}\n" + f"\n" + f"{INDENT}def __repr__(self):\n" + ) + self.ret += ( + f"{gen_vector_repr(info, INDENT*2)}\n" + if name.endswith("_vec_t") + else f"{gen_repr(info, INDENT*2)}\n" + ) + self.ret += "\n" + + else: + self.ret += f"class {name}(Structure):\n{INDENT}pass\n" + + self.ret += "\n" + + def visit_Union(self, node): + # pylint: disable=invalid-name + print(f"Union: {node.show()}") + + def visit_Typedef(self, node): + # pylint: disable=invalid-name + # system defined + if not node.name: + return + + if not node.name.startswith("wasm_"): + return + + if node.name in IGNORE_SYMOLS: + return + + self.visit(node.type) + + if node.name == self.get_type_name(node.type): + return + else: + self.ret += f"{node.name} = {self.get_type_name(node.type)}\n" + self.ret += "\n" + + def visit_FuncDecl(self, node): + # pylint: disable=invalid-name + restype = self.get_type_name(node.type) + + if isinstance(node.type, c_ast.TypeDecl): + func_name = node.type.declname + elif isinstance(node.type, c_ast.PtrDecl): + func_name = node.type.type.declname + else: + raise RuntimeError(f"unexpected type in FuncDecl: {type}") + + if not func_name.startswith("wasm_") or func_name.endswith("_t"): + return + + if func_name in IGNORE_SYMOLS: + return + + params_len = 0 + for arg in node.args.params: + # ignore void but not void* + if isinstance(arg.type, c_ast.TypeDecl): + type_name = self.get_type_name(arg.type) + if "None" == type_name: + continue + + params_len += 1 + + args = ( + "" if not params_len else ",".join([f"arg{i}" for i in range(params_len)]) + ) + argtypes = f"[{self.get_type_name(node.args)}]" if params_len else "None" + + self.ret += ( + f"def {func_name}({args}):\n" + f"{INDENT}_{func_name} = libiwasm.{func_name}\n" + f"{INDENT}_{func_name}.restype = {restype}\n" + f"{INDENT}_{func_name}.argtypes = {argtypes}\n" + f"{INDENT}return _{func_name}({args})\n" + ) + self.ret += "\n" + + def visit_Enum(self, node): + # pylint: disable=invalid-name + elem_value = 0 + # generate enum elementes directly as consts with values + for i, elem in enumerate(node.values.enumerators): + self.ret += f"{elem.name}" + + if elem.value: + elem_value = int(elem.value.value) + else: + if 0 == i: + elem_value = 0 + else: + elem_value += 1 + + self.ret += f" = {elem_value}\n" + + self.ret += "\n" + + +def preflight_check(workspace): + wamr_repo = workspace + file_check_list = [ + wamr_repo.exists(), + wamr_repo.joinpath(WASM_C_API_HEADER).exists(), + ] + + if not all(file_check_list): + print( + "please run utils/download_wamr.py to download the repo, or re-download the repo" + ) + return False + + if not shutil.which("gcc"): + print("please install gcc") + return False + + return True + + +def do_parse(workspace): + filename = workspace.joinpath(WASM_C_API_HEADER) + filename = str(filename) + + ast = parse_file( + filename, + use_cpp=True, + cpp_path="gcc", + cpp_args=[ + "-E", + "-D__attribute__(x)=", + "-D__asm__(x)=", + "-D__asm(x)=", + "-D__builtin_va_list=int", + "-D__extension__=", + "-D__inline__=", + "-D__restrict=", + "-D__restrict__=", + "-D_Static_assert(x, y)=", + "-D__signed=", + "-D__volatile__(x)=", + "-Dstatic_assert(x, y)=", + ], + ) + + ast_visitor = Visitor() + ast_visitor.visit(ast) + return ast_visitor.ret + + +def main(): + current_file = pathlib.Path(__file__) + if current_file.is_symlink(): + current_file = pathlib.Path(os.readlink(current_file)) + + current_dir = current_file.parent.resolve() + root_dir = current_dir.joinpath("../../../..").resolve() + + if not preflight_check(root_dir): + return False + + wamr_repo = root_dir + binding_file_path = root_dir.joinpath(BINDING_PATH) + with open(binding_file_path, "wt", encoding="utf-8") as binding_file: + binding_file.write(do_parse(wamr_repo)) + + return True + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/wasm-micro-runtime/language-bindings/rust/README.md b/wasm-micro-runtime/language-bindings/rust/README.md new file mode 100644 index 0000000..06bdbd2 --- /dev/null +++ b/wasm-micro-runtime/language-bindings/rust/README.md @@ -0,0 +1 @@ +We use a separate repository to host the WAMR Rust SDK. Please refer to [WAMR Rust SDK](https://github.com/bytecodealliance/wamr-rust-sdk) for more details. diff --git a/wasm-micro-runtime/product-mini/README.md b/wasm-micro-runtime/product-mini/README.md new file mode 100644 index 0000000..18813be --- /dev/null +++ b/wasm-micro-runtime/product-mini/README.md @@ -0,0 +1,473 @@ +# Build iwasm + +iwasm is the executable binary built with WAMR VMcore supports WASI and command line interface. Refer to [**how to build wamr vmcore**](../doc/build_wamr.md) for all the supported CMAKE compilation variables. + +If you are building for ARM architecture on a X86 development machine, you can use the `CMAKE_TOOLCHAIN_FILE` to set the toolchain file for cross compling. + +``` +cmake .. -DCMAKE_TOOLCHAIN_FILE=$TOOL_CHAIN_FILE \ + -DWAMR_BUILD_PLATFORM=linux \ + -DWAMR_BUILD_TARGET=ARM +``` + +Refer to toolchain sample file [`wamr-app-framework/samples/simple/profiles/arm-interp/toolchain.cmake`](https://github.com/bytecodealliance/wamr-app-framework/blob/main/samples/simple/profiles/arm-interp/toolchain.cmake) for how to build mini product for ARM target architecture. + +If you compile for ESP-IDF, make sure to set the right toolchain file for the chip you're using (e.g. `$IDF_PATH/tools/cmake/toolchain-esp32c3.cmake`). +Note that all ESP-IDF toolchain files live under `$IDF_PATH/tools/cmake/`. + +## Linux + +First of all please install the dependent packages. +Run command below in Ubuntu-22.04: +``` Bash +sudo apt install build-essential cmake g++-multilib libgcc-11-dev lib32gcc-11-dev ccache +``` +Or in Ubuntu-20.04 +``` Bash +sudo apt install build-essential cmake g++-multilib libgcc-9-dev lib32gcc-9-dev ccache +``` +Or in Ubuntu-18.04: +``` Bash +sudo apt install build-essential cmake g++-multilib libgcc-8-dev lib32gcc-8-dev ccache +``` +Or in Fedora: +``` Bash +sudo dnf install glibc-devel.i686 +``` + +After installing dependencies, build the source code: +``` Bash +cd product-mini/platforms/linux/ +mkdir build && cd build +cmake .. +make +# iwasm is generated under current directory +``` + +By default in Linux, the `fast interpreter`, `AOT` and `Libc WASI` are enabled, and JIT is disabled. +And the build target is set to X86_64 or X86_32 depending on the platform's bitwidth. + +There are total 6 running modes supported: fast interpreter, classi interpreter, AOT, LLVM JIT, Fast JIT and Multi-tier JIT. + +(1) To run a wasm file with `fast interpreter` mode - build iwasm with default build and then: +```Bash +iwasm +``` +Or +```Bash +mkdir build && cd build +cmake .. -DWAMR_BUILD_INTERP=1 +make +``` + +(2) To disable `fast interpreter` and enable `classic interpreter` instead: +``` Bash +mkdir build && cd build +cmake .. -DWAMR_BUILD_FAST_INTERP=0 +make +``` + +(3) To run an AOT file, firstly please refer to [Build wamrc AOT compiler](../wamr-compiler/README.md) to build wamrc, and then: +```Bash +wamrc -o +iwasm +``` + +(4) To enable the `LLVM JIT` mode, firstly we should build the LLVM library: +``` Bash +cd product-mini/platforms/linux/ +./build_llvm.sh (The llvm source code is cloned under /core/deps/llvm and auto built) +``` + +Then pass argument `-DWAMR_BUILD_JIT=1` to cmake to enable LLVM JIT: +``` Bash +mkdir build && cd build +cmake .. -DWAMR_BUILD_JIT=1 +make +``` + +Note: +By default, the LLVM Orc JIT with Lazy compilation is enabled to speedup the lanuching process and reduce +the JIT compilation time by creating backend threads to compile the WASM functions parallely, and for the +main thread, the functions in the module will not be compiled until they are firstly called and haven't been +compiled by the compilation threads. + +If developer wants to disable the Lazy compilation, we can: +``` Bash +mkdir build && cd build +cmake .. -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0 +make +``` +In which all the WASM functions will be previously compiled before main thread starts to run the wasm module. + +(5) To enable the `Fast JIT` mode: +``` Bash +mkdir build && cd build +cmake .. -DWAMR_BUILD_FAST_JIT=1 +make +``` +The Fast JIT is a lightweight JIT engine with quick startup, small footprint and good portability, and gains ~50% performance of AOT. + +(6) To enable the `Multi-tier JIT` mode: +``` Bash +mkdir build && cd build +cmake .. -DWAMR_BUILD_FAST_JTI=1 -DWAMR_BUILD_JIT=1 +make +``` +The Multi-tier JIT is a two level JIT tier-up engine, which launchs Fast JIT to run the wasm module as soon as possible and creates backend threads to compile the LLVM JIT functions at the same time, and when the LLVM JIT functions are compiled, the runtime will switch the extecution from the Fast JIT jitted code to LLVM JIT jitted code gradually, so as to gain the best performance. + +## Linux SGX (Intel Software Guard Extension) + + +Please see [Build and Port WAMR vmcore for Linux SGX](../doc/linux_sgx.md) for the details. + +## MacOS + + +Make sure to install Xcode from App Store firstly, and install cmake. + +If you use Homebrew, install cmake from the command line: +``` Bash +brew install cmake +``` + +Then build the source codes: +``` Bash +cd product-mini/platforms/darwin/ +mkdir build +cd build +cmake .. +make +# iwasm is generated under current directory +``` +By default in MacOS, the `fast interpreter`, `AOT` and `Libc WASI` are enabled, and JIT is disabled. +And the build target is set to X86_64 or X86_32 depending on the platform's bitwidth. + +To run a wasm file with interpreter mode: +```Bash +iwasm +``` +To run an AOT file, firstly please refer to [Build wamrc AOT compiler](../wamr-compiler/README.md) to build wamrc, and then: +```Bash +wamrc -o +iwasm +``` +Note: +For how to build the `JIT` mode and `classic interpreter` mode, please refer to [Build iwasm on Linux](../doc/build_wamr.md#linux). + +WAMR provides some features which can be easily configured by passing options to cmake, please see [WAMR vmcore cmake building configurations](../doc/build_wamr.md#wamr-vmcore-cmake-building-configurations) for details. Currently in MacOS, interpreter, AOT, and builtin libc are enabled by default. + +## Windows + + +Make sure `MSVC` and `cmake` are installed and available in the command line environment + +Then build the source codes: +``` Bash +cd product-mini/platforms/windows/ +mkdir build +cd build +cmake .. +cmake --build . --config Release +# ./Release/iwasm.exe is generated +``` + +By default in Windows, the `fast interpreter`, `AOT` and `Libc WASI` are enabled, and JIT is disabled. + +To run a wasm file with interpreter mode: +```Bash +iwasm.exe +``` +To run an AOT file, firstly please refer to [Build wamrc AOT compiler](../wamr-compiler/README.md) to build wamrc, and then: +```Bash +wamrc.exe -o +iwasm.exe +``` +Note: +For how to build the `JIT` mode and `classic interpreter` mode, please refer to [Build iwasm on Linux](../doc/build_wamr.md#linux). + +WAMR provides some features which can be easily configured by passing options to cmake, please see [WAMR vmcore cmake building configurations](../doc/build_wamr.md#wamr-vmcore-cmake-building-configurations) for details. Currently in Windows, interpreter, AOT, and builtin libc are enabled by default. + +## MinGW + + +First make sure the correct CMake package is installed; the following commands +are valid for the MSYS2 build environment: + +```Bash +pacman -R cmake +pacman -S mingw-w64-x86_64-cmake +pacman -S mingw-w64-x86_64-gcc +pacman -S make git +``` + +Then follow the build instructions for Windows above, and add the following +arguments for cmake: + +```Bash +cmake .. -G"Unix Makefiles" \ + -DWAMR_DISABLE_HW_BOUND_CHECK=1 +```` + +Note that WASI will be disabled until further work is done towards full MinGW support. + +- Since memory access boundary check with hardware trap feature is disabled, when generating the AOT file with `wamrc`, the `--bounds-checks=1` flag should be added to generate the memory access boundary check instructions to ensure the sandbox security: +```bash +wamrc --bounds-checks=1 -o +``` +- Compiler complaining about missing `UnwindInfoAddress` field in `RUNTIME_FUNCTION` +struct (winnt.h). + + +## VxWorks + +VxWorks 7 SR0620 release is validated. + +First you need to build a VSB. Make sure *UTILS_UNIX* layer is added in the VSB. +After the VSB is built, export the VxWorks toolchain path by: +```bash +export /host/vx-compiler/bin:$PATH +``` +Now switch to iwasm source tree to build the source code: +```bash +cd product-mini/platforms/vxworks/ +mkdir build +cd build +cmake .. +make +``` +Create a VIP based on the VSB. Make sure the following components are added: +* INCLUDE_POSIX_PTHREADS +* INCLUDE_POSIX_PTHREAD_SCHEDULER +* INCLUDE_SHARED_DATA +* INCLUDE_SHL + +Copy the generated iwasm executable, the test WASM binary as well as the needed +shared libraries (libc.so.1, libllvm.so.1 or libgnu.so.1 depending on the VSB, +libunix.so.1) to a supported file system (eg: romfs). + +Note: +WAMR provides some features which can be easily configured by passing options to cmake, please see [WAMR vmcore cmake building configurations](../doc/build_wamr.md#wamr-vmcore-cmake-building-configurations) for details. Currently in VxWorks, interpreter and builtin libc are enabled by default. + +## Zephyr +Please refer to this [README](./platforms/zephyr/simple/README.md) under the Zephyr sample directory for details. + +Note: +WAMR provides some features which can be easily configured by passing options to cmake, please see [WAMR vmcore cmake building configurations](../doc/build_wamr.md#wamr-vmcore-cmake-building-configurations) for details. Currently in Zephyr, interpreter, AOT and builtin libc are enabled by default. + + +## RT-Thread + + +1. Get rt-thread [system codes](https://github.com/RT-Thread/rt-thread). + +2. Enable WAMR software package with menuconfig tool which provided by RT-Thread. + + * Environment in Linux, run command below: + + ```bash + scons --menuconfig + ``` + + * Environment in Windows ConEmu, run command below: + + ```bash + menuconfig + ``` + + Select and enable `WAMR` in: + + * RT-Thread online packages + * tools packages + * WebAssembly Micro Runtime (WAMR) + +3. Configure `WAMR` with menuconfig tool. + + you can choice features of iwasm below: + + * Enable testing parameters of iwasm + * Enable interpreter Mode / Fast interpreter Mode + * Use built-libc + * Enable AOT + +4. Exit menuconfig tool and save configure, update and download package. + + ```bash + pkgs --update + ``` + +5. build project and download the binary to boards. + + ```bash + scons + ``` + + or build project with 8-thread by using command below: + + ```bash + scons -j8 + ``` + + after project building, you can got an binary file named `rtthread.bin`, then you can download this file to the MCU board. + +## Android + +Able to generate a shared library support Android platform. +- need an [android SDK](https://developer.android.com/studio). Go and get the "Command line tools only" +- look for a command named *sdkmanager* and download below components. version numbers might need to check and pick others + - "build-tools;29.0.3" + - "cmake;3.10.2.4988404" + - "ndk;latest" + - "patcher;v4" + - "platform-tools" + - "platforms;android-29" +- add bin/ of the downloaded cmake to $PATH +- export ANDROID_HOME=/the/path/of/downloaded/sdk/ +- export ANDROID_NDK_LATEST_HOME=/the/path/of/downloaded/sdk/ndk/2x.xxx/ +- ready to go + +Use such commands, you are able to compile with default configurations. + +``` shell +$ cd product-mini/platforms/android/ +$ mkdir build +$ cd build +$ cmake .. +$ make +$ # check output in distribution/wasm +$ # include/ includes all necesary head files +$ # lib includes libiwasm.so +``` + +To change the target architecture and ABI, you can define `WAMR_BUILD_TARGET` or `ANDROID_ABI` respectively. To build for [supported Android ABIs](https://developer.android.com/ndk/guides/abis#sa): + +```shell +$ cmake .. -DWAMR_BUILD_TARGET=X86_32 -DANDROID_ABI=x86 # 32-bit Intel CPU +$ cmake .. -DWAMR_BUILD_TARGET=X86_64 -DANDROID_ABI=x86_64 # 64-bit Intel CPU +$ cmake .. -DWAMR_BUILD_TARGET=ARMV7A -DANDROID_ABI=armeabi-v7a # 32-bit ARM CPU +$ cmake .. -DWAMR_BUILD_TARGET=AARCH64 -DANDROID_ABI=arm64-v8a # 64-bit ARM CPU +``` + +## NuttX + +WAMR is intergrated with NuttX, just enable the WAMR in Kconfig option (Application Configuration/Interpreters). + +## ESP-IDF + +WAMR integrates with ESP-IDF both for the XTENSA and RISC-V chips (esp32x and esp32c3 respectively). + +In order to use this, you need at least version 4.3.1 of ESP-IDF. +If you don't have it installed, follow the instructions [here](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/#get-started-get-prerequisites). +ESP-IDF also installs the toolchains needed for compiling WAMR and ESP-IDF. +A small demonstration of how to use WAMR and ESP-IDF can be found under [product_mini](./platforms/esp-idf). +The demo builds WAMR for ESP-IDF and runs a small wasm program. +In order to run it for your specific Espressif chip, edit the [build_and_run.sh](./platforms/esp-idf/build_and_run.sh) file and put the correct toolchain file (see #Cross-compilation) and `IDF_TARGET`. +Before compiling it is also necessary to call ESP-IDF's `export.sh` script to bring all compile time relevant information in scope. + +## Docker + +[Docker](https://www.docker.com/) will download all the dependencies and build WAMR Core on your behalf. + +Make sure you have Docker installed on your machine: [macOS](https://docs.docker.com/docker-for-mac/install/), [Windows](https://docs.docker.com/docker-for-windows/install/) or [Linux](https://docs.docker.com/install/linux/docker-ce/ubuntu/). + +Build *iwasm* with the Docker image: + +``` Bash +$ cd ci +$ ./build_wamr.sh +$ ls ../build_out/ +``` + +*build_wamr.sh* will generate *linux* compatible libraries ( libiwasm.so and +libvmlib.a ) and an executable binary (*iwasm*) and copy *iwasm* to +*build_out*. All original generated files are still under +*product-mini/platforms/linux/build*. + +## FreeBSD + +First, install the dependent packages: +```shell +sudo pkg install gcc cmake wget +``` + +Then you can run the following commands to build iwasm with default configurations: +```shell +cd product-mini/platforms/freebsd +mkdir build && cd build +cmake .. +make +``` + + +## AliOS-Things + +1. a developerkit board id needed for testing +2. download the AliOS-Things code + ``` Bash + git clone https://github.com/alibaba/AliOS-Things.git + ``` +3. copy /product-mini/platforms/alios-things directory to AliOS-Things/middleware, and rename it as iwasm + ``` Bash + cp -a /product-mini/platforms/alios-things middleware/iwasm + ``` +4. create a link to in middleware/iwasm/ and rename it to wamr + ``` Bash + ln -s middleware/iwasm/wamr + ``` +5. modify file app/example/helloworld/helloworld.c, patch as: + ``` C + #include + #include + extern bool iwasm_init(); + int application_start(int argc, char *argv[]) + { + int count = 0; + iwasm_init(); + ... + } + ``` +6. modify file app/example/helloworld/aos.mk + ``` C + $(NAME)_COMPONENTS := osal_aos iwasm + ``` +7. build source code and run + For linux host: + + ``` Bash + aos make helloworld@linuxhost -c config + aos make + ./out/helloworld@linuxhost/binary/helloworld@linuxhost.elf + ``` + + For developerkit: + Modify file middleware/iwasm/aos.mk, patch as: + + ``` C + WAMR_BUILD_TARGET := THUMBV7M + ``` + + ``` Bash + aos make helloworld@developerkit -c config + aos make + ``` + download the binary to developerkit board, check the output from serial port + +## Cosmopolitan Libc +Currently, only x86_64 architecture with interpreter modes is supported. + +Setup `cosmocc` as described in [Getting Started](https://github.com/jart/cosmopolitan/#getting-started) being sure to get its `bin` directory into `PATH`. + +Build iwasm +``` Bash +export CC=x86_64-unknown-cosmo-cc +export CXX=x86_64-unknown-cosmo-c++ +rm -rf build +mkdir build +cmake -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=1 -B build +cmake --build build -j +``` + +Run like +``` Bash +./build/iwasm.com +``` diff --git a/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/CMakeLists.txt b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/CMakeLists.txt new file mode 100644 index 0000000..b41fe0a --- /dev/null +++ b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/CMakeLists.txt @@ -0,0 +1,13 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.5) +project(hello_world) + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -Wno-unused-command-line-argument") +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS},--export=main") + +add_library(print print.c) + +add_executable(hello_world main.c) +target_link_libraries(hello_world print) \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/build.sh b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/build.sh new file mode 100755 index 0000000..7b37eb2 --- /dev/null +++ b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/build.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +rm -rf build +mkdir build +cd build +cmake .. -DCMAKE_TOOLCHAIN_FILE=../../../../wamr-sdk/app/wamr_toolchain.cmake +make \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/main.c b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/main.c new file mode 100644 index 0000000..b6dfd28 --- /dev/null +++ b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/main.c @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "stdio.h" + +void +print_line(char *str); + +int +main() +{ + print_line("Hello World!"); + print_line("Wasm Micro Runtime"); + return 0; +} \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/print.c b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/print.c new file mode 100644 index 0000000..d98a826 --- /dev/null +++ b/wasm-micro-runtime/product-mini/app-samples/hello-world-cmake/print.c @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "stdio.h" +#include "string.h" + +void +print_line(char *str) +{ + printf("%s\n", str); +} \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/app-samples/hello-world/build.sh b/wasm-micro-runtime/product-mini/app-samples/hello-world/build.sh new file mode 100755 index 0000000..5dc612b --- /dev/null +++ b/wasm-micro-runtime/product-mini/app-samples/hello-world/build.sh @@ -0,0 +1,25 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +WAMR_DIR=${PWD}/../../.. + +echo "Build wasm app .." +/opt/wasi-sdk/bin/clang -O3 \ + -z stack-size=4096 -Wl,--initial-memory=65536 \ + -o test.wasm main.c \ + -Wl,--export=main -Wl,--export=__main_argc_argv \ + -Wl,--export=__data_end -Wl,--export=__heap_base \ + -Wl,--strip-all,--no-entry \ + -Wl,--allow-undefined \ + -nostdlib \ + +echo "Build binarydump tool .." +rm -fr build && mkdir build && cd build +cmake ../../../../test-tools/binarydump-tool +make +cd .. + +echo "Generate test_wasm.h .." +./build/binarydump -o test_wasm.h -n wasm_test_file test.wasm + +echo "Done" diff --git a/wasm-micro-runtime/product-mini/app-samples/hello-world/main.c b/wasm-micro-runtime/product-mini/app-samples/hello-world/main.c new file mode 100644 index 0000000..4778a4d --- /dev/null +++ b/wasm-micro-runtime/product-mini/app-samples/hello-world/main.c @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +int +main(int argc, char **argv) +{ + char *buf; + + printf("Hello world!\n"); + + buf = malloc(1024); + if (!buf) { + printf("malloc buf failed\n"); + return -1; + } + + printf("buf ptr: %p\n", buf); + + snprintf(buf, 1024, "%s", "1234\n"); + printf("buf: %s", buf); + + free(buf); + return 0; +} diff --git a/wasm-micro-runtime/product-mini/platforms/alios-things/aos.mk b/wasm-micro-runtime/product-mini/platforms/alios-things/aos.mk new file mode 100644 index 0000000..3f25cb9 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/alios-things/aos.mk @@ -0,0 +1,131 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +NAME := iwasm +CORE_ROOT := wamr/core +IWASM_ROOT := wamr/core/iwasm +SHARED_ROOT := wamr/core/shared + +GLOBAL_DEFINES += BH_MALLOC=wasm_runtime_malloc +GLOBAL_DEFINES += BH_FREE=wasm_runtime_free + +# Change it to THUMBV7M if you want to build for developerkit +WAMR_BUILD_TARGET := X86_32 + +WAMR_BUILD_PLATFORM := alios-things + +ifeq (${WAMR_BUILD_TARGET}, X86_32) + GLOBAL_DEFINES += BUILD_TARGET_X86_32 + INVOKE_NATIVE := invokeNative_ia32.s + AOT_RELOC := aot_reloc_x86_32.c +else ifeq (${WAMR_BUILD_TARGET}, X86_64) + GLOBAL_DEFINES += BUILD_TARGET_X86_64 + INVOKE_NATIVE := invokeNative_em64.s + AOT_RELOC := aot_reloc_x86_64.c +else ifeq ($(findstring ARM,$(WAMR_BUILD_TARGET)), ARM) + GLOBAL_DEFINES += BUILD_TARGET_ARM + GLOBAL_DEFINES += BUILD_TARGET=\"$(WAMR_BUILD_TARGET)\" + INVOKE_NATIVE := invokeNative_arm.s + AOT_RELOC := aot_reloc_arm.c +else ifeq ($(findstring THUMB,$(WAMR_BUILD_TARGET)), THUMB) + GLOBAL_DEFINES += BUILD_TARGET_THUMB + GLOBAL_DEFINES += BUILD_TARGET=\"$(WAMR_BUILD_TARGET)\" + INVOKE_NATIVE := invokeNative_thumb.s + AOT_RELOC := aot_reloc_thumb.c +else ifeq (${WAMR_BUILD_TARGET}, MIPS) + GLOBAL_DEFINES += BUILD_TARGET_MIPS + INVOKE_NATIVE := invokeNative_mips.s + AOT_RELOC := aot_reloc_mips.c +else ifeq (${WAMR_BUILD_TARGET}, XTENSA) + GLOBAL_DEFINES += BUILD_TARGET_XTENSA + INVOKE_NATIVE := invokeNative_xtensa.s + AOT_RELOC := aot_reloc_xtensa.c +else + $(error Build target isn't set) +endif + +# Enable Interpreter by default. +WAMR_BUILD_INTERP = 1 + +# Enable AOT by default. +WAMR_BUILD_AOT = 1 + +# Override the global heap usage +ifndef WAMR_BUILD_GLOBAL_HEAP_POOL +WAMR_BUILD_GLOBAL_HEAP_POOL=1 +endif +GLOBAL_DEFINES += WASM_ENABLE_GLOBAL_HEAP_POOL=${WAMR_BUILD_GLOBAL_HEAP_POOL} + +# Override the global heap size for small devices +ifndef WAMR_BUILD_GLOBAL_HEAP_SIZE +WAMR_BUILD_GLOBAL_HEAP_SIZE = 262144 # 256 kB +endif +GLOBAL_DEFINES += WASM_GLOBAL_HEAP_SIZE=${WAMR_BUILD_GLOBAL_HEAP_SIZE} + +ifeq (${WAMR_BUILD_INTERP}, 1) +GLOBAL_DEFINES += WASM_ENABLE_INTERP=1 +endif + +ifeq (${WAMR_BUILD_AOT}, 1) +GLOBAL_DEFINES += WASM_ENABLE_AOT=1 +endif + +GLOBAL_DEFINES += WASM_ENABLE_LIBC_BUILTIN=1 + +GLOBAL_INCLUDES += ${CORE_ROOT} \ + ${IWASM_ROOT}/include \ + ${IWASM_ROOT}/common \ + ${SHARED_ROOT}/include \ + ${SHARED_ROOT}/platform/include \ + ${SHARED_ROOT}/utils \ + ${SHARED_ROOT}/mem-alloc \ + ${SHARED_ROOT}/platform/alios + +ifeq (${WAMR_BUILD_INTERP}, 1) +GLOBAL_INCLUDES += ${IWASM_ROOT}/interpreter +endif + +ifeq (${WAMR_BUILD_AOT}, 1) +GLOBAL_INCLUDES += ${IWASM_ROOT}/aot +endif + +$(NAME)_SOURCES := ${SHARED_ROOT}/platform/alios/alios_platform.c \ + ${SHARED_ROOT}/platform/alios/alios_thread.c \ + ${SHARED_ROOT}/platform/alios/alios_time.c \ + ${SHARED_ROOT}/platform/common/math/math.c \ + ${SHARED_ROOT}/mem-alloc/mem_alloc.c \ + ${SHARED_ROOT}/mem-alloc/ems/ems_kfc.c \ + ${SHARED_ROOT}/mem-alloc/ems/ems_alloc.c \ + ${SHARED_ROOT}/mem-alloc/ems/ems_hmu.c \ + ${SHARED_ROOT}/utils/bh_assert.c \ + ${SHARED_ROOT}/utils/bh_bitmap.c \ + ${SHARED_ROOT}/utils/bh_common.c \ + ${SHARED_ROOT}/utils/bh_hashmap.c \ + ${SHARED_ROOT}/utils/bh_list.c \ + ${SHARED_ROOT}/utils/bh_log.c \ + ${SHARED_ROOT}/utils/bh_queue.c \ + ${SHARED_ROOT}/utils/bh_vector.c \ + ${SHARED_ROOT}/utils/runtime_timer.c \ + ${IWASM_ROOT}/libraries/libc-builtin/libc_builtin_wrapper.c \ + ${IWASM_ROOT}/common/wasm_application.c \ + ${IWASM_ROOT}/common/wasm_runtime_common.c \ + ${IWASM_ROOT}/common/wasm_native.c \ + ${IWASM_ROOT}/common/wasm_exec_env.c \ + ${IWASM_ROOT}/common/wasm_memory.c \ + ${IWASM_ROOT}/common/wasm_c_api.c \ + ${IWASM_ROOT}/common/arch/${INVOKE_NATIVE} \ + src/main.c + +ifeq (${WAMR_BUILD_INTERP}, 1) +$(NAME)_SOURCES += ${IWASM_ROOT}/interpreter/wasm_interp_classic.c \ + ${IWASM_ROOT}/interpreter/wasm_loader.c \ + ${IWASM_ROOT}/interpreter/wasm_runtime.c +endif + +ifeq (${WAMR_BUILD_AOT}, 1) +$(NAME)_SOURCES += ${IWASM_ROOT}/aot/aot_loader.c \ + ${IWASM_ROOT}/aot/arch/${AOT_RELOC} \ + ${IWASM_ROOT}/aot/aot_runtime.c \ + ${IWASM_ROOT}/aot/aot_intrinsic.c +endif + diff --git a/wasm-micro-runtime/product-mini/platforms/alios-things/src/main.c b/wasm-micro-runtime/product-mini/platforms/alios-things/src/main.c new file mode 100644 index 0000000..f2c6f85 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/alios-things/src/main.c @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include "bh_platform.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "test_wasm.h" + +static int app_argc; +static char **app_argv; + +/** + * Find the unique main function from a WASM module instance + * and execute that function. + * + * @param module_inst the WASM module instance + * @param argc the number of arguments + * @param argv the arguments array + * + * @return true if the main function is called, false otherwise. + */ +bool +wasm_application_execute_main(wasm_module_inst_t module_inst, int32_t argc, + char *argv[]); + +static void * +app_instance_main(wasm_module_inst_t module_inst) +{ + const char *exception; + + wasm_application_execute_main(module_inst, app_argc, app_argv); + if ((exception = wasm_runtime_get_exception(module_inst))) + printf("%s\n", exception); + return NULL; +} + +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 +static char global_heap_buf[WASM_GLOBAL_HEAP_SIZE] = { 0 }; +#endif + +void +iwasm_main(void *arg1) +{ + uint8 *wasm_file_buf = NULL; + uint32 wasm_file_size; + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + RuntimeInitArgs init_args; + char error_buf[128]; +#if WASM_ENABLE_LOG != 0 + int log_verbose_level = 2; +#endif + + (void)arg1; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); +#else +#error Another memory allocation scheme than global heap pool is not implemented yet for Alios. +#endif + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return; + } + +#if WASM_ENABLE_LOG != 0 + bh_log_set_verbose_level(log_verbose_level); +#endif + + /* load WASM byte buffer from byte buffer of include file */ + wasm_file_buf = (uint8 *)wasm_test_file; + wasm_file_size = sizeof(wasm_test_file); + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail1; + } + + /* instantiate the module */ + if (!(wasm_module_inst = wasm_runtime_instantiate( + wasm_module, 8 * 1024, 8 * 1024, error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail2; + } + + app_instance_main(wasm_module_inst); + + /* destroy the module instance */ + wasm_runtime_deinstantiate(wasm_module_inst); + +fail2: + /* unload the module */ + wasm_runtime_unload(wasm_module); + +fail1: + /* destroy runtime environment */ + wasm_runtime_destroy(); +} + +#define DEFAULT_THREAD_STACKSIZE (6 * 1024) +#define DEFAULT_THREAD_PRIORITY 50 + +bool +iwasm_init(void) +{ + int ret = + aos_task_new("wasm-main", iwasm_main, NULL, DEFAULT_THREAD_STACKSIZE); + return ret == 0 ? true : false; +} diff --git a/wasm-micro-runtime/product-mini/platforms/alios-things/src/test_wasm.h b/wasm-micro-runtime/product-mini/platforms/alios-things/src/test_wasm.h new file mode 100644 index 0000000..0c34302 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/alios-things/src/test_wasm.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * The byte array buffer is the file content of a test wasm binary file, + * which is compiled by wasi-sdk toolchain from C source file of: + * product-mini/app-samples/hello-world/main.c. + */ +unsigned char wasm_test_file[] = { + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x10, 0x03, 0x60, + 0x01, 0x7F, 0x01, 0x7F, 0x60, 0x02, 0x7F, 0x7F, 0x01, 0x7F, 0x60, 0x01, + 0x7F, 0x00, 0x02, 0x31, 0x04, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x70, 0x75, + 0x74, 0x73, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x6D, 0x61, 0x6C, + 0x6C, 0x6F, 0x63, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x70, 0x72, + 0x69, 0x6E, 0x74, 0x66, 0x00, 0x01, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x66, + 0x72, 0x65, 0x65, 0x00, 0x02, 0x03, 0x02, 0x01, 0x01, 0x04, 0x05, 0x01, + 0x70, 0x01, 0x01, 0x01, 0x05, 0x03, 0x01, 0x00, 0x01, 0x06, 0x13, 0x03, + 0x7F, 0x01, 0x41, 0xC0, 0x28, 0x0B, 0x7F, 0x00, 0x41, 0xBA, 0x08, 0x0B, + 0x7F, 0x00, 0x41, 0xC0, 0x28, 0x0B, 0x07, 0x2C, 0x04, 0x06, 0x6D, 0x65, + 0x6D, 0x6F, 0x72, 0x79, 0x02, 0x00, 0x0A, 0x5F, 0x5F, 0x64, 0x61, 0x74, + 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x03, 0x01, 0x0B, 0x5F, 0x5F, 0x68, 0x65, + 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, 0x65, 0x03, 0x02, 0x04, 0x6D, 0x61, + 0x69, 0x6E, 0x00, 0x04, 0x0A, 0xB2, 0x01, 0x01, 0xAF, 0x01, 0x01, 0x03, + 0x7F, 0x23, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, + 0x24, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x9B, 0x88, 0x80, 0x80, 0x00, + 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x02, 0x40, 0x02, 0x40, 0x41, + 0x80, 0x08, 0x10, 0x81, 0x80, 0x80, 0x80, 0x00, 0x22, 0x03, 0x0D, 0x00, + 0x41, 0xA8, 0x88, 0x80, 0x80, 0x00, 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, + 0x1A, 0x41, 0x7F, 0x21, 0x04, 0x0C, 0x01, 0x0B, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x10, 0x41, 0x80, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x41, + 0x10, 0x6A, 0x10, 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x41, 0x00, 0x21, + 0x04, 0x20, 0x03, 0x41, 0x04, 0x6A, 0x41, 0x00, 0x2F, 0x00, 0x91, 0x88, + 0x80, 0x80, 0x00, 0x3B, 0x00, 0x00, 0x20, 0x03, 0x41, 0x00, 0x28, 0x00, + 0x8D, 0x88, 0x80, 0x80, 0x00, 0x36, 0x00, 0x00, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x00, 0x41, 0x93, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x10, + 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x20, 0x03, 0x10, 0x83, 0x80, 0x80, + 0x80, 0x00, 0x0B, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x80, 0x80, 0x80, + 0x80, 0x00, 0x20, 0x04, 0x0B, 0x0B, 0x41, 0x01, 0x00, 0x41, 0x80, 0x08, + 0x0B, 0x3A, 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, 0x20, 0x25, + 0x70, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, 0x75, 0x66, + 0x3A, 0x20, 0x25, 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, + 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, 0x6F, 0x63, + 0x20, 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, 0x64, 0x00 +}; diff --git a/wasm-micro-runtime/product-mini/platforms/android/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/android/CMakeLists.txt new file mode 100644 index 0000000..db60e86 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/android/CMakeLists.txt @@ -0,0 +1,139 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +if (NOT DEFINED WAMR_BUILD_TARGET) + message (FATAL_ERROR "WAMR_BUILD_TARGET isn't set") +endif () + +if (NOT (WAMR_BUILD_TARGET STREQUAL "X86_64" + OR WAMR_BUILD_TARGET STREQUAL "X86_32" + OR WAMR_BUILD_TARGET MATCHES "AARCH64.*" + OR WAMR_BUILD_TARGET MATCHES "ARM.*" + OR WAMR_BUILD_TARGET MATCHES "RISCV64.*")) + message (FATAL_ERROR "Unsupported build target platform ${WAMR_BUILD_TARGET}!") +endif () + +if (NOT DEFINED ANDROID_ABI) + if (WAMR_BUILD_TARGET STREQUAL "X86_64") + set (ANDROID_ABI "x86_64") + elseif (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (ANDROID_ABI "x86") + elseif (WAMR_BUILD_TARGET MATCHES "AARCH64.*") + set (ANDROID_ABI "arm64-v8a") + elseif (WAMR_BUILD_TARGET MATCHES "ARM.*") + set (ANDROID_ABI "armeabi-v7a") + else () + set (ANDROID_ABI "riscv64") + endif () +endif () + +if (NOT DEFINED ANDROID_LD) + set (ANDROID_LD lld) +endif () + +if (NOT DEFINED ANDROID_PLATFORM) + set (ANDROID_PLATFORM 24) +endif () + +# https://android.googlesource.com/platform/ndk/+/master/build/cmake/android.toolchain.cmake +set (CMAKE_TOOLCHAIN_FILE "$ENV{ANDROID_NDK_LATEST_HOME}/build/cmake/android.toolchain.cmake") +set (ANDROID_SDK $ENV{ANDROID_HOME}) +set (ANDROID_NDK $ENV{ANDROID_NDK_LATEST_HOME}) + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "android") + +set (CMAKE_VERBOSE_MAKEFILE ON) + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Disable multiple modules by default + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS) + # Disable wasi threads library by default + set (WAMR_BUILD_LIB_WASI_THREADS 0) +endif() + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections -pie -fPIE") + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") + +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () +endif () + +# The following flags are to enhance security, but it may impact performance, +# we disable them by default. +#if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftrapv -D_FORTIFY_SOURCE=2") +#endif () +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-strong --param ssp-buffer-size=4") +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack,-z,relro,-z,now") + +add_library (iwasm SHARED ${WAMR_RUNTIME_LIB_SOURCE}) +if (CMAKE_BUILD_TYPE STREQUAL Release) +target_link_libraries (iwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -landroid -llog -s) +else() +target_link_libraries (iwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -landroid -llog) +endif() + +set (distribution_DIR ${CMAKE_BINARY_DIR}/distribution) +set_target_properties (iwasm PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${distribution_DIR}/wasm/lib") + +add_custom_command (TARGET iwasm POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy_directory "${WAMR_ROOT_DIR}/core/iwasm/include" "${distribution_DIR}/wasm/include/" + COMMENT "Copying iwasm to output directory") diff --git a/wasm-micro-runtime/product-mini/platforms/android/build_jit.sh b/wasm-micro-runtime/product-mini/platforms/android/build_jit.sh new file mode 100755 index 0000000..ffa440e --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/android/build_jit.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +rm -fr build && mkdir build +cd build +cmake .. -DWAMR_BUILD_JIT=1 +make -j ${nroc} +cd .. diff --git a/wasm-micro-runtime/product-mini/platforms/android/build_llvm.sh b/wasm-micro-runtime/product-mini/platforms/android/build_llvm.sh new file mode 100755 index 0000000..145e2db --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/android/build_llvm.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copyright (C) 2020 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt +/usr/bin/env python3 ../../../build-scripts/build_llvm.py --platform android "$@" diff --git a/wasm-micro-runtime/product-mini/platforms/android/wasm-jni.cpp b/wasm-micro-runtime/product-mini/platforms/android/wasm-jni.cpp new file mode 100644 index 0000000..c2eff3a --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/android/wasm-jni.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include +#include + +#define LOGI(...) \ + ((void)__android_log_print(ANDROID_LOG_INFO, "wasm_jni::", __VA_ARGS__)) + +static void * +app_instance_main(wasm_module_inst_t module_inst) +{ + const char *exception; + + wasm_application_execute_main(module_inst, 0, NULL); + if ((exception = wasm_runtime_get_exception(module_inst))) + LOGI("%s\n", exception); + return NULL; +} + +// WARNING! CAN NOT BE READ ONLY!!! +static unsigned char wasm_test_file[] = { + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x10, 0x03, 0x60, + 0x01, 0x7F, 0x01, 0x7F, 0x60, 0x02, 0x7F, 0x7F, 0x01, 0x7F, 0x60, 0x01, + 0x7F, 0x00, 0x02, 0x31, 0x04, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x70, 0x75, + 0x74, 0x73, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x6D, 0x61, 0x6C, + 0x6C, 0x6F, 0x63, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x70, 0x72, + 0x69, 0x6E, 0x74, 0x66, 0x00, 0x01, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x66, + 0x72, 0x65, 0x65, 0x00, 0x02, 0x03, 0x02, 0x01, 0x01, 0x04, 0x05, 0x01, + 0x70, 0x01, 0x01, 0x01, 0x05, 0x03, 0x01, 0x00, 0x01, 0x06, 0x13, 0x03, + 0x7F, 0x01, 0x41, 0xC0, 0x28, 0x0B, 0x7F, 0x00, 0x41, 0xBA, 0x08, 0x0B, + 0x7F, 0x00, 0x41, 0xC0, 0x28, 0x0B, 0x07, 0x2C, 0x04, 0x06, 0x6D, 0x65, + 0x6D, 0x6F, 0x72, 0x79, 0x02, 0x00, 0x0A, 0x5F, 0x5F, 0x64, 0x61, 0x74, + 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x03, 0x01, 0x0B, 0x5F, 0x5F, 0x68, 0x65, + 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, 0x65, 0x03, 0x02, 0x04, 0x6D, 0x61, + 0x69, 0x6E, 0x00, 0x04, 0x0A, 0xB2, 0x01, 0x01, 0xAF, 0x01, 0x01, 0x03, + 0x7F, 0x23, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, + 0x24, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x9B, 0x88, 0x80, 0x80, 0x00, + 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x02, 0x40, 0x02, 0x40, 0x41, + 0x80, 0x08, 0x10, 0x81, 0x80, 0x80, 0x80, 0x00, 0x22, 0x03, 0x0D, 0x00, + 0x41, 0xA8, 0x88, 0x80, 0x80, 0x00, 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, + 0x1A, 0x41, 0x7F, 0x21, 0x04, 0x0C, 0x01, 0x0B, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x10, 0x41, 0x80, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x41, + 0x10, 0x6A, 0x10, 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x41, 0x00, 0x21, + 0x04, 0x20, 0x03, 0x41, 0x04, 0x6A, 0x41, 0x00, 0x2F, 0x00, 0x91, 0x88, + 0x80, 0x80, 0x00, 0x3B, 0x00, 0x00, 0x20, 0x03, 0x41, 0x00, 0x28, 0x00, + 0x8D, 0x88, 0x80, 0x80, 0x00, 0x36, 0x00, 0x00, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x00, 0x41, 0x93, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x10, + 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x20, 0x03, 0x10, 0x83, 0x80, 0x80, + 0x80, 0x00, 0x0B, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x80, 0x80, 0x80, + 0x80, 0x00, 0x20, 0x04, 0x0B, 0x0B, 0x41, 0x01, 0x00, 0x41, 0x80, 0x08, + 0x0B, 0x3A, 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, 0x20, 0x25, + 0x70, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, 0x75, 0x66, + 0x3A, 0x20, 0x25, 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, + 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, 0x6F, 0x63, + 0x20, 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, 0x64, 0x00 +}; + +extern "C" JNIEXPORT void JNICALL +Java_com_intel_wasm_api_Runtime_run(JNIEnv *env, jclass thiz) +{ + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + RuntimeInitArgs init_args; + uint wasm_file_size = 0; + uint8_t *wasm_file_buf = NULL; + char error_buf[128] = { 0 }; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + +#if WASM_ENABLE_GLOBAL_HEAP_POOL == 0 + init_args.mem_alloc_type = Alloc_With_Allocator; + init_args.mem_alloc_option.allocator.malloc_func = (void *)malloc; + init_args.mem_alloc_option.allocator.realloc_func = (void *)realloc; + init_args.mem_alloc_option.allocator.free_func = (void *)free; +#else +#error The usage of a global heap pool is not implemented yet for Android. +#endif + + LOGI("wasm_runtime_full_init"); + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + LOGI("Init runtime failed.\n"); + return; + } + + /* load WASM byte buffer from a preinstall WASM bin file */ + LOGI("use an internal test file, gona to output Hello World in logcat\n"); + wasm_file_buf = (uint8_t *)wasm_test_file; + wasm_file_size = sizeof(wasm_test_file); + + /* load WASM module */ + LOGI("wasm_runtime_load"); + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + LOGI("in wasm_runtime_load %s\n", error_buf); + LOGI("goto fail1\n"); + goto fail1; + } + + /* instantiate the module */ + LOGI("wasm_runtime_instantiate"); + if (!(wasm_module_inst = + wasm_runtime_instantiate(wasm_module, 64 * 1024, /* stack size */ + 64 * 1024, /* heap size */ + error_buf, sizeof(error_buf)))) { + LOGI("%s\n", error_buf); + LOGI("goto fail2\n"); + goto fail2; + } + + LOGI("run main() of the application"); + app_instance_main(wasm_module_inst); + + /* destroy the module instance */ + LOGI("wasm_runtime_deinstantiate"); + wasm_runtime_deinstantiate(wasm_module_inst); + +fail2: + /* unload the module */ + LOGI("wasm_runtime_unload"); + wasm_runtime_unload(wasm_module); + +fail1: + // in our case, we don't need a free, but it is not a typical one + /* free the file buffer */ + // bh_free((void *) wasm_file_buf); + + /* destroy runtime environment */ + LOGI("wasm_runtime_destroy"); + wasm_runtime_destroy(); + return; +} diff --git a/wasm-micro-runtime/product-mini/platforms/common/libc_wasi.c b/wasm-micro-runtime/product-mini/platforms/common/libc_wasi.c new file mode 100644 index 0000000..84e133b --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/common/libc_wasi.c @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include "bh_platform.h" +#include "wasm_export.h" + +typedef struct { + const char *dir_list[8]; + uint32 dir_list_size; + const char *map_dir_list[8]; + uint32 map_dir_list_size; + const char *env_list[8]; + uint32 env_list_size; + const char *addr_pool[8]; + uint32 addr_pool_size; + const char *ns_lookup_pool[8]; + uint32 ns_lookup_pool_size; +} libc_wasi_parse_context_t; + +typedef enum { + LIBC_WASI_PARSE_RESULT_OK = 0, + LIBC_WASI_PARSE_RESULT_NEED_HELP, + LIBC_WASI_PARSE_RESULT_BAD_PARAM +} libc_wasi_parse_result_t; + +static void +libc_wasi_print_help() +{ + printf(" --env= Pass wasi environment variables with " + "\"key=value\"\n"); + printf(" to the program, for example:\n"); + printf(" --env=\"key1=value1\" " + "--env=\"key2=value2\"\n"); + printf(" --dir= Grant wasi access to the given host " + "directories\n"); + printf(" to the program, for example:\n"); + printf(" --dir= --dir=\n"); + printf(" --map-dir= Grant wasi access to the given host " + "directories\n"); + printf(" to the program at a specific guest " + "path, for example:\n"); + printf(" --map-dir= " + "--map-dir=\n"); + printf(" --addr-pool= Grant wasi access to the given network " + "addresses in\n"); + printf(" CIDR notation to the program, seperated " + "with ',',\n"); + printf(" for example:\n"); + printf(" --addr-pool=1.2.3.4/15,2.3.4.5/16\n"); + printf(" --allow-resolve= Allow the lookup of the specific domain " + "name or domain\n"); + printf(" name suffixes using a wildcard, for " + "example:\n"); + printf(" --allow-resolve=example.com # allow the " + "lookup of the specific domain\n"); + printf(" --allow-resolve=*.example.com # allow " + "the lookup of all subdomains\n"); + printf(" --allow-resolve=* # allow any lookup\n"); +} + +static bool +validate_env_str(char *env) +{ + char *p = env; + int key_len = 0; + + while (*p != '\0' && *p != '=') { + key_len++; + p++; + } + + if (*p != '=' || key_len == 0) + return false; + + return true; +} + +libc_wasi_parse_result_t +libc_wasi_parse(char *arg, libc_wasi_parse_context_t *ctx) +{ + if (!strncmp(arg, "--dir=", 6)) { + if (arg[6] == '\0') + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + if (ctx->dir_list_size >= sizeof(ctx->dir_list) / sizeof(char *)) { + printf("Only allow max dir number %d\n", + (int)(sizeof(ctx->dir_list) / sizeof(char *))); + return LIBC_WASI_PARSE_RESULT_BAD_PARAM; + } + ctx->dir_list[ctx->dir_list_size++] = arg + 6; + } + else if (!strncmp(arg, "--map-dir=", 10)) { + if (arg[10] == '\0') + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + if (ctx->map_dir_list_size + >= sizeof(ctx->map_dir_list) / sizeof(char *)) { + printf("Only allow max map dir number %d\n", + (int)(sizeof(ctx->map_dir_list) / sizeof(char *))); + return 1; + } + ctx->map_dir_list[ctx->map_dir_list_size++] = arg + 10; + } + else if (!strncmp(arg, "--env=", 6)) { + char *tmp_env; + + if (arg[6] == '\0') + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + if (ctx->env_list_size >= sizeof(ctx->env_list) / sizeof(char *)) { + printf("Only allow max env number %d\n", + (int)(sizeof(ctx->env_list) / sizeof(char *))); + return LIBC_WASI_PARSE_RESULT_BAD_PARAM; + } + tmp_env = arg + 6; + if (validate_env_str(tmp_env)) + ctx->env_list[ctx->env_list_size++] = tmp_env; + else { + printf("Wasm parse env string failed: expect \"key=value\", " + "got \"%s\"\n", + tmp_env); + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + } + } + /* TODO: parse the configuration file via --addr-pool-file */ + else if (!strncmp(arg, "--addr-pool=", strlen("--addr-pool="))) { + /* like: --addr-pool=100.200.244.255/30 */ + char *token = NULL; + + if ('\0' == arg[12]) + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + + token = strtok(arg + strlen("--addr-pool="), ","); + while (token) { + if (ctx->addr_pool_size + >= sizeof(ctx->addr_pool) / sizeof(char *)) { + printf("Only allow max address number %d\n", + (int)(sizeof(ctx->addr_pool) / sizeof(char *))); + return LIBC_WASI_PARSE_RESULT_BAD_PARAM; + } + + ctx->addr_pool[ctx->addr_pool_size++] = token; + token = strtok(NULL, ";"); + } + } + else if (!strncmp(arg, "--allow-resolve=", 16)) { + if (arg[16] == '\0') + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + if (ctx->ns_lookup_pool_size + >= sizeof(ctx->ns_lookup_pool) / sizeof(ctx->ns_lookup_pool[0])) { + printf("Only allow max ns lookup number %d\n", + (int)(sizeof(ctx->ns_lookup_pool) + / sizeof(ctx->ns_lookup_pool[0]))); + return LIBC_WASI_PARSE_RESULT_BAD_PARAM; + } + ctx->ns_lookup_pool[ctx->ns_lookup_pool_size++] = arg + 16; + } + else { + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + } + return LIBC_WASI_PARSE_RESULT_OK; +} + +void +libc_wasi_init(wasm_module_t wasm_module, int argc, char **argv, + libc_wasi_parse_context_t *ctx) +{ + wasm_runtime_set_wasi_args(wasm_module, ctx->dir_list, ctx->dir_list_size, + ctx->map_dir_list, ctx->map_dir_list_size, + ctx->env_list, ctx->env_list_size, argv, argc); + + wasm_runtime_set_wasi_addr_pool(wasm_module, ctx->addr_pool, + ctx->addr_pool_size); + wasm_runtime_set_wasi_ns_lookup_pool(wasm_module, ctx->ns_lookup_pool, + ctx->ns_lookup_pool_size); +} diff --git a/wasm-micro-runtime/product-mini/platforms/cosmopolitan/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/cosmopolitan/CMakeLists.txt new file mode 100644 index 0000000..8533ea6 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/cosmopolitan/CMakeLists.txt @@ -0,0 +1,175 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# Copyright (C) 2023 Dylibso. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +project (iwasm) + +set (CMAKE_VERBOSE_MAKEFILE OFF) + +set (WAMR_BUILD_PLATFORM "cosmopolitan") + +set(CMAKE_EXECUTABLE_SUFFIX ".com") + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +set (CMAKE_C_STANDARD 99) +set (CMAKE_CXX_STANDARD 17) + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_JIT) + # Disable Fast JIT by default + set (WAMR_BUILD_FAST_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Disable multiple modules by default + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS) + # Disable wasi threads library by default + set (WAMR_BUILD_LIB_WASI_THREADS 0) +endif() + + +if (NOT DEFINED WAMR_BUILD_MINI_LOADER) + # Disable wasm mini loader by default + set (WAMR_BUILD_MINI_LOADER 0) +endif () + +if (NOT DEFINED WAMR_BUILD_SIMD) + # Enable SIMD by default + set (WAMR_BUILD_SIMD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_REF_TYPES) + # Disable reference types by default + set (WAMR_BUILD_REF_TYPES 0) +endif () + +if (NOT DEFINED WAMR_BUILD_DEBUG_INTERP) + # Disable Debug feature by default + set (WAMR_BUILD_DEBUG_INTERP 0) +endif () + +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + set (WAMR_BUILD_FAST_INTERP 0) + set (WAMR_BUILD_MINI_LOADER 0) + set (WAMR_BUILD_SIMD 0) +endif () + +set (WAMR_DISABLE_STACK_HW_BOUND_CHECK 1) +set (WAMR_BUILD_AOT 0) +set (WAMR_DISABLE_WRITE_GS_BASE 1) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +check_pie_supported() +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +set_target_properties (vmlib PROPERTIES POSITION_INDEPENDENT_CODE ON) + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") + +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") + +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mindirect-branch-register") + # UNDEFINED BEHAVIOR, refer to https://en.cppreference.com/w/cpp/language/ub + endif () +endif () + +# The following flags are to enhance security, but it may impact performance, +# we disable them by default. +#if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftrapv -D_FORTIFY_SOURCE=2") +#endif () +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-strong --param ssp-buffer-size=4") +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack,-z,relro,-z,now") + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (iwasm main.c ${UNCOMMON_SHARED_SOURCE}) + +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) + +install (TARGETS iwasm DESTINATION bin) + +target_link_libraries (iwasm vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} ${WASI_NN_LIBS} -lm -ldl -lpthread) + +add_library (libiwasm STATIC ${WAMR_RUNTIME_LIB_SOURCE}) + +install (TARGETS libiwasm DESTINATION lib) + +set_target_properties (libiwasm PROPERTIES OUTPUT_NAME iwasm) + +target_link_libraries (libiwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} ${WASI_NN_LIBS} -lm -ldl -lpthread) diff --git a/wasm-micro-runtime/product-mini/platforms/cosmopolitan/build_cosmocc.sh b/wasm-micro-runtime/product-mini/platforms/cosmopolitan/build_cosmocc.sh new file mode 100755 index 0000000..8d96027 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/cosmopolitan/build_cosmocc.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Copyright (C) 2023 Dylibso. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +export CC=x86_64-unknown-cosmo-cc +export CXX=x86_64-unknown-cosmo-c++ +rm -rf build +mkdir build +cmake -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=1 -B build +cmake --build build -j diff --git a/wasm-micro-runtime/product-mini/platforms/cosmopolitan/main.c b/wasm-micro-runtime/product-mini/platforms/cosmopolitan/main.c new file mode 100644 index 0000000..8f0e84a --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/cosmopolitan/main.c @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../posix/main.c" diff --git a/wasm-micro-runtime/product-mini/platforms/darwin/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/darwin/CMakeLists.txt new file mode 100644 index 0000000..865e516 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/darwin/CMakeLists.txt @@ -0,0 +1,130 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "darwin") + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +set(CMAKE_CXX_STANDARD 17) + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_JIT) + # Disable Fast JIT by default + set (WAMR_BUILD_FAST_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Disable multiple module by default + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_MINI_LOADER) + # Disable wasm mini loader by default + set (WAMR_BUILD_MINI_LOADER 0) +endif () + +if (NOT DEFINED WAMR_BUILD_SIMD) + # Enable SIMD by default + set (WAMR_BUILD_SIMD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_DEBUG_INTERP) + # Disable Debug feature by default + set (WAMR_BUILD_DEBUG_INTERP 0) +endif () + +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + set (WAMR_BUILD_FAST_INTERP 0) + set (WAMR_BUILD_MINI_LOADER 0) + set (WAMR_BUILD_SIMD 0) +endif () + +set (CMAKE_SHARED_LINKER_FLAGS "-Wl,-U,_get_ext_lib_export_apis") +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") + +set (CMAKE_MACOSX_RPATH True) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (iwasm main.c ${UNCOMMON_SHARED_SOURCE}) + +install (TARGETS iwasm DESTINATION bin) + +target_link_libraries (iwasm vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) + +add_library (libiwasm SHARED ${WAMR_RUNTIME_LIB_SOURCE}) + +install (TARGETS libiwasm DESTINATION lib) + +set_target_properties (libiwasm PROPERTIES OUTPUT_NAME iwasm) + +target_link_libraries (libiwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) + diff --git a/wasm-micro-runtime/product-mini/platforms/darwin/build_jit.sh b/wasm-micro-runtime/product-mini/platforms/darwin/build_jit.sh new file mode 100755 index 0000000..a7d5591 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/darwin/build_jit.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +rm -fr build && mkdir build +cd build +cmake .. -DWAMR_BUILD_JIT=1 +make -j ${nproc} +cd .. diff --git a/wasm-micro-runtime/product-mini/platforms/darwin/build_llvm.sh b/wasm-micro-runtime/product-mini/platforms/darwin/build_llvm.sh new file mode 100755 index 0000000..b8a9761 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/darwin/build_llvm.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copyright (C) 2020 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt +/usr/bin/env python3 ../../../build-scripts/build_llvm.py --platform darwin "$@" diff --git a/wasm-micro-runtime/product-mini/platforms/darwin/main.c b/wasm-micro-runtime/product-mini/platforms/darwin/main.c new file mode 100644 index 0000000..8f0e84a --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/darwin/main.c @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../posix/main.c" diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/.gitignore b/wasm-micro-runtime/product-mini/platforms/esp-idf/.gitignore new file mode 100644 index 0000000..c260f41 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/.gitignore @@ -0,0 +1,2 @@ +sdkconfig +sdkconfig.old \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/esp-idf/CMakeLists.txt new file mode 100644 index 0000000..8472df8 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright (C) 2019-21 Intel Corporation and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# from ESP-IDF 4.0 examples/build_system/cmake/idf_as_lib +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +project(wamr-simple) \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/build_and_run.sh b/wasm-micro-runtime/product-mini/platforms/esp-idf/build_and_run.sh new file mode 100755 index 0000000..7ce1b57 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/build_and_run.sh @@ -0,0 +1,37 @@ +#!/bin/bash -e + +# Copyright (C) 2019-21 Intel Corporation and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +ESP32_TARGET="esp32" +ESP32C3_TARGET="esp32c3" +ESP32S3_TARGET="esp32s3" +ESP32C6_TARGET="esp32c6" + +usage () +{ + echo "USAGE:" + echo "$0 $ESP32_TARGET|$ESP32C3_TARGET|$ESP32S3_TARGET" + echo "Example:" + echo " $0 $ESP32_TARGET" + echo " $0 $ESP32C3_TARGET" + echo " $0 $ESP32S3_TARGET" + echo " $0 $ESP32C6_TARGET" + exit 1 +} + +if [ $# != 1 ] ; then + usage +fi + +TARGET=$1 + +if [[ -z "${WAMR_PATH}" ]]; then + export WAMR_PATH=$PWD/../../.. +fi + +rm -rf build +idf.py set-target $TARGET +idf.py build +idf.py flash + diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/main/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/CMakeLists.txt new file mode 100644 index 0000000..1bb61ba --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (C) 2021 Intel Corporation and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +idf_component_register(SRCS "main.c" + INCLUDE_DIRS ".") diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/main/idf_component.yml b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/idf_component.yml new file mode 100644 index 0000000..1c05f47 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/idf_component.yml @@ -0,0 +1,7 @@ +## IDF Component Manager Manifest File +dependencies: + wasm-micro-runtime: + version: "^2" + override_path: "../../../.." + idf: + version: ">=4.4" \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/main/main.c b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/main.c new file mode 100644 index 0000000..1a34096 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/main.c @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2019-21 Intel Corporation and others. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "wasm_export.h" +#include "bh_platform.h" +#include "test_wasm.h" + +#include "esp_log.h" + +#define IWASM_MAIN_STACK_SIZE 5120 + +#define LOG_TAG "wamr" + +static void * +app_instance_main(wasm_module_inst_t module_inst) +{ + const char *exception; + + wasm_application_execute_main(module_inst, 0, NULL); + if ((exception = wasm_runtime_get_exception(module_inst))) + printf("%s\n", exception); + return NULL; +} + +void * +iwasm_main(void *arg) +{ + (void)arg; /* unused */ + /* setup variables for instantiating and running the wasm module */ + uint8_t *wasm_file_buf = NULL; + unsigned wasm_file_buf_size = 0; + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + char error_buf[128]; + void *ret; + RuntimeInitArgs init_args; + + /* configure memory allocation */ + memset(&init_args, 0, sizeof(RuntimeInitArgs)); +#if WASM_ENABLE_GLOBAL_HEAP_POOL == 0 + init_args.mem_alloc_type = Alloc_With_Allocator; + init_args.mem_alloc_option.allocator.malloc_func = (void *)os_malloc; + init_args.mem_alloc_option.allocator.realloc_func = (void *)os_realloc; + init_args.mem_alloc_option.allocator.free_func = (void *)os_free; +#else +#error The usage of a global heap pool is not implemented yet for esp-idf. +#endif + + ESP_LOGI(LOG_TAG, "Initialize WASM runtime"); + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + ESP_LOGE(LOG_TAG, "Init runtime failed."); + return NULL; + } + +#if WASM_ENABLE_INTERP != 0 + ESP_LOGI(LOG_TAG, "Run wamr with interpreter"); + + wasm_file_buf = (uint8_t *)wasm_test_file_interp; + wasm_file_buf_size = sizeof(wasm_test_file_interp); + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_buf_size, + error_buf, sizeof(error_buf)))) { + ESP_LOGE(LOG_TAG, "Error in wasm_runtime_load: %s", error_buf); + goto fail1interp; + } + + ESP_LOGI(LOG_TAG, "Instantiate WASM runtime"); + if (!(wasm_module_inst = + wasm_runtime_instantiate(wasm_module, 32 * 1024, // stack size + 32 * 1024, // heap size + error_buf, sizeof(error_buf)))) { + ESP_LOGE(LOG_TAG, "Error while instantiating: %s", error_buf); + goto fail2interp; + } + + ESP_LOGI(LOG_TAG, "run main() of the application"); + ret = app_instance_main(wasm_module_inst); + assert(!ret); + + /* destroy the module instance */ + ESP_LOGI(LOG_TAG, "Deinstantiate WASM runtime"); + wasm_runtime_deinstantiate(wasm_module_inst); + +fail2interp: + /* unload the module */ + ESP_LOGI(LOG_TAG, "Unload WASM module"); + wasm_runtime_unload(wasm_module); + +fail1interp: +#endif +#if WASM_ENABLE_AOT != 0 + ESP_LOGI(LOG_TAG, "Run wamr with AoT"); + + wasm_file_buf = (uint8_t *)wasm_test_file_aot; + wasm_file_buf_size = sizeof(wasm_test_file_aot); + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_buf_size, + error_buf, sizeof(error_buf)))) { + ESP_LOGE(LOG_TAG, "Error in wasm_runtime_load: %s", error_buf); + goto fail1aot; + } + + ESP_LOGI(LOG_TAG, "Instantiate WASM runtime"); + if (!(wasm_module_inst = + wasm_runtime_instantiate(wasm_module, 32 * 1024, // stack size + 32 * 1024, // heap size + error_buf, sizeof(error_buf)))) { + ESP_LOGE(LOG_TAG, "Error while instantiating: %s", error_buf); + goto fail2aot; + } + + ESP_LOGI(LOG_TAG, "run main() of the application"); + ret = app_instance_main(wasm_module_inst); + assert(!ret); + + /* destroy the module instance */ + ESP_LOGI(LOG_TAG, "Deinstantiate WASM runtime"); + wasm_runtime_deinstantiate(wasm_module_inst); + +fail2aot: + /* unload the module */ + ESP_LOGI(LOG_TAG, "Unload WASM module"); + wasm_runtime_unload(wasm_module); +fail1aot: +#endif + + /* destroy runtime environment */ + ESP_LOGI(LOG_TAG, "Destroy WASM runtime"); + wasm_runtime_destroy(); + + return NULL; +} + +void +app_main(void) +{ + pthread_t t; + int res; + + pthread_attr_t tattr; + pthread_attr_init(&tattr); + pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE); + pthread_attr_setstacksize(&tattr, IWASM_MAIN_STACK_SIZE); + + res = pthread_create(&t, &tattr, iwasm_main, (void *)NULL); + assert(res == 0); + + res = pthread_join(t, NULL); + assert(res == 0); + + ESP_LOGI(LOG_TAG, "Exiting..."); +} diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/main/test_wasm.h b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/test_wasm.h new file mode 100644 index 0000000..fc2d76f --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/main/test_wasm.h @@ -0,0 +1,288 @@ +/* + * Copyright (C) 2019-21 Intel Corporation and others. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#if WASM_ENABLE_INTERP != 0 +unsigned char __aligned(4) wasm_test_file_interp[] = { + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x10, 0x03, 0x60, + 0x01, 0x7F, 0x01, 0x7F, 0x60, 0x02, 0x7F, 0x7F, 0x01, 0x7F, 0x60, 0x01, + 0x7F, 0x00, 0x02, 0x31, 0x04, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x70, 0x75, + 0x74, 0x73, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x6D, 0x61, 0x6C, + 0x6C, 0x6F, 0x63, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x70, 0x72, + 0x69, 0x6E, 0x74, 0x66, 0x00, 0x01, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x66, + 0x72, 0x65, 0x65, 0x00, 0x02, 0x03, 0x02, 0x01, 0x01, 0x04, 0x05, 0x01, + 0x70, 0x01, 0x01, 0x01, 0x05, 0x03, 0x01, 0x00, 0x01, 0x06, 0x13, 0x03, + 0x7F, 0x01, 0x41, 0xC0, 0x28, 0x0B, 0x7F, 0x00, 0x41, 0xBA, 0x08, 0x0B, + 0x7F, 0x00, 0x41, 0xC0, 0x28, 0x0B, 0x07, 0x2C, 0x04, 0x06, 0x6D, 0x65, + 0x6D, 0x6F, 0x72, 0x79, 0x02, 0x00, 0x0A, 0x5F, 0x5F, 0x64, 0x61, 0x74, + 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x03, 0x01, 0x0B, 0x5F, 0x5F, 0x68, 0x65, + 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, 0x65, 0x03, 0x02, 0x04, 0x6D, 0x61, + 0x69, 0x6E, 0x00, 0x04, 0x0A, 0xB2, 0x01, 0x01, 0xAF, 0x01, 0x01, 0x03, + 0x7F, 0x23, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, + 0x24, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x9B, 0x88, 0x80, 0x80, 0x00, + 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x02, 0x40, 0x02, 0x40, 0x41, + 0x80, 0x08, 0x10, 0x81, 0x80, 0x80, 0x80, 0x00, 0x22, 0x03, 0x0D, 0x00, + 0x41, 0xA8, 0x88, 0x80, 0x80, 0x00, 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, + 0x1A, 0x41, 0x7F, 0x21, 0x04, 0x0C, 0x01, 0x0B, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x10, 0x41, 0x80, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x41, + 0x10, 0x6A, 0x10, 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x41, 0x00, 0x21, + 0x04, 0x20, 0x03, 0x41, 0x04, 0x6A, 0x41, 0x00, 0x2F, 0x00, 0x91, 0x88, + 0x80, 0x80, 0x00, 0x3B, 0x00, 0x00, 0x20, 0x03, 0x41, 0x00, 0x28, 0x00, + 0x8D, 0x88, 0x80, 0x80, 0x00, 0x36, 0x00, 0x00, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x00, 0x41, 0x93, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x10, + 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x20, 0x03, 0x10, 0x83, 0x80, 0x80, + 0x80, 0x00, 0x0B, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x80, 0x80, 0x80, + 0x80, 0x00, 0x20, 0x04, 0x0B, 0x0B, 0x41, 0x01, 0x00, 0x41, 0x80, 0x08, + 0x0B, 0x3A, 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, 0x20, 0x25, + 0x70, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, 0x75, 0x66, + 0x3A, 0x20, 0x25, 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, + 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, 0x6F, 0x63, + 0x20, 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, 0x64, 0x00 +}; +#endif + +#if WASM_ENABLE_AOT != 0 +#if BUILD_TARGET_XTENSA != 0 +// XTENSA +unsigned char __aligned(4) wasm_test_file_aot[] = { + 0x00, 0x61, 0x6F, 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x5E, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x74, 0x65, 0x6E, 0x73, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x38, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x14, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x41, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x39, 0x00, 0x00, 0x00, 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, + 0x20, 0x25, 0x70, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, + 0x75, 0x66, 0x3A, 0x20, 0x25, 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, + 0x20, 0x77, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, + 0x6F, 0x63, 0x20, 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x7F, 0x7F, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x7F, 0x7F, 0x7F, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x7F, 0x01, 0x41, 0x00, 0x40, 0x14, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x41, 0x00, 0x3A, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x41, 0x00, 0x40, 0x14, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, + 0x65, 0x6E, 0x76, 0x00, 0x04, 0x00, 0x70, 0x75, 0x74, 0x73, 0x00, 0x00, + 0x03, 0x00, 0x65, 0x6E, 0x76, 0x00, 0x06, 0x00, 0x6D, 0x61, 0x6C, 0x6C, + 0x6F, 0x63, 0x01, 0x00, 0x03, 0x00, 0x65, 0x6E, 0x76, 0x00, 0x06, 0x00, + 0x70, 0x72, 0x69, 0x6E, 0x74, 0x66, 0x02, 0x00, 0x03, 0x00, 0x65, 0x6E, + 0x76, 0x00, 0x04, 0x00, 0x66, 0x72, 0x65, 0x65, 0x01, 0x00, 0x00, 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x70, 0x02, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0xC1, 0x00, 0x98, + 0x62, 0x78, 0x22, 0x82, 0x27, 0x6A, 0x32, 0xC8, 0xE0, 0x0C, 0xD4, 0x37, + 0xB9, 0x08, 0xA8, 0x72, 0x0C, 0xEB, 0x37, 0xBA, 0x0F, 0x4D, 0x0B, 0x81, + 0xFF, 0xFF, 0xAD, 0x07, 0xBD, 0x04, 0xE0, 0x08, 0x00, 0x0C, 0x02, 0x1D, + 0xF0, 0xB9, 0x51, 0x49, 0x81, 0xA9, 0x61, 0x99, 0x91, 0x89, 0xC1, 0x68, + 0x32, 0x82, 0x27, 0x64, 0x89, 0xB1, 0x82, 0x27, 0x62, 0x89, 0x71, 0x82, + 0x27, 0x56, 0x89, 0xA1, 0x0C, 0x05, 0x32, 0x67, 0x6A, 0x52, 0x46, 0x03, + 0x52, 0x46, 0x02, 0x0C, 0x48, 0x89, 0xE1, 0x82, 0x46, 0x01, 0x1C, 0xB8, + 0x82, 0x46, 0x00, 0x0C, 0x1C, 0x41, 0xFF, 0xFF, 0xAD, 0x02, 0xBD, 0x05, + 0xDD, 0x06, 0xE0, 0x04, 0x00, 0x92, 0xA0, 0xFF, 0x90, 0x8A, 0x10, 0x16, + 0xA8, 0x1E, 0x0C, 0x05, 0x52, 0x46, 0x03, 0x52, 0x46, 0x02, 0x88, 0xE1, + 0x82, 0x46, 0x01, 0x52, 0x46, 0x00, 0x0C, 0x1B, 0xAD, 0x02, 0xCD, 0x0B, + 0x99, 0xD1, 0xDD, 0x06, 0xE0, 0x04, 0x00, 0x98, 0xD1, 0x90, 0x8A, 0x10, + 0x16, 0x58, 0x1C, 0x49, 0x41, 0xE8, 0x06, 0x16, 0xCE, 0x17, 0x88, 0xC1, + 0x82, 0xC8, 0xF0, 0x0C, 0x24, 0x37, 0xB8, 0x02, 0xC6, 0xDB, 0xFF, 0x98, + 0xB1, 0x87, 0xB9, 0x02, 0xC6, 0xD9, 0xFF, 0x98, 0xA1, 0x8A, 0x99, 0x1C, + 0x8B, 0x00, 0x0B, 0x40, 0xE0, 0xA0, 0x91, 0xA2, 0x49, 0x03, 0x1C, 0x0C, + 0x00, 0x0C, 0x40, 0xE0, 0xA0, 0x91, 0xA2, 0x49, 0x02, 0xE0, 0xA8, 0x41, + 0xA9, 0x01, 0xA2, 0x49, 0x01, 0xE2, 0x49, 0x00, 0xC9, 0x11, 0x00, 0x0C, + 0x40, 0x80, 0x90, 0x91, 0x92, 0x46, 0x06, 0xB9, 0x21, 0x00, 0x0B, 0x40, + 0x80, 0x90, 0x91, 0x92, 0x46, 0x07, 0x82, 0x46, 0x04, 0x80, 0x88, 0x41, + 0x82, 0x46, 0x05, 0x0C, 0x05, 0x52, 0x46, 0x02, 0x52, 0x46, 0x03, 0x52, + 0x46, 0x00, 0x88, 0xE1, 0x82, 0x46, 0x01, 0x0C, 0x24, 0xAD, 0x02, 0xBD, + 0x04, 0xCD, 0x04, 0xDD, 0x06, 0x88, 0x41, 0xE9, 0x31, 0xE0, 0x08, 0x00, + 0x88, 0xD1, 0x80, 0x8A, 0x10, 0x16, 0xC8, 0x13, 0xF8, 0x31, 0x4B, 0x8F, + 0x98, 0x71, 0x87, 0xB9, 0x02, 0x86, 0xBB, 0xFF, 0x92, 0xA4, 0x11, 0xD8, + 0xA1, 0x9A, 0x9D, 0x92, 0x09, 0x00, 0x8A, 0x8D, 0xA2, 0xA4, 0x12, 0xAA, + 0xAD, 0xA2, 0x0A, 0x00, 0xA2, 0x48, 0x01, 0x92, 0x48, 0x00, 0xE8, 0xB1, + 0xF7, 0xBE, 0x02, 0x06, 0xB3, 0xFF, 0x82, 0xA4, 0x0E, 0x8A, 0x8D, 0x82, + 0x08, 0x00, 0x92, 0xA4, 0x0D, 0x9A, 0x9D, 0x92, 0x09, 0x00, 0xA2, 0xA4, + 0x10, 0xAA, 0xAD, 0xA2, 0x0A, 0x00, 0xFA, 0xBD, 0xC2, 0xA4, 0x0F, 0xCA, + 0xCD, 0xC2, 0x0C, 0x00, 0xC2, 0x4B, 0x02, 0xA2, 0x4B, 0x03, 0x92, 0x4B, + 0x00, 0x82, 0x4B, 0x01, 0x37, 0xBE, 0x02, 0x06, 0xA6, 0xFF, 0x88, 0xA1, + 0x3A, 0x88, 0xA8, 0x21, 0x00, 0x0A, 0x40, 0xF0, 0x90, 0x91, 0x92, 0x48, + 0x03, 0x98, 0x11, 0x00, 0x09, 0x40, 0xF0, 0x90, 0x91, 0x92, 0x48, 0x02, + 0x98, 0x01, 0x92, 0x48, 0x01, 0xF2, 0x48, 0x00, 0x30, 0x80, 0x91, 0x82, + 0x46, 0x06, 0x00, 0x0A, 0x40, 0x30, 0x80, 0x91, 0x82, 0x46, 0x07, 0x32, + 0x46, 0x04, 0x30, 0x88, 0x41, 0x82, 0x46, 0x05, 0x0C, 0x05, 0x52, 0x46, + 0x02, 0x52, 0x46, 0x03, 0x1C, 0x38, 0x82, 0x46, 0x00, 0x88, 0xE1, 0x82, + 0x46, 0x01, 0x0C, 0x2B, 0xAD, 0x02, 0xCD, 0x0B, 0xDD, 0x06, 0x48, 0x41, + 0x3D, 0x0F, 0xE0, 0x04, 0x00, 0x98, 0xD1, 0x90, 0x8A, 0x10, 0x16, 0x78, + 0x07, 0x32, 0x46, 0x00, 0x88, 0x21, 0x00, 0x08, 0x40, 0x30, 0x80, 0x91, + 0x82, 0x46, 0x03, 0x88, 0x11, 0x00, 0x08, 0x40, 0x30, 0x80, 0x91, 0x82, + 0x46, 0x02, 0x88, 0x01, 0x82, 0x46, 0x01, 0x0C, 0x3B, 0x0C, 0x1C, 0xAD, + 0x02, 0x5D, 0x09, 0xDD, 0x06, 0xE0, 0x04, 0x00, 0x50, 0x8A, 0x10, 0x0C, + 0x05, 0x56, 0xB8, 0x02, 0x46, 0x10, 0x00, 0x0C, 0x05, 0x52, 0x46, 0x03, + 0x52, 0x46, 0x02, 0x88, 0xE1, 0x82, 0x46, 0x01, 0x2C, 0x88, 0x82, 0x46, + 0x00, 0x0C, 0x1C, 0xAD, 0x02, 0xBD, 0x05, 0x4D, 0x09, 0xDD, 0x06, 0x88, + 0x41, 0xE0, 0x08, 0x00, 0x40, 0x8A, 0x10, 0x16, 0xA8, 0x01, 0x7C, 0xF5, + 0x48, 0x81, 0x88, 0xC1, 0x98, 0x91, 0x87, 0x39, 0x02, 0x86, 0x72, 0xFF, + 0x48, 0x51, 0x98, 0x61, 0x87, 0xB9, 0x02, 0x06, 0x70, 0xFF, 0x82, 0x67, + 0x6A, 0x2D, 0x05, 0x1D, 0xF0, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x06, 0x00, 0x6D, 0x65, 0x6D, 0x6F, + 0x72, 0x79, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x6D, 0x61, 0x69, 0x6E, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x0A, 0x00, + 0x5F, 0x5F, 0x64, 0x61, 0x74, 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x0B, 0x00, 0x5F, 0x5F, 0x68, 0x65, + 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, 0x65, 0x00, 0x05, 0x00, 0x00, 0x00, + 0xC8, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0C, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, + 0x3A, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x2E, 0x72, + 0x65, 0x6C, 0x61, 0x2E, 0x74, 0x65, 0x78, 0x74, 0x08, 0x00, 0x2E, 0x6C, + 0x69, 0x74, 0x65, 0x72, 0x61, 0x6C, 0x0D, 0x00, 0x2E, 0x72, 0x65, 0x6C, + 0x61, 0x2E, 0x6C, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6C, 0x00, 0x11, 0x00, + 0x61, 0x6F, 0x74, 0x5F, 0x69, 0x6E, 0x76, 0x6F, 0x6B, 0x65, 0x5F, 0x6E, + 0x61, 0x74, 0x69, 0x76, 0x65, 0x00, 0x19, 0x00, 0x61, 0x6F, 0x74, 0x5F, + 0x73, 0x65, 0x74, 0x5F, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6F, + 0x6E, 0x5F, 0x77, 0x69, 0x74, 0x68, 0x5F, 0x69, 0x64, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x1B, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00 +}; +#else +// RISC-V +unsigned char __aligned(4) wasm_test_file_aot[] = { + 0x00, 0x61, 0x6F, 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0xF3, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x72, 0x69, 0x73, 0x63, 0x76, 0x33, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x38, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x14, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x41, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x39, 0x00, 0x00, 0x00, 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, + 0x20, 0x25, 0x70, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, + 0x75, 0x66, 0x3A, 0x20, 0x25, 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, + 0x20, 0x77, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, + 0x6F, 0x63, 0x20, 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x7F, 0x7F, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x7F, 0x7F, 0x7F, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x7F, 0x01, 0x41, 0x00, 0x40, 0x14, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x41, 0x00, 0x3A, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x41, 0x00, 0x40, 0x14, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, + 0x65, 0x6E, 0x76, 0x00, 0x04, 0x00, 0x70, 0x75, 0x74, 0x73, 0x00, 0x00, + 0x03, 0x00, 0x65, 0x6E, 0x76, 0x00, 0x06, 0x00, 0x6D, 0x61, 0x6C, 0x6C, + 0x6F, 0x63, 0x01, 0x00, 0x03, 0x00, 0x65, 0x6E, 0x76, 0x00, 0x06, 0x00, + 0x70, 0x72, 0x69, 0x6E, 0x74, 0x66, 0x02, 0x00, 0x03, 0x00, 0x65, 0x6E, + 0x76, 0x00, 0x04, 0x00, 0x66, 0x72, 0x65, 0x65, 0x01, 0x00, 0x00, 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x01, 0x01, 0xFC, 0x23, 0x2E, 0x11, 0x02, 0x23, 0x2C, 0x81, 0x02, + 0x23, 0x2A, 0x91, 0x02, 0x23, 0x28, 0x21, 0x03, 0x23, 0x26, 0x31, 0x03, + 0x23, 0x24, 0x41, 0x03, 0x23, 0x22, 0x51, 0x03, 0x23, 0x20, 0x61, 0x03, + 0x23, 0x2E, 0x71, 0x01, 0x23, 0x2C, 0x81, 0x01, 0x23, 0x2A, 0x91, 0x01, + 0x23, 0x28, 0xA1, 0x01, 0x23, 0x26, 0xB1, 0x01, 0x83, 0x29, 0x85, 0x00, + 0x03, 0xAA, 0x89, 0x1A, 0x83, 0x2A, 0x85, 0x01, 0x93, 0x0B, 0x0A, 0xFE, + 0x13, 0x04, 0xD0, 0x00, 0x63, 0xFA, 0x7A, 0x01, 0x93, 0x04, 0x05, 0x00, + 0x03, 0x2B, 0xC5, 0x01, 0x13, 0x04, 0xE0, 0x00, 0x63, 0x7A, 0x7B, 0x05, + 0x13, 0x85, 0x09, 0x00, 0x93, 0x05, 0x04, 0x00, 0x97, 0x00, 0x00, 0x00, + 0xE7, 0x80, 0x00, 0x00, 0x13, 0x05, 0x00, 0x00, 0x83, 0x2D, 0xC1, 0x00, + 0x03, 0x2D, 0x01, 0x01, 0x83, 0x2C, 0x41, 0x01, 0x03, 0x2C, 0x81, 0x01, + 0x83, 0x2B, 0xC1, 0x01, 0x03, 0x2B, 0x01, 0x02, 0x83, 0x2A, 0x41, 0x02, + 0x03, 0x2A, 0x81, 0x02, 0x83, 0x29, 0xC1, 0x02, 0x03, 0x29, 0x01, 0x03, + 0x83, 0x24, 0x41, 0x03, 0x03, 0x24, 0x81, 0x03, 0x83, 0x20, 0xC1, 0x03, + 0x13, 0x01, 0x01, 0x04, 0x67, 0x80, 0x00, 0x00, 0x03, 0xA9, 0xC4, 0x00, + 0x03, 0xAC, 0x89, 0x15, 0x83, 0xAD, 0x89, 0x18, 0x83, 0xAC, 0x09, 0x19, + 0x23, 0xA4, 0x79, 0x1B, 0xA3, 0x01, 0x09, 0x00, 0x23, 0x01, 0x09, 0x00, + 0x13, 0x04, 0x40, 0x00, 0xA3, 0x00, 0x89, 0x00, 0x13, 0x05, 0xB0, 0x01, + 0x23, 0x00, 0xA9, 0x00, 0x13, 0x06, 0x10, 0x00, 0x13, 0x85, 0x04, 0x00, + 0x93, 0x05, 0x00, 0x00, 0x93, 0x06, 0x09, 0x00, 0x97, 0x00, 0x00, 0x00, + 0xE7, 0x80, 0x00, 0x00, 0x13, 0x75, 0xF5, 0x0F, 0xE3, 0x0C, 0x05, 0xF6, + 0xA3, 0x01, 0x09, 0x00, 0x23, 0x01, 0x09, 0x00, 0xA3, 0x00, 0x89, 0x00, + 0x23, 0x00, 0x09, 0x00, 0x93, 0x05, 0x10, 0x00, 0x13, 0x06, 0x10, 0x00, + 0x13, 0x85, 0x04, 0x00, 0x93, 0x06, 0x09, 0x00, 0x97, 0x00, 0x00, 0x00, + 0xE7, 0x80, 0x00, 0x00, 0x13, 0x75, 0xF5, 0x0F, 0xE3, 0x04, 0x05, 0xF4, + 0x03, 0x2D, 0x09, 0x00, 0x63, 0x08, 0x0D, 0x18, 0x13, 0x05, 0x0A, 0xFF, + 0xB3, 0x35, 0x75, 0x01, 0x33, 0xB6, 0xAC, 0x00, 0xB3, 0xE5, 0xC5, 0x00, + 0x13, 0x04, 0x20, 0x00, 0xE3, 0x9C, 0x05, 0xF0, 0xB3, 0x05, 0xAC, 0x00, + 0x13, 0x56, 0x8D, 0x01, 0x23, 0x24, 0xC1, 0x00, 0xA3, 0x81, 0xC5, 0x00, + 0x13, 0x56, 0x0D, 0x01, 0x23, 0x22, 0xC1, 0x00, 0x23, 0x81, 0xC5, 0x00, + 0x13, 0x56, 0x8D, 0x00, 0x23, 0x20, 0xC1, 0x00, 0xA3, 0x80, 0xC5, 0x00, + 0x23, 0x80, 0xA5, 0x01, 0x23, 0x01, 0x09, 0x00, 0xA3, 0x01, 0x09, 0x00, + 0x23, 0x00, 0x09, 0x00, 0x93, 0x05, 0x40, 0x00, 0xA3, 0x00, 0xB9, 0x00, + 0x93, 0x55, 0x05, 0x01, 0x23, 0x03, 0xB9, 0x00, 0x93, 0x55, 0x85, 0x01, + 0xA3, 0x03, 0xB9, 0x00, 0x23, 0x02, 0xA9, 0x00, 0x13, 0x55, 0x85, 0x00, + 0xA3, 0x02, 0xA9, 0x00, 0x93, 0x05, 0x20, 0x00, 0x13, 0x06, 0x20, 0x00, + 0x13, 0x04, 0x20, 0x00, 0x13, 0x85, 0x04, 0x00, 0x93, 0x06, 0x09, 0x00, + 0x97, 0x00, 0x00, 0x00, 0xE7, 0x80, 0x00, 0x00, 0x13, 0x75, 0xF5, 0x0F, + 0xE3, 0x04, 0x05, 0xEA, 0x13, 0x05, 0x4D, 0x00, 0xE3, 0xE8, 0xAD, 0xE8, + 0x83, 0x05, 0x2C, 0x41, 0x03, 0x46, 0x1C, 0x41, 0x33, 0x05, 0xAC, 0x00, + 0xA3, 0x00, 0xB5, 0x00, 0x23, 0x00, 0xC5, 0x00, 0xE3, 0xEC, 0xAC, 0xE7, + 0x03, 0x45, 0xEC, 0x40, 0x83, 0x45, 0xFC, 0x40, 0x03, 0x46, 0x0C, 0x41, + 0x83, 0x46, 0xDC, 0x40, 0x33, 0x07, 0xAC, 0x01, 0x23, 0x01, 0xB7, 0x00, + 0xA3, 0x01, 0xC7, 0x00, 0x23, 0x00, 0xD7, 0x00, 0xA3, 0x00, 0xA7, 0x00, + 0xE3, 0xE8, 0x7C, 0xE5, 0x33, 0x05, 0x7C, 0x01, 0x03, 0x24, 0x81, 0x00, + 0xA3, 0x01, 0x85, 0x00, 0x03, 0x2C, 0x41, 0x00, 0x23, 0x01, 0x85, 0x01, + 0x83, 0x2C, 0x01, 0x00, 0xA3, 0x00, 0x95, 0x01, 0x23, 0x00, 0xA5, 0x01, + 0x23, 0x01, 0x09, 0x00, 0xA3, 0x01, 0x09, 0x00, 0x13, 0x05, 0x30, 0x01, + 0x23, 0x00, 0xA9, 0x00, 0x13, 0x05, 0x40, 0x00, 0xA3, 0x00, 0xA9, 0x00, + 0x13, 0xD5, 0x0B, 0x01, 0x23, 0x03, 0xA9, 0x00, 0x13, 0xD5, 0x8B, 0x01, + 0xA3, 0x03, 0xA9, 0x00, 0x23, 0x02, 0x79, 0x01, 0x13, 0xD5, 0x8B, 0x00, + 0xA3, 0x02, 0xA9, 0x00, 0x93, 0x05, 0x20, 0x00, 0x13, 0x06, 0x20, 0x00, + 0x13, 0x85, 0x04, 0x00, 0x93, 0x06, 0x09, 0x00, 0x97, 0x00, 0x00, 0x00, + 0xE7, 0x80, 0x00, 0x00, 0x13, 0x75, 0xF5, 0x0F, 0xE3, 0x06, 0x05, 0xDE, + 0x23, 0x00, 0xA9, 0x01, 0xA3, 0x01, 0x89, 0x00, 0x23, 0x01, 0x89, 0x01, + 0xA3, 0x00, 0x99, 0x01, 0x93, 0x05, 0x30, 0x00, 0x13, 0x06, 0x10, 0x00, + 0x13, 0x85, 0x04, 0x00, 0x93, 0x06, 0x09, 0x00, 0x97, 0x00, 0x00, 0x00, + 0xE7, 0x80, 0x00, 0x00, 0x93, 0x75, 0xF5, 0x0F, 0x13, 0x05, 0x00, 0x00, + 0x63, 0x92, 0x05, 0x04, 0x6F, 0xF0, 0x9F, 0xDB, 0xA3, 0x01, 0x09, 0x00, + 0x23, 0x01, 0x09, 0x00, 0x13, 0x05, 0x40, 0x00, 0xA3, 0x00, 0xA9, 0x00, + 0x13, 0x05, 0x80, 0x02, 0x23, 0x00, 0xA9, 0x00, 0x13, 0x06, 0x10, 0x00, + 0x13, 0x85, 0x04, 0x00, 0x93, 0x05, 0x00, 0x00, 0x93, 0x06, 0x09, 0x00, + 0x97, 0x00, 0x00, 0x00, 0xE7, 0x80, 0x00, 0x00, 0x93, 0x75, 0xF5, 0x0F, + 0x13, 0x05, 0xF0, 0xFF, 0xE3, 0x8C, 0x05, 0xD6, 0x13, 0x04, 0xD0, 0x00, + 0xE3, 0xF0, 0x4A, 0xD7, 0x13, 0x04, 0xE0, 0x00, 0xE3, 0x6C, 0x4B, 0xD5, + 0x23, 0xA4, 0x49, 0x1B, 0x6F, 0xF0, 0x5F, 0xD6, 0x03, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x06, 0x00, 0x6D, 0x65, 0x6D, 0x6F, + 0x72, 0x79, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x6D, 0x61, 0x69, 0x6E, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x0A, 0x00, + 0x5F, 0x5F, 0x64, 0x61, 0x74, 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x0B, 0x00, 0x5F, 0x5F, 0x68, 0x65, + 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, 0x65, 0x00, 0x05, 0x00, 0x00, 0x00, + 0xCC, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0C, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, + 0x0A, 0x00, 0x2E, 0x72, 0x65, 0x6C, 0x61, 0x2E, 0x74, 0x65, 0x78, 0x74, + 0x19, 0x00, 0x61, 0x6F, 0x74, 0x5F, 0x73, 0x65, 0x74, 0x5F, 0x65, 0x78, + 0x63, 0x65, 0x70, 0x74, 0x69, 0x6F, 0x6E, 0x5F, 0x77, 0x69, 0x74, 0x68, + 0x5F, 0x69, 0x64, 0x00, 0x11, 0x00, 0x61, 0x6F, 0x74, 0x5F, 0x69, 0x6E, + 0x76, 0x6F, 0x6B, 0x65, 0x5F, 0x6E, 0x61, 0x74, 0x69, 0x76, 0x65, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xEC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x1C, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0xBC, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x78, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0xA8, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0xE8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00 +}; +#endif +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults b/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults new file mode 100644 index 0000000..bb158ee --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults @@ -0,0 +1,2 @@ +CONFIG_FREERTOS_USE_TRACE_FACILITY=y +# CONFIG_ESP_SYSTEM_MEMPROT_FEATURE is not set diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults.esp32 b/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults.esp32 new file mode 100644 index 0000000..538ea3d --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults.esp32 @@ -0,0 +1 @@ +# CONFIG_ESP32_MEMPROT_FEATURE is not set diff --git a/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults.esp32c3 b/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults.esp32c3 new file mode 100644 index 0000000..29b82c4 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/esp-idf/sdkconfig.defaults.esp32c3 @@ -0,0 +1 @@ +# CONFIG_ESP32C3_MEMPROT_FEATURE is not set diff --git a/wasm-micro-runtime/product-mini/platforms/freebsd/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/freebsd/CMakeLists.txt new file mode 100644 index 0000000..dd1bbc4 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/freebsd/CMakeLists.txt @@ -0,0 +1,132 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "freebsd") + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +set(CMAKE_CXX_STANDARD 17) + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_JIT) + # Disable Fast JIT by default + set (WAMR_BUILD_FAST_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Disable multiple module by default + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_MINI_LOADER) + # Disable wasm mini loader by default + set (WAMR_BUILD_MINI_LOADER 0) +endif () + +if (NOT DEFINED WAMR_BUILD_SIMD) + # Enable SIMD by default + set (WAMR_BUILD_SIMD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_DEBUG_INTERP) + # Disable Debug feature by default + set (WAMR_BUILD_DEBUG_INTERP 0) +endif () + +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + set (WAMR_BUILD_FAST_INTERP 0) + set (WAMR_BUILD_MINI_LOADER 0) + set (WAMR_BUILD_SIMD 0) +endif () + +set (WAMR_DISABLE_HW_BOUND_CHECK 1) + +set (CMAKE_SHARED_LINKER_FLAGS "-Wl") +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") + +set (CMAKE_MACOSX_RPATH True) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (iwasm main.c ${UNCOMMON_SHARED_SOURCE}) + +install (TARGETS iwasm DESTINATION bin) + +target_link_libraries (iwasm vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) + +add_library (libiwasm SHARED ${WAMR_RUNTIME_LIB_SOURCE}) + +install (TARGETS libiwasm DESTINATION lib) + +set_target_properties (libiwasm PROPERTIES OUTPUT_NAME iwasm) + +target_link_libraries (libiwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) + diff --git a/wasm-micro-runtime/product-mini/platforms/freebsd/build_jit.sh b/wasm-micro-runtime/product-mini/platforms/freebsd/build_jit.sh new file mode 100755 index 0000000..e6bee75 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/freebsd/build_jit.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +rm -fr build && mkdir build +cd build +cmake .. -DWAMR_BUILD_JIT=1 +nproc=$(sysctl -n hw.ncpu) +make -j ${nproc} +cd .. diff --git a/wasm-micro-runtime/product-mini/platforms/freebsd/build_llvm.sh b/wasm-micro-runtime/product-mini/platforms/freebsd/build_llvm.sh new file mode 100755 index 0000000..c5666b7 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/freebsd/build_llvm.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copyright (C) 2020 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt +/usr/bin/env python3 ../../../build-scripts/build_llvm.py "$@" diff --git a/wasm-micro-runtime/product-mini/platforms/freebsd/main.c b/wasm-micro-runtime/product-mini/platforms/freebsd/main.c new file mode 100644 index 0000000..8f0e84a --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/freebsd/main.c @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../posix/main.c" diff --git a/wasm-micro-runtime/product-mini/platforms/ios/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/ios/CMakeLists.txt new file mode 100644 index 0000000..4bbff4c --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/ios/CMakeLists.txt @@ -0,0 +1,147 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.21) + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfloat-abi=hard") + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "darwin") +set (WAMR_BUILD_TYPE Release) +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_LIBC_WASI 1) + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +set(CMAKE_CXX_STANDARD 17) + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_JIT) + # Disable Fast JIT by default + set (WAMR_BUILD_FAST_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Disable multiple module by default + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MINI_LOADER) + # Disable wasm mini loader by default + set (WAMR_BUILD_MINI_LOADER 0) +endif () + +if (NOT DEFINED WAMR_BUILD_SIMD) + # Enable SIMD by default + set (WAMR_BUILD_SIMD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_DEBUG_INTERP) + # Disable Debug feature by default + set (WAMR_BUILD_DEBUG_INTERP 0) +endif () + +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + set (WAMR_BUILD_FAST_INTERP 0) + set (WAMR_BUILD_MINI_LOADER 0) + set (WAMR_BUILD_SIMD 0) +endif () + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections -pie -fPIE") + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") + +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () +endif () + +# The following flags are to enhance security, but it may impact performance, +# we disable them by default. +#if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftrapv -D_FORTIFY_SOURCE=2") +#endif () +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-strong --param ssp-buffer-size=4") +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack,-z,relro,-z,now") + +add_library (iwasm SHARED ${WAMR_RUNTIME_LIB_SOURCE}) +if (CMAKE_BUILD_TYPE STREQUAL Release) +target_link_libraries (iwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -s) +else() +target_link_libraries (iwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl) +endif() + +set (distribution_DIR ${CMAKE_BINARY_DIR}/distribution) +set_target_properties (iwasm PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${distribution_DIR}/wasm/lib") + +add_custom_command (TARGET iwasm POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy_directory "${WAMR_ROOT_DIR}/core/iwasm/include" "${distribution_DIR}/wasm/include/" + COMMENT "Copying iwasm to output directory") diff --git a/wasm-micro-runtime/product-mini/platforms/ios/generate_xcodeproj.sh b/wasm-micro-runtime/product-mini/platforms/ios/generate_xcodeproj.sh new file mode 100755 index 0000000..1686081 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/ios/generate_xcodeproj.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Copyright (C) 2022 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +rm -rf ./iwasm-proj +git clone https://github.com/leetal/ios-cmake.git ios-cmake +cmake -Biwasm-proj -G Xcode -DDEPLOYMENT_TARGET=11.0 -DPLATFORM=OS64 -DENABLE_BITCODE=0 -DCMAKE_TOOLCHAIN_FILE=ios-cmake/ios.toolchain.cmake . diff --git a/wasm-micro-runtime/product-mini/platforms/linux-sgx/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/linux-sgx/CMakeLists.txt new file mode 100644 index 0000000..20b3fdf --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/linux-sgx/CMakeLists.txt @@ -0,0 +1,189 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "linux-sgx") + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR_BUILD_TARGET +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + if (NOT DEFINED WAMR_BUILD_SIMD) + # Enable SIMD by default in 64-bit platform + set (WAMR_BUILD_SIMD 1) + endif () + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default + # Please install Intel SGX SDKv2.8 or later. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_JIT) + # Disable Fast JIT by default + set (WAMR_BUILD_FAST_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_RATS) + # Disable lib rats support by default + set (WAMR_BUILD_LIB_RATS 0) +endif() + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Enable multiple modules + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Enable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_SIMD) + # Disable SIMD by default + set (WAMR_BUILD_SIMD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_SGX_IPFS) + # Disable SGX IPFS by default + set (WAMR_BUILD_SGX_IPFS 0) +endif () + +if (NOT DEFINED WAMR_BUILD_STATIC_PGO) + # Disable static PGO by default + set (WAMR_BUILD_STATIC_PGO 0) +endif () + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \ + -Wall -Wno-unused-parameter -Wno-pedantic \ + -nostdinc -fvisibility=hidden -fpie" ) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +add_custom_command ( + OUTPUT libvmlib_untrusted.a + COMMAND mkdir -p untrusted && cd untrusted && + ${CMAKE_C_COMPILER} -c ${PLATFORM_SHARED_SOURCE_UNTRUSTED} + COMMAND ${CMAKE_AR} rc libvmlib_untrusted.a untrusted/*.o) + +add_custom_target (vmlib_untrusted ALL DEPENDS libvmlib_untrusted.a) + +if ((WAMR_BUILD_STATIC_PGO EQUAL 1) AND (WAMR_BUILD_AOT EQUAL 1)) + execute_process( + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_STATIC_PGO = 0/WAMR_BUILD_STATIC_PGO = 1/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +else() + execute_process( + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_STATIC_PGO = 1/WAMR_BUILD_STATIC_PGO = 0/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +endif() + +if (DEFINED WAMR_BUILD_GLOBAL_HEAP_POOL) + execute_process( + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_GLOBAL_HEAP_POOL = .*/WAMR_BUILD_GLOBAL_HEAP_POOL = ${WAMR_BUILD_GLOBAL_HEAP_POOL}/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) + if (DEFINED WAMR_BUILD_GLOBAL_HEAP_SIZE) + execute_process( + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_GLOBAL_HEAP_SIZE = .*/WAMR_BUILD_GLOBAL_HEAP_SIZE = ${WAMR_BUILD_GLOBAL_HEAP_SIZE}/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) + endif() +endif() + +if (WAMR_BUILD_LIB_RATS EQUAL 1) + execute_process( + COMMAND bash -c "sed -i -E 's/^#define WASM_ENABLE_LIB_RATS 0/#define WASM_ENABLE_LIB_RATS 1/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.edl" + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_LIB_RATS = 0/WAMR_BUILD_LIB_RATS = 1/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +else() + execute_process( + COMMAND bash -c "sed -i -E 's/^#define WASM_ENABLE_LIB_RATS 1/#define WASM_ENABLE_LIB_RATS 0/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.edl" + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_LIB_RATS = 1/WAMR_BUILD_LIB_RATS = 0/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +endif() + +if (WAMR_BUILD_SGX_IPFS EQUAL 1) + execute_process( + COMMAND bash -c "sed -i -E 's/^#define WASM_ENABLE_SGX_IPFS 0/#define WASM_ENABLE_SGX_IPFS 1/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.edl" + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_SGX_IPFS = 0/WAMR_BUILD_SGX_IPFS = 1/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +else() + execute_process( + COMMAND bash -c "sed -i -E 's/^#define WASM_ENABLE_SGX_IPFS 1/#define WASM_ENABLE_SGX_IPFS 0/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.edl" + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_SGX_IPFS = 1/WAMR_BUILD_SGX_IPFS = 0/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +endif() + +if (WAMR_BUILD_LIBC_WASI EQUAL 1) + execute_process( + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_LIBC_WASI = 0/WAMR_BUILD_LIBC_WASI = 1/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +else() + execute_process( + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_LIBC_WASI = 1/WAMR_BUILD_LIBC_WASI = 0/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput + ) +endif() + +if (WAMR_BUILD_SPEC_TEST EQUAL 1) + execute_process( + COMMAND bash -c "sed -i -E 's/0x1000000<\\/ReservedMemMaxSize>/0x8000000<\\/ReservedMemMaxSize>/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.config.xml" + OUTPUT_VARIABLE cmdOutput + ) +endif() diff --git a/wasm-micro-runtime/product-mini/platforms/linux-sgx/CMakeLists_minimal.txt b/wasm-micro-runtime/product-mini/platforms/linux-sgx/CMakeLists_minimal.txt new file mode 100644 index 0000000..aa3de6d --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/linux-sgx/CMakeLists_minimal.txt @@ -0,0 +1,88 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "linux-sgx") + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR_BUILD_TARGET +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default + # Please install Intel SGX SDKv2.8 or later. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Enable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 0) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Enable multiple modules + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Enable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -ffunction-sections -fdata-sections \ + -Wall -Wno-unused-parameter -Wno-pedantic \ + -nostdinc -fvisibility=hidden -fpie" ) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +add_custom_command ( + OUTPUT libvmlib_untrusted.a + COMMAND mkdir -p untrusted && cd untrusted && + ${CMAKE_C_COMPILER} -c ${PLATFORM_SHARED_SOURCE_UNTRUSTED} + COMMAND ${CMAKE_AR} rc libvmlib_untrusted.a untrusted/*.o) + +add_custom_target (vmlib_untrusted ALL DEPENDS libvmlib_untrusted.a) diff --git a/wasm-micro-runtime/product-mini/platforms/linux-sgx/enclave-sample/Makefile b/wasm-micro-runtime/product-mini/platforms/linux-sgx/enclave-sample/Makefile new file mode 100644 index 0000000..8fd053a --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/linux-sgx/enclave-sample/Makefile @@ -0,0 +1,269 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +######## SGX SDK Settings ######## + +SGX_SDK ?= /opt/intel/sgxsdk +SGX_SSL ?= /opt/intel/sgxssl +SGX_MODE ?= SIM +SGX_ARCH ?= x64 +SGX_DEBUG ?= 0 +SPEC_TEST ?= 0 + +# These variables are automatically set by CMakeLists.txt +WAMR_BUILD_SGX_IPFS = 0 +WAMR_BUILD_LIB_RATS = 0 +WAMR_BUILD_GLOBAL_HEAP_POOL = 0 +WAMR_BUILD_GLOBAL_HEAP_SIZE = 10485760 +WAMR_BUILD_STATIC_PGO = 0 +WAMR_BUILD_LIBC_WASI = 1 + +VMLIB_BUILD_DIR ?= $(CURDIR)/../build +LIB_RATS_SRC ?= $(VMLIB_BUILD_DIR)/_deps/librats-build +LIB_RATS_INSTALL_DIR := $(VMLIB_BUILD_DIR)/librats/lib/librats +LIB_RATS_INCLUDE_DIR := $(VMLIB_BUILD_DIR)/librats/include + +ifeq ($(shell getconf LONG_BIT), 32) + SGX_ARCH := x86 +else ifeq ($(findstring -m32, $(CXXFLAGS)), -m32) + SGX_ARCH := x86 +endif + +ifeq ($(SGX_ARCH), x86) + SGX_COMMON_CFLAGS := -m32 + SGX_LIBRARY_PATH := $(SGX_SDK)/lib + SGX_ENCLAVE_SIGNER := $(SGX_SDK)/bin/x86/sgx_sign + SGX_EDGER8R := $(SGX_SDK)/bin/x86/sgx_edger8r +else + SGX_COMMON_CFLAGS := -m64 + SGX_LIBRARY_PATH := $(SGX_SDK)/lib64 + SGX_ENCLAVE_SIGNER := $(SGX_SDK)/bin/x64/sgx_sign + SGX_EDGER8R := $(SGX_SDK)/bin/x64/sgx_edger8r +endif + +ifeq ($(SGX_DEBUG), 1) +ifeq ($(SGX_PRERELEASE), 1) +$(error Cannot set SGX_DEBUG and SGX_PRERELEASE at the same time!!) +endif +endif + +ifeq ($(SGX_DEBUG), 1) + SGX_COMMON_CFLAGS += -O0 -g +else + SGX_COMMON_CFLAGS += -O2 +endif + +######## App Settings ######## + +ifneq ($(SGX_MODE), HW) + Urts_Library_Name := sgx_urts_sim +else + Urts_Library_Name := sgx_urts +endif + +App_Cpp_Files := App/App.cpp +App_Include_Paths := -IApp -I$(SGX_SDK)/include +ifeq ($(WAMR_BUILD_LIB_RATS), 1) + App_Include_Paths += -I$(LIB_RATS_INCLUDE_DIR) +endif + +App_C_Flags := $(SGX_COMMON_CFLAGS) -fPIC -Wno-attributes $(App_Include_Paths) \ + -DWASM_ENABLE_STATIC_PGO=$(WAMR_BUILD_STATIC_PGO) \ + -DWASM_ENABLE_LIBC_WASI=$(WAMR_BUILD_LIBC_WASI) + +# Three configuration modes - Debug, prerelease, release +# Debug - Macro DEBUG enabled. +# Prerelease - Macro NDEBUG and EDEBUG enabled. +# Release - Macro NDEBUG enabled. +ifeq ($(SGX_DEBUG), 1) + App_C_Flags += -DDEBUG -UNDEBUG -UEDEBUG +else ifeq ($(SGX_PRERELEASE), 1) + App_C_Flags += -DNDEBUG -DEDEBUG -UDEBUG +else + App_C_Flags += -DNDEBUG -UEDEBUG -UDEBUG +endif + +ifeq ($(SPEC_TEST), 1) + App_C_Flags += -DWASM_ENABLE_SPEC_TEST=1 +else + App_C_Flags += -DWASM_ENABLE_SPEC_TEST=0 +endif + +App_Cpp_Flags := $(App_C_Flags) -std=c++11 +App_Link_Flags := $(SGX_COMMON_CFLAGS) libvmlib_untrusted.a -L$(SGX_LIBRARY_PATH) -l$(Urts_Library_Name) -lpthread + +ifneq ($(SGX_MODE), HW) + App_Link_Flags += -lsgx_uae_service_sim +else + App_Link_Flags += -lsgx_uae_service +endif + +ifeq ($(WAMR_BUILD_LIB_RATS), 1) + App_Link_Flags += -L$(LIB_RATS_INSTALL_DIR) -L$(SGX_SSL)/lib64 -lrats_u -lsgx_dcap_ql -lsgx_dcap_quoteverify -lsgx_ukey_exchange -lsgx_usgxssl +endif + +App_Cpp_Objects := $(App_Cpp_Files:.cpp=.o) + +App_Name := iwasm + +######## Enclave Settings ######## + +ifneq ($(SGX_MODE), HW) + Trts_Library_Name := sgx_trts_sim + Service_Library_Name := sgx_tservice_sim +else + Trts_Library_Name := sgx_trts + Service_Library_Name := sgx_tservice +endif + +ifeq ($(WAMR_BUILD_SGX_IPFS), 1) + Intel_Ipfs_Trusted_Flag = -lsgx_tprotected_fs + App_Link_Flags += -lsgx_uprotected_fs +endif + +Crypto_Library_Name := sgx_tcrypto + +WAMR_ROOT := $(CURDIR)/../../../../ + +Enclave_Cpp_Files := Enclave/Enclave.cpp + +Enclave_Include_Paths := -IEnclave -I$(WAMR_ROOT)/core/iwasm/include \ + -I$(WAMR_ROOT)/core/shared/utils \ + -I$(WAMR_ROOT)/core/shared/platform/linux-sgx \ + -I$(SGX_SDK)/include \ + -I$(SGX_SDK)/include/tlibc \ + -I$(SGX_SDK)/include/stlport + +ifeq ($(WAMR_BUILD_LIB_RATS), 1) + Enclave_Include_Paths += -I$(LIB_RATS_INCLUDE_DIR) -I$(SGX_SSL)/include +endif + +Enclave_C_Flags := $(SGX_COMMON_CFLAGS) -nostdinc -fvisibility=hidden \ + -fpie -fstack-protector $(Enclave_Include_Paths) \ + -DWASM_GLOBAL_HEAP_SIZE=$(WAMR_BUILD_GLOBAL_HEAP_SIZE) \ + -DWASM_ENABLE_GLOBAL_HEAP_POOL=$(WAMR_BUILD_GLOBAL_HEAP_POOL) \ + -DWASM_ENABLE_LIB_RATS=$(WAMR_BUILD_LIB_RATS) \ + -DWASM_ENABLE_STATIC_PGO=$(WAMR_BUILD_STATIC_PGO) \ + -DWASM_ENABLE_LIBC_WASI=$(WAMR_BUILD_LIBC_WASI) +ifeq ($(SPEC_TEST), 1) + Enclave_C_Flags += -DWASM_ENABLE_SPEC_TEST=1 +else + Enclave_C_Flags += -DWASM_ENABLE_SPEC_TEST=0 +endif + +ifeq ($(WAMR_BUILD_LIB_RATS), 1) + Rats_Lib_Link_Dirs := -L$(LIB_RATS_INSTALL_DIR) -L$(LIB_RATS_INSTALL_DIR)/attesters -L$(LIB_RATS_INSTALL_DIR)/verifiers -L$(SGX_SSL)/lib64 -L$(VMLIB_BUILD_DIR)/external/libcbor/src/libcbor/lib -L$(LIB_RATS_INSTALL_DIR)/crypto_wrappers + Rats_Lib_W_Link_libs := -lattester_nullattester -lattester_sgx_ecdsa -lattester_sgx_la \ + -lverifier_nullverifier -lverifier_sgx_la -lverifier_sgx_ecdsa_qve -lcbor \ + -lrats_lib -lsgx_tsgxssl -lcrypto_wrapper_nullcrypto -lcrypto_wrapper_openssl + Rats_Lib_NW_Link_libs := -lsgx_dcap_tvl -lsgx_tsgxssl_crypto +endif + +Enclave_Cpp_Flags := $(Enclave_C_Flags) -std=c++11 -nostdinc++ +Enclave_Link_Flags := $(SGX_COMMON_CFLAGS) -Wl,--no-undefined -nostdlib -nodefaultlibs -nostartfiles -L$(SGX_LIBRARY_PATH) ${Rats_Lib_Link_Dirs} \ + -Wl,--whole-archive -l$(Trts_Library_Name) ${Rats_Lib_W_Link_libs} $(Intel_Ipfs_Trusted_Flag) -Wl,--no-whole-archive \ + -Wl,--start-group -lsgx_tstdc -lsgx_tcxx -lsgx_pthread -lsgx_tkey_exchange -l$(Crypto_Library_Name) -l$(Service_Library_Name) $(Rats_Lib_NW_Link_libs) -Wl,--end-group \ + -Wl,-Bstatic -Wl,-Bsymbolic -Wl,--no-undefined \ + -Wl,-pie,-eenclave_entry -Wl,--export-dynamic \ + -Wl,--defsym,__ImageBase=0 + +Enclave_Edl_Search_Path = --search-path ../Enclave \ + --search-path $(SGX_SDK)/include \ + --search-path $(WAMR_ROOT)/core/shared/platform/linux-sgx +ifeq ($(WAMR_BUILD_LIB_RATS), 1) + Enclave_Edl_Search_Path += --search-path $(LIB_RATS_INCLUDE_DIR)/librats/edl --search-path $(SGX_SSL)/include +endif + + +Enclave_Cpp_Objects := $(Enclave_Cpp_Files:.cpp=.o) + +Enclave_Name := enclave.so +Signed_Enclave_Name := enclave.signed.so +Enclave_Config_File := Enclave/Enclave.config.xml + +ifeq ($(SGX_MODE), HW) +ifneq ($(SGX_DEBUG), 1) +ifneq ($(SGX_PRERELEASE), 1) +Build_Mode = HW_RELEASE +endif +endif +endif + + +.PHONY: all run + +ifeq ($(Build_Mode), HW_RELEASE) +all: $(App_Name) $(Enclave_Name) + @echo "The project has been built in release hardware mode." + @echo "Please sign the $(Enclave_Name) first with your signing key before you run the $(App_Name) to launch and access the enclave." + @echo "To sign the enclave use the command:" + @echo " $(SGX_ENCLAVE_SIGNER) sign -key -enclave $(Enclave_Name) -out <$(Signed_Enclave_Name)> -config $(Enclave_Config_File)" + @echo "You can also sign the enclave using an external signing tool. See User's Guide for more details." + @echo "To build the project in simulation mode set SGX_MODE=SIM. To build the project in prerelease mode set SGX_PRERELEASE=1 and SGX_MODE=HW." +else +all: $(App_Name) $(Signed_Enclave_Name) +endif + +run: all +ifneq ($(Build_Mode), HW_RELEASE) + @$(CURDIR)/$(App_Name) + @echo "RUN => $(App_Name) [$(SGX_MODE)|$(SGX_ARCH), OK]" +endif + +######## App Objects ######## +librats: +ifeq ($(WAMR_BUILD_LIB_RATS), 1) + @cd $(LIB_RATS_SRC) && make install + @echo "librats build success" +endif + +App/Enclave_u.c: $(SGX_EDGER8R) Enclave/Enclave.edl librats + @cd App && $(SGX_EDGER8R) --untrusted ../Enclave/Enclave.edl $(Enclave_Edl_Search_Path) + @echo "GEN => $@" + +App/Enclave_u.o: App/Enclave_u.c + @$(CC) $(App_C_Flags) -c $< -o $@ + @echo "CC <= $<" + +App/%.o: App/%.cpp + @$(CXX) $(App_Cpp_Flags) -c $< -o $@ + @echo "CXX <= $<" + +libvmlib_untrusted.a: $(VMLIB_BUILD_DIR)/libvmlib_untrusted.a + @cp $< $@ + @echo "CP $@ <= $<" + +$(App_Name): App/Enclave_u.o $(App_Cpp_Objects) libvmlib_untrusted.a + @$(CXX) $^ -o $@ $(App_Link_Flags) + @echo "LINK => $@" + + +######## Enclave Objects ######## +Enclave/Enclave_t.c: $(SGX_EDGER8R) Enclave/Enclave.edl librats + @cd Enclave && $(SGX_EDGER8R) --trusted ../Enclave/Enclave.edl $(Enclave_Edl_Search_Path) + @echo "GEN => $@" + +Enclave/Enclave_t.o: Enclave/Enclave_t.c + @$(CC) $(Enclave_C_Flags) -c $< -o $@ + @echo "CC <= $<" + +Enclave/%.o: Enclave/%.cpp + @$(CXX) $(Enclave_Cpp_Flags) -c $< -o $@ + @echo "CXX <= $<" + +libvmlib.a: $(VMLIB_BUILD_DIR)/libvmlib.a + @cp $< $@ + @echo "CP $@ <= $<" + +$(Enclave_Name): Enclave/Enclave_t.o $(Enclave_Cpp_Objects) libvmlib.a + @$(CXX) $^ -o $@ $(Enclave_Link_Flags) + @echo "LINK => $@" + +$(Signed_Enclave_Name): $(Enclave_Name) + @$(SGX_ENCLAVE_SIGNER) sign -key Enclave/Enclave_private.pem -enclave $(Enclave_Name) -out $@ -config $(Enclave_Config_File) + @echo "SIGN => $@" + +.PHONY: clean + +clean: + @rm -f $(App_Name) $(Enclave_Name) $(Signed_Enclave_Name) $(App_Cpp_Objects) App/Enclave_u.* $(Enclave_Cpp_Objects) Enclave/Enclave_t.* libvmlib.a libvmlib_untrusted.a diff --git a/wasm-micro-runtime/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal b/wasm-micro-runtime/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal new file mode 100644 index 0000000..07b640d --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal @@ -0,0 +1,210 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +######## SGX SDK Settings ######## + +SGX_SDK ?= /opt/intel/sgxsdk +SGX_MODE ?= SIM +SGX_ARCH ?= x64 +SGX_DEBUG ?= 0 +SPEC_TEST ?= 0 + +ifeq ($(shell getconf LONG_BIT), 32) + SGX_ARCH := x86 +else ifeq ($(findstring -m32, $(CXXFLAGS)), -m32) + SGX_ARCH := x86 +endif + +ifeq ($(SGX_ARCH), x86) + SGX_COMMON_CFLAGS := -m32 + SGX_LIBRARY_PATH := $(SGX_SDK)/lib + SGX_ENCLAVE_SIGNER := $(SGX_SDK)/bin/x86/sgx_sign + SGX_EDGER8R := $(SGX_SDK)/bin/x86/sgx_edger8r +else + SGX_COMMON_CFLAGS := -m64 + SGX_LIBRARY_PATH := $(SGX_SDK)/lib64 + SGX_ENCLAVE_SIGNER := $(SGX_SDK)/bin/x64/sgx_sign + SGX_EDGER8R := $(SGX_SDK)/bin/x64/sgx_edger8r +endif + +ifeq ($(SGX_DEBUG), 1) +ifeq ($(SGX_PRERELEASE), 1) +$(error Cannot set SGX_DEBUG and SGX_PRERELEASE at the same time!!) +endif +endif + +ifeq ($(SGX_DEBUG), 1) + SGX_COMMON_CFLAGS += -O0 -g +else + SGX_COMMON_CFLAGS += -O2 +endif + +######## App Settings ######## + +ifneq ($(SGX_MODE), HW) + Urts_Library_Name := sgx_urts_sim +else + Urts_Library_Name := sgx_urts +endif + +App_Cpp_Files := App/App.cpp +App_Include_Paths := -IApp -I$(SGX_SDK)/include + +App_C_Flags := $(SGX_COMMON_CFLAGS) -fPIC -Wno-attributes $(App_Include_Paths) + +# Three configuration modes - Debug, prerelease, release +# Debug - Macro DEBUG enabled. +# Prerelease - Macro NDEBUG and EDEBUG enabled. +# Release - Macro NDEBUG enabled. +ifeq ($(SGX_DEBUG), 1) + App_C_Flags += -DDEBUG -UNDEBUG -UEDEBUG +else ifeq ($(SGX_PRERELEASE), 1) + App_C_Flags += -DNDEBUG -DEDEBUG -UDEBUG +else + App_C_Flags += -DNDEBUG -UEDEBUG -UDEBUG +endif + +App_Cpp_Flags := $(App_C_Flags) -std=c++11 +App_Link_Flags := $(SGX_COMMON_CFLAGS) -L$(SGX_LIBRARY_PATH) -l$(Urts_Library_Name) -lpthread + +ifneq ($(SGX_MODE), HW) + App_Link_Flags += -lsgx_uae_service_sim +else + App_Link_Flags += -lsgx_uae_service +endif + +App_Cpp_Objects := $(App_Cpp_Files:.cpp=.o) + +App_Name := iwasm + +######## Enclave Settings ######## + +ifneq ($(SGX_MODE), HW) + Trts_Library_Name := sgx_trts_sim + Service_Library_Name := sgx_tservice_sim +else + Trts_Library_Name := sgx_trts + Service_Library_Name := sgx_tservice +endif +Crypto_Library_Name := sgx_tcrypto + +WAMR_ROOT := $(CURDIR)/../../../../ + +Enclave_Cpp_Files := Enclave/Enclave.cpp + +Enclave_Include_Paths := -IEnclave -I$(WAMR_ROOT)/core/iwasm/include \ + -I$(WAMR_ROOT)/core/shared/utils \ + -I$(WAMR_ROOT)/core/shared/platform/linux-sgx \ + -I$(SGX_SDK)/include \ + -I$(SGX_SDK)/include/tlibc \ + -I$(SGX_SDK)/include/stlport + +Enclave_C_Flags := $(SGX_COMMON_CFLAGS) -nostdinc -fvisibility=hidden -fpie -fstack-protector $(Enclave_Include_Paths) + +# disable wasi +Enclave_C_Flags += -DWASM_ENABLE_LIBC_WASI=0 + +ifeq ($(SPEC_TEST), 1) + Enclave_C_Flags += -DWASM_ENABLE_SPEC_TEST=1 +else + Enclave_C_Flags += -DWASM_ENABLE_SPEC_TEST=0 +endif +Enclave_Cpp_Flags := $(Enclave_C_Flags) -std=c++03 -nostdinc++ +Enclave_Link_Flags := $(SGX_COMMON_CFLAGS) -Wl,--no-undefined -nostdlib -nodefaultlibs -nostartfiles -L$(SGX_LIBRARY_PATH) \ + -Wl,--whole-archive -l$(Trts_Library_Name) -Wl,--no-whole-archive \ + libvmlib.a \ + -Wl,--start-group -lsgx_tstdc -lsgx_tcxx -l$(Crypto_Library_Name) -l$(Service_Library_Name) -Wl,--end-group \ + -Wl,-Bstatic -Wl,-Bsymbolic -Wl,--no-undefined \ + -Wl,-pie,-eenclave_entry -Wl,--export-dynamic \ + -Wl,--defsym,__ImageBase=0 + +Enclave_Cpp_Objects := $(Enclave_Cpp_Files:.cpp=.o) + +Enclave_Name := enclave.so +Signed_Enclave_Name := enclave.signed.so +Enclave_Config_File := Enclave/Enclave.config.xml + +ifeq ($(SGX_MODE), HW) +ifneq ($(SGX_DEBUG), 1) +ifneq ($(SGX_PRERELEASE), 1) +Build_Mode = HW_RELEASE +endif +endif +endif + + +.PHONY: all run + +ifeq ($(Build_Mode), HW_RELEASE) +all: $(App_Name) $(Enclave_Name) + @echo "The project has been built in release hardware mode." + @echo "Please sign the $(Enclave_Name) first with your signing key before you run the $(App_Name) to launch and access the enclave." + @echo "To sign the enclave use the command:" + @echo " $(SGX_ENCLAVE_SIGNER) sign -key -enclave $(Enclave_Name) -out <$(Signed_Enclave_Name)> -config $(Enclave_Config_File)" + @echo "You can also sign the enclave using an external signing tool. See User's Guide for more details." + @echo "To build the project in simulation mode set SGX_MODE=SIM. To build the project in prerelease mode set SGX_PRERELEASE=1 and SGX_MODE=HW." +else +all: $(App_Name) $(Signed_Enclave_Name) +endif + +run: all +ifneq ($(Build_Mode), HW_RELEASE) + @$(CURDIR)/$(App_Name) + @echo "RUN => $(App_Name) [$(SGX_MODE)|$(SGX_ARCH), OK]" +endif + +######## App Objects ######## + +App/Enclave_u.c: $(SGX_EDGER8R) Enclave/Enclave.edl + @cd App && $(SGX_EDGER8R) --untrusted ../Enclave/Enclave.edl \ + --search-path ../Enclave \ + --search-path $(SGX_SDK)/include \ + --search-path $(WAMR_ROOT)/core/shared/platform/linux-sgx + @echo "GEN => $@" + +App/Enclave_u.o: App/Enclave_u.c + @$(CC) $(App_C_Flags) -c $< -o $@ + @echo "CC <= $<" + +App/%.o: App/%.cpp + @$(CXX) $(App_Cpp_Flags) -c $< -o $@ + @echo "CXX <= $<" + +$(App_Name): App/Enclave_u.o $(App_Cpp_Objects) + @$(CXX) $^ -o $@ $(App_Link_Flags) + @echo "LINK => $@" + + +######## Enclave Objects ######## + +Enclave/Enclave_t.c: $(SGX_EDGER8R) Enclave/Enclave.edl + @cd Enclave && $(SGX_EDGER8R) --trusted ../Enclave/Enclave.edl \ + --search-path ../Enclave \ + --search-path $(SGX_SDK)/include \ + --search-path $(WAMR_ROOT)/core/shared/platform/linux-sgx + @echo "GEN => $@" + +Enclave/Enclave_t.o: Enclave/Enclave_t.c + @$(CC) $(Enclave_C_Flags) -c $< -o $@ + @echo "CC <= $<" + +Enclave/%.o: Enclave/%.cpp + @$(CXX) $(Enclave_Cpp_Flags) -c $< -o $@ + @echo "CXX <= $<" + +libvmlib.a: ../build/libvmlib.a + @cp $< $@ + @echo "CP $@ <= $<" + +$(Enclave_Name): Enclave/Enclave_t.o $(Enclave_Cpp_Objects) libvmlib.a + @$(CXX) $^ -o $@ $(Enclave_Link_Flags) + @echo "LINK => $@" + +$(Signed_Enclave_Name): $(Enclave_Name) + @$(SGX_ENCLAVE_SIGNER) sign -key Enclave/Enclave_private.pem -enclave $(Enclave_Name) -out $@ -config $(Enclave_Config_File) + @echo "SIGN => $@" + +.PHONY: clean + +clean: + @rm -f $(App_Name) $(Enclave_Name) $(Signed_Enclave_Name) $(App_Cpp_Objects) App/Enclave_u.* $(Enclave_Cpp_Objects) Enclave/Enclave_t.* libvmlib.a diff --git a/wasm-micro-runtime/product-mini/platforms/nuttx/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/nuttx/CMakeLists.txt new file mode 100644 index 0000000..e9fe5a9 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/nuttx/CMakeLists.txt @@ -0,0 +1,203 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# select platform configuration setting WAMR_BUILD_TARGET +set(WAMR_BUILD_PLATFORM nuttx) + +if(CONFIG_ARCH_ARMV6M) + set(WAMR_BUILD_TARGET THUMBV6M) +elseif(CONFIG_ARCH_ARMV7A) + set(WAMR_BUILD_TARGET THUMBV7) +elseif(CONFIG_ARCH_ARMV7M) + set(WAMR_BUILD_TARGET THUMBV7EM) +elseif(CONFIG_ARCH_ARMV8M) + set(WAMR_BUILD_TARGET THUMBV8M) +elseif(CONFIG_ARCH_X86) + set(WAMR_BUILD_TARGET X86_32) +elseif(CONFIG_ARCH_X86_64) + set(WAMR_BUILD_TARGET X86_64) +elseif(CONFIG_ARCH_XTENSA) + set(WAMR_BUILD_TARGET XTENSA) +elseif(CONFIG_ARCH_RV64GC OR CONFIG_ARCH_RV64) + set(WAMR_BUILD_TARGET RISCV64) +elseif(CONFIG_ARCH_RV32IM OR CONFIG_ARCH_RV32) + set(WAMR_BUILD_TARGET RISCV32) +elseif(CONFIG_ARCH_SIM) + if(CONFIG_SIM_M32 OR CONFIG_HOST_X86) + set(WAMR_BUILD_TARGET X86_32) + elseif(CONFIG_HOST_ARM) + set(WAMR_BUILD_TARGET ARM) + elseif(CONFIG_HOST_ARM64) + set(WAMR_BUILD_TARGET AARCH64) + else() + set(WAMR_BUILD_TARGET X86_64) + endif() + if(CONFIG_HOST_MACOS) + add_definitions(-DBH_PLATFORM_DARWIN) + endif() +endif() + +if(CONFIG_INTERPRETERS_WAMR_LOG) + add_definitions(-DWASM_ENABLE_LOG=1) +else() + add_definitions(-DWASM_ENABLE_LOG=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_AOT_WORD_ALIGN_READ) + add_definitions(-DWASM_ENABLE_WORD_ALIGN_READ=1) +else() + add_definitions(-DWASM_ENABLE_WORD_ALIGN_READ=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_STACK_GUARD_SIZE) + add_definitions(-DWASM_STACK_GUARD_SIZE=0) +else() + add_definitions( + -DWASM_STACK_GUARD_SIZE=${CONFIG_INTERPRETERS_WAMR_STACK_GUARD_SIZE}) +endif() + +if(CONFIG_INTERPRETERS_WAMR_MEMORY_TRACING) + add_definitions(-DWASM_ENABLE_MEMORY_TRACING=1) +else() + add_definitions(-DWASM_ENABLE_MEMORY_TRACING=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_SHARED_MEMORY) + set(WAMR_BUILD_SHARED_MEMORY 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_BULK_MEMORY) + set(WAMR_BUILD_BULK_MEMORY 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_AOT_STACK_FRAME) + set(WAMR_BUILD_AOT_STACK_FRAME 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_PERF_PROFILING) + set(WAMR_BUILD_PERF_PROFILING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GC) + set(WAMR_BUILD_GC 1) + set(WAMR_BUILD_STRINGREF 1) + set(WAMR_BUILD_REF_TYPES 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GC_MANUALLY) + add_definitions(-DWASM_GC_MANUALLY=1) +else() + add_definitions(-DWASM_GC_MANUALLY=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GC_PERF_PROFILING) + set(WAMR_BUILD_GC_PERF_PROFILING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_ENABLE_EXCE_HANDLING) + set(WAMR_BUILD_EXCE_HANDLING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_TAIL_CALL) + set(WAMR_BUILD_TAIL_CALL 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_MEMORY_PROFILING) + set(WAMR_BUILD_MEMORY_PROFILING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_MULTI_MODULE) + set(WAMR_BUILD_MULTI_MODULE 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIB_PTHREAD_SEMAPHORE) + set(WAMR_BUILD_LIB_PTHREAD_SEMAPHORE 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_DISABLE_HW_BOUND_CHECK) + set(WAMR_DISABLE_HW_BOUND_CHECK 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_CUSTOM_NAME_SECTIONS) + set(WAMR_BUILD_CUSTOM_NAME_SECTION 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL) + set(WAMR_BUILD_GLOBAL_HEAP_POOL 1) + math(EXPR _HEAP_SIZE_ + "${CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL_SIZE} * 1024") + set(WAMR_BUILD_GLOBAL_HEAP_SIZE ${_HEAP_SIZE_}) +endif() + +if (CONFIG_INTERPRETERS_WAMR_MEM_ALLOC_WITH_USAGE) + set(WAMR_BUILD_MEM_ALLOC_WITH_USAGE 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST) + set(WAMR_BUILD_SPEC_TEST 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_REF_TYPES) + set(WAMR_BUILD_REF_TYPES 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_CLASSIC) + # include iwasm_interp.cmake + set(WAMR_BUILD_INTERP 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_FAST) + # enable iwasm_interp.cmake + set(WAMR_BUILD_FAST_INTERP 1) +endif() + +if((CONFIG_INTERPRETERS_WAMR_FAST OR CONFIG_INTERPRETERS_WAMR_CLASSIC) + AND CONFIG_INTERPRETERS_WAMR_MINILOADER) + # enable iwasm_interp.cmake + set(WAMR_BUILD_MINI_LOADER 1) +else() + set(WAMR_BUILD_MINI_LOADER 0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_AOT) + # include iwasm_aot.cmake + set(WAMR_BUILD_AOT 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_DEBUG_INTERP) + # include debug_engine.cmake + set(WAMR_BUILD_DEBUG_INTERP 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN) + # include libc_builtin.cmake + set(WAMR_BUILD_LIBC_BUILTIN 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIBC_WASI) + # include libc_wasi.cmake + set(WAMR_BUILD_LIBC_WASI 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_THREAD_MGR) + # include thread_mgr.cmake + set(WAMR_BUILD_THREAD_MGR 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIB_PTHREAD) + # include lib_pthread.cmake + set(WAMR_BUILD_LIB_PTHREAD 1) +endif() + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../..) + +# enable WAMR build system +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +# NuttX wamr lib complie required: `WAMR_SOURCES` `WAMR_CFLAGS` `WAMR_INCDIRS` +# `WAMR_DEFINITIONS` +set(WAMR_SOURCES ${WAMR_RUNTIME_LIB_SOURCE}) +set(WAMR_CFLAGS -Wno-strict-prototypes -Wno-shadow -Wno-unused-variable + -Wno-int-conversion -Wno-implicit-function-declaration) +get_directory_property(WAMR_INCDIRS INCLUDE_DIRECTORIES) +get_directory_property(WAMR_DEFINITIONS COMPILE_DEFINITIONS) diff --git a/wasm-micro-runtime/product-mini/platforms/nuttx/main.c b/wasm-micro-runtime/product-mini/platforms/nuttx/main.c new file mode 100644 index 0000000..8f0e84a --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/nuttx/main.c @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../posix/main.c" diff --git a/wasm-micro-runtime/product-mini/platforms/nuttx/wamr.mk b/wasm-micro-runtime/product-mini/platforms/nuttx/wamr.mk new file mode 100644 index 0000000..7aac0e3 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/nuttx/wamr.mk @@ -0,0 +1,474 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CORE_ROOT := wamr/core +IWASM_ROOT := wamr/core/iwasm +SHARED_ROOT := wamr/core/shared + +ifeq ($(CONFIG_ARCH_ARMV6M),y) +WAMR_BUILD_TARGET := THUMBV6M +else ifeq ($(CONFIG_ARCH_ARMV7A),y) +WAMR_BUILD_TARGET := THUMBV7 +else ifeq ($(CONFIG_ARCH_ARMV7M),y) +WAMR_BUILD_TARGET := THUMBV7EM +else ifeq ($(CONFIG_ARCH_ARMV8M),y) +WAMR_BUILD_TARGET := THUMBV8M +else ifeq ($(CONFIG_ARCH_ARM64),y) +WAMR_BUILD_TARGET := AARCH64 +else ifeq ($(CONFIG_ARCH_X86),y) +WAMR_BUILD_TARGET := X86_32 +else ifeq ($(CONFIG_ARCH_X86_64),y) +WAMR_BUILD_TARGET := X86_64 +else ifeq ($(CONFIG_ARCH_XTENSA),y) +WAMR_BUILD_TARGET := XTENSA +# RV64GC and RV32IM used in older +# version NuttX +else ifeq ($(CONFIG_ARCH_RV64GC),y) +WAMR_BUILD_TARGET := RISCV64 +else ifeq ($(CONFIG_ARCH_RV32IM),y) +WAMR_BUILD_TARGET := RISCV32 +else ifeq ($(CONFIG_ARCH_RV64),y) +WAMR_BUILD_TARGET := RISCV64 +else ifeq ($(CONFIG_ARCH_RV32),y) +WAMR_BUILD_TARGET := RISCV32 +else ifeq ($(CONFIG_ARCH_SIM),y) +ifeq ($(CONFIG_SIM_M32),y) +WAMR_BUILD_TARGET := X86_32 +else ifeq ($(CONFIG_HOST_X86),y) +WAMR_BUILD_TARGET := X86_32 +else ifeq ($(CONFIG_HOST_ARM),y) +WAMR_BUILD_TARGET := ARM +else ifeq ($(CONFIG_HOST_ARM64),y) +WAMR_BUILD_TARGET := AARCH64 +else +WAMR_BUILD_TARGET := X86_64 +endif +ifeq ($(CONFIG_HOST_MACOS),y) +# Note: invokeNative_em64.s needs BH_PLATFORM_DARWIN +AFLAGS += -DBH_PLATFORM_DARWIN +endif +endif + +WAMR_BUILD_PLATFORM := nuttx + +CFLAGS += -DBH_MALLOC=wasm_runtime_malloc +CFLAGS += -DBH_FREE=wasm_runtime_free + +ifeq ($(WAMR_BUILD_TARGET), X86_32) + CFLAGS += -DBUILD_TARGET_X86_32 + INVOKE_NATIVE := invokeNative_ia32.s + AOT_RELOC := aot_reloc_x86_32.c +else ifeq ($(WAMR_BUILD_TARGET), X86_64) + CFLAGS += -DBUILD_TARGET_X86_64 + INVOKE_NATIVE := invokeNative_em64.s + AOT_RELOC := aot_reloc_x86_64.c +else ifeq ($(WAMR_BUILD_TARGET), AARCH64) + CFLAGS += -DBUILD_TARGET_AARCH64 + CFLAGS += -DBUILD_TARGET=\"$(WAMR_BUILD_TARGET)\" + INVOKE_NATIVE := invokeNative_aarch64.s + AOT_RELOC := aot_reloc_aarch64.c +else ifeq ($(findstring ARM,$(WAMR_BUILD_TARGET)), ARM) + CFLAGS += -DBUILD_TARGET_ARM + CFLAGS += -DBUILD_TARGET=\"$(WAMR_BUILD_TARGET)\" + INVOKE_NATIVE := invokeNative_arm.s + AOT_RELOC := aot_reloc_arm.c +else ifeq ($(findstring THUMB,$(WAMR_BUILD_TARGET)), THUMB) + CFLAGS += -DBUILD_TARGET=\"$(WAMR_BUILD_TARGET)\" + ifeq ($(CONFIG_ARCH_FPU),y) + CFLAGS += -DBUILD_TARGET_THUMB_VFP + INVOKE_NATIVE := invokeNative_thumb_vfp.s + else + CFLAGS += -DBUILD_TARGET_THUMB + INVOKE_NATIVE := invokeNative_thumb.s + endif + AOT_RELOC := aot_reloc_thumb.c +else ifeq (${WAMR_BUILD_TARGET}, MIPS) + CFLAGS += -DBUILD_TARGET_MIPS + INVOKE_NATIVE := invokeNative_mips.s + AOT_RELOC := aot_reloc_mips.c +else ifeq (${WAMR_BUILD_TARGET}, XTENSA) + CFLAGS += -DBUILD_TARGET_XTENSA + INVOKE_NATIVE := invokeNative_xtensa.s + AOT_RELOC := aot_reloc_xtensa.c +else ifeq (${WAMR_BUILD_TARGET}, RISCV64) + +ifeq (${CONFIG_ARCH_DPFPU},y) + CFLAGS += -DBUILD_TARGET_RISCV64_LP64D +else ifneq (${CONFIG_ARCH_FPU},y) + CFLAGS += -DBUILD_TARGET_RISCV64_LP64 +else + $(error riscv64 lp64f is unsupported) +endif + INVOKE_NATIVE += invokeNative_riscv.S + + AOT_RELOC := aot_reloc_riscv.c + +else ifeq (${WAMR_BUILD_TARGET}, RISCV32) + +ifeq (${CONFIG_ARCH_DPFPU},y) + CFLAGS += -DBUILD_TARGET_RISCV32_ILP32D +else ifneq (${CONFIG_ARCH_FPU},y) + CFLAGS += -DBUILD_TARGET_RISCV32_ILP32 +else + $(error riscv32 ilp32f is unsupported) +endif + + INVOKE_NATIVE += invokeNative_riscv.S + AOT_RELOC := aot_reloc_riscv.c + +else + $(error Build target is unsupported) +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_LOG),y) +CFLAGS += -DWASM_ENABLE_LOG=1 +else +CFLAGS += -DWASM_ENABLE_LOG=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_AOT),y) +CFLAGS += -I$(IWASM_ROOT)/aot +CFLAGS += -DWASM_ENABLE_AOT=1 +CSRCS += aot_loader.c \ + $(AOT_RELOC) \ + aot_intrinsic.c \ + aot_runtime.c +ifeq ($(CONFIG_INTERPRETERS_WAMR_DEBUG_AOT),y) +CFLAGS += -DWASM_ENABLE_DEBUG_AOT=1 +CSRCS += elf_parser.c \ + jit_debug.c +endif +else +CFLAGS += -DWASM_ENABLE_AOT=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_AOT_QUICK_ENTRY),y) +CFLAGS += -DWASM_ENABLE_QUICK_AOT_ENTRY=1 +else +CFLAGS += -DWASM_ENABLE_QUICK_AOT_ENTRY=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_AOT_WORD_ALIGN_READ),y) +CFLAGS += -DWASM_ENABLE_WORD_ALIGN_READ=1 +else +CFLAGS += -DWASM_ENABLE_WORD_ALIGN_READ=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_MEM_DUAL_BUS_MIRROR),y) +CFLAGS += -DWASM_MEM_DUAL_BUS_MIRROR=1 +else +CFLAGS += -DWASM_MEM_DUAL_BUS_MIRROR=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_FAST), y) +CFLAGS += -DWASM_ENABLE_FAST_INTERP=1 +CFLAGS += -DWASM_ENABLE_INTERP=1 +CSRCS += wasm_interp_fast.c +CSRCS += wasm_runtime.c +else +CFLAGS += -DWASM_ENABLE_FAST_INTERP=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_CLASSIC), y) +CFLAGS += -DWASM_ENABLE_INTERP=1 +CSRCS += wasm_interp_classic.c +CSRCS += wasm_runtime.c +endif + +ifeq ($(findstring y,$(CONFIG_INTERPRETERS_WAMR_FAST)$(CONFIG_INTERPRETERS_WAMR_CLASSIC)), y) +ifeq ($(CONFIG_INTERPRETERS_WAMR_MINILOADER),y) +CFLAGS += -DWASM_ENABLE_MINI_LOADER=1 +CSRCS += wasm_mini_loader.c +else +CFLAGS += -DWASM_ENABLE_MINI_LOADER=0 +CSRCS += wasm_loader.c +endif +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_DEBUG_INTERP),y) +# Note: INTERPRETERS_WAMR_CLASSIC/INTERPRETERS_WAMR_THREAD_MGR +# dependencies are already handled in NuttX apps Kconfig +CFLAGS += -DWASM_ENABLE_DEBUG_INTERP=1 +CFLAGS += -I$(IWASM_ROOT)/libraries/debug-engine +CSRCS += debug_engine.c +CSRCS += gdbserver.c +CSRCS += handler.c +CSRCS += packets.c +CSRCS += utils.c +VPATH += $(IWASM_ROOT)/libraries/debug-engine +endif + +ifneq ($(CONFIG_INTERPRETERS_WAMR_STACK_GUARD_SIZE),) +CFLAGS += -DWASM_STACK_GUARD_SIZE=CONFIG_INTERPRETERS_WAMR_STACK_GUARD_SIZE +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_SHARED_MEMORY),y) +CFLAGS += -DWASM_ENABLE_SHARED_MEMORY=1 +CSRCS += wasm_shared_memory.c +else +CFLAGS += -DWASM_ENABLE_SHARED_MEMORY=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_BULK_MEMORY),y) +CFLAGS += -DWASM_ENABLE_BULK_MEMORY=1 +else +CFLAGS += -DWASM_ENABLE_BULK_MEMORY=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_AOT_STACK_FRAME), y) +CFLAGS += -DWASM_ENABLE_AOT_STACK_FRAME=1 +else +CFLAGS += -DWASM_ENABLE_AOT_STACK_FRAME=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_PERF_PROFILING),y) +CFLAGS += -DWASM_ENABLE_PERF_PROFILING=1 +CFLAGS += -DWASM_ENABLE_AOT_STACK_FRAME=1 +else +CFLAGS += -DWASM_ENABLE_PERF_PROFILING=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_GC_PERF_PROFILING),y) +CFLAGS += -DWASM_ENABLE_GC_PERF_PROFILING=1 +else +CFLAGS += -DWASM_ENABLE_GC_PERF_PROFILING=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_MEMORY_PROFILING),y) +CFLAGS += -DWASM_ENABLE_MEMORY_PROFILING=1 +else +CFLAGS += -DWASM_ENABLE_MEMORY_PROFILING=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_MEMORY_TRACING),y) +CFLAGS += -DWASM_ENABLE_MEMORY_TRACING=1 +else +CFLAGS += -DWASM_ENABLE_MEMORY_TRACING=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_DUMP_CALL_STACK),y) +CFLAGS += -DWASM_ENABLE_DUMP_CALL_STACK=1 +CFLAGS += -DWASM_ENABLE_AOT_STACK_FRAME=1 +else +CFLAGS += -DWASM_ENABLE_DUMP_CALL_STACK=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN),y) +CFLAGS += -DWASM_ENABLE_LIBC_BUILTIN=1 +CSRCS += libc_builtin_wrapper.c +VPATH += $(IWASM_ROOT)/libraries/libc-builtin +else +CFLAGS += -DWASM_ENABLE_LIBC_BUILTIN=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_CONFIGURABLE_BOUNDS_CHECKS),y) +CFLAGS += -DWASM_CONFIGURABLE_BOUNDS_CHECKS=1 +else +CFLAGS += -DWASM_CONFIGURABLE_BOUNDS_CHECKS=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_LIBC_WASI),y) +CFLAGS += -DWASM_ENABLE_LIBC_WASI=1 +CFLAGS += -I$(IWASM_ROOT)/libraries/libc-wasi/sandboxed-system-primitives/src +CFLAGS += -I$(IWASM_ROOT)/libraries/libc-wasi/sandboxed-system-primitives/include +CFLAGS += -I${SHARED_ROOT}/platform/common/libc-util +CSRCS += blocking_op.c +CSRCS += posix_socket.c +CSRCS += posix_file.c +CSRCS += posix_clock.c +CSRCS += libc_errno.c +CSRCS += libc_wasi_wrapper.c +VPATH += $(IWASM_ROOT)/libraries/libc-wasi +CSRCS += posix.c +CSRCS += random.c +CSRCS += str.c +VPATH += $(IWASM_ROOT)/libraries/libc-wasi/sandboxed-system-primitives/src +# todo: use Kconfig select instead +CONFIG_INTERPRETERS_WAMR_MODULE_INSTANCE_CONTEXT = y +else +CFLAGS += -DWASM_ENABLE_LIBC_WASI=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_MODULE_INSTANCE_CONTEXT),y) +CFLAGS += -DWASM_ENABLE_MODULE_INST_CONTEXT=1 +else +CFLAGS += -DWASM_ENABLE_MODULE_INST_CONTEXT=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_MULTI_MODULE),y) +CFLAGS += -DWASM_ENABLE_MULTI_MODULE=1 +else +CFLAGS += -DWASM_ENABLE_MULTI_MODULE=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_THREAD_MGR),y) +CFLAGS += -DWASM_ENABLE_THREAD_MGR=1 +CSRCS += thread_manager.c +VPATH += $(IWASM_ROOT)/libraries/thread-mgr +else +CFLAGS += -DWASM_ENABLE_THREAD_MGR=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_LIB_WASI_THREADS),y) +CFLAGS += -DWASM_ENABLE_LIB_WASI_THREADS=1 +CSRCS += lib_wasi_threads_wrapper.c +CSRCS += tid_allocator.c +VPATH += $(IWASM_ROOT)/libraries/lib-wasi-threads +else +CFLAGS += -DWASM_ENABLE_LIB_WASI_THREADS=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_GC),y) +CFLAGS += -DWASM_ENABLE_GC=1 +CSRCS += gc_common.c gc_type.c gc_object.c +VPATH += $(IWASM_ROOT)/common/gc +else +CFLAGS += -DWASM_ENABLE_GC=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_GC_MANUALLY),y) +CFLAGS += -DWASM_GC_MANUALLY=1 +else +CFLAGS += -DWASM_GC_MANUALLY=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_LIB_PTHREAD),y) +CFLAGS += -DWASM_ENABLE_LIB_PTHREAD=1 +CSRCS += lib_pthread_wrapper.c +else +CFLAGS += -DWASM_ENABLE_LIB_PTHREAD=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_LIB_PTHREAD_SEMAPHORE),y) +CFLAGS += -DWASM_ENABLE_LIB_PTHREAD_SEMAPHORE=1 +else +CFLAGS += -DWASM_ENABLE_LIB_PTHREAD_SEMAPHORE=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_DISABLE_HW_BOUND_CHECK),y) +CFLAGS += -DWASM_DISABLE_HW_BOUND_CHECK=1 +CFLAGS += -DWASM_DISABLE_STACK_HW_BOUND_CHECK=1 +else +CFLAGS += -DWASM_DISABLE_HW_BOUND_CHECK=0 +CFLAGS += -DWASM_DISABLE_STACK_HW_BOUND_CHECK=0 +endif + +# REVISIT: is this worth to have a Kconfig? +CFLAGS += -DWASM_DISABLE_WAKEUP_BLOCKING_OP=0 + +ifeq ($(CONFIG_INTERPRETERS_WAMR_LOAD_CUSTOM_SECTIONS),y) +CFLAGS += -DWASM_ENABLE_LOAD_CUSTOM_SECTION=1 +else +CFLAGS += -DWASM_ENABLE_LOAD_CUSTOM_SECTION=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_CUSTOM_NAME_SECTIONS),y) +CFLAGS += -DWASM_ENABLE_CUSTOM_NAME_SECTION=1 +else +CFLAGS += -DWASM_ENABLE_CUSTOM_NAME_SECTION=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL),y) +CFLAGS += -DWASM_ENABLE_GLOBAL_HEAP_POOL=1 +CFLAGS += -DWASM_GLOBAL_HEAP_SIZE="$(CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL_SIZE) * 1024" +else +CFLAGS += -DWASM_ENABLE_GLOBAL_HEAP_POOL=0 +ifeq ($(CONFIG_INTERPRETERS_WAMR_MEM_ALLOC_WITH_USAGE),y) +CFLAGS += -DWASM_MEM_ALLOC_WITH_USAGE=1 +else +CFLAGS += -DWASM_MEM_ALLOC_WITH_USAGE=0 +endif +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST),y) +CFLAGS += -DWASM_ENABLE_SPEC_TEST=1 +else +CFLAGS += -DWASM_ENABLE_SPEC_TEST=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_REF_TYPES),y) +CFLAGS += -DWASM_ENABLE_REF_TYPES=1 +else +CFLAGS += -DWASM_ENABLE_REF_TYPES=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_TAIL_CALL),y) +CFLAGS += -DWASM_ENABLE_TAIL_CALL=1 +else +CFLAGS += -DWASM_ENABLE_TAIL_CALL=0 +endif + +ifeq ($(CONFIG_INTERPRETERS_WAMR_ENABLE_EXCE_HANDLING),y) +CFLAGS += -DWASM_ENABLE_EXCE_HANDLING=1 +CFLAGS += -DWASM_ENABLE_TAGS=1 +else +CFLAGS += -DWASM_ENABLE_EXCE_HANDLING=0 +CFLAGS += -DWASM_ENABLE_TAGS=0 +endif + +CFLAGS += -Wno-strict-prototypes -Wno-shadow -Wno-unused-variable +CFLAGS += -Wno-int-conversion -Wno-implicit-function-declaration + +CFLAGS += -I${CORE_ROOT} \ + -I${IWASM_ROOT}/include \ + -I${IWASM_ROOT}/interpreter \ + -I${IWASM_ROOT}/common \ + -I${IWASM_ROOT}/libraries/thread-mgr \ + -I${SHARED_ROOT}/include \ + -I${SHARED_ROOT}/platform/include \ + -I${SHARED_ROOT}/utils \ + -I${SHARED_ROOT}/utils/uncommon \ + -I${SHARED_ROOT}/mem-alloc \ + -I${SHARED_ROOT}/platform/nuttx + +ifeq ($(WAMR_BUILD_INTERP), 1) +CFLAGS += -I$(IWASM_ROOT)/interpreter +endif + +CSRCS += nuttx_platform.c \ + posix_blocking_op.c \ + posix_thread.c \ + posix_time.c \ + posix_sleep.c \ + mremap.c \ + mem_alloc.c \ + ems_kfc.c \ + ems_alloc.c \ + ems_hmu.c \ + ems_gc.c \ + bh_assert.c \ + bh_bitmap.c \ + bh_common.c \ + bh_hashmap.c \ + bh_list.c \ + bh_log.c \ + bh_queue.c \ + bh_vector.c \ + bh_read_file.c \ + runtime_timer.c \ + wasm_application.c \ + wasm_blocking_op.c \ + wasm_runtime_common.c \ + wasm_native.c \ + wasm_exec_env.c \ + wasm_memory.c \ + wasm_c_api.c + +ASRCS += $(INVOKE_NATIVE) + +VPATH += $(SHARED_ROOT)/platform/nuttx +VPATH += $(SHARED_ROOT)/platform/common/memory +VPATH += $(SHARED_ROOT)/platform/common/posix +VPATH += $(SHARED_ROOT)/platform/common/libc-util +VPATH += $(SHARED_ROOT)/mem-alloc +VPATH += $(SHARED_ROOT)/mem-alloc/ems +VPATH += $(SHARED_ROOT)/utils +VPATH += $(SHARED_ROOT)/utils/uncommon +VPATH += $(IWASM_ROOT)/common +VPATH += $(IWASM_ROOT)/interpreter +VPATH += $(IWASM_ROOT)/libraries +VPATH += $(IWASM_ROOT)/libraries/lib-pthread +VPATH += $(IWASM_ROOT)/common/arch +VPATH += $(IWASM_ROOT)/aot +VPATH += $(IWASM_ROOT)/aot/arch +VPATH += $(IWASM_ROOT)/aot/debug diff --git a/wasm-micro-runtime/product-mini/platforms/posix/main.c b/wasm-micro-runtime/product-mini/platforms/posix/main.c new file mode 100644 index 0000000..cb0581c --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/posix/main.c @@ -0,0 +1,1044 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include + +#include "bh_platform.h" +#include "bh_read_file.h" +#include "wasm_export.h" + +#if WASM_ENABLE_LIBC_WASI != 0 +#include "../common/libc_wasi.c" +#endif + +#if BH_HAS_DLFCN +#include +#endif + +static int app_argc; +static char **app_argv; + +/* clang-format off */ +static int +print_help() +{ + printf("Usage: iwasm [-options] wasm_file [args...]\n"); + printf("options:\n"); + printf(" -f|--function name Specify a function name of the module to run rather\n" + " than main\n"); +#if WASM_ENABLE_LOG != 0 + printf(" -v=n Set log verbose level (0 to 5, default is 2) larger\n" + " level with more log\n"); +#endif +#if WASM_ENABLE_INTERP != 0 + printf(" --interp Run the wasm app with interpreter mode\n"); +#endif +#if WASM_ENABLE_FAST_JIT != 0 + printf(" --fast-jit Run the wasm app with fast jit mode\n"); +#endif +#if WASM_ENABLE_JIT != 0 + printf(" --llvm-jit Run the wasm app with llvm jit mode\n"); +#endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + printf(" --multi-tier-jit Run the wasm app with multi-tier jit mode\n"); +#endif + printf(" --stack-size=n Set maximum stack size in bytes, default is 64 KB\n"); + printf(" --heap-size=n Set maximum heap size in bytes, default is 16 KB\n"); +#if WASM_ENABLE_FAST_JIT != 0 + printf(" --jit-codecache-size=n Set fast jit maximum code cache size in bytes,\n"); + printf(" default is %u KB\n", FAST_JIT_DEFAULT_CODE_CACHE_SIZE / 1024); +#endif +#if WASM_ENABLE_GC != 0 + printf(" --gc-heap-size=n Set maximum gc heap size in bytes,\n"); + printf(" default is %u KB\n", GC_HEAP_SIZE_DEFAULT / 1024); +#endif +#if WASM_ENABLE_JIT != 0 + printf(" --llvm-jit-size-level=n Set LLVM JIT size level, default is 3\n"); + printf(" --llvm-jit-opt-level=n Set LLVM JIT optimization level, default is 3\n"); +#if defined(os_writegsbase) + printf(" --enable-segue[=] Enable using segment register GS as the base address of\n"); + printf(" linear memory, which may improve performance, flags can be:\n"); + printf(" i32.load, i64.load, f32.load, f64.load, v128.load,\n"); + printf(" i32.store, i64.store, f32.store, f64.store, v128.store\n"); + printf(" Use comma to separate, e.g. --enable-segue=i32.load,i64.store\n"); + printf(" and --enable-segue means all flags are added.\n"); +#endif +#endif /* WASM_ENABLE_JIT != 0*/ +#if WASM_ENABLE_LINUX_PERF != 0 + printf(" --enable-linux-perf Enable linux perf support. It works in aot and llvm-jit.\n"); +#endif + printf(" --repl Start a very simple REPL (read-eval-print-loop) mode\n" + " that runs commands in the form of \"FUNC ARG...\"\n"); +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + printf(" --disable-bounds-checks Disable bounds checks for memory accesses\n"); +#endif +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_print_help(); +#endif +#if BH_HAS_DLFCN + printf(" --native-lib= Register native libraries to the WASM module, which\n"); + printf(" are shared object (.so) files, for example:\n"); + printf(" --native-lib=test1.so --native-lib=test2.so\n"); +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + printf(" --module-path= Indicate a module search path. default is current\n" + " directory('./')\n"); +#endif +#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + printf(" --max-threads=n Set maximum thread number per cluster, default is 4\n"); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + printf(" --timeout=ms Set the maximum execution time in ms.\n"); + printf(" If it expires, the runtime aborts the execution\n"); + printf(" with a trap.\n"); +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + printf(" -g=ip:port Set the debug sever address, default is debug disabled\n"); + printf(" if port is 0, then a random port will be used\n"); +#endif +#if WASM_ENABLE_STATIC_PGO != 0 + printf(" --gen-prof-file= Generate LLVM PGO (Profile-Guided Optimization) profile file\n"); +#endif + printf(" --version Show version information\n"); + return 1; +} +/* clang-format on */ + +static const void * +app_instance_main(wasm_module_inst_t module_inst) +{ + const char *exception; + + wasm_application_execute_main(module_inst, app_argc, app_argv); + exception = wasm_runtime_get_exception(module_inst); + return exception; +} + +static const void * +app_instance_func(wasm_module_inst_t module_inst, const char *func_name) +{ + wasm_application_execute_func(module_inst, func_name, app_argc - 1, + app_argv + 1); + /* The result of wasm function or exception info was output inside + wasm_application_execute_func(), here we don't output them again. */ + return wasm_runtime_get_exception(module_inst); +} + +/** + * Split a string into an array of strings + * Returns NULL on failure + * Memory must be freed by caller + * Based on: http://stackoverflow.com/a/11198630/471795 + */ +static char ** +split_string(char *str, int *count, const char *delimer) +{ + char **res = NULL, **res1; + char *p; + int idx = 0; + + /* split string and append tokens to 'res' */ + do { + p = strtok(str, delimer); + str = NULL; + res1 = res; + res = (char **)realloc(res1, sizeof(char *) * (uint32)(idx + 1)); + if (res == NULL) { + free(res1); + return NULL; + } + res[idx++] = p; + } while (p); + + /** + * Due to the function name, + * res[0] might contain a '\' to indicate a space + * func\name -> func name + */ + p = strchr(res[0], '\\'); + while (p) { + *p = ' '; + p = strchr(p, '\\'); + } + + if (count) { + *count = idx - 1; + } + return res; +} + +static void * +app_instance_repl(wasm_module_inst_t module_inst) +{ + char *cmd = NULL; + size_t len = 0; + ssize_t n; + + while ((printf("webassembly> "), fflush(stdout), + n = getline(&cmd, &len, stdin)) + != -1) { + bh_assert(n > 0); + if (cmd[n - 1] == '\n') { + if (n == 1) + continue; + else + cmd[n - 1] = '\0'; + } + if (!strcmp(cmd, "__exit__")) { + printf("exit repl mode\n"); + break; + } + app_argv = split_string(cmd, &app_argc, " "); + if (app_argv == NULL) { + LOG_ERROR("Wasm prepare param failed: split string failed.\n"); + break; + } + if (app_argc != 0) { + const char *exception; + wasm_application_execute_func(module_inst, app_argv[0], + app_argc - 1, app_argv + 1); + if ((exception = wasm_runtime_get_exception(module_inst))) + printf("%s\n", exception); + } + free(app_argv); + } + free(cmd); + return NULL; +} + +#if WASM_ENABLE_JIT != 0 +static uint32 +resolve_segue_flags(char *str_flags) +{ + uint32 segue_flags = 0; + int32 flag_count, i; + char **flag_list; + + flag_list = split_string(str_flags, &flag_count, ","); + if (flag_list) { + for (i = 0; i < flag_count; i++) { + if (!strcmp(flag_list[i], "i32.load")) { + segue_flags |= 1 << 0; + } + else if (!strcmp(flag_list[i], "i64.load")) { + segue_flags |= 1 << 1; + } + else if (!strcmp(flag_list[i], "f32.load")) { + segue_flags |= 1 << 2; + } + else if (!strcmp(flag_list[i], "f64.load")) { + segue_flags |= 1 << 3; + } + else if (!strcmp(flag_list[i], "v128.load")) { + segue_flags |= 1 << 4; + } + else if (!strcmp(flag_list[i], "i32.store")) { + segue_flags |= 1 << 8; + } + else if (!strcmp(flag_list[i], "i64.store")) { + segue_flags |= 1 << 9; + } + else if (!strcmp(flag_list[i], "f32.store")) { + segue_flags |= 1 << 10; + } + else if (!strcmp(flag_list[i], "f64.store")) { + segue_flags |= 1 << 11; + } + else if (!strcmp(flag_list[i], "v128.store")) { + segue_flags |= 1 << 12; + } + else { + /* invalid flag */ + segue_flags = (uint32)-1; + break; + } + } + free(flag_list); + } + return segue_flags; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if BH_HAS_DLFCN +struct native_lib { + void *handle; + + uint32 (*get_native_lib)(char **p_module_name, + NativeSymbol **p_native_symbols); + int (*init_native_lib)(void); + void (*deinit_native_lib)(void); + + char *module_name; + NativeSymbol *native_symbols; + uint32 n_native_symbols; +}; + +struct native_lib * +load_native_lib(const char *name) +{ + struct native_lib *lib = wasm_runtime_malloc(sizeof(*lib)); + if (lib == NULL) { + LOG_WARNING("warning: failed to load native library %s because of " + "allocation failure", + name); + goto fail; + } + memset(lib, 0, sizeof(*lib)); + + /* open the native library */ + if (!(lib->handle = dlopen(name, RTLD_NOW | RTLD_GLOBAL)) + && !(lib->handle = dlopen(name, RTLD_LAZY))) { + LOG_WARNING("warning: failed to load native library %s", name); + goto fail; + } + + lib->init_native_lib = dlsym(lib->handle, "init_native_lib"); + lib->get_native_lib = dlsym(lib->handle, "get_native_lib"); + lib->deinit_native_lib = dlsym(lib->handle, "deinit_native_lib"); + + if (!lib->get_native_lib) { + LOG_WARNING("warning: failed to lookup `get_native_lib` function " + "from native lib %s", + name); + goto fail; + } + + if (lib->init_native_lib) { + int ret = lib->init_native_lib(); + if (ret != 0) { + LOG_WARNING("warning: `init_native_lib` function from native " + "lib %s failed with %d", + name, ret); + goto fail; + } + } + + lib->n_native_symbols = + lib->get_native_lib(&lib->module_name, &lib->native_symbols); + + /* register native symbols */ + if (!(lib->n_native_symbols > 0 && lib->module_name && lib->native_symbols + && wasm_runtime_register_natives( + lib->module_name, lib->native_symbols, lib->n_native_symbols))) { + LOG_WARNING("warning: failed to register native lib %s", name); + if (lib->deinit_native_lib) { + lib->deinit_native_lib(); + } + goto fail; + } + return lib; +fail: + if (lib != NULL) { + if (lib->handle != NULL) { + dlclose(lib->handle); + } + wasm_runtime_free(lib); + } + return NULL; +} + +static uint32 +load_and_register_native_libs(const char **native_lib_list, + uint32 native_lib_count, + struct native_lib **native_lib_loaded_list) +{ + uint32 i, native_lib_loaded_count = 0; + + for (i = 0; i < native_lib_count; i++) { + struct native_lib *lib = load_native_lib(native_lib_list[i]); + if (lib == NULL) { + continue; + } + native_lib_loaded_list[native_lib_loaded_count++] = lib; + } + + return native_lib_loaded_count; +} + +static void +unregister_and_unload_native_libs(uint32 native_lib_count, + struct native_lib **native_lib_loaded_list) +{ + uint32 i; + + for (i = 0; i < native_lib_count; i++) { + struct native_lib *lib = native_lib_loaded_list[i]; + + /* unregister native symbols */ + if (!wasm_runtime_unregister_natives(lib->module_name, + lib->native_symbols)) { + LOG_WARNING("warning: failed to unregister native lib %p", + lib->handle); + continue; + } + + if (lib->deinit_native_lib) { + lib->deinit_native_lib(); + } + + dlclose(lib->handle); + wasm_runtime_free(lib); + } +} +#endif /* BH_HAS_DLFCN */ + +#if WASM_ENABLE_MULTI_MODULE != 0 +static char * +handle_module_path(const char *module_path) +{ + /* next character after = */ + return (strchr(module_path, '=')) + 1; +} + +static char *module_search_path = "."; + +static bool +module_reader_callback(package_type_t module_type, const char *module_name, + uint8 **p_buffer, uint32 *p_size) +{ + char *file_format = NULL; +#if WASM_ENABLE_INTERP != 0 + if (module_type == Wasm_Module_Bytecode) + file_format = ".wasm"; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_type == Wasm_Module_AoT) + file_format = ".aot"; +#endif + bh_assert(file_format); + const char *format = "%s/%s%s"; + int sz = strlen(module_search_path) + strlen("/") + strlen(module_name) + + strlen(file_format) + 1; + char *wasm_file_name = wasm_runtime_malloc(sz); + if (!wasm_file_name) { + return false; + } + snprintf(wasm_file_name, sz, format, module_search_path, module_name, + file_format); + *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_name, p_size); + + wasm_runtime_free(wasm_file_name); + return *p_buffer != NULL; +} + +static void +moudle_destroyer(uint8 *buffer, uint32 size) +{ + if (!buffer) { + return; + } + + wasm_runtime_free(buffer); + buffer = NULL; +} +#endif /* WASM_ENABLE_MULTI_MODULE */ + +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 +static char global_heap_buf[WASM_GLOBAL_HEAP_SIZE] = { 0 }; +#else +static void * +malloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + unsigned int size) +{ + return malloc(size); +} + +static void * +realloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, bool full_size_mmaped, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr, unsigned int size) +{ + return realloc(ptr, size); +} + +static void +free_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr) +{ + free(ptr); +} +#endif /* end of WASM_ENABLE_GLOBAL_HEAP_POOL */ + +#if WASM_ENABLE_STATIC_PGO != 0 +static void +dump_pgo_prof_data(wasm_module_inst_t module_inst, const char *path) +{ + char *buf; + uint32 len; + FILE *file; + + if (!(len = wasm_runtime_get_pgo_prof_data_size(module_inst))) { + printf("failed to get LLVM PGO profile data size\n"); + return; + } + + if (!(buf = wasm_runtime_malloc(len))) { + printf("allocate memory failed\n"); + return; + } + + if (len != wasm_runtime_dump_pgo_prof_data_to_buf(module_inst, buf, len)) { + printf("failed to dump LLVM PGO profile data\n"); + wasm_runtime_free(buf); + return; + } + + if (!(file = fopen(path, "wb"))) { + printf("failed to create file %s", path); + wasm_runtime_free(buf); + return; + } + fwrite(buf, len, 1, file); + fclose(file); + + wasm_runtime_free(buf); + + printf("LLVM raw profile file %s was generated.\n", path); +} +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +struct timeout_arg { + uint32 timeout_ms; + wasm_module_inst_t inst; +#if defined(BH_HAS_STD_ATOMIC) + _Atomic +#endif + bool cancel; +}; + +void * +timeout_thread(void *vp) +{ + const struct timeout_arg *arg = vp; + uint32 left = arg->timeout_ms; + while (!arg->cancel) { + uint32 ms; + if (left >= 100) { + ms = 100; + } + else { + ms = left; + } + os_usleep((uint64)ms * 1000); + left -= ms; + if (left == 0) { + wasm_runtime_terminate(arg->inst); + break; + } + } + return NULL; +} +#endif + +int +main(int argc, char *argv[]) +{ + int32 ret = -1; + char *wasm_file = NULL; + const char *func_name = NULL; + uint8 *wasm_file_buf = NULL; + uint32 wasm_file_size; + uint32 stack_size = 64 * 1024; +#if WASM_ENABLE_LIBC_WASI != 0 + uint32 heap_size = 0; +#else + uint32 heap_size = 16 * 1024; +#endif +#if WASM_ENABLE_FAST_JIT != 0 + uint32 jit_code_cache_size = FAST_JIT_DEFAULT_CODE_CACHE_SIZE; +#endif +#if WASM_ENABLE_GC != 0 + uint32 gc_heap_size = GC_HEAP_SIZE_DEFAULT; +#endif +#if WASM_ENABLE_JIT != 0 + uint32 llvm_jit_size_level = 3; + uint32 llvm_jit_opt_level = 3; + uint32 segue_flags = 0; +#endif +#if WASM_ENABLE_LINUX_PERF != 0 + bool enable_linux_perf = false; +#endif + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + RunningMode running_mode = 0; + RuntimeInitArgs init_args; + char error_buf[128] = { 0 }; +#if WASM_ENABLE_LOG != 0 + int log_verbose_level = 2; +#endif + bool is_repl_mode = false; + bool is_xip_file = false; +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + bool disable_bounds_checks = false; +#endif +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_parse_context_t wasi_parse_ctx; +#endif +#if BH_HAS_DLFCN + const char *native_lib_list[8] = { NULL }; + uint32 native_lib_count = 0; + struct native_lib *native_lib_loaded_list[8]; + uint32 native_lib_loaded_count = 0; +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + char *ip_addr = NULL; + int instance_port = 0; +#endif +#if WASM_ENABLE_STATIC_PGO != 0 + const char *gen_prof_file = NULL; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + int timeout_ms = -1; +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 + memset(&wasi_parse_ctx, 0, sizeof(wasi_parse_ctx)); +#endif + + /* Process options. */ + for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { + if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--function")) { + argc--, argv++; + if (argc < 2) { + return print_help(); + } + func_name = argv[0]; + } +#if WASM_ENABLE_INTERP != 0 + else if (!strcmp(argv[0], "--interp")) { + running_mode = Mode_Interp; + } +#endif +#if WASM_ENABLE_FAST_JIT != 0 + else if (!strcmp(argv[0], "--fast-jit")) { + running_mode = Mode_Fast_JIT; + } +#endif +#if WASM_ENABLE_JIT != 0 + else if (!strcmp(argv[0], "--llvm-jit")) { + running_mode = Mode_LLVM_JIT; + } +#endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + else if (!strcmp(argv[0], "--multi-tier-jit")) { + running_mode = Mode_Multi_Tier_JIT; + } +#endif +#if WASM_ENABLE_LOG != 0 + else if (!strncmp(argv[0], "-v=", 3)) { + log_verbose_level = atoi(argv[0] + 3); + if (log_verbose_level < 0 || log_verbose_level > 5) + return print_help(); + } +#endif + else if (!strcmp(argv[0], "--repl")) { + is_repl_mode = true; + } +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + else if (!strcmp(argv[0], "--disable-bounds-checks")) { + disable_bounds_checks = true; + } +#endif + else if (!strncmp(argv[0], "--stack-size=", 13)) { + if (argv[0][13] == '\0') + return print_help(); + stack_size = atoi(argv[0] + 13); + } + else if (!strncmp(argv[0], "--heap-size=", 12)) { + if (argv[0][12] == '\0') + return print_help(); + heap_size = atoi(argv[0] + 12); + } +#if WASM_ENABLE_FAST_JIT != 0 + else if (!strncmp(argv[0], "--jit-codecache-size=", 21)) { + if (argv[0][21] == '\0') + return print_help(); + jit_code_cache_size = atoi(argv[0] + 21); + } +#endif +#if WASM_ENABLE_GC != 0 + else if (!strncmp(argv[0], "--gc-heap-size=", 15)) { + if (argv[0][15] == '\0') + return print_help(); + gc_heap_size = atoi(argv[0] + 15); + } +#endif +#if WASM_ENABLE_JIT != 0 + else if (!strncmp(argv[0], "--llvm-jit-size-level=", 22)) { + if (argv[0][22] == '\0') + return print_help(); + llvm_jit_size_level = atoi(argv[0] + 22); + if (llvm_jit_size_level < 1) { + printf("LLVM JIT size level shouldn't be smaller than 1, " + "setting it to 1\n"); + llvm_jit_size_level = 1; + } + else if (llvm_jit_size_level > 3) { + printf("LLVM JIT size level shouldn't be greater than 3, " + "setting it to 3\n"); + llvm_jit_size_level = 3; + } + } + else if (!strncmp(argv[0], "--llvm-jit-opt-level=", 21)) { + if (argv[0][21] == '\0') + return print_help(); + llvm_jit_opt_level = atoi(argv[0] + 21); + if (llvm_jit_opt_level < 1) { + printf("LLVM JIT opt level shouldn't be smaller than 1, " + "setting it to 1\n"); + llvm_jit_opt_level = 1; + } + else if (llvm_jit_opt_level > 3) { + printf("LLVM JIT opt level shouldn't be greater than 3, " + "setting it to 3\n"); + llvm_jit_opt_level = 3; + } + } + else if (!strcmp(argv[0], "--enable-segue")) { + /* all flags are enabled */ + segue_flags = 0x1F1F; + } + else if (!strncmp(argv[0], "--enable-segue=", 15)) { + segue_flags = resolve_segue_flags(argv[0] + 15); + if (segue_flags == (uint32)-1) + return print_help(); + } +#endif /* end of WASM_ENABLE_JIT != 0 */ +#if BH_HAS_DLFCN + else if (!strncmp(argv[0], "--native-lib=", 13)) { + if (argv[0][13] == '\0') + return print_help(); + if (native_lib_count >= sizeof(native_lib_list) / sizeof(char *)) { + printf("Only allow max native lib number %d\n", + (int)(sizeof(native_lib_list) / sizeof(char *))); + return 1; + } + native_lib_list[native_lib_count++] = argv[0] + 13; + } +#endif +#if WASM_ENABLE_LINUX_PERF != 0 + else if (!strncmp(argv[0], "--enable-linux-perf", 19)) { + enable_linux_perf = true; + } +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + else if (!strncmp(argv[0], + "--module-path=", strlen("--module-path="))) { + module_search_path = handle_module_path(argv[0]); + if (!strlen(module_search_path)) { + return print_help(); + } + } +#endif +#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + else if (!strncmp(argv[0], "--max-threads=", 14)) { + if (argv[0][14] == '\0') + return print_help(); + wasm_runtime_set_max_thread_num(atoi(argv[0] + 14)); + } +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + else if (!strncmp(argv[0], "--timeout=", 10)) { + if (argv[0][10] == '\0') + return print_help(); + timeout_ms = atoi(argv[0] + 10); + } +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + else if (!strncmp(argv[0], "-g=", 3)) { + char *port_str = strchr(argv[0] + 3, ':'); + char *port_end; + if (port_str == NULL) + return print_help(); + *port_str = '\0'; + instance_port = strtoul(port_str + 1, &port_end, 10); + if (port_str[1] == '\0' || *port_end != '\0') + return print_help(); + ip_addr = argv[0] + 3; + } +#endif +#if WASM_ENABLE_STATIC_PGO != 0 + else if (!strncmp(argv[0], "--gen-prof-file=", 16)) { + if (argv[0][16] == '\0') + return print_help(); + gen_prof_file = argv[0] + 16; + } +#endif + else if (!strcmp(argv[0], "--version")) { + uint32 major, minor, patch; + wasm_runtime_get_version(&major, &minor, &patch); + printf("iwasm %" PRIu32 ".%" PRIu32 ".%" PRIu32 "\n", major, minor, + patch); + return 0; + } + else { +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_parse_result_t result = + libc_wasi_parse(argv[0], &wasi_parse_ctx); + switch (result) { + case LIBC_WASI_PARSE_RESULT_OK: + continue; + case LIBC_WASI_PARSE_RESULT_NEED_HELP: + return print_help(); + case LIBC_WASI_PARSE_RESULT_BAD_PARAM: + return 1; + } +#else + return print_help(); +#endif + } + } + + if (argc == 0) + return print_help(); + + wasm_file = argv[0]; + app_argc = argc; + app_argv = argv; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.running_mode = running_mode; +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); +#else + init_args.mem_alloc_type = Alloc_With_Allocator; +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + /* Set user data for the allocator is needed */ + /* init_args.mem_alloc_option.allocator.user_data = user_data; */ +#endif + init_args.mem_alloc_option.allocator.malloc_func = malloc_func; + init_args.mem_alloc_option.allocator.realloc_func = realloc_func; + init_args.mem_alloc_option.allocator.free_func = free_func; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + init_args.fast_jit_code_cache_size = jit_code_cache_size; +#endif + +#if WASM_ENABLE_GC != 0 + init_args.gc_heap_size = gc_heap_size; +#endif + +#if WASM_ENABLE_JIT != 0 + init_args.llvm_jit_size_level = llvm_jit_size_level; + init_args.llvm_jit_opt_level = llvm_jit_opt_level; + init_args.segue_flags = segue_flags; +#endif +#if WASM_ENABLE_LINUX_PERF != 0 + init_args.enable_linux_perf = enable_linux_perf; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + init_args.instance_port = instance_port; + if (ip_addr) + /* ensure that init_args.ip_addr is null terminated */ + strncpy(init_args.ip_addr, ip_addr, sizeof(init_args.ip_addr) - 1); +#endif + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + +#if WASM_ENABLE_LOG != 0 + bh_log_set_verbose_level(log_verbose_level); +#endif + +#if BH_HAS_DLFCN + native_lib_loaded_count = load_and_register_native_libs( + native_lib_list, native_lib_count, native_lib_loaded_list); +#endif + + /* load WASM byte buffer from WASM bin file */ + if (!(wasm_file_buf = + (uint8 *)bh_read_file_to_buffer(wasm_file, &wasm_file_size))) + goto fail1; + +#if WASM_ENABLE_AOT != 0 + if (wasm_runtime_is_xip_file(wasm_file_buf, wasm_file_size)) { + uint8 *wasm_file_mapped; + int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC; + int map_flags = MMAP_MAP_32BIT; + + if (!(wasm_file_mapped = os_mmap(NULL, (uint32)wasm_file_size, map_prot, + map_flags, os_get_invalid_handle()))) { + printf("mmap memory failed\n"); + wasm_runtime_free(wasm_file_buf); + goto fail1; + } + + bh_memcpy_s(wasm_file_mapped, wasm_file_size, wasm_file_buf, + wasm_file_size); + wasm_runtime_free(wasm_file_buf); + wasm_file_buf = wasm_file_mapped; + is_xip_file = true; + } +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + wasm_runtime_set_module_reader(module_reader_callback, moudle_destroyer); +#endif + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail2; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_init(wasm_module, argc, argv, &wasi_parse_ctx); +#endif + + /* instantiate the module */ + if (!(wasm_module_inst = + wasm_runtime_instantiate(wasm_module, stack_size, heap_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail3; + } + +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + if (disable_bounds_checks) { + wasm_runtime_set_bounds_checks(wasm_module_inst, false); + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (ip_addr != NULL) { + wasm_exec_env_t exec_env = + wasm_runtime_get_exec_env_singleton(wasm_module_inst); + uint32_t debug_port; + if (exec_env == NULL) { + printf("%s\n", wasm_runtime_get_exception(wasm_module_inst)); + goto fail4; + } + debug_port = wasm_runtime_start_debug_instance(exec_env); + if (debug_port == 0) { + printf("Failed to start debug instance\n"); + goto fail4; + } + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + struct timeout_arg timeout_arg; + korp_tid timeout_tid; + if (timeout_ms >= 0) { + timeout_arg.timeout_ms = timeout_ms; + timeout_arg.inst = wasm_module_inst; + timeout_arg.cancel = false; + ret = os_thread_create(&timeout_tid, timeout_thread, &timeout_arg, + APP_THREAD_STACK_SIZE_DEFAULT); + if (ret != 0) { + printf("Failed to start timeout\n"); + goto fail5; + } + } +#endif + + ret = 0; + const char *exception = NULL; + if (is_repl_mode) { + app_instance_repl(wasm_module_inst); + } + else if (func_name) { + exception = app_instance_func(wasm_module_inst, func_name); + if (exception) { + /* got an exception */ + ret = 1; + } + } + else { + exception = app_instance_main(wasm_module_inst); + if (exception) { + /* got an exception */ + ret = 1; + } + } + +#if WASM_ENABLE_LIBC_WASI != 0 + if (ret == 0) { + /* propagate wasi exit code. */ + ret = wasm_runtime_get_wasi_exit_code(wasm_module_inst); + } +#endif + + if (exception) + printf("%s\n", exception); + +#if WASM_ENABLE_STATIC_PGO != 0 && WASM_ENABLE_AOT != 0 + if (get_package_type(wasm_file_buf, wasm_file_size) == Wasm_Module_AoT + && gen_prof_file) + dump_pgo_prof_data(wasm_module_inst, gen_prof_file); +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + if (timeout_ms >= 0) { + timeout_arg.cancel = true; + os_thread_join(timeout_tid, NULL); + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +fail5: +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 +fail4: +#endif + /* destroy the module instance */ + wasm_runtime_deinstantiate(wasm_module_inst); + +fail3: + /* unload the module */ + wasm_runtime_unload(wasm_module); + +fail2: + /* free the file buffer */ + if (!is_xip_file) + wasm_runtime_free(wasm_file_buf); + else + os_munmap(wasm_file_buf, wasm_file_size); + +fail1: +#if BH_HAS_DLFCN + /* unload the native libraries */ + unregister_and_unload_native_libs(native_lib_loaded_count, + native_lib_loaded_list); +#endif + + /* destroy runtime environment */ + wasm_runtime_destroy(); + + return ret; +} diff --git a/wasm-micro-runtime/product-mini/platforms/riot/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/riot/CMakeLists.txt new file mode 100644 index 0000000..0032832 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/riot/CMakeLists.txt @@ -0,0 +1,64 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# Copyright (C) 2020 TU Bergakademie Freiberg Karl Fessel +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.8.2) + +set(CMAKE_TRY_COMPILE_TARGET_TYPE "STATIC_LIBRARY") + +project(NONE) + +enable_language (ASM) + +set (WAMR_BUILD_PLATFORM "riot") + +# Build as X86_32 by default, change to "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", "MIPS" or "XTENSA" +# if we want to support arm, thumb, mips or xtensa + + +if (NOT DEFINED WAMR_BUILD_TARGET) + set (WAMR_BUILD_TARGET "X86_32") +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Disable AOT by default. + set (WAMR_BUILD_AOT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Disable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 0) +endif () + +if (NOT DEFINED WAMR_ROOT_DIR) + # this assumption is true if this file is copied to WAMR_ROOT + set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +endif () + +# Override the global heap size for small devices +if (NOT DEFINED WAMR_BUILD_GLOBAL_HEAP_SIZE) + add_definitions (-DWASM_GLOBAL_HEAP_SIZE=262144) # 256 kB +endif () + + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +# need includes from RIOT prepare them as a cmake list +string(REGEX MATCHALL "([^\ ]+\ |[^\ ]+$)" RIOT_INCLUDES_LIST "${RIOT_INCLUDES}") + +include_directories(SYSTEM ${RIOT_INCLUDES_LIST}) + +# target_sources( ${WAMR_RUNTIME_LIB_SOURCE} ) +# executable linking is done by RIOT build system + +add_library( wamr ${WAMR_RUNTIME_LIB_SOURCE}) diff --git a/wasm-micro-runtime/product-mini/platforms/riot/Makefile b/wasm-micro-runtime/product-mini/platforms/riot/Makefile new file mode 100644 index 0000000..8c7aef3 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/riot/Makefile @@ -0,0 +1,97 @@ +APPLICATION = wamr-mini +# If no BOARD is defined in the environment, use this default: +BOARD ?= native + +# This has to be the absolute path to the RIOT base directory: +RIOTBASE ?= $(CURDIR)/../../../../RIOT + +USEMODULE += ztimer64_msec +USEMODULE += ztimer_usec +USEMODULE += sema + +WPEDANTIC := 0 +WERROR := 0 + +# Comment this out to disable code in RIOT that does safety checking +# which is not needed in a production environment but helps in the +# development process: +DEVELHELP ?= 1 + +# Change this to 0 show compiler invocation lines by default: +QUIET ?= 1 + +ARCHIVES += $(BINDIR)/libwamr.a + +#Load the usual RIOT make infastructure + +include $(RIOTBASE)/Makefile.include + + +WAMR_SOURCE = $(CURDIR)/../../.. +WAMR_BUILD_DIR = $(BINDIR)/wamr + +#less Wall TODO: get things fixed +CFLAGS := $(filter-out -pedantic, $(CFLAGS)) +CFLAGS += -Wno-format +CFLAGS += -Wno-strict-prototypes +CFLAGS += -Wno-old-style-definition +CFLAGS += -Wno-cast-function-type + +WAMR_CORE = $(WAMR_SOURCE)/core +IWASM_ROOT = $(WAMR_CORE)/iwasm +SHARED_LIB_ROOT = $(WAMR_CORE)/shared + +IWASM_INCLUDES += ${IWASM_ROOT}/include \ + ${SHARED_LIB_ROOT}/platform/include \ + ${SHARED_LIB_ROOT}/platform/riot \ + + +INCLUDES += $(addprefix -I,${IWASM_INCLUDES}) + + + +RIOT_INCLUDES = $(filter-out -%,$(subst -I,,$(INCLUDES))) + +#WAMR_BUILD_TARGET is "X86_32" "AARCH64[sub]", "ARM[sub]", +# "THUMB[sub]", "MIPS" or "XTENSA" +#no msp430, no AVR support for now + +#translate (CPU_ARCH) to Build Target +ifeq ($(CPU),native) +#$(CPU) is defined for every CPU +#Riot native is x86_32 + WAMR_BUILD_TARGET = X86_32 +else ifeq ($(findstring arm,$(CPU_ARCH)),arm) + WAMR_BUILD_TARGET = THUMB +else ifeq ($(CPU_ARCH),mips32r2) + WAMR_BUILD_TARGET = MIPS +else ifeq ($(CPU_ARCH),xtensa) + WAMR_BUILD_TARGET = XTENSA +endif + +ifeq ($(QUIET), 0) + CMAKEMAKEFLAGS += VERBOSE=1 +endif + + +$(BINDIR)/libwamr.a: $(WAMR_BUILD_DIR)/libwamr.a + cp $< $@ + +$(WAMR_BUILD_DIR)/libwamr.a: $(WAMR_BUILD_DIR)/Makefile + $(MAKE) -C $(WAMR_BUILD_DIR) $(CMAKEMAKEFLAGS) + +$(WAMR_BUILD_DIR)/Makefile: CMakeLists.txt + cmake -B$(WAMR_BUILD_DIR) \ + "-DRIOT_INCLUDES=$(RIOT_INCLUDES)"\ + -DWAMR_ROOT_DIR=$(WAMR_SOURCE)\ + -DWAMR_BUILD_TARGET=$(WAMR_BUILD_TARGET)\ + -DCMAKE_SYSTEM_NAME=Generic \ + -DCMAKE_SYSTEM_PROCESSOR="$(MCPU)" \ + -DCMAKE_C_COMPILER=$(CC) \ + -DCMAKE_C_COMPILER_WORKS=TRUE \ + +print_build_target: + @echo CPU_ARCH: $(CPU_ARCH) + @echo CPU: $(CPU) + @echo CFLAGS: $(CFLAGS) + @echo WAMR_BUILD_TARGET: $(WAMR_BUILD_TARGET) diff --git a/wasm-micro-runtime/product-mini/platforms/riot/iwasmt.c b/wasm-micro-runtime/product-mini/platforms/riot/iwasmt.c new file mode 100644 index 0000000..633b2b4 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/riot/iwasmt.c @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * Copyright (C) 2020 TU Bergakademie Freiberg Karl Fessel + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include + +#include + +#include "wasm_export.h" + +#include + +/* provide some test program */ +#include "test_wasm.h" + +#define DEFAULT_THREAD_STACKSIZE (6 * 1024) +#define DEFAULT_THREAD_PRIORITY 50 + +static int app_argc; +static char **app_argv; + +static void * +app_instance_main(wasm_module_inst_t module_inst) +{ + const char *exception; + + wasm_application_execute_main(module_inst, app_argc, app_argv); + if ((exception = wasm_runtime_get_exception(module_inst))) { + puts(exception); + } + return NULL; +} + +void * +iwasm_t(void *arg1) +{ + wasm_module_t wasm_module = (wasm_module_t)arg1; + wasm_module_inst_t wasm_module_inst = NULL; + char error_buf[128]; + + /* instantiate the module */ + if (!(wasm_module_inst = wasm_runtime_instantiate( + wasm_module, 8 * 1024, 8 * 1024, error_buf, sizeof(error_buf)))) { + puts(error_buf); + } + else { + app_instance_main(wasm_module_inst); + /* destroy the module instance */ + wasm_runtime_deinstantiate(wasm_module_inst); + } + return NULL; +} + +/* enable FUNC_ALLOC to use custom memory allocation functions */ +#define FUNC_ALLOC + +void * +iwasm_main(void *arg1) +{ + (void)arg1; /* unused */ + uint8_t *wasm_file_buf = NULL; + unsigned wasm_file_buf_size = 0; + wasm_module_t wasm_module = NULL; + char error_buf[128]; + + RuntimeInitArgs init_args; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); +#if defined(FUNC_ALLOC) && WASM_ENABLE_GLOBAL_HEAP_POOL == 0 + init_args.mem_alloc_type = Alloc_With_Allocator; + init_args.mem_alloc_option.allocator.malloc_func = malloc; + init_args.mem_alloc_option.allocator.realloc_func = realloc; + init_args.mem_alloc_option.allocator.free_func = free; +#elif WASM_ENABLE_GLOBAL_HEAP_POOL != 0 + static char global_heap_buf[WASM_GLOBAL_HEAP_SIZE] = { 0 }; + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); +#else + init_args.mem_alloc_type = Alloc_With_System_Allocator; +#endif + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + puts("Init runtime environment failed."); + return NULL; + } + + /* load WASM byte buffer from byte buffer of include file */ + wasm_file_buf = (uint8_t *)wasm_test_file; + wasm_file_buf_size = sizeof(wasm_test_file); + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_buf_size, + error_buf, sizeof(error_buf)))) { + puts(error_buf); + } + else { + iwasm_t(wasm_module); + wasm_runtime_unload(wasm_module); + } + + wasm_runtime_destroy(); + return NULL; +} + +bool +iwasm_init(void) +{ + /* clang-format off */ + struct { + char *stack; + int stacksize; + uint8_t priority; + int flags; + thread_task_func_t task_func; + void *arg; + const char *name; + } b = { + .stacksize = DEFAULT_THREAD_STACKSIZE, + .priority = 8, + .flags = 0, + .task_func = iwasm_main, + .arg = NULL, + .name = "simple_wamr" + }; + /* clang-format on */ + + b.stack = malloc(b.stacksize); + kernel_pid_t tpid = thread_create(b.stack, b.stacksize, b.priority, b.flags, + b.task_func, b.arg, b.name); + + return tpid != 0 ? true : false; + ; +} + +#define telltruth(X) ((X) ? "true" : "false") + +int +main(void) +{ + printf("iwasm_initilised: %s\n", telltruth(iwasm_init())); +} diff --git a/wasm-micro-runtime/product-mini/platforms/riot/test_wasm.h b/wasm-micro-runtime/product-mini/platforms/riot/test_wasm.h new file mode 100644 index 0000000..0c34302 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/riot/test_wasm.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * The byte array buffer is the file content of a test wasm binary file, + * which is compiled by wasi-sdk toolchain from C source file of: + * product-mini/app-samples/hello-world/main.c. + */ +unsigned char wasm_test_file[] = { + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x10, 0x03, 0x60, + 0x01, 0x7F, 0x01, 0x7F, 0x60, 0x02, 0x7F, 0x7F, 0x01, 0x7F, 0x60, 0x01, + 0x7F, 0x00, 0x02, 0x31, 0x04, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x70, 0x75, + 0x74, 0x73, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x6D, 0x61, 0x6C, + 0x6C, 0x6F, 0x63, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x70, 0x72, + 0x69, 0x6E, 0x74, 0x66, 0x00, 0x01, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x66, + 0x72, 0x65, 0x65, 0x00, 0x02, 0x03, 0x02, 0x01, 0x01, 0x04, 0x05, 0x01, + 0x70, 0x01, 0x01, 0x01, 0x05, 0x03, 0x01, 0x00, 0x01, 0x06, 0x13, 0x03, + 0x7F, 0x01, 0x41, 0xC0, 0x28, 0x0B, 0x7F, 0x00, 0x41, 0xBA, 0x08, 0x0B, + 0x7F, 0x00, 0x41, 0xC0, 0x28, 0x0B, 0x07, 0x2C, 0x04, 0x06, 0x6D, 0x65, + 0x6D, 0x6F, 0x72, 0x79, 0x02, 0x00, 0x0A, 0x5F, 0x5F, 0x64, 0x61, 0x74, + 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x03, 0x01, 0x0B, 0x5F, 0x5F, 0x68, 0x65, + 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, 0x65, 0x03, 0x02, 0x04, 0x6D, 0x61, + 0x69, 0x6E, 0x00, 0x04, 0x0A, 0xB2, 0x01, 0x01, 0xAF, 0x01, 0x01, 0x03, + 0x7F, 0x23, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, + 0x24, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x9B, 0x88, 0x80, 0x80, 0x00, + 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x02, 0x40, 0x02, 0x40, 0x41, + 0x80, 0x08, 0x10, 0x81, 0x80, 0x80, 0x80, 0x00, 0x22, 0x03, 0x0D, 0x00, + 0x41, 0xA8, 0x88, 0x80, 0x80, 0x00, 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, + 0x1A, 0x41, 0x7F, 0x21, 0x04, 0x0C, 0x01, 0x0B, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x10, 0x41, 0x80, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x41, + 0x10, 0x6A, 0x10, 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x41, 0x00, 0x21, + 0x04, 0x20, 0x03, 0x41, 0x04, 0x6A, 0x41, 0x00, 0x2F, 0x00, 0x91, 0x88, + 0x80, 0x80, 0x00, 0x3B, 0x00, 0x00, 0x20, 0x03, 0x41, 0x00, 0x28, 0x00, + 0x8D, 0x88, 0x80, 0x80, 0x00, 0x36, 0x00, 0x00, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x00, 0x41, 0x93, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x10, + 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x20, 0x03, 0x10, 0x83, 0x80, 0x80, + 0x80, 0x00, 0x0B, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x80, 0x80, 0x80, + 0x80, 0x00, 0x20, 0x04, 0x0B, 0x0B, 0x41, 0x01, 0x00, 0x41, 0x80, 0x08, + 0x0B, 0x3A, 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, 0x20, 0x25, + 0x70, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, 0x75, 0x66, + 0x3A, 0x20, 0x25, 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, + 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, 0x6F, 0x63, + 0x20, 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, 0x64, 0x00 +}; diff --git a/wasm-micro-runtime/product-mini/platforms/rt-thread/SConscript b/wasm-micro-runtime/product-mini/platforms/rt-thread/SConscript new file mode 100644 index 0000000..4f7ffe6 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/rt-thread/SConscript @@ -0,0 +1,14 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +from building import * + +cwd = GetCurrentDir() +src = Glob('*.c') + +group = DefineGroup('iwasm', src, depend = ['PKG_USING_WAMR']) + +Return('group') diff --git a/wasm-micro-runtime/product-mini/platforms/rt-thread/iwasm.c b/wasm-micro-runtime/product-mini/platforms/rt-thread/iwasm.c new file mode 100644 index 0000000..9a21301 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/rt-thread/iwasm.c @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include "wasm_export.h" +#include "platform_api_vmcore.h" +#include +#include +#include + +#ifdef WAMR_ENABLE_RTT_EXPORT + +#ifdef WAMR_RTT_EXPORT_VPRINTF +static int +wasm_vprintf(wasm_exec_env_t env, const char *fmt, va_list va) +{ + return vprintf(fmt, va); +} + +static int +wasm_vsprintf(wasm_exec_env_t env, char *buf, const char *fmt, va_list va) +{ + return vsprintf(buf, fmt, va); +} + +static int +wasm_vsnprintf(wasm_exec_env_t env, char *buf, int n, const char *fmt, + va_list va) +{ + return vsnprintf(buf, n, fmt, va); +} + +#endif /* WAMR_RTT_EXPORT_VPRINTF */ + +#ifdef WAMR_RTT_EXPORT_DEVICE_OPS +static rt_device_t +wasm_rt_device_find(wasm_exec_env_t env, const char *name) +{ + return rt_device_find(name); +} + +static rt_err_t +wasm_rt_device_open(wasm_exec_env_t env, rt_device_t dev, rt_uint16_t o_flag) +{ + return rt_device_open(dev, o_flag); +} + +static rt_size_t +wasm_rt_device_write(wasm_exec_env_t env, rt_device_t dev, rt_off_t offset, + const void *buf, rt_size_t size) +{ + return rt_device_write(dev, offset, buf, size); +} + +static rt_size_t +wasm_rt_device_read(wasm_exec_env_t env, rt_device_t dev, rt_off_t offset, + void *buf, rt_size_t size) +{ + return rt_device_read(dev, offset, buf, size); +} + +static rt_err_t +wasm_rt_device_close(wasm_exec_env_t env, rt_device_t dev) +{ + return rt_device_close(dev); +} + +static rt_err_t +wasm_rt_device_control(wasm_exec_env_t env, rt_device_t dev, int cmd, void *arg) +{ + return rt_device_control(dev, cmd, arg); +} + +#endif /* WAMR_RTT_EXPORT_DEVICE_OPS */ + +/* clang-format off */ +static NativeSymbol native_export_symbols[] = { +#ifdef WAMR_RTT_EXPORT_VPRINTF + { + "vprintf", + wasm_vprintf, + "($*)i" + }, + { + "vsprintf", + wasm_vsprintf, + "($$*)i" + }, + { + "vsnprintf", + wasm_vsnprintf, + "($i$*)i" + }, +#endif /* WAMR_RTT_EXPORT_VPRINTF */ + +#ifdef WAMR_RTT_EXPORT_DEVICE_OPS + { + "rt_device_find", + wasm_rt_device_find, + "($)i" + }, + { + "rt_device_open", + wasm_rt_device_open, + "(ii)i" + }, + { + "rt_device_write", + wasm_rt_device_write, + "(ii*~)i" + }, + { + "rt_device_read", + wasm_rt_device_read, + "(ii*~)i" + }, + { + "rt_device_close", + wasm_rt_device_close, + "(i)i" + }, + { + "rt_device_control", + wasm_rt_device_control, + "(ii*)i" + }, +#ifdef WAMR_RTT_EXPORT_DEVICE_OPS_CPP + { + "_Z15rt_device_closeP9rt_device", + wasm_rt_device_close, + "(i)i" + }, + { + "_Z14rt_device_readP9rt_devicejPvj", + wasm_rt_device_read, + "(ii*~)i" + }, + { + "_Z15rt_device_writeP9rt_devicejPKvj", + wasm_rt_device_write, + "(ii*~)i" + }, + { + "_Z14rt_device_openP9rt_devicet", + wasm_rt_device_open, + "(ii)i" + }, + { + "_Z14rt_device_findPKc", + wasm_rt_device_find, + "($)i" + }, +#endif /* WAMR_RTT_EXPORT_DEVICE_OPS_CPP */ +#endif /* WAMR_RTT_EXPORT_DEVICE_OPS */ +}; +/* clang-format on */ + +#endif /* WAMR_ENABLE_RTT_EXPORT */ + +/** + * run WASM module instance. + * @param module_inst instance of wasm module + * @param app_argc wasm argument count + * @param app_argv wasm arguments + * @return NULL + */ +static void * +app_instance_main(wasm_module_inst_t module_inst, int app_argc, char **app_argv) +{ + const char *exception; + + wasm_application_execute_main(module_inst, app_argc, app_argv); + if ((exception = wasm_runtime_get_exception(module_inst))) + rt_kprintf("%s\n", exception); + return NULL; +} + +rt_uint8_t * +my_read_file_to_buffer(char *filename, rt_uint32_t *size) +{ + struct stat f_stat; + + rt_uint8_t *buff = rt_malloc(f_stat.st_size); + *size = 0; + if (!buff) { + rt_set_errno(-ENOMEM); + return RT_NULL; + } + + int fd = open(filename, O_RDONLY); + if (fd < 0) { + rt_free(buff); + rt_set_errno(fd); + return RT_NULL; + } + + *size = read(fd, buff, f_stat.st_size); + + close(fd); + + if (*size != f_stat.st_size) { + rt_free(buff); + rt_set_errno(-EBADF); + return RT_NULL; + } + + return buff; +} + +void +iwasm_help(void) +{ +#ifdef WAMR_ENABLE_IWASM_PARAMS + rt_kputs("wrong input: iwasm [-t] [-m] [-s] <*.wasm> \n" + " iwasm [-h]\n"); + rt_kputs("\t -h: show this tips.\n"); + rt_kputs("\t -t: show time taking to run this app.\n"); + rt_kputs("\t -m: show memory taking to run this app\n"); + rt_kputs("\t wasm file name and exec params must behind of all vm-param\n"); +#else + rt_kputs("wrong input: iwasm <*.wasm> \n"); +#endif /* WAMR_ENABLE_PARAMS */ +} + +int +iwasm(int argc, char **argv) +{ + rt_uint8_t *wasm_file_buf = NULL; + rt_uint32_t wasm_file_size; + rt_uint32_t stack_size = 4 * 1024, heap_size = 4 * 1024; + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + RuntimeInitArgs init_args; + static char error_buf[128] = { 0 }; + /* avoid stack overflow */ + +#ifdef WAMR_ENABLE_IWASM_PARAMS + int i_arg_begin; + bool show_mem = false; + bool show_stack = false; + bool show_time_exec = false; + for (i_arg_begin = 1; i_arg_begin < argc; i_arg_begin++) { + if (argv[i_arg_begin][0] != '-') { + break; + } + + if (argv[i_arg_begin][1] == 'm') { + show_mem = true; + } + else if (argv[i_arg_begin][1] == 's') { + show_stack = true; + } + else if (argv[i_arg_begin][1] == 't') { + show_time_exec = true; + } + else if (argv[i_arg_begin][1] == 'h') { + iwasm_help(); + return 0; + } + else if (argv[i_arg_begin][1] == 0x00) { + continue; + } + else { + rt_kprintf("[iwasm] unknown param: %s\n", argv[i_arg_begin]); + } + } +#else /* WAMR_ENABLE_PARAMS */ +#define i_arg_begin 1 +#endif /* WAMR_ENABLE_PARAMS */ + + if (argc - i_arg_begin < 1) { + iwasm_help(); + return -1; + } + + rt_memset(&init_args, 0, sizeof(RuntimeInitArgs)); + init_args.mem_alloc_type = Alloc_With_Allocator; + init_args.mem_alloc_option.allocator.malloc_func = os_malloc; + init_args.mem_alloc_option.allocator.realloc_func = os_realloc; + init_args.mem_alloc_option.allocator.free_func = os_free; +#ifdef WAMR_ENABLE_RTT_EXPORT + init_args.native_symbols = native_export_symbols; + init_args.n_native_symbols = + sizeof(native_export_symbols) / sizeof(NativeSymbol); + init_args.native_module_name = "env"; +#endif /* WAMR_ENABLE_RTT_EXPORT */ + +#ifdef WAMR_ENABLE_IWASM_PARAMS +#if defined(RT_USING_HEAP) && defined(RT_USING_MEMHEAP_AS_HEAP) + extern long list_memheap(void); + if (show_mem) { + list_memheap(); + } +#else + rt_uint32_t total, max, used; + if (show_mem) { + rt_memory_info(&total, &used, &max); + } +#endif + rt_thread_t tid; + if (show_stack) { + tid = rt_thread_self(); + printf("thread stack addr: %p, size: %u, sp: %p\n", tid->stack_addr, + tid->stack_size, tid->sp); + } +#endif /* WAMR_ENABLE_PARAMS */ + + if (wasm_runtime_full_init(&init_args) == false) { + rt_kprintf("Init WASM runtime environment failed.\n"); + return -1; + } + + wasm_file_buf = my_read_file_to_buffer(argv[i_arg_begin], &wasm_file_size); + if (!wasm_file_buf) { + rt_err_t err = rt_get_errno(); + rt_kprintf("WASM load file to RAM failed: %d\n", err); + goto fail1; + } + rt_memset(error_buf, 0x00, sizeof(error_buf)); + wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, error_buf, + sizeof(error_buf)); + if (!wasm_module) { + rt_kprintf("%s\n", error_buf); + goto fail2; + } + rt_memset(error_buf, 0x00, sizeof(error_buf)); + wasm_module_inst = wasm_runtime_instantiate( + wasm_module, stack_size, heap_size, error_buf, sizeof(error_buf)); + if (!wasm_module_inst) { + rt_kprintf("%s\n", error_buf); + goto fail3; + } + +#ifdef WAMR_ENABLE_IWASM_PARAMS + rt_tick_t ticks_exec; + if (show_time_exec) { + ticks_exec = rt_tick_get(); + } +#endif /* WAMR_ENABLE_PARAMS */ + + app_instance_main(wasm_module_inst, argc - i_arg_begin, &argv[i_arg_begin]); + +#ifdef WAMR_ENABLE_IWASM_PARAMS + if (show_time_exec) { + ticks_exec = rt_tick_get() - ticks_exec; + printf("[iwasm] execute ticks took: %u [ticks/s = %u]\n", ticks_exec, + RT_TICK_PER_SECOND); + } +#if defined(RT_USING_HEAP) && defined(RT_USING_MEMHEAP_AS_HEAP) + if (show_mem) { + list_memheap(); + } +#else + rt_uint32_t total_after, max_after, used_after; + if (show_mem) { + rt_memory_info(&total_after, &used_after, &max_after); + rt_kprintf("[iwasm] memory took: %u\n", used_after - used); + } +#endif + if (show_stack) { + printf("[iwasm] thread stack addr: %p, size: %u, sp: %p\n", + tid->stack_addr, tid->stack_size, tid->sp); + } + +#endif /* WAMR_ENABLE_PARAMS */ + + /* destroy the module instance */ + wasm_runtime_deinstantiate(wasm_module_inst); + +fail3: + /* unload the module */ + wasm_runtime_unload(wasm_module); + +fail2: + /* free the file buffer */ + rt_free(wasm_file_buf); + +fail1: + /* destroy runtime environment */ + wasm_runtime_destroy(); + return 0; +} +MSH_CMD_EXPORT(iwasm, Embeded VM of WebAssembly); diff --git a/wasm-micro-runtime/product-mini/platforms/rt-thread/iwasm.cmake b/wasm-micro-runtime/product-mini/platforms/rt-thread/iwasm.cmake new file mode 100644 index 0000000..0ec54ae --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/rt-thread/iwasm.cmake @@ -0,0 +1,66 @@ +# +# Copyright (c) 2021, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +set(CMAKE_ASM_COMPILER_WORKS 1) + +set(WAMR_BUILD_PLATFORM "rt-thread") +set(WAMR_BUILD_TARGET "ARM") + +#set(WAMR_BUILD_INTERP 1) +#set(WAMR_BUILD_FAST_INTERP 1) +#set(WAMR_BUILD_AOT 0) +#set(WAMR_BUILD_JIT 0) +#set(WAMR_BUILD_LIBC_BUILTIN 1) +#set(WAMR_BUILD_LIBC_WASI 0) + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 0) +endif () + +# Disable JIT by default. +set (WAMR_BUILD_JIT 0) + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +set (WAMR_BUILD_LIBC_WASI 0) + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +set (WAMR_BUILD_MULTI_MODULE 0) +set (WAMR_BUILD_LIB_PTHREAD 0) +set (WAMR_BUILD_MINI_LOADER 0) +set (WAMR_BUILD_SIMD 0) + + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../..) + +set(CMAKE_ASM_COMPILER_WORKS 1) + +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +file (GLOB wamr_entry_src + ${WAMR_ROOT_DIR}/product-mini/platforms/rt-thread/rtt_wamr_entry.c + ) + +set(WAMR_SOURCE ${wamr_entry_src} ${WAMR_RUNTIME_LIB_SOURCE}) + + diff --git a/wasm-micro-runtime/product-mini/platforms/vxworks/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/vxworks/CMakeLists.txt new file mode 100644 index 0000000..0dc5d96 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/vxworks/CMakeLists.txt @@ -0,0 +1,97 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "vxworks") + +# Specify the compiler driver provided in the VSB +SET(CMAKE_C_COMPILER vx-cc) +SET(CMAKE_AR vx-ar) +SET(CMAKE_RANLIB vx-ranlib) + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "ARM[sub]", "THUMB[sub]", "MIPS", "XTENSA" +#set (WAMR_BUILD_TARGET "X86_64") + +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Disable AOT by default. + set (WAMR_BUILD_AOT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Disable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 0) +endif () + +if (NOT DEFINED WAMR_BUILD_DEBUG_INTERP) + # Disable Debug feature by default + set (WAMR_BUILD_DEBUG_INTERP 0) +endif () + +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + set (WAMR_BUILD_FAST_INTERP 0) + set (WAMR_BUILD_MINI_LOADER 0) + set (WAMR_BUILD_SIMD 0) +endif () + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (iwasm main.c ${UNCOMMON_SHARED_SOURCE}) + +install (TARGETS iwasm DESTINATION bin) + +target_link_libraries (iwasm vmlib ${LLVM_AVAILABLE_LIBS} -lm -ldl -lunix) + +add_library (libiwasm SHARED ${WAMR_RUNTIME_LIB_SOURCE}) + +install (TARGETS libiwasm DESTINATION lib) + +set_target_properties (libiwasm PROPERTIES OUTPUT_NAME iwasm) + +target_link_libraries (libiwasm ${LLVM_AVAILABLE_LIBS} -lm -ldl -lunix) + diff --git a/wasm-micro-runtime/product-mini/platforms/vxworks/main.c b/wasm-micro-runtime/product-mini/platforms/vxworks/main.c new file mode 100644 index 0000000..8f0e84a --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/vxworks/main.c @@ -0,0 +1,6 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "../posix/main.c" diff --git a/wasm-micro-runtime/product-mini/platforms/windows/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/windows/CMakeLists.txt new file mode 100644 index 0000000..02aa3f3 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/windows/CMakeLists.txt @@ -0,0 +1,161 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project (iwasm C ASM CXX) +# set (CMAKE_VERBOSE_MAKEFILE 1) + +set (WAMR_BUILD_PLATFORM "windows") + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +add_definitions(-DCOMPILING_WASM_RUNTIME_API=1) + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", "MIPS", "XTENSA" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(FATAL_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_JIT) + # Disable JIT by default. + set (WAMR_BUILD_JIT 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_UVWASI) + # Enable libc uvwasi support by default + set (WAMR_BUILD_LIBC_UVWASI 1) +endif () + +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Enable multiple modules + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_MINI_LOADER) + # Disable wasm mini loader by default + set (WAMR_BUILD_MINI_LOADER 0) +endif () + +if (NOT DEFINED WAMR_BUILD_SIMD) + # Enable SIMD by default + set (WAMR_BUILD_SIMD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_DEBUG_INTERP) + # Disable Debug feature by default + set (WAMR_BUILD_DEBUG_INTERP 0) +endif () + +if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) + set (WAMR_BUILD_FAST_INTERP 0) + set (WAMR_BUILD_MINI_LOADER 0) + set (WAMR_BUILD_SIMD 0) +endif () + +if (WAMR_BUILD_LIBC_WASI EQUAL 1) + set (CMAKE_C_STANDARD 11) + if (MSVC) + add_compile_options(/experimental:c11atomics) + endif() +else() + set (CMAKE_C_STANDARD 99) +endif() + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN") +if (NOT MINGW) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SAFESEH:NO") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SAFESEH:NO") +endif () + +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") + +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang" OR MSVC)) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () +endif () + +# The following flags are to enhance security, but it may impact performance, +# we disable them by default. +#if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftrapv -D_FORTIFY_SOURCE=2") +#endif () +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-strong --param ssp-buffer-size=4") +#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack,-z,relro,-z,now") + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (iwasm main.c ${UNCOMMON_SHARED_SOURCE}) + +install (TARGETS iwasm DESTINATION bin) + +target_link_libraries (iwasm vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS}) + +if (MINGW) + target_link_libraries (iwasm ws2_32) +endif () + +add_library (libiwasm SHARED ${WAMR_RUNTIME_LIB_SOURCE}) + +install (TARGETS libiwasm DESTINATION lib) + +set_target_properties (libiwasm PROPERTIES OUTPUT_NAME libiwasm) + +target_link_libraries (libiwasm ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS}) + +if (MINGW) + target_link_libraries (libiwasm ws2_32) +endif () + +if (WIN32) + target_link_libraries(libiwasm ntdll) + + target_link_libraries(iwasm ntdll) +endif() diff --git a/wasm-micro-runtime/product-mini/platforms/windows/build_llvm.py b/wasm-micro-runtime/product-mini/platforms/windows/build_llvm.py new file mode 100644 index 0000000..9325a02 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/windows/build_llvm.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import pathlib +import subprocess +import sys + +script = ( + pathlib.Path(__file__) + .parent.joinpath("../../../build-scripts/build_llvm.py") + .resolve() +) +subprocess.check_call([sys.executable, script]) diff --git a/wasm-micro-runtime/product-mini/platforms/windows/main.c b/wasm-micro-runtime/product-mini/platforms/windows/main.c new file mode 100644 index 0000000..35a4897 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/windows/main.c @@ -0,0 +1,600 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include "bh_platform.h" +#include "bh_read_file.h" +#include "wasm_export.h" + +#if WASM_ENABLE_LIBC_WASI != 0 +#include "../common/libc_wasi.c" +#endif + +static int app_argc; +static char **app_argv; + +#define MODULE_PATH ("--module-path=") + +/* clang-format off */ +static int +print_help() +{ + printf("Usage: iwasm [-options] wasm_file [args...]\n"); + printf("options:\n"); + printf(" -f|--function name Specify a function name of the module to run rather\n" + " than main\n"); +#if WASM_ENABLE_LOG != 0 + printf(" -v=n Set log verbose level (0 to 5, default is 2) larger\n" + " level with more log\n"); +#endif +#if WASM_ENABLE_INTERP != 0 + printf(" --interp Run the wasm app with interpreter mode\n"); +#endif +#if WASM_ENABLE_FAST_JIT != 0 + printf(" --fast-jit Run the wasm app with fast jit mode\n"); +#endif +#if WASM_ENABLE_JIT != 0 + printf(" --llvm-jit Run the wasm app with llvm jit mode\n"); +#endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + printf(" --multi-tier-jit Run the wasm app with multi-tier jit mode\n"); +#endif + printf(" --stack-size=n Set maximum stack size in bytes, default is 64 KB\n"); + printf(" --heap-size=n Set maximum heap size in bytes, default is 16 KB\n"); +#if WASM_ENABLE_JIT != 0 + printf(" --llvm-jit-size-level=n Set LLVM JIT size level, default is 3\n"); + printf(" --llvm-jit-opt-level=n Set LLVM JIT optimization level, default is 3\n"); +#endif + printf(" --repl Start a very simple REPL (read-eval-print-loop) mode\n" + " that runs commands in the form of `FUNC ARG...`\n"); +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_print_help(); +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + printf(" --module-path= Indicate a module search path. default is current\n" + " directory('./')\n"); +#endif +#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + printf(" --max-threads=n Set maximum thread number per cluster, default is 4\n"); +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + printf(" -g=ip:port Set the debug sever address, default is debug disabled\n"); + printf(" if port is 0, then a random port will be used\n"); +#endif + printf(" --version Show version information\n"); + return 1; +} +/* clang-format on */ + +static const void * +app_instance_main(wasm_module_inst_t module_inst) +{ + const char *exception; + + wasm_application_execute_main(module_inst, app_argc, app_argv); + exception = wasm_runtime_get_exception(module_inst); + return exception; +} + +static const void * +app_instance_func(wasm_module_inst_t module_inst, const char *func_name) +{ + wasm_application_execute_func(module_inst, func_name, app_argc - 1, + app_argv + 1); + /* The result of wasm function or exception info was output inside + wasm_application_execute_func(), here we don't output them again. */ + return wasm_runtime_get_exception(module_inst); +} + +/** + * Split a space separated strings into an array of strings + * Returns NULL on failure + * Memory must be freed by caller + * Based on: http://stackoverflow.com/a/11198630/471795 + */ +static char ** +split_string(char *str, int *count) +{ + char **res = NULL, **res1; + char *p, *next_token; + int idx = 0; + + /* split string and append tokens to 'res' */ + do { + p = strtok_s(str, " ", &next_token); + str = NULL; + res1 = res; + res = (char **)realloc(res1, sizeof(char *) * (uint32)(idx + 1)); + if (res == NULL) { + free(res1); + return NULL; + } + res[idx++] = p; + } while (p); + + /** + * Due to the function name, + * res[0] might contain a '\' to indicate a space + * func\name -> func name + */ + p = strchr(res[0], '\\'); + while (p) { + *p = ' '; + p = strchr(p, '\\'); + } + + if (count) { + *count = idx - 1; + } + return res; +} + +static void * +app_instance_repl(wasm_module_inst_t module_inst) +{ + char buffer[4096]; + char *cmd; + size_t n; + + while ((printf("webassembly> "), fflush(stdout), + cmd = fgets(buffer, sizeof(buffer), stdin)) + != NULL) { + bh_assert(cmd); + n = strlen(cmd); + if (cmd[n - 1] == '\n') { + if (n == 1) + continue; + else + cmd[n - 1] = '\0'; + } + if (!strcmp(cmd, "__exit__")) { + printf("exit repl mode\n"); + break; + } + app_argv = split_string(cmd, &app_argc); + if (app_argv == NULL) { + LOG_ERROR("Wasm prepare param failed: split string failed.\n"); + break; + } + if (app_argc != 0) { + const char *exception; + wasm_application_execute_func(module_inst, app_argv[0], + app_argc - 1, app_argv + 1); + if ((exception = wasm_runtime_get_exception(module_inst))) + printf("%s\n", exception); + } + free(app_argv); + } + + return NULL; +} + +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 +static char global_heap_buf[WASM_GLOBAL_HEAP_SIZE] = { 0 }; +#else +static void * +malloc_func( +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + unsigned int size) +{ + return malloc(size); +} + +static void * +realloc_func( +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr, unsigned int size) +{ + return realloc(ptr, size); +} + +static void +free_func( +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr) +{ + free(ptr); +} +#endif /* end of WASM_ENABLE_GLOBAL_HEAP_POOL */ + +#if WASM_ENABLE_MULTI_MODULE != 0 +static char * +handle_module_path(const char *module_path) +{ + /* next character after '=' */ + return (strchr(module_path, '=')) + 1; +} + +static char *module_search_path = "."; +static bool +module_reader_callback(package_type_t module_type, const char *module_name, + uint8 **p_buffer, uint32 *p_size) +{ + char *file_format = NULL; +#if WASM_ENABLE_INTERP != 0 + if (module_type == Wasm_Module_Bytecode) + file_format = ".wasm"; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_type == Wasm_Module_AoT) + file_format = ".aot"; +#endif + bh_assert(file_format); + const char *format = "%s/%s%s"; + int sz = strlen(module_search_path) + strlen("/") + strlen(module_name) + + strlen(file_format) + 1; + char *wasm_file_name = wasm_runtime_malloc(sz); + if (!wasm_file_name) { + return false; + } + snprintf(wasm_file_name, sz, format, module_search_path, module_name, + file_format); + *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_name, p_size); + + wasm_runtime_free(wasm_file_name); + return *p_buffer != NULL; +} + +static void +moudle_destroyer(uint8 *buffer, uint32 size) +{ + if (!buffer) { + return; + } + + wasm_runtime_free(buffer); + buffer = NULL; +} +#endif /* WASM_ENABLE_MULTI_MODULE */ + +int +main(int argc, char *argv[]) +{ + int32 ret = -1; + char *wasm_file = NULL; + const char *func_name = NULL; + uint8 *wasm_file_buf = NULL; + uint32 wasm_file_size; + uint32 stack_size = 64 * 1024; +#if WASM_ENABLE_LIBC_WASI != 0 + uint32 heap_size = 0; +#else + uint32 heap_size = 16 * 1024; +#endif +#if WASM_ENABLE_JIT != 0 + uint32 llvm_jit_size_level = 3; + uint32 llvm_jit_opt_level = 3; +#endif + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + RunningMode running_mode = 0; + RuntimeInitArgs init_args; + char error_buf[128] = { 0 }; +#if WASM_ENABLE_LOG != 0 + int log_verbose_level = 2; +#endif + bool is_repl_mode = false; + bool is_xip_file = false; +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_parse_context_t wasi_parse_ctx; +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + char *ip_addr = NULL; + int instance_port = 0; +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 + memset(&wasi_parse_ctx, 0, sizeof(wasi_parse_ctx)); +#endif + + /* Process options. */ + for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { + if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--function")) { + argc--, argv++; + if (argc < 2) { + return print_help(); + } + func_name = argv[0]; + } +#if WASM_ENABLE_INTERP != 0 + else if (!strcmp(argv[0], "--interp")) { + running_mode = Mode_Interp; + } +#endif +#if WASM_ENABLE_FAST_JIT != 0 + else if (!strcmp(argv[0], "--fast-jit")) { + running_mode = Mode_Fast_JIT; + } +#endif +#if WASM_ENABLE_JIT != 0 + else if (!strcmp(argv[0], "--llvm-jit")) { + running_mode = Mode_LLVM_JIT; + } +#endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 + else if (!strcmp(argv[0], "--multi-tier-jit")) { + running_mode = Mode_Multi_Tier_JIT; + } +#endif +#if WASM_ENABLE_LOG != 0 + else if (!strncmp(argv[0], "-v=", 3)) { + log_verbose_level = atoi(argv[0] + 3); + if (log_verbose_level < 0 || log_verbose_level > 5) + return print_help(); + } +#endif + else if (!strcmp(argv[0], "--repl")) { + is_repl_mode = true; + } + else if (!strncmp(argv[0], "--stack-size=", 13)) { + if (argv[0][13] == '\0') + return print_help(); + stack_size = atoi(argv[0] + 13); + } + else if (!strncmp(argv[0], "--heap-size=", 12)) { + if (argv[0][12] == '\0') + return print_help(); + heap_size = atoi(argv[0] + 12); + } +#if WASM_ENABLE_JIT != 0 + else if (!strncmp(argv[0], "--llvm-jit-size-level=", 22)) { + if (argv[0][22] == '\0') + return print_help(); + llvm_jit_size_level = atoi(argv[0] + 22); + if (llvm_jit_size_level < 1) { + printf("LLVM JIT size level shouldn't be smaller than 1, " + "setting it to 1\n"); + llvm_jit_size_level = 1; + } + else if (llvm_jit_size_level > 3) { + printf("LLVM JIT size level shouldn't be greater than 3, " + "setting it to 3\n"); + llvm_jit_size_level = 3; + } + } + else if (!strncmp(argv[0], "--llvm-jit-opt-level=", 21)) { + if (argv[0][21] == '\0') + return print_help(); + llvm_jit_opt_level = atoi(argv[0] + 21); + if (llvm_jit_opt_level < 1) { + printf("LLVM JIT opt level shouldn't be smaller than 1, " + "setting it to 1\n"); + llvm_jit_opt_level = 1; + } + else if (llvm_jit_opt_level > 3) { + printf("LLVM JIT opt level shouldn't be greater than 3, " + "setting it to 3\n"); + llvm_jit_opt_level = 3; + } + } +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + else if (!strncmp(argv[0], MODULE_PATH, strlen(MODULE_PATH))) { + module_search_path = handle_module_path(argv[0]); + if (!strlen(module_search_path)) { + return print_help(); + } + } +#endif +#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + else if (!strncmp(argv[0], "--max-threads=", 14)) { + if (argv[0][14] == '\0') + return print_help(); + wasm_runtime_set_max_thread_num(atoi(argv[0] + 14)); + } +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + else if (!strncmp(argv[0], "-g=", 3)) { + char *port_str = strchr(argv[0] + 3, ':'); + char *port_end; + if (port_str == NULL) + return print_help(); + *port_str = '\0'; + instance_port = strtoul(port_str + 1, &port_end, 10); + if (port_str[1] == '\0' || *port_end != '\0') + return print_help(); + ip_addr = argv[0] + 3; + } +#endif + else if (!strcmp(argv[0], "--version")) { + uint32 major, minor, patch; + wasm_runtime_get_version(&major, &minor, &patch); + printf("iwasm %" PRIu32 ".%" PRIu32 ".%" PRIu32 "\n", major, minor, + patch); + return 0; + } + else { +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_parse_result_t result = + libc_wasi_parse(argv[0], &wasi_parse_ctx); + switch (result) { + case LIBC_WASI_PARSE_RESULT_OK: + continue; + case LIBC_WASI_PARSE_RESULT_NEED_HELP: + return print_help(); + case LIBC_WASI_PARSE_RESULT_BAD_PARAM: + return 1; + } +#else + return print_help(); +#endif + } + } + + if (argc == 0) + return print_help(); + + wasm_file = argv[0]; + app_argc = argc; + app_argv = argv; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.running_mode = running_mode; +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); +#else + init_args.mem_alloc_type = Alloc_With_Allocator; +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + /* Set user data for the allocator is needed */ + /* init_args.mem_alloc_option.allocator.user_data = user_data; */ +#endif + init_args.mem_alloc_option.allocator.malloc_func = malloc_func; + init_args.mem_alloc_option.allocator.realloc_func = realloc_func; + init_args.mem_alloc_option.allocator.free_func = free_func; +#endif + +#if WASM_ENABLE_JIT != 0 + init_args.llvm_jit_size_level = llvm_jit_size_level; + init_args.llvm_jit_opt_level = llvm_jit_opt_level; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + init_args.instance_port = instance_port; + if (ip_addr) + /* ensure that init_args.ip_addr is null terminated */ + strncpy_s(init_args.ip_addr, sizeof(init_args.ip_addr) - 1, ip_addr, + strlen(ip_addr)); +#endif + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + +#if WASM_ENABLE_LOG != 0 + bh_log_set_verbose_level(log_verbose_level); +#endif + + /* load WASM byte buffer from WASM bin file */ + if (!(wasm_file_buf = + (uint8 *)bh_read_file_to_buffer(wasm_file, &wasm_file_size))) + goto fail1; + +#if WASM_ENABLE_AOT != 0 + if (wasm_runtime_is_xip_file(wasm_file_buf, wasm_file_size)) { + uint8 *wasm_file_mapped; + int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC; + int map_flags = MMAP_MAP_32BIT; + + if (!(wasm_file_mapped = os_mmap(NULL, (uint32)wasm_file_size, map_prot, + map_flags, os_get_invalid_handle()))) { + printf("mmap memory failed\n"); + wasm_runtime_free(wasm_file_buf); + goto fail1; + } + + bh_memcpy_s(wasm_file_mapped, wasm_file_size, wasm_file_buf, + wasm_file_size); + wasm_runtime_free(wasm_file_buf); + wasm_file_buf = wasm_file_mapped; + is_xip_file = true; + } +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + wasm_runtime_set_module_reader(module_reader_callback, moudle_destroyer); +#endif + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail2; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + libc_wasi_init(wasm_module, argc, argv, &wasi_parse_ctx); +#endif + + /* instantiate the module */ + if (!(wasm_module_inst = + wasm_runtime_instantiate(wasm_module, stack_size, heap_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail3; + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (ip_addr != NULL) { + wasm_exec_env_t exec_env = + wasm_runtime_get_exec_env_singleton(wasm_module_inst); + uint32_t debug_port; + if (exec_env == NULL) { + printf("%s\n", wasm_runtime_get_exception(wasm_module_inst)); + goto fail4; + } + debug_port = wasm_runtime_start_debug_instance(exec_env); + if (debug_port == 0) { + printf("Failed to start debug instance\n"); + goto fail4; + } + } +#endif + + ret = 0; + const char *exception = NULL; + if (is_repl_mode) { + app_instance_repl(wasm_module_inst); + } + else if (func_name) { + exception = app_instance_func(wasm_module_inst, func_name); + if (exception) { + /* got an exception */ + ret = 1; + } + } + else { + exception = app_instance_main(wasm_module_inst); + if (exception) { + /* got an exception */ + ret = 1; + } + } + +#if WASM_ENABLE_LIBC_WASI != 0 + if (ret == 0) { + /* propagate wasi exit code. */ + ret = wasm_runtime_get_wasi_exit_code(wasm_module_inst); + } +#endif + + if (exception) + printf("%s\n", exception); + +#if WASM_ENABLE_DEBUG_INTERP != 0 +fail4: +#endif + /* destroy the module instance */ + wasm_runtime_deinstantiate(wasm_module_inst); + +fail3: + /* unload the module */ + wasm_runtime_unload(wasm_module); + +fail2: + /* free the file buffer */ + if (!is_xip_file) + wasm_runtime_free(wasm_file_buf); + else + os_munmap(wasm_file_buf, wasm_file_size); + +fail1: + /* destroy runtime environment */ + wasm_runtime_destroy(); + + return ret; +} diff --git a/wasm-micro-runtime/product-mini/platforms/windows/wasi_filtered_tests.json b/wasm-micro-runtime/product-mini/platforms/windows/wasi_filtered_tests.json new file mode 100644 index 0000000..6391043 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/windows/wasi_filtered_tests.json @@ -0,0 +1,23 @@ +{ + "WASI Rust tests": { + "poll_oneoff_stdio": "Not implemented" + }, + "WASI threads proposal": { + "wasi_threads_exit_main_wasi_read": "Blocking ops not implemented", + "wasi_threads_exit_nonmain_wasi_read": "Blocking ops not implemented", + "wasi_threads_return_main_wasi_read": "Blocking ops not implemented", + "wasi_threads_return_main_wasi": "Blocking ops not implemented", + "wasi_threads_exit_nonmain_wasi": "Blocking ops not implemented", + "wasi_threads_exit_main_wasi": "Blocking ops not implemented" + }, + "WAMR lib-socket tests": { + "nslookup": "Not implemented", + "tcp_udp": "Not implemented" + }, + "lib-wasi-threads tests": { + "nonmain_proc_exit_sleep": "poll_oneoff not implemented", + "main_proc_exit_sleep": "poll_oneoff not implemented", + "main_trap_sleep": "poll_oneoff not implemented", + "nonmain_trap_sleep": "poll_oneoff not implemented" + } +} \ No newline at end of file diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/CMakeLists.txt b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/CMakeLists.txt new file mode 100644 index 0000000..8b2af15 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/CMakeLists.txt @@ -0,0 +1,60 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.8.2) + +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(wamr) + +enable_language (ASM) + +set (WAMR_BUILD_PLATFORM "zephyr") + +# Build as X86_32 by default, change to "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", "MIPS" or "XTENSA" +# if we want to support arm, thumb, mips or xtensa +if (NOT DEFINED WAMR_BUILD_TARGET) + set (WAMR_BUILD_TARGET "X86_32") +endif () + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + # Enable libc builtin support by default + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_WASI) + # Disable libc wasi support by default + set (WAMR_BUILD_LIBC_WASI 0) +endif () + +if (WAMR_BUILD_TARGET STREQUAL "RISCV64_LP64" OR WAMR_BUILD_TARGET STREQUAL "RISCV32_ILP32") + set (WAMR_BUILD_FAST_INTERP 1) +endif () + +# Override the global heap usage +if (NOT DEFINED WAMR_BUILD_GLOBAL_HEAP_POOL) + set (WAMR_BUILD_GLOBAL_HEAP_POOL 1) +endif () + +# Override the global heap size for small devices +if (NOT DEFINED WAMR_BUILD_GLOBAL_HEAP_SIZE) + set (WAMR_BUILD_GLOBAL_HEAP_SIZE 131072) # 128 KB +endif () + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../..) + +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +target_sources(app PRIVATE + ${WAMR_RUNTIME_LIB_SOURCE} + src/main.c) + diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/Dockerfile b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/Dockerfile new file mode 100644 index 0000000..a4c69a8 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/Dockerfile @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Asian/Shanghai + +# Install dependencies for Zephyr +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends git cmake ninja-build gperf \ + ccache dfu-util device-tree-compiler wget \ + python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \ + make gcc gcc-multilib g++-multilib libsdl2-dev libmagic1 \ + && apt-get clean -y && rm -rf /var/lib/apt/lists/* + +# Install the Zephyr Software Development Kit (SDK) +WORKDIR /opt +# hadolint ignore=DL4006 +RUN wget --progress=dot:giga https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.3/zephyr-sdk-0.16.3_linux-x86_64.tar.xz \ + && wget --progress=dot:giga -O - https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.3/sha256.sum | shasum --check --ignore-missing \ + && tar xvf zephyr-sdk-0.16.3_linux-x86_64.tar.xz && rm zephyr-sdk-0.16.3_linux-x86_64.tar.xz + +WORKDIR /opt/zephyr-sdk-0.16.3 +# hadolint ignore=DL4006 +RUN yes | ./setup.sh + +# Get Zephyr +# hadolint ignore=DL3013 +RUN pip3 install --no-cache-dir west && west init -m https://github.com/zephyrproject-rtos/zephyr --mr v3.5.0 /root/zephyrproject + +WORKDIR /root/zephyrproject +RUN west update + +WORKDIR /root/zephyrproject/zephyr +RUN west zephyr-export && pip install --no-cache-dir -r ~/zephyrproject/zephyr/scripts/requirements.txt + +# Git clone wamr +WORKDIR /root +RUN git clone https://github.com/bytecodealliance/wasm-micro-runtime.git + +WORKDIR /root/wasm-micro-runtime/product-mini/platforms/zephyr/simple + +ENV ZEPHYR_BASE="/root/zephyrproject/zephyr" diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/README.md b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/README.md new file mode 100644 index 0000000..5f8bd41 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/README.md @@ -0,0 +1,108 @@ +# How to use WAMR with Zephyr + +[Zephyr](https://www.zephyrproject.org/) is an open source real-time operating +system (RTOS) with a focus on security and broad hardware support. WAMR is +compatible with Zephyr via the [Zephyr WAMR +port](../../../../core/shared/platform/zephyr). + +## Setup + +Using WAMR with Zephyr can be accomplished by either using the provided Docker +image, or by installing Zephyr locally. Both approaches are described below. + +### Docker + +The provided Docker image sets up Zephyr and its dependencies for all +architectures, meaning that you are ready to build for any [supported +board](https://docs.zephyrproject.org/latest/boards/index.html). This comes at +the expense of building a rather large image, which both can take a long time to +build and uses up a large amount of storage (~15 GB). + +Execute the following command to build the Docker image. This may take an +extended period of time to complete. + +```shell +docker build -t wamr-zephyr . +``` + +Execute the following command to run the image built in the previous step. If +you are planning to flash a device after building, make sure to specify it with +[`--device`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). + +```shell +docker run -it --rm --device=/dev/ttyUSB0 wamr-zephyr +``` + +### Local Environment + +Zephyr can also be setup locally to enable building this sample application. +This allows you have have more control over what modules and tools are +installed, which can drastically reduce the storage required compared to the +Docker image. + +Follow the steps provided in the [Zephyr Getting Started +guide](https://docs.zephyrproject.org/latest/develop/getting_started/index.html) +to setup for local development. + +## Building for a Specific Board + +With an environment setup either locally or in a Docker container, you can build +for a Zephyr suppported board using +[`west`](https://docs.zephyrproject.org/latest/develop/west/index.html). There +are already [configuaration files](./boards) for a few boards in this sample. +However, if you are using a new board, you will need to add your own file for +the board, or define configuration in the [`prj.conf](./prj.conf). After doing +so, use the following command with your board identifier to build the sample +application. + +```shell +west build . -b --pristine -- -DWAMR_BUILD_TARGET= +``` + +The `` can be found in the Zephyr supported boards +documentation. It must also be used as the name of the board configuration file. +You must define the architecture for WAMR to target with `WAMR_BUILD_TARGET`. +The list of supported architectures can be found in the main project +[README.md](../../../../README.md#supported-architectures-and-platforms). + +It may be necessary to define additional symbols for some boards. For example, +WAMR AOT execution may not be supported on all architectures. It and other +options can be disabled by modifying the [CMakeLists.txt](./CMakeLists.txt) +file, or by passing additional arguments at build (e.g. `-DWAMR_BUILD_AOT=0`). + +### Example Targets + +[ESP32-C3](https://docs.zephyrproject.org/latest/boards/riscv/esp32c3_devkitm/doc/index.html) +is a 32-bit RISC-V target that does not currently support AOT. + +```shell +west build . -b esp32c3_devkitm -p always -- -DWAMR_BUILD_TARGET=RISCV32_ILP32 -DWAMR_BUILD_AOT=0 +``` + +[ARM Cortex-A53 QEMU +(ARM)](https://docs.zephyrproject.org/latest/boards/arm64/qemu_cortex_a53/doc/index.html) +is a 64-bit ARM target for emulating the Cortex-A53 platform. + +```shell +west build . -b qemu_cortex_a53 -p always -- -DWAMR_BUILD_TARGET=AARCH64 +``` + + +## Flashing or Running Image + +The board can be flashed with the built image with the following command. + +```shell +west flash +``` + +`west` will automatically identify the board if it is connected to the host +machine. + +When using emulated targets, such as those that utilize QEMU, there is no +physical device to flash, but `west` can be used to run the image under +emulation. + +```shell +west build -t run +``` diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/boards/nucleo_f767zi.conf b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/boards/nucleo_f767zi.conf new file mode 100644 index 0000000..c920e6f --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/boards/nucleo_f767zi.conf @@ -0,0 +1,4 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CONFIG_ARM_MPU=y diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/boards/qemu_cortex_a53.conf b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/boards/qemu_cortex_a53.conf new file mode 100644 index 0000000..2cc2dd4 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/boards/qemu_cortex_a53.conf @@ -0,0 +1,4 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CONFIG_ARM_MMU=n diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/build_and_run.sh b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/build_and_run.sh new file mode 100755 index 0000000..6b8fb4f --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/build_and_run.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +X86_TARGET="x86" +STM32_TARGET="stm32" +ESP32C3_TARGET="esp32c3" +PARTICLE_ARGON_TARGET="particle_argon" +QEMU_CORTEX_A53="qemu_cortex_a53" +QEMU_XTENSA_TARGET="qemu_xtensa" +QEMU_RISCV64_TARGET="qemu_riscv64" +QEMU_RISCV32_TARGET="qemu_riscv32" +QEMU_ARC_TARGET="qemu_arc" + +usage () +{ + echo "USAGE:" + echo "$0 $X86_TARGET|$STM32_TARGET|$ESP32C3_TARGET|$PARTICLE_ARGON_TARGET|$QEMU_CORTEX_A53|$QEMU_XTENSA_TARGET|$QEMU_RISCV64_TARGET|$QEMU_RISCV32_TARGET|$QEMU_ARC_TARGET" + echo "Example:" + echo " $0 $X86_TARGET" + echo " $0 $STM32_TARGET" + echo " $0 $ESP32C3_TARGET" + echo " $0 $PARTICLE_ARGON_TARGET" + echo " $0 $QEMU_CORTEX_A53" + echo " $0 $QEMU_XTENSA_TARGET" + echo " $0 $QEMU_RISCV64_TARGET" + echo " $0 $QEMU_RISCV32_TARGET" + echo " $0 $QEMU_ARC_TARGET" + exit 1 +} + +if [ $# != 1 ] ; then + usage +fi + +TARGET=$1 + +case $TARGET in + $X86_TARGET) + west build -b qemu_x86_nommu \ + . -p always -- \ + -DWAMR_BUILD_TARGET=X86_32 + west build -t run + ;; + $STM32_TARGET) + west build -b nucleo_f767zi \ + . -p always -- \ + -DWAMR_BUILD_TARGET=THUMBV7 + west flash + ;; + $ESP32C3_TARGET) + west build -b esp32c3_devkitm \ + . -p always -- \ + -DWAMR_BUILD_TARGET=RISCV32_ILP32 \ + -DWAMR_BUILD_AOT=0 + # west flash will discover the device + west flash + ;; + $PARTICLE_ARGON_TARGET) + west build -b particle_argon \ + . -p always -- \ + -DWAMR_BUILD_TARGET=THUMBV7 + # west flash will discover the device + west flash + ;; + $QEMU_XTENSA_TARGET) + west build -b qemu_xtensa \ + . -p always -- \ + -DWAMR_BUILD_TARGET=XTENSA + west build -t run + ;; + $QEMU_CORTEX_A53) + west build -b qemu_cortex_a53 \ + . -p always -- \ + -DWAMR_BUILD_TARGET=AARCH64 + west build -t run + ;; + $QEMU_RISCV64_TARGET) + west build -b qemu_riscv64 \ + . -p always -- \ + -DWAMR_BUILD_TARGET=RISCV64_LP64 \ + -DWAMR_BUILD_AOT=0 + west build -t run + ;; + $QEMU_RISCV32_TARGET) + west build -b qemu_riscv32 \ + . -p always -- \ + -DWAMR_BUILD_TARGET=RISCV32_ILP32 \ + -DWAMR_BUILD_AOT=0 + west build -t run + ;; + $QEMU_ARC_TARGET) + west build -b qemu_arc_em \ + . -p always -- \ + -DWAMR_BUILD_TARGET=ARC \ + -DWAMR_BUILD_AOT=0 + west build -t run + ;; + *) + echo "unsupported target: $TARGET" + usage + exit 1 + ;; +esac + diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/prj.conf b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/prj.conf new file mode 100644 index 0000000..7f4a328 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/prj.conf @@ -0,0 +1,6 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CONFIG_STACK_SENTINEL=y +CONFIG_PRINTK=y +CONFIG_LOG=y diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/main.c b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/main.c new file mode 100644 index 0000000..3b38982 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/main.c @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include "bh_platform.h" +#include "bh_assert.h" +#include "bh_log.h" +#include "wasm_export.h" +#if defined(BUILD_TARGET_RISCV64_LP64) || defined(BUILD_TARGET_RISCV32_ILP32) +#include "test_wasm_riscv64.h" +#else +#include "test_wasm.h" +#endif /* end of BUILD_TARGET_RISCV64_LP64 || BUILD_TARGET_RISCV32_ILP32 */ + +#if defined(BUILD_TARGET_RISCV64_LP64) || defined(BUILD_TARGET_RISCV32_ILP32) +#if defined(BUILD_TARGET_RISCV64_LP64) +#define CONFIG_GLOBAL_HEAP_BUF_SIZE 4360 +#define CONFIG_APP_STACK_SIZE 288 +#define CONFIG_MAIN_THREAD_STACK_SIZE 2400 +#else +#define CONFIG_GLOBAL_HEAP_BUF_SIZE 5120 +#define CONFIG_APP_STACK_SIZE 512 +#define CONFIG_MAIN_THREAD_STACK_SIZE 4096 +#endif +#define CONFIG_APP_HEAP_SIZE 256 +#else /* else of BUILD_TARGET_RISCV64_LP64 || BUILD_TARGET_RISCV32_ILP32 */ + +#define CONFIG_GLOBAL_HEAP_BUF_SIZE WASM_GLOBAL_HEAP_SIZE +#define CONFIG_APP_STACK_SIZE 8192 +#define CONFIG_APP_HEAP_SIZE 8192 + +#ifdef CONFIG_NO_OPTIMIZATIONS +#define CONFIG_MAIN_THREAD_STACK_SIZE 8192 +#else +#define CONFIG_MAIN_THREAD_STACK_SIZE 4096 +#endif + +#endif /* end of BUILD_TARGET_RISCV64_LP64 || BUILD_TARGET_RISCV32_ILP32 */ + +static int app_argc; +static char **app_argv; + +/** + * Find the unique main function from a WASM module instance + * and execute that function. + * + * @param module_inst the WASM module instance + * @param argc the number of arguments + * @param argv the arguments array + * + * @return true if the main function is called, false otherwise. + */ +bool +wasm_application_execute_main(wasm_module_inst_t module_inst, int argc, + char *argv[]); + +static void * +app_instance_main(wasm_module_inst_t module_inst) +{ + const char *exception; + wasm_function_inst_t func; + wasm_exec_env_t exec_env; + unsigned argv[2] = { 0 }; + + if (wasm_runtime_lookup_function(module_inst, "main") + || wasm_runtime_lookup_function(module_inst, "__main_argc_argv")) { + LOG_VERBOSE("Calling main function\n"); + wasm_application_execute_main(module_inst, app_argc, app_argv); + } + else if ((func = wasm_runtime_lookup_function(module_inst, "app_main"))) { + exec_env = + wasm_runtime_create_exec_env(module_inst, CONFIG_APP_HEAP_SIZE); + if (!exec_env) { + os_printf("Create exec env failed\n"); + return NULL; + } + + LOG_VERBOSE("Calling app_main function\n"); + wasm_runtime_call_wasm(exec_env, func, 0, argv); + + if (!wasm_runtime_get_exception(module_inst)) { + os_printf("result: 0x%x\n", argv[0]); + } + + wasm_runtime_destroy_exec_env(exec_env); + } + else { + os_printf("Failed to lookup function main or app_main to call\n"); + return NULL; + } + + if ((exception = wasm_runtime_get_exception(module_inst))) + os_printf("%s\n", exception); + + return NULL; +} + +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 +static char global_heap_buf[CONFIG_GLOBAL_HEAP_BUF_SIZE] = { 0 }; +#endif + +void +iwasm_main(void *arg1, void *arg2, void *arg3) +{ + int start, end; + start = k_uptime_get_32(); + uint8 *wasm_file_buf = NULL; + uint32 wasm_file_size; + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + RuntimeInitArgs init_args; + char error_buf[128]; +#if WASM_ENABLE_LOG != 0 + int log_verbose_level = 2; +#endif + + (void)arg1; + (void)arg2; + (void)arg3; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + +#if WASM_ENABLE_GLOBAL_HEAP_POOL != 0 + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); +#elif (defined(CONFIG_COMMON_LIBC_MALLOC) \ + && CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE != 0) \ + || defined(CONFIG_NEWLIB_LIBC) + init_args.mem_alloc_type = Alloc_With_System_Allocator; +#else +#error "memory allocation scheme is not defined." +#endif + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return; + } + +#if WASM_ENABLE_LOG != 0 + bh_log_set_verbose_level(log_verbose_level); +#endif + + /* load WASM byte buffer from byte buffer of include file */ + wasm_file_buf = (uint8 *)wasm_test_file; + wasm_file_size = sizeof(wasm_test_file); + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail1; + } + + /* instantiate the module */ + if (!(wasm_module_inst = wasm_runtime_instantiate( + wasm_module, CONFIG_APP_STACK_SIZE, CONFIG_APP_HEAP_SIZE, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail2; + } + + /* invoke the main function */ + app_instance_main(wasm_module_inst); + + /* destroy the module instance */ + wasm_runtime_deinstantiate(wasm_module_inst); + +fail2: + /* unload the module */ + wasm_runtime_unload(wasm_module); + +fail1: + /* destroy runtime environment */ + wasm_runtime_destroy(); + + end = k_uptime_get_32(); + + printf("elapsed: %d\n", (end - start)); +} + +#define MAIN_THREAD_STACK_SIZE (CONFIG_MAIN_THREAD_STACK_SIZE) +#define MAIN_THREAD_PRIORITY 5 + +K_THREAD_STACK_DEFINE(iwasm_main_thread_stack, MAIN_THREAD_STACK_SIZE); +static struct k_thread iwasm_main_thread; + +bool +iwasm_init(void) +{ + k_tid_t tid = k_thread_create( + &iwasm_main_thread, iwasm_main_thread_stack, MAIN_THREAD_STACK_SIZE, + iwasm_main, NULL, NULL, NULL, MAIN_THREAD_PRIORITY, 0, K_NO_WAIT); + return tid ? true : false; +} + +#if KERNEL_VERSION_NUMBER < 0x030400 /* version 3.4.0 */ +void +main(void) +{ + iwasm_init(); +} +#else +int +main(void) +{ + iwasm_init(); + return 0; +} +#endif diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/test_wasm.h b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/test_wasm.h new file mode 100644 index 0000000..a729cad --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/test_wasm.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * The byte array buffer is the file content of a test wasm binary file, + * which is compiled by wasi-sdk toolchain from C source file of: + * product-mini/app-samples/hello-world/main.c. + */ +unsigned char __aligned(4) wasm_test_file[] = { + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x10, 0x03, 0x60, + 0x01, 0x7F, 0x01, 0x7F, 0x60, 0x02, 0x7F, 0x7F, 0x01, 0x7F, 0x60, 0x01, + 0x7F, 0x00, 0x02, 0x31, 0x04, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x70, 0x75, + 0x74, 0x73, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x6D, 0x61, 0x6C, + 0x6C, 0x6F, 0x63, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x70, 0x72, + 0x69, 0x6E, 0x74, 0x66, 0x00, 0x01, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x66, + 0x72, 0x65, 0x65, 0x00, 0x02, 0x03, 0x02, 0x01, 0x01, 0x04, 0x05, 0x01, + 0x70, 0x01, 0x01, 0x01, 0x05, 0x03, 0x01, 0x00, 0x01, 0x06, 0x13, 0x03, + 0x7F, 0x01, 0x41, 0xC0, 0x28, 0x0B, 0x7F, 0x00, 0x41, 0xBA, 0x08, 0x0B, + 0x7F, 0x00, 0x41, 0xC0, 0x28, 0x0B, 0x07, 0x2C, 0x04, 0x06, 0x6D, 0x65, + 0x6D, 0x6F, 0x72, 0x79, 0x02, 0x00, 0x0A, 0x5F, 0x5F, 0x64, 0x61, 0x74, + 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x03, 0x01, 0x0B, 0x5F, 0x5F, 0x68, 0x65, + 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, 0x65, 0x03, 0x02, 0x04, 0x6D, 0x61, + 0x69, 0x6E, 0x00, 0x04, 0x0A, 0xB2, 0x01, 0x01, 0xAF, 0x01, 0x01, 0x03, + 0x7F, 0x23, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, + 0x24, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x9B, 0x88, 0x80, 0x80, 0x00, + 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x02, 0x40, 0x02, 0x40, 0x41, + 0x80, 0x08, 0x10, 0x81, 0x80, 0x80, 0x80, 0x00, 0x22, 0x03, 0x0D, 0x00, + 0x41, 0xA8, 0x88, 0x80, 0x80, 0x00, 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, + 0x1A, 0x41, 0x7F, 0x21, 0x04, 0x0C, 0x01, 0x0B, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x10, 0x41, 0x80, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x41, + 0x10, 0x6A, 0x10, 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x41, 0x00, 0x21, + 0x04, 0x20, 0x03, 0x41, 0x04, 0x6A, 0x41, 0x00, 0x2F, 0x00, 0x91, 0x88, + 0x80, 0x80, 0x00, 0x3B, 0x00, 0x00, 0x20, 0x03, 0x41, 0x00, 0x28, 0x00, + 0x8D, 0x88, 0x80, 0x80, 0x00, 0x36, 0x00, 0x00, 0x20, 0x02, 0x20, 0x03, + 0x36, 0x02, 0x00, 0x41, 0x93, 0x88, 0x80, 0x80, 0x00, 0x20, 0x02, 0x10, + 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x20, 0x03, 0x10, 0x83, 0x80, 0x80, + 0x80, 0x00, 0x0B, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x80, 0x80, 0x80, + 0x80, 0x00, 0x20, 0x04, 0x0B, 0x0B, 0x41, 0x01, 0x00, 0x41, 0x80, 0x08, + 0x0B, 0x3A, 0x62, 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, 0x20, 0x25, + 0x70, 0x0A, 0x00, 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, 0x75, 0x66, + 0x3A, 0x20, 0x25, 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, + 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, 0x6F, 0x63, + 0x20, 0x62, 0x75, 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, 0x64, 0x00 +}; diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/test_wasm_riscv64.h b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/test_wasm_riscv64.h new file mode 100644 index 0000000..1b45211 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/test_wasm_riscv64.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +unsigned char __aligned(4) wasm_test_file[] = { + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x10, 0x03, 0x60, + 0x01, 0x7F, 0x01, 0x7F, 0x60, 0x02, 0x7F, 0x7F, 0x01, 0x7F, 0x60, 0x01, + 0x7F, 0x00, 0x02, 0x31, 0x04, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x70, 0x75, + 0x74, 0x73, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x6D, 0x61, 0x6C, + 0x6C, 0x6F, 0x63, 0x00, 0x00, 0x03, 0x65, 0x6E, 0x76, 0x06, 0x70, 0x72, + 0x69, 0x6E, 0x74, 0x66, 0x00, 0x01, 0x03, 0x65, 0x6E, 0x76, 0x04, 0x66, + 0x72, 0x65, 0x65, 0x00, 0x02, 0x03, 0x02, 0x01, 0x01, 0x04, 0x05, 0x01, + 0x70, 0x01, 0x01, 0x01, 0x05, 0x03, 0x01, 0x00, 0x01, 0x06, 0x12, 0x03, + 0x7F, 0x01, 0x41, 0xC0, 0x01, 0x0B, 0x7F, 0x00, 0x41, 0x3A, 0x0B, 0x7F, + 0x00, 0x41, 0xC0, 0x01, 0x0B, 0x07, 0x2C, 0x04, 0x06, 0x6D, 0x65, 0x6D, + 0x6F, 0x72, 0x79, 0x02, 0x00, 0x04, 0x6D, 0x61, 0x69, 0x6E, 0x00, 0x04, + 0x0A, 0x5F, 0x5F, 0x64, 0x61, 0x74, 0x61, 0x5F, 0x65, 0x6E, 0x64, 0x03, + 0x01, 0x0B, 0x5F, 0x5F, 0x68, 0x65, 0x61, 0x70, 0x5F, 0x62, 0x61, 0x73, + 0x65, 0x03, 0x02, 0x0A, 0xB1, 0x01, 0x01, 0xAE, 0x01, 0x01, 0x03, 0x7F, + 0x23, 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x20, 0x6B, 0x22, 0x02, 0x24, + 0x80, 0x80, 0x80, 0x80, 0x00, 0x41, 0x9B, 0x80, 0x80, 0x80, 0x00, 0x10, + 0x80, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x02, 0x40, 0x02, 0x40, 0x41, 0x10, + 0x10, 0x81, 0x80, 0x80, 0x80, 0x00, 0x22, 0x03, 0x0D, 0x00, 0x41, 0xA8, + 0x80, 0x80, 0x80, 0x00, 0x10, 0x80, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x41, + 0x7F, 0x21, 0x04, 0x0C, 0x01, 0x0B, 0x20, 0x02, 0x20, 0x03, 0x36, 0x02, + 0x10, 0x41, 0x80, 0x80, 0x80, 0x80, 0x00, 0x20, 0x02, 0x41, 0x10, 0x6A, + 0x10, 0x82, 0x80, 0x80, 0x80, 0x00, 0x1A, 0x41, 0x00, 0x21, 0x04, 0x20, + 0x03, 0x41, 0x04, 0x6A, 0x41, 0x00, 0x2F, 0x00, 0x91, 0x80, 0x80, 0x80, + 0x00, 0x3B, 0x00, 0x00, 0x20, 0x03, 0x41, 0x00, 0x28, 0x00, 0x8D, 0x80, + 0x80, 0x80, 0x00, 0x36, 0x00, 0x00, 0x20, 0x02, 0x20, 0x03, 0x36, 0x02, + 0x00, 0x41, 0x93, 0x80, 0x80, 0x80, 0x00, 0x20, 0x02, 0x10, 0x82, 0x80, + 0x80, 0x80, 0x00, 0x1A, 0x20, 0x03, 0x10, 0x83, 0x80, 0x80, 0x80, 0x00, + 0x0B, 0x20, 0x02, 0x41, 0x20, 0x6A, 0x24, 0x80, 0x80, 0x80, 0x80, 0x00, + 0x20, 0x04, 0x0B, 0x0B, 0x40, 0x01, 0x00, 0x41, 0x00, 0x0B, 0x3A, 0x62, + 0x75, 0x66, 0x20, 0x70, 0x74, 0x72, 0x3A, 0x20, 0x25, 0x70, 0x0A, 0x00, + 0x31, 0x32, 0x33, 0x34, 0x0A, 0x00, 0x62, 0x75, 0x66, 0x3A, 0x20, 0x25, + 0x73, 0x00, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, 0x6F, 0x72, 0x6C, + 0x64, 0x21, 0x00, 0x6D, 0x61, 0x6C, 0x6C, 0x6F, 0x63, 0x20, 0x62, 0x75, + 0x66, 0x20, 0x66, 0x61, 0x69, 0x6C, 0x65, 0x64, 0x00 +}; diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/wasm-app-riscv64/build.sh b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/wasm-app-riscv64/build.sh new file mode 100755 index 0000000..73e7349 --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/wasm-app-riscv64/build.sh @@ -0,0 +1,27 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +WAMR_DIR=${PWD}/../../.. + +echo "Build wasm app .." +/opt/wasi-sdk/bin/clang -O3 \ + -z stack-size=128 -Wl,--initial-memory=65536 \ + -Wl,--global-base=0 \ + -o test.wasm main.c \ + -Wl,--export=main -Wl,--export=__main_argc_argv \ + -Wl,--export=__data_end -Wl,--export=__heap_base \ + -Wl,--strip-all,--no-entry \ + -Wl,--allow-undefined \ + -nostdlib \ + +echo "Build binarydump tool .." +rm -fr build && mkdir build && cd build +cmake ../../../../../../../test-tools/binarydump-tool +make +cd .. + +echo "Generate test_wasm.h .." +./build/binarydump -o test_wasm.h -n wasm_test_file test.wasm +cp -a test_wasm.h ../test_wasm_riscv64.h + +echo "Done" diff --git a/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/wasm-app-riscv64/main.c b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/wasm-app-riscv64/main.c new file mode 100644 index 0000000..026cb8f --- /dev/null +++ b/wasm-micro-runtime/product-mini/platforms/zephyr/simple/src/wasm-app-riscv64/main.c @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +int +main(int argc, char **argv) +{ + char *buf; + + printf("Hello world!\n"); + + buf = malloc(16); + if (!buf) { + printf("malloc buf failed\n"); + return -1; + } + + printf("buf ptr: %p\n", buf); + + snprintf(buf, 1024, "%s", "1234\n"); + printf("buf: %s", buf); + + free(buf); + return 0; +} diff --git a/wasm-micro-runtime/samples/README.md b/wasm-micro-runtime/samples/README.md new file mode 100644 index 0000000..872e179 --- /dev/null +++ b/wasm-micro-runtime/samples/README.md @@ -0,0 +1,15 @@ +# Samples + +- [**basic**](./basic): Demonstrating how to use runtime exposed API's to call WASM functions, how to register native functions and call them, and how to call WASM function from native function. +- **[file](./file/README.md)**: Demonstrating the supported file interaction API of WASI. This sample can also demonstrate the SGX IPFS (Intel Protected File System), enabling an enclave to seal and unseal data at rest. +- **[multi-thread](./multi-thread/)**: Demonstrating how to run wasm application which creates multiple threads to execute wasm functions concurrently, and uses mutex/cond by calling pthread related API's. +- **[spawn-thread](./spawn-thread)**: Demonstrating how to execute wasm functions of the same wasm application concurrently, in threads created by host embedder or runtime, but not the wasm application itself. +- **[wasi-threads](./wasi-threads/README.md)**: Demonstrating how to run wasm application which creates multiple threads to execute wasm functions concurrently based on lib wasi-threads. +- **[multi-module](./multi-module)**: Demonstrating the [multiple modules as dependencies](./doc/multi_module.md) feature which implements the [load-time dynamic linking](https://webassembly.org/docs/dynamic-linking/). +- **[ref-types](./ref-types)**: Demonstrating how to call wasm functions with argument of externref type introduced by [reference types proposal](https://github.com/WebAssembly/reference-types). +- **[wasm-c-api](./wasm-c-api/README.md)**: Demonstrating how to run some samples from [wasm-c-api proposal](https://github.com/WebAssembly/wasm-c-api) and showing the supported API's. +- **[socket-api](./socket-api/README.md)**: Demonstrating how to run wasm tcp server and tcp client applications, and how they communicate with each other. +- **[native-lib](./native-lib/README.md)**: Demonstrating how to write required interfaces in native library, build it into a shared library and register the shared library to iwasm. +- **[sgx-ra](./sgx-ra/README.md)**: Demonstrating how to execute Remote Attestation on SGX with [librats](https://github.com/inclavare-containers/librats), which enables mutual attestation with other runtimes or other entities that support librats to ensure that each is running within the TEE. +- **[workload](./workload/README.md)**: Demonstrating how to build and run some complex workloads, e.g. tensorflow-lite, XNNPACK, wasm-av1, meshoptimizer and bwa. +- **[debug-tools](./debug-tools/README.md)**: Demonstrating how to symbolicate a stack trace. diff --git a/wasm-micro-runtime/samples/basic/.gitignore b/wasm-micro-runtime/samples/basic/.gitignore new file mode 100644 index 0000000..0fa8a76 --- /dev/null +++ b/wasm-micro-runtime/samples/basic/.gitignore @@ -0,0 +1 @@ +/out/ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/basic/CMakeLists.txt b/wasm-micro-runtime/samples/basic/CMakeLists.txt new file mode 100644 index 0000000..62edf08 --- /dev/null +++ b/wasm-micro-runtime/samples/basic/CMakeLists.txt @@ -0,0 +1,90 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (basic) +else() + project (basic C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) + +if (NOT MSVC) + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (basic src/main.c src/native_impl.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (basic PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (basic vmlib -lm -ldl -lpthread) +else () + target_link_libraries (basic vmlib -lm -ldl -lpthread -lrt) +endif () diff --git a/wasm-micro-runtime/samples/basic/README.md b/wasm-micro-runtime/samples/basic/README.md new file mode 100644 index 0000000..32e7ed6 --- /dev/null +++ b/wasm-micro-runtime/samples/basic/README.md @@ -0,0 +1,51 @@ + + +The "basic" sample project +============== + +This sample demonstrates a few basic usages of embedding WAMR: +- initialize runtime +- load wasm app and instantiate the module +- call wasm function and pass arguments +- export native functions to the WASM apps +- wasm function calls native function and pass arguments +- deinitialize runtime + +Build this sample +============== +Execute the ```build.sh``` script then all binaries including wasm application files would be generated in 'out' directory. + +``` +$ ./build.sh +``` + +Run the sample +========================== +Enter the out directory. +``` +$ cd ./out/ +$ +$ ./basic -f wasm-apps/testapp.wasm +calling into WASM function: generate_float +Native finished calling wasm function generate_float(), returned a float value: 102009.921875f +calling into WASM function: float_to_string +calling into native function: intToStr +calling into native function: get_pow +calling into native function: intToStr +Native finished calling wasm function: float_to_string, returned a formatted string: 102009.921 +``` +Or execute the ```run.sh``` script in ```samples/basic``` folder. +``` +$ ./run.sh +calling into WASM function: generate_float +Native finished calling wasm function generate_float(), returned a float value: 102009.921875f +calling into WASM function: float_to_string +calling into native function: intToStr +calling into native function: get_pow +calling into native function: intToStr +Native finished calling wasm function: float_to_string, returned a formatted string: 102009.921 +``` + + + + diff --git a/wasm-micro-runtime/samples/basic/build.sh b/wasm-micro-runtime/samples/basic/build.sh new file mode 100755 index 0000000..c1d598a --- /dev/null +++ b/wasm-micro-runtime/samples/basic/build.sh @@ -0,0 +1,63 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +CURR_DIR=$PWD +WAMR_DIR=${PWD}/../.. +OUT_DIR=${PWD}/out + +WASM_APPS=${PWD}/wasm-apps + + +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +mkdir ${OUT_DIR}/wasm-apps + + +echo "#####################build basic project" +cd ${CURR_DIR} +mkdir -p cmake_build +cd cmake_build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BH_VPRINTF=my_vprintf -DWAMR_BH_LOG=my_log +make -j ${nproc} +if [ $? != 0 ];then + echo "BUILD_FAIL basic exit as $?\n" + exit 2 +fi + +cp -a basic ${OUT_DIR} + +echo -e "\n" + +echo "#####################build wasm apps" + +cd ${WASM_APPS} + +for i in `ls *.c` +do +APP_SRC="$i" +OUT_FILE=${i%.*}.wasm + +# use WAMR SDK to build out the .wasm binary +/opt/wasi-sdk/bin/clang \ + --target=wasm32 -O0 -z stack-size=4096 -Wl,--initial-memory=65536 \ + --sysroot=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot \ + -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt \ + -Wl,--strip-all,--no-entry -nostdlib \ + -Wl,--export=generate_float \ + -Wl,--export=float_to_string \ + -Wl,--export=calculate\ + -Wl,--allow-undefined \ + -o ${OUT_DIR}/wasm-apps/${OUT_FILE} ${APP_SRC} + + +if [ -f ${OUT_DIR}/wasm-apps/${OUT_FILE} ]; then + echo "build ${OUT_FILE} success" +else + echo "build ${OUT_FILE} fail" +fi +done +echo "####################build wasm apps done" diff --git a/wasm-micro-runtime/samples/basic/run.sh b/wasm-micro-runtime/samples/basic/run.sh new file mode 100755 index 0000000..a5fb291 --- /dev/null +++ b/wasm-micro-runtime/samples/basic/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +out/basic -f out/wasm-apps/testapp.wasm \ No newline at end of file diff --git a/wasm-micro-runtime/samples/basic/src/main.c b/wasm-micro-runtime/samples/basic/src/main.c new file mode 100644 index 0000000..406b242 --- /dev/null +++ b/wasm-micro-runtime/samples/basic/src/main.c @@ -0,0 +1,242 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" +#include "bh_getopt.h" + +int +intToStr(int x, char *str, int str_len, int digit); +int +get_pow(int x, int y); +int32_t +calculate_native(int32_t n, int32_t func1, int32_t func2); + +void +my_log(uint32 log_level, const char *file, int line, const char *fmt, ...) +{ + char buf[200]; + snprintf(buf, 200, + log_level == WASM_LOG_LEVEL_VERBOSE ? "[WamrLogger - VERBOSE] %s" + : "[WamrLogger] %s", + fmt); + + va_list ap; + va_start(ap, fmt); + vprintf(buf, ap); + va_end(ap); +} + +int +my_vprintf(const char *format, va_list ap) +{ + /* Print in blue */ + char buf[200]; + snprintf(buf, 200, "\x1b[34m%s\x1b[0m", format); + return vprintf(buf, ap); +} + +void +print_usage(void) +{ + fprintf(stdout, "Options:\r\n"); + fprintf(stdout, " -f [path of wasm file] \n"); +} + +int +main(int argc, char *argv_main[]) +{ + static char global_heap_buf[512 * 1024]; + char *buffer, error_buf[128]; + int opt; + char *wasm_path = NULL; + + wasm_module_t module = NULL; + wasm_module_inst_t module_inst = NULL; + wasm_exec_env_t exec_env = NULL; + uint32 buf_size, stack_size = 8092, heap_size = 8092; + wasm_function_inst_t func = NULL; + wasm_function_inst_t func2 = NULL; + char *native_buffer = NULL; + uint64_t wasm_buffer = 0; + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + while ((opt = getopt(argc, argv_main, "hf:")) != -1) { + switch (opt) { + case 'f': + wasm_path = optarg; + break; + case 'h': + print_usage(); + return 0; + case '?': + print_usage(); + return 0; + } + } + if (optind == 1) { + print_usage(); + return 0; + } + + // Define an array of NativeSymbol for the APIs to be exported. + // Note: the array must be static defined since runtime + // will keep it after registration + // For the function signature specifications, goto the link: + // https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/export_native_api.md + + static NativeSymbol native_symbols[] = { + { + "intToStr", // the name of WASM function name + intToStr, // the native function pointer + "(i*~i)i", // the function prototype signature, avoid to use i32 + NULL // attachment is NULL + }, + { + "get_pow", // the name of WASM function name + get_pow, // the native function pointer + "(ii)i", // the function prototype signature, avoid to use i32 + NULL // attachment is NULL + }, + { "calculate_native", calculate_native, "(iii)i", NULL } + }; + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + // Native symbols need below registration phase + init_args.n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); + init_args.native_module_name = "env"; + init_args.native_symbols = native_symbols; + + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + wasm_runtime_set_log_level(WASM_LOG_LEVEL_VERBOSE); + + buffer = bh_read_file_to_buffer(wasm_path, &buf_size); + + if (!buffer) { + printf("Open wasm app file [%s] failed.\n", wasm_path); + goto fail; + } + + module = wasm_runtime_load((uint8 *)buffer, buf_size, error_buf, + sizeof(error_buf)); + if (!module) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + + if (!module_inst) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + exec_env = wasm_runtime_create_exec_env(module_inst, stack_size); + if (!exec_env) { + printf("Create wasm execution environment failed.\n"); + goto fail; + } + + if (!(func = wasm_runtime_lookup_function(module_inst, "generate_float"))) { + printf("The generate_float wasm function is not found.\n"); + goto fail; + } + + wasm_val_t results[1] = { { .kind = WASM_F32, .of.f32 = 0 } }; + wasm_val_t arguments[3] = { + { .kind = WASM_I32, .of.i32 = 10 }, + { .kind = WASM_F64, .of.f64 = 0.000101 }, + { .kind = WASM_F32, .of.f32 = 300.002 }, + }; + + // pass 4 elements for function arguments + if (!wasm_runtime_call_wasm_a(exec_env, func, 1, results, 3, arguments)) { + printf("call wasm function generate_float failed. %s\n", + wasm_runtime_get_exception(module_inst)); + goto fail; + } + + float ret_val; + ret_val = results[0].of.f32; + printf("Native finished calling wasm function generate_float(), returned a " + "float value: %ff\n", + ret_val); + + // Next we will pass a buffer to the WASM function + uint32 argv2[5]; + + // must allocate buffer from wasm instance memory space (never use pointer + // from host runtime) + wasm_buffer = + wasm_runtime_module_malloc(module_inst, 100, (void **)&native_buffer); + + memcpy(argv2, &ret_val, sizeof(float)); // the first argument + argv2[1] = wasm_buffer; // the second argument is the wasm buffer address + argv2[3] = 100; // the third argument is the wasm buffer size + argv2[4] = 3; // the last argument is the digits after decimal point for + // converting float to string + + if (!(func2 = + wasm_runtime_lookup_function(module_inst, "float_to_string"))) { + printf( + "The wasm function float_to_string wasm function is not found.\n"); + goto fail; + } + + if (wasm_runtime_call_wasm(exec_env, func2, 4, argv2)) { + printf("Native finished calling wasm function: float_to_string, " + "returned a formatted string: %s\n", + native_buffer); + } + else { + printf("call wasm function float_to_string failed. error: %s\n", + wasm_runtime_get_exception(module_inst)); + goto fail; + } + + wasm_function_inst_t func3 = + wasm_runtime_lookup_function(module_inst, "calculate"); + if (!func3) { + printf("The wasm function calculate is not found.\n"); + goto fail; + } + + uint32_t argv3[1] = { 3 }; + if (wasm_runtime_call_wasm(exec_env, func3, 1, argv3)) { + uint32_t result = *(uint32_t *)argv3; + printf("Native finished calling wasm function: calculate, return: %d\n", + result); + } + else { + printf("call wasm function calculate failed. error: %s\n", + wasm_runtime_get_exception(module_inst)); + goto fail; + } + +fail: + if (exec_env) + wasm_runtime_destroy_exec_env(exec_env); + if (module_inst) { + if (wasm_buffer) + wasm_runtime_module_free(module_inst, (uint64)wasm_buffer); + wasm_runtime_deinstantiate(module_inst); + } + if (module) + wasm_runtime_unload(module); + if (buffer) + BH_FREE(buffer); + wasm_runtime_destroy(); + return 0; +} diff --git a/wasm-micro-runtime/samples/basic/src/native_impl.c b/wasm-micro-runtime/samples/basic/src/native_impl.c new file mode 100644 index 0000000..1374c8d --- /dev/null +++ b/wasm-micro-runtime/samples/basic/src/native_impl.c @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" +#include "wasm_export.h" +#include "math.h" + +// The first parameter is not exec_env because it is invoked by native funtions +void +reverse(char *str, int len) +{ + int i = 0, j = len - 1, temp; + while (i < j) { + temp = str[i]; + str[i] = str[j]; + str[j] = temp; + i++; + j--; + } +} + +// The first parameter exec_env must be defined using type wasm_exec_env_t +// which is the calling convention for exporting native API by WAMR. +// +// Converts a given integer x to string str[]. +// digit is the number of digits required in the output. +// If digit is more than the number of digits in x, +// then 0s are added at the beginning. +int +intToStr(wasm_exec_env_t exec_env, int x, char *str, int str_len, int digit) +{ + int i = 0; + + printf("calling into native function: %s\n", __FUNCTION__); + + while (x) { + // native is responsible for checking the str_len overflow + if (i >= str_len) { + return -1; + } + str[i++] = (x % 10) + '0'; + x = x / 10; + } + + // If number of digits required is more, then + // add 0s at the beginning + while (i < digit) { + if (i >= str_len) { + return -1; + } + str[i++] = '0'; + } + + reverse(str, i); + + if (i >= str_len) + return -1; + str[i] = '\0'; + return i; +} + +int +get_pow(wasm_exec_env_t exec_env, int x, int y) +{ + printf("calling into native function: %s\n", __FUNCTION__); + return (int)pow(x, y); +} + +int32_t +calculate_native(wasm_exec_env_t exec_env, int32_t n, int32_t func1, + int32_t func2) +{ + printf("calling into native function: %s, n=%d, func1=%d, func2=%d\n", + __FUNCTION__, n, func1, func2); + + uint32_t argv[] = { n }; + if (!wasm_runtime_call_indirect(exec_env, func1, 1, argv)) { + printf("call func1 failed\n"); + return 0xDEAD; + } + + uint32_t n1 = argv[0]; + printf("call func1 and return n1=%d\n", n1); + + if (!wasm_runtime_call_indirect(exec_env, func2, 1, argv)) { + printf("call func2 failed\n"); + return 0xDEAD; + } + + uint32_t n2 = argv[0]; + printf("call func2 and return n2=%d\n", n2); + return n1 + n2; +} diff --git a/wasm-micro-runtime/samples/basic/wasm-apps/testapp.c b/wasm-micro-runtime/samples/basic/wasm-apps/testapp.c new file mode 100644 index 0000000..8db2930 --- /dev/null +++ b/wasm-micro-runtime/samples/basic/wasm-apps/testapp.c @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +int +intToStr(int x, char *str, int str_len, int digit); +int +get_pow(int x, int y); +int32_t +calculate_native(int32_t n, int32_t func1, int32_t func2); + +// +// Primitive parameters functions +// +float +generate_float(int iteration, double seed1, float seed2) +{ + float ret; + + printf("calling into WASM function: %s\n", __FUNCTION__); + + for (int i = 0; i < iteration; i++) { + ret += 1.0f / seed1 + seed2; + } + + return ret; +} + +// Converts a floating-point/double number to a string. +// intToStr() is implemented outside wasm app +void +float_to_string(float n, char *res, int res_size, int afterpoint) +{ + + printf("calling into WASM function: %s\n", __FUNCTION__); + + // Extract integer part + int ipart = (int)n; + + // Extract floating part + float fpart = n - (float)ipart; + + // convert integer part to string + int i = intToStr(ipart, res, res_size, 0); + + // check for display option after point + if (afterpoint != 0) { + res[i] = '.'; // add dot + + // Get the value of fraction part upto given no. + // of points after dot. The third parameter + // is needed to handle cases like 233.007 + fpart = fpart * get_pow(10, afterpoint); + + intToStr((int)fpart, res + i + 1, res_size - i - 1, afterpoint); + } +} + +int32_t +mul7(int32_t n) +{ + printf("calling into WASM function: %s,", __FUNCTION__); + n = n * 7; + printf(" %s return %d \n", __FUNCTION__, n); + return n; +} + +int32_t +mul5(int32_t n) +{ + printf("calling into WASM function: %s,", __FUNCTION__); + n = n * 5; + printf(" %s return %d \n", __FUNCTION__, n); + return n; +} + +int32_t +calculate(int32_t n) +{ + printf("calling into WASM function: %s\n", __FUNCTION__); + int32_t (*f1)(int32_t) = &mul5; + int32_t (*f2)(int32_t) = &mul7; + return calculate_native(n, (uint32_t)f1, (uint32_t)f2); +} diff --git a/wasm-micro-runtime/samples/bh-atomic/CMakeLists.txt b/wasm-micro-runtime/samples/bh-atomic/CMakeLists.txt new file mode 100644 index 0000000..f690527 --- /dev/null +++ b/wasm-micro-runtime/samples/bh-atomic/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (C) 2023 Midokura Japan KK. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.0) +project(bh_atomic) + +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if(APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif() + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_LIBC_BUILTIN 0) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../..) +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_executable(bh_atomic main.c) + +target_link_libraries(bh_atomic) diff --git a/wasm-micro-runtime/samples/bh-atomic/main.c b/wasm-micro-runtime/samples/bh-atomic/main.c new file mode 100644 index 0000000..61c5280 --- /dev/null +++ b/wasm-micro-runtime/samples/bh-atomic/main.c @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include "bh_platform.h" +#include "bh_atomic.h" + +int +main(int argc, char **argv) +{ + bh_atomic_32_t v; + uint32 o; + + v = 0x00ff00ff; + o = BH_ATOMIC_32_LOAD(v); + assert(o == 0x00ff00ff); + + v = 0x00ff00ff; + o = BH_ATOMIC_32_FETCH_OR(v, 0xffff0000); + assert(o == 0x00ff00ff); + assert(v == 0xffff00ff); + + v = 0x00ff00ff; + o = BH_ATOMIC_32_FETCH_AND(v, 0xffff0000); + assert(o == 0x00ff00ff); + assert(v == 0x00ff0000); + + v = 0x00ff00ff; + o = BH_ATOMIC_32_FETCH_ADD(v, 0x10101); + assert(o == 0x00ff00ff); + assert(v == 0x00ff00ff + 0x10101); + + v = 0x00ff00ff; + o = BH_ATOMIC_32_FETCH_SUB(v, 0x10101); + assert(o == 0x00ff00ff); + assert(v == 0x00ff00ff - 0x10101); + + return 0; +} diff --git a/wasm-micro-runtime/samples/debug-tools/CMakeLists.txt b/wasm-micro-runtime/samples/debug-tools/CMakeLists.txt new file mode 100644 index 0000000..ce06029 --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/CMakeLists.txt @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +include(CheckPIESupported) + +project(debug_tools_sample) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +find_package(WASISDK REQUIRED) + +option(SOURCE_MAP_DEMO "Enable source map demo" OFF) +if (SOURCE_MAP_DEMO) + find_package(EMSCRIPTEN 3.1.50 REQUIRED) +endif () + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Resetdefault linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_LIBC_WASI 1) +set(WAMR_BUILD_FAST_INTERP 0) # Otherwise addresses don't match llvm-dwarfdump (addr2line) +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_DUMP_CALL_STACK 1) # Otherwise stack trace is not printed (addr2line) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ wasm application ################ +include(ExternalProject) + +# wasm32-wasi +ExternalProject_Add(wasm33-wasi + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps -B build + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} +) + +if (EMSCRIPTEN_FOUND) + # wasm32-emscripten + ExternalProject_Add(wasm32-emscripten + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps -B build + -DCMAKE_TOOLCHAIN_FILE=${EMSCRIPTEN_TOOLCHAIN} + -DCMAKE_VERBOSE_MAKEFILE=On + -DSOURCE_MAP_DEMO=On + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR}/emscripten + ) +endif () + +################ wamr runtime ################ +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +set (RUNTIME_SOURCE_ALL + ${CMAKE_CURRENT_LIST_DIR}/../../product-mini/platforms/linux/main.c + ${UNCOMMON_SHARED_SOURCE} +) +add_executable (iwasm ${RUNTIME_SOURCE_ALL}) +check_pie_supported() +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(iwasm vmlib -lm -ldl) diff --git a/wasm-micro-runtime/samples/debug-tools/README.md b/wasm-micro-runtime/samples/debug-tools/README.md new file mode 100644 index 0000000..b0358b9 --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/README.md @@ -0,0 +1,133 @@ +# "debug-tools" sample introduction + +Tool to symoblicate stack traces. When using wasm in production, debug info are usually stripped using tools like `wasm-opt`, to decrease the binary size. If a corresponding unstripped wasm file is kept, location information (function, file, line, column) can be retrieved from the stripped stack trace. + +## Build and run the sample + +### Generate the stack trace + +Build `iwasm` with `WAMR_BUILD_DUMP_CALL_STACK=1` and `WAMR_BUILD_FAST_INTERP=0` and the wasm file with debug info (e.g. `clang -g`). As it is done in [CMakeLists.txt](./CMakeLists.txt) and [wasm-apps/CMakeLists.txt](./wasm-apps/CMakeLists.txt) (look for `addr2line`): + +```bash +$ mkdir build && cd build +$ cmake .. +$ make +$ ./iwasm wasm-apps/trap.wasm +``` + +The output should be something like + +```text +#00: 0x0159 - $f5 +#01: 0x01b2 - $f6 +#02: 0x0200 - $f7 +#03: 0x026b - $f8 +#04: 0x236b - $f15 +#05: 0x011f - _start + +Exception: unreachable +``` + +Copy the stack trace printed to stdout into a separate file (`call_stack.txt`): + +```bash +$ ./iwasm wasm-apps/trap.wasm | grep "#" > call_stack.txt +``` + +Same for AOT. The AOT binary has to be generated using the `--enable-dump-call-stack` option of `wamrc`, as in [CMakeLists.txt](./wasm-apps/CMakeLists.txt). Then run: + +```bash +$ ./iwasm wasm-apps/trap.aot | grep "#" > call_stack.txt +``` + +### Symbolicate the stack trace + +Run the [addr2line](../../test-tools/addr2line/addr2line.py) script to symbolicate the stack trace: + +```bash +$ python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file wasm-apps/trap.wasm \ + call_stack.txt +``` + +The output should be something like: + +```text +0: c + at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:5:1 +1: b + at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:11:12 +2: a + at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:17:12 +3: main + at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:24:5 +4: __main_void + at unknown:?:? +5: _start +``` + +If WAMR is run in fast interpreter mode (`WAMR_BUILD_FAST_INTERP=1`), addresses in the stack trace cannot be tracked back to location info. +If WAMR <= `1.3.2` is used, the stack trace does not contain addresses. +In those two cases, run the script with `--no-addr`: the line info returned refers to the start of the function + +```bash +$ python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file wasm-apps/trap.wasm \ + call_stack.txt --no-addr +``` + +#### sourcemap + +This script also supports _sourcemap_ which is produced by [_emscripten_](https://emscripten.org/docs/tools_reference/emcc.html). The _sourcemap_ is used to map the wasm function to the original source file. To use it, add `-gsource-map` option to _emcc_ command line. The output should be a section named "sourceMappingURL" and a separated file named "_.map_. + +If the wasm file is with _sourcemap_, the script will use it to get the source file and line info. It needs an extra command line option `--emsdk` to specify the path of _emsdk_. The script will use _emsymbolizer_ to query the source file and line info. + +````bash +$ python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file emscripten/wasm-apps/trap.wasm \ + --emsdk /opt/emsdk \ + call_stack.from_wasm_w_sourcemap.txt + +The output should be something like: + +```text +1: c + at ../../../../../wasm-apps/trap.c:5:1 +2: b + at ../../../../../wasm-apps/trap.c:11:12 +3: a + at ../../../../../wasm-apps/trap.c:17:12 +4: main + at ../../../../../wasm-apps/trap.c:24:5 +5: __main_void + at ../../../../../../../../../emsdk/emscripten/system/lib/standalone/__main_void.c:53:10 +6: _start + at ../../../../../../../../../emsdk/emscripten/system/lib/libc/crt1.c:27:3 +```` + +> The script assume the separated map file _.map_ is in the same directory as the wasm file. + +### Another approach + +If the wasm file is with "name" section, it is able to output function name in the stack trace. To achieve that, need to enable `WAMR_BUILD_LOAD_CUSTOM_SECTION` and `WAMR_BUILD_CUSTOM_NAME_SECTION`. If using .aot file, need to add `--emit-custom-sections=name` into wamrc command line options. + +Then the output should be something like + +```text +#00: 0x0159 - c +#01: 0x01b2 - b +#02: 0x0200 - a +#03: 0x026b - main +#04: 0x236b - __main_void +#05: 0x011f - _start + +Exception: unreachable +``` + +Also, it is able to use _addr2line.py_ to add file and line info to the stack trace. diff --git a/wasm-micro-runtime/samples/debug-tools/cmake/FindEMSCRIPTEN.cmake b/wasm-micro-runtime/samples/debug-tools/cmake/FindEMSCRIPTEN.cmake new file mode 100644 index 0000000..8f63ec5 --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/cmake/FindEMSCRIPTEN.cmake @@ -0,0 +1,45 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +find_path(EMSCRIPTEN_HOME + NAMES upstream/emscripten + PATHS /opt/emsdk + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_file(EMSCRIPTEN_VERSION_FILE + NAMES emscripten-version.txt + PATHS ${EMSCRIPTEN_HOME}/upstream/emscripten + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +file(READ ${EMSCRIPTEN_VERSION_FILE} EMSCRIPTEN_VERSION_FILE_CONTENT) + +string(REGEX + MATCH + "[0-9]+\.[0-9]+(\.[0-9]+)*" + EMSCRIPTEN_VERSION + ${EMSCRIPTEN_VERSION_FILE_CONTENT} +) + +find_package_handle_standard_args(EMSCRIPTEN + REQUIRED_VARS EMSCRIPTEN_HOME + VERSION_VAR EMSCRIPTEN_VERSION + HANDLE_VERSION_RANGE +) + +if(EMSCRIPTEN_FOUND) + set(EMSCRIPTEN_TOOLCHAIN ${EMSCRIPTEN_HOME}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake) + set(EMCC ${EMSCRIPTEN_HOME}/upstream/emscripten/emcc) +endif() +mark_as_advanced(EMSCRIPTEN_TOOLCHAIN EMCC) diff --git a/wasm-micro-runtime/samples/debug-tools/cmake/FindWAMRC.cmake b/wasm-micro-runtime/samples/debug-tools/cmake/FindWAMRC.cmake new file mode 100644 index 0000000..20f9416 --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/cmake/FindWAMRC.cmake @@ -0,0 +1,27 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +find_path(WAMRC_HOME + wamr-compiler + PATHS ${CMAKE_CURRENT_SOURCE_DIR}/../../.. + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_file(WAMRC_BIN + wamrc + HINTS ${WAMRC_HOME}/wamr-compiler/build + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_package_handle_standard_args(WAMRC REQUIRED_VARS WAMRC_BIN) +mark_as_advanced(WAMRC_BIN) diff --git a/wasm-micro-runtime/samples/debug-tools/cmake/FindWASISDK.cmake b/wasm-micro-runtime/samples/debug-tools/cmake/FindWASISDK.cmake new file mode 100644 index 0000000..0caf374 --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/cmake/FindWASISDK.cmake @@ -0,0 +1,24 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*") +find_path(WASISDK_HOME + NAMES share/wasi-sysroot + PATHS ${WASISDK_SEARCH_PATH} + NO_DEFAULT_PATH + REQUIRED +) + +string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) + +find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) + +if(WASISDK_FOUND) + set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang) + set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++) + set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake) + set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot) +endif() +mark_as_advanced(WASISDK_CC_COMMAND WASISDK_CXX_COMMAND WASISDK_TOOLCHAIN WASISDK_SYSROOT WASISDK_HOME) diff --git a/wasm-micro-runtime/samples/debug-tools/symbolicate.sh b/wasm-micro-runtime/samples/debug-tools/symbolicate.sh new file mode 100644 index 0000000..709622f --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/symbolicate.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euox pipefail + +# Symbolicate .wasm +python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file wasm-apps/trap.wasm \ + call_stack.txt + +# Symbolicate .wasm with `--no-addr` +python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file wasm-apps/trap.wasm \ + call_stack.txt --no-addr + +# Symbolicate .aot +python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file wasm-apps/trap.wasm \ + call_stack_aot.txt \ No newline at end of file diff --git a/wasm-micro-runtime/samples/debug-tools/wasm-apps/CMakeLists.txt b/wasm-micro-runtime/samples/debug-tools/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000..527b5f3 --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/wasm-apps/CMakeLists.txt @@ -0,0 +1,58 @@ +# Copyright (C) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project (debut_tools_wasm) + +set (CMAKE_BUILD_TYPE Debug) # Otherwise no debug symbols (addr2line) + +list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake) +find_package (WAMRC REQUIRED) + +option(SOURCE_MAP_DEMO "Enable source map demo" OFF) +if (SOURCE_MAP_DEMO) + find_package(EMSCRIPTEN 3.1.50 REQUIRED) +endif () + +################ wasm and aot compilation ################ +function (compile_sample SOURCE_FILE) + get_filename_component (FILE_NAME ${SOURCE_FILE} NAME_WLE) + + ## wasm + set (WASM_FILE ${FILE_NAME}.wasm) + add_executable (${FILE_NAME} ${SOURCE_FILE}) + set_target_properties (${FILE_NAME} PROPERTIES SUFFIX .wasm) + + ## aot + set (AOT_FILE ${FILE_NAME}.aot) + add_custom_target ( + ${FILE_NAME}_aot + ALL + DEPENDS ${WAMRC_BIN} ${WASM_FILE} + # Use --enable-dump-call-stack to generate stack trace (addr2line) + COMMAND ${WAMRC_BIN} --size-level=0 --enable-dump-call-stack -o ${AOT_FILE} ${WASM_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + ## wasm + sourcemap + if (DEFINED EMSCRIPTEN) + add_custom_target( + ${FILE_NAME}_w_sourcemap + ALL + DEPENDS ${SOURCE_FILE} + COMMAND ${EMCC} -O0 -gsource-map -o ${FILE_NAME}.sourcemap.wasm ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + endif () + + ## install both + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${WASM_FILE} DESTINATION wasm-apps) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${AOT_FILE} DESTINATION wasm-apps) + if (DEFINED EMSCRIPTEN) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.sourcemap.wasm DESTINATION wasm-apps) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.sourcemap.wasm.map DESTINATION wasm-apps) + endif () +endfunction () + +compile_sample(trap.c) diff --git a/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c b/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c new file mode 100644 index 0000000..364c430 --- /dev/null +++ b/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c @@ -0,0 +1,27 @@ +int +c(int n) +{ + __builtin_trap(); +} + +int +b(int n) +{ + n += 3; + return c(n); +} + +int +a(int n) +{ + return b(n); +} + +int +main(int argc, char **argv) +{ + int i = 5; + a(i); + + return 0; +} diff --git a/wasm-micro-runtime/samples/file/CMakeLists.txt b/wasm-micro-runtime/samples/file/CMakeLists.txt new file mode 100644 index 0000000..c3a69cc --- /dev/null +++ b/wasm-micro-runtime/samples/file/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright (C) 2022 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.0) +project(file) + +################ wasm application ############### +add_subdirectory(src) +add_subdirectory(wasm-app) diff --git a/wasm-micro-runtime/samples/file/README.md b/wasm-micro-runtime/samples/file/README.md new file mode 100644 index 0000000..98f0cc3 --- /dev/null +++ b/wasm-micro-runtime/samples/file/README.md @@ -0,0 +1,112 @@ +# "file" sample introduction + +This sample demonstrates the supported file interaction API of WASI. +This sample can also demonstrate the SGX IPFS (Intel Protected File System), enabling an enclave to seal and unseal data at rest. + +## Preparation + +Please install WASI SDK, download the [wasi-sdk release](https://github.com/WebAssembly/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. +For testing with SGX IPFS, follow the instructions in [the documentation of SGX for WAMR](../../doc/linux_sgx.md#sgx-intel-protected-file-system). + +## Build the sample + +```bash +mkdir build +cd build +cmake .. +make +``` + +The WebAssembly application is the file located at `wasm-app/file.wasm`. + +## Run workload + +Either use [iwasm-sample](../../product-mini/platforms/linux/) for Linux, or [enclave-sample](../../product-mini/platforms/linux-sgx/enclave-sample/) for Intel SGX to run the sample, with the argument to allow the file system interaction with the current folder (`--dir=.`). + +The output with Linux and POSIX is like: + +```bash +Opening a file.. +[Test] File opening passed. +Writing to the file.. +[Test] File writing passed. +Moving the cursor to the start of the file.. +Reading from the file, up to 1000 characters.. +Text read: Hello, world! +[Test] File reading passed. +Determine whether we reach the end of the file.. +Is the end of file? 1 +[Test] End of file detection passed. +Getting the plaintext size.. +The plaintext size is 13. +[Test] Retrieving file offset passed. +Force actual write of all the cached data to the disk.. +[Test] Retrieving file offset passed. +Writing 5 characters at offset 7.. +File current offset: 13 +[Test] Writing at specified offset passed. +Reading 5 characters at offset 7.. +Text read: James +File current offset: 13 +[Test] Reading at specified offset passed. +Allocate more space to the file.. +File current offset: 13 +Moving to the end.. +File current offset: 23 +[Test] Allocation or more space passed. +Extend the file size of 10 bytes using ftruncate.. +File current offset: 23 +Moving to the end.. +File current offset: 33 +[Test] Extension of the file size passed. +Closing from the file.. +[Test] Closing file passed. +Getting the size of the file on disk.. +The file size is 33. +All the tests passed! +``` + +The output with SGX and IPFS is like: + +```bash +Opening a file.. +[Test] File opening passed. +Writing to the file.. +[Test] File writing passed. +Moving the cursor to the start of the file.. +Reading from the file, up to 1000 characters.. +Text read: Hello, world! +[Test] File reading passed. +Determine whether we reach the end of the file.. +Is the end of file? 1 +[Test] End of file detection passed. +Getting the plaintext size.. +The plaintext size is 13. +[Test] Retrieving file offset passed. +Force actual write of all the cached data to the disk.. +[Test] Retrieving file offset passed. +Writing 5 characters at offset 7.. +File current offset: 13 +[Test] Writing at specified offset passed. +Reading 5 characters at offset 7.. +Text read: James +File current offset: 13 +[Test] Reading at specified offset passed. +Allocate more space to the file.. +File current offset: 23 +Moving to the end.. +File current offset: 23 +[Test] Allocation or more space passed. +Extend the file size of 10 bytes using ftruncate.. +File current offset: 23 +Moving to the end.. +File current offset: 33 +[Test] Extension of the file size passed. +Closing from the file.. +[Test] Closing file passed. +Getting the size of the file on disk.. +The file size is 4096. +All the tests passed! +``` + +For SGX IPFS, refer to [SGX Intel Protected File System](../../doc/linux_sgx.md#sgx-intel-protected-file-system) for more details. diff --git a/wasm-micro-runtime/samples/file/src/CMakeLists.txt b/wasm-micro-runtime/samples/file/src/CMakeLists.txt new file mode 100644 index 0000000..19775a0 --- /dev/null +++ b/wasm-micro-runtime/samples/file/src/CMakeLists.txt @@ -0,0 +1,91 @@ +# Copyright (C) 2022 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (iwasm) +else() + project (iwasm C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_REF_TYPES 1) + +if (NOT MSVC) + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (iwasm main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (iwasm vmlib -lm -ldl -lpthread) +else () + target_link_libraries (iwasm vmlib -lm -ldl -lpthread -lrt) +endif () diff --git a/wasm-micro-runtime/samples/file/src/main.c b/wasm-micro-runtime/samples/file/src/main.c new file mode 100644 index 0000000..3675ee0 --- /dev/null +++ b/wasm-micro-runtime/samples/file/src/main.c @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2022 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" + +void +print_usage(void) +{ + fprintf(stdout, "Required arguments:\r\n"); + fprintf(stdout, " -f [path of wasm file] \n"); + fprintf(stdout, " -d [path of host directory] \n"); +} + +int +main(int argc, char *argv_main[]) +{ + static char global_heap_buf[512 * 1024]; + char *buffer, error_buf[128]; + const char *wasm_path = NULL, *wasi_dir = NULL; + int opt, main_result = 1; + + wasm_module_t module = NULL; + wasm_module_inst_t module_inst = NULL; + wasm_exec_env_t exec_env = NULL; + uint32 buf_size, stack_size = 8092, heap_size = 8092; + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + while ((opt = getopt(argc, argv_main, "hf:d:")) != -1) { + switch (opt) { + case 'f': + wasm_path = optarg; + break; + case 'd': + wasi_dir = optarg; + break; + case 'h': + print_usage(); + return 0; + case '?': + print_usage(); + return 0; + } + } + if (wasm_path == NULL || wasi_dir == NULL) { + print_usage(); + return 0; + } + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + buffer = bh_read_file_to_buffer(wasm_path, &buf_size); + + if (!buffer) { + printf("Open wasm app file [%s] failed.\n", wasm_path); + goto fail; + } + + module = wasm_runtime_load(buffer, buf_size, error_buf, sizeof(error_buf)); + if (!module) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + wasm_runtime_set_wasi_args_ex(module, &wasi_dir, 1, NULL, 0, NULL, 0, NULL, + 0, 0, 1, 2); + + module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + + if (!module_inst) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + exec_env = wasm_runtime_create_exec_env(module_inst, stack_size); + if (!exec_env) { + printf("Create wasm execution environment failed.\n"); + goto fail; + } + + if (wasm_application_execute_main(module_inst, 0, NULL)) { + main_result = wasm_runtime_get_wasi_exit_code(module_inst); + } + else { + printf("call wasm function main failed. error: %s\n", + wasm_runtime_get_exception(module_inst)); + goto fail; + } + +fail: + if (exec_env) + wasm_runtime_destroy_exec_env(exec_env); + if (module_inst) + wasm_runtime_deinstantiate(module_inst); + if (module) + wasm_runtime_unload(module); + if (buffer) + BH_FREE(buffer); + wasm_runtime_destroy(); + return main_result; +} diff --git a/wasm-micro-runtime/samples/file/wasm-app/CMakeLists.txt b/wasm-micro-runtime/samples/file/wasm-app/CMakeLists.txt new file mode 100644 index 0000000..4af87a3 --- /dev/null +++ b/wasm-micro-runtime/samples/file/wasm-app/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (C) 2022 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.0) +project(wasm-app) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () + +set (CMAKE_SYSTEM_PROCESSOR wasm32) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_COMPILER_TARGET "wasm32-wasi") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -Wno-unused-command-line-argument") + +add_executable(file.wasm main.c) +target_link_libraries(file.wasm) diff --git a/wasm-micro-runtime/samples/file/wasm-app/main.c b/wasm-micro-runtime/samples/file/wasm-app/main.c new file mode 100644 index 0000000..6e6204c --- /dev/null +++ b/wasm-micro-runtime/samples/file/wasm-app/main.c @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define PATH_TEST_FOLDER "./test" +#define PATH_TEST_FILE (PATH_TEST_FOLDER "/test.txt") +#define FILE_TEXT "Hello, world!" +#define WORLD_OFFSET 7 +#define NAME_REPLACMENT "James" +#define NAME_REPLACMENT_LEN (sizeof(NAME_REPLACMENT) - 1) +#define ADDITIONAL_SPACE 1 * 1024 * 1024 + +int +main(int argc, char **argv) +{ + FILE *file; + const char *text = FILE_TEXT; + char buffer[1000]; + int ret; + long long stat_size; + + // Test: Create a folder to store the file, if it does not exist yet + ret = mkdir(PATH_TEST_FOLDER, 777); + assert(ret == 0 || (ret == -1 && errno == EEXIST)); + + // Test: File opening (fopen) + printf("Opening a file..\n"); + file = fopen(PATH_TEST_FILE, "w+"); + if (file == NULL) { + printf("Error! errno: %d\n", errno); + } + assert(file != NULL); + printf("[Test] File opening passed.\n"); + + // Test: Writing to a file (fprintf) + printf("Writing to the file..\n"); + ret = fprintf(file, "%s", text); + assert(ret == strlen(text)); + printf("[Test] File writing passed.\n"); + + // Test: Reading from a file (fseek) + printf("Moving the cursor to the start of the file..\n"); + ret = fseek(file, 0, SEEK_SET); + assert(ret == 0); + + printf("Reading from the file, up to 1000 characters..\n"); + fread(buffer, 1, sizeof(buffer), file); + printf("Text read: %s\n", buffer); + assert(strncmp(text, buffer, strlen(text)) == 0); + printf("[Test] File reading passed.\n"); + + // Test: end of file detection (feof) + printf("Determine whether we reach the end of the file..\n"); + int is_end_of_file = feof(file); + printf("Is the end of file? %d\n", is_end_of_file); + assert(is_end_of_file == 1); + printf("[Test] End of file detection passed.\n"); + + // Test: retrieving file offset (ftell) + printf("Getting the plaintext size..\n"); + long plaintext_size = ftell(file); + printf("The plaintext size is %ld.\n", plaintext_size); + assert(plaintext_size == 13); + printf("[Test] Retrieving file offset passed.\n"); + + // Test: persist changes on disk (fflush) + printf("Force actual write of all the cached data to the disk..\n"); + ret = fflush(file); + assert(ret == 0); + printf("[Test] Retrieving file offset passed.\n"); + + // Test: writing at specified offset (pwrite) + printf("Writing 5 characters at offset %d..\n", WORLD_OFFSET); + ret = pwrite(fileno(file), NAME_REPLACMENT, NAME_REPLACMENT_LEN, + WORLD_OFFSET); + printf("File current offset: %ld\n", ftell(file)); + assert(ret == NAME_REPLACMENT_LEN); + assert(ftell(file) == strlen(FILE_TEXT)); + printf("[Test] Writing at specified offset passed.\n"); + + // Test: reading at specified offset (pread) + printf("Reading %ld characters at offset %d..\n", NAME_REPLACMENT_LEN, + WORLD_OFFSET); + buffer[NAME_REPLACMENT_LEN] = '\0'; + pread(fileno(file), buffer, NAME_REPLACMENT_LEN, WORLD_OFFSET); + printf("Text read: %s\n", buffer); + printf("File current offset: %ld\n", ftell(file)); + assert(strcmp(NAME_REPLACMENT, buffer) == 0); + assert(ftell(file) == strlen(FILE_TEXT)); + printf("[Test] Reading at specified offset passed.\n"); + + // Test: moving at the start of the file (fseek) + printf("Move at the start of the file (fseek)..\n"); + printf("File current offset: %ld\n", ftell(file)); + fseek(file, 0, SEEK_SET); + printf("File current offset: %ld\n", ftell(file)); + assert(ftell(file) == 0); + + // Test: moving at the end of the file (fseek) + printf("Move at the end of the file (fseek)..\n"); + printf("File current offset: %ld\n", ftell(file)); + fseek(file, 0, SEEK_END); + printf("File current offset: %ld\n", ftell(file)); + assert(ftell(file) == strlen(FILE_TEXT)); + int end_position = ftell(file) / 2; + + // Test: moving at the middle of the file (fseek) + printf("Move at the middle of the file (fseek)..\n"); + printf("File current offset: %ld\n", ftell(file)); + fseek(file, 0, SEEK_SET); + fseek(file, end_position, SEEK_CUR); + printf("File current offset: %ld\n", ftell(file)); + assert(ftell(file) == end_position); + + // Test: allocate more space to the file (posix_fallocate) + printf("Allocate more space to the file (posix_fallocate)..\n"); + fseek(file, 0, SEEK_END); + posix_fallocate(fileno(file), ftell(file), ADDITIONAL_SPACE); + printf("File current offset: %ld\n", ftell(file)); + printf("Moving to the end..\n"); + fseek(file, 0, SEEK_END); + printf("File current offset: %ld\n", ftell(file)); + assert(ftell(file) == strlen(text) + ADDITIONAL_SPACE); + printf("[Test] Allocation or more space passed.\n"); + + // Test: allocate more space to the file (ftruncate) + printf("Allocate more space to the file (ftruncate)..\n"); + ftruncate(fileno(file), ftell(file) + ADDITIONAL_SPACE); + assert(ftell(file) == strlen(text) + ADDITIONAL_SPACE); + printf("File current offset: %ld\n", ftell(file)); + printf("Moving to the end..\n"); + fseek(file, 0, SEEK_END); + printf("File current offset: %ld\n", ftell(file)); + assert(ftell(file) == strlen(text) + 2 * ADDITIONAL_SPACE); + printf("[Test] Extension of the file size passed.\n"); + + // Test: allocate more space to the file (fseek, from the start) + printf("Allocate more space to the file (fseek) from the start..\n"); + printf("File current offset: %ld\n", ftell(file)); + fseek(file, 3 * ADDITIONAL_SPACE, SEEK_SET); + printf("File current offset: %ld\n", ftell(file)); + assert(ftell(file) == 3 * ADDITIONAL_SPACE); + printf("[Test] Extension of the file size passed.\n"); + + // Test: allocate more space to the file (fseek, from the middle) + printf("Allocate more space to the file (fseek) from the middle..\n"); + fseek(file, 3 * ADDITIONAL_SPACE, SEEK_SET); + printf("File current offset: %ld\n", ftell(file)); + fseek(file, 2 * ADDITIONAL_SPACE, SEEK_CUR); + printf("File current offset: %ld\n", ftell(file)); + assert(ftell(file) == 5 * ADDITIONAL_SPACE); + printf("[Test] Extension of the file size passed.\n"); + + // Display some debug information + printf("Getting the size of the file on disk..\n"); + struct stat st; + stat(PATH_TEST_FILE, &st); + stat_size = st.st_size; + assert(stat_size != 0); + + // Compare with the size from fstat + fstat(fileno(file), &st); + printf("The file size is: %lld (stat), %lld (fstat).\n", stat_size, + st.st_size); + assert(stat_size != 0); + assert(stat_size == st.st_size); + + // Test: closing the file (fclose) + printf("Closing from the file..\n"); + ret = fclose(file); + assert(ret == 0); + printf("[Test] Closing file passed.\n"); + + printf("All the tests passed!\n"); + + return 0; +} diff --git a/wasm-micro-runtime/samples/gui/wasm-apps/decrease/src/main.c b/wasm-micro-runtime/samples/gui/wasm-apps/decrease/src/main.c new file mode 100644 index 0000000..9635ebe --- /dev/null +++ b/wasm-micro-runtime/samples/gui/wasm-apps/decrease/src/main.c @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include "wasm_app.h" +#include "wa-inc/lvgl/lvgl.h" +#include "wa-inc/timer_wasm_app.h" + +extern char g_widget_text[]; + +static void +btn_event_cb(lv_obj_t *btn, lv_event_t event); + +uint32_t count = 0; +char count_str[11] = { 0 }; +lv_obj_t *hello_world_label; +lv_obj_t *count_label; +lv_obj_t *btn1; +lv_obj_t *label_count1; +int label_count1_value = 100; +char label_count1_str[11] = { 0 }; + +void +timer1_update(user_timer_t timer1) +{ + if ((count % 100) == 0) { + snprintf(count_str, sizeof(count_str), "%d", count / 100); + lv_label_set_text(count_label, count_str); + } + ++count; +} + +void +on_init() +{ + char *text; + + hello_world_label = lv_label_create(NULL, NULL); + lv_label_set_text(hello_world_label, "Hello world!"); + text = lv_label_get_text(hello_world_label); + printf("Label text %lu %s \n", strlen(text), text); + lv_obj_align(hello_world_label, NULL, LV_ALIGN_IN_TOP_LEFT, 0, 0); + + count_label = lv_label_create(NULL, NULL); + lv_obj_align(count_label, NULL, LV_ALIGN_IN_TOP_MID, 0, 0); + + /* Create a button on the currently loaded screen */ + btn1 = lv_btn_create(NULL, NULL); + /* Set function to be called when the button is released */ + lv_obj_set_event_cb(btn1, (lv_event_cb_t)btn_event_cb); + /* Align below the label */ + lv_obj_align(btn1, NULL, LV_ALIGN_CENTER, 0, 0); + + /* Create a label on the button */ + lv_obj_t *btn_label = lv_label_create(btn1, NULL); + lv_label_set_text(btn_label, "Click --"); + + label_count1 = lv_label_create(NULL, NULL); + lv_label_set_text(label_count1, "100"); + lv_obj_align(label_count1, NULL, LV_ALIGN_IN_BOTTOM_MID, 0, 0); + + /* Set up a timer */ + user_timer_t timer; + timer = api_timer_create(10, true, false, timer1_update); + if (timer) + api_timer_restart(timer, 10); + else + printf("Fail to create timer.\n"); +} + +static void +btn_event_cb(lv_obj_t *btn, lv_event_t event) +{ + if (event == LV_EVENT_RELEASED) { + label_count1_value--; + snprintf(label_count1_str, sizeof(label_count1_str), "%d", + label_count1_value); + lv_label_set_text(label_count1, label_count1_str); + if (label_count1_value == 0) + label_count1_value = 100; + } +} diff --git a/wasm-micro-runtime/samples/gui/wasm-apps/increase/src/main.c b/wasm-micro-runtime/samples/gui/wasm-apps/increase/src/main.c new file mode 100644 index 0000000..31118f7 --- /dev/null +++ b/wasm-micro-runtime/samples/gui/wasm-apps/increase/src/main.c @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include "wasm_app.h" +#include "wa-inc/lvgl/lvgl.h" +#include "wa-inc/timer_wasm_app.h" + +extern char g_widget_text[]; + +static void +btn_event_cb(lv_obj_t *btn, lv_event_t event); + +uint32_t count = 0; +char count_str[11] = { 0 }; +lv_obj_t *hello_world_label; +lv_obj_t *count_label; +lv_obj_t *btn1; +lv_obj_t *label_count1; +int label_count1_value = 1; +char label_count1_str[11] = { 0 }; + +void +timer1_update(user_timer_t timer1) +{ + if ((count % 100) == 0) { + snprintf(count_str, sizeof(count_str), "%d", count / 100); + lv_label_set_text(count_label, count_str); + } + ++count; +} + +void +on_init() +{ + char *text; + + hello_world_label = lv_label_create(NULL, NULL); + lv_label_set_text(hello_world_label, "Hello world!"); + text = lv_label_get_text(hello_world_label); + printf("Label text %lu %s \n", strlen(text), text); + lv_obj_align(hello_world_label, NULL, LV_ALIGN_IN_TOP_LEFT, 0, 0); + + count_label = lv_label_create(NULL, NULL); + lv_obj_align(count_label, NULL, LV_ALIGN_IN_TOP_MID, 0, 0); + + /* Create a button on the current loaded screen */ + btn1 = lv_btn_create(NULL, NULL); + /* Set function to be called when the button is released */ + lv_obj_set_event_cb(btn1, (lv_event_cb_t)btn_event_cb); + /* Align below the label */ + lv_obj_align(btn1, NULL, LV_ALIGN_CENTER, 0, 0); + + /* Create a label on the button */ + lv_obj_t *btn_label = lv_label_create(btn1, NULL); + lv_label_set_text(btn_label, "Click ++"); + + label_count1 = lv_label_create(NULL, NULL); + lv_label_set_text(label_count1, "1"); + lv_obj_align(label_count1, NULL, LV_ALIGN_IN_BOTTOM_MID, 0, 0); + + /* Set up a timer */ + user_timer_t timer; + timer = api_timer_create(10, true, false, timer1_update); + if (timer) + api_timer_restart(timer, 10); + else + printf("Fail to create timer.\n"); +} + +static void +btn_event_cb(lv_obj_t *btn, lv_event_t event) +{ + if (event == LV_EVENT_RELEASED) { + label_count1_value++; + snprintf(label_count1_str, sizeof(label_count1_str), "%d", + label_count1_value); + lv_label_set_text(label_count1, label_count1_str); + if (label_count1_value == 100) + label_count1_value = 0; + } +} diff --git a/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c b/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c new file mode 100644 index 0000000..5502cfd --- /dev/null +++ b/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c @@ -0,0 +1,348 @@ +/** + * @file XPT2046.c + */ +/********************* + * INCLUDES + *********************/ +#include "XPT2046.h" +#include "board_config.h" +#include "stdio.h" +#include +#include "drivers/spi.h" + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#include +#else +#include +#endif + +#if USE_XPT2046 + +#include + +#define abs(x) ((x) < 0 ? -(x) : (x)) + +/********************* + * DEFINES + *********************/ + +/********************** + * TYPEDEFS + **********************/ + +/********************** + * STATIC PROTOTYPES + **********************/ +static void +xpt2046_corr(int16_t *x, int16_t *y); +#if 0 +static void xpt2046_avg(int16_t * x, int16_t * y); +#endif + +/********************** + * STATIC VARIABLES + **********************/ +int16_t avg_buf_x[XPT2046_AVG]; +int16_t avg_buf_y[XPT2046_AVG]; +uint8_t avg_last; + +/********************** + * MACROS + **********************/ + +/********************** + * GLOBAL FUNCTIONS + **********************/ + +/** + * Initialize the XPT2046 + */ +struct device *input_dev; + +struct spi_config spi_conf_xpt2046; +struct spi_cs_control xpt2046_cs_ctrl; +struct device *xpt2046_pen_gpio_dev; +static struct gpio_callback gpio_cb; +lv_indev_data_t touch_point; +lv_indev_data_t last_touch_point; + +#define TOUCH_READ_THREAD_STACK_SIZE 4096 +static K_THREAD_STACK_DEFINE(touch_read_thread_stack, + TOUCH_READ_THREAD_STACK_SIZE); +static struct k_thread touch_thread_data; +static struct k_sem sem_touch_read; + +K_MUTEX_DEFINE(spi_display_touch_mutex); + +int cnt = 0; +int touch_read_times = 0; +int last_pen_interrupt_time = 0; +void +xpt2046_pen_gpio_callback(struct device *port, struct gpio_callback *cb, + u32_t pins) +{ + cnt++; + if ((k_uptime_get_32() - last_pen_interrupt_time) > 500) { + k_sem_give(&sem_touch_read); + touch_read_times++; + last_pen_interrupt_time = k_uptime_get_32(); + } +} + +void +disable_pen_interrupt() +{ + int ret = 0; + ret = gpio_disable_callback(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN); + if (ret != 0) { + printf("gpio_pin_configure GPIO_INPUT failed\n"); + } +} +void +enable_pen_interrupt() +{ + int ret = 0; + ret = gpio_enable_callback(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN); + if (ret != 0) { + printf("gpio_pin_configure failed\n"); + } +} + +void +touch_screen_read_thread() +{ + int i; + bool ret = false; + + for (;;) { + k_sem_take(&sem_touch_read, K_FOREVER); + memset(&last_touch_point, 0, sizeof(lv_indev_data_t)); + memset(&touch_point, 0, sizeof(lv_indev_data_t)); + memset(avg_buf_x, 0, sizeof(avg_buf_x)); + memset(avg_buf_y, 0, sizeof(avg_buf_y)); + k_mutex_lock(&spi_display_touch_mutex, K_FOREVER); + disable_pen_interrupt(); + for (i = 0; i < 100; i++) { + ret = xpt2046_read(&touch_point); + if (ret) { + if ((abs(last_touch_point.point.x - touch_point.point.x) < 4) + && (abs(last_touch_point.point.y - touch_point.point.y) + < 4)) { + break; + } + last_touch_point = touch_point; + } + } + enable_pen_interrupt(); + k_mutex_unlock(&spi_display_touch_mutex); + } +} + +void +xpt2046_init(void) +{ + int ret; + input_dev = device_get_binding(XPT2046_SPI_DEVICE_NAME); + + if (input_dev == NULL) { + printf("device not found. Aborting test."); + return; + } + memset((void *)&touch_point, 0, sizeof(lv_indev_data_t)); + + spi_conf_xpt2046.frequency = XPT2046_SPI_MAX_FREQUENCY; + spi_conf_xpt2046.operation = SPI_OP_MODE_MASTER | SPI_WORD_SET(8); + spi_conf_xpt2046.slave = 0; + spi_conf_xpt2046.cs = NULL; +#ifdef XPT2046_CS_GPIO_CONTROLLER + xpt2046_cs_ctrl.gpio_dev = device_get_binding(XPT2046_CS_GPIO_CONTROLLER); + if (xpt2046_cs_ctrl.gpio_dev == NULL) { + printk("Cannot find %s!\n", XPT2046_CS_GPIO_CONTROLLER); + return; + } + gpio_pin_configure(xpt2046_cs_ctrl.gpio_dev, XPT2046_CS_GPIO_PIN, + GPIO_OUTPUT); + gpio_pin_set(xpt2046_cs_ctrl.gpio_dev, XPT2046_CS_GPIO_PIN, 1); + xpt2046_cs_ctrl.gpio_pin = XPT2046_CS_GPIO_PIN; + xpt2046_cs_ctrl.delay = 0; + spi_conf_xpt2046.cs = &xpt2046_cs_ctrl; + +#endif + +#ifdef XPT2046_PEN_GPIO_CONTROLLER + + xpt2046_pen_gpio_dev = device_get_binding(XPT2046_PEN_GPIO_CONTROLLER); + if (!xpt2046_pen_gpio_dev) { + printk("Cannot find %s!\n", XPT2046_PEN_GPIO_CONTROLLER); + return; + } + /* Setup GPIO input */ + ret = gpio_pin_configure(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN, + (GPIO_INPUT | GPIO_INT_ENABLE | GPIO_INT_EDGE + | GPIO_INT_LOW_0 | GPIO_INT_DEBOUNCE)); + if (ret) { + printk("Error configuring pin %d!\n", XPT2046_PEN_GPIO_PIN); + } + + gpio_init_callback(&gpio_cb, xpt2046_pen_gpio_callback, + BIT(XPT2046_PEN_GPIO_PIN)); + + ret = gpio_add_callback(xpt2046_pen_gpio_dev, &gpio_cb); + if (ret) { + printk("gpio_add_callback error\n"); + } + ret = gpio_enable_callback(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN); + if (ret) { + printk("gpio_enable_callback error\n"); + } +#endif + + k_sem_init(&sem_touch_read, 0, 1); + + k_thread_create(&touch_thread_data, touch_read_thread_stack, + TOUCH_READ_THREAD_STACK_SIZE, touch_screen_read_thread, + NULL, NULL, NULL, 5, 0, K_NO_WAIT); + printf("xpt2046_init ok \n"); +} + +/** + * Get the current position and state of the touchpad + * @param data store the read data here + * @return false: because no ore data to be read + */ +bool +xpt2046_read(lv_indev_data_t *data) +{ + static int16_t last_x = 0; + static int16_t last_y = 0; + bool valid = true; + int s32_ret = 0; + + int16_t x = 0; + int16_t y = 0; + + char tx1[16] = { 0 }; + char rx1[16] = { 0 }; + + struct spi_buf tx_buf = { .buf = &tx1, .len = 3 }; + struct spi_buf_set tx_bufs = { .buffers = &tx_buf, .count = 1 }; + struct spi_buf rx_buf = { .buf = &rx1, .len = 3 }; + struct spi_buf_set rx_bufs = { .buffers = &rx_buf, .count = 1 }; + + tx1[0] = CMD_X_READ; + s32_ret = spi_transceive(input_dev, &spi_conf_xpt2046, &tx_bufs, &rx_bufs); + if (s32_ret != 0) { + printf("spi_transceive return failed:%d\n", s32_ret); + } + x = rx1[1] << 8; + x += rx1[2]; + + tx1[0] = CMD_Y_READ; + s32_ret = spi_transceive(input_dev, &spi_conf_xpt2046, &tx_bufs, &rx_bufs); + if (s32_ret != 0) { + printf("spi_transceive return failed:%d\n", s32_ret); + } + y = rx1[1] << 8; + y += rx1[2]; + x = x >> 3; + y = y >> 3; + + xpt2046_corr(&x, &y); + if (y <= 0 || (x > 320)) { + valid = false; + } + + last_x = x; + last_y = y; + + data->point.x = x; + data->point.y = y; + data->state = valid == false ? LV_INDEV_STATE_REL : LV_INDEV_STATE_PR; + + return valid; +} + +/********************** + * STATIC FUNCTIONS + **********************/ +static void +xpt2046_corr(int16_t *x, int16_t *y) +{ +#if XPT2046_XY_SWAP != 0 + int16_t swap_tmp; + swap_tmp = *x; + *x = *y; + *y = swap_tmp; +#endif + + if ((*x) > XPT2046_X_MIN) + (*x) -= XPT2046_X_MIN; + else + (*x) = 0; + + if ((*y) > XPT2046_Y_MIN) + (*y) -= XPT2046_Y_MIN; + else + (*y) = 0; + + (*x) = (uint32_t)((uint32_t)(*x) * XPT2046_HOR_RES) + / (XPT2046_X_MAX - XPT2046_X_MIN); + + (*y) = (uint32_t)((uint32_t)(*y) * XPT2046_VER_RES) + / (XPT2046_Y_MAX - XPT2046_Y_MIN); + +#if XPT2046_X_INV != 0 + (*x) = XPT2046_HOR_RES - (*x); +#endif + +#if XPT2046_Y_INV != 0 + (*y) = XPT2046_VER_RES - (*y); +#endif +} + +#if 0 +static void xpt2046_avg(int16_t * x, int16_t * y) +{ + /*Shift out the oldest data*/ + uint8_t i; + for (i = XPT2046_AVG - 1; i > 0; i--) { + avg_buf_x[i] = avg_buf_x[i - 1]; + avg_buf_y[i] = avg_buf_y[i - 1]; + } + + /*Insert the new point*/ + avg_buf_x[0] = *x; + avg_buf_y[0] = *y; + if (avg_last < XPT2046_AVG) + avg_last++; + + /*Sum the x and y coordinates*/ + int32_t x_sum = 0; + int32_t y_sum = 0; + for (i = 0; i < avg_last; i++) { + x_sum += avg_buf_x[i]; + y_sum += avg_buf_y[i]; + } + + /*Normalize the sums*/ + (*x) = (int32_t) x_sum / avg_last; + (*y) = (int32_t) y_sum / avg_last; +} +#endif + +bool +touchscreen_read(lv_indev_data_t *data) +{ + /*Store the collected data*/ + data->point.x = last_touch_point.point.x; + data->point.y = last_touch_point.point.y; + data->state = last_touch_point.state; + + if (last_touch_point.state == LV_INDEV_STATE_PR) { + last_touch_point.state = LV_INDEV_STATE_REL; + } + return false; +} + +#endif diff --git a/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/display_ili9340.h b/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/display_ili9340.h new file mode 100644 index 0000000..ddc396e --- /dev/null +++ b/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/display_ili9340.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 Jan Van Winkel + * + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ +#define ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ +#include "board_config.h" +#include + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#else +#include +#endif + +#define ILI9340_CMD_ENTER_SLEEP 0x10 +#define ILI9340_CMD_EXIT_SLEEP 0x11 +#define ILI9340_CMD_GAMMA_SET 0x26 +#define ILI9340_CMD_DISPLAY_OFF 0x28 +#define ILI9340_CMD_DISPLAY_ON 0x29 +#define ILI9340_CMD_COLUMN_ADDR 0x2a +#define ILI9340_CMD_PAGE_ADDR 0x2b +#define ILI9340_CMD_MEM_WRITE 0x2c +#define ILI9340_CMD_MEM_ACCESS_CTRL 0x36 +#define ILI9340_CMD_PIXEL_FORMAT_SET 0x3A +#define ILI9340_CMD_FRAME_CTRL_NORMAL_MODE 0xB1 +#define ILI9340_CMD_DISPLAY_FUNCTION_CTRL 0xB6 +#define ILI9340_CMD_POWER_CTRL_1 0xC0 +#define ILI9340_CMD_POWER_CTRL_2 0xC1 +#define ILI9340_CMD_VCOM_CTRL_1 0xC5 +#define ILI9340_CMD_VCOM_CTRL_2 0xC7 +#define ILI9340_CMD_POSITVE_GAMMA_CORRECTION 0xE0 +#define ILI9340_CMD_NEGATIVE_GAMMA_CORRECTION 0xE1 + +#define ILI9340_DATA_MEM_ACCESS_CTRL_MY 0x80 +#define ILI9340_DATA_MEM_ACCESS_CTRL_MX 0x40 +#define ILI9340_DATA_MEM_ACCESS_CTRL_MV 0x20 +#define ILI9340_DATA_MEM_ACCESS_CTRL_ML 0x10 +#define ILI9340_DATA_MEM_ACCESS_CTRL_BGR 0x08 +#define ILI9340_DATA_MEM_ACCESS_CTRL_MH 0x04 + +#define ILI9340_DATA_PIXEL_FORMAT_RGB_18_BIT 0x60 +#define ILI9340_DATA_PIXEL_FORMAT_RGB_16_BIT 0x50 +#define ILI9340_DATA_PIXEL_FORMAT_MCU_18_BIT 0x06 +#define ILI9340_DATA_PIXEL_FORMAT_MCU_16_BIT 0x05 + +struct ili9340_data; + +/** + * Send data to ILI9340 display controller + * + * @param data Device data structure + * @param cmd Command to send to display controller + * @param tx_data Data to transmit to the display controller + * In case no data should be transmitted pass a NULL pointer + * @param tx_len Number of bytes in tx_data buffer + * + */ +void +ili9340_transmit(struct ili9340_data *data, u8_t cmd, void *tx_data, + size_t tx_len); + +/** + * Perform LCD specific initialization + * + * @param data Device data structure + */ +void +ili9340_lcd_init(struct ili9340_data *data); + +#define DT_ILITEK_ILI9340_0_LABEL "DISPLAY" +#define CONFIG_DISPLAY_LOG_LEVEL 0 + +#endif /* ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ */ diff --git a/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c b/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c new file mode 100644 index 0000000..9ce0c11 --- /dev/null +++ b/wasm-micro-runtime/samples/gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include "bh_platform.h" +#include "runtime_lib.h" +#include "native_interface.h" +#include "app_manager_export.h" +#include "board_config.h" +#include "bh_common.h" +#include "bh_queue.h" +#include "runtime_sensor.h" +#include "bi-inc/attr_container.h" +#include "module_wasm_app.h" +#include "wasm_export.h" +#include "display.h" +#include "lvgl.h" + +extern bool +init_sensor_framework(); +extern void +exit_sensor_framework(); +extern int +aee_host_msg_callback(void *msg, uint32_t msg_len); +extern bool +touchscreen_read(lv_indev_data_t *data); +extern int +ili9340_init(); +extern void +xpt2046_init(void); +extern void +wgl_init(); + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#else +#include +#endif + +#include +#include + +int uart_char_cnt = 0; + +static void +uart_irq_callback(struct device *dev) +{ + unsigned char ch; + + while (uart_poll_in(dev, &ch) == 0) { + uart_char_cnt++; + aee_host_msg_callback(&ch, 1); + } +} + +struct device *uart_dev = NULL; + +static bool +host_init() +{ + uart_dev = device_get_binding(HOST_DEVICE_COMM_UART_NAME); + if (!uart_dev) { + printf("UART: Device driver not found.\n"); + return false; + } + uart_irq_rx_enable(uart_dev); + uart_irq_callback_set(uart_dev, uart_irq_callback); + return true; +} + +int +host_send(void *ctx, const char *buf, int size) +{ + if (!uart_dev) + return 0; + + for (int i = 0; i < size; i++) + uart_poll_out(uart_dev, buf[i]); + + return size; +} + +void +host_destroy() +{} + +/* clang-format off */ +host_interface interface = { + .init = host_init, + .send = host_send, + .destroy = host_destroy +}; +/* clang-format on */ + +timer_ctx_t timer_ctx; + +static char global_heap_buf[270 * 1024] = { 0 }; + +static uint8_t color_copy[320 * 10 * 3]; + +static void +display_flush(lv_disp_drv_t *disp_drv, const lv_area_t *area, lv_color_t *color) +{ + u16_t w = area->x2 - area->x1 + 1; + u16_t h = area->y2 - area->y1 + 1; + struct display_buffer_descriptor desc; + int i; + uint8_t *color_p = color_copy; + + desc.buf_size = 3 * w * h; + desc.width = w; + desc.pitch = w; + desc.height = h; + + for (i = 0; i < w * h; i++, color++) { + color_p[i * 3] = color->ch.red; + color_p[i * 3 + 1] = color->ch.green; + color_p[i * 3 + 2] = color->ch.blue; + } + + display_write(NULL, area->x1, area->y1, &desc, (void *)color_p); + + lv_disp_flush_ready(disp_drv); /* in v5.3 is lv_flush_ready */ +} + +static bool +display_input_read(lv_indev_drv_t *indev_drv, lv_indev_data_t *data) +{ + return touchscreen_read(data); +} + +/** + * Initialize the Hardware Abstraction Layer (HAL) for the Littlev graphics + * library + */ +static void +hal_init(void) +{ + xpt2046_init(); + ili9340_init(); + display_blanking_off(NULL); + + /*Create a display buffer*/ + static lv_disp_buf_t disp_buf1; + static lv_color_t buf1_1[320 * 10]; + lv_disp_buf_init(&disp_buf1, buf1_1, NULL, 320 * 10); + + /*Create a display*/ + lv_disp_drv_t disp_drv; + lv_disp_drv_init(&disp_drv); /*Basic initialization*/ + disp_drv.buffer = &disp_buf1; + disp_drv.flush_cb = display_flush; + // disp_drv.hor_res = 200; + // disp_drv.ver_res = 100; + lv_disp_drv_register(&disp_drv); + + lv_indev_drv_t indev_drv; + lv_indev_drv_init(&indev_drv); /*Basic initialization*/ + indev_drv.type = LV_INDEV_TYPE_POINTER; + indev_drv.read_cb = display_input_read; + lv_indev_drv_register(&indev_drv); +} + +int +iwasm_main() +{ + RuntimeInitArgs init_args; + host_init(); + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + wgl_init(); + hal_init(); + + /* timer manager */ + if (!init_wasm_timer()) { + goto fail; + } + + app_manager_startup(&interface); + +fail: + wasm_runtime_destroy(); + return -1; +} diff --git a/wasm-micro-runtime/samples/inst-context-threads/.gitignore b/wasm-micro-runtime/samples/inst-context-threads/.gitignore new file mode 100644 index 0000000..0fa8a76 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/.gitignore @@ -0,0 +1 @@ +/out/ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/inst-context-threads/CMakeLists.txt b/wasm-micro-runtime/samples/inst-context-threads/CMakeLists.txt new file mode 100644 index 0000000..5ce8696 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/CMakeLists.txt @@ -0,0 +1,91 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (inst-context) +else() + project (inst-context C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 0) +set (WAMR_BUILD_LIB_WASI_THREADS 1) + +if (NOT MSVC) + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (inst-context src/main.c src/native_impl.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (inst-context PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (inst-context vmlib -lm -ldl -lpthread) +else () + target_link_libraries (inst-context vmlib -lm -ldl -lpthread -lrt) +endif () diff --git a/wasm-micro-runtime/samples/inst-context-threads/README.md b/wasm-micro-runtime/samples/inst-context-threads/README.md new file mode 100644 index 0000000..d64cbff --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/README.md @@ -0,0 +1,5 @@ +The "inst-context-threads" sample project +========================================= + +This sample demonstrates some interactions between +module instance context API and wasi-threads. diff --git a/wasm-micro-runtime/samples/inst-context-threads/build.sh b/wasm-micro-runtime/samples/inst-context-threads/build.sh new file mode 100755 index 0000000..35f76ec --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/build.sh @@ -0,0 +1,61 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +CURR_DIR=$PWD +WAMR_DIR=${PWD}/../.. +OUT_DIR=${PWD}/out + +WASM_APPS=${PWD}/wasm-apps + + +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +mkdir ${OUT_DIR}/wasm-apps + + +echo "#####################build inst-context project" +cd ${CURR_DIR} +mkdir -p cmake_build +cd cmake_build +cmake .. +make -j ${nproc} +if [ $? != 0 ];then + echo "BUILD_FAIL inst-context exit as $?\n" + exit 2 +fi + +cp -a inst-context ${OUT_DIR} + +echo -e "\n" + +echo "#####################build wasm apps" + +cd ${WASM_APPS} + +for i in `ls *.c` +do +APP_SRC="$i" +OUT_FILE=${i%.*}.wasm + +# use WAMR SDK to build out the .wasm binary +# require wasi-sdk with wasi-threads support. (wasi-sdk-20.0 or later) +/opt/wasi-sdk/bin/clang \ + --target=wasm32-wasi-threads \ + -pthread \ + -Wl,--import-memory \ + -Wl,--export-memory \ + -Wl,--max-memory=655360 \ + -o ${OUT_DIR}/wasm-apps/${OUT_FILE} ${APP_SRC} + + +if [ -f ${OUT_DIR}/wasm-apps/${OUT_FILE} ]; then + echo "build ${OUT_FILE} success" +else + echo "build ${OUT_FILE} fail" +fi +done +echo "####################build wasm apps done" diff --git a/wasm-micro-runtime/samples/inst-context-threads/run.sh b/wasm-micro-runtime/samples/inst-context-threads/run.sh new file mode 100755 index 0000000..919ed01 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +out/inst-context -f out/wasm-apps/testapp.wasm diff --git a/wasm-micro-runtime/samples/inst-context-threads/src/main.c b/wasm-micro-runtime/samples/inst-context-threads/src/main.c new file mode 100644 index 0000000..2a20363 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/src/main.c @@ -0,0 +1,151 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" +#include "bh_getopt.h" +#include "my_context.h" + +void +set_context(wasm_exec_env_t exec_env, int32_t n); +int32_t +get_context(wasm_exec_env_t exec_env); + +void *my_context_key; +struct my_context my_context; +int my_dtor_called; + +wasm_module_inst_t module_inst = NULL; + +void +print_usage(void) +{ + fprintf(stdout, "Options:\r\n"); + fprintf(stdout, " -f [path of wasm file] \n"); +} + +void +my_context_dtor(wasm_module_inst_t inst, void *ctx) +{ + printf("%s called\n", __func__); + my_dtor_called++; + bh_assert(ctx == &my_context); + bh_assert(inst == module_inst); +} + +int +main(int argc, char *argv_main[]) +{ + static char global_heap_buf[512 * 1024]; + char *buffer; + char error_buf[128]; + int opt; + char *wasm_path = NULL; + int exit_code = 1; + + wasm_module_t module = NULL; + uint32 buf_size, stack_size = 8092, heap_size = 8092; + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + while ((opt = getopt(argc, argv_main, "hf:")) != -1) { + switch (opt) { + case 'f': + wasm_path = optarg; + break; + case 'h': + print_usage(); + return 0; + case '?': + print_usage(); + return 0; + } + } + if (optind == 1) { + print_usage(); + return 0; + } + + // Define an array of NativeSymbol for the APIs to be exported. + // Note: the array must be static defined since runtime + // will keep it after registration + // For the function signature specifications, goto the link: + // https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/export_native_api.md + + static NativeSymbol native_symbols[] = { + { "set_context", set_context, "(i)", NULL }, + { "get_context", get_context, "()i", NULL }, + }; + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + // Native symbols need below registration phase + init_args.n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); + init_args.native_module_name = "env"; + init_args.native_symbols = native_symbols; + + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + my_context_key = wasm_runtime_create_context_key(my_context_dtor); + if (!my_context_key) { + printf("wasm_runtime_create_context_key failed.\n"); + return -1; + } + + buffer = bh_read_file_to_buffer(wasm_path, &buf_size); + + if (!buffer) { + printf("Open wasm app file [%s] failed.\n", wasm_path); + goto fail; + } + + module = wasm_runtime_load((uint8 *)buffer, buf_size, error_buf, + sizeof(error_buf)); + if (!module) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + + if (!module_inst) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + char *args[] = { + "testapp", + }; + wasm_application_execute_main(module_inst, 1, args); + const char *exc = wasm_runtime_get_exception(module_inst); + if (exc != NULL) { + printf("call wasm function calculate failed. error: %s\n", exc); + goto fail; + } + + exit_code = 0; +fail: + if (module_inst) { + bh_assert(my_dtor_called == 0); + wasm_runtime_deinstantiate(module_inst); + bh_assert(my_dtor_called == 1); + } + if (module) + wasm_runtime_unload(module); + if (buffer) + BH_FREE(buffer); + if (my_context_key) + wasm_runtime_destroy_context_key(my_context_key); + wasm_runtime_destroy(); + return exit_code; +} diff --git a/wasm-micro-runtime/samples/inst-context-threads/src/my_context.h b/wasm-micro-runtime/samples/inst-context-threads/src/my_context.h new file mode 100644 index 0000000..008be71 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/src/my_context.h @@ -0,0 +1,11 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +struct my_context { + int x; +}; + +extern void *my_context_key; +extern struct my_context my_context; diff --git a/wasm-micro-runtime/samples/inst-context-threads/src/native_impl.c b/wasm-micro-runtime/samples/inst-context-threads/src/native_impl.c new file mode 100644 index 0000000..0733e19 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/src/native_impl.c @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include "wasm_export.h" +#include "my_context.h" + +void +set_context(wasm_exec_env_t exec_env, int32_t n) +{ + wasm_module_inst_t inst = wasm_runtime_get_module_inst(exec_env); + printf("%s called on module inst %p\n", __func__, inst); + struct my_context *ctx = &my_context; + ctx->x = n; + wasm_runtime_set_context_spread(inst, my_context_key, ctx); +} + +int32_t +get_context(wasm_exec_env_t exec_env) +{ + wasm_module_inst_t inst = wasm_runtime_get_module_inst(exec_env); + printf("%s called on module inst %p\n", __func__, inst); + struct my_context *ctx = wasm_runtime_get_context(inst, my_context_key); + if (ctx == NULL) { + return -1; + } + return ctx->x; +} diff --git a/wasm-micro-runtime/samples/inst-context-threads/wasm-apps/testapp.c b/wasm-micro-runtime/samples/inst-context-threads/wasm-apps/testapp.c new file mode 100644 index 0000000..429b087 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context-threads/wasm-apps/testapp.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +void +set_context(int32_t n) __attribute__((import_module("env"))) +__attribute__((import_name("set_context"))); + +int32_t +get_context() __attribute__((import_module("env"))) +__attribute__((import_name("get_context"))); + +void * +start(void *vp) +{ + int32_t v; + + printf("thread started\n"); + + printf("confirming the initial state on thread\n"); + v = get_context(); + assert(v == -1); + + printf("setting the context on thread\n"); + set_context(1234); + + printf("confirming the context on thread\n"); + v = get_context(); + assert(v == 1234); + return NULL; +} + +int +main() +{ + pthread_t t1; + int32_t v; + int ret; + + printf("confirming the initial state on main\n"); + v = get_context(); + assert(v == -1); + + printf("creating a thread\n"); + ret = pthread_create(&t1, NULL, start, NULL); + assert(ret == 0); + void *val; + ret = pthread_join(t1, &val); + assert(ret == 0); + printf("joined the thread\n"); + + printf("confirming the context propagated from the thread on main\n"); + v = get_context(); + assert(v == 1234); + + printf("success\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/inst-context/.gitignore b/wasm-micro-runtime/samples/inst-context/.gitignore new file mode 100644 index 0000000..0fa8a76 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/.gitignore @@ -0,0 +1 @@ +/out/ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/inst-context/CMakeLists.txt b/wasm-micro-runtime/samples/inst-context/CMakeLists.txt new file mode 100644 index 0000000..af387ca --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/CMakeLists.txt @@ -0,0 +1,90 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (inst-context) +else() + project (inst-context C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) + +if (NOT MSVC) + set (WAMR_BUILD_LIBC_WASI 1) +endif () + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (inst-context src/main.c src/native_impl.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (inst-context PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (inst-context vmlib -lm -ldl -lpthread) +else () + target_link_libraries (inst-context vmlib -lm -ldl -lpthread -lrt) +endif () diff --git a/wasm-micro-runtime/samples/inst-context/README.md b/wasm-micro-runtime/samples/inst-context/README.md new file mode 100644 index 0000000..43b13c6 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/README.md @@ -0,0 +1,4 @@ +The "inst-context" sample project +================================= + +This sample demonstrates module instance context API. diff --git a/wasm-micro-runtime/samples/inst-context/build.sh b/wasm-micro-runtime/samples/inst-context/build.sh new file mode 100755 index 0000000..816e1cc --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/build.sh @@ -0,0 +1,63 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +CURR_DIR=$PWD +WAMR_DIR=${PWD}/../.. +OUT_DIR=${PWD}/out + +WASM_APPS=${PWD}/wasm-apps + + +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +mkdir ${OUT_DIR}/wasm-apps + + +echo "#####################build inst-context project" +cd ${CURR_DIR} +mkdir -p cmake_build +cd cmake_build +cmake .. +make -j ${nproc} +if [ $? != 0 ];then + echo "BUILD_FAIL inst-context exit as $?\n" + exit 2 +fi + +cp -a inst-context ${OUT_DIR} + +echo -e "\n" + +echo "#####################build wasm apps" + +cd ${WASM_APPS} + +for i in `ls *.c` +do +APP_SRC="$i" +OUT_FILE=${i%.*}.wasm + +# use WAMR SDK to build out the .wasm binary +/opt/wasi-sdk/bin/clang \ + --target=wasm32 -O0 -z stack-size=4096 -Wl,--initial-memory=65536 \ + --sysroot=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot \ + -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt \ + -Wl,--strip-all,--no-entry -nostdlib \ + -Wl,--export=generate_float \ + -Wl,--export=float_to_string \ + -Wl,--export=calculate\ + -Wl,--allow-undefined \ + -o ${OUT_DIR}/wasm-apps/${OUT_FILE} ${APP_SRC} + + +if [ -f ${OUT_DIR}/wasm-apps/${OUT_FILE} ]; then + echo "build ${OUT_FILE} success" +else + echo "build ${OUT_FILE} fail" +fi +done +echo "####################build wasm apps done" diff --git a/wasm-micro-runtime/samples/inst-context/run.sh b/wasm-micro-runtime/samples/inst-context/run.sh new file mode 100755 index 0000000..919ed01 --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +out/inst-context -f out/wasm-apps/testapp.wasm diff --git a/wasm-micro-runtime/samples/inst-context/src/main.c b/wasm-micro-runtime/samples/inst-context/src/main.c new file mode 100644 index 0000000..f5068de --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/src/main.c @@ -0,0 +1,166 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" +#include "bh_getopt.h" +#include "my_context.h" + +int32_t +add_native(int32_t n); +void *my_context_key; +struct my_context my_context; +int my_dtor_called; + +wasm_module_inst_t module_inst = NULL; + +void +print_usage(void) +{ + fprintf(stdout, "Options:\r\n"); + fprintf(stdout, " -f [path of wasm file] \n"); +} + +void +my_context_dtor(wasm_module_inst_t inst, void *ctx) +{ + printf("%s called\n", __func__); + my_dtor_called++; + bh_assert(ctx == &my_context); + bh_assert(inst == module_inst); +} + +int +main(int argc, char *argv_main[]) +{ + static char global_heap_buf[512 * 1024]; + char *buffer; + char error_buf[128]; + int opt; + char *wasm_path = NULL; + + wasm_module_t module = NULL; + wasm_exec_env_t exec_env = NULL; + uint32 buf_size, stack_size = 8092, heap_size = 8092; + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + while ((opt = getopt(argc, argv_main, "hf:")) != -1) { + switch (opt) { + case 'f': + wasm_path = optarg; + break; + case 'h': + print_usage(); + return 0; + case '?': + print_usage(); + return 0; + } + } + if (optind == 1) { + print_usage(); + return 0; + } + + // Define an array of NativeSymbol for the APIs to be exported. + // Note: the array must be static defined since runtime + // will keep it after registration + // For the function signature specifications, goto the link: + // https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/export_native_api.md + + static NativeSymbol native_symbols[] = { { "add_native", add_native, "(i)i", + NULL } }; + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + // Native symbols need below registration phase + init_args.n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); + init_args.native_module_name = "env"; + init_args.native_symbols = native_symbols; + + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + my_context_key = wasm_runtime_create_context_key(my_context_dtor); + if (!my_context_key) { + printf("wasm_runtime_create_context_key failed.\n"); + return -1; + } + + buffer = bh_read_file_to_buffer(wasm_path, &buf_size); + + if (!buffer) { + printf("Open wasm app file [%s] failed.\n", wasm_path); + goto fail; + } + + module = wasm_runtime_load((uint8 *)buffer, buf_size, error_buf, + sizeof(error_buf)); + if (!module) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + + if (!module_inst) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + my_context.x = 100; + wasm_runtime_set_context(module_inst, my_context_key, &my_context); + + exec_env = wasm_runtime_create_exec_env(module_inst, stack_size); + if (!exec_env) { + printf("Create wasm execution environment failed.\n"); + goto fail; + } + + wasm_function_inst_t func3 = + wasm_runtime_lookup_function(module_inst, "calculate"); + if (!func3) { + printf("The wasm function calculate is not found.\n"); + goto fail; + } + + uint32_t argv3[1] = { 3 }; + if (wasm_runtime_call_wasm(exec_env, func3, 1, argv3)) { + uint32_t result = *(uint32_t *)argv3; + printf("Native finished calling wasm function: calculate, return: %d\n", + result); + bh_assert(result == 103); /* argv3[0] + my_context.x */ + } + else { + printf("call wasm function calculate failed. error: %s\n", + wasm_runtime_get_exception(module_inst)); + goto fail; + } + +fail: + if (exec_env) + wasm_runtime_destroy_exec_env(exec_env); + if (module_inst) { + bh_assert(my_dtor_called == 0); + wasm_runtime_deinstantiate(module_inst); + bh_assert(my_dtor_called == 1); + } + if (module) + wasm_runtime_unload(module); + if (buffer) + BH_FREE(buffer); + if (my_context_key) + wasm_runtime_destroy_context_key(my_context_key); + wasm_runtime_destroy(); + return 0; +} diff --git a/wasm-micro-runtime/samples/inst-context/src/my_context.h b/wasm-micro-runtime/samples/inst-context/src/my_context.h new file mode 100644 index 0000000..db49c1e --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/src/my_context.h @@ -0,0 +1,10 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +struct my_context { + int x; +}; + +extern void *my_context_key; diff --git a/wasm-micro-runtime/samples/inst-context/src/native_impl.c b/wasm-micro-runtime/samples/inst-context/src/native_impl.c new file mode 100644 index 0000000..1254b4a --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/src/native_impl.c @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "my_context.h" + +int32_t +add_native(wasm_exec_env_t exec_env, int32_t n) +{ + wasm_module_inst_t inst = wasm_runtime_get_module_inst(exec_env); + struct my_context *ctx = wasm_runtime_get_context(inst, my_context_key); + return n + ctx->x; +} diff --git a/wasm-micro-runtime/samples/inst-context/wasm-apps/testapp.c b/wasm-micro-runtime/samples/inst-context/wasm-apps/testapp.c new file mode 100644 index 0000000..1774dcd --- /dev/null +++ b/wasm-micro-runtime/samples/inst-context/wasm-apps/testapp.c @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +int32_t +add_native(int32_t n); + +int32_t +calculate(int32_t n) +{ + printf("calling into WASM function: %s\n", __FUNCTION__); + return add_native(n); +} diff --git a/wasm-micro-runtime/samples/linux-perf/CMakeLists.txt b/wasm-micro-runtime/samples/linux-perf/CMakeLists.txt new file mode 100644 index 0000000..e9882c8 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/CMakeLists.txt @@ -0,0 +1,63 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(linux_perf_sample) + +if(NOT CMAKE_HOST_LINUX) + message(FATAL_ERROR "This sample only works on linux") +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +set(CMAKE_CXX_STANDARD 17) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +find_package(WASISDK REQUIRED) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +include(CheckPIESupported) + +# AOT and JIT byd default +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_INTERP 0) +set(WAMR_BUILD_JIT 1) +# wasm32-wasi +set(WAMR_BUILD_LIBC_BUILTIN 0) +set(WAMR_BUILD_LIBC_WASI 1) +# mvp +set(WAMR_BUILD_BULK_MEMORY 1) +set(WAMR_BUILD_REF_TYPES 1) +set(WAMR_BUILD_SIMD 1) +set(WAMR_BUILD_TAIL_CALL 1) +# trap information +set(WAMR_BUILD_DUMP_CALL_STACK 1) +# linux perf +set(WAMR_BUILD_LINUX_PERF 1) +# +#set(WAMR_BUILD_THREAD_MGR 0) + +# vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib SHARED ${WAMR_RUNTIME_LIB_SOURCE}) +target_include_directories(vmlib INTERFACE ${WAMR_ROOT_DIR}/core/iwasm/include) +target_link_libraries (vmlib ${LLVM_AVAILABLE_LIBS} -lm -ldl) + +################ host ################ +add_executable(${PROJECT_NAME} host/demo.c) +target_link_libraries(${PROJECT_NAME} vmlib) + +################ aot + wasm ################ +include(ExternalProject) +ExternalProject_Add(wasm + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasm" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm -B build + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} +) \ No newline at end of file diff --git a/wasm-micro-runtime/samples/linux-perf/README.md b/wasm-micro-runtime/samples/linux-perf/README.md new file mode 100644 index 0000000..5a8dc57 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/README.md @@ -0,0 +1,90 @@ +# linux perf sample introduction + +This is a sample to show how to use the Linux perf tool to profile the execution of a WebAssembly application. And how to use the [Flamegraph](https://www.brendangregg.com/flamegraphs.html) tool to visualize the profiling result. + +## Build and run the sample + +There are two Wasm modules and their instance will be created and run in the sample. [The first module](./wasm/fib.c) is a simple Wasm module that calculates the Fibonacci number. [The second module](./wasm/ackermann.c) is a simple Wasm module that execute the Ackermann function. The target is enable to profile the execution of both two modules separately. + +```bash +$ cmake -S . -B build +$ cmake --build build +``` + +### Profile the execution + +```bash +$ cd build +$ perf record -k mono -g --output=perf.data -- ./linux_perf_sample +``` + +Enable to use `perf report --stdio` to do a quick analysis of the profiling result. + +### Visualize the profiling result + +Need to download Flamegraph tool from [Flamegraph](https://github.com/brendangregg/FlameGraph/releases/tag/v1.0) firstly. + +```bash +$ perf script > out.perf +$ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded +$ ./FlameGraph/flamegraph.pl out.folded > perf.svg +``` + +In this result, you'll see two modules's profiling result and all wasm functions are named as "aot_func#N" which is a little hard to distinguish. + +![perf.png](./pics/perf.png) + +### Separate profiling result + +[process_folded_data.py](../../test-tools/flame-graph-helper/process_folded_data.py) is a script can a) translate "aot_func#N" into its original function name in name sections, b) separate the profiling result of different modules. + +In this sample, we want to separate `fib` and `ackermann` profiling data from _out.folded_. In [demo](host/demo.c), we decide to name the module of `fib1.wasm` as `fib2` and the module of `ackermann1.wasm` as `ackermann2`. + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm_names fib2=./fib1.wasm,ackermann2=./ackermann1.wasm out.folded +-> write into out.fib2.translated +-> write into out.ackermann2.translated +-> write into out.translated +``` + +More scenarios: + +if only using one wasm during profiling, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm --folded +``` + +if only using one wasm during profiling and specify the module name via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm_names = --folded +``` + +if only using one wasm during profiling and specify the module name, which is same with the basename of wasm file, via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm --folded +``` + +if using multiple wasm during profiling and specify module names, which are same with basename of wasm files, via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm --wasm --wasm --folded +``` + +if using multiple wasm during profiling and specify module names via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm_names =,=,= --folded +``` + +Now we have two flame-graphs for two wasm modules: + +![fib.svg](./pics/perf.fib.svg) + +![ackermann.svg](./pics/perf.ackermann.svg) + +## Reference + +- [perf_tune](../../doc/perf_tune.md) diff --git a/wasm-micro-runtime/samples/linux-perf/cmake/FindWAMRC.cmake b/wasm-micro-runtime/samples/linux-perf/cmake/FindWAMRC.cmake new file mode 100644 index 0000000..586263e --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/cmake/FindWAMRC.cmake @@ -0,0 +1,14 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +find_file(WAMRC_BIN + NAMES wamrc + DOC "search wamrc" + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/../../../wamr-compiler/build + REQUIRED +) + +find_package_handle_standard_args(WAMRC REQUIRED_VARS WAMRC_BIN) +mark_as_advanced(WAMRC_BIN) \ No newline at end of file diff --git a/wasm-micro-runtime/samples/linux-perf/cmake/FindWASISDK.cmake b/wasm-micro-runtime/samples/linux-perf/cmake/FindWASISDK.cmake new file mode 100644 index 0000000..5cdfea4 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/cmake/FindWASISDK.cmake @@ -0,0 +1,23 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*") +find_path(WASISDK_HOME + NAMES share/wasi-sysroot + PATHS ${WASISDK_SEARCH_PATH} + NO_DEFAULT_PATH + REQUIRED +) + +string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) + +find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) + +if(WASISDK_FOUND) + set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang) + set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++) + set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake) + set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot) +endif() diff --git a/wasm-micro-runtime/samples/linux-perf/host/demo.c b/wasm-micro-runtime/samples/linux-perf/host/demo.c new file mode 100644 index 0000000..8ea446e --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/host/demo.c @@ -0,0 +1,198 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +/* return a copy of the file stem of a file path */ +static own char * +stem(const char *file_path) +{ + char *base_name = basename(file_path); + char *s = strdup(base_name); + char *dot = strchr(s, '.'); + assert(dot); + *dot = '\0'; + return s; +} + +static void +guest_i32_to_wasm_i32_array(int *args, unsigned argc, wasm_val_t *data, + unsigned datac) +{ + for (unsigned i = 0; i < argc && i < datac; i++) { + memset(&data[i], 0, sizeof(wasm_val_t)); + data[i].kind = WASM_I32; + data[i].of.i32 = args[i]; + } +} + +int +load_run_wasm_file(wasm_engine_t *engine, const char *file_path, int *args, + unsigned argc) +{ + wasm_store_t *store = wasm_store_new(engine); + // Load binary. + printf("Loading binary...\n"); + FILE *file = fopen(file_path, "rb"); + assert(file); + + int ret = fseek(file, 0L, SEEK_END); + assert(ret == 0); + + long file_size = ftell(file); + assert(file_size != -1); + + ret = fseek(file, 0L, SEEK_SET); + assert(ret == 0); + + wasm_byte_vec_t binary = { 0 }; + wasm_byte_vec_new_uninitialized(&binary, file_size); + + size_t nread = fread(binary.data, file_size, 1, file); + fclose(file); + + // Compile. + printf("Compiling module...\n"); + + // Use its file name as the module name + char *file_name = stem(file_path); + assert(file_name); + + LoadArgs load_args = { 0 }; + load_args.name = file_name; + own wasm_module_t *module = wasm_module_new_ex(store, &binary, &load_args); + wasm_byte_vec_delete(&binary); + assert(module); + + // Use export type to find the function index to call later + wasm_exporttype_vec_t export_types = { 0 }; + wasm_module_exports(module, &export_types); + int func_to_call = -1; + for (unsigned i = 0; i < export_types.num_elems; i++) { + const wasm_name_t *name = wasm_exporttype_name(export_types.data[i]); + if (strncmp(name->data, "run", 3) == 0) { + func_to_call = i; + break; + } + } + assert(func_to_call != -1); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + own wasm_instance_t *instance = wasm_instance_new_with_args( + store, module, &imports, NULL, 16 * 1024 * 1024, 1 * 1024 * 1024); + assert(instance); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + assert(exports.size); + + assert(wasm_extern_kind(exports.data[func_to_call]) == WASM_EXTERN_FUNC); + const wasm_func_t *run_func = + wasm_extern_as_func(exports.data[func_to_call]); + assert(run_func); + + wasm_module_delete(module); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_t as[4] = { 0 }; + guest_i32_to_wasm_i32_array(args, argc, as, 4); + + wasm_val_vec_t params = WASM_ARRAY_VEC(as); + wasm_val_t rs[1] = { WASM_I32_VAL(0) }; + wasm_val_vec_t results = WASM_ARRAY_VEC(rs); + wasm_trap_t *trap = wasm_func_call(run_func, ¶ms, &results); + assert(!trap); + + wasm_extern_vec_delete(&exports); + free(file_name); + wasm_store_delete(store); + + { + nread = nread; + ret = ret; + trap = trap; + } + return 0; +} + +void * +load_run_fib_wasm(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 40 }; + load_run_wasm_file(engine, "./fib1.wasm", args, 1); + return NULL; +} + +void * +load_run_fib_aot(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 40 }; + load_run_wasm_file(engine, "./fib2.aot", args, 1); + return NULL; +} + +void * +load_run_ackermann_wasm(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 3, 12 }; + load_run_wasm_file(engine, "./ackermann1.wasm", args, 2); + return NULL; +} + +void * +load_run_ackermann_aot(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 3, 12 }; + load_run_wasm_file(engine, "./ackermann2.aot", args, 2); + return NULL; +} + +int +main(int argc, const char *argv[]) +{ + // Initialize. + printf("Initializing...\n"); + wasm_config_t *config = wasm_config_new(); + wasm_config_set_linux_perf_opt(config, true); + wasm_engine_t *engine = wasm_engine_new_with_config(config); + + pthread_t tid[4] = { 0 }; + /* FIXME: uncomment when it is able to run two modules with llvm-jit */ + // pthread_create(&tid[0], NULL, load_run_fib_wasm, (void *)engine); + // pthread_create(&tid[2], NULL, load_run_ackermann_wasm, (void *)engine); + + pthread_create(&tid[1], NULL, load_run_fib_aot, (void *)engine); + pthread_create(&tid[3], NULL, load_run_ackermann_aot, (void *)engine); + + for (unsigned i = 0; i < sizeof(tid) / sizeof(tid[0]); i++) + pthread_join(tid[i], NULL); + + // Shut down. + printf("Shutting down...\n"); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/linux-perf/pics/perf.ackermann.svg b/wasm-micro-runtime/samples/linux-perf/pics/perf.ackermann.svg new file mode 100644 index 0000000..c2e1d87 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/pics/perf.ackermann.svg @@ -0,0 +1,1349 @@ + + + + + + + + + + + + + + +Flame Graph + +Reset Zoom +Search +ic + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +load_run_wasm_file (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +invoke_ii_i (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +load_run_ackermann_aot (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] run (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +wasm_runtime_call_wasm (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +all (11,485,868,643 samples, 100%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +aot_call_function (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +linux_perf_samp (11,485,868,643 samples, 100.00%) +linux_perf_samp + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +wasm_func_call (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +invoke_native_with_hw_bound_check (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +start_thread (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + + diff --git a/wasm-micro-runtime/samples/linux-perf/pics/perf.fib.svg b/wasm-micro-runtime/samples/linux-perf/pics/perf.fib.svg new file mode 100644 index 0000000..a1db059 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/pics/perf.fib.svg @@ -0,0 +1,605 @@ + + + + + + + + + + + + + + +Flame Graph + +Reset Zoom +Search +ic + + + +[Wasm] [fib2] fibonacci (1,321,095,222 samples, 93.80%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (382,407,564 samples, 27.15%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (15,340,273 samples, 1.09%) + + + +[Wasm] [fib2] fibonacci (1,359,552,763 samples, 96.53%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (27,274,310 samples, 1.94%) +[.. + + +[Wasm] [fib2] fibonacci (62,450,767 samples, 4.43%) +[Wasm.. + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,388,674,508 samples, 98.59%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,170,751,868 samples, 83.12%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (120,820,158 samples, 8.58%) +[Wasm] [fib2.. + + +invoke_i_i (1,408,481,525 samples, 100.00%) +invoke_i_i + + +[Wasm] [fib2] fibonacci (1,375,872,224 samples, 97.68%) +[Wasm] [fib2] fibonacci + + +load_run_wasm_file (1,408,481,525 samples, 100.00%) +load_run_wasm_file + + +load_run_fib_aot (1,408,481,525 samples, 100.00%) +load_run_fib_aot + + +[Wasm] [fib2] run (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] run + + +[Wasm] [fib2] fibonacci (42,420,273 samples, 3.01%) +[Wa.. + + +[Wasm] [fib2] fibonacci (1,266,323,684 samples, 89.91%) +[Wasm] [fib2] fibonacci + + +linux_perf_samp (1,408,481,525 samples, 100.00%) +linux_perf_samp + + +[Wasm] [fib2] fibonacci (280,259,464 samples, 19.90%) +[Wasm] [fib2] fibonacci + + +start_thread (1,408,481,525 samples, 100.00%) +start_thread + + +[Wasm] [fib2] fibonacci (2,334,521 samples, 0.17%) + + + +[Wasm] [fib2] fibonacci (666,394,609 samples, 47.31%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (943,121,736 samples, 66.96%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,169,581 samples, 0.08%) + + + +[Wasm] [fib2] fibonacci (1,169,581 samples, 0.08%) + + + +[Wasm] [fib2] fibonacci (194,755,877 samples, 13.83%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,406,148,966 samples, 99.83%) +[Wasm] [fib2] fibonacci + + +all (1,408,481,525 samples, 100%) + + + +wasm_func_call (1,408,481,525 samples, 100.00%) +wasm_func_call + + +[Wasm] [fib2] fibonacci (5,943,602 samples, 0.42%) + + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +wasm_runtime_call_wasm (1,408,481,525 samples, 100.00%) +wasm_runtime_call_wasm + + +[Wasm] [fib2] fibonacci (1,401,486,191 samples, 99.50%) +[Wasm] [fib2] fibonacci + + +aot_call_function (1,408,481,525 samples, 100.00%) +aot_call_function + + +[Wasm] [fib2] fibonacci (531,941,563 samples, 37.77%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,406,148,966 samples, 99.83%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,061,055,435 samples, 75.33%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,403,816,880 samples, 99.67%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (800,646,766 samples, 56.84%) +[Wasm] [fib2] fibonacci + + +invoke_native_with_hw_bound_check (1,408,481,525 samples, 100.00%) +invoke_native_with_hw_bound_check + + + diff --git a/wasm-micro-runtime/samples/linux-perf/pics/perf.png b/wasm-micro-runtime/samples/linux-perf/pics/perf.png new file mode 100755 index 0000000..fc4c093 Binary files /dev/null and b/wasm-micro-runtime/samples/linux-perf/pics/perf.png differ diff --git a/wasm-micro-runtime/samples/linux-perf/wasm/CMakeLists.txt b/wasm-micro-runtime/samples/linux-perf/wasm/CMakeLists.txt new file mode 100644 index 0000000..8d36e28 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/wasm/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(linux_perf_sample_wasm) + +include(CMakePrintHelpers) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/../cmake) +find_package(WAMRC REQUIRED) + +################ wasm ################ +add_executable(fib_wasm fib.c) +set_target_properties(fib_wasm PROPERTIES SUFFIX .wasm) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/fib_wasm.wasm DESTINATION . RENAME fib1.wasm) + +add_executable(ackermann_wasm ackermann.c) +set_target_properties(ackermann_wasm PROPERTIES SUFFIX .wasm) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ackermann_wasm.wasm DESTINATION . RENAME ackermann1.wasm) + + +################ aot ################ +add_custom_target(fib_aot + ALL + COMMAND ${WAMRC_BIN} --enable-linux-perf -o fib2.aot fib_wasm.wasm + DEPENDS fib_wasm + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/fib2.aot DESTINATION .) + +add_custom_target(ackermann_aot + ALL + COMMAND ${WAMRC_BIN} --enable-linux-perf -o ackermann2.aot ackermann_wasm.wasm + DEPENDS ackermann_wasm + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ackermann2.aot DESTINATION .) diff --git a/wasm-micro-runtime/samples/linux-perf/wasm/ackermann.c b/wasm-micro-runtime/samples/linux-perf/wasm/ackermann.c new file mode 100644 index 0000000..d6eb434 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/wasm/ackermann.c @@ -0,0 +1,38 @@ +#include + +// Ackermann function +unsigned long +ackermann(unsigned long m, unsigned long n) +{ + if (m == 0) { + return n + 1; + } + else if (n == 0) { + return ackermann(m - 1, 1); + } + else { + return ackermann(m - 1, ackermann(m, n - 1)); + } +} + +__attribute__((export_name("run"))) int +run(int m, int n) +{ + int result = ackermann(m, n); + printf("ackermann(%d, %d)=%d\n", m, n, result); + return result; +} + +int +main() +{ + unsigned long m, n, result; + + // Example usage: + m = 3; + n = 2; + result = ackermann(m, n); + printf("Ackermann(%lu, %lu) = %lu\n", m, n, result); + + return 0; +} diff --git a/wasm-micro-runtime/samples/linux-perf/wasm/fib.c b/wasm-micro-runtime/samples/linux-perf/wasm/fib.c new file mode 100644 index 0000000..cb928f6 --- /dev/null +++ b/wasm-micro-runtime/samples/linux-perf/wasm/fib.c @@ -0,0 +1,32 @@ +#include +#include + +int +fibonacci(int n) +{ + if (n <= 0) + return 0; + + if (n == 1) + return 1; + + return fibonacci(n - 1) + fibonacci(n - 2); +} + +__attribute__((export_name("run"))) int +run(int n) +{ + int result = fibonacci(n); + printf("fibonacci(%d)=%d\n", n, result); + return result; +} + +int +main(int argc, char **argv) +{ + int n = atoi(argv[1]); + + printf("fibonacci(%d)=%d\n", n, fibonacci(n)); + + return 0; +} diff --git a/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/XPT2046.c b/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/XPT2046.c new file mode 100644 index 0000000..e3948e2 --- /dev/null +++ b/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/XPT2046.c @@ -0,0 +1,349 @@ +/** + * @file XPT2046.c + */ +/********************* + * INCLUDES + *********************/ +#include "XPT2046.h" +#include "board_config.h" +#include "stdio.h" +#include +#include "drivers/spi.h" + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#include +#else +#include +#endif + +#if USE_XPT2046 + +#include + +#define abs(x) ((x) < 0 ? -(x) : (x)) + +/********************* + * DEFINES + *********************/ + +/********************** + * TYPEDEFS + **********************/ + +/********************** + * STATIC PROTOTYPES + **********************/ +static void +xpt2046_corr(int16_t *x, int16_t *y); +#if 0 +static void xpt2046_avg(int16_t * x, int16_t * y); +#endif + +/********************** + * STATIC VARIABLES + **********************/ +int16_t avg_buf_x[XPT2046_AVG]; +int16_t avg_buf_y[XPT2046_AVG]; +uint8_t avg_last; + +/********************** + * MACROS + **********************/ + +/********************** + * GLOBAL FUNCTIONS + **********************/ + +/** + * Initialize the XPT2046 + */ +struct device *input_dev; + +struct spi_config spi_conf_xpt2046; +struct spi_cs_control xpt2046_cs_ctrl; +struct device *xpt2046_pen_gpio_dev; +static struct gpio_callback gpio_cb; +lv_indev_data_t touch_point; +lv_indev_data_t last_touch_point; + +#define TOUCH_READ_THREAD_STACK_SIZE 4096 +static K_THREAD_STACK_DEFINE(touch_read_thread_stack, + TOUCH_READ_THREAD_STACK_SIZE); +static struct k_thread touch_thread_data; +static struct k_sem sem_touch_read; + +K_MUTEX_DEFINE(spi_display_touch_mutex); + +int cnt = 0; +int touch_read_times = 0; +int last_pen_interrupt_time = 0; + +void +xpt2046_pen_gpio_callback(struct device *port, struct gpio_callback *cb, + uint32_t pins) +{ + cnt++; + if ((k_uptime_get_32() - last_pen_interrupt_time) > 500) { + k_sem_give(&sem_touch_read); + touch_read_times++; + last_pen_interrupt_time = k_uptime_get_32(); + } +} + +void +disable_pen_interrupt() +{ + int ret = 0; + ret = gpio_disable_callback(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN); + if (ret != 0) { + printf("gpio_pin_configure GPIO_INPUT failed\n"); + } +} +void +enable_pen_interrupt() +{ + int ret = 0; + ret = gpio_enable_callback(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN); + if (ret != 0) { + printf("gpio_pin_configure failed\n"); + } +} + +void +touch_screen_read_thread() +{ + int i; + bool ret = false; + + for (;;) { + k_sem_take(&sem_touch_read, K_FOREVER); + memset(&last_touch_point, 0, sizeof(lv_indev_data_t)); + memset(&touch_point, 0, sizeof(lv_indev_data_t)); + memset(avg_buf_x, 0, sizeof(avg_buf_x)); + memset(avg_buf_y, 0, sizeof(avg_buf_y)); + k_mutex_lock(&spi_display_touch_mutex, K_FOREVER); + disable_pen_interrupt(); + for (i = 0; i < 100; i++) { + ret = xpt2046_read(&touch_point); + if (ret) { + if ((abs(last_touch_point.point.x - touch_point.point.x) < 4) + && (abs(last_touch_point.point.y - touch_point.point.y) + < 4)) { + break; + } + last_touch_point = touch_point; + } + } + enable_pen_interrupt(); + k_mutex_unlock(&spi_display_touch_mutex); + } +} + +void +xpt2046_init(void) +{ + int ret; + input_dev = device_get_binding(XPT2046_SPI_DEVICE_NAME); + + if (input_dev == NULL) { + printf("device not found. Aborting test."); + return; + } + memset((void *)&touch_point, 0, sizeof(lv_indev_data_t)); + + spi_conf_xpt2046.frequency = XPT2046_SPI_MAX_FREQUENCY; + spi_conf_xpt2046.operation = SPI_OP_MODE_MASTER | SPI_WORD_SET(8); + spi_conf_xpt2046.slave = 0; + spi_conf_xpt2046.cs = NULL; +#ifdef XPT2046_CS_GPIO_CONTROLLER + xpt2046_cs_ctrl.gpio_dev = device_get_binding(XPT2046_CS_GPIO_CONTROLLER); + if (xpt2046_cs_ctrl.gpio_dev == NULL) { + printk("Cannot find %s!\n", XPT2046_CS_GPIO_CONTROLLER); + return; + } + gpio_pin_configure(xpt2046_cs_ctrl.gpio_dev, XPT2046_CS_GPIO_PIN, + GPIO_OUTPUT); + gpio_pin_set(xpt2046_cs_ctrl.gpio_dev, XPT2046_CS_GPIO_PIN, 1); + xpt2046_cs_ctrl.gpio_pin = XPT2046_CS_GPIO_PIN; + xpt2046_cs_ctrl.delay = 0; + spi_conf_xpt2046.cs = &xpt2046_cs_ctrl; + +#endif + +#ifdef XPT2046_PEN_GPIO_CONTROLLER + + xpt2046_pen_gpio_dev = device_get_binding(XPT2046_PEN_GPIO_CONTROLLER); + if (!xpt2046_pen_gpio_dev) { + printk("Cannot find %s!\n", XPT2046_PEN_GPIO_CONTROLLER); + return; + } + /* Setup GPIO input */ + ret = gpio_pin_configure(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN, + (GPIO_INPUT | GPIO_INT_ENABLE | GPIO_INT_EDGE + | GPIO_INT_LOW_0 | GPIO_INT_DEBOUNCE)); + if (ret) { + printk("Error configuring pin %d!\n", XPT2046_PEN_GPIO_PIN); + } + + gpio_init_callback(&gpio_cb, xpt2046_pen_gpio_callback, + BIT(XPT2046_PEN_GPIO_PIN)); + + ret = gpio_add_callback(xpt2046_pen_gpio_dev, &gpio_cb); + if (ret) { + printk("gpio_add_callback error\n"); + } + ret = gpio_enable_callback(xpt2046_pen_gpio_dev, XPT2046_PEN_GPIO_PIN); + if (ret) { + printk("gpio_enable_callback error\n"); + } +#endif + + k_sem_init(&sem_touch_read, 0, 1); + + k_thread_create(&touch_thread_data, touch_read_thread_stack, + TOUCH_READ_THREAD_STACK_SIZE, touch_screen_read_thread, + NULL, NULL, NULL, 5, 0, K_NO_WAIT); + printf("xpt2046_init ok \n"); +} + +/** + * Get the current position and state of the touchpad + * @param data store the read data here + * @return false: because no ore data to be read + */ +bool +xpt2046_read(lv_indev_data_t *data) +{ + static int16_t last_x = 0; + static int16_t last_y = 0; + bool valid = true; + int s32_ret = 0; + + int16_t x = 0; + int16_t y = 0; + + char tx1[16] = { 0 }; + char rx1[16] = { 0 }; + + struct spi_buf tx_buf = { .buf = &tx1, .len = 3 }; + struct spi_buf_set tx_bufs = { .buffers = &tx_buf, .count = 1 }; + struct spi_buf rx_buf = { .buf = &rx1, .len = 3 }; + struct spi_buf_set rx_bufs = { .buffers = &rx_buf, .count = 1 }; + + tx1[0] = CMD_X_READ; + s32_ret = spi_transceive(input_dev, &spi_conf_xpt2046, &tx_bufs, &rx_bufs); + if (s32_ret != 0) { + printf("spi_transceive return failed:%d\n", s32_ret); + } + x = rx1[1] << 8; + x += rx1[2]; + + tx1[0] = CMD_Y_READ; + s32_ret = spi_transceive(input_dev, &spi_conf_xpt2046, &tx_bufs, &rx_bufs); + if (s32_ret != 0) { + printf("spi_transceive return failed:%d\n", s32_ret); + } + y = rx1[1] << 8; + y += rx1[2]; + x = x >> 3; + y = y >> 3; + + xpt2046_corr(&x, &y); + if (y <= 0 || (x > 320)) { + valid = false; + } + + last_x = x; + last_y = y; + + data->point.x = x; + data->point.y = y; + data->state = valid == false ? LV_INDEV_STATE_REL : LV_INDEV_STATE_PR; + + return valid; +} + +/********************** + * STATIC FUNCTIONS + **********************/ +static void +xpt2046_corr(int16_t *x, int16_t *y) +{ +#if XPT2046_XY_SWAP != 0 + int16_t swap_tmp; + swap_tmp = *x; + *x = *y; + *y = swap_tmp; +#endif + + if ((*x) > XPT2046_X_MIN) + (*x) -= XPT2046_X_MIN; + else + (*x) = 0; + + if ((*y) > XPT2046_Y_MIN) + (*y) -= XPT2046_Y_MIN; + else + (*y) = 0; + + (*x) = (uint32_t)((uint32_t)(*x) * XPT2046_HOR_RES) + / (XPT2046_X_MAX - XPT2046_X_MIN); + + (*y) = (uint32_t)((uint32_t)(*y) * XPT2046_VER_RES) + / (XPT2046_Y_MAX - XPT2046_Y_MIN); + +#if XPT2046_X_INV != 0 + (*x) = XPT2046_HOR_RES - (*x); +#endif + +#if XPT2046_Y_INV != 0 + (*y) = XPT2046_VER_RES - (*y); +#endif +} + +#if 0 +static void xpt2046_avg(int16_t * x, int16_t * y) +{ + /*Shift out the oldest data*/ + uint8_t i; + for (i = XPT2046_AVG - 1; i > 0; i--) { + avg_buf_x[i] = avg_buf_x[i - 1]; + avg_buf_y[i] = avg_buf_y[i - 1]; + } + + /*Insert the new point*/ + avg_buf_x[0] = *x; + avg_buf_y[0] = *y; + if (avg_last < XPT2046_AVG) + avg_last++; + + /*Sum the x and y coordinates*/ + int32_t x_sum = 0; + int32_t y_sum = 0; + for (i = 0; i < avg_last; i++) { + x_sum += avg_buf_x[i]; + y_sum += avg_buf_y[i]; + } + + /*Normalize the sums*/ + (*x) = (int32_t) x_sum / avg_last; + (*y) = (int32_t) y_sum / avg_last; +} +#endif + +bool +touchscreen_read(lv_indev_data_t *data) +{ + /*Store the collected data*/ + data->point.x = last_touch_point.point.x; + data->point.y = last_touch_point.point.y; + data->state = last_touch_point.state; + + if (last_touch_point.state == LV_INDEV_STATE_PR) { + last_touch_point.state = LV_INDEV_STATE_REL; + } + return false; +} + +#endif diff --git a/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/display_ili9340.h b/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/display_ili9340.h new file mode 100644 index 0000000..39257a2 --- /dev/null +++ b/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/display_ili9340.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 Jan Van Winkel + * + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ +#define ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ +#include "board_config.h" +#include + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#else +#include +#endif + +#define ILI9340_CMD_ENTER_SLEEP 0x10 +#define ILI9340_CMD_EXIT_SLEEP 0x11 +#define ILI9340_CMD_GAMMA_SET 0x26 +#define ILI9340_CMD_DISPLAY_OFF 0x28 +#define ILI9340_CMD_DISPLAY_ON 0x29 +#define ILI9340_CMD_COLUMN_ADDR 0x2a +#define ILI9340_CMD_PAGE_ADDR 0x2b +#define ILI9340_CMD_MEM_WRITE 0x2c +#define ILI9340_CMD_MEM_ACCESS_CTRL 0x36 +#define ILI9340_CMD_PIXEL_FORMAT_SET 0x3A +#define ILI9340_CMD_FRAME_CTRL_NORMAL_MODE 0xB1 +#define ILI9340_CMD_DISPLAY_FUNCTION_CTRL 0xB6 +#define ILI9340_CMD_POWER_CTRL_1 0xC0 +#define ILI9340_CMD_POWER_CTRL_2 0xC1 +#define ILI9340_CMD_VCOM_CTRL_1 0xC5 +#define ILI9340_CMD_VCOM_CTRL_2 0xC7 +#define ILI9340_CMD_POSITVE_GAMMA_CORRECTION 0xE0 +#define ILI9340_CMD_NEGATIVE_GAMMA_CORRECTION 0xE1 + +#define ILI9340_DATA_MEM_ACCESS_CTRL_MY 0x80 +#define ILI9340_DATA_MEM_ACCESS_CTRL_MX 0x40 +#define ILI9340_DATA_MEM_ACCESS_CTRL_MV 0x20 +#define ILI9340_DATA_MEM_ACCESS_CTRL_ML 0x10 +#define ILI9340_DATA_MEM_ACCESS_CTRL_BGR 0x08 +#define ILI9340_DATA_MEM_ACCESS_CTRL_MH 0x04 + +#define ILI9340_DATA_PIXEL_FORMAT_RGB_18_BIT 0x60 +#define ILI9340_DATA_PIXEL_FORMAT_RGB_16_BIT 0x50 +#define ILI9340_DATA_PIXEL_FORMAT_MCU_18_BIT 0x06 +#define ILI9340_DATA_PIXEL_FORMAT_MCU_16_BIT 0x05 + +struct ili9340_data; + +/** + * Send data to ILI9340 display controller + * + * @param data Device data structure + * @param cmd Command to send to display controller + * @param tx_data Data to transmit to the display controller + * In case no data should be transmitted pass a NULL pointer + * @param tx_len Number of bytes in tx_data buffer + * + */ +void +ili9340_transmit(struct ili9340_data *data, uint8_t cmd, void *tx_data, + size_t tx_len); + +/** + * Perform LCD specific initialization + * + * @param data Device data structure + */ +void +ili9340_lcd_init(struct ili9340_data *data); + +#define DT_ILITEK_ILI9340_0_LABEL "DISPLAY" +#define CONFIG_DISPLAY_LOG_LEVEL 0 + +#endif /* ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ */ diff --git a/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c b/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c new file mode 100644 index 0000000..10f4980 --- /dev/null +++ b/wasm-micro-runtime/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include "bh_platform.h" +#include "runtime_lib.h" +#include "native_interface.h" +#include "app_manager_export.h" +#include "board_config.h" +#include "bh_platform.h" +#include "runtime_sensor.h" +#include "bi-inc/attr_container.h" +#include "module_wasm_app.h" +#include "wasm_export.h" +#include "sensor_native_api.h" +#include "connection_native_api.h" +#include "display_indev.h" + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#else +#include +#endif + +#include +#include + +extern bool +init_sensor_framework(); +extern void +exit_sensor_framework(); +extern int +aee_host_msg_callback(void *msg, uint32_t msg_len); + +int uart_char_cnt = 0; + +static void +uart_irq_callback(const struct device *dev, void *user_data) +{ + unsigned char ch; + + while (uart_poll_in(dev, &ch) == 0) { + uart_char_cnt++; + aee_host_msg_callback(&ch, 1); + } + (void)user_data; +} + +const struct device *uart_dev = NULL; + +static bool +host_init() +{ + uart_dev = device_get_binding(HOST_DEVICE_COMM_UART_NAME); + if (!uart_dev) { + printf("UART: Device driver not found.\n"); + return false; + } + uart_irq_rx_enable(uart_dev); + uart_irq_callback_set(uart_dev, uart_irq_callback); + return true; +} + +int +host_send(void *ctx, const char *buf, int size) +{ + if (!uart_dev) + return 0; + + for (int i = 0; i < size; i++) + uart_poll_out(uart_dev, buf[i]); + + return size; +} + +void +host_destroy() +{} + +/* clang-format off */ +host_interface interface = { + .init = host_init, + .send = host_send, + .destroy = host_destroy +}; +/* clang-format on */ + +timer_ctx_t timer_ctx; + +static char global_heap_buf[350 * 1024] = { 0 }; + +static NativeSymbol native_symbols[] = { + EXPORT_WASM_API_WITH_SIG(display_input_read, "(*)i"), + EXPORT_WASM_API_WITH_SIG(display_flush, "(iiii*)"), + EXPORT_WASM_API_WITH_SIG(display_fill, "(iiii*)"), + EXPORT_WASM_API_WITH_SIG(display_vdb_write, "(*iii*i)"), + EXPORT_WASM_API_WITH_SIG(display_map, "(iiii*)"), + EXPORT_WASM_API_WITH_SIG(time_get_ms, "()i") +}; + +int +iwasm_main() +{ + RuntimeInitArgs init_args; + + host_init(); + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + init_args.native_module_name = "env"; + init_args.n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); + init_args.native_symbols = native_symbols; + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + display_init(); + + /* timer manager */ + if (!init_wasm_timer()) { + goto fail; + } + + app_manager_startup(&interface); + +fail: + wasm_runtime_destroy(); + return -1; +} diff --git a/wasm-micro-runtime/samples/mem-allocator/CMakeLists.txt b/wasm-micro-runtime/samples/mem-allocator/CMakeLists.txt new file mode 100644 index 0000000..f157dfb --- /dev/null +++ b/wasm-micro-runtime/samples/mem-allocator/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright (C) 2023 Midokura Japan KK. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.0) +project(mem_allocator_create) + +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if(APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif() + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_LIBC_BUILTIN 0) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../..) +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +add_executable(mem_alloc_test main.c) + +target_link_libraries(mem_alloc_test vmlib -lm -lpthread) diff --git a/wasm-micro-runtime/samples/mem-allocator/main.c b/wasm-micro-runtime/samples/mem-allocator/main.c new file mode 100644 index 0000000..a309d2e --- /dev/null +++ b/wasm-micro-runtime/samples/mem-allocator/main.c @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2023 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include + +#include "mem_alloc.h" + +char store[1000]; + +int +main(int argc, char **argv) +{ + mem_allocator_t a = mem_allocator_create(store, sizeof(store)); + uint8_t *p; + uint8_t *p2; + + p = mem_allocator_malloc(a, 256); + printf("%p\n", p); + if (p == NULL) { + exit(1); + } + p = mem_allocator_realloc(a, p, 256 + 12); + printf("%p\n", p); + if (p == NULL) { + exit(1); + } + + /* + * write some values to confuse the ems allocator. + * + * hmu = p + 256 + * hmu_set_ut(hmu, HMU_FC) + * hmu_set_size(hmu, 256) + * hmu_set_free_size(hmu) + */ + *(uint32_t *)(p + 256) = (1 << 30) | 0x20; + *(uint32_t *)(p + 256 + 12 - 4) = 12; + + p2 = mem_allocator_malloc(a, 256); + printf("%p\n", p2); + if (p2 == NULL) { + exit(1); + } + mem_allocator_free(a, p2); + + p2 = mem_allocator_malloc(a, 256); + printf("%p\n", p2); + if (p2 == NULL) { + exit(1); + } + mem_allocator_free(a, p2); + + mem_allocator_free(a, p); +} diff --git a/wasm-micro-runtime/samples/multi-module/CMakeLists.txt b/wasm-micro-runtime/samples/multi-module/CMakeLists.txt new file mode 100644 index 0000000..7b3fdb8 --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/CMakeLists.txt @@ -0,0 +1,203 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +project(multi_module) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Resetdefault linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set(WAMR_BUILD_INTERP 1) +if (NOT DEFINED WAMR_BUILD_AOT) + set(WAMR_BUILD_AOT 0) +endif () +if (NOT DEFINED WAMR_BUILD_JIT) + set(WAMR_BUILD_JIT 0) +endif () +set(WAMR_BUILD_SIMD 1) +set(WAMR_BUILD_REF_TYPES 1) +set(WAMR_BUILD_LIBC_BUILTIN 1) +set(WAMR_BUILD_LIBC_WASI 1) +set(WAMR_BUILD_MULTI_MODULE 1) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () +endif () + +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +################################################ + +################ application related ################ + +################ WASM MODULES +include(ExternalProject) + +message(CHECK_START "Detecting WASI-SDK") +if(NOT (DEFINED WASI_SDK_DIR OR DEFINED CACHE{WASI_SDK_DIR})) + find_path(WASI_SDK_PARENT + wasi-sdk + PATHS /opt + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if(WASI_SDK_PARENT) + set(WASI_SDK_DIR ${WASI_SDK_PARENT}/wasi-sdk) + endif() +endif() +if(WASI_SDK_DIR) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +message(CHECK_START "Detecting WASI_TOOLCHAIN_FILE at ${WASI_SDK_DIR}") +find_file(WASI_TOOLCHAIN_FILE + wasi-sdk.cmake + PATHS "${WASI_SDK_DIR}/share/cmake" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) +if(WASI_TOOLCHAIN_FILE) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +message(CHECK_START "Detecting WASI_SYS_ROOT at ${WASI_SDK_DIR}") +find_path(WASI_SYS_ROOT + wasi-sysroot + PATHS "${WASI_SDK_DIR}/share" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) +if(WASI_SYS_ROOT) + message(CHECK_PASS "found") + set(WASI_SYS_ROOT ${WASI_SYS_ROOT}/wasi-sysroot) +else() + message(CHECK_FAIL "not found") +endif() + +if((NOT EXISTS ${WASI_SDK_DIR}) OR (NOT EXISTS ${WASI_TOOLCHAIN_FILE}) OR (NOT EXISTS ${WASI_SYS_ROOT})) + message(FATAL_ERROR "Please set the absolute path of wasi-sdk with \'cmake -DWASI_SDK_DIR=XXX\'") +else() + message(STATUS "WASI_SDK_DIR is ${WASI_SDK_DIR}") + message(STATUS "WASI_TOOLCHAIN_FILE is ${WASI_TOOLCHAIN_FILE}") + message(STATUS "WASI_SYS_ROOT is ${WASI_SYS_ROOT}") +endif() + +# .c -> .wasm +ExternalProject_Add(WASM_MODULE + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps + BUILD_ALWAYS TRUE + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_TOOLCHAIN_FILE} + -DCMAKE_SYSROOT=${WASI_SYS_ROOT} + -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps + BUILD_COMMAND ${CMAKE_COMMAND} --build . + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy + ./mA.wasm ${CMAKE_BINARY_DIR} + ./mB.wasm ${CMAKE_BINARY_DIR} + ./mC.wasm ${CMAKE_BINARY_DIR} + ./mD.wasm ${CMAKE_BINARY_DIR} + ./mE.wasm ${CMAKE_BINARY_DIR} +) + +################ WASM MODULES TO AOT +if (WAMR_BUILD_AOT EQUAL 1) + set(WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) + message(CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") + find_file(WAMR_COMPILER + wamrc + PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if(WAMR_COMPILER) + message(CHECK_PASS "found") + else() + message(CHECK_FAIL "not found") + endif() + if((NOT EXISTS ${WAMR_COMPILER}) ) + message(FATAL_ERROR "Please build wamrc under the path=${WAMR_ROOT_DIR}/wamr-compiler/ ") + else() + message(STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") + endif() + + add_custom_target( + wasm_to_aot + ALL + DEPENDS + WASM_MODULE ${WAMR_COMPILER} + COMMAND + ${WAMR_COMPILER} -o mA.aot ./mA.wasm + COMMAND + ${WAMR_COMPILER} -o mB.aot ./mB.wasm + COMMAND + ${WAMR_COMPILER} -o mC.aot ./mC.wasm + WORKING_DIRECTORY + ${CMAKE_BINARY_DIR} + ) +endif() + +################ NATIVE +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable(multi_module src/main.c ${UNCOMMON_SHARED_SOURCE}) + +add_dependencies(multi_module vmlib WASM_MODULE) + +check_pie_supported() +set_target_properties (multi_module PROPERTIES POSITION_INDEPENDENT_CODE ON) + +# libraries +target_link_libraries(multi_module PRIVATE vmlib -lpthread -lm) diff --git a/wasm-micro-runtime/samples/multi-module/README.md b/wasm-micro-runtime/samples/multi-module/README.md new file mode 100644 index 0000000..731ba62 --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/README.md @@ -0,0 +1,20 @@ +# WAMR MULTI-MODUEL SAMPLE +**WAMR supports *multi-module* in both *interpreter* mode and *aot* mode.** + +Multi-modules will determine the running mode based on the type of the main module. + + +``` shell +$ mkdir build +$ cd build +$ cmake .. +$ make +$ # It will build multi_module runtime and +$ # wasm file under the ./build . +$ # If you have built wamrc, +$ # aot file will also genrate. +$ ./multi_module mC.wasm +$ ... +$ ./multi_module mC.aot +$ ... + diff --git a/wasm-micro-runtime/samples/multi-module/src/main.c b/wasm-micro-runtime/samples/multi-module/src/main.c new file mode 100644 index 0000000..c63cf6b --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/src/main.c @@ -0,0 +1,172 @@ +#include +#include +#include +#include "bh_read_file.h" +#include "platform_common.h" +#include "wasm_export.h" + +#if WASM_ENABLE_MULTI_MODULE != 0 +static char *module_search_path = "."; +static bool +module_reader_callback(package_type_t module_type, const char *module_name, + uint8 **p_buffer, uint32 *p_size) +{ + char *file_format = NULL; +#if WASM_ENABLE_INTERP != 0 + if (module_type == Wasm_Module_Bytecode) + file_format = ".wasm"; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_type == Wasm_Module_AoT) + file_format = ".aot"; + +#endif + bh_assert(file_format != NULL); + const char *format = "%s/%s%s"; + int sz = strlen(module_search_path) + strlen("/") + strlen(module_name) + + strlen(file_format) + 1; + char *wasm_file_name = wasm_runtime_malloc(sz); + if (!wasm_file_name) { + return false; + } + snprintf(wasm_file_name, sz, format, module_search_path, module_name, + file_format); + *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_name, p_size); + + wasm_runtime_free(wasm_file_name); + return *p_buffer != NULL; +} + +static void +moudle_destroyer(uint8 *buffer, uint32 size) +{ + if (!buffer) { + return; + } + + wasm_runtime_free(buffer); + buffer = NULL; +} +#endif /* WASM_ENABLE_MULTI_MODULE */ + +/* 10M */ +static char sandbox_memory_space[10 * 1024 * 1024] = { 0 }; +int +main(int argc, char *argv[]) +{ + bool ret = false; + if (argc != 2) { + return -1; + } + char *wasm_file = argv[1]; + /* 16K */ + const uint32 stack_size = 16 * 1024; + const uint32 heap_size = 16 * 1024; + + RuntimeInitArgs init_args = { 0 }; + char error_buf[128] = { 0 }; + /* parameters and return values */ + char *args[1] = { 0 }; + + uint8 *file_buf = NULL; + uint32 file_buf_size = 0; + wasm_module_t module = NULL; + wasm_module_t module1; + wasm_module_inst_t module_inst = NULL; + + /* all malloc() only from the given buffer */ + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = sandbox_memory_space; + init_args.mem_alloc_option.pool.heap_size = sizeof(sandbox_memory_space); + + printf("- wasm_runtime_full_init\n"); + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + goto EXIT; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + printf("- wasm_runtime_set_module_reader\n"); + /* set module reader and destroyer */ + wasm_runtime_set_module_reader(module_reader_callback, moudle_destroyer); +#endif + + /* load WASM byte buffer from WASM bin file */ + if (!(file_buf = + (uint8 *)bh_read_file_to_buffer(wasm_file, &file_buf_size))) + goto RELEASE_RUNTIME; + /* load mC and let WAMR load mA and mB */ + printf("- wasm_runtime_load\n"); + + if (!(module = wasm_runtime_load(file_buf, file_buf_size, error_buf, + sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto RELEASE_BINARY; + } + + /* instantiate the module */ + printf("- wasm_runtime_instantiate\n"); + if (!(module_inst = wasm_runtime_instantiate( + module, stack_size, heap_size, error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto UNLOAD_MODULE; + } + + /* call functions of mC */ + printf("\n----------------------------------------\n"); + printf("call \"C1\", it will return 0x1f:i32, ===> "); + wasm_application_execute_func(module_inst, "C1", 0, args); + printf("call \"C2\", it will call B1() of mB and return 0x15:i32, ===> "); + wasm_application_execute_func(module_inst, "C2", 0, args); + printf("call \"C3\", it will call A1() of mA and return 0xb:i32, ===> "); + wasm_application_execute_func(module_inst, "C3", 0, args); + printf("call \"C4\", it will call B2() of mB and call A1() of mA and " + "return 0xb:i32, ===> "); + wasm_application_execute_func(module_inst, "C4", 0, args); + printf( + "call \"C5\", it will be failed since it is a export function, ===> "); + wasm_application_execute_func(module_inst, "C5", 0, args); + + /* examine module registration a bit */ + module1 = wasm_runtime_find_module_registered("mC"); + if (module1 != NULL) { + printf("unexpected module mC %p != NULL\n", module1); + goto UNLOAD_MODULE; + } + module1 = wasm_runtime_find_module_registered("mA"); + if (module1 == NULL) { + printf("unexpected module mA\n"); + goto UNLOAD_MODULE; + } + module1 = wasm_runtime_find_module_registered("mB"); + if (module1 == NULL) { + printf("unexpected module mB\n"); + goto UNLOAD_MODULE; + } + if (!wasm_runtime_register_module("mC", module, error_buf, + sizeof(error_buf))) { + printf("%s\n", error_buf); + goto UNLOAD_MODULE; + } + module1 = wasm_runtime_find_module_registered("mC"); + if (module1 != module) { + printf("unexpected module mC %p != %p\n", module1, module); + goto UNLOAD_MODULE; + } + + ret = true; + + printf("- wasm_runtime_deinstantiate\n"); + wasm_runtime_deinstantiate(module_inst); +UNLOAD_MODULE: + printf("- wasm_runtime_unload\n"); + wasm_runtime_unload(module); +RELEASE_BINARY: + moudle_destroyer(file_buf, file_buf_size); +RELEASE_RUNTIME: + printf("- wasm_runtime_destroy\n"); + wasm_runtime_destroy(); +EXIT: + return ret ? 0 : 1; +} diff --git a/wasm-micro-runtime/samples/multi-module/wasm-apps/CMakeLists.txt b/wasm-micro-runtime/samples/multi-module/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000..0dbfcc2 --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/wasm-apps/CMakeLists.txt @@ -0,0 +1,89 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.8...3.16) +project(wasm-apps) + +message(CHECK_START "Detecting WABT") +if(NOT (DEFINED WABT_DIR OR DEFINED CACHE{WABT_DIR})) + find_path(WABT_DIR + wabt + PATHS /opt + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if(DEFINED WABT_DIR) + set(WABT_DIR ${WABT_DIR}/wabt) + endif() +endif() +if(WABT_DIR) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +message(CHECK_START "Detecting WASM_OBJDUMP at ${WABT_DIR}") +find_program(WASM_OBJDUMP + wasm-objdump + PATHS "${WABT_DIR}/bin" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) +if(WASM_OBJDUMP) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +message(CHECK_START "Detecting WASM2WAT at ${WABT_DIR}") +find_program(WASM2WAT + wasm2wat + PATHS "${WABT_DIR}/bin" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) +if(WASM2WAT) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +function(COMPILE_WITH_CLANG SOURCE_FILE COMMAND) + get_filename_component(FILE_NAME ${SOURCE_FILE} NAME_WLE) + + set(WASM_MODULE ${FILE_NAME}.wasm) + + set(MAIN_TARGET_NAME MODULE_${FILE_NAME}) + + add_executable(${MAIN_TARGET_NAME} ${SOURCE_FILE}) + set_target_properties(${MAIN_TARGET_NAME} PROPERTIES OUTPUT_NAME ${WASM_MODULE}) + + if(${COMMAND}) + message(STATUS "Generating ${WASM_MODULE} as COMMAND...") + else() + message(STATUS "Generating ${WASM_MODULE} as REACTOR...") + target_link_options(${MAIN_TARGET_NAME} PRIVATE -mexec-model=reactor) + endif() + + if(EXISTS ${WASM2WAT}) + message(STATUS "Dumping ${WASM_MODULE}...") + set(WASM_WAT ${FILE_NAME}.wat) + set(DUMP_TARGET_NAME DUMP_${FILE_NAME}) + add_custom_command(OUTPUT ${WASM_WAT} + COMMAND ${WASM2WAT} --enable-all -o ${WASM_WAT} ${WASM_MODULE} + COMMENT "Dumping ${WASM_MODULE}..." + DEPENDS ${MAIN_TARGET_NAME} + ) + + add_custom_target(${DUMP_TARGET_NAME} ALL + DEPENDS ${WASM_WAT} + ) + endif() +endfunction() + +compile_with_clang(mA.c OFF) +compile_with_clang(mB.c OFF) +compile_with_clang(mC.c ON) +compile_with_clang(mD.cpp ON) +compile_with_clang(mE.cpp OFF) + diff --git a/wasm-micro-runtime/samples/multi-module/wasm-apps/mA.c b/wasm-micro-runtime/samples/multi-module/wasm-apps/mA.c new file mode 100644 index 0000000..663ec81 --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/wasm-apps/mA.c @@ -0,0 +1,13 @@ +__attribute__((export_name("A1"))) int +A1() +{ + return 11; +} + +int +A2() +{ + return 12; +} + +/* mA is a reactor. it doesn't need a main() */ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/multi-module/wasm-apps/mB.c b/wasm-micro-runtime/samples/multi-module/wasm-apps/mB.c new file mode 100644 index 0000000..a912d1d --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/wasm-apps/mB.c @@ -0,0 +1,23 @@ +__attribute__((import_module("mA"))) +__attribute__((import_name("A1"))) extern int +A1(); + +__attribute__((export_name("B1"))) int +B1() +{ + return 21; +} + +__attribute__((export_name("B2"))) int +B2() +{ + return A1(); +} + +int +B3() +{ + return 23; +} + +/* mA is a reactor. it doesn't need a main() */ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/multi-module/wasm-apps/mC.c b/wasm-micro-runtime/samples/multi-module/wasm-apps/mC.c new file mode 100644 index 0000000..8b19a5b --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/wasm-apps/mC.c @@ -0,0 +1,51 @@ +#include +#include + +__attribute__((import_module("mA"))) +__attribute__((import_name("A1"))) extern int +A1(); + +__attribute__((import_module("mB"))) +__attribute__((import_name("B1"))) extern int +B1(); + +__attribute__((import_module("mB"))) +__attribute__((import_name("B2"))) extern int +B2(); + +__attribute__((export_name("C1"))) int +C1() +{ + return 31; +} + +__attribute__((export_name("C2"))) int +C2() +{ + return B1(); +} + +__attribute__((export_name("C3"))) int +C3() +{ + return A1(); +} + +__attribute__((export_name("C4"))) int +C4() +{ + return B2(); +} + +int +C5() +{ + return C1() + C2() + C3() + 35; +} + +int +main() +{ + printf("%u\n", C5()); + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/wasm-micro-runtime/samples/multi-module/wasm-apps/mD.cpp b/wasm-micro-runtime/samples/multi-module/wasm-apps/mD.cpp new file mode 100644 index 0000000..d70998c --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/wasm-apps/mD.cpp @@ -0,0 +1,74 @@ +#include +#include +#include + +static void +bye_main() +{ + std::cout << "mD " << __FUNCTION__ << std::endl; +} + +static void +bye_setup() +{ + std::cout << "mD " << __FUNCTION__ << std::endl; +} + +static void +bye_func() +{ + std::cout << "mD " << __FUNCTION__ << std::endl; +} + +void +func3() __attribute__((__import_module__("mE"), __import_name__("func1"))); + +void +func4() __attribute__((__import_module__("mE"), __import_name__("func2"))); + +void +func1() +{ + std::printf("mD %s\n", __FUNCTION__); + if (std::atexit(bye_func) != 0) { + std::perror("register an atexit handler failed"); + } + func3(); +} + +void +func2() +{ + std::printf("mD %s\n", __FUNCTION__); + func4(); +} + +__attribute__((constructor)) void +setup() +{ + std::cout << "mD " << __FUNCTION__ << std::endl; + if (std::atexit(bye_setup) != 0) { + std::perror("register an atexit handler failed"); + } +} + +__attribute__((destructor)) void +teardown() +{ + std::cout << "mD " << __FUNCTION__ << std::endl; +} + +int +main() +{ + std::printf("mD %s\n", __FUNCTION__); + + if (std::atexit(bye_main) != 0) { + std::perror("register an atexit handler failed"); + return EXIT_FAILURE; + } + + func1(); + func2(); + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/samples/multi-module/wasm-apps/mE.cpp b/wasm-micro-runtime/samples/multi-module/wasm-apps/mE.cpp new file mode 100644 index 0000000..11e70af --- /dev/null +++ b/wasm-micro-runtime/samples/multi-module/wasm-apps/mE.cpp @@ -0,0 +1,45 @@ +#include +#include +#include + +static void +bye_setup() +{ + std::cout << "mE " << __FUNCTION__ << std::endl; +} + +static void +bye_func() +{ + std::cout << "mE " << __FUNCTION__ << std::endl; +} + +__attribute__((constructor)) void +setup() +{ + std::cout << "mE " << __FUNCTION__ << std::endl; + if (std::atexit(bye_setup) != 0) { + std::perror("register an atexit handler failed"); + } +} + +__attribute__((destructor)) void +teardown() +{ + std::cout << "mE " << __FUNCTION__ << std::endl; +} + +__attribute__((export_name("func1"))) void +func1() +{ + std::cout << "mE " << __FUNCTION__ << std::endl; + if (std::atexit(bye_func) != 0) { + std::perror("register an atexit handler failed"); + } +} + +__attribute__((export_name("func2"))) void +func2() +{ + std::cout << "mE " << __FUNCTION__ << std::endl; +} diff --git a/wasm-micro-runtime/samples/multi-thread/CMakeLists.txt b/wasm-micro-runtime/samples/multi-thread/CMakeLists.txt new file mode 100644 index 0000000..67819ec --- /dev/null +++ b/wasm-micro-runtime/samples/multi-thread/CMakeLists.txt @@ -0,0 +1,81 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +include(CheckPIESupported) + +project(pthread) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Resetdefault linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_JIT 0) +set(WAMR_BUILD_LIBC_BUILTIN 1) +set(WAMR_BUILD_FAST_INTERP 1) +set(WAMR_BUILD_LIB_PTHREAD 1) +set(WAMR_BUILD_LIB_PTHREAD_SEMAPHORE 1) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +################################################ + + +################ wasm application ################ +add_subdirectory(wasm-apps) + +################ wamr runtime ################ +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +set (RUNTIME_SOURCE_ALL + ${CMAKE_CURRENT_LIST_DIR}/../../product-mini/platforms/linux/main.c + ${UNCOMMON_SHARED_SOURCE} +) +add_executable (iwasm ${RUNTIME_SOURCE_ALL}) +check_pie_supported() +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(iwasm vmlib -lpthread -lm -ldl) + diff --git a/wasm-micro-runtime/samples/multi-thread/wasm-apps/CMakeLists.txt b/wasm-micro-runtime/samples/multi-thread/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000..d7352e4 --- /dev/null +++ b/wasm-micro-runtime/samples/multi-thread/wasm-apps/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 2.8) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () +set(CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=32768") +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set (DEFINED_SYMBOLS +"${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set (CMAKE_EXE_LINKER_FLAGS + "-Wl,--shared-memory,--max-memory=131072, \ + -Wl,--no-entry,--strip-all, \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors \ + -Wl,--export=main -Wl,--export=__main_argc_argv \ + -Wl,--allow-undefined" + #-Wl,--allow-undefined-file=${DEFINED_SYMBOLS}" +) + +add_executable(test.wasm main.c) +target_link_libraries(test.wasm) + +add_executable(main_thread_exception.wasm main_thread_exception.c) +target_link_libraries(main_thread_exception.wasm) + +add_executable(main_global_atomic.wasm main_global_atomic.c) +target_link_libraries(main_global_atomic.wasm) \ No newline at end of file diff --git a/wasm-micro-runtime/samples/multi-thread/wasm-apps/main.c b/wasm-micro-runtime/samples/multi-thread/wasm-apps/main.c new file mode 100644 index 0000000..2b9581f --- /dev/null +++ b/wasm-micro-runtime/samples/multi-thread/wasm-apps/main.c @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include + +static pthread_mutex_t mutex; +static pthread_cond_t cond; +static sem_t *sem; + +static void * +thread(void *arg) +{ + int *num = (int *)arg; + + pthread_mutex_lock(&mutex); + printf("thread start \n"); + + for (int i = 0; i < 10; i++) { + *num = *num + 1; + printf("num: %d\n", *num); + } + + pthread_cond_signal(&cond); + pthread_mutex_unlock(&mutex); + sem_post(sem); + + printf("thread exit \n"); + + return NULL; +} + +int +main(int argc, char *argv[]) +{ + pthread_t tid; + int num = 0, ret = -1; + + if (pthread_mutex_init(&mutex, NULL) != 0) { + printf("Failed to init mutex.\n"); + return -1; + } + if (pthread_cond_init(&cond, NULL) != 0) { + printf("Failed to init cond.\n"); + goto fail1; + } + + // O_CREAT and S_IRGRPS_IRGRP | S_IWGRP on linux (glibc), initial value is 0 + + if (!(sem = sem_open("tessstsem", 0100, 0x10 | 0x20, 0))) { + printf("Failed to open sem. %p\n", sem); + goto fail2; + } + + pthread_mutex_lock(&mutex); + if (pthread_create(&tid, NULL, thread, &num) != 0) { + printf("Failed to create thread.\n"); + pthread_mutex_unlock(&mutex); + goto fail3; + } + + printf("cond wait start\n"); + pthread_cond_wait(&cond, &mutex); + pthread_mutex_unlock(&mutex); + printf("cond wait success.\n"); + + if (sem_wait(sem) != 0) { + printf("Failed to wait sem.\n"); + } + else { + printf("sem wait success.\n"); + } + + if (pthread_join(tid, NULL) != 0) { + printf("Failed to join thread.\n"); + } + + ret = 0; + +fail3: + sem_close(sem); + sem_unlink("tessstsem"); +fail2: + pthread_cond_destroy(&cond); +fail1: + pthread_mutex_destroy(&mutex); + + return ret; +} \ No newline at end of file diff --git a/wasm-micro-runtime/samples/multi-thread/wasm-apps/main_global_atomic.c b/wasm-micro-runtime/samples/multi-thread/wasm-apps/main_global_atomic.c new file mode 100644 index 0000000..dafbea8 --- /dev/null +++ b/wasm-micro-runtime/samples/multi-thread/wasm-apps/main_global_atomic.c @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#define MAX_NUM_THREADS 4 +#define NUM_ITER 1000 + +int g_count = 0; + +static void * +thread(void *arg) +{ + for (int i = 0; i < NUM_ITER; i++) { + __atomic_fetch_add(&g_count, 1, __ATOMIC_SEQ_CST); + } + + return NULL; +} + +int +main(int argc, char **argv) +{ + pthread_t tids[MAX_NUM_THREADS]; + + for (int i = 0; i < MAX_NUM_THREADS; i++) { + if (pthread_create(&tids[i], NULL, thread, NULL) != 0) { + printf("Thread creation failed\n"); + } + } + + for (int i = 0; i < MAX_NUM_THREADS; i++) { + if (pthread_join(tids[i], NULL) != 0) { + printf("Thread join failed\n"); + } + } + + printf("Value of counter after update: %d (expected=%d)\n", g_count, + MAX_NUM_THREADS * NUM_ITER); + if (g_count != MAX_NUM_THREADS * NUM_ITER) { + __builtin_trap(); + } + + return -1; +} \ No newline at end of file diff --git a/wasm-micro-runtime/samples/multi-thread/wasm-apps/main_thread_exception.c b/wasm-micro-runtime/samples/multi-thread/wasm-apps/main_thread_exception.c new file mode 100644 index 0000000..80f170d --- /dev/null +++ b/wasm-micro-runtime/samples/multi-thread/wasm-apps/main_thread_exception.c @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +typedef struct ThreadArgs { + int start; + int length; +} ThreadArgs; + +void * +thread(void *args) +{ + while (1) { + /* When other threads (including main thread) throw exception, + this thread can successfully exit the dead loop */ + } +} + +int +main() +{ + pthread_t tids; + + if (pthread_create(&tids, NULL, thread, NULL) != 0) { + printf("pthread_create failed\n"); + } + + /* Trigger an exception */ + __builtin_trap(); + + return 0; +} diff --git a/wasm-micro-runtime/samples/native-lib/CMakeLists.txt b/wasm-micro-runtime/samples/native-lib/CMakeLists.txt new file mode 100644 index 0000000..d8201ba --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/CMakeLists.txt @@ -0,0 +1,91 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +include(CheckPIESupported) + +project(native_lib) + +################ runtime settings ############## +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported are: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_FAST_INTERP 1) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + +# build out libiwasm +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +# Note: we build libiwasm as a shared library here so that it can be +# shared between iwasm and native libraries. +add_library(libiwasm SHARED ${WAMR_RUNTIME_LIB_SOURCE}) +set_target_properties (libiwasm PROPERTIES OUTPUT_NAME iwasm) + +################ wamr runtime ################### +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +set (RUNTIME_SOURCE_ALL + ${WAMR_ROOT_DIR}/product-mini/platforms/posix/main.c + ${UNCOMMON_SHARED_SOURCE} +) + +add_executable (iwasm ${RUNTIME_SOURCE_ALL}) + +check_pie_supported() +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) + +target_link_libraries(iwasm libiwasm -lpthread -lm -ldl) + +################ native libraries ############### +add_library (test_add SHARED test_add.c) +add_library (test_sqrt SHARED test_sqrt.c) +add_library (test_hello SHARED test_hello.c) +# Note: Unlike simpler examples above, test_hello2 directly uses +# the API provided by the libiwasm library. +add_library (test_hello2 SHARED test_hello2.c) +target_link_libraries(test_hello2 libiwasm) + +################ wasm application ############### +add_subdirectory(wasm-app) diff --git a/wasm-micro-runtime/samples/native-lib/README.md b/wasm-micro-runtime/samples/native-lib/README.md new file mode 100644 index 0000000..be1cc6e --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/README.md @@ -0,0 +1,78 @@ +# "native-lib" sample introduction + +This sample demonstrates how to write required interfaces in native library, build it into a shared library and register the shared library to iwasm. + +The native library should provide `get_native_lib` API for iwasm to return the native library info, including the module name, the native symbol list and the native symbol count, so that iwasm can use them to regiter the native library, for example: + +```C +static int +foo_wrapper(wasm_exec_env_t exec_env, int x, int y) +{ + return x + y; +} + +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } + +static NativeSymbol native_symbols[] = { + REG_NATIVE_FUNC(foo, "(ii)i") +}; + +uint32_t +get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) +{ + *p_module_name = "env"; + *p_native_symbols = native_symbols; + return sizeof(native_symbols) / sizeof(NativeSymbol); +} +``` + +## Preparation + +Please install WASI SDK, download the [wasi-sdk release](https://github.com/WebAssembly/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. + +## Build the sample + +```bash +mkdir build +cd build +cmake .. +make +``` + +`iwasm`, one wasm module `test.wasm` and two shared libraries `libtest_add.so`, `libtest_sqrt.so` +will be generated. + +## Run workload + +### Linux + +```bash +cd build +./iwasm --native-lib=./libtest_add.so --native-lib=./libtest_sqrt.so --native-lib=./libtest_hello.so --native-lib=./libtest_hello2.so wasm-app/test.wasm +``` + +### macOS + +```bash +cd build +./iwasm --native-lib=libtest_add.dylib --native-lib=libtest_sqrt.dylib --native-lib=libtest_hello.dylib --native-lib=libtest_hello2.dylib wasm-app/test.wasm +``` + +The output is: + +```bash +init_native_lib in test_hello2.c called +Hello World! +10 + 20 = 30 +sqrt(10, 20) = 500 +test_hello("main", 0x0, 0) = 41 +malloc(42) = 0x24e8 +test_hello("main", 0x24e8, 42) = 41 +Message from test_hello: Hello, main. This is test_hello_wrapper! +test_hello2("main", 0x0, 0) = 85 +malloc(86) = 0x24e8 +test_hello2("main", 0x24e8, 86) = 85 +Message from test_hello2: Hello, main. This is test_hello2_wrapper! Your wasm_module_inst_t is 0x7fe0e6804280. +deinit_native_lib in test_hello2.c called +``` diff --git a/wasm-micro-runtime/samples/native-lib/test_add.c b/wasm-micro-runtime/samples/native-lib/test_add.c new file mode 100644 index 0000000..ac44c61 --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/test_add.c @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include "wasm_export.h" + +static int +test_add_wrapper(wasm_exec_env_t exec_env, int x, int y) +{ + return x + y; +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } + +static NativeSymbol native_symbols[] = { + REG_NATIVE_FUNC(test_add, "(ii)i") +}; +/* clang-format on */ + +uint32_t +get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) +{ + *p_module_name = "env"; + *p_native_symbols = native_symbols; + return sizeof(native_symbols) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/samples/native-lib/test_hello.c b/wasm-micro-runtime/samples/native-lib/test_hello.c new file mode 100644 index 0000000..c322ec2 --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/test_hello.c @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include "wasm_export.h" + +static int +test_hello_wrapper(wasm_exec_env_t exec_env, const char *name, char *result, + size_t resultlen) +{ + return snprintf(result, resultlen, "Hello, %s. This is %s!\n", name, + __func__); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } + +static NativeSymbol native_symbols[] = { + REG_NATIVE_FUNC(test_hello, "($*~)i") +}; +/* clang-format on */ + +uint32_t +get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) +{ + *p_module_name = "env"; + *p_native_symbols = native_symbols; + return sizeof(native_symbols) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/samples/native-lib/test_hello2.c b/wasm-micro-runtime/samples/native-lib/test_hello2.c new file mode 100644 index 0000000..53ea663 --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/test_hello2.c @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* + * This example basically does the same thing as test_hello.c, + * using wasm_export.h API. + */ + +#include +#include + +#include "wasm_export.h" + +static int +test_hello2_wrapper(wasm_exec_env_t exec_env, uint32_t nameaddr, + uint32_t resultaddr, uint32_t resultlen) +{ + /* + * Perform wasm_runtime_malloc to check if the runtime has been + * initialized as expected. + * This would fail with "memory hasn't been initialize" error + * unless we are not sharing a runtime with the loader app. (iwasm) + */ + void *p = wasm_runtime_malloc(1); + if (p == NULL) { + return -1; + } + wasm_runtime_free(p); + + wasm_module_inst_t inst = wasm_runtime_get_module_inst(exec_env); + if (!wasm_runtime_validate_app_str_addr(inst, (uint64_t)nameaddr) + || !wasm_runtime_validate_app_addr(inst, (uint64_t)resultaddr, + (uint64_t)resultlen)) { + return -1; + } + const char *name = + wasm_runtime_addr_app_to_native(inst, (uint64_t)nameaddr); + char *result = wasm_runtime_addr_app_to_native(inst, (uint64_t)resultaddr); + return snprintf(result, resultlen, + "Hello, %s. This is %s! Your wasm_module_inst_t is %p.\n", + name, __func__, inst); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } + +static NativeSymbol native_symbols[] = { + REG_NATIVE_FUNC(test_hello2, "(iii)i") +}; +/* clang-format on */ + +uint32_t +get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) +{ + *p_module_name = "env"; + *p_native_symbols = native_symbols; + return sizeof(native_symbols) / sizeof(NativeSymbol); +} + +int +init_native_lib() +{ + printf("%s in test_hello2.c called\n", __func__); + return 0; +} + +void +deinit_native_lib() +{ + printf("%s in test_hello2.c called\n", __func__); +} diff --git a/wasm-micro-runtime/samples/native-lib/test_sqrt.c b/wasm-micro-runtime/samples/native-lib/test_sqrt.c new file mode 100644 index 0000000..ecf7c98 --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/test_sqrt.c @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include "wasm_export.h" + +static int +test_sqrt_wrapper(wasm_exec_env_t exec_env, int x, int y) +{ + return x * x + y * y; +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } + +static NativeSymbol native_symbols[] = { + REG_NATIVE_FUNC(test_sqrt, "(ii)i") +}; +/* clang-format on */ + +uint32_t +get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) +{ + *p_module_name = "env"; + *p_native_symbols = native_symbols; + return sizeof(native_symbols) / sizeof(NativeSymbol); +} diff --git a/wasm-micro-runtime/samples/native-lib/wasm-app/CMakeLists.txt b/wasm-micro-runtime/samples/native-lib/wasm-app/CMakeLists.txt new file mode 100644 index 0000000..ffcd900 --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/wasm-app/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.0) +project(wasm-app) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () + +set (CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_FLAGS "-nostdlib") +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set (CMAKE_EXE_LINKER_FLAGS + "-Wl,--max-memory=131072 -z stack-size=8192 \ + -Wl,--no-entry,--strip-all \ + -Wl,--export=__main_argc_argv \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--allow-undefined" +) + +add_executable(test.wasm main.c) +target_link_libraries(test.wasm) diff --git a/wasm-micro-runtime/samples/native-lib/wasm-app/main.c b/wasm-micro-runtime/samples/native-lib/wasm-app/main.c new file mode 100644 index 0000000..dba652d --- /dev/null +++ b/wasm-micro-runtime/samples/native-lib/wasm-app/main.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +int +test_add(int x, int y); + +int +test_sqrt(int x, int y); + +int +test_hello(const char *name, char *buf, size_t buflen); + +int +test_hello2(const char *name, char *buf, size_t buflen); + +int +main(int argc, char **argv) +{ + const char *name = __func__; + char *buf; + size_t buflen; + int x = 10, y = 20, res; + + printf("Hello World!\n"); + + res = test_add(x, y); + printf("%d + %d = %d\n", x, y, res); + + res = test_sqrt(x, y); + printf("sqrt(%d, %d) = %d\n", x, y, res); + + res = test_hello(name, NULL, 0); + printf("test_hello(\"%s\", %p, %zu) = %d\n", name, NULL, (size_t)0, res); + if (res == -1) { + return -1; + } + buflen = res + 1; + buf = malloc(buflen); + printf("malloc(%zu) = %p\n", buflen, buf); + res = test_hello(__func__, buf, buflen); + if (res == -1) { + return -1; + } + printf("test_hello(\"%s\", %p, %zu) = %d\n", name, buf, buflen, res); + printf("Message from test_hello: %s", buf); + free(buf); + + res = test_hello2(name, NULL, 0); + printf("test_hello2(\"%s\", %p, %zu) = %d\n", name, NULL, (size_t)0, res); + if (res == -1) { + return -1; + } + buflen = res + 1; + buf = malloc(buflen); + printf("malloc(%zu) = %p\n", buflen, buf); + res = test_hello2(__func__, buf, buflen); + if (res == -1) { + return -1; + } + printf("test_hello2(\"%s\", %p, %zu) = %d\n", name, buf, buflen, res); + printf("Message from test_hello2: %s", buf); + free(buf); + + return 0; +} diff --git a/wasm-micro-runtime/samples/native-stack-overflow/.gitignore b/wasm-micro-runtime/samples/native-stack-overflow/.gitignore new file mode 100644 index 0000000..0fa8a76 --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/.gitignore @@ -0,0 +1 @@ +/out/ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/native-stack-overflow/CMakeLists.txt b/wasm-micro-runtime/samples/native-stack-overflow/CMakeLists.txt new file mode 100644 index 0000000..cdc6acc --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/CMakeLists.txt @@ -0,0 +1,87 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (native-stack-overflow) +else() + project (native-stack-overflow C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 0) +set (WAMR_BUILD_LIBC_WASI 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (native-stack-overflow src/main.c src/native_impl.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (native-stack-overflow PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (native-stack-overflow vmlib -lm -ldl -lpthread) +else () + target_link_libraries (native-stack-overflow vmlib -lm -ldl -lpthread -lrt) +endif () diff --git a/wasm-micro-runtime/samples/native-stack-overflow/README.md b/wasm-micro-runtime/samples/native-stack-overflow/README.md new file mode 100644 index 0000000..431c587 --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/README.md @@ -0,0 +1,4 @@ +The "native-stack-overflow" sample project +========================================== + +This sample examines native stack overflow detection mechanisms. diff --git a/wasm-micro-runtime/samples/native-stack-overflow/build.sh b/wasm-micro-runtime/samples/native-stack-overflow/build.sh new file mode 100755 index 0000000..3961098 --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/build.sh @@ -0,0 +1,75 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +CURR_DIR=$PWD +WAMR_DIR=${PWD}/../.. +OUT_DIR=${PWD}/out + +WASM_APPS=${PWD}/wasm-apps + + +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +mkdir ${OUT_DIR}/wasm-apps + + +echo "##################### build (default)" +cd ${CURR_DIR} +mkdir -p cmake_build +cd cmake_build +cmake .. +make -j 4 +if [ $? != 0 ];then + echo "BUILD_FAIL native-stack-overflow exit as $?\n" + exit 2 +fi +cp -a native-stack-overflow ${OUT_DIR} + +echo "##################### build (WAMR_DISABLE_HW_BOUND_CHECK=1)" +cd ${CURR_DIR} +mkdir -p cmake_build_disable_hw_bound +cd cmake_build_disable_hw_bound +cmake -D WAMR_DISABLE_HW_BOUND_CHECK=1 .. +make -j 4 +if [ $? != 0 ];then + echo "BUILD_FAIL native-stack-overflow exit as $?\n" + exit 2 +fi +cp -a native-stack-overflow ${OUT_DIR}/native-stack-overflow.WAMR_DISABLE_HW_BOUND_CHECK + +echo + +echo "##################### build wasm apps" + +cd ${WASM_APPS} + +for i in `ls *.c` +do +APP_SRC="$i" +OUT_FILE=${i%.*}.wasm + +# use WAMR SDK to build out the .wasm binary +/opt/wasi-sdk/bin/clang \ + -mexec-model=reactor \ + -Os -z stack-size=4096 -Wl,--initial-memory=65536 \ + -Wl,--allow-undefined \ + -o ${OUT_DIR}/wasm-apps/${OUT_FILE} ${APP_SRC} + +if [ -f ${OUT_DIR}/wasm-apps/${OUT_FILE} ]; then + echo "build ${OUT_FILE} success" +else + echo "build ${OUT_FILE} fail" +fi +done +echo "#################### build wasm apps done" + +echo "#################### aot-compile" +WAMRC=${WAMR_DIR}/wamr-compiler/build/wamrc +${WAMRC} -o ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot ${OUT_DIR}/wasm-apps/${OUT_FILE} + +echo "#################### aot-compile (--bounds-checks=1)" +${WAMRC} -o ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot.bounds-checks --bounds-checks=1 ${OUT_DIR}/wasm-apps/${OUT_FILE} diff --git a/wasm-micro-runtime/samples/native-stack-overflow/clean.sh b/wasm-micro-runtime/samples/native-stack-overflow/clean.sh new file mode 100755 index 0000000..5eba9bc --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/clean.sh @@ -0,0 +1 @@ +rm -r cmake_build cmake_build_disable_hw_bound out diff --git a/wasm-micro-runtime/samples/native-stack-overflow/run.sh b/wasm-micro-runtime/samples/native-stack-overflow/run.sh new file mode 100755 index 0000000..aa2baa5 --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/run.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "====== Interpreter" +out/native-stack-overflow out/wasm-apps/testapp.wasm + +echo +echo "====== AOT" +out/native-stack-overflow out/wasm-apps/testapp.wasm.aot + +echo +echo "====== AOT WAMR_DISABLE_HW_BOUND_CHECK=1" +out/native-stack-overflow.WAMR_DISABLE_HW_BOUND_CHECK out/wasm-apps/testapp.wasm.aot.bounds-checks diff --git a/wasm-micro-runtime/samples/native-stack-overflow/src/main.c b/wasm-micro-runtime/samples/native-stack-overflow/src/main.c new file mode 100644 index 0000000..fa86e4d --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/src/main.c @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2024 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" + +uint32_t +host_consume_stack_and_call_indirect(wasm_exec_env_t exec_env, uint32_t funcidx, + uint32_t x, uint32_t stack); +uint32_t +host_consume_stack(wasm_exec_env_t exec_env, uint32_t stack); + +extern unsigned int nest; + +static NativeSymbol native_symbols[] = { + { "host_consume_stack_and_call_indirect", + host_consume_stack_and_call_indirect, "(iii)i", NULL }, + { "host_consume_stack", host_consume_stack, "(i)i", NULL }, +}; + +struct record { + bool failed; + bool leaked; + char exception[128]; /* EXCEPTION_BUF_LEN */ +}; + +void +print_record(unsigned int start, unsigned int end, const struct record *rec) +{ + printf("%5u - %5u | %6s | %6s | %s\n", start, end, + rec->failed ? "failed" : "ok", rec->leaked ? "leaked" : "ok", + rec->exception); +} + +int +main(int argc, char **argv) +{ + char *buffer; + char error_buf[128]; + + if (argc != 2) { + return 2; + } + char *module_path = argv[1]; + + wasm_module_t module = NULL; + uint32 buf_size; + uint32 stack_size = 4096; + /* + * disable app heap. + * - we use wasi + * - https://github.com/bytecodealliance/wasm-micro-runtime/issues/2275 + */ + uint32 heap_size = 0; + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + init_args.mem_alloc_type = Alloc_With_System_Allocator; + init_args.n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); + init_args.native_module_name = "env"; + init_args.native_symbols = native_symbols; + if (!wasm_runtime_full_init(&init_args)) { + printf("wasm_runtime_full_init failed.\n"); + return -1; + } + + buffer = bh_read_file_to_buffer(module_path, &buf_size); + if (!buffer) { + printf("bh_read_file_to_buffer failed\n"); + goto fail; + } + + module = wasm_runtime_load((uint8 *)buffer, buf_size, error_buf, + sizeof(error_buf)); + if (!module) { + printf("wasm_runtime_load failed: %s\n", error_buf); + goto fail; + } + + /* header */ + printf(" stack size | fail? | leak? | exception\n"); + printf("-------------------------------------------------------------------" + "--------\n"); + + unsigned int stack; + unsigned int prevstack; + unsigned int stack_range_start = 0; + unsigned int stack_range_end = 4096 * 6; + unsigned int step = 16; + struct record rec0; + struct record rec1; + struct record *rec = &rec0; + struct record *prevrec = &rec1; + bool have_prevrec = false; + for (stack = stack_range_start; stack < stack_range_end; stack += step) { + wasm_module_inst_t module_inst = NULL; + wasm_exec_env_t exec_env = NULL; + bool failed = true; + const char *exception = NULL; + nest = 0; + + module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + if (!module_inst) { + printf("wasm_runtime_instantiate failed: %s\n", error_buf); + goto fail2; + } + + exec_env = wasm_runtime_create_exec_env(module_inst, stack_size); + if (!exec_env) { + printf("wasm_runtime_create_exec_env failed\n"); + goto fail2; + } + + const char *funcname = "test"; + wasm_function_inst_t func = + wasm_runtime_lookup_function(module_inst, funcname); + if (!func) { + printf("wasm_runtime_lookup_function failed for %s\n", funcname); + goto fail2; + } + + /* note: the function type is (ii)i */ + uint32_t wasm_argv[] = { + stack, + 30, + }; + uint32_t wasm_argc = 2; + if (!wasm_runtime_call_wasm(exec_env, func, wasm_argc, wasm_argv)) { + exception = wasm_runtime_get_exception(module_inst); + goto fail2; + } + failed = false; + fail2: + /* + * note: non-zero "nest" here demonstrates resource leak on longjmp + * from signal handler. + * cf. + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/3320 + */ + memset(rec, 0, sizeof(*rec)); + rec->failed = failed; + rec->leaked = nest != 0; + strncpy(rec->exception, exception ? exception : "", + sizeof(rec->exception)); + if (have_prevrec && memcmp(prevrec, rec, sizeof(*rec))) { + print_record(prevstack, stack, prevrec); + have_prevrec = false; + } + if (!have_prevrec) { + prevstack = stack; + struct record *tmp = prevrec; + prevrec = rec; + rec = tmp; + have_prevrec = true; + } + if (exec_env) { + wasm_runtime_destroy_exec_env(exec_env); + } + if (module_inst) { + wasm_runtime_deinstantiate(module_inst); + } + } + if (have_prevrec) { + print_record(prevstack, stack, prevrec); + } + +fail: + if (module) { + wasm_runtime_unload(module); + } + if (buffer) { + BH_FREE(buffer); + } + wasm_runtime_destroy(); +} diff --git a/wasm-micro-runtime/samples/native-stack-overflow/src/native_impl.c b/wasm-micro-runtime/samples/native-stack-overflow/src/native_impl.c new file mode 100644 index 0000000..6eb349c --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/src/native_impl.c @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include "wasm_export.h" +#include "bh_platform.h" + +/* + * this "nest" var has two purposes: + * - prevent tail-call optimization + * - detect possible resource leak + */ +unsigned int nest = 0; +ptrdiff_t prev_diff = 0; + +uint32_t +call_indirect(wasm_exec_env_t exec_env, uint32_t funcidx, uint32_t x) +{ + uint32_t argv[1] = { + x, + }; + uint32_t argc = 1; + if (!wasm_runtime_call_indirect(exec_env, funcidx, argc, argv)) { + /* failed */ + return 0; + } + return argv[0]; +} + +uint32_t +host_consume_stack_and_call_indirect(wasm_exec_env_t exec_env, uint32_t funcidx, + uint32_t x, uint32_t stack) +{ + void *boundary = os_thread_get_stack_boundary(); + void *fp = __builtin_frame_address(0); + ptrdiff_t diff = fp - boundary; + if (diff > stack) { + prev_diff = diff; + nest++; + uint32_t ret = + host_consume_stack_and_call_indirect(exec_env, funcidx, x, stack); + nest--; + return ret; + } + return call_indirect(exec_env, funcidx, x); +} + +static uint32_t +consume_stack1(wasm_exec_env_t exec_env, void *base, uint32_t stack) +{ + void *fp = __builtin_frame_address(0); + ptrdiff_t diff = (unsigned char *)base - (unsigned char *)fp; + assert(diff > 0); + char buf[16]; + memset_s(buf, sizeof(buf), 0, sizeof(buf)); + if (diff > stack) { + return diff; + } + return consume_stack1(exec_env, base, stack); +} + +uint32_t +host_consume_stack(wasm_exec_env_t exec_env, uint32_t stack) +{ + void *base = __builtin_frame_address(0); + return consume_stack1(exec_env, base, stack); +} diff --git a/wasm-micro-runtime/samples/native-stack-overflow/wasm-apps/testapp.c b/wasm-micro-runtime/samples/native-stack-overflow/wasm-apps/testapp.c new file mode 100644 index 0000000..bc517c3 --- /dev/null +++ b/wasm-micro-runtime/samples/native-stack-overflow/wasm-apps/testapp.c @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Midokura Japan KK. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +uint32_t +host_consume_stack_and_call_indirect(int (*)(int), uint32_t, uint32_t); +uint32_t host_consume_stack(uint32_t); + +int +cb(int x) +{ + return x * x; +} + +int +consume_stack_cb(int x) +{ + /* + * intentions: + * + * - consume native stack by making recursive calls + * + * - avoid tail-call optimization (either by the C compiler or + * aot-compiler) + */ + if (x == 0) { + return 0; + } + return consume_stack_cb(x - 1) + 1; +} + +int +host_consume_stack_cb(int x) +{ + return host_consume_stack(x); +} + +__attribute__((export_name("test"))) uint32_t +test(uint32_t native_stack, uint32_t recurse_count) +{ + uint32_t ret; + ret = host_consume_stack_and_call_indirect(cb, 321, native_stack); + ret = host_consume_stack_and_call_indirect(consume_stack_cb, recurse_count, + native_stack); +#if 0 /* notyet */ + ret = host_consume_stack_and_call_indirect(host_consume_stack_cb, 1000000, + native_stack); +#endif + return 42; +} diff --git a/wasm-micro-runtime/samples/ref-types/CMakeLists.txt b/wasm-micro-runtime/samples/ref-types/CMakeLists.txt new file mode 100644 index 0000000..fd18e63 --- /dev/null +++ b/wasm-micro-runtime/samples/ref-types/CMakeLists.txt @@ -0,0 +1,138 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project(ref-types) +else() + project (ref-types C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Resetdefault linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +if(NOT DEFINED WAMR_BUILD_INTERP) + set(WAMR_BUILD_INTERP 1) +endif() + +if(NOT DEFINED WAMR_BUILD_AOT) + set(WAMR_BUILD_AOT 0) +endif() + +if(NOT DEFINED WAMR_BUILD_JOT) + set(WAMR_BUILD_JIT 0) +endif() + +if(NOT DEFINED WAMR_BUILD_FAST_INTERP) + set(WAMR_BUILD_FAST_INTERP 1) +endif() + +if(NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + set(WAMR_BUILD_LIBC_BUILTIN 1) +endif() + +# Enable reference-types feature +set(WAMR_BUILD_REF_TYPES 1) + +if (NOT MSVC) + # compiling and linking flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif() +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +set(WAMRC ${WAMR_ROOT_DIR}/wamr-compiler/build/wamrc) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +if (MSVC) + target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) +endif() + +################ application related ################ +## locate wat2wasm +find_program(WAT2WASM + wat2wasm + PATHS /opt/wabt/bin + REQUIRED +) + +if(NOT WAT2WASM) + message(SEND_ERROR "can not find wat2wasm") +else () + execute_process(COMMAND ${WAT2WASM} --version + OUTPUT_VARIABLE WAT2WASM_VERSION_OUTPUT) + string(STRIP ${WAT2WASM_VERSION_OUTPUT} WAT2WASM_VERSION) + message("-- Found wat2wasm ${WAT2WASM_VERSION}") +endif() + +if (${WAT2WASM_VERSION} VERSION_LESS 1.0.26) + set(WAT2WASM_FLAGS "--enable-reference-types") +endif () + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable(hello src/hello.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (hello PROPERTIES POSITION_INDEPENDENT_CODE ON) + +target_include_directories(hello PRIVATE ${UNCOMMON_SHARED_DIR}) + +target_link_libraries(hello vmlib -lpthread -lm) + +if (MSVC) + target_compile_definitions(hello PRIVATE WASM_API_EXTERN=) +endif() + +# wat to wasm +file(GLOB WAT_FILE src/hello.wat) +add_custom_target(hello_wasm ALL + COMMAND ${WAT2WASM} ${WAT_FILE} ${WAT2WASM_FLAGS} -o ${PROJECT_BINARY_DIR}/hello.wasm + DEPENDS ${WAT_FILE} + BYPRODUCTS ${PROJECT_BINARY_DIR}/hello.wasm + VERBATIM + SOURCES ${WAT_FILE} +) diff --git a/wasm-micro-runtime/samples/ref-types/src/hello.c b/wasm-micro-runtime/samples/ref-types/src/hello.c new file mode 100644 index 0000000..8a6f968 --- /dev/null +++ b/wasm-micro-runtime/samples/ref-types/src/hello.c @@ -0,0 +1,288 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" +#include "bh_read_file.h" +#include "wasm_export.h" + +#define USE_GLOBAL_HEAP_BUF 0 + +#if USE_GLOBAL_HEAP_BUF != 0 +static char global_heap_buf[10 * 1024 * 1024] = { 0 }; +#endif + +static uintptr_t global_objects[10] = { 0 }; + +int32 +local_cmp_externref(wasm_exec_env_t exec_env, uintptr_t externref_a, + uintptr_t externref_b) +{ + return externref_a == externref_b; +} + +int32 +local_chk_externref(wasm_exec_env_t exec_env, int32 index, uintptr_t externref) +{ + return externref == global_objects[index]; +} + +/* clang-format off */ +static NativeSymbol native_symbols[] = { + { "native-cmp-externref", local_cmp_externref, "(rr)i", NULL }, + { "native-chk-externref", local_chk_externref, "(ir)i", NULL }, +}; +/* clang-format on */ + +static inline void +local_set_externref(int32 index, uintptr_t externref) +{ + global_objects[index] = externref; +} + +static WASMFunctionInstanceCommon *wasm_set_externref_ptr; +static WASMFunctionInstanceCommon *wasm_get_externref_ptr; +static WASMFunctionInstanceCommon *wasm_cmp_externref_ptr; + +static bool +wasm_set_externref(wasm_exec_env_t exec_env, wasm_module_inst_t inst, + int32 index, uintptr_t externref) +{ + union { + uintptr_t val; + uint32 parts[2]; + } u; + uint32 argv[3] = { 0 }; + + if (!exec_env || !wasm_set_externref_ptr) { + return false; + } + + u.val = externref; + argv[0] = index; + argv[1] = u.parts[0]; + argv[2] = u.parts[1]; + if (!wasm_runtime_call_wasm(exec_env, wasm_set_externref_ptr, 2, argv)) { + const char *exception; + if ((exception = wasm_runtime_get_exception(inst))) { + printf("Exception: %s\n", exception); + } + return false; + } + + return true; +} + +static bool +wasm_get_externref(wasm_exec_env_t exec_env, wasm_module_inst_t inst, + int32 index, uintptr_t *ret_externref) +{ + wasm_val_t results[1] = { 0 }; + + if (!exec_env || !wasm_get_externref_ptr || !ret_externref) { + return false; + } + + if (!wasm_runtime_call_wasm_v(exec_env, wasm_get_externref_ptr, 1, results, + 1, index)) { + const char *exception; + if ((exception = wasm_runtime_get_exception(inst))) { + printf("Exception: %s\n", exception); + } + return false; + } + + if (WASM_ANYREF != results[0].kind) { + return false; + } + + *ret_externref = results[0].of.foreign; + return true; +} + +static bool +wasm_cmp_externref(wasm_exec_env_t exec_env, wasm_module_inst_t inst, + int32 index, uintptr_t externref, int32 *ret_result) +{ + wasm_val_t results[1] = { 0 }; + wasm_val_t arguments[2] = { + { .kind = WASM_I32, .of.i32 = index }, + { .kind = WASM_ANYREF, .of.foreign = externref }, + }; + + if (!exec_env || !wasm_cmp_externref_ptr || !ret_result) { + return false; + } + + if (!wasm_runtime_call_wasm_a(exec_env, wasm_cmp_externref_ptr, 1, results, + 2, arguments)) { + const char *exception; + if ((exception = wasm_runtime_get_exception(inst))) { + printf("Exception: %s\n", exception); + } + return false; + } + + if (results[0].kind != WASM_I32) { + return false; + } + + *ret_result = results[0].of.i32; + return true; +} + +static bool +set_and_cmp(wasm_exec_env_t exec_env, wasm_module_inst_t inst, int32 i, + uintptr_t externref) +{ + int32 cmp_result = 0; + uintptr_t wasm_externref = 0; + + wasm_set_externref(exec_env, inst, i, externref); + local_set_externref(i, externref); + + wasm_get_externref(exec_env, inst, i, &wasm_externref); + if (!local_chk_externref(exec_env, i, wasm_externref)) { + printf("#%d, In host language world Wasm Externref 0x%lx Vs. Native " + "Externref 0x%lx FAILED\n", + i, wasm_externref, externref); + return false; + } + + if (!wasm_cmp_externref(exec_env, inst, i, global_objects[i], &cmp_result) + || !cmp_result) { + printf("#%d, In Wasm world Native Externref 0x%lx Vs, Wasm Externref " + "FAILED\n", + i, global_objects[i]); + return false; + } + + return true; +} + +int +main(int argc, char *argv[]) +{ + char *wasm_file = "hello.wasm"; + uint8 *wasm_file_buf = NULL; + uint32 wasm_file_size; + uint32 stack_size = 16 * 1024, heap_size = 16 * 1024; + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + wasm_exec_env_t exec_env = NULL; + RuntimeInitArgs init_args; + char error_buf[128] = { 0 }; +#if WASM_ENABLE_LOG != 0 + int log_verbose_level = 2; +#endif + const uint64 big_number = 0x123456789abc; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + +#if USE_GLOBAL_HEAP_BUF != 0 + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); +#else + init_args.mem_alloc_type = Alloc_With_Allocator; + init_args.mem_alloc_option.allocator.malloc_func = malloc; + init_args.mem_alloc_option.allocator.realloc_func = realloc; + init_args.mem_alloc_option.allocator.free_func = free; +#endif + + init_args.n_native_symbols = sizeof(native_symbols) / sizeof(NativeSymbol); + init_args.native_module_name = "env"; + init_args.native_symbols = native_symbols; + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + +#if WASM_ENABLE_LOG != 0 + bh_log_set_verbose_level(log_verbose_level); +#endif + + /* load WASM byte buffer from WASM bin file */ + if (!(wasm_file_buf = + (uint8 *)bh_read_file_to_buffer(wasm_file, &wasm_file_size))) + goto fail; + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail; + } + + /* instantiate the module */ + if (!(wasm_module_inst = + wasm_runtime_instantiate(wasm_module, stack_size, heap_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail; + } + + /* create an execution env */ + if (!(exec_env = + wasm_runtime_create_exec_env(wasm_module_inst, stack_size))) { + printf("%s\n", "create exec env failed"); + goto fail; + } + + /* lookup function instance */ + if (!(wasm_cmp_externref_ptr = wasm_runtime_lookup_function( + wasm_module_inst, "cmp-externref"))) { + printf("%s\n", "lookup function cmp-externref failed"); + goto fail; + } + + if (!(wasm_get_externref_ptr = wasm_runtime_lookup_function( + wasm_module_inst, "get-externref"))) { + printf("%s\n", "lookup function get-externref failed"); + goto fail; + } + + if (!(wasm_set_externref_ptr = wasm_runtime_lookup_function( + wasm_module_inst, "set-externref"))) { + printf("%s\n", "lookup function set-externref failed"); + goto fail; + } + + /* test with NULL */ + if (!set_and_cmp(exec_env, wasm_module_inst, 0, 0) + || !set_and_cmp(exec_env, wasm_module_inst, 1, big_number + 1) + || !set_and_cmp(exec_env, wasm_module_inst, 2, big_number + 2) + || !set_and_cmp(exec_env, wasm_module_inst, 3, big_number + 3)) { + goto fail; + } + + printf("GREAT! PASS ALL CHKs\n"); + +fail: + /* destroy exec env */ + if (exec_env) { + wasm_runtime_destroy_exec_env(exec_env); + } + + /* destroy the module instance */ + if (wasm_module_inst) { + wasm_runtime_deinstantiate(wasm_module_inst); + } + + /* unload the module */ + if (wasm_module) { + wasm_runtime_unload(wasm_module); + } + + /* free the file buffer */ + if (wasm_file_buf) { + wasm_runtime_free(wasm_file_buf); + } + + /* destroy runtime environment */ + wasm_runtime_destroy(); + return 0; +} diff --git a/wasm-micro-runtime/samples/ref-types/src/hello.wat b/wasm-micro-runtime/samples/ref-types/src/hello.wat new file mode 100644 index 0000000..3c2d2f9 --- /dev/null +++ b/wasm-micro-runtime/samples/ref-types/src/hello.wat @@ -0,0 +1,44 @@ +;; Copyright (C) 2019 Intel Corporation. All rights reserved. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(module + (type $t0 (func (param i32 externref) (result i32))) + + (import "env" "native-cmp-externref" + (func $native-cmp-externref (param externref externref) (result i32)) + ) + + (import "env" "native-chk-externref" + (func $native-chk-externref (param i32 externref) (result i32)) + ) + + (table $t1 8 8 externref) + (table $t2 funcref + (elem + $native-cmp-externref + $native-chk-externref + ) + ) + + (func (export "set-externref") (param $i i32) (param $r externref) + (table.set $t1 (local.get $i) (local.get $r)) + ) + + (func (export "get-externref") (param $i i32) (result externref) + (table.get $t1 (local.get $i)) + ) + + (func (export "cmp-externref") (param $i i32) (param $r externref) (result i32) + (table.get $t1 (local.get $i)) + (local.get $r) + (call $native-cmp-externref) + ) + + (func (export "chk-externref") (param $i i32) (param $r externref) (result i32) + (call_indirect $t2 (type $t0) + (local.get $i) + (local.get $r) + (i32.const 1) + ) + ) +) diff --git a/wasm-micro-runtime/samples/sgx-ra/CMakeLists.txt b/wasm-micro-runtime/samples/sgx-ra/CMakeLists.txt new file mode 100644 index 0000000..3f5f23e --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/CMakeLists.txt @@ -0,0 +1,79 @@ +# Copyright (c) 2022 Intel Corporation +# Copyright (c) 2020-2021 Alibaba Cloud +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.1.4) +project(sgx-ra) + +################ runtime settings ############## +set (WAMR_BUILD_PLATFORM "linux-sgx") + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR_BUILD_TARGET +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_LIBC_WASI 1) +set (WAMR_BUILD_LIB_PTHREAD 1) +set (WAMR_BUILD_FAST_INTERP 1) +set (WAMR_BUILD_LIB_RATS 1) + +# compiling and linking flags +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \ + -Wall -Wno-unused-parameter -Wno-pedantic \ + -nostdinc -fvisibility=hidden -fpie" ) + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +set (SGX_PLATFORM_DIR ${WAMR_ROOT_DIR}/product-mini/platforms/linux-sgx) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +add_custom_command ( + OUTPUT libvmlib_untrusted.a + COMMAND mkdir -p untrusted && cd untrusted && + ${CMAKE_C_COMPILER} -c ${PLATFORM_SHARED_SOURCE_UNTRUSTED} + COMMAND ${CMAKE_AR} rc libvmlib_untrusted.a untrusted/*.o) + +add_custom_target (vmlib_untrusted ALL DEPENDS libvmlib_untrusted.a) + +execute_process ( + COMMAND bash -c "sed -i -E 's/^#define WASM_ENABLE_LIB_RATS 0/#define WASM_ENABLE_LIB_RATS 1/g' ${SGX_PLATFORM_DIR}/enclave-sample/Enclave/Enclave.edl" + COMMAND bash -c "sed -i -E 's/^WAMR_BUILD_LIB_RATS = 0/WAMR_BUILD_LIB_RATS = 1/g' ${SGX_PLATFORM_DIR}/enclave-sample/Makefile" + OUTPUT_VARIABLE cmdOutput +) + +################ wamr runtime ################### +add_custom_target ( + iwasm ALL + DEPENDS vmlib_untrusted vmlib_untrusted vmlib + COMMAND make -C ${SGX_PLATFORM_DIR}/enclave-sample clean + COMMAND make -C ${SGX_PLATFORM_DIR}/enclave-sample SGX_MODE=HW SGX_DEBUG=1 VMLIB_BUILD_DIR=${CMAKE_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${SGX_PLATFORM_DIR}/enclave-sample/enclave.signed.so ${CMAKE_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${SGX_PLATFORM_DIR}/enclave-sample/iwasm ${CMAKE_BINARY_DIR} + COMMAND make -C ${SGX_PLATFORM_DIR}/enclave-sample clean) + +################ wasm application ############### +add_subdirectory(wasm-app) diff --git a/wasm-micro-runtime/samples/sgx-ra/README.md b/wasm-micro-runtime/samples/sgx-ra/README.md new file mode 100644 index 0000000..179a074 --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/README.md @@ -0,0 +1,270 @@ +"sgx-ra" sample introduction +============== + +This sample demonstrates how to execute Remote Attestation on SGX with [librats](https://github.com/inclavare-containers/librats) and run it with iwasm. It can only build on [SGX supported processors](https://www.intel.com/content/www/us/en/support/articles/000028173/processors.html), please check it. + +## Preparation + +SGX-RA requires to have installed: + - the WASI-SDK, located in `/opt/wasi-sdk` + +### Intel SGX dependencies + +Before starting, we need to download and install [SGX SDK](https://download.01.org/intel-sgx/latest/linux-latest/distro) and [SGX DCAP Library](https://download.01.org/intel-sgx/latest/dcap-latest) referring to this [guide](https://download.01.org/intel-sgx/sgx-dcap/1.8/linux/docs/Intel_SGX_DCAP_Linux_SW_Installation_Guide.pdf). + +The following commands are an example of the SGX environment installation on Ubuntu 20.04. +``` shell +# Set your platform, you can get the platforms list on +# https://download.01.org/intel-sgx/latest/linux-latest/distro +$ cd $HOME +$ OS_PLATFORM=ubuntu20.04 +$ OS_CODE_NAME=`lsb_release -sc` +$ SGX_PLATFORM=$OS_PLATFORM-server +$ SGX_RELEASE_VERSION=1.17 +$ SGX_DRIVER_VERSION=1.41 +$ SGX_SDK_VERSION=2.20.100.4 + +# install the dependencies +$ sudo apt-get update +$ sudo apt-get install -y build-essential ocaml automake autoconf libtool wget python3 libssl-dev dkms zip cmake +$ sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +# install SGX Driver +$ wget https://download.01.org/intel-sgx/sgx-dcap/$SGX_RELEASE_VERSION/linux/distro/$SGX_PLATFORM/sgx_linux_x64_driver_$SGX_DRIVER_VERSION.bin +$ chmod +x sgx_linux_x64_driver_$SGX_DRIVER_VERSION.bin +$ sudo ./sgx_linux_x64_driver_$SGX_DRIVER_VERSION.bin + +# install SGX SDK +$ wget https://download.01.org/intel-sgx/sgx-dcap/$SGX_RELEASE_VERSION/linux/distro/$SGX_PLATFORM/sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin +$ chmod +x sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin +$ sudo ./sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin --prefix /opt/intel + +# install SGX DCAP Library +$ echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu $OS_CODE_NAME main" | sudo tee /etc/apt/sources.list.d/intel-sgx.list +$ wget -O - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add +$ sudo apt-get update +$ sudo apt-get install -y libsgx-epid libsgx-quote-ex libsgx-dcap-ql libsgx-enclave-common-dev libsgx-dcap-ql-dev libsgx-dcap-default-qpl-dev libsgx-dcap-quote-verify-dev + +# install SGX SSL Library +$ git clone https://github.com/intel/linux-sgx.git +$ cd linux-sgx && make preparation +$ sudo cp external/toolset/$OS_PLATFORM/* /usr/local/bin +$ # Verify that the paths are correctly set +$ which ar as ld objcopy objdump ranlib +$ cd ../ +$ git clone https://github.com/intel/intel-sgx-ssl.git +$ wget https://www.openssl.org/source/openssl-1.1.1v.tar.gz -O intel-sgx-ssl/openssl_source/openssl-1.1.1v.tar.gz +$ cd intel-sgx-ssl/Linux +$ source /opt/intel/sgxsdk/environment +$ make all +$ sudo make install +``` + +You can optionally grant users to communicate with the SDK platform using the following command. +Otherwise, enclaves must be launched with root privileges. + +```shell +sudo usermod -a -G sgx_prv +``` + +### Intel Provisioning Certification Service (Intel PCS) + +Intel DCAP connects to Intel PCS to download the attestation collateral for SGX-enabled machines. +Intel provides a [quick install guide](https://www.intel.com/content/www/us/en/developer/articles/guide/intel-software-guard-extensions-data-center-attestation-primitives-quick-install-guide.html) to set up a simplified environment. +This section summarizes the commands to issue for setting up a working environment on Ubuntu 20.04. + +### Subscribe to Intel PCS Web services + +Intel SGX DCAP requires a complimentary subscription to the Intel PCS. +To subscribe to the service, browse the [Intel SGX Software Services](https://api.portal.trustedservices.intel.com/) page. +A the end of the subscription process, save the primary and the secondary keys. + +### Set up the Intel Provisioning Certification Caching Service (Intel PCCS) + +Intel PCCS is a caching mechanism for attestation collateral, preventing continuously communicating with Intel PCS during attestation. +Intel provides an implementation of the cache mechanism. + +The following commands set up Intel PCCS. +```shell +# install Node.js +$ sudo apt install -y curl cracklib-runtime +$ curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - && sudo apt-get install -y nodejs +# install PCCS software +$ sudo apt-get install -y sgx-dcap-pccs +``` + +The installation will run the PCCS setup script, asking you several questions. + +``` +Do you want to configure PCCS now? (Y/N) +``` + +Answer "Y" to this question. + +``` +Set HTTPS listening port [8081] (1024-65535) +``` + +Accept the default listening port of 8081. + +``` +Set the PCCS service to accept local connections only? [Y] (Y/N) +``` + +Answer "N" to this question. We want the PCCS service to accept connections from other systems. + +``` +Set your Intel PCS API key (Press ENTER to skip) +``` + +Enter either your primary or secondary key retrieved from the previous subsection. +If you already subscribed, you can retrieve them [here](https://api.portal.trustedservices.intel.com/developer). + +``` +Choose caching fill method : [LAZY] (LAZY/OFFLINE/REQ) +``` + +Answer "REQ" to this question. This places the caching service in the "on request" mode, which means it will fetch the attestation collateral for hosts as provisioning requests are received. + +``` +Set PCCS server administrator password: +Re-enter administrator password: +Set PCCS server user password: +Re-enter user password: +``` + +Enter two passwords for the PCCS server. + +``` +Do you want to generate insecure HTTPS key and cert for PCCS service? [Y] (Y/N) +``` + +Answer "Y" to this question. + +### Provisioning the current system's Intel SGX collateral into the PCCS + +Now that the PCCS is up and running, it's time to provision an Intel SGX-enabled platform. +We use the tool `PCKIDRetrievalTool` to get the attestation collateral of the current machine. + +``` shell +$ sudo apt-get install -y sgx-pck-id-retrieval-tool +``` + +Adapt the configuration file of `PCKIDRetrievalTool` located in `/opt/intel/sgx-pck-id-retrieval-tool/network_setting.conf` and make the following changes: +- Change the **PCCS_URL** to match your caching service's location. +- Uncomment the **user_token** parameter, and set it to the user password you created when configuring the PCCS. +- Set the **proxy_type** to fit your environment (most likely, this will be `direct`) +- Ensure **USE_SECURE_CERT** is set to `FALSE` since we're using a self-signed certificate for testing purposes. + +Save your changes and run the provisioning tool. + +```shell +$ sudo PCKIDRetrievalTool +Intel(R) Software Guard Extensions PCK Cert ID Retrieval Tool Version 1.17.100.4 + +Registration status has been set to completed status. +the data has been sent to cache server successfully and pckid_retrieval.csv has been generated successfully! +``` + +You may get some warnings during this execution of the tool. +A correct insertion into the cache server usually means the retrieval of the attestation collateral worked. +Execute the following command to verify the collateral could be stored in your instance of Intel PCCS: + +``` +curl -k https://localhost:8081/sgx/certification/v3/qe/identity +``` + +This should print a JSON value with the attestation collateral. + +### Runtime configuration + +Edit the configuration file, `/etc/sgx_default_qcnl.conf`, and make the following changes: +- Set the **PCCS_URL** parameter to the location of our PCCS server. +- Set **USE_SECURE_CERT** to `FALSE` since we're using a self-signed certificate for testing purposes. + +This system is now ready to run Intel SGX workloads with generate evidence for remote attestation. + +## Build and executing the sample + +``` shell +$ mkdir build && cd build +$ cmake .. +$ make +$ # run the sample +$ ./iwasm wasm-app/test.wasm +``` + +The sample will print the evidence in JSON and the message: *Evidence is trusted.* + +In case of validation issues expressed as a value of `0xeXXX`, the corresponding error reason is explained in [this header file](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/QuoteGeneration/quote_wrapper/common/inc/sgx_ql_lib_common.h). + +## Validate quotes on non-SGX platforms +Quotes created on an Intel SGX platform can also be verified on systems that do not support SGX (e.g., a different CPU architecture). +This scenario typically arises when deploying trusted applications in a cloud environment, which provides confidential computing. + +For that purpose, we are required to install a subset of Intel SGX libraries to support quote validation. +The steps below highlight how to set up such an environment. + + +### Intel SGX dependencies +```shell +$ OS_CODE_NAME=`lsb_release -sc` +# install SGX DCAP Library +$ echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu $OS_CODE_NAME main" | sudo tee /etc/apt/sources.list.d/intel-sgx.list +$ wget -O - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add +$ sudo apt-get update +$ sudo apt-get install -y libsgx-quote-ex libsgx-dcap-ql libsgx-dcap-quote-verify libsgx-dcap-default-qpl +``` + +### Set up the Intel Provisioning Certification Caching Service (Intel PCCS) +Follow the steps described in the section _Set up the Intel Provisioning Certification Caching Service (Intel PCCS)_. + +### Runtime configuration +Follow the steps described in the section _Runtime configuration_. + +### Provisioning all the Intel SGX collateral into the PCCS +We must finally fetch and configure the SGX collaterals into the PCCS for all the SGX-enabled CPUs. + +```shell +# Set up the Intel PCCS administration tool +$ git clone https://github.com/intel/SGXDataCenterAttestationPrimitives.git +$ cd SGXDataCenterAttestationPrimitives/tools/PccsAdminTool +$ sudo apt-get install -y python3 python3-pip +$ pip3 install -r requirements.txt + +# Configuring the Intel PCCS. Input the PCS/PCCS password as requested. +# 1. Get registration data from PCCS service +./pccsadmin.py get +# 2. Fetch platform collateral data from Intel PCS based on the registration data +./pccsadmin.py fetch +# 3. Put platform collateral data or appraisal policy files to PCCS cache db +./pccsadmin.py put +# 4. Request PCCS to refresh certificates or collateral in cache database +./pccsadmin.py refresh +``` + +### Validation of the quotes +The Wasm application can then be modified to validate precomputed quotes using the exposed function `librats_verify`. + +Alternatively, the underlying library `librats` may be directly used if the non-SGX platforms do not execute WebAssembly code (without WAMR). +Examples are provided in the directory [non-sgx-verify/](non-sgx-verify/). + +### Claims validation +Once the runtime has validated the signature of the quote, the application must also check the other claims embedded in the quote to ensure they match their expected value. + +The documentation _Data Center Attestation Primitives: Library API_ describes in Section _3.8 Enclave Identity Checking_ defines the claims for the user to check. +Here is a summary of them: + +- **Enclave Identity Checking**: either check the hash _MRENCLAVE_ (the enclave identity) or _MRSIGNER_ and the _product id_ (the software provider identity). +- **Verify Attributes**: production enclaves should not have the _Debug_ flag set to 1. +- **Verify SSA Frame extended feature set** +- **Verify the ISV_SVN level of the enclave**: whenever there is a security update to an enclave, the ISV_SVN value should be increased to reflect the higher security level. +- **Verify that the ReportData contains the expected value**: This can be used to provide specific data from the enclave or it can be used to hold a hash of a larger block of data which is provided with the quote. Note that the verification of the quote signature confirms the integrity of the report data (and the rest of the REPORT body). + + +## Further readings + +- [Intel SGX Software Installation Guide For Linux OS](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/Intel_SGX_SW_Installation_Guide_for_Linux.pdf) +- [Intel Software Guard Extensions (Intel® SGX) Data Center Attestation Primitives: Library API](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/Intel_SGX_ECDSA_QuoteLibReference_DCAP_API.pdf) +- [Remote Attestation for Multi-Package Platforms using Intel SGX Datacenter Attestation Primitives (DCAP)](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/Intel_SGX_DCAP_Multipackage_SW.pdf) +- [Documentation of the PCCS administration tool](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/tools/PccsAdminTool/README.txt) diff --git a/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/README.md b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/README.md new file mode 100644 index 0000000..e4e9d87 --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/README.md @@ -0,0 +1,5 @@ +# Examples of evidence verification without Intel SGX +Intel SGX evidence generated using WAMR can be validated on trusted plaforms without Intel SGX, or an Intel processors. + +## Using C# +The sample [csharp/](csharp/) demonstrates such validation using C# as a managed language. diff --git a/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/.gitignore b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/.gitignore new file mode 100644 index 0000000..27e48df --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/.gitignore @@ -0,0 +1,3 @@ +.idea/ +bin/ +obj/ diff --git a/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/Program.cs b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/Program.cs new file mode 100644 index 0000000..6d7753a --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/Program.cs @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2024 Intel Corporation. + * Copyright (C) 2024 University of Neuchatel, Switzerland. + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +// Set the reference values below +byte[] mrEnclaveReference = +{ + 0xDA, 0xE0, 0xDA, 0x2F, 0x8A, 0x53, 0xA0, 0xB4, 0x8F, 0x92, 0x6A, 0x3B, 0xC0, 0x48, 0xD6, 0xA9, + 0x67, 0xD4, 0x7C, 0x86, 0x19, 0x86, 0x76, 0x6F, 0x8F, 0x5A, 0xB1, 0xC0, 0xA8, 0xD8, 0x8E, 0x44 +}; +byte[] mrSignerReference = +{ + 0x83, 0xD7, 0x19, 0xE7, 0x7D, 0xEA, 0xCA, 0x14, 0x70, 0xF6, 0xBA, 0xF6, 0x2A, 0x4D, 0x77, 0x43, + 0x03, 0xC8, 0x99, 0xDB, 0x69, 0x02, 0x0F, 0x9C, 0x70, 0xEE, 0x1D, 0xFC, 0x08, 0xC7, 0xCE, 0x9E +}; +const ushort securityVersionReference = 0; +const ushort productIdReference = 0; +string nonce = "This is a sample.\0"; // Notice the \0 at the end, which is mandatory as C-strings are terminated with this char +string evidenceAsString = """{"type":"sgx_ecdsa","report_base64":"[..]","report_len":[..]}"""; +string wasmFilePath = "../build/wasm-app/test.wasm"; + +// Parse and compute the claims +EvidenceJson? evidenceAsJson = JsonSerializer.Deserialize(evidenceAsString, new JsonSerializerOptions +{ + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower +}); +Debug.Assert(evidenceAsJson != null, "The evidence cannot be parsed."); + +byte[] wasmFileContent = await File.ReadAllBytesAsync(wasmFilePath); +byte[] nonceAsBytes = Encoding.UTF8.GetBytes(nonce); +byte[] computedUserData = await ComputeUserData(wasmFileContent, nonceAsBytes); +byte[] evidenceAsBytes = Convert.FromBase64String(evidenceAsJson.ReportBase64); +Evidence evidence = new(evidenceAsBytes); +int libRatsReturnValue = LibRats.VerifyEvidenceFromJson(evidenceAsString, await ComputeUserData(wasmFileContent, nonceAsBytes)); + +// Compare and display the results +Console.WriteLine($"User data, evidence: {BitConverter.ToString(evidence.UserData)}"); +Console.WriteLine($"User Data, computed: {BitConverter.ToString(computedUserData)}"); +Console.WriteLine($"Do the two user data match? {evidence.UserData.SequenceEqual(computedUserData)}"); +Console.WriteLine($"MrEnclave: {BitConverter.ToString(evidence.MrEnclave)}"); +Console.WriteLine($"Do the MrEnclave match? {mrEnclaveReference.SequenceEqual(evidence.MrEnclave)}"); +Console.WriteLine($"MrSigner: {BitConverter.ToString(evidence.MrSigner)}"); +Console.WriteLine($"Do the MrSigner match? {mrSignerReference.SequenceEqual(evidence.MrSigner)}"); +Console.WriteLine($"Security Version: {evidence.SecurityVersion}, expected: {securityVersionReference}"); +Console.WriteLine($"Product ID: {evidence.ProductId}, expected: {productIdReference}"); +Console.WriteLine($"VerifyJsonUsingLibrats returned: {libRatsReturnValue:X}"); + +// Compute the user data as computed by WAMR +static async ValueTask ComputeUserData(byte[] wasmFileContent, byte[] nonce) +{ + using var sha256 = SHA256.Create(); + var wasmFileContentHash = sha256.ComputeHash(wasmFileContent); + + using MemoryStream stream = new(); + await stream.WriteAsync(wasmFileContentHash); + await stream.WriteAsync(nonce); + stream.Position = 0; + + byte[] computedUserData = await sha256.ComputeHashAsync(stream); + return computedUserData; +} + +/// +/// The layout of the JSON is given by librats. +/// +class EvidenceJson +{ + public required string Type { get; init; } + public required string ReportBase64 { get; init; } + public required int ReportLen { get; init; } +} + +/// +/// The start of the _report_body_t struct from Intel SGX is at offset 0x30. +/// +/// +/// _report_body_t struct: https://github.com/intel/linux-sgx/blob/a1eeccba5a72b3b9b342569d2cc469ece106d3e9/common/inc/sgx_report.h#L93-L111 +/// Attestation flow: https://www.intel.com/content/www/us/en/developer/articles/code-sample/software-guard-extensions-remote-attestation-end-to-end-example.html +/// +class Evidence(byte[] evidenceAsBytes) +{ + public byte[] MrEnclave => evidenceAsBytes[0x70..0x90]; + public byte[] MrSigner => evidenceAsBytes[0xB0..0xD0]; + public ushort ProductId => BitConverter.ToUInt16(evidenceAsBytes.AsSpan(0x130, 2)); + public ushort SecurityVersion => BitConverter.ToUInt16(evidenceAsBytes.AsSpan(0x132, 2)); + public byte[] UserData => evidenceAsBytes[0x170..0x190]; +} + +static class LibRats +{ + /// + /// Verifies the evidence using librats native function. + /// + /// + /// Original signature: int librats_verify_evidence_from_json(const char *json_string, const uint8_t *hash); + /// + [DllImport("/usr/local/lib/librats/librats_lib.so", EntryPoint = "librats_verify_evidence_from_json")] + public static extern int VerifyEvidenceFromJson(string json, byte[] hash); +} diff --git a/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/README.md b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/README.md new file mode 100644 index 0000000..689c8da --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/README.md @@ -0,0 +1,18 @@ +# Examples of evidence verification without Intel SGX using C# +This sample demonstrates how to validate WAMR-generated evidence without using an Intel SGX-enabled platform. +A typical use case is a Web service hosted on trusted premises. + +## Prerequisites + - [dotnet-sdk](https://learn.microsoft.com/en-us/dotnet/core/install/linux) (8+) + - [librats](https://github.com/inclavare-containers/librats) + - Intel infrastructure for validating evidence, [see here](../../README.md#validate-quotes-on-non-sgx-platforms) + +This sample has been tested on Linux Ubuntu 20.04+. +Any other Linux platforms should be supported. +This sample should also work on other OS, provided librats can be compiled on those other OS. + +## How to use + - Supply the reference values to consider trustworthy in [Program.cs](Program.cs#L15-L27). + - Generate a valid JSON evidence using WAMR on an Intel SGX-enabled platform. + - Fill in the JSON evidence in [Program.cs](Program.cs#L28). + - Run the command `dotnet run` in this directory. diff --git a/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/VerifyEvidence.csproj b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/VerifyEvidence.csproj new file mode 100644 index 0000000..206b89a --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/non-sgx-verify/csharp/VerifyEvidence.csproj @@ -0,0 +1,10 @@ + + + + Exe + net8.0 + enable + enable + + + diff --git a/wasm-micro-runtime/samples/sgx-ra/wasm-app/CMakeLists.txt b/wasm-micro-runtime/samples/sgx-ra/wasm-app/CMakeLists.txt new file mode 100644 index 0000000..afba7df --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/wasm-app/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright (c) 2022 Intel Corporation +# Copyright (c) 2020-2021 Alibaba Cloud +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.0) +project(wasm-app) + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) +set (LIB_RATS_DIR ${WAMR_ROOT_DIR}/core/iwasm/libraries/lib-rats) + +set (CMAKE_C_LINK_FLAGS "") +set (CMAKE_CXX_LINK_FLAGS "") +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) +endif () + +set (CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_FLAGS "-nostdlib") +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set (CMAKE_EXE_LINKER_FLAGS + "-Wl,--max-memory=131072 -z stack-size=8192 \ + -Wl,--no-entry,--strip-all \ + -Wl,--export=__main_argc_argv \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--allow-undefined" +) + +add_executable(test.wasm main.c) +set_target_properties(test.wasm PROPERTIES INCLUDE_DIRECTORIES ${LIB_RATS_DIR}) +target_link_libraries(test.wasm) diff --git a/wasm-micro-runtime/samples/sgx-ra/wasm-app/main.c b/wasm-micro-runtime/samples/sgx-ra/wasm-app/main.c new file mode 100644 index 0000000..ab61505 --- /dev/null +++ b/wasm-micro-runtime/samples/sgx-ra/wasm-app/main.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2022 Intel Corporation + * Copyright (c) 2020-2021 Alibaba Cloud + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include "lib_rats_wrapper.h" + +#define __is_print(ch) ((unsigned int)((ch) - ' ') < 127u - ' ') + +/** + * hex_dump + * + * @brief dump data in hex format + * + * @param title: Title + * @param buf: User buffer + * @param size: Dump data size + * @param number: The number of outputs per line + * + * @return void + */ +void +hex_dump(const char *title, const uint8_t *buf, uint32_t size, uint32_t number) +{ + int i, j; + if (title) { + printf("\n\t%s:\n\n", title); + } + + for (i = 0; i < size; i += number) { + printf("%08X: ", i); + + for (j = 0; j < number; j++) { + if (j % 8 == 0) { + printf(" "); + } + if (i + j < size) + printf("%02X ", buf[i + j]); + else + printf(" "); + } + printf(" "); + + for (j = 0; j < number; j++) { + if (i + j < size) { + printf("%c", __is_print(buf[i + j]) ? buf[i + j] : '.'); + } + } + printf("\n"); + } +} + +int +main(int argc, char **argv) +{ + int ret_code = -1; + char *evidence_json = NULL; + + // Generate user_data by SHA256 buffer and the wasm module. + // user_data = SHA256(sha256_wasm_module || buffer) + const char *buffer = "This is a sample."; + + // If you want to declare the evidence of type rats_sgx_evidence_t on the + // stack, you should modify the stack size of the CMAKE_EXE_LINKER_FLAGS in + // CMakeLists.txt to 51200 at least. + rats_sgx_evidence_t *evidence = + (rats_sgx_evidence_t *)malloc(sizeof(rats_sgx_evidence_t)); + if (!evidence) { + printf("ERROR: No memory to allocate.\n"); + goto err; + } + + int rats_err = librats_collect(&evidence_json, buffer); + if (rats_err != 0) { + printf("ERROR: Collect evidence failed, error code: %#x\n", rats_err); + goto err; + } + + if (librats_parse_evidence(evidence_json, evidence) != 0) { + printf("ERROR: Parse evidence failed.\n"); + goto err; + } + + // You could use these parameters for further verification. + hex_dump("Quote", evidence->quote, evidence->quote_size, 32); + hex_dump("User Data", evidence->user_data, SGX_USER_DATA_SIZE, 32); + hex_dump("MRENCLAVE", evidence->mr_enclave, SGX_MEASUREMENT_SIZE, 32); + hex_dump("MRSIGNER", evidence->mr_signer, SGX_MEASUREMENT_SIZE, 32); + printf("\n\tProduct ID:\t\t\t\t%u\n", evidence->product_id); + printf("\tSecurity Version:\t\t\t%u\n", evidence->security_version); + printf("\tAttributes.flags:\t\t\t%llu\n", evidence->att_flags); + printf("\tAttributes.flags[INITTED]:\t\t%d\n", + (evidence->att_flags & SGX_FLAGS_INITTED) != 0); + printf("\tAttributes.flags[DEBUG]:\t\t%d\n", + (evidence->att_flags & SGX_FLAGS_DEBUG) != 0); + printf("\tAttributes.flags[MODE64BIT]:\t\t%d\n", + (evidence->att_flags & SGX_FLAGS_MODE64BIT) != 0); + printf("\tAttributes.flags[PROVISION_KEY]:\t%d\n", + (evidence->att_flags & SGX_FLAGS_PROVISION_KEY) != 0); + printf("\tAttributes.flags[EINITTOKEN_KEY]:\t%d\n", + (evidence->att_flags & SGX_FLAGS_EINITTOKEN_KEY) != 0); + printf("\tAttributes.flags[KSS]:\t\t\t%d\n", + (evidence->att_flags & SGX_FLAGS_KSS) != 0); + printf("\tAttributes.flags[AEX_NOTIFY]:\t\t%d\n", + (evidence->att_flags & SGX_FLAGS_AEX_NOTIFY) != 0); + printf("\tAttribute.xfrm:\t\t\t\t%llu\n", evidence->att_xfrm); + + rats_err = librats_verify((const char *)evidence_json, evidence->user_data); + if (rats_err != 0) { + printf("ERROR: Evidence is not trusted, error code: %#x.\n", rats_err); + goto err; + } + + ret_code = 0; + printf("Evidence is trusted.\n"); + +err: + if (evidence_json) { + librats_dispose_evidence_json(evidence_json); + } + + if (evidence) { + free(evidence); + } + + return ret_code; +} diff --git a/wasm-micro-runtime/samples/shared-module/.gitignore b/wasm-micro-runtime/samples/shared-module/.gitignore new file mode 100644 index 0000000..0fa8a76 --- /dev/null +++ b/wasm-micro-runtime/samples/shared-module/.gitignore @@ -0,0 +1 @@ +/out/ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/shared-module/CMakeLists.txt b/wasm-micro-runtime/samples/shared-module/CMakeLists.txt new file mode 100644 index 0000000..c094b07 --- /dev/null +++ b/wasm-micro-runtime/samples/shared-module/CMakeLists.txt @@ -0,0 +1,95 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +project (shared-module) + +set (CMAKE_CXX_STANDARD 17) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) + +# fast interpreter +# set (WAMR_BUILD_FAST_INTERP 1) + +# fast-jit +# set (WAMR_BUILD_FAST_JIT 1) + +# llvm jit +# set (WAMR_BUILD_JIT 1) +# set (LLVM_DIR /usr/local/opt/llvm@14/lib/cmake/llvm) + +set (WAMR_BUILD_REF_TYPES 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (shared-module src/main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (shared-module PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (shared-module vmlib -lm -ldl -lpthread ${LLVM_AVAILABLE_LIBS}) +else () + target_link_libraries (shared-module vmlib -lm -ldl -lpthread -lrt ${LLVM_AVAILABLE_LIBS}) +endif () diff --git a/wasm-micro-runtime/samples/shared-module/README.md b/wasm-micro-runtime/samples/shared-module/README.md new file mode 100644 index 0000000..14baa8c --- /dev/null +++ b/wasm-micro-runtime/samples/shared-module/README.md @@ -0,0 +1,5 @@ +The "shared-module" sample project +================================== + +This sample demonstrates a bug described in: +https://github.com/bytecodealliance/wasm-micro-runtime/issues/2735. diff --git a/wasm-micro-runtime/samples/shared-module/build.sh b/wasm-micro-runtime/samples/shared-module/build.sh new file mode 100755 index 0000000..e6c1047 --- /dev/null +++ b/wasm-micro-runtime/samples/shared-module/build.sh @@ -0,0 +1,63 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +CURR_DIR=$PWD +WAMR_DIR=${PWD}/../.. +OUT_DIR=${PWD}/out + +WASM_APPS=${PWD}/wasm-apps + + +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +mkdir ${OUT_DIR}/wasm-apps + + +echo "##################### build shared-module project" +cd ${CURR_DIR} +mkdir -p cmake_build +cd cmake_build +cmake .. -DCMAKE_BUILD_TYPE=Debug +make -j ${nproc} +if [ $? != 0 ];then + echo "BUILD_FAIL shared-module exit as $?\n" + exit 2 +fi + +cp -a shared-module ${OUT_DIR} + +printf "\n" + +echo "##################### build wasm apps" + +cd ${WASM_APPS} + +for i in `ls *.wat` +do +APP_SRC="$i" +OUT_FILE=${i%.*}.wasm + +# Note: the CI installs wabt in /opt/wabt +if type wat2wasm; then + WAT2WASM=${WAT2WASM:-wat2wasm} +elif [ -x /opt/wabt/bin/wat2wasm ]; then + WAT2WASM=${WAT2WASM:-/opt/wabt/bin/wat2wasm} +fi + +${WAT2WASM} -o ${OUT_DIR}/wasm-apps/${OUT_FILE} ${APP_SRC} + +# aot +# wamrc -o ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot ${OUT_DIR}/wasm-apps/${OUT_FILE} +# mv ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot ${OUT_DIR}/wasm-apps/${OUT_FILE} + +if [ -f ${OUT_DIR}/wasm-apps/${OUT_FILE} ]; then + echo "build ${OUT_FILE} success" +else + echo "build ${OUT_FILE} fail" +fi +done +echo "##################### build wasm apps done" diff --git a/wasm-micro-runtime/samples/shared-module/run.sh b/wasm-micro-runtime/samples/shared-module/run.sh new file mode 100755 index 0000000..3cb2e62 --- /dev/null +++ b/wasm-micro-runtime/samples/shared-module/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +out/shared-module -f out/wasm-apps/testapp.wasm diff --git a/wasm-micro-runtime/samples/shared-module/src/main.c b/wasm-micro-runtime/samples/shared-module/src/main.c new file mode 100644 index 0000000..7efbc4d --- /dev/null +++ b/wasm-micro-runtime/samples/shared-module/src/main.c @@ -0,0 +1,206 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" +#include "bh_getopt.h" + +void +print_usage(void) +{ + fprintf(stdout, "Options:\r\n"); + fprintf(stdout, " -f [path of wasm file] \n"); +} + +int +main(int argc, char *argv_main[]) +{ + int exit_code = 1; + static char global_heap_buf[512 * 1024]; + char *buffer; + char error_buf[128]; + int opt; + char *wasm_path = NULL; + + const unsigned int N = 4; + wasm_module_t module = NULL; + wasm_module_inst_t module_inst[N]; + wasm_exec_env_t exec_env[N]; + const char *name_test_data_drop = "test_data_drop"; + const char *name_test_elem_drop = "test_elem_drop"; + wasm_function_inst_t func_test_data_drop[N]; + wasm_function_inst_t func_test_elem_drop[N]; + unsigned int i; + unsigned int iter; + uint32 buf_size, stack_size = 8092, heap_size = 8092; + + for (i = 0; i < N; i++) { + module_inst[i] = NULL; + exec_env[i] = NULL; + func_test_data_drop[i] = NULL; + func_test_elem_drop[i] = NULL; + } + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + while ((opt = getopt(argc, argv_main, "hf:")) != -1) { + switch (opt) { + case 'f': + wasm_path = optarg; + break; + case 'h': + print_usage(); + return 0; + case '?': + print_usage(); + return 0; + } + } + if (optind == 1) { + print_usage(); + return 0; + } + + memset(&init_args, 0, sizeof(init_args)); + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + buffer = bh_read_file_to_buffer(wasm_path, &buf_size); + + if (!buffer) { + printf("Open wasm app file [%s] failed.\n", wasm_path); + goto fail; + } + + module = wasm_runtime_load((uint8 *)buffer, buf_size, error_buf, + sizeof(error_buf)); + if (!module) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + for (i = 0; i < N; i++) { + module_inst[i] = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + + if (!module_inst[i]) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + exec_env[i] = wasm_runtime_create_exec_env(module_inst[i], stack_size); + if (!exec_env[i]) { + printf("Create wasm execution environment failed.\n"); + goto fail; + } + + func_test_data_drop[i] = + wasm_runtime_lookup_function(module_inst[i], name_test_data_drop); + if (!func_test_data_drop[i]) { + printf("The wasm function %s is not found.\n", name_test_data_drop); + goto fail; + } + + func_test_elem_drop[i] = + wasm_runtime_lookup_function(module_inst[i], name_test_elem_drop); + if (!func_test_elem_drop[i]) { + printf("The wasm function %s is not found.\n", name_test_elem_drop); + goto fail; + } + } + + for (iter = 0; iter < 2; iter++) { + /* + * as we drop data/table in the first iteration, + * the later iterations should trap. + */ + const bool should_trap = iter > 0; + + for (i = 0; i < N; i++) { + uint32_t argv[1] = {}; + if (wasm_runtime_call_wasm(exec_env[i], func_test_data_drop[i], 0, + argv)) { + uint32_t result = argv[0]; + printf( + "Native finished calling wasm function: %s, return: %x\n", + name_test_data_drop, result); + if (result != 0x64636261) { /* "abcd" */ + printf("unexpected return value\n"); + goto fail; + } + if (should_trap) { + printf("a trap is expected\n"); + goto fail; + } + } + else if (should_trap) { + printf("call wasm function %s failed as expected. error: %s\n", + name_test_data_drop, + wasm_runtime_get_exception(module_inst[i])); + } + else { + printf("call wasm function %s failed. error: %s\n", + name_test_data_drop, + wasm_runtime_get_exception(module_inst[i])); + goto fail; + } + } + + for (i = 0; i < N; i++) { + wasm_runtime_clear_exception(module_inst[i]); + + uint32_t argv[1] = {}; + if (wasm_runtime_call_wasm(exec_env[i], func_test_elem_drop[i], 0, + argv)) { + uint32_t result = argv[0]; + printf( + "Native finished calling wasm function: %s, return: %x\n", + name_test_elem_drop, result); + if (result != 0) { + printf("unexpected return value\n"); + goto fail; + } + if (should_trap) { + printf("a trap is expected\n"); + goto fail; + } + } + else if (should_trap) { + printf("call wasm function %s failed as expected. error: %s\n", + name_test_elem_drop, + wasm_runtime_get_exception(module_inst[i])); + } + else { + printf("call wasm function %s failed. error: %s\n", + name_test_elem_drop, + wasm_runtime_get_exception(module_inst[i])); + goto fail; + } + } + } + + exit_code = 0; +fail: + for (i = 0; i < N; i++) { + if (exec_env[i]) + wasm_runtime_destroy_exec_env(exec_env[i]); + if (module_inst[i]) + wasm_runtime_deinstantiate(module_inst[i]); + } + if (module) + wasm_runtime_unload(module); + if (buffer) + BH_FREE(buffer); + wasm_runtime_destroy(); + return exit_code; +} diff --git a/wasm-micro-runtime/samples/shared-module/wasm-apps/testapp.wat b/wasm-micro-runtime/samples/shared-module/wasm-apps/testapp.wat new file mode 100644 index 0000000..263a870 --- /dev/null +++ b/wasm-micro-runtime/samples/shared-module/wasm-apps/testapp.wat @@ -0,0 +1,22 @@ +;; Copyright (C) 2023 Midokura Japan KK. All rights reserved. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(module + (func (export "test_data_drop") (result i32) + (memory.init 0 (i32.const 0) (i32.const 0) (i32.const 4)) + data.drop 0 + (i32.load (i32.const 0)) + ) + (func (export "test_elem_drop") (result i32) + (table.init 0 (i32.const 0) (i32.const 0) (i32.const 4)) + elem.drop 0 + i32.const 3 + table.get 0 + ref.is_null + ) + (func $f) + (memory 1 1) + (table 4 4 funcref) + (data "abcd") + (elem func $f $f $f $f) +) diff --git a/wasm-micro-runtime/samples/socket-api/CMakeLists.txt b/wasm-micro-runtime/samples/socket-api/CMakeLists.txt new file mode 100644 index 0000000..e68a63e --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/CMakeLists.txt @@ -0,0 +1,195 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +include(CheckPIESupported) + +project(socket_api_sample) + +####################################### +## Detect toolchain +####################################### +message(CHECK_START "Detecting WASI-SDK at /opt/wasi-sdk") +if(NOT (DEFINED WASI_SDK_DIR OR DEFINED CACHE{WASI_SDK_DIR})) + find_path(WASI_SDK_PARENT + wasi-sdk + PATHS /opt + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if(WASI_SDK_PARENT) + set(WASI_SDK_DIR ${WASI_SDK_PARENT}/wasi-sdk) + endif() +endif() +if(WASI_SDK_DIR) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +if(NOT EXISTS ${WASI_SDK_DIR}) + message(FATAL_ERROR "Please install WASI-SDK under /opt/wasi-sdk") +endif() + +message(CHECK_START "Detecting WASI_TOOLCHAIN_FILE at ${WASI_SDK_DIR}") +find_file(WASI_TOOLCHAIN_FILE + wasi-sdk.cmake + PATHS "${WASI_SDK_DIR}/share/cmake" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) +if(WASI_TOOLCHAIN_FILE) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +if(WASI_TOOLCHAIN_FILE-NOTFOUND) + message(FATAL_ERROR "Can not find wasi-sdk.cmake under ${WASI_SDK_DIR}") +endif() + +message(CHECK_START "Detecting WASI_SYS_ROOT at ${WASI_SDK_DIR}") +find_path(WASI_SYS_ROOT + wasi-sysroot + PATHS "${WASI_SDK_DIR}/share" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) +if(WASI_SYS_ROOT) + message(CHECK_PASS "found") + set(WASI_SYS_ROOT ${WASI_SYS_ROOT}/wasi-sysroot) +else() + message(CHECK_FAIL "not found") +endif() + +if(WASI_SYS_ROOT-NOTFOUND) + message(FATAL_ERROR "Can not find wasi-sysroot/ under ${WASI_SDK_DIR}") +endif() + +message(STATUS "WASI_SDK_DIR is ${WASI_SDK_DIR}") +message(STATUS "WASI_TOOLCHAIN_FILE is ${WASI_TOOLCHAIN_FILE}") +message(STATUS "WASI_SYS_ROOT is ${WASI_SYS_ROOT}") + +############################################################### +## Build socket applications of wasm version and native version +############################################################### +include(ExternalProject) + +ExternalProject_Add(wasm-app + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-sdk/app/libc-builtin-sysroot/include/pthread.h + ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/inc + && ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_TOOLCHAIN_FILE} + -DCMAKE_SYSROOT=${WASI_SYS_ROOT} + ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src + BUILD_COMMAND ${CMAKE_COMMAND} --build . + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy + addr_resolve.wasm ${CMAKE_BINARY_DIR} + tcp_client.wasm ${CMAKE_BINARY_DIR} + tcp_server.wasm ${CMAKE_BINARY_DIR} + send_recv.wasm ${CMAKE_BINARY_DIR} + socket_opts.wasm ${CMAKE_BINARY_DIR} + udp_client.wasm ${CMAKE_BINARY_DIR} + udp_server.wasm ${CMAKE_BINARY_DIR} + multicast_client.wasm ${CMAKE_BINARY_DIR} + multicast_server.wasm ${CMAKE_BINARY_DIR} + timeout_client.wasm ${CMAKE_BINARY_DIR} + timeout_server.wasm ${CMAKE_BINARY_DIR} +) + +add_executable(tcp_server ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/tcp_server.c) +target_link_libraries(tcp_server pthread) + +add_executable(tcp_client ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/tcp_client.c) + +add_executable(send_recv ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/send_recv.c) +target_link_libraries(send_recv pthread) + +add_executable(addr_resolve ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/addr_resolve.c) + +add_executable(socket_opts ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/socket_opts.c) + +add_executable(udp_client ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/udp_client.c) + +add_executable(udp_server ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/udp_server.c) + +add_executable(multicast_client ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/multicast_client.c) + +add_executable(multicast_server ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/multicast_server.c) + +add_executable(timeout_client ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/timeout_client.c) + +add_executable(timeout_server ${CMAKE_CURRENT_SOURCE_DIR}/wasm-src/timeout_server.c) + +############################################ +## Build iwasm with wasi and pthread support +############################################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set WAMR features + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_FAST_INTERP 1) +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_JIT 0) +set(WAMR_BUILD_LIBC_BUILTIN 1) +set(WAMR_BUILD_LIBC_WASI 1) +set(WAMR_BUILD_LIB_PTHREAD 1) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + +# build vmlib static lib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +# build iwasm +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) +set (RUNTIME_SOURCE_ALL + ${CMAKE_CURRENT_LIST_DIR}/../../product-mini/platforms/linux/main.c + ${UNCOMMON_SHARED_SOURCE} +) +add_executable (iwasm ${RUNTIME_SOURCE_ALL}) +check_pie_supported() +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(iwasm vmlib -lpthread -lm -ldl) diff --git a/wasm-micro-runtime/samples/socket-api/README.md b/wasm-micro-runtime/samples/socket-api/README.md new file mode 100644 index 0000000..911dfb7 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/README.md @@ -0,0 +1,212 @@ +# "socket-api" sample introduction + +This sample demonstrates how to use WAMR socket-api to develop wasm network applications. +Two wasm applications are provided: tcp-server and tcp-client, and this sample demonstrates +how they communicate with each other. + +## Preparation + +Please install WASI SDK, download the [wasi-sdk release](https://github.com/WebAssembly/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. +And install wabt, download the [wabt release](https://github.com/WebAssembly/wabt/releases) and extract the archive to default path `/opt/wabt` + +## Build the sample + +```bash +mkdir build +cd build +cmake .. +make +``` + +`iwasm` and the following Wasm modules (along with their corresponding native version) will be generated: + * `addr_resolve.wasm`, `addr_resolve` + * `send_recv.wasm`, `send_recv` + * `socket_opts.wasm`, `socket_opts` + * `tcp_client.wasm`, `tcp_client` + * `tcp_server.wasm`, `tcp_server` + * `udp_client.wasm`, `udp_client` + * `udp_server.wasm`, `udp_server` + +> Note that iwasm is built with libc-wasi and lib-pthread enabled. + +## Run workload + +### TCP client/server + +Start the tcp server, which opens port 1234 and waits for clients to connect. + +```bash +cd build +./iwasm --addr-pool=0.0.0.0/15 tcp_server.wasm +``` + +Start the tcp client, which connects the server and receives message. + +```bash +cd build +./iwasm --addr-pool=127.0.0.1/15 tcp_client.wasm +``` + +The output of client is like: + +```bash +[Client] Create socket +[Client] Connect socket +[Client] Client receive +[Client] 115 bytes received: +Buffer recieved: +Say Hi from the Server +Say Hi from the Server +Say Hi from the Server +Say Hi from the Server +Say Hi from the Server + +[Client] BYE +``` + +`send_recv.wasm` contains a thread as a server and a thread as a client. They +send and receive data via 127.0.0.1:1234. + +```bash +$ ./iwasm --addr-pool=127.0.0.1/0 ./send_recv.wasm +``` + +The output is: + +```bash +Server is online ... +Client is running... +Start receiving. +Start sending. +Send 106 bytes successfully! +Receive 106 bytes successlly! +Data: + The stars shine down + It brings us light + Light comes down + To make us paths + It watches us + And mourns for us +``` + +### Socket options + +`socket_opts.wasm` shows an example of getting and setting various supported socket options +```bash +$ ./iwasm socket_opts.wasm +``` +The output is: +```bash +[Client] Create TCP socket +[Client] Create UDP socket +[Client] Create UDP IPv6 socket +setsockopt SO_RCVTIMEO result is expected +getsockopt SO_RCVTIMEO result is expected +... +[Client] Close sockets +``` + +The `timeout_client.wasm` and `timeout_server.wasm` examples demonstrate socket send and receive timeouts using the socket options. Start the server, then start the client. + +```bash +$ ./iwasm --addr-pool=0.0.0.0/15 timeout_server.wasm +``` + +The output is: + +```bash +Wait for client to connect +Client connected, sleeping for 10s +Shuting down +``` + +```bash +$ ./iwasm --addr-pool=127.0.0.1/15 timeout_client.wasm +``` + +The output is: + +```bash +Waiting on recv, which should timeout +Waiting on send, which should timeout +Success. Closing socket +``` + +The `multicast_client` and `multicast_server` examples demonstrate receiving multicast packets in WASM. Start the client and then the server with a multicast IP address and port. + +```bash +$ ./iwasm --addr-pool=0.0.0.0/0,::/0 multicast_client.wasm +$ ./iwasm --addr-pool=0.0.0.0/0,::/0 multicast_client.wasm 224.0.0.1 +$ ./iwasm --addr-pool=0.0.0.0/0,::/0 multicast_client.wasm FF02:113D:6FDD:2C17:A643:FFE2:1BD1:3CD2 +``` + +The output should be + +```bash +Joined multicast group. Waiting for datagram... +Reading datagram message...OK. +The message from multicast server is: "Test message" +``` + +```bash +$ ./multicast_server +$ ./multicast_server 224.0.0.1 +$ ./multicast_server FF02:113D:6FDD:2C17:A643:FFE2:1BD1:3CD2 +``` + +The output should be + +```bash +Datagram sent +``` + +### Domain name server resolution + +`addr_resolve.wasm` demonstrates the usage of resolving a domain name +``` +$ ./iwasm --allow-resolve=*.com addr_resolve.wasm github.com +``` + +The command displays the host name and its corresponding IP address: +``` +Host: github.com +IPv4 address: 140.82.121.4 (TCP) +``` + +### UDP client/server + +Start the UDP server, which opens port 1234 and waits for clients to send a message. + +```bash +cd build +./iwasm --addr-pool=0.0.0.0/15 udp_server.wasm +``` + +Start the tcp client, which sends a message to the server and waits for the response. + +```bash +cd build +./iwasm --addr-pool=127.0.0.1/15 udp_client.wasm +``` + +The output of client is like: + +```bash +[Client] Create socket +[Client] Client send +[Client] Client receive +[Client] Buffer recieved: Hello from server +[Client] BYE +``` + +The output of the server is like: +``` +[Server] Create socket +[Server] Bind socket +[Server] Wait for clients to connect .. +[Server] received 17 bytes from 127.0.0.1:60927: Hello from client +``` + +## Documentation + +Refer to [socket api document](../../doc/socket_api.md) for more details. diff --git a/wasm-micro-runtime/samples/socket-api/sample_test_run.py b/wasm-micro-runtime/samples/socket-api/sample_test_run.py new file mode 100755 index 0000000..ec00602 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/sample_test_run.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import argparse +import shlex +import subprocess +import sys +import time +import traceback +import glob + +WAMRC_CMD = "../../wamr-compiler/build/wamrc" + +def compile_wasm_files_to_aot(wasm_apps_dir): + wasm_files = glob.glob(wasm_apps_dir + "/*.wasm") + print("Compile wasm app into aot files") + for wasm_file in wasm_files: + aot_file = wasm_file[0 : len(wasm_file) - 5] + ".aot"; + cmd = [ WAMRC_CMD, "-o", aot_file, wasm_file ] + subprocess.check_call(cmd) + +def start_server(cmd, cwd): + app_server = subprocess.Popen(shlex.split(cmd), cwd=cwd) + return app_server + +def run_cmd(cmd, cwd): + qry_prc = subprocess.run( + shlex.split(cmd), cwd=cwd, check=False, capture_output=True + ) + if (qry_prc.returncode != 0): + print("Run {} failed, return {}".format(cmd), qry_prc.returncode) + return + print("return code: {}, output:\n{}".format(qry_prc.returncode, + qry_prc.stdout.decode())) + +def main(): + """ + GO!GO!!GO!!! + """ + parser = argparse.ArgumentParser(description="run the sample and examine outputs") + parser.add_argument("working_directory", type=str) + parser.add_argument("--aot", action='store_true', help="Test with AOT") + args = parser.parse_args() + + test_aot = False + suffix = ".wasm" + if not args.aot: + print("Test with interpreter mode") + else: + print("Test with AOT mode") + test_aot = True + suffix = ".aot" + wasm_apps_dir = args.working_directory + compile_wasm_files_to_aot(wasm_apps_dir) + + ret = 1 + app_server = None + try: + print("\n================================") + print("Test TCP server and client") + cmd = "./iwasm --addr-pool=0.0.0.0/15 tcp_server" + suffix + app_server = start_server(cmd, args.working_directory) + # wait for a second + time.sleep(1) + cmd = "./iwasm --addr-pool=127.0.0.1/15 tcp_client" + suffix + for i in range(5): + run_cmd(cmd, args.working_directory) + + print("\n================================") + print("Test UDP server and client") + cmd = "./iwasm --addr-pool=0.0.0.0/15 udp_server" + suffix + app_server = start_server(cmd, args.working_directory) + # wait for a second + time.sleep(1) + cmd = "./iwasm --addr-pool=127.0.0.1/15 udp_client" + suffix + for i in range(5): + run_cmd(cmd, args.working_directory) + + print("\n=====================================================") + print("Sleep 80 seconds to wait TCP server port actually close") + time.sleep(80) + + print("\n================================") + print("Test send and receive") + cmd = "./iwasm --addr-pool=127.0.0.1/0 ./send_recv" + suffix + run_cmd(cmd, args.working_directory) + + print("\n================================") + print("Test socket options") + cmd = "./iwasm socket_opts" + suffix + run_cmd(cmd, args.working_directory) + + print("\n================================") + print("Test timeout server and client") + cmd = "./iwasm --addr-pool=0.0.0.0/15 timeout_server" + suffix + app_server = start_server(cmd, args.working_directory) + # wait for a second + time.sleep(1) + cmd = "./iwasm --addr-pool=127.0.0.1/15 timeout_client" + suffix + run_cmd(cmd, args.working_directory) + + print("\n==========================================") + print("Test multicast_client and multicast_server") + cmd = "./iwasm --addr-pool=0.0.0.0/0,::/0 multicast_client.wasm 224.0.0.1" + app_server = start_server(cmd, args.working_directory) + # wait for a second + time.sleep(1) + cmd = "./multicast_server 224.0.0.1" + run_cmd(cmd, args.working_directory) + + cmd = "./iwasm --addr-pool=0.0.0.0/0,::/0 multicast_client.wasm FF02:113D:6FDD:2C17:A643:FFE2:1BD1:3CD2" + app_server = start_server(cmd, args.working_directory) + # wait for a second + time.sleep(1) + cmd = "./multicast_server FF02:113D:6FDD:2C17:A643:FFE2:1BD1:3CD2" + run_cmd(cmd, args.working_directory) + + print("\n================================") + print("Test address resolving") + cmd = "./iwasm --allow-resolve=*.com addr_resolve.wasm github.com" + cmd = "./multicast_server FF02:113D:6FDD:2C17:A643:FFE2:1BD1:3CD2" + run_cmd(cmd, args.working_directory) + + # wait for a second + time.sleep(1) + + print("--> All pass") + ret = 0 + except AssertionError: + traceback.print_exc() + finally: + app_server.kill() + + return ret + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/CMakeLists.txt b/wasm-micro-runtime/samples/socket-api/wasm-src/CMakeLists.txt new file mode 100644 index 0000000..8f11e0a --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/CMakeLists.txt @@ -0,0 +1,91 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 2.8...3.18) +project(socket_api_sample_wasm_app) + +message(CHECK_START "Detecting WABT") +if(NOT (DEFINED WABT_DIR OR DEFINED CACHE{WABT_DIR})) + find_path(WABT_DIR + wabt + PATHS /opt + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if(DEFINED WABT_DIR) + set(WABT_DIR ${WABT_DIR}/wabt) + endif() +endif() +if(WABT_DIR) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +message(CHECK_START "Detecting WASM_OBJDUMP at ${WABT_DIR}") +find_program(WASM_OBJDUMP + wasm-objdump + PATHS "${WABT_DIR}/bin" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) +if(WASM_OBJDUMP) + message(CHECK_PASS "found") +else() + message(CHECK_FAIL "not found") +endif() + +set(SRC ${CMAKE_CURRENT_SOURCE_DIR}) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../core/iwasm/libraries/lib-socket/lib_socket_wasi.cmake) + +function(COMPILE_WITH_CLANG SOURCE_FILE) + get_filename_component(FILE_NAME ${SOURCE_FILE} NAME_WLE) + + set(WASM_MODULE ${FILE_NAME}.wasm) + + set(MAIN_TARGET_NAME MODULE_${FILE_NAME}) + + add_executable(${MAIN_TARGET_NAME} ${SOURCE_FILE}) + set_target_properties(${MAIN_TARGET_NAME} PROPERTIES OUTPUT_NAME ${WASM_MODULE}) + target_include_directories(${MAIN_TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc) + target_compile_options(${MAIN_TARGET_NAME} INTERFACE -pthread) + target_link_libraries(${MAIN_TARGET_NAME} socket_wasi_ext) + target_link_options(${MAIN_TARGET_NAME} PRIVATE + LINKER:--export=__heap_base + LINKER:--export=__data_end + LINKER:--export=malloc + LINKER:--export=free + LINKER:--shared-memory,--max-memory=10485760 + LINKER:--no-check-features + LINKER:--allow-undefined + ) + + if(EXISTS ${WASM_OBJDUMP}) + message(STATUS "Dumping ${WASM_MODULE}...") + set(WASM_DUMP ${WASM_MODULE}.dump) + set(DUMP_TARGET_NAME DUMP_${FILE_NAME}) + + add_custom_command(OUTPUT ${WASM_DUMP} + COMMAND ${WASM_OBJDUMP} -dx ${WASM_MODULE} > ${WASM_DUMP} + COMMENT "Dumping ${WASM_MODULE}..." + DEPENDS ${MAIN_TARGET_NAME} + ) + + add_custom_target(${DUMP_TARGET_NAME} ALL + DEPENDS ${WASM_DUMP} + ) + endif() +endfunction() + +compile_with_clang(tcp_server.c) +compile_with_clang(tcp_client.c) +compile_with_clang(send_recv.c) +compile_with_clang(addr_resolve.c) +compile_with_clang(socket_opts.c) +compile_with_clang(udp_client.c) +compile_with_clang(udp_server.c) +compile_with_clang(multicast_client.c) +compile_with_clang(multicast_server.c) +compile_with_clang(timeout_client.c) +compile_with_clang(timeout_server.c) diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/addr_resolve.c b/wasm-micro-runtime/samples/socket-api/wasm-src/addr_resolve.c new file mode 100644 index 0000000..87734de --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/addr_resolve.c @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#ifdef __wasi__ +#include +#else +#include +#endif + +int +lookup_host(const char *host) +{ + struct addrinfo hints, *res, *result; + int errcode; + char addrstr[100]; + void *ptr; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + errcode = getaddrinfo(host, NULL, &hints, &result); + if (errcode != 0) { + perror("getaddrinfo"); + return -1; + } + + res = result; + + printf("Host: %s\n", host); + while (res) { + switch (res->ai_family) { + case AF_INET: + ptr = &((struct sockaddr_in *)res->ai_addr)->sin_addr; + break; + case AF_INET6: + ptr = &((struct sockaddr_in6 *)res->ai_addr)->sin6_addr; + break; + default: + printf("Unsupported address family: %d\n", res->ai_family); + continue; + } + inet_ntop(res->ai_family, ptr, addrstr, 100); + printf("IPv%d address: %s (%s)\n", res->ai_family == AF_INET6 ? 6 : 4, + addrstr, res->ai_socktype == SOCK_STREAM ? "TCP" : "UDP"); + res = res->ai_next; + } + + freeaddrinfo(result); + + return EXIT_SUCCESS; +} + +int +main(int argc, char *argv[]) +{ + if (argc < 2) { + printf("Usage: %s DOMAIN\n", argv[0]); + return EXIT_FAILURE; + } + + return lookup_host(argv[1]); +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/inc/.gitkeep b/wasm-micro-runtime/samples/socket-api/wasm-src/inc/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/multicast_client.c b/wasm-micro-runtime/samples/socket-api/wasm-src/multicast_client.c new file mode 100644 index 0000000..43201dc --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/multicast_client.c @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +static void +init_sockaddr_inet(struct sockaddr_in *addr) +{ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); +} + +static void +init_sockaddr_inet6(struct sockaddr_in6 *addr) +{ + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(1234); +} + +static int +get_ip_addr_type(char *addr, char *buf) +{ + if (inet_pton(AF_INET6, addr, buf)) { + return AF_INET6; + } + if (inet_pton(AF_INET, addr, buf)) { + return AF_INET; + } + return -1; +} + +static int +is_valid_addr_type(int addr_type) +{ + return !(addr_type == -1 + || (addr_type != AF_INET && addr_type != AF_INET6)); +} + +int +main(int argc, char *argv[]) +{ + struct ipv6_mreq ipv6_group; + struct ip_mreq ipv4_group; + int sd; + int datalen; + char databuf[1024] = { 0 }; + char multicast_addr_buffer[16]; + struct sockaddr_storage local_address = { 0 }; + int addr_type = -1; + int read_result; + int bool_opt = 1; + + if (argc < 2) { + printf("Usage is \n"); + return EXIT_FAILURE; + } + + addr_type = get_ip_addr_type(argv[1], multicast_addr_buffer); + + if (!is_valid_addr_type(addr_type)) { + printf("Not a valid ipv4 or ipv6 address\n"); + return EXIT_FAILURE; + } + + if ((sd = socket(addr_type, SOCK_DGRAM, 0)) == -1) { + perror("Failed opening socket"); + return EXIT_FAILURE; + } + + if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &bool_opt, sizeof(bool_opt)) + == -1) { + perror("Failed setting SO_REUSEADDR"); + goto fail; + } + + if (addr_type == AF_INET) { + init_sockaddr_inet((struct sockaddr_in *)&local_address); + memcpy(&(ipv4_group.imr_multiaddr), multicast_addr_buffer, 4); + ipv4_group.imr_interface.s_addr = htonl(INADDR_ANY); + + if (setsockopt(sd, IPPROTO_IP, IP_ADD_MEMBERSHIP, &ipv4_group, + sizeof(ipv4_group)) + == -1) { + perror("Failed joining IPv4 multicast group"); + goto fail; + } + } + else { + init_sockaddr_inet6((struct sockaddr_in6 *)&local_address); + memcpy(&(ipv6_group.ipv6mr_multiaddr), multicast_addr_buffer, 16); + ipv6_group.ipv6mr_interface = 0; + + if (setsockopt(sd, IPPROTO_IPV6, IPV6_JOIN_GROUP, &ipv6_group, + sizeof(ipv6_group)) + == -1) { + perror("Failed joining IPv6 multicast group"); + goto fail; + } + } + + if (bind(sd, (struct sockaddr *)&local_address, sizeof(local_address)) + == -1) { + perror("Failed binding socket"); + goto fail; + } + + printf("Joined multicast group. Waiting for datagram...\n"); + + datalen = sizeof(databuf) - 1; + read_result = read(sd, databuf, datalen); + + if (read_result < 0) { + perror("Failed binding socket"); + goto fail; + } + + printf("Reading datagram message...OK.\n"); + printf("The message from multicast server is: \"%s\"\n", databuf); + close(sd); + return EXIT_SUCCESS; + +fail: + close(sd); + return EXIT_FAILURE; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/multicast_server.c b/wasm-micro-runtime/samples/socket-api/wasm-src/multicast_server.c new file mode 100644 index 0000000..cd33557 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/multicast_server.c @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +static int +get_ip_addr_type(char *addr, char *buf) +{ + if (inet_pton(AF_INET6, addr, buf)) { + return AF_INET6; + } + if (inet_pton(AF_INET, addr, buf)) { + return AF_INET; + } + return -1; +} + +static int +is_valid_addr_type(int addr_type) +{ + return !(addr_type == -1 + || (addr_type != AF_INET && addr_type != AF_INET6)); +} + +static void +init_sockaddr_inet(struct sockaddr_in *addr, char *addr_buffer) +{ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); + memcpy(&(addr->sin_addr), addr_buffer, 4); +} + +static void +init_sockaddr_inet6(struct sockaddr_in6 *addr, char *addr_buffer) +{ + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(1234); + memcpy(&(addr->sin6_addr), addr_buffer, 16); +} + +int +main(int argc, char *argv[]) +{ + struct sockaddr_storage addr = { 0 }; + int sd; + char *databuf = "Test message"; + int datalen = strlen(databuf) + 1; + char multicast_addr_buffer[16]; + int addr_type = -1; + int multicast_interface; + int bool_opt = 1; + + if (argc < 2) { + printf("Usage is \n"); + return EXIT_FAILURE; + } + + addr_type = get_ip_addr_type(argv[1], multicast_addr_buffer); + + if (!is_valid_addr_type(addr_type)) { + printf("Not a valid ipv4 or ipv6 address\n"); + return EXIT_FAILURE; + } + + if ((sd = socket(addr_type, SOCK_DGRAM, 0)) == -1) { + return EXIT_FAILURE; + } + + if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &bool_opt, sizeof(bool_opt)) + == -1) { + perror("Failed setting SO_REUSEADDR"); + goto fail; + } + + if (addr_type == AF_INET) { + multicast_interface = htonl(INADDR_ANY); + if (setsockopt(sd, IPPROTO_IP, IP_MULTICAST_IF, + (char *)&multicast_interface, + sizeof(multicast_interface))) { + perror("Failed setting local interface"); + goto fail; + } + init_sockaddr_inet((struct sockaddr_in *)&addr, multicast_addr_buffer); + } + else { + multicast_interface = 0; + if (setsockopt(sd, IPPROTO_IPV6, IPV6_MULTICAST_IF, + (char *)&multicast_interface, + sizeof(multicast_interface))) { + perror("Failed setting local interface"); + goto fail; + } + init_sockaddr_inet6((struct sockaddr_in6 *)&addr, + multicast_addr_buffer); + } + + if (sendto(sd, databuf, datalen, 0, (struct sockaddr *)&addr, sizeof(addr)) + == -1) { + perror("Failed sending datagram"); + goto fail; + } + + printf("Datagram sent\n"); + close(sd); + return EXIT_SUCCESS; + +fail: + close(sd); + return EXIT_FAILURE; +} \ No newline at end of file diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/send_recv.c b/wasm-micro-runtime/samples/socket-api/wasm-src/send_recv.c new file mode 100644 index 0000000..0071b2a --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/send_recv.c @@ -0,0 +1,219 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +static pthread_mutex_t lock = { 0 }; +static pthread_cond_t cond = { 0 }; +static bool server_create_failed = false; +static bool server_is_ready = false; + +void * +run_as_server(void *arg) +{ + int sock = -1, on = 1; + struct sockaddr_in addr = { 0 }; + int addrlen = 0; + int new_sock = -1; + char *buf[] = { + "The stars shine down", "It brings us light", "Light comes down", + "To make us paths", "It watches us", "And mourns for us", + }; + struct iovec iov[] = { + { .iov_base = buf[0], .iov_len = strlen(buf[0]) + 1 }, + { .iov_base = buf[1], .iov_len = strlen(buf[1]) + 1 }, + { .iov_base = buf[2], .iov_len = strlen(buf[2]) + 1 }, + { .iov_base = buf[3], .iov_len = strlen(buf[3]) + 1 }, + { .iov_base = buf[4], .iov_len = strlen(buf[4]) + 1 }, + { .iov_base = buf[5], .iov_len = strlen(buf[5]) + 1 }, + }; + struct msghdr msg = { .msg_iov = iov, .msg_iovlen = 6 }; + ssize_t send_len = 0; + + pthread_mutex_lock(&lock); + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Create a socket failed"); + return NULL; + } + +#ifndef __wasi__ + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on))) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Setsockopt failed"); + goto fail1; + } +#endif + + /* 0.0.0.0:1234 */ + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + addrlen = sizeof(addr); + if (bind(sock, (struct sockaddr *)&addr, addrlen) < 0) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Bind failed"); + goto fail1; + } + + if (listen(sock, 0) < 0) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Listen failed"); + goto fail1; + } + + server_is_ready = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + + printf("Server is online ... \n"); + + new_sock = accept(sock, (struct sockaddr *)&addr, (socklen_t *)&addrlen); + if (new_sock < 0) { + perror("Accept failed"); + goto fail1; + } + + printf("Start sending. \n"); + send_len = sendmsg(new_sock, &msg, 0); + if (send_len < 0) { + perror("Sendmsg failed"); + goto fail2; + } + printf("Send %ld bytes successfully!\n", send_len); + +fail2: + close(new_sock); +fail1: + shutdown(sock, SHUT_RD); + close(sock); + return NULL; +} + +void * +run_as_client(void *arg) +{ + int sock = -1; + struct sockaddr_in addr = { 0 }; + /* buf of server is 106 bytes */ + char buf[110] = { 0 }; + struct iovec iov = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct msghdr msg = { .msg_iov = &iov, .msg_iovlen = 1 }; + ssize_t recv_len = 0; + + pthread_mutex_lock(&lock); + while (!server_create_failed && !server_is_ready) { + pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + + if (server_create_failed) { + return NULL; + } + + printf("Client is running...\n"); + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + perror("Create a socket failed"); + return NULL; + } + + /* 127.0.0.1:1234 */ + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + perror("Connect failed"); + goto fail; + } + + printf("Start receiving. \n"); + recv_len = recvmsg(sock, &msg, 0); + if (recv_len < 0) { + perror("Recvmsg failed"); + goto fail; + } + + printf("Receive %ld bytes successlly!\n", recv_len); + assert(recv_len == 106); + + printf("Data:\n"); + char *s = msg.msg_iov->iov_base; + while (strlen(s) > 0) { + printf(" %s\n", s); + s += strlen(s) + 1; + } + +fail: + shutdown(sock, SHUT_RD); + close(sock); + return NULL; +} + +int +main(int argc, char *argv[]) +{ + pthread_t cs[2] = { 0 }; + uint8_t i = 0; + int ret = EXIT_SUCCESS; + + if (pthread_mutex_init(&lock, NULL)) { + perror("Initialize mutex failed"); + ret = EXIT_FAILURE; + goto RETURN; + } + + if (pthread_cond_init(&cond, NULL)) { + perror("Initialize condition failed"); + ret = EXIT_FAILURE; + goto DESTROY_MUTEX; + } + + if (pthread_create(&cs[0], NULL, run_as_server, NULL)) { + perror("Create a server thread failed"); + ret = EXIT_FAILURE; + goto DESTROY_COND; + } + + if (pthread_create(&cs[1], NULL, run_as_client, NULL)) { + perror("Create a client thread failed"); + ret = EXIT_FAILURE; + goto DESTROY_COND; + } + + for (i = 0; i < 2; i++) { + pthread_join(cs[i], NULL); + } + +DESTROY_COND: + pthread_cond_destroy(&cond); +DESTROY_MUTEX: + pthread_mutex_destroy(&lock); +RETURN: + return ret; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/socket_opts.c b/wasm-micro-runtime/samples/socket-api/wasm-src/socket_opts.c new file mode 100644 index 0000000..890cc0c --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/socket_opts.c @@ -0,0 +1,299 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +#define MULTICAST_ADDR 16777440 +#define OPTION_ASSERT(A, B, OPTION) \ + if (A == B) { \ + printf("%s is expected\n", OPTION); \ + } \ + else { \ + printf("%s is unexpected\n", OPTION); \ + perror("assertion failed"); \ + return EXIT_FAILURE; \ + } + +static struct timeval +to_timeval(time_t tv_sec, suseconds_t tv_usec) +{ + struct timeval tv = { tv_sec, tv_usec }; + return tv; +} + +static int +set_and_get_bool_opt(int socket_fd, int level, int optname, int val) +{ + int bool_opt = val; + int ret = -1; + socklen_t opt_len = sizeof(bool_opt); + + ret = setsockopt(socket_fd, level, optname, &bool_opt, sizeof(bool_opt)); + if (ret != 0) + return !val; + + bool_opt = !bool_opt; + ret = getsockopt(socket_fd, level, optname, &bool_opt, &opt_len); + if (ret != 0) + return !val; + + return bool_opt; +} + +int +main(int argc, char *argv[]) +{ + int tcp_socket_fd = 0; + int udp_socket_fd = 0; + int udp_ipv6_socket_fd = 0; + struct timeval tv; + socklen_t opt_len; + int buf_len; + int result; + struct linger linger_opt; + uint32_t time_s; + int ttl; + + printf("[Client] Create TCP socket\n"); + tcp_socket_fd = socket(AF_INET, SOCK_STREAM, 0); + if (tcp_socket_fd == -1) { + perror("Create socket failed"); + return EXIT_FAILURE; + } + + printf("[Client] Create UDP socket\n"); + udp_socket_fd = socket(AF_INET, SOCK_DGRAM, 0); + if (udp_socket_fd == -1) { + perror("Create socket failed"); + return EXIT_FAILURE; + } + + printf("[Client] Create UDP IPv6 socket\n"); + udp_ipv6_socket_fd = socket(AF_INET6, SOCK_DGRAM, 0); + if (udp_ipv6_socket_fd == -1) { + perror("Create socket failed"); + return EXIT_FAILURE; + } + + // SO_RCVTIMEO + tv = to_timeval(123, 1000); + result = + setsockopt(tcp_socket_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + OPTION_ASSERT(result, 0, "setsockopt SO_RCVTIMEO result") + + tv = to_timeval(0, 0); + opt_len = sizeof(tv); + result = getsockopt(tcp_socket_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, &opt_len); + OPTION_ASSERT(result, 0, "getsockopt SO_RCVTIMEO result") + OPTION_ASSERT(tv.tv_sec, 123, "SO_RCVTIMEO tv_sec"); + // OPTION_ASSERT(tv.tv_usec, 1000, "SO_RCVTIMEO tv_usec"); + + // SO_SNDTIMEO + tv = to_timeval(456, 2000); + result = + setsockopt(tcp_socket_fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + OPTION_ASSERT(result, 0, "setsockopt SO_SNDTIMEO result") + + tv = to_timeval(0, 0); + opt_len = sizeof(tv); + result = getsockopt(tcp_socket_fd, SOL_SOCKET, SO_SNDTIMEO, &tv, &opt_len); + OPTION_ASSERT(result, 0, "getsockopt SO_SNDTIMEO result") + OPTION_ASSERT(tv.tv_sec, 456, "SO_SNDTIMEO tv_sec"); + // OPTION_ASSERT(tv.tv_usec, 2000, "SO_SNDTIMEO tv_usec"); + + // SO_SNDBUF + buf_len = 8192; + result = setsockopt(tcp_socket_fd, SOL_SOCKET, SO_SNDBUF, &buf_len, + sizeof(buf_len)); + OPTION_ASSERT(result, 0, "setsockopt SO_SNDBUF result") + + buf_len = 0; + opt_len = sizeof(buf_len); + result = + getsockopt(tcp_socket_fd, SOL_SOCKET, SO_SNDBUF, &buf_len, &opt_len); + OPTION_ASSERT(result, 0, "getsockopt SO_SNDBUF result") + OPTION_ASSERT((buf_len == 16384 || buf_len == 8192), 1, + "SO_SNDBUF buf_len"); + + // SO_RCVBUF + buf_len = 4096; + result = setsockopt(tcp_socket_fd, SOL_SOCKET, SO_RCVBUF, &buf_len, + sizeof(buf_len)); + OPTION_ASSERT(result, 0, "setsockopt SO_RCVBUF result") + + buf_len = 0; + opt_len = sizeof(buf_len); + result = + getsockopt(tcp_socket_fd, SOL_SOCKET, SO_RCVBUF, &buf_len, &opt_len); + OPTION_ASSERT(result, 0, "getsockopt SO_RCVBUF result") + OPTION_ASSERT((buf_len == 8192 || buf_len == 4096), 1, "SO_SNDBUF buf_len"); + + // SO_KEEPALIVE + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, SOL_SOCKET, SO_KEEPALIVE, 1), 1, + "SO_KEEPALIVE enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, SOL_SOCKET, SO_KEEPALIVE, 0), 0, + "SO_KEEPALIVE disabled"); + + // SO_REUSEADDR + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, SOL_SOCKET, SO_REUSEADDR, 1), 1, + "SO_REUSEADDR enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, SOL_SOCKET, SO_REUSEADDR, 0), 0, + "SO_REUSEADDR disabled"); + + // SO_REUSEPORT + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, SOL_SOCKET, SO_REUSEPORT, 1), 1, + "SO_REUSEPORT enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, SOL_SOCKET, SO_REUSEPORT, 0), 0, + "SO_REUSEPORT disabled"); + + // SO_LINGER + linger_opt.l_onoff = 1; + linger_opt.l_linger = 10; + result = setsockopt(tcp_socket_fd, SOL_SOCKET, SO_LINGER, &linger_opt, + sizeof(linger_opt)); + OPTION_ASSERT(result, 0, "setsockopt SO_LINGER result") + + linger_opt.l_onoff = 0; + linger_opt.l_linger = 0; + opt_len = sizeof(linger_opt); + result = + getsockopt(tcp_socket_fd, SOL_SOCKET, SO_LINGER, &linger_opt, &opt_len); + OPTION_ASSERT(result, 0, "getsockopt SO_LINGER result") + OPTION_ASSERT(linger_opt.l_onoff, 1, "SO_LINGER l_onoff"); + OPTION_ASSERT(linger_opt.l_linger, 10, "SO_LINGER l_linger"); + + // SO_BROADCAST + OPTION_ASSERT( + set_and_get_bool_opt(udp_socket_fd, SOL_SOCKET, SO_BROADCAST, 1), 1, + "SO_BROADCAST enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(udp_socket_fd, SOL_SOCKET, SO_BROADCAST, 0), 0, + "SO_BROADCAST disabled"); + + // TCP_KEEPIDLE +#ifdef TCP_KEEPIDLE + time_s = 16; + result = setsockopt(tcp_socket_fd, IPPROTO_TCP, TCP_KEEPIDLE, &time_s, + sizeof(time_s)); + OPTION_ASSERT(result, 0, "setsockopt TCP_KEEPIDLE result") + + time_s = 0; + opt_len = sizeof(time_s); + result = + getsockopt(tcp_socket_fd, IPPROTO_TCP, TCP_KEEPIDLE, &time_s, &opt_len); + OPTION_ASSERT(result, 0, "getsockopt TCP_KEEPIDLE result") + OPTION_ASSERT(time_s, 16, "TCP_KEEPIDLE"); +#endif + + // TCP_KEEPINTVL + time_s = 8; + result = setsockopt(tcp_socket_fd, IPPROTO_TCP, TCP_KEEPINTVL, &time_s, + sizeof(time_s)); + OPTION_ASSERT(result, 0, "setsockopt TCP_KEEPINTVL result") + + time_s = 0; + opt_len = sizeof(time_s); + result = getsockopt(tcp_socket_fd, IPPROTO_TCP, TCP_KEEPINTVL, &time_s, + &opt_len); + OPTION_ASSERT(result, 0, "getsockopt TCP_KEEPINTVL result") + OPTION_ASSERT(time_s, 8, "TCP_KEEPINTVL"); + + // TCP_FASTOPEN_CONNECT +#ifdef TCP_FASTOPEN_CONNECT + OPTION_ASSERT(set_and_get_bool_opt(tcp_socket_fd, IPPROTO_TCP, + TCP_FASTOPEN_CONNECT, 1), + 1, "TCP_FASTOPEN_CONNECT enabled"); + OPTION_ASSERT(set_and_get_bool_opt(tcp_socket_fd, IPPROTO_TCP, + TCP_FASTOPEN_CONNECT, 0), + 0, "TCP_FASTOPEN_CONNECT disabled"); +#endif + + // TCP_NODELAY + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, IPPROTO_TCP, TCP_NODELAY, 1), 1, + "TCP_NODELAY enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, IPPROTO_TCP, TCP_NODELAY, 0), 0, + "TCP_NODELAY disabled"); + + // TCP_QUICKACK +#ifdef TCP_QUICKACK + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, IPPROTO_TCP, TCP_QUICKACK, 1), 1, + "TCP_QUICKACK enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(tcp_socket_fd, IPPROTO_TCP, TCP_QUICKACK, 0), 0, + "TCP_QUICKACK disabled"); +#endif + + // IP_TTL + ttl = 8; + result = setsockopt(tcp_socket_fd, IPPROTO_IP, IP_TTL, &ttl, sizeof(ttl)); + OPTION_ASSERT(result, 0, "IP_TIL"); + ttl = 0; + opt_len = sizeof(ttl); + result = getsockopt(tcp_socket_fd, IPPROTO_IP, IP_TTL, &ttl, &opt_len); + OPTION_ASSERT(ttl, 8, "IP_TTL"); + OPTION_ASSERT(result, 0, "IP_TIL"); + + // IPV6_V6ONLY + OPTION_ASSERT( + set_and_get_bool_opt(udp_ipv6_socket_fd, IPPROTO_IPV6, IPV6_V6ONLY, 1), + 1, "IPV6_V6ONLY enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(udp_ipv6_socket_fd, IPPROTO_IPV6, IPV6_V6ONLY, 0), + 0, "IPV6_V6ONLY disabled"); + + // IP_MULTICAST_LOOP + OPTION_ASSERT( + set_and_get_bool_opt(udp_socket_fd, IPPROTO_IP, IP_MULTICAST_LOOP, 1), + 1, "IP_MULTICAST_LOOP enabled"); + OPTION_ASSERT( + set_and_get_bool_opt(udp_socket_fd, IPPROTO_IP, IP_MULTICAST_LOOP, 0), + 0, "IP_MULTICAST_LOOP disabled"); + + // IP_MULTICAST_TTL + ttl = 8; + result = setsockopt(udp_socket_fd, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, + sizeof(ttl)); + OPTION_ASSERT(result, 0, "IP_MULTICAST_TTL"); + ttl = 0; + opt_len = sizeof(ttl); + result = + getsockopt(udp_socket_fd, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, &opt_len); + OPTION_ASSERT(ttl, 8, "IP_MULTICAST_TTL"); + OPTION_ASSERT(result, 0, "IP_MULTICAST_TTL"); + + // IPV6_MULTICAST_LOOP + OPTION_ASSERT(set_and_get_bool_opt(udp_ipv6_socket_fd, IPPROTO_IPV6, + IPV6_MULTICAST_LOOP, 1), + 1, "IPV6_MULTICAST_LOOP enabled"); + OPTION_ASSERT(set_and_get_bool_opt(udp_ipv6_socket_fd, IPPROTO_IPV6, + IPV6_MULTICAST_LOOP, 0), + 0, "IPV6_MULTICAST_LOOP disabled"); + + printf("[Client] Close sockets\n"); + close(tcp_socket_fd); + close(udp_socket_fd); + close(udp_ipv6_socket_fd); + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/socket_utils.h b/wasm-micro-runtime/samples/socket-api/wasm-src/socket_utils.h new file mode 100644 index 0000000..b69135b --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/socket_utils.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef TCP_UTILS_H +#define TCP_UTILS_H + +#include +#include +#include + +int +sockaddr_to_string(struct sockaddr *addr, char *str, size_t len) +{ + uint16_t port; + char ip_string[64]; + void *addr_buf; + int ret; + + switch (addr->sa_family) { + case AF_INET: + { + struct sockaddr_in *addr_in = (struct sockaddr_in *)addr; + port = addr_in->sin_port; + addr_buf = &addr_in->sin_addr; + break; + } + case AF_INET6: + { + struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr; + port = addr_in6->sin6_port; + addr_buf = &addr_in6->sin6_addr; + break; + } + default: + return -1; + } + + inet_ntop(addr->sa_family, addr_buf, ip_string, + sizeof(ip_string) / sizeof(ip_string[0])); + + ret = snprintf(str, len, "%s:%d", ip_string, ntohs(port)); + + return ret > 0 && (size_t)ret < len ? 0 : -1; +} + +#endif /* TCP_UTILS_H */ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/tcp_client.c b/wasm-micro-runtime/samples/socket-api/wasm-src/tcp_client.c new file mode 100644 index 0000000..aad4494 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/tcp_client.c @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include "socket_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +static void +init_sockaddr_inet(struct sockaddr_in *addr) +{ + /* 127.0.0.1:1234 */ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); + addr->sin_addr.s_addr = htonl(INADDR_LOOPBACK); +} + +static void +init_sockaddr_inet6(struct sockaddr_in6 *addr) +{ + /* [::1]:1234 */ + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(1234); + addr->sin6_addr = in6addr_loopback; +} + +int +main(int argc, char *argv[]) +{ + int socket_fd, ret, total_size = 0, af; + char buffer[1024] = { 0 }; + char ip_string[64] = { 0 }; + socklen_t len; + struct sockaddr_storage server_address = { 0 }; + struct sockaddr_storage local_address = { 0 }; + + if (argc > 1 && strcmp(argv[1], "inet6") == 0) { + af = AF_INET6; + len = sizeof(struct sockaddr_in6); + init_sockaddr_inet6((struct sockaddr_in6 *)&server_address); + } + else { + af = AF_INET; + len = sizeof(struct sockaddr_in); + init_sockaddr_inet((struct sockaddr_in *)&server_address); + } + + printf("[Client] Create socket\n"); + socket_fd = socket(af, SOCK_STREAM, 0); + if (socket_fd == -1) { + perror("Create socket failed"); + return EXIT_FAILURE; + } + + printf("[Client] Connect socket\n"); + if (connect(socket_fd, (struct sockaddr *)&server_address, len) == -1) { + perror("Connect failed"); + close(socket_fd); + return EXIT_FAILURE; + } + + len = sizeof(local_address); + ret = getsockname(socket_fd, (struct sockaddr *)&local_address, &len); + if (ret == -1) { + perror("Failed to retrieve socket address"); + close(socket_fd); + return EXIT_FAILURE; + } + + if (sockaddr_to_string((struct sockaddr *)&local_address, ip_string, + sizeof(ip_string) / sizeof(ip_string[0])) + != 0) { + printf("[Client] failed to parse local address\n"); + close(socket_fd); + return EXIT_FAILURE; + } + + printf("[Client] Local address is: %s\n", ip_string); + + printf("[Client] Client receive\n"); + while (1) { + ret = recv(socket_fd, buffer + total_size, sizeof(buffer) - total_size, + 0); + if (ret <= 0) + break; + total_size += ret; + } + + printf("[Client] %d bytes received:\n", total_size); + if (total_size > 0) { + printf("Buffer recieved:\n%s\n", buffer); + } + + close(socket_fd); + printf("[Client] BYE \n"); + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/tcp_server.c b/wasm-micro-runtime/samples/socket-api/wasm-src/tcp_server.c new file mode 100644 index 0000000..2798dc2 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/tcp_server.c @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include "socket_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +#define WORKER_NUM 5 + +void * +run(void *arg) +{ + const char *message = "Say Hi from the Server\n"; + int new_socket = *(int *)arg; + int i; + + printf("[Server] Communicate with the new connection #%u @ %p ..\n", + new_socket, (void *)(uintptr_t)pthread_self()); + + for (i = 0; i < 5; i++) { + if (send(new_socket, message, strlen(message), 0) < 0) { + perror("Send failed"); + break; + } + } + + printf("[Server] Shuting down the new connection #%u ..\n", new_socket); + shutdown(new_socket, SHUT_RDWR); + + return NULL; +} + +static void +init_sockaddr_inet(struct sockaddr_in *addr) +{ + /* 0.0.0.0:1234 */ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); + addr->sin_addr.s_addr = htonl(INADDR_ANY); +} + +static void +init_sockaddr_inet6(struct sockaddr_in6 *addr) +{ + /* [::]:1234 */ + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(1234); + addr->sin6_addr = in6addr_any; +} + +int +main(int argc, char *argv[]) +{ + int socket_fd = -1, addrlen = 0, af; + struct sockaddr_storage addr = { 0 }; + unsigned connections = 0; + pthread_t workers[WORKER_NUM] = { 0 }; + int client_sock_fds[WORKER_NUM] = { 0 }; + char ip_string[64]; + + if (argc > 1 && strcmp(argv[1], "inet6") == 0) { + af = AF_INET6; + addrlen = sizeof(struct sockaddr_in6); + init_sockaddr_inet6((struct sockaddr_in6 *)&addr); + } + else { + af = AF_INET; + addrlen = sizeof(struct sockaddr_in6); + init_sockaddr_inet((struct sockaddr_in *)&addr); + } + + printf("[Server] Create socket\n"); + socket_fd = socket(af, SOCK_STREAM, 0); + if (socket_fd < 0) { + perror("Create socket failed"); + goto fail; + } + + printf("[Server] Bind socket\n"); + if (bind(socket_fd, (struct sockaddr *)&addr, addrlen) < 0) { + perror("Bind failed"); + goto fail; + } + + printf("[Server] Listening on socket\n"); + if (listen(socket_fd, 3) < 0) { + perror("Listen failed"); + goto fail; + } + + printf("[Server] Wait for clients to connect ..\n"); + while (connections < WORKER_NUM) { + addrlen = sizeof(struct sockaddr); + client_sock_fds[connections] = + accept(socket_fd, (struct sockaddr *)&addr, (socklen_t *)&addrlen); + if (client_sock_fds[connections] < 0) { + perror("Accept failed"); + break; + } + + if (sockaddr_to_string((struct sockaddr *)&addr, ip_string, + sizeof(ip_string) / sizeof(ip_string[0])) + != 0) { + printf("[Server] failed to parse client address\n"); + goto fail; + } + + printf("[Server] Client connected (%s)\n", ip_string); + if (pthread_create(&workers[connections], NULL, run, + &client_sock_fds[connections])) { + perror("Create a worker thread failed"); + shutdown(client_sock_fds[connections], SHUT_RDWR); + break; + } + + connections++; + } + + if (connections == WORKER_NUM) { + printf("[Server] Achieve maximum amount of connections\n"); + } + + for (int i = 0; i < WORKER_NUM; i++) { + pthread_join(workers[i], NULL); + } + + printf("[Server] Shuting down ..\n"); + shutdown(socket_fd, SHUT_RDWR); + sleep(3); + printf("[Server] BYE \n"); + return EXIT_SUCCESS; + +fail: + printf("[Server] Shuting down ..\n"); + if (socket_fd >= 0) + close(socket_fd); + sleep(3); + return EXIT_FAILURE; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/timeout_client.c b/wasm-micro-runtime/samples/socket-api/wasm-src/timeout_client.c new file mode 100644 index 0000000..652021b --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/timeout_client.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +int +main(int argc, char *argv[]) +{ + int socket_fd; + struct sockaddr_in addr; + struct timeval tv = { 0, 1 }; + const int snd_buf_len = 8; + const int data_buf_len = 1000000; + char *buffer = (char *)malloc(sizeof(char) * data_buf_len); + int result; + socklen_t opt_len = sizeof(snd_buf_len); + struct timeval snd_start_time, snd_end_time; + int bool_opt = 1; + + if (buffer == NULL) { + perror("Allocation failed, please re-run with larger heap size"); + return EXIT_FAILURE; + } + + /* 127.0.0.1:1234 */ + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + if ((socket_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { + perror("Create socket failed"); + goto fail1; + } + + if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, &bool_opt, + sizeof(bool_opt)) + == -1) { + perror("Failed setting SO_REUSEADDR"); + goto fail2; + } + + if (setsockopt(socket_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) { + perror("Failed setting SO_RCVTIMEO"); + goto fail2; + } + + if (setsockopt(socket_fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) == -1) { + perror("Failed setting SO_SNDTIMEO"); + goto fail2; + } + + if (setsockopt(socket_fd, SOL_SOCKET, SO_SNDBUF, &data_buf_len, + sizeof(data_buf_len)) + == -1) { + perror("Failed setting SO_SNDBUF"); + goto fail2; + } + + if (connect(socket_fd, (struct sockaddr *)&addr, sizeof(addr)) == -1) { + perror("Connect failed"); + goto fail2; + } + + if (getsockopt(socket_fd, SOL_SOCKET, SO_SNDBUF, (void *)&data_buf_len, + &opt_len) + == -1) { + perror("Failed getting SO_SNDBUF"); + goto fail2; + } + + printf("Waiting on recv, which should timeout\n"); + result = recv(socket_fd, buffer, 1, 0); + + if (result != -1 || errno != EAGAIN) { + perror("Recv did not timeout as expected"); + goto fail2; + } + + printf("Waiting on send, which should timeout\n"); + gettimeofday(&snd_start_time, NULL); + result = send(socket_fd, buffer, data_buf_len, 0); + gettimeofday(&snd_end_time, NULL); + + if (result >= data_buf_len + || snd_start_time.tv_sec != snd_end_time.tv_sec) { + perror("Send did not timeout as expected"); + goto fail2; + } + + printf("Success. Closing socket \n"); + close(socket_fd); + free(buffer); + return EXIT_SUCCESS; + +fail2: + close(socket_fd); +fail1: + free(buffer); + return EXIT_FAILURE; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/timeout_server.c b/wasm-micro-runtime/samples/socket-api/wasm-src/timeout_server.c new file mode 100644 index 0000000..c71cf45 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/timeout_server.c @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +int +main(int argc, char *argv[]) +{ + int socket_fd; + int client_socket_fd; + struct sockaddr_in addr = { 0 }; + int addrlen = sizeof(addr); + int bool_opt = 1; + + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_ANY); + if ((socket_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { + perror("Create socket failed"); + return EXIT_FAILURE; + } + + if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, &bool_opt, + sizeof(bool_opt)) + == -1) { + perror("Failed setting SO_REUSEADDR"); + goto fail; + } + + if (bind(socket_fd, (struct sockaddr *)&addr, addrlen) == -1) { + perror("Bind socket failed"); + goto fail; + } + + if (listen(socket_fd, 1) == -1) { + perror("Listen failed"); + goto fail; + } + + if ((client_socket_fd = + accept(socket_fd, (struct sockaddr *)&addr, (socklen_t *)&addrlen)) + == -1) { + perror("Accept failed"); + goto fail; + } + + printf("Client connected, sleeping for 10s\n"); + sleep(10); + + printf("Shuting down\n"); + shutdown(client_socket_fd, SHUT_RDWR); + close(client_socket_fd); + shutdown(socket_fd, SHUT_RDWR); + close(socket_fd); + return EXIT_SUCCESS; + +fail: + close(socket_fd); + return EXIT_FAILURE; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/udp_client.c b/wasm-micro-runtime/samples/socket-api/wasm-src/udp_client.c new file mode 100644 index 0000000..810a455 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/udp_client.c @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +static void +init_sockaddr_inet(struct sockaddr_in *addr) +{ + /* 127.0.0.1:1234 */ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); + addr->sin_addr.s_addr = htonl(INADDR_LOOPBACK); +} + +static void +init_sockaddr_inet6(struct sockaddr_in6 *addr) +{ + /* [::1]:1234 */ + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(1234); + addr->sin6_addr = in6addr_loopback; +} + +int +main(int argc, char *argv[]) +{ + int socket_fd, ret, af; + char buffer[1024] = { 0 }; + socklen_t serverlen; + struct sockaddr_storage server_address = { 0 }; + const char *message = "Hello from client"; + + if (argc > 1 && strcmp(argv[1], "inet6") == 0) { + af = AF_INET6; + init_sockaddr_inet6((struct sockaddr_in6 *)&server_address); + serverlen = sizeof(struct sockaddr_in6); + } + else { + af = AF_INET; + init_sockaddr_inet((struct sockaddr_in *)&server_address); + serverlen = sizeof(struct sockaddr_in); + } + + printf("[Client] Create socket\n"); + socket_fd = socket(af, SOCK_DGRAM, 0); + if (socket_fd == -1) { + perror("Create socket failed"); + return EXIT_FAILURE; + } + + printf("[Client] Client send\n"); + ret = sendto(socket_fd, message, strlen(message), 0, + (struct sockaddr *)&server_address, serverlen); + if (ret < 0) { + close(socket_fd); + perror("Send failed"); + return EXIT_FAILURE; + } + + printf("[Client] Client receive\n"); + serverlen = sizeof(server_address); + /* make sure there is space for the string terminator */ + ret = recvfrom(socket_fd, buffer, sizeof(buffer) - 1, 0, + (struct sockaddr *)&server_address, &serverlen); + + if (ret > 0) { + buffer[ret] = '\0'; + printf("[Client] Buffer recieved: %s\n", buffer); + } + + close(socket_fd); + printf("[Client] BYE \n"); + return EXIT_SUCCESS; +} diff --git a/wasm-micro-runtime/samples/socket-api/wasm-src/udp_server.c b/wasm-micro-runtime/samples/socket-api/wasm-src/udp_server.c new file mode 100644 index 0000000..5889641 --- /dev/null +++ b/wasm-micro-runtime/samples/socket-api/wasm-src/udp_server.c @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "socket_utils.h" + +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#endif + +#define MAX_CONNECTIONS_COUNT 5 + +static void +init_sockaddr_inet(struct sockaddr_in *addr) +{ + /* 0.0.0.0:1234 */ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); + addr->sin_addr.s_addr = htonl(INADDR_ANY); +} + +static void +init_sockaddr_inet6(struct sockaddr_in6 *addr) +{ + /* [::]:1234 */ + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(1234); + addr->sin6_addr = in6addr_any; +} + +int +main(int argc, char *argv[]) +{ + int socket_fd = -1, af; + socklen_t addrlen = 0; + struct sockaddr_storage addr = { 0 }; + char *reply_message = "Hello from server"; + unsigned connections = 0; + char ip_string[64] = { 0 }; + char buffer[1024] = { 0 }; + + if (argc > 1 && strcmp(argv[1], "inet6") == 0) { + af = AF_INET6; + addrlen = sizeof(struct sockaddr_in6); + init_sockaddr_inet6((struct sockaddr_in6 *)&addr); + } + else { + af = AF_INET; + addrlen = sizeof(struct sockaddr_in); + init_sockaddr_inet((struct sockaddr_in *)&addr); + } + + printf("[Server] Create socket\n"); + socket_fd = socket(af, SOCK_DGRAM, 0); + if (socket_fd < 0) { + perror("Create socket failed"); + goto fail; + } + + printf("[Server] Bind socket\n"); + if (bind(socket_fd, (struct sockaddr *)&addr, addrlen) < 0) { + perror("Bind failed"); + goto fail; + } + + printf("[Server] Wait for clients to connect ..\n"); + while (connections < MAX_CONNECTIONS_COUNT) { + addrlen = sizeof(addr); + /* make sure there is space for the string terminator */ + int ret = recvfrom(socket_fd, buffer, sizeof(buffer) - 1, 0, + (struct sockaddr *)&addr, &addrlen); + if (ret < 0) { + perror("Read failed"); + goto fail; + } + buffer[ret] = '\0'; + + if (sockaddr_to_string((struct sockaddr *)&addr, ip_string, + sizeof(ip_string) / sizeof(ip_string[0])) + != 0) { + printf("[Server] failed to parse client address\n"); + goto fail; + } + + printf("[Server] received %d bytes from %s: %s\n", ret, ip_string, + buffer); + + if (sendto(socket_fd, reply_message, strlen(reply_message), 0, + (struct sockaddr *)&addr, addrlen) + < 0) { + perror("Send failed"); + break; + } + + connections++; + } + + if (connections == MAX_CONNECTIONS_COUNT) { + printf("[Server] Achieve maximum amount of connections\n"); + } + + printf("[Server] Shuting down ..\n"); + shutdown(socket_fd, SHUT_RDWR); + close(socket_fd); + sleep(3); + printf("[Server] BYE \n"); + return EXIT_SUCCESS; + +fail: + printf("[Server] Shuting down ..\n"); + if (socket_fd >= 0) + close(socket_fd); + sleep(3); + return EXIT_FAILURE; +} diff --git a/wasm-micro-runtime/samples/spawn-thread/CMakeLists.txt b/wasm-micro-runtime/samples/spawn-thread/CMakeLists.txt new file mode 100644 index 0000000..29c4dc4 --- /dev/null +++ b/wasm-micro-runtime/samples/spawn-thread/CMakeLists.txt @@ -0,0 +1,79 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +include(CheckPIESupported) + +project(spawn_thread) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Resetdefault linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_JIT 0) +set(WAMR_BUILD_FAST_INTERP 1) +set(WAMR_BUILD_THREAD_MGR 1) +set(WAMR_BUILD_SHARED_MEMORY 1) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +################################################ + + +################ wasm application ################ +add_subdirectory(wasm-apps) + +################ wamr runtime ################ +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +set (RUNTIME_SOURCE_ALL + ${CMAKE_CURRENT_LIST_DIR}/src/main.c + ${UNCOMMON_SHARED_SOURCE} +) +add_executable (spawn_thread ${RUNTIME_SOURCE_ALL}) +check_pie_supported() +set_target_properties (spawn_thread PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(spawn_thread vmlib -lpthread -lm) diff --git a/wasm-micro-runtime/samples/spawn-thread/src/main.c b/wasm-micro-runtime/samples/spawn-thread/src/main.c new file mode 100644 index 0000000..ecdf679 --- /dev/null +++ b/wasm-micro-runtime/samples/spawn-thread/src/main.c @@ -0,0 +1,242 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" +#include "pthread.h" + +#define THREAD_NUM 10 + +typedef struct ThreadArgs { + wasm_exec_env_t exec_env; + int start; + int length; +} ThreadArgs; + +void * +thread(void *arg) +{ + ThreadArgs *thread_arg = (ThreadArgs *)arg; + wasm_exec_env_t exec_env = thread_arg->exec_env; + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasm_function_inst_t func; + uint32 argv[2]; + + if (!wasm_runtime_init_thread_env()) { + printf("failed to initialize thread environment"); + return NULL; + } + + func = wasm_runtime_lookup_function(module_inst, "sum"); + if (!func) { + printf("failed to lookup function sum"); + wasm_runtime_destroy_thread_env(); + return NULL; + } + argv[0] = thread_arg->start; + argv[1] = thread_arg->length; + + /* call the WASM function */ + if (!wasm_runtime_call_wasm(exec_env, func, 2, argv)) { + printf("%s\n", wasm_runtime_get_exception(module_inst)); + wasm_runtime_destroy_thread_env(); + return NULL; + } + + wasm_runtime_destroy_thread_env(); + return (void *)(uintptr_t)argv[0]; +} + +void * +wamr_thread_cb(wasm_exec_env_t exec_env, void *arg) +{ + ThreadArgs *thread_arg = (ThreadArgs *)arg; + wasm_module_inst_t module_inst = get_module_inst(exec_env); + wasm_function_inst_t func; + uint32 argv[2]; + + func = wasm_runtime_lookup_function(module_inst, "sum"); + if (!func) { + printf("failed to lookup function sum"); + return NULL; + } + argv[0] = thread_arg->start; + argv[1] = thread_arg->length; + + /* call the WASM function */ + if (!wasm_runtime_call_wasm(exec_env, func, 2, argv)) { + printf("%s\n", wasm_runtime_get_exception(module_inst)); + return NULL; + } + + return (void *)(uintptr_t)argv[0]; +} + +int +main(int argc, char *argv[]) +{ + char *wasm_file = "wasm-apps/test.wasm"; + uint8 *wasm_file_buf = NULL; + uint32 wasm_file_size, wasm_argv[2], i, threads_created; + uint32 stack_size = 16 * 1024, heap_size = 16 * 1024; + wasm_module_t wasm_module = NULL; + wasm_module_inst_t wasm_module_inst = NULL; + wasm_exec_env_t exec_env = NULL; + RuntimeInitArgs init_args; + ThreadArgs thread_arg[THREAD_NUM]; + pthread_t tid[THREAD_NUM]; + wasm_thread_t wasm_tid[THREAD_NUM]; + uint32 result[THREAD_NUM], sum; + wasm_function_inst_t func; + char error_buf[128] = { 0 }; + + memset(thread_arg, 0, sizeof(ThreadArgs) * THREAD_NUM); + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + init_args.mem_alloc_type = Alloc_With_Allocator; + init_args.mem_alloc_option.allocator.malloc_func = malloc; + init_args.mem_alloc_option.allocator.realloc_func = realloc; + init_args.mem_alloc_option.allocator.free_func = free; + init_args.max_thread_num = THREAD_NUM; + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + /* load WASM byte buffer from WASM bin file */ + if (!(wasm_file_buf = + (uint8 *)bh_read_file_to_buffer(wasm_file, &wasm_file_size))) + goto fail1; + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail2; + } + + /* instantiate the module */ + if (!(wasm_module_inst = + wasm_runtime_instantiate(wasm_module, stack_size, heap_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail3; + } + + /* Create the first exec_env */ + if (!(exec_env = + wasm_runtime_create_exec_env(wasm_module_inst, stack_size))) { + printf("failed to create exec_env\n"); + goto fail4; + } + + func = wasm_runtime_lookup_function(wasm_module_inst, "sum"); + if (!func) { + printf("failed to lookup function sum"); + goto fail5; + } + wasm_argv[0] = 0; + wasm_argv[1] = THREAD_NUM * 10; + + /* + * Execute the wasm function in current thread, get the expect result + */ + if (!wasm_runtime_call_wasm(exec_env, func, 2, wasm_argv)) { + printf("%s\n", wasm_runtime_get_exception(wasm_module_inst)); + } + printf("expect result: %d\n", wasm_argv[0]); + + /* + * Run wasm function in multiple thread created by pthread_create + */ + memset(thread_arg, 0, sizeof(ThreadArgs) * THREAD_NUM); + for (i = 0; i < THREAD_NUM; i++) { + wasm_exec_env_t new_exec_env; + thread_arg[i].start = 10 * i; + thread_arg[i].length = 10; + + /* spawn a new exec_env to be executed in other threads */ + new_exec_env = wasm_runtime_spawn_exec_env(exec_env); + if (new_exec_env) + thread_arg[i].exec_env = new_exec_env; + else { + printf("failed to spawn exec_env\n"); + break; + } + + /* If we use: + thread_arg[i].exec_env = exec_env, + we may get wrong result */ + + if (0 != pthread_create(&tid[i], NULL, thread, &thread_arg[i])) { + printf("failed to create thread.\n"); + wasm_runtime_destroy_spawned_exec_env(new_exec_env); + break; + } + } + + threads_created = i; + + sum = 0; + memset(result, 0, sizeof(uint32) * THREAD_NUM); + for (i = 0; i < threads_created; i++) { + pthread_join(tid[i], (void **)&result[i]); + sum += result[i]; + /* destroy the spawned exec_env */ + if (thread_arg[i].exec_env) + wasm_runtime_destroy_spawned_exec_env(thread_arg[i].exec_env); + } + + printf("[pthread]sum result: %d\n", sum); + + /* + * Run wasm function in multiple thread created by wamr spawn API + */ + memset(thread_arg, 0, sizeof(ThreadArgs) * THREAD_NUM); + for (i = 0; i < THREAD_NUM; i++) { + thread_arg[i].start = 10 * i; + thread_arg[i].length = 10; + + /* No need to spawn exec_env manually */ + if (0 + != wasm_runtime_spawn_thread(exec_env, &wasm_tid[i], wamr_thread_cb, + &thread_arg[i])) { + printf("failed to spawn thread.\n"); + break; + } + } + + threads_created = i; + + sum = 0; + memset(result, 0, sizeof(uint32) * THREAD_NUM); + for (i = 0; i < threads_created; i++) { + wasm_runtime_join_thread(wasm_tid[i], (void **)&result[i]); + sum += result[i]; + /* No need to destroy the spawned exec_env */ + } + printf("[spwan_thread]sum result: %d\n", sum); + +fail5: + wasm_runtime_destroy_exec_env(exec_env); + +fail4: + /* destroy the module instance */ + wasm_runtime_deinstantiate(wasm_module_inst); + +fail3: + /* unload the module */ + wasm_runtime_unload(wasm_module); + +fail2: + /* free the file buffer */ + wasm_runtime_free(wasm_file_buf); + +fail1: + /* destroy runtime environment */ + wasm_runtime_destroy(); + return 0; +} diff --git a/wasm-micro-runtime/samples/spawn-thread/wasm-apps/CMakeLists.txt b/wasm-micro-runtime/samples/spawn-thread/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000..0996d58 --- /dev/null +++ b/wasm-micro-runtime/samples/spawn-thread/wasm-apps/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 2.8) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () +set(CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=32768") +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set (DEFINED_SYMBOLS +"${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set (CMAKE_EXE_LINKER_FLAGS + "-Wl,--shared-memory,--max-memory=131072, \ + -Wl,--no-entry,--strip-all,--export=sum, \ + -Wl,--export=return_bss, \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors \ + -Wl,--allow-undefined-file=${DEFINED_SYMBOLS}" +) + +add_executable(test.wasm sum.c) +target_link_libraries(test.wasm) diff --git a/wasm-micro-runtime/samples/spawn-thread/wasm-apps/sum.c b/wasm-micro-runtime/samples/spawn-thread/wasm-apps/sum.c new file mode 100644 index 0000000..6f8b8f2 --- /dev/null +++ b/wasm-micro-runtime/samples/spawn-thread/wasm-apps/sum.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* + * have something in bss so that llvm synthesizes + * wasm start function for this module. + */ +char * +return_bss() +{ + static char bss[4096]; + return bss; +} + +int +sum(int start, int length) +{ + int sum = 0, i; + + for (i = start; i < start + length; i++) { + sum += i; + } + + return sum; +} diff --git a/wasm-micro-runtime/samples/terminate/.gitignore b/wasm-micro-runtime/samples/terminate/.gitignore new file mode 100644 index 0000000..0fa8a76 --- /dev/null +++ b/wasm-micro-runtime/samples/terminate/.gitignore @@ -0,0 +1 @@ +/out/ \ No newline at end of file diff --git a/wasm-micro-runtime/samples/terminate/CMakeLists.txt b/wasm-micro-runtime/samples/terminate/CMakeLists.txt new file mode 100644 index 0000000..246b835 --- /dev/null +++ b/wasm-micro-runtime/samples/terminate/CMakeLists.txt @@ -0,0 +1,98 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +project (terminate) + +set (CMAKE_CXX_STANDARD 17) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_LIBC_WASI 1) +set (WAMR_BUILD_LIB_WASI_THREADS 1) +set (WAMR_BUILD_THREAD_MGR 1) +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) + +# fast interpreter +# set (WAMR_BUILD_FAST_INTERP 1) + +# fast-jit +# set (WAMR_BUILD_FAST_JIT 1) + +# llvm jit +# set (WAMR_BUILD_JIT 1) +# set (LLVM_DIR /usr/local/opt/llvm@14/lib/cmake/llvm) + +set (WAMR_BUILD_REF_TYPES 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (terminate src/main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (terminate PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (terminate vmlib -lm -ldl -lpthread ${LLVM_AVAILABLE_LIBS}) +else () + target_link_libraries (terminate vmlib -lm -ldl -lpthread -lrt ${LLVM_AVAILABLE_LIBS}) +endif () diff --git a/wasm-micro-runtime/samples/terminate/README.md b/wasm-micro-runtime/samples/terminate/README.md new file mode 100644 index 0000000..89a9c16 --- /dev/null +++ b/wasm-micro-runtime/samples/terminate/README.md @@ -0,0 +1,4 @@ +The "terminate" sample project +============================== + +This sample demonstrates wasm_runtime_terminate API. diff --git a/wasm-micro-runtime/samples/terminate/build.sh b/wasm-micro-runtime/samples/terminate/build.sh new file mode 100755 index 0000000..4c882e9 --- /dev/null +++ b/wasm-micro-runtime/samples/terminate/build.sh @@ -0,0 +1,63 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +CURR_DIR=$PWD +WAMR_DIR=${PWD}/../.. +OUT_DIR=${PWD}/out + +WASM_APPS=${PWD}/wasm-apps + + +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +mkdir ${OUT_DIR}/wasm-apps + + +echo "##################### build terminate project" +cd ${CURR_DIR} +mkdir -p cmake_build +cd cmake_build +cmake .. -DCMAKE_BUILD_TYPE=Debug +make -j ${nproc} +if [ $? != 0 ];then + echo "BUILD_FAIL terminate exit as $?\n" + exit 2 +fi + +cp -a terminate ${OUT_DIR} + +printf "\n" + +echo "##################### build wasm apps" + +cd ${WASM_APPS} + +for i in `ls *.wat` +do +APP_SRC="$i" +OUT_FILE=${i%.*}.wasm + +# Note: the CI installs wabt in /opt/wabt +if type wat2wasm; then + WAT2WASM=${WAT2WASM:-wat2wasm} +elif [ -x /opt/wabt/bin/wat2wasm ]; then + WAT2WASM=${WAT2WASM:-/opt/wabt/bin/wat2wasm} +fi + +${WAT2WASM} -o ${OUT_DIR}/wasm-apps/${OUT_FILE} ${APP_SRC} + +# aot +# wamrc -o ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot ${OUT_DIR}/wasm-apps/${OUT_FILE} +# mv ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot ${OUT_DIR}/wasm-apps/${OUT_FILE} + +if [ -f ${OUT_DIR}/wasm-apps/${OUT_FILE} ]; then + echo "build ${OUT_FILE} success" +else + echo "build ${OUT_FILE} fail" +fi +done +echo "##################### build wasm apps done" diff --git a/wasm-micro-runtime/samples/terminate/run.sh b/wasm-micro-runtime/samples/terminate/run.sh new file mode 100755 index 0000000..0a75fea --- /dev/null +++ b/wasm-micro-runtime/samples/terminate/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +out/terminate -f out/wasm-apps/testapp.wasm diff --git a/wasm-micro-runtime/samples/terminate/src/main.c b/wasm-micro-runtime/samples/terminate/src/main.c new file mode 100644 index 0000000..fb0842a --- /dev/null +++ b/wasm-micro-runtime/samples/terminate/src/main.c @@ -0,0 +1,219 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +#include "wasm_export.h" +#include "bh_read_file.h" +#include "bh_getopt.h" + +void +print_usage(void) +{ + fprintf(stdout, "Options:\r\n"); + fprintf(stdout, " -f [path of wasm file] \n"); +} + +static void * +runner_with_sigleton_exec_env(void *vp) +{ + wasm_module_inst_t inst = vp; + bool ok = wasm_runtime_init_thread_env(); + assert(ok); + wasm_application_execute_main(inst, 0, NULL); + wasm_runtime_destroy_thread_env(); + return inst; +} + +static void * +runner_with_spawn_exec_env(void *vp) +{ + wasm_exec_env_t env = vp; + wasm_module_inst_t inst = wasm_runtime_get_module_inst(env); + wasm_function_inst_t func; + bool ok = wasm_runtime_init_thread_env(); + assert(ok); + func = wasm_runtime_lookup_function(inst, "block_forever"); + assert(func != NULL); + wasm_runtime_call_wasm(env, func, 0, NULL); + wasm_runtime_destroy_spawned_exec_env(env); + wasm_runtime_destroy_thread_env(); + return inst; +} + +int +main(int argc, char *argv_main[]) +{ + int exit_code = 1; + static char global_heap_buf[512 * 1024]; + char *buffer; + char error_buf[128]; + int opt; + char *wasm_path = NULL; + int ret; + int pipe_fds[2]; + + const unsigned int N = 4; + wasm_module_t module = NULL; + wasm_module_inst_t module_inst[N]; + pthread_t th[N]; + unsigned int i; + uint32 buf_size, stack_size = 8092, heap_size = 8092; + + for (i = 0; i < N; i++) { + module_inst[i] = NULL; + } + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + while ((opt = getopt(argc, argv_main, "hf:")) != -1) { + switch (opt) { + case 'f': + wasm_path = optarg; + break; + case 'h': + print_usage(); + return 0; + case '?': + print_usage(); + return 0; + } + } + if (optind == 1) { + print_usage(); + return 0; + } + + memset(&init_args, 0, sizeof(init_args)); + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + buffer = bh_read_file_to_buffer(wasm_path, &buf_size); + + if (!buffer) { + printf("Open wasm app file [%s] failed.\n", wasm_path); + goto fail; + } + + module = wasm_runtime_load((uint8 *)buffer, buf_size, error_buf, + sizeof(error_buf)); + if (!module) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* Ensure that fd_read on FD 0 blocks. */ + ret = pipe(pipe_fds); + if (ret != 0) { + goto fail; + } + wasm_runtime_set_wasi_args_ex(module, NULL, 0, NULL, 0, NULL, 0, NULL, 0, + pipe_fds[0], -1, -1); + + for (i = 0; i < N; i++) { + bool use_wasm_runtime_spawn_exec_env = i / 2 == 0; + wasm_exec_env_t env; + + module_inst[i] = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + + if (!module_inst[i]) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* Note: ensure that module inst has an exec env so that + * it can receive the termination request. + */ + env = wasm_runtime_get_exec_env_singleton(module_inst[i]); + assert(env != NULL); + if (use_wasm_runtime_spawn_exec_env) { + env = wasm_runtime_spawn_exec_env(env); + assert(env != NULL); + } + + if ((i % 2) == 0) { + printf("terminating thread %u before starting\n", i); + wasm_runtime_terminate(module_inst[i]); + } + + if (use_wasm_runtime_spawn_exec_env) { + printf("starting thread %u (spawn_exec_env)\n", i); + ret = pthread_create(&th[i], NULL, runner_with_spawn_exec_env, env); + if (ret != 0) { + wasm_runtime_destroy_spawned_exec_env(env); + goto fail; + } + } + else { + printf("starting thread %u (singleton exec_env)\n", i); + ret = pthread_create(&th[i], NULL, runner_with_sigleton_exec_env, + module_inst[i]); + if (ret != 0) { + goto fail; + } + } + } + + printf("sleeping a bit to ensure that the threads actually started\n"); + sleep(1); + + for (i = 0; i < N; i++) { + if ((i % 2) != 0) { + printf("terminating thread %u\n", i); + wasm_runtime_terminate(module_inst[i]); + } + } + + for (i = 0; i < N; i++) { + printf("joining thread %u\n", i); + void *status; + ret = pthread_join(th[i], &status); + if (ret != 0) { + printf("pthread_join failed for thread %u\n", i); + goto fail; + } + } + + for (i = 0; i < N; i++) { + const char *exception = wasm_runtime_get_exception(module_inst[i]); + if (exception != NULL) { + if (!strstr(exception, "terminated by user")) { + printf("thread %u got an exception: %s (unexpected)\n", i, + exception); + goto fail; + } + printf("thread %u got an exception: %s (expected)\n", i, exception); + } + else { + printf("thread %u got no exception (unexpected)\n", i); + goto fail; + } + } + + exit_code = 0; +fail: + for (i = 0; i < N; i++) { + if (module_inst[i]) + wasm_runtime_deinstantiate(module_inst[i]); + } + if (module) + wasm_runtime_unload(module); + if (buffer) + BH_FREE(buffer); + wasm_runtime_destroy(); + return exit_code; +} diff --git a/wasm-micro-runtime/samples/terminate/wasm-apps/testapp.wat b/wasm-micro-runtime/samples/terminate/wasm-apps/testapp.wat new file mode 100644 index 0000000..349535d --- /dev/null +++ b/wasm-micro-runtime/samples/terminate/wasm-apps/testapp.wat @@ -0,0 +1,53 @@ +;; Copyright (C) 2024 YAMAMOTO Takashi +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(module + (func $fd_read (import "wasi_snapshot_preview1" "fd_read") (param i32 i32 i32 i32) (result i32)) + (func $block_forever (export "block_forever") + ;; read from FD 0 + i32.const 100 ;; iov_base + i32.const 200 ;; buffer + i32.store + i32.const 104 ;; iov_len + i32.const 1 + i32.store + i32.const 0 ;; fd 0 + i32.const 100 ;; iov_base + i32.const 1 ;; iov count + i32.const 300 ;; retp (out) + call $fd_read + unreachable + ) + (func (export "_start") + call $block_forever + ) + + ;; a dumb malloc/free implementation + (func (export "malloc") (param i32) (result i32) + local.get 0 + i32.const 65535 + i32.add + i32.const 65536 + i32.div_u + memory.grow + local.set 0 + local.get 0 + i32.const -1 + i32.eq + if + i32.const 0 + return + end + local.get 0 + i32.const 65536 + i32.mul + ) + (func (export "free") (param i32)) + + (memory (export "memory") 1) + + ;; fake globals to make wasm_set_aux_stack happy + (global (export "__heap_base") i32 (i32.const 0x10000)) + (global (export "__data_end") i32 (i32.const 0x10000)) + (global (mut i32) (i32.const 0x10000)) +) diff --git a/wasm-micro-runtime/samples/wasi-threads/CMakeLists.txt b/wasm-micro-runtime/samples/wasi-threads/CMakeLists.txt new file mode 100644 index 0000000..467f5fd --- /dev/null +++ b/wasm-micro-runtime/samples/wasi-threads/CMakeLists.txt @@ -0,0 +1,80 @@ +# Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +include(CheckPIESupported) + +project(wasi_threads_sample) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Resetdefault linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_JIT 0) +set(WAMR_BUILD_LIBC_BUILTIN 1) +set(WAMR_BUILD_FAST_INTERP 1) +set(WAMR_BUILD_LIBC_WASI 1) +set(WAMR_BUILD_LIB_WASI_THREADS 1) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +################################################ + + +################ wasm application ################ +add_subdirectory(wasm-apps) + +################ wamr runtime ################ +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +set (RUNTIME_SOURCE_ALL + ${CMAKE_CURRENT_LIST_DIR}/../../product-mini/platforms/linux/main.c + ${UNCOMMON_SHARED_SOURCE} +) +add_executable (iwasm ${RUNTIME_SOURCE_ALL}) +check_pie_supported() +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(iwasm vmlib -lpthread -lm -ldl) diff --git a/wasm-micro-runtime/samples/wasi-threads/README.md b/wasm-micro-runtime/samples/wasi-threads/README.md new file mode 100644 index 0000000..a79a3cd --- /dev/null +++ b/wasm-micro-runtime/samples/wasi-threads/README.md @@ -0,0 +1,22 @@ +# "WASI threads" sample introduction + +To run the sample, `wasi-sdk` >= 20 is required. + +## Build and run the samples + +```shell +$ mkdir build +$ cd build +$ cmake .. +$ make +... +$ ./iwasm wasm-apps/no_pthread.wasm +``` + +## Run samples in AOT mode +```shell +$ ../../../wamr-compiler/build/wamrc \ + --enable-multi-thread \ + -o wasm-apps/no_pthread.aot wasm-apps/no_pthread.wasm +$ ./iwasm wasm-apps/no_pthread.aot +``` diff --git a/wasm-micro-runtime/samples/wasi-threads/wasm-apps/CMakeLists.txt b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000..87f21e9 --- /dev/null +++ b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +if (DEFINED WASI_SYSROOT) + set (CMAKE_SYSROOT "${WASI_SYSROOT}") +endif () + +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") +set (CMAKE_ASM_COMPILER "${WASI_SDK_DIR}/bin/clang") +set (CMAKE_EXE_LINKER_FLAGS "-target wasm32-wasi-threads") + +if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) + set (CMAKE_C_FLAGS "") + set (CMAKE_CXX_FLAGS "") +endif () + +function (compile_sample SOURCE_FILE) + get_filename_component (FILE_NAME ${SOURCE_FILE} NAME_WLE) + set (WASM_MODULE ${FILE_NAME}.wasm) + add_executable (${WASM_MODULE} ${SOURCE_FILE} ${ARGN}) + + target_compile_options (${WASM_MODULE} PRIVATE + -pthread -ftls-model=local-exec) + + target_link_options (${WASM_MODULE} PRIVATE + -z stack-size=32768 + LINKER:--export=__heap_base + LINKER:--export=__data_end + LINKER:--shared-memory,--max-memory=1966080 + LINKER:--export=wasi_thread_start + LINKER:--export=malloc + LINKER:--export=free + ) +endfunction () + +compile_sample(no_pthread.c wasi_thread_start.S) \ No newline at end of file diff --git a/wasm-micro-runtime/samples/wasi-threads/wasm-apps/no_pthread.c b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/no_pthread.c new file mode 100644 index 0000000..16661bd --- /dev/null +++ b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/no_pthread.c @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include + +#include "wasi_thread_start.h" + +static const int64_t SECOND = 1000 * 1000 * 1000; + +typedef struct { + start_args_t base; + int th_ready; + int value; + int thread_id; +} shared_t; + +void +__wasi_thread_start_C(int thread_id, int *start_arg) +{ + shared_t *data = (shared_t *)start_arg; + + printf("New thread ID: %d, starting parameter: %d\n", thread_id, + data->value); + + data->thread_id = thread_id; + data->value += 8; + printf("Updated value: %d\n", data->value); + + __atomic_store_n(&data->th_ready, 1, __ATOMIC_SEQ_CST); + __builtin_wasm_memory_atomic_notify(&data->th_ready, 1); +} + +int +main(int argc, char **argv) +{ + shared_t data = { { NULL }, 0, 52, -1 }; + int thread_id; + int ret = EXIT_SUCCESS; + + if (!start_args_init(&data.base)) { + printf("Stack allocation for thread failed\n"); + return EXIT_FAILURE; + } + + thread_id = __wasi_thread_spawn(&data); + ASSERT_VALID_TID(thread_id); + + if (__builtin_wasm_memory_atomic_wait32(&data.th_ready, 0, SECOND) == 2) { + printf("Timeout\n"); + return EXIT_FAILURE; + } + + printf("Thread completed, new value: %d, thread id: %d\n", data.value, + data.thread_id); + + assert(thread_id == data.thread_id); + + start_args_deinit(&data.base); + + return ret; +} diff --git a/wasm-micro-runtime/samples/wasi-threads/wasm-apps/wasi_thread_start.S b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/wasi_thread_start.S new file mode 100644 index 0000000..ea8fd14 --- /dev/null +++ b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/wasi_thread_start.S @@ -0,0 +1,22 @@ +# A slightly modified copy of the wasi-libc implementation +# https://github.com/WebAssembly/wasi-libc/pull/376/ + .globaltype __stack_pointer, i32 + .functype __wasi_thread_start_C (i32, i32) -> () + + .globl wasi_thread_start + +wasi_thread_start: + .functype wasi_thread_start (i32, i32) -> () + + # Set up the minimum C environment. + # Note: offsetof(start_arg, stack) == 0 + local.get 1 # start_arg + i32.load 0 # stack + global.set __stack_pointer + + # Make the C function do the rest of work. + local.get 0 # tid + local.get 1 # start_arg + call __wasi_thread_start_C + + end_function \ No newline at end of file diff --git a/wasm-micro-runtime/samples/wasi-threads/wasm-apps/wasi_thread_start.h b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/wasi_thread_start.h new file mode 100644 index 0000000..5eae293 --- /dev/null +++ b/wasm-micro-runtime/samples/wasi-threads/wasm-apps/wasi_thread_start.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#ifndef WASI_THREAD_START_H +#define WASI_THREAD_START_H + +#define STACK_SIZE 32 * 1024 // same as the main stack + +/* See https://github.com/WebAssembly/wasi-threads#design-choice-thread-ids */ +#define ASSERT_VALID_TID(TID) \ + (void)TID; \ + assert(TID >= 1 && TID <= 0x1FFFFFFF && "Invalid thread ID") + +typedef struct { + void *stack; +} start_args_t; + +static inline int +start_args_init(start_args_t *start_args) +{ + start_args->stack = malloc(STACK_SIZE); + if (!start_args->stack) { + return 0; + } + + start_args->stack += STACK_SIZE; + return 1; +} + +static inline void +start_args_deinit(start_args_t *start_args) +{ + free(start_args->stack - STACK_SIZE); +} + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/.gitignore b/wasm-micro-runtime/samples/wasm-c-api-imports/.gitignore new file mode 100644 index 0000000..ab998a6 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api-imports/.gitignore @@ -0,0 +1,2 @@ +/wasm/inc/** +!/wasm/inc/.* diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/CMakeLists.txt b/wasm-micro-runtime/samples/wasm-c-api-imports/CMakeLists.txt new file mode 100644 index 0000000..1325e11 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api-imports/CMakeLists.txt @@ -0,0 +1,174 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(how-to-deal-with-import) + +include(CMakePrintHelpers) +include(CTest) +include(ExternalProject) +include(FetchContent) + +# +# dependencies +# +set(WAMR_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../) +# wasm required headers +execute_process( + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${WARM_ROOT}/${WAMR_ROOT}/wamr-sdk/app/libc-builtin-sysroot/include/pthread.h + ${CMAKE_CURRENT_LIST_DIR}/wasm/inc +) + +# vmlib +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Resetdefault linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +if (NOT DEFINED WAMR_BUILD_AOT) + # Enable AOT by default. + set (WAMR_BUILD_AOT 1) +endif () +if (NOT DEFINED WAMR_BUILD_INTERP) + # Disable Interpreter by default + set (WAMR_BUILD_INTERP 0) +endif () +set(WAMR_BUILD_JIT 0) +set(WAMR_BUILD_FAST_INTERP 1) +set(WAMR_BUILD_LIB_PTHREAD 1) +set(WAMR_BUILD_LIBC_BUILTIN 1) +set(WAMR_BUILD_LIBC_WASI 1) +set(WAMR_BUILD_SIMD 0) + +# compiling and linking flags +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +target_link_libraries(vmlib INTERFACE dl m pthread) +if(WAMR_BUILD_AOT EQUAL 1) + target_compile_definitions(vmlib INTERFACE -DWASM_ENABLE_AOT=1) +else() + target_compile_definitions(vmlib INTERFACE -DWASM_ENABLE_AOT=0) +endif() + +if(WAMR_BUILD_INTERP EQUAL 1) + target_compile_definitions(vmlib INTERFACE -DWASM_ENABLE_INTERP=1) +else() + target_compile_definitions(vmlib INTERFACE -DWASM_ENABLE_INTERP=0) +endif() + +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + # ASAN + UBSAN + target_compile_options(vmlib INTERFACE -fsanitize=address,undefined) + target_link_options(vmlib INTERFACE -fsanitize=address,undefined) +endif() + +# # MSAN +# target_compile_options(vmlib INTERFACE -fsanitize=memory -fno-optimize-sibling-calls -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer) +# target_link_options(vmlib INTERFACE -fsanitize=memory) + +# wamrc +if(WAMR_BUILD_AOT EQUAL 1 AND WAMR_BUILD_INTERP EQUAL 0) + ExternalProject_Add(wamrc + PREFIX wamrc-build + SOURCE_DIR ${WAMR_ROOT}/wamr-compiler + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${WAMR_ROOT}/wamr-compiler -B build + BUILD_COMMAND ${CMAKE_COMMAND} --build build --target wamrc + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different build/wamrc ${CMAKE_CURRENT_BINARY_DIR}/wamrc + ) +endif() + +# +# host +add_subdirectory(host) +add_custom_target( + install_host ALL + COMMAND ${CMAKE_COMMAND} -E copy_if_different ./host/example1 . + DEPENDS example1 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + +# TODO: replace it with a find_package() +set(WASI_SDK_DIR /opt/wasi-sdk-19.0/) +set(WASI_TOOLCHAIN_FILE ${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake) +set(WASI_SYS_ROOT ${WASI_SDK_DIR}/share/wasi-sysroot) + +# +# wasm +if(WAMR_BUILD_AOT EQUAL 1 AND WAMR_BUILD_INTERP EQUAL 0) + ExternalProject_Add(wasm + PREFIX wasm-build + DEPENDS wamrc + BUILD_ALWAYS TRUE + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/wasm + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_LIST_DIR}/wasm -B build + -DWASI_SDK_PREFIX=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_TOOLCHAIN_FILE} + -DCMAKE_SYSROOT=${WASI_SYS_ROOT} + -DWASM_TO_AOT=ON + -DWAMRC_PATH=${CMAKE_CURRENT_BINARY_DIR}/wamrc + -DSOCKET_WASI_CMAKE=${WAMR_ROOT}/core/iwasm/libraries/lib-socket/lib_socket_wasi.cmake + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} + ) +else() + ExternalProject_Add(wasm + PREFIX wasm-build + BUILD_ALWAYS TRUE + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/wasm + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_LIST_DIR}/wasm -B build + -DWASI_SDK_PREFIX=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_TOOLCHAIN_FILE} + -DCMAKE_SYSROOT=${WASI_SYS_ROOT} + -DSOCKET_WASI_CMAKE=${WAMR_ROOT}/core/iwasm/libraries/lib-socket/lib_socket_wasi.cmake + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} + ) +endif() + +# +# Test +# +add_test( + NAME run_example1 + COMMAND ./example1 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/README.md b/wasm-micro-runtime/samples/wasm-c-api-imports/README.md new file mode 100644 index 0000000..9b61a6e --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api-imports/README.md @@ -0,0 +1,174 @@ +# How to create `imports` for wasm_instance_new() properly + +It's always been asked how to create `wasm_extern_vec_t *imports` for +`wasm_instance_new()`? + +```c +WASM_API_EXTERN own wasm_instance_t* wasm_instance_new( + wasm_store_t*, const wasm_module_t*, const wasm_extern_vec_t *imports, + own wasm_trap_t** trap +); +``` + +`wasm_extern_vec_t *imports` is required to match the requirement of _the +import section_ of a .wasm. + +```bash +$ /opt/wabt-1.0.31/bin/wasm-objdump -j Import -x .wasm + +Section Details: + +Import[27]: + - func[0] sig=2 <- env.pthread_mutex_lock + - func[1] sig=2 <- env.pthread_mutex_unlock + - func[2] sig=2 <- env.pthread_cond_signal + - func[3] sig=3 <- env.log + ... + - func[11] sig=4 <__imported_wasi_snapshot_preview1_sock_bind> <- wasi_snapshot_preview1.sock_bind + - func[12] sig=4 <__imported_wasi_snapshot_preview1_sock_connect> <- wasi_snapshot_preview1.sock_connect + - func[13] sig=4 <__imported_wasi_snapshot_preview1_sock_listen> <- wasi_snapshot_preview1.sock_listen + - func[14] sig=5 <__imported_wasi_snapshot_preview1_sock_open> <- wasi_snapshot_preview1.sock_open + - func[15] sig=4 <__imported_wasi_snapshot_preview1_sock_addr_remote> <- wasi_snapshot_preview1.sock_addr_remote + - func[16] sig=4 <__imported_wasi_snapshot_preview1_args_get> <- wasi_snapshot_preview1.args_get + - func[17] sig=4 <__imported_wasi_snapshot_preview1_args_sizes_get> <- wasi_snapshot_preview1.args_sizes_get + ... +``` + +Developers should fill in _imports_ with enough host functions and make sure +there are no linking problems during instantiation. + +```bash +TODO: linking warnings +``` + +## A natural way + +One natural answer is "to create a list which matches every item in _the import +section_" of the .wasm. Since developers can see the section details of +a .wasm by tools like _wasm-objdump_, the answer is doable. Most of the time, +if they also prepare Wasm modules, developers have full control over import +requirements, and they only need to take a look at the order of _the import +section_. + +Yes, _the order_. A proper `wasm_extern_vec_t *imports` includes two things: + +1. how many `wasm_extern_t` +2. and order of those + +Because there is no "name information" in a `wasm_extern_t`. The only way is let +`wasm_instance_new()` to tell which item in _the import section_ of a .wasm +should match any item in `wasm_extern_vec_t *imports` is based on **_index_**. + +The algorithm is quite straightforward. The first one of _the import section_ matches +`wasm_extern_vec_t *imports->data[0] `. The second one matches `wasm_extern_vec_t *imports->data[1]`. +And so on. + +So the order of `wasm_extern_vec_t *imports` becomes quite a burden. It requires +developers always checking _the import section_ visually. + +Until here, the natural way is still workable although involving some handy work. +Right? + +## A blocker + +Sorry, the situation changes a lot when driving wasm32-wasi Wasm modules with +wasm-c-api. + +As you know, WASI provides _a set of crossing-platform standard libraries_ for +Wasm modules, and leaves some _interfaces_ for native platform-dependent supports. +Those _interfaces_ are those import items with the module name `wasi_snapshot_preview1` +in a Wasm module. + +It seems not economical to let developers provide their version of host +implementations of the `wasi_snapshot_preview1.XXX` functions. All those support +should be packed into a common library and shared in different Wasm modules. +Like a [cargo WASI](https://github.com/bytecodealliance/cargo-wasi). + +WAMR chooses to integrate the WASI support library in the runtime to reduce +developers' compilation work. It brings developers a new thing of a proper +`wasm_extern_vec_t *imports` that developers should avoid overwriting those items +of _the import section_ of a Wasm module that will be provided by the runtime. It +also not economical to code for those functions. + +Using module names as a filter seems to be a simple way. But some private +additional c/c++ libraries are supported in WAMR. Those supporting will bring +more import items that don't use `wasi_snapshot_preview1` as module names but are still +covered by the WASM runtime. Like `env.pthread_`. Plus, [the native lib registeration](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/export_native_api.md) +provides another possible way to fill in the requirement of _the import section_. + +Let's take summarize. A proper `wasm_extern_vec_t *imports` should include: + +1. provides all necessary host implementations for items in _the import section_ +2. should not override runtime provided implementation or covered by native + registrations. functinal or econmical. +3. keep them in a right order + +## A recommendation + +The recommendation is: + +- use `wasm_module_imports()` to build the order +- use `wasm_importtype_is_linked()` to avoid overwriting + +[wasm-c-api-imports](.) is a simple showcase of how to do that. + +First, let's take a look at the Wasm module. [send_recv](./wasm/send_recv.c) +uses both standard WASI and WAMR_BUILD_LIB_PTHREAD supporting. Plus a private +native function `host_log`. + +So, `wasm_extern_vec_t *imports` should only include the host implementation of +`host_log` and avoid WASI related(`wasm-c-api-imports.XXX`) and pthread related(`env.pthread_XXX`). + +[Here is how to do](./host/example1.c): + +- get import types with `wasm_module_imports(0)`. it contains name information + +```c + wasm_importtype_vec_t importtypes = { 0 }; + wasm_module_imports(module, &importtypes); +``` + +- traversal import types. The final `wasm_importvec_t *imports` should have the + same order with `wasm_importtype_vec_t` + +```c + for (unsigned i = 0; i < importtypes.num_elems; i++) +``` + +- use `wasm_importtype_is_linked()` to avoid those covered by the runtime and + registered natives. A little tip is use "wasm_extern_new_empty()" to create + a placeholder. + +```c + /* use wasm_extern_new_empty() to create a placeholder */ + if (wasm_importtype_is_linked(importtype)) { + externs[i] = wasm_extern_new_empty( + store, wasm_externtype_kind(wasm_importtype_type(importtype))); + continue; + } +``` + +- use `wasm_importtype_module()` to get the module name, use `wasm_importtype_name()` + to get the field name. + +```c + const wasm_name_t *module_name = + wasm_importtype_module(importtypes.data[i]); + const wasm_name_t *field_name = + wasm_importtype_name(importtypes.data[i]); +``` + +- fill in `wasm_externvec_t *imports` dynamically and programmatically. + +```c + if (strncmp(module_name->data, "env", strlen("env")) == 0 + && strncmp(field_name->data, "log", strlen("log")) == 0) { + wasm_functype_t *log_type = wasm_functype_new_2_0( + wasm_valtype_new_i64(), wasm_valtype_new_i32()); + wasm_func_t *log_func = wasm_func_new(store, log_type, host_logs); + wasm_functype_delete(log_type); + + externs[i] = wasm_func_as_extern(log_func); + } + } +``` diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/host/CMakeLists.txt b/wasm-micro-runtime/samples/wasm-c-api-imports/host/CMakeLists.txt new file mode 100644 index 0000000..e2636f0 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api-imports/host/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(host) + +set(CMAKE_BUILD_TYPE Debug) + +# +# host +add_executable(example1 ./example1.c) +target_link_libraries(example1 vmlib) diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/host/example1.c b/wasm-micro-runtime/samples/wasm-c-api-imports/host/example1.c new file mode 100644 index 0000000..ccf574d --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api-imports/host/example1.c @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include "wasm_c_api.h" +#include "wasm_export.h" + +static wasm_trap_t * +host_logs(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + return NULL; +} + +static bool +build_imports(wasm_store_t *store, const wasm_module_t *module, + wasm_extern_vec_t *out) +{ + wasm_importtype_vec_t importtypes = { 0 }; + wasm_module_imports(module, &importtypes); + + wasm_extern_t *externs[32] = { 0 }; + + for (unsigned i = 0; i < importtypes.num_elems; i++) { + wasm_importtype_t *importtype = importtypes.data[i]; + + /* use wasm_extern_new_empty() to create a placeholder */ + if (wasm_importtype_is_linked(importtype)) { + externs[i] = wasm_extern_new_empty( + store, wasm_externtype_kind(wasm_importtype_type(importtype))); + continue; + } + + const wasm_name_t *module_name = + wasm_importtype_module(importtypes.data[i]); + const wasm_name_t *field_name = + wasm_importtype_name(importtypes.data[i]); + + if (strncmp(module_name->data, "env", strlen("env")) == 0 + && strncmp(field_name->data, "log", strlen("log")) == 0) { + wasm_functype_t *log_type = wasm_functype_new_2_0( + wasm_valtype_new_i64(), wasm_valtype_new_i32()); + wasm_func_t *log_func = wasm_func_new(store, log_type, host_logs); + wasm_functype_delete(log_type); + + externs[i] = wasm_func_as_extern(log_func); + } + } + + wasm_extern_vec_new(out, importtypes.num_elems, externs); + wasm_importtype_vec_delete(&importtypes); + return true; +} + +int +main() +{ + int main_ret = EXIT_FAILURE; + + // Initialize. + printf("Initializing...\n"); + wasm_engine_t *engine = wasm_engine_new(); + if (!engine) + goto quit; + + wasm_store_t *store = wasm_store_new(engine); + if (!store) + goto delete_engine; + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE *file = fopen("send_recv.aot", "rb"); + printf("> Load .aot\n"); +#else + FILE *file = fopen("send_recv.wasm", "rb"); + printf("> Load .wasm\n"); +#endif + if (!file) { + printf("> Error loading module!\n"); + goto delete_store; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + goto close_file; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + goto close_file; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + goto close_file; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + goto delete_binary; + } + + // Compile. + printf("Compiling module...\n"); + wasm_module_t *module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + goto delete_binary; + } + + // Set Wasi Context + const char *addr_pool[1] = { "127.0.0.1" }; + wasm_runtime_set_wasi_addr_pool(*module, addr_pool, 1); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_vec_t imports = { 0 }; + ret = build_imports(store, module, &imports); + if (!ret) { + printf("> Error building imports!\n"); + goto delete_module; + } + + wasm_instance_t *instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + goto delete_imports; + } + + // Extract export. + printf("Extracting export...\n"); + wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + goto delete_instance; + } + + /** + * should use information from wasm_module_exports to avoid hard coding "1" + */ + const wasm_func_t *start_func = wasm_extern_as_func(exports.data[1]); + if (start_func == NULL) { + printf("> Error accessing export!\n"); + goto delete_exports; + } + + // Call. "_start(nil) -> i32" + printf("Calling _start ...\n"); + wasm_val_t rs[1] = { WASM_I32_VAL(0) }; + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_ARRAY_VEC(rs); + wasm_trap_t *trap = wasm_func_call(start_func, &args, &results); + if (trap) { + wasm_name_t message = { 0 }; + wasm_trap_message(trap, &message); + + printf("> Error calling function! %s\n", message.data); + + wasm_name_delete(&message); + wasm_trap_delete(trap); + goto delete_exports; + } + + // Print result. + printf("Printing result...\n"); + printf("> %u\n", rs[0].of.i32); + + // Shut down. + printf("Shutting down...\n"); + + // All done. + printf("Done.\n"); + main_ret = EXIT_SUCCESS; + +delete_exports: + wasm_extern_vec_delete(&exports); +delete_instance: + wasm_instance_delete(instance); +delete_imports: + wasm_extern_vec_delete(&imports); +delete_module: + wasm_module_delete(module); +delete_binary: + wasm_byte_vec_delete(&binary); +close_file: + fclose(file); +delete_store: + wasm_store_delete(store); +delete_engine: + wasm_engine_delete(engine); +quit: + return main_ret; +} \ No newline at end of file diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/CMakeLists.txt b/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/CMakeLists.txt new file mode 100644 index 0000000..bf23584 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/CMakeLists.txt @@ -0,0 +1,47 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) +project(wasm_modules) + +if(NOT SOCKET_WASI_CMAKE) + message(FATAL_ERROR "Require SOCKET_WASI_CMAKE") +endif() + +option(WASM_TO_AOT "transfer wasm to aot" OFF) +if(WASM_TO_AOT AND NOT WAMRC_PATH) + message(FATAL_ERROR "Require WAMRC_PATH when WASM_TO_AOT is ON") +endif() + +# +# c -> wasm +include(${SOCKET_WASI_CMAKE}) +add_executable(send_recv ${CMAKE_CURRENT_LIST_DIR}/send_recv.c) +set_target_properties(send_recv PROPERTIES SUFFIX .wasm) +target_include_directories(send_recv PUBLIC ${CMAKE_CURRENT_LIST_DIR}/inc) +target_link_libraries(send_recv socket_wasi_ext) +target_link_options(send_recv PRIVATE + LINKER:--export=__heap_base + LINKER:--export=__data_end + LINKER:--shared-memory,--max-memory=196608 + LINKER:--no-check-features + LINKER:--allow-undefined +) + +if(WASM_TO_AOT) + # wasm -> aot + add_custom_target(send_recv_aot ALL + COMMAND pwd && ${WAMRC_PATH} --invoke-c-api-import --enable-multi-thread -o ./send_recv.aot ./send_recv.wasm + DEPENDS send_recv + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) +endif() + +# +# install +if(WASM_TO_AOT) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/send_recv.aot DESTINATION . ) +else() + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/send_recv.wasm DESTINATION . ) +endif() + diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/inc/.gitkeep b/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/inc/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/send_recv.c b/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/send_recv.c new file mode 100644 index 0000000..d9f9557 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api-imports/wasm/send_recv.c @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __wasi__ +#include +#include "pthread.h" +#else +#include +#endif + +static pthread_mutex_t lock = { 0 }; +static pthread_cond_t cond = { 0 }; +static bool server_create_failed = false; +static bool server_is_ready = false; + +#ifdef __wasi__ +__attribute__((import_name("log"))) extern void +host_log(uint64_t message, uint32_t length); +#endif + +static void +local_printf(const char *formatter, ...) +{ + char buffer[128] = { 0 }; + va_list args; + + va_start(args, formatter); + vsnprintf(buffer, 128, formatter, args); + va_end(args); + +#ifdef __wasi__ + host_log((uint64_t)(void *)buffer, strlen(buffer)); +#endif + printf("--> %s", buffer); +} + +void * +run_as_server(void *arg) +{ + int sock = -1, on = 1; + struct sockaddr_in addr = { 0 }; + int addrlen = 0; + int new_sock = -1; + char *buf[] = { + "The stars shine down", "It brings us light", "Light comes down", + "To make us paths", "It watches us", "And mourns for us", + }; + struct iovec iov[] = { + { .iov_base = buf[0], .iov_len = strlen(buf[0]) + 1 }, + { .iov_base = buf[1], .iov_len = strlen(buf[1]) + 1 }, + { .iov_base = buf[2], .iov_len = strlen(buf[2]) + 1 }, + { .iov_base = buf[3], .iov_len = strlen(buf[3]) + 1 }, + { .iov_base = buf[4], .iov_len = strlen(buf[4]) + 1 }, + { .iov_base = buf[5], .iov_len = strlen(buf[5]) + 1 }, + }; + struct msghdr msg = { .msg_iov = iov, .msg_iovlen = 6 }; + ssize_t send_len = 0; + + pthread_mutex_lock(&lock); + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Create a socket failed"); + return NULL; + } + +#ifndef __wasi__ + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on))) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Setsockopt failed"); + goto fail1; + } +#endif + + /* 0.0.0.0:1234 */ + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + addrlen = sizeof(addr); + if (bind(sock, (struct sockaddr *)&addr, addrlen) < 0) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Bind failed"); + goto fail1; + } + + if (listen(sock, 0) < 0) { + server_create_failed = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + perror("Listen failed"); + goto fail1; + } + + server_is_ready = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + + local_printf("Server is online ... \n"); + + new_sock = accept(sock, (struct sockaddr *)&addr, (socklen_t *)&addrlen); + if (new_sock < 0) { + perror("Accept failed"); + goto fail1; + } + + local_printf("Start sending. \n"); + send_len = sendmsg(new_sock, &msg, 0); + if (send_len < 0) { + perror("Sendmsg failed"); + goto fail2; + } + local_printf("Send %ld bytes successfully!\n", send_len); + +fail2: + close(new_sock); +fail1: + shutdown(sock, SHUT_RD); + close(sock); + return NULL; +} + +void * +run_as_client(void *arg) +{ + int sock = -1; + struct sockaddr_in addr = { 0 }; + /* buf of server is 106 bytes */ + char buf[110] = { 0 }; + struct iovec iov = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct msghdr msg = { .msg_iov = &iov, .msg_iovlen = 1 }; + ssize_t recv_len = 0; + + pthread_mutex_lock(&lock); + while (!server_create_failed && !server_is_ready) { + pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + + if (server_create_failed) { + return NULL; + } + + local_printf("Client is running...\n"); + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + perror("Create a socket failed"); + return NULL; + } + + /* 127.0.0.1:1234 */ + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + perror("Connect failed"); + goto fail; + } + + local_printf("Start receiving. \n"); + recv_len = recvmsg(sock, &msg, 0); + if (recv_len < 0) { + perror("Recvmsg failed"); + goto fail; + } + + local_printf("Receive %ld bytes successlly!\n", recv_len); + assert(recv_len == 106); + + local_printf("Data:\n"); + char *s = msg.msg_iov->iov_base; + while (strlen(s) > 0) { + local_printf(" %s\n", s); + s += strlen(s) + 1; + } + +fail: + shutdown(sock, SHUT_RD); + close(sock); + return NULL; +} + +int +main(int argc, char *argv[]) +{ + pthread_t cs[2] = { 0 }; + uint8_t i = 0; + int ret = EXIT_SUCCESS; + + if (pthread_mutex_init(&lock, NULL)) { + perror("Initialize mutex failed"); + ret = EXIT_FAILURE; + goto RETURN; + } + + if (pthread_cond_init(&cond, NULL)) { + perror("Initialize condition failed"); + ret = EXIT_FAILURE; + goto DESTROY_MUTEX; + } + + if (pthread_create(&cs[0], NULL, run_as_server, NULL)) { + perror("Create a server thread failed"); + ret = EXIT_FAILURE; + goto DESTROY_COND; + } + + if (pthread_create(&cs[1], NULL, run_as_client, NULL)) { + perror("Create a client thread failed"); + ret = EXIT_FAILURE; + goto DESTROY_COND; + } + + for (i = 0; i < 2; i++) { + pthread_join(cs[i], NULL); + } + +DESTROY_COND: + pthread_cond_destroy(&cond); +DESTROY_MUTEX: + pthread_mutex_destroy(&lock); +RETURN: + return ret; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/CMakeLists.txt b/wasm-micro-runtime/samples/wasm-c-api/CMakeLists.txt new file mode 100644 index 0000000..b8783f4 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/CMakeLists.txt @@ -0,0 +1,216 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project(c-api) +else() + project (c-api C ASM) +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +set(CMAKE_CXX_STANDARD 17) +################ runtime settings ################ + +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + if (NOT DEFINED WAMR_BUILD_SIMD) + set (WAMR_BUILD_SIMD 1) + endif () + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + if (NOT DEFINED WAMR_BUILD_SIMD) + set (WAMR_BUILD_SIMD 1) + endif () + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if(NOT DEFINED WAMR_BUILD_INTERP) + set(WAMR_BUILD_INTERP 1) +endif() + +if(NOT DEFINED WAMR_BUILD_AOT) + set(WAMR_BUILD_AOT 0) +endif() + +if(NOT DEFINED WAMR_BUILD_JIT) + set(WAMR_BUILD_JIT 0) +endif() + +set(WAMR_BUILD_LIBC_BUILTIN 1) +set(WAMR_BUILD_LIBC_WASI 0) +set(WAMR_BUILD_MULTI_MODULE 1) +set(WAMR_BUILD_DUMP_CALL_STACK 1) +set(WAMR_BUILD_REF_TYPES 1) + +# If not defined WAMR_BUILD_GC, set it to 0 +if(NOT DEFINED WAMRC_BUILD_WITH_GC) + set(WAMRC_BUILD_WITH_GC 0) +endif() + +if(NOT DEFINED WAMR_BUILD_FAST_INTERP) + set(WAMR_BUILD_FAST_INTERP 1) +endif() + +if (NOT MSVC) + # compiling and linking flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif() +# build out vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +if (MSVC) + target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) +endif() +target_link_libraries (vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) + +if (WAMR_BUILD_WASM_CACHE EQUAL 1) + target_link_libraries(vmlib boringssl_crypto) +endif () +################################################ + +################ application related ################ +## locate wat2wasm +find_program(WAT2WASM + wat2wasm + PATHS /opt/wabt/bin + REQUIRED +) + +if(NOT WAT2WASM) + message(SEND_ERROR "can not find wat2wasm") +else () + execute_process(COMMAND ${WAT2WASM} --version + OUTPUT_VARIABLE WAT2WASM_VERSION_OUTPUT) + string(STRIP ${WAT2WASM_VERSION_OUTPUT} WAT2WASM_VERSION) + message("-- Found wat2wasm ${WAT2WASM_VERSION}") +endif() + +if (${WAT2WASM_VERSION} VERSION_LESS 1.0.26) + set(WAT2WASM_FLAGS "--enable-reference-types") +endif () + +if(${WAMR_BUILD_AOT} EQUAL 1) + ## locate wamrc + find_program(WAMRC + wamrc + PATHS ${WAMR_ROOT_DIR}/wamr-compiler/build/ + ) + + if(NOT WAMRC) + message(SEND_ERROR "can not find wamrc. refer to \ + https://github.com/bytecodealliance/wasm-micro-runtime#build-wamrc-aot-compiler" + ) + endif() +endif() +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +set(MM_UTIL src/utils/multi_module_utils.c) +# build executable for each .c +list(APPEND EXAMPLES callback callback_chain empty_imports global hello hostref memory reflect table trap) +# FIXME enable both in the future +#list(APPEND EXAMPLES clone threads) +# FIXME +# if(WAMR_BUILD_JIT EQUAL 1 AND WAMR_BUILD_LAZY_JIT EQUAL 0) +# list(APPEND EXAMPLES serialize) +# endif() + +check_pie_supported() + +include(CTest) +enable_testing() + +foreach(EX ${EXAMPLES}) + set(SRC ${CMAKE_CURRENT_LIST_DIR}/src/${EX}.c) + + add_executable(${EX} ${SRC} ${UNCOMMON_SHARED_SOURCE} ${MM_UTIL}) + set_target_properties (${EX} PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(${EX} PRIVATE ${UNCOMMON_SHARED_DIR}) + target_link_libraries(${EX} vmlib) + if (MSVC) + target_compile_definitions(${EX} PRIVATE WASM_API_EXTERN=) + endif() + + # wat to wasm + set(WAT ${CMAKE_CURRENT_LIST_DIR}/src/${EX}.wat) + + add_custom_target(${EX}_WASM + COMMAND ${WAT2WASM} ${WAT} ${WAT2WASM_FLAGS} -o ${PROJECT_BINARY_DIR}/${EX}.wasm + DEPENDS ${WAT} + BYPRODUCTS ${PROJECT_BINARY_DIR}/${EX}.wasm + VERBATIM + ) + add_dependencies(${EX} ${EX}_WASM) + + # generate .aot file + if(${WAMR_BUILD_AOT} EQUAL 1) + if(${WAMRC_BUILD_WITH_GC} EQUAL 1) + set(WAMRC_GC_FLAGS "--enable-gc") + else() + set(WAMRC_GC_FLAGS "") + endif() + add_custom_target(${EX}_AOT + COMMAND ${WAMRC} ${WAMRC_GC_FLAGS} -o ${PROJECT_BINARY_DIR}/${EX}.aot + ${PROJECT_BINARY_DIR}/${EX}.wasm + DEPENDS ${EX}_WASM + BYPRODUCTS ${PROJECT_BINARY_DIR}/${EX}.aot + VERBATIM + COMMENT "generate a aot file ${PROJECT_BINARY_DIR}/${EX}.aot" + ) + add_dependencies(${EX} ${EX}_AOT) + endif() + + # run `ctest --test-dir build` + add_test(NAME Test_${EX} + COMMAND ./${EX} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) +endforeach() + +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + find_program(VALGRIND + valgrind + REQUIRED + ) + + # run `ctest -T memcheck -V --test-dir build` +endif() diff --git a/wasm-micro-runtime/samples/wasm-c-api/README.md b/wasm-micro-runtime/samples/wasm-c-api/README.md new file mode 100644 index 0000000..b232767 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/README.md @@ -0,0 +1,50 @@ +WAMR supports *wasm-c-api* in both *interpreter* mode and *aot* mode. + +Before staring, we need to download and intall [WABT](https://github.com/WebAssembly/wabt/releases/latest). + +``` shell +$ cd /opt +$ wget https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz +$ tar -xzf wabt-1.0.31-ubuntu.tar.gz +$ mv wabt-1.0.31 wabt +``` + +By default, all samples are compiled and run in "interpreter" mode. + + +``` shell +$ mkdir build +$ cd build +$ cmake .. +$ make +$ # it will build a library with c-api supporting. +$ # Also copy *.wasm from ../src/ +$ # and generate executable files +$ # now, it is ok to run samples +$ ./hello +$ ... +$ ./global +$ ... +$ ./callback +$ ... +``` + +They can be compiled and run in *aot* mode when some compiling flags are given. + +``` shell +$ mkdir build +$ cd build +$ cmake -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_AOT=1 .. +$ make +$ # it will build a library with c-api supporting. +$ # Also copy *.wasm from ../src/ +$ # and transform *.wasm to *.aot +$ # and generate executable files +$ # now, it is ok to run samples +$ ./hello +$ ... +$ ./global +$ ... +$ ./callback +$ ... +``` diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/LICENSE b/wasm-micro-runtime/samples/wasm-c-api/src/LICENSE new file mode 100644 index 0000000..8f71f43 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/callback.c b/wasm-micro-runtime/samples/wasm-c-api/src/callback.c new file mode 100644 index 0000000..ccb9ec0 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/callback.c @@ -0,0 +1,193 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// Print a Wasm value +void wasm_val_print(wasm_val_t val) { + switch (val.kind) { + case WASM_I32: { + printf("%" PRIu32, val.of.i32); + } break; + case WASM_I64: { + printf("%" PRIu64, val.of.i64); + } break; + case WASM_F32: { + printf("%f", val.of.f32); + } break; + case WASM_F64: { + printf("%g", val.of.f64); + } break; + case WASM_ANYREF: + case WASM_FUNCREF: { + if (val.of.ref == NULL) { + printf("null"); + } else { + printf("ref(%p)", val.of.ref); + } + } break; + } +} + +// A function to be called from Wasm code. +own wasm_trap_t* print_callback( + const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + printf("Calling back...\n> "); + wasm_val_print(args->data[0]); + printf("\n"); + + wasm_val_copy(&results->data[0], &args->data[0]); + return NULL; +} + + +// A function closure. +own wasm_trap_t* closure_callback( + void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + int i = *(int*)env; + printf("Calling back closure...\n"); + printf("> %d\n", i); + + results->data[0].kind = WASM_I32; + results->data[0].of.i32 = (int32_t)i; + return NULL; +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("callback.aot", "rb"); +#else + FILE* file = fopen("callback.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Create external print functions. + printf("Creating callback...\n"); + own wasm_functype_t* print_type = wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32()); + own wasm_func_t* print_func = wasm_func_new(store, print_type, print_callback); + + int i = 42; + own wasm_functype_t* closure_type = wasm_functype_new_0_1(wasm_valtype_new_i32()); + own wasm_func_t* closure_func = wasm_func_new_with_env(store, closure_type, closure_callback, &i, NULL); + + wasm_functype_delete(print_type); + wasm_functype_delete(closure_type); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_t* externs[] = { + wasm_func_as_extern(print_func), wasm_func_as_extern(closure_func) + }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(print_func); + wasm_func_delete(closure_func); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return 1; + } + const wasm_func_t* run_func = wasm_extern_as_func(exports.data[0]); + if (run_func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + wasm_module_delete(module); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_t as[2] = { WASM_I32_VAL(3), WASM_I32_VAL(4) }; + wasm_val_t rs[1] = { WASM_INIT_VAL }; + wasm_val_vec_t args = WASM_ARRAY_VEC(as); + wasm_val_vec_t results = WASM_ARRAY_VEC(rs); + wasm_trap_t *trap = wasm_func_call(run_func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Print result. + printf("Printing result...\n"); + printf("> %u\n", rs[0].of.i32); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/callback.wat b/wasm-micro-runtime/samples/wasm-c-api/src/callback.wat new file mode 100644 index 0000000..d86195f --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/callback.wat @@ -0,0 +1,10 @@ +(module + (func $print (import "" "print") (param i32) (result i32)) + (func $closure (import "" "closure") (result i32)) + (func (export "run") (param $x i32) (param $y i32) (result i32) + (i32.add + (call $print (i32.add (local.get $x) (local.get $y))) + (call $closure) + ) + ) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/callback_chain.c b/wasm-micro-runtime/samples/wasm-c-api/src/callback_chain.c new file mode 100644 index 0000000..e4f5801 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/callback_chain.c @@ -0,0 +1,294 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +static const byte_t * +get_memory_data(uint32_t offset, uint32_t length); + +static bool +call_wasm_function(uint32_t export_id, const wasm_val_vec_t *args, + wasm_val_vec_t *results, const char *name); + +/************************ IMPORTED FUNCTIONS **************************/ + +// (nil) -> i32 +#define FUNCTION_TYPE_NIL_I32 wasm_functype_new_0_1(wasm_valtype_new_i32()) +// (i32, i32) -> nil +#define FUNCTION_TYPE_I32X2_NIL \ + wasm_functype_new_2_0(wasm_valtype_new_i32(), wasm_valtype_new_i32()) + +/* IMPORT FUNCTION LIST */ +#define IMPORT_FUNCTION_LIST(V) \ + V(get_pairs, 0, FUNCTION_TYPE_NIL_I32) \ + V(log, 1, FUNCTION_TYPE_I32X2_NIL) + +/* EXPORT FUNCTION LIST */ +#define EXPORT_FUNCTION_LIST(V) \ + V(on_start) \ + V(on_stop) \ + V(malloc) \ + V(free) + +enum EXPORT_ITEM_NAME { +#define DEFINE_ENUM(name) e_##name, + EXPORT_FUNCTION_LIST(DEFINE_ENUM) +#undef DEFINE_ENUM + e_MEMORY, +}; + +#define DEFINE_FUNCTION(name) \ + wasm_trap_t *STUB_##name(const wasm_val_vec_t *args, \ + wasm_val_vec_t *results) + +#define DEFINE_EMPTY_FUNCTION(name) \ + DEFINE_FUNCTION(name) \ + { \ + printf("[WASM -> NATIVE] calling back %s\n", __FUNCTION__); \ + return NULL; \ + } +#undef DEFINE_EMPTY_FUNCTION + +DEFINE_FUNCTION(get_pairs) +{ + wasm_val_vec_t as = { 0 }; + wasm_val_t data[1] = { WASM_I32_VAL(10) }; + wasm_val_vec_new(&as, 1, data); + if (as.data == NULL) { + printf("ERROR: create parameters failed\n"); + return NULL; + } + + call_wasm_function(e_malloc, &as, results, "malloc"); + + wasm_val_vec_delete(&as); + return NULL; +} + +DEFINE_FUNCTION(log) +{ + wasm_val_t offset = args->data[0]; + wasm_val_t length = args->data[1]; + const byte_t *data = NULL; + + printf("[WASM -> NATIVE] calling back %s\n", __FUNCTION__); + + if (offset.kind != WASM_I32 || length.kind != WASM_I32) { + printf("> Error value type!\n"); + } + + if (!(data = get_memory_data(offset.of.i32, length.of.i32))) { + return NULL; + } + + if (data[length.of.i32 - 1]) { + printf("> Error terminated character\n"); + return NULL; + } + + printf("[WASM_LOG] %s\n", data); + return NULL; +} + +/**********************************************************************/ +// all exportted wasm functions. check with "/opt/wabt/bin/wasm-objdump -x -j +// Export X.wasm" -1: memory 0-32: functions +static own wasm_extern_vec_t exports = { 0 }; + +static const byte_t * +get_memory_data(uint32_t offset, uint32_t length) +{ + wasm_memory_t *memory; + + if (!(memory = wasm_extern_as_memory(exports.data[e_MEMORY]))) { + return NULL; + } + + byte_t *base = wasm_memory_data(memory); + size_t size = wasm_memory_data_size(memory); + if (!base || offset + length > size) { + return NULL; + } + + printf("[NATIVE -> WASM] accessing the memory...\n"); + + return base + offset; +} + +static bool +call_wasm_function(uint32_t export_id, const wasm_val_vec_t *args, + wasm_val_vec_t *results, const char *name) +{ + const wasm_func_t *function; + wasm_trap_t *trap; + + printf("[NATIVE -> WASM] calling func %s...\n", name); + + if (!(function = wasm_extern_as_func(exports.data[export_id]))) { + printf("> Error get export function %u\n", export_id); + return false; + } + + if ((trap = wasm_func_call(function, args, results))) { + own wasm_message_t message = { 0 }; + wasm_trap_message(trap, &message); + + if (message.data) { + printf("> Error calling function %s\n", message.data); + } + else { + printf("> Error calling function"); + } + + wasm_name_delete(&message); + wasm_trap_delete(trap); + return false; + } + return true; +} + +int +main(int argc, const char *argv[]) +{ + // Initialize. + printf("Initializing...\n"); + wasm_engine_t *engine = wasm_engine_new(); + wasm_store_t *store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE *file = fopen("callback_chain.aot", "rb"); +#else + FILE *file = fopen("callback_chain.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t *module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Instantiate. + printf("Instantiating module...\n"); + + // Create external functions. + printf("Creating callback...\n"); +#define IMPORT_FUNCTION_VARIABLE_NAME(name, ...) \ + own wasm_func_t *function_##name = NULL; + IMPORT_FUNCTION_LIST(IMPORT_FUNCTION_VARIABLE_NAME) +#undef IMPORT_FUNCTION_VARIABLE_NAME + +#define CREATE_WASM_FUNCTION(name, index, CREATE_FUNC_TYPE) \ + { \ + own wasm_functype_t *type = CREATE_FUNC_TYPE; \ + if (!(function_##name = wasm_func_new(store, type, STUB_##name))) { \ + printf("> Error creating new function\n"); \ + return 1; \ + } \ + wasm_functype_delete(type); \ + } + IMPORT_FUNCTION_LIST(CREATE_WASM_FUNCTION) +#undef CREATE_WASM_FUNCTION + + wasm_extern_t *fs[2] = { 0 }; +#define ADD_TO_FUNCTION_LIST(name, index, ...) \ + fs[index] = wasm_func_as_extern(function_##name); + IMPORT_FUNCTION_LIST(ADD_TO_FUNCTION_LIST) +#undef ADD_TO_FUNCTION_LIST + + wasm_extern_vec_t imports = WASM_ARRAY_VEC(fs); + own wasm_instance_t *instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + +#define DESTROY_WASM_FUNCITON(name, index, ...) \ + wasm_func_delete(function_##name); + IMPORT_FUNCTION_LIST(DESTROY_WASM_FUNCITON) +#undef DESTROY_WASM_FUNCITON + + // Extract export. + printf("Extracting export...\n"); + wasm_instance_exports(instance, &exports); + if (!exports.size) { + printf("> Error accessing exports!\n"); + return 1; + } + + wasm_module_delete(module); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + + if (!call_wasm_function(e_on_start, NULL, NULL, "on_start")) { + printf("> Error calling on_start\n"); + return 1; + } + + if (!call_wasm_function(e_on_stop, NULL, NULL, "on_stop")) { + printf("> Error calling on_stop\n"); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/callback_chain.wat b/wasm-micro-runtime/samples/wasm-c-api/src/callback_chain.wat new file mode 100644 index 0000000..80bbf4f --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/callback_chain.wat @@ -0,0 +1,32 @@ +;; Copyright (C) 2019 Intel Corporation. All rights reserved. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(module + (func $get_pairs (import "" "get_pairs") (result i32)) + (func $log (import "" "log") (param i32 i32)) + + (func $on_start (export "on_start") + (call $log (i32.const 0) (i32.const 9)) + (call $get_pairs) + (drop) + ) + + (func $on_stop (export "on_stop") + (call $log (i32.const 9) (i32.const 8)) + ) + + (func $malloc (export "malloc") (param i32) (result i32) + (call $log (i32.const 17) (i32.const 7)) + (i32.const 64) + ) + + (func $free(export "free") (param i32) + (call $log (i32.const 24) (i32.const 5)) + ) + + (memory (export "memory") 1) + (data (i32.const 0) "on_start") + (data (i32.const 9) "on_stop") + (data (i32.const 17) "malloc") + (data (i32.const 24) "free") +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/clone.c b/wasm-micro-runtime/samples/wasm-c-api/src/clone.c new file mode 100644 index 0000000..da83b37 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/clone.c @@ -0,0 +1,535 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define WORKER_NUMBER 10 + +/******************************* VM *******************************/ +/* Use wasm_vm_t and vm_xxx to simulate a minimal Wasm VM in Envoy */ + +typedef struct _vm { + wasm_engine_t *engine; + wasm_store_t *store; + wasm_module_t *module; + wasm_shared_module_t *shared_module; + wasm_instance_t *instance; + wasm_func_t **function_list; + wasm_memory_t *memory; + wasm_table_t *table; + wasm_extern_vec_t *exports; +} wasm_vm_t; + +typedef enum _clone_level { + not_cloneable = 0, + compiled_bytecode, + instantiated_module +} clone_level; + +typedef struct _thread_arg_t { + char name[32]; + bool *ready_go_flag; + pthread_mutex_t *ready_go_lock; + pthread_cond_t *ready_go_cond; + const wasm_vm_t *base_vm; +} thread_arg_t; + +wasm_vm_t * +vm_new() +{ + wasm_vm_t *vm = NULL; + + vm = malloc(sizeof(struct _vm)); + if (!vm) + goto fail; + + memset(vm, 0, sizeof(wasm_vm_t)); + + vm->engine = wasm_engine_new(); + if (!vm->engine) + goto fail; + + vm->store = wasm_store_new(vm->engine); + if (!vm->store) + goto fail; + + return vm; + +fail: + if (vm) { + if (vm->engine) + wasm_engine_delete(vm->engine); + + free(vm); + } + return NULL; +} + +wasm_vm_t * +vm_release(wasm_vm_t *vm) +{ + if (!vm) + return NULL; + + if (vm->function_list) { + free(vm->function_list); + vm->function_list = NULL; + } + + vm->memory = NULL; + + if (vm->exports) { + wasm_extern_vec_delete(vm->exports); + free(vm->exports); + vm->exports = NULL; + } + + wasm_instance_delete(vm->instance); + vm->instance = NULL; + + wasm_shared_module_delete(vm->shared_module); + vm->shared_module = NULL; + + wasm_module_delete(vm->module); + vm->module = NULL; + + wasm_store_delete(vm->store); + vm->store = NULL; + + wasm_engine_delete(vm->engine); + vm->engine = NULL; + + free(vm); + return NULL; +} + +bool +vm_load(wasm_vm_t *vm, const wasm_byte_vec_t *binary) +{ + vm->module = wasm_module_new(vm->store, binary); + vm->shared_module = wasm_module_share(vm->module); + return vm->module != NULL; +} + +bool +vm_link(wasm_vm_t *vm, wasm_extern_vec_t *imports) +{ + vm->instance = wasm_instance_new(vm->store, vm->module, imports, NULL); + if (!vm->instance) + goto fail; + + vm->exports = malloc(sizeof(wasm_extern_vec_t)); + if (!vm->exports) + goto fail; + + memset(vm->exports, 0, sizeof(wasm_extern_vec_t)); + wasm_instance_exports(vm->instance, vm->exports); + /* an exported memory, and two exported functions */ + assert(vm->exports->size == 3); + + /* bind memory */ + assert(wasm_extern_kind(vm->exports->data[0]) == WASM_EXTERN_MEMORY); + vm->memory = wasm_extern_as_memory(vm->exports->data[0]); + + vm->function_list = malloc(2 * sizeof(wasm_func_t *)); + if (!vm->function_list) + goto fail; + + memset(vm->function_list, 0, 2 * sizeof(wasm_func_t *)); + + /* bind wasm_set_byte(...) */ + assert(wasm_extern_kind(vm->exports->data[1]) == WASM_EXTERN_FUNC); + vm->function_list[0] = wasm_extern_as_func(vm->exports->data[1]); + + /* bind wasm_get_byte(...) */ + assert(wasm_extern_kind(vm->exports->data[2]) == WASM_EXTERN_FUNC); + vm->function_list[1] = wasm_extern_as_func(vm->exports->data[2]); + + return true; +fail: + return false; +} + +wasm_vm_t * +vm_clone_from_module(const wasm_vm_t *base) +{ + printf("Initializing...\n"); + wasm_vm_t *secondary = NULL; + + secondary = vm_new(); + if (secondary) { + printf("Reuse module and bypass vm_load()..."); + secondary->module = + wasm_module_obtain(base->store, base->shared_module); + if (!secondary->module) + secondary = vm_release(secondary); + } + + return secondary; +} + +wasm_vm_t * +vm_clone_from_instance(const wasm_vm_t *base) +{ + /** + * if do a clone of the level instantiated_module, need to malloc and + * initialie + * - global. WASMGlobalIntance and global data + * - memory. WAAMMemoryInstance, memory_data and heap + * - table. WASMTableInstance, table_data + * - exports. all global, memory and table + * + * it is almost everything in wasm_instantiate() except funciton. + */ + (void)base; + printf("Unsupported\n"); + return NULL; +} + +wasm_vm_t * +vm_clone(const wasm_vm_t *base, clone_level level) +{ + if (level == not_cloneable) + return NULL; + + if (level == compiled_bytecode) + return vm_clone_from_module(base); + else + return vm_clone_from_instance(base); +} + +bool +vm_memory_set_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t byte) +{ + byte_t *data = wasm_memory_data(vm->memory); + assert(data); + *(data + offset) = byte; + return true; +} + +bool +vm_memory_get_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t *byte) +{ + byte_t *data = wasm_memory_data(vm->memory); + assert(data); + *byte = *(data + offset); + return true; +} + +bool +vm_function_set_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t byte) +{ + wasm_val_t a_v[2] = { WASM_I32_VAL(offset), WASM_I32_VAL(byte) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(a_v); + wasm_val_vec_t results = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(vm->function_list[0], &args, &results); + if (trap) { + printf("call wasm_set_byte failed"); + wasm_trap_delete(trap); + return false; + } + + return true; +} + +bool +vm_function_get_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t *byte) +{ + wasm_val_t a_v[1] = { WASM_I32_VAL(offset) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(a_v); + wasm_val_t r_v[1] = { WASM_INIT_VAL }; + wasm_val_vec_t results = WASM_ARRAY_VEC(r_v); + wasm_trap_t *trap = wasm_func_call(vm->function_list[1], &args, &results); + if (trap) { + printf("call wasm_get_byte failed"); + wasm_trap_delete(trap); + return false; + } + + assert(results.data->kind == WASM_I32); + *byte = results.data->of.i32; + return true; +} + +static bool +load_wasm_file_content(const char *file_name, wasm_byte_vec_t *out) +{ + bool ret = false; +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE *file = fopen(file_name, "rb"); +#else + FILE *file = fopen(file_name, "rb"); +#endif + if (!file) { + printf("> Error loading .wasm!\n"); + goto quit; + } + + int offset = fseek(file, 0L, SEEK_END); + if (offset == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + offset = fseek(file, 0L, SEEK_SET); + if (offset == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + wasm_byte_vec_new_uninitialized(out, file_size); + if (fread(out->data, file_size, 1, file) != 1) { + printf("> Error loading content!\n"); + goto close_file; + } + + ret = true; +close_file: + fclose(file); +quit: + return ret; +} + +static pthread_key_t name_key; + +wasm_trap_t * +report_cb(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + (void)results; + + assert(args->data[0].kind == WASM_I32); + uint32_t chk_pnt_no = args->data[0].of.i32; + + char *name = pthread_getspecific(name_key); + printf("[%s] Pass CHK POINT #%u\n", name, chk_pnt_no); + + return NULL; +} + +bool +run_code_start(wasm_vm_t **out) +{ + bool ret = false; + + printf("Initializing...\n"); + wasm_vm_t *vm = vm_new(); + if (!vm) + goto fail; + + printf("Loading binary...\n"); + wasm_byte_vec_t binary = { 0 }; +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + const char *file_name = "clone.aot"; +#else + const char *file_name = "clone.wasm"; +#endif + if (!load_wasm_file_content(file_name, &binary)) + goto release_vm; + + printf("Compiling module...\n"); + ret = vm_load(vm, &binary); + wasm_byte_vec_delete(&binary); + if (!ret) + goto release_vm; + + printf("Creating callback...\n"); + wasm_functype_t *callback_type = + wasm_functype_new_1_0(wasm_valtype_new_i32()); + if (!callback_type) + goto release_vm; + + wasm_func_t *callback = wasm_func_new(vm->store, callback_type, report_cb); + wasm_functype_delete(callback_type); + if (!callback) + goto release_vm; + + printf("Instantiating module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(callback) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + ret = vm_link(vm, &imports); + wasm_func_delete(callback); + if (!ret) + goto release_vm; + + *out = vm; + return true; + +release_vm: + vm_release(vm); +fail: + return false; +} + +bool +run_warm_start_w_compiled_bytecode(const wasm_vm_t *first, wasm_vm_t **out) +{ + bool ret; + wasm_vm_t *secondary = vm_clone(first, compiled_bytecode); + if (!secondary) + goto fail; + + printf("Creating callback...\n"); + wasm_functype_t *callback_type = + wasm_functype_new_1_0(wasm_valtype_new_i32()); + if (!callback_type) + goto release_vm; + + wasm_func_t *callback = + wasm_func_new(secondary->store, callback_type, report_cb); + wasm_functype_delete(callback_type); + if (!callback) + goto release_vm; + + printf("Instantiating module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(callback) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + ret = vm_link(secondary, &imports); + wasm_func_delete(callback); + if (!ret) + goto release_vm; + + *out = secondary; + return true; + +release_vm: + vm_release(secondary); +fail: + return false; +} + +bool +run_warm_start_w_instantiated_module(const wasm_vm_t *first, wasm_vm_t **out) +{ + wasm_vm_t *secondary = vm_clone(first, instantiated_module); + if (!secondary) + return false; + + *out = secondary; + return true; +} + +void +run_test(const wasm_vm_t *vm) +{ + uint8_t byte = 0xFF; + + /* read initialization */ + vm_function_get_byte(vm, 10, &byte); + assert(byte == 0x0); + vm_memory_get_byte(vm, 10, &byte); + assert(byte == 0x0); + + /* read after writing */ + vm_function_set_byte(vm, 16, 0xab); + vm_function_get_byte(vm, 16, &byte); + assert(byte == 0xab); + + vm_memory_set_byte(vm, 16, 0xcd); + vm_memory_get_byte(vm, 16, &byte); + assert(byte == 0xcd); + + /* reading and writing across */ + vm_function_set_byte(vm, 16, 0xef); + vm_memory_get_byte(vm, 16, &byte); + assert(byte == 0xef); + + vm_memory_set_byte(vm, 16, 0x67); + vm_function_get_byte(vm, 16, &byte); + assert(byte == 0x67); + + printf("All Passed ...\n"); +} + +static void * +thrd_func(void *arg) +{ + thread_arg_t *thrd_arg = (thread_arg_t *)arg; + + sleep(rand() % 5); + printf("Running warm start at %s...\n", thrd_arg->name); + + pthread_setspecific(name_key, thrd_arg->name); + + wasm_vm_t *vm; + if (!run_warm_start_w_compiled_bytecode(thrd_arg->base_vm, &vm)) + return NULL; + + pthread_mutex_trylock(thrd_arg->ready_go_lock); + while (!(*thrd_arg->ready_go_flag)) { + pthread_cond_wait(thrd_arg->ready_go_cond, thrd_arg->ready_go_lock); + } + pthread_mutex_unlock(thrd_arg->ready_go_lock); + + printf("Running test at %s...\n", thrd_arg->name); + run_test(vm); + + vm_release(vm); + pthread_exit(NULL); + return NULL; +} + +int +main() +{ + int ret = EXIT_FAILURE; + bool ready_go_flag = false; + pthread_mutex_t ready_go_lock = PTHREAD_MUTEX_INITIALIZER; + pthread_cond_t ready_go_cond = PTHREAD_COND_INITIALIZER; + pthread_key_create(&name_key, NULL); + pthread_setspecific(name_key, "Execution Thread"); + + printf("Running cold start at the execution thread...\n"); + wasm_vm_t *base_vm; + if (!run_code_start(&base_vm)) + goto quit; + run_test(base_vm); + + printf("Running warm start at other threads...\n"); + + pthread_t tids[WORKER_NUMBER] = { 0 }; + thread_arg_t thrd_args[WORKER_NUMBER] = { 0 }; + for (size_t i = 0; i < sizeof(tids) / sizeof(tids[0]); i++) { + thread_arg_t *thrd_arg = thrd_args + i; + + snprintf(thrd_arg->name, 32, "Worker#%lu", i); + thrd_arg->ready_go_cond = &ready_go_cond; + thrd_arg->ready_go_lock = &ready_go_lock; + thrd_arg->ready_go_flag = &ready_go_flag; + thrd_arg->base_vm = base_vm; + + int ret = pthread_create(&tids[i], NULL, thrd_func, thrd_arg); + if (ret != 0) + break; + } + + sleep(1); + pthread_mutex_trylock(&ready_go_lock); + ready_go_flag = true; + pthread_mutex_unlock(&ready_go_lock); + pthread_cond_broadcast(&ready_go_cond); + + sleep(3); + for (size_t i = 0; i < sizeof(tids) / sizeof(tids[0]); i++) { + if (tids[i] != 0) + pthread_join(tids[i], NULL); + } + + vm_release(base_vm); + ret = EXIT_SUCCESS; +quit: + return ret; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/clone.wat b/wasm-micro-runtime/samples/wasm-c-api/src/clone.wat new file mode 100644 index 0000000..e9934cc --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/clone.wat @@ -0,0 +1,15 @@ +(module + (func $report (import "" "report") (param i32)) + + (memory (export "mem") 1 1) + + (func $wasm_set_byte (export "set_byte") (param i32 i32) + (call $report (i32.const 1)) + (i32.store8 (local.get 0) (local.get 1)) + ) + + (func $wasm_get_byte (export "get_byte") (param i32) (result i32) + (call $report (i32.const 2)) + (i32.load(local.get 0)) + ) +) \ No newline at end of file diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/empty_imports.c b/wasm-micro-runtime/samples/wasm-c-api/src/empty_imports.c new file mode 100644 index 0000000..c8788b5 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/empty_imports.c @@ -0,0 +1,122 @@ +#include + +#include "wasm_c_api.h" + +#define own + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("empty_imports.aot", "rb"); +#else + FILE* file = fopen("empty_imports.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Instantiate with non-null but empty imports array. + printf("Instantiating module...\n"); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + // Run an exported function to verify that the instance was created correctly. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return 1; + } + + const wasm_func_t* add_func = wasm_extern_as_func(exports.data[0]); + if (add_func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + wasm_module_delete(module); + wasm_instance_delete(instance); + + printf("Calling export...\n"); + wasm_val_t args[2] = { WASM_I32_VAL(3), WASM_I32_VAL(4) }; + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + + wasm_val_t results[1] = { WASM_INIT_VAL }; + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); + + wasm_trap_t *trap = wasm_func_call(add_func, &args_vec, &results_vec); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + return 1; + } + + if (results_vec.data[0].of.i32 != 7) { + printf("> Error calling function!\n"); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/empty_imports.wat b/wasm-micro-runtime/samples/wasm-c-api/src/empty_imports.wat new file mode 100644 index 0000000..3863979 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/empty_imports.wat @@ -0,0 +1,5 @@ +(module + (func (export "add") (param i32 i32) (result i32) + (i32.add (local.get 0) (local.get 1)) + ) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/global.c b/wasm-micro-runtime/samples/wasm-c-api/src/global.c new file mode 100644 index 0000000..91c8cb6 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/global.c @@ -0,0 +1,278 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +wasm_global_t* get_export_global(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_global(exports->data[i])) { + printf("> Error accessing global export %zu!\n", i); + exit(1); + } + return wasm_extern_as_global(exports->data[i]); +} + +wasm_func_t* get_export_func(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_func(exports->data[i])) { + printf("> Error accessing function export %zu!\n", i); + exit(1); + } + return wasm_extern_as_func(exports->data[i]); +} + + +#define check(val, type, expected) \ + if (val.of.type != expected) { \ + printf("> Error reading value\n"); \ + exit(1); \ + } + +#define check_global(global, type, expected) \ + { \ + wasm_val_t val; \ + wasm_global_get(global, &val); \ + check(val, type, expected); \ + } + +#define check_trap(trap) \ + if (trap) { \ + printf("> Error calling function\n"); \ + wasm_trap_delete(trap); \ + exit(1); \ + } + +#define check_call(func, type, expected) \ + { \ + wasm_val_t vs[1]; \ + wasm_val_vec_t args = WASM_EMPTY_VEC; \ + wasm_val_vec_t results = WASM_ARRAY_VEC(vs); \ + wasm_trap_t *trap = wasm_func_call(func, &args, &results); \ + check_trap(trap); \ + check(vs[0], type, expected); \ + } + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("global.aot", "rb"); +#else + FILE* file = fopen("global.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Create external globals. + printf("Creating globals...\n"); + own wasm_globaltype_t* const_f32_type = wasm_globaltype_new( + wasm_valtype_new(WASM_F32), WASM_CONST); + own wasm_globaltype_t* const_i64_type = wasm_globaltype_new( + wasm_valtype_new(WASM_I64), WASM_CONST); + own wasm_globaltype_t* var_f32_type = wasm_globaltype_new( + wasm_valtype_new(WASM_F32), WASM_VAR); + own wasm_globaltype_t* var_i64_type = wasm_globaltype_new( + wasm_valtype_new(WASM_I64), WASM_VAR); + + wasm_val_t val_f32_1 = WASM_F32_VAL(1); + own wasm_global_t* const_f32_import = + wasm_global_new(store, const_f32_type, &val_f32_1); + wasm_val_t val_i64_2 = WASM_I64_VAL(2); + own wasm_global_t* const_i64_import = + wasm_global_new(store, const_i64_type, &val_i64_2); + wasm_val_t val_f32_3 = WASM_F32_VAL(3); + own wasm_global_t* var_f32_import = + wasm_global_new(store, var_f32_type, &val_f32_3); + wasm_val_t val_i64_4 = WASM_I64_VAL(4); + own wasm_global_t* var_i64_import = + wasm_global_new(store, var_i64_type, &val_i64_4); + + wasm_globaltype_delete(const_f32_type); + wasm_globaltype_delete(const_i64_type); + wasm_globaltype_delete(var_f32_type); + wasm_globaltype_delete(var_i64_type); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_t* externs[] = { + wasm_global_as_extern(const_f32_import), + wasm_global_as_extern(const_i64_import), + wasm_global_as_extern(var_f32_import), + wasm_global_as_extern(var_i64_import) + }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_module_delete(module); + + // Extract export. + printf("Extracting exports...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + size_t i = 0; + wasm_global_t* const_f32_export = get_export_global(&exports, i++); + wasm_global_t* const_i64_export = get_export_global(&exports, i++); + wasm_global_t* var_f32_export = get_export_global(&exports, i++); + wasm_global_t* var_i64_export = get_export_global(&exports, i++); + wasm_func_t* get_const_f32_import = get_export_func(&exports, i++); + wasm_func_t* get_const_i64_import = get_export_func(&exports, i++); + wasm_func_t* get_var_f32_import = get_export_func(&exports, i++); + wasm_func_t* get_var_i64_import = get_export_func(&exports, i++); + wasm_func_t* get_const_f32_export = get_export_func(&exports, i++); + wasm_func_t* get_const_i64_export = get_export_func(&exports, i++); + wasm_func_t* get_var_f32_export = get_export_func(&exports, i++); + wasm_func_t* get_var_i64_export = get_export_func(&exports, i++); + wasm_func_t* set_var_f32_import = get_export_func(&exports, i++); + wasm_func_t* set_var_i64_import = get_export_func(&exports, i++); + wasm_func_t* set_var_f32_export = get_export_func(&exports, i++); + wasm_func_t* set_var_i64_export = get_export_func(&exports, i++); + + // Try cloning. + own wasm_global_t* copy = wasm_global_copy(var_f32_import); + assert(wasm_global_same(var_f32_import, copy)); + wasm_global_delete(copy); + + // Interact. + printf("Accessing globals...\n"); + + // Check initial values. + check_global(const_f32_import, f32, 1); + check_global(const_i64_import, i64, 2); + check_global(var_f32_import, f32, 3); + check_global(var_i64_import, i64, 4); + check_global(const_f32_export, f32, 5); + check_global(const_i64_export, i64, 6); + check_global(var_f32_export, f32, 7); + check_global(var_i64_export, i64, 8); + + check_call(get_const_f32_import, f32, 1); + check_call(get_const_i64_import, i64, 2); + check_call(get_var_f32_import, f32, 3); + check_call(get_var_i64_import, i64, 4); + check_call(get_const_f32_export, f32, 5); + check_call(get_const_i64_export, i64, 6); + check_call(get_var_f32_export, f32, 7); + check_call(get_var_i64_export, i64, 8); + + // Modify variables through API and check again. + wasm_val_t val33 = WASM_F32_VAL(33); + wasm_global_set(var_f32_import, &val33); + wasm_val_t val34 = WASM_I64_VAL(34); + wasm_global_set(var_i64_import, &val34); + wasm_val_t val37 = WASM_F32_VAL(37); + wasm_global_set(var_f32_export, &val37); + wasm_val_t val38 = WASM_I64_VAL(38); + wasm_global_set(var_i64_export, &val38); + + check_global(var_f32_import, f32, 33); + check_global(var_i64_import, i64, 34); + check_global(var_f32_export, f32, 37); + check_global(var_i64_export, i64, 38); + + check_call(get_var_f32_import, f32, 33); + check_call(get_var_i64_import, i64, 34); + check_call(get_var_f32_export, f32, 37); + check_call(get_var_i64_export, i64, 38); + + // Modify variables through calls and check again. + wasm_val_vec_t res = WASM_EMPTY_VEC; + wasm_val_t vs73[] = { WASM_F32_VAL(73) }; + wasm_val_vec_t args73 = WASM_ARRAY_VEC(vs73); + wasm_trap_t *trap = wasm_func_call(set_var_f32_import, &args73, &res); + check_trap(trap); + + wasm_val_t vs74[] = { WASM_I64_VAL(74) }; + wasm_val_vec_t args74 = WASM_ARRAY_VEC(vs74); + trap = wasm_func_call(set_var_i64_import, &args74, &res); + check_trap(trap); + + wasm_val_t vs77[] = { WASM_F32_VAL(77) }; + wasm_val_vec_t args77 = WASM_ARRAY_VEC(vs77); + trap = wasm_func_call(set_var_f32_export, &args77, &res); + check_trap(trap); + + wasm_val_t vs78[] = { WASM_I64_VAL(78) }; + wasm_val_vec_t args78 = WASM_ARRAY_VEC(vs78); + trap = wasm_func_call(set_var_i64_export, &args78, &res); + check_trap(trap); + + check_global(var_f32_import, f32, 73); + check_global(var_i64_import, i64, 74); + check_global(var_f32_export, f32, 77); + check_global(var_i64_export, i64, 78); + + check_call(get_var_f32_import, f32, 73); + check_call(get_var_i64_import, i64, 74); + check_call(get_var_f32_export, f32, 77); + check_call(get_var_i64_export, i64, 78); + + wasm_global_delete(const_f32_import); + wasm_global_delete(const_i64_import); + wasm_global_delete(var_f32_import); + wasm_global_delete(var_i64_import); + wasm_extern_vec_delete(&exports); + wasm_instance_delete(instance); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/global.wat b/wasm-micro-runtime/samples/wasm-c-api/src/global.wat new file mode 100644 index 0000000..dea0857 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/global.wat @@ -0,0 +1,27 @@ +(module + (global $f32_import (import "" "const f32") f32) + (global $i64_import (import "" "const i64") i64) + (global $mut_f32_import (import "" "var f32") (mut f32)) + (global $mut_i64_import (import "" "var i64") (mut i64)) + + (global $f32_export (export "const f32") f32 (f32.const 5)) + (global $i64_export (export "const i64") i64 (i64.const 6)) + (global $mut_f32_export (export "var f32") (mut f32) (f32.const 7)) + (global $mut_i64_export (export "var i64") (mut i64) (i64.const 8)) + + (func (export "get const f32 import") (result f32) (global.get $f32_import)) + (func (export "get const i64 import") (result i64) (global.get $i64_import)) + (func (export "get var f32 import") (result f32) (global.get $mut_f32_import)) + (func (export "get var i64 import") (result i64) (global.get $mut_i64_import)) + + (func (export "get const f32 export") (result f32) (global.get $f32_export)) + (func (export "get const i64 export") (result i64) (global.get $i64_export)) + (func (export "get var f32 export") (result f32) (global.get $mut_f32_export)) + (func (export "get var i64 export") (result i64) (global.get $mut_i64_export)) + + (func (export "set var f32 import") (param f32) (global.set $mut_f32_import (local.get 0))) + (func (export "set var i64 import") (param i64) (global.set $mut_i64_import (local.get 0))) + + (func (export "set var f32 export") (param f32) (global.set $mut_f32_export (local.get 0))) + (func (export "set var f64 export") (param i64) (global.set $mut_i64_export (local.get 0))) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport-0.wat b/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport-0.wat new file mode 100644 index 0000000..7ab897c --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport-0.wat @@ -0,0 +1,8 @@ +;; Copyright (C) 2019 Intel Corporation. All rights reserved. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(module + (global $mut_f32_export (export "var f32") (mut f32) (f32.const 7)) + (func (export "get var f32 export") (result f32) (global.get $mut_f32_export)) + (func (export "set var f32 export") (param f32) (global.set $mut_f32_export (local.get 0))) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport-1.wat b/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport-1.wat new file mode 100644 index 0000000..927e420 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport-1.wat @@ -0,0 +1,10 @@ +;; Copyright (C) 2019 Intel Corporation. All rights reserved. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(module + (global $mut_f32_import (export "var f32") (import "globalexportimport-0" "var f32") (mut f32)) + (func (export "get var f32 export") (import "globalexportimport-0" "get var f32 export") (result f32)) + (func (export "set var f32 export") (import "globalexportimport-0" "set var f32 export") (param f32)) + (func (export "get var f32 import") (result f32) (global.get $mut_f32_import)) + (func (export "set var f32 import") (param f32) (global.set $mut_f32_import (local.get 0))) +) \ No newline at end of file diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport.c b/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport.c new file mode 100644 index 0000000..1c1715f --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/globalexportimport.c @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +#include "wasm_c_api.h" +#include "wasm_export.h" +#include "bh_platform.h" + +extern bool +reader(const char *module_name, uint8 **p_buffer, uint32 *p_size); + +extern void +destroyer(uint8 *buffer, uint32 size); + +#define own + +wasm_global_t* get_export_global(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_global(exports->data[i])) { + printf("> Error accessing global export %zu!\n", i); + exit(1); + } + return wasm_extern_as_global(exports->data[i]); +} + +wasm_func_t* get_export_func(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_func(exports->data[i])) { + printf("> Error accessing function export %zu!\n", i); + exit(1); + } + return wasm_extern_as_func(exports->data[i]); +} + + +#define check(val, type, expected) \ + if (val.of.type != expected) { \ + printf("> Expected reading value %f or %f \n", expected, expected); \ + printf("> Error reading value %f or %f\n", val.of.type, val.of.type); \ + } + +#define check_global(global, type, expected) \ + { \ + wasm_val_t val; \ + wasm_global_get(global, &val); \ + check(val, type, expected); \ + } + +#define check_trap(trap) \ + if (trap) { \ + printf("> Error calling function\n"); \ + wasm_trap_delete(trap); \ + exit(1); \ + } + +#define check_call(func, type, expected) \ + { \ + wasm_val_vec_t results; \ + wasm_val_vec_new_uninitialized(&results, 1); \ + wasm_trap_t *trap = wasm_func_call(func, NULL, &results); \ + check_trap(trap); \ + check(results.data[0], type, expected); \ + } + +wasm_module_t * create_module_from_file(wasm_store_t* store, const char * filename) +{ + FILE* file = fopen(filename, "rb"); + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + fseek(file, 0L, SEEK_SET); + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return NULL; + } + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return NULL; + } + wasm_byte_vec_delete(&binary); + fclose(file); + return module; +} + + +int main(int argc, const char* argv[]) { + wasm_runtime_set_module_reader(reader, destroyer); + + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + wasm_module_t* moduleimport = + create_module_from_file(store, "globalimport.aot"); +#else + wasm_module_t* moduleimport = + create_module_from_file(store, "globalexportimport-1.wasm"); +#endif + + if (!moduleimport) { + return 1; + } + + // Instantiate. + printf("Instantiating Import module...\n"); + own wasm_instance_t* instance_import = + wasm_instance_new(store, moduleimport, NULL, NULL); //after this var_f32_export->inst_comm_rt is module_import + if (!instance_import) { + printf("> Error instantiating Import module!\n"); + return 1; + } + wasm_module_delete(moduleimport); + + // Extract export. + printf("Extracting exports from Import module...\n"); + own wasm_extern_vec_t exports_of_import; + wasm_instance_exports(instance_import, &exports_of_import); + int i = 0; + wasm_global_t *var_f32_export = get_export_global(&exports_of_import, i++); + wasm_func_t *get_var_f32_export = get_export_func(&exports_of_import, i++); + wasm_func_t* set_var_f32_export = get_export_func(&exports_of_import, i++); + wasm_func_t* get_var_f32_import = get_export_func(&exports_of_import, i++); + wasm_func_t* set_var_f32_import = get_export_func(&exports_of_import, i++); + + // Interact. + + // Check initial values. + printf("Check initial values...\n"); + check_global(var_f32_export, f32, 7.0); + check_call(get_var_f32_export, f32, 7.0); //Call to module export + check_call(get_var_f32_import, f32, 7.0); //Call to module import + + + // Modify variables through API and check again. + printf("Modify the variable to 37.0...\n"); + wasm_val_t val37 = {.kind = WASM_F32, .of = {.f32 = 37.0}}; + wasm_global_set(var_f32_export, &val37); // var_f32_export->inst_comm_rt is module_import now + + check_global(var_f32_export, f32, 37.0); + check_call(get_var_f32_export, f32, 37.0); //Call to module export Failed here, still 7 + check_call(get_var_f32_import, f32, 37.0); //Call to module import + + // Modify variables through calls and check again. + printf("Modify the variable to 77.0...\n"); + wasm_val_vec_t args77; + wasm_val_vec_new(&args77, 1, (wasm_val_t []){ {.kind = WASM_F32, .of = {.f32 = 77.0}} }); + wasm_trap_t *trap = wasm_func_call(set_var_f32_export, &args77, + NULL); // Call to module export + check_trap(trap); + check_call(get_var_f32_export, f32, 77.0); //Call to module export + check_global(var_f32_export, f32, 77.0); //Failed here, still 37 + check_call(get_var_f32_import, f32, 77.0); //Call to module import Failed here, still 37 + + + printf("Modify the variable to 78.0...\n"); + wasm_val_vec_t args78; + wasm_val_vec_new(&args78, 1, (wasm_val_t []){ {.kind = WASM_F32, .of = {.f32 = 78.0}} }); + trap = wasm_func_call(set_var_f32_import, &args78, NULL); + check_trap(trap); + check_global(var_f32_export, f32, 78.0); + check_call(get_var_f32_export, f32, 78.0); //Call to module export Failed here, still 77 + check_call(get_var_f32_import, f32, 78.0); //Call to module import + + + // wasm_extern_vec_delete(&exports_of_export); + //wasm_instance_delete(instance_export); + wasm_extern_vec_delete(&exports_of_import); + //wasm_instance_delete(instance_import); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; + +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/hello.c b/wasm-micro-runtime/samples/wasm-c-api/src/hello.c new file mode 100644 index 0000000..6650f34 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/hello.c @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// A function to be called from Wasm code. +own wasm_trap_t* hello_callback( + const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + printf("Calling back...\n"); + printf("> Hello World!\n"); + return NULL; +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("hello.aot", "rb"); +#else + FILE* file = fopen("hello.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + if (!wasm_module_validate(store, &binary)) { + printf("> Error validate module!\n"); + wasm_byte_vec_delete(&binary); + return 1; + } + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + assert(wasm_module_set_name(module, "hello")); + + // Create external print functions. + printf("Creating callback...\n"); + own wasm_functype_t* hello_type = wasm_functype_new_0_0(); + own wasm_func_t* hello_func = + wasm_func_new(store, hello_type, hello_callback); + + wasm_functype_delete(hello_type); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_t* externs[] = { wasm_func_as_extern(hello_func) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(hello_func); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return 1; + } + + const wasm_func_t* run_func = wasm_extern_as_func(exports.data[0]); + if (run_func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + { + const char* name = wasm_module_get_name(module); + assert(strncmp(name, "hello", 5) == 0); + printf("> removing module %s \n", name); + } + + wasm_module_delete(module); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(run_func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/hello.wat b/wasm-micro-runtime/samples/wasm-c-api/src/hello.wat new file mode 100644 index 0000000..1c56c55 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/hello.wat @@ -0,0 +1,4 @@ +(module + (func $hello (import "" "hello")) + (func (export "run") (call $hello)) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/hostref.c b/wasm-micro-runtime/samples/wasm-c-api/src/hostref.c new file mode 100644 index 0000000..219c862 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/hostref.c @@ -0,0 +1,308 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + + +// A function to be called from Wasm code. +own wasm_trap_t* callback( + const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + printf("Calling back...\n> "); + printf("> %p\n", + args->data[0].of.ref ? wasm_ref_get_host_info(args->data[0].of.ref) : NULL); + wasm_val_copy(&results->data[0], &args->data[0]); + return NULL; +} + + +wasm_func_t* get_export_func(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_func(exports->data[i])) { + printf("> Error accessing function export %zu!\n", i); + exit(1); + } + return wasm_extern_as_func(exports->data[i]); +} + +wasm_global_t* get_export_global(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_global(exports->data[i])) { + printf("> Error accessing global export %zu!\n", i); + exit(1); + } + return wasm_extern_as_global(exports->data[i]); +} + +wasm_table_t* get_export_table(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_table(exports->data[i])) { + printf("> Error accessing table export %zu!\n", i); + exit(1); + } + return wasm_extern_as_table(exports->data[i]); +} + + +own wasm_ref_t* call_v_r(const wasm_func_t* func) { + printf("call_v_r... "); fflush(stdout); + wasm_val_t rs[] = { WASM_INIT_VAL }; + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_ARRAY_VEC(rs); + wasm_trap_t *trap = wasm_func_call(func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + exit(1); + } + printf("okay\n"); + return rs[0].of.ref; +} + +void call_r_v(const wasm_func_t* func, wasm_ref_t* ref) { + printf("call_r_v... "); fflush(stdout); + wasm_val_t vs[1] = { WASM_REF_VAL(ref) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(vs); + wasm_val_vec_t results = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + exit(1); + } + printf("okay\n"); +} + +own wasm_ref_t* call_r_r(const wasm_func_t* func, wasm_ref_t* ref) { + printf("call_r_r... "); fflush(stdout); + wasm_val_t vs[1] = { WASM_REF_VAL(ref) }; + wasm_val_t rs[1] = { WASM_INIT_VAL }; + wasm_val_vec_t args = WASM_ARRAY_VEC(vs); + wasm_val_vec_t results = WASM_ARRAY_VEC(rs); + wasm_trap_t *trap = wasm_func_call(func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + exit(1); + } + printf("okay\n"); + return rs[0].of.ref; +} + +void call_ir_v(const wasm_func_t* func, int32_t i, wasm_ref_t* ref) { + printf("call_ir_v... "); fflush(stdout); + wasm_val_t vs[2] = { WASM_I32_VAL(i), WASM_REF_VAL(ref) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(vs); + wasm_val_vec_t results = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + exit(1); + } + printf("okay\n"); +} + +own wasm_ref_t* call_i_r(const wasm_func_t* func, int32_t i) { + printf("call_i_r... "); fflush(stdout); + wasm_val_t vs[1] = { WASM_I32_VAL(i) }; + wasm_val_t rs[1] = { WASM_INIT_VAL }; + wasm_val_vec_t args = WASM_ARRAY_VEC(vs); + wasm_val_vec_t results = WASM_ARRAY_VEC(rs); + wasm_trap_t *trap = wasm_func_call(func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + exit(1); + } + printf("okay\n"); + return rs[0].of.ref; +} + +void check(own wasm_ref_t* actual, const wasm_ref_t* expected) { + if (actual != expected && + !(actual && expected && wasm_ref_same(actual, expected))) { + printf("> Error reading reference, expected %p, got %p\n", + expected ? wasm_ref_get_host_info(expected) : NULL, + actual ? wasm_ref_get_host_info(actual) : NULL); + exit(1); + } + // if (actual) wasm_ref_delete(actual); +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("hostref.aot", "rb"); +#else + FILE* file = fopen("hostref.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Create external callback function. + printf("Creating callback...\n"); + own wasm_functype_t* callback_type = wasm_functype_new_1_1( + wasm_valtype_new(WASM_ANYREF), wasm_valtype_new(WASM_ANYREF)); + own wasm_func_t* callback_func = + wasm_func_new(store, callback_type, callback); + + wasm_functype_delete(callback_type); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_t* externs[] = { wasm_func_as_extern(callback_func) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(callback_func); + wasm_module_delete(module); + + // Extract export. + printf("Extracting exports...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + size_t i = 0; + wasm_global_t* global = get_export_global(&exports, i++); + wasm_table_t* table = get_export_table(&exports, i++); + wasm_func_t* global_set = get_export_func(&exports, i++); + wasm_func_t* global_get = get_export_func(&exports, i++); + wasm_func_t* table_set = get_export_func(&exports, i++); + wasm_func_t* table_get = get_export_func(&exports, i++); + wasm_func_t* func_call = get_export_func(&exports, i++); + + wasm_instance_delete(instance); + + // Create host references. + printf("Creating host references...\n"); + own wasm_ref_t* host1 = wasm_foreign_as_ref(wasm_foreign_new(store)); + own wasm_ref_t* host2 = wasm_foreign_as_ref(wasm_foreign_new(store)); + wasm_ref_set_host_info(host1, (void*)1); + wasm_ref_set_host_info(host2, (void*)2); + + // Some sanity checks. + check(NULL, NULL); + wasm_ref_t *host1_cp = wasm_ref_copy(host1); + wasm_ref_t *host2_cp = wasm_ref_copy(host2); + check(host1_cp, host1); + check(host2_cp, host2); + wasm_ref_delete(host1_cp); + wasm_ref_delete(host2_cp); + + own wasm_val_t val; + val.kind = WASM_ANYREF; + val.of.ref = wasm_ref_copy(host1); + wasm_ref_t *ref_cp = wasm_ref_copy(val.of.ref); + check(ref_cp, host1); + check(val.of.ref, host1); + wasm_ref_delete(val.of.ref); + wasm_ref_delete(ref_cp); + + // Interact. + printf("Accessing global...\n"); + check(call_v_r(global_get), NULL); + call_r_v(global_set, host1); + check(call_v_r(global_get), host1); + call_r_v(global_set, host2); + check(call_v_r(global_get), host2); + call_r_v(global_set, NULL); + check(call_v_r(global_get), NULL); + + wasm_global_get(global, &val); + assert(val.kind == WASM_ANYREF); + assert(val.of.ref == NULL); + val.of.ref = host2; + wasm_global_set(global, &val); + wasm_global_get(global, &val); + assert(val.kind == WASM_ANYREF); + assert(val.of.ref == host2); + + printf("Accessing table...\n"); + check(call_i_r(table_get, 0), NULL); + check(call_i_r(table_get, 1), NULL); + call_ir_v(table_set, 0, host1); + call_ir_v(table_set, 1, host2); + check(call_i_r(table_get, 0), host1); + check(call_i_r(table_get, 1), host2); + call_ir_v(table_set, 0, NULL); + check(call_i_r(table_get, 0), NULL); + + check(wasm_table_get(table, 2), NULL); + wasm_table_set(table, 2, host1); + check(call_i_r(table_get, 2), host1); + check(wasm_table_get(table, 2), host1); + + printf("Accessing function...\n"); + check(call_r_r(func_call, NULL), NULL); + check(call_r_r(func_call, host1), host1); + check(call_r_r(func_call, host2), host2); + + wasm_ref_delete(host1); + wasm_ref_delete(host2); + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/hostref.wat b/wasm-micro-runtime/samples/wasm-c-api/src/hostref.wat new file mode 100644 index 0000000..9ed43db --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/hostref.wat @@ -0,0 +1,24 @@ +(module + (import "" "f" (func $fun (param externref) (result externref))) + + (global $glob (export "global") (mut externref) (ref.null extern)) + (table $tab (export "table") 10 externref) + + (func (export "global.set") (param $r externref) + (global.set $glob (local.get $r)) + ) + (func (export "global.get") (result externref) + (global.get $glob) + ) + + (func (export "table.set") (param $i i32) (param $r externref) + (table.set $tab (local.get $i) (local.get $r)) + ) + (func (export "table.get") (param $i i32) (result externref) + (table.get $tab (local.get $i)) + ) + + (func (export "func.call") (param $r externref) (result externref) + (call $fun (local.get $r)) + ) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/memory.c b/wasm-micro-runtime/samples/wasm-c-api/src/memory.c new file mode 100644 index 0000000..737dd85 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/memory.c @@ -0,0 +1,235 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + + +wasm_memory_t* get_export_memory(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_memory(exports->data[i])) { + printf("> Error accessing memory export %zu!\n", i); + exit(1); + } + return wasm_extern_as_memory(exports->data[i]); +} + +wasm_func_t* get_export_func(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_func(exports->data[i])) { + printf("> Error accessing function export %zu!\n", i); + exit(1); + } + return wasm_extern_as_func(exports->data[i]); +} + + +void check(bool success) { + if (!success) { + printf("> Error, expected success\n"); + exit(1); + } +} + +void check_call(wasm_func_t* func, int i, wasm_val_t args[], int32_t expected) { + wasm_val_t r[] = {WASM_INIT_VAL}; + wasm_val_vec_t args_ = {i, args, i, sizeof(wasm_val_t), NULL}; + wasm_val_vec_t results = WASM_ARRAY_VEC(r); + wasm_trap_t *trap = wasm_func_call(func, &args_, &results); + if (trap) { + printf("> Error on result\n"); + wasm_trap_delete(trap); + exit(1); + } + + if (r[0].of.i32 != expected) { + printf("> Error on result\n"); + exit(1); + } +} + +void check_call0(wasm_func_t* func, int32_t expected) { + check_call(func, 0, NULL, expected); +} + +void check_call1(wasm_func_t* func, int32_t arg, int32_t expected) { + wasm_val_t args[] = { WASM_I32_VAL(arg) }; + check_call(func, 1, args, expected); +} + +void check_call2(wasm_func_t* func, int32_t arg1, int32_t arg2, int32_t expected) { + wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + check_call(func, 2, args, expected); +} + +void check_ok(wasm_func_t* func, int i, wasm_val_t args[]) { + wasm_val_vec_t args_ = {i, args, i, sizeof(wasm_val_t), NULL}; + wasm_val_vec_t results = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(func, &args_, &results); + if (trap) { + printf("> Error on result, expected empty\n"); + wasm_trap_delete(trap); + exit(1); + } +} + +void check_ok2(wasm_func_t* func, int32_t arg1, int32_t arg2) { + wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + check_ok(func, 2, args); +} + +void check_trap(wasm_func_t* func, int i, wasm_val_t args[]) { + wasm_val_t r[] = {WASM_INIT_VAL}; + wasm_val_vec_t args_ = {i, args, i, sizeof(wasm_val_t), NULL}; + wasm_val_vec_t results = WASM_ARRAY_VEC(r); + own wasm_trap_t* trap = wasm_func_call(func, &args_, &results); + if (! trap) { + printf("> Error on result, expected trap\n"); + exit(1); + } + wasm_trap_delete(trap); +} + +void check_trap1(wasm_func_t* func, int32_t arg) { + wasm_val_t args[] = { WASM_I32_VAL(arg) }; + check_trap(func, 1, args); +} + +void check_trap2(wasm_func_t* func, int32_t arg1, int32_t arg2) { + wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + check_trap(func, 2, args); +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("memory.aot", "rb"); +#else + FILE* file = fopen("memory.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + own wasm_instance_t *instance = wasm_instance_new_with_args( + store, module, &imports, NULL, KILOBYTE(32), 0); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + // Extract export. + printf("Extracting exports...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + size_t i = 0; + wasm_memory_t* memory = get_export_memory(&exports, i++); + wasm_func_t* size_func = get_export_func(&exports, i++); + wasm_func_t* load_func = get_export_func(&exports, i++); + wasm_func_t* store_func = get_export_func(&exports, i++); + + wasm_module_delete(module); + + // Try cloning. + own wasm_memory_t* copy = wasm_memory_copy(memory); + assert(wasm_memory_same(memory, copy)); + wasm_memory_delete(copy); + + // Check initial memory. + printf("Checking memory...\n"); + check(wasm_memory_size(memory) == 2); + check(wasm_memory_data_size(memory) == 0x20000); + check(wasm_memory_data(memory)[0] == 0); + check(wasm_memory_data(memory)[0x1000] == 1); + check(wasm_memory_data(memory)[0x1003] == 4); + + check_call0(size_func, 2); + check_call1(load_func, 0, 0); + check_call1(load_func, 0x1000, 1); + check_call1(load_func, 0x1003, 4); + check_call1(load_func, 0x1ffff, 0); + check_trap1(load_func, 0x20000); + + // Mutate memory. + printf("Mutating memory...\n"); + wasm_memory_data(memory)[0x1003] = 5; + check_ok2(store_func, 0x1002, 6); + check_trap2(store_func, 0x20000, 0); + + check(wasm_memory_data(memory)[0x1002] == 6); + check(wasm_memory_data(memory)[0x1003] == 5); + check_call1(load_func, 0x1002, 6); + check_call1(load_func, 0x1003, 5); + + // Grow memory. + // DO NOT SUPPORT + printf("Bypass Growing memory...\n"); + wasm_extern_vec_delete(&exports); + wasm_instance_delete(instance); + + // Create stand-alone memory. + // DO NOT SUPPORT + // TODO(wasm+): Once Wasm allows multiple memories, turn this into import. + printf("Bypass Creating stand-alone memory...\n"); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/memory.wat b/wasm-micro-runtime/samples/wasm-c-api/src/memory.wat new file mode 100644 index 0000000..4cf43e2 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/memory.wat @@ -0,0 +1,11 @@ +(module + (memory (export "memory") 2 3) + + (func (export "size") (result i32) (memory.size)) + (func (export "load") (param i32) (result i32) (i32.load8_s (local.get 0))) + (func (export "store") (param i32 i32) + (i32.store8 (local.get 0) (local.get 1)) + ) + + (data (i32.const 0x1000) "\01\02\03\04") +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/multi.c b/wasm-micro-runtime/samples/wasm-c-api/src/multi.c new file mode 100644 index 0000000..a1c8fa9 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/multi.c @@ -0,0 +1,163 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// A function to be called from Wasm code. +own wasm_trap_t* callback( + const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + printf("Calling back...\n> "); + printf("> %"PRIu32" %"PRIu64" %"PRIu64" %"PRIu32"\n", + args->data[0].of.i32, args->data[1].of.i64, + args->data[2].of.i64, args->data[3].of.i32); + printf("\n"); + + wasm_val_copy(&results->data[0], &args->data[3]); + wasm_val_copy(&results->data[1], &args->data[1]); + wasm_val_copy(&results->data[2], &args->data[2]); + wasm_val_copy(&results->data[3], &args->data[0]); + return NULL; +} + + +// A function closure. +own wasm_trap_t* closure_callback( + void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + int i = *(int*)env; + printf("Calling back closure...\n"); + printf("> %d\n", i); + + results->data[0].kind = WASM_I32; + results->data[0].of.i32 = (int32_t)i; + return NULL; +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("multi.aot", "rb"); +#else + FILE* file = fopen("multi.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + fseek(file, 0L, SEEK_SET); + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Create external print functions. + printf("Creating callback...\n"); + wasm_valtype_t* types[4] = { + wasm_valtype_new_i32(), wasm_valtype_new_i64(), + wasm_valtype_new_i64(), wasm_valtype_new_i32() + }; + own wasm_valtype_vec_t tuple1, tuple2; + wasm_valtype_vec_new(&tuple1, 4, types); + wasm_valtype_vec_copy(&tuple2, &tuple1); + own wasm_functype_t* callback_type = wasm_functype_new(&tuple1, &tuple2); + own wasm_func_t* callback_func = + wasm_func_new(store, callback_type, callback); + + wasm_functype_delete(callback_type); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_t* externs[] = { wasm_func_as_extern(callback_func) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(callback_func); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return 1; + } + const wasm_func_t* run_func = wasm_extern_as_func(exports.data[0]); + if (run_func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + wasm_module_delete(module); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_t vals[4] = { + WASM_I32_VAL(1), WASM_I64_VAL(2), WASM_I64_VAL(3), WASM_I32_VAL(4) + }; + wasm_val_t res[4] = { + WASM_INIT_VAL, WASM_INIT_VAL, WASM_INIT_VAL, WASM_INIT_VAL + }; + wasm_val_vec_t args = WASM_ARRAY_VEC(vals); + wasm_val_vec_t results = WASM_ARRAY_VEC(res); + wasm_trap_t *trap = wasm_func_call(run_func, &args, &results); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Print result. + printf("Printing result...\n"); + printf("> %"PRIu32" %"PRIu64" %"PRIu64" %"PRIu32"\n", + res[0].of.i32, res[1].of.i64, res[2].of.i64, res[3].of.i32); + + assert(res[0].of.i32 == 4); + assert(res[1].of.i64 == 3); + assert(res[2].of.i64 == 2); + assert(res[3].of.i32 == 1); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/multi.wat b/wasm-micro-runtime/samples/wasm-c-api/src/multi.wat new file mode 100644 index 0000000..e7fb331 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/multi.wat @@ -0,0 +1,7 @@ +(module + (func $f (import "" "f") (param i32 i64 i64 i32) (result i32 i64 i64 i32)) + + (func $g (export "g") (param i32 i64 i64 i32) (result i32 i64 i64 i32) + (call $f (local.get 0) (local.get 2) (local.get 1) (local.get 3)) + ) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/reflect.c b/wasm-micro-runtime/samples/wasm-c-api/src/reflect.c new file mode 100644 index 0000000..a595fd3 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/reflect.c @@ -0,0 +1,207 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +void print_mutability(wasm_mutability_t mut) { + switch (mut) { + case WASM_VAR: printf("var"); break; + case WASM_CONST: printf("const"); break; + } +} + +void print_limits(const wasm_limits_t* limits) { + if (!limits) { + printf("unknown limits"); + return; + } + printf("%ud", limits->min); + if (limits->max < wasm_limits_max_default) printf(" %ud", limits->max); +} + +void print_valtype(const wasm_valtype_t* type) { + switch (wasm_valtype_kind(type)) { + case WASM_I32: printf("i32"); break; + case WASM_I64: printf("i64"); break; + case WASM_F32: printf("f32"); break; + case WASM_F64: printf("f64"); break; + case WASM_ANYREF: printf("anyref"); break; + case WASM_FUNCREF: printf("funcref"); break; + } +} + +void print_valtypes(const wasm_valtype_vec_t* types) { + bool first = true; + for (size_t i = 0; i < types->size; ++i) { + if (first) { + first = false; + } else { + printf(" "); + } + print_valtype(types->data[i]); + } +} + +void print_externtype(const wasm_externtype_t* type) { + if (!type) { + printf("unknown extern type"); + return; + } + switch (wasm_externtype_kind(type)) { + case WASM_EXTERN_FUNC: { + const wasm_functype_t* functype = + wasm_externtype_as_functype_const(type); + printf("func "); + print_valtypes(wasm_functype_params(functype)); + printf(" -> "); + print_valtypes(wasm_functype_results(functype)); + } break; + case WASM_EXTERN_GLOBAL: { + const wasm_globaltype_t* globaltype = + wasm_externtype_as_globaltype_const(type); + printf("global "); + print_mutability(wasm_globaltype_mutability(globaltype)); + printf(" "); + print_valtype(wasm_globaltype_content(globaltype)); + } break; + case WASM_EXTERN_TABLE: { + const wasm_tabletype_t* tabletype = + wasm_externtype_as_tabletype_const(type); + printf("table "); + print_limits(wasm_tabletype_limits(tabletype)); + printf(" "); + print_valtype(wasm_tabletype_element(tabletype)); + } break; + case WASM_EXTERN_MEMORY: { + const wasm_memorytype_t* memorytype = + wasm_externtype_as_memorytype_const(type); + printf("memory "); + print_limits(wasm_memorytype_limits(memorytype)); + } break; + } +} + +void print_name(const wasm_name_t* name) { + if (!name) { + printf("unknown name"); + return; + } + printf("\"%.*s\"", (int)name->size, name->data); +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("reflect.aot", "rb"); +#else + FILE* file = fopen("reflect.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + // Extract export. + printf("Extracting export...\n"); + own wasm_exporttype_vec_t export_types; + own wasm_extern_vec_t exports; + wasm_module_exports(module, &export_types); + wasm_instance_exports(instance, &exports); + assert(exports.size == export_types.size); + + for (size_t i = 0; i < exports.size; ++i) { + assert(wasm_extern_kind(exports.data[i]) == + wasm_externtype_kind(wasm_exporttype_type(export_types.data[i]))); + printf("> export %zu ", i); + print_name(wasm_exporttype_name(export_types.data[i])); + printf("\n"); + printf(">> initial: "); + print_externtype(wasm_exporttype_type(export_types.data[i])); + printf("\n"); + printf(">> current: "); + own wasm_externtype_t* current = wasm_extern_type(exports.data[i]); + print_externtype(current); + wasm_externtype_delete(current); + printf("\n"); + if (wasm_extern_kind(exports.data[i]) == WASM_EXTERN_FUNC) { + wasm_func_t* func = wasm_extern_as_func(exports.data[i]); + printf(">> in-arity: %zu", wasm_func_param_arity(func)); + printf(", out-arity: %zu\n", wasm_func_result_arity(func)); + } + } + + wasm_module_delete(module); + wasm_instance_delete(instance); + wasm_extern_vec_delete(&exports); + wasm_exporttype_vec_delete(&export_types); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/reflect.wat b/wasm-micro-runtime/samples/wasm-c-api/src/reflect.wat new file mode 100644 index 0000000..7a80e86 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/reflect.wat @@ -0,0 +1,6 @@ +(module + (func (export "func") (param i32 f64 f32) (result i32) (unreachable)) + (global (export "global") f64 (f64.const 0)) + (table (export "table") 0 50 funcref) + (memory (export "memory") 1) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/serialize.c b/wasm-micro-runtime/samples/wasm-c-api/src/serialize.c new file mode 100644 index 0000000..8343681 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/serialize.c @@ -0,0 +1,131 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// A function to be called from Wasm code. +own wasm_trap_t * +hello_callback(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + printf("Calling back...\n"); + printf("> Hello World!\n"); + return NULL; +} + +int +main(int argc, const char *argv[]) +{ + // Initialize. + printf("Initializing...\n"); + wasm_engine_t *engine = wasm_engine_new(); + wasm_store_t *store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); + FILE *file = fopen("serialize.wasm", "rb"); + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + fseek(file, 0L, SEEK_SET); + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t *module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Serialize module. + printf("Serializing module...\n"); + own wasm_byte_vec_t serialized; + wasm_module_serialize(module, &serialized); + + wasm_module_delete(module); + + // Deserialize module. + printf("Deserializing module...\n"); + own wasm_module_t *deserialized = + wasm_module_deserialize(store, &serialized); + if (!deserialized) { + printf("> Error deserializing module!\n"); + return 1; + } + + wasm_byte_vec_delete(&serialized); + + // Create external print functions. + printf("Creating callback...\n"); + own wasm_functype_t *hello_type = wasm_functype_new_0_0(); + own wasm_func_t *hello_func = + wasm_func_new(store, hello_type, hello_callback); + + wasm_functype_delete(hello_type); + + // Instantiate. + printf("Instantiating deserialized module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(hello_func) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t *instance = + wasm_instance_new(store, deserialized, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(hello_func); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return 1; + } + const wasm_func_t *run_func = wasm_extern_as_func(exports.data[0]); + if (run_func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + wasm_module_delete(deserialized); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_vec_t empty = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(run_func, &empty, &empty); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/serialize.wat b/wasm-micro-runtime/samples/wasm-c-api/src/serialize.wat new file mode 100644 index 0000000..1c56c55 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/serialize.wat @@ -0,0 +1,4 @@ +(module + (func $hello (import "" "hello")) + (func (export "run") (call $hello)) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/table.c b/wasm-micro-runtime/samples/wasm-c-api/src/table.c new file mode 100644 index 0000000..c4a7da8 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/table.c @@ -0,0 +1,218 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// A function to be called from Wasm code. +own wasm_trap_t* neg_callback( + const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + printf("Calling back...\n"); + results->data[0].kind = WASM_I32; + results->data[0].of.i32 = -args->data[0].of.i32; + return NULL; +} + + +wasm_table_t* get_export_table(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_table(exports->data[i])) { + printf("> Error accessing table export %zu!\n", i); + exit(1); + } + return wasm_extern_as_table(exports->data[i]); +} + +wasm_func_t* get_export_func(const wasm_extern_vec_t* exports, size_t i) { + if (exports->size <= i || !wasm_extern_as_func(exports->data[i])) { + printf("> Error accessing function export %zu!\n", i); + exit(1); + } + return wasm_extern_as_func(exports->data[i]); +} + + +void check(bool success) { + if (!success) { + printf("> Error, expected success\n"); + exit(1); + } +} + +void check_table(wasm_table_t* table, int32_t i, bool expect_set) { + own wasm_ref_t* ref = wasm_table_get(table, i); + check((ref != NULL) == expect_set); + if (ref) wasm_ref_delete(ref); +} + +void check_call(wasm_func_t* func, int32_t arg1, int32_t arg2, int32_t expected) { + wasm_val_t vs[2] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + wasm_val_t r[1] = { WASM_INIT_VAL }; + wasm_val_vec_t args = WASM_ARRAY_VEC(vs); + wasm_val_vec_t results = WASM_ARRAY_VEC(r); + wasm_trap_t *trap = wasm_func_call(func, &args, &results); + if (trap) { + printf("> Error on calling\n"); + wasm_trap_delete(trap); + exit(1); + } + + if (r[0].of.i32 != expected) { + printf("> Error on result\n"); + exit(1); + } +} + +void check_trap(wasm_func_t* func, int32_t arg1, int32_t arg2) { + wasm_val_t vs[2] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + wasm_val_t r[1] = { WASM_INIT_VAL }; + wasm_val_vec_t args = WASM_ARRAY_VEC(vs); + wasm_val_vec_t results = WASM_ARRAY_VEC(r); + own wasm_trap_t* trap = wasm_func_call(func, &args, &results); + if (! trap) { + printf("> Error on result, expected trap\n"); + exit(1); + } + wasm_trap_delete(trap); +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("table.aot", "rb"); +#else + FILE* file = fopen("table.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + // Extract export. + printf("Extracting exports...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + size_t i = 0; + wasm_table_t* table = get_export_table(&exports, i++); + wasm_func_t* call_indirect = get_export_func(&exports, i++); + wasm_func_t* f = get_export_func(&exports, i++); + wasm_func_t* g = get_export_func(&exports, i++); + + wasm_module_delete(module); + + // Create external function. + printf("Creating callback...\n"); + own wasm_functype_t* neg_type = wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32()); + own wasm_func_t* h = wasm_func_new(store, neg_type, neg_callback); + + wasm_functype_delete(neg_type); + + // Try cloning. + own wasm_table_t* copy = wasm_table_copy(table); + assert(wasm_table_same(table, copy)); + wasm_table_delete(copy); + + // Check initial table. + printf("Checking table...\n"); + check(wasm_table_size(table) == 2); + check_table(table, 0, false); + check_table(table, 1, true); + check_trap(call_indirect, 0, 0); + check_call(call_indirect, 7, 1, 7); + check_trap(call_indirect, 0, 2); + + // Mutate table. + printf("Mutating table...\n"); + check(wasm_table_set(table, 0, wasm_func_as_ref(g))); + check(wasm_table_set(table, 1, NULL)); + wasm_ref_t *ref_f = wasm_func_as_ref(f); + check(! wasm_table_set(table, 2, ref_f)); + wasm_ref_delete(ref_f); + check_table(table, 0, true); + check_table(table, 1, false); + check_call(call_indirect, 7, 0, 666); + check_trap(call_indirect, 0, 1); + check_trap(call_indirect, 0, 2); + + // Grow table. + // DO NOT SUPPORT + printf("Bypass Growing table...\n"); + + wasm_func_delete(h); + wasm_extern_vec_delete(&exports); + wasm_instance_delete(instance); + + // Create stand-alone table. + // DO NOT SUPPORT + // TODO(wasm+): Once Wasm allows multiple tables, turn this into import. + printf("Bypass Creating stand-alone table...\n"); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/table.wat b/wasm-micro-runtime/samples/wasm-c-api/src/table.wat new file mode 100644 index 0000000..d3e3a94 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/table.wat @@ -0,0 +1,12 @@ +(module + (table (export "table") 2 10 funcref) + + (func (export "call_indirect") (param i32 i32) (result i32) + (call_indirect (param i32) (result i32) (local.get 0) (local.get 1)) + ) + + (func $f (export "f") (param i32) (result i32) (local.get 0)) + (func (export "g") (param i32) (result i32) (i32.const 666)) + + (elem (i32.const 1) $f) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/threads.c b/wasm-micro-runtime/samples/wasm-c-api/src/threads.c new file mode 100644 index 0000000..cbe4aaf --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/threads.c @@ -0,0 +1,187 @@ +#include +#include +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +const int N_THREADS = 10; +const int N_REPS = 3; + +// A function to be called from Wasm code. +own wasm_trap_t * +callback(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + assert(args->data[0].kind == WASM_I32); + printf("> Thread %d running\n", args->data[0].of.i32); + return NULL; +} + +typedef struct { + wasm_engine_t *engine; + wasm_shared_module_t *module; + int id; +} thread_args; + +void * +run(void *args_abs) +{ + thread_args *args = (thread_args *)args_abs; + + // Rereate store and module. + own wasm_store_t *store = wasm_store_new(args->engine); + own wasm_module_t *module = wasm_module_obtain(store, args->module); + + // Run the example N times. + for (int i = 0; i < N_REPS; ++i) { + usleep(100000); + + // Create imports. + own wasm_functype_t *func_type = + wasm_functype_new_1_0(wasm_valtype_new_i32()); + own wasm_func_t *func = wasm_func_new(store, func_type, callback); + wasm_functype_delete(func_type); + + wasm_val_t val = WASM_I32_VAL((int32_t)args->id); + own wasm_globaltype_t *global_type = + wasm_globaltype_new(wasm_valtype_new_i32(), WASM_CONST); + own wasm_global_t *global = wasm_global_new(store, global_type, &val); + wasm_globaltype_delete(global_type); + + // Instantiate. + wasm_extern_t *externs[] = { + wasm_func_as_extern(func), + wasm_global_as_extern(global), + }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t *instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return NULL; + } + + wasm_func_delete(func); + wasm_global_delete(global); + + // Extract export. + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return NULL; + } + const wasm_func_t *run_func = wasm_extern_as_func(exports.data[0]); + if (run_func == NULL) { + printf("> Error accessing export!\n"); + return NULL; + } + + wasm_instance_delete(instance); + + // Call. + wasm_val_vec_t empty = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(run_func, &empty, &empty); + if (trap) { + printf("> Error calling function!\n"); + wasm_trap_delete(trap); + return NULL; + } + + wasm_extern_vec_delete(&exports); + } + + wasm_module_delete(module); + wasm_store_delete(store); + + free(args_abs); + + return NULL; +} + +int +main(int argc, const char *argv[]) +{ + // Initialize. + wasm_engine_t *engine = wasm_engine_new(); + + // Load binary. +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE *file = fopen("threads.aot", "rb"); +#else + FILE *file = fopen("threads.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + return 1; + } + fclose(file); + + // Compile and share. + own wasm_store_t *store = wasm_store_new(engine); + own wasm_module_t *module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + own wasm_shared_module_t *shared = wasm_module_share(module); + + wasm_module_delete(module); + wasm_store_delete(store); + + // Spawn threads. + pthread_t threads[N_THREADS]; + for (int i = 0; i < N_THREADS; i++) { + thread_args *args = malloc(sizeof(thread_args)); + args->id = i; + args->engine = engine; + args->module = shared; + printf("Initializing thread %d...\n", i); + pthread_create(&threads[i], NULL, &run, args); + } + + for (int i = 0; i < N_THREADS; i++) { + printf("Waiting for thread: %d\n", i); + pthread_join(threads[i], NULL); + } + + wasm_shared_module_delete(shared); + wasm_engine_delete(engine); + + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/threads.wat b/wasm-micro-runtime/samples/wasm-c-api/src/threads.wat new file mode 100644 index 0000000..29a3bbc --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/threads.wat @@ -0,0 +1,5 @@ +(module + (func $message (import "" "hello") (param i32)) + (global $id (import "" "id") i32) + (func (export "run") (call $message (global.get $id))) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/trap.c b/wasm-micro-runtime/samples/wasm-c-api/src/trap.c new file mode 100644 index 0000000..2637b73 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/trap.c @@ -0,0 +1,184 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// A function to be called from Wasm code. +own wasm_trap_t* fail_callback( + void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results +) { + printf("Calling back...\n"); + own wasm_name_t message; + wasm_name_new_from_string_nt(&message, "callback abort"); + own wasm_trap_t* trap = wasm_trap_new((wasm_store_t*)env, &message); + wasm_name_delete(&message); + return trap; +} + + +void print_frame(wasm_frame_t* frame) { + printf("> %p @ 0x%zx = %"PRIu32".0x%zx\n", + wasm_frame_instance(frame), + wasm_frame_module_offset(frame), + wasm_frame_func_index(frame), + wasm_frame_func_offset(frame) + ); +} + + +int main(int argc, const char* argv[]) { + // Initialize. + printf("Initializing...\n"); + wasm_engine_t* engine = wasm_engine_new(); + wasm_store_t* store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("trap.aot", "rb"); +#else + FILE* file = fopen("trap.wasm", "rb"); +#endif + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + + int ret = fseek(file, 0L, SEEK_END); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + ret = fseek(file, 0L, SEEK_SET); + if (ret == -1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + fclose(file); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t* module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Create external print functions. + printf("Creating callback...\n"); + own wasm_functype_t* fail_type = + wasm_functype_new_0_1(wasm_valtype_new_i32()); + own wasm_func_t* fail_func = + wasm_func_new_with_env(store, fail_type, fail_callback, store, NULL); + + wasm_functype_delete(fail_type); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_t* externs[] = { wasm_func_as_extern(fail_func) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t* instance = + wasm_instance_new(store, module, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(fail_func); + + // Extract export. + printf("Extracting exports...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size < 2) { + printf("> Error accessing exports!\n"); + return 1; + } + + wasm_module_delete(module); + wasm_instance_delete(instance); + + // Call. + for (int i = 0; i < 2; ++i) { + const wasm_func_t* func = wasm_extern_as_func(exports.data[i]); + if (func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + printf("Calling export %d...\n", i); + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_EMPTY_VEC; + own wasm_trap_t* trap = wasm_func_call(func, &args, &results); + if (!trap) { + printf("> Error calling function, expected trap!\n"); + return 1; + } + + printf("Printing message...\n"); + own wasm_name_t message; + wasm_trap_message(trap, &message); + printf("> %s\n", message.data); + assert(message.num_elems > 0); + assert(strncmp(message.data, "Exception: ", strlen("Exception: ")) == 0); + + printf("Printing origin...\n"); + own wasm_frame_t* frame = wasm_trap_origin(trap); + if (frame) { + print_frame(frame); + wasm_frame_delete(frame); + } else { + printf("> Empty origin.\n"); + } + + printf("Printing trace...\n"); + own wasm_frame_vec_t trace; + wasm_trap_trace(trap, &trace); + if (trace.size > 0) { + for (size_t i = 0; i < trace.size; ++i) { + print_frame(trace.data[i]); + } + } else { + printf("> Empty trace.\n"); + } + + wasm_frame_vec_delete(&trace); + wasm_trap_delete(trap); + wasm_name_delete(&message); + } + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/trap.wat b/wasm-micro-runtime/samples/wasm-c-api/src/trap.wat new file mode 100644 index 0000000..dfd20fb --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/trap.wat @@ -0,0 +1,5 @@ +(module + (func $callback (import "" "callback") (result i32)) + (func (export "callback") (result i32) (call $callback)) + (func (export "unreachable") (result i32) (unreachable) (i32.const 1)) +) diff --git a/wasm-micro-runtime/samples/wasm-c-api/src/utils/multi_module_utils.c b/wasm-micro-runtime/samples/wasm-c-api/src/utils/multi_module_utils.c new file mode 100644 index 0000000..6701091 --- /dev/null +++ b/wasm-micro-runtime/samples/wasm-c-api/src/utils/multi_module_utils.c @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_read_file.h" +#include "wasm_export.h" + +static char * +build_module_path(const char *module_name) +{ + const char *module_search_path = "."; + const char *format = "%s/%s.wasm"; + int sz = strlen(module_search_path) + strlen("/") + strlen(module_name) + + strlen(".wasm") + 1; + char *wasm_file_name = BH_MALLOC(sz); + if (!wasm_file_name) { + return NULL; + } + + snprintf(wasm_file_name, sz, format, module_search_path, module_name); + return wasm_file_name; +} + +bool +reader(const char *module_name, uint8 **p_buffer, uint32 *p_size) +{ + char *wasm_file_path = build_module_path(module_name); + if (!wasm_file_path) { + return false; + } + + *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_path, p_size); + BH_FREE(wasm_file_path); + return *p_buffer != NULL; +} + +void +destroyer(uint8 *buffer, uint32 size) +{ + if (!buffer) { + return; + } + + BH_FREE(buffer); + buffer = NULL; +} diff --git a/wasm-micro-runtime/samples/workload/CMakeLists.txt b/wasm-micro-runtime/samples/workload/CMakeLists.txt new file mode 100644 index 0000000..667f0b4 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/CMakeLists.txt @@ -0,0 +1,116 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(wasm_workloads) + +####################################### +add_subdirectory(bwa) +add_subdirectory(meshoptimizer) +add_subdirectory(wasm-av1) + +####################################### +include(ExternalProject) + +################ iwasm ################ +ExternalProject_Add(iwasm + PREFIX + iwasm-build + BUILD_ALWAYS + YES + SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux -B build -DWAMR_BUILD_LIBC_EMCC=1 + BUILD_COMMAND + ${CMAKE_COMMAND} --build build --parallel 4 + INSTALL_COMMAND + # FIXME: replace with --install + ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/iwasm-build/src/iwasm-build/build/iwasm + ${CMAKE_CURRENT_BINARY_DIR}/iwasm +) + +################ wamrc ################ +ExternalProject_Add(wamrc + PREFIX + wamrc-build + BUILD_ALWAYS + YES + SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler -B build + BUILD_COMMAND + ${CMAKE_COMMAND} --build build --parallel 4 + INSTALL_COMMAND + # FIXME: replace with --install + ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/wamrc-build/src/wamrc-build/build/wamrc + ${CMAKE_CURRENT_BINARY_DIR}/wamrc +) + +################ .aot ################ +add_custom_target( + bwa_to_aot + ALL + DEPENDS + bwa wamrc + COMMAND + ./wamrc -o bwa.aot ./bwa/bwa.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_custom_target( + codecbench_to_aot + ALL + DEPENDS + codecbench wamrc + COMMAND + ./wamrc -o codecbench.aot ./meshoptimizer/codecbench.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_custom_target( + av1_to_aot + ALL + DEPENDS + av1 wamrc + COMMAND + ./wamrc -o testavx.aot ./wasm-av1/testavx.opt.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +################ smoking test ################ +include(CTest) + +add_test( + NAME + run_bwa + COMMAND + ./iwasm --dir=. ./bwa.aot index ./bwa/hs38DH-extra.fa + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_test( + NAME + run_codecbench + COMMAND + ./iwasm codecbench.aot + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_test( + NAME + run_av1 + COMMAND + ./iwasm --dir=. testavx.aot ./wasm-av1/elephants_dream_480p24.ivf + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/wasm-micro-runtime/samples/workload/README.md b/wasm-micro-runtime/samples/workload/README.md new file mode 100644 index 0000000..0e6a3a4 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/README.md @@ -0,0 +1,34 @@ +All workloads have similar requirment of software dependencies, including **emsdk** and **binaryen** + +> There might be slight differences when using MacOS and other Linux distro than Ubuntu. This document targets +Ubuntu 20.04 as an example. + +## Installation instructions + +use [preparation.sh](./preparation.sh) to install all dependencies before compiling any workload. Or use [*vscode DevContainer*](../../.devcontainer/) + +The script installs below software: + +- **emsdk**. Refer to [the guide](https://emscripten.org/docs/getting_started/downloads.html). Don't forget to activate + emsdk and set up environment variables. Verify it with `echo ${EMSDK}`. Please be sure to install and activate the building + of 3.0.0 + +``` bash +$ cd /opt +$ git clone https://github.com/emscripten-core/emsdk.git +$ cd emsdk +$ git pull +$ ./emsdk install 3.0.0 +$ ./emsdk activate 3.0.0 +$ echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc +``` + +- **binaryen**. Install + [latest release](https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-linux.tar.gz) + to */opt/binaryen* + +``` bash +$ wget https://github.com/WebAssembly/binaryen/releases/download/${BINARYEN_VER}/${BINARYEN_FILE} +$ tar zxf ${BINARYEN_FILE} -C /opt +$ ln -sf /opt/binaryen-${BINARYEN_VER} /opt/binaryen +``` diff --git a/wasm-micro-runtime/samples/workload/XNNPACK/.gitignore b/wasm-micro-runtime/samples/workload/XNNPACK/.gitignore new file mode 100644 index 0000000..09f3fe9 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/XNNPACK/.gitignore @@ -0,0 +1,2 @@ +xnnpack +build diff --git a/wasm-micro-runtime/samples/workload/XNNPACK/CMakeLists.txt b/wasm-micro-runtime/samples/workload/XNNPACK/CMakeLists.txt new file mode 100644 index 0000000..41cdd58 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/XNNPACK/CMakeLists.txt @@ -0,0 +1,198 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(xnnpack_wasm) + +################ EMCC ################ +include(ExternalProject) + +# grep xnnpack_benchmark -A 1 BUILD.bazel \ +# | grep "name =" \ +# | awk '{print $3}' \ +# | sed -e 's/\"//g; s/,//g; s/^/\"/g; s/$/\"/g' +list(APPEND NATIVE_BENCHMARKS + "qs8_dwconv_bench" + "qs8_f32_vcvt_bench" + "qs8_gemm_bench" + "qs8_requantization_bench" + "qs8_vadd_bench" + "qs8_vaddc_bench" + "qs8_vcvt_bench" + "qs16_qs8_vcvt_bench" + "qs8_vlrelu_bench" + "qs8_vmul_bench" + "qs8_vmulc_bench" + "qu8_f32_vcvt_bench" + "qu8_gemm_bench" + "qu8_requantization_bench" + "qu8_vadd_bench" + "qu8_vaddc_bench" + "qu8_vcvt_bench" + "qu8_vlrelu_bench" + "qu8_vmul_bench" + "qu8_vmulc_bench" + "bf16_gemm_bench" + "f16_f32acc_igemm_bench" + "f16_igemm_bench" + "f16_f32acc_gemm_bench" + "f16_gemm_bench" + "f16_raddstoreexpminusmax_bench" + "f16_spmm_bench" + "f16_vsigmoid_bench" + "f16_vtanh_bench" + "f16_f32_vcvt_bench" + "f32_igemm_bench" + "f32_conv_hwc_bench" + "f16_conv_hwc2chw_bench" + "f16_gavgpool_cw_bench" + "f32_gavgpool_cw_bench" + "f32_conv_hwc2chw_bench" + "f16_dwconv_bench" + "f32_dwconv_bench" + "f32_dwconv2d_chw_bench" + "f16_dwconv2d_chw_bench" + "f32_f16_vcvt_bench" + "xx_transpose_bench" + "x8_transpose_bench" + "x16_transpose_bench" + "x24_transpose_bench" + "x32_transpose_bench" + "x64_transpose_bench" + "f32_bgemm_bench" + "f32_gemm_bench" + "f32_qs8_vcvt_bench" + "f32_qu8_vcvt_bench" + "f32_raddexpminusmax_bench" + "f32_raddextexp_bench" + "f32_raddstoreexpminusmax_bench" + "f32_rmax_bench" + "f32_spmm_bench" + "f32_softmax_bench" + "f16_velu_bench" + "f32_velu_bench" + "f32_vhswish_bench" + "f32_vlrelu_bench" + "f32_vrelu_bench" + "f32_vscaleexpminusmax_bench" + "f32_vscaleextexp_bench" + "f32_vsigmoid_bench" + "f16_vsqrt_bench" + "f32_vsqrt_bench" + "f32_vtanh_bench" + "f32_im2col_gemm_bench" + "rounding_bench" + "s16_rmaxabs_bench" + "s16_window_bench" + "u32_filterbank_accumulate_bench" + "u32_filterbank_subtract_bench" + "u32_vlog_bench" + "u64_u32_vsqrtshift_bench" + "i16_vlshift_bench" + "cs16_vsquareabs_bench" + "cs16_bfly4_bench" + "cs16_fftr_bench" + "x8_lut_bench" + "x32_packw_bench" + "x16_packw_bench" + "abs_bench" + "average_pooling_bench" + "bankers_rounding_bench" + "ceiling_bench" + "channel_shuffle_bench" + "convert_bench" + "convolution_bench" + "deconvolution_bench" + "elu_bench" + "floor_bench" + "global_average_pooling_bench" + "hardswish_bench" + "leaky_relu_bench" + "max_pooling_bench" + "negate_bench" + "prelu_bench" + "sigmoid_bench" + "softmax_bench" + "square_bench" + "square_root_bench" + "tanh_bench" + "truncation_bench" + "f16_dwconv_e2e_bench" + "f16_gemm_e2e_bench" + "f32_dwconv_e2e_bench" + "f32_gemm_e2e_bench" + "qs8_dwconv_e2e_bench" + "qs8_gemm_e2e_bench" + "qu8_gemm_e2e_bench" + "qu8_dwconv_e2e_bench" + "end2end_bench" + "f16_exp_ulp_eval" + "f16_expminus_ulp_eval" + "f16_expm1minus_ulp_eval" + "f16_sigmoid_ulp_eval" + "f16_sqrt_ulp_eval" + "f16_tanh_ulp_eval" + "f32_exp_ulp_eval" + "f32_expminus_ulp_eval" + "f32_expm1minus_ulp_eval" + "f32_extexp_ulp_eval" + "f32_sigmoid_ulp_eval" + "f32_sqrt_ulp_eval" + "f32_tanh_ulp_eval" +) + +# Only Download +ExternalProject_Add(xnnpack-download + PREFIX xnnpack + GIT_REPOSITORY https://github.com/google/XNNPACK.git + GIT_TAG b9d4073a6913891ce9cbd8965c8d506075d2a45a + GIT_PROGRESS ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack + UPDATE_COMMAND "" + PATCH_COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack.patch + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) + +set(WAMRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../wamr-compiler/build/wamrc") +if(EXISTS ${WAMRC}) + message("-- Will generate .aot") +else() + message("Will generate .wasm") +endif() + +foreach(BENCHMARK IN LISTS NATIVE_BENCHMARKS) + string(CONCAT WASM_BENCHMARK "//:" ${BENCHMARK} "-wasm") + string(CONCAT WASM_OUTPUT ${BENCHMARK} ".wasm") + + add_custom_command( + OUTPUT ${WASM_OUTPUT} + COMMAND bazel --output_user_root=build-user-output build -c opt --config=wasm ${WASM_BENCHMARK} + && ${CMAKE_COMMAND} -E copy_if_different ./bazel-bin/${WASM_OUTPUT} ${CMAKE_CURRENT_BINARY_DIR}/${WASM_OUTPUT} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack + DEPENDS xnnpack-download + COMMENT "Generating ${WASM_OUTPUT} ..." + ) + + set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_CLEAN_FILES ${CMAKE_CURRENT_BINARY_DIR}/${WASM_OUTPUT}) + + if(EXISTS ${WAMRC}) + string(CONCAT AOT_OUTPUT ${BENCHMARK} ".aot") + + add_custom_command( + OUTPUT ${AOT_OUTPUT} + COMMAND ${WAMRC} -o ${AOT_OUTPUT} ${WASM_OUTPUT} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${WASM_OUTPUT} + COMMENT "Generating ${AOT_OUTPUT} ..." + ) + + add_custom_target(${BENCHMARK} ALL DEPENDS ${AOT_OUTPUT}) + else() + add_custom_target(${BENCHMARK} ALL DEPENDS ${WASM_OUTPUT}) + endif() +endforeach() + diff --git a/wasm-micro-runtime/samples/workload/XNNPACK/README.md b/wasm-micro-runtime/samples/workload/XNNPACK/README.md new file mode 100644 index 0000000..625ef7f --- /dev/null +++ b/wasm-micro-runtime/samples/workload/XNNPACK/README.md @@ -0,0 +1,49 @@ +"XNNPACK" sample introduction +============== + +This sample demonstrates how to build [XNNPACK](https://github.com/google/XNNPACK) benchmarks into WebAssembly with emsdk toolchain and run them with iwasm. + +## Installation toolchains + +please refer to [installation instructions](../README.md). + +## Build XNNPACK + +please build wamrc: + +``` bash +cd /wamr-compiler +./build_llvm.sh +mkdir build && cd build +cmake .. +make +``` + +And then build xnnpack standalone wasm files + +```bash +$ cd /samples/workload/XNNPACK +$ cmake -S . -B build +$ cmake --build build +``` + +Generated .wasm(and .aot) files are under *samples/workload/XNNPACK/build*. + +## Run benchmarks + +Firstly please build iwasm with simd, libc-emcc and lib-pthread supporting: + +``` bash +$ cd /product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1 -DWAMR_BUILD_LIB_PTHREAD=1 +$ make +``` + +Then run: + +``` shell +$ cd /samples/workload/XNNPACK/build +$ iwasm average_pooling_bench.aot # (or other aot files) +``` + diff --git a/wasm-micro-runtime/samples/workload/XNNPACK/benchmark.patch b/wasm-micro-runtime/samples/workload/XNNPACK/benchmark.patch new file mode 100644 index 0000000..713b476 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/XNNPACK/benchmark.patch @@ -0,0 +1,14 @@ +diff --git include/benchmark/benchmark.h include/benchmark/benchmark.h +index 9b54802..baa5938 100755 +--- include/benchmark/benchmark.h ++++ include/benchmark/benchmark.h +@@ -364,7 +364,9 @@ template + inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + internal::UseCharPointer(&reinterpret_cast(value)); + } ++ + // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers ++inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { } + #endif + + // This class is used for user-defined counters. diff --git a/wasm-micro-runtime/samples/workload/XNNPACK/xnnpack.patch b/wasm-micro-runtime/samples/workload/XNNPACK/xnnpack.patch new file mode 100644 index 0000000..d7680d3 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/XNNPACK/xnnpack.patch @@ -0,0 +1,138 @@ +diff --git a/.bazelrc b/.bazelrc +index fcaff1063..e61d53337 100644 +--- a/.bazelrc ++++ b/.bazelrc +@@ -1,6 +1,7 @@ + # Basic build settings + build --jobs 128 + build --cxxopt='-std=gnu++14' ++build --incompatible_enable_cc_toolchain_resolution + + # Sets the default Apple platform to macOS. + build --apple_platform_type=macos +@@ -55,3 +56,10 @@ build:macos --apple_platform_type=macos + + build:macos_arm64 --config=macos + build:macos_arm64 --cpu=darwin_arm64 ++ ++# Emscripten configs ++build:wasm --copt="-Wno-unused" ++build:wasm --copt="-Wno-unused-function" ++build:wasm --copt="-Wno-unused-but-set-variable" ++build:wasm --cpu=wasm ++build:wasm --features=wasm_simd +diff --git a/WORKSPACE b/WORKSPACE +index 2e568088b..3961371ca 100644 +--- a/WORKSPACE ++++ b/WORKSPACE +@@ -83,7 +83,23 @@ http_archive( + ) + + # Android NDK location and version is auto-detected from $ANDROID_NDK_HOME environment variable +-android_ndk_repository(name = "androidndk") ++# android_ndk_repository(name = "androidndk") + + # Android SDK location and API is auto-detected from $ANDROID_HOME environment variable +-android_sdk_repository(name = "androidsdk") ++# android_sdk_repository(name = "androidsdk") ++ ++http_archive( ++ name = "emsdk", ++ sha256 = "5fa6f5eb45a4d50264610c4c9e1c155535359b63bfaad69b4e5101d16c1e7e32", ++ strip_prefix = "emsdk-a896e3d066448b3530dbcaa48869fafefd738f57/bazel", ++ url = "https://github.com/emscripten-core/emsdk/archive/a896e3d066448b3530dbcaa48869fafefd738f57.tar.gz", ++) ++ ++load("@emsdk//:deps.bzl", emsdk_deps = "deps") ++emsdk_deps() ++ ++load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps") ++emsdk_emscripten_deps(emscripten_version = "3.1.44") ++ ++load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") ++register_emscripten_toolchains() +diff --git a/bench/utils.cc b/bench/utils.cc +index 3b32503a7..656845336 100644 +--- a/bench/utils.cc ++++ b/bench/utils.cc +@@ -456,3 +456,13 @@ CodeMemoryHelper::~CodeMemoryHelper() { + + } // namespace utils + } // namespace benchmark ++ ++ ++extern "C" ++__attribute__((import_module("env"), import_name("getentropy"))) int import_getentropy(void* buffer, size_t length); ++ ++extern "C" ++int getentropy(void* buffer, size_t length) ++{ ++ return import_getentropy(buffer, length); ++} +diff --git a/build_defs.bzl b/build_defs.bzl +index 01b436eb7..2738fd50a 100644 +--- a/build_defs.bzl ++++ b/build_defs.bzl +@@ -1,6 +1,7 @@ + """Build definitions and rules for XNNPACK.""" + +-load(":emscripten.bzl", "xnnpack_emscripten_benchmark_linkopts", "xnnpack_emscripten_deps", "xnnpack_emscripten_minimal_linkopts", "xnnpack_emscripten_test_linkopts") ++load(":emscripten.bzl", "xnnpack_emscripten_benchmark_linkopts", "xnnpack_emscripten_deps", "xnnpack_emscripten_minimal_linkopts", "xnnpack_emscripten_test_linkopts", "xnnpack_emscripten_standalone_benchmark_linkopts") ++load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") + + def xnnpack_visibility(): + """Visibility of :XNNPACK target. +@@ -393,7 +394,8 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []): + "//conditions:default": ["-Wno-unused-function"], + }) + copts, + linkopts = select({ +- ":emscripten": xnnpack_emscripten_benchmark_linkopts(), ++ ":emscripten": xnnpack_emscripten_standalone_benchmark_linkopts(), ++ ":emscripten_wasmsimd": xnnpack_emscripten_standalone_benchmark_linkopts(), + ":windows_x86_64_mingw": ["-lshlwapi"], + ":windows_x86_64_msys": ["-lshlwapi"], + "//conditions:default": [], +@@ -405,5 +407,16 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []): + ":emscripten": xnnpack_emscripten_deps(), + "//conditions:default": [], + }), +- tags = tags, ++ tags = tags, ++ ) ++ ++ wasm_cc_binary( ++ name = name + "-wasm", ++ cc_target = ":" + name, ++ threads = "off", ++ simd = True, ++ standalone= True, ++ outputs = [ ++ name + ".wasm", ++ ] + ) +diff --git a/emscripten.bzl b/emscripten.bzl +index f1557a7b1..a3c4f93b9 100644 +--- a/emscripten.bzl ++++ b/emscripten.bzl +@@ -33,6 +33,21 @@ def xnnpack_emscripten_benchmark_linkopts(): + "--pre-js $(location :preamble.js.lds)", + ] + ++def xnnpack_emscripten_standalone_benchmark_linkopts(): ++ return [ ++ "-s ASSERTIONS=1", ++ "-s ERROR_ON_UNDEFINED_SYMBOLS=0", ++ "-s ALLOW_MEMORY_GROWTH=1", ++ "-s TOTAL_MEMORY=536870912", # 512M ++ "-s USE_PTHREADS=0", ++ "-s STANDALONE_WASM=1", ++ "-Wl,--export=__heap_base", ++ "-Wl,--export=__data_end", ++ "-Wl,--export=malloc", ++ "-Wl,--export=free", ++ ] ++ ++ + def xnnpack_emscripten_deps(): + """Emscripten-specific dependencies for unit tests and benchmarks.""" + return [ diff --git a/wasm-micro-runtime/samples/workload/bwa/.gitignore b/wasm-micro-runtime/samples/workload/bwa/.gitignore new file mode 100644 index 0000000..cd72095 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/bwa/.gitignore @@ -0,0 +1,4 @@ +build +libz +bwa +include diff --git a/wasm-micro-runtime/samples/workload/bwa/CMakeLists.bwa_wasm.txt b/wasm-micro-runtime/samples/workload/bwa/CMakeLists.bwa_wasm.txt new file mode 100644 index 0000000..a8d1d88 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/bwa/CMakeLists.bwa_wasm.txt @@ -0,0 +1,124 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(bwa_wasm C) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake) + +################ dependencies ################ +find_package(Binaryen 111 REQUIRED) + +################ LIBZ ################ +set(LIBZ_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../libz) +add_library(z_wasm STATIC + ${LIBZ_SRC_DIR}/adler32.c + ${LIBZ_SRC_DIR}/compress.c + ${LIBZ_SRC_DIR}/crc32.c + ${LIBZ_SRC_DIR}/deflate.c + ${LIBZ_SRC_DIR}/gzclose.c + ${LIBZ_SRC_DIR}/gzlib.c + ${LIBZ_SRC_DIR}/gzread.c + ${LIBZ_SRC_DIR}/gzwrite.c + ${LIBZ_SRC_DIR}/infback.c + ${LIBZ_SRC_DIR}/inffast.c + ${LIBZ_SRC_DIR}/inflate.c + ${LIBZ_SRC_DIR}/inftrees.c + ${LIBZ_SRC_DIR}/trees.c + ${LIBZ_SRC_DIR}/uncompr.c + ${LIBZ_SRC_DIR}/zutil.c +) + +set_target_properties(z_wasm PROPERTIES LINKER_LANGUAGE C) + +target_compile_definitions(z_wasm PRIVATE Z_HAVE_UNISTD_H _LARGEFILE64_SOURCE=1) + +target_compile_options(z_wasm + PRIVATE + -Wno-unused-function + -Wno-unused-variable +) + +target_include_directories(z_wasm + PUBLIC + ${LIBZ_SRC_DIR} +) + +################ BWA_WASM ################ +set(BWA_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(BWA_SOURCE + ${BWA_SRC_DIR}/utils.c + ${BWA_SRC_DIR}/kthread.c + ${BWA_SRC_DIR}/kstring.c + ${BWA_SRC_DIR}/ksw.c + ${BWA_SRC_DIR}/bwt.c + ${BWA_SRC_DIR}/bntseq.c + ${BWA_SRC_DIR}/bwa.c + ${BWA_SRC_DIR}/bwamem.c + ${BWA_SRC_DIR}/bwamem_pair.c + ${BWA_SRC_DIR}/bwamem_extra.c + ${BWA_SRC_DIR}/malloc_wrap.c + ${BWA_SRC_DIR}/QSufSort.c + ${BWA_SRC_DIR}/bwt_gen.c + ${BWA_SRC_DIR}/rope.c + ${BWA_SRC_DIR}/rle.c + ${BWA_SRC_DIR}/is.c + ${BWA_SRC_DIR}/bwtindex.c + ${BWA_SRC_DIR}/bwashm.c + ${BWA_SRC_DIR}/bwase.c + ${BWA_SRC_DIR}/bwaseqio.c + ${BWA_SRC_DIR}/bwtgap.c + ${BWA_SRC_DIR}/bwtaln.c + ${BWA_SRC_DIR}/bamlite.c + ${BWA_SRC_DIR}/bwape.c + ${BWA_SRC_DIR}/kopen.c + ${BWA_SRC_DIR}/pemerge.c + ${BWA_SRC_DIR}/maxk.c + ${BWA_SRC_DIR}/bwtsw2_core.c + ${BWA_SRC_DIR}/bwtsw2_main.c + ${BWA_SRC_DIR}/bwtsw2_aux.c + ${BWA_SRC_DIR}/bwt_lite.c + ${BWA_SRC_DIR}/bwtsw2_chain.c + ${BWA_SRC_DIR}/fastmap.c + ${BWA_SRC_DIR}/bwtsw2_pair.c + ${BWA_SRC_DIR}/main.c +) + +add_executable(${PROJECT_NAME} ${BWA_SOURCE}) + +set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME bwa.wasm) + +target_compile_definitions(${PROJECT_NAME} + PRIVATE + USE_MALLOC_WRAPPERS + __SSE__ __SSE2__ __SSE4_1__ + _WASI_EMULATED_MMAN _WASI_EMULATED_SIGNAL _WASI_EMULATED_PROCESS_CLOCKS +) + +target_compile_options(${PROJECT_NAME} + PRIVATE + -Wno-unused-function + -Wno-unused-variable + -msimd128 +) + +target_link_options(${PROJECT_NAME} + PRIVATE + -Wno-unused-command-line-argument + LINKER:--allow-undefined,--export=__heap_base,--export=__data_end + LINKER:-z,stack-size=1048576 +) + +target_link_libraries(${PROJECT_NAME} z_wasm wasi-emulated-process-clocks) + +add_custom_target(bwa_wasm_opt ALL + COMMAND + ${Binaryen_WASM_OPT} -Oz --enable-simd -o bwa.opt.wasm bwa.wasm + BYPRODUCTS + ${CMAKE_CURRENT_BINARY_DIR}/bwa.opt.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_dependencies(bwa_wasm_opt ${PROJECT_NAME}) diff --git a/wasm-micro-runtime/samples/workload/bwa/CMakeLists.txt b/wasm-micro-runtime/samples/workload/bwa/CMakeLists.txt new file mode 100644 index 0000000..5db52a3 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/bwa/CMakeLists.txt @@ -0,0 +1,73 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(bwa_wasm) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +####################################### +include(ExternalProject) + +################ libz ################ +ExternalProject_Add(libz_src + GIT_REPOSITORY https://github.com/madler/zlib.git + GIT_TAG 04f42ceca40f73e2978b50e93806c2a18c1281fc + GIT_PROGRESS ON + GIT_SHALLOW ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libz + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" +) + +################ bwa ################ +ExternalProject_Add(bwa + GIT_REPOSITORY https://github.com/lh3/bwa.git + GIT_TAG 139f68fc4c3747813783a488aef2adc86626b01b + GIT_PROGRESS ON + GIT_SHALLOW ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/bwa + DEPENDS libz_src + UPDATE_COMMAND git clean -ffdx && git checkout -- * + && ${CMAKE_COMMAND} -E echo "Copying pre-installed CMakeLists.txt" + && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.bwa_wasm.txt CMakeLists.txt + && git apply ../bwa.patch + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + -DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl + ${CMAKE_CURRENT_SOURCE_DIR}/bwa + BUILD_COMMAND make bwa_wasm_opt -j 4 + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./bwa.opt.wasm ${CMAKE_CURRENT_BINARY_DIR}/bwa.wasm +) + +################ bwa data ################ +ExternalProject_Add(bwa-kit + PREFIX bwa-kit + URL https://sourceforge.net/projects/bio-bwa/files/bwakit/bwakit-0.7.15_x64-linux.tar.bz2/download + URL_HASH SHA256=0a7b11971bc7916b68e9df35a364afe77cb3000df02ffb3a6fbd1aff9be5878c + DOWNLOAD_NAME bwakit-0.7.15_x64-linux.tar.bz2 + DOWNLOAD_EXTRACT_TIMESTAMP ON + DOWNLOAD_NO_EXTRACT OFF + DOWNLOAD_NO_PROGRESS ON + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/bwa-kit/src/bwa-kit/resource-GRCh38/hs38DH-extra.fa + ${CMAKE_CURRENT_BINARY_DIR}/hs38DH-extra.fa +) diff --git a/wasm-micro-runtime/samples/workload/bwa/README.md b/wasm-micro-runtime/samples/workload/bwa/README.md new file mode 100644 index 0000000..a8fbe3e --- /dev/null +++ b/wasm-micro-runtime/samples/workload/bwa/README.md @@ -0,0 +1,46 @@ +"bwa" sample introduction +============== + +This sample demonstrates how to build [bwa](https://github.com/lh3/bwa) into +WebAssembly with simd support and run it with iwasm. + +## Preparation + +please refer to [installation instructions](../README.md). + +## Build + +``` shell +$ mkdir build && cd build +$ cmake .. +$ make +# to verify +$ ls bwa.wasm +``` + +## Download sample data + +Download the bwa-0.7.15 binary package from +[such an address](https://sourceforge.net/projects/bio-bwa/files/bwakit/bwakit-0.7.15_x64-linux.tar.bz2/download), +a sample data file named **hs38DH.fa** will be used later. + +If want more data, please refer to http://hgdownload.cse.ucsc.edu/goldenpath/hg19/bigZips/ + +## Run workload + +Firstly please build iwasm with simd support: + +``` shell +$ cd /product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. +$ make +``` + +Then compile wasm file to aot file and run: + +``` shell +$ cd /samples/workload/bwa/build +$ /wamr-compiler/build/wamrc -o bwa.aot bwa.wasm +$ /product-mini/platforms/linux/iwasm --dir=. bwa.aot index hs38DH.fa +``` diff --git a/wasm-micro-runtime/samples/workload/bwa/bwa.patch b/wasm-micro-runtime/samples/workload/bwa/bwa.patch new file mode 100644 index 0000000..5599ca3 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/bwa/bwa.patch @@ -0,0 +1,13 @@ +diff --git a/utils.c b/utils.c +index 9ceb1be..323299f 100644 +--- a/utils.c ++++ b/utils.c +@@ -301,6 +301,7 @@ long peakrss(void) + #ifdef __linux__ + return r.ru_maxrss * 1024; + #else +- return r.ru_maxrss; ++ /*return r.ru_maxrss;*/ ++ return 0; + #endif + } diff --git a/wasm-micro-runtime/samples/workload/cmake/FindBinaryen.cmake b/wasm-micro-runtime/samples/workload/cmake/FindBinaryen.cmake new file mode 100644 index 0000000..b4a6478 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/cmake/FindBinaryen.cmake @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Output below variables: +# - Binaryen_HOME. the installation location +# + +include(CMakePrintHelpers) +include(FindPackageHandleStandardArgs) + +file(GLOB Binaryen_SEARCH_PATH "/opt/binaryen*") +find_path(Binaryen_HOME + NAMES bin/wasm-opt + PATHS ${Binaryen_SEARCH_PATH} + NO_CMAKE_FIND_ROOT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + REQUIRED +) + +execute_process( + COMMAND ${Binaryen_HOME}/bin/wasm-opt --version + OUTPUT_VARIABLE WASM_OPT_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +string(REGEX MATCH version_[0-9]+ Binaryen_VERSION_tmp ${WASM_OPT_OUTPUT}) +string(REGEX MATCH [0-9]+ Binaryen_VERSION ${Binaryen_VERSION_tmp}) + +#cmake_print_variables(Binaryen_VERSION_tmp Binaryen_VERSION) + +find_package_handle_standard_args(Binaryen REQUIRED_VARS Binaryen_HOME VERSION_VAR Binaryen_VERSION) + +if(Binaryen_FOUND) + mark_as_advanced(Binaryen_SEARCH_PATH) + mark_as_advanced(Binaryen_VERSION_tmp) + mark_as_advanced(Binaryen_VERSION) + mark_as_advanced(WASM_OPT_OUTPUT) + + set(Binaryen_WASM_OPT ${Binaryen_HOME}/bin/wasm-opt) +else() + # TODO: install WASISDK +endif() diff --git a/wasm-micro-runtime/samples/workload/cmake/FindWASISDK.cmake b/wasm-micro-runtime/samples/workload/cmake/FindWASISDK.cmake new file mode 100644 index 0000000..fff8aea --- /dev/null +++ b/wasm-micro-runtime/samples/workload/cmake/FindWASISDK.cmake @@ -0,0 +1,38 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Output below variables: +# - WASISDK_HOME. the installation location +# - WASISDK_SYSROOT. where wasi-sysroot is +# - WASISDK_TOOLCHAIN. where wasi-sdk.cmake is +# + +include(CMakePrintHelpers) +include(FindPackageHandleStandardArgs) + +file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*") +find_path(WASISDK_HOME + NAMES share/wasi-sysroot + PATHS ${WASISDK_SEARCH_PATH} + NO_CMAKE_FIND_ROOT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + REQUIRED +) + +string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) + +#cmake_print_variables(WASISDK_HOME WASISDK_VERSION) +find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) + +if(WASISDK_FOUND) + mark_as_advanced(WASISDK_SEARCH_PATH) + mark_as_advanced(WASISDK_VERSION) + + set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang) + set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++) + set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot) + set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake) +else() + # TODO: install WASISDK +endif() diff --git a/wasm-micro-runtime/samples/workload/include/.gitkeep b/wasm-micro-runtime/samples/workload/include/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/wasm-micro-runtime/samples/workload/meshoptimizer/.gitignore b/wasm-micro-runtime/samples/workload/meshoptimizer/.gitignore new file mode 100644 index 0000000..dd97754 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/meshoptimizer/.gitignore @@ -0,0 +1,2 @@ +build +meshoptimizer \ No newline at end of file diff --git a/wasm-micro-runtime/samples/workload/meshoptimizer/CMakeLists.txt b/wasm-micro-runtime/samples/workload/meshoptimizer/CMakeLists.txt new file mode 100644 index 0000000..d6a1c35 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/meshoptimizer/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(bench-meshoptimizer) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +################ MESHOPTIMIZER ################ +include(ExternalProject) + +ExternalProject_Add(codecbench + PREFIX codecbench + GIT_REPOSITORY https://github.com/zeux/meshoptimizer.git + GIT_TAG f734fd572aed5bf76e84d9ed62ca6f4f6c47d84e + GIT_SHALLOW OFF + GIT_PROGRESS ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer + UPDATE_COMMAND git clean -fd && git checkout -- * + && ${CMAKE_COMMAND} -E echo "Applying patch" + && git apply ${CMAKE_CURRENT_SOURCE_DIR}/codecbench.patch + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer + BUILD_COMMAND make codecbench -j 4 + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./codecbench.wasm ${CMAKE_CURRENT_BINARY_DIR}/codecbench.wasm +) diff --git a/wasm-micro-runtime/samples/workload/meshoptimizer/README.md b/wasm-micro-runtime/samples/workload/meshoptimizer/README.md new file mode 100644 index 0000000..466cd87 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/meshoptimizer/README.md @@ -0,0 +1,57 @@ +"codecbench of meshoptimizer" sample introduction +============== + +This sample demonstrates how to build [codecbench of messoptimizer](https://github.com/zeux/meshoptimizer) into +WebAssembly with simd support and run it with iwasm. + +## Preparation + +please refer to [installation instructions](../README.md). + +## Build with wasi-sdk + +``` shell +$ mkdir build && cd build +$ cmake .. +$ make +# to verify +$ ls codecbench.wasm +``` + +## Or build with EMSDK + +EMSDK is another toolchain to compile C/C++ code to WASM. In this case, the output wasm file +might have a higher performance than the file generated by wasi-sdk. + +``` shell +$ git clone https://github.com/zeux/meshoptimizer.git +$ cd messoptimizer +$ em++ tools/codecbench.cpp src/vertexcodec.cpp src/vertexfilter.cpp \ + src/overdrawanalyzer.cpp src/indexgenerator.cpp src/vcacheoptimizer.cpp \ + src/clusterizer.cpp src/indexcodec.cpp src/vfetchanalyzer.cpp \ + src/spatialorder.cpp src/allocator.cpp src/vcacheanalyzer.cpp \ + src/vfetchoptimizer.cpp src/overdrawoptimizer.cpp src/simplifier.cpp \ + src/stripifier.cpp -O3 -msimd128 \ + -s TOTAL_MEMORY=268435456 \ + -o codecbench.wasm +$ ls -l codecbench.wasm +``` + +## Run workload + +Firstly please build iwasm with simd support: + +``` shell +$ cd /product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. +$ make +``` + +Then compile wasm file to aot file and run: + +``` shell +$ /wamr-compiler/build/wamrc -o codecbench.aot codecbench.wasm +$ /product-mini/platforms/linux/build/iwasm codecbench.aot +``` + diff --git a/wasm-micro-runtime/samples/workload/meshoptimizer/codecbench.patch b/wasm-micro-runtime/samples/workload/meshoptimizer/codecbench.patch new file mode 100644 index 0000000..19db792 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/meshoptimizer/codecbench.patch @@ -0,0 +1,119 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 612cf3b..22a365a 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -158,3 +158,43 @@ install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/meshoptimizerConfigVersion.cmake + COMPONENT meshoptimizer + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/meshoptimizer) ++ ++################################################## ++# codecbench ++################################################## ++add_executable(codecbench tools/codecbench.cpp ${SOURCES}) ++ ++set_target_properties(codecbench PROPERTIES OUTPUT_NAME codecbench.wasm) ++ ++target_compile_options(codecbench ++ PUBLIC ++ -O3 -msimd128 ++ -std=c++11 ++ -Wno-unused-function ++ -Wno-unused-variable ++) ++ ++target_link_options(codecbench ++ PUBLIC ++ LINKER:-allow-undefined,--demangle,--export=malloc,--export=free ++) ++ ++find_program(WASM_OPT ++ NAMES wasm-opt ++ PATHS /opt/binaryen-version_97/bin /opt/binaryen/bin ++) ++ ++if (NOT WASM_OPT) ++ message(WARNING "can not find wasm-opt and will not optimize any wasm module") ++endif() ++ ++add_custom_target(codecbench.opt ALL ++ COMMAND ++ ${WASM_OPT} -Oz --enable-simd -o codecbench.opt.wasm codecbench.wasm ++ BYPRODUCTS ++ ${CMAKE_CURRENT_BINARY_DIR}/codecbench.opt.wasm ++ WORKING_DIRECTORY ++ ${CMAKE_CURRENT_BINARY_DIR} ++) ++ ++add_dependencies(codecbench.opt codecbench) +diff --git a/src/vertexcodec.cpp b/src/vertexcodec.cpp +index 4bd1112..257c258 100644 +--- a/src/vertexcodec.cpp ++++ b/src/vertexcodec.cpp +@@ -89,13 +89,13 @@ + #endif + + #ifdef SIMD_WASM +-#define wasmx_splat_v32x4(v, i) wasm_v32x4_shuffle(v, v, i, i, i, i) +-#define wasmx_unpacklo_v8x16(a, b) wasm_v8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23) +-#define wasmx_unpackhi_v8x16(a, b) wasm_v8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31) +-#define wasmx_unpacklo_v16x8(a, b) wasm_v16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) +-#define wasmx_unpackhi_v16x8(a, b) wasm_v16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) +-#define wasmx_unpacklo_v64x2(a, b) wasm_v64x2_shuffle(a, b, 0, 2) +-#define wasmx_unpackhi_v64x2(a, b) wasm_v64x2_shuffle(a, b, 1, 3) ++#define wasmx_splat_v32x4(v, i) wasm_i32x4_shuffle(v, v, i, i, i, i) ++#define wasmx_unpacklo_v8x16(a, b) wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23) ++#define wasmx_unpackhi_v8x16(a, b) wasm_i8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31) ++#define wasmx_unpacklo_v16x8(a, b) wasm_i16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) ++#define wasmx_unpackhi_v16x8(a, b) wasm_i16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) ++#define wasmx_unpacklo_v64x2(a, b) wasm_i64x2_shuffle(a, b, 0, 2) ++#define wasmx_unpackhi_v64x2(a, b) wasm_i64x2_shuffle(a, b, 1, 3) + #endif + + namespace meshopt +@@ -757,7 +757,7 @@ static v128_t decodeShuffleMask(unsigned char mask0, unsigned char mask1) + v128_t sm1 = wasm_v128_load(&kDecodeBytesGroupShuffle[mask1]); + + v128_t sm1off = wasm_v128_load(&kDecodeBytesGroupCount[mask0]); +- sm1off = wasm_v8x16_shuffle(sm1off, sm1off, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); ++ sm1off = wasm_i8x16_shuffle(sm1off, sm1off, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + v128_t sm1r = wasm_i8x16_add(sm1, sm1off); + +@@ -807,7 +807,7 @@ static const unsigned char* decodeBytesGroupSimd(const unsigned char* data, unsi + + v128_t shuf = decodeShuffleMask(mask0, mask1); + +- v128_t result = wasm_v128_bitselect(wasm_v8x16_swizzle(rest, shuf), sel, mask); ++ v128_t result = wasm_v128_bitselect(wasm_i8x16_swizzle(rest, shuf), sel, mask); + + wasm_v128_store(buffer, result); + +@@ -829,7 +829,7 @@ static const unsigned char* decodeBytesGroupSimd(const unsigned char* data, unsi + + v128_t shuf = decodeShuffleMask(mask0, mask1); + +- v128_t result = wasm_v128_bitselect(wasm_v8x16_swizzle(rest, shuf), sel, mask); ++ v128_t result = wasm_v128_bitselect(wasm_i8x16_swizzle(rest, shuf), sel, mask); + + wasm_v128_store(buffer, result); + +diff --git a/src/vertexfilter.cpp b/src/vertexfilter.cpp +index 5c7589c..c79cad4 100644 +--- a/src/vertexfilter.cpp ++++ b/src/vertexfilter.cpp +@@ -57,10 +57,10 @@ + #endif + + #ifdef SIMD_WASM +-#define wasmx_unpacklo_v16x8(a, b) wasm_v16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) +-#define wasmx_unpackhi_v16x8(a, b) wasm_v16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) +-#define wasmx_unziplo_v32x4(a, b) wasm_v32x4_shuffle(a, b, 0, 2, 4, 6) +-#define wasmx_unziphi_v32x4(a, b) wasm_v32x4_shuffle(a, b, 1, 3, 5, 7) ++#define wasmx_unpacklo_v16x8(a, b) wasm_i16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) ++#define wasmx_unpackhi_v16x8(a, b) wasm_i16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) ++#define wasmx_unziplo_v32x4(a, b) wasm_i32x4_shuffle(a, b, 0, 2, 4, 6) ++#define wasmx_unziphi_v32x4(a, b) wasm_i32x4_shuffle(a, b, 1, 3, 5, 7) + #endif + + #ifndef __has_builtin diff --git a/wasm-micro-runtime/samples/workload/preparation.sh b/wasm-micro-runtime/samples/workload/preparation.sh new file mode 100755 index 0000000..47b11ac --- /dev/null +++ b/wasm-micro-runtime/samples/workload/preparation.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +readonly BUILD_CONTENT="/tmp/build_content" +readonly WABT_VER=1.0.31 +readonly WABT_FILE="wabt-${WABT_VER}-ubuntu.tar.gz" +readonly CMAKE_VER=3.25.1 +readonly CMAKE_FILE="cmake-${CMAKE_VER}-Linux-x86_64.sh" +readonly BINARYEN_VER=version_111 +readonly BINARYEN_FILE="binaryen-${BINARYEN_VER}-x86_64-linux.tar.gz" +readonly BAZEL_VER=6.0.0 +readonly BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh + +function DEBUG() { + env | grep -q "\" +} + +# +# install dependency +function install_deps() { + apt update + apt install -y lsb-release wget software-properties-common \ + build-essential git tree zip unzip +} + +# +# install wabt +function install_wabt() { + if [[ ! -f ${WABT_FILE} ]]; then + wget https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/${WABT_FILE} + fi + + tar zxf ${WABT_FILE} -C /opt + ln -sf /opt/wabt-${WABT_VER} /opt/wabt +} + +# +# install cmake +function install_cmake() { + if [[ ! -f cmake-${CMAKE_VER}-Linux-x86_64.sh ]]; then + wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/${CMAKE_FILE} + fi + + chmod a+x ${CMAKE_FILE} + mkdir /opt/cmake + ./${CMAKE_FILE} --prefix=/opt/cmake --skip-license + ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake +} + +# +# install emsdk +function install_emsdk() { + cd /opt + git clone https://github.com/emscripten-core/emsdk.git + cd emsdk + git pull + ./emsdk install 3.1.28 + ./emsdk activate 3.1.28 + echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc +} + +# +# install binaryen +function install_binaryen() { + if [[ ! -f ${BINARYEN_FILE} ]]; then + wget https://github.com/WebAssembly/binaryen/releases/download/${BINARYEN_VER}/${BINARYEN_FILE} + fi + + tar zxf ${BINARYEN_FILE} -C /opt + ln -sf /opt/binaryen-${BINARYEN_VER} /opt/binaryen +} + +# +# install bazel +function install_bazel() { + if [[ ! -f ${BAZEL_FILE} ]]; then + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VER}/${BAZEL_FILE} + fi + + chmod a+x ${BAZEL_FILE} + ./${BAZEL_FILE} +} + +# +# MAIN +DEBUG && set -xevu +if [[ ! -d ${BUILD_CONTENT} ]]; then + mkdir ${BUILD_CONTENT} +fi + +cd ${BUILD_CONTENT} || exit +if DEBUG; then + "$@" +else + install_deps \ + && install_bazel \ + && install_binaryen \ + && install_cmake \ + && install_emsdk \ + && install_wabt \ + && install_wasi-sdk +fi +cd - > /dev/null || exit +DEBUG && set +xevu diff --git a/wasm-micro-runtime/samples/workload/tensorflow/README.md b/wasm-micro-runtime/samples/workload/tensorflow/README.md new file mode 100644 index 0000000..7bc7dd2 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/tensorflow/README.md @@ -0,0 +1,19 @@ +"tensorflow" sample introduction +============== + +This sample demonstrates how to build [tensorflow](https://github.com/tensorflow/tensorflow) into WebAssembly with emsdk toolchain and run it with iwasm.: +```bash +./build.sh +# for linux platform, or +./build.sh --threads +# for multi-threading on linux platform +./build.sh --sgx +# for linux-sgx platform +``` +to build tensorflow and run it with iwasm, which basically contains the following steps: +- clone emsdk under `/core/deps`, install and activate 2.0.26 +- hack emcc to delete some objects in libc.a +- build tf-lite with emcc compiler +- build iwasm with lib-pthread and libc-emcc enabled +- run benchmark model with iwasm: + --max-secs 300: means the max training time cost is 5 minutes, you can adjust it by yourself diff --git a/wasm-micro-runtime/samples/workload/tensorflow/build.sh b/wasm-micro-runtime/samples/workload/tensorflow/build.sh new file mode 100755 index 0000000..6df8db4 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/tensorflow/build.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#################################### +# build tensorflow-lite sample # +#################################### +BUILD_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +WAMR_DIR="${BUILD_SCRIPT_DIR}/../../.." +WAMR_PLATFORM_DIR="${WAMR_DIR}/product-mini/platforms" +WAMRC_DIR="${WAMR_DIR}/wamr-compiler" +CORE_DEPS_DIR="${WAMR_DIR}/core/deps" +EMSDK_DIR="${CORE_DEPS_DIR}/emsdk" + +EMSDK_WASM_DIR="${EMSDK_DIR}/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten" +OUT_DIR="${BUILD_SCRIPT_DIR}/out" +TENSORFLOW_DIR="${BUILD_SCRIPT_DIR}/tensorflow" +TF_LITE_BUILD_DIR="${TENSORFLOW_DIR}/tensorflow/lite/tools/make" + +function Clear_Before_Exit() +{ + [[ -f ${TENSORFLOW_DIR}/tf_lite.patch ]] && + rm -f ${TENSORFLOW_DIR}/tf_lite.patch + # resume the libc.a under EMSDK_WASM_DIR + cd ${EMSDK_WASM_DIR} + mv libc.a.bak libc.a +} + +set -xe + +# 1.clone emsdk +cd ${CORE_DEPS_DIR} +rm -fr emsdk +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install 2.0.26 +./emsdk activate 2.0.26 +source emsdk_env.sh + +# 2.hack emcc +cd ${EMSDK_WASM_DIR} +# back up libc.a +cp libc.a libc.a.bak +# delete some objects in libc.a +emar d libc.a open.o +emar d libc.a mmap.o +emar d libc.a munmap.o +emar d libc.a library_pthread_stub.o +emar d libc.a pthread_self.o +emranlib libc.a + +# 3. build tf-lite +cd ${BUILD_SCRIPT_DIR} +# 3.1 clone tf repo from Github and checkout to 2303ed commit +if [ ! -d "tensorflow" ]; then + git clone https://github.com/tensorflow/tensorflow.git +fi + +cd ${TENSORFLOW_DIR} +git checkout 2303ed4bdb344a1fc4545658d1df6d9ce20331dd + +# 3.2 copy the tf-lite.patch to tensorflow_root_dir and apply it +cd ${TENSORFLOW_DIR} +cp ${BUILD_SCRIPT_DIR}/tf_lite.patch . +git checkout tensorflow/lite/tools/make/Makefile +git checkout tensorflow/lite/tools/make/targets/linux_makefile.inc + +if [[ $(git apply tf_lite.patch 2>&1) =~ "error" ]]; then + echo "git apply patch failed, please check tf-lite related changes..." + Clear_Before_Exit + exit 0 +fi + +cd ${TF_LITE_BUILD_DIR} +# 3.3 download dependencies +if [ ! -d "${TF_LITE_BUILD_DIR}/downloads" ]; then + source download_dependencies.sh +fi + +# 3.4 build tf-lite target +if [ -d "${TF_LITE_BUILD_DIR}/gen" ]; then + rm -fr ${TF_LITE_BUILD_DIR}/gen +fi + +make -j 4 -C "${TENSORFLOW_DIR}" -f ${TF_LITE_BUILD_DIR}/Makefile + +# remove patch file and recover emcc libc.a after building +Clear_Before_Exit + +# 3.5 copy /make/gen target files to out/ +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +cp -r ${TF_LITE_BUILD_DIR}/gen/linux_x86_64/bin/. ${OUT_DIR}/ + +# 4. compile tf-model.wasm to tf-model.aot with wamrc +# 4.1 build wamr-compiler +cd ${WAMRC_DIR} +./build_llvm.sh +rm -fr build && mkdir build +cd build && cmake .. +make +# 4.2 compile tf-mode.wasm to tf-model.aot +WAMRC_CMD="$(pwd)/wamrc" +cd ${OUT_DIR} +if [[ $1 == '--sgx' ]]; then + ${WAMRC_CMD} -sgx -o benchmark_model.aot benchmark_model.wasm +elif [[ $1 == '--threads' ]]; then + ${WAMRC_CMD} --enable-multi-thread -o benchmark_model.aot benchmark_model.wasm +else + ${WAMRC_CMD} -o benchmark_model.aot benchmark_model.wasm +fi + +# 5. build iwasm with pthread and libc_emcc enable +# platform: +# linux by default +# linux-sgx if $1 equals '--sgx' +if [[ $1 == '--sgx' ]]; then + cd ${WAMR_PLATFORM_DIR}/linux-sgx + rm -fr build && mkdir build + cd build && cmake .. -DWAMR_BUILD_LIBC_EMCC=1 + make + cd ../enclave-sample + make +else + cd ${WAMR_PLATFORM_DIR}/linux + rm -fr build && mkdir build + cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 + make +fi + +# 6. run tensorflow with iwasm +cd ${OUT_DIR} +# 6.1 download tf-lite model +wget "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip" +unzip mobilenet_v1_224_android_quant_2017_11_08.zip + +# 6.2 run tf-lite model with iwasm +echo "---> run tensorflow benchmark model with iwasm" +if [[ $1 == '--sgx' ]]; then + IWASM_CMD="${WAMR_PLATFORM_DIR}/linux-sgx/enclave-sample/iwasm" +else + IWASM_CMD="${WAMR_PLATFORM_DIR}/linux/build/iwasm" +fi + +if [[ $1 == '--threads' ]]; then + ${IWASM_CMD} --heap-size=10475860 \ + benchmark_model.aot --num_threads=4 \ + --graph=mobilenet_quant_v1_224.tflite --max_secs=300 +else + ${IWASM_CMD} --heap-size=10475860 \ + benchmark_model.aot \ + --graph=mobilenet_quant_v1_224.tflite --max_secs=300 +fi diff --git a/wasm-micro-runtime/samples/workload/tensorflow/tf_lite.patch b/wasm-micro-runtime/samples/workload/tensorflow/tf_lite.patch new file mode 100644 index 0000000..b6224d5 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/tensorflow/tf_lite.patch @@ -0,0 +1,84 @@ +diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile +index c7ddff58440..ebfebaead35 100644 +--- a/tensorflow/lite/tools/make/Makefile ++++ b/tensorflow/lite/tools/make/Makefile +@@ -48,11 +48,7 @@ INCLUDES += -I/usr/local/include + + # These are the default libraries needed, but they can be added to or + # overridden by the platform-specific settings in target makefiles. +-LIBS := \ +--lstdc++ \ +--lpthread \ +--lm \ +--lz \ ++LIBS := -lm \ + -ldl + + # There are no rules for compiling objects for the host system (since we don't +@@ -84,14 +80,24 @@ endif # ifeq ($(HOST_ARCH),$(TARGET_ARCH)) + endif # ifeq ($(HOST_OS),$(TARGET)) + endif + ++CFLAGS+=-msimd128 -mbulk-memory -matomics ++CXXFLAGS+=-msimd128 -mbulk-memory -matomics ++ ++LIBFLAGS += -s TOTAL_STACK=1048576 -s MALLOC="none" \ ++ -s INITIAL_MEMORY=16777216 \ ++ -s MAXIMUM_MEMORY=167772160 \ ++ -s ALLOW_MEMORY_GROWTH=1 \ ++ -Wl,--export=__data_end -Wl,--export=__heap_base,--shared-memory,--no-check-features \ ++ -s ERROR_ON_UNDEFINED_SYMBOLS=0 ++ + # This library is the main target for this makefile. It will contain a minimal + # runtime that can be linked in to other programs. + LIB_NAME := libtensorflow-lite.a + + # Benchmark static library and binary + BENCHMARK_LIB_NAME := benchmark-lib.a +-BENCHMARK_BINARY_NAME := benchmark_model +-BENCHMARK_PERF_OPTIONS_BINARY_NAME := benchmark_model_performance_options ++BENCHMARK_BINARY_NAME := benchmark_model.wasm ++BENCHMARK_PERF_OPTIONS_BINARY_NAME := benchmark_model_performance_options.wasm + + # A small example program that shows how to link against the library. + MINIMAL_SRCS := \ +@@ -277,12 +283,16 @@ LIB_PATH := $(LIBDIR)$(LIB_NAME) + BENCHMARK_LIB := $(LIBDIR)$(BENCHMARK_LIB_NAME) + BENCHMARK_BINARY := $(BINDIR)$(BENCHMARK_BINARY_NAME) + BENCHMARK_PERF_OPTIONS_BINARY := $(BINDIR)$(BENCHMARK_PERF_OPTIONS_BINARY_NAME) +-MINIMAL_BINARY := $(BINDIR)minimal ++MINIMAL_BINARY := $(BINDIR)minimal.wasm + LABEL_IMAGE_BINARY := $(BINDIR)label_image + +-CXX := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}g++ +-CC := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}gcc +-AR := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}ar ++# CXX := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}g++ ++# CC := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}gcc ++# AR := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}ar ++ ++CXX := em++ ++CC := emcc ++AR := emar + + MINIMAL_OBJS := $(addprefix $(OBJDIR), \ + $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MINIMAL_SRCS)))) +diff --git a/tensorflow/lite/tools/make/targets/linux_makefile.inc b/tensorflow/lite/tools/make/targets/linux_makefile.inc +index 222cef9e5ff..eea89a38f01 100644 +--- a/tensorflow/lite/tools/make/targets/linux_makefile.inc ++++ b/tensorflow/lite/tools/make/targets/linux_makefile.inc +@@ -2,12 +2,10 @@ + ifeq ($(TARGET), linux) + CXXFLAGS += \ + -fPIC \ +- -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \ +- -pthread ++ -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK + CFLAGS += \ + -fPIC \ +- -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \ +- -pthread ++ -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK + # TODO(petewarden): In the future we may want to add architecture-specific + # flags like -msse4.2 + LIBS += -ldl diff --git a/wasm-micro-runtime/samples/workload/wasm-av1/.gitignore b/wasm-micro-runtime/samples/workload/wasm-av1/.gitignore new file mode 100644 index 0000000..e8bec70 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/wasm-av1/.gitignore @@ -0,0 +1,8 @@ +# from CMakeLists +av1 +build +include + +# from build.sh +wasm-av1 +out diff --git a/wasm-micro-runtime/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt b/wasm-micro-runtime/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt new file mode 100644 index 0000000..409665b --- /dev/null +++ b/wasm-micro-runtime/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt @@ -0,0 +1,72 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(testavx) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake) + +################ dependencies ################ +find_package(Binaryen 111 REQUIRED) + +################ AOM ################ +set(ENABLE_CCACHE ON) +set(ENABLE_DOCS OFF CACHE BOOL "ENABLE_DOCS" FORCE) +set(ENABLE_EXAMPLES OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_NEON OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_NEON_ASM OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_VSX OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_MMX OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(AOM_TARGET_CPU generic) +set(CONFIG_ACCOUNTING 1 CACHE NUMBER "" FORCE) +set(CONFIG_INSPECTION 1 CACHE NUMBER "" FORCE) +set(CONFIG_MULTITHREAD 0 CACHE NUMBER "" FORCE) +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE NUMBER "" FORCE) +set(CONFIG_UNIT_TESTS 0 CACHE NUMBER "" FORCE) +set(CONFIG_WEBM_IO 0 CACHE NUMBER "" FORCE) +add_subdirectory(third_party/aom third_party/aom/bin EXCLUDE_FROM_ALL) + +################ AV ################ +add_executable(${PROJECT_NAME} + test.c + decode-av1.c +) + +target_include_directories(${PROJECT_NAME} + PRIVATE + third_party/aom/ + ${CMAKE_CURRENT_BINARY_DIR}/third_party/aom/bin +) + +set_target_properties(${PROJECT_NAME} + PROPERTIES + OUTPUT_NAME ${PROJECT_NAME}.wasm +) + +target_link_options(${PROJECT_NAME} + PRIVATE + LINKER:--allow-undefined + LINKER:--export=__heap_base + LINKER:--export=__data_end + LINKER:--initial-memory=33554432 + LINKER:-z,stack-size=25165824 +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE + aom +) + +add_dependencies(${PROJECT_NAME} aom) + +add_custom_target(${PROJECT_NAME}_opt ALL + COMMAND + ${Binaryen_WASM_OPT} -Oz --enable-simd -o ${PROJECT_NAME}.opt.wasm ${PROJECT_NAME}.wasm + BYPRODUCTS + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.opt.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_dependencies(${PROJECT_NAME}_opt ${PROJECT_NAME}) diff --git a/wasm-micro-runtime/samples/workload/wasm-av1/CMakeLists.txt b/wasm-micro-runtime/samples/workload/wasm-av1/CMakeLists.txt new file mode 100644 index 0000000..3d263bf --- /dev/null +++ b/wasm-micro-runtime/samples/workload/wasm-av1/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(av1_wasm) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +####################################### +include(ExternalProject) + +################ av1 ################ +ExternalProject_Add(av1 + PREFIX av1 + GIT_REPOSITORY https://github.com/GoogleChromeLabs/wasm-av1.git + GIT_TAG master + GIT_PROGRESS ON + GIT_SHALLOW ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/av1 + UPDATE_COMMAND git clean -fd && git checkout -- * + && ${CMAKE_COMMAND} -E echo "Copying pre-installed CMakeLists.txt" + && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.avx_wasm.txt CMakeLists.txt + && git apply ../av1-clang.patch + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + -DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl + ${CMAKE_CURRENT_SOURCE_DIR}/av1 + BUILD_COMMAND make testavx_opt -j 4 + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different + testavx.opt.wasm + ${CMAKE_CURRENT_SOURCE_DIR}/av1/third_party/samples/elephants_dream_480p24.ivf + ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/wasm-micro-runtime/samples/workload/wasm-av1/README.md b/wasm-micro-runtime/samples/workload/wasm-av1/README.md new file mode 100644 index 0000000..2166fe6 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/wasm-av1/README.md @@ -0,0 +1,54 @@ +"wasm-av1" sample introduction +============== + +This sample demonstrates how to build [wasm-av1](https://github.com/GoogleChromeLabs/wasm-av1) into +WebAssembly with simd support and run it with iwasm. + +## Preparation + +please refer to [installation instructions](../README.md). + +## Build with wasi-sdk + +``` shell +$ mkdir build && cd build +$ cmake .. +$ make +# to verify +$ ls testavx.wasm +``` + +## Or build with EMSDK + +just run the convenience script: + +```bash +./build.sh +``` + +the script builds wasm-av1 and runs it with iwasm, which basically contains the following steps: +- hack emcc to delete some objects in libc.a +- patch wasm-av1 and build it with emcc compiler +- build iwasm with simd and libc-emcc support +- run testav1.aot with iwasm + +### Run workload + +Firstly please build iwasm with simd support: + +``` shell +$ cd /product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1 +$ make +``` + +Then compile wasm file to aot file and run: + +``` shell +$ cd +$ /wamr-compiler/build/wamrc -o testavx.aot testavx.wasm +# copy sample data like /samples/workload/wasm-av1/av1/third_party/samples/elephants_dream_480p24.ivf +# make sure you declare the access priority of the directory in which the sample data is +$ /product-mini/platforms/linux/build/iwasm --dir=. testavx.aot elephants_dream_480p24.ivf +``` diff --git a/wasm-micro-runtime/samples/workload/wasm-av1/av1-clang.patch b/wasm-micro-runtime/samples/workload/wasm-av1/av1-clang.patch new file mode 100644 index 0000000..97e7954 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/wasm-av1/av1-clang.patch @@ -0,0 +1,19 @@ +diff --git a/test.c b/test.c +index df2d44b..520bf13 100644 +--- a/test.c ++++ b/test.c +@@ -63,9 +63,14 @@ main(int argc, char *argv[]) { + static int i = 0; + + ++i; ++ printf("Decoding frame #%d\n", i); + if (30 <= i && i < 40) { ++ printf("Dumping frame #%d\n", i); + dump_raw_frame(af, i); + } ++ if (i >= 1000) { ++ break; ++ } + } + /* + * Run the decoder every time, so that we keep diff --git a/wasm-micro-runtime/samples/workload/wasm-av1/build.sh b/wasm-micro-runtime/samples/workload/wasm-av1/build.sh new file mode 100755 index 0000000..efa17ec --- /dev/null +++ b/wasm-micro-runtime/samples/workload/wasm-av1/build.sh @@ -0,0 +1,100 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +#################################### +# build wasm-av1 sample # +#################################### +if [ ! -d "${EMSDK}" ]; then + echo "can not find emsdk. " + echo "please refer to https://emscripten.org/docs/getting_started/downloads.html " + echo "to install it, or active it by 'source emsdk_env.sh'" + exit +fi + +set -xe + +EMSDK_WASM_DIR="${EMSDK}/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten" +BUILD_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${BUILD_SCRIPT_DIR}/out" +WASM_AV1_DIR="${BUILD_SCRIPT_DIR}/wasm-av1" + +WAMR_PLATFORM_DIR="${BUILD_SCRIPT_DIR}/../../../product-mini/platforms" +IWASM_CMD="${WAMR_PLATFORM_DIR}/linux/build/iwasm" + +WAMRC_DIR="${BUILD_SCRIPT_DIR}/../../../wamr-compiler" +WAMRC_CMD="${WAMRC_DIR}/build/wamrc" + +function Clear_Before_Exit +{ + [[ -f ${WASM_AV1_DIR}/wasm-av1.patch ]] && + rm -f ${WASM_AV1_DIR}/wasm-av1.patch + # resume the libc.a under EMSDK_WASM_DIR + cd ${EMSDK_WASM_DIR} + mv libc.a.bak libc.a +} + +# 1.hack emcc +cd ${EMSDK_WASM_DIR} +# back up libc.a +cp libc.a libc.a.bak +# delete some objects in libc.a +emar d libc.a fopen.o +emar d libc.a fread.o +emar d libc.a feof.o +emar d libc.a fclose.o + +# 2. build wasm-av1 +cd ${BUILD_SCRIPT_DIR} +# 2.1 clone wasm-av1 repo from Github +if [ ! -d "wasm-av1" ]; then + git clone https://github.com/GoogleChromeLabs/wasm-av1.git +fi + +# 2.2 copy the wasm-av1.patch to wasm-av1 and apply the patch +cd ${WASM_AV1_DIR} +cp -a ${BUILD_SCRIPT_DIR}/wasm-av1.patch . +git checkout Makefile +git checkout test.c +git checkout third_party/aom + +if [[ $(git apply wasm-av1.patch 2>&1) =~ "error" ]]; then + echo "git apply patch failed, please check wasm-av1 related changes..." + Clear_Before_Exit + exit 0 +fi + +make testavx -j 4 + +# remove patch file and recover emcc libc.a after building +Clear_Before_Exit + +# 2.3 copy /make/gen target files to out/ +rm -rf ${OUT_DIR} && mkdir ${OUT_DIR} +cp -a ${WASM_AV1_DIR}/testavx.wasm ${OUT_DIR}/ + +# 3. compile wasm-av1.wasm to wasm-av1.aot with wamrc +# 3.1 build wamr-compiler +cd ${WAMRC_DIR} +./build_llvm.sh +rm -fr build && mkdir build +cd build && cmake .. +make +# 3.2 compile wasm-av1.wasm to wasm-av1.aot +cd ${OUT_DIR} +${WAMRC_CMD} -o testavx.aot testavx.wasm + +# 4. build iwasm with pthread and libc_emcc enable +cd ${WAMR_PLATFORM_DIR}/linux +rm -fr build && mkdir build +cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 +make + +# 5. run wasm-av1 with iwasm +echo "---> run testav1.aot with iwasm" +cd ${OUT_DIR} +${IWASM_CMD} testavx.aot ../wasm-av1/third_party/samples/elephants_dream_480p24.ivf + diff --git a/wasm-micro-runtime/samples/workload/wasm-av1/wasm-av1.patch b/wasm-micro-runtime/samples/workload/wasm-av1/wasm-av1.patch new file mode 100644 index 0000000..98f2625 --- /dev/null +++ b/wasm-micro-runtime/samples/workload/wasm-av1/wasm-av1.patch @@ -0,0 +1,701 @@ +diff --git a/Makefile b/Makefile +index c39fff6..4682d43 100644 +--- a/Makefile ++++ b/Makefile +@@ -59,11 +59,13 @@ $(TARGET): $(DEPS) blob-api.c yuv-to-rgb.c $(EMLIBAV1) + ]" \ + blob-api.c yuv-to-rgb.c $(SRCS) $(INC) -L $(LIBDIR) -l$(LIB) + +-$(TESTTARGET): test.c $(DEPS) $(X86LIBAV1) +- cc -o $@ -O3 test.c $(SRCS) $(INC) -L $(X86LIBDIR) -l$(LIB) ++$(TESTTARGET): test.c $(DEPS) $(EMLIBAV1) ++ emcc -o $@.wasm -O3 test.c $(SRCS) $(INC) -L $(LIBDIR) -l$(LIB) \ ++ -s TOTAL_MEMORY=104857600 -s ERROR_ON_UNDEFINED_SYMBOLS=0 + +-$(TESTTARGET)g: test.c $(DEPS) $(X86LIBAV1) +- cc -o $@ -g test.c $(SRCS) $(INC) -L $(X86LIBDIR) -l$(LIB) ++$(TESTTARGET)g: test.c $(DEPS) $(EMLIBAV1) ++ emcc -o $@.wasm -g test.c $(SRCS) $(INC) -L $(LIBDIR) -l$(LIB) \ ++ -s TOTAL_MEMORY=104857600 -s ERROR_ON_UNDEFINED_SYMBOLS=0 + + clean: + -rm $(TARGET) $(TESTTARGET) $(TESTTARGET)g +@@ -80,7 +82,7 @@ $(EMLIBAV1): $(LIBDIR) + -DCONFIG_RUNTIME_CPU_DETECT=0 \ + -DCONFIG_UNIT_TESTS=0 \ + -DCONFIG_WEBM_IO=0 \ +- -DCMAKE_TOOLCHAIN_FILE=`../../get-emcmake.sh`; \ ++ -DCMAKE_TOOLCHAIN_FILE=${EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake; \ + make \ + ) + +diff --git a/test.c b/test.c +index df2d44b..cb270de 100644 +--- a/test.c ++++ b/test.c +@@ -18,6 +18,9 @@ + + #include "decode-av1-priv.h" + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static void + dump_raw_frame(AVX_Video_Frame *avf, int id) { + FILE *f; +@@ -26,12 +29,13 @@ dump_raw_frame(AVX_Video_Frame *avf, int id) { + void *buf; + + sprintf(name, "frame%04d.yuv", id); ++ printf("writing %s ..\n", name); + if ((f = fopen(name, "wb")) == NULL) { + return; + } + buf = AVX_Video_Frame_get_buffer(avf); + size = AVX_Video_Frame_get_size(avf); +- fwrite(buf, size, 1, f); ++ emcc_fwrite(buf, size, 1, f); + fclose(f); + } + +@@ -63,9 +67,12 @@ main(int argc, char *argv[]) { + static int i = 0; + + ++i; ++ printf("##decode raw frame %d\n", i); + if (30 <= i && i < 40) { + dump_raw_frame(af, i); + } ++ if (i >= 1000) ++ break; + } + /* + * Run the decoder every time, so that we keep +diff --git a/third_party/aom/CMakeLists.txt b/third_party/aom/CMakeLists.txt +index 9dbe301..20c7be4 100644 +--- a/third_party/aom/CMakeLists.txt ++++ b/third_party/aom/CMakeLists.txt +@@ -56,6 +56,10 @@ option(BUILD_SHARED_LIBS "CMake should generate a shared library build." OFF) + + project(AOM C CXX) + ++set(CMAKE_C_FLAGS "-msimd128 -msse2 -msse3 -msse4.1 -msse4.2 ${CMAKE_C_FLAGS}") ++set(CMAKE_CXX_FLAGS "-msimd128 -msse2 -msse3 -msse4.1 -msse4.2 ${CMAKE_CXX_FLAGS}") ++set(CMAKE_VERBOSE_MAKEFILE on) ++ + set(AOM_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + set(AOM_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}") + set(INCLUDE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/include" +@@ -347,7 +351,7 @@ if(CONFIG_AV1_DECODER AND ENABLE_EXAMPLES) + em_link_post_js(inspect "${AOM_ROOT}/tools/inspect-post.js") + # Force generation of Wasm instead of asm.js + append_link_flag_to_target("inspect" "-s WASM=1") +- append_compiler_flag("-s WASM=1") ++ append_compiler_flag("-O3 -s WASM=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0") + endif() + endif() + +diff --git a/third_party/aom/aom/src/aom_codec.c b/third_party/aom/aom/src/aom_codec.c +index dbd6fa5..a8d2a49 100644 +--- a/third_party/aom/aom/src/aom_codec.c ++++ b/third_party/aom/aom/src/aom_codec.c +@@ -132,6 +132,7 @@ void aom_internal_error(struct aom_internal_error_info *info, + info->detail[sz - 1] = '\0'; + } + ++ printf("##aom internal error: %s\n", info->detail); + if (info->setjmp) longjmp(info->jmp, info->error_code); + } + +diff --git a/third_party/aom/aom_dsp/grain_table.c b/third_party/aom/aom_dsp/grain_table.c +index 0d6a73f..4b05833 100644 +--- a/third_party/aom/aom_dsp/grain_table.c ++++ b/third_party/aom/aom_dsp/grain_table.c +@@ -293,6 +293,9 @@ aom_codec_err_t aom_film_grain_table_read( + return error_info->error_code; + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + aom_codec_err_t aom_film_grain_table_write( + const aom_film_grain_table_t *t, const char *filename, + struct aom_internal_error_info *error_info) { +@@ -305,7 +308,7 @@ aom_codec_err_t aom_film_grain_table_write( + return error_info->error_code; + } + +- if (!fwrite(kFileMagic, 8, 1, file)) { ++ if (!emcc_fwrite(kFileMagic, 8, 1, file)) { + aom_internal_error(error_info, AOM_CODEC_ERROR, + "Unable to write file magic"); + fclose(file); +diff --git a/third_party/aom/aomdec.c b/third_party/aom/aomdec.c +index 4addee8..f850147 100644 +--- a/third_party/aom/aomdec.c ++++ b/third_party/aom/aomdec.c +@@ -274,6 +274,9 @@ static void update_image_md5(const aom_image_t *img, const int planes[3], + } + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static void write_image_file(const aom_image_t *img, const int *planes, + const int num_planes, FILE *file) { + int i, y; +@@ -287,7 +290,7 @@ static void write_image_file(const aom_image_t *img, const int *planes, + const int h = aom_img_plane_height(img, plane); + + for (y = 0; y < h; ++y) { +- fwrite(buf, bytes_per_sample, w, file); ++ emcc_fwrite(buf, bytes_per_sample, w, file); + buf += stride; + } + } +diff --git a/third_party/aom/aomenc.c b/third_party/aom/aomenc.c +index 64155b0..3ed5080 100644 +--- a/third_party/aom/aomenc.c ++++ b/third_party/aom/aomenc.c +@@ -59,9 +59,12 @@ static size_t wrap_fread(void *ptr, size_t size, size_t nmemb, FILE *stream) { + } + #define fread wrap_fread + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static size_t wrap_fwrite(const void *ptr, size_t size, size_t nmemb, + FILE *stream) { +- return fwrite(ptr, size, nmemb, stream); ++ return emcc_fwrite(ptr, size, nmemb, stream); + } + #define fwrite wrap_fwrite + +diff --git a/third_party/aom/aomstats.c b/third_party/aom/aomstats.c +index 0cfeea2..6833776 100644 +--- a/third_party/aom/aomstats.c ++++ b/third_party/aom/aomstats.c +@@ -80,9 +80,12 @@ void stats_close(stats_io_t *stats, int last_pass) { + } + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void stats_write(stats_io_t *stats, const void *pkt, size_t len) { + if (stats->file) { +- (void)fwrite(pkt, 1, len, stats->file); ++ (void)emcc_fwrite(pkt, 1, len, stats->file); + } else { + if (stats->buf.sz + len > stats->buf_alloc_sz) { + size_t new_sz = stats->buf_alloc_sz + 64 * 1024; +diff --git a/third_party/aom/av1/common/debugmodes.c b/third_party/aom/av1/common/debugmodes.c +index 868f341..c44258c 100644 +--- a/third_party/aom/av1/common/debugmodes.c ++++ b/third_party/aom/av1/common/debugmodes.c +@@ -89,10 +89,13 @@ void av1_print_modes_and_motion_vectors(AV1_COMMON *cm, const char *file) { + fclose(mvs); + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void av1_print_uncompressed_frame_header(const uint8_t *data, int size, + const char *filename) { + FILE *hdrFile = fopen(filename, "w"); +- fwrite(data, size, sizeof(uint8_t), hdrFile); ++ emcc_fwrite(data, size, sizeof(uint8_t), hdrFile); + fclose(hdrFile); + } + +diff --git a/third_party/aom/av1/encoder/encoder.c b/third_party/aom/av1/encoder/encoder.c +index a557380..d709d26 100644 +--- a/third_party/aom/av1/encoder/encoder.c ++++ b/third_party/aom/av1/encoder/encoder.c +@@ -2799,6 +2799,9 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, + snprintf((H) + strlen(H), sizeof(H) - strlen(H), (T), (V)) + #endif // CONFIG_INTERNAL_STATS + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void av1_remove_compressor(AV1_COMP *cpi) { + AV1_COMMON *cm; + unsigned int i; +@@ -2814,7 +2817,7 @@ void av1_remove_compressor(AV1_COMP *cpi) { + if (cpi->oxcf.pass != 1) { + fprintf(stderr, "Writing counts.stt\n"); + FILE *f = fopen("counts.stt", "wb"); +- fwrite(&aggregate_fc, sizeof(aggregate_fc), 1, f); ++ emcc_fwrite(&aggregate_fc, sizeof(aggregate_fc), 1, f); + fclose(f); + } + #endif // CONFIG_ENTROPY_STATS +@@ -3013,7 +3016,7 @@ void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { + int h = s->y_height; + + do { +- fwrite(src, s->y_width, 1, f); ++ emcc_fwrite(src, s->y_width, 1, f); + src += s->y_stride; + } while (--h); + +@@ -3021,7 +3024,7 @@ void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, f); ++ emcc_fwrite(src, s->uv_width, 1, f); + src += s->uv_stride; + } while (--h); + +@@ -3029,7 +3032,7 @@ void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, f); ++ emcc_fwrite(src, s->uv_width, 1, f); + src += s->uv_stride; + } while (--h); + } +@@ -3121,7 +3124,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + uint16_t *src16 = CONVERT_TO_SHORTPTR(s->y_buffer); + + do { +- fwrite(src16, s->y_width, 2, yuv_rec_file); ++ emcc_fwrite(src16, s->y_width, 2, yuv_rec_file); + src16 += s->y_stride; + } while (--h); + +@@ -3129,7 +3132,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src16, s->uv_width, 2, yuv_rec_file); ++ emcc_fwrite(src16, s->uv_width, 2, yuv_rec_file); + src16 += s->uv_stride; + } while (--h); + +@@ -3137,7 +3140,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src16, s->uv_width, 2, yuv_rec_file); ++ emcc_fwrite(src16, s->uv_width, 2, yuv_rec_file); + src16 += s->uv_stride; + } while (--h); + +@@ -3146,7 +3149,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + } + + do { +- fwrite(src, s->y_width, 1, yuv_rec_file); ++ emcc_fwrite(src, s->y_width, 1, yuv_rec_file); + src += s->y_stride; + } while (--h); + +@@ -3154,7 +3157,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, yuv_rec_file); ++ emcc_fwrite(src, s->uv_width, 1, yuv_rec_file); + src += s->uv_stride; + } while (--h); + +@@ -3162,7 +3165,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, yuv_rec_file); ++ emcc_fwrite(src, s->uv_width, 1, yuv_rec_file); + src += s->uv_stride; + } while (--h); + +@@ -3241,16 +3244,16 @@ static int dump_one_image(AV1_COMMON *cm, + + // --- Y --- + for (h = 0; h < cm->height; ++h) { +- fwrite(&ref_buf->y_buffer[h * ref_buf->y_stride], 1, cm->width, f_ref); ++ emcc_fwrite(&ref_buf->y_buffer[h * ref_buf->y_stride], 1, cm->width, f_ref); + } + // --- U --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&ref_buf->u_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&ref_buf->u_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), + f_ref); + } + // --- V --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&ref_buf->v_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&ref_buf->v_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), + f_ref); + } + +@@ -4692,17 +4695,17 @@ static void dump_filtered_recon_frames(AV1_COMP *cpi) { + + // --- Y --- + for (h = 0; h < cm->height; ++h) { +- fwrite(&recon_buf->y_buffer[h * recon_buf->y_stride], 1, cm->width, ++ emcc_fwrite(&recon_buf->y_buffer[h * recon_buf->y_stride], 1, cm->width, + f_recon); + } + // --- U --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&recon_buf->u_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&recon_buf->u_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), + f_recon); + } + // --- V --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&recon_buf->v_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&recon_buf->v_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), + f_recon); + } + +diff --git a/third_party/aom/av1/encoder/firstpass.c b/third_party/aom/av1/encoder/firstpass.c +index bb73fde..b963043 100644 +--- a/third_party/aom/av1/encoder/firstpass.c ++++ b/third_party/aom/av1/encoder/firstpass.c +@@ -476,6 +476,9 @@ static double raw_motion_error_stdev(int *raw_motion_err_list, + return raw_err_stdev; + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + #define UL_INTRA_THRESH 50 + #define INVALID_ROW -1 + void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { +@@ -1077,7 +1080,7 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { + else + recon_file = fopen(filename, "ab"); + +- (void)fwrite(lst_yv12->buffer_alloc, lst_yv12->frame_size, 1, recon_file); ++ (void)emcc_fwrite(lst_yv12->buffer_alloc, lst_yv12->frame_size, 1, recon_file); + fclose(recon_file); + } + +diff --git a/third_party/aom/build/cmake/aom_configure.cmake b/third_party/aom/build/cmake/aom_configure.cmake +index 9220a32..fb8bf9f 100644 +--- a/third_party/aom/build/cmake/aom_configure.cmake ++++ b/third_party/aom/build/cmake/aom_configure.cmake +@@ -260,7 +260,7 @@ if(MSVC) + add_compiler_flag_if_supported("/WX") + endif() + else() +- require_c_flag("-std=c99" YES) ++ #require_c_flag("-std=c99" YES) + add_compiler_flag_if_supported("-Wall") + add_compiler_flag_if_supported("-Wdisabled-optimization") + add_compiler_flag_if_supported("-Wextra") +diff --git a/third_party/aom/examples/resize_util.c b/third_party/aom/examples/resize_util.c +index 5485691..e60ed86 100644 +--- a/third_party/aom/examples/resize_util.c ++++ b/third_party/aom/examples/resize_util.c +@@ -45,6 +45,9 @@ static int parse_dim(char *v, int *width, int *height) { + return 1; + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + int main(int argc, char *argv[]) { + char *fin, *fout; + FILE *fpin, *fpout; +@@ -111,7 +114,7 @@ int main(int argc, char *argv[]) { + av1_resize_frame420(inbuf, width, inbuf_u, inbuf_v, width / 2, height, + width, outbuf, target_width, outbuf_u, outbuf_v, + target_width / 2, target_height, target_width); +- fwrite(outbuf, target_width * target_height * 3 / 2, 1, fpout); ++ emcc_fwrite(outbuf, target_width * target_height * 3 / 2, 1, fpout); + f++; + } + printf("%d frames processed\n", f); +diff --git a/third_party/aom/examples/scalable_encoder.c b/third_party/aom/examples/scalable_encoder.c +index 10d647e..fcf31e1 100644 +--- a/third_party/aom/examples/scalable_encoder.c ++++ b/third_party/aom/examples/scalable_encoder.c +@@ -91,6 +91,9 @@ void usage_exit(void) { + exit(EXIT_FAILURE); + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + int frame_index, int flags, FILE *outfile) { + int got_pkts = 0; +@@ -105,7 +108,7 @@ static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; +- if (fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile) != ++ if (emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile) != + pkt->data.frame.sz) { + die_codec(codec, "Failed to write compressed frame"); + } +diff --git a/third_party/aom/ivfenc.c b/third_party/aom/ivfenc.c +index 80f4d14..d0e4e34 100644 +--- a/third_party/aom/ivfenc.c ++++ b/third_party/aom/ivfenc.c +@@ -14,6 +14,9 @@ + #include "aom/aom_encoder.h" + #include "aom_ports/mem_ops.h" + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void ivf_write_file_header(FILE *outfile, const struct aom_codec_enc_cfg *cfg, + unsigned int fourcc, int frame_cnt) { + char header[32]; +@@ -32,7 +35,7 @@ void ivf_write_file_header(FILE *outfile, const struct aom_codec_enc_cfg *cfg, + mem_put_le32(header + 24, frame_cnt); // length + mem_put_le32(header + 28, 0); // unused + +- fwrite(header, 1, 32, outfile); ++ emcc_fwrite(header, 1, 32, outfile); + } + + void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size) { +@@ -41,12 +44,12 @@ void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size) { + mem_put_le32(header, (int)frame_size); + mem_put_le32(header + 4, (int)(pts & 0xFFFFFFFF)); + mem_put_le32(header + 8, (int)(pts >> 32)); +- fwrite(header, 1, 12, outfile); ++ emcc_fwrite(header, 1, 12, outfile); + } + + void ivf_write_frame_size(FILE *outfile, size_t frame_size) { + char header[4]; + + mem_put_le32(header, (int)frame_size); +- fwrite(header, 1, 4, outfile); ++ emcc_fwrite(header, 1, 4, outfile); + } +diff --git a/third_party/aom/test/decode_perf_test.cc b/third_party/aom/test/decode_perf_test.cc +index 3c93e7d..2d364ae 100644 +--- a/third_party/aom/test/decode_perf_test.cc ++++ b/third_party/aom/test/decode_perf_test.cc +@@ -24,6 +24,11 @@ + + using ::testing::make_tuple; + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace { + + #define VIDEO_NAME 0 +@@ -153,7 +158,7 @@ class AV1NewEncodeDecodePerfTest + + // Write frame header and data. + ivf_write_frame_header(outfile_, out_frames_, pkt->data.frame.sz); +- ASSERT_EQ(fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_), ++ ASSERT_EQ(emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_), + pkt->data.frame.sz); + } + +diff --git a/third_party/aom/test/film_grain_table_test.cc b/third_party/aom/test/film_grain_table_test.cc +index 0688146..dbb8e6b 100644 +--- a/third_party/aom/test/film_grain_table_test.cc ++++ b/third_party/aom/test/film_grain_table_test.cc +@@ -5,6 +5,11 @@ + #include "av1/encoder/grain_test_vectors.h" + #include "test/video_source.h" + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + void grain_equal(const aom_film_grain_t *expected, + const aom_film_grain_t *actual) { + EXPECT_EQ(expected->apply_grain, actual->apply_grain); +@@ -168,7 +173,7 @@ TEST_F(FilmGrainTableIOTest, ReadTruncatedFile) { + + std::string grain_file; + FILE *file = libaom_test::GetTempOutFile(&grain_file); +- fwrite("deadbeef", 8, 1, file); ++ emcc_fwrite("deadbeef", 8, 1, file); + fclose(file); + ASSERT_EQ(AOM_CODEC_ERROR, + aom_film_grain_table_read(&table, grain_file.c_str(), &error_)); +diff --git a/third_party/aom/test/resize_test.cc b/third_party/aom/test/resize_test.cc +index e1c4e9f..9c2bce8 100644 +--- a/third_party/aom/test/resize_test.cc ++++ b/third_party/aom/test/resize_test.cc +@@ -22,6 +22,11 @@ + // Enable(1) or Disable(0) writing of the compressed bitstream. + #define WRITE_COMPRESSED_STREAM 0 + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace { + + #if WRITE_COMPRESSED_STREAM +@@ -55,13 +60,13 @@ static void write_ivf_file_header(const aom_codec_enc_cfg_t *const cfg, + mem_put_le32(header + 24, frame_cnt); /* length */ + mem_put_le32(header + 28, 0); /* unused */ + +- (void)fwrite(header, 1, 32, outfile); ++ (void)emcc_fwrite(header, 1, 32, outfile); + } + + static void write_ivf_frame_size(FILE *const outfile, const size_t size) { + char header[4]; + mem_put_le32(header, static_cast(size)); +- (void)fwrite(header, 1, 4, outfile); ++ (void)emcc_fwrite(header, 1, 4, outfile); + } + + static void write_ivf_frame_header(const aom_codec_cx_pkt_t *const pkt, +@@ -76,7 +81,7 @@ static void write_ivf_frame_header(const aom_codec_cx_pkt_t *const pkt, + mem_put_le32(header + 4, pts & 0xFFFFFFFF); + mem_put_le32(header + 8, pts >> 32); + +- (void)fwrite(header, 1, 12, outfile); ++ (void)emcc_fwrite(header, 1, 12, outfile); + } + #endif // WRITE_COMPRESSED_STREAM + +@@ -309,7 +314,7 @@ class ResizeInternalTestLarge : public ResizeTest { + + // Write frame header and data. + write_ivf_frame_header(pkt, outfile_); +- (void)fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); ++ (void)emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); + } + #endif + +@@ -608,7 +613,7 @@ class ResizeCspTest : public ResizeTest { + + // Write frame header and data. + write_ivf_frame_header(pkt, outfile_); +- (void)fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); ++ (void)emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); + } + #endif + +diff --git a/third_party/aom/test/y4m_test.cc b/third_party/aom/test/y4m_test.cc +index ad901d9..f24093f 100644 +--- a/third_party/aom/test/y4m_test.cc ++++ b/third_party/aom/test/y4m_test.cc +@@ -19,6 +19,11 @@ + #include "test/util.h" + #include "test/y4m_video_source.h" + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace { + + using std::string; +@@ -68,7 +73,7 @@ static void write_image_file(const aom_image_t *img, FILE *file) { + (plane ? (img->d_w + img->x_chroma_shift) >> img->x_chroma_shift + : img->d_w); + for (y = 0; y < h; ++y) { +- fwrite(buf, bytes_per_sample, w, file); ++ emcc_fwrite(buf, bytes_per_sample, w, file); + buf += stride; + } + } +diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc +index 5a8932c..ac2c435 100644 +--- a/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc ++++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc +@@ -146,6 +146,11 @@ + # define vsnprintf _vsnprintf + #endif // GTEST_OS_WINDOWS + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace testing { + + using internal::CountIf; +@@ -3867,7 +3872,7 @@ class ScopedPrematureExitFile { + // errors are ignored as there's nothing better we can do and we + // don't want to fail the test because of this. + FILE* pfile = posix::FOpen(premature_exit_filepath, "w"); +- fwrite("0", 1, 1, pfile); ++ emcc_fwrite("0", 1, 1, pfile); + fclose(pfile); + } + } +diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc +index 84655d8..0004093 100644 +--- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc ++++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc +@@ -14,6 +14,11 @@ + #include // for _SH_DENYWR + #endif + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace mkvmuxer { + + MkvWriter::MkvWriter() : file_(NULL), writer_owns_file_(true) {} +@@ -32,7 +37,7 @@ int32 MkvWriter::Write(const void* buffer, uint32 length) { + if (buffer == NULL) + return -1; + +- const size_t bytes_written = fwrite(buffer, 1, length, file_); ++ const size_t bytes_written = emcc_fwrite(buffer, 1, length, file_); + + return (bytes_written == length) ? 0 : -1; + } +diff --git a/third_party/aom/tools_common.c b/third_party/aom/tools_common.c +index 7abc20c..fbc30bc 100644 +--- a/third_party/aom/tools_common.c ++++ b/third_party/aom/tools_common.c +@@ -185,6 +185,9 @@ const AvxInterface *get_aom_decoder_by_fourcc(uint32_t fourcc) { + } + #endif // CONFIG_AV1_DECODER + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void aom_img_write(const aom_image_t *img, FILE *file) { + int plane; + +@@ -197,7 +200,7 @@ void aom_img_write(const aom_image_t *img, FILE *file) { + int y; + + for (y = 0; y < h; ++y) { +- fwrite(buf, 1, w, file); ++ emcc_fwrite(buf, 1, w, file); + buf += stride; + } + } +diff --git a/third_party/aom/video_writer.c b/third_party/aom/video_writer.c +index 4e072c7..6b1ca54 100644 +--- a/third_party/aom/video_writer.c ++++ b/third_party/aom/video_writer.c +@@ -66,10 +66,13 @@ void aom_video_writer_close(AvxVideoWriter *writer) { + } + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + int aom_video_writer_write_frame(AvxVideoWriter *writer, const uint8_t *buffer, + size_t size, int64_t pts) { + ivf_write_frame_header(writer->file, pts, size); +- if (fwrite(buffer, 1, size, writer->file) != size) return 0; ++ if (emcc_fwrite(buffer, 1, size, writer->file) != size) return 0; + + ++writer->frame_count; + diff --git a/wasm-micro-runtime/test-tools/.gitignore b/wasm-micro-runtime/test-tools/.gitignore new file mode 100644 index 0000000..6aa8dc0 --- /dev/null +++ b/wasm-micro-runtime/test-tools/.gitignore @@ -0,0 +1 @@ +/wasi-sdk diff --git a/wasm-micro-runtime/test-tools/addr2line/addr2line.py b/wasm-micro-runtime/test-tools/addr2line/addr2line.py new file mode 100644 index 0000000..421b0bd --- /dev/null +++ b/wasm-micro-runtime/test-tools/addr2line/addr2line.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +import argparse +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys + +""" +This is a tool to convert addresses, which are from a call-stack dump generated by iwasm, into line info for a wasm file. + +When a wasm file is compiled with debug info, it is possible to transfer the address to line info. + +For example, there is a call-stack dump: + +``` +#00: 0x0a04 - $f18 +#01: 0x08e4 - $f11 +#02: 0x096f - $f12 +#03: 0x01aa - _start +``` + +- store the call-stack dump into a file, e.g. call_stack.txt +- run the following command to convert the address into line info: + ``` + $ cd test-tools/addr2line + $ python3 addr2line.py --wasi-sdk --wabt --wasm-file call_stack.txt + ``` + The script will use *wasm-objdump* in wabt to transform address, then use *llvm-dwarfdump* to lookup the line info for each address + in the call-stack dump. +- if addresses are not available in the stack trace (i.e. iwasm <= 1.3.2) or iwasm is used in fast interpreter mode, + run the following command to convert the function index into line info (passing the `--no-addr` option): + ``` + $ python3 addr2line.py --wasi-sdk --wabt --wasm-file call_stack.txt --no-addr + ``` + The script will use *wasm-objdump* in wabt to get the function names corresponding to function indexes, then use *llvm-dwarfdump* to lookup the line info for each + function index in the call-stack dump. +""" + + +def locate_sourceMappingURL_section(wasm_objdump: Path, wasm_file: Path) -> bool: + """ + Figure out if the wasm file has a sourceMappingURL section. + """ + cmd = f"{wasm_objdump} -h {wasm_file}" + p = subprocess.run( + shlex.split(cmd), + check=True, + capture_output=True, + text=True, + universal_newlines=True, + ) + outputs = p.stdout.split(os.linesep) + + for line in outputs: + line = line.strip() + if "sourceMappingURL" in line: + return True + + return False + + +def get_code_section_start(wasm_objdump: Path, wasm_file: Path) -> int: + """ + Find the start offset of Code section in a wasm file. + + if the code section header likes: + Code start=0x0000017c end=0x00004382 (size=0x00004206) count: 47 + + the start offset is 0x0000017c + """ + cmd = f"{wasm_objdump} -h {wasm_file}" + p = subprocess.run( + shlex.split(cmd), + check=True, + capture_output=True, + text=True, + universal_newlines=True, + ) + outputs = p.stdout.split(os.linesep) + + for line in outputs: + line = line.strip() + if "Code" in line: + return int(line.split()[1].split("=")[1], 16) + + return -1 + + +def get_line_info_from_function_addr_dwarf( + dwarf_dump: Path, wasm_file: Path, offset: int +) -> tuple[str, str, str, str]: + """ + Find the location info of a given offset in a wasm file. + """ + cmd = f"{dwarf_dump} --lookup={offset} {wasm_file}" + p = subprocess.run( + shlex.split(cmd), + check=False, + capture_output=True, + text=True, + universal_newlines=True, + ) + outputs = p.stdout.split(os.linesep) + + function_name, function_file = "", "unknown" + function_line, function_column = "?", "?" + + for line in outputs: + line = line.strip() + + if "DW_AT_name" in line: + function_name = get_dwarf_tag_value("DW_AT_name", line) + + if "DW_AT_decl_file" in line: + function_file = get_dwarf_tag_value("DW_AT_decl_file", line) + + if "Line info" in line: + _, function_line, function_column = parse_line_info(line) + + return (function_name, function_file, function_line, function_column) + + +def get_dwarf_tag_value(tag: str, line: str) -> str: + # Try extracting value as string + STR_PATTERN = rf"{tag}\s+\(\"(.*)\"\)" + m = re.match(STR_PATTERN, line) + if m: + return m.groups()[0] + + # Try extracting value as integer + INT_PATTERN = rf"{tag}\s+\((\d+)\)" + m = re.match(INT_PATTERN, line) + return m.groups()[0] + + +def get_line_info_from_function_name_dwarf( + dwarf_dump: Path, wasm_file: Path, function_name: str +) -> tuple[str, str, str]: + """ + Find the location info of a given function in a wasm file. + """ + cmd = f"{dwarf_dump} --name={function_name} {wasm_file}" + p = subprocess.run( + shlex.split(cmd), + check=False, + capture_output=True, + text=True, + universal_newlines=True, + ) + outputs = p.stdout.split(os.linesep) + + function_name, function_file = "", "unknown" + function_line = "?" + + for line in outputs: + line = line.strip() + + if "DW_AT_name" in line: + function_name = get_dwarf_tag_value("DW_AT_name", line) + + if "DW_AT_decl_file" in line: + function_file = get_dwarf_tag_value("DW_AT_decl_file", line) + + if "DW_AT_decl_line" in line: + function_line = get_dwarf_tag_value("DW_AT_decl_line", line) + + return (function_name, function_file, function_line) + + +def get_line_info_from_function_addr_sourcemapping( + emsymbolizer: Path, wasm_file: Path, offset: int +) -> tuple[str, str, str, str]: + """ + Find the location info of a given offset in a wasm file which is compiled with emcc. + + {emsymbolizer} {wasm_file} {offset of file} + + there usually are two lines: + ?? + relative path to source file:line:column + """ + debug_info_source = wasm_file.with_name(f"{wasm_file.name}.map") + cmd = f"{emsymbolizer} -t code -f {debug_info_source} {wasm_file} {offset}" + p = subprocess.run( + shlex.split(cmd), + check=False, + capture_output=True, + text=True, + universal_newlines=True, + cwd=Path.cwd(), + ) + outputs = p.stdout.split(os.linesep) + + function_name, function_file = "", "unknown" + function_line, function_column = "?", "?" + + for line in outputs: + line = line.strip() + + if not line: + continue + + m = re.match("(.*):(\d+):(\d+)", line) + if m: + function_file, function_line, function_column = m.groups() + continue + else: + # it's always ??, not sure about that + if "??" != line: + function_name = line + + return (function_name, function_file, function_line, function_column) + + +def parse_line_info(line_info: str) -> tuple[str, str, str]: + """ + line_info -> [file, line, column] + """ + PATTERN = r"Line info: file \'(.+)\', line ([0-9]+), column ([0-9]+)" + m = re.search(PATTERN, line_info) + assert m is not None + + file, line, column = m.groups() + return (file, int(line), int(column)) + + +def parse_call_stack_line(line: str) -> tuple[str, str, str]: + """ + New format (WAMR > 1.3.2): + #00: 0x0a04 - $f18 => (00, 0x0a04, $f18) + Old format: + #00 $f18 => (00, _, $f18) + Text format (-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1 -DWAMR_BUILD_CUSTOM_NAME_SECTION=1): + #02: 0x0200 - a => (02, 0x0200, a) + _start (always): + #05: 0x011f - _start => (05, 0x011f, _start) + """ + + # New format and Text format and _start + PATTERN = r"#([0-9]+): 0x([0-9a-f]+) - (\S+)" + m = re.match(PATTERN, line) + if m is not None: + return m.groups() + + # Old format + PATTERN = r"#([0-9]+) (\S+)" + m = re.match(PATTERN, line) + if m is not None: + return (m.groups()[0], None, m.groups()[1]) + + return None + + +def parse_module_functions(wasm_objdump: Path, wasm_file: Path) -> dict[str, str]: + function_index_to_name = {} + + cmd = f"{wasm_objdump} -x {wasm_file} --section=function" + p = subprocess.run( + shlex.split(cmd), + check=True, + capture_output=True, + text=True, + universal_newlines=True, + ) + outputs = p.stdout.split(os.linesep) + + for line in outputs: + if not f"func[" in line: + continue + + PATTERN = r".*func\[([0-9]+)\].*<(.*)>" + m = re.match(PATTERN, line) + assert m is not None + + index = m.groups()[0] + name = m.groups()[1] + function_index_to_name[index] = name + + return function_index_to_name + + +def demangle(cxxfilt: Path, function_name: str) -> str: + cmd = f"{cxxfilt} -n {function_name}" + p = subprocess.run( + shlex.split(cmd), + check=True, + capture_output=True, + text=True, + universal_newlines=True, + ) + return p.stdout.strip() + + +def main(): + parser = argparse.ArgumentParser(description="addr2line for wasm") + parser.add_argument("--wasi-sdk", type=Path, help="path to wasi-sdk") + parser.add_argument("--wabt", type=Path, help="path to wabt") + parser.add_argument("--wasm-file", type=Path, help="path to wasm file") + parser.add_argument("call_stack_file", type=Path, help="path to a call stack file") + parser.add_argument( + "--no-addr", + action="store_true", + help="use call stack without addresses or from fast interpreter mode", + ) + parser.add_argument("--emsdk", type=Path, help="path to emsdk") + args = parser.parse_args() + + wasm_objdump = args.wabt.joinpath("bin/wasm-objdump") + assert wasm_objdump.exists() + + llvm_dwarf_dump = args.wasi_sdk.joinpath("bin/llvm-dwarfdump") + assert llvm_dwarf_dump.exists() + + llvm_cxxfilt = args.wasi_sdk.joinpath("bin/llvm-cxxfilt") + assert llvm_cxxfilt.exists() + + emcc_production = locate_sourceMappingURL_section(wasm_objdump, args.wasm_file) + if emcc_production: + if args.emsdk is None: + print("Please provide the path to emsdk via --emsdk") + return -1 + + emsymbolizer = args.emsdk.joinpath("upstream/emscripten/emsymbolizer") + assert emsymbolizer.exists() + + code_section_start = get_code_section_start(wasm_objdump, args.wasm_file) + if code_section_start == -1: + return -1 + + function_index_to_name = parse_module_functions(wasm_objdump, args.wasm_file) + + assert args.call_stack_file.exists() + with open(args.call_stack_file, "rt", encoding="ascii") as f: + for i, line in enumerate(f): + line = line.strip() + if not line: + continue + + splitted = parse_call_stack_line(line) + if splitted is None: + print(f"{line}") + continue + + _, offset, index = splitted + if args.no_addr: + # FIXME: w/ emcc production + if not index.startswith("$f"): # E.g. _start or Text format + print(f"{i}: {index}") + continue + index = index[2:] + + if index not in function_index_to_name: + print(f"{i}: {line}") + continue + + if not emcc_production: + _, function_file, function_line = ( + get_line_info_from_function_name_dwarf( + llvm_dwarf_dump, + args.wasm_file, + function_index_to_name[index], + ) + ) + else: + _, function_file, function_line = _, "unknown", "?" + + function_name = demangle(llvm_cxxfilt, function_index_to_name[index]) + print(f"{i}: {function_name}") + print(f"\tat {function_file}:{function_line}") + else: + offset = int(offset, 16) + # match the algorithm in wasm_interp_create_call_stack() + # either a *offset* to *code* section start + # or a *offset* in a file + assert offset > code_section_start + offset = offset - code_section_start + + if emcc_production: + function_name, function_file, function_line, function_column = ( + get_line_info_from_function_addr_sourcemapping( + emsymbolizer, args.wasm_file, offset + ) + ) + else: + function_name, function_file, function_line, function_column = ( + get_line_info_from_function_addr_dwarf( + llvm_dwarf_dump, args.wasm_file, offset + ) + ) + + # if can't parse function_name, use name section or + if function_name == "": + if index.startswith("$f"): + function_name = function_index_to_name.get(index[2:], index) + else: + function_name = index + + function_name = demangle(llvm_cxxfilt, function_name) + + print(f"{i}: {function_name}") + print(f"\tat {function_file}:{function_line}:{function_column}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wasm-micro-runtime/test-tools/append-aot-to-wasm/append_aot_to_wasm.py b/wasm-micro-runtime/test-tools/append-aot-to-wasm/append_aot_to_wasm.py new file mode 100644 index 0000000..4829fbb --- /dev/null +++ b/wasm-micro-runtime/test-tools/append-aot-to-wasm/append_aot_to_wasm.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +""" +It is used to append a .aot to a .wasm as a custom section. +The custom section name is "aot". + +e.g. +$ python3 append_aot_to_wasm.py --wasm quicksort.wasm --aot quicksort.aot --output quicksort.aot.wasm +""" + +import argparse +from pathlib import Path + + +def leb128_encode_uint(value: int) -> bytes: + """ + encode unsigned int into a leb128 bytes + """ + binary = [] + while value != 0: + lower_7_bits = value & 0x7F + value >>= 7 + + if value != 0: + current_byte = 0x80 | lower_7_bits + else: + current_byte = 0x00 | lower_7_bits + + binary.append(current_byte) + + return bytes(binary) + + +def leb128_decode_uint(binary: bytes) -> (int, int): + """ + decode binary unsigned from a leb128 bytes + """ + + result = 0 + shift = 0 + for i, b in enumerate(binary): + lower_7_bits = b & 0x7F + result |= lower_7_bits << shift + + highest_bit = b & 0x80 + if not highest_bit: + break + + shift += 7 + + return i + 1, result + + +def is_aligned(n: int, alignment: int): + return (n & (alignment - 1)) == 0 + + +def align_up(n: int, alignment: int): + return n + (alignment - 1) & ~(alignment - 1) + + +def present_as_vector(content: bytes) -> bytes: + v_l = len(content) + v_bin = leb128_encode_uint(v_l) if v_l else b"\x00" + return v_bin + content + + +def calc_padding( + alignment: int, name_bin_len: int, content_len: int, start_pos: int +) -> bytes: + for padding in range(alignment * 2): + padding_bin = present_as_vector(b"\x00" * padding) + section_length = name_bin_len + len(padding_bin) + content_len + section_length_bin = leb128_encode_uint(section_length) + + pos = start_pos + 1 + len(section_length_bin) + name_bin_len + len(padding_bin) + if is_aligned(pos, alignment): + return padding_bin + + +def build_content(content: bytes, pos: int, adding: bytes) -> (int, bytes): + return pos + len(adding), content + adding + + +def create_custom_section_aligned( + start_pos: int, name: str, content: bytes, alignment: int = 4 +) -> bytes: + """ + be sure the section_content starts at a X alignment position + + 1B + | \x00 | length | name vec | padding vec | content | + ^ ^ + | | + start address aligned address + """ + + name_bin = present_as_vector(name.encode("ascii")) + padding_bin = calc_padding(alignment, len(name_bin), len(content), start_pos) + + full_content_bin = b"" + pos = start_pos + + # custome section id 0 + pos, full_content_bin = build_content(full_content_bin, pos, b"\x00") + + # custom section length + section_length = len(name_bin) + len(padding_bin) + len(content) + section_length_bin = leb128_encode_uint(section_length) + pos, full_content_bin = build_content(full_content_bin, pos, section_length_bin) + + # custom section name + pos, full_content_bin = build_content(full_content_bin, pos, name_bin) + + # padding + pos, full_content_bin = build_content(full_content_bin, pos, padding_bin) + assert is_aligned(pos, alignment), f"{pos} is not aligned to {alignment}" + + print(f"append .aot @ offset {pos}(0x{pos:X})") + _, full_content_bin = build_content(full_content_bin, pos, content) + + return full_content_bin + + +def main(wasm_file: str, aot_file: str, output: str) -> None: + cwd = Path.cwd() + wasm_file = cwd.joinpath(wasm_file).resolve() + aot_file = cwd.joinpath(aot_file).resolve() + output = cwd.joinpath(output).resolve() + + assert wasm_file.exists() + assert aot_file.exists() + output.unlink(missing_ok=True) + + # read aot content + with open(aot_file, "rb") as f: + aot_content = f.read() + + # append to .wasm + with open(wasm_file, "rb") as f_in, open(output, "wb") as f_out: + wasm_content = f_in.read(1024) + while wasm_content: + f_out.write(wasm_content) + wasm_content = f_in.read(1024) + + f_out.write(create_custom_section_aligned(f_out.tell(), "aot", aot_content, 4)) + + print(f"{wasm_file.name} + {aot_file.name} ==> {output}") + + +if __name__ == "__main__": + argparse = argparse.ArgumentParser() + argparse.add_argument("--wasm", help="a .wasm") + argparse.add_argument("--aot", help="a .aot") + argparse.add_argument("-o", "--output", help="the output, still be a .wasm") + + args = argparse.parse_args() + main(args.wasm, args.aot, args.output) diff --git a/wasm-micro-runtime/test-tools/binarydump-tool/CMakeLists.txt b/wasm-micro-runtime/test-tools/binarydump-tool/CMakeLists.txt new file mode 100644 index 0000000..d322b42 --- /dev/null +++ b/wasm-micro-runtime/test-tools/binarydump-tool/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project(binarydump) + +add_definitions (-Wextra -pedantic -Wno-unused-parameter) + +add_executable (binarydump binarydump.c) + diff --git a/wasm-micro-runtime/test-tools/binarydump-tool/binarydump.c b/wasm-micro-runtime/test-tools/binarydump-tool/binarydump.c new file mode 100644 index 0000000..050de6d --- /dev/null +++ b/wasm-micro-runtime/test-tools/binarydump-tool/binarydump.c @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +static unsigned char * +read_file_to_buffer(const char *filename, int *ret_size) +{ + unsigned char *buffer; + FILE *file; + int file_size, read_size; + + if (!(file = fopen(filename, "r"))) + return NULL; + + fseek(file, 0, SEEK_END); + file_size = ftell(file); + fseek(file, 0, SEEK_SET); + + if (!(buffer = malloc(file_size))) { + fclose(file); + return NULL; + } + + read_size = fread(buffer, 1, file_size, file); + fclose(file); + + if (read_size < file_size) { + free(buffer); + return NULL; + } + + *ret_size = file_size; + + return buffer; +} + +static int +print_help() +{ + printf("Usage: binarydump -o -n input_file\n"); + printf("Options:\n"); + printf(" -o Place the output into \n"); + printf(" -n The name of array \n"); + + return -1; +} + +static bool +bin_file_dump(const unsigned char *file, int size, const char *bin_file_output, + const char *array_name) +{ + unsigned i = 0; + const unsigned char *p = file, *p_end = file + size; + FILE *file_output = fopen(bin_file_output, "wb+"); + + if (!file_output) + return false; + + fprintf(file_output, "\nunsigned char __aligned(4) %s[] = {\n ", + array_name); + + while (p < p_end) { + fprintf(file_output, "0x%02X", *p++); + + if (p == p_end) + break; + + fprintf(file_output, ","); + + if ((++i % 12) != 0) + fprintf(file_output, " "); + else + fprintf(file_output, "\n "); + } + + fprintf(file_output, "\n};\n"); + + fclose(file_output); + return true; +} + +int +main(int argc, char *argv[]) +{ + unsigned char *file; + int size; + bool ret; + const char *bin_file_input, *array_file_output = NULL, *array_name = NULL; + + for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { + if (!strcmp(argv[0], "-o")) { + ++argv; + if (--argc == 0) + return print_help(); + array_file_output = *argv; + } + else if (!strcmp(argv[0], "-n")) { + ++argv; + if (--argc == 0) + return print_help(); + array_name = *argv; + } + else + return print_help(); + } + + if (!array_file_output || !array_name) + return print_help(); + + bin_file_input = *argv; + + if (!(file = read_file_to_buffer(bin_file_input, &size))) + return -1; + + ret = bin_file_dump(file, size, array_file_output, array_name); + + free(file); + + return ret ? 0 : -1; +} diff --git a/wasm-micro-runtime/test-tools/flame-graph-helper/process_folded_data.py b/wasm-micro-runtime/test-tools/flame-graph-helper/process_folded_data.py new file mode 100644 index 0000000..e4650fe --- /dev/null +++ b/wasm-micro-runtime/test-tools/flame-graph-helper/process_folded_data.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +""" +It is used to process *out.folded* file generated by [FlameGraph](https://github.com/brendangregg/FlameGraph). + +- translate jitted function names, which are in a form like `aot_func#N` or `[module name]#aot_func#N`, into corresponding names in a name section in .wasm +- divide the translated functions into different modules if the module name is specified in the symbol + +Usage: + +After +``` bash +# collect profiling data in perf.data + +$ perf script -i perf.data > out.perf + +$ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded +``` + +Use this script to translate the function names in out.folded + +``` +$ python translate_wasm_function_name.py --wabt_home --folded out.folded <.wasm> +# out.folded -> out.folded.translated +``` + +""" + +import argparse +import os +from pathlib import Path +import re +import shlex +import subprocess +from typing import Dict, List + + +# parse arguments like "foo=bar,fiz=biz" into a dictatory {foo:bar,fiz=biz} +class ParseKVArgs(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, dict()) + for value in values.split(","): + k, v = value.split("=") + getattr(namespace, self.dest)[k] = v + + +def calculate_import_function_count( + wasm_objdump_bin: Path, module_names: Dict[str, Path] +) -> Dict[str, int]: + """ + for every wasm file in , calculate the number of functions in the import section. + + using " -j Import -x " + """ + + assert wasm_objdump_bin.exists() + + import_function_counts = {} + for module_name, wasm_path in module_names.items(): + assert wasm_path.exists() + command = f"{wasm_objdump_bin} -j Import -x {wasm_path}" + p = subprocess.run( + shlex.split(command), + capture_output=True, + check=False, + text=True, + universal_newlines=True, + ) + + if p.stderr: + print("No content in import section") + import_function_counts[module_name] = 0 + continue + + import_function_count = 0 + for line in p.stdout.split(os.linesep): + line = line.strip() + + if not line: + continue + + if not " func" in line: + continue + + m = re.search(r"^-\s+func", line) + assert m + + import_function_count += 1 + + # print(f"! there are {import_function_count} import function in {module_name}") + import_function_counts[module_name] = import_function_count + + return import_function_counts + + +def collect_name_section_content( + wasm_objdump_bin: Path, module_names: Dict[str, Path] +) -> Dict[str, Dict[int, str]]: + """ + for every wasm file in , get the content of name section. + + execute "wasm_objdump_bin -j name -x wasm_file" + """ + assert wasm_objdump_bin.exists() + + name_sections = {} + for module_name, wasm_path in module_names.items(): + assert wasm_path.exists() + command = f"{wasm_objdump_bin} -j name -x {wasm_path}" + p = subprocess.run( + shlex.split(command), + capture_output=True, + check=False, + text=True, + universal_newlines=True, + ) + + if p.stderr: + print("No content in name section") + name_sections[module_name] = {} + continue + + name_section = {} + for line in p.stdout.split(os.linesep): + line = line.strip() + + if not line: + continue + + if not " func" in line: + continue + + # - func[N] <__imported_wasi_snapshot_preview1_fd_close> + m = re.match(r"- func\[(\d+)\] <(.+)>", line) + assert m + + func_index, func_name = m.groups() + name_section.update({int(func_index): func_name}) + + name_sections[module_name] = name_section + + return name_sections + + +def is_stack_check_mode(folded: Path) -> bool: + """ + check if there is a function name looks like "aot_func_internal#N", it means that WAMR adds a stack check function before the original function. + """ + with open(folded, "rt", encoding="utf-8") as f: + for line in f: + line = line.strip() + if "aot_func_internal" in line: + return True + return False + + +def replace_function_name( + import_function_counts: Dict[str, int], + name_sections: Dict[str, Dict[int, str]], + folded_in: Path, + module_names: Dict[str, Path], +) -> None: + """ + read content in . every line contains symbols which are separated by ";". + + Usually, all jitted functions are in the form of "aot_func#N". N is its function index. Use the index to find the corresponding function name in the name section. + + if there is a function name looks like "aot_func_internal#N", it means that WAMR adds a stack check function before the original function. + In this case, "aot_func#N" should be translated with "_precheck" as a suffix and "aot_func_internal#N" should be treated as the original one + """ + + assert folded_in.exists(), f"{folded_in} doesn't exist" + + stack_check_mode = is_stack_check_mode(folded_in) + + # every wasm has a translated out.folded, like out..folded.translated + folded_out_files = {} + for module_name in module_names.keys(): + wasm_folded_out_path = folded_in.with_suffix(f".{module_name}.translated") + print(f"-> write into {wasm_folded_out_path}") + folded_out_files[module_name] = wasm_folded_out_path.open( + "wt", encoding="utf-8" + ) + # Plus a default translated out.folded + default_folded_out_path = folded_in.with_suffix(".translated") + print(f"-> write into {default_folded_out_path}") + default_folded_out = default_folded_out_path.open("wt", encoding="utf-8") + + with folded_in.open("rt", encoding="utf-8") as f_in: + for line in f_in: + line = line.strip() + + m = re.match(r"(.*) (\d+)", line) + assert m + syms, samples = m.groups() + + new_line = [] + last_function_module_name = "" + for sym in syms.split(";"): + if not "aot_func" in sym: + new_line.append(sym) + continue + + # [module_name]#aot_func#N or aot_func#N + splitted = sym.split("#") + module_name = "" if splitted[0] == "aot_func" else splitted[0] + # remove [ and ] + module_name = module_name[1:-1] + + if len(module_name) == 0 and len(module_names) > 1: + raise RuntimeError( + f"❌ {sym} doesn't have a module name, but there are multiple wasm files" + ) + + if not module_name in module_names: + raise RuntimeError( + f"❌ can't find corresponds wasm file for {module_name}" + ) + + last_function_module_name = module_name + + func_idx = int(splitted[-1]) + # adjust index + func_idx = func_idx + import_function_counts[module_name] + + # print(f"🔍 {module_name} {splitted[1]} {func_idx}") + + if func_idx in name_sections[module_name]: + if len(module_name) > 0: + wasm_func_name = f"[Wasm] [{module_name}] {name_sections[module_name][func_idx]}" + else: + wasm_func_name = ( + f"[Wasm] {name_sections[module_name][func_idx]}" + ) + else: + if len(module_name) > 0: + wasm_func_name = f"[Wasm] [{module_name}] func[{func_idx}]" + else: + wasm_func_name = f"[Wasm] func[{func_idx}]" + + if stack_check_mode: + # aot_func_internal -> xxx + # aot_func --> xxx_precheck + if "aot_func" == splitted[1]: + wasm_func_name += "_precheck" + + new_line.append(wasm_func_name) + + line = ";".join(new_line) + line += f" {samples}" + + # always write into the default output + default_folded_out.write(line + os.linesep) + # based on the module name of last function, write into the corresponding output + if len(last_function_module_name) > 0: + folded_out_files[last_function_module_name].write(line + os.linesep) + + default_folded_out.close() + for f in folded_out_files.values(): + f.close() + + +def main(wabt_home: str, folded: str, module_names: Dict[str, Path]) -> None: + wabt_home = Path(wabt_home) + assert wabt_home.exists() + + folded = Path(folded) + assert folded.exists() + + wasm_objdump_bin = wabt_home.joinpath("bin", "wasm-objdump") + import_function_counts = calculate_import_function_count( + wasm_objdump_bin, module_names + ) + + name_sections = collect_name_section_content(wasm_objdump_bin, module_names) + + replace_function_name(import_function_counts, name_sections, folded, module_names) + + +if __name__ == "__main__": + argparse = argparse.ArgumentParser() + argparse.add_argument( + "--wabt_home", required=True, help="wabt home, like /opt/wabt-1.0.33" + ) + argparse.add_argument( + "--wasm", + action="append", + default=[], + help="wasm files for profiling before. like --wasm apple.wasm --wasm banana.wasm", + ) + argparse.add_argument( + "--wasm_names", + action=ParseKVArgs, + default={}, + metavar="module_name=wasm_file, ...", + help="multiple wasm files and their module names, like a=apple.wasm,b=banana.wasm,c=cake.wasm", + ) + argparse.add_argument( + "folded_file", + help="a out.folded generated by flamegraph/stackcollapse-perf.pl", + ) + + args = argparse.parse_args() + + if not args.wasm and not args.wasm_names: + print("Please specify wasm files with either --wasm or --wasm_names") + exit(1) + + # - only one wasm file. And there is no [module name] in out.folded + # - multiple wasm files. via `--wasm X --wasm Y --wasm Z`. And there is [module name] in out.folded. use the basename of wasm as the module name + # - multiple wasm files. via `--wasm_names X=x,Y=y,Z=z`. And there is [module name] in out.folded. use the specified module name + module_names = {} + if args.wasm_names: + for name, wasm_path in args.wasm_names.items(): + module_names[name] = Path(wasm_path) + else: + # use the basename of wasm as the module name + for wasm in args.wasm: + wasm_path = Path(wasm) + module_names[wasm_path.stem] = wasm_path + + main(args.wabt_home, args.folded_file, module_names) diff --git a/wasm-micro-runtime/test-tools/pick-up-emscripten-headers/collect_files.py b/wasm-micro-runtime/test-tools/pick-up-emscripten-headers/collect_files.py new file mode 100755 index 0000000..7e83214 --- /dev/null +++ b/wasm-micro-runtime/test-tools/pick-up-emscripten-headers/collect_files.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +""" +The script operates on such directories and files +|-- core +| `-- deps +| |-- emscripten +|-- samples +| `-- workloads +| |-- include +`-- test-tools + |-- pick-up-emscripten_headers + | |-- collect_files.py +""" + +import argparse +import hashlib +import logging +import os +import pathlib +import shutil +import sys +import tarfile +import tempfile +import urllib +import urllib.request + +logger = logging.getLogger("pick-up-emscripten-headers") + +external_repos = { + "emscripten": { + "sha256": "c5524755b785d8f4b83eb3214fdd3ac4b2e1b1a4644df4c63f06e5968f48f90e", + "store_dir": "core/deps/emscripten", + "strip_prefix": "emscripten-3.0.0", + "url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/3.0.0.tar.gz", + } +} + +# TOOD: can we use headers from wasi-libc and clang directly ? +emscripten_headers_src_dst = [ + ("include/compat/emmintrin.h", "sse/emmintrin.h"), + ("include/compat/immintrin.h", "sse/immintrin.h"), + ("include/compat/smmintrin.h", "sse/smmintrin.h"), + ("include/compat/xmmintrin.h", "sse/xmmintrin.h"), + ("lib/libc/musl/include/pthread.h", "libc/musl/pthread.h"), + ("lib/libc/musl/include/signal.h", "libc/musl/signal.h"), + ("lib/libc/musl/include/netdb.h", "libc/musl/netdb.h"), + ("lib/libc/musl/include/sys/wait.h", "libc/musl/sys/wait.h"), + ("lib/libc/musl/include/sys/socket.h", "libc/musl/sys/socket.h"), + ("lib/libc/musl/include/setjmp.h", "libc/musl/setjmp.h"), + ("lib/libc/musl/arch/emscripten/bits/setjmp.h", "libc/musl/bits/setjmp.h"), +] + + +def checksum(name, local_file): + sha256 = hashlib.sha256() + with open(local_file, "rb") as f: + bytes = f.read(4096) + while bytes: + sha256.update(bytes) + bytes = f.read(4096) + + return sha256.hexdigest() == external_repos[name]["sha256"] + + +def download(url, local_file): + logger.debug(f"download from {url}") + urllib.request.urlretrieve(url, local_file) + return local_file.exists() + + +def unpack(tar_file, strip_prefix, dest_dir): + # extract .tar.gz to /tmp, then move back without strippred prefix directories + with tempfile.TemporaryDirectory() as tmp: + with tarfile.open(tar_file) as tar: + logger.debug(f"extract to {tmp}") + + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + safe_extract(tar, tmp) + + strip_prefix_dir = ( + pathlib.Path(tmp).joinpath(strip_prefix + os.path.sep).resolve() + ) + if not strip_prefix_dir.exists(): + logger.error(f"extract {tar_file.name} failed") + return False + + # mv /tmp/${strip_prefix} dest_dir/* + logger.debug(f"move {strip_prefix_dir} to {dest_dir}") + shutil.copytree( + str(strip_prefix_dir), + str(dest_dir), + copy_function=shutil.move, + dirs_exist_ok=True, + ) + + return True + + +def download_repo(name, root): + if not name in external_repos: + logger.error(f"{name} is not a known repository") + return False + + store_dir = root.joinpath(f'{external_repos[name]["store_dir"]}').resolve() + download_flag = store_dir.joinpath("DOWNLOADED") + if store_dir.exists() and download_flag.exists(): + logger.info( + f"bypass downloading '{store_dir.relative_to(root)}'. Or to remove it and try again if needs a new release" + ) + return True + + # download only when the target is neither existed nor broken + download_dir = pathlib.Path("/tmp/pick-up-emscripten-headers/") + download_dir.mkdir(exist_ok=True) + + tar_name = pathlib.Path(external_repos[name]["url"]).name + tar_file = download_dir.joinpath(tar_name) + if tar_file.exists(): + if checksum(name, tar_file): + logger.debug(f"use pre-downloaded {tar_file}") + else: + logger.debug(f"{tar_file} is broken, remove it") + tar_file.unlink() + + if not tar_file.exists(): + if not download(external_repos[name]["url"], tar_file) or not checksum( + name, tar_file + ): + logger.error(f"download {name} failed") + return False + + # unpack and removing *strip_prefix* + if not unpack(tar_file, external_repos[name]["strip_prefix"], store_dir): + return False + + # leave a FLAG + download_flag.touch() + + # leave download files in /tmp + logger.info(f"Has downloaed and stored in {store_dir.relative_to(root)}") + return True + + +def collect_headers(root, install_location): + if not install_location.exists(): + logger.error(f"{install_location} does not found") + return False + + install_flag = install_location.joinpath("INSTALLED").resolve() + if install_flag.exists(): + logger.info( + f"bypass downloading '{install_location}'. Or to remove it and try again if needs a new one" + ) + return True + + emscripten_home = root.joinpath( + f'{external_repos["emscripten"]["store_dir"]}' + ).resolve() + if not emscripten_home.exists(): + logger.error(f"{emscripten_home} does not found") + return False + + emscripten_headers = emscripten_home.joinpath("system").resolve() + for (src, dst) in emscripten_headers_src_dst: + src = emscripten_headers.joinpath(src) + dst = install_location.joinpath(dst) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(src, dst) + + install_flag.touch() + logger.info(f"Has installed in {install_location}") + return True + + +def main(): + parser = argparse.ArgumentParser( + description="collect headers from emscripten for workload compilation" + ) + parser.add_argument( + "--install", + type=str, + required=True, + help="identify installation location", + ) + parser.add_argument( + "--loglevel", + type=str, + default="INFO", + choices=[ + "ERROR", + "WARNING", + "INFO", + ], + help="the logging level", + ) + options = parser.parse_args() + + console = logging.StreamHandler() + console.setFormatter(logging.Formatter("%(asctime)s - %(message)s")) + logger.setLevel(getattr(logging, options.loglevel)) + logger.addHandler(console) + logger.propagate = False + + # locate the root of WAMR + current_file = pathlib.Path(__file__) + if current_file.is_symlink(): + current_file = pathlib.Path(os.readlink(current_file)) + root = current_file.parent.joinpath("../..").resolve() + logger.info(f"The root of WAMR is {root}") + + # download repos + for repo in external_repos.keys(): + if not download_repo(repo, root): + return False + + if not collect_headers(root, pathlib.Path(options.install)): + return False + + return True + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/wasm-micro-runtime/test-tools/wamr-ide/.gitattributes b/wasm-micro-runtime/test-tools/wamr-ide/.gitattributes new file mode 100644 index 0000000..dcd444f --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/.gitattributes @@ -0,0 +1,2 @@ +# Convert to LF line endings on checkout. +*.sh text eol=lf \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/Config_building_target.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/Config_building_target.png new file mode 100644 index 0000000..18db0c8 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/Config_building_target.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/build_folder.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/build_folder.png new file mode 100644 index 0000000..0ca05ff Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/build_folder.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/build_terminal.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/build_terminal.png new file mode 100644 index 0000000..24fe26e Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/build_terminal.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/change_workspace_dialog.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/change_workspace_dialog.png new file mode 100644 index 0000000..9f6a614 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/change_workspace_dialog.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/compilation_config.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/compilation_config.png new file mode 100644 index 0000000..346244d Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/compilation_config.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/compilation_config_2.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/compilation_config_2.png new file mode 100644 index 0000000..7bf46b2 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/compilation_config_2.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/debug.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/debug.png new file mode 100644 index 0000000..1d8636d Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/debug.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/decoration_for_files.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/decoration_for_files.png new file mode 100644 index 0000000..dd5c0bd Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/decoration_for_files.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_config.jpg b/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_config.jpg new file mode 100644 index 0000000..50cea71 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_config.jpg differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_engine_config.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_engine_config.png new file mode 100644 index 0000000..c375707 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_engine_config.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_images.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_images.png new file mode 100644 index 0000000..67d38e7 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/docker_images.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/install_from_vsix.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/install_from_vsix.png new file mode 100644 index 0000000..f313ec5 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/install_from_vsix.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/new_project_page.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/new_project_page.png new file mode 100644 index 0000000..b498aca Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/new_project_page.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/open_project_page.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/open_project_page.png new file mode 100644 index 0000000..d4e5343 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/open_project_page.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/project_template.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/project_template.png new file mode 100644 index 0000000..e10b102 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/project_template.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/right_click_menus_1.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/right_click_menus_1.png new file mode 100644 index 0000000..0649f2f Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/right_click_menus_1.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/right_click_menus_2.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/right_click_menus_2.png new file mode 100644 index 0000000..9d8e322 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/right_click_menus_2.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/run.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/run.png new file mode 100644 index 0000000..0213dca Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/run.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/save_configuration.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/save_configuration.png new file mode 100644 index 0000000..39a5c1c Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/save_configuration.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/set_up_workspace_message.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/set_up_workspace_message.png new file mode 100644 index 0000000..c812a59 Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/set_up_workspace_message.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Media/wamr_ide_main_menu.png b/wasm-micro-runtime/test-tools/wamr-ide/Media/wamr_ide_main_menu.png new file mode 100644 index 0000000..89c56fa Binary files /dev/null and b/wasm-micro-runtime/test-tools/wamr-ide/Media/wamr_ide_main_menu.png differ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/README.md b/wasm-micro-runtime/test-tools/wamr-ide/README.md new file mode 100644 index 0000000..7788922 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/README.md @@ -0,0 +1,315 @@ +# WAMR-IDE (Experimental) + +## Introduction + +The WAMR-IDE is an Integrated Development Environment to develop WebAssembly application with coding, compiling and source debugging support. It contains 3 components: `VSCode extension`, `wasm-toolchain docker image` and `wasm-debug-server docker image`. + +- `VSCode extension` is an `vscode` extension, with which user can build and manage projects, develop `wasm application`, including `building`, `running` and `debugging`. + +- `WASM-toolchain` is a docker image which provides building environment for wasm. + +- `WASM source debug server` is a docker image which provides running and source debugging environment for wasm application. + +--- + +## How to setup WAMR IDE + +Now, the most straightforward way to install the WAMR IDE extension is by searching for WAMR-IDE in the VS Code extension marketplace and installing it directly. So, if you simply want to use WAMR debugging features in VS Code, this is the ideal (and effortless) way. And you are ready to [use WAMR IDE](#how-to-use-wamr-ide). + +> It is only recommended to download versions after 1.3.2 from the marketplace. + +Also, we have same version tagged docker images, lldb binaries and VS Code installation file(.vsix file) packed for each GitHub release. You can following the tutorial in [this section](#21-download-wamr-vs-code-extension-from-the-github-releaserecommended-approach). + +Alternatively, if you want to build lldb, docker images, or .vsix file locally so that you can try the effect of your modification, you could refer to the tutorial in [this section](#22-build-wamr-vs-code-extension-locallyalternative-approach). + +### 1. Preparation + +#### 1.1. Install `VSCode` on host + +- make sure the version of [vscode](https://code.visualstudio.com/Download) you installed is at least _1.59.0_ + +#### 1.2. Install `Docker` on host + + 1. [Windows: Docker Desktop](https://docs.docker.com/desktop/windows/install/) + 2. [Ubuntu: Docker Engine](https://docs.docker.com/engine/install/ubuntu) + ```xml + OS requirements: + To install Docker Engine, you need the 64-bit version of one of these Ubuntu versions: + - Ubuntu Impish 21.10 + - Ubuntu Hirsute 21.04 + - Ubuntu Focal 20.04(LTS) + - Ubuntu Bionic 18.04(LTS) + ``` + +### 2. WAMR VS Code extension: download from the GitHub release or build locally + +#### 2.1 Download WAMR VS Code extension from the GitHub release(Recommended approach) + +##### 2.1.1 Load docker images from the GitHub release tar file + +From now on, for each GitHub release, we have the same version tagged docker image saved as a tar file, which you can find and download in the GitHub release. + +You could download the tar archive files for docker images from the release, and then load them using the following commands: + +```sh +# download the zip or tar.gz from release depending on your platform +# decompress and get the tar file + +# on Linux/MacOS, you could use tar +tar xf wasm-toolchain-{version number}.tar.gz +tar xf wasm-debug-server-{version number}.tar.gz +# or you could use unzip +unzip wasm-toolchain-{version number}.zip +unzip wasm-debug-server-{version number}.zip +# load wasm-toolchain +docker load --input wasm-toolchain.tar +# load wasm-debug-server +docker load --input wasm-debug-server.tar + +# on Windows, you could use any unzip software you like +# then loading docker images using powershell or git bash +# load wasm-toolchain +docker load --input ./wasm-toolchain.tar +# load wasm-debug-server +docker load --input ./wasm-debug-server.tar +``` + +##### 2.1.2 Download the VS Code extension installation file from the GitHub release + +From now on, for each GitHub release, we have the same version tagged zip/tar.gz file. For example, in release version 1.1.2, you can easily download and decompress `wamr-ide-1.1.2.tar.gz` `wamr-ide-1.1.2.zip`, which contains `wamr-ide.vsix` VS Code extension installation file. As you can imagine, in the future, when new releases are available, you can freely choose whichever version(for example, 1.2.0, 1.3.0, etc.) you prefer. It should work as long as you download the same version tagged docker image and .vsix file. + +##### 2.1.3 Install extension from vsix + +![install_from_vsix](./Media/install_from_vsix.png "install wamr-ide from vsix") + +select `wamr-ide.vsix` which you have decompressed from `.tar.gz` or `.zip` file. + +#### 2.2 Build WAMR VS Code extension locally(Alternative approach) + +You could also build the VS Code extension locally, the following instruction provides a thorough tutorial. It's worth noting that in the local build tutorial we use hard-coded tag version 1.0 other than the semantic version of WAMR. + +Note: Please ensure that the scripts under `resource` directories have execution permission. While on git they have x bits, you might have dropped them eg. by copying them from Windows. Similarly, do not drop execution permission when copying `lldb` binaries under `resource/debug/bin`. + +##### 2.2.1 Build docker images on host + +We have 2 docker images which should be built or loaded on your host, `wasm-toolchain` and `wasm-debug-server`. To build these 2 images, please enter the `WASM-Debug-Server/Docker` & `WASM-Toolchain/Docker`, then execute the `build_docker_image` script respectively. + +Windows (powershell): + +```batch +cd .\WASM-Toolchain\Docker +.\build_docker_image.bat +cd .\WASM-Debug-Server\Docker +.\build_docker_image.bat +``` + +Linux: + +```shell +cd ./WASM-Toolchain/Docker +./build_docker_image.sh +cd ./WASM-Debug-Server/Docker +./build_docker_image.sh +``` + +##### 2.2.2 After building, you can find `wasm-toolchain` and `wasm-debug-server` docker images on your local + +![docker-images](./Media/docker_images.png) + +##### 2.2.3 If building docker images fail during the process + +Sometimes building the Docker images may fail due to bad network conditions. If the `wasm-toolchain` and `wasm-debug-server` images do not exist after building, please build them manually. Fix the proxy setting if needed and execute the following command to build docker images. + +![docker-engine-config](./Media/docker_config.jpg) + +> Note: please correctly replace example proxy address with your own before you run manually. + +```xml +$ cd .\docker_images\wasm-debug-server +$ docker build --no-cache --build-arg http_proxy=http://proxy.example.com:1234 +--build-arg https_proxy=http://proxy.example.com:1234 -t wasm-debug-server:1.0 . +``` + +```xml +$ cd .\docker_images\wasm-toolchain +$ docker build --no-cache --build-arg http_proxy=http://proxy.example.com:1234 +--build-arg https_proxy=http://proxy.example.com:1234 -t wasm-toolchain:1.0 . +``` + +##### 2.2.4 If you encounter the problem `failed to solve with frontend dockerfile.v0: failed to create LLB definition`, please config your docker desktop + +![docker-engine-config](./Media/docker_engine_config.png) + +##### 2.2.5 Points To Remember + +- Make sure that the `wasm-toolchain:1.0` and `wasm-debug-server:1.0` docker images are both successfully built before using `WAMR IDE`, otherwise `Build`, `Run` and `Debug` will not work. + +##### 2.2.6 Generate wamride extension package file + +`wamride-1.0.0.vsix` can be packaged by [`npm vsce`](https://code.visualstudio.com/api/working-with-extensions/publishing-extension). + +```shell +npm install -g vsce +cd VSCode-Extension +rm -rf node_modules +npm install +vsce package +``` + +##### 2.2.7 Enable VS Code debugging feature + +By default, when you build .vsix locally, the debugging feature is off. Suppose you want to enable the source debugging feature. In that case, you could download `lldb` binaries from our GitHub release (for example, `wamr-lldb-1.1.2-x86_64-ubuntu-20.04.tar.gz`), decompress and put every subdirectory and file to the installed directory of your VS Code extension. + +For example, let's say you are on an Ubuntu 20.04 machine. You first download and decompress `wamr-lldb-1.1.2-x86_64-ubuntu-20.04.tar.gz`, and you will get a `wamr-lldb` folder (or `inst` folder in our earlier release). Then, you can simply copy the files and directory inside that folder to the relative path `resource/debug/linux/` under your VS Code extension installation directory. + +Example commands on an Ubuntu 20.04 machine: + +```shell +# decompress .tar.gz file and get the folder +$ ls wamr-lldb +bin lib package.json syntaxes +# copy everything to the vscode extension installation path(in this case, it's /home/{usrname}/.vscode-server/extensions/wamr.wamride-1.0.0/) +$ cp inst/* /home/{usrname}/.vscode-server/extensions/wamr.wamride-1.0.0/resource/debug/linux/ +``` + +If you want to use your own patched `lldb`, you could follow this [instruction](../../doc/source_debugging.md#debugging-with-interpreter) to build `lldb`. And follow this [instruction](./VSCode-Extension/resource/debug/README.md) +to copy the binaries to replace the existing ones. + +> **You can also debug the extension directly follow this [instruction](./VSCode-Extension/README.md) without packing the extension.** + +##### 2.2.7 Install extension from vsix + +![install_from_vsix](./Media/install_from_vsix.png "install wamr-ide from vsix") + +select `wamride-1.0.0.vsix` which you have packed on your host. + +--- + +## How to use `wamr-ide` + +#### `WAMR-IDE` extension contains 2 components as following picture showing. `WAMR IDE` for workspace and project management and `Current Project` for project's execution + +![wamr_ide_main_menu](./Media/wamr_ide_main_menu.png "wamr-ide main menu") + +### Project Execution + +#### 1. New project + +When you click `New project` button, the extension will pop up a message box at the bottom right of the screen as following: + +![set-up-workspace-message](./Media/set_up_workspace_message.png "set up workspace message box") + +You can click `Set up now` and select the target folder to create project workspace, or you click `Maybe later` to close the message box. + +> Note that your selected workspace folder should be **empty** or the folder you have set up as workspace. + +After setting up workspace, extension will prompt successful message: + +```xml +Workspace has been set up successfully! +``` + +Then click `New project` button again, a new page will show as following. + +![new-project-page](./Media/new_project_page.png "new project page") + +Enter the `Project name` and select the `Template`, then click `Create` button. A new project will be generated and opened in your current `VS Code window` or in a new `VS Code window`. + +> Opening in current windows or a new one depends on whether your `vscode's explorer` is empty or not. If empty, open in current window, or open in the new vscode window. + +A new initialized project is as following picture shows. + +![project-template](./Media/project_template.png "default project template") + +`.wamr` is the project configuration folder which contains 3 files, `CMakeLists.txt`, `project.cmake`, and `compilation_config.json`. `CMakeLists.txt` is used to build `wasm target` and the `project.cmake` is included in `CMakeLists.txt`. `compilation_config.json` includes the user's customized configuration such as folders which should be added in the include path. + +#### 2. Open project + +Click `Open project` button, `quick-pick-box` will show as following. All projects under your current workspace will be shown and can be selected. + +![configuration file](./Media/open_project_page.png "configuration file") + +#### 3. Change workspace + +Click `Change workspace` button, a dialog will show as following. You can select 1 folder in file system as workspace, and the new workspace path will override previous workspace, and all new created projects will be generated in the new workspace. + +![change workspace ](./Media/change_workspace_dialog.png "change workspace dialog") + +#### 4. Customize `include paths` and `exclude source files` + + Extension supports adding header file folder to `include path` and excluding source file from build. + +- `Add to include path` + + - Move the cursor to the `folder` and right click, then `menus` will be shown as following. Click `Toggle state of path including`. + + ![right click menus](./Media/right_click_menus_1.png "right click menus") + +- `Exclude source file from build` + + - Move the cursor to the `source file` and right click, then `menus` will be shown as following. Click `Toggle state of excluding`. + + ![right click menus](./Media/right_click_menus_2.png "right click menus") + +#### After setting up `include path` and `exclude files`, the corresponding folder and files will be decorated with color and icon as following picture shows + + ![decoration for files](./Media/decoration_for_files.png "decoration for files") + + At the same time, all added `include path` and `exclude files` will be saved in `.wamr/compilation_config.json` as json array. + + ![compilation config](./Media/compilation_config.png "compilation config") + +> `Toggle state of path including` just shows when selecting `folder` and hides with other resources. +> +> `Toggle state of excluding` just shows when selecting `[.c | .cpp | .cxx] source files` and hides with other resources. + +### Current Project Management + +#### 1. Configuration + +Click `Configuration` button, a new page will be shown as following. You can config building target with `Include paths`, `Initial & Max linear memory`, `stack size`, `exported_symbols` and `include paths`, `exclude files`. + +![config building target](./Media/Config_building_target.png "config building target") + +Short Explanation of the Fields Above: + +- Output file name: The compiled wasm file name of your program. +- Initial linear memory size, Max linear memory size, Stack size: The wasi-sdk clang compile options. +- Exported symbols: The symbols your wasm program wants to export. **Multiple symbols are separated by commas without spaces**. +- Host managed heap size: The running configuration for the host managed heap size of iwasm. In most cases, the default size would be fine, but in some scenarios, let's say you want to allocate more memory using `malloc`, you should increase it here accordingly. + +> Note that due to the current implementation limitation, after changing the `Output file name` or `Host managed heap size`, you need to close and reopen VSCode (to reactivate the extension) so that the running config will be correctly updated. + +Then click `Modify` button to confirm, if configurations are modified successfully and following message will pop. Click `OK`, the page will be auto closed. + +![save configuration](./Media/save_configuration.png "save configuration") + +And all configuration will be saved in `.wamr/compilation_config.json`. + +![configuration file](./Media/compilation_config_2.png "configuration file") + +#### 2. `Build` + +When you have completed coding and ready to build target, click `build` button and the `wasm-toolchain` will auto start a container and execute the building process. + +![build terminal output](./Media/build_terminal.png "build terminal output") + +After successful building execution, `build` folder will be generated in `explorer`, in which `${output_file_name}.wasm` is exist. + +![build folder](./Media/build_folder.png "build folder") + +> Note that to start `docker service` firstly. + +#### 3. Run + +Click `Run` button and `wasm-debug-server` docker image will auto start a container and execute the running process. + +![run](./Media/run.png "run wasm") + +#### 4. Debug + +Click `Debug` button will trigger start ip `wamr-debug-server` docker image, and boot up `lldb debug server` inside of iwasm. Then start a debugging session with configuration to connect. Tap `F11` or click `step into` to start debugging. + +![debug](./Media/debug.png "source debugging") + +> Docker containers will be auto stopped and removed after the execution. diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Script/build.bat b/wasm-micro-runtime/test-tools/wamr-ide/Script/build.bat new file mode 100644 index 0000000..5274dad --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/Script/build.bat @@ -0,0 +1,41 @@ +@REM Copyright (C) 2019 Intel Corporation. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +@echo off +set DIR_ROOT=%cd%\.. + +echo "=== Verify the vscode status ===" +call code --version +IF %ERRORLEVEL%==0 ( + echo "vscode is ready." +) ELSE ( + echo "VSCode is not installed, please install firstly." + exit /b 1 +) + +echo "=== Verify the docker status ===" +call docker --version +IF %ERRORLEVEL%==0 ( + echo "docker is ready." +) ELSE ( + echo "Docker is not installed, please install firstly." + exit /b 1 +) + +cd %DIR_ROOT%\WASM-Debug-Server\Docker +call docker build -t wasm-debug-server:1.0 . +IF %ERRORLEVEL%==0 ( + echo "wasm-debug-server image is ready." +) ELSE ( + echo "build wasm-debug-server image failed." + exit /b 1 +) + +cd %DIR_ROOT%\WASM-Toolchain\Docker +call docker build -t wasm-toolchain:1.0 . +IF %ERRORLEVEL%==0 ( + echo "wasm-toolchain image is ready." +) ELSE ( + echo "build wasm-toolchain image failed." + exit /b 1 +) \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/Script/build.sh b/wasm-micro-runtime/test-tools/wamr-ide/Script/build.sh new file mode 100755 index 0000000..c30cb5a --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/Script/build.sh @@ -0,0 +1,46 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#!/bin/bash + +# 1. verify the environment: vscode & docker +# 1.1 if docker is installed, config docker command execution without sudo, promp if not installed and exit. +# 1.2 if vscode is not installed, promp and exit. +# 2. build wasm-toolchain & wasm-debug-server docker image + +DIR_ROOT=$(pwd)/.. + +echo "=== Verify the vscode status ===" +if [ "$(code --version)" ]; then + echo "VSCode is ready." +else + echo "VSCode is not installed, please install firstly." + exit 1 +fi + +echo "=== Verify the docker status ===" +if [ "$(docker --version)" ]; then + echo "Docker is ready." +else + echo "Docker is not installed, please install firstly." + exit 1 +fi + +# setup docker command exectuion without sudo permission +sudo groupadd docker +sudo gpasswd -a ${USER} docker +sudo service docker restart + +# create new group and execute the rest commands +newgrp - docker << REST + +# 2. build wasm-debug-server docker image +cd ${DIR_ROOT}/WASM-Debug-Server/Docker +docker build -t wasm-debug-server:1.0 . + +# 3. build wasm-toolchain docker image +cd ${DIR_ROOT}/WASM-Toolchain/Docker +docker pull ubuntu:20.04 +docker build -t wasm-toolchain:1.0 . + +REST \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.eslintrc.json b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.eslintrc.json new file mode 100644 index 0000000..5c1fd46 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.eslintrc.json @@ -0,0 +1,19 @@ +{ + "root": true, + "parser": "@typescript-eslint/parser", + "extends": ["plugin:@typescript-eslint/recommended"], + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint"], + "rules": { + "@typescript-eslint/naming-convention": "warn", + "@typescript-eslint/semi": "warn", + "curly": "warn", + "eqeqeq": "warn", + "no-throw-literal": "warn", + "semi": "off" + }, + "ignorePatterns": ["out", "dist", "**/*.d.ts"] +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.gitignore b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.gitignore new file mode 100644 index 0000000..782f2e4 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.gitignore @@ -0,0 +1,12 @@ +out +dist +node_modules +.vscode-test/ +*.vsix +package-lock.json +.vscode +resource/debug/** +!resource/debug/darwin/.placeholder +!resource/debug/linux/.placeholder +!resource/debug/windows/.placeholder +resource/test/test.wasm \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.npmrc b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.npmrc new file mode 100644 index 0000000..4fd0219 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.npmrc @@ -0,0 +1 @@ +engine-strict=true \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.prettierrc.json b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.prettierrc.json new file mode 100644 index 0000000..b2b00da --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.prettierrc.json @@ -0,0 +1,12 @@ +{ + "printWidth": 80, + "tabWidth": 4, + "useTabs": false, + "semi": true, + "singleQuote": true, + "trailingComma": "es5", + "bracketSpacing": true, + "jsxBracketSameLine": false, + "arrowParens": "avoid", + "proseWrap": "always" +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.vscodeignore b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.vscodeignore new file mode 100644 index 0000000..2727ff5 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/.vscodeignore @@ -0,0 +1,19 @@ +.gitignore +.yarnrc + +.vscode/** +.vscode-test/** +out/test/** + +**/tsconfig.json +**/.eslintrc.json +**/*.map +**/*.ts + +src + +resource/test +resource/debug/** +!resource/debug/darwin/.placeholder +!resource/debug/linux/.placeholder +!resource/debug/windows/.placeholder \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/CONTRIBUTING.md b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/CONTRIBUTING.md new file mode 100644 index 0000000..f70a959 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# CONTRIBUTING + +## Pull requests + +To submit your change: + +- Make sure your code is in line with our + [coding conventions](##Coding-conventions). +- Create an [issue] describing the bug the PR fixes or the feature you intend + to implement. +- Submit a [pull request] into the main branch. + +## Coding conventions + +#### Format + +The codebase is formatted by `Prettier` and the `.prettierrc.json` has been +configured. + +- VSCode along with `Format on Save` configuration could easily format your + code during development. +- You can run `prettier-format-check` and `prettier-format-apply` to check and + format your codebase with `prettier` in terminal. + +#### Lint + +`ESlint` is used as linter for the codebase and the `.eslintrc.json` has been +configured. + +- It's suggested to run `npm run lint` then fix errors and warnings before + committing. + +[issue]: https://github.com/bytecodealliance/wasm-micro-runtime/issues +[pull request]: https://github.com/bytecodealliance/wasm-micro-runtime/pulls diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/LICENSE b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/LICENSE new file mode 100644 index 0000000..c6bd7e0 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/LICENSE @@ -0,0 +1,219 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/README.md b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/README.md new file mode 100644 index 0000000..739e39a --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/README.md @@ -0,0 +1,40 @@ +# Introduction + +### An integrated development environment for WASM. + +# How to debug this extension +> Note that when you download and +> decompress to get .vsix file from [our release](https://github.com/bytecodealliance/wasm-micro-runtime/releases). +> It's by default that the `source debugging` feature is not enabled. +> If you want to enable the `source debugging` feature of this extension, +> you could download `lldb` from [our release](https://github.com/bytecodealliance/wasm-micro-runtime/releases) +> (This is the recommended way, and you could do it with a single click in VS Code). +> Then if you want to use your customized lldb patch, +> you could build your own version of `lldb` +> and then follow this [instruction](./resource/debug/README.md) +> to put them in the correct path + +### 1. open `VSCode_Extension` directory with the `vscode` + +```xml +File -> Open Folder -> select `VSCode_Extension` +``` + +### 2. run `npm install` in `terminal` to install necessary dependencies. + +### 3. click `F5` or `ctrl+shift+D` switch to `Run and Debug` panel and click `Run Extension` to boot. + +# Code Format + +`prettier` is recommended and `.prettierrc.json` has been provided in workspace. +More details and usage guidance please refer [prettier](https://prettier.io/docs/en/install.html) + +You can run following commands in current extension directory to check and apply +```shell +# install prettier firstly +npm install --save-dev prettier +# check format +npm run prettier-format-check +# apply +npm run prettier-format-apply +``` \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/formatters/rust.py b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/formatters/rust.py new file mode 100644 index 0000000..6fd5f16 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/formatters/rust.py @@ -0,0 +1,747 @@ +''' +Copyright (c) 2016 Vadim Chugunov +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +''' + +from __future__ import print_function, division +import sys +import logging +import lldb +import weakref + +if sys.version_info[0] == 2: + # python2-based LLDB accepts utf8-encoded ascii strings only. + def to_lldb_str(s): return s.encode('utf8', 'backslashreplace') if isinstance(s, unicode) else s + range = xrange +else: + to_lldb_str = str + +log = logging.getLogger(__name__) + +module = sys.modules[__name__] +rust_category = None + + +def initialize_category(debugger, internal_dict): + global module, rust_category + + rust_category = debugger.CreateCategory('Rust') + # rust_category.AddLanguage(lldb.eLanguageTypeRust) + rust_category.SetEnabled(True) + + attach_summary_to_type(tuple_summary_provider, r'^\(.*\)$', True) + attach_synthetic_to_type(MsvcTupleSynthProvider, r'^tuple\$?<.+>$', + True) # *-windows-msvc uses this name since 1.47 + + attach_synthetic_to_type(StrSliceSynthProvider, '&str') + attach_synthetic_to_type(StrSliceSynthProvider, 'str*') + attach_synthetic_to_type(StrSliceSynthProvider, 'str') # *-windows-msvc uses this name since 1.5? + + attach_synthetic_to_type(StdStringSynthProvider, '^(collections|alloc)::string::String$', True) + attach_synthetic_to_type(StdVectorSynthProvider, r'^(collections|alloc)::vec::Vec<.+>$', True) + attach_synthetic_to_type(StdVecDequeSynthProvider, + r'^(collections|alloc::collections)::vec_deque::VecDeque<.+>$', True) + + attach_synthetic_to_type(MsvcEnumSynthProvider, r'^enum\$<.+>$', True) + attach_synthetic_to_type(MsvcEnum2SynthProvider, r'^enum2\$<.+>$', True) + + attach_synthetic_to_type(SliceSynthProvider, r'^&(mut *)?\[.*\]$', True) + attach_synthetic_to_type(MsvcSliceSynthProvider, r'^(mut *)?slice\$?<.+>.*$', True) + + attach_synthetic_to_type(StdCStringSynthProvider, '^(std|alloc)::ffi::c_str::CString$', True) + attach_synthetic_to_type(StdCStrSynthProvider, '^&?(std|core)::ffi::c_str::CStr$', True) + + attach_synthetic_to_type(StdOsStringSynthProvider, 'std::ffi::os_str::OsString') + attach_synthetic_to_type(StdOsStrSynthProvider, '^&?std::ffi::os_str::OsStr', True) + + attach_synthetic_to_type(StdPathBufSynthProvider, 'std::path::PathBuf') + attach_synthetic_to_type(StdPathSynthProvider, '^&?std::path::Path', True) + + attach_synthetic_to_type(StdRcSynthProvider, r'^alloc::rc::Rc<.+>$', True) + attach_synthetic_to_type(StdRcSynthProvider, r'^alloc::rc::Weak<.+>$', True) + attach_synthetic_to_type(StdArcSynthProvider, r'^alloc::(sync|arc)::Arc<.+>$', True) + attach_synthetic_to_type(StdArcSynthProvider, r'^alloc::(sync|arc)::Weak<.+>$', True) + attach_synthetic_to_type(StdMutexSynthProvider, r'^std::sync::mutex::Mutex<.+>$', True) + + attach_synthetic_to_type(StdCellSynthProvider, r'^core::cell::Cell<.+>$', True) + attach_synthetic_to_type(StdRefCellSynthProvider, r'^core::cell::RefCell<.+>$', True) + attach_synthetic_to_type(StdRefCellBorrowSynthProvider, r'^core::cell::Ref<.+>$', True) + attach_synthetic_to_type(StdRefCellBorrowSynthProvider, r'^core::cell::RefMut<.+>$', True) + + attach_synthetic_to_type(StdHashMapSynthProvider, r'^std::collections::hash::map::HashMap<.+>$', True) + attach_synthetic_to_type(StdHashSetSynthProvider, r'^std::collections::hash::set::HashSet<.+>$', True) + + attach_synthetic_to_type(GenericEnumSynthProvider, r'^core::option::Option<.+>$', True) + attach_synthetic_to_type(GenericEnumSynthProvider, r'^core::result::Result<.+>$', True) + attach_synthetic_to_type(GenericEnumSynthProvider, r'^alloc::borrow::Cow<.+>$', True) + + if 'rust' in internal_dict.get('source_languages', []): + lldb.SBDebugger.SetInternalVariable('target.process.thread.step-avoid-regexp', + '^ +def read_unique_ptr(valobj): + pointer = valobj.GetChildMemberWithName('pointer') + if pointer.TypeIsPointerType(): # Between 1.33 and 1.63 pointer was just *const T + return pointer + return pointer.GetChildAtIndex(0) + + +def string_from_ptr(pointer, length): + if length <= 0: + return u'' + error = lldb.SBError() + process = pointer.GetProcess() + data = process.ReadMemory(pointer.GetValueAsUnsigned(), length, error) + if error.Success(): + return data.decode('utf8', 'replace') + else: + log.error('ReadMemory error: %s', error.GetCString()) + + +def get_template_params(type_name): + params = [] + level = 0 + start = 0 + for i, c in enumerate(type_name): + if c == '<': + level += 1 + if level == 1: + start = i + 1 + elif c == '>': + level -= 1 + if level == 0: + params.append(type_name[start:i].strip()) + elif c == ',' and level == 1: + params.append(type_name[start:i].strip()) + start = i + 1 + return params + + +def obj_summary(valobj, unavailable='{...}'): + summary = valobj.GetSummary() + if summary is not None: + return summary + summary = valobj.GetValue() + if summary is not None: + return summary + return unavailable + + +def sequence_summary(childern, maxsize=32): + s = '' + for child in childern: + if len(s) > 0: + s += ', ' + s += obj_summary(child) + if len(s) > maxsize: + s += ', ...' + break + return s + + +def tuple_summary(obj, skip_first=0): + fields = [obj_summary(obj.GetChildAtIndex(i)) for i in range(skip_first, obj.GetNumChildren())] + return '(%s)' % ', '.join(fields) + + +# ----- Summaries ----- + +def tuple_summary_provider(valobj, dict={}): + return tuple_summary(valobj) + + +# ----- Synth providers ------ + + +class RustSynthProvider(object): + synth_by_id = weakref.WeakValueDictionary() + next_id = 0 + + def __init__(self, valobj, dict={}): + self.valobj = valobj + self.obj_id = RustSynthProvider.next_id + RustSynthProvider.synth_by_id[self.obj_id] = self + RustSynthProvider.next_id += 1 + + def update(self): + return True + + def has_children(self): + return False + + def num_children(self): + return 0 + + def get_child_at_index(self, index): + return None + + def get_child_index(self, name): + if name == '$$object-id$$': + return self.obj_id + + try: + return self.get_index_of_child(name) + except Exception as e: + log.exception('%s', e) + raise + + def get_summary(self): + return None + + +class ArrayLikeSynthProvider(RustSynthProvider): + '''Base class for providers that represent array-like objects''' + + def update(self): + self.ptr, self.len = self.ptr_and_len(self.valobj) # type: ignore + self.item_type = self.ptr.GetType().GetPointeeType() + self.item_size = self.item_type.GetByteSize() + + def ptr_and_len(self, obj): + pass # abstract + + def num_children(self): + return self.len + + def has_children(self): + return True + + def get_child_at_index(self, index): + try: + if not 0 <= index < self.len: + return None + offset = index * self.item_size + return self.ptr.CreateChildAtOffset('[%s]' % index, offset, self.item_type) + except Exception as e: + log.exception('%s', e) + raise + + def get_index_of_child(self, name): + return int(name.lstrip('[').rstrip(']')) + + def get_summary(self): + return '(%d)' % (self.len,) + + +class StdVectorSynthProvider(ArrayLikeSynthProvider): + def ptr_and_len(self, vec): + return ( + read_unique_ptr(gcm(vec, 'buf', 'ptr')), + gcm(vec, 'len').GetValueAsUnsigned() + ) + + def get_summary(self): + return '(%d) vec![%s]' % (self.len, sequence_summary((self.get_child_at_index(i) for i in range(self.len)))) + + +class StdVecDequeSynthProvider(RustSynthProvider): + def update(self): + self.ptr = read_unique_ptr(gcm(self.valobj, 'buf', 'ptr')) + self.cap = gcm(self.valobj, 'buf', 'cap').GetValueAsUnsigned() + + head = gcm(self.valobj, 'head').GetValueAsUnsigned() + + # rust 1.67 changed from a head, tail implementation to a head, length impl + # https://github.com/rust-lang/rust/pull/102991 + vd_len = gcm(self.valobj, 'len') + if vd_len.IsValid(): + self.len = vd_len.GetValueAsUnsigned() + self.startptr = head + else: + tail = gcm(self.valobj, 'tail').GetValueAsUnsigned() + self.len = head - tail + self.startptr = tail + + self.item_type = self.ptr.GetType().GetPointeeType() + self.item_size = self.item_type.GetByteSize() + + def num_children(self): + return self.len + + def has_children(self): + return True + + def get_child_at_index(self, index): + try: + if not 0 <= index < self.num_children(): + return None + offset = ((self.startptr + index) % self.cap) * self.item_size + return self.ptr.CreateChildAtOffset('[%s]' % index, offset, self.item_type) + except Exception as e: + log.exception('%s', e) + raise + + def get_index_of_child(self, name): + return int(name.lstrip('[').rstrip(']')) + + def get_summary(self): + return '(%d) VecDeque[%s]' % (self.num_children(), sequence_summary((self.get_child_at_index(i) for i in range(self.num_children())))) + +################################################################################################################## + + +class SliceSynthProvider(ArrayLikeSynthProvider): + def ptr_and_len(self, vec): + return ( + gcm(vec, 'data_ptr'), + gcm(vec, 'length').GetValueAsUnsigned() + ) + + def get_summary(self): + return '(%d) &[%s]' % (self.len, sequence_summary((self.get_child_at_index(i) for i in range(self.len)))) + + +class MsvcSliceSynthProvider(SliceSynthProvider): + def get_type_name(self): + tparams = get_template_params(self.valobj.GetTypeName()) + return '&[' + tparams[0] + ']' + + +# Base class for *String providers +class StringLikeSynthProvider(ArrayLikeSynthProvider): + def get_child_at_index(self, index): + ch = ArrayLikeSynthProvider.get_child_at_index(self, index) + ch.SetFormat(lldb.eFormatChar) + return ch + + def get_summary(self): + # Limit string length to 1000 characters to cope with uninitialized values whose + # length field contains garbage. + strval = string_from_ptr(self.ptr, min(self.len, 1000)) + if strval == None: + return None + if self.len > 1000: + strval += u'...' + return u'"%s"' % strval + + +class StrSliceSynthProvider(StringLikeSynthProvider): + def ptr_and_len(self, valobj): + return ( + gcm(valobj, 'data_ptr'), + gcm(valobj, 'length').GetValueAsUnsigned() + ) + + +class StdStringSynthProvider(StringLikeSynthProvider): + def ptr_and_len(self, valobj): + vec = gcm(valobj, 'vec') + return ( + read_unique_ptr(gcm(vec, 'buf', 'ptr')), + gcm(vec, 'len').GetValueAsUnsigned() + ) + + +class StdCStringSynthProvider(StringLikeSynthProvider): + def ptr_and_len(self, valobj): + vec = gcm(valobj, 'inner') + return ( + gcm(vec, 'data_ptr'), + gcm(vec, 'length').GetValueAsUnsigned() - 1 + ) + + +class StdOsStringSynthProvider(StringLikeSynthProvider): + def ptr_and_len(self, valobj): + vec = gcm(valobj, 'inner', 'inner') + tmp = gcm(vec, 'bytes') # Windows OSString has an extra layer + if tmp.IsValid(): + vec = tmp + return ( + read_unique_ptr(gcm(vec, 'buf', 'ptr')), + gcm(vec, 'len').GetValueAsUnsigned() + ) + + +class FFISliceSynthProvider(StringLikeSynthProvider): + def ptr_and_len(self, valobj): + process = valobj.GetProcess() + slice_ptr = valobj.GetLoadAddress() + data_ptr_type = valobj.GetTarget().GetBasicType(lldb.eBasicTypeChar).GetPointerType() + # Unsized slice objects have incomplete debug info, so here we just assume standard slice + # reference layout: [, ] + error = lldb.SBError() + pointer = valobj.CreateValueFromAddress('data', slice_ptr, data_ptr_type) + length = process.ReadPointerFromMemory(slice_ptr + process.GetAddressByteSize(), error) + return pointer, length + + +class StdCStrSynthProvider(FFISliceSynthProvider): + def ptr_and_len(self, valobj): + ptr, len = FFISliceSynthProvider.ptr_and_len(self, valobj) + return (ptr, len-1) # drop terminaing '\0' + + +class StdOsStrSynthProvider(FFISliceSynthProvider): + pass + + +class StdPathBufSynthProvider(StdOsStringSynthProvider): + def ptr_and_len(self, valobj): + return StdOsStringSynthProvider.ptr_and_len(self, gcm(valobj, 'inner')) + + +class StdPathSynthProvider(FFISliceSynthProvider): + pass + +################################################################################################################## + + +class DerefSynthProvider(RustSynthProvider): + deref = lldb.SBValue() + + def has_children(self): + return self.deref.MightHaveChildren() + + def num_children(self): + return self.deref.GetNumChildren() + + def get_child_at_index(self, index): + return self.deref.GetChildAtIndex(index) + + def get_index_of_child(self, name): + return self.deref.GetIndexOfChildWithName(name) + + def get_summary(self): + return obj_summary(self.deref) + +# Base for Rc and Arc + + +class StdRefCountedSynthProvider(DerefSynthProvider): + weak = 0 + strong = 0 + + def get_summary(self): + if self.weak != 0: + s = '(refs:%d,weak:%d) ' % (self.strong, self.weak) + else: + s = '(refs:%d) ' % self.strong + if self.strong > 0: + s += obj_summary(self.deref) + else: + s += '' + return s + + +class StdRcSynthProvider(StdRefCountedSynthProvider): + def update(self): + inner = read_unique_ptr(gcm(self.valobj, 'ptr')) + self.strong = gcm(inner, 'strong', 'value', 'value').GetValueAsUnsigned() + self.weak = gcm(inner, 'weak', 'value', 'value').GetValueAsUnsigned() + if self.strong > 0: + self.deref = gcm(inner, 'value') + self.weak -= 1 # There's an implicit weak reference communally owned by all the strong pointers + else: + self.deref = lldb.SBValue() + self.deref.SetPreferSyntheticValue(True) + + +class StdArcSynthProvider(StdRefCountedSynthProvider): + def update(self): + inner = read_unique_ptr(gcm(self.valobj, 'ptr')) + self.strong = gcm(inner, 'strong', 'v', 'value').GetValueAsUnsigned() + self.weak = gcm(inner, 'weak', 'v', 'value').GetValueAsUnsigned() + if self.strong > 0: + self.deref = gcm(inner, 'data') + self.weak -= 1 # There's an implicit weak reference communally owned by all the strong pointers + else: + self.deref = lldb.SBValue() + self.deref.SetPreferSyntheticValue(True) + + +class StdMutexSynthProvider(DerefSynthProvider): + def update(self): + self.deref = gcm(self.valobj, 'data', 'value') + self.deref.SetPreferSyntheticValue(True) + + +class StdCellSynthProvider(DerefSynthProvider): + def update(self): + self.deref = gcm(self.valobj, 'value', 'value') + self.deref.SetPreferSyntheticValue(True) + + +class StdRefCellSynthProvider(DerefSynthProvider): + def update(self): + self.deref = gcm(self.valobj, 'value', 'value') + self.deref.SetPreferSyntheticValue(True) + + def get_summary(self): + borrow = gcm(self.valobj, 'borrow', 'value', 'value').GetValueAsSigned() + s = '' + if borrow < 0: + s = '(borrowed:mut) ' + elif borrow > 0: + s = '(borrowed:%d) ' % borrow + return s + obj_summary(self.deref) + + +class StdRefCellBorrowSynthProvider(DerefSynthProvider): + def update(self): + self.deref = gcm(self.valobj, 'value', 'pointer').Dereference() + self.deref.SetPreferSyntheticValue(True) + +################################################################################################################## + + +class EnumSynthProvider(RustSynthProvider): + variant = lldb.SBValue() + summary = '' + skip_first = 0 + + def has_children(self): + return self.variant.MightHaveChildren() + + def num_children(self): + return self.variant.GetNumChildren() - self.skip_first + + def get_child_at_index(self, index): + return self.variant.GetChildAtIndex(index + self.skip_first) + + def get_index_of_child(self, name): + return self.variant.GetIndexOfChildWithName(name) - self.skip_first + + def get_summary(self): + return self.summary + + +class GenericEnumSynthProvider(EnumSynthProvider): + def update(self): + dyn_type_name = self.valobj.GetTypeName() + variant_name = dyn_type_name[dyn_type_name.rfind(':')+1:] + self.variant = self.valobj + + if self.variant.IsValid() and self.variant.GetNumChildren() > self.skip_first: + if self.variant.GetChildAtIndex(self.skip_first).GetName() in ['0', '__0']: + self.summary = variant_name + tuple_summary(self.variant) + else: + self.summary = variant_name + '{...}' + else: + self.summary = variant_name + + +class MsvcTupleSynthProvider(RustSynthProvider): + def update(self): + tparams = get_template_params(self.valobj.GetTypeName()) + self.type_name = '(' + ', '.join(tparams) + ')' + + def has_children(self): + return self.valobj.MightHaveChildren() + + def num_children(self): + return self.valobj.GetNumChildren() + + def get_child_at_index(self, index): + child = self.valobj.GetChildAtIndex(index) + return child.CreateChildAtOffset(str(index), 0, child.GetType()) + + def get_index_of_child(self, name): + return str(name) + + def get_summary(self): + return tuple_summary(self.valobj) + + def get_type_name(self): + return self.type_name + + +class MsvcEnumSynthProvider(EnumSynthProvider): + is_tuple_variant = False + + def update(self): + tparams = get_template_params(self.valobj.GetTypeName()) + if len(tparams) == 1: # Regular enum + discr = gcm(self.valobj, 'discriminant') + self.variant = gcm(self.valobj, 'variant' + str(discr.GetValueAsUnsigned())) + variant_name = discr.GetValue() + else: # Niche enum + dataful_min = int(tparams[1]) + dataful_max = int(tparams[2]) + dataful_var = tparams[3] + discr = gcm(self.valobj, 'discriminant') + if dataful_min <= discr.GetValueAsUnsigned() <= dataful_max: + self.variant = gcm(self.valobj, 'dataful_variant') + variant_name = dataful_var + else: + variant_name = discr.GetValue() + + self.type_name = tparams[0] + + if self.variant.IsValid() and self.variant.GetNumChildren() > self.skip_first: + if self.variant.GetChildAtIndex(self.skip_first).GetName() == '__0': + self.is_tuple_variant = True + self.summary = variant_name + tuple_summary(self.variant, skip_first=self.skip_first) + else: + self.summary = variant_name + '{...}' + else: + self.summary = variant_name + + def get_child_at_index(self, index): + child = self.variant.GetChildAtIndex(index + self.skip_first) + if self.is_tuple_variant: + return child.CreateChildAtOffset(str(index), 0, child.GetType()) + else: + return child + + def get_index_of_child(self, name): + if self.is_tuple_variant: + return int(name) + else: + return self.variant.GetIndexOfChildWithName(name) - self.skip_first + + def get_type_name(self): + return self.type_name + + +class MsvcEnum2SynthProvider(EnumSynthProvider): + is_tuple_variant = False + + def update(self): + tparams = get_template_params(self.valobj.GetTypeName()) + self.type_name = tparams[0] + + def get_child_at_index(self, index): + return self.valobj.GetChildAtIndex(index) + + def get_index_of_child(self, name): + return self.valobj.GetChildIndex(name) + + def get_type_name(self): + return self.type_name + + +################################################################################################################## + + +class StdHashMapSynthProvider(RustSynthProvider): + def update(self): + self.initialize_table(gcm(self.valobj, 'base', 'table')) + + def initialize_table(self, table): + assert table.IsValid() + + if table.type.GetNumberOfTemplateArguments() > 0: + item_ty = table.type.GetTemplateArgumentType(0) + else: # we must be on windows-msvc - try to look up item type by name + table_ty_name = table.GetType().GetName() # "hashbrown::raw::RawTable" + item_ty_name = get_template_params(table_ty_name)[0] + item_ty = table.GetTarget().FindTypes(item_ty_name).GetTypeAtIndex(0) + + if item_ty.IsTypedefType(): + item_ty = item_ty.GetTypedefedType() + + inner_table = table.GetChildMemberWithName('table') + if inner_table.IsValid(): + self.initialize_hashbrown_v2(inner_table, item_ty) # 1.52 <= std_version + else: + if not table.GetChildMemberWithName('data'): + self.initialize_hashbrown_v2(table, item_ty) # ? <= std_version < 1.52 + else: + self.initialize_hashbrown_v1(table, item_ty) # 1.36 <= std_version < ? + + def initialize_hashbrown_v2(self, table, item_ty): + self.num_buckets = gcm(table, 'bucket_mask').GetValueAsUnsigned() + 1 + ctrl_ptr = gcm(table, 'ctrl', 'pointer') + ctrl = ctrl_ptr.GetPointeeData(0, self.num_buckets) + # Buckets are located above `ctrl`, in reverse order. + start_addr = ctrl_ptr.GetValueAsUnsigned() - item_ty.GetByteSize() * self.num_buckets + buckets_ty = item_ty.GetArrayType(self.num_buckets) + self.buckets = self.valobj.CreateValueFromAddress('data', start_addr, buckets_ty) + error = lldb.SBError() + self.valid_indices = [] + for i in range(self.num_buckets): + if ctrl.GetUnsignedInt8(error, i) & 0x80 == 0: + self.valid_indices.append(self.num_buckets - 1 - i) + + def initialize_hashbrown_v1(self, table, item_ty): + self.num_buckets = gcm(table, 'bucket_mask').GetValueAsUnsigned() + 1 + ctrl_ptr = gcm(table, 'ctrl', 'pointer') + ctrl = ctrl_ptr.GetPointeeData(0, self.num_buckets) + buckets_ty = item_ty.GetArrayType(self.num_buckets) + self.buckets = gcm(table, 'data', 'pointer').Dereference().Cast(buckets_ty) + error = lldb.SBError() + self.valid_indices = [] + for i in range(self.num_buckets): + if ctrl.GetUnsignedInt8(error, i) & 0x80 == 0: + self.valid_indices.append(i) + + def has_children(self): + return True + + def num_children(self): + return len(self.valid_indices) + + def get_child_at_index(self, index): + bucket_idx = self.valid_indices[index] + item = self.buckets.GetChildAtIndex(bucket_idx) + return item.CreateChildAtOffset('[%d]' % index, 0, item.GetType()) + + def get_index_of_child(self, name): + return int(name.lstrip('[').rstrip(']')) + + def get_summary(self): + return 'size=%d, capacity=%d' % (self.num_children(), self.num_buckets) + + +class StdHashSetSynthProvider(StdHashMapSynthProvider): + def update(self): + table = gcm(self.valobj, 'base', 'map', 'table') # std_version >= 1.48 + if not table.IsValid(): + table = gcm(self.valobj, 'map', 'base', 'table') # std_version < 1.48 + self.initialize_table(table) + + def get_child_at_index(self, index): + bucket_idx = self.valid_indices[index] + item = self.buckets.GetChildAtIndex(bucket_idx).GetChildAtIndex(0) + return item.CreateChildAtOffset('[%d]' % index, 0, item.GetType()) + +################################################################################################################## + + +def __lldb_init_module(debugger_obj, internal_dict): + log.info('Initializing') + initialize_category(debugger_obj, internal_dict) \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/package.json b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/package.json new file mode 100644 index 0000000..77f9653 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/package.json @@ -0,0 +1,263 @@ +{ + "name": "wamride", + "publisher": "wamr-ide", + "repository": { + "url": "https://github.com/bytecodealliance/wasm-micro-runtime/tree/main/test-tools/wamr-ide" + }, + "displayName": "WAMR-IDE", + "description": "An Integrated Development Environment for WASM", + "version": "1.3.2", + "engines": { + "vscode": "^1.59.0", + "node": ">=16.0.0" + }, + "engineStrict": true, + "categories": [ + "Other" + ], + "activationEvents": [ + "onStartupFinished" + ], + "main": "./out/extension.js", + "contributes": { + "commands": [ + { + "command": "wamride.newProject", + "title": "Create new project", + "category": "New project" + }, + { + "command": "wamride.changeWorkspace", + "title": "Change workspace", + "category": "Change Workspace" + }, + { + "command": "wamride.build", + "title": "WAMRIDE:Build Wasm" + }, + { + "command": "wamride.run", + "title": "WAMRIDE:Run Wasm" + }, + { + "command": "wamride.debug", + "title": "WAMRIDE:Source Debug" + }, + { + "command": "wamride.openFolder", + "title": "WAMRIDE:openWorkspace" + }, + { + "command": "wamride.build.toggleStateIncludePath", + "title": "Toggle state of path including" + }, + { + "command": "wamride.build.toggleStateExclude", + "title": "Toggle state of excluding" + }, + { + "command": "wamride.targetConfig", + "title": "Target Configuration" + } + ], + "viewsContainers": { + "activitybar": [ + { + "id": "wamride", + "title": "WAMRIDE", + "icon": "$(star)" + } + ] + }, + "views": { + "wamride": [ + { + "id": "wamride.views.welcome", + "name": "Quick Access" + } + ] + }, + "viewsWelcome": [ + { + "view": "wamride.views.welcome", + "contents": "[ WAMR IDE ]\n[$(project)New project](command:wamride.newProject)\n[$(files)Open project](command:wamride.openFolder)\n[$(book)Change workspace](command:wamride.changeWorkspace)" + }, + { + "view": "wamride.views.welcome", + "contents": "[ Current Project ]\n[$(pencil)Configuration](command:wamride.targetConfig)\n[$(gear)Build](command:wamride.build)\n[$(run)Run](command:wamride.run)\n[$(debug-alt) Debug](command:wamride.debug)", + "enablement": "ext.isWasmProject" + } + ], + "menus": { + "explorer/context": [ + { + "command": "wamride.build.toggleStateIncludePath", + "alt": "wamride.build.toggleStateIncludePath", + "group": "config", + "when": "explorerResourceIsFolder" + }, + { + "command": "wamride.build.toggleStateExclude", + "alt": "wamride.build.toggleStateExclude", + "group": "config", + "when": "!explorerResourceIsFolder && resourceExtname in ext.supportedFileType" + } + ] + }, + "debuggers": [ + { + "type": "wamr-debug", + "label": "WAMR lldb debugger", + "enableBreakpointsFor": { + "languageIds": [ + "ada", + "arm", + "asm", + "c", + "cpp", + "crystal", + "d", + "fortan", + "fortran-modern", + "nim", + "objective-c", + "objectpascal", + "pascal", + "rust", + "swift" + ] + }, + "windows": { + "program": "./resource/debug/windows/bin/lldb-vscode.exe" + }, + "osx": { + "program": "./resource/debug/darwin/bin/lldb-vscode" + }, + "linux": { + "program": "./resource/debug/linux/bin/lldb-vscode" + }, + "configurationAttributes": { + "attach": { + "properties": { + "sourcePath": { + "type": "string", + "description": "Specify a source path to remap \"./\" to allow full paths to be used when setting breakpoints in binaries that have relative source paths." + }, + "sourceMap": { + "type": "array", + "description": "Specify an array of path remappings; each element must itself be a two element array containing a source and destination pathname. Overrides sourcePath.", + "default": [] + }, + "debuggerRoot": { + "type": "string", + "description": "Specify a working directory to set the debug adaptor to so relative object files can be located." + }, + "attachCommands": { + "type": "array", + "description": "Custom commands that are executed instead of attaching to a process ID or to a process by name. These commands may optionally create a new target and must perform an attach. A valid process must exist after these commands complete or the \"attach\" will fail.", + "default": [] + }, + "initCommands": { + "type": "array", + "description": "Initialization commands executed upon debugger startup.", + "default": [] + }, + "preRunCommands": { + "type": "array", + "description": "Commands executed just before the program is attached to.", + "default": [] + }, + "stopCommands": { + "type": "array", + "description": "Commands executed each time the program stops.", + "default": [] + }, + "exitCommands": { + "type": "array", + "description": "Commands executed at the end of debugging session.", + "default": [] + } + } + } + }, + "initialConfigurations": [ + { + "type": "wamr-debug", + "request": "attach", + "name": "Debug", + "stopOnEntry": true, + "attachCommands": [ + "process connect -p wasm connect://127.0.0.1:1234" + ] + } + ], + "configurationSnippets": [ + { + "label": "WAMR: Attach", + "description": "", + "body": { + "type": "wamr-debug", + "request": "attach", + "name": "${2:Attach}", + "stopOnEntry": true, + "attachCommands": [ + "process connect -p wasm connect://${3:127.0.0.1}:${4:1234}" + ] + } + } + ] + } + ], + "configuration": [ + { + "title": "WAMR-IDE", + "properties": { + "WAMR-IDE.configWorkspace": { + "type": "string", + "description": "Config the workspace for WebAssembly project." + } + } + } + ], + "taskDefinitions": [ + { + "type": "wasm" + } + ] + }, + "scripts": { + "vscode:prepublish": "npm run compile", + "compile": "tsc -p ./", + "watch": "tsc -watch -p ./", + "pretest": "npm run compile && npm run lint", + "lint": "eslint src --ext ts", + "lint-fix": "eslint --fix src --ext ts", + "test": "node ./out/test/runTest.js", + "prettier-format-check": "prettier --config .prettierrc.json 'src/**/*.ts' --check", + "prettier-format-apply": "prettier --config .prettierrc.json 'src/**/*.ts' --write" + }, + "devDependencies": { + "@types/chai": "^4.3.5", + "@types/glob": "^7.1.3", + "@types/mocha": "^8.2.2", + "@types/node": "14.x", + "@types/request": "^2.48.8", + "@types/vscode": "^1.54.0", + "@types/yauzl": "^2.10.0", + "@typescript-eslint/eslint-plugin": "^4.26.0", + "@typescript-eslint/parser": "^4.26.0", + "@vscode/debugprotocol": "^1.61.0", + "@vscode/test-electron": "^2.3.3", + "chai": "^4.3.7", + "eslint": "^7.32.0", + "glob": "^7.1.7", + "mocha": "^10.2.0", + "prettier": "2.5.1", + "typescript": "^4.3.2" + }, + "dependencies": { + "@vscode/webview-ui-toolkit": "^0.8.4", + "request": "^2.88.2", + "yauzl": "^2.10.0" + } +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/CMakeLists.txt b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/CMakeLists.txt new file mode 100644 index 0000000..81d998b --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/CMakeLists.txt @@ -0,0 +1,32 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.9) + +project(Main) + +include(${CMAKE_CURRENT_SOURCE_DIR}/project.cmake) + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wno-unused-command-line-argument " CACHE INTERNAL "") + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=${STACK_SIZE}") + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdebug-prefix-map=/mnt='$ENV{PROJ_PATH}'") + +set (CMAKE_EXE_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS} -Wl,--initial-memory=${INIT_MEM_SIZE},--max-memory=${MAX_MEM_SIZE},") + +set (CMAKE_EXE_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS} \ + ${EXPORTED_SYMBOLS},") + +set (SRC_LIST + ${PROJECT_SRC_LIST}) + +set (HEADER_LIST + ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${PROJECT_INCLUDES}) + +include_directories(${HEADER_LIST}) + +add_executable (${OUTPUT_FILE_NAME} ${SRC_LIST}) \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/boot_debugger_server.bat b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/boot_debugger_server.bat new file mode 100644 index 0000000..4d3a2c3 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/boot_debugger_server.bat @@ -0,0 +1,10 @@ +@REM Copyright (C) 2019 Intel Corporation. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +@echo off + +docker run --rm -it --name=wasm-debug-server-ctr ^ + -v "%cd%":/mnt ^ + -p 1234:1234 ^ + wasm-debug-server:%2 ^ + /bin/bash -c "./debug.sh %1 %3" diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/boot_debugger_server.sh b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/boot_debugger_server.sh new file mode 100755 index 0000000..e065865 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/boot_debugger_server.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -e + +docker run --rm -it --name=wasm-debug-server-ctr \ + -v "$(pwd)":/mnt \ + -p 1234:1234 \ + wasm-debug-server:$2 \ + /bin/bash -c "./debug.sh $1 $3" diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/build.bat b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/build.bat new file mode 100644 index 0000000..de41510 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/build.bat @@ -0,0 +1,11 @@ +@REM Copyright (C) 2019 Intel Corporation. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +@echo off + +@REM start a container, mount current project path to container/mnt +docker run --rm --name=wasm-toolchain-ctr ^ + -it -v "%cd%":/mnt ^ + --env=PROJ_PATH="%cd%" ^ + wasm-toolchain:%2 ^ + /bin/bash -c "./build_wasm.sh %1" diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/build.sh b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/build.sh new file mode 100755 index 0000000..a8a42cc --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -e + +docker run --rm --name=wasm-toolchain-ctr \ + -it -v "$(pwd)":/mnt \ + --env=PROJ_PATH="$(pwd)" \ + wasm-toolchain:$2 \ + /bin/bash -c "./build_wasm.sh $1" diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/destroy.bat b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/destroy.bat new file mode 100644 index 0000000..faf316a --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/destroy.bat @@ -0,0 +1,36 @@ +@REM Copyright (C) 2019 Intel Corporation. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +@echo off + +call docker --version>nul 2>nul +IF %ERRORLEVEL% GTR 0 ( + echo "Docker is not installed, please install docker desktop firstly." + echo + exit /b 1 +) + +call docker images>nul 2>nul +IF %ERRORLEVEL% GTR 0 ( + echo "Docker is not ready, please lanuch docker desktop firstly." + echo + exit /b 2 +) + +echo "Prepare to clean up the docker containers..." + +call docker inspect wasm-toolchain-ctr>nul 2>nul +IF %ERRORLEVEL% EQU 0 ( + echo "Stopping and removing wasm-toolchain-ctr container..." + docker rm -f wasm-toolchain-ctr>nul 2>nul + echo "Done." +) + +call docker inspect wasm-debug-server-ctr>nul 2>nul +IF %ERRORLEVEL% EQU 0 ( + echo "Stopping and removing wasm-debug-server-ctr container..." + docker rm -f wasm-debug-server-ctr>nul 2>nul + echo "Done." +) + +echo "Clean up docker containers successfully." diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/destroy.sh b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/destroy.sh new file mode 100755 index 0000000..41faf3e --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/destroy.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -e + +docker -v>/dev/null +if [ $? -ne 0 ]; then + echo "\nDocker is not installed, please install docker firstly.\n" + exit 1 +fi + +docker images>/dev/null +if [ $? -ne 0 ]; then + echo "\nDocker service is not running, please start your docker service firstly.\n" + exit 2 +fi + +echo "Prepare to clean up the docker containers..." + +if test ! -z "$(docker ps -a | grep wasm-toolchain-ctr)"; then + echo "Stopping and removing wasm-toolchain-ctr container..." + docker rm -f wasm-toolchain-ctr>/dev/null + echo "Done." +fi + +if test ! -z "$(docker ps -a | grep wasm-debug-server-ctr)"; then + echo "Stopping and removing wasm-debug-server-ctr container..." + docker rm -f wasm-debug-server-ctr>/dev/null + echo "Done." +fi + +echo "Clean up docker containers successfully." diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/project.cmake b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/project.cmake new file mode 100644 index 0000000..20b080e --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/project.cmake @@ -0,0 +1,17 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (OUTPUT_FILE_NAME) + +set (INIT_MEM_SIZE) + +set (MAX_MEM_SIZE) + +set (STACK_SIZE) + +set (EXPORTED_SYMBOLS) + +set (PROJECT_SRC_LIST) + +set (PROJECT_INCLUDES) + diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/run.bat b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/run.bat new file mode 100644 index 0000000..387a8e6 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/run.bat @@ -0,0 +1,9 @@ +@REM Copyright (C) 2019 Intel Corporation. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +@echo off + +docker run --rm -it --name=wasm-debug-server-ctr ^ + -v "%cd%":/mnt ^ + wasm-debug-server:%2 ^ + /bin/bash -c "./run.sh %1 %3" diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/run.sh b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/run.sh new file mode 100755 index 0000000..2526b95 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/scripts/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -e + +docker run --rm -it --name=wasm-debug-server-ctr \ + -v "$(pwd)":/mnt \ + wasm-debug-server:$2 \ + /bin/bash -c "./run.sh $1 $3" diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/test/build.sh b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/test/build.sh new file mode 100755 index 0000000..e175c91 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/test/build.sh @@ -0,0 +1,2 @@ +# compile with debug symbols and no optimization +rustc --target wasm32-wasi ./test.rs -g -C opt-level=0 \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/test/test.rs b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/test/test.rs new file mode 100644 index 0000000..59760f9 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/test/test.rs @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +use std::collections::HashMap; +use std::collections::VecDeque; +use std::cell::RefCell; + +fn main() { + let mut vector = Vec::from([1, 2, 3, 4]); + vector.push(12); + + let mut map: HashMap<&str, f64> = HashMap::from([ + ("Mercury", 0.4), + ("Venus", 0.7), + ("Earth", 1.0), + ("Mars", 1.5), + ]); + map.insert("Venus", 2.5); + map.insert("Sun", 312.2); + + let string = "this is a string"; + + let tmp = String::from("hello world"); + let slice = &tmp[1..5]; + + let mut deque = VecDeque::from([1, 2, 3]); + deque.push_back(4); + deque.push_back(5); + + let ref_cell = RefCell::new(5); + + println!("Hello, world!"); // BP_MARKER_1 +} \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/assert.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/assert.h new file mode 100644 index 0000000..86fdefe --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/assert.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_ASSERT_H +#define _WAMR_LIBC_ASSERT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/ctype.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/ctype.h new file mode 100644 index 0000000..846e7c8 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/ctype.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_CTYPE_H +#define _WAMR_LIBC_CTYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +int +isupper(int c); +int +isalpha(int c); +int +isspace(int c); +int +isgraph(int c); +int +isprint(int c); +int +isdigit(int c); +int +isxdigit(int c); +int +tolower(int c); +int +toupper(int c); +int +isalnum(int c); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/errno.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/errno.h new file mode 100644 index 0000000..8883bf8 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/errno.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_ERRNO_H +#define _WAMR_LIBC_ERRNO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/fcntl.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/fcntl.h new file mode 100644 index 0000000..b7e292f --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/fcntl.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_FCNTL_H +#define _WAMR_LIBC_FCNTL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/inttypes.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/inttypes.h new file mode 100644 index 0000000..2e59ca3 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/inttypes.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_INTTYPES_H +#define _WAMR_LIBC_INTTYPES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/limits.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/limits.h new file mode 100644 index 0000000..3859b05 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/limits.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_LIMITS_H +#define _WAMR_LIBC_LIMITS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define CHAR_BIT 8 +#define SCHAR_MIN -128 +#define SCHAR_MAX 127 +#define UCHAR_MAX 255 +#define CHAR_MIN 0 +#define CHAR_MAX 127 +#define MB_LEN_MAX 1 +#define SHRT_MIN -32768 +#define SHRT_MAX +32767 +#define USHRT_MAX 65535 +#define INT_MIN -32768 +#define INT_MAX +32767 +#define UINT_MAX 65535 +#define LONG_MIN -2147483648 +#define LONG_MAX +2147483647 +#define ULONG_MAX 4294967295 + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/pthread.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/pthread.h new file mode 100644 index 0000000..10b3978 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/pthread.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIB_PTHREAD_H +#define _WAMR_LIB_PTHREAD_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Data type define of pthread, mutex, cond and key */ +typedef unsigned int pthread_t; +typedef unsigned int pthread_mutex_t; +typedef unsigned int pthread_cond_t; +typedef unsigned int pthread_key_t; + +/* Thread APIs */ +int +pthread_create(pthread_t *thread, const void *attr, + void *(*start_routine)(void *), void *arg); + +int +pthread_join(pthread_t thread, void **retval); + +int +pthread_detach(pthread_t thread); + +int +pthread_cancel(pthread_t thread); + +pthread_t +pthread_self(void); + +void +pthread_exit(void *retval); + +/* Mutex APIs */ +int +pthread_mutex_init(pthread_mutex_t *mutex, const void *attr); + +int +pthread_mutex_lock(pthread_mutex_t *mutex); + +int +pthread_mutex_unlock(pthread_mutex_t *mutex); + +int +pthread_mutex_destroy(pthread_mutex_t *mutex); + +/* Cond APIs */ +int +pthread_cond_init(pthread_cond_t *cond, const void *attr); + +int +pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex); + +int +pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, + uint64_t useconds); + +int +pthread_cond_signal(pthread_cond_t *cond); + +int +pthread_cond_broadcast(pthread_cond_t *cond); + +int +pthread_cond_destroy(pthread_cond_t *cond); + +/* Pthread key APIs */ +int +pthread_key_create(pthread_key_t *key, void (*destructor)(void *)); + +int +pthread_setspecific(pthread_key_t key, const void *value); + +void * +pthread_getspecific(pthread_key_t key); + +int +pthread_key_delete(pthread_key_t key); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WAMR_LIB_PTHREAD_H */ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdarg.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdarg.h new file mode 100644 index 0000000..5095957 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdarg.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDARG_H +#define _WAMR_LIBC_STDARG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _VA_LIST +typedef __builtin_va_list va_list; +#define _VA_LIST +#endif +#define va_start(ap, param) __builtin_va_start(ap, param) +#define va_end(ap) __builtin_va_end(ap) +#define va_arg(ap, type) __builtin_va_arg(ap, type) + +#define __va_copy(d, s) __builtin_va_copy(d, s) + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WAMR_LIBC_STDARG_H */ diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdbool.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdbool.h new file mode 100644 index 0000000..2d1f8cd --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdbool.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDBOOL_H +#define _WAMR_LIBC_STDBOOL_H + +#define __bool_true_false_are_defined 1 + +#ifndef __cplusplus + +#define bool _Bool +#define false 0 +#define true 1 + +#endif /* __cplusplus */ + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdint.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdint.h new file mode 100644 index 0000000..8c55bff --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdint.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDINT_H +#define _WAMR_LIBC_STDINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* clang-format off */ +typedef char int8_t; +typedef short int int16_t; +typedef int int32_t; +typedef long long int int64_t; + +/* Unsigned. */ +typedef unsigned char uint8_t; +typedef unsigned short int uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long int uint64_t; + +typedef __INTPTR_TYPE__ intptr_t; +typedef __UINTPTR_TYPE__ uintptr_t; + +/* Minimum of signed integral types. */ +# define INT8_MIN (-128) +# define INT16_MIN (-32767-1) +# define INT32_MIN (-2147483647-1) +# define INT64_MIN (-__INT64_C(9223372036854775807)-1) +/* Maximum of signed integral types. */ +# define INT8_MAX (127) +# define INT16_MAX (32767) +# define INT32_MAX (2147483647) +# define INT64_MAX (__INT64_C(9223372036854775807)) + +/* Maximum of unsigned integral types. */ +# define UINT8_MAX (255) +# define UINT16_MAX (65535) +# define UINT32_MAX (4294967295U) +# define UINT64_MAX (__UINT64_C(18446744073709551615)) +/* clang-format on */ + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdio.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdio.h new file mode 100644 index 0000000..f9f0366 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdio.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDIO_H +#define _WAMR_LIBC_STDIO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef NULL +#define NULL ((void *)0) +#endif + +typedef unsigned long size_t; + +int +printf(const char *format, ...); +int +putchar(int c); +int +snprintf(char *str, size_t size, const char *format, ...); +int +sprintf(char *str, const char *format, ...); +int +puts(char *string); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdlib.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdlib.h new file mode 100644 index 0000000..302c896 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/stdlib.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDLIB_H +#define _WAMR_LIBC_STDLIB_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned long size_t; + +int +atoi(const char *s); +void +exit(int status); +long +strtol(const char *nptr, char **endptr, register int base); +unsigned long +strtoul(const char *nptr, char **endptr, register int base); +void * +malloc(size_t size); +void * +calloc(size_t n, size_t size); +void +free(void *ptr); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/string.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/string.h new file mode 100644 index 0000000..7a1a93d --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/string.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STRING_H +#define _WAMR_LIBC_STRING_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned long size_t; + +int +memcmp(const void *s1, const void *s2, size_t n); +void * +memcpy(void *dest, const void *src, size_t n); +void * +memmove(void *dest, const void *src, size_t n); +void * +memset(void *s, int c, size_t n); +void * +memchr(const void *s, int c, size_t n); +int +strncasecmp(const char *s1, const char *s2, size_t n); +size_t +strspn(const char *s, const char *accept); +size_t +strcspn(const char *s, const char *reject); +char * +strstr(const char *s, const char *find); +char * +strchr(const char *s, int c); +int +strcmp(const char *s1, const char *s2); +char * +strcpy(char *dest, const char *src); +size_t +strlen(const char *s); +int +strncmp(const char *str1, const char *str2, size_t n); +char * +strncpy(char *dest, const char *src, unsigned long n); +char * +strdup(const char *s); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/strings.h b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/strings.h new file mode 100644 index 0000000..3fe6ff6 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/wamr-sdk/libc-builtin-sysroot/include/strings.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STRINGS_H +#define _WAMR_LIBC_STRINGS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/css/style.css b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/css/style.css new file mode 100644 index 0000000..bff10e5 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/css/style.css @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +.box_wrapper { + display: flex; + justify-content: center; + align-items: center; +} + +.form_heading { + text-align: left; +} + +.form_heading vscode-divider, +.config_form_heading vscode-divider, +.form_bottom vscode-divider, +.config_form_bottom vscode-divider { + padding-bottom: 0.5rem; +} + +.form_body, +.config_form_body { + display: grid; + grid-row-gap: 1rem; + padding-bottom: 0.5rem; +} + +.config_form_body div, +.config_submit_btn_wrapper { + display: grid; + grid-template-columns: 4fr 8fr; + grid-column-gap: 0.5rem; +} + +.form_heading, +.form_body, +.form_bottom { + width: 400px; +} + +.config_form_body, +.config_form_heading, +.config_form_bottom { + width: 550px; +} + +#btn_submit { + border-radius: 5px; +} + +#select_div, +#text_filed_div, +.proj_submit_btn_wrapper { + display: grid; + grid-template-columns: 3fr 9fr; + grid-column-gap: 0.5rem; +} + +#ipt_projName, +#select_dropdown, +.config_form_body vscode-text-field, +.config_form_body vscode-text-area { + width: 100%; +} + +#btn { + text-align: center; +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/js/configbuildtarget.js b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/js/configbuildtarget.js new file mode 100644 index 0000000..1c146d9 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/js/configbuildtarget.js @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +const vscode = acquireVsCodeApi(); + +document.getElementById('btn_submit').onclick = () => { + submitFunc(); +}; + +function submitFunc() { + let outputFileName = document.getElementById('output_file_name').value; + let initMemSize = document.getElementById('initial_mem_size').value; + let maxMemSize = document.getElementById('max_mem_size').value; + let stackSize = document.getElementById('stack_size').value; + let exportedSymbols = document.getElementById('exported_symbols').value; + let hostManagedHeapSize = document.getElementById('host_managed_heap_size').value; + + vscode.postMessage({ + command: 'config_build_target', + outputFileName: outputFileName, + initMemSize: initMemSize, + maxMemSize: maxMemSize, + stackSize: stackSize, + exportedSymbols: exportedSymbols, + hostManagedHeapSize: hostManagedHeapSize, + }); +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/js/newproj.js b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/js/newproj.js new file mode 100644 index 0000000..30e1697 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/js/newproj.js @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +const vscode = acquireVsCodeApi(); + +document.getElementById('btn_submit').onclick = () => { + submitFunc(); +}; + +function submitFunc() { + let projectName = document.getElementById('ipt_projName').value; + let template = document.getElementById('select_dropdown').value; + + vscode.postMessage({ + command: 'create_new_project', + projectName: projectName, + template: template, + }); + + /* get msg from ext */ + window.addEventListener('message', event => { + const message = event.data; + switch (message.command) { + /* send command to open the project */ + case 'proj_creation_finish': + vscode.postMessage({ + command: 'open_project', + projectName: message.prjName, + }); + break; + default: + break; + } + }); +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/page/configBuildTarget.html b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/page/configBuildTarget.html new file mode 100644 index 0000000..877356a --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/page/configBuildTarget.html @@ -0,0 +1,81 @@ + + + + + + + + + + + New project + + + +
+
+

Config building target

+ +
+
+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+
+

Config iwasm running option

+ +
+
+ +
+
+
+ + +
+
+
+ +
+
+ +
+
+
+ Modify +
+
+
+
+ + + diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/page/newProject.html b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/page/newProject.html new file mode 100644 index 0000000..71e67bd --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/resource/webview/page/newProject.html @@ -0,0 +1,55 @@ + + + + + + + + + + + Create project + + + +
+
+

Create project

+ +
+
+ +
+
+
+ + +
+ +
+ + + + default + +
+
+
+ +
+
+ +
+
+
+ Create +
+
+
+
+ + + diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/constants.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/constants.ts new file mode 100644 index 0000000..cf8bb71 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/constants.ts @@ -0,0 +1,7 @@ +export const enum SelectionOfPrompt { + skip = 'skip', + setUp = 'setup', +} +export const enum Status { + done = 'done', +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/debugConfigurationProvider.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/debugConfigurationProvider.ts new file mode 100644 index 0000000..d24abe6 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/debugConfigurationProvider.ts @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as vscode from 'vscode'; +import * as os from 'os'; + +/* see https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-vscode#attaching-settings */ +export interface WasmDebugConfig { + type: string; + name: string; + request: string; + program?: string; + pid?: string; + stopOnEntry?: boolean; + waitFor?: boolean; + initCommands?: string[]; + preRunCommands?: string[]; + stopCommands?: string[]; + exitCommands?: string[]; + terminateCommands?: string[]; + attachCommands?: string[]; +} + +export class WasmDebugConfigurationProvider + implements vscode.DebugConfigurationProvider +{ + private wasmDebugConfig: WasmDebugConfig = { + type: 'wamr-debug', + name: 'Attach', + request: 'attach', + stopOnEntry: true, + attachCommands: [ + /* default port 1234 */ + 'process connect -p wasm connect://127.0.0.1:1234', + ], + }; + + constructor(extensionPath: string) { + this.wasmDebugConfig.initCommands = [ + /* Add rust formatters -> https://lldb.llvm.org/use/variable.html */ + `command script import ${extensionPath}/formatters/rust.py`, + ]; + + if (os.platform() === 'win32' || os.platform() === 'darwin') { + this.wasmDebugConfig.initCommands.push( + 'platform select remote-linux' + ); + } + } + + public resolveDebugConfiguration( + _: vscode.WorkspaceFolder | undefined, + debugConfiguration: vscode.DebugConfiguration + ): vscode.ProviderResult { + this.wasmDebugConfig = { + ...this.wasmDebugConfig, + ...debugConfiguration, + }; + + return this.wasmDebugConfig; + } + + public getDebugConfig(): vscode.DebugConfiguration { + return this.wasmDebugConfig; + } +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/decorationProvider.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/decorationProvider.ts new file mode 100644 index 0000000..46efcc9 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/decorationProvider.ts @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as vscode from 'vscode'; +import { readFromFile } from './utilities/directoryUtilities'; +import * as path from 'path'; +import * as os from 'os'; + +const DECORATION_INCLUDE_PATHS: vscode.FileDecoration = + new vscode.FileDecoration( + '✔', + 'Included', + new vscode.ThemeColor('list.highlightForeground') + ); +const DECORATION_EXCLUDE_FILES: vscode.FileDecoration = + new vscode.FileDecoration( + '✗', + 'Excluded', + new vscode.ThemeColor('list.errorForeground') + ); + +export class DecorationProvider implements vscode.FileDecorationProvider { + private disposables: vscode.Disposable[] = []; + public onDidChangeFileDecorations: vscode.Event< + vscode.Uri | vscode.Uri[] | undefined + >; + private eventEmitter: vscode.EventEmitter; + + constructor() { + this.eventEmitter = new vscode.EventEmitter(); + this.onDidChangeFileDecorations = this.eventEmitter.event; + this.disposables.push( + vscode.window.registerFileDecorationProvider(this) + ); + } + + public provideFileDecoration( + uri: vscode.Uri + ): vscode.ProviderResult { + const currentPrjDir = + os.platform() === 'win32' + ? (vscode.workspace.workspaceFolders?.[0].uri.fsPath as string) + : os.platform() === 'linux' || os.platform() === 'darwin' + ? (vscode.workspace.workspaceFolders?.[0].uri.path as string) + : ''; + + const pathRelative = (uri.fsPath ? uri.fsPath : uri.toString()).replace( + currentPrjDir, + '..' + ); + + const prjConfigDir = path.join(currentPrjDir, '.wamr'); + const configFilePath = path.join( + prjConfigDir, + 'compilation_config.json' + ); + if (readFromFile(configFilePath) !== '') { + const configData = JSON.parse(readFromFile(configFilePath)); + const includePathArr = configData['includePaths']; + const excludeFileArr = configData['excludeFiles']; + + if (includePathArr.indexOf(pathRelative) > -1) { + return DECORATION_INCLUDE_PATHS; + } else if (excludeFileArr.indexOf(pathRelative) > -1) { + return DECORATION_EXCLUDE_FILES; + } + } + } + + public dispose(): void { + this.disposables.forEach(d => d.dispose()); + } + + public updateDecorationsForSource(uri: vscode.Uri): void { + this.eventEmitter.fire(uri); + } +} + +export const decorationProvider: DecorationProvider = new DecorationProvider(); diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/extension.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/extension.ts new file mode 100644 index 0000000..ab549fc --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/extension.ts @@ -0,0 +1,1030 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as fileSystem from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import * as vscode from 'vscode'; + +import { WasmTaskProvider } from './taskProvider'; +import { TargetConfigPanel } from './view/TargetConfigPanel'; +import { NewProjectPanel } from './view/NewProjectPanel'; +import { + checkIfDirectoryExists, + writeIntoFile, + readFromFile, +} from './utilities/directoryUtilities'; +import { decorationProvider } from './decorationProvider'; +import { WasmDebugConfigurationProvider } from './debugConfigurationProvider'; +import { + isLLDBInstalled, + promptInstallLLDB, + getWAMRExtensionVersion, +} from './utilities/lldbUtilities'; + +import { + checkIfDockerStarted, + checkIfDockerImagesExist, + promptSetupDockerImages, +} from './utilities/dockerUtilities'; +import { SelectionOfPrompt } from './constants'; + +let wasmTaskProvider: WasmTaskProvider; +let wasmDebugConfigProvider: WasmDebugConfigurationProvider; +let currentPrjDir = ''; +let isWasmProject = false; + +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export async function activate(context: vscode.ExtensionContext) { + const extensionPath = context.extensionPath; + const osPlatform = os.platform(); + const wamrVersion = getWAMRExtensionVersion(context.extensionPath); + const typeMap = new Map(); + const scriptMap = new Map(); + /* set relative path of build.bat|sh script */ + const scriptPrefix = 'resource/scripts/'; + + let buildScript = '', + runScript = '', + debugScript = '', + destroyScript = '', + buildScriptFullPath = '', + runScriptFullPath = '', + debugScriptFullPath = '', + destroyScriptFullPath = '', + /* include paths array used for written into config file */ + includePathArr = new Array(), + /* exclude files array used for written into config file */ + excludeFileArr = new Array(); + + /** + * Provide Build & Run Task with Task Provider instead of "tasks.json" + */ + + if (osPlatform === 'win32') { + buildScript = scriptPrefix.concat('build.bat'); + runScript = scriptPrefix.concat('run.bat'); + debugScript = scriptPrefix.concat('boot_debugger_server.bat'); + destroyScript = scriptPrefix.concat('destroy.bat'); + } else if (osPlatform === 'linux' || osPlatform === 'darwin') { + buildScript = scriptPrefix.concat('build.sh'); + runScript = scriptPrefix.concat('run.sh'); + debugScript = scriptPrefix.concat('boot_debugger_server.sh'); + destroyScript = scriptPrefix.concat('destroy.sh'); + } + + buildScriptFullPath = path.join(extensionPath, buildScript); + runScriptFullPath = path.join(extensionPath, runScript); + debugScriptFullPath = path.join(extensionPath, debugScript); + destroyScriptFullPath = path.join(extensionPath, destroyScript); + + scriptMap.set('buildScript', buildScriptFullPath); + scriptMap.set('runScript', runScriptFullPath); + scriptMap.set('debugScript', debugScriptFullPath); + scriptMap.set('destroyScript', destroyScriptFullPath); + + typeMap.set('Build', 'Build'); + typeMap.set('Run', 'Run'); + typeMap.set('Debug', 'Debug'); + typeMap.set('Destroy', 'Destroy'); + + wasmTaskProvider = new WasmTaskProvider(typeMap, scriptMap, wamrVersion); + + vscode.tasks.registerTaskProvider('wasm', wasmTaskProvider); + + if (vscode.workspace.workspaceFolders?.[0]) { + if (osPlatform === 'win32') { + currentPrjDir = vscode.workspace.workspaceFolders?.[0].uri + .fsPath as string; + } else if (osPlatform === 'linux' || osPlatform === 'darwin') { + currentPrjDir = vscode.workspace.workspaceFolders?.[0].uri + .path as string; + } + + /** + * check whether current project opened in vscode workspace is wasm project + * it not, `build`, `run` and `debug` will be disabled + */ + if (currentPrjDir !== '') { + const wamrFolder = fileSystem + .readdirSync(currentPrjDir, { + withFileTypes: true, + }) + .filter( + folder => folder.isDirectory() && folder.name === '.wamr' + ); + + if (wamrFolder.length !== 0) { + isWasmProject = true; + vscode.commands.executeCommand( + 'setContext', + 'ext.isWasmProject', + isWasmProject + ); + if ( + vscode.workspace + .getConfiguration() + .has('C_Cpp.default.systemIncludePath') + ) { + let newIncludeInCppArr: string[] | undefined | null; + + newIncludeInCppArr = vscode.workspace + .getConfiguration() + .get('C_Cpp.default.systemIncludePath'); + + const libcBuiltinHeaderPath = path.join( + extensionPath, + 'resource/wamr-sdk/libc-builtin-sysroot/include' + ); + + if (newIncludeInCppArr !== undefined) { + /* in case the configuration has not been set up, push directly */ + if (newIncludeInCppArr === null) { + newIncludeInCppArr = []; + newIncludeInCppArr.push(libcBuiltinHeaderPath); + } else { + /* if the configuration has been set up, check the condition */ + if ( + /* include libc-builtin-sysroot */ + newIncludeInCppArr.indexOf( + libcBuiltinHeaderPath + ) < 0 + ) { + newIncludeInCppArr.push(libcBuiltinHeaderPath); + } + } + + vscode.workspace + .getConfiguration() + .update( + 'C_Cpp.default.systemIncludePath', + newIncludeInCppArr, + vscode.ConfigurationTarget.Workspace + ); + } + } + } + } + } + + /* register debug configuration */ + wasmDebugConfigProvider = new WasmDebugConfigurationProvider( + context.extensionPath + ); + + vscode.debug.registerDebugConfigurationProvider( + 'wamr-debug', + wasmDebugConfigProvider + ); + + /* update ext.includePaths to show or hide 'Remove' button in menus */ + vscode.commands.executeCommand('setContext', 'ext.supportedFileType', [ + '.c', + '.cpp', + '.cxx', + ]); + + if (readFromConfigFile() !== '') { + const configData = JSON.parse(readFromConfigFile()); + includePathArr = configData['includePaths']; + excludeFileArr = configData['excludeFiles']; + + if (Object.keys(configData['buildArgs']).length !== 0) { + TargetConfigPanel.buildArgs = configData['buildArgs']; + } + } + + const disposableNewProj = vscode.commands.registerCommand( + 'wamride.newProject', + () => { + const okStr = 'Set up now'; + const cancelStr = 'Maybe later'; + const curWorkspace = vscode.workspace + .getConfiguration() + .get('WAMR-IDE.configWorkspace'); + + /* if user has not set up workspace yet, prompt to set up */ + if (curWorkspace === '' || curWorkspace === undefined) { + vscode.window + .showWarningMessage( + 'Please setup your workspace firstly.', + okStr, + cancelStr + ) + .then(item => { + if (item === okStr) { + vscode.commands.executeCommand( + 'wamride.changeWorkspace' + ); + } else { + return; + } + }); + } else if (!checkIfDirectoryExists(curWorkspace as string)) { + vscode.window + .showWarningMessage( + 'Invalid workspace:', + { + modal: true, + detail: + '' + + vscode.workspace + .getConfiguration() + .get('WAMR-IDE.configWorkspace') + + '', + }, + okStr + ) + .then(item => { + if (item === okStr) { + vscode.commands.executeCommand( + 'wamride.changeWorkspace' + ); + } else { + return; + } + }); + } else { + NewProjectPanel.render(context); + } + } + ); + + const disposableTargetConfig = vscode.commands.registerCommand( + 'wamride.targetConfig', + () => { + if (currentPrjDir !== '') { + TargetConfigPanel.render(context); + } else { + vscode.window.showWarningMessage( + 'Please create and open project firstly.', + 'OK' + ); + } + } + ); + + const disposableChangeWorkspace = vscode.commands.registerCommand( + 'wamride.changeWorkspace', + async () => { + const options: vscode.OpenDialogOptions = { + canSelectFiles: false, + canSelectFolders: true, + openLabel: 'Select Workspace', + }; + + const workSpace = await vscode.window + .showOpenDialog(options) + .then(res => { + if (res) { + return res[0].fsPath as string; + } else { + return ''; + } + }); + + /* update workspace value to vscode global settings */ + if (workSpace !== '' && workSpace !== undefined) { + await vscode.workspace + .getConfiguration() + .update( + 'WAMR-IDE.configWorkspace', + workSpace.trim(), + vscode.ConfigurationTarget.Global + ) + .then( + () => { + vscode.window.showInformationMessage( + 'Workspace has been set up successfully!' + ); + }, + () => { + vscode.window.showErrorMessage( + 'Set up Workspace failed!' + ); + } + ); + } + } + ); + + const disposableBuild = vscode.commands.registerCommand( + 'wamride.build', + async () => { + if (!isWasmProject) { + vscode.window.showErrorMessage('Build failed', { + modal: true, + detail: 'Current project is not wasm project, please open wasm project and try again.', + }); + return; + } + + try { + /* check if docker images are ready before building */ + if ( + (await checkIfDockerStarted()) && + !(await checkIfDockerImagesExist(context)) + ) { + /**NOTE - if users select to skip install, + * we should return rather than continue + * the execution + */ + if ( + (await promptSetupDockerImages(context)) === + SelectionOfPrompt.skip + ) { + return; + } + } + } catch (e) { + vscode.window.showWarningMessage((e as Error).message); + return; + } + + generateCMakeFile(includePathArr, excludeFileArr); + /* destroy the wasm-toolchain-ctr if it exists */ + vscode.commands + .executeCommand( + 'workbench.action.tasks.runTask', + 'Destroy: Wasm-Container-Before-Build' + ) + .then(() => { + const disposable = vscode.tasks.onDidEndTaskProcess(t => { + if ( + t.execution.task.name === + 'Wasm-Container-Before-Build' + ) { + if (t.exitCode !== 0) { + disposable.dispose(); + return; + } + + /* execute the build task */ + vscode.commands + .executeCommand( + 'workbench.action.tasks.runTask', + 'Build: Wasm' + ) + .then(() => { + /* destroy the wasm-toolchain-ctr after building */ + const disposableAft = + vscode.tasks.onDidEndTask(a => { + if ( + a.execution.task.name === + 'Wasm' && + a.execution.task.source === + 'Build' + ) { + vscode.commands + .executeCommand( + 'workbench.action.tasks.runTask', + 'Destroy: Wasm-Container-After-Build' + ) + .then(() => { + /* dispose the event after this building process + */ + disposableAft.dispose(); + }); + } + }); + }); + /* dispose the event after this building process */ + disposable.dispose(); + } + }); + }); + } + ); + + const disposableDebug = vscode.commands.registerCommand( + 'wamride.debug', + async () => { + if (!isWasmProject) { + vscode.window.showErrorMessage('debug failed', { + modal: true, + detail: 'Current project is not wasm project, please open wasm project and try again.', + }); + return; + } + + /* we should check again whether the user installed lldb, as this can be skipped during activation */ + try { + if (!isLLDBInstalled(context.extensionPath)) { + /**NOTE - if users select to skip install, + * we should return rather than continue + * the execution + */ + if ( + (await promptInstallLLDB(context.extensionPath)) === + SelectionOfPrompt.skip + ) { + return; + } + } + + if ( + (await checkIfDockerStarted()) && + !(await checkIfDockerImagesExist(context)) + ) { + /**NOTE - save as above lldb, should return if + * users select to skip set up + */ + if ( + (await promptSetupDockerImages(context)) === + SelectionOfPrompt.skip + ) { + return; + } + } + } catch (e) { + vscode.window.showWarningMessage((e as Error).message); + return; + } + + /* refuse to debug if build process failed */ + if (!checkIfBuildSuccess()) { + vscode.window.showErrorMessage('Debug failed', { + modal: true, + detail: 'Can not find WASM binary, please build WASM firstly.', + }); + return; + } + + /* show debug view */ + vscode.commands.executeCommand('workbench.view.debug'); + + /* should destroy the wasm-debug-server-ctr before debugging */ + vscode.commands + .executeCommand( + 'workbench.action.tasks.runTask', + 'Destroy: Wasm-Container-Before-Debug' + ) + .then(() => { + /* execute the debug task when destroy task finish */ + const disposableBfr = vscode.tasks.onDidEndTask(t => { + if ( + t.execution.task.name === + 'Wasm-Container-Before-Debug' + ) { + vscode.commands + .executeCommand( + 'workbench.action.tasks.runTask', + 'Debug: Wasm' + ) + .then(() => { + vscode.debug + .startDebugging( + undefined, + wasmDebugConfigProvider.getDebugConfig() + ) + .then(() => { + /* register to listen debug session finish event */ + const disposableAft = + vscode.debug.onDidTerminateDebugSession( + s => { + if ( + s.type !== + 'wamr-debug' + ) { + return; + } + + /* execute the task to destroy + * wasm-debug-server-ctr */ + vscode.commands.executeCommand( + 'workbench.action.tasks.runTask', + 'Destroy: Wasm-Container-After-Debug' + ); + + /* execute the task to kill the terminal */ + vscode.commands.executeCommand( + 'workbench.action.terminal.kill', + 'Debug: Wasm' + ); + + disposableAft.dispose(); + } + ); + }); + }); + } + disposableBfr.dispose(); + }); + }); + } + ); + + const disposableRun = vscode.commands.registerCommand( + 'wamride.run', + async () => { + if (!isWasmProject) { + vscode.window.showErrorMessage('run failed', { + modal: true, + detail: 'Current project is not wasm project, please open wasm project and try again.', + }); + return; + } + + try { + /* check if docker images are set up before building */ + if ( + (await checkIfDockerStarted()) && + !(await checkIfDockerImagesExist(context)) + ) { + await promptSetupDockerImages(context); + } + } catch (e) { + vscode.window.showWarningMessage((e as Error).message); + return; + } + + /* refuse to debug if build process failed */ + if (!checkIfBuildSuccess()) { + vscode.window.showErrorMessage('Debug failed', { + modal: true, + detail: 'Can not find WASM binary, please build WASM firstly.', + }); + return; + } + + vscode.commands + .executeCommand( + 'workbench.action.tasks.runTask', + 'Destroy: Wasm-Container-Before-Run' + ) + .then(() => { + const disposableAft = vscode.tasks.onDidEndTaskProcess( + e => { + if ( + e.execution.task.name === + 'Wasm-Container-Before-Run' + ) { + /* make sure that run wasm task will be executed when destroy task finish */ + vscode.commands + .executeCommand( + 'workbench.action.tasks.runTask', + 'Run: Wasm' + ) + .then(() => { + if (e.exitCode !== 0) { + disposableAft.dispose(); + return; + } + }); + disposableAft.dispose(); + } + } + ); + }); + } + ); + + const disposableToggleIncludePath = vscode.commands.registerCommand( + 'wamride.build.toggleStateIncludePath', + fileUri => { + const path = + fileUri._fsPath !== null && fileUri._fsPath !== undefined + ? fileUri._fsPath + : vscode.Uri.parse(fileUri.path as string).fsPath; + const pathRelative = path.replace(currentPrjDir, '..'); + + if (includePathArr.indexOf(pathRelative) > -1) { + /* this folder has been added to include path, remove it */ + includePathArr = includePathArr.filter(value => { + return value !== pathRelative; + }); + } else { + includePathArr.push(pathRelative); + } + + writeIntoConfigFile( + includePathArr, + excludeFileArr, + TargetConfigPanel.buildArgs + ); + + decorationProvider.updateDecorationsForSource(fileUri); + } + ); + + const disposableToggleExcludeFile = vscode.commands.registerCommand( + 'wamride.build.toggleStateExclude', + fileUri => { + const path = + fileUri._fsPath !== null && fileUri._fsPath !== undefined + ? fileUri._fsPath + : vscode.Uri.parse(fileUri.path as string).fsPath; + + /* replace the current project absolute path with .. to change to relative path */ + const pathRelative = path.replace(currentPrjDir, '..'); + + if (excludeFileArr.indexOf(pathRelative) > -1) { + excludeFileArr = excludeFileArr.filter(val => { + return val !== pathRelative; + }); + } else { + excludeFileArr.push(pathRelative); + } + + writeIntoConfigFile( + includePathArr, + excludeFileArr, + TargetConfigPanel.buildArgs + ); + + /* update decoration for this source file */ + decorationProvider.updateDecorationsForSource(fileUri); + } + ); + + const disposableOpenFolder = vscode.commands.registerCommand( + 'wamride.openFolder', + () => { + /* get projects list under current workspace */ + const okStr = 'Set up now'; + const cancelStr = 'Maybe later'; + const createStr = 'Create now'; + const curWorkspace = vscode.workspace + .getConfiguration() + .get('WAMR-IDE.configWorkspace') as string; + + /* if user has not set up workspace yet, prompt to set up */ + if (curWorkspace === '' || curWorkspace === undefined) { + vscode.window + .showWarningMessage( + 'Please setup your workspace firstly.', + okStr, + cancelStr + ) + .then(item => { + if (item === okStr) { + vscode.commands.executeCommand( + 'wamride.changeWorkspace' + ); + } else { + return; + } + }); + } else if (!checkIfDirectoryExists(curWorkspace as string)) { + vscode.window + .showWarningMessage( + 'Invalid workspace:', + { + modal: true, + detail: + '' + + vscode.workspace + .getConfiguration() + .get('WAMR-IDE.configWorkspace') + + '', + }, + okStr + ) + .then(item => { + if (item === okStr) { + vscode.commands.executeCommand( + 'wamride.changeWorkspace' + ); + } else { + return; + } + }); + } else { + /* get all directories within directory, ignore files */ + let directoryArrDirent, directoryArr; + try { + directoryArrDirent = fileSystem.readdirSync(curWorkspace, { + withFileTypes: true, + }); + } catch (err) { + vscode.window.showErrorMessage( + 'Read projects from current workspace failed.' + ); + } + + if (directoryArrDirent !== undefined) { + directoryArr = directoryArrDirent + .filter(dirent => dirent.isDirectory()) + .map(dirent => dirent.name); + + const projFilesArr = directoryArr.filter(obj => { + if (checkIfWasmProj(path.join(curWorkspace, obj))) { + return true; + } + }); + + if (projFilesArr.length === 0) { + vscode.window + .showWarningMessage( + 'Current workspace is empty, please create your project firstly.', + createStr, + cancelStr + ) + .then(item => { + if (item === createStr) { + vscode.commands.executeCommand( + 'wamride.newProject' + ); + } else { + return; + } + }); + } else { + vscode.window + .showQuickPick(projFilesArr, { + title: 'Select project', + placeHolder: 'Please select project', + }) + .then(option => { + if (!option) { + return; + } + + const path = curWorkspace.concat( + osPlatform === 'win32' + ? '\\' + : osPlatform === 'linux' || + osPlatform === 'darwin' + ? '/' + : '', + option + ); + + /* open the selected wasm project */ + openWindowWithSituation(vscode.Uri.file(path)); + }); + } + } + } + } + ); + + context.subscriptions.push( + disposableNewProj, + disposableTargetConfig, + disposableChangeWorkspace, + disposableBuild, + disposableRun, + disposableToggleIncludePath, + disposableOpenFolder, + disposableToggleExcludeFile, + disposableDebug + ); + + try { + if (!isLLDBInstalled(context.extensionPath)) { + await promptInstallLLDB(context.extensionPath); + } + + if ( + (await checkIfDockerStarted()) && + !(await checkIfDockerImagesExist(context)) + ) { + await promptSetupDockerImages(context); + } + } catch (e) { + vscode.window.showWarningMessage((e as Error).message); + } +} + +function openWindowWithSituation(uri: vscode.Uri) { + /** + * check if the workspace folder is empty, + * if yes, open new window, else open in current window + */ + const isWorkspaceEmpty = !vscode.workspace.workspaceFolders?.[0] + ? true + : false; + + isWorkspaceEmpty === false + ? vscode.commands.executeCommand('vscode.openFolder', uri, { + forceNewWindow: true, + }) + : vscode.commands.executeCommand('vscode.openFolder', uri); + + return; +} + +interface BuildArgs { + outputFileName: string; + initMemorySize: string; + maxMemorySize: string; + stackSize: string; + exportedSymbols: string; + hostManagedHeapSize: string; +} + +/** + * @param: includePathArr + * @param: excludeFileArr + * Get current includePathArr and excludeFileArr from the json string that + * will be written into compilation_config.json + */ +export function writeIntoConfigFile( + includePathArr: string[], + excludeFileArr: string[], + buildArgs?: BuildArgs +): void { + const jsonStr = JSON.stringify( + { + includePaths: includePathArr, + excludeFiles: excludeFileArr, + buildArgs: buildArgs ? buildArgs : '{}', + }, + null, + '\t' + ); + + const prjConfigDir = path.join(currentPrjDir, '.wamr'); + const configFilePath = path.join(prjConfigDir, 'compilation_config.json'); + writeIntoFile(configFilePath, jsonStr); +} + +export function readFromConfigFile(): string { + const prjConfigDir = path.join(currentPrjDir, '.wamr'); + const configFilePath = path.join(prjConfigDir, 'compilation_config.json'); + return readFromFile(configFilePath); +} + +/** + * will be triggered when the user clicking `build` button + */ +function generateCMakeFile( + includePathArr: string[], + excludeFileArr: string[] +): void { + // -Wl,--export=${EXPORT_SYMBOLS} + const srcFilePath = path.join(currentPrjDir, 'src'); + const prjConfigDir = path.join(currentPrjDir, '.wamr'); + const cmakeFilePath = path.join(prjConfigDir, 'project.cmake'); + + let strIncludeList = 'set (PROJECT_INCLUDES'; + let strSrcList = 'set (PROJECT_SRC_LIST'; + + let strOutputFileName = 'set (OUTPUT_FILE_NAME'; + let strInitMemSize = 'set (INIT_MEM_SIZE'; + let strMaxMemSize = 'set (MAX_MEM_SIZE'; + let strStackSize = 'set (STACK_SIZE'; + let strExportedSymbols = 'set (EXPORTED_SYMBOLS'; + + let fullStr = ''; + let i, s, e: number; + + /* change the absolute path into relative path */ + const _re = currentPrjDir; + const _substr = '${CMAKE_CURRENT_SOURCE_DIR}/..'; + + /** + * set PROJECT_SRC_LIST + * default ADD every c OR c++ OR cpp under the src/ path + * except the files saved in the excludeFiles array + */ + + const srcPathArr = getAllSrcFiles(srcFilePath); + + if (srcPathArr === undefined) { + return; + } + + for (s = 0; s < srcPathArr.length; s++) { + if ( + excludeFileArr.indexOf( + srcPathArr[s].path.replace(currentPrjDir, '..') + ) === -1 + ) { + /* replace currentPrjDir with ${CMAKE_CURRENT_SOURCE_DIR} */ + const newStr = srcPathArr[s].path + .replace(_re, _substr) + .replace(/\\/g, '/'); + + strSrcList = strSrcList.concat(' ', newStr); + } + } + strSrcList = strSrcList.concat(' )'); + + for (i = 0; i < includePathArr.length; i++) { + const newStr = includePathArr[i] + .replace(/../, _substr) + .replace(/\\/g, '/'); + strIncludeList = strIncludeList.concat(' ', newStr); + } + strIncludeList = strIncludeList.concat(' )'); + + /* set up user customized input in configBuildArgs webview */ + strOutputFileName = strOutputFileName.concat( + ' ', + TargetConfigPanel.buildArgs.outputFileName + ')' + ); + + strInitMemSize = strInitMemSize.concat( + ' ', + TargetConfigPanel.buildArgs.initMemorySize + ')' + ); + + strMaxMemSize = strMaxMemSize.concat( + ' ', + TargetConfigPanel.buildArgs.maxMemorySize + ')' + ); + + strStackSize = strStackSize.concat( + ' ', + TargetConfigPanel.buildArgs.stackSize + ')' + ); + + const exportedSymbolArr = + TargetConfigPanel.buildArgs.exportedSymbols.split(','); + + strExportedSymbols = strExportedSymbols.concat(' "'); + + for (e = 0; e < exportedSymbolArr.length; e++) { + strExportedSymbols = strExportedSymbols.concat( + ' -Wl,', + '--export=', + exportedSymbolArr[e] + ); + } + + strExportedSymbols = strExportedSymbols.concat('")'); + + fullStr = strOutputFileName + .concat('\n', strInitMemSize) + .concat('\n', strMaxMemSize) + .concat('\n', strStackSize) + .concat('\n', strExportedSymbols) + .concat('\n', strSrcList) + .concat('\n', strIncludeList); + + writeIntoFile(cmakeFilePath, fullStr); +} + +function getAllSrcFiles(_path: string) { + try { + const entries = fileSystem.readdirSync(_path, { + withFileTypes: true, + }); + + const files = entries + .filter( + /* filter files mismatch .c |.cpp |.cxx */ + file => + !file.isDirectory() && file.name.match('(.c|.cpp|.cxx)$') + ) + .map(file => ({ + path: path.join(_path, file.name), + })); + + const folders = entries.filter(folder => folder.isDirectory()); + + for (const folder of folders) { + const fileArr = getAllSrcFiles(path.join(_path, folder.name)); + fileArr ? files.push(...fileArr) : ''; + } + + return files; + } catch (error) { + vscode.window.showErrorMessage(error as string); + } +} + +function checkIfBuildSuccess(): boolean { + try { + let wasmExist = false; + const entries = fileSystem.readdirSync( + path.join(currentPrjDir, 'build'), + { + withFileTypes: true, + } + ); + + entries.map(e => { + if (e.name.match('(.wasm)$')) { + wasmExist = true; + } + }); + + return wasmExist; + } catch { + return false; + } +} + +function checkIfWasmProj(path: string): boolean { + try { + let isWasmProj = false; + const entries = fileSystem.readdirSync(path, { + withFileTypes: true, + }); + + entries.map(e => { + if (e.isDirectory() && e.name === '.wamr') { + isWasmProj = true; + } + }); + + return isWasmProj; + } catch { + return false; + } +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/taskProvider.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/taskProvider.ts new file mode 100644 index 0000000..ee8ba5d --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/taskProvider.ts @@ -0,0 +1,229 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as vscode from 'vscode'; +import * as os from 'os'; +import { TargetConfigPanel } from './view/TargetConfigPanel'; + +export interface OwnShellOption { + cmd: string; + options: vscode.ShellExecutionOptions; +} + +export class WasmTaskProvider implements vscode.TaskProvider { + constructor( + public _type: Map, + public _script: Map, + public _wamrVersion: string + ) {} + + buildShellOption: OwnShellOption | undefined; + runShellOption: OwnShellOption | undefined; + debugShellOption: OwnShellOption | undefined; + destroyShellOption: OwnShellOption | undefined; + + private wasmPromise: Thenable | undefined = undefined; + + public provideTasks(): Thenable | undefined { + if (!this.wasmPromise) { + /* target name is used for generated aot target */ + const targetName = + TargetConfigPanel.buildArgs.outputFileName.split('.')[0]; + const heapSize = TargetConfigPanel.buildArgs.hostManagedHeapSize; + + if ( + os.platform() === 'linux' || + os.platform() === 'darwin' || + os.platform() === 'win32' + ) { + /* build */ + this.buildShellOption = { + cmd: + os.platform() === 'linux' || os.platform() === 'darwin' + ? 'bash' + : (this._script.get('buildScript') as string), + options: { + executable: this._script.get('buildScript'), + shellArgs: [targetName, this._wamrVersion], + }, + }; + + /* debug */ + this.debugShellOption = { + cmd: + os.platform() === 'linux' || os.platform() === 'darwin' + ? 'bash' + : (this._script.get('debugScript') as string), + options: { + executable: this._script.get('debugScript'), + shellArgs: [targetName, this._wamrVersion, heapSize], + }, + }; + + /* run */ + this.runShellOption = { + cmd: + os.platform() === 'linux' || os.platform() === 'darwin' + ? 'bash' + : (this._script.get('runScript') as string), + options: { + executable: this._script.get('runScript'), + shellArgs: [targetName, this._wamrVersion, heapSize], + }, + }; + + /* destroy */ + /* run */ + this.destroyShellOption = { + cmd: + os.platform() === 'linux' || os.platform() === 'darwin' + ? 'bash' + : (this._script.get('destroyScript') as string), + options: { + executable: this._script.get('destroyScript'), + shellArgs: [targetName], + }, + }; + } else { + this.buildShellOption = { + cmd: "echo 'os platform is not supported yet'", + options: {}, + }; + + this.debugShellOption = { + cmd: "echo 'os platform is not supported yet'", + options: {}, + }; + + this.runShellOption = { + cmd: "echo 'os platform is not supported yet'", + options: {}, + }; + + this.destroyShellOption = { + cmd: "echo 'os platform is not supported yet'", + options: {}, + }; + } + + this.wasmPromise = Promise.resolve([ + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm', + this._type.get('Build') as string, + new vscode.ShellExecution( + this.buildShellOption.cmd, + this.buildShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm', + this._type.get('Run') as string, + new vscode.ShellExecution( + this.runShellOption.cmd, + this.runShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm', + this._type.get('Debug') as string, + new vscode.ShellExecution( + this.debugShellOption.cmd, + this.debugShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm-Container-Before-Build', + this._type.get('Destroy') as string, + new vscode.ShellExecution( + this.destroyShellOption.cmd, + this.destroyShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm-Container-Before-Debug', + this._type.get('Destroy') as string, + new vscode.ShellExecution( + this.destroyShellOption.cmd, + this.destroyShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm-Container-Before-Run', + this._type.get('Destroy') as string, + new vscode.ShellExecution( + this.destroyShellOption.cmd, + this.destroyShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm-Container-After-Build', + this._type.get('Destroy') as string, + new vscode.ShellExecution( + this.destroyShellOption.cmd, + this.destroyShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm-Container-After-Debug', + this._type.get('Destroy') as string, + new vscode.ShellExecution( + this.destroyShellOption.cmd, + this.destroyShellOption.options + ) + ), + + new vscode.Task( + { type: 'wasm' }, + vscode.TaskScope.Workspace, + 'Wasm-Container-After-Run', + this._type.get('Destroy') as string, + new vscode.ShellExecution( + this.destroyShellOption.cmd, + this.destroyShellOption.options + ) + ), + ]); + } + + return this.wasmPromise; + } + + /** + * if the task or task in tasks.json does not set command, ` + * resolveTask` will be invoked, + * otherwise, `provideTasks` will be invoked + * @param _task + * @returns + */ + public resolveTask(task: vscode.Task): vscode.Task | undefined { + if (task) { + return task; + } + return undefined; + } +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/runTest.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/runTest.ts new file mode 100644 index 0000000..635e02e --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/runTest.ts @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as path from 'path'; +import * as os from 'os'; + +import { runTests } from '@vscode/test-electron'; + +async function main() { + try { + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + const extensionDevelopmentPath = path.resolve(__dirname, '../../'); + + // The path to the extension test script + // Passed to --extensionTestsPath + const extensionTestsPath = path.resolve(__dirname, './suite/index'); + + // Download VS Code, unzip it and run the integration test + await runTests({ + extensionDevelopmentPath, + extensionTestsPath, + launchArgs: ['--user-data-dir', `${os.tmpdir()}`], + }); + } catch (err) { + console.error('Failed to run tests'); + process.exit(1); + } +} + +main(); diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/extension.test.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/extension.test.ts new file mode 100644 index 0000000..d1420df --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/extension.test.ts @@ -0,0 +1,243 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import { DebugProtocol } from '@vscode/debugprotocol'; +import { after, before, test, suite } from 'mocha'; +import { assert } from 'chai'; +import * as vscode from 'vscode'; +import * as cp from 'child_process'; +import * as path from 'path'; +import * as os from 'os'; +import { + WasmDebugConfig, + WasmDebugConfigurationProvider, +} from '../../debugConfigurationProvider'; +import { + EXTENSION_PATH, + clearAllBp, + setBpAtMarker, + compileRustToWasm, +} from './utils'; +import { downloadLldb, isLLDBInstalled } from '../../utilities/lldbUtilities'; + +suite('Unit Tests', function () { + test('DebugConfigurationProvider init commands', function () { + const testExtensionPath = '/test/path/'; + const provider = new WasmDebugConfigurationProvider(testExtensionPath); + + assert.includeMembers( + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + provider.getDebugConfig().initCommands!, + [`command script import ${testExtensionPath}/formatters/rust.py`], + 'Debugger init commands did not contain ' + ); + }); + + test('DebugConfigurationProvider resolve configuration', function () { + const testExtensionPath = '/test/path/'; + const provider = new WasmDebugConfigurationProvider(testExtensionPath); + + const actual = provider.resolveDebugConfiguration(undefined, { + type: 'wamr-debug', + name: 'Attach', + request: 'attach', + initCommands: [], + attachCommands: [ + 'process connect -p wasm connect://123.456.789.1:1237', + ], + }); + + assert.deepEqual( + actual, + { + type: 'wamr-debug', + name: 'Attach', + request: 'attach', + stopOnEntry: true, + initCommands: [], + attachCommands: [ + 'process connect -p wasm connect://123.456.789.1:1237', + ], + }, + 'Configuration did not match the expected configuration after calling resolveDebugConfiguration()' + ); + }); +}); + +suite('Inegration Tests', function () { + let debuggerProcess: cp.ChildProcessWithoutNullStreams; + const port = 1239; + const downloadTimeout = 60 * 1000; + + before(async function () { + // timeout of 20 seconds + this.timeout(20 * 1000); + // Download LLDB if necessary. Should be available in the CI. Only for local execution. + if (!isLLDBInstalled(EXTENSION_PATH)) { + this.timeout(downloadTimeout); + console.log('Downloading LLDB. This might take a moment...'); + await downloadLldb(EXTENSION_PATH); + assert.isTrue( + isLLDBInstalled(EXTENSION_PATH), + 'LLDB was not installed correctly' + ); + } + + compileRustToWasm(); + + const platform = os.platform(); + assert.isTrue( + platform === 'darwin' || platform === 'linux', + `Tests do not support your platform: ${platform}` + ); + const iWasmPath = path.resolve( + `${EXTENSION_PATH}/../../../product-mini/platforms/${platform}/build/iwasm` + ); + const testWasmFilePath = `${EXTENSION_PATH}/resource/test/test.wasm`; + + debuggerProcess = cp.spawn( + iWasmPath, + [`-g=127.0.0.1:${port}`, testWasmFilePath], + {} + ); + + debuggerProcess.stderr.on('data', data => { + console.log(`Error from debugger process: ${data}`); + }); + }); + + after(async function () { + await vscode.debug.stopDebugging(); + debuggerProcess.kill(); + }); + + test('Rust formatters', async function () { + // timeout of 1 minutes + this.timeout(60 * 1000); + clearAllBp(); + setBpAtMarker(`${EXTENSION_PATH}/resource/test/test.rs`, 'BP_MARKER_1'); + + const getVariables = new Promise( + (resolve, reject) => { + vscode.debug.registerDebugAdapterTrackerFactory('wamr-debug', { + createDebugAdapterTracker: function () { + return { + // The debug adapter has sent a Debug Adapter Protocol message to the editor. + onDidSendMessage: ( + message: DebugProtocol.ProtocolMessage + ) => { + if (message.type === 'response') { + const m = message as DebugProtocol.Response; + if (m.command === 'variables') { + const res = + m as DebugProtocol.VariablesResponse; + resolve(res.body.variables); + } + } + }, + onError: (error: Error) => { + reject( + 'An error occurred before vscode reached the breakpoint: ' + + error + ); + }, + onExit: (code: number | undefined) => { + reject( + `Debugger exited before vscode reached the breakpoint with code: ${code}` + ); + }, + }; + }, + }); + } + ); + + const config: WasmDebugConfig = { + type: 'wamr-debug', + request: 'attach', + name: 'Attach Debugger', + stopOnEntry: false, + initCommands: [ + `command script import ${EXTENSION_PATH}/formatters/rust.py`, + ], + attachCommands: [ + `process connect -p wasm connect://127.0.0.1:${port}`, + ], + }; + + if (os.platform() === 'win32' || os.platform() === 'darwin') { + config.initCommands?.push('platform select remote-linux'); + } + + try { + await vscode.debug.startDebugging(undefined, config); + } catch (e) { + assert.fail('Could not connect to debug adapter'); + } + + // wait until vs code has reached breakpoint and has requested the variables. + const variables = await getVariables; + const namesToVariables = variables.reduce( + (acc: { [name: string]: DebugProtocol.Variable }, c) => { + if (c.evaluateName) { + acc[c.evaluateName] = c; + } + return acc; + }, + {} + ); + + assert.includeMembers( + Object.keys(namesToVariables), + ['vector', 'map', 'string', 'slice', 'deque', 'ref_cell'], + 'The Debugger did not return all expected debugger variables.' + ); + + // Vector + assert.equal( + namesToVariables['vector'].value, + ' (5) vec![1, 2, 3, 4, 12]', + 'The Vector summary string looks different than expected' + ); + + // Map + assert.equal( + namesToVariables['map'].value, + ' size=5, capacity=8', + 'The Map summary string looks different than expected' + ); + + // String + assert.equal( + namesToVariables['string'].value, + ' "this is a string"', + 'The String summary string looks different than expected' + ); + + // Slice + assert.equal( + namesToVariables['slice'].value, + ' "ello"', + 'The Slice summary string looks different than expected' + ); + + // Deque + // TODO: The deque format conversion have some problem now + // -alloc::collections::vec_deque::VecDeque @ 0xfff1c + // + (5) VecDeque[1, 2, 3, 4, 5] + // assert.equal( + // namesToVariables['deque'].value, + // ' (5) VecDeque[1, 2, 3, 4, 5]', + // 'The Deque summary string looks different than expected' + // ); + + // RefCell + assert.equal( + namesToVariables['ref_cell'].value, + ' 5', + 'The RefCell summary string looks different than expected' + ); + }); +}); diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/index.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/index.ts new file mode 100644 index 0000000..3b7d271 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/index.ts @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as path from 'path'; +import * as Mocha from 'mocha'; +import * as glob from 'glob'; + +export function run(): Promise { + // Create the mocha test + const mocha = new Mocha({ + ui: 'tdd', + }); + + const testsRoot = path.resolve(__dirname, '..'); + + return new Promise((c, e) => { + glob('**/**.test.js', { cwd: testsRoot }, (err, files) => { + if (err) { + return e(err); + } + + // Add files to the test suite + files.forEach(f => mocha.addFile(path.resolve(testsRoot, f))); + + try { + // Run the mocha test + mocha.run(failures => { + if (failures > 0) { + e(new Error(`${failures} tests failed.`)); + } else { + c(); + } + }); + } catch (err) { + console.error(err); + e(err); + } + }); + }); +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/utils.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/utils.ts new file mode 100644 index 0000000..3f40596 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/test/suite/utils.ts @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import { assert } from 'chai'; +import * as vscode from 'vscode'; +import { Range, SourceBreakpoint } from 'vscode'; +import * as fs from 'fs'; +import path = require('path'); +import * as cp from 'child_process'; + +export const EXTENSION_PATH = path.resolve(`${__dirname}/../../..`); + +// clears all set breakpoints +export function clearAllBp(): void { + vscode.debug.removeBreakpoints(vscode.debug.breakpoints); +} + +// Inserts a breakpoint in a file at the first occurrence of bpMarker +export function setBpAtMarker(file: string, bpMarker: string): void { + const uri = vscode.Uri.file(file); + const data = fs.readFileSync(uri.path, 'utf8'); + const line = data.split('\n').findIndex(line => line.includes(bpMarker)); + assert.notStrictEqual( + line, + -1, + 'Could not find breakpoint marker in source file' + ); + const position = new vscode.Position(line, 0); + const bp = new SourceBreakpoint( + new vscode.Location(uri, new Range(position, position)), + true + ); + vscode.debug.addBreakpoints([bp]); +} + +// compiles resources/test/test.rs to test.wasm +export function compileRustToWasm(): void { + const testResourceFolder = `${EXTENSION_PATH}/resource/test`; + // compile with debug symbols and no optimization + const cmd = `rustc --target wasm32-wasi ${testResourceFolder}/test.rs -g -C opt-level=0 -o ${testResourceFolder}/test.wasm`; + + try { + cp.execSync(cmd, { stdio: [null, null, process.stderr] }); + } catch (e) { + assert.fail(`Compilation of example rust file failed with error: ${e}`); + } + assert.isTrue( + fs.existsSync(`${testResourceFolder}/test.wasm`), + 'Could not find wasm file WASM file to run debugger on.' + ); +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/directoryUtilities.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/directoryUtilities.ts new file mode 100644 index 0000000..0efbea5 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/directoryUtilities.ts @@ -0,0 +1,211 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import fileSystem = require('fs'); +import vscode = require('vscode'); +import path = require('path'); +import os = require('os'); +import request = require('request'); +import yauzl = require('yauzl'); + +/** + * + * @param path destination path + */ +export function createDirectory( + dest: string, + mode: string | number | null | undefined = undefined +): boolean { + try { + if (fileSystem.existsSync(dest)) { + if (fileSystem.lstatSync(dest).isDirectory()) { + return true; + } else { + return false; + } + } + + if (!path) { + return false; + } + + const parent = path.dirname(dest); + if (!createDirectory(parent, mode)) { + return false; + } + + fileSystem.mkdirSync(dest, mode); + return true; + } catch (error) { + vscode.window.showErrorMessage(error as string); + return false; + } +} + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export function copyFiles(src: string, dest: string, flags?: number): boolean { + try { + fileSystem.copyFileSync(src, dest); + return true; + } catch (error) { + vscode.window.showErrorMessage(error as string); + return false; + } +} + +export function writeIntoFile(path: string, data: string): void { + try { + fileSystem.writeFileSync(path, data, null); + } catch (err) { + vscode.window.showErrorMessage(err as string); + } +} + +export function readFromFile(path: string): string { + try { + const data = fileSystem.readFileSync(path, { encoding: 'utf-8' }); + return data as string; + } catch (err) { + vscode.window.showErrorMessage(err as string); + return ''; + } +} + +export function writeIntoFileAsync( + path: string, + data: string, + callback: fileSystem.NoParamCallback +): void { + try { + fileSystem.writeFile(path, data, callback); + } catch (err) { + vscode.window.showErrorMessage(err as string); + return; + } +} + +export function checkIfPathExists(path: string): boolean { + try { + if (fileSystem.existsSync(path)) { + return true; + } else { + return false; + } + } catch (err) { + vscode.window.showErrorMessage(err as string); + return false; + } +} + +export function checkIfDirectoryExists(path: string): boolean { + const doesPathExist = checkIfPathExists(path); + if (doesPathExist) { + return fileSystem.lstatSync(path).isDirectory(); + } + return false; +} + +export function checkIfFileExists(path: string): boolean { + const doesPathExist = checkIfPathExists(path); + if (doesPathExist) { + return fileSystem.lstatSync(path).isFile(); + } + return false; +} + +export function checkFolderName(folderName: string): boolean { + let invalidCharacterArr: string[] = []; + let valid = true; + + if (folderName.length > 255) { + valid = false; + } + + if (os.platform() === 'win32') { + invalidCharacterArr = ['\\', '/', ':', '?', '*', '"', '|', '<', '>']; + } else if (os.platform() === 'linux' || os.platform() === 'darwin') { + invalidCharacterArr = ['/']; + } + + invalidCharacterArr.forEach(function (c) { + if (folderName.indexOf(c) !== -1) { + valid = false; + } + }); + + return valid; +} + +export function downloadFile( + url: string, + destinationPath: string +): Promise { + return new Promise((resolve, reject) => { + const file = fileSystem.createWriteStream(destinationPath); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const stream = request(url, undefined, (error, response, body) => { + if (response.statusCode !== 200) { + reject( + new Error( + `Download from ${url} failed with ${response.statusMessage}` + ) + ); + } + }).pipe(file); + stream.on('close', resolve); + stream.on('error', reject); + }); +} + +export function unzipFile( + sourcePath: string, + getDestinationFileName: (entryName: string) => string +): Promise { + return new Promise((resolve, reject) => { + const unzippedFilePaths: string[] = []; + yauzl.open( + sourcePath, + { lazyEntries: true }, + function (error, zipfile) { + if (error) { + reject(error); + return; + } + zipfile.readEntry(); + zipfile.on('entry', function (entry) { + // This entry is a directory so skip it + if (/\/$/.test(entry.fileName)) { + zipfile.readEntry(); + return; + } + + zipfile.openReadStream(entry, function (error, readStream) { + if (error) { + reject(error); + return; + } + readStream.on('end', () => zipfile.readEntry()); + const destinationFileName = getDestinationFileName( + entry.fileName + ); + fileSystem.mkdirSync( + path.dirname(destinationFileName), + { recursive: true } + ); + + const file = + fileSystem.createWriteStream(destinationFileName); + readStream.pipe(file).on('error', reject); + unzippedFilePaths.push(destinationFileName); + }); + }); + zipfile.on('end', function () { + zipfile.close(); + resolve(unzippedFilePaths); + }); + } + ); + }); +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/dockerUtilities.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/dockerUtilities.ts new file mode 100644 index 0000000..4c2b40e --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/dockerUtilities.ts @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as vscode from 'vscode'; +import * as cp from 'child_process'; +import * as path from 'path'; +import * as fs from 'fs'; +import { getWAMRExtensionVersion } from './lldbUtilities'; +import { downloadFile, unzipFile } from './directoryUtilities'; +import { SelectionOfPrompt, Status } from '../constants'; + +const DOCKER_IMAGES_TEM_FOLDER_NAME = 'docker-resource'; + +type SelectionStatus = SelectionOfPrompt | Status; + +const execShell = (cmd: string) => + new Promise((resolve, reject) => { + cp.exec(cmd, (error, result) => { + if (error) { + return reject(error); + } + return resolve(result); + }); + }); + +export async function promptSetupDockerImages( + context: vscode.ExtensionContext +): Promise { + const extensionPath = context.extensionPath; + const response = await vscode.window.showWarningMessage( + 'Necessary docker images are not found. Setup now?', + SelectionOfPrompt.setUp, + SelectionOfPrompt.skip + ); + + if (response === SelectionOfPrompt.skip) { + return response; + } + + const downloadUrlArray = getDockerImagesDownloadUrl(context); + + const destinationFolder = path.resolve( + extensionPath, + 'resource', + DOCKER_IMAGES_TEM_FOLDER_NAME + ); + + if (!fs.existsSync(destinationFolder)) { + fs.mkdirSync(destinationFolder); + } + + vscode.window.showInformationMessage(`Downloading Docker Images...`); + + for (const url of downloadUrlArray) { + const imageZipName = path.basename(url); + const imageStorePath = path.join(destinationFolder, imageZipName); + await downloadFile(url, imageStorePath); + + /** + * extract docker image tar package to + * '${destinationFolder}' + */ + const dockerImageFile = await unzipFile(imageStorePath, filename => + path.join(destinationFolder, filename) + ); + /* give access before loading */ + dockerImageFile.forEach(file => fs.chmodSync(file, '0775')); + + /**NOTE - load docker image tar package to host + * right now there are just one file + * `docker-image-name.tar` inside so we can + * directly use files[0] here, should be modified + * if the package's files change + */ + await execShell(`docker load -i ${dockerImageFile[0]}`); + } + + /* remove the DOCKER_IMAGES_TEM_FOLDER */ + fs.rmSync(destinationFolder, { recursive: true, force: true }); + + vscode.window.showInformationMessage( + `Docker images are ready, please run '$docker images' to check.` + ); + + return Status.done; +} + +export async function checkIfDockerStarted(): Promise { + try { + await execShell('docker images'); + return true; + } catch (e) { + vscode.window.showWarningMessage((e as Error).message); + return false; + } +} + +export async function checkIfDockerImagesExist( + context: vscode.ExtensionContext +): Promise { + try { + /* the tag of images is equal to extension's version */ + const imageTag = getWAMRExtensionVersion(context.extensionPath); + await execShell( + `docker image inspect wasm-debug-server:${imageTag} wasm-toolchain:${imageTag}` + ); + return true; + } catch (e) { + return false; + } +} + +function getDockerImagesDownloadUrl( + context: vscode.ExtensionContext +): string[] { + const wamrVersion = getWAMRExtensionVersion(context.extensionPath); + const wamrReleaseUrl = `https://github.com/bytecodealliance/wasm-micro-runtime/releases/download/WAMR`; + + return [ + `${wamrReleaseUrl}-${wamrVersion}/wasm-debug-server-${wamrVersion}.zip`, + `${wamrReleaseUrl}-${wamrVersion}/wasm-toolchain-${wamrVersion}.zip`, + ]; +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/getUri.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/getUri.ts new file mode 100644 index 0000000..93a7eef --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/getUri.ts @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import { Uri, Webview } from 'vscode'; + +export function getUri( + webview: Webview, + extensionUri: Uri, + pathList: string[] +): Uri { + return webview.asWebviewUri(Uri.joinPath(extensionUri, ...pathList)); +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/lldbUtilities.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/lldbUtilities.ts new file mode 100644 index 0000000..9544766 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/utilities/lldbUtilities.ts @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as vscode from 'vscode'; +import * as os from 'os'; +import * as path from 'path'; +import * as fs from 'fs'; +import { + checkIfFileExists, + downloadFile, + unzipFile, +} from './directoryUtilities'; +import { SelectionOfPrompt } from '../constants'; + +const LLDB_RESOURCE_DIR = 'resource/debug'; +const LLDB_OS_DOWNLOAD_URL_SUFFIX_MAP: Partial< + Record +> = { + linux: 'x86_64-ubuntu-20.04', + darwin: 'universal-macos-latest', +}; + +const WAMR_LLDB_NOT_SUPPORTED_ERROR = new Error( + 'WAMR LLDB is not supported on this platform' +); + +function getLLDBUnzipFilePath(destinationFolder: string, filename: string) { + const dirs = filename.split('/'); + if (dirs[0] === 'wamr-lldb') { + dirs.shift(); + } + + return path.join(destinationFolder, ...dirs); +} + +export function getWAMRExtensionVersion(extensionPath: string): string { + // eslint-disable-next-line @typescript-eslint/no-var-requires + return require(path.join(extensionPath, 'package.json')).version; +} + +function getLLDBDownloadUrl(extensionPath: string): string { + const wamrVersion = getWAMRExtensionVersion(extensionPath); + const lldbOsUrlSuffix = LLDB_OS_DOWNLOAD_URL_SUFFIX_MAP[os.platform()]; + + if (!lldbOsUrlSuffix) { + throw WAMR_LLDB_NOT_SUPPORTED_ERROR; + } + + return `https://github.com/bytecodealliance/wasm-micro-runtime/releases/download/WAMR-${wamrVersion}/wamr-lldb-${wamrVersion}-${lldbOsUrlSuffix}.zip`; +} + +export function isLLDBInstalled(extensionPath: string): boolean { + const lldbOSDir = os.platform(); + const lldbBinaryPath = path.join( + extensionPath, + LLDB_RESOURCE_DIR, + lldbOSDir, + 'bin', + 'lldb' + ); + return checkIfFileExists(lldbBinaryPath); +} + +export async function promptInstallLLDB( + extensionPath: string +): Promise { + const response = await vscode.window.showWarningMessage( + 'No LLDB instance found. Setup now?', + SelectionOfPrompt.setUp, + SelectionOfPrompt.skip + ); + + if (response === SelectionOfPrompt.skip) { + return response; + } + + await downloadLldb(extensionPath); + + return SelectionOfPrompt.setUp; +} + +export async function downloadLldb(extensionPath: string): Promise { + const downloadUrl = getLLDBDownloadUrl(extensionPath); + const destinationDir = os.platform(); + + if (!downloadUrl) { + throw WAMR_LLDB_NOT_SUPPORTED_ERROR; + } + + const lldbDestinationFolder = path.join( + extensionPath, + LLDB_RESOURCE_DIR, + destinationDir + ); + const lldbZipPath = path.join(lldbDestinationFolder, 'bundle.zip'); + + vscode.window.showInformationMessage(`Downloading LLDB...`); + + await downloadFile(downloadUrl, lldbZipPath); + + vscode.window.showInformationMessage( + `LLDB downloaded to ${lldbZipPath}. Installing...` + ); + + const lldbFiles = await unzipFile(lldbZipPath, filename => + getLLDBUnzipFilePath(lldbDestinationFolder, filename) + ); + // Allow execution of lldb + lldbFiles.forEach(file => fs.chmodSync(file, '0775')); + + vscode.window.showInformationMessage( + `LLDB installed at ${lldbDestinationFolder}` + ); + + // Remove the bundle.zip + fs.unlinkSync(lldbZipPath); +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/view/NewProjectPanel.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/view/NewProjectPanel.ts new file mode 100644 index 0000000..79671f6 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/view/NewProjectPanel.ts @@ -0,0 +1,260 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as vscode from 'vscode'; +import * as path from 'path'; +import * as fs from 'fs'; +import * as os from 'os'; +import { + createDirectory, + copyFiles, + checkFolderName, +} from '../utilities/directoryUtilities'; +import { getUri } from '../utilities/getUri'; + +export class NewProjectPanel { + public static userSetWorkSpace: string; + public static currentPanel: NewProjectPanel | undefined; + private readonly viewPanel: vscode.WebviewPanel; + private disposableArr: vscode.Disposable[] = []; + + private static readonly executionSuccess = 0; + private static readonly dirExistedError = -1; + private static readonly userInputError = -2; + private static readonly dirPathInvalidError = -3; + + constructor(extensionUri: vscode.Uri, panel: vscode.WebviewPanel) { + this.viewPanel = panel; + this.viewPanel.webview.html = this.getHtmlForWebview( + this.viewPanel.webview, + extensionUri, + 'resource/webview/page/newProject.html' + ); + this._setWebviewMessageListener(this.viewPanel.webview, extensionUri); + this.viewPanel.onDidDispose(this.dispose, null, this.disposableArr); + } + + public static render(context: vscode.ExtensionContext): void { + NewProjectPanel.userSetWorkSpace = vscode.workspace + .getConfiguration() + .get('WAMR-IDE.configWorkspace') as string; + + /* check if current panel is initialized */ + if (NewProjectPanel.currentPanel) { + NewProjectPanel.currentPanel.viewPanel.reveal( + vscode.ViewColumn.One + ); + } else { + const panel = vscode.window.createWebviewPanel( + 'newProject', + 'Create project', + vscode.ViewColumn.One, + { + enableScripts: true, + retainContextWhenHidden: true, + } + ); + + NewProjectPanel.currentPanel = new NewProjectPanel( + context.extensionUri, + panel + ); + } + } + + private createNewProject( + projName: string, + template: string, + extensionUri: vscode.Uri + ): number { + if (projName === '' || template === '') { + return NewProjectPanel.userInputError; + } + + if (!checkFolderName(projName)) { + return NewProjectPanel.dirPathInvalidError; + } + + const ROOT_PATH = path.join(NewProjectPanel.userSetWorkSpace, projName); + const EXT_PATH = extensionUri.fsPath; + + if (fs.existsSync(ROOT_PATH)) { + if (fs.lstatSync(ROOT_PATH).isDirectory()) { + return NewProjectPanel.dirExistedError; + } + } + + createDirectory(path.join(ROOT_PATH, '.wamr')); + createDirectory(path.join(ROOT_PATH, 'include')); + createDirectory(path.join(ROOT_PATH, 'src')); + + copyFiles( + path.join(EXT_PATH, 'resource/scripts/CMakeLists.txt'), + path.join(ROOT_PATH, '.wamr/CMakeLists.txt') + ); + + copyFiles( + path.join(EXT_PATH, 'resource/scripts/project.cmake'), + path.join(ROOT_PATH, '.wamr/project.cmake') + ); + + return NewProjectPanel.executionSuccess; + } + + public getHtmlForWebview( + webview: vscode.Webview, + extensionUri: vscode.Uri, + templatePath: string + ): string { + const toolkitUri = getUri(webview, extensionUri, [ + 'node_modules', + '@vscode', + 'webview-ui-toolkit', + 'dist', + 'toolkit.js', + ]); + + const styleUri = getUri(webview, extensionUri, [ + 'resource', + 'webview', + 'css', + 'style.css', + ]); + + const mainUri = getUri(webview, extensionUri, [ + 'resource', + 'webview', + 'js', + 'newproj.js', + ]); + + const resourcePath = path.join(extensionUri.fsPath, templatePath); + let html = fs.readFileSync(resourcePath, 'utf-8'); + html = html + .replace(/(\${toolkitUri})/, toolkitUri.toString()) + .replace(/(\${mainUri})/, mainUri.toString()) + .replace(/(\${styleUri})/, styleUri.toString()); + + return html; + } + + private _setWebviewMessageListener( + webview: vscode.Webview, + extensionUri: vscode.Uri + ) { + webview.onDidReceiveMessage( + message => { + switch (message.command) { + case 'create_new_project': + const createNewProjectStatus = this.createNewProject( + message.projectName, + message.template, + extensionUri + ); + if ( + createNewProjectStatus === + NewProjectPanel.executionSuccess + ) { + webview.postMessage({ + command: 'proj_creation_finish', + prjName: message.projectName, + }); + } else if ( + createNewProjectStatus === + NewProjectPanel.dirExistedError + ) { + vscode.window.showErrorMessage( + 'Project : ' + + message.projectName + + ' exists in your current root path, please change project name or root path!' + ); + return; + } else if ( + createNewProjectStatus === + NewProjectPanel.userInputError + ) { + vscode.window.showErrorMessage( + 'Please fill chart before your submit!' + ); + return; + } else if ( + createNewProjectStatus === + NewProjectPanel.dirPathInvalidError + ) { + if (os.platform() === 'win32') { + vscode.window.showErrorMessage( + "A file name can't contain any of the following characters: ' / \\ : * ? < > | ' and the length should be less than 255" + ); + } else if ( + os.platform() === 'linux' || + os.platform() === 'darwin' + ) { + vscode.window.showErrorMessage( + "A file name can't contain following characters: '/' and the length should be less than 255" + ); + } + return; + } + return; + + case 'open_project': + vscode.window.showInformationMessage( + 'Project : ' + + message.projectName + + ' will be opened!' + ); + + const projPath = path.join( + NewProjectPanel.userSetWorkSpace, + message.projectName + ); + const uri = vscode.Uri.file(projPath); + + /** + * check if the vscode workspace folder is empty, + * if yes, open new window, else open in current window + */ + const isWorkspaceEmpty = !vscode.workspace + .workspaceFolders?.[0] + ? true + : false; + isWorkspaceEmpty === false + ? vscode.commands.executeCommand( + 'vscode.openFolder', + uri, + { + forceNewWindow: true, + } + ) + : vscode.commands.executeCommand( + 'vscode.openFolder', + uri + ); + + case 'close_webview': + this.viewPanel.dispose(); + return; + + default: + break; + } + }, + undefined, + this.disposableArr + ); + } + + private dispose() { + NewProjectPanel.currentPanel = undefined; + this.viewPanel.dispose(); + + while (this.disposableArr.length) { + const disposable = this.disposableArr.pop(); + if (disposable) { + disposable.dispose(); + } + } + } +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/view/TargetConfigPanel.ts b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/view/TargetConfigPanel.ts new file mode 100644 index 0000000..8efa245 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/src/view/TargetConfigPanel.ts @@ -0,0 +1,248 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +import * as vscode from 'vscode'; +import * as path from 'path'; +import * as fs from 'fs'; +import { readFromConfigFile, writeIntoConfigFile } from '../extension'; +import { getUri } from '../utilities/getUri'; + +export class TargetConfigPanel { + public static currentPanel: TargetConfigPanel | undefined; + private readonly viewPanel: vscode.WebviewPanel; + + private _disposables: vscode.Disposable[] = []; + public static buildArgs = { + outputFileName: 'main.wasm', + initMemorySize: '131072', + maxMemorySize: '131072', + stackSize: '4096', + exportedSymbols: 'main', + hostManagedHeapSize: '4096', + }; + + private static readonly userInputError: number = -2; + private static readonly executionSuccess: number = 0; + + /** + * + * @param context extension context from extension.ts active func + * @param panelName + */ + constructor(panel: vscode.WebviewPanel, extensionUri: vscode.Uri) { + this.viewPanel = panel; + this.viewPanel.webview.html = this._getHtmlForWebview( + this.viewPanel.webview, + extensionUri, + 'resource/webview/page/configBuildTarget.html' + ); + this.viewPanel.onDidDispose(this.dispose, null, this._disposables); + this._setWebviewMessageListener(this.viewPanel.webview); + } + + /** + * + * @param context + */ + public static render(context: vscode.ExtensionContext): void { + /* check if current panel is initialized */ + if (TargetConfigPanel.currentPanel) { + TargetConfigPanel.currentPanel.viewPanel.reveal( + vscode.ViewColumn.One + ); + } else { + const panel = vscode.window.createWebviewPanel( + 'targetConfig', + 'Config building target', + vscode.ViewColumn.One, + { + enableScripts: true, + retainContextWhenHidden: true, + } + ); + + TargetConfigPanel.currentPanel = new TargetConfigPanel( + panel, + context.extensionUri + ); + } + } + + private configBuildArgs( + outputFileName: string, + initMemSize: string, + maxMemSize: string, + stackSize: string, + exportedSymbols: string, + hostManagedHeapSize: string + ): number { + if ( + outputFileName === '' || + initMemSize === '' || + maxMemSize === '' || + stackSize === '' || + exportedSymbols === '' || + hostManagedHeapSize === '' + ) { + return TargetConfigPanel.userInputError; + } + + let includePathArr = []; + let excludeFileArr = []; + + const configObj = { + outputFileName: outputFileName, + initMemorySize: initMemSize, + maxMemorySize: maxMemSize, + stackSize: stackSize, + exportedSymbols: exportedSymbols, + hostManagedHeapSize: hostManagedHeapSize, + }; + const configStr = readFromConfigFile(); + + TargetConfigPanel.buildArgs = configObj; + + if (configStr !== '' && configStr !== undefined) { + const configJson = JSON.parse(configStr); + includePathArr = + configJson['includePaths'] === undefined + ? [] + : configJson['includePaths']; + excludeFileArr = + configJson['excludeFiles'] === undefined + ? [] + : configJson['excludeFiles']; + } + + writeIntoConfigFile( + includePathArr, + excludeFileArr, + TargetConfigPanel.buildArgs + ); + + return TargetConfigPanel.executionSuccess; + } + + private _getHtmlForWebview( + webview: vscode.Webview, + extensionUri: vscode.Uri, + templatePath: string + ) { + /* get toolkit uri */ + const toolkitUri = getUri(webview, extensionUri, [ + 'node_modules', + '@vscode', + 'webview-ui-toolkit', + 'dist', + 'toolkit.js', + ]); + + const styleUri = getUri(webview, extensionUri, [ + 'resource', + 'webview', + 'css', + 'style.css', + ]); + + const mainUri = getUri(webview, extensionUri, [ + 'resource', + 'webview', + 'js', + 'configbuildtarget.js', + ]); + + const resourcePath = path.join(extensionUri.fsPath, templatePath); + let html = fs.readFileSync(resourcePath, 'utf-8'); + html = html + .replace(/(\${toolkitUri})/, toolkitUri.toString()) + .replace(/(\${mainUri})/, mainUri.toString()) + .replace(/(\${styleUri})/, styleUri.toString()) + .replace( + /(\${output_file_val})/, + TargetConfigPanel.buildArgs.outputFileName + ) + .replace( + /(\${initial_mem_size_val})/, + TargetConfigPanel.buildArgs.initMemorySize + ) + .replace( + /(\${max_mem_size_val})/, + TargetConfigPanel.buildArgs.maxMemorySize + ) + .replace( + /(\${stack_size_val})/, + TargetConfigPanel.buildArgs.stackSize + ) + .replace( + /(\${exported_symbols_val})/, + TargetConfigPanel.buildArgs.exportedSymbols + ) + .replace( + /(\${host_managed_heap_size_val})/, + TargetConfigPanel.buildArgs.hostManagedHeapSize + ); + + return html; + } + + private _setWebviewMessageListener(webview: vscode.Webview) { + webview.onDidReceiveMessage( + message => { + switch (message.command) { + case 'config_build_target': + if ( + message.outputFileName === '' || + message.initMemSize === '' || + message.maxMemSize === '' || + message.stackSize === '' || + message.exportedSymbols === '' || + message.hostManagedHeapSize === '' + ) { + vscode.window.showErrorMessage( + 'Please fill chart before your submit!' + ); + return; + } else if ( + this.configBuildArgs( + message.outputFileName, + message.initMemSize, + message.maxMemSize, + message.stackSize, + message.exportedSymbols, + message.hostManagedHeapSize + ) === TargetConfigPanel.executionSuccess + ) { + vscode.window + .showInformationMessage( + 'Configurations have been saved!', + 'OK' + ) + .then(() => { + this.viewPanel.dispose(); + return; + }); + } + + default: + break; + } + }, + undefined, + this._disposables + ); + } + + private dispose() { + TargetConfigPanel.currentPanel = undefined; + this.viewPanel.dispose(); + + while (this._disposables.length) { + const disposable = this._disposables.pop(); + if (disposable) { + disposable.dispose(); + } + } + } +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/tsconfig.json b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/tsconfig.json new file mode 100644 index 0000000..c75039e --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/VSCode-Extension/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es6", + "outDir": "out", + "lib": ["es6"], + "sourceMap": true, + "rootDir": "src", + "strict": true /* enable all strict type-checking options */ + /* Additional Checks */ + // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ + // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + // "noUnusedParameters": true, /* Report errors on unused parameters. */ + }, + "exclude": ["node_modules", ".vscode-test"] +} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/Dockerfile b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/Dockerfile new file mode 100644 index 0000000..8165bd5 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/Dockerfile @@ -0,0 +1,30 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +FROM gcc:12.2.0 AS BASE + +## set work directory +WORKDIR /root/ +COPY resource /root/ + +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get -y install make cmake --no-install-recommends + +## -clone wamr-repo and build iwasm +RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git \ + && mkdir -p /root/wasm-micro-runtime/product-mini/platforms/linux/build + +WORKDIR /root/wasm-micro-runtime/product-mini/platforms/linux/build +RUN cmake .. -DWAMR_BUILD_DEBUG_INTERP=1 \ + && make \ + && cp /root/wasm-micro-runtime/product-mini/platforms/linux/build/iwasm /root/iwasm \ + && rm -fr /root/wasm-micro-runtime + +FROM ubuntu:22.04 +# COPY files from BASE image +COPY --from=BASE /root/iwasm /root +COPY --from=BASE /root/debug.sh /root +COPY --from=BASE /root/run.sh /root + +WORKDIR /root/ \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/README.md b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/README.md new file mode 100644 index 0000000..b4738e8 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/README.md @@ -0,0 +1,20 @@ +### Build Docker Image + +- Linux + + ```shell + ./build_docker_image.sh + ``` + +- Windows + + ```shell + ./build_docker_image.bat + ``` + + +### Resource Details + +- `Dockerflie` is the source file to build `wasm-debug-server` docker image +- `resource/debug.sh` is the script to execute the wasm app in debug mod, will start up the debugger server inside of the `iwasm` and hold to wait for connecting. +- `resource/run.sh` is the script to execute the wasm app directly. diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/build_docker_image.bat b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/build_docker_image.bat new file mode 100644 index 0000000..2861105 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/build_docker_image.bat @@ -0,0 +1,8 @@ +@REM Copyright (C) 2019 Intel Corporation. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +@echo off +docker build -t wasm-debug-server:1.0 . + +@REM delete intermediate docker image +docker image prune -f \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/build_docker_image.sh b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/build_docker_image.sh new file mode 100755 index 0000000..6beedd6 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/build_docker_image.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +docker build -t wasm-debug-server:1.0 . + +# delete intermediate docker image +docker image prune -f diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/resource/debug.sh b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/resource/debug.sh new file mode 100755 index 0000000..33cdb58 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/resource/debug.sh @@ -0,0 +1,7 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#!/bin/bash +TARGET=$1 +HEAP_SIZE=$2 +./iwasm -g=0.0.0.0:1234 --heap-size=${HEAP_SIZE} /mnt/build/${TARGET}.wasm diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/resource/run.sh b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/resource/run.sh new file mode 100755 index 0000000..f652cfc --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Debug-Server/Docker/resource/run.sh @@ -0,0 +1,7 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#!/bin/bash +TARGET=$1 +HEAP_SIZE=$2 +./iwasm --heap-size=${HEAP_SIZE} /mnt/build/${TARGET}.wasm diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/.dockerignore b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/.dockerignore new file mode 100644 index 0000000..920e099 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/.dockerignore @@ -0,0 +1,4 @@ +# remove unnecessary files here to save build time cost and image size +*.md +*.sh +*.bat \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile new file mode 100644 index 0000000..cd8da38 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile @@ -0,0 +1,71 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +## Build docker image that consists of gcc, cmake, wasi-sdk & zephyr sdk +FROM gcc:12.2.0 AS BASE + +## set work directory +WORKDIR /root/ + +COPY resource /root/ + +# - download cmake with wget and set up +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get -y install ccache ninja-build make cmake python3-pip --no-install-recommends + +# set compilation environment for wamrc +# - wamr repo +# - cmake +# - wasi-sdk +# - wamr-sdk + +# - download wasi-sdk with wget and set up to /opt/wasi-sdk +RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz \ + && tar -zxf wasi-sdk-*-linux.tar.gz \ + && mv wasi-sdk-19.0 /opt/wasi-sdk/ \ + && rm -f wasi-sdk-*-linux.tar.gz + +## - clone wamr repo +RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git + +WORKDIR /root/wasm-micro-runtime/build-scripts +RUN pip3 install --no-cache-dir --user -r requirements.txt + +WORKDIR /root/wasm-micro-runtime/wamr-compiler +RUN ./build_llvm.sh \ + && mkdir build + +WORKDIR /root/wasm-micro-runtime/wamr-compiler/build +RUN cmake .. \ + && make \ + # - copy the wamrc to /root + && cp /root/wasm-micro-runtime/wamr-compiler/build/wamrc /root/wamrc \ + && mkdir -p /opt/wamr-sdk \ + && cp -r /root/wasm-micro-runtime/wamr-sdk/app /opt/wamr-sdk/ \ + && mv /root/wamr_toolchain.cmake /opt/wamr-sdk/app \ + # - remove the wamr repo to save the size + && rm -fr /root/wasm-micro-runtime + +# ## STAGE 2 +FROM ubuntu:22.04 +ENV HOME_DIR=/home/wasm-toolchain + +RUN mkdir -p /opt/wasi-sdk \ + && mkdir -p /opt/wamr-sdk/app \ + && mkdir -p /home/wasm-toolchain + +# COPY files from BASE image +COPY --from=BASE /opt/wamr-sdk/app/ /opt/wamr-sdk/app/ +COPY --from=BASE /opt/wasi-sdk /opt/wasi-sdk/ +COPY --from=BASE /root/wamrc ${HOME_DIR} +COPY --from=BASE /root/build_wasm.sh ${HOME_DIR} + +RUN ln -s ${HOME_DIR}/wamrc /usr/bin/wamrc + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y cmake make --no-install-recommends \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR ${HOME_DIR} diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/README.md b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/README.md new file mode 100644 index 0000000..f5ecb95 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/README.md @@ -0,0 +1,18 @@ +### Build Docker Image + +- Linux + + ```shell + ./build_docker_image.sh + ``` + +- Windows + + ```shell + ./build_docker_image.bat + ``` + +### Resource Details + +- `Dockerflie` is the source file to build `wasm-debug-server` docker image. +- `resource/build_wasm.sh` is the script to compile the wasm app with `wasi-sdk`. diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/build_docker_image.bat b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/build_docker_image.bat new file mode 100644 index 0000000..96fb028 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/build_docker_image.bat @@ -0,0 +1,9 @@ +@REM Copyright (C) 2019 Intel Corporation. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +@echo off + +docker build -t wasm-toolchain:1.0 . + +@REM delete intermediate docker image +docker image prune -f diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/build_docker_image.sh b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/build_docker_image.sh new file mode 100755 index 0000000..aa1e2f5 --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/build_docker_image.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +docker build -t wasm-toolchain:1.0 . + +# delete intermediate docker image +docker image prune -f diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/resource/build_wasm.sh b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/resource/build_wasm.sh new file mode 100755 index 0000000..3d8c06c --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/resource/build_wasm.sh @@ -0,0 +1,25 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#!/bin/bash +export CC=/opt/wasi-sdk/bin/clang +export CXX=/opt/wasi-sdk/bin/clang++ + +cd /mnt +if [ -d build ];then + rm -fr build +fi + +mkdir -p build && cd build +echo "========> compile wasm with wasi-sdk" +cmake -DWASI_SDK_DIR=/opt/wasi-sdk -DCMAKE_TOOLCHAIN_FILE=/opt/wamr-sdk/app/wamr_toolchain.cmake ../.wamr && make + +if [ $? -eq 0 ]; then + echo "========> compile wasm to AoT with wamrc" + # target name will be provided: + # - user input the target name in IDE + # - generated wasm binary name will be set as user's input target name + # - aot binary name should be the same as wasm binary name + # - target name will be provided through 1st parameter + wamrc -o $1.aot $1.wasm +fi \ No newline at end of file diff --git a/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/resource/wamr_toolchain.cmake b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/resource/wamr_toolchain.cmake new file mode 100755 index 0000000..97b82cc --- /dev/null +++ b/wasm-micro-runtime/test-tools/wamr-ide/WASM-Toolchain/Docker/resource/wamr_toolchain.cmake @@ -0,0 +1,33 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +SET (CMAKE_SYSTEM_NAME Linux) +SET (CMAKE_SYSTEM_PROCESSOR wasm32) +SET (CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR}/libc-builtin-sysroot) + +IF (NOT (DEFINED WASI_SDK_DIR OR DEFINED CACHE{WASI_SDK_DIR})) + SET (WASI_SDK_DIR "/opt/wasi-sdk") +ELSE () + MESSAGE (STATUS "WASI_SDK_DIR=${WASI_SDK_DIR}") + LIST (APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES "WASI_SDK_DIR") +ENDIF () + + +SET (CMAKE_C_FLAGS "-nostdlib" CACHE INTERNAL "") +SET (CMAKE_C_COMPILER_TARGET "wasm32") +SET (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +SET (CMAKE_CXX_FLAGS "-nostdlib" CACHE INTERNAL "") +SET (CMAKE_CXX_COMPILER_TARGET "wasm32") +SET (CMAKE_CXX_COMPILER "${WASI_SDK_DIR}/bin/clang++") + +SET (CMAKE_EXE_LINKER_FLAGS + "-Wl,--no-entry,--fatal-warnings" CACHE INTERNAL "") + +SET (CMAKE_LINKER "${WASI_SDK_DIR}/bin/wasm-ld" CACHE INTERNAL "") +SET (CMAKE_AR "${WASI_SDK_DIR}/bin/llvm-ar" CACHE INTERNAL "") +SET (CMAKE_NM "${WASI_SDK_DIR}/bin/llvm-nm" CACHE INTERNAL "") +SET (CMAKE_OBJDUMP "${WASI_SDK_DIR}/bin/llvm-dwarfdump" CACHE INTERNAL "") +SET (CMAKE_RANLIB "${WASI_SDK_DIR}/bin/llvm-ranlib" CACHE INTERNAL "") +SET (CMAKE_EXE_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS},--allow-undefined-file=${CMAKE_SYSROOT}/share/defined-symbols.txt" CACHE INTERNAL "") \ No newline at end of file diff --git a/wasm-micro-runtime/tests/benchmarks/README.md b/wasm-micro-runtime/tests/benchmarks/README.md new file mode 100644 index 0000000..2112829 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/README.md @@ -0,0 +1,62 @@ +# WAMR test benchmarks + +This folder contains test benchmarks for wamr. + +## Build and Run + +Refer to the `README.md` under each folder for how to build and run the benchmark. + +## Install `llvm-profdata` + +The tool `llvm-profdata` is used when running the `test_pgo.sh` script under the benchmark folder. There are two ways to install it: + +1. Refer to https://apt.llvm.org/, e.g. in Ubuntu 20.04, add lines below to /etc/apt/source.list + +```bash +deb http://apt.llvm.org/focal/ llvm-toolchain-focal main +deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main +# 15 +deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main +deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main +``` + +Then run `sudo apt update`, `sudo apt install llvm`. And after installing: + +```bash +cd /usr/bin +sudo ln -s llvm-profdata-15 llvm-profdata +``` + +2. Build manually + +```bash +git clone --depth 1 --branch release/15.x https://github.com/llvm/llvm-project.git +cd llvm-project +mkdir build && cd build +cmake ../llvm \ + -DCMAKE_BUILD_TYPE:STRING="Release" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DLLVM_APPEND_VC_REV:BOOL=ON \ + -DLLVM_BUILD_EXAMPLES:BOOL=OFF \ + -DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF \ + -DLLVM_BUILD_TESTS:BOOL=OFF \ + -DLLVM_CCACHE_BUILD:BOOL=ON \ + -DLLVM_ENABLE_BINDINGS:BOOL=OFF \ + -DLLVM_ENABLE_IDE:BOOL=OFF \ + -DLLVM_ENABLE_LIBEDIT=OFF \ + -DLLVM_ENABLE_TERMINFO:BOOL=OFF \ + -DLLVM_ENABLE_ZLIB:BOOL=ON \ + -DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF \ + -DLLVM_INCLUDE_DOCS:BOOL=OFF \ + -DLLVM_INCLUDE_EXAMPLES:BOOL=OFF \ + -DLLVM_INCLUDE_UTILS:BOOL=OFF \ + -DLLVM_INCLUDE_TESTS:BOOL=OFF \ + -DLLVM_BUILD_TESTS:BOOL=OFF \ + -DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON \ + -DLLVM_ENABLE_LIBXML2:BOOL=OFF \ + -DLLVM_TARGETS_TO_BUILD:STRING="X86" \ + -DLLVM_INCLUDE_TOOLS:BOOL=ON \ + -G'Ninja' +ninja -j 8 +# tool `llvm-profdata` is generated under this folder. +``` diff --git a/wasm-micro-runtime/tests/benchmarks/coremark/README.md b/wasm-micro-runtime/tests/benchmarks/coremark/README.md new file mode 100644 index 0000000..5b874e9 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/coremark/README.md @@ -0,0 +1,25 @@ +# Introduction + +[CoreMark's](https://www.eembc.org/coremark) primary goals are simplicity and providing a method for testing only a processor's core features. + +**Source**: https://github.com/eembc/coremark + +# Building + +Please build iwasm and wamrc, refer to: +- [Build iwasm on Linux](../../../doc/build_wamr.md#linux), or [Build iwasm on MacOS](../../../doc/build_wamr.md#macos) +- [Build wamrc AOT compiler](../../../README.md#build-wamrc-aot-compiler) + +And install WASI SDK, please download the [wasi-sdk release](https://github.com/WebAssembly/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. + +And then run `./build.sh` to build the source code, file `coremark.exe`, `coremark.wasm` and `coremark.aot` will be generated. + +# Running + +Run `./run.sh` to test the benchmark, the native mode, iwasm aot mode and iwasm interpreter mode will be tested respectively. + +Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`. + +- For Linux, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled. + +- For Linux-sgx, similarly, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then `make` in the directory `enclave-sample`. And run `./test_pgo.sh --sgx` to test the benchmark. diff --git a/wasm-micro-runtime/tests/benchmarks/coremark/build.sh b/wasm-micro-runtime/tests/benchmarks/coremark/build.sh new file mode 100755 index 0000000..ecada10 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/coremark/build.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +WAMRC="../../../wamr-compiler/build/wamrc" + +if [ ! -d coremark ]; then + git clone https://github.com/eembc/coremark.git +fi + +cd coremark + +echo "Build coremark with gcc .." +gcc -O3 -Iposix -I. -DFLAGS_STR=\""-O3 -DPERFORMANCE_RUN=1 -lrt"\" \ + -DITERATIONS=400000 -DSEED_METHOD=SEED_VOLATILE -DPERFORMANCE_RUN=1 \ + core_list_join.c core_main.c core_matrix.c core_state.c \ + core_util.c posix/core_portme.c \ + -o ../coremark.exe -lrt + +echo "Build coremark with wasi-sdk .." +/opt/wasi-sdk/bin/clang -O3 -Iposix -I. -DFLAGS_STR=\""-O3 -DPERFORMANCE_RUN=1"\" \ + -Wl,--export=main \ + -DITERATIONS=400000 -DSEED_METHOD=SEED_VOLATILE -DPERFORMANCE_RUN=1 \ + -Wl,--allow-undefined \ + core_list_join.c core_main.c core_matrix.c core_state.c \ + core_util.c posix/core_portme.c \ + -o ../coremark.wasm + +cd .. + +echo "Compile coremark.wasm to coremark.aot .." +${WAMRC} -o coremark.aot coremark.wasm + +if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile coremark.wasm to coremark_segue.aot .." + ${WAMRC} --enable-segue -o coremark_segue.aot coremark.wasm +fi + +echo "Done" diff --git a/wasm-micro-runtime/tests/benchmarks/coremark/run.sh b/wasm-micro-runtime/tests/benchmarks/coremark/run.sh new file mode 100755 index 0000000..0d308bb --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/coremark/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +IWASM="../../../product-mini/platforms/${PLATFORM}/build/iwasm" +WAMRC="../../../wamr-compiler/build/wamrc" + +echo "Run coremark with native .." +./coremark.exe + +echo "Run coremark with iwasm aot mode .." +${IWASM} coremark.aot + +if [[ ${PLATFORM} == "linux" ]]; then + echo "Run coremark with iwasm aot-segue mode .." + ${IWASM} coremark_segue.aot +fi + +echo "Run coremark with iwasm interpreter mode .." +${IWASM} coremark.wasm diff --git a/wasm-micro-runtime/tests/benchmarks/coremark/test_pgo.sh b/wasm-micro-runtime/tests/benchmarks/coremark/test_pgo.sh new file mode 100755 index 0000000..1c63131 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/coremark/test_pgo.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +if [ "$1" = "--sgx" ] && [ "$PLATFORM" = "linux" ]; then + IWASM="../../../product-mini/platforms/${PLATFORM}-sgx/enclave-sample/iwasm" + WAMRC="../../../wamr-compiler/build/wamrc -sgx" +else + IWASM="../../../product-mini/platforms/${PLATFORM}/build/iwasm" + WAMRC="../../../wamr-compiler/build/wamrc" +fi + +if [ ! -e "coremark.wasm" ]; then + echo "coremark.wasm doesn't exist, please run build.sh first" + exit +fi + +echo "" +echo "Compile coremark.wasm to coremark.aot .." +${WAMRC} -o coremark.aot coremark.wasm + +echo "" +echo "Compile coremark.wasm to coremark_pgo.aot .." +${WAMRC} --enable-llvm-pgo -o coremark_pgo.aot coremark.wasm + +echo "" +echo "Run coremark_pgo.aot to generate the raw profile data .." +${IWASM} --gen-prof-file=coremark.profraw coremark_pgo.aot + +echo "" +echo "Merge the raw profile data to coremark.profdata .." +rm -f coremark.profdata && llvm-profdata merge -output=coremark.profdata coremark.profraw + +echo "" +echo "Compile coremark.wasm to coremark_opt.aot with the profile data .." +${WAMRC} --use-prof-file=coremark.profdata -o coremark_opt.aot coremark.wasm + +echo "" +echo "Run the coremark native" +./coremark.exe + +echo "" +echo "Run the original aot file coremark.aot" +${IWASM} coremark.aot + +echo "" +echo "Run the PGO optimized aot file coremark_opt.aot" +${IWASM} coremark_opt.aot + +# Show the profile data: +# llvm-profdata show --all-functions --detailed-summary --binary-ids --counts \ +# --hot-func-list --memop-sizes --show-prof-sym-list coremark.profraw diff --git a/wasm-micro-runtime/tests/benchmarks/dhrystone/LICENSE b/wasm-micro-runtime/tests/benchmarks/dhrystone/LICENSE new file mode 100644 index 0000000..9b3a7b2 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/dhrystone/LICENSE @@ -0,0 +1,7 @@ +Dhrystone +------------------------------------------------------------------------------ +There is no explicit license defined. They were originally +written in ADA by Reinhold P. Weicker and translated to C by Rick Richardson . + +The source obtained from the following site: +https://fossies.org/linux/privat/old/dhrystone-2.1.tar.gz diff --git a/wasm-micro-runtime/tests/benchmarks/dhrystone/build.sh b/wasm-micro-runtime/tests/benchmarks/dhrystone/build.sh new file mode 100755 index 0000000..eea33d5 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/dhrystone/build.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc + +echo "===> compile dhrystone src to dhrystone_native" +gcc -O3 -o dhrystone_native src/dhry_1.c src/dhry_2.c -I include + +echo "===> compile dhrystone src to dhrystone.wasm" +/opt/wasi-sdk/bin/clang -O3 \ + -o dhrystone.wasm src/dhry_1.c src/dhry_2.c -I include \ + -Wl,--export=__heap_base -Wl,--export=__data_end + +echo "===> compile dhrystone.wasm to dhrystone.aot" +${WAMRC_CMD} -o dhrystone.aot dhrystone.wasm + +if [[ ${PLATFORM} == "linux" ]]; then + echo "===> compile dhrystone.wasm to dhrystone_segue.aot" + ${WAMRC_CMD} --enable-segue -o dhrystone_segue.aot dhrystone.wasm +fi diff --git a/wasm-micro-runtime/tests/benchmarks/dhrystone/include/dhry.h b/wasm-micro-runtime/tests/benchmarks/dhrystone/include/dhry.h new file mode 100644 index 0000000..0eb5ec6 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/dhrystone/include/dhry.h @@ -0,0 +1,306 @@ +/* + ************************************************************************** + * DHRYSTONE 2.1 BENCHMARK PC VERSION + ************************************************************************** + * + * "DHRYSTONE" Benchmark Program + * ----------------------------- + * + * Version: C, Version 2.1 + * + * File: dhry.h (part 1 of 3) + * + * Date: May 25, 1988 + * + * Author: Reinhold P. Weicker + * Siemens AG, AUT E 51 + * Postfach 3220 + * 8520 Erlangen + * Germany (West) + * Phone: [+49]-9131-7-20330 + * (8-17 Central European Time) + * Usenet: ..!mcsun!unido!estevax!weicker + * + * Original Version (in Ada) published in + * "Communications of the ACM" vol. 27., no. 10 (Oct. 1984), + * pp. 1013 - 1030, together with the statistics + * on which the distribution of statements etc. is based. + * + * In this C version, the following C library functions are used: + * - strcpy, strcmp (inside the measurement loop) + * - printf, scanf (outside the measurement loop) + * In addition, Berkeley UNIX system calls "times ()" or "time ()" + * are used for execution time measurement. For measurements + * on other systems, these calls have to be changed. + * + * Collection of Results: + * Reinhold Weicker (address see above) and + * + * Rick Richardson + * PC Research. Inc. + * 94 Apple Orchard Drive + * Tinton Falls, NJ 07724 + * Phone: (201) 389-8963 (9-17 EST) + * Usenet: ...!uunet!pcrat!rick + * + * Please send results to Rick Richardson and/or Reinhold Weicker. + * Complete information should be given on hardware and software used. + * Hardware information includes: Machine type, CPU, type and size + * of caches; for microprocessors: clock frequency, memory speed + * (number of wait states). + * Software information includes: Compiler (and runtime library) + * manufacturer and version, compilation switches, OS version. + * The Operating System version may give an indication about the + * compiler; Dhrystone itself performs no OS calls in the measurement + * loop. + * + * The complete output generated by the program should be mailed + * such that at least some checks for correctness can be made. + * + ************************************************************************** + * + * This version has changes made by Roy Longbottom to conform to a common + * format for a series of standard benchmarks for PCs: + * + * Running time greater than 5 seconds due to inaccuracy of the PC clock. + * + * Automatic adjustment of run time, no manually inserted parameters. + * + * Initial display of calibration times to confirm linearity. + * + * Display of results within one screen (or at a slow speed as the test + * progresses) so that it can be seen to have run successfully. + * + * Facilities to type in details of system used etc. + * + * All results and details appended to a results file. + * + * + * Roy Longbottom + * 101323.2241@compuserve.com + * + ************************************************************************** + * + * For details of history, changes, other defines, benchmark construction + * statistics see official versions from ftp.nosc.mil/pub/aburto where + * the latest table of results (dhry.tbl) are available. See also + * netlib@ornl.gov + * + ************************************************************************** + * + * Defines: The following "Defines" are possible: + * -DREG=register (default: Not defined) + * As an approximation to what an average C programmer + * might do, the "register" storage class is applied + * (if enabled by -DREG=register) + * - for local variables, if they are used (dynamically) + * five or more times + * - for parameters if they are used (dynamically) + * six or more times + * Note that an optimal "register" strategy is + * compiler-dependent, and that "register" declarations + * do not necessarily lead to faster execution. + * -DNOSTRUCTASSIGN (default: Not defined) + * Define if the C compiler does not support + * assignment of structures. + * -DNOENUMS (default: Not defined) + * Define if the C compiler does not support + * enumeration types. + *************************************************************************** + * + * Compilation model and measurement (IMPORTANT): + * + * This C version of Dhrystone consists of three files: + * - dhry.h (this file, containing global definitions and comments) + * - dhry_1.c (containing the code corresponding to Ada package Pack_1) + * - dhry_2.c (containing the code corresponding to Ada package Pack_2) + * + * The following "ground rules" apply for measurements: + * - Separate compilation + * - No procedure merging + * - Otherwise, compiler optimizations are allowed but should be indicated + * - Default results are those without register declarations + * See the companion paper "Rationale for Dhrystone Version 2" for a more + * detailed discussion of these ground rules. + * + * For 16-Bit processors (e.g. 80186, 80286), times for all compilation + * models ("small", "medium", "large" etc.) should be given if possible, + * together with a definition of these models for the compiler system used. + * + ************************************************************************** + * Examples of Pentium Results + * + * Dhrystone Benchmark Version 2.1 (Language: C) + * + * Month run 4/1996 + * PC model Escom + * CPU Pentium + * Clock MHz 100 + * Cache 256K + * Options Neptune chipset + * OS/DOS Windows 95 + * Compiler Watcom C/ C++ 10.5 Win386 + * OptLevel -otexan -zp8 -fp5 -5r + * Run by Roy Longbottom + * From UK + * Mail 101323.2241@compuserve.com + * + * Final values (* implementation-dependent): + * + * Int_Glob: O.K. 5 + * Bool_Glob: O.K. 1 + * Ch_1_Glob: O.K. A + * Ch_2_Glob: O.K. B + * Arr_1_Glob[8]: O.K. 7 + * Arr_2_Glob8/7: O.K. 1600010 + * Ptr_Glob-> + * Ptr_Comp: * 98008 + * Discr: O.K. 0 + * Enum_Comp: O.K. 2 + * Int_Comp: O.K. 17 + * Str_Comp: O.K. DHRYSTONE PROGRAM, SOME STRING + * Next_Ptr_Glob-> + * Ptr_Comp: * 98008 same as above + * Discr: O.K. 0 + * Enum_Comp: O.K. 1 + * Int_Comp: O.K. 18 + * Str_Comp: O.K. DHRYSTONE PROGRAM, SOME STRING + * Int_1_Loc: O.K. 5 + * Int_2_Loc: O.K. 13 + * Int_3_Loc: O.K. 7 + * Enum_Loc: O.K. 1 + * Str_1_Loc: O.K. DHRYSTONE PROGRAM, 1'ST STRING + * Str_2_Loc: O.K. DHRYSTONE PROGRAM, 2'ND STRING + * + * Register option Selected. + * + * Microseconds 1 loop: 4.53 + * Dhrystones / second: 220690 + * VAX MIPS rating: 125.61 + * + * + * Dhrystone Benchmark Version 2.1 (Language: C) + * + * Month run 4/1996 + * PC model Escom + * CPU Pentium + * Clock MHz 100 + * Cache 256K + * Options Neptune chipset + * OS/DOS Windows 95 + * Compiler Watcom C/ C++ 10.5 Win386 + * OptLevel No optimisation + * Run by Roy Longbottom + * From UK + * Mail 101323.2241@compuserve.com + * + * Final values (* implementation-dependent): + * + * Int_Glob: O.K. 5 + * Bool_Glob: O.K. 1 + * Ch_1_Glob: O.K. A + * Ch_2_Glob: O.K. B + * Arr_1_Glob[8]: O.K. 7 + * Arr_2_Glob8/7: O.K. 320010 + * Ptr_Glob-> + * Ptr_Comp: * 98004 + * Discr: O.K. 0 + * Enum_Comp: O.K. 2 + * Int_Comp: O.K. 17 + * Str_Comp: O.K. DHRYSTONE PROGRAM, SOME STRING + * Next_Ptr_Glob-> + * Ptr_Comp: * 98004 same as above + * Discr: O.K. 0 + * Enum_Comp: O.K. 1 + * Int_Comp: O.K. 18 + * Str_Comp: O.K. DHRYSTONE PROGRAM, SOME STRING + * Int_1_Loc: O.K. 5 + * Int_2_Loc: O.K. 13 + * Int_3_Loc: O.K. 7 + * Enum_Loc: O.K. 1 + * Str_1_Loc: O.K. DHRYSTONE PROGRAM, 1'ST STRING + * Str_2_Loc: O.K. DHRYSTONE PROGRAM, 2'ND STRING + * + * Register option Not selected. + * + * Microseconds 1 loop: 20.06 + * Dhrystones / second: 49844 + * VAX MIPS rating: 28.37 + * + ************************************************************************** + */ + +/* Compiler and system dependent definitions: */ + +#ifndef TIME +#define TIMES +#endif +/* Use times(2) time function unless */ +/* explicitly defined otherwise */ + +#ifdef TIMES +/* #include + #include */ +/* for "times" */ +#endif + +#define Mic_secs_Per_Second 1000000.0 +/* Berkeley UNIX C returns process times in seconds/HZ */ + +#ifdef NOSTRUCTASSIGN +#define structassign(d, s) memcpy(&(d), &(s), sizeof(d)) +#else +#define structassign(d, s) d = s +#endif + +#ifdef NOENUM +#define Ident_1 0 +#define Ident_2 1 +#define Ident_3 2 +#define Ident_4 3 +#define Ident_5 4 +typedef int Enumeration; +#else +typedef enum { Ident_1, Ident_2, Ident_3, Ident_4, Ident_5 } Enumeration; +#endif +/* for boolean and enumeration types in Ada, Pascal */ + +/* General definitions: */ + +#include +#include + +/* for strcpy, strcmp */ + +#define Null 0 +/* Value of a Null pointer */ +#define true 1 +#define false 0 + +typedef int One_Thirty; +typedef int One_Fifty; +typedef char Capital_Letter; +typedef int Boolean; +typedef char Str_30[31]; +typedef int Arr_1_Dim[50]; +typedef int Arr_2_Dim[50][50]; + +typedef struct record { + struct record *Ptr_Comp; + Enumeration Discr; + union { + struct { + Enumeration Enum_Comp; + int Int_Comp; + char Str_Comp[31]; + } var_1; + struct { + Enumeration E_Comp_2; + char Str_2_Comp[31]; + } var_2; + struct { + char Ch_1_Comp; + char Ch_2_Comp; + } var_3; + } variant; +} Rec_Type, *Rec_Pointer; diff --git a/wasm-micro-runtime/tests/benchmarks/dhrystone/run.sh b/wasm-micro-runtime/tests/benchmarks/dhrystone/run.sh new file mode 100755 index 0000000..a9ac1d0 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/dhrystone/run.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +readonly IWASM_CMD="../../../product-mini/platforms/${PLATFORM}/build/iwasm" + +echo "============> run dhrystone native" +./dhrystone_native + +echo "============> run dhrystone.aot" +${IWASM_CMD} dhrystone.aot + +if [[ ${PLATFORM} == "linux" ]]; then + echo "============> run dhrystone_segue.aot" + ${IWASM_CMD} dhrystone_segue.aot +fi diff --git a/wasm-micro-runtime/tests/benchmarks/dhrystone/src/dhry_1.c b/wasm-micro-runtime/tests/benchmarks/dhrystone/src/dhry_1.c new file mode 100644 index 0000000..92f6e7e --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/dhrystone/src/dhry_1.c @@ -0,0 +1,485 @@ +/* + ************************************************************************* + * + * "DHRYSTONE" Benchmark Program + * ----------------------------- + * + * Version: C, Version 2.1 + * + * File: dhry_1.c (part 2 of 3) + * + * Date: May 25, 1988 + * + * Author: Reinhold P. Weicker + * + ************************************************************************* + */ + +#include +#include +#include +#include "dhry.h" + +/* Global Variables: */ + +Rec_Pointer Ptr_Glob, Next_Ptr_Glob; +int Int_Glob; +Boolean Bool_Glob; +char Ch_1_Glob, Ch_2_Glob; +int Arr_1_Glob[50]; +int Arr_2_Glob[50][50]; + +Enumeration +Func_1(Capital_Letter Ch_1_Par_Val, Capital_Letter Ch_2_Par_Val); +/* +forward declaration necessary since Enumeration may not simply be int +*/ + +#ifndef ROPT +#define REG +/* REG becomes defined as empty */ +/* i.e. no register variables */ +#else +#define REG register +#endif + +void +Proc_1(REG Rec_Pointer Ptr_Val_Par); +void +Proc_2(One_Fifty *Int_Par_Ref); +void +Proc_3(Rec_Pointer *Ptr_Ref_Par); +void +Proc_4(); +void +Proc_5(); +void +Proc_6(Enumeration Enum_Val_Par, Enumeration *Enum_Ref_Par); +void +Proc_7(One_Fifty Int_1_Par_Val, One_Fifty Int_2_Par_Val, + One_Fifty *Int_Par_Ref); +void +Proc_8(Arr_1_Dim Arr_1_Par_Ref, Arr_2_Dim Arr_2_Par_Ref, int Int_1_Par_Val, + int Int_2_Par_Val); + +Boolean +Func_2(Str_30 Str_1_Par_Ref, Str_30 Str_2_Par_Ref); + +/* variables for time measurement: */ + +#define Too_Small_Time 2 +/* Measurements should last at least 2 seconds */ + +#define BILLION 1000000000L +#define MILLION 1000000 +struct timespec Begin_Time, End_Time; +double User_Time; + +double Microseconds, Dhrystones_Per_Second, Vax_Mips; + +/* end of variables for time measurement */ + +int +main(int argc, char *argv[]) +/*****/ + +/* main program, corresponds to procedures */ +/* Main and Proc_0 in the Ada version */ +{ + One_Fifty Int_1_Loc; + REG One_Fifty Int_2_Loc; + One_Fifty Int_3_Loc; + REG char Ch_Index; + Enumeration Enum_Loc; + Str_30 Str_1_Loc; + Str_30 Str_2_Loc; + REG int Run_Index; + REG int Number_Of_Runs; + int endit, count = 10; + char general[9][80] = { " " }; + + /*********************************************************************** + * Change for compiler and optimisation used * + ***********************************************************************/ + + Next_Ptr_Glob = (Rec_Pointer)malloc(sizeof(Rec_Type)); + Ptr_Glob = (Rec_Pointer)malloc(sizeof(Rec_Type)); + + Ptr_Glob->Ptr_Comp = Next_Ptr_Glob; + Ptr_Glob->Discr = Ident_1; + Ptr_Glob->variant.var_1.Enum_Comp = Ident_3; + Ptr_Glob->variant.var_1.Int_Comp = 40; + strcpy(Ptr_Glob->variant.var_1.Str_Comp, "DHRYSTONE PROGRAM, SOME STRING"); + strcpy(Str_1_Loc, "DHRYSTONE PROGRAM, 1'ST STRING"); + + Arr_2_Glob[8][7] = 10; + /* Was missing in published program. Without this statement, */ + /* Arr_2_Glob [8][7] would have an undefined value. */ + /* Warning: With 16-Bit processors and Number_Of_Runs > 32000, */ + /* overflow may occur for this array element. */ + + printf("\n"); + printf("Dhrystone Benchmark, Version 2.1 (Language: C or C++)\n"); + printf("\n"); + + Number_Of_Runs = 5000; + + do { + + Number_Of_Runs = Number_Of_Runs * 2; + count = count - 1; + Arr_2_Glob[8][7] = 10; + + /***************/ + /* Start timer */ + /***************/ + + clock_gettime(CLOCK_MONOTONIC, &Begin_Time); + + for (Run_Index = 1; Run_Index <= Number_Of_Runs; ++Run_Index) { + + Proc_5(); + Proc_4(); + /* Ch_1_Glob == 'A', Ch_2_Glob == 'B', Bool_Glob == true */ + Int_1_Loc = 2; + Int_2_Loc = 3; + strcpy(Str_2_Loc, "DHRYSTONE PROGRAM, 2'ND STRING"); + Enum_Loc = Ident_2; + Bool_Glob = !Func_2(Str_1_Loc, Str_2_Loc); + /* Bool_Glob == 1 */ + while (Int_1_Loc < Int_2_Loc) /* loop body executed once */ + { + Int_3_Loc = 5 * Int_1_Loc - Int_2_Loc; + /* Int_3_Loc == 7 */ + Proc_7(Int_1_Loc, Int_2_Loc, &Int_3_Loc); + /* Int_3_Loc == 7 */ + Int_1_Loc += 1; + } /* while */ + /* Int_1_Loc == 3, Int_2_Loc == 3, Int_3_Loc == 7 */ + Proc_8(Arr_1_Glob, Arr_2_Glob, Int_1_Loc, Int_3_Loc); + /* Int_Glob == 5 */ + Proc_1(Ptr_Glob); + for (Ch_Index = 'A'; Ch_Index <= Ch_2_Glob; ++Ch_Index) + /* loop body executed twice */ + { + if (Enum_Loc == Func_1(Ch_Index, 'C')) + /* then, not executed */ + { + Proc_6(Ident_1, &Enum_Loc); + strcpy(Str_2_Loc, "DHRYSTONE PROGRAM, 3'RD STRING"); + Int_2_Loc = Run_Index; + Int_Glob = Run_Index; + } + } + /* Int_1_Loc == 3, Int_2_Loc == 3, Int_3_Loc == 7 */ + Int_2_Loc = Int_2_Loc * Int_1_Loc; + Int_1_Loc = Int_2_Loc / Int_3_Loc; + Int_2_Loc = 7 * (Int_2_Loc - Int_3_Loc) - Int_1_Loc; + /* Int_1_Loc == 1, Int_2_Loc == 13, Int_3_Loc == 7 */ + Proc_2(&Int_1_Loc); + /* Int_1_Loc == 5 */ + + } /* loop "for Run_Index" */ + + /**************/ + /* Stop timer */ + /**************/ + + clock_gettime(CLOCK_MONOTONIC, &End_Time); + + User_Time = (End_Time.tv_sec - Begin_Time.tv_sec) * MILLION + + (End_Time.tv_nsec - Begin_Time.tv_nsec) / 1000; + User_Time = User_Time / MILLION; /* convert to seconds */ + + printf("%ld runs %lf seconds \n", (long)Number_Of_Runs, User_Time); + if (User_Time > 5.0) { + count = 0; + } + else { + if (User_Time < 0.1) { + Number_Of_Runs = Number_Of_Runs * 5; + } + } + } /* calibrate/run do while */ + while (count > 0); + + printf("\n"); + printf("Final values (* implementation-dependent):\n"); + printf("\n"); + printf("Int_Glob: "); + if (Int_Glob == 5) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Int_Glob); + + printf("Bool_Glob: "); + if (Bool_Glob == 1) + printf("O.K. "); + else + printf("WRONG "); + printf("%d\n", Bool_Glob); + + printf("Ch_1_Glob: "); + if (Ch_1_Glob == 'A') + printf("O.K. "); + else + printf("WRONG "); + printf("%c ", Ch_1_Glob); + + printf("Ch_2_Glob: "); + if (Ch_2_Glob == 'B') + printf("O.K. "); + else + printf("WRONG "); + printf("%c\n", Ch_2_Glob); + + printf("Arr_1_Glob[8]: "); + if (Arr_1_Glob[8] == 7) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Arr_1_Glob[8]); + + printf("Arr_2_Glob8/7: "); + if (Arr_2_Glob[8][7] == Number_Of_Runs + 10) + printf("O.K. "); + else + printf("WRONG "); + printf("%10d\n", Arr_2_Glob[8][7]); + + printf("Ptr_Glob-> "); + printf(" Ptr_Comp: * %p\n", Ptr_Glob->Ptr_Comp); + + printf(" Discr: "); + if (Ptr_Glob->Discr == 0) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Ptr_Glob->Discr); + + printf("Enum_Comp: "); + if (Ptr_Glob->variant.var_1.Enum_Comp == 2) + printf("O.K. "); + else + printf("WRONG "); + printf("%d\n", Ptr_Glob->variant.var_1.Enum_Comp); + + printf(" Int_Comp: "); + if (Ptr_Glob->variant.var_1.Int_Comp == 17) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Ptr_Glob->variant.var_1.Int_Comp); + + printf("Str_Comp: "); + if (strcmp(Ptr_Glob->variant.var_1.Str_Comp, + "DHRYSTONE PROGRAM, SOME STRING") + == 0) + printf("O.K. "); + else + printf("WRONG "); + printf("%s\n", Ptr_Glob->variant.var_1.Str_Comp); + + printf("Next_Ptr_Glob-> "); + printf(" Ptr_Comp: * %p", Next_Ptr_Glob->Ptr_Comp); + printf(" same as above\n"); + + printf(" Discr: "); + if (Next_Ptr_Glob->Discr == 0) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Next_Ptr_Glob->Discr); + + printf("Enum_Comp: "); + if (Next_Ptr_Glob->variant.var_1.Enum_Comp == 1) + printf("O.K. "); + else + printf("WRONG "); + printf("%d\n", Next_Ptr_Glob->variant.var_1.Enum_Comp); + + printf(" Int_Comp: "); + if (Next_Ptr_Glob->variant.var_1.Int_Comp == 18) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Next_Ptr_Glob->variant.var_1.Int_Comp); + + printf("Str_Comp: "); + if (strcmp(Next_Ptr_Glob->variant.var_1.Str_Comp, + "DHRYSTONE PROGRAM, SOME STRING") + == 0) + printf("O.K. "); + else + printf("WRONG "); + printf("%s\n", Next_Ptr_Glob->variant.var_1.Str_Comp); + + printf("Int_1_Loc: "); + if (Int_1_Loc == 5) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Int_1_Loc); + + printf("Int_2_Loc: "); + if (Int_2_Loc == 13) + printf("O.K. "); + else + printf("WRONG "); + printf("%d\n", Int_2_Loc); + + printf("Int_3_Loc: "); + if (Int_3_Loc == 7) + printf("O.K. "); + else + printf("WRONG "); + printf("%d ", Int_3_Loc); + + printf("Enum_Loc: "); + if (Enum_Loc == 1) + printf("O.K. "); + else + printf("WRONG "); + printf("%d\n", Enum_Loc); + + printf("Str_1_Loc: "); + if (strcmp(Str_1_Loc, "DHRYSTONE PROGRAM, 1'ST STRING") == 0) + printf("O.K. "); + else + printf("WRONG "); + printf("%s\n", Str_1_Loc); + + printf("Str_2_Loc: "); + if (strcmp(Str_2_Loc, "DHRYSTONE PROGRAM, 2'ND STRING") == 0) + printf("O.K. "); + else + printf("WRONG "); + printf("%s\n", Str_2_Loc); + + printf("\n"); + + if (User_Time < Too_Small_Time) { + printf("Measured time too small to obtain meaningful results\n"); + printf("Please increase number of runs\n"); + printf("\n"); + } + else { + Microseconds = User_Time * Mic_secs_Per_Second / (double)Number_Of_Runs; + Dhrystones_Per_Second = (double)Number_Of_Runs / User_Time; + Vax_Mips = Dhrystones_Per_Second / 1757.0; + + printf("Microseconds for one run through Dhrystone: "); + printf("%lf \n", Microseconds); + printf("Dhrystones per Second: "); + printf("%lf \n", Dhrystones_Per_Second); + printf("VAX MIPS rating = "); + printf("%lf \n", Vax_Mips); + printf("\n"); + } + + free(Next_Ptr_Glob); + free(Ptr_Glob); + return 1; +} + +void +Proc_1(REG Rec_Pointer Ptr_Val_Par) +/******************/ + +/* executed once */ +{ + REG Rec_Pointer Next_Record = Ptr_Val_Par->Ptr_Comp; + /* == Ptr_Glob_Next */ + /* Local variable, initialized with Ptr_Val_Par->Ptr_Comp, */ + /* corresponds to "rename" in Ada, "with" in Pascal */ + + structassign(*Ptr_Val_Par->Ptr_Comp, *Ptr_Glob); + Ptr_Val_Par->variant.var_1.Int_Comp = 5; + Next_Record->variant.var_1.Int_Comp = Ptr_Val_Par->variant.var_1.Int_Comp; + Next_Record->Ptr_Comp = Ptr_Val_Par->Ptr_Comp; + Proc_3(&Next_Record->Ptr_Comp); + /* Ptr_Val_Par->Ptr_Comp->Ptr_Comp + == Ptr_Glob->Ptr_Comp */ + if (Next_Record->Discr == Ident_1) + /* then, executed */ + { + Next_Record->variant.var_1.Int_Comp = 6; + Proc_6(Ptr_Val_Par->variant.var_1.Enum_Comp, + &Next_Record->variant.var_1.Enum_Comp); + Next_Record->Ptr_Comp = Ptr_Glob->Ptr_Comp; + Proc_7(Next_Record->variant.var_1.Int_Comp, 10, + &Next_Record->variant.var_1.Int_Comp); + } + else { /* not executed */ + structassign(*Ptr_Val_Par, *Ptr_Val_Par->Ptr_Comp); + } +} /* Proc_1 */ + +void +Proc_2(One_Fifty *Int_Par_Ref) +/******************/ +/* executed once */ +/* *Int_Par_Ref == 1, becomes 4 */ + +{ + One_Fifty Int_Loc; + Enumeration Enum_Loc; + + Int_Loc = *Int_Par_Ref + 10; + do /* executed once */ + if (Ch_1_Glob == 'A') + /* then, executed */ + { + Int_Loc -= 1; + *Int_Par_Ref = Int_Loc - Int_Glob; + Enum_Loc = Ident_1; + } /* if */ + while (Enum_Loc != Ident_1); /* true */ +} /* Proc_2 */ + +void +Proc_3(Rec_Pointer *Ptr_Ref_Par) +/******************/ +/* executed once */ +/* Ptr_Ref_Par becomes Ptr_Glob */ + +{ + if (Ptr_Glob != Null) + /* then, executed */ + *Ptr_Ref_Par = Ptr_Glob->Ptr_Comp; + Proc_7(10, Int_Glob, &Ptr_Glob->variant.var_1.Int_Comp); +} /* Proc_3 */ + +void +Proc_4() /* without parameters */ +/*******/ +/* executed once */ +{ + Boolean Bool_Loc; + + Bool_Loc = Ch_1_Glob == 'A'; + Bool_Glob = Bool_Loc | Bool_Glob; + Ch_2_Glob = 'B'; +} /* Proc_4 */ + +void +Proc_5() /* without parameters */ +/*******/ +/* executed once */ +{ + Ch_1_Glob = 'A'; + Bool_Glob = false; +} /* Proc_5 */ + +/* Procedure for the assignment of structures, */ +/* if the C compiler doesn't support this feature */ +#ifdef NOSTRUCTASSIGN +memcpy(d, s, l) register char *d; +register char *s; +register int l; +{ + while (l--) + *d++ = *s++; +} +#endif diff --git a/wasm-micro-runtime/tests/benchmarks/dhrystone/src/dhry_2.c b/wasm-micro-runtime/tests/benchmarks/dhrystone/src/dhry_2.c new file mode 100644 index 0000000..5378799 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/dhrystone/src/dhry_2.c @@ -0,0 +1,187 @@ +/* + ************************************************************************* + * + * "DHRYSTONE" Benchmark Program + * ----------------------------- + * + * Version: C, Version 2.1 + * + * File: dhry_2.c (part 3 of 3) + * + * Date: May 25, 1988 + * + * Author: Reinhold P. Weicker + * + ************************************************************************* + */ + +#include "dhry.h" + +#ifndef REG +#define REG +/* REG becomes defined as empty */ +/* i.e. no register variables */ +#else +#define REG register +#endif + +extern int Int_Glob; +extern char Ch_1_Glob; + +Boolean +Func_3(Enumeration Enum_Par_Val); + +void +Proc_6(Enumeration Enum_Val_Par, Enumeration *Enum_Ref_Par) +/*********************************/ +/* executed once */ +/* Enum_Val_Par == Ident_3, Enum_Ref_Par becomes Ident_2 */ + +{ + *Enum_Ref_Par = Enum_Val_Par; + if (!Func_3(Enum_Val_Par)) + /* then, not executed */ + *Enum_Ref_Par = Ident_4; + switch (Enum_Val_Par) { + case Ident_1: + *Enum_Ref_Par = Ident_1; + break; + case Ident_2: + if (Int_Glob > 100) + /* then */ + *Enum_Ref_Par = Ident_1; + else + *Enum_Ref_Par = Ident_4; + break; + case Ident_3: /* executed */ + *Enum_Ref_Par = Ident_2; + break; + case Ident_4: + break; + case Ident_5: + *Enum_Ref_Par = Ident_3; + break; + } /* switch */ +} /* Proc_6 */ + +void +Proc_7(One_Fifty Int_1_Par_Val, One_Fifty Int_2_Par_Val, One_Fifty *Int_Par_Ref) +/**********************************************/ +/* executed three times */ +/* first call: Int_1_Par_Val == 2, Int_2_Par_Val == 3, */ +/* Int_Par_Ref becomes 7 */ +/* second call: Int_1_Par_Val == 10, Int_2_Par_Val == 5, */ +/* Int_Par_Ref becomes 17 */ +/* third call: Int_1_Par_Val == 6, Int_2_Par_Val == 10, */ +/* Int_Par_Ref becomes 18 */ + +{ + One_Fifty Int_Loc; + + Int_Loc = Int_1_Par_Val + 2; + *Int_Par_Ref = Int_2_Par_Val + Int_Loc; +} /* Proc_7 */ + +void +Proc_8(Arr_1_Dim Arr_1_Par_Ref, Arr_2_Dim Arr_2_Par_Ref, int Int_1_Par_Val, + int Int_2_Par_Val) +/*********************************************************************/ +/* executed once */ +/* Int_Par_Val_1 == 3 */ +/* Int_Par_Val_2 == 7 */ + +{ + REG One_Fifty Int_Index; + REG One_Fifty Int_Loc; + + Int_Loc = Int_1_Par_Val + 5; + Arr_1_Par_Ref[Int_Loc] = Int_2_Par_Val; + Arr_1_Par_Ref[Int_Loc + 1] = Arr_1_Par_Ref[Int_Loc]; + Arr_1_Par_Ref[Int_Loc + 30] = Int_Loc; + for (Int_Index = Int_Loc; Int_Index <= Int_Loc + 1; ++Int_Index) + Arr_2_Par_Ref[Int_Loc][Int_Index] = Int_Loc; + Arr_2_Par_Ref[Int_Loc][Int_Loc - 1] += 1; + Arr_2_Par_Ref[Int_Loc + 20][Int_Loc] = Arr_1_Par_Ref[Int_Loc]; + Int_Glob = 5; +} /* Proc_8 */ + +Enumeration +Func_1(Capital_Letter Ch_1_Par_Val, Capital_Letter Ch_2_Par_Val) +/*************************************************/ +/* executed three times */ +/* first call: Ch_1_Par_Val == 'H', Ch_2_Par_Val == 'R' */ +/* second call: Ch_1_Par_Val == 'A', Ch_2_Par_Val == 'C' */ +/* third call: Ch_1_Par_Val == 'B', Ch_2_Par_Val == 'C' */ + +{ + Capital_Letter Ch_1_Loc; + Capital_Letter Ch_2_Loc; + + Ch_1_Loc = Ch_1_Par_Val; + Ch_2_Loc = Ch_1_Loc; + if (Ch_2_Loc != Ch_2_Par_Val) + /* then, executed */ + return (Ident_1); + else /* not executed */ + { + Ch_1_Glob = Ch_1_Loc; + return (Ident_2); + } +} /* Func_1 */ + +Boolean +Func_2(Str_30 Str_1_Par_Ref, Str_30 Str_2_Par_Ref) +/*************************************************/ +/* executed once */ +/* Str_1_Par_Ref == "DHRYSTONE PROGRAM, 1'ST STRING" */ +/* Str_2_Par_Ref == "DHRYSTONE PROGRAM, 2'ND STRING" */ + +{ + REG One_Thirty Int_Loc; + Capital_Letter Ch_Loc; + + Int_Loc = 2; + while (Int_Loc <= 2) /* loop body executed once */ + if (Func_1(Str_1_Par_Ref[Int_Loc], Str_2_Par_Ref[Int_Loc + 1]) + == Ident_1) + /* then, executed */ + { + Ch_Loc = 'A'; + Int_Loc += 1; + } /* if, while */ + if (Ch_Loc >= 'W' && Ch_Loc < 'Z') + /* then, not executed */ + Int_Loc = 7; + if (Ch_Loc == 'R') + /* then, not executed */ + return (true); + else /* executed */ + { + if (strcmp(Str_1_Par_Ref, Str_2_Par_Ref) > 0) + /* then, not executed */ + { + Int_Loc += 7; + Int_Glob = Int_Loc; + return (true); + } + else /* executed */ + return (false); + } /* if Ch_Loc */ +} /* Func_2 */ + +Boolean +Func_3(Enumeration Enum_Par_Val) +/***************************/ +/* executed once */ +/* Enum_Par_Val == Ident_3 */ + +{ + Enumeration Enum_Loc; + + Enum_Loc = Enum_Par_Val; + if (Enum_Loc == Ident_3) + /* then, executed */ + return (true); + else /* not executed */ + return (false); +} /* Func_3 */ diff --git a/wasm-micro-runtime/tests/benchmarks/dhrystone/test_pgo.sh b/wasm-micro-runtime/tests/benchmarks/dhrystone/test_pgo.sh new file mode 100755 index 0000000..8bca197 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/dhrystone/test_pgo.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +if [ "$1" = "--sgx" ] && [ "$PLATFORM" = "linux" ]; then + IWASM="../../../product-mini/platforms/${PLATFORM}-sgx/enclave-sample/iwasm" + WAMRC="../../../wamr-compiler/build/wamrc -sgx" +else + IWASM="../../../product-mini/platforms/${PLATFORM}/build/iwasm" + WAMRC="../../../wamr-compiler/build/wamrc" +fi + +if [ ! -e "dhrystone.wasm" ]; then + echo "dhrystone.wasm doesn't exist, please run build.sh first" + exit +fi + +echo "" +echo "Compile dhrystone.wasm to dhrystone.aot .." +${WAMRC} -o dhrystone.aot dhrystone.wasm + +echo "" +echo "Compile dhrystone.wasm to dhrystone_pgo.aot .." +${WAMRC} --enable-llvm-pgo -o dhrystone_pgo.aot dhrystone.wasm + +echo "" +echo "Run dhrystone_pgo.aot to generate the raw profile data .." +${IWASM} --gen-prof-file=dhrystone.profraw dhrystone_pgo.aot + +echo "" +echo "Merge the raw profile data to dhrystone.profdata .." +rm -f dhrystone.profdata && llvm-profdata merge -output=dhrystone.profdata dhrystone.profraw + +echo "" +echo "Compile dhrystone.wasm to dhrystone_opt.aot with the profile data .." +${WAMRC} --use-prof-file=dhrystone.profdata -o dhrystone_opt.aot dhrystone.wasm + +echo "" +echo "Run the dhrystone native" +./dhrystone_native + +echo "" +echo "Run the original aot file dhrystone.aot" +${IWASM} dhrystone.aot + +echo "" +echo "Run the PGO optimized aot file dhrystone_opt.aot" +${IWASM} dhrystone_opt.aot + +# Show the profile data: +# llvm-profdata show --all-functions --detailed-summary --binary-ids --counts \ +# --hot-func-list --memop-sizes --show-prof-sym-list dhrystone.profraw diff --git a/wasm-micro-runtime/tests/benchmarks/jetstream/README.md b/wasm-micro-runtime/tests/benchmarks/jetstream/README.md new file mode 100644 index 0000000..1bf438c --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/jetstream/README.md @@ -0,0 +1,35 @@ +# Introduction + +[JetStream 2](https://browserbench.org/JetStream) is a JavaScript and WebAssembly benchmark suite focused on the most advanced web applications. It rewards browsers that start up quickly, execute code quickly, and run smoothly. + +**Source**: https://browserbench.org/JetStream/in-depth.html + +# Building + +Please build iwasm and wamrc, refer to: +- [Build iwasm on Linux](../../../doc/build_wamr.md#linux), or [Build iwasm on MacOS](../../../doc/build_wamr.md#macos) +- [Build wamrc AOT compiler](../../../README.md#build-wamrc-aot-compiler) + +And install emsdk, refer to [the guide](https://emscripten.org/docs/getting_started/downloads.html). Don't forget to activate + emsdk and set up environment variables. For example, use instructions below to install it under /opt and activate it: +``` bash +$ cd /opt +$ git clone https://github.com/emscripten-core/emsdk.git +$ cd emsdk +$ git pull +$ ./emsdk install latest +$ ./emsdk activate latest +$ echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc +``` + +And then run `./build.sh` to build the source code, the folder `out` will be created and files will be generated under it. + +# Running + +Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested for each workload, and the file `report.txt` will be generated. + +Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`. + +- For Linux, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled. + +- For Linux-sgx, similarly, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then `make` in the directory `enclave-sample`. And run `./test_pgo.sh --sgx` to test the benchmark. diff --git a/wasm-micro-runtime/tests/benchmarks/jetstream/build.sh b/wasm-micro-runtime/tests/benchmarks/jetstream/build.sh new file mode 100755 index 0000000..019b407 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/jetstream/build.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +source /opt/emsdk/emsdk_env.sh + +PLATFORM=$(uname -s | tr A-Z a-z) + +WORK_DIR=$PWD +OUT_DIR=$PWD/out +WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc +JETSTREAM_DIR=$PWD/perf-automation/benchmarks/JetStream2/wasm + +mkdir -p ${OUT_DIR} + +if [[ $1 != "--no-simd" ]];then + NATIVE_SIMD_FLAGS="-msse2 -msse3 -msse4" + WASM_SIMD_FLAGS="-msimd128 -msse2 -msse3 -msse4" +else + NATIVE_SIMD_FLAGS="" + WASM_SIMD_FLAGS="" +fi + +if [ ! -d perf-automation ]; then + git clone https://github.com/mozilla/perf-automation.git + echo "Patch source files .." + cd perf-automation + patch -p1 -N < ../jetstream.patch +fi + +cd ${JETSTREAM_DIR} + +# Build gcc-loops + +echo "Build gcc-loops with g++ .." +g++ -O3 ${NATIVE_SIMD_FLAGS} -o ${OUT_DIR}/gcc-loops_native gcc-loops.cpp + +echo "Build gcc-loops with em++ .." +em++ -O3 -s STANDALONE_WASM=1 ${WASM_SIMD_FLAGS} \ + -s INITIAL_MEMORY=1048576 \ + -s TOTAL_STACK=32768 \ + -s "EXPORTED_FUNCTIONS=['_main']" \ + -s ERROR_ON_UNDEFINED_SYMBOLS=0 \ + -o ${OUT_DIR}/gcc-loops.wasm gcc-loops.cpp + +echo "Compile gcc-loops.wasm to gcc-loops.aot" +${WAMRC_CMD} -o ${OUT_DIR}/gcc-loops.aot ${OUT_DIR}/gcc-loops.wasm + +if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile gcc-loops.wasm to gcc-loops_segue.aot" + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/gcc-loops_segue.aot ${OUT_DIR}/gcc-loops.wasm +fi + +# Build quicksort + +echo "Build quicksort with gcc .." +gcc -O3 ${NATIVE_SIMD_FLAGS} -o ${OUT_DIR}/quicksort_native quicksort.c + +echo "Build quicksort with emcc .." +emcc -O3 -s STANDALONE_WASM=1 ${WASM_SIMD_FLAGS} \ + -s INITIAL_MEMORY=1048576 \ + -s TOTAL_STACK=32768 \ + -s "EXPORTED_FUNCTIONS=['_main']" \ + -o ${OUT_DIR}/quicksort.wasm quicksort.c + +echo "Compile quicksort.wasm to quicksort.aot" +${WAMRC_CMD} -o ${OUT_DIR}/quicksort.aot ${OUT_DIR}/quicksort.wasm + +if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile quicksort.wasm to quicksort_segue.aot" + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/quicksort_segue.aot ${OUT_DIR}/quicksort.wasm +fi + +# Build HashSet + +echo "Build HashSet with g++ .." +g++ -O3 ${NATIVE_SIMD_FLAGS} -o ${OUT_DIR}/HashSet_native HashSet.cpp \ + -lstdc++ + +echo "Build HashSet with em++ .." +em++ -O3 -s STANDALONE_WASM=1 ${WASM_SIMD_FLAGS} \ + -s INITIAL_MEMORY=1048576 \ + -s TOTAL_STACK=32768 \ + -s "EXPORTED_FUNCTIONS=['_main']" \ + -o ${OUT_DIR}/HashSet.wasm HashSet.cpp + +echo "Compile HashSet.wasm to HashSet.aot" +${WAMRC_CMD} -o ${OUT_DIR}/HashSet.aot ${OUT_DIR}/HashSet.wasm + +if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile HashSet.wasm to HashSet_segue.aot" + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/HashSet_segue.aot ${OUT_DIR}/HashSet.wasm +fi + +# Build float-mm + +cd ${JETSTREAM_DIR}/../simple + +echo "Build float-mm with gcc .." +gcc -O3 ${NATIVE_SIMD_FLAGS} -o ${OUT_DIR}/float-mm_native float-mm.c + +echo "Build float-mm with emcc .." +emcc -O3 -s STANDALONE_WASM=1 ${WASM_SIMD_FLAGS} \ + -s INITIAL_MEMORY=1048576 \ + -s TOTAL_STACK=32768 \ + -s "EXPORTED_FUNCTIONS=['_main']" \ + -o ${OUT_DIR}/float-mm.wasm float-mm.c + +echo "Compile float-mm.wasm to float-mm.aot" +${WAMRC_CMD} -o ${OUT_DIR}/float-mm.aot ${OUT_DIR}/float-mm.wasm + +if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile float-mm.wasm to float-mm_segue.aot" + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/float-mm_segue.aot ${OUT_DIR}/float-mm.wasm +fi + +# Build tsf + +cd ${JETSTREAM_DIR}/TSF + +tsf_srcs="tsf_asprintf.c tsf_buffer.c tsf_error.c tsf_reflect.c tsf_st.c \ + tsf_type.c tsf_io.c tsf_native.c tsf_generator.c tsf_st_typetable.c \ + tsf_parser.c tsf_buf_writer.c tsf_buf_reader.c tsf_primitive.c \ + tsf_type_table.c tsf_copier.c tsf_destructor.c tsf_gpc_code_gen.c \ + gpc_code_gen_util.c gpc_threaded.c gpc_intable.c gpc_instruction.c \ + gpc_program.c gpc_proto.c gpc_stack_height.c tsf_serial_in_man.c \ + tsf_serial_out_man.c tsf_type_in_map.c tsf_type_out_map.c \ + tsf_stream_file_input.c tsf_stream_file_output.c tsf_sort.c \ + tsf_version.c tsf_named_type.c tsf_io_utils.c tsf_zip_attr.c \ + tsf_zip_reader.c tsf_zip_writer.c tsf_zip_abstract.c tsf_limits.c \ + tsf_ra_type_man.c tsf_adaptive_reader.c tsf_sha1.c tsf_sha1_writer.c \ + tsf_fsdb.c tsf_fsdb_protocol.c tsf_define_helpers.c tsf_ir.c \ + tsf_ir_different.c tsf_ir_speed.c" + +echo "Build tsf with gcc .." +gcc \ + -o ${OUT_DIR}/tsf_native -O3 ${NATIVE_SIMD_FLAGS} \ + -I. -DTSF_BUILD_SYSTEM=1 \ + ${tsf_srcs} -lm + +echo "Build tsf standalone with wasi-sdk .." +/opt/wasi-sdk/bin/clang -O3 ${WASM_SIMD_FLAGS} -z stack-size=1048576 \ + -Wl,--initial-memory=52428800 \ + -Wl,--export=main \ + -Wl,--export=__heap_base,--export=__data_end \ + -I. -DTSF_BUILD_SYSTEM=1 \ + -Wl,--allow-undefined \ + -o ${OUT_DIR}/tsf.wasm \ + ${tsf_srcs} + +echo "Compile tsf.wasm to tsf.aot" +${WAMRC_CMD} -o ${OUT_DIR}/tsf.aot ${OUT_DIR}/tsf.wasm + +if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile tsf.wasm to tsf_segue.aot" + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/tsf_segue.aot ${OUT_DIR}/tsf.wasm +fi diff --git a/wasm-micro-runtime/tests/benchmarks/jetstream/jetstream.patch b/wasm-micro-runtime/tests/benchmarks/jetstream/jetstream.patch new file mode 100644 index 0000000..8799fb6 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/jetstream/jetstream.patch @@ -0,0 +1,63 @@ +diff --git a/benchmarks/JetStream2/wasm/HashSet.cpp b/benchmarks/JetStream2/wasm/HashSet.cpp +index eca979b0..d1bf4d3d 100644 +--- a/benchmarks/JetStream2/wasm/HashSet.cpp ++++ b/benchmarks/JetStream2/wasm/HashSet.cpp +@@ -22,8 +22,10 @@ + + #include + #include ++#include + #include + #include ++#include + #include + + // Compile with: xcrun clang++ -o HashSet HashSet.cpp -O2 -W -framework Foundation -licucore -std=c++11 -fvisibility=hidden -DNDEBUG=1 +@@ -76,7 +78,7 @@ template + inline ToType bitwise_cast(FromType from) + { + typename std::remove_const::type to { }; +- std::memcpy(&to, &from, sizeof(to)); ++ memcpy(&to, &from, sizeof(to)); + return to; + } + +diff --git a/benchmarks/JetStream2/wasm/TSF/gpc_code_gen_util.c b/benchmarks/JetStream2/wasm/TSF/gpc_code_gen_util.c +index 56220fa7..7e3a365b 100644 +--- a/benchmarks/JetStream2/wasm/TSF/gpc_code_gen_util.c ++++ b/benchmarks/JetStream2/wasm/TSF/gpc_code_gen_util.c +@@ -34,6 +34,8 @@ + #include + #include + ++int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); ++ + /* code generation debugging */ + + /* NOTE: It is now the case that the count may be incremented multiple times, +diff --git a/benchmarks/JetStream2/wasm/TSF/tsf_internal.h b/benchmarks/JetStream2/wasm/TSF/tsf_internal.h +index 225a248b..ae39d3d3 100644 +--- a/benchmarks/JetStream2/wasm/TSF/tsf_internal.h ++++ b/benchmarks/JetStream2/wasm/TSF/tsf_internal.h +@@ -429,6 +429,7 @@ struct tsf_fsdb { + #endif + tsf_fsdb_connection_t *connection; + #endif ++ uint32_t __padding; + } remote; + } u; + tsf_limits_t *limits; +diff --git a/benchmarks/JetStream2/wasm/TSF/tsf_ir_speed.c b/benchmarks/JetStream2/wasm/TSF/tsf_ir_speed.c +index dd75c43e..79435c42 100644 +--- a/benchmarks/JetStream2/wasm/TSF/tsf_ir_speed.c ++++ b/benchmarks/JetStream2/wasm/TSF/tsf_ir_speed.c +@@ -63,6 +63,9 @@ static void writeTest(const char *filename, + Program_t *program; + unsigned elementIndex; + ++ if (!(programIndex % 100)) ++ printf("##programIndex: %u\n", programIndex); ++ + CS(program = tsf_region_create(sizeof(Program_t))); + + program->globals.len = numDecls + numDefns; diff --git a/wasm-micro-runtime/tests/benchmarks/jetstream/run_aot.sh b/wasm-micro-runtime/tests/benchmarks/jetstream/run_aot.sh new file mode 100755 index 0000000..85ef3fb --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/jetstream/run_aot.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm + +BENCH_NAME_MAX_LEN=20 + +JETSTREAM_CASES="gcc-loops HashSet tsf float-mm quicksort" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +if [[ ${PLATFORM} == "linux" ]]; then + echo -en "\t\t\t\t\t native\tiwasm-aot\tiwasm-aot-segue\n" >> $REPORT +else + echo -en "\t\t\t\t\t native\tiwasm-aot\n" >> $REPORT +fi + +for t in $JETSTREAM_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD --dir=. ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + if [[ ${PLATFORM} == "linux" ]]; then + echo "run $t with iwasm aot segue .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD --dir=. ${t}_segue.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + fi + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/benchmarks/jetstream/test_pgo.sh b/wasm-micro-runtime/tests/benchmarks/jetstream/test_pgo.sh new file mode 100755 index 0000000..a110182 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/jetstream/test_pgo.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +if [ "$1" = "--sgx" ] && [ "$PLATFORM" = "linux" ]; then + IWASM_CMD="$CUR_DIR/../../../product-mini/platforms/${PLATFORM}-sgx/enclave-sample/iwasm" + WAMRC_CMD="$CUR_DIR/../../../wamr-compiler/build/wamrc -sgx" +else + IWASM_CMD="$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm" + WAMRC_CMD="$CUR_DIR/../../../wamr-compiler/build/wamrc" +fi + +BENCH_NAME_MAX_LEN=20 + +JETSTREAM_CASES="gcc-loops HashSet tsf float-mm quicksort" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +pushd $OUT_DIR > /dev/null 2>&1 +for t in $JETSTREAM_CASES +do + if [ ! -e "${t}.wasm" ]; then + echo "${t}.wasm doesn't exist, please run build.sh first" + exit + fi + + echo "" + echo "Compile ${t}.wasm to ${t}.aot .." + ${WAMRC_CMD} -o ${t}.aot ${t}.wasm + + echo "" + echo "Compile ${t}.wasm to ${t}_pgo.aot .." + ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm + + echo "" + echo "Run ${t}_pgo.aot to generate the raw profile data .." + ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot + + echo "" + echo "Merge the raw profile data to ${t}.profdata .." + rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw + + echo "" + echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .." + ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm +done +popd > /dev/null 2>&1 + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +echo -en "\t\t\t\t\t native\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT + +for t in $JETSTREAM_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD --dir=. ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot opt .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD --dir=. ${t}_opt.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/benchmarks/libsodium/README.md b/wasm-micro-runtime/tests/benchmarks/libsodium/README.md new file mode 100644 index 0000000..a21e679 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/libsodium/README.md @@ -0,0 +1,29 @@ +# Introduction + +[libsodium](https://github.com/jedisct1/libsodium) is a new, easy-to-use software library for encryption, decryption, signatures, password hashing and more. + +**Source**: https://github.com/jedisct1/libsodium + +# Building + +Please build iwasm and wamrc, refer to: +- [Build iwasm on Linux](../../../doc/build_wamr.md#linux), or [Build iwasm on MacOS](../../../doc/build_wamr.md#macos) +- [Build wamrc AOT compiler](../../../README.md#build-wamrc-aot-compiler) + +And install [zig toolchain](https://ziglang.org/learn/getting-started), refer to [Install Zig from a Package Manager](https://github.com/ziglang/zig/wiki/Install-Zig-from-a-Package-Manager) for how to install it. + +And then run `./build.sh` to build the source code, the libsodium source code will be cloned, and test benchmarks of native version, wasm files and AOT files will be generated under `libsodium/zig-out/bin`. + +# Running + +Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested respectively. + +Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`. + +- For Linux, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled. + +- For Linux-sgx, similarly, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then `make` in the directory `enclave-sample`. And run `./test_pgo.sh --sgx` to test the benchmark. + +# Others + +Refer to [Performance of WebAssembly runtimes in 2023](https://00f.net/2023/01/04/webassembly-benchmark-2023) for more about the performance comparison of wasm runtimes on running the libsodium benchmarks. diff --git a/wasm-micro-runtime/tests/benchmarks/libsodium/build.sh b/wasm-micro-runtime/tests/benchmarks/libsodium/build.sh new file mode 100755 index 0000000..c949375 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/libsodium/build.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +libsodium_CASES="aead_aes256gcm2 aead_aes256gcm aead_chacha20poly13052 aead_chacha20poly1305 \ + aead_xchacha20poly1305 auth2 auth3 auth5 auth6 auth7 auth box2 box7 box8 \ + box_easy2 box_easy box_seal box_seed box chacha20 codecs core1 core2 core3 \ + core4 core5 core6 core_ed25519 core_ristretto255 ed25519_convert generichash2 \ + generichash3 generichash hash3 hash kdf keygen kx metamorphic misuse \ + onetimeauth2 onetimeauth7 onetimeauth pwhash_argon2id pwhash_argon2i \ + pwhash_scrypt_ll pwhash_scrypt randombytes scalarmult2 scalarmult5 \ + scalarmult6 scalarmult7 scalarmult8 scalarmult_ed25519 scalarmult_ristretto255 \ + scalarmult secretbox2 secretbox7 secretbox8 secretbox_easy2 secretbox_easy \ + secretbox secretstream_xchacha20poly1305 shorthash sign siphashx24 sodium_core \ + sodium_utils2 sodium_utils3 sodium_utils sodium_version stream2 stream3 stream4 \ + stream verify1 xchacha20" + +PLATFORM=$(uname -s | tr A-Z a-z) + +readonly WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc +readonly OUT_DIR=$PWD/libsodium/zig-out/bin + +if [ ! -d libsodium ]; then + git clone https://github.com/jedisct1/libsodium.git + cd libsodium + git checkout 1.0.19 + cd .. +fi + +cd libsodium + +echo "Build libsodium native" +zig build -Doptimize=ReleaseFast -Denable_benchmarks=true + +echo "Build libsodium wasm32-wasi" +zig build -Doptimize=ReleaseFast -Denable_benchmarks=true -Dtarget=wasm32-wasi + +for case in ${libsodium_CASES} +do + ${WAMRC_CMD} -o ${OUT_DIR}/${case}.aot ${OUT_DIR}/${case}.wasm + if [ "$?" != 0 ]; then + echo -e "Error while compiling ${case}.wasm to ${case}.aot" + exit + fi + + if [[ ${PLATFORM} == "linux" ]]; then + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/${case}_segue.aot ${OUT_DIR}/${case}.wasm + if [ "$?" != 0 ]; then + echo -e "Error while compiling ${case}.wasm to ${case}_segue.aot" + exit + fi + fi +done diff --git a/wasm-micro-runtime/tests/benchmarks/libsodium/run_aot.sh b/wasm-micro-runtime/tests/benchmarks/libsodium/run_aot.sh new file mode 100755 index 0000000..3623deb --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/libsodium/run_aot.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +libsodium_CASES="aead_aes256gcm2 aead_aes256gcm aead_chacha20poly13052 aead_chacha20poly1305 \ + aead_xchacha20poly1305 auth2 auth3 auth5 auth6 auth7 auth box2 box7 box8 \ + box_easy2 box_easy box_seal box_seed box chacha20 codecs core1 core2 core3 \ + core4 core5 core6 core_ed25519 core_ristretto255 ed25519_convert generichash2 \ + generichash3 generichash hash3 hash kdf keygen kx metamorphic misuse \ + onetimeauth2 onetimeauth7 onetimeauth pwhash_argon2id pwhash_argon2i \ + pwhash_scrypt_ll pwhash_scrypt randombytes scalarmult2 scalarmult5 \ + scalarmult6 scalarmult7 scalarmult8 scalarmult_ed25519 scalarmult_ristretto255 \ + scalarmult secretbox2 secretbox7 secretbox8 secretbox_easy2 secretbox_easy \ + secretbox secretstream_xchacha20poly1305 shorthash sign siphashx24 sodium_core \ + sodium_utils2 sodium_utils stream2 stream3 stream4 stream verify1 xchacha20" + +PLATFORM=$(uname -s | tr A-Z a-z) + +readonly OUT_DIR=$PWD/libsodium/zig-out/bin +readonly REPORT=$PWD/report.txt +readonly IWASM_CMD=$PWD/../../../product-mini/platforms/${PLATFORM}/build/iwasm +readonly TIME=/usr/bin/time + +BENCH_NAME_MAX_LEN=20 + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +# run benchmarks +cd $OUT_DIR + +if [[ ${PLATFORM} == "linux" ]]; then + echo -en "\t\t\t\t\t\tnative\tiwasm-aot\tiwasm-aot-segue\n" >> $REPORT +else + echo -en "\t\t\t\t\t\tnative\tiwasm-aot\n" >> $REPORT +fi + +for t in $libsodium_CASES +do + print_bench_name $t + + echo "run $t with native..." + echo -en "\t" >> $REPORT + if [[ $t != "sodium_utils2" ]]; then + ./${t} | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT + else + # sodium_utils2 doesn't print the result, + # use time command to get result instead + $TIME -f "real-%e-time" ./${t} 2>&1 | grep "real-.*-time" | + awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT + fi + + echo "run $t with iwasm aot..." + echo -en "\t \t" >> $REPORT + if [[ $t != "sodium_utils2" ]]; then + $IWASM_CMD ${t}.aot | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT + else + # sodium_utils2 doesn't print the result, + # use time command to get result instead + $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | + awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT + fi + + if [[ ${PLATFORM} == "linux" ]]; then + echo "run $t with iwasm aot segue..." + echo -en "\t \t" >> $REPORT + if [[ $t != "sodium_utils2" ]]; then + $IWASM_CMD ${t}_segue.aot | awk '{printf "%.2f", $0/1000000.0}' >> $REPORT + else + # sodium_utils2 doesn't print the result, + # use time command to get result instead + $TIME -f "real-%e-time" $IWASM_CMD ${t}_segue.aot 2>&1 | grep "real-.*-time" | + awk -F '-' '{printf "%.2f", $2}' >> $REPORT + fi + fi + + echo -en "\n" >> $REPORT +done + diff --git a/wasm-micro-runtime/tests/benchmarks/libsodium/test_pgo.sh b/wasm-micro-runtime/tests/benchmarks/libsodium/test_pgo.sh new file mode 100755 index 0000000..3f9ee2e --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/libsodium/test_pgo.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +libsodium_CASES="aead_aes256gcm2 aead_aes256gcm aead_chacha20poly13052 aead_chacha20poly1305 \ + aead_xchacha20poly1305 auth2 auth3 auth5 auth6 auth7 auth box2 box7 box8 \ + box_easy2 box_easy box_seal box_seed box chacha20 codecs core1 core2 core3 \ + core4 core5 core6 core_ed25519 core_ristretto255 ed25519_convert generichash2 \ + generichash3 generichash hash3 hash kdf keygen kx metamorphic misuse \ + onetimeauth2 onetimeauth7 onetimeauth pwhash_argon2id pwhash_argon2i \ + pwhash_scrypt_ll pwhash_scrypt randombytes scalarmult2 scalarmult5 \ + scalarmult6 scalarmult7 scalarmult8 scalarmult_ed25519 scalarmult_ristretto255 \ + scalarmult secretbox2 secretbox7 secretbox8 secretbox_easy2 secretbox_easy \ + secretbox secretstream_xchacha20poly1305 shorthash sign siphashx24 sodium_core \ + sodium_utils2 sodium_utils stream2 stream3 stream4 stream verify1 xchacha20" + +PLATFORM=$(uname -s | tr A-Z a-z) + +readonly OUT_DIR=$PWD/libsodium/zig-out/bin +readonly REPORT=$PWD/report.txt +if [ "$1" = "--sgx" ] && [ "$PLATFORM" = "linux" ]; then + readonly IWASM_CMD="$PWD/../../../product-mini/platforms/${PLATFORM}-sgx/enclave-sample/iwasm" + readonly WAMRC_CMD="$PWD/../../../wamr-compiler/build/wamrc -sgx" +else + readonly IWASM_CMD="$PWD/../../../product-mini/platforms/${PLATFORM}/build/iwasm" + readonly WAMRC_CMD="$PWD/../../../wamr-compiler/build/wamrc" +fi +readonly TIME=/usr/bin/time + +BENCH_NAME_MAX_LEN=20 + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +pushd $OUT_DIR > /dev/null 2>&1 +for t in $libsodium_CASES +do + if [ ! -e "${t}.wasm" ]; then + echo "${t}.wasm doesn't exist, please run build.sh first" + exit + fi + + echo "" + echo "Compile ${t}.wasm to ${t}.aot .." + ${WAMRC_CMD} -o ${t}.aot ${t}.wasm + + echo "" + echo "Compile ${t}.wasm to ${t}_pgo.aot .." + ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm + + echo "" + echo "Run ${t}_pgo.aot to generate the raw profile data .." + ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot + + echo "" + echo "Merge the raw profile data to ${t}.profdata .." + rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw + + echo "" + echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .." + ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm +done + +# run benchmarks +cd $OUT_DIR + +echo -en "\t\t\t\t\t\tnative\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT + +for t in $libsodium_CASES +do + print_bench_name $t + + echo "run $t with native..." + echo -en "\t" >> $REPORT + if [[ $t != "sodium_utils2" ]]; then + ./${t} | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT + else + # sodium_utils2 doesn't print the result, + # use time command to get result instead + $TIME -f "real-%e-time" ./${t} 2>&1 | grep "real-.*-time" | + awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT + fi + + echo "run $t with iwasm aot..." + echo -en "\t \t" >> $REPORT + if [[ $t != "sodium_utils2" ]]; then + $IWASM_CMD ${t}.aot | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT + else + # sodium_utils2 doesn't print the result, + # use time command to get result instead + $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | + awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT + fi + + echo "run $t with iwasm aot opt..." + echo -en "\t \t" >> $REPORT + if [[ $t != "sodium_utils2" ]]; then + $IWASM_CMD ${t}_opt.aot | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT + else + # sodium_utils2 doesn't print the result, + # use time command to get result instead + $TIME -f "real-%e-time" $IWASM_CMD ${t}_opt.aot 2>&1 | grep "real-.*-time" | + awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT + fi + + echo -en "\n" >> $REPORT +done + diff --git a/wasm-micro-runtime/tests/benchmarks/polybench/README.md b/wasm-micro-runtime/tests/benchmarks/polybench/README.md new file mode 100644 index 0000000..7b6623f --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/polybench/README.md @@ -0,0 +1,27 @@ +# Introduction + +[PolyBench](https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1) is a benchmark suite of 30 numerical computations with static control flow, extracted from operations in various application domains (linear algebra computations, image processing, physics simulation, dynamic programming, statistics, etc.). + +**Source**: https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1 + +# Building + +Please build iwasm and wamrc, refer to: +- [Build iwasm on Linux](../../../doc/build_wamr.md#linux), or [Build iwasm on MacOS](../../../doc/build_wamr.md#macos) +- [Build wamrc AOT compiler](../../../README.md#build-wamrc-aot-compiler) + +And install WASI SDK, please download the [wasi-sdk release](https://github.com/WebAssembly/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. + +And then run `./build.sh` to build the source code, the folder `out` will be created and files will be generated under it. + +# Running + +Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested for each workload, and the file `report.txt` will be generated. + +Run `./run_interp.sh` to test the benchmark, the native mode and iwasm interpreter mode will be tested for each workload, and the file `report.txt` will be generated. + +Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`. + +- For Linux, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled. + +- For Linux-sgx, similarly, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then `make` in the directory `enclave-sample`. And run `./test_pgo.sh --sgx` to test the benchmark. diff --git a/wasm-micro-runtime/tests/benchmarks/polybench/build.sh b/wasm-micro-runtime/tests/benchmarks/polybench/build.sh new file mode 100755 index 0000000..3e59a9b --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/polybench/build.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +OUT_DIR=$PWD/out +WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc +POLYBENCH_CASES="datamining linear-algebra medley stencils" + +if [ ! -d PolyBenchC-4.2.1 ]; then + git clone https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1.git +fi + +mkdir -p ${OUT_DIR} + +cd PolyBenchC-4.2.1 + +for case in $POLYBENCH_CASES +do + files=`find ${case} -name "*.c"` + for file in ${files} + do + file_name=${file##*/} + if [[ ${file_name} == "Nussinov.orig.c" ]]; then + continue + fi + + echo "Build ${file_name%.*}_native" + gcc -O3 -I utilities -I ${file%/*} utilities/polybench.c ${file} \ + -DPOLYBENCH_TIME -lm -o ${OUT_DIR}/${file_name%.*}_native + + echo "Build ${file_name%.*}.wasm" + /opt/wasi-sdk/bin/clang -O3 -I utilities -I ${file%/*} \ + utilities/polybench.c ${file} \ + -Wl,--export=__heap_base -Wl,--export=__data_end \ + -Wl,--export=malloc -Wl,--export=free \ + -DPOLYBENCH_TIME -o ${OUT_DIR}/${file_name%.*}.wasm \ + -D_WASI_EMULATED_PROCESS_CLOCKS + + echo "Compile ${file_name%.*}.wasm into ${file_name%.*}.aot" + ${WAMRC_CMD} -o ${OUT_DIR}/${file_name%.*}.aot \ + ${OUT_DIR}/${file_name%.*}.wasm + + if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile ${file_name%.*}.wasm into ${file_name%.*}_segue.aot" + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/${file_name%.*}_segue.aot \ + ${OUT_DIR}/${file_name%.*}.wasm + fi + done +done + +cd .. + +echo "Done" diff --git a/wasm-micro-runtime/tests/benchmarks/polybench/run_aot.sh b/wasm-micro-runtime/tests/benchmarks/polybench/run_aot.sh new file mode 100755 index 0000000..7eb301b --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/polybench/run_aot.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm + +BENCH_NAME_MAX_LEN=20 + +POLYBENCH_CASES="2mm 3mm adi atax bicg cholesky correlation covariance \ + deriche doitgen durbin fdtd-2d floyd-warshall gemm gemver \ + gesummv gramschmidt heat-3d jacobi-1d jacobi-2d ludcmp lu \ + mvt nussinov seidel-2d symm syr2k syrk trisolv trmm" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +if [[ ${PLATFORM} == "linux" ]]; then + echo -en "\t\t\t\t\t native\tiwasm-aot\tiwasm-aot-segue\n" >> $REPORT +else + echo -en "\t\t\t\t\t native\tiwasm-aot\n" >> $REPORT +fi + +for t in $POLYBENCH_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + if [[ ${PLATFORM} == "linux" ]]; then + echo "run $t with iwasm aot segue .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}_segue.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + fi + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/benchmarks/polybench/run_interp.sh b/wasm-micro-runtime/tests/benchmarks/polybench/run_interp.sh new file mode 100755 index 0000000..5dfe760 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/polybench/run_interp.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm + +BENCH_NAME_MAX_LEN=20 + +POLYBENCH_CASES="2mm 3mm adi atax bicg cholesky correlation covariance \ + deriche doitgen durbin fdtd-2d floyd-warshall gemm gemver \ + gesummv gramschmidt heat-3d jacobi-1d jacobi-2d ludcmp lu \ + mvt nussinov seidel-2d symm syr2k syrk trisolv trmm" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +echo -en "\t\t\t\t\t native\tiwasm-interp\n" >> $REPORT + +for t in $POLYBENCH_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm interp .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}.wasm 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/benchmarks/polybench/test_pgo.sh b/wasm-micro-runtime/tests/benchmarks/polybench/test_pgo.sh new file mode 100755 index 0000000..6451a5b --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/polybench/test_pgo.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +if [ "$1" = "--sgx" ] && [ "$PLATFORM" = "linux" ]; then + IWASM_CMD="$CUR_DIR/../../../product-mini/platforms/${PLATFORM}-sgx/enclave-sample/iwasm" + WAMRC_CMD="$CUR_DIR/../../../wamr-compiler/build/wamrc -sgx" +else + IWASM_CMD="$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm" + WAMRC_CMD="$CUR_DIR/../../../wamr-compiler/build/wamrc" +fi + +BENCH_NAME_MAX_LEN=20 + +POLYBENCH_CASES="2mm 3mm adi atax bicg cholesky correlation covariance \ + deriche doitgen durbin fdtd-2d floyd-warshall gemm gemver \ + gesummv gramschmidt heat-3d jacobi-1d jacobi-2d ludcmp lu \ + mvt nussinov seidel-2d symm syr2k syrk trisolv trmm" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +pushd $OUT_DIR > /dev/null 2>&1 +for t in $POLYBENCH_CASES +do + if [ ! -e "${t}.wasm" ]; then + echo "${t}.wasm doesn't exist, please run build.sh first" + exit + fi + + echo "" + echo "Compile ${t}.wasm to ${t}.aot .." + ${WAMRC_CMD} -o ${t}.aot ${t}.wasm + + echo "" + echo "Compile ${t}.wasm to ${t}_pgo.aot .." + ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm + + echo "" + echo "Run ${t}_pgo.aot to generate the raw profile data .." + ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot + + echo "" + echo "Merge the raw profile data to ${t}.profdata .." + rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw + + echo "" + echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .." + ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm +done +popd > /dev/null 2>&1 + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +echo -en "\t\t\t\t\t native\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT + +for t in $POLYBENCH_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot opt .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}_opt.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/benchmarks/sightglass/README.md b/wasm-micro-runtime/tests/benchmarks/sightglass/README.md new file mode 100644 index 0000000..3f54fda --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/sightglass/README.md @@ -0,0 +1,27 @@ +# Introduction + +[Sightglass](https://github.com/bytecodealliance/sightglass) is a benchmarking suite and tooling to test WebAssembly applications. + +**Source**: https://github.com/bytecodealliance/sightglass + +# Building + +Please build iwasm and wamrc, refer to: +- [Build iwasm on Linux](../../../doc/build_wamr.md#linux), or [Build iwasm on MacOS](../../../doc/build_wamr.md#macos) +- [Build wamrc AOT compiler](../../../README.md#build-wamrc-aot-compiler) + +And install WASI SDK, please download the [wasi-sdk release](https://github.com/WebAssembly/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. + +And then run `./build.sh` to build the source code, the folder `out` will be created and files will be generated under it. + +# Running + +Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested for each workload, and the file `report.txt` will be generated. + +Run `./run_interp.sh` to test the benchmark, the native mode and iwasm interpreter mode will be tested for each workload, and the file `report.txt` will be generated. + +Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`. + +- For Linux, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled. + +- For Linux-sgx, similarly, build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`, then `make` in the directory `enclave-sample`. And run `./test_pgo.sh --sgx` to test the benchmark. diff --git a/wasm-micro-runtime/tests/benchmarks/sightglass/build.sh b/wasm-micro-runtime/tests/benchmarks/sightglass/build.sh new file mode 100755 index 0000000..5408822 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/sightglass/build.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +PLATFORM=$(uname -s | tr A-Z a-z) + +OUT_DIR=$PWD/out +WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc +SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \ + nestedloop2 nestedloop3 random seqhash sieve strchr \ + switch2" + +if [ ! -d sightglass ]; then + git clone https://github.com/wasm-micro-runtime/sightglass.git +fi + +mkdir -p ${OUT_DIR} + +cd sightglass/benchmarks/shootout + +for bench in $SHOOTOUT_CASES +do + echo "Build ${bench}_native" + gcc -O3 -o ${OUT_DIR}/${bench}_native -Dblack_box=set_res -Dbench=${bench} \ + -I../../include ${bench}.c main/main_${bench}.c main/my_libc.c + + echo "Build ${bench}.wasm" + /opt/wasi-sdk/bin/clang -O3 -nostdlib \ + -Wno-unknown-attributes \ + -Dblack_box=set_res \ + -I../../include -DNOSTDLIB_MODE \ + -Wl,--initial-memory=1310720,--allow-undefined \ + -Wl,--strip-all,--no-entry \ + -o ${OUT_DIR}/${bench}.wasm \ + -Wl,--export=app_main -Wl,--export=_start \ + ${bench}.c main/main_${bench}.c main/my_libc.c + + echo "Compile ${bench}.wasm into ${bench}.aot" + ${WAMRC_CMD} -o ${OUT_DIR}/${bench}.aot ${OUT_DIR}/${bench}.wasm + if [[ ${PLATFORM} == "linux" ]]; then + echo "Compile ${bench}.wasm into ${bench}_segue.aot" + ${WAMRC_CMD} --enable-segue -o ${OUT_DIR}/${bench}_segue.aot ${OUT_DIR}/${bench}.wasm + fi +done + +cd .. + +echo "Done" diff --git a/wasm-micro-runtime/tests/benchmarks/sightglass/run_aot.sh b/wasm-micro-runtime/tests/benchmarks/sightglass/run_aot.sh new file mode 100755 index 0000000..44945b9 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/sightglass/run_aot.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm + +BENCH_NAME_MAX_LEN=20 + +SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \ + nestedloop2 nestedloop3 random seqhash sieve strchr \ + switch2" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +if [[ ${PLATFORM} == "linux" ]]; then + echo -en "\t\t\t\t\t native\tiwasm-aot\tiwasm-aot-segue\n" >> $REPORT +else + echo -en "\t\t\t\t\t native\tiwasm-aot\n" >> $REPORT +fi + +for t in $SHOOTOUT_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + if [[ ${PLATFORM} == "linux" ]]; then + echo "run $t with iwasm aot segue .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}_segue.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + fi + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/benchmarks/sightglass/run_interp.sh b/wasm-micro-runtime/tests/benchmarks/sightglass/run_interp.sh new file mode 100755 index 0000000..c3dbb30 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/sightglass/run_interp.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm + +BENCH_NAME_MAX_LEN=20 + +SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \ + nestedloop2 nestedloop3 random seqhash sieve strchr \ + switch2" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +echo -en "\t\t\t\t\t native\tiwasm-interp\n" >> $REPORT + +for t in $SHOOTOUT_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm interp .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}.wasm 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/benchmarks/sightglass/test_pgo.sh b/wasm-micro-runtime/tests/benchmarks/sightglass/test_pgo.sh new file mode 100755 index 0000000..4baedf9 --- /dev/null +++ b/wasm-micro-runtime/tests/benchmarks/sightglass/test_pgo.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +CUR_DIR=$PWD +OUT_DIR=$CUR_DIR/out +REPORT=$CUR_DIR/report.txt +TIME=/usr/bin/time + +PLATFORM=$(uname -s | tr A-Z a-z) +if [ "$1" = "--sgx" ] && [ "$PLATFORM" = "linux" ]; then + IWASM_CMD="$CUR_DIR/../../../product-mini/platforms/${PLATFORM}-sgx/enclave-sample/iwasm" + WAMRC_CMD="$CUR_DIR/../../../wamr-compiler/build/wamrc -sgx" +else + IWASM_CMD="$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm" + WAMRC_CMD="$CUR_DIR/../../../wamr-compiler/build/wamrc" +fi + +BENCH_NAME_MAX_LEN=20 + +SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \ + nestedloop2 nestedloop3 random seqhash sieve strchr \ + switch2" + +rm -f $REPORT +touch $REPORT + +function print_bench_name() +{ + name=$1 + echo -en "$name" >> $REPORT + name_len=${#name} + if [ $name_len -lt $BENCH_NAME_MAX_LEN ] + then + spaces=$(( $BENCH_NAME_MAX_LEN - $name_len )) + for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done + fi +} + +pushd $OUT_DIR > /dev/null 2>&1 +for t in $SHOOTOUT_CASES +do + if [ ! -e "${t}.wasm" ]; then + echo "${t}.wasm doesn't exist, please run build.sh first" + exit + fi + + echo "" + echo "Compile ${t}.wasm to ${t}.aot .." + ${WAMRC_CMD} -o ${t}.aot ${t}.wasm + + echo "" + echo "Compile ${t}.wasm to ${t}_pgo.aot .." + ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm + + echo "" + echo "Run ${t}_pgo.aot to generate the raw profile data .." + ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot + + echo "" + echo "Merge the raw profile data to ${t}.profdata .." + rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw + + echo "" + echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .." + ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm +done +popd > /dev/null 2>&1 + +echo "Start to run cases, the result is written to report.txt" + +#run benchmarks +cd $OUT_DIR +echo -en "\t\t\t\t\t native\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT + +for t in $SHOOTOUT_CASES +do + print_bench_name $t + + echo "run $t with native .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo "run $t with iwasm aot opt .." + echo -en "\t" >> $REPORT + $TIME -f "real-%e-time" $IWASM_CMD ${t}_opt.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT + + echo -en "\n" >> $REPORT +done diff --git a/wasm-micro-runtime/tests/unit/CMakeLists.txt b/wasm-micro-runtime/tests/unit/CMakeLists.txt new file mode 100644 index 0000000..a9a7b87 --- /dev/null +++ b/wasm-micro-runtime/tests/unit/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project (wamr_unit_tests) + +include (CTest) + +if (NOT DEFINED WAMR_BUILD_INTERP) + # Enable Interpreter by default + set (WAMR_BUILD_INTERP 1) +endif () + +if (NOT DEFINED WAMR_BUILD_PLATFORM) + string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +endif () + +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library (vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +include (FetchContent) +FetchContent_Declare ( + googletest + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip +) +# For Windows: Prevent overriding the parent project's compiler/linker settings +set (gtest_force_shared_crt ON CACHE BOOL "" FORCE) +FetchContent_MakeAvailable (googletest) + +include (GoogleTest) + +add_library (wamr_gtest_main main.cpp) +target_link_libraries (wamr_gtest_main PUBLIC gtest vmlib) + +function (create_wamr_unit_test test_name) + set (sources ${ARGN}) + add_executable (${test_name} ${sources}) + target_link_libraries ( + ${test_name} + wamr_gtest_main + vmlib + ${LLVM_AVAILABLE_LIBS} + ) + gtest_discover_tests (${test_name}) + endfunction () + +if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) + include (${IWASM_DIR}/libraries/lib-wasi-threads/unit-test/lib_wasi_threads_unit_tests.cmake) +endif () diff --git a/wasm-micro-runtime/tests/unit/main.cpp b/wasm-micro-runtime/tests/unit/main.cpp new file mode 100644 index 0000000..63695d9 --- /dev/null +++ b/wasm-micro-runtime/tests/unit/main.cpp @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include +#include "wasm_runtime_common.h" + +int +main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + + if (!wasm_runtime_init()) { + return -1; + } + + int ret = RUN_ALL_TESTS(); + wasm_runtime_destroy(); + + return ret; +} \ No newline at end of file diff --git a/wasm-micro-runtime/tests/wamr-compiler/.gitignore b/wasm-micro-runtime/tests/wamr-compiler/.gitignore new file mode 100644 index 0000000..5425c41 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-compiler/.gitignore @@ -0,0 +1,2 @@ +*.aot +*.wasm \ No newline at end of file diff --git a/wasm-micro-runtime/tests/wamr-compiler/README.md b/wasm-micro-runtime/tests/wamr-compiler/README.md new file mode 100644 index 0000000..d48ea98 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-compiler/README.md @@ -0,0 +1,3 @@ +# WAMR test benchmarks + +This folder contains tests for WAMR AOT compiler and its generated code. diff --git a/wasm-micro-runtime/tests/wamr-compiler/test_shift_negative_constants.wat b/wasm-micro-runtime/tests/wamr-compiler/test_shift_negative_constants.wat new file mode 100644 index 0000000..83bc3c4 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-compiler/test_shift_negative_constants.wat @@ -0,0 +1,78 @@ +;; Copyright (C) 2023 Amazon Inc. All rights reserved. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +;; +;; Those tests verify if passing constant negative value +;; as a right parameter of the shift operator (along +;; with a constant value of the left operator) causes +;; any problems. See: https://github.com/bytecodealliance/wasm-micro-runtime/pull/2619 +(module + (memory (export "memory") 1 1) + + (func $assert_eq (param i32 i32) + (i32.ne (local.get 0) (local.get 1)) + if + unreachable + end + ) + + (func $i32_shr_u + (call $assert_eq + (i32.shr_u (i32.const -1) (i32.const -5)) + (i32.const 31) + ) + ) + + (func $i32_shr_s + (call $assert_eq + (i32.shr_s (i32.const 32) (i32.const -30)) + (i32.const 8) + ) + ) + + (func $i32_shl + (call $assert_eq + (i32.shl (i32.const -1) (i32.const -30)) + (i32.const -4) + ) + ) + + (func $const_ret (result i32) + i32.const -5 + ) + + ;; *_func_call tests validate the potential LLVM optimizations + ;; where the right parameter of the shift operation is an + ;; indirect constant value. + (func $i32_shr_u_func_call + (call $assert_eq + (i32.shr_u (i32.const -1) (call $const_ret)) + (i32.const 31) + ) + ) + + (func $i32_shr_s_func_call + (call $assert_eq + (i32.shr_s + (i32.const 1073741824) ;; 2^30 + (call $const_ret) + ) + (i32.const 8) + ) + ) + + (func $i32_shl_func_call + (call $assert_eq + (i32.shl (i32.const -1) (call $const_ret)) + (i32.const -134217728) + ) + ) + + (func (export "_start") + call $i32_shr_u + call $i32_shr_s + call $i32_shl + call $i32_shr_u_func_call + call $i32_shr_s_func_call + call $i32_shl_func_call + ) +) diff --git a/wasm-micro-runtime/tests/wamr-test-suites/README.md b/wasm-micro-runtime/tests/wamr-test-suites/README.md new file mode 100644 index 0000000..e33649c --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/README.md @@ -0,0 +1,40 @@ +# WAMR test suites + +This folder contains test scripts and cases for wamr. + +## Help +``` +./test_wamr.sh --help +``` + +## Examples +Test spec cases with fast interpreter mode, which will create folder `workspace`, download the `spec` and `wabt` repo, and build `iwasm` automatically to test spec cases: +``` +./test_wamr.sh -s spec -t fast-interp +``` + +Test spec cases with aot mode, and use the wabt binary release package instead of compiling wabt from the source code: +``` +./test_wamr.sh -s spec -t aot -b +``` + +Test spec cases with all modes (classic-interp/fast-interp/aot/jit): +``` +./test_wamr.sh -s spec +``` + +Test spec cases with aot mode and pthread enabled: +``` +./test_wamr.sh -s spec -t aot -p +``` + +Test spec cases with aot mode and SIMD enabled: +``` +./test_wamr.sh -s spec -t aot -S +``` + +Test spec cases with fast-interp on target x86_32: +``` +./test_wamr.sh -s spec -t fast-interp -m x86_32 +``` + diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/CHANGES b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/CHANGES new file mode 100644 index 0000000..0b0cc9a --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/CHANGES @@ -0,0 +1,5 @@ +#################### +2021-9-8 + +Modify runtest.py from https://github.com/kanaka/wac/blob/master/runtest.py +to enable testing spec cases with more checks and support more runtime modes. diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/all.py b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/all.py new file mode 100644 index 0000000..13db472 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/all.py @@ -0,0 +1,625 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import argparse +import multiprocessing as mp +import platform +import pathlib +import subprocess +import sys +import time + +""" +The script itself has to be put under the same directory with the "spec". +To run a single non-GC and non-memory64 case with interpreter mode: + cd workspace + python3 runtest.py --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \ + spec/test/core/xxx.wast +To run a single non-GC case with aot mode: + cd workspace + python3 runtest.py --aot --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \ + --aot-compiler wamrc spec/test/core/xxx.wast +To run a single GC case or single memory64 case: + cd workspace + python3 runtest.py --wast2wasm spec/interpreter/wasm --interpreter iwasm \ + --aot-compiler wamrc --gc spec/test/core/xxx.wast +""" + +def exe_file_path(base_path: str) -> str: + if platform.system().lower() == "windows": + base_path += ".exe" + return base_path + +def get_iwasm_cmd(platform: str) -> str: + build_path = "../../../product-mini/platforms/" + platform + "/build/" + exe_name = "iwasm" + + if platform == "windows": + build_path += "RelWithDebInfo/" + + return exe_file_path(build_path + exe_name) + +PLATFORM_NAME = platform.uname().system.lower() +IWASM_CMD = get_iwasm_cmd(PLATFORM_NAME) +IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm" +IWASM_QEMU_CMD = "iwasm" +SPEC_TEST_DIR = "spec/test/core" +EXCE_HANDLING_DIR = "exception-handling/test/core" +WAST2WASM_CMD = exe_file_path("./wabt/out/gcc/Release/wat2wasm") +SPEC_INTERPRETER_CMD = "spec/interpreter/wasm" +WAMRC_CMD = "../../../wamr-compiler/build/wamrc" +AVAILABLE_TARGETS = [ + "I386", + "X86_32", + "X86_64", + "AARCH64", + "AARCH64_VFP", + "ARMV7", + "ARMV7_VFP", + "RISCV32", + "RISCV32_ILP32F", + "RISCV32_ILP32D", + "RISCV64", + "RISCV64_LP64F", + "RISCV64_LP64D", + "THUMBV7", + "THUMBV7_VFP", +] + +def ignore_the_case( + case_name, + target, + aot_flag=False, + sgx_flag=False, + multi_module_flag=False, + multi_thread_flag=False, + simd_flag=False, + gc_flag=False, + memory64_flag=False, + xip_flag=False, + eh_flag=False, + qemu_flag=False, +): + + if case_name in ["comments", "inline-module", "names"]: + return True + + if not multi_module_flag and case_name in ["imports", "linking"]: + return True + + # Note: x87 doesn't preserve sNaN and makes some relevant tests fail. + if "i386" == target and case_name in ["float_exprs", "conversions"]: + return True + + if gc_flag: + if case_name in ["array_init_elem", "array_init_data"]: + return True + + if sgx_flag: + if case_name in ["conversions", "f32_bitwise", "f64_bitwise"]: + return True + + if aot_flag and case_name in [ + "call_indirect", + "call", + "fac", + "skip-stack-guard-page", + ]: + return True + + if qemu_flag: + if case_name in [ + "f32_bitwise", + "f64_bitwise", + "loop", + "f64", + "f64_cmp", + "conversions", + "f32", + "f32_cmp", + "float_exprs", + "float_misc", + "select", + "memory_grow", + ]: + return True + + return False + + +def preflight_check(aot_flag, eh_flag): + if not pathlib.Path(SPEC_TEST_DIR).resolve().exists(): + print(f"Can not find {SPEC_TEST_DIR}") + return False + + if not pathlib.Path(WAST2WASM_CMD).resolve().exists(): + print(f"Can not find {WAST2WASM_CMD}") + return False + + if aot_flag and not pathlib.Path(WAMRC_CMD).resolve().exists(): + print(f"Can not find {WAMRC_CMD}") + return False + + if eh_flag and not pathlib.Path(EXCE_HANDLING_DIR).resolve().exists(): + print(f"Can not find {EXCE_HANDLING_DIR}") + return False + + return True + + +def test_case( + case_path, + target, + aot_flag=False, + sgx_flag=False, + multi_module_flag=False, + multi_thread_flag=False, + simd_flag=False, + xip_flag=False, + eh_flag=False, + clean_up_flag=True, + verbose_flag=True, + gc_flag=False, + memory64_flag=False, + qemu_flag=False, + qemu_firmware="", + log="", + no_pty=False +): + CMD = [sys.executable, "runtest.py"] + CMD.append("--wast2wasm") + CMD.append(WAST2WASM_CMD if not gc_flag and not memory64_flag else SPEC_INTERPRETER_CMD) + CMD.append("--interpreter") + if sgx_flag: + CMD.append(IWASM_SGX_CMD) + elif qemu_flag: + CMD.append(IWASM_QEMU_CMD) + else: + CMD.append(IWASM_CMD) + if no_pty: + CMD.append("--no-pty") + CMD.append("--aot-compiler") + CMD.append(WAMRC_CMD) + + if aot_flag: + CMD.append("--aot") + + CMD.append("--target") + CMD.append(target) + + if multi_module_flag: + CMD.append("--multi-module") + + if multi_thread_flag: + CMD.append("--multi-thread") + + if sgx_flag: + CMD.append("--sgx") + + if simd_flag: + CMD.append("--simd") + + if xip_flag: + CMD.append("--xip") + + if eh_flag: + CMD.append("--eh") + + if qemu_flag: + CMD.append("--qemu") + CMD.append("--qemu-firmware") + CMD.append(qemu_firmware) + + if not clean_up_flag: + CMD.append("--no_cleanup") + + if gc_flag: + CMD.append("--gc") + + if memory64_flag: + CMD.append("--memory64") + + if log != "": + CMD.append("--log-dir") + CMD.append(log) + + case_path = pathlib.Path(case_path).resolve() + case_name = case_path.stem + + CMD.append(str(case_path)) + # print(f"============> use {' '.join(CMD)}") + print(f"============> run {case_name} ", end="") + with subprocess.Popen( + CMD, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) as p: + try: + case_last_words = [] + while not p.poll(): + output = p.stdout.readline() + + if not output: + break + + if verbose_flag: + print(output, end="") + else: + if len(case_last_words) == 16: + case_last_words.pop(0) + case_last_words.append(output) + + p.wait(60) + + if p.returncode: + print(f"failed with a non-zero return code {p.returncode}") + if not verbose_flag: + print( + f"\n==================== LAST LOG of {case_name} ====================\n" + ) + print("".join(case_last_words)) + print("\n==================== LAST LOG END ====================\n") + raise Exception(case_name) + else: + print("successful") + return True + except subprocess.CalledProcessError: + print("failed with CalledProcessError") + raise Exception(case_name) + except subprocess.TimeoutExpired: + print("failed with TimeoutExpired") + raise Exception(case_name) + + +def test_suite( + target, + aot_flag=False, + sgx_flag=False, + multi_module_flag=False, + multi_thread_flag=False, + simd_flag=False, + xip_flag=False, + eh_flag=False, + clean_up_flag=True, + verbose_flag=True, + gc_flag=False, + memory64_flag=False, + parl_flag=False, + qemu_flag=False, + qemu_firmware="", + log="", + no_pty=False, +): + suite_path = pathlib.Path(SPEC_TEST_DIR).resolve() + if not suite_path.exists(): + print(f"can not find spec test cases at {suite_path}") + return False + + case_list = sorted(suite_path.glob("*.wast")) + if simd_flag: + simd_case_list = sorted(suite_path.glob("simd/*.wast")) + case_list.extend(simd_case_list) + + if gc_flag: + gc_case_list = sorted(suite_path.glob("gc/*.wast")) + case_list.extend(gc_case_list) + + if eh_flag: + eh_path = pathlib.Path(EXCE_HANDLING_DIR).resolve() + if not eh_path.exists(): + print(f"can not find spec test cases at {eh_path}") + return False + eh_case_list = sorted(eh_path.glob("*.wast")) + eh_case_list_include = [test for test in eh_case_list if test.stem in ["throw", "tag", "try_catch", "rethrow", "try_delegate"]] + case_list.extend(eh_case_list_include) + + # ignore based on command line options + filtered_case_list = [] + for case_path in case_list: + case_name = case_path.stem + if not ignore_the_case( + case_name, + target, + aot_flag, + sgx_flag, + multi_module_flag, + multi_thread_flag, + simd_flag, + gc_flag, + memory64_flag, + xip_flag, + eh_flag, + qemu_flag, + ): + filtered_case_list.append(case_path) + print(f"---> {len(case_list)} --filter--> {len(filtered_case_list)}") + case_list = filtered_case_list + + case_count = len(case_list) + failed_case = 0 + successful_case = 0 + + if parl_flag: + print(f"----- Run the whole spec test suite on {mp.cpu_count()} cores -----") + with mp.Pool() as pool: + results = {} + for case_path in case_list: + results[case_path.stem] = pool.apply_async( + test_case, + [ + str(case_path), + target, + aot_flag, + sgx_flag, + multi_module_flag, + multi_thread_flag, + simd_flag, + xip_flag, + eh_flag, + clean_up_flag, + verbose_flag, + gc_flag, + memory64_flag, + qemu_flag, + qemu_firmware, + log, + no_pty, + ], + ) + + for case_name, result in results.items(): + try: + if qemu_flag: + # 60 min / case, testing on QEMU may be very slow + result.wait(7200) + else: + # 5 min / case + result.wait(300) + if not result.successful(): + failed_case += 1 + else: + successful_case += 1 + except mp.TimeoutError: + print(f"{case_name} meets TimeoutError") + failed_case += 1 + else: + print(f"----- Run the whole spec test suite -----") + for case_path in case_list: + print(case_path) + try: + test_case( + str(case_path), + target, + aot_flag, + sgx_flag, + multi_module_flag, + multi_thread_flag, + simd_flag, + xip_flag, + eh_flag, + clean_up_flag, + verbose_flag, + gc_flag, + memory64_flag, + qemu_flag, + qemu_firmware, + log, + no_pty, + ) + successful_case += 1 + except Exception as e: + failed_case += 1 + raise e + + print( + f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP" + ) + + return 0 == failed_case + + +def main(): + parser = argparse.ArgumentParser(description="run the whole spec test suite") + + parser.add_argument( + "-M", + action="store_true", + default=False, + dest="multi_module_flag", + help="Running with the Multi-Module feature", + ) + parser.add_argument( + "-m", + choices=AVAILABLE_TARGETS, + type=str, + dest="target", + default="X86_64", + help="Specify Target ", + ) + parser.add_argument( + "-p", + action="store_true", + default=False, + dest="multi_thread_flag", + help="Running with the Multi-Thread feature", + ) + parser.add_argument( + "-S", + action="store_true", + default=False, + dest="simd_flag", + help="Running with the SIMD feature", + ) + parser.add_argument( + "-X", + action="store_true", + default=False, + dest="xip_flag", + help="Running with the XIP feature", + ) + # added to support WASM_ENABLE_EXCE_HANDLING + parser.add_argument( + "-e", + action="store_true", + default=False, + dest="eh_flag", + help="Running with the exception-handling feature", + ) + parser.add_argument( + "-t", + action="store_true", + default=False, + dest="aot_flag", + help="Running with AOT mode", + ) + parser.add_argument( + "-x", + action="store_true", + default=False, + dest="sgx_flag", + help="Running with SGX environment", + ) + parser.add_argument( + "--no_clean_up", + action="store_false", + default=True, + dest="clean_up_flag", + help="Does not remove tmpfiles. But it will be enabled while running parallelly", + ) + parser.add_argument( + "--parl", + action="store_true", + default=False, + dest="parl_flag", + help="To run whole test suite parallelly", + ) + parser.add_argument( + "--qemu", + action="store_true", + default=False, + dest="qemu_flag", + help="To run whole test suite in qemu", + ) + parser.add_argument( + "--qemu-firmware", + default="", + dest="qemu_firmware", + help="Firmware required by qemu", + ) + parser.add_argument( + "--log", + default="", + dest="log", + help="Log directory", + ) + parser.add_argument( + "--quiet", + action="store_false", + default=True, + dest="verbose_flag", + help="Close real time output while running cases, only show last words of failed ones", + ) + parser.add_argument( + "--gc", + action="store_true", + default=False, + dest="gc_flag", + help="Running with GC feature", + ) + parser.add_argument( + "--memory64", + action="store_true", + default=False, + dest="memory64_flag", + help="Running with memory64 feature", + ) + parser.add_argument( + "cases", + metavar="path_to__case", + type=str, + nargs="*", + help=f"Specify all wanted cases. If not the script will go through all cases under {SPEC_TEST_DIR}", + ) + parser.add_argument('--no-pty', action='store_true', + help="Use direct pipes instead of pseudo-tty") + + options = parser.parse_args() + + # Convert target to lower case for internal use, e.g. X86_64 -> x86_64 + # target is always exist, so no need to check it + options.target = options.target.lower() + + if options.target == "x86_32": + options.target = "i386" + + if not preflight_check(options.aot_flag, options.eh_flag): + return False + + if not options.cases: + if options.parl_flag: + # several cases might share the same workspace/tempfile at the same time + # so, disable it while running parallelly + options.clean_up_flag = False + options.verbose_flag = False + + start = time.time_ns() + ret = test_suite( + options.target, + options.aot_flag, + options.sgx_flag, + options.multi_module_flag, + options.multi_thread_flag, + options.simd_flag, + options.xip_flag, + options.eh_flag, + options.clean_up_flag, + options.verbose_flag, + options.gc_flag, + options.memory64_flag, + options.parl_flag, + options.qemu_flag, + options.qemu_firmware, + options.log, + options.no_pty + ) + end = time.time_ns() + print( + f"It takes {((end - start) / 1000000):,} ms to run test_suite {'parallelly' if options.parl_flag else ''}" + ) + else: + try: + for case in options.cases: + test_case( + case, + options.target, + options.aot_flag, + options.sgx_flag, + options.multi_module_flag, + options.multi_thread_flag, + options.simd_flag, + options.xip_flag, + options.eh_flag, + options.clean_up_flag, + options.verbose_flag, + options.gc_flag, + options.memory64_flag, + options.qemu_flag, + options.qemu_firmware, + options.log, + options.no_pty, + ) + else: + ret = True + except Exception: + ret = False + + return ret + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/all.sh b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/all.sh new file mode 100755 index 0000000..09d8681 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/all.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +# exit if meet an exception +function DEBUG() { + [[ -n $(env | grep "\") ]] && $@ +} +DEBUG set -xevu + +# Run the following command to test a single wast file: +# ./spec-test-script/runtest.py --wast2wasm ./workspace/wabt/out/gcc/Release/wat2wasm \ +# --interpreter iwasm + +readonly SPEC_TEST_DIR="spec/test/core" +readonly WAST2WASM_CMD="./wabt/out/gcc/Release/wat2wasm" +readonly WAMRC_CMD="../../../wamr-compiler/build/wamrc" +PLATFORM=$(uname -s | tr A-Z a-z) +IWASM_CMD="../../../product-mini/platforms/${PLATFORM}/build/iwasm" + +# "imports" and "linking" are only avilable when enabling multi modules +# "comments" is for runtest.py + +IGNORE_LIST=( + "comments" "inline-module" "imports" "linking" "names" +) + +readonly -a MULTI_MODULE_LIST=( + "imports" "linking" +) + +SGX_IGNORE_LIST=("conversions" "f32_bitwise" "f64_bitwise") + +# these cases run failed due to native stack overflow check failed +SGX_AOT_IGNORE_LIST=("call_indirect" "call" "fac" "skip-stack-guard-page") + +function usage() { + echo "Usage: all.sh [-t] [-m ] [-M] [-x] [-S] [-r]" + exit 1 +} + +function run_case_w_aot() { + local test_case=$1 + echo "============> run ${test_case} with AOT" + python2.7 runtest.py \ + --wast2wasm ${WAST2WASM_CMD} \ + --interpreter ${IWASM_CMD} \ + ${SPEC_TEST_DIR}/${test_case} \ + --aot-compiler ${WAMRC_CMD} \ + --aot --aot-target ${TARGET} \ + ${SGX_OPT} \ + ${SIMD_OPT} \ + ${REF_TYPES_OPT} + #--no_cleanup + if [[ $? != 0 ]]; then + echo "============> run ${test_case} failed" + exit 1 + fi +} + +function run_case_wo_aot() { + local test_case=$1 + echo "============> run ${test_case}" + python2.7 runtest.py \ + --wast2wasm ${WAST2WASM_CMD} \ + --interpreter ${IWASM_CMD} \ + ${SPEC_TEST_DIR}/${test_case} \ + --aot-compiler ${WAMRC_CMD} \ + ${SGX_OPT} \ + ${SIMD_OPT} \ + ${REF_TYPES_OPT} + #--no_cleanup + if [[ $? != 0 ]]; then + echo "============> run ${test_case} failed" + exit 1 + fi +} + +ENABLE_MULTI_MODULE=0 +TARGET="X86_64" +SGX_OPT="" +AOT=false +SIMD_OPT="" +REF_TYPES_OPT="" +while getopts ":Mm:txSr" opt; do + case $opt in + t) AOT=true ;; + m) + TARGET=$OPTARG + if [[ ${TARGET} == 'X86_32' ]]; then + TARGET='i386' + elif [[ ${TARGET} == 'X86_64' ]]; then + TARGET='x86_64' + elif [[ ${TARGET} == 'ARMV7_VFP' ]]; then + TARGET='armv7' + elif [[ ${TARGET} == 'THUMBV7_VFP' ]]; then + TARGET='thumbv7' + elif [[ ${TARGET} == 'RISCV64' || ${TARGET} == 'RISCV64_LP64D' ]]; then + TARGET='riscv64_lp64d' + elif [[ ${TARGET} == 'RISCV64_LP64' ]]; then + TARGET='riscv64_lp64' + else + usage + fi ;; + M) ENABLE_MULTI_MODULE=1 ;; + x) SGX_OPT="--sgx" ;; + S) SIMD_OPT="--simd" ;; + r) REF_TYPES_OPT="--ref_types" ;; + *) usage ;; + esac +done + +function contain() { + # [$1, $-1) + local list=${@:0:${#}} + # [$-1] + local item=${@:${#}} + [[ ${list} =~ (^| )${item}($| ) ]] && return 0 || return 1 +} + +if [[ ${SGX_OPT} ]]; then + IWASM_CMD="../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm" + IGNORE_LIST+=("${SGX_IGNORE_LIST[@]}") + if [[ "true" == ${AOT} ]]; then + IGNORE_LIST+=("${SGX_AOT_IGNORE_LIST[@]}") + fi +fi + +if [[ ${TARGET} == "i386" ]]; then + IGNORE_LIST+=("float_exprs") +fi + +declare -i COUNTER=0 +for wast in $(find ${SPEC_TEST_DIR} -name "*.wast" -type f | sort -n); do + # remove a prefix spec/test/core/ + wast=${wast#${SPEC_TEST_DIR}/} + # ${wast%.wast} will remove a surfix .wast + if contain "${IGNORE_LIST[@]}" ${wast%.wast}; then + echo "============> ignore ${wast}" + continue + else + [[ "true" == ${AOT} ]] && run_case_w_aot ${wast} || + run_case_wo_aot ${wast} + ((COUNTER += 1)) + fi +done + +# for now, Multi_Module is always disabled while AOT is true +if [[ "false" == ${AOT} && 1 == ${ENABLE_MULTI_MODULE} ]]; then + echo "============> run cases about multi module" + for wast in ${MULTI_MODULE_LIST[@]}; do + run_case_wo_aot ${wast}.wast + ((COUNTER += 1)) + done +fi + +echo "PASS ALL ${COUNTER} SPEC CASES" +DEBUG set -xevu +exit 0 diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/collect_coverage.sh b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/collect_coverage.sh new file mode 100755 index 0000000..0091e76 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/collect_coverage.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +readonly WORK_DIR=$PWD +readonly WAMR_DIR=${WORK_DIR}/../../.. +readonly DST_COV_FILE=$1 +readonly SRC_COV_DIR=$2 +readonly SRC_TEMP_COV_FILE=wamr_temp.lcov +readonly SRC_COV_FILE=wamr.lcov + +# get dest folder +dir=$(dirname ${DST_COV_FILE}) +pushd ${dir} > /dev/null 2>&1 +readonly DST_COV_DIR=${PWD} +popd > /dev/null 2>&1 + +if [[ ! -d ${SRC_COV_DIR} ]]; then + echo "${SRC_COV_DIR} doesn't exist, ignore code coverage collection" + exit +fi + +echo "Start to collect code coverage of ${SRC_COV_DIR} .." + +pushd ${SRC_COV_DIR} > /dev/null 2>&1 + +# collect all code coverage data +lcov -q -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1 +# extract code coverage data of WAMR source files +lcov -q -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \ + -rc lcov_branch_coverage=1 \ + "*/usr/*" "*/_deps/*" "*/deps/*" "*/tests/unit/*" \ + "*/llvm/include/*" "*/include/llvm/*" "*/samples/*" \ + "*/test-tools/*" "*/tests/standalone/*" "*/tests/*" + +if [[ -s ${SRC_TEMP_COV_FILE} ]]; then + if [[ -s ${DST_COV_FILE} ]]; then + # merge code coverage data + lcov --rc lcov_branch_coverage=1 \ + --add-tracefile ${SRC_TEMP_COV_FILE} \ + -a ${DST_COV_FILE} -o ${SRC_COV_FILE} + # backup the original lcov file + cp -a ${DST_COV_FILE} "${DST_COV_FILE}.orig" + # replace the lcov file + cp -a ${SRC_COV_FILE} ${DST_COV_FILE} + echo "Code coverage file ${DST_COV_FILE} was appended" + else + cp -a ${SRC_TEMP_COV_FILE} ${SRC_COV_FILE} + cp -a ${SRC_COV_FILE} ${DST_COV_FILE} + echo "Code coverage file ${DST_COV_FILE} was generated" + fi + + # get ignored prefix path + dir=$(dirname ${WAMR_DIR}/../..) + pushd ${dir} > /dev/null 2>&1 + prefix_full_path=${PWD} + popd > /dev/null 2>&1 + + # generate html output for merged code coverage data + rm -fr ${DST_COV_DIR}/wamr-lcov + genhtml -q -t "WAMR Code Coverage" \ + --rc lcov_branch_coverage=1 --prefix=${prefix_full_path} \ + -o ${DST_COV_DIR}/wamr-lcov \ + ${DST_COV_FILE} + + cd ${DST_COV_DIR} + rm -f wamr-lcov.zip + zip -r -q -o wamr-lcov.zip wamr-lcov + rm -fr wamr-lcov + + echo "Code coverage html ${DST_COV_DIR}/wamr-lcov.zip was generated" +else + echo "generate code coverage html failed" +fi + +echo "" + +popd > /dev/null 2>&1 diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/exception_handling.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/exception_handling.patch new file mode 100644 index 0000000..0c9e8d4 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/exception_handling.patch @@ -0,0 +1,20 @@ +diff --git a/test/core/try_catch.wast b/test/core/try_catch.wast +index 2a0e9ff6..f243489d 100644 +--- a/test/core/try_catch.wast ++++ b/test/core/try_catch.wast +@@ -203,7 +203,6 @@ + + (assert_return (invoke "catch-param-i32" (i32.const 5)) (i32.const 5)) + +-(assert_return (invoke "catch-imported") (i32.const 2)) + + (assert_return (invoke "catchless-try" (i32.const 0)) (i32.const 0)) + (assert_return (invoke "catchless-try" (i32.const 1)) (i32.const 1)) +@@ -231,7 +230,6 @@ + ) + ) + +-(assert_return (invoke "imported-mismatch") (i32.const 3)) + + (assert_malformed + (module quote "(module (func (catch_all)))") diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/gc_ignore_cases.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/gc_ignore_cases.patch new file mode 100644 index 0000000..bc91d6b --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/gc_ignore_cases.patch @@ -0,0 +1,1007 @@ +diff --git a/test/core/binary-leb128.wast b/test/core/binary-leb128.wast +index 335496f0..5b975028 100644 +--- a/test/core/binary-leb128.wast ++++ b/test/core/binary-leb128.wast +@@ -1078,5 +1078,5 @@ + "\e0\7f" ;; Malformed functype, -0x20 in signed LEB128 encoding + "\00\00" + ) +- "integer representation too long" ++ "invalid type flag" ;; In GC extension, the first byte in rectype define is just one byte, not LEB128 encoded. + ) +diff --git a/test/core/elem.wast b/test/core/elem.wast +index df1610f6..32c1d8b3 100644 +--- a/test/core/elem.wast ++++ b/test/core/elem.wast +@@ -400,7 +400,7 @@ + ) + + (assert_invalid +- (module ++ (module + (table 1 funcref) + (elem (offset (;empty instruction sequence;))) + ) +@@ -476,7 +476,7 @@ + ) + + (assert_invalid +- (module ++ (module + (table 1 funcref) + (elem (global.get 0)) + ) +@@ -493,7 +493,7 @@ + ) + + (assert_invalid +- (module ++ (module + (global (import "test" "global-mut-i32") (mut i32)) + (table 1 funcref) + (elem (global.get 0)) +@@ -603,12 +603,13 @@ + ) + ) + +-(register "module1" $module1) ++(; (register "module1" $module1) ;) + +-(assert_trap (invoke $module1 "call-7") "uninitialized element") +-(assert_return (invoke $module1 "call-8") (i32.const 65)) +-(assert_return (invoke $module1 "call-9") (i32.const 66)) ++(assert_trap (invoke "call-7") "uninitialized element") ++(assert_return (invoke "call-8") (i32.const 65)) ++(assert_return (invoke "call-9") (i32.const 66)) + ++(; + (module $module2 + (type $out-i32 (func (result i32))) + (import "module1" "shared-table" (table 10 funcref)) +@@ -617,11 +618,15 @@ + (func $const-i32-c (type $out-i32) (i32.const 67)) + (func $const-i32-d (type $out-i32) (i32.const 68)) + ) ++;) + ++(; + (assert_return (invoke $module1 "call-7") (i32.const 67)) + (assert_return (invoke $module1 "call-8") (i32.const 68)) + (assert_return (invoke $module1 "call-9") (i32.const 66)) ++;) + ++(; + (module $module3 + (type $out-i32 (func (result i32))) + (import "module1" "shared-table" (table 10 funcref)) +@@ -634,6 +639,7 @@ + (assert_return (invoke $module1 "call-7") (i32.const 67)) + (assert_return (invoke $module1 "call-8") (i32.const 69)) + (assert_return (invoke $module1 "call-9") (i32.const 70)) ++;) + + ;; Element segments must match element type of table + +@@ -666,6 +672,7 @@ + + ;; Initializing a table with an externref-type element segment + ++(; + (module $m + (table $t (export "table") 2 externref) + (func (export "get") (param $i i32) (result externref) +@@ -713,3 +720,5 @@ + ) + + (assert_return (invoke "call_imported_elem") (i32.const 42)) ++ ++;) +diff --git a/test/core/gc/array.wast b/test/core/gc/array.wast +index f5888cb2..b4a2dc0a 100644 +--- a/test/core/gc/array.wast ++++ b/test/core/gc/array.wast +@@ -95,7 +95,7 @@ + ) + + (assert_return (invoke "new") (ref.array)) +-(assert_return (invoke "new") (ref.eq)) ++;; (assert_return (invoke "new") (ref.eq)) + (assert_return (invoke "get" (i32.const 0)) (f32.const 0)) + (assert_return (invoke "set_get" (i32.const 1) (f32.const 7)) (f32.const 7)) + (assert_return (invoke "len") (i32.const 3)) +@@ -140,7 +140,7 @@ + ) + + (assert_return (invoke "new") (ref.array)) +-(assert_return (invoke "new") (ref.eq)) ++;; (assert_return (invoke "new") (ref.eq)) + (assert_return (invoke "get" (i32.const 0)) (f32.const 1)) + (assert_return (invoke "set_get" (i32.const 1) (f32.const 7)) (f32.const 7)) + (assert_return (invoke "len") (i32.const 2)) +@@ -185,7 +185,7 @@ + ) + + (assert_return (invoke "new") (ref.array)) +-(assert_return (invoke "new") (ref.eq)) ++;; (assert_return (invoke "new") (ref.eq)) + (assert_return (invoke "get" (i32.const 0)) (i32.const 1)) + (assert_return (invoke "set_get" (i32.const 1) (i32.const 7)) (i32.const 7)) + (assert_return (invoke "len") (i32.const 3)) +@@ -193,6 +193,7 @@ + (assert_trap (invoke "get" (i32.const 10)) "out of bounds array access") + (assert_trap (invoke "set_get" (i32.const 10) (i32.const 7)) "out of bounds array access") + ++(; array.new_elem not supported + (module + (type $bvec (array i8)) + (type $vec (array (ref $bvec))) +@@ -251,6 +252,7 @@ + + (assert_trap (invoke "get" (i32.const 10) (i32.const 0)) "out of bounds array access") + (assert_trap (invoke "set_get" (i32.const 10) (i32.const 0) (i32.const 0)) "out of bounds array access") ++;) + + (assert_invalid + (module +diff --git a/test/core/gc/extern.wast b/test/core/gc/extern.wast +index abf31669..9ef86506 100644 +--- a/test/core/gc/extern.wast ++++ b/test/core/gc/extern.wast +@@ -43,7 +43,7 @@ + (assert_return (invoke "externalize-i" (i32.const 1)) (ref.extern)) + (assert_return (invoke "externalize-i" (i32.const 2)) (ref.extern)) + (assert_return (invoke "externalize-i" (i32.const 3)) (ref.extern)) +-(assert_return (invoke "externalize-i" (i32.const 4)) (ref.extern)) ++(assert_return (invoke "externalize-i" (i32.const 4)) (ref.extern 0)) + (assert_return (invoke "externalize-i" (i32.const 5)) (ref.null extern)) + + (assert_return (invoke "externalize-ii" (i32.const 0)) (ref.null any)) +diff --git a/test/core/gc/initializer.wast b/test/core/gc/initializer.wast +new file mode 100644 +index 00000000..32650644 +--- /dev/null ++++ b/test/core/gc/initializer.wast +@@ -0,0 +1,34 @@ ++;; added cases to test constant expressions ++ ++(module ++ (type $struct (struct (field f32) (field $y (mut f32)) (field $z f32))) ++ (type $vec (array f32)) ++ ++ ;; table initializer ++ (table 10 anyref) ++ ++ ;; global initializer ++ (global (ref $vec) (array.new_fixed $vec 2 (f32.const 1) (f32.const 2))) ++ (global (ref $struct) (struct.new_default $struct)) ++ ++ ;; elem initializer ++ (elem (i32.const 0) (ref $vec) (array.new_default $vec (i32.const 2))) ++ (elem (i32.const 1) (ref $vec) (array.new $vec (f32.const 1) (i32.const 3))) ++ (elem (i32.const 2) (ref $struct) (struct.new_default $struct)) ++ ++ (func (export "get_table") (param $i i32) (result anyref) ++ (table.get (local.get $i)) ++ ) ++) ++ ++(assert_return (invoke "get_table" (i32.const 0)) (ref.array)) ++(assert_return (invoke "get_table" (i32.const 1)) (ref.array)) ++(assert_return (invoke "get_table" (i32.const 2)) (ref.struct)) ++ ++(assert_invalid ++ (module ++ (type $struct (struct (field f32) (field $y (mut f32)) (field $z f32))) ++ (table 10 anyref (struct.new_default $struct)) ++ ) ++ "unsupported initializer expression for table" ++) +diff --git a/test/core/gc/type-subtyping.wast b/test/core/gc/type-subtyping.wast +index a9022fc3..4aa36e2a 100644 +--- a/test/core/gc/type-subtyping.wast ++++ b/test/core/gc/type-subtyping.wast +@@ -740,7 +740,7 @@ + "sub type" + ) + +-(assert_invalid ++(assert_invalid + (module + (type $f0 (sub (func (param i32) (result i32)))) + (type $s0 (sub $f0 (struct))) +@@ -764,7 +764,7 @@ + "sub type" + ) + +-(assert_invalid ++(assert_invalid + (module + (type $s0 (sub (struct))) + (type $f0 (sub $s0 (func (param i32) (result i32)))) +@@ -772,7 +772,7 @@ + "sub type" + ) + +-(assert_invalid ++(assert_invalid + (module + (type $a0 (sub (array i32))) + (type $f0 (sub $a0 (func (param i32) (result i32)))) +diff --git a/test/core/global.wast b/test/core/global.wast +index 8c47fde2..1a8cc7e3 100644 +--- a/test/core/global.wast ++++ b/test/core/global.wast +@@ -644,7 +644,7 @@ + ) + ) + +-(assert_return (invoke "get-elem" (i32.const 0)) (ref.null)) ++(assert_return (invoke "get-elem" (i32.const 0)) (ref.null func)) + (assert_return (invoke "get-elem" (i32.const 4)) (ref.func)) + (assert_return (invoke "get-elem" (i32.const 8)) (ref.func)) + +@@ -652,7 +652,7 @@ + (assert_return (invoke "get-data" (i32.const 8)) (i32.const 0x88888888)) + + (assert_invalid +- (module ++ (module + (global $g1 i32 (global.get $g2)) + (global $g2 i32 (i32.const 0)) + ) +diff --git a/test/core/imports.wast b/test/core/imports.wast +index 69f76a0b..a3844c65 100644 +--- a/test/core/imports.wast ++++ b/test/core/imports.wast +@@ -572,6 +572,7 @@ + (assert_return (invoke "grow" (i32.const 1)) (i32.const -1)) + (assert_return (invoke "grow" (i32.const 0)) (i32.const 2)) + ++(; unsupported by multi-module currently + (module $Mgm + (memory (export "memory") 1) ;; initial size is 1 + (func (export "grow") (result i32) (memory.grow (i32.const 1))) +@@ -591,6 +592,7 @@ + (func (export "size") (result i32) (memory.size)) + ) + (assert_return (invoke $Mgim2 "size") (i32.const 3)) ++;) + + + ;; Syntax errors +diff --git a/test/core/linking.wast b/test/core/linking.wast +index 6a8ba1d0..a45534fd 100644 +--- a/test/core/linking.wast ++++ b/test/core/linking.wast +@@ -64,6 +64,7 @@ + (export "Mg.set_mut" (func $set_mut)) + ) + ++(; + (assert_return (get $Mg "glob") (i32.const 42)) + (assert_return (get $Ng "Mg.glob") (i32.const 42)) + (assert_return (get $Ng "glob") (i32.const 43)) +@@ -81,6 +82,7 @@ + (assert_return (get $Ng "Mg.mut_glob") (i32.const 241)) + (assert_return (invoke $Mg "get_mut") (i32.const 241)) + (assert_return (invoke $Ng "Mg.get_mut") (i32.const 241)) ++;) + + + (assert_unlinkable +@@ -300,6 +302,7 @@ + ) + ) + ++(; + (assert_return (invoke $Mt "call" (i32.const 2)) (i32.const 4)) + (assert_return (invoke $Nt "Mt.call" (i32.const 2)) (i32.const 4)) + (assert_return (invoke $Nt "call" (i32.const 2)) (i32.const 5)) +@@ -322,6 +325,7 @@ + + (assert_return (invoke $Nt "call" (i32.const 3)) (i32.const -4)) + (assert_trap (invoke $Nt "call" (i32.const 4)) "indirect call type mismatch") ++;) + + (module $Ot + (type (func (result i32))) +@@ -336,6 +340,7 @@ + ) + ) + ++(; + (assert_return (invoke $Mt "call" (i32.const 3)) (i32.const 4)) + (assert_return (invoke $Nt "Mt.call" (i32.const 3)) (i32.const 4)) + (assert_return (invoke $Nt "call Mt.call" (i32.const 3)) (i32.const 4)) +@@ -360,6 +365,7 @@ + (assert_trap (invoke $Ot "call" (i32.const 0)) "uninitialized element") + + (assert_trap (invoke $Ot "call" (i32.const 20)) "undefined element") ++;) + + (module + (table (import "Mt" "tab") 0 funcref) +@@ -398,6 +404,7 @@ + + ;; Unlike in the v1 spec, active element segments stored before an + ;; out-of-bounds access persist after the instantiation failure. ++(; + (assert_trap + (module + (table (import "Mt" "tab") 10 funcref) +@@ -409,7 +416,9 @@ + ) + (assert_return (invoke $Mt "call" (i32.const 7)) (i32.const 0)) + (assert_trap (invoke $Mt "call" (i32.const 8)) "uninitialized element") ++;) + ++(; + (assert_trap + (module + (table (import "Mt" "tab") 10 funcref) +@@ -421,6 +430,7 @@ + "out of bounds memory access" + ) + (assert_return (invoke $Mt "call" (i32.const 7)) (i32.const 0)) ++;) + + + (module $Mtable_ex +@@ -503,10 +513,12 @@ + ) + ) + ++(; + (assert_return (invoke $Mm "load" (i32.const 12)) (i32.const 0xa7)) + (assert_return (invoke $Nm "Mm.load" (i32.const 12)) (i32.const 0xa7)) + (assert_return (invoke $Nm "load" (i32.const 12)) (i32.const 0xf2)) + (assert_return (invoke $Om "load" (i32.const 12)) (i32.const 0xa7)) ++;) + + (module + (memory (import "Mm" "mem") 0) +@@ -529,6 +541,7 @@ + ) + ) + ++(; + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 1)) + (assert_return (invoke $Pm "grow" (i32.const 2)) (i32.const 1)) + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 3)) +@@ -537,6 +550,7 @@ + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 5)) + (assert_return (invoke $Pm "grow" (i32.const 1)) (i32.const -1)) + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 5)) ++;) + + (assert_unlinkable + (module +@@ -560,8 +574,10 @@ + ) + "out of bounds memory access" + ) ++(; + (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97)) + (assert_return (invoke $Mm "load" (i32.const 327670)) (i32.const 0)) ++;) + + (assert_trap + (module +@@ -573,7 +589,9 @@ + ) + "out of bounds table access" + ) ++(; + (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97)) ++;) + + ;; Store is modified if the start function traps. + (module $Ms +@@ -589,6 +607,7 @@ + ) + (register "Ms" $Ms) + ++(; + (assert_trap + (module + (import "Ms" "memory" (memory 1)) +@@ -608,3 +627,4 @@ + + (assert_return (invoke $Ms "get memory[0]") (i32.const 104)) ;; 'h' + (assert_return (invoke $Ms "get table[0]") (i32.const 0xdead)) ++;) +diff --git a/test/core/ref_func.wast b/test/core/ref_func.wast +index adb5cb78..590f6262 100644 +--- a/test/core/ref_func.wast ++++ b/test/core/ref_func.wast +@@ -4,7 +4,8 @@ + (register "M") + + (module +- (func $f (import "M" "f") (param i32) (result i32)) ++ (; aot mode does not support module linking ;) ++ (func $f (param $x i32) (result i32) (local.get $x)) + (func $g (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + ) +diff --git a/test/core/ref_null.wast b/test/core/ref_null.wast +index 1ffd03f8..bdf7471f 100644 +--- a/test/core/ref_null.wast ++++ b/test/core/ref_null.wast +@@ -11,7 +11,7 @@ + + (assert_return (invoke "anyref") (ref.null any)) + (assert_return (invoke "funcref") (ref.null func)) +-(assert_return (invoke "ref") (ref.null)) ++(assert_return (invoke "ref") (ref.null func)) ;; we alwasy require type information + + + (module +@@ -41,23 +41,23 @@ + ) + + (assert_return (invoke "anyref") (ref.null any)) +-(assert_return (invoke "anyref") (ref.null none)) +-(assert_return (invoke "anyref") (ref.null)) ++;; (assert_return (invoke "anyref") (ref.null none)) ++;; (assert_return (invoke "anyref") (ref.null func)) + (assert_return (invoke "nullref") (ref.null any)) +-(assert_return (invoke "nullref") (ref.null none)) +-(assert_return (invoke "nullref") (ref.null)) ++;; (assert_return (invoke "nullref") (ref.null none)) ++;; (assert_return (invoke "nullref") (ref.null func)) + (assert_return (invoke "funcref") (ref.null func)) +-(assert_return (invoke "funcref") (ref.null nofunc)) +-(assert_return (invoke "funcref") (ref.null)) ++;; (assert_return (invoke "funcref") (ref.null nofunc)) ++(assert_return (invoke "funcref") (ref.null func)) ++(assert_return (invoke "nullfuncref") (ref.null func)) ++;; (assert_return (invoke "nullfuncref") (ref.null nofunc)) + (assert_return (invoke "nullfuncref") (ref.null func)) +-(assert_return (invoke "nullfuncref") (ref.null nofunc)) +-(assert_return (invoke "nullfuncref") (ref.null)) + (assert_return (invoke "externref") (ref.null extern)) +-(assert_return (invoke "externref") (ref.null noextern)) +-(assert_return (invoke "externref") (ref.null)) ++;; (assert_return (invoke "externref") (ref.null noextern)) ++;; (assert_return (invoke "externref") (ref.null func)) + (assert_return (invoke "nullexternref") (ref.null extern)) +-(assert_return (invoke "nullexternref") (ref.null noextern)) +-(assert_return (invoke "nullexternref") (ref.null)) ++;; (assert_return (invoke "nullexternref") (ref.null noextern)) ++;; (assert_return (invoke "nullexternref") (ref.null func)) ++(assert_return (invoke "ref") (ref.null func)) ++;; (assert_return (invoke "ref") (ref.null nofunc)) + (assert_return (invoke "ref") (ref.null func)) +-(assert_return (invoke "ref") (ref.null nofunc)) +-(assert_return (invoke "ref") (ref.null)) +diff --git a/test/core/return_call.wast b/test/core/return_call.wast +index 2f91f4de..ad66acca 100644 +--- a/test/core/return_call.wast ++++ b/test/core/return_call.wast +@@ -102,20 +102,20 @@ + + (assert_return (invoke "count" (i64.const 0)) (i64.const 0)) + (assert_return (invoke "count" (i64.const 1000)) (i64.const 0)) +-(assert_return (invoke "count" (i64.const 1_000_000)) (i64.const 0)) ++(assert_return (invoke "count" (i64.const 100_000)) (i64.const 0)) + + (assert_return (invoke "even" (i64.const 0)) (i32.const 44)) + (assert_return (invoke "even" (i64.const 1)) (i32.const 99)) + (assert_return (invoke "even" (i64.const 100)) (i32.const 44)) + (assert_return (invoke "even" (i64.const 77)) (i32.const 99)) +-(assert_return (invoke "even" (i64.const 1_000_000)) (i32.const 44)) +-(assert_return (invoke "even" (i64.const 1_000_001)) (i32.const 99)) ++(assert_return (invoke "even" (i64.const 100_000)) (i32.const 44)) ++(assert_return (invoke "even" (i64.const 100_001)) (i32.const 99)) + (assert_return (invoke "odd" (i64.const 0)) (i32.const 99)) + (assert_return (invoke "odd" (i64.const 1)) (i32.const 44)) + (assert_return (invoke "odd" (i64.const 200)) (i32.const 99)) + (assert_return (invoke "odd" (i64.const 77)) (i32.const 44)) +-(assert_return (invoke "odd" (i64.const 1_000_000)) (i32.const 99)) +-(assert_return (invoke "odd" (i64.const 999_999)) (i32.const 44)) ++(assert_return (invoke "odd" (i64.const 100_000)) (i32.const 99)) ++(assert_return (invoke "odd" (i64.const 99_999)) (i32.const 44)) + + + ;; Invalid typing +diff --git a/test/core/return_call_indirect.wast b/test/core/return_call_indirect.wast +index acf0a72e..6b95c24b 100644 +--- a/test/core/return_call_indirect.wast ++++ b/test/core/return_call_indirect.wast +@@ -263,8 +263,8 @@ + (assert_return (invoke "odd" (i32.const 1)) (i32.const 44)) + (assert_return (invoke "odd" (i32.const 200)) (i32.const 99)) + (assert_return (invoke "odd" (i32.const 77)) (i32.const 44)) +-(assert_return (invoke "odd" (i32.const 200_002)) (i32.const 99)) +-(assert_return (invoke "odd" (i32.const 300_003)) (i32.const 44)) ++(assert_return (invoke "odd" (i32.const 100_002)) (i32.const 99)) ++(assert_return (invoke "odd" (i32.const 100_003)) (i32.const 44)) + + + ;; Invalid syntax +diff --git a/test/core/return_call_ref.wast b/test/core/return_call_ref.wast +index 353811f0..f79975b4 100644 +--- a/test/core/return_call_ref.wast ++++ b/test/core/return_call_ref.wast +@@ -192,20 +192,20 @@ + + (assert_return (invoke "count" (i64.const 0)) (i64.const 0)) + (assert_return (invoke "count" (i64.const 1000)) (i64.const 0)) +-(assert_return (invoke "count" (i64.const 1_000_000)) (i64.const 0)) ++(assert_return (invoke "count" (i64.const 1200)) (i64.const 0)) + + (assert_return (invoke "even" (i64.const 0)) (i64.const 44)) + (assert_return (invoke "even" (i64.const 1)) (i64.const 99)) + (assert_return (invoke "even" (i64.const 100)) (i64.const 44)) + (assert_return (invoke "even" (i64.const 77)) (i64.const 99)) +-(assert_return (invoke "even" (i64.const 1_000_000)) (i64.const 44)) +-(assert_return (invoke "even" (i64.const 1_000_001)) (i64.const 99)) ++(assert_return (invoke "even" (i64.const 1200)) (i64.const 44)) ++(assert_return (invoke "even" (i64.const 1201)) (i64.const 99)) + (assert_return (invoke "odd" (i64.const 0)) (i64.const 99)) + (assert_return (invoke "odd" (i64.const 1)) (i64.const 44)) + (assert_return (invoke "odd" (i64.const 200)) (i64.const 99)) + (assert_return (invoke "odd" (i64.const 77)) (i64.const 44)) +-(assert_return (invoke "odd" (i64.const 1_000_000)) (i64.const 99)) +-(assert_return (invoke "odd" (i64.const 999_999)) (i64.const 44)) ++(assert_return (invoke "odd" (i64.const 1200)) (i64.const 99)) ++(assert_return (invoke "odd" (i64.const 1119)) (i64.const 44)) + + + ;; More typing +diff --git a/test/core/select.wast b/test/core/select.wast +index 61e4dc22..b0b1344c 100644 +--- a/test/core/select.wast ++++ b/test/core/select.wast +@@ -277,7 +277,7 @@ + (assert_return (invoke "select-f64-t" (f64.const 2) (f64.const nan:0x20304) (i32.const 0)) (f64.const nan:0x20304)) + + (assert_return (invoke "join-funcnull" (i32.const 1)) (ref.func)) +-(assert_return (invoke "join-funcnull" (i32.const 0)) (ref.null)) ++(assert_return (invoke "join-funcnull" (i32.const 0)) (ref.null func)) ;; we require type in expected results + + (assert_trap (invoke "select-trap-left" (i32.const 1)) "unreachable") + (assert_trap (invoke "select-trap-left" (i32.const 0)) "unreachable") +diff --git a/test/core/table.wast b/test/core/table.wast +index a11dce56..ace19ac8 100644 +--- a/test/core/table.wast ++++ b/test/core/table.wast +@@ -103,11 +103,11 @@ + (func (export "get5") (result funcref) (table.get $t5 (i32.const 9))) + ) + +-(assert_return (invoke "get1") (ref.null)) ++(assert_return (invoke "get1") (ref.null func)) + (assert_return (invoke "get2") (ref.func)) + (assert_return (invoke "get3") (ref.func)) +-(assert_return (invoke "get4") (ref.func)) +-(assert_return (invoke "get5") (ref.func)) ++(assert_return (invoke "get4") (ref.null func)) ;; We don't give a value to the imported global ++(assert_return (invoke "get5") (ref.null func)) ;; So these two tables are initialized as ref.null + + + (assert_invalid +diff --git a/test/core/table_copy.wast b/test/core/table_copy.wast +index 380e84ee..f37e745c 100644 +--- a/test/core/table_copy.wast ++++ b/test/core/table_copy.wast +@@ -14,11 +14,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -106,11 +107,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (export "ef0") (result i32) (i32.const 0)) ;; index 0 ++ (func (export "ef1") (result i32) (i32.const 1)) ++ (func (export "ef2") (result i32) (i32.const 2)) ++ (func (export "ef3") (result i32) (i32.const 3)) ++ (func (export "ef4") (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -198,11 +199,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -290,11 +291,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -382,11 +383,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -474,11 +475,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -566,11 +567,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -658,11 +659,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -750,11 +751,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -842,11 +843,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -934,11 +935,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1026,11 +1027,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1118,11 +1119,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1210,11 +1211,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1302,11 +1303,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1394,11 +1395,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1486,11 +1487,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1578,11 +1579,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +diff --git a/test/core/table_init.wast b/test/core/table_init.wast +index 0b2d26f7..bdab6a01 100644 +--- a/test/core/table_init.wast ++++ b/test/core/table_init.wast +@@ -14,11 +14,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -72,11 +73,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -130,11 +132,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -196,11 +199,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -254,11 +258,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -312,11 +317,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/gc_nuttx_tail_call.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/gc_nuttx_tail_call.patch new file mode 100644 index 0000000..efbd9e1 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/gc_nuttx_tail_call.patch @@ -0,0 +1,53 @@ +diff --git a/test/core/return_call.wast b/test/core/return_call.wast +index ad66acca..b27af19b 100644 +--- a/test/core/return_call.wast ++++ b/test/core/return_call.wast +@@ -102,20 +102,20 @@ + + (assert_return (invoke "count" (i64.const 0)) (i64.const 0)) + (assert_return (invoke "count" (i64.const 1000)) (i64.const 0)) +-(assert_return (invoke "count" (i64.const 100_000)) (i64.const 0)) ++(assert_return (invoke "count" (i64.const 1001)) (i64.const 0)) + + (assert_return (invoke "even" (i64.const 0)) (i32.const 44)) + (assert_return (invoke "even" (i64.const 1)) (i32.const 99)) + (assert_return (invoke "even" (i64.const 100)) (i32.const 44)) + (assert_return (invoke "even" (i64.const 77)) (i32.const 99)) +-(assert_return (invoke "even" (i64.const 100_000)) (i32.const 44)) +-(assert_return (invoke "even" (i64.const 100_001)) (i32.const 99)) ++(assert_return (invoke "even" (i64.const 1000)) (i32.const 44)) ++(assert_return (invoke "even" (i64.const 1001)) (i32.const 99)) + (assert_return (invoke "odd" (i64.const 0)) (i32.const 99)) + (assert_return (invoke "odd" (i64.const 1)) (i32.const 44)) + (assert_return (invoke "odd" (i64.const 200)) (i32.const 99)) + (assert_return (invoke "odd" (i64.const 77)) (i32.const 44)) +-(assert_return (invoke "odd" (i64.const 100_000)) (i32.const 99)) +-(assert_return (invoke "odd" (i64.const 99_999)) (i32.const 44)) ++(assert_return (invoke "odd" (i64.const 1000)) (i32.const 99)) ++(assert_return (invoke "odd" (i64.const 999)) (i32.const 44)) + + + ;; Invalid typing +diff --git a/test/core/return_call_indirect.wast b/test/core/return_call_indirect.wast +index 6b95c24b..a9e86d42 100644 +--- a/test/core/return_call_indirect.wast ++++ b/test/core/return_call_indirect.wast +@@ -257,14 +257,14 @@ + (assert_return (invoke "even" (i32.const 1)) (i32.const 99)) + (assert_return (invoke "even" (i32.const 100)) (i32.const 44)) + (assert_return (invoke "even" (i32.const 77)) (i32.const 99)) +-(assert_return (invoke "even" (i32.const 100_000)) (i32.const 44)) +-(assert_return (invoke "even" (i32.const 111_111)) (i32.const 99)) ++(assert_return (invoke "even" (i32.const 1000)) (i32.const 44)) ++(assert_return (invoke "even" (i32.const 1111)) (i32.const 99)) + (assert_return (invoke "odd" (i32.const 0)) (i32.const 99)) + (assert_return (invoke "odd" (i32.const 1)) (i32.const 44)) + (assert_return (invoke "odd" (i32.const 200)) (i32.const 99)) + (assert_return (invoke "odd" (i32.const 77)) (i32.const 44)) +-(assert_return (invoke "odd" (i32.const 100_002)) (i32.const 99)) +-(assert_return (invoke "odd" (i32.const 100_003)) (i32.const 44)) ++(assert_return (invoke "odd" (i32.const 1002)) (i32.const 99)) ++(assert_return (invoke "odd" (i32.const 1003)) (i32.const 44)) + + + ;; Invalid syntax diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/ignore_cases.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/ignore_cases.patch new file mode 100644 index 0000000..bfb4383 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/ignore_cases.patch @@ -0,0 +1,785 @@ +diff --git a/test/core/binary.wast b/test/core/binary.wast +index 891aad3..07356a3 100644 +--- a/test/core/binary.wast ++++ b/test/core/binary.wast +@@ -206,7 +206,7 @@ + ) + + ;; Type section with signed LEB128 encoded type +-(assert_malformed ++(;assert_malformed + (module binary + "\00asm" "\01\00\00\00" + "\01" ;; Type section id +@@ -216,7 +216,7 @@ + "\00\00" + ) + "integer representation too long" +-) ++;) + + ;; Unsigned LEB128 must not be overlong + (assert_malformed +@@ -1683,7 +1683,7 @@ + ) + + ;; 2 elem segment declared, 1 given +-(assert_malformed ++(;assert_malformed + (module binary + "\00asm" "\01\00\00\00" + "\01\04\01" ;; type section +@@ -1696,7 +1696,7 @@ + ;; "\00\41\00\0b\01\00" ;; elem 1 (missed) + ) + "unexpected end" +-) ++;) + + ;; 2 elem segment declared, 1.5 given + (assert_malformed +@@ -1813,7 +1813,7 @@ + ) + + ;; 1 br_table target declared, 2 given +-(assert_malformed ++(;assert_malformed + (module binary + "\00asm" "\01\00\00\00" + "\01\04\01" ;; type section +@@ -1832,7 +1832,7 @@ + "\0b\0b\0b" ;; end + ) + "unexpected end" +-) ++;) + + ;; Start section + (module binary +diff --git a/test/core/data.wast b/test/core/data.wast +index 4f339be..0b5b3e6 100644 +--- a/test/core/data.wast ++++ b/test/core/data.wast +@@ -306,9 +306,10 @@ + "\02\01\41\00\0b" ;; active data segment 0 for memory 1 + "\00" ;; empty vec(byte) + ) +- "unknown memory 1" ++ "unknown memory" + ) + ++(; not supported by wat2wasm + ;; Data segment with memory index 0 (no memory section) + (assert_invalid + (module binary +@@ -317,7 +318,7 @@ + "\00\41\00\0b" ;; active data segment 0 for memory 0 + "\00" ;; empty vec(byte) + ) +- "unknown memory 0" ++ "unknown memory" + ) + + ;; Data segment with memory index 1 (no memory section) +@@ -328,7 +329,7 @@ + "\02\01\41\00\0b" ;; active data segment 0 for memory 1 + "\00" ;; empty vec(byte) + ) +- "unknown memory 1" ++ "unknown memory" + ) + + ;; Data segment with memory index 1 and vec(byte) as above, +@@ -348,7 +349,7 @@ + "\20\21\22\23\24\25\26\27\28\29\2a\2b\2c\2d\2e\2f" + "\30\31\32\33\34\35\36\37\38\39\3a\3b\3c\3d" + ) +- "unknown memory 1" ++ "unknown memory" + ) + + ;; Data segment with memory index 1 and specially crafted vec(byte) after. +@@ -368,8 +369,9 @@ + "\20\21\22\23\24\25\26\27\28\29\2a\2b\2c\2d\2e\2f" + "\30\31\32\33\34\35\36\37\38\39\3a\3b\3c\3d" + ) +- "unknown memory 1" ++ "unknown memory" + ) ++;) + + + ;; Invalid offsets +diff --git a/test/core/elem.wast b/test/core/elem.wast +index 575ecef..6eecab9 100644 +--- a/test/core/elem.wast ++++ b/test/core/elem.wast +@@ -571,9 +571,11 @@ + (func $const-i32-d (type $out-i32) (i32.const 68)) + ) + ++(; + (assert_return (invoke $module1 "call-7") (i32.const 67)) + (assert_return (invoke $module1 "call-8") (i32.const 68)) + (assert_return (invoke $module1 "call-9") (i32.const 66)) ++;) + + (module $module3 + (type $out-i32 (func (result i32))) +@@ -584,6 +586,8 @@ + (func $const-i32-f (type $out-i32) (i32.const 70)) + ) + ++(; + (assert_return (invoke $module1 "call-7") (i32.const 67)) + (assert_return (invoke $module1 "call-8") (i32.const 69)) + (assert_return (invoke $module1 "call-9") (i32.const 70)) ++;) +diff --git a/test/core/global.wast b/test/core/global.wast +index 9fa5e22..8c4b949 100644 +--- a/test/core/global.wast ++++ b/test/core/global.wast +@@ -328,10 +328,12 @@ + "type mismatch" + ) + ++(; + (assert_invalid + (module (global (import "" "") externref) (global funcref (global.get 0))) + "type mismatch" + ) ++;) + + (assert_invalid + (module (global (import "test" "global-i32") i32) (global i32 (global.get 0) (global.get 0))) +diff --git a/test/core/imports.wast b/test/core/imports.wast +index 35e8c91..a7a459d 100644 +--- a/test/core/imports.wast ++++ b/test/core/imports.wast +@@ -577,6 +577,7 @@ + (assert_return (invoke "grow" (i32.const 1)) (i32.const -1)) + (assert_return (invoke "grow" (i32.const 0)) (i32.const 2)) + ++(; unsupported by multi-module currently + (module $Mgm + (memory (export "memory") 1) ;; initial size is 1 + (func (export "grow") (result i32) (memory.grow (i32.const 1))) +@@ -596,6 +597,7 @@ + (func (export "size") (result i32) (memory.size)) + ) + (assert_return (invoke $Mgim2 "size") (i32.const 3)) ++;) + + + ;; Syntax errors +diff --git a/test/core/linking.wast b/test/core/linking.wast +index 994e0f4..d0bfb5f 100644 +--- a/test/core/linking.wast ++++ b/test/core/linking.wast +@@ -64,6 +64,7 @@ + (export "Mg.set_mut" (func $set_mut)) + ) + ++(; + (assert_return (get $Mg "glob") (i32.const 42)) + (assert_return (get $Ng "Mg.glob") (i32.const 42)) + (assert_return (get $Ng "glob") (i32.const 43)) +@@ -81,6 +82,7 @@ + (assert_return (get $Ng "Mg.mut_glob") (i32.const 241)) + (assert_return (invoke $Mg "get_mut") (i32.const 241)) + (assert_return (invoke $Ng "Mg.get_mut") (i32.const 241)) ++;) + + + (assert_unlinkable +@@ -165,6 +167,7 @@ + ) + ) + ++(; + (assert_return (invoke $Mt "call" (i32.const 2)) (i32.const 4)) + (assert_return (invoke $Nt "Mt.call" (i32.const 2)) (i32.const 4)) + (assert_return (invoke $Nt "call" (i32.const 2)) (i32.const 5)) +@@ -187,6 +190,7 @@ + + (assert_return (invoke $Nt "call" (i32.const 3)) (i32.const -4)) + (assert_trap (invoke $Nt "call" (i32.const 4)) "indirect call type mismatch") ++;) + + (module $Ot + (type (func (result i32))) +@@ -201,6 +205,7 @@ + ) + ) + ++(; + (assert_return (invoke $Mt "call" (i32.const 3)) (i32.const 4)) + (assert_return (invoke $Nt "Mt.call" (i32.const 3)) (i32.const 4)) + (assert_return (invoke $Nt "call Mt.call" (i32.const 3)) (i32.const 4)) +@@ -225,6 +230,7 @@ + (assert_trap (invoke $Ot "call" (i32.const 0)) "uninitialized element") + + (assert_trap (invoke $Ot "call" (i32.const 20)) "undefined element") ++;) + + (module + (table (import "Mt" "tab") 0 funcref) +@@ -263,6 +269,7 @@ + + ;; Unlike in the v1 spec, active element segments stored before an + ;; out-of-bounds access persist after the instantiation failure. ++(; + (assert_trap + (module + (table (import "Mt" "tab") 10 funcref) +@@ -274,7 +281,9 @@ + ) + (assert_return (invoke $Mt "call" (i32.const 7)) (i32.const 0)) + (assert_trap (invoke $Mt "call" (i32.const 8)) "uninitialized element") ++;) + ++(; + (assert_trap + (module + (table (import "Mt" "tab") 10 funcref) +@@ -286,6 +295,7 @@ + "out of bounds memory access" + ) + (assert_return (invoke $Mt "call" (i32.const 7)) (i32.const 0)) ++;) + + + (module $Mtable_ex +@@ -299,6 +309,7 @@ + (table (import "Mtable_ex" "t-extern") 1 externref) + ) + ++(; + (assert_unlinkable + (module (table (import "Mtable_ex" "t-func") 1 externref)) + "incompatible import type" +@@ -307,6 +318,7 @@ + (module (table (import "Mtable_ex" "t-extern") 1 funcref)) + "incompatible import type" + ) ++;) + + + ;; Memories +@@ -346,10 +358,12 @@ + ) + ) + ++(; + (assert_return (invoke $Mm "load" (i32.const 12)) (i32.const 0xa7)) + (assert_return (invoke $Nm "Mm.load" (i32.const 12)) (i32.const 0xa7)) + (assert_return (invoke $Nm "load" (i32.const 12)) (i32.const 0xf2)) + (assert_return (invoke $Om "load" (i32.const 12)) (i32.const 0xa7)) ++;) + + (module + (memory (import "Mm" "mem") 0) +@@ -372,6 +386,7 @@ + ) + ) + ++(; + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 1)) + (assert_return (invoke $Pm "grow" (i32.const 2)) (i32.const 1)) + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 3)) +@@ -380,6 +395,7 @@ + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 5)) + (assert_return (invoke $Pm "grow" (i32.const 1)) (i32.const -1)) + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 5)) ++;) + + (assert_unlinkable + (module +@@ -403,8 +419,10 @@ + ) + "out of bounds memory access" + ) ++(; + (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97)) + (assert_return (invoke $Mm "load" (i32.const 327670)) (i32.const 0)) ++;) + + (assert_trap + (module +@@ -416,7 +434,9 @@ + ) + "out of bounds table access" + ) ++(; + (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97)) ++;) + + ;; Store is modified if the start function traps. + (module $Ms +@@ -432,6 +452,7 @@ + ) + (register "Ms" $Ms) + ++(; + (assert_trap + (module + (import "Ms" "memory" (memory 1)) +@@ -451,3 +472,4 @@ + + (assert_return (invoke $Ms "get memory[0]") (i32.const 104)) ;; 'h' + (assert_return (invoke $Ms "get table[0]") (i32.const 0xdead)) ++;) +diff --git a/test/core/ref_func.wast b/test/core/ref_func.wast +index adb5cb7..590f626 100644 +--- a/test/core/ref_func.wast ++++ b/test/core/ref_func.wast +@@ -4,7 +4,8 @@ + (register "M") + + (module +- (func $f (import "M" "f") (param i32) (result i32)) ++ (; aot mode does not support module linking ;) ++ (func $f (param $x i32) (result i32) (local.get $x)) + (func $g (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + ) +diff --git a/test/core/table_copy.wast b/test/core/table_copy.wast +index 380e84e..f37e745 100644 +--- a/test/core/table_copy.wast ++++ b/test/core/table_copy.wast +@@ -14,11 +14,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -106,11 +107,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (export "ef0") (result i32) (i32.const 0)) ;; index 0 ++ (func (export "ef1") (result i32) (i32.const 1)) ++ (func (export "ef2") (result i32) (i32.const 2)) ++ (func (export "ef3") (result i32) (i32.const 3)) ++ (func (export "ef4") (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -198,11 +199,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -290,11 +291,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -382,11 +383,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -474,11 +475,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -566,11 +567,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -658,11 +659,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -750,11 +751,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -842,11 +843,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -934,11 +935,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1026,11 +1027,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1118,11 +1119,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1210,11 +1211,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1302,11 +1303,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1394,11 +1395,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1486,11 +1487,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1578,11 +1579,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +diff --git a/test/core/table_init.wast b/test/core/table_init.wast +index 0b2d26f..bdab6a0 100644 +--- a/test/core/table_init.wast ++++ b/test/core/table_init.wast +@@ -14,11 +14,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -72,11 +73,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -130,11 +132,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -196,11 +199,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -254,11 +258,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -312,11 +317,12 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ ;; aot mode does not support module linking ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +diff --git a/test/core/unreached-valid.wast b/test/core/unreached-valid.wast +index b7ebabf..4f2abfb 100644 +--- a/test/core/unreached-valid.wast ++++ b/test/core/unreached-valid.wast +@@ -46,6 +46,7 @@ + + ;; Validation after unreachable + ++(; + (module + (func (export "meet-bottom") + (block (result f64) +@@ -61,3 +62,4 @@ + ) + + (assert_trap (invoke "meet-bottom") "unreachable") ++;) diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/memory64.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/memory64.patch new file mode 100644 index 0000000..739a1df --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/memory64.patch @@ -0,0 +1,28 @@ +diff --git a/test/core/memory.wast b/test/core/memory.wast +index 1dd5b84..497b69f 100644 +--- a/test/core/memory.wast ++++ b/test/core/memory.wast +@@ -76,17 +76,17 @@ + "memory size must be at most 65536 pages (4GiB)" + ) + +-(assert_invalid ++(assert_malformed + (module quote "(memory 0x1_0000_0000)") +- "memory size must be at most 65536 pages (4GiB)" ++ "i32 constant out of range" + ) +-(assert_invalid ++(assert_malformed + (module quote "(memory 0x1_0000_0000 0x1_0000_0000)") +- "memory size must be at most 65536 pages (4GiB)" ++ "i32 constant out of range" + ) +-(assert_invalid ++(assert_malformed + (module quote "(memory 0 0x1_0000_0000)") +- "memory size must be at most 65536 pages (4GiB)" ++ "i32 constant out of range" + ) + + (module diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/multi_module_aot_ignore_cases.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/multi_module_aot_ignore_cases.patch new file mode 100644 index 0000000..4494e08 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/multi_module_aot_ignore_cases.patch @@ -0,0 +1,174 @@ +diff --git a/test/core/linking.wast b/test/core/linking.wast +index d0bfb5f..6617945 100644 +--- a/test/core/linking.wast ++++ b/test/core/linking.wast +@@ -35,7 +35,7 @@ + + + ;; Globals +- ++(; + (module $Mg + (global $glob (export "glob") i32 (i32.const 42)) + (func (export "get") (result i32) (global.get $glob)) +@@ -63,7 +63,7 @@ + (export "Mg.get_mut" (func $get_mut)) + (export "Mg.set_mut" (func $set_mut)) + ) +- ++;) + (; + (assert_return (get $Mg "glob") (i32.const 42)) + (assert_return (get $Ng "Mg.glob") (i32.const 42)) +@@ -84,7 +84,7 @@ + (assert_return (invoke $Ng "Mg.get_mut") (i32.const 241)) + ;) + +- ++(; + (assert_unlinkable + (module (import "Mg" "mut_glob" (global i32))) + "incompatible import type" +@@ -166,7 +166,7 @@ + (call_indirect (type 1) (local.get 0)) + ) + ) +- ++;) + (; + (assert_return (invoke $Mt "call" (i32.const 2)) (i32.const 4)) + (assert_return (invoke $Nt "Mt.call" (i32.const 2)) (i32.const 4)) +@@ -191,7 +191,7 @@ + (assert_return (invoke $Nt "call" (i32.const 3)) (i32.const -4)) + (assert_trap (invoke $Nt "call" (i32.const 4)) "indirect call type mismatch") + ;) +- ++(; + (module $Ot + (type (func (result i32))) + +@@ -204,7 +204,7 @@ + (call_indirect (type 0) (local.get 0)) + ) + ) +- ++;) + (; + (assert_return (invoke $Mt "call" (i32.const 3)) (i32.const 4)) + (assert_return (invoke $Nt "Mt.call" (i32.const 3)) (i32.const 4)) +@@ -231,7 +231,7 @@ + + (assert_trap (invoke $Ot "call" (i32.const 20)) "undefined element") + ;) +- ++(; + (module + (table (import "Mt" "tab") 0 funcref) + (elem (i32.const 9) $f) +@@ -266,7 +266,7 @@ + "unknown import" + ) + (assert_trap (invoke $Mt "call" (i32.const 7)) "uninitialized element") +- ++;) + ;; Unlike in the v1 spec, active element segments stored before an + ;; out-of-bounds access persist after the instantiation failure. + (; +@@ -297,7 +297,7 @@ + (assert_return (invoke $Mt "call" (i32.const 7)) (i32.const 0)) + ;) + +- ++(; + (module $Mtable_ex + (table $t1 (export "t-func") 1 funcref) + (table $t2 (export "t-extern") 1 externref) +@@ -308,7 +308,7 @@ + (table (import "Mtable_ex" "t-func") 1 funcref) + (table (import "Mtable_ex" "t-extern") 1 externref) + ) +- ++;) + (; + (assert_unlinkable + (module (table (import "Mtable_ex" "t-func") 1 externref)) +@@ -322,7 +322,7 @@ + + + ;; Memories +- ++(; + (module $Mm + (memory (export "mem") 1 5) + (data (i32.const 10) "\00\01\02\03\04\05\06\07\08\09") +@@ -357,14 +357,14 @@ + (i32.load8_u (local.get 0)) + ) + ) +- ++;) + (; + (assert_return (invoke $Mm "load" (i32.const 12)) (i32.const 0xa7)) + (assert_return (invoke $Nm "Mm.load" (i32.const 12)) (i32.const 0xa7)) + (assert_return (invoke $Nm "load" (i32.const 12)) (i32.const 0xf2)) + (assert_return (invoke $Om "load" (i32.const 12)) (i32.const 0xa7)) + ;) +- ++(; + (module + (memory (import "Mm" "mem") 0) + (data (i32.const 0xffff) "a") +@@ -385,7 +385,7 @@ + (memory.grow (local.get 0)) + ) + ) +- ++;) + (; + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 1)) + (assert_return (invoke $Pm "grow" (i32.const 2)) (i32.const 1)) +@@ -396,7 +396,7 @@ + (assert_return (invoke $Pm "grow" (i32.const 1)) (i32.const -1)) + (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 5)) + ;) +- ++(; + (assert_unlinkable + (module + (func $host (import "spectest" "print")) +@@ -419,11 +419,12 @@ + ) + "out of bounds memory access" + ) ++;) + (; + (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97)) + (assert_return (invoke $Mm "load" (i32.const 327670)) (i32.const 0)) + ;) +- ++(; + (assert_trap + (module + (memory (import "Mm" "mem") 1) +@@ -434,10 +435,11 @@ + ) + "out of bounds table access" + ) ++;) + (; + (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97)) + ;) +- ++(; + ;; Store is modified if the start function traps. + (module $Ms + (type $t (func (result i32))) +@@ -451,7 +453,7 @@ + ) + ) + (register "Ms" $Ms) +- ++;) + (; + (assert_trap + (module diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/runtest.py b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/runtest.py new file mode 100755 index 0000000..13229d9 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/runtest.py @@ -0,0 +1,1506 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import argparse +import array +import atexit +import math +import os +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import time +import threading +import traceback +from select import select +from queue import Queue +from subprocess import PIPE, STDOUT, Popen +from typing import BinaryIO, Optional, Tuple + +if sys.version_info[0] == 2: + IS_PY_3 = False +else: + IS_PY_3 = True + +test_aot = False +# Available targets: +# "aarch64" "aarch64_vfp" "armv7" "armv7_vfp" "thumbv7" "thumbv7_vfp" +# "riscv32" "riscv32_ilp32f" "riscv32_ilp32d" "riscv64" "riscv64_lp64f" "riscv64_lp64d" +test_target = "x86_64" + +debug_file = None +log_file = None + +# to save the register module with self-define name +temp_file_repo = [] + +# to save the mapping of module files in /tmp by name +temp_module_table = {} + +# AOT compilation options mapping +aot_target_options_map = { + "i386": ["--target=i386"], + "x86_32": ["--target=i386"], + "x86_64": ["--target=x86_64", "--cpu=skylake"], + "aarch64": ["--target=aarch64", "--target-abi=eabi", "--cpu=cortex-a53"], + "aarch64_vfp": ["--target=aarch64", "--target-abi=gnueabihf", "--cpu=cortex-a53"], + "armv7": ["--target=armv7", "--target-abi=eabi", "--cpu=cortex-a9", "--cpu-features=-neon"], + "armv7_vfp": ["--target=armv7", "--target-abi=gnueabihf", "--cpu=cortex-a9"], + "thumbv7": ["--target=thumbv7", "--target-abi=eabi", "--cpu=cortex-a9", "--cpu-features=-neon,-vfpv3"], + "thumbv7_vfp": ["--target=thumbv7", "--target-abi=gnueabihf", "--cpu=cortex-a9", "--cpu-features=-neon"], + "riscv32": ["--target=riscv32", "--target-abi=ilp32", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"], + "riscv32_ilp32f": ["--target=riscv32", "--target-abi=ilp32f", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f"], + "riscv32_ilp32d": ["--target=riscv32", "--target-abi=ilp32d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f,+d"], + "riscv64": ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c"], + "riscv64_lp64f": ["--target=riscv64", "--target-abi=lp64f", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f"], + "riscv64_lp64d": ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f,+d"], +} + +def debug(data): + if debug_file: + debug_file.write(data) + debug_file.flush() + +def log(data, end='\n'): + if log_file: + log_file.write(data + end) + log_file.flush() + print(data, end=end) + sys.stdout.flush() + +def create_tmp_file(suffix: str) -> str: + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp_file: + return tmp_file.name + +# TODO: do we need to support '\n' too +import platform + +if platform.system().find("CYGWIN_NT") >= 0: + # TODO: this is weird, is this really right on Cygwin? + sep = "\n\r\n" +else: + sep = "\r\n" +rundir = None + + +class AsyncStreamReader: + def __init__(self, stream: BinaryIO) -> None: + self._queue = Queue() + self._reader_thread = threading.Thread( + daemon=True, + target=AsyncStreamReader._stdout_reader, + args=(self._queue, stream)) + self._reader_thread.start() + + def read(self) -> Optional[bytes]: + return self._queue.get() + + def cleanup(self) -> None: + self._reader_thread.join() + + @staticmethod + def _stdout_reader(queue: Queue, stdout: BinaryIO) -> None: + while True: + try: + queue.put(stdout.read(1)) + except ValueError as e: + if stdout.closed: + queue.put(None) + break + raise e + + +class Runner(): + def __init__(self, args, no_pty=False): + self.no_pty = no_pty + + # Cleanup child process on exit + atexit.register(self.cleanup) + + self.process = None + env = os.environ + env['TERM'] = 'dumb' + env['INPUTRC'] = '/dev/null' + env['PERL_RL'] = 'false' + if no_pty: + self.process = Popen(args, bufsize=0, + stdin=PIPE, stdout=PIPE, stderr=STDOUT, + env=env) + self.stdin = self.process.stdin + self.stdout = self.process.stdout + else: + import fcntl + # Pseudo-TTY and terminal manipulation + import pty + import termios + # Use tty to setup an interactive environment + master, slave = pty.openpty() + + # Set terminal size large so that readline will not send + # ANSI/VT escape codes when the lines are long. + buf = array.array('h', [100, 200, 0, 0]) + fcntl.ioctl(master, termios.TIOCSWINSZ, buf, True) + + self.process = Popen(args, bufsize=0, + stdin=slave, stdout=slave, stderr=STDOUT, + preexec_fn=os.setsid, + env=env) + # Now close slave so that we will get an exception from + # read when the child exits early + # http://stackoverflow.com/questions/11165521 + os.close(slave) + self.stdin = os.fdopen(master, 'r+b', 0) + self.stdout = self.stdin + + if platform.system().lower() == "windows": + self._stream_reader = AsyncStreamReader(self.stdout) + else: + self._stream_reader = None + + self.buf = "" + + def _read_stdout_byte(self) -> Tuple[bool, Optional[bytes]]: + if self._stream_reader: + return True, self._stream_reader.read() + else: + # select doesn't work on file descriptors on Windows. + # however, this method is much faster than using + # queue, so we keep it for non-windows platforms. + [outs,_,_] = select([self.stdout], [], [], 1) + if self.stdout in outs: + return True, self.stdout.read(1) + else: + return False, None + + def read_to_prompt(self, prompts, timeout): + wait_until = time.time() + timeout + while time.time() < wait_until: + has_value, read_byte = self._read_stdout_byte() + if not has_value: + continue + if not read_byte: + # EOF on macOS ends up here. + break + read_byte = read_byte.decode('utf-8') if IS_PY_3 else read_byte + + debug(read_byte) + if self.no_pty: + self.buf += read_byte.replace('\n', '\r\n') + else: + self.buf += read_byte + self.buf = self.buf.replace('\r\r', '\r') + + # filter the prompts + for prompt in prompts: + pattern = re.compile(prompt) + match = pattern.search(self.buf) + if match: + end = match.end() + buf = self.buf[0:end-len(prompt)] + self.buf = self.buf[end:] + return buf + return None + + def writeline(self, str): + str_to_write = str + '\n' + str_to_write = bytes( + str_to_write, 'utf-8') if IS_PY_3 else str_to_write + + self.stdin.write(str_to_write) + + def cleanup(self): + atexit.unregister(self.cleanup) + + if self.process: + try: + self.writeline("__exit__") + time.sleep(.020) + self.process.kill() + except OSError: + pass + except IOError: + pass + self.process = None + self.stdin.close() + if self.stdin != self.stdout: + self.stdout.close() + self.stdin = None + self.stdout = None + if not IS_PY_3: + sys.exc_clear() + if self._stream_reader: + self._stream_reader.cleanup() + +def assert_prompt(runner, prompts, timeout, is_need_execute_result): + # Wait for the initial prompt + header = runner.read_to_prompt(prompts, timeout=timeout) + if not header and is_need_execute_result: + log(" ---------- will terminate cause the case needs result while there is none inside of buf. ----------") + sys.exit(1) + if not header == None: + if header: + log("Started with:\n%s" % header) + else: + log("Did not one of following prompt(s): %s" % repr(prompts)) + log(" Got : %s" % repr(r.buf)) + sys.exit(1) + + +### WebAssembly specific + +parser = argparse.ArgumentParser( + description="Run a test file against a WebAssembly interpreter") +parser.add_argument('--wast2wasm', type=str, + default=os.environ.get("WAST2WASM", "wast2wasm"), + help="Path to wast2wasm program") +parser.add_argument('--interpreter', type=str, + default=os.environ.get("IWASM_CMD", "iwasm"), + help="Path to WebAssembly interpreter") +parser.add_argument('--aot-compiler', type=str, + default=os.environ.get("WAMRC_CMD", "wamrc"), + help="Path to WebAssembly AoT compiler") + +parser.add_argument('--no_cleanup', action='store_true', + help="Keep temporary *.wasm files") + +parser.add_argument('--rundir', + help="change to the directory before running tests") +parser.add_argument('--start-timeout', default=30, type=int, + help="default timeout for initial prompt") +parser.add_argument('--test-timeout', default=20, type=int, + help="default timeout for each individual test action") +parser.add_argument('--no-pty', action='store_true', + help="Use direct pipes instead of pseudo-tty") +parser.add_argument('--log-file', type=str, + help="Write messages to the named file in addition the screen") +parser.add_argument('--log-dir', type=str, + help="The log directory to save the case file if test failed") +parser.add_argument('--debug-file', type=str, + help="Write all test interaction the named file") + +parser.add_argument('test_file', type=argparse.FileType('r'), + help="a WebAssembly *.wast test file") + +parser.add_argument('--aot', action='store_true', + help="Test with AOT") + +parser.add_argument('--target', type=str, + default="x86_64", + help="Set running target") + +parser.add_argument('--sgx', action='store_true', + help="Test SGX") + +parser.add_argument('--simd', default=False, action='store_true', + help="Enable SIMD") + +parser.add_argument('--xip', default=False, action='store_true', + help="Enable XIP") + +parser.add_argument('--eh', default=False, action='store_true', + help="Enable Exception Handling") + +parser.add_argument('--multi-module', default=False, action='store_true', + help="Enable Multi-thread") + +parser.add_argument('--multi-thread', default=False, action='store_true', + help="Enable Multi-thread") + +parser.add_argument('--gc', default=False, action='store_true', + help='Test with GC') + +parser.add_argument('--memory64', default=False, action='store_true', + help='Test with Memory64') + +parser.add_argument('--qemu', default=False, action='store_true', + help="Enable QEMU") + +parser.add_argument('--qemu-firmware', default='', help="Firmware required by qemu") + +parser.add_argument('--verbose', default=False, action='store_true', + help='show more logs') + +# regex patterns of tests to skip +C_SKIP_TESTS = () +PY_SKIP_TESTS = ( + # names.wast + 'invoke \"~!', + # conversions.wast + '18446742974197923840.0', + '18446744073709549568.0', + '9223372036854775808', + 'reinterpret_f.*nan', + # endianness + '.const 0x1.fff' ) + +def read_forms(string): + forms = [] + form = "" + depth = 0 + line = 0 + pos = 0 + while pos < len(string): + # Keep track of line number + if string[pos] == '\n': line += 1 + + # Handle top-level elements + if depth == 0: + # Add top-level comments + if string[pos:pos+2] == ";;": + end = string.find("\n", pos) + if end == -1: end == len(string) + forms.append(string[pos:end]) + pos = end + continue + + # TODO: handle nested multi-line comments + if string[pos:pos+2] == "(;": + # Skip multi-line comment + end = string.find(";)", pos) + if end == -1: + raise Exception("mismatch multiline comment on line %d: '%s'" % ( + line, string[pos:pos+80])) + pos = end+2 + continue + + # Ignore whitespace between top-level forms + if string[pos] in (' ', '\n', '\t'): + pos += 1 + continue + + # Read a top-level form + if string[pos] == '(': depth += 1 + if string[pos] == ')': depth -= 1 + if depth == 0 and not form: + raise Exception("garbage on line %d: '%s'" % ( + line, string[pos:pos+80])) + form += string[pos] + if depth == 0 and form: + forms.append(form) + form = "" + pos += 1 + return forms + +def get_module_exp_from_assert(string): + depth = 0 + pos = 0 + module = "" + exception = "" + start_record = False + result = [] + while pos < len(string): + # record from the " (module " + if string[pos:pos+7] == "(module": + start_record = True + if start_record: + if string[pos] == '(' : depth += 1 + if string[pos] == ')' : depth -= 1 + module += string[pos] + # if we get all (module ) . + if depth == 0 and module: + result.append(module) + start_record = False + # get expected exception + if string[pos] == '"': + end = string.find("\"", pos+1) + if end != -1: + end_rel = string.find("\"",end+1) + if end_rel == -1: + result.append(string[pos+1:end]) + pos += 1 + return result + +def string_to_unsigned(number_in_string, lane_type): + if not lane_type in ['i8x16', 'i16x8', 'i32x4', 'i64x2']: + raise Exception("invalid value {} and type {} and lane_type {}".format(number_in_string, type, lane_type)) + + number = int(number_in_string, 16) if '0x' in number_in_string else int(number_in_string) + + if "i8x16" == lane_type: + if number < 0: + packed = struct.pack('b', number) + number = struct.unpack('B', packed)[0] + elif "i16x8" == lane_type: + if number < 0: + packed = struct.pack('h', number) + number = struct.unpack('H', packed)[0] + elif "i32x4" == lane_type: + if number < 0: + packed = struct.pack('i', number) + number = struct.unpack('I', packed)[0] + else: # "i64x2" == lane_type: + if number < 0: + packed = struct.pack('q', number) + number = struct.unpack('Q', packed)[0] + + return number + +def cast_v128_to_i64x2(numbers, type, lane_type): + numbers = [n.replace("_", "") for n in numbers] + + if "i8x16" == lane_type: + assert(16 == len(numbers)), "{} should like {}".format(numbers, lane_type) + # str -> int + numbers = [string_to_unsigned(n, lane_type) for n in numbers] + # i8 -> i64 + packed = struct.pack(16 * "B", *numbers) + elif "i16x8" == lane_type: + assert(8 == len(numbers)), "{} should like {}".format(numbers, lane_type) + # str -> int + numbers = [string_to_unsigned(n, lane_type) for n in numbers] + # i16 -> i64 + packed = struct.pack(8 * "H", *numbers) + elif "i32x4" == lane_type: + assert(4 == len(numbers)), "{} should like {}".format(numbers, lane_type) + # str -> int + numbers = [string_to_unsigned(n, lane_type) for n in numbers] + # i32 -> i64 + packed = struct.pack(4 * "I", *numbers) + elif "i64x2" == lane_type: + assert(2 == len(numbers)), "{} should like {}".format(numbers, lane_type) + # str -> int + numbers = [string_to_unsigned(n, lane_type) for n in numbers] + # i64 -> i64 + packed = struct.pack(2 * "Q", *numbers) + elif "f32x4" == lane_type: + assert(4 == len(numbers)), "{} should like {}".format(numbers, lane_type) + # str -> int + numbers = [parse_simple_const_w_type(n, "f32")[0] for n in numbers] + # f32 -> i64 + packed = struct.pack(4 * "f", *numbers) + elif "f64x2" == lane_type: + assert(2 == len(numbers)), "{} should like {}".format(numbers, lane_type) + # str -> int + numbers = [parse_simple_const_w_type(n, "f64")[0] for n in numbers] + # f64 -> i64 + packed = struct.pack(2 * "d", *numbers) + else: + raise Exception("invalid value {} and type {} and lane_type {}".format(numbers, type, lane_type)) + + assert(packed) + unpacked = struct.unpack("Q Q", packed) + return unpacked, f"[{unpacked[0]:#x} {unpacked[1]:#x}]:{lane_type}:v128" + +def parse_simple_const_w_type(number, type): + number = number.replace('_', '') + number = re.sub(r"nan\((ind|snan)\)", "nan", number) + if type in ["i32", "i64"]: + number = int(number, 16) if '0x' in number else int(number) + return number, "0x{:x}:{}".format(number, type) \ + if number >= 0 \ + else "-0x{:x}:{}".format(0 - number, type) + elif type in ["f32", "f64"]: + if "nan:" in number: + return float('nan'), "nan:{}".format(type) + else: + number = float.fromhex(number) if '0x' in number else float(number) + return number, "{:.7g}:{}".format(number, type) + elif type == "ref.null": + if number == "func": + return "func", "func:ref.null" + elif number == "extern": + return "extern", "extern:ref.null" + elif number == "any": + return "any", "any:ref.null" + else: + raise Exception("invalid value {} and type {}".format(number, type)) + elif type == "ref.extern": + number = int(number, 16) if '0x' in number else int(number) + return number, "0x{:x}:ref.extern".format(number) + elif type == "ref.host": + number = int(number, 16) if '0x' in number else int(number) + return number, "0x{:x}:ref.host".format(number) + else: + raise Exception("invalid value {} and type {}".format(number, type)) + +def parse_assertion_value(val): + """ + Parse something like: + "ref.null extern" in (assert_return (invoke "get-externref" (i32.const 0)) (ref.null extern)) + "ref.extern 1" in (assert_return (invoke "get-externref" (i32.const 1)) (ref.extern 1)) + "i32.const 0" in (assert_return (invoke "is_null-funcref" (i32.const 1)) (i32.const 0)) + + in summary: + type.const (sub-type) (val1 val2 val3 val4) ... + type.const val + ref.extern val + ref.null ref_type + ref.array + ref.struct + ref.func + ref.i31 + """ + if not val: + return None, "" + + splitted = re.split('\s+', val) + splitted = [s for s in splitted if s] + type = splitted[0].split(".")[0] + lane_type = splitted[1] if len(splitted) > 2 else "" + numbers = splitted[2:] if len(splitted) > 2 else splitted[1:] + + if type in ["i32", "i64", "f32", "f64"]: + return parse_simple_const_w_type(numbers[0], type) + elif type == "ref": + if splitted[0] in ["ref.array", "ref.struct", "ref.func", "ref.i31"]: + return splitted[0] + # need to distinguish between "ref.null" and "ref.extern" + return parse_simple_const_w_type(numbers[0], splitted[0]) + else: + return cast_v128_to_i64x2(numbers, type, lane_type) + +def int2uint32(i): + return i & 0xffffffff + +def int2int32(i): + val = i & 0xffffffff + if val & 0x80000000: + return val - 0x100000000 + else: + return val + +def int2uint64(i): + return i & 0xffffffffffffffff + +def int2int64(i): + val = i & 0xffffffffffffffff + if val & 0x8000000000000000: + return val - 0x10000000000000000 + else: + return val + + +def num_repr(i): + if isinstance(i, int) or isinstance(i, long): + return re.sub("L$", "", hex(i)) + else: + return "%.16g" % i + +def hexpad16(i): + return "0x%04x" % i + +def hexpad24(i): + return "0x%06x" % i + +def hexpad32(i): + return "0x%08x" % i + +def hexpad64(i): + return "0x%016x" % i + +def invoke(r, args, cmd): + r.writeline(cmd) + + return r.read_to_prompt(['\r\nwebassembly> ', '\nwebassembly> '], + timeout=args.test_timeout) + +def vector_value_comparison(out, expected): + """ + out likes ":v128" + expected likes "[number number]:v128" + """ + # print("vector value comparision {} vs {}".format(out, expected)) + + out_val, out_type = out.split(':') + # => number number + out_val = out_val[1:-1] + + expected_val, lane_type, expected_type = expected.split(':') + # [number nubmer] => number number + expected_val = expected_val[1:-1] + + assert("v128" == out_type), "out_type should be v128" + assert("v128" == expected_type), "expected_type should be v128" + + if out_type != expected_type: + return False + + out_val = out_val.split(" ") + expected_val = expected_val.split(" ") + + # since i64x2 + out_packed = struct.pack("QQ", int(out_val[0], 16), int(out_val[1], 16)) + expected_packed = struct.pack("QQ", + int(expected_val[0]) if not "0x" in expected_val[0] else int(expected_val[0], 16), + int(expected_val[1]) if not "0x" in expected_val[1] else int(expected_val[1], 16)) + + if lane_type in ["i8x16", "i16x8", "i32x4", "i64x2"]: + return out_packed == expected_packed + else: + assert(lane_type in ["f32x4", "f64x2"]), "unexpected lane_type" + + if "f32x4" == lane_type: + out_unpacked = struct.unpack("ffff", out_packed) + expected_unpacked = struct.unpack("ffff", expected_packed) + else: + out_unpacked = struct.unpack("dd", out_packed) + expected_unpacked = struct.unpack("dd", expected_packed) + + out_is_nan = [math.isnan(o) for o in out_unpacked] + expected_is_nan = [math.isnan(e) for e in expected_unpacked] + if any(out_is_nan): + nan_comparision = [o == e for o, e in zip(out_is_nan, expected_is_nan)] + if all(nan_comparision): + print(f"Pass NaN comparision") + return True + + # print(f"compare {out_unpacked} and {expected_unpacked}") + result = [o == e for o, e in zip(out_unpacked, expected_unpacked)] + if not all(result): + result = [ + "{:.7g}".format(o) == "{:.7g}".format(e) + for o, e in zip(out_unpacked, expected_packed) + ] + + return all(result) + + +def simple_value_comparison(out, expected): + """ + compare out of simple types which may like val:i32, val:f64 and so on + """ + if expected == "2.360523e+13:f32" and out == "2.360522e+13:f32": + # one case in float_literals.wast, due to float precision of python + return True + + if expected == "1.797693e+308:f64" and out == "inf:f64": + # one case in float_misc.wast: + # (assert_return (invoke "f64.add" (f64.const 0x1.fffffffffffffp+1023) + # (f64.const 0x1.fffffffffffffp+969)) + # (f64.const 0x1.fffffffffffffp+1023)) + # the add result in x86_32 is inf + return True + + out_val, out_type = out.split(':') + expected_val, expected_type = expected.split(':') + + if not out_type == expected_type: + return False + + out_val, _ = parse_simple_const_w_type(out_val, out_type) + expected_val, _ = parse_simple_const_w_type(expected_val, expected_type) + + if out_val == expected_val \ + or (math.isnan(out_val) and math.isnan(expected_val)): + return True + + if "i32" == expected_type: + out_val_binary = struct.pack('I', out_val) if out_val > 0 \ + else struct.pack('i', out_val) + expected_val_binary = struct.pack('I', expected_val) \ + if expected_val > 0 \ + else struct.pack('i', expected_val) + elif "i64" == expected_type: + out_val_binary = struct.pack('Q', out_val) if out_val > 0 \ + else struct.pack('q', out_val) + expected_val_binary = struct.pack('Q', expected_val) \ + if expected_val > 0 \ + else struct.pack('q', expected_val) + elif "f32" == expected_type: + out_val_binary = struct.pack('f', out_val) + expected_val_binary = struct.pack('f', expected_val) + elif "f64" == expected_type: + out_val_binary = struct.pack('d', out_val) + expected_val_binary = struct.pack('d', expected_val) + elif "ref.extern" == expected_type: + out_val_binary = out_val + expected_val_binary = expected_val + elif "ref.host" == expected_type: + out_val_binary = out_val + expected_val_binary = expected_val + else: + assert(0), "unknown 'expected_type' {}".format(expected_type) + + if out_val_binary == expected_val_binary: + return True + + if expected_type in ["f32", "f64"]: + # compare with a lower precision + out_str = "{:.7g}".format(out_val) + expected_str = "{:.7g}".format(expected_val) + if out_str == expected_str: + return True + + return False + +def value_comparison(out, expected): + if out == expected: + return True + + if not expected: + return False + + if not out in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]: + assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out) + if not expected in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]: + assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected) + + if 'v128' in out: + return vector_value_comparison(out, expected) + else: + return simple_value_comparison(out, expected) + +def is_result_match_expected(out, expected): + # compare value instead of comparing strings of values + return value_comparison(out, expected) + +def test_assert(r, opts, mode, cmd, expected): + log("Testing(%s) %s = %s" % (mode, cmd, expected)) + out = invoke(r, opts, cmd) + if '\n' in out or ' ' in out: + outs = [''] + out.split('\n')[1:] + out = outs[-1] + + if mode=='trap': + o = re.sub('^Exception: ', '', out) + e = re.sub('^Exception: ', '', expected) + if o.find(e) >= 0 or e.find(o) >= 0: + return True + + if mode=='exhaustion': + o = re.sub('^Exception: ', '', out) + expected = 'Exception: stack overflow' + e = re.sub('^Exception: ', '', expected) + if o.find(e) >= 0 or e.find(o) >= 0: + return True + + # wasm-exception thrown out of function call, not a trap + if mode=='wasmexception': + o = re.sub('^Exception: ', '', out) + e = re.sub('^Exception: ', '', expected) + if o.find(e) >= 0 or e.find(o) >= 0: + return True + + ## 0x9:i32,-0x1:i32 -> ['0x9:i32', '-0x1:i32'] + expected_list = re.split(',', expected) + out_list = re.split(',', out) + if len(expected_list) != len(out_list): + raise Exception("Failed:\n Results count incorrect:\n expected: '%s'\n got: '%s'" % (expected, out)) + for i in range(len(expected_list)): + if not is_result_match_expected(out_list[i], expected_list[i]): + raise Exception("Failed:\n Result %d incorrect:\n expected: '%s'\n got: '%s'" % (i, expected_list[i], out_list[i])) + + return True + +def test_assert_return(r, opts, form): + """ + m. to search a pattern like (assert_return (invoke function_name ... ) ...) + n. to search a pattern like (assert_return (invoke $module_name function_name ... ) ...) + """ + # params, return + m = re.search('^\(assert_return\s+\(invoke\s+"((?:[^"]|\\\")*)"\s+(\(.*\))\s*\)\s*(\(.*\))\s*\)\s*$', form, re.S) + # judge if assert_return cmd includes the module name + n = re.search('^\(assert_return\s+\(invoke\s+\$((?:[^\s])*)\s+"((?:[^"]|\\\")*)"\s+(\(.*\))\s*\)\s*(\(.*\))\s*\)\s*$', form, re.S) + + # print("assert_return with {}".format(form)) + + if not m: + # no params, return + m = re.search('^\(assert_return\s+\(invoke\s+"((?:[^"]|\\\")*)"\s*\)\s+()(\(.*\))\s*\)\s*$', form, re.S) + if not m: + # params, no return + m = re.search('^\(assert_return\s+\(invoke\s+"([^"]*)"\s+(\(.*\))()\s*\)\s*\)\s*$', form, re.S) + if not m: + # no params, no return + m = re.search('^\(assert_return\s+\(invoke\s+"([^"]*)"\s*()()\)\s*\)\s*$', form, re.S) + if not m: + # params, return + if not n: + # no params, return + n = re.search('^\(assert_return\s+\(invoke\s+\$((?:[^\s])*)\s+"((?:[^"]|\\\")*)"\s*\)\s+()(\(.*\))\s*\)\s*$', form, re.S) + if not n: + # params, no return + n = re.search('^\(assert_return\s+\(invoke\s+\$((?:[^\s])*)\s+"([^"]*)"\s+(\(.*\))()\s*\)\s*\)\s*$', form, re.S) + if not n: + # no params, no return + n = re.search('^\(assert_return\s+\(invoke\s+\$((?:[^\s])*)\s+"([^"]*)"*()()\)\s*\)\s*$', form, re.S) + if not m and not n: + if re.search('^\(assert_return\s+\(get.*\).*\)$', form, re.S): + log("ignoring assert_return get") + return + else: + raise Exception("unparsed assert_return: '%s'" % form) + if m and not n: + func = m.group(1) + if ' ' in func: + func = func.replace(' ', '\\') + + if m.group(2) == '': + args = [] + else: + #args = [re.split(' +', v)[1].replace('_', "") for v in re.split("\)\s*\(", m.group(2)[1:-1])] + # split arguments with ')spaces(', remove leading and tailing ) and ( + args_type_and_value = re.split(r'\)\s+\(', m.group(2)[1:-1]) + args_type_and_value = [s.replace('_', '') for s in args_type_and_value] + # args are in two forms: + # f32.const -0x1.000001fffffffffffp-50 + # v128.const i32x4 0 0 0 0 + args = [] + for arg in args_type_and_value: + # remove leading and tailing spaces, it might confuse following assertions + arg = arg.strip() + splitted = re.split('\s+', arg) + splitted = [s for s in splitted if s] + + if splitted[0] in ["i32.const", "i64.const"]: + assert(2 == len(splitted)), "{} should have two parts".format(splitted) + # in wast 01234 means 1234 + # in c 0123 means 83 in oct + number, _ = parse_simple_const_w_type(splitted[1], splitted[0][:3]) + args.append(str(number)) + elif splitted[0] in ["f32.const", "f64.const"]: + # let strtof or strtod handle original arguments + assert(2 == len(splitted)), "{} should have two parts".format(splitted) + args.append(splitted[1]) + elif "v128.const" == splitted[0]: + assert(len(splitted) > 2), "{} should have more than two parts".format(splitted) + numbers, _ = cast_v128_to_i64x2(splitted[2:], 'v128', splitted[1]) + + assert(len(numbers) == 2), "has to reform arguments into i64x2" + args.append(f"{numbers[0]:#x}\{numbers[1]:#x}") + elif "ref.null" == splitted[0]: + args.append("null") + elif "ref.extern" == splitted[0]: + number, _ = parse_simple_const_w_type(splitted[1], splitted[0]) + args.append(str(number)) + elif "ref.host" == splitted[0]: + number, _ = parse_simple_const_w_type(splitted[1], splitted[0]) + args.append(str(number)) + else: + assert(0), "an unkonwn parameter type" + + if m.group(3) == '': + returns= [] + else: + returns = re.split("\)\s*\(", m.group(3)[1:-1]) + # processed numbers in strings + if len(returns) == 1 and returns[0] in ["ref.array", "ref.struct", "ref.i31", + "ref.eq", "ref.any", "ref.extern", + "ref.func", "ref.null"]: + expected = [returns[0]] + elif len(returns) == 1 and returns[0] in ["func:ref.null", "any:ref.null", + "extern:ref.null"]: + expected = [returns[0]] + else: + expected = [parse_assertion_value(v)[1] for v in returns] + expected = ",".join(expected) + + test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected) + elif not m and n: + module = temp_module_table[n.group(1)].split(".wasm")[0] + # assume the cmd is (assert_return(invoke $ABC "func")). + # run the ABC.wasm firstly + if test_aot: + r = compile_wasm_to_aot(module+".wasm", module+".aot", True, opts, r) + try: + assert_prompt(r, ['Compile success'], opts.start_timeout, False) + except: + _, exc, _ = sys.exc_info() + log("Run wamrc failed:\n got: '%s'" % r.buf) + ret_code = 1 + sys.exit(1) + r = run_wasm_with_repl(module+".wasm", module+".aot" if test_aot else module, opts, r) + # Wait for the initial prompt + try: + assert_prompt(r, ['webassembly> '], opts.start_timeout, False) + except: + _, exc, _ = sys.exc_info() + raise Exception("Failed:\n expected: '%s'\n got: '%s'" % \ + (repr(exc), r.buf)) + func = n.group(2) + if ' ' in func: + func = func.replace(' ', '\\') + + if n.group(3) == '': + args=[] + else: + # convert (ref.null extern/func) into (ref.null null) + n1 = n.group(3).replace("(ref.null extern)", "(ref.null null)") + n1 = n1.replace("ref.null func)", "(ref.null null)") + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n1[1:-1])] + + _, expected = parse_assertion_value(n.group(4)[1:-1]) + test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected) + +def test_assert_trap(r, opts, form): + # params + m = re.search('^\(assert_trap\s+\(invoke\s+"([^"]*)"\s+(\(.*\))\s*\)\s*"([^"]+)"\s*\)\s*$', form) + # judge if assert_return cmd includes the module name + n = re.search('^\(assert_trap\s+\(invoke\s+\$((?:[^\s])*)\s+"([^"]*)"\s+(\(.*\))\s*\)\s*"([^"]+)"\s*\)\s*$', form, re.S) + if not m: + # no params + m = re.search('^\(assert_trap\s+\(invoke\s+"([^"]*)"\s*()\)\s*"([^"]+)"\s*\)\s*$', form) + if not m: + if not n: + # no params + n = re.search('^\(assert_trap\s+\(invoke\s+\$((?:[^\s])*)\s+"([^"]*)"\s*()\)\s*"([^"]+)"\s*\)\s*$', form, re.S) + if not m and not n: + raise Exception("unparsed assert_trap: '%s'" % form) + + if m and not n: + func = m.group(1) + if m.group(2) == '': + args = [] + else: + # convert (ref.null extern/func) into (ref.null null) + m1 = m.group(2).replace("(ref.null extern)", "(ref.null null)") + m1 = m1.replace("ref.null func)", "(ref.null null)") + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m1[1:-1])] + + expected = "Exception: %s" % m.group(3) + test_assert(r, opts, "trap", "%s %s" % (func, " ".join(args)), expected) + + elif not m and n: + module = n.group(1) + module = tempfile.gettempdir() + "/" + module + + # will trigger the module named in assert_return(invoke $ABC). + # run the ABC.wasm firstly + if test_aot: + r = compile_wasm_to_aot(module+".wasm", module+".aot", True, opts, r) + try: + assert_prompt(r, ['Compile success'], opts.start_timeout, False) + except: + _, exc, _ = sys.exc_info() + log("Run wamrc failed:\n got: '%s'" % r.buf) + ret_code = 1 + sys.exit(1) + r = run_wasm_with_repl(module+".wasm", module+".aot" if test_aot else module, opts, r) + # Wait for the initial prompt + try: + assert_prompt(r, ['webassembly> '], opts.start_timeout, False) + except: + _, exc, _ = sys.exc_info() + raise Exception("Failed:\n expected: '%s'\n got: '%s'" % \ + (repr(exc), r.buf)) + + func = n.group(2) + if n.group(3) == '': + args = [] + else: + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n.group(3)[1:-1])] + expected = "Exception: %s" % n.group(4) + test_assert(r, opts, "trap", "%s %s" % (func, " ".join(args)), expected) + +def test_assert_exhaustion(r,opts,form): + # params + m = re.search('^\(assert_exhaustion\s+\(invoke\s+"([^"]*)"\s+(\(.*\))\s*\)\s*"([^"]+)"\s*\)\s*$', form) + if not m: + # no params + m = re.search('^\(assert_exhaustion\s+\(invoke\s+"([^"]*)"\s*()\)\s*"([^"]+)"\s*\)\s*$', form) + if not m: + raise Exception("unparsed assert_exhaustion: '%s'" % form) + func = m.group(1) + if m.group(2) == '': + args = [] + else: + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m.group(2)[1:-1])] + expected = "Exception: %s\n" % m.group(3) + test_assert(r, opts, "exhaustion", "%s %s" % (func, " ".join(args)), expected) + + +# added to support WASM_ENABLE_EXCE_HANDLING +def test_assert_wasmexception(r,opts,form): + # params + + # ^ + # \(assert_exception\s+ + # \(invoke\s+"([^"]+)"\s+ + # (\(.*\))\s* + # () + # \)\s* + # \)\s* + # $ + m = re.search('^\(assert_exception\s+\(invoke\s+"([^"]+)"\s+(\(.*\))\s*\)\s*\)\s*$', form) + if not m: + # no params + + # ^ + # \(assert_exception\s+ + # \(invoke\s+"([^"]+)"\s* + # () + # \)\s* + # \)\s* + # $ + m = re.search('^\(assert_exception\s+\(invoke\s+"([^"]+)"\s*()\)\s*\)\s*$', form) + if not m: + raise Exception("unparsed assert_exception: '%s'" % form) + func = m.group(1) # function name + if m.group(2) == '': # arguments + args = [] + else: + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m.group(2)[1:-1])] + + expected = "Exception: uncaught wasm exception\n" + test_assert(r, opts, "wasmexception", "%s %s" % (func, " ".join(args)), expected) + +def do_invoke(r, opts, form): + # params + m = re.search('^\(invoke\s+"([^"]+)"\s+(\(.*\))\s*\)\s*$', form) + if not m: + # no params + m = re.search('^\(invoke\s+"([^"]+)"\s*()\)\s*$', form) + if not m: + raise Exception("unparsed invoke: '%s'" % form) + func = m.group(1) + + if ' ' in func: + func = func.replace(' ', '\\') + + if m.group(2) == '': + args = [] + else: + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m.group(2)[1:-1])] + + log("Invoking %s(%s)" % ( + func, ", ".join([str(a) for a in args]))) + + invoke(r, opts, "%s %s" % (func, " ".join(args))) + +def skip_test(form, skip_list): + for s in skip_list: + if re.search(s, form): + return True + return False + +def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts): + log("Writing WAST module to '%s'" % wast_tempfile) + with open(wast_tempfile, 'w') as file: + file.write(form) + log("Compiling WASM to '%s'" % wasm_tempfile) + + # default arguments + if opts.gc or opts.memory64: + cmd = [opts.wast2wasm, "-u", "-d", wast_tempfile, "-o", wasm_tempfile] + elif opts.eh: + cmd = [opts.wast2wasm, "--enable-thread", "--no-check", "--enable-exceptions", "--enable-tail-call", wast_tempfile, "-o", wasm_tempfile ] + else: + cmd = [opts.wast2wasm, "--enable-thread", "--no-check", + wast_tempfile, "-o", wasm_tempfile ] + + # remove reference-type and bulk-memory enabling options since a WABT + # commit 30c1e983d30b33a8004b39fd60cbd64477a7956c + # Enable reference types by default (#1729) + + log("Running: %s" % " ".join(cmd)) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + print(str(e)) + return False + + return True + +def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r, output = 'default'): + log("Compiling '%s' to '%s'" % (wasm_tempfile, aot_tempfile)) + cmd = [opts.aot_compiler] + + if test_target in aot_target_options_map: + cmd += aot_target_options_map[test_target] + + if opts.sgx: + cmd.append("-sgx") + + if not opts.simd: + cmd.append("--disable-simd") + + if opts.xip: + cmd.append("--enable-indirect-mode") + cmd.append("--disable-llvm-intrinsics") + + if opts.multi_thread: + cmd.append("--enable-multi-thread") + + if opts.gc: + cmd.append("--enable-gc") + cmd.append("--enable-tail-call") + + if opts.memory64: + cmd.append("--enable-memory64") + + if output == 'object': + cmd.append("--format=object") + elif output == 'ir': + cmd.append("--format=llvmir-opt") + + # disable llvm link time optimization as it might convert + # code of tail call into code of dead loop, and stack overflow + # exception isn't thrown in several cases + cmd.append("--disable-llvm-lto") + + # Bounds checks is disabled by default for 64-bit targets, to + # use the hardware based bounds checks. But it is not supported + # in QEMU with NuttX. + # Enable bounds checks explicitly for all targets if running in QEMU. + if opts.qemu: + cmd.append("--bounds-checks=1") + + # RISCV64 requires -mcmodel=medany, which can be set by --size-level=1 + if test_target.startswith("riscv64"): + cmd.append("--size-level=1") + + cmd += ["-o", aot_tempfile, wasm_tempfile] + + log("Running: %s" % " ".join(cmd)) + if not runner: + subprocess.check_call(cmd) + else: + if (r != None): + r.cleanup() + r = Runner(cmd, no_pty=opts.no_pty) + return r + +def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): + tmpfile = aot_tempfile if test_aot else wasm_tempfile + log("Starting interpreter for module '%s'" % tmpfile) + + cmd_iwasm = [opts.interpreter, "--heap-size=0", "-v=5" if opts.verbose else "-v=0", "--repl", tmpfile] + + if opts.multi_module: + cmd_iwasm.insert(1, "--module-path=" + (tempfile.gettempdir() if not opts.qemu else "/tmp" )) + + if opts.qemu: + if opts.qemu_firmware == '': + raise Exception("QEMU firmware missing") + + if opts.target.startswith("aarch64"): + cmd = "qemu-system-aarch64 -cpu cortex-a53 -nographic -machine virt,virtualization=on,gic-version=3 -net none -chardev stdio,id=con,mux=on -serial chardev:con -mon chardev=con,mode=readline -kernel".split() + cmd.append(opts.qemu_firmware) + elif opts.target.startswith("thumbv7"): + cmd = "qemu-system-arm -semihosting -M sabrelite -m 1024 -smp 1 -nographic -kernel".split() + cmd.append(opts.qemu_firmware) + elif opts.target.startswith("riscv32"): + cmd = "qemu-system-riscv32 -semihosting -M virt,aclint=on -cpu rv32 -smp 1 -nographic -bios none -kernel".split() + cmd.append(opts.qemu_firmware) + elif opts.target.startswith("riscv64"): + cmd = "qemu-system-riscv64 -semihosting -M virt,aclint=on -cpu rv64 -smp 1 -nographic -bios none -kernel".split() + cmd.append(opts.qemu_firmware) + else: + raise Exception("Unknwon target for QEMU: %s" % opts.target) + + else: + cmd = cmd_iwasm + + log("Running: %s" % " ".join(cmd)) + if (r != None): + r.cleanup() + r = Runner(cmd, no_pty=opts.no_pty) + + if opts.qemu: + r.read_to_prompt(['nsh> '], 10) + r.writeline("mount -t hostfs -o fs={} /tmp".format(tempfile.gettempdir())) + r.read_to_prompt(['nsh> '], 10) + r.writeline(" ".join(cmd_iwasm)) + + return r + +def create_tmpfiles(wast_name): + tempfiles = [] + + tempfiles.append(create_tmp_file(".wast")) + tempfiles.append(create_tmp_file(".wasm")) + if test_aot: + tempfiles.append(create_tmp_file(".aot")) + + # add these temp file to temporal repo, will be deleted when finishing the test + temp_file_repo.extend(tempfiles) + return tempfiles + +def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, opts, r, loadable = True): + details_inside_ast = get_module_exp_from_assert(form) + log("module is ....'%s'"%details_inside_ast[0]) + log("exception is ....'%s'"%details_inside_ast[1]) + # parse the module + module = details_inside_ast[0] + expected = details_inside_ast[1] + + if not compile_wast_to_wasm(module, wast_tempfile, wasm_tempfile, opts): + raise Exception("compile wast to wasm failed") + + if test_aot: + r = compile_wasm_to_aot(wasm_tempfile, aot_tempfile, True, opts, r) + try: + assert_prompt(r, ['Compile success'], opts.start_timeout, True) + except: + _, exc, _ = sys.exc_info() + if (r.buf.find(expected) >= 0): + log("Out exception includes expected one, pass:") + log(" Expected: %s" % expected) + log(" Got: %s" % r.buf) + return + else: + log("Run wamrc failed:\n expected: '%s'\n got: '%s'" % \ + (expected, r.buf)) + ret_code = 1 + sys.exit(1) + + r = run_wasm_with_repl(wasm_tempfile, aot_tempfile if test_aot else None, opts, r) + + # Some module couldn't load so will raise an error directly, so shell prompt won't show here + + if loadable: + # Wait for the initial prompt + try: + assert_prompt(r, ['webassembly> '], opts.start_timeout, True) + except: + _, exc, _ = sys.exc_info() + if (r.buf.find(expected) >= 0): + log("Out exception includes expected one, pass:") + log(" Expected: %s" %expected) + log(" Got: %s" % r.buf) + else: + raise Exception("Failed:\n expected: '%s'\n got: '%s'" % \ + (expected, r.buf)) + +if __name__ == "__main__": + opts = parser.parse_args(sys.argv[1:]) + # print('Input param :',opts) + + if opts.aot: test_aot = True + # default x86_64 + test_target = opts.target + + if opts.rundir: os.chdir(opts.rundir) + + if opts.log_file: log_file = open(opts.log_file, "a") + if opts.debug_file: debug_file = open(opts.debug_file, "a") + + if opts.interpreter.endswith(".py"): + SKIP_TESTS = PY_SKIP_TESTS + else: + SKIP_TESTS = C_SKIP_TESTS + + wast_tempfile = create_tmp_file(".wast") + wasm_tempfile = create_tmp_file(".wasm") + if test_aot: + aot_tempfile = create_tmp_file(".aot") + + ret_code = 0 + try: + log("\n################################################") + log("### Testing %s" % opts.test_file.name) + log("################################################") + forms = read_forms(opts.test_file.read()) + r = None + + for form in forms: + # log("\n### Current Case is " + form + "\n") + if ";;" == form[0:2]: + log(form) + elif skip_test(form, SKIP_TESTS): + log("Skipping test: %s" % form[0:60]) + elif re.match("^\(assert_trap\s+\(module", form): + test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile if test_aot else None, opts, r) + elif re.match("^\(assert_exhaustion\\b.*", form): + test_assert_exhaustion(r, opts, form) + elif re.match("^\(assert_exception\\b.*", form): + test_assert_wasmexception(r, opts, form) + elif re.match("^\(assert_unlinkable\\b.*", form): + test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile if test_aot else None, opts, r, False) + elif re.match("^\(assert_malformed\\b.*", form): + # remove comments in wast + form,n = re.subn(";;.*\n", "", form) + m = re.match("^\(assert_malformed\s*\(module binary\s*(\".*\").*\)\s*\"(.*)\"\s*\)$", form, re.DOTALL) + + if m: + # workaround: spec test changes error message to "malformed" while iwasm still use "invalid" + error_msg = m.group(2).replace("malformed", "invalid") + log("Testing(malformed)") + with open(wasm_tempfile, 'wb') as f: + s = m.group(1) + while s: + res = re.match("[^\"]*\"([^\"]*)\"(.*)", s, re.DOTALL) + if IS_PY_3: + context = res.group(1).replace("\\", "\\x").encode("latin1").decode("unicode-escape").encode("latin1") + f.write(context) + else: + f.write(res.group(1).replace("\\", "\\x").decode("string-escape")) + s = res.group(2) + + # compile wasm to aot + if test_aot: + r = compile_wasm_to_aot(wasm_tempfile, aot_tempfile, True, opts, r) + try: + assert_prompt(r, ['Compile success'], opts.start_timeout, True) + except: + _, exc, _ = sys.exc_info() + if (r.buf.find(error_msg) >= 0): + log("Out exception includes expected one, pass:") + log(" Expected: %s" % error_msg) + log(" Got: %s" % r.buf) + continue + # one case in binary.wast + elif (error_msg == "unexpected end of section or function" + and r.buf.find("unexpected end")): + continue + # one case in binary.wast + elif (error_msg == "invalid value type" + and r.buf.find("unexpected end")): + continue + # one case in binary.wast + elif (error_msg == "integer too large" + and r.buf.find("tables cannot be shared")): + continue + # one case in binary.wast + elif (error_msg == "zero byte expected" + and r.buf.find("unknown table")): + continue + # one case in binary.wast + elif (error_msg == "invalid section id" + and r.buf.find("unexpected end of section or function")): + continue + # one case in binary.wast + elif (error_msg == "illegal opcode" + and r.buf.find("unexpected end of section or function")): + continue + # one case in custom.wast + elif (error_msg == "length out of bounds" + and r.buf.find("unexpected end")): + continue + # several cases in binary-leb128.wast + elif (error_msg == "integer representation too long" + and r.buf.find("invalid section id")): + continue + else: + log("Run wamrc failed:\n expected: '%s'\n got: '%s'" % \ + (error_msg, r.buf)) + ret_code = 1 + sys.exit(1) + + r = run_wasm_with_repl(wasm_tempfile, aot_tempfile if test_aot else None, opts, r) + + elif re.match("^\(assert_malformed\s*\(module quote", form): + log("ignoring assert_malformed module quote") + else: + log("unrecognized assert_malformed") + elif re.match("^\(assert_return[_a-z]*_nan\\b.*", form): + log("ignoring assert_return_.*_nan") + pass + elif re.match(".*\(invoke\s+\$\\b.*", form): + # invoke a particular named module's function + if form.startswith("(assert_return"): + test_assert_return(r,opts,form) + elif form.startswith("(assert_trap"): + test_assert_trap(r,opts,form) + elif re.match("^\(module\\b.*", form): + # if the module includes the particular name startswith $ + m = re.search("^\(module\s+\$.\S+", form) + if m: + # get module name + module_name = re.split('\$', m.group(0).strip())[1] + if module_name: + # create temporal files + temp_files = create_tmpfiles(module_name) + if not compile_wast_to_wasm(form, temp_files[0], temp_files[1], opts): + raise Exception("compile wast to wasm failed") + + if test_aot: + r = compile_wasm_to_aot(temp_files[1], temp_files[2], True, opts, r) + try: + assert_prompt(r, ['Compile success'], opts.start_timeout, False) + except: + _, exc, _ = sys.exc_info() + log("Run wamrc failed:\n got: '%s'" % r.buf) + ret_code = 1 + sys.exit(1) + temp_module_table[module_name] = temp_files[1] + r = run_wasm_with_repl(temp_files[1], temp_files[2] if test_aot else None, opts, r) + else: + if not compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts): + raise Exception("compile wast to wasm failed") + + if test_aot: + r = compile_wasm_to_aot(wasm_tempfile, aot_tempfile, True, opts, r) + try: + assert_prompt(r, ['Compile success'], opts.start_timeout, False) + except: + _, exc, _ = sys.exc_info() + log("Run wamrc failed:\n got: '%s'" % r.buf) + ret_code = 1 + sys.exit(1) + + r = run_wasm_with_repl(wasm_tempfile, aot_tempfile if test_aot else None, opts, r) + + # Wait for the initial prompt + try: + assert_prompt(r, ['webassembly> '], opts.start_timeout, False) + except: + _, exc, _ = sys.exc_info() + raise Exception("Failed:\n expected: '%s'\n got: '%s'" % \ + (repr(exc), r.buf)) + + elif re.match("^\(assert_return\\b.*", form): + assert(r), "iwasm repl runtime should be not null" + test_assert_return(r, opts, form) + elif re.match("^\(assert_trap\\b.*", form): + test_assert_trap(r, opts, form) + elif re.match("^\(invoke\\b.*", form): + assert(r), "iwasm repl runtime should be not null" + do_invoke(r, opts, form) + elif re.match("^\(assert_invalid\\b.*", form): + test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile if test_aot else None, opts, r) + elif re.match("^\(register\\b.*", form): + # get module's new name from the register cmd + name_new =re.split('\"',re.search('\".*\"',form).group(0))[1] + if name_new: + new_module = os.path.join(tempfile.gettempdir(), name_new + ".wasm") + shutil.copyfile(temp_module_table.get(name_new, wasm_tempfile), new_module) + + # add new_module copied from the old into temp_file_repo[] + temp_file_repo.append(new_module) + + if test_aot: + new_module_aot = os.path.join(tempfile.gettempdir(), name_new + ".aot") + r = compile_wasm_to_aot(new_module, new_module_aot, True, opts, r) + try: + assert_prompt(r, ['Compile success'], opts.start_timeout, True) + except: + raise Exception("compile wasm to aot failed") + # add aot module into temp_file_repo[] + temp_file_repo.append(new_module_aot) + else: + # there is no name defined in register cmd + raise Exception("can not find module name from the register") + else: + raise Exception("unrecognized form '%s...'" % form[0:40]) + except Exception as e: + traceback.print_exc() + print("THE FINAL EXCEPTION IS {}".format(e)) + ret_code = 101 + + shutil.copyfile(wasm_tempfile, os.path.join(opts.log_dir, os.path.basename(wasm_tempfile))) + + if opts.aot or opts.xip: + shutil.copyfile(aot_tempfile, os.path.join(opts.log_dir,os.path.basename(aot_tempfile))) + if "indirect-mode" in str(e): + compile_wasm_to_aot(wasm_tempfile, aot_tempfile, None, opts, None, "object") + shutil.copyfile(aot_tempfile, os.path.join(opts.log_dir,os.path.basename(aot_tempfile)+'.o')) + subprocess.check_call(["llvm-objdump", "-r", aot_tempfile]) + compile_wasm_to_aot(wasm_tempfile, aot_tempfile, None, opts, None, "ir") + shutil.copyfile(aot_tempfile, os.path.join(opts.log_dir,os.path.basename(aot_tempfile)+".ir")) + + else: + ret_code = 0 + finally: + if not opts.no_cleanup: + log("Removing tempfiles") + os.remove(wast_tempfile) + os.remove(wasm_tempfile) + if test_aot: + os.remove(aot_tempfile) + + # remove the files under /tempfiles/ and copy of .wasm files + if temp_file_repo: + for t in temp_file_repo: + if(len(str(t))!=0 and os.path.exists(t)): + os.remove(t) + + log("### End testing %s" % opts.test_file.name) + else: + log("Leaving tempfiles: %s" % ([wast_tempfile, wasm_tempfile])) + + sys.exit(ret_code) diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/simd_ignore_cases.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/simd_ignore_cases.patch new file mode 100644 index 0000000..87d3871 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/simd_ignore_cases.patch @@ -0,0 +1,73 @@ +diff --git a/test/core/simd/simd_lane.wast b/test/core/simd/simd_lane.wast +index 9d4b5fd7..4656dd2b 100644 +--- a/test/core/simd/simd_lane.wast ++++ b/test/core/simd/simd_lane.wast +@@ -602,23 +602,23 @@ + "(v128.const i8x16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0) " + "(v128.const i8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)))") "invalid lane length") + (assert_malformed (module quote "(func (result v128) " +- "(i8x16.shuffle 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15.0) " ++ "(i8x16.shuffle 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15.0 " + "(v128.const i8x16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0) " + "(v128.const i8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)))") "malformed lane index") + (assert_malformed (module quote "(func (result v128) " +- "(i8x16.shuffle 0.5 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) " ++ "(i8x16.shuffle 0.5 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 " + "(v128.const i8x16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0) " + "(v128.const i8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)))") "malformed lane index") + (assert_malformed (module quote "(func (result v128) " +- "(i8x16.shuffle -inf 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) " ++ "(i8x16.shuffle -inf 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 " + "(v128.const i8x16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0) " + "(v128.const i8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)))") "malformed lane index") + (assert_malformed (module quote "(func (result v128) " +- "(i8x16.shuffle 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 inf) " ++ "(i8x16.shuffle 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 inf " + "(v128.const i8x16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0) " + "(v128.const i8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)))") "malformed lane index") + (assert_malformed (module quote "(func (result v128) " +- "(i8x16.shuffle nan 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) " ++ "(i8x16.shuffle nan 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 " + "(v128.const i8x16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0) " + "(v128.const i8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)))") "malformed lane index") + +@@ -858,7 +858,7 @@ + (assert_return (invoke "as-if-condition-value" (v128.const i8x16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0)) (i32.const 0)) + (assert_return (invoke "as-return-value-1" (v128.const i16x8 0 0 0 0 0 0 0 0) (i32.const 1)) (v128.const i16x8 1 0 0 0 0 0 0 0)) + (assert_return (invoke "as-local_set-value" (v128.const i32x4 -1 -1 -1 -1)) (i32.const -1)) +-(assert_return (invoke "as-global_set-value-1" (v128.const f32x4 0 0 0 0)(f32.const 3.14)) (v128.const f32x4 3.14 0 0 0)) ++(assert_return (invoke "as-global_set-value-1" (v128.const f32x4 0 0 0 0) (f32.const 3.14)) (v128.const f32x4 3.14 0 0 0)) + + (assert_return (invoke "as-return-value-2" + (v128.const i8x16 -16 -15 -14 -13 -12 -11 -10 -9 -8 -7 -6 -5 -4 -3 -2 -1) +@@ -870,7 +870,7 @@ + (v128.const i8x16 -16 -15 -14 -13 -12 -11 -10 -9 8 7 6 5 4 3 2 1)) + + (assert_return (invoke "as-local_set-value-1" (v128.const i64x2 -1 -1)) (i64.const -1)) +-(assert_return (invoke "as-global_set-value-3" (v128.const f64x2 0 0)(f64.const 3.14)) (v128.const f64x2 3.14 0)) ++(assert_return (invoke "as-global_set-value-3" (v128.const f64x2 0 0) (f64.const 3.14)) (v128.const f64x2 3.14 0)) + + ;; Non-nat lane index + +diff --git a/test/core/simd/simd_load.wast b/test/core/simd/simd_load.wast +index 4b2edc16..c7639218 100644 +--- a/test/core/simd/simd_load.wast ++++ b/test/core/simd/simd_load.wast +@@ -124,7 +124,7 @@ + (i8x16.swizzle (v128.load (i32.const 0)) (v128.load offset=15 (i32.const 1))) + ) + ) +-(assert_return(invoke "as-i8x16.swizzle-operand") (v128.const i8x16 115 114 113 112 111 110 109 108 107 106 105 104 103 102 101 100)) ++(assert_return (invoke "as-i8x16.swizzle-operand") (v128.const i8x16 115 114 113 112 111 110 109 108 107 106 105 104 103 102 101 100)) + + (module (memory 1) + (data (i32.const 0) "\00\01\02\03\04\05\06\07\08\09\0a\0b\0c\0d\0e\0f\00\01\02\03") +@@ -180,7 +180,7 @@ + + (assert_invalid + (module (memory 1) (func (drop (v128.load (local.get 2))))) +- "unknown local 2" ++ "unknown local" + ) + (assert_invalid + (module (memory 1) (func (drop (v128.load)))) diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/thread_proposal_fix_atomic_case.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/thread_proposal_fix_atomic_case.patch new file mode 100644 index 0000000..a64dd17 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/thread_proposal_fix_atomic_case.patch @@ -0,0 +1,49 @@ +diff --git a/test/core/atomic.wast b/test/core/atomic.wast +index 66ad0eb..40259a9 100644 +--- a/test/core/atomic.wast ++++ b/test/core/atomic.wast +@@ -324,7 +324,7 @@ + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i32.atomic.rmw8.cmpxchg_u" (i32.const 0) (i32.const 0x11111111) (i32.const 0xcdcdcdcd)) (i32.const 0x11)) +-(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x1111111111111111)) ++(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x11111111111111cd)) + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i32.atomic.rmw16.cmpxchg_u" (i32.const 0) (i32.const 0) (i32.const 0xcafecafe)) (i32.const 0x1111)) +@@ -332,7 +332,7 @@ + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i32.atomic.rmw16.cmpxchg_u" (i32.const 0) (i32.const 0x11111111) (i32.const 0xcafecafe)) (i32.const 0x1111)) +-(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x1111111111111111)) ++(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x111111111111cafe)) + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i64.atomic.rmw8.cmpxchg_u" (i32.const 0) (i64.const 0) (i64.const 0x4242424242424242)) (i64.const 0x11)) +@@ -340,7 +340,7 @@ + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i64.atomic.rmw8.cmpxchg_u" (i32.const 0) (i64.const 0x1111111111111111) (i64.const 0x4242424242424242)) (i64.const 0x11)) +-(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x1111111111111111)) ++(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x1111111111111142)) + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i64.atomic.rmw16.cmpxchg_u" (i32.const 0) (i64.const 0) (i64.const 0xbeefbeefbeefbeef)) (i64.const 0x1111)) +@@ -348,7 +348,7 @@ + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i64.atomic.rmw16.cmpxchg_u" (i32.const 0) (i64.const 0x1111111111111111) (i64.const 0xbeefbeefbeefbeef)) (i64.const 0x1111)) +-(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x1111111111111111)) ++(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x111111111111beef)) + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i64.atomic.rmw32.cmpxchg_u" (i32.const 0) (i64.const 0) (i64.const 0xcabba6e5cabba6e5)) (i64.const 0x11111111)) +@@ -356,7 +356,7 @@ + + (invoke "init" (i64.const 0x1111111111111111)) + (assert_return (invoke "i64.atomic.rmw32.cmpxchg_u" (i32.const 0) (i64.const 0x1111111111111111) (i64.const 0xcabba6e5cabba6e5)) (i64.const 0x11111111)) +-(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x1111111111111111)) ++(assert_return (invoke "i64.atomic.load" (i32.const 0)) (i64.const 0x11111111cabba6e5)) + + ;; *.atomic.rmw*.cmpxchg (compare true) + diff --git a/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/thread_proposal_ignore_cases.patch b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/thread_proposal_ignore_cases.patch new file mode 100644 index 0000000..b2f4ab3 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/spec-test-script/thread_proposal_ignore_cases.patch @@ -0,0 +1,265 @@ +diff --git a/test/core/atomic_wait_notify.wast b/test/core/atomic_wait_notify.wast +index 2e312c3..4f35ac5 100644 +--- a/test/core/atomic_wait_notify.wast ++++ b/test/core/atomic_wait_notify.wast +@@ -70,6 +70,7 @@ + (memory (export "shared") 1 1 shared) + ) + ++(; + (thread $T1 (shared (module $Mem)) + (register "mem" $Mem) + (module +@@ -106,3 +107,4 @@ + + (wait $T1) + (wait $T2) ++;) +diff --git a/test/core/binary.wast b/test/core/binary.wast +index b9fa438..a5711dd 100644 +--- a/test/core/binary.wast ++++ b/test/core/binary.wast +@@ -45,7 +45,7 @@ + (assert_malformed (module binary "\00asm\00\00\00\01") "unknown binary version") + + ;; Invalid section id. +-(assert_malformed (module binary "\00asm" "\01\00\00\00" "\0c\00") "malformed section id") ++;; (assert_malformed (module binary "\00asm" "\01\00\00\00" "\0c\00") "malformed section id") + (assert_malformed (module binary "\00asm" "\01\00\00\00" "\7f\00") "malformed section id") + (assert_malformed (module binary "\00asm" "\01\00\00\00" "\80\00\01\00") "malformed section id") + (assert_malformed (module binary "\00asm" "\01\00\00\00" "\81\00\01\00") "malformed section id") +@@ -68,7 +68,7 @@ + "\01" ;; call_indirect reserved byte is not equal to zero! + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; call_indirect reserved byte should not be a "long" LEB128 zero. +@@ -87,7 +87,7 @@ + "\80\00" ;; call_indirect reserved byte + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; Same as above for 3, 4, and 5-byte zero encodings. +@@ -106,7 +106,7 @@ + "\80\80\00" ;; call_indirect reserved byte + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + (assert_malformed +@@ -124,7 +124,7 @@ + "\80\80\80\00" ;; call_indirect reserved byte + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + (assert_malformed +@@ -142,7 +142,7 @@ + "\80\80\80\80\00" ;; call_indirect reserved byte + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; memory.grow reserved byte equal to zero. +@@ -162,7 +162,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; memory.grow reserved byte should not be a "long" LEB128 zero. +@@ -182,7 +182,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; Same as above for 3, 4, and 5-byte zero encodings. +@@ -202,7 +202,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + (assert_malformed +@@ -221,7 +221,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + (assert_malformed +@@ -240,7 +240,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; memory.size reserved byte equal to zero. +@@ -259,7 +259,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; memory.size reserved byte should not be a "long" LEB128 zero. +@@ -278,7 +278,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; Same as above for 3, 4, and 5-byte zero encodings. +@@ -297,7 +297,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + (assert_malformed +@@ -315,7 +315,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + (assert_malformed +@@ -333,7 +333,7 @@ + "\1a" ;; drop + "\0b" ;; end + ) +- "zero flag expected" ++ "zero byte expected" + ) + + ;; No more than 2^32 locals. +@@ -745,6 +745,7 @@ + ) + + ;; 2 elem segment declared, 1 given ++(; + (assert_malformed + (module binary + "\00asm" "\01\00\00\00" +@@ -761,6 +762,7 @@ + ) + "unexpected end" + ) ++;) + + ;; 1 elem segment declared, 2 given + (assert_malformed +diff --git a/test/core/elem.wast b/test/core/elem.wast +index 1ea2b06..8eded37 100644 +--- a/test/core/elem.wast ++++ b/test/core/elem.wast +@@ -12,10 +12,10 @@ + (elem 0x0 (i32.const 0) $f $f) + (elem 0x000 (offset (i32.const 0))) + (elem 0 (offset (i32.const 0)) $f $f) +- (elem $t (i32.const 0)) +- (elem $t (i32.const 0) $f $f) +- (elem $t (offset (i32.const 0))) +- (elem $t (offset (i32.const 0)) $f $f) ++ (elem (i32.const 0)) ++ (elem (i32.const 0) $f $f) ++ (elem (offset (i32.const 0))) ++ (elem (offset (i32.const 0)) $f $f) + ) + + ;; Basic use +@@ -354,6 +354,7 @@ + (assert_return (invoke $module1 "call-8") (i32.const 65)) + (assert_return (invoke $module1 "call-9") (i32.const 66)) + ++(; + (module $module2 + (type $out-i32 (func (result i32))) + (import "module1" "shared-table" (table 10 funcref)) +@@ -379,3 +380,4 @@ + (assert_return (invoke $module1 "call-7") (i32.const 67)) + (assert_return (invoke $module1 "call-8") (i32.const 69)) + (assert_return (invoke $module1 "call-9") (i32.const 70)) ++;) +diff --git a/test/core/table.wast b/test/core/table.wast +index 0bc43ca6..ee5209ec 100644 +--- a/test/core/table.wast ++++ b/test/core/table.wast +@@ -8,8 +8,8 @@ + (module (table 0 65536 funcref)) + (module (table 0 0xffff_ffff funcref)) + +-(assert_invalid (module (table 0 funcref) (table 0 funcref)) "multiple tables") +-(assert_invalid (module (table (import "spectest" "table") 0 funcref) (table 0 funcref)) "multiple tables") ++(module (table 0 funcref) (table 0 funcref)) ++(module (table (import "spectest" "table") 0 funcref) (table 0 funcref)) + + (assert_invalid (module (elem (i32.const 0))) "unknown table") + (assert_invalid (module (elem (i32.const 0) $f) (func $f)) "unknown table") +diff --git a/test/core/thread.wast b/test/core/thread.wast +index c3456a6..83fc281 100644 +--- a/test/core/thread.wast ++++ b/test/core/thread.wast +@@ -2,6 +2,7 @@ + (memory (export "shared") 1 1 shared) + ) + ++(; + (thread $T1 (shared (module $Mem)) + (register "mem" $Mem) + (module +@@ -26,3 +27,4 @@ + + (wait $T1) + (wait $T2) ++;) +diff --git a/test/core/unreached-invalid.wast b/test/core/unreached-invalid.wast +index 6ef4ac55..9a2387a3 100644 +--- a/test/core/unreached-invalid.wast ++++ b/test/core/unreached-invalid.wast +@@ -535,6 +535,7 @@ + )) + "type mismatch" + ) ++(; invalid case, the module is fine for the latest spec interpreter + (assert_invalid + (module (func $type-br_table-label-num-vs-label-num-after-unreachable + (block (result f64) +@@ -549,6 +550,7 @@ + )) + "type mismatch" + ) ++;) + + (assert_invalid + (module (func $type-block-value-nested-unreachable-num-vs-void diff --git a/wasm-micro-runtime/tests/wamr-test-suites/test_wamr.sh b/wasm-micro-runtime/tests/wamr-test-suites/test_wamr.sh new file mode 100755 index 0000000..0c56aca --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/test_wamr.sh @@ -0,0 +1,1063 @@ +#!/usr/bin/env bash + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +function DEBUG() { + [[ -n $(env | grep "\") ]] && $@ +} +DEBUG set -xv pipefail + +function help() +{ + echo "test_wamr.sh [options]" + echo "-c clean previous test results, not start test" + echo "-s {suite_name} test only one suite (spec|wasi_certification|wamr_compiler)" + echo "-m set compile target of iwasm(x86_64|x86_32|armv7|armv7_vfp|thumbv7|thumbv7_vfp|" + echo " riscv32|riscv32_ilp32f|riscv32_ilp32d|riscv64|" + echo " riscv64_lp64f|riscv64_lp64d|aarch64|aarch64_vfp)" + echo "-t set compile type of iwasm(classic-interp|fast-interp|jit|aot|fast-jit|multi-tier-jit)" + echo "-M enable multi module feature" + echo "-p enable multi thread feature" + echo "-S enable SIMD feature" + echo "-G enable GC feature" + echo "-W enable memory64 feature" + echo "-X enable XIP feature" + echo "-e enable exception handling" + echo "-x test SGX" + echo "-w enable WASI threads" + echo "-b use the wabt binary release package instead of compiling from the source code" + echo "-g build iwasm with debug version" + echo "-v enable GC heap verification" + echo "-P run the spec test parallelly" + echo "-Q enable qemu" + echo "-F set the firmware path used by qemu" + echo "-C enable code coverage collect" + echo "-j set the platform to test" + echo "-T set sanitizer to use in tests(ubsan|tsan|asan)" +} + +OPT_PARSED="" +WABT_BINARY_RELEASE="NO" +#default type +TYPE=("classic-interp" "fast-interp" "jit" "aot" "fast-jit" "multi-tier-jit") +#default target +TARGET="X86_64" +ENABLE_WASI_THREADS=0 +ENABLE_MULTI_MODULE=0 +ENABLE_MULTI_THREAD=0 +COLLECT_CODE_COVERAGE=0 +ENABLE_SIMD=0 +ENABLE_GC=0 +ENABLE_MEMORY64=0 +ENABLE_XIP=0 +ENABLE_EH=0 +ENABLE_DEBUG_VERSION=0 +ENABLE_GC_HEAP_VERIFY=0 +#unit test case arrary +TEST_CASE_ARR=() +SGX_OPT="" +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then + PLATFORM=windows + PYTHON_EXE=python +else + PLATFORM=$(uname -s | tr A-Z a-z) + PYTHON_EXE=python3 +fi +PARALLELISM=0 +ENABLE_QEMU=0 +QEMU_FIRMWARE="" +# prod/testsuite-all branch +WASI_TESTSUITE_COMMIT="ee807fc551978490bf1c277059aabfa1e589a6c2" +TARGET_LIST=("AARCH64" "AARCH64_VFP" "ARMV7" "ARMV7_VFP" "THUMBV7" "THUMBV7_VFP" \ + "RISCV32" "RISCV32_ILP32F" "RISCV32_ILP32D" "RISCV64" "RISCV64_LP64F" "RISCV64_LP64D") + +while getopts ":s:cabgvt:m:MCpSXexwWPGQF:j:T:" opt +do + OPT_PARSED="TRUE" + case $opt in + s) + TEST_CASE_ARR+=($OPTARG) + # get next suite if there are multiple vaule in -s + eval "nxarg=\${$((OPTIND))}" + # just get test cases, loop until the next symbol '-' + # IN ====> -s spec wasi unit -t fast-classic + # GET ====> spec wasi unit + while [[ "${nxarg}" != -* && ${nxarg} ]]; + do + TEST_CASE_ARR+=(${nxarg}) + OPTIND=$((OPTIND+1)) + eval "nxarg=\${$((OPTIND))}" + done + echo "test following cases: ${TEST_CASE_ARR[@]}" + ;; + c) + read -t 5 -p "Are you sure to delete all reports. y/n " cmd + if [[ $cmd == "y" && $(ls -A workspace/report) ]];then + rm -fr workspace/report/* + rm -fr /tmp/*.wasm /tmp/*.wast /tmp/*.aot + echo "cleaned all reports and temp files" + fi + exit 0;; + a) + TEST_ALL_AOT_RUNTIME="all" + echo "test all runtimes in sightglass_aot" + ;; + b) + WABT_BINARY_RELEASE="YES" + echo "use a WABT binary release instead of compiling from source code" + ;; + t) + echo "set compile type of wamr " ${OPTARG} + if [[ ${OPTARG} != "classic-interp" && ${OPTARG} != "fast-interp" \ + && ${OPTARG} != "jit" && ${OPTARG} != "aot" + && ${OPTARG} != "fast-jit" && ${OPTARG} != "multi-tier-jit" ]]; then + echo "*----- please varify a type of compile when using -t! -----*" + help + exit 1 + fi + + TYPE=(${OPTARG}) + ;; + m) + echo "set compile target of wamr" ${OPTARG} + TARGET=$(echo "$OPTARG" | tr '[a-z]' '[A-Z]') # set target to uppercase if input x86_32 or x86_64 --> X86_32 and X86_64 + ;; + w) + echo "enable WASI threads" + ENABLE_WASI_THREADS=1 + ;; + M) + echo "enable multi module feature" + ENABLE_MULTI_MODULE=1 + ;; + W) + echo "enable wasm64(memory64) feature" + ENABLE_MEMORY64=1 + ;; + C) + echo "enable code coverage" + COLLECT_CODE_COVERAGE=1 + ;; + p) + echo "enable multi thread feature" + ENABLE_MULTI_THREAD=1 + ;; + S) + echo "enable SIMD feature" + ENABLE_SIMD=1 + ;; + X) + echo "enable XIP feature" + ENABLE_XIP=1 + ;; + e) + echo "enable exception handling feature" + ENABLE_EH=1 + ;; + x) + echo "test SGX" + SGX_OPT="--sgx" + ;; + g) + echo "enable build iwasm with debug version" + ENABLE_DEBUG_VERSION=1 + ;; + v) + echo "enable GC heap verification" + ENABLE_GC_HEAP_VERIFY=1 + ;; + G) + echo "enable GC feature" + ENABLE_GC=1 + ;; + P) + PARALLELISM=1 + ;; + Q) + echo "enable QEMU" + ENABLE_QEMU=1 + ;; + F) + echo "QEMU firmware" ${OPTARG} + QEMU_FIRMWARE=${OPTARG} + ;; + j) + echo "test platform " ${OPTARG} + PLATFORM=${OPTARG} + ;; + T) + echo "sanitizer is " ${OPTARG} + WAMR_BUILD_SANITIZER=${OPTARG} + ;; + ?) + help + exit 1 + ;; + esac +done + +# Parameters are not allowed, use options instead +if [ -z "$OPT_PARSED" ]; +then + if [ ! -z "$1" ]; + then + help + exit 1 + fi +fi + +mkdir -p workspace +cd workspace + +readonly WORK_DIR=$PWD + +readonly DATE=$(date +%Y-%m-%d_%H:%M:%S) +readonly REPORT_DIR=${WORK_DIR}/report/${DATE} +mkdir -p ${REPORT_DIR} + +readonly WAMR_DIR=${WORK_DIR}/../../.. + +if [[ ${SGX_OPT} == "--sgx" ]];then + readonly IWASM_LINUX_ROOT_DIR="${WAMR_DIR}/product-mini/platforms/linux-sgx" + readonly IWASM_CMD="${WAMR_DIR}/product-mini/platforms/linux-sgx/enclave-sample/iwasm" +else + readonly IWASM_LINUX_ROOT_DIR="${WAMR_DIR}/product-mini/platforms/${PLATFORM}" + readonly IWASM_CMD="${WAMR_DIR}/product-mini/platforms/${PLATFORM}/build/iwasm" +fi + +readonly WAMRC_CMD="${WAMR_DIR}/wamr-compiler/build/wamrc" + +readonly CLASSIC_INTERP_COMPILE_FLAGS="\ + -DWAMR_BUILD_TARGET=${TARGET} \ + -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \ + -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=0 \ + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" + +readonly FAST_INTERP_COMPILE_FLAGS="\ + -DWAMR_BUILD_TARGET=${TARGET} \ + -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=1 \ + -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=0 \ + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" + +# jit: report linking error if set COLLECT_CODE_COVERAGE, +# now we don't collect code coverage of jit type +readonly ORC_EAGER_JIT_COMPILE_FLAGS="\ + -DWAMR_BUILD_TARGET=${TARGET} \ + -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \ + -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \ + -DWAMR_BUILD_LAZY_JIT=0 \ + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" + +readonly ORC_LAZY_JIT_COMPILE_FLAGS="\ + -DWAMR_BUILD_TARGET=${TARGET} \ + -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \ + -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \ + -DWAMR_BUILD_LAZY_JIT=1 \ + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" + +readonly AOT_COMPILE_FLAGS="\ + -DWAMR_BUILD_TARGET=${TARGET} \ + -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \ + -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=1 \ + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" + +readonly FAST_JIT_COMPILE_FLAGS="\ + -DWAMR_BUILD_TARGET=${TARGET} \ + -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \ + -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=0 \ + -DWAMR_BUILD_FAST_JIT=1 \ + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" + +readonly MULTI_TIER_JIT_COMPILE_FLAGS="\ + -DWAMR_BUILD_TARGET=${TARGET} \ + -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \ + -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \ + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" + +readonly COMPILE_FLAGS=( + "${CLASSIC_INTERP_COMPILE_FLAGS}" + "${FAST_INTERP_COMPILE_FLAGS}" + "${ORC_EAGER_JIT_COMPILE_FLAGS}" + "${ORC_LAZY_JIT_COMPILE_FLAGS}" + "${AOT_COMPILE_FLAGS}" + "${FAST_JIT_COMPILE_FLAGS}" + "${MULTI_TIER_JIT_COMPILE_FLAGS}" + ) + +function unit_test() +{ + echo "Now start unit tests" + + cd ${WORK_DIR} + rm -fr unittest-build && mkdir unittest-build + cd unittest-build + + echo "Build unit test" + touch ${REPORT_DIR}/unit_test_report.txt + cmake ${WORK_DIR}/../../unit -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} + make -j + make test | tee -a ${REPORT_DIR}/unit_test_report.txt + + echo "Finish unit tests" +} + +function sightglass_test() +{ + echo "Now start sightglass benchmark tests" + + cd ${WORK_DIR}/../sightglass/benchmarks + + # build iwasm first + if [[ $1 == "classic-interp" || $1 == "fast-interp" ]];then + ./test_interp.sh ${SGX_OPT} + cp report.txt ${REPORT_DIR}/sightglass_$1_test_report.txt + fi + + if [[ $1 == "aot" ]];then + ./test_aot.sh ${SGX_OPT} + cp report.txt ${REPORT_DIR}/sightglass_aot_test_report.txt + fi + + if [[ $1 == "jit" ]];then + [[ $TEST_ALL_AOT_RUNTIME ]] && ./test_aot.sh ${TEST_ALL_AOT_RUNTIME} ${SGX_OPT} \ + || ./test_aot.sh jit ${SGX_OPT} + cp report.txt ${REPORT_DIR}/sightglass_jit_test_report.txt + fi + + echo "Finish sightglass benchmark tests" +} + +function setup_wabt() +{ + if [ ${WABT_BINARY_RELEASE} == "YES" ]; then + echo "download a binary release and install" + local WAT2WASM=${WORK_DIR}/wabt/out/gcc/Release/wat2wasm + if [ ! -f ${WAT2WASM} ]; then + case ${PLATFORM} in + cosmopolitan) + ;; + linux) + WABT_PLATFORM=ubuntu + ;; + darwin) + WABT_PLATFORM=macos-12 + ;; + windows) + WABT_PLATFORM=windows + ;; + *) + echo "wabt platform for ${PLATFORM} in unknown" + exit 1 + ;; + esac + if [ ! -f /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz ]; then + curl -L \ + https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-${WABT_PLATFORM}.tar.gz \ + -o /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz + fi + + cd /tmp \ + && tar zxf wabt-1.0.31-${WABT_PLATFORM}.tar.gz \ + && mkdir -p ${WORK_DIR}/wabt/out/gcc/Release/ \ + && install wabt-1.0.31/bin/wa* ${WORK_DIR}/wabt/out/gcc/Release/ \ + && cd - + fi + else + echo "download source code and compile and install" + if [ ! -d "wabt" ];then + echo "wabt not exist, clone it from github" + git clone --recursive https://github.com/WebAssembly/wabt + fi + echo "upate wabt" + cd wabt + git pull + git reset --hard origin/main + cd .. + make -C wabt gcc-release -j 4 + fi +} + +# TODO: with iwasm only +function spec_test() +{ + echo "Now start spec tests" + touch ${REPORT_DIR}/spec_test_report.txt + + cd ${WORK_DIR} + if [ ! -d "spec" ];then + echo "spec not exist, clone it from github" + git clone -b master --single-branch https://github.com/WebAssembly/spec + fi + + pushd spec + + # restore and clean everything + git reset --hard HEAD + + # update basic test cases + echo "update spec test cases" + git fetch origin main + # restore from XX_ignore_cases.patch + # resotre branch + git checkout -B main + # [spec] Update note on module initialization trapping (#1493) + git reset --hard 044d0d2e77bdcbe891f7e0b9dd2ac01d56435f0b + git apply ../../spec-test-script/ignore_cases.patch + if [[ ${ENABLE_SIMD} == 1 ]]; then + git apply ../../spec-test-script/simd_ignore_cases.patch + fi + if [[ ${ENABLE_MULTI_MODULE} == 1 && $1 == 'aot' ]]; then + git apply ../../spec-test-script/multi_module_aot_ignore_cases.patch + fi + + # udpate thread cases + if [ ${ENABLE_MULTI_THREAD} == 1 ]; then + echo "checkout spec for threads proposal" + if [[ -z $(git remote -v | grep "\") ]]; then + git remote add threads https://github.com/WebAssembly/threads + fi + + # fetch spec for threads proposal + git fetch threads + # Fix error in Web embedding desc for atomic.notify (#185) + git reset --hard 85b562cd6805947876ec5e8b975ab0127c55a0a2 + git checkout threads/main + + git apply ../../spec-test-script/thread_proposal_ignore_cases.patch + git apply ../../spec-test-script/thread_proposal_fix_atomic_case.patch + fi + + if [ ${ENABLE_EH} == 1 ]; then + echo "checkout exception-handling test cases" + popd + if [ ! -d "exception-handling" ];then + echo "exception-handling not exist, clone it from github" + git clone -b master --single-branch https://github.com/WebAssembly/exception-handling + fi + pushd exception-handling + + # restore and clean everything + git reset --hard 51c721661b671bb7dc4b3a3acb9e079b49778d36 + + if [[ ${ENABLE_MULTI_MODULE} == 0 ]]; then + git apply ../../spec-test-script/exception_handling.patch + fi + + popd + echo $(pwd) + fi + + # update GC cases + if [[ ${ENABLE_GC} == 1 ]]; then + echo "checkout spec for GC proposal" + + popd + rm -fr spec + # check spec test cases for GC + git clone -b main --single-branch https://github.com/WebAssembly/gc.git spec + pushd spec + + git restore . && git clean -ffd . + # Reset to commit: "[test] Unify the error message." + git reset --hard 0caaadc65b5e1910512d8ae228502edcf9d60390 + git apply ../../spec-test-script/gc_ignore_cases.patch + + if [[ ${ENABLE_QEMU} == 1 ]]; then + # Decrease the recursive count for tail call cases as nuttx qemu's + # native stack size is much smaller + git apply ../../spec-test-script/gc_nuttx_tail_call.patch + fi + + echo "compile the reference intepreter" + pushd interpreter + make + popd + fi + + # update memory64 cases + if [[ ${ENABLE_MEMORY64} == 1 ]]; then + echo "checkout spec for memory64 proposal" + + popd + rm -fr spec + # check spec test cases for memory64 + git clone -b main --single-branch https://github.com/WebAssembly/memory64.git spec + pushd spec + + git restore . && git clean -ffd . + # Reset to commit: "Merge remote-tracking branch 'upstream/main' into merge2" + git reset --hard 48e69f394869c55b7bbe14ac963c09f4605490b6 + git checkout 044d0d2e77bdcbe891f7e0b9dd2ac01d56435f0b -- test/core/elem.wast + git apply ../../spec-test-script/ignore_cases.patch + git apply ../../spec-test-script/memory64.patch + + echo "compile the reference intepreter" + pushd interpreter + make + popd + fi + + popd + echo $(pwd) + + setup_wabt + + ln -sf ${WORK_DIR}/../spec-test-script/all.py . + ln -sf ${WORK_DIR}/../spec-test-script/runtest.py . + + local ARGS_FOR_SPEC_TEST="" + + # multi-module only enable in interp mode and aot mode + if [[ 1 == ${ENABLE_MULTI_MODULE} ]]; then + if [[ $1 == 'classic-interp' || $1 == 'fast-interp' || $1 == 'aot' ]]; then + ARGS_FOR_SPEC_TEST+="-M " + fi + fi + + if [[ 1 == ${ENABLE_EH} ]]; then + ARGS_FOR_SPEC_TEST+="-e " + fi + + # sgx only enable in interp mode and aot mode + if [[ ${SGX_OPT} == "--sgx" ]];then + if [[ $1 == 'classic-interp' || $1 == 'fast-interp' || $1 == 'aot' || $1 == 'fast-jit' ]]; then + ARGS_FOR_SPEC_TEST+="-x " + fi + fi + + # simd only enable in jit mode and aot mode + if [[ ${ENABLE_SIMD} == 1 ]]; then + if [[ $1 == 'jit' || $1 == 'aot' ]]; then + ARGS_FOR_SPEC_TEST+="-S " + fi + fi + + if [[ ${ENABLE_MULTI_THREAD} == 1 ]]; then + ARGS_FOR_SPEC_TEST+="-p " + fi + + if [[ ${ENABLE_XIP} == 1 ]]; then + ARGS_FOR_SPEC_TEST+="-X " + fi + + # set the current running target + ARGS_FOR_SPEC_TEST+="-m ${TARGET} " + + # require warmc only in aot mode + if [[ $1 == 'aot' ]]; then + ARGS_FOR_SPEC_TEST+="-t " + fi + + if [[ ${PARALLELISM} == 1 ]]; then + ARGS_FOR_SPEC_TEST+="--parl " + fi + + if [[ ${ENABLE_GC} == 1 ]]; then + ARGS_FOR_SPEC_TEST+="--gc " + fi + + # wasm64(memory64) is only enabled in interp and aot mode + if [[ 1 == ${ENABLE_MEMORY64} ]]; then + if [[ $1 == 'classic-interp' || $1 == 'aot' ]]; then + ARGS_FOR_SPEC_TEST+="--memory64 " + fi + fi + + if [[ ${ENABLE_QEMU} == 1 ]]; then + ARGS_FOR_SPEC_TEST+="--qemu " + ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} " + fi + + if [[ ${PLATFORM} == "windows" ]]; then + ARGS_FOR_SPEC_TEST+="--no-pty " + fi + + # set log directory + ARGS_FOR_SPEC_TEST+="--log ${REPORT_DIR}" + + cd ${WORK_DIR} + echo "${PYTHON_EXE} ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt" + ${PYTHON_EXE} ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt + if [[ ${PIPESTATUS[0]} -ne 0 ]];then + echo -e "\nspec tests FAILED" | tee -a ${REPORT_DIR}/spec_test_report.txt + exit 1 + fi + cd - + + echo -e "\nFinish spec tests" | tee -a ${REPORT_DIR}/spec_test_report.txt +} + +function wasi_test() +{ + echo "Now start wasi tests" + touch ${REPORT_DIR}/wasi_test_report.txt + + cd ${WORK_DIR}/../../wasi + [[ $1 != "aot" ]] && \ + python wasi_test.py --interpreter ${IWASM_CMD} ${SGX_OPT}\ + | tee ${REPORT_DIR}/wasi_test_report.txt \ + || \ + python wasi_test.py --aot --aot-compiler ${WAMRC_CMD} ${SGX_OPT}\ + --interpreter ${IWASM_CMD} \ + | tee ${REPORT_DIR}/wasi_test_report.txt + echo "Finish wasi tests" +} + +function wamr_compiler_test() +{ + if [[ $1 != "aot" ]]; then + echo "WAMR compiler tests only support AOT mode" + exit 1 + fi + + echo "Now start WAMR compiler tests" + setup_wabt + cd ${WORK_DIR}/../wamr-compiler-test-script + ./run_wamr_compiler_tests.sh ${WORK_DIR}/wabt/out/gcc/Release/wat2wasm $WAMRC_CMD $IWASM_CMD \ + | tee -a ${REPORT_DIR}/wamr_compiler_test_report.txt + + ret=${PIPESTATUS[0]} + + if [[ ${ret} -ne 0 ]];then + echo -e "\nWAMR compiler tests FAILED" | tee -a ${REPORT_DIR}/wamr_compiler_test_report.txt + exit 1 + fi + echo -e "\nFinish WAMR compiler tests" | tee -a ${REPORT_DIR}/wamr_compiler_test_report.txt +} + +function wasi_certification_test() +{ + echo "Now start wasi certification tests" + + cd ${WORK_DIR} + if [ ! -d "wasi-testsuite" ]; then + echo "wasi-testsuite not exist, clone it from github" + git clone -b prod/testsuite-all \ + --single-branch https://github.com/WebAssembly/wasi-testsuite.git + fi + cd wasi-testsuite + git reset --hard ${WASI_TESTSUITE_COMMIT} + + TSAN_OPTIONS=${TSAN_OPTIONS} bash ../../wasi-test-script/run_wasi_tests.sh $1 $TARGET $WASI_TEST_FILTER \ + | tee -a ${REPORT_DIR}/wasi_test_report.txt + ret=${PIPESTATUS[0]} + + if [[ ${ret} -ne 0 ]];then + echo -e "\nwasi tests FAILED" | tee -a ${REPORT_DIR}/wasi_test_report.txt + exit 1 + fi + echo -e "\nFinish wasi tests" | tee -a ${REPORT_DIR}/wasi_test_report.txt +} + +function polybench_test() +{ + echo "Now start polybench tests" + + cd ${WORK_DIR}/../polybench + if [[ $1 == "aot" || $1 == "jit" ]];then + ./build.sh AOT ${SGX_OPT} + ./test_aot.sh $1 ${SGX_OPT} + else + ./build.sh + ./test_interp.sh ${SGX_OPT} + fi + cp report.txt ${REPORT_DIR}/polybench_$1_test_report.txt + + echo "Finish polybench tests" +} + +function libsodium_test() +{ + echo "Now start libsodium tests" + + cd ${WORK_DIR}/../libsodium + if [[ $1 == "aot" || $1 == "jit" ]];then + ./build.sh ${SGX_OPT} + ./test_aot.sh $1 ${SGX_OPT} + else + ./test_interp.sh ${SGX_OPT} + fi + cp report.txt ${REPORT_DIR}/libsodium_$1_test_report.txt + + echo "Finish libsodium tests" +} + +function malformed_test() +{ + # build iwasm firstly + cd ${WORK_DIR}/../../malformed + ./malformed_test.py --run ${IWASM_CMD} | tee ${REPORT_DIR}/malfomed_$1_test_report.txt +} + +function collect_standalone() +{ + if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then + pushd ${WORK_DIR} > /dev/null 2>&1 + + CODE_COV_FILE="" + if [[ -z "${CODE_COV_FILE}" ]]; then + CODE_COV_FILE="${WORK_DIR}/wamr.lcov" + else + CODE_COV_FILE="${CODE_COV_FILE}" + fi + + STANDALONE_DIR=${WORK_DIR}/../../standalone + + echo "Collect code coverage of standalone dump-call-stack" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-call-stack/build" + echo "Collect code coverage of standalone dump-mem-profiling" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-mem-profiling/build" + echo "Collect code coverage of standalone dump-perf-profiling" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-perf-profiling/build" + if [[ $1 == "aot" ]]; then + echo "Collect code coverage of standalone pad-test" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/pad-test/build" + fi + echo "Collect code coverage of standalone test-invoke-native" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-invoke-native/build" + echo "Collect code coverage of standalone test-running-modes" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-running-modes/build" + echo "Collect code coverage of standalone test-running-modes/c-embed" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-running-modes/c-embed/build" + echo "Collect code coverage of standalone test-ts2" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-ts2/build" + + popd > /dev/null 2>&1 + fi +} + +function standalone_test() +{ + if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then + export COLLECT_CODE_COVERAGE=1 + fi + + cd ${WORK_DIR}/../../standalone + + args="--$1" + + [[ ${SGX_OPT} == "--sgx" ]] && args="$args --sgx" || args="$args --no-sgx" + + [[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" || args="$args --no-thread" + + [[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" || args="$args --no-simd" + + args="$args ${TARGET}" + + ./standalone.sh $args | tee ${REPORT_DIR}/standalone_$1_test_report.txt + + collect_standalone "$1" +} + +function build_iwasm_with_cfg() +{ + echo "Build iwasm with compile flags " $* " for spec test" \ + | tee -a ${REPORT_DIR}/spec_test_report.txt + + if [[ ${SGX_OPT} == "--sgx" ]];then + cd ${WAMR_DIR}/product-mini/platforms/linux-sgx \ + && if [ -d build ]; then rm -rf build/*; else mkdir build; fi \ + && cd build \ + && cmake $* .. \ + && make -j 4 + cd ${WAMR_DIR}/product-mini/platforms/linux-sgx/enclave-sample \ + && make clean \ + && make SPEC_TEST=1 + else + cd ${WAMR_DIR}/product-mini/platforms/${PLATFORM} \ + && if [ -d build ]; then rm -rf build/*; else mkdir build; fi \ + && cd build \ + && cmake $* .. \ + && cmake --build . -j 4 --config RelWithDebInfo --target iwasm + fi + + if [ "$?" != 0 ];then + echo -e "build iwasm failed" + exit 1 + fi + + if [[ ${PLATFORM} == "cosmopolitan" ]]; then + # convert from APE to ELF so it can be ran easier + # HACK: link to linux so tests work when platform is detected by uname + cp iwasm.com iwasm \ + && ./iwasm --assimilate \ + && rm -rf ../../linux/build \ + && mkdir ../../linux/build \ + && ln -s ../../cosmopolitan/build/iwasm ../../linux/build/iwasm + if [ "$?" != 0 ];then + echo -e "build iwasm failed (cosmopolitan)" + exit 1 + fi + fi +} + +function build_wamrc() +{ + if [[ "${TARGET_LIST[*]}" =~ "${TARGET}" ]]; then + echo "suppose wamrc is already built" + return + fi + + echo "Build wamrc for spec test under aot compile type" + cd ${WAMR_DIR}/wamr-compiler \ + && ./build_llvm.sh \ + && if [ -d build ]; then rm -r build/*; else mkdir build; fi \ + && cd build \ + && cmake .. -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} \ + && make -j 4 +} + +### Need to add a test suite? +### The function name should be ${suite_name}_test +# function xxx_test() +# { +# +# } + +function collect_coverage() +{ + if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then + ln -sf ${WORK_DIR}/../spec-test-script/collect_coverage.sh ${WORK_DIR} + + CODE_COV_FILE="" + if [[ -z "${CODE_COV_FILE}" ]]; then + CODE_COV_FILE="${WORK_DIR}/wamr.lcov" + else + CODE_COV_FILE="${CODE_COV_FILE}" + fi + + pushd ${WORK_DIR} > /dev/null 2>&1 + echo "Collect code coverage of iwasm" + ./collect_coverage.sh ${CODE_COV_FILE} ${IWASM_LINUX_ROOT_DIR}/build + if [[ $1 == "llvm-aot" ]]; then + echo "Collect code coverage of wamrc" + ./collect_coverage.sh ${CODE_COV_FILE} ${WAMR_DIR}/wamr-compiler/build + fi + for suite in "${TEST_CASE_ARR[@]}"; do + if [[ ${suite} = "unit" ]]; then + echo "Collect code coverage of unit test" + ./collect_coverage.sh ${CODE_COV_FILE} ${WORK_DIR}/unittest-build + break + fi + done + popd > /dev/null 2>&1 + else + echo "code coverage isn't collected" + fi +} + +function trigger() +{ + local EXTRA_COMPILE_FLAGS="" + # default enabled features + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_BULK_MEMORY=1" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_REF_TYPES=1" + + if [[ ${ENABLE_MULTI_MODULE} == 1 ]];then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_MULTI_MODULE=1" + else + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_MULTI_MODULE=0" + fi + + if [[ ${ENABLE_MEMORY64} == 1 ]];then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_MEMORY64=1" + else + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_MEMORY64=0" + fi + + if [[ ${ENABLE_MULTI_THREAD} == 1 ]];then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_PTHREAD=1" + fi + + if [[ ${ENABLE_SIMD} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SIMD=1" + else + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SIMD=0" + fi + + if [[ ${ENABLE_GC} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC=1" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_BULK_MEMORY=1" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_TAIL_CALL=1" + fi + + if [[ ${ENABLE_DEBUG_VERSION} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DCMAKE_BUILD_TYPE=Debug" + fi + + if [[ ${ENABLE_GC_HEAP_VERIFY} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC_HEAP_VERIFY=1" + fi + + if [[ ${ENABLE_WASI_THREADS} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=1" + fi + + if [[ ${ENABLE_EH} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_EXCE_HANDLING=1" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_TAIL_CALL=1" + fi + echo "SANITIZER IS" $WAMR_BUILD_SANITIZER + + if [[ "$WAMR_BUILD_SANITIZER" == "ubsan" ]]; then + echo "Setting run with ubsan" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SANITIZER=ubsan" + fi + + if [[ "$WAMR_BUILD_SANITIZER" == "asan" ]]; then + echo "Setting run with asan" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SANITIZER=asan" + fi + + if [[ "$WAMR_BUILD_SANITIZER" == "tsan" ]]; then + echo "Setting run with tsan" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SANITIZER=tsan" + fi + + # Make sure we're using the builtin WASI libc implementation + # if we're running the wasi certification tests. + if [[ $TEST_CASE_ARR ]]; then + for test in "${TEST_CASE_ARR[@]}"; do + if [[ "$test" == "wasi_certification" ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIBC_UVWASI=0 -DWAMR_BUILD_LIBC_WASI=1" + break + fi + done + fi + + for t in "${TYPE[@]}"; do + case $t in + "classic-interp") + if [[ ${ENABLE_SIMD} == 1 ]]; then + echo "does not support SIMD in interp mode, bypass" + continue + fi + + echo "work in classic-interp mode" + # classic-interp + BUILD_FLAGS="$CLASSIC_INTERP_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" + if [[ ${ENABLE_QEMU} == 0 ]]; then + build_iwasm_with_cfg $BUILD_FLAGS + fi + for suite in "${TEST_CASE_ARR[@]}"; do + $suite"_test" classic-interp + done + collect_coverage classic-interp + ;; + + "fast-interp") + if [[ ${ENABLE_SIMD} == 1 ]]; then + echo "does not support SIMD in interp mode, bypass" + continue + fi + + echo "work in fast-interp mode" + # fast-interp + BUILD_FLAGS="$FAST_INTERP_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" + if [[ ${ENABLE_QEMU} == 0 ]]; then + build_iwasm_with_cfg $BUILD_FLAGS + fi + for suite in "${TEST_CASE_ARR[@]}"; do + $suite"_test" fast-interp + done + collect_coverage fast-interp + ;; + + "jit") + if [[ ${TARGET} == "X86_32" ]]; then + echo "does not support an X86_32 target in JIT mode, bypass" + continue + fi + + echo "work in orc jit eager compilation mode" + BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" + build_iwasm_with_cfg $BUILD_FLAGS + for suite in "${TEST_CASE_ARR[@]}"; do + $suite"_test" jit + done + collect_coverage llvm-jit + + echo "work in orc jit lazy compilation mode" + BUILD_FLAGS="$ORC_LAZY_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" + build_iwasm_with_cfg $BUILD_FLAGS + for suite in "${TEST_CASE_ARR[@]}"; do + $suite"_test" jit + done + collect_coverage llvm-jit + ;; + + "aot") + echo "work in aot mode" + # aot + BUILD_FLAGS="$AOT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" + if [[ ${ENABLE_QEMU} == 0 ]]; then + build_iwasm_with_cfg $BUILD_FLAGS + fi + build_wamrc + for suite in "${TEST_CASE_ARR[@]}"; do + $suite"_test" aot + done + collect_coverage llvm-aot + ;; + + "fast-jit") + echo "work in fast-jit mode" + # fast-jit + BUILD_FLAGS="$FAST_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" + build_iwasm_with_cfg $BUILD_FLAGS + for suite in "${TEST_CASE_ARR[@]}"; do + $suite"_test" fast-jit + done + collect_coverage fast-jit + ;; + + "multi-tier-jit") + echo "work in multi-tier-jit mode" + # multi-tier-jit + BUILD_FLAGS="$MULTI_TIER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" + build_iwasm_with_cfg $BUILD_FLAGS + for suite in "${TEST_CASE_ARR[@]}"; do + $suite"_test" multi-tier-jit + done + collect_coverage multi-tier-jit + ;; + + *) + echo "unexpected mode, do nothing" + ;; + esac + done +} + +# if collect code coverage, ignore -s, test all test cases. +if [[ $TEST_CASE_ARR ]];then + trigger || (echo "TEST FAILED"; exit 1) +else + # test all suite, ignore polybench and libsodium because of long time cost + TEST_CASE_ARR=("spec") + : ' + if [[ $COLLECT_CODE_COVERAGE == 1 ]];then + # add polybench if collecting code coverage data + TEST_CASE_ARR+=("polybench") + # add libsodium if needed, which takes long time to run + TEST_CASE_ARR+=("libsodium") + fi + ' + trigger || (echo "TEST FAILED"; exit 1) + # Add more suites here +fi + +echo -e "Test finish. Reports are under ${REPORT_DIR}" +DEBUG set +xv pipefail +echo "TEST SUCCESSFUL" +exit 0 diff --git a/wasm-micro-runtime/tests/wamr-test-suites/tsan_suppressions.txt b/wasm-micro-runtime/tests/wamr-test-suites/tsan_suppressions.txt new file mode 100644 index 0000000..dd903a6 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/tsan_suppressions.txt @@ -0,0 +1,8 @@ +# Proposing to accept this risk for now. It might be wasi-libc related. +# https://github.com/bytecodealliance/wasm-micro-runtime/pull/1963#issuecomment-1455342931 +race:STORE_U32 +race:STORE_U8 + +# Suppressing signal-unsafe inside of a signal for AOT mode +# see https://github.com/bytecodealliance/wasm-micro-runtime/issues/2248#issuecomment-1630189656 +signal:* \ No newline at end of file diff --git a/wasm-micro-runtime/tests/wamr-test-suites/wamr-compiler-test-script/run_wamr_compiler_tests.sh b/wasm-micro-runtime/tests/wamr-test-suites/wamr-compiler-test-script/run_wamr_compiler_tests.sh new file mode 100755 index 0000000..19c8030 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/wamr-compiler-test-script/run_wamr_compiler_tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright (C) 2023 Amazon Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -e + +WAT2WASM_CMD=$1 +WAMRC_CMD=$2 +IWASM_CMD=$3 + +for wat_file in ../../wamr-compiler/*.wat; do + wasm_file="${wat_file%.wat}.wasm" + aot_file="${wat_file%.wat}.aot" + + echo "Compiling $wat_file to $wasm_file" + $WAT2WASM_CMD "$wat_file" -o "$wasm_file" + echo "Compiling $wasm_file to $aot_file" + $WAMRC_CMD -o $aot_file $wasm_file + echo "Testing $aot_file" + $IWASM_CMD "$aot_file" +done diff --git a/wasm-micro-runtime/tests/wamr-test-suites/wasi-test-script/pipe.py b/wasm-micro-runtime/tests/wamr-test-suites/wasi-test-script/pipe.py new file mode 100644 index 0000000..e374087 --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/wasi-test-script/pipe.py @@ -0,0 +1,19 @@ +#! /usr/bin/env python3 + +# Copyright (C) 2023 YAMAMOTO Takashi +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This is a copy of https://github.com/yamt/toywasm/blob/master/test/pipe.py + +# keep stdout open until the peer closes it + +import sys +import select + +p = select.poll() +p.register(sys.stdout, select.POLLHUP) +# http://gnats.netbsd.org/cgi-bin/query-pr-single.pl?number=57369 +while True: + l = p.poll(1) + if l: + break diff --git a/wasm-micro-runtime/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh b/wasm-micro-runtime/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh new file mode 100755 index 0000000..9ea733c --- /dev/null +++ b/wasm-micro-runtime/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +# +# Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +THIS_DIR=$(cd $(dirname $0) && pwd -P) + +readonly MODE=$1 +readonly TARGET=$2 +readonly TEST_FILTER=$3 + +readonly WORK_DIR=$PWD + +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then + readonly PLATFORM=windows + readonly PYTHON_EXE=python + # see https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1 + readonly VENV_BIN_DIR=Scripts + readonly IWASM_EXE=$(cygpath -m "${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/RelWithDebInfo/iwasm.exe") +else + readonly PLATFORM=$(uname -s | tr A-Z a-z) + readonly VENV_BIN_DIR=bin + readonly PYTHON_EXE=python3 + readonly IWASM_EXE="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm" +fi + +readonly WAMR_DIR="${WORK_DIR}/../../../.." +readonly IWASM_CMD="${IWASM_EXE} \ + --allow-resolve=google-public-dns-a.google.com \ + --addr-pool=::1/128,127.0.0.1/32" + +readonly IWASM_CMD_STRESS="${IWASM_CMD} --max-threads=12" +readonly WAMRC_CMD="${WORK_DIR}/../../../../wamr-compiler/build/wamrc" +readonly C_TESTS="tests/c/testsuite/" +readonly RUST_TESTS="tests/rust/testsuite/" +readonly ASSEMBLYSCRIPT_TESTS="tests/assemblyscript/testsuite/" +readonly THREAD_PROPOSAL_TESTS="tests/proposals/wasi-threads/" +readonly THREAD_INTERNAL_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-wasi-threads/test/" +readonly THREAD_STRESS_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-wasi-threads/stress-test/" +readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/" + +add_env_key_to_test_config_file() { + filepath="tests/$2/testsuite/$3.json" + modified_contents=$(jq ".env.$1 = 1" "$filepath") + echo "$modified_contents" > "$filepath" +} + +run_aot_tests () { + local -n tests=$1 + local -n excluded_tests=$2 + + for test_wasm in ${tests[@]}; do + # get the base file name from the filepath + local test_name=${test_wasm##*/} + test_name=${test_name%.wasm} + + for excluded_test in "${excluded_tests[@]}"; do + if [[ $excluded_test == "\"$test_name\"" ]]; then + echo "Skipping test $test_name" + continue 2 + fi + done + + local iwasm="${IWASM_CMD}" + if [[ $test_wasm =~ "stress" ]]; then + iwasm="${IWASM_CMD_STRESS}" + fi + + test_aot="${test_wasm%.wasm}.aot" + test_json="${test_wasm%.wasm}.json" + + if [ -f ${test_wasm} ]; then + expected=$(jq .exit_code ${test_json}) + fi + + echo "Compiling $test_wasm to $test_aot" + ${WAMRC_CMD} --enable-multi-thread ${target_option} \ + -o ${test_aot} ${test_wasm} + + echo "Running $test_aot" + expected=0 + if [ -f ${test_json} ]; then + expected=$(jq .exit_code ${test_json}) + fi + + $PYTHON_EXE ${THIS_DIR}/pipe.py | ${iwasm} $test_aot + ret=${PIPESTATUS[1]} + + echo "expected=$expected, actual=$ret" + if [[ $expected != "" ]] && [[ $expected != $ret ]];then + exit_code=1 + fi + done +} + +if [[ $MODE != "aot" ]];then + $PYTHON_EXE -m venv wasi-env && source wasi-env/${VENV_BIN_DIR}/activate + $PYTHON_EXE -m pip install -r test-runner/requirements.txt + + export TEST_RUNTIME_EXE="${IWASM_CMD}" + + # Some of the WASI test assertions can be controlled via environment + # variables. The following ones are set on Windows so that the tests pass. + if [ "$PLATFORM" == "windows" ]; then + add_env_key_to_test_config_file NO_DANGLING_FILESYSTEM rust symlink_loop + add_env_key_to_test_config_file NO_DANGLING_FILESYSTEM rust dangling_symlink + add_env_key_to_test_config_file ERRNO_MODE_WINDOWS rust path_open_preopen + add_env_key_to_test_config_file NO_RENAME_DIR_TO_EMPTY_DIR rust path_rename + fi + + TEST_OPTIONS="-r adapters/wasm-micro-runtime.py \ + -t \ + ${C_TESTS} \ + ${RUST_TESTS} \ + ${ASSEMBLYSCRIPT_TESTS} \ + ${THREAD_PROPOSAL_TESTS} \ + ${THREAD_INTERNAL_TESTS} \ + ${LIB_SOCKET_TESTS}" + + if [ -n "$TEST_FILTER" ]; then + TEST_OPTIONS="${TEST_OPTIONS} --exclude-filter ${TEST_FILTER}" + fi + + $PYTHON_EXE ${THIS_DIR}/pipe.py | TSAN_OPTIONS=${TSAN_OPTIONS} $PYTHON_EXE test-runner/wasi_test_runner.py $TEST_OPTIONS + + ret=${PIPESTATUS[1]} + + TEST_RUNTIME_EXE="${IWASM_CMD_STRESS}" TSAN_OPTIONS=${TSAN_OPTIONS} $PYTHON_EXE test-runner/wasi_test_runner.py \ + -r adapters/wasm-micro-runtime.py \ + -t \ + ${THREAD_STRESS_TESTS} + + if [ "${ret}" -eq 0 ]; then + ret=${PIPESTATUS[0]} + fi + + exit_code=${ret} + + deactivate +else + target_option="" + if [[ $TARGET == "X86_32" ]];then + target_option="--target=i386" + fi + + exit_code=0 + for testsuite in ${THREAD_STRESS_TESTS} ${THREAD_PROPOSAL_TESTS} ${THREAD_INTERNAL_TESTS}; do + tests=$(ls ${testsuite}*.wasm) + tests_array=($tests) + + if [ -n "$TEST_FILTER" ]; then + readarray -t excluded_tests_array < <(jq -c \ + --slurpfile testsuite_manifest $testsuite/manifest.json \ + '.[$testsuite_manifest[0].name] // {} | keys[]' \ + $TEST_FILTER) + else + excluded_tests_array=() + fi + + run_aot_tests tests_array excluded_tests_array + done +fi + +exit ${exit_code} diff --git a/wasm-micro-runtime/wamr-compiler/CMakeLists.txt b/wasm-micro-runtime/wamr-compiler/CMakeLists.txt new file mode 100644 index 0000000..6a3f975 --- /dev/null +++ b/wasm-micro-runtime/wamr-compiler/CMakeLists.txt @@ -0,0 +1,396 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT DEFINED WAMR_BUILD_PLATFORM) + if (CMAKE_SYSTEM_NAME) + string(TOLOWER ${CMAKE_SYSTEM_NAME} WAMR_BUILD_PLATFORM) + else() + string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) + endif() +endif() + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (aot-compiler) +else() + project (aot-compiler C ASM CXX) + enable_language (ASM_MASM) + add_definitions(-DCOMPILING_WASM_RUNTIME_API=1) +endif() + +set (CMAKE_CXX_STANDARD 17) + +if (NOT DEFINED WAMR_BUILD_PLATFORM) + set (WAMR_BUILD_PLATFORM "linux") +endif() + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +add_definitions(-DWASM_ENABLE_INTERP=1) +add_definitions(-DWASM_ENABLE_WAMR_COMPILER=1) +add_definitions(-DWASM_ENABLE_BULK_MEMORY=1) +add_definitions(-DWASM_DISABLE_HW_BOUND_CHECK=1) +add_definitions(-DWASM_ENABLE_SHARED_MEMORY=1) +add_definitions(-DWASM_ENABLE_THREAD_MGR=1) +add_definitions(-DWASM_ENABLE_TAIL_CALL=1) +add_definitions(-DWASM_ENABLE_SIMD=1) +add_definitions(-DWASM_ENABLE_REF_TYPES=1) +add_definitions(-DWASM_ENABLE_CUSTOM_NAME_SECTION=1) +add_definitions(-DWASM_ENABLE_AOT_STACK_FRAME=1) +add_definitions(-DWASM_ENABLE_DUMP_CALL_STACK=1) +add_definitions(-DWASM_ENABLE_PERF_PROFILING=1) +add_definitions(-DWASM_ENABLE_LOAD_CUSTOM_SECTION=1) +add_definitions(-DWASM_ENABLE_MODULE_INST_CONTEXT=1) + +add_definitions(-DWASM_ENABLE_GC=1) + +set (WAMR_BUILD_STRINGREF 1) +set (WAMR_STRINGREF_IMPL_SOURCE "STUB") + +if (WAMR_BUILD_LLVM_LEGACY_PM EQUAL 1) + add_definitions(-DWASM_ENABLE_LLVM_LEGACY_PM=1) +endif () + +if (LINUX) + add_definitions(-DWASM_ENABLE_LINUX_PERF=1) +endif () + +if (DEFINED WAMR_BUILD_AOT_FUNC_PREFIX) + add_definitions(-DAOT_FUNC_PREFIX="${WAMR_BUILD_AOT_FUNC_PREFIX}") +endif () + +if (NOT WAMR_BUILD_TARGET) + string(TOLOWER "${CMAKE_HOST_SYSTEM_PROCESSOR}" HOST_SYSTEM_PROCESSOR) + if (${HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64") + set (WAMR_BUILD_TARGET "X86_64") + elseif (${HOST_SYSTEM_PROCESSOR} STREQUAL "i686") + set (WAMR_BUILD_TARGET "X86_32") + elseif (${HOST_SYSTEM_PROCESSOR} STREQUAL "amd64") + set (WAMR_BUILD_TARGET "AMD_64") + elseif (${HOST_SYSTEM_PROCESSOR} STREQUAL "aarch64" + OR ${HOST_SYSTEM_PROCESSOR} STREQUAL "arm64") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (${HOST_SYSTEM_PROCESSOR} MATCHES "arm.*") + message(STATUS "Assuming ${CMAKE_HOST_SYSTEM_PROCESSOR} as ARM_32") + set (WAMR_BUILD_TARGET "ARM_32") + elseif (${HOST_SYSTEM_PROCESSOR} STREQUAL "mips") + set (WAMR_BUILD_TARGET "MIPS_32") + elseif (${HOST_SYSTEM_PROCESSOR} STREQUAL "xtensa") + set (WAMR_BUILD_TARGET "XTENSA_32") + elseif (${HOST_SYSTEM_PROCESSOR} STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (${HOST_SYSTEM_PROCESSOR} STREQUAL "riscv") + message(STATUS "Assuming ${CMAKE_HOST_SYSTEM_PROCESSOR} as RISCV32") + set (WAMR_BUILD_TARGET "RISCV32") + else () + message (FATAL_ERROR "Unsupported CMAKE_HOST_SYSTEM_PROCESSOR " + "${CMAKE_HOST_SYSTEM_PROCESSOR}") + endif() + + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + if (("${CMAKE_GENERATOR_PLATFORM}" STREQUAL "Win32")) + set (WAMR_BUILD_TARGET "X86_32") + endif() + endif () +endif () + +string(TOUPPER ${WAMR_BUILD_TARGET} WAMR_BUILD_TARGET) + +# Add definitions for the build target +if (WAMR_BUILD_TARGET STREQUAL "X86_64") + add_definitions(-DBUILD_TARGET_X86_64) +elseif (WAMR_BUILD_TARGET STREQUAL "AMD_64") + add_definitions(-DBUILD_TARGET_AMD_64) +elseif (WAMR_BUILD_TARGET STREQUAL "X86_32") + add_definitions(-DBUILD_TARGET_X86_32) +elseif (WAMR_BUILD_TARGET MATCHES "AARCH64.*") + add_definitions(-DBUILD_TARGET_AARCH64) + add_definitions(-DBUILD_TARGET="${WAMR_BUILD_TARGET}") +elseif (WAMR_BUILD_TARGET MATCHES "ARM.*") + add_definitions(-DBUILD_TARGET_ARM) + add_definitions(-DBUILD_TARGET="${WAMR_BUILD_TARGET}") +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV64" OR WAMR_BUILD_TARGET STREQUAL "RISCV64_LP64D") + add_definitions(-DBUILD_TARGET_RISCV64_LP64D) +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV64_LP64") + add_definitions(-DBUILD_TARGET_RISCV64_LP64) +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV32" OR WAMR_BUILD_TARGET STREQUAL "RISCV32_ILP32D") + add_definitions(-DBUILD_TARGET_RISCV32_ILP32D) +elseif (WAMR_BUILD_TARGET STREQUAL "RISCV32_ILP32") + add_definitions(-DBUILD_TARGET_RISCV32_ILP32) +else () + message (FATAL_ERROR "-- Build target isn't set") +endif () + +message ("-- Build as target ${WAMR_BUILD_TARGET}") + +if (CMAKE_SIZEOF_VOID_P EQUAL 8) + if (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64" + OR WAMR_BUILD_TARGET MATCHES "AARCH64.*" OR WAMR_BUILD_TARGET MATCHES "RISCV64.*") + if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + # Add -fPIC flag if build as 64-bit + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "${CMAKE_SHARED_LIBRARY_LINK_C_FLAGS} -fPIC") + set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS} -fPIC") + endif () + else () + add_definitions (-m32) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif (NOT CMAKE_BUILD_TYPE) +message ("-- CMAKE_BUILD_TYPE = " ${CMAKE_BUILD_TYPE}) + +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + add_definitions(-DBH_DEBUG=1) +endif () +if (WAMR_BUILD_DEBUG_AOT EQUAL 1) + add_definitions(-DWASM_ENABLE_DEBUG_AOT=1) +endif() + +# Enable LLVM +if (NOT WAMR_BUILD_WITH_CUSTOM_LLVM) + set (LLVM_SRC_ROOT "${PROJECT_SOURCE_DIR}/../core/deps/llvm") + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + if (NOT EXISTS "${LLVM_SRC_ROOT}/win32build") + message (FATAL_ERROR "Cannot find LLVM dir: ${LLVM_SRC_ROOT}/win32build") + endif () + set (CMAKE_PREFIX_PATH "${LLVM_SRC_ROOT}/win32build;${CMAKE_PREFIX_PATH}") + else() + if (NOT EXISTS "${LLVM_SRC_ROOT}/build") + message (FATAL_ERROR "Cannot find LLVM dir: ${LLVM_SRC_ROOT}/build") + endif () + set (CMAKE_PREFIX_PATH "${LLVM_SRC_ROOT}/build;${CMAKE_PREFIX_PATH}") + endif () +endif () + +find_package(LLVM REQUIRED CONFIG) +include_directories(${LLVM_INCLUDE_DIRS}) +add_definitions(${LLVM_DEFINITIONS}) + +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + +if (WAMR_BUILD_DEBUG_AOT EQUAL 1) + if(LLVM_BUILD_MAIN_SRC_DIR) + include_directories(${LLVM_BUILD_MAIN_SRC_DIR}/../lldb/include) + include_directories(${LLVM_BUILD_BINARY_DIR}/tools/lldb/include) + endif() + link_directories(${LLVM_LIBRARY_DIRS}) + find_library(lib_lldb NAMES lldb HINTS ${LLVM_LIBRARY_DIRS} REQUIRED) + message(STATUS "find lldb ${LLDB_ALL_PLUGINS} in: ${LLVM_LIBRARY_DIRS}") +endif() + +if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + message ("-- Collect code coverage enabled") +endif () + +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + if(NOT MSVC) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + else() + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SAFESEH:NO") + endif() +endif() + +if (NOT MSVC) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security \ + -ffunction-sections -fdata-sections \ + -Wno-unused-parameter -Wno-pedantic") + # Remove the extra spaces for better make log + string (REGEX REPLACE " *" " " CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) +endif() + +set (SHARED_DIR ../core/shared) +set (IWASM_DIR ../core/iwasm) + +include_directories (${SHARED_DIR}/include + ${IWASM_DIR}/include) + +enable_language (ASM) + +if (NOT MINGW AND NOT MSVC) + if ((NOT DEFINED WAMR_BUILD_LIBC_WASI) AND (NOT DEFINED WAMR_BUILD_LIBC_UVWASI)) + set (WAMR_BUILD_LIBC_WASI 1) + endif () + + if ((WAMR_BUILD_LIBC_WASI EQUAL 1) AND (WAMR_BUILD_LIBC_UVWASI EQUAL 1)) + message (WARNING "-- pick WAMR_BULID_LIBC_UVWASI when both are enabled") + set (WAMR_BUILD_LIBC_WASI 0) + endif () +else () + if (NOT DEFINED WAMR_BUILD_LIBC_UVWASI) + set (WAMR_BUILD_LIBC_UVWASI 1) + endif () + + if (WAMR_BUILD_LIBC_WASI EQUAL 1) + message (WARNING "-- don't accept WAMR_BUILD_LIBC_WASI=1 on MINGW or MSVC") + set (WAMR_BUILD_LIBC_WASI 0) + endif () +endif () + +if (NOT DEFINED WAMR_BUILD_LIBC_BUILTIN) + set (WAMR_BUILD_LIBC_BUILTIN 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + set (WAMR_BUILD_LIB_PTHREAD 1) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS) + set (WAMR_BUILD_LIB_WASI_THREADS 1) +endif () + +if (WAMR_BUILD_LIBC_UVWASI EQUAL 1) + message ("-- Libc WASI enabled with uvwasi implementation") +endif () + +if (WAMR_BUILD_LIBC_WASI EQUAL 1) + message ("-- Libc WASI enabled") +endif () + +if ((NOT WAMR_BUILD_LIBC_WASI) AND (NOT WAMR_BUILD_LIBC_UVWASI)) + message ("-- Libc WASI disabled") +endif () + +if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) + message ("-- Libc builtin enabled") +else () + message ("-- Libc builtin disabled") +endif () + +if (WAMR_BUILD_LIB_PTHREAD EQUAL 1) + message ("-- Lib pthread enabled") +else () + message ("-- Lib pthread disabled") +endif () + +if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) + message ("-- Lib wasi-threads enabled") +else () + message ("-- Lib wasi-threads disabled") +endif () + +include (${SHARED_DIR}/platform/${WAMR_BUILD_PLATFORM}/shared_platform.cmake) +include (${SHARED_DIR}/mem-alloc/mem_alloc.cmake) +include (${SHARED_DIR}/utils/shared_utils.cmake) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) +include (${IWASM_DIR}/libraries/thread-mgr/thread_mgr.cmake) +include (${IWASM_DIR}/common/iwasm_common.cmake) +include (${IWASM_DIR}/common/gc/iwasm_gc.cmake) +include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) +include (${IWASM_DIR}/aot/iwasm_aot.cmake) +include (${IWASM_DIR}/compilation/iwasm_compl.cmake) + +if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) + include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) +endif () + +if (WAMR_BUILD_LIBC_UVWASI EQUAL 1) + include (${IWASM_DIR}/libraries/libc-uvwasi/libc_uvwasi.cmake) +endif () + +if (WAMR_BUILD_LIBC_WASI EQUAL 1) + include (${IWASM_DIR}/libraries/libc-wasi/libc_wasi.cmake) +endif () + +if (WAMR_BUILD_LIB_PTHREAD EQUAL 1) + include (${IWASM_DIR}/libraries/lib-pthread/lib_pthread.cmake) +endif () + +if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) + include (${IWASM_DIR}/libraries/lib-wasi-threads/lib_wasi_threads.cmake) +endif () + +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") +if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang" OR MSVC)) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + # UNDEFINED BEHAVIOR, refer to https://en.cppreference.com/w/cpp/language/ub + if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT WAMR_BUILD_JIT EQUAL 1) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined \ + -fno-sanitize=bounds,bounds-strict,alignment \ + -fno-sanitize-recover") + set(lib_ubsan -fsanitize=undefined) + endif() + else () + # UNDEFINED BEHAVIOR, refer to https://en.cppreference.com/w/cpp/language/ub + if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT WAMR_BUILD_JIT EQUAL 1) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined \ + -fno-sanitize=bounds,alignment \ + -fno-sanitize-recover") + set(lib_ubsan -fsanitize=undefined) + endif() + endif() +endif () + +if (NOT MSVC) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-strong --param ssp-buffer-size=4") +endif() + +# We disable these flags by default to stay the same with wasm runtime +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch=thunk -mfunction-return=thunk") + +if (NOT MSVC) + add_definitions(-D_FORTIFY_SOURCE=2) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftrapv") +endif() + +if (WIN32) + add_definitions(-D_WINSOCK_DEPRECATED_NO_WARNINGS) +endif() + +# message ("-- CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") + +add_library (vmlib + ${PLATFORM_SHARED_SOURCE} + ${MEM_ALLOC_SHARED_SOURCE} + ${UTILS_SHARED_SOURCE} + ${UNCOMMON_SHARED_SOURCE} + ${THREAD_MGR_SOURCE} + ${LIBC_BUILTIN_SOURCE} + ${LIBC_WASI_SOURCE} + ${LIB_PTHREAD_SOURCE} + ${LIB_WASI_THREADS_SOURCE} + ${IWASM_COMMON_SOURCE} + ${IWASM_INTERP_SOURCE} + ${IWASM_AOT_SOURCE} + ${IWASM_GC_SOURCE}) + +add_library (aotclib ${IWASM_COMPL_SOURCE}) + +add_executable (wamrc main.c) +check_pie_supported() +set_target_properties (wamrc PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (LLVM_LINK_LLVM_DYLIB) + set(WAMRC_LINK_LLVM_LIBS LLVM) +else() + set(WAMRC_LINK_LLVM_LIBS ${LLVM_AVAILABLE_LIBS}) +endif() + +if (NOT MSVC) + target_link_libraries (wamrc aotclib vmlib ${WAMRC_LINK_LLVM_LIBS} ${lib_ubsan} + -lm -lpthread ${lib_lldb} ${UV_A_LIBS}) + if (MINGW) + target_link_libraries (wamrc ssp.a ws2_32) + else() + target_link_libraries (wamrc -ldl) + endif() +else() + target_link_libraries (wamrc aotclib vmlib ${lib_lldb} ${WAMRC_LINK_LLVM_LIBS} ${lib_ubsan} + ${UV_A_LIBS}) +endif() + +install (TARGETS wamrc DESTINATION bin) diff --git a/wasm-micro-runtime/wamr-compiler/README.md b/wasm-micro-runtime/wamr-compiler/README.md new file mode 100644 index 0000000..b9e566a --- /dev/null +++ b/wasm-micro-runtime/wamr-compiler/README.md @@ -0,0 +1,32 @@ + +### Build wamrc AOT compiler + +Both wasm binary file and AOT file are supported by iwasm. The wamrc AOT compiler is to compile wasm binary file to AOT file which can also be run by iwasm. You can execute following commands to build **wamrc** compiler: + +For **Linux**(Ubuntu 20.04 as an example): + +First, make sure necessary dependency are installed: + +```shell +sudo apt-get install git build-essential cmake g++-multilib libgcc-9-dev lib32gcc-9-dev ccache +``` + +```shell +cd wamr-compiler +./build_llvm.sh (or "./build_llvm_xtensa.sh" to support xtensa target) +mkdir build && cd build +cmake .. (or "cmake .. -DWAMR_BUILD_PLATFORM=darwin" for MacOS) +make +# wamrc is generated under current directory +``` + +For **Windows**： + +```shell +cd wamr-compiler +python build_llvm.py +mkdir build && cd build +cmake .. +cmake --build . --config Release +# wamrc.exe is generated under .\Release directory +``` diff --git a/wasm-micro-runtime/wamr-compiler/build_llvm.py b/wasm-micro-runtime/wamr-compiler/build_llvm.py new file mode 100644 index 0000000..6597f61 --- /dev/null +++ b/wasm-micro-runtime/wamr-compiler/build_llvm.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import pathlib +import subprocess +import sys + +script = ( + pathlib.Path(__file__).parent.joinpath("../build-scripts/build_llvm.py").resolve() +) +subprocess.check_call([sys.executable, script]) diff --git a/wasm-micro-runtime/wamr-compiler/build_llvm.sh b/wasm-micro-runtime/wamr-compiler/build_llvm.sh new file mode 100755 index 0000000..c3ec54b --- /dev/null +++ b/wasm-micro-runtime/wamr-compiler/build_llvm.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copyright (C) 2020 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt +/usr/bin/env python3 ../build-scripts/build_llvm.py "$@" diff --git a/wasm-micro-runtime/wamr-compiler/build_llvm_arc.sh b/wasm-micro-runtime/wamr-compiler/build_llvm_arc.sh new file mode 100755 index 0000000..d148e11 --- /dev/null +++ b/wasm-micro-runtime/wamr-compiler/build_llvm_arc.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copyright (C) 2020 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt +/usr/bin/env python3 ../build-scripts/build_llvm.py --platform arc "$@" diff --git a/wasm-micro-runtime/wamr-compiler/build_llvm_xtensa.sh b/wasm-micro-runtime/wamr-compiler/build_llvm_xtensa.sh new file mode 100755 index 0000000..183ea37 --- /dev/null +++ b/wasm-micro-runtime/wamr-compiler/build_llvm_xtensa.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copyright (C) 2020 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt +/usr/bin/env python3 ../build-scripts/build_llvm.py --platform xtensa "$@" diff --git a/wasm-micro-runtime/wamr-compiler/main.c b/wasm-micro-runtime/wamr-compiler/main.c new file mode 100644 index 0000000..1044014 --- /dev/null +++ b/wasm-micro-runtime/wamr-compiler/main.c @@ -0,0 +1,744 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include "bh_platform.h" +#include "bh_read_file.h" +#include "wasm_export.h" +#include "aot_export.h" + +#if BH_HAS_DLFCN +#include + +typedef uint32 (*get_native_lib_func)(char **p_module_name, + NativeSymbol **p_native_symbols); + +static uint32 +load_and_register_native_libs(const char **native_lib_list, + uint32 native_lib_count, + void **native_handle_list) +{ + uint32 i, native_handle_count = 0, n_native_symbols; + NativeSymbol *native_symbols; + char *module_name; + void *handle; + + for (i = 0; i < native_lib_count; i++) { + /* open the native library */ + if (!(handle = dlopen(native_lib_list[i], RTLD_NOW | RTLD_GLOBAL)) + && !(handle = dlopen(native_lib_list[i], RTLD_LAZY))) { + LOG_WARNING("warning: failed to load native library %s", + native_lib_list[i]); + continue; + } + + /* lookup get_native_lib func */ + get_native_lib_func get_native_lib = dlsym(handle, "get_native_lib"); + if (!get_native_lib) { + LOG_WARNING("warning: failed to lookup `get_native_lib` function " + "from native lib %s", + native_lib_list[i]); + dlclose(handle); + continue; + } + + n_native_symbols = get_native_lib(&module_name, &native_symbols); + + /* register native symbols */ + if (!(n_native_symbols > 0 && module_name && native_symbols + && wasm_runtime_register_natives(module_name, native_symbols, + n_native_symbols))) { + LOG_WARNING("warning: failed to register native lib %s", + native_lib_list[i]); + dlclose(handle); + continue; + } + + native_handle_list[native_handle_count++] = handle; + } + + return native_handle_count; +} + +static void +unregister_and_unload_native_libs(uint32 native_lib_count, + void **native_handle_list) +{ + uint32 i, n_native_symbols; + NativeSymbol *native_symbols; + char *module_name; + void *handle; + + for (i = 0; i < native_lib_count; i++) { + handle = native_handle_list[i]; + + /* lookup get_native_lib func */ + get_native_lib_func get_native_lib = dlsym(handle, "get_native_lib"); + if (!get_native_lib) { + LOG_WARNING("warning: failed to lookup `get_native_lib` function " + "from native lib %p", + handle); + continue; + } + + n_native_symbols = get_native_lib(&module_name, &native_symbols); + if (n_native_symbols == 0 || module_name == NULL + || native_symbols == NULL) { + LOG_WARNING("warning: get_native_lib returned different values for " + "native lib %p", + handle); + continue; + } + + /* unregister native symbols */ + if (!wasm_runtime_unregister_natives(module_name, native_symbols)) { + LOG_WARNING("warning: failed to unregister native lib %p", handle); + continue; + } + + dlclose(handle); + } +} +#endif + +/* clang-format off */ +static void +print_help() +{ + printf("Usage: wamrc [options] -o output_file wasm_file\n"); + printf(" --target= Set the target arch, which has the general format: \n"); + printf(" = x86_64, i386, aarch64, arm, thumb, xtensa, mips,\n"); + printf(" riscv64, riscv32.\n"); + printf(" Default is host arch, e.g. x86_64\n"); + printf(" = for ex. on arm or thumb: v5, v6m, v7a, v7m, etc.\n"); + printf(" Use --target=help to list supported targets\n"); + printf(" --target-abi= Set the target ABI, e.g. gnu, eabi, gnueabihf, msvc, etc.\n"); + printf(" Default is gnu if target isn't riscv64 or riscv32\n"); + printf(" For target riscv64 and riscv32, default is lp64d and ilp32d\n"); + printf(" Use --target-abi=help to list all the ABI supported\n"); + printf(" --cpu= Set the target CPU (default: host CPU, e.g. skylake)\n"); + printf(" Use --cpu=help to list all the CPU supported\n"); + printf(" --cpu-features= Enable or disable the CPU features\n"); + printf(" Use +feature to enable a feature, or -feature to disable it\n"); + printf(" For example, --cpu-features=+feature1,-feature2\n"); + printf(" Use --cpu-features=+help to list all the features supported\n"); + printf(" --opt-level=n Set the optimization level (0 to 3, default is 3)\n"); + printf(" --size-level=n Set the code size level (0 to 3, default is 3)\n"); + printf(" -sgx Generate code for SGX platform (Intel Software Guard Extensions)\n"); + printf(" --bounds-checks=1/0 Enable or disable the bounds checks for memory access:\n"); + printf(" by default it is disabled in all 64-bit platforms except SGX and\n"); + printf(" in these platforms runtime does bounds checks with hardware trap,\n"); + printf(" and by default it is enabled in all 32-bit platforms\n"); + printf(" CAVEAT: --bounds-checks=0 enables some optimizations\n"); + printf(" which make the compiled AOT module incompatible\n"); + printf(" with a runtime without the hardware bounds checks.\n"); + printf(" --stack-bounds-checks=1/0 Enable or disable the bounds checks for native stack:\n"); + printf(" if the option isn't set, the status is same as `--bounds-check`,\n"); + printf(" if the option is set:\n"); + printf(" (1) it is always enabled when `--bounds-checks` is enabled,\n"); + printf(" (2) else it is enabled/disabled according to the option value\n"); + printf(" --stack-usage= Generate a stack-usage file.\n"); + printf(" Similarly to `clang -fstack-usage`.\n"); + printf(" --format= Specifies the format of the output file\n"); + printf(" The format supported:\n"); + printf(" aot (default) AoT file\n"); + printf(" object Native object file\n"); + printf(" llvmir-unopt Unoptimized LLVM IR\n"); + printf(" llvmir-opt Optimized LLVM IR\n"); + printf(" --disable-bulk-memory Disable the MVP bulk memory feature\n"); + printf(" --enable-multi-thread Enable multi-thread feature, the dependent features bulk-memory and\n"); + printf(" thread-mgr will be enabled automatically\n"); + printf(" --enable-tail-call Enable the post-MVP tail call feature\n"); + printf(" --disable-simd Disable the post-MVP 128-bit SIMD feature:\n"); + printf(" currently 128-bit SIMD is supported for x86-64 and aarch64 targets,\n"); + printf(" and by default it is enabled in them and disabled in other targets\n"); + printf(" --disable-ref-types Disable the MVP reference types feature, it will be disabled forcibly if\n"); + printf(" GC is enabled\n"); + printf(" --disable-aux-stack-check Disable auxiliary stack overflow/underflow check\n"); + printf(" --enable-dump-call-stack Enable stack trace feature\n"); + printf(" --enable-perf-profiling Enable function performance profiling\n"); + printf(" --enable-memory-profiling Enable memory usage profiling\n"); + printf(" --xip A shorthand of --enalbe-indirect-mode --disable-llvm-intrinsics\n"); + printf(" --enable-indirect-mode Enalbe call function through symbol table but not direct call\n"); + printf(" --enable-gc Enalbe GC (Garbage Collection) feature\n"); + printf(" --disable-llvm-intrinsics Disable the LLVM built-in intrinsics\n"); + printf(" --enable-builtin-intrinsics=\n"); + printf(" Enable the specified built-in intrinsics, it will override the default\n"); + printf(" settings. It only takes effect when --disable-llvm-intrinsics is set.\n"); + printf(" Available flags: all, i32.common, i64.common, f32.common, f64.common,\n"); + printf(" i32.clz, i32.ctz, etc, refer to doc/xip.md for full list\n"); + printf(" Use comma to separate, please refer to doc/xip.md for full list.\n"); + printf(" --disable-llvm-lto Disable the LLVM link time optimization\n"); + printf(" --enable-llvm-pgo Enable LLVM PGO (Profile-Guided Optimization)\n"); + printf(" --enable-llvm-passes=\n"); + printf(" Enable the specified LLVM passes, using comma to separate\n"); + printf(" --use-prof-file= Use profile file collected by LLVM PGO (Profile-Guided Optimization)\n"); + printf(" --enable-segue[=] Enable using segment register GS as the base address of linear memory,\n"); + printf(" only available on linux x86-64, which may improve performance,\n"); + printf(" flags can be: i32.load, i64.load, f32.load, f64.load, v128.load,\n"); + printf(" i32.store, i64.store, f32.store, f64.store, v128.store\n"); + printf(" Use comma to separate, e.g. --enable-segue=i32.load,i64.store\n"); + printf(" and --enable-segue means all flags are added.\n"); + printf(" --emit-custom-sections=
\n"); + printf(" Emit the specified custom sections to AoT file, using comma to separate\n"); + printf(" multiple names, e.g.\n"); + printf(" --emit-custom-sections=section1,section2,sectionN\n"); +#if BH_HAS_DLFCN + printf(" --native-lib= Register native libraries to the WASM module, which\n"); + printf(" are shared object (.so) files, for example:\n"); + printf(" --native-lib=test1.so --native-lib=test2.so\n"); +#endif + printf(" --invoke-c-api-import Treat unknown import function as wasm-c-api import function and\n"); + printf(" quick call it from AOT code\n"); +#if WASM_ENABLE_LINUX_PERF != 0 + printf(" --enable-linux-perf Enable linux perf support\n"); +#endif + printf(" -v=n Set log verbose level (0 to 5, default is 2), larger with more log\n"); + printf(" --version Show version information\n"); + printf("Examples: wamrc -o test.aot test.wasm\n"); + printf(" wamrc --target=i386 -o test.aot test.wasm\n"); + printf(" wamrc --target=i386 --format=object -o test.o test.wasm\n"); + printf(" wamrc --target-abi=help\n"); + printf(" wamrc --target=x86_64 --cpu=help\n"); +} +/* clang-format on */ + +#define PRINT_HELP_AND_EXIT() \ + do { \ + print_help(); \ + goto fail0; \ + } while (0) + +/** + * Split a string into an array of strings + * Returns NULL on failure + * Memory must be freed by caller + * Based on: http://stackoverflow.com/a/11198630/471795 + */ +static char ** +split_string(char *str, int *count, const char *delimer) +{ + char **res = NULL, **res1; + char *p; + int idx = 0; + + /* split string and append tokens to 'res' */ + do { + p = strtok(str, delimer); + str = NULL; + res1 = res; + res = (char **)realloc(res1, sizeof(char *) * (uint32)(idx + 1)); + if (res == NULL) { + free(res1); + return NULL; + } + res[idx++] = p; + } while (p); + + /** + * Due to the section name, + * res[0] might contain a '\' to indicate a space + * func\name -> func name + */ + p = strchr(res[0], '\\'); + while (p) { + *p = ' '; + p = strchr(p, '\\'); + } + + if (count) { + *count = idx - 1; + } + return res; +} + +static uint32 +resolve_segue_flags(char *str_flags) +{ + uint32 segue_flags = 0; + int32 flag_count, i; + char **flag_list; + + flag_list = split_string(str_flags, &flag_count, ","); + if (flag_list) { + for (i = 0; i < flag_count; i++) { + if (!strcmp(flag_list[i], "i32.load")) { + segue_flags |= 1 << 0; + } + else if (!strcmp(flag_list[i], "i64.load")) { + segue_flags |= 1 << 1; + } + else if (!strcmp(flag_list[i], "f32.load")) { + segue_flags |= 1 << 2; + } + else if (!strcmp(flag_list[i], "f64.load")) { + segue_flags |= 1 << 3; + } + else if (!strcmp(flag_list[i], "v128.load")) { + segue_flags |= 1 << 4; + } + else if (!strcmp(flag_list[i], "i32.store")) { + segue_flags |= 1 << 8; + } + else if (!strcmp(flag_list[i], "i64.store")) { + segue_flags |= 1 << 9; + } + else if (!strcmp(flag_list[i], "f32.store")) { + segue_flags |= 1 << 10; + } + else if (!strcmp(flag_list[i], "f64.store")) { + segue_flags |= 1 << 11; + } + else if (!strcmp(flag_list[i], "v128.store")) { + segue_flags |= 1 << 12; + } + else { + /* invalid flag */ + segue_flags = (uint32)-1; + break; + } + } + free(flag_list); + } + return segue_flags; +} + +/* When print help info for target/cpu/target-abi/cpu-features, load this dummy + * wasm file content rather than from an input file, the dummy wasm file content + * is: magic header + version number */ +static unsigned char dummy_wasm_file[8] = { 0x00, 0x61, 0x73, 0x6D, + 0x01, 0x00, 0x00, 0x00 }; + +int +main(int argc, char *argv[]) +{ + char *wasm_file_name = NULL, *out_file_name = NULL; + uint8 *wasm_file = NULL; + uint32 wasm_file_size; + wasm_module_t wasm_module = NULL; + aot_comp_data_t comp_data = NULL; + aot_comp_context_t comp_ctx = NULL; + RuntimeInitArgs init_args; + AOTCompOption option = { 0 }; + char error_buf[128]; + int log_verbose_level = 2; + bool sgx_mode = false, size_level_set = false, use_dummy_wasm = false; + int exit_status = EXIT_FAILURE; +#if BH_HAS_DLFCN + const char *native_lib_list[8] = { NULL }; + uint32 native_lib_count = 0; + void *native_handle_list[8] = { NULL }; + uint32 native_handle_count = 0; +#endif +#if WASM_ENABLE_LINUX_PERF != 0 + bool enable_linux_perf = false; +#endif + + option.opt_level = 3; + option.size_level = 3; + option.output_format = AOT_FORMAT_FILE; + /* default value, enable or disable depends on the platform */ + option.bounds_checks = 2; + /* default value, enable or disable depends on the platform */ + option.stack_bounds_checks = 2; + option.enable_simd = true; + option.enable_aux_stack_check = true; + option.enable_bulk_memory = true; + option.enable_ref_types = true; + option.enable_gc = false; + + /* Process options */ + for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { + if (!strcmp(argv[0], "-o")) { + argc--, argv++; + if (argc < 2) + PRINT_HELP_AND_EXIT(); + out_file_name = argv[0]; + } + else if (!strncmp(argv[0], "--target=", 9)) { + if (argv[0][9] == '\0') + PRINT_HELP_AND_EXIT(); + option.target_arch = argv[0] + 9; + if (!strcmp(option.target_arch, "help")) { + use_dummy_wasm = true; + } + } + else if (!strncmp(argv[0], "--target-abi=", 13)) { + if (argv[0][13] == '\0') + PRINT_HELP_AND_EXIT(); + option.target_abi = argv[0] + 13; + if (!strcmp(option.target_abi, "help")) { + use_dummy_wasm = true; + } + } + else if (!strncmp(argv[0], "--cpu=", 6)) { + if (argv[0][6] == '\0') + PRINT_HELP_AND_EXIT(); + option.target_cpu = argv[0] + 6; + if (!strcmp(option.target_cpu, "help")) { + use_dummy_wasm = true; + } + } + else if (!strncmp(argv[0], "--cpu-features=", 15)) { + if (argv[0][15] == '\0') + PRINT_HELP_AND_EXIT(); + option.cpu_features = argv[0] + 15; + if (!strcmp(option.cpu_features, "+help")) { + use_dummy_wasm = true; + } + } + else if (!strncmp(argv[0], "--opt-level=", 12)) { + if (argv[0][12] == '\0') + PRINT_HELP_AND_EXIT(); + option.opt_level = (uint32)atoi(argv[0] + 12); + if (option.opt_level > 3) + option.opt_level = 3; + } + else if (!strncmp(argv[0], "--size-level=", 13)) { + if (argv[0][13] == '\0') + PRINT_HELP_AND_EXIT(); + option.size_level = (uint32)atoi(argv[0] + 13); + if (option.size_level > 3) + option.size_level = 3; + size_level_set = true; + } + else if (!strcmp(argv[0], "-sgx")) { + sgx_mode = true; + } + else if (!strncmp(argv[0], "--bounds-checks=", 16)) { + option.bounds_checks = (atoi(argv[0] + 16) == 1) ? 1 : 0; + } + else if (!strncmp(argv[0], "--stack-bounds-checks=", 22)) { + option.stack_bounds_checks = (atoi(argv[0] + 22) == 1) ? 1 : 0; + } + else if (!strncmp(argv[0], "--stack-usage=", 14)) { + option.stack_usage_file = argv[0] + 14; + } + else if (!strncmp(argv[0], "--format=", 9)) { + if (argv[0][9] == '\0') + PRINT_HELP_AND_EXIT(); + if (!strcmp(argv[0] + 9, "aot")) + option.output_format = AOT_FORMAT_FILE; + else if (!strcmp(argv[0] + 9, "object")) + option.output_format = AOT_OBJECT_FILE; + else if (!strcmp(argv[0] + 9, "llvmir-unopt")) + option.output_format = AOT_LLVMIR_UNOPT_FILE; + else if (!strcmp(argv[0] + 9, "llvmir-opt")) + option.output_format = AOT_LLVMIR_OPT_FILE; + else { + printf("Invalid format %s.\n", argv[0] + 9); + PRINT_HELP_AND_EXIT(); + } + } + else if (!strncmp(argv[0], "-v=", 3)) { + log_verbose_level = atoi(argv[0] + 3); + if (log_verbose_level < 0 || log_verbose_level > 5) + PRINT_HELP_AND_EXIT(); + } + else if (!strcmp(argv[0], "--disable-bulk-memory")) { + option.enable_bulk_memory = false; + } + else if (!strcmp(argv[0], "--enable-multi-thread")) { + option.enable_bulk_memory = true; + option.enable_thread_mgr = true; + } + else if (!strcmp(argv[0], "--enable-tail-call")) { + option.enable_tail_call = true; + } + else if (!strcmp(argv[0], "--enable-simd")) { + /* obsolete option, kept for compatibility */ + option.enable_simd = true; + } + else if (!strcmp(argv[0], "--disable-simd")) { + option.enable_simd = false; + } + else if (!strcmp(argv[0], "--disable-ref-types")) { + option.enable_ref_types = false; + } + else if (!strcmp(argv[0], "--disable-aux-stack-check")) { + option.enable_aux_stack_check = false; + } + else if (!strcmp(argv[0], "--enable-dump-call-stack")) { + option.enable_aux_stack_frame = true; + } + else if (!strcmp(argv[0], "--enable-perf-profiling")) { + option.enable_aux_stack_frame = true; + option.enable_perf_profiling = true; + } + else if (!strcmp(argv[0], "--enable-memory-profiling")) { + option.enable_memory_profiling = true; + option.enable_stack_estimation = true; + } + else if (!strcmp(argv[0], "--xip")) { + option.is_indirect_mode = true; + option.disable_llvm_intrinsics = true; + } + else if (!strcmp(argv[0], "--enable-indirect-mode")) { + option.is_indirect_mode = true; + } + else if (!strcmp(argv[0], "--enable-gc")) { + option.enable_aux_stack_frame = true; + option.enable_gc = true; + } + else if (!strcmp(argv[0], "--disable-llvm-intrinsics")) { + option.disable_llvm_intrinsics = true; + } + else if (!strncmp(argv[0], "--enable-builtin-intrinsics=", 28)) { + if (argv[0][28] == '\0') + PRINT_HELP_AND_EXIT(); + option.builtin_intrinsics = argv[0] + 28; + } + else if (!strcmp(argv[0], "--disable-llvm-lto")) { + option.disable_llvm_lto = true; + } + else if (!strcmp(argv[0], "--enable-llvm-pgo")) { + option.enable_llvm_pgo = true; + } + else if (!strncmp(argv[0], "--enable-llvm-passes=", 21)) { + if (argv[0][21] == '\0') + PRINT_HELP_AND_EXIT(); + option.llvm_passes = argv[0] + 21; + } + else if (!strncmp(argv[0], "--use-prof-file=", 16)) { + if (argv[0][16] == '\0') + PRINT_HELP_AND_EXIT(); + option.use_prof_file = argv[0] + 16; + } + else if (!strcmp(argv[0], "--enable-segue")) { + /* all flags are enabled */ + option.segue_flags = 0x1F1F; + } + else if (!strncmp(argv[0], "--enable-segue=", 15)) { + option.segue_flags = resolve_segue_flags(argv[0] + 15); + if (option.segue_flags == (uint32)-1) + PRINT_HELP_AND_EXIT(); + } + else if (!strncmp(argv[0], "--emit-custom-sections=", 23)) { + int len = 0; + if (option.custom_sections) { + free(option.custom_sections); + } + + option.custom_sections = split_string(argv[0] + 23, &len, ","); + if (!option.custom_sections) { + printf("Failed to process emit-custom-sections: alloc " + "memory failed\n"); + PRINT_HELP_AND_EXIT(); + } + + option.custom_sections_count = len; + } +#if BH_HAS_DLFCN + else if (!strncmp(argv[0], "--native-lib=", 13)) { + if (argv[0][13] == '\0') + PRINT_HELP_AND_EXIT(); + if (native_lib_count >= sizeof(native_lib_list) / sizeof(char *)) { + printf("Only allow max native lib number %d\n", + (int)(sizeof(native_lib_list) / sizeof(char *))); + goto fail0; + } + native_lib_list[native_lib_count++] = argv[0] + 13; + } +#endif + else if (!strcmp(argv[0], "--invoke-c-api-import")) { + option.quick_invoke_c_api_import = true; + } +#if WASM_ENABLE_LINUX_PERF != 0 + else if (!strcmp(argv[0], "--enable-linux-perf")) { + enable_linux_perf = true; + } +#endif + else if (!strcmp(argv[0], "--version")) { + uint32 major, minor, patch; + wasm_runtime_get_version(&major, &minor, &patch); + printf("wamrc %u.%u.%u\n", major, minor, patch); + return 0; + } + else + PRINT_HELP_AND_EXIT(); + } + + if (!use_dummy_wasm && (argc == 0 || !out_file_name)) + PRINT_HELP_AND_EXIT(); + + if (!size_level_set) { + /** + * Set opt level to 1 by default for Windows and MacOS as + * they can not memory map out 0-2GB memory and might not + * be able to meet the requirements of some AOT relocation + * operations. + */ + if (option.target_abi && !strcmp(option.target_abi, "msvc")) { + LOG_VERBOSE("Set size level to 1 for Windows AOT file"); + option.size_level = 1; + } +#if defined(_WIN32) || defined(_WIN32_) || defined(__APPLE__) \ + || defined(__MACH__) + if (!option.target_abi) { + LOG_VERBOSE("Set size level to 1 for Windows or MacOS AOT file"); + option.size_level = 1; + } +#endif + } + + if (sgx_mode) { + option.size_level = 1; + option.is_sgx_platform = true; + } + + if (option.enable_gc) { + option.enable_ref_types = false; + } + + if (!use_dummy_wasm) { + wasm_file_name = argv[0]; + + if (!strcmp(wasm_file_name, out_file_name)) { + printf("Error: input file and output file are the same"); + return -1; + } + } + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Allocator; + init_args.mem_alloc_option.allocator.malloc_func = malloc; + init_args.mem_alloc_option.allocator.realloc_func = realloc; + init_args.mem_alloc_option.allocator.free_func = free; +#if WASM_ENABLE_LINUX_PERF != 0 + init_args.enable_linux_perf = enable_linux_perf; +#endif + + /* initialize runtime environment */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + bh_log_set_verbose_level(log_verbose_level); + +#if BH_HAS_DLFCN + bh_print_time("Begin to load native libs"); + native_handle_count = load_and_register_native_libs( + native_lib_list, native_lib_count, native_handle_list); +#endif + + bh_print_time("Begin to load wasm file"); + + if (use_dummy_wasm) { + /* load WASM byte buffer from dummy buffer */ + wasm_file_size = sizeof(dummy_wasm_file); + wasm_file = dummy_wasm_file; + } + else { + /* load WASM byte buffer from WASM bin file */ + if (!(wasm_file = (uint8 *)bh_read_file_to_buffer(wasm_file_name, + &wasm_file_size))) + goto fail1; + } + + if (wasm_file_size >= 4 /* length of MAGIC NUMBER */ + && get_package_type(wasm_file, wasm_file_size) + != Wasm_Module_Bytecode) { + printf("Invalid wasm file: magic header not detected\n"); + goto fail2; + } + + /* load WASM module */ + if (!(wasm_module = wasm_runtime_load(wasm_file, wasm_file_size, error_buf, + sizeof(error_buf)))) { + printf("%s\n", error_buf); + goto fail2; + } + + if (!(comp_data = aot_create_comp_data(wasm_module, option.target_arch, + option.enable_gc))) { + printf("%s\n", aot_get_last_error()); + goto fail3; + } + +#if WASM_ENABLE_DEBUG_AOT != 0 + if (!create_dwarf_extractor(comp_data, wasm_file_name)) { + printf("%s:create dwarf extractor failed\n", wasm_file_name); + } +#endif + + bh_print_time("Begin to create compile context"); + + if (!(comp_ctx = aot_create_comp_context(comp_data, &option))) { + printf("%s\n", aot_get_last_error()); + goto fail4; + } + + bh_print_time("Begin to compile"); + + if (!aot_compile_wasm(comp_ctx)) { + printf("%s\n", aot_get_last_error()); + goto fail5; + } + + switch (option.output_format) { + case AOT_LLVMIR_UNOPT_FILE: + case AOT_LLVMIR_OPT_FILE: + if (!aot_emit_llvm_file(comp_ctx, out_file_name)) { + printf("%s\n", aot_get_last_error()); + goto fail5; + } + break; + case AOT_OBJECT_FILE: + if (!aot_emit_object_file(comp_ctx, out_file_name)) { + printf("%s\n", aot_get_last_error()); + goto fail5; + } + break; + case AOT_FORMAT_FILE: + if (!aot_emit_aot_file(comp_ctx, comp_data, out_file_name)) { + printf("%s\n", aot_get_last_error()); + goto fail5; + } + break; + default: + break; + } + + bh_print_time("Compile end"); + + printf("Compile success, file %s was generated.\n", out_file_name); + exit_status = EXIT_SUCCESS; + +fail5: + /* Destroy compiler context */ + aot_destroy_comp_context(comp_ctx); + +fail4: + /* Destroy compile data */ + aot_destroy_comp_data(comp_data); + +fail3: + /* Unload WASM module */ + wasm_runtime_unload(wasm_module); + +fail2: + /* free the file buffer */ + if (!use_dummy_wasm) { + wasm_runtime_free(wasm_file); + } + +fail1: +#if BH_HAS_DLFCN + unregister_and_unload_native_libs(native_handle_count, native_handle_list); +#endif + /* Destroy runtime environment */ + wasm_runtime_destroy(); + +fail0: + /* free option.custom_sections */ + if (option.custom_sections) { + free(option.custom_sections); + } + + bh_print_time("wamrc return"); + return exit_status; +} diff --git a/wasm-micro-runtime/wamr-sdk/README.md b/wasm-micro-runtime/wamr-sdk/README.md new file mode 100644 index 0000000..ef0cfd2 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/README.md @@ -0,0 +1,3 @@ +# Notice +`wamr-sdk` has been migrated to **[wamr-app-framework/wamr-sdk](https://github.com/bytecodealliance/wamr-app-framework/tree/main/wamr-sdk)**. +Only some necessary files that wasm app will use are retained here diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/assert.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/assert.h new file mode 100644 index 0000000..534a90d --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/assert.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_ASSERT_H +#define _WAMR_LIBC_ASSERT_H + +#ifdef __cplusplus +extern "C" { +#endif + + + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/ctype.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/ctype.h new file mode 100644 index 0000000..1a3cd4e --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/ctype.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_CTYPE_H +#define _WAMR_LIBC_CTYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +int isupper(int c); +int isalpha(int c); +int isspace(int c); +int isgraph(int c); +int isprint(int c); +int isdigit(int c); +int isxdigit(int c); +int tolower(int c); +int toupper(int c); +int isalnum(int c); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/errno.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/errno.h new file mode 100644 index 0000000..9e9cca1 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/errno.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_ERRNO_H +#define _WAMR_LIBC_ERRNO_H + +#ifdef __cplusplus +extern "C" { +#endif + + + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/fcntl.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/fcntl.h new file mode 100644 index 0000000..a67c24a --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/fcntl.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_FCNTL_H +#define _WAMR_LIBC_FCNTL_H + +#ifdef __cplusplus +extern "C" { +#endif + + + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/inttypes.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/inttypes.h new file mode 100644 index 0000000..a04ec7b --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/inttypes.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_INTTYPES_H +#define _WAMR_LIBC_INTTYPES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/limits.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/limits.h new file mode 100644 index 0000000..d46fb4f --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/limits.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_LIMITS_H +#define _WAMR_LIBC_LIMITS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define CHAR_BIT 8 +#define SCHAR_MIN -128 +#define SCHAR_MAX 127 +#define UCHAR_MAX 255 +#define CHAR_MIN 0 +#define CHAR_MAX 127 +#define MB_LEN_MAX 1 +#define SHRT_MIN -32768 +#define SHRT_MAX +32767 +#define USHRT_MAX 65535 +#define INT_MIN -32768 +#define INT_MAX +32767 +#define UINT_MAX 65535 +#define LONG_MIN -2147483648 +#define LONG_MAX +2147483647 +#define ULONG_MAX 4294967295 + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/pthread.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/pthread.h new file mode 100644 index 0000000..10b3978 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/pthread.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIB_PTHREAD_H +#define _WAMR_LIB_PTHREAD_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Data type define of pthread, mutex, cond and key */ +typedef unsigned int pthread_t; +typedef unsigned int pthread_mutex_t; +typedef unsigned int pthread_cond_t; +typedef unsigned int pthread_key_t; + +/* Thread APIs */ +int +pthread_create(pthread_t *thread, const void *attr, + void *(*start_routine)(void *), void *arg); + +int +pthread_join(pthread_t thread, void **retval); + +int +pthread_detach(pthread_t thread); + +int +pthread_cancel(pthread_t thread); + +pthread_t +pthread_self(void); + +void +pthread_exit(void *retval); + +/* Mutex APIs */ +int +pthread_mutex_init(pthread_mutex_t *mutex, const void *attr); + +int +pthread_mutex_lock(pthread_mutex_t *mutex); + +int +pthread_mutex_unlock(pthread_mutex_t *mutex); + +int +pthread_mutex_destroy(pthread_mutex_t *mutex); + +/* Cond APIs */ +int +pthread_cond_init(pthread_cond_t *cond, const void *attr); + +int +pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex); + +int +pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, + uint64_t useconds); + +int +pthread_cond_signal(pthread_cond_t *cond); + +int +pthread_cond_broadcast(pthread_cond_t *cond); + +int +pthread_cond_destroy(pthread_cond_t *cond); + +/* Pthread key APIs */ +int +pthread_key_create(pthread_key_t *key, void (*destructor)(void *)); + +int +pthread_setspecific(pthread_key_t key, const void *value); + +void * +pthread_getspecific(pthread_key_t key); + +int +pthread_key_delete(pthread_key_t key); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WAMR_LIB_PTHREAD_H */ diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/semaphore.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/semaphore.h new file mode 100644 index 0000000..2e086bd --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/semaphore.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIB_SEMAPHORE_H +#define _WAMR_LIB_SEMAPHORE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef unsigned int sem_t; + +/* Semaphore APIs */ + +sem_t * +sem_open(const char *name, int oflag, int mode, int val); + +int +sem_wait(sem_t *sem); + +int +sem_trywait(sem_t *sem); + +int +sem_post(sem_t *sem); + +int +sem_getvalue(sem_t *restrict sem, int *sval); + +int +sem_unlink(const char *name); + +int +sem_close(sem_t *sem); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WAMR_LIB_SEMAPHORE_H */ diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdarg.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdarg.h new file mode 100644 index 0000000..5095957 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdarg.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDARG_H +#define _WAMR_LIBC_STDARG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _VA_LIST +typedef __builtin_va_list va_list; +#define _VA_LIST +#endif +#define va_start(ap, param) __builtin_va_start(ap, param) +#define va_end(ap) __builtin_va_end(ap) +#define va_arg(ap, type) __builtin_va_arg(ap, type) + +#define __va_copy(d, s) __builtin_va_copy(d, s) + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WAMR_LIBC_STDARG_H */ diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdbool.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdbool.h new file mode 100644 index 0000000..29e03f9 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdbool.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDBOOL_H +#define _WAMR_LIBC_STDBOOL_H + +#define __bool_true_false_are_defined 1 + +#ifndef __cplusplus + +#define bool _Bool +#define false 0 +#define true 1 + +#endif /* __cplusplus */ + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdint.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdint.h new file mode 100644 index 0000000..802e9ac --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdint.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDINT_H +#define _WAMR_LIBC_STDINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* clang-format off */ +/* The word size of platform */ +#ifdef __wasm64__ +#define __WORDSIZE 64 +#else +#define __WORDSIZE 32 +#endif + +typedef char int8_t; +typedef short int int16_t; +typedef int int32_t; +typedef long long int int64_t; + +/* Unsigned. */ +typedef unsigned char uint8_t; +typedef unsigned short int uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long int uint64_t; + +typedef __INTPTR_TYPE__ intptr_t; +typedef __UINTPTR_TYPE__ uintptr_t; + +/* Signed and unsigned */ +#if __WORDSIZE == 64 +#define INT64_C(c) c ## L +#define UINT64_C(c) c ## UL +#define INTMAX_C(c) c ## L +#define UINTMAX_C(c) c ## UL +#else +#define INT64_C(c) c ## LL +#define UINT64_C(c) c ## ULL +#define INTMAX_C(c) c ## LL +#define UINTMAX_C(c) c ## ULL +#endif + + +/* Minimum of signed integral types. */ +# define INT8_MIN (-128) +# define INT16_MIN (-32767-1) +# define INT32_MIN (-2147483647-1) +# define INT64_MIN (-INT64_C(9223372036854775807)-1) + +/* Maximum of signed integral types. */ +# define INT8_MAX (127) +# define INT16_MAX (32767) +# define INT32_MAX (2147483647) +# define INT64_MAX (INT64_C(9223372036854775807)) + +/* Maximum of unsigned integral types. */ +# define UINT8_MAX (255) +# define UINT16_MAX (65535) +# define UINT32_MAX (4294967295U) +# define UINT64_MAX (UINT64_C(18446744073709551615)) + +/* Values to test for integral types holding `void *' pointer. */ +#if __WORDSIZE == 64 +#define INTPTR_MIN INT64_MIN +#define INTPTR_MAX INT64_MAX +#define UINTPTR_MAX UINT64_MAX +#else +#define INTPTR_MIN INT32_MIN +#define INTPTR_MAX INT32_MAX +#define UINTPTR_MAX UINT32_MAX +#endif + +/* Limit of `size_t' type. */ +#if __WORDSIZE == 64 +#define SIZE_MAX UINT64_MAX +#else +#define SIZE_MAX UINT32_MAX +#endif + +/* clang-format on */ + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdio.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdio.h new file mode 100644 index 0000000..1d51e6d --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdio.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDIO_H +#define _WAMR_LIBC_STDIO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef NULL +# define NULL ((void*) 0) +#endif + +typedef unsigned long size_t; + +int printf(const char *format, ...); +int putchar(int c); +int snprintf(char *str, size_t size, const char *format, ...); +int sprintf(char *str, const char *format, ...); +int puts(char *string); + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdlib.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdlib.h new file mode 100644 index 0000000..7eb2cc4 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/stdlib.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STDLIB_H +#define _WAMR_LIBC_STDLIB_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned long size_t; + +int atoi(const char *s); +void exit(int status); +long strtol(const char *nptr, char **endptr, register int base); +unsigned long strtoul(const char *nptr, char **endptr, register int base); +void *malloc(size_t size); +void *calloc(size_t n, size_t size); +void free(void *ptr); + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/string.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/string.h new file mode 100644 index 0000000..e71a168 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/string.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STRING_H +#define _WAMR_LIBC_STRING_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned long size_t; + +int memcmp(const void *s1, const void *s2, size_t n); +void *memcpy(void *dest, const void *src, size_t n); +void *memmove(void *dest, const void *src, size_t n); +void *memset(void *s, int c, size_t n); +void *memchr(const void *s, int c, size_t n); +int strncasecmp(const char *s1, const char *s2, size_t n); +size_t strspn(const char *s, const char *accept); +size_t strcspn(const char *s, const char *reject); +char *strstr(const char *s, const char *find); +char *strchr(const char *s, int c); +int strcmp(const char *s1, const char *s2); +char *strcpy(char *dest, const char *src); +size_t strlen(const char *s); +int strncmp(const char * str1, const char * str2, size_t n); +char *strncpy(char *dest, const char *src, unsigned long n); +char * strdup(const char *s); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/strings.h b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/strings.h new file mode 100644 index 0000000..7f1bec6 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/include/strings.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WAMR_LIBC_STRINGS_H +#define _WAMR_LIBC_STRINGS_H + +#ifdef __cplusplus +extern "C" { +#endif + + + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt new file mode 100644 index 0000000..fc9c400 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt @@ -0,0 +1,87 @@ +wasm_open_connection +wasm_close_connection +wasm_send_on_connection +wasm_config_connection +wasm_sensor_open +wasm_sensor_config +wasm_sensor_config_with_attr_container +wasm_sensor_close +wasm_btn_native_call +wasm_obj_native_call +wasm_label_native_call +wasm_cont_native_call +wasm_page_native_call +wasm_list_native_call +wasm_ddlist_native_call +wasm_cb_native_call +wasm_register_resource +wasm_response_send +wasm_post_request +wasm_sub_event +wasm_create_timer +wasm_timer_destroy +wasm_timer_cancel +wasm_timer_restart +wasm_get_sys_tick_ms +printf +sprintf +snprintf +vprintf +vsprintf +vsnprintf +puts +putchar +memcmp +memcpy +memmove +memset +strchr +strcmp +strcpy +strlen +strncmp +strncpy +malloc +calloc +realloc +strdup +free +atoi +bsearch +exit +strtol +strtoul +memchr +strncasecmp +strspn +strcspn +strstr +isupper +isalpha +isspace +isgraph +isprint +isdigit +isxdigit +tolower +toupper +isalnum +pthread_create +pthread_join +pthread_detach +pthread_cancel +pthread_self +pthread_exit +pthread_mutex_init +pthread_mutex_lock +pthread_mutex_unlock +pthread_mutex_destroy +pthread_cond_init +pthread_cond_wait +pthread_cond_timedwait +pthread_cond_signal +pthread_cond_destroy +pthread_key_create +pthread_setspecific +pthread_getspecific +pthread_key_delete diff --git a/wasm-micro-runtime/wamr-sdk/app/wamr_toolchain.cmake b/wasm-micro-runtime/wamr-sdk/app/wamr_toolchain.cmake new file mode 100644 index 0000000..58fc883 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/wamr_toolchain.cmake @@ -0,0 +1,34 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +SET (CMAKE_SYSTEM_NAME Linux) +SET (CMAKE_SYSTEM_PROCESSOR wasm32) +SET (CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR}/libc-builtin-sysroot) + + +IF (NOT (DEFINED WASI_SDK_DIR OR DEFINED CACHE{WASI_SDK_DIR})) + MESSAGE (FATAL_ERROR "WASI_SDK_DIR is not defined") +ELSE () + MESSAGE (STATUS "WASI_SDK_DIR=${WASI_SDK_DIR}") + LIST (APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES "WASI_SDK_DIR") +ENDIF () + + +SET (CMAKE_C_FLAGS "-nostdlib -z stack-size=4096" CACHE INTERNAL "") +SET (CMAKE_C_COMPILER_TARGET "wasm32") +SET (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +SET (CMAKE_CXX_FLAGS "-nostdlib -z stack-size=4096" CACHE INTERNAL "") +SET (CMAKE_CXX_COMPILER_TARGET "wasm32") +SET (CMAKE_CXX_COMPILER "${WASI_SDK_DIR}/bin/clang++") + +SET (CMAKE_EXE_LINKER_FLAGS + "-Wl,--initial-memory=65536,--no-entry,--strip-all" CACHE INTERNAL "") + +SET (CMAKE_LINKER "${WASI_SDK_DIR}/bin/wasm-ld" CACHE INTERNAL "") +SET (CMAKE_AR "${WASI_SDK_DIR}/bin/llvm-ar" CACHE INTERNAL "") +SET (CMAKE_NM "${WASI_SDK_DIR}/bin/llvm-nm" CACHE INTERNAL "") +SET (CMAKE_OBJDUMP "${WASI_SDK_DIR}/bin/llvm-dwarfdump" CACHE INTERNAL "") +SET (CMAKE_RANLIB "${WASI_SDK_DIR}/bin/llvm-ranlib" CACHE INTERNAL "") +SET (CMAKE_EXE_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS},--allow-undefined-file=${CMAKE_SYSROOT}/share/defined-symbols.txt" CACHE INTERNAL "") \ No newline at end of file diff --git a/wasm-micro-runtime/wamr-sdk/app/wasi_toolchain.cmake b/wasm-micro-runtime/wamr-sdk/app/wasi_toolchain.cmake new file mode 100644 index 0000000..f80c737 --- /dev/null +++ b/wasm-micro-runtime/wamr-sdk/app/wasi_toolchain.cmake @@ -0,0 +1,17 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +SET(CMAKE_SYSTEM_NAME Linux) + +if (NOT DEFINED WASI_SDK_DIR) + SET (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +SET (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") +SET (CMAKE_CXX_COMPILER "${WASI_SDK_DIR}/bin/clang++") + +SET (CMAKE_LINKER "${WASI_SDK_DIR}/bin/wasm-ld" CACHE INTERNAL "") +SET (CMAKE_AR "${WASI_SDK_DIR}/bin/llvm-ar" CACHE INTERNAL "") +SET (CMAKE_NM "${WASI_SDK_DIR}/bin/llvm-nm" CACHE INTERNAL "") +SET (CMAKE_OBJDUMP "${WASI_SDK_DIR}/bin/llvm-dwarfdump" CACHE INTERNAL "") +SET (CMAKE_RANLIB "${WASI_SDK_DIR}/bin/llvm-ranlib" CACHE INTERNAL "") diff --git a/wasm-micro-runtime/zephyr/module.yml b/wasm-micro-runtime/zephyr/module.yml new file mode 100644 index 0000000..059c736 --- /dev/null +++ b/wasm-micro-runtime/zephyr/module.yml @@ -0,0 +1,5 @@ +name: wasm-micro-runtime + +build: + cmake-ext: True + kconfig-ext: True